diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 2bf603228a8..37abdcc25d1 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -849,7 +849,7 @@ py_library( visibility = ["//visibility:public"], deps = select({ "api_version_2": [], - "//conditions:default": ["//tensorflow/contrib:contrib_py"], + "//conditions:default": [], }) + [ ":tensorflow_py_no_contrib", "//tensorflow/python/estimator:estimator_py", diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD deleted file mode 100644 index ee001a6e6fc..00000000000 --- a/tensorflow/contrib/BUILD +++ /dev/null @@ -1,175 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -load("//tensorflow:tensorflow.bzl", "if_not_windows") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "contrib_py", - srcs = glob( - ["**/*.py"], - exclude = [ - "**/*_test.py", - ], - ), - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/contrib/all_reduce", - "//tensorflow/contrib/batching:batch_py", - "//tensorflow/contrib/bayesflow:bayesflow_py", - "//tensorflow/contrib/boosted_trees:init_py", - "//tensorflow/contrib/checkpoint/python:checkpoint", - "//tensorflow/contrib/cluster_resolver:cluster_resolver_py", - "//tensorflow/contrib/compiler:compiler_py", - "//tensorflow/contrib/compiler:xla", - "//tensorflow/contrib/autograph", - "//tensorflow/contrib/constrained_optimization", - "//tensorflow/contrib/copy_graph:copy_graph_py", - "//tensorflow/contrib/crf:crf_py", - "//tensorflow/contrib/cudnn_rnn:cudnn_rnn_py", - "//tensorflow/contrib/data", - "//tensorflow/contrib/deprecated:deprecated_py", - "//tensorflow/contrib/distribute:distribute", - "//tensorflow/contrib/distributions:distributions_py", - "//tensorflow/contrib/eager/python:tfe", - "//tensorflow/contrib/estimator:estimator_py", - "//tensorflow/contrib/factorization:factorization_py", - "//tensorflow/contrib/feature_column:feature_column_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/graph_editor:graph_editor_py", - "//tensorflow/contrib/grid_rnn:grid_rnn_py", - "//tensorflow/contrib/hadoop", - "//tensorflow/contrib/hooks", - "//tensorflow/contrib/image:distort_image_py", - "//tensorflow/contrib/image:image_py", - "//tensorflow/contrib/image:single_image_random_dot_stereograms_py", - "//tensorflow/contrib/input_pipeline:input_pipeline_py", - "//tensorflow/contrib/integrate:integrate_py", - "//tensorflow/contrib/keras", - "//tensorflow/contrib/kernel_methods", - "//tensorflow/contrib/labeled_tensor", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn", - "//tensorflow/contrib/learn:head_test_lib", - "//tensorflow/contrib/legacy_seq2seq:seq2seq_py", - "//tensorflow/contrib/libsvm", - "//tensorflow/contrib/linear_optimizer:sdca_estimator_py", - "//tensorflow/contrib/linear_optimizer:sdca_ops_py", - "//tensorflow/contrib/lookup:lookup_py", - "//tensorflow/contrib/losses:losses_py", - "//tensorflow/contrib/losses:metric_learning_py", - "//tensorflow/contrib/memory_stats:memory_stats_py", - "//tensorflow/contrib/meta_graph_transform", - "//tensorflow/contrib/metrics:metrics_py", - "//tensorflow/contrib/mixed_precision:mixed_precision", - "//tensorflow/contrib/model_pruning", - "//tensorflow/contrib/nearest_neighbor:nearest_neighbor_py", - "//tensorflow/contrib/nn:nn_py", - "//tensorflow/contrib/opt:opt_py", - "//tensorflow/contrib/optimizer_v2:optimizer_v2_py", - "//tensorflow/contrib/periodic_resample:init_py", - "//tensorflow/contrib/predictor", - "//tensorflow/contrib/proto", - "//tensorflow/contrib/quantization:quantization_py", - "//tensorflow/contrib/quantize:quantize_graph", - "//tensorflow/contrib/receptive_field:receptive_field_py", - "//tensorflow/contrib/recurrent:recurrent_py", - "//tensorflow/contrib/reduce_slice_ops:reduce_slice_ops_py", - "//tensorflow/contrib/remote_fused_graph/pylib:remote_fused_graph_ops_py", - "//tensorflow/contrib/resampler:resampler_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/contrib/rpc", - "//tensorflow/contrib/saved_model:saved_model_py", - "//tensorflow/contrib/seq2seq:seq2seq_py", - "//tensorflow/contrib/signal:signal_py", - "//tensorflow/contrib/slim", - "//tensorflow/contrib/slim:nets", - "//tensorflow/contrib/solvers:solvers_py", - "//tensorflow/contrib/sparsemax:sparsemax_py", - "//tensorflow/contrib/specs", - "//tensorflow/contrib/staging", - "//tensorflow/contrib/stat_summarizer:stat_summarizer_py", - "//tensorflow/contrib/stateless", - "//tensorflow/contrib/summary:summary", - "//tensorflow/contrib/tensor_forest:init_py", - "//tensorflow/contrib/tensorboard", - "//tensorflow/contrib/testing:testing_py", - "//tensorflow/contrib/text:text_py", - "//tensorflow/contrib/tfprof", - "//tensorflow/contrib/timeseries", - "//tensorflow/contrib/tpu", - "//tensorflow/contrib/training:training_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:util", - "//tensorflow/python/estimator:estimator_py", - ] + select({ - "//tensorflow:android": [], - "//tensorflow:ios": [], - "//tensorflow:linux_s390x": [], - "//tensorflow:windows": [], - "//conditions:default": [ - "//tensorflow/contrib/fused_conv:fused_conv_py", - "//tensorflow/contrib/tensorrt:init_py", - "//tensorflow/contrib/ffmpeg:ffmpeg_ops_py", - ], - }) + select({ - "//tensorflow:android": [], - "//tensorflow:ios": [], - "//tensorflow:linux_s390x": [], - "//tensorflow:windows": [], - "//tensorflow:no_gcp_support": [], - "//conditions:default": [ - "//tensorflow/contrib/bigtable", - "//tensorflow/contrib/cloud:cloud_py", - ], - }), -) - -cc_library( - name = "contrib_kernels", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/contrib/boosted_trees:boosted_trees_kernels", - "//tensorflow/contrib/factorization/kernels:all_kernels", - "//tensorflow/contrib/hadoop:dataset_kernels", - "//tensorflow/contrib/image:image_ops_kernels", - "//tensorflow/contrib/input_pipeline:input_pipeline_ops_kernels", - "//tensorflow/contrib/layers:sparse_feature_cross_op_kernel", - "//tensorflow/contrib/nearest_neighbor:nearest_neighbor_ops_kernels", - "//tensorflow/contrib/rnn:all_kernels", - "//tensorflow/contrib/seq2seq:beam_search_ops_kernels", - "//tensorflow/contrib/tensor_forest:model_ops_kernels", - "//tensorflow/contrib/tensor_forest:stats_ops_kernels", - "//tensorflow/contrib/tensor_forest:tensor_forest_kernels", - "//tensorflow/contrib/text:all_kernels", - ] + if_not_windows([ - "//tensorflow/contrib/tensorrt:trt_op_kernels", - ]), -) - -cc_library( - name = "contrib_ops_op_lib", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/contrib/boosted_trees:boosted_trees_ops_op_lib", - "//tensorflow/contrib/factorization:all_ops", - "//tensorflow/contrib/framework:all_ops", - "//tensorflow/contrib/hadoop:dataset_ops_op_lib", - "//tensorflow/contrib/input_pipeline:input_pipeline_ops_op_lib", - "//tensorflow/contrib/layers:sparse_feature_cross_op_op_lib", - "//tensorflow/contrib/nearest_neighbor:nearest_neighbor_ops_op_lib", - "//tensorflow/contrib/rnn:all_ops", - "//tensorflow/contrib/seq2seq:beam_search_ops_op_lib", - "//tensorflow/contrib/tensor_forest:model_ops_op_lib", - "//tensorflow/contrib/tensor_forest:stats_ops_op_lib", - "//tensorflow/contrib/tensor_forest:tensor_forest_ops_op_lib", - "//tensorflow/contrib/text:all_ops", - ] + if_not_windows([ - "//tensorflow/compiler/tf2tensorrt:trt_op_libs", - ]), -) diff --git a/tensorflow/contrib/README.md b/tensorflow/contrib/README.md deleted file mode 100644 index fa84e68006f..00000000000 --- a/tensorflow/contrib/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# TensorFlow contrib - -Any code in this directory is not officially supported, and may change or be -removed at any time without notice. - -The contrib directory contains project directories, each of which has designated -owners. It is meant to contain features and contributions that eventually should -get merged into core TensorFlow, but whose interfaces may still change, or which -require some testing to see whether they can find broader acceptance. We are -trying to keep duplication within contrib to a minimum, so you may be asked to -refactor code in contrib to use some feature inside core or in another project -in contrib rather than reimplementing the feature. - -When adding a project, please stick to the following directory structure: -Create a project directory in `contrib/`, and mirror the portions of the -TensorFlow tree that your project requires underneath `contrib/my_project/`. - -For example, let's say you create foo ops in two files: `foo_ops.py` and -`foo_ops_test.py`. If you were to merge those files directly into TensorFlow, -they would live in `tensorflow/python/ops/foo_ops.py` and -`tensorflow/python/kernel_tests/foo_ops_test.py`. In `contrib/`, they are part -of project `foo`, and their full paths are `contrib/foo/python/ops/foo_ops.py` -and `contrib/foo/python/kernel_tests/foo_ops_test.py`. diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py deleted file mode 100644 index 1611cf4f338..00000000000 --- a/tensorflow/contrib/__init__.py +++ /dev/null @@ -1,122 +0,0 @@ -# pylint: disable=g-import-not-at-top -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contrib module containing volatile or experimental code. - -Warning: The `tf.contrib` module will not be included in TensorFlow 2.0. Many -of its submodules have been integrated into TensorFlow core, or spun-off into -other projects like [`tensorflow_io`](https://github.com/tensorflow/io), or -[`tensorflow_addons`](https://github.com/tensorflow/addons). For instructions -on how to upgrade see the -[Migration guide](https://www.tensorflow.org/beta/guide/migration_guide). -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import platform - -# Add projects here, they will show up under tf.contrib. -from tensorflow.contrib import autograph -from tensorflow.contrib import batching -from tensorflow.contrib import bayesflow -from tensorflow.contrib import checkpoint -from tensorflow.contrib import cluster_resolver -from tensorflow.contrib import compiler -from tensorflow.contrib import constrained_optimization -from tensorflow.contrib import copy_graph -from tensorflow.contrib import crf -from tensorflow.contrib import cudnn_rnn -from tensorflow.contrib import data -from tensorflow.contrib import deprecated -from tensorflow.contrib import distribute -from tensorflow.contrib import distributions -from tensorflow.contrib import estimator -from tensorflow.contrib import factorization -from tensorflow.contrib import feature_column -from tensorflow.contrib import framework -from tensorflow.contrib import graph_editor -from tensorflow.contrib import grid_rnn -from tensorflow.contrib import image -from tensorflow.contrib import input_pipeline -from tensorflow.contrib import integrate -from tensorflow.contrib import keras -from tensorflow.contrib import kernel_methods -from tensorflow.contrib import labeled_tensor -from tensorflow.contrib import layers -from tensorflow.contrib import learn -from tensorflow.contrib import legacy_seq2seq -from tensorflow.contrib import linear_optimizer -from tensorflow.contrib import lookup -from tensorflow.contrib import losses -from tensorflow.contrib import memory_stats -from tensorflow.contrib import metrics -from tensorflow.contrib import mixed_precision -from tensorflow.contrib import model_pruning -from tensorflow.contrib import nn -from tensorflow.contrib import opt -from tensorflow.contrib import periodic_resample -from tensorflow.contrib import predictor -from tensorflow.contrib import proto -from tensorflow.contrib import quantization -from tensorflow.contrib import quantize -from tensorflow.contrib import reduce_slice_ops -from tensorflow.contrib import resampler -from tensorflow.contrib import rnn -from tensorflow.contrib import rpc -from tensorflow.contrib import saved_model -from tensorflow.contrib import seq2seq -from tensorflow.contrib import signal -from tensorflow.contrib import slim -from tensorflow.contrib import solvers -from tensorflow.contrib import sparsemax -from tensorflow.contrib import staging -from tensorflow.contrib import stat_summarizer -from tensorflow.contrib import stateless -from tensorflow.contrib import tensor_forest -from tensorflow.contrib import tensorboard -from tensorflow.contrib import testing -from tensorflow.contrib import tfprof -from tensorflow.contrib import timeseries -from tensorflow.contrib import tpu -from tensorflow.contrib import training -from tensorflow.contrib import util -from tensorflow.contrib.eager.python import tfe as eager -from tensorflow.contrib.optimizer_v2 import optimizer_v2_symbols as optimizer_v2 -from tensorflow.contrib.receptive_field import receptive_field_api as receptive_field -from tensorflow.contrib.recurrent.python import recurrent_api as recurrent -from tensorflow.contrib.remote_fused_graph import pylib as remote_fused_graph -from tensorflow.contrib.specs import python as specs -from tensorflow.contrib.summary import summary - -if os.name != "nt" and platform.machine() != "s390x": - try: - from tensorflow.contrib import cloud - except ImportError: - pass - -from tensorflow.python.util.lazy_loader import LazyLoader -ffmpeg = LazyLoader("ffmpeg", globals(), - "tensorflow.contrib.ffmpeg") -del os -del platform - -del LazyLoader - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/all_reduce/BUILD b/tensorflow/contrib/all_reduce/BUILD deleted file mode 100644 index 2ebb821f0b3..00000000000 --- a/tensorflow/contrib/all_reduce/BUILD +++ /dev/null @@ -1,33 +0,0 @@ -# Description: -# All-reduce implementations. -# APIs are subject to change. Eventually to be replaced by equivalent -# functionality within TensorFlow core. - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "all_reduce_py", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - deps = [ - ":all_reduce", - "//tensorflow/python:util", - ], -) - -py_library( - name = "all_reduce", - srcs = [ - "python/all_reduce.py", - ], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/python/distribute:all_reduce", - ], -) diff --git a/tensorflow/contrib/all_reduce/__init__.py b/tensorflow/contrib/all_reduce/__init__.py deleted file mode 100644 index f9824f4cfbf..00000000000 --- a/tensorflow/contrib/all_reduce/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""All-reduce implementations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,line-too-long,wildcard-import -from tensorflow.contrib.all_reduce.python.all_reduce import * - -from tensorflow.python.util.all_util import remove_undocumented -# pylint: enable=unused-import,line-too-long,wildcard-import - -_allowed_symbols = [ - 'build_ring_all_reduce', - 'build_recursive_hd_all_reduce', - 'build_shuffle_all_reduce', - 'build_nccl_all_reduce', - 'build_nccl_then_ring', - 'build_nccl_then_recursive_hd', - 'build_nccl_then_shuffle', - 'build_shuffle_then_ring', - 'build_shuffle_then_shuffle' -] - -remove_undocumented(__name__, allowed_exception_list=_allowed_symbols) diff --git a/tensorflow/contrib/all_reduce/python/all_reduce.py b/tensorflow/contrib/all_reduce/python/all_reduce.py deleted file mode 100644 index 238cdaf8a79..00000000000 --- a/tensorflow/contrib/all_reduce/python/all_reduce.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities to construct a TF subgraph implementing distributed All-Reduce.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.python.distribute.all_reduce import * diff --git a/tensorflow/contrib/android/BUILD b/tensorflow/contrib/android/BUILD deleted file mode 100644 index 092aa4593a8..00000000000 --- a/tensorflow/contrib/android/BUILD +++ /dev/null @@ -1,87 +0,0 @@ -# Description: -# JNI-based Java inference interface for TensorFlow. - -load("@build_bazel_rules_android//android:rules.bzl", "android_library") -load( - "//tensorflow:tensorflow.bzl", - "if_android", - "tf_copts", -) - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files([ - "LICENSE", - "jni/version_script.lds", -]) - -filegroup( - name = "android_tensorflow_inference_jni_srcs", - srcs = glob([ - "**/*.cc", - "**/*.h", - ]), - visibility = ["//visibility:public"], -) - -cc_library( - name = "android_tensorflow_inference_jni", - srcs = if_android([":android_tensorflow_inference_jni_srcs"]), - copts = tf_copts(), - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/core:android_tensorflow_lib_lite", - "//tensorflow/java/src/main/native", - ], - alwayslink = 1, -) - -# JAR with Java bindings to TF. -android_library( - name = "android_tensorflow_inference_java", - srcs = glob(["java/**/*.java"]) + ["//tensorflow/java:java_sources"], - tags = [ - "manual", - "notap", - ], -) - -# Build the native .so. -# bazel build //tensorflow/contrib/android:libtensorflow_inference.so \ -# --crosstool_top=//external:android/crosstool \ -# --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ -# --cpu=armeabi-v7a -LINKER_SCRIPT = "//tensorflow/contrib/android:jni/version_script.lds" - -cc_binary( - name = "libtensorflow_inference.so", - copts = tf_copts() + [ - "-ffunction-sections", - "-fdata-sections", - ], - linkopts = if_android([ - "-landroid", - "-latomic", - "-ldl", - "-llog", - "-lm", - "-z defs", - "-s", - "-Wl,--gc-sections", - "-Wl,--version-script,$(location {})".format(LINKER_SCRIPT), - ]), - linkshared = 1, - linkstatic = 1, - tags = [ - "manual", - "notap", - ], - deps = [ - ":android_tensorflow_inference_jni", - "//tensorflow/core:android_tensorflow_lib", - LINKER_SCRIPT, - ], -) diff --git a/tensorflow/contrib/android/README.md b/tensorflow/contrib/android/README.md deleted file mode 100644 index 27f8ac21323..00000000000 --- a/tensorflow/contrib/android/README.md +++ /dev/null @@ -1,95 +0,0 @@ -# Android TensorFlow support - -This directory defines components (a native `.so` library and a Java JAR) -geared towards supporting TensorFlow on Android. This includes: - -- The [TensorFlow Java API](../../java/README.md) -- A `TensorFlowInferenceInterface` class that provides a smaller API - surface suitable for inference and summarizing performance of model execution. - -For example usage, see [TensorFlowImageClassifier.java](../../examples/android/src/org/tensorflow/demo/TensorFlowImageClassifier.java) -in the [TensorFlow Android Demo](../../examples/android). - -For prebuilt libraries, see the -[nightly Android build artifacts](https://ci.tensorflow.org/view/Nightly/job/nightly-android/) -page for a recent build. - -The TensorFlow Inference Interface is also available as a -[JCenter package](https://bintray.com/google/tensorflow/tensorflow) -(see the tensorflow-android directory) and can be included quite simply in your -android project with a couple of lines in the project's `build.gradle` file: - -``` -allprojects { - repositories { - jcenter() - } -} - -dependencies { - compile 'org.tensorflow:tensorflow-android:+' -} -``` - -This will tell Gradle to use the -[latest version](https://bintray.com/google/tensorflow/tensorflow/_latestVersion) -of the TensorFlow AAR that has been released to -[JCenter](https://jcenter.bintray.com/org/tensorflow/tensorflow-android/). -You may replace the `+` with an explicit version label if you wish to -use a specific release of TensorFlow in your app. - -To build the libraries yourself (if, for example, you want to support custom -TensorFlow operators), pick your preferred approach below: - -### Bazel - -First follow the Bazel setup instructions described in -[tensorflow/examples/android/README.md](../../examples/android/README.md) - -Then, to build the native TF library: - -``` -bazel build -c opt //tensorflow/contrib/android:libtensorflow_inference.so \ - --crosstool_top=//external:android/crosstool \ - --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ - --cxxopt=-std=c++11 \ - --cpu=armeabi-v7a -``` - -Replacing `armeabi-v7a` with your desired target architecture. - -The library will be located at: - -``` -bazel-bin/tensorflow/contrib/android/libtensorflow_inference.so -``` - -To build the Java counterpart: - -``` -bazel build //tensorflow/contrib/android:android_tensorflow_inference_java -``` - -You will find the JAR file at: - -``` -bazel-bin/tensorflow/contrib/android/libandroid_tensorflow_inference_java.jar -``` - -### CMake - -For documentation on building a self-contained AAR file with cmake, see -[tensorflow/contrib/android/cmake](cmake). - - -### Makefile - -For documentation on building native TF libraries with make, including a CUDA-enabled variant for devices like the Nvidia Shield TV, see [tensorflow/contrib/makefile/README.md](../makefile/README.md) - - -## AssetManagerFileSystem - -This directory also contains a TensorFlow filesystem supporting the Android -asset manager. This may be useful when writing native (C++) code that is tightly -coupled with TensorFlow. For typical usage, the library above will be -sufficient. diff --git a/tensorflow/contrib/android/asset_manager_filesystem.cc b/tensorflow/contrib/android/asset_manager_filesystem.cc deleted file mode 100644 index a5aa950bff6..00000000000 --- a/tensorflow/contrib/android/asset_manager_filesystem.cc +++ /dev/null @@ -1,272 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/android/asset_manager_filesystem.h" - -#include - -#include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/file_system_helper.h" - -namespace tensorflow { -namespace { - -string RemoveSuffix(const string& name, const string& suffix) { - string output(name); - StringPiece piece(output); - absl::ConsumeSuffix(&piece, suffix); - return string(piece); -} - -// Closes the given AAsset when variable is destructed. -class ScopedAsset { - public: - ScopedAsset(AAsset* asset) : asset_(asset) {} - ~ScopedAsset() { - if (asset_ != nullptr) { - AAsset_close(asset_); - } - } - - AAsset* get() const { return asset_; } - - private: - AAsset* asset_; -}; - -// Closes the given AAssetDir when variable is destructed. -class ScopedAssetDir { - public: - ScopedAssetDir(AAssetDir* asset_dir) : asset_dir_(asset_dir) {} - ~ScopedAssetDir() { - if (asset_dir_ != nullptr) { - AAssetDir_close(asset_dir_); - } - } - - AAssetDir* get() const { return asset_dir_; } - - private: - AAssetDir* asset_dir_; -}; - -class ReadOnlyMemoryRegionFromAsset : public ReadOnlyMemoryRegion { - public: - ReadOnlyMemoryRegionFromAsset(std::unique_ptr data, uint64 length) - : data_(std::move(data)), length_(length) {} - ~ReadOnlyMemoryRegionFromAsset() override = default; - - const void* data() override { return reinterpret_cast(data_.get()); } - uint64 length() override { return length_; } - - private: - std::unique_ptr data_; - uint64 length_; -}; - -// Note that AAssets are not thread-safe and cannot be used across threads. -// However, AAssetManager is. Because RandomAccessFile must be thread-safe and -// used across threads, new AAssets must be created for every access. -// TODO(tylerrhodes): is there a more efficient way to do this? -class RandomAccessFileFromAsset : public RandomAccessFile { - public: - RandomAccessFileFromAsset(AAssetManager* asset_manager, const string& name) - : asset_manager_(asset_manager), file_name_(name) {} - ~RandomAccessFileFromAsset() override = default; - - Status Read(uint64 offset, size_t to_read, StringPiece* result, - char* scratch) const override { - auto asset = ScopedAsset(AAssetManager_open( - asset_manager_, file_name_.c_str(), AASSET_MODE_RANDOM)); - if (asset.get() == nullptr) { - return errors::NotFound("File ", file_name_, " not found."); - } - - off64_t new_offset = AAsset_seek64(asset.get(), offset, SEEK_SET); - off64_t length = AAsset_getLength64(asset.get()); - if (new_offset < 0) { - *result = StringPiece(scratch, 0); - return errors::OutOfRange("Read after file end."); - } - const off64_t region_left = - std::min(length - new_offset, static_cast(to_read)); - int read = AAsset_read(asset.get(), scratch, region_left); - if (read < 0) { - return errors::Internal("Error reading from asset."); - } - *result = StringPiece(scratch, region_left); - return (region_left == to_read) - ? Status::OK() - : errors::OutOfRange("Read less bytes than requested."); - } - - private: - AAssetManager* asset_manager_; - string file_name_; -}; - -} // namespace - -AssetManagerFileSystem::AssetManagerFileSystem(AAssetManager* asset_manager, - const string& prefix) - : asset_manager_(asset_manager), prefix_(prefix) {} - -Status AssetManagerFileSystem::FileExists(const string& fname) { - string path = RemoveAssetPrefix(fname); - auto asset = ScopedAsset( - AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_RANDOM)); - if (asset.get() == nullptr) { - return errors::NotFound("File ", fname, " not found."); - } - return Status::OK(); -} - -Status AssetManagerFileSystem::NewRandomAccessFile( - const string& fname, std::unique_ptr* result) { - string path = RemoveAssetPrefix(fname); - auto asset = ScopedAsset( - AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_RANDOM)); - if (asset.get() == nullptr) { - return errors::NotFound("File ", fname, " not found."); - } - result->reset(new RandomAccessFileFromAsset(asset_manager_, path)); - return Status::OK(); -} - -Status AssetManagerFileSystem::NewReadOnlyMemoryRegionFromFile( - const string& fname, std::unique_ptr* result) { - string path = RemoveAssetPrefix(fname); - auto asset = ScopedAsset( - AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_STREAMING)); - if (asset.get() == nullptr) { - return errors::NotFound("File ", fname, " not found."); - } - - off64_t start, length; - int fd = AAsset_openFileDescriptor64(asset.get(), &start, &length); - std::unique_ptr data; - if (fd >= 0) { - data.reset(new char[length]); - ssize_t result = pread(fd, data.get(), length, start); - if (result < 0) { - return errors::Internal("Error reading from file ", fname, - " using 'read': ", result); - } - if (result != length) { - return errors::Internal("Expected size does not match size read: ", - "Expected ", length, " vs. read ", result); - } - close(fd); - } else { - length = AAsset_getLength64(asset.get()); - data.reset(new char[length]); - const void* asset_buffer = AAsset_getBuffer(asset.get()); - if (asset_buffer == nullptr) { - return errors::Internal("Error reading ", fname, " from asset manager."); - } - memcpy(data.get(), asset_buffer, length); - } - result->reset(new ReadOnlyMemoryRegionFromAsset(std::move(data), length)); - return Status::OK(); -} - -Status AssetManagerFileSystem::GetChildren(const string& prefixed_dir, - std::vector* r) { - std::string path = NormalizeDirectoryPath(prefixed_dir); - auto dir = - ScopedAssetDir(AAssetManager_openDir(asset_manager_, path.c_str())); - if (dir.get() == nullptr) { - return errors::NotFound("Directory ", prefixed_dir, " not found."); - } - const char* next_file = AAssetDir_getNextFileName(dir.get()); - while (next_file != nullptr) { - r->push_back(next_file); - next_file = AAssetDir_getNextFileName(dir.get()); - } - return Status::OK(); -} - -Status AssetManagerFileSystem::GetFileSize(const string& fname, uint64* s) { - // If fname corresponds to a directory, return early. It doesn't map to an - // AAsset, and would otherwise return NotFound. - if (DirectoryExists(fname)) { - *s = 0; - return Status::OK(); - } - string path = RemoveAssetPrefix(fname); - auto asset = ScopedAsset( - AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_RANDOM)); - if (asset.get() == nullptr) { - return errors::NotFound("File ", fname, " not found."); - } - *s = AAsset_getLength64(asset.get()); - return Status::OK(); -} - -Status AssetManagerFileSystem::Stat(const string& fname, FileStatistics* stat) { - uint64 size; - stat->is_directory = DirectoryExists(fname); - TF_RETURN_IF_ERROR(GetFileSize(fname, &size)); - stat->length = size; - return Status::OK(); -} - -string AssetManagerFileSystem::NormalizeDirectoryPath(const string& fname) { - return RemoveSuffix(RemoveAssetPrefix(fname), "/"); -} - -string AssetManagerFileSystem::RemoveAssetPrefix(const string& name) { - StringPiece piece(name); - absl::ConsumePrefix(&piece, prefix_); - return string(piece); -} - -bool AssetManagerFileSystem::DirectoryExists(const std::string& fname) { - std::string path = NormalizeDirectoryPath(fname); - auto dir = - ScopedAssetDir(AAssetManager_openDir(asset_manager_, path.c_str())); - // Note that openDir will return something even if the directory doesn't - // exist. Therefore, we need to ensure one file exists in the folder. - return AAssetDir_getNextFileName(dir.get()) != NULL; -} - -Status AssetManagerFileSystem::GetMatchingPaths(const string& pattern, - std::vector* results) { - return internal::GetMatchingPaths(this, Env::Default(), pattern, results); -} - -Status AssetManagerFileSystem::NewWritableFile( - const string& fname, std::unique_ptr* result) { - return errors::Unimplemented("Asset storage is read only."); -} -Status AssetManagerFileSystem::NewAppendableFile( - const string& fname, std::unique_ptr* result) { - return errors::Unimplemented("Asset storage is read only."); -} -Status AssetManagerFileSystem::DeleteFile(const string& f) { - return errors::Unimplemented("Asset storage is read only."); -} -Status AssetManagerFileSystem::CreateDir(const string& d) { - return errors::Unimplemented("Asset storage is read only."); -} -Status AssetManagerFileSystem::DeleteDir(const string& d) { - return errors::Unimplemented("Asset storage is read only."); -} -Status AssetManagerFileSystem::RenameFile(const string& s, const string& t) { - return errors::Unimplemented("Asset storage is read only."); -} - -} // namespace tensorflow diff --git a/tensorflow/contrib/android/asset_manager_filesystem.h b/tensorflow/contrib/android/asset_manager_filesystem.h deleted file mode 100644 index a87ff42ae21..00000000000 --- a/tensorflow/contrib/android/asset_manager_filesystem.h +++ /dev/null @@ -1,85 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_ANDROID_ASSET_MANAGER_FILESYSTEM_H_ -#define TENSORFLOW_CONTRIB_ANDROID_ASSET_MANAGER_FILESYSTEM_H_ - -#include -#include - -#include "tensorflow/core/platform/file_system.h" - -namespace tensorflow { - -// FileSystem that uses Android's AAssetManager. Once initialized with a given -// AAssetManager, files in the given AAssetManager can be accessed through the -// prefix given when registered with the TensorFlow Env. -// Note that because APK assets are immutable, any operation that tries to -// modify the FileSystem will return tensorflow::error::code::UNIMPLEMENTED. -class AssetManagerFileSystem : public FileSystem { - public: - // Initialize an AssetManagerFileSystem. Note that this does not register the - // file system with TensorFlow. - // asset_manager - Non-null Android AAssetManager that backs this file - // system. The asset manager is not owned by this file system, and must - // outlive this class. - // prefix - Common prefix to strip from all file URIs before passing them to - // the asset_manager. This is required because TensorFlow gives the entire - // file URI (file:///my_dir/my_file.txt) and AssetManager only knows paths - // relative to its base directory. - AssetManagerFileSystem(AAssetManager* asset_manager, const string& prefix); - ~AssetManagerFileSystem() override = default; - - Status FileExists(const string& fname) override; - Status NewRandomAccessFile( - const string& filename, - std::unique_ptr* result) override; - Status NewReadOnlyMemoryRegionFromFile( - const string& filename, - std::unique_ptr* result) override; - - Status GetFileSize(const string& f, uint64* s) override; - // Currently just returns size. - Status Stat(const string& fname, FileStatistics* stat) override; - Status GetChildren(const string& dir, std::vector* r) override; - - // All these functions return Unimplemented error. Asset storage is - // read only. - Status NewWritableFile(const string& fname, - std::unique_ptr* result) override; - Status NewAppendableFile(const string& fname, - std::unique_ptr* result) override; - Status DeleteFile(const string& f) override; - Status CreateDir(const string& d) override; - Status DeleteDir(const string& d) override; - Status RenameFile(const string& s, const string& t) override; - - Status GetMatchingPaths(const string& pattern, - std::vector* results) override; - - private: - string RemoveAssetPrefix(const string& name); - - // Return a string path that can be passed into AAssetManager functions. - // For example, 'my_prefix://some/dir/' would return 'some/dir'. - string NormalizeDirectoryPath(const string& fname); - bool DirectoryExists(const std::string& fname); - - AAssetManager* asset_manager_; - string prefix_; -}; - -} // namespace tensorflow -#endif // TENSORFLOW_CONTRIB_ANDROID_ASSET_MANAGER_FILESYSTEM_H_ diff --git a/tensorflow/contrib/android/cmake/CMakeLists.txt b/tensorflow/contrib/android/cmake/CMakeLists.txt deleted file mode 100644 index ecf1a103d29..00000000000 --- a/tensorflow/contrib/android/cmake/CMakeLists.txt +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -cmake_minimum_required(VERSION 3.4.1) -include(ExternalProject) - -# TENSORFLOW_ROOT_DIR: -# root directory of tensorflow repo -# used for shared source files and pre-built libs -get_filename_component(TENSORFLOW_ROOT_DIR ../../../.. ABSOLUTE) -set(PREBUILT_DIR ${TENSORFLOW_ROOT_DIR}/tensorflow/contrib/makefile/gen) - -add_library(lib_proto STATIC IMPORTED ) -set_target_properties(lib_proto PROPERTIES IMPORTED_LOCATION - ${PREBUILT_DIR}/protobuf/lib/libprotobuf.a) - -add_library(lib_nsync STATIC IMPORTED ) -set_target_properties(lib_nsync PROPERTIES IMPORTED_LOCATION - ${TARGET_NSYNC_LIB}/lib/libnsync.a) - -add_library(lib_tf STATIC IMPORTED ) -set_target_properties(lib_tf PROPERTIES IMPORTED_LOCATION - ${PREBUILT_DIR}/lib/libtensorflow-core.a) -# Change to compile flags should be replicated into bazel build file -# TODO: Consider options other than -O2 for binary size. -# e.g. -Os for gcc, and -Oz for clang. -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DIS_SLIM_BUILD \ - -std=c++11 -fno-rtti -fno-exceptions \ - -O2 -Wno-narrowing -fomit-frame-pointer \ - -mfpu=neon -mfloat-abi=softfp -fPIE -fPIC \ - -ftemplate-depth=900 \ - -DGOOGLE_PROTOBUF_NO_RTTI \ - -DGOOGLE_PROTOBUF_NO_STATIC_INITIALIZER") - -set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} \ - -Wl,--allow-multiple-definition \ - -Wl,--whole-archive \ - -fPIE -pie -v") -file(GLOB tensorflow_inference_sources - ${CMAKE_CURRENT_SOURCE_DIR}/../jni/*.cc) -file(GLOB java_api_native_sources - ${TENSORFLOW_ROOT_DIR}/tensorflow/java/src/main/native/*.cc) - -add_library(tensorflow_inference SHARED - ${tensorflow_inference_sources} - ${TENSORFLOW_ROOT_DIR}/tensorflow/c/tf_status_helper.cc - ${TENSORFLOW_ROOT_DIR}/tensorflow/c/checkpoint_reader.cc - ${TENSORFLOW_ROOT_DIR}/tensorflow/c/test_op.cc - ${TENSORFLOW_ROOT_DIR}/tensorflow/c/c_api.cc - ${java_api_native_sources}) - -# Include libraries needed for hello-jni lib -target_link_libraries(tensorflow_inference - android - dl - log - m - z - lib_tf - lib_proto - lib_nsync) - -include_directories( - ${PREBUILT_DIR}/proto - ${PREBUILT_DIR}/protobuf/include - ${TENSORFLOW_ROOT_DIR}/tensorflow/contrib/makefile/downloads/eigen - ${TENSORFLOW_ROOT_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/..) diff --git a/tensorflow/contrib/android/cmake/README.md b/tensorflow/contrib/android/cmake/README.md deleted file mode 100644 index 934b58c7242..00000000000 --- a/tensorflow/contrib/android/cmake/README.md +++ /dev/null @@ -1,48 +0,0 @@ -TensorFlow-Android-Inference -============================ -This directory contains CMake support for building the Android Java Inference -interface to the TensorFlow native APIs. - -See [tensorflow/contrib/android](..) for more details about the library, and -instructions for building with Bazel. - -Usage ------ -Add TensorFlow-Android-Inference as a dependency of your Android application - -* settings.gradle - -``` -include ':TensorFlow-Android-Inference' -findProject(":TensorFlow-Android-Inference").projectDir = - new File("${/path/to/tensorflow_repo}/contrib/android/cmake") -``` - -* application's build.gradle (adding dependency): - -``` -debugCompile project(path: ':tensorflow_inference', configuration: 'debug') -releaseCompile project(path: ':tensorflow_inference', configuration: 'release') -``` -Note: this makes native code in the lib traceable from your app. - -Dependencies ------------- -TensorFlow-Android-Inference depends on the TensorFlow static libs already built -in your local TensorFlow repo directory. For Linux/Mac OS, build_all_android.sh -is used in build.gradle to build it. It DOES take time to build the core libs; -so, by default, it is commented out to avoid confusion (otherwise -Android Studio would appear to hang during opening the project). -To enable it, refer to the comment in - -* build.gradle - -Output ------- -- TensorFlow-Inference-debug.aar -- TensorFlow-Inference-release.aar - -File libtensorflow_inference.so should be packed under jni/${ANDROID_ABI}/ -in the above aar, and it is transparent to the app as it will access them via -equivalent java APIs. - diff --git a/tensorflow/contrib/android/cmake/build.gradle b/tensorflow/contrib/android/cmake/build.gradle deleted file mode 100644 index ddec08894f3..00000000000 --- a/tensorflow/contrib/android/cmake/build.gradle +++ /dev/null @@ -1,105 +0,0 @@ -apply plugin: 'com.android.library' - -// TensorFlow repo root dir on local machine -def TF_SRC_DIR = projectDir.toString() + "/../../../.." - -android { - compileSdkVersion 24 - // Check local build_tools_version as this is liable to change within Android Studio. - buildToolsVersion '25.0.2' - - // for debugging native code purpose - publishNonDefault true - - defaultConfig { - archivesBaseName = "Tensorflow-Android-Inference" - minSdkVersion 21 - targetSdkVersion 23 - versionCode 1 - versionName "1.0" - ndk { - abiFilters 'armeabi-v7a' - } - externalNativeBuild { - cmake { - arguments '-DANDROID_TOOLCHAIN=clang', - '-DANDROID_STL=c++_static' - } - } - } - sourceSets { - main { - java { - srcDir "${TF_SRC_DIR}/tensorflow/contrib/android/java" - srcDir "${TF_SRC_DIR}/tensorflow/java/src/main/java" - exclude '**/examples/**' - } - } - } - - externalNativeBuild { - cmake { - path 'CMakeLists.txt' - } - } - buildTypes { - release { - minifyEnabled false - proguardFiles getDefaultProguardFile('proguard-android.txt'), - 'proguard-rules.pro' - } - } -} - -// Build libtensorflow-core.a if necessary -// Note: the environment needs to be set up already -// [ such as installing autoconfig, make, etc ] -// How to use: -// 1) install all of the necessary tools to build libtensorflow-core.a -// 2) inside Android Studio IDE, uncomment buildTensorFlow in -// whenTaskAdded{...} -// 3) re-sync and re-build. It could take a long time if NOT building -// with multiple processes. -import org.apache.tools.ant.taskdefs.condition.Os - -Properties properties = new Properties() -properties.load(project.rootProject.file('local.properties') - .newDataInputStream()) -def ndkDir = properties.getProperty('ndk.dir') -if (ndkDir == null || ndkDir == "") { - ndkDir = System.getenv('ANDROID_NDK_HOME') -} - -if (!Os.isFamily(Os.FAMILY_WINDOWS)) { - // This script is for non-Windows OS. For Windows OS, MANUALLY build - // (or copy the built) libs/headers to the - // ${TENSORFLOW_ROOT_DIR}/tensorflow/contrib/makefile/gen - // refer to CMakeLists.txt about lib and header directories for details - task buildTensorflow(type: Exec) { - group 'buildTensorflowLib' - workingDir getProjectDir().toString() + '/../../../../' - environment PATH: '/opt/local/bin:/opt/local/sbin:' + - System.getenv('PATH') - environment NDK_ROOT: ndkDir - commandLine 'tensorflow/contrib/makefile/build_all_android.sh' - } - - tasks.whenTaskAdded { task -> - group 'buildTensorflowLib' - if (task.name.toLowerCase().contains('sources')) { - def tensorflowTarget = new File(getProjectDir().toString() + - '/../../makefile/gen/lib/libtensorflow-core.a') - if (!tensorflowTarget.exists()) { - // Note: - // just uncomment this line to use it: - // it can take long time to build by default - // it is disabled to avoid false first impression - task.dependsOn buildTensorflow - } - } - } -} - -dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) -} diff --git a/tensorflow/contrib/android/cmake/src/main/AndroidManifest.xml b/tensorflow/contrib/android/cmake/src/main/AndroidManifest.xml deleted file mode 100644 index c17110a78be..00000000000 --- a/tensorflow/contrib/android/cmake/src/main/AndroidManifest.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - diff --git a/tensorflow/contrib/android/cmake/src/main/res/values/strings.xml b/tensorflow/contrib/android/cmake/src/main/res/values/strings.xml deleted file mode 100644 index 92dc3a1baf0..00000000000 --- a/tensorflow/contrib/android/cmake/src/main/res/values/strings.xml +++ /dev/null @@ -1,3 +0,0 @@ - - TensorFlowInference - diff --git a/tensorflow/contrib/android/java/org/tensorflow/contrib/android/RunStats.java b/tensorflow/contrib/android/java/org/tensorflow/contrib/android/RunStats.java deleted file mode 100644 index 39996f6ab03..00000000000 --- a/tensorflow/contrib/android/java/org/tensorflow/contrib/android/RunStats.java +++ /dev/null @@ -1,63 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -package org.tensorflow.contrib.android; - -/** Accumulate and analyze stats from metadata obtained from Session.Runner.run. */ -public class RunStats implements AutoCloseable { - - /** - * Options to be provided to a {@link org.tensorflow.Session.Runner} to enable stats accumulation. - */ - public static byte[] runOptions() { - return fullTraceRunOptions; - } - - public RunStats() { - nativeHandle = allocate(); - } - - @Override - public void close() { - if (nativeHandle != 0) { - delete(nativeHandle); - } - nativeHandle = 0; - } - - /** Accumulate stats obtained when executing a graph. */ - public synchronized void add(byte[] runMetadata) { - add(nativeHandle, runMetadata); - } - - /** Summary of the accumulated runtime stats. */ - public synchronized String summary() { - return summary(nativeHandle); - } - - private long nativeHandle; - - // Hack: This is what a serialized RunOptions protocol buffer with trace_level: FULL_TRACE ends - // up as. - private static byte[] fullTraceRunOptions = new byte[] {0x08, 0x03}; - - private static native long allocate(); - - private static native void delete(long handle); - - private static native void add(long handle, byte[] runMetadata); - - private static native String summary(long handle); -} diff --git a/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java b/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java deleted file mode 100644 index abddadac5bc..00000000000 --- a/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java +++ /dev/null @@ -1,650 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -package org.tensorflow.contrib.android; - -import android.content.res.AssetManager; -import android.os.Build.VERSION; -import android.os.Trace; -import android.text.TextUtils; -import android.util.Log; -import java.io.ByteArrayOutputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.DoubleBuffer; -import java.nio.FloatBuffer; -import java.nio.IntBuffer; -import java.nio.LongBuffer; -import java.util.ArrayList; -import java.util.List; -import org.tensorflow.Graph; -import org.tensorflow.Operation; -import org.tensorflow.Session; -import org.tensorflow.Tensor; -import org.tensorflow.TensorFlow; -import org.tensorflow.Tensors; -import org.tensorflow.types.UInt8; - -/** - * Wrapper over the TensorFlow API ({@link Graph}, {@link Session}) providing a smaller API surface - * for inference. - * - *

See tensorflow/examples/android/src/org/tensorflow/demo/TensorFlowImageClassifier.java for an - * example usage. - */ -public class TensorFlowInferenceInterface { - private static final String TAG = "TensorFlowInferenceInterface"; - private static final String ASSET_FILE_PREFIX = "file:///android_asset/"; - - /* - * Load a TensorFlow model from the AssetManager or from disk if it is not an asset file. - * - * @param assetManager The AssetManager to use to load the model file. - * @param model The filepath to the GraphDef proto representing the model. - */ - public TensorFlowInferenceInterface(AssetManager assetManager, String model) { - prepareNativeRuntime(); - - this.modelName = model; - this.g = new Graph(); - this.sess = new Session(g); - this.runner = sess.runner(); - - final boolean hasAssetPrefix = model.startsWith(ASSET_FILE_PREFIX); - InputStream is = null; - try { - String aname = hasAssetPrefix ? model.split(ASSET_FILE_PREFIX)[1] : model; - is = assetManager.open(aname); - } catch (IOException e) { - if (hasAssetPrefix) { - throw new RuntimeException("Failed to load model from '" + model + "'", e); - } - // Perhaps the model file is not an asset but is on disk. - try { - is = new FileInputStream(model); - } catch (IOException e2) { - throw new RuntimeException("Failed to load model from '" + model + "'", e); - } - } - - try { - if (VERSION.SDK_INT >= 18) { - Trace.beginSection("initializeTensorFlow"); - Trace.beginSection("readGraphDef"); - } - - // TODO(ashankar): Can we somehow mmap the contents instead of copying them? - byte[] graphDef = new byte[is.available()]; - final int numBytesRead = is.read(graphDef); - if (numBytesRead != graphDef.length) { - throw new IOException( - "read error: read only " - + numBytesRead - + " of the graph, expected to read " - + graphDef.length); - } - - if (VERSION.SDK_INT >= 18) { - Trace.endSection(); // readGraphDef. - } - - loadGraph(graphDef, g); - is.close(); - Log.i(TAG, "Successfully loaded model from '" + model + "'"); - - if (VERSION.SDK_INT >= 18) { - Trace.endSection(); // initializeTensorFlow. - } - } catch (IOException e) { - throw new RuntimeException("Failed to load model from '" + model + "'", e); - } - } - - /* - * Load a TensorFlow model from provided InputStream. - * Note: The InputStream will not be closed after loading model, users need to - * close it themselves. - * - * @param is The InputStream to use to load the model. - */ - public TensorFlowInferenceInterface(InputStream is) { - prepareNativeRuntime(); - - // modelName is redundant for model loading from input stream, here is for - // avoiding error in initialization as modelName is marked final. - this.modelName = ""; - this.g = new Graph(); - this.sess = new Session(g); - this.runner = sess.runner(); - - try { - if (VERSION.SDK_INT >= 18) { - Trace.beginSection("initializeTensorFlow"); - Trace.beginSection("readGraphDef"); - } - - int baosInitSize = is.available() > 16384 ? is.available() : 16384; - ByteArrayOutputStream baos = new ByteArrayOutputStream(baosInitSize); - int numBytesRead; - byte[] buf = new byte[16384]; - while ((numBytesRead = is.read(buf, 0, buf.length)) != -1) { - baos.write(buf, 0, numBytesRead); - } - byte[] graphDef = baos.toByteArray(); - - if (VERSION.SDK_INT >= 18) { - Trace.endSection(); // readGraphDef. - } - - loadGraph(graphDef, g); - Log.i(TAG, "Successfully loaded model from the input stream"); - - if (VERSION.SDK_INT >= 18) { - Trace.endSection(); // initializeTensorFlow. - } - } catch (IOException e) { - throw new RuntimeException("Failed to load model from the input stream", e); - } - } - - /* - * Construct a TensorFlowInferenceInterface with provided Graph - * - * @param g The Graph to use to construct this interface. - */ - public TensorFlowInferenceInterface(Graph g) { - prepareNativeRuntime(); - - // modelName is redundant here, here is for - // avoiding error in initialization as modelName is marked final. - this.modelName = ""; - this.g = g; - this.sess = new Session(g); - this.runner = sess.runner(); - } - - /** - * Runs inference between the previously registered input nodes (via feed*) and the requested - * output nodes. Output nodes can then be queried with the fetch* methods. - * - * @param outputNames A list of output nodes which should be filled by the inference pass. - */ - public void run(String[] outputNames) { - run(outputNames, false); - } - - /** - * Runs inference between the previously registered input nodes (via feed*) and the requested - * output nodes. Output nodes can then be queried with the fetch* methods. - * - * @param outputNames A list of output nodes which should be filled by the inference pass. - */ - public void run(String[] outputNames, boolean enableStats) { - run(outputNames, enableStats, new String[] {}); - } - - /** An overloaded version of runInference that allows supplying targetNodeNames as well */ - public void run(String[] outputNames, boolean enableStats, String[] targetNodeNames) { - // Release any Tensors from the previous run calls. - closeFetches(); - - // Add fetches. - for (String o : outputNames) { - fetchNames.add(o); - TensorId tid = TensorId.parse(o); - runner.fetch(tid.name, tid.outputIndex); - } - - // Add targets. - for (String t : targetNodeNames) { - runner.addTarget(t); - } - - // Run the session. - try { - if (enableStats) { - Session.Run r = runner.setOptions(RunStats.runOptions()).runAndFetchMetadata(); - fetchTensors = r.outputs; - - if (runStats == null) { - runStats = new RunStats(); - } - runStats.add(r.metadata); - } else { - fetchTensors = runner.run(); - } - } catch (RuntimeException e) { - // Ideally the exception would have been let through, but since this interface predates the - // TensorFlow Java API, must return -1. - Log.e( - TAG, - "Failed to run TensorFlow inference with inputs:[" - + TextUtils.join(", ", feedNames) - + "], outputs:[" - + TextUtils.join(", ", fetchNames) - + "]"); - throw e; - } finally { - // Always release the feeds (to save resources) and reset the runner, this run is - // over. - closeFeeds(); - runner = sess.runner(); - } - } - - /** Returns a reference to the Graph describing the computation run during inference. */ - public Graph graph() { - return g; - } - - public Operation graphOperation(String operationName) { - final Operation operation = g.operation(operationName); - if (operation == null) { - throw new RuntimeException( - "Node '" + operationName + "' does not exist in model '" + modelName + "'"); - } - return operation; - } - - /** Returns the last stat summary string if logging is enabled. */ - public String getStatString() { - return (runStats == null) ? "" : runStats.summary(); - } - - /** - * Cleans up the state associated with this Object. - * - *

The TenosrFlowInferenceInterface object is no longer usable after this method returns. - */ - public void close() { - closeFeeds(); - closeFetches(); - sess.close(); - g.close(); - if (runStats != null) { - runStats.close(); - } - runStats = null; - } - - @Override - protected void finalize() throws Throwable { - try { - close(); - } finally { - super.finalize(); - } - } - - // Methods for taking a native Tensor and filling it with values from Java arrays. - - /** - * Given a source array with shape {@link dims} and content {@link src}, copy the contents into - * the input Tensor with name {@link inputName}. The source array {@link src} must have at least - * as many elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, boolean[] src, long... dims) { - byte[] b = new byte[src.length]; - - for (int i = 0; i < src.length; i++) { - b[i] = src[i] ? (byte) 1 : (byte) 0; - } - - addFeed(inputName, Tensor.create(Boolean.class, dims, ByteBuffer.wrap(b))); - } - - /** - * Given a source array with shape {@link dims} and content {@link src}, copy the contents into - * the input Tensor with name {@link inputName}. The source array {@link src} must have at least - * as many elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, float[] src, long... dims) { - addFeed(inputName, Tensor.create(dims, FloatBuffer.wrap(src))); - } - - /** - * Given a source array with shape {@link dims} and content {@link src}, copy the contents into - * the input Tensor with name {@link inputName}. The source array {@link src} must have at least - * as many elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, int[] src, long... dims) { - addFeed(inputName, Tensor.create(dims, IntBuffer.wrap(src))); - } - - /** - * Given a source array with shape {@link dims} and content {@link src}, copy the contents into - * the input Tensor with name {@link inputName}. The source array {@link src} must have at least - * as many elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, long[] src, long... dims) { - addFeed(inputName, Tensor.create(dims, LongBuffer.wrap(src))); - } - - /** - * Given a source array with shape {@link dims} and content {@link src}, copy the contents into - * the input Tensor with name {@link inputName}. The source array {@link src} must have at least - * as many elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, double[] src, long... dims) { - addFeed(inputName, Tensor.create(dims, DoubleBuffer.wrap(src))); - } - - /** - * Given a source array with shape {@link dims} and content {@link src}, copy the contents into - * the input Tensor with name {@link inputName}. The source array {@link src} must have at least - * as many elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, byte[] src, long... dims) { - addFeed(inputName, Tensor.create(UInt8.class, dims, ByteBuffer.wrap(src))); - } - - /** - * Copy a byte sequence into the input Tensor with name {@link inputName} as a string-valued - * scalar tensor. In the TensorFlow type system, a "string" is an arbitrary sequence of bytes, not - * a Java {@code String} (which is a sequence of characters). - */ - public void feedString(String inputName, byte[] src) { - addFeed(inputName, Tensors.create(src)); - } - - /** - * Copy an array of byte sequences into the input Tensor with name {@link inputName} as a - * string-valued one-dimensional tensor (vector). In the TensorFlow type system, a "string" is an - * arbitrary sequence of bytes, not a Java {@code String} (which is a sequence of characters). - */ - public void feedString(String inputName, byte[][] src) { - addFeed(inputName, Tensors.create(src)); - } - - // Methods for taking a native Tensor and filling it with src from Java native IO buffers. - - /** - * Given a source buffer with shape {@link dims} and content {@link src}, both stored as - * direct and native ordered java.nio buffers, copy the contents into the input - * Tensor with name {@link inputName}. The source buffer {@link src} must have at least as many - * elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, FloatBuffer src, long... dims) { - addFeed(inputName, Tensor.create(dims, src)); - } - - /** - * Given a source buffer with shape {@link dims} and content {@link src}, both stored as - * direct and native ordered java.nio buffers, copy the contents into the input - * Tensor with name {@link inputName}. The source buffer {@link src} must have at least as many - * elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, IntBuffer src, long... dims) { - addFeed(inputName, Tensor.create(dims, src)); - } - - /** - * Given a source buffer with shape {@link dims} and content {@link src}, both stored as - * direct and native ordered java.nio buffers, copy the contents into the input - * Tensor with name {@link inputName}. The source buffer {@link src} must have at least as many - * elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, LongBuffer src, long... dims) { - addFeed(inputName, Tensor.create(dims, src)); - } - - /** - * Given a source buffer with shape {@link dims} and content {@link src}, both stored as - * direct and native ordered java.nio buffers, copy the contents into the input - * Tensor with name {@link inputName}. The source buffer {@link src} must have at least as many - * elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, DoubleBuffer src, long... dims) { - addFeed(inputName, Tensor.create(dims, src)); - } - - /** - * Given a source buffer with shape {@link dims} and content {@link src}, both stored as - * direct and native ordered java.nio buffers, copy the contents into the input - * Tensor with name {@link inputName}. The source buffer {@link src} must have at least as many - * elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, ByteBuffer src, long... dims) { - addFeed(inputName, Tensor.create(UInt8.class, dims, src)); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into a Java array. {@link - * dst} must have length greater than or equal to that of the source Tensor. This operation will - * not affect dst's content past the source Tensor's size. - */ - public void fetch(String outputName, float[] dst) { - fetch(outputName, FloatBuffer.wrap(dst)); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into a Java array. {@link - * dst} must have length greater than or equal to that of the source Tensor. This operation will - * not affect dst's content past the source Tensor's size. - */ - public void fetch(String outputName, int[] dst) { - fetch(outputName, IntBuffer.wrap(dst)); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into a Java array. {@link - * dst} must have length greater than or equal to that of the source Tensor. This operation will - * not affect dst's content past the source Tensor's size. - */ - public void fetch(String outputName, long[] dst) { - fetch(outputName, LongBuffer.wrap(dst)); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into a Java array. {@link - * dst} must have length greater than or equal to that of the source Tensor. This operation will - * not affect dst's content past the source Tensor's size. - */ - public void fetch(String outputName, double[] dst) { - fetch(outputName, DoubleBuffer.wrap(dst)); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into a Java array. {@link - * dst} must have length greater than or equal to that of the source Tensor. This operation will - * not affect dst's content past the source Tensor's size. - */ - public void fetch(String outputName, byte[] dst) { - fetch(outputName, ByteBuffer.wrap(dst)); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into the direct and - * native ordered java.nio buffer {@link dst}. {@link dst} must have capacity greater than - * or equal to that of the source Tensor. This operation will not affect dst's content past the - * source Tensor's size. - */ - public void fetch(String outputName, FloatBuffer dst) { - getTensor(outputName).writeTo(dst); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into the direct and - * native ordered java.nio buffer {@link dst}. {@link dst} must have capacity greater than - * or equal to that of the source Tensor. This operation will not affect dst's content past the - * source Tensor's size. - */ - public void fetch(String outputName, IntBuffer dst) { - getTensor(outputName).writeTo(dst); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into the direct and - * native ordered java.nio buffer {@link dst}. {@link dst} must have capacity greater than - * or equal to that of the source Tensor. This operation will not affect dst's content past the - * source Tensor's size. - */ - public void fetch(String outputName, LongBuffer dst) { - getTensor(outputName).writeTo(dst); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into the direct and - * native ordered java.nio buffer {@link dst}. {@link dst} must have capacity greater than - * or equal to that of the source Tensor. This operation will not affect dst's content past the - * source Tensor's size. - */ - public void fetch(String outputName, DoubleBuffer dst) { - getTensor(outputName).writeTo(dst); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into the direct and - * native ordered java.nio buffer {@link dst}. {@link dst} must have capacity greater than - * or equal to that of the source Tensor. This operation will not affect dst's content past the - * source Tensor's size. - */ - public void fetch(String outputName, ByteBuffer dst) { - getTensor(outputName).writeTo(dst); - } - - private void prepareNativeRuntime() { - Log.i(TAG, "Checking to see if TensorFlow native methods are already loaded"); - try { - // Hack to see if the native libraries have been loaded. - new RunStats(); - Log.i(TAG, "TensorFlow native methods already loaded"); - } catch (UnsatisfiedLinkError e1) { - Log.i( - TAG, "TensorFlow native methods not found, attempting to load via tensorflow_inference"); - try { - System.loadLibrary("tensorflow_inference"); - Log.i(TAG, "Successfully loaded TensorFlow native methods (RunStats error may be ignored)"); - } catch (UnsatisfiedLinkError e2) { - throw new RuntimeException( - "Native TF methods not found; check that the correct native" - + " libraries are present in the APK."); - } - } - } - - private void loadGraph(byte[] graphDef, Graph g) throws IOException { - final long startMs = System.currentTimeMillis(); - - if (VERSION.SDK_INT >= 18) { - Trace.beginSection("importGraphDef"); - } - - try { - g.importGraphDef(graphDef); - } catch (IllegalArgumentException e) { - throw new IOException("Not a valid TensorFlow Graph serialization: " + e.getMessage()); - } - - if (VERSION.SDK_INT >= 18) { - Trace.endSection(); // importGraphDef. - } - - final long endMs = System.currentTimeMillis(); - Log.i( - TAG, - "Model load took " + (endMs - startMs) + "ms, TensorFlow version: " + TensorFlow.version()); - } - - private void addFeed(String inputName, Tensor t) { - // The string format accepted by TensorFlowInferenceInterface is node_name[:output_index]. - TensorId tid = TensorId.parse(inputName); - runner.feed(tid.name, tid.outputIndex, t); - feedNames.add(inputName); - feedTensors.add(t); - } - - private static class TensorId { - String name; - int outputIndex; - - // Parse output names into a TensorId. - // - // E.g., "foo" --> ("foo", 0), while "foo:1" --> ("foo", 1) - public static TensorId parse(String name) { - TensorId tid = new TensorId(); - int colonIndex = name.lastIndexOf(':'); - if (colonIndex < 0) { - tid.outputIndex = 0; - tid.name = name; - return tid; - } - try { - tid.outputIndex = Integer.parseInt(name.substring(colonIndex + 1)); - tid.name = name.substring(0, colonIndex); - } catch (NumberFormatException e) { - tid.outputIndex = 0; - tid.name = name; - } - return tid; - } - } - - private Tensor getTensor(String outputName) { - int i = 0; - for (String n : fetchNames) { - if (n.equals(outputName)) { - return fetchTensors.get(i); - } - ++i; - } - throw new RuntimeException( - "Node '" + outputName + "' was not provided to run(), so it cannot be read"); - } - - private void closeFeeds() { - for (Tensor t : feedTensors) { - t.close(); - } - feedTensors.clear(); - feedNames.clear(); - } - - private void closeFetches() { - for (Tensor t : fetchTensors) { - t.close(); - } - fetchTensors.clear(); - fetchNames.clear(); - } - - // Immutable state. - private final String modelName; - private final Graph g; - private final Session sess; - - // State reset on every call to run. - private Session.Runner runner; - private List feedNames = new ArrayList(); - private List> feedTensors = new ArrayList>(); - private List fetchNames = new ArrayList(); - private List> fetchTensors = new ArrayList>(); - - // Mutable state. - private RunStats runStats; -} diff --git a/tensorflow/contrib/android/jni/run_stats_jni.cc b/tensorflow/contrib/android/jni/run_stats_jni.cc deleted file mode 100644 index 30de7b59af7..00000000000 --- a/tensorflow/contrib/android/jni/run_stats_jni.cc +++ /dev/null @@ -1,83 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/android/jni/run_stats_jni.h" - -#include - -#include - -#include "tensorflow/core/protobuf/config.pb.h" -#include "tensorflow/core/util/stat_summarizer.h" - -using tensorflow::RunMetadata; -using tensorflow::StatSummarizer; - -namespace { -StatSummarizer* requireHandle(JNIEnv* env, jlong handle) { - if (handle == 0) { - env->ThrowNew(env->FindClass("java/lang/IllegalStateException"), - "close() has been called on the RunStats object"); - return nullptr; - } - return reinterpret_cast(handle); -} -} // namespace - -#define RUN_STATS_METHOD(name) \ - JNICALL Java_org_tensorflow_contrib_android_RunStats_##name - -JNIEXPORT jlong RUN_STATS_METHOD(allocate)(JNIEnv* env, jclass clazz) { - static_assert(sizeof(jlong) >= sizeof(StatSummarizer*), - "Cannot package C++ object pointers as a Java long"); - tensorflow::StatSummarizerOptions opts; - return reinterpret_cast(new StatSummarizer(opts)); -} - -JNIEXPORT void RUN_STATS_METHOD(delete)(JNIEnv* env, jclass clazz, - jlong handle) { - if (handle == 0) return; - delete reinterpret_cast(handle); -} - -JNIEXPORT void RUN_STATS_METHOD(add)(JNIEnv* env, jclass clazz, jlong handle, - jbyteArray run_metadata) { - StatSummarizer* s = requireHandle(env, handle); - if (s == nullptr) return; - jbyte* data = env->GetByteArrayElements(run_metadata, nullptr); - int size = static_cast(env->GetArrayLength(run_metadata)); - tensorflow::RunMetadata proto; - if (!proto.ParseFromArray(data, size)) { - env->ThrowNew(env->FindClass("java/lang/IllegalArgumentException"), - "runMetadata does not seem to be a serialized RunMetadata " - "protocol message"); - } else if (proto.has_step_stats()) { - s->ProcessStepStats(proto.step_stats()); - } - env->ReleaseByteArrayElements(run_metadata, data, JNI_ABORT); -} - -JNIEXPORT jstring RUN_STATS_METHOD(summary)(JNIEnv* env, jclass clazz, - jlong handle) { - StatSummarizer* s = requireHandle(env, handle); - if (s == nullptr) return nullptr; - std::stringstream ret; - ret << s->GetStatsByMetric("Top 10 CPU", tensorflow::StatsCalculator::BY_TIME, - 10) - << s->GetStatsByNodeType() << s->ShortSummary(); - return env->NewStringUTF(ret.str().c_str()); -} - -#undef RUN_STATS_METHOD diff --git a/tensorflow/contrib/android/jni/run_stats_jni.h b/tensorflow/contrib/android/jni/run_stats_jni.h deleted file mode 100644 index de3bceff0a1..00000000000 --- a/tensorflow/contrib/android/jni/run_stats_jni.h +++ /dev/null @@ -1,40 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef ORG_TENSORFLOW_JNI_RUN_STATS_JNI_H_ -#define ORG_TENSORFLOW_JNI_RUN_STATS_JNI_H_ - -#include - -#ifdef __cplusplus -extern "C" { -#endif // __cplusplus - -#define RUN_STATS_METHOD(name) \ - Java_org_tensorflow_contrib_android_RunStats_##name - -JNIEXPORT JNICALL jlong RUN_STATS_METHOD(allocate)(JNIEnv*, jclass); -JNIEXPORT JNICALL void RUN_STATS_METHOD(delete)(JNIEnv*, jclass, jlong); -JNIEXPORT JNICALL void RUN_STATS_METHOD(add)(JNIEnv*, jclass, jlong, - jbyteArray); -JNIEXPORT JNICALL jstring RUN_STATS_METHOD(summary)(JNIEnv*, jclass, jlong); - -#undef RUN_STATS_METHOD - -#ifdef __cplusplus -} // extern "C" -#endif // __cplusplus - -#endif // ORG_TENSORFLOW_JNI_RUN_STATS_JNI_H_ diff --git a/tensorflow/contrib/android/jni/version_script.lds b/tensorflow/contrib/android/jni/version_script.lds deleted file mode 100644 index 38c93dda730..00000000000 --- a/tensorflow/contrib/android/jni/version_script.lds +++ /dev/null @@ -1,11 +0,0 @@ -VERS_1.0 { - # Export JNI symbols. - global: - Java_*; - JNI_OnLoad; - JNI_OnUnload; - - # Hide everything else. - local: - *; -}; diff --git a/tensorflow/contrib/autograph/BUILD b/tensorflow/contrib/autograph/BUILD deleted file mode 100644 index c286632ca0c..00000000000 --- a/tensorflow/contrib/autograph/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -package( - licenses = ["notice"], # Apache 2.0 -) - -filegroup( - name = "all_files", - srcs = glob( - ["**/*"], - exclude = [ - "**/METADATA", - "**/OWNERS", - ], - ), - visibility = ["//tensorflow:__subpackages__"], -) - -py_library( - name = "autograph", - srcs = [ - "__init__.py", - ], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - # This module is kept for backward compatibility only. To depend on AutoGraph, - # use //third_party/tensorflow/python/autograph instead. - deps = [ - "//tensorflow/python/autograph", - ], -) diff --git a/tensorflow/contrib/autograph/README.md b/tensorflow/contrib/autograph/README.md deleted file mode 100644 index 8c277b59e8f..00000000000 --- a/tensorflow/contrib/autograph/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# AutoGraph - -**NOTE: As tensorflow.contrib is being -[deprecated](https://github.com/tensorflow/community/pull/18), AutoGraph is -moving into TensorFlow core. - -The new code location is `tensorflow/python/autograph`. Please refer to the -README.md file in that directory. -** diff --git a/tensorflow/contrib/autograph/__init__.py b/tensorflow/contrib/autograph/__init__.py deleted file mode 100644 index 137bc59202b..00000000000 --- a/tensorflow/contrib/autograph/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""This is the legacy module for AutoGraph, kept for backward compatibility. - -New users should instead use `tensorflow.python.autograph`. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.autograph import * # pylint:disable=wildcard-import diff --git a/tensorflow/contrib/autograph/examples/benchmarks/BUILD b/tensorflow/contrib/autograph/examples/benchmarks/BUILD deleted file mode 100644 index 2cbff6b4d12..00000000000 --- a/tensorflow/contrib/autograph/examples/benchmarks/BUILD +++ /dev/null @@ -1,39 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "py_test") -load("//tensorflow/tools/test:performance.bzl", "tf_py_logged_benchmark") - -package( - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "benchmark_base", - srcs = [ - "benchmark_base.py", - ], - deps = [ - "//tensorflow:tensorflow_py", - ], -) - -py_test( - name = "cartpole_benchmark", - size = "enormous", - srcs = ["cartpole_benchmark.py"], - python_version = "PY2", - tags = [ - "local", - "manual", - "no_oss", - "notap", - "nozapfhahn", - ], - deps = [ - ":benchmark_base", - # Note: required gym dependency may need to be added here. - ], -) - -tf_py_logged_benchmark( - name = "cartpole_logged_benchmark", - target = "//tensorflow/contrib/autograph/examples/benchmarks:cartpole_benchmark", -) diff --git a/tensorflow/contrib/autograph/examples/benchmarks/benchmark_base.py b/tensorflow/contrib/autograph/examples/benchmarks/benchmark_base.py deleted file mode 100644 index 25414fbda62..00000000000 --- a/tensorflow/contrib/autograph/examples/benchmarks/benchmark_base.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Common benchmarking code. - -See https://www.tensorflow.org/community/benchmarks for usage. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time - -import numpy as np - -import tensorflow as tf - - -class ReportingBenchmark(tf.test.Benchmark): - """Base class for a benchmark that reports general performance metrics. - - Subclasses only need to call one of the _profile methods, and optionally - report_results. - """ - - def time_execution(self, name, target, iters, warm_up_iters=5): - for _ in range(warm_up_iters): - target() - - all_times = [] - for _ in range(iters): - iter_time = time.time() - target() - all_times.append(time.time() - iter_time) - - avg_time = np.average(all_times) - - extras = {} - extras['all_times'] = all_times - - if isinstance(name, tuple): - extras['name'] = name - name = '_'.join(str(piece) for piece in name) - - self.report_benchmark( - iters=iters, wall_time=avg_time, name=name, extras=extras) - - -if __name__ == '__main__': - tf.test.main() diff --git a/tensorflow/contrib/autograph/examples/benchmarks/cartpole_benchmark.py b/tensorflow/contrib/autograph/examples/benchmarks/cartpole_benchmark.py deleted file mode 100644 index 4f553be58e9..00000000000 --- a/tensorflow/contrib/autograph/examples/benchmarks/cartpole_benchmark.py +++ /dev/null @@ -1,492 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A basic RL cartpole benchmark. - -The RL model uses the OpenAI Gym environment to train a simple network using -the policy gradients method. The training scales the gradients for each step -by the episode's cumulative discounted reward and averages these gradients over -a fixed number of games before applying the optimization step. - -For benchmarking purposes, we replace the OpenAI Gym environment to a fake -that returns random actions and rewards and never ends the episode. This way -the benchmarks compare the same amount of computation at each step. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import gym -import numpy as np -import tensorflow as tf - -from tensorflow.contrib import eager -from tensorflow.contrib.autograph.examples.benchmarks import benchmark_base -from tensorflow.python import autograph as ag -from tensorflow.python.eager import context - -# -# AutoGraph implementation -# - - -@ag.convert() -def graph_append_discounted_rewards(destination, rewards, discount_rate): - """Discounts episode rewards and appends them to destination.""" - ag.set_element_type(rewards, tf.float32) - - cdr = 0.0 - reverse_discounted = [] - ag.set_element_type(reverse_discounted, tf.float32) - - for i in range(len(rewards) - 1, -1, -1): - cdr = cdr * discount_rate + rewards[i] - cdr.set_shape(()) - reverse_discounted.append(cdr) - - retval = destination - # Note: AutoGraph doesn't yet support reversed() so we use a loop instead. - for i in range(len(reverse_discounted) - 1, -1, -1): - retval.append(reverse_discounted[i]) - - return retval - - -class GraphPolicyNetwork(tf.keras.Model): - """Policy network for the cart-pole reinforcement learning problem. - - The forward path of the network takes an observation from the cart-pole - environment (length-4 vector) and outputs an action. - """ - - def __init__(self, hidden_size): - super(GraphPolicyNetwork, self).__init__() - self._hidden_layer = tf.keras.layers.Dense( - hidden_size, activation=tf.nn.elu) - self._output_layer = tf.keras.layers.Dense(1) - - def call(self, inputs): - """Calculates logits and action. - - Args: - inputs: Observations from a step in the cart-pole environment, of shape - `(batch_size, input_size)` - - Returns: - logits: the logits output by the output layer. This can be viewed as the - likelihood vales of choosing the left (0) action. Shape: - `(batch_size, 1)`. - actions: randomly selected actions ({0, 1}) based on the logits. Shape: - `(batch_size, 1)`. - """ - hidden = self._hidden_layer(inputs) - logits = self._output_layer(hidden) - - left_prob = tf.nn.sigmoid(logits) - action_probs = tf.concat([left_prob, 1.0 - left_prob], 1) - - actions = tf.multinomial(tf.log(action_probs), 1) - return logits, actions - - # TODO(mdan): Move this method out of the class. - @ag.convert() - def train(self, cart_pole_env, optimizer, discount_rate, num_games, - max_steps_per_game): - var_list = tf.trainable_variables() - grad_list = [ - tf.TensorArray(tf.float32, 0, dynamic_size=True) for _ in var_list - ] - - step_counts = [] - discounted_rewards = [] - ag.set_element_type(discounted_rewards, tf.float32) - ag.set_element_type(step_counts, tf.int32) - - # Note: we use a shared object, cart_pole_env here. Because calls to the - # object's method are made through py_func, TensorFlow cannot detect its - # data dependencies. Hence we must manually synchronize access to it - # and ensure the control dependencies are set in such a way that - # calls to reset(), take_one_step, etc. are made in the correct order. - sync_counter = tf.constant(0) - - for _ in tf.range(num_games): - with tf.control_dependencies([sync_counter]): - obs = cart_pole_env.reset() - with tf.control_dependencies([obs]): - sync_counter += 1 - - game_rewards = [] - ag.set_element_type(game_rewards, tf.float32) - - for step in tf.range(max_steps_per_game): - logits, actions = self(obs) # pylint:disable=not-callable - logits = tf.reshape(logits, ()) - actions = tf.reshape(actions, ()) - - labels = 1.0 - tf.cast(actions, tf.float32) - loss = tf.nn.sigmoid_cross_entropy_with_logits( - labels=labels, logits=logits) - grads = tf.gradients(loss, var_list) - - for i in range(len(grads)): - grad_list[i].append(grads[i]) - - with tf.control_dependencies([sync_counter]): - obs, reward, done = cart_pole_env.step(actions) - with tf.control_dependencies([obs]): - sync_counter += 1 - obs = tf.reshape(obs, (1, 4)) - - game_rewards.append(reward) - if reward < 0.1 or done: - step_counts.append(step + 1) - break - - discounted_rewards = graph_append_discounted_rewards( - discounted_rewards, game_rewards, discount_rate) - - discounted_rewards = ag.stack(discounted_rewards) - discounted_rewards.set_shape((None,)) - mean, variance = tf.nn.moments(discounted_rewards, [0]) - normalized_rewards = (discounted_rewards - mean) / tf.sqrt(variance) - - for i in range(len(grad_list)): - g = ag.stack(grad_list[i]) - - # This block just adjusts the shapes to match for multiplication. - r = normalized_rewards - if r.shape.ndims < g.shape.ndims: - r = tf.expand_dims(r, -1) - if r.shape.ndims < g.shape.ndims: - r = tf.expand_dims(r, -1) - - grad_list[i] = tf.reduce_mean(g * r, axis=0) - - optimizer.apply_gradients( - zip(grad_list, var_list), global_step=tf.train.get_global_step()) - - return ag.stack(step_counts) - - -@ag.convert() -def graph_train_model(policy_network, cart_pole_env, optimizer, iterations): - """Trains the policy network for a given number of iterations.""" - i = tf.constant(0) - mean_steps_per_iteration = [] - ag.set_element_type(mean_steps_per_iteration, tf.int32) - - while i < iterations: - steps_per_game = policy_network.train( - cart_pole_env, - optimizer, - discount_rate=0.95, - num_games=20, - max_steps_per_game=200) - mean_steps_per_iteration.append(tf.reduce_mean(steps_per_game)) - i += 1 - - return ag.stack(mean_steps_per_iteration) - - -class GraphGymCartpoleEnv(object): - """An env backed by OpenAI Gym's CartPole environment. - - Used to confirm a functional model only. - """ - - def __init__(self): - cart_pole_env = gym.make('CartPole-v1') - cart_pole_env.seed(0) - cart_pole_env.reset() - self.env = cart_pole_env - - def reset(self): - obs = ag.utils.wrap_py_func(self.env.reset, tf.float64, ()) - obs = tf.reshape(obs, (1, 4)) - obs = tf.cast(obs, tf.float32) - return obs - - def step(self, actions): - - def take_one_step(actions): - obs, reward, done, _ = self.env.step(actions) - obs = obs.astype(np.float32) - reward = np.float32(reward) - return obs, reward, done - - return ag.utils.wrap_py_func(take_one_step, - (tf.float32, tf.float32, tf.bool), (actions,)) - - -class GraphRandomCartpoleEnv(object): - """An environment that returns random actions and never finishes. - - Used during benchmarking, it will cause training to run a constant number of - steps. - """ - - def reset(self): - return tf.random.normal((1, 4)) - - def step(self, actions): - with tf.control_dependencies([actions]): - random_obs = tf.random.normal((1, 4)) - fixed_reward = tf.constant(0.001) - done = tf.constant(False) - return random_obs, fixed_reward, done - - -# -# Eager implementation -# - - -def eager_append_discounted_rewards(discounted_rewards, rewards, discount_rate): - cdr = 0.0 - reverse_discounted = [] - - for i in range(len(rewards) - 1, -1, -1): - cdr = cdr * discount_rate + rewards[i] - reverse_discounted.append(cdr) - - discounted_rewards.extend(reversed(reverse_discounted)) - return discounted_rewards - - -class EagerPolicyNetwork(tf.keras.Model): - """Policy network for the cart-pole reinforcement learning problem. - - The forward path of the network takes an observation from the cart-pole - environment (length-4 vector) and outputs an action. - """ - - def __init__(self, hidden_size): - super(EagerPolicyNetwork, self).__init__() - self._hidden_layer = tf.keras.layers.Dense( - hidden_size, activation=tf.nn.elu) - self._output_layer = tf.keras.layers.Dense(1) - - def call(self, inputs): - """Calculates logits and action. - - Args: - inputs: Observations from a step in the cart-pole environment, of shape - `(batch_size, input_size)` - - Returns: - logits: the logits output by the output layer. This can be viewed as the - likelihood vales of choosing the left (0) action. Shape: - `(batch_size, 1)`. - actions: randomly selected actions ({0, 1}) based on the logits. Shape: - `(batch_size, 1)`. - """ - hidden = self._hidden_layer(inputs) - logits = self._output_layer(hidden) - - left_prob = tf.nn.sigmoid(logits) - action_probs = tf.concat([left_prob, 1.0 - left_prob], 1) - - self._grad_fn = eager.implicit_gradients( - self._get_cross_entropy_and_save_actions) - - actions = tf.multinomial(tf.log(action_probs), 1) - return logits, actions - - def _get_cross_entropy_and_save_actions(self, inputs): - logits, actions = self(inputs) # pylint:disable=not-callable - self._current_actions = actions - labels = 1.0 - tf.cast(actions, tf.float32) - return tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits) - - def train(self, cart_pole_env, optimizer, discount_rate, num_games, - max_steps_per_game): - grad_list = None - - step_counts = [] - discounted_rewards = [] - - for _ in range(num_games): - obs = cart_pole_env.reset() - - game_rewards = [] - - for step in range(max_steps_per_game): - grads_and_vars = self._grad_fn(tf.constant([obs], dtype=tf.float32)) - grads, var_list = zip(*grads_and_vars) - actions = self._current_actions.numpy()[0][0] - - if grad_list is None: - grad_list = [[g] for g in grads] - else: - for i in range(len(grads)): - grad_list[i].append(grads[i]) - - obs, reward, done = cart_pole_env.step(actions) - - game_rewards.append(reward) - if reward < 0.1 or done: - step_counts.append(step + 1) - break - - discounted_rewards = eager_append_discounted_rewards( - discounted_rewards, game_rewards, discount_rate) - - discounted_rewards = tf.stack(discounted_rewards) - mean, variance = tf.nn.moments(discounted_rewards, [0]) - normalized_rewards = (discounted_rewards - mean) / tf.sqrt(variance) - - for i in range(len(grad_list)): - g = tf.stack(grad_list[i]) - - r = normalized_rewards - while r.shape.ndims < g.shape.ndims: - r = tf.expand_dims(r, -1) - - grad_list[i] = tf.reduce_mean(g * r, axis=0) - - optimizer.apply_gradients( - zip(grad_list, var_list), global_step=tf.train.get_global_step()) - - return tf.stack(step_counts) - - -def eager_train_model(policy_network, cart_pole_env, optimizer, iterations): - """Trains the policy network for a given number of iterations.""" - mean_steps_per_iteration = [] - - for _ in range(iterations): - steps_per_game = policy_network.train( - cart_pole_env, - optimizer, - discount_rate=0.95, - num_games=20, - max_steps_per_game=200) - mean_steps_per_iteration.append(tf.reduce_mean(steps_per_game)) - - return mean_steps_per_iteration - - -class EagerGymCartpoleEnv(object): - """An env backed by OpenAI Gym's CartPole environment. - - Used to confirm a functional model only. - """ - - def __init__(self): - cart_pole_env = gym.make('CartPole-v1') - cart_pole_env.seed(0) - cart_pole_env.reset() - self.env = cart_pole_env - - def reset(self): - return self.env.reset() - - def step(self, actions): - obs, reward, done, _ = self.env.step(actions) - return obs, reward, done - - -class EagerRandomCartpoleEnv(object): - """An environment that returns random actions and never finishes. - - Used during benchmarking, it will cause training to run a constant number of - steps. - """ - - def reset(self): - return np.random.normal(size=(4,)) - - def step(self, actions): - with tf.control_dependencies([actions]): - random_obs = np.random.normal(size=(4,)) - fixed_reward = 0.001 - done = False - return random_obs, fixed_reward, done - - -def graph_demo_training(): - """Not used in the benchmark. Used to confirm a functional model.""" - with tf.Graph().as_default(): - tf.set_random_seed(0) - - network = GraphPolicyNetwork(hidden_size=5) - network.build((1, 4)) - env = GraphGymCartpoleEnv() - opt = tf.train.AdamOptimizer(0.05) - - train_ops = graph_train_model(network, env, opt, iterations=5) - - with tf.Session() as sess: - sess.run(tf.global_variables_initializer()) - sess.run(tf.local_variables_initializer()) - steps_per_iteration = sess.run(train_ops) - for i, steps in enumerate(steps_per_iteration): - print('Step {} iterations: {}'.format(i, steps)) - - -def eager_demo_training(): - with context.eager_mode(): - network = EagerPolicyNetwork(hidden_size=5) - network.build((1, 4)) - env = EagerGymCartpoleEnv() - opt = tf.train.AdamOptimizer(0.05) - - steps_per_iteration = eager_train_model(network, env, opt, iterations=5) - for i, steps in enumerate(steps_per_iteration): - print('Step {} iterations: {}'.format(i, steps)) - - -class RLCartPoleBenchmark(benchmark_base.ReportingBenchmark): - """Actual benchmark. - - Trains the RL agent a fixed number of times, on random environments that - result in constant number of steps. - """ - - def benchmark_cartpole(self): - - def train_session(sess, ops): - return lambda: sess.run(ops) - - def train_eager(network, env, opt): - return lambda: eager_train_model(network, env, opt, iterations=10) - - for model_size in (10, 100, 1000): - with tf.Graph().as_default(): - network = GraphPolicyNetwork(hidden_size=model_size) - network.build((1, 4)) - env = GraphRandomCartpoleEnv() - opt = tf.train.AdamOptimizer(0.05) - train_ops = graph_train_model(network, env, opt, iterations=10) - - with tf.Session() as sess: - sess.run(tf.global_variables_initializer()) - sess.run(tf.local_variables_initializer()) - - self.time_execution(('cartpole', 'autograph', model_size), - train_session(sess, train_ops), 20) - - with context.eager_mode(): - network = EagerPolicyNetwork(hidden_size=model_size) - network.build((1, 4)) - env = EagerRandomCartpoleEnv() - opt = tf.train.AdamOptimizer(0.05) - - self.time_execution(('cartpole', 'eager', model_size), - train_eager(network, env, opt), 20) - - -if __name__ == '__main__': - tf.test.main() diff --git a/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_collatz_speed_test.ipynb b/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_collatz_speed_test.ipynb deleted file mode 100644 index c10a5741f64..00000000000 --- a/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_collatz_speed_test.ipynb +++ /dev/null @@ -1,299 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aQkTGc-d8I1k" - }, - "source": [ - "This notebook runs a basic speed test for a simple algorithm that implements the process described in Collatz Conjecture.\n", - "\n", - "https://en.wikipedia.org/wiki/Collatz_conjecture" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x5ChBlH09jk_" - }, - "source": [ - "### Imports" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "X-QAUpWdPxUh" - }, - "outputs": [], - "source": [ - "!pip install -U -q tf-nightly" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "wiKQu3w05eCa" - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "from matplotlib import pyplot as plt\n", - "import tensorflow as tf\n", - "from tensorflow.contrib import autograph as ag\n", - "from tensorflow.python.eager import context" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_cRFTcwT9mnn" - }, - "source": [ - "### Plotting helpers" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "ww7rc0GQ9pMu" - }, - "outputs": [], - "source": [ - "def plot_results(counts, times, title):\n", - " plt.plot(counts, np.array(times) * 1000., 'o')\n", - " plt.ylabel('Time (milliseconds)')\n", - " plt.xlabel('Collatz counter')\n", - " plt.title(title)\n", - " plt.ylim(0, 30)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ESZGw9s9-Y5_" - }, - "source": [ - "### Collatz function definition" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "qeunWm9m-dT7" - }, - "outputs": [], - "source": [ - "def collatz(a):\n", - " count = 0\n", - " while a \u003e 1.1:\n", - " if a % 2 \u003c 0.1:\n", - " a //= 2\n", - " else:\n", - " a = 3 * a + 1\n", - " count += 1\n", - " return count\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nnFmPDvScsDo" - }, - "source": [ - "# AutoGraph" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 301 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 9153, - "status": "ok", - "timestamp": 1531757473651, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "6fU4vlxYcsDe", - "outputId": "11b50f28-aced-4506-a743-4b749e9645c3" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEcCAYAAAAydkhNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtcVGXCB/DfGRBUQA0ZURQvyIspm1reQkxNSPICgoqW\npWZu1vbmjZJV3Jc+axappVLu7guV25rU5g3wlq3iBd1wXHSN3hXy9ZaCgoOIIKAzMOf9g5dZkTkz\nB5i7v+9fzJlzzjzPHD2/Oc/znOcIoiiKICIiMkBh6wIQEZH9YkgQEZEkhgQREUliSBARkSSGBBER\nSWJIEBGRJIYEkQNYsWIFkpOTbV0MegQxJMipzJ49G8OHD4dWq5W9zeOPP45r164163O2bt2KqKgo\nDB48GKNGjcKcOXOwf//+5haXyO4xJMhpFBUV4fTp0xAEAYcPH5a9nSAIzfqc9957D1999RVWrFiB\nU6dO4fjx41iyZAmOHz8uuQ3vWSVHxZAgp5GRkYHBgwdj6tSpSE9P1y+fPXs2duzYoX+dnp6OWbNm\nAQBefvlliKKIqKgoPPXUU/juu+8AANu2bcP48eMxYsQIvPnmm7h58yYA4PLly/jmm2+wYcMGhISE\nwM3NDYIg4KmnnkJSUlKjz9ywYQNefPFFDB48GIWFhdi1axcmTpyIp556Cs899xy+/fZb/fqnTp3C\nmDFjkJKSgqeffhphYWHYs2dPo/rduXMHr7/+Op566inMnDmz2Vc/RC3BkCCnkZmZiaioKEyePBkn\nTpxAWVmZ5LoNVw9bt24FAOzevRtnzpzBhAkTkJOTg/Xr1+OTTz7BiRMn4Ofnh7i4OACASqVCt27d\nMGDAAJPl2bNnD1avXo0zZ86gW7du6Ny5M1JTU3HmzBkkJSUhKSkJ+fn5+vVLS0tRXl6O48eP48MP\nP0RiYiKuXLmif3/fvn1YuHAhcnNz4e/vj40bN7bkayJqFoYEOYXc3Fxcv34dEyZMQHBwMHr27Nnk\nl7hce/fuxfTp0/H444+jTZs2iIuLw9mzZ3H9+nXcvn0bSqWy0fpjxozBsGHDMHDgQNy4cUO/PCYm\nBn379oVCoYCrqyvGjBmDHj16AACGDh2K0NBQ5Obm6tcXBAFLlixBmzZtMGzYMIwZM0Z/ZQMA48eP\nx69+9SsoFApERkY2ChgiS2FIkFPIzMzEqFGj0LFjRwDApEmTkJGR0aJ93bx5E35+fvrX7du3R6dO\nnVBSUoJOnTrpm54aHDt2DCdPnoRWq23U99C1a9cm682cORMjRozAsGHDkJ2djdu3b+vf79ChA9zd\n3fWv/fz8Gn2Wj4+P/u927dqhurq6RfUjag5XWxeAqLXu37+P7777DjqdDqNGjQIAaDQaVFZWoqCg\nAO3bt8e9e/f066vVaqP769KlC65fv65/XV1djfLycvj6+qJTp05YvXo1/vWvfyE4OLjRdg93Tj/Y\nIa7RaLB48WKsW7cOYWFhUCgU+M///M9G21RUVODevXto27YtAODGjRsICgpq5rdBZF68kiCHd/Dg\nQbi4uOC7775DZmYmMjMz8d1332Ho0KHIzMxE//798be//Q337t3DL7/8gp07dzba3sfHp1En8OTJ\nk7Fr1y4UFBRAo9Fg/fr1GDRoEPz8/NCnTx/MnDkTcXFx+OGHH3D//n3odDqcOXPG6CgprVYLrVaL\nxx57DAqFAseOHcPf//73RuuIoohPPvkEWq0Wubm5OHr0KCZMmGDeL4uomXglQQ4vIyMD06ZNg6+v\nb6Pls2bNwvvvv4+9e/fip59+QmhoKPr164fIyEjk5OTo11u4cCHi4+Oh0WiwatUqPP/881i8eDEW\nLlyIiooKPPnkk1i/fr1+/cTERGzduhVJSUm4du0avLy80Lt3b2zcuFHfTPVwYHh4eGDlypVYvHgx\ntFotnn32WYSFhTVaR6lUomPHjnjmmWfQvn17rFq1Cr179zbzt0XUPIIlHzqk0Wjw0ksvQavVoq6u\nDhEREXjrrbdQWFiIuLg43LlzB8HBwVi7di1cXZlX9Og6deoU4uPjcfToUVsXhagRizY3ubm5YcuW\nLcjIyEBGRgays7Px448/4qOPPsK8efPw/fffw8vLq9EYdiIish8W75No164dgPqritraWgiCAJVK\nhYiICAD1wwQPHjxo6WIQEVELWDwkdDodoqOjERoaitDQUPj7+6NDhw5QKOo/umvXrk2GFBI9aoYP\nH86mJrJLFg8JhUKhb2rKy8vDxYsXm6zT3LlziIjIOqw2BNbT0xPDhg3Djz/+iIqKCuh0OgBAcXEx\nunTpYnJ7TpBGRGR9Fh1SVFZWhjZt2sDLywv37t1DTk4OFixYgBEjRuDAgQOYOHEi0tPTmwwFNEQQ\nBKjVlZYsrk0plV6sn4Ny5roBrJ+jUyq9WrW9RUNCrVZj+fLl0Ol00Ol0mDhxIsaMGYOAgADExcUh\nOTkZ/fv3x/Tp0y1ZDCIiaiGL3idhbs6e9qyfY3LmugGsn6Nr7ZUEp+UgIiJJDAkiIpLEkCAiIkkM\nCSIiksSQICIiSQwJIiKSxJAgIiJJDAkiIpLEkCAiIkkMCSIiksSQICIiSQwJIiKSxJAgIiJJDAki\nIpLEkCAiIkkMCSIiksSQICIiSQwJIiKSxJAgIiJJDAkiIpLEkCAiIkkMCSIiksSQICIiSQwJIiKS\nxJAgIiJJDAkiIpLEkCAiIkkMCSIikuRqyZ0XFxcjPj4epaWlcHFxwYwZMzB79mxs2rQJ27ZtQ+fO\nnQEAS5cuxejRoy1ZFCIiagGLhoSLiwtWrFiB/v37o6qqClOnTsXIkSMBAPPmzcO8efMs+fFERNRK\nFg0JpVIJpVIJAPDw8EDfvn1x8+ZNAIAoipb8aCIiMgOr9UkUFhaioKAAAwcOBACkpaVhypQpWLly\nJSorK61VDCIiagarhERVVRUWLVqEhIQEeHh4YNasWTh06BAyMzPh4+ODpKQkaxSDiIiaSRAt3O5T\nW1uL119/HaNHj8bcuXObvF9UVIQ33ngDe/bssWQxiIioBSzaJwEACQkJCAwMbBQQarVa31dx8OBB\nBAUFydqXWu28zVJKpRfr56CcuW4A6+folEqvVm1v0ZA4ffo09uzZg6CgIERHR0MQBCxduhR79+5F\nfn4+FAoFunfvjlWrVlmyGERE1EIWDYkhQ4YgPz+/yXLeE0FE5Bh4xzUREUliSBARkSSGBBERSWJI\nEBGRJIYEERFJYkgQEZEkhgQREUliSBARkSSGBBERSWJIEBGRJIYEERFJYkgQEZEkhgQREUliSBAR\nkSSGBBERSWJIEBGRJIYEERFJYkgQEZEk2Y8vvXfvHtRqNdzd3dGlSxdLlomIiOyE0ZDQ6XTIyMjA\n9u3bUVBQAE9PT2g0Gri6uiI8PByvvPIK+vTpY62yEhGRlRkNiRdffBGDBw/GihUrEBwcDBcXFwDA\nrVu3cPz4cSQmJuKFF17ApEmTrFJYIiKyLkEURVHqzbKyMnh7exvdgZx1zEWtrrTK59iCUunF+jko\nZ64bwPo5OqXSq1XbG+24NnTyv3XrFs6ePWt0HSIicg6yRjfNmjULlZWVqKioQHR0NFauXIk1a9ZY\numxERGRjskKiuroaXl5eOHLkCCIjI7Fnzx6cOHHC0mUjIiIbkxUSGo0GAKBSqTBy5EgoFAp9JzYR\nETkvWSExfPhwREREIDc3F8OHD0dFRQUUCt6HR0Tk7GTdTPfuu++ioKAA/v7+cHNzw927d7F69WpL\nl42IiGzMaEhcuHBB/3ebNm1QXFysf+3m5ma5UhERkV0wGhILFiyAIAgQRRE3btyAp6cnAODu3bvo\n1q0bDh8+bHTnxcXFiI+PR2lpKVxcXBAbG4s5c+bgzp07WLp0KYqKitCjRw9s3LgRXl6tG8tLRETm\nZzQkGkJg9erVGDJkCCZMmAAAOHDgAM6dO2dy5y4uLlixYgX69++PqqoqTJ06FaGhodi1axdCQkLw\n2muvITU1FSkpKXjnnXfMUB0iIjInWb3PeXl5+oAAgOeffx4nT540uZ1SqUT//v0BAB4eHujbty9K\nSkqQlZWFmJgYAEBMTAwOHTrUkrITEZGFyQqJmpoa5Obm6l/n5uaipqamWR9UWFiIgoICDBo0CLdu\n3YKPjw+A+iC5fft2s/ZFRETWIXt0U1xcHNq1awcAuH//Pj7++GPZH1JVVYVFixYhISEBHh4eEASh\nRYVt7Rwk9o71c1zOXDeA9XuUyQqJoUOH4tChQ7h8+TJEUURAQIDs0U21tbVYtGgRpkyZgvDwcABA\n586dUVpaCh8fH6jVatnzPzn7JFysn2Ny5roBrJ+js+gEfw+qq6uDm5sbXF1dcfXq1UbDY41JSEhA\nYGAg5s6dq182btw47Nq1CwCQnp6OsLCwZhabiIisQdaVRFpaGj766CN06tRJ31QkCAKysrKMbnf6\n9Gns2bMHQUFBiI6OhiAIWLp0KV577TUsWbIEO3fuhJ+fH5KTk1tfEyIiMjtZIbF582bs3bsX3bt3\nb9bOhwwZgvz8fIPvffnll83aFxERWZ+s5ialUtnsgCAiIscn60pi5MiRWLt2LSZNmgR3d3f98sDA\nQIsVjIiIbE9WSGRkZACov9O6gZw+CSIicmyyQsLUHE1EROScZIUEUD8jrEqlAgA8/fTT6Nu3r8UK\nRURE9kFWx3VGRgZeeeUV5OfnIz8/H/PmzcPu3bstXTYiIrIx2UNg09PToVQqAQBqtRrz589HVFSU\nRQtHRES2JfuO64aAePhvIiJyXrJComfPnvjkk09QUlKCmzdvYtOmTfD397d02YiIyMZkhcTvf/97\nXL58GVFRUYiKisKlS5ewatUqS5eNiIhsTFafROfOnbFhwwZLl4WIiOyMrCuJ1NRUlJeX61/fvn0b\nn3/+ucUKRURE9kFWSOzbtw+dOnXSv37sscewd+9eixWKiIjsg6yQEEWxybK6ujqzF4aIiOyLrJDo\n3bs3/vznP0MUReh0OmzevBk9e/a0dNmIiMjGZIXEypUrceTIEQwcOBCDBw/GsWPHkJiYaOmyERGR\njcka3eTr64stW7aguroaANC+fXuLFoqIiOyD7D6J7du3449//CPat2+PwsJCnDlzxtJlIyIiG5MV\nEklJSTh58iQOHToEAPDw8MAHH3xg0YIREZHtyQoJlUqFjz76CG3btgVQPwT2/v37Fi0YERHZnqyQ\ncHd3hyAI+tc6nc5iBSIiIvshq+M6KCgIu3fvhiiKKCwsRGpqKoYMGWLpshERkY3JupJYvnw5Tp06\nBbVajdjYWNTV1WHZsmWWLhsREdmYrCsJT09PrF692tJlISIiOyPrSmL//v24e/cuACA5ORnz58/H\n//zP/1i0YEREZHuyQuJPf/oTPD09kZeXhxMnTiA6OppXFkREjwBZIeHqWt8q9fe//x2xsbGIjIzk\nEFgiokeArJAQBAG7d+/Gvn37EBISAgDQarUWLRgREdmerJD43e9+hwMHDiA2Nhb+/v64cuUKRowY\nYXK7hIQEjBw5EpGRkfplmzZtwujRoxETE4OYmBhkZ2e3vPRERGRRgmjoYRFmkpubCw8PD8THx2PP\nnj0A6kPCw8MD8+bNa/b+1OpKcxfRbiiVXqyfg3LmugGsn6NTKr1atb3RIbB/+ctfMHfuXKxdu9bg\n+/Hx8UZ3PnToUBQVFTVZbsFcIiIiMzIaEu7u7gDMPzV4WloaMjMz8atf/QrLly+Hl1frko6IiCzD\nos1NAFBUVIQ33nhD39xUVlaGxx57DIIgYMOGDVCr1ZxRlojIThm9kkhLSzO68UsvvdTsD/T29tb/\nPWPGDLzxxhuyt3X2dkPWzzE5c90A1s/RWbRPwhx3VT98oaJWq6FUKgEABw8eRFBQUKs/g4iILMNo\nSCQlJbVq52+//TZUKhXKy8sxduxYLFy4ECqVCvn5+VAoFOjevTtWrVrVqs8gIiLLMRoSx44dM7rx\nmDFjjL7/8ccfN1k2bdo0GcUiIiJ7YDQkPv/8c8n3BEEwGRJEROTYjIbEV199Za1yEBGRHTIaEteu\nXYO/vz8uXLhg8P3AwECLFIqIiOyD0ZBYvXo1UlJSsGDBgibvCYKArKwsixWMiIhsz2hIpKSkAAAO\nHz5slcIQEZF9kfX4UgCoqalBcXEx6urq9MvY3ERE5NxkhcSWLVuwYcMGdOzYEQpF/ezibG4iInJ+\nskLiL3/5Cw4cOABfX19Ll4eIiOyIrIcOde3alQFBRPQIknUlsXDhQqxcuRJjxozRTx8OmL7jmoiI\nHJuskDhy5AiOHDmCK1euNOqTYEgQETk3WSFx8OBBHD58GG3btrV0eYiIyI7I6pPw9/eHq6vs0bJE\nROQkZJ35e/Xqhblz5yI8PBxubm765S156BARETkOWSGh1WrRs2dPnD9/3tLlISIiOyIrJFr78CEi\nInJMRvskTD2+VKPR4OLFi2YtEBER2Q+TE/zV1NRg8uTJGDRoEHx8fHD//n1cvnwZx48fx7Fjx7B8\n+XL07dvXWuUlIiIrMhoSn376KfLy8vDtt9/iD3/4A4qLi9GuXTsEBQUhPDwcaWlp8PT0tFZZiYjI\nykz2SQwcOBADBw60RlmIiMjOyLpPgoiIHk0MCSIiksSQICIiSQwJIiKSJCskbt26hXfeeUc/DUdB\nQQG++eYbixaMiIhsT1ZI/O53v8OQIUNQUVEBAAgICMDXX39t0YIREZHtyQqJkpISvPjii3BxcQEA\nuLm56Z8rQUREzkvWmf7hacIrKiogiqJFCkRERPZDVkiMHz8eiYmJqKqqwq5du/Dqq69i2rRpJrdL\nSEjAyJEjERkZqV92584dvPrqq4iIiMD8+fNRWVnZ8tITEZFFyQqJX//61xg6dCiCg4Nx7NgxzJ49\nG3PnzjW53dSpU/HFF180WpaamoqQkBB8//33GDFiBFJSUlpWciIisjjZj5uLiopCVFRUs3Y+dOhQ\nFBUVNVqWlZWFrVu3AgBiYmIwe/ZsvPPOO83aLxERWYeskLh16xa2bt2Kq1evora2Vr88OTm52R9Y\nVlYGHx8fAIBSqcTt27ebvQ8iIrIOWSHx5ptvYsCAAQgJCdGPcLIFpdLLZp9tDayf43LmugGs36NM\nVkjU1NTg3XffNcsHdu7cGaWlpfDx8YFarYa3t7fsbdVq5+3kViq9WD8H5cx1A1g/R9faAJTVcT1o\n0CD8/PPPLfqAh4fKjhs3Drt27QIApKenIywsrEX7JSIiy5N1JfHCCy/g5ZdfRteuXeHu7q5fvmPH\nDqPbvf3221CpVCgvL8fYsWOxcOFCLFiwAIsXL8bOnTvh5+fXon4NIiKyDlkhsWzZMrzxxhsYMGBA\ns/okPv74Y4PLv/zyS9n7ICIi25EVEu7u7pg/f76ly0JERHZGVp/EM888g+zsbEuXhYiI7IysK4lt\n27YhNTUVHh4ecHNzgyiKEAQBOTk5li4fERHZkKyQ2Llzp6XLQUREdkhWSHTv3t3S5SAiIjtkNCSW\nLVuGdevWYdq0aRAEocn7pobAEhGRYzMaEg0zvf72t7+1SmGIiMi+GA2Jr7/+Gh988AGGDx9urfIQ\nEZEdMToENj8/31rlICIiO8QHVRMRkSSjzU3nz59HSEhIk+W8T4KI6NFgNCR69+6N1NRUa5WFiIjs\njNGQcHNz4z0SRESPMKN9Em3atLFWOYiIyA4ZDYlt27ZZqxxERGSHOLqJiIgkMSSIiEgSQ4KIiCQx\nJIiISBJDgoiIJDEkiIhIEkOCiIgkMSSIiEgSQ4KIiCTJesY1EdGjTnWuBPtyruB6aTX8fNpjUkhv\njBjga+tiWRxDgojIBNW5EqTs/pf+daG6Sv/a2YOCIUFEVuHIv8T35VyRWP6Lw9ShpRgSRGRxjv5L\n/HpptcHlN25VWbkk1mezkBg3bhw8PT2hUCjg6uqKHTt22KooRGRhjv5L3M+nPQrVTQOhW2cPG5TG\numwWEoIg4KuvvkLHjh1tVQQishJH/yU+KaR3oyuhfy/v1ei1IzepSbFZSIiiCJ1OZ6uPJ3I69nyC\nsuYvcUt8Dw3b78v5BTduVaFbZw9MCunVaL+O3qQmxaZXEvPnz4cgCJg5cyZmzJhhq6IQOTx7P0HJ\n/SXeWpb8HkYM8DW6D0dvUpNis5D461//CqVSibKyMsybNw8BAQEYOnSorYpDZHXm/MVr7ycoOb/E\nzcGW34OjN6lJsVlIKJVKAIC3tzeee+45/PTTTyZDQqn0skbRbIb1c1zNrVv2PwsN/uLt0KEtRj/Z\no9F627P+F1dLKtHT1wuxYf/R6P0G129Jn6DM8b2bYx+Tx3hh8pjAZm0jt/4NWvo9mKN+Pbt64cqN\niibL/X29HPrfvk1CoqamBjqdDh4eHqiursaJEyfw1ltvmdxOra60QulsQ6n0Yv0cVEvq9s33BRLL\nf0b/HvWDOR5uOrlyowLrtp5GRcW9Jr+K/TpLt/m39nu31bFrTv0btOR7MFf9Iob5G2xSixjmb9N/\n+60NKJuERGlpKd566y0IgoC6ujpERkZi1KhRtigKkU3IaZpoTtOJsTZ/e+7QNqYlTUfW6vswxFpN\natZmk5Dw9/dHZmamLT6ayCKaeyI2NdpHda7E4PuA4TZuqRMUALvu0DbGVJAa+85tdaI21bntiHjH\nNVErSfUvANInYlO//A2916Cjp5vB5YZOUIlfqAyuay8d2sYYC1JTo5jsvW6OhFOF0yNLda4EiV+o\n8Os1R5D4hQqqcyUt2s/2rP81uHxfzi+S24wY4IvXo4LRQ+kJF4WAHkpPvB4VjBEDfCWbWRqUVdyX\nXVZHHnEzKaS3xPJeRpuiyLx4JUGPJHOOp79aYrhT0tSJWOoXr9SJ/UFyrwQceToJY01Hn+05Z3Ab\nRwg/R8OQoEeSOcfT9/Q1PPSxpSdiqRP7gwrVd/HrNUdM9n/YsiPXHKSC1JHDz9GwuYkeSVK/1otK\n7za7CSo27D8MLm/piViqmeVhOlHUXwFJldNYs5YjM9YURebFKwl6JEn9EhVF6JfLbYIa/WQPVFTc\nM9uImobtth+5gLLK+7K2MXYF5IwdubYexfQoYUhQqznSOPyGshaVym+7bugMNVZHS5yI5QYE0LQt\n3pGOSUs5Y/jZI4YEtZjqXEmTX7v2NA7/4RNlv56PIet0YZP1BAF4zNNd8qRcVHrX6vcaSPWZtHFR\nQFvXdPbkB9vi7X2yP3IsDAlqEVNj+c05Dr8lv4oNnSilOoMf83JHWYX0r3ZXheETsyXvNZDqM6mV\nmF7/wbZ4e5/sjxwLQ4JaxNRYfnMNRZTzq9hQiJgq34OMBQQgfWK25HBLqT6T7j6e/3+fgHRbvCPf\nG0H2hyHhpCzdJm1qLL+5hiJuP3LB8PKjFzBigK9kiAiCWT4er0cFY1/OFbMPtzR1fIwNXTXVFs/h\noWRODAknZI02aVNj+Zs7FFHqpCnVT9Dw61/qikGqicgQby/D/RE9lJ7678uc9xrIOT6tGb3j6PdG\nkH1hSDgha7RJS52IvDu4I3ZsYLM+x9hJ09R2zW27DxvSAz9fLTc6CV6DhvfMPdxS7vFp6egdDg8l\nc2JIOCFrtEmb80Rk7KQp9SsfqD+xu0g0K8lpuzf0eVLrmnO4pbWOD0OBzIEh4YRMzZ7Zmr4KS/R1\nSJ00C9V3YaproU40vFxO2/2DrHlSZZ8BORKGhBOSagrq17NTq/oqpJqFUnf/C92VHvrASDt4Htln\ni6CtE+EiAO3atkH1/Vr4dTYcKsb6NyQyoIk2LgroRNEhmlbYZ0COhCHhhKSaglrTV6E6V4LN+wzP\nvCni34Fx4qcb+NflMv17dSJwt0YLQDqUpE6azaETRXwW/2yr9mEt7DMgR8KQcFKGmk9aOr2yqRvn\nHvRgQEgx1EELGO48lsvRmmrYZ0COgiHhhKT6DVraFt6cG9PkkHr8ptT9CA28vdwBwfDNb2yqIbIM\nThXuZBp+9Reqq5pMJd3S6ZXlPASnOaRCydQU2bHPBuKjN0OdcuprInvFKwknY6zfYdX84fq/m9MW\nLuchOA2C+3ibbHKSCqUH2+qLSu/CVaFAnU4Hv/8fzvrgjWYMBSLrYEg4GVNj8FtygpXqWDZ0Y9q/\nRzddh7ZOBxdBQLu2rqi5XysrlBgARPaFIWFhDz6/QCEIqNPVD+r09nJH7LPNuzNZDkuMwW/uaJyX\nngvCS88FNVqmVHpBrTb8LGgisl8MiYcYegbBz1dvm7x5zFBnMdB4xE6d+O9R/2WV9y0yx7+lxuDz\nFz7Ro4kh8QBTzyCQGucvdZOZt5e7yc809xz/HINPRObEkHiA3KGeD5/YpbaT8/hJS8zxz1/9RGQu\nHAL7ALlDPR8+sbdmiKij3QRGRI8WhsQD/Hzay1rv4RO71HZympt4ExgR2TObhUR2djaef/55RERE\nIDU1tUX7UJ0rQeIXKvx6zREkfqGC6lxJq8pk6mauf6/X66HXhreLfTZQf+OXQgBcFP+e09Tby503\ngRGR3bNJn4ROp8N7772HL7/8El26dMH06dMRFhaGvn37Sm4zZdnuRrOIWuLpa4Y6ffv17GTwXgBT\n2z188xcRkSOySUjk5eWhV69e6N69OwBg0qRJyMrKMhoSOp3YKAgs9fS11jwNjGFARM7GJs1NJSUl\n6Natm/61r68vbt68KXv7fTm/WOXpXkREjzqbhIQoyn2UjGE3blVJdhZztBARkfnYpLmpa9euuH79\nuv51SUkJunTpInt7f18vxIb9B9ZtPd3kvRcj+kGp9DJLOa3NUcstlzPXz5nrBrB+jzJBbO3P+hao\nq6vD888/jy+//BJKpRKxsbFYv3690T4JIiKyPptcSbi4uOC//uu/8Oqrr0IURUyfPp0BQURkh2xy\nJUFERI6Bd1wTEZEkhgQREUliSBARkSS7DwlzzPFkb8aNG4eoqChER0dj+vTpAIA7d+7g1VdfRURE\nBObPn4/KSsd5iltCQgJGjhyJyMhI/TJj9Vm9ejXGjx+PKVOmID8/3xZFbhZD9du0aRNGjx6NmJgY\nxMTEIDs7W/9eSkoKxo8fjwkTJuDEiRO2KLJsxcXFmDNnDiZOnIjIyEhs2bIFgPMcv4fr99VXXwFw\nnuOn0WiMkXxQAAAKiUlEQVQQGxuL6OhoREZGYtOmTQCAwsJCzJgxAxEREYiLi0Ntba1+/aVLl2L8\n+PGYOXNmo1sRJIl2rK6uTgwPDxcLCwtFjUYjRkVFiRcuXLB1sVpt3LhxYnl5eaNla9euFVNTU0VR\nFMWUlBRx3bp1tihai/zjH/8Qz507J06ePFm/TKo+R48eFV977TVRFEXx7NmzYmxsrPUL3EyG6vfp\np5+KmzdvbrLuhQsXxClTpoharVa8du2aGB4eLup0OmsWt1lu3rwpnjt3ThRFUbx79644fvx48cKF\nC05z/KTq5yzHTxRFsbq6WhRFUaytrRVjY2PFs2fPiosXLxb3798viqIoJiYmit98840oiqKYlpYm\nvvvuu6IoiuK+ffvEJUuWmNy/XV9JPDjHU5s2bfRzPDk6URSh0+kaLcvKykJMTAwAICYmBocOHbJF\n0Vpk6NCh6NChQ6NlD9en4bhlZWUhOjoaADBo0CBUVlaitLTUugVuJkP1AwzPHJCVlYWJEyfC1dUV\nPXr0QK9evZCXl2eNYraIUqlE//79AQAeHh7o27cvSkpKnOb4GapfwxRAznD8AKBdu3YA6q8Samtr\nIQgCVCoVIiIiADQ+nzx4XCMiIpCTk2Ny/3YdEq2d48leCYKA+fPnY9q0adi+fTsA4NatW/Dx8QFQ\n/w/79u3btixiq5WVlTWqT1lZGQDg5s2b6Nq1q349X19flJS0bop3W0lLS8OUKVOwcuVKfXOMoX+z\njlK/wsJCFBQUYNCgQU3+PTrD8Wuo38CBAwE4z/HT6XSIjo5GaGgoQkND4e/vjw4dOkChqD+9d+3a\nVV+HB4+fi4sLOnTogPLycqP7t+uQMJT0zuCvf/0rdu3ahc8++wxpaWnIzc2FIAimN3QCho6pI9Z9\n1qxZOHToEDIzM+Hj44MPP/wQgOPWr6qqCosWLUJCQgI8PDwky+ws9XOm46dQKJCRkYHs7Gzk5eXh\n4sWLTdZpqMPD9RNF0WT97DokWjvHk71SKpUAAG9vb4SHhyMvLw+dO3fWX7ar1Wp4e3vbsoitJlUf\nX19fFBcX69crLi52yGPq7e2t/881Y8YMfZNE165dcePGDf16jlC/2tpaLFq0CFOmTEF4eDgA5zp+\nhurnTMevgaenJ4YNG4Yff/wRFRUV+ibtB+vw4PGrq6vD3bt30bFjR6P7teuQeOKJJ3D16lUUFRVB\no9Fg3759CAsLs3WxWqWmpgZVVfXTmVdXV+PEiRMICgrCuHHjsGvXLgBAenq6w9Xz4V8oUvUJCwtD\nRkYGAODs2bPo0KGDvlnDnj1cP7Varf/74MGDCAoKAlBf7/3790Oj0eDatWu4evWqvnnDXiUkJCAw\nMBBz587VL3Om42eofs5y/MrKyvRNZffu3UNOTg4CAwMxYsQIHDhwAEDj4zdu3Dikp6cDAA4cOICn\nn37a5GfY/bQc2dnZeP/99/VzPC1YsMDWRWqVa9eu4a233oIgCKirq0NkZCQWLFiA8vJyLFmyBDdu\n3ICfnx+Sk5MNdpbao7fffhsqlQrl5eXw8fHBwoULER4ejsWLFxusz6pVq3D8+HG0a9cOSUlJCA4O\ntnENjDNUP5VKhfz8fCgUCnTv3h2rVq3SnyxTUlKwY8cOuLq6YuXKlRg1apSNayDt9OnTePnllxEU\nFARBECAIApYuXYqBAwdK/nt0pOMnVb+9e/c6xfH7+eefsXz5cuh0Ouh0OkycOBG/+c1vcO3aNcTF\nxaGiogL9+/fHunXr0KZNG2g0Gixbtgz5+fno1KkT1q9fjx49ehj9DLsPCSIish27bm4iIiLbYkgQ\nEZEkhgQREUliSBARkSSGBBERSWJIEBGRJIYE2b3a2lokJycjIiICkZGRmDRpEtasWYO6ujqj261Y\nsQJpaWkA6qeGXrt2rcnPOnToEH766SezlNsSioqKsG3bNlsXgx4hDAmye8uXL8fFixeRkZGBPXv2\nYPfu3QgICIBGozH7Z2VlZdn1rJ+FhYX49ttvW7StqVAlMsTV1gUgMuaXX35BVlaW/g5foH72ytjY\nWAD1M2CuW7dO/3CYUaNGIT4+3uikZefPn8fvf/971NTUQKPRYMaMGZgzZw5OnDiBw4cPIycnBzt2\n7MArr7yCwsJCHDx4EIIgQKPR4NKlS/jHP/4BT0/PRvv85z//iXXr1qGqqgqCICA+Ph4jR45EXl4e\nPvjgA9TU1KBdu3ZYuXIlnnjiCZw6dQpr1qzBzp07AaDR61OnTuGDDz7AwIEDcfbsWSgUCqxfvx4B\nAQF47733UFRUhJiYGPTs2RPJycm4dOkSkpKSUF5eDq1Wizlz5mDq1KkAgMcffxzLli3D0aNHMWzY\nMCxatMjsx4icnFmeekFkIfv37xejo6Ml3//666/FefPmibW1taJWqxXnzp2rf8DK8uXLxa1bt4qi\nWP+QoDVr1oiiKIpVVVWiRqPR/z1x4kTx4sWLTbZ52LJly8QPP/ywyfLy8nIxNDRUPHv2rCiKoqjT\n6cSKigpRo9GIY8eOFXNyckRRFMUffvhBHDt2rKjVakWVSiVOmzZNv48HX6tUKjE4OFjMz88XRVEU\n//SnP4nvvPNOk/VEsf5BMzExMeKlS5dEUax/sE5ERIT+db9+/cTPP/9c8vsjMoVXEmTXRBOzxuTk\n5CAmJgYuLi4AgKlTp+LQoUN44YUXJLepqanBu+++i4KCAigUCqjVahQUFCAgIEBym40bN6Kmpga/\n/e1vm7x39uxZBAYGYtCgQQDqp2X28vLC+fPn4ebmpp9ELSQkBG5ubrh8+bLJevfp0wePP/44gPqH\n+xw9etTgeleuXMGlS5cQFxen/660Wi0uXryIPn36AID+IUFELcGQILsWHByMK1euoLKyEl5eXk3e\nFw3Mh29qfvz169dDqVRi7dq1+gdAGevf2LlzJ06ePKl//rOhMshd3lBeFxeXRk8nvH//fqP13N3d\n9X+7uLjon1FsaH/e3t76mT0fJggC2rdvb/A9IjnYcU12rVevXhg3bhwSExP1U6zX1dVhy5YtqKmp\nwciRI5Geno7a2lpotVpkZGQgNDTU6D4rKyvRrVs3CIKA8+fPIzc3V/+eh4cH7t69q3/9ww8/4LPP\nPsMf//hHuLm5Gdzfk08+iQsXLuDHH38EUN9PUlFRgYCAAGi1Wpw6dQoAcPLkSdTW1qJ3797o0aMH\nCgsLUVlZCVEUsW/fPlnfh6enp35qaKD+iqNt27bIzMzUL7t06ZL+uzJ1JUZkCq8kyO6tWbMGn376\nKaZOnQo3NzeIoojRo0fDzc0NM2fOxNWrV/XP7X3mmWf0ndpSfvOb3yA+Ph67d+9Gz549MWzYMP17\nU6ZMwYoVK3DgwAG88sor2LlzJ2pqajB//nz9VUBaWlqjX+cdO3bEpk2bkJSUhOrqari4uCA+Ph4h\nISH45JNPsHr1an3H9aeffgpXV1f4+vpi3rx5iImJgb+/P5544glcuHDB5HfRr18/9OnTB5GRkQgI\nCEBycjL++7//G++//z42b96Muro6+Pj4YOPGjQDs/6lqZP84VTgREUlicxMREUliSBARkSSGBBER\nSWJIEBGRJIYEERFJYkgQEZEkhgQREUliSBARkaT/AzLfG+oMx+5pAAAAAElFTkSuQmCC\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7fc3b259add0\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "counts = []\n", - "times = []\n", - "for n in np.logspace(0, 7, 50):\n", - "\n", - " with tf.Graph().as_default():\n", - " tf_collatz = ag.to_graph(collatz)\n", - " count = tf_collatz(tf.constant(n, dtype=tf.float32))\n", - " with tf.Session() as sess:\n", - " count_value = sess.run(count)\n", - "\n", - " res = %timeit -n10 -r1 -o -q sess.run(count)\n", - " counts.append(count_value)\n", - " times.append(res.best)\n", - " \n", - "plot_results(counts, times, 'AutoGraph')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RRENYzLRF_f3" - }, - "source": [ - "# Eager" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 301 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 5003, - "status": "ok", - "timestamp": 1531757478713, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "dhDf8LLdF_f-", - "outputId": "3de0a5a5-7a11-4b41-8ab0-e4e21ce8d59b" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEcCAYAAAAydkhNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtYVWW+B/Dv2hshBdSQHaighhwas7Qeb6GFDjIyI3LZ\nGphdJLLMzqSlKaPQsTPm5KhZkZ7moKOnManGK17wsUfIS87QNj2jnEnIg5cQEtyAyDWBvdf5g4d9\nBPbaLGCvfeP7+QvW2mvt38tGvq71vut9BVEURRAREZmhsncBRETkuBgSREQkiSFBRESSGBJERCSJ\nIUFERJIYEkREJIkhQUREktzsXQCRvYWHh6OiogJqtRqiKEIQBMyePRtvv/22vUsjsjuGBBGA9PR0\nPPHEE3Z5b4PBALVabZf3JuoMbzcRATA38cCNGzeQmJiISZMmITQ0FMuXL0dtba1p//fffw+tVotx\n48bhjTfewNKlS5GWlmbaf+LECcTFxWHChAmYN28efvjhB9O+8PBwbNu2DTExMXj88cdhNBqVbSBR\nNzEkiCSIoohFixbhb3/7G44ePYqysjJs3rwZANDU1ITFixdjzpw5OHv2LGbNmoXjx4+bjv3++++R\nmpqKd999F2fPnsXcuXPx2muvoampyfSao0ePYtu2bTh37hxUKv5TJMfE30wiAL/97W8xceJETJgw\nARMnTsSePXswbNgwhIaGws3NDffffz8SExPx3XffAQAuXLgAg8GA559/Hmq1Gr/61a8wZswY0/n2\n7NmDZ555Bo8++igEQUBcXBzc3d1x8eJF02vmz58PPz8/uLu727y9RHKxT4IIwCeffNKhT6KyshJr\n167FuXPnUF9fD4PBgIEDBwIA9Ho9/Pz82rx+8ODBpq9/+uknHDx4ELt27QLQclXS3NyMW7dumV7j\n7++vVHOIrIYhQQTzfRKbNm2CIAg4cuQI+vfvj+zsbKxduxYAoNFoUFZW1ub1N2/exLBhwwC0BMCi\nRYvw6quvKl88kYJ4u4lIQl1dHTw9PeHl5YWysjJs377dtO+xxx6DWq1GRkYGDAYDsrOzkZeXZ9qf\nkJCAL7/80rStvr4ep06dQn19vc3bQdQTvJIgAvDaa69BpVKZnpOYMmUK3nzzTaxYsQLjx4/H8OHD\nERsbi08//RQA0KdPH2zevBmpqanYtGkTwsLCEB4ebupfeOSRR/Duu+9izZo1KCoqgoeHB8aNG4cJ\nEyYAAARBsFdTibpEUHLRocbGRjz33HNoamqCwWBAZGQkXn/9dRQXF2PZsmW4c+cORo8ejQ0bNsDN\njXlFzi0hIQHz5s2DVqu1dylEVqPo7SZ3d3fs3LkTmZmZyMzMxOnTp3Hx4kW8//77SEpKwldffQVv\nb2/s3btXyTKIFPHdd9+hvLwcBoMBBw4cwOXLl/HUU0/Zuywiq1K8T6Jv374AWq4qmpubIQgCdDod\nIiMjAQBarbbN+HIiZ3Ht2jXExsZi/Pjx+PTTT/Hxxx/D19fX3mURWZXi93iMRiNmz56NoqIiPPfc\ncwgMDET//v1NDw/5+/u3GRZI5CwSEhKQkJBg7zKIFKX4lYRKpTLdasrLy8OVK1c6vIadeEREjslm\nQ2C9vLwwYcIEXLx4EdXV1aa5akpLS/HAAw90eryC/etERCRB0dtNlZWV6NOnD7y9vfHzzz8jNzcX\nCxcuxKRJk3Ds2DHMnDkTBw4cwPTp0zs9lyAI0OtrlCzXrjQab7bPSbly2wC2z9lpNN49Ol7RkNDr\n9Vi5ciWMRiOMRiNmzpyJqVOnIigoCMuWLUNaWhpGjRqFp59+WskyiIiomxR9TsLaXD3t2T7n5Mpt\nA9g+Z9fTKwlOy0FERJIYEkREJIkhQUREkhgSREQkiSFBRESSGBJERCSJIUFERJIYEkREJIkhQURE\nkhgSREQkiSFBRESSGBJERCSJIUFERJIYEkREJIkhQUREkhgSREQkiSFBRESSGBJERCSJIUFERJIY\nEkREJIkhQUREkhgSREQkiSFBRESSGBJERCSJIUFERJIYEkREJIkhQUREkhgSREQkyU3Jk5eWliI5\nORnl5eVQq9VISEjACy+8gC1btmD37t0YNGgQAGDp0qUICwtTshQiIuoGRUNCrVZj1apVGDVqFOrq\n6jB79mxMnjwZAJCUlISkpCQl356IiHpI0ZDQaDTQaDQAAE9PT4wcORK3bt0CAIiiqORbExGRFdis\nT6K4uBgFBQUYM2YMACAjIwOxsbFITU1FTU2NrcogIqIusElI1NXVYcmSJUhJSYGnpyeeffZZZGdn\n4+DBg/D19cW6detsUQYREXWRICp836e5uRmvvvoqwsLCkJiY2GF/SUkJFi1ahMOHDytZBhERdYOi\nfRIAkJKSguDg4DYBodfrTX0Vx48fR0hIiKxz6fWue1tKo/Fm+5yUK7cNYPucnUbj3aPjFQ2J8+fP\n4/DhwwgJCUFcXBwEQcDSpUtx5MgR5OfnQ6VSYejQoVizZo2SZRARUTcpGhLjxo1Dfn5+h+18JoKI\nyDnwiWsiIpLEkCAiIkkMCSIiksSQICIiSQwJIiKSxJAgIiJJDAkiIpLEkCAiIkkMCSIiksSQICIi\nSQwJIiKSxJAgIiJJDAkiIpLEkCAiIkkMCSIiksSQICIiSQwJIiKSxJAgIiJJspcv/fnnn6HX6+Hh\n4YEHHnhAyZqIiMhBWAwJo9GIzMxM7NmzBwUFBfDy8kJjYyPc3NwQERGBF198EQ8++KCtaiUiIhuz\nGBLz5s3DY489hlWrVmH06NFQq9UAgIqKCnzzzTdYvXo1nnnmGURFRdmkWCIisi1BFEVRamdlZSV8\nfHwsnkDOa6xFr6+xyfvYg0bjzfY5KVduG8D2OTuNxrtHx1vsuDb3x7+iogIXLlyw+BoiInINskY3\nPfvss6ipqUF1dTXi4uKQmpqK9evXK10bERHZmayQqK+vh7e3N06cOIHo6GgcPnwYZ86cUbo2IiKy\nM1kh0djYCADQ6XSYPHkyVCqVqRObiIhcl6yQmDhxIiIjI3Hu3DlMnDgR1dXVUKn4HB4RkauT9TDd\nO++8g4KCAgQGBsLd3R21tbVYu3at0rUREZGdWQyJwsJC09d9+vRBaWmp6Xt3d3flqiIiIodgMSQW\nLlwIQRAgiiJu3rwJLy8vAEBtbS0GDx6Mr7/+2uLJS0tLkZycjPLycqjVasTHx2P+/Pm4c+cOli5d\nipKSEgQEBOCjjz6Ct3fPxvISEZH1WQyJ1hBYu3Ytxo0bh9/85jcAgGPHjuHSpUudnlytVmPVqlUY\nNWoU6urqMHv2bEyZMgX79+9HaGgoXnnlFWzduhXp6elYvny5FZpDRETWJKv3OS8vzxQQAPDrX/8a\n3377bafHaTQajBo1CgDg6emJkSNHoqysDDk5OdBqtQAArVaL7Ozs7tROREQKkxUSDQ0NOHfunOn7\nc+fOoaGhoUtvVFxcjIKCAowdOxYVFRXw9fUF0BIkt2/f7tK5iIjINmSPblq2bBn69u0LALh79y42\nbdok+03q6uqwZMkSpKSkwNPTE4IgdKvYns5B4ujYPuflym0D2D65Tv+jGHty/hdFZTUY5ueN+On/\ngrDHA6xybnuRFRLjx49HdnY2rl27BlEUERQUJHt0U3NzM5YsWYLY2FhEREQAAAYNGoTy8nL4+vpC\nr9fLnv/J1SfhYvuckyu3DWD75NJdKkP6oe9N31+/WY2Nu86juvpnTHrYr8fn7y5FJ/i7l8FggLu7\nO9zc3FBUVNRmeKwlKSkpCA4ORmJiomlbeHg49u/fDwA4cOAApk+f3sWyiYgcS1budYntP9q0DmuT\ndSWRkZGB999/HwMHDjTdKhIEATk5ORaPO3/+PA4fPoyQkBDExcVBEAQsXboUr7zyCt58803s27cP\nQ4YMQVpaWs9bQkRkRz+V15vdfrOizsaVWJeskNixYweOHDmCoUOHdunk48aNQ35+vtl9n376aZfO\nRUTkyIb49kOxvmMgDB7kaYdqrEfW7SaNRtPlgCAi6k2iQkdIbB9u20KsTNaVxOTJk7FhwwZERUXB\nw8PDtD04OFixwoiIHInuUhmycq/jp/J6DPHth6jQEW06pFu/zsr9ETcr6jB4kCeiQofbtdPaGmSF\nRGZmJoCWJ61byemTICJyBe1HLhXr60zftw8KZw+F9mSFRGdzNBERuTJLI5dcLRTakxUSQMuMsDqd\nDgDwxBNPYOTIkYoVRUTUXZ3dFuoOVx25JIesjuvMzEy8+OKLyM/PR35+PpKSknDo0CGlayMi6pLW\n20LF+joYRdF0W0h3qaxH5x3i28/sdmcfuSSH7CGwBw4cgEajAQDo9XosWLAAMTExihZHRNQVSt0W\nigod0aZP4v+3O/fIJTlk325qDYj2XxMROQqlbgu56sglOWSFxLBhw/Dxxx9j7ty5EAQBu3fvRmBg\noNK1ERF1iZIPtLniyCU5ZPVJ/P73v8e1a9cQExODmJgYXL16FWvWrFG6NiKiLnHVB9rsSdaVxKBB\ng/Dhhx8qXQsRUY/05ttCSpEVElu3bkVCQgIGDhwIALh9+zb27duHl19+WdHiiIi6qrfeFlKKrNtN\nWVlZpoAAgPvvvx9HjhxRrCgiInIMskJCFMUO2wwGg9WLISIixyIrJEaMGIH/+q//giiKMBqN2LFj\nB4YNG6Z0bUREZGeyQiI1NRUnTpzAmDFj8Nhjj+HUqVNYvXq10rUREZGdyeq49vPzw86dO1Ff3/Kg\nSr9+5h9RJyIi1yK7T2LPnj345JNP0K9fPxQXF+O///u/la6NiIjsTFZIrFu3Dt9++y2ys7MBAJ6e\nnnjvvfcULYyIiOxP1u0mnU6HzMxMaLVaAC1DYO/evatoYUREligxJTh1JCskPDw8IAiC6Xuj0ahY\nQUREnZG7Uhz1nKzbTSEhITh06BBEUURxcTH+/d//HePGjVO6NiIisyxNCU7WJSskVq5cibNnz0Kv\n1yM+Ph4GgwErVqxQujYiIrN680pxtibrdpOXlxfWrl2rdC1ERLIoOSU4tSXrSuLo0aOora0FAKSl\npWHBggX45z//qWhhRES6S2VYvV2Hl9efwOrtOtMypJwS3HZkhcSf/vQneHl5IS8vD2fOnEFcXByv\nLIhIUZbWq570sB9ejRmNAI0X1CoBARovvBozmp3WCpB1u8nNreVlf/vb3xAfH4/o6Gjs2LFD0cKI\nqHfrbL1qTgluG7KuJARBwKFDh5CVlYXQ0FAAQFNTk6KFEVHvxs5pxyArJN5++20cO3YM8fHxCAwM\nxPXr1zFp0qROj0tJScHkyZMRHR1t2rZlyxaEhYVBq9VCq9Xi9OnT3a+eiFzWEF/zc8Sxc9q2BNHc\nYhFWcu7cOXh6eiI5ORmHDx8G0BISnp6eSEpK6vL59Poaa5foMDQab7bPSbly2wD7ta/9A3OtrN33\n0Bs+v56w2Cfxl7/8BYmJidiwYYPZ/cnJyRZPPn78eJSUlHTYrmAuEZGD6uo0Glyv2jFYDAkPDw8A\n1p8aPCMjAwcPHsQjjzyClStXwtu7Z0lHRI6tu9NosHPa/hS93QQAJSUlWLRokel2U2VlJe6//34I\ngoAPP/wQer2eM8oSubjF75/A9ZvVHbaPGNwfm5f/0g4VkVwWryQyMjIsHvzcc891+Q19fHxMXyck\nJGDRokWyj3X1+4Zsn3Ny5bYB1mlfUan542+U1dj9Z9cbPr+esBgS1niquv2Fil6vh0ajAQAcP34c\nISEhPX4PInJsnEbDeVkMiXXr1vXo5G+99RZ0Oh2qqqowbdo0LF68GDqdDvn5+VCpVBg6dCjWrFnT\no/cgIscXFTrC7EglTqPh+CyGxKlTpywePHXqVIv7N23a1GHbnDlzZJRFRK6EI5Wcl8WQ+POf/yy5\nTxCETkOCiKgVRyo5J4sh8dlnn9mqDiIickAWQ+LGjRsIDAxEYWGh2f3BwcGKFEVERI7BYkisXbsW\n6enpWLhwYYd9giAgJydHscKIiMj+LIZEeno6AODrr7+2STFERORYZK0nAQANDQ0oLS2FwWAwbePt\nJiLH1NV5koikyAqJnTt34sMPP8SAAQOgUrXMLs7bTUSOqbvzJBGZIysk/vKXv+DYsWPw8+MvGJGj\n62xFN6KukLXokL+/PwOCyElIrehWrK/F6u066C6V2bgicmayriQWL16M1NRUTJ061TR9OND5E9dE\nJM3a/Qat5zNamNiZt56oq2SFxIkTJ3DixAlcv369TZ8EQ4Koe6zdbyC1ipsU3noiuWSFxPHjx/H1\n11/jvvvuU7oeol6hJ/0G5q5ApM4n5WZFxxlZicyRFRKBgYFwc5M9WpaIOiHVb9DZH2+pKxBB6Nr7\nc4pukkvWX/7hw4cjMTERERERcHd3N23vzqJDRNT5+gpS/RVSVwxuKhWaDMYO2328PVBZc7fDdk7R\nTXLJCommpiYMGzYMly9fVroeol7B0voKlvorpK5Amo0dAwIA4n/Z8sArp+im7pIVEj1dfIiI2rK0\nvsLq7Tqzx6Qf+h591AKMho77hvp6ISp0uGQYMBSouzpdvvSRRx6R3N/Y2IgbN25g5MiRVi+MyNVJ\nra8gdbUAAE0G88NbWwOBYUDW1ukEfw0NDZg1axbGjh0LX19f3L17F9euXcM333yDU6dOYeXKlQwJ\nIiuS6q+4Vx+1CkZR5O0jUpzFkNi8eTPy8vLw17/+Ff/xH/+B0tJS9O3bFyEhIYiIiEBGRga8vLxs\nVStRryDVX3EvoyhiW/IvbVQR9Wad9kmMGTMGY8aMsUUtRISW21B7ThSaHZXUikNYyVZkzd1ERLbV\nOipJCoewkq3wCTkiO+hs3qZJD/uhsOQOcs4Xdzh2+rgA9kGQzTAkiGxM7rxNz/0qBMFDB/AZB7Ir\nhgSRjXVl3iYOayV7k9UnUVFRgeXLl5um4SgoKMAXX3yhaGFErqq78zYR2YOskHj77bcxbtw4VFdX\nAwCCgoLw+eefK1oYkasa4tvP7HaOWCJHJCskysrKMG/ePKjVagCAu7u7aV0JIuqaqNAREts5Yokc\nj6w+ifbThFdXV0O0sPoVUW/TlVXmLM3bRORoZIXEjBkzsHr1atTV1WH//v34/PPPMWfOnE6PS0lJ\nwcmTJzFo0CAcPnwYAHDnzh0sXboUJSUlCAgIwEcffQRvb++etYLIjk7/o7jLq8yxQ5qchax7Ri+/\n/DLGjx+P0aNH49SpU3jhhReQmJjY6XGzZ8/G9u3b22zbunUrQkND8dVXX2HSpElIT0/vXuVEDmJP\nzv+a3Z6V+6ONKyGyPtlDYGNiYhATE9Olk48fPx4lJSVttuXk5GDXrl0AAK1WixdeeAHLly/v0nmJ\nHElRWY3Z7RytRK5AVkhUVFRg165dKCoqQnNzs2l7Wlpal9+wsrISvr6+AACNRoPbt293+RxE1tCV\nfgRLrx3m543rN6s7HMPRSuQKZIXEv/7rv+Lhhx9GaGioaYSTPWg0rt13wfbZjlQ/wtbD32O4f3/E\nT/8XhD0eYPG1/fvfh7DHAxA//V+wcdf5Du8xL/Ihh2pzT7hKO6S4evt6QlZINDQ04J133rHKGw4a\nNAjl5eXw9fWFXq+Hj4+P7GP1evOX9a5Ao/Fm+2zoi68KzG4XReD6zWps3HUe2w/+E/G/DJZ8QvqL\nr37AqIABCHs8ANXVP3cYrTQqYIBDtbm7HO2zs7be0L6ekBUSY8eOxQ8//ICHHnqoy2/QfqhseHg4\n9u/fj4ULF+LAgQOYPn16l89J1FOWVn9rVVlzF+mHvocgmN9/b58DRyuRq5IVEs888wyef/55+Pv7\nw8PDw7R97969Fo976623oNPpUFVVhWnTpmHx4sVYuHAh3njjDezbtw9DhgzpVr8GUU/JWf2tlZtK\nhSaDscN29jlQbyArJFasWIFFixbh4Ycf7lKfxKZNm8xu//TTT2Wfg8iaWjugS8rljzxqNnYMCIBP\nSFPvICskPDw8sGDBAqVrIeoWuaOU2k/RLddQXy9EhQ7nE9LUK8kKiaeeegqnT59GWFiY0vUQdYnc\ntRkA6Sm6AzQtIbDnZCEqqzsuGdoaCAwF6o1khcTu3buxdetWeHp6wt3dHaIoQhAE5ObmKl0fkUWW\n1mZo3d96hSF1i+lmRZ0pBFquSnjFQNRKVkjs27dP6TqIukVqlFJJeW2HKwwp93ZA84qBqC1ZITF0\n6FCl6yDqFqlRSlIjksxhBzSRNIshsWLFCmzcuBFz5syBYGaweGdDYImUcG9H9UAvd7OvkRqRJAgt\nHdG8nUQkj8WQaJ3p9Xe/+51NiiEyp30oVNb8f+dy69c+3h64U9do+sOflXvd7BXGUF8vrFkw0UaV\nEzk/iyHx+eef47333sPEifxHRfbRfvTSvQFxr3739cH7v53SZpu54a68tUTUNRZDIj8/31Z1EJm1\n50ShrNe1n5abq78RWYfs9SSIlNT+gbiHht2PH4puS145tGduigyOVCLqOYshcfnyZYSGhnbYzuck\nyJrMPRAnd16lVryNRKQMiyExYsQIbN261Va1UC8l9UCcHH3UKrwUNYpXDEQKsRgS7u7ufEaCFCdn\n2m4pDAgiZaks7ezTp4+t6qBebIhvP9mv7aNWQSW0zLf0asxoBgSRwixeSezevdtWdVAvFhU6Qtbs\nrAwFItvj6Cayu9Y//FKzsPr090D8tGAGBJEdMCTI6syt7zBrquV1djkLK5FjEsT2i1A7MFdfrNzZ\n2mcuDADzTzq3zJnkKbkgkDNzxs+uK9g+56bRWP4PWmd4JUHdIrXYj4+3h9nXi6LlBYGIyDFZHN1E\nJEXq2QY5T0i3LghERI6PIUHd0pNnG9rPs0REjou3m0iW9v0P/e5zQ21DU7fOZW6eJSJyTAwJ6pS5\n/oee4DxLRM6DIUGd6sncSq1UAjDE14tDWomcDEOCOtWd/of2K8XNmhrs0sMMiVwVQ4I6NcS3n+xb\nTHw6msi1MCSoU3LmVgrQ8FYSkStiSFCnTHMrnSg0+xwEJ94jcl12C4nw8HB4eXlBpVLBzc0Ne/fu\ntVcpJAPnViLqnewWEoIg4LPPPsOAAQPsVUKvZG6+pa78kee60US9i91CQhRFGI1Ge719ryQ13xLA\nuZSIyDy7XkksWLAAgiBg7ty5SEhIsFcpLkfqakHqeYes3B8ZEkRklt1C4ssvv4RGo0FlZSWSkpIQ\nFBSE8ePH26sclyF1tVBYckfyeQfOpUREUhxiPYktW7bA09MTSUlJ9i7F6S1+/wSu36w2u893YF+U\nVzV02D5icH9sXv5LpUsjIidklyuJhoYGGI1GeHp6or6+HmfOnMHrr7/e6XGu/MSutRY+KSqVPodU\nH1DkhEDFf7auvLCLK7cNYPucnVMuOlReXo7XX38dgiDAYDAgOjoaTz75pD1KcTmWno6+U9uIV2NG\ncwgrEclml5AIDAzEwYMH7fHWLs/S09GDB3lyCCsRdQkXHXIxkx72w/RxAWb3cYpuIuoqTsvhgp77\nVQiChw7gbSUi6jGGhIvibSUisgbebiIiIkkMCSIiksSQICIiSeyTsKGezsBKRGRrDAkbyTh+GTnn\ni03fcwZWInIGDAmF6S6VSa7oBnAGViJybAwJBbWfkdUczsBKRI6MIWEl5vobpNZvuNfgQZ5Kl0ZE\n1G0MCSuQWsNBEDo/llNlEJEj4xBYK5C6YnBTWf7xTh8XwP4IInJovJKwAqkV35ol1m/w8fZA/C+D\nGRBE5PAYElYgtYbDUF8vRIUO50R7ROS0GBKdkPMAnNQaDq2BwFAgImfFkLBAqkMaaPsAXOvXvGIg\nIlfDkGjn3isHtUS/s7kH4HjFQESuiCFxj/ZXDkaD+dfxATgi6i04BPYech5+A/gAHBH1HgyJe0gN\nZW2PD8ARUW/B2033kBrK2ketglEU2SFNRL0OQ+IeUkNZX4oaxWAgol6JIXEPDmUlImqLIdEOh7IS\nEf0/dlwTEZEkp76S4JrRRETKctqQkDtlBhERdZ/dbjedPn0av/71rxEZGYmtW7d2+XipB9+ycn/s\nWWFERGRil5AwGo149913sX37dhw5cgRZWVm4cuVKl84h9eAbp8wgIrIeu4REXl4ehg8fjqFDh6JP\nnz6IiopCTk6OxWNiVxzC6u066C6VAWh58M0cTplBRGQ9dgmJsrIyDB482PS9n58fbt26ZfEYo1E0\n9TvoLpUhKnSE2ddxygwiIuuxS0iIotij41un6n41ZjQCNF5QqwQEaLzwasxodloTEVmRXUY3+fv7\n46effjJ9X1ZWhgceeED28Tcr6qDReGPWVG/MmhqsRIl2odF427sERbly+1y5bQDb15vZJSQeffRR\nFBUVoaSkBBqNBllZWfjggw8sHnN4U6yNqiMiolZ2CQm1Wo1/+7d/w0svvQRRFPH0009j5MiR9iiF\niIgsEMSedhAQEZHL4txNREQkiSFBRESSGBJERCTJ4UOip3M8OaLw8HDExMQgLi4OTz/9NADgzp07\neOmllxAZGYkFCxagpqbGzlXKl5KSgsmTJyM6Otq0zVJ71q5dixkzZiA2Nhb5+fn2KLlLzLVvy5Yt\nCAsLg1arhVarxenTp0370tPTMWPGDPzmN7/BmTNn7FGybKWlpZg/fz5mzpyJ6Oho7Ny5E4DrfH7t\n2/fZZ58BcJ3Pr7GxEfHx8YiLi0N0dDS2bNkCACguLkZCQgIiIyOxbNkyNDc3m16/dOlSzJgxA3Pn\nzm3zKIIk0YEZDAYxIiJCLC4uFhsbG8WYmBixsLDQ3mX1WHh4uFhVVdVm24YNG8StW7eKoiiK6enp\n4saNG+1RWrd899134qVLl8RZs2aZtkm15+TJk+Irr7wiiqIoXrhwQYyPj7d9wV1krn2bN28Wd+zY\n0eG1hYWFYmxsrNjU1CTeuHFDjIiIEI1Goy3L7ZJbt26Jly5dEkVRFGtra8UZM2aIhYWFLvP5SbXP\nVT4/URTF+vp6URRFsbm5WYyPjxcvXLggvvHGG+LRo0dFURTF1atXi1988YUoiqKYkZEhvvPOO6Io\nimJWVpb45ptvdnp+h76S6M4cT85AFEUYjcY223JycqDVagEAWq0W2dnZ9iitW8aPH4/+/fu32da+\nPa2fW04mDIcbAAAJLUlEQVRODuLi4gAAY8eORU1NDcrLy21bcBeZax9gfuaAnJwczJw5E25ubggI\nCMDw4cORl5dnizK7RaPRYNSoUQAAT09PjBw5EmVlZS7z+ZlrX+sUQK7w+QFA3759AbRcJTQ3N0MQ\nBOh0OkRGRgJo+/fk3s81MjISubm5nZ7foUOiO3M8OQNBELBgwQLMmTMHe/bsAQBUVFTA19cXQMsv\n9u3bt+1ZYo9VVla2aU9lZSUA4NatW/D39ze9zs/PD2VlZXapsacyMjIQGxuL1NRU0+0Yc7+zztK+\n4uJiFBQUYOzYsR1+H13h82tt35gxYwC4zudnNBoRFxeHKVOmYMqUKQgMDET//v2hUrX8eff39ze1\n4d7PT61Wo3///qiqqrJ4focOCXNJ7wq+/PJL7N+/H9u2bUNGRgbOnTsHQRDsXZZNmPtMnbHtzz77\nLLKzs3Hw4EH4+vrij3/8IwDnbV9dXR2WLFmClJQUeHp6StbsKu1zpc9PpVIhMzMTp0+fRl5entll\nF1rb0L59oih22j6HDomezvHkqDQaDQDAx8cHERERyMvLw6BBg0yX7Xq9Hj4+PvYsscek2uPn54fS\n0lLT60pLS53yM/Xx8TH940pISDDdkvD398fNmzdNr3OG9jU3N2PJkiWIjY1FREQEANf6/My1z5U+\nv1ZeXl6YMGECLl68iOrqatMt7XvbcO/nZzAYUFtbiwEDBlg8r0OHxL1zPDU2NiIrKwvTp0+3d1k9\n0tDQgLq6loWR6uvrcebMGYSEhCA8PBz79+8HABw4cMDp2tn+fyhS7Zk+fToyMzMBABcuXED//v1N\ntzUcWfv26fV609fHjx9HSEgIgJZ2Hz16FI2Njbhx4waKiopMtzccVUpKCoKDg5GYmGja5kqfn7n2\nucrnV1lZabpV9vPPPyM3NxfBwcGYNGkSjh07BqDt5xceHo4DBw4AAI4dO4Ynnnii0/dw+Gk5Tp8+\njT/84Q+mOZ4WLlxo75J65MaNG3j99dchCAIMBgOio6OxcOFCVFVV4c0338TNmzcxZMgQpKWlme0s\ndURvvfUWdDodqqqq4Ovri8WLFyMiIgJvvPGG2fasWbMG33zzDfr27Yt169Zh9OjRdm6BZebap9Pp\nkJ+fD5VKhaFDh2LNmjWmP5bp6enYu3cv3NzckJqaiieffNLOLZB2/vx5PP/88wgJCYEgCBAEAUuX\nLsWYMWMkfx+d6fOTat+RI0dc4vP74YcfsHLlShiNRhiNRsycOROvvfYabty4gWXLlqG6uhqjRo3C\nxo0b0adPHzQ2NmLFihXIz8/HwIED8cEHHyAgIMDiezh8SBARkf049O0mIiKyL4YEERFJYkgQEZEk\nhgQREUliSBARkSSGBBERSWJIkMNrbm5GWloaIiMjER0djaioKKxfvx4Gg8HicatWrUJGRgaAlqmh\nN2zY0Ol7ZWdn43/+53+sUrcSSkpKsHv3bnuXQb0IQ4Ic3sqVK3HlyhVkZmbi8OHDOHToEIKCgtDY\n2Gj198rJyXHoWT+Li4vx17/+tVvHdhaqROa42bsAIkt+/PFH5OTkmJ7wBVpmr4yPjwfQMgPmxo0b\nTYvDPPnkk0hOTrY4adnly5fx+9//Hg0NDWhsbERCQgLmz5+PM2fO4Ouvv0Zubi727t2LF198EcXF\nxTh+/DgEQUBjYyOuXr2K7777Dl5eXm3O+Y9//AMbN25EXV0dBEFAcnIyJk+ejLy8PLz33ntoaGhA\n3759kZqaikcffRRnz57F+vXrsW/fPgBo8/3Zs2fx3nvvYcyYMbhw4QJUKhU++OADBAUF4d1330VJ\nSQm0Wi2GDRuGtLQ0XL16FevWrUNVVRWampowf/58zJ49GwDwi1/8AitWrMDJkycxYcIELFmyxOqf\nEbk4q6x6QaSQo0ePinFxcZL7P//8czEpKUlsbm4Wm5qaxMTERNMCKytXrhR37dolimLLIkHr168X\nRVEU6+rqxMbGRtPXM2fOFK9cudLhmPZWrFgh/vGPf+ywvaqqSpwyZYp44cIFURRF0Wg0itXV1WJj\nY6M4bdo0MTc3VxRFUfz73/8uTps2TWxqahJ1Op04Z84c0znu/V6n04mjR48W8/PzRVEUxT/96U/i\n8uXLO7xOFFsWmtFqteLVq1dFUWxZWCcyMtL0/UMPPST++c9/lvz5EXWGVxLk0MROZo3Jzc2FVquF\nWq0GAMyePRvZ2dl45plnJI9paGjAO++8g4KCAqhUKuj1ehQUFCAoKEjymI8++ggNDQ343e9+12Hf\nhQsXEBwcjLFjxwJomZbZ29sbly9fhru7u2kStdDQULi7u+PatWudtvvBBx/EL37xCwAti/ucPHnS\n7OuuX7+Oq1evYtmyZaafVVNTE65cuYIHH3wQAEyLBBF1B0OCHNro0aNx/fp11NTUwNvbu8N+0cx8\n+J3Nj//BBx9Ao9Fgw4YNpgWgLPVv7Nu3D99++61p/WdzNcjd3lqvWq1uszrh3bt327zOw8PD9LVa\nrTatUWzufD4+PqaZPdsTBAH9+vUzu49IDnZck0MbPnw4wsPDsXr1atMU6waDATt37kRDQwMmT56M\nAwcOoLm5GU1NTcjMzMSUKVMsnrOmpgaDBw+GIAi4fPkyzp07Z9rn6emJ2tpa0/d///vfsW3bNnzy\nySdwd3c3e77HH38chYWFuHjxIoCWfpLq6moEBQWhqakJZ8+eBQB8++23aG5uxogRIxAQEIDi4mLU\n1NRAFEVkZWXJ+nl4eXmZpoYGWq447rvvPhw8eNC07erVq6afVWdXYkSd4ZUEObz169dj8+bNmD17\nNtzd3SGKIsLCwuDu7o65c+eiqKjItG7vU089ZerUlvLaa68hOTkZhw4dwrBhwzBhwgTTvtjYWKxa\ntQrHjh3Diy++iH379qGhoQELFiwwXQVkZGS0+d/5gAEDsGXLFqxbtw719fVQq9VITk5GaGgoPv74\nY6xdu9bUcb1582a4ubnBz88PSUlJ0Gq1CAwMxKOPPorCwsJOfxYPPfQQHnzwQURHRyMoKAhpaWn4\nz//8T/zhD3/Ajh07YDAY4Ovri48++giA46+qRo6PU4UTEZEk3m4iIiJJDAkiIpLEkCAiIkkMCSIi\nksSQICIiSQwJIiKSxJAgIiJJDAkiIpL0f3zF2/hGE4QYAAAAAElFTkSuQmCC\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7fc3af690a50\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "with context.eager_mode():\n", - "\n", - " counts = []\n", - " times = [] \n", - " for n in np.logspace(0, 7, 50):\n", - "\n", - " n_tensor = tf.constant(n, dtype=tf.float32)\n", - " count = collatz(n_tensor)\n", - "\n", - " res = %timeit -n10 -r1 -o -q collatz(n_tensor)\n", - " times.append(res.best)\n", - " counts.append(count)\n", - " \n", - "plot_results(counts, times, 'Eager')\n" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [ - "x5ChBlH09jk_", - "_cRFTcwT9mnn" - ], - "default_view": {}, - "last_runtime": { - "build_target": "", - "kind": "local" - }, - "name": "Autograph vs. Eager Collatz speed test", - "provenance": [ - { - "file_id": "0B8bm7KvwJklpMUQtbnVpYkdJUjRtOTRyWVVfSEhpRl9HYm5n", - "timestamp": 1531512047714 - } - ], - "version": "0.3.2", - "views": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_mnist_speed_test.ipynb b/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_mnist_speed_test.ipynb deleted file mode 100644 index 952ec091fb1..00000000000 --- a/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_mnist_speed_test.ipynb +++ /dev/null @@ -1,652 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "etTmZVFN8fYO" - }, - "source": [ - "This notebook runs a basic speed test for a short training loop of a neural network training on the MNIST dataset." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eqOvRhOz8SWs" - }, - "source": [ - "### Imports" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "nHY0tntRizGb" - }, - "outputs": [], - "source": [ - "!pip install -U -q tf-nightly" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "Pa2qpEmoVOGe" - }, - "outputs": [], - "source": [ - "import gzip\n", - "import os\n", - "import shutil\n", - "import time\n", - "\n", - "import numpy as np\n", - "import six\n", - "from six.moves import urllib\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.contrib import autograph as ag\n", - "from tensorflow.contrib.eager.python import tfe\n", - "from tensorflow.python.eager import context\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PZWxEJFM9A7b" - }, - "source": [ - "### Testing boilerplate" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "kfZk9EFZ5TeQ" - }, - "outputs": [], - "source": [ - "# Test-only parameters. Test checks successful completion not correctness. \n", - "burn_ins = 1\n", - "trials = 1\n", - "max_steps = 2\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k0GKbZBJ9Gt9" - }, - "source": [ - "### Speed test configuration" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "gWXV8WHn43iZ" - }, - "outputs": [], - "source": [ - "#@test {\"skip\": true} \n", - "burn_ins = 3\n", - "trials = 10\n", - "max_steps = 500\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kZV_3pGy8033" - }, - "source": [ - "### Data source setup" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "YfnHJbBOBKae" - }, - "outputs": [], - "source": [ - "def download(directory, filename):\n", - " filepath = os.path.join(directory, filename)\n", - " if tf.gfile.Exists(filepath):\n", - " return filepath\n", - " if not tf.gfile.Exists(directory):\n", - " tf.gfile.MakeDirs(directory)\n", - " url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'\n", - " zipped_filepath = filepath + '.gz'\n", - " print('Downloading %s to %s' % (url, zipped_filepath))\n", - " urllib.request.urlretrieve(url, zipped_filepath)\n", - " with gzip.open(zipped_filepath, 'rb') as f_in, open(filepath, 'wb') as f_out:\n", - " shutil.copyfileobj(f_in, f_out)\n", - " os.remove(zipped_filepath)\n", - " return filepath\n", - "\n", - "\n", - "def dataset(directory, images_file, labels_file):\n", - " images_file = download(directory, images_file)\n", - " labels_file = download(directory, labels_file)\n", - "\n", - " def decode_image(image):\n", - " # Normalize from [0, 255] to [0.0, 1.0]\n", - " image = tf.decode_raw(image, tf.uint8)\n", - " image = tf.cast(image, tf.float32)\n", - " image = tf.reshape(image, [784])\n", - " return image / 255.0\n", - "\n", - " def decode_label(label):\n", - " label = tf.decode_raw(label, tf.uint8)\n", - " label = tf.reshape(label, [])\n", - " return tf.to_int32(label)\n", - "\n", - " images = tf.data.FixedLengthRecordDataset(\n", - " images_file, 28 * 28, header_bytes=16).map(decode_image)\n", - " labels = tf.data.FixedLengthRecordDataset(\n", - " labels_file, 1, header_bytes=8).map(decode_label)\n", - " return tf.data.Dataset.zip((images, labels))\n", - "\n", - "\n", - "def mnist_train(directory):\n", - " return dataset(directory, 'train-images-idx3-ubyte',\n", - " 'train-labels-idx1-ubyte')\n", - "\n", - "def mnist_test(directory):\n", - " return dataset(directory, 't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte')\n", - "\n", - "def setup_mnist_data(is_training, hp, batch_size):\n", - " if is_training:\n", - " ds = mnist_train('/tmp/autograph_mnist_data')\n", - " ds = ds.cache()\n", - " ds = ds.shuffle(batch_size * 10)\n", - " else:\n", - " ds = mnist_test('/tmp/autograph_mnist_data')\n", - " ds = ds.cache()\n", - " ds = ds.repeat()\n", - " ds = ds.batch(batch_size)\n", - " return ds\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qzkZyZcS9THu" - }, - "source": [ - "### Keras model definition" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "x_MU13boiok2" - }, - "outputs": [], - "source": [ - "def mlp_model(input_shape):\n", - " model = tf.keras.Sequential((\n", - " tf.keras.layers.Dense(100, activation='relu', input_shape=input_shape),\n", - " tf.keras.layers.Dense(100, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')))\n", - " model.build()\n", - " return model\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DXt4GoTxtvn2" - }, - "source": [ - "# AutoGraph" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "W51sfbONiz_5" - }, - "outputs": [], - "source": [ - "def predict(m, x, y):\n", - " y_p = m(x)\n", - " losses = tf.keras.losses.categorical_crossentropy(y, y_p)\n", - " l = tf.reduce_mean(losses)\n", - " accuracies = tf.keras.metrics.categorical_accuracy(y, y_p)\n", - " accuracy = tf.reduce_mean(accuracies)\n", - " return l, accuracy\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "CsAD0ajbi9iZ" - }, - "outputs": [], - "source": [ - "def fit(m, x, y, opt):\n", - " l, accuracy = predict(m, x, y)\n", - " opt.minimize(l)\n", - " return l, accuracy\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "RVw57HdTjPzi" - }, - "outputs": [], - "source": [ - "def get_next_batch(ds):\n", - " itr = ds.make_one_shot_iterator()\n", - " image, label = itr.get_next()\n", - " x = tf.to_float(tf.reshape(image, (-1, 28 * 28)))\n", - " y = tf.one_hot(tf.squeeze(label), 10)\n", - " return x, y\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "UUI0566FjZPx" - }, - "outputs": [], - "source": [ - "def train(train_ds, test_ds, hp):\n", - " m = mlp_model((28 * 28,))\n", - " opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n", - "\n", - " train_losses = []\n", - " test_losses = []\n", - " train_accuracies = []\n", - " test_accuracies = []\n", - " ag.set_element_type(train_losses, tf.float32)\n", - " ag.set_element_type(test_losses, tf.float32)\n", - " ag.set_element_type(train_accuracies, tf.float32)\n", - " ag.set_element_type(test_accuracies, tf.float32)\n", - "\n", - " i = tf.constant(0)\n", - " while i \u003c hp.max_steps:\n", - " train_x, train_y = get_next_batch(train_ds)\n", - " test_x, test_y = get_next_batch(test_ds)\n", - " step_train_loss, step_train_accuracy = fit(m, train_x, train_y, opt)\n", - " step_test_loss, step_test_accuracy = predict(m, test_x, test_y)\n", - "\n", - " train_losses.append(step_train_loss)\n", - " test_losses.append(step_test_loss)\n", - " train_accuracies.append(step_train_accuracy)\n", - " test_accuracies.append(step_test_accuracy)\n", - "\n", - " i += 1\n", - " return (ag.stack(train_losses), ag.stack(test_losses),\n", - " ag.stack(train_accuracies), ag.stack(test_accuracies))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 215 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 12156, - "status": "ok", - "timestamp": 1531752050611, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "K1m8TwOKjdNd", - "outputId": "bd5746f2-bf91-44aa-9eff-38eb11ced33f" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "('Duration:', 0.6226680278778076)\n", - "('Duration:', 0.6082069873809814)\n", - "('Duration:', 0.6223258972167969)\n", - "('Duration:', 0.6176440715789795)\n", - "('Duration:', 0.6309840679168701)\n", - "('Duration:', 0.6180410385131836)\n", - "('Duration:', 0.6219630241394043)\n", - "('Duration:', 0.6183009147644043)\n", - "('Duration:', 0.6176400184631348)\n", - "('Duration:', 0.6476900577545166)\n", - "('Mean duration:', 0.62254641056060789, '+/-', 0.0099792188690656976)\n" - ] - } - ], - "source": [ - "#@test {\"timeout\": 90}\n", - "with tf.Graph().as_default():\n", - " hp = tf.contrib.training.HParams(\n", - " learning_rate=0.05,\n", - " max_steps=max_steps,\n", - " )\n", - " train_ds = setup_mnist_data(True, hp, 500)\n", - " test_ds = setup_mnist_data(False, hp, 100)\n", - " tf_train = ag.to_graph(train)\n", - " losses = tf_train(train_ds, test_ds, hp)\n", - "\n", - " with tf.Session() as sess:\n", - " durations = []\n", - " for t in range(burn_ins + trials):\n", - " sess.run(tf.global_variables_initializer())\n", - "\n", - " start = time.time()\n", - " (train_losses, test_losses, train_accuracies,\n", - " test_accuracies) = sess.run(losses)\n", - "\n", - " if t \u003c burn_ins:\n", - " continue\n", - "\n", - " duration = time.time() - start\n", - " durations.append(duration)\n", - " print('Duration:', duration)\n", - "\n", - " print('Mean duration:', np.mean(durations), '+/-', np.std(durations))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "A06kdgtZtlce" - }, - "source": [ - "# Eager" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "hBKOKGrWty4e" - }, - "outputs": [], - "source": [ - "def predict(m, x, y):\n", - " y_p = m(x)\n", - " losses = tf.keras.losses.categorical_crossentropy(tf.cast(y, tf.float32), y_p)\n", - " l = tf.reduce_mean(losses)\n", - " accuracies = tf.keras.metrics.categorical_accuracy(y, y_p)\n", - " accuracy = tf.reduce_mean(accuracies)\n", - " return l, accuracy\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "HCgTZ0MTt6vt" - }, - "outputs": [], - "source": [ - "def train(ds, hp):\n", - " m = mlp_model((28 * 28,))\n", - " opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n", - "\n", - " train_losses = []\n", - " test_losses = []\n", - " train_accuracies = []\n", - " test_accuracies = []\n", - "\n", - " i = 0\n", - " train_test_itr = tfe.Iterator(ds)\n", - " for (train_x, train_y), (test_x, test_y) in train_test_itr:\n", - " train_x = tf.to_float(tf.reshape(train_x, (-1, 28 * 28)))\n", - " train_y = tf.one_hot(tf.squeeze(train_y), 10)\n", - " test_x = tf.to_float(tf.reshape(test_x, (-1, 28 * 28)))\n", - " test_y = tf.one_hot(tf.squeeze(test_y), 10)\n", - "\n", - " if i \u003e hp.max_steps:\n", - " break\n", - "\n", - " with tf.GradientTape() as tape:\n", - " step_train_loss, step_train_accuracy = predict(m, train_x, train_y)\n", - " grad = tape.gradient(step_train_loss, m.variables)\n", - " opt.apply_gradients(zip(grad, m.variables))\n", - " step_test_loss, step_test_accuracy = predict(m, test_x, test_y)\n", - "\n", - " train_losses.append(step_train_loss)\n", - " test_losses.append(step_test_loss)\n", - " train_accuracies.append(step_train_accuracy)\n", - " test_accuracies.append(step_test_accuracy)\n", - "\n", - " i += 1\n", - " return train_losses, test_losses, train_accuracies, test_accuracies\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 215 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 52499, - "status": "ok", - "timestamp": 1531752103279, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "plv_yrn_t8Dy", - "outputId": "55d5ab3d-252d-48ba-8fb4-20ec3c3e6d00" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "('Duration:', 3.9973549842834473)\n", - "('Duration:', 4.018772125244141)\n", - "('Duration:', 3.9740989208221436)\n", - "('Duration:', 3.9922947883605957)\n", - "('Duration:', 3.9795801639556885)\n", - "('Duration:', 3.966722011566162)\n", - "('Duration:', 3.986541986465454)\n", - "('Duration:', 3.992305040359497)\n", - "('Duration:', 4.012261867523193)\n", - "('Duration:', 4.004716157913208)\n", - "('Mean duration:', 3.9924648046493529, '+/-', 0.015681688635624851)\n" - ] - } - ], - "source": [ - "#@test {\"timeout\": 90}\n", - "with context.eager_mode():\n", - " durations = []\n", - " for t in range(burn_ins + trials):\n", - " hp = tf.contrib.training.HParams(\n", - " learning_rate=0.05,\n", - " max_steps=max_steps,\n", - " )\n", - " train_ds = setup_mnist_data(True, hp, 500)\n", - " test_ds = setup_mnist_data(False, hp, 100)\n", - " ds = tf.data.Dataset.zip((train_ds, test_ds))\n", - " start = time.time()\n", - " (train_losses, test_losses, train_accuracies,\n", - " test_accuracies) = train(ds, hp)\n", - " \n", - " train_losses[-1].numpy()\n", - " test_losses[-1].numpy()\n", - " train_accuracies[-1].numpy()\n", - " test_accuracies[-1].numpy()\n", - "\n", - " if t \u003c burn_ins:\n", - " continue\n", - "\n", - " duration = time.time() - start\n", - " durations.append(duration)\n", - " print('Duration:', duration)\n", - "\n", - " print('Mean duration:', np.mean(durations), '+/-', np.std(durations))\n" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [ - "eqOvRhOz8SWs", - "PZWxEJFM9A7b", - "kZV_3pGy8033" - ], - "default_view": {}, - "name": "Autograph vs. Eager MNIST speed test", - "provenance": [ - { - "file_id": "1tAQW5tHUgAc8M4-iwwJm6Xs6dV9nEqtD", - "timestamp": 1530297010607 - }, - { - "file_id": "18dCjshrmHiPTIe1CNsL8tnpdGkuXgpM9", - "timestamp": 1530289467317 - }, - { - "file_id": "1DcfimonWU11tmyivKBGVrbpAl3BIOaRG", - "timestamp": 1522272821237 - }, - { - "file_id": "1wCZUh73zTNs1jzzYjqoxMIdaBWCdKJ2K", - "timestamp": 1522238054357 - }, - { - "file_id": "1_HpC-RrmIv4lNaqeoslUeWaX8zH5IXaJ", - "timestamp": 1521743157199 - }, - { - "file_id": "1mjO2fQ2F9hxpAzw2mnrrUkcgfb7xSGW-", - "timestamp": 1520522344607 - } - ], - "version": "0.3.2", - "views": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/autograph/examples/notebooks/algorithms.ipynb b/tensorflow/contrib/autograph/examples/notebooks/algorithms.ipynb deleted file mode 100644 index c51d2124920..00000000000 --- a/tensorflow/contrib/autograph/examples/notebooks/algorithms.ipynb +++ /dev/null @@ -1,935 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "b9R-4ezU3NH0" - }, - "source": [ - "## AutoGraph: examples of simple algorithms\n", - "\n", - "This notebook shows how you can use AutoGraph to compile simple algorithms and run them in TensorFlow.\n", - "\n", - "It requires the nightly build of TensorFlow, which is installed below." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "TuWj26KWz1fZ" - }, - "outputs": [], - "source": [ - "!pip install -U -q tf-nightly-2.0-preview" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Cp7iTarmz62Y" - }, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "\n", - "tf = tf.compat.v2\n", - "tf.enable_v2_behavior()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3kudk1elq0Gh" - }, - "source": [ - "### Fibonacci numbers\n", - "\n", - "https://en.wikipedia.org/wiki/Fibonacci_number\n" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "colab": { - "height": 187 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 709, - "status": "ok", - "timestamp": 1563825398552, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "H7olFlMXqrHe", - "outputId": "25243e7b-99a7-4a6d-ad00-e97c52be7d97" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 : 1\n", - "1 : 2\n", - "2 : 3\n", - "3 : 5\n", - "4 : 8\n", - "5 : 13\n", - "6 : 21\n", - "7 : 34\n", - "8 : 55\n", - "9 : 89\n" - ] - } - ], - "source": [ - "@tf.function\n", - "def fib(n):\n", - " f1 = 0\n", - " f2 = 1\n", - " for i in tf.range(n):\n", - " tmp = f2\n", - " f2 = f2 + f1\n", - " f1 = tmp\n", - " tf.print(i, ': ', f2)\n", - " return f2\n", - "\n", - "\n", - "_ = fib(tf.constant(10))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "p8zZyj-tq4K3" - }, - "source": [ - "#### Generated code" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "UeWjK8rHq6Cj" - }, - "outputs": [], - "source": [ - "print(tf.autograph.to_code(fib.python_function))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eIfVy6ZTrFEH" - }, - "source": [ - "### Fizz Buzz\n", - "\n", - "https://en.wikipedia.org/wiki/Fizz_buzz" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "colab": { - "height": 119 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 663, - "status": "ok", - "timestamp": 1563825401385, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "33CAheYsrEQ7", - "outputId": "2a88b65d-4fed-4d96-8770-0c68ffece861" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Buzz\n", - "11\n", - "Fizz\n", - "13\n", - "14\n", - "FizzBuzz\n" - ] - } - ], - "source": [ - "import tensorflow as tf\n", - "\n", - "\n", - "@tf.function(experimental_autograph_options=tf.autograph.experimental.Feature.EQUALITY_OPERATORS)\n", - "def fizzbuzz(i, n):\n", - " while i \u003c n:\n", - " msg = ''\n", - " if i % 3 == 0:\n", - " msg += 'Fizz'\n", - " if i % 5 == 0:\n", - " msg += 'Buzz'\n", - " if msg == '':\n", - " msg = tf.as_string(i)\n", - " tf.print(msg)\n", - " i += 1\n", - " return i\n", - "\n", - "_ = fizzbuzz(tf.constant(10), tf.constant(16))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Lkq3DBGOv3fA" - }, - "source": [ - "#### Generated code" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bBhFIIaZrxvx" - }, - "outputs": [], - "source": [ - "print(tf.autograph.to_code(fizzbuzz.python_function))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BNRtprSvwJgk" - }, - "source": [ - "### Conway's Game of Life\n", - "\n", - "https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r8_0ioEuAI-a" - }, - "source": [ - "#### Testing boilerplate" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7moIlf8VABkl" - }, - "outputs": [], - "source": [ - "NUM_STEPS = 1" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QlEvfIQPAYF5" - }, - "source": [ - "#### Game of Life for AutoGraph\n", - "\n", - "Note: the code may take a while to run." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5pCK2qQSAAK4" - }, - "outputs": [], - "source": [ - "#@test {\"skip\": true} \n", - "NUM_STEPS = 75" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GPZANPdhMagD" - }, - "source": [ - "Note: This code uses a non-vectorized algorithm, which is quite slow. For 75 steps, it will take a few minutes to run. " - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "colab": { - "height": 309 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 147654, - "status": "ok", - "timestamp": 1563825336196, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "hC3qMqryPDHS", - "outputId": "56a095a3-28a3-455d-e95e-2c4c9dcd97d2" - }, - "outputs": [ - { - "data": { - "text/html": [ - "\u003cvideo width=\"432\" height=\"288\" controls autoplay loop\u003e\n", - " \u003csource type=\"video/mp4\" src=\"data:video/mp4;base64,AAAAHGZ0eXBNNFYgAAACAGlzb21pc28yYXZjMQAAAAhmcmVlAABdAG1kYXQAAAKuBgX//6rcRem9\n", - "5tlIt5Ys2CDZI+7veDI2NCAtIGNvcmUgMTUyIHIyODU0IGU5YTU5MDMgLSBILjI2NC9NUEVHLTQg\n", - "QVZDIGNvZGVjIC0gQ29weWxlZnQgMjAwMy0yMDE3IC0gaHR0cDovL3d3dy52aWRlb2xhbi5vcmcv\n", - "eDI2NC5odG1sIC0gb3B0aW9uczogY2FiYWM9MSByZWY9MyBkZWJsb2NrPTE6MDowIGFuYWx5c2U9\n", - "MHgzOjB4MTEzIG1lPWhleCBzdWJtZT03IHBzeT0xIHBzeV9yZD0xLjAwOjAuMDAgbWl4ZWRfcmVm\n", - "PTEgbWVfcmFuZ2U9MTYgY2hyb21hX21lPTEgdHJlbGxpcz0xIDh4OGRjdD0xIGNxbT0wIGRlYWR6\n", - "b25lPTIxLDExIGZhc3RfcHNraXA9MSBjaHJvbWFfcXBfb2Zmc2V0PS0yIHRocmVhZHM9OSBsb29r\n", - "YWhlYWRfdGhyZWFkcz0xIHNsaWNlZF90aHJlYWRzPTAgbnI9MCBkZWNpbWF0ZT0xIGludGVybGFj\n", - "ZWQ9MCBibHVyYXlfY29tcGF0PTAgY29uc3RyYWluZWRfaW50cmE9MCBiZnJhbWVzPTMgYl9weXJh\n", - "bWlkPTIgYl9hZGFwdD0xIGJfYmlhcz0wIGRpcmVjdD0xIHdlaWdodGI9MSBvcGVuX2dvcD0wIHdl\n", - "aWdodHA9MiBrZXlpbnQ9MjUwIGtleWludF9taW49MTAgc2NlbmVjdXQ9NDAgaW50cmFfcmVmcmVz\n", - "aD0wIHJjX2xvb2thaGVhZD00MCByYz1jcmYgbWJ0cmVlPTEgY3JmPTIzLjAgcWNvbXA9MC42MCBx\n", - "cG1pbj0wIHFwbWF4PTY5IHFwc3RlcD00IGlwX3JhdGlvPTEuNDAgYXE9MToxLjAwAIAAAAQZZYiE\n", - "ABH//veIHzLLafk613IR560urR9Q7kZxXqS9/iAAAAMAFpyZQ/thx05aw0AAQoAAjZrf0Z7SQAFS\n", - "RBmrGveunhOj4JFso/zYXaRjQ18w/5BhxFIRpIkBeRXl9T8OOtGMbM52JtIMXIY7KRr49/IsKi0w\n", - "jJUK8Z7XIFmlAjIU+jSbWER5LmeK+6/diSLijDB3co/ebDgChTdnt/smJJAlFMJhzTUcdwoA8NQo\n", - "YBnpXwCtHd9MDNyz4x4zrqfgfXAXtVDOuKqK+ZIROmkudESU5HAc84NxG9mIFkHTHpfRFX0vfuvN\n", - "v30XneTe8IilYhOJYkyOcVBz9L5D3N5P2RHbPf8d2Ia4qkwGurGLJl8PxjFsKE4dm+f6WYtxh4/M\n", - "EbibuuIVHuFVTrhDBdjGsnlvGJ613cHSu4frv4bqhIfOz9nOKI/zhLw9zlvfAkAek0G+jTz8be7+\n", - "o/ndntGdno6L1LXJpdgGJYFOyZwDpk3suJqu9FKdCFsjDfQ4s5OYpZkBRm/h6ksvqs/jKOI7H7Eu\n", - "JEDtMn0Px1875SS+KLSHaHwtTCNzTTTEE83rjSnRcLH2qekoCAzC/F7u+tWoo8/5q7AU8ZwbFyde\n", - "C0AcLGLOTLX2dctD5sMzDYlYtX/lYiEND4SUALBVfbetB5IH67pM/22hp7cM4zkyUfekvXZeKUpq\n", - "ihxpjZ/b0GfRGel+eaIkRAMer8l0HHBl4xOpdwEUiGEQqacmsmAKA7/Wn0I4FZAkAeHbrP6JQw8G\n", - "T6oLn8jHc2YBwe6YY+t5SuugRFwnijdFTQ2IYMHZ9spzZjJhn/lftFm13UY9ay8CDty2j8dXZfss\n", - "pdN3RSB6EMFrirN6yUkoxa8UPGBKHs9MUFO5MnKDgADHT4JhBGInxUASlDV0lsFB0GH9ED4tkRc6\n", - "7SnaMmZwf9T2i4a1NSsheM+jHEQWr9fgPDBABuIyToLYLrnVeLXqSC8JMeZigh4GOpQKyiIsG8oa\n", - "f6kiBTwG/5RebTqU6O7rrQLj5Wd5YFdqaacUZGByo8AxJ60NHIoQcxeNjsWAj6m8SKd2+g3en70+\n", - "zVQW9HkvHI7nnRF3FhwhZYu/LvproEPyWSYykJIx75ojR14WE7oWSjYs0X2AFiwEouayVGii6owJ\n", - "gdlCmnN8HoqT5PPnaOWG7mPgq/3meUuz982ZX4+4VMage3Fe0K3cqRdKLTge+gs4pyQbSUIdrgo3\n", - "4P4R1ejF0wAW1R8YjLZz6fQUzzzchgNN0t7aa8tlO2yDCmII5BbaYJXJrRvBm8Lb1m7TLILNalgu\n", - "RMjYD4Pf/P4iQqWsBEdgB3p334RMzrBfcviq+49N2SRQlYxV0SbSMdybZaH+vxuw+VyvLt3ulEcF\n", - "rmBwnxL4kpGATPv8mogAAAMAUMEAAAI7QZokbEEf/rUqgAYz+kaAoYS6oZnCZBWChU49QzRvBVh/\n", - "3Pl1tY/3h6ui3wW2qKCfpdwQ1h/uuKhRazpong7+Xsbw5g3mv3E7I0N68sUiey8Dbt0hMUrR6zYj\n", - "YtzMQ7gEdgcbbOEgu3H73w44JvEzvgZ4iO4Q2Kwp7BHY2uxxtdUENoG1kHXqnnQawFSCHZ9W6pRZ\n", - "ZX580jW/ekv7tzX5SLrr2mknIiIEL/9OqO/hdKRWyIS92L0VbeMgboQPIpdXZEemH8ScfWR641oo\n", - "Kb2ZqixayrynX4qeQdDAXvtKdnTPfgTsOJHs6zrnaaKb6SpoCg9ffzFUfiQ1YwLPZpLhwkJ1F58m\n", - "QtliSU1LCArOxcL0CdX1xv0PO1XbIga8mvD2ON78HrYIlpd7r9MIJUgGiGlRxLTUITjvxtxjLYBG\n", - "TBzSQ2Mqy08Y4xvBh9/AZrWGoBvplKVOooBAXsS/J3OngcAaMApnGniTlEgacIB/4ihqQm9Zync1\n", - "WrLEldONGr9K6gbteZcFnK/hoe6B53agN6YwjF+Hm1IYltzK42eiNQbmeo0nT6xx724Sek57Pcpp\n", - "/+64lZEYNhMLw61j8cLCmWJLqJ9+OlV3Tu4kvqWM5A7mBmXunK5EElFvFoiaHvfKnFzVKUZHVN47\n", - "dwwOu2bQK/GEFcs57H1A4Ddl2JAlJt4ZWgrJx+vzAgyhhcl1LtQgQcd3rX3aPisDf1CYETnay05i\n", - "xe8yUL0AVMzI07+lqERP6auGU//nlrslfAAAAS1BnkJ4h38AGAsZbANezx+IWo4Ni9MoMfKTC08P\n", - "cqaDTyueeuPLGgFgW9U33kZ+Bw1xhP+VnfaIAfTxYvkb1WNMMRMsh5PjwSMCmaFIlQvFeKZwdgkf\n", - "0eHuoCcg/XQXRqCvEyyYU7Kr945fY16Tu/18Zd8NU8RAJRLFspmBVoIPZ/aTPmSIXDq8KOSzL6TG\n", - "sWN+V8RKxGwExIfHZpdEvHu1tOeg+pVzKTracfnYiBxxlkuVIyzOz2mFv1LQ72jZQGocAdWS14tD\n", - "EtCsmNljiTGQDRggnoajq8kpnFHws9ZMWmcsV4dQvczexFmx4YibNvvMPauj3CH/KK6FXvQFumid\n", - "ftiga3Uno6si2epmOuEVTuVQwXsgCmOyejpjAiAjZuUS1zq40WginD1EPNgRAAAAXQGeYXRDfwAh\n", - "r6zZu6OyBrfB5mVsAz3QNRRqvrwAcnFznD7NXanOaWlAADNOwlJX/xGmO79sH9XeNRT/FnLuEPBH\n", - "1GJhJV/Xt2R0YziQPpgXV9BLMr5IaMaU9R2CpgAAAPgBnmNqQ38AHhCAmS1kGlkSnBkADoOXdXaF\n", - "NGZr+Q4fCvQ7bHDsrrZk+gghfDnB3EgAw+hgyCz7QjPCBdm4Oua2VioU2d4nUZ+UABLNnRNNghIa\n", - "znH4EU6++iAxhcURNicOGGgil2sQO5YirsL6J7S/TznXYcILcn91E9qrSkdqAKeiqMttbt/NlBlt\n", - "zFtTLIQV87eeTgQtRSaGjNkYcjtT9zsSroMxdQkaS/rgzWfPKqioru5///iiFvV7FHhGNapsB8Ep\n", - "xA6YqLEIyfxd3iBKiJ3g/96H/WMQrMVl8ykLYh6g9L/mEknpMxDRuX+/d5vuR5TJpN2l4QAAAY9B\n", - "mmdJqEFomUwII//+tSqABipnkgGrJGhoF2xhqIGFJgrTiV28TOHP6iMSZwA4LzauSvgcy42/qpKz\n", - "PF+GKWIn2EJeWsQWOqhnFWAeu8Qy08RHEYzw2BIfhXKPnsvQ1D45gRUsCZjYq85tliORVeVqHlvt\n", - "fzWrMqI5f+favhs74Q/1bo2ebSMVUSFuP3HPqFVDjXrf/wjJSgWTFPNzCZtjDghfnhYgAzPVh4sd\n", - "mfpnfQi7UGcAu+X0SPRW+sCzjBKyZsabYXRLvCvcRgXcWHRJnqJZ7DbIL5Ahmra4MUmiAdrDqxi1\n", - "yixz8Ge2MnwDKePhHbASj9FgVyabApZmODkYAk9x2eNsu3NC/GWuEsOYUEJXb3NkJ3H0Ehpogb5q\n", - "/7IADF2Rk2r94PZTFE6TdqRa+DeKrhf1PoBJxN2bNx2sA7Pci476Sn+ZpPsAPTlXaikJNRAhO4tD\n", - "lakPd29Edmfvk34bCqY6rFMuCfUJ3yzCy+VRKB59CtgS68dVzaJO/FxZ2Of18yjXsScM2fL16/kA\n", - "AADDQZ6FRREsN/8AHa60qBaQmR4IRAA6Dl3Sc6VtGJbtr5vbN23f25BY5Mbt9ZodJaqeGLgSZDt5\n", - "tMt3+exLq/o1or+DyDOaUjfDuI6HO9EMKVIFrK5bBNySwYGQ9ZOLXviohcSZAskgQCT8YbljWqgY\n", - "W5O+m+Ip3OoA9JMxAp4EiGRPR1hmuQDeRomyGX7bvvzp+lmhQcgx50Gtf2FsWph71RE5OIfz3vbU\n", - "YPJzvstNoHMLjQVN28uexbTk/wUswGjCQ8u5AAABFwGepmpDfwAhvaAbJNR/9ddNI1ZNZPr5vm6q\n", - "XTetXH7Eo8GqFltKJbOb+WxFxg1OZ9LY7Pm4G1n+FvJzAc9iMK3kbM6geeeFIdRl75A0UZYsXIff\n", - "dQXiQxB/kP/GUeJS/ghHdsFXhovY2ei0jBYXhl7XCQdiM+OxqVpdBNYdLY+vhvtTydDweWAQhmfY\n", - "3fYN3w2o0+YtvleCAQNIu+tN7OfSeOifT7EOLQk4YDYkvT1QcI6scYDf1en6ihiP1DSq11Clzx8a\n", - "ja6cddGuoMqDaNkxCF1dzf2Jvz1VA4BpWPjukcCUvSBL5Hjn5IenmZHNevhC9Ri5TKMMAK1OUZos\n", - "eUJttkHLI36Z4EqqgVQeXc7fMR78LG9GpQAAATJBmqlJqEFsmUwUTBH//rUqgAcd7WUAG1wL+eMP\n", - "5NbNjI1PanDtCkQqkSzemsYEjSdqyjDQBhMRhcVkBjrLnQ37QRY6anUo9HtaOXKEvV3Oq3t3zJnU\n", - "VnRnO4+DsYDha+hVjf2RQfz8iIHBAMZBzDCidKRjdK++FyTTJT//wjjoyDzrLD81EvvOEfP1hNq1\n", - "E7Mf/LNi4VzZp3xaz5k3oYD4Uh8itElOoUglEcP1/ghF2UcJA9hOtkSUpVhA8+T8Ytc1zpVMfYyg\n", - "QqbyRa4EvI2+PCgNWtypZmPOW/fUb8LPNYTg5GLhzbOmSjYpenEUzkib0QksNLKbj/E9aHrV1qHX\n", - "qXiny+3UUPxYGvj/pDuYRozh1EchMNkv/eHEkrQhTQjnyxDirLtyAwkvICbz8w9UK2AAAAC1AZ7I\n", - "akN/ACK9oCBuM4cceanCEEWpV8cuy27lpLcHp0RFJ/onjSEljOG8VqS2Rkf30kIRre+KMlNGVcvp\n", - "cL4orO6Yp5KjC/RRBwQz/yE8UKLNeO0Y0FFhQfICXcBtO9ndieTXXlspFHuGf4S6CeBKlAO/lDFn\n", - "Bm6rf4RqP1vvLrD8KUBlig+AFH77l/U3BNsHxmcjURJ4rz9SBUp3dWhkBmKNCP57UtC9bKnqFyE+\n", - "YvACZ+sMCAAAAZlBms1J4QpSZTAgj//+tSqAClE1egBKEwbZY3t792fWy96pbeQQCnoXHta8keYB\n", - "6YD4iyrisk5RAGXAP8hftXkqsIp3gIADtqeyulunIxMvA+tHyMYI4mH7Ktx24JQCDLGwr+SW5Lfl\n", - "LFzLN5Z5EpfMBtjuN1e5MGJfkKE7RLofReD1fgshPg5Hiu3eNzKNtXPqCUQOQrANHyjLVDHW1On8\n", - "GbpMg//3+EW5h//MyUrV8C3bm65GCPAdr+IiAQS5PLqRpJaqPFXYImLzCfEF4IcxGqfKzcnaOGUe\n", - "P5zhUa+at6SYruNLfSBlr3+mvyhAAxPUBpQBX3a2ZIbz3QLaxiA/KmUnrCDmuWAQmEAoRWFYDkhB\n", - "vSu304LzlIj5BSPPqNvyTdiIsLpzAu+SwxleN8rOU8p84R24aRhgQwchoF64pWQkYvhDlixS1XkC\n", - "+1BFsz/ugThqWNrj6DMWcUAmd8tN3JWA8raGQmJpBH1Zjd5483GFE2+DssYAdvIzFktdYvwqJy33\n", - "xqAAiKb/jZmChnRmwaKmyp+usNPBAAAA+UGe60U0TDv/ABgTM0cFpiU9S5COo+Eq1a5EDpKRq+6p\n", - "lSs4dhBzMdhHGYju3Syu9sir+n5TA4S4EozXRjp4djOH9s6Ebl4mnuRqUkAVVyRRxloLXXdAVwvm\n", - "Kw2kt3nH3KtGiXPZtoKRlLMwsYrakek54VGjJMSSK7z2j4bZfzdU5fWILhtGELYhukSGMv6CXtq0\n", - "ugZLCx24z5CJjXHZ6aJugoOXVvLE5AMKcYDe/LowGji7OLeFgeB849mfSaUGlnh7jxuhBOU+fRS4\n", - "p0ITI4vXzUUR4XVTQrOXBNie8HQwoivm+WRv0nW15Zl5mZ7wAnqm6XldppA1IAAAAMIBnwp0Q38A\n", - "Ir2gIG4zgb64sxYLzhi9P+r7lwy6Wa7RRkAjTYM9mY6ueOaRzgw6T2RlVKQ/Wnw9OUPsoB+98v3K\n", - "7Ai/8Ku9oiX4fIaC4XxFxl+0lQDznNsd4UfPo3AQh6FoBHug176P/7mBbtXW9HioX3mZhTRXJOlh\n", - "Psk7HP1i1klJ4f63KMPuZvFOjkq75Z+u+/aiOQvmn6+lP0r2vSaqs7nxNSGwPqSwNXaUgQz58aD0\n", - "pB2v6eKf+Yy3eGu8f7HHrAAAANkBnwxqQ38AH77opN4Quy1TZxAAOg5d0nOlbRa1oa+CUrbGUKO9\n", - "s1K1K60LxAZlk8ZQWiHU0UUuQDnHAAyjelIcwOj4NipQdTlRBT+HrLVCVEK5smCT4WEyhlST21vf\n", - "pS9QIx6rrJJt1ZwRk3fLMy3lh+GbSU8p/deKiRgvPKu2y5xljT8HokdUfoJBN0b+9AYNdPwZxzfv\n", - "wRj3rjB+XbCQdH7rLOmVBWtc7YBBcmnLfJ50Xx9vsPrIGyT/orCu88gDS7Q97WNMWaRoINuEV0SN\n", - "7lASQ8YC8xeRAAAByEGbEUmoQWiZTAgj//61KoAGg+KazAhO48Rk+mELCfGa3jedcL7j4gDd4k3m\n", - "hfDQA786lCeWa51/s1J2qe/kkvnBjg4L/5tqqnPuWzD5CtqsuCrBZfD9tieYn0V6h2QRjHTgf2S7\n", - "KbBJVduRkgXz0DCyLCsDRdQx7ZVeilFNQPYHPpL3dFbV2ZQLhZ15DCVv0ijUbfdtbaCxQWk4hFwi\n", - "4Cl7Vcv5eumMKNjbBf29eX+p4vfxRMeLxQVGLH+o2FLpf2SZwh6nFX8ReHwFB2aNAZojees14KLO\n", - "dDXVOKLwRfawG/F4iTHLNjIHr9KJ7RMP+ZW2v4UodTEwj2IkfoeugjPYygxsYBEN/HIWo7Lp4BiH\n", - "W+sGNW6nzMrLHeZnfPrIXJzjKMZ2dMe3r2TPoxLKTVgPHlFgXbB9gOVEkvjr1YtxEt3sHivjr7TH\n", - "zrmzrXSS01xk914HSqt/CnYSKPxa2MF69g9I/BNJSHdHCdNGwRVm5U4w/DYDySkJOTHhPK5xLTdI\n", - "6pomON2J7Snu3IFO1cMuZQAgHAwoynkWURtTVoyQbA1o0XW4HcVte0xmLSUrxW27KPhiReLpDIah\n", - "P07+6UwIug2Iw2yxWwAAAP1Bny9FESw7/wAZUxOT3tiejYgyJDRrCYHaMUHhX+buBbaoqZ/1iUWs\n", - "Jb7slI/imiQ6OnWj09SEskbfc/zlMQQ4SNXZauWfHJ95XYh7wMFGgh1p51IG9qMewyJwQS444Zn2\n", - "viLgUg5+yrpXHCf0t8/9jDlbqwjDulbT62pdxpAyxuynsO8RFT3dUKeSE5htp/jbraDowEdpXZyE\n", - "hG0WYkl+RbztI/PQNZCwZsz+nvpxvKr5XHM1hBpXHcYTolc3yg25EknXG5iovx0Y9EuSqthrt+Xw\n", - "mK43mYVJUVC/Oh8GeZYMuS8/kSjScKjb9J2cbfyAxgmK23G/LX345QQtAAAA2AGfTnRDfwAc/TTk\n", - "s3FNYSmNHdPgDfXQC1GBEwJGCqSU6MsmeFhDrrArJ4DXkS7h5Olwl5LsAdAjNSMWnsyuwfwlhiS4\n", - "Iu9nXiMR2gsFQTdJfxAGWv/oGKrfOpY9OM+oH5mmAEYRbo0uYIZjYyyv9H1tg0RX725ktocEeT9I\n", - "3B3Tp4qYCOAxN7JPiw1LGqnL098ntFu5ng1+yPoA7ayjGtnhqUNzDdxHw06qdCQZykRFXaAS2mFv\n", - "lmomA2wH7gnlU4hH+9/QtYxMog0PKOypGE94HJSUfoT7gAAAAEEBn1BqQ38AHE7WHA5VnN1RP/m4\n", - "B17wBGTsyVXKs9N7WlI9AxsJJ7v9zVkMjf6pvv+Cg6JoQ3BLOK7r3bcONYUtZQAAAddBm1VJqEFs\n", - "mUwII//+tSqABlJow5npTNmtYD16z8AGI7v0s/GnfyqOWKggEMwd90EmHsgCWksYKFE4Qru8Yv50\n", - "LqOKJvWMLHGzKIf1mWoops1hD8q4hCLJMEdRItKEcO/AvOw75DCgogAQMHz94YdBlV1FB7/3PGw/\n", - "kvp11c7Zd3bjgbTV5f9wCrj5V98Wrk1QkXKTao3xn1WeAORpyCtFJo3KIIzvry0ktsvXmShsZdHK\n", - "SF2Q6qY6Id0i1QRrrPRdF2iq2m2rhv1eY7FLgTuR+kimJsshiQFr/qQ4tOO2msQRBI4huY4JSA+L\n", - "KftHgweMeBwJfCg9ocoILqar/ZxuCC1Kx59hrQRJPfm8amRIkwU/k+wKJNYh9fLLSBsxlrg4XoMn\n", - "PzXBXS36HS/Vq/PUU0Saj0Ks8oGCHCVcz3eoIxgiU+QJY/DixHlF4+MYR1JrL+dYLi5XU6rOa8uy\n", - "cymZbC8fCrT8nFmCuYcD3DNSzmKt2Ypk8ahqcNxMHCCE377w4QcAAK8hLicCDiuo9KVio6ugqDQM\n", - "DiWya9QmBn0ClIbSCznyVdfSZyODo1gjrJ9IiCMcnWI45hcgB0F/w3f4fUDX3TFD/vbMoTmxwMKV\n", - "hWEq4XvI4IEAAAE5QZ9zRRUsO/8AFKVUcHl/E43Gt6o4RZvBs+iAp/X/n7d7Pz7RdmO0J7CPEDVr\n", - "YOGCwg4aa5sRnK1DwPx5sIYzP38566ezpK1+yb8tpnK38Otysb+fPORXq89pSQ+5zLmadq08PRPq\n", - "ft5b+CuHdsaohxgMdfr5HBiNNodd0VK8TNpXmgIXzYR5RpK7ScM1kMS9Nv/EnJHMV/HrvGwgTDTj\n", - "k64XWbP6seQRZKb98opQD+okWzwHsAFj5ehr/ekl0IlB4NOOkEs2vqjJoc0vIcwkba8FSFkLe2wm\n", - "HNG8c/q9E5Tipy3avrHlLTvT0bjPkjeD4HLfC3isImW2RvjzyyF2TiLuxINvE8y7u04RbyNnhNhC\n", - "J15BQDsVja0XtFDfnnr/h18foOkLRpLJ1yQTMBboYsOrVzSZ9GDWwAAAAM0Bn5J0Q38AHQXz6rvN\n", - "uarixND043ZCNdAAIHUCWbOjp5TUpZdEciERk/s2Hj36k/1QHuy5AO7bU6FcTtkwLNXpp4kEhhr2\n", - "pj14tuqcy7uq8XfveV+qzHFw516IWJuk3fnleTKVnyg4EmdGVkh8uUm8KAFIin8/UzurGkP5FXB1\n", - "JS0uIqtx2mbD94hCpeHMsXHXmWbW3GUD6bwQzUCwUdgGFWWOBIzHIH3jzzxIIZ0rnTzx6fd8zSRM\n", - "hMrhmhy9AElVESMBSl9RUVwHxFBAAAABSgGflGpDfwAhvaB1qIOto5yaJpOYSSkbksLCkPuZStd4\n", - "LeT7CV/DcB+jLm/y8AhlFfeod4crFEXxelJR/fWiWC5cEAQJB3xoICKkbqYOm6EmFwfhOJrnHL3F\n", - "i7egoJ4YJywxTcfWExKLj/7q5Qta5s9pQnji3v49xEhquy1bNbsP/0r8degDcM/eCvveCCuWJP4W\n", - "kmgZOsTL6w2RcANA9FiGFsZYFgwwIJNSoi5uPhHUWhw8DgpZUJJwhbcwAlrJ/XkpDgMQdv8+KTaK\n", - "5RNrXWUI+DQboZuQqh0EP6Ucm1iy8BiBubHVtPfvfM6aTMlQH2sGDo7kxk+QnIaS5zzgTFrv32D9\n", - "yKVtBoqoPJ0AuZgM4FsUTuUjy7Mb8fU+FNoSPESiOFS3CYbvMWBzWtiplx16c8G+2sTGiL+yia5h\n", - "U5UjqF9tl+DCrXkPmQAAAhVBm5lJqEFsmUwII//+tSqABlvipo+ln6jP3YEZZAIeN2gdAdBG93Am\n", - "88+PBAP+pBG1b08i0fIFrYTfZkz4SYTuxIQ1JlthBpef+blJppNwqif1piWVs/t6bCj9Z+mNxSeq\n", - "fY1/wgLfvSZhz+cH951YQ+3lZMxDj+AnlpOYgaA5ONYw7fbC4eXvAp07e1QLTwt7AKsxs6j/dp/S\n", - "ROqifCEiS8aS31tyrNd0WUbq8QssOlpj1+9+m64Uuc7+f7EFYNlp0SQRRU2ux+5kBFuUthOQf/99\n", - "ODAIvGEvExgFy7U9xycg96i+XWorpOkUsmc8UuZbMVhIEf4MYVuxmTzjhiOVDlxwcksj2gNb3xa2\n", - "pmXlh1zp/jlUP6lnJbCcR5jJhGaBJ/wuH3P+rOiJDpAwjSIE4agxxO9XGnmQRqhYjiBkbby/Qs/C\n", - "0p6IlpvwhBITpwXRBm1mH+MtJEskEccmYaNT1YNO6b966q1ndwWmG4wqG8yXMOLAMIGnxTjTIpRG\n", - "9a5Z9Xdl+HR4ndQhvFfQ+mQNsGUdDPAaOtDr9NfsDESdrHz/VFsWMxlbozv6ME9/FBsTE8SLTZxK\n", - "uKA7LtdEmFdsikvrVwkDRWs6mlddIWSLEJey878D400I9Bm2F1YzYF8hIer8urpKTRWH3dl5Pnql\n", - "OkpPyvm3RplNwN8DaGYvFB3ajEHHx79ej7jTTF7j2dZAVPOuzAAAAQNBn7dFFSw7/wAYtYg8t2YJ\n", - "aBl5mT7LoVquTMWPsAY8JEk7n2Ltj2VU9Y6yhnUjGblNmyV5I1tDP1WCa31R20KBx8ZAPYjEjgAl\n", - "IBPsF6gwEF1mGQPgwIt+DQ7Ltrn+WWljoOZe6qmL3ODaEJKUCy9wZy8Qi5WMsDYzpEybVU1vipuE\n", - "rsjD5epFom/S3CRpP+JRc2SuBGV9X135AtKz2dAbEFqb0f/DUfvRpyE/xar90tpMsUisBmDyfPqC\n", - "QCIWsyVA62u0XX4SHuuo3VkmdASLaLWJS0hWsThucD2h8t0xx4j3t8tQeFkAoX+vhWm72BA6IAOh\n", - "cP5AynBLYvgLjkBSaw6ZAAABWgGf1nRDfwAgt5i6arm7oDsF+i9EHiOJ6m6rVkYAHTQbG9yseMuo\n", - "2+jJx58xpeovc881Wv+6nIPwZiRTONb2IQaBwPwYP/UAnKjoweUWtNn8yjj61Yi1F5n9oYReT9vo\n", - "YNykd6+UIhqXBR69VB8JEqms6DNcB++Z+7S8cRY1PTjUFRAm3tXpZtcqOC46Yje8Z3mZdWtke57d\n", - "wfIWf/bjH+PQoHPWtMGigrlGqEUElC6TETXz+nB7X3pF40yVazdjxa5pCPS8j1Bqo/RmILtftGxN\n", - "Yu+1c8QTzG5+3qHYIB5lZeEW8bNhQmHlV1zck8pKhAWM+UMUo8Yo1gMDIjGuUuNGCTYOoVand7oO\n", - "JxBESUm+840sI50gEtqO5mhNaTQVfGrhYgQvynil8I63rBmEOncCHtkN57Vx9gduQDjk6aOyO6bY\n", - "qsBt2jiwg3SW9pmMOjEKBDS6IfMiAxcAAAD/AZ/YakN/ACK6K1xrl4Eswd4/m5m3eDoe6aKYRGzt\n", - "qScyJrEz0/YMsioeM46osJc2N8un8CXkVjpps6zgsf8LlkG70ab3ccrB+um/wXzisesiYCwJDgAm\n", - "D8ODYrLA2f4XQyaEvxMLwdPggFdV9SLGW7IaDs1Gj2MKL95CD69ggFd4PlXdr+MMXaKnRfCfYej6\n", - "jyRkJ6YHIJryGsscniQRwJ0d+J+1KTOriJZQomY6moOkqhpxON7UIyt9lzU6HlHOyQJ+oRH5iOIM\n", - "+hKNz7H8znQxxv6dKCBY67rZbPlwYKywoLx2OIjAEQohlh7LdbGhKMy/zzEiJYFobhp2mH1gAAAB\n", - "WkGb3UmoQWyZTAgj//61KoAGC/pGgJ9CubE/Hy/U90CEEMEEbF2Q4cnB3oAeksXBYLQl6DX56J1l\n", - "w/mHq8WxaGt2MnAvQ41YNYO39iE6FvpuFKpW712yS65PLr83LJiqo7HZlMfRzKZN59Hb83g9Yzjb\n", - "LItfty44d54BI12++V5xh28HT7V7r0Y3bFC5OovybNWx1HQWDmvmM+uWQT6BKmA1pblkm0jWUuJ0\n", - "KAyepKH6sPnyIzz9TF/cTcVBDLcJ0ebq4QoNf0i/efDFq1nH+LtoZFDiLpeCwZkCLTOE+JMjcVxC\n", - "aWP/XfyRHhNANFDKtoVePLPasXuBVFa5xCh3bB99SWFmaQdxLlk9zHTMNOyCWoiRa9OkdBShrOe1\n", - "dfGrU6t4YEao5nNo7umRhNJMptOYWcUtCbSBQmV/4G3c/zgmpJb1N+5bNROg3nNApsFhNWPnDxXX\n", - "YEcAkKEAAADvQZ/7RRUsO/8AGBSepWN8xnNsxE4oE6H3s58lr1m+iqw+EfUFRD+Jna0+Uvzz41Eu\n", - "ATVBokoBIC1dZOqsBeTj8Ij9FIuxNitjsFqDL+DuZwvmGihDa0HIS79MTSVw/f89Ulk3p2M2jbij\n", - "TpCkIItiAXbWCZspatvMx2+GoOmu0/Pjqc6iwrXWXyi9/N9Jj+yY/ClUEyj7sTv82Y9nVf++GCrf\n", - "1w5ltOrH9rRQKpUQaVxp4gxcgxC4qFFOgMxs83r/WkZSqY9kO/9UmmCqExD/ljnRMUJvxp8FxL1d\n", - "H7PGv4WLI5AeltB+MOGIOr9NYMAAAADwAZ4adEN/ACG6NY+qIzQfcYKCb0AhP1JJtQboSZcB2Ux6\n", - "0kAZypUjTcd/OmJjJuZBZL4W6I8Qwzms0HJLp8KRrHdk5GfU6sWQ2Z+fhfAzgzC1XgPD4QBqkDkc\n", - "T0sPX8iasgf4/DARkJP486Pq1cqH5kOYBwnnR907+n/qb/xaeHwouVk6h00s/qlqepq0S1p/xGR/\n", - "GdINVBgCemrU+PPAyI+EQBjfU66sma3ahiVaLQtsD7mxr/vZVvwLqa7Chr1J9NZveiHKnAzIMG16\n", - "G9Gmkk/8FUHgdrIbZ2heuBDh1KQSBCztE11k+ocodRJkiMj5AAABBQGeHGpDfwAhujWPq8KUOIXq\n", - "Yi8pfsfzwlVQDEG6igccpABq5mcqZlBxZf6f05WsPP5oiGUHFHfSykAR60y9PVPsKziKYov/dHwR\n", - "Kft2Arvz4qT56TCewQ06i1++DP3k7arAvxqk9+C83xiDX/XWrTHQ1+jT9fNei76g+LJLvs+Z4UVk\n", - "oEaQ3c6fXvOR9+Md7sWQeZnYPXpC/0w6s38iG8bM/+n0jsTdTFeBwE6YfrCAsv/ybSEXYS5eoPM3\n", - "f/HRzfWrUb9MZw2WEuoxs0K4qVyNiDTxcyb1DdadbkuzwkaFG7T2ZM6Pebp0YyXRqckmxx6YTGzB\n", - "LlKwKmWHeooj6Lm9LlzVgQAAAaFBmh9JqEFsmUwUTBH//rUqgAYrWZggqZs1s6MH6FUT684nhne8\n", - "ykZKf89h+0voVegpTcVlgsFoS6xwNTcMDCv9PiwISM3bG5gmdpPxwsd2af4u9VMbVGyE78HSQ5M/\n", - "nbkySYm5CPjed6c1fzFNEjUv+hlxYNfv3cPYnGT/Yav/5erFhxatniKB++1xw2wwwm3hwteUjAt3\n", - "Bi79ySg16ijYqJM5fa8+vosVJZysXRlnbW7/ITdmkkl3c8ndruo8FzJ7m8m8z0kOYciXI4QIL6Xh\n", - "qroOcvOVcWB7Uug78ZH3AowGQXzMbzVMrLD5Q7gJi2vHbYwWBG8EpVzYFtaj2m+v5trtiq/wJKtt\n", - "WosqXvVBFnxrWYQFjXg41D/ASyQHPzn2WsqemfWG6/EDepgeax6MAFQfxyDScuq3fNmr8jf0net2\n", - "tjnK9AbUeZfaZDCLHpnptMZuk8clMx5Y+UVSA4sRK6q5yL86vVu3TWQ+TGs9ZFdT4m8kNBPSkwSz\n", - "rQpsGSml5JPzqe84pJi6yJhqfYRsb2q5mJ8tkrUntJCF8lR106wAAACuAZ4+akN/AB1RsSI82HuA\n", - "EDVZr5mUHFl/p/ZTcmoRWj4TfRvTsYw8OlDJB7dvZ/vcXyur4LGUumPqBQUBQHfGq57+bI/8tRzs\n", - "Z+nHU7WH8qJ9BM8/NBixjH12m2oVcRb4XvfrX32V+Y0hU+0j88MNPEcdX4rv7aeeep8jA96PadWJ\n", - "mSmtmcZfJIFp4fz7nGsOeHvsRUbV0MKDUYmKN+mrh03bThLfJGXI3U9Tnh+UAAABmUGaI0nhClJl\n", - "MCCP//61KoAFm+ceSLbmAtKM+jG0tYuAZBSWLg59auQBOS8BoT1gHMsjZkIU234iG6WAeSbLJEu0\n", - "KCLhFA+AqaJQGzw142KKgdSAFtORqvq8YepvegTzCCnS1DU11oB/GUVDtDnboQEryLd0x6NUSSMN\n", - "cECL9Mzb9QebAeTbVcgtE4xPKr7FEgVH4vbNIioC6rYN5svm+n7fErwoxd1c4B0MbzpTJ9ypWCIt\n", - "jDqP/6ecCXKe8Ac6gqcpyPRaKmFcKdx7byHCFs3Y36UHxsmpasB5iKonQtfou1T7ViPEDD+TNshw\n", - "6ncI9FQOyx3EYxNs7CdmXQjjuiQ/hVztgan/8HWeS5jp2zgzBv5BXUEnWn+A7+FBONSn2LL/uQ/w\n", - "xRZTcRa0x52ow/V5cvgKu7FATp/RCkX/G+w1Qnp+0VyZbVkCutQ1yOnQYxf79Uw65C1zWPQdQMP/\n", - "K+VS6vPAs27IKeqUeSeiBKHv/3isIgE+rjxQbN9Lh1YW9R/9r++mSeHrs60NzUtdlXFG/VIZkaKd\n", - "XMkAAADXQZ5BRTRMO/8AFlm8HmElw5CLBq61UEezfOfwLuaBDj371pFQE2TaGfrDL2cPvWN1QZqb\n", - "tmH36IVd+buOk4nAS7OK6LGtZWekVP+ro0ezqUL6LNjplSKI15AkcuTQweCsbYhrSLoTsRiawYgs\n", - "mv975sfbTCY9L8bxROvDNcwG30R1+JWvK+o/hwf/xA32LhBb08HGKIsZFejSCR/ZACyPMiASYPKQ\n", - "KnKHiabUDVxwGq+/saT475SIsPn2KAHPd1oy/JYI5la+DZBAp1lqCWQj4yUkciIB5BAAAABzAZ5g\n", - "dEN/AB8V9DqLglnogAnlbAbcaeEM/+Dr1d94BLu23/b924ZA1vKLZ+NWO2PdXQ6go3Sf7NA4nwhe\n", - "Jfk07l2+PnIu+kI9sd8bYLUmTTByKGfoyEUnQqTPIf5dfjB+AgnVTc5y8pWcKU354gRsJCt4lQAA\n", - "AO0BnmJqQ38AHxX0OouCWHEND0XeNAIAEOFUWlDAA6yKdnA6h0XJ5AHh6k3PwK41LuRgTA6dFitc\n", - "eGcLOFImUAXmZeNXd8BBiP4Y7WDb/nj/8t7UR/ChuIYJmbMzvyMcttz9Od2nvufuLeTpnnGxlC5D\n", - "sKIQ4TiAF1Zf6Jjc46nP71VK4g2t6fmiQijizaslPXbGXByTezIrwT4YraOsiMH4GMwabs58JhIR\n", - "tYealSfNunZO0jU9FNwqBbfEknuQIRSATwmWr49+JU7MtkfWDJ9lAsDVu2W/43LTVqxccM6dY8NC\n", - "EBnYMhV6U9uYbKYAAAGwQZpnSahBaJlMCCP//rUqgAZTWZgI3NAzNytjReukCJhCqRIQrgVE5TFG\n", - "RpO1ZRhoAw39KCX0FTF/pEpCWlYTREK0RX8M+i/Zkz6IOh5zRR0GMJniH0SeRA8U+ZBIRrL9Hl62\n", - "8kZwKv6q5Netv/8gTYt8wrrWIwWANbXHJaruY4G39urxvB/yx7ozBV54M/wmK8P5AgF0ljjPQAUZ\n", - "DnLEHwmopi3rWM++lGz+7pSmghGU/3PNF3AxzoRutm1cdRdLqAFKdPRrKeDtflDHW39dHMmsizA0\n", - "JAD4HEW4vO3o1CbLX2IxlZFPJGuT1QOtzPR7lO7pJCxfeGJXFchlosXXXbYjZoXRMBBKcHqbIWa+\n", - "lcjl1FcSEXbk84/WCNR/hEiDPBQ56Zc4Yg/Uu5te5H7B3WBkQkc5+tttienjQao2TkWT/tLarBIb\n", - "fSMA+83k8gbv1oyeFIIWqR6ZYarMVbzfFtnH/fWhWkYB/el6Kk3P0OPSTUOVwdEnhQ/ztu0l8Ij9\n", - "PRLg28jDAaygyMt+MtthW/hM1h+aETPrMcrgZoJoV2dKCm8mLdDu/CmksDfLJBRBAAABQkGehUUR\n", - "LDv/ABi1i6Ag4bMBZUwXqVJnyx2PYc2F7FCjvy82YHTp5//HJrbZhCcYERymRfl1ah1T5z9noaM6\n", - "FqCYiKh/nb1NKcv6lay4yu1An9EGWzEXMRaTXWcwehWRMZky6GX2Elv0mAOhcWIk8WVG2FWKKMhd\n", - "27a8KH0mx5CnVDu76Igw2moc1+yPfDPZnRGymeVWDMSj1/TY3hGgb5hmSfANHPp4nyrFETtH62Dy\n", - "FIZnfZ2tua96PI/858zqXLfYaSaEy66elRjPHGSUQ+kLj7sT6e2TgQoh23asg1dvl0lw6aW2KtOQ\n", - "yQVjdxBZzehiTDj2VDDo/FI5LuGH/jfe71B2giPdfSUEN0GwZPmh+oBJ3YPtBDdEXjvqGtPnj9YN\n", - "o2RsGDqkSW3oa8BY1cptmQPEHp1SMBrX83w6xtQW5X0AAAD0AZ6kdEN/ACG9oBtcoOCFYVPj9Yn2\n", - "v/zfoFr4rWL2j9A7ZlqQHr0ZVpbLuAQJB33EyTSBNnFvVuljxMl3V6GA7Dl0BClPwL31OrTpG1l7\n", - "a7ghzL0atyS5ApCJWtp2wOBNzezTQ3N+Y1tH+luIT/i1PP0KLgniqnzZyMrwKfZeXoYEIl7twi0H\n", - "PJVeAcAdd8vPtJ2LywfKZ3u1S3on0S/4f7cj446r85qt7SkU/lr6c/+gK5erYXiPq/kf9oXoMNwY\n", - "9h0XgCkkY0ibuAMW3BGf/tJy6AGuO11Q5hQVr9nNkIcjB8Plen8B0nqwKQkOaIEp5QYqYQAAAQkB\n", - "nqZqQ38AIr2gIG4zhxx5qcIQ9c2Osw5+uNtUP7c8wH627Nk93kOS5kJwZOUsa/GuB8LSJPcgk4rv\n", - "NNy4X5Kv65LRXZpkjxKOzss2V4BAkHf3fdjwk53/8IYs8s8oIvwVKvgR9wljv8Ag07Nf+XJo681q\n", - "NbSzOUK6bv18ql/byQhgzEpF9gyeKzBYpIes4Jq5ygJqsHenGCQnuZZGCejK/v7YZig/zrXj2vhG\n", - "gCib7VW/rlAZYnZRYtYW6jN8+34R58oAelpNik7qpp/KkHdSQspzMHjVSAa9yHgI/KVEUfAeaSTC\n", - "N1Z3u1GIF1TdZRU1zNyC6xbuAxPXtz6Ez91WiAF1zBDEIltBAAABt0Gaq0moQWyZTAgj//61KoAG\n", - "e1mYdETW3g4OxfplN37UKMHTaFqDxb+9ytAjpKDc3XnMw/MxT04D0MH+PToJ4KWEuN7AocErZRv2\n", - "Rz2GQBbpS8lS31542pk6xM8YYh0/yeF1AnMnBxO2+HilOPhojFg3EW0klIcf/AybMYAo9NSuBD9C\n", - "s4e75EU0t8atdvYkg/yfik+FMNyFYTUg/mi4EKL8VgLWVSi8mxQ1+/EWE53/+fwb7K+j+527pMW9\n", - "VCj1B/8oEXG8oxyHRw/TQGPoBS7lGz9zLwh8gXusGZBvY9Xy0pnRdJKDkZLO/YjZFLNiCRPsHTqL\n", - "i2GYmJ9itG9pRnevDN9cAKQP0fgHBe/nvlXFVK7JMen+RKub1gCuPtFfO/y6rA2fstwepz1bap4Z\n", - "wJXzTLHNbeZ6/jnjul1UTQDo+Wyv2+WNy23qAxLYAQV2nquSCySITwJSTVvg+SdePIAmj5UPClGF\n", - "OrJIf0RX1xfSrhrpF0W0EhW8ceypgG4+dXb+bPwXKBwbO3GymyW89X2WJwubd13etWWTwju8K204\n", - "+w8LWTwxqMyJaP52mExMi4W5Yjr9AyAAAAElQZ7JRRUsO/8AGBMzRwWmJT1LkI6j4SrVrkQOkpGr\n", - "7qmVB6agtU/P7NMI3vz5LIs62lee9zlMDhLgStRXRkKeHaPAGaY9hwFwZg4RZnlEijsKiC6r+GA3\n", - "jOJMGPR2G+iEvFq9JqYdk0b1d9ABTX/7oiMKav8zTfVNhhkqe32oj6u1ioYXU2U/9Y4cH3f/N9Gx\n", - "JhjbFALTGuJMdeB2a/pmxPSRSx2DhwUwXe3BT4iK5IJF2QdQUjRydlTK56i3AOElSAfT6NVqnLr8\n", - "mfbO/AiWtC7ZCdSKqLQrBheoCisxuwRDc+0Qj4IlPLBawyneGpiLaece3KMzpKTos+5YxlSYlKtg\n", - "/Me6PG+fH2sUI9B09T2Px/9ucFTXTUC5j4ELLv01D5MY2VAAAADfAZ7odEN/ACK9oCBuM4G+uLMW\n", - "L2dP1lfTvDhmlpluM7IE4yEUJKicqu4KM5OijIBGmwd/fv/FYUE8C16mNefQ0Uy/D+0+Hpx1ZFAP\n", - "3vl+5XYGW/hV3tVz6fpDmClx2VYPTKI+QsHyxc+qQa6raGV2rQAFnERDWDAoPELDpD0DBzrtQ9Gj\n", - "f1X0zbjtJNpqrwp/hRbaIrr15pQNp8wHXKVl3vyz9d+FD2rUtkJQVzj6V7XpNVWdz4mpDYH1JRGS\n", - "i2MURr0RotwXgP3Qnz/8L/EyxM0Sb/CNWw8xQFPmbCgpDwAAAOUBnupqQ38AH77opN4Quy1TZxAA\n", - "Og5d0nOlbRa1c67qPfhIW7P+8Av3GtFE0HFQCvcwO1xKybwlnguY0Nqo5bzwqVZ4m1UebapfH7JG\n", - "d9M94gSTzLBzp+7XrhnquJ9dwfh5fBCyLWBt8xSfTcJZr1HXGrAMOw+Jv+pCMMogCsMVlWbHeQuT\n", - "mD3/yuQp5lDob+9AYNdyDEIT/fV+2vxg/LuQxTIX08ne1pWMu28zMsHEcHxols+2LTEYzIWCi8BU\n", - "K3ZtJRE3rAjZxLOQ4w3m2m/D157HitClmlKcP9jJchoyWV95Jy2gAAABu0Ga70moQWyZTAgj//61\n", - "KoAGg+KazAhO48Rk+mELCfGa3jedcL7j4i4wMKqReszSNQj5h17BpSVMT9hX+zPhBrSs6Vj7HyaE\n", - "qm6lvw7kPbwwNhW67XEllpB7/AB7Dtmc/Lsrl2N4BzMZzIFVEJCqVkWDwHz0DCyLCsDRdQx8uGEg\n", - "Ikolt9wM9AgzvQ7TxR98jTrIYP8SP9CCVhDDASOwwiUKcH0pWRrgAYwjw8Gf7OlbogYj/no1BpFx\n", - "lYglvem+TH822s9SIsjJ3EA1IN/sTGSWgAXqwMREDl6rGx1E4un7krghrGWUm+/7j4jDoGqrYrQI\n", - "g7E+ktnqOLNELPNyQd8WQ/umSuXC1xL1umwA8X5+yPqMMHEIeQL1fzz/JWAXyMH93QMSzGumbhKw\n", - "Zwg0U+25Tvu4PnK5VQHbV0zvOU2Pj+MGf/nsDxqxrqZsD9S4YY9rcTfMxz/MkkzIgfRGQF/OgLHr\n", - "joIjF7P6XCeWe+XUgCwqZQG68PRNzfXkn+zUJpMMk0jjnoYnDkQ975Dz0Z65i4o7OdZtwLEOfaoE\n", - "pB0fo5td4PyA9vYIFlRo3xi7uvrQcih7/M7KbZFgAAAA9kGfDUUVLDv/ABlUeHLsmGHl+OQZEho1\n", - "hMDtEgrgr/N3AttUVM/7crMT5dwlm5uvzGVCn6w/p670sqgr5PJ6oiWC1npINQXp4CRzsctCmXzn\n", - "Ugai5K7NbwfaQcfbZKrjzT/10H2u4nhhcuuZyNqUHfbG94mETU3kKDy9A89Il0BA9I1A+R3yjNfc\n", - "+Nz5BwP3DN+ZYjka/GHLl0y68JgPyPoe9w8jyG5IXdu2vCa+LYvH9kU234z4psgT4qxlrdkhxxyP\n", - "UJXN8nPpx6cXDiQznv0L2owqy0csZbCzUw4CVJ98G+4T1R39bjI9WT0YHLigorskW6Eh4QAAAMYB\n", - "nyx0Q38AHP005LNxTWEpiZ1J9di26t3EruDGda0AVBouFN0G1ywEJMXJZuIMxrfHCac7PtwdnQsN\n", - "5ABPxruKApfvrd4v1WFO3Cl2Zd1SOG3/r1ORn6HwtueiSFcG0RNU2EL7iLFK3PfYpxwH299J2sER\n", - "9fENVpZ0Q3jjs6HsM0edV/QB07Ofn+R5vOS4TYLqhcaZAnuosw5RlS5g1Q8CuW9BZXMHWP4TGLry\n", - "nY5Y9ez3m8FrqVUEclyyvuywjGI3odTE+j8AAABPAZ8uakN/ABxO1hwOVZzdUT/5uAde8ARk7MlV\n", - "yrPTe1pSPQMTQCpdw5z/lBFmnGZwxWyqh+3IqkDkhpoxeW8ZCVdNB2x/1RnvvpDhcO3MwQAAAbZB\n", - "mzNJqEFsmUwII//+tSqABlJow5npTNmtYD16z8AGI7v0s/GnfyqOWOrIj7MzWLMA+5yFNFLu1hTu\n", - "dlbGlkD8jL3ONezhs0gurnHp2pFLsP3djo3BgKHcLr5q4kg5WMX28rT11jnIH4bHAuJDI0/Gub5+\n", - "542H8l9OurnbLu7ccDaau7k+AVcLYmIJfjhEaissSRpn2usY/14Z8WeJwbzUwclx5b0pufbMDj2m\n", - "E4jonmtfVQvsVKXSLVBGus9F0XUey7wsw1/Hxpa1Dj6X89JFMTZZDEgLc8SXNlb52uC+3SYuA3pO\n", - "yIZ3zYRDkwb5/sIpC9s/jtT+DR4JrFHAg/zOLQvdBHh2BZ/H88Qk1FOi1nkBwtogVwTsAvTRwaaM\n", - "L+Fy6Vw65xxtt2p06IrGo+vGB6Ev7rBsQ1lA5dJTwIES1/HSnI96cCqyJNRkq8io7XoKHq1jP8jJ\n", - "K8KCILcbnjTzWMILhY3EuZ8pRzEGblkg+ofcWDech+PkwDbk4flJvQ1eVGNBBbzkH58MbHNkp5C1\n", - "pRDfsnIb9VIwGZIgexRK5GP0EM8ZveKhcNpqg0C7EdFVGM7dDkwAAAFMQZ9RRRUsO/8AFKVU3AQX\n", - "TKYCKlUskM896ABcbpuBaq23+VbIBAleYM+Uh2fmC8hKxXufvA+Jyd8ERfcMKq2QBuOeaw8cG8nv\n", - "l00dW9FnZ2ewlISmCmZ99L0bw0GXPORXq89pSQ+5zLmGTJWLpbqXg/Gg/k26eFQ7yctp0OrjpANw\n", - "gpKfTmSwqfpdIyAO4i1HmWAczC/dxtyvK6EJns7ev/M+uhg/UBsLPdCc4ktjYaoFvgpYJl8v+SaB\n", - "iW6/qJFs8B7ABY+Xoa/3pJdDPx7Wo16RIr9F0VKx7gY2CroKhVZyesK3QK039pTJworswqeMoYtQ\n", - "SxUGWdIlnZAh/LxAqJSAgdbCea7vV7Jw7UJ3RZWLCaN03DO0g6FTEO0PNlB/y2w2d5hCS2yZtMLR\n", - "726poAjDu+5lgVHjodzIR1vHcKS57NpFhydymmBuCPgAAAD3AZ9wdEN/AB0F8+qoYAk/JkWPAABe\n", - "eS/K4R2z8W8rEZ4Es2dHO2B1xqZeWERk/2j9D35SD32hnizfkl5AQkKu7sKMRtxB0qUTg/5Ai8ci\n", - "ewPsEvh0cTnE+UnVVZQsy2FhpSkguxSgj2GzhV7H4B4oQdASRatW+4ge9XWWDwbNzKDfs2ikSZGn\n", - "ZK2J2cdk5ZNdF/NbhHS0c6vDp3S53pob/1OoP8UOX13YMuZJYtnSstfaINj9HWvrLOMusuMgy0ge\n", - "hr00WpqM4G4LNFMeeHMWs3VdDioqjp1BlI0pyKTUMl2eH+Urm0ENGx6u7gM90gDkOBdN7tgm4QAA\n", - "ASkBn3JqQ38AIb2gdaiDraOcmiaTmEkpG5LCwpD7mwoBhbYx9hK/huA/Rlz76MMOi96iXfBz3DSh\n", - "vG5XYVehGnggzBAkHfGgYDsO5F3SWLpvAiWuQYgw379rpdMwhqWoBgIHHe7UqoU3PiKCUX8CUwon\n", - "PUuq8JY4AYYztu7mmGelokJyoAJS97RU/X6H+RdsNNzitkC1d8I6jDPIy7qqN4tCnL3rY6Yesfv1\n", - "e8kTaN9S190RCoZyxCFd2JzsfgZhniY0nZmfUb/Ilr3HhSfAoNjT9YPJpZU0gCEN/XEjzBiwlPnv\n", - "oPqWZP16sXNdepP+5XR/WuewqnrAjpV8x4yn9rFVK/AamriL1xzzEUk66pD3JF3R2TNlp/oPgGf2\n", - "3Zht7rWDs3F41xpI2UAAAAHmQZt3SahBbJlMCCP//rUqgAZb4qaPpZ+oz92BGWQCHjdoHQHQRvdw\n", - "JuWMeCAf9SCNq3pRzo+QLWwm+zJnwkwndhEvWHQ/SujctvY5pe+lS1QEjQXzeizSF8k6tO14eAtl\n", - "F+Mync2FH/YIAKwBXgDqn6AXOHpWQcynHtaJryxWYm270/11pJpJLJP1UcyORiPI54DPlbzdu+l/\n", - "jiFd4hpdaoZTSIPUh6A6ClqPxEqekFrNjAxud2WiOSd4IE7Kaf//vpwZ0mh9bmck4Z3rAu3/6Cvy\n", - "KA3WyoqAFX4UT0ZjH4z6LrUYRBEZElMEZc4snCHRyZf+tjKnoDXWOrVFpzxu69dV7GJ+V1irRKox\n", - "Pd1LRXYUoYi+P14fumR2pYbtX+VBW+m+c7NAd8Z01d3TTKV7Mg7nTZdtCA/oFcETl7++5b2EIheP\n", - "k2Fg+5ToPyynpqzSsvv9vWMyfYTJnDg6PojbFsxSs0nRUvqnP5QCdr6QHBhWXFOG60F0RsLzEsNc\n", - "wpNcPfKeYjjdCfe8YUIVjq0PBSvcnC+B/ETQWaX7IFbWhPaknWILlx3KsiYwYSMVn5rwfQd4Jkdd\n", - "9H+fdht5f/EJHYCK5IGupAjPxHpu+QiB/iUSmCHkkTiMqsG8twzlljjsl22n8veAAAABCEGflUUV\n", - "LDv/ABi1iDy3ZgloGXmZPsuhVsylb+qqNi7GSIfQ+OHuoRwObuWCiDJsleSNbQz9VgmS3f493Q1l\n", - "fk0LSjQ0QBKQCe3UmCkV8vYYHcKN9CZn1L0i/3IstLHQcy91VMXucG0IQjYMvd5K4nw1TsRQ+zNt\n", - "c33OM7wT4gTiFbFnfUP6sORkbyxKD8+9VWHRCKkGnoAnjqhwkHV3YzaNKz290rB0XwxFDvsi8iqf\n", - "z+DNrf49LxpvDCniJY8b921MDAhjoaXQisEELwuIkEG2MG16iA+xn4KZIc8cifkUnLKYTAHTEosc\n", - "/geFGHZmG9d/0Ad4ehB1+UFj3eeT8gc12jWX2ySdSQAAAUIBn7R0Q38AHbXz6qhgDdTYSzAi1h3K\n", - "16Xr3JTVUajJdHP4n1zwK/61yxZ9pP4QSRtJbkJZWH6vivN5vckWYfjVoaQoNcq3qWx+bI+OTtrh\n", - "UNznJnNVmMngQpK+748FuR69zyCunCVVntkmuIrtQvOCVbqBuRz5Qxvz7t49H+VL6IAp+Rh2gf74\n", - "0j/UPUfosZ/ElbvCMu7rvOP7cWI+JN6KUOE+/AXQCyHGSkSvvSc5FsX0fFal2fQXaEkH67EHfCc5\n", - "xhdseiByl+PiqAs8A9zuy4qmXDeeIj+3Yojnw30fZXbmjymzKitBenCylofDP0QjYedpgwNVFWxv\n", - "pKDrpf57i5C5JHBxrkMOZNs3TkoKjfQLvKDT/j1Fvw02tHitRU1MR1mnPja0zhtM0e5b68dpKMZ6\n", - "9AO+761c+Ba/40Js4HhAAAABBwGftmpDfwAiuitca5eBLMHeP5uZuF9cX0/VXhqHcuiBABGdnZlB\n", - "vvbdh+1A3f4uQyVZizhw70/9zDh2nx3tQGn11M/7g3e0ETDcFJMpuy3pyqZj8OhCsFXcJg/Dg2Ky\n", - "wNn+F0Nd65xqPmrT4IAWVNyWgNuyHhWrg80hH2qe3n3QFTH+AG0t1LUQWRwdt8cDbAi+8IGZZrTn\n", - "QzKAGB5g+jkMrZS2t5af/14Dikh/TUO9x6vp3udUZwfEqX9x43nyKd2KkcrjEt0VxTQ1LHt4TKTU\n", - "ov9g2wymXIrIg/m2cGScMEoY8xa4E2v0IBu8Siv364Oh7cF3cjWG+ZJkZ6xGCUsmpmsJt4n9AAAB\n", - "cEGbu0moQWyZTAgj//61KoAGC/pGgJ9CubE/Hy/U90CEEMEEbF2P5yKT5EQsPLolJYuDn1q5ANTN\n", - "SJwpmVcvZVK2Tco4v2Comd7hwZPuuXhX+lvh+l6ZtjrC3czf1ZVbdumb3r3D/ioYe7qcFNf7aS5r\n", - "2YnlPFx/ox3Po4uR9L227Pa5JPu/JVHojzbyIvC2hUPLYoK3yo8EFTOEx9VW2Kka/dDqBAClQEXM\n", - "coaHOVrqvWOBlx0SmrR2Fn5qD0ttjA+wKyG9Ww/+/fxdGsIy8lThxbGnpYEDoqIDxAPPdyC1j/7C\n", - "x1S6SZ6cX8TWD+edELbCVScHr4twowGayNRkN1sGJ3ChzFZqefnm592USWq1KVPalCkn+IgAbkI0\n", - "gf8crEnxuQcz5L3ov1loEzryk4ptgt40vN/cUUrwi49uNdXDzDlba6ntBbOYIPKYQqVbRsWX//V3\n", - "7VjjZzb0fU2VitbTbNlERmPP5obsCvIRmiOfAAAA7EGf2UUVLDv/ABgUnqVjfMZzbMROTbEr98Ov\n", - "G6hTv8LwbEOVBTuoZFwTL9eOUuW51yt7Pk5XoOwvCITHjPxM0+ACPLC5p8LXGPLXOMFwxyKNAOm2\n", - "+bVnL7eC/eonqWYHV7ElnGiaPE4DZvhksvIAUMvT1hgYsLWg5pHxPTMEf4vPc7k/U4gx+qn0dLIb\n", - "xLE6WPqhOli4SJOCHhekKlwgxlnM6S8wIxjTrZQVP6tyjUXc7nRDpn5+4xHTB5JTQd/Y+v5uYYim\n", - "vSxL9Lp9+sJa/YqUqQ0UFcQR3Tlp/PCrTJ5gUcQmlTDSjEV8pdpwAAABAgGf+HRDfwAhujWPq7Ze\n", - "gCJPvLBRhSSbcG6El3BFXKqbl3V6+XLJCsWmxwO7Xskzh85D3/GGBbxCjXU3okqTeEYfyjkOl+SH\n", - "4VGFs6uGeBXI6FuyUdCktochZVIQW+D6bukSQtQ9xBoZWqRH4hlWFBiT6bV+GQGerlgKyeaNsqD5\n", - "s+IDfM/wce0dikHUV0++Nr2rHe3jcRRrSy2FHjFSMdnyldmaj1iFauYYGv6d3l/8LPJtc5g5u4Q0\n", - "WerxF6DQAN+WlQUAod5dWuqnUKOySujKDQh4Sh1bNoaribkhCngsbjiJUpnyDzJfWcRyF47YB87L\n", - "Omkfy8ijCTvweGsJYAgScQAAAQUBn/pqQ38AIbo1j6vClDiF6mIvKX7IDWIXdy1QyeJm7hwAhKrN\n", - "5ZQTH6lrtJ9D3xtslHyvy2ywnd5a5/owLJHRc2EtkPadJ8Uji+G9O7CT6ooBM3rAgAWaKgWADHof\n", - "Rk55HzZ+V8DMw4S4pnRLudTRFnX1DyLXHV3VXMnhAeP+ewFDtdkUHGMhcSI0U8KajX0wWNdBGeGb\n", - "D8Ns9BH8mxfhSu/SqyYkA2AIdaTRVyL0w7XOVFH3DXljVqrcwMdXPvGgiBcw6chMaLbepo7nSmh1\n", - "vAbwAQYruBhNTN0eawky0jofbme4HocI40c1sz31wjy2n2/uelK4XikXYFYmVtl4Kdutz8YAAAGb\n", - "QZv9SahBbJlMFEwR//61KoAGK1mYIKmbNbOjB+hVE+vOJ4Z3vMpGSn/PYftL6FXoKU3FZYLBaEus\n", - "cDU8hX8r/T4sCEjN2tKC+to/+IoDOzT/F3qpjao2Qnfg6SHJn87cmSTE3IR8bzvTmr+Ye4Ac/+hl\n", - "xYNmjmRG01XaPV08JLNnbV2zuL5cn/7CsR7I4pKAadGKE6UheVLfqn0i791ThTaaO2OCRjsSWF8e\n", - "1o7SXLcWHdmh1WCFSlfjet1S/FkIphxf8M1ZQjLPF96/W7wlOpiP6jEis8o6251YpmdqxS3VSmv/\n", - "s9Bv3ISLvkMspiZj+iQwr28MINay/7syEY2A7ZiKqNUJX069yti8CuYwd1gGvQZSlufV+auVaTNU\n", - "xocXs0XuFW0e/AWENf2i3yxrLFTHW9CCBeoKH21CafAHq6hi+H/e9DkZU77nSidgvmP6DIx/XjI4\n", - "Sp9anaBxYwcylzQtEH2XN+nrwpDPp45KYG9LI0xieadJ2QOTHIvADfNhP/PY2gqE0NQ2qkvQc0a7\n", - "Xw6JCi5LfZz745MNAAAA8QGeHGpDfwAdo0DVwAgarNdw1dyEo22Z+2voCmn3MepWOJpNH9uE22Fc\n", - "UAf4fo25DS3VGYdH0kZ3bYGxdzd+R7awrh1yiW2ItRU9+fbZ+7eJ43X/1GQK2tLeuYX+rXNnNYVn\n", - "3JiyKGKiuk48G4gEpBGTo6LBxeBZg0OXhUHfR3yB3h9X56ir+g4EbNusZoLNQh23BaGzc9/s1PO9\n", - "1PPSEqrUiAosSTAygJNCJGqMs5yCqcS+EZopY3ntHhRp/rTMQhL4aAxAb8XQkEJtEmWrzD4p1eX6\n", - "QEZh/6hTVX/Gz191R2H/Dtkpg79J3GkssFm0vPkAAAH+QZoBSeEKUmUwII///rUqgAWb5x2D2a6r\n", - "t0Z9OpYFG2tABdnWLgsFoSkhKeOGdpZQLTxZJNtdR1o3VEUaCsJe7TDcWLiNBjbFk4iCHCNTwP1B\n", - "ET8aIdy/mqBaPrTdtuT/6FMRex7yXV0X/b0t3IdDKZDeFLpQzjHVkdbvbm3BNwCciVQUNcJ7Sjbw\n", - "T4hbhPp0oEDMMYqhG0FXqi8cqsDNhwZenV4L974lIjS1k1BRVCVuxIwrhHZ+ZNeKQOVccqtyU7fb\n", - "1nmmkdbnAEav9V5tnQTxoYHQvrZLL4f7C+LE0IOtSnKggNbex2Xp0FNi9T/+fjTgmF5bW9OJ+WCx\n", - "leyLvNiQF8k0bwSPMh7702+7OB9yXypsT0VFN+3fNlolLg4yJ7ye2ijeDcs0TyR0KI9OqHHwk9VT\n", - "lv0R4DjKMuNtxv3yyDdQ02ld84rRe/IbVoqtujoBlwArv27SRkTybmrwQddynU1vfFNgJ2tkTxsX\n", - "EuhAyTUDk1pdyrePvO3Kyjq07E+ZdqW1unVDCL0p2PAM0Bdj+ozOm4QJPGRq3YEQjJpnk1BNx6E0\n", - "yZMxRvkyW2tYZosgoDR8rW5jEN/sH3PsICgk/jLYhgpsvFfXxjf0NPxMCt81bgYfKxBAoUrGuF/8\n", - "Gb453zLMx96NgDfHj/3/yVULmADuEWX3e7X8vwCYAAAA+kGeP0U0TDv/ABZZvB5hJcOQiwautVBH\n", - "s3zn8C7pn/fvWkU93yxomewKAdw+9VXghKzj8nMy4EQ6n26QhvOvN3ZOGl4wrl9GlrTzWwgssqXz\n", - "oLBd9XVA4LrC7D/kDb3CEAYvcHCWxuhsk3WHFeLlRhwB95RghbDR4boSp+CQz3CY9L8bxC9Ohf/r\n", - "dy9+xoLX1H7kyaZJ3YehTdM+5Wu6Hpc4XocPo/ogFns0WlfgVPekkiZdh228q3p+OFEAyCsprsbc\n", - "bh4x6zwYau0C11ECccZga0PS18ku4j08dAfMYirHksImmVD9Aw8yto6D9YLwntF8IaA+FPG9VagA\n", - "AAB+AZ5edEN/AB8T3aVQEVcYwT0kXXzzDP4yP2lC7bONTcb6acU9HQ87UdrkSLI4+OHKFlU0EAFz\n", - "P/GPhcZ5NOIVfnz6vsVd3DH3XZLg43PF1cMypwOcG8sbzfthjMA4FQSgVvJe40X2MhECJet9t2G/\n", - "XdWa+YBzkUuLdbRPBeGTAAABAAGeQGpDfwAfDtYNiNYeWLJ1JGi8AHLac8oZrJR5tDRFy80bn36g\n", - "01RfxVuWBDFeUQUU4VHoswV2zHbq6MzAloc0SM3f88f/qXApn5tj32GTO8MmdjG+5h2BlZLr7lVk\n", - "BcTdEueULRCVgGF4dFB9PX4Y3jYyGQfKH/BWnAEfbs4hEQ8ebrGB8mSRpcKz5q1oNG7pkp8qNfsq\n", - "nkhG1h5qVJ826dklpNvhQDQdQnVi0zusZWH7g9GItx1/0euTzo8U/z7D4DrbASMUmgB0DC8TSqJd\n", - "xZ+UMAYbubxMdW+iPv2N1tIKXHdcOVBHhDDt1MeY4rBQavQwdjpFZBiUMt5ya+AAAAH0QZpFSahB\n", - "aJlMCCH//qpVAAyk+dgiPwCMdRFSufgoxGSIR+/0rSe9Cp9hy8WpEfkfjpu1RSHWd3zlulcFC+Nh\n", - "XPR//hjTft5KlTxkfWUjrzSX8Q8sCzZTRHqzVvb/rscPsXHQf0E6taB/yJXWDm9ZR5fbjX3mwQRc\n", - "72p/7Nk/lJUO//4LM1qLgtlckFFvGA4aviZYHpBb9w1OJg/Jqwvkkixar7ua0LNG3ane8+4yu/5g\n", - "n8krsqxREhrpsaI39b317zkKj6KVaeKiNvQ1KBsts5QsX+yTO1tzmbv5PRxGS8tz2hKf4zB8fbWM\n", - "XhqB6Gi2mMVEo6jXnv5vErjT3e551EcovqLpcSnuFBTI4jT6V7ZqZq5zqsmn23ZqFTbBnXJfy5qg\n", - "Xc1RIbUSG7SAPcicWIbuNtZ4GQS+WKAEZUxr++6VPQD3gpW4BeKCxEy910wCA11VXaqCgcSgS5FA\n", - "dwACIPfrp0NhEyPCvA4qNFC9NitDM1I8HthEGAjfRL6imFuJfW4+Sk08ZcO8JNBK0/bkkNG7XFo7\n", - "Hs15nZek/o+FGsRiwki6FYqc1HBc8skTelrrFiYgicL9M/ehriAlP3GGSVQdD58oSyTAbR/XOwHh\n", - "/k7736bu5rnUg2SpAi/FdrWUFq0zx+C7UUDgbK+SgABs/nsA2PEAAAE5QZ5jRREsO/8AGLWLoCDh\n", - "swFlTBepUmfLHY9h6nZJebQZXCAk5QrW0LEqJOc6Tf3RfmBa+BH+trXpxDsoWsYBGGxFB6vHNSw7\n", - "QTuHxSINvJ7kINONdsnA7unyZfe+/dUQpBab4cd9DfyyBJrHeEf61R0Nfn0RkLu3bt6BWIYQlYtM\n", - "K9Nfs/vIPwJSfjpXcON5DPtNNDffXZk4RydlgN+S/E7EUmDtA6DaeTT9v6cz5zUd9DSGZ32drbmv\n", - "ejyP/MmN69TJZPy1fo/BndGgtSNNbFsKVeTDjxqdcz9cfjIrJ3P86/aSSTu++gY85cN7L+QFkn5k\n", - "/lX20+90kKxSs6X+x+u7me+jslyG1ZQaBGKwDx+RwViiPwDARZocg2yGxzRByDsEM59E93SHlUl9\n", - "GT+PqBiUfn848MoVbAAAAOoBnoJ0Q38AIb2gG1yg4IVhU+P1ifa//N+gWvitYvaP0DtmWpAevRlW\n", - "lsu4BAkHfcTJNIE2cW9WUS6DTP7xEfhthE/Au7/XTkrYH5bPnHuWMD+L4E2Ys7TDv/WnXsb8WMjs\n", - "GVKLefmxcZqtW10iMABVusPZiYCVoxR1g16JAWeZ7iIjTKxZ0g1yWUY7SYbSh6LLTrvWvhE7lU5U\n", - "CdpswEmIpPdhoFfYojayY1ypJuWbbU1PB5nvwD9t85tVUeFQcQm5aN4kQawNooLXHpvRUW63Gqd8\n", - "iY0WiZheEXu2JHmP8XM7t/dfyrk3Fx0AAAEdAZ6EakN/ACK9oCBuM4cceanCEPXT9ZV29ukUDhUK\n", - "Q43qY97tIKPQ4ZLk+xSOgxBfQxL7yIrZscfkKmKCSoYxQfZ+tSzvOZ1GhW2ifFuVzAIEg7+77ixc\n", - "Kx//CGLPLPJ464HVUHGkhcx37PQ+kbQrXlUbN3cWUp0Qf4LtEibFhZ+LpSZJ4udEDKi6Q/S18Psl\n", - "/qmdcccWROb1W4f/Xy9V+lMS0Du/XhxzsIhWccm/rlAZXG9J5NMLdRfS734QHwqLqFpe0KPTU/Mz\n", - "iY1ev2MPDzHxs95uiDK6gRc1gvD7TgXhVki57ReTigwP0Vcnsm9mMNHj3Nt6/RMhlMwCLQhy6qqL\n", - "YC7Z58RnNbEutfWZAa9Y2SYIcplB+x/e/c7TAAABlkGaiUmoQWyZTAh3//6plgAykehDX8oAigHL\n", - "uS7e5BiYpAhLP0Zp72qQ9WFfih2hD6ViubvwAAAy+5vuYY1yi1tJuPfBi/DL0xvClymIwqUp5EK2\n", - "pijOf291KPaqRN5kbJjB/2wfKr1+XMiKLX6DysREeFfQlDwQLBucvt+vNOXQokOSOb4yTYfCyIZ/\n", - "GHqmX89FI8GoC7SVJ8dqrGOCOpcjHfvSY2QsrqBh9dhAV5Sl9v/BQKeopbgb9Qoepn/uEMh2fyEW\n", - "JmX+JgRFJalJclAgIlVBNaF+FoinY0YPKhqMcuoH+rtaEk2LTWu4NHdn9ysTAkHlBR2G+58hU289\n", - "8X49s9CJy7d2oeKmsapTwnIxxJ2LNCm+TxMniHit0ZHqI5VMxQ+5ZJ2tPHM7/cT3gdae3yVR8+YM\n", - "/KU5H6oISvxSd8TybIcXMyYVHn6O+gwy4SKx3AkMYLFpKRIO1eI3ZmEPll+L/2Ahp3aDBQxulIlY\n", - "Qc1v4+BSAHSjYxY/VpZwrkFkWgmuXijX9pnceU+eCQb0BkKKYYEAAAE7QZ6nRRUsO/8AGBMrmVrk\n", - "4p6lyIABr6JcUvWGXYV0DKg9NQWqfn9mmEaqxk2L7hAoVLAefd2AT3uOnaK6MhbcdSJ0jbOgAdky\n", - "1NCtoTFYEK1L3oNAlJW78V3WE6NttmJ67HTQFhc7jbPt6n2fAdknrF4tehh2ttPPRj0ZMNDck2O/\n", - "Og/0bAxzaaL7DSYz/qGCfH6ue/8E9mejEEqzP8HffVv8Obhn2u8eQxOotWj4hO+DblITeYVYJXny\n", - "h4Mo9PoOPQCtWY4pEEbVZmokYfc6NrhoTMJC8d+WVfQUp/9dQN2FtoGBhQPHEwvVbIcYhR7B4iO2\n", - "lHuM7fr8Nz2PLRQOuR4Lhle59+tgw9IpLSJGfVu5u0NIKILKM/viNoDYYuKxIDdR/J6apnFKAoah\n", - "uk9v6if+0v3ru/qsdmBBAAAA3wGexnRDfwAivaAgbjOBvrizFgvOGL0/6vuXDLpZruFaiDwd2rdX\n", - "jHVzx9p+aFelpPZGVUpD9afD05Q+ygH73y/cGcCL/wq72iJds0hr5PUpNV/aSoB5zpjnS1krIC0g\n", - "xgvcsTNLJd1aFsq1w5umkQK05c9QgDPa1eUOrMmn+/YlpdytXE6u+4FAjIpYVgn74StUYfcT8IT8\n", - "SGX5Wru0UB/4BiwZwXDYz0r2pPySvTt1TUg57ubb0S/BqMvEVZ5rArNFw0GaRO5EmmTuHjFK31Ed\n", - "ZcrudMiOWUSCfSesj44AAADfAZ7IakN/AB++6KTdC0Gg2vR2G3QAHQcu6TnStota0MGq57eEms8e\n", - "GSZ8YTYymFLgl7YZGG1YXmh3orKEBl6b97W6tU9/+wsf9/cg00EpDLAMwmuhlqrl+tcaP161PaCT\n", - "db1JjfLZ6rQlIR/u8Lq+hDMPBrZgZ6lFmsHEDUzmL1vhrC/Eg5wjH+dLR3xJpn70Bg13IMQhP99X\n", - "7a/GD8u5DFMhlFEykeU8M0AF5LVwxauGljyJ2PG9wt/W7GNjLNgsX4aFTR897+cKWdUMsr13pC8x\n", - "KjWMpGHXcQ2lKSkGzAAAAR5BmspJqEFsmUwIb//+p4QAYn+ayCPJyJ7QOf/irXuB3I7yUvrv3Wd8\n", - "OLQaJBb/+EMR1r6SAeh0um3VtQPrwYoZU0zDlMzZlECRYSRYOAqgamI/sUVWVEYaYAVab8QpucQ/\n", - "sSTh0wVtYsFYYkt/gr7uhkEpx1NPSuJ9CqWeDhMsefol+oaGZkPTooDGiCB29X8Zubhk7s13xY5c\n", - "l2KWl6cdQs8QOBu4PKBLJa04v3ctO+FHUCNJTXN7J5YnaOHn+BLPFy7A6HoUxVmuK9kB/hB9j6ln\n", - "0nykP3r6vgXJiVxtga3Ek+Zj3edZUHSAUux6bbxkCgdvPWLgxmKM0iIQ0SZS+9McjsqW/5Kw1hL5\n", - "sobdDT0GsHJ+I+IDODn9/vmRAAAGqm1vb3YAAABsbXZoZAAAAAAAAAAAAAAAAAAAA+gAAB1MAAEA\n", - "AAEAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAA\n", - "AAAAAAAAAAAAAAAAAAAAAAAAAAIAAAXUdHJhawAAAFx0a2hkAAAAAwAAAAAAAAAAAAAAAQAAAAAA\n", - "AB1MAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAGw\n", - "AAABIAAAAAAAJGVkdHMAAAAcZWxzdAAAAAAAAAABAAAdTAAACAAAAQAAAAAFTG1kaWEAAAAgbWRo\n", - "ZAAAAAAAAAAAAAAAAAAAKAAAASwAVcQAAAAAAC1oZGxyAAAAAAAAAAB2aWRlAAAAAAAAAAAAAAAA\n", - "VmlkZW9IYW5kbGVyAAAABPdtaW5mAAAAFHZtaGQAAAABAAAAAAAAAAAAAAAkZGluZgAAABxkcmVm\n", - "AAAAAAAAAAEAAAAMdXJsIAAAAAEAAAS3c3RibAAAALNzdHNkAAAAAAAAAAEAAACjYXZjMQAAAAAA\n", - "AAABAAAAAAAAAAAAAAAAAAAAAAGwASAASAAAAEgAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAA\n", - "AAAAAAAAAAAAAAAAABj//wAAADFhdmNDAWQAFf/hABhnZAAVrNlBsJaEAAADAAQAAAMAUDxYtlgB\n", - "AAZo6+PLIsAAAAAcdXVpZGtoQPJfJE/FujmlG88DI/MAAAAAAAAAGHN0dHMAAAAAAAAAAQAAAEsA\n", - "AAQAAAAAFHN0c3MAAAAAAAAAAQAAAAEAAAJgY3R0cwAAAAAAAABKAAAAAQAACAAAAAABAAAUAAAA\n", - "AAEAAAgAAAAAAQAAAAAAAAABAAAEAAAAAAEAABAAAAAAAgAABAAAAAABAAAMAAAAAAEAAAQAAAAA\n", - "AQAAFAAAAAABAAAIAAAAAAEAAAAAAAAAAQAABAAAAAABAAAUAAAAAAEAAAgAAAAAAQAAAAAAAAAB\n", - "AAAEAAAAAAEAABQAAAAAAQAACAAAAAABAAAAAAAAAAEAAAQAAAAAAQAAFAAAAAABAAAIAAAAAAEA\n", - "AAAAAAAAAQAABAAAAAABAAAUAAAAAAEAAAgAAAAAAQAAAAAAAAABAAAEAAAAAAEAAAwAAAAAAQAA\n", - "BAAAAAABAAAUAAAAAAEAAAgAAAAAAQAAAAAAAAABAAAEAAAAAAEAABQAAAAAAQAACAAAAAABAAAA\n", - "AAAAAAEAAAQAAAAAAQAAFAAAAAABAAAIAAAAAAEAAAAAAAAAAQAABAAAAAABAAAUAAAAAAEAAAgA\n", - "AAAAAQAAAAAAAAABAAAEAAAAAAEAABQAAAAAAQAACAAAAAABAAAAAAAAAAEAAAQAAAAAAQAAFAAA\n", - "AAABAAAIAAAAAAEAAAAAAAAAAQAABAAAAAABAAAUAAAAAAEAAAgAAAAAAQAAAAAAAAABAAAEAAAA\n", - "AAEAAAwAAAAAAQAABAAAAAABAAAUAAAAAAEAAAgAAAAAAQAAAAAAAAABAAAEAAAAAAEAABQAAAAA\n", - "AQAACAAAAAABAAAAAAAAAAEAAAQAAAAAAQAAFAAAAAABAAAIAAAAAAEAAAAAAAAAAQAABAAAAAAB\n", - "AAAIAAAAABxzdHNjAAAAAAAAAAEAAAABAAAASwAAAAEAAAFAc3RzegAAAAAAAAAAAAAASwAABs8A\n", - "AAI/AAABMQAAAGEAAAD8AAABkwAAAMcAAAEbAAABNgAAALkAAAGdAAAA/QAAAMYAAADdAAABzAAA\n", - "AQEAAADcAAAARQAAAdsAAAE9AAAA0QAAAU4AAAIZAAABBwAAAV4AAAEDAAABXgAAAPMAAAD0AAAB\n", - "CQAAAaUAAACyAAABnQAAANsAAAB3AAAA8QAAAbQAAAFGAAAA+AAAAQ0AAAG7AAABKQAAAOMAAADp\n", - "AAABvwAAAPoAAADKAAAAUwAAAboAAAFQAAAA+wAAAS0AAAHqAAABDAAAAUYAAAELAAABdAAAAPAA\n", - "AAEGAAABCQAAAZ8AAAD1AAACAgAAAP4AAACCAAABBAAAAfgAAAE9AAAA7gAAASEAAAGaAAABPwAA\n", - "AOMAAADjAAABIgAAABRzdGNvAAAAAAAAAAEAAAAsAAAAYnVkdGEAAABabWV0YQAAAAAAAAAhaGRs\n", - "cgAAAAAAAAAAbWRpcmFwcGwAAAAAAAAAAAAAAAAtaWxzdAAAACWpdG9vAAAAHWRhdGEAAAABAAAA\n", - "AExhdmY1Ny44My4xMDA=\n", - "\"\u003e\n", - " Your browser does not support the video tag.\n", - "\u003c/video\u003e" - ], - "text/plain": [ - "\u003cIPython.core.display.HTML at 0x7f1286b190b8\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "import time\n", - "import traceback\n", - "import sys\n", - "\n", - "from matplotlib import pyplot as plt\n", - "from matplotlib import animation as anim\n", - "import numpy as np\n", - "from IPython import display\n", - "\n", - "\n", - "@tf.autograph.experimental.do_not_convert\n", - "def render(boards):\n", - " fig = plt.figure()\n", - "\n", - " ims = []\n", - " for b in boards:\n", - " im = plt.imshow(b, interpolation='none')\n", - " im.axes.get_xaxis().set_visible(False)\n", - " im.axes.get_yaxis().set_visible(False)\n", - " ims.append([im])\n", - "\n", - " try:\n", - " ani = anim.ArtistAnimation(\n", - " fig, ims, interval=100, blit=True, repeat_delay=5000)\n", - " plt.close()\n", - "\n", - " display.display(display.HTML(ani.to_html5_video()))\n", - " except RuntimeError:\n", - " print('Coult not render animation:')\n", - " traceback.print_exc()\n", - " return 1\n", - " return 0\n", - "\n", - "\n", - "def gol_episode(board):\n", - " new_board = tf.TensorArray(tf.int32, 0, dynamic_size=True)\n", - "\n", - " for i in tf.range(len(board)):\n", - " for j in tf.range(len(board[i])):\n", - " num_neighbors = tf.reduce_sum(\n", - " board[tf.maximum(i-1, 0):tf.minimum(i+2, len(board)),\n", - " tf.maximum(j-1, 0):tf.minimum(j+2, len(board[i]))]\n", - " ) - board[i][j]\n", - " \n", - " if num_neighbors == 2:\n", - " new_cell = board[i][j]\n", - " elif num_neighbors == 3:\n", - " new_cell = 1\n", - " else:\n", - " new_cell = 0\n", - " \n", - " new_board.append(new_cell)\n", - " final_board = new_board.stack()\n", - " final_board = tf.reshape(final_board, board.shape)\n", - " return final_board\n", - " \n", - "\n", - "@tf.function(experimental_autograph_options=(\n", - " tf.autograph.experimental.Feature.EQUALITY_OPERATORS,\n", - " tf.autograph.experimental.Feature.BUILTIN_FUNCTIONS,\n", - " tf.autograph.experimental.Feature.LISTS,\n", - " ))\n", - "def gol(initial_board):\n", - " board = initial_board\n", - " boards = tf.TensorArray(tf.int32, size=0, dynamic_size=True)\n", - "\n", - " i = 0\n", - " for i in tf.range(NUM_STEPS):\n", - " board = gol_episode(board)\n", - " boards.append(board)\n", - " boards = boards.stack()\n", - " tf.py_function(render, (boards,), (tf.int64,))\n", - " return i\n", - " \n", - "\n", - "# Gosper glider gun\n", - "# Adapted from http://www.cplusplus.com/forum/lounge/75168/\n", - "_ = 0\n", - "initial_board = tf.constant((\n", - " ( _,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_ ),\n", - " ( _,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,1,_,_,_,_,_,_,_,_,_,_,_,_ ),\n", - " ( _,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,1,_,1,_,_,_,_,_,_,_,_,_,_,_,_ ),\n", - " ( _,_,_,_,_,_,_,_,_,_,_,_,_,1,1,_,_,_,_,_,_,1,1,_,_,_,_,_,_,_,_,_,_,_,_,1,1,_ ),\n", - " ( _,_,_,_,_,_,_,_,_,_,_,_,1,_,_,_,1,_,_,_,_,1,1,_,_,_,_,_,_,_,_,_,_,_,_,1,1,_ ),\n", - " ( _,1,1,_,_,_,_,_,_,_,_,1,_,_,_,_,_,1,_,_,_,1,1,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_ ),\n", - " ( _,1,1,_,_,_,_,_,_,_,_,1,_,_,_,1,_,1,1,_,_,_,_,1,_,1,_,_,_,_,_,_,_,_,_,_,_,_ ),\n", - " ( _,_,_,_,_,_,_,_,_,_,_,1,_,_,_,_,_,1,_,_,_,_,_,_,_,1,_,_,_,_,_,_,_,_,_,_,_,_ ),\n", - " ( _,_,_,_,_,_,_,_,_,_,_,_,1,_,_,_,1,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_ ),\n", - " ( _,_,_,_,_,_,_,_,_,_,_,_,_,1,1,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_ ),\n", - " ( _,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_ ),\n", - " ( _,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_ ),\n", - "))\n", - "initial_board = tf.pad(initial_board, ((0, 10), (0, 5)))\n", - "\n", - "_ = gol(initial_board)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NgrSPCZxs3h" - }, - "source": [ - "#### Generated code" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hIGYeX0Cxs3i" - }, - "outputs": [], - "source": [ - "print(tf.autograph.to_code(gol.python_function))" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "last_runtime": { - "build_target": "", - "kind": "local" - }, - "name": "Simple algorithms using AutoGraph", - "provenance": [ - { - "file_id": "19q8KdVF8Cb_fDd13i-WDOG_6n_QGNW5-", - "timestamp": 1528465909719 - } - ], - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb b/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb deleted file mode 100644 index ed5b196e95a..00000000000 --- a/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb +++ /dev/null @@ -1,1933 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "g7nGs4mzVUHP" - }, - "source": [ - "# Experimental: TF AutoGraph\n", - "**TensorFlow Dev Summit, 2018.**\n", - "\n", - "This interactive notebook demonstrates **AutoGraph**, an experimental source-code transformation library to automatically convert Python, TensorFlow and NumPy code to TensorFlow graphs.\n", - "\n", - "**Note: this is pre-alpha software!** The notebook works best with Python 2, for now.\n", - "\n", - "\u003e ![alt text](https://lh3.googleusercontent.com/QOvy0clmg7siaVKzwmSPAjicWWNQ0OeyaB16plDjSJMf35WD3vLjF6mz4CGrhSHw60HnlZPJjkyDCBzw5XOI0oBGSewyYw=s688)\n", - "\n", - "### Table of Contents\n", - "1. _Write Eager code that is fast and scalable._\n", - "2. _Case study: complex control flow._\n", - "3. _Case study: training MNIST with Keras._\n", - "4. _Case study: building an RNN._" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "uFcgBENZqkB2" - }, - "outputs": [], - "source": [ - "# Install TensorFlow; note that Colab notebooks run remotely, on virtual\n", - "# instances provided by Google.\n", - "!pip install -U -q tf-nightly" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "Pa2qpEmoVOGe" - }, - "outputs": [], - "source": [ - "import os\n", - "import time\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow.contrib import autograph\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import six\n", - "\n", - "from google.colab import widgets" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZVKfj5ttVkqz" - }, - "source": [ - "# 1. Write Eager code that is fast and scalable\n", - "\n", - "TF.Eager gives you more flexibility while coding, but at the cost of losing the benefits of TensorFlow graphs. For example, Eager does not currently support distributed training, exporting models, and a variety of memory and computation optimizations.\n", - "\n", - "AutoGraph gives you the best of both worlds: you can write your code in an Eager style, and we will automatically transform it into the equivalent TF graph code. The graph code can be executed eagerly (as a single op), included as part of a larger graph, or exported." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "snaZRFdWd9ym" - }, - "source": [ - "For example, AutoGraph can convert a function like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "9__n8cSIeDnD" - }, - "outputs": [], - "source": [ - "def g(x):\n", - " if x \u003e 0:\n", - " x = x * x\n", - " else:\n", - " x = 0\n", - " return x" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gq0eQcuReHET" - }, - "source": [ - "... into a TF graph-building function:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 431 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 69, - "status": "ok", - "timestamp": 1531750911837, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "sELSn599ePUF", - "outputId": "2858bde5-ae05-4c32-be01-7770ac914f02" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "from __future__ import print_function\n", - "import tensorflow as tf\n", - "\n", - "def tf__g(x):\n", - " try:\n", - " with tf.name_scope('g'):\n", - "\n", - " def if_true():\n", - " with tf.name_scope('if_true'):\n", - " x_1, = x,\n", - " x_1 = x_1 * x_1\n", - " return x_1,\n", - "\n", - " def if_false():\n", - " with tf.name_scope('if_false'):\n", - " x_2, = x,\n", - " x_2 = 0\n", - " return x_2,\n", - " x = ag__.utils.run_cond(tf.greater(x, 0), if_true, if_false)\n", - " return x\n", - " except:\n", - " ag__.rewrite_graph_construction_error(ag_source_map__)\n", - "\n" - ] - } - ], - "source": [ - "print(autograph.to_code(g))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "j74n-8hEe6dk" - }, - "source": [ - "You can then use the converted function as you would any regular TF op -- you can pass `Tensor` arguments and it will return `Tensor`s:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 53 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 83, - "status": "ok", - "timestamp": 1531750911965, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "AkVaY0-dfEbH", - "outputId": "f04541ad-b1d3-4663-bf27-4d902648283d" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "g(9) = 81\n", - "tf_g(9) = 81\n" - ] - } - ], - "source": [ - "tf_g = autograph.to_graph(g)\n", - "\n", - "with tf.Graph().as_default(): \n", - "\n", - " g_ops = tf_g(tf.constant(9))\n", - "\n", - " with tf.Session() as sess:\n", - " tf_g_result = sess.run(g_ops)\n", - "\n", - " print('g(9) = %s' % g(9))\n", - " print('tf_g(9) = %s' % tf_g_result)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "trrHQBM1VnD0" - }, - "source": [ - "# 2. Case study: complex control flow\n", - "\n", - "Autograph can convert a large subset of the Python language into graph-equivalent code, and we're adding new supported language features all the time. In this section, we'll give you a taste of some of the functionality in AutoGraph.\n", - "AutoGraph will automatically convert most Python control flow statements into their graph equivalent.\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "u0YG3DPgZxoW" - }, - "source": [ - "We support common statements like `while`, `for`, `if`, `break`, `return` and more. You can even nest them as much as you like. Imagine trying to write the graph version of this code by hand:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 35 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 169, - "status": "ok", - "timestamp": 1531750912183, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "xJYDzOcrZ8pI", - "outputId": "f392b475-bf87-4d90-919d-44f895ee9fc7" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Sum of even numbers: 42\n" - ] - } - ], - "source": [ - "def sum_even(numbers):\n", - " s = 0\n", - " for n in numbers:\n", - " if n % 2 \u003e 0:\n", - " continue\n", - " s += n\n", - " return s\n", - "\n", - "\n", - "tf_sum_even = autograph.to_graph(sum_even)\n", - "\n", - "with tf.Graph().as_default(): \n", - " with tf.Session() as sess:\n", - " result = sess.run(tf_sum_even(tf.constant([10, 12, 15, 20])))\n", - "\n", - " print('Sum of even numbers: %s' % result)\n", - " \n", - "# Uncomment the line below to print the generated graph code\n", - "# print(autograph.to_code(sum_even))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_YXo4KOcbKrn" - }, - "source": [ - "Try replacing the `continue` in the above code with `break` -- Autograph supports that as well!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xHmC0rBIavW_" - }, - "source": [ - "The Python code above is much more readable than the matching graph code. Autograph takes care of tediously converting every piece of Python code into the matching TensorFlow graph version for you, so that you can quickly write maintainable code, but still benefit from the optimizations and deployment benefits of graphs." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UEHWGpBXbS7g" - }, - "source": [ - "Let's try some other useful Python constructs, like `print` and `assert`. We automatically convert Python `assert` statements into the equivalent `tf.Assert` code. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 53 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 56, - "status": "ok", - "timestamp": 1531750912292, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "qUU57xlEbauI", - "outputId": "c9cd536a-4a95-4eb0-98c0-aafce5d79580" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Got error message: assertion failed: [Do not pass zero!]\n", - "\t [[{{node f/Assert/Assert}} = Assert[T=[DT_STRING], summarize=3, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](f/NotEqual, f/Assert/Assert/data_0)]]\n" - ] - } - ], - "source": [ - "def f(x):\n", - " assert x != 0, 'Do not pass zero!'\n", - " return x * x\n", - "\n", - "tf_f = autograph.to_graph(f)\n", - "with tf.Graph().as_default(): \n", - " with tf.Session() as sess:\n", - " try:\n", - " print(sess.run(tf_f(tf.constant(0))))\n", - " except tf.errors.InvalidArgumentError as e:\n", - " print('Got error message: %s' % e.message)\n", - " \n", - "# Uncomment the line below to print the generated graph code\n", - "# print(autograph.to_code(f))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w5hBZaVJbck4" - }, - "source": [ - "You can also use `print` functions in-graph:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "6NdzRKLEboRv" - }, - "outputs": [], - "source": [ - "def print_sign(n):\n", - " if n \u003e= 0:\n", - " print(n, 'is positive!')\n", - " else:\n", - " print(n, 'is negative!')\n", - " return n\n", - "\n", - "\n", - "tf_print_sign = autograph.to_graph(print_sign)\n", - "with tf.Graph().as_default():\n", - " with tf.Session() as sess:\n", - " sess.run(tf_print_sign(tf.constant(1)))\n", - " \n", - "# Uncomment the line below to print the generated graph code\n", - "# print(autograph.to_code(print_sign))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9u_Z3i3AivLA" - }, - "source": [ - "Appending to lists also works, with a few modifications:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 35 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 148, - "status": "ok", - "timestamp": 1531750912595, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "MjhCQJVuiTNR", - "outputId": "96bf9131-c7c1-4359-ee82-9c38575e7ab4" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[0 1 2 3 4]\n" - ] - } - ], - "source": [ - "def f(n):\n", - " numbers = []\n", - " # We ask you to tell us about the element dtype.\n", - " autograph.set_element_type(numbers, tf.int32)\n", - " for i in range(n):\n", - " numbers.append(i)\n", - " return autograph.stack(numbers) # Stack the list so that it can be used as a Tensor\n", - "\n", - "\n", - "tf_f = autograph.to_graph(f)\n", - "with tf.Graph().as_default():\n", - " with tf.Session() as sess:\n", - " print(sess.run(tf_f(tf.constant(5))))\n", - " \n", - "# Uncomment the line below to print the generated graph code\n", - "# print(autograph.to_code(f))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UdG8ZFrkTAF2" - }, - "source": [ - "And all of these functionalities, and more, can be composed into more complicated code:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 53 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 555, - "status": "ok", - "timestamp": 1531750913176, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "DVs6wt8NKaGQ", - "outputId": "8729229c-4f08-4640-d3a1-0d3f9c697a87" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The prime numbers less than 50 are:\n", - "[ 2 3 5 7 11 13 17 19 23 29 31 37 41 43 47]\n" - ] - } - ], - "source": [ - "def print_primes(n):\n", - " \"\"\"Returns all the prime numbers less than n.\"\"\"\n", - " assert n \u003e 0\n", - " \n", - " primes = []\n", - " autograph.set_element_type(primes, tf.int32)\n", - " for i in range(2, n):\n", - " is_prime = True\n", - " for k in range(2, i):\n", - " if i % k == 0:\n", - " is_prime = False\n", - " break\n", - " if not is_prime:\n", - " continue\n", - " primes.append(i)\n", - " all_primes = autograph.stack(primes)\n", - "\n", - " print('The prime numbers less than', n, 'are:')\n", - " print(all_primes)\n", - " return tf.no_op()\n", - "\n", - " \n", - "tf_print_primes = autograph.to_graph(print_primes)\n", - "with tf.Graph().as_default(): \n", - " with tf.Session() as sess:\n", - " n = tf.constant(50)\n", - " sess.run(tf_print_primes(n))\n", - " \n", - "# Uncomment the line below to print the generated graph code\n", - "# print(autograph.to_code(print_primes))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JQ8kQT99VqDk" - }, - "source": [ - "# 3. Case study: training MNIST with Keras\n", - "\n", - "As we've seen, writing control flow in AutoGraph is easy. So running a training loop in graph should be easy as well!\n", - "\n", - "Here, we show an example of such a training loop for a simple Keras model that trains on MNIST." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "0CrtGWgwuLJr" - }, - "outputs": [], - "source": [ - "import gzip\n", - "import shutil\n", - "\n", - "from six.moves import urllib\n", - "\n", - "\n", - "def download(directory, filename):\n", - " filepath = os.path.join(directory, filename)\n", - " if tf.gfile.Exists(filepath):\n", - " return filepath\n", - " if not tf.gfile.Exists(directory):\n", - " tf.gfile.MakeDirs(directory)\n", - " url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'\n", - " zipped_filepath = filepath + '.gz'\n", - " print('Downloading %s to %s' % (url, zipped_filepath))\n", - " urllib.request.urlretrieve(url, zipped_filepath)\n", - " with gzip.open(zipped_filepath, 'rb') as f_in, open(filepath, 'wb') as f_out:\n", - " shutil.copyfileobj(f_in, f_out)\n", - " os.remove(zipped_filepath)\n", - " return filepath\n", - "\n", - "\n", - "def dataset(directory, images_file, labels_file):\n", - " images_file = download(directory, images_file)\n", - " labels_file = download(directory, labels_file)\n", - "\n", - " def decode_image(image):\n", - " # Normalize from [0, 255] to [0.0, 1.0]\n", - " image = tf.decode_raw(image, tf.uint8)\n", - " image = tf.cast(image, tf.float32)\n", - " image = tf.reshape(image, [784])\n", - " return image / 255.0\n", - "\n", - " def decode_label(label):\n", - " label = tf.decode_raw(label, tf.uint8)\n", - " label = tf.reshape(label, [])\n", - " return tf.to_int32(label)\n", - "\n", - " images = tf.data.FixedLengthRecordDataset(\n", - " images_file, 28 * 28, header_bytes=16).map(decode_image)\n", - " labels = tf.data.FixedLengthRecordDataset(\n", - " labels_file, 1, header_bytes=8).map(decode_label)\n", - " return tf.data.Dataset.zip((images, labels))\n", - "\n", - "\n", - "def mnist_train(directory):\n", - " return dataset(directory, 'train-images-idx3-ubyte',\n", - " 'train-labels-idx1-ubyte')\n", - "\n", - "def mnist_test(directory):\n", - " return dataset(directory, 't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2zu1U9Nqir6L" - }, - "source": [ - "First, we'll define a small three-layer neural network using the Keras API" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "x_MU13boiok2" - }, - "outputs": [], - "source": [ - "def mlp_model(input_shape):\n", - " model = tf.keras.Sequential((\n", - " tf.keras.layers.Dense(100, activation='relu', input_shape=input_shape),\n", - " tf.keras.layers.Dense(100, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax'),\n", - " ))\n", - " model.build()\n", - " return model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wuqg3H8mi0Xj" - }, - "source": [ - "Let's connect the model definition (here abbreviated as `m`) to a loss function, so that we can train our model." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "W51sfbONiz_5" - }, - "outputs": [], - "source": [ - "def predict(m, x, y):\n", - " y_p = m(x)\n", - " losses = tf.keras.losses.categorical_crossentropy(y, y_p)\n", - " l = tf.reduce_mean(losses)\n", - " accuracies = tf.keras.metrics.categorical_accuracy(y, y_p)\n", - " accuracy = tf.reduce_mean(accuracies)\n", - " return l, accuracy" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "035tNWQki9tr" - }, - "source": [ - "Now the final piece of the problem specification (before loading data, and clicking everything together) is backpropagating the loss through the model, and optimizing the weights using the gradient." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "CsAD0ajbi9iZ" - }, - "outputs": [], - "source": [ - "def fit(m, x, y, opt):\n", - " l, accuracy = predict(m, x, y)\n", - " opt.minimize(l)\n", - " return l, accuracy" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PcVRIacKjSwb" - }, - "source": [ - "These are some utility functions to download data and generate batches for training" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "RVw57HdTjPzi" - }, - "outputs": [], - "source": [ - "def setup_mnist_data(is_training, hp, batch_size):\n", - " if is_training:\n", - " ds = mnist_train('/tmp/autograph_mnist_data')\n", - " ds = ds.shuffle(batch_size * 10)\n", - " else:\n", - " ds = mnist_test('/tmp/autograph_mnist_data')\n", - " ds = ds.repeat()\n", - " ds = ds.batch(batch_size)\n", - " return ds\n", - "\n", - "def get_next_batch(ds):\n", - " itr = ds.make_one_shot_iterator()\n", - " image, label = itr.get_next()\n", - " x = tf.to_float(tf.reshape(image, (-1, 28 * 28)))\n", - " y = tf.one_hot(tf.squeeze(label), 10)\n", - " return x, y" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2zEJH5XNjgFz" - }, - "source": [ - "This function specifies the main training loop. We instantiate the model (using the code above), instantiate an optimizer (here we'll use SGD with momentum, nothing too fancy), and we'll instantiate some lists to keep track of training and test loss and accuracy over time.\n", - "\n", - "In the loop inside this function, we'll grab a batch of data, apply an update to the weights of our model to improve its performance, and then record its current training loss and accuracy. Every so often, we'll log some information about training as well." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "UUI0566FjZPx" - }, - "outputs": [], - "source": [ - "def train(train_ds, test_ds, hp):\n", - " m = mlp_model((28 * 28,))\n", - " opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n", - "\n", - " train_losses = []\n", - " autograph.set_element_type(train_losses, tf.float32)\n", - " test_losses = []\n", - " autograph.set_element_type(test_losses, tf.float32)\n", - " train_accuracies = []\n", - " autograph.set_element_type(train_accuracies, tf.float32)\n", - " test_accuracies = []\n", - " autograph.set_element_type(test_accuracies, tf.float32)\n", - "\n", - " i = 0\n", - " while i \u003c hp.max_steps:\n", - " train_x, train_y = get_next_batch(train_ds)\n", - " test_x, test_y = get_next_batch(test_ds)\n", - " step_train_loss, step_train_accuracy = fit(m, train_x, train_y, opt)\n", - " step_test_loss, step_test_accuracy = predict(m, test_x, test_y)\n", - " if i % (hp.max_steps // 10) == 0:\n", - " print('Step', i, 'train loss:', step_train_loss, 'test loss:',\n", - " step_test_loss, 'train accuracy:', step_train_accuracy,\n", - " 'test accuracy:', step_test_accuracy)\n", - " train_losses.append(step_train_loss)\n", - " test_losses.append(step_test_loss)\n", - " train_accuracies.append(step_train_accuracy)\n", - " test_accuracies.append(step_test_accuracy)\n", - " i += 1\n", - " return (autograph.stack(train_losses), autograph.stack(test_losses),\n", - " autograph.stack(train_accuracies),\n", - " autograph.stack(test_accuracies))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cYiUQ1ppkHzk" - }, - "source": [ - "Everything is ready to go, let's train the model and plot its performance!" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 585 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 17094, - "status": "ok", - "timestamp": 1531750930585, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "K1m8TwOKjdNd", - "outputId": "9f63da19-c3bf-498b-cf00-29090bf3b4f0" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEcCAYAAADUX4MJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXeAVNXd//++ZdrONsqyNBUECxZQRBHUoKLoE+lP0F+i\nxMT4tRDFWBKVxG7UJPaK8mBBE40lQBAVFAQE6bAU6WWBZXvf6bec3x+3zu7M7iw7w+4Onxd/MDO3\nnXtn9rzPp5zP4RhjDARBEATRAnx7N4AgCILoHJBgEARBEAlBgkEQBEEkBAkGQRAEkRAkGARBEERC\nkGAQBEEQCUGCQRDtzLp16zBq1KiE9n399dfxxz/+sc3nIYhjgQSDaHeuvPJKnHvuuaitrY36fMKE\nCTjzzDNRXFwMAHjooYdw5plnYtu2beY+hw8fxplnnmm+nzp1Kj7//HPz/cyZMzF69GgMHToUl19+\nOe677z4AwNixYzF06FAMHToUZ511FgYPHozzzz8fQ4cOxTvvvJPK240Jx3FJ2bc15yGI1iK2dwMI\nAgD69u2LhQsX4sYbbwQA7NmzB+FwOKoD5DgOubm5ePnllzF79uyoz2Mxd+5cLFiwAB988AH69u2L\nqqoqLF26FADw5ZdfmvtNnToVEydOxP/+7/+m4tYIIm0gC4PoEEyYMAFz584138+dOxeTJk1qst+k\nSZOwe/dubNiwocVzbt++HZdeein69u0LAOjWrRumTJkSc9+WCh68/vrruOeee/DHP/4RQ4cOxfjx\n41FYWIh33nkHI0eOxBVXXIEff/zR3L+8vBx33nknhg8fjmuuuQafffaZuS0cDuOhhx7CRRddhLFj\nx0ZZTMax06dPx4gRI3DVVVfhww8/bPFeY7F//35MnToVF154IcaNG2eKJQAsX74c1113HYYOHYpR\no0bhvffeAwDU1NTgjjvuwIUXXojhw4fjpptuOqZrE+kJCQbRIRgyZAj8fj8OHDgAVVXxzTffYPz4\n8U06crfbjTvuuAMvvvhiQuecN28eZs+eje3bt0NV1Ta1cdmyZZg0aRI2bNiAQYMG4Xe/+x0YY/jh\nhx8wbdo0PPLII+a+9913H3r37o2VK1filVdewYsvvog1a9YAAF577TUUFRVhyZIlmD17NubNm2ce\nxxjDHXfcgUGDBmHlypV4//33MWfOHKxatapVbZVlGXfeeScuu+wyrF69Gn/+85/xwAMPoLCwEADw\n5z//GU899RQ2bdqEL7/8EhdffDEA4L333kPPnj2xdu1a/Pjjj7j33nvb9MyI9IIEg+gwTJgwAfPm\nzcOqVatw6qmnokePHjH3u/7661FSUoIffvih2fONHz8ejzzyCFatWoWpU6di5MiRbYpPDBs2DCNH\njgTP87j22mtRU1OD2267DYIg4Oc//zmKi4vh8/lQUlKCzZs344EHHoDD4cCZZ56JKVOmYP78+QCA\nb775BnfeeSeysrKQn5+PqVOnmtfYunUramtrceedd0IQBPTt2xdTpkzBwoULW9XWgoICBAIB3Hbb\nbRBFERdffDGuuOIK0xXndDqxb98++Hw+ZGVlYdCgQQAAURRRUVGBoqIiCIKACy644JifF5F+kGAQ\nHYbx48fjyy+/xNy5czFhwoS4+zmdTkybNg2vvPJKi66ksWPH4t1338WGDRvwxBNP4NVXX231aN2g\nW7du5mu3240uXbqY8RO32w3GGPx+PyoqKpCTkwOPx2Pu37t3b5SXlwPQXE49e/aM2mZQXFyMsrIy\nXHTRRbjoootw4YUX4u2330Z1dXWr2lpeXo5evXpFfWZvw6uvvoply5bhyiuvxNSpU1FQUAAAuPXW\nW3HyySfjlltuwdVXX90uCQBEx4UEg+gw9O7dG3369MGKFSswZsyYZvedPHkyGhoa8O233yZ0bkEQ\ncM011+CMM87A3r17k9HcuPTo0QN1dXUIBALmZyUlJabFlJeXh5KSEnObkQUGAL169ULfvn2xbt06\nrFu3DuvXr8fGjRsxc+bMVrfBfg3jOkYbzjnnHLz55ptYvXo1Ro8ejT/84Q8AgIyMDDz44IP47rvv\nMHPmTLz//vumK40gSDCIDsUzzzyDDz74AG63u9n9BEHAXXfdhVmzZsXdZ+7cuVi+fDn8fj8YY1i+\nfDn279+PwYMHJ7vZUfTs2RPnn38+XnzxRUQiEezatQuff/45xo8fDwD4n//5H7z99tuor69HaWkp\nPvroI/PYwYMHIzMzE7NmzUI4HIaiKNi7d2+TwHhLDBkyBBkZGZg1axZkWcbatWuxbNkyjB07FpIk\nYcGCBfD5fBAEAV6vF4IgANDiNIcPHwagiYcgCOY2gqC0WqLdsafFnnTSSXG3NWbs2LF455130NDQ\nEHP/zMxMzJw5EwcOHICiKOjduzcef/xxDB06NOFrtAb7eV544QU89thjuOyyy5CTk4N77rkHI0aM\nAADcddddeOyxxzB69Gjk5+dj8uTJmDNnDgCA53nMnDkTzz33HEaPHg1JktC/f3/cc889rWqLw+HA\nW2+9hccffxxvv/02evbsib///e/o168fJEnC/Pnz8fTTT0NRFPTv3x/PP/88AKCwsBBPPvkkampq\nkJOTgxtvvBEXXnhhUp4P0fnhUrmAUmlpKf70pz+hsrISgiBgypQp+PWvfx21z7p16zBt2jSzo7j6\n6qsxbdq0VDWJIAiCOEZSamEIgoCHH34YgwYNgt/vx+TJk3HJJZdgwIABUfsNGzas1T5agiAI4viS\n0hhGXl6ema7n9XoxYMAAM0uDIAiC6Fwct6B3UVERdu3aFTPgWFBQgIkTJ+K2227Dvn37jleTCIIg\niFaQ0hiGgd/vx9SpUzFt2jRcddVVTbbxPA+Px4Ply5fjmWeewaJFi1LdJIIgCKKVpNzCkGUZ06dP\nx4QJE5qIBaC5qowJTqNGjYIkSU2qljbmOGgcQRAE0YiUp9XOmDEDAwcOxM033xxze2VlJbp37w5A\nK4sAALm5uc2ek+M4VFQ0NLvPiUJeXhY9Cx16Fhb0LCzoWVjk5WW16fiUCsbGjRuxYMECnH766Zg4\ncSI4jsO9996L4uJicByHG264AYsWLcLHH38MURThdrvx0ksvpbJJBEEQxDFyXGIYqYBGDBo0erKg\nZ2FBz8KCnoVFWy0MKg1CEARBJAQJBkEQBJEQJBgEQRBEQpBgEARBEAlBgkEQBEEkBAkGQRCEDZ/P\nh7lzPz+mY//0pz/A7/clvP+7776DTz75qOUdOwgkGARBEDYaGuoxd+5nMbepqtrssX//+8vwejNT\n0awOQadcQOmR715AqDIHd1w8CdleZ3s3hyCINGLmzNdRXHwUt9xyI4YNG44RIy7Be+/NQrdu3bFv\n3x58+OGnePjhB1BRUY5IJIwpU36JceMmAgCmTBmP2bM/RCAQwAMPTMe5556H7du3IC8vH8899wKc\nzvj91d69u/H8888hHA6jT58+ePjhx5CZmYnPPvsE8+f/B6Iool+//nj88b9i8+aNePXVF/RFuzi8\n8casqDXkU0WnFIzdVfsADvhg0SDcPTm1y20SBNF+fLp0H9bvatuSCILAQVGs+ckXntkD1185MO7+\nd955NwoLD+Ddd/8JANi8eSN27tyBDz/8FD179gQAzJjxGLKyshAOh/H//t+vMWrUlcjOzgZgrbpY\nVHQETzzxLB588M949NGHsWzZUowZc23c6z799OO4774HMWTIeZg9+2289947uPvu+/DPf36Azz9f\nAFEUTXfXJ598hPvvfwjnnDMYoVCoWSFKJp3aJVUfab5IIUEQRDI466yzTbEAgE8//Rd+85tf4fbb\nf4vy8nIUFR3Wt1jC1KtXbwwYoAnTGWecidLS4rjn9/t98Pt9GDLkPADAtddeh4KCzQCAgQNPw+OP\n/xmLF38NntfWVz/33CF49dUX8fnnn6ChoR48f3y68k5pYUQOnQnnKbvAvDXt3RSCIFLI9VcObNYa\nSIRklAZxu93m682bN2LTpg1455334XQ6cffdtyMSiTQ5xj7q53kh5j524lVp+sc/XkFBwSasXLkc\n77//f/joo89w002/wciRl2H16pW4/fbf4uWX38TJJ59yjHeXOJ3SwmD+HACA5CDBIAgiuWRkZCAQ\nCMTd7vf7kJWVBafTiUOHCvHTT9tj7teaMn1ebyays7OxdWsBAGDRoq9w3nlDAQBlZaU4//wLcOed\n0+H3+xAMBnD0aBFOPXUAbrzxZpxxxiAcPlyY+A22gU5pYagBrYCWH1Xt3BKCINKN7OwcnHvuENx8\n8/+H4cNHYsSIS6K2Dx8+EvPmfYHf/OZXOPnkU3DOOefatloxDC0gnTgzZjyO559/FuFwGL1798GM\nGY9BlmU8+eQj8Pv9ABhuuOFGeL2ZmDXrLWzatAGCIKBfv1Nx8cWXtHj+ZNApq9Xe/MQi+PsthuCQ\n8dpVT7X6i0knqBKnBT0LC3oWFvQsLE7IarXvPzoGQiQHTIigLlLf3s0hCII4IeiUgsFxHFyqppSV\nwep2bg1BEMSJQacUDADwIBsAUBGgOAZBEMTxoNMKRqagZUrtrYif20wQBEEkj04rGJeeoeVmbz96\npJ1bQhAEcWLQaQVjxOn9AAaEWOKVIQmCIIhjp9MKhsAL4FQnFK752ZMEQRCtoS3lzQHg008/Rjgc\njrnt7rtvx+7du4753O1NpxUMABCYE4yPQFaaLzlMEASRKM2VN0+Ezz77GOFwKIkt6jh0ypneBg7O\nBUkMoCEgoUuWq72bQxBEGtC4vPm0adPxr399iO+//xaSJONnP7sct9xyG0KhEB599CFUVJRDVVXc\nfPOtqK6uRGVlBe6++w7k5ubilVfeinudb7/9Bh999D4A4OKLL8Gdd94NVVXx3HNPYffunQA4XHfd\neFx//S9jljhvDzq1YLh4D4JcFap9fhIMgkhD/rPvS2wu39amcwg8B0W1Clqc3+NcTB44Nu7+jcub\nr1+/BkVFhzFr1hwwxvDgg/dhy5YC1NZWo3v3PPz97y8DAAIBPzIyvPj3vz/Ga6+9rZc7j01lZSVm\nznwd7733T2RmZuHee3+PlSuXIy8vHxUV5fjgg08AwCxnHqvEeXvQqV1SHkFbMKTCR7O9CYJIDevW\nrcX69etwyy034pZbbsThw4dQVHQYp546EBs2rMPMma9jy5YCZGR49SMY7GXOY7Fr108YOnQYsrNz\nwPM8rr76WhQUbEbv3n1QUlKMl19+HmvXrjbPGavEeXvQqS2MDIcHCAM17ai4BEGkjskDxzZrDSRC\nW2tJMcYwdepvMH78pCbbZs/+CKtXr8Lbb7+Oiy66GL/5za0JnzNWGb+srCy8//7HWLt2Nf7zn0+x\ndOm3ePjhR2OWOD9ea2DY6dQWRpZTU9/qABUWIwgiOTQubz58+MVYuPC/CAaDAIDKygrU1NSgsrIS\nLpcLY8Zci1/+8ibs2bNbP96rV5eNz1lnnYMtWzajvr4OiqLgu+8W4bzzhqKurhaqqmDUqCtw6613\nYu9e7ZyxSpy3B53awsh2e4EGoC7U/JdDEASRKI3Lm0+bNh2FhYW4447fAtAE5ZFHnkJR0RG88cYr\n4HkOoujAAw88DAAYP34iHnhgOrp3z2sS9DYqa3fr1h233/573H337QCAESMuxaWX/gz79u3FM888\nAcZUcByHO+64O26J8/agU5Y3B4CKigYs3rcK8w/PxymRS/Cnaye0d5PaBSrdbEHPwoKehQU9C4sT\nsry5QY+sLgCABpl+DARBEKmmUwtG36x8AECA0VKtBEEQqaZTC0ZXTxdAFRAWKK2WIAgi1XRqweA5\nHqKcBdXZAEVV2rs5BEEQaU2nFgwAcLMccLyKcl9tezeFIAgiren0gpHt1KL+e0vL2rklBEEQ6U2n\nF4zeOV0BAHtKy9u5JQRBEOlNSgWjtLQUv/71r/Hzn/8c48aNw5w5c2Lu9/TTT2PMmDGYMGECdu7c\n2apr9M/LAwAcqa5sc3sJgiCI+KR0prcgCHj44YcxaNAg+P1+TJ48GZdccgkGDBhg7rN8+XIcPnwY\nixcvxpYtW/DYY4/h008/TfgaeZm5AID6CNWTIgiCSCUptTDy8vIwaNAgAIDX68WAAQNQXh7tOlqy\nZAkmTpwIABgyZAgaGhpQWZm4tZDt1KbIB1U/1M45aZ0gCKJTcNxiGEVFRdi1axcGDx4c9Xl5eTl6\n9uxpvs/Pz0dZWeIBbCPozYQwGgJSchpLEARBNOG4FB/0+/2YPn06ZsyYAa/XG7UtVikro0BXcxg1\nUbqqGdoxjjCYwLe5Vkpn5ES853jQs7CgZ2FBzyI5pFwwZFnG9OnTMWHCBFx11VVNtufn56O0tNR8\nX1paih49erR4XnsxMSc8CDnC2H+oBrnuTl2At9VQYTULehYW9Cws6FlYdPjigzNmzMDAgQNx8803\nx9w+evRozJs3DwBQUFCA7OxsdO/evVXX8IpecI4I9hyhyXsEQRCpIqXD8Y0bN2LBggU4/fTTMXHi\nRHAch3vvvRfFxcXgOA433HADRo0aheXLl+Pqq6+Gx+PBs88+2+rr5GXmokauxNKNhzDh0n7IcDtS\ncDcEQRAnNikVjAsuuCCheRWPPvpom66T49LMLFUIo6o+TIJBEASRAjr9TG/AypSCI4x6f6R9G0MQ\nBJGmpIVgZOlzMThHBHX+cDu3hiAIIj1JC8EwLAzOEUIdWRgEQRApIS0EIy+jGwCA9/hR5yPBIAiC\nSAVpIRi9vb3AgQOfW47qAOVbEwRBpIK0EAy36AIDA+8O4oC4or2bQxAEkZakhWAAwPCeFwAAgq7i\ndm4JQRBEepI2gvGL08YBAHjJ28KeBEEQxLGQNoKR4cgAwplgnNzeTSEIgkhL0kYwAIBXBTCeBIMg\nCCIVpJVgcMwB8ApUprZ3UwiCINKOtBIMXi+NFVFoISWCIIhkk16CwbSig2GFJu8RBEEkm7QSDEG3\nMMIK1ZMiCIJINmkmGJqFEZRC7dwSgiCI9COtBEPkNMHwR0gwCIIgkk2aCYYTABCUyCVFEASRbNJM\nMDQLI0AuKYIgiKSTVoLh5HXBkMnCIAiCSDZpJRgO3SUVIguDIAgi6aSVYLh4FwAgKJNgEARBJJv0\nEgzBAwAISMF2bglBEET6kVaC4RbcAICgTIJBEASRbNJLMETNwggq5JIiCIJINmklGB7dwggpZGEQ\nBEEkm7QSDJdDBFMEqiVFEASRAtJKMESBA5MdCKvkkiIIgkg2aSUYDpEHFAcijASDIAgi2aSVYIgC\nDyaLkFiEVt0jCIJIMmklGE6HACh6iXOavEcQBJFU0kowsjwOMFkvcS4F2rk1BEEQ6UV6CUaGA0zS\n6kn5JH87t4YgCCK9SDPBcAKyLhgRXzu3hiAIIr1IK8FwOwVwClkYBEEQqSCtBIPjOHjEDACAL0KC\nQRAEkUxSKhgzZszAyJEjMW7cuJjb161bh2HDhmHSpEmYNGkS3nzzzTZf0yt4AQANErmkCIIgkomY\nypNPnjwZU6dOxZ/+9Ke4+wwbNgwzZ85M2jWznF7UAqgPk2AQBEEkk5RaGMOGDUN2dnYqL9GEHHcW\nAKAu1HBcr0sQBJHutHsMo6CgABMnTsRtt92Gffv2tfl8mW43mCKgLkKCQRAEkUxS6pJqibPPPhvf\nf/89PB4Pli9fjt///vdYtGhRm87pdTvA6jNQI1aDMQaO45LUWoIgiBObdhUMr9drvh41ahSeeOIJ\n1NbWIjc3t8Vj8/KyYn/eNROsPANSRgMcWQxdPMfXJdYexHsWJyL0LCzoWVjQs0gOKRcMxljcbZWV\nlejevTsAYOvWrQCQkFgAQEVFbJcTUxSooQwIAHYeKcRpXU5tXYM7GXl5WXGfxYkGPQsLehYW9Cws\n2iqcKRWM+++/H2vXrkVtbS0uv/xy3H333ZAkCRzH4YYbbsCiRYvw8ccfQxRFuN1uvPTSS22+ZoZL\nBAtplktFsDLtBYMgCOJ4kVLBeOGFF5rdfuONN+LGG29M6jW9btGsJ0UFCAmCIJJHu2dJJZsMtwNg\n2m3JqtLOrSEIgkgf0k4wPG7RFAyFyQkdwxhrNtZCEARBpKFgeN0imNo6C+Optc/jlc1vp7JZBEEQ\nnZ52TatNBR6nCDBt7oWcoIVRFqhAWaAilc0iCILo9KSdhcHzHDwOLeitUAyDIAgiaaSdYABAhtsF\nAJDVxCwMgiAIomXSUjCy3JqFISVgYahMTXVzCIIg0oI0FQw3ACAsSy3uS24rgiCIxEhLwcj2aC6p\nkNSyYMiMBIMgCCIR0lIwMj2ahREhC4MgCCJpJCQYX331FXw+bQW7V155Bb/73e+wffv2lDasLeRk\naBZGRG456J1o6i1BEMSJTkKC8dZbbyEzMxNbt27FypUrMXHiRDz99NOpbtsxk5OhWxhK6ywMCoAT\nBEHEJyHBEEVtft+qVaswZcoUjBs3DuFwOKUNawtZXieYykFKIK3WHsMg9xRBEER8EhIMjuPw3//+\nFwsXLsSIESMAAFICAeX2IlMvQJhIaRC7SCgUACcIgohLQoLxl7/8Bd988w2mTJmCk046CYWFhRg+\nfHiq23bMeN0ioPIJCYA9hqGQS4ogCCIuCdWSGjp0KN58803zfb9+/fDII4+krFFtxevRLIxEBIMs\nDIIgiMRIyMJ47rnn0NDQAFmW8atf/QrnnXce5s+fn+q2HTNupwAwHmoiFoZKMQyCIIhESEgwfvzx\nR2RlZWHlypXIz8/HokWL8O6776a6bccMx3HgwENFAhYGIwuDIAgiEVo1cW/9+vW4+uqrkZ+fD47j\nUtWmpMBDAEPLMQl7gUKyMAiCIOKTkGB069YNf/nLX/DVV1/hkksugSzLUJSO3bkKnADGqVBbWEkv\n2sKgoDdBEEQ8EhKMF154AQMHDsRLL72EnJwclJaW4re//W2q29YmBE4AOBWhcPPCJlPQmyAIIiES\nEoyuXbvipptugtfrxb59+9CzZ09Mnjw51W1rEyIvgOMZGgLNTzC0i0SiS7oSBEGciCSUVrtt2zZM\nnz4dTqcTjDHIsozXXnsNZ599dqrbd8w4BAcAoMYfQn5Xb9z9yMIgCIJIjIQE469//SueeeYZc5b3\nmjVr8NRTT+GTTz5JaePaglMQAQWo9Yea3U+xTdxLJA2XIAjiRCUhl1QwGDTFAgAuvvhiBIPBlDUq\nGTj1+ld1gebbGT0Pg4LeBEEQ8UhIMDweD9asWWO+X7duHTweT8oalQzcorZMa12gJQvDFsOgUucE\nQRBxScglNWPGDNxzzz1wOvW1siUJr776akob1lY8DkMwAs3uFx3DIAuDIAgiHgkJxuDBg7F48WIc\nPHgQjDH0798fY8aMwbJly1LcvGOniycbqAXqwr5m91No4h5BEERCJCQYAOBwOHD66aeb71kLE+La\nm64ZWQCA+hYEQ6bSIARBEAlxzGt6d/TSINkuTTBCavMuKYWKDxIEQSREsxbGvn374m6TE1gvuz3J\ncmYCACJoIYZBFgZBEERCNCsYt912W9xtLpcr6Y1JJlkOTTBkNJ8lJavWyoEU9CYIgohPs4KxdOnS\n49WOpJPp1GZ3K0LzpUHCSsR8TRYGQRBEfI45htHRyXRogsGEMBRVhSSrqGloKh4h2fqMYhgEQRDx\nSVvBEHkRHBPBCRJCEQUvf7YF97+xCrW+aNE4UFpjviYLgyAIIj5pKxgAIDAR4BWEIwp2HtKEoSEg\nRe3jC1ulQ+yLKREEQRDRpFQwZsyYgZEjR2LcuHFx93n66acxZswYTJgwATt37kzq9QXOAfAqghHL\nchD4RunAvLUtKDcfICcIgjiRSalgTJ48GbNnz467ffny5Th8+DAWL16MJ598Eo899lhSry9wIjhB\nwaqtJRC6lsB5xjqE5UjUPpxoWRUkGARBEPFJqWAMGzYM2dnZcbcvWbIEEydOBAAMGTIEDQ0NqKys\nTNr1Rc4B8Aq+WXcYzoFbIORUY1/DXgDAtxuO4I7nlwG8AjXsBgAE5I5dgZcgCKI9adcYRnl5OXr2\n7Gm+z8/PR1lZWdLO7+Qd4HgVgFXGpCpUDQD4+Lu9iMgqwMuA5AIYcNRXgtpwXdKuTxAEkU60q2DE\nqkeVzJIjDl5bdc8epygLldgupoLjGZgigmMOVIdq8OdVf03a9QmCINKJhIsPpoL8/HyUlpaa70tL\nS9GjR4+Ejs3Ly2pxnwyXG5AAzmW5mioj5fDk8HCduR5SST/tQ0UAVN6Uz0TO3ZHobO1NJfQsLOhZ\nWNCzSA4pF4zmqtqOHj0a//znP/Hzn/8cBQUFyM7ORvfu3RM6b0VFQ4v78EzQ/s+w9q0JV2PJzjXg\ns6vgyq7S2qiKYLyVbpvIuTsKeXlZnaq9qYSehQU9Cwt6FhZtFc6UCsb999+PtWvXora2Fpdffjnu\nvvtuSJIEjuNwww03YNSoUVi+fDmuvvpqeDwePPvss0m9vkvQFlHiXFYBQgaGf+76PHpHRQD42HWk\n/FIAHtENnkvrKSsEQRAtklLBeOGFF1rc59FHH03Z9Z2GYDi1dFk1kBVlbRgwVYh5fGWwGk+u+Qem\nnD4el/UZEXMfgiCIE4W0Hja7BC3obcQwlOr82DvGEYxSfxkUpqDEX56S9hEEQXQm0lswRK0EO+fU\nBaO2B/p5Tmu6oyqALxze5OMGyQ8ACMvNV7wlCII4EUhrwXCLRgxDn8EtO9DV0TQLi6k8uIYe6J99\nSlSswhfRlncNKyQYbUVVGRSV1hshiM5MmguGbmHoAW0mO83MqShUHqrKIPA8VKaamV0+3cIIkWC0\nmfvfWIXpr6xs72YQBNEG2nUeRqrxOJzWG1XQXE9cDMFgAhTGIOjbVKZC4ARTMMjCaDt1/kjLOxEE\n0aFJawvD47CWkeUUTTyatTB0wTDWxfBFdAsjRgxj/9E6vDF3G8IRWkODIIgTg7QWjAyn23zNq5p4\n8FxTo4rZXFKATTBMC6Pp6PivH27Ext0VWLmtpMk2giCIdCStBcMreszXAtMD4LEsDMZDsVkYsmpY\nGC0HvRWFArkEQZwYpLVguEXLwjAEg48VtlG1x8A3dklJ2gzx5oLe8QufEARBpBdpLRgem2A4OD2e\nwZresjHDQ/F7AAAgAElEQVTTW9HDEXuO1EBSZYQULR1XVmUoKsUqCII4sTlhBEPUBSNm0FsXkcMl\nWsziixV74dfjFwaUKUUQxIlOWguGyFvuJ5cuGDFjGLpLqs6nVazNyXKgIRItGDQXgyCIE520Fgw7\nTkGzNjjEF4yIpEUkvB6hiYURK7WWIAjiROKEEQw3r1sYMQoNMt3qYLpwhCXZzJDyihkArBTbpscm\nvakEQRAdkhNGMFy6haEqMZaA1YUCTNsWliWz8ODJ2X0BANWhmqhDhG7FcA9bBL+anmuAl9UEUOtL\nvlXV3IJaBEF0bE4YwRD1dNqv1xxtutEUDO3/kCybFsXJWbEFw3HqVnA8Q6G0LUUtbl8efnsN7nt9\nVdLPS3pBEJ2XE0YwOF63LNQYt8yiLYyILCMoaym1fTJ7AQCqQ7XRh4S1SYFBRks/tgaVFIMgOi1p\nLxhy2ckAgGxeXys8xjwMQBcTm2AoqgwA6JGhHdfYwjAEI8Dqk9zi9IZcUgTReUl7wegdvgjBddeg\nb9cu+icxYhgGuphEFBmyPtvbI7qR5chsIhjGefxqHVRG5UGawy4SKukFQXRa0rq8OQA89KsLUFUf\nQqhRVVnGAK6RdjDdwpAUGbKqvRZ5EV3dXXDUVwyVqeYCS5ygWSAyIjjqK8VJWb1TfCfHj2RbAXY3\nlEqKQRCdlrS3MFxOAb27e8Hb7jS48UqENl7VdGfdwuA4hoisTeITOAFO5oXMFDToqbbaBtl8uat6\nT0ra3l4kO85gPx25pAii85L2gmHA280JxQmoMYwr3cIApyKiaIIg8gJ27NUC4Ha3FCfIYLIIDjzW\nlW5Kq44w2Sup2q2KjmxgbNxdgcNllMRAEPE4cQSDbyZ2YWAExDkGSRcMgRPBIlqA2xAMSVYAQQYL\nZyCf749ifymK/aUpaXd7kGy3kV1LO2qWlKyoeGPuNjz+3vr2bgpBdFjSPoZhIMQQjOCmK6PTbG0W\nRlh3SfEcb2ZEVYdqMe+HA/jvqoPwXKRAVQR4oQXTG5cS6cwku1O3n491UBND6aDtIoiOxIljYTSO\ncAOAHO2aMkqDgGOIyDI4cJAkBiZpa2k0RHz476pCK36hiDAeoWwrf76zeg8KKran4jaOC8mPYXR8\nl1Q6uRQJIlWcOIIRw8JwORvVlTItDIaIIkHkRS27Sq8/JTNNKIwMKaaI4Fj0sq4A8HrB/2HWtjnJ\nvoXjht0llYyOVO0EQe9kx20IIh05cQQjhoXhaSIYepYUr0JSZIi8gFBENoVE1ifzRVkYrKmFYWDM\nFu9s2AUjGa6aqLTajioYHbRdBNGROHEEI4aF4XE1CuHYLAxZVSBwAgJh2YxzGIFwu4VxqFSLXZhi\nYqM23DkLE9o1QlHa3pGyTpAlRfNDCKJlTmjBcDtjC4bgrYfMFPgDCv46ZyMYixYMiFpAHLIT/oBm\nWRgzw+2zvjutYERZGG331US5pFrRMW/YVY7f/W0pSqpSn1BAFgZBtMyJIxgxYt4ZrmiXlOrPQaYj\nE3yXMiiiD4oSXbAwohoWhiYYTHaYLimj9lRYiZjn++7QcqhMBWMMSzcVobQ6kNR7ShX2zlNOhoVx\njC6pd7/aCcaAZZuL29yGlkimhaGoKpZtPor6QKTlnQmiE3HCCEastFp3Y5eUKmJkrwut92YV29gW\nBlMcZmaVYWGEbHGLXTV7sbN6Dw6U1OOjxXvwl1lrk3ErKSdeDONQaQO+WL6/1aNxtY1ZUrES3Foi\nLDWNKTVHMgXjhy0lmLNoN96c23kz5QgiFieMYHAxg95Np6FkOb3WG0MwDFFQNaHgTJeUPehtWBjR\niw4V+0oRCGnbVCh4o2A2NpVvPfYbOQ7YO3hFsVxSywuOYuHqQyipap2ldKwuqWPtwn/YWow7X1iO\nzXsrEj4mmS6pitogAOBAMVUyJtKLE0YwYloYjbOkAHgcGdYb1TiGB2OApGdCGYLBZKdtlT5NFEK6\nYAicdu4Sf5lZH5f31mFH9W7M3v5RW28npcSzMCKyJh6hSNMAf3Mcq0vqWBXj2/VFAICVW0sSPiap\nMW/zp0ZxESK9OGEEI1bQWxCafpYheszXzL52hsrb0mptMQzd+vAFNVdUSNYEY8wpV0DkRZT4y6zl\nNljT69U0hBFppfsk1dhFQra/1q2NcOTY3T0dNbaczJnenP6F2+91b1EtZi/cYT7D9mDVthJ8vmx/\nu12f6Pyc0ILBxVgbwy4YUYstMR6SEfQ2XFK2eRiBiPaZYWFkODzIEbvgaH05Xvz3Fu043upoGWOo\nrg/h/jdW4c15HcvXbe/o7C4pST42wYhXSyoQkrBxd0XcyXxMH6G3NobBmQLdijYmUzBiXP/ZjzZh\n1bZSbNlXlbTrtJbZC3fiqzWH2u36RGIEQhLmfLML5bprsyNx4ghGgr1OhsMuGBxGnN0TYy48CVB5\nax6GKIGpHKAKtpRbXTD0oLdbcKG8SoLMJJiuCcHqaINyEAdLtMqoW/e3XycSi6gYRpSFob1ubUA5\nnkvqrXnb8cbcbVi/qxwAsPtwDZ58fz3qfNFxoFjC3hzHECNPWgxj/9E6LFwdv1OWlPa3JimFuGPz\n5epDWFZQjDf/s629m9KElAvGihUrcO211+Kaa67BO++802T73LlzMWLECEyaNAmTJk3C559/nuom\nIcMl4vSTcmP2LN6oGAaPrAwHHCIPxmwuKVECZAcAzoxzGGJiWBhu0Q2oAjieAZw+UrZZGIX1R9Bg\nS7sMyiGsOroWSowZ45v2lmHBmn1tueVWERXDUJq6pEKtzUCKCnpbr38q1Kr/Fldq8yx2FNagsLTB\nFNLGIYBV20pQVpOa1ORkdaJ//XCj+Zp10BgGTVLs2BhJMnX+jpeWndJqtaqq4qmnnsL777+PHj16\n4Be/+AVGjx6NAQMGRO133XXX4S9/+UsqmxLFq3+4DDzH4bPvm3bCjV1STgevWSeqzSXFK2BG0ULT\nwtA60bAew3AJLr04IbRSIrIzatGlN7bMxmnCRQC6AgDm7luIVcVrURWqwfgB10a16Z0d70LIqsGo\nwFPIznA1afPhsgYcrfBjxDk9W/8wYqDEmbgnHWMMI56FwXGa28b4LCJr5/WHpOgTcEBRhQ+zF+4E\nALz70JXNX9B0CSXeMaakWm2sU3aAvlpRGMSm+R5EB8HwnneAn0oTUmphbN26Faeccgr69OkDh8OB\n6667DkuWLGmy3/EuSGe6p2JYGA7eYb5mjIfLIUAUeIDxCMu64vNWQUJTMHQx8ellzr0OD5i+j2lZ\n6P93dWsl0asjWtqnxyWgMqi5pbZX7WzSJiFLG4nvKyuLeT+Pv7ces77c0bSjPUZYnIl78jHGMKLK\nm9teG9+DoUkRSXvhD+pJBbZzGKOu+NdQ8fyqt7GudJMVdNa3lQcq8fKmmagKNl6X3SIVy7J3UL1I\nyuz9YyEQknCwhFKNW4QzkiY6wq8lmpQKRllZGXr16mW+z8/PR3l5eZP9Fi9ejAkTJuCee+5BaWnq\nFiK6dvjJ+OVVp8XdfuGZPaLna6gcnKIuGCoPcCq6ZLnA8aqZHWVO3NNdSUY5kFxXDqDooqILhSEc\nkwZeBwAI6gKU5XGaIlIeqIzbvgOVzT8bo8NtK/FKg5hZUq2OYcQ+t5GIYFoY+nl9IQm+oCV+HGIn\nLdgpD1RgXVEBPtjxSZNtH+38DHtrD+CLvf+Ne/xx8+u34jLBsIxguPUpzC11NO219seTH2zAUx9s\nMOepELExfukdUC9S65JKRCGvvPJKjB07Fg6HA5988gkefPBBfPDBBy0el5eX1er2/P7686Pee3X3\njihw+Ntdl6Ffr2w4HQLcQgZCiuYr79Y1AxFJBSvlwfEq8rt5cIhXwQwxMFJleRV5eVnwq35wHIeT\n8/NNC8N0RelB7755eQAAf1hzXzkcAjiH9qwkVcL+0F5c1Oc88Hy0nleEapq97+wcD/K6eWNuKyyp\nxyNv/4iHfn0hzj61W7PPqaja+oP2ZrrNaxrfJifwLT7/3YeqwXEcTj+5C6oCVuefneMxjxV4DhIA\nt9uhfSZo9/vlj4fw5Y9W4Dgjw4luXa37inXtBsGyHhwO7TxOp4i8vCwonL4YliP+76akzpqh35rf\nViAkweMSY04MjXWuzCztee46VI1Fqw9h2i+GwCHGHreNu38+AGDBCxMSaosvEMEvH/kaYy/tj9sn\nDY66tj2dN7eLF12z3QmdM5mU12i/KyYIx/T32xaO9/XagsejeTk4jutw7U6pYPTs2RPFxVYdoLKy\nMvTo0SNqn5ycHPP19ddfj+effz6hc1dUtH3t5WBQG+EzBnTxiKir1UQii89BSAmAc4YQDkmaC0bl\nwXGAIOqdfyOXVEiSUFHRgEpfNbIdWSgr85kxDI5XwABwvHZsxMfAgYfC6YHysIQan2Wqv/jjLNx1\n3q0Y1PV0RGy1qUrrK5rct/0HVVJWDyGOu+H9/25HbUMYr/17M566dXizz6XaFliuqQmY1wyFdSuq\nPhTVjj1HarFs81H89ueDzM7vgVd/AKDFG2qqY5/P6GN9/jAqKhrQ0Cg7yiAQjKC21jrHrP9swbkD\numFAb+u3c7TassxM11lYRkVFAyKS9pyliBL3d2O/55Z+W2t+KkVOpgsOkcczH27EL0efhqsvPCnm\nvo3PtftgFQ4eqcG8lQcBAKf3ycawM7W/CZWxmNl8if7W9xdr1u2XKw/i9kmDo46zWyrl5Q1Qwslx\nXx4LtbUBVFQ4j9v18vKyktJfHC9CumtZVdWkt7utApRSl9S5556Lw4cP4+jRo4hEIli4cCFGjx4d\ntU9FhVW+YcmSJRg4cGAqmxTFWf20gPMVQ/tEfZ7j1DoizhWEUxS0CX66MDhdWmdkWg+qVXyQMYa6\ncD1yXTnarGjTwlCi/j941A+m8KaLKhRRcKAsOrW2Pqz9UHy2pV+DsERlR2E1bnluKXYVVpuf2V1S\nQTmIz/bMR0PEpzWTGXMaWk46jSo+aA96c34ArIlL6rl/bsKaHWUo2NfUnSYratxaUlYMI9ol1aQ9\nKovK1vrvqkL8dc7GqH0CsmUVMS76PEYFYaWZQEVr5mG8s2AH/vHxZmzao/12P16yF8s2H03o2K/W\nHDLFArC+j6/XHsKtf/selbq7Jl7cpzliVTMwsD/b9ophdAa2H6jCii2pL3bZHJwZw2jXZsQkpRaG\nIAh45JFHcMstt4Axhl/84hcYMGAAXn31VZx77rm44oor8OGHH2Lp0qUQRRE5OTl49tlnU9mkKAad\n0gUv/P4S5GZGj3Z6ZuZhn38XOF6By8EjIvOmMBRlaCPnxhaGwhT4JD9kpsDDe3GkrMF0WxmWhRHL\neG/hPrjPFgBBQddsF6rrw3ApQXDMiQGu83CArTNX9/NL1sjX7zqChogPWc5MfLxkLwDgw+WrAUcI\nkNxRncJXB7/DsqJVOOorwR+G3mF2ynwCQwSmMkCMACpvdtQH6gohnfEtxOL+CEeiXVqc2w+hSxlU\n9awm56ppCMfNkrJiGNp7o/RIYxSFtdjJ2Z+TKuiuD92JVhfSRLesPn7QO9EYhr0dWR4rQWLOot24\n/Pw+sQ5pFpdT+0I++16bgb1lfxVGX9A3KsgvySqcjpbTmuwDhsaps2Hbs23v9cuPpZjk8eLFT7VJ\ntj8b0rvd2mDGMIz/GcPhMh/65Hm1eGo7klLBAICf/exn+NnPfhb12fTp083X9913H+67775UNyMu\nXbKapqlOPP1qLN+5B1JJfzjO4iEKvDlBz8fpFpEhGODAmCYYxmh++x4/Nh/aAqGbkVarB70NS0MR\nwFTNwuia5UZ1fRicIIHJTuzcE4HrNGteh3FOJjnAOSSsKdmAq0+5XAsK8zL2ur+B53wguO7aqJG/\nsWTsoYYi/b0uGLYMjHjWhqKq8AxdCjXshqKeAwDYXa11aI7eBxEqvihqf9fZP4ITFBRHDgKITu39\nfN9cfW6J1pnaR/JxLQxBgpBbAaWqFwAOispa7OQCNsFo6FIAFA0BGKCoCoKKH+CA2kj8DB1FjR7R\nx3s2stx0XkpbaDwp0RBXvy3oH4ooCQmGZBOFmobo1R6jLYz2FYyOOHJujKqyFhMtUkajWmTrd5Vj\n5vyfcOXQPrhpzBnt0yadE2amd2vwONyI7DsfzJ8LABB5DpwjehKNkR2lveGhMsWsVMv02AVrlCUF\n3hb/UAWAV3WfPwNEOao2VUTVrlcf0gRDLusHMA4bywoAaB1K4zbZR+heUZuAaMRAmGlhcFh86HvM\nWPV0lLvLjk/Wrsm7QmZpEJG3OqzGabWGEMpoOtFob8Mu7Pb9BHDaeaJcUk2ypLR9nP23wTlgK4S8\nIwA0AWtRMGwuqbCnBJxX8+f7pID5B6jy4ZgrIwLRa3o3Z21INpEoScL6Jo1Fx7i0L2QXjMQypYx5\nLABQ2qiisN36SMYqiolSWh1oMgGtvQUrEaQ41u7xoHEtst1HagEA63Y2zTA93pBgxOGZ2y7GpMv6\n47STciEIPDhno1RA1TbiU3koULG7qCp6m/4/7w4AnArOFdRFhNMFQ4HLof3PcQxQHOYxRqmRyqDW\n8alBL5i/C474iiGpsjY/QrT9IYqRqFFkWLW2qUw1O2qe4zB//9eojzRgdfH6mPdeI1nxFKP4oGHp\nAEBIit2BSUwTTCvXniHMQlChgvPolpIhDkoEkZNWg8+pMMXM6PB4fd4J79XOoygsZicXDMtm525Y\nGMPyz9MeR14RGLRYjp36SHQQsSZUi/21hdFus2Y6NPszLqlsWTBueW4pNu6O/4feeIEqNY6FkQj2\nTs5+vLbNOsfxSiFmjGHGO2tw72sroz5XErTMUj0PISwp2FFYbT4P+/Xs4nu8MWuR6e+NZnUEVx4J\nRhx6ds3AuEv6g+c4OAQOnKsZwWA8GBSs31MStc2wNMT8w3CctFsbsVf1wvCztJRbjlchijAtBfsK\nfuV1flTUBrFgk7Z2BgtmQY1o5zOsBrMIIgBnv59QHCwy39sXclpyeAVUPSbCc0A3txbs31a5I+a9\n19kEw+ioa8OWOyckWokKhrABgE9uwM5DNXjqgw3aB7b28Rna8cYf5+6afVAzy+E6Y6M54jRGwcw+\nQx7QXVJNO5nfv7QCr3+h1dvx68Iw/lRtljznCAGMNRGMunC0W+qx1X/Di5vehF+2xUD0Symq0qRU\ni6SoAKdC7HkQxbVWTCTbGz/rZ86i3XG3Nb4v08I4BsGwWxGSrKLWlnUWbgcLI968oHirODYEIman\nXV4bxO/+9j1WbYsuUb+vqM7MBmsrc77Zjec/KcCan0qbtPd4WBjltUE88OYq7C2qjfrcKl4ZLWSJ\nJKykGhKMBBAEHkp1tG+eRQkGh7As4XBFbfQ22z5Cdy2LRqnqhT7dvRh0UncAgCgycE6tc2cRt+nG\nWrPzKB6cuRp8Rj2YIoCFMsxyJIYY2F1SQtcyfF8zD/uOan9MQZtgzNv/FeoytE6L5zm4BK1zO1h/\nOGo/QAtu7w/9ZL43Rlp1YWtkLuVr1XUDIQlPrPmH+Xm1VInVRwrM95zNAuIztOMNwbB3xEfFjTjS\nUGyN6owZ8oarq5kYhpGZZVgYOa5s/YLasQH9/pik3XNdIwvDiPUEFMuCUvXJbw+seBSvFcyK2l+S\nVYh99sJx8m6gj/accjKdZipvLLo1M+ehqUvKsDAsKy5Rl5TdXbZ43SHc9/oqFJZqAtkeWVLxKg/E\niv3sOVKLe15dif+sOAAAWLNd68SNcjAGz3y0sUmGXCzqfGHUNMRO0zbYul/77Rws1n4TgXB0okGq\nWbDqIKrrw5g5/6eoz824lv6TtwSj9dc4WuHD58v2J+07J8FIgByvE9LBcxAptGUBKZYYaAFs1YpV\nGBZGyAsW0YLqnD5/g6kC3E4BXpfWiQgOm2CE3QAz4h4qwCngPH6ogSyYbiwADSFdMMRGMQNexfwf\ntD84Y10OA5nXOlSO40x/v8pU7KiyRr+MMby48S1Uy5YLxReQsLNqD/bVaUFvpa4buIw6bDlUhLte\n/gE1YWt0dCS0HxsjX4PTrQm7BcRn1gK8bJbgsLuGKt3b8fGuL6yRp25lGfenKGqLo+KAHESGwwOR\n10rOc4IClVkuKTWoTfybtW0OygNNV+Lzyz4Yf6EqYwjIQURUCXtrD5hpuYDWkQi52vGcU3vGWR5H\nVPpxY5oXjNguqZYsjFjuGskmCkfKtOdbU6+10e5ikY9TDMEXjC0YscR/2wHNqv16zeGkXPve11fh\n/jdWNbuPEUMzEkKOt2Bk6ll2TZ6TEXMzBUP7P9GK23aemrMBX605hI27E199sjlIMBIgv2sGHvv1\nCPx53HXWh3pw+qW7LwWLeMA5w+Yo+srzTtb2YTxCWy+LPpkqwO0U4eK1Ea8gKmZ8hEU85nnBK4Ao\ngeMYWETrcAzrwxcOAWBaJwwgvGeotl1yIcOt/QgbWw6GEPE8h5AcAqf/+2zPfEiKBEVVsXD9XjMV\nlelVeOsCIawqXqt/xkOp1WapL9tfgHhwDv3adgsjsw6us9aYHWLjWIKTt7l09OM4dwDgFCgqMztk\nvksphPzCJtf0SwFkOrVAP8e0uJCiqKZgsGCmue+yoqYdyfd1/4F76BKAU6GqDA229hX7rJIskqxq\n7QIA2QEOgNspQpaZfm/MDPAbZGVo34nj1K1wnBztBmw82pbMVQ2tDn7TngoU7LXmuDw6ex2mv/JD\nk3uwJz3U+bRnaIjDdxssd+Xxqlbrj1P/K5aFITRKgEgWzcVBGmfpBW3ttT/LzXsrzPkxycSr/602\nFifr+7EGMEDiFgZjDDsLqxGKyKabraVabIlCgpEgp/TMQj+9pAegFSn87f+ciRyvE0q5NstX6KGN\njjJdthGlKkIN299rFoZT0Kf/Cyo4l80lZRYsVK2ihYZPX9+29WAZ+NwKCF21YoQs5IUayAQnSvC6\ntX2DSgheMQMPXHCXdgpO74Q5hpAShlzfBVJlTzRIPtSG6/HV6kOYu0brzOSyk6HWafdaFwiA5/TM\nrf2DwfzapMafiovMjlGp64bIviHmLRqussZZXHyGDypjqAnV4utCrQilGtKqAwucNafBsEw4UYLn\nwm8R5GrNUanrtAI4T9llrnpoEJACyNTXY+f0DDRZZQhKIf06lmB0cWnZb/b4i3Y9GRAjUBlQbwvy\n76zeY7tOSLMmAUCMgEErLaMyBklS4Th5FzwXLgbfpRTiSbugTXRUAV6G2L0YYs/oEbSsMESUCBwD\nCsB5a82OKmxLLFi3sxyvfmGtA19U4YM/pE0U3by3An94bSWq6kJRnZzhglIUFTUNYRSWWgJ4vGIY\ngTguqVgWRpMU1iS565uL/5hZemosC0M7rrIuiNe+2IYH3159zG2org/h9f9sQ3mj0vweV+xZDcbz\nYY0sjETXhdm6vwr/+KQgytWVLCEmwThGpk0cjMv0yT0PTbgaALS1LwBkuT3RO0vWXA+m8nA7BTh0\nwdiozrO5pBpZGKaLS/9MtzCWFhwC77UCf0xyaIFiQTLrKIXkENyiG909WoBb5YxKu/ofsSzqa3oA\nYSWMo5V+0zLQhEs7T00giCMVesC6vqs1GVGQo1xwSnUvXOwZq20zBEO3FOQqK/7DGPCZrQhgeMcI\nvb36WuiCas1X0QnxNXonZ/3oxV4HIHTT4kKSIiGiSqZggIngeAWKwkz3GwtYgmEUioyVVszxShML\nY8XR1WY6bnXIcsFxDq3NxmSqyrogxJ5aDSzXaQVw9CoE5/Fh9U+lZsaXdpNWR6ooKjaUbYHYrRTu\ns9eYHX04gUKSEUnFa19sQ70/ghVbiqMyoczzq6xJxlQy/NmL1x/B9gPNL/zVGgujsbulpa4x0Tkw\nDXHcYgDA8wycK2B20AFbuRQjHmTEkuz9bWFpfdRaNi3xr+/2YtOeCnzwTXTyQ7y4XGMLsLUxjCPl\n2mDHvjBbsgw3EoxjxO7yObVHHpyC5VLJz8mE02E9WjPrBzBdUkbgWoEEzhnSOmhVNK0IIbcSYm+t\nhARTBW3tCHswWD+nQ83U0nEVBzgOCEohHK30IygH4RHd5voeKq9nYhnrkSsOK4iuhBGKKFHBd6Md\nlfV+lNQYCxoJWhsBbU6JLhiGiDiYdi2tI2Xgs7UfrFw8AEqdJlyyKkfFBKDPPTHmsLgztG1yVU8z\n/lNcXY/NeysAm8Xi6H0QzgHbAFiikOnMwIotxZAlDuAVyKrNJSW5ENl9AQCY14o5D0VQwBhDQ0Tb\nluPMRnWoBosOfQ8AqI1YmVG8JwDnoDWQnFp5lkdmr2tyOk7P9LILPOeyRpolVQFsO2C5mzbzX6Cg\nfJs114VTwGXoqdVqdCVae1B5wY+F2HbAKBOjwtF/G/icCpTXBPHOAm2kaYxo2zoPIiwp+GTJXrz4\n6ZZmR66+QMQaVNiIlSUVq6wJn10F8DIW/FioXVeWwLm076W5GIO9w20sllHt6LYP7iErUMNrIh8M\nW22V4gh2MCzjr3M24tMYa+nEw6jj1bjN9vRi+/ca18JIUDBifSfJckOSYLSS+4ZOw+DuZ2NI3tnm\nZxzHobueqgoAORkZeHX6ZfjddYO0D6LmbAhwOQUInPUZ5wrYgujWr0LsZqXpCjxvncdmfXRvGA6A\n01JyAazacQRPfPkJwkoEHtENgRfgFJxgumWh8LovVhHNa+4trsK2I0Vw9tdcUizianQt/Yet8qb4\ncYJiWQL6viLTXW9iBJzHByGnGkpdNy1+oAuNpESQ49QzmcKZ2v2qIsJ6qrA3Wxe0iBuRQ/rzE2Ts\nKKwB74ox74FXTMEoLAri/a93aRYZr0BWmCnsTHFA1WNBxrViWxgyFJuF8asz/xcCJ2BbxU6tVlgk\nuryIkFWLqowt+sEx3B+6YHA2weAzGsx9V24rwcb9tnRovhabyreas/Yd/XbAfY42XyUsKVEj68bB\nUmPlQj6rBmLeUbjO2IgFPxaiqEJfo8V9bILhC0pRKx3a/eH7iuKnuG4J/gDPsG8BRzjKqoll4TRO\nGdUBXG0AACAASURBVC1R9sN15no4T92GuXrm1OJD38M95AfwXUpRVF/S5BwG9ooHDXql5IgiYfb2\nj7CzynIvylnac68QdkNVWVSBRimOBdMQiEB116ImmHhRQEMMGmui/bu0W2OqKRjG/CRtv+bSarcf\nrDLvO5Y4JGtOCwlGKxmQ2w+3D745yqIAgG4eSzCcghNOh2CO6KIsDKa5pK7rP8b8iBOU6DTdxqgC\nnCJvm59gCUbEGHTLxjbJHNn3d2mi5hE8UFy14Lx1OJS1WGuGKpjn++KHPVEdGot4rJnsvKKvMMgB\n4CAw2xyJRllhX63UgsOcI2LGIVRfLgDOtEIiTMKuI1r7QruG6seL5qi/78n6XIxAllXt1yitos+F\nUX1WlVoICvy6NXDoqCEO2tK4ZTU+lNbVWc9Hb0NIDmPzngrUhzXTfdzJ43CaMNw8X3lNED/sKAQA\ndPd0g4Nz4VBFDZYXFKNe1s7X3WG52VzQ3F1mMNyGZWFYLinnqdvhOtcKvBvZVgYO3mH63oXuWiE8\nMf8wnl/xEd7+cru5X60vjltEjD2qNrJyYsUwGGPYWbUHSw+vaLLtyffX4+G315hCYff1Hy5rwJqd\nxZAVFXW+6LphRdDmyAi5ZVGxhFgWRuNRcbWi/Zb4HMv6MuYNuU4rwMvbXjMXHWuMPYXYsDCe+Wou\nNpVvxZtb3zW3CbL2vQWcxXhj0cqoCgZGsLix66vSXwf3OatxOGdhzGvHwhDoxnEa+3Owx3tMC6PR\n/cQTjK37K/Hiv7fgrXnabyOWNiQrz4EEI0nYLQxXIzGxp+ACgNspINPpxbhTr7HtE7+sF1MFnHda\nd2t+Aq+YgdeI3tf07qIFcjlRMjvy+Qsi+GFrMer8WkfrPtsWuJOtWeWcIJsBdqUhFyycYbuWNlHN\neJ/tdWvioQuJ0T7thQAmi5pLqpGYGBZGfTCA0toG/Tjt56fKvFkKJcRrI3g1mNVkAp8xW1wqOg1y\neV+z7XVhv3VP9mtyKg5X1gJMqzZsuOB2HKnAa//ZhrV7tNIjny85gp8O6B06r+CNudvMLK5sZxYU\nSQAnyPh+81EE9LIpl+Zei1OyTjKvA6Dp5E6j7YIE3h2AGrTWi+dt4mK4Ag3CShgRSdFcEJJ2T0Ju\nBUqFHdhSaQW/lx9dYQbW7fEd3tYOzuU3S9JogsFQK0V3tC99ugW/+9v3eH3L/+GLfV/CF4m2vCrr\nAnCduwKf7JoHILpU+vyCDfiw5GX88V+f497XV2LpJqtqL6enR/NZtVHHxJrpbc1jYfj20DL4mR4r\nsv1d5Dpzo46xx5PsRFkYumCUBLV2uQUrAYVj1t/lT8HVUccZFkYTwQhoAwZFaJSFqBOQAiitDuDl\nz7Y0qTzcuMO3p2LbRbixBRavirNBlb6WixGzUE2LxroeBb07GD0yojOoAFvVyUZi4HZqP9QshxWI\nRTMWxjmn5GHcyH4xXVL6GkxwGX8IeufPFB4Ah/e+2hXl+zeQy0+K7pANwSg7Obo9hktK/+PPyXAB\nqhgd9LYJourPBp/hM+cqGGJiWBhLtxy2soxsM+IlFobY8wB8rBo8eM2NZVoY+ig9s0ZbA9yfY2uf\njJqAXnZEFwxr4SpN1DgmAuDQv4fW4dQFtM66rF7vcGSneQ+cIOvxnDB4JsIjujWxE2R4XCIkpnVA\nua4c3HXerdrt81qasxEEz/QPtGacCzJ4XeiU2vwm3wOXUQehizbvxUiPNmJKOV5nk98ObDXMdsmr\n4ehVCPf5S+GyDQbswuUe8gOcp23W2uVxQMg/hK/r5mBd6Sb4ghIqa4PmHAiDssbzVBwR8J4ANlZr\nMZoo102u5i6K9NkA99Al+HanlW4tqppA8hn15loqQPTI+l+7Psc3hUvMDlroWoJ5+79CNbRsMuO7\nNObG2JHi1AWzz9iuD4Qwc/1HELtrLqwcwVZpmbfNE8poiBKMQCQIX8TfpIJybSh+ActN5Vvxxx8e\nxxNz/4ut+6uwYqt2TaP/53kOBeXb8O/d88BYdLmbYEiGrKhY81NplHvqQHE99uhuP9mW1LC35gD+\ntesLKKrSRIhCakBzC9s+TkahTOA4VKs9UejltToDI2X27P5dMbBvDnqc0RubGw6a242smkynJRjN\nuaT69chFZobDCi732W8GkcMh7Ufn5LSAM++t1zp42/mYypkZXAAwKOdsbLIF2DleAThj/oXuRtM7\nJtcZG8Fk69ouhwCmCOAzfHD029Gk7XLJAAg51RC66nMXDDExr2V3ZfFR2xwn70GY5SLT6YWf8dGC\nxqngM+t0V5XDFkuR8e/lO+E8xSbM9vsSFPD6SHLMsH74oASmxcIEW0kW0Qjm6+a/IwQnMsBxnGY1\neRR4XDwa5AjAAR6HC27RBQ4c6vijcJ3lg1KjLw5Wn49Tc06x2m6Ufom4oFTnm+nQAIOYd9Rsg1qb\nBzAtxhKWFGRlOBBq7F4y53hYHQDnkMA5bEvaNrJ0hBxNELxuh/m9fLjzU8gNXyOy7zwA0Vl9ZYEK\nDMjthyUbi7CjsNoUQkCb7GkXDHsRTk6UEcksQllNAHm5HiicdhznDkRlIBmdl0/yY1WxJkKXsf+n\nP/9GI2H9O919uBZ1IT+YIkAqOg3OU3aZKdONsXf8FeEybD9guf/s7WC8BMY4qHXdIORWwu+zLKvl\ngU/x1cpaBNddA3tc0V5aRlEVCLainKuOavOVkLcfqOhhDgzNET8PzNr+IQDg2n5XRnXitb4IPl6y\nF9/bLDQAeHrOBvO1PWj+8uaZAICzup0Bvy1lPBiWsZZ9As9QCcr+oWAKoNb2SNpERLIwkkSvTEsw\njHkLToeAGTddgFN7Wu6qHrkec0SQZROMU/O7xD232+HS4iH2UiM5WkZMSP9bzuP6AbILYs9CcKIU\n1YkbqasGP+3X/fpGp+uIxO3EAd3Npb93OQXzOMOtkuG0pQ0bM9sNq8ZMCbbiL11zdNcR49Et22Va\nLwAQVANwCa6oY3i3D5zbD45X9Vnv0efjmri/LOuI4xWAieA5Dh6XA1BEy5VmCIbkjC5HwqmAIwIH\n00bITI8PFVXVotrvB1M5uByi+T0D2sREo2MNBUS49ew0TpDBGWm0iojIgcHWvBxeMcUrvPsCaAkA\nDoSVMMKSApcTUUJg3C/QNO6hfWi4xmIXRfR6RG1yKLSOn8+sg5CnB9xtAfsVR3/E90dW4p/f7sHm\nvZVR82nW7i1sNAksenTr8zE8/PYarN5eAlUfwXO8ioKqzeY+isowf//X+GjnZ+ZnZufJR1sNxiDg\nHx9vRqWvQRN3Q0SKLMuoMliF8oAW77ALRkiJFpWGUNDswFVeAhQRql9LwqhnVrzEcInxuRXgnNbz\ntE84/XL38qhzu0Xtd8tn1kHIPwSfXpGBqQyctxZlmZZwheSQaWnxORX4v6VrmoiF7SmAzy2HxJp6\nCurD9VHJD5V1ISjQ3gsDNsF1+iYA8et6tRayMJJEpiP2WtoA4BasDvW5O6zO236M8WMDADWUEeXj\ndotO8BzX1D0BmCN4t+CCw5cPKfuwlhoatM7NAtlQ6rtAyNbjA4rW0ZnFEXseMjv6Jp2u2SjtmHNP\n7YbdpdGjwAyHC8afUeM2mi4po3JvVg04XgWTNZdZbpYLDbZ5CRKLWM/COMbbAOdA3dWhOKKuY3eN\n9evRBQfqbBaP7rrjVDdEgYPLoWea6Z1uhOkjcVs8B7wCzhEGxwG8rHWuiqRtq/b74OQVPQlBvy9b\n7MCwJP535CB4jHsQ5P+/vTMPr6LK8/631rvl3pt9D1khJEAgAcIWdmQTJGkWhRe1WxRFWxRwQXrU\nntHWmcYHp/vpx8exfbrtxWec0R573ufFcXoGX0VfEW1axBZwWFQSIAkhZM9dquq8f5yqU1X3XiAo\niCT1+QdS66lTt36/81vO77AEAKLfR+tJBu9qpsrOsHZCPvbM/dEQrVnl1kujdKRDSG4znxdm3EPr\nCYJP6mT3giLFVVY2FF6SR2Lns316jS2rUmjsPonG7pMAP4+6Hy0Wxnst/w/DBLNEjnWftX0ff3EK\nsHhc32l/E5BmAVE3FFXBn/Q0ZQNjBMydJ2BPrx21pXwrxDz28T3/AABQT1SiZkQGDKupX6XfEYnI\nAE+gIIrX/u8x1FXlQOOiIKoIEnXrx/YDoHEeA9eIv4BEJYQ+ngPALP0PAH86/SamFVYj2RXEZ1+0\nQ+bNb1guPISmaAaAcmiEwD3qA1jzqvpVY+kADa5yWhur/8OF9gcWopCKPgNUEWJmE5TmQrx3oBJ1\nVTnskLZQO3r7zcFmbyix1ZVojs7XwbEwLiO3jVqDtRWr4rbHBcF1rBaGWzSPCf91KrIj49jfXsk+\n4rahj84FgYdILItBxQp8y7lMoFpiD8aIlQniGEFg3Ke2IhMen/3H55XNQGJlfoZtH2LuJeV8gQ6t\nlSkgr0sCHxNjMRWsOXrlPb0xbTfdVUYW1aIpZbhr2Sh2bcMlBU2EKPCQJUGvEkyPjyIEmXMB4G0T\nEg1hzOmCRInw7F6Gu08U4z8do8+mVRSy4CqnB71pmyVb2zneuk93BSoCzhquEd1S0HqDCB80srgM\nhUGVgtKWixJ3JbuX26uAEzSo58z3YMR2fB7pvAI+UZzLmHhpVSZfRj/F7tC/xD0zQxf41gWtzOvR\nfbGjfoAuGMbJ5sRHdg6bx0EAga4Zw+IaMCbVmQJeGHYQn4TMkf8XZ6gVEvmqki5Cxit488MT+JsX\n99K5SYrI+j6il+ePrSLASVHW7/2qPSHgz6cP4L/+3IifvXYAB0/YYz/tfZ147vVPcbYr3hrsV0K0\nbIulb901u2wWlpjRCDGtGWImtQI5bxd+9QYtxmik5Z/oakJbxKz91hEyFZoJGdBE0IHgKIzLyPis\ncZiSMyFuO88JCY6mglHk6Y/VI1mFvYhUyfzgPYbLRxMR+nQa1G5rtggVqqLAsTgGEB8TIYawAphA\nNeIVNoyPsTdo20w0ARyoHzys2T94aymUZdOG29P6mIURG7wVWLtjS4hYra04Yhan4t294EX6MYwp\nzqJ+Y0MR6rW4NJWn/SPy9HyjbDofhgS97WxCogroQlDTrS6j76Sig3pCgZBwohnn6odP8kLgBfMZ\nBMUsPBmbxaUrO7ZGivFcutDo1Vd31HoDccv9GnEKEvGYmT+CAneSns4c8qH/41n6OVTo+twSIEbg\nhh/jhRvYOQDAiQlcXEaJllilYH1mfaARbRyuX4ee09qjB2rbzNGwoZxCoEJ3Zv5UjEmnc23CSgTy\nyA/Z+ZEv9Tk4hsIQFBrEVU3XrFEdIOEETMM9Z9QzUyQ9ecFUQLTvRWaBRbWw7ZlslxMj0DQN3Rq1\n9JRmGqP6n45jaGmn76IzRli3dnfhz5+fscV8DPqVEBRVs/32OTFqm+BpxBUNSJi6SOn6NvT5jnQc\nx1fBnWwA0R1O3BeOhXENcb4qkxzHsUwpm8IAkOo2zUyfvu+Bm8ZhTd14WyE9A1HgIXNm2mZsKq/N\nOlHjLQwDluranYolefXmDo2H1y0mXLbS6zKVkUcWY6wZe1mT2O0CzyGpc7Rtn0s4v8Jgqxhqpjst\nJZd+ZC5RhiQKTFm6hlM3lhrlIQg8C9hTIURAhAibbMgEsqufCS0lIlF/uP48gr+DChNNQDDB+he8\nKwS/TGMsIi9C4kXwvk5zBr0S605T6WjW1l8iOEEDQNBJ6MhR6w3aLCoA4APt+r4AzeQCVSYun24p\nhj1A1A2tJ8DOccs8OCkMGR5oUaOWma4wdMGlfFUJF9HnlRgKw514ZUYIUXBiFEJvJpTTpbRqs369\n3sAh1nZDmRjtCGnU+gjIfrgFOsiJkAh4t+lKU9vyqJvUSEKwVihg82ki6O6L4Me/j587wgpYWtyB\nRBV0a5SwthhVEgAgrIXAwVSQysnhiJ4u0q8XxeftxxHiu6C05SB6YiQIAXojvTQOZ7lXibfc9nei\n2e4hJQRVJXHK2FYRQE5sDfZG+0BAQIjlW9Sv0xOhfcsSMAA6V0nrwtuNF67eOxAchfEtYA2OxmLU\nP/LGKIwMrxko9+gun8qiVMwdn4/a4QVx1xF5Dh7eku0SY2EE3JaYBps3kaBdFuFV4LPcR+Ph0yd+\nzR1mX6PdWgbFJQuILYUCIP6jMUqgCDwy1QpEjlWxXdaYT13WjITnWe/Rqa/V7RJkyBIPrSPTdoqq\n6BaGJACKTOMT/nZwHEE0bM+sEoJn2WS5aJhHR3fY/mECyEsNskmZPxi1xrYvYEmVLg0WgxMVljbL\n+tbmTlPs/WVZ1pfz9FDfe9TNLLTcLBe23TIWQuAcDdZG3eZgQ1DBu43yLh7WT5yg4YHVVYiQMDie\nQIIHasS0cgBTKah9SVCbS+g2MQLe3w4huQ2C6mGVio1Kxka1ZD90a1iRqFXCaeCDbbS6cVuePd4E\noF9feyQgm8ouooXNwYD+PoxFxgBAyGw076G/q7AaxpHTbQgVvQ0AiDaOgHKWWjRs5C4amXAyJE42\n+1ZXQD7Zg1HDMlkbkv0uJqi1sAuwxHi+7KJtUM9lAeAAVaJl8I0MJDEKLezBkgK9phqz0CxlhCTq\nau6N9OsWhm6hnSzV+9SqMOh5N5Wstr0rowqB2poPtbnYdq8e3RWodadAOZPH+r1d/hyvHvl3fFMc\nhfEtkJdEf8RjM0bH7TMsDLfooi4TnaJMM188yWVXJiWZMXECUMHrEUylIPEinrl7KsaU0OvwmtUl\nZZYhCR+eyASAfR/g4k2LhRAeyUm0HfWli/HszCcROlCHaONw5LoKzXMkIa4UCkAtFhvEtDACXpmu\nBWJcw+KSWla6ALnuYeZpTOjGW0cuQYZLFEAiHuYyMNpAYxg8lDN0wp9r2BEAQJeRJWkR2oK+RGxP\nN/DBwRZz3ghrn2ldTMgahxJ+PPvbGpdaVW5aaETjLMrO4l4SFHbv5CSZKYZ5tdmQ3SqIItvbxysQ\nfL0Ap0HrplaoVzLjJWYBSSPuRd/7sFwP2iI0pdZDUqBEeXYOADZXROtPQl+v/nsQo+D0kv1ZkXGI\nHKmG1pcEcATJfpkpjGx3nv5cIjghimBArwLbka6P4O3W0alOah0F5CTmumtKfSMmRsfRd6wPNIz0\nY63Pb1EYUXzRaVb/JRE3KzLJSWHw/rMQ0/T0bkWCZJTQFxQ2CVRWk1i9NQgKAl6ZlaAhETfrf06M\nsNG7UUyUKCJCagjhqAo+0AbeFQJUAZnBII2b68rKmOXv66zAoQP0fby57zhaO/qZwtB6kvX0bdou\nztcBIaUVhAATc8aA0wSmFA6dpXWsSNQFrd/IxqP7enWFYc0mg6BAwYUXkxoojsL4Fkh1p+Dv6x7D\n7aPXxu0zBIwsyCxv+4l1tSjLM2MIse4qq1Ay6Asr8ImmgJ9YnoPUgNucvGONIVhy57WuNFbKnGIq\nD06zWwrpQSqYeI6n9alCSVBOl7J2A/a0W8AapJYQ+mSGZTttgyhwtFS3Ygphq4XhcYnwy5YMNEPo\nRjzQQubzcuAgCRJy032oHp6OCaWmdURUEQLPQxYFaJ3pVGD79NURozK7QviwPf5EVBH//t4XUFqG\n2awMq8IAAAHm3wHdJQUAGZ40UxmrElyyiCdvn2S+CykCjifI8Adw45wyOjlTVyZzJmYjrIbN2JOx\ndK/SiLcaqQtGC1NhkSSbAk8TdJeUIdQs5VBO9dMRcpKWgUiEp7Emyyx6EpUBRbYIySgTRElCgM6W\nD3vAccC9KyvY+i9FQdrXdD6LgtxMe+wn1sIw4i9BVwAe68zrGBcM0QRbDEMLeaC25bPf1LmeXvzH\nXjNIThTJJuDFLEspeSLArSefcLzKhLhXS4NsJCiICqKqBjFJXx2yz2/ri96IRRgDgCIhrPUjElUh\nDTvM3lPQ6wKnSbqA1yCX6bXGoh7WFz3hPpwJtUIqOKK/Lxkk5NNTogmkAlrZluMAt0tEwO2DKNP+\n+/cj/wWoEtS2XNbHctkBCKmn0aN0mX1h6XcFA6+ueyEchfEt4ZeTErqmDJcUVRj0BWemUEG4bvRa\nLCmeb5scBMTMENcJeCX4JGvWlV3J8Kop1OIC4uGYcuw6GiFmmwmf0G8PmAvBALDXvALsEwiVeCtH\n4HlMr8qlAslou+DC/SvH4o4lleA4DgGX5XmNaxMe4QPTWbaIJEjgOA48z+He5VUYV2h1pwkQBU6P\nv3BmCitAZ3kbh/WZwt7WXk1E1OYys/cDT8y/rcqc53hz8StFgqpqyE33MSEuF1IhU5CejAW1w+Bx\ni6zv2kPnQECQ4be3CaAzigGwa/tc9P2JGY1QjOKSMZbJrz57Gaf08hj97X5EIhqdk6KnJfPufmj9\nSRg5LBmV+XROESdGzPiHYMR6aJ8E/BxdD4RwyPYnY9a4XL1iMkEwzZ75Zfw7rzYbnLsHQkYTBM2N\nTG9GwgQHI1gPjdYE41x0Do4R9F08kbphQmrETEuOuOhgQFeUvByBqCtytZNa2Sw2JqjwpOjVAfqC\nOHUmRAcEQhSn2nrpipFRN6C4zHIzYoSt+24qQgkqVOw+0MQC1NHGEeB5DjxxUYUhmgFvviPPzJQT\nFPsSBRE3tLCXPq8cYuO26CnqHvRJXmhSH4TMr6DyIag9AZCI1/atyWWf4AuiL1+rSKYVLijQuPOn\nK18KjsK4yhiL+filJDZSFwX6a6nJrMKi4nlx5/gkc2RdNzYX96+swrQxObYJdMYo2Ai4c+r54xux\nghIApo3JRllekNXI4sRInMJ49NYJmFOTh+oR6Wwbx3F214LFmkm03e+VML48A//4w5mWtrtQVZqG\nKaNpgb9kt2VGfIzbIsVFLTGZtygjAMkuUwEZLin2p8UymVJRgBEFetaZItvjFVZLyaLs3DEWn6BZ\nFYa9L41ArtaTjPJhKXHXBQCvaFhunF6sEfiwmU64Ks1Kwy/un2GLE7E26QojoK+/wid1oVdoocrC\niE/pM6e/6mpEe+QsSFTG4S/68HljB+1L0eLGCnuQnuzBD+brylGMMjcIc9vo/dCn9FOFokjweiTc\nsnAkND1773/wLr0ey2jTkwYkDby3GxwH5ChVcAkyunrtgkwLeYGoG5nJHowtobEF99h3bf2W7NXf\nn2XiY7SxHADHLEZXcjdLj40co4t7ifpvhE86B9HTD6JIaDsLLJtabCpPMQIihaD20PdoXE/K/QLH\nO7+09QH7Tej127SwG1oX/RYkuMHJYXNNmDN5UBXebm3pbZd68gDFBaL/LqXSA+AFDUTlkROh5WKM\n9VjkokO2e9sGYRZIxLRmeF8HVC5qq5D9dXEUxlVmWm4tvl+5GuWpZSjM9iMvw3fBMsaAXWE8fMtE\nVJWmQxR4uCXLaD6m2JgxpwBAvMJIkHW17vpK8DzHsrU4Vz8CMQqjOCeAtfPLael1C7bgpW0mMGdZ\n4J6ek5mi+2CtzxxTHSLda5kFH9P2FDcVUrFzXZJdZuoxUQWmhAEzPREAJpTms7LfAMeCnDzHo7LQ\ndNVZLTSPZL9Xrs+sXBuIcRdGvhoJEpUw0lVL54ggPmXZrSuMmhEZmFkyDi7ehY9a6Mxoj+iB1y3i\nb2+rxTBSYzvPUBheyW4hcqpl5r3lXXRGOi0uOIDjCHhXP3PdkKgMTSPwy0m0/Iv/HBN4xj0MIdQX\npdlkRJFoZhyA22uXQiAy+jU9vVQVMboklQn6kBKCKOuz7BUZR5o68L8/OGJruzEq9rpFdITtpdON\nZ/G4JJqRxZsTH3m9ijLpC0DrS4LqPwWS1MbaAQBpoHEtKfc4iEDb3tMfxeiSNKT5/OBFBYvqqJIy\nrG4jeQAAQoSWJmHKWBfW1HWnmNYDAJ8eT5RLP2FtiCrUqiME4LzdZu2xvjL9nvR3KfjPgfN2IuD2\n4cHV1QCA1n5zFjq9d0xsy0L0VIm+3DPdJ+UdB/GcYwkG3wRHYVxlZEHGxOxq8ByP7y8aiR//YOJF\nz0lxJ2Pl8GXYMv5u+7VkgZWdOBemPvrKIipsq4vy2XFG/MDjElGSG7C7aGIo8NOApkeSUD0iPtie\nkPNU3r11YTmCxshfz5PPTDaFtzECCql2X3a2pU6XITSeuXsq/v7OyUyhSTEKI9Vtr2wqWCwM6yz4\nJMlnsz4Ml4ZHcCPoNa+Z7DUtB3dM2u+SCebaKF6LMgcAtaUIoY/nYFp5iem600SED9WyY9J0K04U\neNx8XSUKg6Y7zbA+slK8eHjuTfh+5Wrz4npbjbXMDXjVFAzKqVJInEWBxKz+CIBNliNRF8JRFSIv\nguvIB+8KQUg5oz+XBwGfzEa0Lx9+FZwUBYnKdAY9gNqKbGQlmckNRJFQlO3HjrtnAQDeP/0R5k2l\n7qHjjf14+vd/gdKab6tHZQT9PS4RTT2nbM/F0psFDhyvlzZJp242v5EFSMzEBgiKTcCnCFkYnlwG\nTg4jTPoARURuuk/vQw8kt4KRZW57PxEeoU+m256J/V+1KAxRQW5yEE+so++1NjgbRBHB+2g8ROZd\ntMAi4aGeyQfv7oOoZ34ZvycjC83AL/tYSfqCpFzbPkPx2+ZX6Rjl/+OKnl4gXX2gOArjOwTHcXGj\n9fMxq2AaSoJFtm1uSWCZM8aodU5NPratHY/66aXmgfoo/b4VVchJ8wKqBKGtDBVCTAorgMXF12F2\nQR22zbyDZkCdh4fXVONHN9NsIbUzsWLRNIKyZOp/NjJsMlLMEdyDE36IMemVmJpbazsv22dJk9U/\nghS/C5kpXqYwYl1SAi+gIf8maCEPtO5UiJb5I2qXmYHmO4/CUImKJI+pMFycKYRjlZMk8phTMB0i\nLyLHZ0/ppXA2C+p7M0qw1KJkioL2NGlrIUt3zKhwQtY4TM6egNrsGty3YizuvGEU0j1p4L+YxALs\ngmY5R5Uw1jeV/TkqP4cJoegXMVl7UReum0DbInXnsc1EFeCWJTyxrhar6uis8rMhfSEpRbYp46DF\nFUhUEa3n+pnSA4C3mvRZ2Cy+ISP8V7N9rNSNLGBa7iRb8wxlIvI8KyjJe6k147aU5bAtiWwRxpkc\nvgAAFtxJREFU8KLAI91jWqsF6SnYciNNcy0JFiGqKdjXoseHrNcIe01LzSqg9Wu7KmgBxaxAEHkZ\n1MIszciB0mJm6vkkD1vRT23T0131+AYb+SsyIkfNWJnVcrxr7A+QKZjXY0kiSoLBGYvL2U11NdGx\nl4ijMAYRLklA9MtRiDaVob50EQBaUrksP2gTikb8QBJ55iKSzoxCTdr4uGvKgoQVw29ga4Ofj/Jh\nKSjVM7u0zgxET5aiyluHm+ePYMeoGsGkbF2p6EI74DU/wAJ/Hu6q+n6c6Wyr06WPFg0BnHoelxQA\njEorR/jATJB+P7xuy8dicc/5ZR8k0bJuQBd9zpAaZusVGGuwGyNhI25i5XtlS/DszCfhERMnEFgd\nc0umFmFpbTn7Oy9m9GhVOt6Y63Ech5srV+HWypswtiwdkyqpciFdGQgfnIwcbTRSQxVI8ZsCLyCb\nQjw3OYWlWmvdabbFqB5YMYnFc1xRM8OLKBJ8bhF+r4yZJTU2F5zEuVj2HGBXGOlJSbhuYgGrZmDF\nNlK3JB4Ygxm3LGLliGUYlzHG3GfUPhN4/GjyJtv1rKnYVrebVcBLAm+LbWX6A6yfjBU0P2r5S/w1\nwLE4H5s5DqA4RtFb331mssfWt92WQlKGK9Fg0cQy9n/N4i61Zj0mu4IYhrHmSUb/kfhBnNG3WmeG\nrSpEW/s3D3w7CmMQ4ZIFQBOhnCpLKLhMU1VflIfnMFUPLC+ZWmRzD31TlJPDUe4ej9k1+Vh/QyUC\nPhkTK7JQmVaOB8bfg++PXoX1SysvGq8xmJozEZWp5Vh3fSVumFbEtpsuqXjT3Mg6AwC/165QNo3d\niDtG3wyP6LEpU8OdUZlWztYrCPpkyIKA0P6ZSGqahYnZ1XH34jguYRbcTXOHQxJ5jCy0VyPmOR4z\n86diTsF0SDECNccikD1SYgUUiyjyIH1BjOCn4oeLpuMnG6axfUGX6U4LyH4MyzLjLJolQ84q7GVB\nBukz/67Q2y8LMlaOWMa2zxtXauu/ZIsy3bC0GqW5QXAch3UxKeW2YK3NzWO4pARIvIiiQEHcPlHg\nkO3LtGULWgcZomYpkWMZVQsCh6ClfdZvpDhQaMbXABDF7r5RWwtsbQAAn5qNGcEl7G+rRZAacGFq\neTH7Oxw2r11XUQTrEGJ4bjqeXj+ZXt+iMGLffapo/i58kg8TR9KBhW0FSgAjcvQkFMIjcthirV9g\nkbaB4lSrHURcyGUEAOHDtXopCl1hCFSQPb9lJmRJQGfv5cnVZugW8eTKbEyuNH/sxcFCFMcP0i/I\n/6pYmXA7C3rz8RaG12X+vA03zMYVVTh2shNlafkAqHKwWV+qBOHQfKy7czr6iglOn+3FzQvK8S+7\njgCKC66o74Iz92OZP7EA8yfGz8wHgFUj6hNutwpJcYCZLRuXV+EP7xzDwknDEPDJyMgwlYTVOgzI\nfvgyLLP+LQLKOodEFHmo57LAJ3XSkicWhZtvsYh8MTGboMWasQrxmswq/IvkM+s+2Xzv1oQH+n9D\n2VvbZMYwaP+nulPQHaUuKTp5sRcuWcDT62bhRx/sjrsPz3M2C8Mq4CVBQoY3jZVIJxEXinMCKM0N\n4PjpLvzo5tlY/+vfQOtJxuiSVBz+6hyWTivCud4e7NZj81ZrkOM4rJo+Gvve/QN9HsENoxwjITy8\nfBL6tG4QjYfEi5CM9FuL8pRjftMLa4vxJz1hbNPy8Th4UMNHh1sRPjgFj24owzP7fgEAaD5jqV1F\neD1BQLvwMtADxLEwBhEu+SI/CE2wuWOIvtCvrCsawz1kdWd8HQwBWT4s+SJHfnMyPelYWDgHM/On\nxu2zpqL69WcbV5aO5TNLbcfZFAYAF+eFW3QhNeDGI2vHIz8jCafO0s89mPTNA4cXQ+RFXDdsFoB4\nd9X5KMkN4MHV1XGZbACQ6gli1Yh6ZHrSURwchorCFEwbnY07llSymFehv8Am4JfPLIHSRu9dmVJp\nu55X8uDGEfVIknwoCgyz7Svw03MkXrQLe1jWvdd4WMvSrJk3nC13y3vpxDPjt2i9htXCAOyJDR7Z\nFLRBj0UhWq0XYs+ei7XCc7xm7GjNrFF4aHU11lw3An9zywRwHAe1dRhIXwCFWX688OBsFOcEkOIz\nrZzYZA2rS2ndItO11tsfZen0RsxGZoM9jrmR+hR7xV+XJGBTzQbMyp+GAn8eJlZkQuA53LGkEmmW\n2MzyGcNt5xluViNu+E1wLIxBxMUsjFnjcvH2fjPzJLZEN8dx+NnGujgBeqncNHc4GqaXXFyBXQY4\njsPS0oXn3WcQ65KyYk25BZBwzkN3H7W+aisSBbUvP8tKF2FR8bzzlsa/FFySgJlZU21Kdd0SqgQq\ni+ohSEvhkz22/hpdnIZfbVmMzvB0c20PCzPyp2J63pQ4l2JxsBBPTfsbcBwXF7BPd6fiq65GWiLe\nQpJHwuphy7Fj33PoPkWVeWYqFbb5fovCVI15SnpKttda0ZkqBlUltjZpFrcaIQRZlnO8Me0zrpfm\nTsG88YWIZerobLz/12akBszz3LKA6MlSSHnH4jKZrO3IS0nB8PwuHGnqRG9/FHmBTJzsb2QLZFnL\nAkWOjkNwxP9gYdHcuDaUJRezxJH0oAe/fGg2ezaDuqocVgZ9XFk6XMHJOID/w9xq3wRHYQwiLqYw\nblk4Emvnl6OxtQefN3YgOzU+ZnEhwXpJbfkWlMWlYLikEiHFKM7Z1Xlxx2xcUYX9R9owZVR23L4r\nAcdxl0VZANbRazwXs5is8Y9Yzhd/ssZCrFRnVmFf6ydx271uEQX+dPy07m9x5wc0iypLz57z25Yx\npuLKr7/L2QV12NW4GwVJuZB66TtU9USFm8ob8Lu3Dphr1INaGLIl1hUbjJ+ZPxUCL2B2QV3C9n9/\n0UhUD09H9XBT6QR8MpSTZfBHCzB+9riE5wF0lvniyYX42WsHMG9CATwZSfiwZR/bb/0NPnvnXEji\nfHuixkXgOA4NZdfHpc5WFKVAVZOx973ZQPT838BAcRTGIOJiCgOgftzCbD8Ks88vCAYjfu/5Pxbr\n8pX/eG9dQrfO6OI0jC5Oi9t+LTCQ38W3wdiMUZiQNQ4dZ2R8atlupJJLotnOjGTTXfS3Ux7Gm4c/\nxFu99L2kJ5vK5KfTfwwOHP5jD11kyBhoT8+bgvTp5XjtnWP44nS3vo/uTPekoa3/LOy5azQetrRk\nwXnbLwo8xpfbLUyfW8LT66cgySslVKC3j74ZRzqOISAnYWyZn8ULVS0NxYFClKdQi4rjOFw/pRA5\nad6v7facN2xm3Dae4yDKvC3V+JvgKIxBhEsWsKF+NBudOZhcyMIw1qj2uMSEyuJaxyV/N0KVPMez\ncvDaTILb/4Eu0+pOYI1a3aLpnjRMzpiKt7APxTl268WwwhItaFVRlIpHi1Jx29+/BcBUJptrNuC9\nU3sxKbsm7pyvQ1YCS92gOnMMqjPN+IVh7Qm8gAcm3GM7Nja2djkQeA6CMLBMxIFwxRXG7t278dRT\nT4EQguXLl2P9+vW2/ZFIBA8//DA+++wzpKSk4Nlnn0Vu7sACfQ7xGKl2DpQlU4vw2RftbC2PRPSF\n9bURLsEFcC0x0Mmg3yY8x+HJ2yfh4JftKMk1lcC9y8fYStwYlOUHsfnGsRienziRYiBC0fDyB10B\nXF983ddq97UGz3PQNHLxAwfIFf1CNE3DE088gZdeegmZmZlYsWIF5s6di9JSU5O+9tprCAaD+NOf\n/oQ33ngD27dvx7PPPnslm+UwhPjejBJ8b0bJBY8xXDaJYjrXMo/eOoEF67+L5Kb7WGkOA2t8IJYL\nuQQvpBR9Hgm9/dG45IahgFsW0NN/eSrVAlc4rfbAgQMoLCxEXl4eJEnC9ddfj127dtmO2bVrFxoa\nGgAACxYswJ49e65kkxwc4lg5uwzXTSjA+htGXfzga4jinACqStMvfuAgIJFLyuAnd03FxJGZmFOT\nf95jBhuP3joBs6vzUDMigxX4HJ5/iZOfEnBFLYyWlhbk5JiLwGdlZeHTTz+1HdPa2orsbJp5IggC\nAoEAOjo6kJx85XP4HRwAGrtYPW/4xQ90+M7i8+iT+hIojtL8ZGyoj1/tcjBTnBNg8Z7RxWnYfONY\nlOR8xxVGbIntgRxDCBlwuQgHBwcHAJhQnonjE7pQNybn4gcPQS5Xht8VVRjZ2dk4dcqcKNbS0oLM\nzMy4Y5qbm5GVlQVVVdHT04Ng8OKa0Fr6YKjj9IWJ0xcmQ60v7lsdXzzTYKj1xZXiisYwxowZgxMn\nTuDkyZOIRCLYuXMn5s61z16cPXs2Xn/9dQDAm2++icmTJ1/JJjk4ODg4fE04MhC/0Tdg9+7d+MlP\nfgJCCFasWIH169fj5z//OcaMGYPZs2cjEongwQcfxKFDh5CcnIwdO3YgP3/oBKccHBwcrhWuuMJw\ncHBwcBgcfPdm9Dg4ODg4fCdxFIaDg4ODw4BwFIaDg4ODw4C45hTG7t27sXDhQixYsAAvvPDC1W7O\nFWfbtm2YOnUqli5dyrZ1dnbitttuw4IFC7Bu3Tp0WxYMfvLJJzF//nwsW7YMhw4duhpNviI0Nzfj\nlltuweLFi7F06VL89re/BTA0+yISiWDlypWor6/H0qVL8Ytf0JXWmpqasGrVKixYsACbN2+Goijs\n+E2bNmH+/Pm48cYbbanugwVN09DQ0IC77roLwNDtizlz5uCGG25AfX09VqxYAeAyfyPkGkJVVTJv\n3jzS1NREIpEIueGGG8jRo0evdrOuKB999BE5ePAgWbJkCdv205/+lLzwwguEEEL+6Z/+iWzfvp0Q\nQsjbb79N7rjjDkIIIfv37ycrV6789ht8hWhtbSUHDx4khBDS09ND5s+fT44ePTok+4IQQvr6+ggh\nhCiKQlauXEn2799P7rvvPvLGG28QQgh57LHHyD//8z8TQgh5+eWXyeOPP04IIWTnzp3k/vvvvypt\nvpL8+te/Jlu2bCF33nknIYQM2b6YM2cO6ejosG27nN/INWVhDKQ21WBjwoQJCATsJZ2t9bcaGhpY\nH+zatQv19XSd6LFjx6K7uxttbW3fboOvEBkZGaioqAAA+Hw+lJaWoqWlZUj2BQB4PLQ+UCQSgaIo\n4DgOe/fuxYIFdD2HhoYG/Pd//zeAwV+vrbm5Ge+88w5WrjTXff/ggw+GZF8QQqBp9hUNL+c3ck0p\njES1qVpbW69ii64O7e3tSE+nReUyMjLQ3t4OwF6XC6D909LSclXaeCVpamrC4cOHMXbsWJw9e3ZI\n9oWmaaivr8e0adMwbdo0FBQUIBAIgNertmZnZ7PnPV+9tsHCU089hYceeoiVFDp37hyCweCQ7AuO\n47Bu3TosX74cr776KgBc1m/kmloAgDhTRi5Iov4ZbHW5ent7sXHjRmzbtg0+n++8zzfY+4Lnefzx\nj39ET08P7rnnHhw7dizuGON5Y/uCDKJ6bW+//TbS09NRUVGBvXv3AqDPF/vMQ6EvAOCVV15hSuG2\n225DcXHxZf1GrimFMZDaVEOBtLQ0tLW1IT09HWfOnEFqaioAOkJobm5mxzU3Nw+q/lEUBRs3bsSy\nZcswb948AEO3LwySkpIwceJEfPLJJ+jq6oKmaeB53va8Rl9car22a4G//OUveOutt/DOO+8gHA6j\nt7cXTz31FLq7u4dcXwDUggCA1NRUzJs3DwcOHLis38g15ZIaSG2qwUjsSGDOnDn4t3/7NwDA66+/\nzvpg7ty5+OMf/wgA2L9/PwKBADNFBwPbtm1DWVkZbr31VrZtKPZFe3s7y3QJhULYs2cPysrKMGnS\nJLz55psA7H0xZ86cQVuvbfPmzXj77bexa9cu7NixA5MmTcIzzzwzJPuiv78fvb29AIC+vj689957\nGDFixGX9Rq650iCJalMNZrZs2YK9e/eio6MD6enpuPfeezFv3jzcd999OH36NHJzc/Gzn/2MBcb/\n7u/+Du+++y48Hg+efvppjBo1OBYF2rdvH9auXYsRI0aA4zhwHIdNmzahqqoK999//5Dqi88//xxb\nt26FpmnQNA2LFy/Ghg0b0NjYiM2bN6OrqwsVFRXYvn07JEkaMvXaPvzwQ/zqV7/C888/PyT7orGx\nET/84Q/BcRxUVcXSpUuxfv16dHR0XLZv5JpTGA4ODg4OV4dryiXl4ODg4HD1cBSGg4ODg8OAcBSG\ng4ODg8OAcBSGg4ODg8OAcBSGg4ODg8OAcBSGg4ODg8OAcBSGwzXNqlWr0NDQgOuvvx6jRo1CQ0MD\nGhoasG3btku+1u233z6gctePPPII9u/f/3Wae0kcPHgQ//mf/3nF7+PgMFCceRgOg4KTJ09ixYoV\nF6w+apSKuFZ49dVXsWfPHuzYseNqN8XBAcA1VkvKweFS2LNnD7Zv345x48bh4MGDuOeee9De3o6X\nX36ZLaizdetW1NbWAgBmzpyJl156CcXFxVizZg2qq6vx8ccfo7W1FUuWLMH9998PAFizZg3uvvtu\n1NXV4cEHH0RSUhKOHTuGlpYW1NTU4OmnnwZAa/M89NBDOHfuHAoKCqCqKubMmYMbb7zR1s62tjZs\n2bIF586dAwDU1dXh9ttvx3PPPYe+vj40NDRg0qRJ2Lp1Kz7++GPs2LED/f39AICNGzdixowZOHHi\nBNasWYMlS5Zg3759iEQiePzxx1FTU/Ot9LXDEOGbLNbh4PBdoampiUyePNm27f333yeVlZXk008/\nZdusi8scPXqUzJo1i/09Y8YMcvz4cUIIIatXryZbtmwhhBDS1dVFamtrSVNTE9v37rvvEkIIeeCB\nB8jatWtJNBol4XCYLFy4kOzdu5cQQsiGDRvIL3/5S0IIIY2NjaS6upq88sorcW1/8cUXyWOPPcb+\n7urqIoQQ8q//+q9k8+bNtrbX19eTs2fPEkIIaW5uJjNmzCA9PT3kq6++IuXl5WTnzp3s2WfNmkUU\nRRl4Jzo4XATHwnAY1JSUlGD06NHs7y+//BI///nP0draCkEQ0Nraio6ODiQnJ8edu2jRIgCA3+9H\ncXExTpw4gby8vLjjrrvuOogi/ZQqKytx4sQJ1NbWYu/evXjyyScBAPn5+cySiWXcuHH4/e9/j2ee\neQYTJ05EXV1dwuP27duHpqYmrFu3jhWkFAQBjY2N8Hq98Hg8WLx4MQBgypQpEAQBX375JUpLSwfa\nXQ4OF8RRGA6DGp/PZ/t706ZNePzxxzFz5kxomoaqqiqEw+GE57pcLvZ/nuehquolHTfQdRbGjx+P\n119/He+//z7+8Ic/4MUXX8Tvfve7uOMIIRg1ahReeumluH0nTpyI26Zp2qBa68Hh6nPtRAAdHC4C\nGUD+Rk9PD6tO+sorr5xXCVwOamtrWVnpkydP4sMPP0x4XFNTE5KSkrB48WJs3boVf/3rXwHQtS6M\nMuYAUFNTg6NHj+LPf/4z23bgwAH2//7+frzxxhsA6BKlAFBYWHh5H8phSONYGA6DhoGMprdt24b1\n69cjJycHkyZNgt/vT3h+7LXOt+9Cxz366KN4+OGHsXPnTpSUlKCmpsZ2P4M9e/bgt7/9LQRBACEE\nTzzxBABg2rRp+M1vfoP6+npMnjwZW7duxXPPPYft27eju7sb0WgUBQUFeP755wEA6enpOHLkCFau\nXIlIJIIdO3ZAEISL9omDw0Bx0modHK4Q4XAYkiSB53m0tLRg5cqVePnll1FQUHDZ72VkSb333nuX\n/doODgaOheHgcIU4fvw4HnnkERBCoGkaNm3adEWUhYPDt4VjYTg4ODg4DAgn6O3g4ODgMCAcheHg\n4ODgMCAcheHg4ODgMCAcheHg4ODgMCAcheHg4ODgMCAcheHg4ODgMCD+P4xSKOOE0RxSAAAAAElF\nTkSuQmCC\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f97f1e98d90\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEcCAYAAADUX4MJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXeAHMWZ/v/pNGlnc5S0ymmFUE6WEAgQ2UJkGxtjsMEG\nbDD+YnNwZ3PnH+fD2GcwnDFHMBmcwETLIiMJ5YByzqvd1eY0eTr9/uie7p7dlRACHQ7z/LM73dVV\n1dXd71NvqLcE0zRNcsghhxxyyOFjIH7eHcghhxxyyOHvAznCyCGHHHLI4ZiQI4wccsghhxyOCTnC\nyCGHHHLI4ZiQI4wccsghhxyOCTnCyCGHHHLI4ZiQI4wccviUWL16NXPmzDmmsg899BC33377Ce5R\nDjmcGOQII4dPjTPPPJNx48bR2dmZdfyiiy6ipqaGhoYGAO68805qamrYvHmzU6a2tpaamhrn99VX\nX81LL73k/H7kkUeYO3cukydP5vTTT+e2224DYN68eUyePJnJkydz0kknMX78eCZNmsTkyZN57LHH\nTuTt9glBEE5I2Rxy+FuC/Hl3IId/DFRXV7NgwQKuuuoqAHbt2kUqlcoSjoIgUFRUxAMPPMATTzyR\ndbwvvPLKK7zxxhs888wzVFdX09bWxvvvvw/AX/7yF6fc1VdfzcUXX8xll112Im7tnwa6riNJ0ufd\njRz+hpHTMHL4THDRRRfxyiuvOL9feeUVLrnkkl7lLrnkEnbu3MnatWs/ts4tW7Ywe/ZsqqurASgt\nLeWKK67os+zHJSx46KGHuPXWW7n99tuZPHky8+fP58CBAzz22GPMmjWLM844g+XLlzvlm5ubuemm\nm5gxYwbnnnsuL774onMulUpx5513Mn36dObNm5elMWWu/d73vsfMmTM566yzeO655z72XgG6u7u5\n8cYbmTlzJjNmzODGG2+kqanJOd/V1cW//uu/cuqppzJjxgxuvvlm59y7777LxRdfzJQpUzjnnHNY\nunQpYGl/K1asyBqHjEmsvr6empoaXnrpJc444wyuvfZaAG699VZmz57NtGnTuPrqq9mzZ0/Wvd97\n772ceeaZTJ06lauuuopUKsUNN9zACy+8kHU/8+fP57333jume8/h7wM5wsjhM8GECROIxWLs27cP\nwzB48803mT9/fi9BHggEuPHGG7n//vuPqc5XX32VJ554gi1btmAYxqfq46JFi7jkkktYu3YtY8aM\n4brrrsM0TT788EO+853vcNdddzllb7vtNvr378/SpUt58MEHuf/++1m5ciUAv/71r6mrq+O9997j\niSee4NVXX3WuM02TG2+8kTFjxrB06VKefvppnn32WZYtW/ax/TMMg8suu4zFixfzwQcfEAgEuPvu\nu53zt99+O6lUioULF7J8+XJHwG/atIk777yTO+64g3Xr1vH8888zYMCAI7bTU6Nbu3YtCxcudLS+\nOXPm8M4777B8+XJOOukkfvjDHzpl7733XrZt28Yf//hH1qxZw+23344oilx88cW89tprTrkdO3bQ\n3Nx8zL6dHP4+kCOMHD4zXHTRRbz66qssW7aMYcOGUVFR0We5L33pSxw+fJgPP/zwqPXNnz+fu+66\ni2XLlnH11Vcza9asT+WfmDp1KrNmzUIURc477zw6Ojr49re/jSRJXHDBBTQ0NBCNRjl8+DDr16/n\nhz/8IYqiUFNTwxVXXOEIxDfffJObbrqJ/Px8Kisrufrqq502Nm3aRGdnJzfddBOSJFFdXc0VV1zB\nggULPrZ/RUVFnH322fh8PkKhEDfccIOjiTU3N7N06VLuvvtuwuEwkiQxdepUAF566SUuv/xyZs6c\nCUBFRQVDhw49pjERBIFbbrmFQCCAz+cD4NJLLyUYDKIoCt/97nfZsWMH0WgU0zR5+eWX+fGPf0x5\neTmCIDBx4kQURWHu3LkcPHiQ2tpaAF577TUuuOACZDln9f5HQu5p5vCZYf78+Xzta1+jrq6Oiy66\n6IjlfD4f3/nOd3jwwQe57777jlrnvHnzmDdvHrqu8+677/KDH/yAsWPHcsopp3zi/pWWljr/BwIB\niouLndl2IBDANE1isRgtLS0UFhYSDAad8v3792fr1q2AJbyrqqqyzmXQ0NBAU1MT06dPByyNwzAM\npk2b9rH9SyaT3HPPPSxdupTu7m5M0yQej2OaJo2NjRQWFhIOh3td19jY+Klm8t57MQyD+++/n7fe\neouOjg4EQUAQBDo6Okin06TTaQYOHNirDp/Px/nnn8/rr7/Od7/7XRYsWMCvf/3r4+5TDn+byGkY\nOXxm6N+/PwMGDGDJkiWcc845Ry176aWXEolEeOedd46pbkmSOPfccxk9ejS7d+/+LLp7RFRUVNDV\n1UU8HneOHT582NGYysvLOXz4sHMuEwUG0K9fP6qrq1m9ejWrV69mzZo1rFu3jkceeeRj233yySc5\ncOAAL730EmvXrnV8AqZp0q9fP7q6uohGo72uq6qq4tChQ33WGQqFSCaTzu+WlpZeZbwmqjfeeIMP\nPviAZ555hrVr1/L+++87ZsXi4mL8fr+jRfTExRdfzOuvv86KFSsIBoNMmDDhY+85h78v5Agjh88U\n99xzD8888wyBQOCo5SRJ4uabb+bxxx8/YplXXnmFxYsXE4vFME2TxYsXs3fvXsaPH/9ZdzsLVVVV\nTJo0ifvvv590Os2OHTt46aWXmD9/PgDnn38+jz76KN3d3TQ2NvL88887144fP55wOMzjjz9OKpVC\n13V2797dyzHeF2KxGIFAgHA4TGdnZ9YMvby8nNNOO42f/OQndHd3o2maY666/PLLefnll1m5ciWm\nadLU1MS+ffsAqKmpYcGCBWiaxubNm3nrrbey2uzpY4rFYvh8PgoKCojH49x3330OoQiCwKWXXsq9\n995Lc3MzhmGwYcMGVFUFYOLEiQiCwL333ntUDTOHv1/kCCOHTw3vDHXgwIGMHTu2z3M9MW/ePCoq\nKnqF3mYQDod55JFHOPPMM5k2bRr33XcfP/nJT5g8efIR2/808NZz3333UVdXx6mnnsr3vvc9br31\nVsdHcPPNN9O/f3/mzp3L9ddfz8UXX+xcJ4oijzzyCDt27GDu3LnMmjWLu+66q0/NoCeuueYaEokE\nM2bM4Morr+xlZvrFL36BLMucf/75nHLKKTz77LOARVL33HMP99xzD1OmTOHrX/+6owHdeuut1NbW\nMn36dH7zm99w4YUXHvGewdIS+vXrx2mnnca8efOYNGlS1vk77riDUaNGcfnllzNjxgzuu+++LNK5\n+OKL2b17t0OuOfxjQTiRGyj927/9G4sWLaK0tJQ33nijzzI//elPWbJkCcFgkHvvvZcxY8acqO7k\nkEMOJxivvvoqL774Yq8Q2xz+MXBCNYxLL700a4FWTyxevJja2lrefvtt7r77bv7jP/7jRHYnhxxy\nOIFIJBL8/ve/58tf/vLn3ZUcThBOKGFMnTqVgoKCI55/7733HHV+woQJRCIRWltbT2SXcsghhxOA\npUuXMmvWLMrLy5k3b97n3Z0cThA+17DanuGJlZWVNDU1UVZW9jn2KocccvikmD17NuvXr/+8u5HD\nCcbn6vTuy32SS8yWQw455PC3ic9Vw6isrKSxsdH53djYeMTVwV6Yppkjlj6w+1AHtz2whLOmDeLW\nKycdsdyFP7BWLL/+y/m5cQSu+veFdMfSnDdzCN+9vO+1A5kxe+JHZ1NREvq/7N5x48ofLSAYUHjq\nrqOvifm/RmYsM3juJ+dRlO//VHXe98I6Fn1UR2VJiN/+6Ow+2/vpjbOYMLL8U7Xz94LMPT/6r3N5\n8vWtrNrayOCqfB66/cxPVe8JJ4yjBWHNnTuXF154gQsuuIANGzZQUFBwTOYoQRBoaYl8lt38u0V5\neb4zFoebrL+xeOqYxudwYzeK/I8TWe0di08Cv2KNQWdX4mOv37anBWFoyXH17/8S5eX5pFQD01T/\n5r+Vg3UdqGV5n6qOZMpaC2IYxhHvt+sYnu8/Glpbo6TTGgC6/ulyscEJJowf/OAHrFq1is7OTk4/\n/XRuueUWVFVFEAS+/OUvM2fOHBYvXszZZ59NMBjkZz/72Ynszj88kpkXwzi2SGlVM/6hCON4EfBZ\nn0EipX1s2cb2OGP/DgjDNE003cAwzL95jTyaUP9vGjpxKwj+ZqHrn+09n1DC+Lg8QQD//u//fiK7\n8HeFV5bsQ5FF5s0aclzXJ9M6AFqPl0TTDR59fSunnNyPiSNdDU7tMeMwTJPH39jG5FHlTKv5eNPg\nkbB6exMbdrdy/YUnIX4Ggmr7wQ4+3NjANy4Y84kJ7u3Vtai6wRdnDgFg4cqDxFMal80Z7pQJ+qw9\nIHYe6uRXf9rI9fPGkB/y9VlfY1u8z+PHCtM0eXrhDkZWFzF7fL9jvm7jnlbW7mzmG+ePQRSPPqYv\nLdpLSZGVB8swTVTNwKdIJFIaD7+yme64ypTR5Zw7bRCPvLaFc6cPomZwcVYde+q7+MvyA5w7fRDv\nravj+nljHGI9XpimydKGlQihbsy4Gz0ZswnDNE1eeGcXowYWMX1MZda1ndEUzyzcwVfPHkV5UZBn\n39zBkH4FnDahP0fC+l0trNja2Ov42wc/YEndCn404zaCspWRIJnWeGLBdubNHMLgqnyn7CtL9iHL\nIudOG8hv/7KNc6YPQhDg7dWH+OYXxyAKAk/+dTtnTBrAqIFFR+zLB+vr2X6gHd0wue6LJxHwSTy5\ncAszavpRVhTkpUV7+cYFYwgHlaOOoWGaPPfWTsYMLu41Ru3dSZ57aydfO2e0c+wXv19Pfsiq87Og\njlzywb8hvLH8AMBxE0bKJoyequfO2k7W7Wxh3c4WnrzTtWFqWna5uuYoq7Y1sWpbE9PuPH5b5yOv\nWUn65s8eStVnYO//799b0TcnDyth1snHLmQB/vC+tZdDhjBeXLQXIIswAn7rM0imdTbva2PBioNc\nOXdkVj2CYE1Qu2Kp47qHDJJpnQ83HebDTYc/EWE8+NImAM6ZNoiBFb0TEHrx15UHs9tUdXyKxKHm\nKFsPdADQHUtTkh9g4942Nu5ty3ovAH7+wkfohsmmvW0ArNhSzBmTq4+5v31hS9t2/rDzFQKjAyTW\nn+4cz2gY0YTK+x/V8/5H9VnCcEf7bl5eu5Y9eytpj2zmrmumsmhDA2xoOCJh7O7Yy0MLt2AmXOGv\n2xrGa3sXAlAfPcyIIiur73vr6li3s4Wt+9t5+DZ3hX3mmxxVXcjanS0E/DLLNh3GBEYMKCQUkFm9\nby8fRRfxmytvQBazReob+94iko7y7lsuIb+5+iCD+iusDz7HmlWDKY9PpaE1RuHivXz9vBqOhuaO\nBIs3NLB4Q0MvwvjtX7axo7YT5QN3/5JoQiWaUBH8cUw+/beYs0f8A8HRMHqYpOJHMLX01DAy139W\n+KyTCLR19y2sNUPrs62jte89F/Rnf+R9jYMsiUc890lgfMoxMY7R3OhFMq2zs30Pf2n4MwhW/6MJ\nFekomkrGrCnkdYKkEgpkz3y70xHePvgBunHs47Hy8DrrHymddTyaUDFMgx1te+hrHvzrDY9TL68H\nOUVje5y0mmnT4L9W3c/Le/6SVd4wDR5Y/yiBccuy6lM1g7Tumr/SejrrnFjYgjFkFRtatvTqQyRu\nXdfUHndqTKQ1UqqO/+RlyFUH2dDcO1/YmwfeY1nDqqx+mCbsi1i5vuSqg6iabtenO/3f3bG3z/e3\nuaNvDVc3dGqL30Cu3oUiZYt1sbCFwIQlJPx1fV77SZAjjL8RaJ+BQyp5BOdW0kMY3pdQ7aFhpNTP\nmjA+m3qCfstk1BntTRgN0UZuXfRvvFu7uNe5tOreX8/xTXvuvafZrOe4eMt8WsI4VptyR7KTrW07\nevUn84w/CZIpjf/Z8Bh7YzsR861913XDxMRECHXjG7GepfUre10nFjcSGLsSZeBOeloWf7n2N7y2\ndyEfNW/qdd2KhjV9Po+D3XZG3VS2hhRNqLy463We3v00UqmV+TcSt4R5d9p1Uguyagl9ezzEgg4a\nYo28V7skS1uOqwn3HvLbnf9VzeBQpN75HVOzha/cfy9SUSsv7sqO4gKIJlUEX5zD7VGHaFXNQNMM\nBNF+pj0GyUtOSO5zkyWRjnRH1m9w39E/7XqNB9Y/yrrmjb364TWJxpJu/S2JNkx/DKX/Pgrzss2p\nUqG1GDoldvWq75MiRxh/IzheQaRqOnsbupw6hGCEpsKlJDUrpfXuuk7W7Gy2CitJNh5yzRVb9reh\n6Qa76zqtvReSfQuj7liaw22xY+pPbVME5DRCsBvdMDnUHKUj0lvQ17VE+WhXizO7OhqKwn7kfnvZ\nl9rE4bYYLZ2uQFh52MrY+vq+N3td53Wm9iTDlGe89R47+Xn71BlN0dQedz7mhJo9O/44dMfTNLS6\nY9fXxKC2KdKLDH+8/B4e3vgknamurPtNqTrN8RY6U9kf//aWvexv7OxTgzkcdVOaC7Lb/2hCQ648\niFTSxO93vuwcb+uy3h2l2kojLxa0O+S7pXU7d3z4/9GWtASxZljvzKHmKA2tMZYcWM/zO17klT0L\n2LinlbSqs35XC3/e9i4dKYusDDF7DOtaoyypt7bHFQKWQFy5rQnTNNnZ7ppXBMW6LqNhSCWufyLz\n/bR0Jlm5y33HMwQJ1thn+g0QVd3nEjHakeyykXSUVdsbWbuj2Tm/uHYVgYlLSBbudrSvnYc62RV3\ntZGE5qaRB9hS56ac9457Q2uMpqSVHNLURYcwDhyO0NAa48N6a0vdPa3ZGoGmG/z10EL8J60ATJra\nrfdibeP6LHI50Nht/aOk8I34CKnEakslu3/Hg5wP428ExzNzBPjv59exYvNhbr9yIsm0hlxeRzJY\nx57O/QwMDONnz3/klA1OWsTjexYB5wHw4gd7WbDccgJ/4/yarFl3Bg3RRn76wROk9kzgkVvOR5Gl\nI/ZFNwx+8tQafKM2IxW1cChaw2//ZM0Ye9rI//2J1QB8/bzRnD7xyNuJAsgSyAP20KqH+NHjpVn1\ntdtCqMTf2+HoJYxkSifPY1ZJpjUK7JmYN6pMHrCLneHFrG5UmV41mcde30pDWxzdMJEqD9A+eAeH\nIoMZmJ9tO09oSVJ6iiJ/YdbxH/5mGZpu8ugPT0eRxSzCeLd2MZXBCn711OGse1IN913oSnXT2elu\n5BRPqfxy3aP0z6vi+5NvBGB72y4e2vxbzHSA6092d//LoDHu7gsu+JKOP+ZQpA653J1xx9UEISXI\n7f+7HDAQApZANVNB0jaJvrxnQZagTWgJuqIp7n56Dbphogzajmwnb3jwlbV8oWYgK3cdJDhpkdsH\nOTsqavPBRoKZva1MS3j+/t3dHPQvYX2LZ5ZtC92uWBowkIotwhBMKWvC9YclW/DXZNpyiVjVDKIe\noo157mOT9h7YCoJu6jz6149AzaToN2kOWxqYWNgKh4cBsC+6C7/hrm6PezSWw20xHv7ravwZ/7Os\ngt2VNTua8U9oQrSXnkiS1XBbd5IfP7GcoL3X1vvrGri8xnAIZdX2w6jFe61ZvpKmuSNOVbnCU9t+\nnzWeO2o7AAGl/16kEpf0eo778SCnYXzO2Nd1gOe2/4l46thnrpqh8dy2P7Gv6wArNlvCpqUraWkY\n9kee0JJEssIVvTNPj/qe0kDUeLnlCbZE1/Zq65FNTyHkdSH320c0cXRSy3y0UpE1o13ZvKJXmc5U\nF49vfg4hYKX7jnn6uKN9Ny9sf7GXXTxhRhBEE1Po3X5H0iKM4kBvwuiMuTPzbEI2Wd+yyTGneM1E\nctVBNCnBxpatmKbJwaYI3bE0CAa+wZaJ6EB3tlO5NdHGv3z4E/5r1f29+q77u5BKGhytRbXbEnwJ\nXtmzgEc2P9Wr33s79zv/d6W6s3xQjYkmYmqcxpgrCFoSrXadSZ7Y9Thi2DV3AFnaiOBLUhS2JNVH\nvJJVzjv7FgJxx8IiyKqjYfQPV2VdE1XjtEdSrs/D7wpNQVGpa445moFzXNJBcN9B7+wbyX0fssgC\nMIwonQdWEImriMXNCIodXSXoJNLudVnt2f/Xr36SSDRKV6o7q+8ZJOjEiBWgHh5i1eFzZ+OZb6on\npEIrIEA9NArINnF1x9IIfvf9y+qToCP4rHOCZCDJ7jsj5nVnXdNqa3sHuw+xtMPdbEzwWd93T7Oa\n1THrfTH17MndsEFH36PmWJAjjM8Z9617mJWH17KtfYdz7OOcxR81b2Jl41ruW/ewcywvoPQgjESW\nDd/7ASBlCzUx3ElajLBLX96rrbakJXxM1Z8l3PtCMmU77aJWyGRnuqNXmUc2PsWGls3I/Syh6J3d\n/3rD4yw/vIZaj50ZIClYH5Ep9m6/w+6fX8peKdyZ6uKJgw+gDLYitrwzUKmsntfrXubJLVYKbs0x\nSZkgWuW60910x9Ik7HvyCkJnKmpjY8tWDNMgriWIqO6+F2lVJ3DycnwjNtEcs4SLrhuI4Q4CE10b\nf0Z4uPW5Zo7udMTtu6RSn7DGLaJGHRu5LGY7pMViV6MQi5pZ3f2+p62ko1n1RGui3Xaqm/hGeyYP\nctrRMCQhW2TE1FiWJpcxKYFFBLGkmkUIercdLeSx6XuFqSBpDKrsOwrM0KN0HlxBJJ7GN3gbpiFg\nJKwFfwnNHXdBSXn+t+oeMP2byEowizAyGoZmaJiCjqkpmGlLqGa+F8GXIDB+qVufl0jsdvRua11O\nTHPvfWf3dnxDtmeNhXeMvO4OFY/PJeya0JDTNLZbdf732oeoVbdl9SMaV4lrvQnDuX+jhzVA+vQa\nRs4k9TeCtMdubpgm0lHWL/S0lYJlDoqnUwglCadMWnDr9M6SBEnD1F0hk0UmR4IufewCq2TGTyBb\nwiClZ9vldUPnUNQyUZmqJeD7cgKrRnY7qhSxZjaiYc1MbbOFZmh02U5Rb8QLQF2kAR0NufIQWtNg\nkmndIWKxwBLeJiYJLen2QUk5H3JXqtv5WCF7jBJq9nh5TRGdqS7HLNXs8T3s7NjDkNIqVN1ALGrO\nul4saEdvtcxyHzVvYkm9q5l1pSOItnYUnPIe2zy32ZnqpCJUTlLLJhzRQ27+UR9lnUNJUZTn46BH\ny0zvOxnfsC00x1qJhVWEvG5Ev0cwejSMuN3WndO+z71rHiCmxok6zl0zezYuq0SjKkKBdX6wOpO9\nyUNQ0IEgq5iaDyGvC/+Y1VltDanKp7ap94ZTzWtXoMbbePBn38McahKuGknzijUopSbJg1sYdOoP\nqV/zDIZQj2molM0cSPFJFqHse+9nbJ/4bbbVr2bfcxvJG1TE/ob1bB24ih/ffbfVgC5jpizzX6xp\nOy2rX8IUUygb0wy+fCxSyI9pxmnc+CeSnXWIqxNUnjEIfypE9+42/vT4U7znf5WioiLU+WEaP9iP\n5JMoP2UQgpzmwOL7GTD9GyhVm9jx65WEhxQTr+vGd9ZYmjavJdl5COQIReNLqDpjKIKSYv2GjTx6\n3wvsbtmHKIsMu2Yi+5/fSMXkcqLJycRUld2/XUf1haMJ2kQrKGnMZLYJSjDFLBPc8SJHGJ8Cj72+\nleqKMBd8YfDHll27o5n31tXx/740AZ/S2w+wfHMTYAmaX/xuPd+/YgKvr9iNIMCX5libSv158V40\n3aB0RG/BrWoGcbPLEXhr9zTQb6ibF0kIel4WMVvD8M4KAXx2qoy31rqmEUSjF2F8uLGBrQfa+fb8\nsTzxl+2I9uRTsGePqulKt39/YhVCqAsyoeO2SSKjYUTSroCIqjF+984u8kMKF8wcjOGLuqqwpIHm\n44/v70aVXVNLysgmjPakO1MT/AmSac1pSwy5kTc/enoRna3WjNtLCm2JLnYeyq7D6V86gW4YPPDi\nJmaMqaRRd/vRmewCe01aoydQYE/3Ps5ltkVOerZGkJkRNnXEeXbVe5AHp/afyYcNK+hOdRNIZZtw\nMuhIdlERKidhE7PW2h9faTOGp6+mJiPYBB4S84kF4hT4fM4MX2+vwIhZ791bm7YxavoUZzast1eC\npCEVtpHUUqRUnX2NbQiKyLLVUZIb57DCFFlt2DNpSSO1cY6nbQXDMKCznOSGOew2/OjmaLSG4Zj2\n7NdMB0husK6RShoRC1sZXFUAG9090zOonD2BZFOCC274Aet5lc51PhLNbQz68jTMhjkYUeh/6lxC\n47dgqDq7H11HwUjL1yQIAjvTFjGl2xMMu2ISgeIyzPc6efbFV2EwmLqMmbSEbv6IPIK+m5FKGoh0\nvUnj2+30O30cTWtWIPoGMmTObfgnLEZPpkltTlP3wQ5GXDeFSmEW3z3lbH665efW/dn3KfhS9t8E\nUkE7qbYEA848jeoL06jpA5TVnIevsgNl2Eb2PrmZxKgkgao2nv3lvdzz01/wh8hLpFIJ9JbhlExp\noWvPFnYWnsaqvRswdYNgZRhTlxAkHd+odSTXneU8Y/XQSIoGtmWZ4I4XOcI4TpimycptTbCt6ZgI\n4+FXLTPDtgMdWautM2iLxsgQxu66LhZtqGeJ8RSmIfIl7sU0TRassGznlw/pHVmk6gYJwRVcta0d\nxPpZAr60wE+XR033mgPAQLRnhULm5bZZ509LNxPIbJ8t6r0I46mFlhlt3qwh2StqbdVX8xBGXUsM\nqbKOjDFEsEkrE6FUH3UFRDQd4911ll3+zMnV2dqRqGHi463Vh5BKDuMbYR1P6T0JwzWHCZJGMq1b\nhCEYCEGXnLrT3UAZYKIM3OVWIBi8unwXYAn3LA0jneJwa5yt+9vZur+dorFNYKdC6vD4Cxq6XZ9A\nl+2c13TDGZ9Qqpq4v85x5m7Z105STyIBs8pO48OGFXSlIwiq3uOZ2fdo15mJiNOaBpNXHEf3R7F8\nVgJmOoAgW/dbKJYTV/bhl9IOqZu6jJkIY2oKCV8zb66qdWamelcpYoF1D3E1wZZ97cTVJAISopKZ\nHRhoug6IzkTE1BS7jky4qfXXMHCseYKoW85eUUQ3PSHO4U5KigVkScA0rUhVpWkcauVmRJsIu9Jd\n4ANT9REor8BXFCDVbOeSalzHzqUbQVNIR1Ko3d3IFbVWqpRoAUIgSaikADlcjq5ECBcNYMee/UiD\nsTSMZAgVe2coAAAgAElEQVQjGUIz66lb9Ti61g5SAtlfweDScvbu66BqwjzARPSlkM0COjtqCQ8p\nwlcUoN3YwMrdE62bMUFrGoSpg1RxyHr37O/QVxBEjp2BEVuBmNdN5PA6utd/CIKK2mmS6ugCOQ+l\nSCQiFqGhQrICvWUoRSfvo+n9j6gfGKH90DpKJvWjguHUbqsiMG4ZgqRbwQ32M9ZaqvEPitOuu+/j\n8SLnwzhO9Ey/cawwj7RAX8wWCM2dthAXrY/Jigyx0BrrnUBN1QxU0bXPIqnUtViC4sJThhIMecxT\nGR+GqBOY/D5SiW3zNi3C0DQDwzSz7bWSSxh7Ovez6NAypLI6lOEbaenwmEQE3YlL1/EIDUCyTUHW\nj8yqdOt8c9wN/Yx6/AD3b3jIcS5a11njJPgS+Ea4TtGeJikvYZAhDN1AGbTD0sJMwa7Hnvn540i2\ncDRVW+OwP24hrwtlwF6nuriazHqKpmchmtdG3p5w+xDXrdmdphvOhyx1DrTqtwV0bVPENhdKiIYP\nRZTpSnWRTGsIHvtzRdCacOzptBZ/ZQgDTYZ0yNIoMuYIu63/N/kmCgQr3Uta6XDMhugKgyrzMbqL\nEf0JyyeUaUtXQLPGIqHHiSbSlilJV/jSmSMITFxMYOJifKOsSKHAycvwjV6LGIoQmLgYuf9+EHQC\n45ZaPht/FLGgzbnu9usHMe2sJue3MmgngqTzSsPvqShREATQO8vpPjgAvbMcMa8bQdSJaDYpaz5E\nyeqff9RHjCnvJNVcx8hvT2XIWTcQrCjC0HR8Q2zbv01oAbEAIxFGEE1SZoqUaj3nYZWlgIDRVUb9\nwu0U10xi+BWXUX3haEzN4LSRYyw3V7iLqgofCAZjq/sxwJM4URAN3k09ab0XyUICYhCtfgSCaODz\nPBdBCFptRYtIdyTorHuHEdePY+T1c8grr0HrtFaoC5LGa13/a9Wn+UD1kxcIkz+8hGjzRiJ12yke\nV4nZWo2ZyEdtsFauC/6EO8nQFQrkEnxS376rT4IcYRwn+lrcdSS0JToITPwAsbAl67jXuS30cEQf\naM+2cze1xwETqayO2m7LKWwa7uPTdANN8URYSBqHbDuwTxadWSzgkJPgSzjmCsg4lU10w6S1K5nt\n2/BoGL/66H95cfdr+IZtQS49TG17b4FuNYBr/pJTiIWtGCnbqZjRMOwxaEm4dURSrkbRmMg2TWSE\nrVdLKPIX9iKMTMy/dY1KMq2hGabjP0jvH2t1K2Bf57H3aq22GcMmE9leTJZBQku6z05Oo/pcYsi0\na5om27QPneMZ56Smm05biYjlx8kQxsHGiDV+ukxKMygJFNOe7LSc3vZzUvQwd07/PmXBUtY1bSCu\nJhyflqnL6HEr/YMYtLQMQUkR0ssYUTSUAsEimqjQ7BCQqcvkh3yOWapN2YNUbI2RqfkcX1Pc7Ka2\nvc0aE122khp2WvZFQUkC1n2Zmg9Ts7UyOY3cf58zjqbmw+ioRK21Yk3rY41Zi+wyaEk24Q+a9jWW\nEURrGYDokzH0JHHDevam6sfUPUYSqRMpKCPKIsm2CPGG7KALQdIsTSqlOSlDkkYcVbfGNuy3828l\n8zBSOr5CGUFWad/QCKbA2LIa8keU0LlvFSm/pVH7NR9SuJrI7iRp22el2d+JEixGSDRhJMLEGyJE\nOzsQ82yys18fMxVET+mIPgnRL5FuDBJr2YnePJBgXn/UaJp4g/Vd63ER0zQZVjCUkmkVtGx/hdCA\nAqSgQn2T6tQHOBqGaYhgikzNP527Z97Za6w/KXImqeNEz7QaALXddeyvTzBr5HAUWcQwTdbvamWX\nsRTBl8I3YgO6foZTPsuM4o0aCUY4rLaR8XRs3NtMR0RFzG/HN2wLhzPWJU8UxKY9bZgFUTAES1BL\nGrV1liaiyFLWLNgRut64bDUAStIS8IZMXXO0F2HEEmqfC+0aOj3CWe5hOpE0MGTE/A4E0URrHYA4\nYK9DJJpmsGxLA+vbXHNQdyoKFPRpt3c0DJsAvzr6Mt7Zv4yo1sq2/W3UHe6iJXWY/V3uoilkjb31\n3fQvy0OQVYxYAUbMcjTkF+qk6tyxUA+NtGZyeKJNMuTWVerY8zNrVvwnrcAUVcsMI2mOZlPb1kZM\nsEjQSAVI+ZNEEylWbWtyxj/SKRMwcQiktjlKoFrDTPtZta2J4tJimuIttGh1jqCRuwdxoD7G2KKx\nLD68hHV1u0lmggt0mURHPkopSOWHEIuaEUSTZERm8YZ65GQZpilRl96NMtg1H+UXKZjddhK+4u3O\ne2dqCoYdAXQgtpe60IcgWOGaq3c0kd49Cd+kd0FWEfwJS3NTfZjpgKUlFbZhJj35izQFENC7SlGA\nDc2b2W1rST1hZhb3ZQhB9SGHFEJVFax78vcU1BTiZwiioZA+WINv8A4aBx4kujRG08OrkcVWAkWD\n7f6bCIL1jI2khKabTnRVVypCLKVRSMhJRGimglSeMYT6hS+jbA4QGlhAyhQp8hcy8NRx7PvrWjb+\n+SEQBUZdWU1Mq6by5MvZ/7s/ASZyno/h10wkXF1Dqm45+xc8R94IEX9pyF33YgduGKkgwUFhgv3C\n7HxoNUqgjGDxUECkIDKRwZcdpv4v2zE0A8HMY8DkGkYVj2Br/21IAZGSSVY+MkO1EwymLcKQ++2z\nfJP2+IV8PvJ9R89BdizIEcZxomfiPt3Q+fna/wFg+66vc+NFJ7N002GeXriD/hNbwQcYEgnPegBv\nDLXgIQwrB46LB1/5iEElJb3CYREM8kM+IvE0O+vbCFRGMOMFVtiepNFt57/xKyKm5PVhZMI0M06x\nUYihbqTSRkfAb2/Zh1LtWWVraxjrdmVrSQDN3RHIiJketnZB0jBVV8CbSVt9t4Xwxr1trGhfhNLf\nDQWNZKI5PGYYPVKElN/pjpMtZNvadRpbU0gFGnc89CEgIA/cgdLPQK0bgVK9B0HS2LCnlQ17mwlO\n0zE0BdNelCVlTFKOTV9xZtWCkrQCADIhlgemwIS3SeopVDsiTAzY8fSyiqiFaEtYZq3/+v1ylLGg\nNVdbZORP8r2H3wfVj2+MhmkIFuHrisfcZIKkYep5vLu2Dt+QBFIFNBa/j8+ORu3qMvn579ZTPcKA\nEvjj7j9jyvYs3RTRuotQALnM1czSCZln3twJgDK8nEhpI2JGjusy4aAbTpoFTcFI52OqCnKFu+pY\nDEb57RvbMUxLQImhCIEJS6wuaAoYMnpbP+sa2+8gdw90TJ5mMoxihhyyEBD4cuVNPL/5dacdof9u\n0HA0iAyJV593CmJBO4KskvxoNGd8fQ5rDlkmp6SQoPqrJxEQQ3SsPM1qt3wHcIARX/4acngHkhxm\nyJzbMFWLgMMnjySUCgHbCSlBQMNMhigcX05hjbvZ0ujpVwKQ5ytm0KUnOcdPH3cm+Wsklqo6pefN\n6hGZlMcPf/Rz/ufljwhOfTdraIecfROyJqLb79qgS6w6UzunYHRZ7QbS+YxWvkno238EQG0Yilbn\n46Sykfxpg/Xe5o8occcd8JsWKYghWwuzAz0CviMvuP0kyJmkjhM9NYxme/EUwOrtlkq/z07Z0RW3\nPhrTELPWA8Q0T+RSH07NDARRo7Y548j0HJd0CsLWiyLmdSGIJka0yI6McV9cRRYxPITRU8MwNcX5\nMDPn6hPZi9MyUVKxPhbvtXt8KkLPWO8+2hJN2THBdcfSjo9CShTjl3xE7Ygp12nXH1oH91lfdwQw\n7HmPTUKZ64yovZgv0yfJ7QOaYglYKZFdRpO5cJplMhF8KQZV5CMoKUxdol9xIaYukVCTpDSjlwYk\nqnl0pSOkdRVDsAlS87k+EZs0BVm1Z36C7SBOO/0XBNOZFRqpIL1gR1jV19rCV/aadATQ/Jjp7DUp\nGQIEMDqzd5zLmKRMtTdhmLZGYESzU5+bhuSkIDHV7Igv2bTqccYea11Ov9gpngpERhinuj8xmTVm\nEN+ddhWTS6xlzo3avqz7dbQ+XxLBH8eI53PV2aO5ft4Yrj9nalYf8hU3Q22GCH1DtyJIumvCsutD\ndgMA8ny2hpHuPe43zLOiP4JiftbxylAFXzt7FP9x7TR8PVwERrSIQRVh9/304AeXTWf2uH4Y8QKM\nuFun939ZEqgqDZHaPg0z7XdCrzd+uJbdj6+l6qxhDM0fglQ3BQyZorCP//z66VntZL6THGF8zujp\nw2iI9g4DzJCDKGUMltmE0Z3yOK+dUNc+nOJStiD0oiDfeoQZk4UeKbY+fiXtCEFJBiTVTQbo+BWs\n8yWhsPsh2W2kNDvssqvUmg3bGkYqbc+OPfDaoTMLpRw7dg+NwNR8Vqih2Nu0JR6YSZG/iC579ud1\nzJaGM05AeywywlfzuStae2hOVeHSrD44JKpbgtBn5qEKcfucq2GcNtZyHKKkGFQVRvAnMdMBqsvD\nln9BT6NqRtaCO61xMELamrZ/cOhDN4eRprjCTk4jD9yBGIw54ZboEoIvbZmQQvYCxcysug/CyNj0\nzXQAI9l3umovQQCOoAHQu0t7FBYI+qS+NQy7jxnzHcDs/jMYFHfNqpnV1hmU5RXafXClp5EMUxTO\nlqaFRjUPnH4PY0tr+GrNZUiiyPjhpVQXZffPebaagmlaa1YEAcx4mJrBxUiiyPQRgykLlCCLMpMq\nxnN6v9Pd6+PZAj4gWfdZErLfJ1l1vpOwL5R131747b1A8noQRnmwFJ8iMbgqH9MT7ZXeOx40n5OO\nRu8qRcKtt6a6wtJeDYnUllMY4B9ivZeeZyeJApUlIYxIKckNZzghv+ef/0V+/L8/Y9bsU7l50nVU\nYKXqLykIUBLOo+DwaaS2zbBuJWb199PuZZJBziR1nPDmBDJNk4Zo741aMoudRMkua4hZSe/aeoR+\nWoV7C1JB0qwgyZ4mKSAUykT72I7PVBCjuwQpvwMxvx2js5LX619yzgmBBMqAvWjNAx1tYES/MtbU\ntme1ldRS4AO1biT+0WudKKmkpiKIJnpnGXpbf3zDN3kcxiZyv/2YJugd5cjlDS7ZZcpoCromZt2n\noKQQ1TxSSYHSQDFN8WYQNQTbOW9qClXFhXR67tNLQJkP3Aq59Ttj6RdDCAh9ajkAQTGPLpoAN9QV\nXabAlw+m1a+64GKEtIqp+qgsCUKHhGqmSau6syq3Mj2eA7X9CA6tBaxEiMpAPP2zSF0qbUSusHwr\n7sI46/n5hm51H6pNGHpXH9sVO2s4BFJbZiGGO/HXrHVs8pBNGKmdUzBTHmJRA4wqqGFXtxUSbSbz\nrDDqPoRkpm+GZ0+Jr9RcxsPbNwMtWWOZQUAMuved6U86QFFpNokZpokiynxnwjezjs+pnoUoiAzM\nH8Db+5ew0dGIBNAUh6CMeIEVzAGIgsh/zPwXTNNEEiUi8TTPYq3OHhQeTO0u3VnAWJIXphsYUFrI\nLl1CUNKIpjVpKA4UAJZ14Cv9b+Cpv+5AGbKNEZXucwhIQSezzpSKCUiiO26Z8GC1YRh6mxU4kVnT\nlN45lSvPG41QVks0HUUUxKy8bFcN/xq/fnkzCVxLgCSJR9xPZu6g05g76DT7/oWsv/5UBUY0yuDO\neezYZ1kxPqudNXMaxnHCq2E89PJmlh305sI3eWt1LRv2WGaqhO7amJdsbOAHv1nGDx9exs5GTwqM\noxBG5pwv4LaZSbFQG7TTPkiuQDZsQZMJE21OWmTmnWlKth0YYOzAKlcjsDWE1qgdhaTLljARdeJJ\njZitTZi67NEi3BBOMRTB6C7FiJTY5/oQ1rpkCUw5TSaSRzaDpFSdPMme+fkTjoYRkP2UBSqsuPxQ\nt3WNrWHEYgJkZqGiq2GYhoBf8hOQ/S5ZeUgLICSFARPfiI1O/itTU5BECUH3I/iTHEpbfhwjUoxf\nkZBQMFDpSHXgG24984BZCAjEunrvlmZqiiOwM2QBONpeJmIo65oMKegKqe3TUBuGec65c7zivDyM\n7jLmyNeS2jrTLWObpExDdOzhXnxp6JepqL2MxPrTMZNhR/CqDUPRGgdz57Tvc8fU7znlja5S9O5i\nrj3pq9b9emar6r5xaI2DLU3UhALJ3r5W9RKGv1fK7SPt6xGQA5w9+HRqSkZy3divY6ZDVBRnk5Bp\nWuG2smffB1EQHeHtTTJZXZ6P0VnhaGahgFWmqiRkmQP9CcTCVoRUPpVhV7vpX1COmQ6S3jWF2cXn\nO8eDinsf3zz5qqy+OyHzpquBC4Jgj69AMmUwp3oWXxx2DuDuJW/1J8+a3HggScIxbUCW2aUv8zez\njsqnF4Lmd+r6LJDTMI4TXsLYcOgAgWKPM1jS+OP7vdMyI2pZi98+OlBL5vtyBGsfZqfMMcVnkAZS\n26chV1o+hiitWAI0EyapYCZsf4Q/AZjEtThiogitYTi+/vsxRZ3+VQqqItEFDC4rwUzYaQWCEaev\nAKP6l9AgKhiiRgroiEUhz1LfC0tLaAIKC0UKggaGqNEMmKmQa4bx234a2TaJ6bITIRKc/L6V7E0A\nn2l9GPFuxel7hoiCcoCg7MfszkMq6CAw5V3XBxLVPFFNacyEPV6GjF+WCEgBYpLttLZJZkBxMeWF\n5ZRVRjncsMddh4IrkAv9BXTq1jM1Yvmoh0YhjxbxC3kkpC7WR62QWb2rlCADgE6MVO+Pe9zgKjZu\nVjENwVmfotYPR++01kQU0p90ohgj6AkB9chSI1JKuTIQv5yiRavP0h5GDypi5dYmGpu1LDt55h6O\ntD+SLIkEfYqTjfULY6vYUduBblQwqDLfycT7pTMMJ6tvLDmKaVVDgGx7uJkOotZamQgQNQomlAP1\nWRqGqAcJBrJFzbFsJBXyy5wzbSBD+xVwsDHCYqx3yegu4dSThvQyczntiQLzZg2hrDDA5FHlmILA\nTsqI0UhC7GDK6GmcMq4fi1f5EPyWGfCU6slZRBgOevrr6eqXJp3G3sV7OXf4ab3a/ebYq3h++4uc\nVDwJIRxkUKU1+fnxNVN5fel+5kzMznA8fngZm/a2MbgqH38fPgZZFKksDvGFkyqtRcJHwNfOGYUg\nwFVnj7Lv3zpumCY3XXwym/a0UlHUhz/sOJAjjOOE1+mdlTAMPLmaTKSyekQ79YaXDARfAqmkCcGU\n0ZN+O1bd7B0JBc7MOUMY86aP5C9rTDd1sZy2BbLghiEaohUWKqvopo6oBQCBkeo57PIvJL9fC/u7\nLRNKcSjsONsy2TIzff32vAk8vHk1TVFLW+mIRyAPJg/vx1nDJnL3ync4eVQeqxvfdrprpv0Y3SWY\nhkigooVo/UhL+NtOVG+4rtLvAAB+wRK2sYgCPpswbNJSBL/luI8VIgZjDlmYhkB7JIUpZaKabHVe\n0jB1GZ8iEZQDCJLtRLcjdvoVFfLtOeN4v7Z3csSM9jG8tD/rmi3C0BqHgO5DlkTyKSNBA43GXkxV\nIb1zCsnBlmTuy+cwdXg1GzfWYkRKHOe+1jDcIc1pNRXUh8vYF3P74kSSYZkZ7vnWDDRjCk+/s5kV\nmvuuDanMZ+XWJg4191jImVmUeAT7gSQKWcJRkUW+deHYXuXOmzGoz+sD/iM4UL3OXc//kh7o5XQ1\njmEZkyAIzla5M06qpHHdaHZ27USrH8k3bh5z1GsvPc3Vyu78+jSW7/LxwPpHOGfw6cz4wjgg25x2\n/qjZ+D0+hnDQoyF5CCPPH+Cn59zUZ5tTKicwpXJCr+PV5WG+c8m4XscHVoT5169NOeI9SJKAKAp8\ne/5Y5k6p5r+eW9dnuZKCALdcNt75ndEwTNN6v6bVVByxjU+KnEnqOOENq80IIjFtxzlnVvKWHsY3\nzLPdo3fXrSorT5NsBDET1voAb8RGFjL1KVabhUHLFKE1WpFD/lEfWZu/2AIZBGQziOCP4x+zyuqj\nZs0mFXsmnyELsGbw6ApGKmjnWXKJK98fxC/50AXLfNSVtMgvpAQJyZaAbI67EWJg29AN2TJlKBGU\nYZusML+MIOuR7hogKFpj195qvZK+wdtRBlob+CiCD0US0Vtck1p6/1jSuyfRGU31zjBqL37zySJB\nOWiNn5J0MuTmS5Y5r6iPlOgZQd4vz90vORMlZJgmJZnNHsBe7CaSyKQf13rPeDOOVL3Ds/+y6X52\niizik+2oqGSIacp8x/4N1kxXEAQUScHXY0/monzL1JPZutbnmDhs34PZt1SWJfFTRc18UgeqpId6\nXXM8W9VeUHUuh1/194raOhaMLB7G7JbxTCw52TmWeV/KxGqK/IX4PTnegh5S/LTb6h4vvCY3fx/5\n546ETD6549nO9+PwT0UYa3c0H/POcQBbD7Q75ZfVr+LB9Y+x6NAyHlz/GCnVE29tE0Y6Ypt1MkK/\n15oETyimLTTDTV/AsGeUYjCWlSJEtGc8gqSBnKbbb4UaFoes8hlBKYatqCKvfdtPHoKiWnWCE32h\nkD0LvnnC9c6MxIgUIShppPI6pIJ2ezcw2UkpoAzZSjRtEUbYZxGGX/JxwEM+Vr9sG3pmEZG9JsBU\nA4693IuS5BiqJWvG2NSH5q0IPkvDiJSSPlhDet/J6C0DMboqSKV1d92Eo6VpmJpHwxCsPToE0UBr\nGkSRYtmqi/wFWe3oHRVkhG2VhzAymkNbV5Iqv0tahm3Gc/ercG1AWtMgpGgV+b48u+5K+xpXewCL\nMBz7sikwrGBYFqH0lagyg4BPptJj4y7Jt94Hvd0itUtGfLHP6yRJ+FSE8UmEF1ihtgGlp4bxyYWZ\npBl07l7/8QWPgJdf+hOplKvdavUj0CNFTAudC1imrAwET7boz5IvdP3Yd9b07rkuHmX/9Z5wNYwc\nYRw3uqIpHn51Cz96fNUxlVc1nfv+sMEp/7udf2ZXxx5e3P0auzr20Kl6NqXxxzFNHD+AQxR9PS8x\n21fR1OheJwSijrlletVkTi+4xDouq/g9+xP0K7EEXc9QSO8aiJ6z0YyJQBEUfPb+CacNmMWY0lFO\nkYxQcyN2rBfvi0MtJ51cUYdUbS0AG5jfD0mUmFHVW6XOxPR7yUnvLkatHU1Bno/UzsluWU1mgDrN\nEaz0CAkFCAhhZJto9KYh6K3V2e1lNAwlZa9lsO7Xp9gaBm5Kc729Etk28hb63N3xUtunkd7t9qsq\nz1Lj8+QQX5w5BIAR1YUUhcKOZpcJLvBubas1DsJI5KEeHENB8yn4MpEwqp/k5lmkdkzP6rsiiRT5\n8537KAhlayleQpgwIjtqKuCTGFLlRjBl9pEw4wXcPv7HzB14GtPHWPcxdog7K5dFkZKCQK/6jxVe\nshk+wHoX+9v5lE4aUuwIuglcSGr3RBRJ7mXGOnlYySdu9+knH0GNt1G37H94+GFrkezvfvcc3/rW\n17n22q/y5JOPAZBMJvmXf/k+3/jGV7nmmitZuHAhL730B1pbW7jllhu59VbLpKS39yO9/QsUBQp4\n+unf8q1vXcOBxffTtOnPAEwaWUY61sbzj/yEa6/9KtdddzUNDVagygsvPMM111zJN77xVR599DcA\n3HLLDezcaUWfdXV1csUV8wFYuPAv3HXXndxxx//jtttuIZFIcOut3+G6667mmmu+wtKl1t4oM0+u\npLtuHQcW/4qDSx5g6YLHiMfjXHHFRYSD1viVF4hcccX8oxKP6DFJfdb4p/FhJNVjZ3bg43eX09zw\nN9GfsNIhZGyits9h6thiNtmLuX1qCWmlHUHWkIUApcUSHYaAronki4VowMk1QbbssGZANcUj6U5b\nAk0IRrN24qosCvOf189gb9ce/njQTcAnKCrhoEI0oeLvQRhCOjOzFcj35dOWbGdcmWsHlkQBvbOc\nfMqJ2CGTGfIaXjSEfKmQiN6F6E+iNQxl9BlW7Pe4spOy9m8A+NcvzaIsXMi6Vh8v77MIRj00GjNW\nREF/H60NFWiNg5GrDmJqlvbgmizcmZR6aBR6exWhYWEUj3r+X9+awdqdLbyyxNK4fvSVmfxqxyLL\nxODJwqrIEtjpHjIRY0a8wJnRF/o9C6aS2TP/fnmVfHvcNQzKH0CRv5DZ4/tRWRwimlBRa2vQmgc6\ncfEZk9SVZ47gD+5eRUii4BAdgJnI1mgAZFnk4hEXsHhjA2r9CAKTJX51y2zASo8+sMJN5zBxRBnX\nnl/D03aW4IBP4tLThjFpZBmSKFKc73cWjQZ9fgRB4LovnsQVp49g+ZbDbD1g+UkkSeC8GYMYNbDo\nmKJweiLgk5EH7kAqaSSWH6BqqDWrrTIMXm1bReUpJqYJO9M6yiCVuLiLx3Z/gH+CvTo56OPt6Cre\n7rFf16SKcVw6Yt4R273pplvYu28vj//2BYJ+mTVrVlJXV8vjjz+LaZrcccdtbNy4gc7OdsrKyvnF\nLx6wxiIoMHWqyR//+Ht+/etHKSjIfg6yJHDZZV/m2muvJxJP88tf3M3y5Uu56eJZfLTgl3ztm9cx\ne/YcVFXFMAxWrlzO0qVLePzxZ/H5fEQivZOBWnDf5a1bN/Pss38kHA5jGAY/+9kvCYVCdHV1csMN\n32D27DlMGGDyx9YVjD3rFmKqzIzRhYRCISZPnsKm9av4+Y0z+fD9vzLo9LlI0pG1vIwyciI0jH8a\nwhCOsiFRX/i4zYISzqY1JigppEQxWo/V0uEwELdCJwdXF3KQ1Yj57ZSZZdZCsbi12tcvBtEAjZSb\nUVP2kySIqUvZ2VptDCjLo6L4JA7qk1m2qd6J9Mns+RASLPu8IsrcOP4b/O/WRkBFECxhmNbTjCwe\n7tTnUyQSKZPx+kWYFftY3vZ+VnsFvgIiiS7MtB+haYwznhUhd9Z77uAzKQ4UMaLKmtUWxz0fph1m\nmZlBqw3DEQJx1EOjUEb3bVM3U0HMVAhZErPiyPuV5hFQ3FTNAyvCBHblEVNSrjlQl/ErIrKc0T7S\nVuJDXXFsw7Loef378D9MKHcdwZXFlmDND1p+ogxZgDXmfkVi3PBS/uCJjpMlIYvo+oIii4SUIOoB\nq62AX3JCUHuGogIMKHeJLeCT8CkSowdZ2oNXQGSISpFFSgsDTsglWEQmCAKjBvbhwzkGeLUFr9lE\nsl7OvssAACAASURBVO9VEAQrd1MmlTnZ35/8KUI8JVEg6Lee2+rVq1izZjXf/OZVmKZJIpGkrq6W\n8eMn8pvfPMgjjzzEzJmzOeusU0kkbN9cH2q/KAqsW7ea3/3uOVKpJJFIhFEjRzJx4mS6OtuYPdva\nr0NRrDFcu3Y1X/zihfjspd35+fm96uyJadNmEA5b74xhGDz66ENs2LAeURRobW2ho6OdDRvWcdbc\ns9mWCoGaxh+w3rl58y7id797jtmz5/D22wu4444fH7Utx8T88cP5ifFPQxifFD0JwycqpD07waV0\n2xYqWInNArKPZIYwZBVREBDkTNK6MvoNHsRBczVSSSO0jUJHdXwOQTFIDEgZSQT7W/RLfuv1TuYh\n2NpFdbg/pw1w4+0VSeG2U77FB6/+GZ+S4ltf+CK/XW9F0fQXxnDR5AmUBIooCRQjmNZaDEEQuOak\nK1ENFcUjMH2ySCJl7fw3INhjNTBWfDxYkSU+z4KjYr8rdMaW1jC8aIjzO19xhWrGz+Bsh6r5SO+y\nzFmyLGaFFRrRAsRwt+MjUGSxVyoW78zdp0j4hSBxOe5ESpmqgiKJ+GSP2c7ug1fI5UtFdGtdWX6D\noyEv2HutBVhC0DsuVjtiVj/7Qk9C+Tj/gN/TRk9HcpZQFrPr9fb7k06eevVBkdAO1aAdquE/7zzz\niOUWbajn2Td30q8qn3/56iS+c7+Vb+rWG77gEPCngWmaXH31tcyff0mvc0888TwrVizj0UcfYteu\nzVxxxdVHrEfXNO6//xc8+eTzlJWV8+STj5FOW0EeR2oXeo+hJEnOam/rehfBoGuefeedN+ns7OSp\np15AFC0TUyqVdgg/U3OG/8eNm0Bj48/ZsOEjDMNg6NBhHA0nUsP4p/Fh6H1klz0avPtX17VEScaz\nBUU0kw7DdmIHFZ8zSxWVNGWFAYdUTE2hUClCNoIIgTiabpA0kgiGveLYFmppM+msp/BLfnyymJX+\n4bwhczllwIzenTVk0tu/wJTKCc5MMi/gY0TRUEoC1uwzbM/sg36JkBKksIfDt7TQ9jvIEiXB3qYT\nE3vmbkieaByyVro6foi+ftuJ5zIC0jtTVmSRoEf4pXZOJbVthpOCWpFEZ9V8xmneU9AGpCCC5K6+\nNhNhEARnbMFNV+FdxPSV6utJrj271/0eCflHIAxJElGU7D7JkpBFTn2h5wrcjyvvHfujOa5TPUyw\n4SP0+0Qi0z9ZErOIsCeZHStCoRDxuJuwc8aML7BgweskEta3aM3UO2htbcXv93POOefxla98jW3b\nttnX5xGL9Q56EdEQBCgoKCQej7No0XtO+YqKSj78cBH8/+3deXxU1f038M+9d2Yy2ReyEjBCEAWM\nAsomNMgiQcKSFKIsVm1Q3BGiCNIifUqr/YHlKTwqlmKlVV7Sal36M6htQUULYl0ALaKCYkggC4Ts\nyyz3PH/cmTuZbDMJmSQz+bxfr76aO3MzOXNk7ne+59zzPQCsVisaGxswdqz2d50T6FVV2he6pKRk\nHD+u/a133/1Xi7/jVFNTg+joGMiyjM8++wTFxdpNIddcMxbvvvsv2B03ljTUu9qakTELv/jFz5CZ\nOddjPzm/HIQEdf1/8z6TYXT0rozqJgHjnY8L9LBvLUqFMfkkyqqrAARr+0xD+7Y/69qh2Ft3CCkD\nTZgWNxifWBxbVzrWBBgagmBVamG122GxW2CUItEAINhkhFkxo1GthxxdqxW6C43HgCtM2HcmAWeh\nZQfNL/JOt994hT7xuiLnKrzz8WnccO1At3Puy74Sez76AZlt7A54X3Ya3vjwe/w4fTAMBoGYE8lI\n6+cakokLjcF31d9DrQ9zK2kAuLKv5uWTw4zux4C2O194iBGzr7sUj2w76Og7GUMHRmHssHiYTQom\npfXHZ9+U4e2PtbuvDAYZE0Yk4lRxNW64Vpvwbn6hHRAdjeLSH/RbcZ3lLNwDhpZhNL1gpQ2Kw/Uj\nB8JkkPGP/zQpid6GfpFmTB2djJIL9YgMNeHAl45V9HbV7ds/oF38w4KNuHHcJahtsKG8ugH9Isw4\nXlDh2N/EFfhWLRyJ4wUViPOwwKpp37dW7mHdbdfi0LESDLs0BufPu/YM6cqAMbh/BDLGDsS1l7d/\nf78zAzIoknv208kyFRERkUhLuxq33bYQ48Zdh3vvXY5Tp07h7rt/CkALKOvWbUBh4Wk8/fQWyLIE\ng8GIX/96AwBg7twsPPzwcsTGxmHLlm149JbROPhlMa4dMRBz5mTj1ltvRlJSfwwb5vp3//Of/x9s\n2vQ4duz4PYxGIzZs+A3GjZuAEye+wdKlt8JkMmL8+IlYtuxeLFq0BOvWPYp33nkL11wzps33MWPG\nTKxenYc777wVQ4ZcjpQUrXbZoEGDceutufjt//t/UIUEUTAE98y/1vE7N2LHjmcxffoMj/20cNpl\nMBpkZE1qPxPpjD4TMOwdDBhNh6TCgo1AowrVUacJySfRqDrKYjvmHAySAbPHXIG97wNh4SrGDkvA\nB582aIvpVAVGgwwDTIChEjZo6WqwYkY1AHOQAaHGYJxvOA85SCuJ7RwCmjUyDc99qU1sR5paHytN\nv9p1335yXBhyM1suakrqF4qlmcNbPO4UHR6E22+8Qj/eMOVBt+dzhs7FocNVsBamICjW/QP/iwmr\nUdFYqd+R5KSViwaCJDOcMz4hZgNuv9G9fUaDjBCzAXfPc90jP2RApB4wjI45jFszXGU0mo+qhAe5\nByfn4rembWotw5BlCbdmXI7PvynzKmBIkoRbZriX8zjwZTEaLPYWGYZzTD9nyhC3x785XYHf7NJq\nGzkvnsMujcGwSz3fOdS0nERrQ0uDkiIwKCmixW2YXRkwZEnCzVMv83hecJMMo6mLmcN47LENbsc5\nOQuRk7PQ7bH+/ZMxdux4/TguLhxlZdWYP/9mzJ9/s/74ZQOicNkAbUj1jjvuxh133N3i7w0YMBBb\ntmxr8fiSJbdhyZLb3B675JJL8ac/vaQfO1/vxhtn48YbXZP5kZFRePbZP7b6/mbOzMS/votCeVUj\nJqa51vwcOfI5rr9+GkJDPe9pERUW1O5n/WIwYLShpt41BilLEiRZdZQB1z54+hyGI8MwyAaYFCOC\nDWZUWbS7Jupt9XoZa5NRhkEKgiQJ2GXt22WoY4evIKOCEGOIXozQVurKDgaEuYJBRBsZRncINgQj\nvOpK1Kv1LdYGRAZFtJr9yJKMX123Fke/rcCfoN3RpLRykfNmYri5pkUcASDM6Brisl+Id5UfaSXD\naHUMv5PXMOeF2K4K/XZGp7aGl5rOJ3h67821ty6jPT0xJBXUZsDoMyPhF8dxyfrd7zbho48O4skn\nt/Rse9CXAkaTPbj3HzmDlIRwpCS2/MYuhMAbJ97BB7YPIIeNhloTjeo6K2CyA6pZ2zcZTSe9HUNS\njrUNEaYIVFq0Mc16W4Ne9MxkUGCStAuW3VAHBUBEkHaRM5sUfW2EWh8CUedaHxAb7PrW2XSSuic4\n747pSOXLaHMUgg1NbkFu5SLqaYiitQtM8zH60CYBo+l6ioSQJsX3HHNMVbUtV5o3L/zmrbYmwYG2\nA0bTi3dHq4h6muNoS2cDzcVoOiTV1MVkGH2B89+i84q1YsWqnmtMM30m1NubFK/Z+dZxvPHh962e\n91Hxp/jn6X2QjFZHZVSgqs4CSCqEKusZhr4VqbPOk+NiHmkKR621DjbVhjpbvV6O2qBISIrSAkGQ\no8rpgOh+UGQJ/WND9Q2Y1NpITBntWk0sSzIWXp6Nm4e2vBOkuzk3u4+NbGXvhHa43XrZygWvMxmG\ncyhh2mhtTsOstFzwd/nAKIQYQ/QsbdwQbf5mUFIrmVonr2FtTYK3J7RJIb6Oftt2ZkfhIZ3LGDob\ncDojMtQEk9G1SND576Z5JkbuenP39J0Mo9mQVFVdy2+ZAHDgzMf6zwYDYAdQWdsAqZ+AJBRXcT/F\nimnXDMC732jrAYyKI8NwLAYrrTsHi90CYXUsvpMkJEVG4kgFYIMFI+PSMOuydMxcriA4yIC3/q39\nK8m6ZjRmDnatvgaAHzW5lbYn/XTWMMwcl4Kkfh27JdKt5EJrAcPTraetPJ+SGI7fPTCp1QvnM3np\nsFhVRDjWMeRdcy++vXASI/pdgYUTbK0Oz3T2M9pehtHWIGjTINGZfQqeWpEOo6HjLX4mL73TmVRn\nBAcZsOme6/Ry448vG49Gq/2ib+vtK3qohFW7+mzAaG1hnipUFNWc0Y8jwhSUAaiu14afTIoBjfq2\nmlaMuiwW755wZhiOW9kM2sX0bK1294yzrpIkAcFG1wRs7ojF2i2pjv8Cd111Gw6e+Q+mX3pdr/1A\nGRTZbeWxtzxlGJ6+ZbeVgUQ0Wdg2Kj4NpxtO49p+Wplqc5M1b0GKCVc6VrV39Vh+cHs1mbz4wHcm\nYISYO/ex7apd1zoivEmpE4Mic/7CC66Pf++LGH0nYNjdO7+2WcAQQuBc3Xk02i0wS2FoEDVw7pVS\n3dAIAwBFcmyPaTVBMjZiQHyYPodhUrTnzAYtQJyt1Uo06HWOmq0JaLp+AQAujbgEl0a0Xk7a3zXN\nMFobjvC0uYs3m78YZAPuGrMEZWVtlWloX2djdHsXYW8+7ryAUlt6X7joSwGjWQH+2gYb7KoKxXFP\n/rbXv8TnZV/ANASIkhJQLGrgqAQAq2prGTDMtQgJUiDJzoDhvgjvrVPawh1nwDAbFZQ3qT/VlzS9\nM6q1DKM3jGl3tAKrU2hwOwHDizGFjlQhpb6hX4QZZRUNCA9ufYOonuTzgLF//348/vjjEEJg/vz5\nWLZsmdvzZ8+exerVq1FdXQ1VVZGXl4fJkyd3eTtau622tsGm1zb65OsyKLGOrVCFNrlrNDqW6jdZ\nawEAsJkgSY6tVx0BI8jgWEltcJ8QXnx9Gi6cicDll0QBF5IAoNUKr4HMLcNoLWB4uGh2xxDd0IFR\nmH1dCkYPbbmlaXuS+oUi+0eD9HpOD908Er/9y2GPv7d68Sh8W1jZar0o6tvumD0c//jPacydeGlP\nN6UFnwYMVVWxYcMG7Ny5E/Hx8ViwYAGmTZuG1FRX0btt27Zh1qxZWLhwIU6ePIk777wT+/bta+dV\nO6fVgFFvRUSIybUK3BEYZKF9iA3O4W5HUHBmGMFKCCwAqi01gKT9jp5hKO4BY2hiIpKHaIHiipjL\nsHrMciSHJnXZ+/IHngKGp3jQHd/BJUnCj9NTPZ/YijkTB+k/jxgUg4HxYThdWtPupOXll0TrQYao\nqZgIs77TYG/j0wHUo0ePIiUlBcnJyTAajcjMzMTevXvdzpEkCTU1WgmDqqoqJCQktPZSF635HAbg\nmvgur3Ls1OYIDJLzVliDM5Boj+sbGtm1eYoaa22TDEP7HXOTDCM18lL0D3Wt1gSAS8IHtJi/CHSe\n5jA8DUn1ghGrDvGz5hJ5zacZRklJCZKSXN+mExIS8MUXX7idc//99yM3NxcvvPACGhoa8Pzzz/uk\nLc3nMABXwCi+4Cho5pjA/vpULUypgKw4Aogji5BaCRh6kHEU12saMG4amtVr73jqTp7u/fc8h+Fn\nfehnzSXylk8DhjeTfvn5+Zg/fz5uv/12HD58GKtWrUJ+fr7H34uL81yDvqngkJYLuyRFQVxcOGzf\nOfZWcFz8nWXHTWatrr/z8eEpcTh9BLhueAr2lR2BMFmR0j8UZwAMHhCLuLhw1BtdpcFTkhIQE9yx\ndnZGR/uiu9VYXcG6aVunXjsQ+z45jauuSEBkWMv/PjdOuBRvHTyFMWlJ6BfZflG+1l6/pxgdE+hG\nk9Kj7ekNfdFbsC+6hk8DRmJiIs6cca1rKCkpQXy8e4XLV155Bc899xwAYOTIkWhsbER5eTliYtov\nxNbR2ycrK+tbPHa2rBplZdU478wwZOdeDY7yHxYLQs1G1Dke7xcWjGcfmoTvq7/HvjKguPw8RgyK\nxJkCQG1UUVZWjfoGV8mKxiqBsprO3ebpLWdhtd6sssJVkrppW5dMG4KbJg+Gpd6CsvqWCylzJg/G\nvOtSoFpsXr3H3tIXNpv276Wx0bt2+0Jv6YvegH3hcrGB06dzGGlpaSgoKEBRUREsFgvy8/Mxbdo0\nt3P69++PAwe0vRpPnjwJi8XiMVh0RmuT3s4hKYujLpHkGJISqgFCAHZhQ2iwUb9LyigbYTIqCHPs\n81BtrYXVsamSwbFwz9xk0tvQw7Wfeou27oKSJMljjaOeqIFERK3z6RVNURSsW7cOubm5EEJgwYIF\nSE1NxdatW5GWloYpU6Zg9erV+PnPf46dO3dClmX8z//8T5e2obSiHmFmY6tzGM7FexbHN0LnXVJQ\nZUBVYBM2hAUbcM6qfft1VkR17vNQY6lBiFFb2e2sJeVcuEcufW2tQfMd04gChc+/AqenpyM9Pd3t\nseXLl+s/p6am4qWXXmr+a13is2/K8NSrXyA4yICMMQNbPF9Tr627cGYYzklvqDIgZNiEHZEhJkj1\n2oK7CMd+FGGOIFFjrdVrSDkDhizJkCAhOaxv3TrbntZKmgeyhJgQnCqu7nCRRqLeLqDHTM5VarfL\n1jfaWpTDBoAaRwFC5/af+qS3UABVhk21Ys7ES1H+5ccoBRDpKCyoyApCDSGottSgzlYPo2xw26ti\n65QnfPiu/E9fyzCW3DAUybGhmOqopEsUKAI6YDTdx7uhlYBRGvI5vjwXAoujUrnzFlmoMoQqw6ra\ncGliBGKLJZSWa3tdOIWZQlFcp9WLuixqsNteFbLE+kBNdWdJ7d4gLFjbgpYo0AT0lc3WZKK7+Q5t\nMDagMeobbDv6vGsOo+mQlKrApmqRpNJShSDF5DY/Ue7YHQ8ALo/unasye4u+lmEQBaqADhhNM4wD\nXxa7PRfUpISPPofhzDCENofhvAOqylKtz184OSe+r44dgWmXuM/RkLu+lmEQBarAHpJqZx/vsHAJ\nztUBFpsKJeYslIhyCFUCIOlDUnbVjhpLLeIj3YvS3X3V7ThR+T0mJ/fe/St6C/YPUWAI6IBhs7e8\nldYpOFi4AobVDtOQIwAASXYEGVWGKlRcaKyAgEC0OdLt9weE98eA8P6+aHbAYYZBFBgCfEiq7QzD\nFOya09DnMJpy1IYqrdP22o4OiuraxvUhnMMgCgyBnWG0MSQlRxfjbJhrzwJLK3dQCUd5kMJqrbRJ\njJkBo7MYMIgCQ4BnGK0PSRmTvnc7Pnu+rsU5wqIVuztZqZ0bzYDRab1hRz0iuniBHTBayTDWLBmN\n/v3Cmj3qfp7JKGP2tcMAACcqTgHgkNTFemB+GtbfPqanm0FEFyGwh6RayTAGxIUhvCwIxU2315bd\nh6TSBvVDapwROAM02LXV4lHNJr2pY0Zd1rGtT4mo9wn4DMOQ9B2MqYfhzCIUWYLavCpcs4DRaLMj\nxuzaPlOChBCDd/sxEBEFqoAOGFa7DcaB38DQrxgwaHWjFEVCjbXG7TypWcCwWlW3gBFiCGa5DyLq\n8wL6KlituDZvks11SEkIhyJLqLI020xFsQNCm5iV6qKxaPplWikQx94WIUZmF0REAT2H0ShX6T9L\nQXVY/9MxsNitqLc1wCAZYLUCksGmD0n1D03E6uuX6xsfRZjC0FDfoO95QUTUlwV0hmEXrm0/JbN2\n62xFYyUAYHjUlbCVpGjPKTZAEgg3hbntkucsNhiscF8DIqLADhiSVf9ZCnIGjAoAQIw5ErBrq7kl\ng3ObVfeES5G051Vw6zQiosAOGHAFDNlchxMV3+O7ygIAQGxIDITq2C/aETCMzQOGrD1vd5Q5JyLq\nywJ6DkN1ZBjCrkAOq8T//Wyb/lxCWAx+lDYQh6q/ajPDMEjasV20XcSQiKivCOgMwxkw1NqIFs9F\nm6NwSZxjMZ7SesBw1o+KbLYXBhFRXxTgGYYNQgCiLhyIuOD2XHRQJIIUbRclSR+SMrqdM/+yOTAb\nzMhImdo9DSYi6sUCNmD885PTsAkLJFWB2hCqPy5Bwk1D58FsMMNs1AKEpGhzFAbHnIWT2WDG/Mvm\ndF+jiYh6sYAdknrpX98Cih2SasSg8MH64z8fl4f0AdcBAMwGxz6tbWQYRETkErABA3BkDnYFa3Mm\n64/FmGP0n4ONQY7zHHMYknuGQURELgE7JAUAUGxQHftaPDZ+FSobK2FSXFlEkJ5haENSzDCIiNoW\nwAFDhSSrUG1a1pAQEoeEEPcS284AoWcYSgB3BxHRRQrcISnHRDbUtoOAHjAcGYZz3QUREbUUsAFD\nMtcDAISl7TpQpmYZRfOV3kRE5BKwAUMO1kqYq3VtL7prPmfRfOEeERG5BG7ACNEChqhvvn+3iyIp\naLr5HjMMIqK2BdwV8u8ffo+TZ6ogBWu76rWXYUiSBKiKtoESmGEQEbUn4K6Qr3/4PQAgaLgNQpVw\n95yr2/8FVWbAICLyQsAOSUFSIUPB2GEJ7Z6mlzgH12EQEbUncAOGrEISXqzcVl1d0LyWFBERuQRu\nwJBUSN68PWYYRERe8XnA2L9/P2bOnImMjAxs37691XP27NmDzMxMzJkzBw8//HCX/F3JywxDbrJY\nj3MYRERt8+kVUlVVbNiwATt37kR8fDwWLFiAadOmITU1VT/nhx9+wI4dO/CXv/wFYWFhKC8v75o/\n7mWGkZoYjZNV2l4ZvK2WiKhtPs0wjh49ipSUFCQnJ8NoNCIzMxN79+51O+evf/0rFi9ejLAwbb1E\nTExMay/VcbI26e2Js2ItwAyDiKg9Pg0YJSUlSEpK0o8TEhJQWlrqds6pU6fw/fffY9GiRVi4cCE+\n+OCDrvnjkgrJi4BhNrgCBjMMIqK2ebxClpSUICGh/VtT2yKaLqNug91uR0FBAXbt2oUzZ85gyZIl\nyM/P1zOOzhGQZAFZ9RwwghRmGERE3vB4hZw/fz5GjRqFxYsXY8KECR168cTERJw5c0Y/LikpQXx8\nvNs5CQkJGDVqFGRZxoABAzBo0CCcOnUKV155ZbuvHRfX9gpuSFqgMshK++cBiC5yPZ8YH6Wt/vYz\nnt5jX8K+cGFfuLAvuobHgLFv3z7s2bMHv/vd77BhwwYsWbIE8+bN8yoDSEtLQ0FBAYqKihAXF4f8\n/Hxs3rzZ7Zzp06cjPz8fWVlZKC8vxw8//ICBAwd6fO2ysuq2n5RU7f+F3P55AFSLK0CcO1fj8e/2\nNnFx4R7fY1/BvnBhX7iwL1wuNnB6DBgmkwlZWVnIysrCZ599hry8PPz2t79FdnY27r33XvTr16/N\n31UUBevWrUNubi6EEFiwYAFSU1OxdetWpKWlYcqUKfjRj36Ef//738jMzISiKHjkkUcQGRnZ6Tck\nSYCQtYCheDPp3WRIioiI2ubVoH1RURF2796NN998ExMmTEBOTg4++ugjLF26FK+//nq7v5ueno70\n9HS3x5YvX+52vGbNGqxZs6aDTW9JCKFVn3VkGLIXGyIFGRgwiIi84fGKevfdd+Obb77BwoUL8eqr\nryI6OhoAMHr0aOzZs8fnDewIu6rNXUgdyDDMzDCIiLziMWDMmzcPM2bMgKK0vPi++eabPmlUZ9nt\njruy9AyjY3dJERFR2zyuw4iMjERdXZ1+XFVVhYMHD/q0UZ3lzDDgyDAMXgQMs6HtLVyJiMjFY8DY\nuHGj2x1RYWFh2Lhxo08b1Vl21XF3lCPDULyYw+CQFBGRdzwGDCGE29oEWZZht9t92qjOcs1haO1T\nOCRFRNRlPAaM0NBQHDlyRD8+cuQIQkJCfNqoztLnMJyT3l4EDO6BQUTkHY9jNqtWrcJ9992HIUOG\nAABOnDiBp556yucN6wzXkJQWOIQX5c0jTOEwK0EYkzjal00jIvJ7HgPGqFGjkJ+fj8OHD0MIgVGj\nRl3Uwjpf0ie99ZXenst8KLKCJ9N/6ZclQYiIupNXC/ciIyMxefJkX7flojVfh+HVFq0AgwURkRc8\nBozjx49j/fr1OH78OCwWi/74V1995dOGdYbdLgDFCslUrz2gBu4OtERE3c1jwPjFL36BFStW4Ikn\nnsCOHTuwa9cuhIaGdkfbOsyuCgSNOAjZrK0bEQwYRERdxuMV1WKxYMKECRBCID4+HitXruy6TY66\nmF1V9WABAAaVi/KIiLqKx4Ahy9opkZGROH78OC5cuICioiKfN6wzVNV9wyazLbaHWkJEFHg8Dkll\nZmbiwoULWLZsGRYtWgRVVVtUm+0tbM0CRlRocA+1hIgo8LQbMFRVxYQJExAdHY309HR8/PHHaGxs\nvMjtU33HbhcQqgxJVjFWmY/Z4y/t6SYREQWMdoekZFnGz372M/3YaDT22mABAFa7DZKsop+cjNsm\nj0NwEPfoJiLqKh7nMFJTU1FYWNgdbbloFrt2269RMvVwS4iIAo/Hr+Dl5eWYO3currnmGrcaUlu2\nbPFpwzqjwd4IADDKDBhERF3Nq0nvzMzM7mjLRWu0OTIM2djDLSEiCjweA0Z2dnZ3tKNLNNgbAAAm\nmSXLiYi6mseAsXz58lZrLfXGISmLygyDiMhXPAaMKVOm6D83NjbinXfeQWpqqk8b1VnOgBHEDIOI\nqMt1eEjqxz/+Me655x6fNehiOO+SMimc9CYi6modrs4nSVKvvc3Womp3SXGfbiKirtehOQwhBL7+\n+mtMmDDB5w3rjEbHbbVmA4sOEhF1tQ7NYSiKgtzcXIwcOdKnjeqsRlXbByPM2Dv3HCci8mcBdVtt\no3AGjN5bvoSIyF95nMNYtGgRKisr9eOKigosWbLEp43qrEahrcMIN/XODZ6IiPyZx4BRV1eHyMhI\n/TgqKgo1NTU+bVRnWVEPoUoIMXIOg4ioq3kMGKqqoq7OtYtdbW0t7Ha7TxvVWVbRANhMMBqUnm4K\nEVHA8TiHMXv2bOTm5mLRokUAgJdeeglz5871ecM6wyo1QNiCoCgtV6YTEdHF8Rgw7rrrLsTHx2Pf\nvn0QQmDhwoXIysrqjrZ1iF21Q5WsELZwGJUOLy8hIiIPvNphKDs7u9ffLVVcVwoAEFYTFAYMF7dU\nkAAAFDhJREFUIqIu5/HK+sADD6CiokI/vnDhAh588EGfNqoz3jq1FwBgP5/EDIOIyAc8XllPnz6N\nqKgo/Tg6OhoFBQU+bVRnlNaVQVKNUCviOYdBROQDHgOG3W53uyvKarXCYrH4tFGdUW9rgKwaIUGC\nIjNgEBF1NY8BY9KkSVi5ciU++eQTfPLJJ8jLy0N6errXf2D//v2YOXMmMjIysH379jbPe/vtt3HF\nFVfgv//9r9ev3VS9rQGSaoSiyK3u30FERBfH46R3Xl4efv/73+M3v/kNAK221Lhx47x6cVVVsWHD\nBuzcuRPx8fFYsGABpk2b1mI/jdraWrz44oudrlElhECDrQGKGgIDh6OIiHzCY4ZhNBpx//334+mn\nn8YNN9yAv//971i7dq1XL3706FGkpKQgOTkZRqMRmZmZ2Lt3b4vztmzZgjvvvBNGY+d2ymu0WyAg\nALsBBk54ExH5RLsZhs1mw759+/C3v/0Nhw8fhs1mw3PPPed1JlBSUoKkpCT9OCEhAV988YXbOV99\n9RWKi4sxefJk7NixoxNvwbWXN+xGZhhERD7S5tfxJ554Atdffz12796N2bNn4/3330dkZGSHho2E\nEB6ff/zxx7FmzRqvf6c1DTYtYAhmGEREPtNmhvHSSy9h1KhRWLZsGcaPHw8AHZ5MTkxMxJkzZ/Tj\nkpISxMfH68e1tbU4ceIEfvKTn0AIgXPnzuHee+/Ftm3bMGLEiHZfOy4uXP/5glSm/WA3IMhkcHuu\nL+hr77c97AsX9oUL+6JrtBkwPvzwQ/zv//4vNm7ciMrKSmRlZXW46GBaWhoKCgpQVFSEuLg45Ofn\nY/PmzfrzYWFhOHjwoH78k5/8BI8++iiGDx/u8bXLyqr1n8+eLwcA2K0KpGbPBbq4uPA+9X7bw75w\nYV+4sC9cLjZwtjl+ExERgSVLluDVV1/F008/jcrKSjQ0NGDJkiXYvXu3Vy+uKArWrVuH3NxczJ49\nG5mZmUhNTcXWrVvx7rvvtjhfkqTODUk5tmZVbQrnMIiIfEQSHbhCW61W/POf/8Rrr72GP/zhD75s\nl0dlZdUoqCrEy9/+HcNiLkP+9/+E5WQaBpmHY+1PrunRtnUnfntyYV+4sC9c2BcuF5theFV80Mlo\nNGLWrFmYNWvWRf3RrrL7m9fwQ9VpfFd5CoA26W2x9c69OoiI/J1f31IUFRTp/oDdgPOVDT3TGCKi\nAOfXASM2OMbtWNgNqG2w9VBriIgCm18HjBbsnVspTkREnvl1wLCr7vMVwm5AanJED7WGiCiwdWjS\nu7exqe7DT4umXIGJIwb0UGuIiAKbX2cYNuGeYQzpH4MQs1/HQCKiXsuvA0bzISmzSemhlhARBT6/\nDhjlNfVux2YTswsiIl/x64BRXe++5oIZBhGR7/h1wLA3m8MIMjJgEBH5SkAFDFlm4UEiIl/x64Ch\nwhUwbGXJPdgSIqLA59ezxKpQIVQZDZ9NBwSzCyIiX/LrgGEXNkDI2v+IiMin/PpKq0JlsCAi6iZ+\nfbUVsEOofv0WiIj8hl9fbbUMg3MXRETdwa/nMATsgDBg2ugBGDs8vqebQ0QU0Pw8YKiAKuOmqakw\nGrhoj4jIl/x6SEpI2qS3ovj12yAi8gt+faUVjrukZInzGEREvua3AUMVKiAJSP77FoiI/IrfXm2d\nu+1JXIdBRNQt/PZqa3NsniSBk91ERN3BbwOGs1Kt7L9vgYjIr/jt1VYfkmKGQUTULfw4YDgzDAYM\nIqLu4LcBwy60DINDUkRE3cNvr7ZWZ4YhMcMgIuoOfhswnHMYCgMGEVG38NuAYbE7h6QYMIiIuoP/\nBgybFQCgyAwYRETdwW8DRqMzYDDDICLqFn4ZMKpqLbDYtCEpAzMMIqJu4Zf7YSx57C0kDq4AYjnp\nTUTUXXyeYezfvx8zZ85ERkYGtm/f3uL5nTt3IjMzE/PmzcNPf/pTnD171qvXLausAwAYZL+MeURE\nfsenAUNVVWzYsAHPPfcc3nzzTeTn5+PkyZNu5wwfPhyvvvoq3njjDcyYMQMbN2707sUlFQADBhFR\nd/FpwDh69ChSUlKQnJwMo9GIzMxM7N271+2csWPHIigoCAAwcuRIlJSUePfishYwjAoDBhFRd/Bp\nwCgpKUFSUpJ+nJCQgNLS0jbPf+WVV5Cenu7di+sZBucwiIi6g0+/ngshvD73jTfewH//+1+88MIL\nXp0vSdprh4cEIy4uvFPtCxR9/f03xb5wYV+4sC+6hk8DRmJiIs6cOaMfl5SUID4+vsV5Bw4cwPbt\n2/Hiiy/CaDR69+KOISnVBpSVVXdJe/1RXFx4n37/TbEvXNgXLuwLl4sNnD4dkkpLS0NBQQGKiopg\nsViQn5+PadOmuZ1z7NgxrF+/Htu2bUN0dLT3L+4YkjJxDoOIqFv49GqrKArWrVuH3NxcCCGwYMEC\npKamYuvWrUhLS8OUKVOwadMm1NfX48EHH4QQAv3798czzzzj+cUl56Q35zCIiLqDz7+ep6ent5jI\nXr58uf7z888/36nXlWRnhuHlEBYREV0UvywNAgBwTHqbDAwYRETdwS8DRnR4EOcwiIi6mV8GDINB\n1u+SCvL2rioiIroo/hkwZFnPMIKYYRARdQv/DBgGSV+4Z2KGQUTULfwyYCiya0jKzElvIqJu4ZcB\nw6BI+pCUmRkGEVG38NOA0WQOw8A5DCKi7uCXAUNRZEiyCqHKMBm40puIqDv4ZcAwKjKg2AC7AqPB\nL98CEZHf8curraJIkIyNENYgBgwiom7il1dbWRGQDDYIaxAUxS/fAhGR3/HPq63SCAAQVhNkSerh\nxhAR9Q1+GTBUgzNgBPVwS4iI+g6/DBh2uR4AINkZMIiIuotfBgxVbgAAGEVwD7eEiKjv8MuAcVoc\nAQAEqRE93BIior7DLwNGg1QJ27n+CFHjeropRER9hl8GDABQq2IQEsSyIERE3cV/A0ZdOIIZMIiI\nuo1/BgwBiPowmE2sI0VE1F38MmAEl6cBQkFYMEubExF1F78MGOvmLsaVg2Iwd+Kgnm4KEVGf4ZeT\nAEMGRCHv5pE93Qwioj7FLzMMIiLqfgwYRETkFQYMIiLyCgMGERF5hQGDiIi8woBBREReYcAgIiKv\nMGAQEZFXGDCIiMgrDBhEROQVBgwiIvKKzwPG/v37MXPmTGRkZGD79u0tnrdYLFi5ciVmzJiBm2++\nGWfOnPF1k4iIqBN8GjBUVcWGDRvw3HPP4c0330R+fj5Onjzpds4rr7yCyMhI/OMf/8Btt92GTZs2\n+bJJRETUST4NGEePHkVKSgqSk5NhNBqRmZmJvXv3up2zd+9eZGdnAwAyMjJw8OBBXzaJiIg6yacB\no6SkBElJSfpxQkICSktL3c4pLS1FYmIiAEBRFERERKCiosKXzSIiok7wacAQQnT4HCEEJEnyVZOI\niKiTfLqBUmJiotskdklJCeLj41ucU1xcjISEBNjtdtTU1CAyMtLja8fFhXd5e/0V+8KFfeHCvnBh\nX3QNn2YYaWlpKCgoQFFRESwWC/Lz8zFt2jS3c6ZMmYLXXnsNAPD2229j/PjxvmwSERF1kiS8GTe6\nCPv378evf/1rCCGwYMECLFu2DFu3bkVaWhqmTJkCi8WCVatW4auvvkJUVBQ2b96MAQMG+LJJRETU\nCT4PGEREFBi40puIiLzCgEFERF5hwCAiIq/4XcDwVJsq0KxduxbXXXcd5syZoz9WWVmJ3NxcZGRk\nYOnSpaiurtaf+9WvfoUZM2Zg3rx5+Oqrr3qiyT5RXFyMW2+9FbNmzcKcOXPw5z//GUDf7AuLxYKc\nnBxkZWVhzpw5eOqppwAAhYWFuOmmm5CRkYG8vDzYbDb9/ECv16aqKrKzs3H33XcD6Lt9MXXqVMyd\nOxdZWVlYsGABgC7+jAg/YrfbxfTp00VhYaGwWCxi7ty54sSJEz3dLJ/6z3/+I44dOyZmz56tP7Zx\n40axfft2IYQQv//978WmTZuEEEK899574s477xRCCHH48GGRk5PT/Q32kdLSUnHs2DEhhBA1NTVi\nxowZ4sSJE32yL4QQoq6uTgghhM1mEzk5OeLw4cPiwQcfFHv27BFCCPHYY4+Jl156SQghxK5du8T6\n9euFEELk5+eLFStW9Eibfen5558XDz30kLjrrruEEKLP9sXUqVNFRUWF22Nd+RnxqwzDm9pUgeba\na69FRESE22NN629lZ2frfbB3715kZWUBAK6++mpUV1fj3Llz3dtgH4mLi8OwYcMAAKGhoUhNTUVJ\nSUmf7AsACA4OBqB9Y7bZbJAkCYcOHUJGRgYArS/+9a9/AQj8em3FxcV4//33kZOToz/20Ucf9cm+\nEEJAVVW3x7ryM+JXAcOb2lR9QXl5OWJjYwFoF9Ly8nIA7nW5AK1/SkpKeqSNvlRYWIjjx4/j6quv\nxvnz5/tkX6iqiqysLEycOBETJ07EwIEDERERAVnWPtKJiYn6+w30em2PP/44HnnkEb2k0IULFxAZ\nGdkn+0KSJCxduhTz58/Hyy+/DABd+hnxaWmQria4ZKRdrfVPoNXlqq2txfLly7F27VqEhoa2+f4C\nvS9kWcbrr7+Ompoa3HfffS22DQBc77d5X4gAqtf23nvvITY2FsOGDcOhQ4cAaO+v+XvuC30BALt3\n79aDQm5uLgYNGtSlnxG/Chje1KbqC/r164dz584hNjYWZWVliImJAaB9QyguLtbPKy4uDqj+sdls\nWL58OebNm4fp06cD6Lt94RQWFoYxY8bgyJEjqKqqgqqqkGXZ7f06+6Kj9dr8wWeffYZ9+/bh/fff\nR2NjI2pra/H444+jurq6z/UFoGUQABATE4Pp06fj6NGjXfoZ8ashKW9qUwWi5t8Epk6dildffRUA\n8Nprr+l9MG3aNLz++usAgMOHDyMiIkJPRQPB2rVrMWTIENx22236Y32xL8rLy/U7XRoaGnDw4EEM\nGTIE48aNw9tvvw3AvS+mTp0asPXa8vLy8N5772Hv3r3YvHkzxo0bhyeffLJP9kV9fT1qa2sBAHV1\ndfjwww8xdOjQLv2M+F1pkNZqUwWyhx56CIcOHUJFRQViY2PxwAMPYPr06XjwwQdx9uxZ9O/fH1u2\nbNEnxn/5y1/igw8+QHBwMJ544gmMGDGih99B1/j0009xyy23YOjQoZAkCZIkYeXKlbjqqquwYsWK\nPtUXX3/9NdasWQNVVaGqKmbNmoV77rkHp0+fRl5eHqqqqjBs2DBs2rQJRqOxz9Rr+/jjj/HHP/4R\nzz77bJ/si9OnT+P++++HJEmw2+2YM2cOli1bhoqKii77jPhdwCAiop7hV0NSRETUcxgwiIjIKwwY\nRETkFQYMIiLyCgMGERF5hQGDiIi8woBBfu2mm25CdnY2MjMzMWLECGRnZyM7Oxtr167t8Gvdcccd\nXpW7fvTRR3H48OHONLdDjh07hnfeecfnf4fIW1yHQQGhqKgICxYsaLf6qLNUhL94+eWXcfDgQWze\nvLmnm0IEwM9qSRF1xMGDB7Fp0yaMHDkSx44dw3333Yfy8nLs2rVL31BnzZo1GDt2LABg8uTJ2Llz\nJwYNGoTFixdj1KhR+Pzzz1FaWorZs2djxYoVAIDFixfj3nvvxaRJk7Bq1SqEhYXh5MmTKCkpwejR\no/HEE08A0GrzPPLII7hw4QIGDhwIu92OqVOn4uabb3Zr57lz5/DQQw/hwoULAIBJkybhjjvuwDPP\nPIO6ujpkZ2dj3LhxWLNmDT7//HNs3rwZ9fX1AIDly5cjPT0dBQUFWLx4MWbPno1PP/0UFosF69ev\nx+jRo7ulr6mPuJjNOoh6i8LCQjF+/Hi3xw4cOCCGDx8uvvjiC/2xppvLnDhxQlx//fX6cXp6uvju\nu++EEEIsWrRIPPTQQ0IIIaqqqsTYsWNFYWGh/twHH3wghBDi4YcfFrfccouwWq2isbFRzJw5Uxw6\ndEgIIcQ999wj/vCHPwghhDh9+rQYNWqU2L17d4u279ixQzz22GP6cVVVlRBCiL/+9a8iLy/Pre1Z\nWVni/PnzQgghiouLRXp6uqipqRE//PCDuPzyy0V+fr7+3q+//nphs9m870QiD5hhUEAbPHgwrrzy\nSv341KlT2Lp1K0pLS6EoCkpLS1FRUYGoqKgWv3vjjTcCAMLDwzFo0CAUFBQgOTm5xXk33HADDAbt\nozR8+HAUFBRg7NixOHToEH71q18BAAYMGKBnMs2NHDkSL774Ip588kmMGTMGkyZNavW8Tz/9FIWF\nhVi6dKlekFJRFJw+fRohISEIDg7GrFmzAAATJkyAoig4deoUUlNTve0uonYxYFBACw0NdTteuXIl\n1q9fj8mTJ0NVVVx11VVobGxs9XeDgoL0n2VZht1u79B53u6zcM011+C1117DgQMH8Le//Q07duzA\nCy+80OI8IQRGjBiBnTt3tniuoKCgxWOqqgbUXg/U8/xnBpDIA+HF/Rs1NTV6ddLdu3e3GQS6wtix\nY/Wy0kVFRfj4449bPa+wsBBhYWGYNWsW1qxZgy+//BKAtteFs4w5AIwePRonTpzAJ598oj929OhR\n/ef6+nrs2bMHgLZFKQCkpKR07ZuiPo0ZBgUMb75Nr127FsuWLUNSUhLGjRuH8PDwVn+/+Wu19Vx7\n561btw6rV69Gfn4+Bg8ejNGjR7v9PaeDBw/iz3/+MxRFgRACGzZsAABMnDgRf/rTn5CVlYXx48dj\nzZo1eOaZZ7Bp0yZUV1fDarVi4MCBePbZZwEAsbGx+Pbbb5GTkwOLxYLNmzdDURSPfULkLd5WS+Qj\njY2NMBqNkGUZJSUlyMnJwa5duzBw4MAu/1vOu6Q+/PDDLn9tIidmGEQ+8t133+HRRx+FEAKqqmLl\nypU+CRZE3YUZBhEReYWT3kRE5BUGDCIi8goDBhEReYUBg4iIvMKAQUREXmHAICIir/x/apbYj523\no60AAAAASUVORK5CYII=\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f97f1330850\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "def plot(train, test, label):\n", - " plt.title('MNIST model %s' % label)\n", - " plt.plot(train, label='train %s' % label)\n", - " plt.plot(test, label='test %s' % label)\n", - " plt.legend()\n", - " plt.xlabel('Training step')\n", - " plt.ylabel(label.capitalize())\n", - " plt.show()\n", - " \n", - "\n", - "with tf.Graph().as_default():\n", - " hp = tf.contrib.training.HParams(\n", - " learning_rate=0.05,\n", - " max_steps=tf.constant(500),\n", - " )\n", - " train_ds = setup_mnist_data(True, hp, 50)\n", - " test_ds = setup_mnist_data(False, hp, 1000)\n", - " tf_train = autograph.to_graph(train)\n", - " all_losses = tf_train(train_ds, test_ds, hp)\n", - "\n", - " with tf.Session() as sess:\n", - " sess.run(tf.global_variables_initializer())\n", - " (train_losses, test_losses, train_accuracies,\n", - " test_accuracies) = sess.run(all_losses)\n", - " \n", - " plot(train_losses, test_losses, 'loss')\n", - " plot(train_accuracies, test_accuracies, 'accuracy')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HNqUFL4deCsL" - }, - "source": [ - "# 4. Case study: building an RNN\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YkC1k4HEQ7rw" - }, - "source": [ - "In this exercise we build and train a model similar to the RNNColorbot model that was used in the main Eager notebook. The model is adapted for converting and training in graph mode." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7nkPDl5CTCNb" - }, - "source": [ - "To get started, we load the colorbot dataset. The code is identical to that used in the other exercise and its details are unimportant." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "A0uREmVXCQEw" - }, - "outputs": [], - "source": [ - "def parse(line):\n", - " \"\"\"Parses a line from the colors dataset.\n", - " \n", - " Args:\n", - " line: A comma-separated string containing four items:\n", - " color_name, red, green, and blue, representing the name and\n", - " respectively the RGB value of the color, as an integer\n", - " between 0 and 255.\n", - "\n", - " Returns:\n", - " A tuple of three tensors (rgb, chars, length), of shapes: (batch_size, 3),\n", - " (batch_size, max_sequence_length, 256) and respectively (batch_size).\n", - " \"\"\"\n", - " items = tf.string_split(tf.expand_dims(line, 0), \",\").values\n", - " rgb = tf.string_to_number(items[1:], out_type=tf.float32) / 255.0\n", - " color_name = items[0]\n", - " chars = tf.one_hot(tf.decode_raw(color_name, tf.uint8), depth=256)\n", - " length = tf.cast(tf.shape(chars)[0], dtype=tf.int64)\n", - " return rgb, chars, length\n", - "\n", - "\n", - "def maybe_download(filename, work_directory, source_url):\n", - " \"\"\"Downloads the data from source url.\"\"\"\n", - " if not tf.gfile.Exists(work_directory):\n", - " tf.gfile.MakeDirs(work_directory)\n", - " filepath = os.path.join(work_directory, filename)\n", - " if not tf.gfile.Exists(filepath):\n", - " temp_file_name, _ = six.moves.urllib.request.urlretrieve(source_url)\n", - " tf.gfile.Copy(temp_file_name, filepath)\n", - " with tf.gfile.GFile(filepath) as f:\n", - " size = f.size()\n", - " print('Successfully downloaded', filename, size, 'bytes.')\n", - " return filepath\n", - "\n", - "\n", - "def load_dataset(data_dir, url, batch_size, training=True):\n", - " \"\"\"Loads the colors data at path into a tf.PaddedDataset.\"\"\"\n", - " path = maybe_download(os.path.basename(url), data_dir, url)\n", - " dataset = tf.data.TextLineDataset(path)\n", - " dataset = dataset.skip(1)\n", - " dataset = dataset.map(parse)\n", - " dataset = dataset.cache()\n", - " dataset = dataset.repeat()\n", - " if training:\n", - " dataset = dataset.shuffle(buffer_size=3000)\n", - " dataset = dataset.padded_batch(batch_size, padded_shapes=((None,), (None, None), ()))\n", - " return dataset\n", - "\n", - "\n", - "train_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/archive/extras/colorbot/data/train.csv\"\n", - "test_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/archive/extras/colorbot/data/test.csv\"\n", - "data_dir = \"tmp/rnn/data\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "waZ89t3DTUla" - }, - "source": [ - "Next, we set up the RNNColobot model, which is very similar to the one we used in the main exercise.\n", - "\n", - "Autograph doesn't fully support classes yet (but it will soon!), so we'll write the model using simple functions." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "9v8AJouiC44V" - }, - "outputs": [], - "source": [ - "def model_components():\n", - " lower_cell = tf.contrib.rnn.LSTMBlockCell(256)\n", - " lower_cell.build(tf.TensorShape((None, 256)))\n", - " upper_cell = tf.contrib.rnn.LSTMBlockCell(128)\n", - " upper_cell.build(tf.TensorShape((None, 256)))\n", - " relu_layer = tf.layers.Dense(3, activation=tf.nn.relu)\n", - " relu_layer.build(tf.TensorShape((None, 128)))\n", - " return lower_cell, upper_cell, relu_layer\n", - "\n", - "\n", - "def rnn_layer(chars, cell, batch_size, training):\n", - " \"\"\"A simple RNN layer.\n", - " \n", - " Args:\n", - " chars: A Tensor of shape (max_sequence_length, batch_size, input_size)\n", - " cell: An object of type tf.contrib.rnn.LSTMBlockCell\n", - " batch_size: Int, the batch size to use\n", - " training: Boolean, whether the layer is used for training\n", - "\n", - " Returns:\n", - " A Tensor of shape (max_sequence_length, batch_size, output_size).\n", - " \"\"\"\n", - " hidden_outputs = tf.TensorArray(tf.float32, size=0, dynamic_size=True)\n", - " state, output = cell.zero_state(batch_size, tf.float32)\n", - " initial_state_shape = state.shape\n", - " initial_output_shape = output.shape\n", - " n = tf.shape(chars)[0]\n", - " i = 0\n", - " while i \u003c n:\n", - " ch = chars[i]\n", - " cell_output, (state, output) = cell.call(ch, (state, output))\n", - " hidden_outputs.append(cell_output)\n", - " i += 1\n", - " hidden_outputs = autograph.stack(hidden_outputs)\n", - " if training:\n", - " hidden_outputs = tf.nn.dropout(hidden_outputs, 0.5)\n", - " return hidden_outputs\n", - "\n", - "\n", - "def model(inputs, lower_cell, upper_cell, relu_layer, batch_size, training):\n", - " \"\"\"RNNColorbot model.\n", - " \n", - " The model consists of two RNN layers (made by lower_cell and upper_cell),\n", - " followed by a fully connected layer with ReLU activation.\n", - " \n", - " Args:\n", - " inputs: A tuple (chars, length)\n", - " lower_cell: An object of type tf.contrib.rnn.LSTMBlockCell\n", - " upper_cell: An object of type tf.contrib.rnn.LSTMBlockCell\n", - " relu_layer: An object of type tf.layers.Dense\n", - " batch_size: Int, the batch size to use\n", - " training: Boolean, whether the layer is used for training\n", - " \n", - " Returns:\n", - " A Tensor of shape (batch_size, 3) - the model predictions.\n", - " \"\"\"\n", - " (chars, length) = inputs\n", - " chars_time_major = tf.transpose(chars, (1, 0, 2))\n", - " chars_time_major.set_shape((None, batch_size, 256))\n", - "\n", - " hidden_outputs = rnn_layer(chars_time_major, lower_cell, batch_size, training)\n", - " final_outputs = rnn_layer(hidden_outputs, upper_cell, batch_size, training)\n", - "\n", - " # Grab just the end-of-sequence from each output.\n", - " indices = tf.stack((length - 1, range(batch_size)), axis=1)\n", - " sequence_ends = tf.gather_nd(final_outputs, indices)\n", - " sequence_ends.set_shape((batch_size, 128))\n", - " return relu_layer(sequence_ends)\n", - "\n", - "def loss_fn(labels, predictions):\n", - " return tf.reduce_mean((predictions - labels) ** 2)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JjK4gXFvFsf4" - }, - "source": [ - "The train and test functions are also similar to the ones used in the Eager notebook. Since the network requires a fixed batch size, we'll train in a single shot, rather than by epoch." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "ZWQMExk0S6X6" - }, - "outputs": [], - "source": [ - "def train(optimizer, train_data, lower_cell, upper_cell, relu_layer, batch_size, num_steps):\n", - " iterator = train_data.make_one_shot_iterator()\n", - " step = 0\n", - " while step \u003c num_steps:\n", - " labels, chars, sequence_length = iterator.get_next()\n", - " predictions = model((chars, sequence_length), lower_cell, upper_cell, relu_layer, batch_size, training=True)\n", - " loss = loss_fn(labels, predictions)\n", - " optimizer.minimize(loss)\n", - " if step % (num_steps // 10) == 0:\n", - " print('Step', step, 'train loss', loss)\n", - " step += 1\n", - " return step\n", - "\n", - "\n", - "def test(eval_data, lower_cell, upper_cell, relu_layer, batch_size, num_steps):\n", - " total_loss = 0.0\n", - " iterator = eval_data.make_one_shot_iterator()\n", - " step = 0\n", - " while step \u003c num_steps:\n", - " labels, chars, sequence_length = iterator.get_next()\n", - " predictions = model((chars, sequence_length), lower_cell, upper_cell, relu_layer, batch_size, training=False)\n", - " total_loss += loss_fn(labels, predictions)\n", - " step += 1\n", - " print('Test loss', total_loss)\n", - " return total_loss\n", - "\n", - "\n", - "def train_model(train_data, eval_data, batch_size, lower_cell, upper_cell, relu_layer, train_steps):\n", - " optimizer = tf.train.AdamOptimizer(learning_rate=0.01)\n", - "\n", - " train(optimizer, train_data, lower_cell, upper_cell, relu_layer, batch_size, num_steps=tf.constant(train_steps))\n", - " test(eval_data, lower_cell, upper_cell, relu_layer, 50, num_steps=tf.constant(2))\n", - "\n", - " print('Colorbot is ready to generate colors!\\n\\n')\n", - " \n", - " # In graph mode, every op needs to be a dependent of another op.\n", - " # Here, we create a no_op that will drive the execution of all other code in\n", - " # this function. Autograph will add the necessary control dependencies.\n", - " return tf.no_op()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iopcs5hXG2od" - }, - "source": [ - "Finally, we add code to run inference on a single input, which we'll read from the input.\n", - "\n", - "Note the `do_not_convert` annotation that lets us disable conversion for certain functions and run them as a `py_func` instead, so you can still call them from compiled code." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "DyU0wnnAFEYj" - }, - "outputs": [], - "source": [ - "@autograph.do_not_convert(run_as=autograph.RunMode.PY_FUNC)\n", - "def draw_prediction(color_name, pred):\n", - " pred = pred * 255\n", - " pred = pred.astype(np.uint8)\n", - " plt.axis('off')\n", - " plt.imshow(pred)\n", - " plt.title(color_name)\n", - " plt.show()\n", - "\n", - "\n", - "def inference(color_name, lower_cell, upper_cell, relu_layer):\n", - " _, chars, sequence_length = parse(color_name)\n", - " chars = tf.expand_dims(chars, 0)\n", - " sequence_length = tf.expand_dims(sequence_length, 0)\n", - " pred = model((chars, sequence_length), lower_cell, upper_cell, relu_layer, 1, training=False)\n", - " pred = tf.minimum(pred, 1.0)\n", - " pred = tf.expand_dims(pred, 0)\n", - " draw_prediction(color_name, pred)\n", - " # Create an op that will drive the entire function.\n", - " return tf.no_op()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Nt0Kv5OCHip0" - }, - "source": [ - "Finally, we put everything together.\n", - "\n", - "Note that the entire training and testing code is all compiled into a single op (`tf_train_model`) that you only execute once! We also still use a `sess.run` loop for the inference part, because that requires keyboard input." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 415 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 15536, - "status": "ok", - "timestamp": 1531750946373, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "-GmWa0GtYWdh", - "outputId": "2e7a9856-9809-43a3-8b43-3c8514ea43e9" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test loss 0.138294\n", - "Colorbot is ready to generate colors!\n", - "\n", - "\n" - ] - }, - { - "data": { - "text/html": [ - "\u003clink rel=stylesheet type=text/css href='/nbextensions/google.colab/tabbar.css'\u003e\u003c/link\u003e" - ], - "text/plain": [ - "\u003cIPython.core.display.HTML at 0x7f97ee42bb90\u003e" - ] - }, - "metadata": { - "tags": [ - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\u003cscript src='/nbextensions/google.colab/tabbar_main.min.js'\u003e\u003c/script\u003e" - ], - "text/plain": [ - "\u003cIPython.core.display.HTML at 0x7f97ee42be10\u003e" - ] - }, - "metadata": { - "tags": [ - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\u003cdiv id=\"id1\"\u003e\u003c/div\u003e" - ], - "text/plain": [ - "\u003cIPython.core.display.HTML at 0x7f97ee42bd90\u003e" - ] - }, - "metadata": { - "tags": [ - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a6045494-8903-11e8-99f9-c8d3ffb5fbe0\"] = colab_lib.createTabBar({\"location\": \"top\", \"borderColor\": [\"#a7a7a7\"], \"initialSelection\": 0, \"elementId\": \"id1\", \"contentHeight\": [\"initial\"], \"contentBorder\": [\"0px\"], \"tabNames\": [\"RNN Colorbot\"]});\n", - "//# sourceURL=js_02f896cbda" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ee2ab810\u003e" - ] - }, - "metadata": { - "tags": [ - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a6045495-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_7e8f9f77a0" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ee2ab710\u003e" - ] - }, - "metadata": { - "tags": [ - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a6045496-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_5531553c2f" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ee2ab6d0\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a6045497-8903-11e8-99f9-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", - "//# sourceURL=js_d1f809ec17" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ee2ab990\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a6045498-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"a6045497-8903-11e8-99f9-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_3a3123cadb" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ee2aba50\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a6045499-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_1a0e1f7d6f" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ee2ab890\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a8e54762-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"a6045496-8903-11e8-99f9-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_6213539615" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ee2abad0\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a8e54763-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_0bd7f95c6e" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ee2ab950\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a8e54764-8903-11e8-99f9-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", - "//# sourceURL=js_215f004f6b" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ee2abb10\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a8e54765-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"a8e54764-8903-11e8-99f9-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_a06186c8ad" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ee2aba90\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a8e54766-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_383fbaae67" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ee2abc50\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQwAAAENCAYAAAD60Fs2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACL9JREFUeJzt3F+IlXUex/Gv2ziiBRGVOQaFd2JBzOg5aiH+IZGoJgmM\n/uhVGIlgFE0QEYHdFQaRGBJ10VX0D5TAi8jKomGmILsYjEAkmBwbRIxKGDV/e7G7w8ouux9jd911\nX6+rcx6e85zveS7e/J7zb0ZrrRVA4A8XewDgf4dgADHBAGKCAcQEA4gJBhATDC6Kp59+urrdbt13\n3301OjpaK1euvNgjERCMS9yaNWtqeHj4Yo9xnq+++qqGh4frs88+q7fffruqqmbMmHGRpyIhGPxH\n/fbbb/XDDz/U9ddfX7NmzbrY43CBBOMS9tRTT9XExERt2bKlBgYG6vXXX69vvvmm7r///up0OrV+\n/foaHR2d3n/Tpk318ssv1wMPPFADAwP18MMP18mTJ6uq6vTp0zU0NFRLly6tTqdTGzZsqBMnTlRV\n1eTkZG3ZsqWWLl1a69atq3feeWf6mDt37qxt27bV0NBQLVmypN5777169tln6+DBgzUwMFA7d+78\nm7kPHz5cmzZtqk6nU3fffXft37+/qqrGx8er0+lM7/fMM8/UrbfeOn1/aGio3nzzzX/tSeR8jUva\n6tWr2/DwcGuttWPHjrVut9sOHDjQWmvtiy++aN1ut504caK11trGjRvb2rVr2/fff9+mpqbaxo0b\n244dO1prrb311lvt0UcfbVNTU+3cuXNtbGys/fLLL6211h566KG2ffv2dvr06Xbo0KG2bNmy6ed8\n5ZVX2k033dQ++uij1lprU1NT7f33328PPvjg9IwjIyNt5cqVrbXWzpw509auXdt2797dzpw504aH\nh1t/f387cuTI9OsZGxtrrbW2bt26dvvtt7fDhw+31lpbtWpVO3To0L/rVNJas8L4P9D+/HOhvXv3\n1qpVq2rFihVVVbV8+fK6+eab69NPP53e9957760bbrihent764477qhDhw5VVVVPT0+dPHmyjhw5\nUjNmzKhFixbV5ZdfXseOHauvv/66nnzyyZo5c2YtXLiwNmzYUHv27Jk+Zn9/f61Zs6aqqnp7e//h\nrAcPHqxTp07VI488Uj09PbVs2bJavXp1ffDBB1VVtWTJkhodHa3jx49XVdW6devqyy+/rPHx8fr1\n119r4cKF/6Kzxt/Tc7EH4D/n6NGjtW/fvvr444+r6k8hOXv2bC1fvnx6n2uuuWb69uzZs+vUqVNV\nVXXPPffUsWPH6oknnqiff/65BgcH6/HHH6/Jycm68sora/bs2dOPmz9/fo2NjU3fnzdvXjzj5ORk\n9fX1nbdt/vz5NTk5WVVVnU6n9u/fX9ddd111u93qdru1Z8+e6u3trcWLF1/A2eD3EIxL3F9/+tDX\n11fr16+v7du3X/Bxenp6auvWrbV169Y6evRobd68uRYsWFC33XZb/fTTT3Xq1KmaM2dOVVVNTEzU\n3Llz/+4M/8zcuXNrYmLivG1Hjx6tBQsWVFVVt9utF198sfr6+qrT6dTAwEA999xz1dvbW91u94Jf\nFxfGJckl7tprr63x8fGqqhocHKz9+/fX559/XufOnaupqakaHR2tH3/88Z8eZ2RkpL777rs6d+5c\nzZkzp3p6euqyyy6refPmVX9/f7300kt1+vTp+vbbb+vdd9+twcHB3zXvLbfcUnPmzKnXXnutzp49\nWyMjI/XJJ5/UnXfeWVVVN954Y82aNav27t1bnU6nrrjiirr66qvrww8/PO8NUf49BOMSt3nz5tq1\na1d1u93at29f7dq1q3bv3l3Lly+v1atX1xtvvDH9Hsc/WgkcP368tm3bVosXL6677rqrli5dOh2F\nHTt21Pj4eK1YsaK2bdtWjz322HmXORdi5syZ9eqrr9aBAwdq2bJl9fzzz9cLL7wwvcKo+tMq46qr\nrpq+1PlLKBYtWvS7npPcjNb8gQ6QscIAYoIBxAQDiAkGEPuv/R7GxN7+iz0C/F/rG/z6b7ZZYQAx\nwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQE\nA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMM\nICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCA\nmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBi\nggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJ\nBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYY\nQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAA\nMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHE\nBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhAT\nDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEww\ngJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEA\nYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOI\nCQYQEwwgNqO11i72EMD/BisMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBi\nggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiP0RoqNMBlokHDIAAAAASUVORK5CYII=\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f97ee42bb90\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a8e54767-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"a8e54763-8903-11e8-99f9-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_28bd08ac10" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ea9efc10\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a8e54768-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_ae2887f57d" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ea9efb50\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a8e54769-8903-11e8-99f9-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", - "//# sourceURL=js_608805a786" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ea9ef710\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a8e5476a-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"a8e54769-8903-11e8-99f9-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_3d87cf7d0f" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ea9efa90\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a8e5476b-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_5e91101199" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ea9efa50\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\u003cdiv class=id_45185901 style=\"margin-right:10px; display:flex;align-items:center;\"\u003e\u003cspan style=\"margin-right: 3px;\"\u003e\u003c/span\u003e\u003c/div\u003e" - ], - "text/plain": [ - "\u003cIPython.core.display.HTML at 0x7f97ee42bd90\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a8e5476c-8903-11e8-99f9-c8d3ffb5fbe0\"] = jQuery(\".id_45185901 span\");\n", - "//# sourceURL=js_f43052a94e" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ea9ef750\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a8e5476d-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"a8e5476c-8903-11e8-99f9-c8d3ffb5fbe0\"].text(\"Give me a color name (or press 'enter' to exit): \");\n", - "//# sourceURL=js_bfc0fb76ce" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ea9efb10\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a9e9b8b0-8903-11e8-99f9-c8d3ffb5fbe0\"] = jQuery(\".id_45185901 input\");\n", - "//# sourceURL=js_7f167283fa" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ea9ef610\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a9e9b8b1-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"a9e9b8b0-8903-11e8-99f9-c8d3ffb5fbe0\"].remove();\n", - "//# sourceURL=js_016ae4bf21" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ea9ef250\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a9e9b8b2-8903-11e8-99f9-c8d3ffb5fbe0\"] = jQuery(\".id_45185901 span\");\n", - "//# sourceURL=js_e666f179bc" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ea9ef550\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a9e9b8b3-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"a9e9b8b2-8903-11e8-99f9-c8d3ffb5fbe0\"].text(\"Give me a color name (or press 'enter' to exit): \");\n", - "//# sourceURL=js_cbb9d14aec" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ea9ef1d0\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"a9e9b8b4-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"a8e54768-8903-11e8-99f9-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_2967a79665" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f97ea9ef1d0\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - } - ], - "source": [ - "def run_input_loop(sess, inference_ops, color_name_placeholder):\n", - " \"\"\"Helper function that reads from input and calls the inference ops in a loop.\"\"\"\n", - "\n", - " tb = widgets.TabBar([\"RNN Colorbot\"])\n", - " while True:\n", - " with tb.output_to(0):\n", - " try:\n", - " color_name = six.moves.input(\"Give me a color name (or press 'enter' to exit): \")\n", - " except (EOFError, KeyboardInterrupt):\n", - " break\n", - " if not color_name:\n", - " break\n", - " with tb.output_to(0):\n", - " tb.clear_tab()\n", - " sess.run(inference_ops, {color_name_placeholder: color_name})\n", - " plt.show()\n", - "\n", - "with tf.Graph().as_default():\n", - " # Read the data.\n", - " batch_size = 64\n", - " train_data = load_dataset(data_dir, train_url, batch_size)\n", - " eval_data = load_dataset(data_dir, test_url, 50, training=False)\n", - " \n", - " # Create the model components.\n", - " lower_cell, upper_cell, relu_layer = model_components()\n", - " # Create the helper placeholder for inference.\n", - " color_name_placeholder = tf.placeholder(tf.string, shape=())\n", - " \n", - " # Compile the train / test code.\n", - " tf_train_model = autograph.to_graph(train_model)\n", - " train_model_ops = tf_train_model(\n", - " train_data, eval_data, batch_size, lower_cell, upper_cell, relu_layer, train_steps=100)\n", - " \n", - " # Compile the inference code.\n", - " tf_inference = autograph.to_graph(inference)\n", - " inference_ops = tf_inference(color_name_placeholder, lower_cell, upper_cell, relu_layer)\n", - " \n", - " with tf.Session() as sess:\n", - " sess.run(tf.global_variables_initializer())\n", - " \n", - " # Run training and testing.\n", - " sess.run(train_model_ops)\n", - " \n", - " # Run the inference loop.\n", - " run_input_loop(sess, inference_ops, color_name_placeholder)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AHJ2c47U-A5W" - }, - "source": [ - "# Where do we go next?\n", - "\n", - "AutoGraph is still in its early stages, but is available in [tensorflow.contrib](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/autograph). We're excited about the possibilities it brings. New versions will be available soon — stay tuned!" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "default_view": {}, - "name": "Dev Summit 2018 - Autograph", - "provenance": [ - { - "file_id": "1wCZUh73zTNs1jzzYjqoxMIdaBWCdKJ2K", - "timestamp": 1522238054357 - }, - { - "file_id": "1_HpC-RrmIv4lNaqeoslUeWaX8zH5IXaJ", - "timestamp": 1521743157199 - }, - { - "file_id": "1mjO2fQ2F9hxpAzw2mnrrUkcgfb7xSGW-", - "timestamp": 1520522344607 - } - ], - "version": "0.3.2", - "views": {} - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/autograph/examples/notebooks/graph_vs_ag_vs_eager_sum_speed_test.ipynb b/tensorflow/contrib/autograph/examples/notebooks/graph_vs_ag_vs_eager_sum_speed_test.ipynb deleted file mode 100644 index 32742bec7ee..00000000000 --- a/tensorflow/contrib/autograph/examples/notebooks/graph_vs_ag_vs_eager_sum_speed_test.ipynb +++ /dev/null @@ -1,519 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "moMkWaT_TTHi" - }, - "source": [ - "This Colab illustrates the differing overhead* between a custom, vectorized graph operation and a loop over a tensor\n", - "that computes the same function. The loop is implemented in TensorFlow Eager mode using Python syntax and control-flow, and using AutoGraph which takes a python function and converts it into graph mode. In AutoGraph the Python loop is converted into a tf.while_loop.\n", - "\n", - "The actual computation, summing a small number of scalar values, takes very little time to compute, so the graphs below are showing the overhead of the differing approaches. As such, this is more of a \"micro-benchmark\" than a representation of real-world performance of the three approaches.\n", - "\n", - "*Note the differing scales of the included plots" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "a0X_rfvuav98" - }, - "source": [ - "### Imports" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "EdxWv4Vn0ync" - }, - "outputs": [], - "source": [ - "!pip install -U -q tf-nightly" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "erq3_S7QsjkU" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import\n", - "from __future__ import division\n", - "from __future__ import print_function\n", - "\n", - "import numpy as np\n", - "import tensorflow as tf\n", - "import matplotlib.pyplot as plt\n", - "import math\n", - "import time\n", - "import random\n", - "from colabtools import adhoc_import\n", - "from tensorflow.contrib import autograph as ag\n", - "from tensorflow.python.framework import function" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1JgnsXooa2RP" - }, - "source": [ - "### Testing boilerplate" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "UyD5LLjVZzny" - }, - "outputs": [], - "source": [ - "# Test-only parameters. Test checks successful completion not correctness. \n", - "burn_ins = 1\n", - "trials = 1\n", - "batches = 2\n", - "max_elements = 2" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4_NBL0RQa8gY" - }, - "source": [ - "### Speed comparison parameters" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "Yq6daecyiJV5" - }, - "outputs": [], - "source": [ - "#@test {\"skip\": true} \n", - "burn_ins = 3 # Batches not counted in the average\n", - "trials = 10 # Batches run per vector-size (and averaged)\n", - "batches = 1000 # Number of random vectors summed over per trial\n", - "max_elements = 100 # Vectors of size 0 to this-1 will be executed and plotted" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fiR8m13CbKH2" - }, - "source": [ - "### Random input" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "d8vrTlyNXuxc" - }, - "outputs": [], - "source": [ - "# Construct a random num x 1 tensor\n", - "def get_elements(num):\n", - " return tf.random_uniform(shape=(num, 1), maxval=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ILJ6SbF3bXFQ" - }, - "source": [ - "## Graph mode" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "vovRf597X55n" - }, - "outputs": [], - "source": [ - "def tf_sum(elements):\n", - " # Using custom vectorized op\n", - " return tf.reduce_sum(elements)\n", - "\n", - "def run_trial(num):\n", - " elements = get_elements(num)\n", - " return tf_sum(elements)\n", - "\n", - "\n", - "\n", - "graph_means = []\n", - "for num in range(max_elements):\n", - " with tf.Graph().as_default():\n", - " durations = []\n", - " foo = run_trial(num)\n", - " \n", - " with tf.Session() as sess:\n", - " \n", - " for _ in range(burn_ins):\n", - " for _ in range(batches):\n", - " sess.run(foo)\n", - " \n", - " for _ in range(trials):\n", - " \n", - " start = time.time()\n", - " for _ in range(batches):\n", - " sess.run(foo)\n", - " \n", - " duration = time.time() - start\n", - " durations.append(duration) \n", - " \n", - " graph_means.append(np.mean(durations)) " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 301 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 278, - "status": "ok", - "timestamp": 1532447361278, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "Jm9Blkyx90Eq", - "outputId": "d83cd51f-7e56-4d73-f7df-bb157dee46df" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa8AAAEcCAYAAABwNTvaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3WdgVFXegPFnZtI7kJ5QQwlCKIGERGroEoSAYZFVEFAR\ngV1XXHvbFWEtK6xlUVgRXMuLiqBSZQUh9E5ChxRIn/ReJjNz3g8hNxkzCUMJkHh+X2DmtnPPnXv+\n95R7ohJCCCRJkiSpGVHf6QRIkiRJ0vWSwUuSJElqdmTwkiRJkpodGbwkSZKkZkcGL0mSJKnZkcFL\nkiRJanaaJHitWLGCV199tSl23WINHz6cAwcONPlxXnzxRd5///0mP87dIikpiUmTJtGvXz++/PLL\nO52cFi0+Pp4HHnjgTifjmsaPH8+RI0du6T5/b/eVObcqX9966y3Wrl17zfWsbmTnffv2RaVSAVBe\nXo6NjQ1qtRqVSsUbb7zBE088cSO7vW5paWmMGDGCs2fPolY3n0rkiy++iLe3N0899dSdTspdoSmv\n46effsqAAQPYsGGD2eVbt27l888/5/z58/Tq1Yv//ve/JsvPnTvHyy+/TGJiIgEBASxevJjAwEBl\n+bvvvsu6detQqVQ88MADPPvssxZv29SmT5/OxIkTiY6Ovi3H++CDD3jsscea9BjDhw9n8eLFhIeH\n3/A+Nm3adAtT1LgVK1bwySefoFKp0Ov16PV67OzsEELg7+/Pxo0bCQwMxN7eHpVKhRACa2trDh8+\nfNvSeCPMlWG3Kl8fffRRpkyZQnR0NFZWDYeoGyopTpw4wfHjxzl+/Di+vr6sWLFC+W78+PE3nOjr\nJYRQLrjUfDXldUxPT6dz584NLndzc2PmzJnMmTOn3rKqqirmz59PVFQUR44cISoqinnz5qHX6wFY\nu3YtO3fuZOPGjfz000/s2rWLb775xqJtm4PruR7Z2dkcOnSIESNGNGGKbo7BYLjtx3ziiSeUsvHv\nf/87ffv25fjx45w4cYKNGzcCoFKp+Omnn5Tv73TguhP5VJeHhwcBAQHs3Lmz0fVu+jFXCFHvR/7R\nRx8pT6BpaWkEBgayfv16hg0bxoABA1i7di2nTp1iwoQJhIaGsmjRIpPt161bx7hx4xgwYACPPfYY\n6enpZo89ffp0APr3709wcDCxsbEIIVi+fDnDhw9n4MCBvPDCC5SUlJjdPj8/n7lz5xISEsKAAQN4\n+OGHlWWBgYGkpKQon+s2Cxw+fJihQ4fy6aefcu+99zJ48GB++eUXdu/ezZgxYxgwYAArVqwwe8xv\nv/2WjRs38umnnxIcHMyTTz6pLDt37hwTJkwgJCSEhQsXotPplGW//vorUVFRhISEMG3aNC5cuGB2\n/wAJCQnMnj2bAQMGcN9997F169YG121sv8OHD2fVqlVMmDCBvn378sorr5Cbm8vjjz9OcHAws2fP\npri4WFn/5MmTPPjgg4SEhBAVFWVyE06fPp3333+fadOmERwczKOPPkpBQYGyDEyvY3JyMtOnT6d/\n//6Eh4ezcOHCBs9hx44djB8/ntDQUGbMmEFiYiIAjzzyCIcOHeKNN94gODiYK1eu1Ns2PDycsWPH\n4uHhUW/Z4cOHMRgMzJgxA2tra6ZPn44QgoMHDwLwww8/MHv2bDw9PfH09GTWrFlKDe/QoUONblvX\nli1b6jW3rVmzhnnz5gGg0+l4++23iYiIYNCgQfztb38z+W388ssvREVF0a9fP0aPHs3evXtZtmwZ\nx44dY9GiRQQHB/Pmm28CcPz4caKjowkJCWHKlCmcOHHC5BotW7aMadOm0adPH1JTU1m/fj0jR44k\nODiYkSNHNvh0vW/fPnr06IGNjQ0AK1eu5M9//rPJOm+++SaLFy8GoKSkhJdffplBgwYxdOhQ/vWv\nf5mUI99++y3jxo0jODiY8ePHc+7cOZ577jkyMjJ48sknCQ4OZtWqVUD965+QkKDsZ/jw4fznP/9R\nfsMGg8GkiT4kJITg4GCCg4Pp27cvgYGBSnnT2L1x9uxZJk+eTL9+/Xj66aeprKw0my+WsPQhobGy\nraac/fbbbxk8eDCDBw9m9erVJtuuXLmSUaNGERYWxtNPP01RUZHJtuvWrSMiIoKZM2cC8NRTTzFo\n0CBCQkKYPn26kq8NlWF181Wn07F48WIGDx7MkCFDWLJkCVVVVUBt+bl69Wql/Fy/fr3JuYaEhLBr\n165rZshNiYiIEPv37zf57sMPPxTPPvusEEKI1NRU0a1bN/H666+LyspKsW/fPhEUFCTmz58v8vLy\nRGZmpggPDxdHjhwRQgjxv//9T4wePVokJiYKg8EgPv74YzF16lSzx05NTRWBgYHCaDQq33333Xdi\n9OjRIjU1VZSVlYkFCxYoafmt9957T7z++uvCYDAIvV4vjh49qiwLDAwUycnJyucXXnhB/Otf/xJC\nCHHo0CFxzz33iOXLlwu9Xi++/fZbERYWJp555hlRVlYmLl26JIKCgkRKSorZ49bdV918nDJlisjO\nzhaFhYXivvvuE2vXrhVCCHH69GkRHh4u4uLihNFoFBs2bBARERFCp9PV23dZWZkYOnSo2LBhgzAa\njeLs2bNiwIABIj4+vt6xr7XfiIgIMXXqVJGbmyu0Wq0IDw8XkyZNEufOnRM6nU7MmDFDfPTRR0II\nITIzM0VoaKiIiYkRQgixf/9+ERoaKvLy8oQQQjz88MNi1KhR4sqVK6KyslI8/PDD4r333mvwOi5c\nuFB88sknQgghKisrxbFjx8zmZWJioujTp4/Yv3+/0Ov14j//+Y8YNWqUqKqqUo773Xffmd22rm+/\n/VZMnz7d5LvVq1eLxx9/3OS7J554QqxevVoIIUS/fv1EbGyssuzUqVMiODjYom3rKi8vF8HBweLK\nlSvKdw888IDYsmWLEEKIN998Uzz55JOiqKhIlJaWirlz54qlS5cKIYSIjY0V/fr1U+5BrVYrEhMT\nzZ57QUGBCAkJET/99JMwGAxi06ZNIiQkRBQUFCjrR0REiPj4eGEwGERxcbEIDg4Wly9fFkIIkZ2d\nrfyOfuvtt98Wb7zxhvI5LS1N9OnTR5SUlAghhDAYDGLgwIFKfj355JPi9ddfFxUVFSI3N1dMmTJF\nfPPNN0IIIbZs2SKGDBkiTp8+LYQQIjk5WaSnpwshqn+TBw4cUI5zresfEREhoqKiRGZmpqisrFS+\n+22ZJYQQS5cuFQ8//LDQ6/WN3hs6nU5ERESIzz//XOj1erFt2zbRo0ePevf0b61fv1788Y9/rPd9\nt27dTMqahjRWttWUswsXLhQVFRXiwoULIiwsTDnP1atXi6lTpwqtVit0Op147bXXxMKFC022ff75\n50V5ebmST99//70oKysTOp1OLFmyREycOFFJS0NlWM3x/vWvf4mpU6eKvLw8kZeXJ6ZOnSref/99\nIURt+fnhhx8KvV4vdu3aJXr37i2KioqUfW3fvl1MmjSp0fy4LR1FKpWK+fPnY2Njw7333ou9vT2R\nkZG0atUKLy8v+vfvz9mzZwH45ptvmDNnDh07dkStVjNnzhzOnz9PRkZGYwFY+f+mTZuYOXMmfn5+\n2Nvbs3DhQrZs2YLRaKy3nZWVFdnZ2aSmpqLRaOjXr5/ZfZpjbW3N3Llz0Wg0jBs3jvz8fB555BHs\n7e3p3LkznTt3brR2ZM6MGTNwd3fHxcWFiIgIzp07B8B3333Hgw8+SFBQECqViqioKGxsbIiNja23\nj19//RV/f3+ioqJQqVR0796d0aNHs23btnrrWrLfhx9+mNatW+Pp6Un//v3p3bs3gYGBWFtbM2rU\nKCWNP/30E8OGDWPw4MFAdY2mZ8+e7N69W9nX5MmTadeuHTY2Ntx3333KtjXq5rmVlRVpaWlotVps\nbGwIDg42m2dbt25l2LBhhIeHo9FoePTRR6moqDCpUdyosrIynJ2dTb5zcnJSnnZ/u9zZ2ZmysjKL\ntq3Lzs6OESNGKLWay5cvk5SUpDTBrVu3jhdffBFnZ2ccHByYM2eOsu66deuIjo5W+oA8PT3p2LGj\n2fPZtWsXHTp04P7770etVhMZGUmnTp349ddflXUmTZpEQEAAarUajUaDRqPh4sWLVFZW4u7uTkBA\ngNl9FxcX4+joqHz29fXlnnvu4ZdffgHgwIEDODg40KtXL3JyctizZw8vvfQStra2tG7dmkceeYTN\nmzcr5/TYY4/Ro0cPANq2bYuPj4+y77q/E0uu/4wZM/Dy8lJqheZs2bKFTZs28eGHH6LRaBq9N2Jj\nY9Hr9cyYMQONRsOYMWPo2bNng/u2xKRJkwgJCSE0NFSpnf6WJWXbn/70J2xtbenatSuTJ09W8vTb\nb7/lL3/5C56enlhbWzN//nx+/vlnZVuVSsWf/vQn7OzslHyaPHky9vb2yvrnz59vsBXLXFrnz59P\nq1ataNWqFQsWLODHH39UlltbWzNv3jw0Gg1Dhw7FwcGBpKQkZbmjo6NJq445NzRg40a0adNG+b+d\nnR3u7u7KZ1tbW+WmT09PZ/Hixbz99ttAbX+IVqs1+QE3JCsrC19fX+Wzn58fer2enJwcPD09TdZ9\n7LHH+PDDD5k9ezYqlYopU6aY7fswx83NTRm0YmdnZ/Yca87JUnW3t7e3Jzs7G6jOkx9//FEZLSeE\nQK/Xk5WVVW8f6enpnDx5ktDQUGVdg8FAVFSU2XWvtd+6abK1ta33ue5127p1q1IQ1uyrbsd63Wtu\nb2/faP4899xz/Otf/yI6OlrplzI3ku2311ulUuHj44NWq21w35ZycHCod7OWlJTg5ORkdnlJSQkO\nDg4WbftbkZGRvPPOO8ybN49NmzYxcuRIbGxsyMvLo7y83OTcjUajUoBnZmYydOhQi87nt3kF1UGm\nbl55e3sr/7e3t2fZsmWsWrWKl156iX79+vHcc8/RqVOnevt2cXGhtLS03jlt3ryZiRMnsmnTJqU/\nPD09Hb1ez6BBg4Daroea+zszM5N27drd0DmZu/51z8mcs2fPsmjRIlavXo2bm5uSxsbuDS8vL5N9\n+Pn5WZTehmzYsIG2bds2uk5jZRtUn3vdc/X19eXSpUvK+SxYsEAZECWEwMrKStkWTPPJaDSydOlS\nfv75Z/Lz81GpVKhUKvLz8xv8DTeWVl9fX5Nyxc3NzWRwlp2dncnvp7S0tN7D32/dtuBlKW9vb558\n8kmLBn7UBI+6PD09TfrI0tLSsLKyMik4azg4OPD888/z/PPPk5CQwPTp0+nVqxdhYWHY29tTXl6u\nrJudnX3Nm6CpeHt7M3fuXItGcfr4+DBgwAClP+BW7deS40ZFRfHGG29c97bmrmObNm2UvtBjx44x\na9YsQkND693gnp6eyg1aIyMj45Zcqy5durBmzRqT7y5evKj00XXu3Jnz588TFBQEVPdZdunSpdFt\n6/ar1jVo0CBefPFFzp8/z+bNm3nppZcAaNWqFfb29mzatKnewxdUX8O6fbN1/TZfPT092b59u8l3\n6enpDBkypMFtBg4cyMCBA9HpdCxbtoxXX32Vr776qt6xunXrZvJkDTB27FjeeecdtFotv/zyizKY\nxcfHB1tbWw4dOmT22nt7e5OcnGzxOd3M9c/Ly2PBggW8/vrrJiNBG7s3jhw5Uu/hKD093eKAe6Ma\nK9syMjIQQpCRkaHUvDMyMpTfjI+PD0uWLKFv37719puWlgaY5u3GjRv59ddf+fzzz/H19aW4uJiQ\nkJDrSmtaWppSU09PTzf7+21IQkLCNUfm3pZmw2s1wdU1bdo0VqxYQXx8PFDdHGGuyQugdevWqNVq\nkx96ZGQka9asITU1ldLSUpYtW0ZkZKTZIdi7du1StnVwcFCaSaB6wMamTZswGo3ExMTc0vdC3N3d\nGyxwzPnDH/7A2rVriYuLA6qbpHbv3m225jJs2DCSkpL48ccf0ev1VFVVcerUKWUQw43u91omTJjA\nzp072bt3L0ajkcrKSg4fPmxRDcjcddy2bZuyrYuLC2q12uw1vO+++9i1axcHDx5Er9ezatUqbG1t\n6dOnj0XpNhqN6HQ69Hq9yf8BQkNDUavVfPHFF+h0OuUpfMCAAQBERUWxZs0atFotWq2WNWvWMHny\n5Ea3DQsLM5uOmuand955h6KiIgYOHAigtAgsWbKEvLw8ALRaLXv37gUgOjqa9evXc/DgQYQQaLVa\n5Vr/9nc2dOhQrly5wubNmzEYDGzZsoXExEQiIiLMpik3N5edO3dSXl6OlZWVco+YM3DgQM6cOWMy\nkKR169aEhITw4osv0rZtW6XG5uHhwcCBA1myZAklJSUIIUhJSVHusSlTpvDZZ59x5swZAJKTk5Vu\nA3d3d1JTU5Vj3Mz1NxgM/OlPf2LChAmMHTvWZFlj90afPn2wsrLiiy++wGAwsH37dk6dOnXN490s\nS8q25cuXU1FRwaVLl1i/fj2RkZEATJ06laVLlyrBLy8vjx07dijb/baMLi0txcbGBhcXF8rKynjv\nvfdMgtu1yrDIyEg+/vhj8vLyyMvLY/ny5UycONHicz1y5IjJQ5U5Nx28zD05XWudxj6PHDmSxx9/\nnKeffpr+/fszYcIE9uzZY3a/dnZ2zJ07l2nTphEaGkpcXBzR0dFMnDiRhx9+mFGjRmFvb88rr7xi\ndvvLly8zc+ZM+vbty7Rp03jooYeUp4uXX36ZnTt3EhISwubNmxk5cuRNnWNd0dHRxMfHExoayoIF\nC665fs+ePVm0aBFvvPEGoaGhjBkzpsH3lhwdHfnss8/YsmWLMurovffeMylULN3v9ZyTt7c3y5cv\nZ8WKFYSHhxMREcFnn32m3BSNbWvuOp46dYopU6YQHBzM/Pnzefnll802zXTs2JF3332XRYsWER4e\nzq5du/jkk0+U90Ou9fv88ccf6dWrF2+88QbHjh2jd+/eygv21tbWLF++nA0bNhAaGsr69etZvny5\nsu8HH3yQiIgIJkyYwIQJE4iIiOAPf/iDRduaExkZyYEDB7jvvvtMCqS//vWvtG/fnj/84Q/079+f\n2bNnc/nyZQB69erFkiVLWLJkCf369WPGjBlKQT9jxgy2bdvGgAEDWLx4MW5ubnzyySesWrWKsLAw\nVq1axYoVK3B1dTWbV0ajkdWrVzNkyBDCwsI4cuQIr7/+utm0t2nThrCwMKWPq8b48eM5cOAA999/\nv8n3b7/9NlVVVURGRhIaGspTTz2lNJOPHTuWuXPn8swzzyjXv7CwEIA5c+awfPlyQkNDWb169Q1d\n/5rvMjMzOX78OJ9//rky2jA4OJjMzMxG7w1ra2s+/PBD1q9fT2hoKNu2bWP06NENXtdrsaQMBSwq\n20JDQxk1ahSzZs3iscceU5rtH3nkEUaMGMHs2bPp168fDz74oBKYzaUhKioKHx8fhgwZwvjx4+vV\n2K5Vhs2bN4+ePXsyYcIEJk6cSM+ePZk7d65FeZCVlUVCQsK1y1xxPdUiSZKkBiQkJPDCCy/w3Xff\n3emk/O6kpaUxcuRIzpw506wmbDDn7bffpl27dkybNq3R9WTwkiRJauaa62xDN+P3cZaSJEktnKXN\njy2FrHlJkiRJzY6seUmSJEnNzl33ntfN0OsN5Odf/zDvlqhVKweZF1fJvKgl86KWzItaHh6NvxB8\nN2pRNS8rK/PvoPweybyoJfOilsyLWjIvmrcWFbwkSZKk3wcZvCSpBSsq1VFcVv8FdUlq7mTwkqQW\n7J9rT/LBurhrryhJzUyLGrAhSVIto1GQnlOKrY3s25FaHlnzkqQWqri8CqMQlFfq0Rvq/z07SWrO\nZPCSpBaqsKT2T9MXl1XdwZRI0q0ng5cktVAFJbUDNeSgDamlkcFLklqoujWvIhm8pBamyYNXTEwM\nY8eOZcyYMaxcubLe8qNHjzJ58mR69OhR76+8QvWfTh8yZAhvvvlmUydVklqUgtI6Na9S2WwotSxN\nGryMRiOLFi1i1apVbNq0ic2bN5OQkGCyjq+vL2+99Va9P1ZX4/333yc0NLQpkylJLZJpn5eseUkt\nS5MGr7i4ONq3b4+fnx/W1tZERkaa/OlpqA5eXbt2NTud/+nTp8nLy2PQoEFNmUxJapEK6/R5FckB\nG1IL06TBS6vV4uPjo3z28vIiKyvLom2FELz99ts899xzyL/aIknXr6BU1ryklqtJX1K+maDz9ddf\nM2zYMLy8vK5rX81xduSmIvOi1u8xL4rL9djbaiivNFCpF0oe/B7zoiEyL5qvJg1e3t7epKenK5+1\nWi2enp4WbXvixAmOHz/O119/TWlpKXq9HkdHRxYuXNjodtnZxTeV5pbCw8NZ5sVVv8e8EEKQV1iB\nn4cjqVkl5BaUkZ1d/LvMi4bIvKjVHIN4kwavoKAgkpOTSUtLw8PDg82bN7N06dIG169bu/rnP/+p\n/H/Dhg2cOXPmmoFLkqRqZVdn1WjlZEthSaUcKi+1OE3a56XRaHj11VeZPXs248ePJzIykoCAAD74\n4AN+/fVXAE6dOsXQoUPZtm0br7/+eoOjDiVJslxBcXV/l6uTDS4ONnLAhtTiqEQLGw0hmwGqySaR\nWr/HvDhzOY/31p5kwsAOJKQVcuZyPp88MxQ/X7ffXV405Pf4u2hIc2w2lDNsSFILVPOOl5uTLc6O\nNoCc31BqWWTwkqQWqOYdL1cnG5ztrwavctnvJbUcMnjdJpU6A2u2nufoecvec5Okm1EzKa+bky0u\njtYAFMkpoqQWRAav2+RkfA4xseks/+E0KzeeobRCFiRS0ym8+oKyq6MNzg41zYay5iW1HPIvKd8m\nlzOLAGjjYsfBM1ouJBfwxIQedG3rdodTdnuUlFdx8lIO+cUV5JfoqKjUM3FQR7xaO9zppLVIBSU6\nVICLow3ODtU1L9nnJbUkMnjdJpczilEBf58dwi9HU/lp32VWbz3PP+aE3emk3RZfbr/A4XOmTaat\nXGyZMqzzHUpRy1ZYUomzgzVWGjUuV2te8l0vqSWRzYa3gVEILmuL8XF3xMHOmgmDOtLZz4WsvDKq\n9IY7nbwmpzcYiUvIpY2LLU//oTcvPhwMQFp26R1OWctVUKrD1ckWoE7Nq+UEr5LyKjnn6e+cDF63\ngTavjEqdgQ7ete9SeLdxQADa/PI7l7Db5GJKARU6A327eBDUqQ1d/N1wdbIhNbvkTietRarQ6anU\nGXB1qq5x1fZ5tYxmw0upBfz5/T38c+1Jsgta/v0jmSeD121wObP6RUiT4NXaEYDM3LI7kqbbKTY+\nF4Dend2V7/w9nMgrqqRMDly55WqGybs5Vte87Gw0WGnULabmde5KvvLvq6sO8cvRFIyyFva7I4PX\nbXA5oyZ4uSjfeV8dqJCZ1/KDV1xCDrY2GpPBKf4e1cE7VTYd3nIFJbVTQwGoVCpcHK1bzFD5mt/M\nlIgArDVqvv7lEv/dduEOp0q63eSAjdvgcmYRKhW09XJSvvNu8/sIXpl5ZWjzy+nX1QNrq9pnJX+P\n6rxIyy753Yy4vF56g5HSCj2uV2fIsFRhae07XjWc7W3IyL27HxTKKqrYeiiZvKJKist1lJRVMaS3\nL8P6+pmsl5Zdgr2tFWND23FvTx+WfHGUg2cyeWhUF6ytNHco9dLtJmteTcxoFFzRFuPn7oitde2N\n5e5qh0atQtvCg1dsfA4AvTq3Mfm+JnjJmlfDvvrfRZ7/ZD9FpdfX3KdMylsn6Dk7WqPTG6mo1N/S\nNN5K2w4ns/nAFQ6cyeR0Yh6XM4v5+UiKyTpVegPavHL8PBxRqVS4OtrQr6snOr2R88kFdyjl0p0g\ng1cTy8gtRVdlNGkyBLDSqPFwsyczr6xFj5pSgleAu8n3Pm0cUKmQgzYaUFpRxf7TmeiqjCSmF13X\ntgVmal41w+VrmhTvNlV6I7tPpuNoZ8U/ngjj44VD6d6+Fdq8MpMX+tNzyjAKQVuP2laMoIDqB6O4\nq32rUjVtfhknLmbf6WQ0GRm8mpgyWMOn/qzN3q0dKK3QU1x+Y30Rd3vQK6uo4lJqIR19XOo1fdlY\na/Bq5UBqduldfx51ZeSWsnzDKfKLmzYIHDidSZXeCNS+4G6pwt/0eUHtcPnrrcXdLkfPZ1FcVsXg\n3r54tXLA1kZDJ9/qB76kOsG75mGnps8UoIu/K/a2GmITcprVb6mprd5yng/XnyIhvfBOJ6VJyODV\nxMwN1qihDNpoZMThtzvj+b9fLtX7Pq+ogqc+2Mv2w8m3KKW33umkPAxGQe+ANmaX+3s4Ul6pNwkE\nCWmFrN1xCYPReLuSeV02xCRy9EI2/zuaUm9ZUZmO9JybbwYVQhATm45GrQJqH4AsVTuvYW3wuttr\nXjuOp6ICIur0b9UEr8SM+sHLr07Ny0qjpkeH1uQUVpDxOxi9a4n84koupVQ3o/58uP5vtSWQwauJ\nXc4sQqNW0dbTsd6yaw3aiE8tZNvhZP53NKVeoRgTm05JeRUHz2pvfaJvQn5xJTmF5RSWVHL8apNF\n3SHyddX2e9U2Ha7deYntR1K4eBf2X+QUlnPs6jntP52J3lAbYIUQfPh9HH9bfZicm3z3KDG9iNTs\nUvp29aCNiy2XM4quq0ZRWKrD0c7KZPCCU03N6y4MXkkZRSSmF9G7szsebvbK9518XQFMmk1r+kjr\n1rygtlk6LkE2HQIcOZ+FADRqFccuZLXI9+Fk8GpCeoOR5KwS/DwczY6Camy4vBCC73cnKJ93nUxT\n/m8wGtkTlwHAFW3xLZvk1yjETTW7nLyUwzP/3sdzHx/g6Y/2cfhcFm5ONrSrM8qyLj9lxGF1gZSZ\nV0ZCWnVBdSn17mvq2Hk8DSGqB9sUleo4lVhbUF5MKSAhrQi9QbD54JWbOs7u2HQAhvb2pYO3C0Vl\nVdfVTFlYUqnMrlGjtuZ19zUb7jyWCsDwfqajCl0dbWjjYkdiem3wTs0uoY2LLQ521ibrKv1eCTm3\nIcV3vyPntKhUED0sACEw21LQ3Mng1YTSc0qp0tcfrFGjsWbDM5fzuJBSQI+OrXF1tGH/qUwqq6qn\nkjqVmEd+cSU21mqE4JbUUsor9Ty7fD///fnG3pcxCsH6mERUKgjr4UVIoCd9u7jz4IguqFQqs9v4\ne9a861WD0HifAAAgAElEQVRd89p/OlNZdjH17qp5VeoMxJxMx8XBmicm9gBg79UHCIBth6qbb53s\nrdkbl0FuYcUNHae8Us/hc1rcXe3o3qGV0ldqadNhld5gdnh9zSwbhXdZzauoTMehc1l4tXbgng6t\n6y3v5OtCSXkV2YUVFJfpKCzRmTQZ1nB1tKGDtzOXUgspq7gzIypv9kXp3zah36icgnIS0osIbNeK\nEf38aeVsy57YjBb3lyxk8GpCjQ3WgOpOdAdbq3o1r+paVyIAU4YFMLi3L2VXCzWAmJPVT+YPDAkA\n4Fxy/k2n9XRSdUDcfTKds5fzrnv7ExdzSM0uYcA9Xsy5vwdPRvXkTw/0IrS7V4PbeLjZY2OtJjW7\nFKMQHDidgZ2NBk83exLSiu6qfq/9pzMoq9QzrK8fAb6utPNyIjY+l8KSStJySolNyKWznysPjuiM\nwXjjta9DZ7XoqowM7u2LWqVSHnwsHbRRaKa/C8DlarPh3RS8Siuq+GFPEnqDkeHBfqjNPOQo/V7p\nhXWaDM3X5HsFtMFgFDf0+71Ze2LTmbd0NxdTbuyhq7xSz5v/Pcqrnx666Vlnjlz9m4ED7vHCSqNm\nZH9/KqsM7L5abrQUMnjdpLKKKtbuuGS2Tfn81WlsOjZQ81KpVHi3cSC7oNykoD52IZsrmcWEdvek\nnZczQ3v7olLBrhPp5BVVEJuQQwdvZ4b19cPaSq0c52bUHVL7xc8XrmvCYCEEG/cloQLGh3eweDu1\nSoWfuyMZuaWcu5xPblEl/QM96d6hFZVVBpK1t3YYfUZuKa+tOsSWRgJLld5ITGw6b311nP/+fIH8\n4kqMQvC/o6lo1CplQMHgXr7VAfeMlp+v1rrGDmjHgHu88Gxlz57Y6mt1PfQGI7+eSEOtUjEoyAeA\n9lenFKsZ+NMQo1EQl5DD59vOA9RrNqyted1cs2FZhf6mA2BiehErN55h4Uf72HUiDRdHGwb29DG7\nbm3wKjI70rCumr7V2NvcdGgUgi0Hr6CrMvLpprOUX+e7dEIIVm0+R0ZuGWWVemJiM669USMOn8tC\no1YR3NUDgKG9/bCz0fDL0RQycku5kJzP0fNZN9w6cLeQM2zcpB/2JvHL0VS0eWU8NaW38n1uYQVH\nzmfh08bBZGaN3/Ju7UBiehE5BRV4tXbAaBRs2JOIWqUianAnANq42tGrUxtiE3L5+pdLCAFD+/hi\nbaWmi78rZy/nU1Sqw+U6Z2KooTcYiU3IpY2LHX27uvPL0VS2HExm4qCOFm1/Mj6H5KwSQrt74utu\nvmBpiJ+HE0kZxayPqa5pDuzpTV5RdQ3wUkoBHX3MB/7rVVSqY9m3seQUVrBuVwKujjYMDKotMCt0\nenYcS+WXo6nKDBUXUwrYfyqD3p3dycwr496e3kpQGHCPF9/svMTO46nkF1fi1dqBPl3cUatUjA/v\nwGdbzrH54BWmj+5mUfqMQvDZ5nOkZFXXXls5Vx/Hyd4ad1c7LmcWI4Qw2wR75nIen289T87VwqiT\nr4vJuQHY2miwsVYrf6SyMWcu53HyYg4TB3fEyb62b6msooq/rT5CaYWev88Kwb3O4ApLZeWX8Y8v\nj2EwCrxa2TOkty/3BvngYGe+KGrv5YxGrSIpvQjd1Wbzhmpe7b2dcXG0ITY+l437krC11mBtrUGv\nN1JRZUBXZSCwfSt6mGmevBnnLuejzS/H0c6KnMIKvv01nkfGBlq8/ZaDVzh+MZvO/q6kaEv45VgK\no0L80ajN1y1yCsrJK640OzONNq+MK9pigjq1Ua6dg50VQ3r7sv1ICi//55Cybr+uHsyfHHSdZ3v3\naPKaV0xMDGPHjmXMmDGsXLmy3vKjR48yefJkevTowfbt25Xvz58/z4MPPsj999/PxIkT2bJlS1Mn\n9bplFZTz6/HqgRSxCbkmo6K2H0nBYBTcN6C92eaQGjX9XhlXmw73nsogI7eMgUHeyjJAmSLn+MVs\nbG00SnNc9/atADh/E02H55PzKa/U07erO5MGd6KVsy2bD1y2aOoqIQQ/7b2MCrh/oGXBri7/q8Eu\nKaMId1c7urR1o0vb6lFmF2/RoI0KnZ7318WRU1jB0D6+ONhasWbreaWJJz6tkL99doTvdydSWWVg\nbGg73pkbzsz7AnG0t1aaYUb1b6vs08nemr5dPMgprMBgFIwJbatc5/CeXni42bEnNp2kjPrNffFp\nhRw4k6k0Dwkh+HL7RQ6e1RLg58LM3xR8HXyq+33MPSnnFJTz8YbTFJRUMqS3D6/PDOGVGf3xM/MQ\n4eJgQ2EjfSpVeiPf7LzEe2tPsuN4Kh//cFoZUSmE4L8/XyCnsILySj2fbjqL0Xj9fTw198VDo7qy\nZE4Y94W1b3T6KxtrDf4eTlzRlnA5sxiNWqWM0v0ttUpFv64elJRXsWFPEmt3xvPFzxf4vx2X2BCT\nyOYDV1i+4dRN9f2cuZxXr1lv5/HqASd/eqAX/h5O7D6ZbvGoxzNJeayPSaSVsy0LJgUxMKj64e3Y\nBfMvF5+8lMNrnx3mra+Om32lpKZrIbS7p8n348LaE9bDi0G9fIgMb88fR3Zh6vDm/bf0mrTmZTQa\nWbRoEWvWrMHT05Po6GhGjBhBQECAso6vry9vvfUWn332mcm29vb2vPPOO7Rr146srCwmT57MkCFD\ncHJquBZzu22IScRgFAzr68euE2n8uDeJp//Qm5LyKmJi02nlbEtYj4b7fMB00EZlOwM/7EnExkqt\n1LpqBHVqQxsXO3KLKhjQ3Qt72+pLF6gEr4JG+5cac+JidTNLcBcP7G2tmDaiC8t/OM0XP1/gmQf7\nNBp8YxNyuaItJiTQ02yBeS1+nrXX896e3qhVKtxd7WntYsul1AKT2sbxi9moVSr6dDE/9N4cg9HI\ne18dIymjiHt7ejNjTDdCAz1Z+m0sH60/xb09vatHYgkYG9qO8fd2UGoBQ9zsCbvHi10n0hDUNuHV\nGNzLhyPns3BxsGZgT2/le41aTdSgTvxn01kWfX6UPp3dGX9vB/KLK9l2+IoyotLaSk2/bh7YWGmI\niU2nracTT0/pja2N6cjUjt7OHD2fxeXMYpPajt5g5OMfz1BWqWfWuEAG9/JtNC+cHayVl8Lr1uCM\nRkFSRhFf/HyB5KwSvFrZ08bVjrOX8/lmZzwPjerKgTOZHD6XRYCfC26Othy7mM22w8mMC2tv8bUo\nLtOxNy6DNi62DO3j2+BAnt/q5OvCFW0xydoS/D0csdI0/Mz94IguhPf0plJnoPJqbcvaSo2tjYaz\nl/PZdiiZnw+nMHlI7f1VXqlnQ0wirVxs6dGhNf6eTmZ/86cTc1n6bSyd/Vx57o99sdKoyS2s4GR8\nDu29neni78rj99/DG2uOsHrrOSYO6kh8aiHxqYU4OVgzc2wg/nV+76cTc1nx0xk0ahXzJvXExdGG\nUf3b8uvxNLYfSTG5n4UQbD5whQ0xiVhZqfFws2P7kRTSckqZO7EH1ho1567ks/dUBlYaFX27eJik\n3cXRhjn397Aov5uLJg1ecXFxtG/fHj+/6lpDZGQkO3bsqBe8gHo/5Pbta28KT09P2rRpQ15eXqPB\nq6lmD9AbjPxn41kc7ayYEtEZe1srkjKKOHRWS3tvZx4e3ZXM3FJOJeYSn1bI2aQ8KqsMRA3u2OiN\nBqbD5bcfTaGgRMf4e9srzUY11GoVYwe045ud8QwPrh1S3MHbGTsbjfJnIqC6U/7AuSw6eTnh1dr8\nU2oNoxCcuJSNk721UuPp182D3gHVzZQ/H07mvgHmC6hKnYFvdsYDcP/ADo0epyF1m4DC6wSALv5u\nHDqrJTOvDJ82jqTnlLJ8w2mMQjDzvkCG9DZfUFfqDOyJS+dCcgEZeWVk5ZehNwgC27kx875AVCoV\n3Tu0ZvqYbqzZep7tR1Jo42LHY+O7061dq3r7s7HWMDq0ndlj3dOhNSOC/Qls71bvVYjwnt44O1jz\n077LnIzP4WR8bT9Mn87udPB25sCZTA6eqX5S9mrtwDNT+9QbAg61f0onKbOI/oG1T9TrdiWQlFFE\neA8vpY+sMc4ONlTpiymr1KPNK+dMUi4XUwtJSCukQlfdJDe4lw/TRnZBCFjyxTF2HEvF0c6K7UdS\nsLPR8Pj9PXCwtSI+vZANMYn06NC6XlBvyK8n0tDpjYwKaXfN+6KuTr4u/HqiuoWjoSbDGtZWajr7\nuZpd1sXfjf2nM/nlaAqjQ9pSU7x/uf0iB85Uj3T9jgScHawZHdKWyN/032692rcZn1bI97sTmDq8\nC7tjq1+fGN7XD5VKRVtPJ6IGd+T73YnKTPf2thqyCspZ9N+jTBvRhUG9fPhhTxJbDl7BSqNi1n3d\nCbj6TptXawd6d3bnZHwO8WmFdPZzpbBUx5fbL3DsQjatXWz50+ReeLjZs3LjGeIScnn5P4eoqNSj\nuzojy5Devg02w7YkTXqGWq0WH5/am8rLy4tTp05d937i4uLQ6/W0a2e+EKnx0GtbmTaiC6NC2ja6\nXg0hBMnaElq52CrvwZizcd9lpenodFIecyb0YMPVPpo/DAtQ+qfe+uo463YlkJ5TiuPVduZr8Wxl\njwpISC/k8Dktzg7WDQaL4cF+DO7lg02dCX41ajVd27oRl5BLXlEFGo2at78+oTT5dW3rxuBePrT1\ndEKjVqFWq2jtbKc83SdlFFFQomNgkLfSxq5SqZg1rjuvrz7M97sS6eLvZrZA+L8dl9DmlTE6pO01\nC5WGuDra0M7TiVbOtni1qg20Xf1dOXRWy6XUQnzaOPLdr/EYhcDGWs3nW89jY6UmrEdtsCutqFL6\nrEquTrdlb6uhracTgR3bEBna1qTAHNLbl0qdgdyiCiYO6qjUZK+HWq3iodFdG1zes1MbenRszfkr\n+ew4noaTfXWhWNMveP/ADlxKLeR0Uh7D+vg22GdpbtDGiUvZbD+SgndrB6aP6WZRLaZmiqjnPt5P\neWXtgBzv1g508XelXzdPetWZDeVP0b1YtOYIP+27DMBj47vjebXm9+i47iz9NpaVG8/w2swQk0mn\nzdFVGdhxLBV7WysG97p2oK2rZtAGgF8DgzUsYWutYVxYe9buuMTPh5OZ2641h85qOXAmk44+zozs\n35azSXnEJuTy/e5EurVtRWf/6t/9lcxizl3Jp4u/K8VlVfx8OIWOPi7EXJ2PMfSe2lrSfQPao1ar\nsLPW0KWtG77ujsTG5/DZ5nP89+cL/Lg3icJSHZ5u9syN6lHvVZpRIW05GZ/DtkPJdGvnxg97Eimv\nNNC1rRvzonoqv5M/P9CLDXsS2XYouTroBbShd2f3BoN3S9OkwetWzDOWlZXFc889xzvvvHPNdVs5\n27J25yU6+LtxbyNNKAaj4NDpDNbviufClXzcnG15ZVYo3drX78i9mJzP5oNX8Gxlz+A+fqzfFc8/\nvjyGEBAc6MmQkOpA4+HhTJ/DKZy8VN1WPXVkV9r513+SN8ejtYPyou7MyHss3q5GSA9v4hJyOZ9W\nxC+Hk8nMK2NYP3/yCiuIi8+pN3zXyd6a52f0p09XT7ZcnTpmWP92eHjUPkF7eMDz00N45ZN9rNx4\nlg+eGaaMWAM4cCqdmNh0Ovq6MDe69039KYoPnx2uzAZQY0AvP77YfpHk7FLSCyqITcilZ0AbHpvQ\nk5c/3senm89hY2dNhc7A8QtZnE7IRVdlwMnemgdHdWP0gPa4u9k1Wqj/cdw9N5zm6+Hp6aL8Tswt\nGxh87YctH3dHkrNKcHd3YvfxVFZuPIuNlZqXZoXS1teywiqwozv7TmXiaGfN4D7+BHfzpGdAm3oj\nE2t4eDjz4sxQ/v7pQQb38WPCsNp39iI8nLmYVsSmfUms3nqBF2eG1KtNGY0C9dVruu3AZYrLqoge\n3uW6f99t2jjhaGdFaYWeHp09TH6n1yt6VDe2H0lmx7FURoZ14MvtF7Cz0fDCI6H4ejgxYRicTcrl\n+Y/28u2ueN57aihqtYrPf74IwEP3dcfd1Z6F78ew8qczGAVEDQ3A39d08MSM8T1NPo/2dKFvdx/+\n+dVRziblMaSPH/On9DZb03Z3d2Ld7gSOX8zm+MVsHO2tmTu5B2PDO5jcIwBzo/vw+OTe9b7/PWjS\n4OXt7U16eu27BVqtFk9Pz0a2MFVSUsLcuXNZuHAhvXr1uub6rz0Wxgsf7eWfXx3jWaPR7BNIfGoh\nqzafRZtfPbS9W1s3LqYW8MK/9zF7XKDJ07yuysA/vzyK0Sh4ZGwg3du3orOPM//ZdJbCEh0TwtuT\nnV37NDxuQDtOXsrG2kpNeHdPk2WN8XS1IyuvDK9W9gR3bmPxdjXaXu3A/vTH0wCMCPbnqWnB5OSU\nkFVQzqGzWopLdRiMAl2VgUPntLy+8iAPjujMvtg0bKzU+Le2r3dcb1dbJgzsyA97k3jn8yPMmXAP\ndjZW5BdX8v7aE1hbqZk9rjsF+bd+Pjk7DTjaWRF3KZv4q4NRJg/uiIuthqem9Oa9tSf56LtYZX1f\nd0cGBnkzrI9fdS1Krycnp3potYeH83Xn6d2mrYcjh3NK+ft/DnDsQjZ2NhrmTOyBk7Xa4nML7+7B\nkL5j0FfolCCkK9eRXd5wc7uvmx3LFgzC3laj5GeN+8Pbk5RWwOGzmbzz+REeHd8dtUpFUkYRq7ec\nI6ugnF4B7oQGevJ9TCIateq67ou6Ovq4cDopDxdbzU1fy7Gh7fj6l0s8/9FedFUGZt4XiDVC2a+H\nkw1h93hx8KyWDTsvck+HVuw5mYa/hyNtW9ujUqmYProrqzafA2BANw+L07RwSm+0+WXVk3IXV1Ba\nbH64emRYez7+4TThPb2JHhaAi4MNeblN9xcYbuaB4E5p0uAVFBREcnIyaWlpeHh4sHnzZpYuXdrg\n+nVralVVVcyfP5+oqChGjx5t0fE6+7vxZFRPPlgXxwfr4njuj31NmrMOn9Py6aZzGI2Cwb18GDug\nHT5tHDmVmMsnP55m5cazxKcV0qNja/zcHdlxLI2M3DJG9vNXRvV1a9eKxY+FUVymqzdUuLO/K9HD\nAnB1tLmuYettPZ04nZRH9LCA6+oLqOHvWftkOqS3D9NG1T4he7rZc/+9HUzWH9rHj482nOLrqxP+\nBnf1aLDZZ/y9HbiQUsDJ+BzmLY3BzckGlUpFaYWe6aO73tAgDUuoVSo6+7kSe3XUVngPb6V5pbOf\nKwun9mZPXAZd/F3p0aE1rV3smiQdd4sO3i4cPpfFsQvZtPV0Yt6knibNrJZQq1S0drEju/L6Rts1\n1H9ibaVm/uQg3lt7kgNnMnGws8LWWsPWQ1cQAtq42HH0fBZHrza5DwryqdeXa6lpI7uQll16S67z\n0D6+bD2UTH5xJcFdPcw2Y06J6MyJSzl8vzuB+FR3jEIwJrSdcl8NDPKpfgfQKK7Zr1yXWq3Cp821\n75ngrh6s+OswpeYq1acSTfw3BGJiYli8eDFCCKKjo5kzZw4ffPABQUFBREREcOrUKRYsWEBRURG2\ntrZ4eHiwceNGfvrpJ1566SW6dOmijI76xz/+QWBg4+9PZGcXExObzpqt51EBfbq4Mzqk7dVO1kTs\nbDTMi+pJz06mM52n55Tywbo4sn7zsrFXawf+Nuvabfo3o6xCT0pWsdkBA5baE5tOdmEFUYM6olar\nrlnbyCuq4IPv40jWljBnwj2E3ePd4LrFZTq2HLxCanYp2rwycgsrCO7qwbxJPS0eMXYjth68wne7\nErC2UvOPOWE3XHC1hJpXSlYJiz4/SngPLx4a1dWk3/N6NEVelJRX8fZXx0m7Onm0u6sds8Z1J7Cd\nGylZJRw+l8UVbTEzxnQzmXj3Tjp+MZtD57N4eGQXk+bwujbtv6y8f9jK2Za354bf0MNlc9Aca15N\nHrxut5ob89iFLDYfuGIyJ1wrZ1v+MqU3bT3NDy6o0Ok5dyWf9JxS0nJKyS+qZOqIzg3OTXg3s6SQ\nqqwycCWzmC7+rtcVhPQGIxq1qkkDF0BqVgmvrz7MxEEdmXAD75DVaAnBC6rz/WYLz6bKi/ziSlb+\ndIa2Xk5MHtIJO5u7f7TbtfKiSm/glU8PkV1QwZSIgAYHUrUEMnjdBer+GIUQJKQVsf1oCuWVemaP\n637DzRbNTUspsEvKq3C0s7qpQNlS8uJWkHlRy5K8iE8tZPfJNP44qusNjUhtLppj8Gq5V4PqId+d\n/V2V4a5S81N3eiJJut1k+XH3apkNuJIkSVKLJoOXJEmS1OzI4CVJkiQ1OzJ4SZIkSc2ODF6SJElS\nsyODlyRJktTsyOAlSZIkNTsyeEmSJEnNjgxekiRJUrMjg5ckSZLU7MjgJUmSJDU7MnhJkiRJzY4M\nXpIkSVKzI4OXJEmS1OzI4CVJkiQ1OzJ4SZIkSc2ODF6SJElSsyODlyRJktTsyOAlSZIkNTtNHrxi\nYmIYO3YsY8aMYeXKlfWWHz16lMmTJ9OjRw+2b99usmzDhg2MGTOGMWPG8MMPPzR1UiVJkqRmwupa\nK6SkpLBu3ToOHTpEZmYmtra2BAYGMmbMGEaPHo2VVcO7MBqNLFq0iDVr1uDp6Ul0dDQjRowgICBA\nWcfX15e33nqLzz77zGTbwsJC/v3vf7NhwwaEEEyePJkRI0bg7Ox8E6crSZIktQSNBq/XXnuNM2fO\nMHbsWP7617/i7u5OZWUlCQkJ7N27l5UrV/K3v/2NPn36mN0+Li6O9u3b4+fnB0BkZCQ7duyoF7wA\nVCqVybZ79+5l4MCBSrAaOHAge/bsYdy4cTd+tpIkSVKL0GjwGjFiBG+88Ua977t168a4ceMoKCgg\nJSWlwe21Wi0+Pj7KZy8vL06dOmVRwsxtq9VqLdpWkiRJatkaDV5Dhw5tdGM3Nzfc3NwaXC6EuLFU\nNbDtb2tn5nh4yGbFGjIvasm8qCXzopbMi+brmn1eAG+99Rbz58/H3t6eGTNmcPbsWf7+978zceLE\nRrfz9vYmPT1d+azVavH09LQoYd7e3hw6dEj5nJmZSVhY2DW3y84utmj/LZ2Hh7PMi6tkXtSSeVFL\n5kWt5hjELRptuH//fpydndm7dy9eXl78/PPP9QZYmBMUFERycjJpaWnodDo2b97MiBEjGly/bm1r\n0KBB7N+/n+LiYgoLC9m/fz+DBg2yJLmSJElSC2dRzavGkSNHGDVqFF5eXhY14Wk0Gl599VVmz56N\nEILo6GgCAgL44IMPCAoKIiIiglOnTrFgwQKKior49ddf+eijj9i4cSOurq7MmzePBx54AJVKxYIF\nC3BxcbnhE5UkSZJaDpWwoGNq1qxZ+Pn5sW/fPn744QccHR2ZNGkSGzduvB1pvC6yGaCabBKpJfOi\nlsyLWjIvarXYZsP33nuPzp07s2zZMlxdXcnMzGTWrFlNnTZJkiRJMsuiZsPWrVszc+ZM5bO/vz/+\n/v5NlSZJkiRJalSjwSssLKzRvq0DBw7c8gRJkiRJ0rU0Gry+//57ANatW0dBQQFTp05FCMH333+P\nl5fXbUmgJEmSJP1Wo8GrZlqnI0eO8OWXXyrfv/LKKzz88MM8/vjjTZs6SZIkSTLDogEbWVlZ5OXl\nKZ/z8vLIzs5uskRJkiRJUmMsGrDxyCOPEBUVxbBhwwDYvXs3TzzxRFOmS5IkSZIaZFHweuihh+jX\nrx9HjhxBCMFDDz1Et27dmjptkiRJkmSWxTNsBAYGEhgY2JRpkSRJkiSLWBS8jh8/zrvvvktKSgoG\ngwEhBCqVSg6VlyRJku4Ii4LXyy+/zLx58+jTpw9qtUVjPCRJkiSpyVgUvOzs7Lj//vubOi2SJEmS\nZBGLqlFDhgxh9+7dTZ0WSZIkSbKIRTWvb775hhUrVuDo6IiNjY3s85IkSZLuKIuCV800UZIkSZJ0\nN7AoePn5+aHX60lKSkKlUtGhQwesrK7r71hKkiRJ0i1jUQQ6deoUf/7zn5UmQ71ez4cffkiPHj2a\nOn2SJEmSVI9FwWvx4sUsWbKE8PBwAA4ePMiiRYtYu3ZtkyZOkiRJksyxaLRheXm5Erig+u98lZeX\nN1miJEmSJKkxFgUve3t7Dh48qHw+fPgw9vb2TZYoSZIkSWqMRc2GL730Ek899RQ2NjYAVFVV8cEH\nH1h0gJiYGJYsWYIQggceeIA5c+aYLNfpdDz//POcOXOGVq1asWzZMnx9fdHr9bzyyiucOXMGo9HI\nxIkT620rSZIk/T5ZFLx69erF9u3bSUpKQghBp06dsLa2vuZ2RqORRYsWsWbNGjw9PYmOjmbEiBEE\nBAQo66xbtw5XV1e2b9/Oli1bePfdd1m2bBnbtm2jqqqKjRs3UlFRwbhx4xg/fjy+vr43fraSJElS\ni2BRs+H+/fupqKiga9eudOvWjfLycoteUI6Li6N9+/b4+flhbW1NZGQkO3bsMFlnx44dTJo0CYAx\nY8YozZMqlYqysjIMBgPl5eXY2Njg5OR0vecnSZIktUAWBa933nnHJHA4OTnxzjvvXHM7rVaLj4+P\n8tnLy4usrCyTdbKysvD29gZAo9Hg7OxMQUEBY8aMwd7enkGDBjF8+HAeffRRXFxcLDopSZIkqWWz\nqNmwZjqoGmq1GoPBYNF217tOzbHi4uLQaDTs27ePgoIC/vjHPxIeHo6/v78lSZYkSZJaMIuCl6Oj\nI7GxsfTu3RuA2NhYHBwcrrmdt7c36enpymetVounp2e9dTIzM/Hy8sJgMFBSUoKrqyubNm1i8ODB\nqNVqWrduTXBwMKdPn75m8PLwcLbklH4XZF7UknlRS+ZFLZkXzZdFwevZZ59l/vz5dO7cGYD4+Hg+\n+uija24XFBREcnIyaWlpeHh4sHnzZpYuXWqyTkREBBs2bKB3795s27aNsLAwAHx8fDh48CATJkyg\nrKyM2NhYZs6cec1jZmcXW3JKLZ6Hh7PMi6tkXtSSeVFL5kWt5hjEVcKStj2gsLCQkydPIoSgb9++\nuLq6WnSAmJgYFi9ejBCC6Oho5syZwwcffEBQUBARERHodDqeffZZzp07h5ubG0uXLsXf35+ysjJe\nfNEYr50AABg9SURBVPFFEhISAHjggQeYNWvWNY8nf4zV5I1ZS+ZFLZkXtWRe1GrRwSspKYmEhARG\njhxJaWkpVVVVuLm5NXX6rpv8MVaTN2YtmRe1ZF7UknlRqzkGL4tGG27YsIEnn3ySf/zjH0B139Vf\n/vKXJk2YJEmSJDXEouD1+eef8/333+PsXB2dO3XqRE5OTpMmTJIkSZIaYlHwsra2xtHR0eQ7jUbT\nJAmSJEmSpGuxKHi5ubkpf4gS4Mcff1ReLJYkSZKk283iiXmfeeYZkpKSGD58OHZ2dnzyySdNnTZJ\nkiRJMsui4NWxY0e+++47Ll++jBCCjh07ymZDSZIk6Y6xqNkwKSkJvV5PQEAAGRkZrFq1isLCwqZO\nmyRJkiSZZVHw+stf/oJarSYlJYXXX3+dlJQUnn/++aZOmyRJkiSZZVHwUqvVWFtbs3v3bqZNm8ai\nRYvIyMho6rRJkiRJklkWBa/Kykq0Wi07d+5U5h60cGIOSZIkSbrlLApejzzyCJGRkTg6OhIUFERK\nSorywrIkSZIk3W4Wz21Yl8FgwGAwYGNj0xRpuilyrrJqct62WjIvasm8qCXzolaLm9vw9OnTZr/X\naDTY2Nig0+mUWd8lSZIk6XZp9D2vFStWUF5ezvjx4+nduzfu7u5UVlaSlJTEnj172L17Ny+88AIB\nAQG3K72SJEmS1Hjw+vDDD4mLi+Obb77h3//+N5mZmdjb29O1a1dGjhzJV199hZOT0+1KqyRJkiQB\nFsyw0atXL3r16nU70iJJkiRJFrFotKEkSZIk3U1k8JIkSZKaHRm8JEmSpGZHBi9JkiSp2bEoeOXm\n5vLXv/6Vhx56CIDz58/zf//3f02aMEmSJElqiEXB65VXXqFfv34UFRUB0KlTJ77++muLDhATE8PY\nsWMZM2YMK1eurLdcp9Px9NNPM3r0aKZOnUp6erqy7Pz58zz44IOMHz+eCRMmoNPpLDqmJEmS1LJZ\nFLy0Wi3Tpk1T/gCljY0NavW1NzUajSxatIhVq1axadMmNm/eXG9GjnXr1uHq6sr27dt55JFHePfd\nd4HqKaiee+453njjDTZt2sQXX3yBtbX19Z6fJEmS1AJZFLysrExfBysqKrJoVvm4uDjat2+Pn58f\n1tbWREZGsmPHDpN1duzYwaRJkwAYM2YMBw8eBGDv3r0EBgbStWtXAFxdXVGpVJYkV5IkSWrhLApe\no0eP5rXXXqO0tJT169cze/ZsHnjggWtup9Vq8fHxUT57eXmRlZVlsk5WVhbe3t5A9ZyJzs7OFBT8\nf3v3HhxVef9x/L1sAlJMgpiQRaS0JraQGqAzKsERIYBZIITsBiIMUsKlpdoBKqFYwck4crXGyUhk\nOhIBKzRMa4HIJRBSgxI6XGy1hZkCRUEn3JJwS5NgypLN8/sjP3YbgrBWNvEkn9df7Nlnz373yzN8\nOGfPPqeKL774AoAZM2aQlpbG6tWrA/1MIiLSxt12hQ2An/70p2zdupXq6mr27NnDT37yE1JTU2/7\nukCOzm4cY4zBZrPh9Xr55JNP2LRpE506dWLq1Kk89NBDvvuJiYhI+xVQeAGMHTuWsWPHfq2dOxyO\nJhdgVFRU0L1792ZjysvLiY6Oxuv1UltbS0REBA6Hg0ceeYSIiAgAnnjiCY4cOXLb8LLi0v7Bol74\nqRd+6oWfemFdAYXXxYsX+f3vf09ZWRn19fW+7StWrLjl6+Lj4ykrK+PMmTNERUVRWFhITk5OkzGJ\niYkUFBTQv39/ioqKfOH0+OOPs3r1aq5evYrdbuevf/0rU6dOvW2tuj9PI92ryE+98FMv/NQLPyuG\neEDh9Ytf/IK4uDgGDRrku+IwEHa7naysLKZPn44xhvHjxxMTE0Nubi7x8fEkJiaSnp7O/PnzSUpK\nomvXrr5wCw8PZ9q0aYwbNw6bzcbQoUMZMmTI//YpRUSkTQnoTspjx45l69atLVHPN6b/STXS/yr9\n1As/9cJPvfCz4pFXQFcb9u/fn3/961/BrkVERCQgAZ02nDhxIpMnT8bhcNCpUyff9o0bNwatMBER\nka8SUHjNnz+fZ555hri4uK/1nZeIiEgwBBRenTp1YsaMGcGuRUREJCABfec1ePBgSktLg12LiIhI\nQAI68nr33XfJy8ujS5cudOzY0bcKxv79+4Ndn4iISDMBhdemTZuCXYeIiEjAAgqvnj17BrsOERGR\ngN0yvObPn092drZvlYsb6VJ5ERFpDbcMr4yMDAB+/etft0gxIiIigbhleG3YsIFly5bx6KOPtlQ9\nIiIit3XLS+WPHj3aUnWIiIgELKDfeYmIiHyb3PK04fHjxxk0aFCz7fqdl4iItKZbhtf3vvc98vLy\nWqoWERGRgNwyvDp27KjfeImIyLfOLb/zCg0Nbak6REREAnbL8Hr33Xdbqg4REZGA6WpDERGxHIWX\niIhYjsJLREQsJ+jhVVpaysiRI3E6nTe97N7j8TB37lySkpKYMGECZ8+ebfL82bNn+fGPf8zbb78d\n7FJFRMQighpeDQ0NLF68mDVr1rB9+3YKCws5ceJEkzEbN24kIiKC4uJiMjIyyM7ObvL8K6+8wpAh\nQ4JZpoiIWExQw+vw4cP07t2bnj17EhoaSnJyMiUlJU3GlJSU4Ha7AXA6nU1W7Xj//ffp1asXsbGx\nwSxTREQsJqjhVVFRQY8ePXyPo6OjqaysbDKmsrISh8MBgN1uJzw8nKqqKurq6li9ejWzZs0KZoki\nImJBAd1J+X9ljPnaY66vm5ibm8vUqVPp3LlzwPsCiIoK+/qFtlHqhZ964ade+KkX1hXU8HI4HE0u\nwKioqKB79+7NxpSXlxMdHY3X66W2tpaIiAgOHz5McXEx2dnZVFdX06FDBzp16sTTTz99y/c8f74m\nKJ/FaqKiwtSL/6de+KkXfuqFnxVDPKjhFR8fT1lZGWfOnCEqKorCwkJycnKajElMTKSgoID+/ftT\nVFREQkICAPn5+b4xK1eupEuXLrcNLhERaR+CGl52u52srCymT5+OMYbx48cTExNDbm4u8fHxJCYm\nkp6ezvz580lKSqJr167Nwk1ERORGNhPol0kWodMAjXRKxE+98FMv/NQLPyueNtQKGyIiYjkKLxER\nsRyFl4iIWI7CS0RELEfhJSIilqPwEhERy1F4iYiI5Si8RETEchReIiJiOQovERGxHIWXiIhYjsJL\nREQsR+ElIiKWo/ASERHLUXiJiIjlKLxERMRyFF4iImI5Ci8REbEchZeIiFiOwktERCxH4SUiIpYT\n9PAqLS1l5MiROJ1O8vLymj3v8XiYO3cuSUlJTJgwgbNnzwKwb98+0tLSGDt2LOPGjePAgQPBLlVE\nRCwiqOHV0NDA4sWLWbNmDdu3b6ewsJATJ040GbNx40YiIiIoLi4mIyOD7OxsALp168aqVavYunUr\nr7zyCs8//3wwSxUREQsJangdPnyY3r1707NnT0JDQ0lOTqakpKTJmJKSEtxuNwBOp5P9+/cD0KdP\nH6KiogB48MEH8Xg8XLt2LZjlioiIRQQ1vCoqKujRo4fvcXR0NJWVlU3GVFZW4nA4ALDb7YSHh1NV\nVdVkTFFREXFxcYSGhgazXBERsYiQYO7cGPO1xxhjsNlsvseffvopOTk5rF27NqD3jIoK+3pFtmHq\nhZ964ade+KkX1hXU8HI4HL4LMKDxSKx79+7NxpSXlxMdHY3X66W2tpaIiAgAysvLmTVrFq+++ir3\n339/QO95/nzNnfsAFhYVFaZe/D/1wk+98FMv/KwY4kE9bRgfH09ZWRlnzpzB4/FQWFjI8OHDm4xJ\nTEykoKAAaDw9mJCQAEB1dTU///nP+dWvfsWAAQOCWaaIiFhMUMPLbreTlZXF9OnTGTNmDMnJycTE\nxJCbm8sHH3wAQHp6OpcvXyYpKYl33nmHefPmAZCfn09ZWRm//e1vcblcuN1uLl26FMxyRUTEImwm\nkC+mLESnARrplIifeuGnXvipF346bSgiItICFF4iImI5Ci8REbEchZeIiFiOwktERCxH4SUiIpaj\n8BIREctReImIiOUovERExHIUXiIiYjkKLxERsRyFl4iIWI7CS0RELEfhJSIilqPwEhERy1F4iYiI\n5Si8RETEchReIiJiOQovERGxHIWXiIhYTtDDq7S0lJEjR+J0OsnLy2v2vMfjYe7cuSQlJTFhwgTO\nnj3re27VqlUkJSUxatQo/vKXvwS7VBERsYighldDQwOLFy9mzZo1bN++ncLCQk6cONFkzMaNG4mI\niKC4uJiMjAyys7MB+Oyzz9i5cyc7duzgrbfe4uWXX8YYE8xyRUTEIoIaXocPH6Z379707NmT0NBQ\nkpOTKSkpaTKmpKQEt9sNgNPp5MCBAwDs3r2b0aNHExISwv3330/v3r05fPhwMMsVERGLCGp4VVRU\n0KNHD9/j6OhoKisrm4yprKzE4XAAYLfbCQsLo6qq6qavraioCGa5IiJiEUENr0BO891sjM1m+8rt\nIiIiIcHcucPhaHIBRkVFBd27d282pry8nOjoaLxeLzU1NUREROBwODh37pxvXHl5ebPX3kxUVNid\n+wAWp174qRd+6oWfemFdQT3yio+Pp6ysjDNnzuDxeCgsLGT48OFNxiQmJlJQUABAUVERCQkJAAwb\nNowdO3bg8Xg4deoUZWVl9OvXL5jlioiIRQT1yMtut5OVlcX06dMxxjB+/HhiYmLIzc0lPj6exMRE\n0tPTmT9/PklJSXTt2pWcnBwAYmNjGTVqFMnJyYSEhPDSSy/ptKGIiABgM7r+XERELEYrbIiIiOUo\nvERExHIUXiIiYjltJrxut4ZiW1ZeXs6UKVMYPXo0KSkprFu3DoB///vfTJ8+HafTyYwZM6ipqWnl\nSltOQ0MDbrebZ555BoDTp0/z1FNP4XQ6yczMpL6+vpUrbBk1NTXMmTPHd/HToUOH2u28+N3vfseY\nMWNISUlh3rx5eDyedjMvFi5cyGOPPUZKSopv263mwZIlS0hKSiI1NZWjR4+2Rsm31SbCK5A1FNsy\nu93OggUL2LFjB3/4wx/Iz8/nxIkT5OXlMWjQIHbt2sXAgQNZtWpVa5faYtatW0dMTIzv8Wuvvca0\nadPYtWsXYWFhbNy4sRWrazlLly5lyJAh7Ny5ky1btvDAAw+0y3lRUVHB+vXr2bx5M9u2bcPr9VJY\nWNhu5kVaWhpr1qxpsu2r5sGePXsoKyujuLiYRYsW8dJLL7VGybfVJsIrkDUU27KoqCj69u0LQJcu\nXYiJiaGioqLJupFut5v333+/NctsMeXl5ezZs4f09HTftgMHDuB0OoHGXvz5z39urfJaTG1tLX/7\n298YN24cACEhIYSFhbXbedHQ0EBdXR319fX85z//oXv37hw8eLBdzIuHH36Y8PDwJttunAfX/80s\nKSnB5XIB0L9/f2pqarhw4ULLFhyANhFegayh2F6cPn2aY8eO0b9/fy5evEhkZCTQGHCXL19u5epa\nxrJly3j++ed9vwu8fPkyERERdOjQON0dDke7mB+nT5/mnnvuYcGCBbjdbrKysqirq2uX8yI6Oppp\n06YxdOhQnnjiCcLCwoiLiyM8PLzdzYvrLl261GQeXLp0CWi63ix8e9eVbRPhpZ+qNbpy5Qpz5sxh\n4cKFdOnSpV3+qPvDDz8kMjKSvn37+uaFMabZHGkPvamvr+fIkSNMmjSJgoICOnfuTF5eXrv47Deq\nrq6mpKSEDz74gL1791JXV0dpaWmzce2xNzeyyrqyQV1ho6UEsoZiW1dfX8+cOXNITU1lxIgRANx7\n771cuHCByMhIzp8/T7du3Vq5yuD75JNP2L17N3v27OHq1atcuXKFZcuWUVNTQ0NDAx06dAh4nUyr\nczgcOBwO4uPjAUhKSuKtt95ql/Ni37599OrVi65duwIwYsQI/v73v1NdXd3u5sV1XzUPoqOjKS8v\n9437tvalTRx5BbKGYlu3cOFCYmNjycjI8G0bNmwYmzdvBqCgoKBd9CQzM5MPP/yQkpIScnJyGDhw\nIK+99hoDBw6kqKgIaD+9iIyMpEePHnz++edA4/d+sbGx7XJe3HfffRw6dIirV69ijOHAgQM8+OCD\n7Wpe3HhE9VXzYPjw4bz33nsA/OMf/yA8PNx3evHbpM0sD1VaWsrSpUt9ayjOnDmztUtqMR9//DGT\nJ0/mBz/4ATabDZvNxty5c+nXrx/PPfcc586d47777mPFihXNvrRtyz766CPWrl3Lm2++yalTp8jM\nzKS6upq+ffuSnZ1NaGhoa5cYdMeOHePFF1+kvr6eXr16sXz5crxeb7ucFytXrqSwsJCQkBDi4uJY\nsmQJ5eXl7WJezJs3j4MHD1JVVUVkZCSzZ89mxIgR/PKXv7zpPFi0aBF79+6lc+fOLF++nB/96Eet\n/AmaazPhJSIi7UebOG0oIiLti8JLREQsR+ElIiKWo/ASERHLUXiJiIjlKLxERMRyFF5iScOGDeOz\nzz5rkfdauXJlk1tlLFiwgPz8/G+83wULFpCSkkJmZuY33tetHDt2jJ07dwb1PURamsJL5DZWrlzJ\ntWvX7ug+L1y4QHFxMdu2bSMnJ+eO7vtGR44c+Z/Dq6Gh4Q5XI3JnKLykTfn888/52c9+Rnp6Oi6X\ny7f8DUCfPn1YtWoV48eP58knn6S4uNj33K5duxg1ahRpaWmsWrWKPn36UFdXx6JFi7DZbEycOBG3\n201tbS0Ax48fJyMjA6fTyQsvvPCV9bz33nukpKSQmprK7NmzuXTpEleuXCEjI4OrV6/idrt55513\nmrxmy5YtzJo1y/fY6/UyePBg3/qdq1ev5qmnniItLY1nn32WixcvAnDt2jV+85vfkJKSgsvlYvbs\n2VRVVfHGG29w4MAB3G43S5cuBRpXpHG73aSmpjJt2jROnToFNK5K4nK5WLJkCRMnTmTv3r3f5K9D\nJHiMiAUlJiaaTz/9tMm2+vp643a7zcmTJ40xxtTW1hqn0+l7/MMf/tDk5+cbY4z5+OOPzeDBg40x\nxly4cME8+uijpqyszBhjzNtvv2369OljvvzyS9/r6urqfO/zwgsvmEmTJhmPx2M8Ho9JTk42+/bt\na1bj8ePHzeOPP24uXLhgjDHm9ddfN88995wxxpjTp0+bhISEm362uro6k5CQYC5fvmyMMWb37t0m\nIyPDGGPMli1bTFZWlm/shg0bzLx584wxxrzxxhtm9uzZpr6+3hhjfK/fvHmzmTNnju81Fy9eNAkJ\nCebEiRPGGGP+9Kc/mfT0dGOMMQcPHjRxcXHm0KFDN61N5NtCR17SZnzxxRecPHmSzMxMXC4XTz/9\nNNeuXWtyV+3Ro0cDMGDAAM6fP4/H4+HQoUM89NBD9OrVC4Dx48c327e5YRW1ESNGEBoaSmhoKHFx\ncZSVlTV7zcGDBxk6dCj33nsvABMnTmTfvn23/Rx33XUXw4cPZ/v27UDjoqnXbyi5e/du9u/fj8vl\nwuVysWHDBs6dOwc03g5mypQp2O12AN8K6jc6dOgQffv25YEHHgBg3LhxHD16lC+//BKA3r17069f\nv9vWKdKa2sQtUUSgMWC6detGQUHBTZ+32Wx06tQJwHcDQq/X2yyYbnx8Mx07dvT92W63N7mg47/3\nc+N9kK6/7+24XC6WL1/OmDFj+Oijj8jOzvbt89lnnyUtLe2m7xeIm9X134+/853vBLQfkdakIy9p\nM77//e9z1113sWXLFt+2kydPcuXKFaD5P+7XHw8YMIB//vOfvu99/vt7MoC7776bmpqar13PoEGD\n2LNnj+87qT/+8Y889thjzd7/Zh5++GFqa2vJycnhySef9IXusGHD2LBhA9XV1QB4PB6OHTsGQGJi\nIuvWrfNdXHL9Dsl3332377u665/36NGjvlulbN68mbi4OIWWWIqOvMSSbDYbU6dOJSQkxHcksW3b\nNt58802WLl3K2rVr8Xq9REZG8vrrr/tec+M+oPGmfC+//DIzZ87knnvuYejQoYSEhNC5c2cApk2b\nxpQpU+jcuTPr168PuMbY2FgyMzOZOnUqHTp0oFevXixatKjZ+38Vl8tFbm4uGzZs8G1LTU2lqqqK\nyZMnY7PZaGhoYNKkSfTp04eZM2eSk5ODy+WiY8eOfPe732XFihUMGjSINWvW4HK5eOSRR3jxxRd5\n9dVXmTdvHl6vl27duvmO7ESsQrdEEQGuXLlCly5dgMYjkU2bNt2R33KJSHDoyEsEWL9+PUVFRXi9\nXrp27crixYtbuyQRuQUdeYmIiOXogg0REbEchZeIiFiOwktERCxH4SUiIpaj8BIREctReImIiOX8\nH4gzFtcS9o9MAAAAAElFTkSuQmCC\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f47b20dd690\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(graph_means)\n", - "plt.ylabel('Time (seconds)')\n", - "plt.xlabel('Length of vector')\n", - "_ = plt.title('Time to sum the elements of 1000 vectors (vectorized TF operation)')\n", - "_ = plt.ylim(ymin=0)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4KZg2WXjbhg5" - }, - "source": [ - "## AutoGraph" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "UQJBQWbCbinm" - }, - "outputs": [], - "source": [ - "# Sum written using for loop and converted with AutoGraph\n", - "def sum_all(elements):\n", - " sum_ = 0.0\n", - " length = len(elements)\n", - " for i in tf.range(length): \n", - " sum_ += elements[i][0]\n", - " return sum_\n", - "\n", - "def run_trial(num):\n", - " elements = get_elements(num)\n", - " return sum_all(elements)\n", - " \n", - "ag_means = []\n", - "ag_run_trial = ag.to_graph(run_trial)\n", - "\n", - "for num in range(max_elements):\n", - " with tf.Graph().as_default():\n", - " durations = []\n", - " foo = ag_run_trial(num)\n", - " with tf.Session() as sess:\n", - " for _ in range(burn_ins):\n", - " for _ in range(batches):\n", - " sess.run(foo)\n", - " \n", - " for _ in range(trials):\n", - " start = time.time()\n", - " for _ in range(batches):\n", - " sess.run(foo)\n", - " \n", - " duration = time.time() - start\n", - " durations.append(duration)\n", - " ag_means.append(np.mean(durations))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 301 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 310, - "status": "ok", - "timestamp": 1532448438694, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "DLDOmrRW99v5", - "outputId": "ae0e0573-39db-4004-a064-efc618dbf867" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEcCAYAAADUX4MJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XdYVFf++PH3DE1AinTEjgULioggNiTYjcZCoiZqjEmM\n6RuzcVc32exPE7O72dTNmmhi1u+abLJqLFFsib1E7GLDjkgbQUSKyDAz5/eHySARcYwMQ/m8nsfn\nce4999zPPXOHzz23nKtRSimEEEKIu9DaOgAhhBC1gyQMIYQQFpGEIYQQwiKSMIQQQlhEEoYQQgiL\nSMIQQghhkTqZMObPn88bb7xh6zBqlQceeICffvrJ6uuZOXMmH330kdXXU1NcuHCBUaNG0a1bN776\n6itbh1OnnT17ljFjxtg6jGpV2e9Jr9czZMgQcnNzq2x9tTJhdO3alfDwcMLDw2nfvj1dunQxT1uz\nZg3PPPMMc+bMsXoc6enphISEYDKZrL6uqlTf/mjfjTW/xy+++IKoqCgOHDjAhAkTbpu/bt06xo0b\nR1hYGJMmTbpt/smTJxk9ejRhYWGMGTOG5OTkcvPfffddoqKi6NGjB+++++49LWttEydOZNmyZdW2\nvo8//pinnnqqwjgiIyMpLS29p/pCQkK4dOnSPS3z1VdfMWLECMLCwujduzeTJk1i7dq191RHVXF0\ndCQ+Pp7PP/+8yuqslQnj0KFDHDx4kIMHD9K4cWPmz59vnvbggw9WWxxKKTQaDfLsY+1mze8xIyOD\n1q1b33G+p6cnkydPZurUqbfNKy0t5fnnn2fkyJHs27ePkSNH8txzz2EwGAD49ttv2bx5M6tXr+b7\n779n69at/O9//7No2drgXr6P7OxsEhMTiYuLKzc9PT2dAwcOoNFo2Lx58z2tX6PR3FP5OXPmsHjx\nYmbOnMnevXvZsWMHv/vd79ixY8cdl7H2344HH3yQFStW3HOyvJNamTBupZS6rdE/+eQTXnvtNaDs\n6HH58uX069ePqKgovv32W44ePcqIESOIjIy8rTeybNkyhg4dSlRUFE899RQZGRkVrnvixIkARERE\nEB4ezpEjR1BKMW/ePB544AF69erFH//4RwoLCytc/urVq0ybNo3u3bsTFRVV7gj010c3t/YK9u7d\nS0xMDF988QU9e/akT58+/Pjjj2zbto1BgwYRFRXF/PnzK1znkiVLWL16NV988QXh4eE8++yz5nkn\nT55kxIgRdO/enenTp6PX683ztmzZwsiRI+nevTvjx4/n1KlTFdYPcO7cOaZMmUJUVBRDhgxh3bp1\ndyxbWb0PPPAACxcuZMSIEXTt2pXXX3+dK1eu8PTTTxMeHs6UKVMoKCgwlz98+DDjxo2je/fujBw5\nkr1795rnTZw4kY8++ojx48cTHh7Ok08+SV5ennkelP8eU1NTmThxIhEREURHRzN9+vQ7bsOmTZt4\n8MEHiYyMZNKkSZw/fx6Axx9/nMTERGbPnk14eDgXL168bdno6GgGDx6Mr6/vbfP27t2L0Whk0qRJ\nODg4MHHiRJRS7NmzB4CVK1cyZcoU/Pz88PPz44knnmDFihUAJCYmVrrsrdauXXvbqZxFixbx3HPP\nATdPbfztb38jNjaW3r1785e//KXcvvHjjz8ycuRIunXrxsCBA9m5cycffPABBw4cYM6cOYSHh/PW\nW28BcPDgQeLj4+nevTsPP/wwhw4dKvcdffDBB4wfP56wsDDS0tJYvnw5/fv3Jzw8nP79+7NmzZoK\nv4Ndu3bRsWNHHB0dy01fuXIlYWFhjB492tw2t67v1h7QihUrePTRRwGYMGECSilGjBhBeHi4eR9e\nsmQJAwcOJCoqiueee47Lly8DN089fvPNN3zwwQdER0fj6OiIRqMhPDycd955567bOHToUMLDwxkw\nYIA56f+yD8TExDB//nx69OhBXFwcq1evLrcd165d45lnniE8PJyxY8eW+7vh7++Ph4cHR44cqbDd\n7pmq5WJjY9Xu3bvLTfvnP/+pXnvtNaWUUmlpaapdu3bqzTffVCUlJWrXrl0qNDRUPf/88yo3N1dl\nZWWp6OhotW/fPqWUUj/88IMaOHCgOn/+vDIajerTTz9VY8eOrXDdaWlpKiQkRJlMJvO0pUuXqoED\nB6q0tDR1/fp19cILL5hj+bX33ntPvfnmm8poNCqDwaD2799vnhcSEqJSU1PNn//4xz+qDz/8UCml\nVGJiourQoYOaN2+eMhgMasmSJapHjx7q1VdfVdevX1dnzpxRoaGh6tKlSxWu99a6bm3Hhx9+WGVn\nZ6tr166pIUOGqG+//VYppdSxY8dUdHS0SkpKUiaTSa1YsULFxsYqvV5/W93Xr19XMTExasWKFcpk\nMqkTJ06oqKgodfbs2dvWfbd6Y2Nj1dixY9WVK1eUTqdT0dHRatSoUerkyZNKr9erSZMmqU8++UQp\npVRWVpaKjIxU27dvV0optXv3bhUZGalyc3OVUkpNmDBBDRgwQF28eFGVlJSoCRMmqPfee++O3+P0\n6dPVZ599ppRSqqSkRB04cKDCtjx//rwKCwtTu3fvVgaDQX3++edqwIABqrS01LzepUuXVrjsrZYs\nWaImTpxYbtq///1v9fTTT5eb9swzz6h///vfSimlunXrpo4cOWKed/ToURUeHm7RsrcqLi5W4eHh\n6uLFi+ZpY8aMUWvXrlVKKfXWW2+pZ599VuXn56uioiI1bdo09f777yullDpy5Ijq1q2b+Teo0+nU\n+fPnK9z2vLw81b17d/X9998ro9Go1qxZo7p3767y8vLM5WNjY9XZs2eV0WhUBQUFKjw8XKWkpCil\nlMrOzjbvR7/2t7/9Tc2ePfu26QMGDFDffPONOnbsmOrYsaO6cuWKed6v41u+fLl69NFHzZ/btWtX\n7je4e/duFRUVZd7/5syZox577DGllFLffPONeuCBByqM7Va/3sbS0lK1detW82913759qkuXLurE\niRNKqbLf+l//+lel1+vV3r17VVhYmLpw4YJS6ubvKTIyUh09elQZjUb16quvqunTp5db57Rp09Ti\nxYvvGpslan0PwxIajYbnn38eR0dHevbsibOzM8OGDaNRo0b4+/sTERHBiRMnAPjf//7H1KlTadmy\nJVqtlqlTp5KcnExmZuYd61e39HDWrFnD5MmTCQoKwtnZmenTp7N27doKz4/b29uTnZ1NWloadnZ2\ndOvWrcI6K+Lg4MC0adOws7Nj6NChXL16lccffxxnZ2dat25N69atK+0FVGTSpEn4+Pjg7u5ObGws\nJ0+eBGDp0qWMGzeO0NBQNBoNI0eOxNHRscKjli1bttCkSRNGjhyJRqOhffv2DBw4kPXr199W1pJ6\nJ0yYgJeXF35+fkRERNClSxdCQkJwcHBgwIAB5hi///57+vXrR58+fYCbR+6dOnVi27Zt5rpGjx5N\ns2bNcHR0ZMiQIeZlf3Frm9vb25Oeno5Op8PR0ZHw8PAK22zdunX069eP6Oho7OzsePLJJ7lx40a5\nI+ff6vr167i5uZWb1rBhQ3OP9dfz3dzcuH79ukXL3qpBgwbExcWZj95TUlK4cOGC+fTOsmXLmDlz\nJm5ubri4uDB16lRz2WXLlhEfH090dDQAfn5+tGzZssLt2bp1Ky1atGD48OFotVqGDRtGq1at2LJl\ni7nMqFGjCA4ORqvVYmdnh52dHadPn6akpAQfHx+Cg4MrrLugoABXV9dy0/bv309GRgZDhgyhY8eO\nNGvW7Laj83uxZs0a4uPjzfvf9OnTOXz4MBkZGVy9evW2XmJMTAzdu3enc+fO5f5+3LqN9vb2xMTE\n0KRJE+BmL7dXr17s37/fXF6j0fC73/0OBwcHunfvTkxMTLle+8CBA+nUqRNarZbhw4fftl+7urqS\nn5//m7f7VvZVUkst4O3tbf5/gwYN8PHxMX92cnIy/9AyMjJ4++23+dvf/gaUnd/W6XQEBgbedT2X\nL1+mcePG5s9BQUEYDAZycnLw8/MrV/app57in//8J1OmTEGj0fDwww9XeC67Ip6enuZzrA0aNKhw\nG3/ZJkvduryzszPZ2dnAzTZZtWqV+S4fpRQGg8HcHb9VRkYGhw8fJjIy0lzWaDQycuTICsverd5b\nY3Jycrrt863f27p168x/fH6p65c/ZEC579zZ2bnS9pkxYwYffvgh8fHx5usMFd2B8+vvW6PREBgY\niE6nu2PdlnJxcbntD3xhYSENGzascH5hYSEuLi4WLftrw4YN4+9//zvPPfcca9asoX///jg6OpKb\nm0txcXG5bTeZTObkmpWVRUxMjEXb8+u2AmjcuHG5tgoICDD/39nZmQ8++ICFCxcya9YsunXrxowZ\nM2jVqtVtdbu7u1NUVFRu2qpVq+jduzceHh7mbVy5ciWPP/64RfFWFH/Hjh3Nn11cXPD09ESn0+Hp\n6Xnb72Hbtm0YjUY6depU7mDk1m38pdy8efNISUnBZDJx48YN2rVrV27bnJyczJ8bN25cbl1326+L\niopwd3f/Tdv8a/UmYVgqICCAZ5991qKL5xVdFPPz8yt3zSM9PR17e/tyX+ovXFxc+MMf/sAf/vAH\nzp07x8SJE+ncuTM9evTA2dmZ4uJic9ns7OzbdrTqEhAQwLRp03jmmWfuWjYwMJCoqCgWLlxYpfVa\nst6RI0cye/bse162ou/R29vbfG3rwIEDPPHEE0RGRtK0adNy5fz8/Dhz5ky5aZmZmVXyXbVp04ZF\nixaVm3b69GnzNZfWrVuTnJxMaGgocPMaVJs2bSpdtqI7tQB69+7NzJkzSU5OJiEhgVmzZgHQqFEj\nnJ2dWbNmzW0HPHDzO7zTnUS/blc/Pz82btxYblpGRgZ9+/a94zK9evWiV69e6PV6PvjgA9544w2+\n/vrr29bVrl07Vq1aZf5cUlLCunXrMJlM9O7dG7h5I0B+fj6nTp2iXbt2uLi4cOPGDfMyvxwg3cmv\nf9vXr18nLy8Pf39/PD09eeuttzh+/Hi5pAK3ny24dRv1ej0vv/wy7777LnFxcWi1Wp5//vlyy+Tn\n53Pjxg3zgWFmZiZt27atNNZbnT9/nieffNLi8pWpF6ek7nZ651bjx49n/vz5nD17FrjZ1a3odAqA\nl5cXWq2W1NRU87Rhw4axaNEi0tLSKCoq4oMPPmDYsGFotbc39datW83Luri4mLvgcPOi95o1azCZ\nTGzfvp19+/ZZvA134+Pjc0+3Cz7yyCN8++23JCUlATd/KNu2bavwCL1fv35cuHCBVatWYTAYKC0t\n5ejRo+YLwb+13rsZMWIEmzdvZufOnZhMJkpKSti7d69FR/oVfY/r1683L+vu7o5Wq63wOxwyZAhb\nt25lz549GAwGFi5ciJOTE2FhYRbFbTKZ0Ov1GAyGcv8HiIyMRKvVsnjxYvR6vbknFhUVBcDIkSNZ\ntGgROp0OnU7HokWLGD16dKXL9ujRo8I47OzsGDRoEH//+9/Jz8+nV69eAOae79y5c8338+t0Onbu\n3AlAfHw8y5cvZ8+ePSil0Ol05u/61/tZTEwMFy9eJCEhAaPRyNq1azl//jyxsbEVxnTlyhU2b95M\ncXEx9vb25t9IRXr16sXx48fNF+N/+OEH7OzsWLduHatWrWLVqlWsXbuWbt26sXLlSuDmb2zjxo3c\nuHGDixcv8t1335Wr89fxP/jggyxfvpzk5GT0ej3vv/8+Xbp0oXHjxrRs2ZKxY8cyffp0du/eTUlJ\nCSaTiYMHD1Z6t1VpaSmlpaU0atQIrVbLtm3b2LVrV7kySik+/vhjSktL2b9/P1u3bmXIkCF3rPNW\nOp2Oa9eu0aVLF4vK302tTxiW3Pr26zKVfe7fvz9PP/00r7zyChEREYwYMeKOt8U1aNCAadOmMX78\neCIjI0lKSiI+Pp6HHnqICRMmMGDAAJydnXn99dcrXD4lJYXJkyfTtWtXxo8fz2OPPUb37t0B+NOf\n/sTmzZvp3r07CQkJ9O/f/7628Vbx8fGcPXuWyMhIXnjhhbuW79SpE3PmzGH27NlERkYyaNCg2+44\n+YWrqytffvkla9eupU+fPvTp04f33nuv3F01ltZ7L9sUEBDAvHnzmD9/PtHR0cTGxvLll1+aDxYq\nW7ai7/Ho0aM8/PDDhIeH8/zzz/OnP/2JoKCg25Zt2bIl7777LnPmzCE6OpqtW7fy2WefYW9vf9f1\nws3TJp07d2b27NkcOHCALl26mB86dXBwYN68eaxYsYLIyEiWL1/OvHnzzHWPGzeO2NhYRowYwYgR\nI4iNjeWRRx6xaNmKDBs2jJ9++okhQ4aUS46///3vad68OY888ggRERFMmTKFlJQUADp37szcuXOZ\nO3cu3bp1Y9KkSebz9ZMmTWL9+vVERUXx9ttv4+npyWeffcbChQvp0aMHCxcuZP78+eZTRr9uK5PJ\nxL///W/69u1Ljx492LdvH2+++WaFsXt7e9OjRw82bdoE3Lw7asyYMfj7++Pt7W3+99hjj7F69WpM\nJhOTJ0/GwcGBXr16MXPmTIYPH16uzhdffJEZM2YQGRnJ+vXriY6O5uWXX+bFF1+kT58+pKWl8f77\n75vL//nPf2bixIm88847REVFERMTw8cff8yHH35oPhX36210dXXlT3/6Ey+//DKRkZGsXbv2tluD\nfX198fDwoE+fPsyYMYPZs2fTokWLO36Pt1q9ejWjRo3CwcHBovJ3o1H3cvh9j2bNmsXWrVvx9va+\n48WmxMRE3nnnHQwGA40aNWLx4sXWCkcIUYedO3eOP/7xjyxdutTWoVSZvXv3MmPGDLZu3XrPy+r1\nekaOHMlXX32Fl5dXlcRj1YSxf/9+XF1dmTFjRoUJo6CggHHjxvHll1/i7+9Pbm5ulW2YEELUdveT\nMKzBqqekIiIiKr06v3r1agYOHIi/vz+AJAshhKjBbHoNIyUlhWvXrjFx4kTGjBljvhglhBDi5s0L\nNaV3ATa+rdZoNHLixAn+7//+j+vXrzNu3Di6du1K8+bNbRmWEEKICtg0Yfj7+9OoUSOcnJxwcnIi\nIiKC5OTkuyaMXx6mE0IIUX2snjAqu6YeFxfHW2+9hdFoRK/Xk5SUxBNPPHHXOjUaDdnZBXctVx/4\n+rpJW/xM2qKMtEUZaYsyvr5udy9UCasmjFdffZXExETy8vLo168fL774IqWlpWg0GsaOHUtwcDC9\ne/dmxIgRaLVaHnnkkUqHghZCCGE7Vr2t1prkiOEmOXoqI21RRtqijLRFmfvtYdT6J72FEEJUD0kY\nQgghLCIJQwghhEUkYQghhLCIJAwhhBAWkYQhhBB1VFXfBCsJQwgh6qD07EJe+HAHe05kVVmdkjCE\nEKIOWrnjAsUlBlwbVM3Lk0AShhBC1GqLN5zi201nyp1+StUVcOB0Nq0au9OpZdW9NsKmgw8KIYT4\n7c6mX2PLoXQA/Bs5ExveBIBVOy8AMLJ3yyodqFUShhBC1FIb9qYC4Oig5ZtNZ2gR6I5Wo+HQmRyC\ng9zpWIW9C5BTUkIIUStdvnqdg6eyaR7gxgujQjEaFZ+uPMaSLWcBGNmnVZW/BkJ6GEIIUcMopZi3\n8hhXC0ro0MKLTi29aNXYHXu7smP8jfsuoYDBkc3o1MqbB3u2YPXuFHKu3aBNEw86NG9U5XFJwhBC\niBrmXEY+B05lA3A+I581u1Nwc3HgyWHt6RzsQ2FxKTuTMvF2dyIixBeAh3q35Gz6NU5evGqV3gVI\nwhBCiBpn19FMAJ4b2Qk7Ow3HLuSy40gmHy5NYlh0c+y0GvQGEwO6N8NOe7PXodVqeGlMZzKuFNEy\n0N0qcUnCEEIIG7mYVcD5jGv06xpk7hHoS43sPamjkZsT4W190Wo1dG3jS9/Ojfl05TESfroIgLOT\nPX06B5arz8nRzmrJAuSitxBC2IS+1Mi/Vhxl8cbTJJ7UmacfOpNDcYmRnp0C0GrLTis1D3Djz5O7\nE9725imouG5BODtV7zG/9DCEEMIGNuxNJefaDQCWbD5Ll2AfnJ3szaejenYKuG0Zlwb2PD+qE5cu\nF9LEt2G1xgvSwxBCiGqXm3+DhD0XcXdxYGD3puQV6lm9K4WrBSUcT8klOMidQG/XCpfVaDQ083cr\n1/uoLlZNGLNmzaJnz54MHz680nJJSUl06NCBjRs3WjMcIYSoEZZtO4e+1MSYmGBG922Fj0cDfth/\nieXbzqEU9AoNvHslNmDVhDF69GgWLlxYaRmTycR7771Hnz59rBmKEELUCGfTrrHnuI7mAW706hyI\no4Md4/u3wWhS7DqWhb2dlsgQP1uHWSGrJoyIiAjc3Su/Yr948WIGDRqEl1fVPsIuhBA1jUkpvtl0\nGoBH+7dB+/OdUWGtfegc7A1AeFsfXKpwhNmqZNNrGDqdjh9//JHx48fbMgwhhKgWe45ncSGzgMj2\nfrRp4mmertFomDCgLV2CvRkW3cJ2Ad6FTe+Smjt3Lq+99pr5/uOqfjuUEELUFCWlRr7bdh4Hey0P\n92t923wfT2defriLDSKznE0TxrFjx3jllVdQSnH16lW2b9+Ovb09cXFxd13W19etGiKsHaQtykhb\nlJG2KFMT2uLbH05xtaCEh+PaENLa19bh/CZWTxiV9Ro2bdpk/v/MmTOJjY21KFkAZGcX3HdsdYGv\nr5u0xc+kLcpIW5SpCW1xtaCEpZtO4+7iQL/OgTaL534Tp1UTxquvvkpiYiJ5eXn069ePF198kdLS\nUjQaDWPHjrXmqoUQosZYseM8+lIT4+PaVPvT2VXJqpG/9957Fpd95513rBiJEEJUv5JSI9sOpbMr\nKZMmvq706dzY1iHdl9qb6oQQwoYMRhMnUq4S0swTRwe7cvOKSwxsOZTOhr2pFFwvxdFBy4SB7Wzy\ndHZVkoQhhBC/wfJt51m/N5UALxeeHNae4CAPlFLsP5XNf388zbVCPc5OdjzYszkDIpri5uJo65Dv\nmyQMIYS4R5lXivhh/yWcnezR5V5n7lcH6N+tKZm5RRw7n4u9nZbhPVswKLJpjX0I77eQhCGEEPfo\n201nMZoUU4aG4ObiyJcJJ/lh/yUAOrb0YsLAtvg3crFxlFVPEoYQQtyDI2dzOHr+Cu2bNyK8rS8a\njYb/NyWSH/ZfIsDLhW7tfK3yetSaQBKGEEJYyGA08e2mM2g1Gsb3b2NODE6OdjzYs4Vtg6sG8j4M\nIYSwgMFoYvm28+iuFhPbNcgmLzCyNelhCCFEJZRSHDqTw9Kt59DlXsfD1ZGH+rS0dVg2IQlDCCF+\n5WpBCeczrnE+I58TF69yMasArUZDv65BPNS7JQ2d686dT/dCEoYQQvys1GDi281n2HIw3TxNA3Rt\n40N8v+A7vja1vpCEIYSol86k5eHsaE9jX1e0Gg05ecXMW3mMlKwCGvu4Et3Rn1aB7rQIdK/V4z9V\nJWkFIUS9s+NIBv9elwxAQ2cH2jTx4PSlPIpuGOjVKYAJg9rh9KvhPoQkDCFEPXMq9Sr/2XAK1wb2\ndGntw6nUqxw6k4O9nZbJQ0Lo0zmwzj5Hcb8kYQgh6g3d1et8svwoAM+PCiWkeSOUUuRcu4GTgx3u\nrrV/vCdrkoQhhKgXim6U8tHSJIpuGJg8JISQ5o2Am+/T9vV0tnF0tYM8uCeEqPOMJhOfrTxGVu51\nBkc2o2+X2v1eCluRhCGEqPOWbjnH8ZSrdA72Jr5fsK3DqbUkYQgh6rQf915k475LBHq78MyIjrX+\nJUa2JAlDCFGrXS0oYfPBNIpulN4270xaHv9aloRrA3teiu8sz1PcJ6u23qxZs9i6dSve3t6sXr36\ntvmrV6/m888/R6PR4OLiwl/+8hfatWtnzZCEEHVIid7I+0sOk55dxIrt5xkW3YK4bkHorhazds9F\nEk/o0Gg0TBvZqU6+n6K6aZRSylqV79+/H1dXV2bMmFFhwjh8+DDBwcG4ubmxfft2PvnkE5YsWWJR\n3dnZBVUdbq3k6+smbfEzaYsy9aEtlFIsWH2CxBM62jdvxMWsAq6XGGjo7EBh8c3eRhNfV6aM6EQL\n3/o9pMcvfH3d7mt5q/YwIiIiSE9Pv+P8sLCwcv/X6XTWDEcIUYf8eCCNxBM6goPceeWRLpSUGkn4\n6SJbDqUTHOTOsOgWdAn2xs/Pvc4nz+pSY07oLV26lL59+9o6DCFELXD6Uh5LNp/F3cWB50aGYm+n\nxd5OyyOxrXkktrWtw6uzakTC2LNnD8uXL+e///2vxcvcb9eqLpG2KCNtUaautsW5tDzmrTyGAv44\nOZK2rXzuukxdbYvqZvOEkZyczJ///Ge++OILPDw8LF5Oupg31Ydz1ZaStihTV9vibPo1PlhyhBsl\nBh4fEkKAu9Ndt7OutsVvUaOvYcDNC1N3kpGRwUsvvcTf//53mjVrZu1QhBC1WPLFq3y0LIlSg4mn\nhncgumOArUOqd6yaMF599VUSExPJy8ujX79+vPjii5SWlqLRaBg7dizz5s3j2rVr/L//9/9QSmFv\nb8+yZcusGZIQopZJyy7kx/1p7D6WiVLw7MhOdGvna+uw6iWr3lZrTdLFvEm622WkLcrU9rYwmkwk\nnb3CjwfSOHnxKgA+Hg2YNLgdnVp631Ndtb0tqlKNPyUlhBCWulpQwq6jmWw9nE5ufgkA7Zs3on+3\nJnRp7SPDetiYJAwhhM2YTIp9yZc5kZLL6Ut56K4WA+DkYEds1yBiuwbRxK+hjaMUv5CEIYSwCX2p\nkc9Xn+DA6WwAGjjaEdrKm87B3vTsFCDjPtVA8o0IIapdwXU9//zuKGfTrxHSzJNHHmhNMz83OeVU\nw0nCEEJUq8t5xXyw5Ai63Ov06ODPE0Pb42AvA2fXBpIwhBDVJj2niH98e4hrhXqG9mjO6JhWaDXS\nq6gtJGEIIapFqq6Af3x7mMLiUsY90JqBkfKwbm0jCUMIYTUmk6KguJSLWQUs+P44xSUGHh/cjpiw\nIFuHJn4DSRhCiCp3IiWX/2w4RU7eDUw/Pxus1WhkSI9aThKGEKJKHTiVzfzvjwHQqrE7Hg0d8XR1\nIrydL+2bN7JxdOJ+SMIQQlRqx5EMDp3JoW9YYzoHe1d6kXrHkQwWrU/G0d6OF8eE0qGFVzVGKqxN\nEoYQ4o5+EQoNAAAgAElEQVRKDSaWbj1HYXEph8/mEOjtQv9uTWjo4oi+1Ii+1Ej+9VKuFtzgyrUb\nHE+5imsDe155JIxWjd1tHb6oYpIwhBB3dOhMNoXFpUS298PeTkviCR2LN56+Y3n/Rs68MDqUIF8Z\nzqMukoQhhLijHUmZAIzo1ZLGPq6M7tuKg6ez0Wg0ONprcXDQ4ubiiJebE54NnWQ4jzpOvl0hRIVy\n8oo5cSGX1kEeNPZxBcDLvQH9I5raODJhK/I8vhCiQjuPZqKAPl0CbR2KqCEkYQghbmMyKXYezaSB\nox3dQ/xsHY6oISRhCCFuczwll9z8EiLb+9PAUc5ci5skYQghbrP9SAYAfbs0tnEkoiaxasKYNWsW\nPXv2ZPjw4Xcs89ZbbzFw4EAeeughTp48ac1whBB3kXmliM9WHePgqWyCfF1pGXh/74AWdYtV+5qj\nR49m4sSJzJgxo8L527ZtIzU1lY0bN3LkyBHefPNNlixZYs2QhKi3SvRGPlt1DEcHO8Ja+xAa7I1r\nA3uuXLtBSlYBh87ksOdEFkpBc383Jg8JQSNDj4tbWDVhREREkJ6efsf5mzZtYuTIkQB06dKFgoIC\ncnJy8PHxsWZYQtRLq3encOTcFQD2JV9Gq9Hg7GRH0Q2DuUwTX1ce6t2K8LY+kizEbWx6Nevy5csE\nBJSNXOnv749Op5OEIUQVy7xSxIa9qXi7N+D50Z04fiGXw2dzKLheSocWXrQIcKNloDttm3nKC43E\nHdk0Yaifhz2+laVHNb6+cm71F9IWZaQtyvzSFkopPvouCaNJMW1MZ7p3CqR7aP16H4XsF1XDpgnD\n39+frKws8+esrCz8/Cy75zs7u8BaYdUqvr5u0hY/k7Yoc2tb7D2p48iZHDoHe9PKz7XetZHsF2Xu\nN3Fa/bbainoRv4iLi2PlypUAHD58GHd3dzkdJUQVKiwu5dtNZ7C30/Jo/zZyXULcF6v2MF599VUS\nExPJy8ujX79+vPjii5SWlqLRaBg7diwxMTFs27aNAQMG4OzszDvvvGPNcISos0r0Rq6XGCg1mjAY\nTBy9mMeWfakcu3AFg1ExolcL/Bq52DpMUctpVGVdgBpMupg3SXe7TH1rC6UUpy/lseVQOgdOZWM0\n3f5TbuLrSlQHfwZFNsPern4+p1vf9ovK3O8pKXnmX4ha6PSlPBZvOEV6ThEAgd4uNPN3w95Og4Od\nlqaBHoQ0cSfQ29XGkYq6RBKGELXMoTPZfLryOCaTIrK9H7Fdg2jb1LPc9Qk5qhbWIAlDiFpk19FM\n/r02GXt7DS+N6UynVt62DknUI5IwhKihDEYTmw+kkX3tBiaT4nqJgcQTOlwb2PO7h7sQHORh6xBF\nPSMJQ4ga6rtt59iw91K5aY3cnHjlkS40kXdmCxuQhCFEDXToTDYb9l7C38uFaSM6Ym+vxU6rwdvd\nCQd7O1uHJ+qpuyaMS5cusWzZMhITE8nKysLJyYmQkBAGDRrEwIEDsbeXnCNEVcrJK2bhmpM42Gt5\nbmQnmvpJb0LUDJX+tf/zn//M8ePHGTx4ML///e/x8fGhpKSEc+fOsXPnThYsWMBf/vIXwsLCqite\nIeo0g9HEp6uOc73EwOQhIZIsRI1SacKIi4tj9uzZt01v164dQ4cOJS8vj0uXLlWwpBDiXhmMJhYm\nnORCZj7RHQPo0znQ1iEJUU6lCSMmJqbShT09PfH09KzSgISoj0oNRj5deZzDZ3MIDnJn4qC2Mu6T\nqHEsGivgr3/9KwUFBRgMBh599FHCwsJYtWqVtWMTol64oTfw4dIkDp/NoUOLRvx+bFcaOMq1QVHz\nWJQwdu/ejZubGzt37sTf358NGzbw5ZdfWjs2Ieq8VF0Bf/3qICcvXqVrGx9eju+Mk6PcBSVqpns6\njNm3bx8DBgzA399fustC3IdSg5Hvd6Wwbk8qJqWICWvMhIFtsdPWzwECRe1gUcLw9vbm9ddfZ9eu\nXUydOhWDwYDRaLR2bELUOfpSI3tO6Fi35yK6q8V4uzfg8cHtZIgPUStYlDDee+89vv/+e+Lj4/Hw\n8CAtLY0nnnjC2rEJUWcUFpeycV8qWw9lUFhcilajoX9EE0b3bSXXK0StYdGe6uXlxeTJk82fmzRp\nQpMmTawVkxA12rXCErRaDW4ujhaV1129zgdLjnD5ajGuDewZFt2c2K5BeLk3sHKkQlStShPGc889\nx7Rp0+jcufNt8woLC/nuu+9o0KABY8eOtVqAQtQkJXojbyzcyw29gZ6dAhkU2bTSd06cS7/GR8uS\nKCwuZUiPZozo1RInB7moLWqnShPGSy+9xHvvvUdKSgqdO3fG29ubkpISzp8/T3p6OuPGjWP8+PHV\nFasQNrc3WUdhcSmODlq2H8lgx5EMQoO96djSi3ZNPWni2xC9wYgut5jzGdf43+azlBpNTBrcjn5h\nQbYOX4j7UmnCCAkJ4fPPPyczM5O9e/ei0+lwcnJi8ODBdOvWDUdHy7rkQtQVO45kogFmPxlFalYB\n6xIvknTuCknnrgDgaK9FbzCZyzs6aHlpTGe6tPaxUcRCVB2LrmEEBgby0EMP/aYVbN++nblz56KU\nYsyYMUydOrXc/MzMTP7whz9QUFCAyWRi+vTpd33CXAhbSM8p4mz6NTq29MLP0xk/T2e6tfMl+9oN\nzlzK49SlPFIyC3B3dcDfy4WARi6EBnsT4OVi69CFqBIWJYwrV67wzjvvkJmZyddff01ycjKHDh26\n6+kok8nEnDlzWLRoEX5+fsTHxxMXF0dwcLC5zKeffsrQoUMZN24c586d4+mnn2bz5s33t1VCWMGO\nIxkA9O3S2DxNo9GYk0evUBn7SdRtFj0l9Prrr9OtWzfy8/MBaNWqFf/973/vulxSUhLNmzcnKCgI\nBwcHhg0bxqZNm8qV0Wg0FBYWApCfn4+/v/+9boMQVldqMLH7WBYNnR3o2kZOL4n6yaKEodPpGD9+\nPHZ2N+/ucHR0RGvBE6k6nY7AwLKjLn9/fy5fvlyuzAsvvMCqVauIiYlh2rRpvPHGG/cSvxDV4vDZ\nHAqLS+kVGoC9nTyNLeoni05J/folSfn5+Sil7rqcJWUSEhIYM2YMkydP5vDhw7z22mskJCTcdTlf\nX7e7lqkvpC3KWKst9iw/CsBD/drUmvauLXFWB2mLqmFRwhg4cCB//vOfKSoqYvny5fz3v/9lzJgx\nd10uICCAjIwM82edToefn1+5MsuWLWPhwoUAhIWFUVJSQm5uLl5eXpXWnZ1dYEnodZ6vr5u0xc+s\n1RapugIOn86mdRMPGmhrx74n+0UZaYsy95s4LepbP/XUU0RERNCxY0e2bdvGxIkTefzxx++6XGho\nKKmpqaSnp6PX60lISCAuLq5cmcaNG7N7924Azp07h16vv2uyEKI6KKXYcjCNtxcfQAEDIpraOiQh\nbEqjLDlvdB+2b9/O22+/jVKK+Ph4pk6dyscff0xoaCixsbGcO3eO119/nevXr6PVapkxYwbR0dF3\nrVeOGG6So6cyVdkW+UV6Fq1L5vDZHFwb2DN5SHu6tfOtkrqrg+wXZaQtytxvD8OihHHlyhW++uor\nUlNTMRgM5ukfffTRfa38fsgOcJP8GMpURVvkX9ezYW8qmw+kU1JqJKSZJ08P70gjN6cqirJ6yH5R\nRtqizP0mDIuuYTz33HN06NCB6Oho851SQtQl6dmF7EjKZNvhDEpKjXg0dCS+XzCxXYPQauXdL0KA\nhQmjuLiYN99809qxCFGtSkqNbD+cwe5jWVzU3TwC9WjoyJiYVsSENcbBXg6OhLiVRQmjS5cunDp1\ninbt2lk7HiGqRfLFqyxal8zlvGLstBq6BHvTMzSQsNbekiiEuAOLEsa4ceOYMGECAQEBODmVnctd\ntmyZ1QIToqqUGkwUXNejN5go0RvZejidbYcz0GhgUGRThkQ1x91VBtIU4m4sShivvfYa06ZNo0OH\nDnINQ9Qql/OK+etXB8gr1Jeb3sTXlSeGtqdloLuNIhOi9rEoYTg5OfHkk09aOxYhqpS+1Mi85UfJ\nK9TTtY0Pbi4OONrb4e/lQkxYYxniQ4h7ZFHC6NOnD9u3b6dv377WjkeIKqGUYvHGU6ReLqRvl8ZM\nHhJi65CEqPUsShhLlixhwYIFuLq64ujoiFIKjUbDTz/9ZO34hPhNth/JYNfRLJoHuPHYgDa2DkeI\nOsGihPHdd99ZOw4hqkROXjE/ndCxetcFXBvY8/yoTnLXkxBVxKKEERQk7yIWNVvSuRx+WHKE4+d/\nflWqg5ZnHuqIj4ezjSMTou6oNGG89tprvPvuu4wZMwaN5vanXeW2WmFrJqX4fucFvt+VAkBIM0+i\nOwbQrZ0fLg0sOh4SQlio0l/ULy87+sMf/lAtwQhxL4pLDCxMOMnB09n4eDTgjSd74OYodz4JYS2V\nJoxfXskaGRlZLcEIYYkSvZHEkzrWJ6aSlXudkGaePDuyE62CPGSQOSGsSPrsotbIKywh4aeL7D6W\nSXGJEY0G+ndrwiMPtJZnKoSoBpUmjNOnT1f4bgq5rVZUtwOnsvm/9ckUFpfi2dCRARFN6dulMV7u\nDWwdmhD1RqUJo0WLFixYsKC6YhHiNsUlBr758Qw7j2biYK/lsQFt5SltIWyk0oTh6Ogot9QKmzAY\nTWw7nMGa3SlcK9LT3N+NqSM6EOjtauvQhKi3Kk0YDg4O1RWHEMDN22R/OpbFqp0XyLl2AycHOx7q\n3ZJh0c2lVyGEjVWaMJYsWVJdcQhBenYh/7fhFGfTrmFvp2Vg96YM7SFDjwtRU1j9Lqnt27czd+5c\nlFKMGTOGqVOn3lZm7dq1/Otf/0Kr1dKuXTv+8Y9/WDssUYPkF+n5Yf8l1iemYjQpurXzZdwDbfD2\nkAvaQtQkVk0YJpOJOXPmsGjRIvz8/IiPjycuLo7g4GBzmYsXL/LFF1/wv//9j4YNG5Kbm2vNkISN\nGIwmVu9KQW8w4uxkj4uTPVfyb3Ai5SqXLhcC4O3uxGMD2hHWxsfG0QohKmLVhJGUlETz5s3NF86H\nDRvGpk2byiWMJUuW8Oijj9KwYUMAvLy8rBmSsJENe1NZvTvltun2dlo6tGhEp5be9OvamAaO8miQ\nEDWVVX+dOp2OwMBA82d/f3+OHj1arkxKSgoA48ePRynF888/T58+fawZlqhmV67dYPWuFNxdHHhh\ndGdKDEau3zDg2sCe1kEeODrIaLJC1AZWTRhKqbuWMRqNpKam8vXXX5ORkcFjjz1GQkKCucdxJ76+\nblUVZq1X09vi84ST6A0mnn+4C9Fdm1h1XTW9LaqTtEUZaYuqYdWEERAQQEZGhvmzTqfDz8+vXBl/\nf3+6du2KVqulSZMmtGzZkpSUFDp16lRp3TJm0E2+vm41ui2Onr/CT0czadPEg07NPK0aa01vi+ok\nbVFG2qLM/SZOq97YHhoaSmpqKunp6ej1ehISEoiLiytXpn///uzZsweA3NxcLl68SNOmTa0Zlqgm\npQYTX/9wGq1Gw4SB7SocIl8IUXtYtYdhZ2fHG2+8wZQpU1BKER8fT3BwMB9//DGhoaHExsbSp08f\ndu3axbBhw7Czs2PGjBl4eHhYMyxhRUopUnWFHDydzcHT2Vy+Wkz/iCY09av8FKMQoubTKEsuNNRA\n0sW8qSZ1t3OuFfOvFce4mHUzHns7LWGtvXliaHucnax/91NNagtbk7YoI21R5n5PSck9jKJKnEnL\n41/Lj5J/vZSubXyI7hhAp1ZecpusEHWI/JrFfTEpxc6kTBZvOIVS8NiAtjwQHiTXK4SogyRhiHtm\nMilOpl7l4Kmb1ymuFelxcbLn2VGd6NhCHrwUoq6ShCHuSeaVIr5Yc5ILmTdf39vQ2YHenQMZFt0c\n/0YuNo5OCGFNkjCERUxKselAGsu2nqPUYKJ7iB+xXYNo09QDO60MOy5EfSAJQ9xVzrVivkw4SXJq\nHg2dHXj6wQ5EhPjdfUEhRJ0iCUPckVKKHUmZfLvpDDf0RsJa+/D44HZ4NHSydWhCCBuQhFHPGYwm\ntBoNWm35u5oycopYsuUsSeeu4Oxkx5PD2tOzU4Dc/SREPSYJox7LL9Lz1n/2U1JqJCLEj6j2/rg2\nsGf17hT2nbyMAjq2aMQTQ9vj5S4vMxKivpOEUU+ZlOLzNSduvjfb0Y4tB9PZcjDdPL+ZX0OG92pJ\neFsf6VUIIQBJGPXW2p8ucvxCLqGtvHlxTCinUvNIPKEjr6iE2K5BhLWWRCGEKE8SRj10+lIeK3ac\np5GbE0892B57Oy0dW3rRsaU8dCeEuDNJGPWIUoqz6df4bNUxNGh4ZkRH3FwcbR2WEKKWkIRRD5Qa\njOw/lc0P+y6R8vNIsg/3C6ZtU08bRyaEqE0kYdRRpy/lcfB0NmfTr3ExqwCjSaEBurbxYWD3prRr\n1sjWIQohahlJGHXQ0fNX+HDJERRgp9XQzL8hIc0aEdM1CD9PZ1uHJ4SopSRh1DGX84pZ8P1x7Oy0\nPPtQRzq09MLJwc7WYQkh6gBJGHVIid7IJ98dpeiGgSeGhNC1ra+tQxJC1CEyzGgdoZRi0fpk0rIL\n6dc1iD5dGts6JCFEHWP1hLF9+3YGDx7MoEGDWLBgwR3LrV+/npCQEI4fP27tkOqU4hIDWw+n8+aX\n+0g8oSO4sTuP9m9j67CEEHWQVU9JmUwm5syZw6JFi/Dz8yM+Pp64uDiCg4PLlSsqKuKrr74iLCzM\nmuHUKSaTImHPRTbsTeX6DQN2Wg0RIX481r8N9nbScRRCVD2rJoykpCSaN29OUFAQAMOGDWPTpk23\nJYyPPvqIp59+mi+++MKa4dQZeYUlLPj+OMmpeXi6OdG/WxNiwoJo5CbDjgshrMeqh6I6nY7AwEDz\nZ39/fy5fvlyuzMmTJ8nKyiImJsaaodQZxy5c4S9f7iU5NY+w1j7Mm/EAI/u0kmQhhLA6q/YwlFJ3\nnT937lz+9re/WbzML3x93e4rttomJTOfxWtPsvdEFvZ2Gp5+qBPD+7RCo9HI8B63qG/7RWWkLcpI\nW1QNqyaMgIAAMjIyzJ91Oh1+fmWv9iwqKuLs2bNMnDgRpRQ5OTk899xzfPrpp3Ts2LHSurOzC6wW\nd02Sqitgw95U9hzXoYC2TTwY178NLQLcyckpxNfXrd60xd1IW5SRtigjbVHmfhOnVRNGaGgoqamp\npKen4+vrS0JCAu+//755fsOGDfnpp5/MnydOnMjMmTPp0KGDNcOq8fSlRvaevMzWw+mcz8gHbr6f\nYnRMMKGtvGTYcSGETVg1YdjZ2fHGG28wZcoUlFLEx8cTHBzMxx9/TGhoKLGxseXKazQai09J1UUm\nk2LXsUxWbD9PXqEeDdA52JuYsMZ0ae2DVhKFEMKGNKqW/oWua13Mkym5fLv5LJcuF+JoryUuogmx\nXYPw8ah87CfpbpeRtigjbVFG2qJMjT4lJe5OX2pkyZazbD6Yjgbo1SmAUX1byTu0hRA1jiQMG0q7\nXMj874+TnlNEkI8rTz7YnhYB7rYOSwghKiQJwwZMSrHpQBpLt5zDYDTxQHgQj8S2xlFGlRVC1GCS\nMKpZbv4Nvlx7khMpV2no7MATQzvStY2MKiuEqPkkYVQTfamR3ceyWLb1HNdLDHQO9uaJISF4NJQn\ntIUQtYMkDCvLyStm86F0dhzJoOiGAUcHLZMGtyOmS2N5nkIIUatIwrASk0mxLvEiK3dcwGhSuLk4\n8GDP5vQLC5I7oIQQtZIkDCu4cu0GX6w5walLeXg0dCQ+JpjI9n442MtFbSFE7SUJowoppdhzQsdX\nG09TXGIgvK0vjw9uJ4MDCiHqBEkYVSS/SM9/Npzi4OlsnBzsmDwkhD6dA+U6hRCizpCEcZ+UUuxL\nvsxXG09TWFxK26aeTBnWHj/Pyof0EEKI2kYSxn1Iu1zIf388TXJqHg72WsbFtaF/RBMZJFAIUSdJ\nwvgNCotLWbnjPFsOpaMUdAn2ZlxcG/y9XGwdmhBCWI0kjHtgMim2HU5n+fbzFN0wEODlwvj+bQht\n5W3r0IQQwuokYVjoVOpV/vvjGS5dLqSBox2PxLamf0QT7O2s+lp0IYSoMSRh3EVu/g2WbDnL3pOX\nAegdGsiYmFYypIcQot6RhHEHJXojG/elkrDnIvpSEy0D3Xh0QFuCG3vYOjQhhLAJSRi/YjCa2JmU\nyaqdF7hWpMfNxYHH+relV+dAuftJCFGvWT1hbN++nblz56KUYsyYMUydOrXc/EWLFrF06VLs7e3x\n8vJi7ty5BAYGWjus2xiMJhJP6Fjz00V0uddxdNDyYM8WDI5shksDyatCCGHVv4Qmk4k5c+awaNEi\n/Pz8iI+PJy4ujuDgYHOZDh06sHz5cpycnPjmm2/4+9//zgcffGDNsMopNZjYmZTBusRUcq7dwE6r\noV/XIEb0aoGnXKcQQggzqyaMpKQkmjdvTlBQEADDhg1j06ZN5RJGZGSk+f9hYWGsXr3amiGVk3ml\niM9WHefS5UIc7LXEhTdhcFQzvD1kNFkhhPg1qyYMnU5X7vSSv78/R48evWP5ZcuW0bdvX2uGZLbr\naCZfbTxNSamRPp0DGd1X7nwSQojKWDVhKKUsLrtq1SqOHz/O4sWLqzyOkxev8p/1yZQaTTg72qPR\naEjLLsTZyY5pD3Uksr1/la9TCCHqGqsmjICAADIyMsyfdTodfn5+t5XbvXs3CxYs4KuvvsLBwcGi\nun193Swqd+R0Nh8tS8JkMuHl4Uz+dT3Xbxjo2Mqb343rSoC3q2UbU4NZ2hb1gbRFGWmLMtIWVcOq\nCSM0NJTU1FTS09Px9fUlISGB999/v1yZEydO8Oabb7Jw4UIaNWpkcd3Z2QV3LXPswhX++d1RlFK8\nMDqUzsE+wM2ej0ajAZPJonpqMl9ft1q/DVVF2qKMtEUZaYsy95s4rZow7OzseOONN5gyZQpKKeLj\n4wkODubjjz8mNDSU2NhY3n33XYqLi3n55ZdRStG4cWPmzZt33+s+fCaHeSuPAfDimM7lxnuSd1QI\nIcS906h7udBQg2RnF1BSamRnUiYdWjQi8OdTS0op1iemsmzrOezttbw4JpROLevu4IBy9FRG2qKM\ntEUZaYsyNbqHYW3fbTvHj/vT0AARIX4MimzGjwcusee4jkZuTrwwOpSWge62DlMIIeqEWpswUnUF\nbDqQho9HA1ydHdiXfJl9yTcHCAxu7M7zo0PlwTshhKhCtTJhmEyKxRtPoRRMGtyOji28OHYhlw17\nU/Fv5MK4uDY42Muw40IIUZVqZcL4cV8q59LziQjxM1+fCG3lLS8yEkIIK6qVh+GL1pzAydGO8XFt\nbB2KEELUG7UyYRRc1zOyd0sauck1CiGEqC61MmE82Lslcd2a2DoMIYSoV2plwnhmVGd5l7YQQlQz\n+asrhBDCIpIwhBBCWEQShhBCCItIwhBCCGERSRhCCCEsIglDCCGERSRhCCGEsIgkDCGEEBaRhCGE\nEMIikjCEEEJYRBKGEEIIi1g9YWzfvp3BgwczaNAgFixYcNt8vV7PK6+8wsCBAxk7diwZGRnWDkkI\nIcRvYNWEYTKZmDNnDgsXLmTNmjUkJCRw7ty5cmWWLVuGh4cHGzdu5PHHH+fdd9+1ZkhCCCF+I6sm\njKSkJJo3b05QUBAODg4MGzaMTZs2lSuzadMmRo0aBcCgQYP46aefrBmSEEKI38iqCUOn0xEYGGj+\n7O/vz+XLl8uVuXz5MgEBAQDY2dnh7u5OXl6eNcMSQgjxG1g1YSil7rmMUgqNRmOtkIQQQvxG9tas\nPCAgoNxFbJ1Oh5+f321lsrKy8Pf3x2g0UlhYiIeHx13r9vV1q/J4aytpizLSFmWkLcpIW1QNq/Yw\nQkNDSU1NJT09Hb1eT0JCAnFxceXKxMbGsmLFCgDWr19Pjx49rBmSEEKI30ijLDlvdB+2b9/O22+/\njVKK+Ph4pk6dyscff0xoaCixsbHo9Xpee+01Tp48iaenJ++//z5Nmsj7uoUQoqaxesIQQghRN8iT\n3kIIISwiCUMIIYRFJGEIIYSwSK1LGHcbm6ouy8rKYtKkSQwdOpThw4fzn//8B4Br164xZcoUBg0a\nxJNPPklBQYGNI60eJpOJUaNGMW3aNADS0tJ45JFHGDRoENOnT8dgMNg4wupTUFDASy+9xJAhQxg2\nbBhHjhypl/vFokWLePDBBxk+fDivvvoqer2+Xu0Xs2bNomfPngwfPtw8rbL94K233mLgwIE89NBD\nnDx58q7116qEYcnYVHWZnZ0dM2fOZO3atXz77bd8/fXXnDt3jgULFhAdHc2GDRuIiopi/vz5tg61\nWvznP/8hODjY/Pkf//gHTzzxBBs2bMDNzY1ly5bZMLrq9fbbbxMTE8O6detYtWoVrVq1qnf7hU6n\nY/HixSxfvpzVq1djNBpJSEioV/vF6NGjWbhwYblpd9oPtm3bRmpqKhs3bmT27Nm8+eabd62/ViUM\nS8amqst8fX1p3749AK6urgQHB6PT6cqNxzVq1Ch+/PFHW4ZZLbKysti2bRsPP/ywedqePXsYNGgQ\ncLMdfvjhB1uFV60KCwvZv38/Y8aMAcDe3h43N7d6uV+YTCaKi4sxGAzcuHEDPz8/EhMT681+ERER\ngbu7e7lpv94PfvmbuWnTJkaOHAlAly5dKCgoICcnp9L6a1XCsGRsqvoiLS2N5ORkunTpwpUrV/Dx\n8QFuJpWrV6/aODrrmzt3LjNmzDAPI3P16lU8PDzQam/u0gEBAfVm30hLS6NRo0bMnDmTUaNG8cYb\nb1BcXFzv9gt/f3+eeOIJ+vXrR9++fXFzc6NDhw64u7vXy/3iF7m5ueX2g9zcXKD8OH5ws/10Ol2l\nddWqhCGPjNxUVFTESy+9xKxZs3B1da13Y29t3boVHx8f2rdvb94nlFK37R/1pV0MBgMnTpzg0Ucf\nZcWKFTg7O7NgwYJ6s/2/yM/PZ9OmTWzZsoUdO3ZQXFzM9u3bbytX39rlTir6e3q3trHqWFJVzZKx\nqcpAbb4AAAdwSURBVOo6g8HASy+9xEMPPUT//v0B8Pb2JicnBx8fH7Kzs/Hy8rJxlNZ18OBBNm/e\nzLZt2ygpKaGoqIi5c+dSUFCAyWRCq9WSlZVVb/aNgIAAAgICCA0NBWDgwIF8/vnn9W6/2L17N02b\nNsXT0xOA/v37c+jQIfLz8+vlfvGLO+0H/v7+ZGVlmctZ0ja1qodhydhUdd2sWbNo3bo1jz/+uHna\nAw88wPLlywFYsWJFnW+T6dOns3XrVjZt2sT7779PVFQU//jHP4iKimL9+vVA/WiHX/j4+BAYGMiF\nCxeAm9dyWrduXe/2i8aNG3PkyBFKSkpQSrFnzx7atGlT7/aLX/cc7rQfxMXFsXLlSgAOHz6Mu7u7\n+dTVndS6oUEqGpuqvjhw4AATJkygbdu2aDQaNBoNr7zyCp07d+Z3v/sdmZmZNG7cmI8++ui2C191\n1d69e/nyyy/57LPPuHTpEtOnTyc/P5/27dvz7rvv4uDgYOsQq0VycjJ/+tOfMBgMNG3alHfeeQej\n0Vjv9otPPvmEhIQE7O3t6dChA2+99RZZWVn1Zr949dVXSUxMJC8vDx8fH1588UX69+/Pyy+/XOF+\nMHv2bHbs2IGzszPvvPMOHTt2rLT+WpcwhBBC2EatOiUlhBDCdiRhCCGEsIgkDCGEEBaRhCGEEMIi\nkjCEEEJYRBKGEEIIi0jCEDXaAw88wNmzZ6tlXZ988km5oa9nzpzJ119/fd/1zpw5k+HDhzN9+vT7\nrqsyycnJrFu3zqrrEPWbJAwhfvbJJ59QWlpapXXm5OTw/9u7v5AmuziA49/ln7S8KOvWoghaI8KL\nihkJWon0R/Y8S2NYOL1IEFqE3gjRRZZEBcPyJqE/lDSIyBp2UV4IEVgGXeyiDKMVFnSRltTmaPr4\ney/Eh3KL9vYG7+vb73O182znnN/DYL+dHfY7fX199Pb2EgwGf+vYcz1//vyXE8b09PRvjkb9H2nC\nUPPS69evOXjwIDU1NRiGYZc+AHA6nXR1dVFdXU1FRQV9fX32c/fv32fnzp14vV66urpwOp0kEgna\n2tpwOBz4fD5M0yQWiwEwPDyM3++nsrKS1tbWH8Zz584dqqqq8Hg8BAIBPn78SDwex+/38/XrV0zT\n5OrVq9/1CYfDHDp0yG5blkVpaaldL+3ixYvs27cPr9dLU1MTY2NjAExOTnL69GmqqqowDINAIMD4\n+DidnZ08fvwY0zRpb28HZiojmKaJx+OhoaGBt2/fAjP/kDcMg5MnT+Lz+Xj48OE/eTvUn0KU+g8r\nLy+Xly9ffndtampKTNOUaDQqIiKxWEwqKyvt9tq1a+X69esiIvL06VMpLS0VEZHR0VHZvHmzjIyM\niIjIlStXxOl0ysTEhN0vkUjY87S2tkptba0kk0lJJpOye/duGRgYSIlxeHhYtm7dKqOjoyIi0tHR\nIUeOHBERkXfv3onb7U57b4lEQtxut3z69ElERPr7+8Xv94uISDgclmPHjtmvDYVC0tLSIiIinZ2d\nEggEZGpqSkTE7t/T0yOHDx+2+4yNjYnb7ZZXr16JiMjNmzelpqZGREQGBwfF5XJJJBJJG5tS6egK\nQ807b968IRqN0tzcjGEY7N+/n8nJye9OX9y1axcAxcXFfPjwgWQySSQSYf369RQVFQFQXV2dMrbM\nqZSzY8cOcnJyyMnJweVyMTIyktJncHCQsrIyli1bBoDP52NgYOCn95GXl8f27du5e/cuMFMYbvYQ\npP7+fh49eoRhGBiGQSgU4v3798BMefe6ujqysrIA7Oqsc0UiEdatW8fq1asB2Lt3L0NDQ0xMTACw\ncuVKNmzY8NM4lZo1r8qbKwUzH+qFhYXcvn077fMOh4OFCxcC2AfnWJaVkgzmttPJzc21H2dlZaU9\nD1pEUs4RmJ33ZwzD4NSpU+zZs4cnT55w9uxZe8ympia8Xm/a+TKRLq5v24sWLcpoHKVm6QpDzTur\nVq0iLy+PcDhsX4tGo8TjcSD1A3W2XVxczLNnz+zf8b/d9wAoKCjgy5cvfzuekpISHjx4YO8x3Lhx\ngy1btqTMn87GjRuJxWIEg0EqKirsRLdt2zZCoRCfP38GIJlM8uLFCwDKy8u5du2avUE/e5JeQUGB\nvfcye79DQ0N22fOenh5cLpcmCvXLdIWh/tMcDgf19fVkZ2fb35h7e3u5cOEC7e3tXL58GcuyWL58\nOR0dHXafuWPAzEEyx48fp7GxkaVLl1JWVkZ2djb5+fkANDQ0UFdXR35+Pt3d3RnHuGbNGpqbm6mv\nr2fBggUUFRXR1taWMv+PGIbB+fPnCYVC9jWPx8P4+DgHDhzA4XAwPT1NbW0tTqeTxsZGgsEghmGQ\nm5vLihUrOHfuHCUlJVy6dAnDMNi0aRNHjx7lzJkztLS0YFkWhYWF9gpGqV+h5c3VHyUej7N48WJg\n5hv3rVu3fst/LZT6E+gKQ/1Ruru7uXfvHpZlsWTJEk6cOPFvh6TUvKErDKWUUhnRTW+llFIZ0YSh\nlFIqI5owlFJKZUQThlJKqYxowlBKKZURTRhKKaUy8hf8CwfjbzhfpQAAAABJRU5ErkJggg==\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f47b218dbd0\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(ag_means)\n", - "plt.ylabel('Time(s)')\n", - "plt.xlabel('Length of vector')\n", - "_ = plt.title('Time to sum the elements of 1000 vectors (AutoGraph)')\n", - "_ = plt.ylim(ymin=0)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "d7IAJ6Bwbk9t" - }, - "source": [ - "## Eager" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "XMu5-12yoOzY" - }, - "outputs": [], - "source": [ - "from tensorflow.python.eager import context" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "_vt9MzpyjQ4T" - }, - "outputs": [], - "source": [ - "# Sum written using for loop and run with tf.eager\n", - "def sum_all(elements):\n", - " sum_ = 0.0\n", - " length = elements.shape[0]\n", - " for i in tf.range(length): \n", - " sum_ += elements[i][0]\n", - " return sum_\n", - "\n", - "eager_means = []\n", - "for num in range(max_elements):\n", - " with context.eager_mode():\n", - " durations = []\n", - " for i in range(trials + burn_ins):\n", - " \n", - " start = time.time()\n", - " for _ in range(batches):\n", - " run_trial(num)\n", - " \n", - " if i \u003c burn_ins:\n", - " continue\n", - " \n", - " duration = time.time() - start\n", - " durations.append(duration)\n", - " eager_means.append(np.mean(durations))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 301 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 422, - "status": "ok", - "timestamp": 1532460024499, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "5gHVdMlD-A-T", - "outputId": "3b581cb7-7ef9-489c-92f1-3e52c0c2dc8a" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEcCAYAAAAydkhNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XlclNX+wPHPsC+CILviCiruhAiiqaAimqnglpZLWpkt\ntmh50+rXvZZ2q9veq5ulZYtlaZp7mQuaorjklkoqiwgKsojsy8yc3x9cBxFUVGBYvu+/mGfmPM93\nzjzMd85zznOORimlEEIIISphYuwAhBBC1F2SJIQQQtyQJAkhhBA3JElCCCHEDUmSEEIIcUOSJIQQ\nQtyQJAlg8eLFvPrqq8YOo14ZOHAge/furfHjzJs3jw8//LDGj1NXxMfHExERQc+ePfnuu++MHU6D\ndvbsWcaMGWPsMCo1btw4YmNjjR0G0EiSxD333IOfnx9+fn506tSJHj16GLZt2LCBxx9/nNdff73G\n40hOTsbHxwe9Xl/jx6pOje2L+lZq8nNcsmQJgYGBHDp0iEmTJlV4fvPmzUyYMAFfX1+mTJlS4flT\np04xevRofH19GTNmDDExMeWef+eddwgMDKR379688847t1W2pk2ePJlVq1bV2vE++ugjHn30UcPj\ngQMH0qNHD/z8/AzfD2+88UatxXOtRx55pM78zzWKJHH48GH+/PNP/vzzT5o3b87ixYsN2+6///5a\ni0MphUajQe5frN9q8nO8cOEC3t7eN3zewcGBhx9+mBkzZlR4rqSkhKeeeorw8HAOHDhAeHg4Tz75\nJFqtFoAVK1awfft21q9fz7p164iMjOTHH3+sUtn64HY+j7S0NKKjoxk0aFC57YsXL+bPP/80fD+8\n8sor1R3mTel0OqA0YUVHR5Oenl6rx69Mo0gS11JKVTiZPvnkE1588UWg7Ffi6tWrCQ4OJjAwkBUr\nVnD8+HFGjhxJQEBAhVbHqlWruO+++wgMDOTRRx/lwoULlR578uTJAPj7++Pn58fRo0dRSvHpp58y\ncOBA+vbty0svvURubm6l5S9fvszMmTPp1asXgYGB5X5p+vj4cP78ecPja3/979+/nwEDBrBkyRL6\n9OlDv3792Lp1Kzt37iQsLIzAwEAWL15c6TF/+ukn1q9fz5IlS/Dz8+OJJ54wPHfq1ClGjhxJr169\nmD17NsXFxYbnduzYQXh4OL169WLixIn8/fffle4fIDY2lunTpxMYGMiwYcPYvHnzDV97s/0OHDiQ\npUuXMnLkSO655x5eeeUVMjIyeOyxx/Dz82P69Onk5OQYXn/kyBEmTJhAr169CA8PZ//+/YbnJk+e\nzIcffsjEiRPx8/PjkUceISsry/AclP8cExMTmTx5Mv7+/gQFBTF79uwbvodt27Zx//33ExAQwJQp\nU4iLiwNg6tSpREdHs2DBAvz8/Dh37lyFskFBQQwdOhQXF5cKz+3fvx+dTseUKVMwNzdn8uTJKKXY\nt28fAL/88gvTp0/H1dUVV1dXpk2bxpo1awCIjo6+adlrbdq0qcJlmmXLlvHkk08CUFxczFtvvUVI\nSAj33nsv//znP8udG1u3biU8PJyePXsyZMgQdu/ezfvvv8+hQ4d4/fXXy/2C//PPPxk7diy9evVi\n3LhxHD58uNxn9P777zNx4kR8fX1JSkpi9erVDB48GD8/PwYPHsyGDRsq/Qz27NlDly5dsLCwKLf9\nRonm/PnzTJ06lcDAQIKCgnjhhRfK/Z+eOHHCcJnw2Wef5fnnny/XErjVefvFF18Yzlu9Xo+FhQVd\nunRh9+7dlcZTq1QjExISoqKiospt+/jjj9WLL76olFIqKSlJdezYUb322muqqKhI7dmzR3Xr1k09\n9dRTKjMzU6WkpKigoCB14MABpZRSv//+uxoyZIiKi4tTOp1O/fe//1UPPPBApcdOSkpSPj4+Sq/X\nG7atXLlSDRkyRCUlJan8/Hz19NNPG2K53rvvvqtee+01pdPplFarVQcPHjQ85+PjoxITEw2PX3rp\nJfXBBx8opZSKjo5WnTt3Vp9++qnSarXqp59+Ur1791Zz5sxR+fn56syZM6pbt27q/PnzlR732n1d\nW4/jxo1TaWlp6sqVK2rYsGFqxYoVSiml/vrrLxUUFKSOHTum9Hq9WrNmjQoJCVHFxcUV9p2fn68G\nDBig1qxZo/R6vTp58qQKDAxUZ8+erXDsW+03JCREPfDAAyojI0OlpqaqoKAgFRERoU6dOqWKi4vV\nlClT1CeffKKUUiolJUUFBASoXbt2KaWUioqKUgEBASozM1MppdSkSZNUaGioOnfunCoqKlKTJk1S\n77777g0/x9mzZ6vPPvtMKaVUUVGROnToUKV1GRcXp3x9fVVUVJTSarXqiy++UKGhoaqkpMRw3JUr\nV1Za9lo//fSTmjx5crltX331lXrsscfKbXv88cfVV199pZRSqmfPnuro0aOG544fP678/PyqVPZa\nBQUFys/PT507d86wbcyYMWrTpk1KKaXeeOMN9cQTT6js7GyVl5enZs6cqd577z2llFJHjx5VPXv2\nNPwPpqamqri4uErfe1ZWlurVq5dat26d0ul0asOGDapXr14qKyvL8PqQkBB19uxZpdPpVE5OjvLz\n81MJCQlKKaXS0tIM59H13nrrLbVgwYJy2yr7brjq3LlzKioqSpWUlKjMzEw1adIktWjRIqWUUsXF\nxSokJER9++23SqvVqi1btqguXbrc1nkbHh6uUlJSVFFRkeGYr7/+uvr3v/9daTy1qdG1JKpCo9Hw\n1FNPYWFhQZ8+fbC2tmb48OE4Ojri5uaGv78/J0+eBODHH39kxowZtG3bFhMTE2bMmEFMTAwXL168\n4f7VNb9WNmzYwMMPP0yLFi2wtrZm9uzZbNq0qdLr3WZmZqSlpZGUlISpqSk9e/asdJ+VMTc3Z+bM\nmZiamnLfffdx+fJlpk6dirW1Nd7e3nh7e9/0135lpkyZgrOzM/b29oSEhHDq1CkAVq5cyYQJE+jW\nrRsajYbw8HAsLCw4evRohX3s2LEDT09PwsPD0Wg0dOrUiSFDhvDrr79WeG1V9jtp0iSaNWuGq6sr\n/v7+9OjRAx8fH8zNzQkNDTXEuG7dOoKDg+nXrx9Q+gu9a9eu7Ny507Cv0aNH06pVKywsLBg2bJih\n7FXX1rmZmRnJycmkpqZiYWGBn59fpXW2efNmgoODCQoKwtTUlEceeYTCwsJyv5DvVH5+PnZ2duW2\nNWnSxPCL9/rn7ezsyM/Pr1LZa1lZWTFo0CDDr/SEhATi4+MNl25WrVrFvHnzsLOzw8bGhhkzZhhe\nu2rVKsaOHUtQUBAArq6utG3bttL3ExkZSZs2bRgxYgQmJiYMHz6cdu3asWPHDsNrIiIi8PLywsTE\nBFNTU0xNTTl9+jRFRUU4Ozvj5eVV6b5zcnKwtbWtsP2pp54iICCAXr16ERAQwMqVKwFo1aoVQUFB\nmJmZ4ejoyNSpUzlw4ABQ2iLV6XRMmjQJU1NTQkND6d69u2GfVTlvp0yZgpubW7mWja2tLdnZ2ZXG\nX5vMjB1AXeXk5GT428rKCmdnZ8NjS0tLwz/XhQsXWLhwIW+99RZQdr06NTUVDw+PWx7n0qVLNG/e\n3PC4RYsWaLVa0tPTcXV1LffaRx99lI8//pjp06ej0WgYN25cpdemK+Pg4IBGozG8n8re49X3VFXX\nlre2tiYtLQ0orZO1a9caRucopdBqtVy6dKnCPi5cuMCRI0cICAgwvFan0xEeHl7pa2+132tjsrS0\nrPD42s9t8+bNhi+cq/u6+uUFlPvMra2tb1o/c+fO5YMPPmDs2LGGfoPKRs5c/3lrNBo8PDxITU29\n4b6rysbGpsKXem5uLk2aNKn0+dzcXGxsbKpU9nrDhw/n7bff5sknn2TDhg0MHjwYCwsLMjMzKSgo\nKPfe9Xq9IaGmpKQwYMCAKr2f6+sKoHnz5uXqyt3d3fC3tbU177//PkuXLmX+/Pn07NmTuXPn0q5d\nuwr7tre3Jy8vr8L2Tz/9lN69e1fYnpmZyRtvvMHBgwfJz89Hp9Ph4OAAlPZvuLm5lXv9tf/7VTlv\nr30fV+Xl5WFvb19he22TJHGX3N3deeKJJ6rUAX71S/parq6u5fowkpOTMTMzK/cFdZWNjQ3/+Mc/\n+Mc//kFsbCyTJ0+me/fu9O7dG2trawoKCgyvTUtLq/TEqw3u7u7MnDmTxx9//Jav9fDwIDAwkKVL\nl1brfqty3PDwcBYsWHDbZSv7HJ2cnAx9VYcOHWLatGkEBATQsmXLcq9zdXXlzJkz5bZdvHixWj6r\n9u3bs2zZsnLbTp8+behD8fb2JiYmhm7dugGlfUrt27e/adnKRlgB3HvvvcybN4+YmBg2btzI/Pnz\nAXB0dMTa2poNGzZU+JEDpZ/htX1n17q+Xl1dXdmyZUu5bRcuXKB///43LNO3b1/69u1LcXEx77//\nPq+++irLly+vcKyOHTuydu3aCttv1CJ/99130Wg0bNiwAXt7e7Zu3WroN3FxcamQ5C9evEirVq0M\n7/lOztu4uDhGjhx5W2VqglxuqsStLt1ca+LEiSxevJizZ88Cpc3Yyi6VADRr1gwTExMSExMN24YP\nH86yZctISkoiLy+P999/n+HDh2NiUvGjiYyMNJS1sbExNK+htON6w4YN6PV6du3aZWgKVwdnZ+cb\n/mNXZvz48axYsYJjx44BpZcydu7cWekv8eDgYOLj41m7di1arZaSkhKOHz9u6My90/3eysiRI9m+\nfTu7d+9Gr9dTVFTE/v37q/SLvrLP8ddffzWUtbe3x8TEpNLPcNiwYURGRrJv3z60Wi1Lly7F0tIS\nX1/fKsWt1+spLi5Gq9WW+xsgICAAExMTvv32W4qLiw2/XAMDAwEIDw9n2bJlpKamkpqayrJlyxg9\nevRNy1b2qxrA1NSUsLAw3n77bbKzs+nbty+AoYW7aNEiMjMzAUhNTTV0wI4dO5bVq1ezb98+lFKk\npqYaPuvrz7MBAwZw7tw5Nm7ciE6nY9OmTcTFxRESElJpTBkZGWzfvp2CggLMzMwM/yOV6du3LydO\nnCjXoX4zeXl52Nra0qRJE1JTU8v9qPH19cXU1JTly5ej0+nYunWr4RyFOztvi4uLOXHihKFejanR\nJYnKfgXe6jU3ezx48GAee+wxnn/+efz9/Rk5ciR//PFHpfu1srJi5syZTJw4kYCAAI4dO8bYsWMZ\nNWoUkyZNIjQ0FGtr6xsOu0tISODhhx/mnnvuYeLEiTz00EP06tULgJdffpnt27fTq1cvNm7cyODB\ng+/qPV5r7NixnD17loCAAJ5++ulbvr5r1668/vrrLFiwgICAAMLCwgyjaK5na2vLl19+yaZNm+jX\nrx/9+vXj3XffrfSf91b7vZ335O7uzqeffsrixYsJCgoiJCSEL7/80vAD4WZlK/scjx8/zrhx4/Dz\n8+Opp57i5ZdfpkWLFhXKtm3blnfeeYfXX3+doKAgIiMj+eyzzzAzM7vlcQHWrl1L9+7dWbBgAYcO\nHaJHjx6GG0HNzc359NNPWbNmDQEBAaxevZpPP/3UsO8JEyYQEhLCyJEjGTlyJCEhIYwfP75KZSsz\nfPhw9u7dy7Bhw8olxBdeeIHWrVszfvx4/P39mT59OgkJCQB0796dRYsWsWjRInr27MmUKVMM/XdT\npkzh119/JTAwkIULF+Lg4MBnn33G0qVL6d27N0uXLmXx4sU0bdq00rrS6/V89dVX9O/fn969e3Pg\nwAFee+21SmN3cnKid+/ebN26tdz2J554wnBPlZ+fH7NmzQLg6aef5q+//sLf35+ZM2cSFhZmKGNu\nbs7HH3/MypUr6dWrFxs2bGDgwIGG/oXbPW+hdARcYGBgpaPYaptG3c7P5ts0f/58IiMjcXJyYv36\n9QDExMTw2muvUVRUhJmZGa+99pqh+SuEELUlNjaWl156ydA5XZ3Gjx/PxIkTiYiIuKPyDzzwAAsX\nLrzpPTO1pUaTxMGDB7G1tWXu3LmGJPHII48wbdo07r33Xnbu3MmSJUv49ttvayoEIYSocQcOHKBt\n27Y4Ojqybt06/vWvf7F169ZK+xbrmxrtuPb39yc5ObncNo1GY7ihKScnp8KoACGEqG/i4+N57rnn\nyM/Pp1WrVnz00UcNIkFADbckoHS0zsyZMw0tidjYWB599FHDnc8rVqyo0lBRIYQQta/WO65/+OEH\nXn75ZSIjI5k3b55h6JwQQoi6p9aTxC+//GIYeTN06NByQ8VupoYbPEIIUe9l5xVz/Gx6tX5f1vjN\ndNcH6+bmxv79+wkICGDv3r20adOmSvvRaDSkpeXc+oWNgIuLndTF/0hdlJG6KNPY6iL9SgFb9p9n\n17ELFJfo+ee0XrRyK51mxcXF7halb65Gk8ScOXOIjo4mKyuL4OBgZs2axeuvv84bb7yBXq/H0tKy\nVtZxEEKIhijjSiG/7I5j71+p6JWimb0lw4Jb09K18ulU7kSNd1xXp8b0y+BmGtuvpJuRuigjdVGm\noddFXmEJG/eeY+vBJLQ6Pc2dbRkW2IrAzm6YmZbvRajTLQkhhBDV61RCJv9de4LcghKa2VsS0a8d\nQV3cMTG59WwSd0KShBBC1BORR5JZvuU0AGODvRjc0xML88rnp6oukiSEEKKO0+n1/LQ9lt8PnqeJ\ntTlPj+5Gh5YOtXJsSRJCCFFHXczIY/fxi+z9K4Ws3GI8nGx4dlwPXB2say0GSRJCCFHHFBRp+WL9\nSY6cTQfA2tKMgX4tGN3fCxur2v3aliQhhBB1SGZ2IR+sPEZSWi7enk0Z5OfJPe2da7zv4UYkSQgh\nRB1x/lIuH6w8yuWcIkL8WvDQ4A41NmqpqiRJCCGEESmlOJt8hT3HU4g+mUpRiY7xId6EBbSs0iJp\nNU2ShBBCGMmhvy+xMjKWS5dL16d3tLNk+vBO9PKpuD64sUiSEEKIWqbXK9b8EcfGvecwNzMhqIsb\nfbp50KmVo9EvL11PkoQQQtSi/MISFq87yfG4DFwdrZk1uhstXKpvrqXqJklCCCFqScaVQt798Qgp\nmfl0bdeMx0d2wdbK3Nhh3ZQkCSGEqAUX0vN498cjXM4pYkivlowP8a5zl5YqI0lCCCFqWNyFbD5Y\neZTcghLGhXgxLLC1sUOqMkkSQghRA5RSJKTksPvYRfb8dZESrZ5pw3zo16O5sUO7LZIkhBCimh2L\nzWBV5FmS0vIAcGhiweQRHbmng4uRI7t9NZok5s+fT2RkJE5OTqxfv96w/dtvv2X58uWYm5szYMAA\nXnjhhZoMQwghas3Wg+f5YdsZTDQaenZ0oV93D7q0bYapicmtC9dBNZokRo8ezeTJk5k7d65hW3R0\nNDt27GDDhg2YmZmRmZlZkyEIIUSt0OsVK7afYevBJOxtLXh2bHfaetgbO6y7VqNJwt/fn+Tk5HLb\nfvjhBx577DHMzEoP3axZs5oMQQghaoRerzgWm0Fyei5pWYWcS83hXEoOzZ1teW5sd5xrcTrvmlTr\nfRIJCQkcPHiQ999/H0tLS+bOnUu3bt1qOwwhhLhjWp2epRtPEX0ytdz27l5OzBjRGZs6fu/D7aj1\nJKHT6cjOzuann37i2LFjPPfcc2zbtq1KZe92Qe+GROqijNRFGamLMjVVF0UlOv799QEOnkqlU5tm\njB3UHrdmNrg52mBl2fDGAtX6O3J3d2fIkCEAdO/eHRMTEy5fvoyjo+Mty6al5dR0ePWCi4ud1MX/\nSF2UkbooU1N1UVCk5eOfjxGTmEWXts14OqIblhal6zzkZBdQF2v/bpNljXe3K6XKPR48eDB79+4F\nID4+Hq1WW6UEIYQQxhR/MZvXvz5ITGIWPTu68MyY7oYE0ZDVaEtizpw5REdHk5WVRXBwMLNmzWLM\nmDHMmzePESNGYG5uzltvvVWTIQghxF3R6fVs3HuO9XsS0OkVQ3q1ZFyIV70d0nq7NOr6n/p1mDSl\nS8llhTJSF2WkLspUV12kXylg8boTxCZn42hnySPDO9G5Tf0akXm3l5saXi+LEEJUgyNn01m64SR5\nhVoCOrkyOaxjnZ+xtSZIkhBCiGsUlehY+0c8v+5PxMzUhKlDO9K/R/M6sZSoMUiSEEIIILeghO2H\nkth6KIncghLcmtnwxKgutHJr3MOKJUkIIRo1vV6xYW8Cm/ado7hEj62VGff3acOwwFZYN8D7Hm6X\n1IAQotG6klfM5+tOcOrcZRyaWDCmf2v69fDAykK+Gq+SmhBCNEp/J17ms7UnuJJXjK+3M4/c36lR\ndkzfiiQJIUSjUqLV88vuOH6NTkSDhnEhXgwNaNVoO6ZvRZKEEKLRSEzNYcmGkySl5eHc1IrHRnSm\nvaeDscOq0yRJCCEavBKtno17E9i49xw6vWKAb3PGh3hLx3QVSA0JIRq00+ez+PrXGC5m5ONoZ8nU\noT5093Iydlj1hiQJIUSDlH6lgLV/xLPnrxQ0wCA/T0YPaCeth9sktSWEaFCu5Baxenc8m6NKJ+Tz\ndLFlylAfvFs0NXZo9ZIkCSFEvafT6zkRn8nu4ykcOZOGVqdwcbAi/N52BHZ2w8RERi7dKUkSQoh6\nq6hYx/Y/k9hy8DxXcosBaO5sS/gAL3zbNcPMtHFM512TJEkIIeqdohIdO/5MZnP0OXLyS7C2NCPE\nrwX3dvOgjbsdrq72Mm16NZEkIYSoV1Iz8/lw1TFSMvOxtjRlZN82DOnVEhu5W7pG1GhbbP78+fTp\n04cRI0ZUeG7p0qX4+PiQlZVVkyEIIRqQUwmZvPHNQVIy8xnU05O3ZvYhvF87SRA1qEaTxOjRo1m6\ndGmF7SkpKURFRdG8efOaPLwQooEo0erYevA87/10lMJiHdPu8+Gh0A40sZbkUNNq9HKTv78/ycnJ\nFbYvWrSIuXPn8sQTT9Tk4YUQ9ZhSiuNxGUSfvMSRs2kUFOloYm3O06O70aGlTKVRW2q9T2L79u14\neHjQsWPH2j60EKIeWRUZy+boRACc7K0Y4NuCQX6eODW1MnJkjUutJonCwkI+++wzvvzyS8M2pVSV\ny9/tgt4NidRFGamLMg2lLk7EZfDr/kQ8nG2Z/aAfHVs53vYsrQ2lLoytVpNEYmIiycnJjBo1CqUU\nqampjBkzhpUrV+LkdOu5VGRIWykXFzupi/+RuijTUOqiqFjHe98fAgXThvrgZGNOenrube2jodRF\ndbjbZFnjSeLalkKHDh3Ys2eP4fHAgQNZs2YNTZvK7fJCiFKrdsZy6XIBQwNa4e0p3w3GVqOjm+bM\nmcOECROIj48nODiYn3/+udzzGo3mti43CSEatphzl9l2KAkPJxvC+7U1djiCGm5JvPvuuzd9ftu2\nbTV5eCFEPXL4TBpfbjyFRgOPDO+MhbmpsUMSyB3XQggj0+r0rIqMZcuB85ibmTD9vk60a25v7LDE\n/0iSEEIYTWJqDl//+jfxF7Nxb2bDE+FdaenaxNhhiWtIkhBC1LrcghLW7Ioj8kgySkFQFzcmh3XE\nykK+kuoa+USEELXm6uytG/cmkFeoxcPJhomD29O1rSwnWldJkhBC1LgSrY7IIxfYtPccV/KKsbY0\nY8JAbwb29JQ1H+o4SRJCiBqhV4qzSVfYdzKVA6dSySvUYmlhyv19WhMW0Apbmbm1XpAkIYSodhcz\n8vho1TFSLxcA0NTWgvt6tyYsoCV2NhZGjk7cDkkSQohqdTmniPd+PEJGdhG9u7jRt6sHnVo7yjrT\n9ZQkCSFEtckvLOG9n0oTRET/dozo08bYIYm7JD1GQohqUVSi46NVx0hOy2OQnyf3B7U2dkiiGkhL\nQghxx/ILtRw9m87hs+n8FZdBYbGOXj6uTBzc/ran9hZ1kyQJIcQdOZt8hU9+PkZ2fgkALg5WDOrp\nyci+baX/oQGRJCGEuG37TqTw5aYY9HrF/X1aE9jJjebOttJ6aIAkSQghqkwpxdrd8azbk4C1pSlP\nhHeTu6UbOEkSQogqUUqx/PfTbP8zGeemVjw7rgctnG2NHZaoYZIkhBC3pFeK5VtOs+NwMp4uTXhh\ngi/2tnJTXGNQo0li/vz5REZG4uTkxPr16wF4++232bFjBxYWFrRq1Yo333yTJk1kamAh6iq9Uny3\n5TSRh5Np6VqaIOSu6cZDo2pw/dCDBw9ia2vL3LlzDUkiKiqK3r17Y2Jiwn/+8x80Gg1z5syp0v5k\nYfNSssh7GamLMtVVF4XFWtbtTiD2whWy84rJzi+moEhHK9cmvDDxHppY1/05l+S8KOPiYndX5Wu0\nJeHv709ycnK5bX369DH87evry2+//VaTIQghbkNs8hW+WH+SS1kFaDRgZ22Ok70VLVya8FBoh3qR\nIET1MmqfxKpVqxg+fLgxQxBCAHq9Yn1UAuv3JKCUYlhgK8L7tcPcTCZlaOyMliT++9//Ym5uzogR\nI6pc5m6bTQ2J1EUZqYsyd1IX+YUlvPPdIQ6eSsXZwZrZE/3o5u1cA9HVLjkvqodRksSaNWvYuXMn\n33zzzW2Vk2uMpeR6axmpizJ3UheZ2YV8uOoY5y/l0qVtM2aO6oKtlXm9r1M5L8rU6T4JKB1bfa1d\nu3axZMkSvvvuOywsZISEEMZyLiWHD1cdJSu3mGDf5jwY2kFWiRMV1GiSmDNnDtHR0WRlZREcHMys\nWbNYvHgxJSUlTJ8+HYAePXrwz3/+sybDEEJc5+/Ey3y46hhFxTrGh3gTFtBSptQQlarRIbDVTZqP\npaQpXUbqokxV6+LI2XT++8tf6PWKx0Z0JqCTWy1EV7vkvChT5y83CSGMK7eghKycIrLzi0lMzWVV\nZCxmphqeGdudbu1k3iVxc5IkhGjA/jh6gW9++xudvuyCgY2lGc+O6057TwcjRibqC0kSQjRQx2Iz\n+PrXv7GxMqNXJ1fsbSywszGnWzsnXBysjR2eqCckSQjRACWkZPPfX/7C1FTDs2O749WiqbFDEvWU\njHcTooFJSsvlg5XHKC7RMWNEF0kQ4q5IS0KIBuDvxMus3h3PgRMppF4uAOCh0A707Ohi5MhEfSdJ\nQoh6rKhEx4/bzxJ5uHQiTUsLU+5p70xAJzcCOze8oa2i9kmSEKKeSrqUy2frTnAhPQ9PF1seH90d\nN3tLuWtXb7olAAAgAElEQVRaVCtJEkLUMyVaPb/tT2TdngS0Oj2D/DwZF+JFi+YOcgOZqHaSJISo\nR04mZPLdltOkZOZjb2PO1GFduKe99DuImiNJQoh6oLBYy/LfT7PneAoaYKBfC0b3b4eNlSwCJGqW\nJAkh6rjE1Bz++8tfpF4uoLWbHVOHdaSNu72xwxKNhCQJIeoopRTb/0zmx+1n0OoUQwNaMXpAO+mY\nFrVKkoQQdZBSip93xrFp3zmaWJvz6P2d6O5V/1eLE/WPJAkh6hi9UqzYeoath5Jwc7TmhQn34NTU\nythhiUbqlkni/PnzrFq1iujoaFJSUrC0tMTHx4ewsDCGDBmCmdmNdzF//nwiIyNxcnJi/fr1AFy5\ncoXnn3+e5ORkPD09+eCDD7Czk7VoReNUotVzObeIgkItCoVSsPNIMruOXqSFiy0vPOBL0yaWxg5T\nNGI3XXTo//7v/zhx4gRDhw7lnnvuwdnZmaKiImJjY9m9ezcnT57kn//8J76+vpWWP3jwILa2tsyd\nO9eQJN555x0cHBx47LHH+Pzzz8nOzuaFF16oUrAyBryULKhSpj7WRVpWAd/8GsP5S7lk55dU+prW\nbnbMmeBLE+uqj16qj3VRU6QuytTookODBg1iwYIFFbZ37NiR++67j6ysLM6fP3/D8v7+/iQnJ5fb\ntm3bNr777jsAIiIimDx5cpWThBD1XWzyFT76+Rg5+SW4OlrTwqUJjnaW2FiZYaLRoNGAjZU5g/xa\nyPBWUSfcNEkMGDDgpoUdHBxwcLi9hUsyMzNxdi7tgHNxceHy5cu3VV6I+upAzCWWbDiJTqeYPKQD\nIX6exg5JiFuq0li6f//73+Tk5KDVannwwQfx9fVl7dq1NR2bEPVadl4xO48k882vMfxr2YHS9R1M\nNDw7rrskCFFvVGl0U1RUFC+99BKRkZG4ubnx/vvvM2PGDEaNGnXbB3RyciI9PR1nZ2fS0tJo1qxZ\nlcve7bW1hkTqokxdrIviEh3zv9hHSkY+AGamJnRq04wnx/agjUfN3QhXF+vCWKQuqsdtDYE9cOAA\noaGhuLm5odFoqlTm+n7xgQMHsnr1ambMmMGaNWsYNGhQlY8vHVGlpFOuTF2ti1+jE0nJyCeoiztD\nerWkhYut4Sa4moq3rtaFMUhdlLnbZFmly01OTk688sorbNq0ib59+6LVatHpdLcsN2fOHCZMmEB8\nfDzBwcH8/PPPzJgxg6ioKMLCwti7dy8zZsy4qzcgRF2TW1DChqgEbK3MeDC0Pa3d7eQuaVFvVakl\n8e6777Ju3TrGjh1L06ZNSUpKYtq0aVUqV5lly5bdVpBC1CcbohLIL9LywEBvbGWEkqjnqpQkmjVr\nxsMPP2x47OnpiaendLwJcb20rAK2/5mEc1MrBkrntGgAbtoGfvLJJzl27Filz+Xm5vL111/z448/\n1khgQtRHa3bFodUpRvdvh7mZXGIS9d9NWxLPPPMM7777LgkJCXTv3h0nJyeKioqIi4sjOTmZCRMm\nMHHixNqKVYg6Sa9XHI1NZ+vBJE6du0xrdzsCZH1p0UDcNEn4+PjwxRdfcPHiRfbv309qaiqWlpYM\nHTqUnj17YmFhUVtxClEnnU26wpINJ7mUVQBAp9aOPBTaAZMqjv4Toq6rUp+Eh4fHHd0TIURDlltQ\nwqe/HCc7r4T+PTwY3LMlnq5NjB2WENWqShdNMzIyeOGFF3jooYcAiImJ4YcffqjRwISo677b8jdZ\nucWE92vLw8M6SYIQDVKVksQrr7xCz549yc7OBqBdu3Z8//33NRqYEHXZvpMp7D91Ce8WTRnWu5Wx\nwxGixlQpSaSmpjJx4kRMTU0BsLCwwMRERm6Ixikzu5DvfjuNpbkpj97fCVP5XxANWJXO7usXFsrO\nzq4w3YYQjUFqZj6frD5OfpGWCYO8cXW0MXZIQtSoKnVcDxkyhP/7v/8jLy+P1atX8/333zNmzJia\njk2IOkOr0/Pb/kTW7k5Aq9PTp6s7/Xs0N3ZYQtS4KiWJRx99lHXr1pGdnc3OnTuZPHmyjHYSjYJS\nimOxGfy8M46ktFzsbS14KLQD/h1dqjzJpRD1WZVngR05ciQjR46syViEqDZanf6uJtXT6vT8FZfJ\nuj3xJKSUziZ6b3cPmY9JNDpVShIZGRl89913JCYmotVqDds//PDDGgtMiDuVmJrDv5Yd4KmIbvh1\ncKlSGaUUv/wRz7HYDC7nFBrWntYAvXxcGdG3DZ4uMsRVND5VShJPPvkknTt3JigoyDDCSYi66kzS\nFZSCY7EZVU4S6/YksD4qAXMzE5rZWdLc2Ra3ZjYM6ukpyUE0alVKEgUFBbz22ms1HYsQ1eLS5dIp\nMs6lVm3RmX0nU1i7Ox7npla8MsUfe1uZbkaIq6qUJHr06MHff/9Nx44dazoeIe5a2v/mUUpOy63Q\nN/H7gfOsj0qgh7cTfbt6YGZmwpcbY7C2NOXZsd0lQQhxnSoliQkTJjBp0iTc3d2xtLQ0bF+1atUd\nH3jZsmWsWrUKjUZDhw4dePPNN2XCQFEtrk62p9UpLqTn0cqtbPnGvSdSyC0oYc/xFPYcTwFAo4Fn\nRvWghVxWEqKCKiWJF198kZkzZ9K5c+dq6ZNITU3l22+/ZfPmzVhYWPDcc8+xadMmwsPD73rfonHT\nK2VoSUDpJaerSaKwWEtiai5eze0ZG+zF7uMXOR6bQXj/dnRt52SskIWo06qUJCwtLXnkkUeq9cB6\nvZ6CggJMTEwoLCzE1dW1WvcvGqcrucWUaPU42VuRkV1IYmqu4bnYC9nolaJDSwc6tnKkYytHI0Yq\nRP1QpYHk/fr1Y9euXdV2UDc3N6ZNm0ZwcDD9+/fHzs6OPn36VNv+ReN16XI+APd0cMZEoynXeX3m\nfBYA7T0djBKbEPVRlVoSP/30E59//jm2trZYWFiglEKj0bB37947Omh2djbbtm1jx44d2NnZ8cwz\nz7B+/XpGjBhxR/sT4qqr/RGeLk3wcLLhfGoueqUw0Wg4k3QFAG/PpsYMUYh6pUpJ4ueff67Wg0ZF\nRdGyZUscHEp/0YWGhnL48OFbJgkXF7ubPt+YSF2UubYu8or1ALRv04xzl3JJPpRECRrcm9kSdzGb\nVu52tG3VzFih1jg5L8pIXVSPKiWJFi1aVOtBmzdvztGjRykqKsLCwoJ9+/bRrVu3W5ZLS6vauPeG\nzsXFTurif66vi4Tk0ktKFoCbgzUAh0+l4OZoQ1GxjnYe9g227uS8KCN1UeZuk+VNk8SLL77IO++8\nw5gxYyqdzOxOh8B2796dsLAwwsPDMTMzo3PnzowfP/6O9iXEtdKyCjAz1eBoZ0lrt9IhrYmpuVzJ\nLQagvVxqEuK23DRJXLp0CYB//OMf1X7gp59+mqeffrra9ysat7SsQpyaWmNioqGla+kvqHMpOVhb\nlp7qHaTTWojbctMkcXW50oCAgFoJRoi7kV+oJbeghLYe9gDYWJnh6mBNYmoOJiYamtlb4tTUyshR\nClG/yLqLosG4ehOd6//6IgBauduRV6glJ79EWhFC3IGbtiROnz5NUFBQhe13OwRWiJpwdfiri2NZ\nkmjt1oSDMaWXTaU/Qojbd9Mk0aZNGz7//PPaikWI25KTX4xdcdn6JldvpLu2JdH6mnmb2reUloQQ\nt+umScLCwqLah78KUR2KS3S8siQab08HZo0uHT6dVklL4uq8TbZWZjR3tq39QIWo526aJMzNZZlG\nUTfFJGaRk1/C4dNpnEvJobW7nWEdCZdrOqftbS3o29UdFwdrTGRNaiFu2007rn/66afaikOI23Is\nNt3w9+8HzwOlLQlHO0sszMvPVPzI/Z0ZeW/bWo1PiIZCRjeJekcpxbHYDKwtzWjh0oTok6mkXykg\nM7sIl2v6I4QQd0+ShKh3Lmbkk36lkC5tmzGqfzt0esWqyFgU4OIg90EIUZ0kSYh651hsBgA9vJwI\n6dkSWysz9p8qHebqKi0JIaqVJAlR71ztj+jazgkrSzMG+JaNwLt2ZJMQ4u5JkhD1Sn6hljNJV2jr\nYUdT29I10Qf6tcDUpHTkkquDjTHDE6LBkSQh6pWTCZno9Ipu16xJ3czeij5d3bG2NMPDSZKEENWp\nSutJCFFXGPojvJ3LbZ8ytCMPDGxvmO1VCFE95D9K1Bt6pTgel4G9jTmt3csvpGJqYoKNlTSMhahu\n8l8l6o0T8ZlcySumWzsnuXtaiFpitJZETk4OL7/8MmfOnMHExIRFixbRo0cPY4Uj6jCtTs+GqAQ2\nRJ1DAwR1dTd2SEI0GkZLEgsXLmTAgAF89NFHaLVaCgsLjRWKqMMupOexdONJ4i/m4GRvySPDO+PT\n2tHYYQnRaBglSeTm5nLw4EH+/e9/lwZhZkaTJk2MEYqoo/ILtazbE8+2Q0no9IqgLu48FNoBGyvp\nRhOiNhnlPy4pKQlHR0fmzZtHTEwMXbt25eWXX8bKSqZUaOwKirTsO5nK2j/iyM4vwbmpFRMHteee\nDi7GDk2IRkmjlFK1fdC//vqLBx54gBUrVtCtWzcWLlyInZ0dzzzzTG2HIuoApRSH/05j+8Hz7P3r\nIsUlOqwsTBk3qAPhA7wqzOoqhKg9RmlJuLu74+7uTrdupYvFhIWFsWTJkluWS0vLqenQ6gUXF7sG\nUxdZuUV8ufEUf8VnAuDmaE1QV3f6dW+Oo50lV7Lyb1q+IdXF3ZK6KCN1UcbFxe7WL7oJoyQJZ2dn\nPDw8iI+Pp23btuzbtw8vLy9jhCKM6PDpNL7aHENuQQld2zVjVN+2tGtuj0aGtwpRZxitF/CVV17h\nhRdeQKvV0rJlS958801jhSJqWVGxjhXbz7DzyAXMzUx4KLQDA/1aSHIQog4yWpLw8fHh559/Ntbh\nhZEkpGSzeN1JUjPz8XRpwuMjO9PCRUa2CVFXyXhCUSu0Oj2/7U/klz/i0ekVYQEtGd3fC3Mzuelf\niLpMkoSoUVeXGv1x+1lSMvNp2sSCR4d3pkvbZsYOTQhRBZIkRI1JTM1hVWQsf8VnotFAiF8LIvq1\no4m1ubFDE0JUkSQJUe0SU3NYtyeBP0+nAdCljSMPDGqPp/Q9CFHvSJIQ1SY9q4CVkbEciCldb9qr\nuT2j7m1Ll7bNZOSSEPWUJAlx1wqKtGzad47f9p9Hq9PT1sOO8H7t6CrJQYh6T5KEuGNKKfaeSGFl\nZCxXcotxtLNkbLAXgZ3dZL0HIRoISRLijpxLyWH576c5m3wFczMTRvZtw7DA1lhayDxLQjQkkiTE\nbSko0rJmVxzbDiWhgJ4dXHhgoDfODtbGDk0IUQMkSYgqOxabwbe/xZCRXYRbMxsmDelAlzZyv4MQ\nDZkkCXFLl3OK+GnHWaJPpmJqouH+Pm0Y0ac15mZyaUmIhk6ShLghrU7P1oNJrN0TT1GxjrYe9kwb\n5oOnq9zvIERjIUlCVKCU4sjZdH7eGceF9DyaWJszYag3/Xo0l1FLQjQykiREOafOXWb1zlhiL2Sj\n0cAA3+aMGeAlU2kI0UhJkhBA6Qpxy7ec5tD/ptLo2cGF8P7taOFsa+TIhBDGJEmikVNK8cexi/y4\n/SwFRVraezZlwqD2tPWwN3ZoQog6wKhJQq/XM2bMGNzc3Pjss8+MGUqjdC4lhxXbzvD3+SysLEyZ\nHNaRAb7S7yCEKGPUJPHNN9/g5eVFbm6uMcNodDKzC1mzK46ov1JQgK+3M5OGdKCZvZWxQxNC1DFG\nSxIpKSns3LmTmTNn8tVXXxkrjEalqETHb9GJbNp3jmKtHk+XJjwwyFtuiBNC3JDRksSiRYuYO3cu\nOTk5xgqh0VBKcSDmEj/tOEtmdhFNbS14KLQdfbt5YGIil5aEEDdmlCQRGRmJs7MznTp1Ijo6usrl\nXFzsajCq+qUqdaGUYv+JFFZsPc3Z81mYmZowdmB7xg1qj41VwxnSKudFGamLMlIX1UOjlFK1fdD3\n3nuPdevWYWpqSlFREXl5eYSGhvL222/ftFxamrQ6oPTkv1VdHDmTzpo/4jh/KRcN4O/jypgB7XB1\ntKmdIGtJVeqisZC6KCN1UeZuk6VRksS19u/fz5dfflml0U3yoZe62T/A5Zwilv9+mj9Pp6HRQGAn\nN4b3adNg73eQL4MyUhdlpC7K3G2SkPskGgi9Uuw8coFVkWcpKNLRoaUDU8I60ryBJgchRO0wepII\nCAggICDA2GHUa38nXuaHbWdITM3F2tKMqUM7yjxLQohqYfQkIe7cpawCVu44y6G/S6fSCOrixthg\nbxztLI0cmRCioZAkUQ/lF5awKjKWLQcS0eoUXi3smTioA+2ay1QaQojqJUmiHlFKEfVXCqt3xXE5\npwhHO0vGhXgR2MkNjVxaEkLUAEkS9URyeh7f/vY3p89nYWFuyqh72zI0sBWW5rI6nBCi5kiSqOOK\ninWsj0rgt/2J6PSKe9o78/QD96DR6owdmhCiEZAkUUcppYg+mcrKyFgu5xThZG/FQ6Ed8G3vjIuj\njYwBF0LUCkkSddC5lByW/36as8lXMDM14f4+bRjeuzWWFnJpSQhRuyRJ1CG5BSWs3hXHzsPJKKBn\nRxceCPHG2cHa2KEJIRopSRJ1gFanZ/exi6zeFUduQQkeTjY8FNqBzjKFtxDCyCRJGJFWpyfqrxQ2\nRCWQfqUQSwtTxod4M9jfEzNTE2OHJ4QQkiSM5fCZNH7Yeob0K4WYmZowuKcn9wW1xqGJ3C0thKg7\nJEnUsit5xXz/+2kOxFzC1ETDoJ6e3Ne7tUylIYSokyRJ1BK9XvHHsQusiowlr1CLVwt7Hh7WqcFO\n4S2EaBgkSdSC0+ez+H7raRJTc7G0MOWh0A6E+LWQWVqFEHWeJIkalJZVwM87Y9l/6hIAQV3cGRvs\nJZeWhBD1hiSJGpBXWMLGqHNsPXQerU7R1sOeBwe3x6tFU2OHJoQQt8UoSSIlJYW5c+eSnp6Oqakp\n48aNY8qUKcYIpVoppfjj2EVW7jhLXqEWJ3tLxgzwIqCzm1xaEkLUS0ZJEqampsybN49OnTqRl5fH\n6NGj6du3L15eXsYIp1qkXs7n680xxCRmYWVhyrhgLwb7e2JuJlNpCCHqL6MkCRcXF1xcXACwtbXF\ny8uLS5cu1cskUVCk5fcD59m47xwlWj2+3s5MGtKBZvZWxg5NCCHumtH7JJKSkoiJiaF79+7GDuW2\nlGh17Dh8gY17E8jJL8He1oJH7++Af0cXWQBICNFgaJRSylgHz8vLY/LkyTz55JMMHjzYWGHclqIS\nHb9Hn+PnHWdJzyrA2tKMiGBvRvVvh42VubHDE0KIamW0JKHVann88cfp378/U6dOrVIZY66hUFyi\nY/ufyfy6P5HsvGIszEwI8WvBfb1bY2djUauxuLjYyXoS/yN1UUbqoozURRkXF7u7Km+0y03z58/H\n29u7ygnCmOIvZrNkw0kuZuRjZWHK8KDWhPq3xN62dpODEELUNqMkiUOHDrF+/Xo6dOhAeHg4Go2G\n559/nv79+xsjnBvS6vSs25PApr3n0CvF4J6ejOrXFlu5rCSEaCSMkiR69uzJqVOnjHHoKjubdIVv\nfoshKS0PJ3srpg/vRKfWjsYOSwghapXRRzfVNbkFJayKPMuuoxcB6N+jOQ8M9MbaUqpKCNH4yDff\nNf48ncayzTHkFpTg6WLL5LCOtPd0MHZYQghhNJIkKL0h7oetZ9h9/CLmZiaMC/Ei1L+lrA4nhGj0\nGn2SOJmQybLNMaRfKaSVWxMeG9FF1ngQQoj/abRJIuNKIT9uP8PBv9PQAMODWjPq3rbSehBCiGs0\nuiRRotXz6/5ENkYlUKzV49XCnkmhHWntfnc3nAghREPUqJLEsdh0vt96hkuXC7C3tWBymBdBXd1l\nGm8hhLiBRpEk0rIKWLHtDIfPpGOi0RDq35JR97bFxqpRvH0hhLhjDfpbsqhEx+Z959i0LxGtTk+H\nlg5MCu2Ap2sTY4cmhBD1QoNMEkopDv2dxo/bz5KRXYhDEwvGD/QmsJObTOMthBC3ocEliYSUbFZs\nPcPppCuYmmgYFtiK+/u0kTumhRDiDjSYb84recX8HBnLnuMXUcA97Z0ZH+KNWzMbY4cmhBD1Vr1P\nElqdnu2Hkli7J56CIh2eLrZMGNSezm2aGTs0IYSo9+p1kvg78TLfbjnNhfQ8bK3MmDSkAwN8m2Nq\nIjfECSFEdaiXSSK3oISfdpxl97GLaIBg3+ZE9G9X6yvECSFEQ2e0JLFr1y4WLVqEUooxY8YwY8aM\nW5ZRShF9MpXvt54ht6CElq5NmDrUh3bN7WshYiGEaHyMkiT0ej2vv/46y5Ytw9XVlbFjxzJo0CC8\nvLxuWCbjSgEfrTrG0dgMLMxNGB/iTWgvT7m0JIQQNcgoSeLYsWO0bt2aFi1aADB8+HC2bdt20yTx\n1NvbySvU0qm1I1OH+eDqYF1b4QohRKNllCSRmpqKh4eH4bGbmxvHjx+/aRm9gqlDO9K/R3O5IU4I\nIWqJUZKEUuq2yyx9JZTCvKIaiEYIIcSNGCVJuLu7c+HCBcPj1NRUXF1db1rGzsZCRi9dw8VFpja/\nSuqijNRFGamL6mGUXt9u3bqRmJhIcnIyxcXFbNy4kUGDBhkjFCGEEDdhlJaEqakpr776KtOnT0cp\nxdixY2/aaS2EEMI4NOpOOgiEEEI0CnKTgRBCiBuSJCGEEOKGJEkIIYS4oTqfJHbt2sXQoUMJCwvj\n888/N3Y4tSolJYUpU6Zw3333MWLECL755hsArly5wvTp0wkLC+ORRx4hJyfHyJHWHr1eT0REBDNn\nzgQgKSmJ8ePHExYWxuzZs9FqtUaOsHbk5OTwzDPPMGzYMIYPH87Ro0cb7XmxbNky7r//fkaMGMGc\nOXMoLi5uNOfF/Pnz6dOnDyNGjDBsu9l58MYbbzBkyBBGjRrFqVOnqnSMOp0krs7xtHTpUjZs2MDG\njRuJjY01dli1xtTUlHnz5rFp0yZWrFjB8uXLiY2N5fPPPycoKIjffvuNwMBAFi9ebOxQa80333xT\nbiTcf/7zH6ZNm8Zvv/2GnZ0dq1atMmJ0tWfhwoUMGDCAzZs3s3btWtq1a9coz4vU1FS+/fZbVq9e\nzfr169HpdGzcuLHRnBejR49m6dKl5bbd6DzYuXMniYmJbNmyhQULFvDaa69V6Rh1OklcO8eTubm5\nYY6nxsLFxYVOnToBYGtri5eXF6mpqWzbto2IiAgAIiIi2Lp1qzHDrDUpKSns3LmTcePGGbbt27eP\nsLAwoLQufv/9d2OFV2tyc3M5ePAgY8aMAcDMzAw7O7tGe17o9XoKCgrQarUUFhbi6upKdHR0ozgv\n/P39sbcvPwv29efB1e/Mbdu2ER4eDkCPHj3IyckhPT39lseo00misjmeLl26ZMSIjCcpKYmYmBh6\n9OhBRkYGzs7OQGkiuXz5spGjqx2LFi1i7ty5hrm7Ll++TNOmTTH530zA7u7ujeL8SEpKwtHRkXnz\n5hEREcGrr75KQUFBozwv3NzcmDZtGsHBwfTv3x87Ozs6d+6Mvb19ozsvrsrMzCx3HmRmZgJw6dIl\n3N3dDa9zc3MjNTX1lvur00lCbuEolZeXxzPPPMP8+fOxtbVtlBMcRkZG4uzsTKdOnQznhVKqwjnS\nGOpGq9Vy8uRJHnzwQdasWYO1tTWff/55o3jv18vOzmbbtm3s2LGDP/74g4KCAnbt2lXhdY2xbq5X\n2fdpVeqlTq9MdydzPDU0Wq2WZ555hlGjRjF48GAAnJycSE9Px9nZmbS0NJo1a/jref/5559s376d\nnTt3UlRURF5eHosWLSInJwe9Xo+JiQkpKSmN4vxwd3fH3d2dbt26ATBkyBC++OKLRnleREVF0bJl\nSxwcHAAYPHgwhw8fJjs7u9GdF1fd6Dxwc3MjJSXF8Lqq1kudbknIHE+loxe8vb2ZOnWqYdvAgQNZ\nvXo1AGvWrGkUdTJ79mwiIyPZtm0b7733HoGBgfznP/8hMDCQX3/9FWg8deHs7IyHhwfx8fFAab+M\nt7d3ozwvmjdvztGjRykqKkIpxb59+2jfvn2jOi+ubyHc6DwYNGgQv/zyCwBHjhzB3t7ecFnqZur8\ntBy7du1i4cKFhjmeqrLMaUNx6NAhJk2aRIcOHdBoNGg0Gp5//nm6d+/Oc889x8WLF2nevDkffvhh\nhc6rhmz//v18+eWXfPbZZ5w/f57Zs2eTnZ1Np06deOeddzA3Nzd2iDUuJiaGl19+Ga1WS8uWLXnz\nzTfR6XSN8rz45JNP2LhxI2ZmZnTu3Jk33niDlJSURnFezJkzh+joaLKysnB2dmbWrFkMHjyYZ599\nttLzYMGCBfzxxx9YW1vz5ptv0qVLl1seo84nCSGEEMZTpy83CSGEMC5JEkIIIW5IkoQQQogbkiQh\nhBDihiRJCCGEuCFJEkIIIW5IkoSocwYOHMjZs2dr5ViffPJJuWmk582bx/Lly+96v/PmzWPEiBHM\nnj37rvd1MzExMWzevLlGjyEaN0kSolH75JNPKCkpqdZ9pqens2XLFtavX897771Xrfu+3smTJ+84\nSej1+mqORjREkiREvREfH89jjz3GuHHjCA8PN0w9AODj48PixYsZO3YsoaGhbNmyxfDcb7/9xrBh\nwxg9ejSLFy/Gx8eHgoICFixYgEajYcKECURERJCbmwvA6dOnmTp1KmFhYbz00ks3jOeXX35hxIgR\njBo1ilmzZpGZmUleXh5Tp06lqKiIiIgIvv7663Jl1q5dy9NPP214rNPp6Nevn2GOsiVLljB+/HhG\njx7NE088QUZGBgAlJSW89dZbjBgxgvDwcGbNmkVWVhYff/wx+/btIyIigoULFwKlsxREREQwatQo\npk2bxvnz54HSO9XDw8N54403mDBhAn/88cfdfByisVBC1DEhISHqzJkz5bZptVoVERGh4uLilFJK\n5ebmqrCwMMPjjh07quXLlyullDp06JDq16+fUkqp9PR0FRAQoBITE5VSSn311VfKx8dH5efnG8oV\nFGMkQHAAAAOxSURBVBQYjvPSSy+pBx98UBUXF6vi4mI1fPhwFRUVVSHG06dPq3vvvVelp6crpZT6\n4IMP1HPPPaeUUiopKUn17t270vdWUFCgevfurS5fvqyUUmr79u1q6tSpSiml1q5dq1599VXDa7//\n/ns1Z84cpZRSH3/8sZo1a5bSarVKKWUov3r1avXMM88YymRkZKjevXur2NhYpZRSK1euVOPGjVNK\nKRUdHa06d+6sjh49WmlsQlRGWhKiXkhISCAuLo7Zs2cTHh7OQw89RElJSbmVCu+77z4AfH19SUtL\no7i4mKNHj9K1a1datmwJwNixYyvsW103M83gwYMxNzfH3Nyczp07k5iYWKFMdHQ0wcHBODk5ATBh\nwgSioqJu+T6srKwYNGgQGzZsAEonYLu6eND27dvZu3cv4eHhhIeH8/3333Px4kWgdKr0KVOmYGpq\nCmCY9fR6R48epVOnTrRr1w6AMWPGcOrUKfLz8wFo3bo13bt3v2WcQlxVp6cKF+IqpRTNmjVjzZo1\nlT6v0WiwtLQEMCw2o9PpKiSA6x9XxsLCwvC3qalppesjK6UqzMV/9bi3Eh4ezptvvsn999/P/v37\neeeddwz7fOKJJxg9enSlx6uKyuK69rGNjU2V9iPEVdKSEPVC27ZtsbKyYu3atYZtcXFx5OXlARW/\nRK8+9vX15cSJE4br8tf2YwA0adKk3ELxVRUUFMTOnTsNfQY//vgjffr0qXD8yvj7+5Obm8t7771H\naGioIbkNHDiQ77//nuzsbACKi4uJiYkBICQkhG+++cbQyX511bkmTZoY+lKuvt9Tp04ZphFfvXo1\nnTt3luQg7pi0JESdo9FoePjhhzEzMzP8Ml6/fj2fffYZCxcu5Msvv0Sn0+Hs7MwHH3xgKHP9PqB0\nAZZ//etfzJgxA0dHR4KDgzEzM8Pa2hqAadOmMWXKFKytrfn222+rHKO3tzezZ8/m4YcfxsTEhJYt\nW7JgwYIKx7+R8PBwPvroo/9v5w5xGASiIAwPBoMhHADNBRCcgtUEzQWQSByChAOgSHB4joVBLqlo\ngnumadK0/T/51LrZyeat1nW9Z2VZ6jgO1XWtIAh0XZeqqlKWZWqaRuM4yjmnMAyVpqmmaVJRFJrn\nWc455Xmurus0DIPatpX3XkmS3E0FeAVfhePnneepKIokPW/W27a9ZRcC+Ac0Cfy8ZVm077u894rj\nWH3ff/pIwNegSQAATDxcAwBMhAQAwERIAABMhAQAwERIAABMhAQAwPQAVSnSA55bZkwAAAAASUVO\nRK5CYII=\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f47b8e3bd90\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(eager_means)\n", - "plt.ylabel('Time(s)')\n", - "plt.xlabel('Length of vector')\n", - "_ = plt.title('Time to sum the elements of 1000 vectors (Eager)')\n", - "_ = plt.ylim(ymin=0)" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "default_view": {}, - "name": "Autograph vs. Eager vs Graph sum", - "provenance": [ - { - "file_id": "1olZkm32B7n7pQwlIAXR0_w8fZhRHCtkX", - "timestamp": 1531755808890 - } - ], - "version": "0.3.2", - "views": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb b/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb deleted file mode 100644 index 7cb2c0dff58..00000000000 --- a/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb +++ /dev/null @@ -1,1055 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "LqNpENf-ec0X", - "slideshow": { - "slide_type": "slide" - } - }, - "outputs": [], - "source": [ - "!pip install -U tf-nightly" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "Pa2qpEmoVOGe", - "slideshow": { - "slide_type": "-" - } - }, - "outputs": [], - "source": [ - "import os\n", - "import time\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow.contrib import autograph\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import six\n", - "\n", - "from google.colab import widgets" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HNqUFL4deCsL", - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "# Case study: training a custom RNN, using Keras and Estimators\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YkC1k4HEQ7rw", - "slideshow": { - "slide_type": "-" - } - }, - "source": [ - "In this section, we show how you can use AutoGraph to build RNNColorbot, an RNN that takes as input names of colors and predicts their corresponding RGB tuples. The model will be trained by a [custom Estimator](https://www.tensorflow.org/get_started/custom_estimators)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7nkPDl5CTCNb", - "slideshow": { - "slide_type": "-" - } - }, - "source": [ - "To get started, set up the dataset. The following cells defines methods that download and format the data needed for RNNColorbot; the details aren't important (read them in the privacy of your own home if you so wish), but make sure to run the cells before proceeding." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "A0uREmVXCQEw", - "slideshow": { - "slide_type": "-" - } - }, - "outputs": [], - "source": [ - "def parse(line):\n", - " \"\"\"Parses a line from the colors dataset.\"\"\"\n", - " items = tf.string_split([line], \",\").values\n", - " rgb = tf.string_to_number(items[1:], out_type=tf.float32) / 255.0\n", - " color_name = items[0]\n", - " chars = tf.one_hot(tf.decode_raw(color_name, tf.uint8), depth=256)\n", - " length = tf.cast(tf.shape(chars)[0], dtype=tf.int64)\n", - " return rgb, chars, length\n", - "\n", - "\n", - "def set_static_batch_shape(batch_size):\n", - " def apply(rgb, chars, length):\n", - " rgb.set_shape((batch_size, None))\n", - " chars.set_shape((batch_size, None, 256))\n", - " length.set_shape((batch_size,))\n", - " return rgb, chars, length\n", - " return apply\n", - "\n", - "\n", - "def load_dataset(data_dir, url, batch_size, training=True):\n", - " \"\"\"Loads the colors data at path into a tf.PaddedDataset.\"\"\"\n", - " path = tf.keras.utils.get_file(os.path.basename(url), url, cache_dir=data_dir)\n", - " dataset = tf.data.TextLineDataset(path)\n", - " dataset = dataset.skip(1)\n", - " dataset = dataset.map(parse)\n", - " dataset = dataset.cache()\n", - " dataset = dataset.repeat()\n", - " if training:\n", - " dataset = dataset.shuffle(buffer_size=3000)\n", - " dataset = dataset.padded_batch(\n", - " batch_size, padded_shapes=((None,), (None, 256), ()))\n", - " # To simplify the model code, we statically set as many of the shapes that we\n", - " # know.\n", - " dataset = dataset.map(set_static_batch_shape(batch_size))\n", - " return dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "waZ89t3DTUla", - "slideshow": { - "slide_type": "-" - } - }, - "source": [ - "To show the use of control flow, we write the RNN loop by hand, rather than using a pre-built RNN model.\n", - "\n", - "Note how we write the model code in Eager style, with regular `if` and `while` statements. Then, we annotate the functions with `@autograph.convert` to have them automatically compiled to run in graph mode.\n", - "We use Keras to define the model, and we will train it using Estimators." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "9v8AJouiC44V", - "slideshow": { - "slide_type": "slide" - } - }, - "outputs": [], - "source": [ - "@autograph.convert()\n", - "class RnnColorbot(tf.keras.Model):\n", - " \"\"\"RNN Colorbot model.\"\"\"\n", - "\n", - " def __init__(self):\n", - " super(RnnColorbot, self).__init__()\n", - " self.lower_cell = tf.contrib.rnn.LSTMBlockCell(256, dtype=tf.float32)\n", - " self.upper_cell = tf.contrib.rnn.LSTMBlockCell(128, dtype=tf.float32)\n", - " self.relu_layer = tf.layers.Dense(3, activation=tf.nn.relu)\n", - "\n", - " def _rnn_layer(self, chars, cell, batch_size, training):\n", - " \"\"\"A single RNN layer.\n", - "\n", - " Args:\n", - " chars: A Tensor of shape (max_sequence_length, batch_size, input_size)\n", - " cell: An object of type tf.contrib.rnn.LSTMBlockCell\n", - " batch_size: Int, the batch size to use\n", - " training: Boolean, whether the layer is used for training\n", - "\n", - " Returns:\n", - " A Tensor of shape (max_sequence_length, batch_size, output_size).\n", - " \"\"\"\n", - " hidden_outputs = tf.TensorArray(tf.float32, 0, True)\n", - " state, output = cell.zero_state(batch_size, tf.float32)\n", - " for ch in chars:\n", - " cell_output, (state, output) = cell.call(ch, (state, output))\n", - " hidden_outputs.append(cell_output)\n", - " hidden_outputs = autograph.stack(hidden_outputs)\n", - " if training:\n", - " hidden_outputs = tf.nn.dropout(hidden_outputs, 0.5)\n", - " return hidden_outputs\n", - "\n", - " def build(self, _):\n", - " \"\"\"Creates the model variables. See keras.Model.build().\"\"\"\n", - " self.lower_cell.build(tf.TensorShape((None, 256)))\n", - " self.upper_cell.build(tf.TensorShape((None, 256)))\n", - " self.relu_layer.build(tf.TensorShape((None, 128))) \n", - " self.built = True\n", - "\n", - "\n", - " def call(self, inputs, training=False):\n", - " \"\"\"The RNN model code. Uses Eager.\n", - "\n", - " The model consists of two RNN layers (made by lower_cell and upper_cell),\n", - " followed by a fully connected layer with ReLU activation.\n", - "\n", - " Args:\n", - " inputs: A tuple (chars, length)\n", - " training: Boolean, whether the layer is used for training\n", - "\n", - " Returns:\n", - " A Tensor of shape (batch_size, 3) - the model predictions.\n", - " \"\"\"\n", - " chars, length = inputs\n", - " batch_size = chars.shape[0]\n", - " seq = tf.transpose(chars, (1, 0, 2))\n", - "\n", - " seq = self._rnn_layer(seq, self.lower_cell, batch_size, training)\n", - " seq = self._rnn_layer(seq, self.upper_cell, batch_size, training)\n", - "\n", - " # Grab just the end-of-sequence from each output.\n", - " indices = (length - 1, list(range(batch_size)))\n", - " indices = tf.stack(indices, 1)\n", - " sequence_ends = tf.gather_nd(seq, indices)\n", - " return self.relu_layer(sequence_ends)\n", - "\n", - "@autograph.convert()\n", - "def loss_fn(labels, predictions):\n", - " return tf.reduce_mean((predictions - labels) ** 2)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JjK4gXFvFsf4", - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "We will now create the model function for the custom Estimator.\n", - "\n", - "In the model function, we simply use the model class we defined above - that's it!" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "-yso_Nx23Gy1", - "slideshow": { - "slide_type": "-" - } - }, - "outputs": [], - "source": [ - "def model_fn(features, labels, mode, params):\n", - " \"\"\"Estimator model function.\"\"\"\n", - " chars = features['chars']\n", - " sequence_length = features['sequence_length']\n", - " inputs = (chars, sequence_length)\n", - "\n", - " # Create the model. Simply using the AutoGraph-ed class just works!\n", - " colorbot = RnnColorbot()\n", - " colorbot.build(None)\n", - "\n", - " if mode == tf.estimator.ModeKeys.TRAIN:\n", - " predictions = colorbot(inputs, training=True)\n", - " loss = loss_fn(labels, predictions)\n", - "\n", - " learning_rate = params['learning_rate']\n", - " optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", - " global_step = tf.train.get_global_step()\n", - " train_op = optimizer.minimize(loss, global_step=global_step)\n", - " return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)\n", - "\n", - " elif mode == tf.estimator.ModeKeys.EVAL:\n", - " predictions = colorbot(inputs)\n", - " loss = loss_fn(labels, predictions)\n", - "\n", - " return tf.estimator.EstimatorSpec(mode, loss=loss)\n", - "\n", - " elif mode == tf.estimator.ModeKeys.PREDICT:\n", - " predictions = colorbot(inputs)\n", - "\n", - " predictions = tf.minimum(predictions, 1.0)\n", - " return tf.estimator.EstimatorSpec(mode, predictions=predictions)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HOQfoBnHC9CP", - "slideshow": { - "slide_type": "-" - } - }, - "source": [ - "We'll create an input function that will feed our training and eval data." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "FJZlx7yG2MP0", - "slideshow": { - "slide_type": "slide" - } - }, - "outputs": [], - "source": [ - "def input_fn(data_dir, data_url, params, training=True):\n", - " \"\"\"An input function for training\"\"\"\n", - " batch_size = params['batch_size']\n", - " \n", - " # load_dataset defined above\n", - " dataset = load_dataset(data_dir, data_url, batch_size, training=training)\n", - "\n", - " # Package the pipeline end in a format suitable for the estimator.\n", - " labels, chars, sequence_length = dataset.make_one_shot_iterator().get_next()\n", - " features = {\n", - " 'chars': chars,\n", - " 'sequence_length': sequence_length\n", - " }\n", - "\n", - " return features, labels" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qsvv-lzbDqXd", - "slideshow": { - "slide_type": "-" - } - }, - "source": [ - "We now have everything in place to build our custom estimator and use it for training and eval!" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 35 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 5454, - "status": "ok", - "timestamp": 1529952160455, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "2pg1AfbxBJQq", - "outputId": "4aef3052-f7c7-4bb1-a0a2-73fef2e96efb", - "slideshow": { - "slide_type": "-" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Eval loss at step 100: 0.0705221\n" - ] - } - ], - "source": [ - "params = {\n", - " 'batch_size': 64,\n", - " 'learning_rate': 0.01,\n", - "}\n", - "\n", - "train_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/archive/extras/colorbot/data/train.csv\"\n", - "test_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/archive/extras/colorbot/data/test.csv\"\n", - "data_dir = \"tmp/rnn/data\"\n", - "\n", - "regressor = tf.estimator.Estimator(\n", - " model_fn=model_fn,\n", - " params=params)\n", - "\n", - "regressor.train(\n", - " input_fn=lambda: input_fn(data_dir, train_url, params),\n", - " steps=100)\n", - "eval_results = regressor.evaluate(\n", - " input_fn=lambda: input_fn(data_dir, test_url, params, training=False),\n", - " steps=2\n", - ")\n", - "\n", - "print('Eval loss at step %d: %s' % (eval_results['global_step'], eval_results['loss']))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zG1YAjB_cUnQ", - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "And here's the same estimator used for inference." - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 343 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 3432, - "status": "ok", - "timestamp": 1529952163923, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "dxHex2tUN_10", - "outputId": "1ff438f2-b045-4f4e-86a0-4dae7503f6b2", - "slideshow": { - "slide_type": "slide" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "\u003clink rel=stylesheet type=text/css href='/nbextensions/google.colab/tabbar.css'\u003e\u003c/link\u003e" - ], - "text/plain": [ - "\u003cIPython.core.display.HTML at 0x7fcd7222a110\u003e" - ] - }, - "metadata": { - "tags": [ - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\u003cscript src='/nbextensions/google.colab/tabbar_main.min.js'\u003e\u003c/script\u003e" - ], - "text/plain": [ - "\u003cIPython.core.display.HTML at 0x7fcd7222a8d0\u003e" - ] - }, - "metadata": { - "tags": [ - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\u003cdiv id=\"id3\"\u003e\u003c/div\u003e" - ], - "text/plain": [ - "\u003cIPython.core.display.HTML at 0x7fcd7222a050\u003e" - ] - }, - "metadata": { - "tags": [ - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8a03307e-78a7-11e8-99f9-c8d3ffb5fbe0\"] = colab_lib.createTabBar({\"contentBorder\": [\"0px\"], \"elementId\": \"id3\", \"contentHeight\": [\"initial\"], \"tabNames\": [\"RNN Colorbot\"], \"location\": \"top\", \"initialSelection\": 0, \"borderColor\": [\"#a7a7a7\"]});\n", - "//# sourceURL=js_dc5d7f2784" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd7222a190\u003e" - ] - }, - "metadata": { - "tags": [ - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8a03307f-78a7-11e8-99f9-c8d3ffb5fbe0\"] = window[\"id3\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_be7950150b" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd7222ac90\u003e" - ] - }, - "metadata": { - "tags": [ - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8a033080-78a7-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_d0c3bd4eaa" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd7222aad0\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8a033081-78a7-11e8-99f9-c8d3ffb5fbe0\"] = document.querySelector(\"#id3_content_0\");\n", - "//# sourceURL=js_f10f6eba86" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd7222aed0\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8a033082-78a7-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"8a033081-78a7-11e8-99f9-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_ff29697179" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd7222abd0\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8a033083-78a7-11e8-99f9-c8d3ffb5fbe0\"] = window[\"id3\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_ff85295dc7" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd7222ab90\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8dc-78a7-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"8a033080-78a7-11e8-99f9-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_ed7aabfedb" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd7222a110\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8dd-78a7-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_c86f8feaf4" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd7222acd0\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8de-78a7-11e8-99f9-c8d3ffb5fbe0\"] = document.querySelector(\"#id3_content_0\");\n", - "//# sourceURL=js_4d0fde6662" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd7222ae50\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8df-78a7-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"8b18d8de-78a7-11e8-99f9-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_3f66d52720" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd7222a210\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8e0-78a7-11e8-99f9-c8d3ffb5fbe0\"] = window[\"id3\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_375f5ae6d7" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd7222a310\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQwAAAENCAYAAAD60Fs2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAABTFJREFUeJzt3C+LV30eh/HP6EZvbP4ZJmkXDA6oQdZRMIhYLIKCMGVA\nyyaLT2ERLMqEDfoUFA2y3WpRrOKoSUSECePcYUEWdsN1OzfOyr5e8ZwT3unie34cfgvb29vbAxDs\n2e0BwK9DMIBMMIBMMIBMMIBMMIBMMPipXrx4MWfOnNntGfwgweCnW1hY2O0J/CDBYEe2trZ2ewI/\nkWDwh509e3bW19fn0qVLc/z48dnY2Jhbt27NyZMn59y5c/Pw4cPvz25ubs7t27dneXl5Ll68OC9f\nvtzF5ezUX3Z7AL+mJ0+ezPr6+uzfv3+uXr0658+fn7t3787GxsbcuHFjjhw5MqdPn5579+7N27dv\n5/nz5/P169dZXV3d7ensgBMGP+T69etz8ODBef369Xz69GnW1tZm7969s7S0NFeuXJnHjx/PzMzT\np09nbW1tfvvttzl48OBcu3Ztl5ezE04Y/JBDhw7NzMy7d+/mw4cPs7y8PDMz29vb8+3btzlx4sTM\nzHz8+PH7szMzi4uLP38sfxrBYEcOHz48S0tL8+zZs/96/8CBA7OxsTFHjx6dmX8Fhl+XVxJ25Nix\nY7Nv375ZX1+fzc3N2dramjdv3nz/cfPChQvz4MGD+fz587x//34ePXq0y4vZCcHgD/v37yj27Nkz\n9+/fn1evXs3KysqcOnVq7ty5M1++fJmZmZs3b87i4uKsrKzM6urqXL58ebdm8ydY8Ac6QOWEAWSC\nAWSCAWSCAWT/s99h/P3GX3d7Avxf+9s//vkf15wwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEww\ngEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEww\ngEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEww\ngEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEww\ngEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEww\ngEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEww\ngEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEww\ngEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEww\ngEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEww\ngEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEww\ngEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEww\ngEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEww\ngEwwgEwwgEwwgGxhe3t7e7dHAL8GJwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwg\nEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwgEwwg+x1QoZHG4XIe4gAAAABJRU5ErkJggg==\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7fcd0d02dc90\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8e1-78a7-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"8b18d8dd-78a7-11e8-99f9-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_34b0509660" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd08e6e850\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8e2-78a7-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_518a0f26fe" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd08e6ec90\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8e3-78a7-11e8-99f9-c8d3ffb5fbe0\"] = document.querySelector(\"#id3_content_0\");\n", - "//# sourceURL=js_17eb3ff612" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd08e6eb50\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8e4-78a7-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"8b18d8e3-78a7-11e8-99f9-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_99da807c8e" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd08e6eb90\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8e5-78a7-11e8-99f9-c8d3ffb5fbe0\"] = window[\"id3\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_dee01cb4b6" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd08e6e610\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\u003cdiv class=id_853612217 style=\"margin-right:10px; display:flex;align-items:center;\"\u003e\u003cspan style=\"margin-right: 3px;\"\u003e\u003c/span\u003e\u003c/div\u003e" - ], - "text/plain": [ - "\u003cIPython.core.display.HTML at 0x7fcd7222aa10\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8e6-78a7-11e8-99f9-c8d3ffb5fbe0\"] = jQuery(\".id_853612217 span\");\n", - "//# sourceURL=js_8c378be329" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd08e6e990\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8e7-78a7-11e8-99f9-c8d3ffb5fbe0\"] = window[\"8b18d8e6-78a7-11e8-99f9-c8d3ffb5fbe0\"].text(\"Give me a color name (or press 'enter' to exit): \");\n", - "//# sourceURL=js_f0b946600c" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd08e6e310\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8e9-78a7-11e8-99f9-c8d3ffb5fbe0\"] = jQuery(\".id_853612217 input\");\n", - "//# sourceURL=js_9e21b1373a" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd08e6ea90\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8ea-78a7-11e8-99f9-c8d3ffb5fbe0\"] = window[\"8b18d8e9-78a7-11e8-99f9-c8d3ffb5fbe0\"].remove();\n", - "//# sourceURL=js_a7764968c6" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd08e6e5d0\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8eb-78a7-11e8-99f9-c8d3ffb5fbe0\"] = jQuery(\".id_853612217 span\");\n", - "//# sourceURL=js_74279d3ff0" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd08e6e890\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8ec-78a7-11e8-99f9-c8d3ffb5fbe0\"] = window[\"8b18d8eb-78a7-11e8-99f9-c8d3ffb5fbe0\"].text(\"Give me a color name (or press 'enter' to exit): \");\n", - "//# sourceURL=js_82b6c34cdb" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd08e6e8d0\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3", - "user_output" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"8b18d8ed-78a7-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"8b18d8e2-78a7-11e8-99f9-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_ff6144734a" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7fcd08e6e8d0\u003e" - ] - }, - "metadata": { - "tags": [ - "id3_content_0", - "outputarea_id3" - ] - }, - "output_type": "display_data" - } - ], - "source": [ - "def predict_input_fn(color_name):\n", - " \"\"\"An input function for prediction.\"\"\"\n", - " _, chars, sequence_length = parse(color_name)\n", - "\n", - " # We create a batch of a single element.\n", - " features = {\n", - " 'chars': tf.expand_dims(chars, 0),\n", - " 'sequence_length': tf.expand_dims(sequence_length, 0)\n", - " }\n", - " return features, None\n", - "\n", - "\n", - "def draw_prediction(color_name, pred):\n", - " pred = pred * 255\n", - " pred = pred.astype(np.uint8)\n", - " plt.axis('off')\n", - " plt.imshow(pred)\n", - " plt.title(color_name)\n", - " plt.show()\n", - "\n", - "\n", - "def predict_with_estimator(color_name, regressor):\n", - " predictions = regressor.predict(\n", - " input_fn=lambda:predict_input_fn(color_name))\n", - " pred = next(predictions)\n", - " predictions.close()\n", - " pred = np.minimum(pred, 1.0)\n", - " pred = np.expand_dims(np.expand_dims(pred, 0), 0)\n", - "\n", - " draw_prediction(color_name, pred)\n", - "\n", - "tb = widgets.TabBar([\"RNN Colorbot\"])\n", - "while True:\n", - " with tb.output_to(0):\n", - " try:\n", - " color_name = six.moves.input(\"Give me a color name (or press 'enter' to exit): \")\n", - " except (EOFError, KeyboardInterrupt):\n", - " break\n", - " if not color_name:\n", - " break\n", - " with tb.output_to(0):\n", - " tb.clear_tab()\n", - " predict_with_estimator(color_name, regressor)\n", - " " - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "default_view": {}, - "last_runtime": { - "build_target": "", - "kind": "local" - }, - "name": "RNN Colorbot using Keras and Estimators", - "version": "0.3.2", - "views": {} - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/autograph/examples/notebooks/workshop.ipynb b/tensorflow/contrib/autograph/examples/notebooks/workshop.ipynb deleted file mode 100644 index e7dfb13e15a..00000000000 --- a/tensorflow/contrib/autograph/examples/notebooks/workshop.ipynb +++ /dev/null @@ -1,1129 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "u3B7Uh50lozN" - }, - "outputs": [], - "source": [ - "!pip install -U -q tf-nightly" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "qWUV0FYjDSKj" - }, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "from tensorflow.contrib import autograph\n", - "\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kGXS3UWBBNoc" - }, - "source": [ - "# 1. AutoGraph writes graph code for you\n", - "\n", - "[AutoGraph](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/autograph/README.md) helps you write complicated graph code using just plain Python -- behind the scenes, AutoGraph automatically transforms your code into the equivalent TF graph code. We support a large chunk of the Python language, which is growing. [Please see this document for what we currently support, and what we're working on](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/autograph/LIMITATIONS.md).\n", - "\n", - "Here's a quick example of how it works:\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "aA3gOodCBkOw" - }, - "outputs": [], - "source": [ - "# Autograph can convert functions like this...\n", - "def g(x):\n", - " if x \u003e 0:\n", - " x = x * x\n", - " else:\n", - " x = 0.0\n", - " return x\n", - "\n", - "# ...into graph-building functions like this:\n", - "def tf_g(x):\n", - " with tf.name_scope('g'):\n", - "\n", - " def if_true():\n", - " with tf.name_scope('if_true'):\n", - " x_1, = x,\n", - " x_1 = x_1 * x_1\n", - " return x_1,\n", - "\n", - " def if_false():\n", - " with tf.name_scope('if_false'):\n", - " x_1, = x,\n", - " x_1 = 0.0\n", - " return x_1,\n", - "\n", - " x = autograph_utils.run_cond(tf.greater(x, 0), if_true, if_false)\n", - " return x" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "I1RtBvoKBxq5" - }, - "outputs": [], - "source": [ - "# You can run your plain-Python code in graph mode,\n", - "# and get the same results out, but with all the benfits of graphs:\n", - "print('Original value: %2.2f' % g(9.0))\n", - "\n", - "# Generate a graph-version of g and call it:\n", - "tf_g = autograph.to_graph(g)\n", - "\n", - "with tf.Graph().as_default():\n", - " # The result works like a regular op: takes tensors in, returns tensors.\n", - " # You can inspect the graph using tf.get_default_graph().as_graph_def()\n", - " g_ops = tf_g(tf.constant(9.0))\n", - " with tf.Session() as sess:\n", - " print('Autograph value: %2.2f\\n' % sess.run(g_ops))\n", - "\n", - "\n", - "# You can view, debug and tweak the generated code:\n", - "print(autograph.to_code(g))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m-jWmsCmByyw" - }, - "source": [ - "#### Automatically converting complex control flow\n", - "\n", - "AutoGraph can convert a large chunk of the Python language into equivalent graph-construction code, and we're adding new supported language features all the time. In this section, we'll give you a taste of some of the functionality in AutoGraph.\n", - "AutoGraph will automatically convert most Python control flow statements into their correct graph equivalent. \n", - " \n", - "We support common statements like `while`, `for`, `if`, `break`, `return` and more. You can even nest them as much as you like. Imagine trying to write the graph version of this code by hand:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "toxKBOXbB1ro" - }, - "outputs": [], - "source": [ - "# Continue in a loop\n", - "def f(l):\n", - " s = 0\n", - " for c in l:\n", - " if c % 2 \u003e 0:\n", - " continue\n", - " s += c\n", - " return s\n", - "\n", - "print('Original value: %d' % f([10,12,15,20]))\n", - "\n", - "tf_f = autograph.to_graph(f)\n", - "with tf.Graph().as_default():\n", - " with tf.Session():\n", - " print('Graph value: %d\\n\\n' % tf_f(tf.constant([10,12,15,20])).eval())\n", - "\n", - "print(autograph.to_code(f))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FUJJ-WTdCGeq" - }, - "source": [ - "Try replacing the `continue` in the above code with `break` -- AutoGraph supports that as well! \n", - " \n", - "Let's try some other useful Python constructs, like `print` and `assert`. We automatically convert Python `assert` statements into the equivalent `tf.Assert` code. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "IAOgh62zCPZ4" - }, - "outputs": [], - "source": [ - "def f(x):\n", - " assert x != 0, 'Do not pass zero!'\n", - " return x * x\n", - "\n", - "tf_f = autograph.to_graph(f)\n", - "with tf.Graph().as_default():\n", - " with tf.Session():\n", - " try:\n", - " print(tf_f(tf.constant(0)).eval())\n", - " except tf.errors.InvalidArgumentError as e:\n", - " print('Got error message:\\n%s' % e.message)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KRu8iIPBCQr5" - }, - "source": [ - "You can also use plain Python `print` functions in in-graph" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "ySTsuxnqCTQi" - }, - "outputs": [], - "source": [ - "def f(n):\n", - " if n \u003e= 0:\n", - " while n \u003c 5:\n", - " n += 1\n", - " print(n)\n", - " return n\n", - "\n", - "tf_f = autograph.to_graph(f)\n", - "with tf.Graph().as_default():\n", - " with tf.Session():\n", - " tf_f(tf.constant(0)).eval()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NqF0GT-VCVFh" - }, - "source": [ - "Appending to lists in loops also works (we create a tensor list ops behind the scenes)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "ABX070KwCczR" - }, - "outputs": [], - "source": [ - "def f(n):\n", - " z = []\n", - " # We ask you to tell us the element dtype of the list\n", - " autograph.set_element_type(z, tf.int32)\n", - " for i in range(n):\n", - " z.append(i)\n", - " # when you're done with the list, stack it\n", - " # (this is just like np.stack)\n", - " return autograph.stack(z)\n", - "\n", - "tf_f = autograph.to_graph(f)\n", - "with tf.Graph().as_default():\n", - " with tf.Session():\n", - " print(tf_f(tf.constant(3)).eval())\n", - "\n", - "print('\\n\\n'+autograph.to_code(f))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "iu5IF7n2Df7C" - }, - "outputs": [], - "source": [ - "def fizzbuzz(num):\n", - " if num % 3 == 0 and num % 5 == 0:\n", - " print('FizzBuzz')\n", - " elif num % 3 == 0:\n", - " print('Fizz')\n", - " elif num % 5 == 0:\n", - " print('Buzz')\n", - " else:\n", - " print(num)\n", - " return num" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "EExAjWuwDPpR" - }, - "outputs": [], - "source": [ - "tf_g = autograph.to_graph(fizzbuzz)\n", - "\n", - "with tf.Graph().as_default():\n", - " # The result works like a regular op: takes tensors in, returns tensors.\n", - " # You can inspect the graph using tf.get_default_graph().as_graph_def()\n", - " g_ops = tf_g(tf.constant(15))\n", - " with tf.Session() as sess:\n", - " sess.run(g_ops) \n", - " \n", - "# You can view, debug and tweak the generated code:\n", - "print('\\n')\n", - "print(autograph.to_code(fizzbuzz))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SzpKGzVpBkph" - }, - "source": [ - "# De-graphify Exercises\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8k23dxcSmmXq" - }, - "source": [ - "#### Easy print statements" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "dE1Vsmp-mlpK" - }, - "outputs": [], - "source": [ - "# See what happens when you turn AutoGraph off.\n", - "# Do you see the type or the value of x when you print it?\n", - "\n", - "# @autograph.convert()\n", - "def square_log(x):\n", - " x = x * x\n", - " print('Squared value of x =', x)\n", - " return x\n", - "\n", - "\n", - "with tf.Graph().as_default():\n", - " with tf.Session() as sess:\n", - " print(sess.run(square_log(tf.constant(4))))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_R-Q7BbxmkBF" - }, - "source": [ - "#### Convert the TensorFlow code into Python code for AutoGraph" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "SwA11tO-yCvg" - }, - "outputs": [], - "source": [ - "def square_if_positive(x):\n", - " x = tf.cond(tf.greater(x, 0), lambda: x * x, lambda: x)\n", - " return x\n", - "\n", - "with tf.Session() as sess:\n", - " print(sess.run(square_if_positive(tf.constant(4))))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "GPmx4CNhyPI_" - }, - "outputs": [], - "source": [ - "@autograph.convert()\n", - "def square_if_positive(x):\n", - "\n", - " pass # TODO: fill it in!\n", - "\n", - "\n", - "with tf.Session() as sess:\n", - " print(sess.run(square_if_positive(tf.constant(4))))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qqsjik-QyA9R" - }, - "source": [ - "#### Uncollapse to see answer" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "DaSmaWUEvMRv" - }, - "outputs": [], - "source": [ - "# Simple cond\n", - "@autograph.convert()\n", - "def square_if_positive(x):\n", - " if x \u003e 0:\n", - " x = x * x\n", - " return x\n", - "\n", - "with tf.Graph().as_default(): \n", - " with tf.Session() as sess:\n", - " print(sess.run(square_if_positive(tf.constant(4))))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qj7am2I_xvTJ" - }, - "source": [ - "#### Nested If statement" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "4yyNOf-Twr6s" - }, - "outputs": [], - "source": [ - "def nearest_odd_square(x):\n", - "\n", - " def if_positive():\n", - " x1 = x * x\n", - " x1 = tf.cond(tf.equal(x1 % 2, 0), lambda: x1 + 1, lambda: x1)\n", - " return x1,\n", - "\n", - " x = tf.cond(tf.greater(x, 0), if_positive, lambda: x)\n", - " return x\n", - "\n", - "with tf.Graph().as_default():\n", - " with tf.Session() as sess:\n", - " print(sess.run(nearest_odd_square(tf.constant(4))))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "hqmh5b2VyU9w" - }, - "outputs": [], - "source": [ - "@autograph.convert()\n", - "def nearest_odd_square(x):\n", - "\n", - " pass # TODO: fill it in!\n", - "\n", - "\n", - "with tf.Session() as sess:\n", - " print(sess.run(nearest_odd_square(tf.constant(4))))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "b9AXIkNLxp6J" - }, - "source": [ - "#### Uncollapse to reveal answer" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "8RlCVEpNxD91" - }, - "outputs": [], - "source": [ - "@autograph.convert()\n", - "def nearest_odd_square(x):\n", - " if x \u003e 0:\n", - " x = x * x\n", - " if x % 2 == 0:\n", - " x = x + 1\n", - " return x\n", - "\n", - "with tf.Graph().as_default():\n", - " with tf.Session() as sess:\n", - " print(sess.run(nearest_odd_square(tf.constant(4))))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jXAxjeBr1qWK" - }, - "source": [ - "#### Convert a while loop" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "kWkv7anlxoee" - }, - "outputs": [], - "source": [ - "# Convert a while loop\n", - "def square_until_stop(x, y):\n", - " x = tf.while_loop(lambda x: tf.less(x, y), lambda x: x * x, [x])\n", - " return x\n", - "\n", - "with tf.Graph().as_default():\n", - " with tf.Session() as sess:\n", - " print(sess.run(square_until_stop(tf.constant(4), tf.constant(100))))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "zVUsc1eA1u2K" - }, - "outputs": [], - "source": [ - "@autograph.convert()\n", - "def square_until_stop(x, y):\n", - "\n", - " pass # TODO: fill it in!\n", - "\n", - "\n", - "with tf.Graph().as_default():\n", - " with tf.Session() as sess:\n", - " print(sess.run(square_until_stop(tf.constant(4), tf.constant(100))))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L2psuzPI02S9" - }, - "source": [ - "#### Uncollapse for the answer\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "ucmZyQVL03bF" - }, - "outputs": [], - "source": [ - "@autograph.convert()\n", - "def square_until_stop(x, y):\n", - " while x \u003c y:\n", - " x = x * x\n", - " return x\n", - "\n", - "with tf.Graph().as_default():\n", - " with tf.Session() as sess:\n", - " print(sess.run(square_until_stop(tf.constant(4), tf.constant(100))))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FXB0Zbwl13PY" - }, - "source": [ - "#### Nested loop and conditional" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "clGymxdf15Ig" - }, - "outputs": [], - "source": [ - "@autograph.convert()\n", - "def argwhere_cumsum(x, threshold):\n", - " current_sum = 0.0\n", - " idx = 0\n", - "\n", - " for i in range(len(x)):\n", - " idx = i\n", - " if current_sum \u003e= threshold:\n", - " break\n", - " current_sum += x[i]\n", - " return idx\n", - "\n", - "n = 10\n", - "with tf.Graph().as_default():\n", - " with tf.Session() as sess:\n", - " idx = argwhere_cumsum(tf.ones(n), tf.constant(float(n / 2)))\n", - " print(sess.run(idx))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "i7PF-uId9lp5" - }, - "outputs": [], - "source": [ - "@autograph.convert()\n", - "def argwhere_cumsum(x, threshold):\n", - "\n", - " pass # TODO: fill it in!\n", - "\n", - "\n", - "n = 10\n", - "with tf.Graph().as_default():\n", - " with tf.Session() as sess:\n", - " idx = argwhere_cumsum(tf.ones(n), tf.constant(float(n / 2)))\n", - " print(sess.run(idx))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "weKFXAb615Vp" - }, - "source": [ - "#### Uncollapse to see answer" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "1sjaFcL717Ig" - }, - "outputs": [], - "source": [ - "@autograph.convert()\n", - "def argwhere_cumsum(x, threshold):\n", - " current_sum = 0.0\n", - " idx = 0\n", - " for i in range(len(x)):\n", - " idx = i\n", - " if current_sum \u003e= threshold:\n", - " break\n", - " current_sum += x[i]\n", - " return idx\n", - "\n", - "n = 10\n", - "with tf.Graph().as_default(): \n", - " with tf.Session() as sess:\n", - " idx = argwhere_cumsum(tf.ones(n), tf.constant(float(n / 2)))\n", - " print(sess.run(idx))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4LfnJjm0Bm0B" - }, - "source": [ - "# 3. Training MNIST in-graph\n", - "\n", - "Writing control flow in AutoGraph is easy, so running a training loop in a TensorFlow graph should be easy as well! \n", - "\n", - "Here, we show an example of training a simple Keras model on MNIST, where the entire training process -- loading batches, calculating gradients, updating parameters, calculating validation accuracy, and repeating until convergence -- is done in-graph." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Em5dzSUOtLRP" - }, - "source": [ - "#### Download data" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "xqoxumv0ssQW" - }, - "outputs": [], - "source": [ - "import gzip\n", - "import os\n", - "import shutil\n", - "\n", - "from six.moves import urllib\n", - "\n", - "\n", - "def download(directory, filename):\n", - " filepath = os.path.join(directory, filename)\n", - " if tf.gfile.Exists(filepath):\n", - " return filepath\n", - " if not tf.gfile.Exists(directory):\n", - " tf.gfile.MakeDirs(directory)\n", - " url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'\n", - " zipped_filepath = filepath + '.gz'\n", - " print('Downloading %s to %s' % (url, zipped_filepath))\n", - " urllib.request.urlretrieve(url, zipped_filepath)\n", - " with gzip.open(zipped_filepath, 'rb') as f_in, open(filepath, 'wb') as f_out:\n", - " shutil.copyfileobj(f_in, f_out)\n", - " os.remove(zipped_filepath)\n", - " return filepath\n", - "\n", - "\n", - "def dataset(directory, images_file, labels_file):\n", - " images_file = download(directory, images_file)\n", - " labels_file = download(directory, labels_file)\n", - "\n", - " def decode_image(image):\n", - " # Normalize from [0, 255] to [0.0, 1.0]\n", - " image = tf.decode_raw(image, tf.uint8)\n", - " image = tf.cast(image, tf.float32)\n", - " image = tf.reshape(image, [784])\n", - " return image / 255.0\n", - "\n", - " def decode_label(label):\n", - " label = tf.decode_raw(label, tf.uint8)\n", - " label = tf.reshape(label, [])\n", - " return tf.to_int32(label)\n", - "\n", - " images = tf.data.FixedLengthRecordDataset(\n", - " images_file, 28 * 28, header_bytes=16).map(decode_image)\n", - " labels = tf.data.FixedLengthRecordDataset(\n", - " labels_file, 1, header_bytes=8).map(decode_label)\n", - " return tf.data.Dataset.zip((images, labels))\n", - "\n", - "\n", - "def mnist_train(directory):\n", - " return dataset(directory, 'train-images-idx3-ubyte',\n", - " 'train-labels-idx1-ubyte')\n", - "\n", - "def mnist_test(directory):\n", - " return dataset(directory, 't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "znmy4l8ntMvW" - }, - "source": [ - "#### Define the model" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "Pe-erWQdBoC5" - }, - "outputs": [], - "source": [ - "def mlp_model(input_shape):\n", - " model = tf.keras.Sequential((\n", - " tf.keras.layers.Dense(100, activation='relu', input_shape=input_shape),\n", - " tf.keras.layers.Dense(100, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')))\n", - " model.build()\n", - " return model\n", - "\n", - "\n", - "def predict(m, x, y):\n", - " y_p = m(x)\n", - " losses = tf.keras.losses.categorical_crossentropy(y, y_p)\n", - " l = tf.reduce_mean(losses)\n", - " accuracies = tf.keras.metrics.categorical_accuracy(y, y_p)\n", - " accuracy = tf.reduce_mean(accuracies)\n", - " return l, accuracy\n", - "\n", - "\n", - "def fit(m, x, y, opt):\n", - " l, accuracy = predict(m, x, y)\n", - " opt.minimize(l)\n", - " return l, accuracy\n", - "\n", - "\n", - "def setup_mnist_data(is_training, hp, batch_size):\n", - " if is_training:\n", - " ds = mnist_train('/tmp/autograph_mnist_data')\n", - " ds = ds.shuffle(batch_size * 10)\n", - " else:\n", - " ds = mnist_test('/tmp/autograph_mnist_data')\n", - " ds = ds.repeat()\n", - " ds = ds.batch(batch_size)\n", - " return ds\n", - "\n", - "\n", - "def get_next_batch(ds):\n", - " itr = ds.make_one_shot_iterator()\n", - " image, label = itr.get_next()\n", - " x = tf.to_float(tf.reshape(image, (-1, 28 * 28)))\n", - " y = tf.one_hot(tf.squeeze(label), 10)\n", - " return x, y" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oeYV6mKnJGMr" - }, - "source": [ - "#### Define the training loop" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "3xtg_MMhJETd" - }, - "outputs": [], - "source": [ - "def train(train_ds, test_ds, hp):\n", - " m = mlp_model((28 * 28,))\n", - " opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n", - "\n", - " # We'd like to save our losses to a list. In order for AutoGraph\n", - " # to convert these lists into their graph equivalent,\n", - " # we need to specify the element type of the lists.\n", - " train_losses = []\n", - " test_losses = []\n", - " train_accuracies = []\n", - " test_accuracies = []\n", - " autograph.set_element_type(train_losses, tf.float32)\n", - " autograph.set_element_type(test_losses, tf.float32)\n", - " autograph.set_element_type(train_accuracies, tf.float32)\n", - " autograph.set_element_type(test_accuracies, tf.float32)\n", - "\n", - " # This entire training loop will be run in-graph.\n", - " i = tf.constant(0)\n", - " while i \u003c hp.max_steps:\n", - " train_x, train_y = get_next_batch(train_ds)\n", - " test_x, test_y = get_next_batch(test_ds)\n", - "\n", - " step_train_loss, step_train_accuracy = fit(m, train_x, train_y, opt)\n", - " step_test_loss, step_test_accuracy = predict(m, test_x, test_y)\n", - "\n", - " if i % (hp.max_steps // 10) == 0:\n", - " print('Step', i, 'train loss:', step_train_loss, 'test loss:',\n", - " step_test_loss, 'train accuracy:', step_train_accuracy,\n", - " 'test accuracy:', step_test_accuracy)\n", - "\n", - " train_losses.append(step_train_loss)\n", - " test_losses.append(step_test_loss)\n", - " train_accuracies.append(step_train_accuracy)\n", - " test_accuracies.append(step_test_accuracy)\n", - "\n", - " i += 1\n", - "\n", - " # We've recorded our loss values and accuracies\n", - " # to a list in a graph with AutoGraph's help.\n", - " # In order to return the values as a Tensor,\n", - " # we need to stack them before returning them.\n", - " return (\n", - " autograph.stack(train_losses),\n", - " autograph.stack(test_losses),\n", - " autograph.stack(train_accuracies),\n", - " autograph.stack(test_accuracies),\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "HYh6MSZyJOag" - }, - "outputs": [], - "source": [ - "with tf.Graph().as_default():\n", - " hp = tf.contrib.training.HParams(\n", - " learning_rate=0.05,\n", - " max_steps=500,\n", - " )\n", - " train_ds = setup_mnist_data(True, hp, 50)\n", - " test_ds = setup_mnist_data(False, hp, 1000)\n", - " tf_train = autograph.to_graph(train)\n", - " loss_tensors = tf_train(train_ds, test_ds, hp)\n", - "\n", - " with tf.Session() as sess:\n", - " sess.run(tf.global_variables_initializer())\n", - " (\n", - " train_losses,\n", - " test_losses,\n", - " train_accuracies,\n", - " test_accuracies\n", - " ) = sess.run(loss_tensors)\n", - "\n", - " plt.title('MNIST train/test losses')\n", - " plt.plot(train_losses, label='train loss')\n", - " plt.plot(test_losses, label='test loss')\n", - " plt.legend()\n", - " plt.xlabel('Training step')\n", - " plt.ylabel('Loss')\n", - " plt.show()\n", - " plt.title('MNIST train/test accuracies')\n", - " plt.plot(train_accuracies, label='train accuracy')\n", - " plt.plot(test_accuracies, label='test accuracy')\n", - " plt.legend(loc='lower right')\n", - " plt.xlabel('Training step')\n", - " plt.ylabel('Accuracy')\n", - " plt.show()" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [ - "qqsjik-QyA9R", - "b9AXIkNLxp6J", - "L2psuzPI02S9", - "weKFXAb615Vp", - "Em5dzSUOtLRP" - ], - "default_view": {}, - "name": "AutoGraph Workshop.ipynb", - "provenance": [ - { - "file_id": "1kE2gz_zuwdYySL4K2HQSz13uLCYi-fYP", - "timestamp": 1530563781803 - } - ], - "version": "0.3.2", - "views": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/batching/BUILD b/tensorflow/contrib/batching/BUILD deleted file mode 100644 index d2528fa1d44..00000000000 --- a/tensorflow/contrib/batching/BUILD +++ /dev/null @@ -1,53 +0,0 @@ -# Description: Batching scheduling library. - -load( - "//tensorflow:tensorflow.bzl", - "py_test", -) - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "batch_py", - srcs = glob(["python/ops/*.py"]) + ["__init__.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:batch_ops", - "//tensorflow/python:batch_ops_gen", - ], -) - -cc_library( - name = "batch_ops_kernels", - deps = [ - "//tensorflow/core:batch_ops_op_lib", - "//tensorflow/core/kernels:batch_kernels", - ], - alwayslink = 1, -) - -py_test( - name = "batch_ops_test", - size = "small", - srcs = ["python/ops/batch_ops_test.py"], - python_version = "PY2", - shard_count = 5, - srcs_version = "PY2AND3", - tags = [ - "manual", - "no_pip", - "nomac", - ], - deps = [ - ":batch_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework", - "//tensorflow/python:gradients", - "//tensorflow/python:script_ops", - ], -) diff --git a/tensorflow/contrib/batching/__init__.py b/tensorflow/contrib/batching/__init__.py deleted file mode 100644 index 1e503a097a7..00000000000 --- a/tensorflow/contrib/batching/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops and modules related to batch. - -@@batch_function_v1 -@@batch_function -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.batching.python.ops.batch_ops import batch_function - -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/batching/python/ops/batch_ops.py b/tensorflow/contrib/batching/python/ops/batch_ops.py deleted file mode 100644 index 2a4f3c36fbf..00000000000 --- a/tensorflow/contrib/batching/python/ops/batch_ops.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Operations for automatic batching and unbatching.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import gen_batch_ops -# pylint: disable=unused-import -from tensorflow.python.ops.batch_ops import batch -from tensorflow.python.ops.batch_ops import batch_function -from tensorflow.python.ops.batch_ops import unbatch -# pylint: enable=unused-import - - -@ops.RegisterGradient("Batch") -def _BatchGrad(op, *out_grads): # pylint: disable=invalid-name - """Gradient for batch op.""" - gradients = [] - for i in range(len(op.inputs)): - gradients.append( - gen_batch_ops.unbatch( - out_grads[i], - op.outputs[-2], - op.outputs[-1], - timeout_micros=op.get_attr("grad_timeout_micros"), - shared_name="batch_gradient_{}_{}".format(op.name, i))) - return gradients - - -@ops.RegisterGradient("Unbatch") -def _UnbatchGrad(op, grad): # pylint: disable=invalid-name - return [ - gen_batch_ops.unbatch_grad( - op.inputs[0], - op.inputs[1], - grad, - op.inputs[2], - shared_name="unbatch_gradient_{}".format(op.name)), None, None - ] - - -def batch_function_v1(num_batch_threads, - max_batch_size, - batch_timeout_micros, - allowed_batch_sizes=None, - grad_timeout_micros=60 * 1000 * 1000, - unbatch_timeout_micros=60 * 1000 * 1000, - max_enqueued_batches=10): - """Batches the computation done by the decorated function. - - This is the older version of batch_function(). Please use the former instead - of this. - - Args: - num_batch_threads: Number of scheduling threads for processing batches - of work. Determines the number of batches processed in parallel. - max_batch_size: Batch sizes will never be bigger than this. - batch_timeout_micros: Maximum number of microseconds to wait before - outputting an incomplete batch. - allowed_batch_sizes: Optional list of allowed batch sizes. If left empty, - does nothing. Otherwise, supplies a list of batch sizes, causing the op - to pad batches up to one of those sizes. The entries must increase - monotonically, and the final entry must equal max_batch_size. - grad_timeout_micros: The timeout to use for the gradient. See the - documentation of the unbatch op for more details. Defaults to 60s. - unbatch_timeout_micros: The timeout to use for unbatching. See the - documentation of the unbatch op for more details. Defaults to 60s. - max_enqueued_batches: The maximum depth of the batch queue. Defaults to 10. - - Returns: - The decorated function will return the unbatched computation output Tensors. - """ - def decorator(f): # pylint: disable=missing-docstring - def decorated(*args): - with ops.name_scope("batch") as name: - for a in args: - if not isinstance(a, ops.Tensor): - raise ValueError("All arguments to functions decorated with " - "`batch_function` are supposed to be Tensors; " - "found %s" % repr(a)) - batched_tensors, batch_index, id_t = gen_batch_ops.batch( - args, - num_batch_threads=num_batch_threads, - max_batch_size=max_batch_size, - batch_timeout_micros=batch_timeout_micros, - max_enqueued_batches=max_enqueued_batches, - allowed_batch_sizes=allowed_batch_sizes, - grad_timeout_micros=grad_timeout_micros, - shared_name=name) - outputs = f(*batched_tensors) - if isinstance(outputs, ops.Tensor): - outputs_list = [outputs] - else: - outputs_list = outputs - with ops.name_scope("unbatch") as unbatch_name: - unbatched = [ - gen_batch_ops.unbatch(t, batch_index, id_t, - timeout_micros=unbatch_timeout_micros, - shared_name=unbatch_name + "/" + t.name) - for t in outputs_list] - if isinstance(outputs, ops.Tensor): - return unbatched[0] - return unbatched - return decorated - return decorator diff --git a/tensorflow/contrib/batching/python/ops/batch_ops_test.py b/tensorflow/contrib/batching/python/ops/batch_ops_test.py deleted file mode 100644 index e224588fa30..00000000000 --- a/tensorflow/contrib/batching/python/ops/batch_ops_test.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for the currently experimental in-graph batch ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import threading -import time - -from tensorflow.contrib.batching.python.ops import batch_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.platform import test - - -def delayed_plus1(x): - """Sleeps for 100ms then returns x+1.""" - time.sleep(0.1) - return x + 1 - - -class BatchOpsTest(test.TestCase): - """Tests for batch_ops.{un,}batch.""" - - def testBasicUnbatchV1Decorated(self): - """Tests that the batch_function_v1 decorator works.""" - with self.cached_session() as sess: - - @batch_ops.batch_function_v1(1, 10, 100000) - def computation(in_t): - return in_t + 1 - - inp = array_ops.placeholder(dtype=dtypes.int32, shape=[1]) - result = computation(inp) - thread_results = [] - - def worker(): - thread_results.extend(sess.run([result], feed_dict={inp: [1]})) - - worker_thread = threading.Thread(target=worker) - worker_thread.start() - main_results = sess.run([result], feed_dict={inp: [2]}) - worker_thread.join() - self.assertEqual(thread_results[0], [2]) - self.assertEqual(main_results[0], [3]) - - def testUnbatchGrad(self): - """Tests that batch and unbatch are differentiable.""" - with self.cached_session() as sess: - inp = array_ops.placeholder(dtype=dtypes.float32, shape=[1]) - batched, index, id_t = batch_ops.batch( - [inp], num_batch_threads=1, max_batch_size=2, - batch_timeout_micros=36000000, grad_timeout_micros=1000000, - batching_queue="") - computation = batched[0] * batched[0] - result = batch_ops.unbatch(computation, index, id_t, - timeout_micros=1000000, shared_name="unbatch") - grad = gradients_impl.gradients(result, inp) - thread_results = [] - - def worker(): - thread_results.extend(sess.run([grad], feed_dict={inp: [1]})) - - worker_thread = threading.Thread(target=worker) - worker_thread.start() - main_results = sess.run([grad], feed_dict={inp: [2]}) - worker_thread.join() - self.assertEqual(thread_results[0], [2]) - self.assertEqual(main_results[0], [4]) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/bayesflow/BUILD b/tensorflow/contrib/bayesflow/BUILD deleted file mode 100644 index 331ab87bf5d..00000000000 --- a/tensorflow/contrib/bayesflow/BUILD +++ /dev/null @@ -1,60 +0,0 @@ -# Description: -# Contains ops for working with statistical distributions, -# particularly useful for Bayesian inference. -# APIs here are meant to evolve over time. - -load("//tensorflow:tensorflow.bzl", "cuda_py_test") - -package( - default_visibility = [ - "//learning/brain/contrib/bayesflow:__subpackages__", - "//tensorflow:__subpackages__", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "bayesflow_py", - srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/distributions:distributions_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:functional_ops", - "//tensorflow/python:gradients", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:util", - "//third_party/py/numpy", - ], -) - -cuda_py_test( - name = "monte_carlo_test", - size = "small", - srcs = ["python/kernel_tests/monte_carlo_test.py"], - additional_deps = [ - ":bayesflow_py", - "//third_party/py/numpy", - "//tensorflow/contrib/distributions:distributions_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python/ops/distributions", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_seed", - ], -) diff --git a/tensorflow/contrib/bayesflow/README.md b/tensorflow/contrib/bayesflow/README.md deleted file mode 100644 index 10323dc6d59..00000000000 --- a/tensorflow/contrib/bayesflow/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Notice - -`tf.contrib.bayesflow` has moved! - -See new code at [github.com/tensorflow/probability]( -https://github.com/tensorflow/probability). - -Switch imports with: - -```python -# old -import tensorflow as tf -tfp = tf.contrib.bayesflow - -# new -import tensorflow_probability as tfp -``` diff --git a/tensorflow/contrib/bayesflow/__init__.py b/tensorflow/contrib/bayesflow/__init__.py deleted file mode 100644 index 493046b3990..00000000000 --- a/tensorflow/contrib/bayesflow/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for representing Bayesian computation. - -Use [tfp](/probability/api_docs/python/tfp) instead. - -## This package provides classes for Bayesian computation with TensorFlow. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,line-too-long -from tensorflow.contrib.bayesflow.python.ops import monte_carlo -# pylint: enable=unused-import,line-too-long - -from tensorflow.python.util.all_util import remove_undocumented - - -_allowed_symbols = [ - 'monte_carlo', -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/bayesflow/python/__init__.py b/tensorflow/contrib/bayesflow/python/__init__.py deleted file mode 100644 index c5ca3a623fb..00000000000 --- a/tensorflow/contrib/bayesflow/python/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ops module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/bayesflow/python/kernel_tests/monte_carlo_test.py b/tensorflow/contrib/bayesflow/python/kernel_tests/monte_carlo_test.py deleted file mode 100644 index 8b6ed9f041b..00000000000 --- a/tensorflow/contrib/bayesflow/python/kernel_tests/monte_carlo_test.py +++ /dev/null @@ -1,260 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Monte Carlo Ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib import layers as layers_lib -from tensorflow.contrib.bayesflow.python.ops import monte_carlo_impl as monte_carlo_lib -from tensorflow.contrib.bayesflow.python.ops.monte_carlo_impl import _get_samples -from tensorflow.contrib.distributions.python.ops import mvn_diag as mvn_diag_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import distribution as distribution_lib -from tensorflow.python.ops.distributions import kullback_leibler -from tensorflow.python.ops.distributions import normal as normal_lib -from tensorflow.python.platform import test - - -layers = layers_lib -mc = monte_carlo_lib - - -class ExpectationImportanceSampleTest(test.TestCase): - - def test_normal_integral_mean_and_var_correctly_estimated(self): - n = int(1e6) - with self.cached_session(): - mu_p = constant_op.constant([-1.0, 1.0], dtype=dtypes.float64) - mu_q = constant_op.constant([0.0, 0.0], dtype=dtypes.float64) - sigma_p = constant_op.constant([0.5, 0.5], dtype=dtypes.float64) - sigma_q = constant_op.constant([1.0, 1.0], dtype=dtypes.float64) - p = normal_lib.Normal(loc=mu_p, scale=sigma_p) - q = normal_lib.Normal(loc=mu_q, scale=sigma_q) - - # Compute E_p[X]. - e_x = mc.expectation_importance_sampler( - f=lambda x: x, log_p=p.log_prob, sampling_dist_q=q, n=n, seed=42) - - # Compute E_p[X^2]. - e_x2 = mc.expectation_importance_sampler( - f=math_ops.square, log_p=p.log_prob, sampling_dist_q=q, n=n, seed=42) - - stddev = math_ops.sqrt(e_x2 - math_ops.square(e_x)) - - # Relative tolerance (rtol) chosen 2 times as large as minimim needed to - # pass. - # Convergence of mean is +- 0.003 if n = 100M - # Convergence of stddev is +- 0.00001 if n = 100M - self.assertEqual(p.batch_shape, e_x.get_shape()) - self.assertAllClose(p.mean().eval(), e_x.eval(), rtol=0.01) - self.assertAllClose(p.stddev().eval(), stddev.eval(), rtol=0.02) - - def test_multivariate_normal_prob_positive_product_of_components(self): - # Test that importance sampling can correctly estimate the probability that - # the product of components in a MultivariateNormal are > 0. - n = 1000 - with self.cached_session(): - p = mvn_diag_lib.MultivariateNormalDiag( - loc=[0.], scale_diag=[1.0, 1.0]) - q = mvn_diag_lib.MultivariateNormalDiag( - loc=[0.5], scale_diag=[3., 3.]) - - # Compute E_p[X_1 * X_2 > 0], with X_i the ith component of X ~ p(x). - # Should equal 1/2 because p is a spherical Gaussian centered at (0, 0). - def indicator(x): - x1_times_x2 = math_ops.reduce_prod(x, axis=[-1]) - return 0.5 * (math_ops.sign(x1_times_x2) + 1.0) - - prob = mc.expectation_importance_sampler( - f=indicator, log_p=p.log_prob, sampling_dist_q=q, n=n, seed=42) - - # Relative tolerance (rtol) chosen 2 times as large as minimim needed to - # pass. - # Convergence is +- 0.004 if n = 100k. - self.assertEqual(p.batch_shape, prob.get_shape()) - self.assertAllClose(0.5, prob.eval(), rtol=0.05) - - -class ExpectationImportanceSampleLogspaceTest(test.TestCase): - - def test_normal_distribution_second_moment_estimated_correctly(self): - # Test the importance sampled estimate against an analytical result. - n = int(1e6) - with self.cached_session(): - mu_p = constant_op.constant([0.0, 0.0], dtype=dtypes.float64) - mu_q = constant_op.constant([-1.0, 1.0], dtype=dtypes.float64) - sigma_p = constant_op.constant([1.0, 2 / 3.], dtype=dtypes.float64) - sigma_q = constant_op.constant([1.0, 1.0], dtype=dtypes.float64) - p = normal_lib.Normal(loc=mu_p, scale=sigma_p) - q = normal_lib.Normal(loc=mu_q, scale=sigma_q) - - # Compute E_p[X^2]. - # Should equal [1, (2/3)^2] - log_e_x2 = mc.expectation_importance_sampler_logspace( - log_f=lambda x: math_ops.log(math_ops.square(x)), - log_p=p.log_prob, - sampling_dist_q=q, - n=n, - seed=42) - e_x2 = math_ops.exp(log_e_x2) - - # Relative tolerance (rtol) chosen 2 times as large as minimim needed to - # pass. - self.assertEqual(p.batch_shape, e_x2.get_shape()) - self.assertAllClose([1., (2 / 3.)**2], e_x2.eval(), rtol=0.02) - - -class GetSamplesTest(test.TestCase): - """Test the private method 'get_samples'.""" - - def test_raises_if_both_z_and_n_are_none(self): - with self.cached_session(): - dist = normal_lib.Normal(loc=0., scale=1.) - z = None - n = None - seed = None - with self.assertRaisesRegexp(ValueError, 'exactly one'): - _get_samples(dist, z, n, seed) - - def test_raises_if_both_z_and_n_are_not_none(self): - with self.cached_session(): - dist = normal_lib.Normal(loc=0., scale=1.) - z = dist.sample(seed=42) - n = 1 - seed = None - with self.assertRaisesRegexp(ValueError, 'exactly one'): - _get_samples(dist, z, n, seed) - - def test_returns_n_samples_if_n_provided(self): - with self.cached_session(): - dist = normal_lib.Normal(loc=0., scale=1.) - z = None - n = 10 - seed = None - z = _get_samples(dist, z, n, seed) - self.assertEqual((10,), z.get_shape()) - - def test_returns_z_if_z_provided(self): - with self.cached_session(): - dist = normal_lib.Normal(loc=0., scale=1.) - z = dist.sample(10, seed=42) - n = None - seed = None - z = _get_samples(dist, z, n, seed) - self.assertEqual((10,), z.get_shape()) - - -class ExpectationTest(test.TestCase): - - def test_works_correctly(self): - with self.cached_session() as sess: - x = constant_op.constant([-1e6, -100, -10, -1, 1, 10, 100, 1e6]) - p = normal_lib.Normal(loc=x, scale=1.) - - # We use the prefex "efx" to mean "E_p[f(X)]". - f = lambda u: u - efx_true = x - samples = p.sample(int(1e5), seed=1) - efx_reparam = mc.expectation(f, samples, p.log_prob) - efx_score = mc.expectation(f, samples, p.log_prob, - use_reparametrization=False) - - [ - efx_true_, - efx_reparam_, - efx_score_, - efx_true_grad_, - efx_reparam_grad_, - efx_score_grad_, - ] = sess.run([ - efx_true, - efx_reparam, - efx_score, - gradients_impl.gradients(efx_true, x)[0], - gradients_impl.gradients(efx_reparam, x)[0], - gradients_impl.gradients(efx_score, x)[0], - ]) - - self.assertAllEqual(np.ones_like(efx_true_grad_), efx_true_grad_) - - self.assertAllClose(efx_true_, efx_reparam_, rtol=0.005, atol=0.) - self.assertAllClose(efx_true_, efx_score_, rtol=0.005, atol=0.) - - self.assertAllEqual(np.ones_like(efx_true_grad_, dtype=np.bool), - np.isfinite(efx_reparam_grad_)) - self.assertAllEqual(np.ones_like(efx_true_grad_, dtype=np.bool), - np.isfinite(efx_score_grad_)) - - self.assertAllClose(efx_true_grad_, efx_reparam_grad_, - rtol=0.03, atol=0.) - # Variance is too high to be meaningful, so we'll only check those which - # converge. - self.assertAllClose(efx_true_grad_[2:-2], - efx_score_grad_[2:-2], - rtol=0.05, atol=0.) - - def test_docstring_example_normal(self): - with self.cached_session() as sess: - num_draws = int(1e5) - mu_p = constant_op.constant(0.) - mu_q = constant_op.constant(1.) - p = normal_lib.Normal(loc=mu_p, scale=1.) - q = normal_lib.Normal(loc=mu_q, scale=2.) - exact_kl_normal_normal = kullback_leibler.kl_divergence(p, q) - approx_kl_normal_normal = monte_carlo_lib.expectation( - f=lambda x: p.log_prob(x) - q.log_prob(x), - samples=p.sample(num_draws, seed=42), - log_prob=p.log_prob, - use_reparametrization=(p.reparameterization_type - == distribution_lib.FULLY_REPARAMETERIZED)) - [exact_kl_normal_normal_, approx_kl_normal_normal_] = sess.run([ - exact_kl_normal_normal, approx_kl_normal_normal]) - self.assertEqual( - True, - p.reparameterization_type == distribution_lib.FULLY_REPARAMETERIZED) - self.assertAllClose(exact_kl_normal_normal_, approx_kl_normal_normal_, - rtol=0.01, atol=0.) - - # Compare gradients. (Not present in `docstring`.) - gradp = lambda fp: gradients_impl.gradients(fp, mu_p)[0] - gradq = lambda fq: gradients_impl.gradients(fq, mu_q)[0] - [ - gradp_exact_kl_normal_normal_, - gradq_exact_kl_normal_normal_, - gradp_approx_kl_normal_normal_, - gradq_approx_kl_normal_normal_, - ] = sess.run([ - gradp(exact_kl_normal_normal), - gradq(exact_kl_normal_normal), - gradp(approx_kl_normal_normal), - gradq(approx_kl_normal_normal), - ]) - self.assertAllClose(gradp_exact_kl_normal_normal_, - gradp_approx_kl_normal_normal_, - rtol=0.01, atol=0.) - self.assertAllClose(gradq_exact_kl_normal_normal_, - gradq_approx_kl_normal_normal_, - rtol=0.01, atol=0.) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/bayesflow/python/ops/monte_carlo.py b/tensorflow/contrib/bayesflow/python/ops/monte_carlo.py deleted file mode 100644 index 28a829d87dd..00000000000 --- a/tensorflow/contrib/bayesflow/python/ops/monte_carlo.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Monte Carlo integration and helpers. - -Use [tfp.monte_carlo](/probability/api_docs/python/tfp/monte_carlo) instead. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# go/tf-wildcard-import -# pylint: disable=wildcard-import -from tensorflow.contrib.bayesflow.python.ops.monte_carlo_impl import * -# pylint: enable=wildcard-import -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'expectation', - 'expectation_importance_sampler', - 'expectation_importance_sampler_logspace', -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py b/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py deleted file mode 100644 index e83a5485119..00000000000 --- a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py +++ /dev/null @@ -1,374 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Monte Carlo integration and helpers. - -@@expectation -@@expectation_importance_sampler -@@expectation_importance_sampler_logspace -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.util import deprecation - -__all__ = [ - 'expectation', - 'expectation_importance_sampler', - 'expectation_importance_sampler_logspace', -] - - -def expectation_importance_sampler(f, - log_p, - sampling_dist_q, - z=None, - n=None, - seed=None, - name='expectation_importance_sampler'): - r"""Monte Carlo estimate of \\(E_p[f(Z)] = E_q[f(Z) p(Z) / q(Z)]\\). - - With \\(p(z) := exp^{log_p(z)}\\), this `Op` returns - - \\(n^{-1} sum_{i=1}^n [ f(z_i) p(z_i) / q(z_i) ], z_i ~ q,\\) - \\(\approx E_q[ f(Z) p(Z) / q(Z) ]\\) - \\(= E_p[f(Z)]\\) - - This integral is done in log-space with max-subtraction to better handle the - often extreme values that `f(z) p(z) / q(z)` can take on. - - If `f >= 0`, it is up to 2x more efficient to exponentiate the result of - `expectation_importance_sampler_logspace` applied to `Log[f]`. - - User supplies either `Tensor` of samples `z`, or number of samples to draw `n` - - Args: - f: Callable mapping samples from `sampling_dist_q` to `Tensors` with shape - broadcastable to `q.batch_shape`. - For example, `f` works "just like" `q.log_prob`. - log_p: Callable mapping samples from `sampling_dist_q` to `Tensors` with - shape broadcastable to `q.batch_shape`. - For example, `log_p` works "just like" `sampling_dist_q.log_prob`. - sampling_dist_q: The sampling distribution. - `tfp.distributions.Distribution`. - `float64` `dtype` recommended. - `log_p` and `q` should be supported on the same set. - z: `Tensor` of samples from `q`, produced by `q.sample` for some `n`. - n: Integer `Tensor`. Number of samples to generate if `z` is not provided. - seed: Python integer to seed the random number generator. - name: A name to give this `Op`. - - Returns: - The importance sampling estimate. `Tensor` with `shape` equal - to batch shape of `q`, and `dtype` = `q.dtype`. - """ - q = sampling_dist_q - with ops.name_scope(name, values=[z, n]): - z = _get_samples(q, z, n, seed) - - log_p_z = log_p(z) - q_log_prob_z = q.log_prob(z) - - def _importance_sampler_positive_f(log_f_z): - # Same as expectation_importance_sampler_logspace, but using Tensors - # rather than samples and functions. Allows us to sample once. - log_values = log_f_z + log_p_z - q_log_prob_z - return _logspace_mean(log_values) - - # With \\(f_{plus}(z) = max(0, f(z)), f_{minus}(z) = max(0, -f(z))\\), - # \\(E_p[f(Z)] = E_p[f_{plus}(Z)] - E_p[f_{minus}(Z)]\\) - # \\( = E_p[f_{plus}(Z) + 1] - E_p[f_{minus}(Z) + 1]\\) - # Without incurring bias, 1 is added to each to prevent zeros in logspace. - # The logarithm is approximately linear around 1 + epsilon, so this is good - # for small values of 'z' as well. - f_z = f(z) - log_f_plus_z = math_ops.log(nn.relu(f_z) + 1.) - log_f_minus_z = math_ops.log(nn.relu(-1. * f_z) + 1.) - - log_f_plus_integral = _importance_sampler_positive_f(log_f_plus_z) - log_f_minus_integral = _importance_sampler_positive_f(log_f_minus_z) - - return math_ops.exp(log_f_plus_integral) - math_ops.exp(log_f_minus_integral) - - -def expectation_importance_sampler_logspace( - log_f, - log_p, - sampling_dist_q, - z=None, - n=None, - seed=None, - name='expectation_importance_sampler_logspace'): - r"""Importance sampling with a positive function, in log-space. - - With \\(p(z) := exp^{log_p(z)}\\), and \\(f(z) = exp{log_f(z)}\\), - this `Op` returns - - \\(Log[ n^{-1} sum_{i=1}^n [ f(z_i) p(z_i) / q(z_i) ] ], z_i ~ q,\\) - \\(\approx Log[ E_q[ f(Z) p(Z) / q(Z) ] ]\\) - \\(= Log[E_p[f(Z)]]\\) - - This integral is done in log-space with max-subtraction to better handle the - often extreme values that `f(z) p(z) / q(z)` can take on. - - In contrast to `expectation_importance_sampler`, this `Op` returns values in - log-space. - - - User supplies either `Tensor` of samples `z`, or number of samples to draw `n` - - Args: - log_f: Callable mapping samples from `sampling_dist_q` to `Tensors` with - shape broadcastable to `q.batch_shape`. - For example, `log_f` works "just like" `sampling_dist_q.log_prob`. - log_p: Callable mapping samples from `sampling_dist_q` to `Tensors` with - shape broadcastable to `q.batch_shape`. - For example, `log_p` works "just like" `q.log_prob`. - sampling_dist_q: The sampling distribution. - `tfp.distributions.Distribution`. - `float64` `dtype` recommended. - `log_p` and `q` should be supported on the same set. - z: `Tensor` of samples from `q`, produced by `q.sample` for some `n`. - n: Integer `Tensor`. Number of samples to generate if `z` is not provided. - seed: Python integer to seed the random number generator. - name: A name to give this `Op`. - - Returns: - Logarithm of the importance sampling estimate. `Tensor` with `shape` equal - to batch shape of `q`, and `dtype` = `q.dtype`. - """ - q = sampling_dist_q - with ops.name_scope(name, values=[z, n]): - z = _get_samples(q, z, n, seed) - log_values = log_f(z) + log_p(z) - q.log_prob(z) - return _logspace_mean(log_values) - - -def _logspace_mean(log_values): - """Evaluate `Log[E[values]]` in a stable manner. - - Args: - log_values: `Tensor` holding `Log[values]`. - - Returns: - `Tensor` of same `dtype` as `log_values`, reduced across dim 0. - `Log[Mean[values]]`. - """ - # center = Max[Log[values]], with stop-gradient - # The center hopefully keep the exponentiated term small. It is canceled - # from the final result, so putting stop gradient on it will not change the - # final result. We put stop gradient on to eliminate unnecessary computation. - center = array_ops.stop_gradient(_sample_max(log_values)) - - # centered_values = exp{Log[values] - E[Log[values]]} - centered_values = math_ops.exp(log_values - center) - - # log_mean_of_values = Log[ E[centered_values] ] + center - # = Log[ E[exp{log_values - E[log_values]}] ] + center - # = Log[E[values]] - E[log_values] + center - # = Log[E[values]] - log_mean_of_values = math_ops.log(_sample_mean(centered_values)) + center - - return log_mean_of_values - - -@deprecation.deprecated( - '2018-10-01', - 'The tf.contrib.bayesflow library has moved to ' - 'TensorFlow Probability (https://github.com/tensorflow/probability). ' - 'Use `tfp.monte_carlo.expectation` instead.', - warn_once=True) -def expectation(f, samples, log_prob=None, use_reparametrization=True, - axis=0, keep_dims=False, name=None): - r"""Computes the Monte-Carlo approximation of \\(E_p[f(X)]\\). - - This function computes the Monte-Carlo approximation of an expectation, i.e., - - \\(E_p[f(X)] \approx= m^{-1} sum_i^m f(x_j), x_j\ ~iid\ p(X)\\) - - where: - - - `x_j = samples[j, ...]`, - - `log(p(samples)) = log_prob(samples)` and - - `m = prod(shape(samples)[axis])`. - - Tricks: Reparameterization and Score-Gradient - - When p is "reparameterized", i.e., a diffeomorphic transformation of a - parameterless distribution (e.g., - `Normal(Y; m, s) <=> Y = sX + m, X ~ Normal(0,1)`), we can swap gradient and - expectation, i.e., - grad[ Avg{ \\(s_i : i=1...n\\) } ] = Avg{ grad[\\(s_i\\)] : i=1...n } where - S_n = Avg{\\(s_i\\)}` and `\\(s_i = f(x_i), x_i ~ p\\). - - However, if p is not reparameterized, TensorFlow's gradient will be incorrect - since the chain-rule stops at samples of non-reparameterized distributions. - (The non-differentiated result, `approx_expectation`, is the same regardless - of `use_reparametrization`.) In this circumstance using the Score-Gradient - trick results in an unbiased gradient, i.e., - - ```none - grad[ E_p[f(X)] ] - = grad[ int dx p(x) f(x) ] - = int dx grad[ p(x) f(x) ] - = int dx [ p'(x) f(x) + p(x) f'(x) ] - = int dx p(x) [p'(x) / p(x) f(x) + f'(x) ] - = int dx p(x) grad[ f(x) p(x) / stop_grad[p(x)] ] - = E_p[ grad[ f(x) p(x) / stop_grad[p(x)] ] ] - ``` - - Unless p is not reparametrized, it is usually preferable to - `use_reparametrization = True`. - - Warning: users are responsible for verifying `p` is a "reparameterized" - distribution. - - Example Use: - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Monte-Carlo approximation of a reparameterized distribution, e.g., Normal. - - num_draws = int(1e5) - p = tfd.Normal(loc=0., scale=1.) - q = tfd.Normal(loc=1., scale=2.) - exact_kl_normal_normal = tfd.kl_divergence(p, q) - # ==> 0.44314718 - approx_kl_normal_normal = tfp.monte_carlo.expectation( - f=lambda x: p.log_prob(x) - q.log_prob(x), - samples=p.sample(num_draws, seed=42), - log_prob=p.log_prob, - use_reparametrization=(p.reparameterization_type - == distribution.FULLY_REPARAMETERIZED)) - # ==> 0.44632751 - # Relative Error: <1% - - # Monte-Carlo approximation of non-reparameterized distribution, e.g., Gamma. - - num_draws = int(1e5) - p = ds.Gamma(concentration=1., rate=1.) - q = ds.Gamma(concentration=2., rate=3.) - exact_kl_gamma_gamma = tfd.kl_divergence(p, q) - # ==> 0.37999129 - approx_kl_gamma_gamma = tfp.monte_carlo.expectation( - f=lambda x: p.log_prob(x) - q.log_prob(x), - samples=p.sample(num_draws, seed=42), - log_prob=p.log_prob, - use_reparametrization=(p.reparameterization_type - == distribution.FULLY_REPARAMETERIZED)) - # ==> 0.37696719 - # Relative Error: <1% - - # For comparing the gradients, see `monte_carlo_test.py`. - ``` - - Note: The above example is for illustration only. To compute approximate - KL-divergence, the following is preferred: - - ```python - approx_kl_p_q = tfp.vi.monte_carlo_csiszar_f_divergence( - f=bf.kl_reverse, - p_log_prob=q.log_prob, - q=p, - num_draws=num_draws) - ``` - - Args: - f: Python callable which can return `f(samples)`. - samples: `Tensor` of samples used to form the Monte-Carlo approximation of - \\(E_p[f(X)]\\). A batch of samples should be indexed by `axis` - dimensions. - log_prob: Python callable which can return `log_prob(samples)`. Must - correspond to the natural-logarithm of the pdf/pmf of each sample. Only - required/used if `use_reparametrization=False`. - Default value: `None`. - use_reparametrization: Python `bool` indicating that the approximation - should use the fact that the gradient of samples is unbiased. Whether - `True` or `False`, this arg only affects the gradient of the resulting - `approx_expectation`. - Default value: `True`. - axis: The dimensions to average. If `None`, averages all - dimensions. - Default value: `0` (the left-most dimension). - keep_dims: If True, retains averaged dimensions using size `1`. - Default value: `False`. - name: A `name_scope` for operations created by this function. - Default value: `None` (which implies "expectation"). - - Returns: - approx_expectation: `Tensor` corresponding to the Monte-Carlo approximation - of \\(E_p[f(X)]\\). - - Raises: - ValueError: if `f` is not a Python `callable`. - ValueError: if `use_reparametrization=False` and `log_prob` is not a Python - `callable`. - """ - - with ops.name_scope(name, 'expectation', [samples]): - if not callable(f): - raise ValueError('`f` must be a callable function.') - if use_reparametrization: - return math_ops.reduce_mean(f(samples), axis=axis, keepdims=keep_dims) - else: - if not callable(log_prob): - raise ValueError('`log_prob` must be a callable function.') - stop = array_ops.stop_gradient # For readability. - x = stop(samples) - logpx = log_prob(x) - fx = f(x) # Call `f` once in case it has side-effects. - # We now rewrite f(x) so that: - # `grad[f(x)] := grad[f(x)] + f(x) * grad[logqx]`. - # To achieve this, we use a trick that - # `h(x) - stop(h(x)) == zeros_like(h(x))` - # but its gradient is grad[h(x)]. - # Note that IEEE754 specifies that `x - x == 0.` and `x + 0. == x`, hence - # this trick loses no precision. For more discussion regarding the - # relevant portions of the IEEE754 standard, see the StackOverflow - # question, - # "Is there a floating point value of x, for which x-x == 0 is false?" - # http://stackoverflow.com/q/2686644 - fx += stop(fx) * (logpx - stop(logpx)) # Add zeros_like(logpx). - return math_ops.reduce_mean(fx, axis=axis, keepdims=keep_dims) - - -def _sample_mean(values): - """Mean over sample indices. In this module this is always [0].""" - return math_ops.reduce_mean(values, axis=[0]) - - -def _sample_max(values): - """Max over sample indices. In this module this is always [0].""" - return math_ops.reduce_max(values, axis=[0]) - - -def _get_samples(dist, z, n, seed): - """Check args and return samples.""" - with ops.name_scope('get_samples', values=[z, n]): - if (n is None) == (z is None): - raise ValueError( - 'Must specify exactly one of arguments "n" and "z". Found: ' - 'n = %s, z = %s' % (n, z)) - if n is not None: - return dist.sample(n, seed=seed) - else: - return ops.convert_to_tensor(z, name='z') diff --git a/tensorflow/contrib/benchmark_tools/README.md b/tensorflow/contrib/benchmark_tools/README.md deleted file mode 100644 index 42b81e32b1a..00000000000 --- a/tensorflow/contrib/benchmark_tools/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Benchmarking Scripts - -This directory tree contains a set of scripts that are useful when benchmarking -TensorFlow. diff --git a/tensorflow/contrib/benchmark_tools/gce/README.md b/tensorflow/contrib/benchmark_tools/gce/README.md deleted file mode 100644 index 66ab8ddc137..00000000000 --- a/tensorflow/contrib/benchmark_tools/gce/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Distributed Tensorflow on Google Compute Engine - -The scripts in this directory automate the work to run distributed TensorFlow on -a cluster of GCE instances. - -## Pre-work - -Before getting started, while GPUs on GCE are in Alpha, you will need to get -your project whitelisted in order to get access. These scripts will not work -until then. diff --git a/tensorflow/contrib/bigtable/BUILD b/tensorflow/contrib/bigtable/BUILD deleted file mode 100644 index 751991bba59..00000000000 --- a/tensorflow/contrib/bigtable/BUILD +++ /dev/null @@ -1,212 +0,0 @@ -# Cloud Bigtable client for TensorFlow - -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") -load( - "//tensorflow:tensorflow.bzl", - "tf_cc_test", - "tf_copts", - "tf_custom_op_library", - "tf_gen_op_libs", - "tf_gen_op_wrapper_py", - "tf_kernel_library", - "tf_py_test", -) - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -tf_custom_op_py_library( - name = "bigtable", - srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - dso = [ - ":python/ops/_bigtable.so", - ], - kernels = [ - ":bigtable_kernels", - ":bigtable_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":bigtable_ops", - "//tensorflow/contrib/data/python/ops:interleave_ops", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:util", - "//tensorflow/python/data", - ], -) - -KERNEL_FILES = [ - "kernels/bigtable_kernels.cc", - "kernels/bigtable_lookup_dataset_op.cc", - "kernels/bigtable_prefix_key_dataset_op.cc", - "kernels/bigtable_range_key_dataset_op.cc", - "kernels/bigtable_sample_keys_dataset_op.cc", - "kernels/bigtable_sample_key_pairs_dataset_op.cc", - "kernels/bigtable_scan_dataset_op.cc", -] - -tf_custom_op_library( - name = "python/ops/_bigtable.so", - srcs = KERNEL_FILES + [ - "ops/bigtable_ops.cc", - ], - deps = [ - ":bigtable_lib_cc", - ":bigtable_range_helpers", - "@com_github_googlecloudplatform_google_cloud_cpp//google/cloud/bigtable:bigtable_client", - ], -) - -tf_gen_op_wrapper_py( - name = "bigtable_ops", - deps = [":bigtable_ops_op_lib"], -) - -tf_gen_op_libs( - op_lib_names = [ - "bigtable_ops", - "bigtable_test_ops", - ], -) - -tf_kernel_library( - name = "bigtable_kernels", - srcs = KERNEL_FILES, - deps = [ - ":bigtable_lib_cc", - ":bigtable_range_helpers", - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - "@com_github_googlecloudplatform_google_cloud_cpp//google/cloud/bigtable:bigtable_client", - ], -) - -# A library for use in the bigtable kernels. -cc_library( - name = "bigtable_lib_cc", - srcs = ["kernels/bigtable_lib.cc"], - hdrs = ["kernels/bigtable_lib.h"], - deps = [ - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - "@com_github_googlecloudplatform_google_cloud_cpp//google/cloud/bigtable:bigtable_client", - ], -) - -cc_library( - name = "bigtable_range_helpers", - srcs = ["kernels/bigtable_range_helpers.cc"], - hdrs = ["kernels/bigtable_range_helpers.h"], - deps = [ - "//tensorflow/core:framework_headers_lib", - ], -) - -cc_library( - name = "bigtable_test_client", - srcs = ["kernels/test_kernels/bigtable_test_client.cc"], - hdrs = ["kernels/test_kernels/bigtable_test_client.h"], - deps = [ - "//tensorflow/core:framework_headers_lib", - "@com_github_googleapis_googleapis//:bigtable_protos", - "@com_github_googlecloudplatform_google_cloud_cpp//google/cloud/bigtable:bigtable_client", - "@com_googlesource_code_re2//:re2", - ], -) - -tf_cc_test( - name = "bigtable_test_client_test", - srcs = ["kernels/test_kernels/bigtable_test_client_test.cc"], - tags = ["manual"], - deps = [ - ":bigtable_test_client", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "@com_github_googlecloudplatform_google_cloud_cpp//google/cloud/bigtable:bigtable_client", - ], -) - -tf_cc_test( - name = "bigtable_range_helpers_test", - size = "small", - srcs = ["kernels/bigtable_range_helpers_test.cc"], - deps = [ - ":bigtable_range_helpers", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -tf_gen_op_wrapper_py( - name = "bigtable_test_ops", - deps = [":bigtable_test_ops_op_lib"], -) - -tf_custom_op_library( - name = "python/kernel_tests/_bigtable_test.so", - srcs = [ - "kernels/test_kernels/bigtable_test_client_op.cc", - "ops/bigtable_test_ops.cc", - ], - deps = [ - ":bigtable_lib_cc", - ":bigtable_test_client", - "@com_googlesource_code_re2//:re2", - ], -) - -# Don't use tf_kernel_library because it prevents access to strings/stringprintf.h -cc_library( - name = "bigtable_test_kernels", - srcs = [ - "kernels/test_kernels/bigtable_test_client_op.cc", - ], - copts = tf_copts(), - linkstatic = 1, - deps = [ - ":bigtable_lib_cc", - ":bigtable_test_client", - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - "@com_googlesource_code_re2//:re2", - ], - alwayslink = 1, -) - -tf_custom_op_py_library( - name = "bigtable_test_py", - dso = [ - ":python/kernel_tests/_bigtable_test.so", - ], - kernels = [ - ":bigtable_test_kernels", - ":bigtable_test_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":bigtable_test_ops", - ], -) - -tf_py_test( - name = "bigtable_ops_test", - size = "small", - srcs = ["python/kernel_tests/bigtable_ops_test.py"], - additional_deps = [ - ":bigtable", - ":bigtable_test_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:util", - ], - tags = ["manual"], -) diff --git a/tensorflow/contrib/bigtable/README.md b/tensorflow/contrib/bigtable/README.md deleted file mode 100644 index 79052bee35c..00000000000 --- a/tensorflow/contrib/bigtable/README.md +++ /dev/null @@ -1,346 +0,0 @@ -# Google Cloud Bigtable - -[Cloud Bigtable](https://cloud.google.com/bigtable/) is a high -performance storage system that can store and serve training data. This contrib -package contains an experimental integration with TensorFlow. - -> **Status: Highly experimental.** The current implementation is very much in -> flux. Please use at your own risk! :-) - -The TensorFlow integration with Cloud Bigtable is optimized for common -TensorFlow usage and workloads. It is currently optimized for reading from Cloud -Bigtable at high speed, in particular to feed modern accelerators. For -general-purpose Cloud Bigtable -APIs, see the [official Cloud Bigtable client library documentation][clientdoc]. - -[clientdoc]: https://cloud.google.com/bigtable/docs/reference/libraries - -## Sample Use - -There are three main reading styles supported by the `BigtableTable` class: - - 1. **Reading keys**: Read only the row keys in a table. Keys are returned in - sorted order from the table. Most key reading operations retrieve all keys - in a contiguous range, however the `sample_keys` operation skips keys, and - operates on the whole table (and not a contiguous subset). - 2. **Retrieving a row's values**: Given a row key, look up the data associated - with a defined set of columns. This operation takes advantage of Cloud - Bigtable's low-latency and excellent support for random access. - 3. **Scanning ranges**: Given a contiguous range of rows retrieve both the row - key and the data associated with a fixed set of columns. This operation - takes advantage of Cloud Bigtable's high throughput scans, and is the most - efficient way to read data. - -When using the Cloud Bigtable API, the workflow is: - - 1. Create a `BigtableClient` object. - 2. Use the `BigtableClient` to create `BigtableTable` objects corresponding to - each table in the Cloud Bigtable instance you would like to access. - 3. Call methods on the `BigtableTable` object to create `tf.data.Dataset`s to - retrieve data. - -The following is an example for how to read all row keys with the prefix -`train-`. - -```python -import tensorflow as tf - -GCP_PROJECT_ID = '' -BIGTABLE_INSTANCE_ID = '' -BIGTABLE_TABLE_NAME = '' -PREFIX = 'train-' - -def main(): - tf.enable_eager_execution() - - client = tf.contrib.cloud.BigtableClient(GCP_PROJECT_ID, BIGTABLE_INSTANCE_ID) - table = client.table(BIGTABLE_TABLE_NAME) - dataset = table.keys_by_prefix_dataset(PREFIX) - - print('Retrieving rows:') - row_index = 0 - for row_key in dataset: - print('Row key %d: %s' % (row_index, row_key)) - row_index += 1 - print('Finished reading data!') - -if __name__ == '__main__': - main() - -``` - -### Reading row keys - -Read only the row keys in a table. Keys are returned in sorted order from the -table. Most key reading operations retrieve all keys in a contiguous range, -however the `sample_keys` operation skips keys, and operates on the whole table -(and not a contiguous subset). - -There are 3 methods to retrieve row keys: - - - `table.keys_by_range_dataset(start, end)`: Retrieve row keys starting with - `start`, and ending with `end`. The range is "half-open", and thus it - includes `start` if `start` is present in the table. It does not include - `end`. - - `table.keys_by_prefix_dataset(prefix)`: Retrieves all row keys that start - with `prefix`. It includes the row key `prefix` if present in the table. - - `table.sample_keys()`: Retrieves a sampling of keys from the underlying - table. This is often useful in conjunction with parallel scans. - -### Reading cell values given a row key - -Given a dataset producing row keys, you can use the `table.lookup_columns` -transformation to retrieve values. Example: - -```python -key_dataset = tf.data.Dataset.from_tensor_slices([ - 'row_key_1', - 'other_row_key', - 'final_row_key', -]) -values_dataset = key_dataset.apply( - table.lookup_columns(('my_column_family', 'column_name'), - ('other_cf', 'col'))) -training_data = values_dataset.map(my_parsing_function) # ... -``` - -### Scanning ranges -Given a contiguous range of rows retrieve both the row key and the data -associated with a fixed set of columns. Scanning is the most efficient way to -retrieve data from Cloud Bigtable and is thus a very common API for high -performance data pipelines. To construct a scanning `tf.data.Dataset` from a -`BigtableTable` object, call one of the following methods: - - - `table.scan_prefix(prefix, ...)` - - `table.scan_range(start, end, ...)` - - `table.parallel_scan_prefix(prefix, ...)` - - `table.parallel_scan_range(start, end, ...)` - -Aside from the specification of the contiguous range of rows, they all take the -following arguments: - - - `probability`: (Optional.) A float between 0 (exclusive) and 1 (inclusive). - A non-1 value indicates to probabilistically sample rows with the - provided probability. - - `columns`: The columns to read. (See below.) - - `**kwargs`: The columns to read. (See below.) - -In addition the two parallel operations accept the following optional argument: -`num_parallel_scans` which configures the number of parallel Cloud Bigtable scan -operations to run. A reasonable default is automatically chosen for small -Cloud Bigtable clusters. If you have a large cluster, or an extremely demanding -workload, you can tune this value to optimize performance. - -#### Specifying columns to read when scanning - -All of the scan operations allow you to specify the column family and columns -in the same ways. - -##### Using `columns` - -The first way to specify the data to read is via the `columns` parameter. The -value should be a tuple (or list of tuples) of strings. The first string in the -tuple is the column family, and the second string in the tuple is the column -qualifier. - -##### Using `**kwargs` - -The second way to specify the data to read is via the `**kwargs` parameter, -which you can use to specify keyword arguments corresponding to the columns that -you want to read. The keyword to use is the column family name, and the argument -value should be either a string, or a tuple of strings, specifying the column -qualifiers (column names). - -Although using `**kwargs` has the advantage of requiring less typing, it is not -future-proof in all cases. (If we add a new parameter to the scan functions that -has the same name as your column family, your code will break.) - -##### Examples - -Below are two equivalent snippets for how to specify which columns to read: - -```python -ds1 = table.scan_range("row_start", "row_end", columns=[("cfa", "c1"), - ("cfa", "c2"), - ("cfb", "c3")]) -ds2 = table.scan_range("row_start", "row_end", cfa=["c1", "c2"], cfb="c3") -``` - -In this example, we are reading 3 columns from a total of 2 column families. -From the `cfa` column family, we are reading columns `c1`, and `c2`. From the -second column family (`cfb`), we are reading `c3`. Both `ds1` and `ds2` will -output elements of the following types (`tf.string`, `tf.string`, `tf.string`, -`tf.string`). The first `tf.string` is the row key, the second `tf.string` is -the latest data in cell `cfa:c1`, the third corresponds to `cfa:c2`, and the -final one is `cfb:c3`. - -#### Determinism when scanning - -While the non-parallel scan operations are fully deterministic, the parallel -scan operations are not. If you would like to scan in parallel without losing -determinism, you can build up the `parallel_interleave` yourself. As an example, -say we wanted to scan all rows between `training_data_00000`, and -`training_data_90000`, we can use the following code snippet: - -```python -table = # ... -columns = [('cf1', 'col1'), ('cf1', 'col2')] -NUM_PARALLEL_READS = # ... -ds = tf.data.Dataset.range(9).shuffle(10) -def interleave_fn(index): - # Given a starting index, create 2 strings to be the start and end - start_idx = index - end_idx = index + 1 - start_idx_str = tf.as_string(start_idx * 10000, width=5, fill='0') - end_idx_str = tf.as_string(end_idx * 10000, width=5, fill='0') - start = tf.string_join(['training_data_', start_idx_str]) - end = tf.string_join(['training_data_', end_idx_str]) - return table.scan_range(start_idx, end_idx, columns=columns) -ds = ds.apply(tf.data.experimental.parallel_interleave( - interleave_fn, cycle_length=NUM_PARALLEL_READS, prefetch_input_elements=1)) -``` - -> Note: you should divide up the key range into more sub-ranges for increased -> parallelism. - -## Writing to Cloud Bigtable - -In order to simplify getting started, this package provides basic support for -writing data into Cloud Bigtable. - -> Note: The implementation is not optimized for performance! Please consider -> using alternative frameworks such as Apache Beam / Cloud Dataflow for -> production workloads. - -Below is an example for how to write a trivial dataset into Cloud Bigtable. - -```python -import tensorflow as tf - -GCP_PROJECT_ID = '' -BIGTABLE_INSTANCE_ID = '' -BIGTABLE_TABLE_NAME = '' -COLUMN_FAMILY = '' -COLUMN_QUALIFIER = '' - -def make_dataset(): - """Makes a dataset to write to Cloud Bigtable.""" - return tf.data.Dataset.from_tensor_slices([ - 'training_data_1', - 'training_data_2', - 'training_data_3', - ]) - -def make_row_key_dataset(): - """Makes a dataset of strings used for row keys. - - The strings are of the form: `fake-data-` followed by a sequential counter. - For example, this dataset would contain the following elements: - - - fake-data-00000001 - - fake-data-00000002 - - ... - - fake-data-23498103 - """ - counter_dataset = tf.data.experimental.Counter() - width = 8 - row_key_prefix = 'fake-data-' - ds = counter_dataset.map(lambda index: tf.as_string(index, - width=width, - fill='0')) - ds = ds.map(lambda idx_str: tf.string_join([row_key_prefix, idx_str])) - return ds - - -def main(): - client = tf.contrib.cloud.BigtableClient(GCP_PROJECT_ID, BIGTABLE_INSTANCE_ID) - table = client.table(BIGTABLE_TABLE_NAME) - dataset = make_dataset() - index_dataset = make_row_key_dataset() - aggregate_dataset = tf.data.Dataset.zip((index_dataset, dataset)) - write_op = table.write(aggregate_dataset, column_families=[COLUMN_FAMILY], - columns=[COLUMN_QUALIFIER]) - - with tf.Session() as sess: - print('Starting transfer.') - sess.run(write_op) - print('Transfer complete.') - -if __name__ == '__main__': - main() -``` - -## Sample applications and architectures - -While most machine learning applications are well suited by a high performance -distributed file system, there are certain applications where using Cloud -Bigtable works extremely well. - -### Perfect Shuffling - -Normally, training data is stored in flat files, and a combination of -(1) `tf.data.Dataset.interleave` (or `parallel_interleave`), (2) -`tf.data.Dataset.shuffle`, and (3) writing the data in an unsorted order in the -data files in the first place, provides enough randomization to ensure models -train efficiently. However, if you would like perfect shuffling, you can use -Cloud Bigtable's low-latency random access capabilities. Create a -`tf.data.Dataset` that generates the keys in a perfectly random order (or read -all the keys into memory and use a shuffle buffer sized to fit all of them for a -perfect random shuffle using `tf.data.Dataset.shuffle`), and then use -`lookup_columns` to retrieve the training data. - -### Distributed Reinforcement Learning - -Sophisticated reinforcement learning algorithms are commonly trained across a -distributed cluster. (See [IMPALA by DeepMind][impala].) One part of the cluster -runs self-play, while the other part of the cluster learns a new version of the -model based on the training data generated by self-play. The new model version -is then distributed to the self-play half of the cluster, and new training data -is generated to continue the cycle. - -In such a configuration, because there is value in training on the freshest -examples, a storage service like Cloud Bigtable can be used to store and -serve the generated training data. When using Cloud Bigtable, there is no need -to aggregate the examples into large batch files, but the examples can instead -be written as soon as they are generated, and then retrieved at high speed. - -[impala]: https://arxiv.org/abs/1802.01561 - -## Common Gotchas! - -### gRPC Certificates - -If you encounter a log line that includes the following: - -``` -"description":"Failed to load file", [...], -"filename":"/usr/share/grpc/roots.pem" -``` - -you can solve it via either of the following approaches: - -* copy the [gRPC `roots.pem` file][grpcPem] to - `/usr/share/grpc/roots.pem` on your local machine, which is the default - location where gRPC will look for this file -* export the environment variable `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH` to point to - the full path of the gRPC `roots.pem` file on your file system if it's in a - different location - -[grpcPem]: https://github.com/grpc/grpc/blob/master/etc/roots.pem - -### Permission denied errors - -The TensorFlow Cloud Bigtable client will search for credentials to use in the -process's environment. It will use the first credentials it finds if multiple -are available. - - - **Compute Engine**: When running on Compute Engine, the client will often use - the service account from the virtual machine's metadata service. Be sure to - authorize your Compute Engine VM to have access to the Cloud Bigtable service - when creating your VM, or [update the VM's scopes][update-vm-scopes] on a - running VM if you run into this issue. - - **Cloud TPU**: Your Cloud TPUs run with the designated Cloud TPU service - account dedicated to your GCP project. Ensure the service account has been - authorized via the Cloud Console to access your Cloud Bigtable instances. - -[update-vm-scopes]: https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#changeserviceaccountandscopes diff --git a/tensorflow/contrib/bigtable/__init__.py b/tensorflow/contrib/bigtable/__init__.py deleted file mode 100644 index b7d89c98420..00000000000 --- a/tensorflow/contrib/bigtable/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Cloud Bigtable Client for TensorFlow. - -This contrib package allows TensorFlow to interface directly with Cloud Bigtable -for high-speed data loading. - -@@BigtableClient -@@BigtableTable - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.bigtable.python.ops.bigtable_api import BigtableClient -from tensorflow.contrib.bigtable.python.ops.bigtable_api import BigtableTable - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'BigtableClient', - 'BigtableTable', -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_kernels.cc b/tensorflow/contrib/bigtable/kernels/bigtable_kernels.cc deleted file mode 100644 index 1e6de7ee17e..00000000000 --- a/tensorflow/contrib/bigtable/kernels/bigtable_kernels.cc +++ /dev/null @@ -1,360 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/bigtable/kernels/bigtable_lib.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/lib/core/refcount.h" -#include "tensorflow/core/lib/core/threadpool.h" - -namespace tensorflow { -namespace { - -class BigtableClientOp : public OpKernel { - public: - explicit BigtableClientOp(OpKernelConstruction* ctx) : OpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("project_id", &project_id_)); - OP_REQUIRES_OK(ctx, ctx->GetAttr("instance_id", &instance_id_)); - OP_REQUIRES(ctx, !project_id_.empty(), - errors::InvalidArgument("project_id must be non-empty")); - OP_REQUIRES(ctx, !instance_id_.empty(), - errors::InvalidArgument("instance_id must be non-empty")); - - OP_REQUIRES_OK( - ctx, ctx->GetAttr("connection_pool_size", &connection_pool_size_)); - // If left unset by the client code, set it to a default of 100. Note: the - // cloud-cpp default of 4 concurrent connections is far too low for high - // performance streaming. - if (connection_pool_size_ == -1) { - connection_pool_size_ = 100; - } - - OP_REQUIRES_OK(ctx, ctx->GetAttr("max_receive_message_size", - &max_receive_message_size_)); - // If left unset by the client code, set it to a default of 100. Note: the - // cloud-cpp default of 4 concurrent connections is far too low for high - // performance streaming. - if (max_receive_message_size_ == -1) { - max_receive_message_size_ = 1 << 24; // 16 MBytes - } - OP_REQUIRES(ctx, max_receive_message_size_ > 0, - errors::InvalidArgument("connection_pool_size must be > 0")); - } - - ~BigtableClientOp() override { - if (cinfo_.resource_is_private_to_kernel()) { - if (!cinfo_.resource_manager() - ->Delete(cinfo_.container(), - cinfo_.name()) - .ok()) { - // Do nothing; the resource can have been deleted by session resets. - } - } - } - - void Compute(OpKernelContext* ctx) override LOCKS_EXCLUDED(mu_) { - mutex_lock l(mu_); - if (!initialized_) { - ResourceMgr* mgr = ctx->resource_manager(); - OP_REQUIRES_OK(ctx, cinfo_.Init(mgr, def())); - BigtableClientResource* resource; - OP_REQUIRES_OK( - ctx, - mgr->LookupOrCreate( - cinfo_.container(), cinfo_.name(), &resource, - [this, ctx]( - BigtableClientResource** ret) EXCLUSIVE_LOCKS_REQUIRED(mu_) { - auto client_options = - google::cloud::bigtable::ClientOptions() - .set_connection_pool_size(connection_pool_size_) - .set_data_endpoint("batch-bigtable.googleapis.com"); - auto channel_args = client_options.channel_arguments(); - channel_args.SetMaxReceiveMessageSize( - max_receive_message_size_); - channel_args.SetUserAgentPrefix("tensorflow"); - channel_args.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 0); - channel_args.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, 60 * 1000); - client_options.set_channel_arguments(channel_args); - std::shared_ptr client = - google::cloud::bigtable::CreateDefaultDataClient( - project_id_, instance_id_, std::move(client_options)); - *ret = new BigtableClientResource(project_id_, instance_id_, - std::move(client)); - return Status::OK(); - })); - core::ScopedUnref resource_cleanup(resource); - initialized_ = true; - } - OP_REQUIRES_OK(ctx, MakeResourceHandleToOutput( - ctx, 0, cinfo_.container(), cinfo_.name(), - MakeTypeIndex())); - } - - private: - string project_id_; - string instance_id_; - int64 connection_pool_size_; - int32 max_receive_message_size_; - - mutex mu_; - ContainerInfo cinfo_ GUARDED_BY(mu_); - bool initialized_ GUARDED_BY(mu_) = false; -}; - -REGISTER_KERNEL_BUILDER(Name("BigtableClient").Device(DEVICE_CPU), - BigtableClientOp); - -class BigtableTableOp : public OpKernel { - public: - explicit BigtableTableOp(OpKernelConstruction* ctx) : OpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("table_name", &table_)); - OP_REQUIRES(ctx, !table_.empty(), - errors::InvalidArgument("table_name must be non-empty")); - } - - ~BigtableTableOp() override { - if (cinfo_.resource_is_private_to_kernel()) { - if (!cinfo_.resource_manager() - ->Delete(cinfo_.container(), - cinfo_.name()) - .ok()) { - // Do nothing; the resource can have been deleted by session resets. - } - } - } - - void Compute(OpKernelContext* ctx) override LOCKS_EXCLUDED(mu_) { - mutex_lock l(mu_); - if (!initialized_) { - ResourceMgr* mgr = ctx->resource_manager(); - OP_REQUIRES_OK(ctx, cinfo_.Init(mgr, def())); - - core::RefCountPtr client_resource; - OP_REQUIRES_OK( - ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &client_resource)); - - BigtableTableResource* resource; - OP_REQUIRES_OK(ctx, - mgr->LookupOrCreate( - cinfo_.container(), cinfo_.name(), &resource, - [this, &client_resource](BigtableTableResource** ret) { - *ret = new BigtableTableResource( - client_resource.get(), table_); - return Status::OK(); - })); - initialized_ = true; - } - OP_REQUIRES_OK(ctx, MakeResourceHandleToOutput( - ctx, 0, cinfo_.container(), cinfo_.name(), - MakeTypeIndex())); - } - - private: - string table_; // Note: this is const after construction. - - mutex mu_; - ContainerInfo cinfo_ GUARDED_BY(mu_); - bool initialized_ GUARDED_BY(mu_) = false; -}; - -REGISTER_KERNEL_BUILDER(Name("BigtableTable").Device(DEVICE_CPU), - BigtableTableOp); - -} // namespace - -namespace data { -namespace { - -class ToBigtableOp : public AsyncOpKernel { - public: - explicit ToBigtableOp(OpKernelConstruction* ctx) - : AsyncOpKernel(ctx), - thread_pool_(new thread::ThreadPool( - ctx->env(), ThreadOptions(), - strings::StrCat("to_bigtable_op_", SanitizeThreadSuffix(name())), - /* num_threads = */ 1, /* low_latency_hint = */ false)) {} - - void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override { - // The call to `iterator->GetNext()` may block and depend on an - // inter-op thread pool thread, so we issue the call from the - // owned thread pool. - thread_pool_->Schedule([this, ctx, done]() { - const Tensor* column_families_tensor; - OP_REQUIRES_OK_ASYNC( - ctx, ctx->input("column_families", &column_families_tensor), done); - OP_REQUIRES_ASYNC( - ctx, column_families_tensor->dims() == 1, - errors::InvalidArgument("`column_families` must be a vector."), done); - - const Tensor* columns_tensor; - OP_REQUIRES_OK_ASYNC(ctx, ctx->input("columns", &columns_tensor), done); - OP_REQUIRES_ASYNC(ctx, columns_tensor->dims() == 1, - errors::InvalidArgument("`columns` must be a vector."), - done); - OP_REQUIRES_ASYNC( - ctx, - columns_tensor->NumElements() == - column_families_tensor->NumElements(), - errors::InvalidArgument("len(column_families) != len(columns)"), - done); - - std::vector column_families; - column_families.reserve(column_families_tensor->NumElements()); - std::vector columns; - columns.reserve(column_families_tensor->NumElements()); - for (uint64 i = 0; i < column_families_tensor->NumElements(); ++i) { - column_families.push_back(column_families_tensor->flat()(i)); - columns.push_back(columns_tensor->flat()(i)); - } - - DatasetBase* dataset; - OP_REQUIRES_OK_ASYNC( - ctx, GetDatasetFromVariantTensor(ctx->input(1), &dataset), done); - - std::unique_ptr iterator; - OP_REQUIRES_OK_ASYNC( - ctx, - dataset->MakeIterator(IteratorContext(ctx), "ToBigtableOpIterator", - &iterator), - done); - - int64 timestamp_int; - OP_REQUIRES_OK_ASYNC( - ctx, ParseScalarArgument(ctx, "timestamp", ×tamp_int), - done); - OP_REQUIRES_ASYNC(ctx, timestamp_int >= -1, - errors::InvalidArgument("timestamp must be >= -1"), - done); - - core::RefCountPtr resource; - OP_REQUIRES_OK_ASYNC( - ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &resource), done); - - std::vector components; - components.reserve(dataset->output_dtypes().size()); - bool end_of_sequence = false; - do { - ::google::cloud::bigtable::BulkMutation mutation; - // TODO(saeta): Make # of mutations configurable. - for (uint64 i = 0; i < 100 && !end_of_sequence; ++i) { - OP_REQUIRES_OK_ASYNC(ctx, - iterator->GetNext(IteratorContext(ctx), - &components, &end_of_sequence), - done); - if (!end_of_sequence) { - OP_REQUIRES_OK_ASYNC( - ctx, - CreateMutation(std::move(components), column_families, columns, - timestamp_int, &mutation), - done); - } - components.clear(); - } - ::google::cloud::Status mutation_status; - std::vector<::google::cloud::bigtable::FailedMutation> failures = - resource->table().BulkApply(mutation); - if (!failures.empty()) { - mutation_status = failures.front().status(); - if (!mutation_status.ok()) { - LOG(ERROR) << "Failure applying mutation: " - << mutation_status.code() << " - " - << mutation_status.message() << "."; - } - ::google::bigtable::v2::MutateRowsRequest request; - mutation.MoveTo(&request); - for (const auto& failure : failures) { - LOG(ERROR) << "Failure applying mutation on row (" - << failure.original_index() << "): " - << request.entries(failure.original_index()).row_key() - << " - error: " << failure.status().message() << "."; - } - } - OP_REQUIRES_ASYNC( - ctx, failures.empty(), - errors::Unknown("Failure while writing to Cloud Bigtable: ", - mutation_status.code(), " - ", - mutation_status.message(), - "; # of mutation failures: ", failures.size(), - ". See the log for the specific error details."), - done); - } while (!end_of_sequence); - done(); - }); - } - - private: - static string SanitizeThreadSuffix(string suffix) { - string clean; - for (int i = 0; i < suffix.size(); ++i) { - const char ch = suffix[i]; - if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || - (ch >= '0' && ch <= '9') || ch == '_' || ch == '-') { - clean += ch; - } else { - clean += '_'; - } - } - return clean; - } - - Status CreateMutation( - std::vector tensors, const std::vector& column_families, - const std::vector& columns, int64 timestamp_int, - ::google::cloud::bigtable::BulkMutation* bulk_mutation) { - if (tensors.size() != column_families.size() + 1) { - return errors::InvalidArgument( - "Iterator produced a set of Tensors shorter than expected"); - } - ::google::cloud::bigtable::SingleRowMutation mutation( - std::move(tensors[0].scalar()())); - std::chrono::milliseconds timestamp(timestamp_int); - for (size_t i = 1; i < tensors.size(); ++i) { - if (!TensorShapeUtils::IsScalar(tensors[i].shape())) { - return errors::Internal("Output tensor ", i, " was not a scalar"); - } - if (timestamp_int == -1) { - mutation.emplace_back(::google::cloud::bigtable::SetCell( - column_families[i - 1], columns[i - 1], - std::move(tensors[i].scalar()()))); - } else { - mutation.emplace_back(::google::cloud::bigtable::SetCell( - column_families[i - 1], columns[i - 1], timestamp, - std::move(tensors[i].scalar()()))); - } - } - bulk_mutation->emplace_back(std::move(mutation)); - return Status::OK(); - } - - template - Status ParseScalarArgument(OpKernelContext* ctx, StringPiece argument_name, - T* output) { - const Tensor* argument_t; - TF_RETURN_IF_ERROR(ctx->input(argument_name, &argument_t)); - if (!TensorShapeUtils::IsScalar(argument_t->shape())) { - return errors::InvalidArgument(argument_name, " must be a scalar"); - } - *output = argument_t->scalar()(); - return Status::OK(); - } - - std::unique_ptr thread_pool_; -}; - -REGISTER_KERNEL_BUILDER(Name("DatasetToBigtable").Device(DEVICE_CPU), - ToBigtableOp); - -} // namespace -} // namespace data -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_lib.cc b/tensorflow/contrib/bigtable/kernels/bigtable_lib.cc deleted file mode 100644 index 13658558bc0..00000000000 --- a/tensorflow/contrib/bigtable/kernels/bigtable_lib.cc +++ /dev/null @@ -1,79 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/bigtable/kernels/bigtable_lib.h" - -namespace tensorflow { -namespace { -::tensorflow::error::Code GcpErrorCodeToTfErrorCode( - ::google::cloud::StatusCode code) { - switch (code) { - case ::google::cloud::StatusCode::kOk: - return ::tensorflow::error::OK; - case ::google::cloud::StatusCode::kCancelled: - return ::tensorflow::error::CANCELLED; - case ::google::cloud::StatusCode::kUnknown: - return ::tensorflow::error::UNKNOWN; - case ::google::cloud::StatusCode::kInvalidArgument: - return ::tensorflow::error::INVALID_ARGUMENT; - case ::google::cloud::StatusCode::kDeadlineExceeded: - return ::tensorflow::error::DEADLINE_EXCEEDED; - case ::google::cloud::StatusCode::kNotFound: - return ::tensorflow::error::NOT_FOUND; - case ::google::cloud::StatusCode::kAlreadyExists: - return ::tensorflow::error::ALREADY_EXISTS; - case ::google::cloud::StatusCode::kPermissionDenied: - return ::tensorflow::error::PERMISSION_DENIED; - case ::google::cloud::StatusCode::kUnauthenticated: - return ::tensorflow::error::UNAUTHENTICATED; - case ::google::cloud::StatusCode::kResourceExhausted: - return ::tensorflow::error::RESOURCE_EXHAUSTED; - case ::google::cloud::StatusCode::kFailedPrecondition: - return ::tensorflow::error::FAILED_PRECONDITION; - case ::google::cloud::StatusCode::kAborted: - return ::tensorflow::error::ABORTED; - case ::google::cloud::StatusCode::kOutOfRange: - return ::tensorflow::error::OUT_OF_RANGE; - case ::google::cloud::StatusCode::kUnimplemented: - return ::tensorflow::error::UNIMPLEMENTED; - case ::google::cloud::StatusCode::kInternal: - return ::tensorflow::error::INTERNAL; - case ::google::cloud::StatusCode::kUnavailable: - return ::tensorflow::error::UNAVAILABLE; - case ::google::cloud::StatusCode::kDataLoss: - return ::tensorflow::error::DATA_LOSS; - } -} -} // namespace - -Status GcpStatusToTfStatus(const ::google::cloud::Status& status) { - if (status.ok()) { - return Status::OK(); - } - return Status( - GcpErrorCodeToTfErrorCode(status.code()), - strings::StrCat("Error reading from Cloud Bigtable: ", status.message())); -} - -string RegexFromStringSet(const std::vector& strs) { - CHECK(!strs.empty()) << "The list of strings to turn into a regex was empty."; - std::unordered_set uniq(strs.begin(), strs.end()); - if (uniq.size() == 1) { - return *uniq.begin(); - } - return absl::StrJoin(uniq, "|"); -} - -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_lib.h b/tensorflow/contrib/bigtable/kernels/bigtable_lib.h deleted file mode 100644 index ce2bea0d759..00000000000 --- a/tensorflow/contrib/bigtable/kernels/bigtable_lib.h +++ /dev/null @@ -1,153 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_BIGTABLE_KERNELS_BIGTABLE_LIB_H_ -#define TENSORFLOW_CONTRIB_BIGTABLE_KERNELS_BIGTABLE_LIB_H_ - -#include "google/cloud/bigtable/data_client.h" -#include "google/cloud/bigtable/table.h" -#include "tensorflow/core/framework/dataset.h" -#include "tensorflow/core/framework/resource_mgr.h" - -namespace tensorflow { - -Status GcpStatusToTfStatus(const ::google::cloud::Status& status); - -string RegexFromStringSet(const std::vector& strs); - -class BigtableClientResource : public ResourceBase { - public: - BigtableClientResource( - string project_id, string instance_id, - std::shared_ptr client) - : project_id_(std::move(project_id)), - instance_id_(std::move(instance_id)), - client_(std::move(client)) {} - - std::shared_ptr get_client() { - return client_; - } - - string DebugString() const override { - return strings::StrCat("BigtableClientResource(project_id: ", project_id_, - ", instance_id: ", instance_id_, ")"); - } - - private: - const string project_id_; - const string instance_id_; - std::shared_ptr client_; -}; - -class BigtableTableResource : public ResourceBase { - public: - BigtableTableResource(BigtableClientResource* client, string table_name) - : client_(client), - table_name_(std::move(table_name)), - table_(client->get_client(), table_name_, - google::cloud::bigtable::AlwaysRetryMutationPolicy()) { - client_->Ref(); - } - - ~BigtableTableResource() override { client_->Unref(); } - - ::google::cloud::bigtable::Table& table() { return table_; } - - string DebugString() const override { - return strings::StrCat( - "BigtableTableResource(client: ", client_->DebugString(), - ", table: ", table_name_, ")"); - } - - private: - BigtableClientResource* client_; // Ownes one ref. - const string table_name_; - ::google::cloud::bigtable::Table table_; -}; - -namespace data { - -// BigtableReaderDatasetIterator is an abstract class for iterators from -// datasets that are "readers" (source datasets, not transformation datasets) -// that read from Bigtable. -template -class BigtableReaderDatasetIterator : public DatasetIterator { - public: - explicit BigtableReaderDatasetIterator( - const typename DatasetIterator::Params& params) - : DatasetIterator(params) {} - - Status GetNextInternal(IteratorContext* ctx, std::vector* out_tensors, - bool* end_of_sequence) override { - mutex_lock l(mu_); - TF_RETURN_IF_ERROR(EnsureIteratorInitialized()); - if (iterator_ == reader_->end()) { - *end_of_sequence = true; - return Status::OK(); - } - if (!*iterator_) { - return GcpStatusToTfStatus(iterator_->status()); - } - *end_of_sequence = false; - google::cloud::bigtable::Row& row = **iterator_; - Status s = ParseRow(ctx, row, out_tensors); - // Ensure we always advance. - ++iterator_; - return s; - } - - protected: - virtual ::google::cloud::bigtable::RowRange MakeRowRange() = 0; - virtual ::google::cloud::bigtable::Filter MakeFilter() = 0; - virtual Status ParseRow(IteratorContext* ctx, - const ::google::cloud::bigtable::Row& row, - std::vector* out_tensors) = 0; - - Status SaveInternal(IteratorStateWriter* writer) override { - return errors::Unimplemented("SaveInternal is currently not supported"); - } - - Status RestoreInternal(IteratorContext* ctx, - IteratorStateReader* reader) override { - return errors::Unimplemented("RestoreInternal is currently not supported"); - } - - private: - Status EnsureIteratorInitialized() EXCLUSIVE_LOCKS_REQUIRED(mu_) { - if (reader_) { - return Status::OK(); - } - - auto rows = MakeRowRange(); - auto filter = MakeFilter(); - - // Note: the this in `this->dataset()` below is necessary due to namespace - // name conflicts. - reader_.reset(new ::google::cloud::bigtable::RowReader( - this->dataset()->table()->table().ReadRows(rows, filter))); - iterator_ = reader_->begin(); - return Status::OK(); - } - - mutex mu_; - std::unique_ptr<::google::cloud::bigtable::RowReader> reader_ GUARDED_BY(mu_); - ::google::cloud::bigtable::RowReader::iterator iterator_ GUARDED_BY(mu_); -}; - -} // namespace data - -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BIGTABLE_KERNELS_BIGTABLE_LIB_H_ diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_lookup_dataset_op.cc b/tensorflow/contrib/bigtable/kernels/bigtable_lookup_dataset_op.cc deleted file mode 100644 index a69936236be..00000000000 --- a/tensorflow/contrib/bigtable/kernels/bigtable_lookup_dataset_op.cc +++ /dev/null @@ -1,248 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/bigtable/kernels/bigtable_lib.h" -#include "tensorflow/core/framework/op_kernel.h" - -namespace tensorflow { -namespace data { -namespace { - -class BigtableLookupDatasetOp : public UnaryDatasetOpKernel { - public: - using UnaryDatasetOpKernel::UnaryDatasetOpKernel; - - void MakeDataset(OpKernelContext* ctx, DatasetBase* input, - DatasetBase** output) override { - core::RefCountPtr table; - OP_REQUIRES_OK(ctx, LookupResource(ctx, HandleFromInput(ctx, 1), &table)); - - std::vector column_families; - std::vector columns; - OP_REQUIRES_OK(ctx, ParseVectorArgument(ctx, "column_families", - &column_families)); - OP_REQUIRES_OK(ctx, ParseVectorArgument(ctx, "columns", &columns)); - OP_REQUIRES( - ctx, column_families.size() == columns.size(), - errors::InvalidArgument("len(columns) != len(column_families)")); - - const uint64 num_outputs = columns.size() + 1; - std::vector output_shapes; - output_shapes.reserve(num_outputs); - DataTypeVector output_types; - output_types.reserve(num_outputs); - for (uint64 i = 0; i < num_outputs; ++i) { - output_shapes.push_back({}); - output_types.push_back(DT_STRING); - } - - *output = - new Dataset(ctx, input, table.get(), std::move(column_families), - std::move(columns), output_types, std::move(output_shapes)); - } - - private: - class Dataset : public DatasetBase { - public: - explicit Dataset(OpKernelContext* ctx, const DatasetBase* input, - BigtableTableResource* table, - std::vector column_families, - std::vector columns, - const DataTypeVector& output_types, - std::vector output_shapes) - : DatasetBase(DatasetContext(ctx)), - input_(input), - table_(table), - column_families_(std::move(column_families)), - columns_(std::move(columns)), - output_types_(output_types), - output_shapes_(std::move(output_shapes)), - filter_(MakeFilter(column_families_, columns_)) { - table_->Ref(); - input_->Ref(); - } - - ~Dataset() override { - table_->Unref(); - input_->Unref(); - } - - std::unique_ptr MakeIteratorInternal( - const string& prefix) const override { - return std::unique_ptr( - new Iterator({this, strings::StrCat(prefix, "::BigtableLookup")})); - } - - const DataTypeVector& output_dtypes() const override { - return output_types_; - } - - const std::vector& output_shapes() const override { - return output_shapes_; - } - - string DebugString() const override { - return "BigtableLookupDatasetOp::Dataset"; - } - - Status CheckExternalState() const override { - return errors::FailedPrecondition(DebugString(), - " depends on external state."); - } - - protected: - Status AsGraphDefInternal(SerializationContext* ctx, - DatasetGraphDefBuilder* b, - Node** output) const override { - return errors::Unimplemented(DebugString(), - " does not support serialization"); - } - - private: - static ::google::cloud::bigtable::Filter MakeFilter( - const std::vector& column_families, - const std::vector& columns) { - string column_family_regex = RegexFromStringSet(column_families); - string column_regex = RegexFromStringSet(columns); - - return ::google::cloud::bigtable::Filter::Chain( - ::google::cloud::bigtable::Filter::Latest(1), - ::google::cloud::bigtable::Filter::FamilyRegex(column_family_regex), - ::google::cloud::bigtable::Filter::ColumnRegex(column_regex)); - } - - class Iterator : public DatasetIterator { - public: - explicit Iterator(const Params& params) - : DatasetIterator(params) {} - - Status Initialize(IteratorContext* ctx) override { - return dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_); - } - - Status GetNextInternal(IteratorContext* ctx, - std::vector* out_tensors, - bool* end_of_sequence) override { - mutex_lock l(mu_); // Sequence requests. - std::vector input_tensors; - TF_RETURN_IF_ERROR( - input_impl_->GetNext(ctx, &input_tensors, end_of_sequence)); - if (*end_of_sequence) { - return Status::OK(); - } - if (input_tensors.size() != 1) { - return errors::InvalidArgument( - "Upstream iterator (", dataset()->input_->DebugString(), - ") did not produce a single `tf.string` `tf.Tensor`. It " - "produced ", - input_tensors.size(), " tensors."); - } - if (input_tensors[0].NumElements() == 0) { - return errors::InvalidArgument("Upstream iterator (", - dataset()->input_->DebugString(), - ") return an empty set of keys."); - } - if (input_tensors[0].NumElements() == 1) { - // Single key lookup. - ::google::cloud::StatusOr< - std::pair> - row = dataset()->table_->table().ReadRow( - input_tensors[0].scalar()(), dataset()->filter_); - if (!row.ok()) { - return GcpStatusToTfStatus(row.status()); - } - if (!row->first) { - return errors::DataLoss("Row key '", - input_tensors[0].scalar()(), - "' not found."); - } - TF_RETURN_IF_ERROR(ParseRow(ctx, row->second, out_tensors)); - } else { - // Batched get. - return errors::Unimplemented( - "BigtableLookupDataset doesn't yet support batched retrieval."); - } - return Status::OK(); - } - - protected: - Status SaveInternal(IteratorStateWriter* writer) override { - return errors::Unimplemented("SaveInternal is currently not supported"); - } - - Status RestoreInternal(IteratorContext* ctx, - IteratorStateReader* reader) override { - return errors::Unimplemented( - "RestoreInternal is currently not supported"); - } - - private: - Status ParseRow(IteratorContext* ctx, - const ::google::cloud::bigtable::Row& row, - std::vector* out_tensors) { - out_tensors->reserve(dataset()->columns_.size() + 1); - Tensor row_key_tensor(ctx->allocator({}), DT_STRING, {}); - row_key_tensor.scalar()() = tstring(row.row_key()); - out_tensors->emplace_back(std::move(row_key_tensor)); - - if (row.cells().size() > 2 * dataset()->columns_.size()) { - LOG(WARNING) << "An excessive number of columns (" - << row.cells().size() - << ") were retrieved when reading row: " - << row.row_key(); - } - - for (uint64 i = 0; i < dataset()->columns_.size(); ++i) { - Tensor col_tensor(ctx->allocator({}), DT_STRING, {}); - bool found_column = false; - for (auto cell_itr = row.cells().begin(); - !found_column && cell_itr != row.cells().end(); ++cell_itr) { - if (cell_itr->family_name() == dataset()->column_families_[i] && - tstring(cell_itr->column_qualifier()) == - dataset()->columns_[i]) { - col_tensor.scalar()() = tstring(cell_itr->value()); - found_column = true; - } - } - if (!found_column) { - return errors::DataLoss("Column ", dataset()->column_families_[i], - ":", dataset()->columns_[i], - " not found in row: ", row.row_key()); - } - out_tensors->emplace_back(std::move(col_tensor)); - } - return Status::OK(); - } - - mutex mu_; - std::unique_ptr input_impl_ GUARDED_BY(mu_); - }; - - const DatasetBase* const input_; - BigtableTableResource* table_; - const std::vector column_families_; - const std::vector columns_; - const DataTypeVector output_types_; - const std::vector output_shapes_; - const ::google::cloud::bigtable::Filter filter_; - }; -}; - -REGISTER_KERNEL_BUILDER(Name("BigtableLookupDataset").Device(DEVICE_CPU), - BigtableLookupDatasetOp); - -} // namespace -} // namespace data -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_prefix_key_dataset_op.cc b/tensorflow/contrib/bigtable/kernels/bigtable_prefix_key_dataset_op.cc deleted file mode 100644 index 6af5c6d0fc2..00000000000 --- a/tensorflow/contrib/bigtable/kernels/bigtable_prefix_key_dataset_op.cc +++ /dev/null @@ -1,121 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/bigtable/kernels/bigtable_lib.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/lib/core/refcount.h" - -namespace tensorflow { -namespace data { -namespace { - -class BigtablePrefixKeyDatasetOp : public DatasetOpKernel { - public: - using DatasetOpKernel::DatasetOpKernel; - - void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override { - tstring prefix; - OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "prefix", &prefix)); - - core::RefCountPtr resource; - OP_REQUIRES_OK(ctx, - LookupResource(ctx, HandleFromInput(ctx, 0), &resource)); - *output = new Dataset(ctx, resource.get(), std::move(prefix)); - } - - private: - class Dataset : public DatasetBase { - public: - explicit Dataset(OpKernelContext* ctx, BigtableTableResource* table, - string prefix) - : DatasetBase(DatasetContext(ctx)), - table_(table), - prefix_(std::move(prefix)) { - table_->Ref(); - } - - ~Dataset() override { table_->Unref(); } - - std::unique_ptr MakeIteratorInternal( - const string& prefix) const override { - return std::unique_ptr( - new Iterator({this, strings::StrCat(prefix, "::BigtablePrefixKey")})); - } - - const DataTypeVector& output_dtypes() const override { - static DataTypeVector* dtypes = new DataTypeVector({DT_STRING}); - return *dtypes; - } - - const std::vector& output_shapes() const override { - static std::vector* shapes = - new std::vector({{}}); - return *shapes; - } - - string DebugString() const override { - return "BigtablePrefixKeyDatasetOp::Dataset"; - } - - BigtableTableResource* table() const { return table_; } - - Status CheckExternalState() const override { - return errors::FailedPrecondition(DebugString(), - " depends on external state."); - } - - protected: - Status AsGraphDefInternal(SerializationContext* ctx, - DatasetGraphDefBuilder* b, - Node** output) const override { - return errors::Unimplemented(DebugString(), - " does not support serialization"); - } - - private: - class Iterator : public BigtableReaderDatasetIterator { - public: - explicit Iterator(const Params& params) - : BigtableReaderDatasetIterator(params) {} - - ::google::cloud::bigtable::RowRange MakeRowRange() override { - return ::google::cloud::bigtable::RowRange::Prefix(dataset()->prefix_); - } - ::google::cloud::bigtable::Filter MakeFilter() override { - return ::google::cloud::bigtable::Filter::Chain( - ::google::cloud::bigtable::Filter::CellsRowLimit(1), - ::google::cloud::bigtable::Filter::StripValueTransformer()); - } - Status ParseRow(IteratorContext* ctx, - const ::google::cloud::bigtable::Row& row, - std::vector* out_tensors) override { - Tensor output_tensor(ctx->allocator({}), DT_STRING, {}); - output_tensor.scalar()() = tstring(row.row_key()); - out_tensors->emplace_back(std::move(output_tensor)); - return Status::OK(); - } - }; - - BigtableTableResource* const table_; - const string prefix_; - }; -}; - -REGISTER_KERNEL_BUILDER(Name("BigtablePrefixKeyDataset").Device(DEVICE_CPU), - BigtablePrefixKeyDatasetOp); - -} // namespace -} // namespace data -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_range_helpers.cc b/tensorflow/contrib/bigtable/kernels/bigtable_range_helpers.cc deleted file mode 100644 index 51965f62144..00000000000 --- a/tensorflow/contrib/bigtable/kernels/bigtable_range_helpers.cc +++ /dev/null @@ -1,68 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/bigtable/kernels/bigtable_range_helpers.h" - -#include "tensorflow/core/platform/logging.h" - -namespace tensorflow { - -namespace { - -string MakePrefixEndKey(const string& prefix) { - string end = prefix; - while (true) { - if (end.empty()) { - return end; - } - ++end[end.size() - 1]; - if (end[end.size() - 1] == 0) { - // Handle wraparound case. - end = end.substr(0, end.size() - 1); - } else { - return end; - } - } -} - -} // namespace - -/* static */ MultiModeKeyRange MultiModeKeyRange::FromPrefix(string prefix) { - string end = MakePrefixEndKey(prefix); - VLOG(1) << "Creating MultiModeKeyRange from Prefix: " << prefix - << ", with end key: " << end; - return MultiModeKeyRange(std::move(prefix), std::move(end)); -} - -/* static */ MultiModeKeyRange MultiModeKeyRange::FromRange(string begin, - string end) { - return MultiModeKeyRange(std::move(begin), std::move(end)); -} - -const string& MultiModeKeyRange::begin_key() const { return begin_; } - -const string& MultiModeKeyRange::end_key() const { return end_; } - -bool MultiModeKeyRange::contains_key(StringPiece key) const { - if (StringPiece(begin_) > key) { - return false; - } - if (StringPiece(end_) <= key && !end_.empty()) { - return false; - } - return true; -} - -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_range_helpers.h b/tensorflow/contrib/bigtable/kernels/bigtable_range_helpers.h deleted file mode 100644 index d6d00476612..00000000000 --- a/tensorflow/contrib/bigtable/kernels/bigtable_range_helpers.h +++ /dev/null @@ -1,68 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_BIGTABLE_KERNELS_BIGTABLE_RANGE_HELPERS_H_ -#define TENSORFLOW_CONTRIB_BIGTABLE_KERNELS_BIGTABLE_RANGE_HELPERS_H_ - -#include -#include - -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { - -// Represents a continuous range of keys defined by either a prefix or a range. -// -// Ranges are represented as "half-open", where the beginning key is included -// in the range, and the end_key is the first excluded key after the range. -// -// The range of keys can be specified either by a key prefix, or by an explicit -// begin key and end key. All methods on this class are valid no matter which -// way the range was specified. -// -// Example: -// MultiModeKeyRange range = MultiModeKeyRange::FromPrefix("myPrefix"); -// if (range.contains_key("myPrefixedKey")) { -// LOG(INFO) << "range from " << range.begin_key() << " to " -// << range.end_key() << "contains \"myPrefixedKey\""; -// } -// if (!range.contains_key("randomKey")) { -// LOG(INFO) << "range does not contain \"randomKey\""; -// } -// range = MultiModeKeyRange::FromRange("a_start_key", "z_end_key"); -class MultiModeKeyRange { - public: - static MultiModeKeyRange FromPrefix(string prefix); - static MultiModeKeyRange FromRange(string begin, string end); - - // The first valid key in the range. - const string& begin_key() const; - // The first invalid key after the valid range. - const string& end_key() const; - // Returns true if the provided key is a part of the range, false otherwise. - bool contains_key(StringPiece key) const; - - private: - MultiModeKeyRange(string begin, string end) - : begin_(std::move(begin)), end_(std::move(end)) {} - - const string begin_; - const string end_; -}; - -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BIGTABLE_KERNELS_BIGTABLE_RANGE_HELPERS_H_ diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_range_helpers_test.cc b/tensorflow/contrib/bigtable/kernels/bigtable_range_helpers_test.cc deleted file mode 100644 index 1bfc547271d..00000000000 --- a/tensorflow/contrib/bigtable/kernels/bigtable_range_helpers_test.cc +++ /dev/null @@ -1,107 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/bigtable/kernels/bigtable_range_helpers.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace { - -TEST(MultiModeKeyRangeTest, SimplePrefix) { - MultiModeKeyRange r = MultiModeKeyRange::FromPrefix("prefix"); - EXPECT_EQ("prefix", r.begin_key()); - EXPECT_EQ("prefiy", r.end_key()); - EXPECT_TRUE(r.contains_key("prefixed_key")); - EXPECT_FALSE(r.contains_key("not-prefixed-key")); - EXPECT_FALSE(r.contains_key("prefi")); - EXPECT_FALSE(r.contains_key("prefiy")); - EXPECT_FALSE(r.contains_key("early")); - EXPECT_FALSE(r.contains_key("")); -} - -TEST(MultiModeKeyRangeTest, Range) { - MultiModeKeyRange r = MultiModeKeyRange::FromRange("a", "b"); - EXPECT_EQ("a", r.begin_key()); - EXPECT_EQ("b", r.end_key()); - EXPECT_TRUE(r.contains_key("a")); - EXPECT_TRUE(r.contains_key("ab")); - EXPECT_FALSE(r.contains_key("b")); - EXPECT_FALSE(r.contains_key("bc")); - EXPECT_FALSE(r.contains_key("A")); - EXPECT_FALSE(r.contains_key("B")); - EXPECT_FALSE(r.contains_key("")); -} - -TEST(MultiModeKeyRangeTest, InvertedRange) { - MultiModeKeyRange r = MultiModeKeyRange::FromRange("b", "a"); - EXPECT_FALSE(r.contains_key("a")); - EXPECT_FALSE(r.contains_key("b")); - EXPECT_FALSE(r.contains_key("")); -} - -TEST(MultiModeKeyRangeTest, EmptyPrefix) { - MultiModeKeyRange r = MultiModeKeyRange::FromPrefix(""); - EXPECT_EQ("", r.begin_key()); - EXPECT_EQ("", r.end_key()); - EXPECT_TRUE(r.contains_key("")); - EXPECT_TRUE(r.contains_key("a")); - EXPECT_TRUE(r.contains_key("z")); - EXPECT_TRUE(r.contains_key("A")); - EXPECT_TRUE(r.contains_key("ZZZZZZ")); -} - -TEST(MultiModeKeyRangeTest, HalfRange) { - MultiModeKeyRange r = MultiModeKeyRange::FromRange("start", ""); - EXPECT_EQ("start", r.begin_key()); - EXPECT_EQ("", r.end_key()); - EXPECT_TRUE(r.contains_key("start")); - EXPECT_TRUE(r.contains_key("starting")); - EXPECT_TRUE(r.contains_key("z-end")); - EXPECT_FALSE(r.contains_key("")); - EXPECT_FALSE(r.contains_key("early")); -} - -TEST(MultiModeKeyRangeTest, PrefixWrapAround) { - string prefix = "abc\xff"; - MultiModeKeyRange r = MultiModeKeyRange::FromPrefix(prefix); - EXPECT_EQ(prefix, r.begin_key()); - EXPECT_EQ("abd", r.end_key()); - - EXPECT_TRUE(r.contains_key("abc\xff\x07")); - EXPECT_TRUE(r.contains_key("abc\xff\x15")); - EXPECT_TRUE(r.contains_key("abc\xff\x61")); - EXPECT_TRUE(r.contains_key("abc\xff\xff")); - EXPECT_FALSE(r.contains_key("abc\0")); - EXPECT_FALSE(r.contains_key("abd")); -} - -TEST(MultiModeKeyRangeTest, PrefixSignedWrapAround) { - string prefix = "abc\x7f"; - MultiModeKeyRange r = MultiModeKeyRange::FromPrefix(prefix); - EXPECT_EQ(prefix, r.begin_key()); - EXPECT_EQ("abc\x80", r.end_key()); - - EXPECT_TRUE(r.contains_key("abc\x7f\x07")); - EXPECT_TRUE(r.contains_key("abc\x7f\x15")); - EXPECT_TRUE(r.contains_key("abc\x7f\x61")); - EXPECT_TRUE(r.contains_key("abc\x7f\xff")); - EXPECT_FALSE(r.contains_key("abc\0")); - EXPECT_FALSE(r.contains_key("abc\x01")); - EXPECT_FALSE(r.contains_key("abd")); - EXPECT_FALSE(r.contains_key("ab\x80")); -} - -} // namespace -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_range_key_dataset_op.cc b/tensorflow/contrib/bigtable/kernels/bigtable_range_key_dataset_op.cc deleted file mode 100644 index 22f7ddfe15d..00000000000 --- a/tensorflow/contrib/bigtable/kernels/bigtable_range_key_dataset_op.cc +++ /dev/null @@ -1,127 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/bigtable/kernels/bigtable_lib.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/lib/core/refcount.h" - -namespace tensorflow { -namespace data { -namespace { - -class BigtableRangeKeyDatasetOp : public DatasetOpKernel { - public: - using DatasetOpKernel::DatasetOpKernel; - - void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override { - tstring start_key; - OP_REQUIRES_OK(ctx, - ParseScalarArgument(ctx, "start_key", &start_key)); - tstring end_key; - OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "end_key", &end_key)); - - core::RefCountPtr resource; - OP_REQUIRES_OK(ctx, - LookupResource(ctx, HandleFromInput(ctx, 0), &resource)); - *output = new Dataset(ctx, resource.get(), std::move(start_key), - std::move(end_key)); - } - - private: - class Dataset : public DatasetBase { - public: - explicit Dataset(OpKernelContext* ctx, BigtableTableResource* table, - string start_key, string end_key) - : DatasetBase(DatasetContext(ctx)), - table_(table), - start_key_(std::move(start_key)), - end_key_(std::move(end_key)) { - table_->Ref(); - } - - ~Dataset() override { table_->Unref(); } - - std::unique_ptr MakeIteratorInternal( - const string& prefix) const override { - return std::unique_ptr( - new Iterator({this, strings::StrCat(prefix, "::BigtableRangeKey")})); - } - - const DataTypeVector& output_dtypes() const override { - static DataTypeVector* dtypes = new DataTypeVector({DT_STRING}); - return *dtypes; - } - - const std::vector& output_shapes() const override { - static std::vector* shapes = - new std::vector({{}}); - return *shapes; - } - - string DebugString() const override { - return "BigtableRangeKeyDatasetOp::Dataset"; - } - - BigtableTableResource* table() const { return table_; } - - Status CheckExternalState() const override { - return errors::FailedPrecondition(DebugString(), - " depends on external state."); - } - - protected: - Status AsGraphDefInternal(SerializationContext* ctx, - DatasetGraphDefBuilder* b, - Node** output) const override { - return errors::Unimplemented(DebugString(), - " does not support serialization"); - } - - private: - class Iterator : public BigtableReaderDatasetIterator { - public: - explicit Iterator(const Params& params) - : BigtableReaderDatasetIterator(params) {} - - ::google::cloud::bigtable::RowRange MakeRowRange() override { - return ::google::cloud::bigtable::RowRange::Range(dataset()->start_key_, - dataset()->end_key_); - } - ::google::cloud::bigtable::Filter MakeFilter() override { - return ::google::cloud::bigtable::Filter::Chain( - ::google::cloud::bigtable::Filter::CellsRowLimit(1), - ::google::cloud::bigtable::Filter::StripValueTransformer()); - } - Status ParseRow(IteratorContext* ctx, - const ::google::cloud::bigtable::Row& row, - std::vector* out_tensors) override { - Tensor output_tensor(ctx->allocator({}), DT_STRING, {}); - output_tensor.scalar()() = tstring(row.row_key()); - out_tensors->emplace_back(std::move(output_tensor)); - return Status::OK(); - } - }; - - BigtableTableResource* const table_; - const string start_key_; - const string end_key_; - }; -}; - -REGISTER_KERNEL_BUILDER(Name("BigtableRangeKeyDataset").Device(DEVICE_CPU), - BigtableRangeKeyDatasetOp); -} // namespace -} // namespace data -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_sample_key_pairs_dataset_op.cc b/tensorflow/contrib/bigtable/kernels/bigtable_sample_key_pairs_dataset_op.cc deleted file mode 100644 index 08bf35f6c23..00000000000 --- a/tensorflow/contrib/bigtable/kernels/bigtable_sample_key_pairs_dataset_op.cc +++ /dev/null @@ -1,227 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/bigtable/kernels/bigtable_lib.h" -#include "tensorflow/contrib/bigtable/kernels/bigtable_range_helpers.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/lib/core/refcount.h" - -namespace tensorflow { -namespace data { -namespace { - -class BigtableSampleKeyPairsDatasetOp : public DatasetOpKernel { - public: - using DatasetOpKernel::DatasetOpKernel; - - void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override { - tstring prefix; - OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "prefix", &prefix)); - - tstring start_key; - OP_REQUIRES_OK(ctx, - ParseScalarArgument(ctx, "start_key", &start_key)); - tstring end_key; - OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "end_key", &end_key)); - - core::RefCountPtr resource; - OP_REQUIRES_OK(ctx, - LookupResource(ctx, HandleFromInput(ctx, 0), &resource)); - - OP_REQUIRES(ctx, prefix.empty() || start_key.empty(), - errors::InvalidArgument( - "Only one of prefix and start_key can be provided")); - if (!prefix.empty()) { - OP_REQUIRES(ctx, end_key.empty(), - errors::InvalidArgument( - "If prefix is specified, end_key must be empty.")); - } - - *output = new Dataset(ctx, resource.get(), std::move(prefix), - std::move(start_key), std::move(end_key)); - } - - private: - class Dataset : public DatasetBase { - public: - explicit Dataset(OpKernelContext* ctx, BigtableTableResource* table, - string prefix, string start_key, string end_key) - : DatasetBase(DatasetContext(ctx)), - table_(table), - key_range_(MakeMultiModeKeyRange( - std::move(prefix), std::move(start_key), std::move(end_key))) { - table_->Ref(); - } - - ~Dataset() override { table_->Unref(); } - - std::unique_ptr MakeIteratorInternal( - const string& prefix) const override { - return std::unique_ptr(new Iterator( - {this, strings::StrCat(prefix, "::BigtableSampleKeyPairs")})); - } - - const DataTypeVector& output_dtypes() const override { - static DataTypeVector* dtypes = - new DataTypeVector({DT_STRING, DT_STRING}); - return *dtypes; - } - - const std::vector& output_shapes() const override { - static std::vector* shapes = - new std::vector({{}, {}}); - return *shapes; - } - - string DebugString() const override { - return "BigtableSampleKeyPairsDatasetOp::Dataset"; - } - - Status CheckExternalState() const override { - return errors::FailedPrecondition(DebugString(), - " depends on external state."); - } - - protected: - Status AsGraphDefInternal(SerializationContext* ctx, - DatasetGraphDefBuilder* b, - Node** output) const override { - return errors::Unimplemented(DebugString(), - " does not support serialization"); - } - - private: - static MultiModeKeyRange MakeMultiModeKeyRange(string prefix, - string start_key, - string end_key) { - if (!start_key.empty()) { - return MultiModeKeyRange::FromRange(std::move(start_key), - std::move(end_key)); - } - return MultiModeKeyRange::FromPrefix(std::move(prefix)); - } - - BigtableTableResource& table() const { return *table_; } - - class Iterator : public DatasetIterator { - public: - explicit Iterator(const Params& params) - : DatasetIterator(params) {} - - // Computes split points (`keys_`) to use when scanning the table. - // - // Initialize first retrieves the sample keys from the table (`row_keys`), - // as these often form good split points within the table. We then iterate - // over them, and copy them to `keys_` if they fall within the requested - // range to scan (`dataset()->key_range_`). Because the requested range - // might start between elements of the sampled keys list, care is taken to - // ensure we don't accidentally miss any subsets of the requested range by - // including `begin_key()` and `end_key()` as appropriate. - Status Initialize(IteratorContext* ctx) override { - ::google::cloud::StatusOr< - std::vector<::google::cloud::bigtable::RowKeySample>> - row_key_samples = dataset()->table().table().SampleRows(); - if (!row_key_samples.ok()) { - return GcpStatusToTfStatus(row_key_samples.status()); - } - - for (const auto& row_key_sample : *row_key_samples) { - string row_key(row_key_sample.row_key); - if (dataset()->key_range_.contains_key(row_key)) { - // First key: check to see if we need to add the begin_key. - if (keys_.empty() && dataset()->key_range_.begin_key() != row_key) { - keys_.push_back(dataset()->key_range_.begin_key()); - } - keys_.push_back(std::move(row_key)); - } else if (!keys_.empty()) { - // If !keys_.empty(), then we have found at least one element of - // `row_keys` that is within our requested range - // (`dataset()->key_range_`). Because `row_keys` is sorted, if we - // have found an element that's not within our key range, then we - // are after our requested range (ranges are contiguous) and can end - // iteration early. - break; - } - } - - // Handle the case where we skip over the selected range entirely. - if (keys_.empty()) { - keys_.push_back(dataset()->key_range_.begin_key()); - } - - // Last key: check to see if we need to add the end_key. - if (keys_.back() != dataset()->key_range_.end_key()) { - keys_.push_back(dataset()->key_range_.end_key()); - } - return Status::OK(); - } - - Status GetNextInternal(IteratorContext* ctx, - std::vector* out_tensors, - bool* end_of_sequence) override { - mutex_lock l(mu_); - if (index_ + 2 > keys_.size()) { - *end_of_sequence = true; - return Status::OK(); - } - - *end_of_sequence = false; - out_tensors->emplace_back(ctx->allocator({}), DT_STRING, - TensorShape({})); - out_tensors->back().scalar()() = keys_[index_]; - - out_tensors->emplace_back(ctx->allocator({}), DT_STRING, - TensorShape({})); - out_tensors->back().scalar()() = keys_[index_ + 1]; - ++index_; - - return Status::OK(); - } - - protected: - Status SaveInternal(IteratorStateWriter* writer) override { - return errors::Unimplemented("SaveInternal is currently not supported"); - } - - Status RestoreInternal(IteratorContext* ctx, - IteratorStateReader* reader) override { - return errors::Unimplemented( - "RestoreInternal is currently not supported"); - } - - private: - mutex mu_; - size_t index_ GUARDED_BY(mu_) = 0; - // Note: we store the keys_ on the iterator instead of the dataset - // because we want to re-sample the row keys in case there have been - // tablet rebalancing operations since the dataset was created. - // - // Note: keys_ is readonly after Initialize, and thus does not need a - // guarding lock. - std::vector keys_; - }; - - BigtableTableResource* const table_; - const MultiModeKeyRange key_range_; - }; -}; - -REGISTER_KERNEL_BUILDER( - Name("BigtableSampleKeyPairsDataset").Device(DEVICE_CPU), - BigtableSampleKeyPairsDatasetOp); - -} // namespace -} // namespace data -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_sample_keys_dataset_op.cc b/tensorflow/contrib/bigtable/kernels/bigtable_sample_keys_dataset_op.cc deleted file mode 100644 index f4498305aa2..00000000000 --- a/tensorflow/contrib/bigtable/kernels/bigtable_sample_keys_dataset_op.cc +++ /dev/null @@ -1,141 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/bigtable/kernels/bigtable_lib.h" -#include "tensorflow/core/framework/op_kernel.h" - -namespace tensorflow { -namespace data { -namespace { - -class BigtableSampleKeysDatasetOp : public DatasetOpKernel { - public: - using DatasetOpKernel::DatasetOpKernel; - - void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override { - core::RefCountPtr resource; - OP_REQUIRES_OK(ctx, - LookupResource(ctx, HandleFromInput(ctx, 0), &resource)); - *output = new Dataset(ctx, resource.get()); - } - - private: - class Dataset : public DatasetBase { - public: - explicit Dataset(OpKernelContext* ctx, BigtableTableResource* table) - : DatasetBase(DatasetContext(ctx)), table_(table) { - table_->Ref(); - } - - ~Dataset() override { table_->Unref(); } - - std::unique_ptr MakeIteratorInternal( - const string& prefix) const override { - return std::unique_ptr(new Iterator( - {this, strings::StrCat(prefix, "::BigtableSampleKeys")})); - } - - const DataTypeVector& output_dtypes() const override { - static DataTypeVector* dtypes = new DataTypeVector({DT_STRING}); - return *dtypes; - } - - const std::vector& output_shapes() const override { - static std::vector* shapes = - new std::vector({{}}); - return *shapes; - } - - string DebugString() const override { - return "BigtableRangeKeyDatasetOp::Dataset"; - } - - BigtableTableResource* table() const { return table_; } - - Status CheckExternalState() const override { - return errors::FailedPrecondition(DebugString(), - " depends on external state."); - } - - protected: - Status AsGraphDefInternal(SerializationContext* ctx, - DatasetGraphDefBuilder* b, - Node** output) const override { - return errors::Unimplemented(DebugString(), - " does not support serialization"); - } - - private: - class Iterator : public DatasetIterator { - public: - explicit Iterator(const Params& params) - : DatasetIterator(params) {} - - Status Initialize(IteratorContext* ctx) override { - ::google::cloud::StatusOr< - std::vector<::google::cloud::bigtable::RowKeySample>> - sampled_rows = dataset()->table()->table().SampleRows(); - if (!sampled_rows.ok()) { - row_keys_.clear(); - return GcpStatusToTfStatus(sampled_rows.status()); - } - row_keys_ = std::move(*sampled_rows); - return Status::OK(); - } - - Status GetNextInternal(IteratorContext* ctx, - std::vector* out_tensors, - bool* end_of_sequence) override { - mutex_lock l(mu_); - if (index_ < row_keys_.size()) { - out_tensors->emplace_back(ctx->allocator({}), DT_STRING, - TensorShape({})); - out_tensors->back().scalar()() = - tstring(row_keys_[index_].row_key); - *end_of_sequence = false; - index_++; - } else { - *end_of_sequence = true; - } - return Status::OK(); - } - - protected: - Status SaveInternal(IteratorStateWriter* writer) override { - return errors::Unimplemented("SaveInternal is currently not supported"); - } - - Status RestoreInternal(IteratorContext* ctx, - IteratorStateReader* reader) override { - return errors::Unimplemented( - "RestoreInternal is currently not supported"); - } - - private: - mutex mu_; - size_t index_ = 0; - std::vector<::google::cloud::bigtable::RowKeySample> row_keys_; - }; - - BigtableTableResource* const table_; - }; -}; - -REGISTER_KERNEL_BUILDER(Name("BigtableSampleKeysDataset").Device(DEVICE_CPU), - BigtableSampleKeysDatasetOp); - -} // namespace -} // namespace data -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_scan_dataset_op.cc b/tensorflow/contrib/bigtable/kernels/bigtable_scan_dataset_op.cc deleted file mode 100644 index d2b6959fef5..00000000000 --- a/tensorflow/contrib/bigtable/kernels/bigtable_scan_dataset_op.cc +++ /dev/null @@ -1,235 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/bigtable/kernels/bigtable_lib.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/lib/core/refcount.h" - -namespace tensorflow { -namespace data { -namespace { - -class BigtableScanDatasetOp : public DatasetOpKernel { - public: - using DatasetOpKernel::DatasetOpKernel; - - void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override { - tstring prefix; - OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "prefix", &prefix)); - tstring start_key; - OP_REQUIRES_OK(ctx, - ParseScalarArgument(ctx, "start_key", &start_key)); - tstring end_key; - OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "end_key", &end_key)); - - OP_REQUIRES(ctx, !(prefix.empty() && start_key.empty()), - errors::InvalidArgument( - "Either prefix or start_key must be specified")); - OP_REQUIRES(ctx, prefix.empty() || start_key.empty(), - errors::InvalidArgument( - "Only one of prefix and start_key can be provided")); - if (!prefix.empty()) { - OP_REQUIRES(ctx, end_key.empty(), - errors::InvalidArgument( - "If prefix is specified, end_key must be empty.")); - } - - std::vector column_families; - std::vector columns; - OP_REQUIRES_OK(ctx, ParseVectorArgument(ctx, "column_families", - &column_families)); - OP_REQUIRES_OK(ctx, ParseVectorArgument(ctx, "columns", &columns)); - OP_REQUIRES( - ctx, column_families.size() == columns.size(), - errors::InvalidArgument("len(columns) != len(column_families)")); - OP_REQUIRES(ctx, !column_families.empty(), - errors::InvalidArgument("`column_families` is empty")); - - float probability = 0; - OP_REQUIRES_OK( - ctx, ParseScalarArgument(ctx, "probability", &probability)); - OP_REQUIRES( - ctx, probability > 0 && probability <= 1, - errors::InvalidArgument( - "Probability outside the range of (0, 1]. Got: ", probability)); - - core::RefCountPtr resource; - OP_REQUIRES_OK(ctx, - LookupResource(ctx, HandleFromInput(ctx, 0), &resource)); - - const uint64 num_outputs = columns.size() + 1; - std::vector output_shapes; - output_shapes.reserve(num_outputs); - DataTypeVector output_types; - output_types.reserve(num_outputs); - for (uint64 i = 0; i < num_outputs; ++i) { - output_shapes.push_back({}); - output_types.push_back(DT_STRING); - } - - *output = new Dataset(ctx, resource.get(), std::move(prefix), - std::move(start_key), std::move(end_key), - std::move(column_families), std::move(columns), - probability, output_types, std::move(output_shapes)); - } - - private: - class Dataset : public DatasetBase { - public: - explicit Dataset(OpKernelContext* ctx, BigtableTableResource* table, - string prefix, string start_key, string end_key, - std::vector column_families, - std::vector columns, float probability, - const DataTypeVector& output_types, - std::vector output_shapes) - : DatasetBase(DatasetContext(ctx)), - table_(table), - prefix_(std::move(prefix)), - start_key_(std::move(start_key)), - end_key_(std::move(end_key)), - column_families_(std::move(column_families)), - columns_(std::move(columns)), - column_family_regex_(RegexFromStringSet(column_families_)), - column_regex_(RegexFromStringSet(columns_)), - probability_(probability), - output_types_(output_types), - output_shapes_(std::move(output_shapes)) { - table_->Ref(); - } - - ~Dataset() override { table_->Unref(); } - - std::unique_ptr MakeIteratorInternal( - const string& prefix) const override { - return std::unique_ptr( - new Iterator({this, strings::StrCat(prefix, "::BigtableScan")})); - } - - const DataTypeVector& output_dtypes() const override { - return output_types_; - } - - const std::vector& output_shapes() const override { - return output_shapes_; - } - - string DebugString() const override { - return "BigtableScanDatasetOp::Dataset"; - } - - BigtableTableResource* table() const { return table_; } - - Status CheckExternalState() const override { - return errors::FailedPrecondition(DebugString(), - " depends on external state."); - } - - protected: - Status AsGraphDefInternal(SerializationContext* ctx, - DatasetGraphDefBuilder* b, - Node** output) const override { - return errors::Unimplemented(DebugString(), - " does not support serialization"); - } - - private: - class Iterator : public BigtableReaderDatasetIterator { - public: - explicit Iterator(const Params& params) - : BigtableReaderDatasetIterator(params) {} - - ::google::cloud::bigtable::RowRange MakeRowRange() override { - if (!dataset()->prefix_.empty()) { - DCHECK(dataset()->start_key_.empty()); - return ::google::cloud::bigtable::RowRange::Prefix( - dataset()->prefix_); - } else { - DCHECK(!dataset()->start_key_.empty()) - << "Both prefix and start_key were empty!"; - return ::google::cloud::bigtable::RowRange::Range( - dataset()->start_key_, dataset()->end_key_); - } - } - ::google::cloud::bigtable::Filter MakeFilter() override { - // TODO(saeta): Investigate optimal ordering here. - return ::google::cloud::bigtable::Filter::Chain( - ::google::cloud::bigtable::Filter::Latest(1), - ::google::cloud::bigtable::Filter::FamilyRegex( - dataset()->column_family_regex_), - ::google::cloud::bigtable::Filter::ColumnRegex( - dataset()->column_regex_), - dataset()->probability_ != 1.0 - ? ::google::cloud::bigtable::Filter::RowSample( - dataset()->probability_) - : ::google::cloud::bigtable::Filter::PassAllFilter()); - } - Status ParseRow(IteratorContext* ctx, - const ::google::cloud::bigtable::Row& row, - std::vector* out_tensors) override { - out_tensors->reserve(dataset()->columns_.size() + 1); - Tensor row_key_tensor(ctx->allocator({}), DT_STRING, {}); - row_key_tensor.scalar()() = tstring(row.row_key()); - out_tensors->emplace_back(std::move(row_key_tensor)); - - if (row.cells().size() > 2 * dataset()->columns_.size()) { - LOG(WARNING) << "An excessive number of columns (" - << row.cells().size() - << ") were retrieved when reading row: " - << row.row_key(); - } - - for (uint64 i = 0; i < dataset()->columns_.size(); ++i) { - Tensor col_tensor(ctx->allocator({}), DT_STRING, {}); - bool found_column = false; - for (auto cell_itr = row.cells().begin(); - !found_column && cell_itr != row.cells().end(); ++cell_itr) { - if (cell_itr->family_name() == dataset()->column_families_[i] && - tstring(cell_itr->column_qualifier()) == - dataset()->columns_[i]) { - col_tensor.scalar()() = tstring(cell_itr->value()); - found_column = true; - } - } - if (!found_column) { - return errors::InvalidArgument( - "Column ", dataset()->column_families_[i], ":", - dataset()->columns_[i], " not found in row: ", row.row_key()); - } - out_tensors->emplace_back(std::move(col_tensor)); - } - return Status::OK(); - } - }; - - BigtableTableResource* table_; - const string prefix_; - const string start_key_; - const string end_key_; - const std::vector column_families_; - const std::vector columns_; - const string column_family_regex_; - const string column_regex_; - const float probability_; - const DataTypeVector output_types_; - const std::vector output_shapes_; - }; -}; - -REGISTER_KERNEL_BUILDER(Name("BigtableScanDataset").Device(DEVICE_CPU), - BigtableScanDatasetOp); - -} // namespace -} // namespace data -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc deleted file mode 100644 index 836b7b8127b..00000000000 --- a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc +++ /dev/null @@ -1,462 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h" - -#include "external/com_github_googleapis_googleapis/google/bigtable/v2/data.pb.h" -#include "google/protobuf/wrappers.pb.h" -#include "re2/re2.h" -#include "tensorflow/core/lib/strings/stringprintf.h" -#include "tensorflow/core/util/ptr_util.h" -// #include "util/task/codes.pb.h" - -namespace tensorflow { -namespace { - -void UpdateRow(const ::google::bigtable::v2::Mutation& mut, - std::map* row) { - if (mut.has_set_cell()) { - CHECK(mut.set_cell().timestamp_micros() >= -1) - << "Timestamp_micros: " << mut.set_cell().timestamp_micros(); - auto col = - strings::Printf("%s:%s", mut.set_cell().family_name().c_str(), - string(mut.set_cell().column_qualifier()).c_str()); - (*row)[col] = string(mut.set_cell().value()); - } else if (mut.has_delete_from_column()) { - auto col = strings::Printf( - "%s:%s", mut.delete_from_column().family_name().c_str(), - string(mut.delete_from_column().column_qualifier()).c_str()); - row->erase(col); - } else if (mut.has_delete_from_family()) { - auto itr = row->lower_bound(mut.delete_from_family().family_name()); - auto prefix = - strings::Printf("%s:", mut.delete_from_family().family_name().c_str()); - while (itr != row->end() && itr->first.substr(0, prefix.size()) == prefix) { - row->erase(itr); - } - } else if (mut.has_delete_from_row()) { - row->clear(); - } else { - LOG(ERROR) << "Unknown mutation: " << mut.ShortDebugString(); - } -} - -} // namespace - -class SampleRowKeysResponse : public grpc::ClientReaderInterface< - google::bigtable::v2::SampleRowKeysResponse> { - public: - explicit SampleRowKeysResponse(BigtableTestClient* client) - : client_(client) {} - - bool NextMessageSize(uint32_t* sz) override { - mutex_lock l(mu_); - mutex_lock l2(client_->mu_); - if (num_messages_sent_ * 2 < client_->table_.rows.size()) { - *sz = 10000; // A sufficiently high enough value to not worry about. - return true; - } - return false; - } - - bool Read(google::bigtable::v2::SampleRowKeysResponse* resp) override { - // Send every other key from the table. - mutex_lock l(mu_); - mutex_lock l2(client_->mu_); - *resp = google::bigtable::v2::SampleRowKeysResponse(); - auto itr = client_->table_.rows.begin(); - for (uint64 i = 0; i < 2 * num_messages_sent_; ++i) { - ++itr; - if (itr == client_->table_.rows.end()) { - return false; - } - } - resp->set_row_key(itr->first); - resp->set_offset_bytes(100 * num_messages_sent_); - num_messages_sent_++; - return true; - } - - grpc::Status Finish() override { return grpc::Status::OK; } - - void WaitForInitialMetadata() override {} // Do nothing. - - private: - mutex mu_; - int64 num_messages_sent_ GUARDED_BY(mu_) = 0; - BigtableTestClient* client_; // Not owned. -}; - -class ReadRowsResponse : public grpc::ClientReaderInterface< - google::bigtable::v2::ReadRowsResponse> { - public: - ReadRowsResponse(BigtableTestClient* client, - google::bigtable::v2::ReadRowsRequest const& request) - : client_(client), request_(request) {} - - bool NextMessageSize(uint32_t* sz) override { - mutex_lock l(mu_); - if (sent_first_message_) { - return false; - } - *sz = 10000000; // A sufficiently high enough value to not worry about. - return true; - } - - bool Read(google::bigtable::v2::ReadRowsResponse* resp) override { - mutex_lock l(mu_); - if (sent_first_message_) { - return false; - } - sent_first_message_ = true; - RowFilter filter = MakeRowFilter(); - - mutex_lock l2(client_->mu_); - *resp = google::bigtable::v2::ReadRowsResponse(); - // Send all contents in first response. - for (auto itr = client_->table_.rows.begin(); - itr != client_->table_.rows.end(); ++itr) { - if (filter.AllowRow(itr->first)) { - ::google::bigtable::v2::ReadRowsResponse_CellChunk* chunk = nullptr; - bool sent_first = false; - for (auto col_itr = itr->second.columns.begin(); - col_itr != itr->second.columns.end(); ++col_itr) { - if (filter.AllowColumn(col_itr->first)) { - chunk = resp->add_chunks(); - if (!sent_first) { - sent_first = true; - chunk->set_row_key(itr->first); - } - auto colon_idx = col_itr->first.find(":"); - CHECK(colon_idx != string::npos) - << "No ':' found in: " << col_itr->first; - chunk->mutable_family_name()->set_value( - string(col_itr->first, 0, colon_idx)); - chunk->mutable_qualifier()->set_value( - string(col_itr->first, ++colon_idx)); - if (!filter.strip_values) { - chunk->set_value(col_itr->second); - } - if (filter.only_one_column) { - break; - } - } - } - if (sent_first) { - // We are sending this row, so set the commit flag on the last chunk. - chunk->set_commit_row(true); - } - } - } - return true; - } - - grpc::Status Finish() override { return grpc::Status::OK; } - - void WaitForInitialMetadata() override {} // Do nothing. - - private: - struct RowFilter { - std::set row_set; - std::vector> row_ranges; - double row_sample = 0.0; // Note: currently ignored. - std::unique_ptr col_filter; - bool strip_values = false; - bool only_one_column = false; - - bool AllowRow(const string& row) { - if (row_set.find(row) != row_set.end()) { - return true; - } - for (const auto& range : row_ranges) { - if (range.first <= row && range.second > row) { - return true; - } - } - return false; - } - - bool AllowColumn(const string& col) { - if (col_filter) { - return RE2::FullMatch(col, *col_filter); - } else { - return true; - } - } - }; - - RowFilter MakeRowFilter() { - RowFilter filter; - for (auto i = request_.rows().row_keys().begin(); - i != request_.rows().row_keys().end(); ++i) { - filter.row_set.insert(string(*i)); - } - for (auto i = request_.rows().row_ranges().begin(); - i != request_.rows().row_ranges().end(); ++i) { - if (i->start_key_case() != - google::bigtable::v2::RowRange::kStartKeyClosed || - i->end_key_case() != google::bigtable::v2::RowRange::kEndKeyOpen) { - LOG(WARNING) << "Skipping row range that cannot be processed: " - << i->ShortDebugString(); - continue; - } - filter.row_ranges.emplace_back(std::make_pair( - string(i->start_key_closed()), string(i->end_key_open()))); - } - if (request_.filter().has_chain()) { - string family_filter; - string qualifier_filter; - for (auto i = request_.filter().chain().filters().begin(); - i != request_.filter().chain().filters().end(); ++i) { - switch (i->filter_case()) { - case google::bigtable::v2::RowFilter::kFamilyNameRegexFilter: - family_filter = i->family_name_regex_filter(); - break; - case google::bigtable::v2::RowFilter::kColumnQualifierRegexFilter: - qualifier_filter = i->column_qualifier_regex_filter(); - break; - case google::bigtable::v2::RowFilter::kCellsPerColumnLimitFilter: - if (i->cells_per_column_limit_filter() != 1) { - LOG(ERROR) << "Unexpected cells_per_column_limit_filter: " - << i->cells_per_column_limit_filter(); - } - break; - case google::bigtable::v2::RowFilter::kStripValueTransformer: - filter.strip_values = i->strip_value_transformer(); - break; - case google::bigtable::v2::RowFilter::kRowSampleFilter: - LOG(INFO) << "Ignoring row sample directive."; - break; - case google::bigtable::v2::RowFilter::kPassAllFilter: - break; - case google::bigtable::v2::RowFilter::kCellsPerRowLimitFilter: - filter.only_one_column = true; - break; - default: - LOG(WARNING) << "Ignoring unknown filter type: " - << i->ShortDebugString(); - } - } - if (family_filter.empty() || qualifier_filter.empty()) { - LOG(WARNING) << "Missing regex!"; - } else { - string regex = strings::Printf("%s:%s", family_filter.c_str(), - qualifier_filter.c_str()); - filter.col_filter.reset(new RE2(regex)); - } - } else { - LOG(WARNING) << "Read request did not have a filter chain specified: " - << request_.filter().DebugString(); - } - return filter; - } - - mutex mu_; - bool sent_first_message_ GUARDED_BY(mu_) = false; - BigtableTestClient* client_; // Not owned. - const google::bigtable::v2::ReadRowsRequest request_; -}; - -class MutateRowsResponse : public grpc::ClientReaderInterface< - google::bigtable::v2::MutateRowsResponse> { - public: - explicit MutateRowsResponse(size_t num_successes) - : num_successes_(num_successes) {} - - bool NextMessageSize(uint32_t* sz) override { - mutex_lock l(mu_); - if (sent_first_message_) { - return false; - } - *sz = 10000000; // A sufficiently high enough value to not worry about. - return true; - } - - bool Read(google::bigtable::v2::MutateRowsResponse* resp) override { - mutex_lock l(mu_); - if (sent_first_message_) { - return false; - } - sent_first_message_ = true; - *resp = google::bigtable::v2::MutateRowsResponse(); - for (size_t i = 0; i < num_successes_; ++i) { - auto entry = resp->add_entries(); - entry->set_index(i); - } - return true; - } - - grpc::Status Finish() override { return grpc::Status::OK; } - - void WaitForInitialMetadata() override {} // Do nothing. - - private: - const size_t num_successes_; - - mutex mu_; - bool sent_first_message_ = false; -}; - -grpc::Status BigtableTestClient::MutateRow( - grpc::ClientContext* context, - google::bigtable::v2::MutateRowRequest const& request, - google::bigtable::v2::MutateRowResponse* response) { - mutex_lock l(mu_); - auto* row = &table_.rows[string(request.row_key())]; - for (int i = 0; i < request.mutations_size(); ++i) { - UpdateRow(request.mutations(i), &row->columns); - } - *response = google::bigtable::v2::MutateRowResponse(); - return grpc::Status::OK; -} -grpc::Status BigtableTestClient::CheckAndMutateRow( - grpc::ClientContext* context, - google::bigtable::v2::CheckAndMutateRowRequest const& request, - google::bigtable::v2::CheckAndMutateRowResponse* response) { - return grpc::Status(grpc::StatusCode::UNIMPLEMENTED, - "CheckAndMutateRow not implemented."); -} -grpc::Status BigtableTestClient::ReadModifyWriteRow( - grpc::ClientContext* context, - google::bigtable::v2::ReadModifyWriteRowRequest const& request, - google::bigtable::v2::ReadModifyWriteRowResponse* response) { - return grpc::Status(grpc::StatusCode::UNIMPLEMENTED, - "ReadModifyWriteRow not implemented."); -} -std::unique_ptr> -BigtableTestClient::AsyncReadModifyWriteRow( - grpc::ClientContext* context, - google::bigtable::v2::ReadModifyWriteRowRequest const& request, - grpc::CompletionQueue* cq) { - LOG(WARNING) << "Call to AsyncReadModifyWriteRow:" << __func__ - << "(); this will likely cause a crash!"; - return nullptr; -} - -std::unique_ptr< - grpc::ClientReaderInterface> -BigtableTestClient::ReadRows( - grpc::ClientContext* context, - google::bigtable::v2::ReadRowsRequest const& request) { - return MakeUnique(this, request); -} - -std::unique_ptr< - grpc::ClientReaderInterface> -BigtableTestClient::SampleRowKeys( - grpc::ClientContext* context, - google::bigtable::v2::SampleRowKeysRequest const& request) { - return MakeUnique(this); -} -std::unique_ptr< - grpc::ClientReaderInterface> -BigtableTestClient::MutateRows( - grpc::ClientContext* context, - google::bigtable::v2::MutateRowsRequest const& request) { - mutex_lock l(mu_); - for (auto i = request.entries().begin(); i != request.entries().end(); ++i) { - auto* row = &table_.rows[string(i->row_key())]; - for (auto mut = i->mutations().begin(); mut != i->mutations().end(); - ++mut) { - UpdateRow(*mut, &row->columns); - } - } - return MakeUnique(request.entries_size()); -} - -std::unique_ptr> -BigtableTestClient::AsyncMutateRow( - grpc::ClientContext* context, - google::bigtable::v2::MutateRowRequest const& request, - grpc::CompletionQueue* cq) { - LOG(WARNING) << "Call to InMemoryDataClient::" << __func__ - << "(); this will likely cause a crash!"; - return nullptr; -} - -std::unique_ptr<::grpc::ClientAsyncReaderInterface< - ::google::bigtable::v2::SampleRowKeysResponse>> -BigtableTestClient::AsyncSampleRowKeys( - ::grpc::ClientContext* context, - const ::google::bigtable::v2::SampleRowKeysRequest& request, - ::grpc::CompletionQueue* cq, void* tag) { - LOG(WARNING) << "Call to InMemoryDataClient::" << __func__ - << "(); this will likely cause a crash!"; - return nullptr; -} - -std::unique_ptr<::grpc::ClientAsyncReaderInterface< - ::google::bigtable::v2::MutateRowsResponse>> -BigtableTestClient::AsyncMutateRows( - ::grpc::ClientContext* context, - const ::google::bigtable::v2::MutateRowsRequest& request, - ::grpc::CompletionQueue* cq, void* tag) { - LOG(WARNING) << "Call to InMemoryDataClient::" << __func__ - << "(); this will likely cause a crash!"; - return nullptr; -} - -std::unique_ptr> -BigtableTestClient::AsyncCheckAndMutateRow( - grpc::ClientContext* context, - const google::bigtable::v2::CheckAndMutateRowRequest& request, - grpc::CompletionQueue* cq) { - LOG(WARNING) << "Call to InMemoryDataClient::" << __func__ - << "(); this will likely cause a crash!"; - return nullptr; -} - -std::unique_ptr< - grpc::ClientAsyncReaderInterface> -BigtableTestClient::AsyncReadRows( - grpc::ClientContext* context, - const google::bigtable::v2::ReadRowsRequest& request, - grpc::CompletionQueue* cq, void* tag) { - LOG(WARNING) << "Call to InMemoryDataClient::" << __func__ - << "(); this will likely cause a crash!"; - return nullptr; -} - -std::unique_ptr< - grpc::ClientAsyncReaderInterface> -BigtableTestClient::PrepareAsyncMutateRows( - grpc::ClientContext* context, - const google::bigtable::v2::MutateRowsRequest& request, - grpc::CompletionQueue* cq) { - LOG(WARNING) << "Call to InMemoryDataClient::" << __func__ - << "(); this will likely cause a crash!"; - return nullptr; -} - -std::unique_ptr<::grpc::ClientAsyncReaderInterface< - ::google::bigtable::v2::ReadRowsResponse>> -BigtableTestClient::PrepareAsyncReadRows( - ::grpc::ClientContext* context, - const ::google::bigtable::v2::ReadRowsRequest& request, - ::grpc::CompletionQueue* cq) { - LOG(WARNING) << "Call to InMemoryDataClient::" << __func__ - << "(); this will likely cause a crash!"; - return nullptr; -} - -std::shared_ptr BigtableTestClient::Channel() { - LOG(WARNING) << "Call to InMemoryDataClient::Channel(); this will likely " - "cause a crash!"; - return nullptr; -} -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h deleted file mode 100644 index 567b98c2414..00000000000 --- a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h +++ /dev/null @@ -1,138 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_BIGTABLE_KERNELS_TEST_KERNELS_BIGTABLE_TEST_CLIENT_H_ -#define TENSORFLOW_CONTRIB_BIGTABLE_KERNELS_TEST_KERNELS_BIGTABLE_TEST_CLIENT_H_ - -#include "google/cloud/bigtable/data_client.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/mutex.h" - -namespace tensorflow { - -class BigtableTestClient : public ::google::cloud::bigtable::DataClient { - public: - std::string const& project_id() const override { return project_id_; } - std::string const& instance_id() const override { return instance_id_; } - void reset() override { - mutex_lock l(mu_); - table_ = Table(); - } - - grpc::Status MutateRow( - grpc::ClientContext* context, - google::bigtable::v2::MutateRowRequest const& request, - google::bigtable::v2::MutateRowResponse* response) override; - - grpc::Status CheckAndMutateRow( - grpc::ClientContext* context, - google::bigtable::v2::CheckAndMutateRowRequest const& request, - google::bigtable::v2::CheckAndMutateRowResponse* response) override; - - grpc::Status ReadModifyWriteRow( - grpc::ClientContext* context, - google::bigtable::v2::ReadModifyWriteRowRequest const& request, - google::bigtable::v2::ReadModifyWriteRowResponse* response) override; - - std::unique_ptr> - AsyncReadModifyWriteRow( - grpc::ClientContext* context, - google::bigtable::v2::ReadModifyWriteRowRequest const& request, - grpc::CompletionQueue* cq) override; - - std::unique_ptr< - grpc::ClientReaderInterface> - ReadRows(grpc::ClientContext* context, - google::bigtable::v2::ReadRowsRequest const& request) override; - std::unique_ptr< - grpc::ClientReaderInterface> - SampleRowKeys( - grpc::ClientContext* context, - google::bigtable::v2::SampleRowKeysRequest const& request) override; - - std::unique_ptr< - grpc::ClientReaderInterface> - MutateRows(grpc::ClientContext* context, - google::bigtable::v2::MutateRowsRequest const& request) override; - - std::unique_ptr> - AsyncMutateRow(grpc::ClientContext* context, - google::bigtable::v2::MutateRowRequest const& request, - grpc::CompletionQueue* cq) override; - - std::unique_ptr<::grpc::ClientAsyncReaderInterface< - ::google::bigtable::v2::SampleRowKeysResponse>> - AsyncSampleRowKeys( - ::grpc::ClientContext* context, - const ::google::bigtable::v2::SampleRowKeysRequest& request, - ::grpc::CompletionQueue* cq, void* tag) override; - - std::unique_ptr<::grpc::ClientAsyncReaderInterface< - ::google::bigtable::v2::MutateRowsResponse>> - AsyncMutateRows(::grpc::ClientContext* context, - const ::google::bigtable::v2::MutateRowsRequest& request, - ::grpc::CompletionQueue* cq, void* tag) override; - - std::unique_ptr> - AsyncCheckAndMutateRow( - grpc::ClientContext* context, - const google::bigtable::v2::CheckAndMutateRowRequest& request, - grpc::CompletionQueue* cq) override; - - std::unique_ptr< - grpc::ClientAsyncReaderInterface> - AsyncReadRows(grpc::ClientContext* context, - const google::bigtable::v2::ReadRowsRequest& request, - grpc::CompletionQueue* cq, void* tag) override; - - std::unique_ptr> - PrepareAsyncMutateRows(grpc::ClientContext* context, - const google::bigtable::v2::MutateRowsRequest& request, - grpc::CompletionQueue* cq) override; - - virtual std::unique_ptr<::grpc::ClientAsyncReaderInterface< - ::google::bigtable::v2::ReadRowsResponse>> - PrepareAsyncReadRows(::grpc::ClientContext* context, - const ::google::bigtable::v2::ReadRowsRequest& request, - ::grpc::CompletionQueue* cq) override; - - std::shared_ptr Channel() override; - - private: - friend class SampleRowKeysResponse; - friend class ReadRowsResponse; - friend class MutateRowsResponse; - - struct Row { - string row_key; - std::map columns; - }; - struct Table { - std::map rows; - }; - - mutex mu_; - const std::string project_id_ = "testproject"; - const std::string instance_id_ = "testinstance"; - Table table_ GUARDED_BY(mu_); -}; - -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BIGTABLE_KERNELS_TEST_KERNELS_BIGTABLE_TEST_CLIENT_H_ diff --git a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client_op.cc b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client_op.cc deleted file mode 100644 index fa3e587b901..00000000000 --- a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client_op.cc +++ /dev/null @@ -1,78 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/bigtable/kernels/bigtable_lib.h" -#include "tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/lib/strings/stringprintf.h" - -namespace tensorflow { - -namespace { - -class BigtableTestClientOp : public OpKernel { - public: - explicit BigtableTestClientOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} - ~BigtableTestClientOp() override { - if (cinfo_.resource_is_private_to_kernel()) { - if (!cinfo_.resource_manager() - ->Delete(cinfo_.container(), - cinfo_.name()) - .ok()) { - // Do nothing; the resource can have been deleted by session resets. - } - } - } - void Compute(OpKernelContext* ctx) override LOCKS_EXCLUDED(mu_) { - mutex_lock l(mu_); - if (!initialized_) { - ResourceMgr* mgr = ctx->resource_manager(); - OP_REQUIRES_OK(ctx, cinfo_.Init(mgr, def())); - BigtableClientResource* resource; - OP_REQUIRES_OK( - ctx, - mgr->LookupOrCreate( - cinfo_.container(), cinfo_.name(), &resource, - [this, ctx](BigtableClientResource** ret) - EXCLUSIVE_LOCKS_REQUIRED(mu_) { - std::shared_ptr client( - new BigtableTestClient()); - // Note: must make explicit copies to sequence - // them before the move of client. - string project_id = client->project_id(); - string instance_id = client->instance_id(); - *ret = new BigtableClientResource(std::move(project_id), - std::move(instance_id), - std::move(client)); - return Status::OK(); - })); - initialized_ = true; - } - OP_REQUIRES_OK(ctx, MakeResourceHandleToOutput( - ctx, 0, cinfo_.container(), cinfo_.name(), - MakeTypeIndex())); - } - - private: - mutex mu_; - ContainerInfo cinfo_ GUARDED_BY(mu_); - bool initialized_ GUARDED_BY(mu_) = false; -}; - -REGISTER_KERNEL_BUILDER(Name("BigtableTestClient").Device(DEVICE_CPU), - BigtableTestClientOp); - -} // namespace -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client_test.cc b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client_test.cc deleted file mode 100644 index c2104604b84..00000000000 --- a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client_test.cc +++ /dev/null @@ -1,349 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h" - -#include "google/cloud/bigtable/table.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace { - -void WriteCell(const string& row, const string& family, const string& column, - const string& value, ::google::cloud::bigtable::Table* table) { - ::google::cloud::bigtable::SingleRowMutation mut(row); - mut.emplace_back(::google::cloud::bigtable::SetCell(family, column, value)); - table->Apply(std::move(mut)); -} - -TEST(BigtableTestClientTest, EmptyRowRead) { - std::shared_ptr<::google::cloud::bigtable::DataClient> client_ptr = - std::make_shared(); - ::google::cloud::bigtable::Table table(client_ptr, "test_table"); - - ::google::cloud::bigtable::RowSet rowset; - rowset.Append("r1"); - auto filter = ::google::cloud::bigtable::Filter::Chain( - ::google::cloud::bigtable::Filter::Latest(1)); - auto rows = table.ReadRows(std::move(rowset), filter); - EXPECT_EQ(rows.begin(), rows.end()) << "Some rows were returned in response!"; -} - -TEST(BigtableTestClientTest, SingleRowWriteAndRead) { - std::shared_ptr<::google::cloud::bigtable::DataClient> client_ptr = - std::make_shared(); - ::google::cloud::bigtable::Table table(client_ptr, "test_table"); - - WriteCell("r1", "f1", "c1", "v1", &table); - - ::google::cloud::bigtable::RowSet rowset("r1"); - auto filter = ::google::cloud::bigtable::Filter::Chain( - ::google::cloud::bigtable::Filter::Latest(1)); - auto rows = table.ReadRows(std::move(rowset), filter); - auto itr = rows.begin(); - EXPECT_NE(itr, rows.end()) << "No rows were returned in response!"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r1"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), "v1"); - - ++itr; - EXPECT_EQ(itr, rows.end()); -} - -TEST(BigtableTestClientTest, MultiRowWriteAndSingleRowRead) { - std::shared_ptr<::google::cloud::bigtable::DataClient> client_ptr = - std::make_shared(); - ::google::cloud::bigtable::Table table(client_ptr, "test_table"); - - WriteCell("r1", "f1", "c1", "v1", &table); - WriteCell("r2", "f1", "c1", "v2", &table); - WriteCell("r3", "f1", "c1", "v3", &table); - - ::google::cloud::bigtable::RowSet rowset("r1"); - auto filter = ::google::cloud::bigtable::Filter::Chain( - ::google::cloud::bigtable::Filter::Latest(1)); - auto rows = table.ReadRows(std::move(rowset), filter); - auto itr = rows.begin(); - - EXPECT_NE(itr, rows.end()) << "Missing rows"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r1"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), "v1"); - - ++itr; - EXPECT_EQ(itr, rows.end()) << "Extra rows in the response."; -} - -TEST(BigtableTestClientTest, MultiRowWriteAndRead) { - std::shared_ptr<::google::cloud::bigtable::DataClient> client_ptr = - std::make_shared(); - ::google::cloud::bigtable::Table table(client_ptr, "test_table"); - - WriteCell("r1", "f1", "c1", "v1", &table); - WriteCell("r2", "f1", "c1", "v2", &table); - WriteCell("r3", "f1", "c1", "v3", &table); - - ::google::cloud::bigtable::RowSet rowset("r1", "r2", "r3"); - auto filter = ::google::cloud::bigtable::Filter::Chain( - ::google::cloud::bigtable::Filter::Latest(1)); - auto rows = table.ReadRows(std::move(rowset), filter); - auto itr = rows.begin(); - - EXPECT_NE(itr, rows.end()) << "Missing rows"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r1"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), "v1"); - - ++itr; - - EXPECT_NE(itr, rows.end()) << "Missing rows"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r2"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), "v2"); - - ++itr; - - EXPECT_NE(itr, rows.end()) << "Missing rows"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r3"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), "v3"); - - ++itr; - EXPECT_EQ(itr, rows.end()) << "Extra rows in the response."; -} - -TEST(BigtableTestClientTest, MultiRowWriteAndPrefixRead) { - std::shared_ptr<::google::cloud::bigtable::DataClient> client_ptr = - std::make_shared(); - ::google::cloud::bigtable::Table table(client_ptr, "test_table"); - - WriteCell("r1", "f1", "c1", "v1", &table); - WriteCell("r2", "f1", "c1", "v2", &table); - WriteCell("r3", "f1", "c1", "v3", &table); - - auto filter = ::google::cloud::bigtable::Filter::Chain( - ::google::cloud::bigtable::Filter::Latest(1)); - auto rows = - table.ReadRows(::google::cloud::bigtable::RowRange::Prefix("r"), filter); - auto itr = rows.begin(); - - EXPECT_NE(itr, rows.end()) << "Missing rows"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r1"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), "v1"); - - ++itr; - - EXPECT_NE(itr, rows.end()) << "Missing rows"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r2"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), "v2"); - - ++itr; - - EXPECT_NE(itr, rows.end()) << "Missing rows"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r3"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), "v3"); - - ++itr; - EXPECT_EQ(itr, rows.end()) << "Extra rows in the response."; -} - -TEST(BigtableTestClientTest, ColumnFiltering) { - std::shared_ptr<::google::cloud::bigtable::DataClient> client_ptr = - std::make_shared(); - ::google::cloud::bigtable::Table table(client_ptr, "test_table"); - - WriteCell("r1", "f1", "c1", "v1", &table); - WriteCell("r2", "f1", "c1", "v2", &table); - WriteCell("r3", "f1", "c1", "v3", &table); - - // Extra cells - WriteCell("r1", "f2", "c1", "v1", &table); - WriteCell("r2", "f2", "c1", "v2", &table); - WriteCell("r3", "f1", "c2", "v3", &table); - - auto filter = ::google::cloud::bigtable::Filter::Chain( - ::google::cloud::bigtable::Filter::Latest(1), - ::google::cloud::bigtable::Filter::FamilyRegex("f1"), - ::google::cloud::bigtable::Filter::ColumnRegex("c1")); - auto rows = - table.ReadRows(::google::cloud::bigtable::RowRange::Prefix("r"), filter); - auto itr = rows.begin(); - - EXPECT_NE(itr, rows.end()) << "Missing rows"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r1"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), "v1"); - - ++itr; - - EXPECT_NE(itr, rows.end()) << "Missing rows"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r2"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), "v2"); - - ++itr; - - EXPECT_NE(itr, rows.end()) << "Missing rows"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r3"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), "v3"); - - ++itr; - EXPECT_EQ(itr, rows.end()) << "Extra rows in the response."; -} - -TEST(BigtableTestClientTest, RowKeys) { - std::shared_ptr<::google::cloud::bigtable::DataClient> client_ptr = - std::make_shared(); - ::google::cloud::bigtable::Table table(client_ptr, "test_table"); - - WriteCell("r1", "f1", "c1", "v1", &table); - WriteCell("r2", "f1", "c1", "v2", &table); - WriteCell("r3", "f1", "c1", "v3", &table); - - // Extra cells - WriteCell("r1", "f2", "c1", "v1", &table); - WriteCell("r2", "f2", "c1", "v2", &table); - WriteCell("r3", "f1", "c2", "v3", &table); - - auto filter = ::google::cloud::bigtable::Filter::Chain( - ::google::cloud::bigtable::Filter::Latest(1), - ::google::cloud::bigtable::Filter::CellsRowLimit(1), - ::google::cloud::bigtable::Filter::StripValueTransformer()); - auto rows = - table.ReadRows(::google::cloud::bigtable::RowRange::Prefix("r"), filter); - auto itr = rows.begin(); - EXPECT_NE(itr, rows.end()) << "Missing rows"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r1"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), ""); - - ++itr; - - EXPECT_NE(itr, rows.end()) << "Missing rows"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r2"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), ""); - - ++itr; - - EXPECT_NE(itr, rows.end()) << "Missing rows"; - EXPECT_TRUE(*itr) << "Error reading row: " << itr->status().message(); - EXPECT_EQ((*itr)->row_key(), "r3"); - EXPECT_EQ((*itr)->cells().size(), 1); - EXPECT_EQ((*itr)->cells()[0].family_name(), "f1"); - EXPECT_EQ((*itr)->cells()[0].column_qualifier(), "c1"); - EXPECT_EQ((*itr)->cells()[0].value(), ""); - - ++itr; - EXPECT_EQ(itr, rows.end()) << "Extra rows in the response."; -} - -TEST(BigtableTestClientTest, SampleKeys) { - std::shared_ptr<::google::cloud::bigtable::DataClient> client_ptr = - std::make_shared(); - ::google::cloud::bigtable::Table table(client_ptr, "test_table"); - - WriteCell("r1", "f1", "c1", "v1", &table); - WriteCell("r2", "f1", "c1", "v2", &table); - WriteCell("r3", "f1", "c1", "v3", &table); - WriteCell("r4", "f1", "c1", "v4", &table); - WriteCell("r5", "f1", "c1", "v5", &table); - - auto resp = table.SampleRows(); - EXPECT_TRUE(resp.ok()); - EXPECT_EQ(3, resp->size()); - EXPECT_EQ("r1", string((*resp)[0].row_key)); - EXPECT_EQ(0, (*resp)[0].offset_bytes); - EXPECT_EQ("r3", string((*resp)[1].row_key)); - EXPECT_EQ(100, (*resp)[1].offset_bytes); - EXPECT_EQ("r5", string((*resp)[2].row_key)); - EXPECT_EQ(200, (*resp)[2].offset_bytes); -} - -TEST(BigtableTestClientTest, SampleKeysShort) { - std::shared_ptr<::google::cloud::bigtable::DataClient> client_ptr = - std::make_shared(); - ::google::cloud::bigtable::Table table(client_ptr, "test_table"); - - WriteCell("r1", "f1", "c1", "v1", &table); - - auto resp = table.SampleRows(); - EXPECT_TRUE(resp.ok()); - EXPECT_EQ(1, resp->size()); - EXPECT_EQ("r1", string((*resp)[0].row_key)); -} - -TEST(BigtableTestClientTest, SampleKeysEvenNumber) { - std::shared_ptr<::google::cloud::bigtable::DataClient> client_ptr = - std::make_shared(); - ::google::cloud::bigtable::Table table(client_ptr, "test_table"); - - WriteCell("r1", "f1", "c1", "v1", &table); - WriteCell("r2", "f1", "c1", "v2", &table); - WriteCell("r3", "f1", "c1", "v3", &table); - WriteCell("r4", "f1", "c1", "v4", &table); - - auto resp = table.SampleRows(); - EXPECT_TRUE(resp.ok()); - EXPECT_EQ(2, resp->size()); - EXPECT_EQ("r1", string((*resp)[0].row_key)); - EXPECT_EQ("r3", string((*resp)[1].row_key)); -} - -} // namespace -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/ops/bigtable_ops.cc b/tensorflow/contrib/bigtable/ops/bigtable_ops.cc deleted file mode 100644 index 39c2a2e775d..00000000000 --- a/tensorflow/contrib/bigtable/ops/bigtable_ops.cc +++ /dev/null @@ -1,107 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" - -namespace tensorflow { - -// TODO(saeta): Add support for setting ClientOptions values. -REGISTER_OP("BigtableClient") - .Attr("project_id: string") - .Attr("instance_id: string") - .Attr("connection_pool_size: int") - .Attr("max_receive_message_size: int = -1") - .Attr("container: string = ''") - .Attr("shared_name: string = ''") - .Output("client: resource") - .SetShapeFn(shape_inference::ScalarShape); - -// TODO(saeta): Add support for Application Profiles. -// See https://cloud.google.com/bigtable/docs/app-profiles for more info. -REGISTER_OP("BigtableTable") - .Input("client: resource") - .Attr("table_name: string") - .Attr("container: string = ''") - .Attr("shared_name: string = ''") - .Output("table: resource") - .SetShapeFn(shape_inference::ScalarShape); - -REGISTER_OP("DatasetToBigtable") - .Input("table: resource") - .Input("input_dataset: variant") - .Input("column_families: string") - .Input("columns: string") - .Input("timestamp: int64") - .SetShapeFn(shape_inference::NoOutputs); - -REGISTER_OP("BigtableLookupDataset") - .Input("keys_dataset: variant") - .Input("table: resource") - .Input("column_families: string") - .Input("columns: string") - .Output("handle: variant") - .SetShapeFn(shape_inference::ScalarShape); - -REGISTER_OP("BigtablePrefixKeyDataset") - .Input("table: resource") - .Input("prefix: string") - .Output("handle: variant") - .SetIsStateful() // TODO(b/123753214): Source dataset ops must be marked - // stateful to inhibit constant folding. - .SetShapeFn(shape_inference::ScalarShape); - -REGISTER_OP("BigtableRangeKeyDataset") - .Input("table: resource") - .Input("start_key: string") - .Input("end_key: string") - .Output("handle: variant") - .SetIsStateful() // TODO(b/123753214): Source dataset ops must be marked - // stateful to inhibit constant folding. - .SetShapeFn(shape_inference::ScalarShape); - -REGISTER_OP("BigtableSampleKeysDataset") - .Input("table: resource") - .Output("handle: variant") - .SetIsStateful() // TODO(b/123753214): Source dataset ops must be marked - // stateful to inhibit constant folding. - .SetShapeFn(shape_inference::ScalarShape); - -REGISTER_OP("BigtableSampleKeyPairsDataset") - .Input("table: resource") - .Input("prefix: string") - .Input("start_key: string") - .Input("end_key: string") - .Output("handle: variant") - .SetIsStateful() // TODO(b/123753214): Source dataset ops must be marked - // stateful to inhibit constant folding. - .SetShapeFn(shape_inference::ScalarShape); - -// TODO(saeta): Support continuing despite bad data (e.g. empty string, or -// skip incomplete row.) -REGISTER_OP("BigtableScanDataset") - .Input("table: resource") - .Input("prefix: string") - .Input("start_key: string") - .Input("end_key: string") - .Input("column_families: string") - .Input("columns: string") - .Input("probability: float") - .Output("handle: variant") - .SetIsStateful() // TODO(b/123753214): Source dataset ops must be marked - // stateful to inhibit constant folding. - .SetShapeFn(shape_inference::ScalarShape); - -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/ops/bigtable_test_ops.cc b/tensorflow/contrib/bigtable/ops/bigtable_test_ops.cc deleted file mode 100644 index f7d02458f63..00000000000 --- a/tensorflow/contrib/bigtable/ops/bigtable_test_ops.cc +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" - -namespace tensorflow { - -REGISTER_OP("BigtableTestClient") - .Attr("container: string = ''") - .Attr("shared_name: string = ''") - .Output("client: resource") - .SetShapeFn(shape_inference::ScalarShape); - -} // namespace tensorflow diff --git a/tensorflow/contrib/bigtable/python/kernel_tests/__init__.py b/tensorflow/contrib/bigtable/python/kernel_tests/__init__.py deleted file mode 100644 index 292d8f4e51a..00000000000 --- a/tensorflow/contrib/bigtable/python/kernel_tests/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""This module contains tests for the bigtable integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py b/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py deleted file mode 100644 index 197f5578eb0..00000000000 --- a/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py +++ /dev/null @@ -1,272 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Bigtable Ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import bigtable -from tensorflow.contrib.bigtable.ops import gen_bigtable_ops -from tensorflow.contrib.bigtable.ops import gen_bigtable_test_ops -from tensorflow.contrib.bigtable.python.ops import bigtable_api -from tensorflow.contrib.util import loader -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import errors -from tensorflow.python.platform import resource_loader -from tensorflow.python.platform import test -from tensorflow.python.util import compat - -_bigtable_so = loader.load_op_library( - resource_loader.get_path_to_datafile("_bigtable_test.so")) - - -def _ListOfTuplesOfStringsToBytes(values): - return [(compat.as_bytes(i[0]), compat.as_bytes(i[1])) for i in values] - - -class BigtableOpsTest(test.TestCase): - COMMON_ROW_KEYS = ["r1", "r2", "r3"] - COMMON_VALUES = ["v1", "v2", "v3"] - - def setUp(self): - self._client = gen_bigtable_test_ops.bigtable_test_client() - table = gen_bigtable_ops.bigtable_table(self._client, "testtable") - self._table = bigtable.BigtableTable("testtable", None, table) - - def _makeSimpleDataset(self): - output_rows = dataset_ops.Dataset.from_tensor_slices(self.COMMON_ROW_KEYS) - output_values = dataset_ops.Dataset.from_tensor_slices(self.COMMON_VALUES) - return dataset_ops.Dataset.zip((output_rows, output_values)) - - def _writeCommonValues(self, sess): - output_ds = self._makeSimpleDataset() - write_op = self._table.write(output_ds, ["cf1"], ["c1"]) - sess.run(write_op) - - def runReadKeyTest(self, read_ds): - itr = dataset_ops.make_initializable_iterator(read_ds) - n = itr.get_next() - expected = list(self.COMMON_ROW_KEYS) - expected.reverse() - with self.cached_session() as sess: - self._writeCommonValues(sess) - sess.run(itr.initializer) - for i in range(3): - output = sess.run(n) - want = expected.pop() - self.assertEqual( - compat.as_bytes(want), compat.as_bytes(output), - "Unequal at step %d: want: %s, got: %s" % (i, want, output)) - - def testReadPrefixKeys(self): - self.runReadKeyTest(self._table.keys_by_prefix_dataset("r")) - - def testReadRangeKeys(self): - self.runReadKeyTest(self._table.keys_by_range_dataset("r1", "r4")) - - def runScanTest(self, read_ds): - itr = dataset_ops.make_initializable_iterator(read_ds) - n = itr.get_next() - expected_keys = list(self.COMMON_ROW_KEYS) - expected_keys.reverse() - expected_values = list(self.COMMON_VALUES) - expected_values.reverse() - with self.cached_session() as sess: - self._writeCommonValues(sess) - sess.run(itr.initializer) - for i in range(3): - output = sess.run(n) - want = expected_keys.pop() - self.assertEqual( - compat.as_bytes(want), compat.as_bytes(output[0]), - "Unequal keys at step %d: want: %s, got: %s" % (i, want, output[0])) - want = expected_values.pop() - self.assertEqual( - compat.as_bytes(want), compat.as_bytes(output[1]), - "Unequal values at step: %d: want: %s, got: %s" % (i, want, - output[1])) - - def testScanPrefixStringCol(self): - self.runScanTest(self._table.scan_prefix("r", cf1="c1")) - - def testScanPrefixListCol(self): - self.runScanTest(self._table.scan_prefix("r", cf1=["c1"])) - - def testScanPrefixTupleCol(self): - self.runScanTest(self._table.scan_prefix("r", columns=("cf1", "c1"))) - - def testScanRangeStringCol(self): - self.runScanTest(self._table.scan_range("r1", "r4", cf1="c1")) - - def testScanRangeListCol(self): - self.runScanTest(self._table.scan_range("r1", "r4", cf1=["c1"])) - - def testScanRangeTupleCol(self): - self.runScanTest(self._table.scan_range("r1", "r4", columns=("cf1", "c1"))) - - def testLookup(self): - ds = self._table.keys_by_prefix_dataset("r") - ds = ds.apply(self._table.lookup_columns(cf1="c1")) - itr = dataset_ops.make_initializable_iterator(ds) - n = itr.get_next() - expected_keys = list(self.COMMON_ROW_KEYS) - expected_values = list(self.COMMON_VALUES) - expected_tuples = zip(expected_keys, expected_values) - with self.cached_session() as sess: - self._writeCommonValues(sess) - sess.run(itr.initializer) - for i, elem in enumerate(expected_tuples): - output = sess.run(n) - self.assertEqual( - compat.as_bytes(elem[0]), compat.as_bytes(output[0]), - "Unequal keys at step %d: want: %s, got: %s" % - (i, compat.as_bytes(elem[0]), compat.as_bytes(output[0]))) - self.assertEqual( - compat.as_bytes(elem[1]), compat.as_bytes(output[1]), - "Unequal values at step %d: want: %s, got: %s" % - (i, compat.as_bytes(elem[1]), compat.as_bytes(output[1]))) - - def testSampleKeys(self): - ds = self._table.sample_keys() - itr = dataset_ops.make_initializable_iterator(ds) - n = itr.get_next() - expected_key = self.COMMON_ROW_KEYS[0] - with self.cached_session() as sess: - self._writeCommonValues(sess) - sess.run(itr.initializer) - output = sess.run(n) - self.assertEqual( - compat.as_bytes(self.COMMON_ROW_KEYS[0]), compat.as_bytes(output), - "Unequal keys: want: %s, got: %s" % (compat.as_bytes( - self.COMMON_ROW_KEYS[0]), compat.as_bytes(output))) - output = sess.run(n) - self.assertEqual( - compat.as_bytes(self.COMMON_ROW_KEYS[2]), compat.as_bytes(output), - "Unequal keys: want: %s, got: %s" % (compat.as_bytes( - self.COMMON_ROW_KEYS[2]), compat.as_bytes(output))) - with self.assertRaises(errors.OutOfRangeError): - sess.run(n) - - def runSampleKeyPairsTest(self, ds, expected_key_pairs): - itr = dataset_ops.make_initializable_iterator(ds) - n = itr.get_next() - with self.cached_session() as sess: - self._writeCommonValues(sess) - sess.run(itr.initializer) - for i, elems in enumerate(expected_key_pairs): - output = sess.run(n) - self.assertEqual( - compat.as_bytes(elems[0]), compat.as_bytes(output[0]), - "Unequal key pair (first element) at step %d; want: %s, got %s" % - (i, compat.as_bytes(elems[0]), compat.as_bytes(output[0]))) - self.assertEqual( - compat.as_bytes(elems[1]), compat.as_bytes(output[1]), - "Unequal key pair (second element) at step %d; want: %s, got %s" % - (i, compat.as_bytes(elems[1]), compat.as_bytes(output[1]))) - with self.assertRaises(errors.OutOfRangeError): - sess.run(n) - - def testSampleKeyPairsSimplePrefix(self): - ds = bigtable_api._BigtableSampleKeyPairsDataset( - self._table, prefix="r", start="", end="") - expected_key_pairs = [("r", "r1"), ("r1", "r3"), ("r3", "s")] - self.runSampleKeyPairsTest(ds, expected_key_pairs) - - def testSampleKeyPairsSimpleRange(self): - ds = bigtable_api._BigtableSampleKeyPairsDataset( - self._table, prefix="", start="r1", end="r3") - expected_key_pairs = [("r1", "r3")] - self.runSampleKeyPairsTest(ds, expected_key_pairs) - - def testSampleKeyPairsSkipRangePrefix(self): - ds = bigtable_api._BigtableSampleKeyPairsDataset( - self._table, prefix="r2", start="", end="") - expected_key_pairs = [("r2", "r3")] - self.runSampleKeyPairsTest(ds, expected_key_pairs) - - def testSampleKeyPairsSkipRangeRange(self): - ds = bigtable_api._BigtableSampleKeyPairsDataset( - self._table, prefix="", start="r2", end="r3") - expected_key_pairs = [("r2", "r3")] - self.runSampleKeyPairsTest(ds, expected_key_pairs) - - def testSampleKeyPairsOffsetRanges(self): - ds = bigtable_api._BigtableSampleKeyPairsDataset( - self._table, prefix="", start="r2", end="r4") - expected_key_pairs = [("r2", "r3"), ("r3", "r4")] - self.runSampleKeyPairsTest(ds, expected_key_pairs) - - def testSampleKeyPairEverything(self): - ds = bigtable_api._BigtableSampleKeyPairsDataset( - self._table, prefix="", start="", end="") - expected_key_pairs = [("", "r1"), ("r1", "r3"), ("r3", "")] - self.runSampleKeyPairsTest(ds, expected_key_pairs) - - def testSampleKeyPairsPrefixAndStartKey(self): - ds = bigtable_api._BigtableSampleKeyPairsDataset( - self._table, prefix="r", start="r1", end="") - itr = dataset_ops.make_initializable_iterator(ds) - with self.cached_session() as sess: - with self.assertRaises(errors.InvalidArgumentError): - sess.run(itr.initializer) - - def testSampleKeyPairsPrefixAndEndKey(self): - ds = bigtable_api._BigtableSampleKeyPairsDataset( - self._table, prefix="r", start="", end="r3") - itr = dataset_ops.make_initializable_iterator(ds) - with self.cached_session() as sess: - with self.assertRaises(errors.InvalidArgumentError): - sess.run(itr.initializer) - - def testParallelScanPrefix(self): - ds = self._table.parallel_scan_prefix(prefix="r", cf1="c1") - itr = dataset_ops.make_initializable_iterator(ds) - n = itr.get_next() - with self.cached_session() as sess: - self._writeCommonValues(sess) - sess.run(itr.initializer) - expected_values = list(zip(self.COMMON_ROW_KEYS, self.COMMON_VALUES)) - actual_values = [] - for _ in range(len(expected_values)): - output = sess.run(n) - actual_values.append(output) - with self.assertRaises(errors.OutOfRangeError): - sess.run(n) - self.assertItemsEqual( - _ListOfTuplesOfStringsToBytes(expected_values), - _ListOfTuplesOfStringsToBytes(actual_values)) - - def testParallelScanRange(self): - ds = self._table.parallel_scan_range(start="r1", end="r4", cf1="c1") - itr = dataset_ops.make_initializable_iterator(ds) - n = itr.get_next() - with self.cached_session() as sess: - self._writeCommonValues(sess) - sess.run(itr.initializer) - expected_values = list(zip(self.COMMON_ROW_KEYS, self.COMMON_VALUES)) - actual_values = [] - for _ in range(len(expected_values)): - output = sess.run(n) - actual_values.append(output) - with self.assertRaises(errors.OutOfRangeError): - sess.run(n) - self.assertItemsEqual( - _ListOfTuplesOfStringsToBytes(expected_values), - _ListOfTuplesOfStringsToBytes(actual_values)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/bigtable/python/ops/__init__.py b/tensorflow/contrib/bigtable/python/ops/__init__.py deleted file mode 100644 index 36d75b0d706..00000000000 --- a/tensorflow/contrib/bigtable/python/ops/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""This module contains the Python API for the Cloud Bigtable integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/bigtable/python/ops/bigtable_api.py b/tensorflow/contrib/bigtable/python/ops/bigtable_api.py deleted file mode 100644 index e55c0dc7806..00000000000 --- a/tensorflow/contrib/bigtable/python/ops/bigtable_api.py +++ /dev/null @@ -1,708 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Python API for TensorFlow's Cloud Bigtable integration. - -TensorFlow has support for reading from and writing to Cloud Bigtable. To use -TensorFlow + Cloud Bigtable integration, first create a BigtableClient to -configure your connection to Cloud Bigtable, and then create a BigtableTable -object to allow you to create numerous `tf.data.Dataset`s to read data, or -write a `tf.data.Dataset` object to the underlying Cloud Bigtable table. - -For background on Cloud Bigtable, see: https://cloud.google.com/bigtable . -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from six import iteritems -from six import string_types - -from tensorflow.contrib.bigtable.ops import gen_bigtable_ops -from tensorflow.contrib.util import loader -from tensorflow.python.data.experimental.ops import interleave_ops -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.util import nest -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_spec -from tensorflow.python.platform import resource_loader - -_bigtable_so = loader.load_op_library( - resource_loader.get_path_to_datafile("_bigtable.so")) - - -class BigtableClient(object): - """BigtableClient is the entrypoint for interacting with Cloud Bigtable in TF. - - BigtableClient encapsulates a connection to Cloud Bigtable, and exposes the - `table` method to open a Bigtable table. - """ - - def __init__(self, - project_id, - instance_id, - connection_pool_size=None, - max_receive_message_size=None): - """Creates a BigtableClient that can be used to open connections to tables. - - Args: - project_id: A string representing the GCP project id to connect to. - instance_id: A string representing the Bigtable instance to connect to. - connection_pool_size: (Optional.) A number representing the number of - concurrent connections to the Cloud Bigtable service to make. - max_receive_message_size: (Optional.) The maximum bytes received in a - single gRPC response. - - Raises: - ValueError: if the arguments are invalid (e.g. wrong type, or out of - expected ranges (e.g. negative).) - """ - if not isinstance(project_id, str): - raise ValueError("`project_id` must be a string") - self._project_id = project_id - - if not isinstance(instance_id, str): - raise ValueError("`instance_id` must be a string") - self._instance_id = instance_id - - if connection_pool_size is None: - connection_pool_size = -1 - elif connection_pool_size < 1: - raise ValueError("`connection_pool_size` must be positive") - - if max_receive_message_size is None: - max_receive_message_size = -1 - elif max_receive_message_size < 1: - raise ValueError("`max_receive_message_size` must be positive") - - self._connection_pool_size = connection_pool_size - - self._resource = gen_bigtable_ops.bigtable_client( - project_id, instance_id, connection_pool_size, max_receive_message_size) - - def table(self, name, snapshot=None): - """Opens a table and returns a `tf.contrib.bigtable.BigtableTable` object. - - Args: - name: A `tf.string` `tf.Tensor` name of the table to open. - snapshot: Either a `tf.string` `tf.Tensor` snapshot id, or `True` to - request the creation of a snapshot. (Note: currently unimplemented.) - - Returns: - A `tf.contrib.bigtable.BigtableTable` Python object representing the - operations available on the table. - """ - # TODO(saeta): Implement snapshot functionality. - table = gen_bigtable_ops.bigtable_table(self._resource, name) - return BigtableTable(name, snapshot, table) - - -class BigtableTable(object): - """Entry point for reading and writing data in Cloud Bigtable. - - This BigtableTable class is the Python representation of the Cloud Bigtable - table within TensorFlow. Methods on this class allow data to be read from and - written to the Cloud Bigtable service in flexible and high performance - manners. - """ - - # TODO(saeta): Investigate implementing tf.contrib.lookup.LookupInterface. - # TODO(saeta): Consider variant tensors instead of resources (while supporting - # connection pooling). - - def __init__(self, name, snapshot, resource): - self._name = name - self._snapshot = snapshot - self._resource = resource - - def lookup_columns(self, *args, **kwargs): - """Retrieves the values of columns for a dataset of keys. - - Example usage: - - ```python - table = bigtable_client.table("my_table") - key_dataset = table.get_keys_prefix("imagenet") - images = key_dataset.apply(table.lookup_columns(("cf1", "image"), - ("cf2", "label"), - ("cf2", "boundingbox"))) - training_data = images.map(parse_and_crop, num_parallel_calls=64).batch(128) - ``` - - Alternatively, you can use keyword arguments to specify the columns to - capture. Example (same as above, rewritten): - - ```python - table = bigtable_client.table("my_table") - key_dataset = table.get_keys_prefix("imagenet") - images = key_dataset.apply(table.lookup_columns( - cf1="image", cf2=("label", "boundingbox"))) - training_data = images.map(parse_and_crop, num_parallel_calls=64).batch(128) - ``` - - Note: certain `kwargs` keys are reserved, and thus, some column families - cannot be identified using the `kwargs` syntax. Instead, please use the - `args` syntax. This list includes: - - - 'name' - - Note: this list can change at any time. - - Args: - *args: A list of tuples containing (column family, column name) pairs. - **kwargs: Column families (keys) and column qualifiers (values). - - Returns: - A function that can be passed to `tf.data.Dataset.apply` to retrieve the - values of columns for the rows. - """ - table = self # Capture self - normalized = args - if normalized is None: - normalized = [] - if isinstance(normalized, tuple): - normalized = list(normalized) - for key, value in iteritems(kwargs): - if key == "name": - continue - if isinstance(value, str): - normalized.append((key, value)) - continue - for col in value: - normalized.append((key, col)) - - def _apply_fn(dataset): - # TODO(saeta): Verify dataset's types are correct! - return _BigtableLookupDataset(dataset, table, normalized) - - return _apply_fn - - def keys_by_range_dataset(self, start, end): - """Retrieves all row keys between start and end. - - Note: it does NOT retrieve the values of columns. - - Args: - start: The start row key. The row keys for rows after start (inclusive) - will be retrieved. - end: (Optional.) The end row key. Rows up to (but not including) end will - be retrieved. If end is None, all subsequent row keys will be retrieved. - - Returns: - A `tf.data.Dataset` containing `tf.string` Tensors corresponding to all - of the row keys between `start` and `end`. - """ - # TODO(saeta): Make inclusive / exclusive configurable? - if end is None: - end = "" - return _BigtableRangeKeyDataset(self, start, end) - - def keys_by_prefix_dataset(self, prefix): - """Retrieves the row keys matching a given prefix. - - Args: - prefix: All row keys that begin with `prefix` in the table will be - retrieved. - - Returns: - A `tf.data.Dataset`. containing `tf.string` Tensors corresponding to all - of the row keys matching that prefix. - """ - return _BigtablePrefixKeyDataset(self, prefix) - - def sample_keys(self): - """Retrieves a sampling of row keys from the Bigtable table. - - This dataset is most often used in conjunction with - `tf.data.experimental.parallel_interleave` to construct a set of ranges for - scanning in parallel. - - Returns: - A `tf.data.Dataset` returning string row keys. - """ - return _BigtableSampleKeysDataset(self) - - def scan_prefix(self, prefix, probability=None, columns=None, **kwargs): - """Retrieves row (including values) from the Bigtable service. - - Rows with row-key prefixed by `prefix` will be retrieved. - - Specifying the columns to retrieve for each row is done by either using - kwargs or in the columns parameter. To retrieve values of the columns "c1", - and "c2" from the column family "cfa", and the value of the column "c3" - from column family "cfb", the following datasets (`ds1`, and `ds2`) are - equivalent: - - ``` - table = # ... - ds1 = table.scan_prefix("row_prefix", columns=[("cfa", "c1"), - ("cfa", "c2"), - ("cfb", "c3")]) - ds2 = table.scan_prefix("row_prefix", cfa=["c1", "c2"], cfb="c3") - ``` - - Note: only the latest value of a cell will be retrieved. - - Args: - prefix: The prefix all row keys must match to be retrieved for prefix- - based scans. - probability: (Optional.) A float between 0 (exclusive) and 1 (inclusive). - A non-1 value indicates to probabilistically sample rows with the - provided probability. - columns: The columns to read. Note: most commonly, they are expressed as - kwargs. Use the columns value if you are using column families that are - reserved. The value of columns and kwargs are merged. Columns is a list - of tuples of strings ("column_family", "column_qualifier"). - **kwargs: The column families and columns to read. Keys are treated as - column_families, and values can be either lists of strings, or strings - that are treated as the column qualifier (column name). - - Returns: - A `tf.data.Dataset` returning the row keys and the cell contents. - - Raises: - ValueError: If the configured probability is unexpected. - """ - probability = _normalize_probability(probability) - normalized = _normalize_columns(columns, kwargs) - return _BigtableScanDataset(self, prefix, "", "", normalized, probability) - - def scan_range(self, start, end, probability=None, columns=None, **kwargs): - """Retrieves rows (including values) from the Bigtable service. - - Rows with row-keys between `start` and `end` will be retrieved. - - Specifying the columns to retrieve for each row is done by either using - kwargs or in the columns parameter. To retrieve values of the columns "c1", - and "c2" from the column family "cfa", and the value of the column "c3" - from column family "cfb", the following datasets (`ds1`, and `ds2`) are - equivalent: - - ``` - table = # ... - ds1 = table.scan_range("row_start", "row_end", columns=[("cfa", "c1"), - ("cfa", "c2"), - ("cfb", "c3")]) - ds2 = table.scan_range("row_start", "row_end", cfa=["c1", "c2"], cfb="c3") - ``` - - Note: only the latest value of a cell will be retrieved. - - Args: - start: The start of the range when scanning by range. - end: (Optional.) The end of the range when scanning by range. - probability: (Optional.) A float between 0 (exclusive) and 1 (inclusive). - A non-1 value indicates to probabilistically sample rows with the - provided probability. - columns: The columns to read. Note: most commonly, they are expressed as - kwargs. Use the columns value if you are using column families that are - reserved. The value of columns and kwargs are merged. Columns is a list - of tuples of strings ("column_family", "column_qualifier"). - **kwargs: The column families and columns to read. Keys are treated as - column_families, and values can be either lists of strings, or strings - that are treated as the column qualifier (column name). - - Returns: - A `tf.data.Dataset` returning the row keys and the cell contents. - - Raises: - ValueError: If the configured probability is unexpected. - """ - probability = _normalize_probability(probability) - normalized = _normalize_columns(columns, kwargs) - return _BigtableScanDataset(self, "", start, end, normalized, probability) - - def parallel_scan_prefix(self, - prefix, - num_parallel_scans=None, - probability=None, - columns=None, - **kwargs): - """Retrieves row (including values) from the Bigtable service at high speed. - - Rows with row-key prefixed by `prefix` will be retrieved. This method is - similar to `scan_prefix`, but by contrast performs multiple sub-scans in - parallel in order to achieve higher performance. - - Note: The dataset produced by this method is not deterministic! - - Specifying the columns to retrieve for each row is done by either using - kwargs or in the columns parameter. To retrieve values of the columns "c1", - and "c2" from the column family "cfa", and the value of the column "c3" - from column family "cfb", the following datasets (`ds1`, and `ds2`) are - equivalent: - - ``` - table = # ... - ds1 = table.parallel_scan_prefix("row_prefix", columns=[("cfa", "c1"), - ("cfa", "c2"), - ("cfb", "c3")]) - ds2 = table.parallel_scan_prefix("row_prefix", cfa=["c1", "c2"], cfb="c3") - ``` - - Note: only the latest value of a cell will be retrieved. - - Args: - prefix: The prefix all row keys must match to be retrieved for prefix- - based scans. - num_parallel_scans: (Optional.) The number of concurrent scans against the - Cloud Bigtable instance. - probability: (Optional.) A float between 0 (exclusive) and 1 (inclusive). - A non-1 value indicates to probabilistically sample rows with the - provided probability. - columns: The columns to read. Note: most commonly, they are expressed as - kwargs. Use the columns value if you are using column families that are - reserved. The value of columns and kwargs are merged. Columns is a list - of tuples of strings ("column_family", "column_qualifier"). - **kwargs: The column families and columns to read. Keys are treated as - column_families, and values can be either lists of strings, or strings - that are treated as the column qualifier (column name). - - Returns: - A `tf.data.Dataset` returning the row keys and the cell contents. - - Raises: - ValueError: If the configured probability is unexpected. - """ - probability = _normalize_probability(probability) - normalized = _normalize_columns(columns, kwargs) - ds = _BigtableSampleKeyPairsDataset(self, prefix, "", "") - return self._make_parallel_scan_dataset(ds, num_parallel_scans, probability, - normalized) - - def parallel_scan_range(self, - start, - end, - num_parallel_scans=None, - probability=None, - columns=None, - **kwargs): - """Retrieves rows (including values) from the Bigtable service. - - Rows with row-keys between `start` and `end` will be retrieved. This method - is similar to `scan_range`, but by contrast performs multiple sub-scans in - parallel in order to achieve higher performance. - - Note: The dataset produced by this method is not deterministic! - - Specifying the columns to retrieve for each row is done by either using - kwargs or in the columns parameter. To retrieve values of the columns "c1", - and "c2" from the column family "cfa", and the value of the column "c3" - from column family "cfb", the following datasets (`ds1`, and `ds2`) are - equivalent: - - ``` - table = # ... - ds1 = table.parallel_scan_range("row_start", - "row_end", - columns=[("cfa", "c1"), - ("cfa", "c2"), - ("cfb", "c3")]) - ds2 = table.parallel_scan_range("row_start", "row_end", - cfa=["c1", "c2"], cfb="c3") - ``` - - Note: only the latest value of a cell will be retrieved. - - Args: - start: The start of the range when scanning by range. - end: (Optional.) The end of the range when scanning by range. - num_parallel_scans: (Optional.) The number of concurrent scans against the - Cloud Bigtable instance. - probability: (Optional.) A float between 0 (exclusive) and 1 (inclusive). - A non-1 value indicates to probabilistically sample rows with the - provided probability. - columns: The columns to read. Note: most commonly, they are expressed as - kwargs. Use the columns value if you are using column families that are - reserved. The value of columns and kwargs are merged. Columns is a list - of tuples of strings ("column_family", "column_qualifier"). - **kwargs: The column families and columns to read. Keys are treated as - column_families, and values can be either lists of strings, or strings - that are treated as the column qualifier (column name). - - Returns: - A `tf.data.Dataset` returning the row keys and the cell contents. - - Raises: - ValueError: If the configured probability is unexpected. - """ - probability = _normalize_probability(probability) - normalized = _normalize_columns(columns, kwargs) - ds = _BigtableSampleKeyPairsDataset(self, "", start, end) - return self._make_parallel_scan_dataset(ds, num_parallel_scans, probability, - normalized) - - def write(self, dataset, column_families, columns, timestamp=None): - """Writes a dataset to the table. - - Args: - dataset: A `tf.data.Dataset` to be written to this table. It must produce - a list of number-of-columns+1 elements, all of which must be strings. - The first value will be used as the row key, and subsequent values will - be used as cell values for the corresponding columns from the - corresponding column_families and columns entries. - column_families: A `tf.Tensor` of `tf.string`s corresponding to the - column names to store the dataset's elements into. - columns: A `tf.Tensor` of `tf.string`s corresponding to the column names - to store the dataset's elements into. - timestamp: (Optional.) An int64 timestamp to write all the values at. - Leave as None to use server-provided timestamps. - - Returns: - A `tf.Operation` that can be run to perform the write. - - Raises: - ValueError: If there are unexpected or incompatible types, or if the - number of columns and column_families does not match the output of - `dataset`. - """ - if timestamp is None: - timestamp = -1 # Bigtable server provided timestamp. - for tensor_type in nest.flatten( - dataset_ops.get_legacy_output_types(dataset)): - if tensor_type != dtypes.string: - raise ValueError("Not all elements of the dataset were `tf.string`") - for shape in nest.flatten(dataset_ops.get_legacy_output_shapes(dataset)): - if not shape.is_compatible_with(tensor_shape.TensorShape([])): - raise ValueError("Not all elements of the dataset were scalars") - if len(column_families) != len(columns): - raise ValueError("len(column_families) != len(columns)") - if len(nest.flatten( - dataset_ops.get_legacy_output_types(dataset))) != len(columns) + 1: - raise ValueError("A column name must be specified for every component of " - "the dataset elements. (e.g.: len(columns) != " - "len(dataset.output_types))") - return gen_bigtable_ops.dataset_to_bigtable( - self._resource, - dataset._variant_tensor, # pylint: disable=protected-access - column_families, - columns, - timestamp) - - def _make_parallel_scan_dataset(self, ds, num_parallel_scans, - normalized_probability, normalized_columns): - """Builds a parallel dataset from a given range. - - Args: - ds: A `_BigtableSampleKeyPairsDataset` returning ranges of keys to use. - num_parallel_scans: The number of concurrent parallel scans to use. - normalized_probability: A number between 0 and 1 for the keep probability. - normalized_columns: The column families and column qualifiers to retrieve. - - Returns: - A `tf.data.Dataset` representing the result of the parallel scan. - """ - if num_parallel_scans is None: - num_parallel_scans = 50 - - ds = ds.shuffle(buffer_size=10000) # TODO(saeta): Make configurable. - - def _interleave_fn(start, end): - return _BigtableScanDataset( - self, - prefix="", - start=start, - end=end, - normalized=normalized_columns, - probability=normalized_probability) - - # Note prefetch_input_elements must be set in order to avoid rpc timeouts. - ds = ds.apply( - interleave_ops.parallel_interleave( - _interleave_fn, - cycle_length=num_parallel_scans, - sloppy=True, - prefetch_input_elements=1)) - return ds - - -def _normalize_probability(probability): - if probability is None: - probability = 1.0 - if isinstance(probability, float) and (probability <= 0.0 or - probability > 1.0): - raise ValueError("probability must be in the range (0, 1].") - return probability - - -def _normalize_columns(columns, provided_kwargs): - """Converts arguments (columns, and kwargs dict) to C++ representation. - - Args: - columns: a datastructure containing the column families and qualifier to - retrieve. Valid types include (1) None, (2) list of tuples, (3) a tuple of - strings. - provided_kwargs: a dictionary containing the column families and qualifiers - to retrieve - - Returns: - A list of pairs of column family+qualifier to retrieve. - - Raises: - ValueError: If there are no cells to retrieve or the columns are in an - incorrect format. - """ - normalized = columns - if normalized is None: - normalized = [] - if isinstance(normalized, tuple): - if len(normalized) == 2: - normalized = [normalized] - else: - raise ValueError("columns was a tuple of inappropriate length") - for key, value in iteritems(provided_kwargs): - if key == "name": - continue - if isinstance(value, string_types): - normalized.append((key, value)) - continue - for col in value: - normalized.append((key, col)) - if not normalized: - raise ValueError("At least one column + column family must be specified.") - return normalized - - -class _BigtableKeyDataset(dataset_ops.DatasetSource): - """_BigtableKeyDataset is an abstract class representing the keys of a table. - """ - - def __init__(self, table, variant_tensor): - """Constructs a _BigtableKeyDataset. - - Args: - table: a Bigtable class. - variant_tensor: DT_VARIANT representation of the dataset. - """ - super(_BigtableKeyDataset, self).__init__(variant_tensor) - self._table = table - - @property - def element_spec(self): - return tensor_spec.TensorSpec([], dtypes.string) - - -class _BigtablePrefixKeyDataset(_BigtableKeyDataset): - """_BigtablePrefixKeyDataset represents looking up keys by prefix. - """ - - def __init__(self, table, prefix): - self._prefix = prefix - variant_tensor = gen_bigtable_ops.bigtable_prefix_key_dataset( - table=table._resource, # pylint: disable=protected-access - prefix=self._prefix) - super(_BigtablePrefixKeyDataset, self).__init__(table, variant_tensor) - - -class _BigtableRangeKeyDataset(_BigtableKeyDataset): - """_BigtableRangeKeyDataset represents looking up keys by range. - """ - - def __init__(self, table, start, end): - self._start = start - self._end = end - variant_tensor = gen_bigtable_ops.bigtable_range_key_dataset( - table=table._resource, # pylint: disable=protected-access - start_key=self._start, - end_key=self._end) - super(_BigtableRangeKeyDataset, self).__init__(table, variant_tensor) - - -class _BigtableSampleKeysDataset(_BigtableKeyDataset): - """_BigtableSampleKeysDataset represents a sampling of row keys. - """ - - # TODO(saeta): Expose the data size offsets into the keys. - - def __init__(self, table): - variant_tensor = gen_bigtable_ops.bigtable_sample_keys_dataset( - table=table._resource) # pylint: disable=protected-access - super(_BigtableSampleKeysDataset, self).__init__(table, variant_tensor) - - -class _BigtableLookupDataset(dataset_ops.DatasetSource): - """_BigtableLookupDataset represents a dataset that retrieves values for keys. - """ - - def __init__(self, dataset, table, normalized): - self._num_outputs = len(normalized) + 1 # 1 for row key - self._dataset = dataset - self._table = table - self._normalized = normalized - self._column_families = [i[0] for i in normalized] - self._columns = [i[1] for i in normalized] - variant_tensor = gen_bigtable_ops.bigtable_lookup_dataset( - keys_dataset=self._dataset._variant_tensor, # pylint: disable=protected-access - table=self._table._resource, # pylint: disable=protected-access - column_families=self._column_families, - columns=self._columns) - super(_BigtableLookupDataset, self).__init__(variant_tensor) - - @property - def element_spec(self): - return tuple([tensor_spec.TensorSpec([], dtypes.string)] * - self._num_outputs) - - -class _BigtableScanDataset(dataset_ops.DatasetSource): - """_BigtableScanDataset represents a dataset that retrieves keys and values. - """ - - def __init__(self, table, prefix, start, end, normalized, probability): - self._table = table - self._prefix = prefix - self._start = start - self._end = end - self._column_families = [i[0] for i in normalized] - self._columns = [i[1] for i in normalized] - self._probability = probability - self._num_outputs = len(normalized) + 1 # 1 for row key - variant_tensor = gen_bigtable_ops.bigtable_scan_dataset( - table=self._table._resource, # pylint: disable=protected-access - prefix=self._prefix, - start_key=self._start, - end_key=self._end, - column_families=self._column_families, - columns=self._columns, - probability=self._probability) - super(_BigtableScanDataset, self).__init__(variant_tensor) - - @property - def element_spec(self): - return tuple([tensor_spec.TensorSpec([], dtypes.string)] * - self._num_outputs) - - -class _BigtableSampleKeyPairsDataset(dataset_ops.DatasetSource): - """_BigtableSampleKeyPairsDataset returns key pairs from a Bigtable table. - """ - - def __init__(self, table, prefix, start, end): - self._table = table - self._prefix = prefix - self._start = start - self._end = end - variant_tensor = gen_bigtable_ops.bigtable_sample_key_pairs_dataset( - table=self._table._resource, # pylint: disable=protected-access - prefix=self._prefix, - start_key=self._start, - end_key=self._end) - super(_BigtableSampleKeyPairsDataset, self).__init__(variant_tensor) - - @property - def element_spec(self): - return (tensor_spec.TensorSpec([], dtypes.string), - tensor_spec.TensorSpec([], dtypes.string)) diff --git a/tensorflow/contrib/boosted_trees/BUILD b/tensorflow/contrib/boosted_trees/BUILD deleted file mode 100644 index 71cca4e10b3..00000000000 --- a/tensorflow/contrib/boosted_trees/BUILD +++ /dev/null @@ -1,614 +0,0 @@ -# TensorFlow code for training gradient boosted trees. - -load("//tensorflow:tensorflow.bzl", "py_test", "tf_custom_op_library", "tf_gen_op_libs", "tf_gen_op_wrapper_py", "tf_kernel_library") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = [ - "//visibility:public", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -package_group(name = "friends") - -cc_library( - name = "boosted_trees_kernels", - deps = [ - ":model_ops_kernels", - ":prediction_ops_kernels", - ":quantile_ops_kernels", - ":split_handler_ops_kernels", - ":stats_accumulator_ops_kernels", - ":training_ops_kernels", - ], - alwayslink = 1, -) - -cc_library( - name = "boosted_trees_ops_op_lib", - deps = [ - ":model_ops_op_lib", - ":prediction_ops_op_lib", - ":quantile_ops_op_lib", - ":split_handler_ops_op_lib", - ":stats_accumulator_ops_op_lib", - ":training_ops_op_lib", - ], -) - -py_library( - name = "init_py", - srcs = [ - "__init__.py", - "python/__init__.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":boosted_trees_ops_py", - ":losses", - ], -) - -py_library( - name = "losses", - srcs = ["python/utils/losses.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - ], -) - -py_test( - name = "losses_test", - size = "small", - srcs = ["python/utils/losses_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":losses", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//third_party/py/numpy", - ], -) - -py_library( - name = "gbdt_batch", - srcs = [ - "python/training/functions/gbdt_batch.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":gen_model_ops_py", - "//tensorflow/contrib/boosted_trees:batch_ops_utils_py", - "//tensorflow/contrib/boosted_trees:boosted_trees_ops_py", - "//tensorflow/contrib/boosted_trees/lib:categorical_split_handler", - "//tensorflow/contrib/boosted_trees/lib:ordinal_split_handler", - "//tensorflow/contrib/boosted_trees/proto:learner_proto_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:gradients", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:stateless_random_ops", - "//tensorflow/python:summary", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/feature_column", - ], -) - -py_test( - name = "gbdt_batch_test", - size = "medium", - srcs = ["python/training/functions/gbdt_batch_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "nofwdcompat", # b/137641346 - "notsan", # b/62863147 - ], - deps = [ - ":gbdt_batch", - ":losses", - ":model_ops_py", - "//tensorflow/contrib/boosted_trees/proto:learner_proto_py", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:resources", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:variables", - ], -) - -# Kernel tests - -py_test( - name = "model_ops_test", - size = "small", - srcs = ["python/kernel_tests/model_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":model_ops_py", - ":prediction_ops_py", - "//tensorflow/contrib/boosted_trees/proto:learner_proto_py", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_py", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:resources", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "prediction_ops_test", - size = "small", - srcs = ["python/kernel_tests/prediction_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":model_ops_py", - ":prediction_ops_py", - "//tensorflow/contrib/boosted_trees/proto:learner_proto_py", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_py", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:resources", - "//third_party/py/numpy", - ], -) - -py_test( - name = "quantile_ops_test", - size = "small", - srcs = ["python/kernel_tests/quantile_ops_test.py"], - python_version = "PY2", - shard_count = 3, - srcs_version = "PY2AND3", - deps = [ - ":quantile_ops_py", - "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:resources", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) - -py_test( - name = "split_handler_ops_test", - size = "small", - srcs = ["python/kernel_tests/split_handler_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":split_handler_ops_py", - "//tensorflow/contrib/boosted_trees/proto:learner_proto_py", - "//tensorflow/contrib/boosted_trees/proto:split_info_proto_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -py_test( - name = "stats_accumulator_ops_test", - size = "small", - srcs = ["python/kernel_tests/stats_accumulator_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":stats_accumulator_ops_py", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:tensor_shape", - ], -) - -py_test( - name = "training_ops_test", - size = "small", - srcs = ["python/kernel_tests/training_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":model_ops_py", - ":training_ops_py", - "//tensorflow/contrib/boosted_trees/proto:learner_proto_py", - "//tensorflow/contrib/boosted_trees/proto:split_info_proto_py", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:resources", - "//third_party/py/numpy", - ], -) - -# Ops - -py_library( - name = "batch_ops_utils_py", - srcs = ["python/ops/batch_ops_utils.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:tensor_shape", - ], -) - -tf_custom_op_py_library( - name = "boosted_trees_ops_loader", - srcs = ["python/ops/boosted_trees_ops_loader.py"], - dso = [":python/ops/_boosted_trees_ops.so"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:errors", - "//tensorflow/python:platform", - ], -) - -py_library( - name = "boosted_trees_ops_py", - srcs_version = "PY2AND3", - deps = [ - ":model_ops_py", - ":prediction_ops_py", - ":quantile_ops_py", - ":split_handler_ops_py", - ":stats_accumulator_ops_py", - ":training_ops_py", - ], -) - -# Model Ops. -tf_gen_op_libs( - op_lib_names = ["model_ops"], -) - -tf_gen_op_wrapper_py( - name = "gen_model_ops_py", - out = "python/ops/gen_model_ops.py", - deps = [":model_ops_op_lib"], -) - -tf_custom_op_py_library( - name = "model_ops_py", - srcs = ["python/ops/model_ops.py"], - kernels = [ - ":model_ops_kernels", - ":model_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":boosted_trees_ops_loader", - ":gen_model_ops_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:resources", - "//tensorflow/python:training", - ], -) - -tf_kernel_library( - name = "model_ops_kernels", - srcs = ["kernels/model_ops.cc"], - deps = [ - "//tensorflow/contrib/boosted_trees/lib:utils", - "//tensorflow/contrib/boosted_trees/resources:decision_tree_ensemble_resource", - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - ], - alwayslink = 1, -) - -tf_custom_op_library( - name = "python/ops/_boosted_trees_ops.so", - srcs = [ - "kernels/model_ops.cc", - "kernels/prediction_ops.cc", - "kernels/quantile_ops.cc", - "kernels/split_handler_ops.cc", - "kernels/stats_accumulator_ops.cc", - "kernels/training_ops.cc", - "ops/model_ops.cc", - "ops/prediction_ops.cc", - "ops/quantile_ops.cc", - "ops/split_handler_ops.cc", - "ops/stats_accumulator_ops.cc", - "ops/training_ops.cc", - ], - deps = [ - "//tensorflow/contrib/boosted_trees/lib:example_partitioner", - "//tensorflow/contrib/boosted_trees/lib:models", - "//tensorflow/contrib/boosted_trees/lib:node-stats", - "//tensorflow/contrib/boosted_trees/lib:utils", - "//tensorflow/contrib/boosted_trees/lib:weighted_quantiles", - "//tensorflow/contrib/boosted_trees/proto:learner_proto_cc", - "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_cc", - "//tensorflow/contrib/boosted_trees/proto:split_info_proto_cc", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc", - "//tensorflow/contrib/boosted_trees/resources:decision_tree_ensemble_resource", - "//tensorflow/contrib/boosted_trees/resources:quantile_stream_resource", - "//tensorflow/contrib/boosted_trees/resources:stamped_resource", - ], -) - -# Split handler Ops. -tf_gen_op_libs( - op_lib_names = ["split_handler_ops"], -) - -tf_gen_op_wrapper_py( - name = "gen_split_handler_ops_py", - out = "python/ops/gen_split_handler_ops.py", - deps = [ - ":split_handler_ops_op_lib", - ], -) - -tf_custom_op_py_library( - name = "split_handler_ops_py", - srcs = ["python/ops/split_handler_ops.py"], - kernels = [ - ":split_handler_ops_kernels", - ":split_handler_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":boosted_trees_ops_loader", - ":gen_split_handler_ops_py", - ], -) - -tf_kernel_library( - name = "split_handler_ops_kernels", - srcs = ["kernels/split_handler_ops.cc"], - deps = [ - "//tensorflow/contrib/boosted_trees/lib:node-stats", - "//tensorflow/contrib/boosted_trees/proto:split_info_proto_cc", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc", - "//tensorflow/core:framework_headers_lib", - "//tensorflow/core:protos_all_cc", - "//third_party/eigen3", - ], - alwayslink = 1, -) - -# Training Ops. -tf_gen_op_libs( - op_lib_names = [ - "training_ops", - ], - deps = ["//tensorflow/contrib/boosted_trees/proto:learner_proto_cc"], -) - -tf_gen_op_wrapper_py( - name = "gen_training_ops_py", - out = "python/ops/gen_training_ops.py", - deps = [ - ":training_ops_op_lib", - ], -) - -tf_custom_op_py_library( - name = "training_ops_py", - srcs = ["python/ops/training_ops.py"], - kernels = [ - ":training_ops_kernels", - ":training_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":boosted_trees_ops_loader", - ":gen_training_ops_py", - ], -) - -tf_kernel_library( - name = "training_ops_kernels", - srcs = ["kernels/training_ops.cc"], - deps = [ - "//tensorflow/contrib/boosted_trees/lib:utils", - "//tensorflow/contrib/boosted_trees/lib:weighted_quantiles", - "//tensorflow/contrib/boosted_trees/proto:learner_proto_cc", - "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_cc", - "//tensorflow/contrib/boosted_trees/proto:split_info_proto_cc", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc", - "//tensorflow/contrib/boosted_trees/resources:decision_tree_ensemble_resource", - "//tensorflow/contrib/boosted_trees/resources:quantile_stream_resource", - "//tensorflow/core:framework_headers_lib", - ], - alwayslink = 1, -) - -# Prediction Ops. -tf_gen_op_libs( - op_lib_names = ["prediction_ops"], - deps = ["//tensorflow/contrib/boosted_trees/proto:learner_proto_cc"], -) - -tf_gen_op_wrapper_py( - name = "gen_prediction_ops_py", - out = "python/ops/gen_prediction_ops.py", - deps = [ - ":prediction_ops_op_lib", - ], -) - -tf_custom_op_py_library( - name = "prediction_ops_py", - srcs = ["python/ops/prediction_ops.py"], - kernels = [ - ":prediction_ops_kernels", - ":prediction_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":boosted_trees_ops_loader", - ":gen_prediction_ops_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:framework_for_generated_wrappers", - ], -) - -tf_kernel_library( - name = "prediction_ops_kernels", - srcs = ["kernels/prediction_ops.cc"], - deps = [ - "//tensorflow/contrib/boosted_trees/lib:example_partitioner", - "//tensorflow/contrib/boosted_trees/lib:models", - "//tensorflow/contrib/boosted_trees/lib:utils", - "//tensorflow/contrib/boosted_trees/proto:learner_proto_cc", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc", - "//tensorflow/contrib/boosted_trees/resources:decision_tree_ensemble_resource", - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - ], - alwayslink = 1, -) - -# Quantile ops -tf_gen_op_libs( - op_lib_names = ["quantile_ops"], -) - -tf_gen_op_wrapper_py( - name = "gen_quantile_ops_py_wrap", - out = "python/ops/gen_quantile_ops.py", - deps = [ - ":quantile_ops_op_lib", - ], -) - -tf_custom_op_py_library( - name = "quantile_ops_py", - srcs = ["python/ops/quantile_ops.py"], - kernels = [ - ":quantile_ops_kernels", - ":quantile_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":batch_ops_utils_py", - ":boosted_trees_ops_loader", - ":gen_quantile_ops_py_wrap", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:resources", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:training", - ], -) - -tf_kernel_library( - name = "quantile_ops_kernels", - srcs = ["kernels/quantile_ops.cc"], - deps = [ - "//tensorflow/contrib/boosted_trees/lib:utils", - "//tensorflow/contrib/boosted_trees/lib:weighted_quantiles", - "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_cc", - "//tensorflow/contrib/boosted_trees/resources:quantile_stream_resource", - "//tensorflow/core:framework_headers_lib", - ], - alwayslink = 1, -) - -# Stats Accumulator ops -tf_gen_op_libs( - op_lib_names = ["stats_accumulator_ops"], -) - -tf_gen_op_wrapper_py( - name = "gen_stats_accumulator_ops_py_wrap", - out = "python/ops/gen_stats_accumulator_ops.py", - deps = [ - ":stats_accumulator_ops_op_lib", - ], -) - -tf_custom_op_py_library( - name = "stats_accumulator_ops_py", - srcs = ["python/ops/stats_accumulator_ops.py"], - kernels = [ - ":stats_accumulator_ops_kernels", - ":stats_accumulator_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":batch_ops_utils_py", - ":boosted_trees_ops_loader", - ":gen_stats_accumulator_ops_py_wrap", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:resources", - "//tensorflow/python:training", - ], -) - -tf_kernel_library( - name = "stats_accumulator_ops_kernels", - srcs = ["kernels/stats_accumulator_ops.cc"], - deps = [ - "//tensorflow/contrib/boosted_trees/lib:utils", - "//tensorflow/contrib/boosted_trees/resources:stamped_resource", - "//tensorflow/core:framework_headers_lib", - ], - alwayslink = 1, -) - -# Pip - -py_library( - name = "boosted_trees_pip", - deps = [ - ":init_py", - "//tensorflow/contrib/boosted_trees:gbdt_batch", - "//tensorflow/contrib/boosted_trees/estimator_batch:custom_export_strategy", - "//tensorflow/contrib/boosted_trees/estimator_batch:dnn_tree_combined_estimator", - "//tensorflow/contrib/boosted_trees/estimator_batch:init_py", - "//tensorflow/contrib/boosted_trees/estimator_batch:trainer_hooks", - "//tensorflow/contrib/boosted_trees/lib:categorical_split_handler", - "//tensorflow/contrib/boosted_trees/lib:ordinal_split_handler", - "//tensorflow/contrib/boosted_trees/proto:learner_proto_py", - "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_py", - "//tensorflow/contrib/boosted_trees/proto:split_info_proto_py", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_py", - ], -) diff --git a/tensorflow/contrib/boosted_trees/README.md b/tensorflow/contrib/boosted_trees/README.md deleted file mode 100644 index 7d30032e539..00000000000 --- a/tensorflow/contrib/boosted_trees/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# TF Boosted Trees (TFBT) - -TF Boosted trees is an implementation of a gradient boosting algorithm with -trees used as weak learners. - -## Examples -Folder "examples" demonstrates how TFBT estimators can be used for various -problems. Namely, it contains: -* binary_mnist.py - an example on how to use TFBT for binary classification. -* mnist.py - a multiclass example. -* boston.py - a regression example. \ No newline at end of file diff --git a/tensorflow/contrib/boosted_trees/__init__.py b/tensorflow/contrib/boosted_trees/__init__.py deleted file mode 100644 index afb2d049b71..00000000000 --- a/tensorflow/contrib/boosted_trees/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Gradient boosted trees implementation in tensorflow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.boosted_trees.python import * -# pylint: enable=unused-import,wildcard-import diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD deleted file mode 100644 index c1323a4b57e..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD +++ /dev/null @@ -1,220 +0,0 @@ -# This directory contains estimators to train and run inference on -# gradient boosted trees on top of TensorFlow. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "init_py", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - deps = [ - ":custom_export_strategy", - ":custom_loss_head", - ":distillation_loss", - ":estimator", - ":model", - ":trainer_hooks", - ], -) - -py_library( - name = "model", - srcs = ["model.py"], - srcs_version = "PY2AND3", - deps = [ - ":estimator_utils", - ":trainer_hooks", - "//tensorflow/contrib/boosted_trees:gbdt_batch", - "//tensorflow/contrib/boosted_trees:model_ops_py", - "//tensorflow/python:framework_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:training_util", - ], -) - -py_library( - name = "trainer_hooks", - srcs = ["trainer_hooks.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/learn", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:platform", - "//tensorflow/python:training", - ], -) - -py_library( - name = "estimator_utils", - srcs = ["estimator_utils.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/learn", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_ops", - ], -) - -py_test( - name = "trainer_hooks_test", - size = "small", - srcs = ["trainer_hooks_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - # TODO(b/134605801): Re-enable this test. - "no_oss", - ], - deps = [ - ":trainer_hooks", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:constant_op", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:session", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], -) - -py_library( - name = "custom_loss_head", - srcs = ["custom_loss_head.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:math_ops", - ], -) - -py_library( - name = "custom_export_strategy", - srcs = ["custom_export_strategy.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/boosted_trees:gbdt_batch", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_py", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_py", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_py", - "//tensorflow/contrib/learn", - "//tensorflow/python:framework_ops", - "//tensorflow/python:platform", - "//tensorflow/python:session", - "//tensorflow/python/saved_model:loader", - "//tensorflow/python/saved_model:tag_constants", - ], -) - -py_test( - name = "custom_export_strategy_test", - size = "small", - srcs = ["custom_export_strategy_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":custom_export_strategy", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_py", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -py_library( - name = "estimator", - srcs = ["estimator.py"], - srcs_version = "PY2AND3", - deps = [ - ":custom_loss_head", - ":estimator_utils", - ":model", - "//tensorflow/contrib/boosted_trees:losses", - "//tensorflow/contrib/learn", - "//tensorflow/python:math_ops", - ], -) - -py_library( - name = "dnn_tree_combined_estimator", - srcs = ["dnn_tree_combined_estimator.py"], - srcs_version = "PY2AND3", - deps = [ - ":distillation_loss", - ":estimator_utils", - ":model", - ":trainer_hooks", - "//tensorflow/contrib/boosted_trees:gbdt_batch", - "//tensorflow/contrib/boosted_trees:model_ops_py", - "//tensorflow/contrib/learn", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - ], -) - -py_library( - name = "distillation_loss", - srcs = ["distillation_loss.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/learn", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - ], -) - -py_test( - name = "dnn_tree_combined_estimator_test", - size = "medium", - timeout = "long", - srcs = ["dnn_tree_combined_estimator_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_gpu", - "no_pip_gpu", - "notsan", - ], - deps = [ - ":dnn_tree_combined_estimator", - "//tensorflow/contrib/boosted_trees:gbdt_batch", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - ], -) - -py_test( - name = "estimator_test", - size = "medium", - srcs = ["estimator_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - tags = [ - "no_gpu", - "no_pip_gpu", - "notsan", - ], - deps = [ - ":estimator", - "//tensorflow/contrib/boosted_trees:gbdt_batch", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - ], -) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/__init__.py b/tensorflow/contrib/boosted_trees/estimator_batch/__init__.py deleted file mode 100644 index 02af4adbd99..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Gradient boosted trees implementation in tensorflow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.boosted_trees.estimator_batch import * -# pylint: enable=unused-import,wildcard-import diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py deleted file mode 100644 index d79b4e0ab39..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py +++ /dev/null @@ -1,267 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Strategy to export custom proto formats.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import os - -from tensorflow.contrib.boosted_trees.proto import tree_config_pb2 -from tensorflow.contrib.boosted_trees.python.training.functions import gbdt_batch -from tensorflow.contrib.decision_trees.proto import generic_tree_model_extensions_pb2 -from tensorflow.contrib.decision_trees.proto import generic_tree_model_pb2 -from tensorflow.contrib.learn.python.learn import export_strategy -from tensorflow.contrib.learn.python.learn.utils import saved_model_export_utils -from tensorflow.python.client import session as tf_session -from tensorflow.python.framework import ops -from tensorflow.python.platform import gfile -from tensorflow.python.saved_model import loader as saved_model_loader -from tensorflow.python.saved_model import tag_constants -from tensorflow.python.util import compat - -_SPARSE_FLOAT_FEATURE_NAME_TEMPLATE = "%s_%d" - - -def make_custom_export_strategy(name, - convert_fn, - feature_columns, - export_input_fn, - use_core_columns=False, - feature_engineering_fn=None, - default_output_alternative_key=None): - """Makes custom exporter of GTFlow tree format. - - Args: - name: A string, for the name of the export strategy. - convert_fn: A function that converts the tree proto to desired format and - saves it to the desired location. Can be None to skip conversion. - feature_columns: A list of feature columns. - export_input_fn: A function that takes no arguments and returns an - `InputFnOps`. - use_core_columns: A boolean, whether core feature columns were used. - feature_engineering_fn: Feature eng function to be called on the input. - default_output_alternative_key: the name of the head to serve when an - incoming serving request does not explicitly request a specific head. - Not needed for single-headed models. - - Returns: - An `ExportStrategy`. - """ - base_strategy = saved_model_export_utils.make_export_strategy( - serving_input_fn=export_input_fn, - strip_default_attrs=True, - default_output_alternative_key=default_output_alternative_key) - input_fn = export_input_fn() - features = input_fn.features - if feature_engineering_fn is not None: - features, _ = feature_engineering_fn(features, labels=None) - (sorted_feature_names, dense_floats, sparse_float_indices, _, _, - sparse_int_indices, _, _) = gbdt_batch.extract_features( - features, feature_columns, use_core_columns) - - def export_fn(estimator, export_dir, checkpoint_path=None, eval_result=None): - """A wrapper to export to SavedModel, and convert it to other formats.""" - result_dir = base_strategy.export(estimator, export_dir, - checkpoint_path, - eval_result) - with ops.Graph().as_default() as graph: - with tf_session.Session(graph=graph) as sess: - saved_model_loader.load( - sess, [tag_constants.SERVING], result_dir) - # Note: This is GTFlow internal API and might change. - ensemble_model = graph.get_operation_by_name( - "ensemble_model/TreeEnsembleSerialize") - _, dfec_str = sess.run(ensemble_model.outputs) - dtec = tree_config_pb2.DecisionTreeEnsembleConfig() - dtec.ParseFromString(dfec_str) - # Export the result in the same folder as the saved model. - if convert_fn: - convert_fn(dtec, sorted_feature_names, - len(dense_floats), - len(sparse_float_indices), - len(sparse_int_indices), result_dir, eval_result) - feature_importances = _get_feature_importances( - dtec, sorted_feature_names, - len(dense_floats), - len(sparse_float_indices), len(sparse_int_indices)) - sorted_by_importance = sorted( - feature_importances.items(), key=lambda x: -x[1]) - assets_dir = os.path.join( - compat.as_bytes(result_dir), compat.as_bytes("assets.extra")) - gfile.MakeDirs(assets_dir) - with gfile.GFile(os.path.join( - compat.as_bytes(assets_dir), - compat.as_bytes("feature_importances")), "w") as f: - f.write("\n".join("%s, %f" % (k, v) for k, v in sorted_by_importance)) - return result_dir - - return export_strategy.ExportStrategy( - name, export_fn, strip_default_attrs=True) - - -def convert_to_universal_format(dtec, sorted_feature_names, - num_dense, num_sparse_float, - num_sparse_int, - feature_name_to_proto=None): - """Convert GTFlow trees to universal format.""" - del num_sparse_int # unused. - model_and_features = generic_tree_model_pb2.ModelAndFeatures() - # TODO(jonasz): Feature descriptions should contain information about how each - # feature is processed before it's fed to the model (e.g. bucketing - # information). As of now, this serves as a list of features the model uses. - for feature_name in sorted_feature_names: - if not feature_name_to_proto: - model_and_features.features[feature_name].SetInParent() - else: - model_and_features.features[feature_name].CopyFrom( - feature_name_to_proto[feature_name]) - model = model_and_features.model - model.ensemble.summation_combination_technique.SetInParent() - for tree_idx in range(len(dtec.trees)): - gtflow_tree = dtec.trees[tree_idx] - tree_weight = dtec.tree_weights[tree_idx] - member = model.ensemble.members.add() - member.submodel_id.value = tree_idx - tree = member.submodel.decision_tree - for node_idx in range(len(gtflow_tree.nodes)): - gtflow_node = gtflow_tree.nodes[node_idx] - node = tree.nodes.add() - node_type = gtflow_node.WhichOneof("node") - node.node_id.value = node_idx - if node_type == "leaf": - leaf = gtflow_node.leaf - if leaf.HasField("vector"): - for weight in leaf.vector.value: - new_value = node.leaf.vector.value.add() - new_value.float_value = weight * tree_weight - else: - for index, weight in zip( - leaf.sparse_vector.index, leaf.sparse_vector.value): - new_value = node.leaf.sparse_vector.sparse_value[index] - new_value.float_value = weight * tree_weight - else: - node = node.binary_node - # Binary nodes here. - if node_type == "dense_float_binary_split": - split = gtflow_node.dense_float_binary_split - feature_id = split.feature_column - inequality_test = node.inequality_left_child_test - inequality_test.feature_id.id.value = sorted_feature_names[feature_id] - inequality_test.type = ( - generic_tree_model_pb2.InequalityTest.LESS_OR_EQUAL) - inequality_test.threshold.float_value = split.threshold - elif node_type == "sparse_float_binary_split_default_left": - split = gtflow_node.sparse_float_binary_split_default_left.split - node.default_direction = (generic_tree_model_pb2.BinaryNode.LEFT) - feature_id = split.feature_column + num_dense - inequality_test = node.inequality_left_child_test - inequality_test.feature_id.id.value = ( - _SPARSE_FLOAT_FEATURE_NAME_TEMPLATE % - (sorted_feature_names[feature_id], split.dimension_id)) - model_and_features.features.pop(sorted_feature_names[feature_id]) - (model_and_features.features[inequality_test.feature_id.id.value] - .SetInParent()) - inequality_test.type = ( - generic_tree_model_pb2.InequalityTest.LESS_OR_EQUAL) - inequality_test.threshold.float_value = split.threshold - elif node_type == "sparse_float_binary_split_default_right": - split = gtflow_node.sparse_float_binary_split_default_right.split - node.default_direction = ( - generic_tree_model_pb2.BinaryNode.RIGHT) - # TODO(nponomareva): adjust this id assignement when we allow multi- - # column sparse tensors. - feature_id = split.feature_column + num_dense - inequality_test = node.inequality_left_child_test - inequality_test.feature_id.id.value = ( - _SPARSE_FLOAT_FEATURE_NAME_TEMPLATE % - (sorted_feature_names[feature_id], split.dimension_id)) - model_and_features.features.pop(sorted_feature_names[feature_id]) - (model_and_features.features[inequality_test.feature_id.id.value] - .SetInParent()) - inequality_test.type = ( - generic_tree_model_pb2.InequalityTest.LESS_OR_EQUAL) - inequality_test.threshold.float_value = split.threshold - elif node_type == "categorical_id_binary_split": - split = gtflow_node.categorical_id_binary_split - node.default_direction = generic_tree_model_pb2.BinaryNode.RIGHT - feature_id = split.feature_column + num_dense + num_sparse_float - categorical_test = ( - generic_tree_model_extensions_pb2.MatchingValuesTest()) - categorical_test.feature_id.id.value = sorted_feature_names[ - feature_id] - matching_id = categorical_test.value.add() - matching_id.int64_value = split.feature_id - node.custom_left_child_test.Pack(categorical_test) - elif (node_type == "oblivious_dense_float_binary_split" or - node_type == "oblivious_categorical_id_binary_split"): - raise ValueError("Universal tree format doesn't support oblivious " - "trees") - else: - raise ValueError("Unexpected node type %s" % node_type) - node.left_child_id.value = split.left_id - node.right_child_id.value = split.right_id - return model_and_features - - -def _get_feature_importances(dtec, feature_names, num_dense_floats, - num_sparse_float, num_sparse_int): - """Export the feature importance per feature column.""" - del num_sparse_int # Unused. - sums = collections.defaultdict(lambda: 0) - for tree_idx in range(len(dtec.trees)): - tree = dtec.trees[tree_idx] - for tree_node in tree.nodes: - node_type = tree_node.WhichOneof("node") - if node_type == "dense_float_binary_split": - split = tree_node.dense_float_binary_split - split_column = feature_names[split.feature_column] - elif node_type == "sparse_float_binary_split_default_left": - split = tree_node.sparse_float_binary_split_default_left.split - split_column = _SPARSE_FLOAT_FEATURE_NAME_TEMPLATE % ( - feature_names[split.feature_column + num_dense_floats], - split.dimension_id) - elif node_type == "sparse_float_binary_split_default_right": - split = tree_node.sparse_float_binary_split_default_right.split - split_column = _SPARSE_FLOAT_FEATURE_NAME_TEMPLATE % ( - feature_names[split.feature_column + num_dense_floats], - split.dimension_id) - elif node_type == "categorical_id_binary_split": - split = tree_node.categorical_id_binary_split - split_column = feature_names[split.feature_column + num_dense_floats + - num_sparse_float] - elif node_type == "oblivious_dense_float_binary_split": - split = tree_node.oblivious_dense_float_binary_split - split_column = feature_names[split.feature_column] - elif node_type == "oblivious_categorical_id_binary_split": - split = tree_node.oblivious_categorical_id_binary_split - split_column = feature_names[split.feature_column + num_dense_floats + - num_sparse_float] - elif node_type == "categorical_id_set_membership_binary_split": - split = tree_node.categorical_id_set_membership_binary_split - split_column = feature_names[split.feature_column + num_dense_floats + - num_sparse_float] - elif node_type == "leaf": - assert tree_node.node_metadata.gain == 0 - continue - else: - raise ValueError("Unexpected split type %s" % node_type) - # Apply shrinkage factor. It is important since it is not always uniform - # across different trees. - sums[split_column] += ( - tree_node.node_metadata.gain * dtec.tree_weights[tree_idx]) - return dict(sums) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy_test.py deleted file mode 100644 index 67ec0e16bf8..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy_test.py +++ /dev/null @@ -1,364 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the conversion code and for feature importances export. - -Tests that cover conversion from TFBT format to a tensorflow.contrib. -decision_tree generic_tree_model format and feature importances export. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from google.protobuf import text_format -from tensorflow.contrib.boosted_trees.estimator_batch import custom_export_strategy -from tensorflow.contrib.boosted_trees.proto import tree_config_pb2 -from tensorflow.python.framework import test_util -from tensorflow.python.platform import googletest - - -class ConvertModelTest(test_util.TensorFlowTestCase): - - def _make_trees(self): - dtec_str = """ - trees { - nodes { - leaf { - vector { - value: -1 - } - } - } - } - trees { - nodes { - dense_float_binary_split { - feature_column: 0 - threshold: 1740.0 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 500 - } - } - nodes { - leaf { - vector { - value: 0.6 - } - } - } - nodes { - sparse_float_binary_split_default_left { - split { - feature_column: 0 - threshold: 1500.0 - left_id: 3 - right_id: 4 - } - } - node_metadata { - gain: 500 - } - } - nodes { - categorical_id_binary_split { - feature_column: 0 - feature_id: 5 - left_id: 5 - right_id: 6 - } - node_metadata { - gain: 500 - } - } - nodes { - leaf { - vector { - value: 0.8 - } - } - } - nodes { - leaf { - vector { - value: 0.5 - } - } - } - nodes { - sparse_float_binary_split_default_right { - split { - feature_column: 1 - dimension_id:3 - threshold: -0.4 - left_id: 7 - right_id: 8 - } - } - node_metadata { - gain: 3600 - } - } - nodes { - leaf { - vector { - value: 0.36 - } - } - } - nodes { - leaf { - vector { - value: 18 - } - } - } - } - tree_weights: 1.0 - tree_weights: 0.1 - """ - dtec = tree_config_pb2.DecisionTreeEnsembleConfig() - text_format.Merge(dtec_str, dtec) - feature_columns = [ - "feature_b", - "feature_a", - "feature_a_m", - "feature_d", - ] - return dtec, feature_columns - - def testConvertModel(self): - dtec, feature_columns = self._make_trees() - # Assume 2 sparse float columns, one with 1 dimension, the second one with - # 5 dimensions. - # The feature columns in the order they were added. - out = custom_export_strategy.convert_to_universal_format( - dtec, feature_columns, 1, 2, 1) - # Features a and a_m are sparse float features, a_m is multidimensional. - expected_tree = """ - features { key: "feature_a_0" } - features { key: "feature_a_m_3" } - features { key: "feature_b" } - features { key: "feature_d" } - model { - ensemble { - summation_combination_technique { - } - members { - submodel { - decision_tree { - nodes { - node_id { - } - leaf { - vector { - value { - float_value: -1.0 - } - } - } - } - } - } - submodel_id { - } - } - members { - submodel { - decision_tree { - nodes { - node_id { - } - binary_node { - left_child_id { - value: 1 - } - right_child_id { - value: 2 - } - inequality_left_child_test { - feature_id { - id { - value: "feature_b" - } - } - threshold { - float_value: 1740.0 - } - } - } - } - nodes { - node_id { - value: 1 - } - leaf { - vector { - value { - float_value: 0.06 - } - } - } - } - nodes { - node_id { - value: 2 - } - binary_node { - left_child_id { - value: 3 - } - right_child_id { - value: 4 - } - inequality_left_child_test { - feature_id { - id { - value: "feature_a_0" - } - } - threshold { - float_value: 1500.0 - } - } - } - } - nodes { - node_id { - value: 3 - } - binary_node { - left_child_id { - value: 5 - } - right_child_id { - value: 6 - } - default_direction: RIGHT - custom_left_child_test { - [type.googleapis.com/tensorflow.decision_trees.MatchingValuesTest] { - feature_id { - id { - value: "feature_d" - } - } - value { - int64_value: 5 - } - } - } - } - } - nodes { - node_id { - value: 4 - } - leaf { - vector { - value { - float_value: 0.08 - } - } - } - } - nodes { - node_id { - value: 5 - } - leaf { - vector { - value { - float_value: 0.05 - } - } - } - } - nodes { - node_id { - value: 6 - } - binary_node { - left_child_id { - value: 7 - } - right_child_id { - value: 8 - } - default_direction: RIGHT - inequality_left_child_test { - feature_id { - id { - value: "feature_a_m_3" - } - } - threshold { - float_value: -0.4 - } - } - } - } - nodes { - node_id { - value: 7 - } - leaf { - vector { - value { - float_value: 0.036 - } - } - } - } - nodes { - node_id { - value: 8 - } - leaf { - vector { - value { - float_value: 1.8 - } - } - } - } - - } - } - submodel_id { - value: 1 - } - } - } - }""" - self.assertProtoEquals(expected_tree, out) - - def testFeatureImportance(self): - dtec, feature_columns = self._make_trees() - feature_importances = custom_export_strategy._get_feature_importances( - dtec, feature_columns, 1, 2, 1) - self.assertItemsEqual( - ["feature_b", "feature_a_0", "feature_a_m_3", "feature_d"], - feature_importances.keys()) - self.assertAlmostEqual(50.0, feature_importances["feature_b"], places=4) - self.assertAlmostEqual(50.0, feature_importances["feature_a_0"], places=4) - self.assertAlmostEqual(50.0, feature_importances["feature_d"], places=4) - self.assertAlmostEqual( - 360.0, feature_importances["feature_a_m_3"], places=4) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_loss_head.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_loss_head.py deleted file mode 100644 index 1f5c6040e57..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_loss_head.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implementation of `head.Head` with custom loss and link function.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops - - -class CustomLossHead(head_lib._RegressionHead): # pylint: disable=protected-access - """A Head object with custom loss function and link function.""" - - def __init__(self, - loss_fn, - link_fn, - logit_dimension, - head_name=None, - weight_column_name=None, - metrics_fn=None): - """`Head` for specifying arbitrary loss function. - - Args: - loss_fn: Loss function. - link_fn: Function that converts logits to prediction. - logit_dimension: Number of dimensions for the logits. - head_name: name of the head. Predictions, summary, metrics keys are - suffixed by `"/" + head_name` and the default variable scope is - `head_name`. - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - metrics_fn: a function that takes predictions dict, labels and weights and - returns a dictionary of metrics to be calculated. - """ - - def loss_wrapper(labels, logits, weight_tensor): - if weight_tensor is None: - weight_tensor = array_ops.ones( - shape=[array_ops.shape(labels)[0], 1], dtype=dtypes.float32) - weighted_loss, _ = loss_fn(labels, weight_tensor, logits) - average_loss = math_ops.reduce_mean(weighted_loss) - return average_loss, average_loss / math_ops.reduce_mean(weight_tensor) - - super(CustomLossHead, self).__init__( - loss_fn=loss_wrapper, - link_fn=link_fn, - head_name=head_name, - weight_column_name=weight_column_name, - enable_centered_bias=False, - label_dimension=logit_dimension) - - self._metrics_fn = metrics_fn - - def _metrics(self, eval_loss, predictions, labels, weights): - if self._metrics_fn is not None: - return self._metrics_fn(predictions, labels, weights) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/distillation_loss.py b/tensorflow/contrib/boosted_trees/estimator_batch/distillation_loss.py deleted file mode 100644 index 9aacc553432..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/distillation_loss.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utill functions for distillation loss. - -The distillation loss_fn will be called with the following: - -Args: - dnn_logits: Tensor of logits from the dnn, treated as the "target". This will - be the output of a call to tf.stop_gradient(). - tree_logits: Tensor of logits from the tree, treated as the "predictions". - example_weights: Tensor of example weights, or a single scalar. - -Returns: - A scalar indicating the reduced loss for that batch of examples. - -Note: we calls the loss_fn defined in contrib head, which is computing two -losses, first one for training and second one for reporting. We only take the -first one here. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn - - -def _logits_to_label_for_tree(logits, n_classes): - if n_classes == 2: - return math_ops.sigmoid(logits) - else: - return nn.softmax(logits) - - -def create_dnn_to_tree_squared_loss_fn(n_classes): - """Returns a squared loss function for dnn to tree distillation.""" - - def _dnn_to_tree_squared_loss(dnn_logits, tree_logits, example_weights): - return head_lib._mean_squared_loss( # pylint: disable=protected-access - labels=_logits_to_label_for_tree(dnn_logits, n_classes), - logits=_logits_to_label_for_tree(tree_logits, n_classes), - weights=example_weights)[0] - - return _dnn_to_tree_squared_loss - - -def create_dnn_to_tree_cross_entropy_loss_fn(n_classes): - """Returns a cross entropy loss function for dnn to tree distillation.""" - - def _dnn_to_tree_cross_entropy_loss(dnn_logits, tree_logits, example_weights): - if n_classes == 2: - return head_lib._log_loss_with_two_classes( # pylint: disable=protected-access - labels=_logits_to_label_for_tree(dnn_logits, n_classes), - logits=tree_logits, - weights=example_weights)[0] - else: - return head_lib._softmax_cross_entropy_loss( # pylint: disable=protected-access - labels=_logits_to_label_for_tree(dnn_logits, n_classes), - logits=tree_logits, - weights=example_weights)[0] - - return _dnn_to_tree_cross_entropy_loss diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py deleted file mode 100644 index 358404cd946..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py +++ /dev/null @@ -1,859 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""TensorFlow estimators for combined DNN + GBDT training model. - -The combined model trains a DNN first, then trains boosted trees to boost the -logits of the DNN. The input layer of the DNN (including the embeddings learned -over sparse features) can optionally be provided to the boosted trees as -an additional input feature. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import six - -from tensorflow.contrib import layers -from tensorflow.contrib.boosted_trees.estimator_batch import model -from tensorflow.contrib.boosted_trees.estimator_batch import distillation_loss -from tensorflow.contrib.boosted_trees.estimator_batch import trainer_hooks -from tensorflow.contrib.boosted_trees.python.ops import model_ops -from tensorflow.contrib.boosted_trees.python.training.functions import gbdt_batch -from tensorflow.contrib.layers.python.layers import optimizers -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.python.estimator import estimator as core_estimator -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.python.feature_column import feature_column_lib -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.summary import summary -from tensorflow.python.training import training_util - -_DNN_LEARNING_RATE = 0.001 - - -def _get_optimizer(optimizer): - if callable(optimizer): - return optimizer() - else: - return optimizer - - -def _add_hidden_layer_summary(value, tag): - summary.scalar("%s_fraction_of_zero_values" % tag, nn.zero_fraction(value)) - summary.histogram("%s_activation" % tag, value) - - -def _dnn_tree_combined_model_fn( - features, - labels, - mode, - head, - dnn_hidden_units, - dnn_feature_columns, - tree_learner_config, - num_trees, - tree_examples_per_layer, - config=None, - dnn_optimizer="Adagrad", - dnn_activation_fn=nn.relu, - dnn_dropout=None, - dnn_input_layer_partitioner=None, - dnn_input_layer_to_tree=True, - dnn_steps_to_train=10000, - predict_with_tree_only=False, - tree_feature_columns=None, - tree_center_bias=False, - dnn_to_tree_distillation_param=None, - use_core_versions=False, - output_type=model.ModelBuilderOutputType.MODEL_FN_OPS, - override_global_step_value=None): - """DNN and GBDT combined model_fn. - - Args: - features: `dict` of `Tensor` objects. - labels: Labels used to train on. - mode: Mode we are in. (TRAIN/EVAL/INFER) - head: A `Head` instance. - dnn_hidden_units: List of hidden units per layer. - dnn_feature_columns: An iterable containing all the feature columns - used by the model's DNN. - tree_learner_config: A config for the tree learner. - num_trees: Number of trees to grow model to after training DNN. - tree_examples_per_layer: Number of examples to accumulate before - growing the tree a layer. This value has a big impact on model - quality and should be set equal to the number of examples in - training dataset if possible. It can also be a function that computes - the number of examples based on the depth of the layer that's - being built. - config: `RunConfig` of the estimator. - dnn_optimizer: string, `Optimizer` object, or callable that defines the - optimizer to use for training the DNN. If `None`, will use the Adagrad - optimizer with default learning rate of 0.001. - dnn_activation_fn: Activation function applied to each layer of the DNN. - If `None`, will use `tf.nn.relu`. - dnn_dropout: When not `None`, the probability to drop out a given - unit in the DNN. - dnn_input_layer_partitioner: Partitioner for input layer of the DNN. - Defaults to `min_max_variable_partitioner` with `min_slice_size` 64 << 20. - dnn_input_layer_to_tree: Whether to provide the DNN's input layer - as a feature to the tree. - dnn_steps_to_train: Number of steps to train dnn for before switching - to gbdt. - predict_with_tree_only: Whether to use only the tree model output as the - final prediction. - tree_feature_columns: An iterable containing all the feature columns - used by the model's boosted trees. If dnn_input_layer_to_tree is - set to True, these features are in addition to dnn_feature_columns. - tree_center_bias: Whether a separate tree should be created for - first fitting the bias. - dnn_to_tree_distillation_param: A Tuple of (float, loss_fn), where the - float defines the weight of the distillation loss, and the loss_fn, for - computing distillation loss, takes dnn_logits, tree_logits and weight - tensor. If the entire tuple is None, no distillation will be applied. If - only the loss_fn is None, we will take the sigmoid/softmax cross entropy - loss be default. When distillation is applied, `predict_with_tree_only` - will be set to True. - use_core_versions: Whether feature columns and loss are from the core (as - opposed to contrib) version of tensorflow. - output_type: Whether to return ModelFnOps (old interface) or EstimatorSpec - (new interface). - override_global_step_value: If after the training is done, global step - value must be reset to this value. This is particularly useful for hyper - parameter tuning, which can't recognize early stopping due to the number - of trees. If None, no override of global step will happen. - - Returns: - A `ModelFnOps` object. - Raises: - ValueError: if inputs are not valid. - """ - if not isinstance(features, dict): - raise ValueError("features should be a dictionary of `Tensor`s. " - "Given type: {}".format(type(features))) - - if not dnn_feature_columns: - raise ValueError("dnn_feature_columns must be specified") - - if dnn_to_tree_distillation_param: - if not predict_with_tree_only: - logging.warning("update predict_with_tree_only to True since distillation" - "is specified.") - predict_with_tree_only = True - - # Build DNN Logits. - dnn_parent_scope = "dnn" - dnn_partitioner = dnn_input_layer_partitioner or ( - partitioned_variables.min_max_variable_partitioner( - max_partitions=config.num_ps_replicas, min_slice_size=64 << 20)) - - if (output_type == model.ModelBuilderOutputType.ESTIMATOR_SPEC and - not use_core_versions): - raise ValueError("You must use core versions with Estimator Spec") - global_step = training_util.get_global_step() - - with variable_scope.variable_scope( - dnn_parent_scope, - values=tuple(six.itervalues(features)), - partitioner=dnn_partitioner): - - with variable_scope.variable_scope( - "input_from_feature_columns", - values=tuple(six.itervalues(features)), - partitioner=dnn_partitioner) as input_layer_scope: - if use_core_versions: - input_layer = feature_column_lib.input_layer( - features=features, - feature_columns=dnn_feature_columns, - weight_collections=[dnn_parent_scope]) - else: - input_layer = layers.input_from_feature_columns( - columns_to_tensors=features, - feature_columns=dnn_feature_columns, - weight_collections=[dnn_parent_scope], - scope=input_layer_scope) - def dnn_logits_fn(): - """Builds the logits from the input layer.""" - previous_layer = input_layer - for layer_id, num_hidden_units in enumerate(dnn_hidden_units): - with variable_scope.variable_scope( - "hiddenlayer_%d" % layer_id, - values=(previous_layer,)) as hidden_layer_scope: - net = layers.fully_connected( - previous_layer, - num_hidden_units, - activation_fn=dnn_activation_fn, - variables_collections=[dnn_parent_scope], - scope=hidden_layer_scope) - if dnn_dropout is not None and mode == model_fn.ModeKeys.TRAIN: - net = layers.dropout(net, keep_prob=(1.0 - dnn_dropout)) - _add_hidden_layer_summary(net, hidden_layer_scope.name) - previous_layer = net - with variable_scope.variable_scope( - "logits", values=(previous_layer,)) as logits_scope: - dnn_logits = layers.fully_connected( - previous_layer, - head.logits_dimension, - activation_fn=None, - variables_collections=[dnn_parent_scope], - scope=logits_scope) - _add_hidden_layer_summary(dnn_logits, logits_scope.name) - return dnn_logits - if predict_with_tree_only and mode == model_fn.ModeKeys.INFER: - dnn_logits = array_ops.constant(0.0) - dnn_train_op_fn = control_flow_ops.no_op - elif predict_with_tree_only and mode == model_fn.ModeKeys.EVAL: - dnn_logits = control_flow_ops.cond( - global_step > dnn_steps_to_train, - lambda: array_ops.constant(0.0), - dnn_logits_fn) - dnn_train_op_fn = control_flow_ops.no_op - else: - dnn_logits = dnn_logits_fn() - def dnn_train_op_fn(loss): - """Returns the op to optimize the loss.""" - return optimizers.optimize_loss( - loss=loss, - global_step=training_util.get_global_step(), - learning_rate=_DNN_LEARNING_RATE, - optimizer=_get_optimizer(dnn_optimizer), - name=dnn_parent_scope, - variables=ops.get_collection( - ops.GraphKeys.TRAINABLE_VARIABLES, scope=dnn_parent_scope), - # Empty summaries to prevent optimizers from logging training_loss. - summaries=[]) - - # Build Tree Logits. - with ops.device(global_step.device): - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config="", # Initialize an empty ensemble. - name="ensemble_model") - - tree_features = features.copy() - if dnn_input_layer_to_tree: - tree_features["dnn_input_layer"] = input_layer - tree_feature_columns.append(layers.real_valued_column("dnn_input_layer")) - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=config.is_chief, - num_ps_replicas=config.num_ps_replicas, - ensemble_handle=ensemble_handle, - center_bias=tree_center_bias, - examples_per_layer=tree_examples_per_layer, - learner_config=tree_learner_config, - feature_columns=tree_feature_columns, - logits_dimension=head.logits_dimension, - features=tree_features, - use_core_columns=use_core_versions) - - with ops.name_scope("gbdt"): - predictions_dict = gbdt_model.predict(mode) - tree_logits = predictions_dict["predictions"] - - def _tree_train_op_fn(loss): - """Returns the op to optimize the loss.""" - if dnn_to_tree_distillation_param: - loss_weight, loss_fn = dnn_to_tree_distillation_param - # pylint: disable=protected-access - if use_core_versions: - weight_tensor = head_lib._weight_tensor(features, head._weight_column) - else: - weight_tensor = head_lib._weight_tensor( - features, head.weight_column_name) - # pylint: enable=protected-access - dnn_logits_fixed = array_ops.stop_gradient(dnn_logits) - - if loss_fn is None: - # we create the loss_fn similar to the head loss_fn for - # multi_class_head used previously as the default one. - n_classes = 2 if head.logits_dimension == 1 else head.logits_dimension - loss_fn = distillation_loss.create_dnn_to_tree_cross_entropy_loss_fn( - n_classes) - - dnn_to_tree_distillation_loss = loss_weight * loss_fn( - dnn_logits_fixed, tree_logits, weight_tensor) - summary.scalar("dnn_to_tree_distillation_loss", - dnn_to_tree_distillation_loss) - loss += dnn_to_tree_distillation_loss - - update_op = gbdt_model.train(loss, predictions_dict, labels) - with ops.control_dependencies( - [update_op]), (ops.colocate_with(global_step)): - update_op = state_ops.assign_add(global_step, 1).op - return update_op - - if predict_with_tree_only: - if mode == model_fn.ModeKeys.TRAIN or mode == model_fn.ModeKeys.INFER: - tree_train_logits = tree_logits - else: - tree_train_logits = control_flow_ops.cond( - global_step > dnn_steps_to_train, - lambda: tree_logits, - lambda: dnn_logits) - else: - tree_train_logits = dnn_logits + tree_logits - - def _no_train_op_fn(loss): - """Returns a no-op.""" - del loss - return control_flow_ops.no_op() - - if tree_center_bias: - num_trees += 1 - finalized_trees, attempted_trees = gbdt_model.get_number_of_trees_tensor() - - if output_type == model.ModelBuilderOutputType.MODEL_FN_OPS: - model_fn_ops = head.create_model_fn_ops( - features=features, - mode=mode, - labels=labels, - train_op_fn=_no_train_op_fn, - logits=tree_train_logits) - if mode != model_fn.ModeKeys.TRAIN: - return model_fn_ops - dnn_train_op = head.create_model_fn_ops( - features=features, - mode=mode, - labels=labels, - train_op_fn=dnn_train_op_fn, - logits=dnn_logits).train_op - tree_train_op = head.create_model_fn_ops( - features=tree_features, - mode=mode, - labels=labels, - train_op_fn=_tree_train_op_fn, - logits=tree_train_logits).train_op - - # Add the hooks - model_fn_ops.training_hooks.extend([ - trainer_hooks.SwitchTrainOp(dnn_train_op, dnn_steps_to_train, - tree_train_op), - trainer_hooks.StopAfterNTrees(num_trees, attempted_trees, - finalized_trees, - override_global_step_value) - ]) - return model_fn_ops - - elif output_type == model.ModelBuilderOutputType.ESTIMATOR_SPEC: - fusion_spec = head.create_estimator_spec( - features=features, - mode=mode, - labels=labels, - train_op_fn=_no_train_op_fn, - logits=tree_train_logits) - if mode != model_fn.ModeKeys.TRAIN: - return fusion_spec - dnn_spec = head.create_estimator_spec( - features=features, - mode=mode, - labels=labels, - train_op_fn=dnn_train_op_fn, - logits=dnn_logits) - tree_spec = head.create_estimator_spec( - features=tree_features, - mode=mode, - labels=labels, - train_op_fn=_tree_train_op_fn, - logits=tree_train_logits) - - training_hooks = [ - trainer_hooks.SwitchTrainOp(dnn_spec.train_op, dnn_steps_to_train, - tree_spec.train_op), - trainer_hooks.StopAfterNTrees(num_trees, attempted_trees, - finalized_trees, - override_global_step_value) - ] - fusion_spec = fusion_spec._replace(training_hooks=training_hooks + - list(fusion_spec.training_hooks)) - return fusion_spec - - -class DNNBoostedTreeCombinedClassifier(estimator.Estimator): - """A classifier that uses a combined DNN/GBDT model.""" - - def __init__(self, - dnn_hidden_units, - dnn_feature_columns, - tree_learner_config, - num_trees, - tree_examples_per_layer, - n_classes=2, - weight_column_name=None, - model_dir=None, - config=None, - label_name=None, - label_keys=None, - feature_engineering_fn=None, - dnn_optimizer="Adagrad", - dnn_activation_fn=nn.relu, - dnn_dropout=None, - dnn_input_layer_partitioner=None, - dnn_input_layer_to_tree=True, - dnn_steps_to_train=10000, - predict_with_tree_only=False, - tree_feature_columns=None, - tree_center_bias=False, - dnn_to_tree_distillation_param=None, - use_core_versions=False, - override_global_step_value=None): - """Initializes a DNNBoostedTreeCombinedClassifier instance. - - Args: - dnn_hidden_units: List of hidden units per layer for DNN. - dnn_feature_columns: An iterable containing all the feature columns - used by the model's DNN. - tree_learner_config: A config for the tree learner. - num_trees: Number of trees to grow model to after training DNN. - tree_examples_per_layer: Number of examples to accumulate before - growing the tree a layer. This value has a big impact on model - quality and should be set equal to the number of examples in - training dataset if possible. It can also be a function that computes - the number of examples based on the depth of the layer that's - being built. - n_classes: The number of label classes. - weight_column_name: The name of weight column. - model_dir: Directory for model exports. - config: `RunConfig` of the estimator. - label_name: String, name of the key in label dict. Can be null if label - is a tensor (single headed models). - label_keys: Optional list of strings with size `[n_classes]` defining the - label vocabulary. Only supported for `n_classes` > 2. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - dnn_optimizer: string, `Optimizer` object, or callable that defines the - optimizer to use for training the DNN. If `None`, will use the Adagrad - optimizer with default learning rate. - dnn_activation_fn: Activation function applied to each layer of the DNN. - If `None`, will use `tf.nn.relu`. - dnn_dropout: When not `None`, the probability to drop out a given - unit in the DNN. - dnn_input_layer_partitioner: Partitioner for input layer of the DNN. - Defaults to `min_max_variable_partitioner` with `min_slice_size` - 64 << 20. - dnn_input_layer_to_tree: Whether to provide the DNN's input layer - as a feature to the tree. - dnn_steps_to_train: Number of steps to train dnn for before switching - to gbdt. - predict_with_tree_only: Whether to use only the tree model output as the - final prediction. - tree_feature_columns: An iterable containing all the feature columns - used by the model's boosted trees. If dnn_input_layer_to_tree is - set to True, these features are in addition to dnn_feature_columns. - tree_center_bias: Whether a separate tree should be created for - first fitting the bias. - dnn_to_tree_distillation_param: A Tuple of (float, loss_fn), where the - float defines the weight of the distillation loss, and the loss_fn, for - computing distillation loss, takes dnn_logits, tree_logits and weight - tensor. If the entire tuple is None, no distillation will be applied. If - only the loss_fn is None, we will take the sigmoid/softmax cross entropy - loss be default. When distillation is applied, `predict_with_tree_only` - will be set to True. - use_core_versions: Whether feature columns and loss are from the core (as - opposed to contrib) version of tensorflow. - override_global_step_value: If after the training is done, global step - value must be reset to this value. This is particularly useful for hyper - parameter tuning, which can't recognize early stopping due to the number - of trees. If None, no override of global step will happen. - """ - head = head_lib.multi_class_head( - n_classes=n_classes, - label_name=label_name, - label_keys=label_keys, - weight_column_name=weight_column_name, - enable_centered_bias=False) - - def _model_fn(features, labels, mode, config): - return _dnn_tree_combined_model_fn( - features=features, - labels=labels, - mode=mode, - head=head, - dnn_hidden_units=dnn_hidden_units, - dnn_feature_columns=dnn_feature_columns, - tree_learner_config=tree_learner_config, - num_trees=num_trees, - tree_examples_per_layer=tree_examples_per_layer, - config=config, - dnn_optimizer=dnn_optimizer, - dnn_activation_fn=dnn_activation_fn, - dnn_dropout=dnn_dropout, - dnn_input_layer_partitioner=dnn_input_layer_partitioner, - dnn_input_layer_to_tree=dnn_input_layer_to_tree, - dnn_steps_to_train=dnn_steps_to_train, - predict_with_tree_only=predict_with_tree_only, - tree_feature_columns=tree_feature_columns, - tree_center_bias=tree_center_bias, - dnn_to_tree_distillation_param=dnn_to_tree_distillation_param, - use_core_versions=use_core_versions, - override_global_step_value=override_global_step_value) - - super(DNNBoostedTreeCombinedClassifier, self).__init__( - model_fn=_model_fn, - model_dir=model_dir, - config=config, - feature_engineering_fn=feature_engineering_fn) - - -class DNNBoostedTreeCombinedRegressor(estimator.Estimator): - """A regressor that uses a combined DNN/GBDT model.""" - - def __init__(self, - dnn_hidden_units, - dnn_feature_columns, - tree_learner_config, - num_trees, - tree_examples_per_layer, - weight_column_name=None, - model_dir=None, - config=None, - label_name=None, - label_dimension=1, - feature_engineering_fn=None, - dnn_optimizer="Adagrad", - dnn_activation_fn=nn.relu, - dnn_dropout=None, - dnn_input_layer_partitioner=None, - dnn_input_layer_to_tree=True, - dnn_steps_to_train=10000, - predict_with_tree_only=False, - tree_feature_columns=None, - tree_center_bias=False, - dnn_to_tree_distillation_param=None, - use_core_versions=False, - override_global_step_value=None): - """Initializes a DNNBoostedTreeCombinedRegressor instance. - - Args: - dnn_hidden_units: List of hidden units per layer for DNN. - dnn_feature_columns: An iterable containing all the feature columns - used by the model's DNN. - tree_learner_config: A config for the tree learner. - num_trees: Number of trees to grow model to after training DNN. - tree_examples_per_layer: Number of examples to accumulate before - growing the tree a layer. This value has a big impact on model - quality and should be set equal to the number of examples in - training dataset if possible. It can also be a function that computes - the number of examples based on the depth of the layer that's - being built. - weight_column_name: The name of weight column. - model_dir: Directory for model exports. - config: `RunConfig` of the estimator. - label_name: String, name of the key in label dict. Can be null if label - is a tensor (single headed models). - label_dimension: Number of regression labels per example. This is the size - of the last dimension of the labels `Tensor` (typically, this has shape - `[batch_size, label_dimension]`). - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - dnn_optimizer: string, `Optimizer` object, or callable that defines the - optimizer to use for training the DNN. If `None`, will use the Adagrad - optimizer with default learning rate. - dnn_activation_fn: Activation function applied to each layer of the DNN. - If `None`, will use `tf.nn.relu`. - dnn_dropout: When not `None`, the probability to drop out a given - unit in the DNN. - dnn_input_layer_partitioner: Partitioner for input layer of the DNN. - Defaults to `min_max_variable_partitioner` with `min_slice_size` - 64 << 20. - dnn_input_layer_to_tree: Whether to provide the DNN's input layer - as a feature to the tree. - dnn_steps_to_train: Number of steps to train dnn for before switching - to gbdt. - predict_with_tree_only: Whether to use only the tree model output as the - final prediction. - tree_feature_columns: An iterable containing all the feature columns - used by the model's boosted trees. If dnn_input_layer_to_tree is - set to True, these features are in addition to dnn_feature_columns. - tree_center_bias: Whether a separate tree should be created for - first fitting the bias. - dnn_to_tree_distillation_param: A Tuple of (float, loss_fn), where the - float defines the weight of the distillation loss, and the loss_fn, for - computing distillation loss, takes dnn_logits, tree_logits and weight - tensor. If the entire tuple is None, no distillation will be applied. If - only the loss_fn is None, we will take the sigmoid/softmax cross entropy - loss be default. When distillation is applied, `predict_with_tree_only` - will be set to True. - use_core_versions: Whether feature columns and loss are from the core (as - opposed to contrib) version of tensorflow. - override_global_step_value: If after the training is done, global step - value must be reset to this value. This is particularly useful for hyper - parameter tuning, which can't recognize early stopping due to the number - of trees. If None, no override of global step will happen. - """ - head = head_lib.regression_head( - label_name=label_name, - label_dimension=label_dimension, - weight_column_name=weight_column_name, - enable_centered_bias=False) - - # num_classes needed for GradientBoostedDecisionTreeModel - if label_dimension == 1: - tree_learner_config.num_classes = 2 - else: - tree_learner_config.num_classes = label_dimension - - def _model_fn(features, labels, mode, config): - return _dnn_tree_combined_model_fn( - features=features, - labels=labels, - mode=mode, - head=head, - dnn_hidden_units=dnn_hidden_units, - dnn_feature_columns=dnn_feature_columns, - tree_learner_config=tree_learner_config, - num_trees=num_trees, - tree_examples_per_layer=tree_examples_per_layer, - config=config, - dnn_optimizer=dnn_optimizer, - dnn_activation_fn=dnn_activation_fn, - dnn_dropout=dnn_dropout, - dnn_input_layer_partitioner=dnn_input_layer_partitioner, - dnn_input_layer_to_tree=dnn_input_layer_to_tree, - dnn_steps_to_train=dnn_steps_to_train, - predict_with_tree_only=predict_with_tree_only, - tree_feature_columns=tree_feature_columns, - tree_center_bias=tree_center_bias, - dnn_to_tree_distillation_param=dnn_to_tree_distillation_param, - use_core_versions=use_core_versions, - override_global_step_value=override_global_step_value) - - super(DNNBoostedTreeCombinedRegressor, self).__init__( - model_fn=_model_fn, - model_dir=model_dir, - config=config, - feature_engineering_fn=feature_engineering_fn) - - -class DNNBoostedTreeCombinedEstimator(estimator.Estimator): - """An estimator that uses a combined DNN/GBDT model. - - Useful for training with user specified `Head`. - """ - - def __init__(self, - dnn_hidden_units, - dnn_feature_columns, - tree_learner_config, - num_trees, - tree_examples_per_layer, - head, - model_dir=None, - config=None, - feature_engineering_fn=None, - dnn_optimizer="Adagrad", - dnn_activation_fn=nn.relu, - dnn_dropout=None, - dnn_input_layer_partitioner=None, - dnn_input_layer_to_tree=True, - dnn_steps_to_train=10000, - predict_with_tree_only=False, - tree_feature_columns=None, - tree_center_bias=False, - dnn_to_tree_distillation_param=None, - use_core_versions=False, - override_global_step_value=None): - """Initializes a DNNBoostedTreeCombinedEstimator instance. - - Args: - dnn_hidden_units: List of hidden units per layer for DNN. - dnn_feature_columns: An iterable containing all the feature columns - used by the model's DNN. - tree_learner_config: A config for the tree learner. - num_trees: Number of trees to grow model to after training DNN. - tree_examples_per_layer: Number of examples to accumulate before - growing the tree a layer. This value has a big impact on model - quality and should be set equal to the number of examples in - training dataset if possible. It can also be a function that computes - the number of examples based on the depth of the layer that's - being built. - head: `Head` instance. - model_dir: Directory for model exports. - config: `RunConfig` of the estimator. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - dnn_optimizer: string, `Optimizer` object, or callable that defines the - optimizer to use for training the DNN. If `None`, will use the Adagrad - optimizer with default learning rate. - dnn_activation_fn: Activation function applied to each layer of the DNN. - If `None`, will use `tf.nn.relu`. - dnn_dropout: When not `None`, the probability to drop out a given - unit in the DNN. - dnn_input_layer_partitioner: Partitioner for input layer of the DNN. - Defaults to `min_max_variable_partitioner` with `min_slice_size` - 64 << 20. - dnn_input_layer_to_tree: Whether to provide the DNN's input layer - as a feature to the tree. - dnn_steps_to_train: Number of steps to train dnn for before switching - to gbdt. - predict_with_tree_only: Whether to use only the tree model output as the - final prediction. - tree_feature_columns: An iterable containing all the feature columns - used by the model's boosted trees. If dnn_input_layer_to_tree is - set to True, these features are in addition to dnn_feature_columns. - tree_center_bias: Whether a separate tree should be created for - first fitting the bias. - dnn_to_tree_distillation_param: A Tuple of (float, loss_fn), where the - float defines the weight of the distillation loss, and the loss_fn, for - computing distillation loss, takes dnn_logits, tree_logits and weight - tensor. If the entire tuple is None, no distillation will be applied. If - only the loss_fn is None, we will take the sigmoid/softmax cross entropy - loss be default. When distillation is applied, `predict_with_tree_only` - will be set to True. - use_core_versions: Whether feature columns and loss are from the core (as - opposed to contrib) version of tensorflow. - override_global_step_value: If after the training is done, global step - value must be reset to this value. This is particularly useful for hyper - parameter tuning, which can't recognize early stopping due to the number - of trees. If None, no override of global step will happen. - """ - - def _model_fn(features, labels, mode, config): - return _dnn_tree_combined_model_fn( - features=features, - labels=labels, - mode=mode, - head=head, - dnn_hidden_units=dnn_hidden_units, - dnn_feature_columns=dnn_feature_columns, - tree_learner_config=tree_learner_config, - num_trees=num_trees, - tree_examples_per_layer=tree_examples_per_layer, - config=config, - dnn_optimizer=dnn_optimizer, - dnn_activation_fn=dnn_activation_fn, - dnn_dropout=dnn_dropout, - dnn_input_layer_partitioner=dnn_input_layer_partitioner, - dnn_input_layer_to_tree=dnn_input_layer_to_tree, - dnn_steps_to_train=dnn_steps_to_train, - predict_with_tree_only=predict_with_tree_only, - tree_feature_columns=tree_feature_columns, - tree_center_bias=tree_center_bias, - dnn_to_tree_distillation_param=dnn_to_tree_distillation_param, - use_core_versions=use_core_versions, - override_global_step_value=override_global_step_value) - - super(DNNBoostedTreeCombinedEstimator, self).__init__( - model_fn=_model_fn, - model_dir=model_dir, - config=config, - feature_engineering_fn=feature_engineering_fn) - - -class CoreDNNBoostedTreeCombinedEstimator(core_estimator.Estimator): - """Initializes a core version of DNNBoostedTreeCombinedEstimator. - - Args: - dnn_hidden_units: List of hidden units per layer for DNN. - dnn_feature_columns: An iterable containing all the feature columns - used by the model's DNN. - tree_learner_config: A config for the tree learner. - num_trees: Number of trees to grow model to after training DNN. - tree_examples_per_layer: Number of examples to accumulate before - growing the tree a layer. This value has a big impact on model - quality and should be set equal to the number of examples in - training dataset if possible. It can also be a function that computes - the number of examples based on the depth of the layer that's - being built. - head: `Head` instance. - model_dir: Directory for model exports. - config: `RunConfig` of the estimator. - dnn_optimizer: string, `Optimizer` object, or callable that defines the - optimizer to use for training the DNN. If `None`, will use the Adagrad - optimizer with default learning rate. - dnn_activation_fn: Activation function applied to each layer of the DNN. - If `None`, will use `tf.nn.relu`. - dnn_dropout: When not `None`, the probability to drop out a given - unit in the DNN. - dnn_input_layer_partitioner: Partitioner for input layer of the DNN. - Defaults to `min_max_variable_partitioner` with `min_slice_size` - 64 << 20. - dnn_input_layer_to_tree: Whether to provide the DNN's input layer - as a feature to the tree. - dnn_steps_to_train: Number of steps to train dnn for before switching - to gbdt. - predict_with_tree_only: Whether to use only the tree model output as the - final prediction. - tree_feature_columns: An iterable containing all the feature columns - used by the model's boosted trees. If dnn_input_layer_to_tree is - set to True, these features are in addition to dnn_feature_columns. - tree_center_bias: Whether a separate tree should be created for - first fitting the bias. - dnn_to_tree_distillation_param: A Tuple of (float, loss_fn), where the - float defines the weight of the distillation loss, and the loss_fn, for - computing distillation loss, takes dnn_logits, tree_logits and weight - tensor. If the entire tuple is None, no distillation will be applied. If - only the loss_fn is None, we will take the sigmoid/softmax cross entropy - loss be default. When distillation is applied, `predict_with_tree_only` - will be set to True. - """ - - def __init__(self, - dnn_hidden_units, - dnn_feature_columns, - tree_learner_config, - num_trees, - tree_examples_per_layer, - head, - model_dir=None, - config=None, - dnn_optimizer="Adagrad", - dnn_activation_fn=nn.relu, - dnn_dropout=None, - dnn_input_layer_partitioner=None, - dnn_input_layer_to_tree=True, - dnn_steps_to_train=10000, - predict_with_tree_only=False, - tree_feature_columns=None, - tree_center_bias=False, - dnn_to_tree_distillation_param=None): - - def _model_fn(features, labels, mode, config): - return _dnn_tree_combined_model_fn( - features=features, - labels=labels, - mode=mode, - head=head, - dnn_hidden_units=dnn_hidden_units, - dnn_feature_columns=dnn_feature_columns, - tree_learner_config=tree_learner_config, - num_trees=num_trees, - tree_examples_per_layer=tree_examples_per_layer, - config=config, - dnn_optimizer=dnn_optimizer, - dnn_activation_fn=dnn_activation_fn, - dnn_dropout=dnn_dropout, - dnn_input_layer_partitioner=dnn_input_layer_partitioner, - dnn_input_layer_to_tree=dnn_input_layer_to_tree, - dnn_steps_to_train=dnn_steps_to_train, - predict_with_tree_only=predict_with_tree_only, - tree_feature_columns=tree_feature_columns, - tree_center_bias=tree_center_bias, - dnn_to_tree_distillation_param=dnn_to_tree_distillation_param, - output_type=model.ModelBuilderOutputType.ESTIMATOR_SPEC, - use_core_versions=True, - override_global_step_value=None) - - super(CoreDNNBoostedTreeCombinedEstimator, self).__init__( - model_fn=_model_fn, model_dir=model_dir, config=config) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator_test.py deleted file mode 100644 index dea19b7c626..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator_test.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for combined DNN + GBDT estimators.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import tempfile -from tensorflow.contrib.boosted_trees.estimator_batch import dnn_tree_combined_estimator as estimator -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.learn.python.learn.estimators import estimator_test_utils -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.python.estimator import exporter -from tensorflow.python.estimator.canned import head as head_lib -from tensorflow.python.estimator.export import export -from tensorflow.python.ops import parsing_ops -from tensorflow.python.feature_column import feature_column_lib as core_feature_column -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import googletest -from tensorflow.python.training import checkpoint_utils - - -def _train_input_fn(): - features = { - "x": constant_op.constant([[2.], [1.], [1.]]) - } - label = constant_op.constant([[1], [0], [0]], dtype=dtypes.int32) - return features, label - - -def _eval_input_fn(): - features = { - "x": constant_op.constant([[1.], [2.], [2.]]) - } - label = constant_op.constant([[0], [1], [1]], dtype=dtypes.int32) - return features, label - - -class DNNBoostedTreeCombinedTest(test_util.TensorFlowTestCase): - - def testClassifierContract(self): - estimator_test_utils.assert_estimator_contract( - self, estimator.DNNBoostedTreeCombinedClassifier) - - def testRegressorContract(self): - estimator_test_utils.assert_estimator_contract( - self, estimator.DNNBoostedTreeCombinedRegressor) - - def testEstimatorContract(self): - estimator_test_utils.assert_estimator_contract( - self, estimator.DNNBoostedTreeCombinedEstimator) - - def testNoDNNFeatureColumns(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - - with self.assertRaisesRegexp( - ValueError, - "dnn_feature_columns must be specified"): - classifier = estimator.DNNBoostedTreeCombinedClassifier( - dnn_hidden_units=[1], - dnn_feature_columns=[], - tree_learner_config=learner_config, - num_trees=1, - tree_examples_per_layer=3, - n_classes=2) - classifier.fit(input_fn=_train_input_fn, steps=5) - - def testFitAndEvaluateDontThrowException(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.DNNBoostedTreeCombinedClassifier( - dnn_hidden_units=[1], - dnn_feature_columns=[feature_column.real_valued_column("x")], - tree_learner_config=learner_config, - num_trees=1, - tree_examples_per_layer=3, - n_classes=2, - model_dir=model_dir, - config=config, - dnn_steps_to_train=10, - dnn_input_layer_to_tree=False, - tree_feature_columns=[feature_column.real_valued_column("x")]) - - classifier.fit(input_fn=_train_input_fn, steps=15) - classifier.evaluate(input_fn=_eval_input_fn, steps=1) - - def testFitAndEvaluateWithDistillation(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.DNNBoostedTreeCombinedClassifier( - dnn_hidden_units=[1], - dnn_feature_columns=[feature_column.real_valued_column("x")], - tree_learner_config=learner_config, - num_trees=1, - tree_examples_per_layer=3, - n_classes=2, - model_dir=model_dir, - config=config, - dnn_steps_to_train=10, - dnn_input_layer_to_tree=False, - tree_feature_columns=[feature_column.real_valued_column("x")], - dnn_to_tree_distillation_param=(1, None)) - - classifier.fit(input_fn=_train_input_fn, steps=15) - classifier.evaluate(input_fn=_eval_input_fn, steps=1) - - -class CoreDNNBoostedTreeCombinedTest(test_util.TensorFlowTestCase): - - def _assert_checkpoint(self, model_dir, global_step): - reader = checkpoint_utils.load_checkpoint(model_dir) - self.assertEqual(global_step, reader.get_tensor(ops.GraphKeys.GLOBAL_STEP)) - - def testTrainEvaluateInferDoesNotThrowErrorWithNoDnnInput(self): - head_fn = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss( - loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS) - - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 3 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - est = estimator.CoreDNNBoostedTreeCombinedEstimator( - head=head_fn, - dnn_hidden_units=[1], - dnn_feature_columns=[core_feature_column.numeric_column("x")], - tree_learner_config=learner_config, - num_trees=1, - tree_examples_per_layer=3, - model_dir=model_dir, - config=config, - dnn_steps_to_train=10, - dnn_input_layer_to_tree=False, - tree_feature_columns=[core_feature_column.numeric_column("x")]) - - # Train for a few steps. - est.train(input_fn=_train_input_fn, steps=1000) - # 10 steps for dnn, 3 for 1 tree of depth 3 + 1 after the tree finished - self._assert_checkpoint(est.model_dir, global_step=14) - res = est.evaluate(input_fn=_eval_input_fn, steps=1) - self.assertLess(0.5, res["auc"]) - est.predict(input_fn=_eval_input_fn) - - def testTrainEvaluateInferDoesNotThrowErrorWithDnnInput(self): - head_fn = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss( - loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS) - - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 3 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - est = estimator.CoreDNNBoostedTreeCombinedEstimator( - head=head_fn, - dnn_hidden_units=[1], - dnn_feature_columns=[core_feature_column.numeric_column("x")], - tree_learner_config=learner_config, - num_trees=1, - tree_examples_per_layer=3, - model_dir=model_dir, - config=config, - dnn_steps_to_train=10, - dnn_input_layer_to_tree=True, - tree_feature_columns=[]) - - # Train for a few steps. - est.train(input_fn=_train_input_fn, steps=1000) - res = est.evaluate(input_fn=_eval_input_fn, steps=1) - self.assertLess(0.5, res["auc"]) - est.predict(input_fn=_eval_input_fn) - - def testTrainEvaluateWithDnnForInputAndTreeForPredict(self): - head_fn = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss( - loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS) - - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 3 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - est = estimator.CoreDNNBoostedTreeCombinedEstimator( - head=head_fn, - dnn_hidden_units=[1], - dnn_feature_columns=[core_feature_column.numeric_column("x")], - tree_learner_config=learner_config, - num_trees=1, - tree_examples_per_layer=3, - model_dir=model_dir, - config=config, - dnn_steps_to_train=10, - dnn_input_layer_to_tree=True, - predict_with_tree_only=True, - dnn_to_tree_distillation_param=(0.5, None), - tree_feature_columns=[]) - - # Train for a few steps. - est.train(input_fn=_train_input_fn, steps=1000) - res = est.evaluate(input_fn=_eval_input_fn, steps=1) - self.assertLess(0.5, res["auc"]) - est.predict(input_fn=_eval_input_fn) - serving_input_fn = ( - export.build_parsing_serving_input_receiver_fn( - feature_spec={"x": parsing_ops.FixedLenFeature( - [1], dtype=dtypes.float32)})) - base_exporter = exporter.FinalExporter( - name="Servo", - serving_input_receiver_fn=serving_input_fn, - assets_extra=None) - export_path = os.path.join(model_dir, "export") - base_exporter.export( - est, - export_path=export_path, - checkpoint_path=None, - eval_result={}, - is_the_final_export=True) - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py deleted file mode 100644 index ca1e968fae1..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py +++ /dev/null @@ -1,837 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""GTFlow Estimator definition.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -from tensorflow.contrib.boosted_trees.estimator_batch import model -from tensorflow.contrib.boosted_trees.python.utils import losses -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.python.estimator.canned import head as core_head_lib -from tensorflow.python.estimator import estimator as core_estimator -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.losses import losses as core_losses -from tensorflow.contrib.boosted_trees.estimator_batch import custom_loss_head -from tensorflow.python.ops import array_ops - -# ================== Old estimator interface=================================== -# The estimators below were designed for old feature columns and old estimator -# interface. They can be used with new feature columns and losses by setting -# use_core_libs = True. - - -class GradientBoostedDecisionTreeClassifier(estimator.Estimator): - """An estimator using gradient boosted decision trees.""" - - def __init__(self, - learner_config, - examples_per_layer, - n_classes=2, - num_trees=None, - feature_columns=None, - weight_column_name=None, - model_dir=None, - config=None, - label_keys=None, - feature_engineering_fn=None, - logits_modifier_function=None, - center_bias=True, - use_core_libs=False, - output_leaf_index=False, - override_global_step_value=None, - num_quantiles=100): - """Initializes a GradientBoostedDecisionTreeClassifier estimator instance. - - Args: - learner_config: A config for the learner. - examples_per_layer: Number of examples to accumulate before growing a - layer. It can also be a function that computes the number of examples - based on the depth of the layer that's being built. - n_classes: Number of classes in the classification. - num_trees: An int, number of trees to build. - feature_columns: A list of feature columns. - weight_column_name: Name of the column for weights, or None if not - weighted. - model_dir: Directory for model exports, etc. - config: `RunConfig` object to configure the runtime settings. - label_keys: Optional list of strings with size `[n_classes]` defining the - label vocabulary. Only supported for `n_classes` > 2. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - logits_modifier_function: A modifier function for the logits. - center_bias: Whether a separate tree should be created for first fitting - the bias. - use_core_libs: Whether feature columns and loss are from the core (as - opposed to contrib) version of tensorflow. - output_leaf_index: whether to output leaf indices along with predictions - during inference. The leaf node indexes are available in predictions - dict by the key 'leaf_index'. It is a Tensor of rank 2 and its shape is - [batch_size, num_trees]. For example, result_iter = - classifier.predict(...) - for result_dict in result_iter: # access leaf index list by - result_dict["leaf_index"] # which contains one leaf index per tree - override_global_step_value: If after the training is done, global step - value must be reset to this value. This should be used to reset global - step to a number > number of steps used to train the current ensemble. - For example, the usual way is to train a number of trees and set a very - large number of training steps. When the training is done (number of - trees were trained), this parameter can be used to set the global step - to a large value, making it look like that number of training steps ran. - If None, no override of global step will happen. - num_quantiles: Number of quantiles to build for numeric feature values. - - Raises: - ValueError: If learner_config is not valid. - """ - if n_classes > 2: - # For multi-class classification, use our loss implementation that - # supports second order derivative. - def loss_fn(labels, logits, weights=None): - result = losses.per_example_maxent_loss( - labels=labels, - logits=logits, - weights=weights, - num_classes=n_classes) - return math_ops.reduce_mean(result[0]) - else: - loss_fn = None - head = head_lib.multi_class_head( - n_classes=n_classes, - weight_column_name=weight_column_name, - enable_centered_bias=False, - loss_fn=loss_fn, - label_keys=label_keys) - if learner_config.num_classes == 0: - learner_config.num_classes = n_classes - elif learner_config.num_classes != n_classes: - raise ValueError("n_classes (%d) doesn't match learner_config (%d)." % - (n_classes, learner_config.num_classes)) - super(GradientBoostedDecisionTreeClassifier, self).__init__( - model_fn=model.model_builder, - params={ - 'head': head, - 'feature_columns': feature_columns, - 'learner_config': learner_config, - 'num_trees': num_trees, - 'weight_column_name': weight_column_name, - 'examples_per_layer': examples_per_layer, - 'center_bias': center_bias, - 'logits_modifier_function': logits_modifier_function, - 'use_core_libs': use_core_libs, - 'output_leaf_index': output_leaf_index, - 'override_global_step_value': override_global_step_value, - 'num_quantiles': num_quantiles, - }, - model_dir=model_dir, - config=config, - feature_engineering_fn=feature_engineering_fn) - - -class GradientBoostedDecisionTreeRegressor(estimator.Estimator): - """An estimator using gradient boosted decision trees.""" - - def __init__(self, - learner_config, - examples_per_layer, - label_dimension=1, - num_trees=None, - feature_columns=None, - label_name=None, - weight_column_name=None, - model_dir=None, - config=None, - feature_engineering_fn=None, - logits_modifier_function=None, - center_bias=True, - use_core_libs=False, - output_leaf_index=False, - override_global_step_value=None, - num_quantiles=100): - """Initializes a GradientBoostedDecisionTreeRegressor estimator instance. - - Args: - learner_config: A config for the learner. - examples_per_layer: Number of examples to accumulate before growing a - layer. It can also be a function that computes the number of examples - based on the depth of the layer that's being built. - label_dimension: Number of regression labels per example. This is the size - of the last dimension of the labels `Tensor` (typically, this has shape - `[batch_size, label_dimension]`). - num_trees: An int, number of trees to build. - feature_columns: A list of feature columns. - label_name: String, name of the key in label dict. Can be null if label is - a tensor (single headed models). - weight_column_name: Name of the column for weights, or None if not - weighted. - model_dir: Directory for model exports, etc. - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - logits_modifier_function: A modifier function for the logits. - center_bias: Whether a separate tree should be created for first fitting - the bias. - use_core_libs: Whether feature columns and loss are from the core (as - opposed to contrib) version of tensorflow. - output_leaf_index: whether to output leaf indices along with predictions - during inference. The leaf node indexes are available in predictions - dict by the key 'leaf_index'. For example, result_dict = - classifier.predict(...) - for example_prediction_result in result_dict: # access leaf index list - by example_prediction_result["leaf_index"] # which contains one leaf - index per tree - override_global_step_value: If after the training is done, global step - value must be reset to this value. This should be used to reset global - step to a number > number of steps used to train the current ensemble. - For example, the usual way is to train a number of trees and set a very - large number of training steps. When the training is done (number of - trees were trained), this parameter can be used to set the global step - to a large value, making it look like that number of training steps ran. - If None, no override of global step will happen. - num_quantiles: Number of quantiles to build for numeric feature values. - """ - head = head_lib.regression_head( - label_name=label_name, - label_dimension=label_dimension, - weight_column_name=weight_column_name, - enable_centered_bias=False) - if label_dimension == 1: - learner_config.num_classes = 2 - else: - learner_config.num_classes = label_dimension - super(GradientBoostedDecisionTreeRegressor, self).__init__( - model_fn=model.model_builder, - params={ - 'head': head, - 'feature_columns': feature_columns, - 'learner_config': learner_config, - 'num_trees': num_trees, - 'weight_column_name': weight_column_name, - 'examples_per_layer': examples_per_layer, - 'logits_modifier_function': logits_modifier_function, - 'center_bias': center_bias, - 'use_core_libs': use_core_libs, - 'output_leaf_index': False, - 'override_global_step_value': override_global_step_value, - 'num_quantiles': num_quantiles, - }, - model_dir=model_dir, - config=config, - feature_engineering_fn=feature_engineering_fn) - - -class GradientBoostedDecisionTreeEstimator(estimator.Estimator): - """An estimator using gradient boosted decision trees. - - Useful for training with user specified `Head`. - """ - - def __init__(self, - learner_config, - examples_per_layer, - head, - num_trees=None, - feature_columns=None, - weight_column_name=None, - model_dir=None, - config=None, - feature_engineering_fn=None, - logits_modifier_function=None, - center_bias=True, - use_core_libs=False, - output_leaf_index=False, - override_global_step_value=None, - num_quantiles=100): - """Initializes a GradientBoostedDecisionTreeEstimator estimator instance. - - Args: - learner_config: A config for the learner. - examples_per_layer: Number of examples to accumulate before growing a - layer. It can also be a function that computes the number of examples - based on the depth of the layer that's being built. - head: `Head` instance. - num_trees: An int, number of trees to build. - feature_columns: A list of feature columns. - weight_column_name: Name of the column for weights, or None if not - weighted. - model_dir: Directory for model exports, etc. - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - logits_modifier_function: A modifier function for the logits. - center_bias: Whether a separate tree should be created for first fitting - the bias. - use_core_libs: Whether feature columns and loss are from the core (as - opposed to contrib) version of tensorflow. - output_leaf_index: whether to output leaf indices along with predictions - during inference. The leaf node indexes are available in predictions - dict by the key 'leaf_index'. For example, result_dict = - classifier.predict(...) - for example_prediction_result in result_dict: # access leaf index list - by example_prediction_result["leaf_index"] # which contains one leaf - index per tree - override_global_step_value: If after the training is done, global step - value must be reset to this value. This should be used to reset global - step to a number > number of steps used to train the current ensemble. - For example, the usual way is to train a number of trees and set a very - large number of training steps. When the training is done (number of - trees were trained), this parameter can be used to set the global step - to a large value, making it look like that number of training steps ran. - If None, no override of global step will happen. - num_quantiles: Number of quantiles to build for numeric feature values. - """ - super(GradientBoostedDecisionTreeEstimator, self).__init__( - model_fn=model.model_builder, - params={ - 'head': head, - 'feature_columns': feature_columns, - 'learner_config': learner_config, - 'num_trees': num_trees, - 'weight_column_name': weight_column_name, - 'examples_per_layer': examples_per_layer, - 'logits_modifier_function': logits_modifier_function, - 'center_bias': center_bias, - 'use_core_libs': use_core_libs, - 'output_leaf_index': False, - 'override_global_step_value': override_global_step_value, - 'num_quantiles': num_quantiles, - }, - model_dir=model_dir, - config=config, - feature_engineering_fn=feature_engineering_fn) - - -class GradientBoostedDecisionTreeRanker(estimator.Estimator): - """A ranking estimator using gradient boosted decision trees.""" - - def __init__(self, - learner_config, - examples_per_layer, - head, - ranking_model_pair_keys, - num_trees=None, - feature_columns=None, - weight_column_name=None, - model_dir=None, - config=None, - label_keys=None, - feature_engineering_fn=None, - logits_modifier_function=None, - center_bias=False, - use_core_libs=False, - output_leaf_index=False, - override_global_step_value=None, - num_quantiles=100): - """Initializes a GradientBoostedDecisionTreeRanker instance. - - This is an estimator that can be trained off the pairwise data and can be - used for inference on non-paired data. This is essentially LambdaMart. - Args: - learner_config: A config for the learner. - examples_per_layer: Number of examples to accumulate before growing a - layer. It can also be a function that computes the number of examples - based on the depth of the layer that's being built. - head: `Head` instance. - ranking_model_pair_keys: Keys to distinguish between features for left and - right part of the training pairs for ranking. For example, for an - Example with features "a.f1" and "b.f1", the keys would be ("a", "b"). - num_trees: An int, number of trees to build. - feature_columns: A list of feature columns. - weight_column_name: Name of the column for weights, or None if not - weighted. - model_dir: Directory for model exports, etc. - config: `RunConfig` object to configure the runtime settings. - label_keys: Optional list of strings with size `[n_classes]` defining the - label vocabulary. Only supported for `n_classes` > 2. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - logits_modifier_function: A modifier function for the logits. - center_bias: Whether a separate tree should be created for first fitting - the bias. - use_core_libs: Whether feature columns and loss are from the core (as - opposed to contrib) version of tensorflow. - output_leaf_index: whether to output leaf indices along with predictions - during inference. The leaf node indexes are available in predictions - dict by the key 'leaf_index'. It is a Tensor of rank 2 and its shape is - [batch_size, num_trees]. For example, result_iter = - classifier.predict(...) - for result_dict in result_iter: # access leaf index list by - result_dict["leaf_index"] # which contains one leaf index per tree - override_global_step_value: If after the training is done, global step - value must be reset to this value. This should be used to reset global - step to a number > number of steps used to train the current ensemble. - For example, the usual way is to train a number of trees and set a very - large number of training steps. When the training is done (number of - trees were trained), this parameter can be used to set the global step - to a large value, making it look like that number of training steps ran. - If None, no override of global step will happen. - num_quantiles: Number of quantiles to build for numeric feature values. - - Raises: - ValueError: If learner_config is not valid. - """ - super(GradientBoostedDecisionTreeRanker, self).__init__( - model_fn=model.ranking_model_builder, - params={ - 'head': head, - 'n_classes': 2, - 'feature_columns': feature_columns, - 'learner_config': learner_config, - 'num_trees': num_trees, - 'weight_column_name': weight_column_name, - 'examples_per_layer': examples_per_layer, - 'center_bias': center_bias, - 'logits_modifier_function': logits_modifier_function, - 'use_core_libs': use_core_libs, - 'output_leaf_index': output_leaf_index, - 'ranking_model_pair_keys': ranking_model_pair_keys, - 'override_global_step_value': override_global_step_value, - 'num_quantiles': num_quantiles, - }, - model_dir=model_dir, - config=config, - feature_engineering_fn=feature_engineering_fn) - - -# When using this estimator, make sure to regularize the hessian (at least l2, -# min_node_weight)! -# TODO(nponomareva): extend to take multiple quantiles in one go. -class GradientBoostedDecisionTreeQuantileRegressor(estimator.Estimator): - """An estimator that does quantile regression and returns quantile estimates.""" - - def __init__(self, - learner_config, - examples_per_layer, - quantiles, - label_dimension=1, - num_trees=None, - feature_columns=None, - weight_column_name=None, - model_dir=None, - config=None, - feature_engineering_fn=None, - logits_modifier_function=None, - center_bias=True, - use_core_libs=False, - output_leaf_index=False, - override_global_step_value=None, - num_quantiles=100): - """Initializes a GradientBoostedDecisionTreeQuantileRegressor instance. - - Args: - learner_config: A config for the learner. - examples_per_layer: Number of examples to accumulate before growing a - layer. It can also be a function that computes the number of examples - based on the depth of the layer that's being built. - quantiles: a list of quantiles for the loss, each between 0 and 1. - label_dimension: Dimension of regression label. This is the size of the - last dimension of the labels `Tensor` (typically, this has shape - `[batch_size, label_dimension]`). When label_dimension>1, it is - recommended to use multiclass strategy diagonal hessian or full hessian. - num_trees: An int, number of trees to build. - feature_columns: A list of feature columns. - weight_column_name: Name of the column for weights, or None if not - weighted. - model_dir: Directory for model exports, etc. - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - logits_modifier_function: A modifier function for the logits. - center_bias: Whether a separate tree should be created for first fitting - the bias. - use_core_libs: Whether feature columns and loss are from the core (as - opposed to contrib) version of tensorflow. - output_leaf_index: whether to output leaf indices along with predictions - during inference. The leaf node indexes are available in predictions - dict by the key 'leaf_index'. For example, result_dict = - classifier.predict(...) - for example_prediction_result in result_dict: # access leaf index list - by example_prediction_result["leaf_index"] # which contains one leaf - index per tree - override_global_step_value: If after the training is done, global step - value must be reset to this value. This should be used to reset global - step to a number > number of steps used to train the current ensemble. - For example, the usual way is to train a number of trees and set a very - large number of training steps. When the training is done (number of - trees were trained), this parameter can be used to set the global step - to a large value, making it look like that number of training steps ran. - If None, no override of global step will happen. - num_quantiles: Number of quantiles to build for numeric feature values. - """ - - if len(quantiles) > 1: - raise ValueError('For now, just one quantile per estimator is supported') - - def _quantile_regression_head(quantile): - # Use quantile regression. - head = custom_loss_head.CustomLossHead( - loss_fn=functools.partial( - losses.per_example_quantile_regression_loss, quantile=quantile), - link_fn=array_ops.identity, - logit_dimension=label_dimension) - return head - - learner_config.num_classes = max(2, label_dimension) - - super(GradientBoostedDecisionTreeQuantileRegressor, self).__init__( - model_fn=model.model_builder, - params={ - 'head': _quantile_regression_head(quantiles[0]), - 'feature_columns': feature_columns, - 'learner_config': learner_config, - 'num_trees': num_trees, - 'weight_column_name': weight_column_name, - 'examples_per_layer': examples_per_layer, - 'logits_modifier_function': logits_modifier_function, - 'center_bias': center_bias, - 'use_core_libs': use_core_libs, - 'output_leaf_index': False, - 'override_global_step_value': override_global_step_value, - 'num_quantiles': num_quantiles, - }, - model_dir=model_dir, - config=config, - feature_engineering_fn=feature_engineering_fn) - - -# ================== New Estimator interface=================================== -# The estimators below use new core Estimator interface and must be used with -# new feature columns and heads. - - -# For multiclass classification, use the following head since it uses loss -# that is twice differentiable. -def core_multiclass_head( - n_classes, - weight_column=None, - loss_reduction=core_losses.Reduction.SUM_OVER_NONZERO_WEIGHTS): - """Core head for multiclass problems.""" - - def loss_fn(labels, logits): - result = losses.per_example_maxent_loss( - # Don't pass the weights: head already multiplies by them. - labels=labels, logits=logits, weights=None, num_classes=n_classes) - return result[0] - - # pylint:disable=protected-access - head_fn = core_head_lib._multi_class_head_with_softmax_cross_entropy_loss( - n_classes=n_classes, - loss_fn=loss_fn, - loss_reduction=loss_reduction, - weight_column=weight_column) - # pylint:enable=protected-access - - return head_fn - - -# For quantile regression, use this head with Core..Estimator, or use -# Core..QuantileRegressor directly, -def core_quantile_regression_head( - quantiles, - label_dimension=1, - weight_column=None, - loss_reduction=core_losses.Reduction.SUM_OVER_NONZERO_WEIGHTS): - """Core head for quantile regression problems.""" - - def loss_fn(labels, logits): - result = losses.per_example_quantile_regression_loss( - labels=labels, - predictions=logits, - # Don't pass the weights: head already multiplies by them. - weights=None, - quantile=quantiles) - return result[0] - - # pylint:disable=protected-access - head_fn = core_head_lib._regression_head( - label_dimension=label_dimension, - loss_fn=loss_fn, - loss_reduction=loss_reduction, - weight_column=weight_column) - # pylint:enable=protected-access - return head_fn - - -class CoreGradientBoostedDecisionTreeEstimator(core_estimator.Estimator): - """An estimator using gradient boosted decision trees. - - Useful for training with user specified `Head`. - """ - - def __init__(self, - learner_config, - examples_per_layer, - head, - num_trees=None, - feature_columns=None, - weight_column_name=None, - model_dir=None, - config=None, - label_keys=None, - feature_engineering_fn=None, - logits_modifier_function=None, - center_bias=True, - output_leaf_index=False, - num_quantiles=100): - """Initializes a core version of GradientBoostedDecisionTreeEstimator. - - Args: - learner_config: A config for the learner. - examples_per_layer: Number of examples to accumulate before growing a - layer. It can also be a function that computes the number of examples - based on the depth of the layer that's being built. - head: `Head` instance. - num_trees: An int, number of trees to build. - feature_columns: A list of feature columns. - weight_column_name: Name of the column for weights, or None if not - weighted. - model_dir: Directory for model exports, etc. - config: `RunConfig` object to configure the runtime settings. - label_keys: Optional list of strings with size `[n_classes]` defining the - label vocabulary. Only supported for `n_classes` > 2. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - logits_modifier_function: A modifier function for the logits. - center_bias: Whether a separate tree should be created for first fitting - the bias. - output_leaf_index: whether to output leaf indices along with predictions - during inference. The leaf node indexes are available in predictions - dict by the key 'leaf_index'. For example, result_dict = - classifier.predict(...) - for example_prediction_result in result_dict: # access leaf index list - by example_prediction_result["leaf_index"] # which contains one leaf - index per tree - num_quantiles: Number of quantiles to build for numeric feature values. - """ - - def _model_fn(features, labels, mode, config): - return model.model_builder( - features=features, - labels=labels, - mode=mode, - config=config, - params={ - 'head': head, - 'feature_columns': feature_columns, - 'learner_config': learner_config, - 'num_trees': num_trees, - 'weight_column_name': weight_column_name, - 'examples_per_layer': examples_per_layer, - 'center_bias': center_bias, - 'logits_modifier_function': logits_modifier_function, - 'use_core_libs': True, - 'output_leaf_index': output_leaf_index, - 'override_global_step_value': None, - 'num_quantiles': num_quantiles, - }, - output_type=model.ModelBuilderOutputType.ESTIMATOR_SPEC) - - super(CoreGradientBoostedDecisionTreeEstimator, self).__init__( - model_fn=_model_fn, model_dir=model_dir, config=config) - - -class CoreGradientBoostedDecisionTreeRanker(core_estimator.Estimator): - """A ranking estimator using gradient boosted decision trees.""" - - def __init__(self, - learner_config, - examples_per_layer, - head, - ranking_model_pair_keys, - num_trees=None, - feature_columns=None, - weight_column_name=None, - model_dir=None, - config=None, - label_keys=None, - logits_modifier_function=None, - center_bias=False, - output_leaf_index=False, - num_quantiles=100): - """Initializes a GradientBoostedDecisionTreeRanker instance. - - This is an estimator that can be trained off the pairwise data and can be - used for inference on non-paired data. This is essentially LambdaMart. - Args: - learner_config: A config for the learner. - examples_per_layer: Number of examples to accumulate before growing a - layer. It can also be a function that computes the number of examples - based on the depth of the layer that's being built. - head: `Head` instance. - ranking_model_pair_keys: Keys to distinguish between features for left and - right part of the training pairs for ranking. For example, for an - Example with features "a.f1" and "b.f1", the keys would be ("a", "b"). - num_trees: An int, number of trees to build. - feature_columns: A list of feature columns. - weight_column_name: Name of the column for weights, or None if not - weighted. - model_dir: Directory for model exports, etc. - config: `RunConfig` object to configure the runtime settings. - label_keys: Optional list of strings with size `[n_classes]` defining the - label vocabulary. Only supported for `n_classes` > 2. - logits_modifier_function: A modifier function for the logits. - center_bias: Whether a separate tree should be created for first fitting - the bias. - output_leaf_index: whether to output leaf indices along with predictions - during inference. The leaf node indexes are available in predictions - dict by the key 'leaf_index'. It is a Tensor of rank 2 and its shape is - [batch_size, num_trees]. For example, result_iter = - classifier.predict(...) - for result_dict in result_iter: # access leaf index list by - result_dict["leaf_index"] # which contains one leaf index per tree - num_quantiles: Number of quantiles to build for numeric feature values. - - Raises: - ValueError: If learner_config is not valid. - """ - - def _model_fn(features, labels, mode, config): - return model.ranking_model_builder( - features=features, - labels=labels, - mode=mode, - config=config, - params={ - 'head': head, - 'n_classes': 2, - 'feature_columns': feature_columns, - 'learner_config': learner_config, - 'num_trees': num_trees, - 'weight_column_name': weight_column_name, - 'examples_per_layer': examples_per_layer, - 'center_bias': center_bias, - 'logits_modifier_function': logits_modifier_function, - 'use_core_libs': True, - 'output_leaf_index': output_leaf_index, - 'ranking_model_pair_keys': ranking_model_pair_keys, - 'override_global_step_value': None, - 'num_quantiles': num_quantiles, - }, - output_type=model.ModelBuilderOutputType.ESTIMATOR_SPEC) - - super(CoreGradientBoostedDecisionTreeRanker, self).__init__( - model_fn=_model_fn, model_dir=model_dir, config=config) - - -# When using this estimator, make sure to regularize the hessian (at least l2, -# min_node_weight)! -# TODO(nponomareva): extend to take multiple quantiles in one go. -class CoreGradientBoostedDecisionTreeQuantileRegressor( - core_estimator.Estimator): - """An estimator that does quantile regression and returns quantile estimates.""" - - def __init__(self, - learner_config, - examples_per_layer, - quantiles, - label_dimension=1, - num_trees=None, - feature_columns=None, - weight_column_name=None, - model_dir=None, - config=None, - label_keys=None, - feature_engineering_fn=None, - logits_modifier_function=None, - center_bias=True, - output_leaf_index=False, - num_quantiles=100): - """Initializes a core version of GradientBoostedDecisionTreeEstimator. - - Args: - learner_config: A config for the learner. - examples_per_layer: Number of examples to accumulate before growing a - layer. It can also be a function that computes the number of examples - based on the depth of the layer that's being built. - quantiles: a list of quantiles for the loss, each between 0 and 1. - label_dimension: Dimension of regression label. This is the size of the - last dimension of the labels `Tensor` (typically, this has shape - `[batch_size, label_dimension]`). When label_dimension>1, it is - recommended to use multiclass strategy diagonal hessian or full hessian. - num_trees: An int, number of trees to build. - feature_columns: A list of feature columns. - weight_column_name: Name of the column for weights, or None if not - weighted. - model_dir: Directory for model exports, etc. - config: `RunConfig` object to configure the runtime settings. - label_keys: Optional list of strings with size `[n_classes]` defining the - label vocabulary. Only supported for `n_classes` > 2. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - logits_modifier_function: A modifier function for the logits. - center_bias: Whether a separate tree should be created for first fitting - the bias. - output_leaf_index: whether to output leaf indices along with predictions - during inference. The leaf node indexes are available in predictions - dict by the key 'leaf_index'. For example, result_dict = - classifier.predict(...) - for example_prediction_result in result_dict: # access leaf index list - by example_prediction_result["leaf_index"] # which contains one leaf - index per tree - num_quantiles: Number of quantiles to build for numeric feature values. - """ - if len(quantiles) > 1: - raise ValueError('For now, just one quantile per estimator is supported') - - def _model_fn(features, labels, mode, config): - return model.model_builder( - features=features, - labels=labels, - mode=mode, - config=config, - params={ - 'head': - core_quantile_regression_head( - quantiles[0], - label_dimension=label_dimension, - weight_column=weight_column_name), - 'feature_columns': - feature_columns, - 'learner_config': - learner_config, - 'num_trees': - num_trees, - 'weight_column_name': - weight_column_name, - 'examples_per_layer': - examples_per_layer, - 'center_bias': - center_bias, - 'logits_modifier_function': - logits_modifier_function, - 'use_core_libs': - True, - 'output_leaf_index': - output_leaf_index, - 'override_global_step_value': - None, - 'num_quantiles': - num_quantiles, - }, - output_type=model.ModelBuilderOutputType.ESTIMATOR_SPEC) - - super(CoreGradientBoostedDecisionTreeQuantileRegressor, self).__init__( - model_fn=_model_fn, model_dir=model_dir, config=config) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py deleted file mode 100644 index 60f92a0ff25..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py +++ /dev/null @@ -1,1030 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for GBDT estimator.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tempfile -import numpy as np - -from google.protobuf import text_format -from tensorflow.contrib.boosted_trees.estimator_batch import estimator -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.boosted_trees.proto import tree_config_pb2 -from tensorflow.contrib.layers.python.layers import feature_column as contrib_feature_column -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.python.estimator.canned import head as head_lib -from tensorflow.python.estimator.inputs import numpy_io -from tensorflow.python.feature_column import feature_column_lib as core_feature_column -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import test_util -from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import gfile -from tensorflow.python.platform import googletest -from tensorflow.python.training import checkpoint_utils - - -def _train_input_fn(): - features = {"x": constant_op.constant([[2.], [1.], [1.]])} - label = constant_op.constant([[1], [0], [0]], dtype=dtypes.int32) - return features, label - - -def _multiclass_train_input_fn(): - features = { - "x": constant_op.constant([[2.], [1.], [1.], [5.], [3.5], [4.6], [3.5]]) - } - label = constant_op.constant([[1], [0], [0], [2], [2], [0], [1]], - dtype=dtypes.int32) - return features, label - - -def _ranking_train_input_fn(): - features = { - "a.f1": constant_op.constant([[3.], [0.3], [1.]]), - "a.f2": constant_op.constant([[0.1], [3.], [1.]]), - "b.f1": constant_op.constant([[13.], [0.4], [5.]]), - "b.f2": constant_op.constant([[1.], [3.], [0.01]]), - } - label = constant_op.constant([[0], [0], [1]], dtype=dtypes.int32) - return features, label - - -def _eval_input_fn(): - features = {"x": constant_op.constant([[1.], [2.], [2.]])} - label = constant_op.constant([[0], [1], [1]], dtype=dtypes.int32) - return features, label - - -def _infer_ranking_train_input_fn(): - features = { - "f1": constant_op.constant([[3.], [2], [1.]]), - "f2": constant_op.constant([[0.1], [3.], [1.]]) - } - return features, None - - -_QUANTILE_REGRESSION_SIZE = 1000 - - -def _quantile_regression_input_fns(two_dimension=False): - # The data generation is taken from - # http://scikit-learn.org/stable/auto_examples/ensemble/plot_gradient_boosting_quantile.html - np.random.seed(1) - - def f(x): - """The function to predict.""" - return x * np.sin(x) - - def g(x): - """The function to predict.""" - return x * np.cos(x) - - # Training data. - x = np.atleast_2d(np.random.uniform(0, 10.0, - size=_QUANTILE_REGRESSION_SIZE)).T - x = x.astype(np.float32) - - # Labels. - if not two_dimension: - y = f(x).ravel() - else: - y = np.column_stack((f(x).ravel(), g(x).ravel())) - - # Add random noise. - dy = 1.5 + 1.0 * np.random.random(y.shape) - noise = np.random.normal(0, dy) - y += noise - y_original = y.astype(np.float32) - if not two_dimension: - y = y.reshape(_QUANTILE_REGRESSION_SIZE, 1) - - train_input_fn = numpy_io.numpy_input_fn( - x=x, - y=y, - batch_size=_QUANTILE_REGRESSION_SIZE, - num_epochs=None, - shuffle=True) - - # Test on the training data to make sure the predictions are calibrated. - test_input_fn = numpy_io.numpy_input_fn( - x=x, - y=y, - batch_size=_QUANTILE_REGRESSION_SIZE, - num_epochs=1, - shuffle=False) - - return train_input_fn, test_input_fn, y_original - - -class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase): - - def setUp(self): - self._export_dir_base = tempfile.mkdtemp() + "export/" - gfile.MkDir(self._export_dir_base) - - def _assert_checkpoint_and_return_model(self, model_dir, global_step): - reader = checkpoint_utils.load_checkpoint(model_dir) - self.assertEqual(global_step, reader.get_tensor(ops.GraphKeys.GLOBAL_STEP)) - serialized = reader.get_tensor("ensemble_model:0_config") - ensemble_proto = tree_config_pb2.DecisionTreeEnsembleConfig() - ensemble_proto.ParseFromString(serialized) - - return ensemble_proto - - def _assert_checkpoint(self, model_dir, global_step): - reader = checkpoint_utils.load_checkpoint(model_dir) - self.assertEqual(global_step, reader.get_tensor(ops.GraphKeys.GLOBAL_STEP)) - - def testFitAndEvaluateDontThrowException(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.GradientBoostedDecisionTreeClassifier( - learner_config=learner_config, - num_trees=1, - examples_per_layer=3, - model_dir=model_dir, - config=config, - feature_columns=[contrib_feature_column.real_valued_column("x")]) - - classifier.fit(input_fn=_train_input_fn, steps=15) - classifier.evaluate(input_fn=_eval_input_fn, steps=1) - classifier.export(self._export_dir_base) - - def testThatLeafIndexIsInPredictions(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.GradientBoostedDecisionTreeClassifier( - learner_config=learner_config, - num_trees=1, - examples_per_layer=3, - model_dir=model_dir, - config=config, - feature_columns=[contrib_feature_column.real_valued_column("x")], - output_leaf_index=True) - - classifier.fit(input_fn=_train_input_fn, steps=15) - result_iter = classifier.predict(input_fn=_eval_input_fn) - for prediction_dict in result_iter: - self.assertTrue("leaf_index" in prediction_dict) - self.assertTrue("logits" in prediction_dict) - - def testFitAndEvaluateDontThrowExceptionWithCoreForEstimator(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - # Use core head - head_fn = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss( - loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE) - - model = estimator.GradientBoostedDecisionTreeEstimator( - head=head_fn, - learner_config=learner_config, - num_trees=1, - examples_per_layer=3, - model_dir=model_dir, - config=config, - feature_columns=[core_feature_column.numeric_column("x")], - use_core_libs=True) - - model.fit(input_fn=_train_input_fn, steps=15) - model.evaluate(input_fn=_eval_input_fn, steps=1) - model.export(self._export_dir_base) - - def testFitAndEvaluateDontThrowExceptionWithCoreForClassifier(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.GradientBoostedDecisionTreeClassifier( - learner_config=learner_config, - num_trees=1, - examples_per_layer=3, - model_dir=model_dir, - config=config, - feature_columns=[core_feature_column.numeric_column("x")], - use_core_libs=True) - - classifier.fit(input_fn=_train_input_fn, steps=15) - classifier.evaluate(input_fn=_eval_input_fn, steps=1) - classifier.export(self._export_dir_base) - - def testFitAndEvaluateDontThrowExceptionWithCoreForRegressor(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - regressor = estimator.GradientBoostedDecisionTreeRegressor( - learner_config=learner_config, - num_trees=1, - examples_per_layer=3, - model_dir=model_dir, - config=config, - feature_columns=[core_feature_column.numeric_column("x")], - use_core_libs=True) - - regressor.fit(input_fn=_train_input_fn, steps=15) - regressor.evaluate(input_fn=_eval_input_fn, steps=1) - regressor.export(self._export_dir_base) - - def testRankingDontThrowExceptionForForEstimator(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - head_fn = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss( - loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS) - - model = estimator.GradientBoostedDecisionTreeRanker( - head=head_fn, - learner_config=learner_config, - num_trees=1, - examples_per_layer=3, - model_dir=model_dir, - config=config, - use_core_libs=True, - feature_columns=[ - core_feature_column.numeric_column("f1"), - core_feature_column.numeric_column("f2") - ], - ranking_model_pair_keys=("a", "b")) - - model.fit(input_fn=_ranking_train_input_fn, steps=1000) - model.evaluate(input_fn=_ranking_train_input_fn, steps=1) - model.predict(input_fn=_infer_ranking_train_input_fn) - - def testDoesNotOverrideGlobalSteps(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 2 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.GradientBoostedDecisionTreeClassifier( - learner_config=learner_config, - num_trees=1, - examples_per_layer=3, - model_dir=model_dir, - config=config, - feature_columns=[contrib_feature_column.real_valued_column("x")], - output_leaf_index=False) - - classifier.fit(input_fn=_train_input_fn, steps=15) - # When no override of global steps, 5 steps were used. - self._assert_checkpoint(classifier.model_dir, global_step=5) - - def testOverridesGlobalSteps(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 2 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.GradientBoostedDecisionTreeClassifier( - learner_config=learner_config, - num_trees=1, - examples_per_layer=3, - model_dir=model_dir, - config=config, - feature_columns=[contrib_feature_column.real_valued_column("x")], - output_leaf_index=False, - override_global_step_value=10000000) - - classifier.fit(input_fn=_train_input_fn, steps=15) - self._assert_checkpoint(classifier.model_dir, global_step=10000000) - - def testFitAndEvaluateMultiClassTreePerClassDontThrowException(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 3 - learner_config.constraints.max_tree_depth = 1 - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.TREE_PER_CLASS) - - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.GradientBoostedDecisionTreeClassifier( - learner_config=learner_config, - n_classes=learner_config.num_classes, - num_trees=1, - examples_per_layer=7, - model_dir=model_dir, - config=config, - feature_columns=[contrib_feature_column.real_valued_column("x")]) - - classifier.fit(input_fn=_multiclass_train_input_fn, steps=100) - classifier.evaluate(input_fn=_eval_input_fn, steps=1) - classifier.export(self._export_dir_base) - result_iter = classifier.predict(input_fn=_eval_input_fn) - for prediction_dict in result_iter: - self.assertTrue("classes" in prediction_dict) - - def testFitAndEvaluateMultiClassDiagonalDontThrowException(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 3 - learner_config.constraints.max_tree_depth = 1 - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) - - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.GradientBoostedDecisionTreeClassifier( - learner_config=learner_config, - n_classes=learner_config.num_classes, - num_trees=1, - examples_per_layer=7, - model_dir=model_dir, - config=config, - center_bias=False, - feature_columns=[contrib_feature_column.real_valued_column("x")]) - - classifier.fit(input_fn=_multiclass_train_input_fn, steps=100) - classifier.evaluate(input_fn=_eval_input_fn, steps=1) - classifier.export(self._export_dir_base) - result_iter = classifier.predict(input_fn=_eval_input_fn) - for prediction_dict in result_iter: - self.assertTrue("classes" in prediction_dict) - - def testFitAndEvaluateMultiClassFullDontThrowException(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 3 - learner_config.constraints.max_tree_depth = 1 - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.FULL_HESSIAN) - - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.GradientBoostedDecisionTreeClassifier( - learner_config=learner_config, - n_classes=learner_config.num_classes, - num_trees=1, - examples_per_layer=7, - model_dir=model_dir, - config=config, - center_bias=False, - feature_columns=[contrib_feature_column.real_valued_column("x")]) - - classifier.fit(input_fn=_multiclass_train_input_fn, steps=100) - classifier.evaluate(input_fn=_eval_input_fn, steps=1) - classifier.export(self._export_dir_base) - result_iter = classifier.predict(input_fn=_eval_input_fn) - for prediction_dict in result_iter: - self.assertTrue("classes" in prediction_dict) - - # One dimensional quantile regression. - def testQuantileRegression(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 6 - learner_config.growing_mode = learner_pb2.LearnerConfig.LAYER_BY_LAYER - learner_config.constraints.min_node_weight = 1 / _QUANTILE_REGRESSION_SIZE - learner_config.regularization.l2 = 1.0 / _QUANTILE_REGRESSION_SIZE - learner_config.regularization.l1 = 1.0 / _QUANTILE_REGRESSION_SIZE - learner_config.regularization.tree_complexity = (1.0 / - _QUANTILE_REGRESSION_SIZE) - - train_input_fn, test_input_fn, y = _quantile_regression_input_fns() - - # 95% percentile. - model_upper = estimator.GradientBoostedDecisionTreeQuantileRegressor( - quantiles=[0.95], - learner_config=learner_config, - num_trees=12, - examples_per_layer=_QUANTILE_REGRESSION_SIZE, - center_bias=False) - - model_upper.fit(input_fn=train_input_fn, steps=1000) - result_iter = model_upper.predict(input_fn=test_input_fn) - upper = [] - for prediction_dict in result_iter: - upper.append(prediction_dict["scores"]) - - frac_below_upper = round(1. * np.count_nonzero(upper > y) / len(y), 3) - # +/- 3% - self.assertTrue(frac_below_upper >= 0.92) - self.assertTrue(frac_below_upper <= 0.98) - - # Multi-dimensional quantile regression. - def testQuantileRegressionMultiDimLabel(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 6 - learner_config.growing_mode = learner_pb2.LearnerConfig.LAYER_BY_LAYER - learner_config.constraints.min_node_weight = 1 / _QUANTILE_REGRESSION_SIZE - learner_config.regularization.l2 = 1.0 / _QUANTILE_REGRESSION_SIZE - learner_config.regularization.l1 = 1.0 / _QUANTILE_REGRESSION_SIZE - learner_config.regularization.tree_complexity = (1.0 / - _QUANTILE_REGRESSION_SIZE) - - train_input_fn, test_input_fn, y = _quantile_regression_input_fns( - two_dimension=True) - - # 95% percentile. - model_upper = estimator.GradientBoostedDecisionTreeQuantileRegressor( - quantiles=[0.95], - learner_config=learner_config, - label_dimension=2, - num_trees=18, - examples_per_layer=_QUANTILE_REGRESSION_SIZE, - center_bias=False) - - model_upper.fit(input_fn=train_input_fn, steps=1000) - result_iter = model_upper.predict(input_fn=test_input_fn) - upper = [] - for prediction_dict in result_iter: - upper.append(prediction_dict["scores"]) - - count_below_upper = np.count_nonzero(upper > y, axis=0) - count_both_below_upper = np.count_nonzero(np.prod(upper > y, axis=1)) - frac_below_upper_0 = round(1. * count_below_upper[0] / len(y), 3) - frac_below_upper_1 = round(1. * count_below_upper[1] / len(y), 3) - frac_both_below_upper = round(1. * count_both_below_upper / len(y), 3) - # +/- 3% - self.assertTrue(frac_below_upper_0 >= 0.92) - self.assertTrue(frac_below_upper_0 <= 0.98) - self.assertTrue(frac_below_upper_1 >= 0.92) - self.assertTrue(frac_below_upper_1 <= 0.98) - self.assertTrue(frac_both_below_upper >= 0.91) - self.assertTrue(frac_both_below_upper <= 0.99) - - def testForcedInitialSplits(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 3 - - initial_subtree = """ - nodes { - dense_float_binary_split { - feature_column: 0 - threshold: -0.5 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 0 - } - } - nodes { - dense_float_binary_split { - feature_column: 0 - threshold: 0.52 - left_id: 3 - right_id: 4 - } - node_metadata { - gain: 0 - } - } - nodes { - dense_float_binary_split { - feature_column: 0 - threshold: 0.554 - left_id: 5 - right_id: 6 - } - node_metadata { - gain: 0 - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - """ - tree_proto = tree_config_pb2.DecisionTreeConfig() - text_format.Merge(initial_subtree, tree_proto) - - # Set initial subtree info. - learner_config.each_tree_start.CopyFrom(tree_proto) - learner_config.each_tree_start_num_layers = 2 - - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.GradientBoostedDecisionTreeClassifier( - learner_config=learner_config, - num_trees=2, - examples_per_layer=6, - model_dir=model_dir, - config=config, - center_bias=False, - feature_columns=[contrib_feature_column.real_valued_column("x")], - output_leaf_index=False) - - classifier.fit(input_fn=_train_input_fn, steps=100) - # When no override of global steps, 5 steps were used. - ensemble = self._assert_checkpoint_and_return_model( - classifier.model_dir, global_step=6) - - # TODO(nponomareva): find a better way to test this. - expected_ensemble = """ - trees { - nodes { - dense_float_binary_split { - threshold: -0.5 - left_id: 1 - right_id: 2 - } - node_metadata { - } - } - nodes { - dense_float_binary_split { - threshold: 0.519999980927 - left_id: 3 - right_id: 4 - } - node_metadata { - } - } - nodes { - dense_float_binary_split { - threshold: 0.554000020027 - left_id: 5 - right_id: 6 - } - node_metadata { - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - nodes { - dense_float_binary_split { - threshold: 1.0 - left_id: 7 - right_id: 8 - } - node_metadata { - gain: 0.888888895512 - } - } - nodes { - leaf { - vector { - value: -2.0 - } - } - } - nodes { - leaf { - vector { - value: 2.00000023842 - } - } - } - } - trees { - nodes { - dense_float_binary_split { - threshold: -0.5 - left_id: 1 - right_id: 2 - } - node_metadata { - } - } - nodes { - dense_float_binary_split { - threshold: 0.519999980927 - left_id: 3 - right_id: 4 - } - node_metadata { - } - } - nodes { - dense_float_binary_split { - threshold: 0.554000020027 - left_id: 5 - right_id: 6 - } - node_metadata { - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - nodes { - dense_float_binary_split { - threshold: 1.0 - left_id: 7 - right_id: 8 - } - node_metadata { - gain: 0.727760672569 - } - } - nodes { - leaf { - vector { - value: -1.81873059273 - } - } - } - nodes { - leaf { - vector { - value: 1.81873047352 - } - } - } - } - trees { - nodes { - dense_float_binary_split { - threshold: -0.5 - left_id: 1 - right_id: 2 - } - node_metadata { - } - } - nodes { - dense_float_binary_split { - threshold: 0.519999980927 - left_id: 3 - right_id: 4 - } - node_metadata { - } - } - nodes { - dense_float_binary_split { - threshold: 0.554000020027 - left_id: 5 - right_id: 6 - } - node_metadata { - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - } - tree_weights: 0.10000000149 - tree_weights: 0.10000000149 - tree_weights: 0.10000000149 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 3 - is_finalized: true - } - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 3 - is_finalized: true - } - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 2 - } - growing_metadata { - num_layers_attempted: 3 - } - """ - self.assertProtoEquals(expected_ensemble, ensemble) - - -class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase): - - def testTrainEvaluateInferDoesNotThrowError(self): - head_fn = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss( - loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS) - - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - est = estimator.CoreGradientBoostedDecisionTreeEstimator( - head=head_fn, - learner_config=learner_config, - num_trees=1, - examples_per_layer=3, - model_dir=model_dir, - config=config, - feature_columns=[core_feature_column.numeric_column("x")]) - - # Train for a few steps. - est.train(input_fn=_train_input_fn, steps=1000) - est.evaluate(input_fn=_eval_input_fn, steps=1) - est.predict(input_fn=_eval_input_fn) - - def testRankingDontThrowExceptionForForEstimator(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - head_fn = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss( - loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS) - - est = estimator.CoreGradientBoostedDecisionTreeRanker( - head=head_fn, - learner_config=learner_config, - num_trees=1, - examples_per_layer=3, - model_dir=model_dir, - config=config, - feature_columns=[ - core_feature_column.numeric_column("f1"), - core_feature_column.numeric_column("f2") - ], - ranking_model_pair_keys=("a", "b")) - - # Train for a few steps. - est.train(input_fn=_ranking_train_input_fn, steps=1000) - est.evaluate(input_fn=_ranking_train_input_fn, steps=1) - est.predict(input_fn=_infer_ranking_train_input_fn) - - def testFitAndEvaluateMultiClassTreePerClasssDontThrowException(self): - n_classes = 3 - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = n_classes - learner_config.constraints.max_tree_depth = 1 - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.TREE_PER_CLASS) - - head_fn = estimator.core_multiclass_head(n_classes=n_classes) - - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.CoreGradientBoostedDecisionTreeEstimator( - learner_config=learner_config, - head=head_fn, - num_trees=1, - center_bias=False, - examples_per_layer=7, - model_dir=model_dir, - config=config, - feature_columns=[core_feature_column.numeric_column("x")]) - - classifier.train(input_fn=_multiclass_train_input_fn, steps=100) - classifier.evaluate(input_fn=_multiclass_train_input_fn, steps=1) - classifier.predict(input_fn=_eval_input_fn) - - def testFitAndEvaluateMultiClassDiagonalDontThrowException(self): - n_classes = 3 - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = n_classes - learner_config.constraints.max_tree_depth = 1 - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) - - head_fn = estimator.core_multiclass_head(n_classes=n_classes) - - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.CoreGradientBoostedDecisionTreeEstimator( - learner_config=learner_config, - head=head_fn, - num_trees=1, - center_bias=False, - examples_per_layer=7, - model_dir=model_dir, - config=config, - feature_columns=[core_feature_column.numeric_column("x")]) - - classifier.train(input_fn=_multiclass_train_input_fn, steps=100) - classifier.evaluate(input_fn=_multiclass_train_input_fn, steps=1) - classifier.predict(input_fn=_eval_input_fn) - - def testFitAndEvaluateMultiClassFullDontThrowException(self): - n_classes = 3 - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = n_classes - learner_config.constraints.max_tree_depth = 1 - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.FULL_HESSIAN) - - head_fn = estimator.core_multiclass_head(n_classes=n_classes) - - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - classifier = estimator.CoreGradientBoostedDecisionTreeEstimator( - learner_config=learner_config, - head=head_fn, - num_trees=1, - center_bias=False, - examples_per_layer=7, - model_dir=model_dir, - config=config, - feature_columns=[core_feature_column.numeric_column("x")]) - - classifier.train(input_fn=_multiclass_train_input_fn, steps=100) - classifier.evaluate(input_fn=_multiclass_train_input_fn, steps=1) - classifier.predict(input_fn=_eval_input_fn) - - def testWeightedCategoricalColumn(self): - head_fn = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss( - loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS) - - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - model_dir = tempfile.mkdtemp() - config = run_config.RunConfig() - - feature_columns = [ - core_feature_column.weighted_categorical_column( - categorical_column=core_feature_column - .categorical_column_with_vocabulary_list( - key="word", vocabulary_list=["the", "cat", "dog"]), - weight_feature_key="weight") - ] - - labels = np.array([[1], [1], [0], [0.]], dtype=np.float32) - - def _make_input_fn(): - - def _input_fn(): - features_dict = {} - # Sparse tensor representing - # example 0: "cat","the" - # examaple 1: "dog" - # example 2: - - # example 3: "the" - # Weights for the words are 5 - cat, 6- dog and 1 -the. - features_dict["word"] = sparse_tensor.SparseTensor( - indices=[[0, 0], [0, 1], [1, 0], [3, 0]], - values=constant_op.constant(["the", "cat", "dog", "the"], - dtype=dtypes.string), - dense_shape=[4, 3]) - features_dict["weight"] = sparse_tensor.SparseTensor( - indices=[[0, 0], [0, 1], [1, 0], [3, 0]], - values=[1., 5., 6., 1.], - dense_shape=[4, 3]) - return features_dict, labels - - return _input_fn - - est = estimator.CoreGradientBoostedDecisionTreeEstimator( - head=head_fn, - learner_config=learner_config, - num_trees=1, - examples_per_layer=3, - model_dir=model_dir, - config=config, - feature_columns=feature_columns) - - input_fn = _make_input_fn() - est.train(input_fn=input_fn, steps=100) - est.evaluate(input_fn=input_fn, steps=1) - est.predict(input_fn=input_fn) - - # Quantile regression in core is the same as in non core estimator, so we - # just check that it does not fail. - def testQuantileRegressionDoesNotThroughException(self): - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE - learner_config.constraints.min_node_weight = 1 / _QUANTILE_REGRESSION_SIZE - learner_config.regularization.l2 = 1.0 / _QUANTILE_REGRESSION_SIZE - learner_config.regularization.l1 = 1.0 / _QUANTILE_REGRESSION_SIZE - learner_config.regularization.tree_complexity = (1.0 / - _QUANTILE_REGRESSION_SIZE) - - train_input_fn, test_input_fn, y = _quantile_regression_input_fns() - y = y.reshape(_QUANTILE_REGRESSION_SIZE, 1) - - # 95% percentile. - model_upper = estimator.CoreGradientBoostedDecisionTreeQuantileRegressor( - quantiles=[0.95], - learner_config=learner_config, - num_trees=1, - examples_per_layer=_QUANTILE_REGRESSION_SIZE, - center_bias=False) - - model_upper.train(input_fn=train_input_fn, steps=1000) - result_iter = model_upper.predict(input_fn=test_input_fn) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_utils.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_utils.py deleted file mode 100644 index c4f94a6554a..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_utils.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities for converting between core and contrib feature columns.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.estimators import model_fn as contrib_model_fn_lib -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.python.estimator import model_fn as model_fn_lib -from tensorflow.python.estimator.export import export_output - -_CORE_MODE_TO_CONTRIB_MODE_ = { - model_fn_lib.ModeKeys.TRAIN: contrib_model_fn_lib.ModeKeys.TRAIN, - model_fn_lib.ModeKeys.EVAL: contrib_model_fn_lib.ModeKeys.EVAL, - model_fn_lib.ModeKeys.PREDICT: contrib_model_fn_lib.ModeKeys.INFER -} - - -def _core_mode_to_contrib_mode(mode): - return _CORE_MODE_TO_CONTRIB_MODE_[mode] - - -def _export_outputs_to_output_alternatives(export_outputs): - """Converts EstimatorSpec.export_outputs to output_alternatives. - - Args: - export_outputs: export_outputs created by create_estimator_spec. - Returns: - converted output_alternatives. - """ - output = {} - if export_outputs is not None: - for key, value in export_outputs.items(): - if isinstance(value, export_output.ClassificationOutput): - exported_predictions = { - prediction_key.PredictionKey.SCORES: value.scores, - prediction_key.PredictionKey.CLASSES: value.classes - } - output[key] = (constants.ProblemType.CLASSIFICATION, - exported_predictions) - return output - return None - - -def estimator_spec_to_model_fn_ops(estimator_spec, export_alternatives=False): - if export_alternatives: - alternatives = _export_outputs_to_output_alternatives( - estimator_spec.export_outputs) - else: - alternatives = [] - - return model_fn.ModelFnOps( - mode=_core_mode_to_contrib_mode(estimator_spec.mode), - predictions=estimator_spec.predictions, - loss=estimator_spec.loss, - train_op=estimator_spec.train_op, - eval_metric_ops=estimator_spec.eval_metric_ops, - output_alternatives=alternatives) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/model.py b/tensorflow/contrib/boosted_trees/estimator_batch/model.py deleted file mode 100644 index 477b191bcb7..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/model.py +++ /dev/null @@ -1,440 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""GTFlow Model definitions.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import copy - -from tensorflow.contrib import learn -from tensorflow.contrib.boosted_trees.estimator_batch import estimator_utils -from tensorflow.contrib.boosted_trees.estimator_batch import trainer_hooks -from tensorflow.contrib.boosted_trees.python.ops import model_ops -from tensorflow.contrib.boosted_trees.python.training.functions import gbdt_batch -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import ops -from tensorflow.python.ops import state_ops -from tensorflow.python.training import training_util -from google.protobuf import text_format -from tensorflow.contrib.boosted_trees.proto import tree_config_pb2 - - -class ModelBuilderOutputType(object): - MODEL_FN_OPS = 0 - ESTIMATOR_SPEC = 1 - - -def model_builder(features, - labels, - mode, - params, - config, - output_type=ModelBuilderOutputType.MODEL_FN_OPS): - """Multi-machine batch gradient descent tree model. - - Args: - features: `Tensor` or `dict` of `Tensor` objects. - labels: Labels used to train on. - mode: Mode we are in. (TRAIN/EVAL/INFER) - params: A dict of hyperparameters. - The following hyperparameters are expected: - * head: A `Head` instance. - * learner_config: A config for the learner. - * feature_columns: An iterable containing all the feature columns used by - the model. - * examples_per_layer: Number of examples to accumulate before growing a - layer. It can also be a function that computes the number of examples - based on the depth of the layer that's being built. - * weight_column_name: The name of weight column. - * center_bias: Whether a separate tree should be created for first fitting - the bias. - * override_global_step_value: If after the training is done, global step - value must be reset to this value. This is particularly useful for hyper - parameter tuning, which can't recognize early stopping due to the number - of trees. If None, no override of global step will happen. - config: `RunConfig` of the estimator. - output_type: Whether to return ModelFnOps (old interface) or EstimatorSpec - (new interface). - - Returns: - A `ModelFnOps` object. - Raises: - ValueError: if inputs are not valid. - """ - head = params["head"] - learner_config = params["learner_config"] - examples_per_layer = params["examples_per_layer"] - feature_columns = params["feature_columns"] - weight_column_name = params["weight_column_name"] - num_trees = params["num_trees"] - use_core_libs = params["use_core_libs"] - logits_modifier_function = params["logits_modifier_function"] - output_leaf_index = params["output_leaf_index"] - override_global_step_value = params.get("override_global_step_value", None) - num_quantiles = params["num_quantiles"] - - if features is None: - raise ValueError("At least one feature must be specified.") - - if config is None: - raise ValueError("Missing estimator RunConfig.") - if config.session_config is not None: - session_config = config.session_config - session_config.allow_soft_placement = True - else: - session_config = config_pb2.ConfigProto(allow_soft_placement=True) - config = config.replace(session_config=session_config) - - center_bias = params["center_bias"] - - if isinstance(features, ops.Tensor): - features = {features.name: features} - - # Make a shallow copy of features to ensure downstream usage - # is unaffected by modifications in the model function. - training_features = copy.copy(features) - training_features.pop(weight_column_name, None) - global_step = training_util.get_global_step() - - initial_ensemble = "" - if learner_config.each_tree_start.nodes: - if learner_config.each_tree_start_num_layers <= 0: - raise ValueError("You must provide each_tree_start_num_layers.") - num_layers = learner_config.each_tree_start_num_layers - initial_ensemble = """ - trees { %s } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: %d - is_finalized: false - } - """ % (text_format.MessageToString( - learner_config.each_tree_start), num_layers) - tree_ensemble_proto = tree_config_pb2.DecisionTreeEnsembleConfig() - text_format.Merge(initial_ensemble, tree_ensemble_proto) - initial_ensemble = tree_ensemble_proto.SerializeToString() - - with ops.device(global_step.device): - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=initial_ensemble, # Initialize the ensemble. - name="ensemble_model") - - # Create GBDT model. - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=config.is_chief, - num_ps_replicas=config.num_ps_replicas, - ensemble_handle=ensemble_handle, - center_bias=center_bias, - examples_per_layer=examples_per_layer, - learner_config=learner_config, - feature_columns=feature_columns, - logits_dimension=head.logits_dimension, - features=training_features, - use_core_columns=use_core_libs, - output_leaf_index=output_leaf_index, - num_quantiles=num_quantiles) - with ops.name_scope("gbdt", "gbdt_optimizer"): - predictions_dict = gbdt_model.predict(mode) - logits = predictions_dict["predictions"] - if logits_modifier_function: - logits = logits_modifier_function(logits, features, mode) - - def _train_op_fn(loss): - """Returns the op to optimize the loss.""" - update_op = gbdt_model.train(loss, predictions_dict, labels) - with ops.control_dependencies( - [update_op]), (ops.colocate_with(global_step)): - update_op = state_ops.assign_add(global_step, 1).op - return update_op - - create_estimator_spec_op = getattr(head, "create_estimator_spec", None) - - training_hooks = [] - if num_trees: - if center_bias: - num_trees += 1 - - finalized_trees, attempted_trees = gbdt_model.get_number_of_trees_tensor() - training_hooks.append( - trainer_hooks.StopAfterNTrees(num_trees, attempted_trees, - finalized_trees, - override_global_step_value)) - - if output_type == ModelBuilderOutputType.MODEL_FN_OPS: - if use_core_libs and callable(create_estimator_spec_op): - model_fn_ops = head.create_estimator_spec( - features=features, - mode=mode, - labels=labels, - train_op_fn=_train_op_fn, - logits=logits) - model_fn_ops = estimator_utils.estimator_spec_to_model_fn_ops( - model_fn_ops) - else: - model_fn_ops = head.create_model_fn_ops( - features=features, - mode=mode, - labels=labels, - train_op_fn=_train_op_fn, - logits=logits) - - if output_leaf_index and gbdt_batch.LEAF_INDEX in predictions_dict: - model_fn_ops.predictions[gbdt_batch.LEAF_INDEX] = predictions_dict[ - gbdt_batch.LEAF_INDEX] - - model_fn_ops.training_hooks.extend(training_hooks) - return model_fn_ops - elif output_type == ModelBuilderOutputType.ESTIMATOR_SPEC: - assert callable(create_estimator_spec_op) - estimator_spec = head.create_estimator_spec( - features=features, - mode=mode, - labels=labels, - train_op_fn=_train_op_fn, - logits=logits) - - if output_leaf_index and gbdt_batch.LEAF_INDEX in predictions_dict: - estimator_spec.predictions[gbdt_batch.LEAF_INDEX] = predictions_dict[ - gbdt_batch.LEAF_INDEX] - - estimator_spec = estimator_spec._replace( - training_hooks=training_hooks + list(estimator_spec.training_hooks)) - return estimator_spec - - return model_fn_ops - - -def ranking_model_builder(features, - labels, - mode, - params, - config, - output_type=ModelBuilderOutputType.MODEL_FN_OPS): - """Multi-machine batch gradient descent tree model for ranking. - - Args: - features: `Tensor` or `dict` of `Tensor` objects. - labels: Labels used to train on. - mode: Mode we are in. (TRAIN/EVAL/INFER) - params: A dict of hyperparameters. - The following hyperparameters are expected: - * head: A `Head` instance. - * learner_config: A config for the learner. - * feature_columns: An iterable containing all the feature columns used by - the model. - * examples_per_layer: Number of examples to accumulate before growing a - layer. It can also be a function that computes the number of examples - based on the depth of the layer that's being built. - * weight_column_name: The name of weight column. - * center_bias: Whether a separate tree should be created for first fitting - the bias. - * ranking_model_pair_keys (Optional): Keys to distinguish between features - for left and right part of the training pairs for ranking. For example, - for an Example with features "a.f1" and "b.f1", the keys would be - ("a", "b"). - * override_global_step_value: If after the training is done, global step - value must be reset to this value. This is particularly useful for hyper - parameter tuning, which can't recognize early stopping due to the number - of trees. If None, no override of global step will happen. - config: `RunConfig` of the estimator. - output_type: Whether to return ModelFnOps (old interface) or EstimatorSpec - (new interface). - - - Returns: - A `ModelFnOps` object. - Raises: - ValueError: if inputs are not valid. - """ - head = params["head"] - learner_config = params["learner_config"] - examples_per_layer = params["examples_per_layer"] - feature_columns = params["feature_columns"] - weight_column_name = params["weight_column_name"] - num_trees = params["num_trees"] - use_core_libs = params["use_core_libs"] - logits_modifier_function = params["logits_modifier_function"] - output_leaf_index = params["output_leaf_index"] - ranking_model_pair_keys = params["ranking_model_pair_keys"] - override_global_step_value = params.get("override_global_step_value", None) - num_quantiles = params["num_quantiles"] - - if features is None: - raise ValueError("At least one feature must be specified.") - - if config is None: - raise ValueError("Missing estimator RunConfig.") - - center_bias = params["center_bias"] - - if isinstance(features, ops.Tensor): - features = {features.name: features} - - # Make a shallow copy of features to ensure downstream usage - # is unaffected by modifications in the model function. - training_features = copy.copy(features) - training_features.pop(weight_column_name, None) - global_step = training_util.get_global_step() - with ops.device(global_step.device): - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config="", # Initialize an empty ensemble. - name="ensemble_model") - - # Extract the features. - if mode == learn.ModeKeys.TRAIN or mode == learn.ModeKeys.EVAL: - # For ranking pairwise training, we extract two sets of features. - if len(ranking_model_pair_keys) != 2: - raise ValueError("You must provide keys for ranking.") - left_pair_key = ranking_model_pair_keys[0] - right_pair_key = ranking_model_pair_keys[1] - if left_pair_key is None or right_pair_key is None: - raise ValueError("Both pair keys should be provided for ranking.") - - features_1 = {} - features_2 = {} - for name in training_features: - feature = training_features[name] - new_name = name[2:] - if name.startswith(left_pair_key + "."): - features_1[new_name] = feature - else: - assert name.startswith(right_pair_key + ".") - features_2[new_name] = feature - - main_features = features_1 - supplementary_features = features_2 - else: - # For non-ranking or inference ranking, we have only 1 set of features. - main_features = training_features - - # Create GBDT model. - gbdt_model_main = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=config.is_chief, - num_ps_replicas=config.num_ps_replicas, - ensemble_handle=ensemble_handle, - center_bias=center_bias, - examples_per_layer=examples_per_layer, - learner_config=learner_config, - feature_columns=feature_columns, - logits_dimension=head.logits_dimension, - features=main_features, - use_core_columns=use_core_libs, - output_leaf_index=output_leaf_index, - num_quantiles=num_quantiles) - - with ops.name_scope("gbdt", "gbdt_optimizer"): - # Logits for inference. - if mode == learn.ModeKeys.INFER: - predictions_dict = gbdt_model_main.predict(mode) - logits = predictions_dict[gbdt_batch.PREDICTIONS] - if logits_modifier_function: - logits = logits_modifier_function(logits, features, mode) - else: - gbdt_model_supplementary = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=config.is_chief, - num_ps_replicas=config.num_ps_replicas, - ensemble_handle=ensemble_handle, - center_bias=center_bias, - examples_per_layer=examples_per_layer, - learner_config=learner_config, - feature_columns=feature_columns, - logits_dimension=head.logits_dimension, - features=supplementary_features, - use_core_columns=use_core_libs, - output_leaf_index=output_leaf_index) - - # Logits for train and eval. - if not supplementary_features: - raise ValueError("Features for ranking must be specified.") - - predictions_dict_1 = gbdt_model_main.predict(mode) - predictions_1 = predictions_dict_1[gbdt_batch.PREDICTIONS] - - predictions_dict_2 = gbdt_model_supplementary.predict(mode) - predictions_2 = predictions_dict_2[gbdt_batch.PREDICTIONS] - - logits = predictions_1 - predictions_2 - if logits_modifier_function: - logits = logits_modifier_function(logits, features, mode) - - predictions_dict = predictions_dict_1 - predictions_dict[gbdt_batch.PREDICTIONS] = logits - - def _train_op_fn(loss): - """Returns the op to optimize the loss.""" - update_op = gbdt_model_main.train(loss, predictions_dict, labels) - with ops.control_dependencies( - [update_op]), (ops.colocate_with(global_step)): - update_op = state_ops.assign_add(global_step, 1).op - return update_op - - create_estimator_spec_op = getattr(head, "create_estimator_spec", None) - - training_hooks = [] - if num_trees: - if center_bias: - num_trees += 1 - - finalized_trees, attempted_trees = ( - gbdt_model_main.get_number_of_trees_tensor()) - training_hooks.append( - trainer_hooks.StopAfterNTrees(num_trees, attempted_trees, - finalized_trees, - override_global_step_value)) - - if output_type == ModelBuilderOutputType.MODEL_FN_OPS: - if use_core_libs and callable(create_estimator_spec_op): - model_fn_ops = head.create_estimator_spec( - features=features, - mode=mode, - labels=labels, - train_op_fn=_train_op_fn, - logits=logits) - model_fn_ops = estimator_utils.estimator_spec_to_model_fn_ops( - model_fn_ops) - else: - model_fn_ops = head.create_model_fn_ops( - features=features, - mode=mode, - labels=labels, - train_op_fn=_train_op_fn, - logits=logits) - - if output_leaf_index and gbdt_batch.LEAF_INDEX in predictions_dict: - model_fn_ops.predictions[gbdt_batch.LEAF_INDEX] = predictions_dict[ - gbdt_batch.LEAF_INDEX] - - model_fn_ops.training_hooks.extend(training_hooks) - return model_fn_ops - - elif output_type == ModelBuilderOutputType.ESTIMATOR_SPEC: - assert callable(create_estimator_spec_op) - estimator_spec = head.create_estimator_spec( - features=features, - mode=mode, - labels=labels, - train_op_fn=_train_op_fn, - logits=logits) - - estimator_spec = estimator_spec._replace( - training_hooks=training_hooks + list(estimator_spec.training_hooks)) - return estimator_spec - - return model_fn_ops diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/trainer_hooks.py b/tensorflow/contrib/boosted_trees/estimator_batch/trainer_hooks.py deleted file mode 100644 index f137ada3552..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/trainer_hooks.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Hooks for use with GTFlow Estimator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from tensorflow.contrib.learn.python.learn import session_run_hook -from tensorflow.contrib.learn.python.learn.session_run_hook import SessionRunArgs -from tensorflow.core.framework.summary_pb2 import Summary -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import training_util -from tensorflow.python.training.summary_io import SummaryWriterCache - - -class FeatureImportanceSummarySaver(session_run_hook.SessionRunHook): - """Hook to save feature importance summaries.""" - - def __init__(self, model_dir, every_n_steps=1): - """Create a FeatureImportanceSummarySaver Hook. - - This hook creates scalar summaries representing feature importance - for each feature column during training. - - Args: - model_dir: model base output directory. - every_n_steps: frequency, in number of steps, for logging summaries. - - Raises: - ValueError: If one of the arguments is invalid. - """ - if model_dir is None: - raise ValueError("model dir must be specified.") - self._model_dir = model_dir - self._every_n_steps = every_n_steps - self._last_triggered_step = None - - def begin(self): - self._global_step_tensor = training_util.get_global_step() - if self._global_step_tensor is None: - raise RuntimeError( - "Global step should be created to use FeatureImportanceSummarySaver.") - graph = ops.get_default_graph() - self._feature_names_tensor = graph.get_tensor_by_name( - "gbdt/feature_names:0") - self._feature_usage_counts_tensor = graph.get_tensor_by_name( - "gbdt/feature_usage_counts:0") - self._feature_gains_tensor = graph.get_tensor_by_name( - "gbdt/feature_gains:0") - - def before_run(self, run_context): - del run_context # Unused by feature importance summary saver hook. - requests = { - "global_step": self._global_step_tensor, - "feature_names": self._feature_names_tensor, - "feature_usage_counts": self._feature_usage_counts_tensor, - "feature_gains": self._feature_gains_tensor - } - return SessionRunArgs(requests) - - def after_run(self, run_context, run_values): - del run_context # Unused by feature importance summary saver hook. - - # Read result tensors. - global_step = run_values.results["global_step"] - feature_names = run_values.results["feature_names"] - feature_usage_counts = run_values.results["feature_usage_counts"] - feature_gains = run_values.results["feature_gains"] - - # Ensure summaries are logged at desired frequency - if (self._last_triggered_step is not None and - global_step < self._last_triggered_step + self._every_n_steps): - return - - # Validate tensors. - if (len(feature_names) != len(feature_usage_counts) or - len(feature_names) != len(feature_gains)): - raise RuntimeError( - "Feature names and importance measures have inconsistent lengths.") - - # Compute total usage. - total_usage_count = 0.0 - for usage_count in feature_usage_counts: - total_usage_count += usage_count - usage_count_norm = 1.0 / total_usage_count if total_usage_count else 1.0 - - # Compute total gain. - total_gain = 0.0 - for gain in feature_gains: - total_gain += gain - gain_norm = 1.0 / total_gain if total_gain else 1.0 - - # Output summary for each feature. - self._last_triggered_step = global_step - for (name, usage_count, gain) in zip(feature_names, feature_usage_counts, - feature_gains): - output_dir = os.path.join(self._model_dir, name.decode("utf-8")) - summary_writer = SummaryWriterCache.get(output_dir) - usage_count_summary = Summary(value=[ - Summary.Value( - tag="feature_importance/usage_counts", simple_value=usage_count) - ]) - usage_fraction_summary = Summary(value=[ - Summary.Value( - tag="feature_importance/usage_fraction", - simple_value=usage_count * usage_count_norm) - ]) - summary_writer.add_summary(usage_count_summary, global_step) - summary_writer.add_summary(usage_fraction_summary, global_step) - gains_summary = Summary(value=[ - Summary.Value(tag="feature_importance/gains", simple_value=gain) - ]) - gains_fraction_summary = Summary(value=[ - Summary.Value( - tag="feature_importance/gains_fraction", - simple_value=gain * gain_norm) - ]) - summary_writer.add_summary(gains_summary, global_step) - summary_writer.add_summary(gains_fraction_summary, global_step) - - -class FeedFnHook(session_run_hook.SessionRunHook): - """Runs feed_fn and sets the feed_dict accordingly.""" - - def __init__(self, feed_fn): - self.feed_fn = feed_fn - - def before_run(self, run_context): - del run_context # unused by FeedFnHook. - return session_run_hook.SessionRunArgs(fetches=None, feed_dict=self.feed_fn) - - -class StopAfterNTrees(session_run_hook.SessionRunHook): - """Stop training after building N full trees.""" - - def __init__(self, n, num_attempted_trees_tensor, num_finalized_trees_tensor, - override_global_step_value=None): - self._num_trees = n - # num_attempted_trees_tensor and num_finalized_trees_tensor are both - # tensors. - self._num_attempted_trees_tensor = num_attempted_trees_tensor - self._num_finalized_trees_tensor = num_finalized_trees_tensor - self._override_global_step_value = override_global_step_value - - def begin(self): - self._global_step_tensor = training_util.get_global_step() - if self._global_step_tensor is None: - raise RuntimeError("Global step should be created.") - - if self._override_global_step_value is not None: - self._override_global_step_op = state_ops.assign( - self._global_step_tensor, self._override_global_step_value) - - def before_run(self, run_context): - del run_context # unused by StopTrainingAfterNTrees. - return session_run_hook.SessionRunArgs({ - "num_attempted_trees": self._num_attempted_trees_tensor, - "num_finalized_trees": self._num_finalized_trees_tensor, - }) - - def after_run(self, run_context, run_values): - num_attempted_trees = run_values.results["num_attempted_trees"] - num_finalized_trees = run_values.results["num_finalized_trees"] - assert num_attempted_trees is not None - assert num_finalized_trees is not None - # Stop when the required number of finalized trees is reached, or when we - # try enough times to build a tree but keep failing. - if (num_finalized_trees >= self._num_trees or - num_attempted_trees > 2 * self._num_trees): - logging.info("Requesting stop since we have reached %d trees.", - num_finalized_trees) - if self._override_global_step_value is not None: - logging.info("Overriding global steps value.") - run_context.session.run(self._override_global_step_op) - run_context.request_stop() - - -class SwitchTrainOp(session_run_hook.SessionRunHook): - """Hook that switches the train op after specified number of steps. - - Hook that replaces the train op depending on the number of steps of training - that have taken place. The first_train_op is used till train_steps steps - are reached. Thereafter the second_train_op is used. - """ - - def __init__(self, first_train_op, train_steps, second_train_op): - """Initializes a `SwitchTrainOp`.""" - self._first_train_op = first_train_op - self._second_train_op = second_train_op - self._train_steps = train_steps - - def _get_train_op_for_global_step(self, current_step): - """Gets train_op for current global step.""" - if current_step < self._train_steps: - return self._first_train_op - return self._second_train_op - - def begin(self): - self._global_step_tensor = training_util.get_global_step() - self._current_train_op = control_flow_ops.no_op() - if self._global_step_tensor is None: - raise RuntimeError( - "Global step should be created to use SwitchTrainOp.") - - def before_run(self, run_context): # pylint: disable=unused-argument - return session_run_hook.SessionRunArgs( - {"global_step": self._global_step_tensor, - "train_op": self._current_train_op}) - - def after_run(self, run_context, run_values): - self._current_train_op = self._get_train_op_for_global_step( - run_values.results["global_step"]) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/trainer_hooks_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/trainer_hooks_test.py deleted file mode 100644 index 47e0eb6d97b..00000000000 --- a/tensorflow/contrib/boosted_trees/estimator_batch/trainer_hooks_test.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for trainer hooks.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import tempfile - -from tensorflow.contrib.boosted_trees.estimator_batch import trainer_hooks -from tensorflow.contrib.framework.python.ops import variables -from tensorflow.python.client import session as tf_session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variables as tf_variables -from tensorflow.python.platform import googletest -from tensorflow.python.training import monitored_session - - -class FeatureImportanceSummarySaverTest(test_util.TensorFlowTestCase): - - def test_invalid_input(self): - with self.assertRaises(ValueError): - trainer_hooks.FeatureImportanceSummarySaver(model_dir=None) - - def test_invalid_graph(self): - # Create inputs. - model_dir = tempfile.mkdtemp() - hook = trainer_hooks.FeatureImportanceSummarySaver(model_dir) - with ops.Graph().as_default(): - # Begin won't be able to find the required tensors in the graph. - _ = variables.get_or_create_global_step() - with self.assertRaises(KeyError): - hook.begin() - - def test_run(self): - # Create inputs. - model_dir = tempfile.mkdtemp() - hook = trainer_hooks.FeatureImportanceSummarySaver(model_dir) - with ops.Graph().as_default(), tf_session.Session() as sess: - global_step = variables.get_or_create_global_step() - with ops.name_scope("gbdt"): - constant_op.constant(["featA", "featB"], name="feature_names") - constant_op.constant([0, 2], name="feature_usage_counts") - constant_op.constant([0, 0.8], name="feature_gains") - # Begin finds tensors in the graph. - hook.begin() - sess.run(tf_variables.global_variables_initializer()) - # Run hook in a monitored session. - train_op = state_ops.assign_add(global_step, 1) - mon_sess = monitored_session._HookedSession(sess, [hook]) - mon_sess.run(train_op) - hook.end(sess) - # Ensure output summary dirs are created. - self.assertTrue(os.path.exists(os.path.join(model_dir, "featA"))) - self.assertTrue(os.path.exists(os.path.join(model_dir, "featB"))) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py deleted file mode 100644 index 47ee3d816f4..00000000000 --- a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -r"""Demonstrates multiclass MNIST TF Boosted trees example. - - This example demonstrates how to run experiments with TF Boosted Trees on - a binary dataset. We use digits 4 and 9 from the original MNIST dataset. - - Example Usage: - python tensorflow/contrib/boosted_trees/examples/binary_mnist.py \ - --output_dir="/tmp/binary_mnist" --depth=4 --learning_rate=0.3 \ - --batch_size=10761 --examples_per_layer=10761 --eval_batch_size=1030 \ - --num_eval_steps=1 --num_trees=10 --l2=1 --vmodule=training_ops=1 - - When training is done, accuracy on eval data is reported. Point tensorboard - to the directory for the run to see how the training progresses: - - tensorboard --logdir=/tmp/binary_mnist - -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import sys - -import numpy as np -import tensorflow as tf -from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeClassifier -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.learn import learn_runner - - -def get_input_fn(data, - batch_size, - capacity=10000, - min_after_dequeue=3000): - """Input function over MNIST data.""" - # Keep only 4 and 9 digits. - ids = np.where((data.labels == 4) | (data.labels == 9)) - images = data.images[ids] - labels = data.labels[ids] - # Make digit 4 label 1, 9 is 0. - labels = labels == 4 - - def _input_fn(): - """Prepare features and labels.""" - images_batch, labels_batch = tf.train.shuffle_batch( - tensors=[images, - labels.astype(np.int32)], - batch_size=batch_size, - capacity=capacity, - min_after_dequeue=min_after_dequeue, - enqueue_many=True, - num_threads=4) - features_map = {"images": images_batch} - return features_map, labels_batch - - return _input_fn - - -# Main config - creates a TF Boosted Trees Estimator based on flags. -def _get_tfbt(output_dir): - """Configures TF Boosted Trees estimator based on flags.""" - learner_config = learner_pb2.LearnerConfig() - - learner_config.learning_rate_tuner.fixed.learning_rate = FLAGS.learning_rate - learner_config.regularization.l1 = 0.0 - learner_config.regularization.l2 = FLAGS.l2 / FLAGS.examples_per_layer - learner_config.constraints.max_tree_depth = FLAGS.depth - - growing_mode = learner_pb2.LearnerConfig.LAYER_BY_LAYER - learner_config.growing_mode = growing_mode - run_config = tf.contrib.learn.RunConfig(save_checkpoints_secs=300) - - # Create a TF Boosted trees estimator that can take in custom loss. - estimator = GradientBoostedDecisionTreeClassifier( - learner_config=learner_config, - examples_per_layer=FLAGS.examples_per_layer, - model_dir=output_dir, - num_trees=FLAGS.num_trees, - center_bias=False, - config=run_config) - return estimator - - -def _make_experiment_fn(output_dir): - """Creates experiment for gradient boosted decision trees.""" - data = tf.contrib.learn.datasets.mnist.load_mnist() - train_input_fn = get_input_fn(data.train, FLAGS.batch_size) - eval_input_fn = get_input_fn(data.validation, FLAGS.eval_batch_size) - - return tf.contrib.learn.Experiment( - estimator=_get_tfbt(output_dir), - train_input_fn=train_input_fn, - eval_input_fn=eval_input_fn, - train_steps=None, - eval_steps=FLAGS.num_eval_steps, - eval_metrics=None) - - -def main(unused_argv): - learn_runner.run( - experiment_fn=_make_experiment_fn, - output_dir=FLAGS.output_dir, - schedule="train_and_evaluate") - - -if __name__ == "__main__": - tf.logging.set_verbosity(tf.logging.INFO) - parser = argparse.ArgumentParser() - # Define the list of flags that users can change. - parser.add_argument( - "--output_dir", - type=str, - required=True, - help="Choose the dir for the output.") - parser.add_argument( - "--batch_size", - type=int, - default=1000, - help="The batch size for reading data.") - parser.add_argument( - "--eval_batch_size", - type=int, - default=1000, - help="Size of the batch for eval.") - parser.add_argument( - "--num_eval_steps", - type=int, - default=1, - help="The number of steps to run evaluation for.") - # Flags for gradient boosted trees config. - parser.add_argument( - "--depth", type=int, default=4, help="Maximum depth of weak learners.") - parser.add_argument( - "--l2", type=float, default=1.0, help="l2 regularization per batch.") - parser.add_argument( - "--learning_rate", - type=float, - default=0.1, - help="Learning rate (shrinkage weight) with which each new tree is added." - ) - parser.add_argument( - "--examples_per_layer", - type=int, - default=1000, - help="Number of examples to accumulate stats for per layer.") - parser.add_argument( - "--num_trees", - type=int, - default=None, - required=True, - help="Number of trees to grow before stopping.") - - FLAGS, unparsed = parser.parse_known_args() - tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/boosted_trees/examples/boston.py b/tensorflow/contrib/boosted_trees/examples/boston.py deleted file mode 100644 index 09b240a7006..00000000000 --- a/tensorflow/contrib/boosted_trees/examples/boston.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -r"""Demonstrates a regression on Boston housing data. - - This example demonstrates how to run experiments with TF Boosted Trees on - a regression dataset. We split all the data into 20% test and 80% train, - and are using l2 loss and l2 regularization. - - Example Usage: - - python tensorflow/contrib/boosted_trees/examples/boston.py \ - --batch_size=404 --output_dir="/tmp/boston" --depth=4 --learning_rate=0.1 \ - --num_eval_steps=1 --num_trees=500 --l2=0.001 \ - --vmodule=training_ops=1 - - When training is done, mean squared error on eval data is reported. - Point tensorboard to the directory for the run to see how the training - progresses: - - tensorboard --logdir=/tmp/boston - -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import os -import sys -import tensorflow as tf -from tensorflow.contrib.boosted_trees.estimator_batch import custom_export_strategy -from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeRegressor -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.learn import learn_runner -from tensorflow.python.util import compat - -_BOSTON_NUM_FEATURES = 13 - - -# Main config - creates a TF Boosted Trees Estimator based on flags. -def _get_tfbt(output_dir, feature_cols): - """Configures TF Boosted Trees estimator based on flags.""" - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = FLAGS.learning_rate - learner_config.regularization.l1 = 0.0 - learner_config.regularization.l2 = FLAGS.l2 - learner_config.constraints.max_tree_depth = FLAGS.depth - - run_config = tf.contrib.learn.RunConfig(save_checkpoints_secs=300) - - # Create a TF Boosted trees regression estimator. - estimator = GradientBoostedDecisionTreeRegressor( - learner_config=learner_config, - # This should be the number of examples. For large datasets it can be - # larger than the batch_size. - examples_per_layer=FLAGS.batch_size, - feature_columns=feature_cols, - label_dimension=1, - model_dir=output_dir, - num_trees=FLAGS.num_trees, - center_bias=False, - config=run_config) - return estimator - - -def _convert_fn(dtec, sorted_feature_names, num_dense, num_sparse_float, - num_sparse_int, export_dir, unused_eval_result): - universal_format = custom_export_strategy.convert_to_universal_format( - dtec, sorted_feature_names, num_dense, num_sparse_float, num_sparse_int) - with tf.gfile.GFile(os.path.join( - compat.as_bytes(export_dir), compat.as_bytes("tree_proto")), "w") as f: - f.write(str(universal_format)) - - -def _make_experiment_fn(output_dir): - """Creates experiment for gradient boosted decision trees.""" - (x_train, y_train), (x_test, - y_test) = tf.keras.datasets.boston_housing.load_data() - - train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( - x={"x": x_train}, - y=y_train, - batch_size=FLAGS.batch_size, - num_epochs=None, - shuffle=True) - eval_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( - x={"x": x_test}, y=y_test, num_epochs=1, shuffle=False) - - feature_columns = [ - feature_column.real_valued_column("x", dimension=_BOSTON_NUM_FEATURES) - ] - feature_spec = tf.contrib.layers.create_feature_spec_for_parsing( - feature_columns) - serving_input_fn = tf.contrib.learn.utils.build_parsing_serving_input_fn( - feature_spec) - # An export strategy that outputs the feature importance and also exports - # the internal tree representation in another format. - export_strategy = custom_export_strategy.make_custom_export_strategy( - "exports", - convert_fn=_convert_fn, - feature_columns=feature_columns, - export_input_fn=serving_input_fn) - return tf.contrib.learn.Experiment( - estimator=_get_tfbt(output_dir, feature_columns), - train_input_fn=train_input_fn, - eval_input_fn=eval_input_fn, - train_steps=None, - eval_steps=FLAGS.num_eval_steps, - eval_metrics=None, - export_strategies=[export_strategy]) - - -def main(unused_argv): - learn_runner.run( - experiment_fn=_make_experiment_fn, - output_dir=FLAGS.output_dir, - schedule="train_and_evaluate") - - -if __name__ == "__main__": - tf.logging.set_verbosity(tf.logging.INFO) - parser = argparse.ArgumentParser() - # Define the list of flags that users can change. - parser.add_argument( - "--batch_size", - type=int, - default=1000, - help="The batch size for reading data.") - parser.add_argument( - "--output_dir", - type=str, - required=True, - help="Choose the dir for the output.") - parser.add_argument( - "--num_eval_steps", - type=int, - default=1, - help="The number of steps to run evaluation for.") - # Flags for gradient boosted trees config. - parser.add_argument( - "--depth", type=int, default=4, help="Maximum depth of weak learners.") - parser.add_argument( - "--l2", type=float, default=1.0, help="l2 regularization per batch.") - parser.add_argument( - "--learning_rate", - type=float, - default=0.1, - help="Learning rate (shrinkage weight) with which each new tree is added." - ) - parser.add_argument( - "--num_trees", - type=int, - default=None, - required=True, - help="Number of trees to grow before stopping.") - - FLAGS, unparsed = parser.parse_known_args() - tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/boosted_trees/examples/boston_combined.py b/tensorflow/contrib/boosted_trees/examples/boston_combined.py deleted file mode 100644 index d640af354f5..00000000000 --- a/tensorflow/contrib/boosted_trees/examples/boston_combined.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -r"""Regression on Boston housing data using DNNBoostedTreeCombinedRegressor. - - Example Usage: - - python tensorflow/contrib/boosted_trees/examples/boston_combined.py \ - --batch_size=404 --output_dir="/tmp/boston" \ - --dnn_hidden_units="8,4" --dnn_steps_to_train=1000 \ - --tree_depth=4 --tree_learning_rate=0.1 \ - --num_trees=100 --tree_l2=0.001 --num_eval_steps=1 \ - --vmodule=training_ops=1 - - When training is done, mean squared error on eval data is reported. - Point tensorboard to the directory for the run to see how the training - progresses: - - tensorboard --logdir=/tmp/boston - -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import sys -import tensorflow as tf - -from tensorflow.contrib.boosted_trees.estimator_batch.dnn_tree_combined_estimator import DNNBoostedTreeCombinedRegressor -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.learn.python.learn import learn_runner -from tensorflow.contrib.learn.python.learn.utils import input_fn_utils -from tensorflow.contrib.learn.python.learn.utils import saved_model_export_utils - -_BOSTON_NUM_FEATURES = 13 - - -def _get_estimator(output_dir, feature_cols): - """Configures DNNBoostedTreeCombinedRegressor based on flags.""" - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = ( - FLAGS.tree_learning_rate) - learner_config.regularization.l1 = 0.0 - learner_config.regularization.l2 = FLAGS.tree_l2 - learner_config.constraints.max_tree_depth = FLAGS.tree_depth - - run_config = tf.contrib.learn.RunConfig(save_summary_steps=1) - - # Create a DNNBoostedTreeCombinedRegressor estimator. - estimator = DNNBoostedTreeCombinedRegressor( - dnn_hidden_units=[int(x) for x in FLAGS.dnn_hidden_units.split(",")], - dnn_feature_columns=feature_cols, - tree_learner_config=learner_config, - num_trees=FLAGS.num_trees, - # This should be the number of examples. For large datasets it can be - # larger than the batch_size. - tree_examples_per_layer=FLAGS.batch_size, - model_dir=output_dir, - config=run_config, - dnn_input_layer_to_tree=True, - dnn_steps_to_train=FLAGS.dnn_steps_to_train) - return estimator - - -def _make_experiment_fn(output_dir): - """Creates experiment for DNNBoostedTreeCombinedRegressor.""" - (x_train, y_train), (x_test, - y_test) = tf.keras.datasets.boston_housing.load_data() - - train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( - x={"x": x_train}, - y=y_train, - batch_size=FLAGS.batch_size, - num_epochs=None, - shuffle=True) - eval_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( - x={"x": x_test}, y=y_test, num_epochs=1, shuffle=False) - - feature_columns = [ - feature_column.real_valued_column("x", dimension=_BOSTON_NUM_FEATURES) - ] - feature_spec = tf.contrib.layers.create_feature_spec_for_parsing( - feature_columns) - serving_input_fn = input_fn_utils.build_parsing_serving_input_fn(feature_spec) - export_strategies = [ - saved_model_export_utils.make_export_strategy(serving_input_fn)] - return tf.contrib.learn.Experiment( - estimator=_get_estimator(output_dir, feature_columns), - train_input_fn=train_input_fn, - eval_input_fn=eval_input_fn, - train_steps=None, - eval_steps=FLAGS.num_eval_steps, - eval_metrics=None, - export_strategies=export_strategies) - - -def main(unused_argv): - learn_runner.run( - experiment_fn=_make_experiment_fn, - output_dir=FLAGS.output_dir, - schedule="train_and_evaluate") - - -if __name__ == "__main__": - tf.logging.set_verbosity(tf.logging.INFO) - parser = argparse.ArgumentParser() - # Define the list of flags that users can change. - parser.add_argument( - "--batch_size", - type=int, - default=1000, - help="The batch size for reading data.") - parser.add_argument( - "--output_dir", - type=str, - required=True, - help="Choose the dir for the output.") - parser.add_argument( - "--num_eval_steps", - type=int, - default=1, - help="The number of steps to run evaluation for.") - # Flags for configuring DNNBoostedTreeCombinedRegressor. - parser.add_argument( - "--dnn_hidden_units", - type=str, - default="8,4", - help="Hidden layers for DNN.") - parser.add_argument( - "--dnn_steps_to_train", - type=int, - default=1000, - help="Number of steps to train DNN.") - parser.add_argument( - "--tree_depth", type=int, default=4, help="Maximum depth of trees.") - parser.add_argument( - "--tree_l2", type=float, default=1.0, help="l2 regularization per batch.") - parser.add_argument( - "--tree_learning_rate", - type=float, - default=0.1, - help=("Learning rate (shrinkage weight) with which each " - "new tree is added.")) - parser.add_argument( - "--num_trees", - type=int, - default=None, - required=True, - help="Number of trees to grow before stopping.") - - FLAGS, unparsed = parser.parse_known_args() - tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/boosted_trees/examples/mnist.py b/tensorflow/contrib/boosted_trees/examples/mnist.py deleted file mode 100644 index 817c6eb3e1a..00000000000 --- a/tensorflow/contrib/boosted_trees/examples/mnist.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -r"""Demonstrates multiclass MNIST TF Boosted trees example. - - This example demonstrates how to run experiments with TF Boosted Trees on - a MNIST dataset. We are using layer by layer boosting with diagonal hessian - strategy for multiclass handling, and cross entropy loss. - - Example Usage: - python tensorflow/contrib/boosted_trees/examples/mnist.py \ - --output_dir="/tmp/mnist" --depth=4 --learning_rate=0.3 --batch_size=60000 \ - --examples_per_layer=60000 --eval_batch_size=10000 --num_eval_steps=1 \ - --num_trees=10 --l2=1 --vmodule=training_ops=1 - - When training is done, accuracy on eval data is reported. Point tensorboard - to the directory for the run to see how the training progresses: - - tensorboard --logdir=/tmp/mnist - -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import sys - -import numpy as np -import tensorflow as tf -from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeClassifier -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.learn import learn_runner - - -def get_input_fn(dataset_split, - batch_size, - capacity=10000, - min_after_dequeue=3000): - """Input function over MNIST data.""" - - def _input_fn(): - """Prepare features and labels.""" - images_batch, labels_batch = tf.train.shuffle_batch( - tensors=[dataset_split.images, - dataset_split.labels.astype(np.int32)], - batch_size=batch_size, - capacity=capacity, - min_after_dequeue=min_after_dequeue, - enqueue_many=True, - num_threads=4) - features_map = {"images": images_batch} - return features_map, labels_batch - - return _input_fn - - -# Main config - creates a TF Boosted Trees Estimator based on flags. -def _get_tfbt(output_dir): - """Configures TF Boosted Trees estimator based on flags.""" - learner_config = learner_pb2.LearnerConfig() - - num_classes = 10 - - learner_config.learning_rate_tuner.fixed.learning_rate = FLAGS.learning_rate - learner_config.num_classes = num_classes - learner_config.regularization.l1 = 0.0 - learner_config.regularization.l2 = FLAGS.l2 / FLAGS.examples_per_layer - learner_config.constraints.max_tree_depth = FLAGS.depth - - growing_mode = learner_pb2.LearnerConfig.LAYER_BY_LAYER - learner_config.growing_mode = growing_mode - run_config = tf.contrib.learn.RunConfig(save_checkpoints_secs=300) - - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) - - # Create a TF Boosted trees estimator that can take in custom loss. - estimator = GradientBoostedDecisionTreeClassifier( - learner_config=learner_config, - n_classes=num_classes, - examples_per_layer=FLAGS.examples_per_layer, - model_dir=output_dir, - num_trees=FLAGS.num_trees, - center_bias=False, - config=run_config) - return estimator - - -def _make_experiment_fn(output_dir): - """Creates experiment for gradient boosted decision trees.""" - data = tf.contrib.learn.datasets.mnist.load_mnist() - train_input_fn = get_input_fn(data.train, FLAGS.batch_size) - eval_input_fn = get_input_fn(data.validation, FLAGS.eval_batch_size) - - return tf.contrib.learn.Experiment( - estimator=_get_tfbt(output_dir), - train_input_fn=train_input_fn, - eval_input_fn=eval_input_fn, - train_steps=None, - eval_steps=FLAGS.num_eval_steps, - eval_metrics=None) - - -def main(unused_argv): - learn_runner.run( - experiment_fn=_make_experiment_fn, - output_dir=FLAGS.output_dir, - schedule="train_and_evaluate") - - -if __name__ == "__main__": - tf.logging.set_verbosity(tf.logging.INFO) - parser = argparse.ArgumentParser() - # Define the list of flags that users can change. - parser.add_argument( - "--output_dir", - type=str, - required=True, - help="Choose the dir for the output.") - parser.add_argument( - "--batch_size", - type=int, - default=1000, - help="The batch size for reading data.") - parser.add_argument( - "--eval_batch_size", - type=int, - default=1000, - help="Size of the batch for eval.") - parser.add_argument( - "--num_eval_steps", - type=int, - default=1, - help="The number of steps to run evaluation for.") - # Flags for gradient boosted trees config. - parser.add_argument( - "--depth", type=int, default=4, help="Maximum depth of weak learners.") - parser.add_argument( - "--l2", type=float, default=1.0, help="l2 regularization per batch.") - parser.add_argument( - "--learning_rate", - type=float, - default=0.1, - help="Learning rate (shrinkage weight) with which each new tree is added." - ) - parser.add_argument( - "--examples_per_layer", - type=int, - default=1000, - help="Number of examples to accumulate stats for per layer.") - parser.add_argument( - "--num_trees", - type=int, - default=None, - required=True, - help="Number of trees to grow before stopping.") - - FLAGS, unparsed = parser.parse_known_args() - tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/boosted_trees/kernels/model_ops.cc b/tensorflow/contrib/boosted_trees/kernels/model_ops.cc deleted file mode 100644 index 5f9976a491c..00000000000 --- a/tensorflow/contrib/boosted_trees/kernels/model_ops.cc +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include - -#include "tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.h" -#include "tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/lib/core/refcount.h" -#include "tensorflow/core/platform/thread_annotations.h" - -namespace tensorflow { -namespace boosted_trees { - -using boosted_trees::models::DecisionTreeEnsembleResource; - -// Creates a tree ensemble variable. -class CreateTreeEnsembleVariableOp : public OpKernel { - public: - explicit CreateTreeEnsembleVariableOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - // Get the stamp token. - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - - // Get the tree ensemble config. - const Tensor* tree_ensemble_config_t; - OP_REQUIRES_OK(context, context->input("tree_ensemble_config", - &tree_ensemble_config_t)); - auto* result = new DecisionTreeEnsembleResource(); - if (!result->InitFromSerialized(tree_ensemble_config_t->scalar()(), - stamp_token)) { - result->Unref(); - OP_REQUIRES( - context, false, - errors::InvalidArgument("Unable to parse tree ensemble config.")); - } - - // Only create one, if one does not exist already. Report status for all - // other exceptions. - auto status = CreateResource(context, HandleFromInput(context, 0), result); - if (!status.ok() && status.code() != tensorflow::error::ALREADY_EXISTS) { - OP_REQUIRES(context, false, status); - } - } -}; - -// Op for retrieving a model stamp token without having to serialize. -class TreeEnsembleStampTokenOp : public OpKernel { - public: - explicit TreeEnsembleStampTokenOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr ensemble_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &ensemble_resource)); - tf_shared_lock l(*ensemble_resource->get_mutex()); - Tensor* output_stamp_token_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape(), - &output_stamp_token_t)); - output_stamp_token_t->scalar()() = ensemble_resource->stamp(); - } -}; - -// Op for serializing a model. -class TreeEnsembleSerializeOp : public OpKernel { - public: - explicit TreeEnsembleSerializeOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr ensemble_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &ensemble_resource)); - tf_shared_lock l(*ensemble_resource->get_mutex()); - Tensor* output_stamp_token_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape(), - &output_stamp_token_t)); - output_stamp_token_t->scalar()() = ensemble_resource->stamp(); - Tensor* output_config_t = nullptr; - OP_REQUIRES_OK( - context, context->allocate_output(1, TensorShape(), &output_config_t)); - output_config_t->scalar()() = - ensemble_resource->SerializeAsString(); - } -}; - -// Op for deserializing a tree ensemble variable from a checkpoint. -class TreeEnsembleDeserializeOp : public OpKernel { - public: - explicit TreeEnsembleDeserializeOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr ensemble_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &ensemble_resource)); - mutex_lock l(*ensemble_resource->get_mutex()); - - // Get the stamp token. - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - - // Get the tree ensemble config. - const Tensor* tree_ensemble_config_t; - OP_REQUIRES_OK(context, context->input("tree_ensemble_config", - &tree_ensemble_config_t)); - // Deallocate all the previous objects on the resource. - ensemble_resource->Reset(); - OP_REQUIRES( - context, - ensemble_resource->InitFromSerialized( - tree_ensemble_config_t->scalar()(), stamp_token), - errors::InvalidArgument("Unable to parse tree ensemble config.")); - } -}; - -class TreeEnsembleUsedHandlersOp : public OpKernel { - public: - explicit TreeEnsembleUsedHandlersOp(OpKernelConstruction* context) - : OpKernel(context) { - OP_REQUIRES_OK(context, - context->GetAttr("num_all_handlers", &num_handlers_)); - } - - void Compute(OpKernelContext* context) override { - core::RefCountPtr ensemble_resource; - - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &ensemble_resource)); - tf_shared_lock l(*ensemble_resource->get_mutex()); - - // Get the stamp token. - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - - // Only the Chief should run this Op and it is guaranteed to be in - // a consistent state so the stamps must always match. - CHECK(ensemble_resource->is_stamp_valid(stamp_token)); - - Tensor* output_used_handlers_t = nullptr; - OP_REQUIRES_OK( - context, context->allocate_output("used_handlers_mask", {num_handlers_}, - &output_used_handlers_t)); - auto output_used_handlers = output_used_handlers_t->vec(); - - Tensor* output_num_used_handlers_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("num_used_handlers", {}, - &output_num_used_handlers_t)); - int handler_idx = 0; - std::vector used_handlers = ensemble_resource->GetUsedHandlers(); - output_num_used_handlers_t->scalar()() = used_handlers.size(); - for (int64 i = 0; i < num_handlers_; ++i) { - if (handler_idx >= used_handlers.size() || - used_handlers[handler_idx] > i) { - output_used_handlers(i) = false; - } else { - OP_REQUIRES(context, used_handlers[handler_idx] == i, - errors::InvalidArgument("Handler IDs should be sorted.")); - ++handler_idx; - output_used_handlers(i) = true; - } - } - } - - private: - int64 num_handlers_; -}; - -REGISTER_RESOURCE_HANDLE_KERNEL(DecisionTreeEnsembleResource); - -REGISTER_KERNEL_BUILDER(Name("TreeEnsembleIsInitializedOp").Device(DEVICE_CPU), - IsResourceInitialized); - -REGISTER_KERNEL_BUILDER(Name("CreateTreeEnsembleVariable").Device(DEVICE_CPU), - CreateTreeEnsembleVariableOp); - -REGISTER_KERNEL_BUILDER(Name("TreeEnsembleStampToken").Device(DEVICE_CPU), - TreeEnsembleStampTokenOp); - -REGISTER_KERNEL_BUILDER(Name("TreeEnsembleSerialize").Device(DEVICE_CPU), - TreeEnsembleSerializeOp); - -REGISTER_KERNEL_BUILDER(Name("TreeEnsembleDeserialize").Device(DEVICE_CPU), - TreeEnsembleDeserializeOp); - -REGISTER_KERNEL_BUILDER(Name("TreeEnsembleUsedHandlers").Device(DEVICE_CPU), - TreeEnsembleUsedHandlersOp); -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/kernels/prediction_ops.cc b/tensorflow/contrib/boosted_trees/kernels/prediction_ops.cc deleted file mode 100644 index b740e0629ad..00000000000 --- a/tensorflow/contrib/boosted_trees/kernels/prediction_ops.cc +++ /dev/null @@ -1,468 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include -#include -#include - -#include "tensorflow/contrib/boosted_trees/lib/learner/common/partitioners/example_partitioner.h" -#include "tensorflow/contrib/boosted_trees/lib/models/multiple_additive_trees.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/batch_features.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.h" -#include "tensorflow/contrib/boosted_trees/proto/learner.pb.h" -#include "tensorflow/contrib/boosted_trees/proto/tree_config.pb.h" -#include "tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h" -#include "tensorflow/core/framework/device_base.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/refcount.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/protobuf.h" -#include "tensorflow/core/platform/types.h" - -using tensorflow::boosted_trees::learner::AveragingConfig; -using tensorflow::boosted_trees::trees::DecisionTreeEnsembleConfig; - -namespace tensorflow { -namespace boosted_trees { - -using boosted_trees::learner::LearnerConfig; -using boosted_trees::learner::LearningRateConfig; -using boosted_trees::learner::LearningRateDropoutDrivenConfig; -using boosted_trees::models::DecisionTreeEnsembleResource; -using boosted_trees::models::MultipleAdditiveTrees; -using boosted_trees::utils::DropoutUtils; -using boosted_trees::utils::TensorUtils; - -namespace { -const char* kLearnerConfigAttributeName = "learner_config"; -const char* kSeedTensorName = "seed"; -const char* kApplyDropoutAttributeName = "apply_dropout"; -const char* kApplyAveragingAttributeName = "apply_averaging"; -const char* kDropoutInfoOutputTensorName = "drop_out_tree_indices_weights"; -const char* kPredictionsTensorName = "predictions"; -const char* kLeafIndexTensorName = "leaf_index"; - -void CalculateTreesToInclude( - const boosted_trees::trees::DecisionTreeEnsembleConfig& config, - const std::vector& trees_to_drop, const int32 num_trees, - const bool only_finalized, const bool center_bias, - std::vector* trees_to_include) { - trees_to_include->reserve(num_trees - trees_to_drop.size()); - - int32 index = 0; - // This assumes that trees_to_drop is a sorted list of tree ids. - for (int32 tree = 0; tree < num_trees; ++tree) { - // Skip the tree if tree is in the list of trees_to_drop. - if (!trees_to_drop.empty() && index < trees_to_drop.size() && - trees_to_drop[index] == tree) { - ++index; - continue; - } - // Or skip if the tree is not finalized and only_finalized is set, - // with the exception of centering bias. - if (only_finalized && !(center_bias && tree == 0) && - config.tree_metadata_size() > 0 && - !config.tree_metadata(tree).is_finalized()) { - continue; - } - trees_to_include->push_back(tree); - } -} -} // namespace - -class GradientTreesPredictionOp : public OpKernel { - public: - explicit GradientTreesPredictionOp(OpKernelConstruction* const context) - : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr("use_locking", &use_locking_)); - - OP_REQUIRES_OK(context, context->GetAttr("center_bias", ¢er_bias_)); - - OP_REQUIRES_OK( - context, context->GetAttr(kApplyDropoutAttributeName, &apply_dropout_)); - - LearnerConfig learner_config; - string learner_config_str; - OP_REQUIRES_OK(context, context->GetAttr(kLearnerConfigAttributeName, - &learner_config_str)); - OP_REQUIRES( - context, ParseProtoUnlimited(&learner_config, learner_config_str), - errors::InvalidArgument("Unable to parse learner config config.")); - - num_classes_ = learner_config.num_classes(); - OP_REQUIRES(context, num_classes_ >= 2, - errors::InvalidArgument("Number of classes must be >=2")); - OP_REQUIRES( - context, ParseProtoUnlimited(&learner_config, learner_config_str), - errors::InvalidArgument("Unable to parse learner config config.")); - - bool reduce_dim; - OP_REQUIRES_OK(context, context->GetAttr("reduce_dim", &reduce_dim)); - prediction_vector_size_ = reduce_dim ? num_classes_ - 1 : num_classes_; - - only_finalized_trees_ = - learner_config.growing_mode() == learner_config.WHOLE_TREE; - if (learner_config.has_learning_rate_tuner() && - learner_config.learning_rate_tuner().tuner_case() == - LearningRateConfig::kDropout) { - dropout_config_ = learner_config.learning_rate_tuner().dropout(); - has_dropout_ = true; - } else { - has_dropout_ = false; - } - - OP_REQUIRES_OK(context, context->GetAttr(kApplyAveragingAttributeName, - &apply_averaging_)); - apply_averaging_ = - apply_averaging_ && learner_config.averaging_config().config_case() != - AveragingConfig::CONFIG_NOT_SET; - if (apply_averaging_) { - averaging_config_ = learner_config.averaging_config(); - - // If there is averaging config, check that the values are correct. - switch (averaging_config_.config_case()) { - case AveragingConfig::kAverageLastNTreesFieldNumber: { - OP_REQUIRES(context, averaging_config_.average_last_n_trees() > 0, - errors::InvalidArgument( - "Average last n trees must be a positive number")); - break; - } - case AveragingConfig::kAverageLastPercentTreesFieldNumber: { - OP_REQUIRES(context, - averaging_config_.average_last_percent_trees() > 0 && - averaging_config_.average_last_percent_trees() <= 1.0, - errors::InvalidArgument( - "Average last percent must be in (0,1] interval.")); - break; - } - case AveragingConfig::CONFIG_NOT_SET: { - LOG(QFATAL) << "We should never get here."; - break; - } - } - } - } - - void Compute(OpKernelContext* const context) override { - core::RefCountPtr ensemble_resource; - // Gets the resource. Grabs the mutex but releases it. - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &ensemble_resource)); - // Release the reference to the resource once we're done using it. - if (use_locking_) { - tf_shared_lock l(*ensemble_resource->get_mutex()); - DoCompute(context, ensemble_resource, - /*return_output_leaf_index=*/false); - } else { - DoCompute(context, ensemble_resource, - /*return_output_leaf_index=*/false); - } - } - - protected: - // return_output_leaf_index is a boolean variable indicating whether to output - // leaf index in prediction. Though this class invokes only with this param - // value as false, the subclass GradientTreesPredictionVerboseOp will invoke - // with the true value. - virtual void DoCompute( - OpKernelContext* context, - const core::RefCountPtr& ensemble_resource, - const bool return_output_leaf_index) { - // Read dense float features list; - OpInputList dense_float_features_list; - OP_REQUIRES_OK(context, TensorUtils::ReadDenseFloatFeatures( - context, &dense_float_features_list)); - - // Read sparse float features list; - OpInputList sparse_float_feature_indices_list; - OpInputList sparse_float_feature_values_list; - OpInputList sparse_float_feature_shapes_list; - OP_REQUIRES_OK(context, TensorUtils::ReadSparseFloatFeatures( - context, &sparse_float_feature_indices_list, - &sparse_float_feature_values_list, - &sparse_float_feature_shapes_list)); - - // Read sparse int features list; - OpInputList sparse_int_feature_indices_list; - OpInputList sparse_int_feature_values_list; - OpInputList sparse_int_feature_shapes_list; - OP_REQUIRES_OK(context, TensorUtils::ReadSparseIntFeatures( - context, &sparse_int_feature_indices_list, - &sparse_int_feature_values_list, - &sparse_int_feature_shapes_list)); - - // Infer batch size. - const int64 batch_size = TensorUtils::InferBatchSize( - dense_float_features_list, sparse_float_feature_shapes_list, - sparse_int_feature_shapes_list); - - // Read batch features. - boosted_trees::utils::BatchFeatures batch_features(batch_size); - OP_REQUIRES_OK( - context, - batch_features.Initialize( - TensorUtils::OpInputListToTensorVec(dense_float_features_list), - TensorUtils::OpInputListToTensorVec( - sparse_float_feature_indices_list), - TensorUtils::OpInputListToTensorVec( - sparse_float_feature_values_list), - TensorUtils::OpInputListToTensorVec( - sparse_float_feature_shapes_list), - TensorUtils::OpInputListToTensorVec( - sparse_int_feature_indices_list), - TensorUtils::OpInputListToTensorVec(sparse_int_feature_values_list), - TensorUtils::OpInputListToTensorVec( - sparse_int_feature_shapes_list))); - - std::vector dropped_trees; - std::vector original_weights; - - // Do dropout if needed. - if (apply_dropout_ && has_dropout_) { - // Read in seed and cast to uint64. - const Tensor* seed_t; - OP_REQUIRES_OK(context, context->input(kSeedTensorName, &seed_t)); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(seed_t->shape()), - errors::InvalidArgument("Seed must be a scalar.")); - const uint64 seed = seed_t->scalar()(); - - std::unordered_set trees_not_to_drop; - if (center_bias_) { - trees_not_to_drop.insert(0); - } - if (ensemble_resource->decision_tree_ensemble().has_growing_metadata()) { - // We are in batch mode, the last tree is the tree that is being built, - // we can't drop it during dropout. - trees_not_to_drop.insert(ensemble_resource->num_trees() - 1); - } - const std::vector weights = ensemble_resource->GetTreeWeights(); - OP_REQUIRES_OK(context, DropoutUtils::DropOutTrees( - seed, dropout_config_, trees_not_to_drop, - weights, &dropped_trees, &original_weights)); - } - - // Prepare the list of trees to include in the prediction. - std::vector trees_to_include; - CalculateTreesToInclude( - ensemble_resource->decision_tree_ensemble(), dropped_trees, - ensemble_resource->decision_tree_ensemble().trees_size(), - only_finalized_trees_, center_bias_, &trees_to_include); - - // Allocate output predictions matrix. - Tensor* output_predictions_t = nullptr; - OP_REQUIRES_OK( - context, context->allocate_output(kPredictionsTensorName, - {batch_size, prediction_vector_size_}, - &output_predictions_t)); - auto output_predictions = output_predictions_t->matrix(); - - // Allocate output leaf index matrix. - Tensor* output_leaf_index_t = nullptr; - if (return_output_leaf_index) { - OP_REQUIRES_OK(context, context->allocate_output( - kLeafIndexTensorName, - {batch_size, ensemble_resource->num_trees()}, - &output_leaf_index_t)); - } - // Run predictor. - thread::ThreadPool* const worker_threads = - context->device()->tensorflow_cpu_worker_threads()->workers; - - if (apply_averaging_) { - DecisionTreeEnsembleConfig adjusted = - ensemble_resource->decision_tree_ensemble(); - const int start_averaging = std::max( - 0.0, - averaging_config_.config_case() == - AveragingConfig::kAverageLastNTreesFieldNumber - ? adjusted.trees_size() - averaging_config_.average_last_n_trees() - : adjusted.trees_size() * - (1.0 - averaging_config_.average_last_percent_trees())); - const int num_ensembles = adjusted.trees_size() - start_averaging; - for (int i = start_averaging; i < adjusted.trees_size(); ++i) { - float weight = adjusted.tree_weights(i); - adjusted.mutable_tree_weights()->Set( - i, weight * (num_ensembles - i + start_averaging) / num_ensembles); - } - MultipleAdditiveTrees::Predict(adjusted, trees_to_include, batch_features, - worker_threads, output_predictions, - output_leaf_index_t); - } else { - MultipleAdditiveTrees::Predict( - ensemble_resource->decision_tree_ensemble(), trees_to_include, - batch_features, worker_threads, output_predictions, - output_leaf_index_t); - } - - // Output dropped trees and original weights. - Tensor* output_dropout_info_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output( - kDropoutInfoOutputTensorName, - {2, static_cast(dropped_trees.size())}, - &output_dropout_info_t)); - auto output_dropout_info = output_dropout_info_t->matrix(); - for (int32 i = 0; i < dropped_trees.size(); ++i) { - output_dropout_info(0, i) = dropped_trees[i]; - output_dropout_info(1, i) = original_weights[i]; - } - } - - private: - LearningRateDropoutDrivenConfig dropout_config_; - AveragingConfig averaging_config_; - bool only_finalized_trees_; - int num_classes_; - // What is the size of the output vector for predictions? - int prediction_vector_size_; - bool apply_dropout_; - bool center_bias_; - bool apply_averaging_; - bool use_locking_; - bool has_dropout_; -}; - -REGISTER_KERNEL_BUILDER(Name("GradientTreesPrediction").Device(DEVICE_CPU), - GradientTreesPredictionOp); - -// GradientTreesPredictionVerboseOp is derived from GradientTreesPredictionOp -// and have an additional output of tensor of rank 2 containing leaf ids for -// each tree where an instance ended up with. -class GradientTreesPredictionVerboseOp : public GradientTreesPredictionOp { - public: - explicit GradientTreesPredictionVerboseOp(OpKernelConstruction* const context) - : GradientTreesPredictionOp(context) {} - - protected: - void DoCompute( - OpKernelContext* context, - const core::RefCountPtr& ensemble_resource, - bool return_output_leaf_index) override { - GradientTreesPredictionOp::DoCompute(context, ensemble_resource, - /*return_output_leaf_index=*/true); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("GradientTreesPredictionVerbose").Device(DEVICE_CPU), - GradientTreesPredictionVerboseOp); - -class GradientTreesPartitionExamplesOp : public OpKernel { - public: - explicit GradientTreesPartitionExamplesOp(OpKernelConstruction* const context) - : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr("use_locking", &use_locking_)); - } - - void Compute(OpKernelContext* const context) override { - core::RefCountPtr ensemble_resource; - // Gets the resource. Grabs the mutex but releases it. - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &ensemble_resource)); - if (use_locking_) { - tf_shared_lock l(*ensemble_resource->get_mutex()); - DoCompute(context, ensemble_resource); - } else { - DoCompute(context, ensemble_resource); - } - } - - private: - void DoCompute( - OpKernelContext* context, - const core::RefCountPtr& resource) { - // The last non-finalized tree in the ensemble is by convention the - // one to partition on. If no such tree exists, a nodeless tree is - // created. - boosted_trees::trees::DecisionTreeConfig empty_tree_config; - const boosted_trees::trees::DecisionTreeConfig& tree_config = - (resource->num_trees() <= 0 || - resource->LastTreeMetadata()->is_finalized()) - ? empty_tree_config - : *resource->LastTree(); - - // Read dense float features list; - OpInputList dense_float_features_list; - OP_REQUIRES_OK(context, TensorUtils::ReadDenseFloatFeatures( - context, &dense_float_features_list)); - - // Read sparse float features list; - OpInputList sparse_float_feature_indices_list; - OpInputList sparse_float_feature_values_list; - OpInputList sparse_float_feature_shapes_list; - OP_REQUIRES_OK(context, TensorUtils::ReadSparseFloatFeatures( - context, &sparse_float_feature_indices_list, - &sparse_float_feature_values_list, - &sparse_float_feature_shapes_list)); - - // Read sparse int features list; - OpInputList sparse_int_feature_indices_list; - OpInputList sparse_int_feature_values_list; - OpInputList sparse_int_feature_shapes_list; - OP_REQUIRES_OK(context, TensorUtils::ReadSparseIntFeatures( - context, &sparse_int_feature_indices_list, - &sparse_int_feature_values_list, - &sparse_int_feature_shapes_list)); - - // Infer batch size. - const int64 batch_size = TensorUtils::InferBatchSize( - dense_float_features_list, sparse_float_feature_shapes_list, - sparse_int_feature_shapes_list); - - // Read batch features. - boosted_trees::utils::BatchFeatures batch_features(batch_size); - OP_REQUIRES_OK( - context, - batch_features.Initialize( - TensorUtils::OpInputListToTensorVec(dense_float_features_list), - TensorUtils::OpInputListToTensorVec( - sparse_float_feature_indices_list), - TensorUtils::OpInputListToTensorVec( - sparse_float_feature_values_list), - TensorUtils::OpInputListToTensorVec( - sparse_float_feature_shapes_list), - TensorUtils::OpInputListToTensorVec( - sparse_int_feature_indices_list), - TensorUtils::OpInputListToTensorVec(sparse_int_feature_values_list), - TensorUtils::OpInputListToTensorVec( - sparse_int_feature_shapes_list))); - - // Allocate output partitions vector. - Tensor* partition_ids_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(0, {batch_size}, &partition_ids_t)); - thread::ThreadPool* const worker_threads = - context->device()->tensorflow_cpu_worker_threads()->workers; - learner::ExamplePartitioner::PartitionExamples( - tree_config, batch_features, worker_threads->NumThreads(), - worker_threads, partition_ids_t->vec().data()); - } - - private: - bool use_locking_; -}; - -REGISTER_KERNEL_BUILDER( - Name("GradientTreesPartitionExamples").Device(DEVICE_CPU), - GradientTreesPartitionExamplesOp); - -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc b/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc deleted file mode 100644 index ee31a4b72c8..00000000000 --- a/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc +++ /dev/null @@ -1,985 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include -#include -#include -#include - -#include "tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_stream.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/parallel_for.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.h" -#include "tensorflow/contrib/boosted_trees/proto/quantiles.pb.h" -#include "tensorflow/contrib/boosted_trees/resources/quantile_stream_resource.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/refcount.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/strings/stringprintf.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -using ::boosted_trees::QuantileConfig; -using boosted_trees::QuantileStreamResource; -using boosted_trees::utils::TensorUtils; - -namespace { -const char* const kExampleWeightsName = "example_weights"; -const char* const kMaxElementsName = "max_elements"; -const char* const kNextStampTokenName = "next_stamp_token"; -const char* const kStampTokenName = "stamp_token"; -const char* const kAreBucketsReadyName = "are_buckets_ready"; -const char* const kGenerateQuantiles = "generate_quantiles"; -// Names for sparse arguments. -const char* const kNumSparseFeaturesName = "num_sparse_features"; -const char* const kSparseBucketsName = "sparse_buckets"; -const char* const kSparseValuesName = "sparse_values"; -const char* const kSparseIndicesName = "sparse_indices"; -const char* const kSparseSummariesName = "sparse_summaries"; -const char* const kSparseConfigName = "sparse_config"; -const char* const kSparseOutputTensorName = "sparse_quantiles"; -// Names for dense arguments. -const char* const kDenseBucketsName = "dense_buckets"; -const char* const kDenseConfigName = "dense_config"; -const char* const kDenseOutputTensorName = "dense_quantiles"; -const char* const kDenseSummariesName = "dense_summaries"; -const char* const kDenseValuesName = "dense_values"; -const char* const kNumDenseFeaturesName = "num_dense_features"; -const char* const kResourceHandlesName = "quantile_accumulator_handles"; -const char* const kNumQuantilesName = "num_quantiles"; -const char* const kEpsilonName = "epsilon"; -const char* const kBucketsName = "buckets"; -const char* const kStreamStateName = "stream_state"; -const char* const kSummariesName = "summaries"; - -using QuantileStream = - boosted_trees::quantiles::WeightedQuantilesStream; -using QuantileSummary = - boosted_trees::quantiles::WeightedQuantilesSummary; -using QuantileSummaryEntry = - boosted_trees::quantiles::WeightedQuantilesSummary::SummaryEntry; - -std::vector GetBuckets(const int32 feature, - const OpInputList& buckets_list) { - const auto& buckets = buckets_list[feature].flat(); - std::vector buckets_vector(buckets.data(), - buckets.data() + buckets.size()); - return buckets_vector; -} - -int32 GetFeatureDimension(const int32 feature_index, const int64 instance, - const OpInputList* const indices_list) { - if (indices_list != nullptr) { - // Sparse multidimensional. - return (*indices_list)[feature_index].matrix()(instance, 1); - } - // No indices, assume one-dimensional tensor. - return 0; -} - -// Allows quantization for each of multiple dimensions of a sparse feature. -void QuantizeFeatures( - const string& output_name, const OpInputList& values_list, - const OpInputList& buckets_list, - const OpInputList* const - indices_list /** Optional, provide for sparse features **/, - OpKernelContext* const context) { - if (values_list.size() == 0) { - return; - } - OpOutputList output_list; - OP_REQUIRES_OK(context, context->output_list(output_name, &output_list)); - - for (int32 feature_index = 0; feature_index < values_list.size(); - ++feature_index) { - const Tensor& values_tensor = values_list[feature_index]; - const int64 num_values = values_tensor.dim_size(0); - - Tensor* output_t = nullptr; - // Output will have bucket id and dimension of the features for that bucket. - OP_REQUIRES_OK( - context, output_list.allocate(feature_index, - TensorShape({num_values, 2}), &output_t)); - - auto output = output_t->matrix(); - - const std::vector& buckets_vector = - GetBuckets(feature_index, buckets_list); - auto flat_values = values_tensor.flat(); - for (int64 instance = 0; instance < num_values; ++instance) { - const float value = flat_values(instance); - CHECK(!buckets_vector.empty()) - << "Got empty buckets for feature " << feature_index; - auto bucket_iter = - std::lower_bound(buckets_vector.begin(), buckets_vector.end(), value); - if (bucket_iter == buckets_vector.end()) { - --bucket_iter; - } - const int32 bucket = - static_cast(bucket_iter - buckets_vector.begin()); - // Bucket id. - output(instance, 0) = bucket; - // Dimension. - output(instance, 1) = - GetFeatureDimension(feature_index, instance, indices_list); - } - } -} - -// Validates attributes for the quantile ops. -Status ReadAndValidateAttributes(OpKernelConstruction* const context, - int* num_dense_features, - int* num_sparse_features) { - TF_RETURN_IF_ERROR( - context->GetAttr(kNumDenseFeaturesName, num_dense_features)); - TF_RETURN_IF_ERROR( - context->GetAttr(kNumSparseFeaturesName, num_sparse_features)); - if ((*num_dense_features) + (*num_sparse_features) == 0) { - return errors::InvalidArgument( - "Please provide at least sparse or dense features."); - } - return Status::OK(); -} - -void ParseConfig(OpKernelConstruction* const context, const string& name, - std::vector* output) { - std::vector serialized_config; - OP_REQUIRES_OK(context, context->GetAttr(name, &serialized_config)); - output->reserve(serialized_config.size()); - QuantileConfig tmp; - for (const auto& serialized_string : serialized_config) { - OP_REQUIRES(context, tmp.ParseFromString(serialized_string), - errors::InvalidArgument("Malformed QuantileConfig passed in.")); - output->push_back(tmp); - } -} - -// Generates quantiles on a finalized QuantileStream. -std::vector GenerateBoundaries(const QuantileStream& stream, - int num_boundaries) { - std::vector boundaries = stream.GenerateBoundaries(num_boundaries); - - // Uniquify elements as we may get dupes. - auto end_it = std::unique(boundaries.begin(), boundaries.end()); - boundaries.resize(std::distance(boundaries.begin(), end_it)); - return boundaries; -} - -// Generates quantiles on a finalized QuantileStream. -std::vector GenerateQuantiles(const QuantileStream& stream, - int num_quantiles) { - // Do not de-dup boundaries. Exactly num_quantiles+1 boundary values - // will be returned. - std::vector boundaries = stream.GenerateQuantiles(num_quantiles); - CHECK_EQ(boundaries.size(), num_quantiles + 1); - return boundaries; -} - -// Copies quantiles to output list. -void CopyBoundaries(OpKernelContext* const context, - const std::vector& boundaries, const int64 index, - OpOutputList* output_list) { - // Output to tensor. - Tensor* output_t = nullptr; - OP_REQUIRES_OK( - context, output_list->allocate( - index, {static_cast(boundaries.size())}, &output_t)); - auto* quantiles_flat = output_t->flat().data(); - memcpy(quantiles_flat, boundaries.data(), sizeof(float) * boundaries.size()); -} - -void CopySummaryToProto(const QuantileSummary& summary, - ::boosted_trees::QuantileSummaryState* summary_proto) { - summary_proto->mutable_entries()->Reserve(summary.Size()); - for (const auto& entry : summary.GetEntryList()) { - auto* new_entry = summary_proto->add_entries(); - new_entry->set_value(entry.value); - new_entry->set_weight(entry.weight); - new_entry->set_min_rank(entry.min_rank); - new_entry->set_max_rank(entry.max_rank); - } -} - -} // namespace - -// Accumulator for Quantile Summaries. -REGISTER_RESOURCE_HANDLE_KERNEL(QuantileStreamResource); - -REGISTER_KERNEL_BUILDER( - Name("QuantileAccumulatorIsInitialized").Device(DEVICE_CPU), - IsResourceInitialized); - -class CreateQuantileAccumulatorOp : public OpKernel { - public: - explicit CreateQuantileAccumulatorOp(OpKernelConstruction* const context) - : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr(kEpsilonName, &epsilon_)); - OP_REQUIRES_OK(context, - context->GetAttr(kNumQuantilesName, &num_quantiles_)); - OP_REQUIRES_OK(context, context->GetAttr(kMaxElementsName, &max_elements_)); - OP_REQUIRES_OK(context, - context->GetAttr(kGenerateQuantiles, &generate_quantiles_)); - } - - void Compute(OpKernelContext* context) override { - // Only create one, if one does not exist already. Report status for all - // other exceptions. If one already exists, it unrefs the new one. - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input(kStampTokenName, &stamp_token_t)); - // An epsilon value of zero could cause perfoamance issues and is therefore, - // disallowed. - OP_REQUIRES( - context, epsilon_ > 0, - errors::InvalidArgument("An epsilon value of zero is not allowed.")); - auto result = new QuantileStreamResource(epsilon_, num_quantiles_, - max_elements_, generate_quantiles_, - stamp_token_t->scalar()()); - auto status = CreateResource(context, HandleFromInput(context, 0), result); - if (!status.ok() && status.code() != tensorflow::error::ALREADY_EXISTS) { - OP_REQUIRES(context, false, status); - } - } - - private: - float epsilon_; - int32 num_quantiles_; - // An upper bound on the number of entries that the summaries might have - // for a feature. - int64 max_elements_; - bool generate_quantiles_; -}; - -REGISTER_KERNEL_BUILDER(Name("CreateQuantileAccumulator").Device(DEVICE_CPU), - CreateQuantileAccumulatorOp); - -// Adds a summary to the quantile summary stream. -class QuantileAccumulatorAddSummariesOp : public OpKernel { - public: - explicit QuantileAccumulatorAddSummariesOp( - OpKernelConstruction* const context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - OpInputList resource_handle_list; - OP_REQUIRES_OK(context, context->input_list(kResourceHandlesName, - &resource_handle_list)); - OpInputList summary_list; - OP_REQUIRES_OK(context, context->input_list(kSummariesName, &summary_list)); - - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input(kStampTokenName, &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - - thread::ThreadPool* const worker_threads = - context->device()->tensorflow_cpu_worker_threads()->workers; - boosted_trees::utils::ParallelFor( - resource_handle_list.size(), worker_threads->NumThreads(), - worker_threads, - [&context, &resource_handle_list, &summary_list, stamp_token]( - int64 start, int64 end) { - for (int resource_handle_idx = start; resource_handle_idx < end; - ++resource_handle_idx) { - const ResourceHandle& handle = - resource_handle_list[resource_handle_idx] - .flat()(0); - core::RefCountPtr streams_resource; - // Create a reference to the underlying resource using the handle. - OP_REQUIRES_OK(context, - LookupResource(context, handle, &streams_resource)); - // Remove the reference at the end of this scope. - mutex_lock l(*streams_resource->mutex()); - - // If the stamp is invalid we drop the update. - if (!streams_resource->is_stamp_valid(stamp_token)) { - VLOG(1) - << "Invalid stamp token in QuantileAccumulatorAddSummariesOp." - << " Passed stamp token: " << stamp_token << " " - << "Current token: " << streams_resource->stamp(); - return; - } - - protobuf::Arena arena; - ::boosted_trees::QuantileSummaryState* summary_proto = - protobuf::Arena::CreateMessage< - ::boosted_trees::QuantileSummaryState>(&arena); - OP_REQUIRES( - context, - ParseProtoUnlimited( - summary_proto, - summary_list[resource_handle_idx].scalar()()), - errors::InvalidArgument("Unable to parse quantile summary.")); - std::vector entries; - entries.reserve(summary_proto->entries_size()); - for (const auto& entry : summary_proto->entries()) { - entries.emplace_back(entry.value(), entry.weight(), - entry.min_rank(), entry.max_rank()); - } - - // Add the summary to the quantile stream. - streams_resource->stream(stamp_token)->PushSummary(entries); - } - }); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("QuantileAccumulatorAddSummaries").Device(DEVICE_CPU), - QuantileAccumulatorAddSummariesOp); - -// Generates summaries for given set of float values, and the given config. -class MakeQuantileSummariesOp : public OpKernel { - public: - explicit MakeQuantileSummariesOp(OpKernelConstruction* const context) - : OpKernel(context) { - OP_REQUIRES_OK(context, - ReadAndValidateAttributes(context, &num_dense_features_, - &num_sparse_features_)); - OP_REQUIRES_OK(context, context->GetAttr(kEpsilonName, &epsilon_)); - } - - void Compute(OpKernelContext* const context) override { - // Read dense float features list; - OpInputList dense_float_features_list; - OP_REQUIRES_OK(context, TensorUtils::ReadDenseFloatFeatures( - context, &dense_float_features_list)); - - // Read sparse float features list; - OpInputList sparse_float_feature_indices_list; - OpInputList sparse_float_feature_values_list; - OpInputList sparse_float_feature_shapes_list; - OP_REQUIRES_OK(context, TensorUtils::ReadSparseFloatFeatures( - context, &sparse_float_feature_indices_list, - &sparse_float_feature_values_list, - &sparse_float_feature_shapes_list)); - - // Parse example weights and get batch size. - const Tensor* example_weights_t; - OP_REQUIRES_OK(context, - context->input(kExampleWeightsName, &example_weights_t)); - auto example_weights = example_weights_t->flat(); - const int64 batch_size = example_weights.size(); - - OpOutputList sparse_summaries_output_list; - OP_REQUIRES_OK(context, - context->output_list(kSparseSummariesName, - &sparse_summaries_output_list)); - OpOutputList dense_summaries_output_list; - OP_REQUIRES_OK(context, context->output_list(kDenseSummariesName, - &dense_summaries_output_list)); - - auto do_quantile_summary_gen = [&](const int64 begin, const int64 end) { - auto copy_over_summaries = [&](const QuantileStream& stream, - const int64 index, - OpOutputList* output_list) { - protobuf::Arena arena; - ::boosted_trees::QuantileSummaryState* summary_proto = - protobuf::Arena::CreateMessage< - ::boosted_trees::QuantileSummaryState>(&arena); - const auto& summary = stream.GetFinalSummary(); - CopySummaryToProto(summary, summary_proto); - // Output to tensor. - Tensor* output_t = nullptr; - OP_REQUIRES_OK(context, output_list->allocate(index, {}, &output_t)); - SerializeToTString(*summary_proto, &output_t->scalar()()); - }; - - // These are blocks of ranges. We are iterating over both sparse and - // dense features i.e. [0, sparse_features.size() + dense_features.size()] - for (int64 i = begin; i < end; ++i) { - if (i < num_dense_features_) { - const int64 dense_index = i; - const auto dense_values = - dense_float_features_list[dense_index].flat(); - QuantileStream stream(epsilon_, batch_size + 1); - // Run quantile summary generation. - for (int64 j = 0; j < batch_size; ++j) { - stream.PushEntry(dense_values(j), example_weights(j)); - } - stream.Finalize(); - // Copy summaries to output. - copy_over_summaries(stream, dense_index, - &dense_summaries_output_list); - } else { - const int64 sparse_index = i - num_dense_features_; - const auto sparse_values = - sparse_float_feature_values_list[sparse_index].flat(); - const auto sparse_indices = - sparse_float_feature_indices_list[sparse_index].matrix(); - const auto dense_shape = - sparse_float_feature_shapes_list[sparse_index].flat(); - OP_REQUIRES(context, batch_size == dense_shape(0), - errors::InvalidArgument( - "Sparse column shape doesn't match the batch size.")); - QuantileStream stream(epsilon_, batch_size + 1); - // Run quantile summary generation. - const int64 num_sparse_rows = - sparse_float_feature_indices_list[sparse_index].dim_size(0); - for (int64 j = 0; j < num_sparse_rows; ++j) { - const int64 example_id = sparse_indices(j, 0); - stream.PushEntry(sparse_values(j), example_weights(example_id)); - } - stream.Finalize(); - // Copy summaries to output. - copy_over_summaries(stream, sparse_index, - &sparse_summaries_output_list); - } - } - }; - const int64 kCostPerUnit = 500 * batch_size; - const int64 num_features = num_sparse_features_ + num_dense_features_; - const DeviceBase::CpuWorkerThreads& worker_threads = - *context->device()->tensorflow_cpu_worker_threads(); - Shard(worker_threads.num_threads, worker_threads.workers, num_features, - kCostPerUnit, do_quantile_summary_gen); - } - - private: - int num_dense_features_; - int num_sparse_features_; - float epsilon_; -}; - -REGISTER_KERNEL_BUILDER(Name("MakeQuantileSummaries").Device(DEVICE_CPU), - MakeQuantileSummariesOp); - -// Serializes the state of streams. -class QuantileAccumulatorSerializeOp : public OpKernel { - public: - explicit QuantileAccumulatorSerializeOp(OpKernelConstruction* const context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr streams_resource; - // Create a reference to the underlying resource using the handle. - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &streams_resource)); - // Remove the reference at the end of this scope. - mutex_lock l(*streams_resource->mutex()); - - int64 stamp_token = streams_resource->stamp(); - Tensor* stream_state_t; - OP_REQUIRES_OK(context, - context->allocate_output(kStreamStateName, TensorShape({}), - &stream_state_t)); - bool are_buckets_ready = streams_resource->are_buckets_ready(); - - // We are iterating over both dense and sparse features. First we go - // through the dense features and then the sparse features. - const QuantileStream& stream = *streams_resource->stream(stamp_token); - const std::vector& boundaries = - are_buckets_ready ? streams_resource->boundaries(stamp_token) - : std::vector(); - protobuf::Arena arena; - ::boosted_trees::QuantileStreamState* stream_proto = - protobuf::Arena::CreateMessage<::boosted_trees::QuantileStreamState>( - &arena); - for (const auto& summary : stream.SerializeInternalSummaries()) { - CopySummaryToProto(summary, stream_proto->add_summaries()); - } - SerializeToTString(*stream_proto, &stream_state_t->scalar()()); - Tensor* buckets_t = nullptr; - OP_REQUIRES_OK( - context, - context->allocate_output( - kBucketsName, {static_cast(boundaries.size())}, &buckets_t)); - auto* quantiles_flat = buckets_t->flat().data(); - memcpy(quantiles_flat, boundaries.data(), - sizeof(float) * boundaries.size()); - Tensor* stamp_token_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(kStampTokenName, TensorShape({}), - &stamp_token_t)); - stamp_token_t->scalar()() = stamp_token; - Tensor* are_buckets_ready_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output(kAreBucketsReadyName, {}, - &are_buckets_ready_t)); - are_buckets_ready_t->scalar()() = are_buckets_ready; - } -}; - -REGISTER_KERNEL_BUILDER(Name("QuantileAccumulatorSerialize").Device(DEVICE_CPU), - QuantileAccumulatorSerializeOp); - -// Serializes the state of streams. -class QuantileAccumulatorDeserializeOp : public OpKernel { - public: - explicit QuantileAccumulatorDeserializeOp(OpKernelConstruction* const context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr streams_resource; - // Create a reference to the underlying resource using the handle. - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &streams_resource)); - // Remove the reference at the end of this scope. - mutex_lock l(*streams_resource->mutex()); - - int64 old_stamp_token = streams_resource->stamp(); - - const Tensor* stream_state_t; - OP_REQUIRES_OK(context, context->input(kStreamStateName, &stream_state_t)); - const Tensor* buckets_t; - OP_REQUIRES_OK(context, context->input(kBucketsName, &buckets_t)); - - QuantileStream* stream = streams_resource->stream(old_stamp_token); - ::boosted_trees::QuantileStreamState state_proto; - OP_REQUIRES( - context, - ParseProtoUnlimited(&state_proto, stream_state_t->scalar()()), - errors::InvalidArgument("Unabnle to parse quantile stream state.")); - std::vector summaries; - summaries.reserve(state_proto.summaries_size()); - std::vector entries; - for (const auto& summary : state_proto.summaries()) { - entries.clear(); - entries.reserve(summary.entries_size()); - for (const auto& entry : summary.entries()) { - entries.emplace_back(entry.value(), entry.weight(), entry.min_rank(), - entry.max_rank()); - } - summaries.emplace_back(); - summaries[summaries.size() - 1].BuildFromSummaryEntries(entries); - } - stream->DeserializeInternalSummaries(summaries); - - const auto& buckets = buckets_t->vec(); - std::vector result; - result.reserve(buckets.size()); - - for (size_t i = 0; i < buckets.size(); ++i) { - result.push_back(buckets(i)); - } - streams_resource->set_boundaries(old_stamp_token, result); - - // Reset the stamp token. - const Tensor* stamp_token_t = nullptr; - OP_REQUIRES_OK(context, context->input(kStampTokenName, &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - streams_resource->set_stamp(stamp_token); - - const Tensor* are_buckets_ready_t = nullptr; - OP_REQUIRES_OK(context, - context->input(kAreBucketsReadyName, &are_buckets_ready_t)); - streams_resource->set_buckets_ready(are_buckets_ready_t->scalar()()); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("QuantileAccumulatorDeserialize").Device(DEVICE_CPU), - QuantileAccumulatorDeserializeOp); - -// Flushes the quantile summary stream resource. -class QuantileAccumulatorFlushOp : public OpKernel { - public: - explicit QuantileAccumulatorFlushOp(OpKernelConstruction* const context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr streams_resource; - // Create a reference to the underlying resource using the handle. - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &streams_resource)); - // Remove the reference at the end of this scope. - mutex_lock l(*streams_resource->mutex()); - - const Tensor* next_stamp_token_t; - OP_REQUIRES_OK(context, - context->input(kNextStampTokenName, &next_stamp_token_t)); - int64 next_stamp_token = next_stamp_token_t->scalar()(); - - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input(kStampTokenName, &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - CHECK(streams_resource->is_stamp_valid(stamp_token)) - << "Invalid stamp token in QuantileAccumulatorFlushOp. " - << "Passed stamp token: " << stamp_token << " " - << "Current token: " << streams_resource->stamp(); - QuantileStream* stream = streams_resource->stream(stamp_token); - bool generate_quantiles = streams_resource->generate_quantiles(); - stream->Finalize(); - - streams_resource->set_boundaries( - stamp_token, - generate_quantiles - ? GenerateQuantiles(*stream, streams_resource->num_quantiles()) - : GenerateBoundaries(*stream, streams_resource->num_quantiles())); - - streams_resource->Reset(next_stamp_token); - } -}; - -REGISTER_KERNEL_BUILDER(Name("QuantileAccumulatorFlush").Device(DEVICE_CPU), - QuantileAccumulatorFlushOp); - -// Flushes the quantile summary stream resource. This version computes the -// summary. -class QuantileAccumulatorFlushSummaryOp : public OpKernel { - public: - explicit QuantileAccumulatorFlushSummaryOp( - OpKernelConstruction* const context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr streams_resource; - // Create a reference to the underlying resource using the handle. - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &streams_resource)); - // Remove the reference at the end of this scope. - mutex_lock l(*streams_resource->mutex()); - - const Tensor* next_stamp_token_t; - OP_REQUIRES_OK(context, - context->input(kNextStampTokenName, &next_stamp_token_t)); - int64 next_stamp_token = next_stamp_token_t->scalar()(); - - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input(kStampTokenName, &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - CHECK(streams_resource->is_stamp_valid(stamp_token)) - << "Invalid stamp token in QuantileAccumulatorFlushSummaryOp. " - << "Passed stamp token: " << stamp_token << " " - << "Current token: " << streams_resource->stamp(); - QuantileStream* stream = streams_resource->stream(stamp_token); - stream->Finalize(); - protobuf::Arena arena; - ::boosted_trees::QuantileSummaryState* summary_proto = - protobuf::Arena::CreateMessage<::boosted_trees::QuantileSummaryState>( - &arena); - const auto& summary = stream->GetFinalSummary(); - CopySummaryToProto(summary, summary_proto); - // Output to tensor. - Tensor* output_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(0, TensorShape({}), &output_t)); - SerializeToTString(*summary_proto, &output_t->scalar()()); - streams_resource->Reset(next_stamp_token); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("QuantileAccumulatorFlushSummary").Device(DEVICE_CPU), - QuantileAccumulatorFlushSummaryOp); - -// Get bucket boundaries from summaries. -class QuantileAccumulatorGetBucketsOp : public OpKernel { - public: - explicit QuantileAccumulatorGetBucketsOp(OpKernelConstruction* const context) - : OpKernel(context) {} - - void Compute(OpKernelContext* const context) override { - OpInputList resource_handle_list; - OP_REQUIRES_OK(context, context->input_list(kResourceHandlesName, - &resource_handle_list)); - OpOutputList are_buckets_ready_list; - OP_REQUIRES_OK(context, context->output_list(kAreBucketsReadyName, - &are_buckets_ready_list)); - OpOutputList buckets_list; - OP_REQUIRES_OK(context, context->output_list(kBucketsName, &buckets_list)); - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input(kStampTokenName, &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - - thread::ThreadPool* const worker_threads = - context->device()->tensorflow_cpu_worker_threads()->workers; - boosted_trees::utils::ParallelFor( - resource_handle_list.size(), worker_threads->NumThreads(), - worker_threads, - [&context, &resource_handle_list, &are_buckets_ready_list, - &buckets_list, stamp_token](int64 start, int64 end) { - for (int resource_handle_idx = start; resource_handle_idx < end; - ++resource_handle_idx) { - const ResourceHandle& handle = - resource_handle_list[resource_handle_idx] - .flat()(0); - core::RefCountPtr streams_resource; - OP_REQUIRES_OK(context, - LookupResource(context, handle, &streams_resource)); - // Remove the reference at the end of this scope. - mutex_lock l(*streams_resource->mutex()); - - bool are_buckets_ready = - streams_resource->is_stamp_valid(stamp_token) && - streams_resource->are_buckets_ready(); - - Tensor* are_buckets_ready_t = nullptr; - OP_REQUIRES_OK(context, - are_buckets_ready_list.allocate( - resource_handle_idx, {}, &are_buckets_ready_t)); - are_buckets_ready_t->scalar()() = are_buckets_ready; - - const std::vector& boundaries = - are_buckets_ready ? streams_resource->boundaries(stamp_token) - : std::vector(); - Tensor* output_t = nullptr; - OP_REQUIRES_OK(context, buckets_list.allocate( - resource_handle_idx, - {static_cast(boundaries.size())}, - &output_t)); - auto* quantiles_flat = output_t->flat().data(); - memcpy(quantiles_flat, boundaries.data(), - sizeof(float) * boundaries.size()); - } - }); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("QuantileAccumulatorGetBuckets").Device(DEVICE_CPU), - QuantileAccumulatorGetBucketsOp); - -// Generates buckets for given set of float values, and the given config. -class QuantileBucketsOp : public OpKernel { - public: - explicit QuantileBucketsOp(OpKernelConstruction* const context) - : OpKernel(context) { - OP_REQUIRES_OK(context, - ReadAndValidateAttributes(context, &num_dense_features_, - &num_sparse_features_)); - - ParseConfig(context, kDenseConfigName, &dense_configs_); - OP_REQUIRES(context, dense_configs_.size() == num_dense_features_, - errors::InvalidArgument( - "Mismatch in number of dense quantile configs.")); - ParseConfig(context, kSparseConfigName, &sparse_configs_); - OP_REQUIRES(context, sparse_configs_.size() == num_sparse_features_, - errors::InvalidArgument( - "Mismatch in number of sparse quantile configs.")); - } - - void Compute(OpKernelContext* const context) override { - // Read dense float features list; - OpInputList dense_float_features_list; - OP_REQUIRES_OK(context, TensorUtils::ReadDenseFloatFeatures( - context, &dense_float_features_list)); - - // Read sparse float features list; - OpInputList sparse_float_feature_indices_list; - OpInputList sparse_float_feature_values_list; - OpInputList sparse_float_feature_shapes_list; - OP_REQUIRES_OK(context, TensorUtils::ReadSparseFloatFeatures( - context, &sparse_float_feature_indices_list, - &sparse_float_feature_values_list, - &sparse_float_feature_shapes_list)); - - // Parse example weights and get batch size. - const Tensor* example_weights_t; - OP_REQUIRES_OK(context, - context->input(kExampleWeightsName, &example_weights_t)); - auto example_weights = example_weights_t->flat(); - const int64 batch_size = example_weights.size(); - - OpOutputList sparse_buckets_output_list; - OP_REQUIRES_OK(context, context->output_list(kSparseBucketsName, - &sparse_buckets_output_list)); - OpOutputList dense_buckets_output_list; - OP_REQUIRES_OK(context, context->output_list(kDenseBucketsName, - &dense_buckets_output_list)); - - auto do_quantile_bucket_gen = [&](const int64 begin, const int64 end) { - // These are blocks of ranges. We are iterating over both sparse and - // dense features i.e. [0, sparse_features.size() + dense_features.size()] - for (int64 i = begin; i < end; ++i) { - if (i < sparse_configs_.size()) { - const int64 sparse_index = i; - const auto sparse_values = - sparse_float_feature_values_list[sparse_index].flat(); - const auto sparse_indices = - sparse_float_feature_indices_list[sparse_index].matrix(); - QuantileStream stream(sparse_configs_[sparse_index].eps(), - batch_size); - // Run quantile summary generation. - const int64 num_sparse_rows = - sparse_float_feature_indices_list[sparse_index].dim_size(0); - for (int64 j = 0; j < num_sparse_rows; ++j) { - const int64 example_id = sparse_indices(j, 0); - stream.PushEntry(sparse_values(j), example_weights(example_id)); - } - stream.Finalize(); - // Create buckets. - const auto boundaries = GenerateBoundaries( - stream, sparse_configs_[sparse_index].num_quantiles()); - CopyBoundaries(context, boundaries, sparse_index, - &sparse_buckets_output_list); - - } else { - const int64 dense_index = i - sparse_configs_.size(); - const auto dense_values = - dense_float_features_list[dense_index].flat(); - QuantileStream stream(dense_configs_[dense_index].eps(), batch_size); - // Run quantile summary generation. - for (int64 j = 0; j < batch_size; ++j) { - stream.PushEntry(dense_values(j), example_weights(j)); - } - stream.Finalize(); - // Create buckets. - const auto boundaries = GenerateBoundaries( - stream, dense_configs_[dense_index].num_quantiles()); - CopyBoundaries(context, boundaries, dense_index, - &dense_buckets_output_list); - } - } - }; - - const int64 kCostPerUnit = 500 * batch_size; - const int64 num_features = sparse_configs_.size() + dense_configs_.size(); - const DeviceBase::CpuWorkerThreads& worker_threads = - *context->device()->tensorflow_cpu_worker_threads(); - Shard(worker_threads.num_threads, worker_threads.workers, num_features, - kCostPerUnit, do_quantile_bucket_gen); - } - - private: - int num_dense_features_; - int num_sparse_features_; - std::vector dense_configs_; - std::vector sparse_configs_; -}; - -REGISTER_KERNEL_BUILDER(Name("QuantileBuckets").Device(DEVICE_CPU), - QuantileBucketsOp); - -// Given the calculated quantiles thresholds and input data, this operation -// converts the input features into the buckets (categorical values), depending -// on which quantile they fall into. -class QuantilesOp : public OpKernel { - public: - explicit QuantilesOp(OpKernelConstruction* const context) - : OpKernel(context) { - int num_dense_features; - int num_sparse_features; - OP_REQUIRES_OK(context, - ReadAndValidateAttributes(context, &num_dense_features, - &num_sparse_features)); - } - - void Compute(OpKernelContext* const context) override { - // Dense features inputs - OpInputList dense_float_features_list; - OP_REQUIRES_OK(context, context->input_list(kDenseValuesName, - &dense_float_features_list)); - OpInputList dense_buckets_list; - OP_REQUIRES_OK(context, - context->input_list(kDenseBucketsName, &dense_buckets_list)); - - if (dense_buckets_list.size() > 0) { - // Check the first tensor to make sure it is the right shape - OP_REQUIRES( - context, - tensorflow::TensorShapeUtils::IsVector(dense_buckets_list[0].shape()), - errors::InvalidArgument( - strings::Printf("Dense buckets should be flat vectors"))); - } - - // Sparse features inputs - OpInputList sparse_float_feature_values_list; - OP_REQUIRES_OK(context, - context->input_list(kSparseValuesName, - &sparse_float_feature_values_list)); - - OpInputList sparse_float_indices_list; - OP_REQUIRES_OK(context, context->input_list(kSparseIndicesName, - &sparse_float_indices_list)); - - OpInputList sparse_buckets_list; - OP_REQUIRES_OK( - context, context->input_list(kSparseBucketsName, &sparse_buckets_list)); - - if (sparse_buckets_list.size() > 0) { - OP_REQUIRES( - context, - tensorflow::TensorShapeUtils::IsVector( - sparse_buckets_list[0].shape()), - errors::InvalidArgument("Sparse buckets should be flat vectors")); - } - - // Quantize the feature values - QuantizeFeatures(kDenseOutputTensorName, dense_float_features_list, - dense_buckets_list, nullptr, context); - - QuantizeFeatures(kSparseOutputTensorName, sparse_float_feature_values_list, - sparse_buckets_list, &sparse_float_indices_list, context); - } -}; - -REGISTER_KERNEL_BUILDER(Name("Quantiles").Device(DEVICE_CPU), QuantilesOp); - -template -class BucketizeWithInputBoundariesOp : public OpKernel { - public: - explicit BucketizeWithInputBoundariesOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - const Tensor& boundaries_tensor = context->input(1); - VLOG(1) << "boundaries has shape: " - << boundaries_tensor.shape().DebugString(); - auto boundaries = boundaries_tensor.flat(); - std::vector boundaries_vector; - boundaries_vector.reserve(boundaries.size()); - for (size_t i = 0; i < boundaries.size(); i++) { - boundaries_vector.push_back(boundaries(i)); - VLOG(1) << "boundaries(" << i << ") : " << boundaries(i); - } - OP_REQUIRES( - context, - std::is_sorted(boundaries_vector.begin(), boundaries_vector.end()), - errors::InvalidArgument("Expected sorted boundaries")); - - const Tensor& input_tensor = context->input(0); - VLOG(1) << "Inputs has shape: " << input_tensor.shape().DebugString() - << " Dtype: " << tensorflow::DataTypeString(input_tensor.dtype()); - auto input = input_tensor.flat(); - - Tensor* output_tensor = nullptr; - OP_REQUIRES_OK(context, context->allocate_output(0, input_tensor.shape(), - &output_tensor)); - auto output = output_tensor->template flat(); - - for (size_t i = 0; i < input.size(); i++) { - output(i) = CalculateBucketIndex(input(i), boundaries_vector); - } - } - - private: - int32 CalculateBucketIndex(const T value, std::vector& boundaries_vector) { - auto first_bigger_it = std::upper_bound(boundaries_vector.begin(), - boundaries_vector.end(), value); - int32 index = first_bigger_it - boundaries_vector.begin(); - CHECK(index >= 0 && index <= boundaries_vector.size()) - << "Invalid bucket index: " << index - << " boundaries_vector.size(): " << boundaries_vector.size(); - return index; - } -}; - -#define REGISTER_KERNEL(T) \ - REGISTER_KERNEL_BUILDER(Name("BucketizeWithInputBoundaries") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T"), \ - BucketizeWithInputBoundariesOp); - -REGISTER_KERNEL(int32); -REGISTER_KERNEL(int64); -REGISTER_KERNEL(float); -REGISTER_KERNEL(double); -#undef REGISTER_KERNEL - -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/kernels/split_handler_ops.cc b/tensorflow/contrib/boosted_trees/kernels/split_handler_ops.cc deleted file mode 100644 index 0afab357414..00000000000 --- a/tensorflow/contrib/boosted_trees/kernels/split_handler_ops.cc +++ /dev/null @@ -1,1003 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include -#include -#include -#include - -#include "tensorflow/contrib/boosted_trees/lib/learner/common/stats/node-stats.h" -#include "tensorflow/contrib/boosted_trees/proto/split_info.pb.h" -#include "tensorflow/contrib/boosted_trees/proto/tree_config.pb.h" -#include "tensorflow/core/framework/device_base.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/random/random.h" -#include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/protobuf.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { - -using boosted_trees::learner::LearnerConfig; -using boosted_trees::learner::LearnerConfig_MultiClassStrategy; -using boosted_trees::learner::ObliviousSplitInfo; -using boosted_trees::learner::SplitInfo; -using boosted_trees::learner::stochastic::GradientStats; -using boosted_trees::learner::stochastic::NodeStats; - -namespace { -const int32 DUMMY_FEATURE_DIMENSION = -1; -} // namespace - -class SplitBuilderState { - public: - explicit SplitBuilderState(OpKernelContext* const context) { - const Tensor* l1_regularization_t; - OP_REQUIRES_OK(context, - context->input("l1_regularization", &l1_regularization_t)); - const Tensor* l2_regularization_t; - OP_REQUIRES_OK(context, - context->input("l2_regularization", &l2_regularization_t)); - const Tensor* tree_complexity_regularization_t; - OP_REQUIRES_OK(context, context->input("tree_complexity_regularization", - &tree_complexity_regularization_t)); - const Tensor* min_node_weight_t; - OP_REQUIRES_OK(context, - context->input("min_node_weight", &min_node_weight_t)); - - const Tensor* feature_column_group_id_t; - OP_REQUIRES_OK(context, context->input("feature_column_group_id", - &feature_column_group_id_t)); - - const Tensor* multiclass_strategy_t; - OP_REQUIRES_OK( - context, context->input("multiclass_strategy", &multiclass_strategy_t)); - int strategy = multiclass_strategy_t->scalar()(); - OP_REQUIRES( - context, - boosted_trees::learner::LearnerConfig_MultiClassStrategy_IsValid( - strategy), - errors::InvalidArgument("Wrong multiclass strategy passed.")); - - multiclass_strategy_ = LearnerConfig_MultiClassStrategy(strategy); - - const Tensor* class_id_t; - OP_REQUIRES_OK(context, context->input("class_id", &class_id_t)); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(class_id_t->shape()), - errors::InvalidArgument("class_id must be a scalar.")); - class_id_ = class_id_t->scalar()(); - - l1_regularization_ = l1_regularization_t->scalar()(); - l2_regularization_ = l2_regularization_t->scalar()(); - tree_complexity_regularization_ = - tree_complexity_regularization_t->scalar()(); - min_node_weight_ = min_node_weight_t->scalar()(); - feature_column_group_id_ = feature_column_group_id_t->scalar()(); - } - - NodeStats ComputeNodeStats(const GradientStats& grad_stats) { - return NodeStats(l1_regularization_, l2_regularization_, min_node_weight_, - multiclass_strategy_, grad_stats); - } - - void FillLeaf(const NodeStats& best_node_stats, - boosted_trees::trees::Leaf* leaf) const { - if (class_id_ == -1) { - // This would be the case either for TREE_PER_CLASS with only 2 classes, - // or for other multiclass strategies. - for (float f : best_node_stats.weight_contribution) { - leaf->mutable_vector()->add_value(f); - } - } else { - CHECK(best_node_stats.weight_contribution.size() == 1) - << "Weight contribution size = " - << best_node_stats.weight_contribution.size(); - leaf->mutable_sparse_vector()->add_index(class_id_); - leaf->mutable_sparse_vector()->add_value( - best_node_stats.weight_contribution[0]); - } - } - - int32 feature_column_group_id() { return feature_column_group_id_; } - float tree_complexity_regularization() { - return tree_complexity_regularization_; - } - - private: - LearnerConfig_MultiClassStrategy multiclass_strategy_; - float l1_regularization_; - float l2_regularization_; - float tree_complexity_regularization_; - float min_node_weight_; - int32 class_id_; - int32 feature_column_group_id_; -}; - -class BuildDenseInequalitySplitsOp : public OpKernel { - public: - explicit BuildDenseInequalitySplitsOp(OpKernelConstruction* const context) - : OpKernel(context) {} - - void Compute(OpKernelContext* const context) override { - const Tensor* num_minibatches_t; - OP_REQUIRES_OK(context, - context->input("num_minibatches", &num_minibatches_t)); - const int64 num_minibatches = num_minibatches_t->scalar()(); - const float normalizer_ratio = (1.0f / num_minibatches); - - const Tensor* bucket_boundaries_t; - OP_REQUIRES_OK(context, - context->input("bucket_boundaries", &bucket_boundaries_t)); - const auto& bucket_boundaries = bucket_boundaries_t->vec(); - - const Tensor* partition_ids_t; - OP_REQUIRES_OK(context, context->input("partition_ids", &partition_ids_t)); - const auto& partition_ids = partition_ids_t->vec(); - - const Tensor* bucket_ids_t; - OP_REQUIRES_OK(context, context->input("bucket_ids", &bucket_ids_t)); - const auto& bucket_ids = bucket_ids_t->matrix(); - - const Tensor* gradients_t; - OP_REQUIRES_OK(context, context->input("gradients", &gradients_t)); - - const Tensor* hessians_t; - OP_REQUIRES_OK(context, context->input("hessians", &hessians_t)); - - const Tensor* weak_learner_type_t; - OP_REQUIRES_OK(context, - context->input("weak_learner_type", &weak_learner_type_t)); - const int32 weak_learner_type = weak_learner_type_t->scalar()(); - - // Find the number of unique partitions before we allocate the output. - std::vector partition_boundaries; - partition_boundaries.push_back(0); - for (int i = 1; i < partition_ids.size(); ++i) { - if (partition_ids(i) != partition_ids(i - 1)) { - // Make sure the input is sorted by partition_ids; - OP_REQUIRES(context, partition_ids(i) >= partition_ids(i - 1), - errors::InvalidArgument("Input should be sorted.")); - partition_boundaries.push_back(i); - } - } - if (partition_ids.size() > 0) { - partition_boundaries.push_back(partition_ids.size()); - } - int32 num_elements = partition_boundaries.size() - 1; - - // When the handler is inactive, no bucket boundaries are built for it. - if (bucket_boundaries.size() == 0) { - num_elements = 0; - } - - Tensor* output_partition_ids_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("output_partition_ids", - TensorShape({num_elements}), - &output_partition_ids_t)); - - tensorflow::TTypes::Vec output_partition_ids = - output_partition_ids_t->vec(); - - // For a normal tree, we output a split per partition. For an oblivious - // tree, we output one split for all partitions of the layer - int32 size_output = num_elements; - if (weak_learner_type == LearnerConfig::OBLIVIOUS_DECISION_TREE && - num_elements > 0) { - size_output = 1; - } - - Tensor* gains_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output( - "gains", TensorShape({size_output}), &gains_t)); - tensorflow::TTypes::Vec gains = gains_t->vec(); - - Tensor* output_splits_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output("split_infos", - TensorShape({size_output}), - &output_splits_t)); - tensorflow::TTypes::Vec output_splits = - output_splits_t->vec(); - - if (num_elements == 0) { - return; - } - SplitBuilderState state(context); - switch (weak_learner_type) { - case LearnerConfig::NORMAL_DECISION_TREE: { - ComputeNormalDecisionTree( - &state, normalizer_ratio, num_elements, partition_boundaries, - bucket_boundaries, partition_ids, bucket_ids, gradients_t, - hessians_t, &output_partition_ids, &gains, &output_splits); - break; - } - case LearnerConfig::OBLIVIOUS_DECISION_TREE: { - ComputeObliviousDecisionTree( - &state, normalizer_ratio, num_elements, partition_boundaries, - bucket_boundaries, partition_ids, bucket_ids, gradients_t, - hessians_t, &output_partition_ids, &gains, &output_splits); - break; - } - } - } - - private: - void ComputeNormalDecisionTree( - SplitBuilderState* state, const float normalizer_ratio, - const int num_elements, const std::vector& partition_boundaries, - const tensorflow::TTypes::ConstVec& bucket_boundaries, - const tensorflow::TTypes::ConstVec& partition_ids, - const tensorflow::TTypes::ConstMatrix& bucket_ids, - const Tensor* gradients_t, const Tensor* hessians_t, - tensorflow::TTypes::Vec* output_partition_ids, - tensorflow::TTypes::Vec* gains, - tensorflow::TTypes::Vec* output_splits) { - for (int root_idx = 0; root_idx < num_elements; ++root_idx) { - float best_gain = std::numeric_limits::lowest(); - int start_index = partition_boundaries[root_idx]; - int end_index = partition_boundaries[root_idx + 1]; - GradientStats root_gradient_stats; - for (int64 bucket_idx = start_index; bucket_idx < end_index; - ++bucket_idx) { - root_gradient_stats += - GradientStats(*gradients_t, *hessians_t, bucket_idx); - } - root_gradient_stats *= normalizer_ratio; - NodeStats root_stats = state->ComputeNodeStats(root_gradient_stats); - int32 best_bucket_idx = 0; - NodeStats best_right_node_stats(0); - NodeStats best_left_node_stats(0); - GradientStats left_gradient_stats; - for (int64 bucket_idx = start_index; bucket_idx < end_index; - ++bucket_idx) { - GradientStats g(*gradients_t, *hessians_t, bucket_idx); - g *= normalizer_ratio; - left_gradient_stats += g; - NodeStats left_stats = state->ComputeNodeStats(left_gradient_stats); - GradientStats right_gradient_stats = - root_gradient_stats - left_gradient_stats; - NodeStats right_stats = state->ComputeNodeStats(right_gradient_stats); - if (left_stats.gain + right_stats.gain > best_gain) { - best_gain = left_stats.gain + right_stats.gain; - best_left_node_stats = left_stats; - best_right_node_stats = right_stats; - best_bucket_idx = bucket_idx; - } - } - SplitInfo split_info; - auto* dense_split = - split_info.mutable_split_node()->mutable_dense_float_binary_split(); - dense_split->set_feature_column(state->feature_column_group_id()); - dense_split->set_threshold( - bucket_boundaries(bucket_ids(best_bucket_idx, 0))); - - auto* left_child = split_info.mutable_left_child(); - auto* right_child = split_info.mutable_right_child(); - - state->FillLeaf(best_left_node_stats, left_child); - state->FillLeaf(best_right_node_stats, right_child); - SerializeToTString(split_info, &(*output_splits)(root_idx)); - (*gains)(root_idx) = - best_gain - root_stats.gain - state->tree_complexity_regularization(); - (*output_partition_ids)(root_idx) = partition_ids(start_index); - } - } - void ComputeObliviousDecisionTree( - SplitBuilderState* state, const float normalizer_ratio, - const int num_elements, const std::vector& partition_boundaries, - const tensorflow::TTypes::ConstVec& bucket_boundaries, - const tensorflow::TTypes::ConstVec& partition_ids, - const tensorflow::TTypes::ConstMatrix& bucket_ids, - const Tensor* gradients_t, const Tensor* hessians_t, - tensorflow::TTypes::Vec* output_partition_ids, - tensorflow::TTypes::Vec* gains, - tensorflow::TTypes::Vec* output_splits) { - // Holds the root stats per each node to be split. - std::vector current_layer_stats; - current_layer_stats.reserve(num_elements); - for (int root_idx = 0; root_idx < num_elements; root_idx++) { - const int start_index = partition_boundaries[root_idx]; - const int end_index = partition_boundaries[root_idx + 1]; - GradientStats root_gradient_stats; - for (int64 bucket_idx = start_index; bucket_idx < end_index; - ++bucket_idx) { - root_gradient_stats += - GradientStats(*gradients_t, *hessians_t, bucket_idx); - } - root_gradient_stats *= normalizer_ratio; - current_layer_stats.push_back(root_gradient_stats); - } - - float best_gain = std::numeric_limits::lowest(); - int64 best_bucket_id = 0; - std::vector best_right_node_stats(num_elements, NodeStats(0)); - std::vector best_left_node_stats(num_elements, NodeStats(0)); - std::vector current_left_node_stats(num_elements, NodeStats(0)); - std::vector current_right_node_stats(num_elements, NodeStats(0)); - int64 current_bucket_id = std::numeric_limits::max(); - int64 last_bucket_id = -1; - // Find the lowest bucket id, this is going to be the first bucket id to - // try. - for (int root_idx = 0; root_idx < num_elements; root_idx++) { - const int start_index = partition_boundaries[root_idx]; - if (bucket_ids(start_index, 0) < current_bucket_id) { - current_bucket_id = bucket_ids(start_index, 0); - } - } - // Indexes offsets for each of the partitions that can be used to access - // gradients of a partition for a current bucket we consider. - std::vector current_layer_offsets(num_elements, 0); - std::vector left_gradient_stats(num_elements); - // The idea is to try every bucket id in increasing order. In each iteration - // we calculate the gain of the layer using the current bucket id as split - // value, and we also obtain the following bucket id to try. - while (current_bucket_id > last_bucket_id) { - last_bucket_id = current_bucket_id; - int64 next_bucket_id = -1; - for (int root_idx = 0; root_idx < num_elements; root_idx++) { - int idx = - current_layer_offsets[root_idx] + partition_boundaries[root_idx]; - const int end_index = partition_boundaries[root_idx + 1]; - if (idx < end_index && bucket_ids(idx, 0) == current_bucket_id) { - GradientStats g(*gradients_t, *hessians_t, idx); - g *= normalizer_ratio; - left_gradient_stats[root_idx] += g; - current_layer_offsets[root_idx]++; - idx++; - } - if (idx < end_index && - (bucket_ids(idx, 0) < next_bucket_id || next_bucket_id == -1)) { - next_bucket_id = bucket_ids(idx, 0); - } - } - float gain_of_split = 0.0; - for (int root_idx = 0; root_idx < num_elements; root_idx++) { - GradientStats right_gradient_stats = - current_layer_stats[root_idx] - left_gradient_stats[root_idx]; - NodeStats left_stat = - state->ComputeNodeStats(left_gradient_stats[root_idx]); - NodeStats right_stat = state->ComputeNodeStats(right_gradient_stats); - gain_of_split += left_stat.gain + right_stat.gain; - current_left_node_stats[root_idx] = left_stat; - current_right_node_stats[root_idx] = right_stat; - } - if (gain_of_split > best_gain) { - best_gain = gain_of_split; - best_left_node_stats = current_left_node_stats; - best_right_node_stats = current_right_node_stats; - best_bucket_id = current_bucket_id; - } - current_bucket_id = next_bucket_id; - } - - for (int root_idx = 0; root_idx < num_elements; root_idx++) { - best_gain -= state->ComputeNodeStats(current_layer_stats[root_idx]).gain; - } - best_gain -= num_elements * state->tree_complexity_regularization(); - - ObliviousSplitInfo oblivious_split_info; - auto* oblivious_dense_split = - oblivious_split_info.mutable_split_node() - ->mutable_oblivious_dense_float_binary_split(); - oblivious_dense_split->set_feature_column(state->feature_column_group_id()); - oblivious_dense_split->set_threshold(bucket_boundaries(best_bucket_id)); - (*gains)(0) = best_gain; - - for (int root_idx = 0; root_idx < num_elements; root_idx++) { - auto* left_child = oblivious_split_info.add_children(); - auto* right_child = oblivious_split_info.add_children(); - - state->FillLeaf(best_left_node_stats[root_idx], left_child); - state->FillLeaf(best_right_node_stats[root_idx], right_child); - - const int start_index = partition_boundaries[root_idx]; - (*output_partition_ids)(root_idx) = partition_ids(start_index); - oblivious_split_info.add_children_parent_id(partition_ids(start_index)); - } - SerializeToTString(oblivious_split_info, &(*output_splits)(0)); - } -}; -REGISTER_KERNEL_BUILDER(Name("BuildDenseInequalitySplits").Device(DEVICE_CPU), - BuildDenseInequalitySplitsOp); - -class BuildSparseInequalitySplitsOp : public OpKernel { - public: - explicit BuildSparseInequalitySplitsOp(OpKernelConstruction* const context) - : OpKernel(context) {} - - void Compute(OpKernelContext* const context) override { - const Tensor* num_minibatches_t; - OP_REQUIRES_OK(context, - context->input("num_minibatches", &num_minibatches_t)); - const int64 num_minibatches = num_minibatches_t->scalar()(); - const float normalizer_ratio = (1.0f / num_minibatches); - - const Tensor* bucket_boundaries_t; - OP_REQUIRES_OK(context, - context->input("bucket_boundaries", &bucket_boundaries_t)); - const auto& bucket_boundaries = bucket_boundaries_t->vec(); - - const Tensor* partition_ids_t; - OP_REQUIRES_OK(context, context->input("partition_ids", &partition_ids_t)); - const auto& partition_ids = partition_ids_t->vec(); - - const Tensor* bucket_ids_t; - OP_REQUIRES_OK(context, context->input("bucket_ids", &bucket_ids_t)); - const auto& bucket_ids_and_dimensions = bucket_ids_t->matrix(); - - const int32 tensor_elements = partition_ids.size(); - - const Tensor* gradients_t; - OP_REQUIRES_OK(context, context->input("gradients", &gradients_t)); - - const Tensor* hessians_t; - OP_REQUIRES_OK(context, context->input("hessians", &hessians_t)); - - const Tensor* bias_feature_id_t; - OP_REQUIRES_OK(context, - context->input("bias_feature_id", &bias_feature_id_t)); - int64 bias_feature_id = bias_feature_id_t->scalar()(); - - // For each partition (tree node), store starting index for each dimension. - PartitionAndDimensionBoundaries partition_boundaries; - // Stores indices in partition_boundaries for those partitions that are - // not empty (have at least one dimension and a bucket apart from catch-all - // bucket of -1 bucket id and dimension 0. - std::vector non_empty_partitions; - bool non_empty_partition = false; - - for (int i = 0; i < partition_ids.size(); ++i) { - // Make sure the input is sorted by partition_ids; - if (i > 0) { - CHECK_LE(partition_ids(i - 1), partition_ids(i)) - << "Partition ids should be sorted. Not sorted for " << i; - } - const int32 dimension = bucket_ids_and_dimensions(i, 1); - - if (i == 0 || (partition_ids(i) != partition_ids(i - 1))) { - if (i != 0) { - // Not the first entry, so partition has changed. - if (non_empty_partition) { - // Saves the id of a previous partition in a list of non empty - // partitions, since it was non empty (had more than just a bias - // bucket -1. - non_empty_partitions.push_back(partition_boundaries.size() - 1); - } - // Add dummy dimension to signify the end for the previous dimension. - partition_boundaries.back().emplace_back(DUMMY_FEATURE_DIMENSION, i); - } - // Allocate for a new partition. - partition_boundaries.emplace_back(); - // Save info about the first dimension for a new partition. - partition_boundaries.back().emplace_back(dimension, i); - - // Each partition has dummy -1 bucket with all gradients and then info - // for all other dimensions -> if we have >1 elements for a partition, - // then it is not empty. - non_empty_partition = (i < partition_ids.size() - 1) && - (partition_ids(i) == partition_ids(i + 1)); - } else if (bucket_ids_and_dimensions(i, 1) != - bucket_ids_and_dimensions(i - 1, 1)) { - // Dimension changed. - partition_boundaries.back().emplace_back(dimension, i); - } - } - if (tensor_elements > 0) { - if (non_empty_partition) { - non_empty_partitions.push_back(partition_boundaries.size() - 1); - } - // Add dummy dimension to signify the end for the previous dimension. - partition_boundaries.back().emplace_back(DUMMY_FEATURE_DIMENSION, - partition_ids.size()); - } - - int num_elements = non_empty_partitions.size(); - Tensor* output_partition_ids_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("output_partition_ids", - TensorShape({num_elements}), - &output_partition_ids_t)); - - tensorflow::TTypes::Vec output_partition_ids = - output_partition_ids_t->vec(); - - Tensor* gains_t = nullptr; - OP_REQUIRES_OK( - context, context->allocate_output("gains", TensorShape({num_elements}), - &gains_t)); - - tensorflow::TTypes::Vec gains = gains_t->vec(); - - Tensor* output_splits_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output( - "split_infos", TensorShape({num_elements}), - &output_splits_t)); - tensorflow::TTypes::Vec output_splits = - output_splits_t->vec(); - SplitBuilderState state(context); - // For each tree node that needs to be split. - for (int root_idx = 0; root_idx < num_elements; ++root_idx) { - const auto& dimension_boundaries = - partition_boundaries[non_empty_partitions[root_idx]]; - - float best_gain = std::numeric_limits::lowest(); - bool default_right = false; - int32 best_element_idx = 0; - - NodeStats best_right_node_stats(0); - NodeStats best_left_node_stats(0); - - // For each partition, the first bucket is dummy catch all. - int32 bias_start_index = dimension_boundaries[0].start_index; - - OP_REQUIRES( - context, - bucket_ids_and_dimensions(bias_start_index, 0) == bias_feature_id, - errors::InvalidArgument("Bias feature ID missing.")); - - // Dimension for bias feature is always 0 - OP_REQUIRES( - context, bucket_ids_and_dimensions(bias_start_index, 1) == 0, - errors::InvalidArgument("Bias feature ID must be with dimension 0.")); - - // For each root, we do two passes over the quantized feature buckets - // accumulating gradients on one side and using the root aggregate - // gradients to get the gradients for the other side. - // Split gains are evaluated for each pass at every threshold and the best - // split is picked. - GradientStats root_gradient_stats(*gradients_t, *hessians_t, - bias_start_index); - root_gradient_stats *= normalizer_ratio; - NodeStats root_stats = state.ComputeNodeStats(root_gradient_stats); - - // Iterate through dimensions. - for (int j = 0; j < dimension_boundaries.size() - 1; ++j) { - const DimensionBoundary& dimension_and_start = dimension_boundaries[j]; - - int start_index = dimension_and_start.start_index; - // Even for the last dimension, we always have additional dummy - // dimension that we can use to find the end index. - const int end_index = - partition_boundaries[non_empty_partitions[root_idx]][j + 1] - .start_index; - if (bucket_ids_and_dimensions(start_index, 0) == bias_feature_id) { - // 0-dimension case which has a first bucket for catch all feature. - CHECK(bucket_ids_and_dimensions(start_index, 1) == 0) - << "Dimension of bias feature should be 0"; - ++start_index; - } - - GradientStats present_gradient_stats; - for (int64 bucket_idx = start_index; bucket_idx < end_index; - ++bucket_idx) { - present_gradient_stats += - GradientStats(*gradients_t, *hessians_t, bucket_idx); - } - present_gradient_stats *= normalizer_ratio; - GradientStats not_present = - root_gradient_stats - present_gradient_stats; - // If there was (almost) no sparsity, fix the default direction to LEFT. - bool fixed_default_direction = not_present.IsAlmostZero(); - - GradientStats left_gradient_stats; - for (int64 element_idx = start_index; element_idx < end_index; - ++element_idx) { - // Check that bucket ids are sorted. - if (element_idx != start_index) { - CHECK(bucket_ids_and_dimensions(element_idx - 1, 0) < - bucket_ids_and_dimensions(element_idx, 0)) - << "Bucket ids must be sorted." - << ", problem on " << element_idx << " and dimension is " << j; - } - - GradientStats g(*gradients_t, *hessians_t, element_idx); - g *= normalizer_ratio; - left_gradient_stats += g; - // We have the sum of all present gradients. Use that to compute the - // backward pass gradients. - GradientStats right_gradient_stats = - present_gradient_stats - left_gradient_stats; - - { - NodeStats left_stats_default_left = state.ComputeNodeStats( - root_gradient_stats - right_gradient_stats); - NodeStats right_stats_default_left = - state.ComputeNodeStats(right_gradient_stats); - if (left_stats_default_left.gain + right_stats_default_left.gain > - best_gain) { - best_gain = - left_stats_default_left.gain + right_stats_default_left.gain; - best_left_node_stats = left_stats_default_left; - best_right_node_stats = right_stats_default_left; - best_element_idx = element_idx; - default_right = false; - } - } - // Consider calculating the default direction only when there were - // enough missing examples. - if (!fixed_default_direction) { - NodeStats left_stats_default_right = - state.ComputeNodeStats(left_gradient_stats); - NodeStats right_stats_default_right = state.ComputeNodeStats( - root_gradient_stats - left_gradient_stats); - if (left_stats_default_right.gain + right_stats_default_right.gain > - best_gain) { - best_gain = left_stats_default_right.gain + - right_stats_default_right.gain; - best_left_node_stats = left_stats_default_right; - best_right_node_stats = right_stats_default_right; - best_element_idx = element_idx; - default_right = true; - } - } - } - } - - SplitInfo split_info; - boosted_trees::trees::DenseFloatBinarySplit* dense_split = nullptr; - if (default_right) { - dense_split = split_info.mutable_split_node() - ->mutable_sparse_float_binary_split_default_right() - ->mutable_split(); - } else { - dense_split = split_info.mutable_split_node() - ->mutable_sparse_float_binary_split_default_left() - ->mutable_split(); - } - dense_split->set_feature_column(state.feature_column_group_id()); - // Set the feature index for the best feature column. - const int64 best_dimension_id = - bucket_ids_and_dimensions(best_element_idx, 1); - const int32 best_bucket_id = - bucket_ids_and_dimensions(best_element_idx, 0); - dense_split->set_dimension_id(best_dimension_id); - dense_split->set_threshold(bucket_boundaries(best_bucket_id)); - - auto* left_child = split_info.mutable_left_child(); - auto* right_child = split_info.mutable_right_child(); - state.FillLeaf(best_left_node_stats, left_child); - state.FillLeaf(best_right_node_stats, right_child); - SerializeToTString(split_info, &output_splits(root_idx)); - gains(root_idx) = - best_gain - root_stats.gain - state.tree_complexity_regularization(); - output_partition_ids(root_idx) = partition_ids(bias_start_index); - } - } - - private: - struct DimensionBoundary { - DimensionBoundary(const int32 dimension_id, const int32 start_index) - : dimension_id(dimension_id), start_index(start_index) {} - - int32 dimension_id; - int32 start_index; - }; - - // For each partition, store start indices of feature column dimensions. - typedef std::vector> - PartitionAndDimensionBoundaries; -}; -REGISTER_KERNEL_BUILDER(Name("BuildSparseInequalitySplits").Device(DEVICE_CPU), - BuildSparseInequalitySplitsOp); - -class BuildCategoricalEqualitySplitsOp : public OpKernel { - public: - explicit BuildCategoricalEqualitySplitsOp(OpKernelConstruction* const context) - : OpKernel(context) {} - - void Compute(OpKernelContext* const context) override { - const Tensor* num_minibatches_t; - OP_REQUIRES_OK(context, - context->input("num_minibatches", &num_minibatches_t)); - const int64 num_minibatches = num_minibatches_t->scalar()(); - const float normalizer_ratio = (1.0f / num_minibatches); - - const Tensor* partition_ids_t; - OP_REQUIRES_OK(context, context->input("partition_ids", &partition_ids_t)); - const auto& partition_ids = partition_ids_t->vec(); - - const Tensor* feature_ids_t; - OP_REQUIRES_OK(context, context->input("feature_ids", &feature_ids_t)); - const auto& feature_ids = feature_ids_t->matrix(); - - const Tensor* gradients_t; - OP_REQUIRES_OK(context, context->input("gradients", &gradients_t)); - - const Tensor* hessians_t; - OP_REQUIRES_OK(context, context->input("hessians", &hessians_t)); - - const Tensor* bias_feature_id_t; - OP_REQUIRES_OK(context, - context->input("bias_feature_id", &bias_feature_id_t)); - int64 bias_feature_id = bias_feature_id_t->scalar()(); - - const Tensor* weak_learner_type_t; - OP_REQUIRES_OK(context, - context->input("weak_learner_type", &weak_learner_type_t)); - const int32 weak_learner_type = weak_learner_type_t->scalar()(); - - // Find the number of unique partitions before we allocate the output. - std::vector partition_boundaries; - partition_boundaries.push_back(0); - for (int i = 1; i < partition_ids.size(); ++i) { - // Make sure the input is sorted by partition_ids; - OP_REQUIRES(context, partition_ids(i - 1) <= partition_ids(i), - errors::InvalidArgument("Partition IDs must be sorted.")); - if (partition_ids(i) != partition_ids(i - 1)) { - partition_boundaries.push_back(i); - } - } - std::vector non_empty_partitions; - partition_boundaries.push_back(partition_ids.size()); - for (int i = 0; i < partition_boundaries.size() - 1; ++i) { - // We want to ignore partitions with only the bias term. - if (partition_boundaries[i + 1] - partition_boundaries[i] >= 2) { - non_empty_partitions.push_back(i); - } - } - int num_elements = non_empty_partitions.size(); - Tensor* output_partition_ids_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("output_partition_ids", - TensorShape({num_elements}), - &output_partition_ids_t)); - - tensorflow::TTypes::Vec output_partition_ids = - output_partition_ids_t->vec(); - - // For a normal tree, we output a split per partition. For an oblivious - // tree, we output one split for all partitions of the layer. - int size_output = num_elements; - if (weak_learner_type == LearnerConfig::OBLIVIOUS_DECISION_TREE && - num_elements > 0) { - size_output = 1; - } - - Tensor* gains_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output( - "gains", TensorShape({size_output}), &gains_t)); - - tensorflow::TTypes::Vec gains = gains_t->vec(); - - Tensor* output_splits_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output("split_infos", - TensorShape({size_output}), - &output_splits_t)); - tensorflow::TTypes::Vec output_splits = - output_splits_t->vec(); - if (num_elements == 0) { - return; - } - SplitBuilderState state(context); - switch (weak_learner_type) { - case LearnerConfig::NORMAL_DECISION_TREE: { - ComputeNormalDecisionTree( - context, &state, normalizer_ratio, num_elements, - partition_boundaries, non_empty_partitions, bias_feature_id, - partition_ids, feature_ids, gradients_t, hessians_t, - &output_partition_ids, &gains, &output_splits); - break; - } - case LearnerConfig::OBLIVIOUS_DECISION_TREE: { - ComputeObliviousDecisionTree( - context, &state, normalizer_ratio, num_elements, - partition_boundaries, non_empty_partitions, bias_feature_id, - partition_ids, feature_ids, gradients_t, hessians_t, - &output_partition_ids, &gains, &output_splits); - break; - } - } - } - - private: - void ComputeNormalDecisionTree( - OpKernelContext* const context, SplitBuilderState* state, - const float normalizer_ratio, const int num_elements, - const std::vector& partition_boundaries, - const std::vector& non_empty_partitions, - const int64 bias_feature_id, - const tensorflow::TTypes::ConstVec& partition_ids, - const tensorflow::TTypes::ConstMatrix& feature_ids, - const Tensor* gradients_t, const Tensor* hessians_t, - tensorflow::TTypes::Vec* output_partition_ids, - tensorflow::TTypes::Vec* gains, - tensorflow::TTypes::Vec* output_splits) { - for (int root_idx = 0; root_idx < num_elements; ++root_idx) { - float best_gain = std::numeric_limits::lowest(); - int start_index = partition_boundaries[non_empty_partitions[root_idx]]; - int end_index = partition_boundaries[non_empty_partitions[root_idx] + 1]; - // First feature ID in each partition should be the bias feature. - OP_REQUIRES(context, feature_ids(start_index, 0) == bias_feature_id, - errors::InvalidArgument("Bias feature ID missing.")); - GradientStats root_gradient_stats(*gradients_t, *hessians_t, start_index); - root_gradient_stats *= normalizer_ratio; - NodeStats root_stats = state->ComputeNodeStats(root_gradient_stats); - int32 best_feature_idx = 0; - bool best_feature_updated = false; - NodeStats best_right_node_stats(0); - NodeStats best_left_node_stats(0); - CHECK(end_index - start_index >= 2) - << "Partition should have a non bias feature. Start index " - << start_index << " and end index " << end_index; - - for (int64 feature_idx = start_index + 1; feature_idx < end_index; - ++feature_idx) { - GradientStats left_gradient_stats(*gradients_t, *hessians_t, - feature_idx); - left_gradient_stats *= normalizer_ratio; - GradientStats right_gradient_stats = - root_gradient_stats - left_gradient_stats; - NodeStats left_stats = state->ComputeNodeStats(left_gradient_stats); - NodeStats right_stats = state->ComputeNodeStats(right_gradient_stats); - if (!best_feature_updated || - left_stats.gain + right_stats.gain > best_gain) { - best_gain = left_stats.gain + right_stats.gain; - best_left_node_stats = left_stats; - best_right_node_stats = right_stats; - best_feature_idx = feature_idx; - best_feature_updated = true; - } - } - SplitInfo split_info; - auto* equality_split = split_info.mutable_split_node() - ->mutable_categorical_id_binary_split(); - equality_split->set_feature_column(state->feature_column_group_id()); - CHECK(feature_ids(best_feature_idx, 0) != bias_feature_id) - << "Unexpected feature ID selected. " - << "Start feature ID: [" << start_index << "] " - << feature_ids(start_index, 0) << ", " << feature_ids(start_index, 1) - << "\nBest feature ID: [" << best_feature_idx << "] " - << feature_ids(best_feature_idx, 0) << ", " - << feature_ids(best_feature_idx, 1) - << "\nPartition IDS: " << partition_ids(start_index) << " " - << partition_ids(best_feature_idx) << " and best gain " << best_gain; - equality_split->set_feature_id(feature_ids(best_feature_idx, 0)); - auto* left_child = split_info.mutable_left_child(); - auto* right_child = split_info.mutable_right_child(); - state->FillLeaf(best_left_node_stats, left_child); - state->FillLeaf(best_right_node_stats, right_child); - SerializeToTString(split_info, &(*output_splits)(root_idx)); - (*gains)(root_idx) = - best_gain - root_stats.gain - state->tree_complexity_regularization(); - (*output_partition_ids)(root_idx) = partition_ids(start_index); - } - } - - void ComputeObliviousDecisionTree( - OpKernelContext* const context, SplitBuilderState* state, - const float normalizer_ratio, const int num_elements, - const std::vector& partition_boundaries, - const std::vector& non_empty_partitions, - const int64 bias_feature_id, - const tensorflow::TTypes::ConstVec& partition_ids, - const tensorflow::TTypes::ConstMatrix& feature_ids, - const Tensor* gradients_t, const Tensor* hessians_t, - tensorflow::TTypes::Vec* output_partition_ids, - tensorflow::TTypes::Vec* gains, - tensorflow::TTypes::Vec* output_splits) { - // Holds the root stats per each node to be split. - std::vector current_layer_stats; - current_layer_stats.reserve(num_elements); - for (int root_idx = 0; root_idx < num_elements; root_idx++) { - const int start_index = partition_boundaries[root_idx]; - // First feature ID in each partition should be the bias feature. - OP_REQUIRES(context, feature_ids(start_index, 0) == bias_feature_id, - errors::InvalidArgument("Bias feature ID missing.")); - GradientStats root_gradient_stats(*gradients_t, *hessians_t, start_index); - root_gradient_stats *= normalizer_ratio; - current_layer_stats.push_back(root_gradient_stats); - } - float best_gain = std::numeric_limits::lowest(); - int64 best_feature_id = 0; - std::vector best_right_node_stats(num_elements, NodeStats(0)); - std::vector best_left_node_stats(num_elements, NodeStats(0)); - std::vector current_left_node_stats(num_elements, NodeStats(0)); - std::vector current_right_node_stats(num_elements, NodeStats(0)); - int64 current_feature_id = std::numeric_limits::max(); - int64 last_feature_id = -1; - // Find the lowest feature id, this is going to be the first feature id to - // try. - for (int root_idx = 0; root_idx < num_elements; root_idx++) { - const int start_index = partition_boundaries[root_idx]; - if (feature_ids(start_index + 1, 0) < current_feature_id) { - current_feature_id = feature_ids(start_index + 1, 0); - } - } - // Indexes offsets for each of the partitions that can be used to access - // gradients of a partition for a current feature we consider. Start at one - // beacuse the zero index is for the bias. - std::vector current_layer_offsets(num_elements, 1); - // The idea is to try every feature id in increasing order. In each - // iteration we calculate the gain of the layer using the current feature id - // as split value, and we also obtain the following feature id to try. - while (current_feature_id > last_feature_id) { - last_feature_id = current_feature_id; - int64 next_feature_id = -1; - // Left gradient stats per node. - std::vector left_gradient_stats(num_elements); - for (int root_idx = 0; root_idx < num_elements; root_idx++) { - int idx = - current_layer_offsets[root_idx] + partition_boundaries[root_idx]; - const int end_index = partition_boundaries[root_idx + 1]; - if (idx < end_index && feature_ids(idx, 0) == current_feature_id) { - GradientStats g(*gradients_t, *hessians_t, idx); - g *= normalizer_ratio; - left_gradient_stats[root_idx] = g; - current_layer_offsets[root_idx]++; - idx++; - } - if (idx < end_index && - (feature_ids(idx, 0) < next_feature_id || next_feature_id == -1)) { - next_feature_id = feature_ids(idx, 0); - } - } - float gain_of_split = 0.0; - for (int root_idx = 0; root_idx < num_elements; root_idx++) { - GradientStats right_gradient_stats = - current_layer_stats[root_idx] - left_gradient_stats[root_idx]; - NodeStats left_stat = - state->ComputeNodeStats(left_gradient_stats[root_idx]); - NodeStats right_stat = state->ComputeNodeStats(right_gradient_stats); - gain_of_split += left_stat.gain + right_stat.gain; - current_left_node_stats[root_idx] = left_stat; - current_right_node_stats[root_idx] = right_stat; - } - if (gain_of_split > best_gain) { - best_gain = gain_of_split; - best_left_node_stats = current_left_node_stats; - best_right_node_stats = current_right_node_stats; - best_feature_id = current_feature_id; - } - current_feature_id = next_feature_id; - } - - for (int root_idx = 0; root_idx < num_elements; root_idx++) { - best_gain -= state->ComputeNodeStats(current_layer_stats[root_idx]).gain; - } - best_gain -= num_elements * state->tree_complexity_regularization(); - - ObliviousSplitInfo oblivious_split_info; - auto* equality_split = - oblivious_split_info.mutable_split_node() - ->mutable_oblivious_categorical_id_binary_split(); - equality_split->set_feature_column(state->feature_column_group_id()); - equality_split->set_feature_id(best_feature_id); - (*gains)(0) = best_gain; - - for (int root_idx = 0; root_idx < num_elements; root_idx++) { - auto* left_child = oblivious_split_info.add_children(); - auto* right_child = oblivious_split_info.add_children(); - - state->FillLeaf(best_left_node_stats[root_idx], left_child); - state->FillLeaf(best_right_node_stats[root_idx], right_child); - - const int start_index = partition_boundaries[root_idx]; - (*output_partition_ids)(root_idx) = partition_ids(start_index); - oblivious_split_info.add_children_parent_id(partition_ids(start_index)); - } - SerializeToTString(oblivious_split_info, &(*output_splits)(0)); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("BuildCategoricalEqualitySplits").Device(DEVICE_CPU), - BuildCategoricalEqualitySplitsOp); - -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/kernels/stats_accumulator_ops.cc b/tensorflow/contrib/boosted_trees/kernels/stats_accumulator_ops.cc deleted file mode 100644 index 3a3836ef937..00000000000 --- a/tensorflow/contrib/boosted_trees/kernels/stats_accumulator_ops.cc +++ /dev/null @@ -1,782 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include -#include -#include -#include -#include - -#include "tensorflow/contrib/boosted_trees/lib/utils/parallel_for.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.h" -#include "tensorflow/contrib/boosted_trees/resources/stamped_resource.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/refcount.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { -namespace boosted_trees { - -namespace { -const char* const kStampTokenName = "stamp_token"; -const char* const kNextStampTokenName = "next_stamp_token"; - -struct PartitionKey { - PartitionKey() : partition_id(-1), feature_id(-1), dimension(-1) {} - - PartitionKey(int32 p, int64 f, int32 d) - : partition_id(p), feature_id(f), dimension(d) {} - - bool operator==(const PartitionKey& other) const { - return (partition_id == other.partition_id) && - (dimension == other.dimension) && (feature_id == other.feature_id); - } - - // Compare for PartitionKey. - struct Less { - bool operator()(const PartitionKey& a, const PartitionKey& b) const { - if (a.partition_id < b.partition_id) { - return true; - } - if ((a.partition_id == b.partition_id) && (a.dimension < b.dimension)) { - return true; - } - if ((a.partition_id == b.partition_id) && (a.dimension == b.dimension) && - (a.feature_id < b.feature_id)) { - return true; - } - return false; - } - }; - - // Tree partition defined by traversing the tree to the leaf. - int32 partition_id; - - // Feature column id. - int64 feature_id; - - // Dimension within feature column. - int32 dimension; -}; - -template -class StatsAccumulatorResource : public boosted_trees::StampedResource { - using StatsByPartition = - std::map, - PartitionKey::Less>; - - public: - StatsAccumulatorResource(const TensorShape& gradient_shape, - const TensorShape& hessian_shape) - : gradient_shape_(gradient_shape), - hessian_shape_(hessian_shape), - num_updates_(0) { - // If GradientType/HessianType is scalar float then the shapes should be - // scalar and vice versa. - CHECK_EQ((std::is_same::value), - TensorShapeUtils::IsScalar(gradient_shape)); - CHECK_EQ((std::is_same::value), - TensorShapeUtils::IsScalar(hessian_shape)); - } - - string DebugString() const override { - return strings::StrCat("StatsAccumulatorResource[size=", values_.size(), - "]"); - } - - void Clear() { - values_.clear(); - num_updates_ = 0; - } - - tensorflow::mutex* mutex() { return &mu_; } - StatsByPartition* mutable_values() { return &values_; } - const StatsByPartition& values() const { return values_; } - const int64& num_updates() const { return num_updates_; } - void set_num_updates(int64 val) { num_updates_ = val; } - const TensorShape& gradient_shape() const { return gradient_shape_; } - const TensorShape& hessian_shape() const { return hessian_shape_; } - - private: - // Key into a specific partition to accumulate stats for the specified feature - // id. - StatsByPartition values_; - const TensorShape gradient_shape_; - const TensorShape hessian_shape_; - int64 num_updates_; - tensorflow::mutex mu_; - TF_DISALLOW_COPY_AND_ASSIGN(StatsAccumulatorResource); -}; - -using StatsAccumulatorScalarResource = StatsAccumulatorResource; -using StatsAccumulatorTensorResource = - StatsAccumulatorResource, std::vector>; - -void SerializeScalarAccumulatorToOutput( - const StatsAccumulatorScalarResource& resource, OpKernelContext* context) { - int64 num_slots = resource.values().size(); - Tensor* partition_ids_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output("output_partition_ids", - TensorShape({num_slots}), - &partition_ids_t)); - auto partition_ids = partition_ids_t->vec(); - - // Feature ids tensor has ids of feature columns and their dimensions. - Tensor* feature_ids_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output("output_feature_ids", - TensorShape({num_slots, 2}), - &feature_ids_t)); - auto feature_ids = feature_ids_t->matrix(); - - Tensor* gradients_t = nullptr; - OP_REQUIRES_OK( - context, context->allocate_output( - "output_gradients", TensorShape({num_slots}), &gradients_t)); - auto gradients = gradients_t->vec(); - - Tensor* hessians_t = nullptr; - OP_REQUIRES_OK( - context, context->allocate_output("output_hessians", - TensorShape({num_slots}), &hessians_t)); - auto hessians = hessians_t->vec(); - - int i = 0; - for (const auto& iter : resource.values()) { - partition_ids(i) = iter.first.partition_id; - feature_ids(i, 0) = iter.first.feature_id; - feature_ids(i, 1) = iter.first.dimension; - - gradients(i) = iter.second.first; - hessians(i) = iter.second.second; - ++i; - } -} - -void SerializeTensorAccumulatorToOutput( - const StatsAccumulatorTensorResource& resource, OpKernelContext* context) { - int64 num_slots = resource.values().size(); - Tensor* partition_ids_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output("output_partition_ids", - TensorShape({num_slots}), - &partition_ids_t)); - auto partition_ids = partition_ids_t->vec(); - - Tensor* feature_ids_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output("output_feature_ids", - TensorShape({num_slots, 2}), - &feature_ids_t)); - auto feature_ids = feature_ids_t->matrix(); - - TensorShape gradient_shape = resource.gradient_shape(); - int64 num_gradient_elements = gradient_shape.num_elements(); - gradient_shape.InsertDim(0, num_slots); - Tensor* gradients_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("output_gradients", gradient_shape, - &gradients_t)); - auto gradients = gradients_t->flat_outer_dims(); - - TensorShape hessian_shape = resource.hessian_shape(); - int64 num_hessian_elements = hessian_shape.num_elements(); - hessian_shape.InsertDim(0, num_slots); - Tensor* hessians_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output("output_hessians", - hessian_shape, &hessians_t)); - auto hessians = hessians_t->flat_outer_dims(); - - int i = 0; - for (const auto& iter : resource.values()) { - partition_ids(i) = iter.first.partition_id; - feature_ids(i, 0) = iter.first.feature_id; - feature_ids(i, 1) = iter.first.dimension; - - for (int j = 0; j < num_gradient_elements; ++j) { - gradients(i, j) = iter.second.first[j]; - } - for (int j = 0; j < num_hessian_elements; ++j) { - hessians(i, j) = iter.second.second[j]; - } - ++i; - } -} - -void AddToScalarAccumulator( - const core::RefCountPtr& resource, - const Tensor& partition_ids_t, const Tensor& feature_ids_t, - const Tensor& gradients_t, const Tensor& hessians_t) { - resource->set_num_updates(resource->num_updates() + 1); - const TensorShape& partition_ids_shape = partition_ids_t.shape(); - const auto& partition_ids = partition_ids_t.vec(); - const auto& feature_ids_and_dimensions = feature_ids_t.matrix(); - const auto& gradients = gradients_t.vec(); - const auto& hessians = hessians_t.vec(); - - int64 num_updates = partition_ids_shape.dim_size(0); - auto stats_map = resource->mutable_values(); - for (int64 i = 0; i < num_updates; ++i) { - const auto key = - PartitionKey(partition_ids(i), feature_ids_and_dimensions(i, 0), - feature_ids_and_dimensions(i, 1)); - auto itr = stats_map->find(key); - if (itr != stats_map->end()) { - itr->second.first += gradients(i); - itr->second.second += hessians(i); - } else { - (*stats_map)[key] = {gradients(i), hessians(i)}; - } - } -} - -void AddToScalarAccumulator( - const core::RefCountPtr& resource, - OpKernelContext* context) { - const Tensor* partition_ids_t; - OP_REQUIRES_OK(context, context->input("partition_ids", &partition_ids_t)); - const Tensor* feature_ids_t; - OP_REQUIRES_OK(context, context->input("feature_ids", &feature_ids_t)); - const Tensor* gradients_t; - OP_REQUIRES_OK(context, context->input("gradients", &gradients_t)); - const Tensor* hessians_t; - OP_REQUIRES_OK(context, context->input("hessians", &hessians_t)); - AddToScalarAccumulator(resource, *partition_ids_t, *feature_ids_t, - *gradients_t, *hessians_t); -} - -void AddToTensorAccumulator( - const core::RefCountPtr& resource, - const Tensor& partition_ids_t, const Tensor& feature_ids_t, - const Tensor& gradients_t, const Tensor& hessians_t, - OpKernelContext* context) { - resource->set_num_updates(resource->num_updates() + 1); - - const TensorShape& partition_ids_shape = partition_ids_t.shape(); - const auto& partition_ids = partition_ids_t.vec(); - const auto& feature_ids_and_dimensions = feature_ids_t.matrix(); - TensorShape gradients_shape = gradients_t.shape(); - const auto& gradients = gradients_t.flat_outer_dims(); - TensorShape hessians_shape = hessians_t.shape(); - const auto& hessians = hessians_t.flat_outer_dims(); - - gradients_shape.RemoveDim(0); - hessians_shape.RemoveDim(0); - - // TODO(soroush): Move gradient and hessian shape check to ShapeFn. - OP_REQUIRES( - context, gradients_shape == resource->gradient_shape(), - errors::InvalidArgument(strings::StrCat( - "Gradients dimensions must match: ", gradients_shape.DebugString(), - ", ", resource->gradient_shape().DebugString()))); - - OP_REQUIRES( - context, hessians_shape == resource->hessian_shape(), - errors::InvalidArgument(strings::StrCat( - "Hessian dimensions must match: ", hessians_shape.DebugString(), ", ", - resource->hessian_shape().DebugString()))); - - int64 num_updates = partition_ids_shape.dim_size(0); - auto stats_map = resource->mutable_values(); - for (int64 i = 0; i < num_updates; ++i) { - const auto key = - PartitionKey(partition_ids(i), feature_ids_and_dimensions(i, 0), - feature_ids_and_dimensions(i, 1)); - auto itr = stats_map->find(key); - if (itr == stats_map->end()) { - std::vector new_gradients(gradients_shape.num_elements()); - for (int j = 0; j < gradients_shape.num_elements(); ++j) { - new_gradients[j] = gradients(i, j); - } - std::vector new_hessians(hessians_shape.num_elements()); - for (int j = 0; j < hessians_shape.num_elements(); ++j) { - new_hessians[j] = hessians(i, j); - } - (*stats_map)[key] = {new_gradients, new_hessians}; - } else { - auto& stored_gradients = itr->second.first; - for (int j = 0; j < gradients_shape.num_elements(); ++j) { - stored_gradients[j] += gradients(i, j); - } - auto& stored_hessians = itr->second.second; - for (int j = 0; j < hessians_shape.num_elements(); ++j) { - stored_hessians[j] += hessians(i, j); - } - } - } -} - -void AddToTensorAccumulator( - const core::RefCountPtr& resource, - OpKernelContext* context) { - const Tensor* partition_ids_t; - OP_REQUIRES_OK(context, context->input("partition_ids", &partition_ids_t)); - const Tensor* feature_ids_t; - OP_REQUIRES_OK(context, context->input("feature_ids", &feature_ids_t)); - const Tensor* gradients_t; - OP_REQUIRES_OK(context, context->input("gradients", &gradients_t)); - const Tensor* hessians_t; - OP_REQUIRES_OK(context, context->input("hessians", &hessians_t)); - AddToTensorAccumulator(resource, *partition_ids_t, *feature_ids_t, - *gradients_t, *hessians_t, context); -} - -} // namespace - -REGISTER_RESOURCE_HANDLE_KERNEL(StatsAccumulatorScalarResource); -REGISTER_RESOURCE_HANDLE_KERNEL(StatsAccumulatorTensorResource); - -REGISTER_KERNEL_BUILDER( - Name("StatsAccumulatorScalarIsInitialized").Device(DEVICE_CPU), - IsResourceInitialized); - -REGISTER_KERNEL_BUILDER( - Name("StatsAccumulatorTensorIsInitialized").Device(DEVICE_CPU), - IsResourceInitialized); - -class CreateStatsAccumulatorScalarOp : public OpKernel { - public: - explicit CreateStatsAccumulatorScalarOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t)); - - TensorShape gradient_shape = TensorShape({}); - TensorShape hessian_shape = TensorShape({}); - - auto* result = - new StatsAccumulatorScalarResource(gradient_shape, hessian_shape); - result->set_stamp(stamp_token_t->scalar()()); - // Only create one, if one does not exist already. Report status for all - // other exceptions. If one already exists, it unrefs the new one. - auto status = CreateResource(context, HandleFromInput(context, 0), result); - if (!status.ok() && status.code() != tensorflow::error::ALREADY_EXISTS) { - OP_REQUIRES(context, false, status); - } - } -}; - -REGISTER_KERNEL_BUILDER(Name("CreateStatsAccumulatorScalar").Device(DEVICE_CPU), - CreateStatsAccumulatorScalarOp); - -class CreateStatsAccumulatorTensorOp : public OpKernel { - public: - explicit CreateStatsAccumulatorTensorOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t)); - - const Tensor* gradient_shape_t; - OP_REQUIRES_OK( - context, context->input("per_slot_gradient_shape", &gradient_shape_t)); - - const Tensor* hessian_shape_t; - OP_REQUIRES_OK(context, - context->input("per_slot_hessian_shape", &hessian_shape_t)); - TensorShape gradient_shape = TensorShape(gradient_shape_t->vec()); - TensorShape hessian_shape = TensorShape(hessian_shape_t->vec()); - auto* result = - new StatsAccumulatorTensorResource(gradient_shape, hessian_shape); - result->set_stamp(stamp_token_t->scalar()()); - - // Only create one, if one does not exist already. Report status for all - // other exceptions. If one already exists, it unrefs the new one. - auto status = CreateResource(context, HandleFromInput(context, 0), result); - if (!status.ok() && status.code() != tensorflow::error::ALREADY_EXISTS) { - OP_REQUIRES(context, false, status); - } - } -}; - -REGISTER_KERNEL_BUILDER(Name("CreateStatsAccumulatorTensor").Device(DEVICE_CPU), - CreateStatsAccumulatorTensorOp); - -class StatsAccumulatorScalarAddOp : public OpKernel { - public: - explicit StatsAccumulatorScalarAddOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - OpInputList resource_handle_list; - OP_REQUIRES_OK(context, context->input_list("stats_accumulator_handles", - &resource_handle_list)); - OpInputList partition_ids_list; - OP_REQUIRES_OK(context, - context->input_list("partition_ids", &partition_ids_list)); - - OpInputList feature_ids_list; - OP_REQUIRES_OK(context, - context->input_list("feature_ids", &feature_ids_list)); - OpInputList gradients_list; - OP_REQUIRES_OK(context, context->input_list("gradients", &gradients_list)); - OpInputList hessians_list; - OP_REQUIRES_OK(context, context->input_list("hessians", &hessians_list)); - - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input(kStampTokenName, &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - - thread::ThreadPool* const worker_threads = - context->device()->tensorflow_cpu_worker_threads()->workers; - boosted_trees::utils::ParallelFor( - resource_handle_list.size(), worker_threads->NumThreads(), - worker_threads, - [&context, &resource_handle_list, &partition_ids_list, - &feature_ids_list, &gradients_list, &hessians_list, - stamp_token](int64 start, int64 end) { - for (int resource_handle_idx = start; resource_handle_idx < end; - ++resource_handle_idx) { - const ResourceHandle& handle = - resource_handle_list[resource_handle_idx] - .flat()(0); - - core::RefCountPtr resource; - OP_REQUIRES_OK(context, LookupResource(context, handle, &resource)); - mutex_lock l(*resource->mutex()); - - // If the stamp is invalid we drop the update. - if (!resource->is_stamp_valid(stamp_token)) { - VLOG(1) << "Invalid stamp token in StatsAccumulatorScalarAddOp. " - << "Passed stamp token: " << stamp_token << " " - << "Current token: " << resource->stamp(); - return; - } - AddToScalarAccumulator(resource, - partition_ids_list[resource_handle_idx], - feature_ids_list[resource_handle_idx], - gradients_list[resource_handle_idx], - hessians_list[resource_handle_idx]); - } - }); - } -}; - -REGISTER_KERNEL_BUILDER(Name("StatsAccumulatorScalarAdd").Device(DEVICE_CPU), - StatsAccumulatorScalarAddOp); - -class StatsAccumulatorTensorAddOp : public OpKernel { - public: - explicit StatsAccumulatorTensorAddOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - OpInputList resource_handle_list; - OP_REQUIRES_OK(context, context->input_list("stats_accumulator_handles", - &resource_handle_list)); - OpInputList partition_ids_list; - OP_REQUIRES_OK(context, - context->input_list("partition_ids", &partition_ids_list)); - - OpInputList feature_ids_list; - OP_REQUIRES_OK(context, - context->input_list("feature_ids", &feature_ids_list)); - OpInputList gradients_list; - OP_REQUIRES_OK(context, context->input_list("gradients", &gradients_list)); - OpInputList hessians_list; - OP_REQUIRES_OK(context, context->input_list("hessians", &hessians_list)); - - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input(kStampTokenName, &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - - thread::ThreadPool* const worker_threads = - context->device()->tensorflow_cpu_worker_threads()->workers; - boosted_trees::utils::ParallelFor( - resource_handle_list.size(), worker_threads->NumThreads(), - worker_threads, - [&context, &resource_handle_list, &partition_ids_list, - &feature_ids_list, &gradients_list, &hessians_list, - stamp_token](int64 start, int64 end) { - for (int resource_handle_idx = start; resource_handle_idx < end; - ++resource_handle_idx) { - const ResourceHandle& handle = - resource_handle_list[resource_handle_idx] - .flat()(0); - - core::RefCountPtr resource; - OP_REQUIRES_OK(context, LookupResource(context, handle, &resource)); - mutex_lock l(*resource->mutex()); - - // If the stamp is invalid we drop the update. - if (!resource->is_stamp_valid(stamp_token)) { - VLOG(1) << "Invalid stamp token in StatsAccumulatorScalarAddOp. " - << "Passed stamp token: " << stamp_token << " " - << "Current token: " << resource->stamp(); - return; - } - AddToTensorAccumulator(resource, - partition_ids_list[resource_handle_idx], - feature_ids_list[resource_handle_idx], - gradients_list[resource_handle_idx], - hessians_list[resource_handle_idx], context); - } - }); - } -}; - -REGISTER_KERNEL_BUILDER(Name("StatsAccumulatorTensorAdd").Device(DEVICE_CPU), - StatsAccumulatorTensorAddOp); - -class StatsAccumulatorScalarFlushOp : public OpKernel { - public: - explicit StatsAccumulatorScalarFlushOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &resource)); - mutex_lock l(*resource->mutex()); - - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input(kStampTokenName, &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - - // If the stamp is invalid we restart the PS. It shouldn't happen since - // only Chief should call this function and chief is guaranteed to be in - // a consistent state. - CHECK(resource->is_stamp_valid(stamp_token)); - - const Tensor* next_stamp_token_t; - OP_REQUIRES_OK(context, - context->input(kNextStampTokenName, &next_stamp_token_t)); - int64 next_stamp_token = next_stamp_token_t->scalar()(); - CHECK(stamp_token != next_stamp_token); - - SerializeScalarAccumulatorToOutput(*resource, context); - Tensor* num_updates_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("num_updates", TensorShape({}), - &num_updates_t)); - num_updates_t->scalar()() = resource->num_updates(); - - resource->Clear(); - resource->set_stamp(next_stamp_token); - } -}; - -REGISTER_KERNEL_BUILDER(Name("StatsAccumulatorScalarFlush").Device(DEVICE_CPU), - StatsAccumulatorScalarFlushOp); - -class StatsAccumulatorTensorFlushOp : public OpKernel { - public: - explicit StatsAccumulatorTensorFlushOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &resource)); - mutex_lock l(*resource->mutex()); - - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input(kStampTokenName, &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - - const Tensor* next_stamp_token_t; - OP_REQUIRES_OK(context, - context->input(kNextStampTokenName, &next_stamp_token_t)); - int64 next_stamp_token = next_stamp_token_t->scalar()(); - - // If the stamp is invalid we restart the PS. It shouldn't happen since - // only Chief should call this function and chief is guaranteed to be in - // a consistent state. - CHECK(resource->is_stamp_valid(stamp_token)); - CHECK(stamp_token != next_stamp_token); - SerializeTensorAccumulatorToOutput(*resource, context); - Tensor* num_updates_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("num_updates", TensorShape({}), - &num_updates_t)); - num_updates_t->scalar()() = resource->num_updates(); - resource->Clear(); - resource->set_stamp(next_stamp_token); - } -}; - -REGISTER_KERNEL_BUILDER(Name("StatsAccumulatorTensorFlush").Device(DEVICE_CPU), - StatsAccumulatorTensorFlushOp); - -class StatsAccumulatorScalarDeserializeOp : public OpKernel { - public: - explicit StatsAccumulatorScalarDeserializeOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &resource)); - mutex_lock l(*resource->mutex()); - - // Check the stamp token. - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input(kStampTokenName, &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - resource->Clear(); - resource->set_stamp(stamp_token); - AddToScalarAccumulator(resource, context); - const Tensor* num_updates_t; - OP_REQUIRES_OK(context, context->input("num_updates", &num_updates_t)); - resource->set_num_updates(num_updates_t->scalar()()); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("StatsAccumulatorScalarDeserialize").Device(DEVICE_CPU), - StatsAccumulatorScalarDeserializeOp); - -class StatsAccumulatorTensorDeserializeOp : public OpKernel { - public: - explicit StatsAccumulatorTensorDeserializeOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &resource)); - mutex_lock l(*resource->mutex()); - - // Check the stamp token. - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input(kStampTokenName, &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - resource->Clear(); - resource->set_stamp(stamp_token); - AddToTensorAccumulator(resource, context); - const Tensor* num_updates_t; - OP_REQUIRES_OK(context, context->input("num_updates", &num_updates_t)); - resource->set_num_updates(num_updates_t->scalar()()); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("StatsAccumulatorTensorDeserialize").Device(DEVICE_CPU), - StatsAccumulatorTensorDeserializeOp); - -class StatsAccumulatorScalarSerializeOp : public OpKernel { - public: - explicit StatsAccumulatorScalarSerializeOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &resource)); - mutex_lock l(*resource->mutex()); - SerializeScalarAccumulatorToOutput(*resource, context); - Tensor* stamp_token_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("stamp_token", TensorShape({}), - &stamp_token_t)); - stamp_token_t->scalar()() = resource->stamp(); - - Tensor* num_updates_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("num_updates", TensorShape({}), - &num_updates_t)); - num_updates_t->scalar()() = resource->num_updates(); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("StatsAccumulatorScalarSerialize").Device(DEVICE_CPU), - StatsAccumulatorScalarSerializeOp); - -class StatsAccumulatorTensorSerializeOp : public OpKernel { - public: - explicit StatsAccumulatorTensorSerializeOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &resource)); - mutex_lock l(*resource->mutex()); - SerializeTensorAccumulatorToOutput(*resource, context); - Tensor* stamp_token_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("stamp_token", TensorShape({}), - &stamp_token_t)); - stamp_token_t->scalar()() = resource->stamp(); - - Tensor* num_updates_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("num_updates", TensorShape({}), - &num_updates_t)); - num_updates_t->scalar()() = resource->num_updates(); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("StatsAccumulatorTensorSerialize").Device(DEVICE_CPU), - StatsAccumulatorTensorSerializeOp); - -class StatsAccumulatorScalarMakeSummaryOp : public OpKernel { - public: - explicit StatsAccumulatorScalarMakeSummaryOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - TensorShape gradient_shape = TensorShape({}); - TensorShape hessian_shape = TensorShape({}); - core::RefCountPtr resource( - new StatsAccumulatorScalarResource(gradient_shape, hessian_shape)); - // Check the stamp token. - AddToScalarAccumulator(resource, context); - SerializeScalarAccumulatorToOutput(*resource, context); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("StatsAccumulatorScalarMakeSummary").Device(DEVICE_CPU), - StatsAccumulatorScalarMakeSummaryOp); - -class StatsAccumulatorTensorMakeSummaryOp : public OpKernel { - public: - explicit StatsAccumulatorTensorMakeSummaryOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - const Tensor* gradients_t; - OP_REQUIRES_OK(context, context->input("gradients", &gradients_t)); - TensorShape gradients_shape = gradients_t->shape(); - gradients_shape.RemoveDim(0); - - const Tensor* hessians_t; - OP_REQUIRES_OK(context, context->input("hessians", &hessians_t)); - TensorShape hessians_shape = hessians_t->shape(); - hessians_shape.RemoveDim(0); - - core::RefCountPtr resource( - new StatsAccumulatorTensorResource(gradients_shape, hessians_shape)); - // Check the stamp token. - AddToTensorAccumulator(resource, context); - SerializeTensorAccumulatorToOutput(*resource, context); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("StatsAccumulatorTensorMakeSummary").Device(DEVICE_CPU), - StatsAccumulatorTensorMakeSummaryOp); - -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/kernels/training_ops.cc b/tensorflow/contrib/boosted_trees/kernels/training_ops.cc deleted file mode 100644 index bf5f5d34457..00000000000 --- a/tensorflow/contrib/boosted_trees/kernels/training_ops.cc +++ /dev/null @@ -1,996 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include - -#include "tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h" -#include "tensorflow/contrib/boosted_trees/proto/learner.pb.h" -#include "tensorflow/contrib/boosted_trees/proto/split_info.pb.h" -#include "tensorflow/contrib/boosted_trees/proto/tree_config.pb.h" -#include "tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/lib/core/refcount.h" - -namespace tensorflow { -using tensorflow::boosted_trees::learner::LearningRateDropoutDrivenConfig; - -namespace boosted_trees { - -namespace { - -using boosted_trees::learner::LearnerConfig; -using boosted_trees::learner::LearningRateConfig; -using boosted_trees::models::DecisionTreeEnsembleResource; -using boosted_trees::trees::DecisionTreeConfig; -using boosted_trees::trees::Leaf; -using boosted_trees::trees::TreeNode; -using boosted_trees::trees::TreeNodeMetadata; -using boosted_trees::utils::DropoutUtils; - -// SplitCandidate holds the split candidate node along with the stats. -struct SplitCandidate { - // Id of handler that generated the split candidate. - int64 handler_id; - - // Split gain. - float gain; - - // Split info. - learner::SplitInfo split_info; - - // Oblivious split info. - learner::ObliviousSplitInfo oblivious_split_info; -}; - -// Checks that the leaf is not empty. -bool IsLeafWellFormed(const Leaf& leaf) { - return leaf.has_sparse_vector() || leaf.has_vector(); -} - -// Helper method to update the best split per partition given -// a current candidate. -void UpdateBestSplit( - const boosted_trees::learner::LearnerConfig& learner_config, - int32 partition_id, SplitCandidate* split, - std::map* best_splits) { - // Don't consider nodeless splits. - if (TF_PREDICT_FALSE(split->split_info.split_node().node_case() == - TreeNode::NODE_NOT_SET)) { - return; - } - - // Don't consider negative splits if we're pre-pruning the tree. - // Note that zero-gain splits are acceptable as they're mostly doing as well - // as what bias centering in that partition would do. - if (learner_config.pruning_mode() == - boosted_trees::learner::LearnerConfig::PRE_PRUNE && - split->gain < 0) { - return; - } - - // If the current node is pure, one of the leafs will be empty, so the split - // is meaningless and we should not split. - if (!(IsLeafWellFormed(split->split_info.right_child()) && - IsLeafWellFormed(split->split_info.left_child()))) { - VLOG(1) << "Split does not actually split anything"; - return; - } - - // Take the split if we don't have a candidate yet. - auto best_split_it = best_splits->find(partition_id); - if (best_split_it == best_splits->end()) { - best_splits->insert(std::make_pair(partition_id, std::move(*split))); - return; - } - - // Determine if best split so far needs to be replaced. - SplitCandidate& best_split = best_split_it->second; - if (TF_PREDICT_FALSE(split->gain == best_split.gain)) { - // Tie break on node case preferring simpler tree node types. - VLOG(2) << "Attempting to tie break with smaller node case. " - << "(current split: " << split->split_info.split_node().node_case() - << ", best split: " - << best_split.split_info.split_node().node_case() << ")"; - if (split->split_info.split_node().node_case() < - best_split.split_info.split_node().node_case()) { - best_split = std::move(*split); - } else if (split->split_info.split_node().node_case() == - best_split.split_info.split_node().node_case()) { - // Tie break on handler Id. - VLOG(2) << "Tie breaking with higher handler Id. " - << "(current split: " << split->handler_id - << ", best split: " << best_split.handler_id << ")"; - if (split->handler_id > best_split.handler_id) { - best_split = std::move(*split); - } - } - } else if (split->gain > best_split.gain) { - best_split = std::move(*split); - } -} - -// Helper method to check whether a node is a terminal node in that it -// only has leaf nodes as children. -bool IsTerminalSplitNode(const size_t node_id, - const std::vector& children, - const std::vector& nodes) { - for (const int32 child_id : children) { - const auto& child_node = nodes[child_id]; - CHECK(child_node.node_case() != TreeNode::NODE_NOT_SET); - if (child_node.node_case() != TreeNode::kLeaf) { - return false; - } - } - return true; -} - -// Helper method to recursively prune the tree in a depth-first fashion. -void RecursivePruneTree(const size_t node_id, std::vector* nodes) { - // Base case when we reach a leaf. - TreeNode& tree_node = (*nodes)[node_id]; - CHECK(tree_node.node_case() != TreeNode::NODE_NOT_SET); - if (tree_node.node_case() == TreeNode::kLeaf) { - return; - } - - // Traverse node children first and recursively prune their sub-trees. - const std::vector children = - boosted_trees::trees::DecisionTree::GetChildren(tree_node); - for (const int32 child_id : children) { - RecursivePruneTree(child_id, nodes); - } - - // Two conditions must be satisfied to prune the node: - // 1- The split gain is negative. - // 2- After depth-first pruning, the node only has leaf children. - TreeNodeMetadata* node_metadata = tree_node.mutable_node_metadata(); - if (node_metadata->gain() < 0 && - IsTerminalSplitNode(node_id, children, (*nodes))) { - // Clear node children. - for (const int32 child_id : children) { - auto& child_node = (*nodes)[child_id]; - child_node.Clear(); - } - - // Change node back into leaf. - (*tree_node.mutable_leaf()) = *node_metadata->mutable_original_leaf(); - - // Clear gain for leaf node. - tree_node.clear_node_metadata(); - } else { - // Clear original leaf as it's no longer needed for back-track pruning. - node_metadata->clear_original_leaf(); - } -} - -} // namespace - -class CenterTreeEnsembleBiasOp : public OpKernel { - public: - explicit CenterTreeEnsembleBiasOp(OpKernelConstruction* const context) - : OpKernel(context) { - // Read learner config. - string serialized_learner_config; - OP_REQUIRES_OK(context, context->GetAttr("learner_config", - &serialized_learner_config)); - OP_REQUIRES(context, - learner_config_.ParseFromString(serialized_learner_config), - errors::InvalidArgument("Unable to parse learner config.")); - - // Read centering epsilon. - OP_REQUIRES_OK(context, - context->GetAttr("centering_epsilon", ¢ering_epsilon_)); - } - - void Compute(OpKernelContext* const context) override { - // Get decision tree ensemble. - core::RefCountPtr ensemble_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &ensemble_resource)); - mutex_lock l(*ensemble_resource->get_mutex()); - - // Get the stamp token. - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - - // Only the Chief should run this Op and it is guaranteed to be in - // a consistent state so the stamps must always match. - CHECK(ensemble_resource->is_stamp_valid(stamp_token)); - - // Get the next stamp token. - const Tensor* next_stamp_token_t; - OP_REQUIRES_OK(context, - context->input("next_stamp_token", &next_stamp_token_t)); - int64 next_stamp_token = next_stamp_token_t->scalar()(); - CHECK(stamp_token != next_stamp_token); - - // Update the ensemble stamp. - ensemble_resource->set_stamp(next_stamp_token); - - // Get the delta updates. - const Tensor* delta_updates_t; - OP_REQUIRES_OK(context, context->input("delta_updates", &delta_updates_t)); - auto delta_updates = delta_updates_t->vec(); - const int64 logits_dimension = delta_updates_t->dim_size(0); - - // Get the bias. - boosted_trees::trees::Leaf* const bias = - RetrieveBias(ensemble_resource, logits_dimension); - CHECK(bias->has_vector()); - - // Update the bias. - float total_delta = 0; - auto* bias_vec = bias->mutable_vector(); - for (size_t idx = 0; idx < bias->vector().value_size(); ++idx) { - float delta = delta_updates(idx); - bias_vec->set_value(idx, bias_vec->value(idx) + delta); - total_delta += std::abs(delta); - } - - // Make a centering continuation decision based on current update. - bool continue_centering = total_delta > centering_epsilon_; - if (continue_centering) { - VLOG(1) << "Continuing to center bias, delta=" << total_delta; - } else { - VLOG(1) << "Done centering bias, delta=" << total_delta; - ensemble_resource->LastTreeMetadata()->set_is_finalized(true); - } - Tensor* continue_centering_t = nullptr; - OP_REQUIRES_OK( - context, context->allocate_output("continue_centering", TensorShape({}), - &continue_centering_t)); - continue_centering_t->scalar()() = continue_centering; - } - - private: - // Helper method to retrieve the bias from the tree ensemble. - Leaf* RetrieveBias( - const core::RefCountPtr& ensemble_resource, - int64 logits_dimension) { - const int32 num_trees = ensemble_resource->num_trees(); - if (num_trees <= 0) { - // Add a new bias leaf. - ensemble_resource->IncrementAttempts(); - boosted_trees::trees::DecisionTreeConfig* const tree_config = - ensemble_resource->AddNewTree(1.0); - auto* const leaf = tree_config->add_nodes()->mutable_leaf(); - for (size_t idx = 0; idx < logits_dimension; ++idx) { - leaf->mutable_vector()->add_value(0.0); - } - return leaf; - } else if (num_trees == 1) { - // Confirms that the only tree is a bias and returns its leaf. - boosted_trees::trees::DecisionTreeConfig* const tree_config = - ensemble_resource->LastTree(); - CHECK_EQ(tree_config->nodes_size(), 1); - CHECK_EQ(tree_config->nodes(0).node_case(), TreeNode::kLeaf); - return tree_config->mutable_nodes(0)->mutable_leaf(); - } else { - LOG(FATAL) << "Unable to center bias on an already grown ensemble"; - } - } - - boosted_trees::learner::LearnerConfig learner_config_; - float centering_epsilon_; -}; - -REGISTER_KERNEL_BUILDER(Name("CenterTreeEnsembleBias").Device(DEVICE_CPU), - CenterTreeEnsembleBiasOp); - -class GrowTreeEnsembleOp : public OpKernel { - public: - explicit GrowTreeEnsembleOp(OpKernelConstruction* const context) - : OpKernel(context) { - // Read number of handlers, note that this is the static number of - // all handlers but any subset of these handlers may be active at a time. - OP_REQUIRES_OK(context, context->GetAttr("num_handlers", &num_handlers_)); - - OP_REQUIRES_OK(context, context->GetAttr("center_bias", ¢er_bias_)); - - // Read learner config. - string serialized_learner_config; - OP_REQUIRES_OK(context, context->GetAttr("learner_config", - &serialized_learner_config)); - OP_REQUIRES(context, - learner_config_.ParseFromString(serialized_learner_config), - errors::InvalidArgument("Unable to parse learner config.")); - - // Determine whether dropout was used when building this tree. - if (learner_config_.has_learning_rate_tuner() && - learner_config_.learning_rate_tuner().tuner_case() == - LearningRateConfig::kDropout) { - dropout_config_ = learner_config_.learning_rate_tuner().dropout(); - dropout_was_applied_ = true; - } else { - dropout_was_applied_ = false; - } - } - - void Compute(OpKernelContext* const context) override { - // Get decision tree ensemble. - core::RefCountPtr ensemble_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &ensemble_resource)); - mutex_lock l(*ensemble_resource->get_mutex()); - - // Get the stamp token. - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - - // Only the Chief should run this Op and it is guaranteed to be in - // a consistent state so the stamps must always match. - CHECK(ensemble_resource->is_stamp_valid(stamp_token)); - - // Get the next stamp token. - const Tensor* next_stamp_token_t; - OP_REQUIRES_OK(context, - context->input("next_stamp_token", &next_stamp_token_t)); - int64 next_stamp_token = next_stamp_token_t->scalar()(); - CHECK(stamp_token != next_stamp_token); - - // Update the ensemble stamp regardless of whether a layer - // or tree is actually grown. - ensemble_resource->set_stamp(next_stamp_token); - - // Read the learning_rate. - const Tensor* learning_rate_t; - OP_REQUIRES_OK(context, context->input("learning_rate", &learning_rate_t)); - float learning_rate = learning_rate_t->scalar()(); - - // Read the weak learner type to use. - const Tensor* weak_learner_type_t; - OP_REQUIRES_OK(context, - context->input("weak_learner_type", &weak_learner_type_t)); - const int32 weak_learner_type = weak_learner_type_t->scalar()(); - - const Tensor* seed_t; - OP_REQUIRES_OK(context, context->input("dropout_seed", &seed_t)); - // Cast seed to uint64. - const uint64 dropout_seed = seed_t->scalar()(); - - // Read partition Ids, gains and split candidates. - OpInputList partition_ids_list; - OpInputList gains_list; - OpInputList splits_list; - OP_REQUIRES_OK(context, - context->input_list("partition_ids", &partition_ids_list)); - OP_REQUIRES_OK(context, context->input_list("gains", &gains_list)); - OP_REQUIRES_OK(context, context->input_list("splits", &splits_list)); - - // Increment attempt stats. - ensemble_resource->IncrementAttempts(); - - // Find best splits for each active partition. - std::map best_splits; - switch (weak_learner_type) { - case LearnerConfig::NORMAL_DECISION_TREE: { - FindBestSplitsPerPartitionNormal(context, partition_ids_list, - gains_list, splits_list, &best_splits); - break; - } - case LearnerConfig::OBLIVIOUS_DECISION_TREE: { - FindBestSplitOblivious(context, gains_list, splits_list, &best_splits); - break; - } - } - // No-op if no new splits can be considered. - if (best_splits.empty()) { - LOG(WARNING) << "Not growing tree ensemble as no good splits were found."; - return; - } - - // Get the max tree depth. - const Tensor* max_tree_depth_t; - OP_REQUIRES_OK(context, - context->input("max_tree_depth", &max_tree_depth_t)); - const int32 max_tree_depth = max_tree_depth_t->scalar()(); - // Update and retrieve the growable tree. - // If the tree is fully built and dropout was applied, it also adjusts the - // weights of dropped and the last tree. - DecisionTreeConfig* const tree_config = UpdateAndRetrieveGrowableTree( - ensemble_resource, learning_rate, dropout_seed, max_tree_depth, - weak_learner_type); - // Split tree nodes. - switch (weak_learner_type) { - case LearnerConfig::NORMAL_DECISION_TREE: { - for (auto& split_entry : best_splits) { - SplitTreeNode(split_entry.first, &split_entry.second, tree_config, - ensemble_resource); - } - break; - } - case LearnerConfig::OBLIVIOUS_DECISION_TREE: { - SplitTreeLayer(&best_splits[0], tree_config, ensemble_resource); - } - } - // Post-prune finalized tree if needed. - if (learner_config_.pruning_mode() == - boosted_trees::learner::LearnerConfig::POST_PRUNE && - ensemble_resource->LastTreeMetadata()->is_finalized()) { - VLOG(2) << "Post-pruning finalized tree."; - if (weak_learner_type == LearnerConfig::OBLIVIOUS_DECISION_TREE) { - LOG(FATAL) << "Post-prunning is not implemented for Oblivious trees."; - } - PruneTree(tree_config); - - // If after post-pruning the whole tree has no gain, remove the tree - // altogether from the ensemble. - if (tree_config->nodes_size() <= 0) { - ensemble_resource->RemoveLastTree(); - } - - if ((ensemble_resource->num_trees() == 0 || - ensemble_resource->LastTreeMetadata()->is_finalized()) && - learner_config_.has_each_tree_start() && - learner_config_.each_tree_start().nodes_size() > 0) { - DCHECK_GT(learner_config_.each_tree_start_num_layers(), 0); - // Add new dummy tree - boosted_trees::trees::DecisionTreeConfig* const tree_config = - ensemble_resource->AddNewTree(learning_rate); - VLOG(1) << "Adding a new forced tree"; - - *tree_config = learner_config_.each_tree_start(); - - boosted_trees::trees::DecisionTreeMetadata* const tree_metadata = - ensemble_resource->LastTreeMetadata(); - - tree_metadata->set_is_finalized(max_tree_depth <= 1); - tree_metadata->set_num_tree_weight_updates(1); - tree_metadata->set_num_layers_grown( - learner_config_.each_tree_start_num_layers()); - } - } - } - - private: - // Helper method which effectively does a reduce over all split candidates - // and finds the best split for each partition. - void FindBestSplitsPerPartitionNormal( - OpKernelContext* const context, const OpInputList& partition_ids_list, - const OpInputList& gains_list, const OpInputList& splits_list, - std::map* best_splits) { - // Find best split per partition going through every feature candidate. - // TODO(salehay): Is this worth parallelizing? - for (int64 handler_id = 0; handler_id < num_handlers_; ++handler_id) { - const auto& partition_ids = partition_ids_list[handler_id].vec(); - const auto& gains = gains_list[handler_id].vec(); - const auto& splits = splits_list[handler_id].vec(); - OP_REQUIRES(context, partition_ids.size() == gains.size(), - errors::InvalidArgument( - "Inconsistent partition Ids and gains tensors: ", - partition_ids.size(), " != ", gains.size())); - OP_REQUIRES(context, partition_ids.size() == splits.size(), - errors::InvalidArgument( - "Inconsistent partition Ids and splits tensors: ", - partition_ids.size(), " != ", splits.size())); - for (size_t candidate_idx = 0; candidate_idx < splits.size(); - ++candidate_idx) { - // Get current split candidate. - const auto& partition_id = partition_ids(candidate_idx); - const auto& gain = gains(candidate_idx); - const auto& serialized_split = splits(candidate_idx); - SplitCandidate split; - split.handler_id = handler_id; - split.gain = gain; - OP_REQUIRES(context, split.split_info.ParseFromString(serialized_split), - errors::InvalidArgument("Unable to parse split info.")); - - // Update best split for partition based on the current candidate. - UpdateBestSplit(learner_config_, partition_id, &split, best_splits); - } - } - } - - void FindBestSplitOblivious(OpKernelContext* const context, - const OpInputList& gains_list, - const OpInputList& splits_list, - std::map* best_splits) { - // Find best split per partition going through every feature candidate. - for (int64 handler_id = 0; handler_id < num_handlers_; ++handler_id) { - const auto& gains = gains_list[handler_id].vec(); - const auto& splits = splits_list[handler_id].vec(); - OP_REQUIRES(context, gains.size() == 1, - errors::InvalidArgument( - "Gains size must be one for oblivious weak learner: ", - gains.size(), " != ", 1)); - OP_REQUIRES(context, splits.size() == 1, - errors::InvalidArgument( - "Splits size must be one for oblivious weak learner: ", - splits.size(), " != ", 1)); - // Get current split candidate. - const auto& gain = gains(0); - const auto& serialized_split = splits(0); - SplitCandidate split; - split.handler_id = handler_id; - split.gain = gain; - OP_REQUIRES( - context, split.oblivious_split_info.ParseFromString(serialized_split), - errors::InvalidArgument("Unable to parse oblivious split info.")); - - auto split_info = split.oblivious_split_info; - CHECK(split_info.children_size() % 2 == 0) - << "The oblivious split should generate an even number of children: " - << split_info.children_size(); - - // If every node is pure, then we shouldn't split. - bool only_pure_nodes = true; - for (int idx = 0; idx < split_info.children_size(); idx += 2) { - if (IsLeafWellFormed(*split_info.mutable_children(idx)) && - IsLeafWellFormed(*split_info.mutable_children(idx + 1))) { - only_pure_nodes = false; - break; - } - } - if (only_pure_nodes) { - VLOG(1) << "The oblivious split does not actually split anything."; - continue; - } - - // Don't consider negative splits if we're pre-pruning the tree. - if (learner_config_.pruning_mode() == learner::LearnerConfig::PRE_PRUNE && - gain < 0) { - continue; - } - - // Take the split if we don't have a candidate yet. - auto best_split_it = best_splits->find(0); - if (best_split_it == best_splits->end()) { - best_splits->insert(std::make_pair(0, std::move(split))); - continue; - } - - // Determine if we should update best split. - SplitCandidate& best_split = best_split_it->second; - trees::TreeNode current_node = split_info.split_node(); - trees::TreeNode best_node = best_split.oblivious_split_info.split_node(); - if (TF_PREDICT_FALSE(gain == best_split.gain)) { - // Tie break on node case preferring simpler tree node types. - VLOG(2) << "Attempting to tie break with smaller node case. " - << "(current split: " << current_node.node_case() - << ", best split: " << best_node.node_case() << ")"; - if (current_node.node_case() < best_node.node_case()) { - best_split = std::move(split); - } else if (current_node.node_case() == best_node.node_case()) { - // Tie break on handler Id. - VLOG(2) << "Tie breaking with higher handler Id. " - << "(current split: " << handler_id - << ", best split: " << best_split.handler_id << ")"; - if (handler_id > best_split.handler_id) { - best_split = std::move(split); - } - } - } else if (gain > best_split.gain) { - best_split = std::move(split); - } - } - } - - void UpdateTreeWeightsIfDropout( - const core::RefCountPtr& ensemble_resource, - const uint64 dropout_seed) { - // It is possible that the tree was built with dropout. If it is the case, - // we need to adjust the tree weight, or bail out. - if (!dropout_was_applied_ || - !ensemble_resource->LastTreeMetadata()->is_finalized()) { - return; - } - const int32 num_trees = ensemble_resource->num_trees(); - - // Based on seed, figure out what trees were dropped before. - std::unordered_set trees_not_to_drop; - if (center_bias_) { - trees_not_to_drop.insert(0); - } - // Last tree is the current tree that is built. - const int32 current_tree = num_trees - 1; - trees_not_to_drop.insert(current_tree); - - // Since only chief builds the trees, we are sure that the other tree - // weights didn't change. - std::vector weights = ensemble_resource->GetTreeWeights(); - std::vector dropped_trees; - std::vector dropped_trees_weights; - const auto dropout_status = DropoutUtils::DropOutTrees( - dropout_seed, dropout_config_, trees_not_to_drop, weights, - &dropped_trees, &dropped_trees_weights); - CHECK(dropout_status.ok()) - << "Can't figure out what trees were dropped out before, error is " - << dropout_status.error_message(); - - // Now we have dropped trees, update their weights and the current tree - // weight. - if (!dropped_trees.empty()) { - std::vector increment_num_updates(num_trees, 0); - DropoutUtils::GetTreesWeightsForAddingTrees( - dropped_trees, dropped_trees_weights, current_tree, - 1 /* only 1 tree was added */, &weights, &increment_num_updates); - - // Update the weights and num of updates for trees. - for (int i = 0; i < num_trees; ++i) { - ensemble_resource->SetTreeWeight(i, weights[i], - increment_num_updates[i]); - } - } - } - - // Helper method to update the growable tree which is by definition the last - // tree in the ensemble. - DecisionTreeConfig* UpdateAndRetrieveGrowableTree( - const core::RefCountPtr& ensemble_resource, - const float learning_rate, const uint64 dropout_seed, - const int32 max_tree_depth, const int32 weak_learner_type) { - const auto num_trees = ensemble_resource->num_trees(); - if (num_trees <= 0 || - ensemble_resource->LastTreeMetadata()->is_finalized()) { - // Create a new tree with a no-op leaf. - boosted_trees::trees::DecisionTreeConfig* const tree_config = - ensemble_resource->AddNewTree(learning_rate); - VLOG(1) << "Adding layer #0 to tree #" << num_trees << " of ensemble of " - << num_trees + 1 << " trees."; - tree_config->add_nodes()->mutable_leaf(); - boosted_trees::trees::DecisionTreeMetadata* const tree_metadata = - ensemble_resource->LastTreeMetadata(); - tree_metadata->set_is_finalized(max_tree_depth <= 1); - tree_metadata->set_num_tree_weight_updates(1); - } else { - // The growable tree is by definition the last tree in the ensemble. - boosted_trees::trees::DecisionTreeMetadata* const tree_metadata = - ensemble_resource->LastTreeMetadata(); - const auto new_num_layers = tree_metadata->num_layers_grown() + 1; - VLOG(1) << "Adding layer #" << new_num_layers - 1 << " to tree #" - << num_trees - 1 << " of ensemble of " << num_trees << " trees."; - // Update growable tree metadata. - tree_metadata->set_num_layers_grown(new_num_layers); - tree_metadata->set_is_finalized(new_num_layers >= max_tree_depth); - } - UpdateTreeWeightsIfDropout(ensemble_resource, dropout_seed); - return ensemble_resource->LastTree(); - } - - // Helper method to merge leaf weights as the tree is being grown. - boosted_trees::trees::Leaf* MergeLeafWeights( - const boosted_trees::trees::Leaf& source, - boosted_trees::trees::Leaf* dest) { - // Resolve leaf merging method based on how the trees are being grown. - if (learner_config_.growing_mode() == - boosted_trees::learner::LearnerConfig::WHOLE_TREE) { - // No merging occurs when building a whole tree at a time. - return dest; - } - - if (dest->leaf_case() == boosted_trees::trees::Leaf::LEAF_NOT_SET) { - // No merging is required. Just copy the source weights; - *dest = source; - return dest; - } - - // Handle leaf merging based on type. - switch (source.leaf_case()) { - case boosted_trees::trees::Leaf::kVector: { - // No-op if source is empty - const auto& src_vec = source.vector(); - if (src_vec.value_size() == 0) { - break; - } - CHECK(source.leaf_case() == dest->leaf_case()); - - // Dense add leaf vectors. - auto* dst_vec = dest->mutable_vector(); - CHECK(src_vec.value_size() == dst_vec->value_size()); - for (size_t idx = 0; idx < source.vector().value_size(); ++idx) { - (*dst_vec->mutable_value()->Mutable(idx)) += src_vec.value(idx); - } - break; - } - case boosted_trees::trees::Leaf::kSparseVector: { - // No-op if source is empty - const auto& src_vec = source.sparse_vector(); - CHECK(src_vec.value_size() == src_vec.index_size()); - if (src_vec.value_size() == 0) { - break; - } - CHECK(source.leaf_case() == dest->leaf_case()); - - // Get mapping of dimension to value for destination. - std::unordered_map dst_map; - auto* dst_vec = dest->mutable_sparse_vector(); - CHECK(dst_vec->value_size() == dst_vec->index_size()); - dst_map.reserve(dst_vec->value_size()); - for (size_t idx = 0; idx < dst_vec->value_size(); ++idx) { - dst_map[dst_vec->index(idx)] = dst_vec->value(idx); - } - // Sparse add source vector to destination vector. - for (size_t idx = 0; idx < src_vec.value_size(); ++idx) { - dst_map[src_vec.index(idx)] += src_vec.value(idx); - } - // Rebuild merged destination leaf. - dst_vec->clear_index(); - dst_vec->clear_value(); - for (const auto& entry : dst_map) { - dst_vec->add_index(entry.first); - dst_vec->add_value(entry.second); - } - break; - } - case boosted_trees::trees::Leaf::LEAF_NOT_SET: { - // No-op as there is nothing to merge. - break; - } - } - return dest; - } - - // Helper method to split a tree node and append its respective - // leaf children given the split candidate. - void SplitTreeNode( - const int32 node_id, SplitCandidate* split, - DecisionTreeConfig* tree_config, - const core::RefCountPtr& resource) { - // No-op if we have no real node. - CHECK(node_id < tree_config->nodes_size()) - << "Invalid node " << node_id << " to split."; - // Ensure new split node is valid. - CHECK(split->split_info.split_node().node_case() != TreeNode::NODE_NOT_SET); - CHECK(tree_config->nodes(node_id).node_case() == TreeNode::kLeaf) - << "Unexpected node type to split " - << tree_config->nodes(node_id).node_case() << " for node_id " << node_id - << ". Tree config: " << tree_config->DebugString(); - - // Add left leaf. - int32 left_id = tree_config->nodes_size(); - (*tree_config->add_nodes()->mutable_leaf()) = - *MergeLeafWeights(tree_config->nodes(node_id).leaf(), - split->split_info.mutable_left_child()); - - // Add right leaf. - int32 right_id = tree_config->nodes_size(); - (*tree_config->add_nodes()->mutable_leaf()) = - *MergeLeafWeights(tree_config->nodes(node_id).leaf(), - split->split_info.mutable_right_child()); - - // Link children and add them as new roots. - boosted_trees::trees::DecisionTree::LinkChildren( - {left_id, right_id}, split->split_info.mutable_split_node()); - - // Add split gain and, if needed, original leaf to node metadata. - TreeNodeMetadata* node_metadata = - split->split_info.mutable_split_node()->mutable_node_metadata(); - node_metadata->set_gain(split->gain); - if (learner_config_.pruning_mode() == - boosted_trees::learner::LearnerConfig::POST_PRUNE) { - (*node_metadata->mutable_original_leaf()) = - *tree_config->mutable_nodes(node_id)->mutable_leaf(); - } - - // Replace node in tree. - (*tree_config->mutable_nodes(node_id)) = - *split->split_info.mutable_split_node(); - if (learner_config_.constraints().max_number_of_unique_feature_columns()) { - resource->MaybeAddUsedHandler(split->handler_id); - } - } - - void SplitTreeLayer( - SplitCandidate* split, DecisionTreeConfig* tree_config, - const core::RefCountPtr& resource) { - int depth = 0; - while (depth < tree_config->nodes_size() && - tree_config->nodes(depth).node_case() != TreeNode::kLeaf) { - depth++; - } - CHECK(tree_config->nodes_size() > 0) - << "A tree must have at least one dummy leaf."; - // The number of new children. - int num_children = 1 << (depth + 1); - auto split_info = split->oblivious_split_info; - CHECK(num_children >= split_info.children_size()) - << "Too many new children, expected <= " << num_children << " and got " - << split_info.children_size(); - std::vector new_leaves; - new_leaves.reserve(num_children); - int next_id = 0; - for (int idx = 0; idx < num_children / 2; idx++) { - trees::Leaf old_leaf = - *tree_config->mutable_nodes(depth + idx)->mutable_leaf(); - // Check if a split was made for this leaf. - if (next_id < split_info.children_parent_id_size() && - depth + idx == split_info.children_parent_id(next_id)) { - // Add left leaf. - new_leaves.push_back(*MergeLeafWeights( - old_leaf, split_info.mutable_children(2 * next_id))); - // Add right leaf. - new_leaves.push_back(*MergeLeafWeights( - old_leaf, split_info.mutable_children(2 * next_id + 1))); - next_id++; - } else { - // If there is no split for this leaf, just duplicate it. - new_leaves.push_back(old_leaf); - new_leaves.push_back(old_leaf); - } - } - CHECK(next_id == split_info.children_parent_id_size()); - TreeNodeMetadata* split_metadata = - split_info.mutable_split_node()->mutable_node_metadata(); - split_metadata->set_gain(split->gain); - - TreeNode new_split = *split_info.mutable_split_node(); - // Move old children to metadata. - for (int idx = depth; idx < tree_config->nodes_size(); idx++) { - *new_split.mutable_node_metadata()->add_original_oblivious_leaves() = - *tree_config->mutable_nodes(idx)->mutable_leaf(); - } - // Add the new split to the tree_config in place before the children start. - *tree_config->mutable_nodes(depth) = new_split; - // Add the new children - int nodes_size = tree_config->nodes_size(); - for (int idx = 0; idx < num_children; idx++) { - if (idx + depth + 1 < nodes_size) { - // Update leaves that were already there. - *tree_config->mutable_nodes(idx + depth + 1)->mutable_leaf() = - new_leaves[idx]; - } else { - // Add new leaves. - *tree_config->add_nodes()->mutable_leaf() = new_leaves[idx]; - } - } - } - void PruneTree(boosted_trees::trees::DecisionTreeConfig* tree_config) { - // No-op if tree is empty. - if (tree_config->nodes_size() <= 0) { - return; - } - - // Copy nodes to temp vector and clear original tree. - std::vector tree_nodes; - tree_nodes.reserve(tree_config->nodes_size()); - for (auto& node : (*tree_config->mutable_nodes())) { - tree_nodes.push_back(node); - node.Clear(); - } - tree_config->clear_nodes(); - - // Prune the tree recursively starting from the root. - RecursivePruneTree(0, &tree_nodes); - - // Rebuild compacted tree. - (*tree_config->add_nodes()) = tree_nodes[0]; - std::unordered_map nodes_map; - nodes_map[0] = 0; - for (size_t node_idx = 0; node_idx < tree_nodes.size(); ++node_idx) { - // Skip pruned nodes. - auto& original_node = tree_nodes[node_idx]; - if (original_node.node_case() == TreeNode::NODE_NOT_SET) { - continue; - } - - // Find node mapped in tree ensemble. - auto mapped_node_it = nodes_map.find(node_idx); - CHECK(mapped_node_it != nodes_map.end()); - auto& mapped_node = (*tree_config->mutable_nodes(mapped_node_it->second)); - - // Get node children - auto children = - boosted_trees::trees::DecisionTree::GetChildren(original_node); - for (int32& child_idx : children) { - auto new_idx = tree_config->nodes_size(); - (*tree_config->add_nodes()) = tree_nodes[child_idx]; - nodes_map[child_idx] = new_idx; - child_idx = new_idx; - } - boosted_trees::trees::DecisionTree::LinkChildren(children, &mapped_node); - } - - // Check if there are any nodes with gain left. - if (tree_config->nodes_size() == 1 && - tree_config->nodes(0).node_metadata().gain() <= 0) { - // The whole tree should be pruned. - VLOG(2) << "No useful nodes left after post-pruning tree."; - tree_config->clear_nodes(); - } - } - - private: - boosted_trees::learner::LearnerConfig learner_config_; - int64 num_handlers_; - LearningRateDropoutDrivenConfig dropout_config_; - bool dropout_was_applied_; - bool center_bias_; -}; - -REGISTER_KERNEL_BUILDER(Name("GrowTreeEnsemble").Device(DEVICE_CPU), - GrowTreeEnsembleOp); - -class TreeEnsembleStatsOp : public OpKernel { - public: - explicit TreeEnsembleStatsOp(OpKernelConstruction* const context) - : OpKernel(context) {} - - void Compute(OpKernelContext* const context) override { - // Get decision tree ensemble. - core::RefCountPtr ensemble_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &ensemble_resource)); - tf_shared_lock l(*ensemble_resource->get_mutex()); - - // Get the stamp token. - const Tensor* stamp_token_t; - OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t)); - int64 stamp_token = stamp_token_t->scalar()(); - - // Only the Chief should run this Op and it is guaranteed to be in - // a consistent state so the stamps must always match. - CHECK(ensemble_resource->is_stamp_valid(stamp_token)); - const boosted_trees::trees::DecisionTreeEnsembleConfig& ensemble_config = - ensemble_resource->decision_tree_ensemble(); - - // Set tree stats. - Tensor* num_trees_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output( - "num_trees", TensorShape({}), &num_trees_t)); - Tensor* active_tree_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("active_tree", TensorShape({}), - &active_tree_t)); - Tensor* attempted_tree_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("attempted_trees", TensorShape({}), - &attempted_tree_t)); - - const int num_trees = ensemble_resource->num_trees(); - active_tree_t->scalar()() = num_trees; - num_trees_t->scalar()() = - (num_trees <= 0 || - ensemble_resource->LastTreeMetadata()->is_finalized()) - ? num_trees - : num_trees - 1; - attempted_tree_t->scalar()() = - ensemble_config.growing_metadata().num_trees_attempted(); - - // Set layer stats. - Tensor* num_layers_t = nullptr; - OP_REQUIRES_OK(context, context->allocate_output( - "num_layers", TensorShape({}), &num_layers_t)); - Tensor* active_layer_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("active_layer", TensorShape({}), - &active_layer_t)); - Tensor* attempted_layers_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output("attempted_layers", TensorShape({}), - &attempted_layers_t)); - - int64 num_layers = 0; - for (const auto& tree_metadata : ensemble_config.tree_metadata()) { - num_layers += tree_metadata.num_layers_grown(); - } - num_layers_t->scalar()() = num_layers; - int tree_metadata_size = ensemble_config.tree_metadata_size(); - active_layer_t->scalar()() = - tree_metadata_size > 0 - ? ensemble_config.tree_metadata(tree_metadata_size - 1) - .num_layers_grown() - : 0; - attempted_layers_t->scalar()() = - ensemble_config.growing_metadata().num_layers_attempted(); - } -}; - -REGISTER_KERNEL_BUILDER(Name("TreeEnsembleStats").Device(DEVICE_CPU), - TreeEnsembleStatsOp); - -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/BUILD b/tensorflow/contrib/boosted_trees/lib/BUILD deleted file mode 100644 index b6ce0f99a7b..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/BUILD +++ /dev/null @@ -1,447 +0,0 @@ -# Description: -# This directory contains common utilities used in boosted_trees. - -load("//tensorflow:tensorflow.bzl", "py_test", "tf_cc_binary", "tf_cc_test") - -package( - default_visibility = [ - "//tensorflow/contrib/boosted_trees:__subpackages__", - "//tensorflow/contrib/boosted_trees:friends", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -# Utils - -cc_library( - name = "utils", - srcs = [ - "utils/batch_features.cc", - "utils/dropout_utils.cc", - "utils/examples_iterable.cc", - "utils/parallel_for.cc", - "utils/sparse_column_iterable.cc", - "utils/tensor_utils.cc", - ], - hdrs = [ - "utils/batch_features.h", - "utils/dropout_utils.h", - "utils/example.h", - "utils/examples_iterable.h", - "utils/macros.h", - "utils/optional_value.h", - "utils/parallel_for.h", - "utils/random.h", - "utils/sparse_column_iterable.h", - "utils/tensor_utils.h", - ], - deps = [ - "//tensorflow/contrib/boosted_trees/proto:learner_proto_cc", - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - ], -) - -tf_cc_test( - name = "sparse_column_iterable_test", - size = "small", - srcs = ["utils/sparse_column_iterable_test.cc"], - deps = [ - ":utils", - "//tensorflow/core:tensor_testutil", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -tf_cc_test( - name = "examples_iterable_test", - size = "small", - srcs = ["utils/examples_iterable_test.cc"], - deps = [ - ":utils", - "//tensorflow/core:tensor_testutil", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "@com_google_absl//absl/algorithm:container", - ], -) - -tf_cc_test( - name = "example_test", - size = "small", - srcs = ["utils/example_test.cc"], - deps = [ - ":utils", - "//tensorflow/core:tensor_testutil", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -tf_cc_test( - name = "batch_features_test", - size = "small", - srcs = ["utils/batch_features_test.cc"], - deps = [ - ":utils", - "//tensorflow/core:lib", - "//tensorflow/core:tensor_testutil", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -tf_cc_test( - name = "dropout_utils_test", - size = "small", - srcs = ["utils/dropout_utils_test.cc"], - deps = [ - ":utils", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc", - "//tensorflow/core:lib", - "//tensorflow/core:tensor_testutil", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -# Models - -cc_library( - name = "models", - srcs = ["models/multiple_additive_trees.cc"], - hdrs = ["models/multiple_additive_trees.h"], - deps = [ - ":trees", - ":utils", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc", - "//tensorflow/core:framework_headers_lib", - ], -) - -tf_cc_test( - name = "multiple_additive_trees_test", - size = "small", - srcs = ["models/multiple_additive_trees_test.cc"], - deps = [ - ":batch_features_testutil", - ":models", - ":random_tree_gen", - "//tensorflow/contrib/boosted_trees/resources:decision_tree_ensemble_resource", - "//tensorflow/core:framework_headers_lib", - "//tensorflow/core:tensor_testutil", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -# Testutil - -cc_library( - name = "batch_features_testutil", - testonly = 1, - srcs = ["testutil/batch_features_testutil.cc"], - hdrs = ["testutil/batch_features_testutil.h"], - deps = [ - ":utils", - "//tensorflow/core:framework_headers_lib", - "//tensorflow/core:test", - "//tensorflow/core:testlib", - ], -) - -cc_library( - name = "random_tree_gen", - srcs = ["testutil/random_tree_gen.cc"], - hdrs = ["testutil/random_tree_gen.h"], - deps = [ - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc", - "//tensorflow/core:lib", - ], -) - -tf_cc_binary( - name = "random_tree_gen_main", - srcs = ["testutil/random_tree_gen_main.cc"], - deps = [ - ":random_tree_gen", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - ], -) - -# Quantiles - -cc_library( - name = "weighted_quantiles", - srcs = [], - hdrs = [ - "quantiles/weighted_quantiles_buffer.h", - "quantiles/weighted_quantiles_stream.h", - "quantiles/weighted_quantiles_summary.h", - ], - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/core:framework_headers_lib", - ], -) - -tf_cc_test( - name = "weighted_quantiles_buffer_test", - size = "small", - srcs = ["quantiles/weighted_quantiles_buffer_test.cc"], - deps = [ - ":weighted_quantiles", - "//tensorflow/core:lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -tf_cc_test( - name = "weighted_quantiles_summary_test", - size = "small", - srcs = ["quantiles/weighted_quantiles_summary_test.cc"], - deps = [ - ":weighted_quantiles", - "//tensorflow/core:lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -tf_cc_test( - name = "weighted_quantiles_stream_test", - size = "small", - srcs = ["quantiles/weighted_quantiles_stream_test.cc"], - deps = [ - ":weighted_quantiles", - "//tensorflow/core:lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -# Trees - -cc_library( - name = "trees", - srcs = ["trees/decision_tree.cc"], - hdrs = ["trees/decision_tree.h"], - deps = [ - "//tensorflow/contrib/boosted_trees/lib:utils", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc", - "//tensorflow/core:framework_headers_lib", - ], -) - -tf_cc_test( - name = "trees_test", - size = "small", - srcs = ["trees/decision_tree_test.cc"], - deps = [ - ":trees", - "//tensorflow/contrib/boosted_trees/lib:utils", - "//tensorflow/core:tensor_testutil", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -# Learner/batch - -py_library( - name = "base_split_handler", - srcs = ["learner/batch/base_split_handler.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/boosted_trees:batch_ops_utils_py", - "//tensorflow/python:control_flow_ops", - ], -) - -py_library( - name = "categorical_split_handler", - srcs = ["learner/batch/categorical_split_handler.py"], - srcs_version = "PY2AND3", - deps = [ - ":base_split_handler", - "//tensorflow/contrib/boosted_trees:split_handler_ops_py", - "//tensorflow/contrib/boosted_trees:stats_accumulator_ops_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:math_ops", - ], -) - -py_test( - name = "categorical_split_handler_test", - srcs = ["learner/batch/categorical_split_handler_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":categorical_split_handler", - "//tensorflow/contrib/boosted_trees/proto:learner_proto_py", - "//tensorflow/contrib/boosted_trees/proto:split_info_proto_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:resources", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:tensor_shape", - ], -) - -py_library( - name = "ordinal_split_handler", - srcs = ["learner/batch/ordinal_split_handler.py"], - srcs_version = "PY2AND3", - deps = [ - ":base_split_handler", - "//tensorflow/contrib/boosted_trees:quantile_ops_py", - "//tensorflow/contrib/boosted_trees:split_handler_ops_py", - "//tensorflow/contrib/boosted_trees:stats_accumulator_ops_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:function", - "//tensorflow/python:math_ops", - "//tensorflow/python:sparse_tensor", - ], -) - -py_test( - name = "ordinal_split_handler_test", - srcs = ["learner/batch/ordinal_split_handler_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":ordinal_split_handler", - "//tensorflow/contrib/boosted_trees/proto:learner_proto_py", - "//tensorflow/contrib/boosted_trees/proto:split_info_proto_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:resources", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:tensor_shape", - ], -) - -# Learner/Common - -cc_library( - name = "class-partition-key", - hdrs = ["learner/common/accumulators/class-partition-key.h"], - deps = [ - "//tensorflow/core:framework_headers_lib", - ], -) - -cc_library( - name = "feature-stats-accumulator", - hdrs = ["learner/common/accumulators/feature-stats-accumulator.h"], - deps = [ - ":class-partition-key", - ], -) - -tf_cc_test( - name = "feature-stats-accumulator_test", - size = "small", - srcs = ["learner/common/accumulators/feature-stats-accumulator_test.cc"], - deps = [ - ":feature-stats-accumulator", - "//tensorflow/core:lib", - "//tensorflow/core:tensor_testutil", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -cc_library( - name = "example_partitioner", - srcs = ["learner/common/partitioners/example_partitioner.cc"], - hdrs = ["learner/common/partitioners/example_partitioner.h"], - deps = [ - "//tensorflow/contrib/boosted_trees/lib:trees", - "//tensorflow/contrib/boosted_trees/lib:utils", - "//tensorflow/core:framework_headers_lib", - ], -) - -tf_cc_test( - name = "example_partitioner_test", - size = "small", - srcs = ["learner/common/partitioners/example_partitioner_test.cc"], - deps = [ - ":example_partitioner", - "//tensorflow/core:tensor_testutil", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -# Learner/stochastic -cc_library( - name = "gradient-stats", - hdrs = ["learner/common/stats/gradient-stats.h"], - deps = [ - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - ], -) - -cc_library( - name = "node-stats", - hdrs = ["learner/common/stats/node-stats.h"], - deps = [ - ":gradient-stats", - "//tensorflow/contrib/boosted_trees/proto:learner_proto_cc", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc", - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - ], -) - -cc_library( - name = "split-stats", - hdrs = ["learner/common/stats/split-stats.h"], - deps = [ - ":node-stats", - ], -) - -cc_library( - name = "feature-split-candidate", - hdrs = ["learner/common/stats/feature-split-candidate.h"], - deps = [ - ":split-stats", - "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc", - ], -) - -tf_cc_test( - name = "node-stats_test", - size = "small", - srcs = ["learner/common/stats/node-stats_test.cc"], - deps = [ - ":node-stats", - "//tensorflow/core:tensor_testutil", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) diff --git a/tensorflow/contrib/boosted_trees/lib/learner/batch/base_split_handler.py b/tensorflow/contrib/boosted_trees/lib/learner/batch/base_split_handler.py deleted file mode 100644 index efa2ab1dad8..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/batch/base_split_handler.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Base class for creating split nodes using one or more features.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - -from tensorflow.contrib.boosted_trees.python.ops import batch_ops_utils -from tensorflow.python.ops import control_flow_ops - - -@six.add_metaclass(abc.ABCMeta) -class BaseSplitHandler(object): - """Abstract Base class defining split handlers interface.""" - - def __init__(self, - l1_regularization, - l2_regularization, - tree_complexity_regularization, - min_node_weight, - feature_column_group_id, - gradient_shape, - hessian_shape, - multiclass_strategy, - loss_uses_sum_reduction=False, - name=None): - """Constructor for BaseSplitHandler. - - Args: - l1_regularization: L1 regularization applied for this split handler. - l2_regularization: L2 regularization applied for this split handler. - tree_complexity_regularization: Tree complexity regularization applied - for this split handler. - min_node_weight: Minimum sum of weights of examples in each partition to - be considered for splitting. - feature_column_group_id: Feature column group index. - gradient_shape: A TensorShape, containing shape of gradients. - hessian_shape: A TensorShape, containing shape of hessians. - multiclass_strategy: Strategy describing how to treat multiclass problems. - loss_uses_sum_reduction: A scalar boolean tensor that specifies whether - SUM or MEAN reduction was used for the loss. - name: An optional handler name. - """ - self._l1_regularization = l1_regularization - self._l2_regularization = l2_regularization - self._tree_complexity_regularization = tree_complexity_regularization - self._min_node_weight = min_node_weight - self._feature_column_group_id = feature_column_group_id - self._name = name or "" - self._multiclass_strategy = multiclass_strategy - self._hessian_shape = hessian_shape - self._gradient_shape = gradient_shape - self._loss_uses_sum_reduction = loss_uses_sum_reduction - - def scheduled_reads(self): - """Returns the list of `ScheduledOp`s required for update_stats.""" - return [] - - @abc.abstractmethod - def update_stats(self, stamp_token, example_partition_ids, gradients, - hessians, empty_gradients, empty_hessians, weights, - is_active, scheduled_reads): - """Updates the state for this split handler. - - Args: - stamp_token: An int32 scalar tensor containing the current stamp token. - example_partition_ids: A dense tensor, containing an int32 for each - example which is the partition id that the example ends up in. - gradients: A dense tensor of gradients. - hessians: A dense tensor of hessians. - empty_gradients: A dense empty tensor of the same shape (for dimensions > - 0) as gradients. - empty_hessians: A dense empty tensor of the same shape (for dimensions > - 0) as hessians. - weights: A dense float32 tensor with a weight for each example. - is_active: A boolean tensor that says if this handler is active or not. - One value for the current layer and one value for the next layer. - scheduled_reads: List of results from the scheduled reads. - - Returns: - A tuple of the op that updates the stats for this handler and a list of - `ScheduledOp`s. - """ - - def update_stats_sync(self, stamp_token, example_partition_ids, gradients, - hessians, empty_gradients, empty_hessians, weights, - is_active): - """Updates the state for this split handler running the scheduled I/O. - - Args: - stamp_token: An int32 scalar tensor containing the current stamp token. - example_partition_ids: A dense tensor, containing an int32 for each - example which is the partition id that the example ends up in. - gradients: A dense tensor of gradients. - hessians: A dense tensor of hessians. - empty_gradients: A dense empty tensor of the same shape (for dimensions > - 0) as gradients. - empty_hessians: A dense empty tensor of the same shape (for dimensions > - 0) as hessians. - weights: A dense float32 tensor with a weight for each example. - is_active: A boolean tensor that says if this handler is active or not. - One value for the current layer and one value for the next layer. - - Returns: - Op that updates the stats for this handler. - """ - handler_reads = {self: self.scheduled_reads()} - handler_results = batch_ops_utils.run_handler_scheduled_ops( - handler_reads, stamp_token, None) - update_1, scheduled_updates = self.update_stats( - stamp_token, example_partition_ids, gradients, hessians, - empty_gradients, empty_hessians, weights, is_active, - handler_results[self]) - update_2 = batch_ops_utils.run_handler_scheduled_ops({ - self: scheduled_updates - }, stamp_token, None) - return control_flow_ops.group(update_1, *update_2[self]) - - @abc.abstractmethod - def reset(self, stamp_token, next_stamp_token): - """Resets the state maintained by the handler.""" - - @abc.abstractmethod - def make_splits(self, stamp_token, next_stamp_token, class_id): - """Create the best split using the accumulated stats and flush the state. - - This should only be called by the master. - - Args: - stamp_token: An int32 scalar tensor containing the current stamp token. - next_stamp_token: An int32 scalar tensor containing the stamp token for - the next iteration. - class_id: what class id the handler gathers stats for (for tree per class - strategy). When class_id=-1, the strategy is not tree per class. - Returns: - A tuple (are_splits_ready, partition_id, gain, split_info) where - are_splits_ready is a scalar boolean tensor, partition_id is a rank 1, - int32 tensor, gain is a rank 1 float32 tensor and split_info is a rank 1 - string tensor containing serialized SplitInfo protos. - """ diff --git a/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler.py b/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler.py deleted file mode 100644 index 22ad181fc3f..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implementation of handler for split nodes for categorical columns.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.boosted_trees.lib.learner.batch import base_split_handler -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.boosted_trees.python.ops import split_handler_ops -from tensorflow.contrib.boosted_trees.python.ops import stats_accumulator_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops - -_BIAS_FEATURE_ID = int(dtypes.int64.min) - - -class EqualitySplitHandler(base_split_handler.BaseSplitHandler): - """Creates equality split type for categorical features.""" - - def __init__(self, - sparse_int_column, - l1_regularization, - l2_regularization, - tree_complexity_regularization, - min_node_weight, - feature_column_group_id, - gradient_shape, - hessian_shape, - multiclass_strategy, - init_stamp_token=0, - loss_uses_sum_reduction=False, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE, - name=None): - """Initialize the internal state for this split handler. - - Args: - sparse_int_column: A `SparseTensor` column with int64 values associated - with this handler. - l1_regularization: L1 regularization applied for this split handler. - l2_regularization: L2 regularization applied for this split handler. - tree_complexity_regularization: Tree complexity regularization applied - for this split handler. - min_node_weight: Minimum sum of weights of examples in each partition to - be considered for splitting. - feature_column_group_id: Feature column group index. - gradient_shape: A TensorShape, containing shape of gradients. - hessian_shape: A TensorShape, containing shape of hessians. - multiclass_strategy: Strategy describing how to treat multiclass problems. - init_stamp_token: A tensor containing an scalar for initial stamp of the - stamped objects. - loss_uses_sum_reduction: A scalar boolean tensor that specifies whether - SUM or MEAN reduction was used for the loss. - weak_learner_type: Specifies the type of weak learner to use. - name: An optional handler name. - """ - super(EqualitySplitHandler, self).__init__( - l1_regularization=l1_regularization, - l2_regularization=l2_regularization, - tree_complexity_regularization=tree_complexity_regularization, - min_node_weight=min_node_weight, - feature_column_group_id=feature_column_group_id, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=multiclass_strategy, - loss_uses_sum_reduction=loss_uses_sum_reduction, - name=name) - self._stats_accumulator = stats_accumulator_ops.StatsAccumulator( - init_stamp_token, - gradient_shape, - hessian_shape, - name="StatsAccumulator/{}".format(self._name)) - self._sparse_int_column = sparse_int_column - self._weak_learner_type = weak_learner_type - - def update_stats(self, stamp_token, example_partition_ids, gradients, - hessians, empty_gradients, empty_hessians, weights, - is_active, scheduled_reads): - """Updates the state for equality split handler. - - Args: - stamp_token: An int32 scalar tensor containing the current stamp token. - example_partition_ids: A dense tensor, containing an int32 for each - example which is the partition id that the example ends up in. - gradients: A dense tensor of gradients. - hessians: A dense tensor of hessians. - empty_gradients: A dense empty tensor of the same shape (for dimensions > - 0) as gradients. - empty_hessians: A dense empty tensor of the same shape (for dimensions > - 0) as hessians. - weights: A dense float32 tensor with a weight for each example. - is_active: A boolean tensor that says if this handler is active or not. - One value for the current layer and one value for the next layer. - scheduled_reads: List of results from the scheduled reads. - Returns: - The op that updates the stats for this handler. - Raises: - ValueError: If example_columns is not a single sparse column. - - """ - del scheduled_reads # Unused by the categorical split handler. - - def not_active_inputs(): - return (constant_op.constant([], dtype=dtypes.int32), - constant_op.constant_v1([], dtype=dtypes.int64, shape=[1, 2]), - empty_gradients, empty_hessians) - - def active_inputs(): - """The normal flow when the handler is active.""" - # Remove the second column of example indices matrix since it is not - # useful. - example_indices, _ = array_ops.split( - self._sparse_int_column.indices, num_or_size_splits=2, axis=1) - example_indices = array_ops.squeeze(example_indices, [1]) - - filtered_gradients = array_ops.gather(gradients, example_indices) - filtered_hessians = array_ops.gather(hessians, example_indices) - filtered_partition_ids = array_ops.gather(example_partition_ids, - example_indices) - unique_partitions, mapped_partitions = array_ops.unique( - example_partition_ids) - - # Compute aggregate stats for each partition. - # The bias is computed on gradients and hessians (and not - # filtered_gradients) which have exactly one value per example, so we - # don't double count a gradient in multivalent columns. - # Since unsorted_segment_sum can be numerically unstable, use 64bit - # operation. - gradients64 = math_ops.cast(gradients, dtypes.float64) - hessians64 = math_ops.cast(hessians, dtypes.float64) - per_partition_gradients = math_ops.unsorted_segment_sum( - gradients64, mapped_partitions, array_ops.size(unique_partitions)) - per_partition_hessians = math_ops.unsorted_segment_sum( - hessians64, mapped_partitions, array_ops.size(unique_partitions)) - per_partition_gradients = math_ops.cast(per_partition_gradients, - dtypes.float32) - per_partition_hessians = math_ops.cast(per_partition_hessians, - dtypes.float32) - # Prepend a bias feature per partition that accumulates the stats for all - # examples in that partition. - # Bias is added to the stats even if there are no examples with values in - # the current sparse column. The reason is that the other example batches - # might have values in these partitions so we have to keep the bias - # updated. - bias_feature_ids = array_ops.fill( - array_ops.shape(unique_partitions), _BIAS_FEATURE_ID) - bias_feature_ids = math_ops.cast(bias_feature_ids, dtypes.int64) - partition_ids = array_ops.concat( - [unique_partitions, filtered_partition_ids], 0) - filtered_gradients = array_ops.concat( - [per_partition_gradients, filtered_gradients], 0) - filtered_hessians = array_ops.concat( - [per_partition_hessians, filtered_hessians], 0) - feature_ids = array_ops.concat( - [bias_feature_ids, self._sparse_int_column.values], 0) - # Dimension is always zero for sparse int features. - dimension_ids = array_ops.zeros_like(feature_ids, dtype=dtypes.int64) - feature_ids_and_dimensions = array_ops.stack( - [feature_ids, dimension_ids], axis=1) - return (partition_ids, feature_ids_and_dimensions, filtered_gradients, - filtered_hessians) - - partition_ids, feature_ids, gradients_out, hessians_out = ( - control_flow_ops.cond(is_active[0], active_inputs, not_active_inputs)) - result = self._stats_accumulator.schedule_add(partition_ids, feature_ids, - gradients_out, hessians_out) - return (control_flow_ops.no_op(), [result]) - - def make_splits(self, stamp_token, next_stamp_token, class_id): - """Create the best split using the accumulated stats and flush the state.""" - # Get the aggregated gradients and hessians per - # pair. - num_minibatches, partition_ids, feature_ids, gradients, hessians = ( - self._stats_accumulator.flush(stamp_token, next_stamp_token)) - # For sum_reduction, we don't need to divide by number of minibatches. - - num_minibatches = control_flow_ops.cond( - ops.convert_to_tensor(self._loss_uses_sum_reduction), - lambda: math_ops.cast(1, dtypes.int64), - lambda: num_minibatches) - partition_ids, gains, split_infos = ( - split_handler_ops.build_categorical_equality_splits( - num_minibatches=num_minibatches, - partition_ids=partition_ids, - feature_ids=feature_ids, - gradients=gradients, - hessians=hessians, - class_id=class_id, - feature_column_group_id=self._feature_column_group_id, - l1_regularization=self._l1_regularization, - l2_regularization=self._l2_regularization, - tree_complexity_regularization=self._tree_complexity_regularization, - min_node_weight=self._min_node_weight, - bias_feature_id=_BIAS_FEATURE_ID, - multiclass_strategy=self._multiclass_strategy, - weak_learner_type=self._weak_learner_type)) - # There are no warm-up rounds needed in the equality column handler. So we - # always return ready. - are_splits_ready = constant_op.constant(True) - return (are_splits_ready, partition_ids, gains, split_infos) - - def reset(self, stamp_token, next_stamp_token): - reset = self._stats_accumulator.flush(stamp_token, next_stamp_token) - return reset diff --git a/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler_test.py b/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler_test.py deleted file mode 100644 index 04dec603667..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler_test.py +++ /dev/null @@ -1,668 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Test for checking stats accumulator related ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.boosted_trees.lib.learner.batch import categorical_split_handler -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.boosted_trees.proto import split_info_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import resources -from tensorflow.python.platform import googletest - - -def get_empty_tensors(gradient_shape, hessian_shape): - empty_hess_shape = [1] + hessian_shape.as_list() - empty_grad_shape = [1] + gradient_shape.as_list() - - empty_gradients = constant_op.constant_v1( - [], dtype=dtypes.float32, shape=empty_grad_shape) - empty_hessians = constant_op.constant_v1( - [], dtype=dtypes.float32, shape=empty_hess_shape) - - return empty_gradients, empty_hessians - - -class EqualitySplitHandlerTest(test_util.TensorFlowTestCase): - - def testGenerateFeatureSplitCandidates(self): - with self.cached_session() as sess: - # The data looks like the following: - # Example | Gradients | Partition | Feature ID | - # i0 | (0.2, 0.12) | 0 | 1,2 | - # i1 | (-0.5, 0.07) | 0 | | - # i2 | (1.2, 0.2) | 0 | 2 | - # i3 | (4.0, 0.13) | 1 | 1 | - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - partition_ids = [0, 0, 0, 1] - indices = [[0, 0], [0, 1], [2, 0], [3, 0]] - values = array_ops.constant([1, 2, 2, 1], dtype=dtypes.int64) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = categorical_split_handler.EqualitySplitHandler( - l1_regularization=0.1, - l2_regularization=1, - tree_complexity_regularization=0, - min_node_weight=0, - sparse_int_column=sparse_tensor.SparseTensor(indices, values, [4, 1]), - feature_column_group_id=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, - init_stamp_token=0) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - update_2 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - - with ops.control_dependencies([update_1, update_2]): - are_splits_ready, partitions, gains, splits = ( - split_handler.make_splits(0, 1, class_id)) - are_splits_ready, partitions, gains, splits = (sess.run( - [are_splits_ready, partitions, gains, splits])) - self.assertTrue(are_splits_ready) - self.assertAllEqual([0, 1], partitions) - - # Check the split on partition 0. - # -(0.2 + 1.2 - 0.1) / (0.12 + 0.2 + 1) - expected_left_weight = -0.9848484848484846 - - # (0.2 + 1.2 - 0.1) ** 2 / (0.12 + 0.2 + 1) - expected_left_gain = 1.2803030303030298 - - # -(-0.5 + 0.1) / (0.07 + 1) - expected_right_weight = 0.37383177570093457 - - # (-0.5 + 0.1) ** 2 / (0.07 + 1) - expected_right_gain = 0.14953271028037385 - - # (0.2 + -0.5 + 1.2 - 0.1) ** 2 / (0.12 + 0.07 + 0.2 + 1) - expected_bias_gain = 0.46043165467625885 - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.categorical_id_binary_split - - self.assertEqual(0, split_node.feature_column) - - self.assertEqual(2, split_node.feature_id) - - self.assertAllClose( - expected_left_gain + expected_right_gain - expected_bias_gain, gains[0], - 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - # Check the split on partition 1. - # (-4 + 0.1) / (0.13 + 1) - expected_left_weight = -3.4513274336283186 - # (-4 + 0.1) ** 2 / (0.13 + 1) - expected_left_gain = 13.460176991150442 - expected_right_weight = 0 - expected_right_gain = 0 - # (-4 + 0.1) ** 2 / (0.13 + 1) - expected_bias_gain = 13.460176991150442 - - # Verify candidate for partition 1, there's only one active feature here - # so zero gain is expected. - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.categorical_id_binary_split - self.assertAllClose(0.0, gains[1], 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - self.assertEqual(0, split_node.feature_column) - - self.assertEqual(1, split_node.feature_id) - - def testObliviousFeatureSplitGeneration(self): - with self.cached_session() as sess: - # The data looks like the following: - # Example | Gradients | Partition | Feature ID | - # i0 | (0.2, 0.12) | 1 | 1 | - # i1 | (-0.5, 0.07) | 1 | 2 | - # i2 | (1.2, 0.2) | 1 | 1 | - # i3 | (4.0, 0.13) | 2 | 2 | - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - partition_ids = [1, 1, 1, 2] - indices = [[0, 0], [1, 0], [2, 0], [3, 0]] - values = array_ops.constant([1, 2, 1, 2], dtype=dtypes.int64) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = categorical_split_handler.EqualitySplitHandler( - l1_regularization=0.1, - l2_regularization=1, - tree_complexity_regularization=0, - min_node_weight=0, - sparse_int_column=sparse_tensor.SparseTensor(indices, values, [4, 1]), - feature_column_group_id=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, - init_stamp_token=0, - weak_learner_type=learner_pb2.LearnerConfig.OBLIVIOUS_DECISION_TREE) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - update_2 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - - with ops.control_dependencies([update_1, update_2]): - are_splits_ready, partitions, gains, splits = ( - split_handler.make_splits(0, 1, class_id)) - are_splits_ready, partitions, gains, splits = ( - sess.run([are_splits_ready, partitions, gains, splits])) - self.assertTrue(are_splits_ready) - self.assertAllEqual([1, 2], partitions) - - # For partition 1. - # -(0.2 + 1.2 - 0.1) / (0.12 + 0.2 + 1) - expected_left_weight1 = -0.9848484848484846 - # (0.2 + 1.2 - 0.1) ** 2 / (0.12 + 0.2 + 1) - expected_left_gain1 = 1.2803030303030298 - - # -(-0.5 + 0.1) / (0.07 + 1) - expected_right_weight1 = 0.37383177570093457 - - # (-0.5 + 0.1) ** 2 / (0.07 + 1) - expected_right_gain1 = 0.14953271028037385 - - # (0.2 + -0.5 + 1.2 - 0.1) ** 2 / (0.12 + 0.07 + 0.2 + 1) - expected_bias_gain1 = 0.46043165467625885 - - split_info = split_info_pb2.ObliviousSplitInfo() - split_info.ParseFromString(splits[0]) - # Children of partition 1. - left_child = split_info.children[0].vector - right_child = split_info.children[1].vector - split_node = split_info.split_node.oblivious_categorical_id_binary_split - - self.assertEqual(0, split_node.feature_column) - self.assertEqual(1, split_node.feature_id) - self.assertAllClose([expected_left_weight1], left_child.value, 0.00001) - self.assertAllClose([expected_right_weight1], right_child.value, 0.00001) - - # For partition2. - expected_left_weight2 = 0 - expected_left_gain2 = 0 - # -(4 - 0.1) / (0.13 + 1) - expected_right_weight2 = -3.4513274336283186 - # (4 - 0.1) ** 2 / (0.13 + 1) - expected_right_gain2 = 13.460176991150442 - # (4 - 0.1) ** 2 / (0.13 + 1) - expected_bias_gain2 = 13.460176991150442 - - # Children of partition 2. - left_child = split_info.children[2].vector - right_child = split_info.children[3].vector - self.assertAllClose([expected_left_weight2], left_child.value, 0.00001) - self.assertAllClose([expected_right_weight2], right_child.value, 0.00001) - - self.assertAllClose( - expected_left_gain1 + expected_right_gain1 - expected_bias_gain1 + - expected_left_gain2 + expected_right_gain2 - expected_bias_gain2, - gains[0], 0.00001) - - def testGenerateFeatureSplitCandidatesSumReduction(self): - with self.cached_session() as sess: - # The data looks like the following: - # Example | Gradients | Partition | Feature ID | - # i0 | (0.2, 0.12) | 0 | 1,2 | - # i1 | (-0.5, 0.07) | 0 | | - # i2 | (1.2, 0.2) | 0 | 2 | - # i3 | (4.0, 0.13) | 1 | 1 | - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - partition_ids = [0, 0, 0, 1] - indices = [[0, 0], [0, 1], [2, 0], [3, 0]] - values = array_ops.constant([1, 2, 2, 1], dtype=dtypes.int64) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = categorical_split_handler.EqualitySplitHandler( - l1_regularization=0.1, - l2_regularization=1, - tree_complexity_regularization=0, - min_node_weight=0, - sparse_int_column=sparse_tensor.SparseTensor(indices, values, [4, 1]), - feature_column_group_id=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, - init_stamp_token=0, - loss_uses_sum_reduction=True) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - update_2 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1, update_2]): - are_splits_ready, partitions, gains, splits = ( - split_handler.make_splits(0, 1, class_id)) - are_splits_ready, partitions, gains, splits = ( - sess.run([are_splits_ready, partitions, gains, splits])) - self.assertTrue(are_splits_ready) - self.assertAllEqual([0, 1], partitions) - - # Check the split on partition 0. - # -(0.4 + 2.4 - 0.1) / (0.24 + 0.4 + 1) - expected_left_weight = -1.6463414634146338 - - # (0.4 + 2.4 - 0.1) ** 2 / (0.24 + 0.4 + 1) - expected_left_gain = 4.445121951219511 - - # -(-1 + 0.1) / (0.14 + 1) - expected_right_weight = 0.789473684211 - - # (-1 + 0.1) ** 2 / (0.14 + 1) - expected_right_gain = 0.710526315789 - - # (0.4 + -1 + 2.4 - 0.1) ** 2 / (0.24 + 0.14 + 0.4 + 1) - expected_bias_gain = 1.6235955056179772 - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.categorical_id_binary_split - - self.assertEqual(0, split_node.feature_column) - - self.assertEqual(2, split_node.feature_id) - - self.assertAllClose( - expected_left_gain + expected_right_gain - expected_bias_gain, gains[0], - 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - # Check the split on partition 1. - # (-8 + 0.1) / (0.26 + 1) - expected_left_weight = -6.26984126984 - # (-8 + 0.1) ** 2 / (0.26 + 1) - expected_left_gain = 49.5317460317 - expected_right_weight = 0 - expected_right_gain = 0 - # (-8 + 0.1) ** 2 / (0.26 + 1) - expected_bias_gain = 49.5317460317 - - # Verify candidate for partition 1, there's only one active feature here - # so zero gain is expected. - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.categorical_id_binary_split - self.assertAllClose(0.0, gains[1], 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - self.assertEqual(0, split_node.feature_column) - - self.assertEqual(1, split_node.feature_id) - - def testGenerateFeatureSplitCandidatesMulticlass(self): - with self.cached_session() as sess: - # Batch size is 4, 2 gradients per each instance. - gradients = array_ops.constant( - [[0.2, 0.1], [-0.5, 0.2], [1.2, 3.4], [4.0, -3.5]], shape=[4, 2]) - # 2x2 matrix for each instance - hessian_0 = [[0.12, 0.02], [0.3, 0.11]] - hessian_1 = [[0.07, -0.2], [-0.5, 0.2]] - hessian_2 = [[0.2, -0.23], [-0.8, 0.9]] - hessian_3 = [[0.13, -0.3], [-1.5, 2.2]] - hessians = array_ops.constant( - [hessian_0, hessian_1, hessian_2, hessian_3]) - - partition_ids = [0, 0, 0, 1] - indices = [[0, 0], [0, 1], [2, 0], [3, 0]] - values = array_ops.constant([1, 2, 2, 1], dtype=dtypes.int64) - - hessians = array_ops.constant( - [hessian_0, hessian_1, hessian_2, hessian_3]) - partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - - gradient_shape = tensor_shape.TensorShape([2]) - hessian_shape = tensor_shape.TensorShape([2, 2]) - class_id = -1 - - split_handler = categorical_split_handler.EqualitySplitHandler( - l1_regularization=0.1, - l2_regularization=1, - tree_complexity_regularization=0, - min_node_weight=0, - sparse_int_column=sparse_tensor.SparseTensor(indices, values, [4, 1]), - feature_column_group_id=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.FULL_HESSIAN, - init_stamp_token=0) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready, partitions, gains, splits = ( - split_handler.make_splits(0, 1, class_id)) - are_splits_ready, partitions, gains, splits = (sess.run( - [are_splits_ready, partitions, gains, splits])) - self.assertTrue(are_splits_ready) - self.assertAllEqual([0, 1], partitions) - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.categorical_id_binary_split - # Each leaf has 2 element vector. - self.assertEqual(2, len(left_child.value)) - self.assertEqual(2, len(right_child.value)) - self.assertEqual(1, split_node.feature_id) - - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.categorical_id_binary_split - self.assertEqual(2, len(left_child.value)) - self.assertEqual(0, len(right_child.value)) - self.assertEqual(1, split_node.feature_id) - - def testEmpty(self): - with self.cached_session() as sess: - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - partition_ids = [0, 0, 0, 1] - indices = constant_op.constant_v1([], dtype=dtypes.int64, shape=[0, 2]) - values = constant_op.constant_v1([], dtype=dtypes.int64) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = categorical_split_handler.EqualitySplitHandler( - l1_regularization=0.1, - l2_regularization=1, - tree_complexity_regularization=0, - min_node_weight=0, - sparse_int_column=sparse_tensor.SparseTensor(indices, values, [4, 1]), - feature_column_group_id=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, - init_stamp_token=0) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready, partitions, gains, splits = ( - split_handler.make_splits(0, 1, class_id)) - are_splits_ready, partitions, gains, splits = (sess.run( - [are_splits_ready, partitions, gains, splits])) - self.assertTrue(are_splits_ready) - self.assertEqual(len(partitions), 0) - self.assertEqual(len(gains), 0) - self.assertEqual(len(splits), 0) - - def testInactive(self): - with self.cached_session() as sess: - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - partition_ids = [0, 0, 0, 1] - indices = [[0, 0], [0, 1], [2, 0], [3, 0]] - values = array_ops.constant([1, 2, 2, 1], dtype=dtypes.int64) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = categorical_split_handler.EqualitySplitHandler( - l1_regularization=0.1, - l2_regularization=1, - tree_complexity_regularization=0, - min_node_weight=0, - sparse_int_column=sparse_tensor.SparseTensor(indices, values, [4, 1]), - feature_column_group_id=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, - init_stamp_token=0) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([False, False])) - with ops.control_dependencies([update_1]): - are_splits_ready, partitions, gains, splits = ( - split_handler.make_splits(0, 1, class_id)) - are_splits_ready, partitions, gains, splits = (sess.run( - [are_splits_ready, partitions, gains, splits])) - self.assertTrue(are_splits_ready) - self.assertEqual(len(partitions), 0) - self.assertEqual(len(gains), 0) - self.assertEqual(len(splits), 0) - - def testLastOneEmpty(self): - with self.cached_session() as sess: - # The data looks like the following: - # Example | Gradients | Partition | Feature ID | - # i0 | (0.2, 0.12) | 0 | 1,2 | - # i1 | (-0.5, 0.07) | 0 | | - # i2 | (1.2, 0.2) | 0 | 2 | - # i3 | (4.0, 0.13) | 1 | | - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - partition_ids = [0, 0, 0, 1] - indices = [[0, 0], [0, 1], [2, 0]] - values = array_ops.constant([1, 2, 2], dtype=dtypes.int64) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = categorical_split_handler.EqualitySplitHandler( - l1_regularization=0.1, - l2_regularization=1, - tree_complexity_regularization=0, - min_node_weight=0, - sparse_int_column=sparse_tensor.SparseTensor(indices, values, [4, 1]), - feature_column_group_id=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, - init_stamp_token=0) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready, partitions, gains, splits = ( - split_handler.make_splits(0, 1, class_id)) - are_splits_ready, partitions, gains, splits = ( - sess.run([are_splits_ready, partitions, gains, splits])) - self.assertTrue(are_splits_ready) - self.assertAllEqual([0], partitions) - - # Check the split on partition 0. - # -(0.2 + 1.2 - 0.1) / (0.12 + 0.2 + 1) - expected_left_weight = -0.9848484848484846 - - # (0.2 + 1.2 - 0.1) ** 2 / (0.12 + 0.2 + 1) - expected_left_gain = 1.2803030303030298 - - # -(-0.5 + 0.1) / (0.07 + 1) - expected_right_weight = 0.37383177570093457 - - # (-0.5 + 0.1) ** 2 / (0.07 + 1) - expected_right_gain = 0.14953271028037385 - - # (0.2 + -0.5 + 1.2 - 0.1) ** 2 / (0.12 + 0.07 + 0.2 + 1) - expected_bias_gain = 0.46043165467625885 - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.categorical_id_binary_split - - self.assertEqual(0, split_node.feature_column) - - self.assertEqual(2, split_node.feature_id) - - self.assertAllClose( - expected_left_gain + expected_right_gain - expected_bias_gain, gains[0], - 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler.py b/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler.py deleted file mode 100644 index 75881945fde..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler.py +++ /dev/null @@ -1,732 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implementation of handler for split nodes for float columns. - -The general idea in batch split finding is that each handler will accumulate its -own statistics on multiple workers. After some steps, the master runs -make_splits() sub-graph of each handler and each handler returns its best split -per partition. - -The way we ensure consistency of statistics is by using stamp_tokens for read -and write operations. During each update of the model, a new stamp token is -created. This stamp token makes sure that updates from the previous iterations -are not included in the statistics for this iteration. - -Inequality splits for float features are created similar to the method described -in Approximate Algorithm described in https://arxiv.org/pdf/1603.02754v3.pdf. -Weighted quantiles of the feature columns are computed in a distributed fashion -using quantile_ops.quantile_accumulator. -After certain number of steps of parallel accumulation of quantile statistics, -we decide on bucket boundaries. These bucket boundaries are then used for the -next N steps to accumulate gradients and hessians per bucket. - -In this implementation, we gather quantile statistics and gradient statistics -concurrently. That means that we don't wait until we have enough quantile -statistics for bucketization before we start gathering gradient stats. Instead -during each step we create quantile stats for the next iteration and use the -previous quantile buckets for gradient stats accumulation. -In make_splits, we do these steps: -1) Get the buckets that were used creating for the gradient stats. -2) Create bucket boundaries for the next N iterations and clear the accumulated - quantile stats. -n3) Get the accumulated gradient stats and clear the accumulator. This step can - run in parallel to step 2. -4) For each leaf node in the current tree (partition): - 4.1) Get the overall gain computed with gradients and hessians of all - examples that end up in this partition. - 4.2) Compute tensors of left and right cumulative sum of gradients, hessians - and gain. The first dimension of these tensors are the bucket - boundaries. - 4.3) Find the gains for all bucket boundaries: - split_gains = left_gain + right_gain - overall_gain. - 4.4) Find the bucket boundary that has the best gain (argmax(split_gains)) - 4.5) For Sparse handler, we also consider the gain for when the examples go - the left child and when the examples go to the right child and pick the - default direction that yields the most gain. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re - -from tensorflow.contrib.boosted_trees.lib.learner.batch import base_split_handler -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.boosted_trees.python.ops import gen_quantile_ops -from tensorflow.contrib.boosted_trees.python.ops import gen_stats_accumulator_ops -from tensorflow.contrib.boosted_trees.python.ops import quantile_ops -from tensorflow.contrib.boosted_trees.python.ops import split_handler_ops -from tensorflow.contrib.boosted_trees.python.ops import stats_accumulator_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import function -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops - - -_BIAS_FEATURE_ID = -1 -# Pattern to remove all non alpha numeric from a string. -_PATTERN = re.compile(r"[\W_]+") - - -class InequalitySplitHandler(base_split_handler.BaseSplitHandler): - """Base class for handlers of inequality splits.""" - - def __init__(self, - l1_regularization, - l2_regularization, - tree_complexity_regularization, - min_node_weight, - feature_column_group_id, - epsilon, - num_quantiles, - gradient_shape, - hessian_shape, - multiclass_strategy, - init_stamp_token=0, - loss_uses_sum_reduction=False, - name=None): - """Initialize the internal state for this split handler. - - Args: - l1_regularization: L1 regularization applied for this split handler. - l2_regularization: L2 regularization applied for this split handler. - tree_complexity_regularization: Tree complexity regularization applied - for this split handler. - min_node_weight: Minimum sum of weights of examples in each partition to - be considered for splitting. - feature_column_group_id: Feature column group index. - epsilon: A float, the error bound for quantile computation. - num_quantiles: An int, the number of buckets to create from the histogram. - gradient_shape: A TensorShape, containing shape of gradients. - hessian_shape: A TensorShape, containing shape of hessians. - multiclass_strategy: Strategy describing how to treat multiclass problems. - init_stamp_token: A tensor containing an scalar for initial stamp of the - stamped objects. - loss_uses_sum_reduction: A scalar boolean tensor that specifies whether - SUM or MEAN reduction was used for the loss. - name: An optional handler name. - """ - super(InequalitySplitHandler, self).__init__( - name=name, - l1_regularization=l1_regularization, - l2_regularization=l2_regularization, - tree_complexity_regularization=tree_complexity_regularization, - min_node_weight=min_node_weight, - feature_column_group_id=feature_column_group_id, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=multiclass_strategy, - loss_uses_sum_reduction=loss_uses_sum_reduction) - self._stats_accumulator = stats_accumulator_ops.StatsAccumulator( - init_stamp_token, - gradient_shape, - hessian_shape, - name="StatsAccumulator/{}".format(self._name)) - # Allocate both stats accumulator and quantile accumulator on the same - # device so that we can build splits with fewer RPCs. - with ops.colocate_with(self._stats_accumulator.resource_handle): - self._quantile_accumulator = quantile_ops.QuantileAccumulator( - init_stamp_token, - epsilon=epsilon, - num_quantiles=num_quantiles, - name="QuantileAccumulator/{}".format(self._name)) - - def reset(self, stamp_token, next_stamp_token): - reset_1 = self._stats_accumulator.flush(stamp_token, next_stamp_token) - reset_2 = self._quantile_accumulator.flush(stamp_token, next_stamp_token) - return control_flow_ops.group([reset_1, reset_2]) - - -class DenseSplitHandler(InequalitySplitHandler): - """Computes stats and finds the best inequality splits on dense columns.""" - - def __init__(self, - dense_float_column, - l1_regularization, - l2_regularization, - tree_complexity_regularization, - min_node_weight, - feature_column_group_id, - epsilon, - num_quantiles, - gradient_shape, - hessian_shape, - multiclass_strategy, - init_stamp_token=0, - loss_uses_sum_reduction=False, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE, - name=None): - """Initialize the internal state for this split handler. - - Args: - dense_float_column: A `Tensor` column associated with this handler. - l1_regularization: L1 regularization applied for this split handler. - l2_regularization: L2 regularization applied for this split handler. - tree_complexity_regularization: Tree complexity regularization applied - for this split handler. - min_node_weight: Minimum sum of weights of examples in each partition to - be considered for splitting. - feature_column_group_id: Feature column group index. - epsilon: A float, the error bound for quantile computation. - num_quantiles: An int, the number of buckets to create from the histogram. - gradient_shape: A TensorShape, containing shape of gradients. - hessian_shape: A TensorShape, containing shape of hessians. - multiclass_strategy: Strategy describing how to treat multiclass problems. - init_stamp_token: A tensor containing an scalar for initial stamp of the - stamped objects. - loss_uses_sum_reduction: A scalar boolean tensor that specifies whether - SUM or MEAN reduction was used for the loss. - weak_learner_type: Specifies the type of weak learner to use. - name: An optional handler name. - """ - super(DenseSplitHandler, self).__init__( - l1_regularization=l1_regularization, - l2_regularization=l2_regularization, - tree_complexity_regularization=tree_complexity_regularization, - min_node_weight=min_node_weight, - feature_column_group_id=feature_column_group_id, - epsilon=epsilon, - num_quantiles=num_quantiles, - init_stamp_token=init_stamp_token, - name=name, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=multiclass_strategy, - loss_uses_sum_reduction=loss_uses_sum_reduction) - self._dense_float_column = dense_float_column - self._weak_learner_type = weak_learner_type - # Register dense_make_stats_update function as an Op to the graph. - g = ops.get_default_graph() - dense_make_stats_update.add_to_graph(g) - - def scheduled_reads(self): - return [self._quantile_accumulator.schedule_get_buckets()] - - def update_stats(self, stamp_token, example_partition_ids, gradients, - hessians, empty_gradients, empty_hessians, weights, - is_active, scheduled_reads): - """Updates the state for dense split handler. - - Args: - stamp_token: An int32 scalar tensor containing the current stamp token. - example_partition_ids: A dense tensor, containing an int32 for each - example which is the partition id that the example ends up in. - gradients: A dense tensor of gradients. - hessians: A dense tensor of hessians. - empty_gradients: A dense empty tensor of the same shape (for dimensions > - 0) as gradients. - empty_hessians: A dense empty tensor of the same shape (for dimensions > - 0) as hessians. - weights: A dense float32 tensor with a weight for each example. - is_active: A boolean tensor that says if this handler is active or not. - One value for the current layer and one value for the next layer. - scheduled_reads: List of scheduled reads for this handler. - - Returns: - The op that updates the stats for this handler. - """ - name = _PATTERN.sub("", self._name) - with ops.name_scope(name, "DenseSplitHandler"): - are_buckets_ready, buckets = scheduled_reads[0] - (quantile_values, quantile_weights, example_partition_ids, - feature_ids, gradients, hessians) = dense_make_stats_update( - is_active, are_buckets_ready, self._dense_float_column, buckets, - example_partition_ids, gradients, hessians, weights, empty_gradients, - empty_hessians) - update_quantiles = self._quantile_accumulator.schedule_add_summary( - stamp_token=stamp_token, - column=quantile_values, - example_weights=quantile_weights) - update_stats = self._stats_accumulator.schedule_add( - example_partition_ids, feature_ids, gradients, hessians) - return control_flow_ops.no_op(), [update_quantiles, update_stats] - - def make_splits(self, stamp_token, next_stamp_token, class_id): - """Create the best split using the accumulated stats and flush the state.""" - if (self._gradient_shape.rank == 0 and self._hessian_shape.rank == 0): - handler = make_dense_split_scalar - else: - handler = make_dense_split_tensor - - are_splits_ready, partition_ids, gains, split_infos = ( - handler(self._quantile_accumulator.resource_handle, - self._stats_accumulator.resource_handle, stamp_token, - next_stamp_token, self._multiclass_strategy, class_id, - self._feature_column_group_id, self._l1_regularization, - self._l2_regularization, self._tree_complexity_regularization, - self._min_node_weight, self._loss_uses_sum_reduction, - self._weak_learner_type)) - return are_splits_ready, partition_ids, gains, split_infos - - -def _make_dense_split(quantile_accumulator_handle, stats_accumulator_handle, - stamp_token, next_stamp_token, multiclass_strategy, - class_id, feature_column_id, l1_regularization, - l2_regularization, tree_complexity_regularization, - min_node_weight, is_multi_dimentional, - loss_uses_sum_reduction, weak_learner_type): - """Function that builds splits for a dense feature column.""" - # Get the bucket boundaries - are_splits_ready, buckets = ( - gen_quantile_ops.quantile_accumulator_get_buckets( - quantile_accumulator_handles=[quantile_accumulator_handle], - stamp_token=stamp_token)) - # quantile_accumulator_get_buckets returns a list of results per handle that - # we pass to it. In this case we're getting results just for one resource. - are_splits_ready = are_splits_ready[0] - buckets = buckets[0] - - # After we receive the boundaries from previous iteration we can flush - # the quantile accumulator. - with ops.control_dependencies([buckets]): - flush_quantiles = gen_quantile_ops.quantile_accumulator_flush( - quantile_accumulator_handle=quantile_accumulator_handle, - stamp_token=stamp_token, - next_stamp_token=next_stamp_token) - - if is_multi_dimentional: - num_minibatches, partition_ids, bucket_ids, gradients, hessians = ( - gen_stats_accumulator_ops.stats_accumulator_tensor_flush( - stats_accumulator_handle, stamp_token, next_stamp_token)) - else: - num_minibatches, partition_ids, bucket_ids, gradients, hessians = ( - gen_stats_accumulator_ops.stats_accumulator_scalar_flush( - stats_accumulator_handle, stamp_token, next_stamp_token)) - # For sum_reduction, we don't need to divide by number of minibatches. - num_minibatches = control_flow_ops.cond( - loss_uses_sum_reduction, - lambda: math_ops.cast(1, dtypes.int64), - lambda: num_minibatches) - # Put quantile and stats accumulator flushing in the dependency path. - with ops.control_dependencies([flush_quantiles, partition_ids]): - are_splits_ready = array_ops.identity(are_splits_ready) - partition_ids, gains, split_infos = ( - split_handler_ops.build_dense_inequality_splits( - num_minibatches=num_minibatches, - bucket_boundaries=buckets, - partition_ids=partition_ids, - bucket_ids=bucket_ids, - gradients=gradients, - hessians=hessians, - class_id=class_id, - feature_column_group_id=feature_column_id, - l1_regularization=l1_regularization, - l2_regularization=l2_regularization, - tree_complexity_regularization=tree_complexity_regularization, - min_node_weight=min_node_weight, - multiclass_strategy=multiclass_strategy, - weak_learner_type=weak_learner_type)) - return are_splits_ready, partition_ids, gains, split_infos - - -class SparseSplitHandler(InequalitySplitHandler): - """Computes stats and finds the best inequality splits on sparse columns.""" - - def __init__(self, - sparse_float_column, - l1_regularization, - l2_regularization, - tree_complexity_regularization, - min_node_weight, - feature_column_group_id, - epsilon, - num_quantiles, - gradient_shape, - hessian_shape, - multiclass_strategy, - init_stamp_token=0, - loss_uses_sum_reduction=False, - name=None): - """Initialize the internal state for this split handler. - - Args: - sparse_float_column: A `SparseTensor` column associated with this handler. - l1_regularization: L1 regularization applied for this split handler. - l2_regularization: L2 regularization applied for this split handler. - tree_complexity_regularization: Tree complexity regularization applied - for this split handler. - min_node_weight: Minimum sum of weights of examples in each partition to - be considered for splitting. - feature_column_group_id: Feature column group index. - epsilon: A float, the error bound for quantile computation. - num_quantiles: An int, the number of buckets to create from the histogram. - gradient_shape: A TensorShape, containing shape of gradients. - hessian_shape: A TensorShape, containing shape of hessians. - multiclass_strategy: Strategy describing how to treat multiclass problems. - init_stamp_token: A tensor containing an scalar for initial stamp of the - stamped objects. - loss_uses_sum_reduction: A scalar boolean tensor that specifies whether - SUM or MEAN reduction was used for the loss. - name: An optional handler name. - """ - super(SparseSplitHandler, self).__init__( - l1_regularization=l1_regularization, - l2_regularization=l2_regularization, - tree_complexity_regularization=tree_complexity_regularization, - min_node_weight=min_node_weight, - feature_column_group_id=feature_column_group_id, - epsilon=epsilon, - num_quantiles=num_quantiles, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=multiclass_strategy, - init_stamp_token=init_stamp_token, - loss_uses_sum_reduction=loss_uses_sum_reduction, - name=name) - self._sparse_float_column = sparse_float_column - - def scheduled_reads(self): - return [self._quantile_accumulator.schedule_get_buckets()] - - def update_stats(self, stamp_token, example_partition_ids, gradients, - hessians, empty_gradients, empty_hessians, weights, - is_active, scheduled_reads): - """Updates the state for dense split handler. - - Args: - stamp_token: An int32 scalar tensor containing the current stamp token. - example_partition_ids: A dense tensor, containing an int32 for each - example which is the partition id that the example ends up in. - gradients: A dense tensor of gradients. - hessians: A dense tensor of hessians. - empty_gradients: A dense empty tensor of the same shape (for dimensions > - 0) as gradients. - empty_hessians: A dense empty tensor of the same shape (for dimensions > - 0) as hessians. - weights: A dense float32 tensor with a weight for each example. - is_active: A boolean tensor that says if this handler is active or not. - One value for the current layer and one value for the next layer. - scheduled_reads: List of results from the scheduled reads. - - Returns: - The op that updates the stats for this handler. - """ - are_buckets_ready, buckets = scheduled_reads[0] - with ops.name_scope(self._name, "SparseSplitHandler"): - (quantile_indices, quantile_values, quantile_shapes, quantile_weights, - example_partition_ids, feature_ids, gradients, - hessians) = sparse_make_stats_update( - is_active, are_buckets_ready, self._sparse_float_column.indices, - self._sparse_float_column.values, - self._sparse_float_column.dense_shape, buckets, - example_partition_ids, gradients, hessians, weights, empty_gradients, - empty_hessians) - update_quantiles = self._quantile_accumulator.schedule_add_summary( - stamp_token=stamp_token, - column=sparse_tensor.SparseTensor(quantile_indices, quantile_values, - quantile_shapes), - example_weights=quantile_weights) - update_stats = self._stats_accumulator.schedule_add( - example_partition_ids, feature_ids, gradients, hessians) - return (control_flow_ops.no_op(), [update_quantiles, update_stats]) - - def make_splits(self, stamp_token, next_stamp_token, class_id): - """Create the best split using the accumulated stats and flush the state.""" - if self._gradient_shape.rank == 0 and self._hessian_shape.rank == 0: - handler = make_sparse_split_scalar - else: - handler = make_sparse_split_tensor - - are_splits_ready, partition_ids, gains, split_infos = ( - handler(self._quantile_accumulator.resource_handle, - self._stats_accumulator.resource_handle, stamp_token, - next_stamp_token, self._multiclass_strategy, class_id, - self._feature_column_group_id, self._l1_regularization, - self._l2_regularization, self._tree_complexity_regularization, - self._min_node_weight, self._loss_uses_sum_reduction)) - return are_splits_ready, partition_ids, gains, split_infos - - -def _make_sparse_split( - quantile_accumulator_handle, stats_accumulator_handle, stamp_token, - next_stamp_token, multiclass_strategy, class_id, feature_column_id, - l1_regularization, l2_regularization, tree_complexity_regularization, - min_node_weight, is_multi_dimentional, loss_uses_sum_reduction): - """Function that builds splits for a sparse feature column.""" - # Get the bucket boundaries - are_splits_ready, buckets = ( - gen_quantile_ops.quantile_accumulator_get_buckets( - quantile_accumulator_handles=[quantile_accumulator_handle], - stamp_token=stamp_token)) - # quantile_accumulator_get_buckets returns a list of results per handle that - # we pass to it. In this case we're getting results just for one resource. - are_splits_ready = are_splits_ready[0] - buckets = buckets[0] - - # After we receive the boundaries from previous iteration we can flush - # the quantile accumulator. - with ops.control_dependencies([buckets]): - flush_quantiles = gen_quantile_ops.quantile_accumulator_flush( - quantile_accumulator_handle=quantile_accumulator_handle, - stamp_token=stamp_token, - next_stamp_token=next_stamp_token) - - if is_multi_dimentional: - num_minibatches, partition_ids, bucket_ids, gradients, hessians = ( - gen_stats_accumulator_ops.stats_accumulator_tensor_flush( - stats_accumulator_handle, stamp_token, next_stamp_token)) - else: - num_minibatches, partition_ids, bucket_ids, gradients, hessians = ( - gen_stats_accumulator_ops.stats_accumulator_scalar_flush( - stats_accumulator_handle, stamp_token, next_stamp_token)) - num_minibatches = control_flow_ops.cond( - loss_uses_sum_reduction, - lambda: math_ops.cast(1, dtypes.int64), - lambda: num_minibatches) - # Put quantile and stats accumulator flushing in the dependency path. - with ops.control_dependencies([flush_quantiles, partition_ids]): - are_splits_ready = array_ops.identity(are_splits_ready) - partition_ids, gains, split_infos = ( - split_handler_ops.build_sparse_inequality_splits( - num_minibatches=num_minibatches, - bucket_boundaries=buckets, - partition_ids=partition_ids, - bucket_ids=bucket_ids, - gradients=gradients, - hessians=hessians, - class_id=class_id, - feature_column_group_id=feature_column_id, - l1_regularization=l1_regularization, - l2_regularization=l2_regularization, - tree_complexity_regularization=tree_complexity_regularization, - min_node_weight=min_node_weight, - bias_feature_id=_BIAS_FEATURE_ID, - multiclass_strategy=multiclass_strategy)) - return are_splits_ready, partition_ids, gains, split_infos - - -def _specialize_make_split_dense(func, is_multi_dimentional): - """Builds a specialized version of the function.""" - - @function.Defun( - dtypes.resource, - dtypes.resource, - dtypes.int64, - dtypes.int64, - dtypes.int32, - dtypes.int32, - dtypes.int32, - dtypes.float32, - dtypes.float32, - dtypes.float32, - dtypes.float32, - dtypes.bool, - dtypes.int32, - noinline=True) - def f(quantile_accumulator_handle, stats_accumulator_handle, stamp_token, - next_stamp_token, multiclass_strategy, class_id, feature_column_id, - l1_regularization, l2_regularization, tree_complexity_regularization, - min_node_weight, loss_uses_sum_reduction, weak_learner_type): - """Function that builds splits for a sparse feature column.""" - return func(quantile_accumulator_handle, stats_accumulator_handle, - stamp_token, next_stamp_token, multiclass_strategy, class_id, - feature_column_id, l1_regularization, l2_regularization, - tree_complexity_regularization, min_node_weight, - is_multi_dimentional, loss_uses_sum_reduction, - weak_learner_type) - - return f - - -def _specialize_make_split_sparse(func, is_multi_dimentional): - """Builds a specialized version of the function.""" - - @function.Defun( - dtypes.resource, - dtypes.resource, - dtypes.int64, - dtypes.int64, - dtypes.int32, - dtypes.int32, - dtypes.int32, - dtypes.float32, - dtypes.float32, - dtypes.float32, - dtypes.float32, - dtypes.bool, - noinline=True) - def f(quantile_accumulator_handle, stats_accumulator_handle, stamp_token, - next_stamp_token, multiclass_strategy, class_id, feature_column_id, - l1_regularization, l2_regularization, tree_complexity_regularization, - min_node_weight, loss_uses_sum_reduction): - """Function that builds splits for a sparse feature column.""" - return func(quantile_accumulator_handle, stats_accumulator_handle, - stamp_token, next_stamp_token, multiclass_strategy, class_id, - feature_column_id, l1_regularization, l2_regularization, - tree_complexity_regularization, min_node_weight, - is_multi_dimentional, loss_uses_sum_reduction) - - return f - - -make_dense_split_scalar = _specialize_make_split_dense( - _make_dense_split, is_multi_dimentional=False) - -make_dense_split_tensor = _specialize_make_split_dense( - _make_dense_split, is_multi_dimentional=True) - -make_sparse_split_scalar = _specialize_make_split_sparse( - _make_sparse_split, is_multi_dimentional=False) -make_sparse_split_tensor = _specialize_make_split_sparse( - _make_sparse_split, is_multi_dimentional=True) - - -@function.Defun( - dtypes.bool, - dtypes.bool, - dtypes.float32, - dtypes.float32, - dtypes.int32, - dtypes.float32, - dtypes.float32, - dtypes.float32, - dtypes.float32, - dtypes.float32, - noinline=True) -def dense_make_stats_update(is_active, are_buckets_ready, float_column, - quantile_buckets, example_partition_ids, gradients, - hessians, weights, empty_gradients, empty_hessians): - """Updates the state for dense split handler.""" - empty_float = constant_op.constant_v1([], dtype=dtypes.float32) - - quantile_values, quantile_weights = control_flow_ops.cond( - is_active[1], # For the next layer, this handler is inactive. - lambda: (float_column, weights), - lambda: (empty_float, empty_float)) - - def ready_inputs_fn(): - """Branch to execute when quantiles are ready.""" - quantized_feature = quantile_ops.quantiles([float_column], [], - [quantile_buckets], [], []) - quantized_feature = math_ops.cast(quantized_feature[0], dtypes.int64) - quantized_feature = array_ops.squeeze(quantized_feature, axis=0) - return (example_partition_ids, quantized_feature, gradients, hessians) - - def not_ready_inputs_fn(): - return (constant_op.constant_v1([], dtype=dtypes.int32), - constant_op.constant_v1([[]], dtype=dtypes.int64, shape=[1, 2]), - empty_gradients, empty_hessians) - - example_partition_ids, feature_ids, gradients, hessians = ( - control_flow_ops.cond( - math_ops.logical_and( - math_ops.logical_and(are_buckets_ready, - array_ops.size(quantile_buckets) > 0), - is_active[0]), ready_inputs_fn, not_ready_inputs_fn)) - return (quantile_values, quantile_weights, example_partition_ids, feature_ids, - gradients, hessians) - - -@function.Defun( - dtypes.bool, - dtypes.bool, - dtypes.int64, - dtypes.float32, - dtypes.int64, - dtypes.float32, - dtypes.int32, - dtypes.float32, - dtypes.float32, - dtypes.float32, - dtypes.float32, - dtypes.float32, - noinline=True) -def sparse_make_stats_update( - is_active, are_buckets_ready, sparse_column_indices, sparse_column_values, - sparse_column_shape, quantile_buckets, example_partition_ids, gradients, - hessians, weights, empty_gradients, empty_hessians): - """Updates the state for this split handler.""" - - def quantiles_ready(): - """The subgraph for when the quantiles are ready.""" - quantized_feature = quantile_ops.quantiles([], [sparse_column_values], [], - [quantile_buckets], - [sparse_column_indices]) - - quantized_feature = math_ops.cast(quantized_feature[1], dtypes.int64) - quantized_feature = array_ops.squeeze(quantized_feature, axis=0) - - example_indices, _ = array_ops.split( - sparse_column_indices, num_or_size_splits=2, axis=1) - example_indices = array_ops.squeeze(example_indices, [1]) - filtered_gradients = array_ops.gather(gradients, example_indices) - filtered_hessians = array_ops.gather(hessians, example_indices) - filtered_partition_ids = array_ops.gather(example_partition_ids, - example_indices) - unique_partitions, mapped_partitions = array_ops.unique( - example_partition_ids) - - # Compute aggregate stats for each partition. - # Since unsorted_segment_sum can be numerically unstable, use 64bit - # operation. - gradients64 = math_ops.cast(gradients, dtypes.float64) - hessians64 = math_ops.cast(hessians, dtypes.float64) - per_partition_gradients = math_ops.unsorted_segment_sum( - gradients64, mapped_partitions, array_ops.size(unique_partitions)) - per_partition_hessians = math_ops.unsorted_segment_sum( - hessians64, mapped_partitions, array_ops.size(unique_partitions)) - per_partition_gradients = math_ops.cast(per_partition_gradients, - dtypes.float32) - per_partition_hessians = math_ops.cast(per_partition_hessians, - dtypes.float32) - # Prepend a bias feature per partition that accumulates the stats for all - # examples in that partition. - bias_feature_ids = array_ops.fill( - array_ops.shape(unique_partitions), _BIAS_FEATURE_ID) - bias_feature_ids = math_ops.cast(bias_feature_ids, dtypes.int64) - zeros = array_ops.zeros_like(bias_feature_ids) - bias_feature_ids = array_ops.stack([bias_feature_ids, zeros], axis=1) - - partition_ids = array_ops.concat( - [unique_partitions, filtered_partition_ids], 0) - filtered_gradients = array_ops.concat( - [per_partition_gradients, filtered_gradients], 0) - filtered_hessians = array_ops.concat( - [per_partition_hessians, filtered_hessians], 0) - - bucket_ids = array_ops.concat([bias_feature_ids, quantized_feature], 0) - - return partition_ids, bucket_ids, filtered_gradients, filtered_hessians - - def quantiles_not_ready(): - """The subgraph for when the quantiles are not ready.""" - return (constant_op.constant_v1([], dtype=dtypes.int32), - constant_op.constant_v1([], dtype=dtypes.int64, shape=[1, 2]), - empty_gradients, empty_hessians) - - empty_float = constant_op.constant_v1([], dtype=dtypes.float32) - handler_not_active = (constant_op.constant( - [], dtype=dtypes.int64, shape=[0, 2]), empty_float, - constant_op.constant([0, 1], dtype=dtypes.int64), - empty_float) - handler_active = (sparse_column_indices, sparse_column_values, - sparse_column_shape, weights) - quantile_indices, quantile_values, quantile_shape, quantile_weights = ( - control_flow_ops.cond(is_active[1], lambda: handler_active, - lambda: handler_not_active)) - - example_partition_ids, feature_ids, gradients, hessians = ( - control_flow_ops.cond( - math_ops.logical_and(are_buckets_ready, - array_ops.size(quantile_buckets) > 0), - quantiles_ready, quantiles_not_ready)) - - return (quantile_indices, quantile_values, quantile_shape, quantile_weights, - example_partition_ids, feature_ids, gradients, hessians) diff --git a/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler_test.py b/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler_test.py deleted file mode 100644 index d41463d002f..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler_test.py +++ /dev/null @@ -1,1725 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Test for checking stats accumulator related ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.boosted_trees.lib.learner.batch import ordinal_split_handler -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.boosted_trees.proto import split_info_pb2 - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import resources -from tensorflow.python.platform import googletest - - -def get_empty_tensors(gradient_shape, hessian_shape): - empty_hess_shape = [1] + hessian_shape.as_list() - empty_grad_shape = [1] + gradient_shape.as_list() - - empty_gradients = constant_op.constant_v1( - [], dtype=dtypes.float32, shape=empty_grad_shape) - empty_hessians = constant_op.constant_v1( - [], dtype=dtypes.float32, shape=empty_hess_shape) - - return empty_gradients, empty_hessians - - -class DenseSplitHandlerTest(test_util.TensorFlowTestCase): - - def testGenerateFeatureSplitCandidates(self): - with self.cached_session() as sess: - # The data looks like the following: - # Example | Gradients | Partition | Dense Quantile | - # i0 | (0.2, 0.12) | 0 | 1 | - # i1 | (-0.5, 0.07) | 0 | 1 | - # i2 | (1.2, 0.2) | 0 | 0 | - # i3 | (4.0, 0.13) | 1 | 1 | - dense_column = array_ops.constant([0.52, 0.52, 0.3, 0.52]) - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - class_id = -1 - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - split_handler = ordinal_split_handler.DenseSplitHandler( - l1_regularization=0.1, - l2_regularization=1., - tree_complexity_regularization=0., - min_node_weight=0., - epsilon=0.001, - num_quantiles=10, - feature_column_group_id=0, - dense_float_column=dense_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - - # During the first iteration, inequality split handlers are not going to - # have any splits. Make sure that we return not_ready in that case. - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - - self.assertAllEqual([0, 1], partitions) - - # Check the split on partition 0. - # -(1.2 - 0.1) / (0.2 + 1) - expected_left_weight = -0.91666 - - # expected_left_weight * -(1.2 - 0.1) - expected_left_gain = 1.0083333333333331 - - # (-0.5 + 0.2 + 0.1) / (0.19 + 1) - expected_right_weight = 0.1680672 - - # expected_right_weight * -(-0.5 + 0.2 + 0.1)) - expected_right_gain = 0.033613445378151252 - - # (0.2 + -0.5 + 1.2 - 0.1) ** 2 / (0.12 + 0.07 + 0.2 + 1) - expected_bias_gain = 0.46043165467625885 - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.dense_float_binary_split - self.assertAllClose( - expected_left_gain + expected_right_gain - expected_bias_gain, gains[0], - 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - self.assertEqual(0, split_node.feature_column) - - self.assertAllClose(0.3, split_node.threshold, 0.00001) - - # Check the split on partition 1. - # (-4 + 0.1) / (0.13 + 1) - expected_left_weight = -3.4513274336283186 - # (-4 + 0.1) ** 2 / (0.13 + 1) - expected_left_gain = 13.460176991150442 - expected_right_weight = 0 - expected_right_gain = 0 - # (-4 + 0.1) ** 2 / (0.13 + 1) - expected_bias_gain = 13.460176991150442 - - # Verify candidate for partition 1, there's only one active bucket here - # so zero gain is expected. - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.dense_float_binary_split - self.assertAllClose(0.0, gains[1], 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - self.assertEqual(0, split_node.feature_column) - - self.assertAllClose(0.52, split_node.threshold, 0.00001) - - def testObliviousFeatureSplitGeneration(self): - with self.cached_session() as sess: - # The data looks like the following: - # Example | Gradients | Partition | Dense Quantile | - # i0 | (0.2, 0.12) | 1 | 3 | - # i1 | (-0.5, 0.07) | 1 | 3 | - # i2 | (1.2, 0.2) | 1 | 1 | - # i3 | (4.0, 0.13) | 2 | 2 | - dense_column = array_ops.placeholder( - dtypes.float32, shape=(4, 1), name="dense_column") - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - partition_ids = array_ops.constant([1, 1, 1, 2], dtype=dtypes.int32) - class_id = -1 - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - split_handler = ordinal_split_handler.DenseSplitHandler( - l1_regularization=0.1, - l2_regularization=1., - tree_complexity_regularization=0., - min_node_weight=0., - epsilon=0.001, - num_quantiles=10, - feature_column_group_id=0, - dense_float_column=dense_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, - weak_learner_type=learner_pb2.LearnerConfig.OBLIVIOUS_DECISION_TREE) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - # Forcing the creation of four buckets. - are_splits_ready = sess.run( - [are_splits_ready], - feed_dict={dense_column: [[0.2], [0.62], [0.3], [0.52]]})[0] - - update_2 = split_handler.update_stats_sync( - 1, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - # Only using the last three buckets. - are_splits_ready2, partitions, gains, splits = ( - sess.run( - [are_splits_ready2, partitions, gains, splits], - feed_dict={dense_column: [[0.62], [0.62], [0.3], [0.52]]})) - - # During the first iteration, inequality split handlers are not going to - # have any splits. Make sure that we return not_ready in that case. - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - - self.assertAllEqual([1, 2], partitions) - - oblivious_split_info = split_info_pb2.ObliviousSplitInfo() - oblivious_split_info.ParseFromString(splits[0]) - split_node = oblivious_split_info.split_node - split_node = split_node.oblivious_dense_float_binary_split - self.assertAllClose(0.3, split_node.threshold, 0.00001) - self.assertEqual(0, split_node.feature_column) - - # Check the split on partition 1. - # -(1.2 - 0.1) / (0.2 + 1) - expected_left_weight_1 = -0.9166666666666666 - - # expected_left_weight_1 * -(1.2 - 0.1) - expected_left_gain_1 = 1.008333333333333 - - # (-0.5 + 0.2 + 0.1) / (0.19 + 1) - expected_right_weight_1 = 0.1680672 - - # expected_right_weight_1 * -(-0.5 + 0.2 + 0.1)) - expected_right_gain_1 = 0.033613445378151252 - - # (0.2 + -0.5 + 1.2 - 0.1) ** 2 / (0.12 + 0.07 + 0.2 + 1) - expected_bias_gain_1 = 0.46043165467625896 - - left_child = oblivious_split_info.children[0].vector - right_child = oblivious_split_info.children[1].vector - - self.assertAllClose([expected_left_weight_1], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight_1], right_child.value, 0.00001) - - # Check the split on partition 2. - expected_left_weight_2 = 0 - expected_left_gain_2 = 0 - # -(4 - 0.1) / (0.13 + 1) - expected_right_weight_2 = -3.4513274336283186 - # expected_right_weight_2 * -(4 - 0.1) - expected_right_gain_2 = 13.460176991150442 - # (-4 + 0.1) ** 2 / (0.13 + 1) - expected_bias_gain_2 = 13.460176991150442 - - left_child = oblivious_split_info.children[2].vector - right_child = oblivious_split_info.children[3].vector - - self.assertAllClose([expected_left_weight_2], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight_2], right_child.value, 0.00001) - - # The layer gain is the sum of the gains of each partition - layer_gain = ( - expected_left_gain_1 + expected_right_gain_1 - expected_bias_gain_1) + ( - expected_left_gain_2 + expected_right_gain_2 - expected_bias_gain_2) - self.assertAllClose(layer_gain, gains[0], 0.00001) - - # We have examples in both partitions, then we get both ids. - self.assertEqual(2, len(oblivious_split_info.children_parent_id)) - self.assertEqual(1, oblivious_split_info.children_parent_id[0]) - self.assertEqual(2, oblivious_split_info.children_parent_id[1]) - - def testGenerateFeatureSplitCandidatesLossUsesSumReduction(self): - with self.cached_session() as sess: - # The data looks like the following: - # Example | Gradients | Partition | Dense Quantile | - # i0 | (0.2, 0.12) | 0 | 1 | - # i1 | (-0.5, 0.07) | 0 | 1 | - # i2 | (1.2, 0.2) | 0 | 0 | - # i3 | (4.0, 0.13) | 1 | 1 | - dense_column = array_ops.constant([0.52, 0.52, 0.3, 0.52]) - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - class_id = -1 - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - split_handler = ordinal_split_handler.DenseSplitHandler( - l1_regularization=0.2, - l2_regularization=2., - tree_complexity_regularization=0., - min_node_weight=0., - epsilon=0.001, - num_quantiles=10, - feature_column_group_id=0, - dense_float_column=dense_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, - loss_uses_sum_reduction=True) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - update_3 = split_handler.update_stats_sync( - 1, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2, update_3]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - - # During the first iteration, inequality split handlers are not going to - # have any splits. Make sure that we return not_ready in that case. - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - - self.assertAllEqual([0, 1], partitions) - - # Check the split on partition 0. - # -(2.4 - 0.2) / (0.4 + 2) - expected_left_weight = -0.91666 - - # expected_left_weight * -(2.4 - 0.2) - expected_left_gain = 2.016666666666666 - - # -(-1 + 0.4 + 0.2) / (0.38 + 2) - expected_right_weight = 0.1680672 - - # expected_right_weight * -(-1 + 0.4 + 0.2) - expected_right_gain = 0.0672268907563025 - - # (0.2 + -0.5 + 1.2 - 0.1) ** 2 / (0.12 + 0.07 + 0.2 + 1) - expected_bias_gain = 0.9208633093525178 - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.dense_float_binary_split - self.assertAllClose( - expected_left_gain + expected_right_gain - expected_bias_gain, gains[0], - 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - self.assertEqual(0, split_node.feature_column) - - self.assertAllClose(0.3, split_node.threshold, 0.00001) - - # Check the split on partition 1. - # (-8 + 0.2) / (0.26 + 2) - expected_left_weight = -3.4513274336283186 - expected_right_weight = 0 - - # Verify candidate for partition 1, there's only one active bucket here - # so zero gain is expected. - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.dense_float_binary_split - self.assertAllClose(0.0, gains[1], 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - self.assertEqual(0, split_node.feature_column) - - self.assertAllClose(0.52, split_node.threshold, 0.00001) - - def testGenerateFeatureSplitCandidatesMulticlassFullHessian(self): - with self.cached_session() as sess: - dense_column = array_ops.constant([0.52, 0.52, 0.3, 0.52]) - # Batch size is 4, 2 gradients per each instance. - gradients = array_ops.constant( - [[0.2, 0.1], [-0.5, 0.2], [1.2, 3.4], [4.0, -3.5]], shape=[4, 2]) - # 2x2 matrix for each instance - hessian_0 = [[0.12, 0.02], [0.3, 0.11]] - hessian_1 = [[0.07, -0.2], [-0.5, 0.2]] - hessian_2 = [[0.2, -0.23], [-0.8, 0.9]] - hessian_3 = [[0.13, -0.3], [-1.5, 2.2]] - - hessians = array_ops.constant( - [hessian_0, hessian_1, hessian_2, hessian_3]) - partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - class_id = -1 - - gradient_shape = tensor_shape.TensorShape([2]) - hessian_shape = tensor_shape.TensorShape([2, 2]) - - split_handler = ordinal_split_handler.DenseSplitHandler( - l1_regularization=0., - l2_regularization=1., - tree_complexity_regularization=0., - min_node_weight=0., - epsilon=0.001, - num_quantiles=3, - feature_column_group_id=0, - dense_float_column=dense_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.FULL_HESSIAN) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - - # During the first iteration, inequality split handlers are not going to - # have any splits. Make sure that we return not_ready in that case. - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.dense_float_binary_split - - # Each leaf has 2 element vector. - self.assertEqual(2, len(left_child.value)) - self.assertEqual(2, len(right_child.value)) - self.assertEqual(0, split_node.feature_column) - self.assertAllClose(0.3, split_node.threshold, 1e-6) - - def testGenerateFeatureSplitCandidatesMulticlassDiagonalHessian(self): - with self.cached_session() as sess: - dense_column = array_ops.constant([0.52, 0.52, 0.3, 0.52]) - # Batch size is 4, 2 gradients per each instance. - gradients = array_ops.constant( - [[0.2, 0.1], [-0.5, 0.2], [1.2, 3.4], [4.0, -3.5]], shape=[4, 2]) - # Each hessian is a diagonal of a full hessian matrix. - hessian_0 = [0.12, 0.11] - hessian_1 = [0.07, 0.2] - hessian_2 = [0.2, 0.9] - hessian_3 = [0.13, 2.2] - - hessians = array_ops.constant( - [hessian_0, hessian_1, hessian_2, hessian_3]) - partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - class_id = -1 - - gradient_shape = tensor_shape.TensorShape([2]) - hessian_shape = tensor_shape.TensorShape([2]) - - split_handler = ordinal_split_handler.DenseSplitHandler( - l1_regularization=0., - l2_regularization=1., - tree_complexity_regularization=0., - min_node_weight=0., - epsilon=0.001, - num_quantiles=3, - feature_column_group_id=0, - dense_float_column=dense_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - - # During the first iteration, inequality split handlers are not going to - # have any splits. Make sure that we return not_ready in that case. - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.dense_float_binary_split - - # Each leaf has 2 element vector. - self.assertEqual(2, len(left_child.value)) - self.assertEqual(2, len(right_child.value)) - self.assertEqual(0, split_node.feature_column) - self.assertAllClose(0.3, split_node.threshold, 1e-6) - - def testGenerateFeatureSplitCandidatesInactive(self): - with self.cached_session() as sess: - # The data looks like the following: - # Example | Gradients | Partition | Dense Quantile | - # i0 | (0.2, 0.12) | 0 | 1 | - # i1 | (-0.5, 0.07) | 0 | 1 | - # i2 | (1.2, 0.2) | 0 | 0 | - # i3 | (4.0, 0.13) | 1 | 1 | - dense_column = array_ops.constant([0.52, 0.52, 0.3, 0.52]) - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = ordinal_split_handler.DenseSplitHandler( - l1_regularization=0.1, - l2_regularization=1., - tree_complexity_regularization=0., - min_node_weight=0., - epsilon=0.001, - num_quantiles=10, - feature_column_group_id=0, - dense_float_column=dense_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, False])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([False, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - - # During the first iteration, inequality split handlers are not going to - # have any splits. Make sure that we return not_ready in that case. - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - # The handler was inactive, so it shouldn't return any splits. - self.assertEqual(len(partitions), 0) - self.assertEqual(len(gains), 0) - self.assertEqual(len(splits), 0) - - def testGenerateFeatureSplitCandidatesWithTreeComplexity(self): - with self.cached_session() as sess: - # The data looks like the following: - # Example | Gradients | Partition | Dense Quantile | - # i0 | (0.2, 0.12) | 0 | 1 | - # i1 | (-0.5, 0.07) | 0 | 1 | - # i2 | (1.2, 0.2) | 0 | 0 | - # i3 | (4.0, 0.13) | 1 | 1 | - dense_column = array_ops.constant([0.52, 0.52, 0.3, 0.52]) - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = ordinal_split_handler.DenseSplitHandler( - l1_regularization=0.1, - l2_regularization=1., - tree_complexity_regularization=0.5, - min_node_weight=0., - epsilon=0.001, - num_quantiles=10, - feature_column_group_id=0, - dense_float_column=dense_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - - # During the first iteration, inequality split handlers are not going to - # have any splits. Make sure that we return not_ready in that case. - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - - self.assertAllEqual([0, 1], partitions) - - # Check the split on partition 0. - # -(1.2 - 0.1) / (0.2 + 1) - expected_left_weight = -0.91666 - - # expected_left_weight * -(1.2 - 0.1) - expected_left_gain = 1.0083333333333331 - - # (-0.5 + 0.2 + 0.1) / (0.19 + 1) - expected_right_weight = 0.1680672 - - # expected_right_weight * -(-0.5 + 0.2 + 0.1)) - expected_right_gain = 0.033613445378151252 - - # (0.2 + -0.5 + 1.2 - 0.1) ** 2 / (0.12 + 0.07 + 0.2 + 1) - expected_bias_gain = 0.46043165467625885 - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.dense_float_binary_split - # Make sure the gain is subtracted by the tree complexity regularization. - self.assertAllClose( - expected_left_gain + expected_right_gain - expected_bias_gain - 0.5, - gains[0], 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - self.assertEqual(0, split_node.feature_column) - - self.assertAllClose(0.3, split_node.threshold, 0.00001) - - # Check the split on partition 1. - # (-4 + 0.1) / (0.13 + 1) - expected_left_weight = -3.4513274336283186 - # (-4 + 0.1) ** 2 / (0.13 + 1) - expected_left_gain = 13.460176991150442 - expected_right_weight = 0 - expected_right_gain = 0 - # (-4 + 0.1) ** 2 / (0.13 + 1) - expected_bias_gain = 13.460176991150442 - - # Verify candidate for partition 1, there's only one active bucket here - # so -0.5 gain is expected (because of tree complexity. - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.dense_float_binary_split - self.assertAllClose(-0.5, gains[1], 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - self.assertEqual(0, split_node.feature_column) - - self.assertAllClose(0.52, split_node.threshold, 0.00001) - - def testGenerateFeatureSplitCandidatesWithMinNodeWeight(self): - with self.cached_session() as sess: - # The data looks like the following: - # Example | Gradients | Partition | Dense Quantile | - # i0 | (0.2, 0.12) | 0 | 1 | - # i1 | (-0.5, 0.07) | 0 | 1 | - # i2 | (1.2, 0.2) | 0 | 0 | - # i3 | (4.0, 2.0) | 1 | 1 | - dense_column = array_ops.constant([0.52, 0.52, 0.3, 0.52]) - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 2]) - partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = ordinal_split_handler.DenseSplitHandler( - l1_regularization=0.1, - l2_regularization=1., - tree_complexity_regularization=0.5, - min_node_weight=1.5, - epsilon=0.001, - num_quantiles=10, - feature_column_group_id=0, - dense_float_column=dense_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - - # During the first iteration, inequality split handlers are not going to - # have any splits. Make sure that we return not_ready in that case. - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - - self.assertAllEqual([0, 1], partitions) - - # Check the gain on partition 0 to be -0.5. - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.dense_float_binary_split - # Make sure the gain is subtracted by the tree complexity regularization. - self.assertAllClose(-0.5, gains[0], 0.00001) - - self.assertEqual(0, split_node.feature_column) - - # Check the split on partition 1. - # (-4 + 0.1) / (2 + 1) - expected_left_weight = -1.3 - expected_right_weight = 0 - - # Verify candidate for partition 1, there's only one active bucket here - # so -0.5 gain is expected (because of tree complexity. - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.dense_float_binary_split - self.assertAllClose(-0.5, gains[1], 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - self.assertEqual(0, split_node.feature_column) - - self.assertAllClose(0.52, split_node.threshold, 0.00001) - - -class SparseSplitHandlerTest(test_util.TensorFlowTestCase): - - def testGenerateFeatureSplitCandidates(self): - with self.cached_session() as sess: - # The data looks like the following: - # Example | Gradients | Partition | Sparse Quantile | - # i0 | (0.2, 0.12) | 0 | 1 | - # i1 | (-0.5, 0.07) | 0 | N/A | - # i2 | (1.2, 0.2) | 0 | 0 | - # i3 | (4.0, 0.13) | 1 | 1 | - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - example_partitions = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - indices = array_ops.constant([[0, 0], [2, 0], [3, 0]], dtype=dtypes.int64) - values = array_ops.constant([0.52, 0.3, 0.52]) - sparse_column = sparse_tensor.SparseTensor(indices, values, [4, 1]) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = ordinal_split_handler.SparseSplitHandler( - l1_regularization=0.0, - l2_regularization=2.0, - tree_complexity_regularization=0.0, - min_node_weight=0.0, - epsilon=0.01, - num_quantiles=2, - feature_column_group_id=0, - sparse_float_column=sparse_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - example_partitions, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - example_partitions, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - - # During the first iteration, inequality split handlers are not going to - # have any splits. Make sure that we return not_ready in that case. - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - - self.assertAllEqual([0, 1], partitions) - # Check the split on partition 0. - # -(0.2 + 1.2) / (0.12 + 0.2 + 2) - expected_left_weight = -0.603448275862069 - # (0.2 + 1.2) ** 2 / (0.12 + 0.2 + 2) - expected_left_gain = 0.8448275862068965 - # 0.5 / (0.07 + 2) - expected_right_weight = 0.24154589371980678 - # 0.5 ** 2 / (0.07 + 2) - expected_right_gain = 0.12077294685990339 - # (0.2 + 1.2 - 0.5) ** 2 / (0.12 + 0.2 + 0.07 + 2) - expected_bias_gain = 0.3389121338912133 - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.sparse_float_binary_split_default_right - self.assertAllClose( - expected_left_gain + expected_right_gain - expected_bias_gain, gains[0]) - - self.assertAllClose([expected_left_weight], left_child.value) - - self.assertAllClose([expected_right_weight], right_child.value) - - self.assertEqual(0, split_node.split.feature_column) - - self.assertAllClose(0.52, split_node.split.threshold) - - # Check the split on partition 1. - expected_left_weight = -1.8779342723004695 - expected_right_weight = 0 - - # Verify candidate for partition 1, there's only one active bucket here - # so zero gain is expected. - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.sparse_float_binary_split_default_left - - self.assertAllClose(0.0, gains[1]) - - self.assertAllClose([expected_left_weight], left_child.value) - - self.assertAllClose([expected_right_weight], right_child.value) - - self.assertEqual(0, split_node.split.feature_column) - - self.assertAllClose(0.52, split_node.split.threshold) - - def testGenerateFeatureSplitCandidatesLossUsesSumReduction(self): - with self.cached_session() as sess: - # The data looks like the following: - # Example | Gradients | Partition | Sparse Quantile | - # i0 | (0.2, 0.12) | 0 | 1 | - # i1 | (-0.5, 0.07) | 0 | N/A | - # i2 | (1.2, 0.2) | 0 | 0 | - # i3 | (4.0, 0.13) | 1 | 1 | - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - example_partitions = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - indices = array_ops.constant([[0, 0], [2, 0], [3, 0]], dtype=dtypes.int64) - values = array_ops.constant([0.52, 0.3, 0.52]) - sparse_column = sparse_tensor.SparseTensor(indices, values, [4, 1]) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = ordinal_split_handler.SparseSplitHandler( - l1_regularization=0.0, - l2_regularization=4.0, - tree_complexity_regularization=0.0, - min_node_weight=0.0, - epsilon=0.01, - num_quantiles=2, - feature_column_group_id=0, - sparse_float_column=sparse_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, - loss_uses_sum_reduction=True) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - example_partitions, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - example_partitions, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - update_3 = split_handler.update_stats_sync( - 1, - example_partitions, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2, update_3]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - - # During the first iteration, inequality split handlers are not going to - # have any splits. Make sure that we return not_ready in that case. - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - - self.assertAllEqual([0, 1], partitions) - # Check the split on partition 0. - # -(0.4 + 2.4) / (0.24 + 0.4 + 4) - expected_left_weight = -0.603448275862069 - # (0.4 + 2.4) ** 2 / (0.24 + 0.4 + 4) - expected_left_gain = 1.689655172413793 - # 1 / (0.14 + 4) - expected_right_weight = 0.24154589371980678 - # 1 ** 2 / (0.14 + 4) - expected_right_gain = 0.24154589371980678 - # (0.4 + 2.4 - 1) ** 2 / (0.24 + 0.4 + 0.14 + 4) - expected_bias_gain = 0.6778242677824265 - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.sparse_float_binary_split_default_right - self.assertAllClose( - expected_left_gain + expected_right_gain - expected_bias_gain, gains[0]) - - self.assertAllClose([expected_left_weight], left_child.value) - - self.assertAllClose([expected_right_weight], right_child.value) - - self.assertEqual(0, split_node.split.feature_column) - - self.assertAllClose(0.52, split_node.split.threshold) - - # Check the split on partition 1. - expected_left_weight = -1.8779342723004695 - expected_right_weight = 0 - - # Verify candidate for partition 1, there's only one active bucket here - # so zero gain is expected. - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.sparse_float_binary_split_default_left - - self.assertAllClose(0.0, gains[1]) - - self.assertAllClose([expected_left_weight], left_child.value) - - self.assertAllClose([expected_right_weight], right_child.value) - - self.assertEqual(0, split_node.split.feature_column) - - self.assertAllClose(0.52, split_node.split.threshold) - - def testGenerateFeatureSplitCandidatesMulticlassFullHessian(self): - with self.cached_session() as sess: - # Batch is 4, 2 classes - gradients = array_ops.constant([[0.2, 1.4], [-0.5, 0.1], [1.2, 3], - [4.0, -3]]) - # 2x2 matrix for each instance - hessian_0 = [[0.12, 0.02], [0.3, 0.11]] - hessian_1 = [[0.07, -0.2], [-0.5, 0.2]] - hessian_2 = [[0.2, -0.23], [-0.8, 0.9]] - hessian_3 = [[0.13, -0.3], [-1.5, 2.2]] - hessians = array_ops.constant( - [hessian_0, hessian_1, hessian_2, hessian_3]) - - example_partitions = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - indices = array_ops.constant([[0, 0], [2, 0], [3, 0]], dtype=dtypes.int64) - values = array_ops.constant([0.52, 0.3, 0.52]) - sparse_column = sparse_tensor.SparseTensor(indices, values, [4, 1]) - - gradient_shape = tensor_shape.TensorShape([2]) - hessian_shape = tensor_shape.TensorShape([2, 2]) - class_id = -1 - - split_handler = ordinal_split_handler.SparseSplitHandler( - l1_regularization=0.0, - l2_regularization=2.0, - tree_complexity_regularization=0.0, - min_node_weight=0.0, - epsilon=0.01, - num_quantiles=2, - feature_column_group_id=0, - sparse_float_column=sparse_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.FULL_HESSIAN) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - example_partitions, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - example_partitions, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.sparse_float_binary_split_default_right - # Each leaf has 2 element vector. - self.assertEqual(2, len(left_child.value)) - self.assertEqual(2, len(right_child.value)) - self.assertEqual(0, split_node.split.feature_column) - self.assertAllClose(0.52, split_node.split.threshold) - - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.sparse_float_binary_split_default_left - self.assertEqual(2, len(left_child.value)) - self.assertEqual(0, split_node.split.feature_column) - self.assertAllClose(0.52, split_node.split.threshold) - - def testGenerateFeatureSplitCandidatesMulticlassDiagonalHessian(self): - with self.cached_session() as sess: - # Batch is 4, 2 classes - gradients = array_ops.constant([[0.2, 1.4], [-0.5, 0.1], [1.2, 3], - [4.0, -3]]) - # Each hessian is a diagonal from a full hessian matrix. - hessian_0 = [0.12, 0.11] - hessian_1 = [0.07, 0.2] - hessian_2 = [0.2, 0.9] - hessian_3 = [0.13, 2.2] - hessians = array_ops.constant( - [hessian_0, hessian_1, hessian_2, hessian_3]) - - example_partitions = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - indices = array_ops.constant([[0, 0], [2, 0], [3, 0]], dtype=dtypes.int64) - values = array_ops.constant([0.52, 0.3, 0.52]) - sparse_column = sparse_tensor.SparseTensor(indices, values, [4, 1]) - - gradient_shape = tensor_shape.TensorShape([2]) - hessian_shape = tensor_shape.TensorShape([2]) - class_id = -1 - - split_handler = ordinal_split_handler.SparseSplitHandler( - l1_regularization=0.0, - l2_regularization=2.0, - tree_complexity_regularization=0.0, - min_node_weight=0.0, - epsilon=0.01, - num_quantiles=2, - feature_column_group_id=0, - sparse_float_column=sparse_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - example_partitions, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - example_partitions, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.sparse_float_binary_split_default_right - # Each leaf has 2 element vector. - self.assertEqual(2, len(left_child.value)) - self.assertEqual(2, len(right_child.value)) - self.assertEqual(0, split_node.split.feature_column) - self.assertAllClose(0.52, split_node.split.threshold) - - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.sparse_float_binary_split_default_left - self.assertEqual(2, len(left_child.value)) - self.assertEqual(0, split_node.split.feature_column) - self.assertAllClose(0.52, split_node.split.threshold) - - def testGenerateFeatureSplitCandidatesInactive(self): - with self.cached_session() as sess: - # The data looks like the following: - # Example | Gradients | Partition | Sparse Quantile | - # i0 | (0.2, 0.12) | 0 | 1 | - # i1 | (-0.5, 0.07) | 0 | N/A | - # i2 | (1.2, 0.2) | 0 | 0 | - # i3 | (4.0, 0.13) | 1 | 1 | - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - example_partitions = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - indices = array_ops.constant([[0, 0], [2, 0], [3, 0]], dtype=dtypes.int64) - values = array_ops.constant([0.52, 0.3, 0.52]) - sparse_column = sparse_tensor.SparseTensor(indices, values, [4, 1]) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = ordinal_split_handler.SparseSplitHandler( - l1_regularization=0.0, - l2_regularization=2.0, - tree_complexity_regularization=0.0, - min_node_weight=0.0, - epsilon=0.01, - num_quantiles=2, - feature_column_group_id=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - sparse_float_column=sparse_column, - init_stamp_token=0, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - example_partitions, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, False])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - example_partitions, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([False, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - - # During the first iteration, inequality split handlers are not going to - # have any splits. Make sure that we return not_ready in that case. - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - # The handler was inactive so it shouldn't any splits. - self.assertEqual(len(partitions), 0) - self.assertEqual(len(gains), 0) - self.assertEqual(len(splits), 0) - - def testEmpty(self): - with self.cached_session() as sess: - indices = constant_op.constant_v1([], dtype=dtypes.int64, shape=[0, 2]) - # No values in this feature column in this mini-batch. - values = constant_op.constant_v1([], dtype=dtypes.float32) - sparse_column = sparse_tensor.SparseTensor(indices, values, [4, 1]) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = ordinal_split_handler.SparseSplitHandler( - l1_regularization=0.0, - l2_regularization=2.0, - tree_complexity_regularization=0.0, - min_node_weight=0.0, - epsilon=0.01, - num_quantiles=2, - feature_column_group_id=0, - sparse_float_column=sparse_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS) - resources.initialize_resources(resources.shared_resources()).run() - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - self.assertEqual(len(partitions), 0) - self.assertEqual(len(gains), 0) - self.assertEqual(len(splits), 0) - - def testEmptyBuckets(self): - """Test that reproduces the case when quantile buckets were empty.""" - with self.cached_session() as sess: - sparse_column = array_ops.sparse_placeholder(dtypes.float32) - - # We have two batches - at first, a sparse feature is empty. - empty_indices = constant_op.constant_v1([], dtype=dtypes.int64, - shape=[0, 2]) - empty_values = constant_op.constant_v1([], dtype=dtypes.float32) - empty_sparse_column = sparse_tensor.SparseTensor(empty_indices, - empty_values, [4, 2]) - empty_sparse_column = empty_sparse_column.eval(session=sess) - - # For the second batch, the sparse feature is not empty. - non_empty_indices = array_ops.constant( - [[0, 0], [2, 1], [3, 2]], dtype=dtypes.int64, shape=[3, 2]) - non_empty_values = array_ops.constant( - [0.52, 0.3, 0.52], dtype=dtypes.float32) - non_empty_sparse_column = sparse_tensor.SparseTensor( - non_empty_indices, non_empty_values, [4, 2]) - non_empty_sparse_column = non_empty_sparse_column.eval(session=sess) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = ordinal_split_handler.SparseSplitHandler( - l1_regularization=0.0, - l2_regularization=2.0, - tree_complexity_regularization=0.0, - min_node_weight=0.0, - epsilon=0.01, - num_quantiles=2, - feature_column_group_id=0, - sparse_float_column=sparse_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS) - resources.initialize_resources(resources.shared_resources()).run() - gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) - hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) - partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([4, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - - # First, calculate quantiles and try to update on an empty data for a - # feature. - are_splits_ready = ( - sess.run( - are_splits_ready, - feed_dict={sparse_column: empty_sparse_column})) - self.assertFalse(are_splits_ready) - - update_2 = split_handler.update_stats_sync( - 1, - partition_ids, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - - # Now the feature in the second batch is not empty, but buckets - # calculated on the first batch are empty. - are_splits_ready2, partitions, gains, splits = ( - sess.run( - [are_splits_ready2, partitions, gains, splits], - feed_dict={sparse_column: non_empty_sparse_column})) - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - # Since the buckets were empty, we can't calculate the splits. - self.assertEqual(len(partitions), 0) - self.assertEqual(len(gains), 0) - self.assertEqual(len(splits), 0) - - def testDegenerativeCase(self): - with self.cached_session() as sess: - # One data example only, one leaf and thus one quantile bucket.The same - # situation is when all examples have the same values. This case was - # causing before a failure. - gradients = array_ops.constant([0.2]) - hessians = array_ops.constant([0.12]) - example_partitions = array_ops.constant([1], dtype=dtypes.int32) - indices = array_ops.constant([[0, 0]], dtype=dtypes.int64) - values = array_ops.constant([0.58]) - sparse_column = sparse_tensor.SparseTensor(indices, values, [1, 1]) - - gradient_shape = tensor_shape.TensorShape([]) - hessian_shape = tensor_shape.TensorShape([]) - class_id = -1 - - split_handler = ordinal_split_handler.SparseSplitHandler( - l1_regularization=0.0, - l2_regularization=2.0, - tree_complexity_regularization=0.0, - min_node_weight=0.0, - epsilon=0.01, - num_quantiles=2, - feature_column_group_id=0, - sparse_float_column=sparse_column, - init_stamp_token=0, - gradient_shape=gradient_shape, - hessian_shape=hessian_shape, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS) - resources.initialize_resources(resources.shared_resources()).run() - - empty_gradients, empty_hessians = get_empty_tensors( - gradient_shape, hessian_shape) - example_weights = array_ops.ones([1, 1], dtypes.float32) - - update_1 = split_handler.update_stats_sync( - 0, - example_partitions, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_1]): - are_splits_ready = split_handler.make_splits( - np.int64(0), np.int64(1), class_id)[0] - - with ops.control_dependencies([are_splits_ready]): - update_2 = split_handler.update_stats_sync( - 1, - example_partitions, - gradients, - hessians, - empty_gradients, - empty_hessians, - example_weights, - is_active=array_ops.constant([True, True])) - with ops.control_dependencies([update_2]): - are_splits_ready2, partitions, gains, splits = ( - split_handler.make_splits(np.int64(1), np.int64(2), class_id)) - are_splits_ready, are_splits_ready2, partitions, gains, splits = ( - sess.run([ - are_splits_ready, are_splits_ready2, partitions, gains, splits - ])) - - # During the first iteration, inequality split handlers are not going to - # have any splits. Make sure that we return not_ready in that case. - self.assertFalse(are_splits_ready) - self.assertTrue(are_splits_ready2) - - self.assertAllEqual([1], partitions) - self.assertAllEqual([0.0], gains) - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - split_node = split_info.split_node.sparse_float_binary_split_default_left - - self.assertEqual(0, split_node.split.feature_column) - - self.assertAllClose(0.58, split_node.split.threshold) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/lib/learner/common/accumulators/class-partition-key.h b/tensorflow/contrib/boosted_trees/lib/learner/common/accumulators/class-partition-key.h deleted file mode 100644 index 3c54868951a..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/common/accumulators/class-partition-key.h +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_ACCUMULATORS_CLASS_PARTITION_KEY_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_ACCUMULATORS_CLASS_PARTITION_KEY_H_ - -#include "tensorflow/core/lib/hash/hash.h" - -namespace tensorflow { -namespace boosted_trees { -namespace learner { - -// Key into a specific class and partition to accumulate stats -// for the specified feature id. A feature id can be the quantile -// for a float feature or the hash/dictionary entry for a string feature. -struct ClassPartitionKey { - ClassPartitionKey() : class_id(-1), partition_id(-1), feature_id(-1) {} - - ClassPartitionKey(uint32 c, uint32 p, uint64 f) - : class_id(c), partition_id(p), feature_id(f) {} - - bool operator==(const ClassPartitionKey& other) const { - return (feature_id == other.feature_id) && - (partition_id == other.partition_id) && (class_id == other.class_id); - } - - // Hasher for ClassPartitionKey. - struct Hash { - size_t operator()(const ClassPartitionKey& key) const { - uint64 class_partition = - (static_cast(key.partition_id) << 32) | (key.class_id); - return Hash64Combine(class_partition, key.feature_id); - } - }; - - // Class to predict for, this is constant for binary tasks. - uint32 class_id; - - // Tree partition defined by traversing the tree to the leaf. - uint32 partition_id; - - // Feature Id within the feature column. - uint64 feature_id; -}; - -} // namespace learner -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_ACCUMULATORS_CLASS_PARTITION_KEY_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/learner/common/accumulators/feature-stats-accumulator.h b/tensorflow/contrib/boosted_trees/lib/learner/common/accumulators/feature-stats-accumulator.h deleted file mode 100644 index ec4e7c52bb5..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/common/accumulators/feature-stats-accumulator.h +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_ACCUMULATORS_FEATURE_STATS_ACCUMULATOR_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_ACCUMULATORS_FEATURE_STATS_ACCUMULATOR_H_ - -#include -#include - -#include "tensorflow/contrib/boosted_trees/lib/learner/common/accumulators/class-partition-key.h" - -namespace tensorflow { -namespace boosted_trees { -namespace learner { - -// Feature stats accumulator to aggregate stats across various -// feature columns. This class is thread compatible not safe, the user -// must ensure proper synchronization if many threads update overlapping -// feature columns. -template -class FeatureStatsAccumulator { - public: - using FeatureStats = - std::unordered_map; - - explicit FeatureStatsAccumulator(size_t num_feature_columns, - Accumulator accumulator = Accumulator()) - : accumulator_(accumulator), feature_column_stats_(num_feature_columns) {} - - // Delete copy and assign. - FeatureStatsAccumulator(const FeatureStatsAccumulator& other) = delete; - FeatureStatsAccumulator& operator=(const FeatureStatsAccumulator& other) = - delete; - - // Adds stats for the specified class, partition and feature within - // the desired slot. - void AddStats(uint32 slot_id, uint32 class_id, uint32 partition_id, - uint64 feature_id, const StatsType& stats) { - accumulator_(stats, &feature_column_stats_[slot_id][ClassPartitionKey( - class_id, partition_id, feature_id)]); - } - - // Retrieves stats for the specified class, partition and feature - // within the desired feature column. Default stats are returned if no match - // can be found. Note that the feature column index must be valid. - StatsType GetStats(uint32 slot_id, uint32 class_id, uint32 partition_id, - uint64 feature_id) const { - auto it = feature_column_stats_[slot_id].find( - ClassPartitionKey(class_id, partition_id, feature_id)); - return it != feature_column_stats_[slot_id].end() ? it->second - : StatsType(); - } - - // Returns feature stats for a given slot. - FeatureStats GetFeatureStats(uint32 slot_id) const { - return feature_column_stats_[slot_id]; - } - - private: - // Accumulator method to use. - const Accumulator accumulator_; - - // Vector of stats indexed by the feature column. - std::vector feature_column_stats_; -}; - -} // namespace learner -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_ACCUMULATORS_FEATURE_STATS_ACCUMULATOR_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/learner/common/accumulators/feature-stats-accumulator_test.cc b/tensorflow/contrib/boosted_trees/lib/learner/common/accumulators/feature-stats-accumulator_test.cc deleted file mode 100644 index fdbd670120f..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/common/accumulators/feature-stats-accumulator_test.cc +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/learner/common/accumulators/feature-stats-accumulator.h" - -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace boosted_trees { -namespace learner { -namespace { - -struct TestStats { - TestStats& operator+=(const TestStats& other) { - s1 += other.s1; - s2 += other.s2; - return (*this); - } - - float s1; - float s2; -}; - -struct TestStatsAccumulator { - void operator()(const TestStats& from, TestStats* to) const { (*to) += from; } -}; -#define EXPECT_STATS_EQ(val1, val2) \ - EXPECT_FLOAT_EQ(val1.s1, val2.s1); \ - EXPECT_FLOAT_EQ(val1.s2, val2.s2); - -using FeatureStatsAccumulator = - FeatureStatsAccumulator; - -class FeatureStatsAccumulatorTest : public ::testing::Test {}; - -TEST_F(FeatureStatsAccumulatorTest, Empty) { - FeatureStatsAccumulator accumulator(1); - TestStats stats = {0, 0}; - - EXPECT_STATS_EQ(stats, accumulator.GetStats(0, 2, 1, 234)); -} - -TEST_F(FeatureStatsAccumulatorTest, OneFeatureOneGrad) { - FeatureStatsAccumulator accumulator(1); - TestStats stats = {-12.023f, 8.2f}; - accumulator.AddStats(0, 2, 1, 234, stats); - - EXPECT_STATS_EQ(stats, accumulator.GetStats(0, 2, 1, 234)); -} - -TEST_F(FeatureStatsAccumulatorTest, OneFeatureAggregateGrad) { - FeatureStatsAccumulator accumulator(1); - TestStats stats1 = {-12.023f, 8.2f}; - accumulator.AddStats(0, 2, 1, 234, stats1); - TestStats stats2 = {4.46f, 1.9f}; - accumulator.AddStats(0, 2, 1, 234, stats2); - TestStats expected = {-7.563f, 10.1f}; - EXPECT_STATS_EQ(expected, accumulator.GetStats(0, 2, 1, 234)); -} - -TEST_F(FeatureStatsAccumulatorTest, TwoFeaturesOneGrad) { - FeatureStatsAccumulator accumulator(1); - TestStats stats1 = {-12.023f, 8.2f}; - accumulator.AddStats(0, 1, 0, 34, stats1); - TestStats stats2 = {4.46f, 1.9f}; - accumulator.AddStats(0, 1, 0, 91, stats2); - - EXPECT_STATS_EQ(stats1, accumulator.GetStats(0, 1, 0, 34)); - EXPECT_STATS_EQ(stats2, accumulator.GetStats(0, 1, 0, 91)); -} - -} // namespace -} // namespace learner -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/learner/common/partitioners/example_partitioner.cc b/tensorflow/contrib/boosted_trees/lib/learner/common/partitioners/example_partitioner.cc deleted file mode 100644 index 1f84b1b06a6..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/common/partitioners/example_partitioner.cc +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/learner/common/partitioners/example_partitioner.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/parallel_for.h" - -namespace tensorflow { -namespace boosted_trees { -namespace learner { - -void ExamplePartitioner::UpdatePartitions( - const boosted_trees::trees::DecisionTreeConfig& tree_config, - const boosted_trees::utils::BatchFeatures& features, - const int desired_parallelism, thread::ThreadPool* const thread_pool, - int32* example_partition_ids) { - // Get batch size. - const int64 batch_size = features.batch_size(); - if (batch_size <= 0) { - return; - } - - // Lambda for doing a block of work. - auto partition_examples_subset = [&tree_config, &features, - &example_partition_ids](const int64 start, - const int64 end) { - if (TF_PREDICT_TRUE(tree_config.nodes_size() > 0)) { - auto examples_iterable = features.examples_iterable(start, end); - for (const auto& example : examples_iterable) { - int32& example_partition = example_partition_ids[example.example_idx]; - example_partition = boosted_trees::trees::DecisionTree::Traverse( - tree_config, example_partition, example); - DCHECK_GE(example_partition, 0); - } - } else { - std::fill(example_partition_ids + start, example_partition_ids + end, 0); - } - }; - - // Parallelize partitioning over the batch. - boosted_trees::utils::ParallelFor(batch_size, desired_parallelism, - thread_pool, partition_examples_subset); -} - -void ExamplePartitioner::PartitionExamples( - const boosted_trees::trees::DecisionTreeConfig& tree_config, - const boosted_trees::utils::BatchFeatures& features, - const int desired_parallelism, thread::ThreadPool* const thread_pool, - int32* example_partition_ids) { - // Get batch size. - const int64 batch_size = features.batch_size(); - if (batch_size <= 0) { - return; - } - - // Lambda for doing a block of work. - auto partition_examples_subset = [&tree_config, &features, - &example_partition_ids](const int64 start, - const int64 end) { - if (TF_PREDICT_TRUE(tree_config.nodes_size() > 0)) { - auto examples_iterable = features.examples_iterable(start, end); - for (const auto& example : examples_iterable) { - uint32 partition = boosted_trees::trees::DecisionTree::Traverse( - tree_config, 0, example); - example_partition_ids[example.example_idx] = partition; - DCHECK_GE(partition, 0); - } - } else { - std::fill(example_partition_ids + start, example_partition_ids + end, 0); - } - }; - - // Parallelize partitioning over the batch. - boosted_trees::utils::ParallelFor(batch_size, desired_parallelism, - thread_pool, partition_examples_subset); -} - -} // namespace learner -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/learner/common/partitioners/example_partitioner.h b/tensorflow/contrib/boosted_trees/lib/learner/common/partitioners/example_partitioner.h deleted file mode 100644 index 37a71037041..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/common/partitioners/example_partitioner.h +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_PARTITIONERS_EXAMPLE_PARTITIONER_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_PARTITIONERS_EXAMPLE_PARTITIONER_H_ - -#include -#include "tensorflow/contrib/boosted_trees/lib/trees/decision_tree.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/batch_features.h" -#include "tensorflow/core/lib/core/threadpool.h" - -namespace tensorflow { -namespace boosted_trees { -namespace learner { - -// Partitions examples based on the path through the current tree. -class ExamplePartitioner { - public: - // Updates partitions from previous set using the current tree structure by - // traversing sub-roots for each example. This method can be optionally - // parallelized using the passed thread pool. - static void UpdatePartitions(const trees::DecisionTreeConfig& tree_config, - const utils::BatchFeatures& features, - int desired_parallelism, - thread::ThreadPool* const thread_pool, - int32* example_partition_ids); - - // Partitions examples using the current tree structure by traversing from - // root for each example. This method can be optionally parallelized using - // the passed thread pool. - static void PartitionExamples(const trees::DecisionTreeConfig& tree_config, - const utils::BatchFeatures& features, - int desired_parallelism, - thread::ThreadPool* const thread_pool, - int32* example_partition_ids); -}; - -} // namespace learner -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_PARTITIONERS_EXAMPLE_PARTITIONER_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/learner/common/partitioners/example_partitioner_test.cc b/tensorflow/contrib/boosted_trees/lib/learner/common/partitioners/example_partitioner_test.cc deleted file mode 100644 index 0d5326aa456..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/common/partitioners/example_partitioner_test.cc +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/learner/common/partitioners/example_partitioner.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace boosted_trees { -namespace learner { -namespace { - -class ExamplePartitionerTest : public ::testing::Test { - protected: - ExamplePartitionerTest() - : thread_pool_(tensorflow::Env::Default(), "test_pool", 2), - batch_features_(2) { - dense_matrix_ = test::AsTensor({7.0f, -2.0f}, {2, 1}); - TF_EXPECT_OK( - batch_features_.Initialize({dense_matrix_}, {}, {}, {}, {}, {}, {})); - } - - thread::ThreadPool thread_pool_; - Tensor dense_matrix_; - boosted_trees::utils::BatchFeatures batch_features_; -}; - -TEST_F(ExamplePartitionerTest, EmptyTree) { - boosted_trees::trees::DecisionTreeConfig tree_config; - std::vector example_partition_ids(2); - ExamplePartitioner::UpdatePartitions(tree_config, batch_features_, 1, - &thread_pool_, - example_partition_ids.data()); - EXPECT_EQ(0, example_partition_ids[0]); - EXPECT_EQ(0, example_partition_ids[1]); -} - -TEST_F(ExamplePartitionerTest, UpdatePartitions) { - // Create tree with one split. - // TODO(salehay): figure out if we can use PARSE_TEXT_PROTO. - boosted_trees::trees::DecisionTreeConfig tree_config; - auto* split = tree_config.add_nodes()->mutable_dense_float_binary_split(); - split->set_feature_column(0); - split->set_threshold(3.0f); - split->set_left_id(1); - split->set_right_id(2); - tree_config.add_nodes()->mutable_leaf(); - tree_config.add_nodes()->mutable_leaf(); - - // Partition input: - // Instance 1 has !(7 <= 3) => go right => leaf 2. - // Instance 2 has (-2 <= 3) => go left => leaf 1. - std::vector example_partition_ids(2); - ExamplePartitioner::UpdatePartitions(tree_config, batch_features_, 1, - &thread_pool_, - example_partition_ids.data()); - EXPECT_EQ(2, example_partition_ids[0]); - EXPECT_EQ(1, example_partition_ids[1]); -} - -TEST_F(ExamplePartitionerTest, PartitionExamples) { - // Create tree with one split. - // TODO(salehay): figure out if we can use PARSE_TEXT_PROTO. - boosted_trees::trees::DecisionTreeConfig tree_config; - auto* split = tree_config.add_nodes()->mutable_dense_float_binary_split(); - split->set_feature_column(0); - split->set_threshold(3.0f); - split->set_left_id(1); - split->set_right_id(2); - tree_config.add_nodes()->mutable_leaf(); - tree_config.add_nodes()->mutable_leaf(); - - // Partition input: - // Instance 1 has !(7 <= 3) => go right => leaf 2. - // Instance 2 has (-2 <= 3) => go left => leaf 1. - std::vector example_partition_ids(2); - ExamplePartitioner::PartitionExamples(tree_config, batch_features_, 1, - &thread_pool_, - example_partition_ids.data()); - EXPECT_EQ(2, example_partition_ids[0]); - EXPECT_EQ(1, example_partition_ids[1]); -} - -} // namespace -} // namespace learner -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/learner/common/stats/feature-split-candidate.h b/tensorflow/contrib/boosted_trees/lib/learner/common/stats/feature-split-candidate.h deleted file mode 100644 index 382b85cf0b2..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/common/stats/feature-split-candidate.h +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_STATS_FEATURE_SPLIT_CANDIDATE_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_STATS_FEATURE_SPLIT_CANDIDATE_H_ - -#include "tensorflow/contrib/boosted_trees/lib/learner/common/stats/split-stats.h" -#include "tensorflow/contrib/boosted_trees/proto/tree_config.pb.h" - -namespace tensorflow { -namespace boosted_trees { -namespace learner { -namespace stochastic { - -// FeatureSplitCandidate holds the split candidate node along with the stats. -struct FeatureSplitCandidate { - // Empty split candidate. - explicit FeatureSplitCandidate(const int output_length) - : feature_column_slot_id(kInvalidFeatureColumnSlot), - split_stats(output_length) {} - - // Feature binary split candidate. - FeatureSplitCandidate(const int64 fc_slot_id, - const boosted_trees::trees::TreeNode& node, - const SplitStats& stats) - : feature_column_slot_id(fc_slot_id), - tree_node(node), - split_stats(stats) {} - - // Globally unique slot Id identifying the feature column - // used in this split candidates. - int64 feature_column_slot_id; - - // Tree node for the candidate split. - boosted_trees::trees::TreeNode tree_node; - - // Split stats. - SplitStats split_stats; - - // Invalid feature column slot reserved value. - static constexpr int64 kInvalidFeatureColumnSlot = -1; -}; - -} // namespace stochastic -} // namespace learner -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_STATS_FEATURE_SPLIT_CANDIDATE_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/learner/common/stats/gradient-stats.h b/tensorflow/contrib/boosted_trees/lib/learner/common/stats/gradient-stats.h deleted file mode 100644 index 3dd03215d88..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/common/stats/gradient-stats.h +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_STATS_GRADIENT_STATS_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_STATS_GRADIENT_STATS_H_ - -#include - -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_util.h" - -namespace tensorflow { -namespace boosted_trees { -namespace learner { -namespace stochastic { - -const double kEps = 1e-6; - -// A data structure for accumulating a Tensor value. -struct TensorStat { - TensorStat() {} - - explicit TensorStat(const float v) : t(DT_FLOAT, TensorShape({1})) { - t.flat()(0) = v; - } - - explicit TensorStat(const Tensor& rt) : t(tensor::DeepCopy(rt)) {} - - TensorStat(const TensorStat& ts) : t(tensor::DeepCopy(ts.t)) {} - - TensorStat& operator+=(const TensorStat& other) { - if (t.NumElements() == 0) { - t = tensor::DeepCopy(other.t); - return (*this); - } - CHECK(t.shape() == other.t.shape()) - << "My shape = " << t.shape().DebugString() - << " Other shape = " << other.t.shape().DebugString(); - auto me_flat = t.unaligned_flat(); - auto other_flat = other.t.unaligned_flat(); - for (int i = 0; i < me_flat.size(); i++) { - me_flat(i) += other_flat(i); - } - return (*this); - } - - TensorStat& operator-=(const TensorStat& other) { - if (other.t.NumElements() == 0) { - return (*this); - } - CHECK(t.shape() == other.t.shape()) - << "My shape = " << t.shape().DebugString() - << " Other shape = " << other.t.shape().DebugString(); - auto me_flat = t.unaligned_flat(); - auto other_flat = other.t.unaligned_flat(); - for (int i = 0; i < me_flat.size(); i++) { - me_flat(i) -= other_flat(i); - } - return (*this); - } - - TensorStat& operator*=(float value) { - auto me_flat = t.unaligned_flat(); - for (size_t i = 0; i < me_flat.size(); i++) { - me_flat(i) *= value; - } - return (*this); - } - - bool IsZero() const { - auto me_flat = t.unaligned_flat(); - for (int i = 0; i < me_flat.size(); i++) { - if (me_flat(i) != 0.0f) { - return false; - } - } - return true; - } - - // Checks if the L^2 magnitude of the tensor is less than eps. - bool IsAlmostZero(const float eps = kEps) const { - auto me_flat = t.unaligned_flat(); - double s = 0.0; - for (int i = 0; i < me_flat.size(); i++) { - s += me_flat(i) * me_flat(i); - if (s > eps * eps) { - return false; - } - } - return true; - } - - float Magnitude() const { - auto me_flat = t.unaligned_flat(); - double s = 0.0; - for (int i = 0; i < me_flat.size(); i++) { - s += me_flat(i) * me_flat(i); - } - return sqrt(s); - } - - string DebugString() const { return t.DebugString(); } - - Tensor t; -}; - -// GradientStats holds first and second order gradient stats. -struct GradientStats { - GradientStats() {} - - // Legacy constructor for tests - GradientStats(float g, float h) : first(g), second(h) {} - - GradientStats(const Tensor& g, const Tensor& h) : first(g), second(h) {} - - GradientStats(const Tensor& g, const Tensor& h, int64 example_index) - : first(g.Slice(example_index, example_index + 1)), - second(h.Slice(example_index, example_index + 1)) {} - - GradientStats& operator+=(const GradientStats& other) { - first += other.first; - second += other.second; - return (*this); - } - - GradientStats& operator*=(float value) { - first *= value; - second *= value; - return (*this); - } - - GradientStats& operator-=(const GradientStats& other) { - first -= other.first; - second -= other.second; - return (*this); - } - - bool IsZero() const { return first.IsZero() && second.IsZero(); } - - bool IsAlmostZero(const float eps = kEps) const { - return first.IsAlmostZero(eps) && second.IsAlmostZero(eps); - } - - float Magnitude() const { return second.Magnitude(); } - - string DebugString() const { - return "First = " + first.DebugString() + - " Second = " + second.DebugString(); - } - - TensorStat first; - TensorStat second; -}; - -struct GradientStatsAccumulator { - void operator()(const GradientStats& from, GradientStats* to) const { - (*to) += from; - } -}; - -inline GradientStats operator+(const GradientStats& a, const GradientStats& b) { - GradientStats ret(a); - ret += b; - return ret; -} - -inline GradientStats operator-(const GradientStats& a, const GradientStats& b) { - GradientStats ret(a); - ret -= b; - return ret; -} - -// Helper macro to check gradient stats approximate equality. -#define EXPECT_GRADIENT_STATS_EQ(val1, val2) \ - EXPECT_TRUE((val1 - val2).IsAlmostZero()); - -} // namespace stochastic -} // namespace learner -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_STATS_GRADIENT_STATS_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/learner/common/stats/node-stats.h b/tensorflow/contrib/boosted_trees/lib/learner/common/stats/node-stats.h deleted file mode 100644 index 4f4456a7535..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/common/stats/node-stats.h +++ /dev/null @@ -1,301 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_STATS_NODE_STATS_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_STATS_NODE_STATS_H_ - -#include "third_party/eigen3/Eigen/Core" -#include "third_party/eigen3/Eigen/Eigenvalues" -#include "tensorflow/contrib/boosted_trees/lib/learner/common/stats/gradient-stats.h" -#include "tensorflow/contrib/boosted_trees/proto/learner.pb.h" -#include "tensorflow/contrib/boosted_trees/proto/tree_config.pb.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/lib/strings/stringprintf.h" - -namespace tensorflow { -namespace boosted_trees { -namespace learner { -namespace stochastic { - -using tensorflow::boosted_trees::learner::LearnerConfig; -using tensorflow::boosted_trees::learner::LearnerConfig_MultiClassStrategy; -using tensorflow::boosted_trees::learner:: - LearnerConfig_MultiClassStrategy_DIAGONAL_HESSIAN; -using tensorflow::boosted_trees::learner:: - LearnerConfig_MultiClassStrategy_FULL_HESSIAN; -using tensorflow::boosted_trees::learner:: - LearnerConfig_MultiClassStrategy_TREE_PER_CLASS; - -// NodeStats holds aggregate gradient stats as well as metadata about the node. -struct NodeStats { - // Initialize the NodeStats with 0 stats. We need the output length - // so that we can make weight_contribution the right length. - explicit NodeStats(const int output_length) - : weight_contribution(output_length, 0.0f), gain(0) {} - - NodeStats(const LearnerConfig& learner_config, - const GradientStats& grad_stats) - : NodeStats(learner_config.regularization().l1(), - learner_config.regularization().l2(), - learner_config.constraints().min_node_weight(), - learner_config.multi_class_strategy(), grad_stats) {} - - NodeStats(float l1_reg, float l2_reg, float min_node_weight, - const LearnerConfig_MultiClassStrategy& strategy, - const GradientStats& grad_stats) - : gradient_stats(grad_stats), gain(0) { - switch (strategy) { - case LearnerConfig::TREE_PER_CLASS: { - float g; - float h; - // Initialize now in case of early return. - weight_contribution.push_back(0.0f); - - if (grad_stats.first.t.NumElements() == 0 || - grad_stats.second.t.NumElements() == 0) { - return; - } - - g = grad_stats.first.t.unaligned_flat()(0); - h = grad_stats.second.t.unaligned_flat()(0); - - if (grad_stats.IsAlmostZero() || h <= min_node_weight) { - return; - } - - // Apply L1 regularization. - if (l1_reg > 0) { - if (g > l1_reg) { - g -= l1_reg; - } else if (g < -l1_reg) { - g += l1_reg; - } else { - return; - } - } - - // The node gain is given by: (l'^2) / (l'' + l2_reg) and the node - // weight - // contribution is given by: (-l') / (l'' + l2_reg). - // Note that l'' can't be zero here because of the min node weight check - // since min node weight must be >= 0. - weight_contribution[0] = -g / (h + l2_reg); - gain = (weight_contribution[0] * -g); - break; - } - case LearnerConfig::FULL_HESSIAN: { - weight_contribution.clear(); - - if (grad_stats.first.t.NumElements() == 0 || - grad_stats.second.t.NumElements() == 0) { - return; - } - const int64 grad_dim = grad_stats.first.t.dim_size(1); - - QCHECK(grad_stats.first.t.dims() == 2) - << strings::Printf("Gradient should be of rank 2, got rank %d", - grad_stats.first.t.dims()); - QCHECK(grad_stats.first.t.dim_size(0) == 1) << strings::Printf( - "Gradient must be of shape 1 x %lld, got %lld x %lld", grad_dim, - grad_stats.first.t.dim_size(0), grad_dim); - QCHECK(grad_stats.second.t.dims() == 3) - << strings::Printf("Hessian should be of rank 3, got rank %d", - grad_stats.second.t.dims()); - QCHECK(grad_stats.second.t.shape() == - TensorShape({1, grad_dim, grad_dim})) - << strings::Printf( - "Hessian must be of shape 1 x %lld x %lld, got %lld x % lld " - " x % lld ", - grad_dim, grad_dim, grad_stats.second.t.shape().dim_size(0), - grad_stats.second.t.shape().dim_size(1), - grad_stats.second.t.shape().dim_size(2)); - - // Check if we're violating min weight constraint. - - if (grad_stats.IsAlmostZero() || - grad_stats.second.Magnitude() <= min_node_weight) { - return; - } - // TODO(nponomareva): figure out l1 in matrix form. - // g is a vector of gradients, H is a hessian matrix. - Eigen::VectorXf g = TensorToEigenVector(grad_stats.first.t, grad_dim); - - Eigen::MatrixXf hessian = - TensorToEigenMatrix(grad_stats.second.t, grad_dim, grad_dim); - // I is an identity matrix. - // The gain in general form is g^T (H+l2 I)^-1 g. - // The node weights are -(H+l2 I)^-1 g. - Eigen::MatrixXf identity; - identity.setIdentity(grad_dim, grad_dim); - - Eigen::MatrixXf hessian_and_reg = hessian + l2_reg * identity; - - CalculateWeightAndGain(hessian_and_reg, g); - break; - } - case LearnerConfig::DIAGONAL_HESSIAN: { - weight_contribution.clear(); - if (grad_stats.first.t.NumElements() == 0 || - grad_stats.second.t.NumElements() == 0) { - return; - } - const int64 grad_dim = grad_stats.first.t.dim_size(1); - - QCHECK(grad_stats.first.t.dims() == 2) - << strings::Printf("Gradient should be of rank 2, got rank %d", - grad_stats.first.t.dims()); - QCHECK(grad_stats.first.t.dim_size(0) == 1) << strings::Printf( - "Gradient must be of shape 1 x %lld, got %lld x %lld", grad_dim, - grad_stats.first.t.dim_size(0), grad_dim); - QCHECK(grad_stats.second.t.dims() == 2) - << strings::Printf("Hessian should be of rank 2, got rank %d", - grad_stats.second.t.dims()); - QCHECK(grad_stats.second.t.shape() == TensorShape({1, grad_dim})) - << strings::Printf( - "Hessian must be of shape 1 x %lld, got %lld x %lld", - grad_dim, grad_stats.second.t.shape().dim_size(0), - grad_stats.second.t.shape().dim_size(1)); - - // Check if we're violating min weight constraint. - if (grad_stats.IsAlmostZero() || - grad_stats.second.Magnitude() <= min_node_weight) { - return; - } - // TODO(nponomareva): figure out l1 in matrix form. - // Diagonal of the hessian. - Eigen::ArrayXf hessian = - TensorToEigenArray(grad_stats.second.t, grad_dim); - Eigen::ArrayXf hessian_and_reg = hessian + l2_reg; - - // Check if any of the elements are zeros. - bool invertible = true; - for (int i = 0; i < hessian_and_reg.size(); ++i) { - if (hessian_and_reg[i] == 0.0) { - invertible = false; - break; - } - } - if (invertible) { - Eigen::ArrayXf g = TensorToEigenArray(grad_stats.first.t, grad_dim); - // Operations on arrays are element wise. The formulas are as for full - // hessian, but for hessian of diagonal form they are simplified. - Eigen::ArrayXf ones = Eigen::ArrayXf::Ones(grad_dim); - Eigen::ArrayXf temp = ones / hessian_and_reg; - Eigen::ArrayXf weight = -temp * g; - - // Copy over weights to weight_contribution. - weight_contribution = - std::vector(weight.data(), weight.data() + weight.rows()); - gain = (-g * weight).sum(); - } else { - Eigen::VectorXf g = TensorToEigenVector(grad_stats.first.t, grad_dim); - // Hessian is not invertible. We will go the same route as in full - // hessian to get an approximate solution. - CalculateWeightAndGain(hessian_and_reg.matrix().asDiagonal(), g); - } - break; - } - default: - LOG(FATAL) << "Unknown multi-class strategy " << strategy; - break; - } - } - - string DebugString() const { - return strings::StrCat( - gradient_stats.DebugString(), "\n", - "Weight_contrib = ", str_util::Join(weight_contribution, ","), - "Gain = ", gain); - } - - // Use these node stats to populate a Leaf's model. - void FillLeaf(const int class_id, boosted_trees::trees::Leaf* leaf) const { - if (class_id == -1) { - for (int i = 0; i < weight_contribution.size(); i++) { - leaf->mutable_vector()->add_value(weight_contribution[i]); - } - } else { - CHECK(weight_contribution.size() == 1) - << "Weight contribution size = " << weight_contribution.size(); - leaf->mutable_sparse_vector()->add_index(class_id); - leaf->mutable_sparse_vector()->add_value(weight_contribution[0]); - } - } - - // Sets the weight_contribution and gain member variables based on the - // given regularized Hessian and gradient vector g. - void CalculateWeightAndGain(const Eigen::MatrixXf& hessian_and_reg, - const Eigen::VectorXf& g) { - // The gain in general form is g^T (Hessian_and_regularization)^-1 g. - // The node weights are -(Hessian_and_regularization)^-1 g. - Eigen::VectorXf weight; - // If we want to calculate x = K^-1 v, instead of explicitly calculating - // K^-1 and multiplying by v, we can solve this matrix equation using - // solve method. - weight = -hessian_and_reg.colPivHouseholderQr().solve(g); - // Copy over weights to weight_contribution. - weight_contribution = - std::vector(weight.data(), weight.data() + weight.rows()); - - gain = -g.transpose() * weight; - } - - static Eigen::MatrixXf TensorToEigenMatrix(const Tensor& tensor, - const int num_rows, - const int num_cols) { - return Eigen::Map(tensor.flat().data(), - num_rows, num_cols); - } - - static Eigen::VectorXf TensorToEigenVector(const Tensor& tensor, - const int num_elements) { - return Eigen::Map(tensor.flat().data(), - num_elements); - } - - static Eigen::ArrayXf TensorToEigenArray(const Tensor& tensor, - const int num_elements) { - return Eigen::Map(tensor.flat().data(), - num_elements); - } - - GradientStats gradient_stats; - std::vector weight_contribution; - float gain; -}; - -// Helper macro to check std::vector approximate equality. -#define EXPECT_VECTOR_FLOAT_EQ(x, y) \ - { \ - EXPECT_EQ((x).size(), (y).size()); \ - for (int i = 0; i < (x).size(); ++i) { \ - EXPECT_FLOAT_EQ((x)[i], (y)[i]); \ - } \ - } - -// Helper macro to check node stats approximate equality. -#define EXPECT_NODE_STATS_EQ(val1, val2) \ - EXPECT_GRADIENT_STATS_EQ(val1.gradient_stats, val2.gradient_stats); \ - EXPECT_VECTOR_FLOAT_EQ(val1.weight_contribution, val2.weight_contribution); \ - EXPECT_FLOAT_EQ(val1.gain, val2.gain); - -} // namespace stochastic -} // namespace learner -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_STATS_NODE_STATS_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/learner/common/stats/node-stats_test.cc b/tensorflow/contrib/boosted_trees/lib/learner/common/stats/node-stats_test.cc deleted file mode 100644 index 8bca132acfd..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/common/stats/node-stats_test.cc +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/learner/common/stats/node-stats.h" - -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/platform/test.h" - -using std::vector; -using tensorflow::test::AsTensor; - -namespace tensorflow { -namespace boosted_trees { -namespace learner { -namespace stochastic { -namespace { - -const double kDelta = 1e-5; - -TEST(NodeStatsTest, AlmostZero) { - LearnerConfig learner_config; - learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); - NodeStats node_stats(learner_config, GradientStats(1e-8f, 1e-8f)); - EXPECT_EQ(0, node_stats.weight_contribution[0]); - EXPECT_EQ(0, node_stats.gain); -} - -TEST(NodeStatsTest, LessThanMinWeightConstraint) { - LearnerConfig learner_config; - learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); - learner_config.mutable_constraints()->set_min_node_weight(3.2f); - NodeStats node_stats(learner_config, GradientStats(7.32f, 1.63f)); - EXPECT_EQ(0, node_stats.weight_contribution[0]); - EXPECT_EQ(0, node_stats.gain); -} - -TEST(NodeStatsTest, L1RegSquashed) { - LearnerConfig learner_config; - learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); - learner_config.mutable_regularization()->set_l1(10.0f); - NodeStats node_stats(learner_config, GradientStats(7.32f, 1.63f)); - EXPECT_EQ(0, node_stats.weight_contribution[0]); - EXPECT_EQ(0, node_stats.gain); -} - -TEST(NodeStatsTest, L1RegPos) { - LearnerConfig learner_config; - learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); - learner_config.mutable_regularization()->set_l1(5.0f); - NodeStats node_stats(learner_config, GradientStats(7.32f, 1.63f)); - const float expected_clipped_grad = 7.32f - 5.0f; - const float expected_weight_contribution = -expected_clipped_grad / 1.63f; - const float expected_gain = - expected_clipped_grad * expected_clipped_grad / 1.63f; - EXPECT_FLOAT_EQ(expected_weight_contribution, - node_stats.weight_contribution[0]); - EXPECT_FLOAT_EQ(expected_gain, node_stats.gain); -} - -TEST(NodeStatsTest, L1RegNeg) { - LearnerConfig learner_config; - learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); - learner_config.mutable_regularization()->set_l1(5.0f); - NodeStats node_stats(learner_config, GradientStats(-7.32f, 1.63f)); - const float expected_clipped_grad = -7.32f + 5.0f; - const float expected_weight_contribution = -expected_clipped_grad / 1.63f; - const float expected_gain = - expected_clipped_grad * expected_clipped_grad / 1.63f; - EXPECT_FLOAT_EQ(expected_weight_contribution, - node_stats.weight_contribution[0]); - EXPECT_FLOAT_EQ(expected_gain, node_stats.gain); -} - -TEST(NodeStatsTest, L2Reg) { - LearnerConfig learner_config; - learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); - learner_config.mutable_regularization()->set_l2(8.0f); - NodeStats node_stats(learner_config, GradientStats(7.32f, 1.63f)); - const float expected_denom = 1.63f + 8.0f; - const float expected_weight_contribution = -7.32f / expected_denom; - const float expected_gain = 7.32f * 7.32f / expected_denom; - EXPECT_FLOAT_EQ(expected_weight_contribution, - node_stats.weight_contribution[0]); - EXPECT_FLOAT_EQ(expected_gain, node_stats.gain); -} - -TEST(NodeStatsTest, L1L2Reg) { - LearnerConfig learner_config; - learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); - learner_config.mutable_regularization()->set_l1(5.0f); - learner_config.mutable_regularization()->set_l2(8.0f); - NodeStats node_stats(learner_config, GradientStats(7.32f, 1.63f)); - const float expected_clipped_grad = 7.32f - 5.0f; - const float expected_denom = 1.63f + 8.0f; - const float expected_weight_contribution = - -expected_clipped_grad / expected_denom; - const float expected_gain = - expected_clipped_grad * expected_clipped_grad / expected_denom; - EXPECT_FLOAT_EQ(expected_weight_contribution, - node_stats.weight_contribution[0]); - EXPECT_FLOAT_EQ(expected_gain, node_stats.gain); -} - -TEST(NodeStatsTest, MulticlassFullHessianTest) { - LearnerConfig learner_config; - learner_config.set_multi_class_strategy(LearnerConfig::FULL_HESSIAN); - learner_config.mutable_regularization()->set_l2(0.3f); - - const int kNumClasses = 4; - const auto& g_shape = TensorShape({1, kNumClasses}); - Tensor g = AsTensor({0.5, 0.33, -9, 1}, g_shape); - const auto& hessian_shape = TensorShape({1, kNumClasses, kNumClasses}); - Tensor h = AsTensor({3, 5, 7, 8, 5, 4, 1, 5, 7, 1, 8, 4, 8, 5, 4, 9}, - hessian_shape); - - NodeStats node_stats(learner_config, GradientStats(g, h)); - - // Index 1 has 0 value because of l1 regularization, - std::vector expected_weight = {0.9607576, 0.4162569, 0.9863192, - -1.5820024}; - - EXPECT_EQ(kNumClasses, node_stats.weight_contribution.size()); - for (int i = 0; i < kNumClasses; ++i) { - EXPECT_NEAR(expected_weight[i], node_stats.weight_contribution[i], kDelta); - } - EXPECT_NEAR(9.841132, node_stats.gain, kDelta); -} - -TEST(NodeStatsTest, MulticlassDiagonalHessianTest) { - // Normal case. - { - LearnerConfig learner_config; - learner_config.set_multi_class_strategy(LearnerConfig::FULL_HESSIAN); - learner_config.mutable_regularization()->set_l2(0.3f); - - const int kNumClasses = 4; - const auto& g_shape = TensorShape({1, kNumClasses}); - Tensor g = AsTensor({0.5, 0.33, -9, 1}, g_shape); - Tensor h; - // Full hessian. - { - const auto& hessian_shape = TensorShape({1, kNumClasses, kNumClasses}); - // Construct full hessian. - h = AsTensor({3, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8, 0, 0, 0, 0, 9}, - hessian_shape); - } - NodeStats full_node_stats(learner_config, GradientStats(g, h)); - - // Diagonal only. - { - const auto& hessian_shape = TensorShape({1, kNumClasses}); - // Construct diagonal of hessian. - h = AsTensor({3, 4, 8, 9}, hessian_shape); - } - learner_config.set_multi_class_strategy(LearnerConfig::DIAGONAL_HESSIAN); - NodeStats diag_node_stats(learner_config, GradientStats(g, h)); - - // Full and diagonal hessian should return the same results. - EXPECT_EQ(full_node_stats.weight_contribution.size(), - diag_node_stats.weight_contribution.size()); - for (int i = 0; i < full_node_stats.weight_contribution.size(); ++i) { - EXPECT_FLOAT_EQ(full_node_stats.weight_contribution[i], - diag_node_stats.weight_contribution[i]); - } - EXPECT_EQ(full_node_stats.gain, diag_node_stats.gain); - } - // Zero entries in diagonal, no regularization - { - LearnerConfig learner_config; - learner_config.set_multi_class_strategy(LearnerConfig::FULL_HESSIAN); - - const int kNumClasses = 4; - const auto& g_shape = TensorShape({1, kNumClasses}); - Tensor g = AsTensor({0.5, 0.33, -9, 1}, g_shape); - Tensor h; - // Full hessian. - { - const auto& hessian_shape = TensorShape({1, kNumClasses, kNumClasses}); - // Construct full hessian. - h = AsTensor({3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0}, - hessian_shape); - } - NodeStats full_node_stats(learner_config, GradientStats(g, h)); - - // Diagonal only. - { - const auto& hessian_shape = TensorShape({1, kNumClasses}); - // Diagonal of hessian, two entries are 0 - h = AsTensor({3, 0, 8, 0}, hessian_shape); - } - learner_config.set_multi_class_strategy(LearnerConfig::DIAGONAL_HESSIAN); - NodeStats diag_node_stats(learner_config, GradientStats(g, h)); - - // Full and diagonal hessian should return the same results. - EXPECT_EQ(full_node_stats.weight_contribution.size(), - diag_node_stats.weight_contribution.size()); - for (int i = 0; i < full_node_stats.weight_contribution.size(); ++i) { - EXPECT_FLOAT_EQ(full_node_stats.weight_contribution[i], - diag_node_stats.weight_contribution[i]); - } - EXPECT_EQ(full_node_stats.gain, diag_node_stats.gain); - } -} - -} // namespace -} // namespace stochastic -} // namespace learner -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/learner/common/stats/split-stats.h b/tensorflow/contrib/boosted_trees/lib/learner/common/stats/split-stats.h deleted file mode 100644 index 81ee2774bda..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/learner/common/stats/split-stats.h +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_STATS_SPLIT_STATS_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_STATS_SPLIT_STATS_H_ - -#include - -#include "tensorflow/contrib/boosted_trees/lib/learner/common/stats/node-stats.h" - -namespace tensorflow { -namespace boosted_trees { -namespace learner { -namespace stochastic { - -// FeatureSplitCandidate holds the split candidate node along with the stats. -struct SplitStats { - // Initialize with 0 stats. - explicit SplitStats(const int output_length) - : root_node_stats(output_length), - left_node_stats(output_length), - right_node_stats(output_length), - gain(0) {} - - // Feature unary split candidate, we don't apply tree complexity - // regularization as no new nodes are being added with this candidate. - SplitStats(const LearnerConfig& learner_config, const NodeStats& root_stats) - : root_node_stats(root_stats), - left_node_stats(root_stats.weight_contribution.size()), - right_node_stats(root_stats.weight_contribution.size()), - gain(0) {} - - // Feature binary split candidate, we apply tree complexity regularization - // over the split gain to trade-off adding new nodes with loss reduction. - SplitStats(const LearnerConfig& learner_config, const NodeStats& root_stats, - const NodeStats& left_stats, const NodeStats& right_stats) - : root_node_stats(root_stats), - left_node_stats(left_stats), - right_node_stats(right_stats), - gain(left_stats.gain + right_stats.gain - root_stats.gain - - learner_config.regularization().tree_complexity()) {} - - // Root Stats. - NodeStats root_node_stats; - - // Children stats. - NodeStats left_node_stats; - NodeStats right_node_stats; - - // Split gain. - float gain; - - string DebugString() const { - return "Root = " + root_node_stats.DebugString() + - "\nLeft = " + left_node_stats.DebugString() + - "\nRight = " + right_node_stats.DebugString() + - "\nGain = " + std::to_string(gain); - } -}; - -// Helper macro to check split stats approximate equality. -#define EXPECT_SPLIT_STATS_EQ(val1, val2) \ - EXPECT_NODE_STATS_EQ(val1.root_node_stats, val2.root_node_stats); \ - EXPECT_NODE_STATS_EQ(val1.left_node_stats, val2.left_node_stats); \ - EXPECT_NODE_STATS_EQ(val1.right_node_stats, val2.right_node_stats); \ - EXPECT_FLOAT_EQ(val1.gain, val2.gain); - -} // namespace stochastic -} // namespace learner -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_LEARNER_COMMON_STATS_SPLIT_STATS_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/models/multiple_additive_trees.cc b/tensorflow/contrib/boosted_trees/lib/models/multiple_additive_trees.cc deleted file mode 100644 index c9223afeab2..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/models/multiple_additive_trees.cc +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/models/multiple_additive_trees.h" -#include "tensorflow/contrib/boosted_trees/lib/trees/decision_tree.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/batch_features.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/parallel_for.h" - -namespace tensorflow { -namespace boosted_trees { -namespace models { - -void MultipleAdditiveTrees::Predict( - const boosted_trees::trees::DecisionTreeEnsembleConfig& config, - const std::vector& trees_to_include, - const boosted_trees::utils::BatchFeatures& features, - tensorflow::thread::ThreadPool* const worker_threads, - tensorflow::TTypes::Matrix output_predictions, - Tensor* const output_leaf_index) { - // Zero out predictions as the model is additive. - output_predictions.setZero(); - - // Get batch size. - const int64 batch_size = features.batch_size(); - if (batch_size <= 0) { - return; - } - - // Lambda for doing a block of work. - auto update_predictions = [&config, &features, &trees_to_include, - &output_predictions, - &output_leaf_index](int64 start, int64 end) { - auto examples_iterable = features.examples_iterable(start, end); - Tensor dummy_tensor(DT_INT32, TensorShape({1, 1})); - tensorflow::TTypes::Matrix output_leaf_index_mat = - output_leaf_index != nullptr ? output_leaf_index->matrix() - : dummy_tensor.matrix(); - for (const auto& example : examples_iterable) { - for (const int32 tree_idx : trees_to_include) { - const boosted_trees::trees::DecisionTreeConfig& tree = - config.trees(tree_idx); - const float tree_weight = config.tree_weights(tree_idx); - const int leaf_idx = trees::DecisionTree::Traverse(tree, 0, example); - QCHECK(leaf_idx >= 0) << "Invalid tree: " << tree.DebugString(); - // Checks if output leaf tree index is required. - if (output_leaf_index != nullptr) { - output_leaf_index_mat(example.example_idx, tree_idx) = leaf_idx; - } - const auto& leaf_node = tree.nodes(leaf_idx); - QCHECK(leaf_node.has_leaf()) - << "Invalid leaf node: " << leaf_node.DebugString(); - if (leaf_node.leaf().has_sparse_vector()) { - const auto& leaf = leaf_node.leaf().sparse_vector(); - QCHECK_EQ(leaf.index_size(), leaf.value_size()); - for (size_t logit_dim = 0; logit_dim < leaf.index_size(); - ++logit_dim) { - const float value = tree_weight * leaf.value(logit_dim); - output_predictions(example.example_idx, leaf.index(logit_dim)) += - value; - } - } else { - QCHECK(leaf_node.leaf().has_vector()) << "Unknown leaf type"; - const auto& leaf = leaf_node.leaf().vector(); - for (size_t i = 0; i < leaf.value_size(); ++i) { - const float value = tree_weight * leaf.value(i); - output_predictions(example.example_idx, i) += value; - } - } - } - } - }; - - // TODO(salehay): parallelize this for low latency in serving path where - // batch size tends to be small but ensemble size tends to be large. - boosted_trees::utils::ParallelFor(batch_size, worker_threads->NumThreads(), - worker_threads, update_predictions); -} - -} // namespace models -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/models/multiple_additive_trees.h b/tensorflow/contrib/boosted_trees/lib/models/multiple_additive_trees.h deleted file mode 100644 index 940531c4ba4..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/models/multiple_additive_trees.h +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_MODELS_MULTIPLE_ADDITIVE_TREES_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_MODELS_MULTIPLE_ADDITIVE_TREES_H_ - -#include - -#include "tensorflow/contrib/boosted_trees/lib/utils/batch_features.h" -#include "tensorflow/contrib/boosted_trees/proto/tree_config.pb.h" // NOLINT -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace boosted_trees { -namespace models { - -// Multiple additive trees prediction model. -// This class does not hold state and is thread safe. -class MultipleAdditiveTrees { - public: - // Predict runs tree ensemble on the given batch and updates - // output predictions accordingly, for the given list of trees. - // output_leaf_indices is a pointer to a 2 dimensional tensor. If it is not - // nullptr, this method fills output_leaf_indices with a per-tree leaf id - // where each of the instances from 'features' ended up in. Its shape is num - // examples X num of trees. - static void Predict( - const boosted_trees::trees::DecisionTreeEnsembleConfig& config, - const std::vector& trees_to_include, - const boosted_trees::utils::BatchFeatures& features, - tensorflow::thread::ThreadPool* const worker_threads, - tensorflow::TTypes::Matrix output_predictions, - Tensor* const output_leaf_index); -}; - -} // namespace models -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_MODELS_MULTIPLE_ADDITIVE_TREES_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/models/multiple_additive_trees_test.cc b/tensorflow/contrib/boosted_trees/lib/models/multiple_additive_trees_test.cc deleted file mode 100644 index 462a9ac86fe..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/models/multiple_additive_trees_test.cc +++ /dev/null @@ -1,298 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/models/multiple_additive_trees.h" - -#include "tensorflow/contrib/boosted_trees/lib/testutil/batch_features_testutil.h" -#include "tensorflow/contrib/boosted_trees/lib/testutil/random_tree_gen.h" -#include "tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/lib/random/philox_random.h" -#include "tensorflow/core/lib/random/simple_philox.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/platform/test_benchmark.h" - -namespace tensorflow { -using boosted_trees::trees::DecisionTreeEnsembleConfig; -using test::AsTensor; - -namespace boosted_trees { -namespace models { -namespace { - -const int32 kNumThreadsMultiThreaded = 6; -const int32 kNumThreadsSingleThreaded = 1; - -class MultipleAdditiveTreesTest : public ::testing::Test { - protected: - MultipleAdditiveTreesTest() : batch_features_(2) { - // Create a batch of two examples having one dense feature each. - // The shape of the dense matrix is therefore 2x1 as in one row per example - // and one column per feature per example. - auto dense_matrix = test::AsTensor({7.0f, -2.0f}, {2, 1}); - TF_EXPECT_OK( - batch_features_.Initialize({dense_matrix}, {}, {}, {}, {}, {}, {})); - } - - boosted_trees::utils::BatchFeatures batch_features_; -}; - -TEST_F(MultipleAdditiveTreesTest, Empty) { - // Create empty tree ensemble. - DecisionTreeEnsembleConfig tree_ensemble_config; - auto output_tensor = AsTensor({9.0f, 23.0f}, {2, 1}); - auto output_matrix = output_tensor.matrix(); - - // Predict for both instances. - tensorflow::thread::ThreadPool threads(tensorflow::Env::Default(), "test", - kNumThreadsSingleThreaded); - MultipleAdditiveTrees::Predict(tree_ensemble_config, {}, batch_features_, - &threads, output_matrix, - /*output_leaf_index=*/nullptr); - EXPECT_EQ(0, output_matrix(0, 0)); - EXPECT_EQ(0, output_matrix(1, 0)); -} - -TEST_F(MultipleAdditiveTreesTest, SingleClass) { - // Add one bias and one stump to ensemble for a single class. - DecisionTreeEnsembleConfig tree_ensemble_config; - auto* tree1 = tree_ensemble_config.add_trees(); - auto* bias_leaf = tree1->add_nodes()->mutable_leaf()->mutable_sparse_vector(); - bias_leaf->add_index(0); - bias_leaf->add_value(-0.4f); - auto* tree2 = tree_ensemble_config.add_trees(); - auto* dense_split = tree2->add_nodes()->mutable_dense_float_binary_split(); - dense_split->set_feature_column(0); - dense_split->set_threshold(5.0f); - dense_split->set_left_id(1); - dense_split->set_right_id(2); - auto* leaf1 = tree2->add_nodes()->mutable_leaf()->mutable_sparse_vector(); - leaf1->add_index(0); - leaf1->add_value(0.9f); - auto* leaf2 = tree2->add_nodes()->mutable_leaf()->mutable_sparse_vector(); - leaf2->add_index(0); - leaf2->add_value(0.2f); - - tree_ensemble_config.add_tree_weights(1.0); - tree_ensemble_config.add_tree_weights(1.0); - - auto output_tensor = AsTensor({0.0f, 0.0f}, {2, 1}); - auto output_matrix = output_tensor.matrix(); - - tensorflow::thread::ThreadPool threads(tensorflow::Env::Default(), "test", - kNumThreadsSingleThreaded); - - // Normal case. - { - MultipleAdditiveTrees::Predict(tree_ensemble_config, {0, 1}, - batch_features_, &threads, output_matrix, - /*output_leaf_index=*/nullptr); - EXPECT_FLOAT_EQ(-0.2f, output_matrix(0, 0)); // -0.4 (bias) + 0.2 (leaf 2). - EXPECT_FLOAT_EQ(0.5f, output_matrix(1, 0)); // -0.4 (bias) + 0.9 (leaf 1). - } - // Normal case with leaf node. - { - // Initialize output leaf index tensor, since leaf index is positive in this - // case, initialize with the value of -1. Since there are 2 examples and - // there are 2 trees, initialize leaf output index by 2 * 2. - Tensor output_leaf_index_tensor(DT_INT32, TensorShape({2, 2})); - MultipleAdditiveTrees::Predict(tree_ensemble_config, {0, 1}, - batch_features_, &threads, output_matrix, - &output_leaf_index_tensor); - EXPECT_FLOAT_EQ(-0.2f, output_matrix(0, 0)); // -0.4 (bias) + 0.2 (leaf 2). - EXPECT_FLOAT_EQ(0.5f, output_matrix(1, 0)); // -0.4 (bias) + 0.9 (leaf 1). - EXPECT_FLOAT_EQ(0, output_leaf_index_tensor.matrix()( - 0, 0)); // 1st leaf for the first example - EXPECT_FLOAT_EQ(0, output_leaf_index_tensor.matrix()( - 1, 0)); // 1st leaf for the second example - EXPECT_FLOAT_EQ(2, output_leaf_index_tensor.matrix()( - 0, 1)); // 2nd leaf for the first example - EXPECT_FLOAT_EQ(1, output_leaf_index_tensor.matrix()( - 1, 1)); // 2nd leaf for the second example - } - // Weighted case - { - DecisionTreeEnsembleConfig weighted = tree_ensemble_config; - weighted.set_tree_weights(0, 6.0); - weighted.set_tree_weights(1, 3.2); - MultipleAdditiveTrees::Predict(weighted, {0, 1}, batch_features_, &threads, - output_matrix, nullptr); - // -0.4 (bias) + 0.2 (leaf 2). - EXPECT_FLOAT_EQ(-0.4f * 6 + 0.2 * 3.2, output_matrix(0, 0)); - // -0.4 (bias) + 0.9 (leaf 1). - EXPECT_FLOAT_EQ(-0.4f * 6 + 0.9 * 3.2, output_matrix(1, 0)); - } - // Drop first tree. - { - MultipleAdditiveTrees::Predict(tree_ensemble_config, {1}, batch_features_, - &threads, output_matrix, nullptr); - EXPECT_FLOAT_EQ(0.2f, output_matrix(0, 0)); // 0.2 (leaf 2). - EXPECT_FLOAT_EQ(0.9f, output_matrix(1, 0)); // 0.9 (leaf 1). - } - // Drop second tree. - { - MultipleAdditiveTrees::Predict(tree_ensemble_config, {0}, batch_features_, - &threads, output_matrix, nullptr); - EXPECT_FLOAT_EQ(-0.4f, output_matrix(0, 0)); // -0.4 (bias). - EXPECT_FLOAT_EQ(-0.4f, output_matrix(1, 0)); // -0.4 (bias). - } - // Drop all trees. - { - MultipleAdditiveTrees::Predict(tree_ensemble_config, {}, batch_features_, - &threads, output_matrix, nullptr); - EXPECT_FLOAT_EQ(0.0, output_matrix(0, 0)); - EXPECT_FLOAT_EQ(0.0, output_matrix(1, 0)); - } -} - -TEST_F(MultipleAdditiveTreesTest, MultiClass) { - // Add one bias and one stump to ensemble for two classes. - DecisionTreeEnsembleConfig tree_ensemble_config; - auto* tree1 = tree_ensemble_config.add_trees(); - auto* bias_leaf = tree1->add_nodes()->mutable_leaf()->mutable_sparse_vector(); - bias_leaf->add_index(0); - bias_leaf->add_value(-0.4f); - bias_leaf->add_index(1); - bias_leaf->add_value(-0.7f); - auto* tree2 = tree_ensemble_config.add_trees(); - auto* dense_split = tree2->add_nodes()->mutable_dense_float_binary_split(); - dense_split->set_feature_column(0); - dense_split->set_threshold(5.0f); - dense_split->set_left_id(1); - dense_split->set_right_id(2); - auto* leaf1 = tree2->add_nodes()->mutable_leaf()->mutable_sparse_vector(); - leaf1->add_index(0); - leaf1->add_value(0.9f); - auto* leaf2 = tree2->add_nodes()->mutable_leaf()->mutable_sparse_vector(); - leaf2->add_index(1); - leaf2->add_value(0.2f); - - tree_ensemble_config.add_tree_weights(1.0); - tree_ensemble_config.add_tree_weights(1.0); - - // Predict for both instances. - tensorflow::thread::ThreadPool threads(tensorflow::Env::Default(), "test", - kNumThreadsSingleThreaded); - auto output_tensor = AsTensor({0.0f, 0.0f, 0.0f, 0.0f}, {2, 2}); - auto output_matrix = output_tensor.matrix(); - - // Normal case. - { - MultipleAdditiveTrees::Predict(tree_ensemble_config, {0, 1}, - batch_features_, &threads, output_matrix, - nullptr); - EXPECT_FLOAT_EQ(-0.4f, output_matrix(0, 0)); // -0.4 (bias) - EXPECT_FLOAT_EQ(-0.5f, output_matrix(0, 1)); // -0.7 (bias) + 0.2 (leaf 2) - EXPECT_FLOAT_EQ(0.5f, output_matrix(1, 0)); // -0.4 (bias) + 0.9 (leaf 1) - EXPECT_FLOAT_EQ(-0.7f, output_matrix(1, 1)); // -0.7 (bias) - } - // Weighted case. - { - DecisionTreeEnsembleConfig weighted = tree_ensemble_config; - weighted.set_tree_weights(0, 6.0); - weighted.set_tree_weights(1, 3.2); - MultipleAdditiveTrees::Predict(weighted, {0, 1}, batch_features_, &threads, - output_matrix, nullptr); - // bias - EXPECT_FLOAT_EQ(-0.4f * 6, output_matrix(0, 0)); - // bias + leaf 2 - EXPECT_FLOAT_EQ(-0.7f * 6 + 0.2f * 3.2, output_matrix(0, 1)); - // bias + leaf 2 - EXPECT_FLOAT_EQ(-0.4f * 6 + 0.9f * 3.2f, output_matrix(1, 0)); - // bias - EXPECT_FLOAT_EQ(-0.7f * 6, output_matrix(1, 1)); - } - // Dropout first tree. - { - MultipleAdditiveTrees::Predict(tree_ensemble_config, {1}, batch_features_, - &threads, output_matrix, nullptr); - EXPECT_FLOAT_EQ(0.0, output_matrix(0, 0)); - EXPECT_FLOAT_EQ(0.2f, output_matrix(0, 1)); // 0.2 (leaf 2) - EXPECT_FLOAT_EQ(0.9f, output_matrix(1, 0)); // 0.9 (leaf 2) - EXPECT_FLOAT_EQ(0.0f, output_matrix(1, 1)); - } - // Dropout second tree. - { - MultipleAdditiveTrees::Predict(tree_ensemble_config, {0}, batch_features_, - &threads, output_matrix, nullptr); - EXPECT_FLOAT_EQ(-0.4f, output_matrix(0, 0)); // -0.4 (bias) - EXPECT_FLOAT_EQ(-0.7f, output_matrix(0, 1)); // -0.7 (bias) - EXPECT_FLOAT_EQ(-0.4f, output_matrix(1, 0)); // -0.4 (bias) - EXPECT_FLOAT_EQ(-0.7f, output_matrix(1, 1)); // -0.7 (bias) - } - // Drop both trees. - { - MultipleAdditiveTrees::Predict(tree_ensemble_config, {}, batch_features_, - &threads, output_matrix, nullptr); - EXPECT_FLOAT_EQ(0.0f, output_matrix(0, 0)); - EXPECT_FLOAT_EQ(0.0f, output_matrix(0, 1)); - EXPECT_FLOAT_EQ(0.0f, output_matrix(1, 0)); - EXPECT_FLOAT_EQ(0.0f, output_matrix(1, 1)); - } -} - -TEST_F(MultipleAdditiveTreesTest, DenseLeaves) { - DecisionTreeEnsembleConfig tree_ensemble_config; - auto* tree1 = tree_ensemble_config.add_trees(); - auto* bias_leaf = tree1->add_nodes()->mutable_leaf()->mutable_vector(); - bias_leaf->add_value(-0.4f); - bias_leaf->add_value(-0.7f); - bias_leaf->add_value(3.0f); - auto* tree2 = tree_ensemble_config.add_trees(); - auto* dense_split = tree2->add_nodes()->mutable_dense_float_binary_split(); - dense_split->set_feature_column(0); - dense_split->set_threshold(5.0f); - dense_split->set_left_id(1); - dense_split->set_right_id(2); - auto* leaf1 = tree2->add_nodes()->mutable_leaf()->mutable_vector(); - leaf1->add_value(0.9f); - leaf1->add_value(0.8f); - leaf1->add_value(0.7f); - auto* leaf2 = tree2->add_nodes()->mutable_leaf()->mutable_vector(); - leaf2->add_value(0.2f); - leaf2->add_value(0.3f); - leaf2->add_value(0.4f); - - tree_ensemble_config.add_tree_weights(1.0); - tree_ensemble_config.add_tree_weights(1.0); - - // Predict for both instances. - tensorflow::thread::ThreadPool threads(tensorflow::Env::Default(), "test", - kNumThreadsSingleThreaded); - auto output_tensor = - AsTensor({0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}, {2, 3}); - auto output_matrix = output_tensor.matrix(); - - // Normal case. - { - MultipleAdditiveTrees::Predict(tree_ensemble_config, {0, 1}, - batch_features_, &threads, output_matrix, - nullptr); - EXPECT_FLOAT_EQ(-0.2f, output_matrix(0, 0)); // -0.4 (tree1) + 0.2 (leaf 2) - EXPECT_FLOAT_EQ(-0.4f, output_matrix(0, 1)); // -0.7 (tree1) + 0.3 (leaf 2) - EXPECT_FLOAT_EQ(3.4f, output_matrix(0, 2)); // 3.0 -(tree1) + 0.4 (leaf 2) - EXPECT_FLOAT_EQ(0.5f, output_matrix(1, 0)); // -0.4 (tree1) + 0.9 (leaf 1) - EXPECT_FLOAT_EQ(0.1f, output_matrix(1, 1)); // -0.7 (tree1) + 0.8 (leaf 1) - EXPECT_FLOAT_EQ(3.7f, output_matrix(1, 2)); // 3.0 (tree1) + 0.7 (leaf 1) - } -} - -} // namespace -} // namespace models -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_buffer.h b/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_buffer.h deleted file mode 100644 index 804b218f1c0..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_buffer.h +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_QUANTILES_WEIGHTED_QUANTILES_BUFFER_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_QUANTILES_WEIGHTED_QUANTILES_BUFFER_H_ - -#include -#include -#include - -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace boosted_trees { -namespace quantiles { - -// Buffering container ideally suited for scenarios where we need -// to sort and dedupe/compact fixed chunks of a stream of weighted elements. -template > -class WeightedQuantilesBuffer { - public: - struct BufferEntry { - BufferEntry(ValueType v, WeightType w) - : value(std::move(v)), weight(std::move(w)) {} - BufferEntry() : value(), weight(0) {} - - bool operator<(const BufferEntry& other) const { - return kCompFn(value, other.value); - } - bool operator==(const BufferEntry& other) const { - return value == other.value && weight == other.weight; - } - friend std::ostream& operator<<(std::ostream& strm, - const BufferEntry& entry) { - return strm << "{" << entry.value << ", " << entry.weight << "}"; - } - ValueType value; - WeightType weight; - }; - - explicit WeightedQuantilesBuffer(int64 block_size, int64 max_elements) - : max_size_(std::min(block_size << 1, max_elements)) { - QCHECK(max_size_ > 0) << "Invalid buffer specification: (" << block_size - << ", " << max_elements << ")"; - vec_.reserve(max_size_); - } - - // Disallow copying as it's semantically non-sensical in the Squawd algorithm - // but enable move semantics. - WeightedQuantilesBuffer(const WeightedQuantilesBuffer& other) = delete; - WeightedQuantilesBuffer& operator=(const WeightedQuantilesBuffer&) = delete; - WeightedQuantilesBuffer(WeightedQuantilesBuffer&& other) = default; - WeightedQuantilesBuffer& operator=(WeightedQuantilesBuffer&& other) = default; - - // Push entry to buffer and maintain a compact representation within - // pre-defined size limit. - void PushEntry(ValueType value, WeightType weight) { - // Callers are expected to act on a full compacted buffer after the - // PushEntry call returns. - QCHECK(!IsFull()) << "Buffer already full: " << max_size_; - - // Ignore zero and negative weight entries. - if (weight <= 0) { - return; - } - - // Push back the entry to the buffer. - vec_.push_back(BufferEntry(std::move(value), std::move(weight))); - } - - // Returns a sorted vector view of the base buffer and clears the buffer. - // Callers should minimize how often this is called, ideally only right after - // the buffer becomes full. - std::vector GenerateEntryList() { - std::vector ret; - if (vec_.size() == 0) { - return ret; - } - ret.swap(vec_); - vec_.reserve(max_size_); - std::sort(ret.begin(), ret.end()); - size_t num_entries = 0; - for (size_t i = 1; i < ret.size(); ++i) { - if (ret[i].value != ret[i - 1].value) { - BufferEntry tmp = ret[i]; - ++num_entries; - ret[num_entries] = tmp; - } else { - ret[num_entries].weight += ret[i].weight; - } - } - ret.resize(num_entries + 1); - return ret; - } - - int64 Size() const { return vec_.size(); } - bool IsFull() const { return vec_.size() >= max_size_; } - void Clear() { vec_.clear(); } - - private: - using BufferVector = typename std::vector; - - // Comparison function. - static constexpr decltype(CompareFn()) kCompFn = CompareFn(); - - // Base buffer. - size_t max_size_; - BufferVector vec_; -}; - -template -constexpr decltype(CompareFn()) - WeightedQuantilesBuffer::kCompFn; - -} // namespace quantiles -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_QUANTILES_WEIGHTED_QUANTILES_BUFFER_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_buffer_test.cc b/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_buffer_test.cc deleted file mode 100644 index 8e403186651..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_buffer_test.cc +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_buffer.h" - -#include "tensorflow/core/lib/random/philox_random.h" -#include "tensorflow/core/lib/random/simple_philox.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/platform/test_benchmark.h" - -namespace tensorflow { -namespace { - -using Buffer = - boosted_trees::quantiles::WeightedQuantilesBuffer; -using BufferEntry = - boosted_trees::quantiles::WeightedQuantilesBuffer::BufferEntry; - -class WeightedQuantilesBufferTest : public ::testing::Test {}; - -TEST_F(WeightedQuantilesBufferTest, Invalid) { - EXPECT_DEATH( - ({ - boosted_trees::quantiles::WeightedQuantilesBuffer - buffer(2, 0); - }), - "Invalid buffer specification"); - EXPECT_DEATH( - ({ - boosted_trees::quantiles::WeightedQuantilesBuffer - buffer(0, 2); - }), - "Invalid buffer specification"); -} - -TEST_F(WeightedQuantilesBufferTest, PushEntryNotFull) { - Buffer buffer(20, 100); - buffer.PushEntry(5, 9); - buffer.PushEntry(2, 3); - buffer.PushEntry(-1, 7); - buffer.PushEntry(3, 0); // This entry will be ignored. - - EXPECT_FALSE(buffer.IsFull()); - EXPECT_EQ(buffer.Size(), 3); -} - -TEST_F(WeightedQuantilesBufferTest, PushEntryFull) { - // buffer capacity is 4. - Buffer buffer(2, 100); - buffer.PushEntry(5, 9); - buffer.PushEntry(2, 3); - buffer.PushEntry(-1, 7); - buffer.PushEntry(2, 1); - - std::vector expected; - expected.emplace_back(-1, 7); - expected.emplace_back(2, 4); - expected.emplace_back(5, 9); - - // At this point, we have pushed 4 entries and we expect the buffer to be - // full. - EXPECT_TRUE(buffer.IsFull()); - EXPECT_EQ(buffer.GenerateEntryList(), expected); - EXPECT_FALSE(buffer.IsFull()); -} - -TEST_F(WeightedQuantilesBufferTest, PushEntryFullDeath) { - // buffer capacity is 4. - Buffer buffer(2, 100); - buffer.PushEntry(5, 9); - buffer.PushEntry(2, 3); - buffer.PushEntry(-1, 7); - buffer.PushEntry(2, 1); - - std::vector expected; - expected.emplace_back(-1, 7); - expected.emplace_back(2, 4); - expected.emplace_back(5, 9); - - // At this point, we have pushed 4 entries and we expect the buffer to be - // full. - EXPECT_TRUE(buffer.IsFull()); - // Can't push any more entries before clearing. - EXPECT_DEATH(({ buffer.PushEntry(6, 6); }), "Buffer already full"); -} - -} // namespace -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_stream.h b/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_stream.h deleted file mode 100644 index f19e5116f58..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_stream.h +++ /dev/null @@ -1,330 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_QUANTILES_WEIGHTED_QUANTILES_STREAM_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_QUANTILES_WEIGHTED_QUANTILES_STREAM_H_ - -#include -#include -#include - -#include "tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_buffer.h" -#include "tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace boosted_trees { -namespace quantiles { - -// Class to compute approximate quantiles with error bound guarantees for -// weighted data sets. -// This implementation is an adaptation of techniques from the following papers: -// * (2001) Space-efficient online computation of quantile summaries. -// * (2004) Power-conserving computation of order-statistics over -// sensor networks. -// * (2007) A fast algorithm for approximate quantiles in high speed -// data streams. -// * (2016) XGBoost: A Scalable Tree Boosting System. -// -// The key ideas at play are the following: -// - Maintain an in-memory multi-level quantile summary in a way to guarantee -// a maximum approximation error of eps * W per bucket where W is the total -// weight across all points in the input dataset. -// - Two base operations are defined: MERGE and COMPRESS. MERGE combines two -// summaries guaranteeing a epsNew = max(eps1, eps2). COMPRESS compresses -// a summary to b + 1 elements guaranteeing epsNew = epsOld + 1/b. -// - b * sizeof(summary entry) must ideally be small enough to fit in an -// average CPU L2 cache. -// - To distribute this algorithm with maintaining error bounds, we need -// the worker-computed summaries to have no more than eps / h error -// where h is the height of the distributed computation graph which -// is 2 for an MR with no combiner. -// -// We mainly want to max out IO bw by ensuring we're not compute-bound and -// using a reasonable amount of RAM. -// -// Complexity: -// Compute: O(n * log(1/eps * log(eps * n))). -// Memory: O(1/eps * log^2(eps * n)) <- for one worker streaming through the -// entire dataset. -// An epsilon value of zero would make the algorithm extremely inefficent and -// therefore, is disallowed. -template > -class WeightedQuantilesStream { - public: - using Buffer = WeightedQuantilesBuffer; - using BufferEntry = typename Buffer::BufferEntry; - using Summary = WeightedQuantilesSummary; - using SummaryEntry = typename Summary::SummaryEntry; - - explicit WeightedQuantilesStream(double eps, int64 max_elements) - : eps_(eps), buffer_(1LL, 2LL), finalized_(false) { - // See the class documentation. An epsilon value of zero could cause - // perfoamance issues. - QCHECK(eps > 0) << "An epsilon value of zero is not allowed."; - std::tie(max_levels_, block_size_) = GetQuantileSpecs(eps, max_elements); - buffer_ = Buffer(block_size_, max_elements); - summary_levels_.reserve(max_levels_); - } - - // Disallow copy and assign but enable move semantics for the stream. - WeightedQuantilesStream(const WeightedQuantilesStream& other) = delete; - WeightedQuantilesStream& operator=(const WeightedQuantilesStream&) = delete; - WeightedQuantilesStream(WeightedQuantilesStream&& other) = default; - WeightedQuantilesStream& operator=(WeightedQuantilesStream&& other) = default; - - // Pushes one entry while maintaining approximation error invariants. - void PushEntry(const ValueType& value, const WeightType& weight) { - // Validate state. - QCHECK(!finalized_) << "Finalize() already called."; - - // Push element to base buffer. - buffer_.PushEntry(value, weight); - - // When compacted buffer is full we need to compress - // and push weighted quantile summary up the level chain. - if (buffer_.IsFull()) { - PushBuffer(buffer_); - } - } - - // Pushes full buffer while maintaining approximation error invariants. - void PushBuffer(Buffer& buffer) { - // Validate state. - QCHECK(!finalized_) << "Finalize() already called."; - - // Create local compressed summary and propagate. - local_summary_.BuildFromBufferEntries(buffer.GenerateEntryList()); - local_summary_.Compress(block_size_, eps_); - PropagateLocalSummary(); - } - - // Pushes full summary while maintaining approximation error invariants. - void PushSummary(const std::vector& summary) { - // Validate state. - QCHECK(!finalized_) << "Finalize() already called."; - - // Create local compressed summary and propagate. - local_summary_.BuildFromSummaryEntries(summary); - local_summary_.Compress(block_size_, eps_); - PropagateLocalSummary(); - } - - // Flushes approximator and finalizes state. - void Finalize() { - // Validate state. - QCHECK(!finalized_) << "Finalize() may only be called once."; - - // Flush any remaining buffer elements. - PushBuffer(buffer_); - - // Create final merged summary. - local_summary_.Clear(); - for (auto& summary : summary_levels_) { - local_summary_.Merge(summary); - summary.Clear(); - } - summary_levels_.clear(); - summary_levels_.shrink_to_fit(); - finalized_ = true; - } - - // Generates requested number of quantiles after finalizing stream. - // The returned quantiles can be queried using std::lower_bound to get - // the bucket for a given value. - std::vector GenerateQuantiles(int64 num_quantiles) const { - // Validate state. - QCHECK(finalized_) - << "Finalize() must be called before generating quantiles."; - return local_summary_.GenerateQuantiles(num_quantiles); - } - - // Generates requested number of boundaries after finalizing stream. - // The returned boundaries can be queried using std::lower_bound to get - // the bucket for a given value. - // The boundaries, while still guaranteeing approximation bounds, don't - // necessarily represent the actual quantiles of the distribution. - // Boundaries are preferable over quantiles when the caller is less - // interested in the actual quantiles distribution and more interested in - // getting a representative sample of boundary values. - std::vector GenerateBoundaries(int64 num_boundaries) const { - // Validate state. - QCHECK(finalized_) - << "Finalize() must be called before generating boundaries."; - return local_summary_.GenerateBoundaries(num_boundaries); - } - - // Calculates approximation error for the specified level. - // If the passed level is negative, the approximation error for the entire - // summary is returned. Note that after Finalize is called, only the overall - // error is available. - WeightType ApproximationError(int64 level = -1) const { - if (finalized_) { - QCHECK(level <= 0) << "Only overall error is available after Finalize()"; - return local_summary_.ApproximationError(); - } - - if (summary_levels_.empty()) { - // No error even if base buffer isn't empty. - return 0; - } - - // If level is negative, we get the approximation error - // for the top-most level which is the max approximation error - // in all summaries by construction. - if (level < 0) { - level = summary_levels_.size() - 1; - } - QCHECK(level < summary_levels_.size()) << "Invalid level."; - return summary_levels_[level].ApproximationError(); - } - - size_t MaxDepth() const { return summary_levels_.size(); } - - // Generates requested number of quantiles after finalizing stream. - const Summary& GetFinalSummary() const { - // Validate state. - QCHECK(finalized_) - << "Finalize() must be called before requesting final summary."; - return local_summary_; - } - - // Helper method which, given the desired approximation error - // and an upper bound on the number of elements, computes the optimal - // number of levels and block size and returns them in the tuple. - static std::tuple GetQuantileSpecs(double eps, - int64 max_elements); - - // Serializes the internal state of the stream. - std::vector

SerializeInternalSummaries() const { - // The buffer should be empty for serialize to work. - QCHECK_EQ(buffer_.Size(), 0); - std::vector result; - result.reserve(summary_levels_.size() + 1); - for (const Summary& summary : summary_levels_) { - result.push_back(summary); - } - result.push_back(local_summary_); - return result; - } - - // Resets the state of the stream with a serialized state. - void DeserializeInternalSummaries(const std::vector& summaries) { - // Clear the state before deserializing. - buffer_.Clear(); - summary_levels_.clear(); - local_summary_.Clear(); - QCHECK_GT(max_levels_, summaries.size() - 1); - for (int i = 0; i < summaries.size() - 1; ++i) { - summary_levels_.push_back(summaries[i]); - } - local_summary_ = summaries[summaries.size() - 1]; - } - - private: - // Propagates local summary through summary levels while maintaining - // approximation error invariants. - void PropagateLocalSummary() { - // Validate state. - QCHECK(!finalized_) << "Finalize() already called."; - - // No-op if there's nothing to add. - if (local_summary_.Size() <= 0) { - return; - } - - // Propagate summary through levels. - size_t level = 0; - for (bool settled = false; !settled; ++level) { - // Ensure we have enough depth. - if (summary_levels_.size() <= level) { - summary_levels_.emplace_back(); - } - - // Merge summaries. - Summary& current_summary = summary_levels_[level]; - local_summary_.Merge(current_summary); - - // Check if we need to compress and propagate summary higher. - if (current_summary.Size() == 0 || - local_summary_.Size() <= block_size_ + 1) { - current_summary = std::move(local_summary_); - settled = true; - } else { - // Compress, empty current level and propagate. - local_summary_.Compress(block_size_, eps_); - current_summary.Clear(); - } - } - } - - // Desired approximation precision. - double eps_; - // Maximum number of levels. - int64 max_levels_; - // Max block size per level. - int64 block_size_; - // Base buffer. - Buffer buffer_; - // Local summary used to minimize memory allocation and cache misses. - // After the stream is finalized, this summary holds the final quantile - // estimates. - Summary local_summary_; - // Summary levels; - std::vector summary_levels_; - // Flag indicating whether the stream is finalized. - bool finalized_; -}; - -template -inline std::tuple -WeightedQuantilesStream::GetQuantileSpecs( - double eps, int64 max_elements) { - int64 max_level = 1LL; - int64 block_size = 2LL; - QCHECK(eps >= 0 && eps < 1); - QCHECK_GT(max_elements, 0); - - if (eps <= std::numeric_limits::epsilon()) { - // Exact quantile computation at the expense of RAM. - max_level = 1; - block_size = std::max(max_elements, int64{2}); - } else { - // The bottom-most level will become full at most - // (max_elements / block_size) times, the level above will become full - // (max_elements / 2 * block_size) times and generally level l becomes - // full (max_elements / 2^l * block_size) times until the last - // level max_level becomes full at most once meaning when the inequality - // (2^max_level * block_size >= max_elements) is satisfied. - // In what follows, we jointly solve for max_level and block_size by - // gradually increasing the level until the inequality above is satisfied. - // We could alternatively set max_level = ceil(log2(eps * max_elements)); - // and block_size = ceil(max_level / eps) + 1 but that tends to give more - // pessimistic bounds and wastes RAM needlessly. - for (max_level = 1, block_size = 2; - (1LL << max_level) * block_size < max_elements; ++max_level) { - // Update upper bound on block size at current level, we always - // increase the estimate by 2 to hold the min/max elements seen so far. - block_size = static_cast(ceil(max_level / eps)) + 1; - } - } - return std::make_tuple(max_level, std::max(block_size, int64{2})); -} - -} // namespace quantiles -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_QUANTILES_WEIGHTED_QUANTILES_STREAM_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_stream_test.cc b/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_stream_test.cc deleted file mode 100644 index 67ac9bf387a..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_stream_test.cc +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_stream.h" - -#include "tensorflow/core/lib/random/philox_random.h" -#include "tensorflow/core/lib/random/simple_philox.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/platform/test_benchmark.h" - -namespace tensorflow { -namespace { -using Tuple = std::tuple; - -using Summary = - boosted_trees::quantiles::WeightedQuantilesSummary; -using SummaryEntry = - boosted_trees::quantiles::WeightedQuantilesSummary::SummaryEntry; -using Stream = - boosted_trees::quantiles::WeightedQuantilesStream; - -TEST(GetQuantileSpecs, InvalidEps) { - EXPECT_DEATH({ Stream::GetQuantileSpecs(-0.01, 0L); }, "eps >= 0"); - EXPECT_DEATH({ Stream::GetQuantileSpecs(1.01, 0L); }, "eps < 1"); -} - -TEST(GetQuantileSpecs, ZeroEps) { - EXPECT_DEATH({ Stream::GetQuantileSpecs(0.0, 0L); }, "max_elements > 0"); - EXPECT_EQ(Stream::GetQuantileSpecs(0.0, 1LL), Tuple(1LL, 2LL)); - EXPECT_EQ(Stream::GetQuantileSpecs(0.0, 20LL), Tuple(1LL, 20LL)); -} - -TEST(GetQuantileSpecs, NonZeroEps) { - EXPECT_DEATH({ Stream::GetQuantileSpecs(0.01, 0L); }, "max_elements > 0"); - EXPECT_EQ(Stream::GetQuantileSpecs(0.1, 320LL), Tuple(4LL, 31LL)); - EXPECT_EQ(Stream::GetQuantileSpecs(0.01, 25600LL), Tuple(6LL, 501LL)); - EXPECT_EQ(Stream::GetQuantileSpecs(0.01, 104857600LL), Tuple(17LL, 1601LL)); - EXPECT_EQ(Stream::GetQuantileSpecs(0.1, 104857600LL), Tuple(20LL, 191LL)); - EXPECT_EQ(Stream::GetQuantileSpecs(0.01, 1LL << 40), Tuple(29LL, 2801LL)); - EXPECT_EQ(Stream::GetQuantileSpecs(0.001, 1LL << 40), Tuple(26LL, 25001LL)); -} - -class WeightedQuantilesStreamTest : public ::testing::Test {}; - -// Stream generators. -void GenerateFixedUniformSummary(int32 worker_id, int64 max_elements, - double *total_weight, Stream *stream) { - for (int64 i = 0; i < max_elements; ++i) { - const double x = static_cast(i) / max_elements; - stream->PushEntry(x, 1.0); - ++(*total_weight); - } - stream->Finalize(); -} - -void GenerateFixedNonUniformSummary(int32 worker_id, int64 max_elements, - double *total_weight, Stream *stream) { - for (int64 i = 0; i < max_elements; ++i) { - const double x = static_cast(i) / max_elements; - stream->PushEntry(x, x); - (*total_weight) += x; - } - stream->Finalize(); -} - -void GenerateRandUniformFixedWeightsSummary(int32 worker_id, int64 max_elements, - double *total_weight, - Stream *stream) { - // Simulate uniform distribution stream. - random::PhiloxRandom philox(13 + worker_id); - random::SimplePhilox rand(&philox); - for (int64 i = 0; i < max_elements; ++i) { - const double x = rand.RandDouble(); - stream->PushEntry(x, 1); - ++(*total_weight); - } - stream->Finalize(); -} - -void GenerateRandUniformRandWeightsSummary(int32 worker_id, int64 max_elements, - double *total_weight, - Stream *stream) { - // Simulate uniform distribution stream. - random::PhiloxRandom philox(13 + worker_id); - random::SimplePhilox rand(&philox); - for (int64 i = 0; i < max_elements; ++i) { - const double x = rand.RandDouble(); - const double w = rand.RandDouble(); - stream->PushEntry(x, w); - (*total_weight) += w; - } - stream->Finalize(); -} - -// Single worker tests. -void TestSingleWorkerStreams( - double eps, int64 max_elements, - const std::function - &worker_summary_generator, - std::initializer_list expected_quantiles, - double quantiles_matcher_epsilon) { - // Generate single stream. - double total_weight = 0; - Stream stream(eps, max_elements); - worker_summary_generator(0, max_elements, &total_weight, &stream); - - // Ensure we didn't lose track of any elements and are - // within approximation error bound. - EXPECT_LE(stream.ApproximationError(), eps); - EXPECT_NEAR(stream.GetFinalSummary().TotalWeight(), total_weight, 1e-6); - - // Verify expected quantiles. - int i = 0; - auto actuals = stream.GenerateQuantiles(expected_quantiles.size() - 1); - for (auto expected_quantile : expected_quantiles) { - EXPECT_NEAR(actuals[i], expected_quantile, quantiles_matcher_epsilon); - ++i; - } -} - -// Stream generators. -void GenerateOneValue(int32 worker_id, int64 max_elements, double *total_weight, - Stream *stream) { - stream->PushEntry(10, 1); - ++(*total_weight); - stream->Finalize(); -} - -void GenerateOneZeroWeightedValue(int32 worker_id, int64 max_elements, - double *total_weight, Stream *stream) { - stream->PushEntry(10, 0); - stream->Finalize(); -} - -TEST(WeightedQuantilesStreamTest, OneValue) { - const double eps = 0.01; - const int64 max_elements = 1 << 16; - TestSingleWorkerStreams(eps, max_elements, GenerateOneValue, - {10.0, 10.0, 10.0, 10.0, 10.0}, 1e-2); -} - -TEST(WeightedQuantilesStreamTest, OneZeroWeightValue) { - const double eps = 0.01; - const int64 max_elements = 1 << 16; - TestSingleWorkerStreams(eps, max_elements, GenerateOneZeroWeightedValue, {}, - 1e-2); -} - -TEST(WeightedQuantilesStreamTest, FixedUniform) { - const double eps = 0.01; - const int64 max_elements = 1 << 16; - TestSingleWorkerStreams(eps, max_elements, GenerateFixedUniformSummary, - {0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0}, - 1e-2); -} - -TEST(WeightedQuantilesStreamTest, FixedNonUniform) { - const double eps = 0.01; - const int64 max_elements = 1 << 16; - TestSingleWorkerStreams(eps, max_elements, GenerateFixedNonUniformSummary, - {0, std::sqrt(0.1), std::sqrt(0.2), std::sqrt(0.3), - std::sqrt(0.4), std::sqrt(0.5), std::sqrt(0.6), - std::sqrt(0.7), std::sqrt(0.8), std::sqrt(0.9), 1.0}, - 1e-2); -} - -TEST(WeightedQuantilesStreamTest, RandUniformFixedWeights) { - const double eps = 0.01; - const int64 max_elements = 1 << 16; - TestSingleWorkerStreams( - eps, max_elements, GenerateRandUniformFixedWeightsSummary, - {0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0}, 1e-2); -} - -TEST(WeightedQuantilesStreamTest, RandUniformRandWeights) { - const double eps = 0.01; - const int64 max_elements = 1 << 16; - TestSingleWorkerStreams( - eps, max_elements, GenerateRandUniformRandWeightsSummary, - {0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0}, 1e-2); -} - -// Distributed tests. -void TestDistributedStreams( - int32 num_workers, double eps, int64 max_elements, - const std::function - &worker_summary_generator, - std::initializer_list expected_quantiles, - double quantiles_matcher_epsilon) { - // Simulate streams on each worker running independently - double total_weight = 0; - std::vector> worker_summaries; - for (int32 i = 0; i < num_workers; ++i) { - Stream stream(eps / 2, max_elements); - worker_summary_generator(i, max_elements / num_workers, &total_weight, - &stream); - worker_summaries.push_back(stream.GetFinalSummary().GetEntryList()); - } - - // In the accumulation phase, we aggregate the summaries from each worker - // and build an overall summary while maintaining error bounds by ensuring we - // don't increase the error by more than eps / 2. - Stream reducer_stream(eps, max_elements); - for (const auto &summary : worker_summaries) { - reducer_stream.PushSummary(summary); - } - reducer_stream.Finalize(); - - // Ensure we didn't lose track of any elements and are - // within approximation error bound. - EXPECT_LE(reducer_stream.ApproximationError(), eps); - EXPECT_NEAR(reducer_stream.GetFinalSummary().TotalWeight(), total_weight, - total_weight); - - // Verify expected quantiles. - int i = 0; - auto actuals = - reducer_stream.GenerateQuantiles(expected_quantiles.size() - 1); - for (auto expected_quantile : expected_quantiles) { - EXPECT_NEAR(actuals[i], expected_quantile, quantiles_matcher_epsilon); - ++i; - } -} - -TEST(WeightedQuantilesStreamTest, FixedUniformDistributed) { - const int32 num_workers = 10; - const double eps = 0.01; - const int64 max_elements = num_workers * (1 << 16); - TestDistributedStreams( - num_workers, eps, max_elements, GenerateFixedUniformSummary, - {0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0}, 1e-2); -} - -TEST(WeightedQuantilesStreamTest, FixedNonUniformDistributed) { - const int32 num_workers = 10; - const double eps = 0.01; - const int64 max_elements = num_workers * (1 << 16); - TestDistributedStreams(num_workers, eps, max_elements, - GenerateFixedNonUniformSummary, - {0, std::sqrt(0.1), std::sqrt(0.2), std::sqrt(0.3), - std::sqrt(0.4), std::sqrt(0.5), std::sqrt(0.6), - std::sqrt(0.7), std::sqrt(0.8), std::sqrt(0.9), 1.0}, - 1e-2); -} - -TEST(WeightedQuantilesStreamTest, RandUniformFixedWeightsDistributed) { - const int32 num_workers = 10; - const double eps = 0.01; - const int64 max_elements = num_workers * (1 << 16); - TestDistributedStreams( - num_workers, eps, max_elements, GenerateRandUniformFixedWeightsSummary, - {0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0}, 1e-2); -} - -TEST(WeightedQuantilesStreamTest, RandUniformRandWeightsDistributed) { - const int32 num_workers = 10; - const double eps = 0.01; - const int64 max_elements = num_workers * (1 << 16); - TestDistributedStreams( - num_workers, eps, max_elements, GenerateRandUniformRandWeightsSummary, - {0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0}, 1e-2); -} - -} // namespace -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h b/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h deleted file mode 100644 index 8d71a6cdbc4..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_QUANTILES_WEIGHTED_QUANTILES_SUMMARY_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_QUANTILES_WEIGHTED_QUANTILES_SUMMARY_H_ - -#include -#include - -#include "tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_buffer.h" - -namespace tensorflow { -namespace boosted_trees { -namespace quantiles { - -// Summary holding a sorted block of entries with upper bound guarantees -// over the approximation error. -template > -class WeightedQuantilesSummary { - public: - using Buffer = WeightedQuantilesBuffer; - using BufferEntry = typename Buffer::BufferEntry; - - struct SummaryEntry { - SummaryEntry(const ValueType& v, const WeightType& w, const WeightType& min, - const WeightType& max) { - value = v; - weight = w; - min_rank = min; - max_rank = max; - } - - SummaryEntry() { - value = ValueType(); - weight = 0; - min_rank = 0; - max_rank = 0; - } - - bool operator==(const SummaryEntry& other) const { - return value == other.value && weight == other.weight && - min_rank == other.min_rank && max_rank == other.max_rank; - } - friend std::ostream& operator<<(std::ostream& strm, - const SummaryEntry& entry) { - return strm << "{" << entry.value << ", " << entry.weight << ", " - << entry.min_rank << ", " << entry.max_rank << "}"; - } - - // Max rank estimate for previous smaller value. - WeightType PrevMaxRank() const { return max_rank - weight; } - - // Min rank estimate for next larger value. - WeightType NextMinRank() const { return min_rank + weight; } - - ValueType value; - WeightType weight; - WeightType min_rank; - WeightType max_rank; - }; - - // Re-construct summary from the specified buffer. - void BuildFromBufferEntries(const std::vector& buffer_entries) { - entries_.clear(); - entries_.reserve(buffer_entries.size()); - WeightType cumulative_weight = 0; - for (const auto& entry : buffer_entries) { - WeightType current_weight = entry.weight; - entries_.emplace_back(entry.value, entry.weight, cumulative_weight, - cumulative_weight + current_weight); - cumulative_weight += current_weight; - } - } - - // Re-construct summary from the specified summary entries. - void BuildFromSummaryEntries( - const std::vector& summary_entries) { - entries_.clear(); - entries_.reserve(summary_entries.size()); - entries_.insert(entries_.begin(), summary_entries.begin(), - summary_entries.end()); - } - - // Merges two summaries through an algorithm that's derived from MergeSort - // for summary entries while guaranteeing that the max approximation error - // of the final merged summary is no greater than the approximation errors - // of each individual summary. - // For example consider summaries where each entry is of the form - // (element, weight, min rank, max rank): - // summary entries 1: (1, 3, 0, 3), (4, 2, 3, 5) - // summary entries 2: (3, 1, 0, 1), (4, 1, 1, 2) - // merged: (1, 3, 0, 3), (3, 1, 3, 4), (4, 3, 4, 7). - void Merge(const WeightedQuantilesSummary& other_summary) { - // Make sure we have something to merge. - const auto& other_entries = other_summary.entries_; - if (other_entries.empty()) { - return; - } - if (entries_.empty()) { - BuildFromSummaryEntries(other_summary.entries_); - return; - } - - // Move current entries to make room for a new buffer. - std::vector base_entries(std::move(entries_)); - entries_.clear(); - entries_.reserve(base_entries.size() + other_entries.size()); - - // Merge entries maintaining ranks. The idea is to stack values - // in order which we can do in linear time as the two summaries are - // already sorted. We keep track of the next lower rank from either - // summary and update it as we pop elements from the summaries. - // We handle the special case when the next two elements from either - // summary are equal, in which case we just merge the two elements - // and simultaneously update both ranks. - auto it1 = base_entries.cbegin(); - auto it2 = other_entries.cbegin(); - WeightType next_min_rank1 = 0; - WeightType next_min_rank2 = 0; - while (it1 != base_entries.cend() && it2 != other_entries.cend()) { - if (kCompFn(it1->value, it2->value)) { // value1 < value2 - // Take value1 and use the last added value2 to compute - // the min rank and the current value2 to compute the max rank. - entries_.emplace_back(it1->value, it1->weight, - it1->min_rank + next_min_rank2, - it1->max_rank + it2->PrevMaxRank()); - // Update next min rank 1. - next_min_rank1 = it1->NextMinRank(); - ++it1; - } else if (kCompFn(it2->value, it1->value)) { // value1 > value2 - // Take value2 and use the last added value1 to compute - // the min rank and the current value1 to compute the max rank. - entries_.emplace_back(it2->value, it2->weight, - it2->min_rank + next_min_rank1, - it2->max_rank + it1->PrevMaxRank()); - // Update next min rank 2. - next_min_rank2 = it2->NextMinRank(); - ++it2; - } else { // value1 == value2 - // Straight additive merger of the two entries into one. - entries_.emplace_back(it1->value, it1->weight + it2->weight, - it1->min_rank + it2->min_rank, - it1->max_rank + it2->max_rank); - // Update next min ranks for both. - next_min_rank1 = it1->NextMinRank(); - next_min_rank2 = it2->NextMinRank(); - ++it1; - ++it2; - } - } - - // Fill in any residual. - while (it1 != base_entries.cend()) { - entries_.emplace_back(it1->value, it1->weight, - it1->min_rank + next_min_rank2, - it1->max_rank + other_entries.back().max_rank); - ++it1; - } - while (it2 != other_entries.cend()) { - entries_.emplace_back(it2->value, it2->weight, - it2->min_rank + next_min_rank1, - it2->max_rank + base_entries.back().max_rank); - ++it2; - } - } - - // Compresses buffer into desired size. The size specification is - // considered a hint as we always keep the first and last elements and - // maintain strict approximation error bounds. - // The approximation error delta is taken as the max of either the requested - // min error or 1 / size_hint. - // After compression, the approximation error is guaranteed to increase - // by no more than that error delta. - // This algorithm is linear in the original size of the summary and is - // designed to be cache-friendly. - void Compress(int64 size_hint, double min_eps = 0) { - // No-op if we're already within the size requirement. - size_hint = std::max(size_hint, int64{2}); - if (entries_.size() <= size_hint) { - return; - } - - // First compute the max error bound delta resulting from this compression. - double eps_delta = TotalWeight() * std::max(1.0 / size_hint, min_eps); - - // Compress elements ensuring approximation bounds and elements diversity - // are both maintained. - int64 add_accumulator = 0, add_step = entries_.size(); - auto write_it = entries_.begin() + 1, last_it = write_it; - for (auto read_it = entries_.begin(); read_it + 1 != entries_.end();) { - auto next_it = read_it + 1; - while (next_it != entries_.end() && add_accumulator < add_step && - next_it->PrevMaxRank() - read_it->NextMinRank() <= eps_delta) { - add_accumulator += size_hint; - ++next_it; - } - if (read_it == next_it - 1) { - ++read_it; - } else { - read_it = next_it - 1; - } - (*write_it++) = (*read_it); - last_it = read_it; - add_accumulator -= add_step; - } - // Write last element and resize. - if (last_it + 1 != entries_.end()) { - (*write_it++) = entries_.back(); - } - entries_.resize(write_it - entries_.begin()); - } - - // To construct the boundaries we first run a soft compress over a copy - // of the summary and retrieve the values. - // The resulting boundaries are guaranteed to both contain at least - // num_boundaries unique elements and maintain approximation bounds. - std::vector GenerateBoundaries(int64 num_boundaries) const { - std::vector output; - if (entries_.empty()) { - return output; - } - - // Generate soft compressed summary. - WeightedQuantilesSummary - compressed_summary; - compressed_summary.BuildFromSummaryEntries(entries_); - // Set an epsilon for compression that's at most 1.0 / num_boundaries - // more than epsilon of original our summary since the compression operation - // adds ~1.0/num_boundaries to final approximation error. - float compression_eps = ApproximationError() + (1.0 / num_boundaries); - compressed_summary.Compress(num_boundaries, compression_eps); - - // Return boundaries. - output.reserve(compressed_summary.entries_.size()); - for (const auto& entry : compressed_summary.entries_) { - output.push_back(entry.value); - } - return output; - } - - // To construct the desired n-quantiles we repetitively query n ranks from the - // original summary. The following algorithm is an efficient cache-friendly - // O(n) implementation of that idea which avoids the cost of the repetitive - // full rank queries O(nlogn). - std::vector GenerateQuantiles(int64 num_quantiles) const { - std::vector output; - if (entries_.empty()) { - return output; - } - num_quantiles = std::max(num_quantiles, int64{2}); - output.reserve(num_quantiles + 1); - - // Make successive rank queries to get boundaries. - // We always keep the first (min) and last (max) entries. - for (size_t cur_idx = 0, rank = 0; rank <= num_quantiles; ++rank) { - // This step boils down to finding the next element sub-range defined by - // r = (rmax[i + 1] + rmin[i + 1]) / 2 where the desired rank d < r. - WeightType d_2 = 2 * (rank * entries_.back().max_rank / num_quantiles); - size_t next_idx = cur_idx + 1; - while (next_idx < entries_.size() && - d_2 >= entries_[next_idx].min_rank + entries_[next_idx].max_rank) { - ++next_idx; - } - cur_idx = next_idx - 1; - - // Determine insertion order. - if (next_idx == entries_.size() || - d_2 < entries_[cur_idx].NextMinRank() + - entries_[next_idx].PrevMaxRank()) { - output.push_back(entries_[cur_idx].value); - } else { - output.push_back(entries_[next_idx].value); - } - } - return output; - } - - // Calculates current approximation error which should always be <= eps. - double ApproximationError() const { - if (entries_.empty()) { - return 0; - } - - WeightType max_gap = 0; - for (auto it = entries_.cbegin() + 1; it < entries_.end(); ++it) { - max_gap = std::max(max_gap, - std::max(it->max_rank - it->min_rank - it->weight, - it->PrevMaxRank() - (it - 1)->NextMinRank())); - } - return static_cast(max_gap) / TotalWeight(); - } - - ValueType MinValue() const { - return !entries_.empty() ? entries_.front().value - : std::numeric_limits::max(); - } - ValueType MaxValue() const { - return !entries_.empty() ? entries_.back().value - : std::numeric_limits::max(); - } - WeightType TotalWeight() const { - return !entries_.empty() ? entries_.back().max_rank : 0; - } - int64 Size() const { return entries_.size(); } - void Clear() { entries_.clear(); } - const std::vector& GetEntryList() const { return entries_; } - - private: - // Comparison function. - static constexpr decltype(CompareFn()) kCompFn = CompareFn(); - - // Summary entries. - std::vector entries_; -}; - -template -constexpr decltype(CompareFn()) - WeightedQuantilesSummary::kCompFn; - -} // namespace quantiles -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_QUANTILES_WEIGHTED_QUANTILES_SUMMARY_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary_test.cc b/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary_test.cc deleted file mode 100644 index 8de154483e6..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary_test.cc +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h" - -#include "tensorflow/core/lib/random/philox_random.h" -#include "tensorflow/core/lib/random/simple_philox.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/platform/test_benchmark.h" - -namespace tensorflow { -namespace { - -using Buffer = boosted_trees::quantiles::WeightedQuantilesBuffer; -using BufferEntry = - boosted_trees::quantiles::WeightedQuantilesBuffer::BufferEntry; -using Summary = - boosted_trees::quantiles::WeightedQuantilesSummary; -using SummaryEntry = - boosted_trees::quantiles::WeightedQuantilesSummary::SummaryEntry; - -class WeightedQuantilesSummaryTest : public ::testing::Test { - protected: - void SetUp() override { - // Constructs a buffer of 10 weighted unique entries. - buffer1_.reset(new Buffer(10, 1000)); - buffer1_->PushEntry(5, 9); - buffer1_->PushEntry(2, 3); - buffer1_->PushEntry(-1, 7); - buffer1_->PushEntry(-7, 1); - buffer1_->PushEntry(3, 2); - buffer1_->PushEntry(-2, 3); - buffer1_->PushEntry(21, 8); - buffer1_->PushEntry(-13, 4); - buffer1_->PushEntry(8, 2); - buffer1_->PushEntry(-5, 6); - - // Constructs a buffer of 7 weighted unique entries. - buffer2_.reset(new Buffer(7, 1000)); - buffer2_->PushEntry(9, 2); - buffer2_->PushEntry(-7, 3); - buffer2_->PushEntry(2, 1); - buffer2_->PushEntry(4, 13); - buffer2_->PushEntry(0, 5); - buffer2_->PushEntry(-5, 3); - buffer2_->PushEntry(11, 3); - } - - void TearDown() override { buffer1_->Clear(); } - - std::unique_ptr buffer1_; - std::unique_ptr buffer2_; - const double buffer1_min_value_ = -13; - const double buffer1_max_value_ = 21; - const double buffer1_total_weight_ = 45; - const double buffer2_min_value_ = -7; - const double buffer2_max_value_ = 11; - const double buffer2_total_weight_ = 30; -}; - -TEST_F(WeightedQuantilesSummaryTest, BuildFromBuffer) { - Summary summary; - summary.BuildFromBufferEntries(buffer1_->GenerateEntryList()); - - // We expect no approximation error because no compress operation occurred. - EXPECT_EQ(summary.ApproximationError(), 0); - - // Check first and last elements in the summary. - const auto& entries = summary.GetEntryList(); - // First element's rmin should be zero. - EXPECT_EQ(summary.MinValue(), buffer1_min_value_); - EXPECT_EQ(entries.front(), SummaryEntry(-13, 4, 0, 4)); - // Last element's rmax should be cumulative weight. - EXPECT_EQ(summary.MaxValue(), buffer1_max_value_); - EXPECT_EQ(entries.back(), SummaryEntry(21, 8, 37, 45)); - // Check total weight. - EXPECT_EQ(summary.TotalWeight(), buffer1_total_weight_); -} - -TEST_F(WeightedQuantilesSummaryTest, CompressSeparately) { - const auto entry_list = buffer1_->GenerateEntryList(); - for (int new_size = 9; new_size >= 2; --new_size) { - Summary summary; - summary.BuildFromBufferEntries(entry_list); - summary.Compress(new_size); - - // Expect a max approximation error of 1 / n - // ie. eps0 + 1/n but eps0 = 0. - EXPECT_TRUE(summary.Size() >= new_size && summary.Size() <= new_size + 2); - EXPECT_LE(summary.ApproximationError(), 1.0 / new_size); - - // Min/Max elements and total weight should not change. - EXPECT_EQ(summary.MinValue(), buffer1_min_value_); - EXPECT_EQ(summary.MaxValue(), buffer1_max_value_); - EXPECT_EQ(summary.TotalWeight(), buffer1_total_weight_); - } -} - -TEST_F(WeightedQuantilesSummaryTest, CompressSequentially) { - Summary summary; - summary.BuildFromBufferEntries(buffer1_->GenerateEntryList()); - for (int new_size = 9; new_size >= 2; new_size -= 2) { - double prev_eps = summary.ApproximationError(); - summary.Compress(new_size); - - // Expect a max approximation error of prev_eps + 1 / n. - EXPECT_TRUE(summary.Size() >= new_size && summary.Size() <= new_size + 2); - EXPECT_LE(summary.ApproximationError(), prev_eps + 1.0 / new_size); - - // Min/Max elements and total weight should not change. - EXPECT_EQ(summary.MinValue(), buffer1_min_value_); - EXPECT_EQ(summary.MaxValue(), buffer1_max_value_); - EXPECT_EQ(summary.TotalWeight(), buffer1_total_weight_); - } -} - -TEST_F(WeightedQuantilesSummaryTest, CompressRandomized) { - // Check multiple size compressions and ensure approximation bounds - // are always respected. - int prev_size = 1; - int size = 2; - float max_value = 1 << 20; - while (size < (1 << 16)) { - // Create buffer of size from uniform random elements. - Buffer buffer(size, size << 4); - random::PhiloxRandom philox(13); - random::SimplePhilox rand(&philox); - for (int i = 0; i < size; ++i) { - buffer.PushEntry(rand.RandFloat() * max_value, - rand.RandFloat() * max_value); - } - - // Create summary and compress. - Summary summary; - summary.BuildFromBufferEntries(buffer.GenerateEntryList()); - int new_size = std::max(rand.Uniform(size), 2u); - summary.Compress(new_size); - - // Ensure approximation error is acceptable. - EXPECT_TRUE(summary.Size() >= new_size && summary.Size() <= new_size + 2); - EXPECT_LE(summary.ApproximationError(), 1.0 / new_size); - - // Update size to next fib number. - size_t last_size = size; - size += prev_size; - prev_size = last_size; - } -} - -TEST_F(WeightedQuantilesSummaryTest, MergeSymmetry) { - // Create two separate summaries and merge. - const auto list_1 = buffer1_->GenerateEntryList(); - const auto list_2 = buffer2_->GenerateEntryList(); - Summary summary1; - summary1.BuildFromBufferEntries(list_1); - Summary summary2; - summary2.BuildFromBufferEntries(list_2); - - // Merge summary 2 into 1 and verify. - summary1.Merge(summary2); - EXPECT_EQ(summary1.ApproximationError(), 0.0); - EXPECT_EQ(summary1.MinValue(), - std::min(buffer1_min_value_, buffer2_min_value_)); - EXPECT_EQ(summary1.MaxValue(), - std::max(buffer1_max_value_, buffer2_max_value_)); - EXPECT_EQ(summary1.TotalWeight(), - buffer1_total_weight_ + buffer2_total_weight_); - EXPECT_EQ(summary1.Size(), 14); // 14 unique values. - - // Merge summary 1 into 2 and verify same result. - summary1.BuildFromBufferEntries(list_1); - summary2.Merge(summary1); - EXPECT_EQ(summary2.ApproximationError(), 0.0); - EXPECT_EQ(summary2.MinValue(), - std::min(buffer1_min_value_, buffer2_min_value_)); - EXPECT_EQ(summary2.MaxValue(), - std::max(buffer1_max_value_, buffer2_max_value_)); - EXPECT_EQ(summary2.TotalWeight(), - buffer1_total_weight_ + buffer2_total_weight_); - EXPECT_EQ(summary2.Size(), 14); // 14 unique values. -} - -TEST_F(WeightedQuantilesSummaryTest, CompressThenMerge) { - // Create two separate summaries and merge. - Summary summary1; - summary1.BuildFromBufferEntries(buffer1_->GenerateEntryList()); - Summary summary2; - summary2.BuildFromBufferEntries(buffer2_->GenerateEntryList()); - - // Compress summaries. - summary1.Compress(5); // max error is 1/5. - const auto eps1 = 1.0 / 5; - EXPECT_LE(summary1.ApproximationError(), eps1); - summary2.Compress(3); // max error is 1/3. - const auto eps2 = 1.0 / 3; - EXPECT_LE(summary2.ApproximationError(), eps2); - - // Merge guarantees an approximation error of max(eps1, eps2). - // Merge summary 2 into 1 and verify. - summary1.Merge(summary2); - EXPECT_LE(summary1.ApproximationError(), std::max(eps1, eps2)); - EXPECT_EQ(summary1.MinValue(), - std::min(buffer1_min_value_, buffer2_min_value_)); - EXPECT_EQ(summary1.MaxValue(), - std::max(buffer1_max_value_, buffer2_max_value_)); - EXPECT_EQ(summary1.TotalWeight(), - buffer1_total_weight_ + buffer2_total_weight_); -} - -} // namespace -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/testutil/batch_features_testutil.cc b/tensorflow/contrib/boosted_trees/lib/testutil/batch_features_testutil.cc deleted file mode 100644 index 39c2fbe9c99..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/testutil/batch_features_testutil.cc +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/testutil/batch_features_testutil.h" - -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/status_test_util.h" - -namespace tensorflow { -namespace boosted_trees { -namespace testutil { - -using tensorflow::Tensor; - -void RandomlyInitializeBatchFeatures( - tensorflow::random::SimplePhilox* rng, uint32 num_dense_float_features, - uint32 num_sparse_float_features, double sparsity_lo, double sparsity_hi, - boosted_trees::utils::BatchFeatures* batch_features) { - const int64 batch_size = static_cast(batch_features->batch_size()); - - // Populate dense features. - std::vector dense_float_features_list; - for (int i = 0; i < num_dense_float_features; ++i) { - std::vector values; - for (int64 j = 0; j < batch_size; ++j) { - values.push_back(rng->RandFloat()); - } - auto dense_tensor = Tensor(tensorflow::DT_FLOAT, {batch_size, 1}); - tensorflow::test::FillValues(&dense_tensor, values); - dense_float_features_list.push_back(dense_tensor); - } - - // Populate sparse features. - std::vector sparse_float_feature_indices_list; - std::vector sparse_float_feature_values_list; - std::vector sparse_float_feature_shapes_list; - for (int i = 0; i < num_sparse_float_features; ++i) { - std::set indices; - const double sparsity = - sparsity_lo + rng->RandDouble() * (sparsity_hi - sparsity_lo); - const double density = 1 - sparsity; - for (int64 k = 0; k < static_cast(density * batch_size) + 1; ++k) { - indices.insert(rng->Uniform64(batch_size)); - } - const int64 sparse_values_size = indices.size(); - std::vector indices_vector; - for (auto idx : indices) { - indices_vector.push_back(idx); - indices_vector.push_back(0); - } - auto indices_tensor = Tensor(tensorflow::DT_INT64, {sparse_values_size, 2}); - tensorflow::test::FillValues(&indices_tensor, indices_vector); - sparse_float_feature_indices_list.push_back(indices_tensor); - - std::vector values; - for (int64 j = 0; j < sparse_values_size; ++j) { - values.push_back(rng->RandFloat()); - } - auto values_tensor = Tensor(tensorflow::DT_FLOAT, {sparse_values_size}); - tensorflow::test::FillValues(&values_tensor, values); - sparse_float_feature_values_list.push_back(values_tensor); - - auto shape_tensor = Tensor(tensorflow::DT_INT64, {2}); - tensorflow::test::FillValues(&shape_tensor, {batch_size, 1}); - sparse_float_feature_shapes_list.push_back(shape_tensor); - } - - // TODO(salehay): Add categorical feature generation support. - TF_EXPECT_OK(batch_features->Initialize( - dense_float_features_list, sparse_float_feature_indices_list, - sparse_float_feature_values_list, sparse_float_feature_shapes_list, {}, - {}, {})); -} - -} // namespace testutil -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/testutil/batch_features_testutil.h b/tensorflow/contrib/boosted_trees/lib/testutil/batch_features_testutil.h deleted file mode 100644 index b98190b10dc..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/testutil/batch_features_testutil.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_TESTUTIL_BATCH_FEATURES_TESTUTIL_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_TESTUTIL_BATCH_FEATURES_TESTUTIL_H_ - -#include "tensorflow/contrib/boosted_trees/lib/utils/batch_features.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/random/simple_philox.h" - -namespace tensorflow { -namespace boosted_trees { -namespace testutil { - -// This method calls Initialize on the given 'batch_features', which will be -// populated with randomly generated feature values when the call returns. -// 'tensors' returns a vector of all tensors used in the initialization, -// because they must outlive 'batch_features'. -// -// All float features will be either missing or uniformly randomly chosen -// from [0, 1). For sparse (float) features, a sparsity is uniformly randomly -// chosen from ['sparsity_lo', 'sparsity_hi') per feature, and each instance -// will have a probability of sparsity of missing that feature, in other words, -// sparsity = 1 - density. -void RandomlyInitializeBatchFeatures( - tensorflow::random::SimplePhilox* rng, uint32 num_dense_float_features, - uint32 num_sparse_float_features, double sparsity_lo, double sparsity_hi, - boosted_trees::utils::BatchFeatures* batch_features); - -} // namespace testutil -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_TESTUTIL_BATCH_FEATURES_TESTUTIL_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/testutil/random_tree_gen.cc b/tensorflow/contrib/boosted_trees/lib/testutil/random_tree_gen.cc deleted file mode 100644 index 705b65e9db9..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/testutil/random_tree_gen.cc +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/testutil/random_tree_gen.h" - -#include "tensorflow/core/lib/random/philox_random.h" -#include "tensorflow/core/lib/random/simple_philox.h" -#include "tensorflow/core/platform/logging.h" - -namespace tensorflow { -namespace boosted_trees { -namespace testutil { - -using boosted_trees::trees::DenseFloatBinarySplit; -using tensorflow::boosted_trees::trees::DecisionTreeConfig; -using tensorflow::boosted_trees::trees::TreeNode; - -namespace { - -// Append the given nodes to tree with transfer of pointer ownership. -// nodes will not be usable upon return. -template -void AppendNodes(DecisionTreeConfig* tree, T* nodes) { - std::reverse(nodes->pointer_begin(), nodes->pointer_end()); - while (!nodes->empty()) { - tree->mutable_nodes()->AddAllocated(nodes->ReleaseLast()); - } -} - -DenseFloatBinarySplit* GetSplit(TreeNode* node) { - switch (node->node_case()) { - case TreeNode::kSparseFloatBinarySplitDefaultLeft: - return node->mutable_sparse_float_binary_split_default_left() - ->mutable_split(); - case TreeNode::kSparseFloatBinarySplitDefaultRight: - return node->mutable_sparse_float_binary_split_default_right() - ->mutable_split(); - case TreeNode::kDenseFloatBinarySplit: - return node->mutable_dense_float_binary_split(); - default: - LOG(FATAL) << "Unknown node type encountered."; - } - return nullptr; -} - -} // namespace - -RandomTreeGen::RandomTreeGen(tensorflow::random::SimplePhilox* rng, - int dense_feature_size, int sparse_feature_size) - : rng_(rng), - dense_feature_size_(dense_feature_size), - sparse_feature_size_(sparse_feature_size) {} - -namespace { -void AddWeightAndMetadata( - boosted_trees::trees::DecisionTreeEnsembleConfig* ret) { - // Assign the weight of the tree to 1 and say that this weight was updated - // only once. - ret->add_tree_weights(1.0); - auto* meta = ret->add_tree_metadata(); - meta->set_num_tree_weight_updates(1); -} - -} // namespace - -boosted_trees::trees::DecisionTreeEnsembleConfig -RandomTreeGen::GenerateEnsemble(int depth, int tree_count) { - boosted_trees::trees::DecisionTreeEnsembleConfig ret; - *(ret.add_trees()) = Generate(depth); - AddWeightAndMetadata(&ret); - for (int i = 1; i < tree_count; ++i) { - *(ret.add_trees()) = Generate(ret.trees(0)); - AddWeightAndMetadata(&ret); - } - return ret; -} - -DecisionTreeConfig RandomTreeGen::Generate(const DecisionTreeConfig& tree) { - DecisionTreeConfig ret = tree; - for (auto& node : *ret.mutable_nodes()) { - if (node.node_case() == TreeNode::kLeaf) { - node.mutable_leaf()->mutable_sparse_vector()->set_value( - 0, rng_->RandFloat()); - continue; - } - // Original node is a split. Re-generate it's type but retain the split node - // indices. - DenseFloatBinarySplit* split = GetSplit(&node); - const int left_id = split->left_id(); - const int right_id = split->right_id(); - GenerateSplit(&node, left_id, right_id); - } - return ret; -} - -DecisionTreeConfig RandomTreeGen::Generate(int depth) { - DecisionTreeConfig ret; - // Add root, - TreeNode* node = ret.add_nodes(); - GenerateSplit(node, 1, 2); - if (depth == 1) { - // Add left and right leaves. - TreeNode* left = ret.add_nodes(); - left->mutable_leaf()->mutable_sparse_vector()->add_index(0); - left->mutable_leaf()->mutable_sparse_vector()->add_value(rng_->RandFloat()); - TreeNode* right = ret.add_nodes(); - right->mutable_leaf()->mutable_sparse_vector()->add_index(0); - right->mutable_leaf()->mutable_sparse_vector()->add_value( - rng_->RandFloat()); - return ret; - } else { - DecisionTreeConfig left_branch = Generate(depth - 1); - DecisionTreeConfig right_branch = Generate(depth - 1); - Combine(&ret, &left_branch, &right_branch); - return ret; - } -} - -void RandomTreeGen::Combine(DecisionTreeConfig* root, - DecisionTreeConfig* left_branch, - DecisionTreeConfig* right_branch) { - const int left_branch_size = left_branch->nodes_size(); - CHECK_EQ(1, root->nodes_size()); - // left_branch starts its index at 1. right_branch starts its index at - // (left_branch_size + 1). - auto* root_node = root->mutable_nodes(0); - DenseFloatBinarySplit* root_split = GetSplit(root_node); - root_split->set_left_id(1); - root_split->set_right_id(left_branch_size + 1); - // Shift left/right branch's indices internally so that everything is - // consistent. - ShiftNodeIndex(left_branch, 1); - ShiftNodeIndex(right_branch, left_branch_size + 1); - - // Complexity O(branch node size). No proto copying though. - AppendNodes(root, left_branch->mutable_nodes()); - AppendNodes(root, right_branch->mutable_nodes()); -} - -void RandomTreeGen::ShiftNodeIndex(DecisionTreeConfig* tree, int shift) { - for (TreeNode& node : *(tree->mutable_nodes())) { - DenseFloatBinarySplit* split = nullptr; - switch (node.node_case()) { - case TreeNode::kLeaf: - break; - case TreeNode::kSparseFloatBinarySplitDefaultLeft: - split = node.mutable_sparse_float_binary_split_default_left() - ->mutable_split(); - break; - case TreeNode::kSparseFloatBinarySplitDefaultRight: - split = node.mutable_sparse_float_binary_split_default_right() - ->mutable_split(); - break; - case TreeNode::kDenseFloatBinarySplit: - split = node.mutable_dense_float_binary_split(); - break; - default: - LOG(FATAL) << "Unknown node type encountered."; - } - if (split) { - split->set_left_id(shift + split->left_id()); - split->set_right_id(shift + split->right_id()); - } - } -} - -void RandomTreeGen::GenerateSplit(TreeNode* node, int left_id, int right_id) { - const double denseSplitProb = - sparse_feature_size_ == 0 - ? 1.0 - : static_cast(dense_feature_size_) / - (dense_feature_size_ + sparse_feature_size_); - // Generate the tree such that it has equal probability of going left and - // right when the feature is missing. - static constexpr float kLeftProb = 0.5; - - DenseFloatBinarySplit* split; - int feature_size; - if (rng_->RandFloat() < denseSplitProb) { - feature_size = dense_feature_size_; - split = node->mutable_dense_float_binary_split(); - } else { - feature_size = sparse_feature_size_; - if (rng_->RandFloat() < kLeftProb) { - split = node->mutable_sparse_float_binary_split_default_left() - ->mutable_split(); - } else { - split = node->mutable_sparse_float_binary_split_default_right() - ->mutable_split(); - } - } - split->set_threshold(rng_->RandFloat()); - split->set_feature_column(rng_->Uniform(feature_size)); - split->set_left_id(left_id); - split->set_right_id(right_id); -} - -} // namespace testutil -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/testutil/random_tree_gen.h b/tensorflow/contrib/boosted_trees/lib/testutil/random_tree_gen.h deleted file mode 100644 index 1838b4cee21..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/testutil/random_tree_gen.h +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_TESTUTIL_RANDOM_TREE_GEN_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_TESTUTIL_RANDOM_TREE_GEN_H_ - -#include - -#include "tensorflow/contrib/boosted_trees/proto/tree_config.pb.h" // NOLINT -#include "tensorflow/core/lib/random/simple_philox.h" -#include "tensorflow/core/platform/macros.h" - -namespace tensorflow { -namespace boosted_trees { -namespace testutil { - -// Randomly generate a balanced tree, for performance benchmarking purposes, -// that assume all features are sparse float features, for now. -class RandomTreeGen { - public: - RandomTreeGen(tensorflow::random::SimplePhilox* rng, int dense_feature_size, - int sparse_feature_size); - - // Required: depth must be >= 1. - // If one wants to generate multiple trees with the same depth, see also the - // overload below. - boosted_trees::trees::DecisionTreeConfig Generate(int depth); - - // Randomly generate a new tree with the same depth (and tree structure) - // as the given tree. This is faster. - boosted_trees::trees::DecisionTreeConfig Generate( - const boosted_trees::trees::DecisionTreeConfig& tree); - - // Required: depth >= 1; tree_count >= 1. - boosted_trees::trees::DecisionTreeEnsembleConfig GenerateEnsemble( - int dept, int tree_count); - - private: - tensorflow::random::SimplePhilox* rng_; - const int dense_feature_size_; - const int sparse_feature_size_; - - // Put together a deeper tree by combining two trees. - void Combine(boosted_trees::trees::DecisionTreeConfig* root, - boosted_trees::trees::DecisionTreeConfig* left_branch, - boosted_trees::trees::DecisionTreeConfig* right_branch); - - // For each node in the provided tree, shift its referenced left/right index - // by shift. - void ShiftNodeIndex(boosted_trees::trees::DecisionTreeConfig* tree, - int shift); - - // Generate a sparse split in the node. - void GenerateSplit(boosted_trees::trees::TreeNode* node, int left_id, - int right_id); - - TF_DISALLOW_COPY_AND_ASSIGN(RandomTreeGen); -}; - -} // namespace testutil -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_TESTUTIL_RANDOM_TREE_GEN_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/testutil/random_tree_gen_main.cc b/tensorflow/contrib/boosted_trees/lib/testutil/random_tree_gen_main.cc deleted file mode 100644 index 5ea81e8d9a4..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/testutil/random_tree_gen_main.cc +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -// Randomly generate a tree ensemble and write to file. - -#include "tensorflow/contrib/boosted_trees/lib/testutil/random_tree_gen.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/random/philox_random.h" -#include "tensorflow/core/lib/random/simple_philox.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/command_line_flags.h" - -using tensorflow::Flag; -using tensorflow::Flags; -using tensorflow::int32; -using tensorflow::string; - -int main(int argc, char* argv[]) { - int32 dense_feature_size = 100; - int32 sparse_feature_size = 100; - int32 depth = 8; - int32 tree_count = 10; - string filename = "/tmp/trees.pb"; - std::vector flag_list = { - Flag("dense_feature_size", &dense_feature_size, "dense feature size"), - Flag("sparse_feature_size", &sparse_feature_size, "sparse_feature_size"), - Flag("depth", &depth, "tree depth"), - Flag("tree_count", &tree_count, "tree count"), - Flag("filename", &filename, "Output filename."), - }; - string usage = Flags::Usage(argv[0], flag_list); - const bool parse_result = Flags::Parse(&argc, argv, flag_list); - // We need to call this to set up global state for TensorFlow. - tensorflow::port::InitMain(usage.c_str(), &argc, &argv); - if (!parse_result) { - LOG(ERROR) << "\n" << usage; - return -1; - } - - tensorflow::random::PhiloxRandom philox(1); - tensorflow::random::SimplePhilox rng(&philox); - tensorflow::boosted_trees::testutil::RandomTreeGen tree_gen( - &rng, dense_feature_size, sparse_feature_size); - const auto& trees = tree_gen.GenerateEnsemble(depth, tree_count); - tensorflow::Status status = - tensorflow::WriteBinaryProto(tensorflow::Env::Default(), filename, trees); - if (!status.ok()) { - LOG(WARNING) << "Failed to write: " << filename << " : " << status; - } else { - LOG(INFO) << "Tree ensemble written to: " << filename; - } - return 0; -} diff --git a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc deleted file mode 100644 index de30a7bde79..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include - -#include "tensorflow/contrib/boosted_trees/lib/trees/decision_tree.h" -#include "tensorflow/core/platform/macros.h" - -namespace tensorflow { -namespace boosted_trees { -namespace trees { - -constexpr int kInvalidLeaf = -1; -int DecisionTree::Traverse(const DecisionTreeConfig& config, - const int32 sub_root_id, - const utils::Example& example) { - if (TF_PREDICT_FALSE(config.nodes_size() <= sub_root_id)) { - return kInvalidLeaf; - } - // Traverse tree starting at the provided sub-root. - int32 node_id = sub_root_id; - // The index of the leave that holds this example in the oblivious case. - int oblivious_leaf_idx = 0; - while (true) { - const auto& current_node = config.nodes(node_id); - switch (current_node.node_case()) { - case TreeNode::kLeaf: { - return node_id + oblivious_leaf_idx; - } - case TreeNode::kDenseFloatBinarySplit: { - const auto& split = current_node.dense_float_binary_split(); - node_id = example.dense_float_features[split.feature_column()] <= - split.threshold() - ? split.left_id() - : split.right_id(); - break; - } - case TreeNode::kSparseFloatBinarySplitDefaultLeft: { - const auto& split = - current_node.sparse_float_binary_split_default_left().split(); - auto sparse_feature = - example.sparse_float_features[split.feature_column()]; - // Feature id for the split when multivalent sparse float column, or 0 - // by default. - const int32 dimension_id = split.dimension_id(); - - node_id = !sparse_feature[dimension_id].has_value() || - sparse_feature[dimension_id].get_value() <= - split.threshold() - ? split.left_id() - : split.right_id(); - break; - } - case TreeNode::kSparseFloatBinarySplitDefaultRight: { - const auto& split = - current_node.sparse_float_binary_split_default_right().split(); - auto sparse_feature = - example.sparse_float_features[split.feature_column()]; - // Feature id for the split when multivalent sparse float column, or 0 - // by default. - const int32 dimension_id = split.dimension_id(); - node_id = sparse_feature[dimension_id].has_value() && - sparse_feature[dimension_id].get_value() <= - split.threshold() - ? split.left_id() - : split.right_id(); - break; - } - case TreeNode::kCategoricalIdBinarySplit: { - const auto& split = current_node.categorical_id_binary_split(); - const auto& features = - example.sparse_int_features[split.feature_column()]; - node_id = (std::find(features.begin(), features.end(), - split.feature_id()) == features.end()) - ? split.right_id() - : split.left_id(); - break; - } - case TreeNode::kCategoricalIdSetMembershipBinarySplit: { - const auto& split = - current_node.categorical_id_set_membership_binary_split(); - // The new node_id = left_id if a feature is found, or right_id. - node_id = split.right_id(); - for (const int64 feature_id : - example.sparse_int_features[split.feature_column()]) { - if (std::binary_search(split.feature_ids().begin(), - split.feature_ids().end(), feature_id)) { - node_id = split.left_id(); - break; - } - } - break; - } - case TreeNode::kObliviousDenseFloatBinarySplit: { - const auto& split = current_node.oblivious_dense_float_binary_split(); - oblivious_leaf_idx <<= 1; - if (example.dense_float_features[split.feature_column()] > - split.threshold()) { - oblivious_leaf_idx++; - } - node_id++; - break; - } - case TreeNode::kObliviousCategoricalIdBinarySplit: { - const auto& split = - current_node.oblivious_categorical_id_binary_split(); - oblivious_leaf_idx <<= 1; - const auto& features = - example.sparse_int_features[split.feature_column()]; - if (std::find(features.begin(), features.end(), split.feature_id()) == - features.end()) { - oblivious_leaf_idx++; - } - node_id++; - break; - } - case TreeNode::NODE_NOT_SET: { - LOG(QFATAL) << "Invalid node in tree: " << current_node.DebugString(); - break; - } - } - DCHECK_NE(node_id, 0) << "Malformed tree, cycles found to root:" - << current_node.DebugString(); - } -} - -void DecisionTree::LinkChildren(const std::vector& children, - TreeNode* parent_node) { - // Decide how to link children depending on the parent node's type. - auto children_it = children.begin(); - switch (parent_node->node_case()) { - case TreeNode::kLeaf: { - // Essentially no-op. - QCHECK(children.empty()) << "A leaf node cannot have children."; - break; - } - case TreeNode::kDenseFloatBinarySplit: { - QCHECK(children.size() == 2) - << "A binary split node must have exactly two children."; - auto* split = parent_node->mutable_dense_float_binary_split(); - split->set_left_id(*children_it); - split->set_right_id(*++children_it); - break; - } - case TreeNode::kSparseFloatBinarySplitDefaultLeft: { - QCHECK(children.size() == 2) - << "A binary split node must have exactly two children."; - auto* split = - parent_node->mutable_sparse_float_binary_split_default_left() - ->mutable_split(); - split->set_left_id(*children_it); - split->set_right_id(*++children_it); - break; - } - case TreeNode::kSparseFloatBinarySplitDefaultRight: { - QCHECK(children.size() == 2) - << "A binary split node must have exactly two children."; - auto* split = - parent_node->mutable_sparse_float_binary_split_default_right() - ->mutable_split(); - split->set_left_id(*children_it); - split->set_right_id(*++children_it); - break; - } - case TreeNode::kCategoricalIdBinarySplit: { - QCHECK(children.size() == 2) - << "A binary split node must have exactly two children."; - auto* split = parent_node->mutable_categorical_id_binary_split(); - split->set_left_id(*children_it); - split->set_right_id(*++children_it); - break; - } - case TreeNode::kCategoricalIdSetMembershipBinarySplit: { - QCHECK(children.size() == 2) - << "A binary split node must have exactly two children."; - auto* split = - parent_node->mutable_categorical_id_set_membership_binary_split(); - split->set_left_id(*children_it); - split->set_right_id(*++children_it); - break; - } - case TreeNode::kObliviousDenseFloatBinarySplit: { - LOG(QFATAL) - << "Not implemented for the ObliviousDenseFloatBinarySplit case."; - break; - } - case TreeNode::kObliviousCategoricalIdBinarySplit: { - LOG(QFATAL) - << "Not implemented for the ObliviousCategoricalIdBinarySplit case."; - break; - } - case TreeNode::NODE_NOT_SET: { - LOG(QFATAL) << "A non-set node cannot have children."; - break; - } - } -} - -std::vector DecisionTree::GetChildren(const TreeNode& node) { - // A node's children depend on its type. - switch (node.node_case()) { - case TreeNode::kLeaf: { - return {}; - } - case TreeNode::kDenseFloatBinarySplit: { - const auto& split = node.dense_float_binary_split(); - return {split.left_id(), split.right_id()}; - } - case TreeNode::kSparseFloatBinarySplitDefaultLeft: { - const auto& split = node.sparse_float_binary_split_default_left().split(); - return {split.left_id(), split.right_id()}; - } - case TreeNode::kSparseFloatBinarySplitDefaultRight: { - const auto& split = - node.sparse_float_binary_split_default_right().split(); - return {split.left_id(), split.right_id()}; - } - case TreeNode::kCategoricalIdBinarySplit: { - const auto& split = node.categorical_id_binary_split(); - return {split.left_id(), split.right_id()}; - } - case TreeNode::kCategoricalIdSetMembershipBinarySplit: { - const auto& split = node.categorical_id_set_membership_binary_split(); - return {split.left_id(), split.right_id()}; - } - case TreeNode::kObliviousDenseFloatBinarySplit: { - LOG(QFATAL) - << "Not implemented for the ObliviousDenseFloatBinarySplit case."; - return {}; - } - case TreeNode::kObliviousCategoricalIdBinarySplit: { - LOG(QFATAL) - << "Not implemented for the ObliviousCategoricalIdBinarySplit case."; - break; - } - case TreeNode::NODE_NOT_SET: { - return {}; - } - } -} - -} // namespace trees -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.h b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.h deleted file mode 100644 index 43526c229a6..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_TREES_DECISION_TREE_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_TREES_DECISION_TREE_H_ - -#include "tensorflow/contrib/boosted_trees/lib/utils/example.h" -#include "tensorflow/contrib/boosted_trees/proto/tree_config.pb.h" // NOLINT - -namespace tensorflow { -namespace boosted_trees { -namespace trees { - -// Decision tree class to encapsulate tree traversal and mutation logic. -// This class does not hold state and is thread safe. -class DecisionTree { - public: - // Traverse given an instance, a sub-root and its set of features - // and return the leaf index or -1 if the tree is empty or - // the sub-root is invalid. - static int Traverse(const DecisionTreeConfig& config, int32 sub_root_id, - const utils::Example& example); - - // Links the specified children to the parent, the children must - // already be added to the decision tree config so this method - // just ensures nodes are re-linked. - static void LinkChildren(const std::vector& children, - TreeNode* parent_node); - - // Retrieves node children indices if any. - static std::vector GetChildren(const TreeNode& node); -}; - -} // namespace trees -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_TREES_DECISION_TREE_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc deleted file mode 100644 index 58fe8e335af..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc +++ /dev/null @@ -1,403 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/trees/decision_tree.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/batch_features.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace boosted_trees { -namespace trees { -namespace { - -class DecisionTreeTest : public ::testing::Test { - protected: - DecisionTreeTest() : batch_features_(2) { - // Create a batch of two examples having one dense float, two sparse float - // and one sparse int features, and one sparse multi-column float feature - // (SparseFM). - // The first example is missing the second sparse feature column and the - // second example is missing the first sparse feature column. - // This looks like the following: - // Instance | DenseF1 | SparseF1 | SparseF2 | SparseI1 | SparseFM (3 cols) - // 0 | 7 | -3 | | 3 | 3.0 | | 1.0 - // 1 | -2 | | 4 | | 1.5 |3.5| - auto dense_float_matrix = test::AsTensor({7.0f, -2.0f}, {2, 1}); - auto sparse_float_indices1 = test::AsTensor({0, 0}, {1, 2}); - auto sparse_float_values1 = test::AsTensor({-3.0f}); - auto sparse_float_shape1 = test::AsTensor({2, 1}); - auto sparse_float_indices2 = test::AsTensor({1, 0}, {1, 2}); - auto sparse_float_values2 = test::AsTensor({4.0f}); - auto sparse_float_shape2 = test::AsTensor({2, 1}); - auto sparse_int_indices1 = test::AsTensor({0, 0}, {1, 2}); - auto sparse_int_values1 = test::AsTensor({3}); - auto sparse_int_shape1 = test::AsTensor({2, 1}); - - // Multivalent sparse feature. - auto multi_sparse_float_indices = - test::AsTensor({0, 0, 0, 2, 1, 0, 1, 1}, {4, 2}); - auto multi_sparse_float_values = - test::AsTensor({3.0f, 1.0f, 1.5f, 3.5f}); - auto multi_sparse_float_shape = test::AsTensor({2, 3}); - - TF_EXPECT_OK(batch_features_.Initialize( - {dense_float_matrix}, - {sparse_float_indices1, sparse_float_indices2, - multi_sparse_float_indices}, - {sparse_float_values1, sparse_float_values2, multi_sparse_float_values}, - {sparse_float_shape1, sparse_float_shape2, multi_sparse_float_shape}, - {sparse_int_indices1}, {sparse_int_values1}, {sparse_int_shape1})); - } - - template - void TestLinkChildrenBinary(TreeNode* node, SplitType* split) { - // Verify children were linked. - DecisionTree::LinkChildren({3, 8}, node); - EXPECT_EQ(3, split->left_id()); - EXPECT_EQ(8, split->right_id()); - - // Invalid cases. - EXPECT_DEATH(DecisionTree::LinkChildren({}, node), - "A binary split node must have exactly two children."); - EXPECT_DEATH(DecisionTree::LinkChildren({3}, node), - "A binary split node must have exactly two children."); - EXPECT_DEATH(DecisionTree::LinkChildren({1, 2, 3}, node), - "A binary split node must have exactly two children."); - } - - void TestGetChildren(const TreeNode& node, - const std::vector& expected_children) { - // Verify children were linked. - auto children = DecisionTree::GetChildren(node); - EXPECT_EQ(children.size(), expected_children.size()); - for (size_t idx = 0; idx < children.size(); ++idx) { - EXPECT_EQ(children[idx], expected_children[idx]); - } - } - - utils::BatchFeatures batch_features_; -}; - -TEST_F(DecisionTreeTest, TraverseEmpty) { - DecisionTreeConfig tree_config; - auto example = (*batch_features_.examples_iterable(0, 1).begin()); - EXPECT_EQ(-1, DecisionTree::Traverse(tree_config, 0, example)); -} - -TEST_F(DecisionTreeTest, TraverseBias) { - DecisionTreeConfig tree_config; - tree_config.add_nodes()->mutable_leaf(); - auto example = (*batch_features_.examples_iterable(0, 1).begin()); - EXPECT_EQ(0, DecisionTree::Traverse(tree_config, 0, example)); -} - -TEST_F(DecisionTreeTest, TraverseInvalidSubRoot) { - DecisionTreeConfig tree_config; - tree_config.add_nodes()->mutable_leaf(); - auto example = (*batch_features_.examples_iterable(0, 1).begin()); - EXPECT_EQ(-1, DecisionTree::Traverse(tree_config, 10, example)); -} - -TEST_F(DecisionTreeTest, TraverseDenseBinarySplit) { - DecisionTreeConfig tree_config; - auto* split_node = - tree_config.add_nodes()->mutable_dense_float_binary_split(); - split_node->set_feature_column(0); - split_node->set_threshold(0.0f); - split_node->set_left_id(1); - split_node->set_right_id(2); - tree_config.add_nodes()->mutable_leaf(); - tree_config.add_nodes()->mutable_leaf(); - auto example_iterable = batch_features_.examples_iterable(0, 2); - - // Expect right child to be picked as !(7 <= 0); - auto example_it = example_iterable.begin(); - EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it)); - - // Expect left child to be picked as (-2 <= 0); - EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it)); -} - -TEST_F(DecisionTreeTest, TraverseSparseBinarySplit) { - auto example_iterable = batch_features_.examples_iterable(0, 2); - // Split on SparseF1. - // Test first sparse feature which is missing for the second example. - { - DecisionTreeConfig tree_config; - auto* split_node = tree_config.add_nodes() - ->mutable_sparse_float_binary_split_default_left() - ->mutable_split(); - split_node->set_feature_column(0); - split_node->set_threshold(-20.0f); - split_node->set_left_id(1); - split_node->set_right_id(2); - tree_config.add_nodes()->mutable_leaf(); - tree_config.add_nodes()->mutable_leaf(); - - // Expect right child to be picked as !(-3 <= -20). - auto example_it = example_iterable.begin(); - EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it)); - - // Expect left child to be picked as default direction. - EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it)); - } - // Split on SparseF2. - // Test second sparse feature which is missing for the first example. - { - DecisionTreeConfig tree_config; - auto* split_node = tree_config.add_nodes() - ->mutable_sparse_float_binary_split_default_right() - ->mutable_split(); - split_node->set_feature_column(1); - split_node->set_threshold(4.0f); - split_node->set_left_id(1); - split_node->set_right_id(2); - tree_config.add_nodes()->mutable_leaf(); - tree_config.add_nodes()->mutable_leaf(); - - // Expect right child to be picked as default direction. - auto example_it = example_iterable.begin(); - EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it)); - - // Expect left child to be picked as (4 <= 4). - EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it)); - } - // Split on SparseFM. - // Test second sparse feature which is missing for the first example. - { - DecisionTreeConfig tree_config; - auto* split_node = tree_config.add_nodes() - ->mutable_sparse_float_binary_split_default_right() - ->mutable_split(); - split_node->set_feature_column(2); - - split_node->set_left_id(1); - split_node->set_right_id(2); - tree_config.add_nodes()->mutable_leaf(); - tree_config.add_nodes()->mutable_leaf(); - - // Split on first column - split_node->set_dimension_id(0); - split_node->set_threshold(2.0f); - - // Both instances have this feature value. - auto example_it = example_iterable.begin(); - EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it)); - EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it)); - - // Split on second column - split_node->set_dimension_id(1); - split_node->set_threshold(5.0f); - - // First instance does not have it (default right), second does have it. - example_it = example_iterable.begin(); - EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it)); - EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it)); - - // Split on third column - split_node->set_dimension_id(2); - split_node->set_threshold(3.0f); - example_it = example_iterable.begin(); - - // First instance has it, second does not (default right). - EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *example_it)); - EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *++example_it)); - } -} - -TEST_F(DecisionTreeTest, TraverseCategoricalIdBinarySplit) { - DecisionTreeConfig tree_config; - auto* split_node = - tree_config.add_nodes()->mutable_categorical_id_binary_split(); - split_node->set_feature_column(0); - split_node->set_feature_id(3); - split_node->set_left_id(1); - split_node->set_right_id(2); - tree_config.add_nodes()->mutable_leaf(); - tree_config.add_nodes()->mutable_leaf(); - auto example_iterable = batch_features_.examples_iterable(0, 2); - - // Expect left child to be picked as 3 == 3; - auto example_it = example_iterable.begin(); - EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *example_it)); - - // Expect right child to be picked as the feature is missing; - EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *++example_it)); -} - -TEST_F(DecisionTreeTest, TraverseCategoricalIdSetMembershipBinarySplit) { - DecisionTreeConfig tree_config; - auto* split_node = tree_config.add_nodes() - ->mutable_categorical_id_set_membership_binary_split(); - split_node->set_feature_column(0); - split_node->add_feature_ids(3); - split_node->set_left_id(1); - split_node->set_right_id(2); - tree_config.add_nodes()->mutable_leaf(); - tree_config.add_nodes()->mutable_leaf(); - auto example_iterable = batch_features_.examples_iterable(0, 2); - - // Expect left child to be picked as 3 in {3}; - auto example_it = example_iterable.begin(); - EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *example_it)); - - // Expect right child to be picked as the feature is missing; - EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *++example_it)); -} - -TEST_F(DecisionTreeTest, TraverseHybridSplits) { - DecisionTreeConfig tree_config; - auto* split_node1 = - tree_config.add_nodes()->mutable_dense_float_binary_split(); - split_node1->set_feature_column(0); - split_node1->set_threshold(9.0f); - split_node1->set_left_id(1); // sparse split. - split_node1->set_right_id(2); // leaf - auto* split_node2 = tree_config.add_nodes() - ->mutable_sparse_float_binary_split_default_left() - ->mutable_split(); - tree_config.add_nodes()->mutable_leaf(); - split_node2->set_feature_column(0); - split_node2->set_threshold(-20.0f); - split_node2->set_left_id(3); - split_node2->set_right_id(4); - auto* split_node3 = - tree_config.add_nodes()->mutable_categorical_id_binary_split(); - split_node3->set_feature_column(0); - split_node3->set_feature_id(2); - split_node3->set_left_id(5); - split_node3->set_right_id(6); - tree_config.add_nodes()->mutable_leaf(); - tree_config.add_nodes()->mutable_leaf(); - tree_config.add_nodes()->mutable_leaf(); - auto example_iterable = batch_features_.examples_iterable(0, 2); - - // Expect will go left through the first dense split as (7.0f <= 9.0f), - // then will go right through the sparse split as !(-3 <= -20). - auto example_it = example_iterable.begin(); - EXPECT_EQ(4, DecisionTree::Traverse(tree_config, 0, *example_it)); - - // Expect will go left through the first dense split as (-2.0f <= 9.0f), - // then will go left the default direction as the sparse feature is missing, - // then will go right as 2 != 3 on the categorical split. - EXPECT_EQ(6, DecisionTree::Traverse(tree_config, 0, *++example_it)); -} - -TEST_F(DecisionTreeTest, LinkChildrenLeaf) { - // Create leaf node. - TreeNode node; - node.mutable_leaf(); - - // No-op. - DecisionTree::LinkChildren({}, &node); - - // Invalid case. - EXPECT_DEATH(DecisionTree::LinkChildren({1}, &node), - "A leaf node cannot have children."); -} - -TEST_F(DecisionTreeTest, LinkChildrenDenseFloatBinarySplit) { - TreeNode node; - auto* split = node.mutable_dense_float_binary_split(); - split->set_left_id(-1); - split->set_right_id(-1); - TestLinkChildrenBinary(&node, split); -} - -TEST_F(DecisionTreeTest, LinkChildrenSparseFloatBinarySplitDefaultLeft) { - TreeNode node; - auto* split = - node.mutable_sparse_float_binary_split_default_left()->mutable_split(); - split->set_left_id(-1); - split->set_right_id(-1); - TestLinkChildrenBinary(&node, split); -} - -TEST_F(DecisionTreeTest, LinkChildrenSparseFloatBinarySplitDefaultRight) { - TreeNode node; - auto* split = - node.mutable_sparse_float_binary_split_default_right()->mutable_split(); - split->set_left_id(-1); - split->set_right_id(-1); - TestLinkChildrenBinary(&node, split); -} - -TEST_F(DecisionTreeTest, LinkChildrenCategoricalSingleIdBinarySplit) { - TreeNode node; - auto* split = node.mutable_categorical_id_binary_split(); - split->set_left_id(-1); - split->set_right_id(-1); - TestLinkChildrenBinary(&node, split); -} - -TEST_F(DecisionTreeTest, LinkChildrenNodeNotSet) { - // Create unset node. - TreeNode node; - - // Invalid case. - EXPECT_DEATH(DecisionTree::LinkChildren({1}, &node), - "A non-set node cannot have children."); -} - -TEST_F(DecisionTreeTest, GetChildrenLeaf) { - TreeNode node; - node.mutable_leaf(); - TestGetChildren(node, {}); -} - -TEST_F(DecisionTreeTest, GetChildrenDenseFloatBinarySplit) { - TreeNode node; - auto* split = node.mutable_dense_float_binary_split(); - split->set_left_id(23); - split->set_right_id(24); - TestGetChildren(node, {23, 24}); -} - -TEST_F(DecisionTreeTest, GetChildrenSparseFloatBinarySplitDefaultLeft) { - TreeNode node; - auto* split = - node.mutable_sparse_float_binary_split_default_left()->mutable_split(); - split->set_left_id(12); - split->set_right_id(13); - TestGetChildren(node, {12, 13}); -} - -TEST_F(DecisionTreeTest, GetChildrenSparseFloatBinarySplitDefaultRight) { - TreeNode node; - auto* split = - node.mutable_sparse_float_binary_split_default_right()->mutable_split(); - split->set_left_id(1); - split->set_right_id(2); - TestGetChildren(node, {1, 2}); -} - -TEST_F(DecisionTreeTest, GetChildrenCategoricalSingleIdBinarySplit) { - TreeNode node; - auto* split = node.mutable_categorical_id_binary_split(); - split->set_left_id(7); - split->set_right_id(8); - TestGetChildren(node, {7, 8}); -} - -TEST_F(DecisionTreeTest, GetChildrenNodeNotSet) { - TreeNode node; - TestGetChildren(node, {}); -} - -} // namespace -} // namespace trees -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc deleted file mode 100644 index 4fab2b0b7de..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include "tensorflow/contrib/boosted_trees/lib/utils/batch_features.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/macros.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.h" -#include "tensorflow/core/lib/core/errors.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -Status BatchFeatures::Initialize( - std::vector dense_float_features_list, - std::vector sparse_float_feature_indices_list, - std::vector sparse_float_feature_values_list, - std::vector sparse_float_feature_shapes_list, - std::vector sparse_int_feature_indices_list, - std::vector sparse_int_feature_values_list, - std::vector sparse_int_feature_shapes_list) { - // Validate number of feature columns. - auto num_dense_float_features = dense_float_features_list.size(); - auto num_sparse_float_features = sparse_float_feature_indices_list.size(); - auto num_sparse_int_features = sparse_int_feature_indices_list.size(); - QCHECK(num_dense_float_features + num_sparse_float_features + - num_sparse_int_features > - 0) - << "Must have at least one feature column."; - - // Read dense float features. - dense_float_feature_columns_.reserve(num_dense_float_features); - for (uint32 dense_feat_idx = 0; dense_feat_idx < num_dense_float_features; - ++dense_feat_idx) { - auto dense_float_feature = dense_float_features_list[dense_feat_idx]; - TF_CHECK_AND_RETURN_IF_ERROR( - TensorShapeUtils::IsMatrix(dense_float_feature.shape()), - errors::InvalidArgument("Dense float feature must be a matrix.")); - TF_CHECK_AND_RETURN_IF_ERROR( - dense_float_feature.dim_size(0) == batch_size_, - errors::InvalidArgument( - "Dense float vector must have batch_size rows: ", batch_size_, - " vs. ", dense_float_feature.dim_size(0))); - TF_CHECK_AND_RETURN_IF_ERROR( - dense_float_feature.dim_size(1) == 1, - errors::InvalidArgument( - "Dense float features may not be multivalent: dim_size(1) = ", - dense_float_feature.dim_size(1))); - dense_float_feature_columns_.emplace_back(dense_float_feature); - } - - // Read sparse float features. - sparse_float_feature_columns_.reserve(num_sparse_float_features); - TF_CHECK_AND_RETURN_IF_ERROR( - sparse_float_feature_values_list.size() == num_sparse_float_features && - sparse_float_feature_shapes_list.size() == num_sparse_float_features, - errors::InvalidArgument("Inconsistent number of sparse float features.")); - for (uint32 sparse_feat_idx = 0; sparse_feat_idx < num_sparse_float_features; - ++sparse_feat_idx) { - auto sparse_float_feature_indices = - sparse_float_feature_indices_list[sparse_feat_idx]; - auto sparse_float_feature_values = - sparse_float_feature_values_list[sparse_feat_idx]; - auto sparse_float_feature_shape = - sparse_float_feature_shapes_list[sparse_feat_idx]; - TF_CHECK_AND_RETURN_IF_ERROR( - TensorShapeUtils::IsMatrix(sparse_float_feature_indices.shape()), - errors::InvalidArgument( - "Sparse float feature indices must be a matrix.")); - TF_CHECK_AND_RETURN_IF_ERROR( - TensorShapeUtils::IsVector(sparse_float_feature_values.shape()), - errors::InvalidArgument( - "Sparse float feature values must be a vector.")); - TF_CHECK_AND_RETURN_IF_ERROR( - TensorShapeUtils::IsVector(sparse_float_feature_shape.shape()), - errors::InvalidArgument( - "Sparse float feature shape must be a vector.")); - auto shape_flat = sparse_float_feature_shape.flat(); - TF_CHECK_AND_RETURN_IF_ERROR( - shape_flat.size() == 2, - errors::InvalidArgument( - "Sparse float feature column must be two-dimensional.")); - TF_CHECK_AND_RETURN_IF_ERROR( - shape_flat(0) == batch_size_, - errors::InvalidArgument( - "Sparse float feature shape incompatible with batch size.")); - auto tensor_shape = TensorShape({shape_flat(0), shape_flat(1)}); - auto order_dims = sparse::SparseTensor::VarDimArray({0, 1}); - sparse::SparseTensor sparse_tensor; - TF_RETURN_IF_ERROR(sparse::SparseTensor::Create( - sparse_float_feature_indices, sparse_float_feature_values, tensor_shape, - order_dims, &sparse_tensor)); - sparse_float_feature_columns_.push_back(std::move(sparse_tensor)); - } - - // Read sparse int features. - sparse_int_feature_columns_.reserve(num_sparse_int_features); - TF_CHECK_AND_RETURN_IF_ERROR( - sparse_int_feature_values_list.size() == num_sparse_int_features && - sparse_int_feature_shapes_list.size() == num_sparse_int_features, - errors::InvalidArgument("Inconsistent number of sparse int features.")); - for (uint32 sparse_feat_idx = 0; sparse_feat_idx < num_sparse_int_features; - ++sparse_feat_idx) { - auto sparse_int_feature_indices = - sparse_int_feature_indices_list[sparse_feat_idx]; - auto sparse_int_feature_values = - sparse_int_feature_values_list[sparse_feat_idx]; - auto sparse_int_feature_shape = - sparse_int_feature_shapes_list[sparse_feat_idx]; - TF_CHECK_AND_RETURN_IF_ERROR( - TensorShapeUtils::IsMatrix(sparse_int_feature_indices.shape()), - errors::InvalidArgument( - "Sparse int feature indices must be a matrix.")); - TF_CHECK_AND_RETURN_IF_ERROR( - TensorShapeUtils::IsVector(sparse_int_feature_values.shape()), - errors::InvalidArgument("Sparse int feature values must be a vector.")); - TF_CHECK_AND_RETURN_IF_ERROR( - TensorShapeUtils::IsVector(sparse_int_feature_shape.shape()), - errors::InvalidArgument("Sparse int feature shape must be a vector.")); - auto shape_flat = sparse_int_feature_shape.flat(); - TF_CHECK_AND_RETURN_IF_ERROR( - shape_flat.size() == 2, - errors::InvalidArgument( - "Sparse int feature column must be two-dimensional.")); - TF_CHECK_AND_RETURN_IF_ERROR( - shape_flat(0) == batch_size_, - errors::InvalidArgument( - "Sparse int feature shape incompatible with batch size.")); - auto tensor_shape = TensorShape({shape_flat(0), shape_flat(1)}); - auto order_dims = sparse::SparseTensor::VarDimArray({0, 1}); - sparse::SparseTensor sparse_tensor; - TF_RETURN_IF_ERROR(sparse::SparseTensor::Create( - sparse_int_feature_indices, sparse_int_feature_values, tensor_shape, - order_dims, &sparse_tensor)); - sparse_int_feature_columns_.push_back(std::move(sparse_tensor)); - } - return Status::OK(); -} - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h deleted file mode 100644 index a3b1b013e3a..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_BATCH_FEATURES_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_BATCH_FEATURES_H_ - -#include -#include "tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/util/sparse/sparse_tensor.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -class BatchFeatures { - public: - // Constructs batch features with a fixed batch size. - explicit BatchFeatures(int64 batch_size) : batch_size_(batch_size) {} - - // Disallow copy and assign. - BatchFeatures(const BatchFeatures& other) = delete; - BatchFeatures& operator=(const BatchFeatures& other) = delete; - - // Method to initialize batch features from op kernel context. - Status Initialize(std::vector dense_float_features_list, - std::vector sparse_float_feature_indices_list, - std::vector sparse_float_feature_values_list, - std::vector sparse_float_feature_shapes_list, - std::vector sparse_int_feature_indices_list, - std::vector sparse_int_feature_values_list, - std::vector sparse_int_feature_shapes_list); - - Status GetFeatureColumnSizes(int64* const num_dense_float_features, - int64* const num_sparse_float_features, - int64* const num_sparse_int_features) const { - QCHECK_NE(num_dense_float_features, static_cast(nullptr)); - QCHECK_NE(num_sparse_float_features, static_cast(nullptr)); - QCHECK_NE(num_sparse_int_features, static_cast(nullptr)); - *num_dense_float_features = dense_float_feature_columns_.size(); - *num_sparse_float_features = sparse_float_feature_columns_.size(); - *num_sparse_int_features = sparse_int_feature_columns_.size(); - if (*num_dense_float_features == 0 && *num_sparse_float_features == 0 && - *num_sparse_int_features == 0) { - return errors::FailedPrecondition("Not initialized yet."); - } - return Status::OK(); - } - - // Creates an example iterable for the requested slice. - ExamplesIterable examples_iterable(int64 example_start, - int64 example_end) const { - QCHECK(example_start >= 0 && example_end >= 0); - QCHECK(example_start < batch_size_ && example_end <= batch_size_); - return ExamplesIterable( - dense_float_feature_columns_, sparse_float_feature_columns_, - sparse_int_feature_columns_, example_start, example_end); - } - - // Returns the fixed batch size. - int64 batch_size() const { return batch_size_; } - - private: - // Total number of examples in the batch. - const int64 batch_size_; - - // Dense float feature columns. - std::vector dense_float_feature_columns_; - - // Sparse float feature columns. - std::vector sparse_float_feature_columns_; - - // Sparse int feature columns. - std::vector sparse_int_feature_columns_; -}; - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_BATCH_FEATURES_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc deleted file mode 100644 index cfe9101e743..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include "tensorflow/contrib/boosted_trees/lib/utils/batch_features.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { -namespace { - -using errors::InvalidArgument; -using test::AsTensor; - -class BatchFeaturesTest : public ::testing::Test {}; - -TEST_F(BatchFeaturesTest, InvalidNumFeatures) { - BatchFeatures batch_features(8); - EXPECT_DEATH(({ batch_features.Initialize({}, {}, {}, {}, {}, {}, {}); }) - .IgnoreError(), - "Must have at least one feature column."); -} - -TEST_F(BatchFeaturesTest, DenseFloatFeatures_WrongShape) { - BatchFeatures batch_features(8); - auto dense_vec = AsTensor({3.0f, 7.0f}); - auto expected_error = - InvalidArgument("Dense float feature must be a matrix."); - EXPECT_EQ(expected_error, - batch_features.Initialize({dense_vec}, {}, {}, {}, {}, {}, {})); -} - -TEST_F(BatchFeaturesTest, DenseFloatFeatures_WrongBatchDimension) { - BatchFeatures batch_features(8); - auto dense_vec = AsTensor({3.0f, 7.0f}, {2, 1}); - auto expected_error = - InvalidArgument("Dense float vector must have batch_size rows: 8 vs. 2"); - EXPECT_EQ(expected_error, - batch_features.Initialize({dense_vec}, {}, {}, {}, {}, {}, {})); -} - -TEST_F(BatchFeaturesTest, DenseFloatFeatures_Multivalent) { - BatchFeatures batch_features(1); - auto dense_vec = AsTensor({3.0f, 7.0f}, {1, 2}); - auto expected_error = InvalidArgument( - "Dense float features may not be multivalent: dim_size(1) = 2"); - EXPECT_EQ(expected_error, - batch_features.Initialize({dense_vec}, {}, {}, {}, {}, {}, {})); -} - -TEST_F(BatchFeaturesTest, SparseFloatFeatures_WrongShapeIndices) { - BatchFeatures batch_features(2); - auto sparse_float_feature_indices = AsTensor({0, 0, 1, 0}); - auto sparse_float_feature_values = AsTensor({3.0f, 7.0f}); - auto sparse_float_feature_shape = AsTensor({2, 1}); - auto expected_error = - InvalidArgument("Sparse float feature indices must be a matrix."); - EXPECT_EQ(expected_error, batch_features.Initialize( - {}, {sparse_float_feature_indices}, - {sparse_float_feature_values}, - {sparse_float_feature_shape}, {}, {}, {})); -} - -TEST_F(BatchFeaturesTest, SparseFloatFeatures_WrongShapeValues) { - BatchFeatures batch_features(2); - auto sparse_float_feature_indices = AsTensor({0, 0, 1, 0}, {2, 2}); - auto sparse_float_feature_values = AsTensor({3.0f, 7.0f}, {1, 2}); - auto sparse_float_feature_shape = AsTensor({2, 1}); - auto expected_error = - InvalidArgument("Sparse float feature values must be a vector."); - EXPECT_EQ(expected_error, batch_features.Initialize( - {}, {sparse_float_feature_indices}, - {sparse_float_feature_values}, - {sparse_float_feature_shape}, {}, {}, {})); -} - -TEST_F(BatchFeaturesTest, SparseFloatFeatures_WrongShapeShape) { - BatchFeatures batch_features(2); - auto sparse_float_feature_indices = AsTensor({0, 0, 1, 0}, {2, 2}); - auto sparse_float_feature_values = AsTensor({3.0f, 7.0f}); - auto sparse_float_feature_shape = AsTensor({2, 1}, {1, 2}); - auto expected_error = - InvalidArgument("Sparse float feature shape must be a vector."); - EXPECT_EQ(expected_error, batch_features.Initialize( - {}, {sparse_float_feature_indices}, - {sparse_float_feature_values}, - {sparse_float_feature_shape}, {}, {}, {})); -} - -TEST_F(BatchFeaturesTest, SparseFloatFeatures_WrongSizeShape) { - BatchFeatures batch_features(2); - auto sparse_float_feature_indices = AsTensor({0, 0, 1, 0}, {2, 2}); - auto sparse_float_feature_values = AsTensor({3.0f, 7.0f}); - auto sparse_float_feature_shape = AsTensor({2, 1, 9}); - auto expected_error = - InvalidArgument("Sparse float feature column must be two-dimensional."); - EXPECT_EQ(expected_error, batch_features.Initialize( - {}, {sparse_float_feature_indices}, - {sparse_float_feature_values}, - {sparse_float_feature_shape}, {}, {}, {})); -} - -TEST_F(BatchFeaturesTest, SparseFloatFeatures_IncompatibleShape) { - BatchFeatures batch_features(2); - auto sparse_float_feature_indices = AsTensor({0, 0, 1, 0}, {2, 2}); - auto sparse_float_feature_values = AsTensor({3.0f, 7.0f}); - auto sparse_float_feature_shape = AsTensor({8, 1}); - auto expected_error = InvalidArgument( - "Sparse float feature shape incompatible with batch size."); - EXPECT_EQ(expected_error, batch_features.Initialize( - {}, {sparse_float_feature_indices}, - {sparse_float_feature_values}, - {sparse_float_feature_shape}, {}, {}, {})); -} - -TEST_F(BatchFeaturesTest, SparseIntFeatures_WrongShapeIndices) { - BatchFeatures batch_features(2); - auto sparse_int_feature_indices = AsTensor({0, 0, 1, 0}); - auto sparse_int_feature_values = AsTensor({3, 7}); - auto sparse_int_feature_shape = AsTensor({2, 1}); - auto expected_error = - InvalidArgument("Sparse int feature indices must be a matrix."); - EXPECT_EQ(expected_error, - batch_features.Initialize( - {}, {}, {}, {}, {sparse_int_feature_indices}, - {sparse_int_feature_values}, {sparse_int_feature_shape})); -} - -TEST_F(BatchFeaturesTest, SparseIntFeatures_WrongShapeValues) { - BatchFeatures batch_features(2); - auto sparse_int_feature_indices = AsTensor({0, 0, 1, 0}, {2, 2}); - auto sparse_int_feature_values = AsTensor({3, 7}, {1, 2}); - auto sparse_int_feature_shape = AsTensor({2, 1}); - auto expected_error = - InvalidArgument("Sparse int feature values must be a vector."); - EXPECT_EQ(expected_error, - batch_features.Initialize( - {}, {}, {}, {}, {sparse_int_feature_indices}, - {sparse_int_feature_values}, {sparse_int_feature_shape})); -} - -TEST_F(BatchFeaturesTest, SparseIntFeatures_WrongShapeShape) { - BatchFeatures batch_features(2); - auto sparse_int_feature_indices = AsTensor({0, 0, 1, 0}, {2, 2}); - auto sparse_int_feature_values = AsTensor({3, 7}); - auto sparse_int_feature_shape = AsTensor({2, 1}, {1, 2}); - auto expected_error = - InvalidArgument("Sparse int feature shape must be a vector."); - EXPECT_EQ(expected_error, - batch_features.Initialize( - {}, {}, {}, {}, {sparse_int_feature_indices}, - {sparse_int_feature_values}, {sparse_int_feature_shape})); -} - -TEST_F(BatchFeaturesTest, SparseIntFeatures_WrongSizeShape) { - BatchFeatures batch_features(2); - auto sparse_int_feature_indices = AsTensor({0, 0, 1, 0}, {2, 2}); - auto sparse_int_feature_values = AsTensor({3, 7}); - auto sparse_int_feature_shape = AsTensor({2, 1, 9}); - auto expected_error = - InvalidArgument("Sparse int feature column must be two-dimensional."); - EXPECT_EQ(expected_error, - batch_features.Initialize( - {}, {}, {}, {}, {sparse_int_feature_indices}, - {sparse_int_feature_values}, {sparse_int_feature_shape})); -} - -TEST_F(BatchFeaturesTest, SparseIntFeatures_IncompatibleShape) { - BatchFeatures batch_features(2); - auto sparse_int_feature_indices = AsTensor({0, 0, 1, 0}, {2, 2}); - auto sparse_int_feature_values = AsTensor({3, 7}); - auto sparse_int_feature_shape = AsTensor({8, 1}); - auto expected_error = - InvalidArgument("Sparse int feature shape incompatible with batch size."); - EXPECT_EQ(expected_error, - batch_features.Initialize( - {}, {}, {}, {}, {sparse_int_feature_indices}, - {sparse_int_feature_values}, {sparse_int_feature_shape})); -} - -} // namespace -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc deleted file mode 100644 index ce67db797de..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h" - -#include -#include -#include - -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/random/philox_random.h" -#include "tensorflow/core/lib/random/simple_philox.h" -#include "tensorflow/core/platform/logging.h" - -using tensorflow::Status; -using tensorflow::boosted_trees::learner::LearningRateDropoutDrivenConfig; -using tensorflow::random::PhiloxRandom; -using tensorflow::random::SimplePhilox; - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -Status DropoutUtils::DropOutTrees( - const uint64 seed, const LearningRateDropoutDrivenConfig& config, - const std::unordered_set& trees_not_to_drop, - const std::vector& weights, std::vector* dropped_trees, - std::vector* original_weights) { - // Verify params. - if (dropped_trees == nullptr) { - return errors::Internal("Dropped trees is nullptr."); - } - if (original_weights == nullptr) { - return errors::InvalidArgument("Original weights is nullptr."); - } - const float dropout_probability = config.dropout_probability(); - if (dropout_probability < 0 || dropout_probability > 1) { - return errors::InvalidArgument( - "Dropout probability must be in [0,1] range"); - } - const float probability_of_skipping_dropout = - config.probability_of_skipping_dropout(); - if (probability_of_skipping_dropout < 0 || - probability_of_skipping_dropout > 1) { - return errors::InvalidArgument( - "Probability of skipping dropout must be in [0,1] range"); - } - const auto num_trees = weights.size(); - - dropped_trees->clear(); - original_weights->clear(); - - // If dropout is no op, return. - if (dropout_probability == 0 || probability_of_skipping_dropout == 1.0) { - return Status::OK(); - } - - // Roll the dice for each tree. - PhiloxRandom philox(seed); - SimplePhilox rng(&philox); - - std::vector trees_to_keep; - - // What is the probability of skipping dropout altogether. - if (probability_of_skipping_dropout != 0) { - // First roll the dice - do we do dropout - double roll = rng.RandDouble(); - if (roll < probability_of_skipping_dropout) { - // don't do dropout - return Status::OK(); - } - } - - for (int32 i = 0; i < num_trees; ++i) { - // We can't drop some of the trees: for example, bias tree in batch mode, - // or current tree that is built, in the batch mode. - if (trees_not_to_drop.find(i) != trees_not_to_drop.end()) { - continue; - } - double roll = rng.RandDouble(); - if (roll >= dropout_probability) { - trees_to_keep.push_back(i); - } else { - dropped_trees->push_back(i); - } - } - - // Sort the dropped trees indices. - std::sort(dropped_trees->begin(), dropped_trees->end()); - for (const int32 dropped_tree : *dropped_trees) { - original_weights->push_back(weights[dropped_tree]); - } - - return Status::OK(); -} - -void DropoutUtils::GetTreesWeightsForAddingTrees( - const std::vector& dropped_trees, - const std::vector& dropped_trees_original_weights, - const int32 new_trees_first_index, const int32 num_trees_to_add, - std::vector* current_weights, std::vector* num_updates) { - CHECK(num_updates->size() == current_weights->size()); - // combined weight of trees that were dropped out - - const float dropped_sum = - std::accumulate(dropped_trees_original_weights.begin(), - dropped_trees_original_weights.end(), 0.0); - - const int num_dropped = dropped_trees.size(); - - // Allocate additional weight for the new tree - const float total_new_trees_weight = dropped_sum / (num_dropped + 1); - - for (int i = 0; i < num_trees_to_add; ++i) { - const int32 new_tree_index = new_trees_first_index + i; - if (new_tree_index < current_weights->size()) { - // We have the entries in weights and updates for this tree already - (*current_weights)[new_tree_index] = - total_new_trees_weight / num_trees_to_add; - (*num_updates)[new_tree_index]++; - } else { - // We need to add a new entry. This is non-batch mode. - current_weights->push_back(total_new_trees_weight / num_trees_to_add); - num_updates->push_back(1); - } - } - - for (int32 i = 0; i < dropped_trees.size(); ++i) { - const int32 dropped = dropped_trees[i]; - const float original_weight = dropped_trees_original_weights[i]; - const float new_weight = original_weight * num_dropped / (num_dropped + 1); - (*current_weights)[dropped] = new_weight; - // Update the number of updates per tree. - ++(*num_updates)[dropped]; - } -} - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h deleted file mode 100644 index 77c16da5410..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_DROPOUT_UTILS_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_DROPOUT_UTILS_H_ - -#include -#include - -#include "tensorflow/contrib/boosted_trees/proto/learner.pb.h" // NOLINT -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -// Utils for deciding on what trees to be/were dropped when building a new tree. -class DropoutUtils { - public: - // This method determines what trees should be dropped and returns their - // indices and the weights they had when this method ran. - // seed: random seed to be used - // config: dropout config, that defines the probability of dropout etc - // trees_not_to_drop: indices of trees that can't be dropped, for example bias - // (0) and the last tree in the batch mode. - // number_of_trees_to_consider: how many trees are currently in the ensemble - // weights: weights of those trees - // Returns sorted vector of indices of trees to be dropped and their original - // weights. - static tensorflow::Status DropOutTrees( - const uint64 seed, const learner::LearningRateDropoutDrivenConfig& config, - const std::unordered_set& trees_not_to_drop, - const std::vector& weights, std::vector* dropped_trees, - std::vector* original_weights); - - // Recalculates the weights of the trees when the new trees are added to - // ensemble. - // dropped_trees: ids of trees that were dropped when trees to add were built. - // dropped_trees_original_weights: the weight dropped trees had during dropout - // new_trees_first_index: index of the last tree. If it is already in the - // ensemble, its weight and num updates are adjusted. Otherwise, its weight - // and num updates are added as new entries to current_weights and - // num_updates. num_trees_to_add: how many trees are being added to the - // ensemble. Returns current_weights: updated vector of the tree weights. - // Weights of dropped trees are updated. Note that the size of returned vector - // will be total_num_trees + num_trees_to_add (the last elements are the - // weights of the new trees to be added) if new_trees_first_index - // >=current_weights.size num_updates: updated vector with increased number of - // updates for dropped trees. - static void GetTreesWeightsForAddingTrees( - const std::vector& dropped_trees, - const std::vector& dropped_trees_original_weights, - const int32 new_trees_first_index, const int32 num_trees_to_add, - // Current weights and num_updates will be updated as a result of this - // func - std::vector* current_weights, - // How many weight assignments have been done for each tree already. - std::vector* num_updates); -}; - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_DROPOUT_UTILS_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils_test.cc deleted file mode 100644 index 1067fe409cf..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils_test.cc +++ /dev/null @@ -1,447 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "tensorflow/contrib/boosted_trees/proto/tree_config.pb.h" // NOLINT -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/env.h" - -using std::unordered_set; -using tensorflow::boosted_trees::learner::LearningRateDropoutDrivenConfig; -using tensorflow::boosted_trees::trees::DecisionTreeEnsembleConfig; - -namespace tensorflow { -namespace boosted_trees { -namespace utils { -namespace { - -const uint32 kSeed = 123; -const int32 kNumTrees = 1000; - -class DropoutUtilsTest : public ::testing::Test { - public: - void SetUp() override { - // Fill an weights. - for (int i = 0; i < kNumTrees; ++i) { - weights_.push_back(1.1 + 0.4 * i); - } - } - - protected: - std::vector weights_; -}; - -TEST_F(DropoutUtilsTest, DropoutProbabilityTest) { - std::vector dropped_trees; - std::vector original_weights; - std::unordered_set trees_not_to_drop; - - // Do not drop any trees - { - LearningRateDropoutDrivenConfig config; - config.set_dropout_probability(0.0); - config.set_learning_rate(1.0); - - TF_EXPECT_OK(DropoutUtils::DropOutTrees(kSeed, config, trees_not_to_drop, - weights_, &dropped_trees, - &original_weights)); - - // Nothing changed - EXPECT_TRUE(dropped_trees.empty()); - EXPECT_TRUE(original_weights.empty()); - } - // Drop out all trees - { - LearningRateDropoutDrivenConfig config; - config.set_dropout_probability(1.0); - config.set_learning_rate(1.0); - - TF_EXPECT_OK(DropoutUtils::DropOutTrees(kSeed, config, trees_not_to_drop, - weights_, &dropped_trees, - &original_weights)); - - // No trees left - EXPECT_EQ(kNumTrees, dropped_trees.size()); - EXPECT_EQ(kNumTrees, original_weights.size()); - EXPECT_EQ(original_weights, weights_); - } - // 50% probability of dropping a tree - { - const int32 kNumRuns = 1000; - LearningRateDropoutDrivenConfig config; - config.set_dropout_probability(0.5); - config.set_learning_rate(1.0); - - int32 total_num_trees = 0; - for (int i = 0; i < kNumRuns; ++i) { - // draw random seeds - uint random_generator_seed = - static_cast(Env::Default()->NowMicros()); - uint32 seed = rand_r(&random_generator_seed) % 100 + i; - TF_EXPECT_OK(DropoutUtils::DropOutTrees(seed, config, trees_not_to_drop, - weights_, &dropped_trees, - &original_weights)); - - // We would expect 400-600 trees left - EXPECT_NEAR(500, kNumTrees - dropped_trees.size(), 100); - total_num_trees += kNumTrees - dropped_trees.size(); - - // Trees dropped are unique - unordered_set ids; - for (const auto& tree : dropped_trees) { - ids.insert(tree); - } - EXPECT_EQ(ids.size(), dropped_trees.size()); - } - EXPECT_NEAR(500, total_num_trees / kNumRuns, 5); - } -} - -TEST_F(DropoutUtilsTest, DropoutIgnoresNotToDropTest) { - std::vector dropped_trees; - std::vector original_weights; - - // Empty do not drop set. - { - std::unordered_set trees_not_to_drop; - - LearningRateDropoutDrivenConfig config; - config.set_dropout_probability(1.0); - config.set_learning_rate(1.0); - - TF_EXPECT_OK(DropoutUtils::DropOutTrees(kSeed, config, trees_not_to_drop, - weights_, &dropped_trees, - &original_weights)); - - // No trees left - EXPECT_EQ(kNumTrees, dropped_trees.size()); - EXPECT_EQ(kNumTrees, original_weights.size()); - EXPECT_EQ(original_weights, weights_); - } - - // Do not drop any trees - { - std::unordered_set trees_not_to_drop; - for (int i = 0; i < kNumTrees; ++i) { - trees_not_to_drop.insert(i); - } - - LearningRateDropoutDrivenConfig config; - config.set_dropout_probability(1.0); - config.set_learning_rate(1.0); - - TF_EXPECT_OK(DropoutUtils::DropOutTrees(kSeed, config, trees_not_to_drop, - weights_, &dropped_trees, - &original_weights)); - - // No trees were dropped - they all were in do not drop set. - EXPECT_EQ(0, dropped_trees.size()); - EXPECT_EQ(0, original_weights.size()); - } - // Do not drop some trees - { - std::unordered_set trees_not_to_drop; - trees_not_to_drop.insert(0); - trees_not_to_drop.insert(34); - - LearningRateDropoutDrivenConfig config; - config.set_dropout_probability(1.0); - config.set_learning_rate(1.0); - - TF_EXPECT_OK(DropoutUtils::DropOutTrees(kSeed, config, trees_not_to_drop, - weights_, &dropped_trees, - &original_weights)); - - // No trees were dropped - they all were in do not drop set. - EXPECT_EQ(kNumTrees - 2, dropped_trees.size()); - EXPECT_EQ(kNumTrees - 2, original_weights.size()); - EXPECT_TRUE(std::find(dropped_trees.begin(), dropped_trees.end(), 0) == - dropped_trees.end()); - EXPECT_TRUE(std::find(dropped_trees.begin(), dropped_trees.end(), 34) == - dropped_trees.end()); - } -} - -TEST_F(DropoutUtilsTest, DropoutSeedTest) { - std::unordered_set trees_not_to_drop; - // Different seeds remove different trees - { - LearningRateDropoutDrivenConfig config; - config.set_dropout_probability(0.5); - config.set_learning_rate(1.0); - - std::vector dropped_trees_1; - std::vector original_weights_1; - std::vector dropped_trees_2; - std::vector original_weights_2; - - TF_EXPECT_OK(DropoutUtils::DropOutTrees( - kSeed + 1, config, trees_not_to_drop, weights_, &dropped_trees_1, - &original_weights_1)); - TF_EXPECT_OK(DropoutUtils::DropOutTrees( - kSeed + 2, config, trees_not_to_drop, weights_, &dropped_trees_2, - &original_weights_2)); - - EXPECT_FALSE(dropped_trees_1 == dropped_trees_2); - EXPECT_FALSE(original_weights_1 == original_weights_2); - } - // The same seed produces the same result - { - LearningRateDropoutDrivenConfig config; - config.set_dropout_probability(0.5); - config.set_learning_rate(1.0); - - std::vector dropped_trees_1; - std::vector original_weights_1; - std::vector dropped_trees_2; - std::vector original_weights_2; - - - TF_EXPECT_OK(DropoutUtils::DropOutTrees(kSeed, config, trees_not_to_drop, - weights_, &dropped_trees_1, - &original_weights_1)); - TF_EXPECT_OK(DropoutUtils::DropOutTrees(kSeed, config, trees_not_to_drop, - weights_, &dropped_trees_2, - &original_weights_2)); - - EXPECT_TRUE(dropped_trees_1 == dropped_trees_2); - EXPECT_TRUE(original_weights_1 == original_weights_2); - } -} - -TEST_F(DropoutUtilsTest, InvalidConfigTest) { - std::vector dropped_trees; - std::vector original_weights; - std::unordered_set trees_not_to_drop; - // Negative prob - { - LearningRateDropoutDrivenConfig config; - config.set_dropout_probability(-1.34); - - EXPECT_FALSE(DropoutUtils::DropOutTrees(kSeed, config, trees_not_to_drop, - weights_, &dropped_trees, - &original_weights) - .ok()); - } - // Larger than 1 prob of dropping a tree. - { - LearningRateDropoutDrivenConfig config; - config.set_dropout_probability(1.34); - - EXPECT_FALSE(DropoutUtils::DropOutTrees(kSeed, config, trees_not_to_drop, - weights_, &dropped_trees, - &original_weights) - .ok()); - } - // Negative probability of skipping dropout. - { - LearningRateDropoutDrivenConfig config; - config.set_dropout_probability(0.5); - config.set_probability_of_skipping_dropout(-10); - - EXPECT_FALSE(DropoutUtils::DropOutTrees(kSeed, config, trees_not_to_drop, - weights_, &dropped_trees, - &original_weights) - .ok()); - } - // Larger than 1 probability of skipping dropout. - { - LearningRateDropoutDrivenConfig config; - config.set_dropout_probability(0.5); - config.set_probability_of_skipping_dropout(1.2); - - EXPECT_FALSE(DropoutUtils::DropOutTrees(kSeed, config, trees_not_to_drop, - weights_, &dropped_trees, - &original_weights) - .ok()); - } -} -namespace { - -void ExpectVecsEquiv(const std::vector& vec1, - const std::vector& vec2) { - EXPECT_EQ(vec1.size(), vec2.size()); - for (int i = 0; i < vec1.size(); ++i) { - EXPECT_NEAR(vec1[i], vec2[i], 1e-3); - } -} - -std::vector GetWeightsByIndex(const std::vector& weights, - const std::vector& indices) { - std::vector res; - res.reserve(indices.size()); - for (const int index : indices) { - res.push_back(weights[index]); - } - return res; -} - -void MergeLastElements(const int32 last_n, std::vector* weights) { - float sum = 0.0; - for (int i = 0; i < last_n; ++i) { - sum += weights->back(); - weights->pop_back(); - } - weights->push_back(sum); -} - -} // namespace - -TEST_F(DropoutUtilsTest, GetTreesWeightsForAddingTreesTest) { - // Adding trees should give the same res in any order - { - std::vector weights = {1.0, 1.0, 1.0, 1.0, 1.0}; - std::vector dropped_1 = {0, 3}; - - std::vector dropped_2 = {0}; - - std::vector res_1; - std::vector res_2; - // Do one order - { - std::vector current_weights = weights; - std::vector num_updates = - std::vector(current_weights.size(), 1); - DropoutUtils::GetTreesWeightsForAddingTrees( - dropped_1, GetWeightsByIndex(current_weights, dropped_1), - current_weights.size(), 1, ¤t_weights, &num_updates); - DropoutUtils::GetTreesWeightsForAddingTrees( - dropped_2, GetWeightsByIndex(current_weights, dropped_2), - current_weights.size(), 1, ¤t_weights, &num_updates); - res_1 = current_weights; - } - // Do another order - { - std::vector current_weights = weights; - std::vector num_updates = - std::vector(current_weights.size(), 1); - - DropoutUtils::GetTreesWeightsForAddingTrees( - dropped_2, GetWeightsByIndex(current_weights, dropped_2), - current_weights.size(), 1, ¤t_weights, &num_updates); - DropoutUtils::GetTreesWeightsForAddingTrees( - dropped_1, GetWeightsByIndex(current_weights, dropped_1), - current_weights.size(), 1, ¤t_weights, &num_updates); - res_2 = current_weights; - } - // The vectors are the same, but the last two elements have the same sum. - EXPECT_EQ(res_1.size(), 7); - EXPECT_EQ(res_2.size(), 7); - - MergeLastElements(2, &res_1); - MergeLastElements(2, &res_2); - - EXPECT_EQ(res_1, res_2); - } - // Now when the weights are not all 1s - { - std::vector weights = {1.1, 2.1, 3.1, 4.1, 5.1}; - std::vector dropped_1 = {0, 3}; - - std::vector dropped_2 = {0}; - - std::vector res_1; - std::vector res_2; - // Do one order - { - std::vector current_weights = weights; - std::vector num_updates = - std::vector(current_weights.size(), 1); - DropoutUtils::GetTreesWeightsForAddingTrees( - dropped_1, GetWeightsByIndex(current_weights, dropped_1), - current_weights.size(), 1, ¤t_weights, &num_updates); - DropoutUtils::GetTreesWeightsForAddingTrees( - dropped_2, GetWeightsByIndex(current_weights, dropped_2), - current_weights.size(), 1, ¤t_weights, &num_updates); - res_1 = current_weights; - } - // Do another order - { - std::vector current_weights = weights; - std::vector num_updates = - std::vector(current_weights.size(), 1); - DropoutUtils::GetTreesWeightsForAddingTrees( - dropped_2, GetWeightsByIndex(current_weights, dropped_2), - current_weights.size(), 1, ¤t_weights, &num_updates); - DropoutUtils::GetTreesWeightsForAddingTrees( - dropped_1, GetWeightsByIndex(current_weights, dropped_1), - current_weights.size(), 1, ¤t_weights, &num_updates); - res_2 = current_weights; - } - EXPECT_EQ(res_1.size(), 7); - EXPECT_EQ(res_2.size(), 7); - - // The vectors are the same, but the last two elements have the same sum. - MergeLastElements(2, &res_1); - MergeLastElements(2, &res_2); - - ExpectVecsEquiv(res_1, res_2); - } -} - -TEST_F(DropoutUtilsTest, GetTreesWeightsForAddingTreesIndexTest) { - std::vector weights = {1.0, 1.0, 1.0, 1.0, 1.0}; - std::vector dropped = {0, 3}; - - std::vector res; - std::vector res_2; - - // The tree that is added does not yet have an entry in weights vector. - { - std::vector current_weights = weights; - std::vector num_updates = - std::vector(current_weights.size(), 1); - DropoutUtils::GetTreesWeightsForAddingTrees( - dropped, GetWeightsByIndex(current_weights, dropped), - current_weights.size(), 1, ¤t_weights, &num_updates); - EXPECT_EQ(current_weights.size(), weights.size() + 1); - EXPECT_EQ(num_updates.size(), weights.size() + 1); - - std::vector expected_num_updates = {2, 1, 1, 2, 1, 1}; - std::vector expected_weights = {2.0 / 3, 1, 1, 2.0 / 3, 1, 2.0 / 3}; - EXPECT_EQ(expected_weights, current_weights); - EXPECT_EQ(expected_num_updates, num_updates); - } - // The tree that is added has already an entry in weights and updates (batch - // mode). - { - std::vector current_weights = weights; - std::vector num_updates = - std::vector(current_weights.size(), 1); - DropoutUtils::GetTreesWeightsForAddingTrees( - dropped, GetWeightsByIndex(current_weights, dropped), - current_weights.size() - 1, 1, ¤t_weights, &num_updates); - EXPECT_EQ(current_weights.size(), weights.size()); - EXPECT_EQ(num_updates.size(), weights.size()); - - std::vector expected_num_updates = {2, 1, 1, 2, 2}; - std::vector expected_weights = {2.0 / 3, 1, 1, 2.0 / 3, 2.0 / 3}; - EXPECT_EQ(expected_weights, current_weights); - EXPECT_EQ(expected_num_updates, num_updates); - } -} - -} // namespace -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/example.h b/tensorflow/contrib/boosted_trees/lib/utils/example.h deleted file mode 100644 index 445ffaaa714..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/example.h +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_EXAMPLE_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_EXAMPLE_H_ - -#include -#include -#include -#include "tensorflow/contrib/boosted_trees/lib/utils/optional_value.h" -#include "tensorflow/core/lib/gtl/inlined_vector.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { -// Represents sparse vector that have a value for some feature indices within -// the feature column. -// Allows subscript access []. -template -class SparseMultidimensionalValues { - public: - void Add(const int32 feature_idx, const T value) { - values_.emplace_back(feature_idx, value); - } - - void Clear() { values_.clear(); } - - void Reserve(const int32 size) { values_.reserve(size); } - - OptionalValue operator[](int feature_idx) const { - auto value_iter = - std::find_if(values_.begin(), values_.end(), - [&feature_idx](const std::pair& element) { - return element.first == feature_idx; - }); - - if (value_iter == values_.end()) { - return OptionalValue(); - } - return OptionalValue(value_iter->second); - } - - private: - std::vector> values_; -}; - -// Represents storage for a sparse float feature column. Can store values either -// for one dimensional or a multivalent (multidimensional) sparse column. -// Allows subscript operator access [feature_id]. -template -class SparseFloatFeatureColumn { - public: - void Reserve(const int32 size) { - if (!single_dimensional_) { - multidimensional_values.Reserve(size); - } - } - - void SetDimension(const int32 dimension) { - single_dimensional_ = dimension <= 1; - } - - void Add(const int32 feature_idx, const float value) { - if (single_dimensional_) { - DCHECK_EQ(0, feature_idx); - single_value_ = value; - } else { - multidimensional_values.Add(feature_idx, value); - } - initialized_ = true; - } - - void Clear() { - single_dimensional_ = false; - initialized_ = false; - multidimensional_values.Clear(); - } - - OptionalValue operator[](int feature_idx) const { - if (!initialized_) { - return OptionalValue(); - } - if (single_dimensional_) { - return OptionalValue(single_value_); - } else { - return multidimensional_values[feature_idx]; - } - } - - private: - bool single_dimensional_; - bool initialized_; - T single_value_; - SparseMultidimensionalValues multidimensional_values; -}; - -// Holds data for one example and enables lookup by feature column. -struct Example { - // Default constructor creates an empty example. - Example() : example_idx(-1) {} - - // Example index. - int64 example_idx; - - // Dense and sparse float features indexed by feature column. - // TODO(salehay): figure out a design to support multivalent float features. - std::vector dense_float_features; - - // Sparse float features columns (can be either single or multivalent - // (multidimensional). - std::vector> sparse_float_features; - - // Sparse integer features indexed by feature column. - // Note that all integer features are assumed to be categorical, i.e. will - // never be compared by order. Also these features can be multivalent. - // By default we allocate a InlinedVector of length 1 though since that is - // the most common case. - std::vector> sparse_int_features; -}; - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_EXAMPLE_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc deleted file mode 100644 index be9d63ee8ae..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/utils/example.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { -namespace { - -class ExampleTest : public ::testing::Test {}; - -TEST_F(ExampleTest, TestSparseMatrix) { - // Create the following matrix (FC is feature column): - // FC | f0 | f1 | f2 - // multidimensional - // 0 | | 0.4 | 0.3 - // 1 | 1 | | 2 - // 2 | 3 | 1 | 5 - // 3 | | | - // one dimensional columns - // 4 | -4 - // 5 | - std::vector> matrix; - matrix.resize(6); - matrix[0].SetDimension(3); - matrix[1].SetDimension(3); - matrix[2].SetDimension(3); - matrix[3].SetDimension(3); - matrix[4].SetDimension(1); - matrix[5].SetDimension(1); - - matrix[0].Add(1, 0.4f); - matrix[0].Add(2, 0.3f); - matrix[1].Add(0, 1.f); - matrix[1].Add(2, 2.f); - matrix[2].Add(0, 3.f); - matrix[2].Add(1, 1.f); - matrix[2].Add(2, 5.f); - matrix[4].Add(0, -4.f); - - // Row 0. - EXPECT_FALSE(matrix[0][0].has_value()); - EXPECT_TRUE(matrix[0][1].has_value()); - EXPECT_EQ(0.4f, matrix[0][1].get_value()); - EXPECT_TRUE(matrix[0][2].has_value()); - EXPECT_EQ(0.3f, matrix[0][2].get_value()); - - // Row 1. - EXPECT_TRUE(matrix[1][0].has_value()); - EXPECT_EQ(1.f, matrix[1][0].get_value()); - EXPECT_FALSE(matrix[1][1].has_value()); - EXPECT_TRUE(matrix[1][2].has_value()); - EXPECT_EQ(2.f, matrix[1][2].get_value()); - - // Row 2. - EXPECT_TRUE(matrix[2][0].has_value()); - EXPECT_EQ(3.f, matrix[2][0].get_value()); - EXPECT_TRUE(matrix[2][1].has_value()); - EXPECT_EQ(1.f, matrix[2][1].get_value()); - EXPECT_TRUE(matrix[2][2].has_value()); - EXPECT_EQ(5.f, matrix[2][2].get_value()); - - // Row 3. - EXPECT_FALSE(matrix[3][0].has_value()); - EXPECT_FALSE(matrix[3][1].has_value()); - EXPECT_FALSE(matrix[3][2].has_value()); - - // Row 4. - EXPECT_TRUE(matrix[4][0].has_value()); - EXPECT_EQ(-4.f, matrix[4][0].get_value()); - - // Row 5. - EXPECT_FALSE(matrix[5][0].has_value()); -} - -} // namespace -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc deleted file mode 100644 index e7e0b568c6f..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -using Iterator = ExamplesIterable::Iterator; - -ExamplesIterable::ExamplesIterable( - const std::vector& dense_float_feature_columns, - const std::vector& sparse_float_feature_columns, - const std::vector& sparse_int_feature_columns, - int64 example_start, int64 example_end) - : example_start_(example_start), example_end_(example_end) { - // Create dense float column values. - dense_float_column_values_.reserve(dense_float_feature_columns.size()); - for (auto& dense_float_column : dense_float_feature_columns) { - dense_float_column_values_.emplace_back( - dense_float_column.template matrix()); - } - - // Create sparse float column iterables and values. - sparse_float_column_iterables_.reserve(sparse_float_feature_columns.size()); - sparse_float_column_values_.reserve(sparse_float_feature_columns.size()); - sparse_float_dimensions_.reserve(sparse_float_feature_columns.size()); - for (auto& sparse_float_column : sparse_float_feature_columns) { - sparse_float_column_iterables_.emplace_back( - sparse_float_column.indices().template matrix(), example_start, - example_end); - sparse_float_column_values_.emplace_back( - sparse_float_column.values().template vec()); - sparse_float_dimensions_.push_back(sparse_float_column.shape()[1]); - } - - // Create sparse int column iterables and values. - sparse_int_column_iterables_.reserve(sparse_int_feature_columns.size()); - sparse_int_column_values_.reserve(sparse_int_feature_columns.size()); - for (auto& sparse_int_column : sparse_int_feature_columns) { - sparse_int_column_iterables_.emplace_back( - sparse_int_column.indices().template matrix(), example_start, - example_end); - sparse_int_column_values_.emplace_back( - sparse_int_column.values().template vec()); - } -} - -Iterator::Iterator(ExamplesIterable* iter, int64 example_idx) - : iter_(iter), example_idx_(example_idx) { - // Create sparse iterators. - sparse_float_column_iterators_.reserve( - iter->sparse_float_column_iterables_.size()); - for (auto& iterable : iter->sparse_float_column_iterables_) { - sparse_float_column_iterators_.emplace_back(iterable.begin()); - } - sparse_int_column_iterators_.reserve( - iter->sparse_int_column_iterables_.size()); - for (auto& iterable : iter->sparse_int_column_iterables_) { - sparse_int_column_iterators_.emplace_back(iterable.begin()); - } - - // Pre-size example features. - example_.dense_float_features.resize( - iter_->dense_float_column_values_.size()); - example_.sparse_int_features.resize(iter_->sparse_int_column_values_.size()); - example_.sparse_float_features.resize( - iter_->sparse_float_column_values_.size()); -} - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h deleted file mode 100644 index 3c5e0fbbb40..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_EXAMPLES_ITERABLE_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_EXAMPLES_ITERABLE_H_ - -#include - -#include "tensorflow/contrib/boosted_trees/lib/utils/example.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/util/sparse/sparse_tensor.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -// Enables row-wise iteration through examples from feature columns. -class ExamplesIterable { - public: - // Constructs an iterable given the desired examples slice and corresponding - // feature columns. - ExamplesIterable( - const std::vector& dense_float_feature_columns, - const std::vector& sparse_float_feature_columns, - const std::vector& sparse_int_feature_columns, - int64 example_start, int64 example_end); - - // Helper class to iterate through examples. - class Iterator { - public: - Iterator(ExamplesIterable* iter, int64 example_idx); - - Iterator& operator++() { - // Advance to next example. - ++example_idx_; - - // Update sparse column iterables. - for (auto& it : sparse_float_column_iterators_) { - ++it; - } - for (auto& it : sparse_int_column_iterators_) { - ++it; - } - return (*this); - } - - Iterator operator++(int) { - Iterator tmp(*this); - ++(*this); - return tmp; - } - - bool operator!=(const Iterator& other) const { - QCHECK_EQ(iter_, other.iter_); - return (example_idx_ != other.example_idx_); - } - - bool operator==(const Iterator& other) const { - QCHECK_EQ(iter_, other.iter_); - return (example_idx_ == other.example_idx_); - } - - const Example& operator*() { - // Set example index based on iterator. - example_.example_idx = example_idx_; - - // Get dense float values per column. - auto& dense_float_features = example_.dense_float_features; - for (size_t dense_float_idx = 0; - dense_float_idx < dense_float_features.size(); ++dense_float_idx) { - dense_float_features[dense_float_idx] = - iter_->dense_float_column_values_[dense_float_idx](example_idx_, 0); - } - - // Get sparse float values per column. - auto& sparse_float_features = example_.sparse_float_features; - // Iterate through each sparse float feature column. - for (size_t sparse_float_idx = 0; - sparse_float_idx < iter_->sparse_float_column_iterables_.size(); - ++sparse_float_idx) { - // Clear info from a previous instance. - sparse_float_features[sparse_float_idx].Clear(); - - // Get range for values tensor. - const auto& row_range = - (*sparse_float_column_iterators_[sparse_float_idx]); - DCHECK_EQ(example_idx_, row_range.example_idx); - - // If the example has this feature column. - if (row_range.start < row_range.end) { - const int32 dimension = - iter_->sparse_float_dimensions_[sparse_float_idx]; - sparse_float_features[sparse_float_idx].SetDimension(dimension); - if (dimension <= 1) { - // single dimensional sparse feature column. - DCHECK_EQ(1, row_range.end - row_range.start); - sparse_float_features[sparse_float_idx].Add( - 0, iter_->sparse_float_column_values_[sparse_float_idx]( - row_range.start)); - } else { - // Retrieve original indices tensor. - const TTypes::ConstMatrix& indices = - iter_->sparse_float_column_iterables_[sparse_float_idx] - .sparse_indices(); - - sparse_float_features[sparse_float_idx].Reserve(row_range.end - - row_range.start); - - // For each value. - for (int64 row_idx = row_range.start; row_idx < row_range.end; - ++row_idx) { - // Get the feature id for the feature column and the value. - const int32 feature_id = indices(row_idx, 1); - DCHECK_EQ(example_idx_, indices(row_idx, 0)); - - // Save the value to our sparse matrix. - sparse_float_features[sparse_float_idx].Add( - feature_id, - iter_->sparse_float_column_values_[sparse_float_idx]( - row_idx)); - } - } - } - } - - // Get sparse int values per column. - auto& sparse_int_features = example_.sparse_int_features; - for (size_t sparse_int_idx = 0; - sparse_int_idx < sparse_int_features.size(); ++sparse_int_idx) { - const auto& row_range = (*sparse_int_column_iterators_[sparse_int_idx]); - DCHECK_EQ(example_idx_, row_range.example_idx); - sparse_int_features[sparse_int_idx].clear(); - if (row_range.start < row_range.end) { - sparse_int_features[sparse_int_idx].reserve(row_range.end - - row_range.start); - for (int64 row_idx = row_range.start; row_idx < row_range.end; - ++row_idx) { - sparse_int_features[sparse_int_idx].push_back( - iter_->sparse_int_column_values_[sparse_int_idx](row_idx)); - } - } - } - - return example_; - } - - private: - // Examples iterable (not owned). - const ExamplesIterable* iter_; - - // Example index. - int64 example_idx_; - - // Sparse float column iterators. - std::vector sparse_float_column_iterators_; - - // Sparse int column iterators. - std::vector sparse_int_column_iterators_; - - // Example placeholder. - Example example_; - }; - - Iterator begin() { return Iterator(this, example_start_); } - Iterator end() { return Iterator(this, example_end_); } - - private: - // Example slice spec. - const int64 example_start_; - const int64 example_end_; - - // Dense float column values. - std::vector::ConstMatrix> dense_float_column_values_; - - // Sparse float column iterables. - std::vector sparse_float_column_iterables_; - - // Sparse float column values. - std::vector::ConstVec> sparse_float_column_values_; - - // Dimensions for sparse float feature columns. - std::vector sparse_float_dimensions_; - - // Sparse int column iterables. - std::vector sparse_int_column_iterables_; - - // Sparse int column values. - std::vector::ConstVec> sparse_int_column_values_; -}; - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_EXAMPLES_ITERABLE_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc deleted file mode 100644 index 2f4f2495eaf..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h" -#include "absl/algorithm/container.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { -namespace { - -class ExamplesIterableTest : public ::testing::Test {}; - -TEST_F(ExamplesIterableTest, Iterate) { - // Create a batch of 8 examples having one dense float, two sparse float and - // two sparse int features. Second sparse float feature is multivalent. - // The data looks like the following: - // Instance | DenseF1 | SparseF1 | SparseF2 | SparseI1 | SparseI2 | - // 0 | 7 | -3 | | 1 | 1, 8 | | - // 1 | -2 | | 4 | | 0 | 7 | - // 2 | 8 | 0 | | 3 | | 13 | - // 3 | 1 | 5 | 7 | | 2, 0 | 4 | - // 4 | 0 | 0 | | 4.3 | | 0 | - // 5 | -4 | | 9 | 0.8 | | | - // 6 | 7 | | | | | | - // 7 | -2 | | -4 | | 5 | | - auto dense_float_tensor = test::AsTensor( - {7.0f, -2.0f, 8.0f, 1.0f, 0.0f, -4.0f, 7.0f, -2.0f}, {8, 1}); - auto sparse_float_indices1 = - test::AsTensor({0, 0, 2, 0, 3, 0, 4, 0}, {4, 2}); - auto sparse_float_values1 = test::AsTensor({-3.0f, 0.0f, 5.0f, 0.0f}); - auto sparse_float_shape1 = TensorShape({8, 1}); - sparse::SparseTensor sparse_float_tensor1; - TF_ASSERT_OK( - sparse::SparseTensor::Create(sparse_float_indices1, sparse_float_values1, - sparse_float_shape1, &sparse_float_tensor1)); - auto sparse_float_indices2 = test::AsTensor( - {0, 1, 1, 0, 2, 1, 3, 0, 4, 1, 5, 0, 5, 1, 7, 0}, {8, 2}); - auto sparse_float_values2 = - test::AsTensor({1.f, 4.0f, 3.f, 7.0f, 4.3f, 9.0f, 0.8f, -4.0f}); - auto sparse_float_shape2 = TensorShape({8, 2}); - sparse::SparseTensor sparse_float_tensor2; - TF_ASSERT_OK( - sparse::SparseTensor::Create(sparse_float_indices2, sparse_float_values2, - sparse_float_shape2, &sparse_float_tensor2)); - auto sparse_int_indices1 = - test::AsTensor({0, 0, 0, 1, 1, 0, 3, 0, 3, 1, 7, 0}, {6, 2}); - auto sparse_int_values1 = test::AsTensor({1, 8, 0, 2, 0, 5}); - auto sparse_int_shape1 = TensorShape({8, 2}); - sparse::SparseTensor sparse_int_tensor1; - TF_ASSERT_OK( - sparse::SparseTensor::Create(sparse_int_indices1, sparse_int_values1, - sparse_int_shape1, &sparse_int_tensor1)); - auto sparse_int_indices2 = - test::AsTensor({1, 0, 2, 0, 3, 0, 4, 0}, {4, 2}); - auto sparse_int_values2 = test::AsTensor({7, 13, 4, 0}); - auto sparse_int_shape2 = TensorShape({8, 1}); - sparse::SparseTensor sparse_int_tensor2; - TF_ASSERT_OK( - sparse::SparseTensor::Create(sparse_int_indices2, sparse_int_values2, - sparse_int_shape2, &sparse_int_tensor2)); - - auto validate_example_features = [](int64 example_idx, - const Example& example) { - EXPECT_EQ(1, example.dense_float_features.size()); - - switch (example_idx) { - case 0: { - EXPECT_EQ(0, example.example_idx); - EXPECT_EQ(7.0f, example.dense_float_features[0]); - // SparseF1. - EXPECT_TRUE(example.sparse_float_features[0][0].has_value()); - EXPECT_EQ(-3.0f, example.sparse_float_features[0][0].get_value()); - // SparseF2 - multivalent. - EXPECT_FALSE(example.sparse_float_features[1][0].has_value()); - EXPECT_TRUE(example.sparse_float_features[1][1].has_value()); - EXPECT_EQ(1.0f, example.sparse_float_features[1][1].get_value()); - - EXPECT_EQ(2, example.sparse_int_features[0].size()); - EXPECT_EQ(1, absl::c_count(example.sparse_int_features[0], 1)); - EXPECT_EQ(1, absl::c_count(example.sparse_int_features[0], 8)); - EXPECT_EQ(0, example.sparse_int_features[1].size()); - } break; - case 1: { - EXPECT_EQ(1, example.example_idx); - EXPECT_EQ(-2.0f, example.dense_float_features[0]); - // SparseF1. - EXPECT_FALSE(example.sparse_float_features[0][0].has_value()); - // SparseF2. - EXPECT_TRUE(example.sparse_float_features[1][0].has_value()); - EXPECT_EQ(4.0f, example.sparse_float_features[1][0].get_value()); - EXPECT_FALSE(example.sparse_float_features[1][1].has_value()); - - EXPECT_EQ(1, example.sparse_int_features[0].size()); - EXPECT_EQ(1, absl::c_count(example.sparse_int_features[0], 0)); - EXPECT_EQ(1, example.sparse_int_features[1].size()); - EXPECT_EQ(1, absl::c_count(example.sparse_int_features[1], 7)); - } break; - case 2: { - EXPECT_EQ(2, example.example_idx); - EXPECT_EQ(8.0f, example.dense_float_features[0]); - // SparseF1. - EXPECT_TRUE(example.sparse_float_features[0][0].has_value()); - EXPECT_EQ(0.0f, example.sparse_float_features[0][0].get_value()); - // SparseF2. - EXPECT_FALSE(example.sparse_float_features[1][0].has_value()); - EXPECT_TRUE(example.sparse_float_features[1][1].has_value()); - EXPECT_EQ(3.f, example.sparse_float_features[1][1].get_value()); - - EXPECT_EQ(0, example.sparse_int_features[0].size()); - EXPECT_EQ(1, example.sparse_int_features[1].size()); - EXPECT_EQ(1, absl::c_count(example.sparse_int_features[1], 13)); - } break; - case 3: { - EXPECT_EQ(3, example.example_idx); - EXPECT_EQ(1.0f, example.dense_float_features[0]); - // SparseF1. - EXPECT_TRUE(example.sparse_float_features[0][0].has_value()); - EXPECT_EQ(5.0f, example.sparse_float_features[0][0].get_value()); - // SparseF2. - EXPECT_TRUE(example.sparse_float_features[1][0].has_value()); - EXPECT_EQ(7.0f, example.sparse_float_features[1][0].get_value()); - EXPECT_FALSE(example.sparse_float_features[1][1].has_value()); - - EXPECT_EQ(2, example.sparse_int_features[0].size()); - EXPECT_EQ(1, absl::c_count(example.sparse_int_features[0], 2)); - EXPECT_EQ(1, absl::c_count(example.sparse_int_features[0], 0)); - EXPECT_EQ(1, example.sparse_int_features[1].size()); - EXPECT_EQ(1, absl::c_count(example.sparse_int_features[1], 4)); - } break; - case 4: { - EXPECT_EQ(4, example.example_idx); - EXPECT_EQ(0.0f, example.dense_float_features[0]); - // SparseF1. - EXPECT_TRUE(example.sparse_float_features[0][0].has_value()); - EXPECT_EQ(0.0f, example.sparse_float_features[0][0].get_value()); - // SparseF2. - EXPECT_FALSE(example.sparse_float_features[1][0].has_value()); - EXPECT_TRUE(example.sparse_float_features[1][1].has_value()); - EXPECT_EQ(4.3f, example.sparse_float_features[1][1].get_value()); - - EXPECT_EQ(0, example.sparse_int_features[0].size()); - EXPECT_EQ(1, example.sparse_int_features[1].size()); - EXPECT_EQ(1, absl::c_count(example.sparse_int_features[1], 0)); - } break; - case 5: { - EXPECT_EQ(5, example.example_idx); - EXPECT_EQ(-4.0f, example.dense_float_features[0]); - // SparseF1. - EXPECT_FALSE(example.sparse_float_features[0][0].has_value()); - // SparseF2. - EXPECT_TRUE(example.sparse_float_features[1][0].has_value()); - EXPECT_EQ(9.0f, example.sparse_float_features[1][0].get_value()); - EXPECT_TRUE(example.sparse_float_features[1][1].has_value()); - EXPECT_EQ(0.8f, example.sparse_float_features[1][1].get_value()); - - EXPECT_EQ(0, example.sparse_int_features[0].size()); - } break; - case 6: { - EXPECT_EQ(6, example.example_idx); - EXPECT_EQ(7.0f, example.dense_float_features[0]); - // SparseF1. - EXPECT_FALSE(example.sparse_float_features[0][0].has_value()); - // SparseF2. - EXPECT_FALSE(example.sparse_float_features[1][0].has_value()); - EXPECT_FALSE(example.sparse_float_features[1][1].has_value()); - - EXPECT_EQ(0, example.sparse_int_features[0].size()); - } break; - case 7: { - EXPECT_EQ(7, example.example_idx); - EXPECT_EQ(-2.0f, example.dense_float_features[0]); - // SparseF1. - EXPECT_FALSE(example.sparse_float_features[0][0].has_value()); - // SparseF2. - EXPECT_TRUE(example.sparse_float_features[1][0].has_value()); - EXPECT_EQ(-4.0f, example.sparse_float_features[1][0].get_value()); - EXPECT_FALSE(example.sparse_float_features[1][1].has_value()); - - EXPECT_EQ(1, example.sparse_int_features[0].size()); - EXPECT_EQ(1, absl::c_count(example.sparse_int_features[0], 5)); - } break; - default: { LOG(QFATAL) << "Invalid example index."; } break; - } - }; - - // Iterate through all examples sequentially. - ExamplesIterable full_iterable( - {dense_float_tensor}, {sparse_float_tensor1, sparse_float_tensor2}, - {sparse_int_tensor1, sparse_int_tensor2}, 0, 8); - int64 example_idx = 0; - - for (const auto& example : full_iterable) { - validate_example_features(example_idx, example); - ++example_idx; - } - EXPECT_EQ(8, example_idx); - - // Iterate through slice (2, 6) of examples. - ExamplesIterable slice_iterable( - {dense_float_tensor}, {sparse_float_tensor1, sparse_float_tensor2}, - {sparse_int_tensor1, sparse_int_tensor2}, 2, 6); - example_idx = 2; - for (const auto& example : slice_iterable) { - validate_example_features(example_idx, example); - ++example_idx; - } - EXPECT_EQ(6, example_idx); -} - -} // namespace -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/macros.h b/tensorflow/contrib/boosted_trees/lib/utils/macros.h deleted file mode 100644 index 9a53fb2ef7d..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/macros.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_MACROS_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_MACROS_H_ - -#include "tensorflow/core/platform/macros.h" - -#define TF_CHECK_AND_RETURN_IF_ERROR(EXP, STATUS) \ - if (!TF_PREDICT_TRUE(EXP)) { \ - return (STATUS); \ - } - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_MACROS_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/utils/optional_value.h b/tensorflow/contrib/boosted_trees/lib/utils/optional_value.h deleted file mode 100644 index b2166f53d7a..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/optional_value.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_OPTIONAL_VALUE_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_OPTIONAL_VALUE_H_ - -#include "tensorflow/core/platform/logging.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -// Utility class holding an optional value. -template -class OptionalValue { - public: - OptionalValue() : value_(), has_value_(false) {} - explicit OptionalValue(T value) : value_(value), has_value_(true) {} - - bool has_value() const { return has_value_; } - const T& get_value() const { - QCHECK(has_value()); - return value_; - } - - private: - T value_; - bool has_value_; -}; - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_OPTIONAL_VALUE_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/utils/parallel_for.cc b/tensorflow/contrib/boosted_trees/lib/utils/parallel_for.cc deleted file mode 100644 index b00d80b522d..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/parallel_for.cc +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/utils/parallel_for.h" -#include "tensorflow/core/lib/core/blocking_counter.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -void ParallelFor(int64 batch_size, int64 desired_parallelism, - thread::ThreadPool* thread_pool, - std::function do_work) { - // Parallelize work over the batch. - if (desired_parallelism <= 0) { - do_work(0, batch_size); - return; - } - const int num_shards = std::max( - 1, std::min(static_cast(desired_parallelism), batch_size)); - const int64 block_size = (batch_size + num_shards - 1) / num_shards; - CHECK_GT(block_size, 0); - const int num_shards_used = (batch_size + block_size - 1) / block_size; - BlockingCounter counter(num_shards_used - 1); - for (int64 start = block_size; start < batch_size; start += block_size) { - auto end = std::min(start + block_size, batch_size); - thread_pool->Schedule([&do_work, &counter, start, end]() { - do_work(start, end); - counter.DecrementCount(); - }); - } - - // Execute first shard on main thread. - do_work(0, std::min(block_size, batch_size)); - counter.Wait(); -} - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/parallel_for.h b/tensorflow/contrib/boosted_trees/lib/utils/parallel_for.h deleted file mode 100644 index 1f3672bf859..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/parallel_for.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_PARALLEL_FOR_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_PARALLEL_FOR_H_ - -#include "tensorflow/core/lib/core/threadpool.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -// Executes a parallel for over the batch for the desired parallelism level. -void ParallelFor(int64 batch_size, int64 desired_parallelism, - thread::ThreadPool* thread_pool, - std::function do_work); - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_PARALLEL_FOR_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/utils/random.h b/tensorflow/contrib/boosted_trees/lib/utils/random.h deleted file mode 100644 index f0eaef24cbb..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/random.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_RANDOM_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_RANDOM_H_ - -#include - -#include "tensorflow/core/lib/random/simple_philox.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -// Generates a poisson distributed number with mean 1 for use in bootstrapping. -inline int32 PoissonBootstrap(random::SimplePhilox* rng) { - // Knuth, special cased for lambda = 1.0 for efficiency. - static const float lbound = std::exp(-1.0f); - int32 n = 0; - for (float r = 1; r > lbound; r *= rng->RandFloat()) { - ++n; - } - return n - 1; -} - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_RANDOM_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/utils/random_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/random_test.cc deleted file mode 100644 index 51162f410e8..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/random_test.cc +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/lib/utils/random.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { -namespace { - -TEST(RandomTest, Poisson) { - random::PhiloxRandom philox(77L); - random::SimplePhilox rng(&philox); - for (int trial = 0; trial < 10; ++trial) { - const int32 num_bootstrap = 10000; - double sum = 0; - double zeros = 0; - double ones = 0; - for (int i = 0; i < num_bootstrap; ++i) { - auto n = PoissonBootstrap(&rng); - sum += n; - zeros += (n == 0) ? 1 : 0; - ones += (n == 1) ? 1 : 0; - } - - // Ensure mean is near expected value. - const double expected_mean = 1.0; // lambda - const double mean_std_error = 1.0 / sqrt(num_bootstrap); - double mean = sum / num_bootstrap; - EXPECT_NEAR(mean, expected_mean, 3 * mean_std_error); - - // Ensure probability mass for values 0 and 1 are near expected value. - const double expected_p = 0.368; - const double proportion_std_error = - sqrt(expected_p * (1 - expected_p) / num_bootstrap); - EXPECT_NEAR(zeros / num_bootstrap, expected_p, 3 * proportion_std_error); - EXPECT_NEAR(ones / num_bootstrap, expected_p, 3 * proportion_std_error); - } -} - -} // namespace -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.cc b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.cc deleted file mode 100644 index 1297aa88493..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.cc +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include "tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -using ExampleRowRange = SparseColumnIterable::ExampleRowRange; -using Iterator = SparseColumnIterable::Iterator; - -namespace { - -// Iterator over indices matrix rows. -class IndicesRowIterator - : public std::iterator { - public: - IndicesRowIterator() : iter_(nullptr), row_idx_(-1) {} - IndicesRowIterator(SparseColumnIterable* iter, int row_idx) - : iter_(iter), row_idx_(row_idx) {} - IndicesRowIterator(const IndicesRowIterator& other) - : iter_(other.iter_), row_idx_(other.row_idx_) {} - - IndicesRowIterator& operator=(const IndicesRowIterator& other) { - iter_ = other.iter_; - row_idx_ = other.row_idx_; - return (*this); - } - - IndicesRowIterator& operator++() { - ++row_idx_; - return (*this); - } - - IndicesRowIterator operator++(int) { - IndicesRowIterator tmp(*this); - ++row_idx_; - return tmp; - } - - reference operator*() const { return iter_->ix()(row_idx_, 0); } - - pointer operator->() { return &iter_->ix()(row_idx_, 0); } - - IndicesRowIterator& operator--() { - --row_idx_; - return (*this); - } - - IndicesRowIterator operator--(int) { - IndicesRowIterator tmp(*this); - --row_idx_; - return tmp; - } - - IndicesRowIterator& operator+=(const difference_type& step) { - row_idx_ += step; - return (*this); - } - IndicesRowIterator& operator-=(const difference_type& step) { - row_idx_ -= step; - return (*this); - } - - IndicesRowIterator operator+(const difference_type& step) const { - IndicesRowIterator tmp(*this); - tmp += step; - return tmp; - } - - IndicesRowIterator operator-(const difference_type& step) const { - IndicesRowIterator tmp(*this); - tmp -= step; - return tmp; - } - - difference_type operator-(const IndicesRowIterator& other) const { - return row_idx_ - other.row_idx_; - } - - bool operator!=(const IndicesRowIterator& other) const { - QCHECK_EQ(iter_, other.iter_); - return (row_idx_ != other.row_idx_); - } - - bool operator<(const IndicesRowIterator& other) const { - return (row_idx_ < other.row_idx_); - } - - bool operator==(const IndicesRowIterator& other) const { - QCHECK_EQ(iter_, other.iter_); - return (row_idx_ == other.row_idx_); - } - - Eigen::Index row_idx() const { return row_idx_; } - - private: - SparseColumnIterable* iter_; - Eigen::Index row_idx_; -}; -} // namespace - -Iterator::Iterator(SparseColumnIterable* iter, int64 example_idx) - : iter_(iter), example_idx_(example_idx), end_(iter->ix_.dimension(0)) { - cur_ = next_ = std::lower_bound(IndicesRowIterator(iter, 0), - IndicesRowIterator(iter, end_), example_idx_) - .row_idx(); - UpdateNext(); -} - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h deleted file mode 100644 index 87fb1fbf5ae..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_SPARSE_COLUMN_ITERABLE_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_SPARSE_COLUMN_ITERABLE_H_ - -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -// Enables row-wise iteration through examples on sparse feature columns. -class SparseColumnIterable { - public: - // Indicates a contiguous range for an example: [start, end). - struct ExampleRowRange { - int64 example_idx; - int64 start; - int64 end; - }; - - // Helper class to iterate through examples and return the corresponding - // indices row range. Note that the row range can be empty in case a given - // example has no corresponding indices. - // An Iterator can be initialized from any example start offset, the - // corresponding range indicators will be initialized in log time. - class Iterator { - public: - Iterator(SparseColumnIterable* iter, int64 example_idx); - - Iterator& operator++() { - ++example_idx_; - if (cur_ < end_ && iter_->ix()(cur_, 0) < example_idx_) { - cur_ = next_; - UpdateNext(); - } - return (*this); - } - - Iterator operator++(int) { - Iterator tmp(*this); - ++(*this); - return tmp; - } - - bool operator!=(const Iterator& other) const { - QCHECK_EQ(iter_, other.iter_); - return (example_idx_ != other.example_idx_); - } - - bool operator==(const Iterator& other) const { - QCHECK_EQ(iter_, other.iter_); - return (example_idx_ == other.example_idx_); - } - - const ExampleRowRange& operator*() { - range_.example_idx = example_idx_; - if (cur_ < end_ && iter_->ix()(cur_, 0) == example_idx_) { - range_.start = cur_; - range_.end = next_; - } else { - range_.start = 0; - range_.end = 0; - } - return range_; - } - - private: - void UpdateNext() { - next_ = std::min(next_ + 1, end_); - while (next_ < end_ && iter_->ix()(cur_, 0) == iter_->ix()(next_, 0)) { - ++next_; - } - } - - const SparseColumnIterable* iter_; - int64 example_idx_; - int64 cur_; - int64 next_; - const int64 end_; - ExampleRowRange range_; - }; - - // Constructs an iterable given the desired examples slice and corresponding - // feature columns. - SparseColumnIterable(TTypes::ConstMatrix ix, int64 example_start, - int64 example_end) - : ix_(ix), example_start_(example_start), example_end_(example_end) { - QCHECK(example_start >= 0 && example_end >= 0); - } - - Iterator begin() { return Iterator(this, example_start_); } - Iterator end() { return Iterator(this, example_end_); } - - const TTypes::ConstMatrix& ix() const { return ix_; } - int64 example_start() const { return example_start_; } - int64 example_end() const { return example_end_; } - - const TTypes::ConstMatrix& sparse_indices() const { return ix_; } - - private: - // Sparse indices matrix. - TTypes::ConstMatrix ix_; - - // Example slice spec. - const int64 example_start_; - const int64 example_end_; -}; - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_SPARSE_COLUMN_ITERABLE_H_ diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc deleted file mode 100644 index cc7604745e6..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include "tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { -namespace { - -using test::AsTensor; -using ExampleRowRange = SparseColumnIterable::ExampleRowRange; - -class SparseColumnIterableTest : public ::testing::Test {}; - -TEST_F(SparseColumnIterableTest, Empty) { - const auto indices = Tensor(DT_INT64, {0, 2}); - SparseColumnIterable iterable(indices.template matrix(), 0, 0); - EXPECT_EQ(iterable.begin(), iterable.end()); -} - -TEST_F(SparseColumnIterableTest, Iterate) { - // 8 examples having 7 sparse features with the 3rd and 7th multivalent. - // This can be visualized like the following: - // Instance | Sparse | - // 0 | x | - // 1 | | - // 2 | | - // 3 | xxx | - // 4 | x | - // 5 | | - // 6 | | - // 7 | x x | - const auto indices = - AsTensor({0, 0, 3, 0, 3, 1, 3, 2, 4, 0, 7, 0, 7, 2}, {7, 2}); - - auto validate_example_range = [](const ExampleRowRange& range) { - switch (range.example_idx) { - case 0: { - EXPECT_EQ(0, range.start); - EXPECT_EQ(1, range.end); - } break; - case 3: { - EXPECT_EQ(1, range.start); - EXPECT_EQ(4, range.end); - } break; - case 4: { - EXPECT_EQ(4, range.start); - EXPECT_EQ(5, range.end); - } break; - case 7: { - EXPECT_EQ(5, range.start); - EXPECT_EQ(7, range.end); - } break; - default: { - // Empty examples. - EXPECT_GE(range.start, range.end); - } break; - } - }; - - // Iterate through all examples sequentially. - SparseColumnIterable full_iterable(indices.template matrix(), 0, 8); - int64 expected_example_idx = 0; - for (const ExampleRowRange& range : full_iterable) { - EXPECT_EQ(expected_example_idx, range.example_idx); - validate_example_range(range); - ++expected_example_idx; - } - EXPECT_EQ(8, expected_example_idx); - - // Iterate through slice (2, 6) of examples. - SparseColumnIterable slice_iterable(indices.template matrix(), 2, 6); - expected_example_idx = 2; - for (const ExampleRowRange& range : slice_iterable) { - EXPECT_EQ(expected_example_idx, range.example_idx); - validate_example_range(range); - ++expected_example_idx; - } - EXPECT_EQ(6, expected_example_idx); -} - -} // namespace -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.cc b/tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.cc deleted file mode 100644 index 326e3943df7..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.cc +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include "tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.h" -#include "tensorflow/contrib/boosted_trees/lib/utils/macros.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -std::vector TensorUtils::OpInputListToTensorVec( - const OpInputList& input_list) { - std::vector tensor_vec; - tensor_vec.reserve(input_list.size()); - for (const Tensor& tensor : input_list) { - tensor_vec.emplace_back(tensor); - } - return tensor_vec; -} - -Status TensorUtils::ReadDenseFloatFeatures(OpKernelContext* const context, - OpInputList* features_list) { - // Constants. - constexpr auto kDenseFloatFeaturesName = "dense_float_features"; - - // Read dense float features list; - TF_RETURN_IF_ERROR( - context->input_list(kDenseFloatFeaturesName, features_list)); - return Status::OK(); -} - -Status TensorUtils::ReadSparseFloatFeatures(OpKernelContext* const context, - OpInputList* features_indices_list, - OpInputList* feature_values_list, - OpInputList* feature_shapes_list) { - // Constants. - constexpr auto kSparseFloatFeatureIndicesName = - "sparse_float_feature_indices"; - constexpr auto kSparseFloatFeatureValuesName = "sparse_float_feature_values"; - constexpr auto kSparseFloatFeatureShapesName = "sparse_float_feature_shapes"; - - // Read sparse float features list; - TF_RETURN_IF_ERROR(context->input_list(kSparseFloatFeatureIndicesName, - features_indices_list)); - TF_RETURN_IF_ERROR( - context->input_list(kSparseFloatFeatureValuesName, feature_values_list)); - TF_RETURN_IF_ERROR( - context->input_list(kSparseFloatFeatureShapesName, feature_shapes_list)); - return Status::OK(); -} - -Status TensorUtils::ReadSparseIntFeatures(OpKernelContext* const context, - OpInputList* features_indices_list, - OpInputList* feature_values_list, - OpInputList* feature_shapes_list) { - // Constants. - constexpr auto kSparseIntFeatureIndicesName = "sparse_int_feature_indices"; - constexpr auto kSparseIntFeatureValuesName = "sparse_int_feature_values"; - constexpr auto kSparseIntFeatureShapesName = "sparse_int_feature_shapes"; - - // Read sparse int features list; - TF_RETURN_IF_ERROR( - context->input_list(kSparseIntFeatureIndicesName, features_indices_list)); - TF_RETURN_IF_ERROR( - context->input_list(kSparseIntFeatureValuesName, feature_values_list)); - TF_RETURN_IF_ERROR( - context->input_list(kSparseIntFeatureShapesName, feature_shapes_list)); - return Status::OK(); -} - -int64 TensorUtils::InferBatchSize( - const OpInputList& dense_float_features_list, - const OpInputList& sparse_float_feature_shapes_list, - const OpInputList& sparse_int_feature_shapes_list) { - if (dense_float_features_list.size() > 0) { - return dense_float_features_list[0].dim_size(0); - } - if (sparse_float_feature_shapes_list.size() > 0) { - return sparse_float_feature_shapes_list[0].flat()(0); - } - if (sparse_int_feature_shapes_list.size() > 0) { - return sparse_int_feature_shapes_list[0].flat()(0); - } - LOG(QFATAL) << "Could not infer batch size due to empty feature set."; -} - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.h b/tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.h deleted file mode 100644 index 475d3718ecc..00000000000 --- a/tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.h +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_TENSOR_UTILS_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_TENSOR_UTILS_H_ - -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_types.h" - -namespace tensorflow { -namespace boosted_trees { -namespace utils { - -class TensorUtils { - public: - // Read an input list into a vector of tensors. - static std::vector OpInputListToTensorVec( - const OpInputList& input_list); - - // Reads the dense float features input list. - static Status ReadDenseFloatFeatures(OpKernelContext* const context, - OpInputList* features_list); - - // Reads the sparse float features input list. - static Status ReadSparseFloatFeatures(OpKernelContext* const context, - OpInputList* features_indices_list, - OpInputList* feature_values_list, - OpInputList* feature_shapes_list); - - // Reads the sparse int features input list. - static Status ReadSparseIntFeatures(OpKernelContext* const context, - OpInputList* features_indices_list, - OpInputList* feature_values_list, - OpInputList* feature_shapes_list); - - // Infers the batch size by looking at the op input features. - static int64 InferBatchSize( - const OpInputList& dense_float_features_list, - const OpInputList& sparse_float_feature_shapes_list, - const OpInputList& sparse_int_feature_shapes_list); -}; - -} // namespace utils -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_TENSOR_UTILS_H_ diff --git a/tensorflow/contrib/boosted_trees/ops/model_ops.cc b/tensorflow/contrib/boosted_trees/ops/model_ops.cc deleted file mode 100644 index 9d6343c7e80..00000000000 --- a/tensorflow/contrib/boosted_trees/ops/model_ops.cc +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { -namespace boosted_trees { - -REGISTER_RESOURCE_HANDLE_OP(DecisionTreeEnsembleResource); - -REGISTER_OP("TreeEnsembleIsInitializedOp") - .Input("tree_ensemble_handle: resource") - .Output("is_initialized: bool") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - c->set_output(0, c->Scalar()); - return Status::OK(); - }) - .Doc(R"doc( -Checks whether a tree ensemble has been initialized. -)doc"); - -REGISTER_OP("CreateTreeEnsembleVariable") - .Input("tree_ensemble_handle: resource") - .Input("stamp_token: int64") - .Input("tree_ensemble_config: string") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused_input)); - return Status::OK(); - }) - .Doc(R"doc( -Creates a tree ensemble model and returns a handle to it. - -tree_ensemble_handle: Handle to the tree ensemble resource to be created. -stamp_token: Token to use as the initial value of the resource stamp. -tree_ensemble_config: Serialized proto of the tree ensemble. -)doc"); - -REGISTER_OP("TreeEnsembleStampToken") - .Input("tree_ensemble_handle: resource") - .Output("stamp_token: int64") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - c->set_output(0, c->Scalar()); - return Status::OK(); - }) - .Doc(R"doc( -Retrieves the tree ensemble resource stamp token. - -tree_ensemble_handle: Handle to the tree ensemble. -stamp_token: Stamp token of the tree ensemble resource. -)doc"); - -REGISTER_OP("TreeEnsembleSerialize") - .Input("tree_ensemble_handle: resource") - .Output("stamp_token: int64") - .Output("tree_ensemble_config: string") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - c->set_output(0, c->Scalar()); - c->set_output(1, c->Scalar()); - return Status::OK(); - }) - .Doc(R"doc( -Serializes the tree ensemble to a proto. - -tree_ensemble_handle: Handle to the tree ensemble. -stamp_token: Stamp token of the tree ensemble resource. -tree_ensemble_config: Serialized proto of the ensemble. -)doc"); - -REGISTER_OP("TreeEnsembleDeserialize") - .Input("tree_ensemble_handle: resource") - .Input("stamp_token: int64") - .Input("tree_ensemble_config: string") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused_input)); - return Status::OK(); - }) - .Doc(R"doc( -Deserializes a serialized tree ensemble config and replaces current tree -ensemble. - -tree_ensemble_handle: Handle to the tree ensemble. -stamp_token: Token to use as the new value of the resource stamp. -tree_ensemble_config: Serialized proto of the ensemble. -)doc"); - -REGISTER_OP("TreeEnsembleUsedHandlers") - .Attr("num_all_handlers: int >= 0") - .Input("tree_ensemble_handle: resource") - .Input("stamp_token: int64") - .Output("num_used_handlers: int64") - .Output("used_handlers_mask: bool") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - c->set_output(0, c->Scalar()); - int num_all_handlers; - c->GetAttr("num_all_handlers", &num_all_handlers).IgnoreError(); - c->set_output(1, {c->Vector(num_all_handlers)}); - - return Status::OK(); - }) - .Doc(R"doc( -Returns the mask of used handlers along with the number of non-zero elements in -this mask. Used in feature selection. - -tree_ensemble_handle: Handle to the tree ensemble. -stamp_token: Token to use as the new value of the resource stamp. -num_used_handlers: number of feature column handlers used in the model. -used_handlers_mask: A boolean vector of showing which handlers are used in the - model. -)doc"); - -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/ops/prediction_ops.cc b/tensorflow/contrib/boosted_trees/ops/prediction_ops.cc deleted file mode 100644 index 6491d587943..00000000000 --- a/tensorflow/contrib/boosted_trees/ops/prediction_ops.cc +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/boosted_trees/proto/learner.pb.h" -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -using tensorflow::boosted_trees::learner::LearnerConfig; - -namespace tensorflow { - -using shape_inference::InferenceContext; - -static Status ApplyGradientTreesPredictionShapeFn(InferenceContext* c) { - string learner_config_str; - // TODO(b/32704451): Don't just ignore the ::tensorflow::Status object! - c->GetAttr("learner_config", &learner_config_str).IgnoreError(); - LearnerConfig learner_config; - ParseProtoUnlimited(&learner_config, learner_config_str); - - bool reduce_dim; - c->GetAttr("reduce_dim", &reduce_dim).IgnoreError(); - // Sets the shape of the output as a matrix. - c->set_output(0, {c->Matrix(InferenceContext::kUnknownDim, - reduce_dim ? learner_config.num_classes() - 1 - : learner_config.num_classes())}); - c->set_output(1, {c->UnknownShape()}); - return Status::OK(); -} - -static Status ApplyGradientTreesPredictionVerboseShapeFn(InferenceContext* c) { - string learner_config_str; - c->GetAttr("learner_config", &learner_config_str).IgnoreError(); - LearnerConfig learner_config; - ParseProtoUnlimited(&learner_config, learner_config_str); - - bool reduce_dim; - c->GetAttr("reduce_dim", &reduce_dim).IgnoreError(); - // Sets the shape of the output as a matrix. - c->set_output(0, {c->Matrix(InferenceContext::kUnknownDim, - reduce_dim ? learner_config.num_classes() - 1 - : learner_config.num_classes())}); - c->set_output(1, {c->UnknownShape()}); - c->set_output(2, {c->Matrix(InferenceContext::kUnknownDim, - InferenceContext::kUnknownDim)}); - return Status::OK(); -} - -REGISTER_OP("GradientTreesPrediction") - .Attr("learner_config: string") - .Attr("num_dense_float_features: int >= 0") - .Attr("num_sparse_float_features: int >= 0") - .Attr("num_sparse_int_features: int >= 0") - .Attr("use_locking: bool = false") - .Attr("apply_dropout: bool") - .Attr("apply_averaging: bool") - .Attr("center_bias: bool") - .Attr("reduce_dim: bool") - .Input("tree_ensemble_handle: resource") - .Input("seed: int64") - .Input("dense_float_features: num_dense_float_features * float") - .Input("sparse_float_feature_indices: num_sparse_float_features * int64") - .Input("sparse_float_feature_values: num_sparse_float_features * float") - .Input("sparse_float_feature_shapes: num_sparse_float_features * int64") - .Input("sparse_int_feature_indices: num_sparse_int_features * int64") - .Input("sparse_int_feature_values: num_sparse_int_features * int64") - .Input("sparse_int_feature_shapes: num_sparse_int_features * int64") - .Output("predictions: float") - .Output("drop_out_tree_indices_weights: float") - .SetShapeFn(ApplyGradientTreesPredictionShapeFn) - .Doc(R"doc( -Runs multiple additive regression forests predictors on input instances -and computes the final prediction for each class. - -learner_config: Config for the learner of type LearnerConfig proto. Prediction -ops for now uses only LearningRateDropoutDrivenConfig config from the learner. -num_dense_float_features: Number of dense float features. -num_sparse_float_features: Number of sparse float features. -num_sparse_int_features: Number of sparse int features. -use_locking: Whether to use locking. -seed: random seed to be used for dropout. -reduce_dim: whether to reduce the dimension (legacy impl) or not. -apply_dropout: whether to apply dropout during prediction. -apply_averaging: whether averaging of tree ensembles should take place. If set -to true, will be based on AveragingConfig from learner_config. -tree_ensemble_handle: The handle to the tree ensemble. -dense_float_features: Rank 2 Tensors containing dense float feature values. -sparse_float_feature_indices: Rank 2 Tensors containing sparse float indices. -sparse_float_feature_values: Rank 1 Tensors containing sparse float values. -sparse_float_feature_shapes: Rank 1 Tensors containing sparse float shapes. -sparse_int_feature_indices: Rank 2 Tensors containing sparse int indices. -sparse_int_feature_values: Rank 1 Tensors containing sparse int values. -sparse_int_feature_shapes: Rank 1 Tensors containing sparse int shapes. -predictions: Rank 2 Tensor containing predictions per example per class. -drop_out_tree_indices_weights: Tensor of Rank 2 containing dropped trees indices -and original weights of those trees during prediction. -)doc"); - -REGISTER_OP("GradientTreesPredictionVerbose") - .Attr("learner_config: string") - .Attr("num_dense_float_features: int >= 0") - .Attr("num_sparse_float_features: int >= 0") - .Attr("num_sparse_int_features: int >= 0") - .Attr("use_locking: bool = false") - .Attr("apply_dropout: bool") - .Attr("apply_averaging: bool") - .Attr("center_bias: bool") - .Attr("reduce_dim: bool") - .Input("tree_ensemble_handle: resource") - .Input("seed: int64") - .Input("dense_float_features: num_dense_float_features * float") - .Input("sparse_float_feature_indices: num_sparse_float_features * int64") - .Input("sparse_float_feature_values: num_sparse_float_features * float") - .Input("sparse_float_feature_shapes: num_sparse_float_features * int64") - .Input("sparse_int_feature_indices: num_sparse_int_features * int64") - .Input("sparse_int_feature_values: num_sparse_int_features * int64") - .Input("sparse_int_feature_shapes: num_sparse_int_features * int64") - .Output("predictions: float") - .Output("drop_out_tree_indices_weights: float") - .Output("leaf_index: int32") - .SetShapeFn(ApplyGradientTreesPredictionVerboseShapeFn) - .Doc(R"doc( -Runs multiple additive regression forests predictors on input instances -and computes the final prediction for each class, and outputs a matrix of -leaf ids per each tree in an ensemble. - -learner_config: Config for the learner of type LearnerConfig proto. Prediction -ops for now uses only LearningRateDropoutDrivenConfig config from the learner. -num_dense_float_features: Number of dense float features. -num_sparse_float_features: Number of sparse float features. -num_sparse_int_features: Number of sparse int features. -use_locking: Whether to use locking. -seed: random seed to be used for dropout. -reduce_dim: whether to reduce the dimension (legacy impl) or not. -apply_dropout: whether to apply dropout during prediction. -apply_averaging: whether averaging of tree ensembles should take place. If set -to true, will be based on AveragingConfig from learner_config. -tree_ensemble_handle: The handle to the tree ensemble. -dense_float_features: Rank 2 Tensors containing dense float feature values. -sparse_float_feature_indices: Rank 2 Tensors containing sparse float indices. -sparse_float_feature_values: Rank 1 Tensors containing sparse float values. -sparse_float_feature_shapes: Rank 1 Tensors containing sparse float shapes. -sparse_int_feature_indices: Rank 2 Tensors containing sparse int indices. -sparse_int_feature_values: Rank 1 Tensors containing sparse int values. -sparse_int_feature_shapes: Rank 1 Tensors containing sparse int shapes. -predictions: Rank 2 Tensor containing predictions per example per class. -drop_out_tree_indices_weights: Tensor of Rank 2 containing dropped trees indices -leaf_index: tensor of rank 2 containing leaf ids for each tree where an instance ended up. -)doc"); - -REGISTER_OP("GradientTreesPartitionExamples") - .Attr("num_dense_float_features: int >= 0") - .Attr("num_sparse_float_features: int >= 0") - .Attr("num_sparse_int_features: int >= 0") - .Attr("use_locking: bool = false") - .Input("tree_ensemble_handle: resource") - .Input("dense_float_features: num_dense_float_features * float") - .Input("sparse_float_feature_indices: num_sparse_float_features * int64") - .Input("sparse_float_feature_values: num_sparse_float_features * float") - .Input("sparse_float_feature_shapes: num_sparse_float_features * int64") - .Input("sparse_int_feature_indices: num_sparse_int_features * int64") - .Input("sparse_int_feature_values: num_sparse_int_features * int64") - .Input("sparse_int_feature_shapes: num_sparse_int_features * int64") - .Output("partition_ids: int32") - .SetShapeFn([](InferenceContext* c) { - return c->set_output("partition_ids", - {c->Vector(InferenceContext::kUnknownDim)}); - }) - .Doc(R"doc( -Splits input examples into the leaves of the tree. - -num_dense_float_features: Number of dense float features. -num_sparse_float_features: Number of sparse float features. -num_sparse_int_features: Number of sparse int features. -use_locking: Whether to use locking. -tree_ensemble_handle: The handle to the tree ensemble. -dense_float_features: Rank 2 Tensors containing dense float feature values. -sparse_float_feature_indices: Rank 2 Tensors containing sparse float indices. -sparse_float_feature_values: Rank 1 Tensors containing sparse float values. -sparse_float_feature_shapes: Rank 1 Tensors containing sparse float shapes. -sparse_int_feature_indices: Rank 2 Tensors containing sparse int indices. -sparse_int_feature_values: Rank 1 Tensors containing sparse int values. -sparse_int_feature_shapes: Rank 1 Tensors containing sparse int shapes. -partition_ids: Rank 1 Tensor containing partition ids per example. -)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/ops/quantile_ops.cc b/tensorflow/contrib/boosted_trees/ops/quantile_ops.cc deleted file mode 100644 index 6aa52463987..00000000000 --- a/tensorflow/contrib/boosted_trees/ops/quantile_ops.cc +++ /dev/null @@ -1,337 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { -namespace boosted_trees { -using shape_inference::DimensionHandle; -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -REGISTER_RESOURCE_HANDLE_OP(QuantileStreamResource); - -REGISTER_OP("QuantileAccumulatorIsInitialized") - .Input("quantile_accumulator_handle: resource") - .Output("is_initialized: bool") - .SetShapeFn(shape_inference::ScalarShape) - .Doc(R"doc( -Checks whether a quantile accumulator has been initialized. -)doc"); - -REGISTER_OP("CreateQuantileAccumulator") - .Attr("container: string = ''") - .Attr("shared_name: string = ''") - .Attr("max_elements: int = 1099511627776") // 1 << 40 - .Attr("epsilon: float") - .Attr("num_quantiles: int") - .Attr("generate_quantiles: bool=False") - .Input("quantile_accumulator_handle: resource") - .Input("stamp_token: int64") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input)); - return Status::OK(); - }) - .Doc(R"doc( -Creates a stateful accumulator for quantile summaries. - -epsilon: Error bound on the quantile summary. -num_quantiles: Number of buckets that we create from the data. -stamp_token: Token to use as the initial value of the resource stamp. -quantile_accumulator_handle: The handle to the accumulator. -)doc"); - -REGISTER_OP("QuantileAccumulatorAddSummaries") - .Attr("num_resource_handles: int >= 1") - .Input("quantile_accumulator_handles: num_resource_handles * resource") - .Input("stamp_token: int64") - .Input("summaries: num_resource_handles * string") - .SetShapeFn([](InferenceContext* c) { - int num_resource_handles; - TF_RETURN_IF_ERROR( - c->GetAttr("num_resource_handles", &num_resource_handles)); - // All the inputs are scalars. - shape_inference::ShapeHandle unused_input; - for (int i = 0; i < 2 * num_resource_handles + 1; ++i) { - TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 0, &unused_input)); - } - return Status::OK(); - }) - .Doc(R"doc( -Adds each quantile summary to its stream. - -quantile_accumulator_handles: The handles to the quantile stream resources. -stamp_token: Stamp token to validate the Read/Write operation. -summaries: A list of serialized QuantileSummaryState. -)doc"); - -REGISTER_OP("QuantileAccumulatorGetBuckets") - .Attr("num_resource_handles: int >= 1") - .Input("quantile_accumulator_handles: num_resource_handles * resource") - .Input("stamp_token: int64") - .Output("are_buckets_ready: num_resource_handles * bool") - .Output("buckets: num_resource_handles * float") - .SetShapeFn([](InferenceContext* c) { - int num_resource_handles; - TF_RETURN_IF_ERROR( - c->GetAttr("num_resource_handles", &num_resource_handles)); - for (int i = 0; i < num_resource_handles; ++i) { - c->set_output(i, c->Scalar()); - c->set_output(i + num_resource_handles, c->Vector(c->UnknownDim())); - } - return Status::OK(); - }) - - .Doc(R"doc( -Returns quantile buckets created during previous flush of the accumulator. - -quantile_accumulator_handles: The handles to the quantile stream resources. -stamp_token: Stamp token to validate the Read/Write operation. -are_buckets_ready: Whether the buckets are ready or not. -buckets: Output quantile summary representing boundaries with "num_quantile" - elements. -)doc"); - -REGISTER_OP("QuantileAccumulatorFlush") - .Input("quantile_accumulator_handle: resource") - .Input("stamp_token: int64") - .Input("next_stamp_token: int64") - .Doc(R"doc( -Resets quantile summary streams for each column with a new token. - -quantile_accumulator_handle: The handle to the accumulator. -stamp_token: Stamp token for Read/Write operations. - Any operation with a mismatching token will be dropped. -next_stamp_token: Stamp token to be used for the next iteration. -)doc"); - -REGISTER_OP("QuantileAccumulatorFlushSummary") - .Input("quantile_accumulator_handle: resource") - .Input("stamp_token: int64") - .Input("next_stamp_token: int64") - .Output("output: string") - .Doc(R"doc( -Resets quantile summary stream and returns the summary. - -quantile_accumulator_handle: The handle to the accumulator. -stamp_token: Stamp token for Read/Write operations. - Any operation with a mismatching token will be dropped. -next_stamp_token: Stamp token to be used for the next iteration. -output: A scalar string that is the a summary of the accumulator. -)doc"); - -REGISTER_OP("QuantileAccumulatorSerialize") - .Input("quantile_accumulator_handle: resource") - .Output("stamp_token: int64") - .Output("stream_state: string") - .Output("are_buckets_ready: bool") - .Output("buckets: float") - .Doc(R"doc( -Serializes the state of the given resource. - -quantile_accumulator_handle: The handle to the accumulator. -stamp_token: Stamp token for Read/Write operations. - Any operation with a mismatching token will be dropped. -stream_state: A serialized QuantileStreamState. -are_buckets_ready: Whether the buckets are ready or not. -buckets: Output quantile buckets representing boundaries with "num_quantile" - elements. -)doc"); - -REGISTER_OP("QuantileAccumulatorDeserialize") - .Input("quantile_accumulator_handle: resource") - .Input("stamp_token: int64") - .Input("stream_state: string") - .Input("are_buckets_ready: bool") - .Input("buckets: float") - .Doc(R"doc( -Serializes the state of the given resource. - -quantile_accumulator_handle: The handle to the accumulator. -stamp_token: Stamp token for Read/Write operations. - Any operation with a mismatching token will be dropped. -stream_state: A serialized QuantileStreamState. -are_buckets_ready: Whether the buckets are ready or not. -buckets: Output quantile summary representing boundaries with "num_quantile" - elements. -)doc"); - -REGISTER_OP("MakeQuantileSummaries") - .Attr("num_dense_features: int >= 0") - .Attr("num_sparse_features: int >= 0") - .Attr("epsilon: float") - .Input("dense_float_features: num_dense_features * float") - .Input("sparse_float_feature_indices: num_sparse_features * int64") - .Input("sparse_float_feature_values: num_sparse_features * float") - .Input("sparse_float_feature_shapes: num_sparse_features * int64") - .Input("example_weights: float") - .Output("dense_summaries: num_dense_features * string") - .Output("sparse_summaries: num_sparse_features * string") - .SetShapeFn([](InferenceContext* c) { - int num_dense_features; - TF_RETURN_IF_ERROR(c->GetAttr("num_dense_features", &num_dense_features)); - int num_sparse_features; - TF_RETURN_IF_ERROR( - c->GetAttr("num_sparse_features", &num_sparse_features)); - ShapeHandle example_weights_shape; - int example_weights_index = num_dense_features + num_sparse_features * 3; - TF_RETURN_IF_ERROR(c->WithRank(c->input(example_weights_index), 2, - &example_weights_shape)); - for (int i = 0; i < num_dense_features; ++i) { - ShapeHandle dense_feature_shape; - DimensionHandle unused_dim; - TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 2, &dense_feature_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(dense_feature_shape, 0), - c->Dim(example_weights_shape, 0), - &unused_dim)); - c->set_output(i, c->Scalar()); - } - for (int i = 0; i < num_sparse_features; ++i) { - c->set_output(i + num_dense_features, c->Scalar()); - } - return Status::OK(); - }) - .Doc(R"doc( -Creates a summary for the given features. - -num_dense_features: Number of dense feature groups to compute quantiles on. -num_sparse_features: Number of sparse feature groups to compute quantiles on. -epsilon: Error bound on the computed summary. -dense_float_features: A list of vectors which contains dense values. -sparse_float_feature_indices: List of rank 2 tensors containing the sparse float -feature indices. -sparse_float_feature_values: List of rank 1 tensors containing the sparse float -feature values. -sparse_float_feature_shapes: List of rank 1 tensors containing the shape of the -float feature. -example_weights: Rank 2 (N, 1) tensor of per-example weights. Should match - dense and sparse features shape. -dense_summaries: A list of serialized QuantileSummaryState for dense columns. -sparse_summaries: A list of serialized QuantileSummaryState for sparse columns. -)doc"); - -REGISTER_OP("QuantileBuckets") - .Attr("num_dense_features: int >= 0") - .Attr("num_sparse_features: int >= 0") - .Attr("dense_config: list(string)") - .Attr("sparse_config: list(string)") - .Input("dense_float_features: num_dense_features * float") - .Input("sparse_float_feature_indices: num_sparse_features * int64") - .Input("sparse_float_feature_values: num_sparse_features * float") - .Input("sparse_float_feature_shapes: num_sparse_features * int64") - .Input("example_weights: float") - .Output("dense_buckets: num_dense_features * float") - .Output("sparse_buckets: num_sparse_features * float") - .Doc(R"doc( -Computes quantile buckets for a given list of dense and sparse features with -given example weights. - -num_dense_features: Number of dense feature groups to compute quantiles on. -num_sparse_features: Number of sparse feature groups to compute quantiles on. -dense_config: Config for computing buckets for dense values. -Each entry is QuantileConfig proto. -sparse_config: Config for computing buckets for sparse feature values. -Each entry is QuantileConfig proto. -dense_float_features: A list of vectors which contains dense values. -sparse_float_feature_indices: List of rank 2 tensors containing the sparse float -feature indices. -sparse_float_feature_values: List of rank 1 tensors containing the sparse float -feature values. -sparse_float_feature_shapes: List of rank 1 tensors containing the shape of the -float feature. -example_weights: Rank 1 tensor containing the example weight tensor. -dense_buckets: Output quantile summary for each dense float tensor -representing boundaries each with "num_quantile" elements. -sparse_buckets: Output quantile summary for each sparse float value tensor -representing boundaries each with "num_quantile" elements. -)doc"); - -REGISTER_OP("Quantiles") - .Attr("num_dense_features: int >= 0") - .Attr("num_sparse_features: int >= 0") - .Input("dense_values: num_dense_features * float") - .Input("sparse_values: num_sparse_features * float") - .Input("dense_buckets: num_dense_features * float") - .Input("sparse_buckets: num_sparse_features * float") - .Input("sparse_indices: num_sparse_features * int64") - .Output("dense_quantiles: num_dense_features * int32") - .Output("sparse_quantiles: num_sparse_features * int32") - .SetShapeFn([](InferenceContext* c) { - int num_dense_features; - TF_RETURN_IF_ERROR(c->GetAttr("num_dense_features", &num_dense_features)); - int num_sparse_features; - TF_RETURN_IF_ERROR( - c->GetAttr("num_sparse_features", &num_sparse_features)); - // Set output shapes (dense_quantiles and sparse_quantiles) by the - // relevant inputs (dense_values and sparse_values). Note that the output - // has an additional dimension for dimension_ids. - for (int i = 0; i < num_dense_features + num_sparse_features; ++i) { - c->set_output(i, c->MakeShape({c->Dim(c->input(i), 0), 2})); - } - return Status::OK(); - }) - .Doc(R"doc( -Computes quantile for each a given list of dense and sparse feature values using -the given buckets. - -num_dense_features: Number of dense feature groups to generate quantiles for. -num_sparse_features: Number of sparse feature groups to generate quantiles for. -dense_values: List of rank 1 tensors containing the dense values. -sparse_values: List of rank 1 tensors containing the sparse feature values. -dense_buckets: Quantile summary for each of the dense float tensor. -sparse_buckets: Quantile summary for each of the sparse feature float tensor. -sparse_indices: List of rank 2 tensors with indices for sparse float -tensors. -dense_quantiles: Rank 2 tensors representing associated quantiles for each of -dense float tensors and the dimension. -sparse_quantiles: Rank 2 tensors representing associated quantiles for each of -the sparse feature tensors for each of sparse feature dimensions: -[quantile id, dimension id]. -)doc"); - -REGISTER_OP("BucketizeWithInputBoundaries") - .Input("input: T") - .Input("boundaries: float") - .Output("output: int32") - .Attr("T: {int32, int64, float, double}") - .SetShapeFn(shape_inference::UnchangedShape) - .Doc(R"doc( -Bucketizes 'input' based on 'boundaries'. This function is similar to Bucketize -op in core math_ops, except that boundaries are specified using an input tensor, -as compared with a fixed attribute in Bucketize(). - -For example, if the inputs are - boundaries = [0, 10, 100] - input = [[-5, 10000] - [150, 10] - [5, 100]] - -then the output will be - output = [[0, 3] - [3, 2] - [1, 3]] - -input: Any shape of Tensor contains with numeric type. -boundaries: A vector Tensor of sorted floats specifies the boundaries -of the buckets. -output: Same shape as 'input', where each value of input is replaced with its corresponding bucket index. -)doc"); - -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/ops/split_handler_ops.cc b/tensorflow/contrib/boosted_trees/ops/split_handler_ops.cc deleted file mode 100644 index f1e12a028a7..00000000000 --- a/tensorflow/contrib/boosted_trees/ops/split_handler_ops.cc +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { - -using shape_inference::DimensionHandle; -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -REGISTER_OP("BuildDenseInequalitySplits") - .Input("num_minibatches: int64") - .Input("partition_ids: int32") - .Input("bucket_ids: int64") - .Input("gradients: float32") - .Input("hessians: float32") - .Input("bucket_boundaries: float32") - .Input("class_id: int32") - .Input("feature_column_group_id: int32") - .Input("l1_regularization: float") - .Input("l2_regularization: float") - .Input("tree_complexity_regularization: float") - .Input("min_node_weight: float") - .Input("multiclass_strategy: int32") - .Input("weak_learner_type: int32") - .Output("output_partition_ids: int32") - .Output("gains: float32") - .Output("split_infos: string") - .SetShapeFn([](InferenceContext* c) { - DimensionHandle unused_dim; - ShapeHandle unused_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_shape)); - - ShapeHandle partition_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &partition_ids_shape)); - ShapeHandle bucket_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 2, &bucket_ids_shape)); - ShapeHandle gradients_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(3), 1, &gradients_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(gradients_shape, 0), &unused_dim)); - ShapeHandle hessians_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(4), 1, &hessians_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(hessians_shape, 0), &unused_dim)); - ShapeHandle bucket_boundaries_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 1, &bucket_boundaries_shape)); - c->set_output(0, c->Vector(c->UnknownDim())); - c->set_output(1, c->Vector(c->UnknownDim())); - c->set_output(2, c->Vector(c->UnknownDim())); - return Status::OK(); - }) - .Doc(R"doc( -Find the split that has the best gain for the accumulated stats. - -num_minibatches: A scalar, the number of times per example gradients & hessians - were accumulated. The stats are divided by this to get per example stats. -partition_ids: A rank 1 tensor of partition IDs. -bucket_ids: A rank 2 tensor of buckets IDs and dimensions. -gradients: A rank 1 tensor of gradients. -hessians: A rank 1 tensor of hessians. -bucket_boundaries: A rank 1 tensor, thresholds that were used for bucketization. -class_id: A scalar, the class id for which we're building the splits. -feature_column_group_id: A scalar, the index of the feature we are spiltting on. -l1_regularization: A scalar, which specifies the l1 regularization term. -l2_regularization: A scalar, which specifies the l2 regularization term. -tree_complexity_regularization: A scalar, which specifies the tree complexity - regularization term. -min_node_weight: A scalar, minimum sum of example hessian needed in a child. - If a split results in a leaf node with a smaller value, the split will not - be considered. -multiclass_strategy: A scalar, specifying the multiclass handling strategy. - See LearnerConfig.MultiClassStrategy for valid values. -weak_learner_type: A scalar, specifying the weak learner type to use. - See LearnerConfig.WeakLearnerType for valid values. -output_partition_ids: A rank 1 tensor, the partition IDs that we created splits - for. -gains: A rank 1 tensor, for the computed gain for the created splits. -split_infos: A rank 1 tensor of serialized protos which contains the - `SplitInfo`s. -)doc"); - -REGISTER_OP("BuildSparseInequalitySplits") - .Input("num_minibatches: int64") - .Input("partition_ids: int32") - .Input("bucket_ids: int64") - .Input("gradients: float32") - .Input("hessians: float32") - .Input("bucket_boundaries: float32") - .Input("class_id: int32") - .Input("feature_column_group_id: int32") - .Input("bias_feature_id: int64") - .Input("l1_regularization: float") - .Input("l2_regularization: float") - .Input("tree_complexity_regularization: float") - .Input("min_node_weight: float") - .Input("multiclass_strategy: int32") - .Output("output_partition_ids: int32") - .Output("gains: float32") - .Output("split_infos: string") - .SetShapeFn([](InferenceContext* c) { - DimensionHandle unused_dim; - ShapeHandle unused_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_shape)); - - ShapeHandle partition_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &partition_ids_shape)); - ShapeHandle bucket_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 2, &bucket_ids_shape)); - ShapeHandle gradients_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(3), 1, &gradients_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(gradients_shape, 0), &unused_dim)); - ShapeHandle hessians_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(4), 1, &hessians_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(hessians_shape, 0), &unused_dim)); - ShapeHandle bucket_boundaries_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 1, &bucket_boundaries_shape)); - c->set_output(0, c->Vector(c->UnknownDim())); - c->set_output(1, c->Vector(c->UnknownDim())); - c->set_output(2, c->Vector(c->UnknownDim())); - return Status::OK(); - }) - .Doc(R"doc( -Find the split that has the best gain for the accumulated stats for a particular -feature column. - -num_minibatches: A scalar, the number of times per example gradients & hessians - were accumulated. The stats are divided by this to get per example stats. -partition_ids: A rank 2 tensor of partition IDs for each dimension of feature column. -bucket_ids: A rank 2 tensor of buckets IDs and dimensions. -gradients: A rank 1 tensor of gradients. -hessians: A rank 1 tensor of hessians. -bucket_boundaries: A rank 1 tensor, thresholds that were used for bucketization. -class_id: A scalar, the class id for which we're building the splits. -feature_column_group_id: A scalar, the index of the feature we are spiltting on. -l1_regularization: A scalar, which specifies the l1 regularization term. -l2_regularization: A scalar, which specifies the l2 regularization term. -tree_complexity_regularization: A scalar, which specifies the tree complexity - regularization term. -min_node_weight: A scalar, minimum sum of example hessian needed in a child. - If a split results in a leaf node with a smaller value, the split will not - be considered. -multiclass_strategy: A scalar, specifying the multiclass handling strategy. - See LearnerConfig.MultiClassStrategy for valid values. -output_partition_ids: A rank 1 tensor, the partition IDs that we created splits - for. -gains: A rank 1 tensor, for the computed gain for the created splits. -split_infos: A rank 1 tensor of serialized protos which contains the - `SplitInfo`s. -)doc"); - -REGISTER_OP("BuildCategoricalEqualitySplits") - .Input("num_minibatches: int64") - .Input("partition_ids: int32") - .Input("feature_ids: int64") - .Input("gradients: float32") - .Input("hessians: float32") - .Input("class_id: int32") - .Input("feature_column_group_id: int32") - .Input("bias_feature_id: int64") - .Input("l1_regularization: float") - .Input("l2_regularization: float") - .Input("tree_complexity_regularization: float") - .Input("min_node_weight: float") - .Input("multiclass_strategy: int32") - .Input("weak_learner_type: int32") - .Output("output_partition_ids: int32") - .Output("gains: float32") - .Output("split_infos: string") - .SetShapeFn([](InferenceContext* c) { - DimensionHandle unused_dim; - ShapeHandle unused_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_shape)); - - ShapeHandle partition_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &partition_ids_shape)); - ShapeHandle bucket_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 2, &bucket_ids_shape)); - ShapeHandle gradients_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(3), 1, &gradients_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(gradients_shape, 0), &unused_dim)); - ShapeHandle hessians_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(4), 1, &hessians_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(hessians_shape, 0), &unused_dim)); - c->set_output(0, c->Vector(c->UnknownDim())); - c->set_output(1, c->Vector(c->UnknownDim())); - c->set_output(2, c->Vector(c->UnknownDim())); - return Status::OK(); - }) - .Doc(R"doc( -Find the split that has the best gain for the accumulated stats. - -num_minibatches: A scalar, the number of times per example gradients & hessians - were accumulated. The stats are divided by this to get per example stats. -partition_ids: A rank 1 tensor of partition IDs. -feature_ids: A rank 2 tensor of feature IDs and dimensions. -gradients: A rank 1 tensor of gradients. -hessians: A rank 1 tensor of hessians. -class_id: A scalar, the class id for which we're building the splits. -feature_column_group_id: A scalar, the index of the feature we are spiltting on. -l1_regularization: A scalar, which specifies the l1 regularization term. -l2_regularization: A scalar, which specifies the l2 regularization term. -tree_complexity_regularization: A scalar, which specifies the tree complexity - regularization term. -min_node_weight: A scalar, minimum sum of example hessian needed in a child. - If a split results in a leaf node with a smaller value, the split will not - be considered. -multiclass_strategy: A scalar, specifying the multiclass handling strategy. - See LearnerConfig.MultiClassStrategy for valid values. -weak_learner_type: A scalar, specifying the weak learner type to use. - See LearnerConfig.WeakLearnerType for valid values. -output_partition_ids: A rank 1 tensor, the partition IDs that we created splits - for. -gains: A rank 1 tensor, for the computed gain for the created splits. -split_infos: A rank 1 tensor of serialized protos which contains the - `SplitInfo`s. -)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/ops/stats_accumulator_ops.cc b/tensorflow/contrib/boosted_trees/ops/stats_accumulator_ops.cc deleted file mode 100644 index 179505eef01..00000000000 --- a/tensorflow/contrib/boosted_trees/ops/stats_accumulator_ops.cc +++ /dev/null @@ -1,473 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { -namespace boosted_trees { -using shape_inference::DimensionHandle; -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -REGISTER_RESOURCE_HANDLE_OP(StatsAccumulatorScalarResource); - -REGISTER_OP("StatsAccumulatorScalarIsInitialized") - .Input("stats_accumulator_handle: resource") - .Output("is_initialized: bool") - .SetShapeFn(tensorflow::shape_inference::ScalarShape) - .Doc(R"doc( -Checks whether a stats accumulator has been initialized. -)doc"); - -REGISTER_OP("CreateStatsAccumulatorScalar") - .Input("stats_accumulator_handle: resource") - .Input("stamp_token: int64") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - // stamp_token is a scalar. - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input)); - return Status::OK(); - }) - .Doc(R"doc( -Creates a scalar stats accumulator. - -stats_accumulator_handle: handle to the stats accumulator. -stamp_token: Token to use as the initial value of the resource stamp. -)doc"); - -REGISTER_OP("StatsAccumulatorScalarAdd") - .Attr("num_resource_handles: int >= 1") - .Input("stats_accumulator_handles: num_resource_handles * resource") - .Input("stamp_token: int64") - .Input("partition_ids: num_resource_handles * int32") - .Input("feature_ids: num_resource_handles * int64") - .Input("gradients: num_resource_handles * float") - .Input("hessians: num_resource_handles * float") - .SetShapeFn([](InferenceContext* c) { - int num_resource_handles; - TF_RETURN_IF_ERROR( - c->GetAttr("num_resource_handles", &num_resource_handles)); - for (int i = 0; i < num_resource_handles; ++i) { - ShapeHandle unused_input; - DimensionHandle unused_dim; - TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 0, &unused_input)); - TF_RETURN_IF_ERROR( - c->WithRank(c->input(num_resource_handles), 0, &unused_input)); - ShapeHandle partition_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(num_resource_handles + i + 1), - 1, &partition_ids_shape)); - ShapeHandle feature_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank( - c->input(num_resource_handles * 2 + i + 1), 2, &feature_ids_shape)); - ShapeHandle gradients_shape; - TF_RETURN_IF_ERROR(c->WithRank( - c->input(num_resource_handles * 3 + i + 1), 1, &gradients_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(gradients_shape, 0), &unused_dim)); - ShapeHandle hessians_shape; - TF_RETURN_IF_ERROR(c->WithRank( - c->input(num_resource_handles * 4 + i + 1), 1, &hessians_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(hessians_shape, 0), &unused_dim)); - } - return Status::OK(); - }) - .Doc(R"doc( -Updates the scalar stats accumulator. - -stamp_token: Stamp token for Read/Write operations. - Any operation with a mismatching token will be dropped. -stats_accumulator_handles: A list of handles to the stats accumulator. -partition_ids: A list of vectors of partition_ids. -feature_ids: Rank 2 tensor of feature id and feature dimension ids. -gradients: A list of vectors of gradients for each slot in - . -hessians: A list of vectors of hessians for each slot in - . -)doc"); - -REGISTER_OP("StatsAccumulatorScalarFlush") - .Input("stats_accumulator_handle: resource") - .Input("stamp_token: int64") - .Input("next_stamp_token: int64") - .Output("num_updates: int64") - .Output("output_partition_ids: int32") - .Output("output_feature_ids: int64") - .Output("output_gradients: float") - .Output("output_hessians: float") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused_input)); - c->set_output(0, c->Scalar()); - c->set_output(1, c->Vector(c->UnknownDim())); - c->set_output(2, c->UnknownShape()); - c->set_output(3, c->Vector(c->UnknownDim())); - c->set_output(4, c->Vector(c->UnknownDim())); - return Status::OK(); - }) - .Doc(R"doc( -Flushes the scalar stats accumulator to output and resets the internal state. - -stats_accumulator_handle: handle to the stats accumulator. -stamp_token: Stamp token for Read/Write operations. - Any operation with a mismatching token will be dropped. -next_stamp_token: Stamp token for the next iteration. -num_updates: Number of times stats were added to this accumulator since last - flush. -output_partition_ids A vector of partition_ids for the slots. -output_feature_ids: Rank 2 tensor of feature id and feature dimension ids. -output_gradients: A vector of gradients, with a value for each slot - in . -output_hessians: A vector of hessians, with a value for each slot - in . -)doc"); - -REGISTER_OP("StatsAccumulatorScalarDeserialize") - .Input("stats_accumulator_handle: resource") - .Input("stamp_token: int64") - .Input("num_updates: int64") - .Input("partition_ids: int32") - .Input("feature_ids: int64") - .Input("gradients: float") - .Input("hessians: float") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle unused_input; - DimensionHandle unused_dim; - // stats_accumulator_handle - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - // stamp_token - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input)); - // num_updates - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused_input)); - ShapeHandle partition_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 1, &partition_ids_shape)); - ShapeHandle feature_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 2, &feature_ids_shape)); - ShapeHandle gradients_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 1, &gradients_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(gradients_shape, 0), &unused_dim)); - ShapeHandle hessians_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(6), 1, &hessians_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(hessians_shape, 0), &unused_dim)); - return Status::OK(); - }) - .Doc(R"doc( -Resets the scalar stats accumulator with the serialized state. - -stats_accumulator_handle: handle to the stats accumulator. -stamp_token: Stamp token for Read/Write operations. - Any operation with a mismatching token will be dropped. -num_updates: Number of times stats were added to this accumulator since last - flush. -partition_ids: A vector of partition_ids. -feature_ids: Rank 2 tensor of feature id and feature dimension ids. -gradients: A vector of gradients for each slot in . -hessians: A vector of hessians for each slot in -)doc"); - -REGISTER_OP("StatsAccumulatorScalarSerialize") - .Input("stats_accumulator_handle: resource") - .Output("stamp_token: int64") - .Output("num_updates: int64") - .Output("output_partition_ids: int32") - .Output("output_feature_ids: int64") - .Output("output_gradients: float") - .Output("output_hessians: float") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - // stamp_token - c->set_output(0, c->Scalar()); - // num_updates - c->set_output(1, c->Scalar()); - c->set_output(2, c->Vector(c->UnknownDim())); - c->set_output(3, c->UnknownShape()); - c->set_output(4, c->Vector(c->UnknownDim())); - c->set_output(5, c->Vector(c->UnknownDim())); - return Status::OK(); - }) - .Doc(R"doc( -Serializes the scalar stats accumulator state. - -stats_accumulator_handle: handle to the stats accumulator. -stamp_token: The current stamp token for the resource. -num_updates: Number of times stats were added to this accumulator since last - flush. -output_partition_ids A vector of partition_ids for the slots. -output_feature_ids: Rank 2 tensor of feature id and feature dimension ids. -output_gradients: A vector of gradients, with a value for each slot - in . -output_hessians: A vector of hessians, with a value for each slot - in . -)doc"); - -REGISTER_OP("StatsAccumulatorScalarMakeSummary") - .Input("partition_ids: int32") - .Input("feature_ids: int64") - .Input("gradients: float") - .Input("hessians: float") - .Output("output_partition_ids: int32") - .Output("output_feature_ids: int64") - .Output("output_gradients: float") - .Output("output_hessians: float") - .Doc(R"doc( -)doc"); - -// Tensor version of the stats accumulator ops. -REGISTER_RESOURCE_HANDLE_OP(StatsAccumulatorTensorResource); - -REGISTER_OP("StatsAccumulatorTensorIsInitialized") - .Input("stats_accumulator_handle: resource") - .Output("is_initialized: bool") - .SetShapeFn(tensorflow::shape_inference::ScalarShape) - .Doc(R"doc( -Checks whether a tensor stats accumulator has been initialized. -)doc"); - -REGISTER_OP("CreateStatsAccumulatorTensor") - .Input("stats_accumulator_handle: resource") - .Input("stamp_token: int64") - .Input("per_slot_gradient_shape: int64") - .Input("per_slot_hessian_shape: int64") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - // stamp_token is a scalar. - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 1, &unused_input)); - return Status::OK(); - }) - .Doc(R"doc( -Creates a tensor stats accumulator. - -stats_accumulator_handle: handle to the tree ensemble resource to be created. -stamp_token: Token to use as the initial value of the resource stamp. -per_slot_gradient_shape: a vector that defines the shape of gradients. -per_slot_hessian_shape: a vector that defines the shape of hessians. -)doc"); - -REGISTER_OP("StatsAccumulatorTensorAdd") - .Attr("num_resource_handles: int >= 1") - .Input("stats_accumulator_handles: num_resource_handles * resource") - .Input("stamp_token: int64") - .Input("partition_ids: num_resource_handles * int32") - .Input("feature_ids: num_resource_handles * int64") - .Input("gradients: num_resource_handles * float") - .Input("hessians: num_resource_handles * float") - .SetShapeFn([](InferenceContext* c) { - int num_resource_handles; - TF_RETURN_IF_ERROR( - c->GetAttr("num_resource_handles", &num_resource_handles)); - for (int i = 0; i < num_resource_handles; ++i) { - ShapeHandle unused_input; - DimensionHandle unused_dim; - TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 0, &unused_input)); - TF_RETURN_IF_ERROR( - c->WithRank(c->input(num_resource_handles), 0, &unused_input)); - ShapeHandle partition_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(num_resource_handles + i + 1), - 1, &partition_ids_shape)); - ShapeHandle feature_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank( - c->input(num_resource_handles * 2 + i + 1), 2, &feature_ids_shape)); - ShapeHandle gradients_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast( - c->input(num_resource_handles * 3 + i + 1), 2, &gradients_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(gradients_shape, 0), &unused_dim)); - ShapeHandle hessians_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast( - c->input(num_resource_handles * 4 + i + 1), 2, &hessians_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(hessians_shape, 0), &unused_dim)); - } - return Status::OK(); - }) - .Doc(R"doc( -Updates the tensor stats accumulator. - -stats_accumulator_handles: A list of handles to the stats accumulator. -stamp_token: Stamp token for Read/Write operations. - Any operation with a mismatching token will be dropped. -partition_ids: A list of vectors of partition_ids. -feature_ids: Rank 2 tensor of feature id and feature dimension ids. -gradients: A list of vectors of gradients for each slot in - . -hessians: A list of vectors of hessians for each slot in - . -)doc"); - -REGISTER_OP("StatsAccumulatorTensorFlush") - .Input("stats_accumulator_handle: resource") - .Input("stamp_token: int64") - .Input("next_stamp_token: int64") - .Output("num_updates: int64") - .Output("output_partition_ids: int32") - .Output("output_feature_ids: int64") - .Output("output_gradients: float") - .Output("output_hessians: float") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused_input)); - // num_updates - c->set_output(0, c->Scalar()); - c->set_output(1, c->Vector(c->UnknownDim())); - c->set_output(2, c->UnknownShape()); - c->set_output(3, c->UnknownShape()); - c->set_output(4, c->UnknownShape()); - return Status::OK(); - }) - .Doc(R"doc( -Flushes the stats accumulator to output and resets the internal state. - -stats_accumulator_handle: handle to the tree ensemble resource to be created. -stamp_token: Stamp token for Read/Write operations. - Any operation with a mismatching token will be dropped. -next_stamp_token: Stamp token to be used for the next iteration. -num_updates: Number of times stats were added to this accumulator since last - flush. -output_partition_ids: A vector of partition_ids for the slots. -output_feature_ids: Rank 2 tensor of feature id and feature dimension ids. -output_gradients: A tensor of gradients, first dimension matches slots - in . -output_hessians: A tensor of hessians, first dimension matches slots - in >. -)doc"); - -REGISTER_OP("StatsAccumulatorTensorDeserialize") - .Input("stats_accumulator_handle: resource") - .Input("stamp_token: int64") - .Input("num_updates: int64") - .Input("partition_ids: int32") - .Input("feature_ids: int64") - .Input("gradients: float") - .Input("hessians: float") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle unused_input; - DimensionHandle unused_dim; - // stats_accumulator_handle - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - // stamp_token - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input)); - // num_updates - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused_input)); - ShapeHandle partition_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 1, &partition_ids_shape)); - ShapeHandle feature_ids_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 2, &feature_ids_shape)); - ShapeHandle gradients_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(5), 2, &gradients_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(gradients_shape, 0), &unused_dim)); - ShapeHandle hessians_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(6), 2, &hessians_shape)); - TF_RETURN_IF_ERROR(c->Merge(c->Dim(partition_ids_shape, 0), - c->Dim(hessians_shape, 0), &unused_dim)); - - return Status::OK(); - }) - .Doc(R"doc( -Resets the tensor stats accumulator with the serialized state. - -stats_accumulator_handle: handle to the tree ensemble resource to be created. -stamp_token: Stamp token for Read/Write operations. - Any operation with a mismatching token will be dropped. -num_updates: Number of times stats were added to this accumulator since last - flush. -partition_ids: A vector of partition_ids. -feature_ids: Rank 2 tensor of feature id and feature dimension ids. -gradients: A vector of gradients for each slot in -hessians: A vector of hessians for each slot in . -)doc"); - -REGISTER_OP("StatsAccumulatorTensorSerialize") - .Input("stats_accumulator_handle: resource") - .Output("stamp_token: int64") - .Output("num_updates: int64") - .Output("output_partition_ids: int32") - .Output("output_feature_ids: int64") - .Output("output_gradients: float") - .Output("output_hessians: float") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - // stamp_token - c->set_output(0, c->Scalar()); - // num_updates - c->set_output(1, c->Scalar()); - c->set_output(2, c->Vector(c->UnknownDim())); - c->set_output(3, c->UnknownShape()); - c->set_output(4, c->UnknownShape()); - c->set_output(5, c->UnknownShape()); - return Status::OK(); - }) - .Doc(R"doc( -Serializes the scalar stats accumulator state. - -stats_accumulator_handle: handle to the tree ensemble resource to be created. -stamp_token: Stamp token for Read/Write operations. - Any operation with a mismatching token will be dropped. -num_updates: Number of times stats were added to this accumulator since last - flush. -output_partition_ids: A vector of partition_ids for the slots. -output_feature_ids: Rank 2 tensor of feature id and feature dimension ids. -output_gradients: A tensor of gradients, first dimension matches slots - in . -output_hessians: A tensor of hessians, first dimension matches slots - in . -)doc"); - -REGISTER_OP("StatsAccumulatorTensorMakeSummary") - .Input("partition_ids: int32") - .Input("feature_ids: int64") - .Input("gradients: float") - .Input("hessians: float") - .Output("output_partition_ids: int32") - .Output("output_feature_ids: int64") - .Output("output_gradients: float") - .Output("output_hessians: float") - .Doc(R"doc( -Summarizes the stats by summing the that are for the same -. - -partition_ids: A vector of partition_ids. -feature_ids: Rank 2 tensor of feature id and feature dimension ids. -gradients: A vector of gradients for each slot in . -hessians: A vector of hessians for each slot in . -output_partition_ids: A vector of partition_ids for the slots. -output_feature_ids: A rank2 tensor of feature_ids and dimensions for the slots. -output_gradients: A tensor of gradients, first dimension matches slots - in . -output_hessians: A tensor of hessians, first dimension matches slots - in . -)doc"); -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/ops/training_ops.cc b/tensorflow/contrib/boosted_trees/ops/training_ops.cc deleted file mode 100644 index 604ec8e0bfa..00000000000 --- a/tensorflow/contrib/boosted_trees/ops/training_ops.cc +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { -namespace boosted_trees { - -REGISTER_OP("CenterTreeEnsembleBias") - .Attr("learner_config: string") - .Attr("centering_epsilon: float = 0.01") - .Input("tree_ensemble_handle: resource") - .Input("stamp_token: int64") - .Input("next_stamp_token: int64") - .Input("delta_updates: float") - .Output("continue_centering: bool") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 1, &unused_input)); - c->set_output(0, c->Scalar()); - return Status::OK(); - }) - .Doc(R"doc( -Centers the tree ensemble bias before adding trees based on feature splits. - -learner_config: Config for the learner of type LearnerConfig proto. -tree_ensemble_handle: Handle to the ensemble variable. -stamp_token: Stamp token for validating operation consistency. -next_stamp_token: Stamp token to be used for the next iteration. -delta_updates: Rank 1 Tensor containing delta updates per bias dimension. -continue_centering: Scalar indicating whether more centering is needed. -)doc"); - -REGISTER_OP("GrowTreeEnsemble") - .Attr("learner_config: string") - .Attr("num_handlers: int >= 0") - .Attr("center_bias: bool") - .Input("tree_ensemble_handle: resource") - .Input("stamp_token: int64") - .Input("next_stamp_token: int64") - .Input("learning_rate: float") - .Input("dropout_seed: int64") - .Input("max_tree_depth: int32") - .Input("weak_learner_type: int32") - .Input("partition_ids: num_handlers * int32") - .Input("gains: num_handlers * float") - .Input("splits: num_handlers * string") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused_input)); - // Dropout seed. - TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused_input)); - // Maximum tree depth. - TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &unused_input)); - return Status::OK(); - }) - .Doc(R"doc( -Grows the tree ensemble by either adding a layer to the last tree being grown -or by starting a new tree. - -learner_config: Config for the learner of type LearnerConfig proto. -num_handlers: Number of handlers generating candidates. -tree_ensemble_handle: Handle to the ensemble variable. -stamp_token: Stamp token for validating operation consistency. -next_stamp_token: Stamp token to be used for the next iteration. -learning_rate: Scalar learning rate. -weak_learner_type: The type of weak learner to use. -partition_ids: List of Rank 1 Tensors containing partition Id per candidate. -gains: List of Rank 1 Tensors containing gains per candidate. -splits: List of Rank 1 Tensors containing serialized SplitInfo protos per candidate. -)doc"); - -REGISTER_OP("TreeEnsembleStats") - .Input("tree_ensemble_handle: resource") - .Input("stamp_token: int64") - .Output("num_trees: int64") - .Output("num_layers: int64") - .Output("active_tree: int64") - .Output("active_layer: int64") - .Output("attempted_trees: int64") - .Output("attempted_layers: int64") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused_input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input)); - c->set_output(0, c->Scalar()); - c->set_output(1, c->Scalar()); - c->set_output(2, c->Scalar()); - c->set_output(3, c->Scalar()); - c->set_output(4, c->Scalar()); - c->set_output(5, c->Scalar()); - return Status::OK(); - }) - .Doc(R"doc( -Retrieves stats related to the tree ensemble. - -tree_ensemble_handle: Handle to the ensemble variable. -stamp_token: Stamp token for validating operation consistency. -num_trees: Scalar indicating the number of finalized trees in the ensemble. -num_layers: Scalar indicating the number of layers in the ensemble. -active_tree: Scalar indicating the active tree being trained. -active_layer: Scalar indicating the active layer being trained. -)doc"); - -} // namespace boosted_trees -} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/proto/BUILD b/tensorflow/contrib/boosted_trees/proto/BUILD deleted file mode 100644 index ca3dd545489..00000000000 --- a/tensorflow/contrib/boosted_trees/proto/BUILD +++ /dev/null @@ -1,45 +0,0 @@ -load("//tensorflow/core/platform:default/build_config.bzl", "tf_proto_library") - -package( - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_proto_library( - name = "learner_proto", - srcs = [ - "learner.proto", - ], - cc_api_version = 2, - protodeps = [ - ":tree_config_proto", - ], - visibility = ["//visibility:public"], -) - -tf_proto_library( - name = "quantiles_proto", - srcs = [ - "quantiles.proto", - ], - cc_api_version = 2, - visibility = ["//visibility:public"], -) - -tf_proto_library( - name = "split_info_proto", - srcs = ["split_info.proto"], - cc_api_version = 2, - protodeps = [ - ":tree_config_proto", - ], - visibility = ["//visibility:public"], -) - -tf_proto_library( - name = "tree_config_proto", - srcs = ["tree_config.proto"], - cc_api_version = 2, - visibility = ["//visibility:public"], -) diff --git a/tensorflow/contrib/boosted_trees/proto/learner.proto b/tensorflow/contrib/boosted_trees/proto/learner.proto deleted file mode 100644 index fc5f158c073..00000000000 --- a/tensorflow/contrib/boosted_trees/proto/learner.proto +++ /dev/null @@ -1,161 +0,0 @@ -syntax = "proto3"; - -package tensorflow.boosted_trees.learner; - -import "tensorflow/contrib/boosted_trees/proto/tree_config.proto"; - -option cc_enable_arenas = true; - -// Tree regularization config. -message TreeRegularizationConfig { - // Classic L1/L2. - float l1 = 1; - float l2 = 2; - - // Tree complexity penalizes overall model complexity effectively - // limiting how deep the tree can grow in regions with small gain. - float tree_complexity = 3; -} - -// Tree constraints config. -message TreeConstraintsConfig { - // Maximum depth of the trees. The default value is 6 if not specified. - uint32 max_tree_depth = 1; - - // Min hessian weight per node. - float min_node_weight = 2; - - // Maximum number of unique features used in the tree. Zero means there is no - // limit. - int64 max_number_of_unique_feature_columns = 3; -} - -// LearningRateConfig describes all supported learning rate tuners. -message LearningRateConfig { - oneof tuner { - LearningRateFixedConfig fixed = 1; - LearningRateDropoutDrivenConfig dropout = 2; - LearningRateLineSearchConfig line_search = 3; - } -} - -// Config for a fixed learning rate. -message LearningRateFixedConfig { - float learning_rate = 1; -} - -// Config for a tuned learning rate. -message LearningRateLineSearchConfig { - // Max learning rate. Must be strictly positive. - float max_learning_rate = 1; - - // Number of learning rate values to consider between [0, max_learning_rate). - int32 num_steps = 2; -} - -// When we have a sequence of trees 1, 2, 3 ... n, these essentially represent -// weights updates in functional space, and thus we can use averaging of weight -// updates to achieve better performance. For example, we can say that our final -// ensemble will be an average of ensembles of tree 1, and ensemble of tree 1 -// and tree 2 etc .. ensemble of all trees. -// Note that this averaging will apply ONLY DURING PREDICTION. The training -// stays the same. -message AveragingConfig { - oneof config { - float average_last_n_trees = 1; - // Between 0 and 1. If set to 1.0, we are averaging ensembles of tree 1, - // ensemble of tree 1 and tree 2, etc ensemble of all trees. If set to 0.5, - // last half of the trees are averaged etc. - float average_last_percent_trees = 2; - } -} - -message LearningRateDropoutDrivenConfig { - // Probability of dropping each tree in an existing so far ensemble. - float dropout_probability = 1; - - // When trees are built after dropout happen, they don't "advance" to the - // optimal solution, they just rearrange the path. However you can still - // choose to skip dropout periodically, to allow a new tree that "advances" - // to be added. - // For example, if running for 200 steps with probability of dropout 1/100, - // you would expect the dropout to start happening for sure for all iterations - // after 100. However you can add probability_of_skipping_dropout of 0.1, this - // way iterations 100-200 will include approx 90 iterations of dropout and 10 - // iterations of normal steps.Set it to 0 if you want just keep building - // the refinement trees after dropout kicks in. - float probability_of_skipping_dropout = 2; - - // Between 0 and 1. - float learning_rate = 3; -} - -message LearnerConfig { - enum PruningMode { - PRUNING_MODE_UNSPECIFIED = 0; - PRE_PRUNE = 1; - POST_PRUNE = 2; - } - - enum GrowingMode { - GROWING_MODE_UNSPECIFIED = 0; - WHOLE_TREE = 1; - LAYER_BY_LAYER = 2; - } - - enum MultiClassStrategy { - MULTI_CLASS_STRATEGY_UNSPECIFIED = 0; - TREE_PER_CLASS = 1; - FULL_HESSIAN = 2; - DIAGONAL_HESSIAN = 3; - } - - enum WeakLearnerType { - NORMAL_DECISION_TREE = 0; - OBLIVIOUS_DECISION_TREE = 1; - } - - // Number of classes. - uint32 num_classes = 1; - - // Fraction of features to consider in each tree sampled randomly - // from all available features. - oneof feature_fraction { - float feature_fraction_per_tree = 2; - float feature_fraction_per_level = 3; - }; - - // Regularization. - TreeRegularizationConfig regularization = 4; - - // Constraints. - TreeConstraintsConfig constraints = 5; - - // Pruning. POST_PRUNE is the default pruning mode. - PruningMode pruning_mode = 8; - - // Growing Mode. LAYER_BY_LAYER is the default growing mode. - GrowingMode growing_mode = 9; - - // Learning rate. By default we use fixed learning rate of 0.1. - LearningRateConfig learning_rate_tuner = 6; - - // Multi-class strategy. By default we use TREE_PER_CLASS for binary - // classification and linear regression. For other cases, we use - // DIAGONAL_HESSIAN as the default. - MultiClassStrategy multi_class_strategy = 10; - - // If you want to average the ensembles (for regularization), provide the - // config below. - AveragingConfig averaging_config = 11; - - // By default we use NORMAL_DECISION_TREE as weak learner. - WeakLearnerType weak_learner_type = 12; - - // If you want to enforce some splits and allow boosting to figure out the - // rest, you can provide a tree that represents the starting splits for each - // tree in the ensemble. - // Set both each_tree_start and each_tree_start_num_layers. - tensorflow.boosted_trees.trees.DecisionTreeConfig each_tree_start = 13; - int32 each_tree_start_num_layers = 14; -} diff --git a/tensorflow/contrib/boosted_trees/proto/quantiles.proto b/tensorflow/contrib/boosted_trees/proto/quantiles.proto deleted file mode 100644 index 7f872d2aa71..00000000000 --- a/tensorflow/contrib/boosted_trees/proto/quantiles.proto +++ /dev/null @@ -1,32 +0,0 @@ -syntax = "proto3"; - -option cc_enable_arenas = true; - -package boosted_trees; - -message QuantileConfig { - // Maximum eps error when computing quantile summaries. - double eps = 1; - // Number of quantiles to generate. - int64 num_quantiles = 2; -} - -message QuantileEntry { - // Value for the entry. - float value = 1; - // Weight for the entry. - float weight = 2; - // We need the minimum and maximum rank possible for this entry. - // Rank is 0.0 for the absolute minimum and sum of the weights for the maximum - // value in the input. - float min_rank = 3; - float max_rank = 4; -} - -message QuantileSummaryState { - repeated QuantileEntry entries = 1; -} - -message QuantileStreamState { - repeated QuantileSummaryState summaries = 1; -} diff --git a/tensorflow/contrib/boosted_trees/proto/split_info.proto b/tensorflow/contrib/boosted_trees/proto/split_info.proto deleted file mode 100644 index 784977af395..00000000000 --- a/tensorflow/contrib/boosted_trees/proto/split_info.proto +++ /dev/null @@ -1,28 +0,0 @@ -syntax = "proto3"; - -option cc_enable_arenas = true; - -package tensorflow.boosted_trees.learner; - -import "tensorflow/contrib/boosted_trees/proto/tree_config.proto"; - -// Gathered information for a split node. -message SplitInfo { - // The split node without the child nodes attached. - tensorflow.boosted_trees.trees.TreeNode split_node = 1; - - // Left Leaf node. - tensorflow.boosted_trees.trees.Leaf left_child = 2; - - // Right Leaf node. - tensorflow.boosted_trees.trees.Leaf right_child = 3; -} - -message ObliviousSplitInfo { - tensorflow.boosted_trees.trees.TreeNode split_node = 1; - repeated tensorflow.boosted_trees.trees.Leaf children = 2; - // For each child, children_parent_id stores the node_id of its parent when it - // was a leaf. For the idx-th child it corresponds the idx/2-th - // children_parent_id. - repeated int32 children_parent_id = 3; -} diff --git a/tensorflow/contrib/boosted_trees/proto/tree_config.proto b/tensorflow/contrib/boosted_trees/proto/tree_config.proto deleted file mode 100644 index 520b4f8b11b..00000000000 --- a/tensorflow/contrib/boosted_trees/proto/tree_config.proto +++ /dev/null @@ -1,172 +0,0 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package tensorflow.boosted_trees.trees; - -// TreeNode describes a node in a tree. -message TreeNode { - oneof node { - Leaf leaf = 1; - DenseFloatBinarySplit dense_float_binary_split = 2; - SparseFloatBinarySplitDefaultLeft sparse_float_binary_split_default_left = - 3; - SparseFloatBinarySplitDefaultRight sparse_float_binary_split_default_right = - 4; - CategoricalIdBinarySplit categorical_id_binary_split = 5; - CategoricalIdSetMembershipBinarySplit - categorical_id_set_membership_binary_split = 6; - ObliviousDenseFloatBinarySplit oblivious_dense_float_binary_split = 7; - ObliviousCategoricalIdBinarySplit oblivious_categorical_id_binary_split = 8; - } - TreeNodeMetadata node_metadata = 777; -} - -// TreeNodeMetadata encodes metadata associated with each node in a tree. -message TreeNodeMetadata { - // The gain associated with this node. - float gain = 1; - - // The original leaf node before this node was split. - Leaf original_leaf = 2; - - // The original layer of leaves before that layer was converted to a split. - repeated Leaf original_oblivious_leaves = 3; -} - -// Leaves can either hold dense or sparse information. -message Leaf { - oneof leaf { - // See third_party/tensorflow/contrib/decision_trees/ - // proto/generic_tree_model.proto - // for a description of how vector and sparse_vector might be used. - Vector vector = 1; - SparseVector sparse_vector = 2; - } -} - -message Vector { - repeated float value = 1; -} - -message SparseVector { - repeated int32 index = 1; - repeated float value = 2; -} - -// Split rule for dense float features. -message DenseFloatBinarySplit { - // Float feature column and split threshold describing - // the rule feature <= threshold. - int32 feature_column = 1; - // If feature column is multivalent, this holds the index of the dimension - // for the split. Defaults to 0. - int32 dimension_id = 5; - float threshold = 2; - - // Node children indexing into a contiguous - // vector of nodes starting from the root. - int32 left_id = 3; - int32 right_id = 4; -} - -// Split rule for sparse float features defaulting left for missing features. -message SparseFloatBinarySplitDefaultLeft { - DenseFloatBinarySplit split = 1; -} - -// Split rule for sparse float features defaulting right for missing features. -message SparseFloatBinarySplitDefaultRight { - DenseFloatBinarySplit split = 1; -} - -// Split rule for categorical features with a single feature Id. -message CategoricalIdBinarySplit { - // Categorical feature column and Id describing - // the rule feature == Id. - int32 feature_column = 1; - int64 feature_id = 2; - - // Node children indexing into a contiguous - // vector of nodes starting from the root. - int32 left_id = 3; - int32 right_id = 4; -} - -// Split rule for categorical features with a set of feature Ids. -message CategoricalIdSetMembershipBinarySplit { - // Categorical feature column and Id describing - // the rule feature ∈ feature_ids. - int32 feature_column = 1; - // Sorted list of Ids in the set. - repeated int64 feature_ids = 2; - - // Node children indexing into a contiguous - // vector of nodes starting from the root. - int32 left_id = 3; - int32 right_id = 4; -} - -// Split rule for dense float features in the oblivious case. -message ObliviousDenseFloatBinarySplit { - // Float feature column and split threshold describing - // the rule feature <= threshold. - int32 feature_column = 1; - float threshold = 2; - // We don't store children ids, because either the next node represents the - // whole next layer of the tree or starting with the next node we only have - // leaves. -} - -// Split rule for categorical features with a single feature Id in the oblivious -// case. -message ObliviousCategoricalIdBinarySplit { - // Categorical feature column and Id describing the rule feature == Id. - int32 feature_column = 1; - int64 feature_id = 2; - // We don't store children ids, because either the next node represents the - // whole next layer of the tree or starting with the next node we only have - // leaves. -} - -// DecisionTreeConfig describes a list of connected nodes. -// Node 0 must be the root and can carry any payload including a leaf -// in the case of representing the bias. -// Note that each node id is implicitly its index in the list of nodes. -message DecisionTreeConfig { - repeated TreeNode nodes = 1; -} - -message DecisionTreeMetadata { - // How many times tree weight was updated (due to reweighting of the final - // ensemble, dropout, shrinkage etc). - int32 num_tree_weight_updates = 1; - - // Number of layers grown for this tree. - int32 num_layers_grown = 2; - - // Whether the tree is finalized in that no more layers can be grown. - bool is_finalized = 3; -} - -message GrowingMetadata { - // Number of trees that we have attempted to build. After pruning, these - // trees might have been removed. - int64 num_trees_attempted = 1; - // Number of layers that we have attempted to build. After pruning, these - // layers might have been removed. - int64 num_layers_attempted = 2; - - // Sorted list of column handlers that have been used in at least one split - // so far. - repeated int64 used_handler_ids = 3; -} - -// DecisionTreeEnsembleConfig describes an ensemble of decision trees. -message DecisionTreeEnsembleConfig { - repeated DecisionTreeConfig trees = 1; - repeated float tree_weights = 2; - repeated DecisionTreeMetadata tree_metadata = 3; - - // Metadata that is used during the training. - GrowingMetadata growing_metadata = 4; -} diff --git a/tensorflow/contrib/boosted_trees/python/__init__.py b/tensorflow/contrib/boosted_trees/python/__init__.py deleted file mode 100644 index 60b0476a442..00000000000 --- a/tensorflow/contrib/boosted_trees/python/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Gradient boosted trees implementation in tensorflow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.boosted_trees.python.ops import * -# pylint: enable=unused-import,wildcard-import diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/model_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/model_ops_test.py deleted file mode 100644 index aa3f24f08a0..00000000000 --- a/tensorflow/contrib/boosted_trees/python/kernel_tests/model_ops_test.py +++ /dev/null @@ -1,331 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the GTFlow model ops. - -The tests cover: -- Loading a model from protobufs. -- Running Predictions using an existing model. -- Serializing the model. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -import numpy as np - -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.boosted_trees.proto import tree_config_pb2 -from tensorflow.contrib.boosted_trees.python.ops import model_ops -from tensorflow.contrib.boosted_trees.python.ops import prediction_ops -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import resources -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest -from tensorflow.python.training import saver - - -def _append_to_leaf(leaf, c_id, w): - """Helper method for building tree leaves. - - Appends weight contributions for the given class index to a leaf node. - - Args: - leaf: leaf node to append to. - c_id: class Id for the weight update. - w: weight contribution value. - """ - leaf.sparse_vector.index.append(c_id) - leaf.sparse_vector.value.append(w) - - -def _set_float_split(split, feat_col, thresh, l_id, r_id): - """Helper method for building tree float splits. - - Sets split feature column, threshold and children. - - Args: - split: split node to update. - feat_col: feature column for the split. - thresh: threshold to split on forming rule x <= thresh. - l_id: left child Id. - r_id: right child Id. - """ - split.feature_column = feat_col - split.threshold = thresh - split.left_id = l_id - split.right_id = r_id - - -class ModelOpsTest(test_util.TensorFlowTestCase): - - def setUp(self): - """Sets up test for model_ops. - - Create a batch of two examples having one dense float, two sparse float and - one sparse int features. - The data looks like the following: - | Instance | Dense0 | SparseF0 | SparseF1 | SparseI0 | - | 0 | 7 | -3 | | | - | 1 | -2 | | 4 | 9,1 | - """ - super(ModelOpsTest, self).setUp() - self._dense_float_tensor = np.array([[7.0], [-2.0]]) - self._sparse_float_indices1 = np.array([[0, 0]]) - self._sparse_float_values1 = np.array([-3.0]) - self._sparse_float_shape1 = np.array([2, 1]) - self._sparse_float_indices2 = np.array([[1, 0]]) - self._sparse_float_values2 = np.array([4.0]) - self._sparse_float_shape2 = np.array([2, 1]) - self._sparse_int_indices1 = np.array([[1, 0], [1, 1]]) - self._sparse_int_values1 = np.array([9, 1]) - self._sparse_int_shape1 = np.array([2, 2]) - self._seed = 123 - - def testCreate(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - tree = tree_ensemble_config.trees.add() - _append_to_leaf(tree.nodes.add().leaf, 0, -0.4) - tree_ensemble_config.tree_weights.append(1.0) - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=3, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="create_tree") - resources.initialize_resources(resources.shared_resources()).run() - - result, _ = prediction_ops.gradient_trees_prediction( - tree_ensemble_handle, - self._seed, [self._dense_float_tensor], [ - self._sparse_float_indices1, self._sparse_float_indices2 - ], [self._sparse_float_values1, self._sparse_float_values2], - [self._sparse_float_shape1, - self._sparse_float_shape2], [self._sparse_int_indices1], - [self._sparse_int_values1], [self._sparse_int_shape1], - learner_config=learner_config.SerializeToString(), - apply_dropout=False, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - self.assertAllClose(result.eval(), [[-0.4], [-0.4]]) - stamp_token = model_ops.tree_ensemble_stamp_token(tree_ensemble_handle) - self.assertEqual(stamp_token.eval(), 3) - - def testSerialization(self): - with ops.Graph().as_default() as graph: - with self.session(graph): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Bias tree only for second class. - tree1 = tree_ensemble_config.trees.add() - _append_to_leaf(tree1.nodes.add().leaf, 1, -0.2) - - tree_ensemble_config.tree_weights.append(1.0) - - # Depth 2 tree. - tree2 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_weights.append(1.0) - _set_float_split(tree2.nodes.add() - .sparse_float_binary_split_default_right.split, 1, 4.0, - 1, 2) - _set_float_split(tree2.nodes.add().dense_float_binary_split, 0, 9.0, 3, - 4) - _append_to_leaf(tree2.nodes.add().leaf, 0, 0.5) - _append_to_leaf(tree2.nodes.add().leaf, 1, 1.2) - _append_to_leaf(tree2.nodes.add().leaf, 0, -0.9) - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=7, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="saver_tree") - stamp_token, serialized_config = model_ops.tree_ensemble_serialize( - tree_ensemble_handle) - resources.initialize_resources(resources.shared_resources()).run() - self.assertEqual(stamp_token.eval(), 7) - serialized_config = serialized_config.eval() - - with ops.Graph().as_default() as graph: - with self.session(graph): - tree_ensemble_handle2 = model_ops.tree_ensemble_variable( - stamp_token=9, - tree_ensemble_config=serialized_config, - name="saver_tree2") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 3 - - result, _ = prediction_ops.gradient_trees_prediction( - tree_ensemble_handle2, - self._seed, [self._dense_float_tensor], [ - self._sparse_float_indices1, self._sparse_float_indices2 - ], [self._sparse_float_values1, self._sparse_float_values2], - [self._sparse_float_shape1, - self._sparse_float_shape2], [self._sparse_int_indices1], - [self._sparse_int_values1], [self._sparse_int_shape1], - learner_config=learner_config.SerializeToString(), - apply_dropout=False, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - - # Re-serialize tree. - stamp_token2, serialized_config2 = model_ops.tree_ensemble_serialize( - tree_ensemble_handle2) - - # The first example will get bias class 1 -0.2 from first tree and - # leaf 2 payload (sparse feature missing) of 0.5 hence [0.5, -0.2], - # the second example will get the same bias class 1 -0.2 and leaf 3 - # payload of class 1 1.2 hence [0.0, 1.0]. - self.assertEqual(stamp_token2.eval(), 9) - - # Class 2 does have scores in the leaf => it gets score 0. - self.assertEqual(serialized_config2.eval(), serialized_config) - self.assertAllClose(result.eval(), [[0.5, -0.2], [0, 1.0]]) - - def testRestore(self): - # Calling self.cached_session() without a graph specified results in - # TensorFlowTestCase caching the session and returning the same one - # every time. In this test, we need to create two different sessions - # which is why we also create a graph and pass it to self.cached_session() - # to ensure no caching occurs under the hood. - save_path = os.path.join(self.get_temp_dir(), "restore-test") - with ops.Graph().as_default() as graph: - with self.session(graph) as sess: - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - - # Add the first tree and save. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - tree = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - tree_ensemble_config.tree_weights.append(1.0) - _append_to_leaf(tree.nodes.add().leaf, 0, -0.1) - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=3, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="restore_tree") - resources.initialize_resources(resources.shared_resources()).run() - variables.global_variables_initializer().run() - my_saver = saver.Saver() - - # Add the second tree and replace the ensemble of the handle. - tree2 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_weights.append(1.0) - _append_to_leaf(tree2.nodes.add().leaf, 0, -1.0) - # Predict to confirm. - with ops.control_dependencies([ - model_ops.tree_ensemble_deserialize( - tree_ensemble_handle, - stamp_token=3, - tree_ensemble_config=tree_ensemble_config.SerializeToString()) - ]): - result, _ = prediction_ops.gradient_trees_prediction( - tree_ensemble_handle, - self._seed, [self._dense_float_tensor], [ - self._sparse_float_indices1, self._sparse_float_indices2 - ], [self._sparse_float_values1, self._sparse_float_values2], - [self._sparse_float_shape1, - self._sparse_float_shape2], [self._sparse_int_indices1], - [self._sparse_int_values1], [self._sparse_int_shape1], - learner_config=learner_config.SerializeToString(), - apply_dropout=False, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - self.assertAllClose([[-1.1], [-1.1]], result.eval()) - # Save before adding other trees. - val = my_saver.save(sess, save_path) - self.assertEqual(save_path, val) - - # Add more trees after saving. - tree3 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_weights.append(1.0) - _append_to_leaf(tree3.nodes.add().leaf, 0, -10.0) - # Predict to confirm. - with ops.control_dependencies([ - model_ops.tree_ensemble_deserialize( - tree_ensemble_handle, - stamp_token=3, - tree_ensemble_config=tree_ensemble_config.SerializeToString()) - ]): - result, _ = prediction_ops.gradient_trees_prediction( - tree_ensemble_handle, - self._seed, [self._dense_float_tensor], [ - self._sparse_float_indices1, self._sparse_float_indices2 - ], [self._sparse_float_values1, self._sparse_float_values2], - [self._sparse_float_shape1, - self._sparse_float_shape2], [self._sparse_int_indices1], - [self._sparse_int_values1], [self._sparse_int_shape1], - learner_config=learner_config.SerializeToString(), - apply_dropout=False, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - self.assertAllClose(result.eval(), [[-11.1], [-11.1]]) - - # Start a second session. In that session the parameter nodes - # have not been initialized either. - with ops.Graph().as_default() as graph: - with self.session(graph) as sess: - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="restore_tree") - my_saver = saver.Saver() - my_saver.restore(sess, save_path) - result, _ = prediction_ops.gradient_trees_prediction( - tree_ensemble_handle, - self._seed, [self._dense_float_tensor], [ - self._sparse_float_indices1, self._sparse_float_indices2 - ], [self._sparse_float_values1, self._sparse_float_values2], - [self._sparse_float_shape1, - self._sparse_float_shape2], [self._sparse_int_indices1], - [self._sparse_int_values1], [self._sparse_int_shape1], - learner_config=learner_config.SerializeToString(), - apply_dropout=False, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - # Make sure we only have the first and second tree. - # The third tree was added after the save. - self.assertAllClose(result.eval(), [[-1.1], [-1.1]]) - - def testUsedHandlers(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - tree_ensemble_config.growing_metadata.used_handler_ids.append(1) - tree_ensemble_config.growing_metadata.used_handler_ids.append(5) - stamp_token = 3 - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=stamp_token, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="create_tree") - resources.initialize_resources(resources.shared_resources()).run() - result = model_ops.tree_ensemble_used_handlers( - tree_ensemble_handle, stamp_token, num_all_handlers=6) - self.assertAllEqual([0, 1, 0, 0, 0, 1], result.used_handlers_mask.eval()) - self.assertEqual(2, result.num_used_handlers.eval()) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py deleted file mode 100644 index 46dfbdefeb0..00000000000 --- a/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py +++ /dev/null @@ -1,1439 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the GTFlow prediction Ops. - -The tests cover tree traversal and additive models for single and -multi class problems. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.boosted_trees.proto import tree_config_pb2 -from tensorflow.contrib.boosted_trees.python.ops import model_ops -from tensorflow.contrib.boosted_trees.python.ops import prediction_ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import resources -from tensorflow.python.platform import googletest - - -def _append_to_leaf(leaf, c_id, w): - """Helper method for building tree leaves. - - Appends weight contributions for the given class index to a leaf node. - - Args: - leaf: leaf node to append to. - c_id: class Id for the weight update. - w: weight contribution value. - """ - leaf.sparse_vector.index.append(c_id) - leaf.sparse_vector.value.append(w) - - -def _append_multi_values_to_leaf(leaf, c_ids, w): - """Helper method for building tree leaves with sparse vector of values. - - Appends weight contributions for the given class index to a leaf node. - - Args: - leaf: leaf node to append to. - c_ids: list of class ids - w: corresponding weight contributions for the classes in c_ids - """ - for i in range(len(c_ids)): - leaf.sparse_vector.index.append(c_ids[i]) - leaf.sparse_vector.value.append(w[i]) - - -def _append_multi_values_to_dense_leaf(leaf, w): - """Helper method for building tree leaves with dense vector of values. - - Appends weight contributions to a leaf. w is assumed to be for all classes. - - Args: - leaf: leaf node to append to. - w: corresponding weight contributions for all classes. - """ - for x in w: - leaf.vector.value.append(x) - - -def _set_float_split(split, feat_col, thresh, l_id, r_id, feature_dim_id=None): - """Helper method for building tree float splits. - - Sets split feature column, threshold and children. - - Args: - split: split node to update. - feat_col: feature column for the split. - thresh: threshold to split on forming rule x <= thresh. - l_id: left child Id. - r_id: right child Id. - feature_dim_id: dimension of the feature column to be used in the split. - """ - split.feature_column = feat_col - split.threshold = thresh - split.left_id = l_id - split.right_id = r_id - if feature_dim_id is not None: - split.dimension_id = feature_dim_id - - -def _set_float_oblivious_split(split, feat_col, thresh): - """Helper method for building tree float splits. - - Sets split feature column and threshold. - - Args: - split: split node to update. - feat_col: feature column for the split. - thresh: threshold to split on forming rule x <= thresh. - """ - split.feature_column = feat_col - split.threshold = thresh - - -def _set_categorical_id_split(split, feat_col, feat_id, l_id, r_id): - """Helper method for building tree categorical id splits. - - Sets split feature column, feature id and children. - - Args: - split: categorical id split node. - feat_col: feature column for the split. - feat_id: feature id forming rule x == id. - l_id: left child Id. - r_id: right child Id. - """ - split.feature_column = feat_col - split.feature_id = feat_id - split.left_id = l_id - split.right_id = r_id - - -class PredictionOpsTest(test_util.TensorFlowTestCase): - - def setUp(self): - """Sets up the prediction tests. - - Creates, a batch of two examples having three dense float, two sparse float - single valued, one sparse float multidimensional and one sparse int - features. The data looks like the following: - |Instance |Dense0 |Dense1 |Dense2 |SparseF0 |SparseF1 |SparseI0 |SparseM - | 0 | 7 | 1 | 2 | -3 | | 9,1 | __, 5.0 - | 1 | -2 | 2 | 0.5 | | 4 | | 3, ___ - """ - super(PredictionOpsTest, self).setUp() - self._dense_float_tensor1 = np.array([[7.0], [-2.0]]) - self._dense_float_tensor2 = np.array([[1.0], [2.0]]) - self._dense_float_tensor3 = np.array([[2.0], [0.5]]) - self._sparse_float_indices1 = np.array([[0, 0]]) - self._sparse_float_values1 = np.array([-3.0]) - self._sparse_float_shape1 = np.array([2, 1]) - self._sparse_float_indices2 = np.array([[1, 0]]) - self._sparse_float_values2 = np.array([4.0]) - self._sparse_float_shape2 = np.array([2, 1]) - # Multi dimensional sparse float - self._sparse_float_indices_m = np.array([[0, 1], [1, 0]]) - self._sparse_float_values_m = np.array([5.0, 3.0]) - self._sparse_float_shape_m = np.array([2, 2]) - - self._sparse_int_indices1 = np.array([[0, 0], [0, 1]]) - self._sparse_int_values1 = np.array([9, 1]) - self._sparse_int_shape1 = np.array([2, 2]) - self._seed = 123 - - def _get_predictions(self, - tree_ensemble_handle, - learner_config, - apply_dropout=False, - apply_averaging=False, - center_bias=False, - reduce_dim=False): - return prediction_ops.gradient_trees_prediction( - tree_ensemble_handle, - self._seed, [self._dense_float_tensor1], - [self._sparse_float_indices1, self._sparse_float_indices2], - [self._sparse_float_values1, self._sparse_float_values2], - [self._sparse_float_shape1, self._sparse_float_shape2], - [self._sparse_int_indices1], [self._sparse_int_values1], - [self._sparse_int_shape1], - learner_config=learner_config, - apply_dropout=apply_dropout, - apply_averaging=apply_averaging, - center_bias=center_bias, - reduce_dim=reduce_dim) - - def _get_predictions_oblivious_case(self, - tree_ensemble_handle, - learner_config, - apply_dropout=False, - apply_averaging=False, - center_bias=False, - reduce_dim=False): - return prediction_ops.gradient_trees_prediction( - tree_ensemble_handle, - self._seed, [ - self._dense_float_tensor1, self._dense_float_tensor2, - self._dense_float_tensor3 - ], [], [], [], [], [], [], - learner_config=learner_config, - apply_dropout=apply_dropout, - apply_averaging=apply_averaging, - center_bias=center_bias, - reduce_dim=reduce_dim) - - def testEmptyEnsemble(self): - with self.cached_session(): - # Empty tree ensenble. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="empty") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - reduce_dim=True) - self.assertAllEqual([[0], [0]], result.eval()) - # Empty dropout. - self.assertAllEqual([[], []], dropout_info.eval()) - - def testBiasEnsembleSingleClass(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - tree = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_to_leaf(tree.nodes.add().leaf, 0, -0.4) - - tree_ensemble_config.tree_weights.append(1.0) - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="bias") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - reduce_dim=True) - self.assertAllClose([[-0.4], [-0.4]], result.eval()) - - # Empty dropout. - self.assertAllEqual([[], []], dropout_info.eval()) - - def testBiasEnsembleMultiClass(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - tree = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - leaf = tree.nodes.add().leaf - _append_to_leaf(leaf, 0, -0.4) - _append_to_leaf(leaf, 1, 0.9) - - tree_ensemble_config.tree_weights.append(1.0) - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="multiclass") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 3 - - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - reduce_dim=True) - self.assertAllClose([[-0.4, 0.9], [-0.4, 0.9]], result.eval()) - - # Empty dropout. - self.assertAllEqual([[], []], dropout_info.eval()) - - def testFullEnsembleSingleClass(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Bias tree. - tree1 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_to_leaf(tree1.nodes.add().leaf, 0, -0.4) - - # Depth 3 tree. - tree2 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _set_float_split(tree2.nodes.add().dense_float_binary_split, 0, 9.0, 1, 2) - _set_float_split(tree2.nodes.add() - .sparse_float_binary_split_default_left.split, 0, -20.0, - 3, 4) - _append_to_leaf(tree2.nodes.add().leaf, 0, 0.5) - _append_to_leaf(tree2.nodes.add().leaf, 0, 1.2) - _set_categorical_id_split(tree2.nodes.add().categorical_id_binary_split, - 0, 9, 5, 6) - _append_to_leaf(tree2.nodes.add().leaf, 0, -0.9) - _append_to_leaf(tree2.nodes.add().leaf, 0, 0.7) - - tree_ensemble_config.tree_weights.append(1.0) - tree_ensemble_config.tree_weights.append(1.0) - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="full_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - reduce_dim=True) - - # The first example will get bias -0.4 from first tree and - # leaf 4 payload of -0.9 hence -1.3, the second example will - # get the same bias -0.4 and leaf 3 payload (sparse feature missing) - # of 1.2 hence 0.8. - self.assertAllClose([[-1.3], [0.8]], result.eval()) - - # Empty dropout. - self.assertAllEqual([[], []], dropout_info.eval()) - - def testObliviousEnsemble(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Bias tree. - tree1 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_to_leaf(tree1.nodes.add().leaf, 0, -0.4) - - # Depth 3 tree. - tree2 = tree_ensemble_config.trees.add() - _set_float_oblivious_split( - tree2.nodes.add().oblivious_dense_float_binary_split, 0, 5.0) - _set_float_oblivious_split( - tree2.nodes.add().oblivious_dense_float_binary_split, 1, 3.0) - _set_float_oblivious_split( - tree2.nodes.add().oblivious_dense_float_binary_split, 2, 1.0) - for i in range(1, 9): - _append_to_leaf(tree2.nodes.add().leaf, 0, i / 10.0) - - tree_ensemble_config.tree_weights.append(1.0) - tree_ensemble_config.tree_weights.append(1.0) - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="full_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - - result, dropout_info = self._get_predictions_oblivious_case( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - reduce_dim=True) - - # The first example will get bias -0.4 from first tree and 0.6 from - # the 5th leaf of the second tree corresponding to node_id = 8, hence a - # prediction of 0.2. - # The second example will get bias -0.4 and 0.1 from the 0th leaf of the - # second tree corresponding to node_id = 3, hence a prediction of -0.3 - self.assertAllClose([[0.2], [-0.3]], result.eval()) - - # Empty dropout. - self.assertAllEqual([[], []], dropout_info.eval()) - - def testFullEnsembleWithMultidimensionalSparseSingleClass(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Bias tree. - tree1 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_to_leaf(tree1.nodes.add().leaf, 0, -0.4) - - # Depth 3 tree. - tree2 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - # Use feature column 2 (sparse multidimensional), split on first value - # node 0. - _set_float_split( - tree2.nodes.add().sparse_float_binary_split_default_right.split, - 2, - 7.0, - 1, - 2, - feature_dim_id=0) - # Leafs split on second dimension of sparse multidimensional feature. - # Node 1. - _set_float_split( - tree2.nodes.add().sparse_float_binary_split_default_left.split, - 2, - 4.5, - 3, - 4, - feature_dim_id=1) - # Node 2. - _set_float_split( - tree2.nodes.add().sparse_float_binary_split_default_right.split, - 2, - 9, - 5, - 6, - feature_dim_id=1) - - # Node 3. - _append_to_leaf(tree2.nodes.add().leaf, 0, 0.6) - # Node 4. - _append_to_leaf(tree2.nodes.add().leaf, 0, 1.3) - - # Node 5. - _append_to_leaf(tree2.nodes.add().leaf, 0, -0.1) - # Node 6. - _append_to_leaf(tree2.nodes.add().leaf, 0, 0.8) - - tree_ensemble_config.tree_weights.append(1.0) - tree_ensemble_config.tree_weights.append(1.0) - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="full_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - - result, dropout_info = prediction_ops.gradient_trees_prediction( - tree_ensemble_handle, - self._seed, [self._dense_float_tensor1], [ - self._sparse_float_indices1, self._sparse_float_indices2, - self._sparse_float_indices_m - ], [ - self._sparse_float_values1, self._sparse_float_values2, - self._sparse_float_values_m - ], [ - self._sparse_float_shape1, self._sparse_float_shape2, - self._sparse_float_shape_m - ], [self._sparse_int_indices1], [self._sparse_int_values1], - [self._sparse_int_shape1], - learner_config=learner_config.SerializeToString(), - apply_dropout=False, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - - # The first example will get bias -0.4 from first tree and - # leaf 5 payload of -0.1 hence -0.5, the second example will - # get the same bias -0.4 and leaf 3 payload (0.6) hence 0.2 - self.assertAllClose([[-0.5], [0.2]], result.eval()) - - # Empty dropout. - self.assertAllEqual([[], []], dropout_info.eval()) - - def testExcludeNonFinalTree(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Bias tree. - tree1 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_to_leaf(tree1.nodes.add().leaf, 0, -0.4) - - # Depth 3 tree. - tree2 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = False - _set_float_split(tree2.nodes.add().dense_float_binary_split, 0, 9.0, 1, 2) - _set_float_split(tree2.nodes.add() - .sparse_float_binary_split_default_left.split, 0, -20.0, - 3, 4) - _append_to_leaf(tree2.nodes.add().leaf, 0, 0.5) - _append_to_leaf(tree2.nodes.add().leaf, 0, 1.2) - _set_categorical_id_split(tree2.nodes.add().categorical_id_binary_split, - 0, 9, 5, 6) - _append_to_leaf(tree2.nodes.add().leaf, 0, -0.9) - _append_to_leaf(tree2.nodes.add().leaf, 0, 0.7) - - tree_ensemble_config.tree_weights.append(1.0) - tree_ensemble_config.tree_weights.append(1.0) - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="full_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - reduce_dim=True) - - # All the examples should get only the bias since the second tree is - # non-finalized - self.assertAllClose([[-0.4], [-0.4]], result.eval()) - - # Empty dropout. - self.assertAllEqual([[], []], dropout_info.eval()) - - def testIncludeNonFinalTree(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Bias tree. - tree1 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_to_leaf(tree1.nodes.add().leaf, 0, -0.4) - - # Depth 3 tree. - tree2 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = False - _set_float_split(tree2.nodes.add().dense_float_binary_split, 0, 9.0, 1, 2) - _set_float_split(tree2.nodes.add() - .sparse_float_binary_split_default_left.split, 0, -20.0, - 3, 4) - _append_to_leaf(tree2.nodes.add().leaf, 0, 0.5) - _append_to_leaf(tree2.nodes.add().leaf, 0, 1.2) - _set_categorical_id_split(tree2.nodes.add().categorical_id_binary_split, - 0, 9, 5, 6) - _append_to_leaf(tree2.nodes.add().leaf, 0, -0.9) - _append_to_leaf(tree2.nodes.add().leaf, 0, 0.7) - - tree_ensemble_config.tree_weights.append(1.0) - tree_ensemble_config.tree_weights.append(1.0) - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="full_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.growing_mode = learner_pb2.LearnerConfig.LAYER_BY_LAYER - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - reduce_dim=True) - - # The first example will get bias -0.4 from first tree and - # leaf 4 payload of -0.9 hence -1.3, the second example will - # get the same bias -0.4 and leaf 3 payload (sparse feature missing) - # of 1.2 hence 0.8. Note that the non-finalized tree is included. - self.assertAllClose([[-1.3], [0.8]], result.eval()) - - # Empty dropout. - self.assertAllEqual([[], []], dropout_info.eval()) - - def testMetadataMissing(self): - # Sometimes we want to do prediction on trees that are not added to ensemble - # (for example in - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Bias tree. - tree1 = tree_ensemble_config.trees.add() - _append_to_leaf(tree1.nodes.add().leaf, 0, -0.4) - - # Depth 3 tree. - tree2 = tree_ensemble_config.trees.add() - # We are not setting the tree_ensemble_config.tree_metadata in this test. - _set_float_split(tree2.nodes.add().dense_float_binary_split, 0, 9.0, 1, 2) - _set_float_split(tree2.nodes.add() - .sparse_float_binary_split_default_left.split, 0, -20.0, - 3, 4) - _append_to_leaf(tree2.nodes.add().leaf, 0, 0.5) - _append_to_leaf(tree2.nodes.add().leaf, 0, 1.2) - _set_categorical_id_split(tree2.nodes.add().categorical_id_binary_split, - 0, 9, 5, 6) - _append_to_leaf(tree2.nodes.add().leaf, 0, -0.9) - _append_to_leaf(tree2.nodes.add().leaf, 0, 0.7) - - tree_ensemble_config.tree_weights.append(1.0) - tree_ensemble_config.tree_weights.append(1.0) - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="full_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - reduce_dim=True) - - # The first example will get bias -0.4 from first tree and - # leaf 4 payload of -0.9 hence -1.3, the second example will - # get the same bias -0.4 and leaf 3 payload (sparse feature missing) - # of 1.2 hence 0.8. - self.assertAllClose([[-1.3], [0.8]], result.eval()) - - # Empty dropout. - self.assertAllEqual([[], []], dropout_info.eval()) - - # For TREE_PER_CLASS strategy, predictions size is num_classes-1 - def testFullEnsembleMultiClassTreePerClassStrategy(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Bias tree only for second class. - tree1 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_to_leaf(tree1.nodes.add().leaf, 1, -0.2) - - # Depth 2 tree. - tree2 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _set_float_split(tree2.nodes.add() - .sparse_float_binary_split_default_right.split, 1, 4.0, - 1, 2) - _set_float_split(tree2.nodes.add().dense_float_binary_split, 0, 9.0, 3, 4) - _append_to_leaf(tree2.nodes.add().leaf, 0, 0.5) - _append_to_leaf(tree2.nodes.add().leaf, 1, 1.2) - _append_to_leaf(tree2.nodes.add().leaf, 0, -0.9) - - tree_ensemble_config.tree_weights.append(1.0) - tree_ensemble_config.tree_weights.append(1.0) - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="ensemble_multi_class") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 3 - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.TREE_PER_CLASS) - - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - reduce_dim=True) - # The first example will get bias class 1 -0.2 from first tree and - # leaf 2 payload (sparse feature missing) of 0.5 hence [0.5, -0.2], - # the second example will get the same bias class 1 -0.2 and leaf 3 - # payload of class 1 1.2 hence [0.0, 1.0]. - self.assertAllClose([[0.5, -0.2], [0, 1.0]], result.eval()) - - # Empty dropout. - self.assertAllEqual([[], []], dropout_info.eval()) - - # For tree-per-class multiclass handling strategies, predictions vec - # will have the size of the number of classes. - # This test is when leafs have SPARSE weights stored (class id and - # contribution). - def testFullEnsembleMultiNotClassTreePerClassStrategySparseVector(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Bias tree only for second class. - tree1 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_to_leaf(tree1.nodes.add().leaf, 1, -0.2) - - # Depth 2 tree. - tree2 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _set_float_split(tree2.nodes.add() - .sparse_float_binary_split_default_right.split, 1, 4.0, - 1, 2) - _set_float_split(tree2.nodes.add().dense_float_binary_split, 0, 9.0, 3, 4) - _append_to_leaf(tree2.nodes.add().leaf, 0, 0.5) - _append_multi_values_to_leaf(tree2.nodes.add().leaf, [1, 2], [1.2, -0.7]) - _append_to_leaf(tree2.nodes.add().leaf, 0, -0.9) - - tree_ensemble_config.tree_weights.append(1.0) - tree_ensemble_config.tree_weights.append(1.0) - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="ensemble_multi_class") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 3 - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.FULL_HESSIAN) - - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - reduce_dim=False) - # The first example will get bias class 1 -0.2 from first tree and - # leaf 2 payload (sparse feature missing) of 0.5 hence [0.5, -0.2], - # the second example will get the same bias class 1 -0.2 and leaf 3 - # payload of class 1 1.2 and class 2-0.7 hence [0.0, 1.0, -0.7]. - self.assertAllClose([[0.5, -0.2, 0.0], [0, 1.0, -0.7]], result.eval()) - - # Empty dropout. - self.assertAllEqual([[], []], dropout_info.eval()) - - # For all non-tree-per class multiclass handling strategies, predictions vec - # will have the size of the number of classes. - # This test is when leafs have DENSE weights stored (weight for each class) - def testFullEnsembleMultiNotClassTreePerClassStrategyDenseVector(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Bias tree only for second class. - tree1 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_multi_values_to_dense_leaf(tree1.nodes.add().leaf, [0, -0.2, -2]) - - # Depth 2 tree. - tree2 = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _set_float_split(tree2.nodes.add() - .sparse_float_binary_split_default_right.split, 1, 4.0, - 1, 2) - _set_float_split(tree2.nodes.add().dense_float_binary_split, 0, 9.0, 3, 4) - _append_multi_values_to_dense_leaf(tree2.nodes.add().leaf, [0.5, 0, 0]) - _append_multi_values_to_dense_leaf(tree2.nodes.add().leaf, [0, 1.2, -0.7]) - _append_multi_values_to_dense_leaf(tree2.nodes.add().leaf, [-0.9, 0, 0]) - - tree_ensemble_config.tree_weights.append(1.0) - tree_ensemble_config.tree_weights.append(1.0) - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="ensemble_multi_class") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 3 - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.FULL_HESSIAN) - - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - reduce_dim=False) - # The first example will get bias class 1 -0.2 and -2 for class 2 from - # first tree and leaf 2 payload (sparse feature missing) of 0.5 hence - # 0.5, -0.2], the second example will get the same bias and leaf 3 payload - # of class 1 1.2 and class 2-0.7 hence [0.0, 1.0, -2.7]. - self.assertAllClose([[0.5, -0.2, -2.0], [0, 1.0, -2.7]], result.eval()) - - # Empty dropout. - self.assertAllEqual([[], []], dropout_info.eval()) - - def testDropout(self): - with self.cached_session(): - # Empty tree ensenble. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Add 1000 trees with some weights. - for i in range(0, 999): - tree = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_to_leaf(tree.nodes.add().leaf, 0, -0.4) - tree_ensemble_config.tree_weights.append(i + 1) - - # Prepare learner/dropout config. - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.dropout.dropout_probability = 0.5 - learner_config.learning_rate_tuner.dropout.learning_rate = 1.0 - learner_config.num_classes = 2 - - # Apply dropout. - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="existing") - resources.initialize_resources(resources.shared_resources()).run() - - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - apply_dropout=True, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - - # We expect approx 500 trees were dropped. - dropout_info = dropout_info.eval() - self.assertIn(dropout_info[0].size, range(400, 601)) - self.assertEqual(dropout_info[0].size, dropout_info[1].size) - - for i in range(dropout_info[0].size): - dropped_index = dropout_info[0][i] - dropped_weight = dropout_info[1][i] - # We constructed the trees so tree number + 1 is the tree weight, so - # we can check here the weights for dropped trees. - self.assertEqual(dropped_index + 1, dropped_weight) - - # Don't apply dropout. - result_no_dropout, no_dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - apply_dropout=False, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - - self.assertEqual(result.eval().size, result_no_dropout.eval().size) - for i in range(result.eval().size): - self.assertNotEqual(result.eval()[i], result_no_dropout.eval()[i]) - - # We expect none of the trees were dropped. - self.assertAllEqual([[], []], no_dropout_info.eval()) - - def testDropoutCenterBiasNoGrowingMeta(self): - # This is for normal non-batch mode where ensemble does not contain the tree - # that is being built currently. - num_trees = 10 - with self.cached_session(): - # Empty tree ensemble. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Add 10 trees with some weights. - for i in range(0, num_trees): - tree = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_to_leaf(tree.nodes.add().leaf, 0, -0.4) - tree_ensemble_config.tree_weights.append(i + 1) - - # Prepare learner/dropout config. - learner_config = learner_pb2.LearnerConfig() - # Drop all the trees. - learner_config.learning_rate_tuner.dropout.dropout_probability = 1.0 - learner_config.learning_rate_tuner.dropout.learning_rate = 1.0 - learner_config.num_classes = 2 - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="existing") - resources.initialize_resources(resources.shared_resources()).run() - - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - apply_dropout=True, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - - result_center, dropout_info_center = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - apply_dropout=True, - apply_averaging=False, - center_bias=True, - reduce_dim=True) - - dropout_info = dropout_info.eval() - dropout_info_center = dropout_info_center.eval() - - # With centering, the bias tree is not dropped. - num_dropped = dropout_info[0].size - self.assertEqual(num_dropped, num_trees) - num_dropped_center = dropout_info_center[0].size - self.assertEqual(num_dropped_center, num_trees - 1) - - result = result.eval() - result_center = result_center.eval() - for i in range(result.size): - self.assertNotEqual(result[i], result_center[i]) - - # First dropped tree is a bias tree 0. - self.assertEqual(0, dropout_info[0][0]) - # Last dropped tree is the last tree. - self.assertEqual(num_trees - 1, dropout_info[0][num_dropped - 1]) - - # First dropped tree is a tree 1. - self.assertEqual(1, dropout_info_center[0][0]) - # Last dropped tree is the last tree. - self.assertEqual(num_trees - 1, dropout_info_center[0][num_dropped_center - - 1]) - - def testDropoutCenterBiasWithGrowingMeta(self): - # This is batch mode where ensemble already contains the tree that we are - # building. This tree should never be dropped. - num_trees = 10 - with self.cached_session(): - # Empty tree ensemble. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Add 10 trees with some weights. - for i in range(0, num_trees): - tree = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_to_leaf(tree.nodes.add().leaf, 0, -0.4) - tree_ensemble_config.tree_weights.append(i + 1) - - # Add growing metadata to indicate batch mode. - tree_ensemble_config.growing_metadata.num_trees_attempted = num_trees - tree_ensemble_config.growing_metadata.num_layers_attempted = num_trees - - # Prepare learner/dropout config. - learner_config = learner_pb2.LearnerConfig() - # Drop all the trees. - learner_config.learning_rate_tuner.dropout.dropout_probability = 1.0 - learner_config.learning_rate_tuner.dropout.learning_rate = 1.0 - learner_config.num_classes = 2 - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="existing") - resources.initialize_resources(resources.shared_resources()).run() - - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - apply_dropout=True, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - - result_center, dropout_info_center = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - apply_dropout=True, - apply_averaging=False, - center_bias=True, - reduce_dim=True) - - dropout_info = dropout_info.eval() - dropout_info_center = dropout_info_center.eval() - - # Last tree is never dropped, the bias tree can be dropped. - num_dropped = dropout_info[0].size - self.assertEqual(num_dropped, num_trees - 1) - num_dropped_center = dropout_info_center[0].size - self.assertEqual(num_dropped_center, num_trees - 2) - - result = result.eval() - result_center = result_center.eval() - for i in range(result.size): - self.assertNotEqual(result[i], result_center[i]) - - # First dropped tree is a bias tree 0. - self.assertEqual(0, dropout_info[0][0]) - # Last dropped tree is not the last tree (not tree num_trees-1). - self.assertNotEqual(num_trees - 1, dropout_info[0][num_dropped - 1]) - # First dropped tree is a tree 1. - self.assertEqual(1, dropout_info_center[0][0]) - # Last dropped tree is not the last tree in ensemble. - self.assertNotEqual(num_trees - 1, - dropout_info_center[0][num_dropped_center - 1]) - - def testDropoutSeed(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Add 10 trees with some weights. - for i in range(0, 999): - tree = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_to_leaf(tree.nodes.add().leaf, 0, -0.4) - tree_ensemble_config.tree_weights.append(i + 1) - - # Prepare learner/dropout config. - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.dropout.dropout_probability = 0.5 - learner_config.learning_rate_tuner.dropout.learning_rate = 1.0 - learner_config.num_classes = 2 - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="empty") - resources.initialize_resources(resources.shared_resources()).run() - - _, dropout_info_1 = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - apply_dropout=True, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - - _, dropout_info_2 = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - apply_dropout=True, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - - # Different seed. - _, dropout_info_3 = prediction_ops.gradient_trees_prediction( - tree_ensemble_handle, - 112314, [self._dense_float_tensor1], - [self._sparse_float_indices1, self._sparse_float_indices2], - [self._sparse_float_values1, self._sparse_float_values2], - [self._sparse_float_shape1, self._sparse_float_shape2], - [self._sparse_int_indices1], [self._sparse_int_values1], - [self._sparse_int_shape1], - learner_config=learner_config.SerializeToString(), - apply_dropout=True, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - - # First seed with centering bias. - _, dropout_info_4 = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - apply_dropout=True, - apply_averaging=False, - center_bias=True, - reduce_dim=True) - - # The same seed returns the same results. - self.assertAllEqual(dropout_info_1.eval(), dropout_info_2.eval()) - # Different seeds give diff results. - self.assertNotEqual(dropout_info_3.eval().shape, - dropout_info_2.eval().shape) - # With centering bias and the same seed does not give the same result. - self.assertNotEqual(dropout_info_4.eval(), dropout_info_1.eval()) - # With centering bias has 1 less tree dropped (bias tree is not dropped). - self.assertEqual( - len(dropout_info_4.eval()[0]) + 1, len(dropout_info_1.eval()[0])) - - def testDropOutZeroProb(self): - with self.cached_session(): - # Empty tree ensemble. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Add 1000 trees with some weights. - for i in range(0, 999): - tree = tree_ensemble_config.trees.add() - tree_ensemble_config.tree_metadata.add().is_finalized = True - _append_to_leaf(tree.nodes.add().leaf, 0, -0.4) - tree_ensemble_config.tree_weights.append(i + 1) - - # Dropout with 0 probability. - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.dropout.dropout_probability = 0.0 - learner_config.learning_rate_tuner.dropout.learning_rate = 1.0 - learner_config.num_classes = 2 - - # Apply dropout, but expect nothing dropped. - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="existing") - resources.initialize_resources(resources.shared_resources()).run() - - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - apply_dropout=True, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - - result_no_dropout, _ = self._get_predictions( - tree_ensemble_handle, - learner_config=learner_config.SerializeToString(), - apply_dropout=False, - apply_averaging=False, - center_bias=False, - reduce_dim=True) - - self.assertAllEqual([[], []], dropout_info.eval()) - self.assertAllClose(result.eval(), result_no_dropout.eval()) - - def testAveragingAllTrees(self): - with self.cached_session(): - # Empty tree ensemble. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - adjusted_tree_ensemble_config = ( - tree_config_pb2.DecisionTreeEnsembleConfig()) - # Add 100 trees with some weights. - # When averaging is applied, the tree weights will essentially change to - # 1, 98/99, 97/99 etc, so lets create the ensemble with such weights. - # too - total_num = 100 - for i in range(0, total_num): - tree = tree_ensemble_config.trees.add() - _append_to_leaf(tree.nodes.add().leaf, 0, -0.4) - - tree_ensemble_config.tree_metadata.add().is_finalized = True - tree_ensemble_config.tree_weights.append(1.0) - # This is how the weight will look after averaging - copy_tree = adjusted_tree_ensemble_config.trees.add() - _append_to_leaf(copy_tree.nodes.add().leaf, 0, -0.4) - - adjusted_tree_ensemble_config.tree_metadata.add().is_finalized = True - adjusted_tree_ensemble_config.tree_weights.append( - 1.0 * (total_num - i) / total_num) - - # Prepare learner config WITH AVERAGING. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.averaging_config.average_last_percent_trees = 1.0 - - # No averaging config. - learner_config_no_averaging = learner_pb2.LearnerConfig() - learner_config_no_averaging.num_classes = 2 - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="existing") - - # This is how our ensemble will "look" during averaging - adjusted_tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=adjusted_tree_ensemble_config.SerializeToString( - ), - name="adjusted") - - resources.initialize_resources(resources.shared_resources()).run() - - # Do averaging. - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config.SerializeToString(), - apply_averaging=True, - reduce_dim=True) - - pattern_result, pattern_dropout_info = self._get_predictions( - adjusted_tree_ensemble_handle, - learner_config_no_averaging.SerializeToString(), - apply_averaging=False, - reduce_dim=True) - - self.assertAllEqual(result.eval(), pattern_result.eval()) - self.assertAllEqual(dropout_info.eval(), pattern_dropout_info.eval()) - - def testAveragingSomeTrees(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - adjusted_tree_ensemble_config = ( - tree_config_pb2.DecisionTreeEnsembleConfig()) - # Add 1000 trees with some weights. - total_num = 100 - num_averaged = 25 - j = 0 - for i in range(0, total_num): - tree = tree_ensemble_config.trees.add() - _append_to_leaf(tree.nodes.add().leaf, 0, -0.4) - - tree_ensemble_config.tree_metadata.add().is_finalized = True - tree_ensemble_config.tree_weights.append(1.0) - - # This is how the weight will look after averaging - we are adjusting - # the weights of the last 25 trees - copy_tree = adjusted_tree_ensemble_config.trees.add() - _append_to_leaf(copy_tree.nodes.add().leaf, 0, -0.4) - - adjusted_tree_ensemble_config.tree_metadata.add().is_finalized = True - if i >= 75: - adjusted_tree_ensemble_config.tree_weights.append( - 1.0 * (num_averaged - j) / num_averaged) - j += 1 - else: - adjusted_tree_ensemble_config.tree_weights.append(1.0) - - # Prepare learner config WITH AVERAGING. - learner_config_1 = learner_pb2.LearnerConfig() - learner_config_1.num_classes = 2 - learner_config_1.averaging_config.average_last_percent_trees = 0.25 - - # This is equivalent. - learner_config_2 = learner_pb2.LearnerConfig() - learner_config_2.num_classes = 2 - learner_config_2.averaging_config.average_last_n_trees = 25 - - # No averaging config. - learner_config_no_averaging = learner_pb2.LearnerConfig() - learner_config_no_averaging.num_classes = 2 - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="existing") - - # This is how our ensemble will "look" during averaging - adjusted_tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=adjusted_tree_ensemble_config.SerializeToString( - ), - name="adjusted") - - resources.initialize_resources(resources.shared_resources()).run() - - result_1, dropout_info_1 = self._get_predictions( - tree_ensemble_handle, - learner_config_1.SerializeToString(), - apply_averaging=True, - reduce_dim=True) - - result_2, dropout_info_2 = self._get_predictions( - tree_ensemble_handle, - learner_config_2.SerializeToString(), - apply_averaging=True, - reduce_dim=True) - - pattern_result, pattern_dropout_info = self._get_predictions( - adjusted_tree_ensemble_handle, - learner_config_no_averaging.SerializeToString(), - apply_averaging=False, - reduce_dim=True) - - self.assertAllEqual(result_1.eval(), pattern_result.eval()) - self.assertAllEqual(result_2.eval(), pattern_result.eval()) - - self.assertAllEqual(dropout_info_1.eval(), pattern_dropout_info.eval()) - self.assertAllEqual(dropout_info_2.eval(), pattern_dropout_info.eval()) - - def testAverageMoreThanNumTreesExist(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - adjusted_tree_ensemble_config = ( - tree_config_pb2.DecisionTreeEnsembleConfig()) - # When we say to average over more trees than possible, it is averaging - # across all trees. - total_num = 100 - for i in range(0, total_num): - tree = tree_ensemble_config.trees.add() - _append_to_leaf(tree.nodes.add().leaf, 0, -0.4) - - tree_ensemble_config.tree_metadata.add().is_finalized = True - tree_ensemble_config.tree_weights.append(1.0) - # This is how the weight will look after averaging - copy_tree = adjusted_tree_ensemble_config.trees.add() - _append_to_leaf(copy_tree.nodes.add().leaf, 0, -0.4) - - adjusted_tree_ensemble_config.tree_metadata.add().is_finalized = True - adjusted_tree_ensemble_config.tree_weights.append( - 1.0 * (total_num - i) / total_num) - - # Prepare learner config WITH AVERAGING. - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - # We have only 100 trees but we ask to average over 250. - learner_config.averaging_config.average_last_n_trees = 250 - - # No averaging config. - learner_config_no_averaging = learner_pb2.LearnerConfig() - learner_config_no_averaging.num_classes = 2 - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="existing") - - # This is how our ensemble will "look" during averaging - adjusted_tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=adjusted_tree_ensemble_config.SerializeToString( - ), - name="adjusted") - - resources.initialize_resources(resources.shared_resources()).run() - - result, dropout_info = self._get_predictions( - tree_ensemble_handle, - learner_config.SerializeToString(), - apply_averaging=True, - reduce_dim=True) - - pattern_result, pattern_dropout_info = self._get_predictions( - adjusted_tree_ensemble_handle, - learner_config_no_averaging.SerializeToString(), - apply_averaging=False, - reduce_dim=True) - - self.assertAllEqual(result.eval(), pattern_result.eval()) - self.assertAllEqual(dropout_info.eval(), pattern_dropout_info.eval()) - - -class PartitionExamplesOpsTest(test_util.TensorFlowTestCase): - - def setUp(self): - """Sets up the prediction tests. - - Create a batch of two examples having three dense float, two sparse float - and one sparse int features. - The data looks like the following: - |Instance |Dense0 |Dense1 |Dense2 |SparseF0 |SparseF1 |SparseI0 | - | 0 | 7 | 1 | 2 | -3 | | 9,1 | - | 1 | -2 | 2 | 0.5 | | 4 | | - - """ - super(PartitionExamplesOpsTest, self).setUp() - self._dense_float_tensor1 = np.array([[7.0], [-2.0]]) - self._dense_float_tensor2 = np.array([[1.0], [2.0]]) - self._dense_float_tensor3 = np.array([[2.0], [0.5]]) - self._sparse_float_indices1 = np.array([[0, 0]]) - self._sparse_float_values1 = np.array([-3.0]) - self._sparse_float_shape1 = np.array([2, 1]) - self._sparse_float_indices2 = np.array([[1, 0]]) - self._sparse_float_values2 = np.array([4.0]) - self._sparse_float_shape2 = np.array([2, 1]) - self._sparse_int_indices1 = np.array([[0, 0], [0, 1]]) - self._sparse_int_values1 = np.array([9, 1]) - self._sparse_int_shape1 = np.array([2, 2]) - - def testEnsembleEmpty(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="full_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - result = prediction_ops.gradient_trees_partition_examples( - tree_ensemble_handle, [self._dense_float_tensor1], - [self._sparse_float_indices1, self._sparse_float_indices2], - [self._sparse_float_values1, self._sparse_float_values2], - [self._sparse_float_shape1, self._sparse_float_shape2], - [self._sparse_int_indices1], [self._sparse_int_values1], - [self._sparse_int_shape1]) - - self.assertAllEqual([0, 0], result.eval()) - - def testTreeNonFinalized(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Depth 3 tree. - tree1 = tree_ensemble_config.trees.add() - _set_float_split(tree1.nodes.add().dense_float_binary_split, 0, 9.0, 1, 2) - _set_float_split(tree1.nodes.add() - .sparse_float_binary_split_default_left.split, 0, -20.0, - 3, 4) - _append_to_leaf(tree1.nodes.add().leaf, 0, 0.2) - _append_to_leaf(tree1.nodes.add().leaf, 0, 0.3) - _set_categorical_id_split(tree1.nodes.add().categorical_id_binary_split, - 0, 9, 5, 6) - _append_to_leaf(tree1.nodes.add().leaf, 0, 0.5) - _append_to_leaf(tree1.nodes.add().leaf, 0, 0.6) - - tree_ensemble_config.tree_weights.append(1.0) - tree_ensemble_config.tree_metadata.add().is_finalized = False - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="full_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - result = prediction_ops.gradient_trees_partition_examples( - tree_ensemble_handle, [self._dense_float_tensor1], - [self._sparse_float_indices1, self._sparse_float_indices2], - [self._sparse_float_values1, self._sparse_float_values2], - [self._sparse_float_shape1, self._sparse_float_shape2], - [self._sparse_int_indices1], [self._sparse_int_values1], - [self._sparse_int_shape1]) - - self.assertAllEqual([5, 3], result.eval()) - - def testTreeFinalized(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Depth 3 tree. - tree1 = tree_ensemble_config.trees.add() - _set_float_split(tree1.nodes.add().dense_float_binary_split, 0, 9.0, 1, 2) - _set_float_split(tree1.nodes.add() - .sparse_float_binary_split_default_left.split, 0, -20.0, - 3, 4) - _append_to_leaf(tree1.nodes.add().leaf, 0, 0.2) - _append_to_leaf(tree1.nodes.add().leaf, 0, 0.3) - _set_categorical_id_split(tree1.nodes.add().categorical_id_binary_split, - 0, 9, 5, 6) - _append_to_leaf(tree1.nodes.add().leaf, 0, 0.5) - _append_to_leaf(tree1.nodes.add().leaf, 0, 0.6) - - tree_ensemble_config.tree_weights.append(1.0) - tree_ensemble_config.tree_metadata.add().is_finalized = True - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="full_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - result = prediction_ops.gradient_trees_partition_examples( - tree_ensemble_handle, [self._dense_float_tensor1], - [self._sparse_float_indices1, self._sparse_float_indices2], - [self._sparse_float_values1, self._sparse_float_values2], - [self._sparse_float_shape1, self._sparse_float_shape2], - [self._sparse_int_indices1], [self._sparse_int_values1], - [self._sparse_int_shape1]) - - self.assertAllEqual([0, 0], result.eval()) - - def testObliviousTreeNonFinalized(self): - with self.cached_session(): - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - # Depth 3 tree. - tree1 = tree_ensemble_config.trees.add() - _set_float_oblivious_split( - tree1.nodes.add().oblivious_dense_float_binary_split, 0, 5.0) - _set_float_oblivious_split( - tree1.nodes.add().oblivious_dense_float_binary_split, 1, 3.0) - _set_float_oblivious_split( - tree1.nodes.add().oblivious_dense_float_binary_split, 2, 1.0) - for i in range(1, 9): - _append_to_leaf(tree1.nodes.add().leaf, 0, i / 10.0) - tree_ensemble_config.tree_weights.append(1.0) - tree_ensemble_config.tree_metadata.add().is_finalized = False - - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="full_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - result = prediction_ops.gradient_trees_partition_examples( - tree_ensemble_handle, [ - self._dense_float_tensor1, - self._dense_float_tensor2, - self._dense_float_tensor3 - ], [], [], [], [], [], []) - - # The first example goes right, left, right in the tree and the second - # example goes lef, left, left. Since the depth of the tree is 3, the - # partition id's are as follows: - # First example: 3 + 5 = 8 - # Second exampel: 3 + 0 = 3 - self.assertAllEqual([8, 3], result.eval()) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py deleted file mode 100644 index 848c42b6865..00000000000 --- a/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py +++ /dev/null @@ -1,629 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Test for checking quantile related ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import os -import tempfile - -import numpy as np - -from tensorflow.contrib.boosted_trees.proto.quantiles_pb2 import QuantileConfig -from tensorflow.contrib.boosted_trees.python.ops import quantile_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import resources -from tensorflow.python.platform import googletest -from tensorflow.python.training import saver - - -class QuantileBucketsOpTest(test_util.TensorFlowTestCase): - - def _gen_config(self, eps, num_quantiles): - config = QuantileConfig() - config.eps = eps - config.num_quantiles = num_quantiles - return config.SerializeToString() - - def testBasicQuantileBuckets(self): - """Sets up the quantile summary op test as follows. - - Create a batch of 6 examples having a dense and sparse features. SparseM is - a sparse multi-dimensional (multivalent) feature. - The data looks like this - | Instance | instance weights | Dense 0 | Sparse 0 | SparseM - | 0 | 10 | 1 | | | | - | 1 | 1 | 2 | 2 | 2 | | - | 2 | 1 | 3 | 3 | 3 | | - | 3 | 1 | 4 | 4 | | 4 | - | 4 | 1 | 4 | 5 | | 5 | - | 5 | 1 | 5 | 6 | | 6 | - """ - - dense_float_tensor_0 = constant_op.constant( - [1, 2, 3, 4, 4, 5], dtype=dtypes.float32) - sparse_indices_0 = constant_op.constant( - [[1, 0], [2, 0], [3, 0], [4, 0], [5, 0]], dtype=dtypes.int64) - sparse_values_0 = constant_op.constant( - [2, 3, 4, 5, 6], dtype=dtypes.float32) - sparse_shape_0 = constant_op.constant([6, 1], dtype=dtypes.int64) - # Multi-dimensional feature that should have the same quantiles as Sparse 0. - sparse_indices_m = constant_op.constant( - [[1, 1], [2, 0], [3, 1], [4, 1], [5, 1]], dtype=dtypes.int64) - sparse_values_m = constant_op.constant( - [2, 3, 4, 5, 6], dtype=dtypes.float32) - sparse_shape_m = constant_op.constant([6, 2], dtype=dtypes.int64) - - example_weights = constant_op.constant( - [10, 1, 1, 1, 1, 1], dtype=dtypes.float32) - - with self.cached_session(): - config = self._gen_config(0.33, 3) - dense_buckets, sparse_buckets = quantile_ops.quantile_buckets( - [dense_float_tensor_0], [sparse_indices_0, sparse_indices_m], - [sparse_values_0, sparse_values_m], [sparse_shape_0, sparse_shape_m], - example_weights=example_weights, - dense_config=[config], - sparse_config=[config, config]) - - self.assertAllEqual([1, 3, 5], dense_buckets[0].eval()) - self.assertAllEqual([2, 4, 6.], sparse_buckets[0].eval()) - # Multidimensional sparse. - self.assertAllEqual([2, 4, 6.], sparse_buckets[1].eval()) - - def testStreamingQuantileBucketsWithVaryingBatch(self): - """Sets up the quantile summary op test as follows. - - Creates batches examples with different number of inputs in each batch. - The input values are dense in the range [1 ... N] - The data looks like this: - | Batch | Start | InputList - | 1 | 1 | [1] - | 2 | 2 | [2, 3] - | 3 | 4 | [4, 5, 6] - | 4 | 7 | [7, 8, 9, 10] - | 5 | 11 | [11, 12, 13, 14, 15] - | 6 | 16 | [16, 17, 18, 19, 20, 21] - """ - - num_quantiles = 3 - with self.cached_session() as sess: - accumulator = quantile_ops.QuantileAccumulator( - init_stamp_token=0, num_quantiles=num_quantiles, - epsilon=0.001, name="q1") - resources.initialize_resources(resources.shared_resources()).run() - input_column = array_ops.placeholder(dtypes.float32) - weights = array_ops.placeholder(dtypes.float32) - update = accumulator.add_summary( - stamp_token=0, - column=input_column, - example_weights=weights) - - with self.cached_session() as sess: - for i in range(1, 23): - # start = 1, 2, 4, 7, 11, 16 ... (see comment above) - start = int((i * (i-1) / 2) + 1) - sess.run(update, - {input_column: range(start, start+i), - weights: [1] * i}) - - with self.cached_session() as sess: - sess.run(accumulator.flush(stamp_token=0, next_stamp_token=1)) - are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1)) - buckets, are_ready_flush = (sess.run( - [buckets, are_ready_flush])) - self.assertEqual(True, are_ready_flush) - self.assertEqual(num_quantiles + 1, len(buckets)) - self.assertAllEqual([1, 86., 170., 253.], buckets) - - def testStreamingQuantileBucketsLowPrecisionInput(self): - """Tests inputs that simulate low precision float16 values.""" - - num_quantiles = 3 - # set generate_quantiles to True since the test will generate fewer - # boundaries otherwise. - with self.cached_session() as sess: - accumulator = quantile_ops.QuantileAccumulator( - init_stamp_token=0, num_quantiles=num_quantiles, - epsilon=0.001, name="q1", generate_quantiles=True) - resources.initialize_resources(resources.shared_resources()).run() - input_column = array_ops.placeholder(dtypes.float32) - weights = array_ops.placeholder(dtypes.float32) - update = accumulator.add_summary( - stamp_token=0, - column=input_column, - example_weights=weights) - - with self.cached_session() as sess: - # This input is generated by integer in the range [2030, 2060] - # but represented by with float16 precision. Integers <= 2048 are - # exactly represented, whereas numbers > 2048 are rounded; and hence - # numbers > 2048 are repeated. For precision loss / rounding, see: - # https://en.wikipedia.org/wiki/Half-precision_floating-point_format. - # - # The intent of the test is not handling of float16 values, but to - # validate the number of buckets is returned, in cases where the input - # may contain repeated values. - inputs = [ - 2030.0, 2031.0, 2032.0, 2033.0, 2034.0, 2035.0, 2036.0, 2037.0, - 2038.0, 2039.0, 2040.0, 2041.0, 2042.0, 2043.0, 2044.0, 2045.0, - 2046.0, 2047.0, 2048.0, 2048.0, 2050.0, 2052.0, 2052.0, 2052.0, - 2054.0, 2056.0, 2056.0, 2056.0, 2058.0, 2060.0 - ] - sess.run(update, - {input_column: inputs, - weights: [1] * len(inputs)}) - - with self.cached_session() as sess: - sess.run(accumulator.flush(stamp_token=0, next_stamp_token=1)) - are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1)) - buckets, are_ready_flush = (sess.run( - [buckets, are_ready_flush])) - self.assertEqual(True, are_ready_flush) - self.assertEqual(num_quantiles + 1, len(buckets)) - self.assertAllEqual([2030, 2040, 2050, 2060], buckets) - - def _testStreamingQuantileBucketsHelper( - self, inputs, num_quantiles=3, expected_buckets=None): - """Helper to test quantile buckets on different inputs.""" - - # set generate_quantiles to True since the test will generate fewer - # boundaries otherwise. - with self.cached_session() as sess: - accumulator = quantile_ops.QuantileAccumulator( - init_stamp_token=0, num_quantiles=num_quantiles, - epsilon=0.001, name="q1", generate_quantiles=True) - resources.initialize_resources(resources.shared_resources()).run() - input_column = array_ops.placeholder(dtypes.float32) - weights = array_ops.placeholder(dtypes.float32) - update = accumulator.add_summary( - stamp_token=0, - column=input_column, - example_weights=weights) - - with self.cached_session() as sess: - sess.run(update, - {input_column: inputs, - weights: [1] * len(inputs)}) - - with self.cached_session() as sess: - sess.run(accumulator.flush(stamp_token=0, next_stamp_token=1)) - are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1)) - buckets, are_ready_flush = (sess.run( - [buckets, are_ready_flush])) - self.assertEqual(True, are_ready_flush) - # By default, use 3 quantiles, 4 boundaries for simplicity. - self.assertEqual(num_quantiles + 1, len(buckets)) - if expected_buckets: - self.assertAllEqual(buckets, expected_buckets) - - def testStreamingQuantileBucketsRepeatedSingleValue(self): - inputs = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] - self._testStreamingQuantileBucketsHelper(inputs) - - def testStreamingQ2antileBucketsRepeatedTwoValues(self): - inputs = [1, 1, 1, 2, 2, 2, 2, 2, 1, 1] - self._testStreamingQuantileBucketsHelper(inputs) - - def testStreamingQ2antileBucketsRepeatedTwoValuesUnbalanced(self): - inputs = [7, 7, 7, 2, 7, 7, 2, 2, 7, 7] - self._testStreamingQuantileBucketsHelper(inputs) - - def testStreamingQuantileBucketsFewerInputstThanBuckets(self): - inputs = [5] - self._testStreamingQuantileBucketsHelper(inputs) - - def testStreamingQuantileBucketsEqualDistributionInSequence(self): - # Input pattern is of the form [1, 1, 1, 2, 2, 2, 3, 3, 3, ...] - ones = 100 * [1] - inputs = [] - for i in range(1, 101): - inputs += [i * k for k in ones] - # Expect 100 equally spaced buckets. - expected_buckets = range(1, 101) - self._testStreamingQuantileBucketsHelper( - inputs, num_quantiles=99, expected_buckets=expected_buckets) - - def testStreamingQuantileBucketsEqualDistributionInterleaved(self): - # Input pattern is of the form [1, 2, 3, 1, 2, 3, 1, 2, 3, ...] - sequence = range(1, 101) - inputs = [] - for _ in range(1, 101): - inputs += sequence - # Expect 100 equally spaced buckets. - expected_buckets = range(1, 101) - self._testStreamingQuantileBucketsHelper( - inputs, num_quantiles=99, expected_buckets=expected_buckets) - - def testStreamingQuantileBuckets(self): - """Sets up the quantile summary op test as follows. - - 100 batches of data is added to the accumulator. The batches are in form: - [0 1 .. 99] - [100 101 .. 200] - ... - [9900 9901 .. 9999] - All the batches have 1 for all the example weights. - """ - with self.cached_session() as sess: - accumulator = quantile_ops.QuantileAccumulator( - init_stamp_token=0, num_quantiles=3, epsilon=0.01, name="q1") - resources.initialize_resources(resources.shared_resources()).run() - weight_placeholder = array_ops.placeholder(dtypes.float32) - dense_placeholder = array_ops.placeholder(dtypes.float32) - update = accumulator.add_summary( - stamp_token=0, - column=dense_placeholder, - example_weights=weight_placeholder) - with self.cached_session() as sess: - for i in range(100): - dense_float = np.linspace( - i * 100, (i + 1) * 100 - 1, num=100).reshape(-1, 1) - sess.run(update, { - dense_placeholder: dense_float, - weight_placeholder: np.ones(shape=(100, 1), dtype=np.float32) - }) - - with self.cached_session() as sess: - sess.run(accumulator.flush(stamp_token=0, next_stamp_token=1)) - are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1)) - buckets, are_ready_flush = (sess.run([buckets, are_ready_flush])) - self.assertEqual(True, are_ready_flush) - self.assertAllEqual([0, 3335., 6671., 9999.], buckets) - - def testStreamingQuantileBucketsTwoLevel(self): - """Sets up the quantile summary op test as follows. - - 100 batches of data is added to the accumulator. The batches are in form: - [0 1 .. 99] - [100 101 .. 200] - ... - [9900 9901 .. 9999] - All the batches have 1 for all the example weights. - """ - with self.cached_session() as sess: - accumulator = quantile_ops.QuantileAccumulator( - init_stamp_token=0, num_quantiles=3, epsilon=0.01, name="q1") - accumulator_2 = quantile_ops.QuantileAccumulator( - init_stamp_token=0, num_quantiles=3, epsilon=0.01, name="q2") - resources.initialize_resources(resources.shared_resources()).run() - weight_placeholder = array_ops.placeholder(dtypes.float32) - dense_placeholder = array_ops.placeholder(dtypes.float32) - update = accumulator.add_summary( - stamp_token=0, - column=dense_placeholder, - example_weights=weight_placeholder) - with self.cached_session() as sess: - for i in range(100): - dense_float = np.linspace( - i * 100, (i + 1) * 100 - 1, num=100).reshape(-1, 1) - sess.run(update, { - dense_placeholder: dense_float, - weight_placeholder: np.ones(shape=(100, 1), dtype=np.float32) - }) - - with self.cached_session() as sess: - summary = sess.run( - accumulator.flush_summary(stamp_token=0, next_stamp_token=1)) - sess.run( - accumulator_2.add_prebuilt_summary( - stamp_token=0, summary=constant_op.constant(summary))) - sess.run(accumulator_2.flush(stamp_token=0, next_stamp_token=1)) - are_ready_flush, buckets = (accumulator_2.get_buckets(stamp_token=1)) - buckets, are_ready_flush = (sess.run([buckets, are_ready_flush])) - self.assertEqual(True, are_ready_flush) - self.assertAllEqual([0, 3337., 6677., 9999.], buckets) - - def testSaveRestoreBeforeFlush(self): - save_dir = os.path.join(self.get_temp_dir(), "save_restore") - save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash") - - with self.session(graph=ops.Graph()) as sess: - accumulator = quantile_ops.QuantileAccumulator( - init_stamp_token=0, num_quantiles=3, epsilon=0.33, name="q0") - - save = saver.Saver() - resources.initialize_resources(resources.shared_resources()).run() - - sparse_indices_0 = constant_op.constant( - [[1, 0], [2, 1], [3, 0], [4, 2], [5, 0]], dtype=dtypes.int64) - sparse_values_0 = constant_op.constant( - [2.0, 3.0, 4.0, 5.0, 6.0], dtype=dtypes.float32) - sparse_shape_0 = constant_op.constant([6, 3], dtype=dtypes.int64) - example_weights = constant_op.constant( - [10, 1, 1, 1, 1, 1], dtype=dtypes.float32, shape=[6, 1]) - update = accumulator.add_summary( - stamp_token=0, - column=sparse_tensor.SparseTensor(sparse_indices_0, sparse_values_0, - sparse_shape_0), - example_weights=example_weights) - update.run() - save.save(sess, save_path) - reset = accumulator.flush(stamp_token=0, next_stamp_token=1) - with ops.control_dependencies([reset]): - are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1)) - buckets, are_ready_flush = (sess.run([buckets, are_ready_flush])) - self.assertEqual(True, are_ready_flush) - self.assertAllEqual([2, 4, 6.], buckets) - - with self.session(graph=ops.Graph()) as sess: - accumulator = quantile_ops.QuantileAccumulator( - init_stamp_token=0, num_quantiles=3, epsilon=0.33, name="q0") - save = saver.Saver() - - # Restore the saved values in the parameter nodes. - save.restore(sess, save_path) - are_ready_noflush = accumulator.get_buckets(stamp_token=0)[0] - with ops.control_dependencies([are_ready_noflush]): - reset = accumulator.flush(stamp_token=0, next_stamp_token=1) - - with ops.control_dependencies([reset]): - are_ready_flush, buckets = accumulator.get_buckets(stamp_token=1) - buckets, are_ready_flush, are_ready_noflush = (sess.run( - [buckets, are_ready_flush, are_ready_noflush])) - self.assertFalse(are_ready_noflush) - self.assertTrue(are_ready_flush) - self.assertAllEqual([2, 4, 6.], buckets) - - def testSaveRestoreAfterFlush(self): - save_dir = os.path.join(self.get_temp_dir(), "save_restore") - save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash") - - with self.session(graph=ops.Graph()) as sess: - accumulator = quantile_ops.QuantileAccumulator( - init_stamp_token=0, num_quantiles=3, epsilon=0.33, name="q0") - - save = saver.Saver() - resources.initialize_resources(resources.shared_resources()).run() - - example_weights = constant_op.constant( - [10, 1, 1, 1, 1, 1], dtype=dtypes.float32, shape=[6, 1]) - dense_float_tensor_0 = constant_op.constant( - [1, 2, 3, 4, 4, 5], dtype=dtypes.float32, shape=[6, 1]) - update = accumulator.add_summary( - stamp_token=0, - column=dense_float_tensor_0, - example_weights=example_weights) - update.run() - reset = accumulator.flush(stamp_token=0, next_stamp_token=1) - with ops.control_dependencies([reset]): - are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1)) - buckets, are_ready_flush = (sess.run([buckets, are_ready_flush])) - self.assertEqual(True, are_ready_flush) - self.assertAllEqual([1, 3, 5], buckets) - save.save(sess, save_path) - - with self.session(graph=ops.Graph()) as sess: - accumulator = quantile_ops.QuantileAccumulator( - init_stamp_token=0, num_quantiles=3, epsilon=0.33, name="q0") - save = saver.Saver() - - # Restore the saved values in the parameter nodes. - save.restore(sess, save_path) - are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1)) - buckets, are_ready_flush = (sess.run([buckets, are_ready_flush])) - self.assertEqual(True, are_ready_flush) - self.assertAllEqual([1, 3, 5], buckets) - - def testFixedUniform(self): - """Sets up the quantile summary op test as follows. - - Creates array dividing range [0, 1] to 1<<16 elements equally spaced - with weight of 1.0. - """ - dense_float_tensor_0 = constant_op.constant( - [(1.0 * i) / math.pow(2.0, 16) - for i in range(0, int(math.pow(2, 16)) + 1)]) - example_weights = constant_op.constant( - [1] * (int(math.pow(2, 16)) + 1), dtype=dtypes.float32) - config = self._gen_config(0.1, 10) - - with self.cached_session(): - dense_buckets, _ = quantile_ops.quantile_buckets( - [dense_float_tensor_0], [], [], [], - example_weights=example_weights, - dense_config=[config], - sparse_config=[]) - self.assertAllClose( - [0] + [(i + 1.0) / 10 for i in range(0, 10)], - dense_buckets[0].eval(), - atol=0.1) - - def testFixedNonUniform(self): - """Sets up the quantile summary op test as follows. - - Creates array dividing range [0, 1] to 1<<16 elements equally spaced - with weight same as the value. - """ - dense_float_tensor_0 = constant_op.constant( - [(1.0 * i) / math.pow(2.0, 16) - for i in range(0, int(math.pow(2, 16)) + 1)]) - example_weights = constant_op.constant( - [(1.0 * i) / math.pow(2.0, 16) - for i in range(0, int(math.pow(2, 16)) + 1)]) - - config = self._gen_config(0.1, 10) - - with self.cached_session(): - dense_buckets, _ = quantile_ops.quantile_buckets( - [dense_float_tensor_0], [], [], [], - example_weights=example_weights, - dense_config=[config], - sparse_config=[]) - self.assertAllClose( - [0] + [math.sqrt((i + 1.0) / 10) for i in range(0, 10)], - dense_buckets[0].eval(), - atol=0.1) - - -class QuantilesOpTest(test_util.TensorFlowTestCase): - - def setUp(self): - """Sets up the quantile op tests. - - Create a batch of 4 examples having 2 dense and 4 sparse features. - Fourth sparse feature is multivalent (3 dimensional) - The data looks like this - | Instance | Dense 0 | Dense 1 | Sparse 0 | Sparse 1 |Sparse 2| SparseM - | 0 | -0.1 | -1 | -2 | 0.1 | |_ ,1,_ - | 1 | 0.4 | -15 | 5.5 | | 2 |2 ,_,_ - | 2 | 3.2 | 18 | 16 | 3 | |__,_,_ - | 3 | 190 | 1000 | 17.5 | -3 | 4 |1 ,8,1 - Quantiles are: - Dense 0: (-inf,0.4], (0.4,5], (5, 190] - Dense 1: (-inf, -9], (-9,15], (15, 1000) - Sparse 0: (-inf, 5], (5,16], (16, 100] - Sparse 1: (-inf, 2], (2, 5] - Sparse 2: (-inf, 100] - SparseM: (-inf, 1], (1,2], (2,1000] - """ - super(QuantilesOpTest, self).setUp() - self._dense_float_tensor_0 = constant_op.constant( - [[-0.1], [0.4], [3.2], [190]], dtype=dtypes.float32) - self._dense_float_tensor_1 = constant_op.constant( - [[-1], [-15], [18], [1000]], dtype=dtypes.float32) - # Sparse feature 0 - self._sparse_indices_0 = constant_op.constant( - [[0, 0], [1, 0], [2, 0], [3, 0]], dtype=dtypes.int64) - self._sparse_values_0 = constant_op.constant([-2, 5.5, 16, 17.5]) - self._sparse_shape_0 = constant_op.constant([4, 1]) - # Sprase feature 1 - self._sparse_indices_1 = constant_op.constant( - [[0, 0], [2, 0], [3, 0]], dtype=dtypes.int64) - self._sparse_values_1 = constant_op.constant([0.1, 3, -3]) - self._sparse_shape_1 = constant_op.constant([4, 1]) - # Sprase feature 2 - self._sparse_indices_2 = constant_op.constant( - [[1, 0], [3, 0]], dtype=dtypes.int64) - self._sparse_values_2 = constant_op.constant([2, 4], dtype=dtypes.float32) - self._sparse_shape_2 = constant_op.constant([4, 1]) - # Sprase feature M - self._sparse_indices_m = constant_op.constant( - [[0, 1], [1, 0], [3, 0], [3, 1], [3, 2]], dtype=dtypes.int64) - self._sparse_values_m = constant_op.constant( - [1, 2, 1, 8, 1], dtype=dtypes.float32) - self._sparse_shape_m = constant_op.constant([4, 1]) - # Quantiles - self._dense_thresholds_0 = [0.4, 5, 190] - self._dense_thresholds_1 = [-9, 15, 1000] - - self._sparse_thresholds_0 = [5, 16, 100] - self._sparse_thresholds_1 = [2, 5] - self._sparse_thresholds_2 = [100] - self._sparse_thresholds_m = [1, 2, 1000] - - def testDenseFeaturesOnly(self): - with self.cached_session(): - dense_quantiles, _ = quantile_ops.quantiles( - [self._dense_float_tensor_0, self._dense_float_tensor_1], [], - [self._dense_thresholds_0, self._dense_thresholds_1], [], []) - - # Dense feature 0 - self.assertAllEqual([[0, 0], [0, 0], [1, 0], [2, 0]], - dense_quantiles[0].eval()) - # Dense feature 1 - self.assertAllEqual([[1, 0], [0, 0], [2, 0], [2, 0]], - dense_quantiles[1].eval()) - - def testSparseFeaturesOnly(self): - with self.cached_session(): - _, sparse_quantiles = quantile_ops.quantiles([], [ - self._sparse_values_0, self._sparse_values_1, self._sparse_values_2, - self._sparse_values_m - ], [], [ - self._sparse_thresholds_0, self._sparse_thresholds_1, - self._sparse_thresholds_2, self._sparse_thresholds_m - ], [ - self._sparse_indices_0, self._sparse_indices_1, - self._sparse_indices_2, self._sparse_indices_m - ]) - - self.assertAllEqual(4, len(sparse_quantiles)) - # Sparse feature 0 - self.assertAllEqual([[0, 0], [1, 0], [1, 0], [2, 0]], - sparse_quantiles[0].eval()) - # Sparse feature 1 - self.assertAllEqual([[0, 0], [1, 0], [0, 0]], sparse_quantiles[1].eval()) - # Sparse feature 2 - self.assertAllEqual([[0, 0], [0, 0]], sparse_quantiles[2].eval()) - # Multidimensional feature. - self.assertAllEqual([[0, 1], [1, 0], [0, 0], [2, 1], [0, 2]], - sparse_quantiles[3].eval()) - - def testDenseAndSparseFeatures(self): - with self.cached_session(): - dense_quantiles, sparse_quantiles = quantile_ops.quantiles( - [self._dense_float_tensor_0, self._dense_float_tensor_1], [ - self._sparse_values_0, self._sparse_values_1, - self._sparse_values_2, self._sparse_values_m - ], [self._dense_thresholds_0, self._dense_thresholds_1], [ - self._sparse_thresholds_0, self._sparse_thresholds_1, - self._sparse_thresholds_2, self._sparse_thresholds_m - ], [ - self._sparse_indices_0, self._sparse_indices_1, - self._sparse_indices_2, self._sparse_indices_m - ]) - - # Dense feature 0 - self.assertAllEqual([[0, 0], [0, 0], [1, 0], [2, 0]], - dense_quantiles[0].eval()) - # Dense feature 1 - self.assertAllEqual([[1, 0], [0, 0], [2, 0], [2, 0]], - dense_quantiles[1].eval()) - # Sparse feature 0 - self.assertAllEqual([[0, 0], [1, 0], [1, 0], [2, 0]], - sparse_quantiles[0].eval()) - # Sparse feature 1 - self.assertAllEqual([[0, 0], [1, 0], [0, 0]], sparse_quantiles[1].eval()) - # Sparse feature 2 - self.assertAllEqual([[0, 0], [0, 0]], sparse_quantiles[2].eval()) - # Multidimensional feature. - self.assertAllEqual([[0, 1], [1, 0], [0, 0], [2, 1], [0, 2]], - sparse_quantiles[3].eval()) - - def testBucketizeWithInputBoundaries(self): - with self.cached_session(): - buckets = quantile_ops.bucketize_with_input_boundaries( - input=[1, 2, 3, 4, 5], - boundaries=[3]) - self.assertAllEqual([0, 0, 1, 1, 1], buckets.eval()) - - def testBucketizeWithInputBoundaries2(self): - with self.cached_session(): - boundaries = constant_op.constant([3], dtype=dtypes.float32) - buckets = quantile_ops.bucketize_with_input_boundaries( - input=[1, 2, 3, 4, 5], - boundaries=boundaries) - self.assertAllEqual([0, 0, 1, 1, 1], buckets.eval()) - - def testBucketizeWithInputBoundaries3(self): - with self.cached_session(): - b = array_ops.placeholder(dtypes.float32) - buckets = quantile_ops.bucketize_with_input_boundaries( - input=[1, 2, 3, 4, 5], - boundaries=b) - self.assertAllEqual([0, 1, 1, 2, 2], - buckets.eval(feed_dict={b: [2, 4]})) - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/split_handler_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/split_handler_ops_test.py deleted file mode 100644 index 74917f7cdea..00000000000 --- a/tensorflow/contrib/boosted_trees/python/kernel_tests/split_handler_ops_test.py +++ /dev/null @@ -1,688 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the GTFlow split handler Ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random - -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.boosted_trees.proto import split_info_pb2 -from tensorflow.contrib.boosted_trees.python.ops import split_handler_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import googletest - - -class SplitHandlerOpsTest(test_util.TensorFlowTestCase): - - def testMakeDenseSplit(self): - """Tests split handler op.""" - with self.cached_session() as sess: - # The data looks like the following after dividing by number of steps (2). - # Gradients | Partition | Dense Quantile | - # (1.2, 0.2) | 0 | 0 | - # (-0.3, 0.19) | 0 | 1 | - # (4.0, 0.13) | 1 | 1 | - partition_ids = array_ops.constant([0, 0, 1], dtype=dtypes.int32) - bucket_ids = array_ops.constant( - [[0, 0], [1, 0], [1, 0]], dtype=dtypes.int64) - gradients = array_ops.constant([2.4, -0.6, 8.0]) - hessians = array_ops.constant([0.4, 0.38, 0.26]) - bucket_boundaries = [0.3, 0.52] - partitions, gains, splits = ( - split_handler_ops.build_dense_inequality_splits( - num_minibatches=2, - partition_ids=partition_ids, - bucket_ids=bucket_ids, - gradients=gradients, - hessians=hessians, - bucket_boundaries=bucket_boundaries, - l1_regularization=0.1, - l2_regularization=1, - tree_complexity_regularization=0, - min_node_weight=0, - class_id=-1, - feature_column_group_id=0, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE)) - partitions, gains, splits = sess.run([partitions, gains, splits]) - self.assertAllEqual([0, 1], partitions) - - # Check the split on partition 0. - # -(1.2 - 0.1) / (0.2 + 1) - expected_left_weight = -0.91666 - - # expected_left_weight * -(1.2 - 0.1) - expected_left_gain = 1.0083333333333331 - - # (-0.3 + 0.1) / (0.19 + 1) - expected_right_weight = 0.1680672 - - # expected_right_weight * -(-0.3 + 0.1) - expected_right_gain = 0.033613445378151252 - - # (-0.3 + 1.2 - 0.1) ** 2 / (0.19 + 0.2 + 1) - expected_bias_gain = 0.46043165467625885 - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.dense_float_binary_split - self.assertAllClose( - expected_left_gain + expected_right_gain - expected_bias_gain, gains[0], - 0.00001) - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - self.assertEqual(0, split_node.feature_column) - self.assertAllClose(0.3, split_node.threshold, 0.00001) - - # Check the split on partition 1. - # (-4 + 0.1) / (0.13 + 1) - expected_left_weight = -3.4513274336283186 - expected_right_weight = 0 - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.dense_float_binary_split - # There's only one active bucket here so zero gain is expected. - self.assertAllClose(0.0, gains[1], 0.00001) - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - self.assertEqual(0, split_node.feature_column) - self.assertAllClose(0.52, split_node.threshold, 0.00001) - - def testMakeMulticlassDenseSplit(self): - """Tests split handler op.""" - with self.cached_session() as sess: - partition_ids = array_ops.constant([0, 0, 1], dtype=dtypes.int32) - bucket_ids = array_ops.constant( - [[0, 0], [1, 0], [1, 0]], dtype=dtypes.int64) - gradients = array_ops.constant([[2.4, 3.0], [-0.6, 0.1], [8.0, 1.0]]) - hessians = array_ops.constant([[[0.4, 1], [1, 1]], [[0.38, 1], [1, 1]], - [[0.26, 1], [1, 1]]]) - bucket_boundaries = [0.3, 0.52] - partitions, gains, splits = ( - split_handler_ops.build_dense_inequality_splits( - num_minibatches=2, - partition_ids=partition_ids, - bucket_ids=bucket_ids, - gradients=gradients, - hessians=hessians, - bucket_boundaries=bucket_boundaries, - l1_regularization=0, - l2_regularization=1, - tree_complexity_regularization=0, - min_node_weight=0, - class_id=-1, - feature_column_group_id=0, - multiclass_strategy=learner_pb2.LearnerConfig.FULL_HESSIAN, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE)) - partitions, gains, splits = sess.run([partitions, gains, splits]) - self.assertAllEqual([0, 1], partitions) - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.dense_float_binary_split - - # Each leaf has 2 element vector. - self.assertEqual(2, len(left_child.value)) - self.assertEqual(2, len(right_child.value)) - self.assertEqual(0, split_node.feature_column) - self.assertAllClose(0.3, split_node.threshold, 1e-6) - - def testMakeDenseSplitEmptyInputs(self): - """Tests empty inputs op.""" - with self.cached_session() as sess: - partition_ids = array_ops.constant([], dtype=dtypes.int32) - bucket_ids = array_ops.constant([[]], dtype=dtypes.int64) - gradients = array_ops.constant([]) - hessians = array_ops.constant([]) - bucket_boundaries = [0.3, 0.52] - partitions, gains, splits = ( - split_handler_ops.build_dense_inequality_splits( - num_minibatches=0, - partition_ids=partition_ids, - bucket_ids=bucket_ids, - gradients=gradients, - hessians=hessians, - bucket_boundaries=bucket_boundaries, - l1_regularization=0.1, - l2_regularization=1, - tree_complexity_regularization=0, - min_node_weight=0, - class_id=-1, - feature_column_group_id=0, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE)) - partitions, gains, splits = sess.run([partitions, gains, splits]) - # .assertEmpty doesn't exist on ubuntu-contrib - self.assertEqual(0, len(partitions)) - self.assertEqual(0, len(gains)) - self.assertEqual(0, len(splits)) - - def testMakeSparseSplit(self): - """Tests split handler op.""" - with self.cached_session() as sess: - # The data looks like the following after dividing by number of steps (2). - # Gradients | Partition | bucket ID | - # (0.9, 0.39) | 0 | -1 | - # (1.2, 0.2) | 0 | 0 | - # (0.2, 0.12) | 0 | 1 | - # (4.0, 0.13) | 1 | -1 | - # (4.0, 0.13) | 1 | 1 | - partition_ids = array_ops.constant([0, 0, 0, 1, 1], dtype=dtypes.int32) - # We have only 1 dimension in our sparse feature column. - bucket_ids = array_ops.constant([-1, 0, 1, -1, 1], dtype=dtypes.int64) - dimension_ids = array_ops.constant([0, 0, 0, 0, 0], dtype=dtypes.int64) - bucket_ids = array_ops.stack([bucket_ids, dimension_ids], axis=1) - - gradients = array_ops.constant([1.8, 2.4, 0.4, 8.0, 8.0]) - hessians = array_ops.constant([0.78, 0.4, 0.24, 0.26, 0.26]) - bucket_boundaries = array_ops.constant([0.3, 0.52]) - partitions, gains, splits = ( - split_handler_ops.build_sparse_inequality_splits( - num_minibatches=2, - partition_ids=partition_ids, - bucket_ids=bucket_ids, - gradients=gradients, - hessians=hessians, - bucket_boundaries=bucket_boundaries, - l1_regularization=0, - l2_regularization=2, - tree_complexity_regularization=0, - min_node_weight=0, - feature_column_group_id=0, - bias_feature_id=-1, - class_id=-1, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS)) - partitions, gains, splits = (sess.run([partitions, gains, splits])) - self.assertAllEqual([0, 1], partitions) - self.assertEqual(2, len(splits)) - # Check the split on partition 0. - # -(0.2 + 1.2) / (0.12 + 0.2 + 2) - expected_left_weight = -0.603448275862069 - # (0.2 + 1.2) ** 2 / (0.12 + 0.2 + 2) - expected_left_gain = 0.8448275862068965 - # 0.5 / (0.07 + 2) - expected_right_weight = 0.24154589371980678 - # 0.5 ** 2 / (0.07 + 2) - expected_right_gain = 0.12077294685990339 - # (0.2 + 1.2 - 0.5) ** 2 / (0.12 + 0.2 + 0.07 + 2) - expected_bias_gain = 0.3389121338912133 - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.sparse_float_binary_split_default_right - self.assertAllClose( - expected_left_gain + expected_right_gain - expected_bias_gain, gains[0]) - - self.assertAllClose([expected_left_weight], left_child.value) - - self.assertAllClose([expected_right_weight], right_child.value) - - self.assertEqual(0, split_node.split.feature_column) - # Sparse is one dimensional. - self.assertEqual(0, split_node.split.dimension_id) - - self.assertAllClose(0.52, split_node.split.threshold) - - # Check the split on partition 1. - expected_left_weight = -1.8779342723004695 - expected_right_weight = 0 - - # Verify candidate for partition 1, there's only one active bucket here - # so zero gain is expected. - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.sparse_float_binary_split_default_left - - self.assertAllClose(0.0, gains[1]) - - self.assertAllClose([expected_left_weight], left_child.value) - - self.assertAllClose([expected_right_weight], right_child.value) - - self.assertEqual(0, split_node.split.feature_column) - # Sparse is one dimensional. - self.assertEqual(0, split_node.split.dimension_id) - - self.assertAllClose(0.52, split_node.split.threshold) - - def testMakeSparseSplitAllEmptyDimensions(self): - """Tests split handler op when all dimensions have only bias bucket id.""" - with self.cached_session() as sess: - # The data looks like the following after dividing by number of steps (2). - # Gradients | Partition | Dimension | bucket ID | - # (0.9, 0.39) | 0 | 0 | -1 | - # (4.0, 0.13) | 1 | 0 | -1 | - partition_ids = array_ops.constant([0, 1], dtype=dtypes.int32) - # We have only 1 dimension in our sparse feature column. - bucket_ids = array_ops.constant([[-1, 0], [-1, 0]], dtype=dtypes.int64) - gradients = array_ops.constant([1.8, 8.0]) - hessians = array_ops.constant([0.78, 0.26]) - bucket_boundaries = array_ops.constant([0.3, 0.52]) - partitions, gains, splits = ( - split_handler_ops.build_sparse_inequality_splits( - num_minibatches=2, - partition_ids=partition_ids, - bucket_ids=bucket_ids, - gradients=gradients, - hessians=hessians, - bucket_boundaries=bucket_boundaries, - l1_regularization=0, - l2_regularization=2, - tree_complexity_regularization=0, - min_node_weight=0, - feature_column_group_id=0, - bias_feature_id=-1, - class_id=-1, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS)) - partitions, gains, splits = (sess.run([partitions, gains, splits])) - self.assertEqual(0, len(partitions)) - self.assertEqual(0, len(splits)) - - def testMakeSparseMultidimensionalSplit(self): - """Tests split handler op.""" - with self.cached_session() as sess: - # Num of steps is 2. - # The feature column is three dimensional. - # First dimension has bias bucket only, the second has bias bucket and - # two valid buckets, the third has just one bias bucket and one valid - # bucket. - # Gradients | Partition | Dimension | bucket ID | - # (0.9, 0.39) | 0 | 0 | -1 | - # (1.2, 0.2) | 0 | 1 | 0 | - # (0.2, 0.12) | 0 | 1 | 2 | - # (0.1, 0.1) | 0 | 2 | 3 | - # Now second node - nothing interesting there, just one dimension. - # Second node has the same bucket ids for all dimensions. - # (4.0, 0.13) | 1 | 0 | -1 | - # (4.0, 0.13) | 1 | 2 | 3 | - - # Tree node ids. - partition_ids = array_ops.constant([0, 0, 0, 0, 1, 1], dtype=dtypes.int32) - - dimension_ids = array_ops.constant([0, 1, 1, 2, 0, 2], dtype=dtypes.int64) - bucket_ids = array_ops.constant([-1, 0, 2, 3, -1, 3], dtype=dtypes.int64) - bucket_ids = array_ops.stack([bucket_ids, dimension_ids], axis=1) - - gradients = array_ops.constant([1.8, 2.4, 0.4, 0.2, 8.0, 8.0]) - hessians = array_ops.constant([0.78, 0.4, 0.24, 0.2, 0.26, 0.26]) - bucket_boundaries = array_ops.constant([0.3, 0.52, 0.58, 0.6]) - partitions, gains, splits = ( - split_handler_ops.build_sparse_inequality_splits( - num_minibatches=2, - partition_ids=partition_ids, - bucket_ids=bucket_ids, - gradients=gradients, - hessians=hessians, - bucket_boundaries=bucket_boundaries, - l1_regularization=0, - l2_regularization=2, - tree_complexity_regularization=0, - min_node_weight=0, - feature_column_group_id=0, - bias_feature_id=-1, - class_id=-1, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS)) - partitions, gains, splits = (sess.run([partitions, gains, splits])) - self.assertAllEqual([0, 1], partitions) - self.assertEqual(2, len(splits)) - # Check the split on node 0 - it should split on second dimension - # -(0.2 + 1.2) / (0.12 + 0.2 + 2) - expected_left_weight = -0.603448275862069 - # (0.2 + 1.2) ** 2 / (0.12 + 0.2 + 2) - expected_left_gain = 0.8448275862068965 - # 0.5 / (0.07 + 2) - expected_right_weight = 0.24154589371980678 - # 0.5 ** 2 / (0.07 + 2) - expected_right_gain = 0.12077294685990339 - # (0.2 + 1.2 - 0.5) ** 2 / (0.12 + 0.2 + 0.07 + 2) - expected_bias_gain = 0.3389121338912133 - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.sparse_float_binary_split_default_right - self.assertAllClose( - expected_left_gain + expected_right_gain - expected_bias_gain, gains[0]) - - self.assertAllClose([expected_left_weight], left_child.value) - - self.assertAllClose([expected_right_weight], right_child.value) - - self.assertEqual(0, split_node.split.feature_column) - # Split happened on second dimension. - self.assertEqual(1, split_node.split.dimension_id) - - self.assertAllClose(0.58, split_node.split.threshold) - - # Check the split on partition 1. - expected_left_weight = -1.8779342723004695 - expected_right_weight = 0 - - # Verify candidate for partition 1, there's only one active bucket here - # so zero gain is expected. - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.sparse_float_binary_split_default_left - - self.assertAllClose(0.0, gains[1]) - - self.assertAllClose([expected_left_weight], left_child.value) - - self.assertAllClose([expected_right_weight], right_child.value) - - self.assertEqual(0, split_node.split.feature_column) - self.assertEqual(2, split_node.split.dimension_id) - - self.assertAllClose(0.6, split_node.split.threshold) - - def testMakeSparseSplitDefaultDirectionIsStable(self): - """Tests default direction is stable when no sparsity.""" - random.seed(1123) - for _ in range(50): - with self.cached_session() as sess: - grad = random.random() - hessian = random.random() - # The data looks like the following (divide by the num of steps 2). - # Gradients | Partition | bucket ID | - # (grad, hessian) | 0 | -1 | - # And then 100 buckets of - # (grad/100, hessian/100), so there is no sparsity. - n_buckets = 100 - - # 1 for the overall sum, and 100 buckets. - partition_ids = array_ops.constant( - [0] * (n_buckets + 1), dtype=dtypes.int32) - # We have only 1 dimension in our sparse feature column. - - bucket_ids = [-1] + [n for n in range(100)] - bucket_ids = array_ops.constant(bucket_ids, dtype=dtypes.int64) - dimension_ids = array_ops.constant( - [0] * (n_buckets + 1), dtype=dtypes.int64) - bucket_ids = array_ops.stack([bucket_ids, dimension_ids], axis=1) - - gradients = [grad] + [grad / n_buckets] * n_buckets - gradients = array_ops.constant(gradients) - hessians = [hessian] + [hessian / n_buckets] * n_buckets - hessians = array_ops.constant(hessians) - - boundaries = [x * 1 for x in range(n_buckets + 1)] - bucket_boundaries = array_ops.constant(boundaries, dtype=dtypes.float32) - - partitions, gains, splits = ( - split_handler_ops.build_sparse_inequality_splits( - num_minibatches=2, - partition_ids=partition_ids, - bucket_ids=bucket_ids, - gradients=gradients, - hessians=hessians, - bucket_boundaries=bucket_boundaries, - l1_regularization=0, - l2_regularization=2, - tree_complexity_regularization=0, - min_node_weight=0, - feature_column_group_id=0, - bias_feature_id=-1, - class_id=-1, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS)) - partitions, gains, splits = (sess.run([partitions, gains, splits])) - self.assertAllEqual([0], partitions) - self.assertEqual(1, len(splits)) - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - self.assertTrue( - split_info.split_node.HasField( - 'sparse_float_binary_split_default_left')) - - def testMakeMulticlassSparseSplit(self): - """Tests split handler op.""" - with self.cached_session() as sess: - partition_ids = array_ops.constant([0, 0, 0, 1, 1], dtype=dtypes.int32) - bucket_ids = array_ops.constant( - [[-1, 0], [0, 0], [1, 0], [-1, 0], [1, 0]], dtype=dtypes.int64) - gradients = array_ops.constant([[1.8, 3.5], [2.4, 1.0], [0.4, 4.0], - [8.0, 3.1], [8.0, 0.8]]) - - hessian_0 = [[0.78, 1], [12, 1]] - hessian_1 = [[0.4, 1], [1, 1]] - hessian_2 = [[0.24, 1], [1, 1]] - hessian_3 = [[0.26, 1], [1, 1]] - hessian_4 = [[0.26, 1], [1, 1]] - - hessians = array_ops.constant( - [hessian_0, hessian_1, hessian_2, hessian_3, hessian_4]) - bucket_boundaries = array_ops.constant([0.3, 0.52]) - partitions, gains, splits = ( - split_handler_ops.build_sparse_inequality_splits( - num_minibatches=2, - partition_ids=partition_ids, - bucket_ids=bucket_ids, - gradients=gradients, - hessians=hessians, - bucket_boundaries=bucket_boundaries, - l1_regularization=0, - l2_regularization=2, - tree_complexity_regularization=0, - min_node_weight=0, - feature_column_group_id=0, - bias_feature_id=-1, - class_id=-1, - multiclass_strategy=learner_pb2.LearnerConfig.FULL_HESSIAN)) - partitions, gains, splits = (sess.run([partitions, gains, splits])) - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.sparse_float_binary_split_default_right - - # Each leaf has 2 element vector. - self.assertEqual(2, len(left_child.value)) - self.assertEqual(2, len(right_child.value)) - - self.assertEqual(0, split_node.split.feature_column) - self.assertAllClose(0.52, split_node.split.threshold) - - def testMakeCategoricalEqualitySplit(self): - """Tests split handler op for categorical equality split.""" - with self.cached_session() as sess: - # The data looks like the following after dividing by number of steps (2). - # Gradients | Partition | Feature ID | - # (0.9, 0.39) | 0 | -1 | - # (0.2, 0.12) | 0 | 1 | - # (1.4, 0.32) | 0 | 2 | - # (4.0, 0.13) | 1 | -1 | - # (4.0, 0.13) | 1 | 1 | - gradients = [1.8, 0.4, 2.8, 8.0, 8.0] - hessians = [0.78, 0.24, 0.64, 0.26, 0.26] - partition_ids = [0, 0, 0, 1, 1] - feature_ids = array_ops.constant( - [[-1, 0], [1, 0], [2, 0], [-1, 0], [1, 0]], dtype=dtypes.int64) - partitions, gains, splits = ( - split_handler_ops.build_categorical_equality_splits( - num_minibatches=2, - partition_ids=partition_ids, - feature_ids=feature_ids, - gradients=gradients, - hessians=hessians, - l1_regularization=0.1, - l2_regularization=1, - tree_complexity_regularization=0, - min_node_weight=0, - feature_column_group_id=0, - bias_feature_id=-1, - class_id=-1, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE)) - partitions, gains, splits = sess.run([partitions, gains, splits]) - self.assertAllEqual([0, 1], partitions) - - # Check the split on partition 0. - # -(0.2 + 1.2 - 0.1) / (0.12 + 0.2 + 1) - expected_left_weight = -0.9848484848484846 - - # (0.2 + 1.2 - 0.1) ** 2 / (0.12 + 0.2 + 1) - expected_left_gain = 1.2803030303030298 - - # -(-0.5 + 0.1) / (0.07 + 1) - expected_right_weight = 0.37383177570093457 - - # (-0.5 + 0.1) ** 2 / (0.07 + 1) - expected_right_gain = 0.14953271028037385 - - # (0.2 + -0.5 + 1.2 - 0.1) ** 2 / (0.12 + 0.07 + 0.2 + 1) - expected_bias_gain = 0.46043165467625885 - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[0]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.categorical_id_binary_split - - self.assertEqual(0, split_node.feature_column) - - self.assertEqual(2, split_node.feature_id) - - self.assertAllClose( - expected_left_gain + expected_right_gain - expected_bias_gain, gains[0], - 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - # Check the split on partition 1. - # (-4 + 0.1) / (0.13 + 1) - expected_left_weight = -3.4513274336283186 - # (-4 + 0.1) ** 2 / (0.13 + 1) - expected_left_gain = 13.460176991150442 - expected_right_weight = 0 - expected_right_gain = 0 - # (-4 + 0.1) ** 2 / (0.13 + 1) - expected_bias_gain = 13.460176991150442 - - # Verify candidate for partition 1, there's only one active feature here - # so zero gain is expected. - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.categorical_id_binary_split - self.assertAllClose(0.0, gains[1], 0.00001) - - self.assertAllClose([expected_left_weight], left_child.value, 0.00001) - - self.assertAllClose([expected_right_weight], right_child.value, 0.00001) - - self.assertEqual(0, split_node.feature_column) - - self.assertEqual(1, split_node.feature_id) - - def testMakeMulticlassCategoricalEqualitySplit(self): - """Tests split handler op for categorical equality split in multiclass.""" - with self.cached_session() as sess: - gradients = array_ops.constant([[1.8, 3.5], [2.4, 1.0], [0.4, 4.0], - [9.0, 3.1], [3.0, 0.8]]) - - hessian_0 = [[0.78, 1], [12, 1]] - hessian_1 = [[0.4, 1], [1, 1]] - hessian_2 = [[0.24, 1], [1, 1]] - hessian_3 = [[0.16, 2], [-1, 1]] - hessian_4 = [[0.6, 1], [2, 1]] - - hessians = array_ops.constant( - [hessian_0, hessian_1, hessian_2, hessian_3, hessian_4]) - partition_ids = [0, 0, 0, 1, 1] - feature_ids = array_ops.constant( - [[-1, 0], [1, 0], [2, 0], [-1, 0], [1, 0]], dtype=dtypes.int64) - partitions, gains, splits = ( - split_handler_ops.build_categorical_equality_splits( - num_minibatches=2, - partition_ids=partition_ids, - feature_ids=feature_ids, - gradients=gradients, - hessians=hessians, - l1_regularization=0.1, - l2_regularization=1, - tree_complexity_regularization=0, - min_node_weight=0, - feature_column_group_id=0, - bias_feature_id=-1, - class_id=-1, - multiclass_strategy=learner_pb2.LearnerConfig.FULL_HESSIAN, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE)) - partitions, gains, splits = sess.run([partitions, gains, splits]) - self.assertAllEqual([0, 1], partitions) - - split_info = split_info_pb2.SplitInfo() - split_info.ParseFromString(splits[1]) - left_child = split_info.left_child.vector - right_child = split_info.right_child.vector - split_node = split_info.split_node.categorical_id_binary_split - - # Each leaf has 2 element vector. - self.assertEqual(2, len(left_child.value)) - self.assertEqual(2, len(right_child.value)) - - self.assertEqual(0, split_node.feature_column) - self.assertEqual(1, split_node.feature_id) - - def testMakeCategoricalEqualitySplitEmptyInput(self): - with self.cached_session() as sess: - gradients = [] - hessians = [] - partition_ids = [] - feature_ids = [[]] - partitions, gains, splits = ( - split_handler_ops.build_categorical_equality_splits( - num_minibatches=0, - partition_ids=partition_ids, - feature_ids=feature_ids, - gradients=gradients, - hessians=hessians, - l1_regularization=0.1, - l2_regularization=1, - tree_complexity_regularization=0, - min_node_weight=0, - feature_column_group_id=0, - bias_feature_id=-1, - class_id=-1, - multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE)) - partitions, gains, splits = (sess.run([partitions, gains, splits])) - self.assertEqual(0, len(partitions)) - self.assertEqual(0, len(gains)) - self.assertEqual(0, len(splits)) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/stats_accumulator_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/stats_accumulator_ops_test.py deleted file mode 100644 index d21a0f16621..00000000000 --- a/tensorflow/contrib/boosted_trees/python/kernel_tests/stats_accumulator_ops_test.py +++ /dev/null @@ -1,460 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Test for checking stats accumulator related ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.boosted_trees.python.ops import stats_accumulator_ops -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import test_util -from tensorflow.python.platform import googletest - - -class StatsAccumulatorScalarTest(test_util.TensorFlowTestCase): - """Tests for scalar gradients and hessians accumulator.""" - - def testSimpleAcculumator(self): - with self.cached_session() as sess: - accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=tensor_shape.TensorShape([]), - hessian_shape=tensor_shape.TensorShape([])) - with ops.control_dependencies([accumulator.initializer]): - op1 = accumulator.add( - stamp_token=0, - partition_ids=[1, 2], - feature_ids=[[2, 0], [3, 0]], - gradients=[0.1, 0.3], - hessians=[0.2, 0.4]) - op2 = accumulator.add(0, [1], [[2, 0]], [0.1], [0.2]) - - with ops.control_dependencies([op1, op2]): - num_updates, partition, bucket_ids, grads, hessians = accumulator.flush( - stamp_token=0, next_stamp_token=1) - num_updates, partition, bucket_ids, grads, hessians = sess.run( - [num_updates, partition, bucket_ids, grads, hessians]) - - result = _AccumulatorResultToDict(partition, bucket_ids, grads, hessians) - self.assertEqual(num_updates, 2) - self.assertEqual(len(result), 2) - # Key is partition, bucket, dimension - self.assertAllClose(result[(1, 2, 0)], [0.2, 0.4]) - self.assertAllClose(result[(2, 3, 0)], [0.3, 0.4]) - - def testMultidimensionalAcculumator(self): - with self.cached_session() as sess: - accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=tensor_shape.TensorShape([]), - hessian_shape=tensor_shape.TensorShape([])) - with ops.control_dependencies([accumulator.initializer]): - op1 = accumulator.add( - stamp_token=0, - partition_ids=[1, 2, 1], - feature_ids=[[2, 2], [3, 0], [2, 2]], - gradients=[0.1, 0.3, 0.8], - hessians=[0.2, 0.4, -9]) - op2 = accumulator.add(0, [2, 1], [[3, 1], [2, 2]], [0.1, 1], [0.2, -1]) - - with ops.control_dependencies([op1, op2]): - num_updates, partition, bucket_ids, grads, hessians = accumulator.flush( - stamp_token=0, next_stamp_token=1) - num_updates, partition, bucket_ids, grads, hessians = sess.run( - [num_updates, partition, bucket_ids, grads, hessians]) - - result = _AccumulatorResultToDict(partition, bucket_ids, grads, hessians) - self.assertEqual(num_updates, 2) - self.assertEqual(len(result), 3) - # Key is partition, bucket, dimension. - self.assertAllClose(result[(1, 2, 2)], [1.9, -9.8]) - self.assertAllClose(result[(2, 3, 0)], [0.3, 0.4]) - self.assertAllClose(result[(2, 3, 1)], [0.1, 0.2]) - - def testDropStaleUpdate(self): - with self.cached_session() as sess: - accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=tensor_shape.TensorShape([]), - hessian_shape=tensor_shape.TensorShape([])) - with ops.control_dependencies([accumulator.initializer]): - op1 = accumulator.add( - stamp_token=0, - partition_ids=[1, 2], - feature_ids=[[2, 0], [3, 0]], - gradients=[0.1, 0.3], - hessians=[0.2, 0.4]) - op2 = accumulator.add( - stamp_token=-1, - partition_ids=[1], - feature_ids=[[2, 0]], - gradients=[0.1], - hessians=[0.2]) - - with ops.control_dependencies([op1, op2]): - num_updates, partition, feature, grads, hessians = accumulator.flush( - stamp_token=0, next_stamp_token=1) - num_updates, partition, feature, grads, hessians = sess.run( - [num_updates, partition, feature, grads, hessians]) - - result = _AccumulatorResultToDict(partition, feature, grads, hessians) - self.assertEqual(num_updates, 1) - self.assertEqual(len(result), 2) - self.assertAllClose(result[(1, 2, 0)], [0.1, 0.2]) - self.assertAllClose(result[(2, 3, 0)], [0.3, 0.4]) - - def testSerialize(self): - with self.cached_session() as sess: - accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=tensor_shape.TensorShape([]), - hessian_shape=tensor_shape.TensorShape([])) - with ops.control_dependencies([accumulator.initializer]): - op1 = accumulator.add( - stamp_token=0, - partition_ids=[1, 2], - feature_ids=[[2, 0], [3, 0]], - gradients=[0.1, 0.3], - hessians=[0.2, 0.4]) - - with ops.control_dependencies([op1]): - (stamp_token, num_updates, partition_1, feature_1, grads_1, - hessians_1) = accumulator.saveable.serialize() - # Make sure that the accumulator hasn't changed during serialization. - with ops.control_dependencies([stamp_token]): - num_updates_2, partition_2, feature_2, grads_2, hessians_2 = ( - accumulator.flush(stamp_token=0, next_stamp_token=1)) - (stamp_token, num_updates, partition_1, feature_1, grads_1, hessians_1, - num_updates_2, partition_2, feature_2, grads_2, hessians_2) = sess.run( - [ - stamp_token, num_updates, partition_1, feature_1, grads_1, - hessians_1, num_updates_2, partition_2, feature_2, grads_2, - hessians_2 - ]) - - result_1 = _AccumulatorResultToDict(partition_1, feature_1, grads_1, - hessians_1) - result_2 = _AccumulatorResultToDict(partition_2, feature_2, grads_2, - hessians_2) - self.assertEqual(num_updates, 1) - self.assertEqual(num_updates_2, 1) - self.assertEqual(len(result_1), 2) - self.assertAllClose(result_1[(1, 2, 0)], [0.1, 0.2]) - self.assertAllClose(result_1[(2, 3, 0)], [0.3, 0.4]) - self.assertAllEqual(result_1, result_2) - self.assertEqual(0, stamp_token) - - def testDeserialize(self): - with self.cached_session() as sess: - accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=tensor_shape.TensorShape([]), - hessian_shape=tensor_shape.TensorShape([])) - with ops.control_dependencies([accumulator.initializer]): - # These will be deleted due to deserialize call. - op1 = accumulator.add( - stamp_token=0, - partition_ids=[1, 2], - feature_ids=[[2, 0], [3, 1]], - gradients=[0.1, 0.3], - hessians=[0.2, 0.4]) - - with ops.control_dependencies([op1]): - deserialize = ( - accumulator.saveable.deserialize( - stamp_token=2, - num_updates=3, - partition_ids=[3, 4], - feature_ids=[[5, 0], [6, 2]], - gradients=[0.4, 0.5], - hessians=[0.6, 0.7])) - with ops.control_dependencies([deserialize]): - num_updates, partition, feature, grads, hessians = accumulator.flush( - stamp_token=2, next_stamp_token=3) - num_updates, partition, feature, grads, hessians = sess.run( - [num_updates, partition, feature, grads, hessians]) - - result = _AccumulatorResultToDict(partition, feature, grads, - hessians) - self.assertEqual(num_updates, 3) - self.assertEqual(len(result), 2) - self.assertAllClose(result[(3, 5, 0)], [0.4, 0.6]) - self.assertAllClose(result[(4, 6, 2)], [0.5, 0.7]) - - def testMakeSummary(self): - with self.cached_session() as sess: - accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=tensor_shape.TensorShape([]), - hessian_shape=tensor_shape.TensorShape([])) - partition, feature, grads, hessians = accumulator._make_summary( - partition_ids=[1, 2, 1], - feature_ids=[[2, 0], [3, 1], [2, 0]], - gradients=[0.1, 0.3, 0.1], - hessians=[0.2, 0.4, 0.2]) - partition, feature, grads, hessians = sess.run( - [partition, feature, grads, hessians]) - result = _AccumulatorResultToDict(partition, feature, grads, hessians) - self.assertEqual(len(result), 2) - self.assertAllClose(result[(1, 2, 0)], [0.2, 0.4]) - self.assertAllClose(result[(2, 3, 1)], [0.3, 0.4]) - - -class StatsAccumulatorTensorTest(test_util.TensorFlowTestCase): - """Tests for tensor gradients and hessians accumulator.""" - - def testSimpleAcculumator(self): - with self.cached_session() as sess: - accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=tensor_shape.TensorShape([2]), - hessian_shape=tensor_shape.TensorShape([2, 2])) - with ops.control_dependencies([accumulator.initializer]): - op1 = accumulator.add( - stamp_token=0, - partition_ids=[1, 2], - feature_ids=[[2, 0], [3, 0]], - # Two values for gradients, - gradients=[[0.1, 0.1], [0.2, 0.2]], - # A 2x2 matrix for each hessian. - hessians=[[[0.01, 0.02], [0.03, 0.04]], [[0.05, 0.06], [0.07, - 0.08]]]) - op2 = accumulator.add( - stamp_token=0, - partition_ids=[1], - feature_ids=[[2, 0]], - gradients=[[0.10, 0.11]], - hessians=[[[0.011, 0.022], [0.033, 0.044]]]) - - with ops.control_dependencies([op1, op2]): - num_updates, partition, feature, grads, hessians = accumulator.flush( - stamp_token=0, next_stamp_token=1) - num_updates, partition, feature, grads, hessians = sess.run( - [num_updates, partition, feature, grads, hessians]) - - result = _AccumulatorResultToDict(partition, feature, grads, hessians) - self.assertEqual(num_updates, 2) - self.assertEqual(len(result), 2) - self.assertAllClose(result[(1, 2, 0)][0], [0.20, 0.21]) - self.assertAllClose(result[(1, 2, 0)][1], - [[0.021, 0.042], [0.063, 0.084]]) - self.assertAllClose(result[(2, 3, 0)][0], [0.2, 0.2]) - self.assertAllClose(result[(2, 3, 0)][1], [[0.05, 0.06], [0.07, 0.08]]) - - def testMultidimensionalAcculumator(self): - with self.cached_session() as sess: - accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=tensor_shape.TensorShape([2]), - hessian_shape=tensor_shape.TensorShape([2, 2])) - with ops.control_dependencies([accumulator.initializer]): - op1 = accumulator.add( - stamp_token=0, - partition_ids=[1, 2], - feature_ids=[[2, 4], [3, 1]], - # Two values for gradients, - gradients=[[0.1, 0.1], [0.2, 0.2]], - # A 2x2 matrix for each hessian. - hessians=[[[0.01, 0.02], [0.03, 0.04]], [[0.05, 0.06], [0.07, - 0.08]]]) - op2 = accumulator.add( - stamp_token=0, - partition_ids=[1], - feature_ids=[[2, 4]], - gradients=[[0.10, 0.11]], - hessians=[[[0.011, 0.022], [0.033, 0.044]]]) - - with ops.control_dependencies([op1, op2]): - num_updates, partition, feature, grads, hessians = accumulator.flush( - stamp_token=0, next_stamp_token=1) - num_updates, partition, feature, grads, hessians = sess.run( - [num_updates, partition, feature, grads, hessians]) - - result = _AccumulatorResultToDict(partition, feature, grads, hessians) - self.assertEqual(num_updates, 2) - self.assertEqual(len(result), 2) - self.assertAllClose(result[(1, 2, 4)][0], [0.20, 0.21]) - self.assertAllClose(result[(1, 2, 4)][1], - [[0.021, 0.042], [0.063, 0.084]]) - self.assertAllClose(result[(2, 3, 1)][0], [0.2, 0.2]) - self.assertAllClose(result[(2, 3, 1)][1], [[0.05, 0.06], [0.07, 0.08]]) - - def testDropStaleUpdate(self): - with self.cached_session() as sess: - accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=tensor_shape.TensorShape([2]), - hessian_shape=tensor_shape.TensorShape([2, 2])) - with ops.control_dependencies([accumulator.initializer]): - op1 = accumulator.add( - stamp_token=0, - partition_ids=[1, 2], - feature_ids=[[2, 5], [3, 0]], - # Two values for gradients, - gradients=[[0.1, 0.1], [0.2, 0.2]], - # A 2x2 matrix for each hessian. - hessians=[[[0.01, 0.02], [0.03, 0.04]], [[0.05, 0.06], [0.07, - 0.08]]]) - op2 = accumulator.add( - stamp_token=-1, - partition_ids=[1], - feature_ids=[[2, 5]], - gradients=[[0.10, 0.11]], - hessians=[[[0.011, 0.022], [0.033, 0.044]]]) - - with ops.control_dependencies([op1, op2]): - num_updates, partition, feature, grads, hessians = accumulator.flush( - stamp_token=0, next_stamp_token=1) - num_updates, partition, feature, grads, hessians = sess.run( - [num_updates, partition, feature, grads, hessians]) - - result = _AccumulatorResultToDict(partition, feature, grads, hessians) - self.assertEqual(num_updates, 1) - self.assertEqual(len(result), 2) - self.assertAllClose(result[(1, 2, 5)][0], [0.1, 0.1]) - self.assertAllClose(result[(1, 2, 5)][1], [[0.01, 0.02], [0.03, 0.04]]) - self.assertAllClose(result[(2, 3, 0)][0], [0.2, 0.2]) - self.assertAllClose(result[(2, 3, 0)][1], [[0.05, 0.06], [0.07, 0.08]]) - - def testSerialize(self): - with self.cached_session() as sess: - accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=tensor_shape.TensorShape([2]), - hessian_shape=tensor_shape.TensorShape([2, 2])) - with ops.control_dependencies([accumulator.initializer]): - op1 = accumulator.add( - stamp_token=0, - partition_ids=[1, 2], - feature_ids=[[2, 0], [3, 0]], - # Two values for gradients, - gradients=[[0.1, 0.1], [0.2, 0.2]], - # A 2x2 matrix for each hessian. - hessians=[[[0.01, 0.02], [0.03, 0.04]], [[0.05, 0.06], [0.07, - 0.08]]]) - - with ops.control_dependencies([op1]): - (stamp_token, num_updates_1, partition_1, feature_1, grads_1, - hessians_1) = accumulator.saveable.serialize() - # Make sure that the accumulator hasn't changed during serialization. - with ops.control_dependencies([stamp_token]): - num_updates_2, partition_2, feature_2, grads_2, hessians_2 = ( - accumulator.flush(stamp_token=0, next_stamp_token=1)) - (stamp_token, num_updates_1, partition_1, feature_1, grads_1, - hessians_1, num_updates_2, partition_2, feature_2, grads_2, - hessians_2) = sess.run([ - stamp_token, num_updates_1, partition_1, feature_1, grads_1, - hessians_1, num_updates_2, partition_2, feature_2, grads_2, - hessians_2 - ]) - - result_1 = _AccumulatorResultToDict(partition_1, feature_1, grads_1, - hessians_1) - result_2 = _AccumulatorResultToDict(partition_2, feature_2, grads_2, - hessians_2) - - self.assertEqual(num_updates_1, 1) - self.assertEqual(num_updates_2, 1) - self.assertEqual(len(result_1), 2) - self.assertAllClose(result_1[(1, 2, 0)][0], [0.1, 0.1]) - self.assertAllClose(result_1[(1, 2, 0)][1], [[0.01, 0.02], [0.03, 0.04]]) - self.assertAllClose(result_1[(2, 3, 0)][0], [0.2, 0.2]) - self.assertAllClose(result_1[(2, 3, 0)][1], [[0.05, 0.06], [0.07, 0.08]]) - - self.assertAllEqual(result_1[1, 2, 0][0], result_2[1, 2, 0][0]) - self.assertAllEqual(result_1[1, 2, 0][1], result_2[1, 2, 0][1]) - self.assertAllEqual(result_1[2, 3, 0][0], result_2[2, 3, 0][0]) - self.assertAllEqual(result_1[2, 3, 0][1], result_2[2, 3, 0][1]) - - def testDeserialize(self): - with self.cached_session() as sess: - accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=tensor_shape.TensorShape([2]), - hessian_shape=tensor_shape.TensorShape([2, 2])) - with ops.control_dependencies([accumulator.initializer]): - # These will be deleted due to deserialize call. - op1 = accumulator.add( - stamp_token=0, - partition_ids=[1, 2], - feature_ids=[[2, 0], [3, 0]], - # Two values for gradients, - gradients=[[0.1, 0.1], [0.2, 0.2]], - # A 2x2 matrix for each hessian. - hessians=[[[0.01, 0.02], [0.03, 0.04]], [[0.05, 0.06], [0.07, - 0.08]]]) - - with ops.control_dependencies([op1]): - deserialize = accumulator.saveable.deserialize( - stamp_token=2, - num_updates=3, - partition_ids=[3, 4], - feature_ids=[[4, 0], [5, 0]], - # Two values for gradients, - gradients=[[0.3, 0.3], [0.5, 0.5]], - # A 2x2 matrix for each hessian. - hessians=[[[0.03, 0.04], [0.05, 0.06]], [[0.07, 0.08], [0.09, - 0.10]]]) - with ops.control_dependencies([deserialize]): - num_updates, partition, feature, grads, hessians = accumulator.flush( - stamp_token=2, next_stamp_token=3) - num_updates, partition, feature, grads, hessians = sess.run( - [num_updates, partition, feature, grads, hessians]) - - result = _AccumulatorResultToDict(partition, feature, grads, - hessians) - self.assertEqual(num_updates, 3) - self.assertEqual(len(result), 2) - self.assertAllClose(result[(3, 4, 0)][0], [0.3, 0.3]) - self.assertAllClose(result[(3, 4, 0)][1], [[0.03, 0.04], [0.05, 0.06]]) - self.assertAllClose(result[(4, 5, 0)][0], [0.5, 0.5]) - self.assertAllClose(result[(4, 5, 0)][1], [[0.07, 0.08], [0.09, 0.10]]) - - def testMakeSummary(self): - with self.cached_session() as sess: - accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=tensor_shape.TensorShape([2]), - hessian_shape=tensor_shape.TensorShape([2, 2])) - partition, feature, grads, hessians = accumulator._make_summary( - partition_ids=[1, 2, 1], - feature_ids=[[2, 0], [3, 2], [2, 0]], - # Two values for gradients, - gradients=[[0.1, 0.1], [0.2, 0.2], [0.10, 0.11]], - # A 2x2 matrix for each hessian. - hessians=[[[0.01, 0.02], [0.03, 0.04]], [[0.05, 0.06], [0.07, 0.08]], - [[0.011, 0.022], [0.033, 0.044]]]) - partition, feature, grads, hessians = sess.run( - [partition, feature, grads, hessians]) - - result = _AccumulatorResultToDict(partition, feature, grads, hessians) - self.assertEqual(len(result), 2) - self.assertAllClose(result[(1, 2, 0)][0], [0.20, 0.21]) - self.assertAllClose(result[(1, 2, 0)][1], - [[0.021, 0.042], [0.063, 0.084]]) - self.assertAllClose(result[(2, 3, 2)][0], [0.2, 0.2]) - self.assertAllClose(result[(2, 3, 2)][1], [[0.05, 0.06], [0.07, 0.08]]) - - -def _AccumulatorResultToDict(partition, feature, grads, hessians): - """Converts the inputs to a dictionary since the ordering changes.""" - return {(partition[i], feature[i, 0], feature[i, 1]): (grads[i], hessians[i]) - for i in range(len(partition))} - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/training_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/training_ops_test.py deleted file mode 100644 index 74a51f4e4d8..00000000000 --- a/tensorflow/contrib/boosted_trees/python/kernel_tests/training_ops_test.py +++ /dev/null @@ -1,2579 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the GTFlow training Ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from google.protobuf import text_format - -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.boosted_trees.proto import split_info_pb2 -from tensorflow.contrib.boosted_trees.proto import tree_config_pb2 -from tensorflow.contrib.boosted_trees.python.ops import model_ops -from tensorflow.contrib.boosted_trees.python.ops import training_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import resources -from tensorflow.python.platform import googletest - - -def _gen_learner_config(num_classes, - l1_reg, - l2_reg, - tree_complexity, - max_depth, - min_node_weight, - pruning_mode, - growing_mode, - dropout_probability=None, - dropout_learning_rate=None, - dropout_prob_of_skipping=None): - """Create a serialized learner config with the desired settings.""" - config = learner_pb2.LearnerConfig() - config.num_classes = num_classes - config.regularization.l1 = l1_reg - config.regularization.l2 = l2_reg - config.regularization.tree_complexity = tree_complexity - config.constraints.max_tree_depth = max_depth - config.constraints.min_node_weight = min_node_weight - config.pruning_mode = pruning_mode - config.growing_mode = growing_mode - - if dropout_probability is not None: - config.learning_rate_tuner.dropout.dropout_probability = dropout_probability - if dropout_learning_rate is not None: - config.learning_rate_tuner.dropout.learning_rate = dropout_learning_rate - if dropout_prob_of_skipping is not None: - config.learning_rate_tuner.dropout.dropout_prob_of_skipping = ( - dropout_prob_of_skipping) - return config - - -def _gen_dense_split_info(fc, threshold, left_weight, right_weight): - split_str = """ - split_node { - dense_float_binary_split { - feature_column: %d - threshold: %f - } - } - left_child { - sparse_vector { - index: 0 - value: %f - } - } - right_child { - sparse_vector { - index: 0 - value: %f - } - }""" % (fc, threshold, left_weight, right_weight) - split = split_info_pb2.SplitInfo() - text_format.Merge(split_str, split) - return split.SerializeToString() - - -def _gen_dense_oblivious_split_info(fc, threshold, leave_weights, - children_parent_id): - split_str = """ - split_node { - oblivious_dense_float_binary_split { - feature_column: %d - threshold: %f - } - }""" % (fc, threshold) - for weight in leave_weights: - split_str += """ - children { - vector { - value: %f - } - }""" % ( - weight) - for x in children_parent_id: - split_str += """ - children_parent_id: %d""" % (x) - split = split_info_pb2.ObliviousSplitInfo() - text_format.Merge(split_str, split) - return split.SerializeToString() - - -def _gen_categorical_split_info(fc, feat_id, left_weight, right_weight): - split_str = """ - split_node { - categorical_id_binary_split { - feature_column: %d - feature_id: %d - } - } - left_child { - sparse_vector { - index: 0 - value: %f - } - } - right_child { - sparse_vector { - index: 0 - value: %f - } - }""" % (fc, feat_id, left_weight, right_weight) - split = split_info_pb2.SplitInfo() - text_format.Merge(split_str, split) - return split.SerializeToString() - - -def _get_bias_update(grads, hess): - return array_ops.where_v2(hess > 0, -grads / hess, - array_ops.zeros_like(grads)) - - -class CenterTreeEnsembleBiasOpTest(test_util.TensorFlowTestCase): - """Tests for centering tree ensemble bias.""" - - def testCenterBias(self): - """Tests bias centering for multiple iterations.""" - with self.cached_session() as session: - # Create empty ensemble. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=3, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=4, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.PRE_PRUNE, - growing_mode=learner_pb2.LearnerConfig.WHOLE_TREE, - # Dropout does not change anything here. - dropout_probability=0.5).SerializeToString() - - # Center bias for the initial step. - grads = constant_op.constant([0.4, -0.3]) - hess = constant_op.constant([2.0, 1.0]) - continue_centering1 = training_ops.center_tree_ensemble_bias( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - delta_updates=_get_bias_update(grads, hess), - learner_config=learner_config) - continue_centering = session.run(continue_centering1) - self.assertEqual(continue_centering, True) - - # Validate ensemble state. - # dim 0 update: -0.4/2.0 = -0.2 - # dim 1 update: +0.3/1.0 = +0.3 - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=1)) - tree_ensemble_config.ParseFromString(serialized) - expected_result = """ - trees { - nodes { - leaf { - vector { - value: -0.2 - value: 0.3 - } - } - } - } - tree_weights: 1.0 - tree_metadata { - num_layers_grown: 1 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 1 - } - """ - self.assertEqual(new_stamp, 1) - self.assertEqual(stats.num_trees, 0) - self.assertEqual(stats.num_layers, 1) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 1) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 1) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - # Center bias for another step. - # dim 0 update: -0.06/0.5 = -0.12 - # dim 1 update: -0.01/0.5 = -0.02 - grads = constant_op.constant([0.06, 0.01]) - hess = constant_op.constant([0.5, 0.5]) - continue_centering2 = training_ops.center_tree_ensemble_bias( - tree_ensemble_handle, - stamp_token=1, - next_stamp_token=2, - delta_updates=_get_bias_update(grads, hess), - learner_config=learner_config) - continue_centering = session.run(continue_centering2) - self.assertEqual(continue_centering, True) - - # Validate ensemble state. - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=2)) - tree_ensemble_config.ParseFromString(serialized) - expected_result = """ - trees { - nodes { - leaf { - vector { - value: -0.32 - value: 0.28 - } - } - } - } - tree_weights: 1.0 - tree_metadata { - num_layers_grown: 1 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 1 - } - """ - self.assertEqual(new_stamp, 2) - self.assertEqual(stats.num_trees, 0) - self.assertEqual(stats.num_layers, 1) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 1) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 1) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - # Center bias for another step, but this time updates are negligible. - grads = constant_op.constant([0.0000001, -0.00003]) - hess = constant_op.constant([0.5, 0.0]) - continue_centering3 = training_ops.center_tree_ensemble_bias( - tree_ensemble_handle, - stamp_token=2, - next_stamp_token=3, - delta_updates=_get_bias_update(grads, hess), - learner_config=learner_config) - continue_centering = session.run(continue_centering3) - self.assertEqual(continue_centering, False) - - # Validate ensemble stamp. - new_stamp, _ = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=3)) - self.assertEqual(new_stamp, 3) - self.assertEqual(stats.num_trees, 1) - self.assertEqual(stats.num_layers, 1) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 1) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 1) - - -class GrowTreeEnsembleOpTest(test_util.TensorFlowTestCase): - """Tests for growing tree ensemble from split candidates.""" - - def testGrowEmptyEnsemble(self): - """Test growing an empty ensemble.""" - with self.cached_session() as session: - # Create empty ensemble. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=1, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.PRE_PRUNE, - growing_mode=learner_pb2.LearnerConfig.WHOLE_TREE, - # Dropout does not change anything here, tree is not finalized. - dropout_probability=0.5) - - # Prepare handler inputs. - # Note that handlers 1 & 3 have the same gain but different splits. - handler1_partitions = np.array([0], dtype=np.int32) - handler1_gains = np.array([7.62], dtype=np.float32) - handler1_split = [_gen_dense_split_info(0, 0.52, -4.375, 7.143)] - handler2_partitions = np.array([0], dtype=np.int32) - handler2_gains = np.array([0.63], dtype=np.float32) - handler2_split = [_gen_dense_split_info(0, 0.23, -0.6, 0.24)] - handler3_partitions = np.array([0], dtype=np.int32) - handler3_gains = np.array([7.62], dtype=np.float32) - handler3_split = [_gen_categorical_split_info(0, 7, -4.375, 7.143)] - - # Grow tree ensemble. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=0.1, - partition_ids=[ - handler1_partitions, handler2_partitions, handler3_partitions - ], - gains=[handler1_gains, handler2_gains, handler3_gains], - splits=[handler1_split, handler2_split, handler3_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE) - session.run(grow_op) - - # Expect the simpler split from handler 1 to be chosen. - # The grown tree should be finalized as max tree depth is 1. - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=1)) - tree_ensemble_config.ParseFromString(serialized) - expected_result = """ - trees { - nodes { - dense_float_binary_split { - threshold: 0.52 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 7.62 - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: -4.375 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 7.143 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - is_finalized: true - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 1 - } - """ - self.assertEqual(new_stamp, 1) - self.assertEqual(stats.num_trees, 1) - self.assertEqual(stats.num_layers, 1) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 1) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 1) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - def testGrowEmptyEnsembleObliviousCase(self): - """Test growing an empty ensemble in the oblivious case.""" - with self.cached_session() as session: - # Create empty ensemble. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=1, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.PRE_PRUNE, - growing_mode=learner_pb2.LearnerConfig.WHOLE_TREE) - - # Prepare handler inputs. - # Note that handlers 1 & 3 have the same gain but different splits. - handler1_partitions = np.array([0], dtype=np.int32) - handler1_gains = np.array([7.62], dtype=np.float32) - handler1_split = [ - _gen_dense_oblivious_split_info(0, 0.52, [-4.375, 7.143], [0]) - ] - handler2_partitions = np.array([0], dtype=np.int32) - handler2_gains = np.array([0.63], dtype=np.float32) - handler2_split = [ - _gen_dense_oblivious_split_info(0, 0.23, [-0.6, 0.24], [0]) - ] - handler3_partitions = np.array([0], dtype=np.int32) - handler3_gains = np.array([7.62], dtype=np.float32) - handler3_split = [ - _gen_dense_oblivious_split_info(0, 7, [-4.375, 7.143], [0]) - ] - - # Grow tree ensemble. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=0.1, - partition_ids=[ - handler1_partitions, handler2_partitions, handler3_partitions - ], - gains=[handler1_gains, handler2_gains, handler3_gains], - splits=[handler1_split, handler2_split, handler3_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.OBLIVIOUS_DECISION_TREE) - session.run(grow_op) - - # Expect the split with bigger handler_id, i.e. handler 3 to be chosen. - # The grown tree should be finalized as max tree depth is 1. - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=1)) - tree_ensemble_config.ParseFromString(serialized) - expected_result = """ - trees { - nodes { - oblivious_dense_float_binary_split { - feature_column: 0 - threshold: 7 - } - node_metadata { - gain: 7.62 - original_oblivious_leaves { - } - } - } - nodes { - leaf { - vector { - value: -4.375 - } - } - } - nodes { - leaf { - vector { - value: 7.143 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - is_finalized: true - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 1 - } - """ - self.assertEqual(new_stamp, 1) - self.assertEqual(stats.num_trees, 1) - self.assertEqual(stats.num_layers, 1) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 1) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 1) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - def testGrowExistingEnsembleTreeNotFinalized(self): - """Test growing an existing ensemble with the last tree not finalized.""" - with self.cached_session() as session: - # Create existing ensemble with one root split - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - text_format.Merge(""" - trees { - nodes { - categorical_id_binary_split { - feature_id: 4 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 7.61999988556 - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 7.14300012589 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: -4.375 - } - } - } - } - tree_weights: 0.10000000149 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 1 - } - """, tree_ensemble_config) - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=3, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.PRE_PRUNE, - growing_mode=learner_pb2.LearnerConfig.WHOLE_TREE, - # Dropout does not change anything here - tree is not finalized. - dropout_probability=0.5) - - # Prepare handler inputs. - # Handler 1 only has a candidate for partition 1, handler 2 has candidates - # for both partitions and handler 3 only has a candidate for partition 2. - handler1_partitions = np.array([1], dtype=np.int32) - handler1_gains = np.array([1.4], dtype=np.float32) - handler1_split = [_gen_dense_split_info(0, 0.21, -6.0, 1.65)] - handler2_partitions = np.array([1, 2], dtype=np.int32) - handler2_gains = np.array([0.63, 2.7], dtype=np.float32) - handler2_split = [ - _gen_dense_split_info(0, 0.23, -0.6, 0.24), - _gen_categorical_split_info(1, 7, -1.5, 2.3) - ] - handler3_partitions = np.array([2], dtype=np.int32) - handler3_gains = np.array([1.7], dtype=np.float32) - handler3_split = [_gen_categorical_split_info(0, 3, -0.75, 1.93)] - - # Grow tree ensemble. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=0.1, - partition_ids=[ - handler1_partitions, handler2_partitions, handler3_partitions - ], - gains=[handler1_gains, handler2_gains, handler3_gains], - splits=[handler1_split, handler2_split, handler3_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE) - session.run(grow_op) - - # Expect the split for partition 1 to be chosen from handler 1 and - # the split for partition 2 to be chosen from handler 2. - # The grown tree should not be finalized as max tree depth is 3 and - # it's only grown 2 layers. - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=1)) - tree_ensemble_config.ParseFromString(serialized) - expected_result = """ - trees { - nodes { - categorical_id_binary_split { - feature_id: 4 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 7.61999988556 - } - } - nodes { - dense_float_binary_split { - threshold: 0.21 - left_id: 3 - right_id: 4 - } - node_metadata { - gain: 1.4 - } - } - nodes { - categorical_id_binary_split { - feature_column: 1 - feature_id: 7 - left_id: 5 - right_id: 6 - } - node_metadata { - gain: 2.7 - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: -6.0 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 1.65 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: -1.5 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 2.3 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 2 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 2 - } - """ - self.assertEqual(new_stamp, 1) - self.assertEqual(stats.num_trees, 0) - self.assertEqual(stats.num_layers, 2) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 2) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 2) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - def testGrowExistingEnsembleTreeFinalized(self): - """Test growing an existing ensemble with the last tree finalized.""" - with self.cached_session() as session: - # Create existing ensemble with one root split - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - text_format.Merge(""" - trees { - nodes { - categorical_id_binary_split { - feature_column: 3 - feature_id: 7 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 1.3 - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 2.3 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: -0.9 - } - } - } - } - tree_weights: 0.10000000149 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - is_finalized: true - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 1 - } - """, tree_ensemble_config) - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=1, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.PRE_PRUNE, - growing_mode=learner_pb2.LearnerConfig.WHOLE_TREE) - - # Prepare handler inputs. - handler1_partitions = np.array([0], dtype=np.int32) - handler1_gains = np.array([7.62], dtype=np.float32) - handler1_split = [_gen_dense_split_info(5, 0.52, -4.375, 7.143)] - handler2_partitions = np.array([0], dtype=np.int32) - handler2_gains = np.array([0.63], dtype=np.float32) - handler2_split = [_gen_dense_split_info(2, 0.23, -0.6, 0.24)] - handler3_partitions = np.array([0], dtype=np.int32) - handler3_gains = np.array([7.62], dtype=np.float32) - handler3_split = [_gen_categorical_split_info(8, 7, -4.375, 7.143)] - - # Grow tree ensemble. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=0.2, - partition_ids=[ - handler1_partitions, handler2_partitions, handler3_partitions - ], - gains=[handler1_gains, handler2_gains, handler3_gains], - splits=[handler1_split, handler2_split, handler3_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE) - session.run(grow_op) - - # Expect a new tree to be added with the split from handler 1. - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=1)) - tree_ensemble_config.ParseFromString(serialized) - expected_result = """ - trees { - nodes { - categorical_id_binary_split { - feature_column: 3 - feature_id: 7 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 1.3 - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 2.3 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: -0.9 - } - } - } - } - trees { - nodes { - dense_float_binary_split { - feature_column: 5 - threshold: 0.52 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 7.62 - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: -4.375 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 7.143 - } - } - } - } - tree_weights: 0.1 - tree_weights: 0.2 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - is_finalized: true - } - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - is_finalized: true - } - growing_metadata { - num_trees_attempted: 2 - num_layers_attempted: 2 - } - """ - self.assertEqual(new_stamp, 1) - self.assertEqual(stats.num_trees, 2) - self.assertEqual(stats.num_layers, 2) - self.assertEqual(stats.active_tree, 2) - self.assertEqual(stats.active_layer, 1) - self.assertEqual(stats.attempted_trees, 2) - self.assertEqual(stats.attempted_layers, 2) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - def testGrowEnsemblePrePrune(self): - """Test growing an ensemble with pre-pruning.""" - with self.cached_session() as session: - # Create empty ensemble. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=1, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.PRE_PRUNE, - growing_mode=learner_pb2.LearnerConfig.WHOLE_TREE) - - # Prepare handler inputs. - # All handlers have negative gain. - handler1_partitions = np.array([0], dtype=np.int32) - handler1_gains = np.array([-0.62], dtype=np.float32) - handler1_split = [_gen_dense_split_info(0, 0.52, 0.01, 0.0143)] - handler2_partitions = np.array([0], dtype=np.int32) - handler2_gains = np.array([-1.3], dtype=np.float32) - handler2_split = [_gen_categorical_split_info(0, 7, 0.013, 0.0143)] - - # Grow tree ensemble. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=0.1, - partition_ids=[handler1_partitions, handler2_partitions], - gains=[handler1_gains, handler2_gains], - splits=[handler1_split, handler2_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE) - session.run(grow_op) - - # Expect the ensemble to be empty. - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=1)) - tree_ensemble_config.ParseFromString(serialized) - self.assertEqual(new_stamp, 1) - self.assertEqual(stats.num_trees, 0) - self.assertEqual(stats.num_layers, 0) - self.assertEqual(stats.active_tree, 0) - self.assertEqual(stats.active_layer, 0) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 1) - self.assertProtoEquals(""" - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 1 - } - """, tree_ensemble_config) - - def testGrowEnsemblePostPruneNone(self): - """Test growing an empty ensemble.""" - with self.cached_session() as session: - # Create empty ensemble. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=1, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.POST_PRUNE, - growing_mode=learner_pb2.LearnerConfig.WHOLE_TREE) - - # Prepare handler inputs. - # Note that handlers 1 & 3 have the same gain but different splits. - handler1_partitions = np.array([0], dtype=np.int32) - handler1_gains = np.array([7.62], dtype=np.float32) - handler1_split = [_gen_dense_split_info(0, 0.52, -4.375, 7.143)] - handler2_partitions = np.array([0], dtype=np.int32) - handler2_gains = np.array([0.63], dtype=np.float32) - handler2_split = [_gen_dense_split_info(0, 0.23, -0.6, 0.24)] - handler3_partitions = np.array([0], dtype=np.int32) - handler3_gains = np.array([7.62], dtype=np.float32) - handler3_split = [_gen_categorical_split_info(0, 7, -4.375, 7.143)] - - # Grow tree ensemble. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=0.1, - partition_ids=[ - handler1_partitions, handler2_partitions, handler3_partitions - ], - gains=[handler1_gains, handler2_gains, handler3_gains], - splits=[handler1_split, handler2_split, handler3_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE) - session.run(grow_op) - - # Expect the simpler split from handler 1 to be chosen. - # The grown tree should be finalized as max tree depth is 1. - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=1)) - tree_ensemble_config.ParseFromString(serialized) - expected_result = """ - trees { - nodes { - dense_float_binary_split { - threshold: 0.52 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 7.62 - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: -4.375 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 7.143 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - is_finalized: true - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 1 - } - """ - self.assertEqual(new_stamp, 1) - self.assertEqual(stats.num_trees, 1) - self.assertEqual(stats.num_layers, 1) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 1) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 1) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - def testGrowEnsemblePostPruneAll(self): - """Test growing an ensemble with post-pruning.""" - with self.cached_session() as session: - # Create empty ensemble. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=2, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.POST_PRUNE, - growing_mode=learner_pb2.LearnerConfig.WHOLE_TREE) - - # Prepare handler inputs. - # All handlers have negative gain. - handler1_partitions = np.array([0], dtype=np.int32) - handler1_gains = np.array([-1.3], dtype=np.float32) - handler1_split = [_gen_categorical_split_info(0, 7, 0.013, 0.0143)] - handler2_partitions = np.array([0], dtype=np.int32) - handler2_gains = np.array([-0.62], dtype=np.float32) - handler2_split = [_gen_dense_split_info(0, 0.33, 0.01, 0.0143)] - - # Grow tree ensemble. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=0.1, - partition_ids=[handler1_partitions, handler2_partitions], - gains=[handler1_gains, handler2_gains], - splits=[handler1_split, handler2_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE) - session.run(grow_op) - - # Expect the split from handler 2 to be chosen despite the negative gain. - # The grown tree should not be finalized as max tree depth is 2 so no - # pruning occurs. - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - tree_ensemble_config.ParseFromString(serialized) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=1)) - expected_result = """ - trees { - nodes { - dense_float_binary_split { - threshold: 0.33 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: -0.62 - original_leaf { - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 0.01 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 0.0143 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 1 - } - """ - self.assertEqual(new_stamp, 1) - self.assertEqual(stats.num_trees, 0) - self.assertEqual(stats.num_layers, 1) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 1) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 1) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - # Prepare handler inputs. - # All handlers have negative gain. - handler1_partitions = np.array([1, 2], dtype=np.int32) - handler1_gains = np.array([-0.2, -0.5], dtype=np.float32) - handler1_split = [ - _gen_categorical_split_info(3, 7, 0.07, 0.083), - _gen_categorical_split_info(3, 5, 0.041, 0.064) - ] - - # Grow tree ensemble. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=1, - next_stamp_token=2, - learning_rate=0.1, - partition_ids=[handler1_partitions], - gains=[handler1_gains], - splits=[handler1_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE) - session.run(grow_op) - - # Expect the ensemble to be empty as post-pruning will prune - # the entire finalized tree. - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=2)) - tree_ensemble_config.ParseFromString(serialized) - self.assertEqual(new_stamp, 2) - self.assertEqual(stats.num_trees, 0) - self.assertEqual(stats.num_layers, 0) - self.assertEqual(stats.active_tree, 0) - self.assertEqual(stats.active_layer, 0) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 2) - self.assertProtoEquals(""" - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 2 - } - """, tree_ensemble_config) - - def testGrowEnsemblePostPrunePartial(self): - """Test growing an ensemble with post-pruning.""" - with self.cached_session() as session: - # Create empty ensemble. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=2, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.POST_PRUNE, - growing_mode=learner_pb2.LearnerConfig.WHOLE_TREE) - - # Prepare handler inputs. - # Second handler has positive gain. - handler1_partitions = np.array([0], dtype=np.int32) - handler1_gains = np.array([-1.3], dtype=np.float32) - handler1_split = [_gen_categorical_split_info(0, 7, 0.013, 0.0143)] - handler2_partitions = np.array([0], dtype=np.int32) - handler2_gains = np.array([-0.2], dtype=np.float32) - handler2_split = [_gen_dense_split_info(0, 0.33, 0.01, 0.0143)] - - # Grow tree ensemble. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=0.1, - partition_ids=[handler1_partitions, handler2_partitions], - gains=[handler1_gains, handler2_gains], - splits=[handler1_split, handler2_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE) - session.run(grow_op) - - # Expect the split from handler 2 to be chosen despite the negative gain. - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=1)) - tree_ensemble_config.ParseFromString(serialized) - expected_result = """ - trees { - nodes { - dense_float_binary_split { - threshold: 0.33 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: -0.2 - original_leaf { - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 0.01 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 0.0143 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 1 - } - """ - self.assertEqual(new_stamp, 1) - self.assertEqual(stats.num_trees, 0) - self.assertEqual(stats.num_layers, 1) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 1) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 1) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - # Prepare handler inputs for second layer. - # Note that partition 1 gain is negative and partition 2 gain is positive. - handler1_partitions = np.array([1, 2], dtype=np.int32) - handler1_gains = np.array([-0.2, 0.5], dtype=np.float32) - handler1_split = [ - _gen_categorical_split_info(3, 7, 0.07, 0.083), - _gen_categorical_split_info(3, 5, 0.041, 0.064) - ] - - # Grow tree ensemble. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=1, - next_stamp_token=2, - learning_rate=0.1, - partition_ids=[handler1_partitions], - gains=[handler1_gains], - splits=[handler1_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE) - session.run(grow_op) - - # Expect the negative gain split of partition 1 to be pruned and the - # positive gain split of partition 2 to be retained. - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=2)) - tree_ensemble_config.ParseFromString(serialized) - expected_result = """ - trees { - nodes { - dense_float_binary_split { - threshold: 0.33 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: -0.2 - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 0.01 - } - } - } - nodes { - categorical_id_binary_split { - feature_column: 3 - feature_id: 5 - left_id: 3 - right_id: 4 - } - node_metadata { - gain: 0.5 - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 0.041 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 0.064 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 2 - is_finalized: true - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 2 - } - """ - self.assertEqual(new_stamp, 2) - self.assertEqual(stats.num_trees, 1) - self.assertEqual(stats.num_layers, 2) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 2) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 2) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - def testGrowEnsembleTreeLayerByLayer(self): - """Test growing an existing ensemble with the last tree not finalized.""" - with self.cached_session() as session: - # Create existing ensemble with one root split - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - text_format.Merge(""" - trees { - nodes { - categorical_id_binary_split { - feature_id: 4 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 7.62 - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 7.143 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: -4.375 - } - } - } - } - tree_weights: 0.10000000149 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 1 - } - """, tree_ensemble_config) - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=3, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.PRE_PRUNE, - growing_mode=learner_pb2.LearnerConfig.LAYER_BY_LAYER, - # Dropout will have no effect, since the tree will not be fully grown. - dropout_probability=1.0) - - # Prepare handler inputs. - # Handler 1 only has a candidate for partition 1, handler 2 has candidates - # for both partitions and handler 3 only has a candidate for partition 2. - handler1_partitions = np.array([1], dtype=np.int32) - handler1_gains = np.array([1.4], dtype=np.float32) - handler1_split = [_gen_dense_split_info(0, 0.21, -6.0, 1.65)] - handler2_partitions = np.array([1, 2], dtype=np.int32) - handler2_gains = np.array([0.63, 2.7], dtype=np.float32) - handler2_split = [ - _gen_dense_split_info(0, 0.23, -0.6, 0.24), - _gen_categorical_split_info(1, 7, -1.5, 2.3) - ] - handler3_partitions = np.array([2], dtype=np.int32) - handler3_gains = np.array([1.7], dtype=np.float32) - handler3_split = [_gen_categorical_split_info(0, 3, -0.75, 1.93)] - - # Grow tree ensemble layer by layer. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=0.1, - partition_ids=[ - handler1_partitions, handler2_partitions, handler3_partitions - ], - gains=[handler1_gains, handler2_gains, handler3_gains], - splits=[handler1_split, handler2_split, handler3_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE) - session.run(grow_op) - - # Expect the split for partition 1 to be chosen from handler 1 and - # the split for partition 2 to be chosen from handler 2. - # The grown tree should not be finalized as max tree depth is 3 and - # it's only grown 2 layers. - # The partition 1 split weights get added to original leaf weight 7.143. - # The partition 2 split weights get added to original leaf weight -4.375. - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=1)) - tree_ensemble_config.ParseFromString(serialized) - expected_result = """ - trees { - nodes { - categorical_id_binary_split { - feature_id: 4 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 7.62 - } - } - nodes { - dense_float_binary_split { - threshold: 0.21 - left_id: 3 - right_id: 4 - } - node_metadata { - gain: 1.4 - } - } - nodes { - categorical_id_binary_split { - feature_column: 1 - feature_id: 7 - left_id: 5 - right_id: 6 - } - node_metadata { - gain: 2.7 - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 1.143 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 8.793 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: -5.875 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: -2.075 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 2 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 2 - } - """ - self.assertEqual(new_stamp, 1) - self.assertEqual(stats.num_trees, 0) - self.assertEqual(stats.num_layers, 2) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 2) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 2) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - def testGrowEnsembleTreeLayerByLayerObliviousCase(self): - """Test growing an existing ensemble with the last tree not finalized.""" - with self.cached_session() as session: - # Create existing ensemble with one root split - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - text_format.Merge( - """ - trees { - nodes { - oblivious_dense_float_binary_split { - feature_column: 4 - threshold: 7 - } - node_metadata { - gain: 7.62 - original_oblivious_leaves { - } - } - } - nodes { - leaf { - vector { - value: 7.143 - } - } - } - nodes { - leaf { - vector { - value: -4.375 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 1 - } - """, tree_ensemble_config) - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=3, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.PRE_PRUNE, - growing_mode=learner_pb2.LearnerConfig.LAYER_BY_LAYER) - - # Prepare handler inputs. - handler1_partitions = np.array([0], dtype=np.int32) - handler1_gains = np.array([1.4], dtype=np.float32) - handler1_split = [ - _gen_dense_oblivious_split_info(0, 0.21, [-6.0, 1.65, 1.0, -0.5], - [1, 2]) - ] - handler2_partitions = np.array([0], dtype=np.int32) - handler2_gains = np.array([2.7], dtype=np.float32) - handler2_split = [ - _gen_dense_oblivious_split_info(0, 0.23, [-0.6, 0.24, 0.3, 0.4], - [1, 2]) - ] - handler3_partitions = np.array([0], dtype=np.int32) - handler3_gains = np.array([1.7], dtype=np.float32) - handler3_split = [ - _gen_dense_oblivious_split_info(0, 3, [-0.75, 1.93, 0.2, -0.1], - [1, 2]) - ] - - # Grow tree ensemble layer by layer. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=0.1, - partition_ids=[ - handler1_partitions, handler2_partitions, handler3_partitions - ], - gains=[handler1_gains, handler2_gains, handler3_gains], - splits=[handler1_split, handler2_split, handler3_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.OBLIVIOUS_DECISION_TREE) - session.run(grow_op) - - # Expect the split for partition 1 to be chosen from handler 1 and - # the split for partition 2 to be chosen from handler 2. - # The grown tree should not be finalized as max tree depth is 3 and - # it's only grown 2 layers. - # The partition 1 split weights get added to original leaf weight 7.143. - # The partition 2 split weights get added to original leaf weight -4.375. - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=1)) - tree_ensemble_config.ParseFromString(serialized) - expected_result = """ - trees { - nodes { - oblivious_dense_float_binary_split { - feature_column: 4 - threshold: 7 - } - node_metadata { - gain: 7.62 - original_oblivious_leaves { - } - } - } - nodes { - oblivious_dense_float_binary_split { - feature_column: 0 - threshold: 0.23 - } - node_metadata { - gain: 2.7 - original_oblivious_leaves { - vector { - value: 7.143 - } - } - original_oblivious_leaves { - vector { - value: -4.375 - } - } - } - } - nodes { - leaf { - vector { - value: 6.543 - } - } - } - nodes { - leaf { - vector { - value: 7.383 - } - } - } - nodes { - leaf { - vector { - value: -4.075 - } - } - } - nodes { - leaf { - vector { - value: -3.975 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 2 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 2 - } - """ - self.assertEqual(new_stamp, 1) - self.assertEqual(stats.num_trees, 0) - self.assertEqual(stats.num_layers, 2) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 2) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 2) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - def testGrowEnsembleWithEmptyNodesMiddleCase(self): - """Test case: The middle existing leaves don't have examples.""" - with self.cached_session() as session: - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - text_format.Merge( - """ - trees { - nodes { - oblivious_dense_float_binary_split { - feature_column: 4 - threshold: 7 - } - node_metadata { - gain: 7.62 - original_oblivious_leaves { - } - } - } - nodes { - oblivious_dense_float_binary_split { - feature_column: 1 - threshold: 0.23 - } - node_metadata { - gain: 2.7 - original_oblivious_leaves { - vector { - value: 7.143 - } - } - original_oblivious_leaves { - vector { - value: -4.375 - } - } - } - } - nodes { - leaf { - vector { - value: 6.543 - } - } - } - nodes { - leaf { - vector { - value: 7.5 - } - } - } - nodes { - leaf { - vector { - value: -4.075 - } - } - } - nodes { - leaf { - vector { - value: -3.975 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 2 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 2 - } - """, tree_ensemble_config) - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=6, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.PRE_PRUNE, - growing_mode=learner_pb2.LearnerConfig.LAYER_BY_LAYER) - - # Prepare handler inputs. - handler1_partitions = np.array([0], dtype=np.int32) - handler1_gains = np.array([1.8], dtype=np.float32) - handler1_split = [ - _gen_dense_oblivious_split_info(0, 0.9, [1.0, 2.0, 3.0, 4.0], [2, 5]) - ] - # The tree currently has depth 2, so the ids for the four leaves are in - # the range [2, 6). In this test case we are assuming that our examples - # only fall in leaves 2 and 5. - - # Grow tree ensemble layer by layer. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=0.1, - partition_ids=[handler1_partitions], - gains=[handler1_gains], - splits=[handler1_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.OBLIVIOUS_DECISION_TREE) - session.run(grow_op) - - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=1)) - tree_ensemble_config.ParseFromString(serialized) - expected_result = """ - trees { - nodes { - oblivious_dense_float_binary_split { - feature_column: 4 - threshold: 7 - } - node_metadata { - gain: 7.62 - original_oblivious_leaves { - } - } - } - nodes { - oblivious_dense_float_binary_split { - feature_column: 1 - threshold: 0.23 - } - node_metadata { - gain: 2.7 - original_oblivious_leaves { - vector { - value: 7.143 - } - } - original_oblivious_leaves { - vector { - value: -4.375 - } - } - } - } - nodes { - oblivious_dense_float_binary_split { - feature_column: 0 - threshold: 0.9 - } - node_metadata { - gain: 1.8 - original_oblivious_leaves { - vector { - value: 6.543 - } - } - original_oblivious_leaves { - vector { - value: 7.5 - } - } - original_oblivious_leaves { - vector { - value: -4.075 - } - } - original_oblivious_leaves { - vector { - value: -3.975 - } - } - } - } - nodes { - leaf { - vector { - value: 7.543 - } - } - } - nodes { - leaf { - vector { - value: 8.543 - } - } - } - nodes { - leaf { - vector { - value: 7.5 - } - } - } - nodes { - leaf { - vector { - value: 7.5 - } - } - } - nodes { - leaf { - vector { - value: -4.075 - } - } - } - nodes { - leaf { - vector { - value: -4.075 - } - } - } - nodes { - leaf { - vector { - value: -0.975 - } - } - } - nodes { - leaf { - vector { - value: 0.025 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 3 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 3 - } - """ - self.assertEqual(new_stamp, 1) - self.assertEqual(stats.num_trees, 0) - self.assertEqual(stats.num_layers, 3) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 3) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 3) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - def testGrowEnsembleWithEmptyNodesBorderCase(self): - """Test case: The first and last existing leaves don't have examples.""" - with self.cached_session() as session: - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - text_format.Merge( - """ - trees { - nodes { - oblivious_dense_float_binary_split { - feature_column: 4 - threshold: 7 - } - node_metadata { - gain: 7.62 - original_oblivious_leaves { - } - } - } - nodes { - oblivious_dense_float_binary_split { - feature_column: 1 - threshold: 0.23 - } - node_metadata { - gain: 2.7 - original_oblivious_leaves { - vector { - value: 7.143 - } - } - original_oblivious_leaves { - vector { - value: -4.375 - } - } - } - } - nodes { - leaf { - vector { - value: 6.543 - } - } - } - nodes { - leaf { - vector { - value: 7.5 - } - } - } - nodes { - leaf { - vector { - value: -4.075 - } - } - } - nodes { - leaf { - vector { - value: -3.975 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 2 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 2 - } - """, tree_ensemble_config) - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=6, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.PRE_PRUNE, - growing_mode=learner_pb2.LearnerConfig.LAYER_BY_LAYER) - - # Prepare handler inputs. - handler1_partitions = np.array([0], dtype=np.int32) - handler1_gains = np.array([1.8], dtype=np.float32) - handler1_split = [ - _gen_dense_oblivious_split_info(0, 0.9, [1.0, 2.0, 3.0, 4.0], [3, 4]) - ] - # The tree currently has depth 2, so the ids for the four leaves are in - # the range [2, 6). In this test case we are assuming that our examples - # only fall in leaves 3 and 4. - - # Grow tree ensemble layer by layer. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=0.1, - partition_ids=[handler1_partitions], - gains=[handler1_gains], - splits=[handler1_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.OBLIVIOUS_DECISION_TREE) - session.run(grow_op) - - new_stamp, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - stats = session.run( - training_ops.tree_ensemble_stats(tree_ensemble_handle, stamp_token=1)) - tree_ensemble_config.ParseFromString(serialized) - expected_result = """ - trees { - nodes { - oblivious_dense_float_binary_split { - feature_column: 4 - threshold: 7 - } - node_metadata { - gain: 7.62 - original_oblivious_leaves { - } - } - } - nodes { - oblivious_dense_float_binary_split { - feature_column: 1 - threshold: 0.23 - } - node_metadata { - gain: 2.7 - original_oblivious_leaves { - vector { - value: 7.143 - } - } - original_oblivious_leaves { - vector { - value: -4.375 - } - } - } - } - nodes { - oblivious_dense_float_binary_split { - feature_column: 0 - threshold: 0.9 - } - node_metadata { - gain: 1.8 - original_oblivious_leaves { - vector { - value: 6.543 - } - } - original_oblivious_leaves { - vector { - value: 7.5 - } - } - original_oblivious_leaves { - vector { - value: -4.075 - } - } - original_oblivious_leaves { - vector { - value: -3.975 - } - } - } - } - nodes { - leaf { - vector { - value: 6.543 - } - } - } - nodes { - leaf { - vector { - value: 6.543 - } - } - } - nodes { - leaf { - vector { - value: 8.5 - } - } - } - nodes { - leaf { - vector { - value: 9.5 - } - } - } - nodes { - leaf { - vector { - value: -1.075 - } - } - } - nodes { - leaf { - vector { - value: -0.075 - } - } - } - nodes { - leaf { - vector { - value: -3.975 - } - } - } - nodes { - leaf { - vector { - value: -3.975 - } - } - } - } - tree_weights: 0.1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 3 - } - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 3 - } - """ - self.assertEqual(new_stamp, 1) - self.assertEqual(stats.num_trees, 0) - self.assertEqual(stats.num_layers, 3) - self.assertEqual(stats.active_tree, 1) - self.assertEqual(stats.active_layer, 3) - self.assertEqual(stats.attempted_trees, 1) - self.assertEqual(stats.attempted_layers, 3) - self.assertProtoEquals(expected_result, tree_ensemble_config) - - def testGrowExistingEnsembleTreeFinalizedWithDropout(self): - """Test growing an existing ensemble with the last tree finalized.""" - with self.cached_session() as session: - # Create existing ensemble with one root split and one bias tree. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - text_format.Merge(""" - trees { - nodes { - leaf { - vector { - value: -0.32 - value: 0.28 - } - } - } - } - trees { - nodes { - categorical_id_binary_split { - feature_column: 3 - feature_id: 7 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 1.3 - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 2.3 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: -0.9 - } - } - } - } - tree_weights: 0.7 - tree_weights: 1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - is_finalized: true - } - tree_metadata { - num_tree_weight_updates: 5 - num_layers_grown: 1 - is_finalized: true - } - growing_metadata { - num_trees_attempted: 2 - num_layers_attempted: 2 - } - """, tree_ensemble_config) - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=1, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.PRE_PRUNE, - growing_mode=learner_pb2.LearnerConfig.WHOLE_TREE, - dropout_probability=1.0) - - # Prepare handler inputs. - handler1_partitions = np.array([0], dtype=np.int32) - handler1_gains = np.array([7.62], dtype=np.float32) - handler1_split = [_gen_dense_split_info(5, 0.52, -4.375, 7.143)] - handler2_partitions = np.array([0], dtype=np.int32) - handler2_gains = np.array([0.63], dtype=np.float32) - handler2_split = [_gen_dense_split_info(2, 0.23, -0.6, 0.24)] - handler3_partitions = np.array([0], dtype=np.int32) - handler3_gains = np.array([7.62], dtype=np.float32) - handler3_split = [_gen_categorical_split_info(8, 7, -4.375, 7.143)] - - # Grow tree ensemble. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=1, - partition_ids=[ - handler1_partitions, handler2_partitions, handler3_partitions - ], - gains=[handler1_gains, handler2_gains, handler3_gains], - splits=[handler1_split, handler2_split, handler3_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE) - session.run(grow_op) - - # Expect a new tree to be added with the split from handler 1. - _, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - tree_ensemble_config.ParseFromString(serialized) - - self.assertEqual(3, len(tree_ensemble_config.trees)) - # Both trees got 0.5 as weights, bias tree is untouched. - self.assertAllClose([0.7, 0.5, 0.5], tree_ensemble_config.tree_weights) - - self.assertEqual( - 1, tree_ensemble_config.tree_metadata[0].num_tree_weight_updates) - self.assertEqual( - 6, tree_ensemble_config.tree_metadata[1].num_tree_weight_updates) - self.assertEqual( - 2, tree_ensemble_config.tree_metadata[2].num_tree_weight_updates) - - def testGrowExistingEnsembleTreeWithFeatureSelectionUsedHandlers(self): - """Test growing a tree with feature selection.""" - with self.cached_session() as session: - # Create existing ensemble with one root split and one bias tree. - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - text_format.Merge(""" - trees { - nodes { - leaf { - vector { - value: -0.32 - value: 0.28 - } - } - } - } - trees { - nodes { - categorical_id_binary_split { - feature_column: 3 - feature_id: 7 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 1.3 - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: 2.3 - } - } - } - nodes { - leaf { - sparse_vector { - index: 0 - value: -0.9 - } - } - } - } - tree_weights: 0.7 - tree_weights: 1 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - is_finalized: true - } - tree_metadata { - num_tree_weight_updates: 5 - num_layers_grown: 1 - is_finalized: true - } - growing_metadata { - num_trees_attempted: 2 - num_layers_attempted: 2 - used_handler_ids: 2 - } - """, tree_ensemble_config) - tree_ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - - # Prepare learner config. - learner_config = _gen_learner_config( - num_classes=2, - l1_reg=0, - l2_reg=0, - tree_complexity=0, - max_depth=1, - min_node_weight=0, - pruning_mode=learner_pb2.LearnerConfig.PRE_PRUNE, - growing_mode=learner_pb2.LearnerConfig.WHOLE_TREE) - - learner_config.constraints.max_number_of_unique_feature_columns = 3 - # Prepare handler inputs. - handler1_partitions = np.array([0], dtype=np.int32) - handler1_gains = np.array([7.62], dtype=np.float32) - handler1_split = [_gen_dense_split_info(5, 0.52, -4.375, 7.143)] - handler2_partitions = np.array([0], dtype=np.int32) - handler2_gains = np.array([0.63], dtype=np.float32) - handler2_split = [_gen_dense_split_info(2, 0.23, -0.6, 0.24)] - handler3_partitions = np.array([0], dtype=np.int32) - handler3_gains = np.array([7.62], dtype=np.float32) - handler3_split = [_gen_categorical_split_info(8, 7, -4.375, 7.143)] - - # Grow tree ensemble. - grow_op = training_ops.grow_tree_ensemble( - tree_ensemble_handle, - stamp_token=0, - next_stamp_token=1, - learning_rate=1, - partition_ids=[ - handler1_partitions, handler2_partitions, handler3_partitions - ], - gains=[handler1_gains, handler2_gains, handler3_gains], - splits=[handler1_split, handler2_split, handler3_split], - learner_config=learner_config.SerializeToString(), - dropout_seed=123, - center_bias=True, - max_tree_depth=learner_config.constraints.max_tree_depth, - weak_learner_type=learner_pb2.LearnerConfig.NORMAL_DECISION_TREE) - session.run(grow_op) - - _, serialized = session.run( - model_ops.tree_ensemble_serialize(tree_ensemble_handle)) - tree_ensemble_config.ParseFromString(serialized) - self.assertEqual(3, len(tree_ensemble_config.trees)) - # 2 was already used. handler 0 is being added in this tree. - self.assertAllEqual( - [0, 2], tree_ensemble_config.growing_metadata.used_handler_ids) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/python/ops/batch_ops_utils.py b/tensorflow/contrib/boosted_trees/python/ops/batch_ops_utils.py deleted file mode 100644 index 8083d8fac85..00000000000 --- a/tensorflow/contrib/boosted_trees/python/ops/batch_ops_utils.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utility for batching remote OPs together to reduce RPC overhead.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc -import collections - -import six - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops - - -@six.add_metaclass(abc.ABCMeta) -class ScheduledOp(object): - """Represents a scheduled remote operation.""" - - @abc.abstractmethod - def batching_key(self): - """Returns the key for batching operations.""" - - @abc.abstractmethod - def batch_runner_fn(self): - """Returns the function that executes the operation on the batch.""" - - -class ScheduledStampedResourceOp(ScheduledOp): - """Wrapper class for batched operations on stamped resources.""" - - def __init__(self, resource_handle, op, **kwargs): - self.resource_handle = resource_handle - self.op = op - self.args = kwargs - - def batching_key(self): - # We want to group the same operations on the same device and run them in - # one batch. So we use (device, operation) as the key. - return self.resource_handle.device, self.op - - def batch_runner_fn(self): - return _scheduled_stamp_resource_op_runner - - -def _move_tensors(tensors, device): - """Moves a list of tensors to a device by concatenating/splitting them.""" - # Reset the device setting to avoid weird interactions with device merging - # logic. - zero = constant_op.constant(0, dtype=dtypes.int32) - with ops.device(None): - if all(tensor.shape.rank == 0 for tensor in tensors): - with ops.device(tensors[0].device): - values = array_ops.stack(tensors) - with ops.device(device): - return array_ops.unstack(values) - else: - with ops.device(tensors[0].device): - sizes = array_ops.stack(array_ops.shape_n(tensors))[:, 0] - values = array_ops.concat(tensors, axis=zero) - with ops.device(device): - sizes = array_ops.unstack(sizes) - return list(array_ops.split(values, sizes, axis=zero)) - - -def _scheduled_stamp_resource_op_runner(batch, stamp): - """Runs a batch operation on a stamped resource.""" - if not batch: - return - arg_keys = set(batch[0].args.keys()) - grouped_args = collections.OrderedDict() - resource_handles = [] - # Check that the set of arguments is the same across all the scheduled ops. - for op in batch: - if set(op.args.keys()) != arg_keys: - raise ValueError("Mismatching arguments: %s, %s.", op.args, arg_keys) - for key in arg_keys: - grouped_args.setdefault(key, []).append(op.args[key]) - resource_handles.append(op.resource_handle) - # Move all the inputs to the op device in one RPC. - grouped_args = collections.OrderedDict( - (k, _move_tensors(v, resource_handles[0].device)) - for k, v in sorted(grouped_args.items())) - with ops.device(resource_handles[0].device): - return batch[0].op(resource_handles, stamp, **grouped_args) - - -def run_handler_scheduled_ops(per_handler_ops, stamp, worker_device): - """Given a dictionary of ops for each handler, runs them in batch.""" - batched_ops = collections.OrderedDict() - # Group the ops by their batching_key. Ops that share the same batching key - # can be executed together. - for handler in per_handler_ops.keys(): - for op in per_handler_ops[handler]: - key = (op.batching_key(), op.batch_runner_fn()) - batched_ops.setdefault(key, []).append(op) - op_results = {} - for batch in batched_ops.values(): - # Run each of the batched ops using its runner. - results = batch[0].batch_runner_fn()(batch, stamp) - # If the result is a tuple, move each entry in the tuple in one RPC. - if isinstance(results, tuple): - results = tuple( - _move_tensors(result, worker_device) for result in results) - # Once all the results are on the worker, create individual tuple for - # each scheduled op request. - for i in range(len(batch)): - op_results[batch[i]] = tuple(result[i] for result in results) - # If the result is a tuple, it didn't have any outputs, so use the - # `ops.Operation` as the result for all the scheduled ops. - elif isinstance(results, ops.Operation): - for i in range(len(batch)): - op_results[batch[i]] = results - else: - raise ValueError("Unknown type of result %s.", results) - handler_results = collections.defaultdict(list) - # Dispatch the results of the ScheduledOps to the handlers that requested - # them. - for handler in per_handler_ops.keys(): - for op in per_handler_ops[handler]: - handler_results[handler].append(op_results[op]) - return handler_results diff --git a/tensorflow/contrib/boosted_trees/python/ops/boosted_trees_ops_loader.py b/tensorflow/contrib/boosted_trees/python/ops/boosted_trees_ops_loader.py deleted file mode 100644 index 6e23c150b24..00000000000 --- a/tensorflow/contrib/boosted_trees/python/ops/boosted_trees_ops_loader.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Loads the _boosted_trees_ops.so when the binary is not statically linked.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.util import loader -from tensorflow.python.framework import errors -from tensorflow.python.platform import resource_loader - -# Conditionally load ops, they might already be statically linked in. -try: - loader.load_op_library( - resource_loader.get_path_to_datafile('_boosted_trees_ops.so')) -except (errors.NotFoundError, IOError): - print('Error loading _boosted_trees_ops.so') diff --git a/tensorflow/contrib/boosted_trees/python/ops/model_ops.py b/tensorflow/contrib/boosted_trees/python/ops/model_ops.py deleted file mode 100644 index f9945959812..00000000000 --- a/tensorflow/contrib/boosted_trees/python/ops/model_ops.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model ops python wrappers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -# pylint: disable=unused-import -from tensorflow.contrib.boosted_trees.python.ops import boosted_trees_ops_loader -# pylint: enable=unused-import -from tensorflow.contrib.boosted_trees.python.ops import gen_model_ops -from tensorflow.contrib.boosted_trees.python.ops.gen_model_ops import tree_ensemble_deserialize -from tensorflow.contrib.boosted_trees.python.ops.gen_model_ops import tree_ensemble_serialize -# pylint: disable=unused-import -from tensorflow.contrib.boosted_trees.python.ops.gen_model_ops import tree_ensemble_stamp_token -from tensorflow.contrib.boosted_trees.python.ops.gen_model_ops import tree_ensemble_used_handlers -# pylint: enable=unused-import - -from tensorflow.python.framework import ops -from tensorflow.python.ops import resources -from tensorflow.python.training import saver -from tensorflow.python.training.tracking import tracking - -ops.NotDifferentiable("TreeEnsembleVariable") -ops.NotDifferentiable("TreeEnsembleSerialize") -ops.NotDifferentiable("TreeEnsembleDeserialize") - - -class TreeEnsembleVariableSavable(saver.BaseSaverBuilder.SaveableObject): - """SaveableObject implementation for TreeEnsembleVariable.""" - - def __init__(self, tree_ensemble_handle, create_op, name): - """Creates a TreeEnsembleVariableSavable object. - - Args: - tree_ensemble_handle: handle to the tree ensemble variable. - create_op: the op to initialize the variable. - name: the name to save the tree ensemble variable under. - """ - stamp_token, ensemble_config = tree_ensemble_serialize(tree_ensemble_handle) - # slice_spec is useful for saving a slice from a variable. - # It's not meaningful the tree ensemble variable. So we just pass an empty - # value. - slice_spec = "" - specs = [ - saver.BaseSaverBuilder.SaveSpec(stamp_token, slice_spec, - name + "_stamp"), - saver.BaseSaverBuilder.SaveSpec(ensemble_config, slice_spec, - name + "_config"), - ] - super(TreeEnsembleVariableSavable, self).__init__(tree_ensemble_handle, - specs, name) - self._tree_ensemble_handle = tree_ensemble_handle - self._create_op = create_op - - def restore(self, restored_tensors, unused_restored_shapes): - """Restores the associated tree ensemble from 'restored_tensors'. - - Args: - restored_tensors: the tensors that were loaded from a checkpoint. - unused_restored_shapes: the shapes this object should conform to after - restore. Not meaningful for trees. - - Returns: - The operation that restores the state of the tree ensemble variable. - """ - with ops.control_dependencies([self._create_op]): - return tree_ensemble_deserialize( - self._tree_ensemble_handle, - stamp_token=restored_tensors[0], - tree_ensemble_config=restored_tensors[1]) - - -class TreeEnsembleVariable(tracking.TrackableResource): - """A Tree ensemble model.""" - - def __init__(self, stamp_token, tree_ensemble_config, name, container=None): - self._stamp_token = stamp_token - self._tree_ensemble_config = tree_ensemble_config - self._name = name - self._container = container - self._init_op = None - super(TreeEnsembleVariable, self).__init__() - - def _create_resource(self): - return gen_model_ops.decision_tree_ensemble_resource_handle_op( - self._container, shared_name=self._name, name=self._name) - - def _initialize(self): - return gen_model_ops.create_tree_ensemble_variable( - self.resource_handle, self._stamp_token, self._tree_ensemble_config) - - @property - def initializer(self): - if self._init_op is None: - self._init_op = self._initialize() - return self._init_op - - def is_initialized(self): - return gen_model_ops.tree_ensemble_is_initialized_op(self.resource_handle) - - def _gather_saveables_for_checkpoint(self): - return { - self.resource_handle.op.name + "/tree_ensemble_variable": - functools.partial( - TreeEnsembleVariableSavable, - tree_ensemble_handle=self.resource_handle, - create_op=self.initializer) - } - - -def tree_ensemble_variable(stamp_token, - tree_ensemble_config, - name, - container=None): - r"""Creates a tree ensemble model and returns a handle to it. - - Args: - stamp_token: The initial stamp token value for the ensemble resource. - tree_ensemble_config: A `Tensor` of type `string`. Serialized proto of the - tree ensemble. - name: A name for the ensemble variable. - container: An optional `string`. Defaults to `""`. - - Returns: - A `Tensor` of type mutable `string`. The handle to the tree ensemble. - """ - with ops.name_scope(name, "TreeEnsembleVariable") as name: - tree_ensemble_var = TreeEnsembleVariable(stamp_token, tree_ensemble_config, - name, container) - resource_handle = tree_ensemble_var.resource_handle - create_op = tree_ensemble_var.initializer - is_initialized_op = tree_ensemble_var.is_initialized() - # Adds the variable to the savable list. - saveable = TreeEnsembleVariableSavable(resource_handle, create_op, - resource_handle.name) - ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) - resources.register_resource(resource_handle, create_op, is_initialized_op) - return resource_handle diff --git a/tensorflow/contrib/boosted_trees/python/ops/prediction_ops.py b/tensorflow/contrib/boosted_trees/python/ops/prediction_ops.py deleted file mode 100644 index 7f6e55ae588..00000000000 --- a/tensorflow/contrib/boosted_trees/python/ops/prediction_ops.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Split handler custom ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.contrib.boosted_trees.python.ops import boosted_trees_ops_loader -from tensorflow.contrib.boosted_trees.python.ops.gen_prediction_ops import gradient_trees_partition_examples -from tensorflow.contrib.boosted_trees.python.ops.gen_prediction_ops import gradient_trees_prediction -from tensorflow.contrib.boosted_trees.python.ops.gen_prediction_ops import gradient_trees_prediction_verbose -# pylint: enable=unused-import diff --git a/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py b/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py deleted file mode 100644 index 82f9b17b330..00000000000 --- a/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py +++ /dev/null @@ -1,233 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Quantile ops python wrappers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import re - -from tensorflow.contrib.boosted_trees.python.ops import batch_ops_utils -# pylint: disable=unused-import -from tensorflow.contrib.boosted_trees.python.ops import boosted_trees_ops_loader -# pylint: enable=unused-import -from tensorflow.contrib.boosted_trees.python.ops import gen_quantile_ops - -# go/tf-wildcard-import -# pylint: disable=wildcard-import,undefined-variable -from tensorflow.contrib.boosted_trees.python.ops.gen_quantile_ops import * -# pylint: enable=wildcard-import,undefined-variable - -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import resources -from tensorflow.python.training import saver -from tensorflow.python.training.tracking import tracking - -# Pattern to remove all non alpha numeric from a string. -_PATTERN = re.compile(r"[\W_]+") - - -class QuantileAccumulatorSaveable(saver.BaseSaverBuilder.SaveableObject): - """SaveableObject implementation for QuantileAccumulator.""" - - def __init__(self, resource_handle, create_op, name): - self._resource_handle = resource_handle - self._create_op = create_op - stamp_token, state, are_buckets_ready, buckets = ( - gen_quantile_ops.quantile_accumulator_serialize(resource_handle)) - # slice_spec is useful for saving a slice from a variable. - # It's not meaningful in quantile accumulator. - slice_spec = "" - def make_save_spec(tensor, suffix): - return saver.BaseSaverBuilder.SaveSpec(tensor, slice_spec, name + suffix) - - specs = [make_save_spec(stamp_token, "_stamp")] - specs += [make_save_spec(state, "_state")] - specs += [make_save_spec(are_buckets_ready, "_are_buckets_ready")] - specs += [make_save_spec(buckets, "buckets")] - super(QuantileAccumulatorSaveable, self).__init__(self._resource_handle, - specs, name) - - def restore(self, restored_tensors, unused_restored_shapes): - """Restores the associated quantile accumulator from 'restored_tensors'. - - Args: - restored_tensors: the tensors that were loaded from a checkpoint. - unused_restored_shapes: the shapes this object should conform to after - restore. - - Returns: - The operation that restores the state of the quantile accumulator. - """ - # Read the restored tensors with the same order that were added to saving - # spec. - stamp_token = restored_tensors[:1] - state = restored_tensors[1:2] - are_buckets_ready = restored_tensors[2:3] - buckets = restored_tensors[3] - with ops.control_dependencies([self._create_op]): - return gen_quantile_ops.quantile_accumulator_deserialize( - self._resource_handle, - stamp_token=stamp_token, - stream_state=state, - are_buckets_ready=are_buckets_ready, - buckets=buckets) - - -class QuantileAccumulator(tracking.TrackableResource): - """A resource that allows distributed quantile computation.""" - - def __init__(self, - init_stamp_token, - epsilon, - num_quantiles, - max_elements=None, - name=None, - container=None, - generate_quantiles=False): - """Creates a QuantileAccumulator object. - - Args: - init_stamp_token: The initial value for the stamp token. - epsilon: Error bound on the quantile computation. - num_quantiles: Number of quantiles to produce from the final summary. - max_elements: Maximum number of elements added to the accumulator. - name: the name to save the accumulator under. - container: An optional `string`. Defaults to `""` - generate_quantiles: Generate quantiles instead of approximate boundaries. - If true, exactly `num_quantiles` will be produced in the final summary. - """ - self._init_stamp_token = init_stamp_token - self._epsilon = epsilon - self._num_quantiles = num_quantiles - self._max_elements = max_elements - self._container = container - self._generate_quantiles = generate_quantiles - super(QuantileAccumulator, self).__init__() - - name = _PATTERN.sub("", name) - with ops.name_scope(name, "QuantileAccumulator") as name: - self._name = name - self._resource_handle = self._create_resource() - self._init_op = self._initialize() - is_initialized_op = self.is_initialized() - resources.register_resource(self.resource_handle, self._init_op, - is_initialized_op) - self._saveable = QuantileAccumulatorSaveable(self.resource_handle, - self._init_op, name) - ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, self._saveable) - - def _create_resource(self): - return gen_quantile_ops.quantile_stream_resource_handle_op( - container=self._container, shared_name=self._name, name=self._name) - - def _initialize(self): - return gen_quantile_ops.create_quantile_accumulator( - self.resource_handle, - self._init_stamp_token, - epsilon=self._epsilon, - max_elements=self._max_elements, - num_quantiles=self._num_quantiles, - generate_quantiles=self._generate_quantiles) - - @property - def initializer(self): - if self._init_op is None: - self._init_op = self._initialize() - return self._init_op - - def is_initialized(self): - return gen_quantile_ops.quantile_accumulator_is_initialized( - self.resource_handle) - - def _gather_saveables_for_checkpoint(self): - return {"quantile_accumulator", self.saveable} - - def get_buckets(self, stamp_token): - """Returns quantile buckets created during previous flush.""" - are_buckets_ready, buckets = ( - gen_quantile_ops.quantile_accumulator_get_buckets( - quantile_accumulator_handles=[self.resource_handle], - stamp_token=stamp_token)) - return are_buckets_ready[0], buckets[0] - - def schedule_get_buckets(self): - """Returns a scheduled read of buckets created during previous flush.""" - return batch_ops_utils.ScheduledStampedResourceOp( - resource_handle=self.resource_handle, - op=gen_quantile_ops.quantile_accumulator_get_buckets) - - def _make_summary(self, column, example_weights): - if isinstance(column, sparse_tensor.SparseTensor): - return gen_quantile_ops.make_quantile_summaries( - dense_float_features=[], - sparse_float_feature_indices=[column.indices], - sparse_float_feature_values=[column.values], - sparse_float_feature_shapes=[column.dense_shape], - example_weights=example_weights, - epsilon=self._epsilon / 2).sparse_summaries[0] - else: - return gen_quantile_ops.make_quantile_summaries( - dense_float_features=[column], - sparse_float_feature_indices=[], - sparse_float_feature_values=[], - sparse_float_feature_shapes=[], - example_weights=example_weights, - epsilon=self._epsilon / 2).dense_summaries[0] - - def add_summary(self, stamp_token, column, example_weights): - """Adds quantile summary to its stream in resource.""" - summary = self._make_summary(column, example_weights) - return gen_quantile_ops.quantile_accumulator_add_summaries( - quantile_accumulator_handles=[self.resource_handle], - stamp_token=stamp_token, - summaries=[summary]) - - def add_prebuilt_summary(self, stamp_token, summary): - """Adds quantile summary to its stream in resource.""" - return gen_quantile_ops.quantile_accumulator_add_summaries( - quantile_accumulator_handles=[self.resource_handle], - stamp_token=stamp_token, - summaries=[summary]) - - def schedule_add_summary(self, stamp_token, column, example_weights): - """Schedules to add a quantile summary to its stream in resource.""" - summary = self._make_summary(column, example_weights) - return batch_ops_utils.ScheduledStampedResourceOp( - op=gen_quantile_ops.quantile_accumulator_add_summaries, - resource_handle=self.resource_handle, - summaries=summary) - - def flush(self, stamp_token, next_stamp_token): - """Finalizes quantile summary stream and resets it for next iteration. - - Args: - stamp_token: Expected current token. - next_stamp_token: Next value for the token. - Returns: - The flush operation. - """ - return gen_quantile_ops.quantile_accumulator_flush( - quantile_accumulator_handle=self.resource_handle, - stamp_token=stamp_token, - next_stamp_token=next_stamp_token) - - def flush_summary(self, stamp_token, next_stamp_token): - """Finalizes quantile summary stream and resets it for next iteration.""" - result = gen_quantile_ops.quantile_accumulator_flush_summary( - quantile_accumulator_handle=self.resource_handle, - stamp_token=stamp_token, - next_stamp_token=next_stamp_token) - return result diff --git a/tensorflow/contrib/boosted_trees/python/ops/split_handler_ops.py b/tensorflow/contrib/boosted_trees/python/ops/split_handler_ops.py deleted file mode 100644 index b80e4918ed6..00000000000 --- a/tensorflow/contrib/boosted_trees/python/ops/split_handler_ops.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Split handler custom ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.contrib.boosted_trees.python.ops import boosted_trees_ops_loader -# pylint: enable=unused-import -# pylint: disable=wildcard-import -from tensorflow.contrib.boosted_trees.python.ops.gen_split_handler_ops import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/boosted_trees/python/ops/stats_accumulator_ops.py b/tensorflow/contrib/boosted_trees/python/ops/stats_accumulator_ops.py deleted file mode 100644 index 62d0d0821b2..00000000000 --- a/tensorflow/contrib/boosted_trees/python/ops/stats_accumulator_ops.py +++ /dev/null @@ -1,243 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stats Accumulator ops python wrappers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import re -from tensorflow.contrib.boosted_trees.python.ops import batch_ops_utils -# pylint: disable=unused-import -from tensorflow.contrib.boosted_trees.python.ops import boosted_trees_ops_loader -# pylint: enable=unused-import -from tensorflow.contrib.boosted_trees.python.ops import gen_stats_accumulator_ops -from tensorflow.python.framework import ops -from tensorflow.python.ops import resources -from tensorflow.python.training import saver -from tensorflow.python.training.tracking import tracking - -# Pattern to remove all non alpha numeric from a string. -_PATTERN = re.compile(r"[\W_]+") - - -class StatsAccumulatorSaveable(saver.BaseSaverBuilder.SaveableObject): - """SaveableObject implementation for StatsAccumulator.""" - - def __init__(self, resource_handle, create_op, is_scalar, name): - self._create_op = create_op - self._resource_handle = resource_handle - self._is_scalar = is_scalar - slice_spec = "" - saver_name = self._resource_handle.name - (stamp_token, num_updates, partition_ids, feature_ids, gradients, - hessians) = self.serialize() - specs = [ - saver.BaseSaverBuilder.SaveSpec(stamp_token, slice_spec, - saver_name + "_stamp"), - saver.BaseSaverBuilder.SaveSpec(num_updates, slice_spec, - saver_name + "_num_updates"), - saver.BaseSaverBuilder.SaveSpec(partition_ids, slice_spec, - saver_name + "_partition_ids"), - saver.BaseSaverBuilder.SaveSpec(feature_ids, slice_spec, - saver_name + "_feature_ids"), - saver.BaseSaverBuilder.SaveSpec(gradients, slice_spec, - saver_name + "_gradients"), - saver.BaseSaverBuilder.SaveSpec(hessians, slice_spec, - saver_name + "hessians"), - ] - super(StatsAccumulatorSaveable, self).__init__(self._resource_handle, specs, - name) - - def serialize(self): - """Serializes the stats accumulator state.""" - if self._is_scalar: - return gen_stats_accumulator_ops.stats_accumulator_scalar_serialize( - self._resource_handle) - else: - return gen_stats_accumulator_ops.stats_accumulator_tensor_serialize( - self._resource_handle) - - def deserialize(self, stamp_token, num_updates, partition_ids, feature_ids, - gradients, hessians): - """Resets the stats accumulator with the serialized state.""" - if self._is_scalar: - return gen_stats_accumulator_ops.stats_accumulator_scalar_deserialize( - self._resource_handle, stamp_token, num_updates, partition_ids, - feature_ids, gradients, hessians) - else: - return gen_stats_accumulator_ops.stats_accumulator_tensor_deserialize( - self._resource_handle, stamp_token, num_updates, partition_ids, - feature_ids, gradients, hessians) - - def restore(self, restored_tensors, unused_restored_shapes): - """Restores the associated tree ensemble from 'restored_tensors'. - - Args: - restored_tensors: the tensors that were loaded from a checkpoint. - unused_restored_shapes: the shapes this object should conform to after - restore. Not meaningful for trees. - - Returns: - The operation that restores the state of the tree ensemble variable. - """ - with ops.control_dependencies([self._create_op]): - return self.deserialize( - stamp_token=restored_tensors[0], - num_updates=restored_tensors[1], - partition_ids=restored_tensors[2], - feature_ids=restored_tensors[3], - gradients=restored_tensors[4], - hessians=restored_tensors[5]) - - -class StatsAccumulator(tracking.TrackableResource): - """A resource that allows to accumulate gradients and hessians. - - For consistency guarantees, we use read and write stamp tokens. - The stamp token on the resource is updated with StatsAccumulator.flush. - Calls to StatsAccumulator.add that don't provide the current stamp token are - ignored. - """ - - def __init__(self, - stamp_token, - gradient_shape, - hessian_shape, - name=None, - container=None): - """Creates a stats accumulator and returns a handle to it. - - Args: - stamp_token: An int64, initial value to use for the stamp token. - gradient_shape: A TensorShape, containing shape of gradients. - hessian_shape: A TensorShape, containing shape of hessians. - name: A name for the stats accumulator variable. - container: An optional `string`. Defaults to `""`. - - Returns: - A `Tensor` of type mutable `string`. The handle to the stats accumulator. - """ - self._stamp_token = stamp_token - self._gradient_shape = gradient_shape - self._hessian_shape = hessian_shape - self._container = container - - if (gradient_shape.rank == 0 and hessian_shape.rank == 0): - self._is_scalar = True - else: - self._is_scalar = False - - if name is not None: - name = _PATTERN.sub("", name) - with ops.name_scope(name, "StatsAccumulator") as name: - self._name = name - self._resource_handle = self._create_resource() - self._init_op = self._initialize() - is_initialized_op = self.is_initialized() - resources.register_resource(self.resource_handle, self.initializer, - is_initialized_op) - self._saveable = StatsAccumulatorSaveable( - self.resource_handle, self.initializer, self._is_scalar, name) - ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, self._saveable) - - def _create_resource(self): - if self._is_scalar: - return ( - gen_stats_accumulator_ops.stats_accumulator_scalar_resource_handle_op( - self._container, self._name, name=self._name)) - else: - return ( - gen_stats_accumulator_ops.stats_accumulator_tensor_resource_handle_op( - self._container, self._name, name=self._name)) - - def _initialize(self): - if self._is_scalar: - return gen_stats_accumulator_ops.create_stats_accumulator_scalar( - self.resource_handle, self._stamp_token) - else: - return gen_stats_accumulator_ops.create_stats_accumulator_tensor( - self.resource_handle, self._stamp_token, - self._gradient_shape.as_list(), self._hessian_shape.as_list()) - - @property - def initializer(self): - if self._init_op is None: - self._init_op = self._initialize() - return self._init_op - - def is_initialized(self): - if self._is_scalar: - return gen_stats_accumulator_ops.stats_accumulator_scalar_is_initialized( - self.resource_handle) - else: - return gen_stats_accumulator_ops.stats_accumulator_tensor_is_initialized( - self.resource_handle) - - @property - def saveable(self): - return self._saveable - - def _gather_saveables_for_checkpoint(self): - return {"stats_accumulator", self.saveable} - - def add(self, stamp_token, partition_ids, feature_ids, gradients, hessians): - """Updates the stats accumulator.""" - partition_ids, feature_ids, gradients, hessians = (self._make_summary( - partition_ids, feature_ids, gradients, hessians)) - if self._is_scalar: - return gen_stats_accumulator_ops.stats_accumulator_scalar_add( - [self.resource_handle], stamp_token, [partition_ids], [feature_ids], - [gradients], [hessians]) - else: - return gen_stats_accumulator_ops.stats_accumulator_tensor_add( - [self.resource_handle], stamp_token, [partition_ids], [feature_ids], - [gradients], [hessians]) - - def schedule_add(self, partition_ids, feature_ids, gradients, hessians): - """Schedules an update to the stats accumulator.""" - partition_ids, feature_ids, gradients, hessians = (self._make_summary( - partition_ids, feature_ids, gradients, hessians)) - if self._is_scalar: - return batch_ops_utils.ScheduledStampedResourceOp( - op=gen_stats_accumulator_ops.stats_accumulator_scalar_add, - resource_handle=self.resource_handle, - partition_ids=partition_ids, - feature_ids=feature_ids, - gradients=gradients, - hessians=hessians) - else: - return batch_ops_utils.ScheduledStampedResourceOp( - op=gen_stats_accumulator_ops.stats_accumulator_tensor_add, - resource_handle=self.resource_handle, - partition_ids=partition_ids, - feature_ids=feature_ids, - gradients=gradients, - hessians=hessians) - - def _make_summary(self, partition_ids, feature_ids, gradients, hessians): - if self._is_scalar: - return gen_stats_accumulator_ops.stats_accumulator_scalar_make_summary( - partition_ids, feature_ids, gradients, hessians) - else: - return gen_stats_accumulator_ops.stats_accumulator_tensor_make_summary( - partition_ids, feature_ids, gradients, hessians) - - def flush(self, stamp_token, next_stamp_token): - """Flushes the stats accumulator.""" - if self._is_scalar: - return gen_stats_accumulator_ops.stats_accumulator_scalar_flush( - self.resource_handle, stamp_token, next_stamp_token) - else: - return gen_stats_accumulator_ops.stats_accumulator_tensor_flush( - self.resource_handle, stamp_token, next_stamp_token) diff --git a/tensorflow/contrib/boosted_trees/python/ops/training_ops.py b/tensorflow/contrib/boosted_trees/python/ops/training_ops.py deleted file mode 100644 index d589a0eaa49..00000000000 --- a/tensorflow/contrib/boosted_trees/python/ops/training_ops.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Import and conditionally load custom ops for training boosted trees.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.contrib.boosted_trees.python.ops import boosted_trees_ops_loader -# pylint: enable=unused-import -# pylint: disable=wildcard-import -from tensorflow.contrib.boosted_trees.python.ops.gen_training_ops import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/boosted_trees/python/training/__init__.py b/tensorflow/contrib/boosted_trees/python/training/__init__.py deleted file mode 100644 index b569ac5fdb6..00000000000 --- a/tensorflow/contrib/boosted_trees/python/training/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""training module under boosted_trees.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/__init__.py b/tensorflow/contrib/boosted_trees/python/training/functions/__init__.py deleted file mode 100644 index c1750117cd7..00000000000 --- a/tensorflow/contrib/boosted_trees/python/training/functions/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""functions module under boosted_trees.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py deleted file mode 100644 index ffad201cbf1..00000000000 --- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py +++ /dev/null @@ -1,1338 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Training functions for Gradient boosted decision trees.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import copy - -from tensorflow.contrib import learn -from tensorflow.contrib.boosted_trees.lib.learner.batch import categorical_split_handler -from tensorflow.contrib.boosted_trees.lib.learner.batch import ordinal_split_handler -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.boosted_trees.python.ops import batch_ops_utils -from tensorflow.contrib.boosted_trees.python.ops import gen_model_ops -from tensorflow.contrib.boosted_trees.python.ops import model_ops -from tensorflow.contrib.boosted_trees.python.ops import prediction_ops -from tensorflow.contrib.boosted_trees.python.ops import stats_accumulator_ops -from tensorflow.contrib.boosted_trees.python.ops import training_ops -from tensorflow.contrib.layers.python.layers import feature_column as feature_column_lib -from tensorflow.contrib.layers.python.layers import feature_column_ops -from tensorflow.python.feature_column import feature_column as fc_core -from tensorflow.python.feature_column import feature_column_v2 as fc_v2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import stateless_random_ops as stateless -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.summary import summary -from tensorflow.python.training import device_setter - - -# Key names for prediction dict. -ENSEMBLE_STAMP = "ensemble_stamp" -PREDICTIONS = "predictions" -PARTITION_IDS = "partition_ids" -NUM_LAYERS_ATTEMPTED = "num_layers" -NUM_TREES_ATTEMPTED = "num_trees" -NUM_USED_HANDLERS = "num_used_handlers" -USED_HANDLERS_MASK = "used_handlers_mask" -LEAF_INDEX = "leaf_index" -_FEATURE_NAME_TEMPLATE = "%s_%d" - -# Keys in Training state. -GBDTTrainingState = collections.namedtuple("GBDTTrainingState", [ - "num_layer_examples", "num_layer_steps", "num_layers", "active_tree", - "active_layer", "continue_centering", "bias_stats_accumulator", - "steps_accumulator", "handlers" -]) - - -def _get_column_by_index(tensor, indices): - """Returns columns from a 2-D tensor by index.""" - shape = array_ops.shape(tensor) - p_flat = array_ops.reshape(tensor, [-1]) - i_flat = array_ops.reshape( - array_ops.reshape(math_ops.range(0, shape[0]) * shape[1], [-1, 1]) + - indices, [-1]) - return array_ops.reshape(array_ops.gather(p_flat, i_flat), [shape[0], -1]) - - -def _make_predictions_dict(stamp, - logits, - partition_ids, - ensemble_stats, - used_handlers, - leaf_index=None): - """Returns predictions for the given logits and n_classes. - - Args: - stamp: The ensemble stamp. - logits: A rank 2 `Tensor` with shape [batch_size, n_classes - 1]. that - contains predictions when no dropout was applied. - partition_ids: A rank 1 `Tensor` with shape [batch_size]. - ensemble_stats: A TreeEnsembleStatsOp result tuple. - used_handlers: A TreeEnsembleUsedHandlerOp result tuple of an int and a - boolean mask. - leaf_index: A rank 2 `Tensor` with shape [batch_size, number of trees]. that - contains leaf id for each example prediction. - - Returns: - A dict of predictions. - """ - result = {} - result[ENSEMBLE_STAMP] = stamp - result[PREDICTIONS] = logits - result[PARTITION_IDS] = partition_ids - result[NUM_LAYERS_ATTEMPTED] = ensemble_stats.attempted_layers - result[NUM_TREES_ATTEMPTED] = ensemble_stats.attempted_trees - result[NUM_USED_HANDLERS] = used_handlers.num_used_handlers - result[USED_HANDLERS_MASK] = used_handlers.used_handlers_mask - if leaf_index is not None: - result[LEAF_INDEX] = leaf_index - return result - - -class _OpRoundRobinStrategy(object): - """Returns the next ps task index for placement via per-Op round-robin order. - - This strategy works slightly better for the GBDT graph because of using - custom resources which vary significantly in compute cost. - """ - - def __init__(self, ps_ops, num_tasks): - """Create a new `_RoundRobinStrategy`. - - Args: - ps_ops: List of Op types to place on PS. - num_tasks: Number of ps tasks to cycle among. - """ - next_task = 0 - self._next_task_per_op = {} - for op in ps_ops: - self._next_task_per_op[op] = next_task - next_task = (next_task + 1) % num_tasks if num_tasks else 0 - self._num_tasks = num_tasks - - def __call__(self, op): - """Choose a ps task index for the given `Operation`. - - Args: - op: An `Operation` to be placed on ps. - - Returns: - The next ps task index to use for the `Operation`. Returns the next - index, in the range `[offset, offset + num_tasks)`. - - Raises: - ValueError: If attempting to place non-PS Op. - """ - if op.type not in self._next_task_per_op: - raise ValueError("Unknown op type '%s' for placement:" % op.type) - task = self._next_task_per_op[op.type] - self._next_task_per_op[op.type] = ((task + 1) % self._num_tasks - if self._num_tasks else 0) - return task - - -def extract_features(features, feature_columns, use_core_columns): - """Extracts columns from a dictionary of features. - - Args: - features: `dict` of `Tensor` objects. - feature_columns: A list of feature_columns. - - Returns: - Seven values: - - A list of all feature column names. - - A list of dense floats. - - A list of sparse float feature indices. - - A list of sparse float feature values. - - A list of sparse float feature shapes. - - A list of sparse int feature indices. - - A list of sparse int feature values. - - A list of sparse int feature shapes. - Raises: - ValueError: if features is not valid. - """ - if not features: - raise ValueError("Features dictionary must be specified.") - - # Make a shallow copy of features to ensure downstream usage - # is unaffected by modifications in the model function. - features = copy.copy(features) - # pylint: disable=protected-access - state_manager = fc_v2._StateManagerImpl(layer=None, trainable=False) - if feature_columns: - scope = "gbdt" - with variable_scope.variable_scope(scope): - feature_columns = list(feature_columns) - transformed_features = collections.OrderedDict() - for fc in feature_columns: - if use_core_columns: - if isinstance(fc, fc_v2.FeatureColumn): - tensor = fc_v2._transform_features_v2( - features, [fc], state_manager)[fc] - else: - tensor = fc_core._transform_features(features, [fc])[fc] - transformed_features[fc.name] = tensor - elif isinstance(fc, feature_column_lib._EmbeddingColumn): - # pylint: enable=protected-access - transformed_features[fc.name] = fc_core.input_layer( - features, [fc], weight_collections=[scope]) - else: - result = feature_column_ops.transform_features(features, [fc]) - if len(result) > 1: - raise ValueError("Unexpected number of output features") - transformed_features[fc.name] = result[list(result.keys())[0]] - features = transformed_features - - dense_float_names = [] - dense_floats = [] - sparse_float_names = [] - sparse_float_indices = [] - sparse_float_values = [] - sparse_float_shapes = [] - sparse_int_names = [] - sparse_int_indices = [] - sparse_int_values = [] - sparse_int_shapes = [] - for key in sorted(features.keys()): - tensor = features[key] - # TODO(nponomareva): consider iterating over feature columns instead. - if isinstance(tensor, tuple): - # Weighted categorical feature. - categorical_tensor = tensor[0] - weight_tensor = tensor[1] - - shape = categorical_tensor.dense_shape - indices = array_ops.concat([ - array_ops.slice(categorical_tensor.indices, [0, 0], [-1, 1]), - array_ops.expand_dims( - math_ops.cast(categorical_tensor.values, dtypes.int64), -1) - ], 1) - tensor = sparse_tensor.SparseTensor( - indices=indices, values=weight_tensor.values, dense_shape=shape) - - if isinstance(tensor, sparse_tensor.SparseTensor): - if tensor.values.dtype == dtypes.float32: - sparse_float_names.append(key) - sparse_float_indices.append(tensor.indices) - sparse_float_values.append(tensor.values) - sparse_float_shapes.append(tensor.dense_shape) - elif tensor.values.dtype == dtypes.int64: - sparse_int_names.append(key) - sparse_int_indices.append(tensor.indices) - sparse_int_values.append(tensor.values) - sparse_int_shapes.append(tensor.dense_shape) - else: - raise ValueError("Unsupported sparse feature %s with dtype %s." % - (tensor.indices.name, tensor.dtype)) - else: - if tensor.dtype == dtypes.float32: - if len(tensor.shape) > 1 and tensor.shape[1] > 1: - unstacked = array_ops.unstack(tensor, axis=1) - for i in range(len(unstacked)): - dense_float_names.append(_FEATURE_NAME_TEMPLATE % (key, i)) - dense_floats.append(array_ops.reshape(unstacked[i], [-1, 1])) - else: - dense_float_names.append(key) - dense_floats.append(tensor) - else: - raise ValueError("Unsupported dense feature %s with dtype %s." % - (tensor.name, tensor.dtype)) - # Feature columns are logically organized into incrementing slots starting - # from dense floats, then sparse floats then sparse ints. - fc_names = (dense_float_names + sparse_float_names + sparse_int_names) - return (fc_names, dense_floats, sparse_float_indices, sparse_float_values, - sparse_float_shapes, sparse_int_indices, sparse_int_values, - sparse_int_shapes) - - -def _dropout_params(mode, ensemble_stats): - """Returns parameters relevant for dropout. - - Args: - mode: Train/Eval/Infer - ensemble_stats: A TreeEnsembleStatsOp result tuple. - - Returns: - Whether to apply dropout and a dropout seed. - """ - if mode == learn.ModeKeys.TRAIN: - # Do dropout only during training. - apply_dropout = True - seed = ensemble_stats.attempted_trees - else: - seed = -1 - apply_dropout = False - return apply_dropout, seed - - -class GradientBoostedDecisionTreeModel(object): - """A GBDT model function.""" - - def __init__(self, - is_chief, - num_ps_replicas, - ensemble_handle, - center_bias, - examples_per_layer, - learner_config, - features, - logits_dimension, - loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS, - feature_columns=None, - use_core_columns=False, - output_leaf_index=False, - output_leaf_index_modes=None, - num_quantiles=100): - """Construct a new GradientBoostedDecisionTreeModel function. - - Args: - is_chief: Whether to build the chief graph. - num_ps_replicas: Number of parameter server replicas, can be 0. - ensemble_handle: A handle to the ensemble variable. - center_bias: Whether to center the bias before growing trees. - examples_per_layer: Number of examples to accumulate before growing a tree - layer. It can also be a function that computes the number of examples - based on the depth of the layer that's being built. - learner_config: A learner config. - features: `dict` of `Tensor` objects. - logits_dimension: An int, the dimension of logits. - loss_reduction: Either `SUM_OVER_NONZERO_WEIGHTS` (mean) or `SUM`. - feature_columns: A list of feature columns. - use_core_columns: A boolean specifying whether core feature columns are - used. - output_leaf_index: A boolean variable indicating whether to output leaf - index into predictions dictionary. - output_leaf_index_modes: A list of modes from (TRAIN, EVAL, INFER) which - dictates when leaf indices will be outputted. By default, leaf indices - are only outputted in INFER mode. - num_quantiles: Number of quantiles to build for numeric feature values. - - Raises: - ValueError: if inputs are not valid. - """ - if ensemble_handle is None: - raise ValueError("ensemble_handle must be specified.") - - if learner_config is None: - raise ValueError("learner_config must be specified.") - - if learner_config.num_classes < 2: - raise ValueError("Number of classes must be >=2") - - self._logits_dimension = logits_dimension - self._is_chief = is_chief - self._num_ps_replicas = num_ps_replicas - self._ensemble_handle = ensemble_handle - self._center_bias = center_bias - self._examples_per_layer = examples_per_layer - - # Check loss reduction value. - if (loss_reduction != losses.Reduction.SUM and - loss_reduction != losses.Reduction.SUM_OVER_NONZERO_WEIGHTS): - raise ValueError( - "Invalid loss reduction is provided: %s." % loss_reduction) - self._loss_reduction = loss_reduction - - # Fill in the defaults. - if (learner_config.multi_class_strategy == - learner_pb2.LearnerConfig.MULTI_CLASS_STRATEGY_UNSPECIFIED): - if logits_dimension == 1: - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.TREE_PER_CLASS) - else: - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) - - if logits_dimension == 1 or learner_config.multi_class_strategy == ( - learner_pb2.LearnerConfig.TREE_PER_CLASS): - self._gradient_shape = tensor_shape.TensorShape([]) - self._hessian_shape = tensor_shape.TensorShape([]) - else: - if center_bias: - raise ValueError("Center bias should be False for multiclass.") - - self._gradient_shape = tensor_shape.TensorShape([logits_dimension]) - if (learner_config.multi_class_strategy == - learner_pb2.LearnerConfig.FULL_HESSIAN): - self._hessian_shape = tensor_shape.TensorShape( - ([logits_dimension, logits_dimension])) - else: - # Diagonal hessian strategy. - self._hessian_shape = tensor_shape.TensorShape(([logits_dimension])) - if (learner_config.growing_mode == - learner_pb2.LearnerConfig.GROWING_MODE_UNSPECIFIED): - learner_config.growing_mode = learner_pb2.LearnerConfig.LAYER_BY_LAYER - - if (learner_config.weak_learner_type == learner_pb2.LearnerConfig - .OBLIVIOUS_DECISION_TREE and learner_config.pruning_mode == learner_pb2 - .LearnerConfig.PRUNING_MODE_UNSPECIFIED): - learner_config.pruning_mode = learner_pb2.LearnerConfig.PRE_PRUNE - - if (learner_config.pruning_mode == - learner_pb2.LearnerConfig.PRUNING_MODE_UNSPECIFIED): - learner_config.pruning_mode = learner_pb2.LearnerConfig.POST_PRUNE - - if (learner_config.weak_learner_type == learner_pb2.LearnerConfig - .OBLIVIOUS_DECISION_TREE and - learner_config.pruning_mode == learner_pb2.LearnerConfig.POST_PRUNE): - raise ValueError( - "Post pruning is not implmented for oblivious decision trees.") - - if learner_config.constraints.max_tree_depth == 0: - # Use 6 as the default maximum depth. - learner_config.constraints.max_tree_depth = 6 - - tuner = learner_config.learning_rate_tuner.WhichOneof("tuner") - if not tuner: - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - - self._learner_config = learner_config - self._feature_columns = feature_columns - self._learner_config_serialized = learner_config.SerializeToString() - self._num_quantiles = num_quantiles - self._max_tree_depth = variables.VariableV1( - initial_value=self._learner_config.constraints.max_tree_depth) - self._attempted_trees = variables.VariableV1( - initial_value=array_ops.zeros([], dtypes.int64), - trainable=False, - name="attempted_trees") - self._finalized_trees = variables.VariableV1( - initial_value=array_ops.zeros([], dtypes.int64), - trainable=False, - name="finalized_trees") - if not features: - raise ValueError("Features dictionary must be specified.") - (fc_names, dense_floats, sparse_float_indices, sparse_float_values, - sparse_float_shapes, sparse_int_indices, - sparse_int_values, sparse_int_shapes) = extract_features( - features, self._feature_columns, use_core_columns) - if (learner_config.weak_learner_type == learner_pb2.LearnerConfig - .OBLIVIOUS_DECISION_TREE and sparse_float_indices): - raise ValueError("Oblivious trees don't handle sparse float features yet." - ) - - logging.info("Active Feature Columns: " + str(fc_names)) - logging.info("Learner config: " + str(learner_config)) - self._fc_names = fc_names - self._dense_floats = dense_floats - self._sparse_float_indices = sparse_float_indices - self._sparse_float_values = sparse_float_values - self._sparse_float_shapes = sparse_float_shapes - self._sparse_int_indices = sparse_int_indices - self._sparse_int_values = sparse_int_values - self._sparse_int_shapes = sparse_int_shapes - self._reduce_dim = ( - self._learner_config.multi_class_strategy == - learner_pb2.LearnerConfig.TREE_PER_CLASS and - learner_config.num_classes == 2) - - if output_leaf_index_modes is None: - output_leaf_index_modes = [learn.ModeKeys.INFER] - elif not all( - mode in (learn.ModeKeys.TRAIN, learn.ModeKeys.EVAL, - learn.ModeKeys.INFER) for mode in output_leaf_index_modes): - raise ValueError("output_leaf_index_modes should only contain ModeKeys.") - - self._output_leaf_index = output_leaf_index - self._output_leaf_index_modes = output_leaf_index_modes - - def _predict_and_return_dict(self, ensemble_handle, ensemble_stamp, mode): - """Runs prediction and returns a dictionary of the prediction results. - - Args: - ensemble_handle: ensemble resource handle. - ensemble_stamp: stamp of ensemble resource. - mode: learn.ModeKeys.TRAIN or EVAL or INFER. - - Returns: - a dictionary of prediction results - - ENSEMBLE_STAMP, PREDICTION, PARTITION_IDS, - NUM_LAYER_ATTEMPTED, NUM_TREES_ATTEMPTED. - """ - ensemble_stats = training_ops.tree_ensemble_stats(ensemble_handle, - ensemble_stamp) - num_handlers = ( - len(self._dense_floats) + len(self._sparse_float_shapes) + len( - self._sparse_int_shapes)) - # Used during feature selection. - used_handlers = model_ops.tree_ensemble_used_handlers( - ensemble_handle, ensemble_stamp, num_all_handlers=num_handlers) - - # We don't need dropout info - we can always restore it based on the - # seed. - apply_dropout, seed = _dropout_params(mode, ensemble_stats) - # Make sure ensemble stats run. This will check that the ensemble has - # the right stamp. - with ops.control_dependencies(ensemble_stats): - leaf_index = None - if self._output_leaf_index and mode in self._output_leaf_index_modes: - predictions, _, leaf_index = ( - prediction_ops).gradient_trees_prediction_verbose( - ensemble_handle, - seed, - self._dense_floats, - self._sparse_float_indices, - self._sparse_float_values, - self._sparse_float_shapes, - self._sparse_int_indices, - self._sparse_int_values, - self._sparse_int_shapes, - learner_config=self._learner_config_serialized, - apply_dropout=apply_dropout, - apply_averaging=mode != learn.ModeKeys.TRAIN, - use_locking=True, - center_bias=self._center_bias, - reduce_dim=self._reduce_dim) - else: - leaf_index = None - predictions, _ = prediction_ops.gradient_trees_prediction( - ensemble_handle, - seed, - self._dense_floats, - self._sparse_float_indices, - self._sparse_float_values, - self._sparse_float_shapes, - self._sparse_int_indices, - self._sparse_int_values, - self._sparse_int_shapes, - learner_config=self._learner_config_serialized, - apply_dropout=apply_dropout, - apply_averaging=mode != learn.ModeKeys.TRAIN, - use_locking=True, - center_bias=self._center_bias, - reduce_dim=self._reduce_dim) - partition_ids = prediction_ops.gradient_trees_partition_examples( - ensemble_handle, - self._dense_floats, - self._sparse_float_indices, - self._sparse_float_values, - self._sparse_float_shapes, - self._sparse_int_indices, - self._sparse_int_values, - self._sparse_int_shapes, - use_locking=True) - - return _make_predictions_dict(ensemble_stamp, predictions, partition_ids, - ensemble_stats, used_handlers, leaf_index) - - def predict(self, mode): - """Returns predictions given the features and mode. - - Args: - mode: Mode the graph is running in (train|predict|eval). - - Returns: - A dict of predictions tensors. - - Raises: - ValueError: if features is not valid. - """ - - # Use the current ensemble to predict on the current batch of input. - # For faster prediction we check if the inputs are on the same device - # as the model. If not, we create a copy of the model on the worker. - input_deps = ( - self._dense_floats + self._sparse_float_indices + - self._sparse_int_indices) - if not input_deps: - raise ValueError("No input tensors for prediction.") - - # Get most current model stamp. - ensemble_stamp = model_ops.tree_ensemble_stamp_token(self._ensemble_handle) - - # Determine if ensemble is colocated with the inputs. - if self._ensemble_handle.device != input_deps[0].device: - # Create a local ensemble and get its local stamp. - with ops.name_scope("local_ensemble", "TreeEnsembleVariable"): - local_ensemble_handle = ( - gen_model_ops.decision_tree_ensemble_resource_handle_op( - self._ensemble_handle.op.name + "/local_ensemble")) - create_op = gen_model_ops.create_tree_ensemble_variable( - local_ensemble_handle, stamp_token=-1, tree_ensemble_config="") - with ops.control_dependencies([create_op]): - local_stamp = model_ops.tree_ensemble_stamp_token( - local_ensemble_handle) - - # Determine whether the local ensemble is stale and update it if needed. - def _refresh_local_ensemble_fn(): - # Serialize the model from parameter server after reading the inputs. - with ops.control_dependencies([input_deps[0]]): - (ensemble_stamp, serialized_model) = ( - model_ops.tree_ensemble_serialize(self._ensemble_handle)) - - # Update local ensemble with the serialized model from parameter server. - with ops.control_dependencies([create_op]): - return model_ops.tree_ensemble_deserialize( - local_ensemble_handle, - stamp_token=ensemble_stamp, - tree_ensemble_config=serialized_model), ensemble_stamp - - with ops.device(local_ensemble_handle.device): - # Need to colocate stamps for cond. - colocated_ensemble_stamp = array_ops.identity(ensemble_stamp) - - refresh_local_ensemble, ensemble_stamp = control_flow_ops.cond( - math_ops.not_equal(colocated_ensemble_stamp, - local_stamp), _refresh_local_ensemble_fn, - lambda: (control_flow_ops.no_op(), colocated_ensemble_stamp)) - - # Once updated, use the local model for prediction. - with ops.control_dependencies([refresh_local_ensemble]): - return self._predict_and_return_dict(local_ensemble_handle, - ensemble_stamp, mode) - else: - # Use ensemble_handle directly, if colocated. - with ops.device(self._ensemble_handle.device): - return self._predict_and_return_dict(self._ensemble_handle, - ensemble_stamp, mode) - - def _get_class_id(self, predictions_dict): - # Handle different multiclass strategies. - if (self._learner_config.multi_class_strategy == - learner_pb2.LearnerConfig.TREE_PER_CLASS and - self._logits_dimension != 1): - # Choose the class for which the tree is built (one vs rest). - return math_ops.cast( - predictions_dict[NUM_TREES_ATTEMPTED] % self._logits_dimension, - dtypes.int32) - return constant_op.constant(-1, dtype=dtypes.int32) - - def update_stats(self, loss, predictions_dict, gradients=None, hessians=None): - """Update the accumulators with stats from this batch. - - Args: - loss: A scalar tensor representing average loss of examples. - predictions_dict: Dictionary of Rank 2 `Tensor` representing information - about predictions per example. - gradients: A tensor with the gradients with the respect to logits from - predictions_dict. If not provided, tensorflow will do - autodifferentiation. - hessians: A tensor with the hessians with the respect to logits from - predictions_dict. If not provided, tensorflow will do - autodifferentiation. - - Returns: - Three values: - - An op that adds a new tree to the ensemble, and - - An op that increments the stamp but removes all the trees and resets - the handlers. This can be used to reset the state of the ensemble. - - A dict containing the training state. - - Raises: - ValueError: if inputs are not valid. - """ - # Get the worker device from input dependencies. - input_deps = ( - self._dense_floats + self._sparse_float_indices + - self._sparse_int_indices) - worker_device = input_deps[0].device - - # Get tensors relevant for training and form the loss. - predictions = predictions_dict[PREDICTIONS] - partition_ids = predictions_dict[PARTITION_IDS] - ensemble_stamp = predictions_dict[ENSEMBLE_STAMP] - if gradients is None: - gradients = gradients_impl.gradients( - loss, - predictions, - name="Gradients", - colocate_gradients_with_ops=False, - gate_gradients=0, - aggregation_method=None)[0] - strategy = self._learner_config.multi_class_strategy - - class_id = self._get_class_id(predictions_dict) - # Handle different multiclass strategies. - if strategy == learner_pb2.LearnerConfig.TREE_PER_CLASS: - # We build one vs rest trees. - if self._logits_dimension == 1: - # We have only 1 score, gradients is of shape [batch, 1]. - if hessians is None: - hessians = gradients_impl.gradients( - gradients, - predictions, - name="Hessian", - colocate_gradients_with_ops=False, - gate_gradients=0, - aggregation_method=None)[0] - - squeezed_gradients = array_ops.squeeze(gradients, axis=[1]) - squeezed_hessians = array_ops.squeeze(hessians, axis=[1]) - else: - if hessians is not None: - raise ValueError("Providing hessians is not yet supported here.") - hessian_list = self._diagonal_hessian(gradients, predictions) - # Assemble hessian list into a tensor. - hessians = array_ops.stack(hessian_list, axis=1) - # Use class id tensor to get the column with that index from gradients - # and hessians. - squeezed_gradients = array_ops.squeeze( - _get_column_by_index(gradients, class_id)) - squeezed_hessians = array_ops.squeeze( - _get_column_by_index(hessians, class_id)) - else: - if hessians is not None: - raise ValueError("Providing hessians is not yet supported here.") - # Other multiclass strategies. - if strategy == learner_pb2.LearnerConfig.FULL_HESSIAN: - hessian_list = self._full_hessian(gradients, predictions) - else: - # Diagonal hessian strategy. - hessian_list = self._diagonal_hessian(gradients, predictions) - - squeezed_gradients = gradients - hessians = array_ops.stack(hessian_list, axis=1) - squeezed_hessians = hessians - - # Get the weights for each example for quantiles calculation, - weights = self._get_weights(self._hessian_shape, squeezed_hessians) - - # Create all handlers ensuring resources are evenly allocated across PS. - fc_name_idx = 0 - handlers = [] - init_stamp_token = constant_op.constant(0, dtype=dtypes.int64) - l1_regularization = constant_op.constant( - self._learner_config.regularization.l1, dtypes.float32) - l2_regularization = constant_op.constant( - self._learner_config.regularization.l2, dtypes.float32) - tree_complexity_regularization = constant_op.constant( - self._learner_config.regularization.tree_complexity, dtypes.float32) - min_node_weight = constant_op.constant( - self._learner_config.constraints.min_node_weight, dtypes.float32) - loss_uses_sum_reduction = self._loss_reduction == losses.Reduction.SUM - loss_uses_sum_reduction = constant_op.constant(loss_uses_sum_reduction) - weak_learner_type = constant_op.constant( - self._learner_config.weak_learner_type) - num_quantiles = self._num_quantiles - epsilon = 1.0 / num_quantiles - strategy_tensor = constant_op.constant(strategy) - with ops.device(self._get_replica_device_setter(worker_device)): - # Create handlers for dense float columns - for dense_float_column_idx in range(len(self._dense_floats)): - fc_name = self._fc_names[fc_name_idx] - handlers.append( - ordinal_split_handler.DenseSplitHandler( - l1_regularization=l1_regularization, - l2_regularization=l2_regularization, - tree_complexity_regularization=tree_complexity_regularization, - min_node_weight=min_node_weight, - feature_column_group_id=constant_op.constant( - dense_float_column_idx), - epsilon=epsilon, - num_quantiles=num_quantiles, - dense_float_column=self._dense_floats[dense_float_column_idx], - name=fc_name, - gradient_shape=self._gradient_shape, - hessian_shape=self._hessian_shape, - multiclass_strategy=strategy_tensor, - init_stamp_token=init_stamp_token, - loss_uses_sum_reduction=loss_uses_sum_reduction, - weak_learner_type=weak_learner_type, - )) - fc_name_idx += 1 - - # Create handlers for sparse float columns. - for sparse_float_column_idx in range(len(self._sparse_float_indices)): - fc_name = self._fc_names[fc_name_idx] - handlers.append( - ordinal_split_handler.SparseSplitHandler( - l1_regularization=l1_regularization, - l2_regularization=l2_regularization, - tree_complexity_regularization=tree_complexity_regularization, - min_node_weight=min_node_weight, - feature_column_group_id=constant_op.constant( - sparse_float_column_idx), - epsilon=epsilon, - num_quantiles=num_quantiles, - sparse_float_column=sparse_tensor.SparseTensor( - self._sparse_float_indices[sparse_float_column_idx], - self._sparse_float_values[sparse_float_column_idx], - self._sparse_float_shapes[sparse_float_column_idx]), - name=fc_name, - gradient_shape=self._gradient_shape, - hessian_shape=self._hessian_shape, - multiclass_strategy=strategy_tensor, - init_stamp_token=init_stamp_token, - loss_uses_sum_reduction=loss_uses_sum_reduction)) - fc_name_idx += 1 - - # Create handlers for sparse int columns. - for sparse_int_column_idx in range(len(self._sparse_int_indices)): - fc_name = self._fc_names[fc_name_idx] - handlers.append( - categorical_split_handler.EqualitySplitHandler( - l1_regularization=l1_regularization, - l2_regularization=l2_regularization, - tree_complexity_regularization=tree_complexity_regularization, - min_node_weight=min_node_weight, - feature_column_group_id=constant_op.constant( - sparse_int_column_idx), - sparse_int_column=sparse_tensor.SparseTensor( - self._sparse_int_indices[sparse_int_column_idx], - self._sparse_int_values[sparse_int_column_idx], - self._sparse_int_shapes[sparse_int_column_idx]), - name=fc_name, - gradient_shape=self._gradient_shape, - hessian_shape=self._hessian_shape, - multiclass_strategy=strategy_tensor, - init_stamp_token=init_stamp_token, - loss_uses_sum_reduction=loss_uses_sum_reduction, - weak_learner_type=weak_learner_type)) - fc_name_idx += 1 - - # Create ensemble stats variables. - num_layer_examples = variables.VariableV1( - initial_value=array_ops.zeros([], dtypes.int64), - name="num_layer_examples", - trainable=False) - num_layer_steps = variables.VariableV1( - initial_value=array_ops.zeros([], dtypes.int64), - name="num_layer_steps", - trainable=False) - num_layers = variables.VariableV1( - initial_value=array_ops.zeros([], dtypes.int64), - name="num_layers", - trainable=False) - active_tree = variables.VariableV1( - initial_value=array_ops.zeros([], dtypes.int64), - name="active_tree", - trainable=False) - active_layer = variables.VariableV1( - initial_value=array_ops.zeros([], dtypes.int64), - name="active_layer", - trainable=False) - # Variable that becomes false once bias centering is done. - continue_centering = variables.VariableV1( - initial_value=self._center_bias, - name="continue_centering", - trainable=False) - # Create bias stats accumulator. - bias_stats_accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=self._gradient_shape, - hessian_shape=self._hessian_shape, - name="BiasAccumulator") - # Create steps accumulator. - steps_accumulator = stats_accumulator_ops.StatsAccumulator( - stamp_token=0, - gradient_shape=tensor_shape.TensorShape([]), - hessian_shape=tensor_shape.TensorShape([]), - name="StepsAccumulator") - # Create ensemble stats summaries. - summary.scalar("layer_stats/num_examples", num_layer_examples) - summary.scalar("layer_stats/num_steps", num_layer_steps) - summary.scalar("ensemble_stats/active_tree", active_tree) - summary.scalar("ensemble_stats/active_layer", active_layer) - - # Update bias stats. - stats_update_ops = [] - - stats_update_ops.append( - control_flow_ops.cond( - continue_centering, - self._make_update_bias_stats_fn(ensemble_stamp, predictions, - gradients, bias_stats_accumulator, - hessians), control_flow_ops.no_op)) - - # Update handler stats. - handler_reads = collections.OrderedDict() - for handler in handlers: - handler_reads[handler] = handler.scheduled_reads() - - handler_results = batch_ops_utils.run_handler_scheduled_ops( - handler_reads, ensemble_stamp, worker_device) - per_handler_updates = collections.OrderedDict() - # Two values per handler. First one is if the handler is active for the - # current layer. The second one is if the handler is going to be active - # for the next layer. - subsampling_type = self._learner_config.WhichOneof("feature_fraction") - if subsampling_type == "feature_fraction_per_level": - seed = predictions_dict[NUM_LAYERS_ATTEMPTED] - active_handlers_current_layer = stateless.stateless_random_uniform( - shape=[len(handlers)], seed=[seed, 1]) - active_handlers_next_layer = stateless.stateless_random_uniform( - shape=[len(handlers)], seed=[seed + 1, 1]) - active_handlers = array_ops.stack( - [active_handlers_current_layer, active_handlers_next_layer], axis=1) - active_handlers = ( - active_handlers < self._learner_config.feature_fraction_per_level) - elif subsampling_type == "feature_fraction_per_tree": - seed = predictions_dict[NUM_TREES_ATTEMPTED] - active_handlers_current_layer = stateless.stateless_random_uniform( - shape=[len(handlers)], seed=[seed, 2]) - active_handlers_current_layer = ( - active_handlers_current_layer < - self._learner_config.feature_fraction_per_tree) - active_handlers = array_ops.stack( - [ - active_handlers_current_layer, - array_ops.ones([len(handlers)], dtype=dtypes.bool) - ], - axis=1) - else: - active_handlers = array_ops.ones([len(handlers), 2], dtype=dtypes.bool) - - if self._learner_config.constraints.max_number_of_unique_feature_columns: - target = ( - self._learner_config.constraints.max_number_of_unique_feature_columns) - - def _feature_selection_active_handlers(): - # The active list for current and the next iteration. - used_handlers = array_ops.reshape(predictions_dict[USED_HANDLERS_MASK], - [-1, 1]) - used_handlers = array_ops.concat([used_handlers, used_handlers], axis=1) - return math_ops.logical_and(used_handlers, active_handlers) - - active_handlers = ( - control_flow_ops.cond(predictions_dict[NUM_USED_HANDLERS] >= target, - _feature_selection_active_handlers, - lambda: active_handlers)) - - # Prepare empty gradients and hessians when handlers are not ready. - empty_hess_shape = [1] + self._hessian_shape.as_list() - empty_grad_shape = [1] + self._gradient_shape.as_list() - - empty_gradients = constant_op.constant_v1( - [], dtype=dtypes.float32, shape=empty_grad_shape) - empty_hessians = constant_op.constant_v1( - [], dtype=dtypes.float32, shape=empty_hess_shape) - - active_handlers = array_ops.unstack(active_handlers, axis=0) - for handler_idx in range(len(handlers)): - handler = handlers[handler_idx] - is_active = active_handlers[handler_idx] - updates, scheduled_updates = handler.update_stats( - ensemble_stamp, partition_ids, squeezed_gradients, squeezed_hessians, - empty_gradients, empty_hessians, weights, is_active, - handler_results[handler]) - stats_update_ops.append(updates) - per_handler_updates[handler] = scheduled_updates - - update_results = batch_ops_utils.run_handler_scheduled_ops( - per_handler_updates, ensemble_stamp, worker_device) - for update in update_results.values(): - stats_update_ops += update - - training_state = GBDTTrainingState( - num_layer_examples=num_layer_examples, - num_layer_steps=num_layer_steps, - num_layers=num_layers, - active_tree=active_tree, - active_layer=active_layer, - continue_centering=continue_centering, - bias_stats_accumulator=bias_stats_accumulator, - steps_accumulator=steps_accumulator, - handlers=handlers) - - reset_op = control_flow_ops.no_op() - if self._is_chief: - # Advance the ensemble stamp to throw away staggered workers. - stamp_token, _ = model_ops.tree_ensemble_serialize(self._ensemble_handle) - next_stamp_token = stamp_token + 1 - - reset_ops = [] - for handler in handlers: - reset_ops.append(handler.reset(stamp_token, next_stamp_token)) - if self._center_bias: - reset_ops.append( - bias_stats_accumulator.flush(stamp_token, next_stamp_token)) - reset_ops.append(steps_accumulator.flush(stamp_token, next_stamp_token)) - reset_ops.append(self._finalized_trees.assign(0).op) - reset_ops.append(self._attempted_trees.assign(0).op) - reset_ops.append( - model_ops.tree_ensemble_deserialize( - self._ensemble_handle, - stamp_token=next_stamp_token, - tree_ensemble_config="", - name="reset_gbdt")) - - reset_op = control_flow_ops.group([reset_ops]) - - return stats_update_ops, reset_op, training_state - - def increment_step_counter_and_maybe_update_ensemble(self, predictions_dict, - training_state): - """Increments number of visited examples and grows the ensemble. - - If the number of visited examples reaches the target examples_per_layer, - ensemble is updated. - - Args: - predictions_dict: Dictionary of Rank 2 `Tensor` representing information - about predictions per example. - training_state: `dict` returned by update_stats. - - Returns: - An op that updates the counters and potientially grows the ensemble. - """ - batch_size = math_ops.cast( - array_ops.shape(predictions_dict[PREDICTIONS])[0], dtypes.float32) - ensemble_stamp = predictions_dict[ENSEMBLE_STAMP] - # Accumulate a step after updating stats. - - steps_accumulator = training_state.steps_accumulator - num_layer_examples = training_state.num_layer_examples - num_layer_steps = training_state.num_layer_steps - active_layer = training_state.active_layer - add_step_op = steps_accumulator.add( - ensemble_stamp, [0], [[0, 0]], [batch_size], [1.0]) - - # After adding the step, decide if further processing is needed. - ensemble_update_ops = [add_step_op] - class_id = self._get_class_id(predictions_dict) - - with ops.control_dependencies([add_step_op]): - if self._is_chief: - dropout_seed = predictions_dict[NUM_TREES_ATTEMPTED] - - # Get accumulated steps and examples for the current layer. - _, _, _, _, acc_examples, acc_steps = ( - steps_accumulator.saveable.serialize()) - acc_examples = math_ops.cast(acc_examples[0], dtypes.int64) - acc_steps = math_ops.cast(acc_steps[0], dtypes.int64) - ensemble_update_ops.append( - num_layer_examples.assign(acc_examples)) - ensemble_update_ops.append(num_layer_steps.assign(acc_steps)) - # Determine whether we need to update tree ensemble. - examples_per_layer = self._examples_per_layer - if callable(examples_per_layer): - examples_per_layer = examples_per_layer(active_layer) - ensemble_update_ops.append( - control_flow_ops.cond( - acc_examples >= examples_per_layer, - self.make_update_ensemble_fn(ensemble_stamp, training_state, - dropout_seed, class_id), - control_flow_ops.no_op)) - - # Note, the loss is calculated from the prediction considering dropouts, so - # that the value might look staggering over steps when the dropout ratio is - # high. eval_loss might be referred instead in the aspect of convergence. - return control_flow_ops.group(*ensemble_update_ops) - - def make_update_ensemble_fn(self, ensemble_stamp, training_state, - dropout_seed, class_id): - """A method to create the function which updates the tree ensemble.""" - # Determine learning rate. - learning_rate_tuner = self._learner_config.learning_rate_tuner.WhichOneof( - "tuner") - if learning_rate_tuner == "fixed" or learning_rate_tuner == "dropout": - tuner = getattr(self._learner_config.learning_rate_tuner, - learning_rate_tuner) - learning_rate = tuner.learning_rate - else: - # TODO(nponomareva, soroush) do the line search. - raise ValueError("Line search learning rate is not yet supported.") - - def _update_ensemble(): - """A method to update the tree ensemble.""" - # Get next stamp token. - next_ensemble_stamp = ensemble_stamp + 1 - # Finalize bias stats. - _, _, _, bias_grads, bias_hess = ( - training_state.bias_stats_accumulator.flush(ensemble_stamp, - next_ensemble_stamp)) - - # Finalize handler splits. - are_splits_ready_list = [] - partition_ids_list = [] - gains_list = [] - split_info_list = [] - - for handler in training_state.handlers: - (are_splits_ready, - partition_ids, gains, split_info) = handler.make_splits( - ensemble_stamp, next_ensemble_stamp, class_id) - are_splits_ready_list.append(are_splits_ready) - partition_ids_list.append(partition_ids) - gains_list.append(gains) - split_info_list.append(split_info) - # Stack all the inputs to one tensor per type. - # This is a workaround for the slowness of graph building in tf.cond. - # See (b/36554864). - split_sizes = array_ops.reshape( - array_ops.shape_n(partition_ids_list), [len(partition_ids_list)]) - partition_ids = array_ops.concat(partition_ids_list, axis=0) - gains = array_ops.concat(gains_list, axis=0) - split_infos = array_ops.concat(split_info_list, axis=0) - - # Determine if all splits are ready. - are_all_splits_ready = math_ops.reduce_all( - array_ops.stack( - are_splits_ready_list, axis=0, name="stack_handler_readiness")) - - # Define bias centering update operation. - def _center_bias_fn(): - # Center tree ensemble bias. - delta_updates = array_ops.where(bias_hess > 0, -bias_grads / bias_hess, - array_ops.zeros_like(bias_grads)) - center_bias = training_ops.center_tree_ensemble_bias( - tree_ensemble_handle=self._ensemble_handle, - stamp_token=ensemble_stamp, - next_stamp_token=next_ensemble_stamp, - delta_updates=delta_updates, - learner_config=self._learner_config_serialized) - return training_state.continue_centering.assign(center_bias) - - # Define ensemble growing operations. - def _grow_ensemble_ready_fn(): - # Grow the ensemble given the current candidates. - sizes = array_ops.unstack(split_sizes) - partition_ids_list = list(array_ops.split(partition_ids, sizes, axis=0)) - # When using the oblivious decision tree as weak learner, it produces - # one gain and one split per handler and not number of partitions. - if self._learner_config.weak_learner_type == ( - learner_pb2.LearnerConfig.OBLIVIOUS_DECISION_TREE): - sizes = len(training_state.handlers) - - gains_list = list(array_ops.split(gains, sizes, axis=0)) - split_info_list = list(array_ops.split(split_infos, sizes, axis=0)) - return training_ops.grow_tree_ensemble( - tree_ensemble_handle=self._ensemble_handle, - stamp_token=ensemble_stamp, - next_stamp_token=next_ensemble_stamp, - learning_rate=learning_rate, - partition_ids=partition_ids_list, - gains=gains_list, - splits=split_info_list, - learner_config=self._learner_config_serialized, - dropout_seed=dropout_seed, - center_bias=self._center_bias, - max_tree_depth=self._max_tree_depth, - weak_learner_type=self._learner_config.weak_learner_type) - - def _grow_ensemble_not_ready_fn(): - # Don't grow the ensemble, just update the stamp. - return training_ops.grow_tree_ensemble( - tree_ensemble_handle=self._ensemble_handle, - stamp_token=ensemble_stamp, - next_stamp_token=next_ensemble_stamp, - learning_rate=0, - partition_ids=[], - gains=[], - splits=[], - learner_config=self._learner_config_serialized, - dropout_seed=dropout_seed, - center_bias=self._center_bias, - max_tree_depth=self._max_tree_depth, - weak_learner_type=self._learner_config.weak_learner_type) - - def _grow_ensemble_fn(): - # Conditionally grow an ensemble depending on whether the splits - # from all the handlers are ready. - return control_flow_ops.cond(are_all_splits_ready, - _grow_ensemble_ready_fn, - _grow_ensemble_not_ready_fn) - - # Update ensemble. - update_ops = [are_all_splits_ready] - if self._center_bias: - update_model = control_flow_ops.cond(training_state.continue_centering, - _center_bias_fn, _grow_ensemble_fn) - else: - update_model = _grow_ensemble_fn() - update_ops.append(update_model) - - # Update ensemble stats. - with ops.control_dependencies([update_model]): - stats = training_ops.tree_ensemble_stats( - self._ensemble_handle, stamp_token=next_ensemble_stamp) - update_ops.append(self._finalized_trees.assign(stats.num_trees)) - update_ops.append(self._attempted_trees.assign(stats.attempted_trees)) - update_ops.append(training_state.num_layers.assign(stats.num_layers)) - update_ops.append(training_state.active_tree.assign(stats.active_tree)) - update_ops.append( - training_state.active_layer.assign(stats.active_layer)) - - # Flush step stats. - update_ops.extend( - training_state.steps_accumulator.flush(ensemble_stamp, - next_ensemble_stamp)) - return control_flow_ops.group(*update_ops, name="update_ensemble") - - return _update_ensemble - - def get_number_of_trees_tensor(self): - return self._finalized_trees, self._attempted_trees - - def get_max_tree_depth(self): - return self._max_tree_depth - - def train(self, loss, predictions_dict, labels, gradients=None, - hessians=None): - """Updates the accumalator stats and grows the ensemble. - - Args: - loss: A scalar tensor representing average loss of examples. - predictions_dict: Dictionary of Rank 2 `Tensor` representing information - about predictions per example. - labels: Rank 2 `Tensor` representing labels per example. Has no effect - on the training and is only kept for backward compatibility. - gradients: A tensor with the gradients with the respect to logits from - predictions_dict. If not provided, tensorflow will do - autodifferentiation. - hessians: A tensor with the hessians with the respect to logits from - predictions_dict. If not provided, tensorflow will do - autodifferentiation. - - Returns: - An op that adds a new tree to the ensemble. - - Raises: - ValueError: if inputs are not valid. - """ - del labels # unused; kept for backward compatibility. - update_op, _, training_state = self.update_stats(loss, predictions_dict, - gradients, hessians) - with ops.control_dependencies(update_op): - return self.increment_step_counter_and_maybe_update_ensemble( - predictions_dict, training_state) - - def _get_weights(self, hessian_shape, hessians): - """Derives weights to be used based on hessians and multiclass strategy.""" - if hessian_shape.rank == 0: - # This is tree per class. - weights = hessians - elif len(hessian_shape.dims) == 1: - # This is diagonal hessian. - weights = math_ops.reduce_sum(hessians, axis=1) - else: - # This is full hessian. - weights = math_ops.trace(hessians) - return weights - - def _full_hessian(self, grads, predictions): - """Prepares hessians for full-hessian multiclass strategy.""" - # Because of - # https://github.com/tensorflow/tensorflow/issues/675, we can't just - # compute the full hessian with a single call to gradients, but instead - # must compute it row-by-row. - gradients_list = array_ops.unstack( - grads, num=self._logits_dimension, axis=1) - hessian_rows = [] - - for row in range(self._logits_dimension): - # If current row is i, K is number of classes,each row returns a tensor of - # size batch_size x K representing for each example dx_i dx_1, dx_i dx_2 - # etc dx_i dx_K - hessian_row = gradients_impl.gradients( - gradients_list[row], - predictions, - name="Hessian_%d" % row, - colocate_gradients_with_ops=False, - gate_gradients=0, - aggregation_method=None) - - # hessian_row is of dimension 1, batch_size, K, => trim first dimension - # to get batch_size x K - hessian_row = array_ops.squeeze(array_ops.unstack(hessian_row), [0]) - hessian_rows.append(hessian_row) - return hessian_rows - - def _diagonal_hessian(self, grads, predictions): - """Prepares hessians for diagonal-hessian multiclass mode.""" - diag_hessian_list = [] - - gradients_list = array_ops.unstack( - grads, num=self._logits_dimension, axis=1) - - for row, row_grads in enumerate(gradients_list): - # If current row is i, K is number of classes,each row returns a tensor of - # size batch_size x K representing for each example dx_i dx_1, dx_1 dx_2 - # etc dx_i dx_K - hessian_row = gradients_impl.gradients( - row_grads, - predictions, - name="Hessian_%d" % row, - colocate_gradients_with_ops=False, - gate_gradients=0, - aggregation_method=None) - - # hessian_row is of dimension 1, batch_size, K, => trim first dimension - # to get batch_size x K - hessian_row = array_ops.squeeze(array_ops.unstack(hessian_row), [0]) - - # Get dx_i^2 for the whole batch. - elem = array_ops.transpose(hessian_row)[row] - diag_hessian_list.append(elem) - - return diag_hessian_list - - def _get_replica_device_setter(self, worker_device): - """Creates a replica device setter.""" - ps_tasks = self._num_ps_replicas - ps_ops = list(device_setter.STANDARD_PS_OPS) - ps_ops.extend([ - "DecisionTreeEnsembleResourceHandleOp", - "StatsAccumulatorScalarResourceHandleOp", - "StatsAccumulatorTensorResourceHandleOp", - ]) - ps_strategy = _OpRoundRobinStrategy(ps_ops, ps_tasks) - return device_setter.replica_device_setter( - worker_device=worker_device, - ps_tasks=ps_tasks, - merge_devices=True, - ps_ops=ps_ops, - ps_strategy=ps_strategy) - - def _make_update_bias_stats_fn(self, - ensemble_stamp, - predictions, - gradients, - bias_stats_accumulator, - hessians=None): - """A method to create the function which updates the bias stats.""" - - def _update_bias_stats(): - """A method to update the bias stats.""" - # Get reduced gradients and hessians. - grads_sum = math_ops.reduce_sum(gradients, 0) - if hessians is not None: - hess = hessians - else: - hess = gradients_impl.gradients( - grads_sum, - predictions, - name="Hessians", - colocate_gradients_with_ops=False, - gate_gradients=0, - aggregation_method=None)[0] - hess_sum = math_ops.reduce_sum(hess, 0) - - # Accumulate gradients and hessians. - partition_ids = math_ops.range(self._logits_dimension) - feature_ids = array_ops.zeros( - [self._logits_dimension, 2], dtype=dtypes.int64) - - add_stats_op = bias_stats_accumulator.add( - ensemble_stamp, partition_ids, feature_ids, grads_sum, hess_sum) - return control_flow_ops.group(*[add_stats_op], name="update_bias_stats") - - return _update_bias_stats diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py deleted file mode 100644 index c9f37508677..00000000000 --- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py +++ /dev/null @@ -1,2141 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for GBDT train function.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from google.protobuf import text_format -from tensorflow.contrib import layers -from tensorflow.contrib import learn -from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.boosted_trees.proto import tree_config_pb2 -from tensorflow.contrib.boosted_trees.python.ops import model_ops -from tensorflow.contrib.boosted_trees.python.training.functions import gbdt_batch -from tensorflow.contrib.boosted_trees.python.utils import losses -from tensorflow.contrib.layers.python.layers import feature_column as feature_column_lib -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.python.feature_column import feature_column_lib as core_feature_column -from tensorflow.python.feature_column import feature_column_v2 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resources -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest - - -def _squared_loss(label, unused_weights, predictions): - """Unweighted loss implementation.""" - loss = math_ops.reduce_sum( - math_ops.squared_difference(predictions, label), 1, keepdims=True) - return loss - - -def _append_to_leaf(leaf, c_id, w): - """Helper method for building tree leaves. - - Appends weight contributions for the given class index to a leaf node. - - Args: - leaf: leaf node to append to. - c_id: class Id for the weight update. - w: weight contribution value. - """ - leaf.sparse_vector.index.append(c_id) - leaf.sparse_vector.value.append(w) - - -def _set_float_split(split, feat_col, thresh, l_id, r_id): - """Helper method for building tree float splits. - - Sets split feature column, threshold and children. - - Args: - split: split node to update. - feat_col: feature column for the split. - thresh: threshold to split on forming rule x <= thresh. - l_id: left child Id. - r_id: right child Id. - """ - split.feature_column = feat_col - split.threshold = thresh - split.left_id = l_id - split.right_id = r_id - - -class GbdtTest(test_util.TensorFlowTestCase): - - def setUp(self): - super(GbdtTest, self).setUp() - - def testExtractFeatures(self): - """Tests feature extraction.""" - with self.cached_session(): - features = {} - features["dense_float"] = array_ops.zeros([2, 1], dtypes.float32) - features["sparse_float"] = sparse_tensor.SparseTensor( - array_ops.zeros([2, 2], dtypes.int64), - array_ops.zeros([2], dtypes.float32), - array_ops.zeros([2], dtypes.int64)) - features["sparse_int"] = sparse_tensor.SparseTensor( - array_ops.zeros([2, 2], dtypes.int64), - array_ops.zeros([2], dtypes.int64), array_ops.zeros([2], - dtypes.int64)) - (fc_names, dense_floats, sparse_float_indices, sparse_float_values, - sparse_float_shapes, sparse_int_indices, sparse_int_values, - sparse_int_shapes) = ( - gbdt_batch.extract_features(features, None, use_core_columns=False)) - self.assertEqual(len(fc_names), 3) - self.assertAllEqual(fc_names, - ["dense_float", "sparse_float", "sparse_int"]) - self.assertEqual(len(dense_floats), 1) - self.assertEqual(len(sparse_float_indices), 1) - self.assertEqual(len(sparse_float_values), 1) - self.assertEqual(len(sparse_float_shapes), 1) - self.assertEqual(len(sparse_int_indices), 1) - self.assertEqual(len(sparse_int_values), 1) - self.assertEqual(len(sparse_int_shapes), 1) - self.assertAllEqual(dense_floats[0].eval(), - features["dense_float"].eval()) - self.assertAllEqual(sparse_float_indices[0].eval(), - features["sparse_float"].indices.eval()) - self.assertAllEqual(sparse_float_values[0].eval(), - features["sparse_float"].values.eval()) - self.assertAllEqual(sparse_float_shapes[0].eval(), - features["sparse_float"].dense_shape.eval()) - self.assertAllEqual(sparse_int_indices[0].eval(), - features["sparse_int"].indices.eval()) - self.assertAllEqual(sparse_int_values[0].eval(), - features["sparse_int"].values.eval()) - self.assertAllEqual(sparse_int_shapes[0].eval(), - features["sparse_int"].dense_shape.eval()) - - def testExtractFeaturesWithTransformation(self): - """Tests feature extraction.""" - with self.cached_session(): - features = {} - features["dense_float"] = array_ops.zeros([2, 1], dtypes.float32) - features["sparse_float"] = sparse_tensor.SparseTensor( - array_ops.zeros([2, 2], dtypes.int64), - array_ops.zeros([2], dtypes.float32), - array_ops.zeros([2], dtypes.int64)) - features["sparse_categorical"] = sparse_tensor.SparseTensor( - array_ops.zeros([2, 2], dtypes.int64), - array_ops.zeros([2], dtypes.string), array_ops.zeros([2], - dtypes.int64)) - feature_columns = set() - feature_columns.add(layers.real_valued_column("dense_float")) - feature_columns.add( - layers.feature_column._real_valued_var_len_column( - "sparse_float", is_sparse=True)) - feature_columns.add( - feature_column_lib.sparse_column_with_hash_bucket( - "sparse_categorical", hash_bucket_size=1000000)) - (fc_names, dense_floats, sparse_float_indices, sparse_float_values, - sparse_float_shapes, sparse_int_indices, sparse_int_values, - sparse_int_shapes) = ( - gbdt_batch.extract_features( - features, feature_columns, use_core_columns=False)) - self.assertEqual(len(fc_names), 3) - self.assertAllEqual(fc_names, - ["dense_float", "sparse_float", "sparse_categorical"]) - self.assertEqual(len(dense_floats), 1) - self.assertEqual(len(sparse_float_indices), 1) - self.assertEqual(len(sparse_float_values), 1) - self.assertEqual(len(sparse_float_shapes), 1) - self.assertEqual(len(sparse_int_indices), 1) - self.assertEqual(len(sparse_int_values), 1) - self.assertEqual(len(sparse_int_shapes), 1) - self.assertAllEqual(dense_floats[0].eval(), - features["dense_float"].eval()) - self.assertAllEqual(sparse_float_indices[0].eval(), - features["sparse_float"].indices.eval()) - self.assertAllEqual(sparse_float_values[0].eval(), - features["sparse_float"].values.eval()) - self.assertAllEqual(sparse_float_shapes[0].eval(), - features["sparse_float"].dense_shape.eval()) - self.assertAllEqual(sparse_int_indices[0].eval(), - features["sparse_categorical"].indices.eval()) - self.assertAllEqual(sparse_int_values[0].eval(), [397263, 397263]) - self.assertAllEqual(sparse_int_shapes[0].eval(), - features["sparse_categorical"].dense_shape.eval()) - - def testExtractFeaturesFromV2FeatureColumns(self): - """Tests feature extraction when using v2 columns.""" - with self.cached_session(): - features = {} - features["dense_float"] = array_ops.zeros([2, 1], dtypes.float32) - features["sparse_categorical"] = sparse_tensor.SparseTensor( - array_ops.zeros([2, 2], dtypes.int64), - array_ops.zeros([2], dtypes.string), array_ops.zeros([2], - dtypes.int64)) - feature_columns = set() - feature_columns.add(feature_column_v2.numeric_column("dense_float")) - feature_columns.add( - feature_column_v2.categorical_column_with_hash_bucket( - "sparse_categorical", hash_bucket_size=1000000)) - (fc_names, dense_floats, _, _, _, sparse_int_indices, sparse_int_values, - sparse_int_shapes) = ( - gbdt_batch.extract_features( - features, feature_columns, use_core_columns=True)) - self.assertEqual(len(fc_names), 2) - self.assertAllEqual(fc_names, ["dense_float", "sparse_categorical"]) - self.assertEqual(len(dense_floats), 1) - self.assertEqual(len(sparse_int_indices), 1) - self.assertEqual(len(sparse_int_values), 1) - self.assertEqual(len(sparse_int_shapes), 1) - self.assertAllEqual(dense_floats[0].eval(), - features["dense_float"].eval()) - self.assertAllEqual(sparse_int_indices[0].eval(), - features["sparse_categorical"].indices.eval()) - self.assertAllEqual(sparse_int_values[0].eval(), [397263, 397263]) - self.assertAllEqual(sparse_int_shapes[0].eval(), - features["sparse_categorical"].dense_shape.eval()) - - def testExtractFeaturesFromCoreFeatureColumns(self): - """Tests feature extraction when using core columns.""" - with self.cached_session(): - features = {} - # Sparse float column does not exist in core, so only dense numeric and - # categorical. - features["dense_float"] = array_ops.zeros([2, 1], dtypes.float32) - features["sparse_categorical"] = sparse_tensor.SparseTensor( - array_ops.zeros([2, 2], dtypes.int64), - array_ops.zeros([2], dtypes.string), array_ops.zeros([2], - dtypes.int64)) - - feature_columns = set() - feature_columns.add(core_feature_column.numeric_column("dense_float")) - feature_columns.add( - core_feature_column.categorical_column_with_hash_bucket( - "sparse_categorical", hash_bucket_size=1000000)) - (fc_names, dense_floats, _, _, _, sparse_int_indices, sparse_int_values, - sparse_int_shapes) = ( - gbdt_batch.extract_features( - features, feature_columns, use_core_columns=True)) - self.assertEqual(len(fc_names), 2) - self.assertAllEqual(fc_names, ["dense_float", "sparse_categorical"]) - self.assertEqual(len(dense_floats), 1) - self.assertEqual(len(sparse_int_indices), 1) - self.assertEqual(len(sparse_int_values), 1) - self.assertEqual(len(sparse_int_shapes), 1) - self.assertAllEqual(dense_floats[0].eval(), - features["dense_float"].eval()) - self.assertAllEqual(sparse_int_indices[0].eval(), - features["sparse_categorical"].indices.eval()) - self.assertAllEqual(sparse_int_values[0].eval(), [397263, 397263]) - self.assertAllEqual(sparse_int_shapes[0].eval(), - features["sparse_categorical"].dense_shape.eval()) - - def testTrainFnChiefNoBiasCentering(self): - """Tests the train function running on chief without bias centering.""" - with self.cached_session() as sess: - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.regularization.l1 = 0 - learner_config.regularization.l2 = 0 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.min_node_weight = 0 - features = {} - features["dense_float"] = array_ops.ones([4, 1], dtypes.float32) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features) - - predictions = array_ops.constant( - [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) - partition_ids = array_ops.zeros([4], dtypes.int32) - ensemble_stamp = variables.VariableV1( - initial_value=0, - name="ensemble_stamp", - trainable=False, - dtype=dtypes.int64) - - predictions_dict = { - "predictions": predictions, - "predictions_no_dropout": predictions, - "partition_ids": partition_ids, - "ensemble_stamp": ensemble_stamp, - "num_trees": 12, - } - - labels = array_ops.ones([4, 1], dtypes.float32) - weights = array_ops.ones([4, 1], dtypes.float32) - # Create train op. - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - _squared_loss(labels, weights, predictions)), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # On first run, expect no splits to be chosen because the quantile - # buckets will not be ready. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 0) - self.assertEquals(len(output.tree_weights), 0) - self.assertEquals(stamp_token.eval(), 1) - - # Update the stamp to be able to run a second time. - sess.run([ensemble_stamp.assign_add(1)]) - - # On second run, expect a trivial split to be chosen to basically - # predict the average. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 1) - self.assertAllClose(output.tree_weights, [0.1]) - self.assertEquals(stamp_token.eval(), 2) - expected_tree = """ - nodes { - dense_float_binary_split { - threshold: 1.0 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 0 - } - } - nodes { - leaf { - vector { - value: 0.25 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - }""" - self.assertProtoEquals(expected_tree, output.trees[0]) - - def testObliviousDecisionTreeAsWeakLearner(self): - with self.cached_session(): - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - learner_config = learner_pb2.LearnerConfig() - learner_config.num_classes = 2 - learner_config.learning_rate_tuner.fixed.learning_rate = 1 - learner_config.regularization.l1 = 0 - learner_config.regularization.l2 = 0 - learner_config.constraints.max_tree_depth = 2 - learner_config.constraints.min_node_weight = 0 - learner_config.weak_learner_type = ( - learner_pb2.LearnerConfig.OBLIVIOUS_DECISION_TREE) - learner_config.pruning_mode = learner_pb2.LearnerConfig.PRE_PRUNE - learner_config.growing_mode = learner_pb2.LearnerConfig.LAYER_BY_LAYER - features = {} - features["dense_float"] = array_ops.constant([[-2], [-1], [1], [2]], - dtypes.float32) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features) - - predictions_dict = gbdt_model.predict(learn.ModeKeys.TRAIN) - predictions = predictions_dict["predictions"] - labels = array_ops.constant([[-2], [-1], [1], [2]], dtypes.float32) - weights = array_ops.ones([4, 1], dtypes.float32) - - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - _squared_loss(labels, weights, predictions)), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # On first run, expect no splits to be chosen because the quantile - # buckets will not be ready. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 0) - self.assertEquals(len(output.tree_weights), 0) - self.assertEquals(stamp_token.eval(), 1) - - # Second run. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 1) - self.assertAllClose(output.tree_weights, [1]) - self.assertEquals(stamp_token.eval(), 2) - expected_tree = """ - nodes { - oblivious_dense_float_binary_split { - threshold: -1.0 - } - node_metadata { - gain: 4.5 - original_oblivious_leaves { - } - } - } - nodes { - leaf { - vector { - value: -1.5 - } - } - } - nodes { - leaf { - vector { - value: 1.5 - } - } - }""" - self.assertProtoEquals(expected_tree, output.trees[0]) - # Third run. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 1) - self.assertAllClose(output.tree_weights, [1]) - self.assertEquals(stamp_token.eval(), 3) - expected_tree = """ - nodes { - oblivious_dense_float_binary_split { - threshold: -1.0 - } - node_metadata { - gain: 4.5 - original_oblivious_leaves { - } - } - } - nodes { - oblivious_dense_float_binary_split { - threshold: -2.0 - } - node_metadata { - gain: 0.25 - original_oblivious_leaves { - vector { - value: -1.5 - } - } - original_oblivious_leaves { - vector { - value: 1.5 - } - } - } - } - nodes { - leaf { - vector { - value: -2.0 - } - } - } - nodes { - leaf { - vector { - value: -1.0 - } - } - } - nodes { - leaf { - vector { - value: 1.5 - } - } - } - nodes { - leaf { - vector { - value: 1.5 - } - } - }""" - self.assertProtoEquals(expected_tree, output.trees[0]) - - def testTrainFnChiefSparseAndDense(self): - """Tests the train function with sparse and dense features.""" - with self.cached_session() as sess: - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.regularization.l1 = 0 - learner_config.regularization.l2 = 0 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.min_node_weight = 0 - features = {} - features["dense_float"] = array_ops.ones([4, 1], dtypes.float32) - features["sparse_float"] = sparse_tensor.SparseTensor( - array_ops.zeros([2, 2], dtypes.int64), - array_ops.zeros([2], dtypes.float32), - array_ops.constant([4, 1], dtypes.int64)) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features) - - predictions = array_ops.constant( - [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) - partition_ids = array_ops.zeros([4], dtypes.int32) - ensemble_stamp = variables.VariableV1( - initial_value=0, - name="ensemble_stamp", - trainable=False, - dtype=dtypes.int64) - - predictions_dict = { - "predictions": predictions, - "predictions_no_dropout": predictions, - "partition_ids": partition_ids, - "ensemble_stamp": ensemble_stamp, - "num_trees": 12, - } - - labels = array_ops.ones([4, 1], dtypes.float32) - weights = array_ops.ones([4, 1], dtypes.float32) - # Create train op. - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - _squared_loss(labels, weights, predictions)), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # On first run, expect no splits to be chosen because the quantile - # buckets will not be ready. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 0) - self.assertEquals(len(output.tree_weights), 0) - self.assertEquals(stamp_token.eval(), 1) - - # Update the stamp to be able to run a second time. - sess.run([ensemble_stamp.assign_add(1)]) - - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 1) - self.assertAllClose(output.tree_weights, [0.1]) - self.assertEquals(stamp_token.eval(), 2) - expected_tree = """ - nodes { - sparse_float_binary_split_default_right { - split{ - left_id: 1 - right_id: 2 - } - } - node_metadata { - gain: 1.125 - } - } - nodes { - leaf { - vector { - value: 1.0 - } - } - } - nodes { - leaf { - vector { - value: -0.5 - } - } - }""" - self.assertProtoEquals(expected_tree, output.trees[0]) - - def testTrainFnChiefScalingNumberOfExamples(self): - """Tests the train function running on chief without bias centering.""" - with self.cached_session() as sess: - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.regularization.l1 = 0 - learner_config.regularization.l2 = 0 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.min_node_weight = 0 - num_examples_fn = ( - lambda layer: math_ops.pow(math_ops.cast(2, dtypes.int64), layer) * 1) - features = {} - features["dense_float"] = array_ops.ones([4, 1], dtypes.float32) - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=num_examples_fn, - learner_config=learner_config, - logits_dimension=1, - features=features) - - predictions = array_ops.constant( - [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) - partition_ids = array_ops.zeros([4], dtypes.int32) - ensemble_stamp = variables.VariableV1( - initial_value=0, - name="ensemble_stamp", - trainable=False, - dtype=dtypes.int64) - - predictions_dict = { - "predictions": predictions, - "predictions_no_dropout": predictions, - "partition_ids": partition_ids, - "ensemble_stamp": ensemble_stamp, - "num_trees": 12, - } - - labels = array_ops.ones([4, 1], dtypes.float32) - weights = array_ops.ones([4, 1], dtypes.float32) - # Create train op. - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - _squared_loss(labels, weights, predictions)), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # On first run, expect no splits to be chosen because the quantile - # buckets will not be ready. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 0) - self.assertEquals(len(output.tree_weights), 0) - self.assertEquals(stamp_token.eval(), 1) - - # Update the stamp to be able to run a second time. - sess.run([ensemble_stamp.assign_add(1)]) - - # On second run, expect a trivial split to be chosen to basically - # predict the average. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 1) - self.assertAllClose(output.tree_weights, [0.1]) - self.assertEquals(stamp_token.eval(), 2) - expected_tree = """ - nodes { - dense_float_binary_split { - threshold: 1.0 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 0 - } - } - nodes { - leaf { - vector { - value: 0.25 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - }""" - self.assertProtoEquals(expected_tree, output.trees[0]) - - def testTrainFnChiefWithBiasCentering(self): - """Tests the train function running on chief with bias centering.""" - with self.cached_session(): - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.regularization.l1 = 0 - learner_config.regularization.l2 = 0 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.min_node_weight = 0 - features = {} - features["dense_float"] = array_ops.ones([4, 1], dtypes.float32) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=True, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features) - - predictions = array_ops.constant( - [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) - partition_ids = array_ops.zeros([4], dtypes.int32) - ensemble_stamp = variables.VariableV1( - initial_value=0, - name="ensemble_stamp", - trainable=False, - dtype=dtypes.int64) - - predictions_dict = { - "predictions": predictions, - "predictions_no_dropout": predictions, - "partition_ids": partition_ids, - "ensemble_stamp": ensemble_stamp, - "num_trees": 12, - } - - labels = array_ops.ones([4, 1], dtypes.float32) - weights = array_ops.ones([4, 1], dtypes.float32) - # Create train op. - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - _squared_loss(labels, weights, predictions)), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # On first run, expect bias to be centered. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - expected_tree = """ - nodes { - leaf { - vector { - value: 0.25 - } - } - }""" - self.assertEquals(len(output.trees), 1) - self.assertAllEqual(output.tree_weights, [1.0]) - self.assertProtoEquals(expected_tree, output.trees[0]) - self.assertEquals(stamp_token.eval(), 1) - - def testTrainFnNonChiefNoBiasCentering(self): - """Tests the train function running on worker without bias centering.""" - with self.cached_session(): - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.regularization.l1 = 0 - learner_config.regularization.l2 = 0 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.min_node_weight = 0 - features = {} - features["dense_float"] = array_ops.ones([4, 1], dtypes.float32) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=False, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features) - - predictions = array_ops.constant( - [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) - partition_ids = array_ops.zeros([4], dtypes.int32) - ensemble_stamp = variables.VariableV1( - initial_value=0, - name="ensemble_stamp", - trainable=False, - dtype=dtypes.int64) - - predictions_dict = { - "predictions": predictions, - "predictions_no_dropout": predictions, - "partition_ids": partition_ids, - "ensemble_stamp": ensemble_stamp - } - - labels = array_ops.ones([4, 1], dtypes.float32) - weights = array_ops.ones([4, 1], dtypes.float32) - # Create train op. - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - _squared_loss(labels, weights, predictions)), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # Regardless of how many times the train op is run, a non-chief worker - # can only accumulate stats so the tree ensemble never changes. - for _ in range(5): - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 0) - self.assertEquals(len(output.tree_weights), 0) - self.assertEquals(stamp_token.eval(), 0) - - def testTrainFnNonChiefWithCentering(self): - """Tests the train function running on worker with bias centering.""" - with self.cached_session(): - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.regularization.l1 = 0 - learner_config.regularization.l2 = 0 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.min_node_weight = 0 - features = {} - features["dense_float"] = array_ops.ones([4, 1], dtypes.float32) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=False, - num_ps_replicas=0, - center_bias=True, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features) - - predictions = array_ops.constant( - [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) - partition_ids = array_ops.zeros([4], dtypes.int32) - ensemble_stamp = variables.VariableV1( - initial_value=0, - name="ensemble_stamp", - trainable=False, - dtype=dtypes.int64) - - predictions_dict = { - "predictions": predictions, - "predictions_no_dropout": predictions, - "partition_ids": partition_ids, - "ensemble_stamp": ensemble_stamp - } - - labels = array_ops.ones([4, 1], dtypes.float32) - weights = array_ops.ones([4, 1], dtypes.float32) - # Create train op. - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - _squared_loss(labels, weights, predictions)), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # Regardless of how many times the train op is run, a non-chief worker - # can only accumulate stats so the tree ensemble never changes. - for _ in range(5): - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 0) - self.assertEquals(len(output.tree_weights), 0) - self.assertEquals(stamp_token.eval(), 0) - - def testPredictFn(self): - """Tests the predict function.""" - with self.cached_session() as sess: - # Create ensemble with one bias node. - ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - text_format.Merge( - """ - trees { - nodes { - leaf { - vector { - value: 0.25 - } - } - } - } - tree_weights: 1.0 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - is_finalized: true - }""", ensemble_config) - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=3, - tree_ensemble_config=ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.regularization.l1 = 0 - learner_config.regularization.l2 = 0 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.min_node_weight = 0 - features = {} - features["dense_float"] = array_ops.ones([4, 1], dtypes.float32) - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=False, - num_ps_replicas=0, - center_bias=True, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features) - - # Create predict op. - mode = model_fn.ModeKeys.EVAL - predictions_dict = sess.run(gbdt_model.predict(mode)) - self.assertEquals(predictions_dict["ensemble_stamp"], 3) - self.assertAllClose(predictions_dict["predictions"], - [[0.25], [0.25], [0.25], [0.25]]) - self.assertAllClose(predictions_dict["partition_ids"], [0, 0, 0, 0]) - - def testPredictFnWithLeafIndexAdvancedLeft(self): - """Tests the predict function with output leaf ids.""" - with self.cached_session() as sess: - # Create ensemble with one bias node. - ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - text_format.Merge( - """ - trees { - nodes { - dense_float_binary_split { - threshold: 1.0 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 0 - } - } - nodes { - leaf { - vector { - value: 0.25 - } - } - } - nodes { - leaf { - vector { - value: 0.15 - } - } - } - } - trees { - nodes { - dense_float_binary_split { - threshold: 0.99 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 00 - } - } - nodes { - leaf { - vector { - value: 0.25 - } - } - } - nodes { - leaf { - vector { - value: 0.23 - } - } - } - } - tree_weights: 1.0 - tree_weights: 1.0 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - is_finalized: true - } - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - is_finalized: true - }""", ensemble_config) - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=3, - tree_ensemble_config=ensemble_config.SerializeToString(), - name="tree_ensemble") - resources.initialize_resources(resources.shared_resources()).run() - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.regularization.l1 = 0 - learner_config.regularization.l2 = 0 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.min_node_weight = 0 - features = {} - features["dense_float"] = array_ops.constant( - [[0.0], [1.0], [1.1], [2.0]], dtype=dtypes.float32) - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=False, - num_ps_replicas=0, - center_bias=True, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features, - output_leaf_index=True) - - # Create predict op. - mode = model_fn.ModeKeys.INFER - predictions_dict = sess.run(gbdt_model.predict(mode)) - self.assertEquals(predictions_dict["ensemble_stamp"], 3) - # here are how the numbers in expected results are calculated, - # 0.5 = 0.25 + 0.25 - # 0.48 = 0.25 + 0.23 - # 0.38 = 0.15 + 0.23 - # 0.38 = 0.15 + 0.23 - self.assertAllClose(predictions_dict["predictions"], - [[0.5], [0.48], [0.38], [0.38]]) - self.assertAllClose(predictions_dict["partition_ids"], [0, 0, 0, 0]) - self.assertAllClose(predictions_dict["leaf_index"], - [[1, 1], [1, 2], [2, 2], [2, 2]]) - - def testTrainFnMulticlassFullHessian(self): - """Tests the GBDT train for multiclass full hessian.""" - with self.cached_session() as sess: - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 1 - # Use full hessian multiclass strategy. - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.FULL_HESSIAN) - learner_config.num_classes = 5 - learner_config.regularization.l1 = 0 - # To make matrix inversible. - learner_config.regularization.l2 = 1e-5 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.min_node_weight = 0 - features = {} - batch_size = 3 - features["dense_float"] = array_ops.constant( - [0.3, 1.5, 1.1], dtype=dtypes.float32) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=5, - features=features) - - predictions = array_ops.constant( - [[0.0, -1.0, 0.5, 1.2, 3.1], [1.0, 0.0, 0.8, 0.3, 1.0], - [0.0, 0.0, 0.0, 0.0, 1.2]], - dtype=dtypes.float32) - - labels = array_ops.constant([[2], [2], [3]], dtype=dtypes.float32) - weights = array_ops.ones([batch_size, 1], dtypes.float32) - - partition_ids = array_ops.zeros([batch_size], dtypes.int32) - ensemble_stamp = variables.VariableV1( - initial_value=0, - name="ensemble_stamp", - trainable=False, - dtype=dtypes.int64) - - predictions_dict = { - "predictions": predictions, - "predictions_no_dropout": predictions, - "partition_ids": partition_ids, - "ensemble_stamp": ensemble_stamp, - "num_trees": 0, - } - - # Create train op. - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - losses.per_example_maxent_loss( - labels, - weights, - predictions, - num_classes=learner_config.num_classes)[0]), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # On first run, expect no splits to be chosen because the quantile - # buckets will not be ready. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 0) - self.assertEquals(len(output.tree_weights), 0) - self.assertEquals(stamp_token.eval(), 1) - - # Update the stamp to be able to run a second time. - sess.run([ensemble_stamp.assign_add(1)]) - # On second run, expect a trivial split to be chosen to basically - # predict the average. - train_op.run() - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output.ParseFromString(serialized.eval()) - self.assertEqual(len(output.trees), 1) - # We got 3 nodes: one parent and 2 leafs. - self.assertEqual(len(output.trees[0].nodes), 3) - self.assertAllClose(output.tree_weights, [1]) - self.assertEquals(stamp_token.eval(), 2) - - # Leafs should have a dense vector of size 5. - expected_leaf_1 = [-3.4480, -3.4429, 13.8490, -3.45, -3.4508] - expected_leaf_2 = [-1.2547, -1.3145, 1.52, 2.3875, -1.3264] - self.assertArrayNear(expected_leaf_1, - output.trees[0].nodes[1].leaf.vector.value, 7e-3) - self.assertArrayNear(expected_leaf_2, - output.trees[0].nodes[2].leaf.vector.value, 7e-3) - - def testTrainFnMulticlassDiagonalHessian(self): - """Tests the GBDT train for multiclass diagonal hessian.""" - with self.cached_session() as sess: - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 1 - # Use full hessian multiclass strategy. - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) - learner_config.num_classes = 5 - learner_config.regularization.l1 = 0 - # To make matrix inversible. - learner_config.regularization.l2 = 1e-5 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.min_node_weight = 0 - batch_size = 3 - features = {} - features["dense_float"] = array_ops.constant( - [0.3, 1.5, 1.1], dtype=dtypes.float32) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=5, - features=features) - - predictions = array_ops.constant( - [[0.0, -1.0, 0.5, 1.2, 3.1], [1.0, 0.0, 0.8, 0.3, 1.0], - [0.0, 0.0, 0.0, 0.0, 1.2]], - dtype=dtypes.float32) - - labels = array_ops.constant([[2], [2], [3]], dtype=dtypes.float32) - weights = array_ops.ones([batch_size, 1], dtypes.float32) - - partition_ids = array_ops.zeros([batch_size], dtypes.int32) - ensemble_stamp = variables.VariableV1( - initial_value=0, - name="ensemble_stamp", - trainable=False, - dtype=dtypes.int64) - - predictions_dict = { - "predictions": predictions, - "predictions_no_dropout": predictions, - "partition_ids": partition_ids, - "ensemble_stamp": ensemble_stamp, - "num_trees": 0, - } - - # Create train op. - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - losses.per_example_maxent_loss( - labels, - weights, - predictions, - num_classes=learner_config.num_classes)[0]), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # On first run, expect no splits to be chosen because the quantile - # buckets will not be ready. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEqual(len(output.trees), 0) - self.assertEqual(len(output.tree_weights), 0) - self.assertEqual(stamp_token.eval(), 1) - - # Update the stamp to be able to run a second time. - sess.run([ensemble_stamp.assign_add(1)]) - # On second run, expect a trivial split to be chosen to basically - # predict the average. - train_op.run() - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output.ParseFromString(serialized.eval()) - self.assertEqual(len(output.trees), 1) - # We got 3 nodes: one parent and 2 leafs. - self.assertEqual(len(output.trees[0].nodes), 3) - self.assertAllClose(output.tree_weights, [1]) - self.assertEqual(stamp_token.eval(), 2) - - # Leafs should have a dense vector of size 5. - expected_leaf_1 = [-1.0354, -1.0107, 17.2976, -1.1313, -4.5023] - expected_leaf_2 = [-1.2924, -1.1376, 2.2042, 3.1052, -1.6269] - self.assertArrayNear(expected_leaf_1, - output.trees[0].nodes[1].leaf.vector.value, 1e-3) - self.assertArrayNear(expected_leaf_2, - output.trees[0].nodes[2].leaf.vector.value, 1e-3) - - def testTrainFnMulticlassDiagonalHessianOblivious(self): - """Tests the GBDT train for multiclass diagonal hessian.""" - with self.cached_session(): - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 1 - # Use full hessian multiclass strategy. - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) - learner_config.num_classes = 5 - learner_config.regularization.l1 = 0 - # To make matrix inversible. - learner_config.regularization.l2 = 1e-5 - learner_config.weak_learner_type = ( - learner_pb2.LearnerConfig.OBLIVIOUS_DECISION_TREE) - learner_config.pruning_mode = learner_pb2.LearnerConfig.PRE_PRUNE - learner_config.constraints.max_tree_depth = 5 - learner_config.constraints.min_node_weight = 0 - batch_size = 3 - features = {} - features["sparse_int"] = sparse_tensor.SparseTensor( - array_ops.constant([[0, 0], [1, 0]], dtypes.int64), - array_ops.constant([1, 2], dtypes.int64), - array_ops.constant([3, 1], dtypes.int64)) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=5, - features=features) - - labels = array_ops.constant([[2], [2], [3]], dtype=dtypes.float32) - weights = array_ops.ones([batch_size, 1], dtypes.float32) - - predictions_dict = gbdt_model.predict(learn.ModeKeys.TRAIN) - predictions = predictions_dict["predictions"] - - # Create train op. - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - losses.per_example_maxent_loss( - labels, - weights, - predictions, - num_classes=learner_config.num_classes)[0]), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - - # Grow 2 layers. - train_op.run() - train_op.run() - - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output.ParseFromString(serialized.eval()) - self.assertEqual(len(output.trees), 1) - # We got 6 nodes: one parent and 4 leafs. - self.assertEqual(len(output.trees[0].nodes), 6) - self.assertAllClose(output.tree_weights, [1]) - self.assertEqual(stamp_token.eval(), 2) - - print(output.trees[0]) - # Leafs should have a dense vector of size 5. - expected_leaf_1 = [-1.2497, -1.24976, 4.999, -1.24976, -1.2497] - expected_leaf_2 = [-2.2362, -2.2362, 6.0028, -2.2362, -2.2362] - expected_leaf_3 = [-2.2694, -2.2694, 4.0064, -0.0084, -2.2694] - expected_leaf_4 = [-2.2694, -2.2694, -0.0084, 4.0064, -2.2694] - self.assertArrayNear(expected_leaf_1, - output.trees[0].nodes[2].leaf.vector.value, 1e-3) - self.assertArrayNear(expected_leaf_2, - output.trees[0].nodes[3].leaf.vector.value, 1e-3) - self.assertArrayNear(expected_leaf_3, - output.trees[0].nodes[4].leaf.vector.value, 1e-3) - self.assertArrayNear(expected_leaf_4, - output.trees[0].nodes[5].leaf.vector.value, 1e-3) - - def testTrainFnMulticlassTreePerClass(self): - """Tests the GBDT train for multiclass tree per class strategy.""" - with self.cached_session() as sess: - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 1 - # Use full hessian multiclass strategy. - learner_config.multi_class_strategy = ( - learner_pb2.LearnerConfig.TREE_PER_CLASS) - learner_config.num_classes = 5 - learner_config.regularization.l1 = 0 - # To make matrix inversible. - learner_config.regularization.l2 = 1e-5 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.min_node_weight = 0 - features = { - "dense_float": - array_ops.constant([[1.0], [1.5], [2.0]], dtypes.float32), - } - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=5, - features=features) - - batch_size = 3 - predictions = array_ops.constant( - [[0.0, -1.0, 0.5, 1.2, 3.1], [1.0, 0.0, 0.8, 0.3, 1.0], - [0.0, 0.0, 0.0, 2.0, 1.2]], - dtype=dtypes.float32) - - labels = array_ops.constant([[2], [2], [3]], dtype=dtypes.float32) - weights = array_ops.ones([batch_size, 1], dtypes.float32) - - partition_ids = array_ops.zeros([batch_size], dtypes.int32) - ensemble_stamp = variables.VariableV1( - initial_value=0, - name="ensemble_stamp", - trainable=False, - dtype=dtypes.int64) - - predictions_dict = { - "predictions": predictions, - "predictions_no_dropout": predictions, - "partition_ids": partition_ids, - "ensemble_stamp": ensemble_stamp, - # This should result in a tree built for a class 2. - "num_trees": 13, - } - - # Create train op. - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - losses.per_example_maxent_loss( - labels, - weights, - predictions, - num_classes=learner_config.num_classes)[0]), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # On first run, expect no splits to be chosen because the quantile - # buckets will not be ready. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEqual(len(output.trees), 0) - self.assertEqual(len(output.tree_weights), 0) - self.assertEqual(stamp_token.eval(), 1) - - # Update the stamp to be able to run a second time. - sess.run([ensemble_stamp.assign_add(1)]) - # On second run, expect a trivial split to be chosen to basically - # predict the average. - train_op.run() - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output.ParseFromString(serialized.eval()) - self.assertEqual(len(output.trees), 1) - self.assertAllClose(output.tree_weights, [1]) - self.assertEqual(stamp_token.eval(), 2) - - # One node for a split, two children nodes. - self.assertEqual(3, len(output.trees[0].nodes)) - - # Leafs will have a sparse vector for class 3. - self.assertEqual(1, - len(output.trees[0].nodes[1].leaf.sparse_vector.index)) - self.assertEqual(3, output.trees[0].nodes[1].leaf.sparse_vector.index[0]) - self.assertAlmostEqual( - -1.13134455681, output.trees[0].nodes[1].leaf.sparse_vector.value[0]) - - self.assertEqual(1, - len(output.trees[0].nodes[2].leaf.sparse_vector.index)) - self.assertEqual(3, output.trees[0].nodes[2].leaf.sparse_vector.index[0]) - self.assertAllClose( - 0.893284678459, - output.trees[0].nodes[2].leaf.sparse_vector.value[0], - atol=1e-4, - rtol=1e-4) - - def testTrainFnChiefFeatureSelectionReachedLimitNoGoodSplit(self): - """Tests the train function running on chief with feature selection.""" - with self.cached_session() as sess: - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.regularization.l1 = 0 - learner_config.regularization.l2 = 0 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.max_number_of_unique_feature_columns = 1 - learner_config.constraints.min_node_weight = 0 - features = {} - features["dense_float_0"] = array_ops.ones([4, 1], dtypes.float32) - # Feature 1 is predictive but it won't be used because we have reached the - # limit of num_used_handlers >= max_number_of_unique_feature_columns - features["dense_float_1"] = array_ops.constant([0, 0, 1, 1], - dtypes.float32) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features) - - predictions = array_ops.constant( - [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) - partition_ids = array_ops.zeros([4], dtypes.int32) - ensemble_stamp = variables.VariableV1( - initial_value=0, - name="ensemble_stamp", - trainable=False, - dtype=dtypes.int64) - - predictions_dict = { - "predictions": - predictions, - "predictions_no_dropout": - predictions, - "partition_ids": - partition_ids, - "ensemble_stamp": - ensemble_stamp, - "num_trees": - 12, - "num_used_handlers": - array_ops.constant(1, dtype=dtypes.int64), - "used_handlers_mask": - array_ops.constant([True, False], dtype=dtypes.bool), - } - - labels = array_ops.constant([0, 0, 1, 1], dtypes.float32) - weights = array_ops.ones([4, 1], dtypes.float32) - # Create train op. - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - _squared_loss(labels, weights, predictions)), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # On first run, expect no splits to be chosen because the quantile - # buckets will not be ready. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 0) - self.assertEquals(len(output.tree_weights), 0) - self.assertEquals(stamp_token.eval(), 1) - - # Update the stamp to be able to run a second time. - sess.run([ensemble_stamp.assign_add(1)]) - - # On second run, expect a trivial split to be chosen to basically - # predict the average. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 1) - self.assertAllClose(output.tree_weights, [0.1]) - self.assertEquals(stamp_token.eval(), 2) - expected_tree = """ - nodes { - dense_float_binary_split { - feature_column: 0 - threshold: 1.0 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 0 - } - } - nodes { - leaf { - vector { - value: -0.25 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - }""" - self.assertProtoEquals(expected_tree, output.trees[0]) - - def testTrainFnChiefFeatureSelectionWithGoodSplits(self): - """Tests the train function running on chief with feature selection.""" - with self.cached_session() as sess: - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.regularization.l1 = 0 - learner_config.regularization.l2 = 0 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.max_number_of_unique_feature_columns = 1 - learner_config.constraints.min_node_weight = 0 - features = {} - features["dense_float_0"] = array_ops.ones([4, 1], dtypes.float32) - # Feature 1 is predictive and is in our selected features so it will be - # used even when we're at the limit. - features["dense_float_1"] = array_ops.constant([0, 0, 1, 1], - dtypes.float32) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features) - - predictions = array_ops.constant( - [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) - partition_ids = array_ops.zeros([4], dtypes.int32) - ensemble_stamp = variables.VariableV1( - initial_value=0, - name="ensemble_stamp", - trainable=False, - dtype=dtypes.int64) - - predictions_dict = { - "predictions": - predictions, - "predictions_no_dropout": - predictions, - "partition_ids": - partition_ids, - "ensemble_stamp": - ensemble_stamp, - "num_trees": - 12, - "num_used_handlers": - array_ops.constant(1, dtype=dtypes.int64), - "used_handlers_mask": - array_ops.constant([False, True], dtype=dtypes.bool), - } - - labels = array_ops.constant([0, 0, 1, 1], dtypes.float32) - weights = array_ops.ones([4, 1], dtypes.float32) - # Create train op. - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - _squared_loss(labels, weights, predictions)), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # On first run, expect no splits to be chosen because the quantile - # buckets will not be ready. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 0) - self.assertEquals(len(output.tree_weights), 0) - self.assertEquals(stamp_token.eval(), 1) - - # Update the stamp to be able to run a second time. - sess.run([ensemble_stamp.assign_add(1)]) - - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - - self.assertEquals(len(output.trees), 1) - self.assertAllClose(output.tree_weights, [0.1]) - self.assertEquals(stamp_token.eval(), 2) - expected_tree = """ - nodes { - dense_float_binary_split { - feature_column: 1 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 0.5 - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - } - nodes { - leaf { - vector { - value: -0.5 - } - } - }""" - self.assertProtoEquals(expected_tree, output.trees[0]) - - def testTrainFnChiefFeatureSelectionReachedLimitIncrementAttemptedLayer(self): - """Tests the train function running on chief with feature selection.""" - with self.cached_session() as sess: - tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - tree = tree_ensemble_config.trees.add() - - _set_float_split( - tree.nodes.add().sparse_float_binary_split_default_right.split, 2, - 4.0, 1, 2) - _append_to_leaf(tree.nodes.add().leaf, 0, 0.5) - _append_to_leaf(tree.nodes.add().leaf, 1, 1.2) - tree_ensemble_config.tree_weights.append(1.0) - metadata = tree_ensemble_config.tree_metadata.add() - metadata.is_finalized = False - metadata.num_layers_grown = 1 - tree_ensemble_config = tree_ensemble_config.SerializeToString() - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=tree_ensemble_config, - name="tree_ensemble") - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.regularization.l1 = 0 - learner_config.regularization.l2 = 0 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.max_number_of_unique_feature_columns = 1 - learner_config.constraints.min_node_weight = 0 - features = {} - # Both features will be disabled since the feature selection limit is - # already reached. - features["dense_float_0"] = array_ops.ones([4, 1], dtypes.float32) - features["dense_float_1"] = array_ops.constant([0, 0, 1, 1], - dtypes.float32) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features) - - predictions = array_ops.constant( - [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) - partition_ids = array_ops.zeros([4], dtypes.int32) - ensemble_stamp = variables.VariableV1( - initial_value=0, - name="ensemble_stamp", - trainable=False, - dtype=dtypes.int64) - - predictions_dict = { - "predictions": - predictions, - "predictions_no_dropout": - predictions, - "partition_ids": - partition_ids, - "ensemble_stamp": - ensemble_stamp, - "num_trees": - 12, - # We have somehow reached our limit 1. Both of the handlers will be - # disabled. - "num_used_handlers": - array_ops.constant(1, dtype=dtypes.int64), - "used_handlers_mask": - array_ops.constant([False, False], dtype=dtypes.bool), - } - - labels = array_ops.constant([0, 0, 1, 1], dtypes.float32) - weights = array_ops.ones([4, 1], dtypes.float32) - # Create train op. - train_op = gbdt_model.train( - loss=math_ops.reduce_mean( - _squared_loss(labels, weights, predictions)), - predictions_dict=predictions_dict, - labels=labels) - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # On first run, expect no splits to be chosen because the quantile - # buckets will not be ready. - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 1) - self.assertEquals(output.growing_metadata.num_layers_attempted, 1) - self.assertEquals(stamp_token.eval(), 1) - - # Update the stamp to be able to run a second time. - sess.run([ensemble_stamp.assign_add(1)]) - - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - # Make sure the trees are not modified, but the num_layers_attempted is - # incremented so that eventually the training stops. - self.assertEquals(len(output.trees), 1) - self.assertEquals(len(output.trees[0].nodes), 3) - - self.assertEquals(output.growing_metadata.num_layers_attempted, 2) - - def testResetModelBeforeAndAfterSplit(self): - """Tests whether resetting works.""" - with self.cached_session(): - # First build a small tree and train it to verify training works. - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - features = {} - features["dense_float"] = array_ops.ones([4, 1], dtypes.float32) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features) - - predictions = array_ops.constant( - [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) - partition_ids = array_ops.zeros([4], dtypes.int32) - ensemble_stamp = model_ops.tree_ensemble_stamp_token(ensemble_handle) - - predictions_dict = { - "predictions": predictions, - "predictions_no_dropout": predictions, - "partition_ids": partition_ids, - "ensemble_stamp": ensemble_stamp, - "num_trees": 12, - "max_tree_depth": 4, - } - - labels = array_ops.ones([4, 1], dtypes.float32) - weights = array_ops.ones([4, 1], dtypes.float32) - loss = math_ops.reduce_mean(_squared_loss(labels, weights, predictions)) - - # Create train op. - update_op, reset_op, training_state = gbdt_model.update_stats( - loss, predictions_dict) - with ops.control_dependencies(update_op): - train_op = gbdt_model.increment_step_counter_and_maybe_update_ensemble( - predictions_dict, training_state) - - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - original_stamp = ensemble_stamp.eval() - expected_tree = """ - nodes { - dense_float_binary_split { - threshold: 1.0 - left_id: 1 - right_id: 2 - } - node_metadata { - gain: 0 - } - } - nodes { - leaf { - vector { - value: 0.25 - } - } - } - nodes { - leaf { - vector { - value: 0.0 - } - } - }""" - - def _train_once_and_check(expect_split): - stamp = ensemble_stamp.eval() - train_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(stamp_token.eval(), stamp + 1) - if expect_split: - # State of the ensemble after a split occurs. - self.assertEquals(len(output.trees), 1) - self.assertProtoEquals(expected_tree, output.trees[0]) - else: - # State of the ensemble after a single accumulation but before any - # splitting occurs - self.assertEquals(len(output.trees), 0) - self.assertProtoEquals(""" - growing_metadata { - num_trees_attempted: 1 - num_layers_attempted: 1 - }""", output) - - def _run_reset(): - stamp_before_reset = ensemble_stamp.eval() - reset_op.run() - stamp_after_reset = ensemble_stamp.eval() - self.assertNotEquals(stamp_after_reset, stamp_before_reset) - - _, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertProtoEquals("", output) - - return stamp_after_reset - - # Exit after one train_op, so no new layer are created but the handlers - # contain enough information to split on the next call to train. - _train_once_and_check(expect_split=False) - self.assertEquals(ensemble_stamp.eval(), original_stamp + 1) - - # Reset the handlers so it still requires two training calls to split. - stamp_after_reset = _run_reset() - - _train_once_and_check(expect_split=False) - _train_once_and_check(expect_split=True) - self.assertEquals(ensemble_stamp.eval(), stamp_after_reset + 2) - - # This time, test that the reset_op works right after splitting. - stamp_after_reset = _run_reset() - - # Test that after resetting, the tree can be trained as normal. - _train_once_and_check(expect_split=False) - _train_once_and_check(expect_split=True) - self.assertEquals(ensemble_stamp.eval(), stamp_after_reset + 2) - - def testResetModelNonChief(self): - """Tests the reset function on a non-chief worker.""" - with self.cached_session(): - # Create ensemble with one bias node. - ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig() - text_format.Merge( - """ - trees { - nodes { - leaf { - vector { - value: 0.25 - } - } - } - } - tree_weights: 1.0 - tree_metadata { - num_tree_weight_updates: 1 - num_layers_grown: 1 - is_finalized: false - }""", ensemble_config) - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, - tree_ensemble_config=ensemble_config.SerializeToString(), - name="tree_ensemble") - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.constraints.max_tree_depth = 1 - features = {} - features["dense_float"] = array_ops.ones([4, 1], dtypes.float32) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=False, - num_ps_replicas=0, - center_bias=False, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features) - - predictions = array_ops.constant( - [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) - partition_ids = array_ops.zeros([4], dtypes.int32) - ensemble_stamp = model_ops.tree_ensemble_stamp_token(ensemble_handle) - - predictions_dict = { - "predictions": predictions, - "predictions_no_dropout": predictions, - "partition_ids": partition_ids, - "ensemble_stamp": ensemble_stamp - } - - labels = array_ops.ones([4, 1], dtypes.float32) - weights = array_ops.ones([4, 1], dtypes.float32) - loss = math_ops.reduce_mean(_squared_loss(labels, weights, predictions)) - - # Create reset op. - _, reset_op, _ = gbdt_model.update_stats( - loss, predictions_dict) - - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # Reset op doesn't do anything because this is a non-chief worker. - reset_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 1) - self.assertEquals(len(output.tree_weights), 1) - self.assertEquals(stamp_token.eval(), 0) - - def testResetModelWithCenterBias(self): - """Tests the reset function running on chief with bias centering.""" - with self.cached_session(): - ensemble_handle = model_ops.tree_ensemble_variable( - stamp_token=0, tree_ensemble_config="", name="tree_ensemble") - learner_config = learner_pb2.LearnerConfig() - learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 - learner_config.num_classes = 2 - learner_config.regularization.l1 = 0 - learner_config.regularization.l2 = 0 - learner_config.constraints.max_tree_depth = 1 - learner_config.constraints.min_node_weight = 0 - features = {} - features["dense_float"] = array_ops.ones([4, 1], dtypes.float32) - - gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( - is_chief=True, - num_ps_replicas=0, - center_bias=True, - ensemble_handle=ensemble_handle, - examples_per_layer=1, - learner_config=learner_config, - logits_dimension=1, - features=features) - - predictions = array_ops.constant( - [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) - partition_ids = array_ops.zeros([4], dtypes.int32) - ensemble_stamp = model_ops.tree_ensemble_stamp_token(ensemble_handle) - - predictions_dict = { - "predictions": predictions, - "predictions_no_dropout": predictions, - "partition_ids": partition_ids, - "ensemble_stamp": ensemble_stamp, - "num_trees": 12, - } - - labels = array_ops.ones([4, 1], dtypes.float32) - weights = array_ops.ones([4, 1], dtypes.float32) - loss = math_ops.reduce_mean(_squared_loss(labels, weights, predictions)) - - # Create train op. - update_op, reset_op, training_state = gbdt_model.update_stats( - loss, predictions_dict) - with ops.control_dependencies(update_op): - train_op = gbdt_model.increment_step_counter_and_maybe_update_ensemble( - predictions_dict, training_state) - - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - - # On first run, expect bias to be centered. - def train_and_check(): - train_op.run() - _, serialized = model_ops.tree_ensemble_serialize(ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - expected_tree = """ - nodes { - leaf { - vector { - value: 0.25 - } - } - }""" - self.assertEquals(len(output.trees), 1) - self.assertAllEqual(output.tree_weights, [1.0]) - self.assertProtoEquals(expected_tree, output.trees[0]) - - train_and_check() - self.assertEquals(ensemble_stamp.eval(), 1) - - reset_op.run() - stamp_token, serialized = model_ops.tree_ensemble_serialize( - ensemble_handle) - output = tree_config_pb2.DecisionTreeEnsembleConfig() - output.ParseFromString(serialized.eval()) - self.assertEquals(len(output.trees), 0) - self.assertEquals(len(output.tree_weights), 0) - self.assertEquals(stamp_token.eval(), 2) - - train_and_check() - self.assertEquals(ensemble_stamp.eval(), 3) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/python/utils/__init__.py b/tensorflow/contrib/boosted_trees/python/utils/__init__.py deleted file mode 100644 index 6ceb150c265..00000000000 --- a/tensorflow/contrib/boosted_trees/python/utils/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""utils module under boosted_trees.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/boosted_trees/python/utils/losses.py b/tensorflow/contrib/boosted_trees/python/utils/losses.py deleted file mode 100644 index 40fdfcf45ac..00000000000 --- a/tensorflow/contrib/boosted_trees/python/utils/losses.py +++ /dev/null @@ -1,319 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Losses for Gtflow Estimator and Batch Estimator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops.losses import losses - - -def per_example_squared_hinge_loss(labels, weights, predictions): - loss = losses.hinge_loss(labels=labels, logits=predictions, weights=weights) - return math_ops.square(loss), control_flow_ops.no_op() - - -def per_example_logistic_loss(labels, weights, predictions): - """Logistic loss given labels, example weights and predictions. - - Args: - labels: Rank 2 (N, 1) tensor of per-example labels. - weights: Rank 2 (N, 1) tensor of per-example weights. - predictions: Rank 2 (N, 1) tensor of per-example predictions. - - Returns: - loss: A Rank 2 (N, 1) tensor of per-example logistic loss. - update_op: An update operation to update the loss's internal state. - """ - labels = math_ops.cast(labels, dtypes.float32) - unweighted_loss = nn.sigmoid_cross_entropy_with_logits( - labels=labels, logits=predictions) - return unweighted_loss * weights, control_flow_ops.no_op() - -# MUST USE WITH HESSIAN REGULARIZATION, -# This loss can have zero hessian, so it must be used with l2 or min_node_weight -# regularization. -# An example config is -# learner_config.constraints.min_node_weight = 1 / num_examples_per_layer -# learner_config.regularization.l2 = 1.0 / num_examples_per_layer -# TODO(nponomareva): make it multidimensional so we can estimate several -# quantiles at once. -def per_example_quantile_regression_loss(labels, weights, predictions, - quantile): - """Smoothed loss for quantile regression. - - The standard quantile regression loss is quantile*(y-y') when y>y' and - (quantile-1)*(y-y') otherwise, y' is a prediction, y is a label. The impl - below is this loss but squared in the region where the loss value < 1. - - Args: - labels: Rank 2 (N, D) tensor of per-example labels. - weights: Rank 2 (N, 1) tensor of per-example weights. - predictions: Rank 2 (N, D) tensor of per-example predictions. - quantile: The quantile to use. - - Returns: - loss: A Rank 2 (N, 1) tensor of per-example quantile loss. - update_op: An update operation to update the loss's internal state. - """ - labels = math_ops.cast(labels, dtypes.float32) - error = labels - predictions - square_loss_right = array_ops.where(error * quantile < 1.0, - math_ops.square(quantile * error), - quantile * error) - square_loss_left = array_ops.where(error * (quantile - 1) < 1, - math_ops.square((quantile - 1) * error), - (quantile - 1) * error) - - unweighted_loss = array_ops.where(error > 0, square_loss_right, - square_loss_left) - if weights is None: - return unweighted_loss, control_flow_ops.no_op() - else: - return unweighted_loss * weights, control_flow_ops.no_op() - -# This is classical form of Maximum entropy loss, that is twice differentiable -# (sparse_softmax_cross_entropy which is what we go for is not twice -# differentiable). -def per_example_maxent_loss(labels, weights, logits, num_classes, eps=1e-15): - """Maximum entropy loss for multiclass problems. - - Maximum entropy is a generalization of logistic loss for the case when more - than 2 classes are present. - - Args: - labels: Rank 2 (N, 1) or Rank 1 (N) tensor of per-example labels. - weights: Rank 2 (N, 1) tensor of per-example weights. - logits: Rank 2 (N, K) tensor of per-example predictions, K - num of - classes. - num_classes: number of classes in classification task. Used to expand label - indices into one-hot encodings. - eps: tolerance, used as a minimum possible value. - - Returns: - loss: A Rank 2 (N, 1) tensor of per-example maxent loss - update_op: An update operation to update the loss's internal state. - """ - labels = math_ops.cast(labels, dtypes.int64) - # If labels are of rank 1, make them rank 2. - labels_shape = labels.get_shape() - if len(labels_shape) != 2: - labels = array_ops.expand_dims(labels, 1) - # Labels are indices of classes, convert them to one hot encodings. - target_one_hot = array_ops.one_hot(indices=labels, depth=num_classes) - labels = math_ops.reduce_sum(input_tensor=target_one_hot, axis=[1]) - labels = math_ops.cast(labels, dtypes.float32) - - # Calculate softmax probabilities for each class. - unnormalized_probs = math_ops.exp(logits) - normalizers = math_ops.reduce_sum(unnormalized_probs, 1, keepdims=True) - softmax_predictions = math_ops.divide(unnormalized_probs, - math_ops.add(normalizers, eps)) - - # Pull out the probabilities for real label. - probs_for_real_class = math_ops.reduce_sum(labels * softmax_predictions, 1) - - # Add handling for values near 0 and 1. - zeros = array_ops.zeros_like(probs_for_real_class, dtype=logits.dtype) + eps - one_minus_eps = array_ops.ones_like( - probs_for_real_class, dtype=logits.dtype) - eps - - # Take maximum(eps, pred) - cond = (probs_for_real_class >= eps) - probs_for_real_class = array_ops.where(cond, probs_for_real_class, zeros) - - # Take minimum(1-eps, pred) - cond = (probs_for_real_class <= 1 - eps) - probs_for_real_class = array_ops.where(cond, probs_for_real_class, - one_minus_eps) - - unweighted_loss = array_ops.expand_dims(-math_ops.log(probs_for_real_class), - 1) - if weights is None: - return unweighted_loss, control_flow_ops.no_op() - else: - return unweighted_loss * weights, control_flow_ops.no_op() - - -def per_example_squared_loss(labels, weights, predictions): - """Squared loss given labels, example weights and predictions. - - Args: - labels: Rank 2 (N, D) tensor of per-example labels. - weights: Rank 2 (N, 1) tensor of per-example weights. - predictions: Rank 2 (N, D) tensor of per-example predictions. - - Returns: - loss: A Rank 2 (N, 1) tensor of per-example squared loss. - update_op: An update operation to update the loss's internal state. - """ - unweighted_loss = math_ops.reduce_sum( - math_ops.squared_difference(predictions, labels), 1, keepdims=True) - - return unweighted_loss * weights, control_flow_ops.no_op() - - -def per_example_exp_loss(labels, weights, predictions, name=None, eps=0.1): - """Trimmed exponential loss given labels, example weights and predictions. - - Note that this is only for binary classification. - If logistic loss tries to make sure that the classifier is certain of its - predictions, exp loss says: "as long as it got it correct, even barely, i - don't care". Can be used on noisy data, or when you don't care about getting - the actual probabilities from the model, just the correct label. - - The loss returns is exp(-targets*modified_predictions), where - modified_predictions are 1 if sigmoid is >= 0.5+eps (eg we predict positive - class), -1 if sigmoid < 0.5-eps (e.g. we predict negative class) and ax+b in - the interval 0.5-eps, 0.5+eps, where a = 1/eps, b=1/(2eps). - - Args: - labels: Rank 2 (N, D) tensor of per-example labels. - weights: Rank 2 (N, 1) tensor of per-example weights. - predictions: Rank 2 (N, D) tensor of per-example predictions. - name: A name for the operation (optional). - eps: For the range (0.5-eps, 0.5+eps) we set the predictions to be ax+b. - - Returns: - loss: A Rank 2 (N, 1) tensor of per-example exp loss - update_op: An update operation to update the loss's internal state. - """ - - def exp_with_logits(name, eps, labels=None, logits=None): - """Computes exponential loss given `logits`. - - The loss returns is exp(-targets*modified_predictions), where - modified_predictions are 1 if sigmoid is >= 0.5+eps (eg we predict positive - class), -1 if sigmoid < 0.5-eps (e.g. we predict negative class) and ax+b in - the interval 0.5-eps, 0.5+eps, where a = 1/eps, b=1/(2eps). - - Args: - name: A name for the operation (optional). - eps: For the range (0.5-eps, 0.5+eps) we set the predictions to be ax+b. - labels: A `Tensor` of the same type and shape as `logits`. - logits: A `Tensor` of type `float32` or `float64`. - - Returns: - A `Tensor` of the same shape as `logits` with the componentwise - exponential losses. - - Raises: - ValueError: If `logits` and `labels` do not have the same shape. - """ - with ops.name_scope(name, "exp_loss", [logits, labels]) as name: - logits = ops.convert_to_tensor(logits, name="logits") - labels = ops.convert_to_tensor(labels, name="labels") - try: - labels.get_shape().merge_with(logits.get_shape()) - except ValueError: - raise ValueError("logits and labels must have the same shape (%s vs %s)" - % (logits.get_shape(), labels.get_shape())) - - # Default threshold to switch between classes - zeros = array_ops.zeros_like(logits, dtype=logits.dtype) - ones = array_ops.ones_like(logits, dtype=logits.dtype) - neg_ones = -array_ops.ones_like(logits, dtype=logits.dtype) - - # Convert labels to 1 and -1 - cond_labels = (labels > zeros) - labels_converted = array_ops.where(cond_labels, ones, neg_ones) - - # Convert predictions to 1 and -1 - # The loss we build is min(1, max(-1,ax+b)) - # where a=1/eps, b=-1/2eps. - - a = 1.0 / eps - b = -1.0 / 2 / eps - probs = math_ops.sigmoid(logits) - y = a * probs + b - # Build max(-1, ax+b) - cond = (y < -1) - max_res = array_ops.where(cond, neg_ones, y) - # Build min part - cond = (max_res > 1) - min_res = array_ops.where(cond, ones, max_res) - preds_converted = min_res - return math_ops.exp(-preds_converted * labels_converted) - - labels = math_ops.cast(labels, dtypes.float32) - unweighted_loss = exp_with_logits( - name=name, eps=eps, labels=labels, logits=predictions) - return unweighted_loss * weights, control_flow_ops.no_op() - - -def per_example_full_exp_loss(labels, weights, predictions, name=None): - """Full exponential loss given labels, example weights and predictions. - - Note that this is only for binary classification. - The loss returns is exp(-targets*logits), where targets are converted to -1 - and 1. - - Args: - labels: Rank 2 (N, D) tensor of per-example labels. - weights: Rank 2 (N, 1) tensor of per-example weights. - predictions: Rank 2 (N, D) tensor of per-example predictions. - name: A name for the operation (optional). - - Returns: - loss: A Rank 2 (N, 1) tensor of per-example exp loss - update_op: An update operation to update the loss's internal state. - """ - - def full_exp_with_logits(name, labels=None, logits=None): - """Computes exponential loss given `logits`. - - Args: - name: A name for the operation (optional). - labels: A `Tensor` of the same type and shape as `logits`. - logits: A `Tensor` of type `float32` or `float64`. - - Returns: - A `Tensor` of the same shape as `logits` with the componentwise - exponential losses. - - Raises: - ValueError: If `logits` and `labels` do not have the same shape. - """ - with ops.name_scope(name, "exp_loss", [logits, labels]) as name: - logits = ops.convert_to_tensor(logits, name="logits") - labels = ops.convert_to_tensor(labels, name="labels") - try: - labels.get_shape().merge_with(logits.get_shape()) - except ValueError: - raise ValueError("logits and labels must have the same shape (%s vs %s)" - % (logits.get_shape(), labels.get_shape())) - - # Default threshold of 0 to switch between classes - zeros = array_ops.zeros_like(logits, dtype=logits.dtype) - ones = array_ops.ones_like(logits, dtype=logits.dtype) - neg_ones = -array_ops.ones_like(logits, dtype=logits.dtype) - - # Convert labels to 1 and -1 - cond_labels = (labels > zeros) - labels_converted = array_ops.where(cond_labels, ones, neg_ones) - - return math_ops.exp(-1.0 * logits * labels_converted) - - labels = math_ops.cast(labels, dtypes.float32) - unweighted_loss = full_exp_with_logits( - name=name, labels=labels, logits=predictions) - return unweighted_loss * weights, control_flow_ops.no_op() diff --git a/tensorflow/contrib/boosted_trees/python/utils/losses_test.py b/tensorflow/contrib/boosted_trees/python/utils/losses_test.py deleted file mode 100644 index cc22504c8f3..00000000000 --- a/tensorflow/contrib/boosted_trees/python/utils/losses_test.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for trainer hooks.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.boosted_trees.python.utils import losses -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import googletest - - -class LossesTest(test_util.TensorFlowTestCase): - - def test_per_example_exp_loss(self): - - def _logit(p): - return np.log(p) - np.log(1 - p) - - labels_positive = array_ops.ones([10, 1], dtypes.float32) - weights = array_ops.ones([10, 1], dtypes.float32) - labels_negative = array_ops.zeros([10, 1], dtypes.float32) - predictions_probs = np.array( - [[0.1], [0.2], [0.3], [0.4], [0.5], [0.6], [0.7], [0.8], [0.9], [0.99]], - dtype=np.float32) - prediction_logits = _logit(predictions_probs) - - eps = 0.2 - - with self.cached_session(): - predictions_tensor = constant_op.constant( - prediction_logits, dtype=dtypes.float32) - loss_for_positives, _ = losses.per_example_exp_loss( - labels_positive, weights, predictions_tensor, eps=eps) - - loss_for_negatives, _ = losses.per_example_exp_loss( - labels_negative, weights, predictions_tensor, eps=eps) - - pos_loss = loss_for_positives.eval() - neg_loss = loss_for_negatives.eval() - # For positive labels, points <= 0.3 get max loss of e. - # For negative labels, these points have minimum loss of 1/e. - self.assertAllClose(np.exp(np.ones([2, 1])), pos_loss[:2], atol=1e-4) - self.assertAllClose(np.exp(-np.ones([2, 1])), neg_loss[:2], atol=1e-4) - - # For positive lables, p oints with predictions 0.7 and larger get minimum - # loss value of 1/e. For negative labels, these points are wrongly - # classified and get loss e. - self.assertAllClose(np.exp(-np.ones([4, 1])), pos_loss[6:10], atol=1e-4) - self.assertAllClose(np.exp(np.ones([4, 1])), neg_loss[6:10], atol=1e-4) - - # Points in between 0.5-eps, 0..5+eps get loss exp(-label_m*y), where - # y = 1/eps *x -1/(2eps), where x is the probability and label_m is either - # 1 or -1 (for label of 0). - self.assertAllClose( - np.exp(-(predictions_probs[2:6] * 1.0 / eps - 0.5 / eps)), - pos_loss[2:6], atol=1e-4) - self.assertAllClose( - np.exp(predictions_probs[2:6] * 1.0 / eps - 0.5 / eps), - neg_loss[2:6], atol=1e-4) - - def test_per_example_squared_loss(self): - - labels = np.array([[0.123], [224.2], [-3], [2], [.3]], dtype=np.float32) - weights = array_ops.ones([5, 1], dtypes.float32) - predictions = np.array( - [[0.123], [23.2], [233], [52], [3]], dtype=np.float32) - - with self.cached_session(): - loss_tensor, _ = losses.per_example_squared_loss(labels, weights, - predictions) - - loss = loss_tensor.eval() - self.assertAllClose( - np.square(labels[:5] - predictions[:5]), loss[:5], atol=1e-4) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/boosted_trees/resources/BUILD b/tensorflow/contrib/boosted_trees/resources/BUILD deleted file mode 100644 index 1205ce55694..00000000000 --- a/tensorflow/contrib/boosted_trees/resources/BUILD +++ /dev/null @@ -1,39 +0,0 @@ -package( - default_visibility = [ - "//tensorflow/contrib/boosted_trees:__subpackages__", - "//tensorflow/contrib/boosted_trees:friends", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -cc_library( - name = "stamped_resource", - hdrs = ["stamped_resource.h"], - deps = [ - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - ], -) - -cc_library( - name = "quantile_stream_resource", - hdrs = ["quantile_stream_resource.h"], - deps = [ - ":stamped_resource", - "//tensorflow/contrib/boosted_trees/lib:weighted_quantiles", - "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_cc", - "//tensorflow/core:framework_headers_lib", - ], -) - -cc_library( - name = "decision_tree_ensemble_resource", - hdrs = ["decision_tree_ensemble_resource.h"], - deps = [ - ":stamped_resource", - "//tensorflow/contrib/boosted_trees/lib:trees", - "//tensorflow/core:framework_headers_lib", - ], -) diff --git a/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h b/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h deleted file mode 100644 index 0fe57c0a4e8..00000000000 --- a/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_RESOURCES_DECISION_TREE_ENSEMBLE_RESOURCE_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_RESOURCES_DECISION_TREE_ENSEMBLE_RESOURCE_H_ - -#include "tensorflow/contrib/boosted_trees/lib/trees/decision_tree.h" -#include "tensorflow/contrib/boosted_trees/resources/stamped_resource.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/protobuf.h" - -namespace tensorflow { -namespace boosted_trees { -namespace models { - -// Keep a tree ensemble in memory for efficient evaluation and mutation. -class DecisionTreeEnsembleResource : public StampedResource { - public: - // Constructor. - explicit DecisionTreeEnsembleResource() - : decision_tree_ensemble_( - protobuf::Arena::CreateMessage< - boosted_trees::trees::DecisionTreeEnsembleConfig>(&arena_)) {} - - string DebugString() const override { - return strings::StrCat("GTFlowDecisionTreeEnsemble[size=", - decision_tree_ensemble_->trees_size(), "]"); - } - - const boosted_trees::trees::DecisionTreeEnsembleConfig& - decision_tree_ensemble() const { - return *decision_tree_ensemble_; - } - - int32 num_trees() const { return decision_tree_ensemble_->trees_size(); } - - bool InitFromSerialized(const string& serialized, const int64 stamp_token) { - CHECK_EQ(stamp(), -1) << "Must Reset before Init."; - if (ParseProtoUnlimited(decision_tree_ensemble_, serialized)) { - set_stamp(stamp_token); - return true; - } - return false; - } - - string SerializeAsString() const { - return decision_tree_ensemble_->SerializeAsString(); - } - - // Increment num_layers_attempted and num_trees_attempted in growing_metadata - // if the tree is finalized. - void IncrementAttempts() { - boosted_trees::trees::GrowingMetadata* const growing_metadata = - decision_tree_ensemble_->mutable_growing_metadata(); - growing_metadata->set_num_layers_attempted( - growing_metadata->num_layers_attempted() + 1); - const int num_trees = decision_tree_ensemble_->trees_size(); - if (num_trees <= 0 || LastTreeMetadata()->is_finalized()) { - growing_metadata->set_num_trees_attempted( - growing_metadata->num_trees_attempted() + 1); - } - } - - boosted_trees::trees::DecisionTreeConfig* AddNewTree(const float weight) { - // Adding a tree as well as a weight and a tree_metadata. - decision_tree_ensemble_->add_tree_weights(weight); - boosted_trees::trees::DecisionTreeMetadata* const metadata = - decision_tree_ensemble_->add_tree_metadata(); - metadata->set_num_layers_grown(1); - return decision_tree_ensemble_->add_trees(); - } - - void RemoveLastTree() { - QCHECK_GT(decision_tree_ensemble_->trees_size(), 0); - decision_tree_ensemble_->mutable_trees()->RemoveLast(); - decision_tree_ensemble_->mutable_tree_weights()->RemoveLast(); - decision_tree_ensemble_->mutable_tree_metadata()->RemoveLast(); - } - - boosted_trees::trees::DecisionTreeConfig* LastTree() { - const int32 tree_size = decision_tree_ensemble_->trees_size(); - QCHECK_GT(tree_size, 0); - return decision_tree_ensemble_->mutable_trees(tree_size - 1); - } - - boosted_trees::trees::DecisionTreeMetadata* LastTreeMetadata() { - const int32 metadata_size = decision_tree_ensemble_->tree_metadata_size(); - QCHECK_GT(metadata_size, 0); - return decision_tree_ensemble_->mutable_tree_metadata(metadata_size - 1); - } - - // Retrieves tree weights and returns as a vector. - std::vector GetTreeWeights() const { - return {decision_tree_ensemble_->tree_weights().begin(), - decision_tree_ensemble_->tree_weights().end()}; - } - - float GetTreeWeight(const int32 index) const { - return decision_tree_ensemble_->tree_weights(index); - } - - void MaybeAddUsedHandler(const int32 handler_id) { - protobuf::RepeatedField* used_ids = - decision_tree_ensemble_->mutable_growing_metadata() - ->mutable_used_handler_ids(); - protobuf::RepeatedField::iterator first = - std::lower_bound(used_ids->begin(), used_ids->end(), handler_id); - if (first == used_ids->end()) { - used_ids->Add(handler_id); - return; - } - if (handler_id == *first) { - // It is a duplicate entry. - return; - } - used_ids->Add(handler_id); - // Keep the list of used handlers sorted. - std::sort(used_ids->begin(), used_ids->end()); - } - - std::vector GetUsedHandlers() const { - std::vector result; - result.reserve( - decision_tree_ensemble_->growing_metadata().used_handler_ids().size()); - for (int64 h : - decision_tree_ensemble_->growing_metadata().used_handler_ids()) { - result.push_back(h); - } - return result; - } - - // Sets the weight of i'th tree, and increment num_updates in tree_metadata. - void SetTreeWeight(const int32 index, const float weight, - const int32 increment_num_updates) { - QCHECK_GE(index, 0); - QCHECK_LT(index, num_trees()); - decision_tree_ensemble_->set_tree_weights(index, weight); - if (increment_num_updates != 0) { - const int32 num_updates = decision_tree_ensemble_->tree_metadata(index) - .num_tree_weight_updates(); - decision_tree_ensemble_->mutable_tree_metadata(index) - ->set_num_tree_weight_updates(num_updates + increment_num_updates); - } - } - - // Resets the resource and frees the protos in arena. - // Caller needs to hold the mutex lock while calling this. - virtual void Reset() { - // Reset stamp. - set_stamp(-1); - - // Clear tree ensemle. - arena_.Reset(); - CHECK_EQ(0, arena_.SpaceAllocated()); - decision_tree_ensemble_ = protobuf::Arena::CreateMessage< - boosted_trees::trees::DecisionTreeEnsembleConfig>(&arena_); - } - - mutex* get_mutex() { return &mu_; } - - protected: - protobuf::Arena arena_; - mutex mu_; - boosted_trees::trees::DecisionTreeEnsembleConfig* decision_tree_ensemble_; -}; - -} // namespace models -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_RESOURCES_DECISION_TREE_ENSEMBLE_RESOURCE_H_ diff --git a/tensorflow/contrib/boosted_trees/resources/quantile_stream_resource.h b/tensorflow/contrib/boosted_trees/resources/quantile_stream_resource.h deleted file mode 100644 index 574e3065e7f..00000000000 --- a/tensorflow/contrib/boosted_trees/resources/quantile_stream_resource.h +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_RESOURCES_QUANTILE_STREAM_RESOURCE_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_RESOURCES_QUANTILE_STREAM_RESOURCE_H_ - -#include "tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_stream.h" -#include "tensorflow/contrib/boosted_trees/proto/quantiles.pb.h" // NOLINT -#include "tensorflow/contrib/boosted_trees/resources/stamped_resource.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/platform/mutex.h" - -namespace tensorflow { -namespace boosted_trees { - -using QuantileStream = - boosted_trees::quantiles::WeightedQuantilesStream; - -// Resource for accumulating summaries for multiple columns. -class QuantileStreamResource : public StampedResource { - public: - QuantileStreamResource(const float epsilon, const int32 num_quantiles, - const int64 max_elements, bool generate_quantiles, - int64 stamp_token) - : stream_(epsilon, max_elements), - are_buckets_ready_(false), - epsilon_(epsilon), - num_quantiles_(num_quantiles), - max_elements_(max_elements), - generate_quantiles_(generate_quantiles) { - set_stamp(stamp_token); - } - - string DebugString() const override { return "QuantileStreamResource"; } - - tensorflow::mutex* mutex() { return &mu_; } - - QuantileStream* stream(int64 stamp) { - CHECK(is_stamp_valid(stamp)); - return &stream_; - } - - const std::vector& boundaries(int64 stamp) { - CHECK(is_stamp_valid(stamp)); - return boundaries_; - } - - void set_boundaries(int64 stamp, const std::vector& boundaries) { - CHECK(is_stamp_valid(stamp)); - are_buckets_ready_ = true; - boundaries_ = boundaries; - } - - float epsilon() const { return epsilon_; } - int32 num_quantiles() const { return num_quantiles_; } - - void Reset(int64 stamp) { - set_stamp(stamp); - stream_ = QuantileStream(epsilon_, max_elements_); - } - - bool are_buckets_ready() const { return are_buckets_ready_; } - void set_buckets_ready(bool are_buckets_ready) { - are_buckets_ready_ = are_buckets_ready; - } - - bool generate_quantiles() const { return generate_quantiles_; } - void set_generate_quantiles(bool generate_quantiles) { - generate_quantiles_ = generate_quantiles; - } - - private: - ~QuantileStreamResource() override {} - - // Mutex for the whole resource. - tensorflow::mutex mu_; - - // Quantile stream. - QuantileStream stream_; - - // Stores the boundaries from the previous iteration. Empty during the first - // iteration. - std::vector boundaries_; - - // Whether boundaries are created. Initially boundaries are empty until - // set_boundaries are called. - bool are_buckets_ready_; - - const float epsilon_; - const int32 num_quantiles_; - // An upper-bound for the number of elements. - int64 max_elements_; - - // Generate quantiles instead of approximate boundaries. - // If true, exactly `num_quantiles` will be produced in the final summary. - bool generate_quantiles_; - - TF_DISALLOW_COPY_AND_ASSIGN(QuantileStreamResource); -}; - -} // namespace boosted_trees -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_RESOURCES_QUANTILE_STREAM_RESOURCE_H_ diff --git a/tensorflow/contrib/boosted_trees/resources/stamped_resource.h b/tensorflow/contrib/boosted_trees/resources/stamped_resource.h deleted file mode 100644 index 957bbe8d61d..00000000000 --- a/tensorflow/contrib/boosted_trees/resources/stamped_resource.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_BOOSTED_TREES_RESOURCES_STAMPED_RESOURCE_H_ -#define TENSORFLOW_CONTRIB_BOOSTED_TREES_RESOURCES_STAMPED_RESOURCE_H_ - -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/platform/mutex.h" - -namespace tensorflow { -namespace boosted_trees { - -// A StampedResource is a resource that has a stamp token associated with it. -// Before reading from or applying updates to the resource, the stamp should -// be checked to verify that the update is not stale. -class StampedResource : public ResourceBase { - public: - StampedResource() : stamp_(-1) {} - - bool is_stamp_valid(int64 stamp) const { return stamp_ == stamp; } - - int64 stamp() const { return stamp_; } - void set_stamp(int64 stamp) { stamp_ = stamp; } - - private: - int64 stamp_; -}; - -} // namespace boosted_trees -} // namespace tensorflow -#endif // TENSORFLOW_CONTRIB_BOOSTED_TREES_RESOURCES_STAMPED_RESOURCE_H_ diff --git a/tensorflow/contrib/checkpoint/README.md b/tensorflow/contrib/checkpoint/README.md deleted file mode 100644 index d35c5bae3b7..00000000000 --- a/tensorflow/contrib/checkpoint/README.md +++ /dev/null @@ -1,2 +0,0 @@ -Tools for working with object-based checkpoints produced by -`tf.train.Checkpoint`. diff --git a/tensorflow/contrib/checkpoint/__init__.py b/tensorflow/contrib/checkpoint/__init__.py deleted file mode 100644 index a416588691f..00000000000 --- a/tensorflow/contrib/checkpoint/__init__.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tools for working with object-based checkpoints. - -Visualization and inspection: -@@dot_graph_from_checkpoint -@@list_objects -@@object_metadata - -Managing dependencies: -@@capture_dependencies -@@Checkpointable -@@CheckpointableBase -@@CheckpointableObjectGraph -@@NoDependency -@@split_dependency - -Trackable data structures: -@@List -@@Mapping -@@UniqueNameTracker - -Checkpoint management: -@@CheckpointManager - -Saving and restoring Python state: -@@NumpyState -@@PythonStateWrapper -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.checkpoint.python.containers import UniqueNameTracker -from tensorflow.contrib.checkpoint.python.python_state import NumpyState -from tensorflow.contrib.checkpoint.python.split_dependency import split_dependency -from tensorflow.contrib.checkpoint.python.visualize import dot_graph_from_checkpoint -from tensorflow.core.protobuf.trackable_object_graph_pb2 import TrackableObjectGraph as CheckpointableObjectGraph -from tensorflow.python.training.checkpoint_management import CheckpointManager -from tensorflow.python.training.tracking.base import Trackable as CheckpointableBase -from tensorflow.python.training.tracking.data_structures import List -from tensorflow.python.training.tracking.data_structures import Mapping -from tensorflow.python.training.tracking.data_structures import NoDependency -from tensorflow.python.training.tracking.python_state import PythonState as PythonStateWrapper -from tensorflow.python.training.tracking.tracking import AutoTrackable as Checkpointable -from tensorflow.python.training.tracking.util import capture_dependencies -from tensorflow.python.training.tracking.util import list_objects -from tensorflow.python.training.tracking.util import object_metadata -from tensorflow.python.util.all_util import remove_undocumented - -remove_undocumented(module_name=__name__) - diff --git a/tensorflow/contrib/checkpoint/python/BUILD b/tensorflow/contrib/checkpoint/python/BUILD deleted file mode 100644 index 24b8ae5447f..00000000000 --- a/tensorflow/contrib/checkpoint/python/BUILD +++ /dev/null @@ -1,124 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "tf_py_test") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "checkpoint", - srcs_version = "PY2AND3", - deps = [ - ":containers", - ":python_state", - ":split_dependency", - ":visualize", - "//tensorflow/python/training/tracking:data_structures", - ], -) - -py_library( - name = "containers", - srcs = ["containers.py"], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python/training/tracking:base", - "//tensorflow/python/training/tracking:data_structures", - ], -) - -tf_py_test( - name = "containers_test", - srcs = ["containers_test.py"], - additional_deps = [ - ":containers", - "@six_archive//:six", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python/training/tracking:base", - "//tensorflow/python/training/tracking:util", - ], -) - -py_library( - name = "python_state", - srcs = ["python_state.py"], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python/training/tracking:base", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -tf_py_test( - name = "python_state_test", - srcs = ["python_state_test.py"], - additional_deps = [ - ":python_state", - "//third_party/py/numpy", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:session", - "//tensorflow/python:variables", - "//tensorflow/python/eager:test", - "//tensorflow/python/training/tracking:util", - ], -) - -py_library( - name = "split_dependency", - srcs = ["split_dependency.py"], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:training", - "//tensorflow/python/training/tracking:base", - ], -) - -tf_py_test( - name = "split_dependency_test", - srcs = ["split_dependency_test.py"], - additional_deps = [ - ":split_dependency", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python/eager:test", - "//tensorflow/python/training/tracking:base", - "//tensorflow/python/training/tracking:util", - ], -) - -py_library( - name = "visualize", - srcs = ["visualize.py"], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python:pywrap_tensorflow", - "//tensorflow/python/training/tracking:base", - "//tensorflow/python/training/tracking:util", - ], -) - -tf_py_test( - name = "visualize_test", - srcs = ["visualize_test.py"], - additional_deps = [ - ":visualize", - "//tensorflow/python:constant_op", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:training", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:test", - "//tensorflow/python/keras:engine", - "//tensorflow/python/keras:layers", - "//tensorflow/python/training/tracking:util", - ], -) diff --git a/tensorflow/contrib/checkpoint/python/containers.py b/tensorflow/contrib/checkpoint/python/containers.py deleted file mode 100644 index a25d51980ea..00000000000 --- a/tensorflow/contrib/checkpoint/python/containers.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Trackable data structures.""" -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.training.tracking import base as trackable_lib -from tensorflow.python.training.tracking import data_structures - - -class UniqueNameTracker(data_structures.TrackableDataStructure): - """Adds dependencies on trackable objects with name hints. - - Useful for creating dependencies with locally unique names. - - Example usage: - ```python - class SlotManager(tf.contrib.checkpoint.Checkpointable): - - def __init__(self): - # Create a dependency named "slotdeps" on the container. - self.slotdeps = tf.contrib.checkpoint.UniqueNameTracker() - slotdeps = self.slotdeps - slots = [] - slots.append(slotdeps.track(tf.Variable(3.), "x")) # Named "x" - slots.append(slotdeps.track(tf.Variable(4.), "y")) - slots.append(slotdeps.track(tf.Variable(5.), "x")) # Named "x_1" - ``` - """ - - def __init__(self): - super(UniqueNameTracker, self).__init__() - self._maybe_initialize_trackable() - self._name_counts = {} - - @property - def _values(self): - return [dep.ref for dep in self._checkpoint_dependencies] - - def track(self, trackable, base_name): - """Add a dependency on `trackable`. - - Args: - trackable: An object to add a checkpoint dependency on. - base_name: A name hint, which is uniquified to determine the dependency - name. - Returns: - `trackable`, for chaining. - Raises: - ValueError: If `trackable` is not a trackable object. - """ - - if not isinstance(trackable, trackable_lib.Trackable): - raise ValueError( - ("Expected a trackable value, got %s which does not inherit " - "from tf.track.Trackable.") % (trackable,)) - - def _format_name(prefix, number): - if number > 0: - return "%s_%d" % (prefix, number) - else: - return prefix - - count = self._name_counts.get(base_name, 0) - candidate = _format_name(base_name, count) - while self._lookup_dependency(candidate) is not None: - count += 1 - candidate = _format_name(base_name, count) - self._name_counts[base_name] = count + 1 - self._track_value(trackable, name=candidate) - return trackable diff --git a/tensorflow/contrib/checkpoint/python/containers_test.py b/tensorflow/contrib/checkpoint/python/containers_test.py deleted file mode 100644 index bace2193960..00000000000 --- a/tensorflow/contrib/checkpoint/python/containers_test.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -import six - -from tensorflow.contrib.checkpoint.python import containers -from tensorflow.python.framework import test_util -from tensorflow.python.keras import layers -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.platform import test -from tensorflow.python.training.tracking import data_structures -from tensorflow.python.training.tracking import tracking -from tensorflow.python.training.tracking import util - - -class UniqueNameTrackerTests(test.TestCase): - - @test_util.run_in_graph_and_eager_modes - def testNames(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - - x1 = resource_variable_ops.ResourceVariable(2.) - x2 = resource_variable_ops.ResourceVariable(3.) - x3 = resource_variable_ops.ResourceVariable(4.) - y = resource_variable_ops.ResourceVariable(5.) - slots = containers.UniqueNameTracker() - slots.track(x1, "x") - slots.track(x2, "x") - slots.track(x3, "x_1") - slots.track(y, "y") - self.evaluate((x1.initializer, x2.initializer, x3.initializer, - y.initializer)) - save_root = util.Checkpoint(slots=slots) - save_path = save_root.save(checkpoint_prefix) - - restore_slots = tracking.AutoTrackable() - restore_root = util.Checkpoint( - slots=restore_slots) - status = restore_root.restore(save_path) - restore_slots.x = resource_variable_ops.ResourceVariable(0.) - restore_slots.x_1 = resource_variable_ops.ResourceVariable(0.) - restore_slots.x_1_1 = resource_variable_ops.ResourceVariable(0.) - restore_slots.y = resource_variable_ops.ResourceVariable(0.) - status.assert_consumed().run_restore_ops() - self.assertEqual(2., self.evaluate(restore_slots.x)) - self.assertEqual(3., self.evaluate(restore_slots.x_1)) - self.assertEqual(4., self.evaluate(restore_slots.x_1_1)) - self.assertEqual(5., self.evaluate(restore_slots.y)) - - @test_util.run_in_graph_and_eager_modes - def testExample(self): - class SlotManager(tracking.AutoTrackable): - - def __init__(self): - self.slotdeps = containers.UniqueNameTracker() - slotdeps = self.slotdeps - slots = [] - slots.append(slotdeps.track( - resource_variable_ops.ResourceVariable(3.), "x")) - slots.append(slotdeps.track( - resource_variable_ops.ResourceVariable(4.), "y")) - slots.append(slotdeps.track( - resource_variable_ops.ResourceVariable(5.), "x")) - self.slots = data_structures.NoDependency(slots) - - manager = SlotManager() - self.evaluate([v.initializer for v in manager.slots]) - checkpoint = util.Checkpoint(slot_manager=manager) - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - save_path = checkpoint.save(checkpoint_prefix) - metadata = util.object_metadata(save_path) - dependency_names = [] - for node in metadata.nodes: - for child in node.children: - dependency_names.append(child.local_name) - six.assertCountEqual( - self, - dependency_names, - ["x", "x_1", "y", "slot_manager", "slotdeps", "save_counter"]) - - @test_util.run_in_graph_and_eager_modes - def testLayers(self): - tracker = containers.UniqueNameTracker() - tracker.track(layers.Dense(3), "dense") - tracker.layers[0](array_ops.zeros([1, 1])) - self.assertEqual(2, len(tracker.trainable_weights)) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/checkpoint/python/python_state.py b/tensorflow/contrib/checkpoint/python/python_state.py deleted file mode 100644 index e9618b972d9..00000000000 --- a/tensorflow/contrib/checkpoint/python/python_state.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Utilities for including Python state in TensorFlow checkpoints.""" -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy - -from tensorflow.python.training.tracking import base -from tensorflow.python.training.tracking import python_state as core_python_state - -# pylint: disable=g-import-not-at-top -try: - # In Python 2.x, use the faster string buffering option. - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO -# pylint: enable=g-import-not-at-top - - -class NumpyState(base.Trackable): - """A trackable object whose NumPy array attributes are saved/restored. - - Example usage: - - ```python - arrays = tf.contrib.checkpoint.NumpyState() - checkpoint = tf.train.Checkpoint(numpy_arrays=arrays) - arrays.x = numpy.zeros([3, 4]) - save_path = checkpoint.save("/tmp/ckpt") - arrays.x[1, 1] = 4. - checkpoint.restore(save_path) - assert (arrays.x == numpy.zeros([3, 4])).all() - - second_checkpoint = tf.train.Checkpoint( - numpy_arrays=tf.contrib.checkpoint.NumpyState()) - # Attributes of NumpyState objects are created automatically by restore() - second_checkpoint.restore(save_path) - assert (second_checkpoint.numpy_arrays.x == numpy.zeros([3, 4])).all() - ``` - - Note that `NumpyState` objects re-create the attributes of the previously - saved object on `restore()`. This is in contrast to TensorFlow variables, for - which a `Variable` object must be created and assigned to an attribute. - - This snippet works both when graph building and when executing eagerly. On - save, the NumPy array(s) are fed as strings to be saved in the checkpoint (via - a placeholder when graph building, or as a string constant when executing - eagerly). When restoring they skip the TensorFlow graph entirely, and so no - restore ops need be run. This means that restoration always happens eagerly, - rather than waiting for `checkpoint.restore(...).run_restore_ops()` like - TensorFlow variables when graph building. - """ - - def _lookup_dependency(self, name): - """Create placeholder NumPy arrays for to-be-restored attributes. - - Typically `_lookup_dependency` is used to check by name whether a dependency - exists. We cheat slightly by creating a trackable object for `name` if - we don't already have one, giving us attribute re-creation behavior when - loading a checkpoint. - - Args: - name: The name of the dependency being checked. - Returns: - An existing dependency if one exists, or a new `_NumpyWrapper` placeholder - dependency (which will generally be restored immediately). - """ - value = super(NumpyState, self)._lookup_dependency(name) - if value is None: - value = _NumpyWrapper(numpy.array([])) - new_reference = base.TrackableReference(name=name, ref=value) - self._unconditional_checkpoint_dependencies.append(new_reference) - self._unconditional_dependency_names[name] = value - super(NumpyState, self).__setattr__(name, value) - return value - - def __getattribute__(self, name): - """Un-wrap `_NumpyWrapper` objects when accessing attributes.""" - value = super(NumpyState, self).__getattribute__(name) - if isinstance(value, _NumpyWrapper): - return value.array - return value - - def __setattr__(self, name, value): - """Automatically wrap NumPy arrays assigned to attributes.""" - # TODO(allenl): Consider supporting lists/tuples, either ad-hoc or by making - # ndarrays trackable natively and using standard trackable list - # tracking. - if isinstance(value, (numpy.ndarray, numpy.generic)): - try: - existing = super(NumpyState, self).__getattribute__(name) - existing.array = value - return - except AttributeError: - value = _NumpyWrapper(value) - self._track_trackable(value, name=name, overwrite=True) - elif (name not in ("_self_setattr_tracking", "_self_update_uid") - and getattr(self, "_self_setattr_tracking", True)): - # Mixing restore()-created attributes with user-added trackable - # objects is tricky, since we can't use the `_lookup_dependency` trick to - # re-create attributes (we might accidentally steal the restoration for - # another trackable object). For now `NumpyState` objects must be - # leaf nodes. Theoretically we could add some extra arguments to - # `_lookup_dependency` to figure out whether we should create a NumPy - # array for the attribute or not. - raise NotImplementedError( - ("Assigned %s to the %s property of %s, which is not a NumPy array. " - "Currently mixing NumPy arrays and other trackable objects is " - "not supported. File a feature request if this limitation bothers " - "you.") - % (value, name, self)) - super(NumpyState, self).__setattr__(name, value) - - -class _NumpyWrapper(core_python_state.PythonState): - """Wraps a NumPy array for storage in an object-based checkpoint.""" - - def __init__(self, array): - """Specify a NumPy array to wrap. - - Args: - array: The NumPy array to save and restore (may be overwritten). - """ - self.array = array - - def serialize(self): - """Callback to serialize the array.""" - string_file = BytesIO() - try: - numpy.save(string_file, self.array, allow_pickle=False) - serialized = string_file.getvalue() - finally: - string_file.close() - return serialized - - def deserialize(self, string_value): - """Callback to deserialize the array.""" - string_file = BytesIO(string_value) - try: - self.array = numpy.load(string_file, allow_pickle=False) - finally: - string_file.close() diff --git a/tensorflow/contrib/checkpoint/python/python_state_test.py b/tensorflow/contrib/checkpoint/python/python_state_test.py deleted file mode 100644 index 40d8fe83640..00000000000 --- a/tensorflow/contrib/checkpoint/python/python_state_test.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -import numpy - -from tensorflow.contrib.checkpoint.python import python_state -from tensorflow.python.client import session -from tensorflow.python.eager import test -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import variables -from tensorflow.python.training.tracking import util - - -class NumpyStateTests(test.TestCase): - - @test_util.run_in_graph_and_eager_modes - def testSaveRestoreNumpyState(self): - directory = self.get_temp_dir() - prefix = os.path.join(directory, "ckpt") - save_state = python_state.NumpyState() - saver = util.Checkpoint(numpy=save_state) - save_state.a = numpy.ones([2, 2]) - save_state.b = numpy.ones([2, 2]) - save_state.b = numpy.zeros([2, 2]) - save_state.c = numpy.int64(3) - self.assertAllEqual(numpy.ones([2, 2]), save_state.a) - self.assertAllEqual(numpy.zeros([2, 2]), save_state.b) - self.assertEqual(3, save_state.c) - first_save_path = saver.save(prefix) - save_state.a[1, 1] = 2. - save_state.c = numpy.int64(4) - second_save_path = saver.save(prefix) - - load_state = python_state.NumpyState() - loader = util.Checkpoint(numpy=load_state) - loader.restore(first_save_path).initialize_or_restore() - self.assertAllEqual(numpy.ones([2, 2]), load_state.a) - self.assertAllEqual(numpy.zeros([2, 2]), load_state.b) - self.assertEqual(3, load_state.c) - load_state.a[0, 0] = 42. - self.assertAllEqual([[42., 1.], [1., 1.]], load_state.a) - loader.restore(first_save_path).run_restore_ops() - self.assertAllEqual(numpy.ones([2, 2]), load_state.a) - loader.restore(second_save_path).run_restore_ops() - self.assertAllEqual([[1., 1.], [1., 2.]], load_state.a) - self.assertAllEqual(numpy.zeros([2, 2]), load_state.b) - self.assertEqual(4, load_state.c) - - def testNoGraphPollution(self): - graph = ops.Graph() - with graph.as_default(), session.Session(): - directory = self.get_temp_dir() - prefix = os.path.join(directory, "ckpt") - save_state = python_state.NumpyState() - saver = util.Checkpoint(numpy=save_state) - save_state.a = numpy.ones([2, 2]) - save_path = saver.save(prefix) - saver.restore(save_path) - graph.finalize() - saver.save(prefix) - save_state.a = numpy.zeros([2, 2]) - saver.save(prefix) - saver.restore(save_path) - - @test_util.run_in_graph_and_eager_modes - def testNoMixedNumpyStateTF(self): - save_state = python_state.NumpyState() - save_state.a = numpy.ones([2, 2]) - with self.assertRaises(NotImplementedError): - save_state.v = variables.Variable(1.) - - @test_util.run_in_graph_and_eager_modes - def testDocstringExample(self): - arrays = python_state.NumpyState() - checkpoint = util.Checkpoint(numpy_arrays=arrays) - arrays.x = numpy.zeros([3, 4]) - save_path = checkpoint.save(os.path.join(self.get_temp_dir(), "ckpt")) - arrays.x[1, 1] = 4. - checkpoint.restore(save_path) - self.assertAllEqual(numpy.zeros([3, 4]), arrays.x) - - second_checkpoint = util.Checkpoint(numpy_arrays=python_state.NumpyState()) - second_checkpoint.restore(save_path) - self.assertAllEqual(numpy.zeros([3, 4]), second_checkpoint.numpy_arrays.x) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/checkpoint/python/split_dependency.py b/tensorflow/contrib/checkpoint/python/split_dependency.py deleted file mode 100644 index aaabe4e3f57..00000000000 --- a/tensorflow/contrib/checkpoint/python/split_dependency.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Utility for creating multiple dependencies with synchronized save/restore.""" -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.training import saver as saver_lib -from tensorflow.python.training.tracking import base as trackable - - -class _CallbackSaveable(saver_lib.BaseSaverBuilder.SaveableObject): - """Wraps save and restore callbacks as a `SaveableObject`.""" - - def __init__(self, name, dtype, device, save_callback, restore_callback): - self._restore_callback = restore_callback - spec = saver_lib.BaseSaverBuilder.SaveSpec( - tensor=save_callback, - slice_spec="", - name=name, - dtype=dtype, - device=device) - super(_CallbackSaveable, self).__init__( - save_callback, [spec], name) - - def restore(self, restored_tensors, restored_shapes): - """Restore the same value into both variables.""" - tensor, = restored_tensors - return self._restore_callback(tensor) - - -class _SplitDependency(trackable.Trackable): - """Looks like a regular variable while synchronizing save/restores.""" - - def __init__(self, save_buffer, restore_buffer, name, dtype, device, - num_components, fill_save_buffer_fn, consume_restore_buffer_fn): - self._save_buffer = save_buffer - self._restore_buffer = restore_buffer - self._name = name - self._dtype = dtype - self._device = device - self._num_components = num_components - self._fill_save_buffer_fn = fill_save_buffer_fn - self._consume_restore_buffer_fn = consume_restore_buffer_fn - - def _save(self): - """Pull from the shared buffer, populating it if necessary.""" - if self._name not in self._save_buffer: - if self._save_buffer: - raise AssertionError( - ("Split dependency %s (%s) unsynchronized. Split dependencies must " - "be saved together.") % (self._name, self)) - self._fill_save_buffer_fn(self._save_buffer) - return self._save_buffer.pop(self._name) - - def _restore(self, tensor): - """Push into the shared buffer, flushing it if necessary.""" - if self._name in self._restore_buffer: - raise AssertionError( - ("Split dependency %s (%s) unsynchronized. Split dependencies must " - "be restored together.") % (self._name, self)) - self._restore_buffer[self._name] = tensor - if len(self._restore_buffer) == self._num_components: - op = self._consume_restore_buffer_fn(self._restore_buffer) - self._restore_buffer.clear() - return op - else: - return control_flow_ops.no_op() - - def _gather_saveables_for_checkpoint(self): - """Looks to Trackable like a regular variable.""" - return { - trackable.VARIABLE_VALUE_KEY: - functools.partial(_CallbackSaveable, - dtype=self._dtype, - device=self._device, - save_callback=self._save, - restore_callback=self._restore) - } - - -def split_dependency(component_names, component_dtypes, - fill_save_buffer_fn, consume_restore_buffer_fn, - device): - """Creates multiple dependencies with a synchronized save/restore. - - Useful when a single op produces `Tensor`s which should each be saved under - different objects, or when `Tensor`s saved with many different objects need to - be restored together as inputs to a single op (i.e. an object which uses a - single fused op may be swapped out for a subgraph of objects, and these two - programs are checkpoint compatible). - - Args: - component_names: A sequence of names for the split - dependencies. `fill_save_buffer_fn` must add these keys to the dictionary - it is passed, and `consume_restore_buffer_fn` will receive a dictionary - with these keys. - component_dtypes: Data types for the `Tensor`s being saved and restored, a - sequence corresponding to `component_names`. - fill_save_buffer_fn: A function which takes an empty dictionary as an - argument and adds `Tensor`s with `component_names` as keys. These - `Tensor`s will be saved as if they were individual variables. - consume_restore_buffer_fn: A function which takes a dictionary with - `component_names` as keys mapping to restored individual `Tensor`s and - returns a restore op (or if executing eagerly, runs the restoration and - may return `None`). - device: The device on which to run save and restore operations. - - Returns: - A dictionary mapping from names to Trackable objects. If one is - reachable from an object as a dependency, the others should be too; adding - dependencies on some but not all of the objects will result in errors. - """ - save_buffer = {} - restore_buffer = {} - split_dependencies = {} - for name, dtype in zip(component_names, component_dtypes): - split_dependencies[name] = _SplitDependency( - save_buffer=save_buffer, - restore_buffer=restore_buffer, - name=name, - dtype=dtype, - device=device, - num_components=len(component_names), - fill_save_buffer_fn=fill_save_buffer_fn, - consume_restore_buffer_fn=consume_restore_buffer_fn) - return split_dependencies diff --git a/tensorflow/contrib/checkpoint/python/split_dependency_test.py b/tensorflow/contrib/checkpoint/python/split_dependency_test.py deleted file mode 100644 index 8660cc12f28..00000000000 --- a/tensorflow/contrib/checkpoint/python/split_dependency_test.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from tensorflow.contrib.checkpoint.python import split_dependency -from tensorflow.python.eager import test -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.training.tracking import base -from tensorflow.python.training.tracking import tracking -from tensorflow.python.training.tracking import util - - -def _split_variable_closure(variable): - def _fill_save_buffer_fn(save_buffer): - save_buffer["first_half"] = variable[:2] - save_buffer["second_half"] = variable[2:] - return _fill_save_buffer_fn - - -def _combine_variable_closure(variable): - def _consume_restore_buffer_fn(restore_buffer): - return variable.assign( - array_ops.concat([restore_buffer["first_half"], - restore_buffer["second_half"]], - axis=0)) - return _consume_restore_buffer_fn - - -class SaveTensorSlicesAsDeps(base.Trackable): - - def __init__(self): - self.combined = resource_variable_ops.ResourceVariable([0., 0., 0., 0.]) - split_dependencies = split_dependency.split_dependency( - component_names=("first_half", "second_half"), - component_dtypes=(self.combined.dtype,) * 2, - fill_save_buffer_fn=_split_variable_closure( - self.combined), - consume_restore_buffer_fn=_combine_variable_closure( - self.combined), - device=self.combined.device) - for name, dep in split_dependencies.items(): - self._track_trackable(dep, name=name) - - -class HasRegularDeps(tracking.AutoTrackable): - - def __init__(self): - self.first_half = resource_variable_ops.ResourceVariable([0., 0.]) - self.second_half = resource_variable_ops.ResourceVariable([0., 0.]) - - -class OnlyOneDep(tracking.AutoTrackable): - - def __init__(self): - self.first_half = resource_variable_ops.ResourceVariable([0., 0.]) - - -class SplitTests(test.TestCase): - - @test_util.run_in_graph_and_eager_modes - def testSaveRestoreSplitDep(self): - save_checkpoint = util.Checkpoint( - dep=SaveTensorSlicesAsDeps()) - self.evaluate(save_checkpoint.dep.combined.assign([1., 2., 3., 4.])) - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - save_path = save_checkpoint.save(checkpoint_prefix) - - regular_deps = HasRegularDeps() - regular_restore_checkpoint = util.Checkpoint( - dep=regular_deps) - regular_restore_checkpoint.restore( - save_path).assert_consumed().run_restore_ops() - self.assertAllEqual([1., 2.], self.evaluate(regular_deps.first_half)) - self.assertAllEqual([3., 4.], self.evaluate(regular_deps.second_half)) - - one_dep = OnlyOneDep() - one_dep_restore_checkpoint = util.Checkpoint(dep=one_dep) - status = one_dep_restore_checkpoint.restore(save_path) - with self.assertRaises(AssertionError): - # Missing the second dependency. - status.assert_consumed() - status.run_restore_ops() - self.assertAllEqual([1., 2.], self.evaluate(one_dep.first_half)) - - restore_checkpoint = util.Checkpoint() - status = restore_checkpoint.restore(save_path) - restore_checkpoint.dep = SaveTensorSlicesAsDeps() - status.assert_consumed().run_restore_ops() - self.assertAllEqual( - [1., 2., 3., 4.], - self.evaluate(restore_checkpoint.dep.combined)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/checkpoint/python/visualize.py b/tensorflow/contrib/checkpoint/python/visualize.py deleted file mode 100644 index faf90f01847..00000000000 --- a/tensorflow/contrib/checkpoint/python/visualize.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Utilities for visualizing dependency graphs.""" -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python import pywrap_tensorflow -from tensorflow.python.training.tracking import base as trackable -from tensorflow.python.training.tracking import util as trackable_utils - - -def dot_graph_from_checkpoint(save_path): - r"""Visualizes an object-based checkpoint (from `tf.train.Checkpoint`). - - Useful for inspecting checkpoints and debugging loading issues. - - Example usage from Python (requires pydot): - ```python - import tensorflow as tf - import pydot - - dot_string = tf.contrib.checkpoint.dot_graph_from_checkpoint('/path/to/ckpt') - parsed, = pydot.graph_from_dot_data(dot_string) - parsed.write_svg('/tmp/tensorflow/visualized_checkpoint.svg') - ``` - - Example command line usage: - ```sh - python -c "import tensorflow as tf;\ - print(tf.contrib.checkpoint.dot_graph_from_checkpoint('/path/to/ckpt'))"\ - | dot -Tsvg > /tmp/tensorflow/checkpoint_viz.svg - ``` - - Args: - save_path: The checkpoint prefix, as returned by `tf.train.Checkpoint.save` - or `tf.train.latest_checkpoint`. - Returns: - A graph in DOT format as a string. - """ - reader = pywrap_tensorflow.NewCheckpointReader(save_path) - object_graph = trackable_utils.object_metadata(save_path) - shape_map = reader.get_variable_to_shape_map() - dtype_map = reader.get_variable_to_dtype_map() - graph = 'digraph {\n' - def _escape(name): - return name.replace('"', '\\"') - slot_ids = set() - for node in object_graph.nodes: - for slot_reference in node.slot_variables: - slot_ids.add(slot_reference.slot_variable_node_id) - for node_id, node in enumerate(object_graph.nodes): - if (len(node.attributes) == 1 - and node.attributes[0].name == trackable.VARIABLE_VALUE_KEY): - if node_id in slot_ids: - color = 'orange' - tooltip_prefix = 'Slot variable' - else: - color = 'blue' - tooltip_prefix = 'Variable' - attribute = node.attributes[0] - graph += ('N_%d [shape=point label="" color=%s width=.25' - ' tooltip="%s %s shape=%s %s"]\n') % ( - node_id, - color, - tooltip_prefix, - _escape(attribute.full_name), - shape_map[attribute.checkpoint_key], - dtype_map[attribute.checkpoint_key].name) - elif node.slot_variables: - graph += ('N_%d [shape=point label="" width=.25 color=red,' - 'tooltip="Optimizer"]\n') % node_id - else: - graph += 'N_%d [shape=point label="" width=.25]\n' % node_id - for reference in node.children: - graph += 'N_%d -> N_%d [label="%s"]\n' % ( - node_id, reference.node_id, _escape(reference.local_name)) - for slot_reference in node.slot_variables: - graph += 'N_%d -> N_%d [label="%s" style=dotted]\n' % ( - node_id, - slot_reference.slot_variable_node_id, - _escape(slot_reference.slot_name)) - graph += 'N_%d -> N_%d [style=dotted]\n' % ( - slot_reference.original_variable_node_id, - slot_reference.slot_variable_node_id) - graph += '}\n' - return graph diff --git a/tensorflow/contrib/checkpoint/python/visualize_test.py b/tensorflow/contrib/checkpoint/python/visualize_test.py deleted file mode 100644 index 98a22d573fd..00000000000 --- a/tensorflow/contrib/checkpoint/python/visualize_test.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import os - -from tensorflow.contrib.checkpoint.python import visualize - -from tensorflow.python.eager import context -from tensorflow.python.eager import test -from tensorflow.python.framework import constant_op -from tensorflow.python.keras.engine import training -from tensorflow.python.keras.layers import core -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.training import adam -from tensorflow.python.training.tracking import util as trackable_utils - -try: - import pydot # pylint: disable=g-import-not-at-top -except ImportError: - pydot = None - - -class MyModel(training.Model): - """A concrete Model for testing.""" - - def __init__(self): - super(MyModel, self).__init__() - self._named_dense = core.Dense(1, use_bias=True) - self._second = core.Dense(1, use_bias=False) - - def call(self, values): - ret = self._second(self._named_dense(values)) - return ret - - -class DotGraphTests(test.TestCase): - - def testMakeDotGraph(self): - with context.eager_mode(): - input_value = constant_op.constant([[3.]]) - model = MyModel() - optimizer = adam.AdamOptimizer(0.001) - optimizer_step = resource_variable_ops.ResourceVariable(12) - save_checkpoint = trackable_utils.Checkpoint( - optimizer=optimizer, model=model, optimizer_step=optimizer_step) - optimizer.minimize(functools.partial(model, input_value)) - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt') - save_path = save_checkpoint.save(checkpoint_prefix) - prefix = save_checkpoint.save(save_path) - - dot_graph_string = visualize.dot_graph_from_checkpoint(prefix) - - # The remainder of this test is more-or-less optional since it's so - # dependent on pydot/platform/Python versions. - if pydot is None: - self.skipTest('pydot is required for the remainder of this test.') - try: - parsed, = pydot.graph_from_dot_data(dot_graph_string) - except NameError as e: - if "name 'dot_parser' is not defined" in str(e): - self.skipTest("pydot isn't working") - else: - raise - # Check that the graph isn't completely trivial - self.assertEqual( - '"model"', - parsed.obj_dict['edges'][('N_0', 'N_1')][0]['attributes']['label']) - image_path = os.path.join(self.get_temp_dir(), 'saved.svg') - try: - parsed.write_svg(image_path) - except Exception as e: # pylint: disable=broad-except - # For some reason PyDot's "dot not available" error is an Exception, not - # something more specific. - if '"dot" not found in path' in str(e): - self.skipTest("pydot won't save SVGs (dot not available)") - else: - raise - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/cloud/BUILD b/tensorflow/contrib/cloud/BUILD deleted file mode 100644 index b64df500ea8..00000000000 --- a/tensorflow/contrib/cloud/BUILD +++ /dev/null @@ -1,85 +0,0 @@ -# Description: -# BigQueryReader implementation - -load( - "//tensorflow:tensorflow.bzl", - "tf_gen_op_libs", - "tf_gen_op_wrapper_py", - "tf_py_test", -) - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -tf_gen_op_libs( - op_lib_names = [ - "bigquery_reader_ops", - "gcs_config_ops", - ], - deps = [ - "//tensorflow/core:lib", - ], -) - -tf_gen_op_wrapper_py( - name = "gen_bigquery_reader_ops", - out = "python/ops/gen_bigquery_reader_ops.py", - deps = [":bigquery_reader_ops_op_lib"], -) - -tf_gen_op_wrapper_py( - name = "gen_gcs_config_ops", - out = "python/ops/gen_gcs_config_ops.py", - visibility = ["//tensorflow:internal"], - deps = [":gcs_config_ops_op_lib"], -) - -py_library( - name = "cloud_py", - srcs = [ - "__init__.py", - "python/ops/bigquery_reader_ops.py", - "python/ops/gcs_config_ops.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":gen_bigquery_reader_ops", - ":gen_gcs_config_ops", - "//tensorflow/contrib/bigtable", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:io_ops", - "//tensorflow/python:util", - ], -) - -tf_py_test( - name = "bigquery_reader_ops_test", - size = "small", - srcs = ["python/ops/bigquery_reader_ops_test.py"], - additional_deps = [ - ":cloud_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:io_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:platform", - "//tensorflow/python:util", - ], - tags = ["manual"], -) - -tf_py_test( - name = "gcs_config_ops_test", - size = "small", - srcs = ["python/ops/gcs_config_ops_test.py"], - additional_deps = [ - ":cloud_py", - "//tensorflow/python:client_testlib", - ], - tags = ["manual"], -) diff --git a/tensorflow/contrib/cloud/README.md b/tensorflow/contrib/cloud/README.md deleted file mode 100644 index a80d8965f3b..00000000000 --- a/tensorflow/contrib/cloud/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Cloud # - -## Cloud Bigtable ## - -[Google Cloud Bigtable](https://cloud.google.com/bigtable/) is a high -performance storage system that can store and serve training data. This contrib -package contains an experimental integration with TensorFlow. - -> **Status: Highly experimental.** The current implementation is very much in -> flux. Please use at your own risk! :-) - - - -## Cloud Storage (GCS) ## - -The Google Cloud Storage ops allow the user to configure the GCS File System. - - diff --git a/tensorflow/contrib/cloud/__init__.py b/tensorflow/contrib/cloud/__init__.py deleted file mode 100644 index 8efd259946b..00000000000 --- a/tensorflow/contrib/cloud/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Module for cloud ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -# pylint: disable=line-too-long,wildcard-import,g-import-not-at-top -from tensorflow.contrib.cloud.python.ops.bigquery_reader_ops import * -from tensorflow.contrib.cloud.python.ops.gcs_config_ops import * - -if os.name != 'nt': - from tensorflow.contrib.bigtable.python.ops.bigtable_api import BigtableClient - from tensorflow.contrib.bigtable.python.ops.bigtable_api import BigtableTable - -del os - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'BigQueryReader', - 'BigtableClient', - 'BigtableTable', - 'BlockCacheParams', - 'configure_colab_session', - 'configure_gcs', - 'ConfigureGcsHook', -] -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/cloud/kernels/BUILD b/tensorflow/contrib/cloud/kernels/BUILD deleted file mode 100644 index d7bbbc10a17..00000000000 --- a/tensorflow/contrib/cloud/kernels/BUILD +++ /dev/null @@ -1,87 +0,0 @@ -# Description: -# BigQueryReader implementation - -load( - "//tensorflow:tensorflow.bzl", - "tf_cc_test", - "tf_copts", - "tf_kernel_library", -) - -# For platform specific build config -load( - "//tensorflow/core/platform:default/build_config.bzl", - "tf_proto_library", -) - -package( - default_visibility = ["//visibility:private"], - licenses = ["notice"], # Apache 2.0 -) - -tf_kernel_library( - name = "bigquery_reader_ops", - srcs = ["bigquery_reader_ops.cc"], - visibility = ["//visibility:public"], - deps = [ - ":bigquery_table_accessor", - ":bigquery_table_partition_proto_cc", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:reader_base", - ], -) - -cc_library( - name = "bigquery_table_accessor", - srcs = ["bigquery_table_accessor.cc"], - hdrs = ["bigquery_table_accessor.h"], - copts = tf_copts(), - linkstatic = 1, - deps = [ - ":bigquery_table_partition_proto_cc", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core/platform/cloud:curl_http_request", - "//tensorflow/core/platform/cloud:google_auth_provider", - ], - alwayslink = 1, -) - -tf_cc_test( - name = "bigquery_table_accessor_test", - size = "small", - srcs = [ - "bigquery_table_accessor_test.cc", - "bigquery_table_accessor_test_data.h", - ], - deps = [ - ":bigquery_table_accessor", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core/platform/cloud:http_request_fake", - ], -) - -tf_proto_library( - name = "bigquery_table_partition_proto", - srcs = ["bigquery_table_partition.proto"], - cc_api_version = 2, -) - -tf_kernel_library( - name = "gcs_config_ops", - srcs = ["gcs_config_ops.cc"], - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core/platform/cloud:curl_http_request", - "//tensorflow/core/platform/cloud:gcs_file_system", - "//tensorflow/core/platform/cloud:oauth_client", - "@jsoncpp_git//:jsoncpp", - ], -) diff --git a/tensorflow/contrib/cloud/kernels/bigquery_reader_ops.cc b/tensorflow/contrib/cloud/kernels/bigquery_reader_ops.cc deleted file mode 100644 index ae6402b391e..00000000000 --- a/tensorflow/contrib/cloud/kernels/bigquery_reader_ops.cc +++ /dev/null @@ -1,192 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include -#include -#include - -#include "tensorflow/contrib/cloud/kernels/bigquery_table_accessor.h" -#include "tensorflow/contrib/cloud/kernels/bigquery_table_partition.pb.h" -#include "tensorflow/core/framework/reader_base.h" -#include "tensorflow/core/framework/reader_op_kernel.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/math/math_util.h" -#include "tensorflow/core/lib/strings/numbers.h" - -namespace tensorflow { -namespace { - -constexpr int64 kDefaultRowBufferSize = 1000; // Number of rows to buffer. - -// This is a helper function for reading table attributes from context. -Status GetTableAttrs(OpKernelConstruction* context, string* project_id, - string* dataset_id, string* table_id, - int64* timestamp_millis, std::vector* columns, - string* test_end_point) { - TF_RETURN_IF_ERROR(context->GetAttr("project_id", project_id)); - TF_RETURN_IF_ERROR(context->GetAttr("dataset_id", dataset_id)); - TF_RETURN_IF_ERROR(context->GetAttr("table_id", table_id)); - TF_RETURN_IF_ERROR(context->GetAttr("timestamp_millis", timestamp_millis)); - TF_RETURN_IF_ERROR(context->GetAttr("columns", columns)); - TF_RETURN_IF_ERROR(context->GetAttr("test_end_point", test_end_point)); - return Status::OK(); -} - -} // namespace - -// Note that overridden methods with names ending in "Locked" are called by -// ReaderBase while a mutex is held. -// See comments for ReaderBase. -class BigQueryReader : public ReaderBase { - public: - explicit BigQueryReader(BigQueryTableAccessor* bigquery_table_accessor, - const string& node_name) - : ReaderBase(strings::StrCat("BigQueryReader '", node_name, "'")), - bigquery_table_accessor_(CHECK_NOTNULL(bigquery_table_accessor)) {} - - Status OnWorkStartedLocked() override { - BigQueryTablePartition partition; - if (!partition.ParseFromString(current_work())) { - return errors::InvalidArgument( - "Could not parse work as valid partition."); - } - TF_RETURN_IF_ERROR(bigquery_table_accessor_->SetPartition(partition)); - return Status::OK(); - } - - Status ReadLocked(tstring* key, tstring* value, bool* produced, - bool* at_end) override { - *at_end = false; - *produced = false; - if (bigquery_table_accessor_->Done()) { - *at_end = true; - return Status::OK(); - } - - Example example; - int64 row_id; - TF_RETURN_IF_ERROR(bigquery_table_accessor_->ReadRow(&row_id, &example)); - - *key = std::to_string(row_id); - *value = example.SerializeAsString(); - *produced = true; - return Status::OK(); - } - - private: - // Not owned. - BigQueryTableAccessor* bigquery_table_accessor_; -}; - -class BigQueryReaderOp : public ReaderOpKernel { - public: - explicit BigQueryReaderOp(OpKernelConstruction* context) - : ReaderOpKernel(context) { - string table_id; - string project_id; - string dataset_id; - int64 timestamp_millis; - std::vector columns; - string test_end_point; - - OP_REQUIRES_OK(context, - GetTableAttrs(context, &project_id, &dataset_id, &table_id, - ×tamp_millis, &columns, &test_end_point)); - OP_REQUIRES_OK(context, - BigQueryTableAccessor::New( - project_id, dataset_id, table_id, timestamp_millis, - kDefaultRowBufferSize, test_end_point, columns, - BigQueryTablePartition(), &bigquery_table_accessor_)); - - SetReaderFactory([this]() { - return new BigQueryReader(bigquery_table_accessor_.get(), name()); - }); - } - - private: - std::unique_ptr bigquery_table_accessor_; -}; - -REGISTER_KERNEL_BUILDER(Name("BigQueryReader").Device(DEVICE_CPU), - BigQueryReaderOp); - -class GenerateBigQueryReaderPartitionsOp : public OpKernel { - public: - explicit GenerateBigQueryReaderPartitionsOp(OpKernelConstruction* context) - : OpKernel(context) { - string project_id; - string dataset_id; - string table_id; - int64 timestamp_millis; - std::vector columns; - string test_end_point; - - OP_REQUIRES_OK(context, - GetTableAttrs(context, &project_id, &dataset_id, &table_id, - ×tamp_millis, &columns, &test_end_point)); - OP_REQUIRES_OK(context, - BigQueryTableAccessor::New( - project_id, dataset_id, table_id, timestamp_millis, - kDefaultRowBufferSize, test_end_point, columns, - BigQueryTablePartition(), &bigquery_table_accessor_)); - OP_REQUIRES_OK(context, InitializeNumberOfPartitions(context)); - OP_REQUIRES_OK(context, InitializeTotalNumberOfRows()); - } - - void Compute(OpKernelContext* context) override { - const int64 partition_size = tensorflow::MathUtil::CeilOfRatio( - total_num_rows_, num_partitions_); - Tensor* output_tensor = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(0, TensorShape({num_partitions_}), - &output_tensor)); - - auto output = output_tensor->template flat(); - for (int64 i = 0; i < num_partitions_; ++i) { - BigQueryTablePartition partition; - partition.set_start_index(i * partition_size); - partition.set_end_index( - std::min(total_num_rows_, (i + 1) * partition_size) - 1); - output(i) = partition.SerializeAsString(); - } - } - - private: - Status InitializeTotalNumberOfRows() { - total_num_rows_ = bigquery_table_accessor_->total_num_rows(); - if (total_num_rows_ <= 0) { - return errors::FailedPrecondition("Invalid total number of rows."); - } - return Status::OK(); - } - - Status InitializeNumberOfPartitions(OpKernelConstruction* context) { - TF_RETURN_IF_ERROR(context->GetAttr("num_partitions", &num_partitions_)); - if (num_partitions_ <= 0) { - return errors::FailedPrecondition("Invalid number of partitions."); - } - return Status::OK(); - } - - int64 num_partitions_; - int64 total_num_rows_; - std::unique_ptr bigquery_table_accessor_; -}; - -REGISTER_KERNEL_BUILDER( - Name("GenerateBigQueryReaderPartitions").Device(DEVICE_CPU), - GenerateBigQueryReaderPartitionsOp); - -} // namespace tensorflow diff --git a/tensorflow/contrib/cloud/kernels/bigquery_table_accessor.cc b/tensorflow/contrib/cloud/kernels/bigquery_table_accessor.cc deleted file mode 100644 index e57a66b99f6..00000000000 --- a/tensorflow/contrib/cloud/kernels/bigquery_table_accessor.cc +++ /dev/null @@ -1,371 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include "tensorflow/contrib/cloud/kernels/bigquery_table_accessor.h" - -#include "tensorflow/core/example/feature.pb.h" -#include "tensorflow/core/lib/strings/numbers.h" - -namespace tensorflow { -namespace { - -constexpr size_t kBufferSize = 1024 * 1024; // In bytes. -const string kBigQueryEndPoint = "https://www.googleapis.com/bigquery/v2"; - -bool IsPartitionEmpty(const BigQueryTablePartition& partition) { - if (partition.end_index() != -1 && - partition.end_index() < partition.start_index()) { - return true; - } - return false; -} - -Status ParseJson(StringPiece json, Json::Value* result) { - Json::Reader reader; - if (!reader.parse(string(json), *result)) { - return errors::Internal("Couldn't parse JSON response from BigQuery."); - } - return Status::OK(); -} - -Status ParseColumnType(const string& type, - BigQueryTableAccessor::ColumnType* enum_type) { - if (type == "RECORD") { - *enum_type = BigQueryTableAccessor::ColumnType::kRecord; - } else if (type == "STRING") { - *enum_type = BigQueryTableAccessor::ColumnType::kString; - } else if (type == "BYTES") { - *enum_type = BigQueryTableAccessor::ColumnType::kBytes; - } else if (type == "INTEGER") { - *enum_type = BigQueryTableAccessor::ColumnType::kInteger; - } else if (type == "FLOAT") { - *enum_type = BigQueryTableAccessor::ColumnType::kFloat; - } else if (type == "BOOLEAN") { - *enum_type = BigQueryTableAccessor::ColumnType::kBoolean; - } else if (type == "TIMESTAMP") { - *enum_type = BigQueryTableAccessor::ColumnType::kTimestamp; - } else if (type == "DATE") { - *enum_type = BigQueryTableAccessor::ColumnType::kDate; - } else if (type == "TIME") { - *enum_type = BigQueryTableAccessor::ColumnType::kTime; - } else if (type == "DATETIME") { - *enum_type = BigQueryTableAccessor::ColumnType::kDatetime; - } else { - return errors::Internal( - strings::StrCat("Could not parse column type ", type)); - } - return Status::OK(); -} - -} // namespace - -Status BigQueryTableAccessor::New( - const string& project_id, const string& dataset_id, const string& table_id, - int64 timestamp_millis, int64 row_buffer_size, const string& end_point, - const std::vector& columns, const BigQueryTablePartition& partition, - std::unique_ptr* accessor) { - return New(project_id, dataset_id, table_id, timestamp_millis, - row_buffer_size, end_point, columns, partition, nullptr, nullptr, - accessor); -} - -Status BigQueryTableAccessor::New( - const string& project_id, const string& dataset_id, const string& table_id, - int64 timestamp_millis, int64 row_buffer_size, const string& end_point, - const std::vector& columns, const BigQueryTablePartition& partition, - std::unique_ptr auth_provider, - std::shared_ptr http_request_factory, - std::unique_ptr* accessor) { - if (timestamp_millis <= 0) { - return errors::InvalidArgument( - "Cannot use zero or negative timestamp to query a table."); - } - const string& big_query_end_point = - end_point.empty() ? kBigQueryEndPoint : end_point; - if (auth_provider == nullptr && http_request_factory == nullptr) { - http_request_factory = std::make_shared(); - auto compute_engine_metadata_client = - std::make_shared(http_request_factory); - auth_provider = std::unique_ptr( - new GoogleAuthProvider(compute_engine_metadata_client)); - } - - accessor->reset(new BigQueryTableAccessor( - project_id, dataset_id, table_id, timestamp_millis, row_buffer_size, - big_query_end_point, columns, partition, std::move(auth_provider), - std::move(http_request_factory))); - - return (*accessor)->ReadSchema(); -} - -BigQueryTableAccessor::BigQueryTableAccessor( - const string& project_id, const string& dataset_id, const string& table_id, - int64 timestamp_millis, int64 row_buffer_size, const string& end_point, - const std::vector& columns, const BigQueryTablePartition& partition, - std::unique_ptr auth_provider, - std::shared_ptr http_request_factory) - : project_id_(project_id), - dataset_id_(dataset_id), - table_id_(table_id), - timestamp_millis_(timestamp_millis), - columns_(columns.begin(), columns.end()), - bigquery_end_point_(end_point), - partition_(partition), - auth_provider_(std::move(auth_provider)), - http_request_factory_(std::move(http_request_factory)) { - row_buffer_.resize(row_buffer_size); - Reset(); -} - -Status BigQueryTableAccessor::SetPartition( - const BigQueryTablePartition& partition) { - if (partition.start_index() < 0) { - return errors::InvalidArgument("Start index cannot be negative."); - } - partition_ = partition; - Reset(); - return Status::OK(); -} - -void BigQueryTableAccessor::Reset() { - first_buffered_row_index_ = partition_.start_index(); - next_row_in_buffer_ = -1; - next_page_token_ = ""; -} - -Status BigQueryTableAccessor::ReadRow(int64* row_id, Example* example) { - if (Done()) { - return errors::OutOfRange("Reached end of table ", FullTableName()); - } - - // If the next row is already fetched and cached, return the row from the - // buffer. Otherwise, fill up the row buffer from BigQuery and return a row. - if (next_row_in_buffer_ != -1 && - next_row_in_buffer_ < ComputeMaxResultsArg()) { - *row_id = first_buffered_row_index_ + next_row_in_buffer_; - *example = row_buffer_[next_row_in_buffer_]; - next_row_in_buffer_++; - } else { - string auth_token; - TF_RETURN_IF_ERROR( - AuthProvider::GetToken(auth_provider_.get(), &auth_token)); - - std::unique_ptr request(http_request_factory_->Create()); - std::vector output_buffer; - output_buffer.reserve(kBufferSize); - - // The first time that we access BigQuery there is no page token. After that - // we use the page token (which returns rows faster). - if (!next_page_token_.empty()) { - request->SetUri(strings::StrCat( - BigQueryUriPrefix(), "data?maxResults=", ComputeMaxResultsArg(), - "&pageToken=", request->EscapeString(next_page_token_))); - first_buffered_row_index_ += row_buffer_.size(); - } else { - request->SetUri(strings::StrCat( - BigQueryUriPrefix(), "data?maxResults=", ComputeMaxResultsArg(), - "&startIndex=", first_buffered_row_index_)); - } - request->AddAuthBearerHeader(auth_token); - request->SetResultBuffer(&output_buffer); - TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), " when reading rows from ", - FullTableName()); - - // Parse the returned row. - StringPiece response_piece = - StringPiece(&output_buffer[0], output_buffer.size()); - Json::Value root; - TF_RETURN_IF_ERROR(ParseJson(response_piece, &root)); - for (unsigned int i = 0; i < root["rows"].size(); ++i) { - row_buffer_[i].Clear(); - TF_RETURN_IF_ERROR( - ParseColumnValues(root["rows"][i], schema_root_, &row_buffer_[i])); - } - - next_page_token_ = root["pageToken"].asString(); - *row_id = first_buffered_row_index_; - *example = row_buffer_[0]; - next_row_in_buffer_ = 1; - } - return Status::OK(); -} - -int64 BigQueryTableAccessor::ComputeMaxResultsArg() { - if (partition_.end_index() == -1) { - return row_buffer_.size(); - } - if (IsPartitionEmpty(partition_)) { - return 0; - } - return std::min(static_cast(row_buffer_.size()), - static_cast(partition_.end_index() - - partition_.start_index() + 1)); -} - -Status BigQueryTableAccessor::ParseColumnValues( - const Json::Value& value, const SchemaNode& root_schema_node, - Example* example) { - if (value.empty()) { - return Status::OK(); - } - if (value["f"].isNull()) { - return Status::OK(); - } - int value_index = 0; - for (const auto& schema_node : root_schema_node.schema_nodes) { - if (value["f"][value_index].isNull()) { - value_index++; - continue; - } - - if (schema_node.type == ColumnType::kRecord) { - TF_RETURN_IF_ERROR(ParseColumnValues(value["f"][value_index]["v"], - schema_node, example)); - } else { - // Append the column value only if user has requested the column. - if (columns_.empty() || - columns_.find(schema_node.name) != columns_.end()) { - TF_RETURN_IF_ERROR(AppendValueToExample(schema_node.name, - value["f"][value_index]["v"], - schema_node.type, example)); - } - } - value_index++; - } - return Status::OK(); -} - -Status BigQueryTableAccessor::ReadSchema() { - string auth_token; - TF_RETURN_IF_ERROR(AuthProvider::GetToken(auth_provider_.get(), &auth_token)); - - // Send a request to read the schema. - std::unique_ptr request(http_request_factory_->Create()); - std::vector output_buffer; - output_buffer.reserve(kBufferSize); - request->SetUri(BigQueryUriPrefix()); - request->AddAuthBearerHeader(auth_token); - request->SetResultBuffer(&output_buffer); - TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), " when reading schema for ", - FullTableName()); - - // Parse the schema. - StringPiece response_piece = - StringPiece(&output_buffer[0], output_buffer.size()); - - Json::Value root; - TF_RETURN_IF_ERROR(ParseJson(response_piece, &root)); - const auto& columns = root["schema"]["fields"]; - string column_name_prefix = ""; - schema_root_ = {"", ColumnType::kNone}; - TF_RETURN_IF_ERROR( - ExtractColumnType(columns, column_name_prefix, &schema_root_)); - if (root["numRows"].isNull()) { - return errors::Internal("Number of rows cannot be extracted for table ", - FullTableName()); - } - strings::safe_strto64(root["numRows"].asString().c_str(), &total_num_rows_); - return Status::OK(); -} - -Status BigQueryTableAccessor::ExtractColumnType( - const Json::Value& columns, const string& column_name_prefix, - SchemaNode* root) { - for (auto columns_it = columns.begin(); columns_it != columns.end(); - ++columns_it) { - if ((*columns_it)["mode"].asString() == "REPEATED") { - return errors::Unimplemented(strings::StrCat( - "Tables with repeated columns are not supported: ", FullTableName())); - } - ColumnType type; - const string current_column_name = strings::StrCat( - column_name_prefix, (*columns_it)["name"].asString().c_str()); - TF_RETURN_IF_ERROR( - ParseColumnType((*columns_it)["type"].asString().c_str(), &type)); - root->schema_nodes.emplace_back(current_column_name, type); - if (type == ColumnType::kRecord) { - const auto new_prefix = strings::StrCat(current_column_name, "."); - TF_RETURN_IF_ERROR(ExtractColumnType((*columns_it)["fields"], new_prefix, - &root->schema_nodes.back())); - } - } - return Status::OK(); -} - -Status BigQueryTableAccessor::AppendValueToExample( - const string& column_name, const Json::Value& column_value, - const BigQueryTableAccessor::ColumnType type, Example* example) { - if (column_value.isNull()) { - return Status::OK(); - } - auto& feature = - (*example->mutable_features()->mutable_feature())[column_name]; - - switch (type) { - case BigQueryTableAccessor::ColumnType::kNone: - case BigQueryTableAccessor::ColumnType::kRecord: - return errors::Unimplemented("Cannot append type to an example."); - case BigQueryTableAccessor::ColumnType::kTimestamp: - case BigQueryTableAccessor::ColumnType::kDate: - case BigQueryTableAccessor::ColumnType::kTime: - case BigQueryTableAccessor::ColumnType::kDatetime: - case BigQueryTableAccessor::ColumnType::kString: - case BigQueryTableAccessor::ColumnType::kBytes: - feature.mutable_bytes_list()->add_value(column_value.asString()); - break; - case BigQueryTableAccessor::ColumnType::kBoolean: - feature.mutable_int64_list()->add_value( - column_value.asString() == "false" ? 0 : 1); - break; - case BigQueryTableAccessor::ColumnType::kInteger: - int64 column_value_int64; - if (!strings::safe_strto64(column_value.asString().c_str(), - &column_value_int64)) { - return errors::Internal("Cannot convert value to integer ", - column_value.asString().c_str()); - } - feature.mutable_int64_list()->add_value(column_value_int64); - break; - case BigQueryTableAccessor::ColumnType::kFloat: - // BigQuery float is actually a double. - double column_value_double; - if (!strings::safe_strtod(column_value.asString().c_str(), - &column_value_double)) { - return errors::Internal("Cannot convert value to double: ", - column_value.asString().c_str()); - } - feature.mutable_float_list()->add_value( - static_cast(column_value_double)); - break; - } - return Status::OK(); -} - -string BigQueryTableAccessor::BigQueryTableAccessor::BigQueryUriPrefix() { - CurlHttpRequest request; - return strings::StrCat(bigquery_end_point_, "/projects/", - request.EscapeString(project_id_), "/datasets/", - request.EscapeString(dataset_id_), "/tables/", - request.EscapeString(table_id_), "/"); -} - -bool BigQueryTableAccessor::Done() { - return (total_num_rows_ <= first_buffered_row_index_ + next_row_in_buffer_) || - IsPartitionEmpty(partition_) || - (partition_.end_index() != -1 && - partition_.end_index() < - first_buffered_row_index_ + next_row_in_buffer_); -} - -} // namespace tensorflow diff --git a/tensorflow/contrib/cloud/kernels/bigquery_table_accessor.h b/tensorflow/contrib/cloud/kernels/bigquery_table_accessor.h deleted file mode 100644 index f1fcaff73be..00000000000 --- a/tensorflow/contrib/cloud/kernels/bigquery_table_accessor.h +++ /dev/null @@ -1,201 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_CLOUD_KERNELS_BIGQUERY_TABLE_ACCESSOR_H_ -#define TENSORFLOW_CONTRIB_CLOUD_KERNELS_BIGQUERY_TABLE_ACCESSOR_H_ - -#include -#include -#include - -#include "tensorflow/contrib/cloud/kernels/bigquery_table_partition.pb.h" -#include "tensorflow/core/example/example.pb.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/platform/cloud/curl_http_request.h" -#include "tensorflow/core/platform/cloud/google_auth_provider.h" - -namespace tensorflow { - -/// This class facilitates accessing BigQuery tables. -/// -/// Notes: -/// - Nested fields are not supported. -/// - BigQuery 'Record's are automatically flattened, -/// - BigQuery float type is a double but is converted to a C++ float in this -/// class. -/// - It is possible for a table snapshot to go out-of-scope in the BigQuery -/// service while accessing the table if a very old timestamp is used. For -/// exact details, see 'Table Decorators' in BigQuery docs. -class BigQueryTableAccessor { - public: - // Column types supported by BigQuery. - enum class ColumnType { - kString = 0, - kBytes, - kInteger, - kFloat, - kBoolean, - kTimestamp, - kDate, - kTime, - kDatetime, - kRecord, - kNone - }; - - /// \brief Creates a new BigQueryTableAccessor object. - // - // We do not allow relative (negative or zero) snapshot times here since we - // want to have a consistent snapshot of the table for the lifetime of this - // object. - // Use end_point if you want to connect to a different end point than the - // official BigQuery end point. Otherwise send an empty string. - static Status New(const string& project_id, const string& dataset_id, - const string& table_id, int64 timestamp_millis, - int64 row_buffer_size, const string& end_point, - const std::vector& columns, - const BigQueryTablePartition& partition, - std::unique_ptr* accessor); - - /// \brief Starts reading a new partition. - Status SetPartition(const BigQueryTablePartition& partition); - - /// \brief Returns true if there are more rows available in the current - /// partition. - bool Done(); - - /// \brief Returns a single row as example proto. - /// - /// This function will return an error if the table snapshot goes out of scope - /// in the BigQuery service. - Status ReadRow(int64* row_id, Example* example); - - /// \brief Returns total number of rows in the table. - int64 total_num_rows() { return total_num_rows_; } - - virtual ~BigQueryTableAccessor() {} - - private: - friend class BigQueryTableAccessorTest; - - // This struct encapsulates schema nodes for a BigQuery table. - struct SchemaNode { - SchemaNode() {} - SchemaNode(const string& name, ColumnType type) : name(name), type(type) {} - - string name; - ColumnType type; - std::vector schema_nodes; - }; - - /// If nullptr is passed for http_request_factory and auth_provider the - /// default production ones are used. This can be used by tests to override - /// these two variables. - static Status New(const string& project_id, const string& dataset_id, - const string& table_id, int64 timestamp_millis, - int64 row_buffer_size, const string& end_point, - const std::vector& columns, - const BigQueryTablePartition& partition, - std::unique_ptr auth_provider, - std::shared_ptr http_request_factory, - std::unique_ptr* accessor); - - /// \brief Constructs an object for a given table and partition. - BigQueryTableAccessor( - const string& project_id, const string& dataset_id, - const string& table_id, int64 timestamp_millis, int64 row_buffer_size, - const string& end_point, const std::vector& columns, - const BigQueryTablePartition& partition, - std::unique_ptr auth_provider, - std::shared_ptr http_request_factory); - - /// \brief Parses column values for a given row. - Status ParseColumnValues(const Json::Value& value, - const SchemaNode& root_schema_node, - Example* example); - - /// \brief Reads the table schema and stores it. - Status ReadSchema(); - - /// \brief Extracts column type from a column in schema. - Status ExtractColumnType(const Json::Value& columns, - const string& column_name_prefix, SchemaNode* root); - - /// \brief Appends a single BigQuery column Value to 'example' for a given - /// column. - Status AppendValueToExample(const string& column_name, - const Json::Value& column_value, - const BigQueryTableAccessor::ColumnType type, - Example* example); - - /// \brief Resets internal counters for reading a partition. - void Reset(); - - /// \brief Helper function that returns BigQuery http endpoint prefix. - string BigQueryUriPrefix(); - - /// \brief Computes the maxResults arg to send to BigQuery. - int64 ComputeMaxResultsArg(); - - /// \brief Returns full name of the underlying table name. - string FullTableName() { - return strings::StrCat(project_id_, ":", dataset_id_, ".", table_id_, "@", - timestamp_millis_); - } - - const string project_id_; - const string dataset_id_; - const string table_id_; - - // Snapshot timestamp. - const int64 timestamp_millis_; - - // Columns that should be read. Empty means all columns. - const std::set columns_; - - // HTTP address of BigQuery end point to use. - const string bigquery_end_point_; - - // Describes the portion of the table that we are currently accessing. - BigQueryTablePartition partition_; - - // Total number of rows in the underlying table. - int64 total_num_rows_ = 0; - - // Offset of the first row in the underlying row_buffer_. - int64 first_buffered_row_index_ = 0; - - // Offset of the next row in the row_buffer_. -1 indicates that this index - // is invalid. - int next_row_in_buffer_ = -1; - - // This buffer holds next rows to improve performance. Its size will be - // based on how much buffering was requested. - std::vector row_buffer_; - - // If next_page is set, it will used to read next batch of data. - string next_page_token_; - - // A tree representing the schema for the underlying table. - SchemaNode schema_root_; - - std::unique_ptr auth_provider_; - std::shared_ptr http_request_factory_; - - TF_DISALLOW_COPY_AND_ASSIGN(BigQueryTableAccessor); -}; - -} // namespace tensorflow -#endif // TENSORFLOW_CONTRIB_CLOUD_KERNELS_BIGQUERY_TABLE_ACCESSOR_H_ diff --git a/tensorflow/contrib/cloud/kernels/bigquery_table_accessor_test.cc b/tensorflow/contrib/cloud/kernels/bigquery_table_accessor_test.cc deleted file mode 100644 index 77ac1b13859..00000000000 --- a/tensorflow/contrib/cloud/kernels/bigquery_table_accessor_test.cc +++ /dev/null @@ -1,513 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/cloud/kernels/bigquery_table_accessor.h" -#include "tensorflow/contrib/cloud/kernels/bigquery_table_accessor_test_data.h" -#include "tensorflow/core/example/feature.pb.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/platform/cloud/http_request_fake.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace { - -constexpr char kTestProject[] = "test-project"; -constexpr char kTestDataset[] = "test-dataset"; -constexpr char kTestTable[] = "test-table"; - -bool HasSubstr(StringPiece base, StringPiece substr) { - bool ok = absl::StrContains(base, substr); - EXPECT_TRUE(ok) << base << ", expected substring " << substr; - return ok; -} - -class FakeAuthProvider : public AuthProvider { - public: - Status GetToken(string* token) override { - *token = "fake_token"; - return Status::OK(); - } -}; - -string DeterministicSerialization(const tensorflow::Example& example) { - const std::size_t size = example.ByteSizeLong(); - string result(size, '\0'); - ::tensorflow::protobuf::io::ArrayOutputStream array_stream(&*result.begin(), - size); - ::tensorflow::protobuf::io::CodedOutputStream output_stream(&array_stream); - - output_stream.SetSerializationDeterministic(true); - example.SerializeWithCachedSizes(&output_stream); - EXPECT_FALSE(output_stream.HadError()); - EXPECT_EQ(size, output_stream.ByteCount()); - return result; -} - -} // namespace - -class BigQueryTableAccessorTest : public ::testing::Test { - protected: - BigQueryTableAccessor::SchemaNode GetSchema() { - return accessor_->schema_root_; - } - - Status CreateTableAccessor(const string& project_id, const string& dataset_id, - const string& table_id, int64 timestamp_millis, - int64 row_buffer_size, - const std::vector& columns, - const BigQueryTablePartition& partition) { - return BigQueryTableAccessor::New( - project_id, dataset_id, table_id, timestamp_millis, row_buffer_size, "", - columns, partition, std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests_)), - &accessor_); - } - - std::vector requests_; - std::unique_ptr accessor_; -}; - -TEST_F(BigQueryTableAccessorTest, NegativeTimestamp) { - const auto status = - CreateTableAccessor(kTestProject, kTestDataset, kTestTable, -1, 3, {}, - BigQueryTablePartition()); - EXPECT_TRUE(errors::IsInvalidArgument(status)); -} - -TEST_F(BigQueryTableAccessorTest, ZeroTimestamp) { - const auto status = - CreateTableAccessor(kTestProject, kTestDataset, kTestTable, 0, 3, {}, - BigQueryTablePartition()); - EXPECT_TRUE(errors::IsInvalidArgument(status)); -} - -TEST_F(BigQueryTableAccessorTest, RepeatedFieldNoAllowedTest) { - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/\n" - "Auth Token: fake_token\n", - R"({ - "kind": "bigquery#table", - "etag": "\"4zcX32ezvFoFzxHoG04qJqKZk6c/MTQ1Nzk3NTgwNzE4Mw\"", - "id": "test-project:test-dataset.test-table", - "schema": { - "fields": [ - { - "name": "int_field", - "type": "INTEGER", - "mode": "REPEATED" - }] - }, - "numRows": "10" - })")); - const auto status = - CreateTableAccessor(kTestProject, kTestDataset, kTestTable, 1, 3, {}, - BigQueryTablePartition()); - EXPECT_TRUE(errors::IsUnimplemented(status)); - EXPECT_TRUE(HasSubstr(status.error_message(), - "Tables with repeated columns are not supported")); -} - -TEST_F(BigQueryTableAccessorTest, ValidSchemaTest) { - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/\n" - "Auth Token: fake_token\n", - kSampleSchema)); - TF_EXPECT_OK(CreateTableAccessor(kTestProject, kTestDataset, kTestTable, 1, 3, - {}, BigQueryTablePartition())); - // Validate total number of rows. - EXPECT_EQ(4, accessor_->total_num_rows()); - - // Validate the schema. - const auto schema_root = GetSchema(); - EXPECT_EQ(schema_root.name, ""); - EXPECT_EQ(schema_root.type, BigQueryTableAccessor::ColumnType::kNone); - EXPECT_EQ(9, schema_root.schema_nodes.size()); - - EXPECT_EQ(schema_root.schema_nodes[0].name, "int_field"); - EXPECT_EQ(schema_root.schema_nodes[0].type, - BigQueryTableAccessor::ColumnType::kInteger); - - EXPECT_EQ(schema_root.schema_nodes[1].name, "str_field"); - EXPECT_EQ(schema_root.schema_nodes[1].type, - BigQueryTableAccessor::ColumnType::kString); - - EXPECT_EQ(1, schema_root.schema_nodes[2].schema_nodes.size()); - EXPECT_EQ(schema_root.schema_nodes[2].name, "rec_field"); - EXPECT_EQ(schema_root.schema_nodes[2].type, - BigQueryTableAccessor::ColumnType::kRecord); - - EXPECT_EQ(schema_root.schema_nodes[2].schema_nodes[0].name, - "rec_field.float_field"); - EXPECT_EQ(schema_root.schema_nodes[2].schema_nodes[0].type, - BigQueryTableAccessor::ColumnType::kFloat); - - EXPECT_EQ(schema_root.schema_nodes[3].name, "bool_field"); - EXPECT_EQ(schema_root.schema_nodes[3].type, - BigQueryTableAccessor::ColumnType::kBoolean); - - EXPECT_EQ(schema_root.schema_nodes[4].name, "bytes_field"); - EXPECT_EQ(schema_root.schema_nodes[4].type, - BigQueryTableAccessor::ColumnType::kBytes); - - EXPECT_EQ(schema_root.schema_nodes[5].name, "timestamp_field"); - EXPECT_EQ(schema_root.schema_nodes[5].type, - BigQueryTableAccessor::ColumnType::kTimestamp); - - EXPECT_EQ(schema_root.schema_nodes[6].name, "date_field"); - EXPECT_EQ(schema_root.schema_nodes[6].type, - BigQueryTableAccessor::ColumnType::kDate); - - EXPECT_EQ(schema_root.schema_nodes[7].name, "time_field"); - EXPECT_EQ(schema_root.schema_nodes[7].type, - BigQueryTableAccessor::ColumnType::kTime); - - EXPECT_EQ(schema_root.schema_nodes[8].name, "datetime_field"); - EXPECT_EQ(schema_root.schema_nodes[8].type, - BigQueryTableAccessor::ColumnType::kDatetime); -} - -TEST_F(BigQueryTableAccessorTest, ReadOneRowTest) { - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/\n" - "Auth Token: fake_token\n", - kSampleSchema)); - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/data?maxResults=1&startIndex=2\n" - "Auth Token: fake_token\n", - kTestRow)); - BigQueryTablePartition partition; - partition.set_start_index(2); - partition.set_end_index(2); - TF_EXPECT_OK(CreateTableAccessor(kTestProject, kTestDataset, kTestTable, 1, 1, - {}, partition)); - int64 row_id; - Example example; - TF_EXPECT_OK(accessor_->ReadRow(&row_id, &example)); - - // Validate returned result. - Example expected_example; - ASSERT_TRUE(protobuf::TextFormat::ParseFromString(kTestExampleProto, - &expected_example)); - EXPECT_EQ(DeterministicSerialization(expected_example), - DeterministicSerialization(example)); - EXPECT_EQ(row_id, 2); - EXPECT_TRUE(accessor_->Done()); -} - -TEST_F(BigQueryTableAccessorTest, ReadOneRowPartialTest) { - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/\n" - "Auth Token: fake_token\n", - kSampleSchema)); - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/data?maxResults=1&startIndex=2\n" - "Auth Token: fake_token\n", - kTestRow)); - BigQueryTablePartition partition; - partition.set_start_index(2); - partition.set_end_index(2); - TF_EXPECT_OK(CreateTableAccessor(kTestProject, kTestDataset, kTestTable, 1, 1, - {"bool_field", "rec_field.float_field"}, - partition)); - int64 row_id; - Example example; - TF_EXPECT_OK(accessor_->ReadRow(&row_id, &example)); - - // Validate returned result. - EXPECT_EQ(row_id, 2); - EXPECT_TRUE(accessor_->Done()); - Example expected_example; - ASSERT_TRUE(protobuf::TextFormat::ParseFromString(kTestPartialExampleProto, - &expected_example)); - EXPECT_EQ(DeterministicSerialization(expected_example), - DeterministicSerialization(example)); -} - -TEST_F(BigQueryTableAccessorTest, ReadOneRowWithNullsTest) { - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/\n" - "Auth Token: fake_token\n", - kSampleSchema)); - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/data?maxResults=1&startIndex=2\n" - "Auth Token: fake_token\n", - kTestRowWithNulls)); - BigQueryTablePartition partition; - partition.set_start_index(2); - partition.set_end_index(2); - TF_EXPECT_OK(CreateTableAccessor(kTestProject, kTestDataset, kTestTable, 1, 1, - {}, partition)); - int64 row_id; - Example example; - TF_EXPECT_OK(accessor_->ReadRow(&row_id, &example)); - - // Validate returned result. - Example expected_example; - ASSERT_TRUE(protobuf::TextFormat::ParseFromString(kTestExampleProtoWithNulls, - &expected_example)); - EXPECT_EQ(DeterministicSerialization(expected_example), - DeterministicSerialization(example)); - EXPECT_EQ(row_id, 2); - EXPECT_TRUE(accessor_->Done()); -} - -TEST_F(BigQueryTableAccessorTest, ReadOneRowTwoRecords) { - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/\n" - "Auth Token: fake_token\n", - kSampleSchemaTwoRecords)); - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/data?maxResults=1&startIndex=2\n" - "Auth Token: fake_token\n", - kTestRowWithTwoRecords)); - BigQueryTablePartition partition; - partition.set_start_index(2); - partition.set_end_index(2); - TF_EXPECT_OK(CreateTableAccessor( - kTestProject, kTestDataset, kTestTable, 1, 1, - {"rec_field2.bool_field", "rec_field1.float_field"}, partition)); - - int64 row_id; - Example example; - TF_EXPECT_OK(accessor_->ReadRow(&row_id, &example)); - - // Validate returned result. - Example expected_example; - ASSERT_TRUE(protobuf::TextFormat::ParseFromString( - kTestExampleProtoWithTwoRecords, &expected_example)); - EXPECT_EQ(DeterministicSerialization(expected_example), - DeterministicSerialization(example)); - EXPECT_EQ(row_id, 2); - EXPECT_TRUE(accessor_->Done()); -} - -TEST_F(BigQueryTableAccessorTest, NonExistentColumns) { - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/\n" - "Auth Token: fake_token\n", - kSampleSchemaTwoRecords)); - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/data?maxResults=1&startIndex=2\n" - "Auth Token: fake_token\n", - kTestRowWithTwoRecords)); - BigQueryTablePartition partition; - partition.set_start_index(2); - partition.set_end_index(2); - TF_EXPECT_OK(CreateTableAccessor(kTestProject, kTestDataset, kTestTable, 1, 1, - {"bool_field", "float_field"}, partition)); - int64 row_id; - Example example; - TF_EXPECT_OK(accessor_->ReadRow(&row_id, &example)); - - // Validate returned result. - EXPECT_EQ(row_id, 2); - EXPECT_TRUE(accessor_->Done()); -} - -TEST_F(BigQueryTableAccessorTest, EmptyRow) { - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/\n" - "Auth Token: fake_token\n", - kSampleSchemaTwoRecords)); - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/data?maxResults=1&startIndex=2\n" - "Auth Token: fake_token\n", - kTestEmptyRow)); - BigQueryTablePartition partition; - partition.set_start_index(2); - partition.set_end_index(2); - TF_EXPECT_OK(CreateTableAccessor(kTestProject, kTestDataset, kTestTable, 1, 1, - {}, partition)); - int64 row_id; - Example example; - TF_EXPECT_OK(accessor_->ReadRow(&row_id, &example)); - - // Validate returned result. - EXPECT_EQ(row_id, 2); - EXPECT_TRUE(accessor_->Done()); -} - -TEST_F(BigQueryTableAccessorTest, BrokenRowTest) { - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/\n" - "Auth Token: fake_token\n", - kSampleSchema)); - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/data?maxResults=1&startIndex=2\n" - "Auth Token: fake_token\n", - kBrokenTestRow)); - BigQueryTablePartition partition; - partition.set_start_index(2); - partition.set_end_index(2); - TF_EXPECT_OK(CreateTableAccessor(kTestProject, kTestDataset, kTestTable, 1, 1, - {}, partition)); - int64 row_id; - Example example; - const auto status = accessor_->ReadRow(&row_id, &example); - EXPECT_TRUE(errors::IsInternal(status)); - EXPECT_TRUE( - HasSubstr(status.error_message(), "Cannot convert value to integer")); -} - -TEST_F(BigQueryTableAccessorTest, MultiplePagesTest) { - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/\n" - "Auth Token: fake_token\n", - kSampleSchema)); - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/data?maxResults=2&startIndex=1\n" - "Auth Token: fake_token\n", - kTestTwoRows)); - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/" - "data?maxResults=2&pageToken=next_page\n" - "Auth Token: fake_token\n", - kTestRowWithNulls)); - - BigQueryTablePartition partition; - partition.set_start_index(1); - partition.set_end_index(-1); - TF_EXPECT_OK(CreateTableAccessor(kTestProject, kTestDataset, kTestTable, 1, 2, - {}, partition)); - - int64 row_id; - Example example; - TF_EXPECT_OK(accessor_->ReadRow(&row_id, &example)); - EXPECT_EQ(1, row_id); - EXPECT_FALSE(accessor_->Done()); - EXPECT_EQ( - (example.features().feature()).at("int_field").int64_list().value(0), - 1111); - - TF_EXPECT_OK(accessor_->ReadRow(&row_id, &example)); - EXPECT_EQ(2, row_id); - EXPECT_FALSE(accessor_->Done()); - EXPECT_EQ(example.features().feature().at("int_field").int64_list().value(0), - 2222); - - TF_EXPECT_OK(accessor_->ReadRow(&row_id, &example)); - EXPECT_EQ(3, row_id); - EXPECT_TRUE(accessor_->Done()); - - Example expected_example; - ASSERT_TRUE(protobuf::TextFormat::ParseFromString(kTestExampleProtoWithNulls, - &expected_example)); - EXPECT_EQ(DeterministicSerialization(expected_example), - DeterministicSerialization(example)); - EXPECT_TRUE(errors::IsOutOfRange(accessor_->ReadRow(&row_id, &example))); -} - -TEST_F(BigQueryTableAccessorTest, SwitchingPartitionsTest) { - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/\n" - "Auth Token: fake_token\n", - kSampleSchema)); - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/data?maxResults=1&startIndex=0\n" - "Auth Token: fake_token\n", - kTestTwoRows)); - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/" - "data?maxResults=2&startIndex=3\n" - "Auth Token: fake_token\n", - kTestRowWithNulls)); - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/data?maxResults=2&startIndex=0\n" - "Auth Token: fake_token\n", - kTestTwoRows)); - - BigQueryTablePartition partition; - partition.set_start_index(0); - partition.set_end_index(0); - TF_EXPECT_OK(CreateTableAccessor(kTestProject, kTestDataset, kTestTable, 1, 2, - {}, partition)); - - int64 row_id; - Example example; - TF_EXPECT_OK(accessor_->ReadRow(&row_id, &example)); - EXPECT_EQ(0, row_id); - EXPECT_TRUE(accessor_->Done()); - EXPECT_EQ(example.features().feature().at("int_field").int64_list().value(0), - 1111); - - partition.set_start_index(3); - partition.set_end_index(-1); - TF_EXPECT_OK(accessor_->SetPartition(partition)); - TF_EXPECT_OK(accessor_->ReadRow(&row_id, &example)); - EXPECT_EQ(3, row_id); - EXPECT_TRUE(accessor_->Done()); - EXPECT_EQ(example.features().feature().at("int_field").int64_list().value(0), - 1234); - - partition.set_start_index(0); - partition.set_end_index(1); - TF_EXPECT_OK(accessor_->SetPartition(partition)); - TF_EXPECT_OK(accessor_->ReadRow(&row_id, &example)); - EXPECT_EQ(0, row_id); - EXPECT_FALSE(accessor_->Done()); - EXPECT_EQ(example.features().feature().at("int_field").int64_list().value(0), - 1111); - TF_EXPECT_OK(accessor_->ReadRow(&row_id, &example)); - EXPECT_EQ(1, row_id); - EXPECT_TRUE(accessor_->Done()); - EXPECT_EQ(example.features().feature().at("int_field").int64_list().value(0), - 2222); -} - -TEST_F(BigQueryTableAccessorTest, EmptyPartitionTest) { - requests_.emplace_back(new FakeHttpRequest( - "Uri: https://www.googleapis.com/bigquery/v2/projects/test-project/" - "datasets/test-dataset/tables/test-table/\n" - "Auth Token: fake_token\n", - kSampleSchema)); - - BigQueryTablePartition partition; - partition.set_start_index(3); - partition.set_end_index(2); - TF_EXPECT_OK(CreateTableAccessor(kTestProject, kTestDataset, kTestTable, 1, 1, - {}, partition)); - EXPECT_TRUE(accessor_->Done()); - - int64 row_id; - Example example; - EXPECT_TRUE(errors::IsOutOfRange(accessor_->ReadRow(&row_id, &example))); -} - -} // namespace tensorflow diff --git a/tensorflow/contrib/cloud/kernels/bigquery_table_accessor_test_data.h b/tensorflow/contrib/cloud/kernels/bigquery_table_accessor_test_data.h deleted file mode 100644 index 6f4d54ae4ab..00000000000 --- a/tensorflow/contrib/cloud/kernels/bigquery_table_accessor_test_data.h +++ /dev/null @@ -1,404 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_CLOUD_KERNELS_BIGQUERY_TABLE_ACCESSOR_TEST_DATA_H_ -#define TENSORFLOW_CONTRIB_CLOUD_KERNELS_BIGQUERY_TABLE_ACCESSOR_TEST_DATA_H_ - -#include - -namespace tensorflow { -namespace { - -const string kSampleSchema = R"({ - "kind": "bigquery#table", - "etag": "\"4zcX32ezvFoFzxHoG04qJqKZk6c/MTQ1Nzk3NTgwNzE4Mw\"", - "id": "test-project:test-dataset.test-table", - "schema": { - "fields": [ - { - "name": "int_field", - "type": "INTEGER", - "mode": "REQUIRED" - },{ - "name": "str_field", - "type": "STRING", - "mode": "NULLABLE" - },{ - "name": "rec_field", - "type": "RECORD", - "fields": [ - { - "name": "float_field", - "type": "FLOAT", - "mode": "NULLABLE" - }] - },{ - "name": "bool_field", - "type": "BOOLEAN", - "mode": "NULLABLE" - },{ - "name": "bytes_field", - "type": "BYTES", - "mode": "NULLABLE" - },{ - "name": "timestamp_field", - "type": "TIMESTAMP", - "mode": "NULLABLE" - },{ - "name": "date_field", - "type": "DATE", - "mode": "NULLABLE" - },{ - "name": "time_field", - "type": "TIME", - "mode": "NULLABLE" - },{ - "name": "datetime_field", - "type": "DATETIME", - "mode": "NULLABLE" - }] - }, - "numRows": "4" -})"; - -const string kSampleSchemaTwoRecords = R"({ - "kind": "bigquery#table", - "etag": "\"4zcX32ezvFoFzxHoG04qJqKZk6c/MTQ1Nzk3NTgwNzE4Mw\"", - "id": "test-project:test-dataset.test-table", - "schema": { - "fields": [ - { - "name": "rec_field1", - "type": "RECORD", - "fields": [ - { - "name": "int_field", - "type": "INTEGER", - "mode": "NULLABLE" - }, { - "name": "float_field", - "type": "FLOAT", - "mode": "NULLABLE" - }] - },{ - "name": "rec_field2", - "type": "RECORD", - "fields": [ - { - "name": "bool_field", - "type": "BOOLEAN", - "mode": "NULLABLE" - },{ - "name": "bytes_field", - "type": "BYTES", - "mode": "NULLABLE" - }] - }] - }, - "numRows": "4" -})"; - -const string kTestRow = R"({ - "kind": "bigquery#table", - "etag": "\"4zcX32ezvFoFzxHoG04qJqKZk6c/MTQ1Nzk3NTgwNzE4Mw\"", - "id": "test-project:test-dataset.test-table", - "rows": [ - { - "f": [ - { - "v": "1234" - },{ - "v": "" - },{ - "v": { - "f": [ - { - "v": "1.23456" - }] - } - },{ - "v": "true" - },{ - "v": "01010100101" - },{ - "v": "timestamp" - },{ - "v": "date" - },{ - "v": "time" - },{ - "v": "datetime" - }]}]})"; - -const string kBrokenTestRow = R"({ - "kind": "bigquery#table", - "etag": "\"4zcX32ezvFoFzxHoG04qJqKZk6c/MTQ1Nzk3NTgwNzE4Mw\"", - "id": "test-project:test-dataset.test-table", - "rows": [ - { - "f": [ - { - "v": "1-234" // This does not parse as integer. - },{ - "v": "" - },{ - },{ - "v": "true" - },{ - "v": "01010100101" - },{ - "v": "timestamp" - },{ - "v": "date" - },{ - "v": "time" - },{ - "v": "datetime" - }]}]})"; - -const string kTestRowWithNulls = R"({ - "kind": "bigquery#table", - "etag": "\"4zcX32ezvFoFzxHoG04qJqKZk6c/MTQ1Nzk3NTgwNzE4Mw\"", - "id": "test-project:test-dataset.test-table", - "rows": [ - { - "f": [ - { - "v": "1234" - },{ - "v": "string" - },{ - "v": null - },{ - "v": "true" - },{ - "v": "01010100101" - },{ - "v": "" - },{ - "v": null - },{ - "v": null - },{ - "v": "datetime" - }]}]})"; - -// Example proto corresponding to kTestRow. -const string kTestExampleProto = R"(features { - feature { - key: "bool_field" - value { - int64_list { - value: 1 - } - } - } - feature { - key: "bytes_field" - value { - bytes_list { - value: "01010100101" - } - } - } - feature { - key: "date_field" - value { - bytes_list { - value: "date" - } - } - } - feature { - key: "datetime_field" - value { - bytes_list { - value: "datetime" - } - } - } - feature { - key: "int_field" - value { - int64_list { - value: 1234 - } - } - } - feature { - key: "rec_field.float_field" - value { - float_list { - value: 1.23456 - } - } - } - feature { - key: "str_field" - value { - bytes_list { - value: "" - } - } - } - feature { - key: "time_field" - value { - bytes_list { - value: "time" - } - } - } - feature { - key: "timestamp_field" - value { - bytes_list { - value: "timestamp" - } - } - } -} -)"; - -// Example proto corresponding to kTestRowWithNulls. -const string kTestExampleProtoWithNulls = R"(features { - feature { - key: "bool_field" - value { - int64_list { - value: 1 - } - } - } - feature { - key: "bytes_field" - value { - bytes_list { - value: "01010100101" - } - } - } - feature { - key: "datetime_field" - value { - bytes_list { - value: "datetime" - } - } - } - feature { - key: "int_field" - value { - int64_list { - value: 1234 - } - } - } - feature { - key: "timestamp_field" - value { - bytes_list { - value: "" - } - } - } - feature { - key: "str_field" - value { - bytes_list { - value: "string" - } - } - } -} -)"; - -// Example proto corresponding to part of kTestRow. -const string kTestPartialExampleProto = R"(features { - feature { - key: "bool_field" - value { - int64_list { - value: 1 - } - } - } - feature { - key: "rec_field.float_field" - value { - float_list { - value: 1.23456 - } - } - } -} -)"; - -const string kTestExampleProtoWithTwoRecords = R"(features { - feature { - key: "rec_field1.float_field" - value { - float_list { - value: 1.23456 - } - } - } - feature { - key: "rec_field2.bool_field" - value { - int64_list { - value: 1 - } - } - } -} -)"; - -const string kTestTwoRows = R"({ - "kind": "bigquery#table", - "etag": "\"4zcX32ezvFoFzxHoG04qJqKZk6c/MTQ1Nzk3NTgwNzE4Mw\"", - "pageToken": "next_page", - "id": "test-project:test-dataset.test-table", - "rows": [ - {"f": [{"v": "1111"},{},{},{},{},{},{},{},{}]}, - {"f": [{"v": "2222"},{},{},{},{},{},{},{},{}]} - ]})"; - -const string kTestRowWithTwoRecords = R"({ - "kind": "bigquery#table", - "etag": "\"4zcX32ezvFoFzxHoG04qJqKZk6c/MTQ1Nzk3NTgwNzE4Mw\"", - "id": "test-project:test-dataset.test-table", - "rows": [ - { - "f": [ - {"v": {"f": [{}, {"v": "1.23456"}]}}, - {"v": {"f": [{"v": "true"}, {}]} - }]}]})"; - -const string kTestEmptyRow = R"({ - "kind": "bigquery#table", - "etag": "\"4zcX32ezvFoFzxHoG04qJqKZk6c/MTQ1Nzk3NTgwNzE4Mw\"", - "id": "test-project:test-dataset.test-table", - "rows": [ - { - "f": [ - {"v": {"f": [{}, {}]}}, - {"v": {"f": [{"v": null}, {}]} - }]}]})"; - -} // namespace -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_CLOUD_KERNELS_BIGQUERY_TABLE_ACCESSOR_TEST_DATA_H_ diff --git a/tensorflow/contrib/cloud/kernels/bigquery_table_partition.proto b/tensorflow/contrib/cloud/kernels/bigquery_table_partition.proto deleted file mode 100644 index 2d9d1380dbd..00000000000 --- a/tensorflow/contrib/cloud/kernels/bigquery_table_partition.proto +++ /dev/null @@ -1,12 +0,0 @@ -syntax = "proto3"; - -package tensorflow; - -// This proto specifies a table partition in BigQuery. -message BigQueryTablePartition { - // [start_index, end_index] specify the boundaries of a partition. - // If end_index is -1, every row starting from start_index is part of the - // partition. - int64 start_index = 1; - int64 end_index = 2; -}; diff --git a/tensorflow/contrib/cloud/kernels/gcs_config_ops.cc b/tensorflow/contrib/cloud/kernels/gcs_config_ops.cc deleted file mode 100644 index 04571348272..00000000000 --- a/tensorflow/contrib/cloud/kernels/gcs_config_ops.cc +++ /dev/null @@ -1,206 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include - -#include "include/json/json.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/platform/cloud/curl_http_request.h" -#include "tensorflow/core/platform/cloud/gcs_file_system.h" -#include "tensorflow/core/platform/cloud/oauth_client.h" -#include "tensorflow/core/util/ptr_util.h" - -namespace tensorflow { -namespace { - -// The default initial delay between retries with exponential backoff. -constexpr int kInitialRetryDelayUsec = 500000; // 0.5 sec - -// The minimum time delta between now and the token expiration time -// for the token to be re-used. -constexpr int kExpirationTimeMarginSec = 60; - -// The URL to retrieve the auth bearer token via OAuth with a refresh token. -constexpr char kOAuthV3Url[] = "https://www.googleapis.com/oauth2/v3/token"; - -// The URL to retrieve the auth bearer token via OAuth with a private key. -constexpr char kOAuthV4Url[] = "https://www.googleapis.com/oauth2/v4/token"; - -// The authentication token scope to request. -constexpr char kOAuthScope[] = "https://www.googleapis.com/auth/cloud-platform"; - -Status RetrieveGcsFs(OpKernelContext* ctx, RetryingGcsFileSystem** fs) { - DCHECK(fs != nullptr); - *fs = nullptr; - - FileSystem* filesystem = nullptr; - TF_RETURN_IF_ERROR( - ctx->env()->GetFileSystemForFile("gs://fake/file.text", &filesystem)); - if (filesystem == nullptr) { - return errors::FailedPrecondition("The GCS file system is not registered."); - } - - *fs = dynamic_cast(filesystem); - if (*fs == nullptr) { - return errors::Internal( - "The filesystem registered under the 'gs://' scheme was not a " - "tensorflow::RetryingGcsFileSystem*."); - } - return Status::OK(); -} - -template -Status ParseScalarArgument(OpKernelContext* ctx, StringPiece argument_name, - T* output) { - const Tensor* argument_t; - TF_RETURN_IF_ERROR(ctx->input(argument_name, &argument_t)); - if (!TensorShapeUtils::IsScalar(argument_t->shape())) { - return errors::InvalidArgument(argument_name, " must be a scalar"); - } - *output = argument_t->scalar()(); - return Status::OK(); -} - -// GcsCredentialsOpKernel overrides the credentials used by the gcs_filesystem. -class GcsCredentialsOpKernel : public OpKernel { - public: - explicit GcsCredentialsOpKernel(OpKernelConstruction* ctx) : OpKernel(ctx) {} - void Compute(OpKernelContext* ctx) override { - // Get a handle to the GCS file system. - RetryingGcsFileSystem* gcs = nullptr; - OP_REQUIRES_OK(ctx, RetrieveGcsFs(ctx, &gcs)); - - tstring json_string; - OP_REQUIRES_OK(ctx, - ParseScalarArgument(ctx, "json", &json_string)); - - Json::Value json; - Json::Reader reader; - std::stringstream json_stream(json_string); - OP_REQUIRES(ctx, reader.parse(json_stream, json), - errors::InvalidArgument("Could not parse json: ", json_string)); - - OP_REQUIRES( - ctx, json.isMember("refresh_token") || json.isMember("private_key"), - errors::InvalidArgument("JSON format incompatible; did not find fields " - "`refresh_token` or `private_key`.")); - - auto provider = - tensorflow::MakeUnique(json, ctx->env()); - - // Test getting a token - string dummy_token; - OP_REQUIRES_OK(ctx, provider->GetToken(&dummy_token)); - OP_REQUIRES(ctx, !dummy_token.empty(), - errors::InvalidArgument( - "Could not retrieve a token with the given credentials.")); - - // Set the provider. - gcs->underlying()->SetAuthProvider(std::move(provider)); - } - - private: - class ConstantAuthProvider : public AuthProvider { - public: - ConstantAuthProvider(const Json::Value& json, - std::unique_ptr oauth_client, Env* env, - int64 initial_retry_delay_usec) - : json_(json), - oauth_client_(std::move(oauth_client)), - env_(env), - initial_retry_delay_usec_(initial_retry_delay_usec) {} - - ConstantAuthProvider(const Json::Value& json, Env* env) - : ConstantAuthProvider(json, tensorflow::MakeUnique(), env, - kInitialRetryDelayUsec) {} - - ~ConstantAuthProvider() override {} - - Status GetToken(string* token) override { - mutex_lock l(mu_); - const uint64 now_sec = env_->NowSeconds(); - - if (!current_token_.empty() && - now_sec + kExpirationTimeMarginSec < expiration_timestamp_sec_) { - *token = current_token_; - return Status::OK(); - } - if (json_.isMember("refresh_token")) { - TF_RETURN_IF_ERROR(oauth_client_->GetTokenFromRefreshTokenJson( - json_, kOAuthV3Url, ¤t_token_, &expiration_timestamp_sec_)); - } else if (json_.isMember("private_key")) { - TF_RETURN_IF_ERROR(oauth_client_->GetTokenFromServiceAccountJson( - json_, kOAuthV4Url, kOAuthScope, ¤t_token_, - &expiration_timestamp_sec_)); - } else { - return errors::FailedPrecondition( - "Unexpected content of the JSON credentials file."); - } - - *token = current_token_; - return Status::OK(); - } - - private: - Json::Value json_; - std::unique_ptr oauth_client_; - Env* env_; - - mutex mu_; - string current_token_ GUARDED_BY(mu_); - uint64 expiration_timestamp_sec_ GUARDED_BY(mu_) = 0; - - // The initial delay for exponential backoffs when retrying failed calls. - const int64 initial_retry_delay_usec_; - TF_DISALLOW_COPY_AND_ASSIGN(ConstantAuthProvider); - }; -}; - -REGISTER_KERNEL_BUILDER(Name("GcsConfigureCredentials").Device(DEVICE_CPU), - GcsCredentialsOpKernel); - -class GcsBlockCacheOpKernel : public OpKernel { - public: - explicit GcsBlockCacheOpKernel(OpKernelConstruction* ctx) : OpKernel(ctx) {} - void Compute(OpKernelContext* ctx) override { - // Get a handle to the GCS file system. - RetryingGcsFileSystem* gcs = nullptr; - OP_REQUIRES_OK(ctx, RetrieveGcsFs(ctx, &gcs)); - - uint64 max_cache_size, block_size, max_staleness; - OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "max_cache_size", - &max_cache_size)); - OP_REQUIRES_OK(ctx, - ParseScalarArgument(ctx, "block_size", &block_size)); - OP_REQUIRES_OK( - ctx, ParseScalarArgument(ctx, "max_staleness", &max_staleness)); - - if (gcs->underlying()->block_size() == block_size && - gcs->underlying()->max_bytes() == max_cache_size && - gcs->underlying()->max_staleness() == max_staleness) { - LOG(INFO) << "Skipping resetting the GCS block cache."; - return; - } - gcs->underlying()->ResetFileBlockCache(block_size, max_cache_size, - max_staleness); - } -}; - -REGISTER_KERNEL_BUILDER(Name("GcsConfigureBlockCache").Device(DEVICE_CPU), - GcsBlockCacheOpKernel); - -} // namespace -} // namespace tensorflow diff --git a/tensorflow/contrib/cloud/ops/bigquery_reader_ops.cc b/tensorflow/contrib/cloud/ops/bigquery_reader_ops.cc deleted file mode 100644 index fbba04a31aa..00000000000 --- a/tensorflow/contrib/cloud/ops/bigquery_reader_ops.cc +++ /dev/null @@ -1,88 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -/* This file registers Bigquery reader ops. */ - -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" -namespace tensorflow { - -using shape_inference::InferenceContext; - -REGISTER_OP("BigQueryReader") - .Attr("container: string = ''") - .Attr("shared_name: string = ''") - .Attr("project_id: string") - .Attr("dataset_id: string") - .Attr("table_id: string") - .Attr("columns: list(string)") - .Attr("timestamp_millis: int") - .Attr("test_end_point: string = ''") - .Output("reader_handle: Ref(string)") - .SetIsStateful() - .SetShapeFn([](InferenceContext* c) { - c->set_output(0, c->Vector(2)); - return Status::OK(); - }) - .Doc(R"doc( -A Reader that outputs rows from a BigQuery table as tensorflow Examples. - -container: If non-empty, this reader is placed in the given container. - Otherwise, a default container is used. -shared_name: If non-empty, this reader is named in the given bucket - with this shared_name. Otherwise, the node name is used instead. -project_id: GCP project ID. -dataset_id: BigQuery Dataset ID. -table_id: Table to read. -columns: List of columns to read. Leave empty to read all columns. -timestamp_millis: Table snapshot timestamp in millis since epoch. Relative -(negative or zero) snapshot times are not allowed. For more details, see -'Table Decorators' in BigQuery docs. -test_end_point: Do not use. For testing purposes only. -reader_handle: The handle to reference the Reader. -)doc"); - -REGISTER_OP("GenerateBigQueryReaderPartitions") - .Attr("project_id: string") - .Attr("dataset_id: string") - .Attr("table_id: string") - .Attr("columns: list(string)") - .Attr("timestamp_millis: int") - .Attr("num_partitions: int") - .Attr("test_end_point: string = ''") - .Output("partitions: string") - .SetShapeFn([](InferenceContext* c) { - c->set_output(0, c->Vector(InferenceContext::kUnknownDim)); - return Status::OK(); - }) - .Doc(R"doc( -Generates serialized partition messages suitable for batch reads. - -This op should not be used directly by clients. Instead, the -bigquery_reader_ops.py file defines a clean interface to the reader. - -project_id: GCP project ID. -dataset_id: BigQuery Dataset ID. -table_id: Table to read. -columns: List of columns to read. Leave empty to read all columns. -timestamp_millis: Table snapshot timestamp in millis since epoch. Relative -(negative or zero) snapshot times are not allowed. For more details, see -'Table Decorators' in BigQuery docs. -num_partitions: Number of partitions to split the table into. -test_end_point: Do not use. For testing purposes only. -partitions: Serialized table partitions. -)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/cloud/ops/gcs_config_ops.cc b/tensorflow/contrib/cloud/ops/gcs_config_ops.cc deleted file mode 100644 index 9cf85f5f181..00000000000 --- a/tensorflow/contrib/cloud/ops/gcs_config_ops.cc +++ /dev/null @@ -1,70 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { - -REGISTER_OP("GcsConfigureCredentials") - .Input("json: string") - .SetShapeFn(shape_inference::NoOutputs) - .Doc(R"doc( -Configures the credentials used by the GCS client of the local TF runtime. - -The json input can be of the format: - -1. Refresh Token: -{ - "client_id": "", - "client_secret": "", - "refresh_token: "", - "type": "authorized_user", -} - -2. Service Account: -{ - "type": "service_account", - "project_id": "", - "private_key_id": "", - "private_key": "------BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY------\n", - "client_email": "@.iam.gserviceaccount.com", - "client_id": "", - # Some additional fields elided -} - -Note the credentials established through this method are shared across all -sessions run on this runtime. - -Note be sure to feed the inputs to this op to ensure the credentials are not -stored in a constant op within the graph that might accidentally be checkpointed -or in other ways be persisted or exfiltrated. -)doc"); - -REGISTER_OP("GcsConfigureBlockCache") - .Input("max_cache_size: uint64") - .Input("block_size: uint64") - .Input("max_staleness: uint64") - .SetShapeFn(shape_inference::NoOutputs) - .Doc(R"doc( -Re-configures the GCS block cache with the new configuration values. - -If the values are the same as already configured values, this op is a no-op. If -they are different, the current contents of the block cache is dropped, and a -new block cache is created fresh. -)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/cloud/python/ops/bigquery_reader_ops.py b/tensorflow/contrib/cloud/python/ops/bigquery_reader_ops.py deleted file mode 100644 index 6606213cbfc..00000000000 --- a/tensorflow/contrib/cloud/python/ops/bigquery_reader_ops.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""BigQuery reading support for TensorFlow.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.cloud.python.ops import gen_bigquery_reader_ops -from tensorflow.python.framework import ops -from tensorflow.python.ops import io_ops - - -class BigQueryReader(io_ops.ReaderBase): - """A Reader that outputs keys and tf.Example values from a BigQuery table. - - Example use: - ```python - # Assume a BigQuery has the following schema, - # name STRING, - # age INT, - # state STRING - - # Create the parse_examples list of features. - features = dict( - name=tf.io.FixedLenFeature([1], tf.string), - age=tf.io.FixedLenFeature([1], tf.int32), - state=tf.io.FixedLenFeature([1], dtype=tf.string, default_value="UNK")) - - # Create a Reader. - reader = bigquery_reader_ops.BigQueryReader(project_id=PROJECT, - dataset_id=DATASET, - table_id=TABLE, - timestamp_millis=TIME, - num_partitions=NUM_PARTITIONS, - features=features) - - # Populate a queue with the BigQuery Table partitions. - queue = tf.compat.v1.train.string_input_producer(reader.partitions()) - - # Read and parse examples. - row_id, examples_serialized = reader.read(queue) - examples = tf.io.parse_example(examples_serialized, features=features) - - # Process the Tensors examples["name"], examples["age"], etc... - ``` - - Note that to create a reader a snapshot timestamp is necessary. This - will enable the reader to look at a consistent snapshot of the table. - For more information, see 'Table Decorators' in BigQuery docs. - - See ReaderBase for supported methods. - """ - - def __init__(self, - project_id, - dataset_id, - table_id, - timestamp_millis, - num_partitions, - features=None, - columns=None, - test_end_point=None, - name=None): - """Creates a BigQueryReader. - - Args: - project_id: GCP project ID. - dataset_id: BigQuery dataset ID. - table_id: BigQuery table ID. - timestamp_millis: timestamp to snapshot the table in milliseconds since - the epoch. Relative (negative or zero) snapshot times are not allowed. - For more details, see 'Table Decorators' in BigQuery docs. - num_partitions: Number of non-overlapping partitions to read from. - features: parse_example compatible dict from keys to `VarLenFeature` and - `FixedLenFeature` objects. Keys are read as columns from the db. - columns: list of columns to read, can be set iff features is None. - test_end_point: Used only for testing purposes (optional). - name: a name for the operation (optional). - - Raises: - TypeError: - If features is neither None nor a dict or - - If columns is neither None nor a list or - - If both features and columns are None or set. - """ - if (features is None) == (columns is None): - raise TypeError("exactly one of features and columns must be set.") - - if features is not None: - if not isinstance(features, dict): - raise TypeError("features must be a dict.") - self._columns = list(features.keys()) - elif columns is not None: - if not isinstance(columns, list): - raise TypeError("columns must be a list.") - self._columns = columns - - self._project_id = project_id - self._dataset_id = dataset_id - self._table_id = table_id - self._timestamp_millis = timestamp_millis - self._num_partitions = num_partitions - self._test_end_point = test_end_point - - reader = gen_bigquery_reader_ops.big_query_reader( - name=name, - project_id=self._project_id, - dataset_id=self._dataset_id, - table_id=self._table_id, - timestamp_millis=self._timestamp_millis, - columns=self._columns, - test_end_point=self._test_end_point) - super(BigQueryReader, self).__init__(reader) - - def partitions(self, name=None): - """Returns serialized BigQueryTablePartition messages. - - These messages represent a non-overlapping division of a table for a - bulk read. - - Args: - name: a name for the operation (optional). - - Returns: - `1-D` string `Tensor` of serialized `BigQueryTablePartition` messages. - """ - return gen_bigquery_reader_ops.generate_big_query_reader_partitions( - name=name, - project_id=self._project_id, - dataset_id=self._dataset_id, - table_id=self._table_id, - timestamp_millis=self._timestamp_millis, - num_partitions=self._num_partitions, - test_end_point=self._test_end_point, - columns=self._columns) - - -ops.NotDifferentiable("BigQueryReader") diff --git a/tensorflow/contrib/cloud/python/ops/bigquery_reader_ops_test.py b/tensorflow/contrib/cloud/python/ops/bigquery_reader_ops_test.py deleted file mode 100644 index 11e177cd0c8..00000000000 --- a/tensorflow/contrib/cloud/python/ops/bigquery_reader_ops_test.py +++ /dev/null @@ -1,294 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for BigQueryReader Op.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import json -import os -import re -import socket -import threading - -from six.moves import SimpleHTTPServer -from six.moves import socketserver - -from tensorflow.contrib.cloud.python.ops import bigquery_reader_ops as cloud -from tensorflow.core.example import example_pb2 -from tensorflow.core.framework import types_pb2 -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util import compat - -_PROJECT = "test-project" -_DATASET = "test-dataset" -_TABLE = "test-table" -# List representation of the test rows in the 'test-table' in BigQuery. -# The schema for each row is: [int64, string, float]. -# The values for rows are generated such that some columns have null values. The -# general formula here is: -# - The int64 column is present in every row. -# - The string column is only available in even rows. -# - The float column is only available in every third row. -_ROWS = [[0, "s_0", 0.1], [1, None, None], [2, "s_2", None], [3, None, 3.1], - [4, "s_4", None], [5, None, None], [6, "s_6", 6.1], [7, None, None], - [8, "s_8", None], [9, None, 9.1]] -# Schema for 'test-table'. -# The schema currently has three columns: int64, string, and float -_SCHEMA = { - "kind": "bigquery#table", - "id": "test-project:test-dataset.test-table", - "schema": { - "fields": [{ - "name": "int64_col", - "type": "INTEGER", - "mode": "NULLABLE" - }, { - "name": "string_col", - "type": "STRING", - "mode": "NULLABLE" - }, { - "name": "float_col", - "type": "FLOAT", - "mode": "NULLABLE" - }] - } -} - - -def _ConvertRowToExampleProto(row): - """Converts the input row to an Example proto. - - Args: - row: Input Row instance. - - Returns: - An Example proto initialized with row values. - """ - - example = example_pb2.Example() - example.features.feature["int64_col"].int64_list.value.append(row[0]) - if row[1] is not None: - example.features.feature["string_col"].bytes_list.value.append( - compat.as_bytes(row[1])) - if row[2] is not None: - example.features.feature["float_col"].float_list.value.append(row[2]) - return example - - -class IPv6TCPServer(socketserver.TCPServer): - address_family = socket.AF_INET6 - - -class FakeBigQueryServer(threading.Thread): - """Fake http server to return schema and data for sample table.""" - - def __init__(self, address, port): - """Creates a FakeBigQueryServer. - - Args: - address: Server address - port: Server port. Pass 0 to automatically pick an empty port. - """ - threading.Thread.__init__(self) - self.handler = BigQueryRequestHandler - try: - self.httpd = socketserver.TCPServer((address, port), self.handler) - self.host_port = "{}:{}".format(*self.httpd.server_address) - except IOError: - self.httpd = IPv6TCPServer((address, port), self.handler) - self.host_port = "[{}]:{}".format(*self.httpd.server_address) - - def run(self): - self.httpd.serve_forever() - - def shutdown(self): - self.httpd.shutdown() - self.httpd.socket.close() - - -class BigQueryRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): - """Responds to BigQuery HTTP requests. - - Attributes: - num_rows: num_rows in the underlying table served by this class. - """ - - num_rows = 0 - - def do_GET(self): - if "data?maxResults=" not in self.path: - # This is a schema request. - _SCHEMA["numRows"] = self.num_rows - response = json.dumps(_SCHEMA) - else: - # This is a data request. - # - # Extract max results and start index. - max_results = int(re.findall(r"maxResults=(\d+)", self.path)[0]) - start_index = int(re.findall(r"startIndex=(\d+)", self.path)[0]) - - # Send the rows as JSON. - rows = [] - for row in _ROWS[start_index:start_index + max_results]: - row_json = { - "f": [{ - "v": str(row[0]) - }, { - "v": str(row[1]) if row[1] is not None else None - }, { - "v": str(row[2]) if row[2] is not None else None - }] - } - rows.append(row_json) - response = json.dumps({ - "kind": "bigquery#table", - "id": "test-project:test-dataset.test-table", - "rows": rows - }) - self.send_response(200) - self.end_headers() - self.wfile.write(compat.as_bytes(response)) - - -def _SetUpQueue(reader): - """Sets up a queue for a reader.""" - queue = data_flow_ops.FIFOQueue(8, [types_pb2.DT_STRING], shapes=()) - key, value = reader.read(queue) - queue.enqueue_many(reader.partitions()).run() - queue.close().run() - return key, value - - -class BigQueryReaderOpsTest(test.TestCase): - - def setUp(self): - super(BigQueryReaderOpsTest, self).setUp() - self.server = FakeBigQueryServer("localhost", 0) - self.server.start() - logging.info("server address is %s", self.server.host_port) - - # An override to bypass the GCP auth token retrieval logic - # in google_auth_provider.cc. - os.environ["GOOGLE_AUTH_TOKEN_FOR_TESTING"] = "not-used" - - def tearDown(self): - self.server.shutdown() - super(BigQueryReaderOpsTest, self).tearDown() - - def _ReadAndCheckRowsUsingFeatures(self, num_rows): - self.server.handler.num_rows = num_rows - - with self.cached_session() as sess: - feature_configs = { - "int64_col": - parsing_ops.FixedLenFeature( - [1], dtype=dtypes.int64), - "string_col": - parsing_ops.FixedLenFeature( - [1], dtype=dtypes.string, default_value="s_default"), - } - reader = cloud.BigQueryReader( - project_id=_PROJECT, - dataset_id=_DATASET, - table_id=_TABLE, - num_partitions=4, - features=feature_configs, - timestamp_millis=1, - test_end_point=self.server.host_port) - - key, value = _SetUpQueue(reader) - - seen_rows = [] - features = parsing_ops.parse_example( - array_ops.reshape(value, [1]), feature_configs) - for _ in range(num_rows): - int_value, str_value = sess.run( - [features["int64_col"], features["string_col"]]) - - # Parse values returned from the session. - self.assertEqual(int_value.shape, (1, 1)) - self.assertEqual(str_value.shape, (1, 1)) - int64_col = int_value[0][0] - string_col = str_value[0][0] - seen_rows.append(int64_col) - - # Compare. - expected_row = _ROWS[int64_col] - self.assertEqual(int64_col, expected_row[0]) - self.assertEqual( - compat.as_str(string_col), ("s_%d" % int64_col) if expected_row[1] - else "s_default") - - self.assertItemsEqual(seen_rows, range(num_rows)) - - with self.assertRaisesOpError("is closed and has insufficient elements " - "\\(requested 1, current size 0\\)"): - sess.run([key, value]) - - def testReadingSingleRowUsingFeatures(self): - self._ReadAndCheckRowsUsingFeatures(1) - - def testReadingMultipleRowsUsingFeatures(self): - self._ReadAndCheckRowsUsingFeatures(10) - - def testReadingMultipleRowsUsingColumns(self): - num_rows = 10 - self.server.handler.num_rows = num_rows - - with self.cached_session() as sess: - reader = cloud.BigQueryReader( - project_id=_PROJECT, - dataset_id=_DATASET, - table_id=_TABLE, - num_partitions=4, - columns=["int64_col", "float_col", "string_col"], - timestamp_millis=1, - test_end_point=self.server.host_port) - key, value = _SetUpQueue(reader) - seen_rows = [] - for row_index in range(num_rows): - returned_row_id, example_proto = sess.run([key, value]) - example = example_pb2.Example() - example.ParseFromString(example_proto) - self.assertIn("int64_col", example.features.feature) - feature = example.features.feature["int64_col"] - self.assertEqual(len(feature.int64_list.value), 1) - int64_col = feature.int64_list.value[0] - seen_rows.append(int64_col) - - # Create our expected Example. - expected_example = example_pb2.Example() - expected_example = _ConvertRowToExampleProto(_ROWS[int64_col]) - - # Compare. - self.assertProtoEquals(example, expected_example) - self.assertEqual(row_index, int(returned_row_id)) - - self.assertItemsEqual(seen_rows, range(num_rows)) - - with self.assertRaisesOpError("is closed and has insufficient elements " - "\\(requested 1, current size 0\\)"): - sess.run([key, value]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/cloud/python/ops/gcs_config_ops.py b/tensorflow/contrib/cloud/python/ops/gcs_config_ops.py deleted file mode 100644 index 806d5630464..00000000000 --- a/tensorflow/contrib/cloud/python/ops/gcs_config_ops.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""GCS file system configuration for TensorFlow.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import json -import os - -from tensorflow.contrib.cloud.python.ops import gen_gcs_config_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.training import training - - -# @tf_export('contrib.cloud.BlockCacheParams') -class BlockCacheParams(object): - """BlockCacheParams is a struct used for configuring the GCS Block Cache.""" - - def __init__(self, block_size=None, max_bytes=None, max_staleness=None): - self._block_size = block_size or 128 * 1024 * 1024 - self._max_bytes = max_bytes or 2 * self._block_size - self._max_staleness = max_staleness or 0 - - @property - def block_size(self): - return self._block_size - - @property - def max_bytes(self): - return self._max_bytes - - @property - def max_staleness(self): - return self._max_staleness - - -# @tf_export('contrib.cloud.ConfigureGcsHook') -class ConfigureGcsHook(training.SessionRunHook): - """ConfigureGcsHook configures GCS when used with Estimator/TPUEstimator. - - Warning: GCS `credentials` may be transmitted over the network unencrypted. - Please ensure that the network is trusted before using this function. For - users running code entirely within Google Cloud, your data is protected by - encryption in between data centers. For more information, please take a look - at https://cloud.google.com/security/encryption-in-transit/. - - Example: - - ``` - sess = tf.compat.v1.Session() - refresh_token = raw_input("Refresh token: ") - client_secret = raw_input("Client secret: ") - client_id = "" - creds = { - "client_id": client_id, - "refresh_token": refresh_token, - "client_secret": client_secret, - "type": "authorized_user", - } - tf.contrib.cloud.configure_gcs(sess, credentials=creds) - ``` - - """ - - def _verify_dictionary(self, creds_dict): - if 'refresh_token' in creds_dict or 'private_key' in creds_dict: - return True - return False - - def __init__(self, credentials=None, block_cache=None): - """Constructs a ConfigureGcsHook. - - Args: - credentials: A json-formatted string. - block_cache: A `BlockCacheParams` - - Raises: - ValueError: If credentials is improperly formatted or block_cache is not a - BlockCacheParams. - """ - if credentials is not None: - if isinstance(credentials, str): - try: - data = json.loads(credentials) - except ValueError as e: - raise ValueError('credentials was not a well formed JSON string.', e) - if not self._verify_dictionary(data): - raise ValueError( - 'credentials has neither a "refresh_token" nor a "private_key" ' - 'field.') - elif isinstance(credentials, dict): - if not self._verify_dictionary(credentials): - raise ValueError('credentials has neither a "refresh_token" nor a ' - '"private_key" field.') - credentials = json.dumps(credentials) - else: - raise ValueError('credentials is of an unknown type') - - self._credentials = credentials - - if block_cache and not isinstance(block_cache, BlockCacheParams): - raise ValueError('block_cache must be an instance of BlockCacheParams.') - self._block_cache = block_cache - - def begin(self): - if self._credentials: - self._credentials_placeholder = array_ops.placeholder(dtypes.string) - self._credentials_op = gen_gcs_config_ops.gcs_configure_credentials( - self._credentials_placeholder) - else: - self._credentials_op = None - - if self._block_cache: - self._block_cache_op = gen_gcs_config_ops.gcs_configure_block_cache( - max_cache_size=self._block_cache.max_bytes, - block_size=self._block_cache.block_size, - max_staleness=self._block_cache.max_staleness) - else: - self._block_cache_op = None - - def after_create_session(self, session, coord): - del coord - if self._credentials_op: - session.run( - self._credentials_op, - feed_dict={self._credentials_placeholder: self._credentials}) - if self._block_cache_op: - session.run(self._block_cache_op) - - -def configure_gcs(session, credentials=None, block_cache=None, device=None): - """Configures the GCS file system for a given a session. - - Warning: GCS `credentials` may be transmitted over the network unencrypted. - Please ensure that the network is trusted before using this function. For - users running code entirely within Google Cloud, your data is protected by - encryption in between data centers. For more information, please take a look - at https://cloud.google.com/security/encryption-in-transit/. - - Args: - session: A `tf.compat.v1.Session` session that should be used to configure - the GCS file system. - credentials: [Optional.] A JSON string - block_cache: [Optional.] A BlockCacheParams to configure the block cache . - device: [Optional.] The device to place the configure ops. - """ - - def configure(credentials, block_cache): - """Helper function to actually configure GCS.""" - if credentials: - if isinstance(credentials, dict): - credentials = json.dumps(credentials) - placeholder = array_ops.placeholder(dtypes.string) - op = gen_gcs_config_ops.gcs_configure_credentials(placeholder) - session.run(op, feed_dict={placeholder: credentials}) - if block_cache: - op = gen_gcs_config_ops.gcs_configure_block_cache( - max_cache_size=block_cache.max_bytes, - block_size=block_cache.block_size, - max_staleness=block_cache.max_staleness) - session.run(op) - - if device: - with ops.device(device): - return configure(credentials, block_cache) - return configure(credentials, block_cache) - - -def configure_colab_session(session): - """ConfigureColabSession configures the GCS file system in Colab. - - Args: - session: A `tf.compat.v1.Session` session. - """ - # Read from the application default credentials (adc). - adc_filename = os.environ.get( - 'GOOGLE_APPLICATION_CREDENTIALS', '/content/adc.json') - with open(adc_filename) as f: - data = json.load(f) - configure_gcs(session, credentials=data) diff --git a/tensorflow/contrib/cloud/python/ops/gcs_config_ops_test.py b/tensorflow/contrib/cloud/python/ops/gcs_config_ops_test.py deleted file mode 100644 index 4f2ecbcb170..00000000000 --- a/tensorflow/contrib/cloud/python/ops/gcs_config_ops_test.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the gcs_config_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.cloud.python.ops import gcs_config_ops -from tensorflow.python.platform import test - - -class GcsConfigOpsTest(test.TestCase): - - def testSetBlockCache(self): - cfg = gcs_config_ops.BlockCacheParams(max_bytes=1024*1024*1024) - with self.cached_session() as sess: - gcs_config_ops.configure_gcs(sess, block_cache=cfg) - - def testConfigureGcsHook(self): - creds = {'client_id': 'fake_client', - 'refresh_token': 'fake_token', - 'client_secret': 'fake_secret', - 'type': 'authorized_user'} - hook = gcs_config_ops.ConfigureGcsHook(credentials=creds) - hook.begin() - with self.cached_session() as sess: - sess.run = lambda _, feed_dict=None, options=None, run_metadata=None: None - hook.after_create_session(sess, None) - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/cluster_resolver/BUILD b/tensorflow/contrib/cluster_resolver/BUILD deleted file mode 100644 index a552173fb55..00000000000 --- a/tensorflow/contrib/cluster_resolver/BUILD +++ /dev/null @@ -1,44 +0,0 @@ -# Description: Operations defined for Cluster Resolvers - -load("//tensorflow:tensorflow.bzl", "tf_py_test") - -package( - default_visibility = [ - "//tensorflow:__subpackages__", - ], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "cluster_resolver_pip", - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":cluster_resolver_py", - ], -) - -py_library( - name = "cluster_resolver_py", - srcs = glob([ - "__init__.py", - "python/training/*.py", - ]), - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = ["//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib"], -) - -tf_py_test( - name = "cluster_resolver_initialization_test", - srcs = ["cluster_resolver_initialization_test.py"], - additional_deps = [ - ":cluster_resolver_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:training", - ], - main = "cluster_resolver_initialization_test.py", -) diff --git a/tensorflow/contrib/cluster_resolver/__init__.py b/tensorflow/contrib/cluster_resolver/__init__.py deleted file mode 100644 index a4dea85efd9..00000000000 --- a/tensorflow/contrib/cluster_resolver/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Standard imports for Cluster Resolvers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver -from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver -from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver -from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GCEClusterResolver -from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver -from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver -from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver -from tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver import TPUClusterResolver -# pylint: enable=wildcard-import,unused-import - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'ClusterResolver', - 'SimpleClusterResolver', - 'UnionClusterResolver', - 'GCEClusterResolver', - 'KubernetesClusterResolver', - 'TFConfigClusterResolver', - 'TPUClusterResolver', - 'SlurmClusterResolver', -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/cluster_resolver/cluster_resolver_initialization_test.py b/tensorflow/contrib/cluster_resolver/cluster_resolver_initialization_test.py deleted file mode 100644 index 01ff1478c69..00000000000 --- a/tensorflow/contrib/cluster_resolver/cluster_resolver_initialization_test.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests to ensure ClusterResolvers are usable via the old contrib path.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.cluster_resolver import SimpleClusterResolver -from tensorflow.contrib.cluster_resolver.python.training import cluster_resolver -from tensorflow.contrib.cluster_resolver.python.training import UnionClusterResolver -from tensorflow.python.platform import test -from tensorflow.python.training import server_lib - - -class ClusterResolverInitializationTest(test.TestCase): - - def testCreateSimpleClusterResolverFromLib(self): - base_cluster_spec = server_lib.ClusterSpec({ - "ps": ["ps0:2222", "ps1:2222"], - "worker": ["worker0:2222", "worker1:2222", "worker2:2222"] - }) - cluster_resolver.SimpleClusterResolver(base_cluster_spec) - - def testCreateSimpleClusterResolver(self): - base_cluster_spec = server_lib.ClusterSpec({ - "ps": ["ps0:2222", "ps1:2222"], - "worker": ["worker0:2222", "worker1:2222", "worker2:2222"] - }) - SimpleClusterResolver(base_cluster_spec) - - def testCreateUnionClusterResolver(self): - base_cluster_spec = server_lib.ClusterSpec({ - "ps": ["ps0:2222", "ps1:2222"], - "worker": ["worker0:2222", "worker1:2222", "worker2:2222"] - }) - simple_cr = SimpleClusterResolver(base_cluster_spec) - UnionClusterResolver(simple_cr) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/cluster_resolver/python/training/__init__.py b/tensorflow/contrib/cluster_resolver/python/training/__init__.py deleted file mode 100644 index ef1e9f11a07..00000000000 --- a/tensorflow/contrib/cluster_resolver/python/training/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Library Imports for Cluster Resolvers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# This file (and all files in this directory in general) is a backwards -# compatibility shim that exists to re-export ClusterResolvers such that -# existing OSS code will not be broken. - -from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver -from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver -from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver -from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GCEClusterResolver -from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver -from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver -from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver -from tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver import TPUClusterResolver - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'cluster_resolver', - 'gce_cluster_resolver', - 'kubernetes_cluster_resolver', - 'slurm_cluster_resolver', - 'tfconfig_cluster_resolver', - 'tpu_cluster_resolver', - 'ClusterResolver', - 'SimpleClusterResolver', - 'UnionClusterResolver', - 'GCEClusterResolver', - 'KubernetesClusterResolver', - 'TFConfigClusterResolver', - 'TPUClusterResolver', - 'SlurmClusterResolver', -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py deleted file mode 100644 index 99840fb5166..00000000000 --- a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file for ClusterResolver to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# This file (and all files in this directory in general) is a backwards -# compatibility shim that exists to re-export ClusterResolvers such that -# existing OSS code will not be broken. - -# pylint: disable=unused-import -from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver -from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver -from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver -# pylint: enable=unused-import - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'ClusterResolver', - 'SimpleClusterResolver', - 'UnionClusterResolver', -] - -remove_undocumented(__name__, _allowed_symbols) - diff --git a/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py deleted file mode 100644 index 5b49116ff6a..00000000000 --- a/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file for GCEClusterResolver to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# This file (and all files in this directory in general) is a backwards -# compatibility shim that exists to re-export ClusterResolvers such that -# existing OSS code will not be broken. - -# pylint: disable=unused-import -from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GCEClusterResolver -# pylint: enable=unused-import - -from tensorflow.python.util.all_util import remove_undocumented - - -_allowed_symbols = [ - 'GCEClusterResolver', -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py deleted file mode 100644 index a8eaf33629a..00000000000 --- a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file for KubernetesClusterResolver for backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# This file (and all files in this directory in general) is a backwards -# compatibility shim that exists to re-export ClusterResolvers such that -# existing OSS code will not be broken. - -# pylint: disable=unused-import -from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver -# pylint: enable=unused-import - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'KubernetesClusterResolver', -] - -remove_undocumented(__name__, _allowed_symbols) - diff --git a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py deleted file mode 100644 index fcd2a846eeb..00000000000 --- a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file for SlurmClusterResolver to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# This file (and all files in this directory in general) is a backwards -# compatibility shim that exists to re-export ClusterResolvers such that -# existing OSS code will not be broken. - -# pylint: disable=unused-import -from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver -# pylint: enable=unused-import - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'SlurmClusterResolver', -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py deleted file mode 100644 index 9db7f47dcb4..00000000000 --- a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file for TFConfigClusterResolver to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# This file (and all files in this directory in general) is a backwards -# compatibility shim that exists to re-export ClusterResolvers such that -# existing OSS code will not be broken. - -# pylint: disable=unused-import -from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver -# pylint: enable=unused-import - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'TFConfigClusterResolver', -] - -remove_undocumented(__name__, _allowed_symbols) - diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py deleted file mode 100644 index 3a1eaccd06e..00000000000 --- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file for TPUClusterResolver to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# This file (and all files in this directory in general) is a backwards -# compatibility shim that exists to re-export ClusterResolvers such that -# existing OSS code will not be broken. - -# pylint: disable=unused-import -from tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver import TPUClusterResolver -# pylint: enable=unused-import - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'TPUClusterResolver', -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt deleted file mode 100644 index 44431d5010d..00000000000 --- a/tensorflow/contrib/cmake/CMakeLists.txt +++ /dev/null @@ -1,619 +0,0 @@ -# Minimum CMake required -cmake_minimum_required(VERSION 3.5) - -if(WIN32) - if(${CMAKE_VERSION} VERSION_LESS "3.8") - message(WARNING "Your current cmake version is ${CMAKE_VERSION} which does not support setting the toolset architecture to x64. This may cause \"compiler out of heap space\" errors when building. Consider upgrading your cmake to > 3.8 and using the flag -Thost=x64 when running cmake. Ignore this if you are on CMake GUI.") - else() - if(NOT CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE OR NOT "${CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE}" STREQUAL "x64") - message(WARNING "Your current cmake generator is set to use 32 bit toolset architecture. This may cause \"compiler out of heap space\" errors when building. Consider using the flag -Thost=x64 when running cmake. Ignore this if you are on CMake GUI.") - endif() - endif() -endif() - -# Project -project(tensorflow VERSION 1.12.0 LANGUAGES C CXX) - -# Set C++14 as standard for the whole project -set(CMAKE_CXX_STANDARD 14) - -# Actual source is the ../../.. directory -get_filename_component(tf_contrib_source_dir ${tensorflow_SOURCE_DIR} PATH) -get_filename_component(tf_tf_source_dir ${tf_contrib_source_dir} PATH) -get_filename_component(tensorflow_source_dir ${tf_tf_source_dir} PATH) - -# [CLEANUP] Not sure if this is needed (copied from Protobuf) -# CMake policies -cmake_policy(SET CMP0022 NEW) - -# Options -option(tensorflow_VERBOSE "Enable for verbose output" OFF) - -if(WIN32) -# BoringSSL is disabled for windows as it currently doesn't build with -# MSBuild. (Ninja is required.) -option(tensorflow_ENABLE_SSL_SUPPORT "Enable boringssl support" OFF) -else() -# BoringSSL is enabled for gRPC. -option(tensorflow_ENABLE_SSL_SUPPORT "Enable boringssl support" ON) -endif() - -option(tensorflow_ENABLE_GRPC_SUPPORT "Enable gRPC support" ON) -option(tensorflow_ENABLE_HDFS_SUPPORT "Enable HDFS support" OFF) -option(tensorflow_BUILD_CC_EXAMPLE "Build the C++ tutorial example" ON) -option(tensorflow_BUILD_PYTHON_BINDINGS "Build the Python bindings" ON) -option(tensorflow_BUILD_ALL_KERNELS "Build all OpKernels" ON) -option(tensorflow_BUILD_CONTRIB_KERNELS "Build OpKernels from tensorflow/contrib/..." ON) -option(tensorflow_BUILD_CC_TESTS "Build cc unit tests " OFF) -option(tensorflow_BUILD_PYTHON_TESTS "Build python unit tests " OFF) -option(tensorflow_BUILD_MORE_PYTHON_TESTS "Build more python unit tests for contrib packages" OFF) -option(tensorflow_BUILD_SHARED_LIB "Build TensorFlow as a shared library" OFF) -option(tensorflow_OPTIMIZE_FOR_NATIVE_ARCH "Enable compiler optimizations for the native processor architecture (if available)" ON) -option(tensorflow_ENABLE_SNAPPY_SUPPORT "Enable SNAPPY compression support" ON) -option(tensorflow_DISABLE_EIGEN_FORCEINLINE "Disable forceinline, to speed up build on windows." OFF) - -if (WIN32) -SET(tensorflow_WIN_CPU_SIMD_OPTIONS "/arch:AVX" CACHE STRING "Enables CPU SIMD instructions") -SET_PROPERTY(CACHE tensorflow_WIN_CPU_SIMD_OPTIONS PROPERTY STRINGS /arch:AVX) -endif() - -# SIMD, MKL and MKLDNN options -option(tensorflow_WIN_CPU_SIMD_OPTIONS "Enables CPU SIMD instructions" OFF) -option(tensorflow_ENABLE_MKL_SUPPORT "Enable Intel MKL support" OFF) -option(tensorflow_ENABLE_MKLDNN_SUPPORT "Enable Intel MKLDNN support, requires MKL enabled" OFF) - - -# GPU, CUDA and cuDNN options -option(tensorflow_ENABLE_GPU "Enable GPU support" OFF) - -if(HAIKU) - option(tensorflow_ENABLE_POSITION_INDEPENDENT_CODE "Enable PIE support" OFF) -else() - option(tensorflow_ENABLE_POSITION_INDEPENDENT_CODE "Enable PIE support" ON) -endif() - - -if (NOT WIN32) - # Threads: defines CMAKE_THREAD_LIBS_INIT and adds -pthread compile option - # for targets that link ${CMAKE_THREAD_LIBS_INIT}. - find_package (Threads REQUIRED) - - # Options for linking CUDA/CUDNN libraries - option(tensorflow_PATH_CUDA_LIB "Additional library search path for cudnn, nccl, culibos" /usr/local/cuda/lib64/) - option(tensorflow_CUDNN_INCLUDE "cudnn.h header install path" /usr/include/) - if (NOT tensorflow_CUDNN_INCLUDE) - # option's default value is OFF. Fill it with real default values - set(tensorflow_CUDNN_INCLUDE /usr/include) - endif (NOT tensorflow_CUDNN_INCLUDE) - option(tensorflow_NCCL_INCLUDE "nccl.h header install path" /usr/include/) - if (NOT tensorflow_NCCL_INCLUDE) - # option's default value is OFF. Fill it with real default values - set(tensorflow_NCCL_INCLUDE /usr/include) - endif (NOT tensorflow_NCCL_INCLUDE) - option(tensorflow_PATH_CUDNN_LIB "Override PATH_CUDA_LIB for cudnn" ${tensorflow_PATH_CUDA_LIB}) - if (NOT tensorflow_PATH_CUDNN_LIB) - # option's default value is OFF. Fill it with real default values - set (tensorflow_PATH_CUDNN_LIB ${tensorflow_PATH_CUDA_LIB}) - endif (NOT tensorflow_PATH_CUDNN_LIB) - option(tensorflow_PATH_NCCL_LIB "Override PATH_CUDA_LIB for nccl" ${tensorflow_PATH_CUDA_LIB}) - if (NOT tensorflow_PATH_NCCL_LIB) - # option's default value is OFF. Fill it with real default values - set (tensorflow_PATH_NCCL_LIB ${tensorflow_PATH_CUDA_LIB}) - endif (NOT tensorflow_PATH_NCCL_LIB) - option(tensorflow_CUDA_LIBRARY_PATH "Designate the default CUDA library paths" /usr/local/cuda/lib64) - if (NOT tensorflow_CUDA_LIBRARY_PATH) - # option's default value is OFF. Fill it with real default values - set(tensorflow_CUDA_LIBRARY_PATH /usr/local/cuda/lib64) - endif (NOT tensorflow_CUDA_LIBRARY_PATH) - - # Options for linking other libraries - option(systemlib_ZLIB "Use the system installed library as shared objects instead of downloading ZLIB and statically linking to it: ZLIB" OFF) - option(systemlib_ABSEIL_CPP "Use the system installed library as shared objects instead of downloading ABSEIL_CPP and statically linking to it: ABSEIL_CPP" OFF) - - option(systemlib_ALL "Turn on every possible systemlib_* options" OFF) - if (systemlib_ALL) - set (systemlib_ZLIB ON) - set (systemlib_ABSEIL_CPP ON) - endif (systemlib_ALL) -endif() - -if (WIN32) - set(BOOL_WIN32 ON) -else (WIN32) - set(BOOL_WIN32 OFF) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") -endif (WIN32) - -# [CLEANUP] Remove when done -# For debugging -function(SHOW_VARIABLES) - get_cmake_property(_variableNames VARIABLES) - foreach (_variableName ${_variableNames}) - message(STATUS "${_variableName}=${${_variableName}}") - endforeach() -endfunction() - -# External dependencies -set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external ${PROJECT_SOURCE_DIR}/modules) - -# Location where external projects will be downloaded -set (DOWNLOAD_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/downloads" - CACHE PATH "Location where external projects will be downloaded.") -mark_as_advanced(DOWNLOAD_LOCATION) - -if (tensorflow_ENABLE_POSITION_INDEPENDENT_CODE) - set(CMAKE_POSITION_INDEPENDENT_CODE ON) -else() - set(CMAKE_POSITION_INDEPENDENT_CODE OFF) -endif() - -# TODO(jart): We should make this only apply to snapfn.cc -add_definitions(-DSQLITE_OMIT_LOAD_EXTENSION) - -if (tensorflow_DISABLE_EIGEN_FORCEINLINE) - add_definitions(-DEIGEN_STRONG_INLINE=inline) -endif() - -add_definitions(-DEIGEN_AVOID_STL_ARRAY) -if(WIN32) - if(CMAKE_SIZEOF_VOID_P EQUAL 8) - # 64 bits - add_definitions(-DWIN64) - elseif(CMAKE_SIZEOF_VOID_P EQUAL 4) - # 32 bits - # temporary fix for #18241 - add_definitions(-DEIGEN_DEFAULT_DENSE_INDEX_TYPE=std::int64_t) - endif() - add_definitions(-DNOMINMAX -D_WIN32_WINNT=0x0A00) - add_definitions(-DWIN32_LEAN_AND_MEAN -DNOGDI -DPLATFORM_WINDOWS) - add_definitions(-DTENSORFLOW_USE_EIGEN_THREADPOOL -DEIGEN_HAS_C99_MATH) - add_definitions(-DTF_COMPILE_LIBRARY) - add_compile_options(/bigobj /GF /MP /Gm-) - # Suppress warnings to reduce build log size. - add_compile_options(/wd4267 /wd4244 /wd4800 /wd4503 /wd4554 /wd4996 /wd4348 /wd4018) - add_compile_options(/wd4099 /wd4146 /wd4267 /wd4305 /wd4307) - add_compile_options(/wd4715 /wd4722 /wd4723 /wd4838 /wd4309 /wd4334) - add_compile_options(/wd4003 /wd4244 /wd4267 /wd4503 /wd4506 /wd4800 /wd4996) - # Suppress linker warnings. - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /ignore:4049 /ignore:4197 /ignore:4217 /ignore:4221") - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} /ignore:4049 /ignore:4197 /ignore:4217 /ignore:4221") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /ignore:4049 /ignore:4197 /ignore:4217 /ignore:4221") - set(CMAKE_CXX_FLAGS_DEBUG "/D_DEBUG /MDd /Ob2") - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /D_ITERATOR_DEBUG_LEVEL=0") - set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} /D_ITERATOR_DEBUG_LEVEL=0") - set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /D_ITERATOR_DEBUG_LEVEL=0") - - set(compiler_flags - CMAKE_CXX_FLAGS - CMAKE_CXX_FLAGS_DEBUG - CMAKE_CXX_FLAGS_RELEASE - CMAKE_C_FLAGS - CMAKE_C_FLAGS_DEBUG - CMAKE_C_FLAGS_RELEASE - ) - # No exception - foreach(flag ${compiler_flags}) - string(REPLACE "/EHsc" "/EHs-c-" ${flag} "${${flag}}") - endforeach() - add_definitions(/D_HAS_EXCEPTIONS=0) - # Suppress 'noexcept used with no exception handling mode specified' warning - add_compile_options(/wd4577) - - # Try to avoid flaky failures due to failed generation of generate.stamp files. - set(CMAKE_SUPPRESS_REGENERATION ON) -endif() - - -if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions -std=c++11") -endif() - -if (tensorflow_OPTIMIZE_FOR_NATIVE_ARCH) - include(CheckCXXCompilerFlag) - CHECK_CXX_COMPILER_FLAG("-march=native" COMPILER_OPT_ARCH_NATIVE_SUPPORTED) - if (COMPILER_OPT_ARCH_NATIVE_SUPPORTED) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") - endif() -endif() - -include(CheckCXXCompilerFlag) - -# OpenMP Support -if (WIN32) - CHECK_CXX_COMPILER_FLAG("/openmp" MSVC_OPENMP_SUPPORT) - if (MSVC_OPENMP_SUPPORT) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /openmp") - endif() -else (WIN32) - CHECK_CXX_COMPILER_FLAG("-fopenmp" GCC_OPENMP_SUPPORT) - if (GCC_OPENMP_SUPPORT) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp") - endif() -endif (WIN32) - -# MSVC SIMD instructions -if (tensorflow_WIN_CPU_SIMD_OPTIONS) - if (WIN32) - CHECK_CXX_COMPILER_FLAG(${tensorflow_WIN_CPU_SIMD_OPTIONS} COMPILER_OPT_WIN_CPU_SIMD_SUPPORTED) - if(COMPILER_OPT_WIN_CPU_SIMD_SUPPORTED) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${tensorflow_WIN_CPU_SIMD_OPTIONS}") - endif() - endif() -endif() - -# External dependencies -include(zlib) -include(gif) -include(png) -include(jpeg) -include(lmdb) -include(eigen) -include(gemmlowp) -include(jsoncpp) -include(farmhash) -include(fft2d) -include(highwayhash) -include(nsync) -include(protobuf) -include(re2) -include(cub) -include(sqlite) -include(double_conversion) -include(abseil_cpp) -if (tensorflow_BUILD_CC_TESTS) - include(googletest) -endif() - -add_definitions(${ADD_CFLAGS}) -link_directories(${ADD_LINK_DIRECTORY}) - -set(tensorflow_EXTERNAL_LIBRARIES - ${tensorflow_EXTERNAL_LIBRARIES} - ${gif_STATIC_LIBRARIES} - ${png_STATIC_LIBRARIES} - ${jpeg_STATIC_LIBRARIES} - ${lmdb_STATIC_LIBRARIES} - ${jsoncpp_STATIC_LIBRARIES} - ${farmhash_STATIC_LIBRARIES} - ${fft2d_STATIC_LIBRARIES} - ${highwayhash_STATIC_LIBRARIES} - ${nsync_STATIC_LIBRARIES} - ${protobuf_STATIC_LIBRARIES} - ${re2_STATIC_LIBRARIES} - ${sqlite_STATIC_LIBRARIES} - ${double_conversion_STATIC_LIBRARIES} -) - -if (systemlib_ZLIB) - set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES} - ${ZLIB_LIBRARIES}) -else (systemlib_ZLIB) - set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES} - ${zlib_STATIC_LIBRARIES}) -endif (systemlib_ZLIB) - -if (systemlib_ABSEIL_CPP) - set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES} - ${abseil_cpp_LIBRARIES}) -else (systemlib_ABSEIL_CPP) - set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES} - ${abseil_cpp_STATIC_LIBRARIES}) -endif (systemlib_ABSEIL_CPP) - -set(tensorflow_EXTERNAL_DEPENDENCIES - zlib_copy_headers_to_destination - gif_copy_headers_to_destination - png_copy_headers_to_destination - jpeg_copy_headers_to_destination - lmdb_copy_headers_to_destination - jsoncpp - farmhash_copy_headers_to_destination - highwayhash_copy_headers_to_destination - nsync_copy_headers_to_destination - protobuf - eigen - gemmlowp - cub - fft2d - re2 - sqlite_copy_headers_to_destination - double_conversion -) - -include_directories( - # Source and generated code. - ${tensorflow_source_dir} - ${CMAKE_CURRENT_BINARY_DIR} - # External dependencies. - ${zlib_INCLUDE_DIR} - ${gif_INCLUDE_DIR} - ${png_INCLUDE_DIR} - ${jpeg_INCLUDE_DIR} - ${lmdb_INCLUDE_DIR} - ${eigen_INCLUDE_DIRS} - ${gemmlowp_INCLUDE_DIR} - ${jsoncpp_INCLUDE_DIR} - ${farmhash_INCLUDE_DIR} - ${highwayhash_INCLUDE_DIR} - ${cub_INCLUDE_DIR} - ${nsync_INCLUDE_DIR} - ${PROTOBUF_INCLUDE_DIRS} - ${re2_INCLUDE_DIR} - ${sqlite_INCLUDE_DIR} - ${double_conversion_INCLUDE_DIR} -) - -if(tensorflow_ENABLE_GRPC_SUPPORT) - if(tensorflow_ENABLE_SSL_SUPPORT) - include(boringssl) - include_directories(${boringssl_INCLUDE_DIR}) - endif() - include(grpc) - include_directories(${GRPC_INCLUDE_DIRS}) - # Place boringssl after grpc as grpc depends on boringssl. - list(APPEND tensorflow_EXTERNAL_LIBRARIES ${grpc_STATIC_LIBRARIES}) - list(APPEND tensorflow_EXTERNAL_DEPENDENCIES grpc) - if(tensorflow_ENABLE_SSL_SUPPORT) - list(APPEND tensorflow_EXTERNAL_LIBRARIES ${boringssl_STATIC_LIBRARIES}) - list(APPEND tensorflow_EXTERNAL_DEPENDENCIES boringssl) - endif() -endif() -if(tensorflow_ENABLE_SNAPPY_SUPPORT) - include(snappy) - list(APPEND tensorflow_EXTERNAL_LIBRARIES ${snappy_STATIC_LIBRARIES}) - list(APPEND tensorflow_EXTERNAL_DEPENDENCIES snappy) - include_directories(${snappy_INCLUDE_DIR}) -endif() -if(WIN32) - list(APPEND tensorflow_EXTERNAL_LIBRARIES wsock32 ws2_32 shlwapi) -endif() -if(UNIX) - list(APPEND tensorflow_EXTERNAL_LIBRARIES ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) -endif() -if(HAIKU) - list(APPEND tensorflow_EXTERNAL_LIBRARIES network) -endif() - -# MKL Support -if (tensorflow_ENABLE_MKL_SUPPORT) - add_definitions(-DINTEL_MKL -DEIGEN_USE_VML -DENABLE_MKL) - include(mkl) - list(APPEND tensorflow_EXTERNAL_LIBRARIES ${mkl_STATIC_LIBRARIES}) - list(APPEND tensorflow_EXTERNAL_DEPENDENCIES mkl_copy_shared_to_destination) - include_directories(${mkl_INCLUDE_DIRS}) - if (tensorflow_ENABLE_MKLDNN_SUPPORT) - include(mkldnn) - list(APPEND tensorflow_EXTERNAL_LIBRARIES ${mkldnn_STATIC_LIBRARIES}) - list(APPEND tensorflow_EXTERNAL_DEPENDENCIES mkldnn_copy_shared_to_destination) - include_directories(${mkldnn_INCLUDE_DIRS}) - endif(tensorflow_ENABLE_MKLDNN_SUPPORT) -endif (tensorflow_ENABLE_MKL_SUPPORT) - -if (tensorflow_ENABLE_GPU) - if (NOT WIN32) - # Default install paths for cuda libraries in Linux - # In some Linux distros, find_package(CUDA) seems to require CMAKE_LIBRARY_PATH to include cuda-lib paths - list(APPEND CMAKE_LIBRARY_PATH "${tensorflow_CUDA_LIBRARY_PATH}") - list(APPEND CMAKE_LIBRARY_PATH "${tensorflow_CUDA_LIBRARY_PATH}/stubs") - endif (NOT WIN32) - - # minimum 9.0 in cuda version - find_package(CUDA 9.0 REQUIRED) - if(NOT CUDA_FOUND) - message(FATAL_ERROR "CUDA not found.") - endif() - - # use cmake internal CUDA_ARCH_NAME switch - # e.g. CUDA_ARCH_NAME="Auto" will autodetect - # CUDA_ARCH_NAME="All" will use all arches - cuda_select_nvcc_arch_flags(NVCC_ARCH_FLAGS ${CUDA_ARCH_NAME}) - list(APPEND CUDA_NVCC_FLAGS ${NVCC_ARCH_FLAGS}) - message(STATUS "Using CUDA arch flags: ${NVCC_ARCH_FLAGS_readable}") - - set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};--include-path ${PROJECT_BINARY_DIR}/$\{build_configuration\};--expt-relaxed-constexpr) - set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-ftz=true) # Flush denormals to zero - set(CUDA_INCLUDE ${CUDA_TOOLKIT_TARGET_DIR} ${CUDA_TOOLKIT_TARGET_DIR}/extras/CUPTI/include) - - include_directories(${CUDA_INCLUDE}) - if (WIN32) - add_definitions(-DGOOGLE_CUDA=1 -DTF_EXTRA_CUDA_CAPABILITIES=3.7,5.2,6.0,6.1,7.0) - else (WIN32) - # Without these double quotes, cmake in Linux makes it "-DTF_EXTRA_CUDA_CAPABILITIES=3.7, -D5.2, ..." for cc, which incurs build breaks - add_definitions(-DGOOGLE_CUDA=1 -D"TF_EXTRA_CUDA_CAPABILITIES=3.7,5.2,6.0,6.1,7.0") - endif (WIN32) - - if (WIN32) - # add cudnn - if(NOT CUDNN_HOME) - set(CUDNN_HOME ${CUDA_TOOLKIT_TARGET_DIR}) - endif(NOT CUDNN_HOME) - set(CUDNN_INCLUDE "${CUDNN_HOME}/include") - - set(CUDA_LIBRARIES ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_CUFFT_LIBRARIES} - ${CUDA_curand_LIBRARY} ${CUDA_cupti_LIBRARY} ${CUDA_cusolver_LIBRARY} ${CUDNN_HOME}/lib/x64/cudnn.lib) - else (WIN32) - set(CUDNN_INCLUDE "${tensorflow_CUDNN_INCLUDE}") - - if (tensorflow_BUILD_SHARED_LIB) - find_library(nccl_LIBRARY NAMES libnccl.so PATHS ${tensorflow_PATH_NCCL_LIB} ${CUDA_TOOLKIT_ROOT_DIR}) - else (tensorflow_BUILD_SHARED_LIB) - find_library(nccl_LIBRARY NAMES libnccl_static.a PATHS ${tensorflow_PATH_NCCL_LIB} ${CUDA_TOOLKIT_ROOT_DIR}) - endif (tensorflow_BUILD_SHARED_LIB) - if (NOT nccl_LIBRARY) - message(FATAL_ERROR "NCCL is required for GPU-build") - else (NOT nccl_LIBRARY) - message("nccl: ${nccl_LIBRARY}") - # something like /usr/lib64/libnccl_static.a - endif (NOT nccl_LIBRARY) - - if (tensorflow_BUILD_SHARED_LIB) - find_library(cudnn_LIBRARY NAMES libcudnn.so PATHS ${tensorflow_PATH_CUDNN_LIB} ${CUDA_TOOLKIT_ROOT_DIR}) - else (tensorflow_BUILD_SHARED_LIB) - find_library(cudnn_LIBRARY NAMES libcudnn_static.a PATHS ${tensorflow_PATH_CUDNN_LIB} ${CUDA_TOOLKIT_ROOT_DIR}) - endif (tensorflow_BUILD_SHARED_LIB) - if (NOT cudnn_LIBRARY) - message(FATAL_ERROR "CUDNN is required for GPU-build") - else (NOT cudnn_LIBRARY) - file(READ ${CUDNN_INCLUDE}/cudnn.h CUDNN_VERSION_FILE_CONTENTS) - # fetch cudnn version - string(REGEX MATCH "define CUDNN_MAJOR * +([0-9]+)" - CUDNN_VERSION_MAJOR "${CUDNN_VERSION_FILE_CONTENTS}") - string(REGEX REPLACE "define CUDNN_MAJOR * +([0-9]+)" "\\1" - CUDNN_VERSION_MAJOR "${CUDNN_VERSION_MAJOR}") - string(REGEX MATCH "define CUDNN_MINOR * +([0-9]+)" - CUDNN_VERSION_MINOR "${CUDNN_VERSION_FILE_CONTENTS}") - string(REGEX REPLACE "define CUDNN_MINOR * +([0-9]+)" "\\1" - CUDNN_VERSION_MINOR "${CUDNN_VERSION_MINOR}") - string(REGEX MATCH "define CUDNN_PATCHLEVEL * +([0-9]+)" - CUDNN_VERSION_PATCH "${CUDNN_VERSION_FILE_CONTENTS}") - string(REGEX REPLACE "define CUDNN_PATCHLEVEL * +([0-9]+)" "\\1" - CUDNN_VERSION_PATCH "${CUDNN_VERSION_PATCH}") - if(NOT CUDNN_VERSION_MAJOR) - set(CUDNN_VERSION "???") - else() - set(CUDNN_VERSION "${CUDNN_VERSION_MAJOR}.${CUDNN_VERSION_MINOR}.${CUDNN_VERSION_PATCH}") - endif() - message(STATUS "cudnn library: ${cudnn_LIBRARY} (found version: \"${CUDNN_VERSION}\")") - endif (NOT cudnn_LIBRARY) - - if (tensorflow_BUILD_SHARED_LIB) - # shared first (if exists) else static one - find_library(culibos_LIBRARY NAMES libculibos.so libculibos.a PATHS ${tensorflow_PATH_CUDA_LIB} ${CUDA_TOOLKIT_ROOT_DIR}) - else (tensorflow_BUILD_SHARED_LIB) - # only static version - find_library(culibos_LIBRARY NAMES libculibos.a PATHS ${tensorflow_PATH_CUDA_LIB} ${CUDA_TOOLKIT_ROOT_DIR}) - endif (tensorflow_BUILD_SHARED_LIB) - if (NOT culibos_LIBRARY) - message(FATAL_ERROR "CULIBOS is required for GPU-build") - else (NOT culibos_LIBRARY) - message("culibos: ${culibos_LIBRARY}") - endif (NOT culibos_LIBRARY) - - set(CUDA_LIBRARIES ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_CUFFT_LIBRARIES} - ${CUDA_curand_LIBRARY} ${CUDA_cupti_LIBRARY} ${CUDA_cusolver_LIBRARY} ${cudnn_LIBRARY} ${culibos_LIBRARY} ${nccl_LIBRARY}) - endif (WIN32) - include_directories(${CUDNN_INCLUDE}) - - # Remove "." from CUDA version variable. - string(REPLACE "." "" short_CUDA_VER ${CUDA_VERSION}) - - # List of enumerated CUDA caps - string(REPLACE " " ";" NVCC_ARCH_LIST "${NVCC_ARCH_FLAGS_readable}") - set(list ${NVCC_ARCH_LIST}) - - # Construct capability string - foreach(NVCC_ARCH ${NVCC_ARCH_LIST}) - if (NVCC_ARCH MATCHES "sm_") - string(REGEX REPLACE "^.sm*" "" NVCC_ARCH ${NVCC_ARCH}) - math(EXPR NVCC_ARCH_MAJOR "${NVCC_ARCH} / 10") - math(EXPR NVCC_ARCH_MINOR "(${NVCC_ARCH} - (${NVCC_ARCH_MAJOR}*10))") - if (TF_CUDA_CAP) - set(TF_CUDA_CAP "${TF_CUDA_CAP},CudaVersion(\"${NVCC_ARCH_MAJOR}.${NVCC_ARCH_MINOR}\")") - else (TF_CUDA_CAP) - set(TF_CUDA_CAP "CudaVersion(\"${NVCC_ARCH_MAJOR}.${NVCC_ARCH_MINOR}\")") - endif (TF_CUDA_CAP) - endif() - endforeach() - - # create cuda_config.h - FILE(WRITE ${tensorflow_source_dir}/third_party/gpus/cuda/cuda_config.h - "#ifndef CUDA_CUDA_CONFIG_H_\n" - "#define CUDA_CUDA_CONFIG_H_\n" - "#define TF_CUDA_CAPABILITIES ${TF_CUDA_CAP}\n" - "#define TF_CUDA_VERSION \"64_${short_CUDA_VER}\"\n" - "#define TF_CUDA_LIB_VERSION \"64_${short_CUDA_VER}\"\n" - "#define TF_CUDNN_VERSION \"64_${CUDNN_VERSION}\"\n" - "#define TF_CUDA_TOOLKIT_PATH \"${CUDA_TOOLKIT_ROOT_DIR}\"\n" - "#endif // CUDA_CUDA_CONFIG_H_\n" - ) - - # tf assumes in various places header files to be in cuda/include. On windows the cuda sdk - # installs them under cuda/version/include and to avoid that we need to change tf we copy a - # few files to cuda/include - FILE(COPY - ${CUDA_TOOLKIT_TARGET_DIR}/include/cuda.h - ${CUDA_TOOLKIT_TARGET_DIR}/include/cuComplex.h - ${CUDA_TOOLKIT_TARGET_DIR}/include/cublas_v2.h - ${CUDA_TOOLKIT_TARGET_DIR}/include/cusolverDn.h - ${CUDA_TOOLKIT_TARGET_DIR}/include/device_functions.h - ${CUDA_TOOLKIT_TARGET_DIR}/include/cufft.h - ${CUDA_TOOLKIT_TARGET_DIR}/include/curand.h - ${CUDA_TOOLKIT_TARGET_DIR}/include/cuda_runtime_api.h - ${CUDNN_INCLUDE}/cudnn.h - DESTINATION ${tensorflow_source_dir}/third_party/gpus/cuda/include - ) - - include_directories(${tensorflow_source_dir}/third_party/gpus) - # add cuda libraries to tensorflow_EXTERNAL_LIBRARIES - list(APPEND tensorflow_EXTERNAL_LIBRARIES ${CUDA_LIBRARIES}) - if(NOT WIN32) - # add gomp to tensorflow_EXTERNAL_LIBRARIES, needed by libcusolver.so - list(APPEND tensorflow_EXTERNAL_LIBRARIES gomp) - endif() - - # NOTE(mrry): Update these flags when the version of CUDA or cuDNN used - # in the default build is upgraded. - if(WIN32) - set(tensorflow_BUILD_INFO_FLAGS --build_config cuda --key_value - msvcp_dll_name=msvcp140.dll - cudart_dll_name=cudart64_${short_CUDA_VER}.dll - cuda_version_number=${CUDA_VERSION} - nvcuda_dll_name=nvcuda.dll - cudnn_dll_name=cudnn64_${tensorflow_CUDNN_VERSION}.dll - cudnn_version_number=${tensorflow_CUDNN_VERSION}) - else(WIN32) - set(tensorflow_BUILD_INFO_FLAGS --build_config cuda --key_value - cuda_version_number=${CUDA_VERSION} - cudnn_version_number=${tensorflow_CUDNN_VERSION}) - endif(WIN32) -else(tensorflow_ENABLE_GPU) - if(WIN32) - set(tensorflow_BUILD_INFO_FLAGS --build_config cpu --key_value - msvcp_dll_name=msvcp140.dll) - else() - set(tensorflow_BUILD_INFO_FLAGS --build_config cpu) - endif() -endif(tensorflow_ENABLE_GPU) - -if(tensorflow_BUILD_PYTHON_BINDINGS) - # Find python executable - include(FindPythonInterp) - if(NOT ${PYTHONINTERP_FOUND}) - message(FATAL_ERROR "CMake was unable to find a python interpreter.") - endif() -endif() - -# Let's get to work! -include(tf_core_framework.cmake) -if (tensorflow_ENABLE_GPU) - include(tf_stream_executor.cmake) -endif() - -include(tf_core_cpu.cmake) -include(tf_core_ops.cmake) -include(tf_core_direct_session.cmake) -include(tf_core_kernels.cmake) -if(tensorflow_ENABLE_GRPC_SUPPORT) - include(tf_core_distributed_runtime.cmake) -endif() -# We include tf_cc_ops first, because tf_c depends on tf_cc. -include(tf_cc_ops.cmake) -include(tf_c.cmake) -include(tf_grappler.cmake) -include(tf_core_profiler.cmake) -include(tf_core_eager_runtime.cmake) -if(tensorflow_BUILD_CC_EXAMPLE) - include(tf_tutorials.cmake) - include(tf_label_image_example.cmake) -endif() -include(tf_tools.cmake) -if(tensorflow_BUILD_PYTHON_BINDINGS) - include(tf_python.cmake) -endif() -if(tensorflow_BUILD_SHARED_LIB) - include(tf_shared_lib.cmake) -endif() -if(tensorflow_BUILD_CC_TESTS OR tensorflow_BUILD_PYTHON_TESTS) - include(tf_tests.cmake) -endif() diff --git a/tensorflow/contrib/cmake/README.md b/tensorflow/contrib/cmake/README.md deleted file mode 100644 index 9e9d85def83..00000000000 --- a/tensorflow/contrib/cmake/README.md +++ /dev/null @@ -1,522 +0,0 @@ -TensorFlow CMake build -====================== - -CMAKE build is deprecated for TensorFlow. Please use `bazel` to build TF for all -platforms. For details, see the -[TensorFlow install guide](https://www.tensorflow.org/install/). - -This directory contains CMake files for building TensorFlow on Microsoft Windows -and Linux. [CMake](https://cmake.org) is a cross-platform tool that can generate -build scripts for multiple build systems, including Microsoft Visual Studio and -GCC. "The method has not been tested on Mac OS X. - -**N.B.** We provide Linux build instructions primarily for the purpose of -testing the build. We recommend using the standard Bazel-based build on -Linux. - -Current Status --------------- - -CMake can be used to build TensorFlow on all platforms. See the -[getting started documentation](https://www.tensorflow.org/install/install_windows) -for instructions on how to install a pre-built TensorFlow package on Windows and -Linux. The procedure in MacOS is similar to the Linux build. - -### Current known limitations - -* It is not possible to load a custom Op library. -* GCS file system is not supported. -* Debug build is not available since Python for Windows is no longer - distributed with a debug library. - -## Building with CMake - -The CMake files in this directory can build the core TensorFlow runtime, an -example C++ binary, and a PIP package containing the runtime and Python -bindings. - -### Prerequisites - -* CMake version 3.5 or later. - -* [Git](https://git-scm.com) - -* [SWIG](http://www.swig.org/download.html) - -* [Perl](https://www.perl.org/get.html) (optional, for SSL support build) - -* [Go](https://golang.org/) (optional, for SSL support build) - -* [NASM](http://www.nasm.us/)/[YASM](http://yasm.tortall.net/) (optional, for - SSL support build) - -* Additional pre-requisites for Microsoft Windows: - - - Visual Studio 2015 (latest version of MSVC 2017 is not supported by CUDA - yet, try it on your own risk) - - - Python 3.5 - -* Additional prerequisites for Linux: - - - Python 2.7 or later - - [Docker](https://www.docker.com/) (for automated testing) - -* Python dependencies: - - - wheel - - NumPy 1.11.0 or later - -### Known-good configurations - -* Microsoft Windows 10 - - - Microsoft Visual Studio Enterprise/ Community 2015 with Visual C++ 2015 - - [Anaconda 4.1.1 (Python 3.5 64-bit)](https://www.anaconda.com/download/) - - [Git for Windows version 2.9.2.windows.1](https://git-scm.com/download/win) - - [swigwin-3.0.10](http://www.swig.org/download.html) - - [NVidia CUDA Toolkit 9.0](https://developer.nvidia.com/cuda-downloads) - - [NVidia CUDNN 7](https://developer.nvidia.com/cudnn) - - [CMake 3.6](https://cmake.org/files/v3.6/cmake-3.6.3-win64-x64.msi) - -* Ubuntu 14.04 - - - Makefile generator - - Docker 1.9.1 (for automated testing) - -### Current known limitations - -- The Python package supports **Python 3.5/3.6 only**, because these are the - only versions for which standard Python binaries exist and those binaries - are compatible with the TensorFlow runtime. (On Windows, the standard Python - binaries for versions earlier than 3.5 were compiled with older compilers - that do not have all of the features (e.g. C++11 support) needed to compile - TensorFlow. We welcome patches for making TensorFlow work with Python 2.7 on - Windows, but have not yet committed to supporting that configuration.) - -- The following Python APIs are not currently implemented: - - * Loading custom op libraries via `tf.load_op_library()`. In order to use - your custom op, please put the source code under the - tensorflow/core/user_ops directory, and a shape function is required - (not optional) for each op. - * Path manipulation functions (such as `tf.gfile.ListDirectory()`) are not - functional. - -- The `tf.contrib` libraries are not currently included in the PIP package. - -- The following operations are not currently implemented: - - * `DepthwiseConv2dNative` - * `Digamma` - * `Erf` - * `Erfc` - * `Igamma` - * `Igammac` - * `ImmutableConst` - * `Lgamma` - * `Polygamma` - * `Zeta` - -- Google Cloud Storage support is not currently implemented. The GCS library - currently depends on `libcurl` and `boringssl`, and the Windows version - could use standard Windows APIs for making HTTP requests and cryptography - (for OAuth). Contributions are welcome for this feature. - -We are actively working on improving CMake and Windows support, and addressing -these limitations. We would appreciate pull requests that implement missing -ops or APIs. - -# CMake GUI build (all platforms) - -Install from CMake GUI would be a convenient way to generate C++ build projects. -The software supports Windows, MacOS and Linux, while the posix platform -provides an extra ccmake binary to run command line GUI. Both working principal -of cmake, ccmake and cmake-gui are the same, the only difference is by providing -suitable interface for project configuration and dependency setting. - -1. Pre-buid checklist: The following binary/libraries should be setted in - system path, otherwise you need to set manualy via cmake. - * Compiler (GCC for Linux, MSVC for Windows) - * Make sure compiler directory has been set to system path - * CUDA 9.0 (GPU build) - * CUDNN (GPU build) - * NCCL (GPU build on Linux) - * SWIG (python binding) - * Perl (required if you need ssl support, optional) - * Go (required if you need ssl support, optional) - * NASM/YASM (required by grpc for ssl support, optional) -2. Start CMake GUI -3. Click on `Browse Source` and direct to the folder - `/tensorflow/contrib/cmake` -4. Click on `Browse Build` and spectify a location that you want tensorflow to - be build -5. Click on `Configure`, a new window will be prompted out, specify the - generator mode for the project generation. For Windows, choose `Visual - Studio Win64`, for Linux, choose `Unix Makefiles`, then - press `Finish`. Wait for a moment, the default project dependency would - automatically generate. -6. There are a few options that you can customize your own build. **The setting - here is crucial for a successful build, please check all items carefully.** - - * `tensorflow_BUILD_ALL_KERNELS` should always be `on` - * `tensorflow_BUILD_CC_EXAMPLE` is default to be `on`. This can help you - to test build (optional) - * `tensorflow_BUILD_CONTRIB_KERNELS` is default to be `on`, but it won't - affect tensorflow function, turn it to `off` if you want a slim build. - (optional) - * `tensorflow_BUILD_PYTHON_BINDING` is default to be `on`. Set to `off` if - you don't need python interaface. If SWIG is not in system path, you - need set it manually. (optional) - * `tensorflow_BUILD_SHARED_LIB` is default to be `off`. Set to `on` if you - want the c++ interface. (optional) - * `tensorflow_ENABLE_GPU` is default to be `off`. Set to `on` if you want - GPU support. It will search CUDA and CUDNN dependecies if you have set - them to system path, otherwise CMake would prompt error and request you - to set it manually. (optional) - * `tensorflow_ENABLE_GRPC_SUPPORT` is default to be `on`. For Linux build, - this option must always be `on`. This need to be `on` for a gpu build. - Reminded that Perl, Go and NASM/YASM are required for this option if you - want to build grpc with offical SSL support. - * `tensorflow_ENABLE_POSITION_INDEPENDENT_CODE` should always be `on` - * `tensorflow_ENABLE_SNAPPY_SUPPORT` should always be `on` - * `tensorflow_OPTIMIZE_FOR_NATIVE_ARCH` should always be `on` - * `CMAKE_INSTALL_PREFIX` is the location where the final package will be - installed. You may change it to your own preferred path (optional) - -7. After changing the configuration in step 5, press `Configure` again - -8. If not error is found, press `Generate` - -#### Windows - -1. Open `tensorflow.sln` in the build folder (Windows). Change build type from - `Debug` to `Release`. Choose `Build`->`Build Solution`. This may take more - than hours of compilation. If everything is alright, the output window would - show no error. - - ##### Python - - In solution explorer, right click on `tf_python_build_pip_package` -> - `build`. It will generate the wheel file in - `/tf_python/dist`. Install with following command: - - `pip install --upgrade tensorflow-.whl` - - ***The wheel name varies depends on you config. Change to your own wheel - filename.*** - - Reminded that some pip installation requires administrator right command - prompt. - - ##### C++ - - You can directly use the build folder tree for C++ interface with cmake. If - you want to do installation for api releasing, right click on `Install` -> - `build`. The headers and library will be installed in the directory specify - by `CMAKE_INSTALL_PREFIX` during configuration. - -1. For smaller RAM computer, it is noticed that out of heap space error - appears. Change to command prompt build is an alternative to do step 1. - - Open `VS2015 x64 Native Tools Command Prompt`. You can open it by press - `Start`, then type the binary name. Use `VS2017 x64 Native Tools Command - Prompt` if you are using MSVC 2017. - - ##### Python - - Directly build python wheel package by following command: - - `MSBuild /p:Configuration=Release - ` - - Remember to change `` to the - actual path of the file, it can be found at the root of build directory - - Install the wheel file generated as instructed by step 1. - - ##### C++ interface - - Build from VS native toolchain with following command: `MSBuild - /p:Configuration=Release ` - - Headers are discretely located in the build folders. Tensorflow library can - be found at `/Release`, namely `tensorflow.dll` and - `tensorflow.lib`. - - * Build to install for api release (optional): `MSBuild - /p:Configuration=Release ` - - Remember to change `` and - `` to the actual path of the file, it can be found - at the root of build directory. - -#### Linux/MacOS (command line GNU build) - -1. Open the terminal, change working directory to the one specified in step 3. - -2. Type the following command: - - `make -sj all` - - ##### Python - - **Important Note** CMake generated python wheel for Linux/MacOs is currently - under development. Please use bazel build. - - Follow code is an expected Linux/MacOS python package build after - development work is completed. - - ``` - make -sj tf_python_build_pip_package - cd tf_python - pip install --upgrade tensorflow-.whl - ``` - - ##### C++ interface - - `make -sj install` - - Where `` is the threads used for the compilation, change - to any integer less or equal to your computer's maximum thread number. - - Headers are discretely located in the build folders. Tensorflow library can - be found at ``, namely `tensorflow.so` (Linux) or - `tensorflow.dylib` (MacOS). - -#### Start a Tensorflow C++ project with CMake - -Here we assume that you have basic knowledge on gathering dependency with -`CMakeLists.txt`. Here we introduce how the C++ api works with -[official hello world tutorial](https://www.tensorflow.org/api_guides/cc/guide). - -1. Create a new working directory and create a new text file named - `CMakeLists.txt` and the c++ file `main.cxx` -2. Fill in the `main.cxx` with the code provided in - [official c++ api basic](https://www.tensorflow.org/api_guides/cc/guide). -3. Fill in the `CMakeLists.txt` with following code: - - ```cmake - cmake_minimum_required (VERSION 2.6) project (tf_hello) - - # Tensorflow - find_package(Tensorflow REQUIRED) - include_directories(${TENSORFLOW_INCLUDE_DIRS}) - - # compiler setting required by tensorflow, to be tested on all compilers - - # currently only tested on MSVC and GCC - - if (${CMAKE_CXX_COMPILER_ID} STREQUAL MSVC) add_definitions(-DCOMPILER_MSVC) - elseif (${CMAKE_CXX_COMPILER_ID} STREQUAL GNU) if - (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS "3") - add_definitions(-DCOMPILER_GCC3) else() add_definitions(-D__GNUC__) endif() - else() message(ERROR " compiler ${CMAKE_CXX_COMPILER_ID} not supported by - this CMakeList.txt, under development") endif() - - add_executable(tf_hello main.cxx) target_link_libraries(tf_hello - ${TENSORFLOW_LIBRARIES}) - ``` - -4. Configure the folder with cmake-gui, an error should be prompted out, - requesting you to locate the folder containing `TensorflowConfig.cmake`. - This file can be found at `` or `` (for - those have build install in previous steps). - -5. Configure again, generate the project. - -6. Compile the project with `Release` config (Windows). For Linux users, just - compile the project. - -7. Copy the `tensorflow.dll`(Windows)/`tensorflow.so`(Linux) from build - directory to the build folder containing `tf_hello` binary. - -8. Run `tf_hello` binary - -# Step-by-step Windows build (command prompt) - -1. Install the prerequisites detailed above, and set up your environment. - - * When building with GPU support after installing the CUDNN zip file from - NVidia, append its bin directory to your PATH environment variable. In - case TensorFlow fails to find the CUDA dll's during initialization, - check your PATH environment variable. It should contain the directory of - the CUDA dlls and the directory of the CUDNN dll. For example: - - ``` - D:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v8.0\bin - D:\local\cuda\bin - ``` - - * When building with MKL support after installing - [MKL](https://software.intel.com/en-us/mkl) from INTEL, append its bin - directories to your PATH environment variable. - - In case TensorFlow fails to find the MKL dll's during initialization, - check your PATH environment variable. It should contain the directory of - the MKL dlls. For example: - - ``` - D:\Tools\IntelSWTools\compilers_and_libraries\windows\redist\intel64\mkl - D:\Tools\IntelSWTools\compilers_and_libraries\windows\redist\intel64\compiler - D:\Tools\IntelSWTools\compilers_and_libraries\windows\redist\intel64\tbb\vc_mt - ``` - - * We assume that `cmake` and `git` are installed and in your `%PATH%`. If - for example `cmake` is not in your path and it is installed in - `C:\Program Files (x86)\CMake\bin\cmake.exe`, you can add this directory - to your `%PATH%` as follows: - - ``` - D:\temp> set PATH="%PATH%;C:\Program Files (x86)\CMake\bin\cmake.exe" - ``` - -2. Clone the TensorFlow repository and create a working directory for your - build: - - ``` - D:\temp> git clone https://github.com/tensorflow/tensorflow.git - D:\temp> cd tensorflow\tensorflow\contrib\cmake - D:\temp\tensorflow\tensorflow\contrib\cmake> mkdir build - D:\temp\tensorflow\tensorflow\contrib\cmake> cd build - D:\temp\tensorflow\tensorflow\contrib\cmake\build> - ``` - -3. Invoke CMake to create Visual Studio solution and project files. - - **N.B.** This assumes that `cmake.exe` is in your `%PATH%` environment - variable. The other paths are for illustrative purposes only, and may be - different on your platform. The `^` character is a line continuation and - must be the last character on each line. - - ``` - D:\...\build> cmake .. -A x64 -Thost=x64 -DCMAKE_BUILD_TYPE=Release ^ - More? -DSWIG_EXECUTABLE=C:/tools/swigwin-3.0.10/swig.exe ^ - More? -DPYTHON_EXECUTABLE=C:/Users/%USERNAME%/AppData/Local/Continuum/Anaconda3/python.exe ^ - More? -DPYTHON_LIBRARIES=C:/Users/%USERNAME%/AppData/Local/Continuum/Anaconda3/libs/python35.lib - ``` - - To build with GPU support add "^" at the end of the last line above - following with: `More? -Dtensorflow_ENABLE_GPU=ON ^ More? - -DCUDNN_HOME="D:\...\cudnn"` To build with MKL support add "^" at the end of - the last line above following with: - - ``` - More? -Dtensorflow_ENABLE_MKL_SUPPORT=ON ^ - More? -DMKL_HOME="D:\...\compilers_and_libraries" - ``` - - To enable SIMD instructions with MSVC, as AVX and SSE, define it as follows: - - ``` - More? -Dtensorflow_WIN_CPU_SIMD_OPTIONS=/arch:AVX - ``` - - Note that the `-DCMAKE_BUILD_TYPE=Release` flag must match the build - configuration that you choose when invoking `msbuild`. The known-good values - are `Release` and `RelWithDebInfo`. The `Debug` build type is not currently - supported, because it relies on a `Debug` library for Python - (`python35d.lib`) that is not distributed by default. - - The `-Thost=x64` flag will ensure that the 64 bit compiler and linker is - used when building. Without this flag, MSBuild will use the 32 bit toolchain - which is prone to compile errors such as "compiler out of heap space". - - There are various options that can be specified when generating the solution - and project files: - - * `-DCMAKE_BUILD_TYPE=(Release|RelWithDebInfo)`: Note that the - `CMAKE_BUILD_TYPE` option must match the build configuration that you - choose when invoking MSBuild in step 4. The known-good values are - `Release` and `RelWithDebInfo`. The `Debug` build type is not currently - supported, because it relies on a `Debug` library for Python - (`python35d.lib`) that is not distributed by default. - - * `-Dtensorflow_BUILD_ALL_KERNELS=(ON|OFF)`. Defaults to `ON`. You can - build a small subset of the kernels for a faster build by setting this - option to `OFF`. - - * `-Dtensorflow_BUILD_CC_EXAMPLE=(ON|OFF)`. Defaults to `ON`. Generate - project files for a simple C++ - [example training program](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/tutorials/example_trainer.cc). - - * `-Dtensorflow_BUILD_PYTHON_BINDINGS=(ON|OFF)`. Defaults to `ON`. - Generate project files for building a PIP package containing the - TensorFlow runtime and its Python bindings. - - * `-Dtensorflow_ENABLE_GRPC_SUPPORT=(ON|OFF)`. Defaults to `ON`. Include - gRPC support and the distributed client and server code in the - TensorFlow runtime. - - * `-Dtensorflow_ENABLE_SSL_SUPPORT=(ON|OFF)`. Defaults to `OFF`. Include - SSL support (for making secure HTTP requests) in the TensorFlow runtime. - This support is incomplete, and will be used for Google Cloud Storage - support. - - * `-Dtensorflow_ENABLE_GPU=(ON|OFF)`. Defaults to `OFF`. Include GPU - support. If GPU is enabled you need to install the CUDA 8.0 Toolkit and - CUDNN 5.1. CMake will expect the location of CUDNN in - -DCUDNN_HOME=path_you_unzipped_cudnn. - - * `-Dtensorflow_BUILD_CC_TESTS=(ON|OFF)`. Defaults to `OFF`. This builds - cc unit tests. There are many of them and building will take a few - hours. After cmake, build and execute the tests with `MSBuild - /p:Configuration=RelWithDebInfo ALL_BUILD.vcxproj ctest -C - RelWithDebInfo` - - * `-Dtensorflow_BUILD_PYTHON_TESTS=(ON|OFF)`. Defaults to `OFF`. This - enables python kernel tests. After building the python wheel, you need - to install the new wheel before running the tests. To execute the tests, - use `ctest -C RelWithDebInfo` - - * `-Dtensorflow_BUILD_MORE_PYTHON_TESTS=(ON|OFF)`. Defaults to `OFF`. This - enables python tests on serveral major packages. This option is only - valid if this and tensorflow_BUILD_PYTHON_TESTS are both set as `ON`. - After building the python wheel, you need to install the new wheel - before running the tests. To execute the tests, use `ctest -C - RelWithDebInfo` - - * `-Dtensorflow_ENABLE_MKL_SUPPORT=(ON|OFF)`. Defaults to `OFF`. Include - MKL support. If MKL is enabled you need to install the - [Intel Math Kernal Library](https://software.intel.com/en-us/mkl). CMake - will expect the location of MKL in -MKL_HOME=path_you_install_mkl. - - * `-Dtensorflow_ENABLE_MKLDNN_SUPPORT=(ON|OFF)`. Defaults to `OFF`. - Include MKL DNN support. MKL DNN is [Intel(R) Math Kernel Library for - Deep Neural Networks (Intel(R) - MKL-DNN)](https://github.com/intel/mkl-dnn). You have to add - `-Dtensorflow_ENABLE_MKL_SUPPORT=ON` before including MKL DNN support. - -4. Invoke MSBuild to build TensorFlow. - - Set up the path to find MSbuild: `D:\temp> "C:\Program Files (x86)\Microsoft - Visual Studio 14.0\VC\bin\amd64\vcvarsall.bat"` - - To build the C++ example program, which will be created as a `.exe` - executable in the subdirectory `.\Release`: - - ``` - D:\...\build> MSBuild /p:Configuration=Release tf_tutorials_example_trainer.vcxproj - D:\...\build> Release\tf_tutorials_example_trainer.exe - ``` - - To build the PIP package, which will be created as a `.whl` file in the - subdirectory `.\tf_python\dist`: - - ``` - D:\...\build> MSBuild /p:Configuration=Release tf_python_build_pip_package.vcxproj - ``` - -Linux Continuous Integration build -================================== - -This build requires [Docker](https://www.docker.com/) to be installed on the -local machine. - -```bash -$ git clone --recursive https://github.com/tensorflow/tensorflow.git -$ cd tensorflow -$ tensorflow/tools/ci_build/ci_build.sh CMAKE tensorflow/tools/ci_build/builds/cmake.sh -``` - -That's it. Dependencies included. diff --git a/tensorflow/contrib/cmake/TensorflowConfig.cmake.in b/tensorflow/contrib/cmake/TensorflowConfig.cmake.in deleted file mode 100644 index cc04db6e952..00000000000 --- a/tensorflow/contrib/cmake/TensorflowConfig.cmake.in +++ /dev/null @@ -1,16 +0,0 @@ -# - Config file for the Tensorflow package -# It defines the following variables -# TENSORFLOW_INCLUDE_DIRS - include directories for FooBar -# TENSORFLOW_LIBRARIES - libraries to link against - -# Compute paths -get_filename_component(TENSORFLOW_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) -set(TENSORFLOW_INCLUDE_DIRS "@CONF_INCLUDE_DIRS@") - -# Our library dependencies (contains definitions for IMPORTED targets) -if(NOT TENSORFLOW_BINARY_DIR) - include("${TENSORFLOW_CMAKE_DIR}/TensorflowTargets.cmake") -endif() - -# These are IMPORTED targets created by TensorflowTargets.cmake -set(TENSORFLOW_LIBRARIES tensorflow) \ No newline at end of file diff --git a/tensorflow/contrib/cmake/TensorflowConfigVersion.cmake.in b/tensorflow/contrib/cmake/TensorflowConfigVersion.cmake.in deleted file mode 100644 index 2a9609ddb9c..00000000000 --- a/tensorflow/contrib/cmake/TensorflowConfigVersion.cmake.in +++ /dev/null @@ -1,11 +0,0 @@ -set(PACKAGE_VERSION "@TENSORFLOW_VERSION@") - -# Check whether the requested PACKAGE_FIND_VERSION is compatible -if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") - set(PACKAGE_VERSION_COMPATIBLE FALSE) -else() - set(PACKAGE_VERSION_COMPATIBLE TRUE) - if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") - set(PACKAGE_VERSION_EXACT TRUE) - endif() -endif() \ No newline at end of file diff --git a/tensorflow/contrib/cmake/external/abseil_cpp.cmake b/tensorflow/contrib/cmake/external/abseil_cpp.cmake deleted file mode 100644 index 53ad3648d61..00000000000 --- a/tensorflow/contrib/cmake/external/abseil_cpp.cmake +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -if (systemlib_ABSEIL_CPP) - - find_package(AbseilCpp REQUIRED - absl_base - absl_spinlock_wait - absl_dynamic_annotations - absl_malloc_internal - absl_throw_delegate - absl_int128 - absl_strings - str_format_internal - absl_bad_optional_access) - - include_directories(${ABSEIL_CPP_INCLUDE_DIR}) - list(APPEND tensorflow_EXTERNAL_LIBRARIES ${ABSEIL_CPP_LIBRARIES}) - - message(STATUS " abseil_cpp includes: ${ABSEIL_CPP_INCLUDE_DIR}") - message(STATUS " abseil_cpp libraries: ${ABSEIL_CPP_LIBRARIES}") - - add_custom_target(abseil_cpp_build) - list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build) - -else (systemlib_ABSEIL_CPP) - - include (ExternalProject) - - set(abseil_cpp_INCLUDE_DIR ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp_build) - set(abseil_cpp_URL https://github.com/abseil/abseil-cpp.git) - set(abseil_cpp_TAG master) - set(abseil_cpp_BUILD ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp_build) - - if(WIN32) - if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(abseil_cpp_STATIC_LIBRARIES - ${abseil_cpp_BUILD}/absl/base/Release/absl_base.lib - ${abseil_cpp_BUILD}/absl/base/Release/absl_dynamic_annotations.lib - ${abseil_cpp_BUILD}/absl/base/Release/absl_malloc_internal.lib - ${abseil_cpp_BUILD}/absl/base/Release/absl_internal_throw_delegate.lib - ${abseil_cpp_BUILD}/absl/numeric/Release/absl_int128.lib - ${abseil_cpp_BUILD}/absl/strings/Release/absl_strings.lib - ${abseil_cpp_BUILD}/absl/strings/Release/str_format_internal.lib - ${abseil_cpp_BUILD}/absl/time/Release/absl_time.lib - ${abseil_cpp_BUILD}/absl/types/Release/absl_bad_optional_access.lib) - else() - set(abseil_cpp_STATIC_LIBRARIES - ${abseil_cpp_BUILD}/absl/base/absl_base.lib - ${abseil_cpp_BUILD}/absl/base/absl_spinlock_wait.lib - ${abseil_cpp_BUILD}/absl/base/absl_dynamic_annotations.lib - ${abseil_cpp_BUILD}/absl/base/absl_malloc_internal.lib - ${abseil_cpp_BUILD}/absl/base/absl_throw_delegate.lib - ${abseil_cpp_BUILD}/absl/numeric/absl_int128.lib - ${abseil_cpp_BUILD}/absl/strings/absl_strings.lib - ${abseil_cpp_BUILD}/absl/strings/str_format_internal.lib - ${abseil_cpp_BUILD}/absl/time/absl_time.lib - ${abseil_cpp_BUILD}/absl/types/absl_bad_optional_access.lib) - endif() - else() - set(abseil_cpp_STATIC_LIBRARIES - ${abseil_cpp_BUILD}/absl/base/libabsl_base.a - ${abseil_cpp_BUILD}/absl/base/libabsl_spinlock_wait.a - ${abseil_cpp_BUILD}/absl/base/libabsl_dynamic_annotations.a - ${abseil_cpp_BUILD}/absl/base/libabsl_malloc_internal.a - ${abseil_cpp_BUILD}/absl/base/libabsl_throw_delegate.a - ${abseil_cpp_BUILD}/absl/numeric/libabsl_int128.a - ${abseil_cpp_BUILD}/absl/strings/libabsl_strings.a - ${abseil_cpp_BUILD}/absl/strings/libstr_format_internal.a - ${abseil_cpp_BUILD}/absl/time/libabsl_time.a - ${abseil_cpp_BUILD}/absl/types/libabsl_bad_optional_access.a) - endif() - - ExternalProject_Add(abseil_cpp_build - PREFIX abseil_cpp - GIT_REPOSITORY ${abseil_cpp_URL} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${abseil_cpp_STATIC_LIBRARIES} - BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release - COMMAND ${CMAKE_COMMAND} --build . --config Release - INSTALL_COMMAND "" - CMAKE_CACHE_ARGS - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE} - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - ) - - include_directories(${abseil_cpp_INCLUDE_DIR}) - message(STATUS ${abseil_cpp_INCLUDE_DIR}) - - list(APPEND tensorflow_EXTERNAL_LIBRARIES ${abseil_cpp_STATIC_LIBRARIES}) - - list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build) - -endif (systemlib_ABSEIL_CPP) diff --git a/tensorflow/contrib/cmake/external/boringssl.cmake b/tensorflow/contrib/cmake/external/boringssl.cmake deleted file mode 100644 index fbb14b2515a..00000000000 --- a/tensorflow/contrib/cmake/external/boringssl.cmake +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(boringssl_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/boringssl/src/boringssl/include) -#set(boringssl_EXTRA_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/boringssl/src) -set(boringssl_URL https://boringssl.googlesource.com/boringssl) -set(boringssl_TAG 7f8c553d7f4db0a6ce727f2986d41bf8fe8ec4bf) -set(boringssl_BUILD ${CMAKE_BINARY_DIR}/boringssl/src/boringssl-build) -#set(boringssl_LIBRARIES ${boringssl_BUILD}/obj/so/libboringssl.so) -set(boringssl_STATIC_LIBRARIES - ${boringssl_BUILD}/ssl/libssl.a - ${boringssl_BUILD}/crypto/libcrypto.a - ${boringssl_BUILD}/decrepit/libdecrepit.a -) -set(boringssl_INCLUDES ${boringssl_BUILD}) - -set(boringssl_HEADERS - "${boringssl_INCLUDE_DIR}/include/*.h" -) - -ExternalProject_Add(boringssl - PREFIX boringssl - GIT_REPOSITORY ${boringssl_URL} - GIT_TAG ${boringssl_TAG} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - # BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${boringssl_STATIC_LIBRARIES} - INSTALL_COMMAND "" - CMAKE_CACHE_ARGS - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE} - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF -) - diff --git a/tensorflow/contrib/cmake/external/cub.cmake b/tensorflow/contrib/cmake/external/cub.cmake deleted file mode 100644 index 98a8c7e736e..00000000000 --- a/tensorflow/contrib/cmake/external/cub.cmake +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(cub_URL https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.8.0.zip) -set(cub_HASH SHA256=6bfa06ab52a650ae7ee6963143a0bbc667d6504822cbd9670369b598f18c58c3) -set(cub_BUILD ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) -set(cub_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) -set(cub_ARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/cub_archive) - -ExternalProject_Add(cub - PREFIX cub - URL ${cub_URL} - URL_HASH ${cub_HASH} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/cub/CMakeLists.txt ${cub_BUILD} - INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory ${cub_INCLUDE_DIR}/cub ${cub_ARCHIVE_DIR}/cub) diff --git a/tensorflow/contrib/cmake/external/double_conversion.cmake b/tensorflow/contrib/cmake/external/double_conversion.cmake deleted file mode 100644 index 5c5adaf5798..00000000000 --- a/tensorflow/contrib/cmake/external/double_conversion.cmake +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(double_conversion_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/double_conversion/src/double_conversion) -set(double_conversion_URL https://github.com/google/double-conversion.git) -set(double_conversion_TAG 3992066a95b823efc8ccc1baf82a1cfc73f6e9b8) -set(double_conversion_BUILD ${double_conversion_INCLUDE_DIR}) -set(double_conversion_LIBRARIES ${double_conversion_BUILD}/double-conversion/libdouble-conversion.so) -set(double_conversion_INCLUDES ${double_conversion_BUILD}) - -if(WIN32) - set(double_conversion_STATIC_LIBRARIES ${double_conversion_BUILD}/$(Configuration)/double-conversion.lib) -else() - set(double_conversion_STATIC_LIBRARIES ${double_conversion_BUILD}/libdouble-conversion.a) -endif() - -set(double_conversion_HEADERS - "${double_conversion_INCLUDE_DIR}/double-conversion/bignum-dtoa.h" - "${double_conversion_INCLUDE_DIR}/double-conversion/cached-powers.h" - "${double_conversion_INCLUDE_DIR}/double-conversion/double-conversion.h" - "${double_conversion_INCLUDE_DIR}/double-conversion/fixed-dtoa.h" - "${double_conversion_INCLUDE_DIR}/double-conversion/strtod.h" - "${double_conversion_INCLUDE_DIR}/double-conversion/bignum.h" - "${double_conversion_INCLUDE_DIR}/double-conversion/diy-fp.h" - "${double_conversion_INCLUDE_DIR}/double-conversion/fast-dtoa.h" - "${double_conversion_INCLUDE_DIR}/double-conversion/ieee.h" - "${double_conversion_INCLUDE_DIR}/double-conversion/utils.h" -) - -ExternalProject_Add(double_conversion - PREFIX double_conversion - GIT_REPOSITORY ${double_conversion_URL} - GIT_TAG ${double_conversion_TAG} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - INSTALL_COMMAND "" - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -) diff --git a/tensorflow/contrib/cmake/external/eigen.cmake b/tensorflow/contrib/cmake/external/eigen.cmake deleted file mode 100644 index 33bb31148d2..00000000000 --- a/tensorflow/contrib/cmake/external/eigen.cmake +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -#new_http_archive( -# name = "eigen_archive", -# urls = ["https://bitbucket.org/eigen/eigen/get/..."], -# sha256 = "...", -# build_file = "eigen.BUILD", -#) - -option(eigen_PATCH_FILE "Patch file to apply to eigen" OFF) -set(eigen_PATCH_COMMAND "") -if(eigen_PATCH_FILE) - set(eigen_PATCH_COMMAND PATCH_COMMAND patch -p0 -i "${eigen_PATCH_FILE}") -endif(eigen_PATCH_FILE) - -include (ExternalProject) - -# We parse the current Eigen version and archive hash from the bazel configuration -file(STRINGS ${PROJECT_SOURCE_DIR}/../../workspace.bzl workspace_contents) -foreach(line ${workspace_contents}) - string(REGEX MATCH ".*\"(https://bitbucket.org/eigen/eigen/get/[^\"]*tar.gz)\"" has_url ${line}) - if(has_url) - set(eigen_url ${CMAKE_MATCH_1}) - break() - endif() -endforeach() - -set(eigen_INCLUDE_DIRS - ${CMAKE_CURRENT_BINARY_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/external/eigen_archive - ${tensorflow_source_dir}/third_party/eigen3 -) -set(eigen_URL ${eigen_url}) -set(eigen_BUILD ${CMAKE_CURRENT_BINARY_DIR}/eigen/src/eigen) -set(eigen_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/eigen/install) - -ExternalProject_Add(eigen - PREFIX eigen - URL ${eigen_URL} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - INSTALL_DIR "${eigen_INSTALL}" - ${eigen_PATCH_COMMAND} - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DCMAKE_INSTALL_PREFIX:STRING=${eigen_INSTALL} - -DINCLUDE_INSTALL_DIR:STRING=${CMAKE_CURRENT_BINARY_DIR}/external/eigen_archive - -DBUILD_TESTING:BOOL=OFF -) diff --git a/tensorflow/contrib/cmake/external/farmhash.cmake b/tensorflow/contrib/cmake/external/farmhash.cmake deleted file mode 100644 index d51569bc213..00000000000 --- a/tensorflow/contrib/cmake/external/farmhash.cmake +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(farmhash_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/farmhash_archive ${CMAKE_CURRENT_BINARY_DIR}/external/farmhash_archive/util) -set(farmhash_URL https://mirror.bazel.build/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz) -set(farmhash_HASH SHA256=6560547c63e4af82b0f202cb710ceabb3f21347a4b996db565a411da5b17aba0) -set(farmhash_BUILD ${CMAKE_CURRENT_BINARY_DIR}/farmhash/src/farmhash) -set(farmhash_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/farmhash/install) -set(farmhash_INCLUDES ${farmhash_BUILD}) -set(farmhash_HEADERS - "${farmhash_BUILD}/src/farmhash.h" -) - -if(WIN32) - set(farmhash_STATIC_LIBRARIES ${farmhash_INSTALL}/lib/farmhash.lib) - - ExternalProject_Add(farmhash - PREFIX farmhash - URL ${farmhash_URL} - URL_HASH ${farmhash_HASH} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${farmhash_STATIC_LIBRARIES} - PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/farmhash/CMakeLists.txt ${farmhash_BUILD} - INSTALL_DIR ${farmhash_INSTALL} - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DCMAKE_INSTALL_PREFIX:STRING=${farmhash_INSTALL}) -else() - set(farmhash_STATIC_LIBRARIES ${farmhash_INSTALL}/lib/libfarmhash.a) - - ExternalProject_Add(farmhash - PREFIX farmhash - URL ${farmhash_URL} - URL_HASH ${farmhash_HASH} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_COMMAND $(MAKE) - INSTALL_COMMAND $(MAKE) install - CONFIGURE_COMMAND - ${farmhash_BUILD}/configure - --prefix=${farmhash_INSTALL} - --libdir=${farmhash_INSTALL}/lib - --enable-shared=yes - CXXFLAGS=-fPIC) - -endif() - -# put farmhash includes in the directory where they are expected -add_custom_target(farmhash_create_destination_dir - COMMAND ${CMAKE_COMMAND} -E make_directory ${farmhash_INCLUDE_DIR} - DEPENDS farmhash) - -add_custom_target(farmhash_copy_headers_to_destination - DEPENDS farmhash_create_destination_dir) - -foreach(header_file ${farmhash_HEADERS}) - add_custom_command(TARGET farmhash_copy_headers_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file} ${farmhash_INCLUDE_DIR}/) -endforeach() diff --git a/tensorflow/contrib/cmake/external/fft2d.cmake b/tensorflow/contrib/cmake/external/fft2d.cmake deleted file mode 100644 index a7bc50d5bcd..00000000000 --- a/tensorflow/contrib/cmake/external/fft2d.cmake +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -include (ExternalProject) - -set(fft2d_URL https://mirror.bazel.build/www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz) -set(fft2d_HASH SHA256=52bb637c70b971958ec79c9c8752b1df5ff0218a4db4510e60826e0cb79b5296) -set(fft2d_BUILD ${CMAKE_CURRENT_BINARY_DIR}/fft2d/) -set(fft2d_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/fft2d/src) - -if(WIN32) - set(fft2d_STATIC_LIBRARIES ${fft2d_BUILD}/src/lib/fft2d.lib) - - ExternalProject_Add(fft2d - PREFIX fft2d - URL ${fft2d_URL} - URL_HASH ${fft2d_HASH} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${fft2d_STATIC_LIBRARIES} - PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/fft2d/CMakeLists.txt ${fft2d_BUILD}/src/fft2d/CMakeLists.txt - INSTALL_DIR ${fft2d_INSTALL} - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DCMAKE_INSTALL_PREFIX:STRING=${fft2d_INSTALL}) -else() - set(fft2d_STATIC_LIBRARIES ${fft2d_BUILD}/src/fft2d/libfft2d.a) - - ExternalProject_Add(fft2d - PREFIX fft2d - URL ${fft2d_URL} - URL_HASH ${fft2d_HASH} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/fft2d/CMakeLists.txt ${fft2d_BUILD}/src/fft2d/CMakeLists.txt - INSTALL_DIR ${fft2d_INSTALL} - INSTALL_COMMAND echo - BUILD_COMMAND $(MAKE)) - -endif() diff --git a/tensorflow/contrib/cmake/external/gemmlowp.cmake b/tensorflow/contrib/cmake/external/gemmlowp.cmake deleted file mode 100644 index cdaa6b73b93..00000000000 --- a/tensorflow/contrib/cmake/external/gemmlowp.cmake +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(gemmlowp_URL https://github.com/google/gemmlowp/archive/38ebac7b059e84692f53e5938f97a9943c120d98.zip) -set(gemmlowp_HASH SHA256=b87faa7294dfcc5d678f22a59d2c01ca94ea1e2a3b488c38a95a67889ed0a658) -set(gemmlowp_BUILD ${CMAKE_CURRENT_BINARY_DIR}/gemmlowp/src/gemmlowp) -set(gemmlowp_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/gemmlowp/src/gemmlowp) - -ExternalProject_Add(gemmlowp - PREFIX gemmlowp - URL ${gemmlowp_URL} - URL_HASH ${gemmlowp_HASH} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/gemmlowp/CMakeLists.txt ${gemmlowp_BUILD} - INSTALL_COMMAND "") diff --git a/tensorflow/contrib/cmake/external/gif.cmake b/tensorflow/contrib/cmake/external/gif.cmake deleted file mode 100644 index e1f8d13f8ea..00000000000 --- a/tensorflow/contrib/cmake/external/gif.cmake +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(gif_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/gif_archive/giflib-5.1.4/) -set(gif_URL https://mirror.bazel.build/ufpr.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz) -set(gif_HASH SHA256=34a7377ba834397db019e8eb122e551a49c98f49df75ec3fcc92b9a794a4f6d1) -set(gif_INSTALL ${CMAKE_BINARY_DIR}/gif/install) -set(gif_BUILD ${CMAKE_BINARY_DIR}/gif/src/gif) - - -set(gif_HEADERS - "${gif_INSTALL}/include/gif_lib.h" -) - -if(WIN32) - - set(gif_STATIC_LIBRARIES ${gif_INSTALL}/lib/giflib.lib) - - ExternalProject_Add(gif - PREFIX gif - URL ${gif_URL} - URL_HASH ${gif_HASH} - BUILD_BYPRODUCTS ${gif_STATIC_LIBRARIES} - PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/patches/gif/CMakeLists.txt ${gif_BUILD} - INSTALL_DIR ${gif_INSTALL} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DCMAKE_INSTALL_PREFIX:STRING=${gif_INSTALL} - ) - - ExternalProject_Add_Step(gif copy_unistd - COMMAND ${CMAKE_COMMAND} -E copy_if_different - ${CMAKE_SOURCE_DIR}/patches/gif/unistd.h ${gif_BUILD}/lib/unistd.h - DEPENDEES patch - DEPENDERS build - ) - -else() - - set(gif_STATIC_LIBRARIES ${gif_INSTALL}/lib/libgif.a) - set(ENV{CFLAGS} "$ENV{CFLAGS} -fPIC") - - ExternalProject_Add(gif - PREFIX gif - URL ${gif_URL} - URL_HASH ${gif_HASH} - INSTALL_DIR ${gif_INSTALL} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_COMMAND $(MAKE) - INSTALL_COMMAND $(MAKE) install - CONFIGURE_COMMAND - ${CMAKE_CURRENT_BINARY_DIR}/gif/src/gif/configure - --with-pic - --prefix=${gif_INSTALL} - --libdir=${gif_INSTALL}/lib - --enable-shared=yes - ) - -endif() - -# put gif includes in the directory where they are expected -add_custom_target(gif_create_destination_dir - COMMAND ${CMAKE_COMMAND} -E make_directory ${gif_INCLUDE_DIR} - DEPENDS gif) - -add_custom_target(gif_copy_headers_to_destination - DEPENDS gif_create_destination_dir) - -foreach(header_file ${gif_HEADERS}) - add_custom_command(TARGET gif_copy_headers_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file} ${gif_INCLUDE_DIR}/) -endforeach() diff --git a/tensorflow/contrib/cmake/external/googletest.cmake b/tensorflow/contrib/cmake/external/googletest.cmake deleted file mode 100644 index 7cc5ae63909..00000000000 --- a/tensorflow/contrib/cmake/external/googletest.cmake +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(googletest_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest/googletest/include) -set(googletest_URL https://github.com/google/googletest.git) -set(googletest_BUILD ${CMAKE_CURRENT_BINARY_DIR}/googletest/) -set(googletest_TAG ec44c6c1675c25b9827aacd08c02433cccde7780) - -if(WIN32) - if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(googletest_STATIC_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest/googletest/$(Configuration)/gtest.lib) - else() - set(googletest_STATIC_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest/googletest/gtest.lib) - endif() -else() - set(googletest_STATIC_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest/googletest/${CMAKE_BUILD_TYPE}/gtest.a) -endif() - -ExternalProject_Add(googletest - PREFIX googletest - GIT_REPOSITORY ${googletest_URL} - GIT_TAG ${googletest_TAG} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${googletest_STATIC_LIBRARIES} - #PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/patches/grpc/CMakeLists.txt ${GRPC_BUILD} - INSTALL_COMMAND "" - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} - -DBUILD_GMOCK:BOOL=OFF - -DBUILD_GTEST:BOOL=ON - -Dgtest_force_shared_crt:BOOL=ON -) diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake deleted file mode 100644 index 22f4ea89e3c..00000000000 --- a/tensorflow/contrib/cmake/external/grpc.cmake +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include) -set(GRPC_URL https://github.com/grpc/grpc.git) -set(GRPC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc) -set(GRPC_TAG 4566c2a29ebec0835643b972eb99f4306c4234a3) - -if(WIN32) - # We use unsecure gRPC because boringssl does not build on windows - set(grpc_TARGET grpc++_unsecure) - set(grpc_DEPENDS protobuf zlib) - set(grpc_SSL_PROVIDER NONE) - if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(grpc_STATIC_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/$(Configuration)/grpc++_unsecure.lib - ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/$(Configuration)/grpc_unsecure.lib - ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/$(Configuration)/gpr.lib) - else() - set(grpc_STATIC_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/grpc++_unsecure.lib - ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/grpc_unsecure.lib - ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/gpr.lib) - endif() -else() - set(grpc_TARGET grpc++) - set(grpc_DEPENDS boringssl protobuf zlib) - set(grpc_SSL_PROVIDER module) - set(grpc_STATIC_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc++.a - ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc.a - ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libaddress_sorting.a - ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgpr.a - ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/third_party/cares/cares/lib/libcares.a - ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/third_party/zlib/libz.a) -endif() - -add_definitions(-DGRPC_ARES=0) - -ExternalProject_Add(grpc - PREFIX grpc - DEPENDS ${grpc_DEPENDS} - GIT_REPOSITORY ${GRPC_URL} - GIT_TAG ${GRPC_TAG} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${grpc_STATIC_LIBRARIES} - BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release --target ${grpc_TARGET} - COMMAND ${CMAKE_COMMAND} --build . --config Release --target grpc_cpp_plugin - INSTALL_COMMAND "" - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DPROTOBUF_INCLUDE_DIRS:STRING=${PROTOBUF_INCLUDE_DIRS} - -DPROTOBUF_LIBRARIES:STRING=${protobuf_STATIC_LIBRARIES} - -DZLIB_ROOT:STRING=${ZLIB_INSTALL} - -DgRPC_SSL_PROVIDER:STRING=${grpc_SSL_PROVIDER} -) - -# grpc/src/core/ext/census/tracing.c depends on the existence of openssl/rand.h. -ExternalProject_Add_Step(grpc copy_rand - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_SOURCE_DIR}/patches/grpc/rand.h ${GRPC_BUILD}/include/openssl/rand.h - DEPENDEES patch - DEPENDERS build -) diff --git a/tensorflow/contrib/cmake/external/highwayhash.cmake b/tensorflow/contrib/cmake/external/highwayhash.cmake deleted file mode 100644 index 7d260b85f21..00000000000 --- a/tensorflow/contrib/cmake/external/highwayhash.cmake +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(highwayhash_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/highwayhash) -set(highwayhash_URL https://github.com/google/highwayhash.git) -set(highwayhash_TAG be5edafc2e1a455768e260ccd68ae7317b6690ee) -set(highwayhash_BUILD ${CMAKE_CURRENT_BINARY_DIR}/highwayhash/src/highwayhash) -set(highwayhash_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/highwayhash/install) - -if(WIN32) - set(highwayhash_HEADERS "${highwayhash_BUILD}/highwayhash/*.h") - set(highwayhash_STATIC_LIBRARIES ${highwayhash_INSTALL}/lib/highwayhash.lib) -else() - set(highwayhash_HEADERS "${highwayhash_BUILD}/highwayhash/*.h") - set(highwayhash_STATIC_LIBRARIES ${highwayhash_INSTALL}/lib/libhighwayhash.a) -endif() - -set(highwayhash_HEADERS - "${highwayhash_INSTALL}/include/code_annotation.h" - "${highwayhash_INSTALL}/include/highway_tree_hash.h" - "${highwayhash_INSTALL}/include/scalar_highway_tree_hash.h" - "${highwayhash_INSTALL}/include/scalar_sip_tree_hash.h" - "${highwayhash_INSTALL}/include/sip_hash.h" - "${highwayhash_INSTALL}/include/sip_tree_hash.h" - "${highwayhash_INSTALL}/include/sse41_highway_tree_hash.h" - "${highwayhash_INSTALL}/include/state_helpers.h" - "${highwayhash_INSTALL}/include/types.h" - "${highwayhash_INSTALL}/include/vec.h" - "${highwayhash_INSTALL}/include/vec2.h" -) - -ExternalProject_Add(highwayhash - PREFIX highwayhash - GIT_REPOSITORY ${highwayhash_URL} - GIT_TAG ${highwayhash_TAG} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${highwayhash_STATIC_LIBRARIES} - PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/highwayhash/CMakeLists.txt ${highwayhash_BUILD} - INSTALL_DIR ${highwayhash_INSTALL} - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DCMAKE_INSTALL_PREFIX:STRING=${highwayhash_INSTALL}) - -# put highwayhash includes in the directory where they are expected -add_custom_target(highwayhash_create_destination_dir - COMMAND ${CMAKE_COMMAND} -E make_directory ${highwayhash_INCLUDE_DIR}/highwayhash - DEPENDS highwayhash) - -add_custom_target(highwayhash_copy_headers_to_destination - DEPENDS highwayhash_create_destination_dir) - -foreach(header_file ${highwayhash_HEADERS}) - add_custom_command(TARGET highwayhash_copy_headers_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file} ${highwayhash_INCLUDE_DIR}/highwayhash/) -endforeach() diff --git a/tensorflow/contrib/cmake/external/jpeg.cmake b/tensorflow/contrib/cmake/external/jpeg.cmake deleted file mode 100644 index c1c5842aa44..00000000000 --- a/tensorflow/contrib/cmake/external/jpeg.cmake +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(jpeg_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/jpeg_archive) -set(jpeg_URL https://mirror.bazel.build/www.ijg.org/files/jpegsrc.v9a.tar.gz) -set(jpeg_HASH SHA256=3a753ea48d917945dd54a2d97de388aa06ca2eb1066cbfdc6652036349fe05a7) -set(jpeg_BUILD ${CMAKE_CURRENT_BINARY_DIR}/jpeg/src/jpeg) -set(jpeg_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/jpeg/install) - -if(WIN32) - set(jpeg_STATIC_LIBRARIES ${jpeg_INSTALL}/lib/libjpeg.lib) -else() - set(jpeg_STATIC_LIBRARIES ${jpeg_INSTALL}/lib/libjpeg.a) -endif() - -set(jpeg_HEADERS - "${jpeg_INSTALL}/include/jconfig.h" - "${jpeg_INSTALL}/include/jerror.h" - "${jpeg_INSTALL}/include/jmorecfg.h" - "${jpeg_INSTALL}/include/jpeglib.h" - "${jpeg_BUILD}/cderror.h" - "${jpeg_BUILD}/cdjpeg.h" - "${jpeg_BUILD}/jdct.h" - "${jpeg_BUILD}/jinclude.h" - "${jpeg_BUILD}/jmemsys.h" - "${jpeg_BUILD}/jpegint.h" - "${jpeg_BUILD}/jversion.h" - "${jpeg_BUILD}/transupp.h" -) - -if (WIN32) - ExternalProject_Add(jpeg - PREFIX jpeg - URL ${jpeg_URL} - URL_HASH ${jpeg_HASH} - BUILD_BYPRODUCTS ${jpeg_STATIC_LIBRARIES} - PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/jpeg/CMakeLists.txt ${jpeg_BUILD} - INSTALL_DIR ${jpeg_INSTALL} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DCMAKE_INSTALL_PREFIX:STRING=${jpeg_INSTALL} - ) - - ExternalProject_Add_Step(jpeg copy_jconfig - COMMAND ${CMAKE_COMMAND} -E copy_if_different - ${jpeg_BUILD}/jconfig.vc ${jpeg_BUILD}/jconfig.h - DEPENDEES patch - DEPENDERS build - ) - -else() - ExternalProject_Add(jpeg - PREFIX jpeg - URL ${jpeg_URL} - URL_HASH ${jpeg_HASH} - INSTALL_DIR ${jpeg_INSTALL} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_COMMAND $(MAKE) - INSTALL_COMMAND $(MAKE) install - CONFIGURE_COMMAND - ${jpeg_BUILD}/configure - --prefix=${jpeg_INSTALL} - --libdir=${jpeg_INSTALL}/lib - --enable-shared=yes - CFLAGS=-fPIC - ) - -endif() - -# put jpeg includes in the directory where they are expected -add_custom_target(jpeg_create_destination_dir - COMMAND ${CMAKE_COMMAND} -E make_directory ${jpeg_INCLUDE_DIR} - DEPENDS jpeg) - -add_custom_target(jpeg_copy_headers_to_destination - DEPENDS jpeg_create_destination_dir) - -foreach(header_file ${jpeg_HEADERS}) - add_custom_command(TARGET jpeg_copy_headers_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file} ${jpeg_INCLUDE_DIR}) -endforeach() diff --git a/tensorflow/contrib/cmake/external/jsoncpp.cmake b/tensorflow/contrib/cmake/external/jsoncpp.cmake deleted file mode 100644 index 84c52e3652f..00000000000 --- a/tensorflow/contrib/cmake/external/jsoncpp.cmake +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(jsoncpp_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/jsoncpp/src/jsoncpp) -#set(jsoncpp_EXTRA_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/jsoncpp/src) -set(jsoncpp_URL https://github.com/open-source-parsers/jsoncpp.git) -set(jsoncpp_TAG 4356d9b) -set(jsoncpp_BUILD ${CMAKE_CURRENT_BINARY_DIR}/jsoncpp/src/jsoncpp/src/lib_json) -set(jsoncpp_LIBRARIES ${jsoncpp_BUILD}/obj/so/libjsoncpp.so) -set(jsoncpp_INCLUDES ${jsoncpp_BUILD}) - -if(WIN32) - if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(jsoncpp_STATIC_LIBRARIES ${jsoncpp_BUILD}/$(Configuration)/jsoncpp.lib) - else() - set(jsoncpp_STATIC_LIBRARIES ${jsoncpp_BUILD}/jsoncpp.lib) - endif() -else() - set(jsoncpp_STATIC_LIBRARIES ${jsoncpp_BUILD}/libjsoncpp.a) -endif() - -# We only need jsoncpp.h in external/jsoncpp/jsoncpp/jsoncpp.h -# For the rest, we'll just add the build dir as an include dir. -set(jsoncpp_HEADERS - "${jsoncpp_INCLUDE_DIR}/include/json/json.h" -) - -ExternalProject_Add(jsoncpp - PREFIX jsoncpp - GIT_REPOSITORY ${jsoncpp_URL} - GIT_TAG ${jsoncpp_TAG} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${jsoncpp_STATIC_LIBRARIES} - INSTALL_COMMAND "" - CMAKE_CACHE_ARGS - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE} - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF -) - diff --git a/tensorflow/contrib/cmake/external/lmdb.cmake b/tensorflow/contrib/cmake/external/lmdb.cmake deleted file mode 100644 index ed5ab788acc..00000000000 --- a/tensorflow/contrib/cmake/external/lmdb.cmake +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(lmdb_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/lmdb) -set(lmdb_URL https://mirror.bazel.build/github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz) -set(lmdb_HASH SHA256=108532fb94c6f227558d45be3f3347b52539f0f58290a7bb31ec06c462d05326) -set(lmdb_BUILD ${CMAKE_BINARY_DIR}/lmdb/src/lmdb) -set(lmdb_INSTALL ${CMAKE_BINARY_DIR}/lmdb/install) - -if(WIN32) - set(lmdb_STATIC_LIBRARIES ${lmdb_INSTALL}/lib/lmdb.lib) -else() - set(lmdb_STATIC_LIBRARIES ${lmdb_INSTALL}/lib/liblmdb.a) -endif() - -ExternalProject_Add(lmdb - PREFIX lmdb - URL ${lmdb_URL} - URL_HASH ${lmdb_HASH} - BUILD_BYPRODUCTS ${lmdb_STATIC_LIBRARIES} - PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different - ${CMAKE_CURRENT_SOURCE_DIR}/patches/lmdb/CMakeLists.txt ${lmdb_BUILD} - INSTALL_DIR ${lmdb_INSTALL} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - CMAKE_CACHE_ARGS - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE} - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DCMAKE_INSTALL_PREFIX:STRING=${lmdb_INSTALL} -) - -set(lmdb_HEADERS - "${lmdb_INSTALL}/include/lmdb.h" - "${lmdb_INSTALL}/include/midl.h" -) - -## put lmdb includes in the directory where they are expected -add_custom_target(lmdb_create_destination_dir - COMMAND ${CMAKE_COMMAND} -E make_directory ${lmdb_INCLUDE_DIR} - DEPENDS lmdb) - -add_custom_target(lmdb_copy_headers_to_destination - DEPENDS lmdb_create_destination_dir) - -foreach(header_file ${lmdb_HEADERS}) - add_custom_command(TARGET lmdb_copy_headers_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file} ${lmdb_INCLUDE_DIR}/) -endforeach() diff --git a/tensorflow/contrib/cmake/external/mkl.cmake b/tensorflow/contrib/cmake/external/mkl.cmake deleted file mode 100644 index a172e3a41a2..00000000000 --- a/tensorflow/contrib/cmake/external/mkl.cmake +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -# NOTE: Different from mkldnn.cmake, this file is meant to download mkl libraries -set(mkl_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/mkl/src/mkl/include) -set(mkl_BIN_DIRS ${CMAKE_CURRENT_BINARY_DIR}/mkl/bin) -set(mkl_WIN mklml_win_2018.0.3.20180406.zip) # match for v0.14 -set(mkl_MAC mklml_mac_2018.0.3.20180406.tgz) -set(mkl_LNX mklml_lnx_2018.0.3.20180406.tgz) -set(mkl_TAG v0.14) -set(mkl_URL https://github.com/intel/mkl-dnn/releases) - -if (WIN32) - set(mkl_DOWNLOAD_URL ${mkl_URL}/download/${mkl_TAG}/${mkl_WIN}) - list(APPEND mkl_STATIC_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/mkl/src/mkl/lib/mklml.lib) - list(APPEND mkl_STATIC_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/mkl/src/mkl/lib/libiomp5md.lib) - list(APPEND mkl_SHARED_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/mkl/src/mkl/lib/mklml.dll) - list(APPEND mkl_SHARED_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/mkl/src/mkl/lib/libiomp5md.dll) -elseif (UNIX) - set(mkl_DOWNLOAD_URL ${mkl_URL}/download/${mkl_TAG}/${mkl_LNX}) - list(APPEND mkl_SHARED_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/mkl/src/mkl/lib/libiomp5.so) - list(APPEND mkl_SHARED_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/mkl/src/mkl/lib/libmklml_gnu.so) - list(APPEND mkl_SHARED_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/mkl/src/mkl/lib/libmklml_intel.so) -elseif (APPLE) - set(mkl_DOWNLOAD_URL ${mkl_URL}/download/${mkl_TAG}/${mkl_MAC}) - #TODO need more information -endif () - -ExternalProject_Add(mkl - PREFIX mkl - URL ${mkl_DOWNLOAD_URL} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - UPDATE_COMMAND "" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "") - -# put mkl dynamic libraries in one bin directory -add_custom_target(mkl_create_destination_dir - COMMAND ${CMAKE_COMMAND} -E make_directory ${mkl_BIN_DIRS} - DEPENDS mkl) - -add_custom_target(mkl_copy_shared_to_destination DEPENDS mkl_create_destination_dir) - -foreach(dll_file ${mkl_SHARED_LIBRARIES}) - add_custom_command(TARGET mkl_copy_shared_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${dll_file} ${mkl_BIN_DIRS}) -endforeach() diff --git a/tensorflow/contrib/cmake/external/mkldnn.cmake b/tensorflow/contrib/cmake/external/mkldnn.cmake deleted file mode 100644 index 8123ee1f393..00000000000 --- a/tensorflow/contrib/cmake/external/mkldnn.cmake +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(mkldnn_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/mkldnn/src/mkldnn/include) -set(mkldnn_URL https://github.com/01org/mkl-dnn.git) -set(mkldnn_BUILD ${CMAKE_CURRENT_BINARY_DIR}/mkldnn/src/mkldnn/src) -set(mkldnn_TAG 3063b2e4c943983f6bf5f2fb9a490d4a998cd291) - -if(WIN32) - if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(mkldnn_STATIC_LIBRARIES ${CMAKE_CURRENT_BINARY_DIR}/mkldnn/src/mkldnn/src/Release/mkldnn.lib) - set(mkldnn_SHARED_LIBRARIES ${CMAKE_CURRENT_BINARY_DIR}/mkldnn/src/mkldnn/src/Release/mkldnn.dll) - set(mkldnn_BUILD ${CMAKE_CURRENT_BINARY_DIR}/mkldnn/src/mkldnn/src/Release) - else() - set(mkldnn_STATIC_LIBRARIES ${CMAKE_CURRENT_BINARY_DIR}/mkldnn/src/mkldnn/src/mkldnn.lib) - set(mkldnn_SHARED_LIBRARIES ${CMAKE_CURRENT_BINARY_DIR}/mkldnn/src/mkldnn/src/mkldnn.dll) - endif() -else() - set(mkldnn_STATIC_LIBRARIES ${CMAKE_CURRENT_BINARY_DIR}/mkldnn/src/mkldnn/src/libmkldnn.a) -endif() - -ExternalProject_Add(mkldnn - PREFIX mkldnn - DEPENDS mkl - GIT_REPOSITORY ${mkldnn_URL} - GIT_TAG ${mkldnn_TAG} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${mkldnn_STATIC_LIBRARIES} - INSTALL_COMMAND "" - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DMKLINC:STRING=${mkl_INCLUDE_DIRS} -) - -# since mkldnn depends on mkl, copy the mkldnn.dll together with mklml.dll to mkl_bin_dirs -add_custom_target(mkldnn_copy_shared_to_destination DEPENDS mkldnn) - -add_custom_command(TARGET mkldnn_copy_shared_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${mkldnn_SHARED_LIBRARIES} ${mkl_BIN_DIRS}) diff --git a/tensorflow/contrib/cmake/external/nsync.cmake b/tensorflow/contrib/cmake/external/nsync.cmake deleted file mode 100644 index 2926889301a..00000000000 --- a/tensorflow/contrib/cmake/external/nsync.cmake +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(nsync_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/nsync/public) -set(nsync_URL https://github.com/google/nsync) -set(nsync_TAG 1.22.0) -set(nsync_BUILD ${CMAKE_CURRENT_BINARY_DIR}/nsync/src/nsync) -set(nsync_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/nsync/install) - -if(WIN32) - set(nsync_HEADERS "${nsync_BUILD}/public/*.h") - set(nsync_STATIC_LIBRARIES ${nsync_INSTALL}/lib/nsync_cpp.lib) -else() - set(nsync_HEADERS "${nsync_BUILD}/public/*.h") - set(nsync_STATIC_LIBRARIES ${nsync_INSTALL}/lib/libnsync_cpp.a) -endif() - -ExternalProject_Add(nsync - PREFIX nsync - GIT_REPOSITORY ${nsync_URL} - GIT_TAG ${nsync_TAG} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${nsync_STATIC_LIBRARIES} - INSTALL_DIR ${nsync_INSTALL} - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DCMAKE_INSTALL_PREFIX:STRING=${nsync_INSTALL} - -DCMAKE_INSTALL_LIBDIR:STRING=lib - -DNSYNC_LANGUAGE:STRING=c++11) - -set(nsync_HEADERS - "${nsync_INSTALL}/include/nsync.h" - "${nsync_INSTALL}/include/nsync_atomic.h" - "${nsync_INSTALL}/include/nsync_counter.h" - "${nsync_INSTALL}/include/nsync_cpp.h" - "${nsync_INSTALL}/include/nsync_cv.h" - "${nsync_INSTALL}/include/nsync_debug.h" - "${nsync_INSTALL}/include/nsync_mu.h" - "${nsync_INSTALL}/include/nsync_mu_wait.h" - "${nsync_INSTALL}/include/nsync_note.h" - "${nsync_INSTALL}/include/nsync_once.h" - "${nsync_INSTALL}/include/nsync_time.h" - "${nsync_INSTALL}/include/nsync_time_internal.h" - "${nsync_INSTALL}/include/nsync_waiter.h" -) - -# put nsync includes in the directory where they are expected -add_custom_target(nsync_create_destination_dir - COMMAND ${CMAKE_COMMAND} -E make_directory ${nsync_INCLUDE_DIR} - DEPENDS nsync) - -add_custom_target(nsync_copy_headers_to_destination - DEPENDS nsync_create_destination_dir) - -foreach(header_file ${nsync_HEADERS}) - add_custom_command(TARGET nsync_copy_headers_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file} ${nsync_INCLUDE_DIR}/) -endforeach() - - diff --git a/tensorflow/contrib/cmake/external/png.cmake b/tensorflow/contrib/cmake/external/png.cmake deleted file mode 100644 index c102b327dce..00000000000 --- a/tensorflow/contrib/cmake/external/png.cmake +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) -include (GNUInstallDirs) - -set(png_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/png_archive) -set(png_URL https://mirror.bazel.build/github.com/glennrp/libpng/archive/v1.6.37.tar.gz) -set(png_HASH SHA256=ca74a0dace179a8422187671aee97dd3892b53e168627145271cad5b5ac81307) -set(png_BUILD ${CMAKE_BINARY_DIR}/png/src/png) -set(png_INSTALL ${CMAKE_BINARY_DIR}/png/install) - -if(WIN32) - if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(png_STATIC_LIBRARIES - debug ${CMAKE_BINARY_DIR}/png/install/lib/libpng16_staticd.lib - optimized ${CMAKE_BINARY_DIR}/png/install/lib/libpng16_static.lib) - else() - if(CMAKE_BUILD_TYPE EQUAL Debug) - set(png_STATIC_LIBRARIES - ${CMAKE_BINARY_DIR}/png/install/lib/libpng16_staticd.lib) - else() - set(png_STATIC_LIBRARIES - ${CMAKE_BINARY_DIR}/png/install/lib/libpng16_static.lib) - endif() - endif() -else() - set(png_STATIC_LIBRARIES ${CMAKE_BINARY_DIR}/png/install/${CMAKE_INSTALL_LIBDIR}/libpng16.a) -endif() - -set(png_HEADERS - "${png_INSTALL}/include/libpng16/png.h" - "${png_INSTALL}/include/libpng16/pngconf.h" - "${png_INSTALL}/include/libpng16/pnglibconf.h" -) - -ExternalProject_Add(png - PREFIX png - DEPENDS zlib - URL ${png_URL} - URL_HASH ${png_HASH} - BUILD_BYPRODUCTS ${png_STATIC_LIBRARIES} - INSTALL_DIR ${png_INSTALL} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - CMAKE_CACHE_ARGS - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE} - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DCMAKE_INSTALL_PREFIX:STRING=${png_INSTALL} - -DZLIB_ROOT:STRING=${ZLIB_INSTALL} - -DPNG_TESTS:BOOL=OFF -) - -## put png includes in the directory where they are expected -add_custom_target(png_create_destination_dir - COMMAND ${CMAKE_COMMAND} -E make_directory ${png_INCLUDE_DIR} - DEPENDS png) - -add_custom_target(png_copy_headers_to_destination - DEPENDS png_create_destination_dir) - -foreach(header_file ${png_HEADERS}) - add_custom_command(TARGET png_copy_headers_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file} ${png_INCLUDE_DIR}/) -endforeach() diff --git a/tensorflow/contrib/cmake/external/protobuf.cmake b/tensorflow/contrib/cmake/external/protobuf.cmake deleted file mode 100644 index 773c37b309b..00000000000 --- a/tensorflow/contrib/cmake/external/protobuf.cmake +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(PROTOBUF_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src) -set(PROTOBUF_URL https://github.com/google/protobuf.git) - -# enable choose protobuf versions -SET(PROTOBUF_VERSION "3.6.1" CACHE STRING "Protobuf version") -SET_PROPERTY(CACHE PROTOBUF_VERSION PROPERTY STRINGS "3.4.0" "3.5.0" "3.6.1") - -if(${PROTOBUF_VERSION} STREQUAL "3.5.1") - set(PROTOBUF_TAG v3.6.1) -elseif(${PROTOBUF_VERSION} STREQUAL "3.5.0") - set(PROTOBUF_TAG 2761122b810fe8861004ae785cc3ab39f384d342) -elseif(${PROTOBUF_VERSION} STREQUAL "3.4.0") - set(PROTOBUF_TAG b04e5cba356212e4e8c66c61bbe0c3a20537c5b9) -endif() - -if(WIN32) - if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(protobuf_STATIC_LIBRARIES - debug ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/$(Configuration)/libprotobufd.lib - optimized ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/$(Configuration)/libprotobuf.lib) - set(PROTOBUF_PROTOC_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/$(Configuration)/protoc.exe) - else() - if(CMAKE_BUILD_TYPE EQUAL Debug) - set(protobuf_STATIC_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/libprotobufd.lib) - else() - set(protobuf_STATIC_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/libprotobuf.lib) - endif() - set(PROTOBUF_PROTOC_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/protoc.exe) - endif() - - # This section is to make sure CONFIGURE_COMMAND use the same generator settings - set(PROTOBUF_GENERATOR_PLATFORM) - if (CMAKE_GENERATOR_PLATFORM) - set(PROTOBUF_GENERATOR_PLATFORM -A ${CMAKE_GENERATOR_PLATFORM}) - endif() - set(PROTOBUF_GENERATOR_TOOLSET) - if (CMAKE_GENERATOR_TOOLSET) - set(PROTOBUF_GENERATOR_TOOLSET -T ${CMAKE_GENERATOR_TOOLSET}) - endif() - set(PROTOBUF_ADDITIONAL_CMAKE_OPTIONS -Dprotobuf_MSVC_STATIC_RUNTIME:BOOL=OFF - -G${CMAKE_GENERATOR} ${PROTOBUF_GENERATOR_PLATFORM} ${PROTOBUF_GENERATOR_TOOLSET}) - # End of section -else() - set(protobuf_STATIC_LIBRARIES ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/libprotobuf.a) - set(PROTOBUF_PROTOC_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/protoc) -endif() - -ExternalProject_Add(protobuf - PREFIX protobuf - DEPENDS zlib - GIT_REPOSITORY ${PROTOBUF_URL} - GIT_TAG ${PROTOBUF_TAG} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${PROTOBUF_PROTOC_EXECUTABLE} ${protobuf_STATIC_LIBRARIES} - SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf - # SOURCE_SUBDIR cmake/ # Requires CMake 3.7, this will allow removal of CONFIGURE_COMMAND - # CONFIGURE_COMMAND resets some settings made in CMAKE_CACHE_ARGS and the generator used - CONFIGURE_COMMAND ${CMAKE_COMMAND} cmake/ - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE} - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -Dprotobuf_BUILD_TESTS:BOOL=OFF - -DZLIB_ROOT=${ZLIB_INSTALL} - ${PROTOBUF_ADDITIONAL_CMAKE_OPTIONS} - INSTALL_COMMAND "" - CMAKE_CACHE_ARGS - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE} - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -Dprotobuf_BUILD_TESTS:BOOL=OFF - -Dprotobuf_MSVC_STATIC_RUNTIME:BOOL=OFF - -DZLIB_ROOT:STRING=${ZLIB_INSTALL} -) diff --git a/tensorflow/contrib/cmake/external/re2.cmake b/tensorflow/contrib/cmake/external/re2.cmake deleted file mode 100644 index c4bc0b1707b..00000000000 --- a/tensorflow/contrib/cmake/external/re2.cmake +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(re2_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/re2/install/include) -set(re2_URL https://github.com/google/re2) -set(re2_BUILD ${CMAKE_CURRENT_BINARY_DIR}/re2/src/re2) -set(re2_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/re2/install) -set(re2_TAG e7efc48) - -if(WIN32) - if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(re2_STATIC_LIBRARIES ${re2_BUILD}/$(Configuration)/re2.lib) - else() - set(re2_STATIC_LIBRARIES ${re2_BUILD}/re2.lib) - endif() -else() - set(re2_STATIC_LIBRARIES ${re2_BUILD}/libre2.a) -endif() - -set(re2_HEADERS - ${re2_INSTALL}/include/re2/re2.h -) - -ExternalProject_Add(re2 - PREFIX re2 - GIT_REPOSITORY ${re2_URL} - GIT_TAG ${re2_TAG} - INSTALL_DIR ${re2_INSTALL} - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${re2_STATIC_LIBRARIES} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - CMAKE_CACHE_ARGS - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE} - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_INSTALL_PREFIX:STRING=${re2_INSTALL} - -DRE2_BUILD_TESTING:BOOL=OFF -) diff --git a/tensorflow/contrib/cmake/external/snappy.cmake b/tensorflow/contrib/cmake/external/snappy.cmake deleted file mode 100644 index f54197643b0..00000000000 --- a/tensorflow/contrib/cmake/external/snappy.cmake +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(snappy_URL https://github.com/google/snappy.git) -set(snappy_TAG "55924d11095df25ab25c405fadfe93d0a46f82eb") -set(snappy_BUILD ${CMAKE_CURRENT_BINARY_DIR}/snappy/src/snappy) -set(snappy_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/snappy/src/snappy) - -if(WIN32) - if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(snappy_STATIC_LIBRARIES ${snappy_BUILD}/$(Configuration)/snappy.lib) - else() - set(snappy_STATIC_LIBRARIES ${snappy_BUILD}/snappy.lib) - endif() -else() - set(snappy_STATIC_LIBRARIES ${snappy_BUILD}/libsnappy.a) -endif() - -set(snappy_HEADERS - "${snappy_INCLUDE_DIR}/snappy.h" -) - -ExternalProject_Add(snappy - PREFIX snappy - GIT_REPOSITORY ${snappy_URL} - GIT_TAG ${snappy_TAG} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${snappy_STATIC_LIBRARIES} - INSTALL_COMMAND "" - LOG_DOWNLOAD ON - LOG_CONFIGURE ON - LOG_BUILD ON - CMAKE_CACHE_ARGS - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE} - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DSNAPPY_BUILD_TESTS:BOOL=OFF -) - -# actually enables snappy in the source code -add_definitions(-DTF_USE_SNAPPY) diff --git a/tensorflow/contrib/cmake/external/sqlite.cmake b/tensorflow/contrib/cmake/external/sqlite.cmake deleted file mode 100644 index ef9226a9388..00000000000 --- a/tensorflow/contrib/cmake/external/sqlite.cmake +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(sqlite_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/sqlite) -set(sqlite_URL https://mirror.bazel.build/www.sqlite.org/2019/sqlite-amalgamation-3280000.zip) -set(sqlite_HASH SHA256=d02fc4e95cfef672b45052e221617a050b7f2e20103661cda88387349a9b1327) -set(sqlite_BUILD ${CMAKE_CURRENT_BINARY_DIR}/sqlite/src/sqlite) -set(sqlite_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/sqlite/install) - -if(WIN32) - set(sqlite_STATIC_LIBRARIES ${sqlite_INSTALL}/lib/sqlite.lib) -else() - set(sqlite_STATIC_LIBRARIES ${sqlite_INSTALL}/lib/libsqlite.a) -endif() - -set(sqlite_HEADERS - "${sqlite_BUILD}/sqlite3.h" - "${sqlite_BUILD}/sqlite3ext.h" -) - -if (WIN32) - ExternalProject_Add(sqlite - PREFIX sqlite - URL ${sqlite_URL} - URL_HASH ${sqlite_HASH} - BUILD_BYPRODUCTS ${sqlite_STATIC_LIBRARIES} - PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/sqlite/CMakeLists.txt ${sqlite_BUILD} - INSTALL_DIR ${sqlite_INSTALL} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DCMAKE_INSTALL_PREFIX:STRING=${sqlite_INSTALL} - ) - -else() - ExternalProject_Add(sqlite - PREFIX sqlite - URL ${sqlite_URL} - URL_HASH ${sqlite_HASH} - PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/sqlite/CMakeLists.txt ${sqlite_BUILD} - INSTALL_DIR ${sqlite_INSTALL} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - CMAKE_CACHE_ARGS - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE} - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DCMAKE_INSTALL_PREFIX:STRING=${sqlite_INSTALL} - ) - -endif() - -# put sqlite includes in the directory where they are expected -add_custom_target(sqlite_create_destination_dir - COMMAND ${CMAKE_COMMAND} -E make_directory ${sqlite_INCLUDE_DIR} - DEPENDS sqlite) - -add_custom_target(sqlite_copy_headers_to_destination - DEPENDS sqlite_create_destination_dir) - -foreach(header_file ${sqlite_HEADERS}) - add_custom_command(TARGET sqlite_copy_headers_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file} ${sqlite_INCLUDE_DIR}) -endforeach() diff --git a/tensorflow/contrib/cmake/external/zlib.cmake b/tensorflow/contrib/cmake/external/zlib.cmake deleted file mode 100644 index 8942f3eecf0..00000000000 --- a/tensorflow/contrib/cmake/external/zlib.cmake +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -if (systemlib_ZLIB) - find_package(PkgConfig) - pkg_search_module(ZLIB REQUIRED zlib) - set(zlib_INCLUDE_DIR ${ZLIB_INCLUDE_DIRS}) - set(ADD_LINK_DIRECTORY ${ADD_LINK_DIRECTORY} ${ZLIB_LIBRARY_DIRS}) - set(ADD_CFLAGS ${ADD_CFLAGS} ${ZLIB_CFLAGS_OTHER}) - - # To meet DEPENDS zlib from other projects. - # If we hit this line, zlib is already built and installed to the system. - add_custom_target(zlib) - add_custom_target(zlib_copy_headers_to_destination) - -else (systemlib_ZLIB) - include (ExternalProject) - - set(zlib_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/zlib_archive) - set(ZLIB_URL https://github.com/madler/zlib) - set(ZLIB_BUILD ${CMAKE_CURRENT_BINARY_DIR}/zlib/src/zlib) - set(ZLIB_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/zlib/install) - # Match zlib version in tensorflow/workspace.bzl - set(ZLIB_TAG v1.2.11) - - if(WIN32) - if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(zlib_STATIC_LIBRARIES - debug ${CMAKE_CURRENT_BINARY_DIR}/zlib/install/lib/zlibstaticd.lib - optimized ${CMAKE_CURRENT_BINARY_DIR}/zlib/install/lib/zlibstatic.lib) - else() - if(CMAKE_BUILD_TYPE EQUAL Debug) - set(zlib_STATIC_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/zlib/install/lib/zlibstaticd.lib) - else() - set(zlib_STATIC_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/zlib/install/lib/zlibstatic.lib) - endif() - endif() - else() - set(zlib_STATIC_LIBRARIES - ${CMAKE_CURRENT_BINARY_DIR}/zlib/install/lib/libz.a) - endif() - - set(ZLIB_HEADERS - "${ZLIB_INSTALL}/include/zconf.h" - "${ZLIB_INSTALL}/include/zlib.h" - ) - - ExternalProject_Add(zlib - PREFIX zlib - GIT_REPOSITORY ${ZLIB_URL} - GIT_TAG ${ZLIB_TAG} - INSTALL_DIR ${ZLIB_INSTALL} - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${zlib_STATIC_LIBRARIES} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - CMAKE_CACHE_ARGS - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE} - -DCMAKE_BUILD_TYPE:STRING=Release - -DCMAKE_INSTALL_PREFIX:STRING=${ZLIB_INSTALL} - ) - - # put zlib includes in the directory where they are expected - add_custom_target(zlib_create_destination_dir - COMMAND ${CMAKE_COMMAND} -E make_directory ${zlib_INCLUDE_DIR} - DEPENDS zlib) - - add_custom_target(zlib_copy_headers_to_destination - DEPENDS zlib_create_destination_dir) - - foreach(header_file ${ZLIB_HEADERS}) - add_custom_command(TARGET zlib_copy_headers_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file} ${zlib_INCLUDE_DIR}) - endforeach() -endif (systemlib_ZLIB) diff --git a/tensorflow/contrib/cmake/make.bat b/tensorflow/contrib/cmake/make.bat deleted file mode 100644 index d52b24e01d6..00000000000 --- a/tensorflow/contrib/cmake/make.bat +++ /dev/null @@ -1,38 +0,0 @@ -%echo off - -cd /d %~dp0 - -if exist _build rd /s /q _build - -mkdir _build -chdir _build - - -rem cmake ../ -G "Visual Studio 15 Win64" -DCMAKE_GENERATOR_TOOLSET=v141,host=x64 -DCMAKE_INSTALL_PREFIX:PATH=.\install - -CALL :NORMALIZEPATH "..\..\..\.." -SET SOURCE_DIR=%RETVAL% - -echo %SOURCE_DIR% - -SET SOURCE_DIR=F:\frameworks\tensorflow\ - -CALL :NORMALIZEPATH "../../../tools/git/gen_git_source.py" -SET SOURCE_PYTHON_SCRIPT=%RETVAL% - -CALL :NORMALIZEPATH "../../../core/util/version_info.cc" -SET SOURCE_VERSION_CC=%RETVAL% - -python %SOURCE_PYTHON_SCRIPT% --raw_generate %SOURCE_VERSION_CC% --source_dir %SOURCE_DIR% --git_tag_override= - -cmake ../ -G "Visual Studio 15 Win64" -DCMAKE_GENERATOR_TOOLSET=v141,host=x64 -DCMAKE_INSTALL_PREFIX:PATH=.\install - -EXIT /B - -:NORMALIZEPATH - SET RETVAL=%~dpfn1 - EXIT /B - - - - \ No newline at end of file diff --git a/tensorflow/contrib/cmake/make.sh b/tensorflow/contrib/cmake/make.sh deleted file mode 100755 index eed3c34aba1..00000000000 --- a/tensorflow/contrib/cmake/make.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -( -cd "$(dirname "$0")" -mkdir -p _build - -( -cd _build -rm -rf -- * -cmake .. -) -) diff --git a/tensorflow/contrib/cmake/modules/FindAbseilCpp.cmake b/tensorflow/contrib/cmake/modules/FindAbseilCpp.cmake deleted file mode 100644 index 944ae3997a9..00000000000 --- a/tensorflow/contrib/cmake/modules/FindAbseilCpp.cmake +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -find_path(ABSEIL_CPP_INCLUDE_DIR absl/base/config.h - HINTS "${ABSEIL_CPP_INCLUDE_DIR_HINTS}" - PATHS "$ENV{PROGRAMFILES}" - "$ENV{PROGRAMW6432}" - PATH_SUFFIXES "") - -if(EXISTS "${ABSEIL_CPP_INCLUDE_DIR}" AND NOT "${ABSEIL_CPP_INCLUDE_DIR}" STREQUAL "") - - if(NOT AbseilCpp_FIND_COMPONENTS) - # search all libraries if no COMPONENTS was requested - set(AbseilCpp_FIND_COMPONENTS - "absl_algorithm;absl_any;absl_bad_any_cast" - "absl_bad_optional_access;absl_base;absl_container;absl_debugging" - "absl_dynamic_annotations;absl_examine_stack;absl_failure_signal_handler" - "absl_int128;absl_leak_check;absl_internal_malloc_internal;absl_memory;absl_meta" - "absl_numeric;absl_optional;absl_span;absl_internal_spinlock_wait;absl_stack_consumption" - "absl_stacktrace;absl_str_format;absl_strings;absl_symbolize;absl_synchronization" - "absl_throw_delegate;absl_time;absl_utility;str_format_extension_internal" - "str_format_internal;test_instance_tracker_lib") - endif() - - foreach(LIBNAME ${AbseilCpp_FIND_COMPONENTS}) - - unset(ABSEIL_CPP_LIBRARY CACHE) - - find_library(ABSEIL_CPP_LIBRARY - NAMES ${LIBNAME} - HINTS ${ABSEIL_CPP_LIBRARIES_DIR_HINTS}) - - if(ABSEIL_CPP_LIBRARY) - list(APPEND ABSEIL_CPP_LIBRARIES ${ABSEIL_CPP_LIBRARY}) - else() - message(FATAL_ERROR "\n" - "abseil_cpp library \"${LIBNAME}\" not found in system path.\n" - "Please provide locations using: -DABSEIL_CPP_LIBRARIES_DIR_HINTS:STRING=\"PATH\"\n") - endif() - - endforeach() - - unset(LIBNAME CACHE) - unset(ABSEIL_CPP_LIBRARY CACHE) - - set(ABSEIL_CPP_FOUND TRUE) - message(STATUS "Found abseil_cpp libraries") - - set(ABSEIL_CPP_INCLUDE_DIR "${ABSEIL_CPP_INCLUDE_DIR}" CACHE PATH "" FORCE) - mark_as_advanced(ABSEIL_CPP_INCLUDE_DIR) - - set(ABSEIL_CPP_LIBRARIES "${ABSEIL_CPP_LIBRARIES}" CACHE PATH "" FORCE) - mark_as_advanced(ABSEIL_CPP_LIBRARIES) - -else() - - message(FATAL_ERROR "\n" - "abseil_cpp headers not found in system path.\n" - "Please provide locations using: -DABSEIL_CPP_INCLUDE_DIR_HINTS:STRING=\"PATH\"\n") - -endif() diff --git a/tensorflow/contrib/cmake/patches/cub/CMakeLists.txt b/tensorflow/contrib/cmake/patches/cub/CMakeLists.txt deleted file mode 100644 index 36890f0ce6c..00000000000 --- a/tensorflow/contrib/cmake/patches/cub/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -cmake_minimum_required(VERSION 2.8.3) - -project(cub) diff --git a/tensorflow/contrib/cmake/patches/farmhash/CMakeLists.txt b/tensorflow/contrib/cmake/patches/farmhash/CMakeLists.txt deleted file mode 100644 index 0286f29ad0c..00000000000 --- a/tensorflow/contrib/cmake/patches/farmhash/CMakeLists.txt +++ /dev/null @@ -1,25 +0,0 @@ -cmake_minimum_required(VERSION 2.8.3) - -project(farmhash) - -set(FARMHASH_SRCS - "src/farmhash.h" - "src/farmhash.cc" -) - -set(FARMHASH_INCLUDES - "src/farmhash.h" -) - -include_directories("${CMAKE_CURRENT_SOURCE_DIR}") - -add_library(farmhash ${FARMHASH_SRCS}) -add_definitions(-DFARMHASH_NO_BUILTIN_EXPECT) - -install(TARGETS farmhash - LIBRARY DESTINATION lib COMPONENT RuntimeLibraries - ARCHIVE DESTINATION lib COMPONENT Development) - -foreach(FARMHASH_INCLUDE ${FARMHASH_INCLUDES}) - install(FILES ${FARMHASH_INCLUDE} DESTINATION include COMPONENT Development) -endforeach() diff --git a/tensorflow/contrib/cmake/patches/fft2d/CMakeLists.txt b/tensorflow/contrib/cmake/patches/fft2d/CMakeLists.txt deleted file mode 100644 index b31ea3ed980..00000000000 --- a/tensorflow/contrib/cmake/patches/fft2d/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -cmake_minimum_required(VERSION 2.8.3) - -project(fft2d) - -set(CMAKE_POSITION_INDEPENDENT_CODE ON) - -set(FFT2D_SRCS - "fftsg.c" -) - -include_directories("${CMAKE_CURRENT_SOURCE_DIR}") - -add_library(fft2d ${FFT2D_SRCS}) - -install(TARGETS fft2d - LIBRARY DESTINATION lib COMPONENT RuntimeLibraries - ARCHIVE DESTINATION lib COMPONENT Development) diff --git a/tensorflow/contrib/cmake/patches/gemmlowp/CMakeLists.txt b/tensorflow/contrib/cmake/patches/gemmlowp/CMakeLists.txt deleted file mode 100644 index e7a37b10f19..00000000000 --- a/tensorflow/contrib/cmake/patches/gemmlowp/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -cmake_minimum_required(VERSION 2.8.3) - -project(gemmlowp) \ No newline at end of file diff --git a/tensorflow/contrib/cmake/patches/gif/CMakeLists.txt b/tensorflow/contrib/cmake/patches/gif/CMakeLists.txt deleted file mode 100644 index e19df937888..00000000000 --- a/tensorflow/contrib/cmake/patches/gif/CMakeLists.txt +++ /dev/null @@ -1,42 +0,0 @@ -cmake_minimum_required(VERSION 2.8.3) - -project(giflib) - -set(GIFLIB_SRCS - "lib/dgif_lib.c" - "lib/egif_lib.c" - "lib/gif_font.c" - "lib/gif_hash.h" - "lib/gifalloc.c" - "lib/openbsd-reallocarray.c" - "lib/gif_err.c" - "lib/quantize.c" - "lib/gif_hash.c" - "lib/gif_lib.h" - "lib/gif_lib_private.h" -) -set(GIFLIB_INCLUDES - "lib/gif_lib.h" -) - -if (WIN32) - # Suppress warnings to reduce build log size. - add_definitions(/wd4267 /wd4244 /wd4800 /wd4503 /wd4554 /wd4996 /wd4348 /wd4018) - add_definitions(/wd4099 /wd4146 /wd4267 /wd4305 /wd4307) - add_definitions(/wd4715 /wd4722 /wd4723 /wd4838 /wd4309 /wd4334) - add_definitions(/wd4003 /wd4244 /wd4267 /wd4503 /wd4506 /wd4800 /wd4996) - add_definitions(/wd8029) -endif() - -include_directories("${CMAKE_CURRENT_SOURCE_DIR}/lib") - -add_library(giflib ${GIFLIB_SRCS}) - -install(TARGETS giflib - RUNTIME DESTINATION bin COMPONENT RuntimeLibraries - LIBRARY DESTINATION lib COMPONENT RuntimeLibraries - ARCHIVE DESTINATION lib COMPONENT Development) - -foreach(GIFLIB_INCLUDE ${GIFLIB_INCLUDES}) - install(FILES ${GIFLIB_INCLUDE} DESTINATION include COMPONENT Development) -endforeach() diff --git a/tensorflow/contrib/cmake/patches/gif/unistd.h b/tensorflow/contrib/cmake/patches/gif/unistd.h deleted file mode 100644 index cd52ce31d4d..00000000000 --- a/tensorflow/contrib/cmake/patches/gif/unistd.h +++ /dev/null @@ -1,14 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ diff --git a/tensorflow/contrib/cmake/patches/grpc/rand.h b/tensorflow/contrib/cmake/patches/grpc/rand.h deleted file mode 100644 index 194cb683fc5..00000000000 --- a/tensorflow/contrib/cmake/patches/grpc/rand.h +++ /dev/null @@ -1,14 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ diff --git a/tensorflow/contrib/cmake/patches/highwayhash/CMakeLists.txt b/tensorflow/contrib/cmake/patches/highwayhash/CMakeLists.txt deleted file mode 100644 index 8313d1c9f29..00000000000 --- a/tensorflow/contrib/cmake/patches/highwayhash/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -cmake_minimum_required(VERSION 2.8.3) - -project(highwayhash) - -set(HIGHWAYHASH_SRCS - "highwayhash/code_annotation.h" - "highwayhash/highway_tree_hash.cc" - "highwayhash/highway_tree_hash.h" - "highwayhash/scalar_highway_tree_hash.cc" - "highwayhash/scalar_highway_tree_hash.h" - "highwayhash/scalar_sip_tree_hash.cc" - "highwayhash/scalar_sip_tree_hash.h" - "highwayhash/sip_hash.cc" - "highwayhash/sip_hash.h" - "highwayhash/sip_tree_hash.cc" - "highwayhash/sip_tree_hash.h" - "highwayhash/sse41_highway_tree_hash.cc" - "highwayhash/sse41_highway_tree_hash.h" - "highwayhash/state_helpers.h" - "highwayhash/types.h" - "highwayhash/vec.h" - "highwayhash/vec2.h" -) - -set(HIGHWAYHASH_INCLUDES - "highwayhash/code_annotation.h" - "highwayhash/highway_tree_hash.h" - "highwayhash/scalar_highway_tree_hash.h" - "highwayhash/scalar_sip_tree_hash.h" - "highwayhash/sip_hash.h" - "highwayhash/sip_tree_hash.h" - "highwayhash/sse41_highway_tree_hash.h" - "highwayhash/state_helpers.h" - "highwayhash/types.h" - "highwayhash/vec.h" - "highwayhash/vec2.h" -) - -include_directories("${CMAKE_CURRENT_SOURCE_DIR}") - -add_library(highwayhash ${HIGHWAYHASH_SRCS}) - -# C++11 -target_compile_features(highwayhash PRIVATE - cxx_rvalue_references -) - -install(TARGETS highwayhash - LIBRARY DESTINATION lib COMPONENT RuntimeLibraries - ARCHIVE DESTINATION lib COMPONENT Development) - -foreach(HIGHWAYHASH_INCLUDE ${HIGHWAYHASH_INCLUDES}) - install(FILES ${HIGHWAYHASH_INCLUDE} DESTINATION include COMPONENT Development) -endforeach() diff --git a/tensorflow/contrib/cmake/patches/jpeg/CMakeLists.txt b/tensorflow/contrib/cmake/patches/jpeg/CMakeLists.txt deleted file mode 100644 index 0baea996eed..00000000000 --- a/tensorflow/contrib/cmake/patches/jpeg/CMakeLists.txt +++ /dev/null @@ -1,85 +0,0 @@ -cmake_minimum_required(VERSION 2.8.3) - -project(libjpeg) - -set(LIBJPEG_SRCS - "jaricom.c" - "jcapimin.c" - "jcapistd.c" - "jcarith.c" - "jccoefct.c" - "jccolor.c" - "jcdctmgr.c" - "jchuff.c" - "jcinit.c" - "jcmainct.c" - "jcmarker.c" - "jcmaster.c" - "jcomapi.c" - "jcparam.c" - "jcprepct.c" - "jcsample.c" - "jctrans.c" - "jdapimin.c" - "jdapistd.c" - "jdarith.c" - "jdatadst.c" - "jdatasrc.c" - "jdcoefct.c" - "jdcolor.c" - "jddctmgr.c" - "jdhuff.c" - "jdinput.c" - "jdmainct.c" - "jdmarker.c" - "jdmaster.c" - "jdmerge.c" - "jdpostct.c" - "jdsample.c" - "jdtrans.c" - "jerror.c" - "jfdctflt.c" - "jfdctfst.c" - "jfdctint.c" - "jidctflt.c" - "jidctfst.c" - "jidctint.c" - "jmemmgr.c" - "jmemnobs.c" - "jquant1.c" - "jquant2.c" - "jutils.c" -) -set(LIBJPEG_INCLUDES - "jconfig.h" - "jdct.h" - "jerror.h" - "jinclude.h" - "jmemsys.h" - "jmorecfg.h" - "jpegint.h" - "jpeglib.h" - "jversion.h" -) - -if (WIN32) - # Suppress warnings to reduce build log size. - add_definitions(/wd4267 /wd4244 /wd4800 /wd4503 /wd4554 /wd4996 /wd4348 /wd4018) - add_definitions(/wd4099 /wd4146 /wd4267 /wd4305 /wd4307) - add_definitions(/wd4715 /wd4722 /wd4723 /wd4838 /wd4309 /wd4334) - add_definitions(/wd4003 /wd4244 /wd4267 /wd4503 /wd4506 /wd4800 /wd4996) - add_definitions(/wd8029) -endif() - -include_directories("${CMAKE_CURRENT_SOURCE_DIR}") - -add_library(libjpeg ${LIBJPEG_SRCS}) - -install(TARGETS libjpeg - RUNTIME DESTINATION bin COMPONENT RuntimeLibraries - LIBRARY DESTINATION lib COMPONENT RuntimeLibraries - ARCHIVE DESTINATION lib COMPONENT Development) - -foreach(LIBJPEG_INCLUDE ${LIBJPEG_INCLUDES}) - install(FILES ${LIBJPEG_INCLUDE} DESTINATION include COMPONENT Development) -endforeach() diff --git a/tensorflow/contrib/cmake/patches/lmdb/CMakeLists.txt b/tensorflow/contrib/cmake/patches/lmdb/CMakeLists.txt deleted file mode 100644 index 2167b4352e1..00000000000 --- a/tensorflow/contrib/cmake/patches/lmdb/CMakeLists.txt +++ /dev/null @@ -1,35 +0,0 @@ -cmake_minimum_required(VERSION 2.8.3) - -project(liblmdb) - -set(LIBLMDB_SRCS - "libraries/liblmdb/mdb.c" - "libraries/liblmdb/midl.c" -) - -set(LIBLMDB_INCLUDES - "libraries/liblmdb/lmdb.h" - "libraries/liblmdb/midl.h" -) - -if (WIN32) - # Suppress warnings to reduce build log size. - add_definitions(/wd4267 /wd4244 /wd4800 /wd4503 /wd4554 /wd4996 /wd4348 /wd4018) - add_definitions(/wd4099 /wd4146 /wd4267 /wd4305 /wd4307) - add_definitions(/wd4715 /wd4722 /wd4723 /wd4838 /wd4309 /wd4334) - add_definitions(/wd4003 /wd4244 /wd4267 /wd4503 /wd4506 /wd4800 /wd4996) - add_definitions(/wd8029) -endif() - -include_directories("${CMAKE_CURRENT_SOURCE_DIR}") - -add_library(lmdb ${LIBLMDB_SRCS}) - -install(TARGETS lmdb - RUNTIME DESTINATION bin COMPONENT RuntimeLibraries - LIBRARY DESTINATION lib COMPONENT RuntimeLibraries - ARCHIVE DESTINATION lib COMPONENT Development) - -foreach(LIBLMDB_INCLUDE ${LIBLMDB_INCLUDES}) - install(FILES ${LIBLMDB_INCLUDE} DESTINATION include COMPONENT Development) -endforeach() diff --git a/tensorflow/contrib/cmake/patches/sqlite/CMakeLists.txt b/tensorflow/contrib/cmake/patches/sqlite/CMakeLists.txt deleted file mode 100644 index e67792afd26..00000000000 --- a/tensorflow/contrib/cmake/patches/sqlite/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -cmake_minimum_required(VERSION 2.8.3) - -project(sqlite) - -set(SQLITE_SRCS - "sqlite3.c" -) - -set(SQLITE_INCLUDES - "sqlite3.h" -) - -include_directories("${CMAKE_CURRENT_SOURCE_DIR}") - -add_library(sqlite ${SQLITE_SRCS}) - -# C++11 -target_compile_features(sqlite PRIVATE - cxx_rvalue_references -) - -install(TARGETS sqlite - LIBRARY DESTINATION lib COMPONENT RuntimeLibraries - ARCHIVE DESTINATION lib COMPONENT Development) - -foreach(SQLITE_INCLUDE ${SQLITE_INCLUDES}) - install(FILES ${SQLITE_INCLUDE} DESTINATION include COMPONENT Development) -endforeach() diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt deleted file mode 100644 index 7b7d5526f7f..00000000000 --- a/tensorflow/contrib/cmake/python_modules.txt +++ /dev/null @@ -1,433 +0,0 @@ -# python_sanity_test.py will complain about invalid or missing entries -# problematic entries can be commented for temporary whitelisting -tensorflow -tensorflow/compiler -tensorflow/compiler/xla -tensorflow/compiler/xla/service -tensorflow/core -tensorflow/core/example -tensorflow/core/framework -tensorflow/core/kernels -tensorflow/core/kernels/boosted_trees -tensorflow/core/lib -tensorflow/core/lib/core -tensorflow/core/profiler -tensorflow/core/protobuf -tensorflow/core/protobuf/tpu -tensorflow/core/util -tensorflow/examples -tensorflow/examples/tutorials -tensorflow/examples/tutorials/mnist -tensorflow/python -tensorflow/python/client -tensorflow/python/compat -tensorflow/python/data -tensorflow/python/data/ops -tensorflow/python/data/util -tensorflow/python/debug -tensorflow/python/debug/cli -tensorflow/python/debug/examples -tensorflow/python/debug/lib -tensorflow/python/debug/wrappers -tensorflow/python/eager -tensorflow/python/estimator -tensorflow/python/estimator/canned -tensorflow/python/estimator/export -tensorflow/python/estimator/inputs -tensorflow/python/estimator/inputs/queues -tensorflow/python/feature_column -tensorflow/python/framework -tensorflow/python/grappler -tensorflow/python/keras -tensorflow/python/keras/applications -tensorflow/python/keras/datasets -tensorflow/python/keras/engine -tensorflow/python/keras/estimator -tensorflow/python/keras/layers -tensorflow/python/keras/preprocessing -tensorflow/python/keras/utils -tensorflow/python/keras/wrappers -tensorflow/python/kernel_tests -tensorflow/python/kernel_tests/boosted_trees -tensorflow/python/kernel_tests/distributions -tensorflow/python/kernel_tests/linalg -tensorflow/python/kernel_tests/random -tensorflow/python/kernel_tests/testdata -tensorflow/python/layers -tensorflow/python/lib -tensorflow/python/lib/core -tensorflow/python/lib/io -tensorflow/python/ops -tensorflow/python/ops/distributions -tensorflow/python/ops/linalg -tensorflow/python/ops/losses -tensorflow/python/ops/signal -tensorflow/python/platform -tensorflow/python/profiler -tensorflow/python/profiler/internal -tensorflow/python/saved_model -tensorflow/python/summary -tensorflow/python/summary/writer -tensorflow/python/tools -tensorflow/python/tools/api -tensorflow/python/tools/api/generator -tensorflow/python/tpu -tensorflow/python/training -tensorflow/python/training/tracking -tensorflow/python/user_ops -tensorflow/python/util -tensorflow/python/util/protobuf -tensorflow/tools -tensorflow/tools/api -tensorflow/tools/graph_transforms -tensorflow/contrib -tensorflow/contrib/all_reduce -tensorflow/contrib/all_reduce/python -tensorflow/tools/android/inference_interface -tensorflow/tools/android/inference_interface/java -tensorflow/tools/android/inference_interface/java/org -tensorflow/tools/android/inference_interface/java/org/tensorflow -tensorflow/tools/android/inference_interface/java/org/tensorflow/contrib -tensorflow/tools/android/inference_interface/java/org/tensorflow/tools/android/inference_interface -tensorflow/tools/android/inference_interface/jni -tensorflow/contrib/batching -tensorflow/contrib/batching/python -tensorflow/contrib/batching/python/ops -tensorflow/contrib/bayesflow -tensorflow/contrib/bayesflow/python -tensorflow/contrib/bayesflow/python/ops -# tensorflow/contrib/bigtable/python -# tensorflow/contrib/bigtable/python/ops -tensorflow/contrib/boosted_trees -tensorflow/contrib/boosted_trees/estimator_batch -tensorflow/contrib/boosted_trees/kernels -tensorflow/contrib/boosted_trees/ops -tensorflow/contrib/boosted_trees/proto -tensorflow/contrib/boosted_trees/python -tensorflow/contrib/boosted_trees/python/kernel_tests -tensorflow/contrib/boosted_trees/python/ops -tensorflow/contrib/boosted_trees/python/training -tensorflow/contrib/boosted_trees/python/training/functions -tensorflow/contrib/boosted_trees/python/utils -tensorflow/contrib/checkpoint -tensorflow/contrib/checkpoint/python -tensorflow/contrib/cloud -tensorflow/contrib/cloud/kernels -tensorflow/contrib/cloud/ops -tensorflow/contrib/cloud/python -tensorflow/contrib/cloud/python/ops -tensorflow/contrib/cluster_resolver -tensorflow/contrib/cluster_resolver/python -tensorflow/contrib/cluster_resolver/python/training -tensorflow/contrib/compiler -tensorflow/contrib/constrained_optimization -tensorflow/contrib/constrained_optimization/python -tensorflow/contrib/copy_graph -tensorflow/contrib/copy_graph/python -tensorflow/contrib/copy_graph/python/util -tensorflow/contrib/crf -tensorflow/contrib/crf/python -tensorflow/contrib/crf/python/ops -tensorflow/contrib/cudnn_rnn -tensorflow/contrib/cudnn_rnn/python -tensorflow/contrib/cudnn_rnn/python/layers -tensorflow/contrib/cudnn_rnn/python/ops -tensorflow/contrib/data -tensorflow/contrib/data/python -tensorflow/contrib/data/python/kernel_tests -tensorflow/contrib/data/python/ops -tensorflow/contrib/decision_trees -tensorflow/contrib/decision_trees/proto -tensorflow/contrib/deprecated -tensorflow/contrib/distribute -tensorflow/contrib/distribute/python -tensorflow/contrib/distribute/python/examples -tensorflow/contrib/distributions -tensorflow/contrib/distributions/python -tensorflow/contrib/distributions/python/ops -tensorflow/contrib/distributions/python/ops/bijectors -tensorflow/contrib/eager -tensorflow/contrib/eager/python -tensorflow/contrib/estimator -tensorflow/contrib/estimator/python -tensorflow/contrib/estimator/python/estimator -tensorflow/contrib/factorization -tensorflow/contrib/factorization/examples -tensorflow/contrib/factorization/kernels -tensorflow/contrib/factorization/ops -tensorflow/contrib/factorization/python -tensorflow/contrib/factorization/python/ops -tensorflow/contrib/feature_column -tensorflow/contrib/feature_column/python -tensorflow/contrib/feature_column/python/feature_column -tensorflow/contrib/ffmpeg -tensorflow/contrib/ffmpeg/default -tensorflow/contrib/framework -tensorflow/contrib/framework/kernels -tensorflow/contrib/framework/ops -tensorflow/contrib/framework/python -tensorflow/contrib/framework/python/framework -tensorflow/contrib/framework/python/ops -tensorflow/contrib/fused_conv -tensorflow/contrib/fused_conv/kernels -tensorflow/contrib/fused_conv/python -tensorflow/contrib/fused_conv/python/ops -tensorflow/contrib/graph_editor -tensorflow/contrib/graph_editor/examples -tensorflow/contrib/grid_rnn -tensorflow/contrib/grid_rnn/python -tensorflow/contrib/grid_rnn/python/ops -tensorflow/contrib/hadoop/python -tensorflow/contrib/hadoop/python/ops -tensorflow/contrib/hooks -tensorflow/contrib/hooks/python -tensorflow/contrib/image -tensorflow/contrib/image/kernels -tensorflow/contrib/image/ops -tensorflow/contrib/image/python -tensorflow/contrib/image/python/ops -tensorflow/contrib/input_pipeline -tensorflow/contrib/input_pipeline/kernels -tensorflow/contrib/input_pipeline/ops -tensorflow/contrib/input_pipeline/python -tensorflow/contrib/input_pipeline/python/ops -tensorflow/contrib/integrate -tensorflow/contrib/integrate/python -tensorflow/contrib/integrate/python/ops -tensorflow/contrib/kafka/python -tensorflow/contrib/kafka/python/ops -tensorflow/contrib/ignite/python -tensorflow/contrib/ignite/python/ops -tensorflow/contrib/keras -tensorflow/contrib/keras/api -tensorflow/contrib/keras/api/keras -tensorflow/contrib/keras/api/keras/activations -tensorflow/contrib/keras/api/keras/applications -tensorflow/contrib/keras/api/keras/applications/inception_v3 -tensorflow/contrib/keras/api/keras/applications/mobilenet -tensorflow/contrib/keras/api/keras/applications/resnet50 -tensorflow/contrib/keras/api/keras/applications/vgg16 -tensorflow/contrib/keras/api/keras/applications/vgg19 -tensorflow/contrib/keras/api/keras/applications/xception -tensorflow/contrib/keras/api/keras/backend -tensorflow/contrib/keras/api/keras/callbacks -tensorflow/contrib/keras/api/keras/constraints -tensorflow/contrib/keras/api/keras/datasets -tensorflow/contrib/keras/api/keras/datasets/boston_housing -tensorflow/contrib/keras/api/keras/datasets/cifar10 -tensorflow/contrib/keras/api/keras/datasets/cifar100 -tensorflow/contrib/keras/api/keras/datasets/imdb -tensorflow/contrib/keras/api/keras/datasets/mnist -tensorflow/contrib/keras/api/keras/datasets/reuters -tensorflow/contrib/keras/api/keras/initializers -tensorflow/contrib/keras/api/keras/layers -tensorflow/contrib/keras/api/keras/losses -tensorflow/contrib/keras/api/keras/metrics -tensorflow/contrib/keras/api/keras/models -tensorflow/contrib/keras/api/keras/optimizers -tensorflow/contrib/keras/api/keras/preprocessing -tensorflow/contrib/keras/api/keras/preprocessing/image -tensorflow/contrib/keras/api/keras/preprocessing/sequence -tensorflow/contrib/keras/api/keras/preprocessing/text -tensorflow/contrib/keras/api/keras/regularizers -tensorflow/contrib/keras/api/keras/utils -tensorflow/contrib/keras/api/keras/wrappers -tensorflow/contrib/keras/api/keras/wrappers/scikit_learn -tensorflow/contrib/kernel_methods -tensorflow/contrib/kernel_methods/python -tensorflow/contrib/kernel_methods/python/mappers -tensorflow/contrib/kinesis/python -tensorflow/contrib/kinesis/python/ops -tensorflow/contrib/labeled_tensor -tensorflow/contrib/labeled_tensor/python -tensorflow/contrib/labeled_tensor/python/ops -tensorflow/contrib/layers -tensorflow/contrib/layers/kernels -tensorflow/contrib/layers/ops -tensorflow/contrib/layers/python -tensorflow/contrib/layers/python/layers -tensorflow/contrib/layers/python/ops -tensorflow/contrib/learn -tensorflow/contrib/learn/python -tensorflow/contrib/learn/python/learn -tensorflow/contrib/learn/python/learn/datasets -tensorflow/contrib/learn/python/learn/datasets/data -tensorflow/contrib/learn/python/learn/estimators -tensorflow/contrib/learn/python/learn/learn_io -tensorflow/contrib/learn/python/learn/ops -tensorflow/contrib/learn/python/learn/preprocessing -tensorflow/contrib/learn/python/learn/utils -tensorflow/contrib/legacy_seq2seq -tensorflow/contrib/legacy_seq2seq/python -tensorflow/contrib/legacy_seq2seq/python/ops -tensorflow/contrib/libsvm -tensorflow/contrib/libsvm/python -tensorflow/contrib/libsvm/python/kernel_tests -tensorflow/contrib/libsvm/python/ops -tensorflow/contrib/linear_optimizer -tensorflow/contrib/linear_optimizer/kernels -tensorflow/contrib/linear_optimizer/kernels/g3doc -tensorflow/contrib/linear_optimizer/python -tensorflow/contrib/linear_optimizer/python/ops -# TODO(drpngx): Fix failing imports -# tensorflow/lite -# tensorflow/lite/python -# tensorflow/lite/toco -# tensorflow/lite/toco/python -tensorflow/contrib/lookup -tensorflow/contrib/losses -tensorflow/contrib/losses/python -tensorflow/contrib/losses/python/losses -tensorflow/contrib/losses/python/metric_learning -tensorflow/contrib/makefile -tensorflow/contrib/memory_stats -tensorflow/contrib/memory_stats/kernels -tensorflow/contrib/memory_stats/ops -tensorflow/contrib/memory_stats/python -tensorflow/contrib/memory_stats/python/ops -tensorflow/contrib/meta_graph_transform -tensorflow/contrib/metrics -tensorflow/contrib/metrics/python -tensorflow/contrib/metrics/python/metrics -tensorflow/contrib/metrics/python/ops -tensorflow/contrib/mixed_precision -tensorflow/contrib/mixed_precision/python -tensorflow/contrib/mpi_collectives/python -tensorflow/contrib/mpi_collectives/python/ops -tensorflow/contrib/model_pruning -tensorflow/contrib/model_pruning/examples -tensorflow/contrib/model_pruning/examples/cifar10 -tensorflow/contrib/model_pruning/python -tensorflow/contrib/model_pruning/python/layers -tensorflow/contrib/nearest_neighbor -tensorflow/contrib/nearest_neighbor/kernels -tensorflow/contrib/nearest_neighbor/ops -tensorflow/contrib/nearest_neighbor/python -tensorflow/contrib/nearest_neighbor/python/ops -tensorflow/contrib/nn -tensorflow/contrib/nn/python -tensorflow/contrib/nn/python/ops -tensorflow/contrib/opt -tensorflow/contrib/opt/python -tensorflow/contrib/opt/python/training -tensorflow/contrib/optimizer_v2 -tensorflow/contrib/pi_examples -tensorflow/contrib/pi_examples/camera -tensorflow/contrib/pi_examples/label_image -tensorflow/contrib/pi_examples/label_image/data -tensorflow/contrib/periodic_resample -tensorflow/contrib/periodic_resample/python -tensorflow/contrib/periodic_resample/python/ops -tensorflow/contrib/predictor -tensorflow/contrib/proto -tensorflow/contrib/proto/python -tensorflow/contrib/proto/python/ops -tensorflow/contrib/quantization -tensorflow/contrib/quantization/python -tensorflow/contrib/quantize -tensorflow/contrib/quantize/python -tensorflow/contrib/receptive_field -tensorflow/contrib/receptive_field/python -tensorflow/contrib/receptive_field/python/util -tensorflow/contrib/receptive_field/python/util/examples -tensorflow/contrib/recurrent -tensorflow/contrib/recurrent/python -tensorflow/contrib/recurrent/python/ops -tensorflow/contrib/recurrent/python/kernel_tests -tensorflow/contrib/reduce_slice_ops -tensorflow/contrib/reduce_slice_ops/kernels -tensorflow/contrib/reduce_slice_ops/ops -tensorflow/contrib/reduce_slice_ops/python -tensorflow/contrib/reduce_slice_ops/python/ops -tensorflow/contrib/remote_fused_graph -tensorflow/contrib/remote_fused_graph/pylib -tensorflow/contrib/remote_fused_graph/pylib/python -tensorflow/contrib/remote_fused_graph/pylib/python/ops -tensorflow/contrib/resampler -tensorflow/contrib/resampler/kernels -tensorflow/contrib/resampler/ops -tensorflow/contrib/resampler/python -tensorflow/contrib/resampler/python/ops -tensorflow/contrib/rnn -tensorflow/contrib/rnn/kernels -tensorflow/contrib/rnn/ops -tensorflow/contrib/rnn/python -tensorflow/contrib/rnn/python/kernel_tests -tensorflow/contrib/rnn/python/ops -tensorflow/contrib/rpc -tensorflow/contrib/rpc/python -tensorflow/contrib/rpc/python/ops -tensorflow/contrib/saved_model -tensorflow/contrib/saved_model/python -tensorflow/contrib/saved_model/python/saved_model -tensorflow/contrib/seq2seq -tensorflow/contrib/seq2seq/kernels -tensorflow/contrib/seq2seq/ops -tensorflow/contrib/seq2seq/python -tensorflow/contrib/seq2seq/python/ops -tensorflow/contrib/session_bundle -tensorflow/contrib/session_bundle/example -tensorflow/contrib/signal -tensorflow/contrib/slim -tensorflow/contrib/slim/python -tensorflow/contrib/slim/python/slim -tensorflow/contrib/slim/python/slim/data -tensorflow/contrib/slim/python/slim/nets -tensorflow/contrib/solvers -tensorflow/contrib/solvers/python -tensorflow/contrib/solvers/python/ops -tensorflow/contrib/sparsemax -tensorflow/contrib/sparsemax/python -tensorflow/contrib/sparsemax/python/ops -tensorflow/contrib/specs -tensorflow/contrib/specs/python -tensorflow/contrib/staging -tensorflow/contrib/stat_summarizer -tensorflow/contrib/stat_summarizer/python -tensorflow/contrib/stateless -tensorflow/contrib/stateless/python -tensorflow/contrib/summary -tensorflow/contrib/tensorboard -tensorflow/contrib/tensorboard/plugins -tensorflow/contrib/tensorboard/plugins/projector -# TODO(sami): Add cmake implementations. -# tensorflow/contrib/tensorrt/python -# tensorflow/contrib/tensorrt/python/ops -tensorflow/contrib/tensor_forest -tensorflow/contrib/tensor_forest/client -tensorflow/contrib/tensor_forest/hybrid -tensorflow/contrib/tensor_forest/hybrid/core -tensorflow/contrib/tensor_forest/hybrid/core/ops -tensorflow/contrib/tensor_forest/hybrid/python -tensorflow/contrib/tensor_forest/hybrid/python/layers -tensorflow/contrib/tensor_forest/hybrid/python/models -tensorflow/contrib/tensor_forest/hybrid/python/ops -tensorflow/contrib/tensor_forest/kernels -tensorflow/contrib/tensor_forest/proto -tensorflow/contrib/tensor_forest/python -tensorflow/contrib/tensor_forest/python/ops -tensorflow/contrib/testing -tensorflow/contrib/testing/python -tensorflow/contrib/testing/python/framework -tensorflow/contrib/text -tensorflow/contrib/text/kernels -tensorflow/contrib/text/ops -tensorflow/contrib/text/python -tensorflow/contrib/text/python/ops -tensorflow/contrib/tfprof -tensorflow/contrib/timeseries -tensorflow/contrib/timeseries/examples -tensorflow/contrib/timeseries/examples/data -tensorflow/contrib/timeseries/python -tensorflow/contrib/timeseries/python/timeseries -tensorflow/contrib/timeseries/python/timeseries/state_space_models -tensorflow/contrib/tpu -tensorflow/contrib/tpu/profiler -tensorflow/contrib/tpu/python -tensorflow/contrib/tpu/python/ops -tensorflow/contrib/tpu/python/profiler -tensorflow/contrib/tpu/python/tpu -tensorflow/contrib/training -tensorflow/contrib/training/python -tensorflow/contrib/training/python/training -tensorflow/contrib/util diff --git a/tensorflow/contrib/cmake/python_protos.txt b/tensorflow/contrib/cmake/python_protos.txt deleted file mode 100644 index b4603206da4..00000000000 --- a/tensorflow/contrib/cmake/python_protos.txt +++ /dev/null @@ -1,18 +0,0 @@ -tensorflow/core -tensorflow/core/kernels/boosted_trees -tensorflow/core/profiler -tensorflow/core/protobuf/tpu -tensorflow/python -tensorflow/contrib/boosted_trees/proto -tensorflow/contrib/cloud/kernels -tensorflow/contrib/decision_trees/proto -tensorflow/contrib/gdr -tensorflow/lite/toco -tensorflow/contrib/mpi -tensorflow/contrib/mpi_collectives -tensorflow/contrib/session_bundle -tensorflow/contrib/tensor_forest/proto -tensorflow/contrib/tensorboard/plugins/projector -tensorflow/contrib/tpu/profiler -tensorflow/contrib/training/python/training -tensorflow/contrib/verbs diff --git a/tensorflow/contrib/cmake/python_protos_cc.txt b/tensorflow/contrib/cmake/python_protos_cc.txt deleted file mode 100644 index d4a257b25c8..00000000000 --- a/tensorflow/contrib/cmake/python_protos_cc.txt +++ /dev/null @@ -1,5 +0,0 @@ -tensorflow/core/profiler -tensorflow/python -tensorflow/contrib/session_bundle -tensorflow/contrib/tensorboard -tensorflow/contrib/training diff --git a/tensorflow/contrib/cmake/python_sanity_test.py b/tensorflow/contrib/cmake/python_sanity_test.py deleted file mode 100644 index e0056823a80..00000000000 --- a/tensorflow/contrib/cmake/python_sanity_test.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Complain about invalid or missing entries in python_*.txt files. - -Problematic entries can be commented for temporary whitelisting. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import unittest - - -def abs_path(path): - root = os.path.dirname(__file__) - - for _ in range(3): - root = os.path.join(root, os.pardir) - - path = os.path.join(root, path) - path = os.path.abspath(path) - return path - - -def read_entries(test): - with open(abs_path(test.entries_file), "r") as f: - lines = f.readlines() - - lines = [line.strip() for line in lines] - lines = [line for line in lines if line] - - test.entries = [] - test.whitelist = [] - - for line in lines: - # line is comment - if line.startswith("#"): - line = line[1:].strip() - # whitelist entry - if line.startswith("tensorflow/"): - test.whitelist.append(line) - # line has comment -> strip comment - elif line.find("#") != -1: - line = line[:line.find("#")].strip() - test.entries.append(line) - else: - test.entries.append(line) - - -def test_invalid_directories(test): - for entry in test.entries: - if not os.path.isdir(abs_path(entry)): - problem = "'" + test.entries_file + "' contains invalid '" + entry + "'" - solution = ("Please remove the invalid entry (or add the missing " - "directory).") - raise AssertionError(problem + "\n" + solution) - - -def test_missing_directory(test, path): - if path in test.whitelist: - return - - dir_exists = os.path.isdir(abs_path(path)) - entry_exists = path in test.entries - - if dir_exists and not entry_exists: - problem = "'" + test.entries_file + "' is missing '" + path + "'" - solution = "Please add the missing entry (comment to whitelist if needed)." - raise AssertionError(problem + "\n" + solution) - - -class PythonModuleTest(unittest.TestCase): - - def setUp(self): - self.entries_file = "tensorflow/contrib/cmake/python_modules.txt" - read_entries(self) - - def testInvalidEntries(self): - test_invalid_directories(self) - - def testMissingModules(self): - module_names = next(os.walk(abs_path("tensorflow/contrib")))[1] - - for module_name in module_names: - path = "tensorflow/contrib/" + module_name - - test_missing_directory(self, path + "/python") - test_missing_directory(self, path + "/python/ops") - test_missing_directory(self, path + "/python/kernels") - test_missing_directory(self, path + "/python/layers") - - -class PythonProtoTest(unittest.TestCase): - - def setUp(self): - self.entries_file = "tensorflow/contrib/cmake/python_protos.txt" - read_entries(self) - - def testInvalidEntries(self): - test_invalid_directories(self) - - -class PythonProtoCCTest(unittest.TestCase): - - def setUp(self): - self.entries_file = "tensorflow/contrib/cmake/python_protos_cc.txt" - read_entries(self) - - def testInvalidEntries(self): - test_invalid_directories(self) - - -if __name__ == "__main__": - unittest.main() diff --git a/tensorflow/contrib/cmake/tests/cuda/compatibility_test.c b/tensorflow/contrib/cmake/tests/cuda/compatibility_test.c deleted file mode 100644 index 9e355da33a7..00000000000 --- a/tensorflow/contrib/cmake/tests/cuda/compatibility_test.c +++ /dev/null @@ -1,22 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// This is a program to test if compiler is compatible with CUDA. -#define __CUDACC__ -#include "crt/host_config.h" - -int main(void) { - return 0; -} diff --git a/tensorflow/contrib/cmake/tests/cuda/compatibility_test.cc b/tensorflow/contrib/cmake/tests/cuda/compatibility_test.cc deleted file mode 100644 index beb574061be..00000000000 --- a/tensorflow/contrib/cmake/tests/cuda/compatibility_test.cc +++ /dev/null @@ -1,22 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -============================================================================*/ - -// This is a program to test if compiler is compatible with CUDA. -#define __CUDACC__ -#include "crt/host_config.h" - -int main(void) { - return 0; -} diff --git a/tensorflow/contrib/cmake/tf_c.cmake b/tensorflow/contrib/cmake/tf_c.cmake deleted file mode 100644 index a04142bd249..00000000000 --- a/tensorflow/contrib/cmake/tf_c.cmake +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -######################################################## -# tf_c_framework library -######################################################## -set(tf_c_srcs - "${tensorflow_source_dir}/tensorflow/c/c_api.cc" - "${tensorflow_source_dir}/tensorflow/c/c_api.h" - "${tensorflow_source_dir}/tensorflow/c/c_api_function.cc" - "${tensorflow_source_dir}/tensorflow/c/eager/c_api.cc" - "${tensorflow_source_dir}/tensorflow/c/eager/c_api.h" - "${tensorflow_source_dir}/tensorflow/c/eager/c_api_debug.cc" - "${tensorflow_source_dir}/tensorflow/c/eager/tape.h" - "${tensorflow_source_dir}/tensorflow/c/checkpoint_reader.cc" - "${tensorflow_source_dir}/tensorflow/c/checkpoint_reader.h" - "${tensorflow_source_dir}/tensorflow/c/tf_status_helper.cc" - "${tensorflow_source_dir}/tensorflow/c/tf_status_helper.h" -) - -add_library(tf_c OBJECT ${tf_c_srcs}) -add_dependencies( - tf_c - tf_cc_framework - tf_cc_while_loop - tf_core_lib - tf_protos_cc) diff --git a/tensorflow/contrib/cmake/tf_cc_ops.cmake b/tensorflow/contrib/cmake/tf_cc_ops.cmake deleted file mode 100644 index 6514ae50a4a..00000000000 --- a/tensorflow/contrib/cmake/tf_cc_ops.cmake +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -######################################################## -# tf_cc_framework library -######################################################## -set(tf_cc_framework_srcs - "${tensorflow_source_dir}/tensorflow/cc/framework/ops.h" - "${tensorflow_source_dir}/tensorflow/cc/framework/ops.cc" - "${tensorflow_source_dir}/tensorflow/cc/framework/scope.h" - "${tensorflow_source_dir}/tensorflow/cc/framework/scope_internal.h" - "${tensorflow_source_dir}/tensorflow/cc/framework/scope.cc" -) - -add_library(tf_cc_framework OBJECT ${tf_cc_framework_srcs}) - -add_dependencies(tf_cc_framework tf_core_framework) - -######################################################## -# tf_cc_op_gen_main library -######################################################## -set(tf_cc_op_gen_main_srcs - "${tensorflow_source_dir}/tensorflow/cc/framework/cc_op_gen.cc" - "${tensorflow_source_dir}/tensorflow/cc/framework/cc_op_gen_main.cc" - "${tensorflow_source_dir}/tensorflow/cc/framework/cc_op_gen.h" -) - -add_library(tf_cc_op_gen_main OBJECT ${tf_cc_op_gen_main_srcs}) - -add_dependencies(tf_cc_op_gen_main tf_core_framework) - -######################################################## -# tf_gen_op_wrapper_cc executables -######################################################## - -# create directory for ops generated files -set(cc_ops_target_dir ${CMAKE_CURRENT_BINARY_DIR}/tensorflow/cc/ops) - -add_custom_target(create_cc_ops_header_dir - COMMAND ${CMAKE_COMMAND} -E make_directory ${cc_ops_target_dir} -) - -set(tf_cc_ops_generated_files) - -set(tf_cc_op_lib_names - ${tf_op_lib_names} - "user_ops" -) -foreach(tf_cc_op_lib_name ${tf_cc_op_lib_names}) - # Using to work around an issue where no ops were - # registered (static initializers dropped by the linker because the ops - # are not used explicitly in the *_gen_cc executables). - add_executable(${tf_cc_op_lib_name}_gen_cc - $ - $ - $ - $ - ) - - target_link_libraries(${tf_cc_op_lib_name}_gen_cc PRIVATE - tf_protos_cc - ${tensorflow_EXTERNAL_LIBRARIES} - ) - - set(cc_ops_include_internal 0) - if(${tf_cc_op_lib_name} STREQUAL "sendrecv_ops") - set(cc_ops_include_internal 1) - endif() - - add_custom_command( - OUTPUT ${cc_ops_target_dir}/${tf_cc_op_lib_name}.h - ${cc_ops_target_dir}/${tf_cc_op_lib_name}.cc - ${cc_ops_target_dir}/${tf_cc_op_lib_name}_internal.h - ${cc_ops_target_dir}/${tf_cc_op_lib_name}_internal.cc - COMMAND ${tf_cc_op_lib_name}_gen_cc ${cc_ops_target_dir}/${tf_cc_op_lib_name}.h ${cc_ops_target_dir}/${tf_cc_op_lib_name}.cc ${cc_ops_include_internal} ${tensorflow_source_dir}/tensorflow/core/api_def/base_api - DEPENDS ${tf_cc_op_lib_name}_gen_cc create_cc_ops_header_dir - ) - - list(APPEND tf_cc_ops_generated_files ${cc_ops_target_dir}/${tf_cc_op_lib_name}.h) - list(APPEND tf_cc_ops_generated_files ${cc_ops_target_dir}/${tf_cc_op_lib_name}.cc) - list(APPEND tf_cc_ops_generated_files ${cc_ops_target_dir}/${tf_cc_op_lib_name}_internal.h) - list(APPEND tf_cc_ops_generated_files ${cc_ops_target_dir}/${tf_cc_op_lib_name}_internal.cc) -endforeach() - - - -######################################################## -# tf_cc_ops library -######################################################## -add_library(tf_cc_ops OBJECT - ${tf_cc_ops_generated_files} - "${tensorflow_source_dir}/tensorflow/cc/ops/const_op.h" - "${tensorflow_source_dir}/tensorflow/cc/ops/const_op.cc" - "${tensorflow_source_dir}/tensorflow/cc/ops/standard_ops.h" -) - -######################################################## -# tf_cc_while_loop library -######################################################## -add_library(tf_cc_while_loop OBJECT - "${tensorflow_source_dir}/tensorflow/cc/ops/while_loop.h" - "${tensorflow_source_dir}/tensorflow/cc/ops/while_loop.cc" -) - -add_dependencies(tf_cc_while_loop tf_core_framework tf_cc_ops) - -######################################################## -# tf_cc library -######################################################## -file(GLOB_RECURSE tf_cc_srcs - "${tensorflow_source_dir}/tensorflow/cc/client/*.h" - "${tensorflow_source_dir}/tensorflow/cc/client/*.cc" - "${tensorflow_source_dir}/tensorflow/cc/gradients/*.h" - "${tensorflow_source_dir}/tensorflow/cc/gradients/*.cc" - "${tensorflow_source_dir}/tensorflow/cc/training/*.h" - "${tensorflow_source_dir}/tensorflow/cc/training/*.cc" -) - -set(tf_cc_srcs - ${tf_cc_srcs} - "${tensorflow_source_dir}/tensorflow/cc/framework/grad_op_registry.h" - "${tensorflow_source_dir}/tensorflow/cc/framework/grad_op_registry.cc" - "${tensorflow_source_dir}/tensorflow/cc/framework/gradient_checker.h" - "${tensorflow_source_dir}/tensorflow/cc/framework/gradient_checker.cc" - "${tensorflow_source_dir}/tensorflow/cc/framework/gradients.h" - "${tensorflow_source_dir}/tensorflow/cc/framework/gradients.cc" - "${tensorflow_source_dir}/tensorflow/cc/framework/while_gradients.h" - "${tensorflow_source_dir}/tensorflow/cc/framework/while_gradients.cc" -) - -file(GLOB_RECURSE tf_cc_test_srcs - "${tensorflow_source_dir}/tensorflow/cc/*test*.cc" -) - -list(REMOVE_ITEM tf_cc_srcs ${tf_cc_test_srcs}) - -add_library(tf_cc OBJECT ${tf_cc_srcs}) -add_dependencies(tf_cc tf_cc_framework tf_cc_ops) - -if (WIN32) - set (pywrap_tensorflow_lib "${CMAKE_CURRENT_BINARY_DIR}/$(Configuration)/pywrap_tensorflow_internal.lib") -else (WIN32) - set (pywrap_tensorflow_lib "${CMAKE_CURRENT_BINARY_DIR}/libpywrap_tensorflow_internal${CMAKE_SHARED_LIBRARY_SUFFIX}") -endif (WIN32) -add_custom_target(tf_extension_ops) - -function(AddUserOps) - cmake_parse_arguments(_AT "" "" "TARGET;SOURCES;GPUSOURCES;DEPENDS;DISTCOPY" ${ARGN}) - if (tensorflow_ENABLE_GPU AND _AT_GPUSOURCES) - # if gpu build is enabled and we have gpu specific code, - # hint to cmake that this needs to go to nvcc - set (gpu_source ${_AT_GPUSOURCES}) - set (gpu_lib "${_AT_TARGET}_gpu") - set_source_files_properties(${gpu_source} PROPERTIES CUDA_SOURCE_PROPERTY_FORMAT OBJ) - cuda_compile(gpu_lib ${gpu_source}) - endif() - # create shared library from source and cuda obj - add_library(${_AT_TARGET} SHARED ${_AT_SOURCES} ${gpu_lib}) - target_link_libraries(${_AT_TARGET} ${pywrap_tensorflow_lib}) - if (tensorflow_ENABLE_GPU AND _AT_GPUSOURCES) - # some ops call out to cuda directly; need to link libs for the cuda dlls - target_link_libraries(${_AT_TARGET} ${CUDA_LIBRARIES}) - endif() - if (_AT_DISTCOPY) - add_custom_command(TARGET ${_AT_TARGET} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy $ ${_AT_DISTCOPY}/) - endif() - if (_AT_DEPENDS) - add_dependencies(${_AT_TARGET} ${_AT_DEPENDS}) - endif() - # make sure TF_COMPILE_LIBRARY is not defined for this target - get_target_property(target_compile_flags ${_AT_TARGET} COMPILE_FLAGS) - if(target_compile_flags STREQUAL "target_compile_flags-NOTFOUND") - if (WIN32) - set(target_compile_flags "/UTF_COMPILE_LIBRARY") - else (WIN32) - # gcc uses UTF as default - set(target_compile_flags "-finput-charset=UTF-8") - endif (WIN32) - else() - if (WIN32) - set(target_compile_flags "${target_compile_flags} /UTF_COMPILE_LIBRARY") - else (WIN32) - # gcc uses UTF as default - set(target_compile_flags "${target_compile_flags} -finput-charset=UTF-8") - endif (WIN32) - endif() - set_target_properties(${_AT_TARGET} PROPERTIES COMPILE_FLAGS ${target_compile_flags}) - add_dependencies(tf_extension_ops ${_AT_TARGET}) -endfunction(AddUserOps) diff --git a/tensorflow/contrib/cmake/tf_core_cpu.cmake b/tensorflow/contrib/cmake/tf_core_cpu.cmake deleted file mode 100644 index d8884d464fb..00000000000 --- a/tensorflow/contrib/cmake/tf_core_cpu.cmake +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -######################################################## -# tf_core_cpu library -######################################################## -file(GLOB_RECURSE tf_core_cpu_srcs - "${tensorflow_source_dir}/tensorflow/cc/saved_model/*.h" - "${tensorflow_source_dir}/tensorflow/cc/saved_model/*.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/*.h" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/*.cc" - "${tensorflow_source_dir}/tensorflow/core/debug/*.h" - "${tensorflow_source_dir}/tensorflow/core/debug/*.cc" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/server_lib.h" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/server_lib.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/*.h" - "${tensorflow_source_dir}/tensorflow/core/graph/*.cc" - "${tensorflow_source_dir}/tensorflow/core/grappler/*.h" - "${tensorflow_source_dir}/tensorflow/core/grappler/*.cc" - "${tensorflow_source_dir}/tensorflow/core/grappler/*/*.h" - "${tensorflow_source_dir}/tensorflow/core/grappler/*/*.cc" - "${tensorflow_source_dir}/tensorflow/core/public/*.h" -) - -file(GLOB_RECURSE tf_core_cpu_exclude_srcs - "${tensorflow_source_dir}/tensorflow/cc/saved_model/*test*.h" - "${tensorflow_source_dir}/tensorflow/cc/saved_model/*test*.cc" - "${tensorflow_source_dir}/tensorflow/core/*test*.h" - "${tensorflow_source_dir}/tensorflow/core/*test*.cc" - "${tensorflow_source_dir}/tensorflow/core/*main.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/eager/*.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/eager/*.h" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/*.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu_device_factory.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/direct_session.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/direct_session.h" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/session.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/session_factory.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/session_options.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/edgeset.h" - "${tensorflow_source_dir}/tensorflow/core/graph/edgeset.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/graph.h" - "${tensorflow_source_dir}/tensorflow/core/graph/graph.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/graph_def_builder.h" - "${tensorflow_source_dir}/tensorflow/core/graph/graph_def_builder.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/node_builder.h" - "${tensorflow_source_dir}/tensorflow/core/graph/node_builder.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/tensor_id.h" - "${tensorflow_source_dir}/tensorflow/core/graph/tensor_id.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/while_context.h" - "${tensorflow_source_dir}/tensorflow/core/graph/while_context.cc" - "${tensorflow_source_dir}/tensorflow/core/grappler/clusters/single_machine.h" - "${tensorflow_source_dir}/tensorflow/core/grappler/clusters/single_machine.cc" - "${tensorflow_source_dir}/tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.h" - "${tensorflow_source_dir}/tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.cc" -) -file(GLOB_RECURSE tf_core_cpu_whitelisted_srcs - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/gpu_id.h" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/gpu_id.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/gpu_id_manager.cc" -) -list(REMOVE_ITEM tf_core_cpu_exclude_srcs ${tf_core_cpu_whitelisted_srcs}) -list(REMOVE_ITEM tf_core_cpu_srcs ${tf_core_cpu_exclude_srcs}) - -if (tensorflow_ENABLE_GPU) - file(GLOB_RECURSE tf_core_gpu_srcs - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/*.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/default/gpu/cupti_wrapper.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/default/device_tracer.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu_device_factory.cc" - "${tensorflow_source_dir}/tensorflow/core/grappler/devices.h" - "${tensorflow_source_dir}/tensorflow/core/grappler/devices.cc" - ) - file(GLOB_RECURSE tf_core_gpu_exclude_srcs - "${tensorflow_source_dir}/tensorflow/core/*test*.cc" - "${tensorflow_source_dir}/tensorflow/core/*test*.cc" - ) - list(REMOVE_ITEM tf_core_gpu_srcs ${tf_core_gpu_exclude_srcs}) - list(REMOVE_ITEM tf_core_gpu_srcs ${tf_core_cpu_whitelisted_srcs}) - list(APPEND tf_core_cpu_srcs ${tf_core_gpu_srcs}) -endif() - -add_library(tf_core_cpu OBJECT ${tf_core_cpu_srcs}) -add_dependencies(tf_core_cpu tf_core_framework) diff --git a/tensorflow/contrib/cmake/tf_core_direct_session.cmake b/tensorflow/contrib/cmake/tf_core_direct_session.cmake deleted file mode 100644 index de2fa866957..00000000000 --- a/tensorflow/contrib/cmake/tf_core_direct_session.cmake +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -######################################################## -# tf_core_direct_session library -######################################################## -file(GLOB tf_core_direct_session_srcs - "${tensorflow_source_dir}/tensorflow/core/common_runtime/direct_session.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/direct_session.h" -) - -file(GLOB_RECURSE tf_core_direct_session_test_srcs - "${tensorflow_source_dir}/tensorflow/core/debug/*test*.h" - "${tensorflow_source_dir}/tensorflow/core/debug/*test*.cc" -) - -list(REMOVE_ITEM tf_core_direct_session_srcs ${tf_core_direct_session_test_srcs}) - -add_library(tf_core_direct_session OBJECT ${tf_core_direct_session_srcs}) - -add_dependencies(tf_core_direct_session tf_core_cpu) \ No newline at end of file diff --git a/tensorflow/contrib/cmake/tf_core_distributed_runtime.cmake b/tensorflow/contrib/cmake/tf_core_distributed_runtime.cmake deleted file mode 100644 index 2c1b6d1f6e5..00000000000 --- a/tensorflow/contrib/cmake/tf_core_distributed_runtime.cmake +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -######################################################## -# tf_core_distributed_runtime library -######################################################## -file(GLOB_RECURSE tf_core_distributed_runtime_srcs - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/*.h" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/*.cc" -) - -file(GLOB_RECURSE tf_core_distributed_runtime_exclude_srcs - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/server_lib.cc" # Build in tf_core_cpu instead. - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/*test*.h" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/*test*.cc" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/rpc/grpc_tensorflow_server.cc" -) - -list(REMOVE_ITEM tf_core_distributed_runtime_srcs ${tf_core_distributed_runtime_exclude_srcs}) - -add_library(tf_core_distributed_runtime OBJECT ${tf_core_distributed_runtime_srcs}) - -add_dependencies(tf_core_distributed_runtime - tf_core_cpu grpc -) - -######################################################## -# grpc_tensorflow_server executable -######################################################## -set(grpc_tensorflow_server_srcs - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/rpc/grpc_tensorflow_server.cc" -) - -add_executable(grpc_tensorflow_server - ${grpc_tensorflow_server_srcs} - $ - $ - $ - $ - $ - $ - $ - $ - $ - $<$:$> -) - -target_link_libraries(grpc_tensorflow_server PUBLIC - tf_protos_cc - ${tf_core_gpu_kernels_lib} - ${tensorflow_EXTERNAL_LIBRARIES} -) diff --git a/tensorflow/contrib/cmake/tf_core_eager_runtime.cmake b/tensorflow/contrib/cmake/tf_core_eager_runtime.cmake deleted file mode 100644 index 78e4c0d3035..00000000000 --- a/tensorflow/contrib/cmake/tf_core_eager_runtime.cmake +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -######################################################## -# tf_core_eager_runtime library -######################################################## -file(GLOB_RECURSE tf_core_eager_runtime_srcs - "${tensorflow_source_dir}/tensorflow/core/common_runtime/eager/*.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/eager/*.h" -) - -file(GLOB_RECURSE tf_core_eager_runtime_exclude_srcs - "${tensorflow_source_dir}/tensorflow/core/common_runtime/eager/*test*.h" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/eager/*test*.cc" -) - -list(REMOVE_ITEM tf_core_eager_runtime_srcs ${tf_core_eager_runtime_exclude_srcs}) - -add_library(tf_core_eager_runtime OBJECT ${tf_core_eager_runtime_srcs}) -add_dependencies( - tf_core_eager_runtime - tf_c - tf_core_lib) - - -file(GLOB_RECURSE tf_c_eager_srcs - "${tensorflow_source_dir}/tensorflow/c/eager/*.cc" - "${tensorflow_source_dir}/tensorflow/c/eager/*.h" -) - -file(GLOB_RECURSE tf_c_eager_exlclude_srcs - "${tensorflow_source_dir}/tensorflow/c/eager/*test*.h" - "${tensorflow_source_dir}/tensorflow/c/eager/*test*.cc" -) - -list(REMOVE_ITEM tf_c_eager_srcs ${tf_c_eager_exlclude_srcs}) - -add_library(tf_c_eager OBJECT ${tf_c_eager_srcs}) -add_dependencies( - tf_c_eager - tf_core_eager_runtime - tf_c - tf_cc_framework - tf_cc_while_loop - tf_core_lib - tf_protos_cc) \ No newline at end of file diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake deleted file mode 100644 index 24e45236a63..00000000000 --- a/tensorflow/contrib/cmake/tf_core_framework.cmake +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -######################################################## -# RELATIVE_PROTOBUF_GENERATE_CPP function -######################################################## -# A variant of PROTOBUF_GENERATE_CPP that keeps the directory hierarchy. -# ROOT_DIR must be absolute, and proto paths must be relative to ROOT_DIR. -function(RELATIVE_PROTOBUF_GENERATE_CPP SRCS HDRS ROOT_DIR) - if(NOT ARGN) - message(SEND_ERROR "Error: RELATIVE_PROTOBUF_GENERATE_CPP() called without any proto files") - return() - endif() - - set(${SRCS}) - set(${HDRS}) - foreach(FIL ${ARGN}) - set(ABS_FIL ${ROOT_DIR}/${FIL}) - get_filename_component(FIL_WE ${FIL} NAME_WE) - get_filename_component(FIL_DIR ${ABS_FIL} PATH) - file(RELATIVE_PATH REL_DIR ${ROOT_DIR} ${FIL_DIR}) - - list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.cc") - list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.h") - - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.cc" - "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.h" - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} - ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} -I ${ROOT_DIR} ${ABS_FIL} -I ${PROTOBUF_INCLUDE_DIRS} - DEPENDS ${ABS_FIL} protobuf - COMMENT "Running C++ protocol buffer compiler on ${FIL}" - VERBATIM ) - endforeach() - - set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) - set(${SRCS} ${${SRCS}} PARENT_SCOPE) - set(${HDRS} ${${HDRS}} PARENT_SCOPE) -endfunction() - -if(NOT WIN32) - function(RELATIVE_PROTOBUF_GENERATE_GRPC_CPP SRCS HDRS ROOT_DIR) - if(NOT ARGN) - message(SEND_ERROR "Error: RELATIVE_PROTOBUF_GENERATE_GRPC_CPP() called without any proto files") - return() - endif() - - set(${SRCS}) - set(${HDRS}) - foreach(FIL ${ARGN}) - set(ABS_FIL ${ROOT_DIR}/${FIL}) - get_filename_component(FIL_WE ${FIL} NAME_WE) - get_filename_component(FIL_DIR ${ABS_FIL} PATH) - file(RELATIVE_PATH REL_DIR ${ROOT_DIR} ${FIL_DIR}) - - list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.grpc.pb.cc") - list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.grpc.pb.h") - list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.cc") - list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.h") - - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.grpc.pb.cc" - "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.grpc.pb.h" - "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.cc" - "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.h" - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} - ARGS --grpc_out ${CMAKE_CURRENT_BINARY_DIR} --cpp_out ${CMAKE_CURRENT_BINARY_DIR} --plugin protoc-gen-grpc=${GRPC_BUILD}/grpc_cpp_plugin -I ${ROOT_DIR} ${ABS_FIL} -I ${PROTOBUF_INCLUDE_DIRS} - DEPENDS ${ABS_FIL} protobuf grpc - COMMENT "Running C++ protocol buffer grpc compiler on ${FIL}" - VERBATIM ) - endforeach() - - set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) - set(${SRCS} ${${SRCS}} PARENT_SCOPE) - set(${HDRS} ${${HDRS}} PARENT_SCOPE) - endfunction() -endif() - -function(RELATIVE_PROTOBUF_TEXT_GENERATE_CPP SRCS HDRS ROOT_DIR) - if(NOT ARGN) - message(SEND_ERROR "Error: RELATIVE_PROTOBUF_TEXT_GENERATE_CPP() called without any proto files") - return() - endif() - - set(${SRCS}) - set(${HDRS}) - foreach(FIL ${ARGN}) - set(ABS_FIL ${ROOT_DIR}/${FIL}) - get_filename_component(FIL_WE ${FIL} NAME_WE) - get_filename_component(FIL_DIR ${ABS_FIL} PATH) - file(RELATIVE_PATH REL_DIR ${ROOT_DIR} ${FIL_DIR}) - - list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb_text.cc") - list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb_text.h") - - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb_text.cc" - "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb_text.h" - COMMAND ${PROTO_TEXT_EXE} - ARGS "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}" ${REL_DIR} ${ABS_FIL} "${ROOT_DIR}/tensorflow/tools/proto_text/placeholder.txt" - DEPENDS ${ABS_FIL} ${PROTO_TEXT_EXE} - COMMENT "Running C++ protocol buffer text compiler (${PROTO_TEXT_EXE}) on ${FIL}" - VERBATIM ) - endforeach() - - set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) - set(${SRCS} ${${SRCS}} PARENT_SCOPE) - set(${HDRS} ${${HDRS}} PARENT_SCOPE) -endfunction() - -######################################################## -# tf_protos_cc library -######################################################## - -file(GLOB_RECURSE tf_protos_cc_srcs RELATIVE ${tensorflow_source_dir} - "${tensorflow_source_dir}/tensorflow/core/*.proto" - "${tensorflow_source_dir}/tensorflow/core/protobuf/tpu/*.proto" - "${tensorflow_source_dir}/tensorflow/compiler/xla/*.proto" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/proto/*.proto" -) - -RELATIVE_PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS - ${tensorflow_source_dir} ${tf_protos_cc_srcs} -) - - -set(PROTO_TEXT_EXE "proto_text") -set(tf_proto_text_srcs - "tensorflow/core/example/example.proto" - "tensorflow/core/example/feature.proto" - "tensorflow/core/framework/allocation_description.proto" - "tensorflow/core/framework/api_def.proto" - "tensorflow/core/framework/attr_value.proto" - "tensorflow/core/framework/cost_graph.proto" - "tensorflow/core/framework/device_attributes.proto" - "tensorflow/core/framework/function.proto" - "tensorflow/core/framework/graph.proto" - "tensorflow/core/framework/graph_transfer_info.proto" - "tensorflow/core/framework/kernel_def.proto" - "tensorflow/core/framework/log_memory.proto" - "tensorflow/core/framework/node_def.proto" - "tensorflow/core/framework/op_def.proto" - "tensorflow/core/framework/reader_base.proto" - "tensorflow/core/framework/remote_fused_graph_execute_info.proto" - "tensorflow/core/framework/resource_handle.proto" - "tensorflow/core/framework/step_stats.proto" - "tensorflow/core/framework/summary.proto" - "tensorflow/core/framework/tensor.proto" - "tensorflow/core/framework/tensor_description.proto" - "tensorflow/core/framework/tensor_shape.proto" - "tensorflow/core/framework/tensor_slice.proto" - "tensorflow/core/framework/types.proto" - "tensorflow/core/framework/variable.proto" - "tensorflow/core/framework/versions.proto" - "tensorflow/core/lib/core/error_codes.proto" - "tensorflow/core/protobuf/cluster.proto" - "tensorflow/core/protobuf/config.proto" - "tensorflow/core/protobuf/debug.proto" - "tensorflow/core/protobuf/device_properties.proto" - "tensorflow/core/protobuf/rewriter_config.proto" - "tensorflow/core/protobuf/tensor_bundle.proto" - "tensorflow/core/protobuf/saver.proto" - "tensorflow/core/util/memmapped_file_system.proto" - "tensorflow/core/util/saved_tensor_slice.proto" -) -RELATIVE_PROTOBUF_TEXT_GENERATE_CPP(PROTO_TEXT_SRCS PROTO_TEXT_HDRS - ${tensorflow_source_dir} ${tf_proto_text_srcs} -) - -if(WIN32) - add_library(tf_protos_cc ${PROTO_SRCS} ${PROTO_HDRS}) -else() - file(GLOB_RECURSE tf_protos_grpc_cc_srcs RELATIVE ${tensorflow_source_dir} - "${tensorflow_source_dir}/tensorflow/core/debug/*.proto" - ) - RELATIVE_PROTOBUF_GENERATE_GRPC_CPP(PROTO_GRPC_SRCS PROTO_GRPC_HDRS - ${tensorflow_source_dir} ${tf_protos_grpc_cc_srcs} - ) - add_library(tf_protos_cc ${PROTO_GRPC_SRCS} ${PROTO_GRPC_HDRS} ${PROTO_SRCS} ${PROTO_HDRS}) -endif() - -######################################################## -# tf_core_lib library -######################################################## -file(GLOB_RECURSE tf_core_lib_srcs - "${tensorflow_source_dir}/tensorflow/core/lib/*.h" - "${tensorflow_source_dir}/tensorflow/core/lib/*.cc" - "${tensorflow_source_dir}/tensorflow/core/public/*.h" -) - -file(GLOB tf_core_platform_srcs - "${tensorflow_source_dir}/tensorflow/core/platform/*.h" - "${tensorflow_source_dir}/tensorflow/core/platform/*.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/default/*.h" - "${tensorflow_source_dir}/tensorflow/core/platform/default/*.cc" - "${tensorflow_source_dir}/tensorflow/core/framework/resource_handle.h" - "${tensorflow_source_dir}/tensorflow/core/framework/resource_handle.cc") -if (NOT tensorflow_ENABLE_GPU) - file(GLOB tf_core_platform_gpu_srcs_exclude - "${tensorflow_source_dir}/tensorflow/core/platform/cuda_libdevice_path.*" - "${tensorflow_source_dir}/tensorflow/core/platform/default/cuda_libdevice_path.*") - list(REMOVE_ITEM tf_core_platform_srcs ${tf_core_platform_gpu_srcs_exclude}) -else() - file(GLOB tf_core_platform_srcs_exclude - "${tensorflow_source_dir}/tensorflow/core/platform/default/device_tracer.cc") - list(REMOVE_ITEM tf_core_platform_srcs ${tf_core_platform_srcs_exclude}) -endif() - -list(APPEND tf_core_lib_srcs ${tf_core_platform_srcs}) - -if(UNIX) - file(GLOB tf_core_platform_posix_srcs - "${tensorflow_source_dir}/tensorflow/core/platform/posix/*.h" - "${tensorflow_source_dir}/tensorflow/core/platform/posix/*.cc" - ) - list(APPEND tf_core_lib_srcs ${tf_core_platform_posix_srcs}) -endif(UNIX) - -if(WIN32) - file(GLOB tf_core_platform_windows_srcs - "${tensorflow_source_dir}/tensorflow/core/platform/windows/*.h" - "${tensorflow_source_dir}/tensorflow/core/platform/windows/*.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/posix/error.h" - "${tensorflow_source_dir}/tensorflow/core/platform/posix/error.cc" - ) - list(APPEND tf_core_lib_srcs ${tf_core_platform_windows_srcs}) -endif(WIN32) - -if (tensorflow_ENABLE_HDFS_SUPPORT) - list(APPEND tf_core_platform_hdfs_srcs - "${tensorflow_source_dir}/tensorflow/core/platform/hadoop/hadoop_file_system.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/hadoop/hadoop_file_system.h" - ) - list(APPEND tf_core_lib_srcs ${tf_core_platform_hdfs_srcs}) -endif() - -file(GLOB_RECURSE tf_core_lib_test_srcs - "${tensorflow_source_dir}/tensorflow/core/lib/*test*.h" - "${tensorflow_source_dir}/tensorflow/core/lib/*test*.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/*test*.h" - "${tensorflow_source_dir}/tensorflow/core/platform/*test*.cc" - "${tensorflow_source_dir}/tensorflow/core/public/*test*.h" -) -list(REMOVE_ITEM tf_core_lib_srcs ${tf_core_lib_test_srcs}) - -add_library(tf_core_lib OBJECT ${tf_core_lib_srcs}) -add_dependencies(tf_core_lib ${tensorflow_EXTERNAL_DEPENDENCIES} tf_protos_cc) - -# Tricky setup to force always rebuilding -# force_rebuild always runs forcing ${VERSION_INFO_CC} target to run -# ${VERSION_INFO_CC} would cache, but it depends on a phony never produced -# target. -# This code forces rebuild every time, not needed as version from git is fetched only once -# move to make.bat which mimicks make.sh - -if (NOT WIN32) - - set(VERSION_INFO_CC ${tensorflow_source_dir}/tensorflow/core/util/version_info.cc) - add_custom_target(force_rebuild_target ALL DEPENDS ${VERSION_INFO_CC}) - add_custom_command(OUTPUT __force_rebuild COMMAND ${CMAKE_COMMAND} -E echo) - add_custom_command(OUTPUT - ${VERSION_INFO_CC} - COMMAND ${PYTHON_EXECUTABLE} ${tensorflow_source_dir}/tensorflow/tools/git/gen_git_source.py - ARGS --raw_generate ${VERSION_INFO_CC} --source_dir ${tensorflow_source_dir} --git_tag_override=${GIT_TAG_OVERRIDE} - DEPENDS __force_rebuild) - set(tf_version_srcs ${tensorflow_source_dir}/tensorflow/core/util/version_info.cc) -endif() - -######################################################## -# tf_core_framework library -######################################################## -file(GLOB_RECURSE tf_core_framework_srcs - "${tensorflow_source_dir}/tensorflow/core/framework/*.h" - "${tensorflow_source_dir}/tensorflow/core/framework/*.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/edgeset.h" - "${tensorflow_source_dir}/tensorflow/core/graph/edgeset.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/graph.h" - "${tensorflow_source_dir}/tensorflow/core/graph/graph.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/graph_def_builder.h" - "${tensorflow_source_dir}/tensorflow/core/graph/graph_def_builder.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/node_builder.h" - "${tensorflow_source_dir}/tensorflow/core/graph/node_builder.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/tensor_id.h" - "${tensorflow_source_dir}/tensorflow/core/graph/tensor_id.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/while_context.h" - "${tensorflow_source_dir}/tensorflow/core/graph/while_context.cc" - "${tensorflow_source_dir}/tensorflow/core/util/*.h" - "${tensorflow_source_dir}/tensorflow/core/util/*.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/session.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/session_factory.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/session_options.cc" - "${tensorflow_source_dir}/tensorflow/core/summary/*.cc" - "${tensorflow_source_dir}/tensorflow/core/summary/*.h" - "${tensorflow_source_dir}/public/*.h" -) - -file(GLOB_RECURSE tf_core_framework_exclude_srcs - "${tensorflow_source_dir}/tensorflow/core/framework/*test*.h" - "${tensorflow_source_dir}/tensorflow/core/framework/*test*.cc" - "${tensorflow_source_dir}/tensorflow/core/framework/*testutil.h" - "${tensorflow_source_dir}/tensorflow/core/framework/*testutil.cc" - "${tensorflow_source_dir}/tensorflow/core/framework/*main.cc" - "${tensorflow_source_dir}/tensorflow/core/framework/resource_handle.cc" - "${tensorflow_source_dir}/tensorflow/core/util/*test*.h" - "${tensorflow_source_dir}/tensorflow/core/util/*test*.cc" - "${tensorflow_source_dir}/tensorflow/core/util/*main.cc" - "${tensorflow_source_dir}/tensorflow/core/summary/*test*.cc" - "${tensorflow_source_dir}/tensorflow/core/summary/loader.cc" - "${tensorflow_source_dir}/tensorflow/core/summary/vacuum.cc" -) - -# TODO(jart): Why doesn't this work? -# set_source_files_properties( -# ${tensorflow_source_dir}/tensorflow/core/lib/db/snapfn.cc -# PROPERTIES COMPILE_FLAGS -DSQLITE_OMIT_LOAD_EXTENSION) - -list(REMOVE_ITEM tf_core_framework_srcs ${tf_core_framework_exclude_srcs}) - -add_library(tf_core_framework OBJECT - ${tf_core_framework_srcs} - ${tf_version_srcs} - ${PROTO_TEXT_HDRS} - ${PROTO_TEXT_SRCS}) -add_dependencies(tf_core_framework - tf_core_lib - proto_text -) diff --git a/tensorflow/contrib/cmake/tf_core_kernels.cmake b/tensorflow/contrib/cmake/tf_core_kernels.cmake deleted file mode 100644 index e8972098c7e..00000000000 --- a/tensorflow/contrib/cmake/tf_core_kernels.cmake +++ /dev/null @@ -1,220 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -######################################################## -# tf_core_kernels library -######################################################## - -if(tensorflow_BUILD_ALL_KERNELS) - file(GLOB_RECURSE tf_core_kernels_srcs - "${tensorflow_source_dir}/tensorflow/core/kernels/*.h" - "${tensorflow_source_dir}/tensorflow/core/kernels/*.cc" - ) -else(tensorflow_BUILD_ALL_KERNELS) - # Build a minimal subset of kernels to be able to run a test program. - set(tf_core_kernels_srcs - "${tensorflow_source_dir}/tensorflow/core/kernels/bounds_check.h" - "${tensorflow_source_dir}/tensorflow/core/kernels/constant_op.h" - "${tensorflow_source_dir}/tensorflow/core/kernels/constant_op.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/fill_functor.h" - "${tensorflow_source_dir}/tensorflow/core/kernels/fill_functor.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/matmul_op.h" - "${tensorflow_source_dir}/tensorflow/core/kernels/matmul_op.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/no_op.h" - "${tensorflow_source_dir}/tensorflow/core/kernels/no_op.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/ops_util.h" - "${tensorflow_source_dir}/tensorflow/core/kernels/ops_util.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/sendrecv_ops.h" - "${tensorflow_source_dir}/tensorflow/core/kernels/sendrecv_ops.cc" - ) -endif(tensorflow_BUILD_ALL_KERNELS) - -if(tensorflow_BUILD_CONTRIB_KERNELS) - set(tf_contrib_kernels_srcs - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/kernels/model_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/kernels/prediction_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/kernels/split_handler_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/kernels/stats_accumulator_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/kernels/training_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/lib/utils/parallel_for.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/lib/learner/common/partitioners/example_partitioner.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/lib/models/multiple_additive_trees.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/model_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/prediction_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/quantile_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/split_handler_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/stats_accumulator_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/training_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/clustering_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/wals_solver_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/factorization/ops/clustering_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/factorization/ops/factorization_ops.cc" - #"${tensorflow_source_dir}/tensorflow/contrib/ffmpeg/decode_audio_op.cc" - #"${tensorflow_source_dir}/tensorflow/contrib/ffmpeg/encode_audio_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/framework/kernels/zero_initializer_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/framework/ops/variable_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/image/kernels/bipartite_match_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/image/kernels/image_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/image/kernels/segmentation_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/image/kernels/single_image_random_dot_stereograms_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/image/ops/distort_image_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/image/ops/image_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/image/ops/single_image_random_dot_stereograms_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/layers/kernels/sparse_feature_cross_kernel.cc" - "${tensorflow_source_dir}/tensorflow/contrib/layers/ops/sparse_feature_cross_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/libsvm/kernels/decode_libsvm_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/libsvm/ops/libsvm_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes.cc" - "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/ops/nearest_neighbor_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/resampler/kernels/resampler_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/resampler/ops/resampler_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/rnn/kernels/blas_gemm.cc" - "${tensorflow_source_dir}/tensorflow/contrib/rnn/kernels/gru_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/rnn/kernels/lstm_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/rnn/ops/gru_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/rnn/ops/lstm_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/ops/tensor_forest_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/kernels/reinterpret_string_to_float_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/kernels/scatter_add_ndim_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/kernels/tree_utils.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/hybrid/core/ops/hard_routing_function_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/hybrid/core/ops/k_feature_gradient_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/hybrid/core/ops/k_feature_routing_function_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/hybrid/core/ops/routing_function_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/hybrid/core/ops/routing_gradient_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_function_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_gradient_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/hybrid/core/ops/unpack_path_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.cc" - "${tensorflow_source_dir}/tensorflow/contrib/text/kernels/skip_gram_kernels.cc" - "${tensorflow_source_dir}/tensorflow/contrib/text/ops/skip_gram_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tpu/ops/cross_replica_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tpu/ops/infeed_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tpu/ops/outfeed_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tpu/ops/replication_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tpu/ops/tpu_configuration_ops.cc" - ) - list(APPEND tf_core_kernels_srcs ${tf_contrib_kernels_srcs}) -endif(tensorflow_BUILD_CONTRIB_KERNELS) - -# Cloud libraries require curl and boringssl. -# Curl is not supported yet anyway so we remove for now. -file(GLOB tf_core_kernels_cloud_srcs - "${tensorflow_source_dir}/tensorflow/contrib/cloud/kernels/*.h" - "${tensorflow_source_dir}/tensorflow/contrib/cloud/kernels/*.cc" -) -list(REMOVE_ITEM tf_core_kernels_srcs ${tf_core_kernels_cloud_srcs}) - -file(GLOB_RECURSE tf_core_kernels_exclude_srcs - "${tensorflow_source_dir}/tensorflow/core/kernels/*test*.h" - "${tensorflow_source_dir}/tensorflow/core/kernels/*test*.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/*testutil.h" - "${tensorflow_source_dir}/tensorflow/core/kernels/*testutil.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/*test_utils.h" - "${tensorflow_source_dir}/tensorflow/core/kernels/*test_utils.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/*main.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/*.cu.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/fuzzing/*" - "${tensorflow_source_dir}/tensorflow/core/kernels/hexagon/*" - "${tensorflow_source_dir}/tensorflow/core/kernels/remote_fused_graph_rewriter_transform*.cc" -) -list(REMOVE_ITEM tf_core_kernels_srcs ${tf_core_kernels_exclude_srcs}) - -if(WIN32) - file(GLOB_RECURSE tf_core_kernels_windows_exclude_srcs - # not working on windows yet - "${tensorflow_source_dir}/tensorflow/core/kernels/neon/*" - # not in core - those are loaded dynamically as dll - "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes.cc" - "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/ops/nearest_neighbor_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/resampler/kernels/resampler_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/rnn/kernels/blas_gemm.cc" - "${tensorflow_source_dir}/tensorflow/contrib/rnn/kernels/gru_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/rnn/kernels/lstm_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/rnn/ops/gru_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/rnn/ops/lstm_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc" - # temporarily disable nccl (nccl itself needs to be ported to windows first) - "${tensorflow_source_dir}/tensorflow/contrib/nccl/kernels/nccl_manager.cc" - "${tensorflow_source_dir}/tensorflow/contrib/nccl/kernels/nccl_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/nccl/ops/nccl_ops.cc" - ) - list(REMOVE_ITEM tf_core_kernels_srcs ${tf_core_kernels_windows_exclude_srcs}) -else(WIN32) - if(tensorflow_ENABLE_GPU) - file(GLOB_RECURSE tf_core_kernels_gpu_exclude_srcs - # temporarily disable nccl as it needs to be ported with gpu - "${tensorflow_source_dir}/tensorflow/contrib/nccl/kernels/nccl_manager.cc" - "${tensorflow_source_dir}/tensorflow/contrib/nccl/kernels/nccl_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/nccl/ops/nccl_ops.cc" - ) - list(REMOVE_ITEM tf_core_kernels_srcs ${tf_core_kernels_gpu_exclude_srcs}) - endif(tensorflow_ENABLE_GPU) -endif(WIN32) - -file(GLOB_RECURSE tf_core_gpu_kernels_srcs - "${tensorflow_source_dir}/tensorflow/core/kernels/*.cu.cc" - "${tensorflow_source_dir}/tensorflow/contrib/framework/kernels/zero_initializer_op_gpu.cu.cc" - "${tensorflow_source_dir}/tensorflow/contrib/image/kernels/*.cu.cc" - "${tensorflow_source_dir}/tensorflow/contrib/rnn/kernels/*.cu.cc" - "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/kernels/*.cu.cc" - "${tensorflow_source_dir}/tensorflow/contrib/resampler/kernels/*.cu.cc" -) - -if(WIN32 AND tensorflow_ENABLE_GPU) - file(GLOB_RECURSE tf_core_kernels_cpu_only_srcs - # GPU implementation not working on Windows yet. - "${tensorflow_source_dir}/tensorflow/core/kernels/matrix_diag_op.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/one_hot_op.cc") - list(REMOVE_ITEM tf_core_kernels_srcs ${tf_core_kernels_cpu_only_srcs}) - add_library(tf_core_kernels_cpu_only OBJECT ${tf_core_kernels_cpu_only_srcs}) - add_dependencies(tf_core_kernels_cpu_only tf_core_cpu) - # Undefine GOOGLE_CUDA to avoid registering unsupported GPU kernel symbols. - get_target_property(target_compile_flags tf_core_kernels_cpu_only COMPILE_FLAGS) - if(target_compile_flags STREQUAL "target_compile_flags-NOTFOUND") - set(target_compile_flags "/UGOOGLE_CUDA") - else() - set(target_compile_flags "${target_compile_flags} /UGOOGLE_CUDA") - endif() - set_target_properties(tf_core_kernels_cpu_only PROPERTIES COMPILE_FLAGS ${target_compile_flags}) -endif(WIN32 AND tensorflow_ENABLE_GPU) - -add_library(tf_core_kernels OBJECT ${tf_core_kernels_srcs}) -add_dependencies(tf_core_kernels tf_core_cpu) - -if (WIN32) - target_compile_options(tf_core_kernels PRIVATE /MP) -endif (WIN32) -if (tensorflow_ENABLE_GPU) - set_source_files_properties(${tf_core_gpu_kernels_srcs} PROPERTIES CUDA_SOURCE_PROPERTY_FORMAT OBJ) - set(tf_core_gpu_kernels_lib tf_core_gpu_kernels) - cuda_add_library(${tf_core_gpu_kernels_lib} ${tf_core_gpu_kernels_srcs}) - set_target_properties(${tf_core_gpu_kernels_lib} - PROPERTIES DEBUG_POSTFIX "" - COMPILE_FLAGS "${TF_REGULAR_CXX_FLAGS}" - ) - add_dependencies(${tf_core_gpu_kernels_lib} tf_core_cpu) -endif() diff --git a/tensorflow/contrib/cmake/tf_core_ops.cmake b/tensorflow/contrib/cmake/tf_core_ops.cmake deleted file mode 100644 index f73f89ce379..00000000000 --- a/tensorflow/contrib/cmake/tf_core_ops.cmake +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -set(tf_op_lib_names - "array_ops" - "audio_ops" - "batch_ops" - "bitwise_ops" - "boosted_trees_ops" - "candidate_sampling_ops" - "checkpoint_ops" - "collective_ops" - "control_flow_ops" - "ctc_ops" - "cudnn_rnn_ops" - "data_flow_ops" - "dataset_ops" - "decode_proto_ops" - "encode_proto_ops" - "function_ops" - "functional_ops" - "image_ops" - "io_ops" - "linalg_ops" - "list_ops" - "logging_ops" - "lookup_ops" - "manip_ops" - "math_ops" - "nn_ops" - "no_op" - "parsing_ops" - "random_ops" - "remote_fused_graph_ops" - "resource_variable_ops" - "rpc_ops" - "scoped_allocator_ops" - "script_ops" - "sdca_ops" - "sendrecv_ops" - "set_ops" - "sparse_ops" - "spectral_ops" - "state_ops" - "stateless_random_ops" - "string_ops" - "summary_ops" - "training_ops" - "word2vec_ops" -) - -foreach(tf_op_lib_name ${tf_op_lib_names}) - ######################################################## - # tf_${tf_op_lib_name} library - ######################################################## - file(GLOB tf_${tf_op_lib_name}_srcs - "${tensorflow_source_dir}/tensorflow/core/ops/${tf_op_lib_name}.cc" - ) - - add_library(tf_${tf_op_lib_name} OBJECT ${tf_${tf_op_lib_name}_srcs}) - - add_dependencies(tf_${tf_op_lib_name} tf_core_framework) -endforeach() - -function(GENERATE_CONTRIB_OP_LIBRARY op_lib_name cc_srcs) - add_library(tf_contrib_${op_lib_name}_ops OBJECT ${cc_srcs}) - add_dependencies(tf_contrib_${op_lib_name}_ops tf_core_framework) -endfunction() - -file(GLOB_RECURSE tensor_forest_hybrid_srcs - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/hybrid/core/ops/*.cc" -) - -file(GLOB_RECURSE tpu_ops_srcs - "${tensorflow_source_dir}/tensorflow/contrib/tpu/ops/*.cc" -) - -GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_model "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/model_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_split_handler "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/split_handler_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_training "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/training_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_prediction "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/prediction_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_quantiles "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/quantile_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_stats_accumulator "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/stats_accumulator_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(factorization_clustering "${tensorflow_source_dir}/tensorflow/contrib/factorization/ops/clustering_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(factorization_factorization "${tensorflow_source_dir}/tensorflow/contrib/factorization/ops/factorization_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(framework_variable "${tensorflow_source_dir}/tensorflow/contrib/framework/ops/variable_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(input_pipeline "${tensorflow_source_dir}/tensorflow/contrib/input_pipeline/ops/input_pipeline_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(image "${tensorflow_source_dir}/tensorflow/contrib/image/ops/image_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(image_distort_image "${tensorflow_source_dir}/tensorflow/contrib/image/ops/distort_image_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(image_sirds "${tensorflow_source_dir}/tensorflow/contrib/image/ops/single_image_random_dot_stereograms_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(layers_sparse_feature_cross "${tensorflow_source_dir}/tensorflow/contrib/layers/ops/sparse_feature_cross_op.cc") -GENERATE_CONTRIB_OP_LIBRARY(memory_stats "${tensorflow_source_dir}/tensorflow/contrib/memory_stats/ops/memory_stats_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(periodic_resample "${tensorflow_source_dir}/tensorflow/contrib/periodic_resample/ops/array_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(nearest_neighbor "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/ops/nearest_neighbor_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(resampler "${tensorflow_source_dir}/tensorflow/contrib/resampler/ops/resampler_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(rnn_gru "${tensorflow_source_dir}/tensorflow/contrib/rnn/ops/gru_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(rnn_lstm "${tensorflow_source_dir}/tensorflow/contrib/rnn/ops/lstm_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(seq2seq_beam_search "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(tensor_forest "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/ops/tensor_forest_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(tensor_forest_hybrid "${tensor_forest_hybrid_srcs}") -GENERATE_CONTRIB_OP_LIBRARY(tensor_forest_model "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/ops/model_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(tensor_forest_stats "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/ops/stats_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(text_skip_gram "${tensorflow_source_dir}/tensorflow/contrib/text/ops/skip_gram_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(tpu "${tpu_ops_srcs}") -GENERATE_CONTRIB_OP_LIBRARY(bigquery_reader "${tensorflow_source_dir}/tensorflow/contrib/cloud/ops/bigquery_reader_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(gcs_config "${tensorflow_source_dir}/tensorflow/contrib/cloud/ops/gcs_config_ops.cc") -GENERATE_CONTRIB_OP_LIBRARY(reduce_slice_ops "${tensorflow_source_dir}/tensorflow/contrib/reduce_slice_ops/ops/reduce_slice_ops.cc") - -######################################################## -# tf_user_ops library -######################################################## -file(GLOB_RECURSE tf_user_ops_srcs - "${tensorflow_source_dir}/tensorflow/core/user_ops/*.cc" -) - -add_library(tf_user_ops OBJECT ${tf_user_ops_srcs}) - -add_dependencies(tf_user_ops tf_core_framework) - -######################################################## -# tf_core_ops library -######################################################## -file(GLOB_RECURSE tf_core_ops_srcs - "${tensorflow_source_dir}/tensorflow/core/ops/*.h" - "${tensorflow_source_dir}/tensorflow/core/ops/*.cc" - "${tensorflow_source_dir}/tensorflow/core/user_ops/*.h" - "${tensorflow_source_dir}/tensorflow/core/user_ops/*.cc" -) - -file(GLOB_RECURSE tf_core_ops_exclude_srcs - "${tensorflow_source_dir}/tensorflow/core/ops/*test*.h" - "${tensorflow_source_dir}/tensorflow/core/ops/*test*.cc" - "${tensorflow_source_dir}/tensorflow/core/ops/*main.cc" - "${tensorflow_source_dir}/tensorflow/core/user_ops/*test*.h" - "${tensorflow_source_dir}/tensorflow/core/user_ops/*test*.cc" - "${tensorflow_source_dir}/tensorflow/core/user_ops/*main.cc" - "${tensorflow_source_dir}/tensorflow/core/user_ops/*.cu.cc" -) - -list(REMOVE_ITEM tf_core_ops_srcs ${tf_core_ops_exclude_srcs}) - -add_library(tf_core_ops OBJECT ${tf_core_ops_srcs}) - -add_dependencies(tf_core_ops tf_core_cpu) - -######################################################## -# tf_debug_ops library -######################################################## - -file(GLOB tf_debug_ops_srcs - "${tensorflow_source_dir}/tensorflow/core/ops/debug_ops.cc" -) - -add_library(tf_debug_ops OBJECT ${tf_debug_ops_srcs}) - -add_dependencies(tf_debug_ops tf_core_framework) diff --git a/tensorflow/contrib/cmake/tf_core_profiler.cmake b/tensorflow/contrib/cmake/tf_core_profiler.cmake deleted file mode 100644 index b91a7f43e5c..00000000000 --- a/tensorflow/contrib/cmake/tf_core_profiler.cmake +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -######################################################## -# tf_core_profiler library -######################################################## -file(GLOB_RECURSE tf_core_profiler_srcs - "${tensorflow_source_dir}/tensorflow/core/profiler/*.proto" - "${tensorflow_source_dir}/tensorflow/core/profiler/tfprof_options.h" - "${tensorflow_source_dir}/tensorflow/core/profiler/tfprof_options.cc" - "${tensorflow_source_dir}/tensorflow/core/profiler/internal/*.h" - "${tensorflow_source_dir}/tensorflow/core/profiler/internal/*.cc" - "${tensorflow_source_dir}/tensorflow/core/profiler/internal/advisor/*.h" - "${tensorflow_source_dir}/tensorflow/core/profiler/internal/advisor/*.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/regexp.h" -) - -file(GLOB_RECURSE tf_core_profiler_exclude_srcs - "${tensorflow_source_dir}/tensorflow/core/profiler/internal/*test.cc" - "${tensorflow_source_dir}/tensorflow/core/profiler/internal/advisor/*test.cc" - "${tensorflow_source_dir}/tensorflow/core/profiler/internal/print_model_analysis.cc" - "${tensorflow_source_dir}/tensorflow/core/profiler/internal/print_model_analysis.h" -) -list(REMOVE_ITEM tf_core_profiler_srcs ${tf_core_profiler_exclude_srcs}) - -add_library(tf_core_profiler OBJECT ${tf_core_profiler_srcs}) -add_dependencies(tf_core_profiler tf_core_lib) \ No newline at end of file diff --git a/tensorflow/contrib/cmake/tf_grappler.cmake b/tensorflow/contrib/cmake/tf_grappler.cmake deleted file mode 100644 index 410490531a3..00000000000 --- a/tensorflow/contrib/cmake/tf_grappler.cmake +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -######################################################## -# tf_grappler library -######################################################## -file(GLOB tf_grappler_srcs - "${tensorflow_source_dir}/tensorflow/core/grappler/clusters/single_machine.cc" - "${tensorflow_source_dir}/tensorflow/core/grappler/clusters/single_machine.h" - "${tensorflow_source_dir}/tensorflow/python/grappler/cost_analyzer.cc" - "${tensorflow_source_dir}/tensorflow/python/grappler/cost_analyzer.h" - "${tensorflow_source_dir}/tensorflow/python/grappler/model_analyzer.cc" - "${tensorflow_source_dir}/tensorflow/python/grappler/model_analyzer.h" - ) - -add_library(tf_grappler OBJECT ${tf_grappler_srcs}) - -add_dependencies(tf_grappler tf_core_cpu) \ No newline at end of file diff --git a/tensorflow/contrib/cmake/tf_label_image_example.cmake b/tensorflow/contrib/cmake/tf_label_image_example.cmake deleted file mode 100644 index 7f2f60b0897..00000000000 --- a/tensorflow/contrib/cmake/tf_label_image_example.cmake +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -set(tf_label_image_example_srcs - "${tensorflow_source_dir}/tensorflow/examples/label_image/main.cc" -) - -add_executable(tf_label_image_example - ${tf_label_image_example_srcs} - $ - $ - $ - $ - $ - $ - $ - $ - $<$:$> -) - -target_link_libraries(tf_label_image_example PUBLIC - tf_protos_cc - ${tf_core_gpu_kernels_lib} - ${tensorflow_EXTERNAL_LIBRARIES} -) - -install(TARGETS tf_label_image_example - RUNTIME DESTINATION bin - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib) \ No newline at end of file diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake deleted file mode 100755 index 40d17372bf1..00000000000 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ /dev/null @@ -1,961 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# CMake rules for generating the TensorFlow Python bindings. -# -# Known limitations: -# * Generates output in a hard-coded path ${CMAKE_CURRENT_BINARY_DIR}/tf_python. -# * No support for dynamic library loading. -# * Limited support for tf.contrib. -# -# The _pywrap_tensorflow_internal target builds everything. - -######################################################## -# Resolve installed dependencies -######################################################## - -# 1. Resolve the installed version of Python (for Python.h and python). -# TODO(mrry): Parameterize the build script to enable Python 3 building. -if(NOT PYTHON_INCLUDE_DIR) - set(PYTHON_NOT_FOUND false) - exec_program("${PYTHON_EXECUTABLE}" - ARGS "-c \"import distutils.sysconfig; print(distutils.sysconfig.get_python_inc())\"" - OUTPUT_VARIABLE PYTHON_INCLUDE_DIR - RETURN_VALUE PYTHON_NOT_FOUND) - if(${PYTHON_NOT_FOUND}) - message(FATAL_ERROR - "Cannot get Python include directory. Is distutils installed?") - endif(${PYTHON_NOT_FOUND}) -endif(NOT PYTHON_INCLUDE_DIR) -FIND_PACKAGE(PythonLibs) - -# 2. Resolve the installed version of NumPy (for numpy/arrayobject.h). -if(NOT NUMPY_INCLUDE_DIR) - set(NUMPY_NOT_FOUND false) - exec_program("${PYTHON_EXECUTABLE}" - ARGS "-c \"import numpy; print(numpy.get_include())\"" - OUTPUT_VARIABLE NUMPY_INCLUDE_DIR - RETURN_VALUE NUMPY_NOT_FOUND) - if(${NUMPY_NOT_FOUND}) - message(FATAL_ERROR - "Cannot get NumPy include directory: Is NumPy installed?") - endif(${NUMPY_NOT_FOUND}) -endif(NOT NUMPY_INCLUDE_DIR) - - -######################################################## -# Build the Python directory structure. -######################################################## - -# TODO(mrry): Configure this to build in a directory other than tf_python/ - -# Generates the Python protobuf wrappers. -# ROOT_DIR must be absolute; subsequent arguments are interpreted as -# paths of .proto files, and must be relative to ROOT_DIR. -function(RELATIVE_PROTOBUF_GENERATE_PYTHON ROOT_DIR SRCS) - if(NOT ARGN) - message(SEND_ERROR "Error: RELATIVE_PROTOBUF_GENERATE_PYTHON() called without any proto files") - return() - endif() - - set(${SRCS}) - foreach(FIL ${ARGN}) - set(ABS_FIL ${ROOT_DIR}/${FIL}) - get_filename_component(FIL_WE ${FIL} NAME_WE) - get_filename_component(FIL_DIR ${ABS_FIL} PATH) - file(RELATIVE_PATH REL_DIR ${ROOT_DIR} ${FIL_DIR}) - - list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/tf_python/${REL_DIR}/${FIL_WE}_pb2.py") - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/tf_python/${REL_DIR}/${FIL_WE}_pb2.py" - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} - ARGS --python_out ${CMAKE_CURRENT_BINARY_DIR}/tf_python/ -I ${ROOT_DIR} -I ${PROTOBUF_INCLUDE_DIRS} ${ABS_FIL} - DEPENDS ${PROTOBUF_PROTOC_EXECUTABLE} protobuf - COMMENT "Running Python protocol buffer compiler on ${FIL}" - VERBATIM ) - endforeach() - set(${SRCS} ${${SRCS}} PARENT_SCOPE) -endfunction() - -function(RELATIVE_PROTOBUF_GENERATE_CPP SRCS HDRS ROOT_DIR) - if(NOT ARGN) - message(SEND_ERROR "Error: RELATIVE_PROTOBUF_GENERATE_CPP() called without any proto files") - return() - endif() - - set(${SRCS}) - set(${HDRS}) - foreach(FIL ${ARGN}) - set(ABS_FIL ${ROOT_DIR}/${FIL}) - get_filename_component(FIL_WE ${FIL} NAME_WE) - get_filename_component(FIL_DIR ${ABS_FIL} PATH) - file(RELATIVE_PATH REL_DIR ${ROOT_DIR} ${FIL_DIR}) - - list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.cc") - list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.h") - - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.cc" - "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb.h" - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} - ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} -I ${ROOT_DIR} ${ABS_FIL} -I ${PROTOBUF_INCLUDE_DIRS} - DEPENDS ${ABS_FIL} protobuf - COMMENT "Running C++ protocol buffer compiler on ${FIL}" - VERBATIM ) - endforeach() - - set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) - set(${SRCS} ${${SRCS}} PARENT_SCOPE) - set(${HDRS} ${${HDRS}} PARENT_SCOPE) -endfunction() - -FILE(READ python_protos.txt python_protos) -# Convert file contents into a CMake list (where each element in the list is one line of the file) -STRING(REGEX REPLACE ";" "\\\\;" python_protos "${python_protos}") -STRING(REGEX REPLACE "\n" ";" python_protos "${python_protos}") - -foreach(python_proto ${python_protos}) - if(NOT python_proto MATCHES "^\#") - STRING(REGEX REPLACE " *\#.*" "" python_proto "${python_proto}") - if(NOT EXISTS "${tensorflow_source_dir}/${python_proto}") - message(SEND_ERROR "Python proto directory not found: ${python_proto}") - endif() - file(GLOB_RECURSE tf_python_protos_src RELATIVE ${tensorflow_source_dir} - "${tensorflow_source_dir}/${python_proto}/*.proto" - ) - list(APPEND tf_python_protos_srcs ${tf_python_protos_src}) - endif() -endforeach(python_proto) - -RELATIVE_PROTOBUF_GENERATE_PYTHON( - ${tensorflow_source_dir} PYTHON_PROTO_GENFILES ${tf_python_protos_srcs} -) - -FILE(READ python_protos_cc.txt python_protos_cc) -# Convert file contents into a CMake list (where each element in the list is one line of the file) -STRING(REGEX REPLACE ";" "\\\\;" python_protos_cc "${python_protos_cc}") -STRING(REGEX REPLACE "\n" ";" python_protos_cc "${python_protos_cc}") - -foreach(python_proto_cc ${python_protos_cc}) - if(NOT python_proto_cc MATCHES "^\#") - STRING(REGEX REPLACE " *\#.*" "" python_proto_cc "${python_proto_cc}") - if(NOT EXISTS "${tensorflow_source_dir}/${python_proto_cc}") - message(SEND_ERROR "Python proto CC directory not found: ${python_proto_cc}") - endif() - file(GLOB_RECURSE tf_python_protos_cc_src RELATIVE ${tensorflow_source_dir} - "${tensorflow_source_dir}/${python_proto_cc}/*.proto" - ) - list(APPEND tf_python_protos_cc_srcs ${tf_python_protos_cc_src}) - endif() -endforeach(python_proto_cc) - -RELATIVE_PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS - ${tensorflow_source_dir} ${tf_python_protos_cc_srcs} -) - -add_library(tf_python_protos_cc ${PROTO_SRCS} ${PROTO_HDRS}) -add_dependencies(tf_python_protos_cc tf_protos_cc) - -# tf_python_touchup_modules adds empty __init__.py files to all -# directories containing Python code, so that Python will recognize -# them as modules. -add_custom_target(tf_python_touchup_modules) - -# tf_python_copy_scripts_to_destination copies all Python files -# (including static source and generated protobuf wrappers, but *not* -# generated TensorFlow op wrappers) into tf_python/. -add_custom_target(tf_python_copy_scripts_to_destination DEPENDS tf_python_touchup_modules) - - -# tf_python_srcs contains all static .py files -function(add_python_module MODULE_NAME) - set(options DONTCOPY) - cmake_parse_arguments(ADD_PYTHON_MODULE "${options}" "" "" ${ARGN}) - add_custom_command(TARGET tf_python_touchup_modules PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/tf_python/${MODULE_NAME}") - add_custom_command(TARGET tf_python_touchup_modules PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E touch "${CMAKE_CURRENT_BINARY_DIR}/tf_python/${MODULE_NAME}/__init__.py") - file(GLOB module_python_srcs RELATIVE ${tensorflow_source_dir} - "${tensorflow_source_dir}/${MODULE_NAME}/*.py" - ) - if(NOT ${ADD_PYTHON_MODULE_DONTCOPY}) - foreach(script ${module_python_srcs}) - get_filename_component(REL_DIR ${script} DIRECTORY) - # NOTE(mrry): This rule may exclude modules that should be part of - # the distributed PIP package - # (e.g. tensorflow/contrib/testing/python/framework/util_test.py), - # so we currently add explicit commands to include those files - # later on in this script. - if (NOT "${script}" MATCHES "_test\.py$") - add_custom_command(TARGET tf_python_copy_scripts_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/${script} ${CMAKE_CURRENT_BINARY_DIR}/tf_python/${script}) - endif() - endforeach() - endif() -endfunction() - -FILE(READ python_modules.txt python_modules) -# Convert file contents into a CMake list (where each element in the list is one line of the file) -STRING(REGEX REPLACE ";" "\\\\;" python_modules "${python_modules}") -STRING(REGEX REPLACE "\n" ";" python_modules "${python_modules}") - -foreach(python_module ${python_modules}) - if(NOT python_module MATCHES "^\#") - STRING(REGEX REPLACE " *\#.*" "" python_module "${python_module}") - if(NOT EXISTS "${tensorflow_source_dir}/${python_module}") - message(SEND_ERROR "Python module not found: ${python_module}") - endif() - add_python_module(${python_module}) - endif() -endforeach(python_module) - -add_custom_command(TARGET tf_python_touchup_modules PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/lite") -add_custom_command(TARGET tf_python_touchup_modules PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/lite/python") -add_custom_command(TARGET tf_python_touchup_modules PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E touch - "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/lite/python/__init__.py") -add_custom_command( - TARGET tf_python_copy_scripts_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E touch - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/lite/python/lite.py) - -# Generate the tensorflow.python.platform.build_info module. -set(BUILD_INFO_PY "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/platform/build_info.py") -add_custom_command(TARGET tf_python_copy_scripts_to_destination PRE_BUILD - COMMAND ${PYTHON_EXECUTABLE} ${tensorflow_source_dir}/tensorflow/tools/build_info/gen_build_info.py --raw_generate ${BUILD_INFO_PY} ${tensorflow_BUILD_INFO_FLAGS}) - - -######################################################## -# tf_python_op_gen_main library -######################################################## -set(tf_python_op_gen_main_srcs - "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen.cc" - "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen.h" - "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen_internal.cc" - "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen_internal.h" - "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen_main.cc" -) - -add_library(tf_python_op_gen_main OBJECT ${tf_python_op_gen_main_srcs}) - -add_dependencies(tf_python_op_gen_main tf_core_framework) - -# create directory for ops generated files -set(python_ops_target_dir ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/ops) - -set(tf_python_ops_generated_files) - -set(tf_python_op_lib_names - ${tf_op_lib_names} - "user_ops" -) - -function(GENERATE_PYTHON_OP_LIB tf_python_op_lib_name) - set(options SHAPE_FUNCTIONS_NOT_REQUIRED) - set(oneValueArgs DESTINATION) - set(multiValueArgs ADDITIONAL_LIBRARIES) - cmake_parse_arguments(GENERATE_PYTHON_OP_LIB - "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - if(NOT DEFINED GENERATE_PYTHON_OP_LIB_DESTINATION) - # Default destination is tf_python/tensorflow/python/ops/gen_<...>.py. - set(GENERATE_PYTHON_OP_LIB_DESTINATION - "${python_ops_target_dir}/gen_${tf_python_op_lib_name}.py") - endif() - if(GENERATE_PYTHON_OP_LIB_SHAPE_FUNCTIONS_NOT_REQUIRED) - set(require_shape_fn 0) - else() - set(require_shape_fn 1) - endif() - - get_filename_component(GENERATE_PYTHON_OP_LIB_MKDIRPATH ${GENERATE_PYTHON_OP_LIB_DESTINATION} PATH) - file(MAKE_DIRECTORY ${GENERATE_PYTHON_OP_LIB_MKDIRPATH}) - - # Create a C++ executable that links in the appropriate op - # registrations and generates Python wrapper code based on the - # registered ops. - add_executable(${tf_python_op_lib_name}_gen_python - $ - $ - $ - $ - ${GENERATE_PYTHON_OP_LIB_ADDITIONAL_LIBRARIES} - ) - target_link_libraries(${tf_python_op_lib_name}_gen_python PRIVATE - tf_protos_cc - tf_python_protos_cc - ${tensorflow_EXTERNAL_LIBRARIES} - ) - - # Use the generated C++ executable to create a Python file - # containing the wrappers. - add_custom_command( - OUTPUT ${GENERATE_PYTHON_OP_LIB_DESTINATION} - COMMAND ${tf_python_op_lib_name}_gen_python ${tensorflow_source_dir}/tensorflow/core/api_def/base_api,${tensorflow_source_dir}/tensorflow/core/api_def/python_api ${require_shape_fn} > ${GENERATE_PYTHON_OP_LIB_DESTINATION} - DEPENDS ${tf_python_op_lib_name}_gen_python - ) - - set(tf_python_ops_generated_files ${tf_python_ops_generated_files} - ${GENERATE_PYTHON_OP_LIB_DESTINATION} PARENT_SCOPE) -endfunction() - -GENERATE_PYTHON_OP_LIB("array_ops") -GENERATE_PYTHON_OP_LIB("audio_ops") -GENERATE_PYTHON_OP_LIB("batch_ops") -GENERATE_PYTHON_OP_LIB("bitwise_ops") -GENERATE_PYTHON_OP_LIB("boosted_trees_ops") -GENERATE_PYTHON_OP_LIB("candidate_sampling_ops") -GENERATE_PYTHON_OP_LIB("checkpoint_ops") -GENERATE_PYTHON_OP_LIB("collective_ops") -GENERATE_PYTHON_OP_LIB("control_flow_ops" - ADDITIONAL_LIBRARIES $) -GENERATE_PYTHON_OP_LIB("ctc_ops") -GENERATE_PYTHON_OP_LIB("cudnn_rnn_ops") -GENERATE_PYTHON_OP_LIB("data_flow_ops") -GENERATE_PYTHON_OP_LIB("dataset_ops") -GENERATE_PYTHON_OP_LIB("decode_proto_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/proto/python/ops/gen_decode_proto_op.py) -GENERATE_PYTHON_OP_LIB("encode_proto_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/proto/python/ops/gen_encode_proto_op.py) -GENERATE_PYTHON_OP_LIB("function_ops") -GENERATE_PYTHON_OP_LIB("functional_ops") -GENERATE_PYTHON_OP_LIB("image_ops") -GENERATE_PYTHON_OP_LIB("io_ops") -GENERATE_PYTHON_OP_LIB("linalg_ops") -GENERATE_PYTHON_OP_LIB("list_ops") -GENERATE_PYTHON_OP_LIB("logging_ops") -GENERATE_PYTHON_OP_LIB("lookup_ops") -GENERATE_PYTHON_OP_LIB("manip_ops") -GENERATE_PYTHON_OP_LIB("math_ops") -GENERATE_PYTHON_OP_LIB("nn_ops") -GENERATE_PYTHON_OP_LIB("no_op") -GENERATE_PYTHON_OP_LIB("parsing_ops") -GENERATE_PYTHON_OP_LIB("random_ops") -GENERATE_PYTHON_OP_LIB("remote_fused_graph_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/remote_fused_graph/pylib/python/ops/gen_remote_fused_graph_ops.py) -GENERATE_PYTHON_OP_LIB("resource_variable_ops") -GENERATE_PYTHON_OP_LIB("rpc_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/rpc/python/ops/gen_rpc_op.py) -GENERATE_PYTHON_OP_LIB("scoped_allocator_ops") -GENERATE_PYTHON_OP_LIB("script_ops") -GENERATE_PYTHON_OP_LIB("sdca_ops") -GENERATE_PYTHON_OP_LIB("sendrecv_ops") -GENERATE_PYTHON_OP_LIB("set_ops") -GENERATE_PYTHON_OP_LIB("sparse_ops") -GENERATE_PYTHON_OP_LIB("spectral_ops") -GENERATE_PYTHON_OP_LIB("state_ops") -GENERATE_PYTHON_OP_LIB("stateless_random_ops") -GENERATE_PYTHON_OP_LIB("string_ops") -GENERATE_PYTHON_OP_LIB("summary_ops") -GENERATE_PYTHON_OP_LIB("user_ops") -GENERATE_PYTHON_OP_LIB("training_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/training/gen_training_ops.py) -GENERATE_PYTHON_OP_LIB("word2vec_ops") - -GENERATE_PYTHON_OP_LIB("contrib_boosted_trees_model_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/boosted_trees/python/ops/gen_model_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_boosted_trees_split_handler_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/boosted_trees/python/ops/gen_split_handler_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_boosted_trees_training_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/boosted_trees/python/ops/gen_training_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_boosted_trees_prediction_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/boosted_trees/python/ops/gen_prediction_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_boosted_trees_quantiles_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/boosted_trees/python/ops/gen_quantile_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_boosted_trees_stats_accumulator_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/boosted_trees/python/ops/gen_stats_accumulator_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_factorization_clustering_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/factorization/python/ops/gen_clustering_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_factorization_factorization_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/factorization/python/ops/gen_factorization_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_framework_variable_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/framework/python/ops/gen_variable_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_input_pipeline_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/input_pipeline/ops/gen_input_pipeline_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_image_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/image/ops/gen_image_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_image_distort_image_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/image/ops/gen_distort_image_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_image_sirds_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/image/ops/gen_single_image_random_dot_stereograms_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_layers_sparse_feature_cross_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/layers/ops/gen_sparse_feature_cross_op.py) -GENERATE_PYTHON_OP_LIB("contrib_memory_stats_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/memory_stats/ops/gen_memory_stats_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_periodic_resample_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/periodic_resample/python/ops/gen_periodic_resample_op.py) -GENERATE_PYTHON_OP_LIB("contrib_nearest_neighbor_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/nearest_neighbor/ops/gen_nearest_neighbor_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_resampler_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/resampler/ops/gen_resampler_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_seq2seq_beam_search_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/seq2seq/ops/gen_beam_search_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_tensor_forest_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/tensor_forest/python/ops/gen_tensor_forest_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_tensor_forest_hybrid_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/tensor_forest/hybrid/ops/gen_training_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_tensor_forest_model_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/tensor_forest/python/ops/gen_model_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_tensor_forest_stats_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/tensor_forest/python/ops/gen_stats_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_text_skip_gram_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/text/python/ops/gen_skip_gram_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_bigquery_reader_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/cloud/python/ops/gen_bigquery_reader_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_gcs_config_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/cloud/python/ops/gen_gcs_config_ops.py) -GENERATE_PYTHON_OP_LIB("debug_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/debug/ops/gen_debug_ops.py) - -add_custom_target(tf_python_ops SOURCES ${tf_python_ops_generated_files} ${PYTHON_PROTO_GENFILES}) -add_dependencies(tf_python_ops tf_python_op_gen_main) - - -############################################################ -# Build the SWIG-wrapped library for the TensorFlow runtime. -############################################################ - -find_package(SWIG REQUIRED) -# Generate the C++ and Python source code for the SWIG wrapper. -# NOTE(mrry): We always regenerate the SWIG wrapper, which means that we must -# always re-link the Python extension, but we don't have to track the -# individual headers on which the SWIG wrapper depends. -add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/pywrap_tensorflow_internal.py" - "${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.cc" - DEPENDS tf_python_touchup_modules __force_rebuild - COMMAND ${SWIG_EXECUTABLE} - ARGS -python -c++ - -I${tensorflow_source_dir} - -I${CMAKE_CURRENT_BINARY_DIR} - -module pywrap_tensorflow_internal - -outdir ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python - -o ${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.cc - -globals '' - ${tensorflow_source_dir}/tensorflow/python/tensorflow.i - COMMENT "Running SWIG to generate Python wrappers" - VERBATIM ) - -add_library(tf_c_python_api OBJECT - "${tensorflow_source_dir}/tensorflow/c/python_api.cc" - "${tensorflow_source_dir}/tensorflow/c/python_api.h" -) -add_dependencies( - tf_c_python_api - tf_c - tf_core_lib - tf_core_framework - tf_protos_cc - tf_python_protos_cc) - -set (pywrap_tensorflow_internal_src - "${tensorflow_source_dir}/tensorflow/core/profiler/internal/print_model_analysis.h" - "${tensorflow_source_dir}/tensorflow/core/profiler/internal/print_model_analysis.cc" - "${tensorflow_source_dir}/tensorflow/python/eager/pywrap_tfe.h" - "${tensorflow_source_dir}/tensorflow/python/eager/pywrap_tensor.cc" - "${tensorflow_source_dir}/tensorflow/python/eager/pywrap_tfe_src.cc" - "${tensorflow_source_dir}/tensorflow/python/client/tf_session_helper.h" - "${tensorflow_source_dir}/tensorflow/python/client/tf_session_helper.cc" - "${tensorflow_source_dir}/tensorflow/python/framework/cpp_shape_inference.h" - "${tensorflow_source_dir}/tensorflow/python/framework/cpp_shape_inference.cc" - "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen.h" - "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen.cc" - "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen_internal.h" - "${tensorflow_source_dir}/tensorflow/python/framework/python_op_gen_internal.cc" - "${tensorflow_source_dir}/tensorflow/python/lib/core/bfloat16.h" - "${tensorflow_source_dir}/tensorflow/python/lib/core/bfloat16.cc" - "${tensorflow_source_dir}/tensorflow/python/lib/core/numpy.h" - "${tensorflow_source_dir}/tensorflow/python/lib/core/numpy.cc" - "${tensorflow_source_dir}/tensorflow/python/lib/core/ndarray_tensor.h" - "${tensorflow_source_dir}/tensorflow/python/lib/core/ndarray_tensor.cc" - "${tensorflow_source_dir}/tensorflow/python/lib/core/ndarray_tensor_bridge.h" - "${tensorflow_source_dir}/tensorflow/python/lib/core/ndarray_tensor_bridge.cc" - "${tensorflow_source_dir}/tensorflow/python/lib/core/py_func.h" - "${tensorflow_source_dir}/tensorflow/python/lib/core/py_func.cc" - "${tensorflow_source_dir}/tensorflow/python/lib/core/py_exception_registry.h" - "${tensorflow_source_dir}/tensorflow/python/lib/core/py_exception_registry.cc" - "${tensorflow_source_dir}/tensorflow/python/lib/core/py_seq_tensor.h" - "${tensorflow_source_dir}/tensorflow/python/lib/core/py_seq_tensor.cc" - "${tensorflow_source_dir}/tensorflow/python/lib/core/py_util.h" - "${tensorflow_source_dir}/tensorflow/python/lib/core/py_util.cc" - "${tensorflow_source_dir}/tensorflow/python/lib/core/safe_ptr.h" - "${tensorflow_source_dir}/tensorflow/python/lib/core/safe_ptr.cc" - "${tensorflow_source_dir}/tensorflow/python/lib/io/py_record_reader.h" - "${tensorflow_source_dir}/tensorflow/python/lib/io/py_record_reader.cc" - "${tensorflow_source_dir}/tensorflow/python/lib/io/py_record_writer.h" - "${tensorflow_source_dir}/tensorflow/python/lib/io/py_record_writer.cc" - "${tensorflow_source_dir}/tensorflow/python/util/kernel_registry.h" - "${tensorflow_source_dir}/tensorflow/python/util/kernel_registry.cc" - "${tensorflow_source_dir}/tensorflow/python/util/util.h" - "${tensorflow_source_dir}/tensorflow/python/util/util.cc" - "${tensorflow_source_dir}/tensorflow/cc/framework/ops.cc" - "${tensorflow_source_dir}/tensorflow/cc/framework/scope.cc" - "${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.cc" -) - -if(WIN32) - # Windows: build a static library with the same objects as tensorflow.dll. - # This can be used to build for a standalone exe and also helps us to - # find all symbols that need to be exported from the dll which is needed - # to provide the tensorflow c/c++ api in tensorflow.dll. - # From the static library we create the def file with all symbols that need to - # be exported from tensorflow.dll. Because there is a limit of 64K sybmols - # that can be exported, we filter the symbols with a python script to the namespaces - # we need. - # - add_library(pywrap_tensorflow_internal_static STATIC - ${pywrap_tensorflow_internal_src} - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $<$:$> - $ - $<$:$> - $<$:$> - ) - - target_include_directories(pywrap_tensorflow_internal_static PUBLIC - ${PYTHON_INCLUDE_DIR} - ${NUMPY_INCLUDE_DIR} - ) - #target_link_libraries(pywrap_tensorflow_internal_static - # tf_protos_cc - # tf_python_protos_cc - #) - add_dependencies(pywrap_tensorflow_internal_static tf_protos_cc tf_python_protos_cc) - set(pywrap_tensorflow_internal_static_dependencies - $ - $ - $ - ${nsync_STATIC_LIBRARIES} - ) - - if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(pywrap_tensorflow_deffile "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/pywrap_tensorflow.def") - else() - set(pywrap_tensorflow_deffile "${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow.def") - endif() - set_source_files_properties(${pywrap_tensorflow_deffile} PROPERTIES GENERATED TRUE) - math(EXPR tensorflow_target_bitness "${CMAKE_SIZEOF_VOID_P}*8") - add_custom_command(TARGET pywrap_tensorflow_internal_static POST_BUILD - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/create_def_file.py - --input "${pywrap_tensorflow_internal_static_dependencies}" - --output "${pywrap_tensorflow_deffile}" - --target _pywrap_tensorflow_internal.pyd - --bitness "${tensorflow_target_bitness}" - BYPRODUCTS ${pywrap_tensorflow_deffile} # Required for Ninja - ) -endif(WIN32) - -# pywrap_tensorflow_internal is a shared library containing all of the -# TensorFlow runtime and the standard ops and kernels. These are installed into -# tf_python/tensorflow/python/. -add_library(pywrap_tensorflow_internal SHARED - ${pywrap_tensorflow_internal_src} - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $<$:$> - $ - $<$:$<$:$>> - $<$:$> - ${pywrap_tensorflow_deffile} -) - -# There is a bug in GCC 5 resulting in undefined reference to a __cpu_model function when -# linking to the tensorflow library. Adding the following libraries fixes it. -if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.0) - target_link_libraries(pywrap_tensorflow_internal PRIVATE gcc_s gcc) -endif() - -if(WIN32) - add_dependencies(pywrap_tensorflow_internal pywrap_tensorflow_internal_static) -endif(WIN32) - -target_include_directories(pywrap_tensorflow_internal PUBLIC - ${PYTHON_INCLUDE_DIR} - ${NUMPY_INCLUDE_DIR} -) - -if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.0) - # There is a bug in GCC 5 resulting in undefined reference to a __cpu_model function when - # linking to the tensorflow library. Adding the following libraries fixes it. - # See issue on github: https://github.com/tensorflow/tensorflow/issues/9593 - target_link_libraries(pywrap_tensorflow_internal PRIVATE - ${tf_core_gpu_kernels_lib} - ${tensorflow_EXTERNAL_LIBRARIES} - tf_protos_cc - tf_python_protos_cc - ${PYTHON_LIBRARIES} - gcc_s - gcc -) -else() - target_link_libraries(pywrap_tensorflow_internal PRIVATE - ${tf_core_gpu_kernels_lib} - ${tensorflow_EXTERNAL_LIBRARIES} - tf_protos_cc - tf_python_protos_cc - ${PYTHON_LIBRARIES} -) -endif() - -if(WIN32) - - # include contrib/periodic_resample as .so - # - set(tf_periodic_resample_srcs - "${tensorflow_source_dir}/tensorflow/contrib/periodic_resample/kernels/periodic_resample_op.cc" - "${tensorflow_source_dir}/tensorflow/contrib/periodic_resample/kernels/periodic_resample_op.h" - "${tensorflow_source_dir}/tensorflow/contrib/periodic_resample/ops/array_ops.cc" - ) - - AddUserOps(TARGET _periodic_resample_op - SOURCES "${tf_periodic_resample_srcs}" - DEPENDS pywrap_tensorflow_internal tf_python_ops - DISTCOPY ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/periodic_resample/python/ops/) - - # include contrib/nearest_neighbor as .so - # - set(tf_nearest_neighbor_srcs - "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/kernels/heap.h" - "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes.h" - "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes.cc" - "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/ops/nearest_neighbor_ops.cc" - ) - - AddUserOps(TARGET _nearest_neighbor_ops - SOURCES "${tf_nearest_neighbor_srcs}" - DEPENDS pywrap_tensorflow_internal tf_python_ops - DISTCOPY ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/nearest_neighbor/python/ops/) -endif(WIN32) - -# include contrib/seq2seq as .so -# -set(tf_beam_search_srcs - "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc" - "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h" - "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc" -) - -set(tf_beam_search_gpu_srcs - "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc" -) - -AddUserOps(TARGET _beam_search_ops - SOURCES "${tf_beam_search_srcs}" - GPUSOURCES ${tf_beam_search_gpu_srcs} - DEPENDS pywrap_tensorflow_internal tf_python_ops - DISTCOPY ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/seq2seq/python/ops/) - -if(WIN32) - if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - add_custom_command(TARGET pywrap_tensorflow_internal POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration)/pywrap_tensorflow_internal.dll - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.pyd - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration)/pywrap_tensorflow_internal.lib - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/) - else() - add_custom_command(TARGET pywrap_tensorflow_internal POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.dll - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.pyd - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.lib - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/) - endif() -else() - add_custom_command(TARGET pywrap_tensorflow_internal POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/libpywrap_tensorflow_internal${CMAKE_SHARED_LIBRARY_SUFFIX} - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.so) -endif() - - -######################################################## -# Generate API __init__.py files. -######################################################## - -# Parse tensorflow/python/tools/api/generator/BUILD to get list of generated files. -FILE(READ ${tensorflow_source_dir}/tensorflow/python/tools/api/generator/api_init_files.bzl api_generator_BUILD_text) -STRING(REGEX MATCH "# BEGIN GENERATED FILES.*# END GENERATED FILES" api_init_files_text ${api_generator_BUILD_text}) -string(REPLACE "# BEGIN GENERATED FILES" "" api_init_files_text ${api_init_files_text}) -string(REPLACE "# END GENERATED FILES" "" api_init_files_text ${api_init_files_text}) -string(REPLACE "," ";" api_init_files_list ${api_init_files_text}) - -set(api_init_files "") -foreach(api_init_file ${api_init_files_list}) - string(STRIP "${api_init_file}" api_init_file) - if(api_init_file) - string(REPLACE "\"" "" api_init_file "${api_init_file}") # Remove quotes - list(APPEND api_init_files "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/${api_init_file}") - endif() -endforeach(api_init_file) -set(api_init_list_file "${tensorflow_source_dir}/api_init_files_list.txt") -file(WRITE "${api_init_list_file}" "${api_init_files}") - -# Run create_python_api.py to generate __init__.py files. - -### TODO -# In order to download and compile MKL/MKL-DNN automatically in cmake script, mkl-built libraries should be added to system path -# to be loaded by python executor. However `add_custom_command` has an issue with `COMMAND ${CMAKE_COMMAND} -E env PATH=`, where -# arguments of multiple paths (such as D:/;D:/mkl) will be parsed in to seperate string without semicolon and that command fail to -# recongnize paths. As CUDA isn't built with MKL, the MKL built directory is the only path to this command to work around that issue. -# To not override the CUDA and system path in other circumstances, `if-else` branch used here to handle this problem, -# and should be removed if the path issue can be resolved. -# UPDATE: Below block appears to handle multiple items in PATH correctly, but risks command line limits if PATH is large. -# If you have issues, try `set(PY_RUNTIME_ENV "PATH=${mkl_BIN_DIRS}")` instead. -### - -set(PY_RUNTIME_ENV "") -if(tensorflow_ENABLE_MKL_SUPPORT) - # add mkl dist dlls to system path for python - file(TO_CMAKE_PATH "$ENV{PATH}" PY_RUNTIME_ENV) - set(PY_RUNTIME_ENV ${mkl_BIN_DIRS} ${PY_RUNTIME_ENV}) - file(TO_NATIVE_PATH "${PY_RUNTIME_ENV}" PY_RUNTIME_ENV) - set(PY_RUNTIME_ENV "PATH=${PY_RUNTIME_ENV}") -endif(tensorflow_ENABLE_MKL_SUPPORT) - -add_custom_command( - OUTPUT ${api_init_files} - DEPENDS tf_python_ops tf_python_copy_scripts_to_destination pywrap_tensorflow_internal tf_python_touchup_modules tf_extension_ops - - # tensorflow/__init__.py depends on files generated in this step. So, remove it while - # this step is running since the files aren't there yet. - COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py - COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py - - # Run create_python_api.py to generate API init files. - COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/tf_python "${PY_RUNTIME_ENV}" ${PYTHON_EXECUTABLE} - "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/tools/api/generator/create_python_api.py" - "--root_init_template=${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/api_template.__init__.py" - "--apidir=${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow" - "--package=tensorflow.python" - "--apiname=tensorflow" - "${api_init_list_file}" - - COMMENT "Generating __init__.py files for Python API." - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tf_python" - VERBATIM -) - -add_custom_target(tf_python_api SOURCES ${api_init_files}) -add_dependencies(tf_python_api tf_python_ops) - -# TODO(mikecase): This can be removed once tf.estimator is moved -# out of TensorFlow. -######################################################## -# Generate API __init__.py files for tf.estimator. -######################################################## - -# Parse tensorflow/python/tools/api/generator/BUILD to get list of generated files. -FILE(READ ${tensorflow_source_dir}/tensorflow/python/tools/api/generator/api_init_files.bzl api_generator_BUILD_text) -STRING(REGEX MATCH "# BEGIN GENERATED FILES.*# END GENERATED FILES" api_init_files_text ${api_generator_BUILD_text}) -string(REPLACE "# BEGIN GENERATED FILES" "" api_init_files_text ${api_init_files_text}) -string(REPLACE "# END GENERATED FILES" "" api_init_files_text ${api_init_files_text}) -string(REPLACE "," ";" api_init_files_list ${api_init_files_text}) - -set(api_init_files "") -foreach(api_init_file ${api_init_files_list}) - string(STRIP "${api_init_file}" api_init_file) - if(api_init_file) - string(REPLACE "\"" "" api_init_file "${api_init_file}") # Remove quotes - list(APPEND api_init_files "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/estimator/api/${api_init_file}") - endif() -endforeach(api_init_file) -set(estimator_api_init_list_file "${tensorflow_source_dir}/estimator_api_init_files_list.txt") -file(WRITE "${estimator_api_init_list_file}" "${api_init_files}") - -# Run create_python_api.py to generate __init__.py files. -add_custom_command( - OUTPUT ${api_init_files} - DEPENDS tf_python_ops tf_python_copy_scripts_to_destination pywrap_tensorflow_internal tf_python_touchup_modules tf_extension_ops - - # Run create_python_api.py to generate API init files. - COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/tf_python "${PY_RUNTIME_ENV}" ${PYTHON_EXECUTABLE} - "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/tools/api/generator/create_python_api.py" - "--apidir=${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/estimator/api" - "--package=tensorflow.python.estimator" - "--apiname=estimator" - "--output_package=tensorflow.python.estimator.api" - "${estimator_api_init_list_file}" - - COMMENT "Generating __init__.py files for Python API." - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tf_python" -) - -add_custom_target(estimator_python_api SOURCES ${api_init_files}) -add_dependencies(estimator_python_api tf_python_ops) -############################################################ -# Build a PIP package containing the TensorFlow runtime. -############################################################ -add_custom_target(tf_python_build_pip_package) -add_dependencies(tf_python_build_pip_package - pywrap_tensorflow_internal - tf_python_copy_scripts_to_destination - tf_python_touchup_modules - tf_python_ops - tf_python_api - estimator_python_api - tf_extension_ops) - -# Fix-up Python files that were not included by the add_python_module() macros. -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/tensorflow/tools/pip_package/setup.py - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/) -# This file is unfortunately excluded by the regex that excludes *_test.py -# files, but it is imported into tf.contrib, so we add it explicitly. -add_custom_command(TARGET tf_python_copy_scripts_to_destination PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/tensorflow/contrib/testing/python/framework/util_test.py - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/testing/python/framework/) -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/tensorflow/tools/pip_package/README - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/) -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/tensorflow/tools/pip_package/MANIFEST.in - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/) - -# Copy datasets for tf.contrib.learn. -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/datasets/data/boston_house_prices.csv - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/learn/python/learn/datasets/data/) -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/datasets/data/iris.csv - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/learn/python/learn/datasets/data/) -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/datasets/data/text_test.csv - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/learn/python/learn/datasets/data/) -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/datasets/data/text_train.csv - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/learn/python/learn/datasets/data/) - -# Create include header directory -add_custom_command(TARGET tf_python_build_pip_package PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/) - -# tensorflow headers -add_custom_command(TARGET tf_python_build_pip_package PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/tensorflow) -add_custom_command(TARGET tf_python_build_pip_package PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/tensorflow/core) -add_custom_command(TARGET tf_python_build_pip_package PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/tensorflow/stream_executor) -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory ${tensorflow_source_dir}/tensorflow/core - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/tensorflow/core) -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_BINARY_DIR}/tensorflow/core - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/tensorflow/core) -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory ${tensorflow_source_dir}/tensorflow/stream_executor - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/tensorflow/stream_executor) - -# google protobuf headers -add_custom_command(TARGET tf_python_build_pip_package PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/google) -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src/google - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/google) - -# Eigen directory -add_custom_command(TARGET tf_python_build_pip_package PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/Eigen) -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_BINARY_DIR}/eigen/src/eigen/Eigen - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/Eigen) - -# external directory -add_custom_command(TARGET tf_python_build_pip_package PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/external) -add_custom_command(TARGET tf_python_build_pip_package PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/external/eigen_archive) -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_BINARY_DIR}/external/eigen_archive - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/external/eigen_archive) - -# third_party eigen directory -add_custom_command(TARGET tf_python_build_pip_package PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/third_party) -add_custom_command(TARGET tf_python_build_pip_package PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/third_party/eigen3) -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory ${tensorflow_source_dir}/third_party/eigen3 - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/third_party/eigen3) - -# unsupported Eigen directory -add_custom_command(TARGET tf_python_build_pip_package PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/unsupported) -add_custom_command(TARGET tf_python_build_pip_package PRE_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/unsupported/Eigen) -add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_BINARY_DIR}/eigen/src/eigen/unsupported/Eigen - ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/include/unsupported/Eigen) - -if(${tensorflow_TF_NIGHTLY}) - if(${tensorflow_ENABLE_GPU}) - add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/tf_python/setup.py bdist_wheel --project_name tf_nightly_gpu - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tf_python) - else() - add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/tf_python/setup.py bdist_wheel --project_name tf_nightly - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tf_python) - endif(${tensorflow_ENABLE_GPU}) -else() - if(${tensorflow_ENABLE_GPU}) - add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/tf_python/setup.py bdist_wheel --project_name tensorflow_gpu - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tf_python) - else() - add_custom_command(TARGET tf_python_build_pip_package POST_BUILD - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/tf_python/setup.py bdist_wheel - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tf_python) - endif(${tensorflow_ENABLE_GPU}) -endif(${tensorflow_TF_NIGHTLY}) diff --git a/tensorflow/contrib/cmake/tf_shared_lib.cmake b/tensorflow/contrib/cmake/tf_shared_lib.cmake deleted file mode 100644 index 62005dd113b..00000000000 --- a/tensorflow/contrib/cmake/tf_shared_lib.cmake +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -if(WIN32) - # Windows: build a static library with the same objects as tensorflow.dll. - # This can be used to build for a standalone exe and also helps us to - # find all symbols that need to be exported from the dll which is needed - # to provide the tensorflow c/c++ api in tensorflow.dll. - # From the static library we create the def file with all symbols that need to - # be exported from tensorflow.dll. Because there is a limit of 64K sybmols - # that can be exported, we filter the symbols with a python script to the namespaces - # we need. - # - add_library(tensorflow_static STATIC - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $<$:$> - $ - $<$:$> - $<$:$> - ) - - add_dependencies(tensorflow_static tf_protos_cc) - set(tensorflow_static_dependencies - $ - $ - ) - - if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(tensorflow_deffile "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/tensorflow.def") - else() - set(tensorflow_deffile "${CMAKE_CURRENT_BINARY_DIR}/tensorflow.def") - endif() - set_source_files_properties(${tensorflow_deffile} PROPERTIES GENERATED TRUE) - math(EXPR tensorflow_target_bitness "${CMAKE_SIZEOF_VOID_P}*8") - add_custom_command(TARGET tensorflow_static POST_BUILD - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/create_def_file.py - --input "${tensorflow_static_dependencies}" - --output "${tensorflow_deffile}" - --target tensorflow.dll - --bitness "${tensorflow_target_bitness}" - ) -endif(WIN32) - -# tensorflow is a shared library containing all of the -# TensorFlow runtime and the standard ops and kernels. -add_library(tensorflow SHARED - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $<$:$> - $ - $<$:$<$:$>> - $<$:$> - ${tensorflow_deffile} -) - -target_link_libraries(tensorflow PRIVATE - ${tf_core_gpu_kernels_lib} - ${tensorflow_EXTERNAL_LIBRARIES} - tf_protos_cc -) - -# There is a bug in GCC 5 resulting in undefined reference to a __cpu_model function when -# linking to the tensorflow library. Adding the following libraries fixes it. -# See issue on github: https://github.com/tensorflow/tensorflow/issues/9593 -if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.0) - target_link_libraries(tensorflow PRIVATE gcc_s gcc) -endif() - -# Offer the user the choice of overriding the installation directories -set(INSTALL_LIB_DIR lib CACHE PATH "Installation directory for libraries") -set(INSTALL_BIN_DIR bin CACHE PATH "Installation directory for executables") -set(INSTALL_INCLUDE_DIR include CACHE PATH - "Installation directory for header files") -if(WIN32 AND NOT CYGWIN) - set(DEF_INSTALL_CMAKE_DIR cmake) -else() - set(DEF_INSTALL_CMAKE_DIR lib/cmake) -endif() -set(INSTALL_CMAKE_DIR ${DEF_INSTALL_CMAKE_DIR} CACHE PATH - "Installation directory for CMake files") - -# Make relative paths absolute (needed later on) -foreach(p LIB BIN INCLUDE CMAKE) - set(var INSTALL_${p}_DIR) - if(NOT IS_ABSOLUTE "${${var}}") - set(${var} "${CMAKE_INSTALL_PREFIX}/${${var}}") - endif() -endforeach() - -if(WIN32) - add_dependencies(tensorflow tensorflow_static) -endif(WIN32) - -target_include_directories(tensorflow PUBLIC - $) - -# Add all targets to build-tree export set -export(TARGETS tensorflow - FILE ${PROJECT_BINARY_DIR}/TensorflowTargets.cmake) - -# Export the package for use from the build-tree -export(PACKAGE Tensorflow) - -# Create the TensorflowConfig.cmake and TensorflowConfigVersion files -file(RELATIVE_PATH REL_INCLUDE_DIR "${INSTALL_CMAKE_DIR}" - "${INSTALL_INCLUDE_DIR}") -# for the build tree -set(CONF_INCLUDE_DIRS "${tensorflow_source_dir}" - "${PROJECT_BINARY_DIR}" - "${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src" - "${CMAKE_CURRENT_BINARY_DIR}/nsync/install/include" # Please if there is a better directory - "${CMAKE_CURRENT_BINARY_DIR}/eigen/src/eigen/Eigen/" - "${CMAKE_CURRENT_BINARY_DIR}/external/eigen_archive/" - "${tensorflow_source_dir}/third_party/eigen3/" - "${CMAKE_CURRENT_BINARY_DIR}/eigen/src/eigen/unsupported/Eigen/") -configure_file(TensorflowConfig.cmake.in - "${PROJECT_BINARY_DIR}/TensorflowConfig.cmake" @ONLY) -# for the install tree, yet to be complete -set(CONF_INCLUDE_DIRS "\${TENSORFLOW_CMAKE_DIR}/${REL_INCLUDE_DIR}") -configure_file(TensorflowConfig.cmake.in - "${PROJECT_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/TensorflowConfig.cmake" @ONLY) -# for both -configure_file(TensorflowConfigVersion.cmake.in - "${PROJECT_BINARY_DIR}/TensorflowConfigVersion.cmake" @ONLY) - -# install(TARGETS tensorflow EXPORT tensorflow_export -# RUNTIME DESTINATION ${INSTALL_BIN_DIR} -# LIBRARY DESTINATION ${INSTALL_LIB_DIR} -# ARCHIVE DESTINATION ${INSTALL_LIB_DIR}) - -# install(EXPORT tensorflow_export -# FILE TensorflowConfig.cmake -# DESTINATION ${INSTALL_CMAKE_DIR}) - -install(FILES - "${PROJECT_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/TensorflowConfig.cmake" - "${PROJECT_BINARY_DIR}/TensorflowConfigVersion.cmake" - DESTINATION "${INSTALL_CMAKE_DIR}" COMPONENT dev) - -# install the export set for use with the install-tree -install(EXPORT TensorflowTargets - DESTINATION ${INSTALL_CMAKE_DIR}) - -install(TARGETS tensorflow EXPORT TensorflowTargets - RUNTIME DESTINATION ${INSTALL_BIN_DIR} - LIBRARY DESTINATION ${INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${INSTALL_LIB_DIR}) - -# install necessary headers -# tensorflow headers -install(DIRECTORY ${tensorflow_source_dir}/tensorflow/cc/ - DESTINATION include/tensorflow/cc - FILES_MATCHING PATTERN "*.h") -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tensorflow/cc/ - DESTINATION include/tensorflow/cc - FILES_MATCHING PATTERN "*.h") -install(DIRECTORY ${tensorflow_source_dir}/tensorflow/core/ - DESTINATION include/tensorflow/core - FILES_MATCHING PATTERN "*.h") -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tensorflow/core/ - DESTINATION include/tensorflow/core - FILES_MATCHING PATTERN "*.h") -install(DIRECTORY ${tensorflow_source_dir}/tensorflow/stream_executor/ - DESTINATION include/tensorflow/stream_executor - FILES_MATCHING PATTERN "*.h") -# google protobuf headers -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src/google/ - DESTINATION include/google - FILES_MATCHING PATTERN "*.h") -# Eigen directory -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/eigen/src/eigen/Eigen/ - DESTINATION include/Eigen) -# external directory -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/external/eigen_archive/ - DESTINATION include/external/eigen_archive) -# third_party eigen directory -install(DIRECTORY ${tensorflow_source_dir}/third_party/eigen3/ - DESTINATION include/third_party/eigen3) -# unsupported Eigen directory -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/eigen/src/eigen/unsupported/Eigen/ - DESTINATION include/unsupported/Eigen) -# absl directory -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/abseil_cpp/src/abseil_cpp/absl/ - DESTINATION include/absl - FILES_MATCHING PATTERN "*.h") -# mkl -if (tensorflow_ENABLE_MKL_SUPPORT) - install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mkl/src/mkl/include/ - DESTINATION include/mkl) -endif (tensorflow_ENABLE_MKL_SUPPORT) diff --git a/tensorflow/contrib/cmake/tf_stream_executor.cmake b/tensorflow/contrib/cmake/tf_stream_executor.cmake deleted file mode 100644 index 6d634cb1709..00000000000 --- a/tensorflow/contrib/cmake/tf_stream_executor.cmake +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -#cc_library( -# name = "stream_executor", -# srcs = glob( -# [ -#XX "*.cc", -# "lib/*.cc", -# ], -# exclude = [ -# "**/*_test.cc", -# ], -# ) + if_cuda( -# glob([ -# "cuda/*.cc", -# ]), -# ), -# hdrs = glob([ -# "*.h", -# "cuda/*.h", -# "lib/*.h", -# "platform/**/*.h", -# ]), -# data = [ -# "//tensorflow/core:cuda", -# "//third_party/gpus/cuda:cublas", -# "//third_party/gpus/cuda:cudnn", -# ], -# linkopts = [ -# "-ldl", -# ], -# visibility = ["//visibility:public"], -# deps = [ -# "//tensorflow/core:lib", -# "//third_party/gpus/cuda:cuda_headers", -# ], -# alwayslink = 1, -#) - -######################################################## -# tf_stream_executor library -######################################################## -file(GLOB tf_stream_executor_srcs - "${tensorflow_source_dir}/tensorflow/stream_executor/*.cc" - "${tensorflow_source_dir}/tensorflow/stream_executor/*.h" - "${tensorflow_source_dir}/tensorflow/stream_executor/lib/*.cc" - "${tensorflow_source_dir}/tensorflow/stream_executor/lib/*.h" - "${tensorflow_source_dir}/tensorflow/stream_executor/platform/*.h" - "${tensorflow_source_dir}/tensorflow/stream_executor/platform/default/*.h" -) - -if (tensorflow_ENABLE_GPU) - file(GLOB tf_stream_executor_gpu_srcs - "${tensorflow_source_dir}/tensorflow/stream_executor/cuda/*.cc" - ) - if (NOT tensorflow_BUILD_CC_TESTS) - file(GLOB tf_stream_executor_gpu_tests - "${tensorflow_source_dir}/tensorflow/stream_executor/cuda/*_test.cc" - ) - list(REMOVE_ITEM tf_stream_executor_gpu_srcs ${tf_stream_executor_gpu_tests}) - endif() - list(APPEND tf_stream_executor_srcs ${tf_stream_executor_gpu_srcs}) -endif() - -file(GLOB_RECURSE tf_stream_executor_test_srcs - "${tensorflow_source_dir}/tensorflow/stream_executor/*test.cc" - "${tensorflow_source_dir}/tensorflow/stream_executor/lib/*test.h" -) -list(REMOVE_ITEM tf_stream_executor_srcs ${tf_stream_executor_test_srcs}) - -if (NOT WIN32) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lgomp") -endif (NOT WIN32) -add_library(tf_stream_executor OBJECT ${tf_stream_executor_srcs}) - -add_dependencies(tf_stream_executor - tf_core_lib -) -#target_link_libraries(tf_stream_executor -# ${CMAKE_THREAD_LIBS_INIT} -# ${PROTOBUF_LIBRARIES} -# tf_protos_cc -# tf_core_lib -#) diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake deleted file mode 100644 index af4483f3cb0..00000000000 --- a/tensorflow/contrib/cmake/tf_tests.cmake +++ /dev/null @@ -1,562 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -enable_testing() - -# -# get a temp path for test data -# -function(GetTestRunPath VAR_NAME OBJ_NAME) - if(WIN32) - if(DEFINED ENV{TMP}) - set(TMPDIR "$ENV{TMP}") - elseif(DEFINED ENV{TEMP}) - set(TMPDIR "$ENV{TEMP}") - endif() - string(REPLACE "\\" "/" TMPDIR ${TMPDIR}) - else() - set(TMPDIR "$ENV{TMPDIR}") - endif() - if(NOT EXISTS "${TMPDIR}") - message(FATAL_ERROR "Unable to determine a path to the temporary directory") - endif() - set(${VAR_NAME} "${TMPDIR}/${OBJ_NAME}" PARENT_SCOPE) -endfunction(GetTestRunPath) - -# -# create test for each source -# -function(AddTests) - cmake_parse_arguments(_AT "" "" "SOURCES;OBJECTS;LIBS;DATA;DEPENDS" ${ARGN}) - foreach(sourcefile ${_AT_SOURCES}) - string(REPLACE "${tensorflow_source_dir}/" "" exename ${sourcefile}) - string(REPLACE ".cc" "" exename ${exename}) - string(REPLACE "/" "_" exename ${exename}) - AddTest( - TARGET ${exename} - SOURCES ${sourcefile} - OBJECTS ${_AT_OBJECTS} - LIBS ${_AT_LIBS} - DATA ${_AT_DATA} - DEPENDS ${_AT_DEPENDS} - ) - endforeach() -endfunction(AddTests) - -# -# create once test -# -function(AddTest) - cmake_parse_arguments(_AT "" "TARGET" "SOURCES;OBJECTS;LIBS;DATA;DEPENDS" ${ARGN}) - - list(REMOVE_DUPLICATES _AT_SOURCES) - list(REMOVE_DUPLICATES _AT_OBJECTS) - list(REMOVE_DUPLICATES _AT_LIBS) - if (_AT_DATA) - list(REMOVE_DUPLICATES _AT_DATA) - endif(_AT_DATA) - if (_AT_DEPENDS) - list(REMOVE_DUPLICATES _AT_DEPENDS) - endif(_AT_DEPENDS) - - add_executable(${_AT_TARGET} ${_AT_SOURCES} ${_AT_OBJECTS}) - target_link_libraries(${_AT_TARGET} ${_AT_LIBS}) - - GetTestRunPath(testdir ${_AT_TARGET}) - set(tempdir "${testdir}/tmp") - file(REMOVE_RECURSE "${testdir}") - file(MAKE_DIRECTORY "${testdir}") - file(MAKE_DIRECTORY "${tempdir}") - add_test(NAME ${_AT_TARGET} COMMAND ${_AT_TARGET} WORKING_DIRECTORY "${testdir}") - set_tests_properties(${_AT_TARGET} - PROPERTIES ENVIRONMENT "TEST_TMPDIR=${tempdir};TEST_SRCDIR=${testdir}" - ) - set_tests_properties(${_AT_TARGET} PROPERTIES TIMEOUT "600") - - foreach(datafile ${_AT_DATA}) - file(RELATIVE_PATH datafile_rel ${tensorflow_source_dir} ${datafile}) - add_custom_command( - TARGET ${_AT_TARGET} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - "${datafile}" - "${testdir}/${datafile_rel}" - DEPENDS "${datafile}" - ) - endforeach() - - if (_AT_DEPENDS) - add_dependencies(${_AT_TARGET} ${_AT_DEPENDS} googletest) - endif() -endfunction(AddTest) - -# -# create python test for each script -# -function(AddPythonTests) - cmake_parse_arguments(_AT "" "" "SOURCES;DATA;DEPENDS" ${ARGN}) - list(REMOVE_DUPLICATES _AT_SOURCES) - if (_AT_DATA) - list(REMOVE_DUPLICATES _AT_DATA) - endif(_AT_DATA) - if (_AT_DEPENDS) - list(REMOVE_DUPLICATES _AT_DEPENDS) - endif(_AT_DEPENDS) - - foreach(sourcefile ${_AT_SOURCES}) - add_test(NAME ${sourcefile} COMMAND ${PYTHON_EXECUTABLE} ${sourcefile} WORKING_DIRECTORY ${tensorflow_source_dir}) - if (_AT_DEPENDS) - add_dependencies(${_AT_TARGET} ${_AT_DEPENDS}) - endif() - set_tests_properties(${sourcefile} PROPERTIES TIMEOUT "600") - endforeach() -endfunction(AddPythonTests) - -# -# ensure that every element is an existing file -# -function(CheckExists TYPE SOURCES) - foreach(source ${SOURCES}) - if(NOT EXISTS ${source}) - message(SEND_ERROR "${TYPE} not found: ${source}") - endif() - endforeach(source) -endfunction(CheckExists) - -if (tensorflow_BUILD_PYTHON_TESTS) - # - # python tests. This assumes that the tensorflow wheel is - # installed on the test system. - # TODO: we currently don't handle tests that need to have - # some environment setup: see AddTest how to add this - # - - # include all test - if (WIN32) - file(GLOB_RECURSE tf_test_rnn_src_py - "${tensorflow_source_dir}/tensorflow/contrib/rnn/python/kernel_tests/*_test.py" - ) - endif() - - file(GLOB_RECURSE tf_test_src_py - ${tf_test_rnn_src_py} - "${tensorflow_source_dir}/tensorflow/python/data/kernel_tests/*.py" - "${tensorflow_source_dir}/tensorflow/python/debug/cli/*_test.py" - "${tensorflow_source_dir}/tensorflow/python/debug/lib/*_test.py" - "${tensorflow_source_dir}/tensorflow/python/debug/wrappers/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/estimator/python/estimator/*_test.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/*.py" - "${tensorflow_source_dir}/tensorflow/python/ops/quantized_conv_ops_test.py" - "${tensorflow_source_dir}/tensorflow/python/ops/quantized_ops_test.py" - "${tensorflow_source_dir}/tensorflow/python/platform/build_info_test.py" - "${tensorflow_source_dir}/tensorflow/python/profiler/*_test.py" - "${tensorflow_source_dir}/tensorflow/python/profiler/internal/*_test.py" - "${tensorflow_source_dir}/tensorflow/python/saved_model/*_test.py" - "${tensorflow_source_dir}/tensorflow/python/training/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/data/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/factorization/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/feature_column/python/feature_column/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/image/*_test.py" - "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/periodic_resample/python/kernel_tests/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/python/kernel_tests/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/python/kernel_tests/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/stateless/python/kernel_tests/*_test.py" - # NOTE: tensor_forest tests in tensor_forest/hybrid/... still don't pass. - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/client/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/*_test.py" - ) - - if (tensorflow_BUILD_MORE_PYTHON_TESTS) - # Adding other major packages - file(GLOB_RECURSE tf_test_src_py - ${tf_test_src_py} - "${tensorflow_source_dir}/tensorflow/contrib/legacy_seq2seq/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/graph_editor/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/bayesflow/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/framework/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/distributions/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/learn/*_test.py" - ) - endif() - - # exclude the ones we don't want - set(tf_test_src_py_exclude - # Not a test. - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/__init__.py" - # Flaky because of port collisions. - "${tensorflow_source_dir}/tensorflow/python/training/localhost_cluster_performance_test.py" - # generally not working - "${tensorflow_source_dir}/tensorflow/python/profiler/pprof_profiler_test.py" - # flaky test - "${tensorflow_source_dir}/tensorflow/python/profiler/internal/run_metadata_test.py" - "${tensorflow_source_dir}/tensorflow/python/profiler/model_analyzer_test.py" - "${tensorflow_source_dir}/tensorflow/python/data/kernel_tests/map_dataset_op_test.py" - # Fails because uses data dependencies with bazel - "${tensorflow_source_dir}/tensorflow/python/saved_model/saved_model_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py" - # requires scipy - "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/pprof_profiler_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py" - # Takes very long to run without sharding (defined in bazel build file). - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cwise_ops_test.py" - # Loading resources in contrib doesn't seem to work on Windows - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/client/random_forest_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/tensor_forest_test.py" - # dask need fix - "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/learn_io/generator_io_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/learn_io/graph_io_test.py" - # Test is flaky on Windows GPU builds (b/38283730). - "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/gmm_test.py" - # Disable following manual tag in BUILD. - "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/layers/convolutional_test.py" - # These tests depend on a .so file - ${tensorflow_source_dir}/tensorflow/python/kernel_tests/duplicate_op_test.py - ${tensorflow_source_dir}/tensorflow/python/kernel_tests/invalid_op_test.py - ${tensorflow_source_dir}/tensorflow/python/kernel_tests/ackermann_test.py - # Tests too large to run. - ${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py - ) - if (WIN32) - set(tf_test_src_py_exclude - ${tf_test_src_py_exclude} - # TODO: failing tests. - # Nothing critical in here but should get this list down to [] - # The failing list is grouped by failure source - # Python source line inspection tests are flaky on Windows (b/36375074). - "${tensorflow_source_dir}/tensorflow/python/debug/cli/analyzer_cli_test.py" - "${tensorflow_source_dir}/tensorflow/python/debug/cli/profile_analyzer_cli_test.py" - # Windows does not have the curses library and uses readline. - "${tensorflow_source_dir}/tensorflow/python/debug/cli/curses_ui_test.py" - # TFDBG grpc:// mode is not yet available on Windows. - "${tensorflow_source_dir}/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py" - "${tensorflow_source_dir}/tensorflow/python/debug/lib/grpc_large_data_test.py" - "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_grpc_test.py" - "${tensorflow_source_dir}/tensorflow/python/debug/lib/source_remote_test.py" - # stl on windows handles overflows different - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/as_string_op_test.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/string_to_number_op_test.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/clip_ops_test.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/list_ops_test.py" # Needs portpicker. - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/tensor_array_ops_test.py" # Needs portpicker. - # Numerical issues, calculations off. - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/wals_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/periodic_resample/python/kernel_tests/periodic_resample_op_test.py" - "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/utils/data_utils_test.py" - "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/backend_test.py" - "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/preprocessing/image_test.py" - # Float division by zero - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/benchmark_test.py" - # Flaky, for unknown reasons. Cannot reproduce in terminal. Revisit once we can get stack traces. - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/batch_matmul_op_test.py" - # Flaky because of local cluster creation. - "${tensorflow_source_dir}/tensorflow/python/training/sync_replicas_optimizer_test.py" - "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_grpc_test.py" - "${tensorflow_source_dir}/tensorflow/python/training/localhost_cluster_performance_test.py" - "${tensorflow_source_dir}/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/functional_ops_test.py" - # Type error in testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU. - "${tensorflow_source_dir}/tensorflow/python/data/kernel_tests/iterator_ops_test.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py" - # IteratorGetMax OutOfRangeError - "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py" - # Depends on gemmlowp -> pthread - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py" - # int32/int64 mixup - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cast_op_test.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/variable_scope_test.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/functional_ops_test.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/py_func_test.py" - # Flaky on Windows cpu with py36 (b/73556968) - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/sparse_reshape_op_test.py" - # Windows file management related issues. - "${tensorflow_source_dir}/tensorflow/python/training/evaluation_test.py" - # training tests - "${tensorflow_source_dir}/tensorflow/python/training/basic_session_run_hooks_test.py" # Needs tf.contrib fix. - "${tensorflow_source_dir}/tensorflow/python/training/quantize_training_test.py" # Needs quantization ops to be included in windows. - "${tensorflow_source_dir}/tensorflow/python/training/supervisor_test.py" # Flaky I/O error on rename. - "${tensorflow_source_dir}/tensorflow/python/training/server_lib_test.py" # Test occasionally deadlocks. - "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py" # Fails on multiple GPUs. - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py" # numerical issues - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg_grad_test.py" # cudaSolver handle creation fails. - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py" # depends on python/framework/test_ops - # Dataset tests - "${tensorflow_source_dir}/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py" # Segfaults on windows - "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py" # Segfaults on Windows. - "${tensorflow_source_dir}/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py" # Deadlocks - "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py" # b/65430561 - "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py" # Segfaults on Windows. - # tensor_forest tests (also note that we exclude the hybrid tests for now) - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/kernel_tests/count_extremely_random_stats_op_test.py" # Results in wrong order. - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/kernel_tests/sample_inputs_op_test.py" # Results in wrong order. - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/kernel_tests/scatter_add_ndim_op_test.py" # Bad placement. - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/topn_test.py" # Results inaccurate - "${tensorflow_source_dir}/tensorflow/python/ops/cloud/bigquery_reader_ops_test.py" # No libcurl support - # Dask.Dataframe bugs on Window Build - "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/tests/dataframe/tensorflow_dataframe_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/learn_io/io_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/graph_actions_test.py" - # Need extra build - "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/conditional_distribution_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/conditional_transformed_distribution_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/estimator_test.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py" # depends on python/framework/test_ops - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/depthtospace_op_test.py" # QuantizeV2 - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/spacetodepth_op_test.py" # QuantizeV2 - # Windows Path - "${tensorflow_source_dir}/tensorflow/contrib/framework/python/ops/checkpoint_ops_test.py" #TODO: Fix path - "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/kmeans_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/estimators/kmeans_test.py" - # Numpy upgrade needed? - "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_test.py" - # Test should only be run manually - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/reduction_ops_test_big.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/svd_op_test.py" - # Depends on python/framework/test_ops - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/control_flow_util_test.py" - # Flaky replicate_model_fn_test - "${tensorflow_source_dir}/tensorflow/contrib/estimator/python/estimator/replicate_model_fn_test.py" # b/71901810 - # Broken io_utils_test - "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/utils/io_utils_test.py" # b/72894325 - ) - endif() - CheckExists(${tf_test_src_py_exclude}) - list(REMOVE_ITEM tf_test_src_py ${tf_test_src_py_exclude}) - - AddPythonTests( - SOURCES ${tf_test_src_py} - ) -endif(tensorflow_BUILD_PYTHON_TESTS) - -if (tensorflow_BUILD_CC_TESTS) - # - # cc unit tests. Be aware that by default we include 250+ tests which - # will take time and space to build. - # If you want to cut this down, for example to a specific test, modify - # tf_test_src_simple to your needs - # - - include_directories(${googletest_INCLUDE_DIRS}) - - # cc tests wrapper - set(tf_src_testlib - "${tensorflow_source_dir}/tensorflow/cc/framework/testutil.cc" - "${tensorflow_source_dir}/tensorflow/cc/gradients/grad_testutil.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc" - "${tensorflow_source_dir}/tensorflow/core/framework/function_testlib.cc" - "${tensorflow_source_dir}/tensorflow/core/framework/shape_inference_testutil.cc" - "${tensorflow_source_dir}/tensorflow/core/framework/tensor_testutil.cc" - "${tensorflow_source_dir}/tensorflow/core/graph/testlib.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/test.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/test_main.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/default/test_benchmark.cc" - "${tensorflow_source_dir}/tensorflow/c/c_api.cc" - "${tensorflow_source_dir}/tensorflow/c/checkpoint_reader.cc" - "${tensorflow_source_dir}/tensorflow/c/tf_status_helper.cc" - ) - - if(WIN32) - set(tf_src_testlib - ${tf_src_testlib} - "${tensorflow_source_dir}/tensorflow/core/platform/windows/test.cc" - ) - else() - set(tf_src_testlib - ${tf_src_testlib} - "${tensorflow_source_dir}/tensorflow/core/platform/posix/test.cc" - ) - endif() - - # include all test - file(GLOB_RECURSE tf_test_src_simple - "${tensorflow_source_dir}/tensorflow/cc/*_test.cc" - "${tensorflow_source_dir}/tensorflow/python/*_test.cc" - "${tensorflow_source_dir}/tensorflow/core/*_test.cc" - "${tensorflow_source_dir}/tensorflow/user_ops/*_test.cc" - "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/*_test.cc" - "${tensorflow_source_dir}/tensorflow/contrib/rnn/*_test.cc" - ) - - # exclude the ones we don't want - set(tf_test_src_simple_exclude - # generally not working - "${tensorflow_source_dir}/tensorflow/cc/client/client_session_test.cc" - "${tensorflow_source_dir}/tensorflow/cc/framework/gradients_test.cc" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/call_options_test.cc" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/tensor_coding_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/remote_fused_graph_rewriter_transform_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/hexagon/graph_transferer_test.cc" - ) - - if (NOT tensorflow_ENABLE_GPU) - # exclude gpu tests if we are not buildig for gpu - set(tf_test_src_simple_exclude - ${tf_test_src_simple_exclude} - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/gpu_allocator_retry_test.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/gpu_debug_allocator_test.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/gpu_stream_util_test.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/pool_allocator_test.cc" - ) - endif() - - if (WIN32) - set(tf_test_src_simple_exclude - ${tf_test_src_simple_exclude} - # generally excluded - "${tensorflow_source_dir}/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_test.cc" - "${tensorflow_source_dir}/tensorflow/cc/framework/cc_ops_test.cc" # test_op.h missing - - # TODO: test failing - "${tensorflow_source_dir}/tensorflow/core/common_runtime/simple_placer_test.cc" - "${tensorflow_source_dir}/tensorflow/core/debug/debug_gateway_test.cc" # hangs - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/executor_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/quantized_reshape_op_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/requantization_range_op_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/requantize_op_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/quantize_op_test.cc" - "${tensorflow_source_dir}/tensorflow/core/lib/strings/str_util_test.cc" - "${tensorflow_source_dir}/tensorflow/core/lib/strings/numbers_test.cc" - "${tensorflow_source_dir}/tensorflow/core/lib/monitoring/collection_registry_test.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/file_system_test.cc" - "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/cudnn_rnn_ops_test.cc" - "${tensorflow_source_dir}/tensorflow/contrib/rnn/ops/gru_ops_test.cc" # status 5 - "${tensorflow_source_dir}/tensorflow/contrib/rnn/ops/lstm_ops_test.cc" # status 5 - - # TODO: not compiling - "${tensorflow_source_dir}/tensorflow/core/kernels/quantization_utils_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/quantize_and_dequantize_op_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/quantize_down_and_shrink_range_op_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/debug_ops_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/quantized_activation_ops_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/quantized_bias_add_op_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/quantized_concat_op_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/quantized_conv_ops_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/quantized_matmul_op_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/quantized_pooling_ops_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/quantized_batch_norm_op_test.cc" - "${tensorflow_source_dir}/tensorflow/core/kernels/cloud/bigquery_table_accessor_test.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/cloud/gcs_file_system_test.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/cloud/google_auth_provider_test.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/cloud/http_request_test.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/cloud/oauth_client_test.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/cloud/retrying_file_system_test.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/cloud/time_util_test.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/hadoop/hadoop_file_system_test.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/profile_utils/cpu_utils_test.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/subprocess_test.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/gpu_allocator_retry_test.cc" - "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/gpu_debug_allocator_test.cc" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/master_test.cc" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/remote_device_test.cc" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/rpc/grpc_channel_test.cc" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/rpc/grpc_session_test.cc" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/rpc/grpc_tensor_coding_test.cc" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr_test.cc" - "${tensorflow_source_dir}/tensorflow/core/distributed_runtime/master_test.cc" - "${tensorflow_source_dir}/tensorflow/core/example/example_parser_configuration_test.cc" - "${tensorflow_source_dir}/tensorflow/core/example/feature_util_test.cc" - "${tensorflow_source_dir}/tensorflow/core/util/reporter_test.cc" - "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc" - "${tensorflow_source_dir}/tensorflow/contrib/session_bundle/bundle_shim_test.cc" - "${tensorflow_source_dir}/tensorflow/contrib/session_bundle/bundle_test.cc" - "${tensorflow_source_dir}/tensorflow/contrib/session_bundle/signature_test.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/core/ops/training_ops_test.cc" - "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/core/ops/tree_utils_test.cc" - ) - endif() - - # Tests for saved_model require data, so need to treat them separately. - file(GLOB tf_cc_saved_model_test_srcs - "${tensorflow_source_dir}/tensorflow/cc/saved_model/*_test.cc" - ) - - CheckExists(${tf_test_src_simple_exclude}) - list(REMOVE_ITEM tf_test_src_simple - ${tf_test_src_simple_exclude} - ${tf_cc_saved_model_test_srcs} - ) - - file(GLOB tf_core_profiler_test_srcs - "${tensorflow_source_dir}/tensorflow/core/profiler/internal/*_test.cc" - "${tensorflow_source_dir}/tensorflow/core/profiler/internal/advisor/*_test.cc" - ) - - list(REMOVE_ITEM tf_test_src_simple - ${tf_core_profiler_test_srcs} - ) - - CheckExists(${tf_src_testlib}) - set(tf_test_lib tf_test_lib) - add_library(${tf_test_lib} STATIC ${tf_src_testlib}) - - # this is giving to much objects and libraries to the linker but - # it makes this script much easier. So for now we do it this way. - set(tf_obj_test - $ - $ - $ - $ - $ - $ - $ - $ - $ - $<$:$> - ) - - set(tf_test_libs - tf_protos_cc - tf_test_lib - ${tf_core_gpu_kernels_lib} - ${googletest_STATIC_LIBRARIES} - ${tensorflow_EXTERNAL_LIBRARIES} - ) - - # All tests that require no data. - AddTests( - SOURCES ${tf_test_src_simple} - OBJECTS ${tf_obj_test} - LIBS ${tf_test_libs} - ) - - # Tests for tensorflow/cc/saved_model. - file(GLOB_RECURSE tf_cc_saved_model_test_data - "${tensorflow_source_dir}/tensorflow/cc/saved_model/testdata/*" - ) - - AddTests( - SOURCES ${tf_cc_saved_model_test_srcs} - DATA ${tf_cc_saved_model_test_data} - OBJECTS ${tf_obj_test} - LIBS ${tf_test_libs} - ) - - file(GLOB_RECURSE tf_core_profiler_test_data - "${tensorflow_source_dir}/tensorflow/core/profiler/testdata/*" - ) - - AddTests( - SOURCES ${tf_core_profiler_test_srcs} - DATA ${tf_core_profiler_test_data} - OBJECTS ${tf_obj_test} - LIBS ${tf_test_libs} - ) - -endif(tensorflow_BUILD_CC_TESTS) diff --git a/tensorflow/contrib/cmake/tf_tools.cmake b/tensorflow/contrib/cmake/tf_tools.cmake deleted file mode 100644 index 58c7df95c82..00000000000 --- a/tensorflow/contrib/cmake/tf_tools.cmake +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -set(tf_tools_proto_text_src_dir "${tensorflow_source_dir}/tensorflow/tools/proto_text") - -file(GLOB tf_tools_proto_text_srcs - "${tf_tools_proto_text_src_dir}/gen_proto_text_functions.cc" - "${tf_tools_proto_text_src_dir}/gen_proto_text_functions_lib.h" - "${tf_tools_proto_text_src_dir}/gen_proto_text_functions_lib.cc" -) - -set(proto_text "proto_text") - -add_executable(${proto_text} - ${tf_tools_proto_text_srcs} - $ -) - -target_link_libraries(${proto_text} PUBLIC - ${tensorflow_EXTERNAL_LIBRARIES} - tf_protos_cc -) - -add_dependencies(${proto_text} tf_core_lib) -if(tensorflow_ENABLE_GRPC_SUPPORT) - add_dependencies(${proto_text} grpc) -endif(tensorflow_ENABLE_GRPC_SUPPORT) - -file(GLOB_RECURSE tf_tools_transform_graph_lib_srcs - "${tensorflow_source_dir}/tensorflow/tools/graph_transforms/*.h" - "${tensorflow_source_dir}/tensorflow/tools/graph_transforms/*.cc" -) - -file(GLOB_RECURSE tf_tools_transform_graph_lib_exclude_srcs - "${tensorflow_source_dir}/tensorflow/tools/graph_transforms/*test*.h" - "${tensorflow_source_dir}/tensorflow/tools/graph_transforms/*test*.cc" - "${tensorflow_source_dir}/tensorflow/tools/graph_transforms/compare_graphs.cc" - "${tensorflow_source_dir}/tensorflow/tools/graph_transforms/summarize_graph_main.cc" - "${tensorflow_source_dir}/tensorflow/tools/graph_transforms/transform_graph_main.cc" -) -list(REMOVE_ITEM tf_tools_transform_graph_lib_srcs ${tf_tools_transform_graph_lib_exclude_srcs}) - -add_library(tf_tools_transform_graph_lib OBJECT ${tf_tools_transform_graph_lib_srcs}) -add_dependencies(tf_tools_transform_graph_lib tf_core_cpu) -add_dependencies(tf_tools_transform_graph_lib tf_core_framework) -add_dependencies(tf_tools_transform_graph_lib tf_core_kernels) -add_dependencies(tf_tools_transform_graph_lib tf_core_lib) -add_dependencies(tf_tools_transform_graph_lib tf_core_ops) - -set(transform_graph "transform_graph") - -add_executable(${transform_graph} - "${tensorflow_source_dir}/tensorflow/tools/graph_transforms/transform_graph_main.cc" - $ - $ - $ - $ - $ - $ - $ - $ - $<$:$<$:$>> - $<$:$> -) - -target_link_libraries(${transform_graph} PUBLIC - tf_protos_cc - ${tf_core_gpu_kernels_lib} - ${tensorflow_EXTERNAL_LIBRARIES} -) - -set(summarize_graph "summarize_graph") - -add_executable(${summarize_graph} - "${tensorflow_source_dir}/tensorflow/tools/graph_transforms/summarize_graph_main.cc" - $ - $ - $ - $ - $ - $ - $ - $ - $<$:$<$:$>> - $<$:$> -) - -target_link_libraries(${summarize_graph} PUBLIC - tf_protos_cc - ${tf_core_gpu_kernels_lib} - ${tensorflow_EXTERNAL_LIBRARIES} -) - -set(compare_graphs "compare_graphs") - -add_executable(${compare_graphs} - "${tensorflow_source_dir}/tensorflow/tools/graph_transforms/compare_graphs.cc" - $ - $ - $ - $ - $ - $ - $ - $ - $<$:$<$:$>> - $<$:$> -) - -target_link_libraries(${compare_graphs} PUBLIC - tf_protos_cc - ${tf_core_gpu_kernels_lib} - ${tensorflow_EXTERNAL_LIBRARIES} -) - -set(benchmark_model "benchmark_model") - -add_executable(${benchmark_model} - "${tensorflow_source_dir}/tensorflow/tools/benchmark/benchmark_model.cc" - "${tensorflow_source_dir}/tensorflow/tools/benchmark/benchmark_model_main.cc" - $ - $ - $ - $ - $ - $ - $<$:$<$:$>> - $<$:$> -) - -target_link_libraries(${benchmark_model} PUBLIC - tf_protos_cc - ${tf_core_gpu_kernels_lib} - ${tensorflow_EXTERNAL_LIBRARIES} -) - -install(TARGETS ${transform_graph} ${summarize_graph} ${compare_graphs} ${benchmark_model} - RUNTIME DESTINATION bin - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib) diff --git a/tensorflow/contrib/cmake/tf_tutorials.cmake b/tensorflow/contrib/cmake/tf_tutorials.cmake deleted file mode 100644 index e63fccc1810..00000000000 --- a/tensorflow/contrib/cmake/tf_tutorials.cmake +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -set(tf_tutorials_example_trainer_srcs - "${tensorflow_source_dir}/tensorflow/cc/tutorials/example_trainer.cc" -) - -add_executable(tf_tutorials_example_trainer - ${tf_tutorials_example_trainer_srcs} - $ - $ - $ - $ - $ - $ - $ - $ - $<$:$> -) - -target_link_libraries(tf_tutorials_example_trainer PUBLIC - tf_protos_cc - ${tf_core_gpu_kernels_lib} - ${tensorflow_EXTERNAL_LIBRARIES} -) - -install(TARGETS tf_tutorials_example_trainer - RUNTIME DESTINATION bin - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib) diff --git a/tensorflow/contrib/cmake/tools/create_def_file.py b/tensorflow/contrib/cmake/tools/create_def_file.py deleted file mode 100644 index 4f957f1e0b4..00000000000 --- a/tensorflow/contrib/cmake/tools/create_def_file.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""create_def_file.py - tool to create a windows def file. - -The def file can be used to export symbols from the tensorflow dll to enable -tf.load_library(). - -Because the linker allows only 64K symbols to be exported per dll -we filter the symbols down to the essentials. The regular expressions -we use for this are specific to tensorflow. - -TODO: this works fine but there is an issue with exporting -'const char * const' and importing it from a user_ops. The problem is -on the importing end and using __declspec(dllimport) works around it. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import codecs -import os -import re -import subprocess -import sys -import tempfile - -# External tools we use that come with visual studio sdk and -# we assume that the caller has the correct PATH to the sdk -UNDNAME = "undname.exe" -DUMPBIN = "dumpbin.exe" - -# Exclude if matched -EXCLUDE_RE = re.compile(r"RTTI|deleting destructor|::internal::|Internal|" - r"python_op_gen_internal|grappler") - -# Include if matched before exclude -INCLUDEPRE_RE = re.compile(r"google::protobuf::internal::ExplicitlyConstructed|" - r"tensorflow::internal::LogMessage|" - r"tensorflow::internal::LogString|" - r"tensorflow::internal::CheckOpMessageBuilder|" - r"tensorflow::internal::PickUnusedPortOrDie|" - r"tensorflow::internal::ValidateDevice|" - r"tensorflow::ops::internal::Enter|" - r"tensorflow::strings::internal::AppendPieces|" - r"tensorflow::strings::internal::CatPieces|" - r"tensorflow::errors::Internal|" - r"tensorflow::Tensor::CopyFromInternal|" - r"tensorflow::kernel_factory::" - r"OpKernelRegistrar::InitInternal|" - r"tensorflow::io::internal::JoinPathImpl") - -# Include if matched after exclude -INCLUDE_RE = re.compile(r"^(TF_\w*)$|" - r"^(TFE_\w*)$|" - r"tensorflow::|" - r"functor::|" - r"\?nsync_|" - r"stream_executor::") - -# We want to identify data members explicitly in the DEF file, so that no one -# can implicitly link against the DLL if they use one of the variables exported -# from the DLL and the header they use does not decorate the symbol with -# __declspec(dllimport). It is easier to detect what a data symbol does -# NOT look like, so doing it with the below regex. -DATA_EXCLUDE_RE = re.compile(r"[)(]|" - r"vftable|" - r"vbtable|" - r"vcall|" - r"RTTI|" - r"protobuf::internal::ExplicitlyConstructed") - -def get_args(): - """Parse command line.""" - filename_list = lambda x: x.split(";") - parser = argparse.ArgumentParser() - parser.add_argument("--input", type=filename_list, - help="paths to input libraries separated by semicolons", - required=True) - parser.add_argument("--output", help="output deffile", required=True) - parser.add_argument("--target", help="name of the target", required=True) - parser.add_argument("--bitness", help="build target bitness", required=True) - args = parser.parse_args() - return args - - -def main(): - """main.""" - args = get_args() - - # Pipe dumpbin to extract all linkable symbols from libs. - # Good symbols are collected in candidates and also written to - # a temp file. - candidates = [] - tmpfile = tempfile.NamedTemporaryFile(mode="w", delete=False) - for lib_path in args.input: - proc = subprocess.Popen([DUMPBIN, "/nologo", "/linkermember:1", lib_path], - stdout=subprocess.PIPE) - for line in codecs.getreader("utf-8")(proc.stdout): - cols = line.split() - if len(cols) < 2: - continue - sym = cols[1] - tmpfile.file.write(sym + "\n") - candidates.append(sym) - exit_code = proc.wait() - if exit_code != 0: - print("{} failed, exit={}".format(DUMPBIN, exit_code)) - return exit_code - tmpfile.file.close() - - # Run the symbols through undname to get their undecorated name - # so we can filter on something readable. - with open(args.output, "w") as def_fp: - # track dupes - taken = set() - - # Header for the def file. - def_fp.write("LIBRARY " + args.target + "\n") - def_fp.write("EXPORTS\n") - if args.bitness == "64": - def_fp.write("\t??1OpDef@tensorflow@@UEAA@XZ\n") - else: - def_fp.write("\t??1OpDef@tensorflow@@UAE@XZ\n") - - # Each symbols returned by undname matches the same position in candidates. - # We compare on undname but use the decorated name from candidates. - dupes = 0 - proc = subprocess.Popen([UNDNAME, tmpfile.name], stdout=subprocess.PIPE) - for idx, line in enumerate(codecs.getreader("utf-8")(proc.stdout)): - decorated = candidates[idx] - if decorated in taken: - # Symbol is already in output, done. - dupes += 1 - continue - - if not INCLUDEPRE_RE.search(line): - if EXCLUDE_RE.search(line): - continue - if not INCLUDE_RE.search(line): - continue - - if "deleting destructor" in line: - # Some of the symbols convered by INCLUDEPRE_RE export deleting - # destructor symbols, which is a bad idea. - # So we filter out such symbols here. - continue - - if DATA_EXCLUDE_RE.search(line): - def_fp.write("\t" + decorated + "\n") - else: - def_fp.write("\t" + decorated + " DATA\n") - taken.add(decorated) - exit_code = proc.wait() - if exit_code != 0: - print("{} failed, exit={}".format(UNDNAME, exit_code)) - return exit_code - - os.unlink(tmpfile.name) - - print("symbols={}, taken={}, dupes={}" - .format(len(candidates), len(taken), dupes)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tensorflow/contrib/compiler/BUILD b/tensorflow/contrib/compiler/BUILD deleted file mode 100644 index e7ba9e3d6a0..00000000000 --- a/tensorflow/contrib/compiler/BUILD +++ /dev/null @@ -1,66 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "cuda_py_test") - -package( - default_visibility = [":friends"], - licenses = ["notice"], # Apache 2.0 -) - -package_group( - name = "friends", - includes = ["//tensorflow/compiler/jit:friends"], - packages = [ - "//tensorflow/...", - "//tensorflow_models/...", - "//third_party/py/tensor2tensor/...", - ], -) - -py_library( - name = "compiler_py", - srcs = [ - "__init__.py", - "jit.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":xla", - "//tensorflow/python/compiler/xla:compiler_py", - ], -) - -py_library( - name = "xla", - srcs = ["xla.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:platform", - "//tensorflow/python:util", - "//tensorflow/python/compiler/xla:compiler_py", - "//tensorflow/python/estimator:estimator_py", - ], -) - -cuda_py_test( - name = "xla_test", - srcs = ["xla_test.py"], - additional_deps = [ - ":xla", - "@absl_py//absl/testing:parameterized", - "//tensorflow/compiler/tests:xla_test", - "//tensorflow/contrib/tpu:tpu_estimator", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:platform", - "//tensorflow/python:summary", - "//tensorflow/python:training", - "//tensorflow/python/data/ops:dataset_ops", - ], - tags = [ - "no_mac", - "no_windows", - ], - xla_enabled = True, -) diff --git a/tensorflow/contrib/compiler/__init__.py b/tensorflow/contrib/compiler/__init__.py deleted file mode 100644 index 797e5e8164e..00000000000 --- a/tensorflow/contrib/compiler/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A module for controlling the Tensorflow/XLA JIT compiler.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.compiler import jit -from tensorflow.contrib.compiler import xla diff --git a/tensorflow/contrib/compiler/jit.py b/tensorflow/contrib/compiler/jit.py deleted file mode 100644 index 70898aeb974..00000000000 --- a/tensorflow/contrib/compiler/jit.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Library for controlling the Tensorflow/XLA JIT compiler.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.compiler.xla import jit - -experimental_jit_scope = jit.experimental_jit_scope diff --git a/tensorflow/contrib/compiler/tests/BUILD b/tensorflow/contrib/compiler/tests/BUILD deleted file mode 100644 index 15147627897..00000000000 --- a/tensorflow/contrib/compiler/tests/BUILD +++ /dev/null @@ -1,52 +0,0 @@ -# These tests depend on contrib, so they can't be in //tensorflow/compiler/tests - -load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test") - -package( - licenses = ["notice"], # Apache 2.0 -) - -tf_xla_py_test( - name = "adamax_test", - size = "small", - srcs = ["adamax_test.py"], - deps = [ - "//tensorflow/compiler/tests:xla_test", - "//tensorflow/contrib/opt:opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) - -tf_xla_py_test( - name = "addsign_test", - size = "small", - srcs = ["addsign_test.py"], - deps = [ - "//tensorflow/compiler/tests:xla_test", - "//tensorflow/contrib/opt:opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) - -tf_xla_py_test( - name = "powersign_test", - size = "small", - srcs = ["powersign_test.py"], - deps = [ - "//tensorflow/compiler/tests:xla_test", - "//tensorflow/contrib/opt:opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/compiler/tests/adamax_test.py b/tensorflow/contrib/compiler/tests/adamax_test.py deleted file mode 100644 index e50b5594a62..00000000000 --- a/tensorflow/contrib/compiler/tests/adamax_test.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for AdaMax optimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.compiler.tests import xla_test -from tensorflow.contrib.opt.python.training import adamax -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def adamax_update_numpy(param, - g_t, - t, - m, - v, - alpha=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8): - m_t = beta1 * m + (1 - beta1) * g_t - v_t = np.maximum(beta2 * v, np.abs(g_t)) - param_t = param - (alpha / (1 - beta1**t)) * (m_t / (v_t + epsilon)) - return param_t, m_t, v_t - - -class AdaMaxOptimizerTest(xla_test.XLATestCase): - - def testBasic(self): - for i, dtype in enumerate(self.float_types): - with self.session(), self.test_scope(): - variable_scope.get_variable_scope().set_use_resource(True) - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype) - - var0 = resource_variable_ops.ResourceVariable( - var0_np, name="var0_%d" % i) - var1 = resource_variable_ops.ResourceVariable( - var1_np, name="var1_%d" % i) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - opt = adamax.AdaMaxOptimizer() - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - opt_variables = opt.variables() - beta1_power = opt._get_beta_accumulators() - self.assertTrue(beta1_power is not None) - self.assertIn(beta1_power, opt_variables) - - with ops.Graph().as_default(): - # Shouldn't return non-slot variables from other graphs. - self.assertEqual(0, len(opt.variables())) - - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - beta1_power = opt._get_beta_accumulators() - - # Run 3 steps of AdaMax - for t in range(1, 4): - update.run() - - self.assertAllCloseAccordingToType(0.9**(t + 1), - self.evaluate(beta1_power)) - - var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType( - var0_np, self.evaluate(var0), rtol=1e-2) - self.assertAllCloseAccordingToType( - var1_np, self.evaluate(var1), rtol=1e-2) - self.assertEqual("var0_%d/AdaMax:0" % (i,), - opt.get_slot(var=var0, name="m").name) - - def testTensorLearningRate(self): - for dtype in self.float_types: - with self.session(), self.test_scope(): - variable_scope.get_variable_scope().set_use_resource(True) - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype) - - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - opt = adamax.AdaMaxOptimizer(constant_op.constant(0.001)) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - beta1_power = opt._get_beta_accumulators() - - # Run 3 steps of AdaMax - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power)) - update.run() - - var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/compiler/tests/addsign_test.py b/tensorflow/contrib/compiler/tests/addsign_test.py deleted file mode 100644 index f55ab75745a..00000000000 --- a/tensorflow/contrib/compiler/tests/addsign_test.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for AddSign.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.compiler.tests import xla_test -from tensorflow.contrib.opt.python.training import addsign -from tensorflow.contrib.opt.python.training import sign_decay -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def py_linear_decay_fn(decay_steps): - def linear_decay(step): - step = min(step, decay_steps) - return float(decay_steps - step) / decay_steps - return linear_decay - - -def addsign_update_numpy(params, - g_t, - m, - lr, - alpha=1.0, - beta=0.9, - py_sign_decay_fn=None, - t=None): - m_t = beta * m + (1 - beta) * g_t - if py_sign_decay_fn is None: - sign_decayed = 1.0 - else: - sign_decayed = py_sign_decay_fn(t-1) - multiplier = alpha + sign_decayed * np.sign(g_t) * np.sign(m_t) - params_t = params - lr * multiplier * g_t - return params_t, m_t - - -class AddSignTest(xla_test.XLATestCase): - - def _testDense(self, - learning_rate=0.1, - sign_decay_fn=None, - py_sign_decay_fn=None, - alpha=1.0, - beta=0.9): - for dtype in self.float_types: - with self.session(), self.test_scope(): - # Initialize variables for numpy implementation. - m0, m1 = 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype) - - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - global_step = resource_variable_ops.ResourceVariable(0, trainable=False) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - opt = addsign.AddSignOptimizer( - learning_rate=learning_rate, - alpha=alpha, - beta=beta, - sign_decay_fn=sign_decay_fn, - ) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - neg_update = opt.apply_gradients(zip([-grads0, -grads1], [var0, var1]), - global_step=global_step) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - # Run 7 steps of AddSign - # first 4 steps with positive gradient - # last 3 steps with negative gradient (sign(gm) should be -1) - for t in range(1, 8): - if t < 5: - update.run() - else: - neg_update.run() - - var0_np, m0 = addsign_update_numpy( - var0_np, - grads0_np if t < 5 else -grads0_np, - m0, - learning_rate, - alpha=alpha, - beta=beta, - py_sign_decay_fn=py_sign_decay_fn, - t=t, - ) - var1_np, m1 = addsign_update_numpy( - var1_np, - grads1_np if t < 5 else -grads1_np, - m1, - learning_rate, - alpha=alpha, - beta=beta, - py_sign_decay_fn=py_sign_decay_fn, - t=t, - ) - - # Validate updated params - self.assertAllCloseAccordingToType( - var0_np, self.evaluate(var0), half_rtol=1e-2) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - - def testDense(self): - decay_steps = 10 - sign_decay_fn = sign_decay.get_linear_decay_fn(decay_steps) - py_sign_decay_fn = py_linear_decay_fn(decay_steps) - self._testDense() - self._testDense(learning_rate=0.01, alpha=0.1, beta=0.8) - self._testDense( - sign_decay_fn=sign_decay_fn, py_sign_decay_fn=py_sign_decay_fn) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/compiler/tests/powersign_test.py b/tensorflow/contrib/compiler/tests/powersign_test.py deleted file mode 100644 index 119b15b6b2c..00000000000 --- a/tensorflow/contrib/compiler/tests/powersign_test.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for PowerSign.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import numpy as np - -from tensorflow.compiler.tests import xla_test -from tensorflow.contrib.opt.python.training import powersign -from tensorflow.contrib.opt.python.training import sign_decay -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def py_linear_decay_fn(decay_steps): - def linear_decay(step): - step = min(step, decay_steps) - return float(decay_steps - step) / decay_steps - return linear_decay - - -def powersign_update_numpy(params, - g_t, - m, - lr, - base=math.e, - beta=0.9, - py_sign_decay_fn=None, - t=None): - m_t = beta * m + (1 - beta) * g_t - if py_sign_decay_fn is None: - sign_decayed = 1.0 - else: - sign_decayed = py_sign_decay_fn(t-1) - multiplier = base ** (sign_decayed * np.sign(g_t) * np.sign(m_t)) - params_t = params - lr * multiplier * g_t - return params_t, m_t - - -class PowerSignTest(xla_test.XLATestCase): - - def _testDense(self, - learning_rate=0.1, - sign_decay_fn=None, - py_sign_decay_fn=None, - base=math.e, - beta=0.9): - for dtype in self.float_types: - with self.session(), self.test_scope(): - # Initialize variables for numpy implementation. - m0, m1 = 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype) - - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - global_step = resource_variable_ops.ResourceVariable(0, trainable=False) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - opt = powersign.PowerSignOptimizer( - learning_rate=learning_rate, - base=base, - beta=beta, - sign_decay_fn=sign_decay_fn, - ) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - neg_update = opt.apply_gradients(zip([-grads0, -grads1], [var0, var1]), - global_step=global_step) - - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - # Run 7 steps of powersign - # first 4 steps with positive gradient - # last 3 steps with negative gradient (sign(gm) should be -1) - for t in range(1, 8): - if t < 5: - update.run() - else: - neg_update.run() - - var0_np, m0 = powersign_update_numpy( - var0_np, - grads0_np if t < 5 else -grads0_np, - m0, - learning_rate, - base=base, - beta=beta, - py_sign_decay_fn=py_sign_decay_fn, - t=t, - ) - var1_np, m1 = powersign_update_numpy( - var1_np, - grads1_np if t < 5 else -grads1_np, - m1, - learning_rate, - base=base, - beta=beta, - py_sign_decay_fn=py_sign_decay_fn, - t=t, - ) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - - def testDense(self): - decay_steps = 10 - sign_decay_fn = sign_decay.get_linear_decay_fn(decay_steps) - py_sign_decay_fn = py_linear_decay_fn(decay_steps) - self._testDense() - self._testDense(learning_rate=0.1, base=10.0, beta=0.8) - self._testDense( - sign_decay_fn=sign_decay_fn, py_sign_decay_fn=py_sign_decay_fn) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/compiler/xla.py b/tensorflow/contrib/compiler/xla.py deleted file mode 100644 index eec0b0ccb09..00000000000 --- a/tensorflow/contrib/compiler/xla.py +++ /dev/null @@ -1,296 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""xla is an experimental library that provides XLA support APIs.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.compiler.xla import xla -from tensorflow.python.estimator import model_fn as model_fn_lib -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util import function_utils -from tensorflow.python.util import tf_decorator - - -compile = xla.compile # pylint: disable=redefined-builtin -check_function_argument_count = xla.check_function_argument_count - -class _CapturedObject(object): - """A placeholder to capture an object.""" - - def __init__(self): - self._object = None - - def capture(self, o): - if self._object: - raise RuntimeError( - 'InternalError: _CapturedObject can capture only once. Please file ' - 'bug.') - - self._object = o - - def get(self): - return self._object - - -def _get_scaffold(captured_scaffold_fn): - """Retrieves the Scaffold from `captured_scaffold_fn`.""" - scaffold_fn = captured_scaffold_fn.get() - - if not scaffold_fn: - return None - - scaffold = scaffold_fn() - if scaffold is None: - raise ValueError( - 'TPUEstimatorSpec.scaffold_fn returns None, which is not allowed') - - return scaffold - - -class _ModelFnWrapper(object): - """_ModelFnWrapper supports executing model_fn with XLA.""" - - def __init__(self, function): - self._model_fn = function - - def __call__(self, features, labels, mode, params): - - # TPUEstimator compiles model_fn when use_tpu=True. To avoid double - # compilation, we use this params['use_tpu'] as a hint. When it is set to - # True, model_fn is called without compilation. - # Note that this condition isn't accurate for the case of exporting a model. - # In that case we should ideally not compile so that user can see detailed - # graph. However, we don't have enough information to tell whether model_fn - # is being called for export mode or not. - # TODO(ycao): Make this condition more accurate when implementing PREDICT - # mode. - if params.get('use_tpu'): - return self._call_model_fn(features, labels, mode, params) - - if mode == model_fn_lib.ModeKeys.TRAIN: - train_step, captured_scaffold_fn = self._make_train_step( - features, labels, params) - (loss,) = compile(train_step) - return model_fn_lib.EstimatorSpec( - mode=mode, - loss=loss, - train_op=array_ops.identity(loss), - scaffold=_get_scaffold(captured_scaffold_fn)) - elif mode == model_fn_lib.ModeKeys.EVAL: - eval_step, captured_eval_metric_fn, captured_scaffold_fn = ( - self._make_eval_step(features, labels, params)) - outputs = compile(eval_step) - loss = outputs[0] - - # Calculate eval_metric_ops if eval_metric_fn is set and captured. - eval_metric_fn = captured_eval_metric_fn.get() - if eval_metric_fn: - eval_metric_fn_tensors = outputs[1:] - eval_metric_ops = eval_metric_fn(*eval_metric_fn_tensors) - else: - eval_metric_ops = None - - return model_fn_lib.EstimatorSpec( - mode=mode, - loss=loss, - eval_metric_ops=eval_metric_ops, - scaffold=_get_scaffold(captured_scaffold_fn)) - else: - raise NotImplementedError('%s is not implemented, only TRAIN and EVAL are' - ' supported' % mode) - - def _make_train_step(self, features, labels, params): - """Creates a single step of training for xla.compile().""" - captured_scaffold_fn = _CapturedObject() - - def train_step(): - """A single step of training.""" - estimator_spec = self._call_model_fn(features, labels, - model_fn_lib.ModeKeys.TRAIN, params) - - try: - captured_scaffold_fn.capture(estimator_spec.scaffold_fn) - except AttributeError: - captured_scaffold_fn.capture(None) - - # train_step will be run by xla.compile(). xla.compile() only supports - # tensor output while train_op can be either an operation or a tensor. - # Even though xla.compile() automatically adds operation-typed train_op as - # control dependency of other tensor outputs, it doesn't do so for - # tensor-typed train_op. Thus, we need to set it explicitly here. - with ops.control_dependencies([estimator_spec.train_op]): - return array_ops.identity(estimator_spec.loss) - - return train_step, captured_scaffold_fn - - def _make_eval_step(self, features, labels, params): - """Creates a single step of evaluation for xla.compile().""" - captured_eval_metric_fn = _CapturedObject() - captured_scaffold_fn = _CapturedObject() - - def eval_step(): - """A single step of evaluation.""" - estimator_spec = self._call_model_fn(features, labels, - model_fn_lib.ModeKeys.EVAL, params) - - try: - captured_scaffold_fn.capture(estimator_spec.scaffold_fn) - except AttributeError: - captured_scaffold_fn.capture(None) - - eval_metric_fn = None - eval_metric_fn_tensors = [] - try: - if estimator_spec.eval_metrics: - (eval_metric_fn, eval_metric_fn_tensors) = estimator_spec.eval_metrics - except AttributeError: - pass - - # If a dictionary is provided, we need to convert it into a list sorted - # according to order of eval_metric_fn positional arguments. - if isinstance(eval_metric_fn_tensors, dict): - eval_metric_fn_args = function_utils.fn_args(eval_metric_fn) - eval_metric_fn_tensors = [ - eval_metric_fn_tensors[i] for i in eval_metric_fn_args - ] - - captured_eval_metric_fn.capture(eval_metric_fn) - - return tuple([estimator_spec.loss] + eval_metric_fn_tensors) - - return eval_step, captured_eval_metric_fn, captured_scaffold_fn - - def _call_model_fn(self, features, labels, mode, params): - """Calls the model_fn with required parameters.""" - model_fn_args = function_utils.fn_args(self._model_fn) - kwargs = {} - - if 'labels' in model_fn_args: - kwargs['labels'] = labels - elif labels is not None: - raise ValueError( - 'model_fn does not take labels, but input_fn returns labels.') - if 'mode' in model_fn_args: - kwargs['mode'] = mode - - if 'params' in model_fn_args: - kwargs['params'] = params - - return self._verify_estimator_spec( - self._model_fn(features=features, **kwargs)) - - def _verify_estimator_spec(self, estimator_spec): - """Verifies estimator spec contains correct data.""" - # TODO(ycao): Implement estimator spec verification for other modes. - - try: - if estimator_spec.scaffold: - logging.warning('EstimatorSpec.scaffold is ignored with XLA compilation' - '. Please use TPUEstimatorSpec.scaffold_fn instead.') - except AttributeError: - pass - - try: - if estimator_spec.eval_metric_ops: - raise ValueError('EstimatorSpec.eval_metric_ops is not supported with ' - 'XLA compilation. Please use ' - 'TPUEstimatorSpec.eval_metrics instead.') - except AttributeError: - pass - - if estimator_spec.mode == model_fn_lib.ModeKeys.EVAL: - # If estimator_spec is of type TPUEstimatorSpec and contains eval_metrics, - # check that eval_metrics contains eval_metric_fn and - # eval_metric_fn_tensors with matching arguments. - try: - eval_metrics = estimator_spec.eval_metrics - except AttributeError: - eval_metrics = None - - if eval_metrics: - (eval_metric_fn, eval_metric_fn_tensors) = eval_metrics - eval_metric_fn_args = function_utils.fn_args(eval_metric_fn) - - if isinstance(eval_metric_fn_tensors, dict): - missing_tensors = [ - i for i in eval_metric_fn_args if i not in eval_metric_fn_tensors - ] - additional_tensors = [ - i for i in eval_metric_fn_tensors if i not in eval_metric_fn_args - ] - - if missing_tensors: - raise ValueError('Arguments %s are needed by metric_fn (first ' - 'element of TPUEstimatorSpec.eval_metrics) but ' - 'they are not provided by evaluation tensors ' - '(second element of TPUEstimatorSpec.eval_metrics)' - '.' % missing_tensors) - - if additional_tensors: - raise ValueError('Arguments %s are provided by evaluation tensors ' - '(second element of TPUEstimatorSpec.eval_metrics)' - ' but they are not needed by metric_fn (first ' - 'element of TPUEstimatorSpec.eval_metrics).' % - additional_tensors) - - return estimator_spec - - -def estimator_model_fn(target_model_fn=None): - """estimator_model_fn decorates a model_fn to be compiled for execution. - - Currently it only works with `TPUEstimator`. If you need to use it with base - `Estimator`, please add `tf.compat.v1.enable_resource_variables()` at the - beginning of your program. - - Example 1, decorating model_fn: - ``` - @xla.estimator_model_fn() - def model_fn(features, labels, mode, params): - ... - return EstimatorSpec(...) - - - est = Estimator(model_fn=model_fn, ...) - est.train(...) - - ``` - - Example 2, decorator as function: - ``` - def model_fn(features, labels, mode, params): - ... - return EstimatorSpec(...) - - est = Estimator(model_fn=xla.estimator_model_fn(model_fn), ...) - est.train(...) - ``` - - Args: - target_model_fn: model_fn to be decorated. This is only needed when - decorator is used in function call form (example 2). - - Returns: - Decorated target_model_fn. - """ - - def decorated(function): - return tf_decorator.make_decorator(function, _ModelFnWrapper(function)) - - return decorated(target_model_fn) if target_model_fn else decorated diff --git a/tensorflow/contrib/compiler/xla_test.py b/tensorflow/contrib/compiler/xla_test.py deleted file mode 100644 index 0df7c3706aa..00000000000 --- a/tensorflow/contrib/compiler/xla_test.py +++ /dev/null @@ -1,369 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Tests for contrib.compiler.xla.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re -from absl.testing import parameterized - -from tensorflow.contrib.compiler import xla -from tensorflow.contrib.tpu.python.tpu import tpu_estimator -from tensorflow.contrib.training.python.training import hparam -from tensorflow.python import summary -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.estimator import model_fn as model_fn_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.platform import test -from tensorflow.python.training import training - - -_TRAIN = model_fn_lib.ModeKeys.TRAIN -_EVAL = model_fn_lib.ModeKeys.EVAL -_EXPECTED_LOSS = 1 -_EXPECTED_FEATURE = 2 -_EXPECTED_LABEL = 3 - - -def _test_train_model_fn(features, labels, mode, params): - """A dummy model_fn for testing purpose.""" - del features, labels, params - loss = constant_op.constant(_EXPECTED_LOSS) - return model_fn_lib.EstimatorSpec( - mode=mode, loss=loss, train_op=array_ops.identity(loss)) - - -@xla.estimator_model_fn -def decorated_model_fn(features, labels, mode, params): - return _test_train_model_fn(features, labels, mode, params) - - -def make_dummy_features_labels(): - # XLA CPU/GPU backend doesn't support guaranteed constant, thus use dataset - # container to work around. - features_dataset = dataset_ops.Dataset.from_tensors( - constant_op.constant(_EXPECTED_FEATURE)).repeat(10) - features_op = features_dataset.make_one_shot_iterator().get_next() - labels_dataset = dataset_ops.Dataset.from_tensors( - constant_op.constant(_EXPECTED_LABEL)).repeat(10) - labels_op = labels_dataset.make_one_shot_iterator().get_next() - return features_op, labels_op - - -class XlaDecoratorTest(test.TestCase, parameterized.TestCase): - - @parameterized.named_parameters( - ('test_use_as_decorator', decorated_model_fn, None), - ('test_use_as_function', xla.estimator_model_fn(_test_train_model_fn), - None), - ('test_use_tpu_false_hparams', decorated_model_fn, - hparam.HParams(use_tpu=False)), - ('test_use_tpu_false_dict_params', decorated_model_fn, { - 'use_tpu': False - }), - ) - def test_compile(self, model_fn, params): - """Calls model_fn and verifies it is compiled.""" - with test.mock.patch.object(xla, 'compile') as mock_xla_compile: - loss = constant_op.constant(_EXPECTED_LOSS) - mock_xla_compile.return_value = [loss] - - features, labels = make_dummy_features_labels() - estimator_spec = model_fn( - features=features, labels=labels, mode=_TRAIN, params=params or {}) - - self.assertEqual(mock_xla_compile.call_count, 1) - self.assertEqual(estimator_spec.mode, _TRAIN) - - with self.test_session() as sess: - self.assertEqual(sess.run(estimator_spec.loss), sess.run(loss)) - self.assertEqual(sess.run(estimator_spec.train_op), sess.run(loss)) - - @parameterized.named_parameters( - ('test_use_tpu_true_hparams', decorated_model_fn, - hparam.HParams(use_tpu=True)), - ('test_use_tpu_true_dict_params', decorated_model_fn, { - 'use_tpu': True - }), - ) - def test_not_compile(self, model_fn, params): - """Calls model_fn and verifies it is NOT compiled.""" - with test.mock.patch.object(xla, 'compile') as mock_xla_compile: - loss = constant_op.constant(_EXPECTED_LOSS) - mock_xla_compile.return_value = [loss] - - features, labels = make_dummy_features_labels() - estimator_spec = model_fn( - features=features, labels=labels, mode=_TRAIN, params=params or {}) - - mock_xla_compile.assert_not_called() - self.assertEqual(estimator_spec.mode, _TRAIN) - - with self.test_session() as sess: - self.assertEqual(sess.run(estimator_spec.loss), sess.run(loss)) - self.assertEqual(sess.run(estimator_spec.train_op), sess.run(loss)) - - def test_model_with_summary(self): - """Tests that summary ops are disabled.""" - - @xla.estimator_model_fn - def model_fn_with_summary(features, labels, mode, params): - del features, labels, params - loss = constant_op.constant(_EXPECTED_LOSS) - summary.scalar('loss_scalar_summary', loss) - summary.histogram('loss_histogram_summary', loss) - summary.image('loss_image_summary', loss) - return model_fn_lib.EstimatorSpec( - mode=mode, loss=loss, train_op=array_ops.identity(loss)) - - features, labels = make_dummy_features_labels() - estimator_spec = model_fn_with_summary( - features=features, labels=labels, mode=_TRAIN, params={}) - - with self.test_session() as sess: - self.assertEqual(sess.run(estimator_spec.loss), _EXPECTED_LOSS) - - -def _test_eval_metric_fn(eval_tensor_1, eval_tensor_2): - return { - 'metric_1': (eval_tensor_1, eval_tensor_1), - 'metric_2': (eval_tensor_2, eval_tensor_2), - } - - -class XlaDecoratorEvaluationTest(test.TestCase): - - def _verify_evaluation_result(self, eval_model_fn): - features, labels = make_dummy_features_labels() - estimator_spec = eval_model_fn( - features=features, labels=labels, mode=_EVAL, params={}) - - with self.test_session() as sess: - self.assertEqual(sess.run(estimator_spec.loss), _EXPECTED_LOSS) - self.assertEqual( - sess.run(estimator_spec.eval_metric_ops['metric_1'][0]), - _EXPECTED_FEATURE + _EXPECTED_LABEL) - self.assertEqual( - sess.run(estimator_spec.eval_metric_ops['metric_1'][1]), - _EXPECTED_FEATURE + _EXPECTED_LABEL) - self.assertEqual( - sess.run(estimator_spec.eval_metric_ops['metric_2'][0]), - _EXPECTED_FEATURE - _EXPECTED_LABEL) - self.assertEqual( - sess.run(estimator_spec.eval_metric_ops['metric_2'][1]), - _EXPECTED_FEATURE - _EXPECTED_LABEL) - - def test_eval_base_estimator_spec_eval_metric_ops_disallowed(self): - - @xla.estimator_model_fn - def eval_model_fn_return_estimator_spec(features, labels, mode, params): - del features, labels, params - loss = constant_op.constant(_EXPECTED_LOSS) - return model_fn_lib.EstimatorSpec( - mode=mode, - loss=loss, - eval_metric_ops={ - 'metric': (array_ops.identity(loss), control_flow_ops.no_op()) - }) - - with self.assertRaisesRegexp( - ValueError, 'EstimatorSpec.eval_metric_ops is not supported with XLA ' - 'compilation. Please use TPUEstimatorSpec.eval_metrics instead.'): - self._verify_evaluation_result(eval_model_fn_return_estimator_spec) - - def test_eval_base_estimator_spec_no_eval_metric_ops(self): - - @xla.estimator_model_fn - def eval_model_fn_no_eval_metric_ops(features, labels, mode, params): - del features, labels, params - return model_fn_lib.EstimatorSpec( - mode=mode, loss=constant_op.constant(_EXPECTED_LOSS)) - - features, labels = make_dummy_features_labels() - estimator_spec = eval_model_fn_no_eval_metric_ops( - features=features, labels=labels, mode=_EVAL, params={}) - with self.test_session() as sess: - self.assertEqual(sess.run(estimator_spec.loss), _EXPECTED_LOSS) - - def test_eval_no_eval_metrics(self): - - @xla.estimator_model_fn - def eval_model_fn_no_eval_metrics(features, labels, mode, params): - del features, labels, params - return tpu_estimator.TPUEstimatorSpec( - mode=mode, loss=constant_op.constant(_EXPECTED_LOSS)) - - features, labels = make_dummy_features_labels() - estimator_spec = eval_model_fn_no_eval_metrics( - features=features, labels=labels, mode=_EVAL, params={}) - - self.assertEqual(estimator_spec.eval_metric_ops, {}) - with self.test_session() as sess: - self.assertEqual(sess.run(estimator_spec.loss), _EXPECTED_LOSS) - - def test_eval_fn_missing_input_tensor(self): - - @xla.estimator_model_fn - def eval_model_fn(features, labels, mode, params): - del params - dummy_eval_metric_fn_tensors_dict = { - 'eval_tensor_1': features + labels, - } - return tpu_estimator.TPUEstimatorSpec( - mode=mode, - loss=constant_op.constant(_EXPECTED_LOSS), - eval_metrics=(_test_eval_metric_fn, - dummy_eval_metric_fn_tensors_dict)) - - with self.assertRaisesRegexp( - ValueError, - re.escape("Arguments ['eval_tensor_2'] are needed by metric_fn (first " - 'element of TPUEstimatorSpec.eval_metrics) but they are not ' - 'provided by evaluation tensors (second element of ' - 'TPUEstimatorSpec.eval_metrics).')): - self._verify_evaluation_result(eval_model_fn) - - def test_eval_fn_extraneous_input_tensor(self): - - @xla.estimator_model_fn - def eval_model_fn(features, labels, mode, params): - del params - dummy_eval_metric_fn_tensors_dict = { - 'eval_tensor_1': features + labels, - 'eval_tensor_2': features - labels, - 'extra_tensor': features * 2 - labels, - } - return tpu_estimator.TPUEstimatorSpec( - mode=mode, - loss=constant_op.constant(_EXPECTED_LOSS), - eval_metrics=(_test_eval_metric_fn, - dummy_eval_metric_fn_tensors_dict)) - - with self.assertRaisesRegexp( - ValueError, - re.escape("Arguments ['extra_tensor'] are provided by evaluation " - 'tensors (second element of TPUEstimatorSpec.eval_metrics) ' - 'but they are not needed by metric_fn (first element of ' - 'TPUEstimatorSpec.eval_metrics).')): - self._verify_evaluation_result(eval_model_fn) - - def test_eval_tensors_as_list(self): - - @xla.estimator_model_fn - def eval_model_fn(features, labels, mode, params): - del params - dummy_eval_metric_fn_tensors = [features + labels, features - labels] - return tpu_estimator.TPUEstimatorSpec( - mode=mode, - loss=constant_op.constant(_EXPECTED_LOSS), - eval_metrics=(_test_eval_metric_fn, dummy_eval_metric_fn_tensors)) - - self._verify_evaluation_result(eval_model_fn) - - def test_eval_tensors_as_dict(self): - - @xla.estimator_model_fn - def eval_model_fn(features, labels, mode, params): - del params - dummy_eval_metric_fn_tensors_dict = { - 'eval_tensor_1': features + labels, - 'eval_tensor_2': features - labels, - } - return tpu_estimator.TPUEstimatorSpec( - mode=mode, - loss=constant_op.constant(_EXPECTED_LOSS), - eval_metrics=(_test_eval_metric_fn, - dummy_eval_metric_fn_tensors_dict)) - - self._verify_evaluation_result(eval_model_fn) - - def test_model_with_summary(self): - """Tests that summary ops are disabled.""" - - @xla.estimator_model_fn - def model_fn_with_summary(features, labels, mode, params): - del features, labels, params - loss = constant_op.constant(_EXPECTED_LOSS) - summary.scalar('loss_scalar_summary', loss) - summary.histogram('loss_histogram_summary', loss) - summary.image('loss_image_summary', loss) - return tpu_estimator.TPUEstimatorSpec(mode=mode, loss=loss) - - features, labels = make_dummy_features_labels() - estimator_spec = model_fn_with_summary( - features=features, labels=labels, mode=_EVAL, params={}) - - with self.test_session() as sess: - self.assertEqual(sess.run(estimator_spec.loss), _EXPECTED_LOSS) - - -class XlaDecoratorScaffoldTest(test.TestCase, parameterized.TestCase): - - def _make_scaffold_fn(self, mode): - - def _scaffold_fn_on_cpu(): - scaffold = training.Scaffold() - self.assertNotIn(mode, self.is_scaffold_fn_called) - self.is_scaffold_fn_called[mode] = True - return scaffold - - return _scaffold_fn_on_cpu - - def test_scaffold_fn_return_none(self): - - @xla.estimator_model_fn - def model_fn(features, labels, mode, params): - del features, labels, params - return tpu_estimator.TPUEstimatorSpec( - mode=mode, - loss=constant_op.constant(_EXPECTED_LOSS), - train_op=control_flow_ops.no_op(), - scaffold_fn=lambda: None) - - features, labels = make_dummy_features_labels() - with self.assertRaisesRegexp( - ValueError, - 'TPUEstimatorSpec.scaffold_fn returns None, which is not allowed'): - model_fn(features=features, labels=labels, mode=_TRAIN, params={}) - - @parameterized.named_parameters( - ('train_mode', _TRAIN), - ('eval_mode', _EVAL), - # TODO(ycao): Add predict_mode test after PREDICT mode is implemented. - ) - def test_scaffold_fn_in_mode(self, mode): - - @xla.estimator_model_fn - def model_fn(features, labels, mode, params): - del features, labels, params - return tpu_estimator.TPUEstimatorSpec( - mode=mode, - loss=constant_op.constant(_EXPECTED_LOSS), - train_op=control_flow_ops.no_op(), - scaffold_fn=self._make_scaffold_fn(mode)) - - features, labels = make_dummy_features_labels() - - self.is_scaffold_fn_called = {} - model_fn(features=features, labels=labels, mode=mode, params={}) - self.assertTrue(self.is_scaffold_fn_called[mode]) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/constrained_optimization/BUILD b/tensorflow/contrib/constrained_optimization/BUILD deleted file mode 100644 index 0054bbbd78c..00000000000 --- a/tensorflow/contrib/constrained_optimization/BUILD +++ /dev/null @@ -1,99 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -# Transitive dependencies of this target will be included in the pip package. -py_library( - name = "constrained_optimization_pip", - deps = [ - ":constrained_optimization", - ":test_util", - ], -) - -py_library( - name = "constrained_optimization", - srcs = [ - "__init__.py", - "python/candidates.py", - "python/constrained_minimization_problem.py", - "python/constrained_optimizer.py", - "python/external_regret_optimizer.py", - "python/swap_regret_optimizer.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework", - "//tensorflow/python:standard_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_test( - name = "candidates_test", - srcs = ["python/candidates_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - # TODO(b/129496144): Re-enable MSAN test. - "nomsan", - ], - deps = [ - ":constrained_optimization", - "//tensorflow/python:client_testlib", - "//third_party/py/numpy", - ], -) - -# NOTE: This library can't be "testonly" since it needs to be included in the -# pip package. -py_library( - name = "test_util", - srcs = ["python/test_util.py"], - srcs_version = "PY2AND3", - deps = [ - ":constrained_optimization", - "//tensorflow/python:dtypes", - "//tensorflow/python:standard_ops", - ], -) - -py_test( - name = "external_regret_optimizer_test", - srcs = ["python/external_regret_optimizer_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":constrained_optimization", - ":test_util", - "//tensorflow/python:client_testlib", - "//tensorflow/python:standard_ops", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) - -py_test( - name = "swap_regret_optimizer_test", - srcs = ["python/swap_regret_optimizer_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":constrained_optimization", - ":test_util", - "//tensorflow/python:client_testlib", - "//tensorflow/python:standard_ops", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/constrained_optimization/README.md b/tensorflow/contrib/constrained_optimization/README.md deleted file mode 100644 index 7ffb6894d37..00000000000 --- a/tensorflow/contrib/constrained_optimization/README.md +++ /dev/null @@ -1,350 +0,0 @@ - - -**NOTE: As tensorflow.contrib is being -[deprecated](https://github.com/tensorflow/community/pull/18), TFCO is moving to -its own repository on -[github](https://github.com/google-research/tensorflow_constrained_optimization).** - -# ConstrainedOptimization (TFCO) - -TFCO is a library for optimizing inequality-constrained problems in TensorFlow. -Both the objective function and the constraints are represented as Tensors, -giving users the maximum amount of flexibility in specifying their optimization -problems. - -This flexibility makes optimization considerably more difficult: on a non-convex -problem, if one uses the "standard" approach of introducing a Lagrange -multiplier for each constraint, and then jointly maximizing over the Lagrange -multipliers and minimizing over the model parameters, then a stable stationary -point might not even *exist*. Hence, in some cases, oscillation, instead of -convergence, is inevitable. - -Thankfully, it turns out that even if, over the course of optimization, no -*particular* iterate does a good job of minimizing the objective while -satisfying the constraints, the *sequence* of iterates, on average, usually -will. This observation suggests the following approach: at training time, we'll -periodically snapshot the model state during optimization; then, at evaluation -time, each time we're given a new example to evaluate, we'll sample one of the -saved snapshots uniformly at random, and apply it to the example. This -*stochastic model* will generally perform well, both with respect to the -objective function, and the constraints. - -In fact, we can do better: it's possible to post-process the set of snapshots to -find a distribution over at most $$m+1$$ snapshots, where $$m$$ is the number of -constraints, that will be at least as good (and will usually be much better) -than the (much larger) uniform distribution described above. If you're unable or -unwilling to use a stochastic model at all, then you can instead use a heuristic -to choose the single best snapshot. - -For full details, motivation, and theoretical results on the approach taken by -this library, please refer to: - -> Cotter, Jiang and Sridharan. "Two-Player Games for Efficient Non-Convex -> Constrained Optimization". -> [https://arxiv.org/abs/1804.06500](https://arxiv.org/abs/1804.06500) - -which will be referred to as [CoJiSr18] throughout the remainder of this -document. - -### Proxy Constraints - -Imagine that we want to constrain the recall of a binary classifier to be at -least 90%. Since the recall is proportional to the number of true positive -classifications, which itself is a sum of indicator functions, this constraint -is non-differentiable, and therefore cannot be used in a problem that will be -optimized using a (stochastic) gradient-based algorithm. - -For this and similar problems, TFCO supports so-called *proxy constraints*, -which are (at least semi-differentiable) approximations of the original -constraints. For example, one could create a proxy recall function by replacing -the indicator functions with sigmoids. During optimization, each proxy -constraint function will be penalized, with the magnitude of the penalty being -chosen to satisfy the corresponding *original* (non-proxy) constraint. - -On a problem including proxy constraints—even a convex problem—the -Lagrangian approach discussed above isn't guaranteed to work. However, a -different algorithm, based on minimizing *swap regret*, does work. Aside from -this difference, the recommended procedure for optimizing a proxy-constrained -problem remains the same: periodically snapshot the model during optimization, -and then either find the best $$m+1$$-sized distribution, or heuristically -choose the single best snapshot. - -## Components - -* [constrained_minimization_problem](https://www.tensorflow.org/code/tensorflow/contrib/constrained_optimization/python/constrained_minimization_problem.py): - contains the `ConstrainedMinimizationProblem` interface. Your own - constrained optimization problems should be represented using - implementations of this interface. - -* [constrained_optimizer](https://www.tensorflow.org/code/tensorflow/contrib/constrained_optimization/python/constrained_optimizer.py): - contains the `ConstrainedOptimizer` interface, which is similar to (but - different from) `tf.train.Optimizer`, with the main difference being that - `ConstrainedOptimizer`s are given `ConstrainedMinimizationProblem`s to - optimize, and perform constrained optimization. - - * [external_regret_optimizer](https://www.tensorflow.org/code/tensorflow/contrib/constrained_optimization/python/external_regret_optimizer.py): - contains the `AdditiveExternalRegretOptimizer` implementation, which is - a `ConstrainedOptimizer` implementing the Lagrangian approach discussed - above (with additive updates to the Lagrange multipliers). You should - use this optimizer for problems *without* proxy constraints. It may also - work for problems with proxy constraints, but we recommend using a swap - regret optimizer, instead. - - This optimizer is most similar to Algorithm 3 in Appendix C.3 of - [CoJiSr18], and is discussed in Section 3. The two differences are that - it uses proxy constraints (if they're provided) in the update of the - model parameters, and uses `tf.train.Optimizer`s, instead of SGD, for - the "inner" updates. - - * [swap_regret_optimizer](https://www.tensorflow.org/code/tensorflow/contrib/constrained_optimization/python/swap_regret_optimizer.py): - contains the `AdditiveSwapRegretOptimizer` and - `MultiplicativeSwapRegretOptimizer` implementations, which are - `ConstrainedOptimizer`s implementing the swap-regret minimization - approach mentioned above (with additive or multiplicative updates, - respectively, to the parameters associated with the - constraints—these parameters are not Lagrange multipliers, but - play a similar role). You should use one of these optimizers (we suggest - `MultiplicativeSwapRegretOptimizer`) for problems *with* proxy - constraints. - - The `MultiplicativeSwapRegretOptimizer` is most similar to Algorithm 2 - in Section 4 of [CoJiSr18], with the difference being that it uses - `tf.train.Optimizer`s, instead of SGD, for the "inner" updates. The - `AdditiveSwapRegretOptimizer` differs further in that it performs - additive (instead of multiplicative) updates of the stochastic matrix. - -* [candidates](https://www.tensorflow.org/code/tensorflow/contrib/constrained_optimization/python/candidates.py): - contains two functions, `find_best_candidate_distribution` and - `find_best_candidate_index`. Both of these functions are given a set of - candidate solutions to a constrained optimization problem, from which the - former finds the best distribution over at most $$m+1$$ candidates, and the - latter heuristically finds the single best candidate. As discussed above, - the set of candidates will typically be model snapshots saved periodically - during optimization. Both of these functions require that scipy be - installed. - - The `find_best_candidate_distribution` function implements the approach - described in Lemma 3 of [CoJiSr18], while `find_best_candidate_index` - implements the heuristic used for hyperparameter search in the experiments - of Section 5.2. - -## Convex Example with Proxy Constraints - -This is a simple example of recall-constrained optimization on simulated data: -we will try to find a classifier that minimizes the average hinge loss while -constraining recall to be at least 90%. - -We'll start with the required imports—notice the definition of `tfco`: - -```python -import math -import numpy as np -import tensorflow as tf - -tfco = tf.contrib.constrained_optimization -``` - -We'll now create an implementation of the `ConstrainedMinimizationProblem` class -for this problem. The constructor takes three parameters: a Tensor containing -the classification labels (0 or 1) for every training example, another Tensor -containing the model's predictions on every training example (sometimes called -the "logits"), and the lower bound on recall that will be enforced using a -constraint. - -This implementation will contain both constraints *and* proxy constraints: the -former represents the constraint that the true recall (defined in terms of the -*number* of true positives) be at least `recall_lower_bound`, while the latter -represents the same constraint, but on a hinge approximation of the recall. - -```python -class ExampleProblem(tfco.ConstrainedMinimizationProblem): - - def __init__(self, labels, predictions, recall_lower_bound): - self._labels = labels - self._predictions = predictions - self._recall_lower_bound = recall_lower_bound - # The number of positively-labeled examples. - self._positive_count = tf.reduce_sum(self._labels) - - @property - def objective(self): - return tf.losses.hinge_loss(labels=self._labels, logits=self._predictions) - - @property - def constraints(self): - true_positives = self._labels * tf.to_float(self._predictions > 0) - true_positive_count = tf.reduce_sum(true_positives) - recall = true_positive_count / self._positive_count - # The constraint is (recall >= self._recall_lower_bound), which we convert - # to (self._recall_lower_bound - recall <= 0) because - # ConstrainedMinimizationProblems must always provide their constraints in - # the form (tensor <= 0). - # - # The result of this function should be a tensor, with each element being - # a quantity that is constrained to be nonpositive. We only have one - # constraint, so we return a one-element tensor. - return self._recall_lower_bound - recall - - @property - def proxy_constraints(self): - # Use 1 - hinge since we're SUBTRACTING recall in the constraint function, - # and we want the proxy constraint function to be convex. - true_positives = self._labels * tf.minimum(1.0, self._predictions) - true_positive_count = tf.reduce_sum(true_positives) - recall = true_positive_count / self._positive_count - # Please see the corresponding comment in the constraints property. - return self._recall_lower_bound - recall -``` - -We'll now create a simple simulated dataset by sampling 1000 random -10-dimensional feature vectors from a Gaussian, finding their labels using a -random "ground truth" linear model, and then adding noise by randomly flipping -200 labels. - -```python -# Create a simulated 10-dimensional training dataset consisting of 1000 labeled -# examples, of which 800 are labeled correctly and 200 are mislabeled. -num_examples = 1000 -num_mislabeled_examples = 200 -dimension = 10 -# We will constrain the recall to be at least 90%. -recall_lower_bound = 0.9 - -# Create random "ground truth" parameters to a linear model. -ground_truth_weights = np.random.normal(size=dimension) / math.sqrt(dimension) -ground_truth_threshold = 0 - -# Generate a random set of features for each example. -features = np.random.normal(size=(num_examples, dimension)).astype( - np.float32) / math.sqrt(dimension) -# Compute the labels from these features given the ground truth linear model. -labels = (np.matmul(features, ground_truth_weights) > - ground_truth_threshold).astype(np.float32) -# Add noise by randomly flipping num_mislabeled_examples labels. -mislabeled_indices = np.random.choice( - num_examples, num_mislabeled_examples, replace=False) -labels[mislabeled_indices] = 1 - labels[mislabeled_indices] -``` - -We're now ready to construct our model, and the corresponding optimization -problem. We'll use a linear model of the form $$f(x) = w^T x - t$$, where $$w$$ -is the `weights`, and $$t$$ is the `threshold`. The `problem` variable will hold -an instance of the `ExampleProblem` class we created earlier. - -```python -# Create variables containing the model parameters. -weights = tf.Variable(tf.zeros(dimension), dtype=tf.float32, name="weights") -threshold = tf.Variable(0.0, dtype=tf.float32, name="threshold") - -# Create the optimization problem. -constant_labels = tf.constant(labels, dtype=tf.float32) -constant_features = tf.constant(features, dtype=tf.float32) -predictions = tf.tensordot(constant_features, weights, axes=(1, 0)) - threshold -problem = ExampleProblem( - labels=constant_labels, - predictions=predictions, - recall_lower_bound=recall_lower_bound, -) -``` - -We're almost ready to train our model, but first we'll create a couple of -functions to measure its performance. We're interested in two quantities: the -average hinge loss (which we seek to minimize), and the recall (which we -constrain). - -```python -def average_hinge_loss(labels, predictions): - num_examples, = np.shape(labels) - signed_labels = (labels * 2) - 1 - total_hinge_loss = np.sum(np.maximum(0.0, 1.0 - signed_labels * predictions)) - return total_hinge_loss / num_examples - -def recall(labels, predictions): - positive_count = np.sum(labels) - true_positives = labels * (predictions > 0) - true_positive_count = np.sum(true_positives) - return true_positive_count / positive_count -``` - -As was mentioned earlier, external regret optimizers suffice for problems -without proxy constraints, but swap regret optimizers are recommended for -problems *with* proxy constraints. Since this problem contains proxy -constraints, we use the `MultiplicativeSwapRegretOptimizer`. - -For this problem, the constraint is fairly easy to satisfy, so we can use the -same "inner" optimizer (an `AdagradOptimizer` with a learning rate of 1) for -optimization of both the model parameters (`weights` and `threshold`), and the -internal parameters associated with the constraints (these are the analogues of -the Lagrange multipliers used by the `MultiplicativeSwapRegretOptimizer`). For -more difficult problems, it will often be necessary to use different optimizers, -with different learning rates (presumably found via a hyperparameter search): to -accomplish this, pass *both* the `optimizer` and `constraint_optimizer` -parameters to `MultiplicativeSwapRegretOptimizer`'s constructor. - -Since this is a convex problem (both the objective and proxy constraint -functions are convex), we can just take the last iterate. Periodic snapshotting, -and the use of the `find_best_candidate_distribution` or -`find_best_candidate_index` functions, is generally only necessary for -non-convex problems (and even then, it isn't *always* necessary). - -```python -with tf.Session() as session: - optimizer = tfco.MultiplicativeSwapRegretOptimizer( - optimizer=tf.train.AdagradOptimizer(learning_rate=1.0)) - train_op = optimizer.minimize(problem) - - session.run(tf.global_variables_initializer()) - for ii in xrange(1000): - session.run(train_op) - - trained_weights, trained_threshold = session.run((weights, threshold)) - -trained_predictions = np.matmul(features, trained_weights) - trained_threshold -print("Constrained average hinge loss = %f" % average_hinge_loss( - labels, trained_predictions)) -print("Constrained recall = %f" % recall(labels, trained_predictions)) -``` - -Running the above code gives the following output (due to the randomness of the -dataset, you'll get a different result when you run it): - -```none -Constrained average hinge loss = 0.710019 -Constrained recall = 0.899811 -``` - -As we hoped, the recall is extremely close to 90%—and, thanks to the use -of proxy constraints, this is the *true* recall, not a hinge approximation. - -For comparison, let's try optimizing the same problem *without* the recall -constraint: - -```python -with tf.Session() as session: - optimizer = tf.train.AdagradOptimizer(learning_rate=1.0) - # For optimizing the unconstrained problem, we just minimize the "objective" - # portion of the minimization problem. - train_op = optimizer.minimize(problem.objective) - - session.run(tf.global_variables_initializer()) - for ii in xrange(1000): - session.run(train_op) - - trained_weights, trained_threshold = session.run((weights, threshold)) - -trained_predictions = np.matmul(features, trained_weights) - trained_threshold -print("Unconstrained average hinge loss = %f" % average_hinge_loss( - labels, trained_predictions)) -print("Unconstrained recall = %f" % recall(labels, trained_predictions)) -``` - -This code gives the following output (again, you'll get a different answer, -since the dataset is random): - -```none -Unconstrained average hinge loss = 0.627271 -Unconstrained recall = 0.793951 -``` - -Because there is no constraint, the unconstrained problem does a better job of -minimizing the average hinge loss, but naturally doesn't approach 90% recall. diff --git a/tensorflow/contrib/constrained_optimization/__init__.py b/tensorflow/contrib/constrained_optimization/__init__.py deleted file mode 100644 index 1e49ba9f179..00000000000 --- a/tensorflow/contrib/constrained_optimization/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A library for performing constrained optimization in TensorFlow.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.constrained_optimization.python.candidates import * -from tensorflow.contrib.constrained_optimization.python.constrained_minimization_problem import * -from tensorflow.contrib.constrained_optimization.python.constrained_optimizer import * -from tensorflow.contrib.constrained_optimization.python.external_regret_optimizer import * -from tensorflow.contrib.constrained_optimization.python.swap_regret_optimizer import * -# pylint: enable=wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - "AdditiveExternalRegretOptimizer", - "AdditiveSwapRegretOptimizer", - "ConstrainedMinimizationProblem", - "ConstrainedOptimizer", - "find_best_candidate_distribution", - "find_best_candidate_index", - "MultiplicativeSwapRegretOptimizer", -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/constrained_optimization/python/candidates.py b/tensorflow/contrib/constrained_optimization/python/candidates.py deleted file mode 100644 index 66d7ebed74d..00000000000 --- a/tensorflow/contrib/constrained_optimization/python/candidates.py +++ /dev/null @@ -1,319 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Code for optimizing over a set of candidate solutions. - -The functions in this file deal with the constrained problem: - -> minimize f(w) -> s.t. g_i(w) <= 0 for all i in {0,1,...,m-1} - -Here, f(w) is the "objective function", and g_i(w) is the ith (of m) "constraint -function". Given the values of the objective and constraint functions for a set -of n "candidate solutions" {w_0,w_1,...,w_{n-1}} (for a total of n objective -function values, and n*m constraint function values), the -`find_best_candidate_distribution` function finds the best DISTRIBUTION over -these candidates, while `find_best_candidate_index' heuristically finds the -single best candidate. - -Both of these functions have dependencies on `scipy`, so if you want to call -them, then you must make sure that `scipy` is available. The imports are -performed inside the functions themselves, so if they're not actually called, -then `scipy` is not needed. - -For more specifics, please refer to: - -> Cotter, Jiang and Sridharan. "Two-Player Games for Efficient Non-Convex -> Constrained Optimization". -> [https://arxiv.org/abs/1804.06500](https://arxiv.org/abs/1804.06500) - -The `find_best_candidate_distribution` function implements the approach -described in Lemma 3, while `find_best_candidate_index` implements the heuristic -used for hyperparameter search in the experiments of Section 5.2. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from six.moves import xrange # pylint: disable=redefined-builtin - - -def _find_best_candidate_distribution_helper(objective_vector, - constraints_matrix, - maximum_violation=0.0): - """Finds a distribution minimizing an objective subject to constraints. - - This function deals with the constrained problem: - - > minimize f(w) - > s.t. g_i(w) <= 0 for all i in {0,1,...,m-1} - - Here, f(w) is the "objective function", and g_i(w) is the ith (of m) - "constraint function". Given a set of n "candidate solutions" - {w_0,w_1,...,w_{n-1}}, this function finds a distribution over these n - candidates that, in expectation, minimizes the objective while violating - the constraints by no more than `maximum_violation`. If no such distribution - exists, it returns an error (using Go-style error reporting). - - The `objective_vector` parameter should be a numpy array with shape (n,), for - which objective_vector[i] = f(w_i). Likewise, `constraints_matrix` should be a - numpy array with shape (m,n), for which constraints_matrix[i,j] = g_i(w_j). - - This function will return a distribution for which at most m+1 probabilities, - and often fewer, are nonzero. - - Args: - objective_vector: numpy array of shape (n,), where n is the number of - "candidate solutions". Contains the objective function values. - constraints_matrix: numpy array of shape (m,n), where m is the number of - constraints and n is the number of "candidate solutions". Contains the - constraint violation magnitudes. - maximum_violation: nonnegative float, the maximum amount by which any - constraint may be violated, in expectation. - - Returns: - A pair (`result`, `message`), exactly one of which is None. If `message` is - None, then the `result` contains the optimal distribution as a numpy array - of shape (n,). If `result` is None, then `message` contains an error - message. - - Raises: - ValueError: If `objective_vector` and `constraints_matrix` have inconsistent - shapes, or if `maximum_violation` is negative. - ImportError: If we're unable to import `scipy.optimize`. - """ - if maximum_violation < 0.0: - raise ValueError("maximum_violation must be nonnegative") - - mm, nn = np.shape(constraints_matrix) - if (nn,) != np.shape(objective_vector): - raise ValueError( - "objective_vector must have shape (n,), and constraints_matrix (m, n)," - " where n is the number of candidates, and m is the number of " - "constraints") - - # We import scipy inline, instead of at the top of the file, so that a scipy - # dependency is only introduced if either find_best_candidate_distribution() - # or find_best_candidate_index() are actually called. - import scipy.optimize # pylint: disable=g-import-not-at-top - - # Feasibility (within maximum_violation) constraints. - a_ub = constraints_matrix - b_ub = np.full((mm, 1), maximum_violation) - # Sum-to-one constraint. - a_eq = np.ones((1, nn)) - b_eq = np.ones((1, 1)) - # Nonnegativity constraints. - bounds = (0, None) - - result = scipy.optimize.linprog( - objective_vector, - A_ub=a_ub, - b_ub=b_ub, - A_eq=a_eq, - b_eq=b_eq, - bounds=bounds) - # Go-style error reporting. We don't raise on error, since - # find_best_candidate_distribution() needs to handle the failure case, and we - # shouldn't use exceptions as flow-control. - if not result.success: - return (None, result.message) - else: - return (result.x, None) - - -def find_best_candidate_distribution(objective_vector, - constraints_matrix, - epsilon=0.0): - """Finds a distribution minimizing an objective subject to constraints. - - This function deals with the constrained problem: - - > minimize f(w) - > s.t. g_i(w) <= 0 for all i in {0,1,...,m-1} - - Here, f(w) is the "objective function", and g_i(w) is the ith (of m) - "constraint function". Given a set of n "candidate solutions" - {w_0,w_1,...,w_{n-1}}, this function finds a distribution over these n - candidates that, in expectation, minimizes the objective while violating - the constraints by the smallest possible amount (with the amount being found - via bisection search). - - The `objective_vector` parameter should be a numpy array with shape (n,), for - which objective_vector[i] = f(w_i). Likewise, `constraints_matrix` should be a - numpy array with shape (m,n), for which constraints_matrix[i,j] = g_i(w_j). - - This function will return a distribution for which at most m+1 probabilities, - and often fewer, are nonzero. - - For more specifics, please refer to: - - > Cotter, Jiang and Sridharan. "Two-Player Games for Efficient Non-Convex - > Constrained Optimization". - > [https://arxiv.org/abs/1804.06500](https://arxiv.org/abs/1804.06500) - - This function implements the approach described in Lemma 3. - - Args: - objective_vector: numpy array of shape (n,), where n is the number of - "candidate solutions". Contains the objective function values. - constraints_matrix: numpy array of shape (m,n), where m is the number of - constraints and n is the number of "candidate solutions". Contains the - constraint violation magnitudes. - epsilon: nonnegative float, the threshold at which to terminate the binary - search while searching for the minimal expected constraint violation - magnitude. - - Returns: - The optimal distribution, as a numpy array of shape (n,). - - Raises: - ValueError: If `objective_vector` and `constraints_matrix` have inconsistent - shapes, or if `epsilon` is negative. - ImportError: If we're unable to import `scipy.optimize`. - """ - if epsilon < 0.0: - raise ValueError("epsilon must be nonnegative") - - # If there is a feasible solution (i.e. with maximum_violation=0), then that's - # what we'll return. - pp, _ = _find_best_candidate_distribution_helper(objective_vector, - constraints_matrix) - if pp is not None: - return pp - - # The bound is the minimum over all candidates, of the maximum per-candidate - # constraint violation. - lower = 0.0 - upper = np.min(np.amax(constraints_matrix, axis=0)) - best_pp, _ = _find_best_candidate_distribution_helper( - objective_vector, constraints_matrix, maximum_violation=upper) - assert best_pp is not None - - # Throughout this loop, a maximum_violation of "lower" is not achievable, - # but a maximum_violation of "upper" is achievable. - while True: - middle = 0.5 * (lower + upper) - if (middle - lower <= epsilon) or (upper - middle <= epsilon): - break - else: - pp, _ = _find_best_candidate_distribution_helper( - objective_vector, constraints_matrix, maximum_violation=middle) - if pp is None: - lower = middle - else: - best_pp = pp - upper = middle - - return best_pp - - -def find_best_candidate_index(objective_vector, - constraints_matrix, - rank_objectives=False): - """Heuristically finds the best candidate solution to a constrained problem. - - This function deals with the constrained problem: - - > minimize f(w) - > s.t. g_i(w) <= 0 for all i in {0,1,...,m-1} - - Here, f(w) is the "objective function", and g_i(w) is the ith (of m) - "constraint function". Given a set of n "candidate solutions" - {w_0,w_1,...,w_{n-1}}, this function finds the "best" solution according - to the following heuristic: - - 1. Across all models, the ith constraint violations (i.e. max{0, g_i(0)}) - are ranked, as are the objectives (if rank_objectives=True). - 2. Each model is then associated its MAXIMUM rank across all m constraints - (and the objective, if rank_objectives=True). - 3. The model with the minimal maximum rank is then identified. Ties are - broken using the objective function value. - 4. The index of this "best" model is returned. - - The `objective_vector` parameter should be a numpy array with shape (n,), for - which objective_vector[i] = f(w_i). Likewise, `constraints_matrix` should be a - numpy array with shape (m,n), for which constraints_matrix[i,j] = g_i(w_j). - - For more specifics, please refer to: - - > Cotter, Jiang and Sridharan. "Two-Player Games for Efficient Non-Convex - > Constrained Optimization". - > [https://arxiv.org/abs/1804.06500](https://arxiv.org/abs/1804.06500) - - This function implements the heuristic used for hyperparameter search in the - experiments of Section 5.2. - - Args: - objective_vector: numpy array of shape (n,), where n is the number of - "candidate solutions". Contains the objective function values. - constraints_matrix: numpy array of shape (m,n), where m is the number of - constraints and n is the number of "candidate solutions". Contains the - constraint violation magnitudes. - rank_objectives: bool, whether the objective function values should be - included in the initial ranking step. If True, both the objective and - constraints will be ranked. If False, only the constraints will be ranked. - In either case, the objective function values will be used for - tiebreaking. - - Returns: - The index (in {0,1,...,n-1}) of the "best" model according to the above - heuristic. - - Raises: - ValueError: If `objective_vector` and `constraints_matrix` have inconsistent - shapes. - ImportError: If we're unable to import `scipy.stats`. - """ - mm, nn = np.shape(constraints_matrix) - if (nn,) != np.shape(objective_vector): - raise ValueError( - "objective_vector must have shape (n,), and constraints_matrix (m, n)," - " where n is the number of candidates, and m is the number of " - "constraints") - - # We import scipy inline, instead of at the top of the file, so that a scipy - # dependency is only introduced if either find_best_candidate_distribution() - # or find_best_candidate_index() are actually called. - import scipy.stats # pylint: disable=g-import-not-at-top - - if rank_objectives: - maximum_ranks = scipy.stats.rankdata(objective_vector, method="min") - else: - maximum_ranks = np.zeros(nn, dtype=np.int64) - for ii in xrange(mm): - # Take the maximum of the constraint functions with zero, since we want to - # rank the magnitude of constraint *violations*. If the constraint is - # satisfied, then we don't care how much it's satisfied by (as a result, we - # we expect all models satisfying a constraint to be tied at rank 1). - ranks = scipy.stats.rankdata( - np.maximum(0.0, constraints_matrix[ii, :]), method="min") - maximum_ranks = np.maximum(maximum_ranks, ranks) - - best_index = None - best_rank = float("Inf") - best_objective = float("Inf") - for ii in xrange(nn): - if maximum_ranks[ii] < best_rank: - best_index = ii - best_rank = maximum_ranks[ii] - best_objective = objective_vector[ii] - elif (maximum_ranks[ii] == best_rank) and (objective_vector[ii] <= - best_objective): - best_index = ii - best_objective = objective_vector[ii] - - return best_index diff --git a/tensorflow/contrib/constrained_optimization/python/candidates_test.py b/tensorflow/contrib/constrained_optimization/python/candidates_test.py deleted file mode 100644 index 280e9acd886..00000000000 --- a/tensorflow/contrib/constrained_optimization/python/candidates_test.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for constrained_optimization.python.candidates.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.constrained_optimization.python import candidates -from tensorflow.python.platform import test - - -class CandidatesTest(test.TestCase): - - def test_inconsistent_shapes_for_best_distribution(self): - """An error is raised when parameters have inconsistent shapes.""" - objective_vector = np.array([1, 2, 3]) - constraints_matrix = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]) - with self.assertRaises(ValueError): - _ = candidates.find_best_candidate_distribution(objective_vector, - constraints_matrix) - - def test_inconsistent_shapes_for_best_index(self): - """An error is raised when parameters have inconsistent shapes.""" - objective_vector = np.array([1, 2, 3]) - constraints_matrix = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]) - with self.assertRaises(ValueError): - _ = candidates.find_best_candidate_index(objective_vector, - constraints_matrix) - - def test_best_distribution(self): - """Distribution should match known solution.""" - objective_vector = np.array( - [0.03053309, -0.06667082, 0.88355145, 0.46529806]) - constraints_matrix = np.array( - [[-0.60164551, 0.36676229, 0.7856454, -0.8441711], - [0.00371592, -0.16392108, -0.59778071, -0.56908492]]) - distribution = candidates.find_best_candidate_distribution( - objective_vector, constraints_matrix) - # Verify that the solution is a probability distribution. - self.assertTrue(np.all(distribution >= -1e-6)) - self.assertAlmostEqual(np.sum(distribution), 1.0) - # Verify that the solution satisfies the constraints. - maximum_constraint_violation = np.amax( - np.dot(constraints_matrix, distribution)) - self.assertLessEqual(maximum_constraint_violation, 1e-6) - # Verify that the solution matches that which we expect. - expected_distribution = np.array([0.37872711, 0.62127289, 0, 0]) - self.assertAllClose(expected_distribution, distribution, rtol=0, atol=1e-6) - - def test_best_index_rank_objectives_true(self): - """Index should match known solution.""" - # Objective ranks = [2, 1, 4, 3]. - objective_vector = np.array( - [0.03053309, -0.06667082, 0.88355145, 0.46529806]) - # Constraint ranks = [[1, 3, 4, 1], [4, 1, 1, 1]]. - constraints_matrix = np.array( - [[-0.60164551, 0.36676229, 0.7856454, -0.8441711], - [0.00371592, -0.16392108, -0.59778071, -0.56908492]]) - # Maximum ranks = [4, 3, 4, 3]. - index = candidates.find_best_candidate_index( - objective_vector, constraints_matrix, rank_objectives=True) - self.assertEqual(1, index) - - def test_best_index_rank_objectives_false(self): - """Index should match known solution.""" - # Objective ranks = [2, 1, 4, 3]. - objective_vector = np.array( - [0.03053309, -0.06667082, 0.88355145, 0.46529806]) - # Constraint ranks = [[1, 3, 4, 1], [4, 1, 1, 1]]. - constraints_matrix = np.array( - [[-0.60164551, 0.36676229, 0.7856454, -0.8441711], - [0.00371592, -0.16392108, -0.59778071, -0.56908492]]) - # Maximum ranks = [4, 3, 4, 1]. - index = candidates.find_best_candidate_index( - objective_vector, constraints_matrix, rank_objectives=False) - self.assertEqual(3, index) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/constrained_optimization/python/constrained_minimization_problem.py b/tensorflow/contrib/constrained_optimization/python/constrained_minimization_problem.py deleted file mode 100644 index 6926c0d03fe..00000000000 --- a/tensorflow/contrib/constrained_optimization/python/constrained_minimization_problem.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Defines abstract class for `ConstrainedMinimizationProblem`s. - -A ConstrainedMinimizationProblem consists of an objective function to minimize, -and a set of constraint functions that are constrained to be nonpositive. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - - -@six.add_metaclass(abc.ABCMeta) -class ConstrainedMinimizationProblem(object): - """Abstract class representing a `ConstrainedMinimizationProblem`. - - A ConstrainedMinimizationProblem consists of an objective function to - minimize, and a set of constraint functions that are constrained to be - nonpositive. - - In addition to the constraint functions, there may (optionally) be proxy - constraint functions: a ConstrainedOptimizer will attempt to penalize these - proxy constraint functions so as to satisfy the (non-proxy) constraints. Proxy - constraints could be used if the constraints functions are difficult or - impossible to optimize (e.g. if they're piecewise constant), in which case the - proxy constraints should be some approximation of the original constraints - that is well-enough behaved to permit successful optimization. - """ - - @abc.abstractproperty - def objective(self): - """Returns the objective function. - - Returns: - A 0d tensor that should be minimized. - """ - pass - - @property - def num_constraints(self): - """Returns the number of constraints. - - Returns: - An int containing the number of constraints. - - Raises: - ValueError: If the constraints (or proxy_constraints, if present) do not - have fully-known shapes, OR if proxy_constraints are present, and the - shapes of constraints and proxy_constraints are fully-known, but they're - different. - """ - constraints_shape = self.constraints.get_shape() - if self.proxy_constraints is None: - proxy_constraints_shape = constraints_shape - else: - proxy_constraints_shape = self.proxy_constraints.get_shape() - - if (constraints_shape.ndims is None or - proxy_constraints_shape.ndims is None or - any(ii is None for ii in constraints_shape.as_list()) or - any(ii is None for ii in proxy_constraints_shape.as_list())): - raise ValueError( - "constraints and proxy_constraints must have fully-known shapes") - if constraints_shape != proxy_constraints_shape: - raise ValueError( - "constraints and proxy_constraints must have the same shape") - - size = 1 - for ii in constraints_shape.as_list(): - size *= ii - return int(size) - - @abc.abstractproperty - def constraints(self): - """Returns the vector of constraint functions. - - Letting g_i be the ith element of the constraints vector, the ith constraint - will be g_i <= 0. - - Returns: - A tensor of constraint functions. - """ - pass - - # This is a property, instead of an abstract property, since it doesn't need - # to be overridden: if proxy_constraints returns None, then there are no - # proxy constraints. - @property - def proxy_constraints(self): - """Returns the optional vector of proxy constraint functions. - - The difference between `constraints` and `proxy_constraints` is that, when - proxy constraints are present, the `constraints` are merely EVALUATED during - optimization, whereas the `proxy_constraints` are DIFFERENTIATED. If there - are no proxy constraints, then the `constraints` are both evaluated and - differentiated. - - For example, if we want to impose constraints on step functions, then we - could use these functions for `constraints`. However, because a step - function has zero gradient almost everywhere, we can't differentiate these - functions, so we would take `proxy_constraints` to be some differentiable - approximation of `constraints`. - - Returns: - A tensor of proxy constraint functions. - """ - return None - - # This is a property, instead of an abstract property, since it doesn't need - # to be overridden: if pre_train_ops returns None, then there are no ops to - # run before train_op. - @property - def pre_train_ops(self): - """Returns a list of `Operation`s to run before the train_op. - - When a `ConstrainedOptimizer` creates a train_op (in `minimize` - `minimize_unconstrained`, or `minimize_constrained`), it will include these - ops before the main training step. - - Returns: - A list of `Operation`s. - """ - return None diff --git a/tensorflow/contrib/constrained_optimization/python/constrained_optimizer.py b/tensorflow/contrib/constrained_optimization/python/constrained_optimizer.py deleted file mode 100644 index 09249884423..00000000000 --- a/tensorflow/contrib/constrained_optimization/python/constrained_optimizer.py +++ /dev/null @@ -1,279 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Defines base class for `ConstrainedOptimizer`s.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import standard_ops -from tensorflow.python.training import optimizer as train_optimizer - - -@six.add_metaclass(abc.ABCMeta) -class ConstrainedOptimizer(object): - """Base class representing a constrained optimizer. - - A ConstrainedOptimizer wraps a tf.compat.v1.train.Optimizer (or more than - one), and applies it to a ConstrainedMinimizationProblem. Unlike a - tf.compat.v1.train.Optimizer, which takes a tensor to minimize as a parameter - to its minimize() method, a constrained optimizer instead takes a - ConstrainedMinimizationProblem. - """ - - def __init__(self, optimizer): - """Constructs a new `ConstrainedOptimizer`. - - Args: - optimizer: tf.compat.v1.train.Optimizer, used to optimize the - ConstraintedMinimizationProblem. - - Returns: - A new `ConstrainedOptimizer`. - """ - self._optimizer = optimizer - - @property - def optimizer(self): - """Returns the `tf.compat.v1.train.Optimizer` used for optimization.""" - return self._optimizer - - @abc.abstractmethod - def _minimize_constrained(self, - minimization_problem, - global_step=None, - var_list=None, - gate_gradients=train_optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - name=None, - grad_loss=None): - """Version of `minimize_constrained` to be overridden by subclasses. - - Implementations of this method should ignore the `pre_train_ops` property of - the `minimization_problem`. The public `minimize_constrained` method will - take care of executing these before the returned train_op. - - Args: - minimization_problem: ConstrainedMinimizationProblem, the problem to - optimize. - global_step: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - var_list: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - gate_gradients: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - aggregation_method: as in `tf.compat.v1.train.Optimizer`'s `minimize` - method. - colocate_gradients_with_ops: as in `tf.compat.v1.train.Optimizer`'s - `minimize` method. - name: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - grad_loss: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - - Returns: - `Operation`, the train_op. - """ - pass - - def minimize_constrained(self, - minimization_problem, - global_step=None, - var_list=None, - gate_gradients=train_optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - name=None, - grad_loss=None): - """Returns an `Operation` for minimizing the constrained problem. - - Unlike `minimize_unconstrained`, this function attempts to find a solution - that minimizes the `objective` portion of the minimization problem while - satisfying the `constraints` portion. - - Args: - minimization_problem: ConstrainedMinimizationProblem, the problem to - optimize. - global_step: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - var_list: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - gate_gradients: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - aggregation_method: as in `tf.compat.v1.train.Optimizer`'s `minimize` - method. - colocate_gradients_with_ops: as in `tf.compat.v1.train.Optimizer`'s - `minimize` method. - name: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - grad_loss: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - - Returns: - `Operation`, the train_op. - """ - - def train_op_callback(): - return self._minimize_constrained( - minimization_problem, - global_step=global_step, - var_list=var_list, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, - name=name, - grad_loss=grad_loss) - - # If we have pre_train_ops, use tf.control_dependencies() to ensure that - # they execute before the train_op. - pre_train_ops = minimization_problem.pre_train_ops - if pre_train_ops: - with ops.control_dependencies(pre_train_ops): - train_op = train_op_callback() - else: - train_op = train_op_callback() - - return train_op - - def minimize_unconstrained(self, - minimization_problem, - global_step=None, - var_list=None, - gate_gradients=train_optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - name=None, - grad_loss=None): - """Returns an `Operation` for minimizing the unconstrained problem. - - Unlike `minimize_constrained`, this function ignores the `constraints` (and - `proxy_constraints`) portion of the minimization problem entirely, and only - minimizes `objective`. - - Args: - minimization_problem: ConstrainedMinimizationProblem, the problem to - optimize. - global_step: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - var_list: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - gate_gradients: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - aggregation_method: as in `tf.compat.v1.train.Optimizer`'s `minimize` - method. - colocate_gradients_with_ops: as in `tf.compat.v1.train.Optimizer`'s - `minimize` method. - name: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - grad_loss: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - - Returns: - `Operation`, the train_op. - """ - - def train_op_callback(): - return self.optimizer.minimize( - minimization_problem.objective, - global_step=global_step, - var_list=var_list, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, - name=name, - grad_loss=grad_loss) - - # If we have pre_train_ops, use tf.control_dependencies() to ensure that - # they execute before the train_op. - pre_train_ops = minimization_problem.pre_train_ops - if pre_train_ops: - with ops.control_dependencies(pre_train_ops): - train_op = train_op_callback() - else: - train_op = train_op_callback() - - return train_op - - def minimize(self, - minimization_problem, - unconstrained_steps=None, - global_step=None, - var_list=None, - gate_gradients=train_optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - name=None, - grad_loss=None): - """Returns an `Operation` for minimizing the constrained problem. - - This method combines the functionality of `minimize_unconstrained` and - `minimize_constrained`. If global_step < unconstrained_steps, it will - perform an unconstrained update, and if global_step >= unconstrained_steps, - it will perform a constrained update. - - The reason for this functionality is that it may be best to initialize the - constrained optimizer with an approximate optimum of the unconstrained - problem. - - Args: - minimization_problem: ConstrainedMinimizationProblem, the problem to - optimize. - unconstrained_steps: int, number of steps for which we should perform - unconstrained updates, before transitioning to constrained updates. - global_step: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - var_list: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - gate_gradients: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - aggregation_method: as in `tf.compat.v1.train.Optimizer`'s `minimize` - method. - colocate_gradients_with_ops: as in `tf.compat.v1.train.Optimizer`'s - `minimize` method. - name: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - grad_loss: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - - Returns: - `Operation`, the train_op. - - Raises: - ValueError: If unconstrained_steps is provided, but global_step is not. - """ - - def unconstrained_fn(): - """Returns an `Operation` for minimizing the unconstrained problem.""" - return self.minimize_unconstrained( - minimization_problem=minimization_problem, - global_step=global_step, - var_list=var_list, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, - name=name, - grad_loss=grad_loss) - - def constrained_fn(): - """Returns an `Operation` for minimizing the constrained problem.""" - return self.minimize_constrained( - minimization_problem=minimization_problem, - global_step=global_step, - var_list=var_list, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, - name=name, - grad_loss=grad_loss) - - if unconstrained_steps is not None: - if global_step is None: - raise ValueError( - "global_step cannot be None if unconstrained_steps is provided") - unconstrained_steps_tensor = ops.convert_to_tensor(unconstrained_steps) - dtype = unconstrained_steps_tensor.dtype - return control_flow_ops.cond( - standard_ops.cast(global_step, dtype) < unconstrained_steps_tensor, - true_fn=unconstrained_fn, - false_fn=constrained_fn) - else: - return constrained_fn() diff --git a/tensorflow/contrib/constrained_optimization/python/external_regret_optimizer.py b/tensorflow/contrib/constrained_optimization/python/external_regret_optimizer.py deleted file mode 100644 index 0cfe354bce0..00000000000 --- a/tensorflow/contrib/constrained_optimization/python/external_regret_optimizer.py +++ /dev/null @@ -1,393 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Defines `AdditiveExternalRegretOptimizer`. - -This optimizer minimizes a `ConstrainedMinimizationProblem` by introducing -Lagrange multipliers, and using `tf.compat.v1.train.Optimizer`s to jointly -optimize over -the model parameters and Lagrange multipliers. - -For the purposes of constrained optimization, at least in theory, -external-regret minimization suffices if the `ConstrainedMinimizationProblem` -we're optimizing doesn't have any `proxy_constraints`, while swap-regret -minimization should be used if `proxy_constraints` are present. - -For more specifics, please refer to: - -> Cotter, Jiang and Sridharan. "Two-Player Games for Efficient Non-Convex -> Constrained Optimization". -> [https://arxiv.org/abs/1804.06500](https://arxiv.org/abs/1804.06500) - -The formulation used by the AdditiveExternalRegretOptimizer--which is simply the -usual Lagrangian formulation--can be found in Definition 1, and is discussed in -Section 3. This optimizer is most similar to Algorithm 3 in Appendix C.3, with -the two differences being that it uses proxy constraints (if they're provided) -in the update of the model parameters, and uses `tf.compat.v1.train.Optimizer`s, -instead of SGD, for the "inner" updates. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - -from tensorflow.contrib.constrained_optimization.python import constrained_optimizer - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import standard_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.training import optimizer as train_optimizer - - -def _project_multipliers_wrt_euclidean_norm(multipliers, radius): - """Projects its argument onto the feasible region. - - The feasible region is the set of all vectors with nonnegative elements that - sum to at most `radius`. - - Args: - multipliers: 1d tensor, the Lagrange multipliers to project. - radius: float, the radius of the feasible region. - - Returns: - The 1d tensor that results from projecting `multipliers` onto the feasible - region w.r.t. the Euclidean norm. - - Raises: - ValueError: if the `multipliers` tensor is not floating-point, does not have - a fully-known shape, or is not one-dimensional. - """ - if not multipliers.dtype.is_floating: - raise ValueError("multipliers must have a floating-point dtype") - multipliers_shape = multipliers.get_shape() - if multipliers_shape.ndims is None: - raise ValueError("multipliers must have known shape") - if multipliers_shape.ndims != 1: - raise ValueError( - "multipliers must be one dimensional (instead is %d-dimensional)" % - multipliers_shape.ndims) - dimension = multipliers_shape.dims[0].value - if dimension is None: - raise ValueError("multipliers must have fully-known shape") - - def while_loop_condition(iteration, multipliers, inactive, old_inactive): - """Returns false if the while loop should terminate.""" - del multipliers # Needed by the body, but not the condition. - not_done = (iteration < dimension) - not_converged = standard_ops.reduce_any( - standard_ops.not_equal(inactive, old_inactive)) - return standard_ops.logical_and(not_done, not_converged) - - def while_loop_body(iteration, multipliers, inactive, old_inactive): - """Performs one iteration of the projection.""" - del old_inactive # Needed by the condition, but not the body. - iteration += 1 - scale = standard_ops.minimum( - 0.0, (radius - standard_ops.reduce_sum(multipliers)) / - standard_ops.maximum(1.0, standard_ops.reduce_sum(inactive))) - multipliers = multipliers + (scale * inactive) - new_inactive = standard_ops.cast(multipliers > 0, multipliers.dtype) - multipliers = multipliers * new_inactive - return (iteration, multipliers, new_inactive, inactive) - - iteration = standard_ops.constant(0) - inactive = standard_ops.ones_like(multipliers, dtype=multipliers.dtype) - - # We actually want a do-while loop, so we explicitly call while_loop_body() - # once before tf.while_loop(). - iteration, multipliers, inactive, old_inactive = while_loop_body( - iteration, multipliers, inactive, inactive) - iteration, multipliers, inactive, old_inactive = control_flow_ops.while_loop( - while_loop_condition, - while_loop_body, - loop_vars=(iteration, multipliers, inactive, old_inactive), - name="euclidean_projection") - - return multipliers - - -@six.add_metaclass(abc.ABCMeta) -class _ExternalRegretOptimizer(constrained_optimizer.ConstrainedOptimizer): - """Base class representing an `_ExternalRegretOptimizer`. - - This class contains most of the logic for performing constrained - optimization, minimizing external regret for the constraints player. What it - *doesn't* do is keep track of the internal state (the Lagrange multipliers). - Instead, the state is accessed via the _initial_state(), - _lagrange_multipliers(), _constraint_grad_and_var() and _projection_op() - methods. - - The reason for this is that we want to make it easy to implement different - representations of the internal state. - - For more specifics, please refer to: - - > Cotter, Jiang and Sridharan. "Two-Player Games for Efficient Non-Convex - > Constrained Optimization". - > [https://arxiv.org/abs/1804.06500](https://arxiv.org/abs/1804.06500) - - The formulation used by `_ExternalRegretOptimizer`s--which is simply the usual - Lagrangian formulation--can be found in Definition 1, and is discussed in - Section 3. Such optimizers are most similar to Algorithm 3 in Appendix C.3. - """ - - def __init__(self, optimizer, constraint_optimizer=None): - """Constructs a new `_ExternalRegretOptimizer`. - - The difference between `optimizer` and `constraint_optimizer` (if the latter - is provided) is that the former is used for learning the model parameters, - while the latter us used for the Lagrange multipliers. If no - `constraint_optimizer` is provided, then `optimizer` is used for both. - - Args: - optimizer: tf.compat.v1.train.Optimizer, used to optimize the objective - and proxy_constraints portion of the ConstrainedMinimizationProblem. If - constraint_optimizer is not provided, this will also be used to optimize - the Lagrange multipliers. - constraint_optimizer: optional tf.compat.v1.train.Optimizer, used to - optimize the Lagrange multipliers. - - Returns: - A new `_ExternalRegretOptimizer`. - """ - super(_ExternalRegretOptimizer, self).__init__(optimizer=optimizer) - self._constraint_optimizer = constraint_optimizer - - @property - def constraint_optimizer(self): - """Returns the `tf.compat.v1.train.Optimizer` used for the Lagrange multipliers.""" - return self._constraint_optimizer - - @abc.abstractmethod - def _initial_state(self, num_constraints): - pass - - @abc.abstractmethod - def _lagrange_multipliers(self, state): - pass - - @abc.abstractmethod - def _constraint_grad_and_var(self, state, gradient): - pass - - @abc.abstractmethod - def _projection_op(self, state, name=None): - pass - - def _minimize_constrained(self, - minimization_problem, - global_step=None, - var_list=None, - gate_gradients=train_optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - name=None, - grad_loss=None): - """Returns an `Operation` for minimizing the constrained problem. - - The `optimizer` constructor parameter will be used to update the model - parameters, while the Lagrange multipliers will be updated using - `constrained_optimizer` (if provided) or `optimizer` (if not). - - Args: - minimization_problem: ConstrainedMinimizationProblem, the problem to - optimize. - global_step: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - var_list: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - gate_gradients: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - aggregation_method: as in `tf.compat.v1.train.Optimizer`'s `minimize` - method. - colocate_gradients_with_ops: as in `tf.compat.v1.train.Optimizer`'s - `minimize` method. - name: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - grad_loss: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - - Raises: - ValueError: If the minimization_problem tensors have different dtypes. - - Returns: - `Operation`, the train_op. - """ - objective = minimization_problem.objective - - constraints = minimization_problem.constraints - proxy_constraints = minimization_problem.proxy_constraints - if proxy_constraints is None: - proxy_constraints = constraints - - # Make sure that the objective, constraints and proxy constraints all have - # the same dtype. - if (objective.dtype.base_dtype != constraints.dtype.base_dtype or - objective.dtype.base_dtype != proxy_constraints.dtype.base_dtype): - raise ValueError("objective, constraints and proxy_constraints must " - "have the same dtype") - - # Flatten both constraints tensors to 1d. - num_constraints = minimization_problem.num_constraints - constraints = standard_ops.reshape(constraints, shape=(num_constraints,)) - proxy_constraints = standard_ops.reshape( - proxy_constraints, shape=(num_constraints,)) - - # We use a lambda to initialize the state so that, if this function call is - # inside the scope of a tf.control_dependencies() block, the dependencies - # will not be applied to the initializer. - state = standard_ops.Variable( - lambda: self._initial_state(num_constraints), - trainable=False, - name="external_regret_optimizer_state") - - multipliers = self._lagrange_multipliers(state) - loss = ( - objective + standard_ops.tensordot( - standard_ops.cast(multipliers, proxy_constraints.dtype), - proxy_constraints, 1)) - multipliers_gradient = standard_ops.cast(constraints, multipliers.dtype) - - update_ops = [] - if self.constraint_optimizer is None: - # If we don't have a separate constraint_optimizer, then we use - # self._optimizer for both the update of the model parameters, and that of - # the internal state. - grads_and_vars = self.optimizer.compute_gradients( - loss, - var_list=var_list, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, - grad_loss=grad_loss) - grads_and_vars.append( - self._constraint_grad_and_var(state, multipliers_gradient)) - update_ops.append( - self.optimizer.apply_gradients(grads_and_vars, name="update")) - else: - # If we have a separate constraint_optimizer, then we use self._optimizer - # for the update of the model parameters, and self._constraint_optimizer - # for that of the internal state. - grads_and_vars = self.optimizer.compute_gradients( - loss, - var_list=var_list, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, - grad_loss=grad_loss) - multiplier_grads_and_vars = [ - self._constraint_grad_and_var(state, multipliers_gradient) - ] - - gradients = [ - gradient for gradient, _ in grads_and_vars + multiplier_grads_and_vars - if gradient is not None - ] - with ops.control_dependencies(gradients): - update_ops.append( - self.optimizer.apply_gradients(grads_and_vars, name="update")) - update_ops.append( - self.constraint_optimizer.apply_gradients( - multiplier_grads_and_vars, name="optimizer_state_update")) - - with ops.control_dependencies(update_ops): - if global_step is None: - # If we don't have a global step, just project, and we're done. - return self._projection_op(state, name=name) - else: - # If we have a global step, then we need to increment it in addition to - # projecting. - projection_op = self._projection_op(state, name="project") - with ops.colocate_with(global_step): - global_step_op = state_ops.assign_add( - global_step, 1, name="global_step_increment") - return control_flow_ops.group(projection_op, global_step_op, name=name) - - -class AdditiveExternalRegretOptimizer(_ExternalRegretOptimizer): - """A `ConstrainedOptimizer` based on external-regret minimization. - - This `ConstrainedOptimizer` uses the given `tf.compat.v1.train.Optimizer`s to - jointly minimize over the model parameters, and maximize over Lagrange - multipliers, with the latter maximization using additive updates and an - algorithm that minimizes external regret. - - For more specifics, please refer to: - - > Cotter, Jiang and Sridharan. "Two-Player Games for Efficient Non-Convex - > Constrained Optimization". - > [https://arxiv.org/abs/1804.06500](https://arxiv.org/abs/1804.06500) - - The formulation used by this optimizer--which is simply the usual Lagrangian - formulation--can be found in Definition 1, and is discussed in Section 3. It - is most similar to Algorithm 3 in Appendix C.3, with the two differences being - that it uses proxy constraints (if they're provided) in the update of the - model parameters, and uses `tf.compat.v1.train.Optimizer`s, instead of SGD, - for the "inner" updates. - """ - - def __init__(self, - optimizer, - constraint_optimizer=None, - maximum_multiplier_radius=None): - """Constructs a new `AdditiveExternalRegretOptimizer`. - - Args: - optimizer: tf.compat.v1.train.Optimizer, used to optimize the objective - and proxy_constraints portion of ConstrainedMinimizationProblem. If - constraint_optimizer is not provided, this will also be used to optimize - the Lagrange multipliers. - constraint_optimizer: optional tf.compat.v1.train.Optimizer, used to - optimize the Lagrange multipliers. - maximum_multiplier_radius: float, an optional upper bound to impose on the - sum of the Lagrange multipliers. - - Returns: - A new `AdditiveExternalRegretOptimizer`. - - Raises: - ValueError: If the maximum_multiplier_radius parameter is nonpositive. - """ - super(AdditiveExternalRegretOptimizer, self).__init__( - optimizer=optimizer, constraint_optimizer=constraint_optimizer) - - if maximum_multiplier_radius and (maximum_multiplier_radius <= 0.0): - raise ValueError("maximum_multiplier_radius must be strictly positive") - - self._maximum_multiplier_radius = maximum_multiplier_radius - - def _initial_state(self, num_constraints): - # For an AdditiveExternalRegretOptimizer, the internal state is simply a - # tensor of Lagrange multipliers with shape (m,), where m is the number of - # constraints. - # - # FUTURE WORK: make the dtype a parameter. - return standard_ops.zeros((num_constraints,), dtype=dtypes.float32) - - def _lagrange_multipliers(self, state): - return state - - def _constraint_grad_and_var(self, state, gradient): - # TODO(acotter): v1.colocate_with(), if colocate_gradients_with_ops is True? - return (-gradient, state) - - def _projection_op(self, state, name=None): - with ops.colocate_with(state): - if self._maximum_multiplier_radius: - projected_multipliers = _project_multipliers_wrt_euclidean_norm( - state, self._maximum_multiplier_radius) - else: - projected_multipliers = standard_ops.maximum(state, 0.0) - return state_ops.assign(state, projected_multipliers, name=name) diff --git a/tensorflow/contrib/constrained_optimization/python/external_regret_optimizer_test.py b/tensorflow/contrib/constrained_optimization/python/external_regret_optimizer_test.py deleted file mode 100644 index 3e25079e02e..00000000000 --- a/tensorflow/contrib/constrained_optimization/python/external_regret_optimizer_test.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for constrained_optimization.python.external_regret_optimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.constrained_optimization.python import external_regret_optimizer -from tensorflow.contrib.constrained_optimization.python import test_util - -from tensorflow.python.ops import standard_ops -from tensorflow.python.platform import test -from tensorflow.python.training import gradient_descent - - -class AdditiveExternalRegretOptimizerWrapper( - external_regret_optimizer.AdditiveExternalRegretOptimizer): - """Testing wrapper class around AdditiveExternalRegretOptimizer. - - This class is identical to AdditiveExternalRegretOptimizer, except that it - caches the internal optimization state when _lagrange_multipliers() is called, - so that we can test that the Lagrange multipliers take on their expected - values. - """ - - def __init__(self, - optimizer, - constraint_optimizer=None, - maximum_multiplier_radius=None): - """Same as AdditiveExternalRegretOptimizer.__init__.""" - super(AdditiveExternalRegretOptimizerWrapper, self).__init__( - optimizer=optimizer, - constraint_optimizer=constraint_optimizer, - maximum_multiplier_radius=maximum_multiplier_radius) - self._cached_lagrange_multipliers = None - - @property - def lagrange_multipliers(self): - """Returns the cached Lagrange multipliers.""" - return self._cached_lagrange_multipliers - - def _lagrange_multipliers(self, state): - """Caches the internal state for testing.""" - self._cached_lagrange_multipliers = super( - AdditiveExternalRegretOptimizerWrapper, - self)._lagrange_multipliers(state) - return self._cached_lagrange_multipliers - - -class ExternalRegretOptimizerTest(test.TestCase): - - def test_project_multipliers_wrt_euclidean_norm(self): - """Tests Euclidean projection routine on some known values.""" - multipliers1 = standard_ops.constant([-0.1, -0.6, -0.3]) - expected_projected_multipliers1 = np.array([0.0, 0.0, 0.0]) - - multipliers2 = standard_ops.constant([-0.1, 0.6, 0.3]) - expected_projected_multipliers2 = np.array([0.0, 0.6, 0.3]) - - multipliers3 = standard_ops.constant([0.4, 0.7, -0.2, 0.5, 0.1]) - expected_projected_multipliers3 = np.array([0.2, 0.5, 0.0, 0.3, 0.0]) - - with self.cached_session() as session: - projected_multipliers1 = session.run( - external_regret_optimizer._project_multipliers_wrt_euclidean_norm( - multipliers1, 1.0)) - projected_multipliers2 = session.run( - external_regret_optimizer._project_multipliers_wrt_euclidean_norm( - multipliers2, 1.0)) - projected_multipliers3 = session.run( - external_regret_optimizer._project_multipliers_wrt_euclidean_norm( - multipliers3, 1.0)) - - self.assertAllClose( - expected_projected_multipliers1, - projected_multipliers1, - rtol=0, - atol=1e-6) - self.assertAllClose( - expected_projected_multipliers2, - projected_multipliers2, - rtol=0, - atol=1e-6) - self.assertAllClose( - expected_projected_multipliers3, - projected_multipliers3, - rtol=0, - atol=1e-6) - - def test_additive_external_regret_optimizer(self): - """Tests that the Lagrange multipliers update as expected.""" - minimization_problem = test_util.ConstantMinimizationProblem( - np.array([0.6, -0.1, 0.4])) - optimizer = AdditiveExternalRegretOptimizerWrapper( - gradient_descent.GradientDescentOptimizer(1.0), - maximum_multiplier_radius=1.0) - train_op = optimizer.minimize_constrained(minimization_problem) - - expected_multipliers = [ - np.array([0.0, 0.0, 0.0]), - np.array([0.6, 0.0, 0.4]), - np.array([0.7, 0.0, 0.3]), - np.array([0.8, 0.0, 0.2]), - np.array([0.9, 0.0, 0.1]), - np.array([1.0, 0.0, 0.0]), - np.array([1.0, 0.0, 0.0]), - ] - - multipliers = [] - with self.cached_session() as session: - session.run(standard_ops.global_variables_initializer()) - while len(multipliers) < len(expected_multipliers): - multipliers.append(session.run(optimizer.lagrange_multipliers)) - session.run(train_op) - - for expected, actual in zip(expected_multipliers, multipliers): - self.assertAllClose(expected, actual, rtol=0, atol=1e-6) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/constrained_optimization/python/swap_regret_optimizer.py b/tensorflow/contrib/constrained_optimization/python/swap_regret_optimizer.py deleted file mode 100644 index fb02106f025..00000000000 --- a/tensorflow/contrib/constrained_optimization/python/swap_regret_optimizer.py +++ /dev/null @@ -1,619 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Defines `{Additive,Multiplicative}SwapRegretOptimizer`s. - -These optimizers minimize a `ConstrainedMinimizationProblem` by using a -swap-regret minimizing algorithm (either SGD or multiplicative weights) to learn -what weights should be associated with the objective function and constraints. -These algorithms do *not* use Lagrange multipliers, but the idea is similar. -The main differences between the formulation used here, and the standard -Lagrangian formulation, are that (i) the objective function is weighted, in -addition to the constraints, and (ii) we learn a matrix of weights, instead of a -vector. - -For the purposes of constrained optimization, at least in theory, -external-regret minimization suffices if the `ConstrainedMinimizationProblem` -we're optimizing doesn't have any `proxy_constraints`, while swap-regret -minimization should be used if `proxy_constraints` are present. - -For more specifics, please refer to: - -> Cotter, Jiang and Sridharan. "Two-Player Games for Efficient Non-Convex -> Constrained Optimization". -> [https://arxiv.org/abs/1804.06500](https://arxiv.org/abs/1804.06500) - -The formulation used by both of the SwapRegretOptimizers can be found in -Definition 2, and is discussed in Section 4. The -`MultiplicativeSwapRegretOptimizer` is most similar to Algorithm 2 in Section 4, -with the difference being that it uses `tf.compat.v1.train.Optimizer`s, instead -of SGD, -for the "inner" updates. The `AdditiveSwapRegretOptimizer` differs further in -that it performs additive (instead of multiplicative) updates of the stochastic -matrix. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc -import math - -import six - -from tensorflow.contrib.constrained_optimization.python import constrained_optimizer - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import standard_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.training import optimizer as train_optimizer - - -def _maximal_eigenvector_power_method(matrix, - epsilon=1e-6, - maximum_iterations=100): - """Returns the maximal right-eigenvector of `matrix` using the power method. - - Args: - matrix: 2D Tensor, the matrix of which we will find the maximal - right-eigenvector. - epsilon: nonnegative float, if two iterations of the power method differ (in - L2 norm) by no more than epsilon, we will terminate. - maximum_iterations: nonnegative int, if we perform this many iterations, we - will terminate. - Result: The maximal right-eigenvector of `matrix`. - - Raises: - ValueError: If the `matrix` tensor is not floating-point, or if the - `epsilon` or `maximum_iterations` parameters violate their bounds. - """ - if not matrix.dtype.is_floating: - raise ValueError("multipliers must have a floating-point dtype") - if epsilon <= 0.0: - raise ValueError("epsilon must be strictly positive") - if maximum_iterations <= 0: - raise ValueError("maximum_iterations must be strictly positive") - - def while_loop_condition(iteration, eigenvector, old_eigenvector): - """Returns false if the while loop should terminate.""" - not_done = (iteration < maximum_iterations) - not_converged = (standard_ops.norm(eigenvector - old_eigenvector) > epsilon) - return standard_ops.logical_and(not_done, not_converged) - - def while_loop_body(iteration, eigenvector, old_eigenvector): - """Performs one iteration of the power method.""" - del old_eigenvector # Needed by the condition, but not the body. - iteration += 1 - # We need to use tf.matmul() and tf.expand_dims(), instead of - # tf.tensordot(), since the former will infer the shape of the result, while - # the latter will not (tf.while_loop() needs the shapes). - new_eigenvector = standard_ops.matmul( - matrix, standard_ops.expand_dims(eigenvector, 1))[:, 0] - new_eigenvector /= standard_ops.norm(new_eigenvector) - return (iteration, new_eigenvector, eigenvector) - - iteration = standard_ops.constant(0) - eigenvector = standard_ops.ones_like(matrix[:, 0]) - eigenvector /= standard_ops.norm(eigenvector) - - # We actually want a do-while loop, so we explicitly call while_loop_body() - # once before tf.while_loop(). - iteration, eigenvector, old_eigenvector = while_loop_body( - iteration, eigenvector, eigenvector) - iteration, eigenvector, old_eigenvector = control_flow_ops.while_loop( - while_loop_condition, - while_loop_body, - loop_vars=(iteration, eigenvector, old_eigenvector), - name="power_method") - - return eigenvector - - -def _project_stochastic_matrix_wrt_euclidean_norm(matrix): - """Projects its argument onto the set of left-stochastic matrices. - - This algorithm is O(n^3) at worst, where `matrix` is n*n. It can be done in - O(n^2 * log(n)) time by sorting each column (and maybe better with a different - algorithm), but the algorithm implemented here is easier to implement in - TensorFlow. - - Args: - matrix: 2d square tensor, the matrix to project. - - Returns: - The 2d square tensor that results from projecting `matrix` onto the set of - left-stochastic matrices w.r.t. the Euclidean norm applied column-wise - (i.e. the Frobenius norm). - - Raises: - ValueError: if the `matrix` tensor is not floating-point, does not have a - fully-known shape, or is not two-dimensional and square. - """ - if not matrix.dtype.is_floating: - raise ValueError("multipliers must have a floating-point dtype") - matrix_shape = matrix.get_shape() - if matrix_shape.ndims is None: - raise ValueError("matrix must have known shape") - if matrix_shape.ndims != 2: - raise ValueError( - "matrix must be two dimensional (instead is %d-dimensional)" % - matrix_shape.ndims) - if matrix_shape[0] != matrix_shape[1]: - raise ValueError("matrix must be square (instead has shape (%d,%d))" % - (matrix_shape[0], matrix_shape[1])) - dimension = matrix_shape.dims[0].value - if dimension is None: - raise ValueError("matrix must have fully-known shape") - - def while_loop_condition(iteration, matrix, inactive, old_inactive): - """Returns false if the while loop should terminate.""" - del matrix # Needed by the body, but not the condition. - not_done = (iteration < dimension) - not_converged = standard_ops.reduce_any( - standard_ops.not_equal(inactive, old_inactive)) - return standard_ops.logical_and(not_done, not_converged) - - def while_loop_body(iteration, matrix, inactive, old_inactive): - """Performs one iteration of the projection.""" - del old_inactive # Needed by the condition, but not the body. - iteration += 1 - scale = (1.0 - standard_ops.reduce_sum( - matrix, axis=0, keepdims=True)) / standard_ops.maximum( - 1.0, standard_ops.reduce_sum(inactive, axis=0, keepdims=True)) - matrix = matrix + (scale * inactive) - new_inactive = standard_ops.cast(matrix > 0, matrix.dtype) - matrix = matrix * new_inactive - return (iteration, matrix, new_inactive, inactive) - - iteration = standard_ops.constant(0) - inactive = standard_ops.ones_like(matrix, dtype=matrix.dtype) - - # We actually want a do-while loop, so we explicitly call while_loop_body() - # once before tf.while_loop(). - iteration, matrix, inactive, old_inactive = while_loop_body( - iteration, matrix, inactive, inactive) - iteration, matrix, inactive, old_inactive = control_flow_ops.while_loop( - while_loop_condition, - while_loop_body, - loop_vars=(iteration, matrix, inactive, old_inactive), - name="euclidean_projection") - - return matrix - - -def _project_log_stochastic_matrix_wrt_kl_divergence(log_matrix): - """Projects its argument onto the set of log-left-stochastic matrices. - - Args: - log_matrix: 2d square tensor, the element-wise logarithm of the matrix to - project. - - Returns: - The 2d square tensor that results from projecting exp(`matrix`) onto the set - of left-stochastic matrices w.r.t. the KL-divergence applied column-wise. - """ - - # For numerical reasons, make sure that the largest matrix element is zero - # before exponentiating. - log_matrix = log_matrix - standard_ops.reduce_max( - log_matrix, axis=0, keepdims=True) - log_matrix = log_matrix - standard_ops.log( - standard_ops.reduce_sum( - standard_ops.exp(log_matrix), axis=0, keepdims=True)) - return log_matrix - - -@six.add_metaclass(abc.ABCMeta) -class _SwapRegretOptimizer(constrained_optimizer.ConstrainedOptimizer): - """Base class representing a `_SwapRegretOptimizer`. - - This class contains most of the logic for performing constrained optimization, - minimizing swap regret for the constraints player. What it *doesn't* do is - keep track of the internal state (the stochastic matrix). Instead, the state - is accessed via the _initial_state(), _stochastic_matrix(), - _constraint_grad_and_var() and _projection_op() methods. - - The reason for this is that we want to make it easy to implement different - representations of the internal state. For example, for additive updates, it's - most natural to store the stochastic matrix directly, whereas for - multiplicative updates, it's most natural to store its element-wise logarithm. - - For more specifics, please refer to: - - > Cotter, Jiang and Sridharan. "Two-Player Games for Efficient Non-Convex - > Constrained Optimization". - > [https://arxiv.org/abs/1804.06500](https://arxiv.org/abs/1804.06500) - - The formulation used by `_SwapRegretOptimizer`s can be found in Definition 2, - and is discussed in Section 4. Such optimizers are most similar to Algorithm - 2 in Section 4. Most notably, the internal state is a left-stochastic matrix - of shape (m+1,m+1), where m is the number of constraints. - """ - - def __init__(self, optimizer, constraint_optimizer=None): - """Constructs a new `_SwapRegretOptimizer`. - - The difference between `optimizer` and `constraint_optimizer` (if the latter - is provided) is that the former is used for learning the model parameters, - while the latter us used for the update to the constraint/objective weight - matrix (the analogue of Lagrange multipliers). If no `constraint_optimizer` - is provided, then `optimizer` is used for both. - - Args: - optimizer: tf.compat.v1.train.Optimizer, used to optimize the objective - and proxy_constraints portion of ConstrainedMinimizationProblem. If - constraint_optimizer is not provided, this will also be used to optimize - the Lagrange multiplier analogues. - constraint_optimizer: optional tf.compat.v1.train.Optimizer, used to - optimize the Lagrange multiplier analogues. - - Returns: - A new `_SwapRegretOptimizer`. - """ - super(_SwapRegretOptimizer, self).__init__(optimizer=optimizer) - self._constraint_optimizer = constraint_optimizer - - @property - def constraint_optimizer(self): - """Returns the `tf.compat.v1.train.Optimizer` used for the matrix.""" - return self._constraint_optimizer - - @abc.abstractmethod - def _initial_state(self, num_constraints): - pass - - @abc.abstractmethod - def _stochastic_matrix(self, state): - pass - - def _distribution(self, state): - distribution = _maximal_eigenvector_power_method( - self._stochastic_matrix(state)) - distribution = standard_ops.abs(distribution) - distribution /= standard_ops.reduce_sum(distribution) - return distribution - - @abc.abstractmethod - def _constraint_grad_and_var(self, state, gradient): - pass - - @abc.abstractmethod - def _projection_op(self, state, name=None): - pass - - def _minimize_constrained(self, - minimization_problem, - global_step=None, - var_list=None, - gate_gradients=train_optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - name=None, - grad_loss=None): - """Returns an `Operation` for minimizing the constrained problem. - - The `optimizer` constructor parameter will be used to update the model - parameters, while the constraint/objective weight matrix (the analogue of - Lagrange multipliers) will be updated using `constrained_optimizer` (if - provided) or `optimizer` (if not). Whether the matrix updates are additive - or multiplicative depends on the derived class. - - Args: - minimization_problem: ConstrainedMinimizationProblem, the problem to - optimize. - global_step: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - var_list: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - gate_gradients: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - aggregation_method: as in `tf.compat.v1.train.Optimizer`'s `minimize` - method. - colocate_gradients_with_ops: as in `tf.compat.v1.train.Optimizer`'s - `minimize` method. - name: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - grad_loss: as in `tf.compat.v1.train.Optimizer`'s `minimize` method. - - Raises: - ValueError: If the minimization_problem tensors have different dtypes. - - Returns: - `Operation`, the train_op. - """ - objective = minimization_problem.objective - - constraints = minimization_problem.constraints - proxy_constraints = minimization_problem.proxy_constraints - if proxy_constraints is None: - proxy_constraints = constraints - - # Make sure that the objective, constraints and proxy constraints all have - # the same dtype. - if (objective.dtype.base_dtype != constraints.dtype.base_dtype or - objective.dtype.base_dtype != proxy_constraints.dtype.base_dtype): - raise ValueError("objective, constraints and proxy_constraints must " - "have the same dtype") - - # Flatten both constraints tensors to 1d. - num_constraints = minimization_problem.num_constraints - constraints = standard_ops.reshape(constraints, shape=(num_constraints,)) - proxy_constraints = standard_ops.reshape( - proxy_constraints, shape=(num_constraints,)) - - # We use a lambda to initialize the state so that, if this function call is - # inside the scope of a tf.control_dependencies() block, the dependencies - # will not be applied to the initializer. - state = standard_ops.Variable( - lambda: self._initial_state(num_constraints), - trainable=False, - name="swap_regret_optimizer_state") - - zero_and_constraints = standard_ops.concat((standard_ops.zeros( - (1,), dtype=constraints.dtype), constraints), - axis=0) - objective_and_proxy_constraints = standard_ops.concat( - (standard_ops.expand_dims(objective, 0), proxy_constraints), axis=0) - - distribution = self._distribution(state) - loss = standard_ops.tensordot( - standard_ops.cast(distribution, objective_and_proxy_constraints.dtype), - objective_and_proxy_constraints, 1) - matrix_gradient = standard_ops.matmul( - standard_ops.expand_dims( - standard_ops.cast(zero_and_constraints, distribution.dtype), 1), - standard_ops.expand_dims(distribution, 0)) - - update_ops = [] - if self.constraint_optimizer is None: - # If we don't have a separate constraint_optimizer, then we use - # self._optimizer for both the update of the model parameters, and that of - # the internal state. - grads_and_vars = self.optimizer.compute_gradients( - loss, - var_list=var_list, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, - grad_loss=grad_loss) - grads_and_vars.append( - self._constraint_grad_and_var(state, matrix_gradient)) - update_ops.append( - self.optimizer.apply_gradients(grads_and_vars, name="update")) - else: - # If we have a separate constraint_optimizer, then we use self._optimizer - # for the update of the model parameters, and self._constraint_optimizer - # for that of the internal state. - grads_and_vars = self.optimizer.compute_gradients( - loss, - var_list=var_list, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, - grad_loss=grad_loss) - matrix_grads_and_vars = [ - self._constraint_grad_and_var(state, matrix_gradient) - ] - - gradients = [ - gradient for gradient, _ in grads_and_vars + matrix_grads_and_vars - if gradient is not None - ] - with ops.control_dependencies(gradients): - update_ops.append( - self.optimizer.apply_gradients(grads_and_vars, name="update")) - update_ops.append( - self.constraint_optimizer.apply_gradients( - matrix_grads_and_vars, name="optimizer_state_update")) - - with ops.control_dependencies(update_ops): - if global_step is None: - # If we don't have a global step, just project, and we're done. - return self._projection_op(state, name=name) - else: - # If we have a global step, then we need to increment it in addition to - # projecting. - projection_op = self._projection_op(state, name="project") - with ops.colocate_with(global_step): - global_step_op = state_ops.assign_add( - global_step, 1, name="global_step_increment") - return control_flow_ops.group(projection_op, global_step_op, name=name) - - -class AdditiveSwapRegretOptimizer(_SwapRegretOptimizer): - """A `ConstrainedOptimizer` based on swap-regret minimization. - - This `ConstrainedOptimizer` uses the given `tf.compat.v1.train.Optimizer`s to - jointly - minimize over the model parameters, and maximize over constraint/objective - weight matrix (the analogue of Lagrange multipliers), with the latter - maximization using additive updates and an algorithm that minimizes swap - regret. - - For more specifics, please refer to: - - > Cotter, Jiang and Sridharan. "Two-Player Games for Efficient Non-Convex - > Constrained Optimization". - > [https://arxiv.org/abs/1804.06500](https://arxiv.org/abs/1804.06500) - - The formulation used by this optimizer can be found in Definition 2, and is - discussed in Section 4. It is most similar to Algorithm 2 in Section 4, with - the differences being that it uses `tf.compat.v1.train.Optimizer`s, instead of - SGD, for - the "inner" updates, and performs additive (instead of multiplicative) updates - of the stochastic matrix. - """ - - def __init__(self, optimizer, constraint_optimizer=None): - """Constructs a new `AdditiveSwapRegretOptimizer`. - - Args: - optimizer: tf.compat.v1.train.Optimizer, used to optimize the objective - and proxy_constraints portion of ConstrainedMinimizationProblem. If - constraint_optimizer is not provided, this will also be used to optimize - the Lagrange multiplier analogues. - constraint_optimizer: optional tf.compat.v1.train.Optimizer, used to - optimize the Lagrange multiplier analogues. - - Returns: - A new `AdditiveSwapRegretOptimizer`. - """ - # TODO(acotter): add a parameter determining the initial values of the - # matrix elements (like initial_multiplier_radius in - # MultiplicativeSwapRegretOptimizer). - super(AdditiveSwapRegretOptimizer, self).__init__( - optimizer=optimizer, constraint_optimizer=constraint_optimizer) - - def _initial_state(self, num_constraints): - # For an AdditiveSwapRegretOptimizer, the internal state is a tensor of - # shape (m+1,m+1), where m is the number of constraints, representing a - # left-stochastic matrix. - dimension = num_constraints + 1 - # Initialize by putting all weight on the objective, and none on the - # constraints. - return standard_ops.concat((standard_ops.ones( - (1, dimension)), standard_ops.zeros((dimension - 1, dimension))), - axis=0) - - def _stochastic_matrix(self, state): - return state - - def _constraint_grad_and_var(self, state, gradient): - # TODO(acotter): tf.compat.v1.colocate_with(), - # if colocate_gradients_with_ops is True? - return (-gradient, state) - - def _projection_op(self, state, name=None): - with ops.colocate_with(state): - return state_ops.assign( - state, - _project_stochastic_matrix_wrt_euclidean_norm(state), - name=name) - - -class MultiplicativeSwapRegretOptimizer(_SwapRegretOptimizer): - """A `ConstrainedOptimizer` based on swap-regret minimization. - - This `ConstrainedOptimizer` uses the given `tf.compat.v1.train.Optimizer`s to - jointly - minimize over the model parameters, and maximize over constraint/objective - weight matrix (the analogue of Lagrange multipliers), with the latter - maximization using multiplicative updates and an algorithm that minimizes swap - regret. - - For more specifics, please refer to: - - > Cotter, Jiang and Sridharan. "Two-Player Games for Efficient Non-Convex - > Constrained Optimization". - > [https://arxiv.org/abs/1804.06500](https://arxiv.org/abs/1804.06500) - - The formulation used by this optimizer can be found in Definition 2, and is - discussed in Section 4. It is most similar to Algorithm 2 in Section 4, with - the difference being that it uses `tf.compat.v1.train.Optimizer`s, instead of - SGD, for - the "inner" updates. - """ - - def __init__(self, - optimizer, - constraint_optimizer=None, - minimum_multiplier_radius=1e-3, - initial_multiplier_radius=None): - """Constructs a new `MultiplicativeSwapRegretOptimizer`. - - Args: - optimizer: tf.compat.v1.train.Optimizer, used to optimize the objective - and proxy_constraints portion of ConstrainedMinimizationProblem. If - constraint_optimizer is not provided, this will also be used to optimize - the Lagrange multiplier analogues. - constraint_optimizer: optional tf.compat.v1.train.Optimizer, used to - optimize the Lagrange multiplier analogues. - minimum_multiplier_radius: float, each element of the matrix will be lower - bounded by `minimum_multiplier_radius` divided by one plus the number of - constraints. - initial_multiplier_radius: float, the initial value of each element of the - matrix associated with a constraint (i.e. excluding those elements - associated with the objective) will be `initial_multiplier_radius` - divided by one plus the number of constraints. Defaults to the value of - `minimum_multiplier_radius`. - - Returns: - A new `MultiplicativeSwapRegretOptimizer`. - - Raises: - ValueError: If the two radius parameters are inconsistent. - """ - super(MultiplicativeSwapRegretOptimizer, self).__init__( - optimizer=optimizer, constraint_optimizer=constraint_optimizer) - - if (minimum_multiplier_radius <= 0.0) or (minimum_multiplier_radius >= 1.0): - raise ValueError("minimum_multiplier_radius must be in the range (0,1)") - if initial_multiplier_radius is None: - initial_multiplier_radius = minimum_multiplier_radius - elif (initial_multiplier_radius < - minimum_multiplier_radius) or (minimum_multiplier_radius > 1.0): - raise ValueError("initial_multiplier_radius must be in the range " - "[minimum_multiplier_radius,1]") - - self._minimum_multiplier_radius = minimum_multiplier_radius - self._initial_multiplier_radius = initial_multiplier_radius - - def _initial_state(self, num_constraints): - # For a MultiplicativeSwapRegretOptimizer, the internal state is a tensor of - # shape (m+1,m+1), where m is the number of constraints, representing the - # element-wise logarithm of a left-stochastic matrix. - dimension = num_constraints + 1 - # Initialize by putting as much weight as possible on the objective, and as - # little as possible on the constraints. - log_initial_one = math.log(1.0 - (self._initial_multiplier_radius * - (dimension - 1) / (dimension))) - log_initial_zero = math.log(self._initial_multiplier_radius / dimension) - # FUTURE WORK: make the dtype a parameter. - return standard_ops.concat((standard_ops.constant( - log_initial_one, dtype=dtypes.float32, shape=(1, dimension)), - standard_ops.constant( - log_initial_zero, - dtype=dtypes.float32, - shape=(dimension - 1, dimension))), - axis=0) - - def _stochastic_matrix(self, state): - return standard_ops.exp(state) - - def _constraint_grad_and_var(self, state, gradient): - # TODO(acotter): tf.compat.v1.colocate_with(), - # if colocate_gradients_with_ops is True? - return (-gradient, state) - - def _projection_op(self, state, name=None): - with ops.colocate_with(state): - # Gets the dimension of the state (num_constraints + 1)--all of these - # assertions are of things that should be impossible, since the state - # passed into this method will have the same shape as that returned by - # _initial_state(). - state_shape = state.get_shape() - assert state_shape is not None - assert state_shape.ndims == 2 - assert state_shape[0] == state_shape[1] - dimension = state_shape.dims[0].value - assert dimension is not None - - minimum_log_multiplier = standard_ops.log( - self._minimum_multiplier_radius / standard_ops.to_float(dimension)) - - return state_ops.assign( - state, - standard_ops.maximum( - _project_log_stochastic_matrix_wrt_kl_divergence(state), - minimum_log_multiplier), - name=name) diff --git a/tensorflow/contrib/constrained_optimization/python/swap_regret_optimizer_test.py b/tensorflow/contrib/constrained_optimization/python/swap_regret_optimizer_test.py deleted file mode 100644 index df0eced6317..00000000000 --- a/tensorflow/contrib/constrained_optimization/python/swap_regret_optimizer_test.py +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for constrained_optimization.python.swap_regret_optimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.constrained_optimization.python import swap_regret_optimizer -from tensorflow.contrib.constrained_optimization.python import test_util - -from tensorflow.python.ops import standard_ops -from tensorflow.python.platform import test -from tensorflow.python.training import gradient_descent - - -class AdditiveSwapRegretOptimizerWrapper( - swap_regret_optimizer.AdditiveSwapRegretOptimizer): - """Testing wrapper class around AdditiveSwapRegretOptimizer. - - This class is identical to AdditiveSwapRegretOptimizer, except that it caches - the internal optimization state when _stochastic_matrix() is called, so that - we can test that the stochastic matrices take on their expected values. - """ - - def __init__(self, optimizer, constraint_optimizer=None): - """Same as AdditiveSwapRegretOptimizer.__init__().""" - super(AdditiveSwapRegretOptimizerWrapper, self).__init__( - optimizer=optimizer, constraint_optimizer=constraint_optimizer) - self._cached_stochastic_matrix = None - - @property - def stochastic_matrix(self): - """Returns the cached stochastic matrix.""" - return self._cached_stochastic_matrix - - def _stochastic_matrix(self, state): - """Caches the internal state for testing.""" - self._cached_stochastic_matrix = super(AdditiveSwapRegretOptimizerWrapper, - self)._stochastic_matrix(state) - return self._cached_stochastic_matrix - - -class MultiplicativeSwapRegretOptimizerWrapper( - swap_regret_optimizer.MultiplicativeSwapRegretOptimizer): - """Testing wrapper class around MultiplicativeSwapRegretOptimizer. - - This class is identical to MultiplicativeSwapRegretOptimizer, except that it - caches the internal optimization state when _stochastic_matrix() is called, so - that we can test that the stochastic matrices take on their expected values. - """ - - def __init__(self, - optimizer, - constraint_optimizer=None, - minimum_multiplier_radius=None, - initial_multiplier_radius=None): - """Same as MultiplicativeSwapRegretOptimizer.__init__().""" - super(MultiplicativeSwapRegretOptimizerWrapper, self).__init__( - optimizer=optimizer, - constraint_optimizer=constraint_optimizer, - minimum_multiplier_radius=1e-3, - initial_multiplier_radius=initial_multiplier_radius) - self._cached_stochastic_matrix = None - - @property - def stochastic_matrix(self): - """Returns the cached stochastic matrix.""" - return self._cached_stochastic_matrix - - def _stochastic_matrix(self, state): - """Caches the internal state for testing.""" - self._cached_stochastic_matrix = super( - MultiplicativeSwapRegretOptimizerWrapper, - self)._stochastic_matrix(state) - return self._cached_stochastic_matrix - - -class SwapRegretOptimizerTest(test.TestCase): - - def test_maximum_eigenvector_power_method(self): - """Tests power method routine on some known left-stochastic matrices.""" - matrix1 = np.matrix([[0.6, 0.1, 0.1], [0.0, 0.6, 0.9], [0.4, 0.3, 0.0]]) - matrix2 = np.matrix([[0.4, 0.4, 0.2], [0.2, 0.1, 0.5], [0.4, 0.5, 0.3]]) - - with self.cached_session() as session: - eigenvector1 = session.run( - swap_regret_optimizer._maximal_eigenvector_power_method( - standard_ops.constant(matrix1))) - eigenvector2 = session.run( - swap_regret_optimizer._maximal_eigenvector_power_method( - standard_ops.constant(matrix2))) - - # Check that eigenvector1 and eigenvector2 are eigenvectors of matrix1 and - # matrix2 (respectively) with associated eigenvalue 1. - matrix_eigenvector1 = np.tensordot(matrix1, eigenvector1, axes=1) - matrix_eigenvector2 = np.tensordot(matrix2, eigenvector2, axes=1) - self.assertAllClose(eigenvector1, matrix_eigenvector1, rtol=0, atol=1e-6) - self.assertAllClose(eigenvector2, matrix_eigenvector2, rtol=0, atol=1e-6) - - def test_project_stochastic_matrix_wrt_euclidean_norm(self): - """Tests Euclidean projection routine on some known values.""" - matrix = standard_ops.constant([[-0.1, -0.1, 0.4], [-0.8, 0.4, 1.2], - [-0.3, 0.1, 0.2]]) - expected_projected_matrix = np.array([[0.6, 0.1, 0.1], [0.0, 0.6, 0.9], - [0.4, 0.3, 0.0]]) - - with self.cached_session() as session: - projected_matrix = session.run( - swap_regret_optimizer._project_stochastic_matrix_wrt_euclidean_norm( - matrix)) - - self.assertAllClose( - expected_projected_matrix, projected_matrix, rtol=0, atol=1e-6) - - def test_project_log_stochastic_matrix_wrt_kl_divergence(self): - """Tests KL-divergence projection routine on some known values.""" - matrix = standard_ops.constant([[0.2, 0.8, 0.6], [0.1, 0.2, 1.5], - [0.2, 1.0, 0.9]]) - expected_projected_matrix = np.array([[0.4, 0.4, 0.2], [0.2, 0.1, 0.5], - [0.4, 0.5, 0.3]]) - - with self.cached_session() as session: - projected_matrix = session.run( - standard_ops.exp( - swap_regret_optimizer. - _project_log_stochastic_matrix_wrt_kl_divergence( - standard_ops.log(matrix)))) - - self.assertAllClose( - expected_projected_matrix, projected_matrix, rtol=0, atol=1e-6) - - def test_additive_swap_regret_optimizer(self): - """Tests that the stochastic matrices update as expected.""" - minimization_problem = test_util.ConstantMinimizationProblem( - np.array([0.6, -0.1, 0.4])) - optimizer = AdditiveSwapRegretOptimizerWrapper( - gradient_descent.GradientDescentOptimizer(1.0)) - train_op = optimizer.minimize_constrained(minimization_problem) - - # Calculated using a numpy+python implementation of the algorithm. - expected_matrices = [ - np.array([[1.0, 1.0, 1.0, 1.0], [0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]), - np.array([[0.66666667, 1.0, 1.0, 1.0], [0.26666667, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0], [0.06666667, 0.0, 0.0, 0.0]]), - np.array([[0.41666667, 0.93333333, 1.0, - 0.98333333], [0.46666667, 0.05333333, 0.0, - 0.01333333], [0.0, 0.0, 0.0, 0.0], - [0.11666667, 0.01333333, 0.0, 0.00333333]]), - ] - - matrices = [] - with self.cached_session() as session: - session.run(standard_ops.global_variables_initializer()) - while len(matrices) < len(expected_matrices): - matrices.append(session.run(optimizer.stochastic_matrix)) - session.run(train_op) - - for expected, actual in zip(expected_matrices, matrices): - self.assertAllClose(expected, actual, rtol=0, atol=1e-6) - - def test_multiplicative_swap_regret_optimizer(self): - """Tests that the stochastic matrices update as expected.""" - minimization_problem = test_util.ConstantMinimizationProblem( - np.array([0.6, -0.1, 0.4])) - optimizer = MultiplicativeSwapRegretOptimizerWrapper( - gradient_descent.GradientDescentOptimizer(1.0), - initial_multiplier_radius=0.8) - train_op = optimizer.minimize_constrained(minimization_problem) - - # Calculated using a numpy+python implementation of the algorithm. - expected_matrices = [ - np.array([[0.4, 0.4, 0.4, 0.4], [0.2, 0.2, 0.2, 0.2], - [0.2, 0.2, 0.2, 0.2], [0.2, 0.2, 0.2, 0.2]]), - np.array([[0.36999014, 0.38528351, 0.38528351, 0.38528351], [ - 0.23517483, 0.21720297, 0.21720297, 0.21720297 - ], [0.17774131, 0.18882719, 0.18882719, 0.18882719], - [0.21709373, 0.20868632, 0.20868632, 0.20868632]]), - np.array([[0.33972109, 0.36811863, 0.37118462, 0.36906575], [ - 0.27114826, 0.23738228, 0.23376693, 0.23626491 - ], [0.15712313, 0.17641793, 0.17858959, 0.17708679], - [0.23200752, 0.21808115, 0.21645886, 0.21758255]]), - ] - - matrices = [] - with self.cached_session() as session: - session.run(standard_ops.global_variables_initializer()) - while len(matrices) < len(expected_matrices): - matrices.append(session.run(optimizer.stochastic_matrix)) - session.run(train_op) - - for expected, actual in zip(expected_matrices, matrices): - self.assertAllClose(expected, actual, rtol=0, atol=1e-6) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/constrained_optimization/python/test_util.py b/tensorflow/contrib/constrained_optimization/python/test_util.py deleted file mode 100644 index 704b36ca4c9..00000000000 --- a/tensorflow/contrib/constrained_optimization/python/test_util.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains helpers used by tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.constrained_optimization.python import constrained_minimization_problem - -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import standard_ops - - -class ConstantMinimizationProblem( - constrained_minimization_problem.ConstrainedMinimizationProblem): - """A `ConstrainedMinimizationProblem` with constant constraint violations. - - This minimization problem is intended for use in performing simple tests of - the Lagrange multiplier (or equivalent) update in the optimizers. There is a - one-element "dummy" model parameter, but it should be ignored. - """ - - def __init__(self, constraints): - """Constructs a new `ConstantMinimizationProblem'. - - Args: - constraints: 1d numpy array, the constant constraint violations. - - Returns: - A new `ConstantMinimizationProblem'. - """ - # We make an fake 1-parameter linear objective so that we don't get a "no - # variables to optimize" error. - self._objective = standard_ops.Variable(0.0, dtype=dtypes.float32) - self._constraints = standard_ops.constant(constraints, dtype=dtypes.float32) - - @property - def objective(self): - """Returns the objective function.""" - return self._objective - - @property - def constraints(self): - """Returns the constant constraint violations.""" - return self._constraints diff --git a/tensorflow/contrib/copy_graph/BUILD b/tensorflow/contrib/copy_graph/BUILD deleted file mode 100644 index 1b94638be7c..00000000000 --- a/tensorflow/contrib/copy_graph/BUILD +++ /dev/null @@ -1,45 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "copy_graph_py", - srcs = [ - "__init__.py", - "python/util/__init__.py", - "python/util/copy_elements.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:session", - "//tensorflow/python:util", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "copy_test", - srcs = glob(["python/util/copy_test.py"]), - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":copy_graph_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:session", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/copy_graph/__init__.py b/tensorflow/contrib/copy_graph/__init__.py deleted file mode 100644 index 61ee39e4be1..00000000000 --- a/tensorflow/contrib/copy_graph/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functions to copy elements between graphs. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.copy_graph.python.util import copy_elements -# pylint: disable=wildcard-import -from tensorflow.contrib.copy_graph.python.util.copy_elements import * -# pylint: enable=wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented - -remove_undocumented(__name__, doc_string_modules=[copy_elements]) diff --git a/tensorflow/contrib/copy_graph/python/__init__.py b/tensorflow/contrib/copy_graph/python/__init__.py deleted file mode 100644 index 5c1048e02a3..00000000000 --- a/tensorflow/contrib/copy_graph/python/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functions for copying elements from one graph to another. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/copy_graph/python/util/__init__.py b/tensorflow/contrib/copy_graph/python/util/__init__.py deleted file mode 100644 index 5c1048e02a3..00000000000 --- a/tensorflow/contrib/copy_graph/python/util/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functions for copying elements from one graph to another. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/copy_graph/python/util/copy_elements.py b/tensorflow/contrib/copy_graph/python/util/copy_elements.py deleted file mode 100644 index 9c5871da343..00000000000 --- a/tensorflow/contrib/copy_graph/python/util/copy_elements.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""## Functions for copying elements from one graph to another. - -These functions allow for recursive copying of elements (ops and variables) -from one graph to another. The copied elements are initialized inside a -user-specified scope in the other graph. There are separate functions to -copy ops and variables. -There is also a function to retrieve the copied version of an op from the -first graph inside a scope in the second graph. - -@@copy_op_to_graph -@@copy_variable_to_graph -@@get_copied_op -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from copy import deepcopy -from tensorflow.python.ops.variables import VariableV1 -from tensorflow.python.client.session import Session -from tensorflow.python.framework import ops - -__all__ = ['copy_op_to_graph', 'copy_variable_to_graph', 'get_copied_op'] - - -def copy_variable_to_graph(org_instance, to_graph, scope=''): - """Given a `Variable` instance from one `Graph`, initializes and returns - a copy of it from another `Graph`, under the specified scope - (default `""`). - - Args: - org_instance: A `Variable` from some `Graph`. - to_graph: The `Graph` to copy the `Variable` to. - scope: A scope for the new `Variable` (default `""`). - - Returns: - The copied `Variable` from `to_graph`. - - Raises: - TypeError: If `org_instance` is not a `Variable`. - """ - - if not isinstance(org_instance, VariableV1): - raise TypeError(str(org_instance) + ' is not a Variable') - - #The name of the new variable - if scope != '': - new_name = (scope + '/' + org_instance.name[:org_instance.name.index(':')]) - else: - new_name = org_instance.name[:org_instance.name.index(':')] - - #Get the collections that the new instance needs to be added to. - #The new collections will also be a part of the given scope, - #except the special ones required for variable initialization and - #training. - collections = [] - for name, collection in org_instance.graph._collections.items(): - if org_instance in collection: - if (name == ops.GraphKeys.GLOBAL_VARIABLES or - name == ops.GraphKeys.TRAINABLE_VARIABLES or scope == ''): - collections.append(name) - else: - collections.append(scope + '/' + name) - - #See if it's trainable. - trainable = ( - org_instance in org_instance.graph.get_collection( - ops.GraphKeys.TRAINABLE_VARIABLES)) - #Get the initial value - with org_instance.graph.as_default(): - temp_session = Session() - init_value = temp_session.run(org_instance.initialized_value()) - - #Initialize the new variable - with to_graph.as_default(): - new_var = VariableV1( - init_value, - trainable, - name=new_name, - collections=collections, - validate_shape=False) - - return new_var - - -def copy_op_to_graph(org_instance, to_graph, variables, scope=''): - """Returns a copy of an operation from another Graph under a specified scope. - - Given an `Operation` `org_instance` from one `Graph`, - initializes and returns a copy of it from another `Graph`, - under the specified scope (default `""`). - - The copying is done recursively, so any `Operation` whose output - is required to evaluate the `org_instance`, is also copied (unless - already done). - - Since `Variable` instances are copied separately, those required - to evaluate `org_instance` must be provided as input. - - Args: - org_instance: An `Operation` from some `Graph`. Could be a - `Placeholder` as well. - to_graph: The `Graph` to copy `org_instance` to. - variables: An iterable of `Variable` instances to copy `org_instance` to. - scope: A scope for the new `Variable` (default `""`). - - Returns: - The copied `Operation` from `to_graph`. - - Raises: - TypeError: If `org_instance` is not an `Operation` or `Tensor`. - """ - - #The name of the new instance - if scope != '': - new_name = scope + '/' + org_instance.name - else: - new_name = org_instance.name - - #Extract names of variables - copied_variables = dict((x.name, x) for x in variables) - - #If a variable by the new name already exists, return the - #correspondng tensor that will act as an input - if new_name in copied_variables: - return to_graph.get_tensor_by_name(copied_variables[new_name].name) - - #If an instance of the same name exists, return appropriately - try: - already_present = to_graph.as_graph_element( - new_name, allow_tensor=True, allow_operation=True) - return already_present - except: - pass - - #Get the collections that the new instance needs to be added to. - #The new collections will also be a part of the given scope. - collections = [] - for name, collection in org_instance.graph._collections.items(): - if org_instance in collection: - if scope == '': - collections.append(name) - else: - collections.append(scope + '/' + name) - - #Take action based on the class of the instance - - if isinstance(org_instance, ops.Tensor): - - #If it's a Tensor, it is one of the outputs of the underlying - #op. Therefore, copy the op itself and return the appropriate - #output. - op = org_instance.op - new_op = copy_op_to_graph(op, to_graph, variables, scope) - output_index = op.outputs.index(org_instance) - new_tensor = new_op.outputs[output_index] - #Add to collections if any - for collection in collections: - to_graph.add_to_collection(collection, new_tensor) - - return new_tensor - - elif isinstance(org_instance, ops.Operation): - - op = org_instance - - #If it has an original_op parameter, copy it - if op._original_op is not None: - new_original_op = copy_op_to_graph(op._original_op, to_graph, variables, - scope) - else: - new_original_op = None - - #If it has control inputs, call this function recursively on each. - new_control_inputs = [ - copy_op_to_graph(x, to_graph, variables, scope) - for x in op.control_inputs - ] - - #If it has inputs, call this function recursively on each. - new_inputs = [ - copy_op_to_graph(x, to_graph, variables, scope) for x in op.inputs - ] - - #Make a new node_def based on that of the original. - #An instance of tensorflow.core.framework.node_def_pb2.NodeDef, it - #stores String-based info such as name, device and type of the op. - #Unique to every Operation instance. - new_node_def = deepcopy(op.node_def) - #Change the name - new_node_def.name = new_name - - #Copy the other inputs needed for initialization - output_types = op._output_types[:] - input_types = op._input_types[:] - - #Make a copy of the op_def too. - #Its unique to every _type_ of Operation. - op_def = deepcopy(op.op_def) - - #Initialize a new Operation instance - new_op = ops.Operation(new_node_def, to_graph, new_inputs, output_types, - new_control_inputs, input_types, new_original_op, - op_def) - #Use Graph's hidden methods to add the op - to_graph._record_op_seen_by_control_dependencies(new_op) - # pylint: disable=protected-access - for device_function in to_graph._device_functions_outer_to_inner: - new_op._set_device(device_function(new_op)) - # pylint: enable=protected-access - - return new_op - - else: - raise TypeError('Could not copy instance: ' + str(org_instance)) - - -def get_copied_op(org_instance, graph, scope=''): - """Given an `Operation` instance from some `Graph`, returns - its namesake from `graph`, under the specified scope - (default `""`). - - If a copy of `org_instance` is present in `graph` under the given - `scope`, it will be returned. - - Args: - org_instance: An `Operation` from some `Graph`. - graph: The `Graph` to be searched for a copr of `org_instance`. - scope: The scope `org_instance` is present in. - - Returns: - The `Operation` copy from `graph`. - """ - - #The name of the copied instance - if scope != '': - new_name = scope + '/' + org_instance.name - else: - new_name = org_instance.name - - return graph.as_graph_element( - new_name, allow_tensor=True, allow_operation=True) diff --git a/tensorflow/contrib/copy_graph/python/util/copy_test.py b/tensorflow/contrib/copy_graph/python/util/copy_test.py deleted file mode 100644 index 4d8651a79fd..00000000000 --- a/tensorflow/contrib/copy_graph/python/util/copy_test.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for contrib.copy_graph.python.util.copy.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.copy_graph.python.util import copy_elements -from tensorflow.python.client import session as session_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class CopyVariablesTest(test.TestCase): - - def setUp(self): - self.graph1 = ops.Graph() - self.graph2 = ops.Graph() - - def testVariableCopy(self): - - with self.graph1.as_default(): - #Define a Variable in graph1 - some_var = variables.VariableV1(2) - #Initialize session - sess1 = session_lib.Session() - #Initialize the Variable - variables.global_variables_initializer().run(session=sess1) - - #Make a copy of some_var in the defsult scope in graph2 - copy1 = copy_elements.copy_variable_to_graph(some_var, self.graph2) - - #Make another copy with different scope - copy2 = copy_elements.copy_variable_to_graph(some_var, - self.graph2, - "test_scope") - - #Initialize both the copies - with self.graph2.as_default(): - #Initialize Session - sess2 = session_lib.Session() - #Initialize the Variables - variables.global_variables_initializer().run(session=sess2) - - #Ensure values in all three variables are the same - v1 = some_var.eval(session=sess1) - v2 = copy1.eval(session=sess2) - v3 = copy2.eval(session=sess2) - - assert isinstance(copy1, variables.Variable) - assert isinstance(copy2, variables.Variable) - assert v1 == v2 == v3 == 2 - - -class CopyOpsTest(test.TestCase): - - def setUp(self): - self.graph1 = ops.Graph() - self.graph2 = ops.Graph() - - def testOpsCopy(self): - - with self.graph1.as_default(): - #Initialize a basic expression y = ax + b - x = array_ops.placeholder("float") - a = variables.VariableV1(3.0) - b = constant_op.constant(4.0) - ax = math_ops.multiply(x, a) - y = math_ops.add(ax, b) - #Initialize session - sess1 = session_lib.Session() - #Initialize the Variable - variables.global_variables_initializer().run(session=sess1) - - #First, initialize a as a Variable in graph2 - a1 = copy_elements.copy_variable_to_graph(a, self.graph2) - - #Initialize a1 in graph2 - with self.graph2.as_default(): - #Initialize session - sess2 = session_lib.Session() - #Initialize the Variable - variables.global_variables_initializer().run(session=sess2) - - #Initialize a copy of y in graph2 - y1 = copy_elements.copy_op_to_graph(y, self.graph2, [a1]) - - #Now that y has been copied, x must be copied too. - #Get that instance - x1 = copy_elements.get_copied_op(x, self.graph2) - - #Compare values of y & y1 for a sample input - #and check if they match - v1 = y.eval({x: 5}, session=sess1) - v2 = y1.eval({x1: 5}, session=sess2) - - assert v1 == v2 - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/crf/BUILD b/tensorflow/contrib/crf/BUILD deleted file mode 100644 index c225b7b904b..00000000000 --- a/tensorflow/contrib/crf/BUILD +++ /dev/null @@ -1,43 +0,0 @@ -# Description: -# Contains classes to construct a CRF layer -# APIs here are meant to evolve over time. - -load("//tensorflow:tensorflow.bzl", "cuda_py_tests") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "crf_py", - srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:rnn", - "//tensorflow/python:rnn_cell", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//third_party/py/numpy", - ], -) - -cuda_py_tests( - name = "crf_test", - srcs = ["python/kernel_tests/crf_test.py"], - additional_deps = [ - ":crf_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) diff --git a/tensorflow/contrib/crf/README.md b/tensorflow/contrib/crf/README.md deleted file mode 100644 index b58cc2dd04c..00000000000 --- a/tensorflow/contrib/crf/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# CRF - -The CRF module implements a linear-chain CRF layer for learning to predict tag sequences. This variant of the CRF is factored into unary potentials for every element in the sequence and binary potentials for every transition between output tags. - -### Usage - -Below is an example of the API, which learns a CRF for some random data. The linear layer in the example can be replaced by any neural network. - - -```python -import numpy as np -import tensorflow as tf - -# Data settings. -num_examples = 10 -num_words = 20 -num_features = 100 -num_tags = 5 - -# Random features. -x = np.random.rand(num_examples, num_words, num_features).astype(np.float32) - -# Random tag indices representing the gold sequence. -y = np.random.randint(num_tags, size=[num_examples, num_words]).astype(np.int32) - -# All sequences in this example have the same length, but they can be variable in a real model. -sequence_lengths = np.full(num_examples, num_words - 1, dtype=np.int32) - -# Train and evaluate the model. -with tf.Graph().as_default(): - with tf.Session() as session: - # Add the data to the TensorFlow graph. - x_t = tf.constant(x) - y_t = tf.constant(y) - sequence_lengths_t = tf.constant(sequence_lengths) - - # Compute unary scores from a linear layer. - weights = tf.get_variable("weights", [num_features, num_tags]) - matricized_x_t = tf.reshape(x_t, [-1, num_features]) - matricized_unary_scores = tf.matmul(matricized_x_t, weights) - unary_scores = tf.reshape(matricized_unary_scores, - [num_examples, num_words, num_tags]) - - # Compute the log-likelihood of the gold sequences and keep the transition - # params for inference at test time. - log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood( - unary_scores, y_t, sequence_lengths_t) - - # Compute the viterbi sequence and score. - viterbi_sequence, viterbi_score = tf.contrib.crf.crf_decode( - unary_scores, transition_params, sequence_lengths_t) - - # Add a training op to tune the parameters. - loss = tf.reduce_mean(-log_likelihood) - train_op = tf.train.GradientDescentOptimizer(0.01).minimize(loss) - - session.run(tf.global_variables_initializer()) - - mask = (np.expand_dims(np.arange(num_words), axis=0) < - np.expand_dims(sequence_lengths, axis=1)) - total_labels = np.sum(sequence_lengths) - - # Train for a fixed number of iterations. - for i in range(1000): - tf_viterbi_sequence, _ = session.run([viterbi_sequence, train_op]) - if i % 100 == 0: - correct_labels = np.sum((y == tf_viterbi_sequence) * mask) - accuracy = 100.0 * correct_labels / float(total_labels) - print("Accuracy: %.2f%%" % accuracy) -``` diff --git a/tensorflow/contrib/crf/__init__.py b/tensorflow/contrib/crf/__init__.py deleted file mode 100644 index d53549048f3..00000000000 --- a/tensorflow/contrib/crf/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Linear-chain CRF layer. - -@@crf_binary_score -@@crf_decode -@@crf_log_likelihood -@@crf_log_norm -@@crf_multitag_sequence_score -@@crf_sequence_score -@@crf_unary_score -@@CrfDecodeBackwardRnnCell -@@CrfDecodeForwardRnnCell -@@CrfForwardRnnCell -@@viterbi_decode -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.crf.python.ops.crf import crf_binary_score -from tensorflow.contrib.crf.python.ops.crf import crf_decode -from tensorflow.contrib.crf.python.ops.crf import crf_log_likelihood -from tensorflow.contrib.crf.python.ops.crf import crf_log_norm -from tensorflow.contrib.crf.python.ops.crf import crf_multitag_sequence_score -from tensorflow.contrib.crf.python.ops.crf import crf_sequence_score -from tensorflow.contrib.crf.python.ops.crf import crf_unary_score -from tensorflow.contrib.crf.python.ops.crf import CrfDecodeBackwardRnnCell -from tensorflow.contrib.crf.python.ops.crf import CrfDecodeForwardRnnCell -from tensorflow.contrib.crf.python.ops.crf import CrfForwardRnnCell -from tensorflow.contrib.crf.python.ops.crf import viterbi_decode - -from tensorflow.python.util.all_util import remove_undocumented - -remove_undocumented(__name__) diff --git a/tensorflow/contrib/crf/python/__init__.py b/tensorflow/contrib/crf/python/__init__.py deleted file mode 100644 index 8439848dd0e..00000000000 --- a/tensorflow/contrib/crf/python/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Linear-chain CRF.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/crf/python/kernel_tests/crf_test.py b/tensorflow/contrib/crf/python/kernel_tests/crf_test.py deleted file mode 100644 index 556d7318402..00000000000 --- a/tensorflow/contrib/crf/python/kernel_tests/crf_test.py +++ /dev/null @@ -1,367 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for CRF.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools - -import numpy as np - -from tensorflow.contrib.crf.python.ops import crf -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class CrfTest(test.TestCase): - - def calculateSequenceScore(self, inputs, transition_params, tag_indices, - sequence_lengths): - expected_unary_score = sum( - inputs[i][tag_indices[i]] for i in range(sequence_lengths)) - expected_binary_score = sum( - transition_params[tag_indices[i], tag_indices[i + 1]] - for i in range(sequence_lengths - 1)) - return expected_unary_score + expected_binary_score - - def testCrfSequenceScore(self): - transition_params = np.array( - [[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=np.float32) - # Test both the length-1 and regular cases. - sequence_lengths_list = [ - np.array(3, dtype=np.int32), - np.array(1, dtype=np.int32) - ] - inputs_list = [ - np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], - dtype=np.float32), - np.array([[4, 5, -3]], - dtype=np.float32), - ] - tag_indices_list = [ - np.array([1, 2, 1, 0], dtype=np.int32), - np.array([1], dtype=np.int32) - ] - for sequence_lengths, inputs, tag_indices in zip(sequence_lengths_list, - inputs_list, - tag_indices_list): - with self.cached_session() as sess: - sequence_score = crf.crf_sequence_score( - inputs=array_ops.expand_dims(inputs, 0), - tag_indices=array_ops.expand_dims(tag_indices, 0), - sequence_lengths=array_ops.expand_dims(sequence_lengths, 0), - transition_params=constant_op.constant(transition_params)) - sequence_score = array_ops.squeeze(sequence_score, [0]) - tf_sequence_score = sess.run(sequence_score) - expected_sequence_score = self.calculateSequenceScore( - inputs, transition_params, tag_indices, sequence_lengths) - self.assertAllClose(tf_sequence_score, expected_sequence_score) - - def testCrfMultiTagSequenceScore(self): - transition_params = np.array( - [[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=np.float32) - # Test both the length-1 and regular cases. - sequence_lengths_list = [ - np.array(3, dtype=np.int32), - np.array(1, dtype=np.int32) - ] - inputs_list = [ - np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], - dtype=np.float32), - np.array([[4, 5, -3]], - dtype=np.float32), - ] - tag_bitmap_list = [ - np.array( - [[True, True, False], [True, False, True], [False, True, True], - [True, False, True]], - dtype=np.bool), - np.array([[True, True, False]], dtype=np.bool) - ] - for sequence_lengths, inputs, tag_bitmap in zip( - sequence_lengths_list, inputs_list, tag_bitmap_list): - with self.cached_session() as sess: - sequence_score = crf.crf_multitag_sequence_score( - inputs=array_ops.expand_dims(inputs, 0), - tag_bitmap=array_ops.expand_dims(tag_bitmap, 0), - sequence_lengths=array_ops.expand_dims(sequence_lengths, 0), - transition_params=constant_op.constant(transition_params)) - sequence_score = array_ops.squeeze(sequence_score, [0]) - tf_sum_sequence_score = sess.run(sequence_score) - all_indices_list = [ - single_index_bitmap.nonzero()[0] - for single_index_bitmap in tag_bitmap[:sequence_lengths] - ] - expected_sequence_scores = [ - self.calculateSequenceScore(inputs, transition_params, indices, - sequence_lengths) - for indices in itertools.product(*all_indices_list) - ] - expected_log_sum_exp_sequence_scores = np.logaddexp.reduce( - expected_sequence_scores) - self.assertAllClose(tf_sum_sequence_score, - expected_log_sum_exp_sequence_scores) - - def testCrfUnaryScore(self): - inputs = np.array( - [[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=np.float32) - for dtype in (np.int32, np.int64): - tag_indices = np.array([1, 2, 1, 0], dtype=dtype) - sequence_lengths = np.array(3, dtype=np.int32) - with self.cached_session() as sess: - unary_score = crf.crf_unary_score( - tag_indices=array_ops.expand_dims(tag_indices, 0), - sequence_lengths=array_ops.expand_dims(sequence_lengths, 0), - inputs=array_ops.expand_dims(inputs, 0)) - unary_score = array_ops.squeeze(unary_score, [0]) - tf_unary_score = sess.run(unary_score) - expected_unary_score = sum(inputs[i][tag_indices[i]] - for i in range(sequence_lengths)) - self.assertAllClose(tf_unary_score, expected_unary_score) - - def testCrfBinaryScore(self): - tag_indices = np.array([1, 2, 1, 0], dtype=np.int32) - transition_params = np.array( - [[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=np.float32) - sequence_lengths = np.array(3, dtype=np.int32) - with self.cached_session() as sess: - binary_score = crf.crf_binary_score( - tag_indices=array_ops.expand_dims(tag_indices, 0), - sequence_lengths=array_ops.expand_dims(sequence_lengths, 0), - transition_params=constant_op.constant(transition_params)) - binary_score = array_ops.squeeze(binary_score, [0]) - tf_binary_score = sess.run(binary_score) - expected_binary_score = sum( - transition_params[tag_indices[i], tag_indices[i + 1]] - for i in range(sequence_lengths - 1)) - self.assertAllClose(tf_binary_score, expected_binary_score) - - def testCrfLogNorm(self): - transition_params = np.array( - [[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=np.float32) - # Test both the length-1 and regular cases. - sequence_lengths_list = [ - np.array(3, dtype=np.int32), - np.array(1, dtype=np.int64) - ] - inputs_list = [ - np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], - dtype=np.float32), - np.array([[3, -1, 3]], - dtype=np.float32), - ] - tag_indices_list = [ - np.array([1, 2, 1, 0], dtype=np.int32), - np.array([2], dtype=np.int32) - ] - - for sequence_lengths, inputs, tag_indices in zip(sequence_lengths_list, - inputs_list, - tag_indices_list): - num_words = inputs.shape[0] - num_tags = inputs.shape[1] - with self.cached_session() as sess: - all_sequence_scores = [] - - # Compare the dynamic program with brute force computation. - for tag_indices in itertools.product( - range(num_tags), repeat=sequence_lengths): - tag_indices = list(tag_indices) - tag_indices.extend([0] * (num_words - sequence_lengths)) - all_sequence_scores.append( - crf.crf_sequence_score( - inputs=array_ops.expand_dims(inputs, 0), - tag_indices=array_ops.expand_dims(tag_indices, 0), - sequence_lengths=array_ops.expand_dims(sequence_lengths, 0), - transition_params=constant_op.constant(transition_params))) - - brute_force_log_norm = math_ops.reduce_logsumexp(all_sequence_scores) - log_norm = crf.crf_log_norm( - inputs=array_ops.expand_dims(inputs, 0), - sequence_lengths=array_ops.expand_dims(sequence_lengths, 0), - transition_params=constant_op.constant(transition_params)) - log_norm = array_ops.squeeze(log_norm, [0]) - tf_brute_force_log_norm, tf_log_norm = sess.run( - [brute_force_log_norm, log_norm]) - - self.assertAllClose(tf_log_norm, tf_brute_force_log_norm) - - def testCrfLogNormZeroSeqLength(self): - """ - Test `crf_log_norm` when `sequence_lengths` contains one or more zeros. - """ - with self.cached_session() as sess: - inputs = constant_op.constant(np.ones([2, 10, 5], - dtype=np.float32)) - transition_params = constant_op.constant(np.ones([5, 5], - dtype=np.float32)) - sequence_lengths = constant_op.constant(np.zeros([2], - dtype=np.int32)) - expected_log_norm = np.zeros([2], dtype=np.float32) - log_norm = crf.crf_log_norm(inputs, sequence_lengths, transition_params) - tf_log_norm = sess.run(log_norm) - self.assertAllClose(tf_log_norm, expected_log_norm) - - def testCrfLogLikelihood(self): - inputs = np.array( - [[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=np.float32) - transition_params = np.array( - [[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=np.float32) - sequence_lengths = np.array(3, dtype=np.int32) - num_words = inputs.shape[0] - num_tags = inputs.shape[1] - with self.cached_session() as sess: - all_sequence_log_likelihoods = [] - - # Make sure all probabilities sum to 1. - for tag_indices in itertools.product( - range(num_tags), repeat=sequence_lengths): - tag_indices = list(tag_indices) - tag_indices.extend([0] * (num_words - sequence_lengths)) - sequence_log_likelihood, _ = crf.crf_log_likelihood( - inputs=array_ops.expand_dims(inputs, 0), - tag_indices=array_ops.expand_dims(tag_indices, 0), - sequence_lengths=array_ops.expand_dims(sequence_lengths, 0), - transition_params=constant_op.constant(transition_params)) - all_sequence_log_likelihoods.append(sequence_log_likelihood) - total_log_likelihood = math_ops.reduce_logsumexp( - all_sequence_log_likelihoods) - tf_total_log_likelihood = sess.run(total_log_likelihood) - self.assertAllClose(tf_total_log_likelihood, 0.0) - - def testViterbiDecode(self): - inputs = np.array( - [[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=np.float32) - transition_params = np.array( - [[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=np.float32) - sequence_lengths = np.array(3, dtype=np.int32) - num_words = inputs.shape[0] - num_tags = inputs.shape[1] - - with self.cached_session() as sess: - all_sequence_scores = [] - all_sequences = [] - - # Compare the dynamic program with brute force computation. - for tag_indices in itertools.product( - range(num_tags), repeat=sequence_lengths): - tag_indices = list(tag_indices) - tag_indices.extend([0] * (num_words - sequence_lengths)) - all_sequences.append(tag_indices) - sequence_score = crf.crf_sequence_score( - inputs=array_ops.expand_dims(inputs, 0), - tag_indices=array_ops.expand_dims(tag_indices, 0), - sequence_lengths=array_ops.expand_dims(sequence_lengths, 0), - transition_params=constant_op.constant(transition_params)) - sequence_score = array_ops.squeeze(sequence_score, [0]) - all_sequence_scores.append(sequence_score) - - tf_all_sequence_scores = sess.run(all_sequence_scores) - - expected_max_sequence_index = np.argmax(tf_all_sequence_scores) - expected_max_sequence = all_sequences[expected_max_sequence_index] - expected_max_score = tf_all_sequence_scores[expected_max_sequence_index] - - actual_max_sequence, actual_max_score = crf.viterbi_decode( - inputs[:sequence_lengths], transition_params) - - self.assertAllClose(actual_max_score, expected_max_score) - self.assertEqual(actual_max_sequence, - expected_max_sequence[:sequence_lengths]) - - def testCrfDecode(self): - transition_params = np.array( - [[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=np.float32) - # Test both the length-1 and regular cases. - sequence_lengths_list = [ - np.array(3, dtype=np.int32), - np.array(1, dtype=np.int64) - ] - inputs_list = [ - np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], - dtype=np.float32), - np.array([[-1, 2, 1]], - dtype=np.float32), - ] - tag_indices_list = [ - np.array([1, 2, 1, 0], dtype=np.int32), - np.array([2], dtype=np.int32) - ] - - for sequence_lengths, inputs, tag_indices in zip(sequence_lengths_list, - inputs_list, - tag_indices_list): - num_words = inputs.shape[0] - num_tags = inputs.shape[1] - - with self.cached_session() as sess: - all_sequence_scores = [] - all_sequences = [] - - # Compare the dynamic program with brute force computation. - for tag_indices in itertools.product( - range(num_tags), repeat=sequence_lengths): - tag_indices = list(tag_indices) - tag_indices.extend([0] * (num_words - sequence_lengths)) - all_sequences.append(tag_indices) - sequence_score = crf.crf_sequence_score( - inputs=array_ops.expand_dims(inputs, 0), - tag_indices=array_ops.expand_dims(tag_indices, 0), - sequence_lengths=array_ops.expand_dims(sequence_lengths, 0), - transition_params=constant_op.constant(transition_params)) - sequence_score = array_ops.squeeze(sequence_score, [0]) - all_sequence_scores.append(sequence_score) - - tf_all_sequence_scores = sess.run(all_sequence_scores) - - expected_max_sequence_index = np.argmax(tf_all_sequence_scores) - expected_max_sequence = all_sequences[expected_max_sequence_index] - expected_max_score = tf_all_sequence_scores[expected_max_sequence_index] - - actual_max_sequence, actual_max_score = crf.crf_decode( - array_ops.expand_dims(inputs, 0), - constant_op.constant(transition_params), - array_ops.expand_dims(sequence_lengths, 0)) - actual_max_sequence = array_ops.squeeze(actual_max_sequence, [0]) - actual_max_score = array_ops.squeeze(actual_max_score, [0]) - tf_actual_max_sequence, tf_actual_max_score = sess.run( - [actual_max_sequence, actual_max_score]) - - self.assertAllClose(tf_actual_max_score, expected_max_score) - self.assertEqual(list(tf_actual_max_sequence[:sequence_lengths]), - expected_max_sequence[:sequence_lengths]) - - def testCrfDecodeZeroSeqLength(self): - """ - Test that crf_decode works when sequence_length contains one or more zeros. - """ - with self.cached_session() as sess: - inputs = constant_op.constant(np.ones([2, 10, 5], - dtype=np.float32)) - transition_params = constant_op.constant(np.ones([5, 5], - dtype=np.float32)) - sequence_lengths = constant_op.constant(np.zeros([2], - dtype=np.int32)) - tags, scores = crf.crf_decode(inputs, transition_params, sequence_lengths) - tf_tags, tf_scores = sess.run([tags, scores]) - self.assertEqual(len(tf_tags.shape), 2) - self.assertEqual(len(tf_scores.shape), 1) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/crf/python/ops/__init__.py b/tensorflow/contrib/crf/python/ops/__init__.py deleted file mode 100644 index 5ab8d7ac4a9..00000000000 --- a/tensorflow/contrib/crf/python/ops/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for building a linear-chain CRF layer.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/crf/python/ops/crf.py b/tensorflow/contrib/crf/python/ops/crf.py deleted file mode 100644 index 974423fec99..00000000000 --- a/tensorflow/contrib/crf/python/ops/crf.py +++ /dev/null @@ -1,596 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Module for constructing a linear-chain CRF. - -The following snippet is an example of a CRF layer on top of a batched sequence -of unary scores (logits for every word). This example also decodes the most -likely sequence at test time. There are two ways to do decoding. One -is using crf_decode to do decoding in Tensorflow , and the other one is using -viterbi_decode in Numpy. - -log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood( - unary_scores, gold_tags, sequence_lengths) - -loss = tf.reduce_mean(-log_likelihood) -train_op = tf.compat.v1.train.GradientDescentOptimizer(0.01).minimize(loss) - -# Decoding in Tensorflow. -viterbi_sequence, viterbi_score = tf.contrib.crf.crf_decode( - unary_scores, transition_params, sequence_lengths) - -tf_viterbi_sequence, tf_viterbi_score, _ = session.run( - [viterbi_sequence, viterbi_score, train_op]) - -# Decoding in Numpy. -tf_unary_scores, tf_sequence_lengths, tf_transition_params, _ = session.run( - [unary_scores, sequence_lengths, transition_params, train_op]) -for tf_unary_scores_, tf_sequence_length_ in zip(tf_unary_scores, - tf_sequence_lengths): - # Remove padding. - tf_unary_scores_ = tf_unary_scores_[:tf_sequence_length_] - - # Compute the highest score and its tag sequence. - tf_viterbi_sequence, tf_viterbi_score = tf.contrib.crf.viterbi_decode( - tf_unary_scores_, tf_transition_params) -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.layers import utils -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import rnn -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import variable_scope as vs - -__all__ = [ - "crf_sequence_score", "crf_log_norm", "crf_log_likelihood", - "crf_unary_score", "crf_binary_score", "CrfForwardRnnCell", - "viterbi_decode", "crf_decode", "CrfDecodeForwardRnnCell", - "CrfDecodeBackwardRnnCell", "crf_multitag_sequence_score" -] - - -def crf_sequence_score(inputs, tag_indices, sequence_lengths, - transition_params): - """Computes the unnormalized score for a tag sequence. - - Args: - inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials - to use as input to the CRF layer. - tag_indices: A [batch_size, max_seq_len] matrix of tag indices for which we - compute the unnormalized score. - sequence_lengths: A [batch_size] vector of true sequence lengths. - transition_params: A [num_tags, num_tags] transition matrix. - Returns: - sequence_scores: A [batch_size] vector of unnormalized sequence scores. - """ - # If max_seq_len is 1, we skip the score calculation and simply gather the - # unary potentials of the single tag. - def _single_seq_fn(): - batch_size = array_ops.shape(inputs, out_type=tag_indices.dtype)[0] - example_inds = array_ops.reshape( - math_ops.range(batch_size, dtype=tag_indices.dtype), [-1, 1]) - sequence_scores = array_ops.gather_nd( - array_ops.squeeze(inputs, [1]), - array_ops.concat([example_inds, tag_indices], axis=1)) - sequence_scores = array_ops.where(math_ops.less_equal(sequence_lengths, 0), - array_ops.zeros_like(sequence_scores), - sequence_scores) - return sequence_scores - - def _multi_seq_fn(): - # Compute the scores of the given tag sequence. - unary_scores = crf_unary_score(tag_indices, sequence_lengths, inputs) - binary_scores = crf_binary_score(tag_indices, sequence_lengths, - transition_params) - sequence_scores = unary_scores + binary_scores - return sequence_scores - - return utils.smart_cond( - pred=math_ops.equal( - tensor_shape.dimension_value( - inputs.shape[1]) or array_ops.shape(inputs)[1], - 1), - true_fn=_single_seq_fn, - false_fn=_multi_seq_fn) - - -def crf_multitag_sequence_score(inputs, tag_bitmap, sequence_lengths, - transition_params): - """Computes the unnormalized score of all tag sequences matching tag_bitmap. - - tag_bitmap enables more than one tag to be considered correct at each time - step. This is useful when an observed output at a given time step is - consistent with more than one tag, and thus the log likelihood of that - observation must take into account all possible consistent tags. - - Using one-hot vectors in tag_bitmap gives results identical to - crf_sequence_score. - - Args: - inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials - to use as input to the CRF layer. - tag_bitmap: A [batch_size, max_seq_len, num_tags] boolean tensor - representing all active tags at each index for which to calculate the - unnormalized score. - sequence_lengths: A [batch_size] vector of true sequence lengths. - transition_params: A [num_tags, num_tags] transition matrix. - Returns: - sequence_scores: A [batch_size] vector of unnormalized sequence scores. - """ - - # If max_seq_len is 1, we skip the score calculation and simply gather the - # unary potentials of all active tags. - def _single_seq_fn(): - filtered_inputs = array_ops.where( - tag_bitmap, inputs, - array_ops.fill(array_ops.shape(inputs), float("-inf"))) - return math_ops.reduce_logsumexp( - filtered_inputs, axis=[1, 2], keepdims=False) - - def _multi_seq_fn(): - # Compute the logsumexp of all scores of sequences matching the given tags. - filtered_inputs = array_ops.where( - tag_bitmap, inputs, - array_ops.fill(array_ops.shape(inputs), float("-inf"))) - return crf_log_norm( - inputs=filtered_inputs, - sequence_lengths=sequence_lengths, - transition_params=transition_params) - - return utils.smart_cond( - pred=math_ops.equal( - tensor_shape.dimension_value( - inputs.shape[1]) or array_ops.shape(inputs)[1], - 1), - true_fn=_single_seq_fn, - false_fn=_multi_seq_fn) - - -def crf_log_norm(inputs, sequence_lengths, transition_params): - """Computes the normalization for a CRF. - - Args: - inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials - to use as input to the CRF layer. - sequence_lengths: A [batch_size] vector of true sequence lengths. - transition_params: A [num_tags, num_tags] transition matrix. - Returns: - log_norm: A [batch_size] vector of normalizers for a CRF. - """ - # Split up the first and rest of the inputs in preparation for the forward - # algorithm. - first_input = array_ops.slice(inputs, [0, 0, 0], [-1, 1, -1]) - first_input = array_ops.squeeze(first_input, [1]) - - # If max_seq_len is 1, we skip the algorithm and simply reduce_logsumexp over - # the "initial state" (the unary potentials). - def _single_seq_fn(): - log_norm = math_ops.reduce_logsumexp(first_input, [1]) - # Mask `log_norm` of the sequences with length <= zero. - log_norm = array_ops.where(math_ops.less_equal(sequence_lengths, 0), - array_ops.zeros_like(log_norm), - log_norm) - return log_norm - - def _multi_seq_fn(): - """Forward computation of alpha values.""" - rest_of_input = array_ops.slice(inputs, [0, 1, 0], [-1, -1, -1]) - - # Compute the alpha values in the forward algorithm in order to get the - # partition function. - forward_cell = CrfForwardRnnCell(transition_params) - # Sequence length is not allowed to be less than zero. - sequence_lengths_less_one = math_ops.maximum( - constant_op.constant(0, dtype=sequence_lengths.dtype), - sequence_lengths - 1) - _, alphas = rnn.dynamic_rnn( - cell=forward_cell, - inputs=rest_of_input, - sequence_length=sequence_lengths_less_one, - initial_state=first_input, - dtype=dtypes.float32) - log_norm = math_ops.reduce_logsumexp(alphas, [1]) - # Mask `log_norm` of the sequences with length <= zero. - log_norm = array_ops.where(math_ops.less_equal(sequence_lengths, 0), - array_ops.zeros_like(log_norm), - log_norm) - return log_norm - - return utils.smart_cond( - pred=math_ops.equal( - tensor_shape.dimension_value( - inputs.shape[1]) or array_ops.shape(inputs)[1], - 1), - true_fn=_single_seq_fn, - false_fn=_multi_seq_fn) - - -def crf_log_likelihood(inputs, - tag_indices, - sequence_lengths, - transition_params=None): - """Computes the log-likelihood of tag sequences in a CRF. - - Args: - inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials - to use as input to the CRF layer. - tag_indices: A [batch_size, max_seq_len] matrix of tag indices for which we - compute the log-likelihood. - sequence_lengths: A [batch_size] vector of true sequence lengths. - transition_params: A [num_tags, num_tags] transition matrix, if available. - Returns: - log_likelihood: A [batch_size] `Tensor` containing the log-likelihood of - each example, given the sequence of tag indices. - transition_params: A [num_tags, num_tags] transition matrix. This is either - provided by the caller or created in this function. - """ - # Get shape information. - num_tags = tensor_shape.dimension_value(inputs.shape[2]) - - # Get the transition matrix if not provided. - if transition_params is None: - transition_params = vs.get_variable("transitions", [num_tags, num_tags]) - - sequence_scores = crf_sequence_score(inputs, tag_indices, sequence_lengths, - transition_params) - log_norm = crf_log_norm(inputs, sequence_lengths, transition_params) - - # Normalize the scores to get the log-likelihood per example. - log_likelihood = sequence_scores - log_norm - return log_likelihood, transition_params - - -def crf_unary_score(tag_indices, sequence_lengths, inputs): - """Computes the unary scores of tag sequences. - - Args: - tag_indices: A [batch_size, max_seq_len] matrix of tag indices. - sequence_lengths: A [batch_size] vector of true sequence lengths. - inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials. - Returns: - unary_scores: A [batch_size] vector of unary scores. - """ - batch_size = array_ops.shape(inputs)[0] - max_seq_len = array_ops.shape(inputs)[1] - num_tags = array_ops.shape(inputs)[2] - - flattened_inputs = array_ops.reshape(inputs, [-1]) - - offsets = array_ops.expand_dims( - math_ops.range(batch_size) * max_seq_len * num_tags, 1) - offsets += array_ops.expand_dims(math_ops.range(max_seq_len) * num_tags, 0) - # Use int32 or int64 based on tag_indices' dtype. - if tag_indices.dtype == dtypes.int64: - offsets = math_ops.cast(offsets, dtypes.int64) - flattened_tag_indices = array_ops.reshape(offsets + tag_indices, [-1]) - - unary_scores = array_ops.reshape( - array_ops.gather(flattened_inputs, flattened_tag_indices), - [batch_size, max_seq_len]) - - masks = array_ops.sequence_mask(sequence_lengths, - maxlen=array_ops.shape(tag_indices)[1], - dtype=dtypes.float32) - - unary_scores = math_ops.reduce_sum(unary_scores * masks, 1) - return unary_scores - - -def crf_binary_score(tag_indices, sequence_lengths, transition_params): - """Computes the binary scores of tag sequences. - - Args: - tag_indices: A [batch_size, max_seq_len] matrix of tag indices. - sequence_lengths: A [batch_size] vector of true sequence lengths. - transition_params: A [num_tags, num_tags] matrix of binary potentials. - Returns: - binary_scores: A [batch_size] vector of binary scores. - """ - # Get shape information. - num_tags = transition_params.get_shape()[0] - num_transitions = array_ops.shape(tag_indices)[1] - 1 - - # Truncate by one on each side of the sequence to get the start and end - # indices of each transition. - start_tag_indices = array_ops.slice(tag_indices, [0, 0], - [-1, num_transitions]) - end_tag_indices = array_ops.slice(tag_indices, [0, 1], [-1, num_transitions]) - - # Encode the indices in a flattened representation. - flattened_transition_indices = start_tag_indices * num_tags + end_tag_indices - flattened_transition_params = array_ops.reshape(transition_params, [-1]) - - # Get the binary scores based on the flattened representation. - binary_scores = array_ops.gather(flattened_transition_params, - flattened_transition_indices) - - masks = array_ops.sequence_mask(sequence_lengths, - maxlen=array_ops.shape(tag_indices)[1], - dtype=dtypes.float32) - truncated_masks = array_ops.slice(masks, [0, 1], [-1, -1]) - binary_scores = math_ops.reduce_sum(binary_scores * truncated_masks, 1) - return binary_scores - - -class CrfForwardRnnCell(rnn_cell.RNNCell): - """Computes the alpha values in a linear-chain CRF. - - See http://www.cs.columbia.edu/~mcollins/fb.pdf for reference. - """ - - def __init__(self, transition_params): - """Initialize the CrfForwardRnnCell. - - Args: - transition_params: A [num_tags, num_tags] matrix of binary potentials. - This matrix is expanded into a [1, num_tags, num_tags] in preparation - for the broadcast summation occurring within the cell. - """ - self._transition_params = array_ops.expand_dims(transition_params, 0) - self._num_tags = tensor_shape.dimension_value(transition_params.shape[0]) - - @property - def state_size(self): - return self._num_tags - - @property - def output_size(self): - return self._num_tags - - def __call__(self, inputs, state, scope=None): - """Build the CrfForwardRnnCell. - - Args: - inputs: A [batch_size, num_tags] matrix of unary potentials. - state: A [batch_size, num_tags] matrix containing the previous alpha - values. - scope: Unused variable scope of this cell. - - Returns: - new_alphas, new_alphas: A pair of [batch_size, num_tags] matrices - values containing the new alpha values. - """ - state = array_ops.expand_dims(state, 2) - - # This addition op broadcasts self._transitions_params along the zeroth - # dimension and state along the second dimension. This performs the - # multiplication of previous alpha values and the current binary potentials - # in log space. - transition_scores = state + self._transition_params - new_alphas = inputs + math_ops.reduce_logsumexp(transition_scores, [1]) - - # Both the state and the output of this RNN cell contain the alphas values. - # The output value is currently unused and simply satisfies the RNN API. - # This could be useful in the future if we need to compute marginal - # probabilities, which would require the accumulated alpha values at every - # time step. - return new_alphas, new_alphas - - -def viterbi_decode(score, transition_params): - """Decode the highest scoring sequence of tags outside of TensorFlow. - - This should only be used at test time. - - Args: - score: A [seq_len, num_tags] matrix of unary potentials. - transition_params: A [num_tags, num_tags] matrix of binary potentials. - - Returns: - viterbi: A [seq_len] list of integers containing the highest scoring tag - indices. - viterbi_score: A float containing the score for the Viterbi sequence. - """ - trellis = np.zeros_like(score) - backpointers = np.zeros_like(score, dtype=np.int32) - trellis[0] = score[0] - - for t in range(1, score.shape[0]): - v = np.expand_dims(trellis[t - 1], 1) + transition_params - trellis[t] = score[t] + np.max(v, 0) - backpointers[t] = np.argmax(v, 0) - - viterbi = [np.argmax(trellis[-1])] - for bp in reversed(backpointers[1:]): - viterbi.append(bp[viterbi[-1]]) - viterbi.reverse() - - viterbi_score = np.max(trellis[-1]) - return viterbi, viterbi_score - - -class CrfDecodeForwardRnnCell(rnn_cell.RNNCell): - """Computes the forward decoding in a linear-chain CRF. - """ - - def __init__(self, transition_params): - """Initialize the CrfDecodeForwardRnnCell. - - Args: - transition_params: A [num_tags, num_tags] matrix of binary - potentials. This matrix is expanded into a - [1, num_tags, num_tags] in preparation for the broadcast - summation occurring within the cell. - """ - self._transition_params = array_ops.expand_dims(transition_params, 0) - self._num_tags = tensor_shape.dimension_value(transition_params.shape[0]) - - @property - def state_size(self): - return self._num_tags - - @property - def output_size(self): - return self._num_tags - - def __call__(self, inputs, state, scope=None): - """Build the CrfDecodeForwardRnnCell. - - Args: - inputs: A [batch_size, num_tags] matrix of unary potentials. - state: A [batch_size, num_tags] matrix containing the previous step's - score values. - scope: Unused variable scope of this cell. - - Returns: - backpointers: A [batch_size, num_tags] matrix of backpointers. - new_state: A [batch_size, num_tags] matrix of new score values. - """ - # For simplicity, in shape comments, denote: - # 'batch_size' by 'B', 'max_seq_len' by 'T' , 'num_tags' by 'O' (output). - state = array_ops.expand_dims(state, 2) # [B, O, 1] - - # This addition op broadcasts self._transitions_params along the zeroth - # dimension and state along the second dimension. - # [B, O, 1] + [1, O, O] -> [B, O, O] - transition_scores = state + self._transition_params # [B, O, O] - new_state = inputs + math_ops.reduce_max(transition_scores, [1]) # [B, O] - backpointers = math_ops.argmax(transition_scores, 1) - backpointers = math_ops.cast(backpointers, dtype=dtypes.int32) # [B, O] - return backpointers, new_state - - -class CrfDecodeBackwardRnnCell(rnn_cell.RNNCell): - """Computes backward decoding in a linear-chain CRF. - """ - - def __init__(self, num_tags): - """Initialize the CrfDecodeBackwardRnnCell. - - Args: - num_tags: An integer. The number of tags. - """ - self._num_tags = num_tags - - @property - def state_size(self): - return 1 - - @property - def output_size(self): - return 1 - - def __call__(self, inputs, state, scope=None): - """Build the CrfDecodeBackwardRnnCell. - - Args: - inputs: A [batch_size, num_tags] matrix of - backpointer of next step (in time order). - state: A [batch_size, 1] matrix of tag index of next step. - scope: Unused variable scope of this cell. - - Returns: - new_tags, new_tags: A pair of [batch_size, num_tags] - tensors containing the new tag indices. - """ - state = array_ops.squeeze(state, axis=[1]) # [B] - batch_size = array_ops.shape(inputs)[0] - b_indices = math_ops.range(batch_size) # [B] - indices = array_ops.stack([b_indices, state], axis=1) # [B, 2] - new_tags = array_ops.expand_dims( - gen_array_ops.gather_nd(inputs, indices), # [B] - axis=-1) # [B, 1] - - return new_tags, new_tags - - -def crf_decode(potentials, transition_params, sequence_length): - """Decode the highest scoring sequence of tags in TensorFlow. - - This is a function for tensor. - - Args: - potentials: A [batch_size, max_seq_len, num_tags] tensor of - unary potentials. - transition_params: A [num_tags, num_tags] matrix of - binary potentials. - sequence_length: A [batch_size] vector of true sequence lengths. - - Returns: - decode_tags: A [batch_size, max_seq_len] matrix, with dtype `tf.int32`. - Contains the highest scoring tag indices. - best_score: A [batch_size] vector, containing the score of `decode_tags`. - """ - # If max_seq_len is 1, we skip the algorithm and simply return the argmax tag - # and the max activation. - def _single_seq_fn(): - squeezed_potentials = array_ops.squeeze(potentials, [1]) - decode_tags = array_ops.expand_dims( - math_ops.argmax(squeezed_potentials, axis=1), 1) - best_score = math_ops.reduce_max(squeezed_potentials, axis=1) - return math_ops.cast(decode_tags, dtype=dtypes.int32), best_score - - def _multi_seq_fn(): - """Decoding of highest scoring sequence.""" - - # For simplicity, in shape comments, denote: - # 'batch_size' by 'B', 'max_seq_len' by 'T' , 'num_tags' by 'O' (output). - num_tags = tensor_shape.dimension_value(potentials.shape[2]) - - # Computes forward decoding. Get last score and backpointers. - crf_fwd_cell = CrfDecodeForwardRnnCell(transition_params) - initial_state = array_ops.slice(potentials, [0, 0, 0], [-1, 1, -1]) - initial_state = array_ops.squeeze(initial_state, axis=[1]) # [B, O] - inputs = array_ops.slice(potentials, [0, 1, 0], [-1, -1, -1]) # [B, T-1, O] - # Sequence length is not allowed to be less than zero. - sequence_length_less_one = math_ops.maximum( - constant_op.constant(0, dtype=sequence_length.dtype), - sequence_length - 1) - backpointers, last_score = rnn.dynamic_rnn( # [B, T - 1, O], [B, O] - crf_fwd_cell, - inputs=inputs, - sequence_length=sequence_length_less_one, - initial_state=initial_state, - time_major=False, - dtype=dtypes.int32) - backpointers = gen_array_ops.reverse_sequence( # [B, T - 1, O] - backpointers, sequence_length_less_one, seq_dim=1) - - # Computes backward decoding. Extract tag indices from backpointers. - crf_bwd_cell = CrfDecodeBackwardRnnCell(num_tags) - initial_state = math_ops.cast(math_ops.argmax(last_score, axis=1), # [B] - dtype=dtypes.int32) - initial_state = array_ops.expand_dims(initial_state, axis=-1) # [B, 1] - decode_tags, _ = rnn.dynamic_rnn( # [B, T - 1, 1] - crf_bwd_cell, - inputs=backpointers, - sequence_length=sequence_length_less_one, - initial_state=initial_state, - time_major=False, - dtype=dtypes.int32) - decode_tags = array_ops.squeeze(decode_tags, axis=[2]) # [B, T - 1] - decode_tags = array_ops.concat([initial_state, decode_tags], # [B, T] - axis=1) - decode_tags = gen_array_ops.reverse_sequence( # [B, T] - decode_tags, sequence_length, seq_dim=1) - - best_score = math_ops.reduce_max(last_score, axis=1) # [B] - return decode_tags, best_score - - return utils.smart_cond( - pred=math_ops.equal(tensor_shape.dimension_value(potentials.shape[1]) or - array_ops.shape(potentials)[1], 1), - true_fn=_single_seq_fn, - false_fn=_multi_seq_fn) diff --git a/tensorflow/contrib/cudnn_rnn/BUILD b/tensorflow/contrib/cudnn_rnn/BUILD deleted file mode 100644 index 3f43f160d5d..00000000000 --- a/tensorflow/contrib/cudnn_rnn/BUILD +++ /dev/null @@ -1,123 +0,0 @@ -# Description: -# A Cudnn RNN wrapper. -# APIs are meant to change over time. - -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = ["//visibility:private"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_custom_op_py_library( - name = "cudnn_rnn_py", - srcs = [ - "__init__.py", - "python/layers/__init__.py", - "python/layers/cudnn_rnn.py", - "python/ops/cudnn_rnn_ops.py", - ], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/contrib/checkpoint/python:split_dependency", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:cudnn_rnn_ops_gen", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:layers_base", - "//tensorflow/python:platform", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - ], -) - -cuda_py_test( - name = "cudnn_rnn_ops_test", - size = "medium", - srcs = ["python/kernel_tests/cudnn_rnn_ops_test.py"], - additional_deps = [ - ":cudnn_rnn_py", - "@absl_py//absl/testing:parameterized", - "//tensorflow/core:protos_all_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python/ops/losses:losses", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], - shard_count = 2, - tags = [ - "noasan", # http://b/62067814 - "requires-gpu-sm35", - ], -) - -cuda_py_test( - name = "cudnn_rnn_test", - size = "enormous", - srcs = ["python/kernel_tests/cudnn_rnn_test.py"], - additional_deps = [ - ":cudnn_rnn_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python/ops/losses:losses", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], - shard_count = 6, - tags = [ - "noasan", # http://b/62067814 - ], -) - -cuda_py_test( - name = "cudnn_rnn_ops_benchmark", - size = "small", - srcs = ["python/kernel_tests/cudnn_rnn_ops_benchmark.py"], - additional_deps = [ - ":cudnn_rnn_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:init_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:variables", - ], - tags = [ - "noasan", # http://b/62067814 - "nomsan", - "notsan", - ], -) diff --git a/tensorflow/contrib/cudnn_rnn/__init__.py b/tensorflow/contrib/cudnn_rnn/__init__.py deleted file mode 100644 index 53202322686..00000000000 --- a/tensorflow/contrib/cudnn_rnn/__init__.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for fused Cudnn RNN models. - -@@CudnnCompatibleGRUCell -@@CudnnCompatibleLSTMCell -@@CudnnGRU -@@CudnnLSTM -@@CudnnRNNRelu -@@CudnnRNNTanh -@@CudnnLSTMSaveable -@@CudnnGRUSaveable -@@CudnnRNNReluSaveable -@@CudnnRNNTanhSaveable -@@CudnnParamsFormatConverterLSTM -@@CudnnParamsFormatConverterGRU -@@CudnnParamsFormatConverterTanh -@@CudnnParamsFormatConverterRelu -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.cudnn_rnn.python.layers import * -# pylint: enable=unused-import,wildcard-import - - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - "CudnnCompatibleGRUCell", - "CudnnCompatibleLSTMCell", - "CudnnGRU", - "CudnnLSTM", - "CudnnRNNRelu", - "CudnnRNNTanh", - "CudnnLSTMSaveable", - "CudnnGRUSaveable", - "CudnnRNNReluSaveable", - "CudnnRNNTanhSaveable", - "CudnnParamsFormatConverterLSTM", - "CudnnParamsFormatConverterGRU", - "CudnnParamsFormatConverterTanh", - "CudnnParamsFormatConverterRelu", -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_benchmark.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_benchmark.py deleted file mode 100644 index 933df6d71dd..00000000000 --- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_benchmark.py +++ /dev/null @@ -1,170 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Benchmarks for Cudnn RNN models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time - -from six.moves import xrange # pylint: disable=redefined-builtin -from tensorflow.contrib import rnn as contrib_rnn -from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops -from tensorflow.contrib.rnn.python.ops import lstm_ops -from tensorflow.python.client import session -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import rnn -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class CudnnRNNBenchmark(test.Benchmark): - """Benchmarks Cudnn LSTM and other related models. - """ - - def _GetTestConfig(self): - return { - "large": { - "num_layers": 4, - "num_units": 1024, - "seq_length": 50, - "batch_size": 64, - }, - "medium": { - "num_layers": 4, - "num_units": 512, - "seq_length": 50, - "batch_size": 64, - }, - "small": { - "num_layers": 4, - "num_units": 128, - "seq_length": 50, - "batch_size": 64, - }, - } - - def _GetConfigDesc(self, config): - num_layers = config["num_layers"] - num_units = config["num_units"] - batch_size = config["batch_size"] - seq_length = config["seq_length"] - - return "y%d_u%d_b%d_q%d" % (num_layers, num_units, batch_size, seq_length) - - def _BenchmarkOp(self, op, desc): - burn_in_steps = 10 - benchmark_steps = 20 - with session.Session() as sess: - sess.run(variables.global_variables_initializer()) - for i in xrange(burn_in_steps + benchmark_steps): - if i == burn_in_steps: - start_time = time.time() - sess.run(op) - total_time = time.time() - start_time - step_time = total_time / benchmark_steps - print("%s takes %.4f sec/step" % (desc, step_time)) - self.report_benchmark( - name=desc, iters=benchmark_steps, wall_time=total_time) - - def benchmarkCudnnLSTMTraining(self): - test_configs = self._GetTestConfig() - for config_name, config in test_configs.items(): - config = test_configs[config_name] - num_layers = config["num_layers"] - num_units = config["num_units"] - batch_size = config["batch_size"] - seq_length = config["seq_length"] - - with ops.Graph().as_default(), ops.device("/device:GPU:0"): - model = cudnn_rnn_ops.CudnnLSTM(num_layers, num_units, num_units) - params_size_t = model.params_size() - input_data = variables.Variable( - array_ops.ones([seq_length, batch_size, num_units])) - input_h = variables.Variable( - array_ops.ones([num_layers, batch_size, num_units])) - input_c = variables.Variable( - array_ops.ones([num_layers, batch_size, num_units])) - params = variables.Variable( - array_ops.ones([params_size_t]), validate_shape=False) - output, output_h, output_c = model( - is_training=True, - input_data=input_data, - input_h=input_h, - input_c=input_c, - params=params) - all_grads = gradients_impl.gradients( - [output, output_h, output_c], - [params, input_data, input_h, input_c]) - training_op = control_flow_ops.group(*all_grads) - self._BenchmarkOp(training_op, "cudnn_lstm %s %s" % - (config_name, self._GetConfigDesc(config))) - - def benchmarkTfRNNLSTMTraining(self): - test_configs = self._GetTestConfig() - for config_name, config in test_configs.items(): - num_layers = config["num_layers"] - num_units = config["num_units"] - batch_size = config["batch_size"] - seq_length = config["seq_length"] - - with ops.Graph().as_default(), ops.device("/device:GPU:0"): - inputs = array_ops.zeros([batch_size, seq_length, num_units], - dtypes.float32) - - multi_cell = contrib_rnn.MultiRNNCell( - [contrib_rnn.BasicLSTMCell(num_units) for _ in range(num_layers)]) - outputs, final_state = rnn.dynamic_rnn( - multi_cell, inputs, dtype=dtypes.float32) - trainable_variables = ops.get_collection( - ops.GraphKeys.TRAINABLE_VARIABLES) - gradients = gradients_impl.gradients([outputs, final_state], - trainable_variables) - training_op = control_flow_ops.group(*gradients) - self._BenchmarkOp(training_op, "tf_rnn_lstm %s %s" % - (config_name, self._GetConfigDesc(config))) - - def benchmarkTfRNNLSTMBlockCellTraining(self): - test_configs = self._GetTestConfig() - for config_name, config in test_configs.items(): - num_layers = config["num_layers"] - num_units = config["num_units"] - batch_size = config["batch_size"] - seq_length = config["seq_length"] - - with ops.Graph().as_default(), ops.device("/device:GPU:0"): - inputs = array_ops.zeros([batch_size, seq_length, num_units], - dtypes.float32) - - multi_cell = contrib_rnn.MultiRNNCell( - [lstm_ops.LSTMBlockCell(num_units) for _ in range(num_layers)]) - outputs, final_state = rnn.dynamic_rnn( - multi_cell, inputs, dtype=dtypes.float32) - trainable_variables = ops.get_collection( - ops.GraphKeys.TRAINABLE_VARIABLES) - gradients = gradients_impl.gradients([outputs, final_state], - trainable_variables) - training_op = control_flow_ops.group(*gradients) - self._BenchmarkOp(training_op, "tf_rnn_lstm_block_cell %s %s" % - (config_name, self._GetConfigDesc(config))) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py deleted file mode 100644 index 7cba7937f1b..00000000000 --- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py +++ /dev/null @@ -1,1346 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Cudnn RNN models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import itertools -import os - -from absl.testing import parameterized -import numpy as np - -from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops -from tensorflow.core.protobuf import saver_pb2 -from tensorflow.python.compat import compat -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import rnn -from tensorflow.python.ops import rnn_cell_impl -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import saver as saver_lib - -CUDNN_RNN_UNIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION -CUDNN_RNN_BIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION - -CUDNN_LSTM = cudnn_rnn_ops.CUDNN_LSTM -CUDNN_GRU = cudnn_rnn_ops.CUDNN_GRU -CUDNN_RNN_RELU = cudnn_rnn_ops.CUDNN_RNN_RELU -CUDNN_RNN_TANH = cudnn_rnn_ops.CUDNN_RNN_TANH - -CUDNN_LSTM_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_LSTM_PARAMS_PER_LAYER -CUDNN_GRU_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_GRU_PARAMS_PER_LAYER -CUDNN_RNN_TANH_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_TANH_PARAMS_PER_LAYER -CUDNN_RNN_RELU_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_RELU_PARAMS_PER_LAYER - - -def RunLSTM(sess, - num_units, - input_size, - batch_size, - time, - num_layers=1, - variable_seq_lengths=False, - time_major=True, - dynamic_shape_input=False, - is_training=True, - dropout=0., - num_dirs=True, - dtype=dtypes.float32, - num_proj=None): - # TODO(jamesqin): add multi-layer tests. - # TODO(jamesqin): add multi-dir tests - assert num_layers == 1 - assert num_dirs == 1 - if is_training and not np.isclose(dropout, 0): - raise ValueError("dropout can not be 0. when test training.") - - # set graph level random seed and numpy random seed. - random_seed.set_random_seed(0) - np.random.seed(0) - - shape = ([time, batch_size, input_size] - if time_major else [batch_size, time, input_size]) - inputs_np = np.random.rand(*shape).astype(dtype.as_numpy_dtype) - inputs_static = variable_scope.get_variable( - "inputs", initializer=inputs_np, dtype=dtype) - inputs_dynamic = array_ops.placeholder( - dtype, shape=[None, None, None], name="inputs") - inputs = inputs_dynamic if dynamic_shape_input else inputs_static - unified_num_units = num_proj if num_proj else num_units - unified_num_proj = num_proj if num_proj else None - initial_h_op = variable_scope.get_variable( - "initial_h_op", - initializer=np.random.rand(batch_size, unified_num_units).astype( - dtype.as_numpy_dtype), - dtype=dtype) - initial_c_op = variable_scope.get_variable( - "initial_c_op", - initializer=np.random.rand(batch_size, - num_units).astype(dtype.as_numpy_dtype), - dtype=dtype) - - if variable_seq_lengths: - lengths_v = np.random.randint(low=1, high=time + 1, size=batch_size) - lengths_v[0] = time # make sure the max sequence has 'time' elems - lengths = ops.convert_to_tensor(lengths_v.astype(np.int32)) - else: - lengths = None - - initializer = init_ops.random_uniform_initializer( - -0.01, 0.01, dtype=dtype, seed=19980904) - - with variable_scope.variable_scope("test", initializer=initializer): - w = variable_scope.get_variable( - "rnn/lstm_cell/kernel", - shape=[input_size + unified_num_units, num_units * 4], - dtype=dtype) - b = variable_scope.get_variable( - "rnn/lstm_cell/bias", shape=[num_units * 4], dtype=dtype) - if num_proj: - pw = variable_scope.get_variable( - "rnn/lstm_cell/projection/kernel", - shape=[num_units, num_proj], - dtype=dtype) - - # canonical lstm. must set forget_bias to 0. to align with cudnn lstm. - cell = rnn_cell_impl.LSTMCell( - num_units, forget_bias=0., reuse=True, num_proj=unified_num_proj) - outputs_op, state_tuple_op = rnn.dynamic_rnn( - cell, - inputs_static, - sequence_length=lengths, - initial_state=rnn_cell_impl.LSTMStateTuple( - h=initial_h_op, c=initial_c_op), - dtype=dtype, - time_major=time_major, - scope=None) - - # Convert to cudnn opaque param. - format_converter = cudnn_rnn_ops.CudnnParamsFormatConverterLSTM( - num_layers, num_units, input_size, num_proj=unified_num_proj) - if num_proj: - opaque_params = format_converter.tf_canonical_to_opaque([w, b], [ - pw, - ]) - else: - opaque_params = format_converter.tf_canonical_to_opaque([w, b]) - - cu_initial_h_op = array_ops.expand_dims( - initial_h_op, axis=(0 if time_major else 1)) - cu_initial_c_op = array_ops.expand_dims( - initial_c_op, axis=(0 if time_major else 1)) - cu_outputs_op, cu_h_op, cu_c_op = cudnn_rnn_ops._cudnn_rnn( - inputs, - cu_initial_h_op, - cu_initial_c_op, - opaque_params, - sequence_lengths=lengths, - time_major=time_major, - dropout=dropout, - is_training=is_training, - rnn_mode=cudnn_rnn_ops.CUDNN_LSTM, - num_proj=unified_num_proj) - # Remove the trivial 1st dimension. - cu_state_tuple_op = rnn_cell_impl.LSTMStateTuple( - c=array_ops.squeeze(cu_c_op, axis=0 if time_major else 1), - h=array_ops.squeeze(cu_h_op, axis=0 if time_major else 1)) - - if is_training: - if num_proj: - (inp_grad_op, hgrad_op, cgrad_op, - wgrad_op, bgrad_op, pwgrad_op) = gradients_impl.gradients( - outputs_op, [inputs_static, initial_h_op, initial_c_op, w, b, pw]) - else: - (inp_grad_op, hgrad_op, - cgrad_op, wgrad_op, bgrad_op) = gradients_impl.gradients( - outputs_op, [inputs_static, initial_h_op, initial_c_op, w, b]) - - (cu_inp_grad_op, cu_hgrad_op, - cu_cgrad_op, opaque_grad_op) = gradients_impl.gradients( - cu_outputs_op, - [inputs, cu_initial_h_op, cu_initial_c_op, opaque_params]) - # Remove the trivial 1st dimension - cu_hgrad_op = array_ops.squeeze(cu_hgrad_op, axis=0 if time_major else 1) - # Remove the trivial 1st dimension - cu_cgrad_op = array_ops.squeeze(cu_cgrad_op, axis=0 if time_major else 1) - - if num_proj: - cu_wgrad_op, cu_bgrad_op, cu_pwgrad_op = \ - format_converter.opaque_to_tf_canonical(opaque_grad_op) - else: - cu_wgrad_op, cu_bgrad_op = format_converter.opaque_to_tf_canonical( - opaque_grad_op) - cu_wgrad_op = cu_wgrad_op[0] - cu_bgrad_op = cu_bgrad_op[0] - if num_proj: - cu_pwgrad_op = cu_pwgrad_op[0] - # cudnn lstm has 2 biases each gate. When converting to tf canonical format, - # the two biases are summed into one. Thus here bias gradient should be - # halved when comparing with tf lstm. - cu_bgrad_op *= 0.5 - - init_op = variables.global_variables_initializer() - sess.run(init_op) - - if is_training: - if num_proj: - (outputs, state_tuple, inp_grad, state_grad, wgrad, bgrad, - pwgrad) = sess.run([ - outputs_op, state_tuple_op, inp_grad_op, (hgrad_op, cgrad_op), - wgrad_op, bgrad_op, pwgrad_op - ]) - (cu_outputs, cu_state_tuple, cu_inp_grad, cu_state_grad, cu_wgrad, - cu_bgrad, cu_pwgrad) = sess.run( - [ - cu_outputs_op, cu_state_tuple_op, cu_inp_grad_op, - (cu_hgrad_op, cu_cgrad_op), cu_wgrad_op, cu_bgrad_op, - cu_pwgrad_op - ], - feed_dict={inputs: inputs_np} if dynamic_shape_input else None) - else: - outputs, state_tuple, inp_grad, state_grad, wgrad, bgrad = sess.run([ - outputs_op, state_tuple_op, inp_grad_op, (hgrad_op, cgrad_op), - wgrad_op, bgrad_op - ]) - (cu_outputs, cu_state_tuple, cu_inp_grad, cu_state_grad, cu_wgrad, - cu_bgrad) = sess.run( - [ - cu_outputs_op, cu_state_tuple_op, cu_inp_grad_op, - (cu_hgrad_op, cu_cgrad_op), cu_wgrad_op, cu_bgrad_op - ], - feed_dict={inputs: inputs_np} if dynamic_shape_input else None) - - logging.vlog(1, "outputs: %s" % outputs) - logging.vlog(1, "cu_outputs: %s" % cu_outputs) - logging.vlog(1, "state_tuple: %s" % str(state_tuple)) - logging.vlog(1, "cu_state_tuple: %s" % str(cu_state_tuple)) - logging.vlog(1, "inp_grad: %s" % inp_grad) - logging.vlog(1, "cu_inp_grad: %s" % cu_inp_grad) - logging.vlog(1, "state_grad: %s" % str(state_grad)) - logging.vlog(1, "cu_state_grad: %s" % str(cu_state_grad)) - logging.vlog(1, "wgrad: %s" % str(wgrad)) - logging.vlog(1, "bgrad: %s" % str(bgrad)) - if num_proj: - logging.vlog(1, "pwgrad: %s" % str(bgrad)) - logging.vlog(1, "cu_wgrad: %s" % str(cu_wgrad)) - logging.vlog(1, "cu_bgrad: %s" % str(cu_bgrad)) - if num_proj: - logging.vlog(1, "cu_pwgrad: %s" % str(cu_bgrad)) - if num_proj: - return (outputs, cu_outputs, state_tuple, cu_state_tuple, inp_grad, - cu_inp_grad, state_grad, cu_state_grad, wgrad, bgrad, pwgrad, - cu_wgrad, cu_bgrad, cu_pwgrad) - else: - return (outputs, cu_outputs, state_tuple, cu_state_tuple, inp_grad, - cu_inp_grad, state_grad, cu_state_grad, wgrad, bgrad, cu_wgrad, - cu_bgrad) - else: - outputs, state_tuple = sess.run([outputs_op, state_tuple_op]) - cu_outputs, cu_state_tuple = sess.run([cu_outputs_op, cu_state_tuple_op], - feed_dict=({ - inputs: inputs_np - } if dynamic_shape_input else None)) - - logging.vlog(1, "outputs: %s" % outputs) - logging.vlog(1, "cu_outputs: %s" % cu_outputs) - logging.vlog(1, "state_tuple: %s" % str(state_tuple)) - logging.vlog(1, "cu_state_tuple: %s" % str(cu_state_tuple)) - return outputs, cu_outputs, state_tuple, cu_state_tuple - - -# Basic set of RNN configs to test. They can be further extended in relevant -# test (e.g. adding num_dirs). -NAMED_RNN_TESTCASES = ({ - "testcase_name": "xsmall", - "num_units": 1, - "input_size": 1, - "batch_size": 1, - "time": 1, - "num_layers": 1, -}, { - "testcase_name": "small", - "num_units": 4, - "input_size": 4, - "batch_size": 4, - "time": 4, - "num_layers": 1, -}, { - "testcase_name": "medium", - "num_units": 128, - "input_size": 64, - "batch_size": 8, - "time": 16, - "num_layers": 1, -}, { - "testcase_name": "large", - "num_units": 128, - "input_size": 128, - "batch_size": 16, - "time": 32, - "num_layers": 1, -}) - -def ExpandNamedTestCases(inputs, *remove_keys, **extra_configs): - """Expands testcase with new config dimensions. - - Example: - inputs = ( - {'testcase_name': 'test1', 'gender': 'male'} - {'testcase_name': 'test2', 'gender': 'female'} - ) - remove_keys: empty - extra_configs = { - 'age': [40, 80] - 'height': [5, 6] - } - - Returns: - ( - {'testcase_name': 'test1_age_40_height_5','gender': 'male', 'age': - 40,'height': 5} - {'testcase_name': 'test1_age_40_height_6', 'gender': 'male', 'age': 40, - 'height': 6} - {'testcase_name': 'test1_age_80_height_5', 'gender': 'male', 'age': 80, - 'height': 5} - {'testcase_name': 'test1_age_80_height_6', 'gender': 'male', 'age': 80, - 'height': 6} - - {'testcase_name': 'test2_age_40_height_5', 'gender': 'female', 'age': - 40, - 'height': 5} - {'testcase_name': 'test2_age_40_height_6', 'gender': 'female', 'age': - 40, - 'height': 6} - {'testcase_name': 'test2_age_80_height_5', 'gender': 'female', 'age': - 80, - 'height': 5} - {'testcase_name': 'test2_age_80_height_6', 'gender': 'female', 'age': - 80, - 'height': 6} - ) - - Args: - inputs: A list of dictionary, each being a testcase. - *remove_keys: A list of keys into testcase which are not needed in new - testcases. - **extra_configs: A dict of new test dimension and applicable values in that - dimension. - - Returns: - A list of dictionary with expanded test cases. - """ - res = [] - ordered_extra_configs = collections.OrderedDict(extra_configs) - keys = ordered_extra_configs.keys() - # A list of list of configs. - # The outer loop is iterating keys, the innner is values of one key. - combined_kv = [[(k, v) for v in ordered_extra_configs[k]] for k in keys] - logging.info("combined_kv: %s", combined_kv) - - for inp in inputs: - # Each inp is a dict - for config in itertools.product(*combined_kv): - new_inp = dict(inp) - # config is a list in the form of [(k_i, v_j), (k_p, v_q), ...] - suffix = ["%s_%s" % (p[0], str(p[1])) for p in config] - suffix = "_".join(suffix) - new_inp["testcase_name"] += "_" + suffix - for k, v in config: - new_inp[k] = v - # Remove not used keys from the new test case. - if remove_keys: - if not isinstance(remove_keys, (list, tuple)): - remove_keys = [remove_keys] - for k in remove_keys: - new_inp.pop(k, None) - logging.info("new_inp: %s", new_inp) - res.append(new_inp) - # Dedup, necessary if `remove_keys` is set. - return [dict(t) for t in {tuple(d.items()) for d in res}] - - -class CudnnLSTMTest(test_util.TensorFlowTestCase, parameterized.TestCase): - - def _test_training_helper(self, - num_units, - input_size, - batch_size, - time, - num_layers, - dtype, - variable_seq_lengths, - time_major, - dynamic_shape_input=False, - rtol=3e-6, - atol=3e-6, - num_proj=None): - with self.session(use_gpu=True) as sess: - if num_proj is not None and num_proj != 0: - (outputs, cu_outputs, state_tuple, cu_state_tuple, inp_grad, - cu_inp_grad, state_grad, cu_state_grad, wgrad, bgrad, pwgrad, cu_wgrad, - cu_bgrad, cu_pwgrad) = RunLSTM( - sess, - num_units, - input_size, - batch_size, - time, - num_layers, - variable_seq_lengths=variable_seq_lengths, - dynamic_shape_input=dynamic_shape_input, - num_proj=num_proj) - else: - (outputs, cu_outputs, state_tuple, cu_state_tuple, inp_grad, - cu_inp_grad, state_grad, cu_state_grad, wgrad, bgrad, cu_wgrad, - cu_bgrad) = RunLSTM( - sess, - num_units, - input_size, - batch_size, - time, - num_layers, - variable_seq_lengths=variable_seq_lengths, - dynamic_shape_input=dynamic_shape_input, - num_proj=num_proj) - - self.assertAllClose(outputs, cu_outputs, rtol=rtol, atol=atol) - for s, cu_s in zip(state_tuple, cu_state_tuple): - self.assertAllClose(s, cu_s, rtol=rtol, atol=atol) - for sg, cu_sg in zip(state_grad, cu_state_grad): - self.assertAllClose(sg, cu_sg, rtol=rtol, atol=atol) - self.assertAllClose(inp_grad, cu_inp_grad, rtol=rtol, atol=atol) - self.assertAllClose(bgrad, cu_bgrad, rtol=rtol, atol=atol) - self.assertAllClose(wgrad, cu_wgrad, rtol=rtol, atol=atol) - if num_proj is not None and num_proj != 0: - self.assertAllClose(pwgrad, cu_pwgrad, rtol=rtol, atol=atol) - - @parameterized.named_parameters( - ExpandNamedTestCases( - NAMED_RNN_TESTCASES, **{ - "variable_seq_lengths": [True, False], - "time_major": [True, False], - "dynamic_shape_input": [True, False], - "use_proj": [True, False], - })) - @test_util.run_gpu_only - def test_training(self, num_units, input_size, batch_size, time, num_layers, - variable_seq_lengths, time_major, dynamic_shape_input, - use_proj): - with compat.forward_compatibility_horizon(2019, 6, 27): - num_proj = num_units // 2 - if use_proj and num_proj == 0: - self.skipTest("num_proj cannot be 0") - self._test_training_helper( - num_units, - input_size, - batch_size, - time, - num_layers, - dtypes.float32, - variable_seq_lengths=variable_seq_lengths, - time_major=time_major, - dynamic_shape_input=dynamic_shape_input, - num_proj=num_proj if use_proj else None) - - @parameterized.named_parameters( - ExpandNamedTestCases( - NAMED_RNN_TESTCASES, **{ - "variable_seq_lengths": [True, False], - "time_major": [True, False], - "dynamic_shape_input": [True, False], - "use_proj": [True, False], - })) - @test_util.run_gpu_only - def test_training_fp16(self, num_units, input_size, batch_size, time, - num_layers, variable_seq_lengths, time_major, - dynamic_shape_input, use_proj): - with compat.forward_compatibility_horizon(2019, 6, 27): - num_proj = num_units // 2 - if use_proj and num_proj == 0: - self.skipTest("num_proj cannot be 0") - self._test_training_helper( - num_units, - input_size, - batch_size, - time, - num_layers, - dtypes.float16, - rtol=5e-3, - atol=5e-4, - variable_seq_lengths=variable_seq_lengths, - time_major=time_major, - dynamic_shape_input=dynamic_shape_input, - num_proj=num_proj if use_proj else None) - - @parameterized.named_parameters( - ExpandNamedTestCases( - NAMED_RNN_TESTCASES, **{ - "variable_seq_lengths": [True, False], - "time_major": [True, False], - "dynamic_shape_input": [True, False], - "use_proj": [True, False], - })) - @test_util.run_gpu_only - def test_inference(self, num_units, input_size, batch_size, time, num_layers, - variable_seq_lengths, time_major, dynamic_shape_input, - use_proj): - with compat.forward_compatibility_horizon(2019, 6, 27): - num_proj = num_units // 2 - if use_proj and num_proj == 0: - self.skipTest("num_proj cannot be 0") - with self.session(use_gpu=True) as sess: - (outputs, cu_outputs, state_tuple, cu_state_tuple) = RunLSTM( - sess, - num_units, - input_size, - batch_size, - time, - num_layers, - is_training=False, - variable_seq_lengths=variable_seq_lengths, - time_major=time_major, - dynamic_shape_input=dynamic_shape_input, - num_proj=num_proj if use_proj else None) - - self.assertAllClose(outputs, cu_outputs) - # h - self.assertAllClose(state_tuple.h, cu_state_tuple.h) - # c - self.assertAllClose(state_tuple.c, cu_state_tuple.c) - - @parameterized.named_parameters( - ExpandNamedTestCases( - NAMED_RNN_TESTCASES, **{ - "variable_seq_lengths": [True, False], - "time_major": [True, False], - "dynamic_shape_input": [True, False], - "use_proj": [True, False], - })) - @test_util.run_gpu_only - def test_inference_fp16(self, num_units, input_size, batch_size, time, - num_layers, variable_seq_lengths, time_major, - dynamic_shape_input, use_proj): - with compat.forward_compatibility_horizon(2019, 6, 27): - num_proj = num_units // 2 - if use_proj and num_proj == 0: - self.skipTest("num_proj cannot be 0") - with self.session(use_gpu=True) as sess: - (outputs, cu_outputs, state_tuple, cu_state_tuple) = RunLSTM( - sess, - num_units, - input_size, - batch_size, - time, - num_layers, - is_training=False, - dtype=dtypes.float16, - variable_seq_lengths=variable_seq_lengths, - time_major=time_major, - dynamic_shape_input=dynamic_shape_input, - num_proj=num_proj if use_proj else None) - - rtol, atol = 5e-3, 5e-4 - self.assertAllClose(outputs, cu_outputs, rtol=rtol, atol=atol) - # h - self.assertAllClose( - state_tuple.h, cu_state_tuple.h, rtol=rtol, atol=atol) - # c - self.assertAllClose( - state_tuple.c, cu_state_tuple.c, rtol=rtol, atol=atol) - - @parameterized.named_parameters( - ExpandNamedTestCases( - NAMED_RNN_TESTCASES, **{ - "variable_seq_lengths": [True, False], - "time_major": [True, False], - "dynamic_shape_input": [True, False], - "use_proj": [True, False], - })) - @test_util.run_gpu_only - def test_inference_with_dropout(self, num_units, input_size, batch_size, time, - num_layers, variable_seq_lengths, time_major, - dynamic_shape_input, use_proj): - """Validates that dropout does not affect Cudnn Rnn inference.""" - with compat.forward_compatibility_horizon(2019, 6, 27): - num_proj = num_units // 2 - if use_proj and num_proj == 0: - self.skipTest("num_proj cannot be 0") - # Hand-picked dropouts are used below (0. and 1.) - with ops.Graph().as_default() as g: - with self.session(use_gpu=True, graph=g) as sess: - # 1st time w/o dropout. - (_, cu_outputs, _, cu_state_tuple) = RunLSTM( - sess, - num_units, - input_size, - batch_size, - time, - num_layers, - is_training=False, - dropout=0., - variable_seq_lengths=variable_seq_lengths, - time_major=time_major, - dynamic_shape_input=dynamic_shape_input, - num_proj=num_proj if use_proj else None) - - with ops.Graph().as_default() as g: - with self.session(use_gpu=True, graph=g) as sess: - (_, cu_outputs2, _, cu_state_tuple2) = RunLSTM( - sess, - num_units, - input_size, - batch_size, - time, - num_layers, - is_training=False, - dropout=1., - variable_seq_lengths=variable_seq_lengths, - time_major=time_major, - dynamic_shape_input=dynamic_shape_input, - num_proj=num_proj if use_proj else None) - - self.assertAllClose(cu_outputs, cu_outputs2) - # h - self.assertAllClose(cu_state_tuple.h, cu_state_tuple2.h) - # c - self.assertAllClose(cu_state_tuple.c, cu_state_tuple2.c) - - -def RunGRU(sess, - num_units, - input_size, - batch_size, - time, - num_layers=1, - is_training=True, - variable_seq_lengths=False, - time_major=True, - dynamic_shape_input=False, - dropout=0., - num_dirs=True, - dtype=dtypes.float32): - # TODO(jamesqin): add multi-layer tests. - # TODO(jamesqin): add multi-dir tests - assert num_layers == 1 - assert num_dirs == 1 - if is_training and not np.isclose(dropout, 0): - raise ValueError("dropout can not be 0. when test training.") - - # set graph level random seed and numpy random seed. - random_seed.set_random_seed(0) - np.random.seed(0) - - shape = ([time, batch_size, input_size] - if time_major else [batch_size, time, input_size]) - inputs_np = np.random.rand(*shape).astype(dtype.as_numpy_dtype) - inputs_static = variable_scope.get_variable( - "inputs", initializer=inputs_np, dtype=dtype) - inputs_dynamic = array_ops.placeholder( - dtype, shape=[None, None, None], name="inputs") - inputs = inputs_dynamic if dynamic_shape_input else inputs_static - initial_h_op = variable_scope.get_variable( - "initial_h_op", - initializer=np.random.rand(batch_size, - num_units).astype(dtype.as_numpy_dtype), - dtype=dtype) - - if variable_seq_lengths: - lengths_v = np.random.randint(low=1, high=time + 1, size=batch_size) - lengths_v[0] = time # make sure the max sequence has 'time' elems - lengths = ops.convert_to_tensor(lengths_v.astype(np.int32)) - else: - lengths = None - - initializer = init_ops.random_uniform_initializer( - -0.01, 0.01, dtype=dtype, seed=19980904) - with variable_scope.variable_scope("test", initializer=initializer): - gate_kernel = variable_scope.get_variable( - "rnn/cudnn_compatible_gru_cell/gates/kernel", - shape=[input_size + num_units, num_units * 2], - dtype=dtype) - gate_bias = variable_scope.get_variable( - "rnn/cudnn_compatible_gru_cell/gates/bias", - shape=[num_units * 2], - dtype=dtype) - candidate_inp_kernel = variable_scope.get_variable( - "rnn/cudnn_compatible_gru_cell/candidate/input_projection/kernel", - shape=[input_size, num_units], - dtype=dtype) - candidate_inp_bias = variable_scope.get_variable( - "rnn/cudnn_compatible_gru_cell/candidate/input_projection/bias", - shape=[num_units], - dtype=dtype) - candidate_hid_kernel = variable_scope.get_variable( - "rnn/cudnn_compatible_gru_cell/candidate/hidden_projection/kernel", - shape=[num_units, num_units], - dtype=dtype) - candidate_hid_bias = variable_scope.get_variable( - "rnn/cudnn_compatible_gru_cell/candidate/hidden_projection/bias", - shape=[num_units], - dtype=dtype) - - cell = cudnn_rnn_ops.CudnnCompatibleGRUCell(num_units, reuse=True) - outputs_op, h_op = rnn.dynamic_rnn( - cell, - inputs_static, - sequence_length=lengths, - initial_state=initial_h_op, - dtype=dtype, - time_major=time_major, - scope=None) - - ws = [gate_kernel, candidate_inp_kernel, candidate_hid_kernel] - bs = [gate_bias, candidate_inp_bias, candidate_hid_bias] - # Convert to cudnn opaque param. - format_converter = cudnn_rnn_ops.CudnnParamsFormatConverterGRU( - num_layers, num_units, input_size) - opaque_params = format_converter.tf_canonical_to_opaque(ws + bs) - - - cu_initial_h_op = array_ops.expand_dims( - initial_h_op, axis=(0 if time_major else 1)) - cu_outputs_op, cu_h_op, _ = cudnn_rnn_ops._cudnn_rnn( - inputs, - cu_initial_h_op, - array_ops.zeros_like(cu_initial_h_op), # not used - opaque_params, - sequence_lengths=lengths, - time_major=time_major, - dropout=dropout, - is_training=is_training, - rnn_mode=cudnn_rnn_ops.CUDNN_GRU) - - if is_training: - (inp_grad_op, hgrad_op, gk_grad_op, cik_grad_op, chk_grad_op, gb_grad_op, - cib_grad_op, chb_grad_op) = gradients_impl.gradients( - outputs_op, [inputs_static, initial_h_op] + ws + bs) - - (cu_inp_grad_op, cu_hgrad_op, opaque_grad_op) = gradients_impl.gradients( - cu_outputs_op, [inputs, cu_initial_h_op, opaque_params]) - # Remove the trivial 1st dimension - cu_hgrad_op = array_ops.squeeze(cu_hgrad_op, axis=0 if time_major else 1) - - cu_wgrad_op, cu_bgrad_op = format_converter.opaque_to_tf_canonical( - opaque_grad_op) - (cu_gk_grad_op, cu_cik_grad_op, cu_chk_grad_op) = cu_wgrad_op - (cu_gb_grad_op, cu_cib_grad_op, cu_chb_grad_op) = cu_bgrad_op - # cudnn gru has 2 biases for reset and update gates. When converting to tf - # canonical format, the two biases are summed into one. Thus here relevant - # bias gradient should be halved before comparing with tf gru. - cu_gb_grad_op *= 0.5 - - init_op = variables.global_variables_initializer() - sess.run(init_op) - - if is_training: - outputs, h, inp_grad, hgrad, wgrad, bgrad = sess.run([ - outputs_op, h_op, inp_grad_op, hgrad_op, - (gk_grad_op, cik_grad_op, chk_grad_op), - (gb_grad_op, cib_grad_op, chb_grad_op) - ]) - (cu_outputs, cu_h, cu_inp_grad, cu_hgrad, cu_wgrad, cu_bgrad) = sess.run( - [ - cu_outputs_op, cu_h_op, cu_inp_grad_op, cu_hgrad_op, - (cu_gk_grad_op, cu_cik_grad_op, cu_chk_grad_op), - (cu_gb_grad_op, cu_cib_grad_op, cu_chb_grad_op) - ], - feed_dict={inputs: inputs_np} if dynamic_shape_input else None) - # Remove the trivial 1st dimension - cu_h = np.squeeze(cu_h, axis=0 if time_major else 1) - - logging.vlog(1, "outputs: %s" % outputs) - logging.vlog(1, "cu_outputs: %s" % cu_outputs) - logging.vlog(1, "h: %s" % h) - logging.vlog(1, "cu_h: %s" % h) - logging.vlog(1, "inp_grad: %s" % inp_grad) - logging.vlog(1, "cu_inp_grad: %s" % cu_inp_grad) - logging.vlog(1, "hgrad: %s" % hgrad) - logging.vlog(1, "cu_hgrad: %s" % cu_hgrad) - logging.vlog(1, "wgrad: %s" % str(wgrad)) - logging.vlog(1, "bgrad: %s" % str(bgrad)) - logging.vlog(1, "cu_wgrad: %s" % str(cu_wgrad)) - logging.vlog(1, "cu_bgrad: %s" % str(cu_bgrad)) - return (outputs, cu_outputs, h, cu_h, inp_grad, cu_inp_grad, hgrad, - cu_hgrad, wgrad, bgrad, cu_wgrad, cu_bgrad) - else: - outputs, h = sess.run([outputs_op, h_op]) - cu_outputs, cu_h = sess.run([cu_outputs_op, cu_h_op], - feed_dict=({ - inputs: inputs_np - } if dynamic_shape_input else None)) - # Remove the trivial 1st dimension. - cu_h = np.squeeze(cu_h, axis=0 if time_major else 1) - - logging.vlog(1, "outputs: %s" % outputs) - logging.vlog(1, "cu_outputs: %s" % cu_outputs) - logging.vlog(1, "h: %s" % h) - logging.vlog(1, "cu_h: %s" % h) - return outputs, cu_outputs, h, cu_h - - -class CudnnGRUTest(test_util.TensorFlowTestCase, parameterized.TestCase): - - def _test_training_helper(self, - num_units, - input_size, - batch_size, - time, - num_layers, - dtype, - variable_seq_lengths, - time_major, - dynamic_shape_input=False, - rtol=3e-6, - atol=3e-6): - with self.session(use_gpu=True) as sess: - (outputs, cu_outputs, h, cu_h, inp_grad, cu_inp_grad, hgrad, cu_hgrad, - wgrad, bgrad, cu_wgrad, cu_bgrad) = RunGRU( - sess, - num_units, - input_size, - batch_size, - time, - num_layers, - variable_seq_lengths=variable_seq_lengths, - time_major=time_major, - dynamic_shape_input=dynamic_shape_input) - - self.assertAllClose(outputs, cu_outputs, rtol=rtol, atol=atol) - self.assertAllClose(h, cu_h, rtol=rtol, atol=atol) - self.assertAllClose(hgrad, cu_hgrad, rtol=rtol, atol=atol) - self.assertAllClose(inp_grad, cu_inp_grad, rtol=rtol, atol=atol) - for bg, cu_bg in zip(bgrad, cu_bgrad): - self.assertAllClose(bg, cu_bg, rtol=rtol, atol=atol) - for wg, cu_wg in zip(wgrad, cu_wgrad): - self.assertAllClose(wg, cu_wg, rtol=rtol, atol=atol) - - @parameterized.named_parameters( - ExpandNamedTestCases( - NAMED_RNN_TESTCASES, **{ - "variable_seq_lengths": [True, False], - "time_major": [True, False], - "dynamic_shape_input": [True, False], - })) - @test_util.run_gpu_only - def test_training(self, num_units, input_size, batch_size, time, num_layers, - variable_seq_lengths, time_major, dynamic_shape_input): - self._test_training_helper( - num_units, - input_size, - batch_size, - time, - num_layers, - dtypes.float32, - variable_seq_lengths=variable_seq_lengths, - time_major=time_major, - dynamic_shape_input=dynamic_shape_input) - - @parameterized.named_parameters( - ExpandNamedTestCases( - NAMED_RNN_TESTCASES, **{ - "variable_seq_lengths": [True, False], - "time_major": [True, False], - "dynamic_shape_input": [True, False], - })) - @test_util.run_gpu_only - def test_training_fp16(self, num_units, input_size, batch_size, time, - num_layers, variable_seq_lengths, time_major, - dynamic_shape_input): - self._test_training_helper( - num_units, - input_size, - batch_size, - time, - num_layers, - dtypes.float16, - rtol=5e-3, - atol=5e-4, - variable_seq_lengths=variable_seq_lengths, - time_major=time_major, - dynamic_shape_input=dynamic_shape_input) - - @parameterized.named_parameters( - ExpandNamedTestCases( - NAMED_RNN_TESTCASES, **{ - "variable_seq_lengths": [True, False], - "time_major": [True, False], - "dynamic_shape_input": [True, False], - })) - @test_util.run_gpu_only - def test_inference(self, num_units, input_size, batch_size, time, num_layers, - variable_seq_lengths, time_major, dynamic_shape_input): - with self.session(use_gpu=True) as sess: - (outputs, cu_outputs, h, cu_h) = RunGRU( - sess, - num_units, - input_size, - batch_size, - time, - num_layers, - is_training=False, - variable_seq_lengths=variable_seq_lengths, - time_major=time_major, - dynamic_shape_input=dynamic_shape_input) - self.assertAllClose(outputs, cu_outputs) - self.assertAllClose(h, cu_h) - - @parameterized.named_parameters( - ExpandNamedTestCases( - NAMED_RNN_TESTCASES, **{ - "variable_seq_lengths": [True, False], - "time_major": [True, False], - "dynamic_shape_input": [True, False], - })) - @test_util.run_gpu_only - def test_inference_fp16(self, num_units, input_size, batch_size, time, - num_layers, variable_seq_lengths, time_major, - dynamic_shape_input): - with self.session(use_gpu=True) as sess: - (outputs, cu_outputs, h, cu_h) = RunGRU( - sess, - num_units, - input_size, - batch_size, - time, - num_layers, - is_training=False, - dtype=dtypes.float16, - variable_seq_lengths=variable_seq_lengths, - time_major=time_major, - dynamic_shape_input=dynamic_shape_input) - - rtol, atol = 5e-3, 5e-4 - self.assertAllClose(outputs, cu_outputs, rtol=rtol, atol=atol) - self.assertAllClose(h, cu_h, rtol=rtol, atol=atol) - - @parameterized.named_parameters( - ExpandNamedTestCases( - NAMED_RNN_TESTCASES, **{ - "variable_seq_lengths": [True, False], - "time_major": [True, False], - "dynamic_shape_input": [True, False], - })) - @test_util.run_gpu_only - def test_inference_with_dropout(self, num_units, input_size, batch_size, time, - num_layers, variable_seq_lengths, time_major, - dynamic_shape_input): - """Validates that dropout does not affect Cudnn Rnn inference.""" - # Hand-picked dropouts are used below (0. and 1.) - with ops.Graph().as_default() as g: - with self.session(use_gpu=True, graph=g) as sess: - # 1st time w/o dropout. - (_, cu_outputs, _, cu_h) = RunGRU( - sess, - num_units, - input_size, - batch_size, - time, - num_layers, - is_training=False, - dropout=0., - variable_seq_lengths=variable_seq_lengths, - time_major=time_major, - dynamic_shape_input=dynamic_shape_input) - - with ops.Graph().as_default() as g: - with self.session(use_gpu=True, graph=g) as sess: - (_, cu_outputs2, _, cu_h2) = RunGRU( - sess, - num_units, - input_size, - batch_size, - time, - num_layers, - is_training=False, - dropout=1., - variable_seq_lengths=variable_seq_lengths, - time_major=time_major, - dynamic_shape_input=dynamic_shape_input) - - self.assertAllClose(cu_outputs, cu_outputs2) - self.assertAllClose(cu_h[0], cu_h2[0]) - - -class CudnnParamsFormatConverterTest(test_util.TensorFlowTestCase, - parameterized.TestCase): - """Class for testing various format converters.""" - - def _test_lstm_helper(self, - num_units, - input_size, - num_layers, - direction, - num_proj=None): - with self.session(use_gpu=True) as sess: - random_seed.set_random_seed(0) - np.random.seed(0) - - num_dirs = 1 if direction == cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION else 2 - format_converter = cudnn_rnn_ops.CudnnParamsFormatConverterLSTM( - num_layers, - num_units, - input_size, - direction=direction, - num_proj=num_proj if num_proj else None) - - ws, bs, pws = [], [], [] - for _ in range(num_layers * num_dirs): - w = constant_op.constant( - np.random.rand(input_size + (num_proj if num_proj else num_units), - 4 * num_units), - dtype=dtypes.float32) - b = constant_op.constant( - np.random.rand(4 * num_units), dtype=dtypes.float32) - ws.append(w) - bs.append(b) - if num_proj: - pw = constant_op.constant( - np.random.rand(num_units, num_proj), dtype=dtypes.float32) - pws.append(pw) - - if num_proj: - opaque_params = format_converter.tf_canonical_to_opaque(ws + bs, pws) - else: - opaque_params = format_converter.tf_canonical_to_opaque(ws + bs) - - opaque_params_size = cudnn_rnn_ops.cudnn_rnn_opaque_params_size( - cudnn_rnn_ops.CUDNN_LSTM, - num_layers, - num_units, - input_size, - direction=direction, - num_proj=num_proj if num_proj else None) - - if num_proj: - ws_r, bs_r, pws_r = format_converter.opaque_to_tf_canonical( - opaque_params) - ws, ws_r, pws, bs, bs_r, pws_r = sess.run( - [ws, ws_r, pws, bs, bs_r, pws_r]) - else: - ws_r, bs_r = format_converter.opaque_to_tf_canonical(opaque_params) - ws, ws_r, bs, bs_r = sess.run([ws, ws_r, bs, bs_r]) - - # Test tf_canonical_to_opaque() followed by opaque_to_tf_canonical() - # returns the original input. - for w, w_r in zip(ws, ws_r): - self.assertAllClose(w, w_r) - if num_proj: - for pw, pw_r in zip(pws, pws_r): - self.assertAllClose(pw, pw_r) - for b, b_r in zip(bs, bs_r): - self.assertAllClose(b, b_r) - - # Test opaque_params size lower bound - opaque_params_size_v = sess.run(opaque_params_size) - min_params_size = sum(x.size for x in ws) + np.sum(x.size for x in bs) - logging.info("min_parm_size: %d vs actual_opaque_param_size: %d", - min_params_size, opaque_params_size_v) - self.assertLessEqual(min_params_size, opaque_params_size_v) - - @parameterized.named_parameters((c["testcase_name"], c["num_units"], - c["input_size"], c["num_layers"]) - for c in NAMED_RNN_TESTCASES) - @test_util.run_gpu_only - def test_lstm(self, num_units, input_size, num_layers): - self._test_lstm_helper(num_units, input_size, num_layers, - cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION) - - @parameterized.named_parameters( - (c["testcase_name"], c["num_units"], c["input_size"], c["num_layers"]) - for c in NAMED_RNN_TESTCASES) - @test_util.run_gpu_only - def test_lstmp(self, num_units, input_size, num_layers): - with compat.forward_compatibility_horizon(2019, 6, 27): - num_proj = num_units // 2 - if num_proj == 0: - self.skipTest("num_proj cannot be 0") - self._test_lstm_helper( - num_units, - input_size, - num_layers, - cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION, - num_proj=num_proj) - - @parameterized.named_parameters((c["testcase_name"], c["num_units"], - c["input_size"], c["num_layers"]) - for c in NAMED_RNN_TESTCASES) - @test_util.run_gpu_only - def test_lstm_bidi(self, num_units, input_size, num_layers): - self._test_lstm_helper(num_units, input_size, num_layers, - cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION) - - @parameterized.named_parameters( - (c["testcase_name"], c["num_units"], c["input_size"], c["num_layers"]) - for c in NAMED_RNN_TESTCASES) - @test_util.run_gpu_only - def test_lstmp_bidi(self, num_units, input_size, num_layers): - with compat.forward_compatibility_horizon(2019, 6, 27): - num_proj = num_units // 2 - if num_proj == 0: - self.skipTest("num_proj cannot be 0") - self._test_lstm_helper( - num_units, - input_size, - num_layers, - cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION, - num_proj=num_proj) - - def _test_gru_helper(self, num_units, input_size, num_layers, direction): - with self.session(use_gpu=True) as sess: - random_seed.set_random_seed(0) - np.random.seed(0) - - num_dirs = 1 if direction == cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION else 2 - format_converter = cudnn_rnn_ops.CudnnParamsFormatConverterGRU( - num_layers, num_units, input_size, direction=direction) - - ws, bs = [], [] - for _ in range(num_layers * num_dirs): - gate_kernel = constant_op.constant( - np.random.rand(input_size + num_units, num_units * 2), - dtype=dtypes.float32) - gate_bias = constant_op.constant( - np.random.rand(num_units * 2), dtype=dtypes.float32) - candidate_inp_kernel = constant_op.constant( - np.random.rand(input_size, num_units), dtype=dtypes.float32) - candidate_inp_bias = constant_op.constant( - np.random.rand(num_units), dtype=dtypes.float32) - candidate_hid_kernel = constant_op.constant( - np.random.rand(num_units, num_units), dtype=dtypes.float32) - candidate_hid_bias = constant_op.constant( - np.random.rand(num_units), dtype=dtypes.float32) - ws.extend([gate_kernel, candidate_inp_kernel, candidate_hid_kernel]) - bs.extend([gate_bias, candidate_inp_bias, candidate_hid_bias]) - - opaque_params = format_converter.tf_canonical_to_opaque(ws + bs) - opaque_params_size = cudnn_rnn_ops.cudnn_rnn_opaque_params_size( - cudnn_rnn_ops.CUDNN_GRU, - num_layers, - num_units, - input_size, - direction=direction) - - ws_r, bs_r = format_converter.opaque_to_tf_canonical(opaque_params) - - # Test tf_canonical_to_opaque() followed by opaque_to_tf_canonical() - # returns the original input. - ws, ws_r, bs, bs_r = sess.run([ws, ws_r, bs, bs_r]) - for w, w_r in zip(ws, ws_r): - self.assertAllClose(w, w_r) - for b, b_r in zip(bs, bs_r): - self.assertAllClose(b, b_r) - - # Test opaque_params size lower bound - opaque_params_size_v = sess.run(opaque_params_size) - min_params_size = sum(x.size for x in ws) + sum(x.size for x in bs) - logging.info("min_parm_size: %d vs actual_opaque_param_size: %d", - min_params_size, opaque_params_size_v) - self.assertLessEqual(min_params_size, opaque_params_size_v) - - @parameterized.named_parameters((c["testcase_name"], c["num_units"], - c["input_size"], c["num_layers"]) - for c in NAMED_RNN_TESTCASES) - @test_util.run_gpu_only - def test_gru(self, num_units, input_size, num_layers): - self._test_gru_helper(num_units, input_size, num_layers, - cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION) - - @parameterized.named_parameters((c["testcase_name"], c["num_units"], - c["input_size"], c["num_layers"]) - for c in NAMED_RNN_TESTCASES) - @test_util.run_gpu_only - def test_gru_bidi(self, num_units, input_size, num_layers): - self._test_gru_helper(num_units, input_size, num_layers, - cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION) - - -class CudnnRnnSaveRestoreTest(test_util.TensorFlowTestCase, - parameterized.TestCase): - """Class for testing various Cudnn Rnn SaveableObjects.""" - - def _create_opaque_param(self, - rnn_mode, - num_units, - input_size, - num_layers, - direction, - name=None): - param_size_t = cudnn_rnn_ops.cudnn_rnn_opaque_params_size( - rnn_mode, num_layers, num_units, input_size, direction=direction) - init_val = random_ops.random_uniform([param_size_t]) - return variable_scope.get_variable( - name or "opaque_param", initializer=init_val, validate_shape=False) - - def _create_saveable(self, opaque_param, rnn_mode, num_units, input_size, - num_layers, direction): - if rnn_mode == CUDNN_LSTM: - fn = cudnn_rnn_ops.CudnnLSTMSaveable - elif rnn_mode == CUDNN_GRU: - fn = cudnn_rnn_ops.CudnnGRUSaveable - elif rnn_mode == CUDNN_RNN_TANH: - fn = cudnn_rnn_ops.CudnnRNNTanhSaveable - elif rnn_mode == CUDNN_RNN_RELU: - fn = cudnn_rnn_ops.CudnnRNNReluSaveable - saveable = fn( - opaque_param, num_layers, num_units, input_size, direction=direction) - return saveable - - def _compare_weights(self, lhs, rhs): - self.assertLen(rhs, len(lhs)) - for lw, rw in zip(lhs, rhs): - self.assertAllEqual(lw, rw) - - def _compare_biases(self, lhs, rhs): - self.assertLen(rhs, len(lhs)) - for lf, rt in zip(lhs, rhs): - self.assertAllEqual(lf, rt) - - @parameterized.named_parameters( - ExpandNamedTestCases( - NAMED_RNN_TESTCASES, "time", "batch_size", **{ - "rnn_mode": [ - CUDNN_LSTM, CUDNN_GRU, CUDNN_RNN_RELU, CUDNN_RNN_TANH - ], - "direction": [CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION] - })) - @test_util.run_gpu_only - def test_save_restore_variable(self, rnn_mode, num_units, input_size, - num_layers, direction): - # Verify the restored opaque param, once converted to tf_canonical format, - # is the same as the tf canonicals of the pre-restored param. - with self.session(use_gpu=True) as sess: - opaque_param = self._create_opaque_param(rnn_mode, num_units, input_size, - num_layers, direction) - saveable = self._create_saveable(opaque_param, rnn_mode, num_units, - input_size, num_layers, direction) - ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) - weights_op, biases_op = saveable.format_converter.opaque_to_tf_canonical( - saveable._variables) - - save_path = os.path.join(self.get_temp_dir(), "save_restore_var_test") - saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2) - - init_op = variables.global_variables_initializer() - reset_op = state_ops.assign(opaque_param, - array_ops.zeros_like(opaque_param)) - sess.run(init_op) - self.assertEqual(save_path, saver.save(sess, save_path)) - - # Get the tf canonical vals before reset-restore - weights, biases = sess.run([weights_op, biases_op]) - - # Reset the opaque param value - sess.run(reset_op) - # Assert reset happened. - weights_z, biases_z = sess.run([weights_op, biases_op]) - for w in weights_z: - self.assertAllClose(w, np.zeros_like(w)) - for b in biases_z: - self.assertAllClose(b, np.zeros_like(b)) - - # Restore opaque param value from checkpoint. - saver.restore(sess, save_path) - weights_r, biases_r = sess.run([weights_op, biases_op]) - self._compare_weights(weights, weights_r) - self._compare_biases(biases, biases_r) - - @parameterized.named_parameters( - ExpandNamedTestCases( - NAMED_RNN_TESTCASES, "time", "batch_size", **{ - "rnn_mode": [ - CUDNN_LSTM, CUDNN_GRU, CUDNN_RNN_RELU, CUDNN_RNN_TANH - ], - "direction": [CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION] - })) - @test_util.run_gpu_only - def test_save_restore_multi_variables(self, rnn_mode, num_units, input_size, - num_layers, direction): - # Verify the restored opaque param, once converted to tf_canonical format, - # is the same as the tf canonicals of the pre-restored param. - with self.session(use_gpu=True) as sess: - opaque_params = [] - saveables = [] - num_opaque_params = 2 - for i in range(num_opaque_params): - opaque_params.append( - self._create_opaque_param( - rnn_mode, - num_units, - input_size, - num_layers, - direction, - name="opaque_param_%d" % i)) - saveable = self._create_saveable(opaque_params[i], rnn_mode, num_units, - input_size, num_layers, direction) - ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) - saveables.append(saveable) - - weights_ops, biases_ops = [], [] - for i in range(num_opaque_params): - weights_op, biases_op = ( - saveables[i].format_converter.opaque_to_tf_canonical( - saveables[i]._variables)) - weights_ops.append(weights_op) - biases_ops.append(biases_op) - - save_path = os.path.join(self.get_temp_dir(), "save_restore_var_test") - saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2) - - init_op = variables.global_variables_initializer() - reset_ops = [] - for i in range(num_opaque_params): - reset_ops.append( - state_ops.assign(opaque_params[i], - array_ops.zeros_like(opaque_params[i]))) - sess.run(init_op) - self.assertEqual(save_path, saver.save(sess, save_path)) - - # Get the tf canonical vals before reset-restore - for i in range(num_opaque_params): - weights, biases = sess.run([weights_ops[i], biases_ops[i]]) - - # Reset the opaque param value - sess.run(reset_ops[i]) - - # Assert reset happened. - weights_z, biases_z = sess.run([weights_ops[i], biases_ops[i]]) - for w in weights_z: - self.assertAllClose(w, np.zeros_like(w)) - for b in biases_z: - self.assertAllClose(b, np.zeros_like(b)) - - # Restore opaque param value from checkpoint. - saver.restore(sess, save_path) - weights_r, biases_r = sess.run([weights_ops[i], biases_ops[i]]) - self._compare_weights(weights, weights_r) - self._compare_biases(biases, biases_r) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py deleted file mode 100644 index 5831781c2ac..00000000000 --- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py +++ /dev/null @@ -1,1536 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Cudnn RNN models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import functools -import itertools -import os -import sys -import unittest - -import numpy as np - -from tensorflow.contrib.cudnn_rnn.python.layers import cudnn_rnn -from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops -from tensorflow.contrib.rnn.python.ops import rnn as contrib_rnn_lib -from tensorflow.python.eager import backprop -from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_nn_ops -from tensorflow.python.ops import gradients_impl as gradients -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import rnn as rnn_lib -from tensorflow.python.ops import rnn_cell_impl -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.ops import variables -from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import googletest -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import adagrad -from tensorflow.python.training import adam -from tensorflow.python.training import gradient_descent -from tensorflow.python.training import momentum -from tensorflow.python.training import rmsprop -from tensorflow.python.training import saver as saver_lib -from tensorflow.python.training.tracking import util as trackable_utils -from tensorflow.python.util.compat import collections_abc - - -CUDNN_LSTM = cudnn_rnn_ops.CUDNN_LSTM -CUDNN_GRU = cudnn_rnn_ops.CUDNN_GRU -CUDNN_RNN_RELU = cudnn_rnn_ops.CUDNN_RNN_RELU -CUDNN_RNN_TANH = cudnn_rnn_ops.CUDNN_RNN_TANH -CUDNN_RNN_UNIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION -CUDNN_RNN_BIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION - -CUDNN_LSTM_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_LSTM_PARAMS_PER_LAYER -CUDNN_GRU_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_GRU_PARAMS_PER_LAYER -CUDNN_RNN_TANH_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_TANH_PARAMS_PER_LAYER -CUDNN_RNN_RELU_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_RELU_PARAMS_PER_LAYER - - -class CudnnTestModel(object): - """Model with convenient APIs for easier building and running test graph. - - The graph built is used by all tests below to avoid repeatedly building - similar test graphs. - """ - - def __init__(self, - rnn_mode, - num_layers, - num_units, - input_size, - direction=CUDNN_RNN_UNIDIRECTION, - dropout=0., - dtype=dtypes.float32, - training=False, - seed=None, - kernel_initializer=None, - bias_initializer=None): - if dtype not in (dtypes.float16, dtypes.float32, dtypes.float64): - raise ValueError("Invalid dtype: %s" % dtype) - self._dtype = dtype - - self._inputs = array_ops.placeholder( - dtype=dtype, shape=[None, None, input_size], name="inputs") - h = array_ops.placeholder( - dtype=dtype, shape=[None, None, num_units], name="h") - c = array_ops.placeholder( - dtype=dtype, shape=[None, None, num_units], name="c") - if rnn_mode == CUDNN_LSTM: - model_fn = cudnn_rnn.CudnnLSTM - self._initial_state = (h, c) - elif rnn_mode == CUDNN_GRU: - model_fn = cudnn_rnn.CudnnGRU - self._initial_state = (h,) - elif rnn_mode == CUDNN_RNN_TANH: - model_fn = cudnn_rnn.CudnnRNNTanh - self._initial_state = (h,) - elif rnn_mode == CUDNN_RNN_RELU: - model_fn = cudnn_rnn.CudnnRNNRelu - self._initial_state = (h,) - else: - raise ValueError("Invalid rnn_mode: %s" % rnn_mode) - self._rnn = model_fn( - num_layers, - num_units, - direction=direction, - dropout=dropout, - dtype=dtype, - seed=seed, - kernel_initializer=kernel_initializer, - bias_initializer=bias_initializer) - self._rnn.build([None, None, input_size]) - - self._outputs, self._output_state = self._rnn( - self._inputs, initial_state=self._initial_state, training=training) - - def _AddUp(self, outputs, output_state): - total = math_ops.reduce_sum(outputs) - for s in output_state: - total += math_ops.reduce_sum(s) - return total - - @property - def inputs(self): - return self._inputs - - @property - def initial_state(self): - return self._initial_state - - @property - def outputs(self): - return self._outputs - - @property - def output_state(self): - return self._output_state - - @property - def rnn(self): - return self._rnn - - @property - def total_sum(self): - return self._AddUp(self.outputs, self.output_state) - - def SynthesizeInput(self, seq_length, batch_size, seed=1234): - """Synthesizes input and initial state values for testing.""" - np.random.seed(seed) - num_layers = self._rnn.num_layers - dir_count = self._rnn.num_dirs - num_units = self._rnn.num_units - input_size = self._rnn.input_size - - np_dtype = np.float32 if self._dtype == dtypes.float32 else np.float64 - inputs = np.random.randn(seq_length, batch_size, - input_size).astype(np_dtype) - input_h = np.random.randn(num_layers * dir_count, batch_size, - num_units).astype(np_dtype) - if self._rnn.rnn_mode == CUDNN_LSTM: - input_c = np.random.randn(num_layers * dir_count, batch_size, - num_units).astype(np_dtype) - initial_state = (input_h, input_c) - else: - initial_state = (input_h,) - return inputs, initial_state - - def ZeroState(self, batch_size): - num_layers = self._rnn.num_layers - dir_count = self._rnn.num_dirs - num_units = self._rnn.num_units - - np_dtype = np.float32 if self._dtype == dtypes.float32 else np.float64 - input_h = np.zeros((num_layers * dir_count, batch_size, - num_units)).astype(np_dtype) - if self._rnn.rnn_mode == CUDNN_LSTM: - input_c = np.zeros((num_layers * dir_count, batch_size, - num_units)).astype(np_dtype) - initial_state = (input_h, input_c) - else: - initial_state = (input_h,) - return initial_state - - def FProp(self, inputs_t, initial_state_t, training): - """Builds additional subgraph with given inputs and state. - - Args: - inputs_t: a tensor. - initial_state_t: a tensor. - training: boolean, true if training mode. - Returns: - A tensor of the forward pass output of the model. - """ - outputs, output_state = self._rnn( - inputs_t, initial_state=initial_state_t, training=training) - return self._AddUp(outputs, output_state) - - def Feed(self, sess, inputs, initial_state=None, return_sum=True): - """Runs graph with given inputs and initial state.""" - batch_size = inputs.shape[1] - if initial_state is None: - initial_state = self.ZeroState(batch_size) - if return_sum: - return sess.run( - self.total_sum, - feed_dict={self.inputs: inputs, - self.initial_state: initial_state}) - else: - return sess.run( - [self.outputs, self.output_state], - feed_dict={self.inputs: inputs, - self.initial_state: initial_state}) - - -def _CreateCudnnCompatibleCanonicalRNN(rnn, inputs, is_bidi=False, scope=None): - mode = rnn.rnn_mode - num_units = rnn.num_units - num_layers = rnn.num_layers - - # To reuse cuDNN-trained models, must use cudnn compatible rnn cells. - if mode == CUDNN_LSTM: - single_cell = lambda: cudnn_rnn_ops.CudnnCompatibleLSTMCell(num_units) - elif mode == CUDNN_GRU: - single_cell = lambda: cudnn_rnn_ops.CudnnCompatibleGRUCell(num_units) - elif mode == CUDNN_RNN_TANH: - single_cell = (lambda: rnn_cell_impl.BasicRNNCell(num_units, math_ops.tanh)) - elif mode == CUDNN_RNN_RELU: - single_cell = ( - lambda: rnn_cell_impl.BasicRNNCell(num_units, gen_nn_ops.relu)) - else: - raise ValueError("%s is not supported!" % mode) - - if not is_bidi: - cell = rnn_cell_impl.MultiRNNCell( - [single_cell() for _ in range(num_layers)]) - return rnn_lib.dynamic_rnn( - cell, inputs, dtype=dtypes.float32, time_major=True, scope=scope) - else: - cells_fw = [single_cell() for _ in range(num_layers)] - cells_bw = [single_cell() for _ in range(num_layers)] - - (outputs, output_state_fw, - output_state_bw) = contrib_rnn_lib.stack_bidirectional_dynamic_rnn( - cells_fw, - cells_bw, - inputs, - dtype=dtypes.float32, - time_major=True, - scope=scope) - return outputs, (output_state_fw, output_state_bw) - - -class CudnnRNNTestBasic(test_util.TensorFlowTestCase): - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testLayerBasic(self): - num_layers = 4 - num_units = 2 - batch_size = 8 - direction = CUDNN_RNN_UNIDIRECTION - dir_count = 1 - - with vs.variable_scope("main"): - kernel_initializer = init_ops.constant_initializer(0.) - bias_initializer = init_ops.constant_initializer(0.) - inputs = random_ops.random_uniform([ - num_layers * dir_count, batch_size, num_units], dtype=dtypes.float32) - - lstm = cudnn_rnn.CudnnLSTM(num_layers, num_units, - direction=direction, - kernel_initializer=kernel_initializer, - bias_initializer=bias_initializer, - name="awesome_lstm") - - # Build the layer - outputs1, _ = lstm(inputs) - # Reuse the layer - outputs2, _ = lstm(inputs) - - total_sum1 = math_ops.reduce_sum(outputs1) - total_sum2 = math_ops.reduce_sum(outputs2) - - with vs.variable_scope("main", reuse=True): - lstm = cudnn_rnn.CudnnLSTM(num_layers, num_units, - direction=direction, - kernel_initializer=kernel_initializer, - bias_initializer=bias_initializer, - name="awesome_lstm") - - # Reuse the layer - outputs3, _ = lstm(inputs) - total_sum3 = math_ops.reduce_sum(outputs3) - - self.assertEqual(1, len(variables.trainable_variables())) - self.assertEqual(1, len(ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS))) - self.assertEqual("main/awesome_lstm/opaque_kernel", - variables.trainable_variables()[0].op.name) - - with self.test_session(use_gpu=True) as sess: - sess.run(variables.global_variables_initializer()) - (total_sum1_v, total_sum2_v, total_sum3_v) = sess.run( - [total_sum1, total_sum2, total_sum3]) - self.assertEqual(0, total_sum1_v) - self.assertEqual(0, total_sum2_v) - self.assertEqual(0, total_sum3_v) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testOptimizersSupport(self): - for opt in ("adagrad", "adam", "rmsprop", "momentum", "sgd"): - self._TestOptimizerSupportHelper(opt) - - def _GetOptimizer(self, opt): - if opt == "adagrad": - return adagrad.AdagradOptimizer(learning_rate=1e-2) - elif opt == "adam": - return adam.AdamOptimizer(learning_rate=1e-2) - elif opt == "rmsprop": - return rmsprop.RMSPropOptimizer(learning_rate=1e-2) - elif opt == "momentum": - return momentum.MomentumOptimizer(learning_rate=1e-2, momentum=0.9) - elif opt == "sgd": - return gradient_descent.GradientDescentOptimizer(learning_rate=1e-2) - else: - raise ValueError("Unsupported optimizer: %s" % opt) - - def _TestOptimizerSupportHelper(self, opt): - num_layers = 4 - num_units = 2 - batch_size = 8 - direction = CUDNN_RNN_UNIDIRECTION - dir_count = 1 - - with ops.Graph().as_default() as g: - kernel_initializer = init_ops.constant_initializer(0.) - bias_initializer = init_ops.constant_initializer(0.) - inputs = random_ops.random_uniform([ - num_layers * dir_count, batch_size, num_units], dtype=dtypes.float32) - - lstm = cudnn_rnn.CudnnLSTM(num_layers, num_units, - direction=direction, - kernel_initializer=kernel_initializer, - bias_initializer=bias_initializer, - name="awesome_lstm") - outputs, _ = lstm(inputs) - loss = math_ops.reduce_sum(outputs) - optimizer = self._GetOptimizer(opt) - train_op = optimizer.minimize(loss) - - with self.test_session(use_gpu=True, graph=g) as sess: - sess.run(variables.global_variables_initializer()) - sess.run(train_op) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSaveableGraphDeviceAssignment(self): - num_layers = 4 - num_units = 2 - batch_size = 8 - direction = CUDNN_RNN_UNIDIRECTION - dir_count = 1 - - def DeviceFn(op): - if op.type in ("Variable", "VariableV2"): - return "/cpu:0" - else: - return "/gpu:0" - - with ops.Graph().as_default() as g: - with ops.device(DeviceFn): - with vs.variable_scope("main"): - kernel_initializer = init_ops.constant_initializer(3.14) - bias_initializer = init_ops.constant_initializer(1.59) - inputs = random_ops.random_uniform( - [num_layers * dir_count, batch_size, num_units], - dtype=dtypes.float32) - - lstm = cudnn_rnn.CudnnLSTM(num_layers, num_units, - direction=direction, - kernel_initializer=kernel_initializer, - bias_initializer=bias_initializer, - name="awesome_lstm") - outputs = lstm(inputs) - - # saver is created in the scope of DeviceFn. - saver = saver_lib.Saver() - - with self.test_session(use_gpu=True, graph=g) as sess: - save_path = os.path.join(self.get_temp_dir(), - "test-saveable-device-assignment") - sess.run(variables.global_variables_initializer()) - - saver.save(sess, save_path) - saver.restore(sess, save_path) - sess.run(outputs) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testDifferentShapesEager(self): - # Checks that kernel caching does not cause sharing of temporary storage - # across different input shapes when executing eagerly. - with context.eager_mode(): - with ops.device("gpu:0"): - first_output, _ = cudnn_rnn.CudnnGRU(1, 100)( - array_ops.zeros([28, 100, 28])) - second_output, _ = cudnn_rnn.CudnnGRU(1, 100)( - array_ops.zeros([28, 100, 100])) - self.assertAllEqual([28, 100, 100], first_output.shape) - self.assertAllEqual([28, 100, 100], second_output.shape) - - def _LossFunc(): - first_output, _ = cudnn_rnn.CudnnGRU(1, 100)( - array_ops.zeros([28, 100, 28])) - second_output, _ = cudnn_rnn.CudnnGRU(1, 100)( - array_ops.zeros([28, 100, 100])) - return (math_ops.reduce_sum(first_output) + - math_ops.reduce_sum(second_output)) - - backprop.implicit_grad(_LossFunc)() - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testDifferentShapesGraph(self): - # Tests that a single kernel instance presented with multiple input shapes - # does not crash with graph execution. - with ops.device("gpu:0"): - layer = cudnn_rnn.CudnnGRU(1, 100) - layer(array_ops.zeros([28, 100, 100])) - - def _Cond(index, accumulation): - del accumulation # unused - return math_ops.less(index, 4) - - def _Body(index, accumulation): - layer_input = accumulation[:, :, 10 * (1 + index % 2):] - output, _ = layer(layer_input) - return index + 1, accumulation + output - - original_input = array_ops.zeros([28, 100, 100]) - _, accumulation = control_flow_ops.while_loop(_Cond, _Body, - [0, original_input]) - grad, = gradients.gradients( - math_ops.reduce_sum(accumulation), (original_input,)) - init_op = variables.global_variables_initializer() - with self.cached_session() as sess: - sess.run(init_op) - accumulation_eval, grad_eval = sess.run((accumulation, grad)) - self.assertAllEqual([28, 100, 100], accumulation_eval.shape) - self.assertAllEqual([28, 100, 100], grad_eval.shape) - - -# TODO(jamesqin): Transform to parameterized test after it is included in the -# TF open source codebase. -class CudnnRNNTestSaveRestore(test_util.TensorFlowTestCase): - - def _CompareWeights(self, lhs, rhs): - self.assertEqual(len(lhs), len(rhs)) - for lw, rw in zip(lhs, rhs): - self.assertAllEqual(lw, rw) - - def _CompareBiases(self, lhs, rhs, rnn_mode, num_layers, direction): - self.assertEqual(len(lhs), len(rhs)) - if rnn_mode == CUDNN_LSTM: - num_params_per_layer = CUDNN_LSTM_PARAMS_PER_LAYER - elif rnn_mode == CUDNN_GRU: - num_params_per_layer = CUDNN_GRU_PARAMS_PER_LAYER - elif rnn_mode == CUDNN_RNN_TANH: - num_params_per_layer = CUDNN_RNN_TANH_PARAMS_PER_LAYER - else: - num_params_per_layer = CUDNN_RNN_RELU_PARAMS_PER_LAYER - num_dirs = 1 if direction == CUDNN_RNN_UNIDIRECTION else 2 - num_params_per_layer *= num_dirs - self.assertEqual(num_params_per_layer * num_layers, len(lhs)) - - for i in range(num_layers): - layer_lhs = lhs[i * num_params_per_layer: (i+1) * num_params_per_layer] - layer_rhs = rhs[i * num_params_per_layer: (i+1) * num_params_per_layer] - if direction == CUDNN_RNN_UNIDIRECTION: - self._CompareSingleLayerBiases(layer_lhs, layer_rhs) - else: - size = len(layer_lhs) - fw_lhs, bw_lhs = layer_lhs[:size//2], layer_lhs[size//2:] - fw_rhs, bw_rhs = layer_rhs[:size//2], layer_rhs[size//2:] - self._CompareSingleLayerBiases(fw_lhs, fw_rhs) - self._CompareSingleLayerBiases(bw_lhs, bw_rhs) - - def _CompareSingleLayerBiases(self, lhs, rhs): - self.assertEqual(len(lhs), len(rhs)) - - lf_lhs, rt_lhs = lhs[:len(lhs)//2], lhs[len(lhs)//2:] - lf_rhs, rt_rhs = rhs[:len(rhs)//2], rhs[len(rhs)//2:] - self.assertEqual(len(lf_lhs), len(rt_lhs)) - self.assertEqual(len(lf_rhs), len(rt_rhs)) - - sum_lhs, sum_rhs = [], [] - for lf, rt in zip(lf_lhs, rt_lhs): - sum_lhs.append(lf + rt) - for lf, rt in zip(lf_rhs, rt_rhs): - sum_rhs.append(lf + rt) - self.assertEqual(len(sum_lhs), len(sum_rhs)) - for lf, rt in zip(sum_lhs, sum_rhs): - self.assertAllEqual(lf, rt) - - def _TestSaveRestoreVariable(self, rnn_mode, direction, dtype): - input_size = 3 - num_layers = 2 - num_units = 7 - with ops.Graph().as_default() as g: - random_seed.set_random_seed(1234) - model = CudnnTestModel( - rnn_mode, - num_layers, - num_units, - input_size, - direction=direction, - dtype=dtype) - rnn = model.rnn - save_path = os.path.join(self.get_temp_dir(), - "save-restore-variable-test") - saver = saver_lib.Saver() - weights, biases = ( - model.rnn.saveable.format_converter._opaque_to_cu_canonical( - model.rnn.saveable._variables)) - opaque_params = rnn.trainable_variables[0] - # CudnnTestModel() creates CudnnOpaqueParamsSaveable that helps saver save - # Cudnn vars in canonical format. - reset_op = state_ops.assign( - opaque_params, - array_ops.zeros(array_ops.shape(opaque_params), dtype=dtype)) - # Passing graph explicitly, otherwise an old sess would be reused. - with self.test_session(use_gpu=True, graph=g) as sess: - sess.run(variables.global_variables_initializer()) - val = saver.save(sess, save_path) - self.assertEqual(save_path, val) - weights_v, biases_v = sess.run([weights, biases]) - - # Reset opaque param - sess.run(reset_op) - saver.restore(sess, save_path) - weights_v_restored, biases_v_restored = sess.run([weights, biases]) - - self._CompareWeights(weights_v, weights_v_restored) - self._CompareBiases(biases_v, biases_v_restored, rnn_mode, num_layers, - direction) - - def _TestSaveRestoreTwoVariables(self, rnn_mode, direction, dtype): - input_size = 3 - num_layers = 2 - num_units = 7 - with ops.Graph().as_default() as g: - random_seed.set_random_seed(1234) - with vs.variable_scope("m1"): - model1 = CudnnTestModel( - rnn_mode, - num_layers, - num_units, - input_size, - direction=direction, - dtype=dtype) - with vs.variable_scope("m2"): - model2 = CudnnTestModel( - rnn_mode, - num_layers, - num_units, - input_size, - direction=direction, - dtype=dtype) - opaque_params = (model1.rnn.trainable_variables[0], - model2.rnn.trainable_variables[0]) - saveable1 = model1.rnn.saveable - weights1, biases1 = saveable1.format_converter._opaque_to_cu_canonical( - saveable1._variables) - saveable2 = model1.rnn.saveable - weights2, biases2 = saveable2.format_converter._opaque_to_cu_canonical( - saveable2._variables) - reset_params = [ - state_ops.assign(params, - array_ops.zeros_like(params, dtype=dtype)) - for params in opaque_params - ] - reset_op = control_flow_ops.group(*reset_params) - save_path = os.path.join(self.get_temp_dir(), - "save-restore-variable-test2") - saver = saver_lib.Saver() - # Passing graph explicitly, otherwise an old sess would be reused. - with self.test_session(use_gpu=True, graph=g) as sess: - sess.run(variables.global_variables_initializer()) - val = saver.save(sess, save_path) - self.assertEqual(save_path, val) - - weights1_v, biases1_v = sess.run([weights1, biases1]) - weights2_v, biases2_v = sess.run([weights2, biases2]) - - sess.run(reset_op) - saver.restore(sess, save_path) - weights1_v_restored, biases1_v_restored = sess.run([weights1, biases1]) - weights2_v_restored, biases2_v_restored = sess.run([weights2, biases2]) - - self._CompareWeights(weights1_v, weights1_v_restored) - self._CompareWeights(weights2_v, weights2_v_restored) - self._CompareBiases(biases1_v, biases1_v_restored, rnn_mode, num_layers, - direction) - self._CompareBiases(biases2_v, biases2_v_restored, rnn_mode, num_layers, - direction) - - def _TestSaveRestoreOutput(self, rnn_mode, direction, dtype): - with ops.Graph().as_default() as g: - num_layers = 2 - num_units = 7 - input_size = 7 - seq_length = 8 - batch_size = 4 - model = CudnnTestModel( - rnn_mode, - num_layers, - num_units, - input_size, - direction=direction, - dtype=dtype, - training=False) - rnn = model.rnn - - save_path = os.path.join(self.get_temp_dir(), "save-restore-output-test") - saver = saver_lib.Saver() - - # Only one opaque var in a cudnn layer. - assert len(rnn.trainable_variables) == 1 - reset_params = state_ops.assign( - rnn.trainable_variables[0], - array_ops.zeros( - array_ops.shape(rnn.trainable_variables[0]), dtype=dtype)) - - # Passing graph explicitly, otherwise an old sess would be reused. - with self.test_session(use_gpu=True, graph=g) as sess: - sess.run(variables.global_variables_initializer()) - inputs, initial_state = model.SynthesizeInput(seq_length, batch_size) - total_sum_v = model.Feed(sess, inputs, initial_state) - val = saver.save(sess, save_path) - self.assertEqual(save_path, val) - - sess.run(reset_params) - saver.restore(sess, save_path) - total_sum_v_restored = model.Feed(sess, inputs, initial_state) - self.assertAllClose(total_sum_v, total_sum_v_restored, atol=1e-5) - - def _TestSaveRestoreHelper(self, rnn_mode): - directions = [CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION] - dtype_list = [dtypes.float16, dtypes.float32, dtypes.float64] - for direction, dtype in itertools.product(directions, dtype_list): - self._TestSaveRestoreVariable(rnn_mode, direction, dtype) - self._TestSaveRestoreTwoVariables(rnn_mode, direction, dtype) - self._TestSaveRestoreOutput(rnn_mode, direction, dtype) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSaveRestoreRepeatedlyCreateCustomSaveable(self): - input_size = 3 - num_layers = 2 - num_units = 7 - with ops.Graph().as_default(): - random_seed.set_random_seed(1234) - model = CudnnTestModel( - CUDNN_LSTM, - num_layers, - num_units, - input_size, - direction=CUDNN_RNN_UNIDIRECTION, - dtype=dtypes.float32) - with self.assertRaisesRegexp(RuntimeError, - "Cudnn saveable already created"): - model.rnn._create_saveable() - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSaveRestoreLSTM(self): - self._TestSaveRestoreHelper(CUDNN_LSTM) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSaveRestoreGRU(self): - self._TestSaveRestoreHelper(CUDNN_GRU) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSaveRestoreRNNTanh(self): - self._TestSaveRestoreHelper(CUDNN_RNN_TANH) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSaveRestoreRNNRelu(self): - self._TestSaveRestoreHelper(CUDNN_RNN_RELU) - - -class CudnnRNNTestSaveRestoreTrackable(test_util.TensorFlowTestCase): - - def _VerifyCheckpoint( - self, checkpoint_path, compatible_cell_fn, cudnn_cell_fn, - num_layers, input_size, expected_variable_values, num_applications=3): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - with ops.device("gpu:0"): - cudnn_layer = cudnn_cell_fn() - cudnn_checkpoint = trackable_utils.Checkpoint(cell=cudnn_layer) - status = cudnn_checkpoint.restore(checkpoint_path) - inputs = 3. * array_ops.ones([num_applications, num_layers, input_size], - dtype=dtypes.float32) - cudnn_output, _ = cudnn_layer(inputs) - status.run_restore_ops() - second_save_path = cudnn_checkpoint.save(checkpoint_prefix) - restore_layer = compatible_cell_fn() - restore_layer_checkpoint = trackable_utils.Checkpoint( - cell=restore_layer) - status = restore_layer_checkpoint.restore(second_save_path) - current_state = restore_layer.zero_state(1, dtypes.float32) - for _ in range(num_applications): - restore_layer_output, current_state = restore_layer( - inputs=3. * array_ops.ones([1, input_size]), - state=current_state) - status.run_restore_ops() - self.assertTrue(restore_layer.variables) - for variable, expected_value in zip( - restore_layer.variables, expected_variable_values): - self.assertAllClose(expected_value, self.evaluate(variable)) - self.assertAllClose(self.evaluate(restore_layer_output), - self.evaluate(cudnn_output)[-1, -1:, ...]) - - def _TrackableSingleCellUnidirectionalTestTemplate( - self, single_cell_fn, cudnn_cell_fn): - # Single-layer cuDNN cells with object-based checkpointing should be - # checkpoint compatible with either single CudnnCompatible cells or - # MultiRnnCells with one cell. - input_size = 3 - save_cell_layer = single_cell_fn() - save_cell_layer( - inputs=array_ops.ones([1, input_size]), - state=save_cell_layer.zero_state(1, dtypes.float32)) - self.assertTrue(save_cell_layer.variables) - expected_values = [] - np.random.seed(10) - for variable in save_cell_layer.variables: - value = np.random.normal(size=variable.shape) - expected_values.append(value) - self.evaluate(variable.assign(value)) - save_checkpoint = trackable_utils.Checkpoint(cell=save_cell_layer) - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - first_save_path = save_checkpoint.save(checkpoint_prefix) - self._VerifyCheckpoint( - checkpoint_path=first_save_path, - compatible_cell_fn= - lambda: rnn_cell_impl.MultiRNNCell([single_cell_fn()]), - cudnn_cell_fn=cudnn_cell_fn, - num_layers=1, - expected_variable_values=expected_values, - input_size=input_size) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - @test_util.run_in_graph_and_eager_modes - def testLSTMTrackableSingleLayer(self): - num_units = 2 - direction = CUDNN_RNN_UNIDIRECTION - self._TrackableSingleCellUnidirectionalTestTemplate( - single_cell_fn=functools.partial( - cudnn_rnn_ops.CudnnCompatibleLSTMCell, num_units=num_units), - cudnn_cell_fn=functools.partial( - cudnn_rnn.CudnnLSTM, num_layers=1, num_units=num_units, - direction=direction, name="awesome_lstm")) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - @test_util.run_in_graph_and_eager_modes - def testGRUTrackableSingleLayer(self): - num_units = 2 - direction = CUDNN_RNN_UNIDIRECTION - with self.assertRaises(NotImplementedError): - # TODO(allenl): Implement object-based saving for GRUs and other cells. - self._TrackableSingleCellUnidirectionalTestTemplate( - single_cell_fn=functools.partial( - cudnn_rnn_ops.CudnnCompatibleGRUCell, num_units=num_units), - cudnn_cell_fn=functools.partial( - cudnn_rnn.CudnnGRU, num_layers=1, num_units=num_units, - direction=direction, name="awesome_gru")) - - def _TrackableMultiLayerTestTemplate( - self, single_cell_fn, cudnn_cell_fn, num_layers): - - def _MultiCellFn(): - return rnn_cell_impl.MultiRNNCell( - [single_cell_fn() for _ in range(num_layers)]) - input_size = 3 - save_graph = ops.Graph() - with save_graph.as_default(), self.session(graph=save_graph): - save_layer = _MultiCellFn() - save_layer(inputs=array_ops.ones([1, input_size]), - state=save_layer.zero_state(1, dtypes.float32)) - self.assertTrue(save_layer.variables) - expected_values = [] - np.random.seed(10) - for variable in save_layer.variables: - value = np.random.normal(size=variable.shape) - expected_values.append(value) - self.evaluate(variable.assign(value)) - save_checkpoint = trackable_utils.Checkpoint(cell=save_layer) - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - first_save_path = save_checkpoint.save(checkpoint_prefix) - self._VerifyCheckpoint( - checkpoint_path=first_save_path, - compatible_cell_fn=_MultiCellFn, cudnn_cell_fn=cudnn_cell_fn, - num_layers=num_layers, - expected_variable_values=expected_values, - input_size=input_size) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - @test_util.run_in_graph_and_eager_modes - def testCudnnCompatibleLSTMCheckpointablMultiLayer(self): - num_units = 2 - num_layers = 3 - direction = CUDNN_RNN_UNIDIRECTION - self._TrackableMultiLayerTestTemplate( - single_cell_fn=functools.partial( - cudnn_rnn_ops.CudnnCompatibleLSTMCell, num_units=num_units), - cudnn_cell_fn=functools.partial( - cudnn_rnn.CudnnLSTM, num_layers=num_layers, num_units=num_units, - direction=direction, name="awesome_lstm"), - num_layers=num_layers) - - -# TODO(jamesqin): Transform to parameterized test after it is included in the -# TF open source codebase. -class CudnnRNNTestCompatibleRNNCells(test_util.TensorFlowTestCase): - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testCudnnCompatibleLSTM(self): - self._TestCudnnCompatibleRnnCellsHelper(CUDNN_LSTM) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testCudnnCompatibleGRU(self): - self._TestCudnnCompatibleRnnCellsHelper(CUDNN_GRU) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testCudnnCompatibleRNNTanh(self): - self._TestCudnnCompatibleRnnCellsHelper(CUDNN_RNN_TANH) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testCudnnCompatibleRNNRelu(self): - self._TestCudnnCompatibleRnnCellsHelper(CUDNN_RNN_RELU) - - def _TestCudnnCompatibleRnnCellsHelper(self, rnn_mode): - configs = [ - { - "num_layers": 1, - "seq_length": 3, - "num_units": 4, - "input_size": 5, - "batch_size": 6, - }, - { - "num_layers": 2, - "seq_length": 8, - "num_units": 4, - "input_size": 8, - "batch_size": 16, - }, - { - "num_layers": 2, - "seq_length": 3, - "num_units": 4, - "input_size": 5, - "batch_size": 6, - }, - { - "num_layers": 1, - "seq_length": 2, - "num_units": 2, - "input_size": 4, - "batch_size": 1, - }, - ] - directions = [CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION] - for cfg, direction in zip(configs, directions): - self._TestCudnnCompatibleRnnCells(cfg["num_layers"], cfg["seq_length"], - cfg["num_units"], cfg["input_size"], - cfg["batch_size"], rnn_mode, direction) - - def _TestCudnnCompatibleRnnCells(self, num_layers, seq_length, num_units, - input_size, batch_size, rnn_mode, direction): - dtype = dtypes.float32 - # Train graph - with ops.Graph().as_default() as g: - model = CudnnTestModel( - rnn_mode, - num_layers, - num_units, - input_size, - direction=direction, - dtype=dtype, - training=True) - target_output = array_ops.placeholder(dtype=dtype) - loss_op = losses.log_loss( - labels=target_output, predictions=model.total_sum) - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1e-2) - train_op = optimizer.minimize(loss_op) - - saver = saver_lib.Saver() - - # Train Cudnn model - seed = 0 - with self.test_session(use_gpu=True, graph=g) as sess: - sess.run(variables.global_variables_initializer()) - # Train 128 steps - num_steps = 128 - for _ in range(num_steps): - inputs, _ = model.SynthesizeInput(seq_length, batch_size, seed) - targets = np.random.rand() - sess.run( - train_op, - feed_dict={ - model.inputs: inputs, - model.initial_state: model.ZeroState(batch_size), - target_output: targets - }) - seed += 1 - - save_path = os.path.join(self.get_temp_dir(), - ("cudnn-rnn-%s-test" % rnn_mode)) - save_v = saver.save(sess, save_path) - self.assertEqual(save_path, save_v) - - # Cudnn inference graph - with ops.Graph().as_default() as g: - model = CudnnTestModel( - rnn_mode, - num_layers, - num_units, - input_size, - direction=direction, - dtype=dtype, - training=False) - rnn = model.rnn - saver = saver_lib.Saver() - - inference_input = np.random.rand(seq_length, batch_size, - input_size).astype(np.float32) - with self.test_session(use_gpu=True, graph=g) as sess: - sess.run(variables.global_variables_initializer()) - saver.restore(sess, save_path) - - # Cudnn inference - cudnn_outputs_v, cudnn_output_states_v = model.Feed( - sess, inference_input, return_sum=False) - - # Canonical RNN inference graph - with ops.Graph().as_default() as g: - cell_inputs = array_ops.placeholder( - dtype, shape=[seq_length, batch_size, input_size]) - if direction == CUDNN_RNN_UNIDIRECTION: - # outputs is one tensor, states are num_layer tuples, each 2 tensors - (outputs, states) = _CreateCudnnCompatibleCanonicalRNN(rnn, cell_inputs) - if rnn_mode == CUDNN_LSTM: - output_h = array_ops.stack([s.h for s in states]) - output_c = array_ops.stack([s.c for s in states]) - else: - output_state = array_ops.stack([s for s in states]) - else: - # outputs is one tensor. - # states is a tuple of 2 tuples: - # each sub tuple is num_layer tuples, each with 2 tensors. - (outputs, states) = _CreateCudnnCompatibleCanonicalRNN( - rnn, cell_inputs, is_bidi=True) - output_state_fw, output_state_bw = states - if rnn_mode == CUDNN_LSTM: - output_h, output_c = [], [] - for s_fw, s_bw in zip(output_state_fw, output_state_bw): - output_h.append(array_ops.stack([s_fw.h, s_bw.h])) - output_c.append(array_ops.stack([s_fw.c, s_bw.c])) - output_h = array_ops.concat(output_h, axis=0) - output_c = array_ops.concat(output_c, axis=0) - else: - output_state = [] - for s_fw, s_bw in zip(output_state_fw, output_state_bw): - output_state.append(array_ops.stack([s_fw, s_bw])) - output_state = array_ops.concat(output_state, axis=0) - saver = saver_lib.Saver() - - with self.test_session(use_gpu=True, graph=g) as sess: - saver.restore(sess, save_path) - - # BlockCell inference - if rnn_mode == CUDNN_LSTM: - outputs_v, output_h_v, output_c_v = sess.run( - [outputs, output_h, output_c], - feed_dict={cell_inputs: inference_input}) - self.assertAllClose(cudnn_outputs_v, outputs_v) - cudnn_output_h_v, cudnn_output_c_v = cudnn_output_states_v - self.assertAllClose(cudnn_output_h_v, output_h_v) - self.assertAllClose(cudnn_output_c_v, output_c_v) - else: - outputs_v, output_state_v = sess.run( - [outputs, output_state], - feed_dict={cell_inputs: inference_input}) - self.assertAllClose(cudnn_outputs_v, outputs_v, atol=1e-4, rtol=2e-4) - (cudnn_output_h_v,) = cudnn_output_states_v - self.assertAllClose(cudnn_output_h_v, output_state_v, atol=2e-5, - rtol=2e-5) - - -class CudnnRNNTestParamsSize(test_util.TensorFlowTestCase): - - def _TestOpaqueParamsSize(self, rnn_mode, num_layers, num_units, input_size, - dtype, direction): - logging.info("Testing one lstm param size with config: %s", locals()) - model = CudnnTestModel( - rnn_mode, - num_layers, - num_units, - input_size, - dtype=dtype, - direction=direction) - rnn = model.rnn - - # Min param size estimate = sum(weights.size) + sum(biases.size) - min_params_size = ( - sum(map(np.prod, rnn.canonical_weight_shapes)) + - sum(sp[0] for sp in rnn.canonical_bias_shapes)) - - opaque_params = rnn.trainable_variables[0] - with self.test_session(use_gpu=True, graph=ops.get_default_graph()): - variables.global_variables_initializer().run() - opaque_params_size_v = opaque_params.eval().size - self.assertLessEqual(min_params_size, opaque_params_size_v) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testOpaqueParamsSize(self): - test_configs = [ - [4, 200, 200], - [4, 200, 300], - [4, 200, 100], - [1, 100, 200], - [2, 200, 100], - [3, 200, 400], - ] - directions = [CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION] - dtype_list = [dtypes.float16, dtypes.float32, dtypes.float64] - rnns = [CUDNN_LSTM, CUDNN_GRU, CUDNN_RNN_RELU, CUDNN_RNN_TANH] - for (rnn, config, dtype, direction) in itertools.product( - rnns, test_configs, dtype_list, directions): - num_layers, num_units, input_size = config - with ops.Graph().as_default(): - self._TestOpaqueParamsSize(rnn, num_layers, num_units, input_size, - dtype, direction) - - -class CudnnRNNTestTraining(test_util.TensorFlowTestCase): - - def setUp(self): - super(CudnnRNNTestTraining, self).setUp() - self._reset_rnd_gen_state = os.environ.get("TF_CUDNN_RESET_RND_GEN_STATE", - str(False)) - self._rnn_use_v2 = os.environ.get("TF_CUDNN_RNN_USE_V2", "0") - - def tearDown(self): - super(CudnnRNNTestTraining, self).tearDown() - os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = self._reset_rnd_gen_state - os.environ["TF_CUDNN_RNN_USE_V2"] = self._rnn_use_v2 - - def _ComputeNumericGrad(self, sess, y, x, delta=1e-4, step=1): - """Compute the numeric gradient of y wrt to x. - - Args: - sess: The TF session constructed with a graph containing x and y. - y: A scalar TF Tensor in the graph constructed in sess. - x: A TF Tensor in the graph constructed in sess. - delta: Gradient checker's small perturbation of x[i]. - step: Only compute numerical gradients for a subset of x values. - I.e. dy/dx[i] is computed if i % step == 0. - Returns: - A Tensor of the same shape and dtype as x. If x[i] is not chosen - to compute the numerical gradient dy/x[i], the corresponding - value is set to 0. - """ - - x_data = sess.run(x) - x_size = x_data.size - x_shape = x_data.shape - - numeric_grad = np.zeros(x_size, dtype=x_data.dtype) - - for i in range(0, x_size, step): - x_pos = x_data.copy() - if x_size == 1: - x_pos += delta - else: - x_pos.flat[i] += delta - y_pos_feed_dict = dict([(x.name, x_pos)]) - y_pos = sess.run(y, feed_dict=y_pos_feed_dict) - - x_neg = x_data.copy() - if x_size == 1: - x_neg -= delta - else: - x_neg.flat[i] -= delta - y_neg_feed_dict = dict([(x.name, x_neg)]) - y_neg = sess.run(y, feed_dict=y_neg_feed_dict) - numeric_grad[i] = (y_pos - y_neg) / (2 * delta) - return numeric_grad.reshape(x_shape) - - def _GetShape(self, sess, inputs): - if not isinstance(inputs, collections_abc.Iterable): - return sess.run(array_ops.shape(inputs)) - else: - return sess.run([array_ops.shape(x) for x in inputs]) - - def _GradientCheckFp16(self, sess, y, xs, num_samples, - tolerance=1e-6, delta=1e-4): - """Gradient check for Fp16. - - Fp16 numerical gradients end up being zeros. Use a new way to check - gradients: - - Given multi-variant function: - y = f(x1, x2, ... xn) - delta_y = f(x1 + delta_x1, x2+delta_x2, ..., xn+delta_xn) - - f(x1, x2, ..., xn) - = f'(x1) * delta_x1 + f'(x2) * delta_x2 + .. + f'(xn) * delta_xn - where: - delta_xi are very small disturbance. - f'(xi) is the gradient of y w.r.t xi. - - The gradient check verifies the expected delta_y calculated by the above - equation is close to the actual delta_y. - Args: - sess: tf.compat.v1.Session object. - y: output tensor. - xs: a tensor or a list of input tensors. - num_samples: number of test samples to run. - tolerance: error tolerance. - delta: the order of magnititued of input disturbance to apply to calculate - the output change w.r.t inputs. - """ - sym_grads = self._ComputeSymGrads(sess, y, xs) - xs_shapes = self._GetShape(sess, xs) - - x_vals = [sess.run(x) for x in xs] - for _ in range(num_samples): - delta_xs = [delta * np.random.rand(*shape.tolist()) - for shape in xs_shapes] - - feed_dict = {} - for x, x_val, delta_x in zip(xs, x_vals, delta_xs): - feed_dict[x] = x_val + delta_x - actual_delta_y = (float(sess.run(y, feed_dict=feed_dict)) - - float(sess.run(y))) - - expected_delta_y = 0. - for sym_grad, delta_x in zip(sym_grads, delta_xs): - expected_delta_y += np.dot( - sym_grad.astype(np.float32).flatten(), - delta_x.astype(np.float32).flatten()) - self.assertAllClose(expected_delta_y, actual_delta_y, - atol=tolerance, rtol=tolerance) - - def _GradientCheck(self, sess, y, xs, tolerance=1e-6, delta=1e-4): - sym_grads = self._ComputeSymGrads(sess, y, xs) - - num_grads = [self._ComputeNumericGrad(sess, y, x, delta) for x in xs] - self.assertEqual(len(sym_grads), len(num_grads)) - for x, sym, num in zip(xs, sym_grads, num_grads): - logging.info("Comparing gradients for input: %s", x.name) - self.assertFalse(np.any(np.isnan(sym))) - self.assertFalse(np.any(np.isnan(num))) - self.assertAllClose(sym, num, atol=tolerance, rtol=tolerance) - - def _ComputeSymGrads(self, sess, y, xs): - sym_grads_t = gradients.gradients(y, xs) - return sess.run(sym_grads_t) - - def _TestOneSimpleTraining(self, rnn_mode, num_layers, num_units, input_size, - batch_size, seq_length, dir_count, dropout, dtype, - use_v2, delta, tolerance): - # Gradient checking runs two forward ops with almost the same input. Need to - # make sure the drop patterns across the two runs are the same. - logging.info("Training test with config: %s", locals()) - os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = str(True) - - np.random.seed(1234) - random_seed.set_random_seed(5678) - has_input_c = (rnn_mode == CUDNN_LSTM) - direction = (CUDNN_RNN_UNIDIRECTION - if dir_count == 1 else CUDNN_RNN_BIDIRECTION) - if use_v2: - os.environ["TF_CUDNN_RNN_USE_V2"] = "1" - else: - os.environ["TF_CUDNN_RNN_USE_V2"] = "0" - model = CudnnTestModel( - rnn_mode, - num_layers, - num_units, - input_size, - direction=direction, - dropout=dropout, - dtype=dtype, - training=True, - bias_initializer=init_ops.random_normal_initializer( - mean=1., dtype=dtype)) - rnn = model.rnn - params = rnn.trainable_variables[0] - - inputs = variables.Variable( - random_ops.random_uniform([seq_length, batch_size, input_size], - dtype=dtype), - dtype=dtype).read_value() - input_h = variables.Variable( - random_ops.random_uniform( - [num_layers * dir_count, batch_size, num_units], dtype=dtype), - dtype=dtype).read_value() - if has_input_c: - input_c = variables.Variable( - random_ops.random_uniform( - [num_layers * dir_count, batch_size, num_units], dtype=dtype), - dtype=dtype).read_value() - initial_state = (input_h, input_c) - else: - initial_state = (input_h,) - total_sum = model.FProp(inputs, initial_state, training=True) - - with self.test_session(use_gpu=True, graph=ops.get_default_graph()) as sess: - sess.run(variables.global_variables_initializer()) - all_inputs = [inputs, params] - for s in initial_state: - all_inputs.append(s) - if dtype == dtypes.float16: - self._GradientCheckFp16( - sess, total_sum, all_inputs, - num_samples=FLAGS.grad_check_num_samples, - tolerance=tolerance, delta=delta) - else: - for _ in range(FLAGS.grad_check_num_samples): - # Each time choose a different set of inputs. - sess.run(variables.global_variables_initializer()) - self._GradientCheck( - sess, total_sum, all_inputs, - tolerance=tolerance, delta=delta) - - def _TestSimpleTrainingHelper(self, rnn_mode, test_configs): - dropouts = [0, 0.5, 1.] - v2_options = [False, True] - for config, dropout, use_v2 in itertools.product(test_configs, dropouts, - v2_options): - dtype = config.get("dtype", dtypes.float32) - delta = config.get("delta", 1e-4) - tolerance = config.get("tolerance", 1e-6) - dir_count = config.get("dir_count", 1) - shape = config["shape"] - if dtype == dtypes.float64: - # TODO(jamesqin): b/117848763 - use_v2 = False - with ops.Graph().as_default(): - self._TestOneSimpleTraining( - rnn_mode, shape["num_layers"], shape["num_units"], - shape["input_size"], shape["batch_size"], shape["seq_length"], - dir_count, dropout, dtype, use_v2, delta, tolerance) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSimpleTrainingLSTMFp64(self): - test_configs = [ - { - "dtype": dtypes.float64, - "tolerance": 5e-6, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - ] - self._TestSimpleTrainingHelper(CUDNN_LSTM, test_configs) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSimpleTrainingLSTMFp32(self): - test_configs = [ - { - "dtype": dtypes.float32, - "delta": 1e-4, - "tolerance": 9e-2, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - ] - self._TestSimpleTrainingHelper(CUDNN_LSTM, test_configs) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSimpleTrainingLSTMFp16(self): - test_configs = [ - { - "dtype": dtypes.float16, - "delta": 1e-3, - "tolerance": 9e-2, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - { - "dtype": dtypes.float16, - "delta": 1e-2, - "tolerance": 9e-2, - "shape": { - "num_layers": 2, - "num_units": 6, - "input_size": 8, - "batch_size": 6, - "seq_length": 4, - }, - }, - ] - self._TestSimpleTrainingHelper(CUDNN_LSTM, test_configs) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSimpleTrainingGRUFp64(self): - test_configs = [ - { - "dtype": dtypes.float64, - "tolerance": 5e-6, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - } - }, - ] - self._TestSimpleTrainingHelper(CUDNN_GRU, test_configs) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSimpleTrainingGRUFp32(self): - test_configs = [ - { - "dtype": dtypes.float32, - "delta": 1e-3, - "tolerance": 4e-3, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - ] - self._TestSimpleTrainingHelper(CUDNN_GRU, test_configs) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSimpleTrainingGRUFp16(self): - test_configs = [ - { - "dtype": dtypes.float16, - "delta": 2e-3, - "tolerance": 6e-2, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - ] - self._TestSimpleTrainingHelper(CUDNN_GRU, test_configs) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSimpleTrainingRNNTanhFp64(self): - test_configs = [ - { - "dtype": dtypes.float64, - "tolerance": 5e-6, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - ] - self._TestSimpleTrainingHelper(CUDNN_RNN_TANH, test_configs) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSimpleTrainingRNNTanhFp32(self): - test_configs = [ - { - "dtype": dtypes.float32, - "delta": 1e-3, - "tolerance": 5e-3, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - ] - self._TestSimpleTrainingHelper(CUDNN_RNN_TANH, test_configs) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSimpleTrainingRNNTanhFp16(self): - test_configs = [ - { - "dtype": dtypes.float16, - "delta": 1e-3, - "tolerance": 5e-2, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - ] - self._TestSimpleTrainingHelper(CUDNN_RNN_TANH, test_configs) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSimpleTrainingRNNReluFp64(self): - test_configs = [ - { - "dtype": dtypes.float64, - "tolerance": 5e-6, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - ] - self._TestSimpleTrainingHelper(CUDNN_RNN_RELU, test_configs) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSimpleTrainingRNNReluFp32(self): - test_configs = [ - { - "dtype": dtypes.float32, - "delta": 1e-4, - "tolerance": 3e-1, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - ] - self._TestSimpleTrainingHelper(CUDNN_RNN_RELU, test_configs) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSimpleTrainingRNNReluFp16(self): - test_configs = [ - { - "dtype": dtypes.float16, - "delta": 1e-3, - "tolerance": 7e-2, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - ] - self._TestSimpleTrainingHelper(CUDNN_RNN_RELU, test_configs) - - -if __name__ == "__main__": - argv0 = sys.argv[0] - parser = argparse.ArgumentParser() - parser.add_argument( - "--grad_check_num_samples", - type=int, - default=1, - help="Number of samples to run for gradient check.") - FLAGS, unparsed = parser.parse_known_args() - sys.argv = [argv0] + unparsed - googletest.main() diff --git a/tensorflow/contrib/cudnn_rnn/python/layers/__init__.py b/tensorflow/contrib/cudnn_rnn/python/layers/__init__.py deleted file mode 100644 index 60229af374b..00000000000 --- a/tensorflow/contrib/cudnn_rnn/python/layers/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""layers module with higher level CudnnRNN primitives.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import sys - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import * -# pylint: enable=unused-import,wildcard-import - -from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnCompatibleGRUCell -from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnCompatibleLSTMCell -from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnGRUSaveable -from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnLSTMSaveable -from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnParamsFormatConverterGRU -from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnParamsFormatConverterLSTM -from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnParamsFormatConverterRelu -from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnParamsFormatConverterTanh -from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNReluSaveable -from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNTanhSaveable - diff --git a/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py b/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py deleted file mode 100644 index 2401870a455..00000000000 --- a/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py +++ /dev/null @@ -1,616 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Cudnn RNN operators.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.keras.engine import input_spec -from tensorflow.python.layers import base as base_layer -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.platform import tf_logging as logging - -CUDNN_RNN_UNIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION -CUDNN_RNN_BIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION -CUDNN_LSTM = cudnn_rnn_ops.CUDNN_LSTM -CUDNN_GRU = cudnn_rnn_ops.CUDNN_GRU -CUDNN_RNN_RELU = cudnn_rnn_ops.CUDNN_RNN_RELU -CUDNN_RNN_TANH = cudnn_rnn_ops.CUDNN_RNN_TANH - -# Half for cell input, half for hidden states. -CUDNN_LSTM_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_LSTM_PARAMS_PER_LAYER -CUDNN_GRU_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_GRU_PARAMS_PER_LAYER -CUDNN_RNN_TANH_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_TANH_PARAMS_PER_LAYER -CUDNN_RNN_RELU_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_RELU_PARAMS_PER_LAYER - -CUDNN_INPUT_LINEAR_MODE = cudnn_rnn_ops.CUDNN_INPUT_LINEAR_MODE -CUDNN_INPUT_SKIP_MODE = cudnn_rnn_ops.CUDNN_INPUT_SKIP_MODE -CUDNN_INPUT_AUTO_MODE = cudnn_rnn_ops.CUDNN_INPUT_AUTO_MODE - -__all__ = ["CudnnLSTM", "CudnnGRU", "CudnnRNNTanh", "CudnnRNNRelu"] - - -class _CudnnRNN(base_layer.Layer): - # pylint:disable=line-too-long - """Abstract class for RNN layers with Cudnn implementation. - - Cudnn RNNs have two major differences from other platform-independent RNNs tf - provides: - * Cudnn LSTM and GRU are mathematically different from their tf counterparts. - (e.g. `tf.contrib.rnn.LSTMBlockCell` and `tf.compat.v1.nn.rnn_cell.GRUCell`. - * Cudnn-trained checkpoints are not directly compatible with tf RNNs: - * They use a single opaque parameter buffer for the entire (possibly) - multi-layer multi-directional RNN; Whereas tf RNN weights are per-cell and - layer. - * The size and layout of the parameter buffers may change between - CUDA/CuDNN/GPU generations. Because of that, the opaque parameter variable - does not have a static shape and is not partitionable. Instead of using - partitioning to alleviate the PS's traffic load, try building a - multi-tower model and do gradient aggregation locally within the host - before updating the PS. See - https://www.tensorflow.org/performance/performance_models#parameter_server_variables - for a detailed performance guide. - - Consequently, if one plans to use Cudnn trained models on both GPU and CPU - for inference and training, one needs to: - * Create a CudnnOpaqueParamsSaveable subclass object to save RNN params in - canonical format. (This is done for you automatically during layer building - process.) - * When not using a Cudnn RNN class, use CudnnCompatibleRNN classes to load the - checkpoints. These classes are platform-independent and perform the same - computation as Cudnn for training and inference. - Similarly, CudnnCompatibleRNN-trained checkpoints can be loaded by CudnnRNN - classes seamlessly. - - Below is a typical workflow(using LSTM as an example): - for detailed performance guide. - - # Use Cudnn-trained checkpoints with CudnnCompatibleRNNs - ```python - with tf.Graph().as_default(): - lstm = CudnnLSTM(num_layers, num_units, direction, ...) - - outputs, output_states = lstm(inputs, initial_states, training=True) - - # If user plans to delay calling the cell with inputs, one can do - # lstm.build(input_shape) - - saver = Saver() - - # training subgraph - ... - - # Once in a while save the model. - saver.save(save_path) - - # Inference subgraph for unidirectional RNN on, e.g., CPU or mobile. - with tf.Graph().as_default(): - single_cell = lambda: - tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell(num_units) - - # NOTE: Even if there's only one layer, the cell needs to be wrapped in - # MultiRNNCell. - cell = tf.compat.v1.nn.rnn_cell.MultiRNNCell( - [single_cell() for _ in range(num_layers)]) - - # Leave the scope arg unset. - outputs, final_state = tf.compat.v1.nn.dynamic_rnn(cell, inputs, - initial_state, ...) - - saver = Saver() - - # Create session - sess = ... - - # Restores - saver.restore(sess, save_path) - - # Inference subgraph for bidirectional RNN - with tf.Graph().as_default(): - single_cell = lambda: - tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell(num_units) - cells_fw = [single_cell() for _ in range(num_layers)] - cells_bw = [single_cell() for _ in range(num_layers)] - - # Leave the scope arg unset. - (outputs, output_state_fw, - output_state_bw) = tf.contrib.rnn.stack_bidirectional_dynamic_rnn( - cells_fw, cells_bw, inputs, ...) - saver = Saver() - - # Create session - sess = ... - - # Restores - saver.restore(sess, save_path) - ``` - """ - # pylint:enable=line-too-long - - # TODO(allenl): Document object-based saving and checkpoint compatibility once - # it's implemented for more cuDNN Layers. - - # The following are constants defined by subclasses. - # Type of RNN cell. - _rnn_mode = None - # Number of cell weights(or biases) per layer. - _num_params_per_layer = None - # Custom SaveableObject class for the CudnnRNN class. - _saveable_cls = None - - def __init__(self, - num_layers, - num_units, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - dropout=0., - seed=None, - dtype=dtypes.float32, - kernel_initializer=None, - bias_initializer=None, - name=None): - """Creates a CudnnRNN model from model spec. - - Args: - num_layers: the number of layers for the RNN model. - num_units: the number of units within the RNN model. - input_mode: indicate whether there is a linear projection between the - input and the actual computation before the first layer. It can be - 'linear_input', 'skip_input' or 'auto_select'. 'linear_input' (default) - always applies a linear projection of input onto RNN hidden state. - (standard RNN behavior). 'skip_input' is only allowed when input_size == - num_units; 'auto_select' implies 'skip_input' when input_size == - num_units; otherwise, it implies 'linear_input'. - direction: the direction model that the model operates. Can be either - 'unidirectional' or 'bidirectional' - dropout: dropout rate, a number between [0, 1]. Dropout is applied between - each layer (no dropout is applied for a model with a single layer). When - set to 0, dropout is disabled. - seed: the op seed used for initializing dropout. See - `tf.compat.v1.set_random_seed` for behavior. - dtype: tf.float16, tf.float32 or tf.float64 - kernel_initializer: starting value to initialize the weight. - bias_initializer: starting value to initialize the bias (default is all - zeros). - name: VariableScope for the created subgraph; defaults to class name. This - only serves the default scope if later no scope is specified when - invoking __call__(). - - Raises: - ValueError: if direction is invalid. Or dtype is not supported. - """ - super(_CudnnRNN, self).__init__(dtype=dtype, name=name) - cudnn_rnn_ops.check_direction(direction) - cudnn_rnn_ops.check_input_mode(input_mode) - - if dtype not in [dtypes.float16, dtypes.float32, dtypes.float64]: - raise ValueError("Only support float16, float32, float64, provided %s" % - dtype) - # Layer self.dtype is type name, the original DType object is kept here. - self._plain_dtype = dtype - self._num_layers = num_layers - self._num_units = num_units - self._input_mode = input_mode - self._direction = direction - self._dropout = dropout - self._seed = seed - self._kernel_initializer = kernel_initializer - self._bias_initializer = bias_initializer - # Init input_size to None, which will be set after build(). - self._input_size = None - self._saveable = None - - @property - def num_layers(self): - return self._num_layers - - @property - def num_units(self): - return self._num_units - - @property - def input_mode(self): - """Input mode of first layer. - - Indicates whether there is a linear projection between the input and the - actual computation before the first layer. It can be - * 'linear_input': (default) always applies a linear projection of input - onto RNN hidden state. (standard RNN behavior) - * 'skip_input': 'skip_input' is only allowed when input_size == num_units. - * 'auto_select'. implies 'skip_input' when input_size == num_units; - otherwise, it implies 'linear_input'. - - Returns: - 'linear_input', 'skip_input' or 'auto_select'. - """ - return self._input_mode - - @property - def input_size(self): - if not self._input_size: - raise ValueError( - "\'input_size\' is unknown since layer has not been built.") - return self._input_size - - @property - def rnn_mode(self): - """Type of RNN cell used. - - Returns: - `lstm`, `gru`, `rnn_relu` or `rnn_tanh`. - """ - return self._rnn_mode - - @property - def direction(self): - """Returns `unidirectional` or `bidirectional`.""" - return self._direction - - @property - def num_dirs(self): - return 1 if self._direction == CUDNN_RNN_UNIDIRECTION else 2 - - @property - def saveable(self): - return self._saveable - - @property - def canonical_weight_shapes(self): - """Shapes of Cudnn canonical weight tensors.""" - if not self._input_size: - raise RuntimeError( - "%s.canonical_weight_shapes invoked before input shape is known" % - type(self).__name__) - - shapes = [] - for i in range(self._num_layers): - shapes.extend(self._canonical_weight_shape(i)) - return shapes - - @property - def canonical_bias_shapes(self): - """Shapes of Cudnn canonical bias tensors.""" - return self._canonical_bias_shape(0) * self._num_layers - - def _update_trainable_weights(self, getter, *args, **kwargs): - """Custom getter for layer variables.""" - # Add variables to layer's `(non_)trainable_weights` list(s). - variable = getter(*args, **kwargs) - trainable = kwargs.get("trainable", True) - if trainable and variable not in self._trainable_weights: - self._trainable_weights.append(variable) - elif not trainable and variable not in self._non_trainable_weights: - self._non_trainable_weights.append(variable) - return variable - - def build(self, input_shape): - """Create variables of the Cudnn RNN. - - It can be called manually before `__call__()` or automatically through - `__call__()`. In the former case, subsequent `__call__()`s will skip - creating variables. - Args: - input_shape: network input tensor shape, a python list or a TensorShape - object with 3 dimensions. - - Raises: - ValueError: if input_shape has wrong dimension or unknown 3rd dimension. - """ - if self.built: - return - - input_shape = tensor_shape.TensorShape(input_shape) - if input_shape.ndims != 3: - raise ValueError("Expecting input_shape with 3 dims, got %d" % - input_shape.ndims) - if input_shape[-1].value is None: - raise ValueError("The last dimension of the inputs to `CudnnRNN` " - "should be defined. Found `None`.") - self._input_size = input_shape[-1].value - self.input_spec = input_spec.InputSpec(ndim=3, axes={-1: self._input_size}) - - self._set_scope(None) - - # Not using base class `add_variable()` since the it calls - # `tf.compat.v1.get_variable()` with a callable initializer whereas here - # with a tensor. The difference is mandated to support forward-compatibility - # with Cudnn. - with vs.variable_scope( - self._scope, - reuse=self.built, - custom_getter=self._update_trainable_weights): - if self._kernel_initializer is None: - self._kernel_initializer = init_ops.glorot_uniform_initializer( - seed=self._seed, dtype=self._plain_dtype) - if self._bias_initializer is None: - self._bias_initializer = init_ops.constant_initializer( - 0.0, dtype=self._plain_dtype) - - weights = [ - self._kernel_initializer(sp, dtype=self._plain_dtype) - for sp in self.canonical_weight_shapes - ] - biases = [ - self._bias_initializer(sp, dtype=self._plain_dtype) - for sp in self.canonical_bias_shapes - ] - opaque_params_t = self._canonical_to_opaque(weights, biases) - - if vs.get_variable_scope().partitioner is not None: - logging.warn( - "Partitioner is not supported for Cudnn RNN layer variables, using " - "it will create forward-compatibility issues with future " - "CUDA/CuDNN generations.") - # Initialize opaque params with a tensor with unknown shape, thus couldn't - # use self.add_variable(name, shape, initializer, ...) - self.kernel = vs.get_variable( - "opaque_kernel", - dtype=self._plain_dtype, - initializer=opaque_params_t, - validate_shape=False) - # Create saveable in the outer scope of the cudnn subgraph, such that - # alternative subgraph with platform-independent rnn cells can load the - # checkpoints directly. - if not (self.built or vs.get_variable_scope().reuse is True): - self._create_saveable() - self.built = True - - def _gather_saveables_for_checkpoint(self): - raise NotImplementedError( - "This cell does not yet support object-based saving. File a feature " - "request if this limitation bothers you.") - - def call(self, - inputs, - initial_state=None, - sequence_lengths=None, - time_major=True, - training=True): - """Runs the forward step for the RNN model. - - Args: - inputs: `3-D` tensor. If `time_major` is True (default), the Tensor shape - is [time_len, batch_size, input_size]. If `time_major` is False, the - shape is [batch_size, time_len, input_size]. - initial_state: a tuple of tensor(s) of shape `[num_layers * num_dirs, - batch_size, num_units]` if `time_major` is True (default) or - `[batch_size, num_layers * num_dirs, num_units]` if `time_major` is - False. If not provided, use zero initial states. The tuple size is 2 for - LSTM and 1 for other RNNs. - sequence_lengths: an int32 array representing the variable sequence - lengths in a batch. The size of the array has to equal the batch_size. - If not provided, the same sequence length will be assumed. - time_major: The shape format of the `inputs` and `outputs` Tensors. If - true, these Tensors must be shaped ['max_time', 'batch_size', 'depth']. - If false, these Tensors must be shaped ['batch_size', 'max_time', - 'depth']. By default this function accepts input and emits output in - time-major form. This param is only effective when 'sequence_lengths' is - used. - training: whether this operation will be used in training or inference. - - Returns: - output: a tensor of shape `[time_len, batch_size, num_dirs * num_units]` - if `time_major` is True (default) or `[batch_size, time_len, - num_dirs * num_units]` if `time_major` is False. - It is a `concat([fwd_output, bak_output], axis=2)`. - output_states: a tuple of tensor(s) of the same shape and structure as - `initial_state`. - Raises: - TypeError: initial_state is not a tuple. - """ - if initial_state is not None and not isinstance(initial_state, tuple): - raise TypeError("Invalid initial_state type: %s, expecting tuple." % - initial_state) - dtype = self.dtype - inputs = ops.convert_to_tensor(inputs, dtype=dtype) - - batch_size = array_ops.shape(inputs)[1] - if initial_state is None: - initial_state = self._zero_state(batch_size) - if self._rnn_mode == CUDNN_LSTM: - h, c = initial_state # pylint:disable=unbalanced-tuple-unpacking,unpacking-non-sequence - else: - h, = initial_state # pylint:disable=unbalanced-tuple-unpacking,unpacking-non-sequence - h = ops.convert_to_tensor(h, dtype=dtype) - if self._rnn_mode == CUDNN_LSTM: - c = ops.convert_to_tensor(c, dtype=dtype) - else: - # For model that doesn't take input_c, replace with a dummy tensor. - c = array_ops.constant([], dtype=dtype) - outputs, (output_h, output_c) = self._forward(inputs, h, c, self.kernel, - sequence_lengths, time_major, - training) - if self._rnn_mode == CUDNN_LSTM: - return outputs, (output_h, output_c) - else: - return outputs, (output_h,) - - def state_shape(self, batch_size): - raise NotImplementedError - - def _zero_state(self, batch_size): - res = [] - for sp in self.state_shape(batch_size): - res.append(array_ops.zeros(sp, dtype=self.dtype)) - return tuple(res) - - def _canonical_weight_shape(self, layer): - """Shapes of Cudnn canonical weight tensors for given layer.""" - if layer < 0 or layer >= self._num_layers: - raise ValueError("\'layer\' is not valid, got %s, expecting [%d, %d]" % - (layer, 0, self._num_layers - 1)) - if not self._input_size: - raise RuntimeError( - "%s._canonical_weight_shape invoked before input shape is known" % - type(self).__name__) - - input_size = self._input_size - num_units = self._num_units - num_gates = self._num_params_per_layer // 2 - is_bidi = self._direction == CUDNN_RNN_BIDIRECTION - - if layer == 0: - wts_applied_on_inputs = [(num_units, input_size)] * num_gates - else: - if is_bidi: - wts_applied_on_inputs = [(num_units, 2 * num_units)] * num_gates - else: - wts_applied_on_inputs = [(num_units, num_units)] * num_gates - wts_applied_on_hidden_states = [(num_units, num_units)] * num_gates - tf_wts = wts_applied_on_inputs + wts_applied_on_hidden_states - return tf_wts if not is_bidi else tf_wts * 2 - - def _canonical_bias_shape(self, unused_layer): - """Shapes of Cudnn canonical bias tensors for given layer.""" - num_dirs = 1 if self._direction == CUDNN_RNN_UNIDIRECTION else 2 - return [[self._num_units]] * num_dirs * self._num_params_per_layer - - def _canonical_to_opaque(self, cu_weights, cu_biases): - if not self._input_size: - raise RuntimeError( - "%s._canonical_to_opaque invoked before input shape is known" % - type(self).__name__) - with ops.device("/gpu:0"): - return cudnn_rnn_ops.cudnn_rnn_canonical_to_opaque_params( - rnn_mode=self._rnn_mode, - num_layers=self._num_layers, - num_units=self._num_units, - input_size=self._input_size, - weights=cu_weights, - biases=cu_biases, - input_mode=self._input_mode, - seed=self._seed, - dropout=self._dropout, - direction=self._direction) - - def _forward(self, inputs, h, c, opaque_params, sequence_lengths, time_major, - training): - output, output_h, output_c = cudnn_rnn_ops._cudnn_rnn( # pylint:disable=protected-access - inputs, - h, - c, - opaque_params, - training, - self._rnn_mode, - sequence_lengths=sequence_lengths, - time_major=time_major, - input_mode=self._input_mode, - direction=self._direction, - dropout=self._dropout, - seed=self._seed) - return output, (output_h, output_c) - - def _create_saveable(self): - """Create custom saveable for the Cudnn layer. - - Called during layer building process to make sharing checkpoints between - Cudnn and Cudnn-compatible RNNs easy. - Returns: - a `CudnnOpaqueParamsSaveable` object. - Raises: - RuntimeError: if any custom saveable is already created for this layer. - """ - if self._saveable is not None: - raise RuntimeError("Cudnn saveable already created.") - self._saveable = self._saveable_cls( # pylint:disable=not-callable - opaque_params=self.trainable_variables[0], - num_layers=self.num_layers, - num_units=self.num_units, - input_size=self.input_size, - input_mode=self.input_mode, - direction=self.direction, - scope=vs.get_variable_scope(), - name="%s_saveable" % self.trainable_variables[0].name.split(":")[0]) - self._saveable._add_trackable_dependencies( # pylint: disable=protected-access - trackable=self, - dtype=self._plain_dtype) - ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, self._saveable) - - -class CudnnLSTM(_CudnnRNN): - """Cudnn implementation of LSTM layer.""" - _rnn_mode = CUDNN_LSTM - _num_params_per_layer = CUDNN_LSTM_PARAMS_PER_LAYER - _saveable_cls = cudnn_rnn_ops.CudnnLSTMSaveable - - def state_shape(self, batch_size): - """Shape of Cudnn LSTM states. - - Shape is a 2-element tuple. Each is - [num_layers * num_dirs, batch_size, num_units] - Args: - batch_size: an int - - Returns: - a tuple of python arrays. - """ - return ([self.num_layers * self.num_dirs, batch_size, self.num_units], - [self.num_layers * self.num_dirs, batch_size, self.num_units]) - - @property - def _gather_saveables_for_checkpoint(self): - if self._direction == CUDNN_RNN_UNIDIRECTION: - # Skip one inheritance level to avoid NotImplementedError. - return super(_CudnnRNN, self)._gather_saveables_for_checkpoint - else: - raise NotImplementedError( - "Object-based saving does not currently support bidirectional LSTM " - "cells. File a feature request if this limitation bothers you.") - - -class _CudnnRNNNoInputC(_CudnnRNN): - """Abstract simple CudnnRNN layer without input_c.""" - - def state_shape(self, batch_size): - """Shape of the state of Cudnn RNN cells w/o. - - input_c. - - Shape is a 1-element tuple, - [num_layers * num_dirs, batch_size, num_units] - Args: - batch_size: an int - - Returns: - a tuple of python arrays. - """ - return [self.num_layers * self.num_dirs, batch_size, self.num_units], - - -class CudnnGRU(_CudnnRNNNoInputC): - """Cudnn implementation of the GRU layer.""" - _rnn_mode = CUDNN_GRU - _num_params_per_layer = CUDNN_GRU_PARAMS_PER_LAYER - _saveable_cls = cudnn_rnn_ops.CudnnGRUSaveable - - -class CudnnRNNTanh(_CudnnRNNNoInputC): - """Cudnn implementation of the RNN-tanh layer.""" - _rnn_mode = CUDNN_RNN_TANH - _num_params_per_layer = CUDNN_RNN_TANH_PARAMS_PER_LAYER - _saveable_cls = cudnn_rnn_ops.CudnnRNNTanhSaveable - - -class CudnnRNNRelu(_CudnnRNNNoInputC): - """Cudnn implementation of the RNN-relu layer.""" - _rnn_mode = CUDNN_RNN_RELU - _num_params_per_layer = CUDNN_RNN_RELU_PARAMS_PER_LAYER - _saveable_cls = cudnn_rnn_ops.CudnnRNNReluSaveable diff --git a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py deleted file mode 100644 index 7ee00ade6d2..00000000000 --- a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py +++ /dev/null @@ -1,2065 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Cudnn RNN operators.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -from tensorflow.contrib.checkpoint.python import split_dependency -from tensorflow.contrib.rnn.python.ops import lstm_ops -from tensorflow.python.compat import compat -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.keras.engine import base_layer -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_cudnn_rnn_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import rnn_cell_impl -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.training import saver -from tensorflow.python.training.tracking import tracking as trackable_lib - -CUDNN_RNN_UNIDIRECTION = "unidirectional" -CUDNN_RNN_BIDIRECTION = "bidirectional" -CUDNN_LSTM = "lstm" -CUDNN_GRU = "gru" -CUDNN_RNN_RELU = "rnn_relu" -CUDNN_RNN_TANH = "rnn_tanh" - -# Half for cell input, half for hidden states. -CUDNN_LSTM_PARAMS_PER_LAYER = 8 -CUDNN_GRU_PARAMS_PER_LAYER = 6 -CUDNN_RNN_TANH_PARAMS_PER_LAYER = 2 -CUDNN_RNN_RELU_PARAMS_PER_LAYER = 2 - -CUDNN_INPUT_LINEAR_MODE = "linear_input" -CUDNN_INPUT_SKIP_MODE = "skip_input" -CUDNN_INPUT_AUTO_MODE = "auto_select" - -# pylint:disable=protected-access -_BIAS_VARIABLE_NAME = rnn_cell_impl._BIAS_VARIABLE_NAME -_WEIGHTS_VARIABLE_NAME = rnn_cell_impl._WEIGHTS_VARIABLE_NAME -# pylint:enable=protected-access - - -class CudnnCompatibleLSTMCell(lstm_ops.LSTMBlockCell): - """Cudnn Compatible LSTMCell. - - A simple wrapper around `tf.contrib.rnn.LSTMBlockCell` to use along with - `tf.contrib.cudnn_rnn.CudnnLSTM`. The latter's params can be used by - this cell seamlessly. - """ - - def __init__(self, num_units, reuse=None): - super(CudnnCompatibleLSTMCell, self).__init__( - num_units, - forget_bias=0, - cell_clip=None, - use_peephole=False, - reuse=reuse, - name="cudnn_compatible_lstm_cell") - self._names.update({"scope": "cudnn_compatible_lstm_cell"}) - - -class CudnnCompatibleGRUCell(rnn_cell_impl.GRUCell): - r"""Cudnn Compatible GRUCell. - - A GRU impl akin to `tf.compat.v1.nn.rnn_cell.GRUCell` to use along with - `tf.contrib.cudnn_rnn.CudnnGRU`. The latter's params can be used by - it seamlessly. - - It differs from platform-independent GRUs in how the new memory gate is - calculated. Nvidia picks this variant based on GRU author's[1] suggestion and - the fact it has no accuracy impact[2]. - [1] https://arxiv.org/abs/1406.1078 - [2] http://svail.github.io/diff_graphs/ - - Cudnn compatible GRU (from Cudnn library user guide): - ```python - # reset gate - $$r_t = \sigma(x_t * W_r + h_t-1 * R_h + b_{Wr} + b_{Rr})$$ - # update gate - $$u_t = \sigma(x_t * W_u + h_t-1 * R_u + b_{Wu} + b_{Ru})$$ - # new memory gate - $$h'_t = tanh(x_t * W_h + r_t .* (h_t-1 * R_h + b_{Rh}) + b_{Wh})$$ - $$h_t = (1 - u_t) .* h'_t + u_t .* h_t-1$$ - ``` - - Other GRU (see `tf.compat.v1.nn.rnn_cell.GRUCell` and - `tf.contrib.rnn.GRUBlockCell`): - ```python - # new memory gate - \\(h'_t = tanh(x_t * W_h + (r_t .* h_t-1) * R_h + b_{Wh})\\) - ``` - which is not equivalent to Cudnn GRU: in addition to the extra bias term b_Rh, - ```python - \\(r .* (h * R) != (r .* h) * R\\) - ``` - """ - - def __init__(self, num_units, reuse=None, kernel_initializer=None): - super(CudnnCompatibleGRUCell, self).__init__( - num_units, - activation=None, - reuse=reuse, - kernel_initializer=kernel_initializer) - - def build(self, inputs_shape): - if inputs_shape[1].value is None: - raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" % - inputs_shape) - - input_depth = inputs_shape[1].value - self._gate_kernel = self.add_variable( - "gates/%s" % _WEIGHTS_VARIABLE_NAME, - shape=[input_depth + self._num_units, 2 * self._num_units], - initializer=self._kernel_initializer) - self._gate_bias = self.add_variable( - "gates/%s" % _BIAS_VARIABLE_NAME, - shape=[2 * self._num_units], - initializer=(self._bias_initializer - if self._bias_initializer is not None else - init_ops.constant_initializer(1.0, dtype=self.dtype))) - - self._candidate_input_kernel = self.add_variable( - "candidate/input_projection/%s" % _WEIGHTS_VARIABLE_NAME, - shape=[input_depth, self._num_units], - initializer=self._kernel_initializer) - self._candidate_hidden_kernel = self.add_variable( - "candidate/hidden_projection/%s" % _WEIGHTS_VARIABLE_NAME, - shape=[self._num_units, self._num_units], - initializer=self._kernel_initializer) - - self._candidate_input_bias = self.add_variable( - "candidate/input_projection/%s" % _BIAS_VARIABLE_NAME, - shape=[self._num_units], - initializer=(self._bias_initializer - if self._bias_initializer is not None else - init_ops.zeros_initializer(dtype=self.dtype))) - self._candidate_hidden_bias = self.add_variable( - "candidate/hidden_projection/%s" % _BIAS_VARIABLE_NAME, - shape=[self._num_units], - initializer=(self._bias_initializer - if self._bias_initializer is not None else - init_ops.zeros_initializer(dtype=self.dtype))) - - def call(self, inputs, state): - """Gated recurrent unit (GRU) with nunits cells.""" - gate_inputs = math_ops.matmul( - array_ops.concat([inputs, state], 1), self._gate_kernel) - gate_inputs = nn_ops.bias_add(gate_inputs, self._gate_bias) - - value = math_ops.sigmoid(gate_inputs) - r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1) - - candidate = nn_ops.bias_add( - math_ops.matmul(inputs, self._candidate_input_kernel), - self._candidate_input_bias) - candidate += r * nn_ops.bias_add( - math_ops.matmul(state, self._candidate_hidden_kernel), - self._candidate_hidden_bias) - candidate = self._activation(candidate) - new_h = (1 - u) * candidate + u * state - return new_h, new_h - - -class CudnnParamsFormatConverter(object): - """Abstract class that converts between params of Cudnn Rnn and TF Rnn.""" - - def __init__(self, - num_layers, - num_units, - input_size, - num_proj=None, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION): - """Constructor. - - Args: - num_layers: the number of layers for the RNN model. - num_units: the number of units within the RNN model. - input_size: the size of the input, it could be different from the - num_units. - num_proj: The output dimensionality for the projection matrices. - If None or 0, no projection is performed. - input_mode: indicate whether there is a linear projection between the - input and the actual computation before the first layer. It could be one - of 'linear_input', 'skip_input' or 'auto_select'. * 'linear_input' - (default) always applies a linear projection of input onto RNN hidden - state. (standard RNN behavior). * 'skip_input' is only allowed when - input_size == num_units; * 'auto_select' implies 'skip_input' when - input_size == num_units; otherwise, it implies 'linear_input'. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - """ - self._num_layers = num_layers - self._input_size = input_size - self._num_units = num_units - self._input_mode = input_mode - self._num_proj = num_proj - self._direction = direction - self._num_dirs = 1 if self._direction == CUDNN_RNN_UNIDIRECTION else 2 - self._num_params = ( - self._num_params_per_layer * self._num_layers * self._num_dirs) - - def tf_canonical_to_opaque(self, tf_canonicals, weights_proj=None): - r"""Converts tf canonical weights to cudnn opaque param.""" - cu_weights, cu_biases = self._tf_canonical_to_cu_canonical( - tf_canonicals, weights_proj) - cu_weights = [array_ops.reshape(w, [-1]) for w in cu_weights] - opaque_params = self._cu_canonical_to_opaque(cu_weights, cu_biases) - return opaque_params - - def opaque_to_tf_canonical(self, opaque_param): - r"""Converts cudnn opaque param to tf canonical weights.""" - cu_weights, cu_biases = self._opaque_to_cu_canonical(opaque_param) - if self._num_proj: - weights, biases, weights_proj = self._cu_canonical_to_tf_canonical( - cu_weights, cu_biases) - return weights, biases, weights_proj - else: - weights, biases = self._cu_canonical_to_tf_canonical( - cu_weights, cu_biases) - return weights, biases - - def _opaque_to_cu_canonical(self, opaque_param): - """Converts opaque params to Cudnn canonical format. - - Args: - opaque_param: An opaque tensor storing cudnn rnn params (weights and - biases). - - Returns: - 2 list for weights and biases respectively. - """ - with ops.device("/gpu:0"): - if compat.forward_compatible(2019, 6, 26) and self._num_proj: - num_params_weights = ( - self._num_params + 1 * self._num_layers * self._num_dirs) - num_params_biases = self._num_params - weights, biases = gen_cudnn_rnn_ops.cudnn_rnn_params_to_canonical_v2( - num_layers=self._num_layers, - num_units=self._num_units, - input_size=self._input_size, - params=opaque_param, - rnn_mode=self._rnn_mode, - input_mode=self._input_mode, - direction=self._direction, - num_params_weights=num_params_weights, - num_params_biases=num_params_biases, - num_proj=self._num_proj) - else: - weights, biases = gen_cudnn_rnn_ops.cudnn_rnn_params_to_canonical( - num_layers=self._num_layers, - num_units=self._num_units, - input_size=self._input_size, - params=opaque_param, - num_params=self._num_params, - rnn_mode=self._rnn_mode, - input_mode=self._input_mode, - direction=self._direction) - return (weights, biases) - - def _cu_canonical_to_opaque(self, cu_weights, cu_biases): - """Converts from Cudnn canonical format to opaque params. - - Args: - cu_weights: a list of tensors, Cudnn canonical weights. - cu_biases: a list of tensors, Cudnn canonical biases. - - Returns: - a single opaque tensor. - """ - with ops.device("/gpu:0"): - if compat.forward_compatible(2019, 6, 26) and self._num_proj: - return gen_cudnn_rnn_ops.cudnn_rnn_canonical_to_params_v2( - num_layers=self._num_layers, - num_units=self._num_units, - input_size=self._input_size, - weights=cu_weights, - biases=cu_biases, - rnn_mode=self._rnn_mode, - input_mode=self._input_mode, - num_proj=self._num_proj, - direction=self._direction) - else: - return gen_cudnn_rnn_ops.cudnn_rnn_canonical_to_params( - num_layers=self._num_layers, - num_units=self._num_units, - input_size=self._input_size, - weights=cu_weights, - biases=cu_biases, - rnn_mode=self._rnn_mode, - input_mode=self._input_mode, - direction=self._direction) - - def _cu_canonical_to_tf_canonical(self, cu_weights, cu_biases): - r"""Transform from Cudnn canonical to tf canonical. - - The elements of argument lists are laid out in the following format: - ------------------------------------------------------------ - | weights | biases | - ------------------------------------------------------------ - \ \ - \ \ - ------------------------------- - | layer1 |layer2 |... | - ------------------------------- - \ \ - --------------- - |fwd |bak | - --------------- - Args: - cu_weights: a list of tensors of Cudnn canonical weights. - cu_biases: a list of tensors of Cudnn canonical biases. - - Returns: - 1 tuple, tf canonical weights and biases. - """ - tf_weights, tf_biases = [], [] - tf_weights_proj = [] - - layer_weights_num = self._num_params_per_layer * self._num_dirs - layer_biases_num = layer_weights_num - layer_weights_num += (1 * self._num_dirs) if self._num_proj else 0 - - for i in range(self._num_layers): - layer_weights = cu_weights[i * layer_weights_num:(i + 1) * - layer_weights_num] - layer_biases = cu_biases[i * layer_biases_num:(i + 1) * layer_biases_num] - if self._direction == CUDNN_RNN_UNIDIRECTION: - self._cu_canonical_to_tf_canonical_single_layer(layer_weights, - layer_biases, - tf_weights, tf_biases, - tf_weights_proj) - else: - fw_weights = layer_weights[:len(layer_weights) // 2] - bw_weights = layer_weights[len(layer_weights) // 2:] - fw_biases = layer_biases[:len(layer_biases) // 2] - bw_biases = layer_biases[len(layer_biases) // 2:] - - self._cu_canonical_to_tf_canonical_single_layer( - fw_weights, - fw_biases, - tf_weights, - tf_biases, - tf_weights_proj, - ) - - self._cu_canonical_to_tf_canonical_single_layer( - bw_weights, - bw_biases, - tf_weights, - tf_biases, - tf_weights_proj, - ) - if self._num_proj: - return (tf_weights, tf_biases, tf_weights_proj) - else: - return (tf_weights, tf_biases) - - def _cu_canonical_to_tf_canonical_single_layer(self, - cu_weights, - cu_biases, - tf_weights, - tf_biases, - tf_weigths_proj=None): - r"""Transform single layer Cudnn canonicals to tf canonicals. - - The elements of cu_weights, cu_biases are laid out in the following format: - ------------------------------------------------------------------------- - | gate0 param on inputs | gate0 param on hidden state | gate1 ..........| - ------------------------------------------------------------------------- - Args: - cu_weights: a list of tensors, single layer weights. - cu_biases: a list of tensors, single layer biases. - tf_weights: a list where transformed weights are stored. - tf_biases: a list where transformed biases are stored. - """ - raise NotImplementedError("Abstract method") - - def _tf_canonical_to_cu_canonical(self, tf_canonicals, weights_proj): - r"""Transform from tf canonical to Cudnn canonical. - - This is the reverse routine of _TransformCanonical(). - Args: - tf_canonicals: a list of tensors of tf canonical params. The elements are - laid out in the following format: - ------------------------------------------------------------ - | weights | biases | - ------------------------------------------------------------ - \ \ - \ \ - ------------------------------- - | layer1 |layer2 |... | - ------------------------------- - \ \ - --------------- - |fwd |bak | - --------------- - weights_proj: (optional) weights matrices for projection - Returns: - 2 lists: the recovered cudnn canonical weights and biases. - """ - weights = tf_canonicals[:len(tf_canonicals) // 2] - biases = tf_canonicals[len(tf_canonicals) // 2:] - - cu_weights, cu_biases = [], [] - layer_weights_num = len(weights) // self._num_layers - layer_biases_num = len(biases) // self._num_layers - for i in range(self._num_layers): - layer_weights = weights[i * layer_weights_num:(i + 1) * layer_weights_num] - layer_biases = biases[i * layer_biases_num:(i + 1) * layer_biases_num] - if self._direction == CUDNN_RNN_UNIDIRECTION: - cu_weights.extend(self._tf_to_cudnn_weights(i, *layer_weights)) - if weights_proj is not None: - pw = array_ops.transpose(weights_proj[i]) - cu_weights.append(pw) - cu_biases.extend(self._tf_to_cudnn_biases(*layer_biases)) - else: - fw_weights, bw_weights = layer_weights[:len(layer_weights) // - 2], layer_weights[ - len(layer_weights) // 2:] - fw_biases, bw_biases = layer_biases[:len(layer_biases) // - 2], layer_biases[len(layer_biases - ) // 2:] - cu_weights.extend(self._tf_to_cudnn_weights(i, *fw_weights)) - if weights_proj is not None: - pw0 = array_ops.transpose(weights_proj[2 * i + 0]) - cu_weights.append(pw0) - cu_biases.extend(self._tf_to_cudnn_biases(*fw_biases)) - - cu_weights.extend(self._tf_to_cudnn_weights(i, *bw_weights)) - if weights_proj is not None: - pw1 = array_ops.transpose(weights_proj[2 * i + 1]) - cu_weights.append(pw1) - cu_biases.extend(self._tf_to_cudnn_biases(*bw_biases)) - return cu_weights, cu_biases - - def _cudnn_to_tf_weights(self, *cu_weights): - r"""Stitches cudnn canonical weights to generate tf canonical weights.""" - raise NotImplementedError("Abstract method") - - def _tf_to_cudnn_weights(self, layer, *tf_weights): - r"""Reverses the operations in StitchWeights().""" - raise NotImplementedError("Abstract method") - - def _cudnn_to_tf_biases(self, *biases): - r"""Stitches cudnn canonical biases to generate tf canonical biases.""" - raise NotImplementedError("Abstract method") - - def _tf_to_cudnn_biases(self, *tf_biases): - r"""Reverses the operations in StitchBiases().""" - raise NotImplementedError("Abstract method") - - -class CudnnParamsFormatConverterLSTM(CudnnParamsFormatConverter): - """Helper class that converts between params of Cudnn and TF LSTM.""" - _rnn_mode = CUDNN_LSTM - _num_params_per_layer = CUDNN_LSTM_PARAMS_PER_LAYER - - def _cudnn_to_tf_gate_params(self, *cu_gate_order): - i_g, f_g, c_g, o_g = cu_gate_order - return [i_g, c_g, f_g, o_g] - - def _tf_to_cudnn_gate_params(self, *tf_gate_order): - i_g, c_g, f_g, o_g = tf_gate_order - return [i_g, f_g, c_g, o_g] - - def _cudnn_to_tf_weights(self, *cu_weights): - r"""Stitching cudnn canonical weights to generate tf canonical weights.""" - if self._num_proj: - w_i, w_f, w_c, w_o, r_i, r_f, r_c, r_o, pw = cu_weights - else: - w_i, w_f, w_c, w_o, r_i, r_f, r_c, r_o = cu_weights - - # pylint: disable=invalid-name - W_i = array_ops.concat([w_i, r_i], axis=1) - W_f = array_ops.concat([w_f, r_f], axis=1) - W_c = array_ops.concat([w_c, r_c], axis=1) - W_o = array_ops.concat([w_o, r_o], axis=1) - # pylint: enable=invalid-name - # Cudnn LSTM weights are in ifco order, other tf LSTMs are in icfo order. - reordered = self._cudnn_to_tf_gate_params(*[W_i, W_f, W_c, W_o]) - if self._num_proj: - return (array_ops.transpose(array_ops.concat(reordered, axis=0)), - array_ops.transpose(pw)) - else: - return (array_ops.transpose(array_ops.concat(reordered, axis=0)),) - - def _tf_to_cudnn_weights(self, layer, *tf_weights): - r"""Reverse the operations in StitchWeights().""" - input_size = self._input_size - num_units = self._num_units - if layer == 0: - input_weight_width = input_size - else: - input_weight_width = self._num_proj if self._num_proj else num_units - if self._direction == CUDNN_RNN_BIDIRECTION: - input_weight_width *= 2 - - (tf_weight,) = tf_weights - w = array_ops.transpose(tf_weight) - # pylint: disable=invalid-name - W_i, W_f, W_c, W_o = self._tf_to_cudnn_gate_params( - *array_ops.split(w, 4, axis=0)) - - hidden_state_width = self._num_proj if self._num_proj else num_units - w_i, r_i = array_ops.split( - W_i, [input_weight_width, hidden_state_width], axis=1) - w_c, r_c = array_ops.split( - W_c, [input_weight_width, hidden_state_width], axis=1) - w_f, r_f = array_ops.split( - W_f, [input_weight_width, hidden_state_width], axis=1) - w_o, r_o = array_ops.split( - W_o, [input_weight_width, hidden_state_width], axis=1) - return w_i, w_f, w_c, w_o, r_i, r_f, r_c, r_o - # pylint: enable=invalid-name - - def _cudnn_to_tf_biases(self, *cu_biases): - r"""Stitching cudnn canonical biases to generate tf canonical biases.""" - b_wi, b_wf, b_wc, b_wo, b_ri, b_rf, b_rc, b_ro = cu_biases - # Save only the sum instead of individual biases. When recovering, return - # two biases each with half the value. Since RNN does not regularize by - # weight decay, it has no side effect in training or inference. - # pylint: disable=invalid-name - B_i = b_wi + b_ri - B_f = b_wf + b_rf - B_c = b_wc + b_rc - B_o = b_wo + b_ro - # pylint: enable=invalid-name - reordered = self._cudnn_to_tf_gate_params(*[B_i, B_f, B_c, B_o]) - return (array_ops.concat(reordered, axis=0),) - - def _tf_to_cudnn_biases(self, *tf_biases): - r"""Reverse the operations in StitchBiases().""" - (tf_bias,) = tf_biases - # pylint: disable=invalid-name - B_i, B_f, B_c, B_o = self._tf_to_cudnn_gate_params( - *array_ops.split(tf_bias, 4, axis=0)) - # pylint: enable=invalid-name - # pylint: disable=unbalanced-tuple-unpacking - b_wi, b_ri = (B_i * 0.5,) * 2 - b_wf, b_rf = (B_f * 0.5,) * 2 - b_wc, b_rc = (B_c * 0.5,) * 2 - b_wo, b_ro = (B_o * 0.5,) * 2 - # pylint: enable=unbalanced-tuple-unpacking - # Return ifco order for Cudnn LSTM. - return b_wi, b_wf, b_wc, b_wo, b_ri, b_rf, b_rc, b_ro - - def _cu_canonical_to_tf_canonical_single_layer(self, - cu_weights, - cu_biases, - tf_weights, - tf_biases, - tf_weights_proj=None): - if self._num_proj: - (w, pw) = self._cudnn_to_tf_weights(*cu_weights) - tf_weights.append(w) - tf_weights_proj.append(pw) - else: - (w,) = self._cudnn_to_tf_weights(*cu_weights) - tf_weights.append(w) - (b,) = self._cudnn_to_tf_biases(*cu_biases) - tf_biases.append(b) - - -class CudnnParamsFormatConverterGRU(CudnnParamsFormatConverter): - """Helper class that converts between params of Cudnn and TF GRU.""" - - _rnn_mode = CUDNN_GRU - _num_params_per_layer = CUDNN_GRU_PARAMS_PER_LAYER - - _rnn_cell_name = base_layer.to_snake_case(CudnnCompatibleGRUCell.__name__) - - def _cudnn_to_tf_weights(self, *cu_weights): - r"""Stitching cudnn canonical weights to generate tf canonical weights.""" - w_i, w_r, w_h, r_i, r_r, r_h = cu_weights - - # pylint: disable=invalid-name - W_i = array_ops.concat([w_i, r_i], axis=1) - W_r = array_ops.concat([w_r, r_r], axis=1) - # pylint: enable=invalid-name - return (array_ops.transpose(array_ops.concat([W_i, W_r], axis=0)), - array_ops.transpose(w_h), array_ops.transpose(r_h)) - - def _tf_to_cudnn_weights(self, layer, *tf_weights): - r"""Reverse the operations in StitchWeights().""" - input_size = self._input_size - num_units = self._num_units - if layer == 0: - input_weight_width = input_size - else: - input_weight_width = num_units - if self._direction == CUDNN_RNN_BIDIRECTION: - input_weight_width *= 2 - # pylint: disable=invalid-name - W_ir, w_h, r_h = tf_weights - W_ir = array_ops.transpose(W_ir) - w_h = array_ops.transpose(w_h) - r_h = array_ops.transpose(r_h) - - W_i, W_r = array_ops.split(W_ir, 2, axis=0) - w_i, r_i = array_ops.split(W_i, [input_weight_width, num_units], axis=1) - w_r, r_r = array_ops.split(W_r, [input_weight_width, num_units], axis=1) - # pylint: enable=invalid-name - return w_i, w_r, w_h, r_i, r_r, r_h - - def _cudnn_to_tf_biases(self, *biases): - r"""Stitching cudnn canonical biases to generate tf canonical biases.""" - b_wi, b_wr, b_wh, b_ri, b_rr, b_rh = biases - return ( - # Save only the sum instead of individual biases. When recovering, - # return two biases each with half the value. Since RNN does not - # regularize by weight decay, it has no side effect in training or - # inference. - array_ops.concat([b_wi, b_wr], axis=0) + - array_ops.concat([b_ri, b_rr], axis=0), - b_wh, - b_rh) - - def _tf_to_cudnn_biases(self, *tf_biases): - r"""Reverse the operations in StitchBiases().""" - # b_ir is the summed bias of reset and update gate. - b_ir, b_wh, b_rh = tf_biases - bi, br = b_ir * 0.5, b_ir * 0.5 - b_wi, b_wr = array_ops.split(bi, 2, axis=0) - b_ri, b_rr = array_ops.split(br, 2, axis=0) - return b_wi, b_wr, b_wh, b_ri, b_rr, b_rh - - def _cu_canonical_to_tf_canonical_single_layer(self, - cu_weights, - cu_biases, - tf_weights, - tf_biases, - tf_weights_proj=None): - # pylint: disable=invalid-name - W_ir, w_h, r_h = self._cudnn_to_tf_weights(*cu_weights) - b_ir, b_wh, b_rh = self._cudnn_to_tf_biases(*cu_biases) - # pylint: enable=invalid-name - tf_weights.extend([W_ir, w_h, r_h]) - tf_biases.extend([b_ir, b_wh, b_rh]) - - -class CudnnParamsFormatConverterBasic(CudnnParamsFormatConverterLSTM): - """Helper class that converts between params of Cudnn and TF Relu/Tanh RNN.""" - - def _cudnn_to_tf_weights(self, *cu_weights): - r"""Stitching cudnn canonical weights to generate tf canonical weights.""" - w_i, w_h = cu_weights - W = array_ops.concat([w_i, w_h], axis=1) # pylint: disable=invalid-name - return (array_ops.transpose(W),) - - def _tf_to_cudnn_weights(self, layer, *tf_weights): - r"""Reverse the operations in StitchWeights().""" - input_size = self._input_size - num_units = self._num_units - if layer == 0: - input_weight_width = input_size - else: - input_weight_width = num_units - if self._direction == CUDNN_RNN_BIDIRECTION: - input_weight_width *= 2 - - (tf_weight,) = tf_weights - # pylint: disable=invalid-name - W = array_ops.transpose(tf_weight) - w_i, w_h = array_ops.split(W, [input_weight_width, num_units], axis=1) - return w_i, w_h - # pylint: enable=invalid-name - - def _cudnn_to_tf_biases(self, *cu_biases): - r"""Stitching cudnn canonical biases to generate tf canonical biases.""" - # Save only the sum instead of individual biases. When recovering, return - # two biases each with half the value. Since RNN does not regularize by - # weight decay, it has no side effect in training or inference. - b_wi, b_wh = cu_biases - return (b_wi + b_wh,) - - def _tf_to_cudnn_biases(self, *tf_biases): - r"""Reverse the operations in StitchBiases().""" - (tf_bias,) = tf_biases - b_i = tf_bias * 0.5 - b_h = tf_bias * 0.5 - return b_i, b_h - - -class CudnnParamsFormatConverterTanh(CudnnParamsFormatConverterBasic): - """Helper class that converts between params of Cudnn and TF Tanh RNN.""" - _rnn_mode = CUDNN_RNN_TANH - _num_params_per_layer = CUDNN_RNN_TANH_PARAMS_PER_LAYER - - -class CudnnParamsFormatConverterRelu(CudnnParamsFormatConverterBasic): - """Helper class that converts between params of Cudnn and TF Relu RNN.""" - _rnn_mode = CUDNN_RNN_RELU - _num_params_per_layer = CUDNN_RNN_RELU_PARAMS_PER_LAYER - - -# TODO(yaozhang): make sure we only save the canonical version of params and -# don't save the platform-specific version to avoid potential race -# conditions where params is updated by both versions when being restored. -# Currently, checkpointing will function properly, despite that we save both -# versions, because Saver restores customized savables after Variables. -# However, it is good to not rely on this restoring order of Saver and to -# avoid unnecessary storage. Add a test to check only the canonical version is -# saved. -class CudnnOpaqueParamsSaveable(saver.BaseSaverBuilder.SaveableObject): - """Abstract SaveableObject implementation handling Cudnn opaque params.""" - - def __init__(self, - opaque_params, - num_layers, - num_units, - input_size, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - scope=None, - name="cudnn_rnn_saveable"): - """Creates a CudnnOpaqueParamsSaveable object. - - CudnnOpaqueParamsSaveable is saveable/restorable in a checkpoint file - and is used to save/restore the weights and biases parameters in a - canonical format which is directly consumable by platform-independent tf - RNN cells. Parameters are saved as tensors layer by layer with weight - tensors followed by bias tensors, and forward direction followed by - backward direction (if applicable). When restoring, a user could name - param_variables as desired, and restore weight and bias tensors to these - variables. - - For CudnnRNNRelu or CudnnRNNTanh, there are 2 tensors per weight and per - bias for each layer: tensor 0 is applied to the input from the previous - layer and tensor 1 to the recurrent input. - - For CudnnLSTM, there are 8 tensors per weight and per bias for each - layer: tensor 0-3 are applied to the input from the previous layer and - tensor 4-7 to the recurrent input. Tensor 0 and 4 are for the input gate; - tensor 1 and 5 the forget gate; tensor 2 and 6 the new memory gate; - tensor 3 and 7 the output gate. - - For CudnnGRU, there are 6 tensors per weight and per bias for each layer: - tensor 0-2 are applied to the input from the previous layer and - tensor 3-5 to the recurrent input. Tensor 0 and 3 are for the reset gate; - tensor 1 and 4 the update gate; tensor 2 and 5 the new memory gate. - - Args: - opaque_params: a variable, Cudnn RNN opaque params. - num_layers: the number of layers for the RNN model. - num_units: the number of units within the RNN model. - input_size: the size of the input, it could be different from the - num_units. - input_mode: indicate whether there is a linear projection between the - input and the actual computation before the first layer. It could be - 'linear_input', 'skip_input' or 'auto_select'. 'linear_input' (default) - always applies a linear projection of input onto RNN hidden state. - (standard RNN behavior). 'skip_input' is only allowed when input_size == - num_units; 'auto_select' implies 'skip_input' when input_size == - num_units; otherwise, it implies 'linear_input'. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - scope: string of VariableScope, the scope of equivalent subgraph - consisting only platform-independent tf RNN cells. - name: the name of the CudnnOpaqueParamsSaveable object. - """ - # Define in subclasses. - self._num_layers = num_layers - self._input_size = input_size - self._num_units = num_units - self._input_mode = input_mode - self._direction = direction - if scope is not None: - scope_name = scope.name if isinstance(scope, vs.VariableScope) else scope - self._scope = scope_name or None - else: - self._scope = None - - self._variables = opaque_params - self._num_dirs = 1 if self._direction == CUDNN_RNN_UNIDIRECTION else 2 - # Defined in subclasses. - self._format_converter = None - - tf_weights, tf_biases = ( - self.format_converter.opaque_to_tf_canonical(self._variables)) - tf_weight_names, tf_bias_names = self._tf_canonical_names() - # We currently don't use slice_spec. It might be useful in a distributed - # setting where each parameter server node stores a slice of variable, - # instead of having the master pull all slices and then save them. - slice_spec = "" - params = tf_weights + tf_biases - self._weight_names = tf_weight_names - self._bias_names = tf_bias_names - self._param_names = tf_weight_names + tf_bias_names - prefixed_param_names = tf_weight_names + tf_bias_names - if self._scope: - prefixed_param_names = [ - "%s/%s" % (self._scope, pn) for pn in prefixed_param_names - ] - specs = [ - saver.BaseSaverBuilder.SaveSpec(param, slice_spec, param_name) - for param, param_name in zip(params, prefixed_param_names) - ] - super(CudnnOpaqueParamsSaveable, - self).__init__(array_ops.identity(self._variables), specs, name) - - @property - def format_converter(self): - if self._format_converter is None: - self._format_converter = self._format_converter_cls( - self._num_layers, - self._num_units, - self._input_size, - input_mode=self._input_mode, - direction=self._direction) - return self._format_converter - - def restore(self, restored_tensors, restored_shapes): - opaque_params = self.format_converter.tf_canonical_to_opaque( - restored_tensors) - return state_ops.assign( - self._variables, opaque_params, validate_shape=False) - - def _trackable_save(self, save_buffer): - weights, biases = self.format_converter.opaque_to_tf_canonical( - self._variables) - for name, tensor in zip(self._param_names, weights + biases): - save_buffer[name] = array_ops.identity(tensor) - - def _trackable_restore(self, restore_buffer): - tensors = [ - array_ops.identity(restore_buffer[name]) for name in self._param_names - ] - return self.restore( - restored_tensors=tensors, - restored_shapes=None # Unused - ) - - def _add_trackable_dependencies(self, trackable, dtype): - """Add canonical weight dependencies to `trackable`. - - When saving or restoring, converts to or from the opaque buffer - format. Weights are saved and loaded in the configuration expected by - cuDNN-compatible cells. - - Args: - trackable: An object inheriting from `Trackable` to add dependencies too - (typically the cuDNN `Layer`). - dtype: The dtype for the canonical parameter Tensors. - """ - split_dependencies = split_dependency.split_dependency( - component_names=self._param_names, - component_dtypes=(dtype,) * len(self._param_names), - fill_save_buffer_fn=self._trackable_save, - consume_restore_buffer_fn=self._trackable_restore, - device=self._variables[0].device) - self._trackable_track_params(trackable, split_dependencies) - - def _trackable_track_params(self, trackable, params): - """Tracks parameters in a canonical configuration.""" - return # NotImplementedError raised by the Layer. - - def _tf_canonical_names(self): - tf_weights_names, tf_biases_names = [], [] - for i in range(self._num_layers): - if self._direction == CUDNN_RNN_UNIDIRECTION: - prefix = self._tf_canonical_name_prefix(i) - self._tf_canonical_names_single_layer(prefix, tf_weights_names, - tf_biases_names) - else: - fwd_prefix = self._tf_canonical_name_prefix(i, is_fwd=True) - bak_prefix = self._tf_canonical_name_prefix(i, is_fwd=False) - - self._tf_canonical_names_single_layer(fwd_prefix, tf_weights_names, - tf_biases_names) - self._tf_canonical_names_single_layer(bak_prefix, tf_weights_names, - tf_biases_names) - return tf_weights_names, tf_biases_names - - def _tf_canonical_name_prefix(self, layer, is_fwd=True): - if self._direction == CUDNN_RNN_UNIDIRECTION: - return "rnn/multi_rnn_cell/cell_%d/%s" % (layer, self._rnn_cell_name) - else: - if is_fwd: - return ("stack_bidirectional_rnn/cell_%d/bidirectional_rnn/fw/%s" % - (layer, self._rnn_cell_name)) - else: - return ("stack_bidirectional_rnn/cell_%d/bidirectional_rnn/bw/%s" % - (layer, self._rnn_cell_name)) - - def _tf_canonical_names_single_layer(self, prefix, tf_weights_names, - tf_biases_names): - raise NotImplementedError("Abstract method") - - -class CudnnLSTMSaveable(CudnnOpaqueParamsSaveable): - """SaveableObject implementation handling Cudnn LSTM opaque params.""" - - _format_converter_cls = CudnnParamsFormatConverterLSTM - _rnn_cell_name = base_layer.to_snake_case(CudnnCompatibleLSTMCell.__name__) - - def _tf_canonical_names_single_layer(self, prefix, tf_weights_names, - tf_bias_names): - tf_weights_names.append(prefix + "/kernel") - tf_bias_names.append(prefix + "/bias") - - def _trackable_track_params(self, trackable, params): - """Track parameters for compatibility with CudnnCompatibleLSTMCell.""" - biases = [] - weights = [] - for name in self._weight_names: - weights.append(params[name]) - for name in self._bias_names: - biases.append(params[name]) - assert len(params) == len(weights) + len(biases) - if len(weights) == 1 and len(biases) == 1: - # For single-layer cells, allow substituting a cell with no MultiRNNCell - # wrapping. - kernel, = weights # pylint: disable=unbalanced-tuple-unpacking - bias, = biases # pylint: disable=unbalanced-tuple-unpacking - trackable._track_trackable(kernel, name="kernel") # pylint: disable=protected-access - trackable._track_trackable(bias, name="bias") # pylint: disable=protected-access - assert len(biases) == len(weights) - for cell_index, (bias, kernel) in enumerate(zip(biases, weights)): - cell = trackable_lib.AutoTrackable() - trackable._track_trackable(cell, name="cell-%d" % cell_index) # pylint: disable=protected-access - cell.bias = bias - cell.kernel = kernel - - -class CudnnGRUSaveable(CudnnOpaqueParamsSaveable): - """SaveableObject implementation handling Cudnn GRU opaque params.""" - - _format_converter_cls = CudnnParamsFormatConverterGRU - _rnn_cell_name = base_layer.to_snake_case(CudnnCompatibleGRUCell.__name__) - - def _tf_canonical_names_single_layer(self, prefix, tf_weights_names, - tf_bias_names): - tf_weights_names.append(prefix + "/gates/kernel") - tf_weights_names.append(prefix + "/candidate/input_projection/kernel") - tf_weights_names.append(prefix + "/candidate/hidden_projection/kernel") - - tf_bias_names.append(prefix + "/gates/bias") - tf_bias_names.append(prefix + "/candidate/input_projection/bias") - tf_bias_names.append(prefix + "/candidate/hidden_projection/bias") - - -class CudnnRNNTanhSaveable(CudnnLSTMSaveable): - _format_converter_cls = CudnnParamsFormatConverterTanh - _rnn_cell_name = base_layer.to_snake_case(rnn_cell_impl.BasicRNNCell.__name__) - - -class CudnnRNNReluSaveable(CudnnLSTMSaveable): - _format_converter_cls = CudnnParamsFormatConverterRelu - _rnn_cell_name = base_layer.to_snake_case(rnn_cell_impl.BasicRNNCell.__name__) - - -_cudnn_rnn_common_doc_string = """ - Cudnn RNN has an opaque parameter buffer that can be used for inference and - training. But it is possible that the layout of the parameter buffers - changes between generations. So it is highly recommended to use - CudnnOpaqueParamsSaveable to save and restore weights and biases in a - canonical format. - - This is a typical use case: - - * The user creates a CudnnRNN model. - * The user query that parameter buffer size. - * The user creates a variable of that size that serves as the parameter - buffers. - * The user either initialize the parameter buffer, or load the canonical - weights into the parameter buffer. - * The user calls the model with the parameter buffer for inference, or - training. - * If training, the user creates a Saver object. - * If training, the user creates a CudnnOpaqueParamsSaveable object from the - parameter buffer for it to be later saved in the canonical format. When - creating a CudnnOpaqueParamsSaveable object, a name could be provided, - which is useful in distinguishing the names of multiple - CudnnOpaqueParamsSaveable objects (e.g. for an encoder-decoder model). - * Once a while, the user saves the parameter buffer into model checkpoints - with Saver.save(). - * When restoring, the user creates a CudnnOpaqueParamsSaveable object and - uses Saver.restore() to restore the parameter buffer from the canonical - format to a user-defined format, as well as to restore other savable - objects in the checkpoint file. -""" - - -def _check_rnn_mode(rnn_mode): - if rnn_mode not in (CUDNN_LSTM, CUDNN_GRU, CUDNN_RNN_TANH, CUDNN_RNN_RELU): - raise ValueError( - "Invalid rnn_mode: %s, expect one of (%s, %s, %s, %s)" % - (rnn_mode, CUDNN_LSTM, CUDNN_GRU, CUDNN_RNN_TANH, CUDNN_RNN_RELU)) - - -def _get_seed(seed): - seed, seed2 = random_seed.get_seed(seed) - if seed is None and seed2 is None: - seed, seed2 = 0, 0 - return seed, seed2 - - -def check_direction(direction): - """Check validity of direction.""" - if direction not in (CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION): - raise ValueError("Invalid direction: %s, expecting %s or %s" % - (direction, CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION)) - - -def check_input_mode(input_mode): - if input_mode not in (CUDNN_INPUT_LINEAR_MODE, CUDNN_INPUT_SKIP_MODE, - CUDNN_INPUT_AUTO_MODE): - raise ValueError("Invalid input_mode: %s, expect one of (%s, %s, %s)" % - (input_mode, CUDNN_INPUT_LINEAR_MODE, - CUDNN_INPUT_SKIP_MODE, CUDNN_INPUT_AUTO_MODE)) - - -def _get_num_params(rnn_mode, num_layers, direction): - """Return num params for given Cudnn config.""" - if rnn_mode == CUDNN_LSTM: - num_params_per_layer = CUDNN_LSTM_PARAMS_PER_LAYER - elif rnn_mode == CUDNN_GRU: - num_params_per_layer = CUDNN_GRU_PARAMS_PER_LAYER - elif rnn_mode == CUDNN_RNN_RELU: - num_params_per_layer = CUDNN_RNN_RELU_PARAMS_PER_LAYER - elif rnn_mode == CUDNN_RNN_TANH: - num_params_per_layer = CUDNN_RNN_TANH_PARAMS_PER_LAYER - else: - raise ValueError("Invalid \'rnn_mode\': %s" % rnn_mode) - num_params = num_layers * num_params_per_layer - if direction != CUDNN_RNN_UNIDIRECTION: - num_params *= 2 - return num_params - - -def _cudnn_rnn(inputs, - input_h, - input_c, - params, - is_training, - rnn_mode, - sequence_lengths=None, - time_major=True, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - dropout=0., - seed=0, - num_proj=None, - name=None): - """Cudnn RNN. - - Args: - inputs: the input sequence to the RNN model. If `time_major` is True - (default), the Tensor shape is [max_time, batch_size, input_size]. If - `time_major` is False, the shape is [batch_size, max_time, input_size]. - input_h: the initial hidden state for h. If `time_major` is True (default), - the Tensor shape is [num_layers, batch_size, num_units]. If `time_major` - is False, the shape is [batch_size, num_layers, num_units]. - input_c: the initial hidden state for c. This is only relevant for LSTM. A - Tensor of the same shape as input_h. - params: the parameter buffer created for this model. - is_training: whether this operation will be used in training or inference - rnn_mode: one of ('lstm', 'gru', 'rnn_relu', 'rnn_tanh'). - sequence_lengths: an int32 array representing the variable sequence lengths - in a batch. The size of the array has to equal the batch_size. Default to - None, in which case sequences in the batch are assumed to have the same - length, which is inferred from inputs. - time_major: The shape format of the `inputs` and `outputs` Tensors. If true, - these Tensors must be shaped ['max_time', 'batch_size', 'depth']. If - false, these Tensors must be shaped ['batch_size', 'max_time', 'depth']. - By default this function accepts input and emits output in time-major - form. This param is only effective when 'sequence_lengths' is used. - input_mode: indicate whether there is a linear projection between the input - and the actual computation before the first layer. It could be - 'linear_input', 'skip_input' or 'auto_select'. 'linear_input' (default) - always applies a linear projection of input onto RNN hidden state. - (standard RNN behavior). 'skip_input' is only allowed when input_size == - num_units; 'auto_select' implies 'skip_input' when input_size == - num_units; otherwise, it implies 'linear_input'. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - dropout: whether to enable dropout. With it is 0, dropout is disabled. - seed: the op seed used for initializing dropout. See - `tf.compat.v1.set_random_seed` for behavior. - num_proj: The output dimensionality for the projection matrices. - If None or 0, no projection is performed. - name: name of the operation. - - Returns: - outputs, output_h, output_c - """ - _check_rnn_mode(rnn_mode) - check_direction(direction) - check_input_mode(input_mode) - seed, seed2 = random_seed.get_seed(seed) - # TODO(jamesqin): switch default value to "1" on May 25th 2018, and get rid - # of V1 ops. - use_cudnn_v2 = os.environ.get("TF_CUDNN_RNN_USE_V2", "0") - args = { - "input": inputs, - "input_h": input_h, - "input_c": input_c, - "params": params, - "is_training": is_training, - "rnn_mode": rnn_mode, - "input_mode": input_mode, - "direction": direction, - "dropout": dropout, - "seed": seed, - "seed2": seed2, - "name": name - } - if sequence_lengths is not None: - args["sequence_lengths"] = sequence_lengths - args["time_major"] = time_major - args["num_proj"] = 0 if num_proj is None else num_proj - outputs, output_h, output_c, _, _ = gen_cudnn_rnn_ops.cudnn_rnnv3(**args) - elif time_major is False or num_proj: - batch_id, time_id = (1, 0) if time_major else (0, 1) - batch_size = array_ops.shape(inputs)[batch_id] - max_time = array_ops.shape(inputs)[time_id] - sequence_lengths = array_ops.fill([batch_size], max_time) - args["sequence_lengths"] = sequence_lengths - args["time_major"] = time_major - args["num_proj"] = 0 if num_proj is None else num_proj - outputs, output_h, output_c, _, _ = gen_cudnn_rnn_ops.cudnn_rnnv3(**args) - elif use_cudnn_v2 != "1": - outputs, output_h, output_c, _ = gen_cudnn_rnn_ops.cudnn_rnn(**args) - else: - outputs, output_h, output_c, _, _ = gen_cudnn_rnn_ops.cudnn_rnnv2(**args) - return (outputs, output_h, output_c) - - -def cudnn_lstm(inputs, - input_h, - input_c, - params, - is_training, - sequence_lengths=None, - time_major=True, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - dropout=0., - seed=0, - num_proj=None, - name=None): - """Cudnn LSTM. - - Args: - inputs: the input sequence to the RNN model. If `time_major` is True - (default), the Tensor shape is [max_time, batch_size, input_size]. If - `time_major` is False, the shape is [batch_size, max_time, input_size]. - input_h: the initial hidden state for h. If `time_major` is True (default), - the Tensor shape is [num_layers, batch_size, num_units]. If `time_major` - is False, the shape is [batch_size, num_layers, num_units]. - input_c: the initial hidden state for c. This is only relevant for LSTM. A - Tensor of the same shape as input_h. - params: the parameter buffer created for this model. - is_training: whether this operation will be used in training or inference - sequence_lengths: an int32 array representing the variable sequence lengths - in a batch. The size of the array has to equal the batch_size. Default to - None, in which case sequences in the batch are assumed to have the same - length, which is inferred from inputs. - time_major: The shape format of the `inputs` and `outputs` Tensors. If true, - these Tensors must be shaped ['max_time', 'batch_size', 'depth']. If - false, these Tensors must be shaped ['batch_size', 'max_time', 'depth']. - By default this function accepts input and emits output in time-major - form. This param is only effective when 'sequence_lengths' is used. - input_mode: indicate whether there is a linear projection between the input - and the actual computation before the first layer. It could be - 'linear_input', 'skip_input' or 'auto_select'. 'linear_input' (default) - always applies a linear projection of input onto RNN hidden state. - (standard RNN behavior). 'skip_input' is only allowed when input_size == - num_units; 'auto_select' implies 'skip_input' when input_size == - num_units; otherwise, it implies 'linear_input'. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - dropout: whether to enable dropout. With it is 0, dropout is disabled. - seed: the op seed used for initializing dropout. See - `tf.compat.v1.set_random_seed` for behavior. - num_proj: The output dimensionality for the projection matrices. - If None or 0, no projection is performed. - name: name of the operation. - - Returns: - outputs, output_h, output_c - """ - return _cudnn_rnn(inputs, input_h, input_c, params, is_training, CUDNN_LSTM, - sequence_lengths, time_major, input_mode, direction, - dropout, seed, num_proj, name) - - -def _cudnn_rnn_no_input_c(inputs, - input_h, - params, - is_training, - rnn_mode, - sequence_lengths=None, - time_major=True, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - dropout=0., - seed=0, - name=None): - """Cudnn RNN w/o input_c. - - Args: - inputs: the input sequence to the RNN model. If `time_major` is True - (default), the Tensor shape is [max_time, batch_size, input_size]. If - `time_major` is False, the shape is [batch_size, max_time, input_size]. - input_h: the initial hidden state for h. If `time_major` is True (default), - the Tensor shape is [num_layers, batch_size, num_units]. If `time_major` - is False, the shape is [batch_size, num_layers, num_units]. - params: the parameter buffer created for this model. - is_training: whether this operation will be used in training or inference - rnn_mode: one of ('lstm', 'gru', 'rnn_relu', 'rnn_tanh'). - sequence_lengths: an int32 array representing the variable sequence lengths - in a batch. The size of the array has to equal the batch_size. Default to - None, in which case sequences in the batch are assumed to have the same - length, which is inferred from inputs. - time_major: The shape format of the `inputs` and `outputs` Tensors. If true, - these Tensors must be shaped ['max_time', 'batch_size', 'depth']. If - false, these Tensors must be shaped ['batch_size', 'max_time', 'depth']. - By default this function accepts input and emits output in time-major - form. This param is only effective when 'sequence_lengths' is used. - input_mode: indicate whether there is a linear projection between the input - and the actual computation before the first layer. It could be - 'linear_input', 'skip_input' or 'auto_select'. 'linear_input' (default) - always applies a linear projection of input onto RNN hidden state. - (standard RNN behavior). 'skip_input' is only allowed when input_size == - num_units; 'auto_select' implies 'skip_input' when input_size == - num_units; otherwise, it implies 'linear_input'. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - dropout: whether to enable dropout. With it is 0, dropout is disabled. - seed: the op seed used for initializing dropout. See - `tf.compat.v1.set_random_seed` for behavior. - name: name of the operation. - - Returns: - outputs, output_h - """ - input_c = array_ops.constant([], dtype=input_h.dtype) - outputs, output_h, _ = _cudnn_rnn(inputs, input_h, input_c, params, - is_training, rnn_mode, sequence_lengths, - time_major, input_mode, direction, dropout, - seed, None, name) - return outputs, output_h - - -def cudnn_gru(inputs, - input_h, - params, - is_training, - sequence_lengths=None, - time_major=True, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - dropout=0., - seed=0, - name=None): - """Cudnn GRU. - - Args: - inputs: the input sequence to the RNN model. If `time_major` is True - (default), the Tensor shape is [max_time, batch_size, input_size]. If - `time_major` is False, the shape is [batch_size, max_time, input_size]. - input_h: the initial hidden state for h. If `time_major` is True (default), - the Tensor shape is [num_layers, batch_size, num_units]. If `time_major` - is False, the shape is [batch_size, num_layers, num_units]. - params: the parameter buffer created for this model. - is_training: whether this operation will be used in training or inference - input_mode: indicate whether there is a linear projection between the - input and the actual computation before the first layer. It could be - 'linear_input', 'skip_input' or 'auto_select'. 'linear_input' (default) - always applies a linear projection of input onto RNN hidden state. - (standard RNN behavior). 'skip_input' is only allowed when input_size == - num_units; 'auto_select' implies 'skip_input' when input_size == - num_units; otherwise, it implies 'linear_input'. - sequence_lengths: an int32 array representing the variable sequence lengths - in a batch. The size of the array has to equal the batch_size. Default to - None, in which case sequences in the batch are assumed to have the same - length, which is inferred from inputs. - time_major: The shape format of the `inputs` and `outputs` Tensors. If true, - these Tensors must be shaped ['max_time', 'batch_size', 'depth']. If - false, these Tensors must be shaped ['batch_size', 'max_time', 'depth']. - By default this function accepts input and emits output in time-major - form. This param is only effective when 'sequence_lengths' is used. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - dropout: whether to enable dropout. With it is 0, dropout is disabled. - seed: the op seed used for initializing dropout. See - `tf.compat.v1.set_random_seed` for behavior. - name: name of the operation. - - Returns: - outputs, output_h - """ - return _cudnn_rnn_no_input_c(inputs, input_h, params, is_training, CUDNN_GRU, - sequence_lengths, time_major, input_mode, - direction, dropout, seed, name) - - -def cudnn_rnn_relu(inputs, - input_h, - params, - is_training, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - dropout=0., - seed=0, - sequence_lengths=None, - time_major=True, - name=None): - """Cudnn RNN Relu. - - Args: - inputs: the input sequence to the RNN model. If `time_major` is True - (default), the Tensor shape is [max_time, batch_size, input_size]. If - `time_major` is False, the shape is [batch_size, max_time, input_size]. - input_h: the initial hidden state for h. If `time_major` is True (default), - the Tensor shape is [num_layers, batch_size, num_units]. If `time_major` - is False, the shape is [batch_size, num_layers, num_units]. - params: the parameter buffer created for this model. - is_training: whether this operation will be used in training or inference - input_mode: indicate whether there is a linear projection between the - input and the actual computation before the first layer. It could be - 'linear_input', 'skip_input' or 'auto_select'. 'linear_input' (default) - always applies a linear projection of input onto RNN hidden state. - (standard RNN behavior). 'skip_input' is only allowed when input_size == - num_units; 'auto_select' implies 'skip_input' when input_size == - num_units; otherwise, it implies 'linear_input'. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - dropout: whether to enable dropout. With it is 0, dropout is disabled. - seed: the op seed used for initializing dropout. See - `tf.compat.v1.set_random_seed` for behavior. - sequence_lengths: an int32 array representing the variable sequence lengths - in a batch. The size of the array has to equal the batch_size. If not - provided, the same sequence length will be assumed. - time_major: The shape format of the `inputs` and `outputs` Tensors. If true, - these Tensors must be shaped ['max_time', 'batch_size', 'depth']. If - false, these Tensors must be shaped ['batch_size', 'max_time', 'depth']. - By default this function accepts input and emits output in time-major - form. This param is only effective when 'sequence_lengths' is used. - name: name of the operation. - - Returns: - outputs, output_h - """ - return _cudnn_rnn_no_input_c(inputs, input_h, params, is_training, - CUDNN_RNN_RELU, sequence_lengths, time_major, - input_mode, direction, dropout, seed, name) - - -def cudnn_rnn_tanh(inputs, - input_h, - params, - is_training, - sequence_lengths=None, - time_major=True, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - dropout=0., - seed=0, - name=None): - """Cudnn RNN Tanh. - - Args: - inputs: the input sequence to the RNN model. If `time_major` is True - (default), the Tensor shape is [max_time, batch_size, input_size]. If - `time_major` is False, the shape is [batch_size, max_time, input_size]. - input_h: the initial hidden state for h. If `time_major` is True (default), - the Tensor shape is [num_layers, batch_size, num_units]. If `time_major` - is False, the shape is [batch_size, num_layers, num_units]. - params: the parameter buffer created for this model. - is_training: whether this operation will be used in training or inference - input_mode: indicate whether there is a linear projection between the - input and the actual computation before the first layer. It could be - 'linear_input', 'skip_input' or 'auto_select'. 'linear_input' (default) - always applies a linear projection of input onto RNN hidden state. - (standard RNN behavior). 'skip_input' is only allowed when input_size == - num_units; 'auto_select' implies 'skip_input' when input_size == - num_units; otherwise, it implies 'linear_input'. - sequence_lengths: an int32 array representing the variable sequence lengths - in a batch. The size of the array has to equal the batch_size. Default to - None, in which case sequences in the batch are assumed to have the same - length, which is inferred from inputs. - time_major: The shape format of the `inputs` and `outputs` Tensors. If true, - these Tensors must be shaped ['max_time', 'batch_size', 'depth']. If - false, these Tensors must be shaped ['batch_size', 'max_time', 'depth']. - By default this function accepts input and emits output in time-major - form. This param is only effective when 'sequence_lengths' is used. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - dropout: whether to enable dropout. With it is 0, dropout is disabled. - seed: the op seed used for initializing dropout. See - `tf.compat.v1.set_random_seed` for behavior. - name: name of the operation. - - Returns: - outputs, output_h - """ - return _cudnn_rnn_no_input_c(inputs, input_h, params, is_training, - CUDNN_RNN_TANH, sequence_lengths, time_major, - input_mode, direction, dropout, seed, name) - - -def cudnn_rnn_opaque_params_to_canonical(rnn_mode, - num_layers, - num_units, - input_size, - params, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - dropout=0, - seed=0, - num_proj=None, - name=None): - """Convert cudnn opaque params to canonical. - - Args: - rnn_mode: a string specifies the mode, under which this RNN model runs. - Could be either 'lstm', 'gru', 'rnn_tanh' or 'rnn_relu'. - num_layers: the number of layers for the RNN model. - num_units: the number of units within the RNN model. - input_size: the size of the input, it could be different from the num_units. - params: opaque cudnn params var. - input_mode: indicate whether there is a linear projection between the input - and the actual computation before the first layer. It could be - 'linear_input', 'skip_input' or 'auto_select'. 'linear_input' (default) - always applies a linear projection of input onto RNN hidden state. - (standard RNN behavior). 'skip_input' is only allowed when input_size == - num_units; 'auto_select' implies 'skip_input' when input_size == - num_units; otherwise, it implies 'linear_input'. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - dropout: whether to enable dropout. With it is 0, dropout is disabled. - seed: the op seed used for initializing dropout. See - `tf.compat.v1.set_random_seed` for behavior. - num_proj: The output dimensionality for the projection matrices. - If None or 0, no projection is performed. - name: name of the operation. - - Returns: - weights list and bias list - Raises: - ValueError: if rnn_mode or direction is invalid. - """ - - _check_rnn_mode(rnn_mode) - check_direction(direction) - check_input_mode(input_mode) - num_params = _get_num_params(rnn_mode, num_layers, direction) - seed, seed2 = random_seed.get_seed(seed) - num_dirs = 1 if direction == CUDNN_RNN_UNIDIRECTION else 2 - if num_proj is not None and num_proj != 0: - num_params_weights = (num_params + 1 * num_layers * num_dirs) - num_params_biases = num_params - weights, biases = gen_cudnn_rnn_ops.cudnn_rnn_params_to_canonical_v2( - rnn_mode=rnn_mode, - num_layers=num_layers, - num_units=num_units, - input_size=input_size, - params=params, - input_mode=input_mode, - direction=direction, - dropout=dropout, - seed=seed, - seed2=seed2, - num_params_weights=num_params_weights, - num_params_biases=num_params_biases, - num_proj=num_proj, - name=name) - else: - weights, biases = gen_cudnn_rnn_ops.cudnn_rnn_params_to_canonical( - rnn_mode=rnn_mode, - num_layers=num_layers, - num_units=num_units, - input_size=input_size, - params=params, - input_mode=input_mode, - direction=direction, - dropout=dropout, - seed=seed, - seed2=seed2, - num_params=num_params, - name=name) - return weights, biases - - -def cudnn_rnn_canonical_to_opaque_params(rnn_mode, - num_layers, - num_units, - input_size, - weights, - biases, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - dropout=0, - seed=0, - num_proj=None, - name=None): - """Converts params from the canonical format to a specific format of cuDNN. - - Args: - rnn_mode: a string specifies the mode, under which this RNN model runs. - Could be either 'lstm', 'gru', 'rnn_tanh' or 'rnn_relu'. - num_layers: the number of layers for the RNN model. - num_units: the number of units within the RNN model. - input_size: the size of the input, it could be different from the num_units. - weights: a Tensor for weight parameters. - biases: a Tensor for bias parameters. - input_mode: indicate whether there is a linear projection between the input - and the actual computation before the first layer. It could be - 'linear_input', 'skip_input' or 'auto_select'. 'linear_input' (default) - always applies a linear projection of input onto RNN hidden state. - (standard RNN behavior). 'skip_input' is only allowed when input_size == - num_units; 'auto_select' implies 'skip_input' when input_size == - num_units; otherwise, it implies 'linear_input'. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - dropout: whether to enable dropout. With it is 0, dropout is disabled. - seed: the op seed used for initializing dropout. See - `tf.compat.v1.set_random_seed` for behavior. - num_proj: The output dimensionality for the projection matrices. - If None or 0, no projection is performed. - name: name of the operation. - - Returns: - an opaque Cudnn param. - Raises: - ValueError: if rnn_mode or direction is invalid. - """ - _check_rnn_mode(rnn_mode) - check_direction(direction) - check_input_mode(input_mode) - seed, seed2 = random_seed.get_seed(seed) - if num_proj is not None and num_proj != 0: - return gen_cudnn_rnn_ops.cudnn_rnn_canonical_to_params_v2( - rnn_mode=rnn_mode, - num_layers=num_layers, - num_units=num_units, - input_size=input_size, - weights=weights, - biases=biases, - input_mode=input_mode, - direction=direction, - dropout=dropout, - seed=seed, - seed2=seed2, - num_proj=num_proj, - name=name) - else: - return gen_cudnn_rnn_ops.cudnn_rnn_canonical_to_params( - rnn_mode=rnn_mode, - num_layers=num_layers, - num_units=num_units, - input_size=input_size, - weights=weights, - biases=biases, - input_mode=input_mode, - direction=direction, - dropout=dropout, - seed=seed, - seed2=seed2, - name=name) - - -def cudnn_rnn_opaque_params_size(rnn_mode, - num_layers, - num_units, - input_size, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - dtype=dtypes.float32, - dropout=0, - seed=0, - num_proj=None, - name=None): - """Returns opaque params size for specific Cudnn config. - - Args: - rnn_mode: a string specifies the mode, under which this RNN model runs. - Could be either 'lstm', 'gru', 'rnn_tanh' or 'rnn_relu'. - num_layers: the number of layers for the RNN model. - num_units: the number of units within the RNN model. - input_size: the size of the input, it could be different from the num_units. - input_mode: indicate whether there is a linear projection between the input - and the actual computation before the first layer. It could be - 'linear_input', 'skip_input' or 'auto_select'. 'linear_input' (default) - always applies a linear projection of input onto RNN hidden state. - (standard RNN behavior). 'skip_input' is only allowed when input_size == - num_units; 'auto_select' implies 'skip_input' when input_size == - num_units; otherwise, it implies 'linear_input'. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - dtype: one of tf.float32 or tf.float64. - dropout: whether to enable dropout. With it is 0, dropout is disabled. - seed: the op seed used for initializing dropout. See - `tf.compat.v1.set_random_seed` for behavior. - num_proj: The output dimensionality for the projection matrices. - If None or 0, no projection is performed. - name: name of the operation. - - Returns: - a int, size of Cudnn opaque params. - Raises: - ValueError: if rnn_mode or direction is invalid. - """ - _check_rnn_mode(rnn_mode) - check_direction(direction) - check_input_mode(input_mode) - seed, seed2 = random_seed.get_seed(seed) - return gen_cudnn_rnn_ops.cudnn_rnn_params_size( - rnn_mode=rnn_mode, - num_layers=num_layers, - num_units=num_units, - input_size=input_size, - num_proj=num_proj, - T=dtype, - S=dtypes.int32, - dropout=dropout, - seed=seed, - seed2=seed2, - input_mode=input_mode, - direction=direction, - name=name)[0] - - -class _CudnnRNN(object): - """Creates an RNN model using the underlying Cudnn implementation. - - Note that self._NUM_PARAMS_PER_LAYER is the number of parameter sets of - weight and bias per layer. It needs to be defined in subclasses. - """ - __doc__ += _cudnn_rnn_common_doc_string - - # TODO(jamesqin): support float16 CuDNN RNN - def __init__(self, - rnn_mode, - num_layers, - num_units, - input_size, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - dtype=dtypes.float32, - dropout=0., - seed=0, - num_proj=None): - """Creates a CudnnRNN model from model spec. - - Args: - rnn_mode: a string specifies the mode, under which this RNN model runs. - Could be either 'lstm', 'gru', 'rnn_tanh' or 'rnn_relu'. - num_layers: the number of layers for the RNN model. - num_units: the number of units within the RNN model. - input_size: the size of the input, it could be different from the - num_units. - input_mode: indicate whether there is a linear projection between the - input and the actual computation before the first layer. It could be - 'linear_input', 'skip_input' or 'auto_select'. 'linear_input' (default) - always applies a linear projection of input onto RNN hidden state. - (standard RNN behavior). 'skip_input' is only allowed when input_size == - num_units; 'auto_select' implies 'skip_input' when input_size == - num_units; otherwise, it implies 'linear_input'. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - dtype: dtype of params, tf.float32 or tf.float64. - dropout: whether to enable dropout. With it is 0, dropout is disabled. - seed: the op seed used for initializing dropout. See - `tf.compat.v1.set_random_seed` for behavior. - num_proj: The output dimensionality for the projection matrices. - If None or 0, no projection is performed. - - Raises: - ValueError: if direction is invalid. - """ - self._num_layers = num_layers - self._num_units = num_units - self._input_size = input_size - self._rnn_mode = rnn_mode - self._input_mode = input_mode - self._direction = direction - self._dtype = dtype - self._dropout = dropout - self._seed = seed - self._num_proj = num_proj - - @property - def input_mode(self): - return self._input_mode - - @property - def input_size(self): - return self._input_size - - @property - def num_units(self): - return self._num_units - - @property - def num_layers(self): - return self._num_layers - - @property - def rnn_mode(self): - return self._rnn_mode - - @property - def direction(self): - return self._direction - - @property - def num_proj(self): - return self._num_proj - - def params_size(self): - """Calculates the size of the opaque parameter buffer needed for this model. - - Returns: - The calculated parameter buffer size. - """ - return cudnn_rnn_opaque_params_size( - rnn_mode=self._rnn_mode, - num_layers=self._num_layers, - num_units=self._num_units, - input_size=self._input_size, - num_proj=self._num_proj, - dtype=self._dtype, - dropout=self._dropout, - seed=self._seed, - input_mode=self._input_mode, - direction=self._direction) - - def __call__(self, - input_data, - input_h, - input_c, - params, - is_training=True, - sequence_lengths=None, - time_major=True): - """Runs the forward step for the RNN model. - - Args: - input_data: the input sequence to the RNN model. If `time_major` is True - (default), the Tensor shape is [max_time, batch_size, input_size]. If - `time_major` is False, the shape is [batch_size, max_time, input_size]. - input_h: the initial hidden state for h. If `time_major` is True - (default), the Tensor shape is [num_layers, batch_size, num_units]. If - `time_major` is False, the shape is [batch_size, num_layers, num_units]. - input_c: the initial hidden state for c. This is only relevant for LSTM. A - Tensor of the same shape as input_h. - params: the parameter buffer created for this model. - is_training: whether this operation will be used in training or inference. - sequence_lengths: an int32 array representing the variable sequence - lengths in a batch. The size of the array has to equal the batch_size. - Default to None, in which case sequences in the batch are assumed to - have the same length, which is inferred from inputs. - time_major: The shape format of the `inputs` and `outputs` Tensors. If - true, these Tensors must be shaped ['max_time', 'batch_size', 'depth']. - If false, these Tensors must be shaped ['batch_size', 'max_time', - 'depth']. By default this function accepts input and emits output in - time-major form. This param is only effective when 'sequence_lengths' is - used. - - Returns: - output: the output sequence. - output_h: the final state for h. - output_c: the final state for c. This is only relevant for LSTM. - """ - return _cudnn_rnn( - input_data, - input_h, - input_c, - params, - is_training, - self._rnn_mode, - sequence_lengths=sequence_lengths, - time_major=time_major, - input_mode=self._input_mode, - direction=self._direction, - dropout=self._dropout, - seed=self._seed, - num_proj=self._num_proj) - - def params_to_canonical(self, params): - """Converts params from a specific format of cuDNN to the canonical format. - - Args: - params: a Variable for weight and bias parameters. - - Returns: - A function for the specific-to-canonical conversion. - """ - return cudnn_rnn_opaque_params_to_canonical( - rnn_mode=self._rnn_mode, - num_layers=self._num_layers, - num_units=self._num_units, - input_size=self._input_size, - params=params, - input_mode=self._input_mode, - direction=self._direction, - dropout=self._dropout, - seed=self._seed, - num_proj=self._num_proj) - - def canonical_to_params(self, weights, biases): - """Converts params from the canonical format to a specific format of cuDNN. - - Args: - weights: a Tensor for weight parameters. - biases: a Tensor for bias parameters. - - Returns: - A function for the canonical-to-params-to-specific conversion.. - """ - return cudnn_rnn_canonical_to_opaque_params( - rnn_mode=self._rnn_mode, - num_layers=self._num_layers, - num_units=self._num_units, - input_size=self._input_size, - weights=weights, - biases=biases, - input_mode=self._input_mode, - direction=self._direction, - dropout=self._dropout, - seed=self._seed, - num_proj=self._num_proj) - - -class CudnnLSTM(_CudnnRNN): - """Cudnn implementation of the LSTM model.""" - __doc__ += _cudnn_rnn_common_doc_string - # 4 sets of weight and bias parameters for the recurrent input, and 4 for the - # previous layer input. - _NUM_PARAMS_PER_LAYER = CUDNN_LSTM_PARAMS_PER_LAYER - - def __init__(self, - num_layers, - num_units, - input_size, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - dtype=dtypes.float32, - dropout=0., - seed=0, - num_proj=None): - """Creates a Cudnn LSTM model from model spec. - - Args: - num_layers: the number of layers for the RNN model. - num_units: the number of units within the RNN model. - input_size: the size of the input, it could be different from the - num_units. - input_mode: indicate whether there is a linear projection between the - input and The actual computation before the first layer. It could be - 'skip_input', 'linear_input' or 'auto_select'. 'skip_input' is only - allowed when input_size == num_units; 'auto_select' implies 'skip_input' - when input_size == num_units; otherwise, it implies 'linear_input'. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - dtype: dtype of params, tf.float32 or tf.float64. - dropout: whether to enable dropout. With it is 0, dropout is disabled. - seed: the seed used for initializing dropout. - num_proj: The output dimensionality for the projection matrices. - If None or 0, no projection is performed. - """ - super(CudnnLSTM, self).__init__( - CUDNN_LSTM, - num_layers, - num_units, - input_size, - input_mode=input_mode, - direction=direction, - dtype=dtype, - dropout=dropout, - seed=seed, - num_proj=num_proj) - - def __call__(self, - input_data, - input_h, - input_c, - params, - sequence_lengths=None, - time_major=True, - is_training=True): - """Runs the forward step for the Cudnn LSTM model. - - Args: - input_data: the input sequence to the RNN model. If `time_major` is True - (default), the Tensor shape is [max_time, batch_size, input_size]. If - `time_major` is False, the shape is [batch_size, max_time, input_size]. - input_h: the initial hidden state for h. If `time_major` is True - (default), the Tensor shape is [num_layers, batch_size, num_units]. If - `time_major` is False, the shape is [batch_size, num_layers, num_units]. - input_c: the initial hidden state for c. A Tensor of the same shape as - input_h. - params: the parameter buffer created for this model. - sequence_lengths: an int32 array representing the variable sequence - lengths in a batch. The size of the array has to equal the batch_size. - Default to None, in which case sequences in the batch are assumed to - have the same length, which is inferred from inputs. - time_major: The shape format of the `inputs` and `outputs` Tensors. If - true, these Tensors must be shaped ['max_time', 'batch_size', 'depth']. - If false, these Tensors must be shaped ['batch_size', 'max_time', - 'depth']. By default this function accepts input and emits output in - time-major form. This param is only effective when 'sequence_lengths' is - used. - is_training: whether this operation will be used in training or inference. - - Returns: - output: the output sequence. - output_h: the final state for h. - output_c: the final state for c. - """ - output, output_h, output_c = super(CudnnLSTM, self).__call__( - input_data, - input_h, - input_c, - params, - sequence_lengths=sequence_lengths, - time_major=time_major, - is_training=is_training) - return (output, output_h, output_c) - - -class _CudnnRNNNoInputC(_CudnnRNN): - """Simple CudnnRNN models without input_c.""" - __doc__ += _cudnn_rnn_common_doc_string - - def __init__(self, - num_layers, - num_units, - input_size, - input_mode=CUDNN_INPUT_LINEAR_MODE, - direction=CUDNN_RNN_UNIDIRECTION, - dtype=dtypes.float32, - dropout=0., - seed=0): - """Creates a Cudnn RNN model from model without hidden-state C. - - Args: - num_layers: the number of layers for the RNN model. - num_units: the number of units within the RNN model. - input_size: the size of the input, it could be different from the - num_units. - input_mode: indicate whether there is a linear projection between the - input and The actual computation before the first layer. It could be - 'skip_input', 'linear_input' or 'auto_select'. 'skip_input' is only - allowed when input_size == num_units; 'auto_select' implies 'skip_input' - when input_size == num_units; otherwise, it implies 'linear_input'. - direction: the direction model that the model operates. Could be either - 'unidirectional' or 'bidirectional' - dtype: dtype of params, tf.float32 or tf.float64. - dropout: whether to enable dropout. With it is 0, dropout is disabled. - seed: the seed used for initializing dropout. - - Raises: - ValueError: if direction is not 'unidirectional' or 'bidirectional'. - """ - - if direction not in (CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION): - raise ValueError("Invalid direction: %s" % direction) - - super(_CudnnRNNNoInputC, self).__init__( - self._rnn_mode, - num_layers, - num_units, - input_size, - input_mode=input_mode, - direction=direction, - dtype=dtype, - dropout=dropout, - seed=seed) - - def __call__(self, - input_data, - input_h, - params, - sequence_lengths=None, - time_major=True, - is_training=True): - """Runs the forward step for the Cudnn LSTM model. - - Args: - input_data: the input sequence to the RNN model. If `time_major` is True - (default), the Tensor shape is [max_time, batch_size, input_size]. If - `time_major` is False, the shape is [batch_size, max_time, input_size]. - input_h: the initial hidden state for h. If `time_major` is True - (default), the Tensor shape is [num_layers, batch_size, num_units]. If - `time_major` is False, the shape is [batch_size, num_layers, num_units]. - params: the parameter buffer created for this model. - sequence_lengths: an int32 array representing the variable sequence - lengths in a batch. The size of the array has to equal the batch_size. - Default to None, in which case sequences in the batch are assumed to - have the same length, which is inferred from inputs. - time_major: The shape format of the `inputs` and `outputs` Tensors. If - true, these Tensors must be shaped ['max_time', 'batch_size', 'depth']. - If false, these Tensors must be shaped ['batch_size', 'max_time', - 'depth']. By default this function accepts input and emits output in - time-major form. This param is only effective when 'sequence_lengths' is - used. - is_training: whether this operation will be used in training or inference. - - Returns: - output: the output sequence. - output_h: the final state for h. - """ - return _cudnn_rnn_no_input_c( - input_data, - input_h, - params, - is_training, - self._rnn_mode, - sequence_lengths=sequence_lengths, - time_major=time_major, - input_mode=self._input_mode, - direction=self._direction, - dropout=self._dropout, - seed=self._seed) - - -class CudnnGRU(_CudnnRNNNoInputC): - """Cudnn implementation of the GRU model.""" - __doc__ += _cudnn_rnn_common_doc_string - _rnn_mode = CUDNN_GRU - # 3 sets of weight and bias parameters for the recurrent input, and 3 for the - # previous layer input. - _NUM_PARAMS_PER_LAYER = CUDNN_GRU_PARAMS_PER_LAYER - - -class CudnnRNNTanh(_CudnnRNNNoInputC): - """Cudnn implementation of the RNN-tanh model.""" - __doc__ += _cudnn_rnn_common_doc_string - _rnn_mode = CUDNN_RNN_TANH - # 1 set of weight and bias parameters for the recurrent input, and 1 for the - # previous layer input. - _NUM_PARAMS_PER_LAYER = CUDNN_RNN_TANH_PARAMS_PER_LAYER - - -class CudnnRNNRelu(_CudnnRNNNoInputC): - """Cudnn implementation of the RNN-relu model.""" - __doc__ += _cudnn_rnn_common_doc_string - _rnn_mode = CUDNN_RNN_RELU - # 1 set of weight and bias parameters for the recurrent input, and 1 for the - # previous layer input. - _NUM_PARAMS_PER_LAYER = CUDNN_RNN_RELU_PARAMS_PER_LAYER diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD deleted file mode 100644 index 74e3ae067d6..00000000000 --- a/tensorflow/contrib/data/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "data", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:iterator_ops", - "//tensorflow/python:util", - ], -) diff --git a/tensorflow/contrib/data/README.md b/tensorflow/contrib/data/README.md deleted file mode 100644 index 90be7a66cac..00000000000 --- a/tensorflow/contrib/data/README.md +++ /dev/null @@ -1,41 +0,0 @@ -`tf.contrib.data` API -===================== - -NOTE: The `tf.contrib.data` module has been deprecated. Use `tf.data` instead, -or `tf.data.experimental` for the experimental transformations previously hosted -in this module. We are continuing to support existing code using the -`tf.contrib.data` APIs in the current version of TensorFlow, but will eventually -remove support. The non-experimental `tf.data` APIs are subject to backwards -compatibility guarantees. - -Porting your code to `tf.data` ------------------------------- - -The `tf.contrib.data.Dataset` class has been renamed to `tf.data.Dataset`, and -the `tf.contrib.data.Iterator` class has been renamed to `tf.data.Iterator`. -Most code can be ported by removing `.contrib` from the names of the classes. -However, there are some small differences, which are outlined below. - -The arguments accepted by the `Dataset.map()` transformation have changed: - -* `dataset.map(..., num_threads=T)` is now `dataset.map(num_parallel_calls=T)`. -* `dataset.map(..., output_buffer_size=B)` is now - `dataset.map(...).prefetch(B)`. - -Some transformations have been removed from `tf.data.Dataset`, and you must -instead apply them using `Dataset.apply()` transformation. The full list of -changes is as follows: - -* `dataset.dense_to_sparse_batch(...)` is now - `dataset.apply(tf.data.experimental.dense_to_sparse_batch(...)`. -* `dataset.enumerate(...)` is now - `dataset.apply(tf.data.experimental.enumerate_dataset(...))`. -* `dataset.group_by_window(...)` is now - `dataset.apply(tf.data.experimental.group_by_window(...))`. -* `dataset.ignore_errors()` is now - `dataset.apply(tf.data.experimental.ignore_errors())`. -* `dataset.unbatch()` is now `dataset.apply(tf.contrib.data.unbatch())`. - -The `Dataset.make_dataset_resource()` and `Iterator.dispose_op()` methods have -been removed from the API. Please open a GitHub issue if you have a need for -either of these. diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py deleted file mode 100644 index 5390538e1f2..00000000000 --- a/tensorflow/contrib/data/__init__.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Experimental API for building input pipelines. - -This module contains experimental `Dataset` sources and transformations that can -be used in conjunction with the `tf.data.Dataset` API. Note that the -`tf.contrib.data` API is not subject to the same backwards compatibility -guarantees as `tf.data`, but we will provide deprecation advice in advance of -removing existing functionality. - -See [Importing Data](https://tensorflow.org/guide/datasets) for an overview. - -@@Counter -@@CheckpointInputPipelineHook -@@CsvDataset -@@LMDBDataset -@@Optional -@@RandomDataset -@@Reducer -@@SqlDataset -@@TFRecordWriter - -@@assert_element_shape -@@batch_and_drop_remainder -@@bucket_by_sequence_length -@@choose_from_datasets -@@copy_to_device -@@dense_to_sparse_batch -@@enumerate_dataset -@@get_next_as_optional -@@get_single_element -@@group_by_reducer -@@group_by_window -@@ignore_errors -@@latency_stats -@@make_batched_features_dataset -@@make_csv_dataset -@@make_saveable_from_iterator -@@map_and_batch -@@padded_batch_and_drop_remainder -@@parallel_interleave -@@parse_example_dataset -@@prefetch_to_device -@@read_batch_features -@@rejection_resample -@@reduce_dataset -@@sample_from_datasets -@@scan -@@set_stats_aggregator -@@shuffle_and_repeat -@@sliding_window_batch -@@sloppy_interleave -@@StatsAggregator -@@unbatch -@@unique - -@@AUTOTUNE -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import - -from tensorflow.contrib.data.python.ops.batching import assert_element_shape -from tensorflow.contrib.data.python.ops.batching import batch_and_drop_remainder -from tensorflow.contrib.data.python.ops.batching import dense_to_sparse_batch -from tensorflow.contrib.data.python.ops.batching import map_and_batch -from tensorflow.contrib.data.python.ops.batching import padded_batch_and_drop_remainder -from tensorflow.contrib.data.python.ops.batching import unbatch -from tensorflow.contrib.data.python.ops.counter import Counter -from tensorflow.contrib.data.python.ops.enumerate_ops import enumerate_dataset -from tensorflow.contrib.data.python.ops.error_ops import ignore_errors -from tensorflow.contrib.data.python.ops.get_single_element import get_single_element -from tensorflow.contrib.data.python.ops.get_single_element import reduce_dataset -from tensorflow.contrib.data.python.ops.grouping import bucket_by_sequence_length -from tensorflow.contrib.data.python.ops.grouping import group_by_reducer -from tensorflow.contrib.data.python.ops.grouping import group_by_window -from tensorflow.contrib.data.python.ops.grouping import Reducer -from tensorflow.contrib.data.python.ops.interleave_ops import choose_from_datasets -from tensorflow.contrib.data.python.ops.interleave_ops import parallel_interleave -from tensorflow.contrib.data.python.ops.interleave_ops import sample_from_datasets -from tensorflow.contrib.data.python.ops.interleave_ops import sloppy_interleave -from tensorflow.contrib.data.python.ops.iterator_ops import CheckpointInputPipelineHook -from tensorflow.contrib.data.python.ops.iterator_ops import make_saveable_from_iterator -from tensorflow.contrib.data.python.ops.parsing_ops import parse_example_dataset -from tensorflow.contrib.data.python.ops.prefetching_ops import copy_to_device -from tensorflow.contrib.data.python.ops.prefetching_ops import prefetch_to_device -from tensorflow.contrib.data.python.ops.random_ops import RandomDataset -from tensorflow.contrib.data.python.ops.readers import CsvDataset -from tensorflow.contrib.data.python.ops.readers import LMDBDataset -from tensorflow.contrib.data.python.ops.readers import make_batched_features_dataset -from tensorflow.contrib.data.python.ops.readers import make_csv_dataset -from tensorflow.contrib.data.python.ops.readers import read_batch_features -from tensorflow.contrib.data.python.ops.readers import SqlDataset -from tensorflow.contrib.data.python.ops.resampling import rejection_resample -from tensorflow.contrib.data.python.ops.scan_ops import scan -from tensorflow.contrib.data.python.ops.shuffle_ops import shuffle_and_repeat -from tensorflow.contrib.data.python.ops.sliding import sliding_window_batch -from tensorflow.contrib.data.python.ops.unique import unique -from tensorflow.contrib.data.python.ops.writers import TFRecordWriter -from tensorflow.python.data.ops.dataset_ops import AUTOTUNE -from tensorflow.python.data.ops.iterator_ops import get_next_as_optional -from tensorflow.python.data.ops.optional_ops import Optional -# pylint: enable=unused-import - -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD deleted file mode 100644 index bc55aa7e10a..00000000000 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ /dev/null @@ -1,112 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_test( - name = "assert_element_shape_test", - srcs = ["assert_element_shape_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/data/python/ops:batching", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", - "//tensorflow/python:script_ops", - "//tensorflow/python:string_ops", - "//tensorflow/python:tensor_shape", - "//tensorflow/python/data/kernel_tests:test_base", - "//tensorflow/python/data/ops:dataset_ops", - "//third_party/py/numpy", - ], -) - -py_test( - name = "lmdb_dataset_op_test", - size = "medium", - srcs = ["lmdb_dataset_op_test.py"], - data = ["//tensorflow/core:lmdb_testdata"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_pip", - "no_windows", - ], - deps = [ - "//tensorflow/contrib/data/python/ops:readers", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:session", - "//tensorflow/python/data/kernel_tests:test_base", - "//third_party/py/numpy", - ], -) - -py_test( - name = "reduce_dataset_test", - size = "small", - srcs = ["reduce_dataset_test.py"], - python_version = "PY2", - deps = [ - "//tensorflow/contrib/data/python/ops:get_single_element", - "//tensorflow/contrib/data/python/ops:grouping", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python/data/kernel_tests:test_base", - "//tensorflow/python/data/ops:dataset_ops", - "@absl_py//absl/testing:parameterized", - ], -) - -py_test( - name = "restructured_dataset_test", - size = "medium", - srcs = ["restructured_dataset_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/data/python/ops:batching", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python/data/experimental/ops:batching", - "//tensorflow/python/data/kernel_tests:test_base", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/util:nest", - ], -) - -py_test( - name = "slide_dataset_op_test", - size = "small", - srcs = ["slide_dataset_op_test.py"], - python_version = "PY2", - deps = [ - "//tensorflow/contrib/data/python/ops:sliding", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:math_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python/data/kernel_tests:test_base", - "//tensorflow/python/data/ops:dataset_ops", - "//third_party/py/numpy", - "@absl_py//absl/testing:parameterized", - ], -) diff --git a/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py b/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py deleted file mode 100644 index 077571fcd20..00000000000 --- a/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py +++ /dev/null @@ -1,235 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.data.python.ops import batching -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import script_ops -from tensorflow.python.platform import test - - -@test_util.run_v1_only("deprecated API, no eager or V2 test coverage") -class AssertElementShapeTest(test_base.DatasetTestBase): - - def test_assert_element_shape(self): - - def create_dataset(_): - return (array_ops.ones(2, dtype=dtypes.float32), - array_ops.zeros((3, 4), dtype=dtypes.int32)) - - dataset = dataset_ops.Dataset.range(5).map(create_dataset) - expected_shapes = (tensor_shape.TensorShape(2), - tensor_shape.TensorShape((3, 4))) - self.assertEqual(expected_shapes, - dataset_ops.get_legacy_output_shapes(dataset)) - - result = dataset.apply(batching.assert_element_shape(expected_shapes)) - self.assertEqual(expected_shapes, - dataset_ops.get_legacy_output_shapes(result)) - - iterator = dataset_ops.make_initializable_iterator(result) - init_op = iterator.initializer - get_next = iterator.get_next() - with self.cached_session() as sess: - sess.run(init_op) - for _ in range(5): - sess.run(get_next) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def test_assert_wrong_element_shape(self): - - def create_dataset(_): - return (array_ops.ones(2, dtype=dtypes.float32), - array_ops.zeros((3, 4), dtype=dtypes.int32)) - - dataset = dataset_ops.Dataset.range(3).map(create_dataset) - wrong_shapes = (tensor_shape.TensorShape(2), - tensor_shape.TensorShape((3, 10))) - with self.assertRaises(ValueError): - dataset.apply(batching.assert_element_shape(wrong_shapes)) - - def test_assert_element_shape_on_unknown_shape_dataset(self): - - def create_unknown_shape_dataset(x): - return script_ops.py_func( - lambda _: ( # pylint: disable=g-long-lambda - np.ones(2, dtype=np.float32), - np.zeros((3, 4), dtype=np.int32)), - [x], - [dtypes.float32, dtypes.int32]) - - dataset = dataset_ops.Dataset.range(5).map(create_unknown_shape_dataset) - unknown_shapes = (tensor_shape.TensorShape(None), - tensor_shape.TensorShape(None)) - self.assertEqual(unknown_shapes, - dataset_ops.get_legacy_output_shapes(dataset)) - - expected_shapes = (tensor_shape.TensorShape(2), - tensor_shape.TensorShape((3, 4))) - result = dataset.apply(batching.assert_element_shape(expected_shapes)) - self.assertEqual(expected_shapes, - dataset_ops.get_legacy_output_shapes(result)) - - iterator = dataset_ops.make_initializable_iterator(result) - init_op = iterator.initializer - get_next = iterator.get_next() - with self.cached_session() as sess: - sess.run(init_op) - for _ in range(5): - sess.run(get_next) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def test_assert_wrong_element_shape_on_unknown_shape_dataset(self): - - def create_unknown_shape_dataset(x): - return script_ops.py_func( - lambda _: ( # pylint: disable=g-long-lambda - np.ones(2, dtype=np.float32), - np.zeros((3, 4), dtype=np.int32)), - [x], - [dtypes.float32, dtypes.int32]) - - dataset = dataset_ops.Dataset.range(3).map(create_unknown_shape_dataset) - unknown_shapes = (tensor_shape.TensorShape(None), - tensor_shape.TensorShape(None)) - self.assertEqual(unknown_shapes, - dataset_ops.get_legacy_output_shapes(dataset)) - - wrong_shapes = (tensor_shape.TensorShape(2), - tensor_shape.TensorShape((3, 10))) - iterator = dataset_ops.make_initializable_iterator( - dataset.apply(batching.assert_element_shape(wrong_shapes))) - init_op = iterator.initializer - get_next = iterator.get_next() - with self.cached_session() as sess: - sess.run(init_op) - with self.assertRaises(errors.InvalidArgumentError): - sess.run(get_next) - - def test_assert_partial_element_shape(self): - - def create_dataset(_): - return (array_ops.ones(2, dtype=dtypes.float32), - array_ops.zeros((3, 4), dtype=dtypes.int32)) - - dataset = dataset_ops.Dataset.range(5).map(create_dataset) - partial_expected_shape = ( - tensor_shape.TensorShape(None), # Unknown shape - tensor_shape.TensorShape((None, 4))) # Partial shape - result = dataset.apply( - batching.assert_element_shape(partial_expected_shape)) - # Partial shapes are merged with actual shapes: - actual_shapes = (tensor_shape.TensorShape(2), - tensor_shape.TensorShape((3, 4))) - self.assertEqual(actual_shapes, - dataset_ops.get_legacy_output_shapes(result)) - - iterator = dataset_ops.make_initializable_iterator(result) - init_op = iterator.initializer - get_next = iterator.get_next() - with self.cached_session() as sess: - sess.run(init_op) - for _ in range(5): - sess.run(get_next) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def test_assert_wrong_partial_element_shape(self): - - def create_dataset(_): - return (array_ops.ones(2, dtype=dtypes.float32), - array_ops.zeros((3, 4), dtype=dtypes.int32)) - - dataset = dataset_ops.Dataset.range(3).map(create_dataset) - wrong_shapes = (tensor_shape.TensorShape(2), - tensor_shape.TensorShape((None, 10))) - with self.assertRaises(ValueError): - dataset.apply(batching.assert_element_shape(wrong_shapes)) - - def test_assert_partial_element_shape_on_unknown_shape_dataset(self): - - def create_unknown_shape_dataset(x): - return script_ops.py_func( - lambda _: ( # pylint: disable=g-long-lambda - np.ones(2, dtype=np.float32), - np.zeros((3, 4), dtype=np.int32)), - [x], - [dtypes.float32, dtypes.int32]) - - dataset = dataset_ops.Dataset.range(5).map(create_unknown_shape_dataset) - unknown_shapes = (tensor_shape.TensorShape(None), - tensor_shape.TensorShape(None)) - self.assertEqual(unknown_shapes, - dataset_ops.get_legacy_output_shapes(dataset)) - - expected_shapes = (tensor_shape.TensorShape(2), - tensor_shape.TensorShape((None, 4))) - result = dataset.apply(batching.assert_element_shape(expected_shapes)) - self.assertEqual(expected_shapes, - dataset_ops.get_legacy_output_shapes(result)) - - iterator = dataset_ops.make_initializable_iterator(result) - init_op = iterator.initializer - get_next = iterator.get_next() - with self.cached_session() as sess: - sess.run(init_op) - for _ in range(5): - sess.run(get_next) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def test_assert_wrong_partial_element_shape_on_unknown_shape_dataset(self): - - def create_unknown_shape_dataset(x): - return script_ops.py_func( - lambda _: ( # pylint: disable=g-long-lambda - np.ones(2, dtype=np.float32), - np.zeros((3, 4), dtype=np.int32)), - [x], - [dtypes.float32, dtypes.int32]) - - dataset = dataset_ops.Dataset.range(3).map(create_unknown_shape_dataset) - unknown_shapes = (tensor_shape.TensorShape(None), - tensor_shape.TensorShape(None)) - self.assertEqual(unknown_shapes, - dataset_ops.get_legacy_output_shapes(dataset)) - - wrong_shapes = (tensor_shape.TensorShape(2), - tensor_shape.TensorShape((None, 10))) - iterator = dataset_ops.make_initializable_iterator( - dataset.apply(batching.assert_element_shape(wrong_shapes))) - init_op = iterator.initializer - get_next = iterator.get_next() - with self.cached_session() as sess: - sess.run(init_op) - with self.assertRaises(errors.InvalidArgumentError): - sess.run(get_next) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py deleted file mode 100644 index d5bcdebf81a..00000000000 --- a/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for LMDBDatasetOp.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import shutil -import sys - -from tensorflow.contrib.data.python.ops import readers -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import test_util -from tensorflow.python.platform import test -from tensorflow.python.util import compat - -prefix_path = "tensorflow/core/lib" - - -@test_util.run_v1_only("deprecated API, no eager or V2 test coverage") -class LMDBDatasetTest(test_base.DatasetTestBase): - - def setUp(self): - super(LMDBDatasetTest, self).setUp() - # Copy database out because we need the path to be writable to use locks. - # The on-disk format of an LMDB database is different on big-endian - # machines, because LMDB is a memory-mapped database. - db_file = "data.mdb" if sys.byteorder == "little" else "data_bigendian.mdb" - path = os.path.join(prefix_path, "lmdb", "testdata", db_file) - self.db_path = os.path.join(self.get_temp_dir(), "data.mdb") - shutil.copy(path, self.db_path) - - def testReadFromFile(self): - filename = self.db_path - - filenames = constant_op.constant([filename], dtypes.string) - num_repeats = 2 - - dataset = readers.LMDBDataset(filenames).repeat(num_repeats) - iterator = dataset_ops.make_initializable_iterator(dataset) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - for _ in range(num_repeats): # Dataset is repeated. - for i in range(10): # 10 records. - k = compat.as_bytes(str(i)) - v = compat.as_bytes(str(chr(ord("a") + i))) - self.assertEqual((k, v), sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/data/python/kernel_tests/reduce_dataset_test.py b/tensorflow/contrib/data/python/kernel_tests/reduce_dataset_test.py deleted file mode 100644 index 8132f81c1cd..00000000000 --- a/tensorflow/contrib/data/python/kernel_tests/reduce_dataset_test.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy as np - -from tensorflow.contrib.data.python.ops import get_single_element -from tensorflow.contrib.data.python.ops import grouping -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import test_util -from tensorflow.python.platform import test - - -@test_util.run_all_in_graph_and_eager_modes -class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): - - @parameterized.named_parameters( - ("SumZero", 0), - ("SumOne", 1), - ("SumFive", 5), - ("SumTen", 10), - ) - def testReduceDataset(self, stop): - def init_fn(_): - return np.int64(0) - - def reduce_fn(state, value): - return state + value - - def finalize_fn(state): - return state - - sum_reducer = grouping.Reducer(init_fn, reduce_fn, finalize_fn) - - dataset = dataset_ops.Dataset.range(stop) - element = get_single_element.reduce_dataset(dataset, sum_reducer) - - self.assertEqual(stop * (stop - 1) / 2, self.evaluate(element)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/data/python/kernel_tests/restructured_dataset_test.py b/tensorflow/contrib/data/python/kernel_tests/restructured_dataset_test.py deleted file mode 100644 index 607ce5e8c8b..00000000000 --- a/tensorflow/contrib/data/python/kernel_tests/restructured_dataset_test.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the private `_RestructuredDataset` transformation.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.data.python.ops import batching -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.util import nest -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -# TODO(b/117581999): Add eager specific test. -class RestructuredDatasetTest(test_base.DatasetTestBase): - - @test_util.run_deprecated_v1 - def testRestructureDataset(self): - components = (array_ops.placeholder(dtypes.int32), - (array_ops.placeholder(dtypes.int32, shape=[None]), - array_ops.placeholder(dtypes.int32, shape=[20, 30]))) - dataset = dataset_ops.Dataset.from_tensors(components) - - i32 = dtypes.int32 - - test_cases = [((i32, i32, i32), None), - (((i32, i32), i32), None), - ((i32, i32, i32), (None, None, None)), - ((i32, i32, i32), ([17], [17], [20, 30]))] - - for new_types, new_shape_lists in test_cases: - # pylint: disable=protected-access - new = batching._RestructuredDataset(dataset, new_types, new_shape_lists) - # pylint: enable=protected-access - self.assertEqual(new_types, dataset_ops.get_legacy_output_types(new)) - if new_shape_lists is not None: - for expected_shape_list, shape in zip( - nest.flatten(new_shape_lists), - nest.flatten(dataset_ops.get_legacy_output_shapes(new))): - if expected_shape_list is None: - self.assertIs(None, shape.ndims) - else: - self.assertEqual(expected_shape_list, shape.as_list()) - - fail_cases = [((i32, dtypes.int64, i32), None), - ((i32, i32, i32, i32), None), - ((i32, i32, i32), ((None, None), None)), - ((i32, i32, i32), (None, None, None, None)), - ((i32, i32, i32), (None, [None], [21, 30]))] - - for new_types, new_shape_lists in fail_cases: - with self.assertRaises(ValueError): - # pylint: disable=protected-access - new = batching._RestructuredDataset(dataset, new_types, new_shape_lists) - # pylint: enable=protected-access - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py deleted file mode 100644 index 95c1e016112..00000000000 --- a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py +++ /dev/null @@ -1,325 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy as np - -from tensorflow.contrib.data.python.ops import sliding -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -@test_util.run_v1_only("deprecated API, no eager or V2 test coverage") -class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): - - @parameterized.named_parameters( - ("1", 20, 14, 7, 1), - ("2", 20, 17, 9, 1), - ("3", 20, 14, 14, 1), - ("4", 20, 10, 14, 1), - ("5", 20, 14, 19, 1), - ("6", 20, 4, 1, 2), - ("7", 20, 2, 1, 6), - ("8", 20, 4, 7, 2), - ("9", 20, 2, 7, 6), - ("10", 1, 10, 4, 1), - ("11", 0, 10, 4, 1), - ) - def testSlideDataset(self, count, window_size, window_shift, window_stride): - """Tests a dataset that slides a window its input elements.""" - components = (np.arange(7), - np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], - np.array(37.0) * np.arange(7)) - - count_t = array_ops.placeholder(dtypes.int64, shape=[]) - window_size_t = array_ops.placeholder(dtypes.int64, shape=[]) - window_shift_t = array_ops.placeholder(dtypes.int64, shape=[]) - window_stride_t = array_ops.placeholder(dtypes.int64, shape=[]) - - def _map_fn(x, y, z): - return math_ops.square(x), math_ops.square(y), math_ops.square(z) - - # The pipeline is TensorSliceDataset -> MapDataset(square_3) -> - # RepeatDataset(count) -> - # _SlideDataset(window_size, window_shift, window_stride). - iterator = dataset_ops.make_initializable_iterator( - dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn) - .repeat(count).apply( - sliding.sliding_window_batch( - window_size=window_size_t, - window_shift=window_shift_t, - window_stride=window_stride_t))) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([[None] + list(c.shape[1:]) for c in components], - [t.shape.as_list() for t in get_next]) - - with self.cached_session() as sess: - sess.run( - init_op, - feed_dict={ - count_t: count, - window_size_t: window_size, - window_shift_t: window_shift, - window_stride_t: window_stride - }) - num_batches = (count * 7 - ( - (window_size - 1) * window_stride + 1)) // window_shift + 1 - for i in range(num_batches): - result = sess.run(get_next) - for component, result_component in zip(components, result): - for j in range(window_size): - self.assertAllEqual( - component[(i * window_shift + j * window_stride) % 7]**2, - result_component[j]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - @parameterized.named_parameters( - ("1", 20, 14, 7, 1), - ("2", 20, 17, 9, 1), - ("3", 20, 14, 14, 1), - ("4", 20, 10, 14, 1), - ("5", 20, 14, 19, 1), - ("6", 20, 4, 1, 2), - ("7", 20, 2, 1, 6), - ("8", 20, 4, 7, 2), - ("9", 20, 2, 7, 6), - ("10", 1, 10, 4, 1), - ("11", 0, 10, 4, 1), - ) - def testSlideDatasetDeprecated(self, count, window_size, stride, - window_stride): - """Tests a dataset that slides a window its input elements.""" - components = (np.arange(7), - np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], - np.array(37.0) * np.arange(7)) - - count_t = array_ops.placeholder(dtypes.int64, shape=[]) - window_size_t = array_ops.placeholder(dtypes.int64, shape=[]) - stride_t = array_ops.placeholder(dtypes.int64, shape=[]) - window_stride_t = array_ops.placeholder(dtypes.int64, shape=[]) - - def _map_fn(x, y, z): - return math_ops.square(x), math_ops.square(y), math_ops.square(z) - - # The pipeline is TensorSliceDataset -> MapDataset(square_3) -> - # RepeatDataset(count) -> _SlideDataset(window_size, stride, window_stride). - iterator = dataset_ops.make_initializable_iterator( - dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn) - .repeat(count).apply( - sliding.sliding_window_batch( - window_size=window_size_t, - stride=stride_t, - window_stride=window_stride_t))) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([[None] + list(c.shape[1:]) for c in components], - [t.shape.as_list() for t in get_next]) - - with self.cached_session() as sess: - sess.run( - init_op, - feed_dict={ - count_t: count, - window_size_t: window_size, - stride_t: stride, - window_stride_t: window_stride - }) - num_batches = (count * 7 - ( - (window_size - 1) * window_stride + 1)) // stride + 1 - for i in range(num_batches): - result = sess.run(get_next) - for component, result_component in zip(components, result): - for j in range(window_size): - self.assertAllEqual( - component[(i * stride + j * window_stride) % 7]**2, - result_component[j]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - @parameterized.named_parameters( - ("1", 14, 0, 3, 1), - ("2", 14, 3, 0, 1), - ("3", 14, 3, 3, 0), - ) - def testSlideDatasetInvalid(self, count, window_size, window_shift, - window_stride): - count_t = array_ops.placeholder(dtypes.int64, shape=[]) - window_size_t = array_ops.placeholder(dtypes.int64, shape=[]) - window_shift_t = array_ops.placeholder(dtypes.int64, shape=[]) - window_stride_t = array_ops.placeholder(dtypes.int64, shape=[]) - - iterator = dataset_ops.make_initializable_iterator( - dataset_ops.Dataset.range(10).map(lambda x: x).repeat(count_t).apply( - sliding.sliding_window_batch( - window_size=window_size_t, - window_shift=window_shift_t, - window_stride=window_stride_t))) - init_op = iterator.initializer - - with self.cached_session() as sess: - with self.assertRaises(errors.InvalidArgumentError): - sess.run( - init_op, - feed_dict={ - count_t: count, - window_size_t: window_size, - window_shift_t: window_shift, - window_stride_t: window_stride - }) - - def testSlideDatasetValueError(self): - with self.assertRaises(ValueError): - dataset_ops.Dataset.range(10).map(lambda x: x).apply( - sliding.sliding_window_batch( - window_size=1, stride=1, window_shift=1, window_stride=1)) - - def testSlideSparse(self): - - def _sparse(i): - return sparse_tensor.SparseTensorValue( - indices=[[0]], values=(i * [1]), dense_shape=[1]) - - iterator = dataset_ops.make_initializable_iterator( - dataset_ops.Dataset.range(10).map(_sparse).apply( - sliding.sliding_window_batch(window_size=5, window_shift=3))) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - num_batches = (10 - 5) // 3 + 1 - for i in range(num_batches): - actual = sess.run(get_next) - expected = sparse_tensor.SparseTensorValue( - indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]], - values=[i * 3, i * 3 + 1, i * 3 + 2, i * 3 + 3, i * 3 + 4], - dense_shape=[5, 1]) - self.assertTrue(sparse_tensor.is_sparse(actual)) - self.assertValuesEqual(actual, expected) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testSlideSparseWithDifferentDenseShapes(self): - - def _sparse(i): - return sparse_tensor.SparseTensorValue( - indices=array_ops.expand_dims( - math_ops.range(i, dtype=dtypes.int64), 1), - values=array_ops.fill([math_ops.cast(i, dtypes.int32)], i), - dense_shape=[i]) - - iterator = dataset_ops.make_initializable_iterator( - dataset_ops.Dataset.range(10).map(_sparse).apply( - sliding.sliding_window_batch(window_size=5, window_shift=3))) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - num_batches = (10 - 5) // 3 + 1 - for i in range(num_batches): - actual = sess.run(get_next) - expected_indices = [] - expected_values = [] - for j in range(5): - for k in range(i * 3 + j): - expected_indices.append([j, k]) - expected_values.append(i * 3 + j) - expected = sparse_tensor.SparseTensorValue( - indices=expected_indices, - values=expected_values, - dense_shape=[5, i * 3 + 5 - 1]) - self.assertTrue(sparse_tensor.is_sparse(actual)) - self.assertValuesEqual(actual, expected) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testNestedSlideSparse(self): - - def _sparse(i): - return sparse_tensor.SparseTensorValue( - indices=[[0]], values=(i * [1]), dense_shape=[1]) - - iterator = dataset_ops.make_initializable_iterator( - dataset_ops.Dataset.range(10).map(_sparse).apply( - sliding.sliding_window_batch(window_size=4, window_shift=2)).apply( - sliding.sliding_window_batch(window_size=3, window_shift=1))) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - # Slide: 1st batch. - actual = sess.run(get_next) - expected = sparse_tensor.SparseTensorValue( - indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0], - [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0], - [2, 2, 0], [2, 3, 0]], - values=[0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7], - dense_shape=[3, 4, 1]) - self.assertTrue(sparse_tensor.is_sparse(actual)) - self.assertValuesEqual(actual, expected) - # Slide: 2nd batch. - actual = sess.run(get_next) - expected = sparse_tensor.SparseTensorValue( - indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0], - [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0], - [2, 2, 0], [2, 3, 0]], - values=[2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9], - dense_shape=[3, 4, 1]) - self.assertTrue(sparse_tensor.is_sparse(actual)) - self.assertValuesEqual(actual, expected) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testSlideShapeError(self): - - def generator(): - yield [1.0, 2.0, 3.0] - yield [4.0, 5.0, 6.0] - yield [7.0, 8.0, 9.0, 10.0] - - iterator = dataset_ops.make_initializable_iterator( - dataset_ops.Dataset.from_generator( - generator, dtypes.float32, output_shapes=[None]).apply( - sliding.sliding_window_batch(window_size=3, window_shift=1))) - next_element = iterator.get_next() - - with self.cached_session() as sess: - sess.run(iterator.initializer) - with self.assertRaisesRegexp( - errors.InvalidArgumentError, - r"Cannot batch tensors with different shapes in component 0. " - r"First element had shape \[3\] and element 2 had shape \[4\]."): - sess.run(next_element) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD deleted file mode 100644 index ff0d3e361e2..00000000000 --- a/tensorflow/contrib/data/python/ops/BUILD +++ /dev/null @@ -1,260 +0,0 @@ -load( - "//tensorflow:tensorflow.bzl", - "tf_gen_op_wrapper_py", - "tf_kernel_library", -) -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "counter", - srcs = ["counter.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:counter", - ], -) - -py_library( - name = "get_single_element", - srcs = ["get_single_element.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:get_single_element", - ], -) - -py_library( - name = "iterator_ops", - srcs = [ - "iterator_ops.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:iterator_ops", - ], -) - -py_library( - name = "random_ops", - srcs = [ - "random_ops.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:random_ops", - ], -) - -py_library( - name = "readers", - srcs = [ - "readers.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":batching", - ":interleave_ops", - ":parsing_ops", - ":shuffle_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:dataset_ops_gen", - "//tensorflow/python:dtypes", - "//tensorflow/python:experimental_dataset_ops_gen", - "//tensorflow/python:framework_ops", - "//tensorflow/python:lib", - "//tensorflow/python:platform", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:util", - "//tensorflow/python/data/experimental/ops:readers", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/ops:readers", - "//tensorflow/python/data/util:convert", - "//tensorflow/python/data/util:nest", - "//third_party/py/numpy", - ], -) - -py_library( - name = "shuffle_ops", - srcs = [ - "shuffle_ops.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:shuffle_ops", - ], -) - -py_library( - name = "batching", - srcs = ["batching.py"], - srcs_version = "PY2AND3", - deps = [ - ":get_single_element", - ":grouping", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:dataset_ops_gen", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:tensor_util", - "//tensorflow/python/data/experimental/ops:batching", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/util:convert", - "//tensorflow/python/data/util:nest", - "//tensorflow/python/data/util:sparse", - "//third_party/py/numpy", - ], -) - -py_library( - name = "enumerate_ops", - srcs = ["enumerate_ops.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:enumerate_ops", - ], -) - -py_library( - name = "error_ops", - srcs = ["error_ops.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:error_ops", - ], -) - -py_library( - name = "grouping", - srcs = ["grouping.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:grouping", - ], -) - -py_library( - name = "interleave_ops", - srcs = ["interleave_ops.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:interleave_ops", - ], -) - -py_library( - name = "parsing_ops", - srcs = ["parsing_ops.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:parsing_ops", - ], -) - -py_library( - name = "resampling", - srcs = ["resampling.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:resampling", - ], -) - -py_library( - name = "scan_ops", - srcs = ["scan_ops.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:scan_ops", - ], -) - -py_library( - name = "sliding", - srcs = ["sliding.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:dataset_ops_gen", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:function", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/util:structure", - ], -) - -py_library( - name = "threadpool", - srcs = ["threadpool.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:threadpool", - ], -) - -py_library( - name = "unique", - srcs = [ - "unique.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:unique", - ], -) - -py_library( - name = "writers", - srcs = [ - "writers.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/data/experimental/ops:writers", - ], -) - -py_library( - name = "prefetching_ops", - srcs = ["prefetching_ops.py"], - deps = [ - "//tensorflow/python/data/experimental/ops:prefetching_ops", - ], -) - -py_library( - name = "dataset_ops", - deps = [ - ":batching", - ":counter", - ":enumerate_ops", - ":error_ops", - ":get_single_element", - ":grouping", - ":interleave_ops", - ":prefetching_ops", - ":random_ops", - ":readers", - ":resampling", - ":scan_ops", - ":shuffle_ops", - ":sliding", - ":threadpool", - ":unique", - ":writers", - "//tensorflow/python:dataset_ops_gen", - "//tensorflow/python:util", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/util:nest", - ], -) diff --git a/tensorflow/contrib/data/python/ops/batching.py b/tensorflow/contrib/data/python/ops/batching.py deleted file mode 100644 index 0f3ed40c574..00000000000 --- a/tensorflow/contrib/data/python/ops/batching.py +++ /dev/null @@ -1,358 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Batching dataset transformations.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework import with_shape -from tensorflow.python.data.experimental.ops import batching -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.util import nest -from tensorflow.python.data.util import structure -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.util import deprecation - - -@deprecation.deprecated( - None, "Use `tf.data.experimental.dense_to_sparse_batch(...)`.") -def dense_to_sparse_batch(batch_size, row_shape): - """A transformation that batches ragged elements into `tf.SparseTensor`s. - - Like `Dataset.padded_batch()`, this transformation combines multiple - consecutive elements of the dataset, which might have different - shapes, into a single element. The resulting element has three - components (`indices`, `values`, and `dense_shape`), which - comprise a `tf.SparseTensor` that represents the same data. The - `row_shape` represents the dense shape of each row in the - resulting `tf.SparseTensor`, to which the effective batch size is - prepended. For example: - - ```python - # NOTE: The following examples use `{ ... }` to represent the - # contents of a dataset. - a = { ['a', 'b', 'c'], ['a', 'b'], ['a', 'b', 'c', 'd'] } - - a.apply(tf.data.experimental.dense_to_sparse_batch(batch_size=2, - row_shape=[6])) == - { - ([[0, 0], [0, 1], [0, 2], [1, 0], [1, 1]], # indices - ['a', 'b', 'c', 'a', 'b'], # values - [2, 6]), # dense_shape - ([[0, 0], [0, 1], [0, 2], [0, 3]], - ['a', 'b', 'c', 'd'], - [1, 6]) - } - ``` - - Args: - batch_size: A `tf.int64` scalar `tf.Tensor`, representing the number of - consecutive elements of this dataset to combine in a single batch. - row_shape: A `tf.TensorShape` or `tf.int64` vector tensor-like object - representing the equivalent dense shape of a row in the resulting - `tf.SparseTensor`. Each element of this dataset must have the same rank as - `row_shape`, and must have size less than or equal to `row_shape` in each - dimension. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - return batching.dense_to_sparse_batch(batch_size, row_shape) - - -@deprecation.deprecated(None, "Use `tf.data.experimental.unbatch()`.") -def unbatch(): - """Splits elements of a dataset into multiple elements on the batch dimension. - - For example, if elements of the dataset are shaped `[B, a0, a1, ...]`, - where `B` may vary for each input element, then for each element in the - dataset, the unbatched dataset will contain `B` consecutive elements - of shape `[a0, a1, ...]`. - - ```python - # NOTE: The following example uses `{ ... }` to represent the contents - # of a dataset. - a = { ['a', 'b', 'c'], ['a', 'b'], ['a', 'b', 'c', 'd'] } - - a.apply(tf.data.experimental.unbatch()) == { - 'a', 'b', 'c', 'a', 'b', 'a', 'b', 'c', 'd'} - ``` - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - return batching.unbatch() - - -@deprecation.deprecated( - None, "Use `tf.data.Dataset.batch(..., drop_remainder=True)`.") -def batch_and_drop_remainder(batch_size): - """A batching transformation that omits the final small batch (if present). - - Like `tf.data.Dataset.batch`, this transformation combines - consecutive elements of this dataset into batches. However, if the batch - size does not evenly divide the input dataset size, this transformation will - drop the final smaller element. - - The following example illustrates the difference between this - transformation and `Dataset.batch()`: - - ```python - dataset = tf.data.Dataset.range(200) - batched = - dataset.apply(tf.contrib.data.batch_and_drop_remainder(128)) - print(batched.output_shapes) # ==> "(128,)" (the batch dimension is known) - ``` - - By contrast, `dataset.batch(128)` would yield a two-element dataset with - shapes `(128,)` and `(72,)`, so the batch dimension would not be statically - known. - - Args: - batch_size: A `tf.int64` scalar `tf.Tensor`, representing the number of - consecutive elements of this dataset to combine in a single batch. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply` - """ - - def _apply_fn(dataset): - """Function from `Dataset` to `Dataset` that applies the transformation.""" - return dataset.batch(batch_size, drop_remainder=True) - - return _apply_fn - - -@deprecation.deprecated( - None, "Use `tf.data.Dataset.padded_batch(..., drop_remainder=True)`.") -def padded_batch_and_drop_remainder(batch_size, - padded_shapes, - padding_values=None): - """A batching and padding transformation that omits the final small batch. - - Like `tf.data.Dataset.padded_batch`, this transformation combines - consecutive elements of this dataset into batches. However, if the batch - size does not evenly divide the input dataset size, this transformation will - drop the final smaller element. - - See `tf.contrib.data.batch_and_drop_remainder` for more details. - - Args: - batch_size: A `tf.int64` scalar `tf.Tensor`, representing the number of - consecutive elements of this dataset to combine in a single batch. - padded_shapes: A nested structure of `tf.TensorShape` or `tf.int64` vector - tensor-like objects. See `tf.data.Dataset.padded_batch` for details. - padding_values: (Optional.) A nested structure of scalar-shaped `tf.Tensor`. - See `tf.data.Dataset.padded_batch` for details. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply` - """ - - def _apply_fn(dataset): - """Function from `Dataset` to `Dataset` that applies the transformation.""" - return dataset.padded_batch( - batch_size, padded_shapes=padded_shapes, padding_values=padding_values, - drop_remainder=True) - - return _apply_fn - - -# TODO(b/116817045): Move this to `tf.data.experimental` when the `with_shape()` -# function is available in the core. -def assert_element_shape(expected_shapes): - """Assert the shape of this `Dataset`. - - ```python - shapes = [tf.TensorShape([16, 256]), tf.TensorShape([None, 2])] - result = dataset.apply(tf.data.experimental.assert_element_shape(shapes)) - print(result.output_shapes) # ==> "((16, 256), (, 2))" - ``` - - If dataset shapes and expected_shape, are fully defined, assert they match. - Otherwise, add assert op that will validate the shapes when tensors are - evaluated, and set shapes on tensors, respectively. - - Note that unknown dimension in `expected_shapes` will be ignored. - - Args: - expected_shapes: A nested structure of `tf.TensorShape` objects. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply` - """ - - def _merge_output_shapes(original_shapes, expected_shapes): - flat_original_shapes = nest.flatten(original_shapes) - flat_new_shapes = nest.flatten_up_to(original_shapes, expected_shapes) - flat_merged_output_shapes = [ - original_shape.merge_with(new_shape) - for original_shape, new_shape in zip(flat_original_shapes, - flat_new_shapes)] - return nest.pack_sequence_as(original_shapes, flat_merged_output_shapes) - - def _check_shape(*elements): - flatten_tensors = nest.flatten(elements) - flatten_shapes = nest.flatten(expected_shapes) - checked_tensors = [ - with_shape(shape, tensor) if shape else tensor # Ignore unknown shape - for shape, tensor in zip(flatten_shapes, flatten_tensors) - ] - return nest.pack_sequence_as(elements, checked_tensors) - - def _apply_fn(dataset): - output_shapes = _merge_output_shapes( - dataset_ops.get_legacy_output_shapes(dataset), expected_shapes) - # pylint: disable=protected-access - return _RestructuredDataset( - dataset.map(_check_shape), - dataset_ops.get_legacy_output_types(dataset), - output_shapes=output_shapes, - output_classes=dataset_ops.get_legacy_output_classes(dataset)) - - return _apply_fn - - -@deprecation.deprecated(None, "Use `tf.data.experimental.map_and_batch(...)`.") -def map_and_batch(map_func, - batch_size, - num_parallel_batches=None, - drop_remainder=False, - num_parallel_calls=None): - """Fused implementation of `map` and `batch`. - - Maps `map_func` across `batch_size` consecutive elements of this dataset - and then combines them into a batch. Functionally, it is equivalent to `map` - followed by `batch`. However, by fusing the two transformations together, the - implementation can be more efficient. Surfacing this transformation in the API - is temporary. Once automatic input pipeline optimization is implemented, - the fusing of `map` and `batch` will happen automatically and this API will be - deprecated. - - Args: - map_func: A function mapping a nested structure of tensors to another nested - structure of tensors. - batch_size: A `tf.int64` scalar `tf.Tensor`, representing the number of - consecutive elements of this dataset to combine in a single batch. - num_parallel_batches: (Optional.) A `tf.int64` scalar `tf.Tensor`, - representing the number of batches to create in parallel. On one hand, - higher values can help mitigate the effect of stragglers. On the other - hand, higher values can increase contention if CPU is scarce. - drop_remainder: (Optional.) A `tf.bool` scalar `tf.Tensor`, representing - whether the last batch should be dropped in case its size is smaller than - desired; the default behavior is not to drop the smaller batch. - num_parallel_calls: (Optional.) A `tf.int32` scalar `tf.Tensor`, - representing the number of elements to process in parallel. If not - specified, `batch_size * num_parallel_batches` elements will be processed - in parallel. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - - Raises: - ValueError: If both `num_parallel_batches` and `num_parallel_calls` are - specified. - """ - return batching.map_and_batch(map_func, batch_size, num_parallel_batches, - drop_remainder, num_parallel_calls) - - -class _RestructuredDataset(dataset_ops.UnaryDataset): - """An internal helper for changing the structure and shape of a dataset.""" - - def __init__(self, - dataset, - output_types, - output_shapes=None, - output_classes=None): - """Creates a new dataset with the given output types and shapes. - - The given `dataset` must have a structure that is convertible: - * `dataset.output_types` must be the same as `output_types` module nesting. - * Each shape in `dataset.output_shapes` must be compatible with each shape - in `output_shapes` (if given). - - Note: This helper permits "unsafe casts" for shapes, equivalent to using - `tf.Tensor.set_shape()` where domain-specific knowledge is available. - - Args: - dataset: A `Dataset` object. - output_types: A nested structure of `tf.DType` objects. - output_shapes: (Optional.) A nested structure of `tf.TensorShape` objects. - If omitted, the shapes will be inherited from `dataset`. - output_classes: (Optional.) A nested structure of class types. If omitted, - the class types will be inherited from `dataset`. - - Raises: - ValueError: If either `output_types` or `output_shapes` is not compatible - with the structure of `dataset`. - """ - self._input_dataset = dataset - - input_types = dataset_ops.get_legacy_output_types(dataset) - # Validate that the types are compatible. - output_types = nest.map_structure(dtypes.as_dtype, output_types) - flat_original_types = nest.flatten(input_types) - flat_new_types = nest.flatten(output_types) - if flat_original_types != flat_new_types: - raise ValueError( - "Dataset with output types %r cannot be restructured to have " - "output types %r" % - (dataset_ops.get_legacy_output_types(dataset), output_types)) - - input_shapes = dataset_ops.get_legacy_output_shapes(dataset) - if output_shapes is None: - # Inherit shapes from the original `dataset`. - output_shapes = nest.pack_sequence_as( - output_types, nest.flatten(input_shapes)) - else: - # Validate that the shapes are compatible. - nest.assert_same_structure(output_types, output_shapes) - flat_original_shapes = nest.flatten(input_shapes) - flat_new_shapes = nest.flatten_up_to(output_types, output_shapes) - - for original_shape, new_shape in zip(flat_original_shapes, - flat_new_shapes): - if not original_shape.is_compatible_with(new_shape): - raise ValueError( - "Dataset with output shapes %r cannot be restructured to have " - "incompatible output shapes %r" % (input_shapes, - output_shapes)) - output_shapes = nest.map_structure_up_to( - output_types, tensor_shape.as_shape, output_shapes) - - input_classes = dataset_ops.get_legacy_output_classes(dataset) - if output_classes is None: - # Inherit class types from the original `dataset`. - output_classes = nest.pack_sequence_as( - output_types, nest.flatten(input_classes)) - - self._element_spec = structure.convert_legacy_structure( - output_types, output_shapes, output_classes) - variant_tensor = self._input_dataset._variant_tensor # pylint: disable=protected-access - super(_RestructuredDataset, self).__init__(dataset, variant_tensor) - - @property - def element_spec(self): - return self._element_spec - - diff --git a/tensorflow/contrib/data/python/ops/counter.py b/tensorflow/contrib/data/python/ops/counter.py deleted file mode 100644 index 4ff5bf3e39d..00000000000 --- a/tensorflow/contrib/data/python/ops/counter.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Counter Dataset.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.experimental.ops import counter -from tensorflow.python.framework import dtypes -from tensorflow.python.util import deprecation - - -@deprecation.deprecated(None, "Use `tf.data.experimental.Counter(...)`.") -def Counter(start=0, step=1, dtype=dtypes.int64): - """Creates a `Dataset` that counts from `start` in steps of size `step`. - - For example: - - ```python - Dataset.count() == [0, 1, 2, ...) - Dataset.count(2) == [2, 3, ...) - Dataset.count(2, 5) == [2, 7, 12, ...) - Dataset.count(0, -1) == [0, -1, -2, ...) - Dataset.count(10, -1) == [10, 9, ...) - ``` - - Args: - start: (Optional.) The starting value for the counter. Defaults to 0. - step: (Optional.) The step size for the counter. Defaults to 1. - dtype: (Optional.) The data type for counter elements. Defaults to - `tf.int64`. - - Returns: - A `Dataset` of scalar `dtype` elements. - """ - return counter.Counter(start, step, dtype) diff --git a/tensorflow/contrib/data/python/ops/enumerate_ops.py b/tensorflow/contrib/data/python/ops/enumerate_ops.py deleted file mode 100644 index a21da4d3eca..00000000000 --- a/tensorflow/contrib/data/python/ops/enumerate_ops.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Enumerate dataset transformations.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -from tensorflow.python.data.experimental.ops import enumerate_ops -from tensorflow.python.util import deprecation - - -@deprecation.deprecated(None, - "Use `tf.data.experimental.enumerate_dataset(...)`.") -def enumerate_dataset(start=0): - """A transformation that enumerate the elements of a dataset. - - It is Similar to python's `enumerate`. - For example: - - ```python - # NOTE: The following examples use `{ ... }` to represent the - # contents of a dataset. - a = { 1, 2, 3 } - b = { (7, 8), (9, 10) } - - # The nested structure of the `datasets` argument determines the - # structure of elements in the resulting dataset. - a.apply(tf.contrib.data.enumerate(start=5)) == { (5, 1), (6, 2), (7, 3) } - b.apply(tf.contrib.data.enumerate()) == { (0, (7, 8)), (1, (9, 10)) } - ``` - - Args: - start: A `tf.int64` scalar `tf.Tensor`, representing the start - value for enumeration. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - return enumerate_ops.enumerate_dataset(start) diff --git a/tensorflow/contrib/data/python/ops/error_ops.py b/tensorflow/contrib/data/python/ops/error_ops.py deleted file mode 100644 index b22e11a7044..00000000000 --- a/tensorflow/contrib/data/python/ops/error_ops.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ignore_errors dataset transformations.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.experimental.ops import error_ops -from tensorflow.python.util import deprecation - - -@deprecation.deprecated(None, "Use `tf.data.experimental.ignore_errors()`.") -def ignore_errors(): - """Creates a `Dataset` from another `Dataset` and silently ignores any errors. - - Use this transformation to produce a dataset that contains the same elements - as the input, but silently drops any elements that caused an error. For - example: - - ```python - dataset = tf.data.Dataset.from_tensor_slices([1., 2., 0., 4.]) - - # Computing `tf.debugging.check_numerics(1. / 0.)` will raise an - InvalidArgumentError. - dataset = dataset.map(lambda x: tf.debugging.check_numerics(1. / x, "error")) - - # Using `ignore_errors()` will drop the element that causes an error. - dataset = - dataset.apply(tf.data.experimental.ignore_errors()) # ==> { 1., 0.5, 0.2 - } - ``` - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - return error_ops.ignore_errors() diff --git a/tensorflow/contrib/data/python/ops/get_single_element.py b/tensorflow/contrib/data/python/ops/get_single_element.py deleted file mode 100644 index 9df55faf291..00000000000 --- a/tensorflow/contrib/data/python/ops/get_single_element.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python wrappers for Datasets and Iterators.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.data.experimental.ops import get_single_element as experimental_get_single_element -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.util import deprecation - - -@deprecation.deprecated(None, - "Use `tf.data.experimental.get_single_element(...)`.") -def get_single_element(dataset): - """Returns the single element in `dataset` as a nested structure of tensors. - - This function enables you to use a `tf.data.Dataset` in a stateless - "tensor-in tensor-out" expression, without creating a - `tf.compat.v1.data.Iterator`. - This can be useful when your preprocessing transformations are expressed - as a `Dataset`, and you want to use the transformation at serving time. - For example: - - ```python - input_batch = tf.compat.v1.placeholder(tf.string, shape=[BATCH_SIZE]) - - def preprocessing_fn(input_str): - # ... - return image, label - - dataset = (tf.data.Dataset.from_tensor_slices(input_batch) - .map(preprocessing_fn, num_parallel_calls=BATCH_SIZE) - .batch(BATCH_SIZE)) - - image_batch, label_batch = tf.data.experimental.get_single_element(dataset) - ``` - - Args: - dataset: A `tf.data.Dataset` object containing a single element. - - Returns: - A nested structure of `tf.Tensor` objects, corresponding to the single - element of `dataset`. - - Raises: - TypeError: if `dataset` is not a `tf.data.Dataset` object. - InvalidArgumentError (at runtime): if `dataset` does not contain exactly - one element. - """ - return experimental_get_single_element.get_single_element(dataset) - - -@deprecation.deprecated(None, "Use `tf.data.Dataset.reduce(...)`.") -def reduce_dataset(dataset, reducer): - """Returns the result of reducing the `dataset` using `reducer`. - - Args: - dataset: A `tf.data.Dataset` object. - reducer: A `tf.data.experimental.Reducer` object representing the reduce - logic. - - Returns: - A nested structure of `tf.Tensor` objects, corresponding to the result - of reducing `dataset` using `reducer`. - - Raises: - TypeError: if `dataset` is not a `tf.data.Dataset` object. - """ - if not isinstance(dataset, dataset_ops.Dataset): - raise TypeError("`dataset` must be a `tf.data.Dataset` object.") - - return dataset.reduce(reducer.init_func(np.int64(0)), reducer.reduce_func) diff --git a/tensorflow/contrib/data/python/ops/grouping.py b/tensorflow/contrib/data/python/ops/grouping.py deleted file mode 100644 index a99dc2f29ae..00000000000 --- a/tensorflow/contrib/data/python/ops/grouping.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Grouping dataset transformations.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -from tensorflow.python.data.experimental.ops import grouping -from tensorflow.python.util import deprecation - - -@deprecation.deprecated(None, - "Use `tf.data.experimental.group_by_reducer(...)`.") -def group_by_reducer(key_func, reducer): - """A transformation that groups elements and performs a reduction. - - This transformation maps element of a dataset to a key using `key_func` and - groups the elements by key. The `reducer` is used to process each group; its - `init_func` is used to initialize state for each group when it is created, the - `reduce_func` is used to update the state every time an element is mapped to - the matching group, and the `finalize_func` is used to map the final state to - an output value. - - Args: - key_func: A function mapping a nested structure of tensors - (having shapes and types defined by `self.output_shapes` and - `self.output_types`) to a scalar `tf.int64` tensor. - reducer: An instance of `Reducer`, which captures the reduction logic using - the `init_func`, `reduce_func`, and `finalize_func` functions. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - return grouping.group_by_reducer(key_func, reducer) - - -@deprecation.deprecated(None, - "Use `tf.data.experimental.group_by_window(...)`.") -def group_by_window(key_func, - reduce_func, - window_size=None, - window_size_func=None): - """A transformation that groups windows of elements by key and reduces them. - - This transformation maps each consecutive element in a dataset to a key - using `key_func` and groups the elements by key. It then applies - `reduce_func` to at most `window_size_func(key)` elements matching the same - key. All except the final window for each key will contain - `window_size_func(key)` elements; the final window may be smaller. - - You may provide either a constant `window_size` or a window size determined by - the key through `window_size_func`. - - Args: - key_func: A function mapping a nested structure of tensors - (having shapes and types defined by `self.output_shapes` and - `self.output_types`) to a scalar `tf.int64` tensor. - reduce_func: A function mapping a key and a dataset of up to `window_size` - consecutive elements matching that key to another dataset. - window_size: A `tf.int64` scalar `tf.Tensor`, representing the number of - consecutive elements matching the same key to combine in a single - batch, which will be passed to `reduce_func`. Mutually exclusive with - `window_size_func`. - window_size_func: A function mapping a key to a `tf.int64` scalar - `tf.Tensor`, representing the number of consecutive elements matching - the same key to combine in a single batch, which will be passed to - `reduce_func`. Mutually exclusive with `window_size`. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - - Raises: - ValueError: if neither or both of {`window_size`, `window_size_func`} are - passed. - """ - return grouping.group_by_window(key_func, reduce_func, window_size, - window_size_func) - - -@deprecation.deprecated( - None, "Use `tf.data.experimental.bucket_by_sequence_length(...)`.") -def bucket_by_sequence_length(element_length_func, - bucket_boundaries, - bucket_batch_sizes, - padded_shapes=None, - padding_values=None, - pad_to_bucket_boundary=False, - no_padding=False): - """A transformation that buckets elements in a `Dataset` by length. - - Elements of the `Dataset` are grouped together by length and then are padded - and batched. - - This is useful for sequence tasks in which the elements have variable length. - Grouping together elements that have similar lengths reduces the total - fraction of padding in a batch which increases training step efficiency. - - Args: - element_length_func: function from element in `Dataset` to `tf.int32`, - determines the length of the element, which will determine the bucket it - goes into. - bucket_boundaries: `list`, upper length boundaries of the buckets. - bucket_batch_sizes: `list`, batch size per bucket. Length should be - `len(bucket_boundaries) + 1`. - padded_shapes: Nested structure of `tf.TensorShape` to pass to - `tf.data.Dataset.padded_batch`. If not provided, will use - `dataset.output_shapes`, which will result in variable length dimensions - being padded out to the maximum length in each batch. - padding_values: Values to pad with, passed to - `tf.data.Dataset.padded_batch`. Defaults to padding with 0. - pad_to_bucket_boundary: bool, if `False`, will pad dimensions with unknown - size to maximum length in batch. If `True`, will pad dimensions with - unknown size to bucket boundary minus 1 (i.e., the maximum length in each - bucket), and caller must ensure that the source `Dataset` does not contain - any elements with length longer than `max(bucket_boundaries)`. - no_padding: `bool`, indicates whether to pad the batch features (features - need to be either of type `tf.SparseTensor` or of same shape). - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - - Raises: - ValueError: if `len(bucket_batch_sizes) != len(bucket_boundaries) + 1`. - """ - return grouping.bucket_by_sequence_length( - element_length_func, bucket_boundaries, bucket_batch_sizes, padded_shapes, - padding_values, pad_to_bucket_boundary, no_padding) - - -class Reducer(grouping.Reducer): - """A reducer is used for reducing a set of elements. - - A reducer is represented as a tuple of the three functions: - 1) initialization function: key => initial state - 2) reduce function: (old state, input) => new state - 3) finalization function: state => result - """ - - @deprecation.deprecated(None, "Use `tf.data.experimental.Reducer(...)`.") - def __init__(self, init_func, reduce_func, finalize_func): - super(Reducer, self).__init__(init_func, reduce_func, finalize_func) diff --git a/tensorflow/contrib/data/python/ops/interleave_ops.py b/tensorflow/contrib/data/python/ops/interleave_ops.py deleted file mode 100644 index 4543bd2ecc3..00000000000 --- a/tensorflow/contrib/data/python/ops/interleave_ops.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Non-deterministic dataset transformations.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.experimental.ops import interleave_ops -from tensorflow.python.util import deprecation - - -@deprecation.deprecated(None, - "Use `tf.data.experimental.parallel_interleave(...)`.") -def parallel_interleave(map_func, - cycle_length, - block_length=1, - sloppy=False, - buffer_output_elements=None, - prefetch_input_elements=None): - """A parallel version of the `Dataset.interleave()` transformation. - - `parallel_interleave()` maps `map_func` across its input to produce nested - datasets, and outputs their elements interleaved. Unlike - `tf.data.Dataset.interleave`, it gets elements from `cycle_length` nested - datasets in parallel, which increases the throughput, especially in the - presence of stragglers. Furthermore, the `sloppy` argument can be used to - improve performance, by relaxing the requirement that the outputs are produced - in a deterministic order, and allowing the implementation to skip over nested - datasets whose elements are not readily available when requested. - - Example usage: - - ```python - # Preprocess 4 files concurrently. - filenames = tf.data.Dataset.list_files("/path/to/data/train*.tfrecords") - dataset = filenames.apply( - tf.data.experimental.parallel_interleave( - lambda filename: tf.data.TFRecordDataset(filename), - cycle_length=4)) - ``` - - WARNING: If `sloppy` is `True`, the order of produced elements is not - deterministic. - - Args: - map_func: A function mapping a nested structure of tensors to a `Dataset`. - cycle_length: The number of input `Dataset`s to interleave from in parallel. - block_length: The number of consecutive elements to pull from an input - `Dataset` before advancing to the next input `Dataset`. - sloppy: If false, elements are produced in deterministic order. Otherwise, - the implementation is allowed, for the sake of expediency, to produce - elements in a non-deterministic order. - buffer_output_elements: The number of elements each iterator being - interleaved should buffer (similar to the `.prefetch()` transformation for - each interleaved iterator). - prefetch_input_elements: The number of input elements to transform to - iterators before they are needed for interleaving. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - return interleave_ops.parallel_interleave( - map_func, cycle_length, block_length, sloppy, buffer_output_elements, - prefetch_input_elements) - - -@deprecation.deprecated( - None, "Use `tf.contrib.data.parallel_interleave(..., sloppy=True)`.") -def sloppy_interleave(map_func, cycle_length, block_length=1): - """A non-deterministic version of the `Dataset.interleave()` transformation. - - `sloppy_interleave()` maps `map_func` across `dataset`, and - non-deterministically interleaves the results. - - The resulting dataset is almost identical to `interleave`. The key - difference is that if retrieving a value from a given output iterator would - cause `get_next` to block, that iterator will be skipped, and consumed - when next available. If consuming from all iterators would cause the - `get_next` call to block, the `get_next` call blocks until the first value is - available. - - If the underlying datasets produce elements as fast as they are consumed, the - `sloppy_interleave` transformation behaves identically to `interleave`. - However, if an underlying dataset would block the consumer, - `sloppy_interleave` can violate the round-robin order (that `interleave` - strictly obeys), producing an element from a different underlying - dataset instead. - - Example usage: - - ```python - # Preprocess 4 files concurrently. - filenames = tf.data.Dataset.list_files("/path/to/data/train*.tfrecords") - dataset = filenames.apply( - tf.contrib.data.sloppy_interleave( - lambda filename: tf.data.TFRecordDataset(filename), - cycle_length=4)) - ``` - - WARNING: The order of elements in the resulting dataset is not - deterministic. Use `Dataset.interleave()` if you want the elements to have a - deterministic order. - - Args: - map_func: A function mapping a nested structure of tensors (having shapes - and types defined by `self.output_shapes` and `self.output_types`) to a - `Dataset`. - cycle_length: The number of input `Dataset`s to interleave from in parallel. - block_length: The number of consecutive elements to pull from an input - `Dataset` before advancing to the next input `Dataset`. Note: - `sloppy_interleave` will skip the remainder of elements in the - `block_length` in order to avoid blocking. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - return interleave_ops.parallel_interleave( - map_func, cycle_length, block_length, sloppy=True) - - -@deprecation.deprecated(None, - "Use `tf.data.experimental.sample_from_datasets(...)`.") -def sample_from_datasets(datasets, weights=None, seed=None): - """Samples elements at random from the datasets in `datasets`. - - Args: - datasets: A list of `tf.data.Dataset` objects with compatible structure. - weights: (Optional.) A list of `len(datasets)` floating-point values where - `weights[i]` represents the probability with which an element should be - sampled from `datasets[i]`, or a `tf.data.Dataset` object where each - element is such a list. Defaults to a uniform distribution across - `datasets`. - seed: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the - random seed that will be used to create the distribution. See - `tf.compat.v1.set_random_seed` for behavior. - - Returns: - A dataset that interleaves elements from `datasets` at random, according to - `weights` if provided, otherwise with uniform probability. - - Raises: - TypeError: If the `datasets` or `weights` arguments have the wrong type. - ValueError: If the `weights` argument is specified and does not match the - length of the `datasets` element. - """ - return interleave_ops.sample_from_datasets(datasets, weights, seed) - - -@deprecation.deprecated(None, - "Use `tf.data.experimental.choose_from_datasets(...)`.") -def choose_from_datasets(datasets, choice_dataset): - """Creates a dataset that deterministically chooses elements from `datasets`. - - For example, given the following datasets: - - ```python - datasets = [tf.data.Dataset.from_tensors("foo").repeat(), - tf.data.Dataset.from_tensors("bar").repeat(), - tf.data.Dataset.from_tensors("baz").repeat()] - - # Define a dataset containing `[0, 1, 2, 0, 1, 2, 0, 1, 2]`. - choice_dataset = tf.data.Dataset.range(3).repeat(3) - - result = tf.data.experimental.choose_from_datasets(datasets, choice_dataset) - ``` - - The elements of `result` will be: - - ``` - "foo", "bar", "baz", "foo", "bar", "baz", "foo", "bar", "baz" - ``` - - Args: - datasets: A list of `tf.data.Dataset` objects with compatible structure. - choice_dataset: A `tf.data.Dataset` of scalar `tf.int64` tensors between - `0` and `len(datasets) - 1`. - - Returns: - A dataset that interleaves elements from `datasets` according to the values - of `choice_dataset`. - - Raises: - TypeError: If the `datasets` or `choice_dataset` arguments have the wrong - type. - """ - return interleave_ops.choose_from_datasets(datasets, choice_dataset) diff --git a/tensorflow/contrib/data/python/ops/iterator_ops.py b/tensorflow/contrib/data/python/ops/iterator_ops.py deleted file mode 100644 index 013fedb6186..00000000000 --- a/tensorflow/contrib/data/python/ops/iterator_ops.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Iterator ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.experimental.ops import iterator_ops -from tensorflow.python.util import deprecation - - -@deprecation.deprecated( - None, "Use `tf.data.experimental.make_saveable_from_iterator(...)`.") -def make_saveable_from_iterator(iterator): - """Returns a SaveableObject for saving/restore iterator state using Saver. - - Args: - iterator: Iterator. - - For example: - - ```python - with tf.Graph().as_default(): - ds = tf.data.Dataset.range(10) - iterator = ds.make_initializable_iterator() - # Build the iterator SaveableObject. - saveable_obj = tf.data.experimental.make_saveable_from_iterator(iterator) - # Add the SaveableObject to the SAVEABLE_OBJECTS collection so - # it can be automatically saved using Saver. - tf.compat.v1.add_to_collection(tf.GraphKeys.SAVEABLE_OBJECTS, saveable_obj) - saver = tf.compat.v1.train.Saver() - - while continue_training: - ... Perform training ... - if should_save_checkpoint: - saver.save() - ``` - - Note: When restoring the iterator, the existing iterator state is completely - discarded. This means that any changes you may have made to the Dataset - graph will be discarded as well! This includes the new Dataset graph - that you may have built during validation. So, while running validation, - make sure to run the initializer for the validation input pipeline after - restoring the checkpoint. - - Note: Not all iterators support checkpointing yet. Attempting to save the - state of an unsupported iterator will throw an error. - """ - return iterator_ops.make_saveable_from_iterator(iterator) - - -class CheckpointInputPipelineHook(iterator_ops.CheckpointInputPipelineHook): - """Checkpoints input pipeline state every N steps or seconds. - - This hook saves the state of the iterators in the `Graph` so that when - training is resumed the input pipeline continues from where it left off. - This could potentially avoid overfitting in certain pipelines where the - number of training steps per eval are small compared to the dataset - size or if the training pipeline is pre-empted. - - Differences from `CheckpointSaverHook`: - 1. Saves only the input pipelines in the "iterators" collection and not the - global variables or other saveable objects. - 2. Does not write the `GraphDef` and `MetaGraphDef` to the summary. - - Example of checkpointing the training pipeline: - - ```python - est = tf.estimator.Estimator(model_fn) - while True: - est.train( - train_input_fn, - hooks=[tf.data.experimental.CheckpointInputPipelineHook(est)], - steps=train_steps_per_eval) - # Note: We do not pass the hook here. - metrics = est.evaluate(eval_input_fn) - if should_stop_the_training(metrics): - break - ``` - - This hook should be used if the input pipeline state needs to be saved - separate from the model checkpoint. Doing so may be useful for a few reasons: - 1. The input pipeline checkpoint may be large, if there are large shuffle - or prefetch buffers for instance, and may bloat the checkpoint size. - 2. If the input pipeline is shared between training and validation, restoring - the checkpoint during validation may override the validation input - pipeline. - - For saving the input pipeline checkpoint alongside the model weights use - `tf.data.experimental.make_saveable_from_iterator` directly to create a - `SaveableObject` and add to the `SAVEABLE_OBJECTS` collection. Note, however, - that you will need to be careful not to restore the training iterator during - eval. You can do that by not adding the iterator to the SAVEABLE_OBJECTS - collector when building the eval graph. - """ - - @deprecation.deprecated( - None, "Use `tf.data.experimental.CheckpointInputPipelineHook(...)`.") - def __init__(self, estimator): - super(CheckpointInputPipelineHook, self).__init__(estimator) diff --git a/tensorflow/contrib/data/python/ops/parsing_ops.py b/tensorflow/contrib/data/python/ops/parsing_ops.py deleted file mode 100644 index 7bc4f0a0193..00000000000 --- a/tensorflow/contrib/data/python/ops/parsing_ops.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Experimental `dataset` API for parsing example.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.experimental.ops import parsing_ops -from tensorflow.python.util import deprecation - - -@deprecation.deprecated( - None, "Use `tf.data.experimental.parse_example_dataset(...)`.") -def parse_example_dataset(features, num_parallel_calls=1): - """A transformation that parses `Example` protos into a `dict` of tensors. - - Parses a number of serialized `Example` protos given in `serialized`. We refer - to `serialized` as a batch with `batch_size` many entries of individual - `Example` protos. - - This op parses serialized examples into a dictionary mapping keys to `Tensor` - and `SparseTensor` objects. `features` is a dict from keys to `VarLenFeature`, - `SparseFeature`, and `FixedLenFeature` objects. Each `VarLenFeature` - and `SparseFeature` is mapped to a `SparseTensor`, and each - `FixedLenFeature` is mapped to a `Tensor`. See `tf.io.parse_example` for more - details about feature dictionaries. - - Args: - features: A `dict` mapping feature keys to `FixedLenFeature`, - `VarLenFeature`, and `SparseFeature` values. - num_parallel_calls: (Optional.) A `tf.int32` scalar `tf.Tensor`, - representing the number of parsing processes to call in parallel. - - Returns: - A dataset transformation function, which can be passed to - `tf.data.Dataset.apply`. - - Raises: - ValueError: if features argument is None. - """ - return parsing_ops.parse_example_dataset(features, num_parallel_calls) diff --git a/tensorflow/contrib/data/python/ops/prefetching_ops.py b/tensorflow/contrib/data/python/ops/prefetching_ops.py deleted file mode 100644 index adfb390cd9a..00000000000 --- a/tensorflow/contrib/data/python/ops/prefetching_ops.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python wrapper for prefetching_ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.experimental.ops import prefetching_ops -from tensorflow.python.util import deprecation - - -@deprecation.deprecated(None, - "Use `tf.data.experimental.prefetch_to_device(...)`.") -def prefetch_to_device(device, buffer_size=None): - """A transformation that prefetches dataset values to the given `device`. - - NOTE: Although the transformation creates a `tf.data.Dataset`, the - transformation must be the final `Dataset` in the input pipeline. - - Args: - device: A string. The name of a device to which elements will be prefetched. - buffer_size: (Optional.) The number of elements to buffer on `device`. - Defaults to an automatically chosen value. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - return prefetching_ops.prefetch_to_device(device, buffer_size) - - -@deprecation.deprecated(None, "Use `tf.data.experimental.copy_to_device(...)`.") -def copy_to_device(target_device, source_device="/cpu:0"): - """A transformation that copies dataset elements to the given `target_device`. - - Args: - target_device: The name of a device to which elements will be copied. - source_device: The original device on which `input_dataset` will be placed. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - return prefetching_ops.copy_to_device(target_device, source_device) diff --git a/tensorflow/contrib/data/python/ops/random_ops.py b/tensorflow/contrib/data/python/ops/random_ops.py deleted file mode 100644 index 2c951256368..00000000000 --- a/tensorflow/contrib/data/python/ops/random_ops.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Datasets for random number generators.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.experimental.ops import random_ops -from tensorflow.python.util import deprecation - - -class RandomDataset(random_ops.RandomDataset): - """A `Dataset` of pseudorandom values.""" - - @deprecation.deprecated( - None, "Use `tf.data.experimental.RandomDataset(...)`.") - def __init__(self, seed=None): - super(RandomDataset, self).__init__(seed) diff --git a/tensorflow/contrib/data/python/ops/readers.py b/tensorflow/contrib/data/python/ops/readers.py deleted file mode 100644 index 92d4820d60a..00000000000 --- a/tensorflow/contrib/data/python/ops/readers.py +++ /dev/null @@ -1,405 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python wrappers for reader Datasets.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.compat import compat -from tensorflow.python.data.experimental.ops import readers -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.ops import readers as core_readers -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_spec -from tensorflow.python.ops import gen_experimental_dataset_ops -from tensorflow.python.util import deprecation - - -@deprecation.deprecated(None, - "Use `tf.data.experimental.make_csv_dataset(...)`.") -def make_csv_dataset( - file_pattern, - batch_size, - column_names=None, - column_defaults=None, - label_name=None, - select_columns=None, - field_delim=",", - use_quote_delim=True, - na_value="", - header=True, - num_epochs=None, - shuffle=True, - shuffle_buffer_size=10000, - shuffle_seed=None, - prefetch_buffer_size=None, - num_parallel_reads=None, - sloppy=False, - num_rows_for_inference=100, - compression_type=None, -): - """Reads CSV files into a dataset. - - Reads CSV files into a dataset, where each element is a (features, labels) - tuple that corresponds to a batch of CSV rows. The features dictionary - maps feature column names to `Tensor`s containing the corresponding - feature data, and labels is a `Tensor` containing the batch's label data. - - Args: - file_pattern: List of files or patterns of file paths containing CSV - records. See `tf.io.gfile.glob` for pattern rules. - batch_size: An int representing the number of records to combine - in a single batch. - column_names: An optional list of strings that corresponds to the CSV - columns, in order. One per column of the input record. If this is not - provided, infers the column names from the first row of the records. - These names will be the keys of the features dict of each dataset element. - column_defaults: A optional list of default values for the CSV fields. One - item per selected column of the input record. Each item in the list is - either a valid CSV dtype (float32, float64, int32, int64, or string), or a - `Tensor` with one of the aforementioned types. The tensor can either be - a scalar default value (if the column is optional), or an empty tensor (if - the column is required). If a dtype is provided instead of a tensor, the - column is also treated as required. If this list is not provided, tries - to infer types based on reading the first num_rows_for_inference rows of - files specified, and assumes all columns are optional, defaulting to `0` - for numeric values and `""` for string values. If both this and - `select_columns` are specified, these must have the same lengths, and - `column_defaults` is assumed to be sorted in order of increasing column - index. - label_name: A optional string corresponding to the label column. If - provided, the data for this column is returned as a separate `Tensor` from - the features dictionary, so that the dataset complies with the format - expected by a `tf.Estimator.train` or `tf.Estimator.evaluate` input - function. - select_columns: An optional list of integer indices or string column - names, that specifies a subset of columns of CSV data to select. If - column names are provided, these must correspond to names provided in - `column_names` or inferred from the file header lines. When this argument - is specified, only a subset of CSV columns will be parsed and returned, - corresponding to the columns specified. Using this results in faster - parsing and lower memory usage. If both this and `column_defaults` are - specified, these must have the same lengths, and `column_defaults` is - assumed to be sorted in order of increasing column index. - field_delim: An optional `string`. Defaults to `","`. Char delimiter to - separate fields in a record. - use_quote_delim: An optional bool. Defaults to `True`. If false, treats - double quotation marks as regular characters inside of the string fields. - na_value: Additional string to recognize as NA/NaN. - header: A bool that indicates whether the first rows of provided CSV files - correspond to header lines with column names, and should not be included - in the data. - num_epochs: An int specifying the number of times this dataset is repeated. - If None, cycles through the dataset forever. - shuffle: A bool that indicates whether the input should be shuffled. - shuffle_buffer_size: Buffer size to use for shuffling. A large buffer size - ensures better shuffling, but increases memory usage and startup time. - shuffle_seed: Randomization seed to use for shuffling. - prefetch_buffer_size: An int specifying the number of feature - batches to prefetch for performance improvement. Recommended value is the - number of batches consumed per training step. Defaults to auto-tune. - num_parallel_reads: Number of threads used to read CSV records from files. - If >1, the results will be interleaved. Defaults to `1`. - sloppy: If `True`, reading performance will be improved at - the cost of non-deterministic ordering. If `False`, the order of elements - produced is deterministic prior to shuffling (elements are still - randomized if `shuffle=True`. Note that if the seed is set, then order - of elements after shuffling is deterministic). Defaults to `False`. - num_rows_for_inference: Number of rows of a file to use for type inference - if record_defaults is not provided. If None, reads all the rows of all - the files. Defaults to 100. - compression_type: (Optional.) A `tf.string` scalar evaluating to one of - `""` (no compression), `"ZLIB"`, or `"GZIP"`. Defaults to no compression. - - Returns: - A dataset, where each element is a (features, labels) tuple that corresponds - to a batch of `batch_size` CSV rows. The features dictionary maps feature - column names to `Tensor`s containing the corresponding column data, and - labels is a `Tensor` containing the column data for the label column - specified by `label_name`. - - Raises: - ValueError: If any of the arguments is malformed. - """ - return readers.make_csv_dataset( - file_pattern, batch_size, column_names, column_defaults, label_name, - select_columns, field_delim, use_quote_delim, na_value, header, - num_epochs, shuffle, shuffle_buffer_size, shuffle_seed, - prefetch_buffer_size, num_parallel_reads, sloppy, num_rows_for_inference, - compression_type) - - -class CsvDataset(readers.CsvDataset): - """A Dataset comprising lines from one or more CSV files.""" - - @deprecation.deprecated(None, "Use `tf.data.experimental.CsvDataset(...)`.") - def __init__(self, - filenames, - record_defaults, - compression_type=None, - buffer_size=None, - header=False, - field_delim=",", - use_quote_delim=True, - na_value="", - select_cols=None): - super(CsvDataset, self).__init__( - filenames, record_defaults, compression_type, buffer_size, header, - field_delim, use_quote_delim, na_value, select_cols) - - -@deprecation.deprecated( - None, "Use `tf.data.experimental.make_batched_features_dataset(...)`.") -def make_batched_features_dataset(file_pattern, - batch_size, - features, - reader=core_readers.TFRecordDataset, - label_key=None, - reader_args=None, - num_epochs=None, - shuffle=True, - shuffle_buffer_size=10000, - shuffle_seed=None, - prefetch_buffer_size=None, - reader_num_threads=None, - parser_num_threads=None, - sloppy_ordering=False, - drop_final_batch=False): - """Returns a `Dataset` of feature dictionaries from `Example` protos. - - If label_key argument is provided, returns a `Dataset` of tuple - comprising of feature dictionaries and label. - - Example: - - ``` - serialized_examples = [ - features { - feature { key: "age" value { int64_list { value: [ 0 ] } } } - feature { key: "gender" value { bytes_list { value: [ "f" ] } } } - feature { key: "kws" value { bytes_list { value: [ "code", "art" ] } } } - }, - features { - feature { key: "age" value { int64_list { value: [] } } } - feature { key: "gender" value { bytes_list { value: [ "f" ] } } } - feature { key: "kws" value { bytes_list { value: [ "sports" ] } } } - } - ] - ``` - - We can use arguments: - - ``` - features: { - "age": FixedLenFeature([], dtype=tf.int64, default_value=-1), - "gender": FixedLenFeature([], dtype=tf.string), - "kws": VarLenFeature(dtype=tf.string), - } - ``` - - And the expected output is: - - ```python - { - "age": [[0], [-1]], - "gender": [["f"], ["f"]], - "kws": SparseTensor( - indices=[[0, 0], [0, 1], [1, 0]], - values=["code", "art", "sports"] - dense_shape=[2, 2]), - } - ``` - - Args: - file_pattern: List of files or patterns of file paths containing - `Example` records. See `tf.io.gfile.glob` for pattern rules. - batch_size: An int representing the number of records to combine - in a single batch. - features: A `dict` mapping feature keys to `FixedLenFeature` or - `VarLenFeature` values. See `tf.io.parse_example`. - reader: A function or class that can be - called with a `filenames` tensor and (optional) `reader_args` and returns - a `Dataset` of `Example` tensors. Defaults to `tf.data.TFRecordDataset`. - label_key: (Optional) A string corresponding to the key labels are stored in - `tf.Examples`. If provided, it must be one of the `features` key, - otherwise results in `ValueError`. - reader_args: Additional arguments to pass to the reader class. - num_epochs: Integer specifying the number of times to read through the - dataset. If None, cycles through the dataset forever. Defaults to `None`. - shuffle: A boolean, indicates whether the input should be shuffled. Defaults - to `True`. - shuffle_buffer_size: Buffer size of the ShuffleDataset. A large capacity - ensures better shuffling but would increase memory usage and startup time. - shuffle_seed: Randomization seed to use for shuffling. - prefetch_buffer_size: Number of feature batches to prefetch in order to - improve performance. Recommended value is the number of batches consumed - per training step. Defaults to auto-tune. - reader_num_threads: Number of threads used to read `Example` records. If >1, - the results will be interleaved. Defaults to `1`. - parser_num_threads: Number of threads to use for parsing `Example` tensors - into a dictionary of `Feature` tensors. Defaults to `2`. - sloppy_ordering: If `True`, reading performance will be improved at - the cost of non-deterministic ordering. If `False`, the order of elements - produced is deterministic prior to shuffling (elements are still - randomized if `shuffle=True`. Note that if the seed is set, then order - of elements after shuffling is deterministic). Defaults to `False`. - drop_final_batch: If `True`, and the batch size does not evenly divide the - input dataset size, the final smaller batch will be dropped. Defaults to - `False`. - - Returns: - A dataset of `dict` elements, (or a tuple of `dict` elements and label). - Each `dict` maps feature keys to `Tensor` or `SparseTensor` objects. - - Raises: - ValueError: If `label_key` is not one of the `features` keys. - """ - return readers.make_batched_features_dataset( - file_pattern, batch_size, features, reader, label_key, reader_args, - num_epochs, shuffle, shuffle_buffer_size, shuffle_seed, - prefetch_buffer_size, reader_num_threads, parser_num_threads, - sloppy_ordering, drop_final_batch) - - -@deprecation.deprecated( - None, "Use `tf.data.experimental.make_batched_features_dataset(...)`") -def read_batch_features(file_pattern, - batch_size, - features, - reader=core_readers.TFRecordDataset, - reader_args=None, - randomize_input=True, - num_epochs=None, - capacity=10000): - """Reads batches of Examples. - - Example: - - ``` - serialized_examples = [ - features { - feature { key: "age" value { int64_list { value: [ 0 ] } } } - feature { key: "gender" value { bytes_list { value: [ "f" ] } } } - feature { key: "kws" value { bytes_list { value: [ "code", "art" ] } } } - }, - features { - feature { key: "age" value { int64_list { value: [] } } } - feature { key: "gender" value { bytes_list { value: [ "f" ] } } } - feature { key: "kws" value { bytes_list { value: [ "sports" ] } } } - } - ] - ``` - - We can use arguments: - - ``` - features: { - "age": FixedLenFeature([], dtype=tf.int64, default_value=-1), - "gender": FixedLenFeature([], dtype=tf.string), - "kws": VarLenFeature(dtype=tf.string), - } - ``` - - And the expected output is: - - ```python - { - "age": [[0], [-1]], - "gender": [["f"], ["f"]], - "kws": SparseTensor( - indices=[[0, 0], [0, 1], [1, 0]], - values=["code", "art", "sports"] - dense_shape=[2, 2]), - } - ``` - - Args: - file_pattern: List of files or patterns of file paths containing - `Example` records. See `tf.io.gfile.glob` for pattern rules. - batch_size: An int representing the number of records to combine - in a single batch. - features: A `dict` mapping feature keys to `FixedLenFeature` or - `VarLenFeature` values. See `tf.io.parse_example`. - reader: A function or class that can be - called with a `filenames` tensor and (optional) `reader_args` and returns - a `Dataset` of `Example` tensors. Defaults to `tf.data.TFRecordDataset`. - reader_args: Additional arguments to pass to the reader class. - randomize_input: Whether the input should be randomized. - num_epochs: Integer specifying the number of times to read through the - dataset. If None, cycles through the dataset forever. - capacity: Buffer size of the ShuffleDataset. A large capacity ensures better - shuffling but would increase memory usage and startup time. - Returns: - A dict from keys in features to `Tensor` or `SparseTensor` objects. - """ - dataset = readers.make_batched_features_dataset( - file_pattern, - batch_size, - features, - reader=reader, - reader_args=reader_args, - shuffle=randomize_input, - num_epochs=num_epochs, - shuffle_buffer_size=capacity) - iterator = dataset_ops.make_one_shot_iterator(dataset) - outputs = iterator.get_next() - return outputs - - -class SqlDataset(readers.SqlDataset): - """A `Dataset` consisting of the results from a SQL query.""" - - @deprecation.deprecated(None, "Use `tf.data.experimental.SqlDataset(...)`.") - def __init__(self, driver_name, data_source_name, query, output_types): - super(SqlDataset, self).__init__( - driver_name, data_source_name, query, output_types) - - -class LMDBDataset(dataset_ops.DatasetSource): - """A LMDB Dataset that reads the lmdb file.""" - - def __init__(self, filenames): - """Create a `LMDBDataset`. - - `LMDBDataset` allows a user to read data from a mdb file as - (key value) pairs sequentially. - For example: - ```python - tf.compat.v1.enable_eager_execution() - - dataset = tf.contrib.lmdb.LMDBDataset("/foo/bar.mdb") - - # Prints the (key, value) pairs inside a lmdb file. - for key, value in dataset: - print(key, value) - ``` - Args: - filenames: A `tf.string` tensor containing one or more filenames. - """ - self._filenames = ops.convert_to_tensor( - filenames, dtype=dtypes.string, name="filenames") - if compat.forward_compatible(2019, 8, 3): - variant_tensor = gen_experimental_dataset_ops.lmdb_dataset( - self._filenames, **self._flat_structure) - else: - variant_tensor = gen_experimental_dataset_ops.experimental_lmdb_dataset( - self._filenames, **self._flat_structure) - super(LMDBDataset, self).__init__(variant_tensor) - - @property - def element_spec(self): - return (tensor_spec.TensorSpec([], dtypes.string), - tensor_spec.TensorSpec([], dtypes.string)) diff --git a/tensorflow/contrib/data/python/ops/resampling.py b/tensorflow/contrib/data/python/ops/resampling.py deleted file mode 100644 index 29d77528d95..00000000000 --- a/tensorflow/contrib/data/python/ops/resampling.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Resampling dataset transformations.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.experimental.ops import resampling -from tensorflow.python.util import deprecation - - -@deprecation.deprecated(None, - "Use `tf.data.experimental.rejection_resample(...)`.") -def rejection_resample(class_func, target_dist, initial_dist=None, seed=None): - """A transformation that resamples a dataset to achieve a target distribution. - - **NOTE** Resampling is performed via rejection sampling; some fraction - of the input values will be dropped. - - Args: - class_func: A function mapping an element of the input dataset to a scalar - `tf.int32` tensor. Values should be in `[0, num_classes)`. - target_dist: A floating point type tensor, shaped `[num_classes]`. - initial_dist: (Optional.) A floating point type tensor, shaped - `[num_classes]`. If not provided, the true class distribution is - estimated live in a streaming fashion. - seed: (Optional.) Python integer seed for the resampler. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - return resampling.rejection_resample(class_func, target_dist, initial_dist, - seed) diff --git a/tensorflow/contrib/data/python/ops/scan_ops.py b/tensorflow/contrib/data/python/ops/scan_ops.py deleted file mode 100644 index 0ca9fddb23b..00000000000 --- a/tensorflow/contrib/data/python/ops/scan_ops.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Scan dataset transformation.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.experimental.ops import scan_ops -from tensorflow.python.util import deprecation - - -@deprecation.deprecated(None, "Use `tf.data.experimental.scan(...)`.") -def scan(initial_state, scan_func): - """A transformation that scans a function across an input dataset. - - This transformation is a stateful relative of `tf.data.Dataset.map`. - In addition to mapping `scan_func` across the elements of the input dataset, - `scan()` accumulates one or more state tensors, whose initial values are - `initial_state`. - - Args: - initial_state: A nested structure of tensors, representing the initial state - of the accumulator. - scan_func: A function that maps `(old_state, input_element)` to - `(new_state, output_element). It must take two arguments and return a - pair of nested structures of tensors. The `new_state` must match the - structure of `initial_state`. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - return scan_ops.scan(initial_state, scan_func) diff --git a/tensorflow/contrib/data/python/ops/shuffle_ops.py b/tensorflow/contrib/data/python/ops/shuffle_ops.py deleted file mode 100644 index ef9944e6143..00000000000 --- a/tensorflow/contrib/data/python/ops/shuffle_ops.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Experimental shuffle ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.experimental.ops import shuffle_ops -from tensorflow.python.util import deprecation - - -@deprecation.deprecated(None, - "Use `tf.data.experimental.shuffle_and_repeat(...)`.") -def shuffle_and_repeat(buffer_size, count=None, seed=None): - """Shuffles and repeats a Dataset returning a new permutation for each epoch. - - `dataset.apply(tf.data.experimental.shuffle_and_repeat(buffer_size, count))` - - is equivalent to - - `dataset.shuffle(buffer_size, reshuffle_each_iteration=True).repeat(count)` - - The difference is that the latter dataset is not serializable. So, - if you need to checkpoint an input pipeline with reshuffling you must use - this implementation. - - Args: - buffer_size: A `tf.int64` scalar `tf.Tensor`, representing the - maximum number elements that will be buffered when prefetching. - count: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the - number of times the dataset should be repeated. The default behavior - (if `count` is `None` or `-1`) is for the dataset be repeated - indefinitely. - seed: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the - random seed that will be used to create the distribution. See - `tf.compat.v1.set_random_seed` for behavior. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - return shuffle_ops.shuffle_and_repeat(buffer_size, count, seed) diff --git a/tensorflow/contrib/data/python/ops/sliding.py b/tensorflow/contrib/data/python/ops/sliding.py deleted file mode 100644 index c329b6559f9..00000000000 --- a/tensorflow/contrib/data/python/ops/sliding.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Sliding dataset transformations.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.compat import compat -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.util import nest -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops -from tensorflow.python.util import deprecation - - -class _SlideDataset(dataset_ops.UnaryDataset): - """A `Dataset` that passes a sliding window over its input.""" - - def __init__(self, input_dataset, window_size, window_shift, window_stride): - """See `sliding_window_batch` for details.""" - self._input_dataset = input_dataset - self._window_size = ops.convert_to_tensor( - window_size, dtype=dtypes.int64, name="window_stride") - self._window_stride = ops.convert_to_tensor( - window_stride, dtype=dtypes.int64, name="window_stride") - self._window_shift = ops.convert_to_tensor( - window_shift, dtype=dtypes.int64, name="window_shift") - - input_structure = dataset_ops.get_structure(input_dataset) - self._element_spec = nest.map_structure( - lambda component_spec: component_spec._batch(None), input_structure) # pylint: disable=protected-access - if compat.forward_compatible(2019, 8, 3): - variant_tensor = ged_ops.sliding_window_dataset( - self._input_dataset._variant_tensor, # pylint: disable=protected-access - window_size=self._window_size, - window_shift=self._window_shift, - window_stride=self._window_stride, - **self._flat_structure) - else: - variant_tensor = ged_ops.experimental_sliding_window_dataset( - self._input_dataset._variant_tensor, # pylint: disable=protected-access - window_size=self._window_size, - window_shift=self._window_shift, - window_stride=self._window_stride, - **self._flat_structure) - super(_SlideDataset, self).__init__(input_dataset, variant_tensor) - - @property - def element_spec(self): - return self._element_spec - - -@deprecation.deprecated_args( - None, "stride is deprecated, use window_shift instead", "stride") -@deprecation.deprecated( - None, "Use `tf.data.Dataset.window(size=window_size, shift=window_shift, " - "stride=window_stride).flat_map(lambda x: x.batch(window_size))` " - "instead.") -def sliding_window_batch(window_size, - stride=None, - window_shift=None, - window_stride=1): - """A sliding window over a dataset. - - This transformation passes a sliding window over this dataset. The window size - is `window_size`, the stride of the input elements is `window_stride`, and the - shift between consecutive windows is `window_shift`. If the remaining elements - cannot fill up the sliding window, this transformation will drop the final - smaller element. For example: - - ```python - # NOTE: The following examples use `{ ... }` to represent the - # contents of a dataset. - a = { [1], [2], [3], [4], [5], [6] } - - a.apply(sliding_window_batch(window_size=3)) == - { [[1], [2], [3]], [[2], [3], [4]], [[3], [4], [5]], [[4], [5], [6]] } - - a.apply(sliding_window_batch(window_size=3, window_shift=2)) == - { [[1], [2], [3]], [[3], [4], [5]] } - - a.apply(sliding_window_batch(window_size=3, window_stride=2)) == - { [[1], [3], [5]], [[2], [4], [6]] } - ``` - - Args: - window_size: A `tf.int64` scalar `tf.Tensor`, representing the number of - elements in the sliding window. It must be positive. - stride: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the - forward shift of the sliding window in each iteration. The default is `1`. - It must be positive. Deprecated alias for `window_shift`. - window_shift: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the - forward shift of the sliding window in each iteration. The default is `1`. - It must be positive. - window_stride: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the - stride of the input elements in the sliding window. The default is `1`. - It must be positive. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - - Raises: - ValueError: if invalid arguments are provided. - """ - if stride is None and window_shift is None: - window_shift = 1 - elif stride is not None and window_shift is None: - window_shift = stride - elif stride is not None and window_shift is not None: - raise ValueError("Cannot specify both `stride` and `window_shift`") - - def _apply_fn(dataset): - return _SlideDataset(dataset, window_size, window_shift, window_stride) - - return _apply_fn diff --git a/tensorflow/contrib/data/python/ops/threadpool.py b/tensorflow/contrib/data/python/ops/threadpool.py deleted file mode 100644 index 20cceb4647a..00000000000 --- a/tensorflow/contrib/data/python/ops/threadpool.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Experimental API for controlling threading in `tf.data` pipelines.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.python.data.experimental.ops.threadpool import override_threadpool -from tensorflow.python.data.experimental.ops.threadpool import PrivateThreadPool diff --git a/tensorflow/contrib/data/python/ops/unique.py b/tensorflow/contrib/data/python/ops/unique.py deleted file mode 100644 index 9129599eb95..00000000000 --- a/tensorflow/contrib/data/python/ops/unique.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Unique element dataset transformations.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.experimental.ops import unique as experimental_unique -from tensorflow.python.util import deprecation - - -@deprecation.deprecated(None, "Use `tf.data.experimental.unique()`.") -def unique(): - """Creates a `Dataset` from another `Dataset`, discarding duplicates. - - Use this transformation to produce a dataset that contains one instance of - each unique element in the input. For example: - - ```python - dataset = tf.data.Dataset.from_tensor_slices([1, 37, 2, 37, 2, 1]) - - # Using `unique()` will drop the duplicate elements. - dataset = dataset.apply(tf.data.experimental.unique()) # ==> { 1, 37, 2 } - ``` - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - return experimental_unique.unique() diff --git a/tensorflow/contrib/data/python/ops/writers.py b/tensorflow/contrib/data/python/ops/writers.py deleted file mode 100644 index 42fb69bf077..00000000000 --- a/tensorflow/contrib/data/python/ops/writers.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python wrappers for tf.data writers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.experimental.ops import writers -from tensorflow.python.util import deprecation - - -class TFRecordWriter(writers.TFRecordWriter): - """Writes data to a TFRecord file.""" - - @deprecation.deprecated( - None, "Use `tf.data.experimental.TFRecordWriter(...)`.") - def __init__(self, filename, compression_type=None): - super(TFRecordWriter, self).__init__(filename, compression_type) diff --git a/tensorflow/contrib/decision_trees/__init__.py b/tensorflow/contrib/decision_trees/__init__.py deleted file mode 100644 index 2c44e4eb59f..00000000000 --- a/tensorflow/contrib/decision_trees/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Shared representations for tree-based models in tensorflow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.decision_trees.proto import * -# pylint: enable=unused-import,wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented - - -remove_undocumented(__name__) diff --git a/tensorflow/contrib/decision_trees/proto/BUILD b/tensorflow/contrib/decision_trees/proto/BUILD deleted file mode 100644 index ebbb9b3c052..00000000000 --- a/tensorflow/contrib/decision_trees/proto/BUILD +++ /dev/null @@ -1,37 +0,0 @@ -load( - "//tensorflow/core/platform:default/build_config.bzl", - "tf_proto_library", - "tf_pyclif_proto_library", -) - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files([ - "LICENSE", - "generic_tree_model_proto.swig", -]) - -tf_proto_library( - name = "generic_tree_model", - srcs = ["generic_tree_model.proto"], - cc_api_version = 2, - visibility = ["//visibility:public"], -) - -tf_proto_library( - name = "generic_tree_model_extensions", - srcs = ["generic_tree_model_extensions.proto"], - cc_api_version = 2, - protodeps = [":generic_tree_model"], - visibility = ["//visibility:public"], -) - -tf_pyclif_proto_library( - name = "generic_tree_model_pyclif", - proto_lib = ":generic_tree_model", - proto_srcfile = "generic_tree_model.proto", - visibility = ["//visibility:public"], -) diff --git a/tensorflow/contrib/decision_trees/proto/generic_tree_model.proto b/tensorflow/contrib/decision_trees/proto/generic_tree_model.proto deleted file mode 100644 index dd80b37f52e..00000000000 --- a/tensorflow/contrib/decision_trees/proto/generic_tree_model.proto +++ /dev/null @@ -1,183 +0,0 @@ -// Generic representation of tree-based models. - -// This proto establishes a shared standard: "fully compatible" projects should -// provide support for all reasonable models expressed through it. Therefore, -// it should be kept as simple as possible, and should never contain -// project-specific design choices. - -// Status: work in progress. This proto can change anytime without notice. - -syntax = "proto3"; -option cc_enable_arenas = true; - -package tensorflow.decision_trees; - -import "google/protobuf/any.proto"; -import "google/protobuf/wrappers.proto"; - -// A generic handle for any type of model. -message Model { - oneof model { - DecisionTree decision_tree = 1; - Ensemble ensemble = 2; - google.protobuf.Any custom_model = 3; - } - repeated google.protobuf.Any additional_data = 4; -} - -message ModelAndFeatures { - message Feature { - // TODO(jonasz): Remove this field, as it's confusing. Ctx: cr/153569450. - FeatureId feature_id = 1 [deprecated = true]; - repeated google.protobuf.Any additional_data = 2; - }; - // Given a FeatureId feature_id, the feature's description is in - // features[feature_id.id.value]. - map features = 1; - Model model = 2; - repeated google.protobuf.Any additional_data = 3; -} - -// An ordered sequence of models. This message can be used to express bagged or -// boosted models, as well as custom ensembles. -message Ensemble { - message Member { - Model submodel = 1; - google.protobuf.Int32Value submodel_id = 2; - repeated google.protobuf.Any additional_data = 3; - } - repeated Member members = 100; // A higher id for more readable printing. - - // The presence of a certain combination_technique indicates how to combine - // the outputs of member models in order to compute the ensemble's output. - oneof combination_technique { - Summation summation_combination_technique = 1; - Averaging averaging_combination_technique = 2; - google.protobuf.Any custom_combination_technique = 3; - } - repeated google.protobuf.Any additional_data = 4; -} - -// When present, the Ensemble's output is the sum of member models' outputs. -message Summation { - repeated google.protobuf.Any additional_data = 1; -}; - - -// When present, the Ensemble's output is the average of member models' outputs. -message Averaging { - repeated google.protobuf.Any additional_data = 1; -}; - - -message DecisionTree { - repeated TreeNode nodes = 1; - repeated google.protobuf.Any additional_data = 2; -}; - - -message TreeNode { - // Following fields are provided for convenience and better readability. - // Filling them in is not required. - google.protobuf.Int32Value node_id = 1; - google.protobuf.Int32Value depth = 2; - google.protobuf.Int32Value subtree_size = 3; - - oneof node_type { - BinaryNode binary_node = 4; - Leaf leaf = 5; - google.protobuf.Any custom_node_type = 6; - } - - repeated google.protobuf.Any additional_data = 7; -} - - -message BinaryNode { - google.protobuf.Int32Value left_child_id = 1; - google.protobuf.Int32Value right_child_id = 2; - enum Direction { - LEFT = 0; - RIGHT = 1; - } - // When left_child_test is undefined for a particular datapoint (e.g. because - // it's not defined when feature value is missing), the datapoint should go - // in this direction. - Direction default_direction = 3; - // When a datapoint satisfies the test, it should be propagated to the left - // child. - oneof left_child_test { - InequalityTest inequality_left_child_test = 4; - google.protobuf.Any custom_left_child_test = 5; - } -}; - -// A SparseVector represents a vector in which only certain select elements -// are non-zero. Maps labels to values (e.g. class id to probability or count). -message SparseVector { - map sparse_value = 1; -} - -message Vector { - repeated Value value = 1; -} - -message Leaf { - oneof leaf { - // The interpretation of the values held in the leaves of a decision tree - // is application specific, but some common cases are: - // 1) len(vector) = 1, and the floating point value[0] holds the class 0 - // probability in a two class classification problem. - // 2) len(vector) = 1, and the integer value[0] holds the class prediction. - // 3) The floating point value[i] holds the class i probability prediction. - // 4) The floating point value[i] holds the i-th component of the - // vector prediction in a regression problem. - // 5) sparse_vector holds the sparse class predictions for a classification - // problem with a large number of classes. - Vector vector = 1; - SparseVector sparse_vector = 2; - } - // For non-standard handling of leaves. - repeated google.protobuf.Any additional_data = 3; -}; - - -message FeatureId { - google.protobuf.StringValue id = 1; - repeated google.protobuf.Any additional_data = 2; -}; - -message ObliqueFeatures { - // total value is sum(features[i] * weights[i]). - repeated FeatureId features = 1; - repeated float weights = 2; -} - - -message InequalityTest { - // When the feature is missing, the test's outcome is undefined. - oneof FeatureSum { - FeatureId feature_id = 1; - ObliqueFeatures oblique = 4; - } - enum Type { - LESS_OR_EQUAL = 0; - LESS_THAN = 1; - GREATER_OR_EQUAL = 2; - GREATER_THAN = 3; - }; - Type type = 2; - Value threshold = 3; -}; - - -// Represents a single value of any type, e.g. 5 or "abc". -message Value { - oneof value { - float float_value = 1; - double double_value = 2; - int32 int32_value = 3; - int64 int64_value = 4; - google.protobuf.Any custom_value = 5; - } -}; diff --git a/tensorflow/contrib/decision_trees/proto/generic_tree_model_extensions.proto b/tensorflow/contrib/decision_trees/proto/generic_tree_model_extensions.proto deleted file mode 100644 index 2a41b321b78..00000000000 --- a/tensorflow/contrib/decision_trees/proto/generic_tree_model_extensions.proto +++ /dev/null @@ -1,30 +0,0 @@ -// Messages in this file are not part of the basic standard established by -// generic_tree_model.proto (see the toplevel comment in that file). - -syntax = "proto3"; - -package tensorflow.decision_trees; - -import "tensorflow/contrib/decision_trees/proto/generic_tree_model.proto"; - -// Used in generic_tree_model.BinaryNode.left_child_test. -// Tests whether the feature's value belongs to the specified list, -// (or does not belong if inverse=True). -// For empty list use ConstResultTest instead. -message MatchingValuesTest { - // When the feature is missing, the test's outcome is undefined. - FeatureId feature_id = 1; - repeated Value value = 2; - bool inverse = 3; -} - -// Used in generic_tree_model.BinaryNode.left_child_test. -// Returns test_result if feature value is not missed. Otherwise -// BinaryNode.default_direction is used. -message ConstResultTest { - FeatureId feature_id = 1; - // value_for_dtype is used to store the type of the feature. The value itself - // should be ignored, only its type is used. - Value value_for_dtype = 2; - bool test_result = 3; -} diff --git a/tensorflow/contrib/decision_trees/proto/generic_tree_model_proto.swig b/tensorflow/contrib/decision_trees/proto/generic_tree_model_proto.swig deleted file mode 100644 index cafb9314cae..00000000000 --- a/tensorflow/contrib/decision_trees/proto/generic_tree_model_proto.swig +++ /dev/null @@ -1,14 +0,0 @@ -////////// SWIG INCLUDE ////////// - -%include "net/proto/swig/protofunc.swig" - -#ifndef ABSL_MUST_USE_RESULT -#error Use this file only as a %include or %import after google.swig. -#endif - -%{ -#include "third_party/tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" -%} - -PROTO_INPUT(tensorflow::decision_trees::DecisionTree, decision_tree); -PROTO_IN_OUT(tensorflow::decision_trees::DecisionTree, decision_tree); diff --git a/tensorflow/contrib/deprecated/BUILD b/tensorflow/contrib/deprecated/BUILD deleted file mode 100644 index 62529afe4c9..00000000000 --- a/tensorflow/contrib/deprecated/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -# Description: -# Contains deprecated functions that we aren't quite ready to remove entirely - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "deprecated_py", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:logging_ops", - "//tensorflow/python:util", - ], -) - -py_test( - name = "summaries_test", - srcs = ["summaries_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:logging_ops", - ], -) diff --git a/tensorflow/contrib/deprecated/__init__.py b/tensorflow/contrib/deprecated/__init__.py deleted file mode 100644 index fc4742bc2c8..00000000000 --- a/tensorflow/contrib/deprecated/__init__.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Non-core alias for the deprecated tf.X_summary ops. - -For TensorFlow 1.0, we have reorganized the TensorFlow summary ops into a -submodule, and made some semantic tweaks. The first thing to note is that we -moved the APIs around as follows: - -```python -tf.scalar_summary -> tf.compat.v1.summary.scalar -tf.histogram_summary -> tf.compat.v1.summary.histogram -tf.audio_summary -> tf.compat.v1.summary.audio -tf.image_summary -> tf.compat.v1.summary.image -tf.merge_summary -> tf.compat.v1.summary.merge -tf.merge_all_summaries -> tf.compat.v1.summary.merge_all -``` - -We think this API is cleaner and will improve long-term discoverability and -clarity of the TensorFlow API. But we also took the opportunity to make an -important change to how summary "tags" work. The "tag" of a summary is the -string that is associated with the output data, i.e. the key for organizing the -generated protobufs. - -Previously, the tag was allowed to be any unique string; it had no relation -to the summary op generating it, and no relation to the TensorFlow name system. -This behavior made it very difficult to write reusable that would add -summary ops to the graph. If you had a function to add summary ops, you would -need to pass in a `tf.name_scope`, manually, to that function to create -deduplicated tags. Otherwise your program would fail with a runtime error due -to tag collision. - -The new summary APIs under `tf.summary` throw away the "tag" as an independent -concept; instead, the first argument is the node name. So summary tags now -automatically inherit the surrounding `tf.name_scope`, and automatically -are deduplicated if there is a conflict. Now however, the only allowed -characters are alphanumerics, underscores, and forward slashes. To make -migration easier, the new APIs automatically convert illegal characters to -underscores. - -Just as an example, consider the following "before" and "after" code snippets: - -```python -# Before -def add_activation_summaries(v, scope): - tf.scalar_summary("%s/fraction_of_zero" % scope, tf.nn.fraction_of_zero(v)) - tf.histogram_summary("%s/activations" % scope, v) - -# After -def add_activation_summaries(v): - tf.compat.v1.summary.scalar("fraction_of_zero", tf.nn.fraction_of_zero(v)) - tf.compat.v1.summary.histogram("activations", v) -``` - -Now, so long as the add_activation_summaries function is called from within the -right `tf.name_scope`, the behavior is the same. - -Because this change does modify the behavior and could break tests, we can't -automatically migrate usage to the new APIs. That is why we are making the old -APIs temporarily available here at `tf.contrib.deprecated`. - -In addition to the name change described above, there are two further changes -to the new summary ops: - -- the "max_images" argument for `tf.image_summary` was renamed to "max_outputs - for `tf.compat.v1.summary.image` -- `tf.scalar_summary` accepted arbitrary tensors of tags and values. But - `tf.compat.v1.summary.scalar` requires a single scalar name and scalar value. - In most - cases, you can create `tf.compat.v1.summary.scalar` in a loop to get the same - behavior - -As before, TensorBoard groups charts by the top-level `tf.name_scope` which may -be inconvenient, for in the new summary ops, the summary will inherit that -`tf.name_scope` without user control. We plan to add more grouping mechanisms -to TensorBoard, so it will be possible to specify the TensorBoard group for -each summary via the summary API. - -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.python.ops.logging_ops import audio_summary -from tensorflow.python.ops.logging_ops import histogram_summary -from tensorflow.python.ops.logging_ops import image_summary -from tensorflow.python.ops.logging_ops import merge_all_summaries -from tensorflow.python.ops.logging_ops import merge_summary -from tensorflow.python.ops.logging_ops import scalar_summary - -from tensorflow.python.util.all_util import remove_undocumented -# pylint: enable=unused-import,line-too-long - -_allowed_symbols = [ - 'audio_summary', 'histogram_summary', 'image_summary', - 'merge_all_summaries', 'merge_summary', 'scalar_summary' -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/deprecated/summaries_test.py b/tensorflow/contrib/deprecated/summaries_test.py deleted file mode 100644 index 4038224a1c7..00000000000 --- a/tensorflow/contrib/deprecated/summaries_test.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the deprecated summary ops in tf.contrib.deprecated.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import logging_ops -from tensorflow.python.platform import test - - -class DeprecatedSummariesTest(test.TestCase): - - def testScalarSummary(self): - with self.cached_session(): - c = constant_op.constant(3) - s = logging_ops.scalar_summary('tag', c) - self.assertEqual(s.op.type, u'ScalarSummary') - - def testHistogramSummary(self): - with self.cached_session(): - c = constant_op.constant(3) - s = logging_ops.histogram_summary('tag', c) - self.assertEqual(s.op.type, u'HistogramSummary') - - def testImageSummary(self): - with self.cached_session(): - i = array_ops.ones((5, 4, 4, 3)) - s = logging_ops.image_summary('tag', i) - self.assertEqual(s.op.type, u'ImageSummary') - - def testAudioSummary(self): - with self.cached_session(): - c = constant_op.constant(3.0) - s = logging_ops.audio_summary('tag', c, sample_rate=8000) - self.assertEqual(s.op.type, u'AudioSummaryV2') - - def testMergeSummary(self): - with self.cached_session(): - c = constant_op.constant(3) - a = logging_ops.scalar_summary('a', c) - b = logging_ops.scalar_summary('b', c) - s = logging_ops.merge_summary([a, b]) - self.assertEqual(s.op.type, u'MergeSummary') - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/distribute/BUILD b/tensorflow/contrib/distribute/BUILD deleted file mode 100644 index 112cb4ac54e..00000000000 --- a/tensorflow/contrib/distribute/BUILD +++ /dev/null @@ -1,40 +0,0 @@ -# Implementation of a prototype TF distributed computation library. - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -filegroup( - name = "all_files", - srcs = glob( - ["**/*"], - exclude = [ - "**/METADATA", - "**/OWNERS", - ], - ), - visibility = ["//tensorflow:__subpackages__"], -) - -py_library( - name = "distribute", - srcs = ["__init__.py"], - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/contrib/distribute/python:collective_all_reduce_strategy", - "//tensorflow/contrib/distribute/python:mirrored_strategy", - "//tensorflow/contrib/distribute/python:monitor", - "//tensorflow/contrib/distribute/python:one_device_strategy", - "//tensorflow/contrib/distribute/python:parameter_server_strategy", - "//tensorflow/contrib/distribute/python:tpu_strategy", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python/distribute:cross_device_ops", - "//tensorflow/python/distribute:distribute_config", - "//tensorflow/python/distribute:distribute_coordinator", - "//tensorflow/python/distribute:step_fn", - ], -) diff --git a/tensorflow/contrib/distribute/README.md b/tensorflow/contrib/distribute/README.md deleted file mode 100644 index 680907252db..00000000000 --- a/tensorflow/contrib/distribute/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Distribution Strategy - -See the guide for overview and examples: -[TensorFlow v1.x](https://www.tensorflow.org/guide/distribute_strategy), -[TensorFlow v2.x](https://www.tensorflow.org/alpha/guide/distribute_strategy). diff --git a/tensorflow/contrib/distribute/__init__.py b/tensorflow/contrib/distribute/__init__.py deleted file mode 100644 index 40a0e773978..00000000000 --- a/tensorflow/contrib/distribute/__init__.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A distributed computation library for TF. - -See [tensorflow/contrib/distribute/README.md]( -https://www.tensorflow.org/code/tensorflow/contrib/distribute/README.md) -for overview and examples. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.distribute.python.collective_all_reduce_strategy import CollectiveAllReduceStrategy -from tensorflow.contrib.distribute.python.mirrored_strategy import MirroredStrategy -from tensorflow.contrib.distribute.python.monitor import Monitor -from tensorflow.contrib.distribute.python.one_device_strategy import OneDeviceStrategy -from tensorflow.contrib.distribute.python.parameter_server_strategy import ParameterServerStrategy -from tensorflow.contrib.distribute.python.tpu_strategy import initialize_tpu_system -from tensorflow.contrib.distribute.python.tpu_strategy import TPUStrategy -from tensorflow.python.distribute.cross_device_ops import * -from tensorflow.python.distribute.distribute_config import DistributeConfig -from tensorflow.python.distribute.distribute_coordinator import run_standard_tensorflow_server -from tensorflow.python.distribute.distribute_lib import * -from tensorflow.python.distribute.distribution_strategy_context import * - -from tensorflow.python.distribute.step_fn import * -from tensorflow.python.util.all_util import remove_undocumented - - -DistributionStrategy = StrategyV1 - - -_allowed_symbols = [ - 'AllReduceCrossDeviceOps', - 'CollectiveAllReduceStrategy', - 'CrossDeviceOps', - 'DistributeConfig', - 'DistributionStrategy', - 'DistributionStrategyExtended', - 'MirroredStrategy', - 'Monitor', - 'MultiWorkerAllReduce', - 'OneDeviceStrategy', - 'ParameterServerStrategy', - 'ReductionToOneDeviceCrossDeviceOps', - 'Step', - 'StandardInputStep', - 'StandardSingleLossStep', - 'ReplicaContext', - 'TPUStrategy', - 'initialize_tpu_system', - 'get_cross_replica_context', - 'get_distribution_strategy', - 'get_loss_reduction', - 'get_replica_context', - 'get_strategy', - 'has_distribution_strategy', - 'has_strategy', - 'in_cross_replica_context', - 'require_replica_context', - 'run_standard_tensorflow_server', - 'UpdateContext', -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD deleted file mode 100644 index 3ae0975f6d0..00000000000 --- a/tensorflow/contrib/distribute/python/BUILD +++ /dev/null @@ -1,285 +0,0 @@ -# Implementation of a prototype TF distributed computation library. - -load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test") -load("//tensorflow/core/platform:default/distribute.bzl", "distribute_py_test") -load("//tensorflow:tensorflow.bzl", "cuda_py_test") - -package( - default_visibility = [ - "//tensorflow:internal", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "distribute_test_lib_pip", - visibility = ["//tensorflow:internal"], - deps = [ - ":keras_multi_worker_test_base", - "//tensorflow/python/distribute:combinations", - "//tensorflow/python/distribute:model_combinations", - "//tensorflow/python/distribute:multi_worker_test_base", - "//tensorflow/python/distribute:saved_model_test_base", - "//tensorflow/python/distribute:single_loss_example", - "//tensorflow/python/distribute:strategy_combinations", - "//tensorflow/python/distribute:strategy_test_lib", - "//tensorflow/python/keras/distribute:keras_correctness_test_lib", - "//tensorflow/python/keras/distribute:keras_test_lib", - ], -) - -py_library( - name = "mirrored_strategy", - srcs = ["mirrored_strategy.py"], - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python/distribute:distribute_lib", - "//tensorflow/python/distribute:input_lib", - "//tensorflow/python/distribute:mirrored_strategy", - ], -) - -py_library( - name = "parameter_server_strategy", - srcs = ["parameter_server_strategy.py"], - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python/distribute:distribute_lib", - "//tensorflow/python/distribute:parameter_server_strategy", - "//tensorflow/python/distribute:values", - "//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib", - ], -) - -cuda_py_test( - name = "parameter_server_strategy_test", - srcs = ["parameter_server_strategy_test.py"], - additional_deps = [ - ":parameter_server_strategy", - "//tensorflow/python/distribute:central_storage_strategy", - "//tensorflow/python/distribute:combinations", - "//tensorflow/python/distribute:parameter_server_strategy", - "//tensorflow/python/distribute:strategy_combinations", - "//tensorflow/python/distribute:multi_worker_test_base", - "//tensorflow/python/distribute:strategy_test_lib", - "@absl_py//absl/testing:parameterized", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:gradients", - "//tensorflow/python:layers", - "//tensorflow/python:session", - "//tensorflow/python:tensor_util", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/distribute:multi_worker_util", - "//tensorflow/python/distribute:values", - "//tensorflow/python/eager:context", - "//tensorflow/python/estimator:estimator_py", - ], - tags = [ - "multi_and_single_gpu", - "no_oss", # TODO(b/133330625) - ], - # TODO(b/141096229): Find problem with AssignAddVariableOps - xla_enable_strict_auto_jit = False, -) - -py_library( - name = "one_device_strategy", - srcs = ["one_device_strategy.py"], - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python/distribute:one_device_strategy", - ], -) - -py_library( - name = "collective_all_reduce_strategy", - srcs = ["collective_all_reduce_strategy.py"], - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python/distribute:collective_all_reduce_strategy", - "//tensorflow/python/distribute:distribute_lib", - "//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib", - ], -) - -cuda_py_test( - name = "contrib_mirrored_strategy_test", - srcs = ["contrib_mirrored_strategy_test.py"], - additional_deps = [ - ":mirrored_strategy", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:test", - "//tensorflow/python/distribute:combinations", - "//tensorflow/python/distribute:values", - ], - shard_count = 1, - tags = [ - "guitar", - "multi_and_single_gpu", - ], -) - -cuda_py_test( - name = "keras_multi_worker_correctness_test", - srcs = ["keras_multi_worker_correctness_test.py"], - additional_deps = [ - ":collective_all_reduce_strategy", - ":mirrored_strategy", - ":parameter_server_strategy", - ":keras_multi_worker_test_base", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:training", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/distribute:distribute_config", - "//tensorflow/python/distribute:distribute_coordinator", - "//tensorflow/python/distribute:multi_worker_util", - "//tensorflow/python/eager:context", - "//tensorflow/python/keras", - ], - tags = [ - "manual", - "multi_and_single_gpu", - "no_oss", - "nomsan", # TODO(b/130299192) - "notap", # TODO(b/130432997) - ], -) - -py_library( - name = "keras_multi_worker_test_base", - srcs = ["keras_multi_worker_test_base.py"], - deps = [ - ":collective_all_reduce_strategy", - ":mirrored_strategy", - ":parameter_server_strategy", - "//tensorflow/python:client_testlib", - "//tensorflow/python/distribute:combinations", - "//tensorflow/python/distribute:multi_worker_test_base", - ], -) - -py_library( - name = "tpu_strategy", - srcs = ["tpu_strategy.py"], - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/contrib/tpu:tpu_lib", - "//tensorflow/python/distribute:tpu_strategy", - ], -) - -cuda_py_test( - name = "collective_all_reduce_strategy_test", - srcs = ["collective_all_reduce_strategy_test.py"], - additional_deps = [ - ":collective_all_reduce_strategy", - "//tensorflow/python/distribute:combinations", - "//tensorflow/python/distribute:strategy_combinations", - "//tensorflow/python/distribute:multi_worker_test_base", - "//tensorflow/python/distribute:strategy_test_lib", - "@absl_py//absl/testing:parameterized", - "//third_party/py/numpy", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:gradients", - "//tensorflow/python:init_ops", - "//tensorflow/python:layers", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/distribute:cross_device_utils", - "//tensorflow/python/eager:context", - "//tensorflow/python/estimator:estimator_py", - ], - tags = [ - "multi_and_single_gpu", - "noguitar", # b/139307796 - ], -) - -cuda_py_test( - name = "optimizer_v2_test", - srcs = ["optimizer_v2_test.py"], - additional_deps = [ - "//tensorflow/python/distribute:combinations", - "//tensorflow/python/distribute:strategy_combinations", - "//tensorflow/python/distribute:single_loss_example", - ":mirrored_strategy", - "@absl_py//absl/testing:parameterized", - "//third_party/py/numpy", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:variables", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:test", - "//tensorflow/contrib/optimizer_v2:training", - ], - tags = [ - "multi_and_single_gpu", - ], -) - -py_library( - name = "monitor", - srcs = ["monitor.py"], - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python:variables", - "//tensorflow/python/eager:context", - ], -) - -cuda_py_test( - name = "monitor_test", - srcs = ["monitor_test.py"], - additional_deps = [ - "//tensorflow/python/distribute:combinations", - "//tensorflow/python/distribute:strategy_combinations", - ":monitor", - "//tensorflow/python/distribute:single_loss_example", - "@absl_py//absl/testing:parameterized", - "//third_party/py/numpy", - "//tensorflow/python/distribute:one_device_strategy", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:test", - "//tensorflow/python:framework_ops", - "//tensorflow/python:training", - ], - tags = [ - "multi_and_single_gpu", - ], -) - -# TODO(b/121200287): Remove this in 2.0 -distribute_py_test( - name = "keras_backward_compat_test", - srcs = ["keras_backward_compat_test.py"], - full_precision = True, - main = "keras_backward_compat_test.py", - shard_count = 31, - tags = [ - "multi_and_single_gpu", - "no_windows_gpu", - "notsan", - ], - xla_enable_strict_auto_jit = False, # Ignoring due to in contrib. - deps = [ - ":mirrored_strategy", - "//tensorflow/python/distribute:tpu_strategy", - "//tensorflow/python/keras/distribute:keras_test_lib", - ], -) diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py deleted file mode 100644 index 5f944e493dc..00000000000 --- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Class CollectiveAllReduceStrategy implementing DistributionStrategy.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.distribute import collective_all_reduce_strategy -from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib -from tensorflow.python.distribute import distribute_lib -from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver -from tensorflow.python.distribute.cluster_resolver import TFConfigClusterResolver - - -# TODO(yuefengz): support in-graph replication. -class CollectiveAllReduceStrategy(distribute_lib.StrategyV1): - """Distribution strategy that uses collective ops for all-reduce. - - *** contrib version *** - - It is similar to the MirroredStrategy but it uses collective ops for - reduction. - - When `cluster_spec` is given by the `configure` method, it turns into the - mulit-worker version that works on multiple workers with between-graph - replication. - - Note: `configure` will be called by higher-level APIs if running in - distributed environment. - """ - - def __init__(self, - num_gpus_per_worker=0, - communication=cross_device_ops_lib.CollectiveCommunication.AUTO): - """Initializes the object. - - Args: - num_gpus_per_worker: number of local GPUs or GPUs per worker, the default - is 0 meaning CPU only. - communication: optional Enum of type - `distribute.experimental.CollectiveCommunication`. This provides a way - for the user to override the choice of collective op communication. - Possible values include `AUTO`, `RING`, and `NCCL`. - """ - super(CollectiveAllReduceStrategy, self).__init__( - CollectiveAllReduceExtended( - self, - num_gpus_per_worker=num_gpus_per_worker, - communication=communication)) - - -class CollectiveAllReduceExtended( - collective_all_reduce_strategy.CollectiveAllReduceExtended): - """Implementation of CollectiveAllReduceStrategy.""" - - def __init__(self, - container_strategy, - num_gpus_per_worker, - communication): - # Use TFConfigClusterResolver to parse TF_CONFIG. We don't want to change - # the constructor's interface to allow customized cluster resolver. Use - # SimpleClusterResolver to override num_accelerators. - tfconfig = TFConfigClusterResolver() - cluster_resolver = SimpleClusterResolver( - cluster_spec=tfconfig.cluster_spec(), - task_type=tfconfig.task_type, - task_id=tfconfig.task_id, - num_accelerators={"GPU": num_gpus_per_worker}, - rpc_layer=tfconfig.rpc_layer) - super(CollectiveAllReduceExtended, self).__init__( - container_strategy, - communication=communication, - cluster_resolver=cluster_resolver) diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py deleted file mode 100644 index 1f527340d8d..00000000000 --- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py +++ /dev/null @@ -1,564 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for CollectiveAllReduceStrategy.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy as np - -from tensorflow.contrib.distribute.python import collective_all_reduce_strategy -from tensorflow.core.protobuf import config_pb2 -from tensorflow.core.protobuf import rewriter_config_pb2 -from tensorflow.python import keras -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.distribute import collective_all_reduce_strategy as core_collective_all_reduce_strategy -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib -from tensorflow.python.distribute import cross_device_utils -from tensorflow.python.distribute import distribute_lib -from tensorflow.python.distribute import multi_worker_test_base -from tensorflow.python.distribute import reduce_util -from tensorflow.python.distribute import strategy_test_lib -from tensorflow.python.distribute import values -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.layers import core -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import test -from tensorflow.python.training import adam -from tensorflow.python.training import training_util - - -class MockCollectiveAllReduceStrategy(distribute_lib.StrategyV1): - """Mock the strategy to allow cluster resolver as an argument.""" - - def __init__(self, cluster_resolver): - super(MockCollectiveAllReduceStrategy, self).__init__( - core_collective_all_reduce_strategy.CollectiveAllReduceExtended( - self, - communication=cross_device_ops_lib.CollectiveCommunication.AUTO, - cluster_resolver=cluster_resolver)) - - -def create_test_objects(cluster_spec=None, - task_type=None, - task_id=None, - num_gpus=None): - sess_config = config_pb2.ConfigProto() - if num_gpus is None: - num_gpus = context.num_gpus() - - strategy = collective_all_reduce_strategy.CollectiveAllReduceStrategy( - num_gpus_per_worker=num_gpus) - if task_type and task_id is not None: - strategy.configure( - session_config=sess_config, - cluster_spec=cluster_spec, - task_type=task_type, - task_id=task_id) - target = 'grpc://' + cluster_spec[task_type][task_id] - else: - target = '' - - return strategy, target, sess_config - - -class CollectiveAllReduceStrategyTestBase( - multi_worker_test_base.MultiWorkerTestBase): - - collective_key_base = 0 - - def setUp(self): - # We use a different key_base for each test so that collective keys won't be - # reused. - # TODO(yuefengz, ayushd): enable it to reuse collective keys in different - # tests. - CollectiveAllReduceStrategyTestBase.collective_key_base += 100000 - super(CollectiveAllReduceStrategyTestBase, self).setUp() - - def _get_test_object(self, task_type, task_id, num_gpus=0): - strategy, target, session_config = create_test_objects( - cluster_spec=self._cluster_spec, - task_type=task_type, - task_id=task_id, - num_gpus=num_gpus) - - collective_keys = cross_device_utils.CollectiveKeys( - group_key_start=10 + - CollectiveAllReduceStrategyTestBase.collective_key_base, - op_instance_key_start=100 + - CollectiveAllReduceStrategyTestBase.collective_key_base, - variable_instance_key_start=10000 + - CollectiveAllReduceStrategyTestBase.collective_key_base) - strategy.extended._collective_keys = collective_keys - strategy.extended._cross_device_ops._collective_keys = (collective_keys) - - return strategy, target, session_config - - def _test_minimize_loss_graph(self, task_type, task_id, num_gpus): - d, master_target, config = self._get_test_object(task_type, task_id, - num_gpus) - with ops.Graph().as_default(), \ - self.cached_session(config=config, - target=master_target) as sess, \ - d.scope(): - l = core.Dense(1, use_bias=False, - name='gpu_%d' % d.extended._num_gpus_per_worker) - - def loss_fn(x): - y = array_ops.reshape(l(x), []) - constant_op.constant(1.) - return y * y - - # TODO(yuefengz, apassos): eager.backprop.implicit_grad is not safe for - # multiple graphs (b/111216820). - def grad_fn(x): - loss = loss_fn(x) - var_list = ( - variables.trainable_variables() + ops.get_collection( - ops.GraphKeys.TRAINABLE_RESOURCE_VARIABLES)) - grads = gradients.gradients(loss, var_list) - ret = list(zip(grads, var_list)) - return ret - - def update(v, g): - return v.assign_sub(0.05 * g, use_locking=True) - - one = constant_op.constant([[1.]]) - - def step(): - """Perform one optimization step.""" - # Run forward & backward to get gradients, variables list. - g_v = d.extended.call_for_each_replica(grad_fn, args=[one]) - # Update the variables using the gradients and the update() function. - before_list = [] - after_list = [] - for g, v in g_v: - fetched = d.extended.read_var(v) - before_list.append(fetched) - with ops.control_dependencies([fetched]): - # TODO(yuefengz): support non-Mirrored variable as destinations. - g = d.extended.reduce_to( - reduce_util.ReduceOp.SUM, g, destinations=v) - with ops.control_dependencies( - d.extended.update(v, update, args=(g,), group=False)): - after_list.append(d.extended.read_var(v)) - return before_list, after_list - - before_out, after_out = step() - - if context.num_gpus() < d.extended._num_gpus_per_worker: - return True - - sess.run(variables.global_variables_initializer()) - - for i in range(10): - b, a = sess.run((before_out, after_out)) - if i == 0: - before, = b - after, = a - - error_before = abs(before - 1) - error_after = abs(after - 1) - # Error should go down - self.assertLess(error_after, error_before) - - def _test_complex_model(self, task_type, task_id, num_gpus): - d, master_target, config = self._get_test_object(task_type, task_id, - num_gpus) - - def model_fn(): - """Mnist model with synthetic input.""" - data_format = 'channels_last' - input_shape = [28, 28, 1] - l = keras.layers - max_pool = l.MaxPooling2D((2, 2), (2, 2), - padding='same', - data_format=data_format) - model = keras.Sequential([ - l.Reshape(target_shape=input_shape, input_shape=(28 * 28,)), - l.Conv2D( - 32, - 5, - padding='same', - data_format=data_format, - activation=nn.relu), max_pool, - l.Conv2D( - 64, - 5, - padding='same', - data_format=data_format, - activation=nn.relu), max_pool, - l.Flatten(), - l.Dense(1024, activation=nn.relu), - l.Dropout(0.4), - l.Dense(10) - ]) - image = random_ops.random_uniform([2, 28, 28]) - label = random_ops.random_uniform([2, 1], maxval=10, dtype=dtypes.int32) - logits = model(image, training=True) - # TODO(yuefengz): make loss a callable for eager mode. - loss = losses.sparse_softmax_cross_entropy(labels=label, logits=logits) - optimizer = adam.AdamOptimizer(learning_rate=1e-4) - train_op = optimizer.minimize(loss, - training_util.get_or_create_global_step()) - return train_op - - with ops.Graph().as_default(), \ - self.cached_session(config=config, - target=master_target) as sess: - with d.scope(): - train_op = d.extended.call_for_each_replica(model_fn) - train_op = d.group(d.experimental_local_results(train_op)) - - sess.run(variables.global_variables_initializer()) - sess.run(train_op) - - def _test_variable_initialization(self, task_type, task_id, num_gpus): - distribution, master_target, config = self._get_test_object( - task_type, task_id, num_gpus) - with ops.Graph().as_default(), \ - self.cached_session(config=config, - target=master_target) as sess, \ - distribution.scope(): - - def model_fn(): - x = variable_scope.get_variable( - 'x', - shape=(2, 3), - initializer=init_ops.random_uniform_initializer( - 1.0, 10.0, dtype=dtypes.float32)) - return array_ops.identity(x) - - x = distribution.extended.call_for_each_replica(model_fn) - reduced_x = distribution.reduce(reduce_util.ReduceOp.MEAN, x, axis=None) - x = distribution.experimental_local_results(x)[0] - - sess.run(variables.global_variables_initializer()) - - x_value, reduced_x_value = sess.run([x, reduced_x]) - self.assertTrue( - np.allclose(x_value, reduced_x_value, atol=1e-5), - msg=('x_value = %r, reduced_x_value = %r' % (x_value, - reduced_x_value))) - - def _test_input_fn_iterator(self, - task_type, - task_id, - num_gpus, - input_fn, - expected_values, - test_reinitialize=True, - ignore_order=False): - distribution, master_target, config = self._get_test_object( - task_type, task_id, num_gpus) - devices = distribution.extended.worker_devices - - with ops.Graph().as_default(), \ - self.cached_session(config=config, - target=master_target) as sess: - iterator = distribution.make_input_fn_iterator(input_fn) - sess.run(iterator.initialize()) - - for expected_value in expected_values: - next_element = iterator.get_next() - computed_value = sess.run([values.select_replica(r, next_element) - for r in range(len(devices))]) - if ignore_order: - self.assertCountEqual(expected_value, computed_value) - else: - self.assertEqual(expected_value, computed_value) - - with self.assertRaises(errors.OutOfRangeError): - next_element = iterator.get_next() - sess.run([values.select_replica(r, next_element) - for r in range(len(devices))]) - - # After re-initializing the iterator, should be able to iterate again. - if test_reinitialize: - sess.run(iterator.initialize()) - - for expected_value in expected_values: - next_element = iterator.get_next() - computed_value = sess.run([values.select_replica(r, next_element) - for r in range(len(devices))]) - if ignore_order: - self.assertCountEqual(expected_value, computed_value) - else: - self.assertEqual(expected_value, computed_value) - - -class DistributedCollectiveAllReduceStrategyTest( - CollectiveAllReduceStrategyTestBase, - strategy_test_lib.DistributionTestBase, - parameterized.TestCase): - - @classmethod - def setUpClass(cls): - """Create a local cluster with 3 workers.""" - cls._cluster_spec = multi_worker_test_base.create_in_process_cluster( - num_workers=3, num_ps=0) - - @combinations.generate(combinations.combine(mode=['graph'])) - def test_num_replicas_in_sync(self): - distribution, _, _ = create_test_objects( - cluster_spec=self._cluster_spec, - task_type='worker', - task_id=0, - num_gpus=2) - num_workers = len(self._cluster_spec.get('chief', []) + - self._cluster_spec.get('worker', [])) - self.assertEqual(2 * num_workers, - distribution.num_replicas_in_sync) - - @combinations.generate( - combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1)) - def testMinimizeLossGraph(self, num_gpus): - self._run_between_graph_clients(self._test_minimize_loss_graph, - self._cluster_spec, num_gpus) - - @combinations.generate( - combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1)) - def testVariableInitialization(self, num_gpus): - if context.num_gpus() < num_gpus: - self.skipTest('Not enough GPUs') - self._run_between_graph_clients( - self._test_variable_initialization, - self._cluster_spec, - num_gpus=num_gpus) - - @combinations.generate( - combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1)) - def testComplexModel(self, num_gpus): - if context.num_gpus() < num_gpus: - self.skipTest('Not enough GPUs') - self._run_between_graph_clients( - self._test_complex_model, self._cluster_spec, num_gpus=num_gpus) - - # TODO(yuefengz): Update how we use num_gpus and required_gpus - @combinations.generate( - combinations.combine( - mode=['graph'], - num_gpus=[0, 1, 2], - required_gpus=1, - use_dataset=[True, False])) - def testMakeInputFnIterator(self, num_gpus, use_dataset): - if context.num_gpus() < num_gpus: - self.skipTest('Not enough GPUs') - if use_dataset: - fn = lambda: dataset_ops.Dataset.range(100) - else: - def fn(): - dataset = dataset_ops.Dataset.range(100) - it = dataset.make_one_shot_iterator() - return it.get_next - # We use CPU as the device when num_gpus = 0 - devices_per_worker = max(1, num_gpus) - expected_values = [[i+j for j in range(devices_per_worker)] - for i in range(0, 100, devices_per_worker)] - - input_fn = self._input_fn_to_test_input_context( - fn, - expected_num_replicas_in_sync=3*devices_per_worker, - expected_num_input_pipelines=3, - expected_input_pipeline_id=1) # because task_id = 1 - self._test_input_fn_iterator( - 'worker', - 1, - num_gpus, - input_fn, - expected_values, - test_reinitialize=use_dataset, - ignore_order=not use_dataset) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testUpdateConfigProto(self): - strategy, _, _ = self._get_test_object( - task_type='worker', task_id=1, num_gpus=2) - - config_proto = config_pb2.ConfigProto(device_filters=['to_be_overridden']) - rewrite_options = config_proto.graph_options.rewrite_options - rewrite_options.scoped_allocator_opts.enable_op.append('to_be_removed') - - new_config = strategy.update_config_proto(config_proto) - - # Verify group leader - self.assertEqual('/job:worker/replica:0/task:0', - new_config.experimental.collective_group_leader) - - # Verify device filters. - self.assertEqual(['/job:worker/task:1'], new_config.device_filters) - - # Verify rewrite options. - new_rewrite_options = new_config.graph_options.rewrite_options - self.assertEqual(rewriter_config_pb2.RewriterConfig.ON, - new_rewrite_options.scoped_allocator_optimization) - self.assertEqual(['CollectiveReduce'], - new_rewrite_options.scoped_allocator_opts.enable_op) - - -class DistributedCollectiveAllReduceStrategyTestWithChief( - CollectiveAllReduceStrategyTestBase, parameterized.TestCase): - - @classmethod - def setUpClass(cls): - """Create a local cluster with 3 workers and 1 chief.""" - cls._cluster_spec = multi_worker_test_base.create_in_process_cluster( - num_workers=3, num_ps=0, has_chief=True) - - @combinations.generate( - combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1)) - def testMinimizeLossGraph(self, num_gpus): - self._run_between_graph_clients(self._test_minimize_loss_graph, - self._cluster_spec, num_gpus) - - @combinations.generate( - combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1)) - def testVariableInitialization(self, num_gpus): - if context.num_gpus() < num_gpus: - return - self._run_between_graph_clients( - self._test_variable_initialization, - self._cluster_spec, - num_gpus=num_gpus) - - @combinations.generate( - combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1)) - def testComplexModel(self, num_gpus): - if context.num_gpus() < num_gpus: - return - self._run_between_graph_clients( - self._test_complex_model, self._cluster_spec, num_gpus=num_gpus) - - -class LocalCollectiveAllReduceStrategy( - CollectiveAllReduceStrategyTestBase, - strategy_test_lib.DistributionTestBase, - strategy_test_lib.TwoDeviceDistributionTestBase, - parameterized.TestCase): - - @combinations.generate( - combinations.combine( - mode=['graph', 'eager'], num_gpus=[2, 4], required_gpus=2)) - def testMinimizeLoss(self, num_gpus): - # Collective ops doesn't support strategy with one device. - if context.num_gpus() < num_gpus: - self.skipTest('Not enough GPUs') - if context.executing_eagerly(): - strategy, _, _ = self._get_test_object(None, None, num_gpus) - self._test_minimize_loss_eager(strategy) - else: - self._test_minimize_loss_graph(None, None, num_gpus) - - @combinations.generate( - combinations.combine(mode=['graph'], num_gpus=[2, 4], required_gpus=2)) - def testComplexModel(self, num_gpus): - if context.num_gpus() < num_gpus: - self.skipTest('Not enough GPUs') - self._test_complex_model(None, None, num_gpus) - - @combinations.generate( - combinations.combine( - mode=['graph', 'eager'], required_gpus=2, use_dataset=[True, False])) - def testMakeInputFnIterator(self, use_dataset): - num_gpus = 2 - if use_dataset: - fn = lambda: dataset_ops.Dataset.range(5 * num_gpus) - else: - def fn(): - dataset = dataset_ops.Dataset.range(5 * num_gpus) - it = dataset.make_one_shot_iterator() - return it.get_next - expected_values = [range(i, i + num_gpus) for i in range(0, 10, num_gpus)] - - input_fn = self._input_fn_to_test_input_context( - fn, - expected_num_replicas_in_sync=num_gpus, - expected_num_input_pipelines=1, - expected_input_pipeline_id=0) - self._test_input_fn_iterator( - None, - None, - num_gpus, - input_fn, - expected_values, - test_reinitialize=use_dataset, - ignore_order=not use_dataset) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testAllReduceSum(self): - if context.num_gpus() < 2: self.skipTest('Not enough GPUs') - distribution, target, config = self._get_test_object(None, None, num_gpus=2) - with self.cached_session(config=config, target=target): - self._test_all_reduce_sum(distribution) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testAllReduceSumGradients(self): - if context.num_gpus() < 2: self.skipTest('Not enough GPUs') - distribution, target, config = self._get_test_object(None, None, num_gpus=2) - with self.cached_session(config=config, target=target): - self._test_all_reduce_sum_gradients(distribution) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testAllReduceSumGradientTape(self): - if context.num_gpus() < 2: self.skipTest('Not enough GPUs') - distribution, target, config = self._get_test_object(None, None, num_gpus=2) - with self.cached_session(config=config, target=target): - self._test_all_reduce_sum_gradient_tape(distribution) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testAllReduceMean(self): - if context.num_gpus() < 2: self.skipTest('Not enough GPUs') - distribution, target, config = self._get_test_object(None, None, num_gpus=2) - with self.cached_session(config=config, target=target): - self._test_all_reduce_mean(distribution) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testAllReduceMeanGradients(self): - if context.num_gpus() < 2: self.skipTest('Not enough GPUs') - distribution, target, config = self._get_test_object(None, None, num_gpus=2) - with self.cached_session(config=config, target=target): - self._test_all_reduce_mean_gradients(distribution) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testAllReduceMeanGradientTape(self): - if context.num_gpus() < 2: self.skipTest('Not enough GPUs') - distribution, target, config = self._get_test_object(None, None, num_gpus=2) - with self.cached_session(config=config, target=target): - self._test_all_reduce_mean_gradient_tape(distribution) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testNumpyIterator(self): - num_gpus = 2 - if context.num_gpus() < num_gpus: - self.skipTest('Not enough GPUs') - strategy, _, _ = self._get_test_object(None, None, num_gpus=num_gpus) - self._test_numpy_iterator(strategy) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/distribute/python/contrib_mirrored_strategy_test.py b/tensorflow/contrib/distribute/python/contrib_mirrored_strategy_test.py deleted file mode 100644 index 8642e0cf4e9..00000000000 --- a/tensorflow/contrib/distribute/python/contrib_mirrored_strategy_test.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests the contrib MirroredStrategy specific features.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy as np - -from tensorflow.contrib.distribute.python import mirrored_strategy -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import values -from tensorflow.python.eager import context -from tensorflow.python.eager import test - -contrib_mirrored_strategies = [ - combinations.NamedDistribution( - "ContribMirrored1CPU", - lambda: mirrored_strategy.MirroredStrategy(["/cpu:0"])), - combinations.NamedDistribution( - "ContribMirrored1GPU", - lambda: mirrored_strategy.MirroredStrategy(["/gpu:0"]), - required_gpus=1), - combinations.NamedDistribution( - "ContribMirroredCPUAndGPU", - lambda: mirrored_strategy.MirroredStrategy(["/cpu:0", "/gpu:0"]), - required_gpus=1), - combinations.NamedDistribution( - "ContribMirrored2GPU", - lambda: mirrored_strategy.MirroredStrategy(["/gpu:0", "/gpu:1"]), - required_gpus=2), -] - - -def all_strategy_and_eager_plus_graph(): - return combinations.times( - combinations.combine(distribution=contrib_mirrored_strategies), - combinations.combine(mode=["eager", "graph"])) - - -class ContribMirroredStrategyTest(test.TestCase, parameterized.TestCase): - - def _initialize_and_evaluate_iterator(self, iterator): - if context.executing_eagerly(): - iterator.initialize() - res = iterator.get_next() - if isinstance(res, values.PerReplica): - res = res.values - else: - with self.cached_session() as sess: - sess.run(iterator.initialize()) - res = iterator.get_next() - if isinstance(res, values.PerReplica): - res = sess.run(res.values) - else: - res = sess.run(res) - - return res - - @combinations.generate(all_strategy_and_eager_plus_graph()) - def test_dataset_iterator(self, distribution): - data = np.array([[1, 1], [2, 1], [3, 1], [4, 1]]) - dataset = dataset_ops.Dataset.from_tensors(data).repeat() - iterator = distribution.make_dataset_iterator(dataset) - res = self._initialize_and_evaluate_iterator(iterator) - - if isinstance(res, tuple): - self.assertLen(res, 2) - self.assertAllEqual(data, res[0]) - self.assertAllEqual(data, res[1]) - else: - self.assertAllEqual(data, res) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distribute/python/examples/BUILD b/tensorflow/contrib/distribute/python/examples/BUILD deleted file mode 100644 index afabba7bfb4..00000000000 --- a/tensorflow/contrib/distribute/python/examples/BUILD +++ /dev/null @@ -1,73 +0,0 @@ -# Example TensorFlow models that use DistributionStrategy for training. - -package( - default_visibility = [ - "//tensorflow:internal", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_binary( - name = "simple_estimator_example", - srcs = ["simple_estimator_example.py"], - python_version = "PY2", - deps = [ - "//tensorflow:tensorflow_py", - ], -) - -py_binary( - name = "keras_model_with_estimator", - srcs = [ - "keras_model_with_estimator.py", - ], - python_version = "PY2", - deps = [ - "//tensorflow:tensorflow_py", - "//third_party/py/numpy", - ], -) - -py_binary( - name = "keras_mnist", - srcs = ["keras_mnist.py"], - python_version = "PY2", - deps = [":keras_mnist_lib"], -) - -py_library( - name = "keras_mnist_lib", - srcs = [ - "keras_mnist.py", - ], - deps = [ - "//tensorflow:tensorflow_py", - "//third_party/py/numpy", - ], -) - -py_binary( - name = "mnist_eager_multigpu", - srcs = [ - "mnist_eager_multigpu.py", - ], - python_version = "PY2", - deps = [ - "//tensorflow:tensorflow_py", - "//third_party/py/numpy", - ], -) - -py_binary( - name = "mnist_tf1_tpu", - srcs = [ - "mnist_tf1_tpu.py", - ], - python_version = "PY2", - deps = [ - "//tensorflow:tensorflow_py", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/distribute/python/examples/keras_mnist.py b/tensorflow/contrib/distribute/python/examples/keras_mnist.py deleted file mode 100644 index 1ce91ecaf22..00000000000 --- a/tensorflow/contrib/distribute/python/examples/keras_mnist.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""An example training a Keras Model using MirroredStrategy and native APIs.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - - -from tensorflow.python.distribute import mirrored_strategy -from tensorflow.python.keras.optimizer_v2 import rmsprop - - -NUM_CLASSES = 10 - - -def get_input_datasets(use_bfloat16=False): - """Downloads the MNIST dataset and creates train and eval dataset objects. - - Args: - use_bfloat16: Boolean to determine if input should be cast to bfloat16 - - Returns: - Train dataset, eval dataset and input shape. - - """ - # input image dimensions - img_rows, img_cols = 28, 28 - cast_dtype = tf.bfloat16 if use_bfloat16 else tf.float32 - - # the data, split between train and test sets - (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() - - if tf.keras.backend.image_data_format() == 'channels_first': - x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) - x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) - input_shape = (1, img_rows, img_cols) - else: - x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) - x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) - input_shape = (img_rows, img_cols, 1) - - x_train = x_train.astype('float32') - x_test = x_test.astype('float32') - x_train /= 255 - x_test /= 255 - - # convert class vectors to binary class matrices - y_train = tf.keras.utils.to_categorical(y_train, NUM_CLASSES) - y_test = tf.keras.utils.to_categorical(y_test, NUM_CLASSES) - - # train dataset - train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train)) - train_ds = train_ds.repeat() - train_ds = train_ds.map(lambda x, y: (tf.cast(x, cast_dtype), y)) - train_ds = train_ds.batch(64, drop_remainder=True) - - # eval dataset - eval_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)) - eval_ds = eval_ds.repeat() - eval_ds = eval_ds.map(lambda x, y: (tf.cast(x, cast_dtype), y)) - eval_ds = eval_ds.batch(64, drop_remainder=True) - - return train_ds, eval_ds, input_shape - - -def get_model(input_shape): - """Builds a Sequential CNN model to recognize MNIST digits. - - Args: - input_shape: Shape of the input depending on the `image_data_format`. - - Returns: - a Keras model - - """ - # Define a CNN model to recognize MNIST digits. - model = tf.keras.models.Sequential() - model.add(tf.keras.layers.Conv2D(32, kernel_size=(3, 3), - activation='relu', - input_shape=input_shape)) - model.add(tf.keras.layers.Conv2D(64, (3, 3), activation='relu')) - model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2))) - model.add(tf.keras.layers.Dropout(0.25)) - model.add(tf.keras.layers.Flatten()) - model.add(tf.keras.layers.Dense(128, activation='relu')) - model.add(tf.keras.layers.Dropout(0.5)) - model.add(tf.keras.layers.Dense(NUM_CLASSES, activation='softmax')) - return model - - -def main(_): - # Build the train and eval datasets from the MNIST data. Also return the - # input shape which is constructed based on the `image_data_format` - # i.e channels_first or channels_last. - tf.enable_eager_execution() - - train_ds, eval_ds, input_shape = get_input_datasets() - - # Instantiate the MirroredStrategy object. If we don't specify `num_gpus` or - # the `devices` argument then all the GPUs available on the machine are used. - # TODO(priyag): Use `tf.distribute.MirroredStrategy` once available. - strategy = mirrored_strategy.MirroredStrategy(['/gpu:0', '/cpu:0']) - - # Create and compile the model under Distribution strategy scope. - # `fit`, `evaluate` and `predict` will be distributed based on the strategy - # model was compiled with. - with strategy.scope(): - model = get_model(input_shape) - optimizer = rmsprop.RMSProp(learning_rate=0.001) - model.compile(loss=tf.keras.losses.categorical_crossentropy, - optimizer=optimizer, - metrics=['accuracy']) - - # Train the model with the train dataset. - model.fit(x=train_ds, epochs=20, steps_per_epoch=468) - - # Evaluate the model with the eval dataset. - score = model.evaluate(eval_ds, steps=10, verbose=0) - print('Test loss:', score[0]) - print('Test accuracy:', score[1]) - - -if __name__ == '__main__': - tf.app.run() diff --git a/tensorflow/contrib/distribute/python/examples/keras_model_with_estimator.py b/tensorflow/contrib/distribute/python/examples/keras_model_with_estimator.py deleted file mode 100644 index 8d117eb7e8f..00000000000 --- a/tensorflow/contrib/distribute/python/examples/keras_model_with_estimator.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""An example of training tf.keras Model using MirroredStrategy.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import sys - -import numpy as np -import tensorflow as tf - - -def input_fn(): - x = np.random.random((1024, 10)) - y = np.random.randint(2, size=(1024, 1)) - x = tf.cast(x, tf.float32) - dataset = tf.data.Dataset.from_tensor_slices((x, y)) - dataset = dataset.repeat(10) - dataset = dataset.batch(32) - return dataset - - -def main(args): - if len(args) < 2: - print('You must specify model_dir for checkpoints such as' - ' /tmp/tfkeras_example/.') - return - - model_dir = args[1] - print('Using %s to store checkpoints.' % model_dir) - - # Define a Keras Model. - model = tf.keras.Sequential() - model.add(tf.keras.layers.Dense(16, activation='relu', input_shape=(10,))) - model.add(tf.keras.layers.Dense(1, activation='sigmoid')) - - # Compile the model. - optimizer = tf.train.GradientDescentOptimizer(0.2) - model.compile(loss='binary_crossentropy', optimizer=optimizer) - model.summary() - tf.keras.backend.set_learning_phase(True) - - # Define a DistributionStrategy and convert the Keras Model to an - # Estimator that utilizes the DistributionStrategy. - strategy = tf.contrib.distribute.MirroredStrategy( - ['/device:GPU:0', '/device:GPU:1']) - config = tf.estimator.RunConfig( - train_distribute=strategy, eval_distribute=strategy) - keras_estimator = tf.keras.estimator.model_to_estimator( - keras_model=model, config=config, model_dir=model_dir) - - # Train and evaluate the model. - keras_estimator.train(input_fn=input_fn, steps=10) - eval_result = keras_estimator.evaluate(input_fn=input_fn) - print('Eval result: {}'.format(eval_result)) - - -if __name__ == '__main__': - tf.app.run(argv=sys.argv) diff --git a/tensorflow/contrib/distribute/python/examples/mnist_eager_multigpu.py b/tensorflow/contrib/distribute/python/examples/mnist_eager_multigpu.py deleted file mode 100644 index 502f94c5728..00000000000 --- a/tensorflow/contrib/distribute/python/examples/mnist_eager_multigpu.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Run MNIST on multiple GPUs on using MirroredStrategy with eager execution. - -By default, runs on all available GPUs, or CPU if no GPUs are available. - -NOTE: Currently, this takes more time than when running MNIST in eager without -MirroredStrategy because of a number overheads. Therefore, this is just a -proof of concept right now and cannot be used to actually scale up training. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl import app -from absl import flags -import numpy as np -import tensorflow.compat.v2 as tf - -flags.DEFINE_integer("num_gpus", None, "How many GPUs should we run on?" - "Defaults to all available GPUs, otherwise CPU.") -flags.DEFINE_integer("batch_size", 64, - "What should be the size of each batch?") -flags.DEFINE_integer("num_epochs", 10, "How many epochs to run?") -flags.DEFINE_float("learning_rate", 0.01, "Learning Rate") -flags.DEFINE_float("momentum", 0.5, "SGD momentum") - -FLAGS = flags.FLAGS -NUM_TRAIN_IMAGES = 60000 - - -def create_model(): - max_pool = tf.keras.layers.MaxPooling2D((2, 2), (2, 2), padding="same") - # The model consists of a sequential chain of layers, so tf.keras.Sequential - # (a subclass of tf.keras.Model) makes for a compact description. - return tf.keras.Sequential([ - tf.keras.layers.Reshape( - target_shape=[28, 28, 1], - input_shape=(28, 28,)), - tf.keras.layers.Conv2D(2, 5, padding="same", activation=tf.nn.relu), - max_pool, - tf.keras.layers.Conv2D(4, 5, padding="same", activation=tf.nn.relu), - max_pool, - tf.keras.layers.Flatten(), - tf.keras.layers.Dense(32, activation=tf.nn.relu), - tf.keras.layers.Dropout(0.4), - tf.keras.layers.Dense(10)]) - - -def compute_loss(logits, labels): - loss = tf.reduce_sum( - tf.nn.sparse_softmax_cross_entropy_with_logits( - logits=logits, labels=labels)) - # Scale loss by global batch size. - return loss * (1. / FLAGS.batch_size) - - -def mnist_datasets(strategy): - (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() - # Numpy defaults to dtype=float64; TF defaults to float32. Stick with float32. - x_train, x_test = x_train / np.float32(255), x_test / np.float32(255) - y_train, y_test = y_train.astype(np.int64), y_test.astype(np.int64) - train_dataset = strategy.experimental_make_numpy_dataset((x_train, y_train)) - test_dataset = strategy.experimental_make_numpy_dataset((x_test, y_test)) - return train_dataset, test_dataset - - -def main(unused_argv): - """Run a CNN model on MNIST data to demonstrate DistributedStrategies.""" - - tf.enable_v2_behavior() - - num_gpus = FLAGS.num_gpus - if num_gpus is None: - devices = None - elif num_gpus == 0: - devices = ["/device:CPU:0"] - else: - devices = ["/device:GPU:{}".format(i) for i in range(num_gpus)] - strategy = tf.distribute.MirroredStrategy(devices) - - with strategy.scope(): - train_ds, test_ds = mnist_datasets(strategy) - train_ds = train_ds.shuffle(NUM_TRAIN_IMAGES).batch(FLAGS.batch_size) - test_ds = test_ds.batch(FLAGS.batch_size) - - model = create_model() - optimizer = tf.keras.optimizers.SGD(FLAGS.learning_rate, FLAGS.momentum) - training_loss = tf.keras.metrics.Mean("training_loss", dtype=tf.float32) - training_accuracy = tf.keras.metrics.SparseCategoricalAccuracy( - "training_accuracy", dtype=tf.float32) - test_loss = tf.keras.metrics.Mean("test_loss", dtype=tf.float32) - test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy( - "test_accuracy", dtype=tf.float32) - - @tf.function - def train_epoch(train_dist_dataset): - """Training Step.""" - def step_fn(images, labels): - with tf.GradientTape() as tape: - logits = model(images, training=True) - loss = compute_loss(logits, labels) - grads = tape.gradient(loss, model.variables) - optimizer.apply_gradients(zip(grads, model.variables)) - training_loss.update_state(loss) - training_accuracy.update_state(labels, logits) - - for images, labels in train_dist_dataset: - strategy.experimental_run_v2(step_fn, args=(images, labels)) - - @tf.function - def test_epoch(test_dist_dataset): - """Testing Step.""" - def step_fn(images, labels): - logits = model(images, training=False) - loss = compute_loss(logits, labels) - test_loss.update_state(loss) - test_accuracy.update_state(labels, logits) - - for images, labels in test_dist_dataset: - strategy.experimental_run_v2(step_fn, args=(images, labels)) - - train_dist_dataset = strategy.experimental_distribute_dataset(train_ds) - test_dist_dataset = strategy.experimental_distribute_dataset(test_ds) - - for epoch in range(FLAGS.num_epochs): - # Train - print("Starting epoch {}".format(epoch)) - train_epoch(train_dist_dataset) - print("Training loss: {:0.4f}, accuracy: {:0.2f}%".format( - training_loss.result(), training_accuracy.result() * 100)) - training_loss.reset_states() - training_accuracy.reset_states() - - # Test - test_epoch(test_dist_dataset) - print("Test loss: {:0.4f}, accuracy: {:0.2f}%".format( - test_loss.result(), test_accuracy.result() * 100)) - test_loss.reset_states() - test_accuracy.reset_states() - - -if __name__ == "__main__": - app.run(main) diff --git a/tensorflow/contrib/distribute/python/examples/mnist_tf1_tpu.py b/tensorflow/contrib/distribute/python/examples/mnist_tf1_tpu.py deleted file mode 100644 index 9845f8a5a1d..00000000000 --- a/tensorflow/contrib/distribute/python/examples/mnist_tf1_tpu.py +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Run MNIST on multiple GPUs on using MirroredStrategy with eager execution. - -By default, runs on all available GPUs, or CPU if no GPUs are available. - -NOTE: Currently, this takes more time than when running MNIST in eager without -MirroredStrategy because of a number overheads. Therefore, this is just a -proof of concept right now and cannot be used to actually scale up training. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl import app -from absl import flags -import numpy as np -import tensorflow as tf - - -flags.DEFINE_string("tpu", None, "Name of the TPU to use.") -flags.DEFINE_integer("batch_size", 64, - "What should be the size of each batch?") -flags.DEFINE_integer("num_epochs", 10, "How many epochs to run?") -flags.DEFINE_float("learning_rate", 0.01, "Learning Rate") -flags.DEFINE_float("momentum", 0.5, "SGD momentum") - -FLAGS = flags.FLAGS -NUM_TRAIN_IMAGES = 60000 - - -def create_model(): - max_pool = tf.keras.layers.MaxPooling2D((2, 2), (2, 2), padding="same") - # The model consists of a sequential chain of layers, so tf.keras.Sequential - # (a subclass of tf.keras.Model) makes for a compact description. - return tf.keras.Sequential([ - tf.keras.layers.Reshape( - target_shape=[28, 28, 1], - input_shape=(28, 28,)), - tf.keras.layers.Conv2D(2, 5, padding="same", activation=tf.nn.relu), - max_pool, - tf.keras.layers.Conv2D(4, 5, padding="same", activation=tf.nn.relu), - max_pool, - tf.keras.layers.Flatten(), - tf.keras.layers.Dense(32, activation=tf.nn.relu), - tf.keras.layers.Dropout(0.4), - tf.keras.layers.Dense(10)]) - - -def compute_loss(logits, labels): - loss = tf.reduce_sum( - tf.nn.sparse_softmax_cross_entropy_with_logits( - logits=logits, labels=labels)) - # Scale loss by global batch size. - return loss * (1. / FLAGS.batch_size) - - -def mnist_datasets(): - (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() - # Numpy defaults to dtype=float64; TF defaults to float32. Stick with float32. - x_train, x_test = x_train / np.float32(255), x_test / np.float32(255) - y_train, y_test = y_train.astype(np.int64), y_test.astype(np.int64) - # TODO(priyag): `strategy.make_numpy_iterator` can be used directly instead of - # converting to datasets. - train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train)) - test_dataset = tf.data.Dataset.from_tensor_slices((x_test, y_test)) - return train_dataset, test_dataset - - -def main(argv): - """Run a CNN model on MNIST data to demonstrate DistributedStrategies.""" - del argv # Unused. - tf.disable_v2_behavior() - - cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( - tpu=FLAGS.tpu) - strategy = tf.contrib.distribute.TPUStrategy(cluster_resolver) - - # TODO(rxsang): This doesn't work on cloud for some reason - strategy.extended.experimental_enable_get_next_as_optional = False - - with strategy.scope(): - train_ds, test_ds = mnist_datasets() - train_ds = train_ds.shuffle(NUM_TRAIN_IMAGES).batch(FLAGS.batch_size) - test_ds = test_ds.batch(FLAGS.batch_size) - - model = create_model() - optimizer = tf.keras.optimizers.SGD(FLAGS.learning_rate, FLAGS.momentum) - training_loss = tf.keras.metrics.Mean("training_loss", dtype=tf.float32) - training_accuracy = tf.keras.metrics.SparseCategoricalAccuracy( - "training_accuracy", dtype=tf.float32) - test_loss = tf.keras.metrics.Mean("test_loss", dtype=tf.float32) - test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy( - "test_accuracy", dtype=tf.float32) - - def train_step(inputs): # pylint: disable=missing-docstring - images, labels = inputs - with tf.GradientTape() as tape: - logits = model(images, training=True) - loss = compute_loss(logits, labels) - grads = tape.gradient(loss, model.variables) - update_vars = optimizer.apply_gradients(zip(grads, model.variables)) - update_loss = training_loss.update_state(loss) - update_accuracy = training_accuracy.update_state(labels, logits) - - with tf.control_dependencies([update_vars, update_loss, update_accuracy]): - return tf.identity(loss) - - def test_step(inputs): - images, labels = inputs - logits = model(images, training=False) - loss = compute_loss(logits, labels) - update_loss = test_loss.update_state(loss) - update_accuracy = test_accuracy.update_state(labels, logits) - - with tf.control_dependencies([update_loss, update_accuracy]): - return tf.identity(loss) - - train_iterator = strategy.experimental_distribute_dataset( - train_ds).make_initializable_iterator() - test_iterator = strategy.experimental_distribute_dataset( - test_ds).make_initializable_iterator() - - dist_train = strategy.experimental_local_results( - strategy.experimental_run_v2(train_step, args=(next(train_iterator),))) - dist_test = strategy.experimental_local_results( - strategy.experimental_run_v2(test_step, args=(next(test_iterator),))) - - training_loss_result = training_loss.result() - training_accuracy_result = training_accuracy.result() - test_loss_result = test_loss.result() - test_accuracy_result = test_accuracy.result() - - tf.contrib.distribute.initialize_tpu_system(cluster_resolver) - - train_iterator_init = train_iterator.initialize() - test_iterator_init = test_iterator.initialize() - - all_variables = ( - tf.global_variables() + - training_loss.variables + - training_accuracy.variables + - test_loss.variables + - test_accuracy.variables) - - config = tf.ConfigProto() - cluster_spec = cluster_resolver.cluster_spec() - if cluster_spec: - config.cluster_def.CopyFrom(cluster_spec.as_cluster_def()) - - with tf.Session(cluster_resolver.master(), config=config) as session: - session.run([v.initializer for v in all_variables]) - - for epoch in range(0, FLAGS.num_epochs): - # Train - print("Starting epoch {}".format(epoch)) - session.run(train_iterator_init) - while True: - try: - session.run(dist_train) - except tf.errors.OutOfRangeError: - break - print("Training loss: {:0.4f}, accuracy: {:0.2f}%".format( - session.run(training_loss_result), - session.run(training_accuracy_result) * 100)) - training_loss.reset_states() - training_accuracy.reset_states() - - # Test - session.run(test_iterator_init) - while True: - try: - session.run(dist_test) - except tf.errors.OutOfRangeError: - break - print("Test loss: {:0.4f}, accuracy: {:0.2f}%".format( - session.run(test_loss_result), - session.run(test_accuracy_result) * 100)) - test_loss.reset_states() - test_accuracy.reset_states() - - -if __name__ == "__main__": - flags.mark_flag_as_required("tpu") - app.run(main) diff --git a/tensorflow/contrib/distribute/python/examples/simple_estimator_example.py b/tensorflow/contrib/distribute/python/examples/simple_estimator_example.py deleted file mode 100644 index cfaee03a200..00000000000 --- a/tensorflow/contrib/distribute/python/examples/simple_estimator_example.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A simple example to test the a DistributionStrategy with Estimators. - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -from tensorflow.python.keras import metrics as metrics_module - - -def build_model_fn_optimizer(): - """Simple model_fn with optimizer.""" - # TODO(anjalisridhar): Move this inside the model_fn once OptimizerV2 is - # done? - optimizer = tf.train.GradientDescentOptimizer(0.2) - - def model_fn(features, labels, mode): # pylint: disable=unused-argument - """model_fn which uses a single unit Dense layer.""" - # You can also use the Flatten layer if you want to test a model without any - # weights. - layer = tf.layers.Dense(1, use_bias=True) - logits = layer(features) - - if mode == tf.estimator.ModeKeys.PREDICT: - predictions = {"logits": logits} - return tf.estimator.EstimatorSpec(mode, predictions=predictions) - - def loss_fn(): - y = tf.reshape(logits, []) - tf.constant(1.) - return y * y - - if mode == tf.estimator.ModeKeys.EVAL: - acc_obj = metrics_module.BinaryAccuracy() - acc_obj.update_state(labels, labels) - return tf.estimator.EstimatorSpec( - mode, loss=loss_fn(), eval_metric_ops={"Accuracy": acc_obj}) - - assert mode == tf.estimator.ModeKeys.TRAIN - - global_step = tf.train.get_global_step() - train_op = optimizer.minimize(loss_fn(), global_step=global_step) - return tf.estimator.EstimatorSpec(mode, loss=loss_fn(), train_op=train_op) - - return model_fn - - -def main(_): - distribution = tf.contrib.distribute.MirroredStrategy( - ["/device:GPU:0", "/device:GPU:1"]) - config = tf.estimator.RunConfig(train_distribute=distribution, - eval_distribute=distribution) - # Since there are 2 devices and 10 samples, we set steps=5. - steps = 5 - - def train_input_fn(): - features = tf.data.Dataset.from_tensors([[1.]]).repeat(10) - labels = tf.data.Dataset.from_tensors([1.]).repeat(10) - return tf.data.Dataset.zip((features, labels)) - - estimator = tf.estimator.Estimator( - model_fn=build_model_fn_optimizer(), config=config) - estimator.train(input_fn=train_input_fn, steps=steps) - - def eval_input_fn(): - features = tf.data.Dataset.from_tensors([[1.]]).repeat(10) - labels = tf.data.Dataset.from_tensors([1.]).repeat(10) - return tf.data.Dataset.zip((features, labels)) - - eval_result = estimator.evaluate(input_fn=eval_input_fn, steps=steps) - print("Eval result: {}".format(eval_result)) - assert eval_result["Accuracy"] == 1.0 - - def predict_input_fn(): - predict_features = tf.data.Dataset.from_tensors([[1.]]).repeat(10) - return predict_features - - prediction_iterable = estimator.predict(input_fn=predict_input_fn) - # Create a list containing each of the prediction dictionaries that map - # the key 'logits' to an array of model outputs. - predictions = [prediction_iterable.next() for _ in range(10)] - print("Prediction results: {}".format(predictions)) - - -if __name__ == "__main__": - tf.app.run() diff --git a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py b/tensorflow/contrib/distribute/python/keras_backward_compat_test.py deleted file mode 100644 index 98195cca3c3..00000000000 --- a/tensorflow/contrib/distribute/python/keras_backward_compat_test.py +++ /dev/null @@ -1,1156 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.keras models using DistributionStrategy.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy as np -from tensorflow.contrib.distribute.python import mirrored_strategy -from tensorflow.python import keras -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import distribute_lib -from tensorflow.python.distribute import strategy_combinations -from tensorflow.python.distribute import tpu_strategy -from tensorflow.python.eager import test -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import test_util -from tensorflow.python.keras import testing_utils -from tensorflow.python.keras.distribute import distributed_training_utils -from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras -from tensorflow.python.keras.utils.mode_keys import ModeKeys -from tensorflow.python.ops.parsing_ops import gen_parsing_ops -from tensorflow.python.training import gradient_descent -from tensorflow.python.training import rmsprop - -_RANDOM_SEED = 1337 -_TRAIN_SIZE = 200 -_INPUT_SIZE = (10,) -_NUM_CLASS = 2 - - -# TODO(anjalisridhar): Add a decorator that will allow us to run these tests as -# part of the tf.keras unit tests suite. -def simple_sequential_model(): - model = keras.models.Sequential() - model.add(keras.layers.Dense(16, activation='relu', input_shape=_INPUT_SIZE)) - model.add(keras.layers.Dropout(0.1)) - model.add(keras.layers.Dense(_NUM_CLASS, activation='softmax')) - return model - - -def simple_functional_model(): - a = keras.layers.Input(shape=_INPUT_SIZE) - b = keras.layers.Dense(16, activation='relu')(a) - b = keras.layers.Dropout(0.1)(b) - b = keras.layers.Dense(_NUM_CLASS, activation='softmax')(b) - model = keras.models.Model(inputs=[a], outputs=[b]) - return model - - -def multi_inputs_multi_outputs_model(): - input_a = keras.layers.Input(shape=(16,), name='input_a') - input_b = keras.layers.Input(shape=(16,), name='input_b') - input_m = keras.layers.Input(shape=(8,), dtype='string', name='input_m') - dense = keras.layers.Dense(8, name='dense_1') - - interm_a = dense(input_a) - # Read m - interm_m = keras.layers.Lambda(gen_parsing_ops.string_to_number)(input_m) - interm_s = keras.layers.Lambda(lambda k: k[0] * k[1])([interm_m, interm_a]) - interm_b = dense(input_b) - merged = keras.layers.concatenate([interm_s, interm_b], name='merge') - output_c = keras.layers.Dense(3, activation='softmax', name='dense_2')(merged) - output_d = keras.layers.Dense(2, activation='softmax', name='dense_3')(merged) - model = keras.models.Model( - inputs=[input_a, input_b, input_m], outputs=[output_c, output_d]) - model.compile( - loss='categorical_crossentropy', - optimizer=gradient_descent.GradientDescentOptimizer(0.001), - metrics={ - 'dense_2': 'categorical_accuracy', - 'dense_3': 'categorical_accuracy' - }) - return model - - -def get_ds_train_input_fn(): - np.random.seed(_RANDOM_SEED) - (x_train, y_train), _ = testing_utils.get_test_data( - train_samples=_TRAIN_SIZE, - test_samples=50, - input_shape=_INPUT_SIZE, - num_classes=_NUM_CLASS) - y_train = keras.utils.to_categorical(y_train) - - dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train)) - dataset = dataset.batch(32) - return dataset - - -def get_ds_test_input_fn(): - np.random.seed(_RANDOM_SEED) - _, (x_test, y_test) = testing_utils.get_test_data( - train_samples=_TRAIN_SIZE, - test_samples=50, - input_shape=_INPUT_SIZE, - num_classes=_NUM_CLASS) - y_test = keras.utils.to_categorical(y_test) - - dataset = dataset_ops.Dataset.from_tensor_slices((x_test, y_test)) - dataset = dataset.batch(32) - return dataset - - -def get_multi_inputs_multi_outputs_data(): - (a_train, c_train), (a_test, c_test) = testing_utils.get_test_data( - train_samples=_TRAIN_SIZE, - test_samples=50, - input_shape=(16,), - num_classes=3, - random_seed=_RANDOM_SEED) - (b_train, d_train), (b_test, d_test) = testing_utils.get_test_data( - train_samples=_TRAIN_SIZE, - test_samples=50, - input_shape=(16,), - num_classes=2, - random_seed=_RANDOM_SEED) - (m_train, _), (m_test, _) = testing_utils.get_test_data( - train_samples=_TRAIN_SIZE, - test_samples=50, - input_shape=(8,), - num_classes=2, - random_seed=_RANDOM_SEED) - - c_train = keras.utils.to_categorical(c_train) - c_test = keras.utils.to_categorical(c_test) - d_train = keras.utils.to_categorical(d_train) - d_test = keras.utils.to_categorical(d_test) - - train_data = { - 'input_a': a_train, - 'input_b': b_train, - 'input_m': m_train, - 'output_c': c_train, - 'output_d': d_train - } - test_data = { - 'input_a': a_test, - 'input_b': b_test, - 'input_m': m_test, - 'output_c': c_test, - 'output_d': d_test - } - - return (train_data, test_data) - - -def batch_wrapper(dataset, batch_size, distribution, repeat=None): - if repeat: - dataset = dataset.repeat(repeat) - # TPUs currently require fully defined input shapes, drop_remainder ensures - # the input will have fully defined shapes. - if isinstance(distribution, (tpu_strategy.TPUStrategy, - tpu_strategy.TPUStrategyV1)): - return dataset.batch(batch_size, drop_remainder=True) - else: - return dataset.batch(batch_size) - - -def get_model(): - x = keras.layers.Input(shape=(3,), name='input') - y = keras.layers.Dense(4, name='dense')(x) - model = keras.Model(x, y) - return model - - -def get_dataset(distribution): - inputs = np.zeros((10, 3), dtype=np.float32) - targets = np.zeros((10, 4), dtype=np.float32) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat(100) - dataset = batch_wrapper(dataset, 10, distribution) - return dataset - - -def get_predict_dataset(distribution): - inputs = np.zeros((10, 3), dtype=np.float32) - dataset = dataset_ops.Dataset.from_tensor_slices(inputs) - dataset = dataset.repeat(100) - dataset = batch_wrapper(dataset, 10, distribution) - return dataset - - -def multi_input_output_model(): - a = keras.layers.Input(shape=(3,), name='input_a') - b = keras.layers.Input(shape=(5,), name='input_b') - # TODO(anjalisridhar): Change the output dimension of the second Dense layer - # once the iterator output validation issue has been fixed. - dense_1 = keras.layers.Dense(7, name='dense_1') - dense_2 = keras.layers.Dense(7, name='dense_2') - c = dense_1(a) - d = dense_2(b) - e = keras.layers.Dropout(0.5, name='dropout')(c) - model = keras.models.Model([a, b], [d, e]) - return model - - -def get_correctness_test_inputs(use_numpy, use_validation_data, - with_distribution, - x_train, y_train, x_predict): - """Generates the inputs for correctness check when enable Keras with DS.""" - training_epochs = 2 - global_batch_size = 64 - batch_size = global_batch_size - # TODO(b/118776054): Use global batch size for Keras/DS support. - use_per_core_batch_size = ( - with_distribution and - not distributed_training_utils.global_batch_size_supported( - with_distribution)) - if use_per_core_batch_size: - batch_size //= with_distribution.num_replicas_in_sync - - if use_numpy: - training_inputs = { - 'batch_size': batch_size, - 'x': x_train, - 'y': y_train, - 'epochs': training_epochs, - 'shuffle': False, - } - - if use_validation_data: - eval_inputs = None - training_inputs['validation_data'] = (x_train, y_train) - else: - eval_inputs = { - 'batch_size': batch_size, - 'x': x_train, - 'y': y_train, - } - predict_inputs = { - 'x': np.array(x_predict, dtype=np.float32), - } - else: - # For dataset inputs, we do not pass batch_size to - # keras.fit/evaluate/predict. The batch size is part of the dataset. - train_dataset = dataset_ops.Dataset.from_tensor_slices( - (x_train, y_train)) - x = batch_wrapper( - train_dataset, batch_size, with_distribution, repeat=training_epochs) - - training_inputs = { - 'batch_size': None, - 'x': x, - 'y': None, - 'epochs': training_epochs, - 'shuffle': False, - 'steps_per_epoch': len(x_train) // global_batch_size, - } - if use_validation_data: - eval_inputs = None # Remove the eval_inputs - eval_dataset = dataset_ops.Dataset.from_tensor_slices( - (x_train, y_train)) - x = batch_wrapper(eval_dataset, batch_size, with_distribution) - training_inputs['validation_data'] = x - training_inputs['validation_steps'] = 5 - else: - eval_inputs = { - 'batch_size': None, - 'x': x, - 'y': None, - 'steps': 20, - } - - predict_batch_size = len(x_predict) - if use_per_core_batch_size: - predict_batch_size //= with_distribution.num_replicas_in_sync - predict_dataset = dataset_ops.Dataset.from_tensor_slices(x_predict) - predict_dataset = batch_wrapper(predict_dataset, - predict_batch_size, with_distribution) - predict_inputs = { - 'steps': 1, - 'x': predict_dataset, - } - - return training_inputs, eval_inputs, predict_inputs - - -strategies_minus_tpu = [ - strategy_combinations.default_strategy, - strategy_combinations.one_device_strategy, - strategy_combinations.mirrored_strategy_with_gpu_and_cpu, - strategy_combinations.mirrored_strategy_with_two_gpus, -] - -tpu_strategies = [ - strategy_combinations.tpu_strategy, # steps_per_run=2 - strategy_combinations.tpu_strategy_one_step -] - - -def strategy_minus_tpu_combinations(): - return combinations.combine( - distribution=strategies_minus_tpu, - mode=['graph', 'eager']) - - -def tpu_strategy_combinations(): - return combinations.combine( - distribution=tpu_strategies, - mode=['graph']) - - -def all_strategy_combinations(): - return strategy_minus_tpu_combinations() + tpu_strategy_combinations() - - -def strategy_and_optimizer_combinations(): - return combinations.times( - all_strategy_combinations(), - combinations.combine(optimizer=[ - strategy_combinations.adagrad_optimizer_v1_fn, - strategy_combinations.adagrad_optimizer_keras_v2_fn, - strategy_combinations.adam_optimizer_v1_fn, - strategy_combinations.adam_optimizer_keras_v2_fn, - strategy_combinations.gradient_descent_optimizer_v1_fn, - strategy_combinations.gradient_descent_optimizer_keras_v2_fn, - strategy_combinations.rmsprop_optimizer_v1_fn, - strategy_combinations.rmsprop_optimizer_keras_v2_fn - ])) - - -def strategy_and_input_combinations(): - return ( - combinations.times( - combinations.combine(distribution=strategies_minus_tpu), - combinations.combine(mode=['graph'], - use_numpy=[True, False], - use_validation_data=[True, False]) - + combinations.combine(mode=['eager'], - use_numpy=[False], - use_validation_data=[False])) + - combinations.times( - combinations.combine(distribution=tpu_strategies), - combinations.combine(mode=['graph'], - use_numpy=[True, False], - use_validation_data=[True, False]))) - - -def strategy_for_numpy_input_combinations(): - return combinations.combine( - distribution=strategies_minus_tpu + tpu_strategies, - mode=['graph']) - - -@test_util.run_v1_only('model.compile(..distribute=..) only works in TF v1') -class TestDistributionStrategyWithNumpyArrays(test.TestCase, - parameterized.TestCase): - - @combinations.generate(strategy_for_numpy_input_combinations()) - def test_calling_model_with_numpy_arrays(self, distribution): - with self.cached_session(): - model = get_model() - - optimizer = gradient_descent.GradientDescentOptimizer(0.001) - loss = 'mse' - metrics = ['mae'] - model.compile( - optimizer, - loss, - metrics=metrics, - distribute=distribution, - experimental_run_tf_function=False) - - inputs = np.zeros((64, 3), dtype=np.float32) - targets = np.zeros((64, 4), dtype=np.float32) - - # Call fit with validation data - model.fit(inputs, targets, epochs=1, batch_size=2, verbose=0, - validation_data=(inputs, targets)) - - # TODO(anjalisridhar): We need tests for when the batch size and steps are - # smaller and results in a 0 batch_size and steps value. - model.evaluate(inputs, targets) - # with steps - model.evaluate(inputs, targets, steps=2) - # with batch_size - model.evaluate(inputs, targets, batch_size=8) - - model.predict(inputs) - # with steps - model.predict(inputs, steps=2) - # with batch_size - model.predict(inputs, batch_size=8) - - @combinations.generate(strategy_for_numpy_input_combinations()) - def test_calling_model_with_nested_numpy_arrays(self, distribution): - with self.cached_session(): - model = multi_input_output_model() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.001) - loss = 'mse' - model.compile( - optimizer, - loss, - distribute=distribution, - experimental_run_tf_function=False) - - input_a_np = np.asarray(np.random.random((64, 3)), dtype=np.float32) - input_b_np = np.asarray(np.random.random((64, 5)), dtype=np.float32) - inputs = [input_a_np, input_b_np] - - output_d_np = np.asarray(np.random.random((64, 7)), dtype=np.float32) - output_e_np = np.asarray(np.random.random((64, 7)), dtype=np.float32) - targets = [output_d_np, output_e_np] - - # Call fit with validation data - model.fit(inputs, targets, epochs=1, batch_size=8, verbose=0) - - # TODO(anjalisridhar): We need tests for when the batch size and steps are - # smaller and results in a 0 batch_size and steps value. - model.evaluate(inputs, targets) - # with steps - model.evaluate(inputs, targets, steps=2) - # with batch_size - model.evaluate(inputs, targets, batch_size=8) - - model.predict(inputs) - # with steps - model.predict(inputs, steps=2) - # with batch_size - model.predict(inputs, batch_size=8) - - @combinations.generate(combinations.combine( - distribution=strategies_minus_tpu, mode=['graph'])) - def test_numpy_with_sample_weights(self, distribution): - model = get_model() - optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001) - loss = 'mse' - model.compile( - optimizer, - loss, - distribute=distribution, - experimental_run_tf_function=False) - - inputs = np.zeros((20, 3), np.float32) - targets = np.zeros((20, 4), np.float32) - sample_weights = np.ones((20), np.float32) - - model.fit(inputs, targets, sample_weight=sample_weights, epochs=1, - steps_per_epoch=2, verbose=1) - - @combinations.generate(strategy_for_numpy_input_combinations()) - def test_flatten_predict_outputs(self, distribution): - with self.cached_session(): - model = multi_input_output_model() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.001) - loss = 'mse' - model.compile( - optimizer, - loss, - distribute=distribution, - experimental_run_tf_function=False) - - # We take 6 input samples with each input having a dimension of 3 or 5. - input_a_np = np.asarray(np.random.random((6, 3)), dtype=np.float32) - input_b_np = np.asarray(np.random.random((6, 5)), dtype=np.float32) - inputs = [input_a_np, input_b_np] - - outs = model.predict(inputs, steps=1) - # `predict` a list that is equal in length to the number of model outputs. - # In this test our model has two outputs and each element of `outs` - # corresponds to all the samples of one of the model outputs. - self.assertLen(outs, 2) - # Each of the output samples have a dimension of 7. We should process all - # the available input samples(6). - self.assertAllEqual([6, 7], outs[0].shape) - self.assertAllEqual([6, 7], outs[1].shape) - - -@test_util.run_v1_only('model.compile(..distribute=..) only works in TF v1') -class TestDistributionStrategyWithDatasets(test.TestCase, - parameterized.TestCase): - - @combinations.generate(all_strategy_combinations()) - def test_calling_model_on_same_dataset(self, distribution): - with self.cached_session(): - model = get_model() - - optimizer = gradient_descent.GradientDescentOptimizer(0.001) - loss = 'mse' - metrics = ['mae', keras.metrics.CategoricalAccuracy()] - model.compile( - optimizer, - loss, - metrics=metrics, - distribute=distribution, - experimental_run_tf_function=False) - - dataset = get_dataset(distribution) - - # Call fit with validation data - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, - validation_data=dataset, validation_steps=2) - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, - validation_data=dataset, validation_steps=2) - model.predict(get_predict_dataset(distribution), steps=2) - - @combinations.generate(all_strategy_combinations()) - def test_model_interleaved_eval_same_as_direct_eval(self, distribution): - with self.cached_session(): - user_controlled_model = get_model() - user_controlled_model.compile( - gradient_descent.GradientDescentOptimizer(0.001), - loss='mse', - metrics=['mae', keras.metrics.CategoricalAccuracy()], - distribute=distribution, - experimental_run_tf_function=False) - - interleaved_model = get_model() - interleaved_model.set_weights(user_controlled_model.get_weights()) - interleaved_model.compile( - gradient_descent.GradientDescentOptimizer(0.001), - loss='mse', - metrics=['mae', keras.metrics.CategoricalAccuracy()], - distribute=distribution, - experimental_run_tf_function=False) - - dataset = get_dataset(distribution) - - # Call fit with validation interleaved - interleaved_output = interleaved_model.fit( - dataset, epochs=2, steps_per_epoch=2, verbose=1, - validation_data=dataset, validation_steps=2, shuffle=False) - - # Manually control the validation running after each epoch. - user_controlled_output = [] - for _ in range(2): - user_controlled_model.fit( - dataset, epochs=1, steps_per_epoch=2, verbose=1, shuffle=False) - user_controlled_output.append( - user_controlled_model.evaluate(dataset, steps=2)) - - self.assertEqual(interleaved_output.history['val_loss'], - [x[0] for x in user_controlled_output]) - self.assertEqual(interleaved_output.history['val_mean_absolute_error'], - [x[1] for x in user_controlled_output]) - self.assertEqual(interleaved_output.history['val_categorical_accuracy'], - [x[2] for x in user_controlled_output]) - - # TODO(priyag): Enable this test for TPU. Currently tuples/dict don't work - # as clone_model's input_tensors argument only seems to accept list and not - # tuples or dict. - - @combinations.generate( - combinations.combine( - distribution=[ - strategy_combinations.mirrored_strategy_with_gpu_and_cpu, - ], - mode=['graph', 'eager'])) - def test_fit_with_tuple_and_dict_dataset_inputs(self, distribution): - with self.cached_session(): - model = multi_input_output_model() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.001) - loss = 'mse' - metrics = ['mae', keras.metrics.CategoricalAccuracy()] - model.compile( - optimizer, - loss, - metrics=metrics, - distribute=distribution, - experimental_run_tf_function=False) - - input_a_np = np.random.random((10, 3)) - input_b_np = np.random.random((10, 5)) - output_d_np = np.random.random((10, 7)) - output_e_np = np.random.random((10, 7)) - - # Test with tuples - dataset_tuple = dataset_ops.Dataset.from_tensor_slices(( - (input_a_np, input_b_np), (output_d_np, output_e_np))) - dataset_tuple = dataset_tuple.repeat(100) - dataset_tuple = dataset_tuple.batch(10) - - model.fit(dataset_tuple, epochs=1, steps_per_epoch=2, verbose=1) - - # Test with dict - dataset_dict = dataset_ops.Dataset.from_tensor_slices(( - {'input_a': input_a_np, 'input_b': input_b_np}, - (output_d_np, output_e_np))) - dataset_dict = dataset_dict.repeat(100) - dataset_dict = dataset_dict.batch(10) - - model.fit(dataset_dict, epochs=1, steps_per_epoch=2, verbose=1) - - @combinations.generate(all_strategy_combinations()) - def test_fit_eval_and_predict_methods_on_dataset(self, distribution): - with self.cached_session(): - model = get_model() - - optimizer = gradient_descent.GradientDescentOptimizer(0.001) - loss = 'mse' - metrics = ['mae', keras.metrics.CategoricalAccuracy()] - model.compile( - optimizer, - loss, - metrics=metrics, - distribute=distribution, - experimental_run_tf_function=False) - - dataset = get_dataset(distribution) - - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1) - model.evaluate(dataset, steps=2, verbose=1) - model.predict(get_predict_dataset(distribution), steps=2) - - @combinations.generate(strategy_and_optimizer_combinations()) - def test_fit_eval_and_predict_with_optimizer(self, distribution, optimizer): - with self.cached_session(): - model = get_model() - - loss = 'mse' - model.compile( - optimizer(), - loss, - distribute=distribution, - experimental_run_tf_function=False) - - dataset = get_dataset(distribution) - - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1) - model.evaluate(dataset, steps=2, verbose=1) - model.predict(get_predict_dataset(distribution), steps=2) - - @combinations.generate(strategy_minus_tpu_combinations()) - def test_dataset_with_sample_weights(self, distribution): - model = get_model() - optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001) - loss = 'mse' - model.compile( - optimizer, - loss, - distribute=distribution, - experimental_run_tf_function=False) - - inputs = np.zeros((10, 3), np.float32) - targets = np.zeros((10, 4), np.float32) - sample_weights = np.ones((10), np.float32) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets, - sample_weights)) - dataset = dataset.repeat() - dataset = dataset.batch(10) - - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1) - model.evaluate(dataset, steps=2, verbose=1) - model.predict(dataset, steps=2) - - @combinations.generate( - combinations.combine( - distribution=[ - strategy_combinations.mirrored_strategy_with_gpu_and_cpu, - ], - mode=['graph', 'eager'])) - # TODO(b/120943676, b/120957836): Re-enable once the validation code is - # restored. - def DISABLED_test_dataset_wrong_input_shape(self, distribution): - with self.cached_session(): - model = get_model() - - optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001) - loss = 'mse' - model.compile( - optimizer, - loss, - distribute=distribution, - experimental_run_tf_function=False) - - # Wrong input shape - inputs = np.zeros((10, 5), dtype=np.float32) - targets = np.zeros((10, 4), dtype=np.float32) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat(100) - dataset = dataset.batch(10) - - with self.assertRaisesRegexp(ValueError, - 'expected input to have shape'): - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0) - - @combinations.generate( - combinations.combine( - distribution=[ - strategy_combinations.mirrored_strategy_with_gpu_and_cpu - ], - mode=['graph', 'eager'])) - # TODO(b/120943676, b/120957836): Re-enable once the validation code is - # restored. - def DISABLED_test_dataset_no_batch_input_validation(self, distribution): - with self.cached_session(): - model = get_model() - - optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001) - loss = 'mse' - model.compile( - optimizer, - loss, - distribute=distribution, - experimental_run_tf_function=False) - - # User forgets to batch the dataset - inputs = np.zeros((10, 3), dtype=np.float32) - targets = np.zeros((10, 4), dtype=np.float32) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat(100) - - with self.assertRaisesRegexp(ValueError, 'expected input to have shape'): - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0) - - @combinations.generate( - combinations.combine( - distribution=[ - strategy_combinations.mirrored_strategy_with_gpu_and_cpu, - strategy_combinations.mirrored_strategy_with_two_gpus, - ], - mode=['graph', 'eager'])) - def test_learning_phase_value(self, distribution): - # TODO(anjalisridhar): Modify this test to use Lambdas since we can compare - # meaningful values. Currently we don't pass the learning phase if the - # Lambda layer uses the learning phase. - with self.cached_session(): - x = keras.layers.Input(shape=(1,), name='input') - y = keras.layers.Dense(1, kernel_initializer='ones')(x) - z = keras.layers.Dropout(0.9999)(y) - model = keras.Model(x, z) - initial_weights = model.get_weights() - - optimizer = gradient_descent.GradientDescentOptimizer(0.005) - loss = 'mse' - metrics = ['acc'] - model.compile( - optimizer, - loss, - metrics=metrics, - distribute=distribution, - experimental_run_tf_function=False) - - batch_size = 8 - if isinstance(distribution, mirrored_strategy.CoreMirroredStrategy): - # CoreMirroredStrategy uses global batch size. - batch_size = 8 * distribution.num_replicas_in_sync - - inputs = np.ones((10, 1), dtype=np.float32) - targets = np.ones((10, 1), dtype=np.float32) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat().batch(batch_size) - hist = model.fit(dataset, epochs=1, steps_per_epoch=20, verbose=1) - self.assertAlmostEqual(hist.history['acc'][0], 0, 0) - - model.set_weights(initial_weights) - # TODO(psv/anjalisridhar): Enable these lines after we fix b/117431185. - # evaluate_output = model.evaluate(dataset, steps=20) - # self.assertAlmostEqual(evaluate_output[1], 1, 0) - - inputs = np.ones((10, 1), dtype=np.float32) - predict_dataset = dataset_ops.Dataset.from_tensor_slices(inputs) - - predict_dataset = predict_dataset.repeat().batch(batch_size) - output = model.predict(predict_dataset, steps=10) - # `predict` runs for 10 steps - ref_output = np.ones((160, 1), dtype=np.float32) - self.assertArrayNear(output, ref_output, 1e-1) - - @combinations.generate(strategy_minus_tpu_combinations()) - def testOptimizerWithCallbacks(self, distribution): - with self.cached_session(): - model = get_model() - - optimizer = gradient_descent_keras.SGD(0.01) - loss = 'mse' - model.compile( - optimizer, - loss, - distribute=distribution, - experimental_run_tf_function=False) - - dataset = get_dataset(distribution) - - def schedule(_): - return 0.001 - - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, - callbacks=[keras.callbacks.LearningRateScheduler(schedule)]) - grouped_models = distribution.experimental_local_results( - distributed_training_utils.get_distributed_model( - model, ModeKeys.TRAIN)) - with distribution.scope(): - for m in grouped_models: - self.assertAllClose(0.001, keras.backend.get_value( - m.optimizer.lr), atol=1e-05, rtol=1e-05) - - -@test_util.run_v1_only('model.compile(..distribute=..) only works in TF v1') -class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase): - - @combinations.generate( - combinations.combine( - distribution=[ - strategy_combinations.mirrored_strategy_with_gpu_and_cpu, - ], - mode=['graph', 'eager'])) - def test_unsupported_features(self, distribution): - with self.cached_session(): - model = get_model() - - optimizer = gradient_descent.GradientDescentOptimizer(0.001) - loss = 'mse' - metrics = ['mae'] - model.compile( - optimizer, - loss, - metrics=metrics, - distribute=distribution, - experimental_run_tf_function=False) - - dataset = get_dataset(distribution) - - # Test with validation split - with self.assertRaisesRegexp( - ValueError, '`validation_split` argument is not ' - 'supported when input `x` is a dataset or a ' - 'dataset iterator.+'): - model.fit(dataset, - epochs=1, steps_per_epoch=2, verbose=0, - validation_split=0.5, validation_steps=2) - - # Test with sample weight. - sample_weight = np.random.random((10,)) - with self.assertRaisesRegexp( - ValueError, '`sample_weight` argument is not supported when input ' - '`x` is a dataset or a dataset iterator.'): - model.fit( - dataset, - epochs=1, - steps_per_epoch=2, - verbose=0, - sample_weight=sample_weight) - - # Test with not specifying the `steps` argument for dataset with - # infinite cardinality. - dataset = dataset.repeat() - with self.assertRaisesRegexp(ValueError, 'When passing an infinitely ' - 'repeating dataset, you must specify the ' - '`steps_per_epoch` argument'): - model.fit(dataset, epochs=1, verbose=0) - with self.assertRaisesRegexp(ValueError, 'When passing an infinitely ' - 'repeating dataset, you must specify the ' - '`steps` argument'): - model.evaluate(dataset, verbose=0) - - with self.assertRaisesRegexp(ValueError, 'When passing an infinitely ' - 'repeating dataset, you must specify the ' - '`steps` argument'): - model.predict(dataset, verbose=0) - - @combinations.generate( - combinations.combine( - distribution=[ - strategy_combinations.mirrored_strategy_with_gpu_and_cpu, - ], - mode=['graph', 'eager'])) - def test_calling_with_unsupported_predefined_callbacks(self, distribution): - with self.cached_session(): - model = get_model() - - optimizer = gradient_descent.GradientDescentOptimizer(0.001) - loss = 'mse' - metrics = ['mae'] - model.compile( - optimizer, - loss, - metrics=metrics, - distribute=distribution, - experimental_run_tf_function=False) - - dataset = get_dataset(distribution) - - def schedule(_): - return 0.001 - with self.assertRaisesRegexp(ValueError, - 'You must specify a Keras Optimizer V2 when ' - 'using'): - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, - callbacks=[keras.callbacks.LearningRateScheduler(schedule)]) - - with self.assertRaisesRegexp(ValueError, - 'You must specify a Keras Optimizer V2 when ' - 'using'): - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, - callbacks=[keras.callbacks.ReduceLROnPlateau()]) - - -@test_util.run_v1_only('model.compile(..distribute=..) only works in TF v1') -class TestDistributionStrategyWithLossMasking(test.TestCase, - parameterized.TestCase): - - # TODO(priyag): Enable all strategies for this test. Currently it does not - # work for TPU due to some invalid datatype. - @combinations.generate( - combinations.combine( - distribution=[ - strategy_combinations.mirrored_strategy_with_gpu_and_cpu, - ], - mode=['graph', 'eager'])) - def test_masking(self, distribution): - with self.cached_session(): - np.random.seed(1337) - x = np.array([[[1], [1]], [[0], [0]]]) - model = keras.models.Sequential() - model.add(keras.layers.Masking(mask_value=0, input_shape=(2, 1))) - model.add( - keras.layers.TimeDistributed( - keras.layers.Dense(1, kernel_initializer='one'))) - model.compile( - loss='mse', - optimizer=gradient_descent.GradientDescentOptimizer(0.01), - distribute=distribution, - experimental_run_tf_function=False) - y = np.array([[[1], [1]], [[1], [1]]]) - dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) - dataset = dataset.repeat(100) - dataset = dataset.batch(10) - hist = model.fit(x=dataset, epochs=1, steps_per_epoch=2) - self.assertEqual(hist.history['loss'][0], 0) - - -@test_util.run_v1_only('model.compile(..distribute=..) only works in TF v1') -class TestDistributionStrategyWithNormalizationLayer( - test.TestCase, parameterized.TestCase): - - @combinations.generate(all_strategy_combinations()) - def test_batchnorm_correctness(self, distribution): - with self.cached_session(): - model = keras.models.Sequential() - norm = keras.layers.BatchNormalization(input_shape=(10,), momentum=0.8) - model.add(norm) - model.compile( - loss='mse', - optimizer=gradient_descent.GradientDescentOptimizer(0.01), - distribute=distribution, - experimental_run_tf_function=False) - - # centered on 5.0, variance 10.0 - x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10)) - x = x.astype('float32') - dataset = dataset_ops.Dataset.from_tensor_slices((x, x)) - dataset = dataset.repeat(100) - dataset = batch_wrapper(dataset, 32, distribution) - - predict_dataset = dataset_ops.Dataset.from_tensor_slices(x) - predict_dataset = predict_dataset.repeat(100) - predict_dataset = batch_wrapper(predict_dataset, 32, distribution) - - model.fit(dataset, epochs=4, verbose=0, steps_per_epoch=10) - out = model.predict(predict_dataset, steps=2) - out -= keras.backend.eval(norm.beta) - out /= keras.backend.eval(norm.gamma) - np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1) - np.testing.assert_allclose(out.std(), 1.0, atol=1e-1) - - -@test_util.run_v1_only('model.compile(..distribute=..) only works in TF v1') -class TestDistributionStrategyCorrectness(test.TestCase, - parameterized.TestCase): - - @combinations.generate(all_strategy_combinations()) - def test_metric_correctness(self, distribution): - with self.cached_session(): - keras.backend.set_image_data_format('channels_last') - num_samples = 10000 - - x_train = np.random.randint(0, 2, num_samples) - x_train = np.reshape(x_train, (num_samples, 1)) - y_train = x_train - x_train = x_train.astype('float32') - y_train = y_train.astype('float32') - - # Create identity model. - model = keras.Sequential() - model.add( - keras.layers.Dense(1, input_shape=(1,), kernel_initializer='ones')) - model.compile( - loss=keras.losses.mean_squared_error, - optimizer=gradient_descent.GradientDescentOptimizer(0.5), - metrics=[keras.metrics.BinaryAccuracy()], - distribute=distribution, - experimental_run_tf_function=False) - - batch_size = 64 - if not distributed_training_utils.global_batch_size_supported( - distribution): - batch_size //= distribution.num_replicas_in_sync - train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train)) - train_dataset = batch_wrapper(train_dataset, batch_size, distribution) - - history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10) - self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0]) - - @combinations.generate(all_strategy_combinations()) - def test_eval_metrics_correctness(self, distribution): - with self.cached_session(): - model = keras.Sequential() - model.add( - keras.layers.Dense( - 3, activation='relu', input_dim=4, kernel_initializer='ones')) - model.add( - keras.layers.Dense( - 1, activation='sigmoid', kernel_initializer='ones')) - model.compile( - loss='mae', - metrics=['accuracy', keras.metrics.BinaryAccuracy()], - optimizer=gradient_descent.GradientDescentOptimizer(0.001), - distribute=distribution, - experimental_run_tf_function=False) - - # verify correctness of stateful and stateless metrics. - x = np.ones((100, 4)).astype('float32') - y = np.ones((100, 1)).astype('float32') - dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat() - dataset = batch_wrapper(dataset, 4, distribution) - outs = model.evaluate(dataset, steps=10) - self.assertEqual(outs[1], 1.) - self.assertEqual(outs[2], 1.) - - y = np.zeros((100, 1)).astype('float32') - dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat() - dataset = batch_wrapper(dataset, 4, distribution) - outs = model.evaluate(dataset, steps=10) - self.assertEqual(outs[1], 0.) - self.assertEqual(outs[2], 0.) - - @combinations.generate(strategy_and_input_combinations()) - def test_correctness(self, distribution, use_numpy, use_validation_data): - with self.cached_session(): - default_tolerance = 1e-5 - tol_table = {} - - if isinstance(distribution, ( - mirrored_strategy.MirroredStrategy, - mirrored_strategy.CoreMirroredStrategy, - distribute_lib._DefaultDistributionStrategy)): # pylint: disable=protected-access - # TODO(b/119257215): Weights are not exactly the same, so use larger - # tolerance for now. Predict should be related to weights. - tol_table = { - 'weights_1': 1e-4, - 'weights_2': 1e-4, - 'predict_result_1': 1e-4, - } - - keras.backend.set_image_data_format('channels_last') - np.random.seed(_RANDOM_SEED) - random_seed.set_random_seed(_RANDOM_SEED) - - # Train, eval, and predict datasets are created with the same input numpy - # arrays. - # TODO(xiejw): Change this back to 10000, once we support final partial - # batch. - num_samples = 9984 - x_train = np.random.rand(num_samples, 1) - y_train = 3 * x_train - x_train = x_train.astype('float32') - y_train = y_train.astype('float32') - x_predict = [[1.], [2.], [3.], [4.]] - - # The model is built once and the initial weights are saved. - # This is used to initialize the model for both the distribution and - # non-distribution run. In addition, we add few non-linear layers to make - # it non-trivial. - def _create_model(): - model = keras.Sequential() - model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,))) - model.add(keras.layers.Dense(10, activation='relu')) - model.add(keras.layers.Dense(10, activation='relu')) - model.add(keras.layers.Dense(1)) - return model - - model = _create_model() - initial_weights = model.get_weights() - del model # avoid accident usage. - - def fit_eval_and_predict(with_distribution=None): - model = _create_model() - # We have initialized the model to the same weight for the distribution - # and non-distribution run. - model.set_weights(initial_weights) - model.compile( - loss=keras.losses.mean_squared_error, - optimizer=gradient_descent_keras.SGD(0.5), - metrics=['mse'], - distribute=with_distribution, - experimental_run_tf_function=False) - - training_inputs, eval_inputs, predict_inputs = ( - get_correctness_test_inputs(use_numpy, use_validation_data, - with_distribution, - x_train, y_train, x_predict)) - - result = {} - result['training_history_1'] = model.fit(**training_inputs).history - - if eval_inputs is not None: - result['eval_result_1'] = model.evaluate(**eval_inputs) - - result['weights_1'] = model.get_weights() - result['predict_result_1'] = model.predict(**predict_inputs) - - # Train and eval again to mimic user's flow. - - result['training_history_2'] = model.fit(**training_inputs).history - - if eval_inputs is not None: - result['eval_result_2'] = model.evaluate(**eval_inputs) - - result['weights_2'] = model.get_weights() - - return result - - results_with_ds = fit_eval_and_predict(with_distribution=distribution) - results_without_ds = fit_eval_and_predict(with_distribution=None) - - # Verify that the weights, training history, eval results, predict outputs - # are the same within some limits of tolerance. - for key in results_with_ds: - if (key.startswith('training_history') and - isinstance(distribution, tpu_strategy.TPUStrategyV1) and - distribution.extended.steps_per_run > 1): - # TODO(b/119894254): Enable this test for all cases once the - # underlying bug is fixed. - continue - - tolerance = tol_table.get(key, default_tolerance) - - self.assertAllClose( - results_with_ds[key], - results_without_ds[key], - atol=tolerance, - rtol=tolerance, - msg='Fail to assert {}.'.format(key)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/distribute/python/keras_multi_worker_correctness_test.py b/tensorflow/contrib/distribute/python/keras_multi_worker_correctness_test.py deleted file mode 100644 index 3eceac0eb50..00000000000 --- a/tensorflow/contrib/distribute/python/keras_multi_worker_correctness_test.py +++ /dev/null @@ -1,229 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for accuracy and mathematical correctness of tf.keras multi-worker.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import sys - -from absl.testing import parameterized -import numpy as np - -# pylint: disable=g-direct-tensorflow-import -from tensorflow.contrib.distribute.python import collective_all_reduce_strategy as collective_strategy -from tensorflow.contrib.distribute.python import keras_multi_worker_test_base -from tensorflow.python import keras -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import multi_worker_util -from tensorflow.python.distribute.cluster_resolver import TFConfigClusterResolver -from tensorflow.python.framework import ops -from tensorflow.python.keras.optimizer_v2 import gradient_descent -from tensorflow.python.platform import test - - -np.random.seed(99) -EMBED_INPUTS = np.random.randint(0, 10, (6400, 1)).astype(np.int32) -EMBED_TARGETS = np.random.normal(0, 0.1, (6400, 1)).astype(np.float32) -IMAGE_INPUTS = np.random.normal(0, 0.1, (6400, 28, 28, 3)).astype(np.float32) -IMAGE_TARGETS = np.random.randint(0, 10, (6400, 1)) -LSTM_INPUTS = np.random.normal(0, 0.1, (6400, 10, 20)).astype(np.float32) -LSTM_TARGETS = np.random.normal(0, 0.1, (6400, 1)).astype(np.float32) - - -def get_num_workers(): - cluster_resolver = TFConfigClusterResolver() - cluster_spec = cluster_resolver.cluster_spec().as_dict() - if cluster_spec: - task_type = cluster_resolver.task_type - return int(multi_worker_util.worker_count(cluster_spec, task_type)) - return 1 - - -class Bias(keras.layers.Layer): - - def build(self, input_shape): - self.bias = self.add_weight(shape=(), initializer='zeros', name='bias') - - def call(self, inputs): - return inputs + self.bias - - -class SimpleBiasTest( - keras_multi_worker_test_base.KerasIndependentWorkerTestBase, - parameterized.TestCase): - - @keras_multi_worker_test_base.run_sync_strategies - def test_multi_worker_simple_bias_fit(self, strategy_cls): - - def _worker_fn(results_without_ds=None): - # Make sure Session is cleared at the start of each run. - keras.backend._SESSION.session = None - - x = ops.convert_to_tensor([[0.], [1.], [2.], [0.], [1.], [2.], [0.], - [1.]]) - y = ops.convert_to_tensor([[0.5], [2.], [3.5], [0.5], [2.], [3.5], [0.5], - [2.]]) - ds = dataset_ops.Dataset.from_tensor_slices((x, y)) - ds = ds.batch(8) - model = keras.Sequential([Bias(input_shape=(1,))]) - model.compile( - keras.optimizer_v2.gradient_descent.SGD(0.1), 'mae', metrics=['mae']) - history = model.fit(ds, epochs=5) - self.assertAllClose(history.history['loss'], - [0.9375, 0.8375, 0.7375, 0.6375, 0.5375]) - self.assertAllClose(history.history['mean_absolute_error'], - [0.9375, 0.8375, 0.7375, 0.6375, 0.5375]) - - results = {'training': history.history} - if results_without_ds: - for key in results: - self.assertAllClose( - results[key], - results_without_ds[key], - msg='Fail to assert {}'.format(key)) - - return results - - results_without_ds = _worker_fn() - self.run_independent_workers( - _worker_fn, - strategy_cls, - num_workers=2, - results_without_ds=results_without_ds) - - -def make_image_model(initial_weights=None): - image = keras.layers.Input(shape=(28, 28, 3), name='image') - c1 = keras.layers.Conv2D( - name='conv1', - filters=16, - kernel_size=(3, 3), - strides=(4, 4), - kernel_regularizer=keras.regularizers.l2(1e-4))( - image) - c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1) - c1 = keras.layers.Flatten()(c1) - logits = keras.layers.Dense(10, activation='softmax', name='pred')(c1) - model = keras.Model(inputs=[image], outputs=[logits]) - - if initial_weights: - model.set_weights(initial_weights) - - model.compile( - 'sgd', - loss='sparse_categorical_crossentropy', - metrics=['sparse_categorical_accuracy']) - - return model, IMAGE_INPUTS, IMAGE_TARGETS - - -def make_lstm_model(initial_weights=None): - inputs = keras.layers.Input(shape=(10, 20)) - rnn_out = keras.layers.LSTM(4)(inputs) - outputs = keras.layers.Dense(1)(rnn_out) - model = keras.Model(inputs, outputs) - - if initial_weights: - model.set_weights(initial_weights) - - model.compile( - gradient_descent.SGD(0.1), - 'sparse_categorical_crossentropy', - metrics=['sparse_categorical_crossentropy']) - - return model, LSTM_INPUTS, LSTM_TARGETS - - -def make_embedding_model(initial_weights=None): - inputs = keras.layers.Input(shape=(1,), dtype='int32') - embeddings = keras.layers.Embedding(100, 5)(inputs) - outputs = keras.layers.Dense(1, activation='softmax')(embeddings) - model = keras.Model(inputs, outputs) - - if initial_weights: - model.set_weights(initial_weights) - - model.compile('rmsprop', 'mae', metrics=['binary_crossentropy']) - - return model, EMBED_INPUTS, EMBED_TARGETS - - -class ModelCorrectnessTest( - keras_multi_worker_test_base.KerasIndependentWorkerTestBase, - parameterized.TestCase): - - def make_dataset(self, inputs, targets, batch_size=64): - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.batch(batch_size) - return dataset - - @combinations.generate( - combinations.combine( - mode=['graph'], - strategy_cls=[ - collective_strategy.CollectiveAllReduceStrategy, - ], - make_model=[make_image_model, make_lstm_model, make_embedding_model], - steps_per_epoch=[50, None], - required_gpus=[0, 1])) - def test_correctness(self, strategy_cls, make_model, steps_per_epoch): - - def _worker_fn(initial_weights=None, results_without_ds=None): - # Make sure Session is cleared at each run - # so that it can be configured properly for the DistributionStrategy. - keras.backend._SESSION.session = None - - results = {} - model, inputs, targets = make_model(initial_weights) - - data = self.make_dataset(inputs, targets) - - # TODO(b/129363441): Remove `steps_per_epoch`. - results['training'] = model.fit( - data, steps_per_epoch=steps_per_epoch, epochs=2).history - results['trained_weights'] = model.get_weights() - - eval_data = self.make_dataset(inputs, targets) - results['evaluation'] = model.evaluate(eval_data, steps=50) - - if results_without_ds: - for key in results: - self.assertAllClose( - results[key], - results_without_ds[key], - rtol=1e-5, - atol=1e-5, - msg='Fail to assert {}'.format(key)) - - return results - - model, _, _ = make_model() - initial_weights = model.get_weights() - results_without_ds = _worker_fn(initial_weights=initial_weights) - self.run_independent_workers( - _worker_fn, - strategy_cls, - num_workers=2, - initial_weights=initial_weights, - results_without_ds=results_without_ds) - - -if __name__ == '__main__': - with test.mock.patch.object(sys, 'exit', os._exit): - test.main() diff --git a/tensorflow/contrib/distribute/python/keras_multi_worker_test_base.py b/tensorflow/contrib/distribute/python/keras_multi_worker_test_base.py deleted file mode 100644 index 324b10fdae1..00000000000 --- a/tensorflow/contrib/distribute/python/keras_multi_worker_test_base.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Test base for tf.keras Models in multi-worker mode.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import threading - -# pylint: disable=g-direct-tensorflow-import -from tensorflow.contrib.distribute.python import collective_all_reduce_strategy as collective_strategy -from tensorflow.contrib.distribute.python import parameter_server_strategy -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import distribute_coordinator as dc -from tensorflow.python.distribute import multi_worker_test_base -from tensorflow.python.eager import context -from tensorflow.python.platform import test - -_original_run_std_server = dc._run_std_server # pylint: disable=protected-access - -# Used as a decorator on test methods. -run_sync_strategies = combinations.generate( - combinations.combine( - mode=['graph'], - strategy_cls=[ - collective_strategy.CollectiveAllReduceStrategy, - ], - required_gpus=[0, 1])) - -# Used as a decorator on test methods. -run_async_strategies = combinations.generate( - combinations.combine( - mode=['graph'], - strategy_cls=[parameter_server_strategy.ParameterServerStrategy], - required_gpus=[0, 1])) - - -def get_strategy_object(strategy_cls): - return strategy_cls(num_gpus_per_worker=context.num_gpus()) - - -# TODO(omalleyt): Merge with keras_multiworker_callback_test -class KerasIndependentWorkerTestBase( - multi_worker_test_base.IndependentWorkerTestBase): - """Test base for simulating Keras Multi-Worker in threads.""" - - def _make_mock_run_std_server(self): - thread_local = threading.local() - - def _mock_run_std_server(*args, **kwargs): - ret = _original_run_std_server(*args, **kwargs) - # Wait for all std servers to be brought up in order to reduce the chance - # of remote sessions taking local ports that have been assigned to std - # servers. Only call this barrier the first time this function is run for - # each thread. - if not getattr(thread_local, 'server_started', False): - self._barrier.wait() - thread_local.server_started = True - return ret - - return _mock_run_std_server - - def run_independent_workers(self, - worker_fn, - strategy_cls, - num_workers, - num_ps=None, - **kwargs): - cluster_spec = multi_worker_test_base.create_cluster_spec( - num_workers=num_workers, num_ps=num_ps) - self._barrier = dc._Barrier(num_workers + (num_ps or 0)) # pylint: disable=protected-access - - def _worker_fn(**kwargs): - """Runs the worker function in a thread.""" - with test.mock.patch.object(dc, '_run_std_server', - self._make_mock_run_std_server()): - strategy = get_strategy_object(strategy_cls) - with strategy.scope(): - return worker_fn(**kwargs) - - threads = self.run_multiple_tasks_in_threads(_worker_fn, cluster_spec, - **kwargs) - strategy = get_strategy_object(strategy_cls) - if strategy.extended.experimental_between_graph: - threads_to_join = threads.get('chief', []) + threads.get('worker', []) - else: - threads_to_join = [ - threads['chief'][0] if 'chief' in threads else threads['worker'][0] - ] - self.join_independent_workers(threads_to_join) diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py deleted file mode 100644 index 8b3c5e546d6..00000000000 --- a/tensorflow/contrib/distribute/python/mirrored_strategy.py +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contrib version of MirroredStrategy.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.distribute import distribute_lib -from tensorflow.python.distribute import input_lib -from tensorflow.python.distribute import mirrored_strategy - - -all_local_devices = mirrored_strategy.all_local_devices -CoreMirroredStrategy = mirrored_strategy.MirroredStrategy -CoreMirroredExtended = mirrored_strategy.MirroredExtended - - -class MirroredStrategy(distribute_lib.StrategyV1): - """Mirrors vars to distribute across multiple devices and machines. - - *** contrib version *** - - This strategy uses one replica per device and sync replication for its - multi-GPU version. - - When `cluster_spec` is given by the `configure` method., it turns into the - mulit-worker version that works on multiple workers with in-graph replication. - Note: `configure` will be called by higher-level APIs if running in - distributed environment. - - There are several important concepts for distributed TensorFlow, e.g. - `client`, `job`, `task`, `cluster`, `in-graph replication` and - `synchronous training` and they have already been defined in the - [TensorFlow's documentation](https://www.tensorflow.org/deploy/distributed). - The distribution strategy inherits these concepts as well and in addition to - that we also clarify several more concepts: - - * **In-graph replication**: the `client` creates a single `tf.Graph` that - specifies tasks for devices on all workers. The `client` then creates a - client session which will talk to the `master` service of a `worker`. Then - the `master` will partition the graph and distribute the work to all - participating workers. - * **Worker**: A `worker` is a TensorFlow `task` that usually maps to one - physical machine. We will have multiple `worker`s with different `task` - index. They all do similar things except for one worker checkpointing model - variables, writing summaries, etc. in addition to its ordinary work. - - The multi-worker version of this class maps one replica to one device on a - worker. It mirrors all model variables on all replicas. For example, if you - have two `worker`s and each `worker` has 4 GPUs, it will create 8 copies of - the model variables on these 8 GPUs. Then like in MirroredStrategy, each - replica performs their computation with their own copy of variables unless in - cross-replica model where variable or tensor reduction happens. - - Args: - devices: a list of device strings. - num_gpus: number of GPUs. For local training, either specify `devices` or - `num_gpus`. In distributed training, this must be specified as number of - GPUs on each worker. - num_gpus_per_worker: number of GPUs per worker. This is the same as - `num_gpus` and only one of `num_gpus` and `num_gpus_per_worker` can be - specified. - cross_device_ops: optional, a descedant of `CrossDeviceOps`. If this is not - set, the `configure` method will try to find the best one. - auto_shard_dataset: whether to auto-shard the dataset when there are - multiple workers. - cross_tower_ops: Deprecated alias for `cross_device_ops`. - """ - - def __init__(self, - devices=None, - num_gpus=None, - num_gpus_per_worker=None, - cross_device_ops=None, - auto_shard_dataset=False, - cross_tower_ops=None): - assert not (cross_device_ops and cross_tower_ops) - if num_gpus is not None and num_gpus_per_worker is not None: - raise ValueError( - "You cannot specify both `num_gpus` and `num_gpus_per_worker`.") - if num_gpus is None: - num_gpus = num_gpus_per_worker - extended = MirroredExtended(self, devices, num_gpus, - cross_device_ops or cross_tower_ops, - auto_shard_dataset) - super(MirroredStrategy, self).__init__(extended) - - # Override to change the documentation to reflect the different handling of - # global vs. local batch size between core and contrib. - def make_dataset_iterator(self, dataset): # pylint: disable=useless-super-delegation - """Makes an iterator for input provided via `dataset`. - - NOTE: The batch size of the `dataset` argument is treated differently for - this contrib version of `MirroredStrategy`. - - Data from the given dataset will be distributed evenly across all the - compute replicas. We will assume that the input dataset is batched by the - per-replica batch size. - - The user could also use `make_input_fn_iterator` if they want to - customize which input is fed to which replica/worker etc. - - Args: - dataset: `tf.data.Dataset` that will be distributed evenly across all - replicas. - - Returns: - An `tf.distribute.InputIterator` which returns inputs for each step of the - computation. User should call `initialize` on the returned iterator. - """ - return super(MirroredStrategy, self).make_dataset_iterator(dataset) - - -class MirroredExtended(CoreMirroredExtended): - """Implementation of (contrib) MirroredStrategy.""" - - def __init__(self, - container_strategy, - devices=None, - num_gpus_per_worker=None, - cross_device_ops=None, - auto_shard_dataset=False): - if devices is None: - devices = mirrored_strategy.all_local_devices(num_gpus_per_worker) - elif num_gpus_per_worker is not None: - raise ValueError( - "Must only specify one of `devices` and `num_gpus_per_worker`.") - super(MirroredExtended, self).__init__(container_strategy, devices, - cross_device_ops) - self._auto_shard_dataset = auto_shard_dataset - - def _make_dataset_iterator(self, dataset): - """Make iterator from dataset without splitting the batch. - - This implementation is different than the one in - `tf.distribute.MirroredStrategy` for purposes of backward compatibility. - We treat the incoming dataset's batch size as per replica batch size. - - Args: - dataset: `tf.data.Dataset` for input. - Returns: - An `InputIterator` which returns inputs for each step of the computation. - """ - return input_lib.DatasetIterator(dataset, self._input_workers, - self._container_strategy()) - - # TODO(priyag): Delete this once all strategies use global batch size. - @property - def _global_batch_size(self): - """The contrib version of Mirrored strategy uses per-replica batch size.""" - return False diff --git a/tensorflow/contrib/distribute/python/monitor.py b/tensorflow/contrib/distribute/python/monitor.py deleted file mode 100644 index 53e35ea6b75..00000000000 --- a/tensorflow/contrib/distribute/python/monitor.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Monitor is responsible for training, checkpointing and recovery.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.eager import context -from tensorflow.python.framework import errors -from tensorflow.python.ops import variables - - -class Monitor(object): - """Executes training steps, recovers and checkpoints. - - Note that this class is particularly preliminary, experimental, and - expected to change. - """ - # TODO(isaprykin): Support step functions that need multiple session calls. - # TODO(isaprykin): Support extra arguments to the step function. - # TODO(isaprykin): Support recovery, checkpointing and summaries. - - def __init__(self, step_callable, session=None): - """Initialize the Monitor with components for executing training steps. - - Args: - step_callable: a training `Step` that's capable of signaling when done. - session: a `Session` instance that's needed for graph mode. - - Raises: - ValueError: if `session` was provided for eager mode or not provided for - graph mode. - """ - if context.executing_eagerly(): - if session is not None: - raise ValueError("Should not provide a `session` in Eager mode.") - self._run_step = step_callable - else: - if session is None: - raise ValueError("Should provide a `session` in Graph mode.") - session.run(step_callable.initialize()) - self._run_step = session.make_callable(step_callable()) - session.run(variables.global_variables_initializer()) - - def run_steps(self, num_steps=None): - step = 0 - while num_steps is None or step < num_steps: - try: - self._run_step() - step += 1 - except errors.OutOfRangeError: - break diff --git a/tensorflow/contrib/distribute/python/monitor_test.py b/tensorflow/contrib/distribute/python/monitor_test.py deleted file mode 100644 index 397ce8743d8..00000000000 --- a/tensorflow/contrib/distribute/python/monitor_test.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for class Monitor.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy -from tensorflow.contrib.distribute.python import monitor as monitor_lib -from tensorflow.python.client import session -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import one_device_strategy -from tensorflow.python.distribute import strategy_combinations -from tensorflow.python.distribute.single_loss_example import single_loss_example -from tensorflow.python.eager import context -from tensorflow.python.eager import test -from tensorflow.python.framework import ops -from tensorflow.python.training import gradient_descent - - -class MonitorTest(test.TestCase, parameterized.TestCase): - - @combinations.generate( - combinations.times( - strategy_combinations.distributions_and_v1_optimizers(), - combinations.combine( - mode=strategy_combinations.graph_and_eager_modes))) - def testTrainNetwork(self, distribution, optimizer_fn): - with distribution.scope(): - single_loss_step, layer = single_loss_example(optimizer_fn, distribution) - - if context.executing_eagerly(): - monitor = monitor_lib.Monitor(single_loss_step, None) - else: - with self.cached_session() as sess: - monitor = monitor_lib.Monitor(single_loss_step, sess) - - monitor.run_steps(1) - - self.assertEqual(1, len(layer.trainable_variables)) - mirrored_weight_variable = layer.trainable_variables[0] - start_error = self.evaluate(mirrored_weight_variable) - start_error = abs(numpy.array(start_error) - 1) - - monitor.run_steps(9) - end_error = self.evaluate(mirrored_weight_variable) - end_error = abs(numpy.array(end_error) - 1) - self.assertGreaterEqual(start_error, end_error) - - def testPassingASessionInEager(self): - distribution = one_device_strategy.OneDeviceStrategy( - "/device:CPU:0") - step_function, _ = single_loss_example( - lambda: gradient_descent.GradientDescentOptimizer(0.2), distribution) - - with session.Session() as sess, context.eager_mode(): - with self.assertRaisesRegexp(ValueError, "Should not provide"): - _ = monitor_lib.Monitor(step_function, sess) - - def testNotPassingASessionInGraph(self): - distribution = one_device_strategy.OneDeviceStrategy( - "/device:CPU:0") - step_function, _ = single_loss_example( - lambda: gradient_descent.GradientDescentOptimizer(0.2), distribution) - - with context.graph_mode(), ops.Graph().as_default(): - with self.assertRaisesRegexp(ValueError, "Should provide"): - _ = monitor_lib.Monitor(step_function, session=None) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py deleted file mode 100644 index 6ae847c2938..00000000000 --- a/tensorflow/contrib/distribute/python/one_device_strategy.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Class OneDeviceStrategy implementing DistributionStrategy.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.distribute import one_device_strategy - -OneDeviceStrategy = one_device_strategy.OneDeviceStrategyV1 diff --git a/tensorflow/contrib/distribute/python/optimizer_v2_test.py b/tensorflow/contrib/distribute/python/optimizer_v2_test.py deleted file mode 100644 index bbae1174e49..00000000000 --- a/tensorflow/contrib/distribute/python/optimizer_v2_test.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for running legacy optimizer code with DistributionStrategy.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy -from tensorflow.contrib.distribute.python import mirrored_strategy as mirrored_lib -from tensorflow.contrib.optimizer_v2 import adagrad as adagrad_v2 -from tensorflow.contrib.optimizer_v2 import gradient_descent as gradient_descent_v2 -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations -from tensorflow.python.distribute.single_loss_example import minimize_loss_example -from tensorflow.python.eager import context -from tensorflow.python.eager import test -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import variables - - -mirrored_strategy_with_gpu_and_cpu = combinations.NamedDistribution( - "MirroredCPUAndGPU", - lambda: mirrored_lib.MirroredStrategy(["/gpu:0", "/cpu:0"]), - required_gpus=1) -mirrored_strategy_with_two_gpus = combinations.NamedDistribution( - "Mirrored2GPUs", - lambda: mirrored_lib.MirroredStrategy(["/gpu:0", "/gpu:1"]), - required_gpus=2) - -# pylint: disable=g-long-lambda -gradient_descent_optimizer_v2_fn = combinations.NamedObject( - "GradientDescentV2", lambda: gradient_descent_v2.GradientDescentOptimizer( - 0.2)) -adagrad_optimizer_v2_fn = combinations.NamedObject( - "AdagradV2", lambda: adagrad_v2.AdagradOptimizer(0.001)) - -optimizers_v2 = [gradient_descent_optimizer_v2_fn, adagrad_optimizer_v2_fn] - - -def distributions_and_v2_optimizers(): - """DistributionStrategies and V2 Optimizers.""" - return combinations.combine( - distribution=[ - strategy_combinations.one_device_strategy, - strategy_combinations.mirrored_strategy_with_gpu_and_cpu, - strategy_combinations.mirrored_strategy_with_two_gpus, - ], - optimizer_fn=optimizers_v2) - - -class MinimizeLossOptimizerV2Test(test.TestCase, parameterized.TestCase): - - @combinations.generate( - combinations.times( - distributions_and_v2_optimizers(), - combinations.combine(mode=["graph"], use_callable_loss=[True, False]) - + combinations.combine(mode=["eager"], use_callable_loss=[True]))) - def testTrainNetwork(self, distribution, optimizer_fn, - use_callable_loss=True): - with distribution.scope(): - optimizer = optimizer_fn() - model_fn, dataset_fn, layer = minimize_loss_example( - optimizer, use_bias=True, use_callable_loss=use_callable_loss) - iterator = distribution.make_input_fn_iterator(lambda _: dataset_fn()) - - def run_step(): - return control_flow_ops.group( - distribution.experimental_local_results( - distribution.extended.call_for_each_replica( - model_fn, args=(iterator.get_next(),)))) - - if not context.executing_eagerly(): - with self.cached_session() as sess: - sess.run(iterator.initialize()) - run_step = sess.make_callable(run_step()) - self.evaluate(variables.global_variables_initializer()) - - weights, biases = [], [] - for _ in range(10): - run_step() - - weights.append(self.evaluate(layer.kernel)) - biases.append(self.evaluate(layer.bias)) - - error = abs(numpy.add(numpy.squeeze(weights), numpy.squeeze(biases)) - 1) - is_not_increasing = all(y <= x for x, y in zip(error, error[1:])) - self.assertTrue(is_not_increasing) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py deleted file mode 100644 index 90f174d0d47..00000000000 --- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Classes implementing a multi-worker ps DistributionStrategy.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.distribute import distribute_lib -from tensorflow.python.distribute import input_lib -from tensorflow.python.distribute import parameter_server_strategy -from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver -from tensorflow.python.distribute.cluster_resolver import TFConfigClusterResolver - -# pylint: disable=protected-access,invalid-name,line-too-long -CoreParameterServerStrategy = parameter_server_strategy.ParameterServerStrategy -CoreParameterServerExtended = parameter_server_strategy.ParameterServerStrategyExtended - -# pylint: enable=protected-access,invalid-name,line-too-long - - -class ParameterServerStrategy(distribute_lib.StrategyV1): - """A parameter server DistributionStrategy. - - *** contrib version *** - - This strategy class works for both local training and between-graph replicated - training for multiple workers. If `cluster_spec` is specified, either passed - in to __init__() method or parsed from the - ["TF_CONFIG" environment - variable](https://www.tensorflow.org/api_docs/python/tf/estimator/RunConfig), - variables and updates to those variables are assigned to parameter servers and - other operations are assigned to workers. If `cluster_spec` is not set, it - becomes local training where variables are assigned to local CPU or the only - GPU. When each worker has more than one GPU, operations will be replicated on - these GPUs. In both cases, operations are replicated but variables are not and - these workers share a common view for which parameter server a variable is - assigned to. - - This class assumes between-graph replication will be used and works on a graph - for a particular worker. Note that each graph and worker is independent. - This means that while each worker will synchronously compute a single gradient - update across all GPUs, updates between workers proceed asynchronously. - Operations that occur only on the first replica (such as incrementing the - global step), will occur on the first replica *of every worker*. - - It is expected to call `call_for_each_replica(fn, ...)` for any - operations which potentially can be replicated across replicas (i.e. multiple - GPUs) even if there is only CPU or one GPU. When defining the `fn`, extra - caution needs to be taken: - - 1) Always use `tf.compat.v1.get_variable` instead of `tf.Variable` which - is not able to refer to the same variable on different replicas. - - 2) It is generally not recommended to open a device scope under the strategy's - scope. A device scope (i.e. calling `tf.device`) will be merged with or - override the device for operations but will not change the device for - variables. - - 3) It is also not recommended to open a colocation scope (i.e. calling - `tf.compat.v1.colocate_with`) under the strategy's scope. For colocating - variables, use `strategy.extended.colocate_vars_with` instead. Colocation of - ops will possibly create conflicts of device assignment. - """ - - def __init__(self, num_gpus_per_worker=0): - """Initializes this strategy. - - Args: - num_gpus_per_worker: number of local GPUs or GPUs per worker, the default - is 0 meaning CPU only. - - Raises: - ValueError: if `cluster_spec` is given but `task_type` or `task_id` is - not. - """ - super(ParameterServerStrategy, self).__init__( - ParameterServerExtended(self, num_gpus_per_worker)) - - # Override to change the documentation to reflect the different handling of - # global vs. local batch size between core and contrib. - def make_dataset_iterator(self, dataset): # pylint: disable=useless-super-delegation - """Makes an iterator for input provided via `dataset`. - - NOTE: The batch size of the `dataset` argument is treated differently for - this contrib version of `ParameterServerStrategy`. - - Data from the given dataset will be distributed evenly across all the - compute replicas. We will assume that the input dataset is batched by the - per-replica batch size. - - The user could also use `make_input_fn_iterator` if they want to - customize which input is fed to which replica/worker etc. - - Args: - dataset: `tf.data.Dataset` that will be distributed evenly across all - replicas. - - Returns: - An `tf.distribute.InputIterator` which returns inputs for each step of the - computation. User should call `initialize` on the returned iterator. - """ - return super(ParameterServerStrategy, self).make_dataset_iterator(dataset) - - -class ParameterServerExtended(CoreParameterServerExtended): - """Implementation of ParameterServerStrategy.""" - - def __init__(self, container_strategy, num_gpus_per_worker): - # Use TFConfigClusterResolver to parse TF_CONFIG. We don't want to change - # the constructor's interface to allow customized cluster resolver. Use - # SimpleClusterResolver to override num_accelerators. - tfconfig = TFConfigClusterResolver() - cluster_resolver = SimpleClusterResolver( - cluster_spec=tfconfig.cluster_spec(), - task_type=tfconfig.task_type, - task_id=tfconfig.task_id, - num_accelerators={'GPU': num_gpus_per_worker}) - super(ParameterServerExtended, self).__init__( - container_strategy, cluster_resolver=cluster_resolver) - - def _make_dataset_iterator(self, dataset): - return input_lib.DatasetIterator(dataset, self._input_workers, - self._container_strategy()) - - # TODO(priyag): Delete this once all strategies use global batch size. - @property - def _global_batch_size(self): - """The contrib version of PS strategy uses per-replica batch size.""" - return False diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py deleted file mode 100644 index a4d5f0cf5a1..00000000000 --- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py +++ /dev/null @@ -1,852 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for ParameterServerStrategy.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import copy -import threading -from absl.testing import parameterized -from tensorflow.contrib.distribute.python import parameter_server_strategy -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import device_util -from tensorflow.python.distribute import distribution_strategy_context as ds_context -from tensorflow.python.distribute import multi_worker_test_base -from tensorflow.python.distribute import multi_worker_util -from tensorflow.python.distribute import reduce_util -from tensorflow.python.distribute import strategy_test_lib -from tensorflow.python.distribute import values -from tensorflow.python.eager import backprop -from tensorflow.python.eager import context -from tensorflow.python.estimator import run_config -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_util -from tensorflow.python.layers import core -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gradients -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import training_util - -CHIEF = run_config.TaskType.CHIEF -WORKER = run_config.TaskType.WORKER -PS = run_config.TaskType.PS - - -def _get_replica_id_integer(): - replica_id = ds_context.get_replica_context().replica_id_in_sync_group - if isinstance(replica_id, ops.Tensor): - replica_id = tensor_util.constant_value(replica_id) - return replica_id - - -def create_test_objects(cluster_spec=None, - task_type=None, - task_id=None, - num_gpus=None, - sess_config=None): - sess_config = sess_config or config_pb2.ConfigProto() - if num_gpus is None: - num_gpus = context.num_gpus() - - distribution = parameter_server_strategy.ParameterServerStrategy( - num_gpus_per_worker=num_gpus) - - if task_type: - sess_config = copy.deepcopy(sess_config) - distribution.configure( - session_config=sess_config, - cluster_spec=cluster_spec, - task_type=task_type, - task_id=task_id) - target = 'grpc://' + cluster_spec[WORKER][task_id] - else: - target = '' - - return distribution, target, sess_config - - -class ParameterServerStrategyTestBase( - multi_worker_test_base.MultiWorkerTestBase): - - def setUp(self): - self._result = 0 - self._lock = threading.Lock() - self._init_condition = threading.Condition() - self._init_reached = 0 - self._finish_condition = threading.Condition() - self._finish_reached = 0 - self._sess_config = config_pb2.ConfigProto(allow_soft_placement=True) - super(ParameterServerStrategyTestBase, self).setUp() - - def _get_test_objects(self, task_type, task_id, num_gpus): - return create_test_objects( - cluster_spec=self._cluster_spec, - task_type=task_type, - task_id=task_id, - num_gpus=num_gpus, - sess_config=self._sess_config) - - def _test_device_assignment_distributed(self, task_type, task_id, num_gpus): - worker_device = '/job:%s/replica:0/task:%d' % (task_type, task_id) - d, _, sess_config = self._get_test_objects(task_type, task_id, num_gpus) - with ops.Graph().as_default(), \ - self.cached_session(target=self._default_target, - config=sess_config) as sess, \ - d.scope(): - - # Define a variable outside the call_for_each_replica scope. - n = variable_scope.get_variable('n', initializer=10.0) - self.assertEqual(n.device, '/job:ps/task:0') - - def model_fn(): - if num_gpus == 0: - last_part_device = 'device:CPU:0' - else: - replica_id = _get_replica_id_integer() - last_part_device = ('device:GPU:%d' % replica_id) - - a = constant_op.constant(1.0) - b = constant_op.constant(2.0) - c = a + b - self.assertEqual(a.device, worker_device + '/' + last_part_device) - self.assertEqual(b.device, worker_device + '/' + last_part_device) - self.assertEqual(c.device, worker_device + '/' + last_part_device) - - # The device scope is ignored for variables but not for normal ops. - with ops.device('/job:worker/task:0'): - x = variable_scope.get_variable( - 'x', initializer=10.0, - aggregation=variable_scope.VariableAggregation.SUM) - x_add = x.assign_add(c) - e = a + c - # The variable x is on the task 1 since the device_function has been - # called once before the model_fn. - self.assertEqual(x.device, '/job:ps/task:1') - self.assertEqual(x_add.device, x.device) - self.assertEqual(e.device, - '/job:worker/replica:0/task:0/%s' % last_part_device) - - # The colocate_vars_with can override the distribution's device. - with d.extended.colocate_vars_with(x): - y = variable_scope.get_variable( - 'y', initializer=20.0, - aggregation=variable_scope.VariableAggregation.SUM) - # We add an identity here to avoid complaints about summing - # non-distributed values. - y_add = y.assign_add(array_ops.identity(x_add)) - self.assertEqual(y.device, '/job:ps/task:1') - self.assertEqual(y_add.device, y.device) - self.assertEqual(y.device, x.device) - - z = variable_scope.get_variable( - 'z', initializer=10.0, - aggregation=variable_scope.VariableAggregation.SUM) - self.assertEqual(z.device, '/job:ps/task:0') - self.assertNotEqual(z.device, x.device) - - with ops.control_dependencies([y_add]): - # We add an identity here to avoid complaints about summing - # non-distributed values. - z_add = z.assign_add(array_ops.identity(y)) - with ops.control_dependencies([z_add]): - f = z + c - self.assertEqual(f.device, worker_device + '/' + last_part_device) - - # The device scope would merge with the default worker device. - with ops.device('/CPU:1'): - g = e + 1.0 - self.assertEqual(g.device, worker_device + '/device:CPU:1') - - # Ths ops.colocate_with will be ignored when defining a variale but not - # for a normal tensor. - with ops.colocate_with(x): - u = variable_scope.get_variable('u', initializer=30.0) - v = variable_scope.get_variable('v', initializer=30.0) - h = f + 1.0 - self.assertIn('/job:ps/', u.device) - self.assertIn('/job:ps/', v.device) - # u and v are on different parameter servers. - self.assertTrue(u.device != x.device or v.device != x.device) - self.assertTrue(u.device == x.device or v.device == x.device) - # Here h is not on one worker. Note h.device is canonical while x.device - # is not but. - self.assertIn('/job:ps/', h.device) - return y_add, z_add, f - - y, z, f = d.extended.call_for_each_replica(model_fn) - self.assertNotEqual(y, None) - self.assertNotEqual(z, None) - self.assertNotEqual(f, None) - - if context.num_gpus() >= 1 and num_gpus <= 1: - variables.global_variables_initializer().run() - y_val, z_val, f_val = sess.run([y, z, f]) - self.assertEqual(y_val, 33.0) - self.assertEqual(z_val, 43.0) - self.assertEqual(f_val, 46.0) - - def _test_device_assignment_distributed_enable_partitioner( - self, task_type, task_id, num_gpus): - d, _, sess_config = self._get_test_objects(task_type, task_id, num_gpus) - num_shards = len(d.extended.parameter_devices) - partitioner = partitioned_variables.fixed_size_partitioner(num_shards) - with ops.Graph().as_default(), \ - self.cached_session(target=self._default_target, - config=sess_config) as sess, \ - d.scope(): - - n = variable_scope.get_variable( - 'n', - initializer=constant_op.constant([10.0, 20.0]), - aggregation=variable_scope.VariableAggregation.SUM, - partitioner=partitioner) - - for part_id, var in enumerate(n): - self.assertEqual(var.device, '/job:ps/task:%d' % part_id) - - def model_fn(): - a = constant_op.constant([3.0, 5.0]) - # The device scope is ignored for variables but not for normal ops. - with ops.device('/job:worker/task:0'): - x = variable_scope.get_variable( - 'x', - initializer=constant_op.constant([10.0, 20.0]), - aggregation=variable_scope.VariableAggregation.SUM, - partitioner=partitioner) - x_add = x.assign_add(a, name='x_add') - # The variable x is on the task 1 since the device_function has been - # called once before the model_fn. - for part_id, var in enumerate(x): - self.assertEqual(var.device, '/job:ps/task:%d' % part_id) - self.assertEqual(var.device, x_add[part_id].device) - - return x_add - - x = d.extended.call_for_each_replica(model_fn) - - if context.num_gpus() >= 1: - variables.global_variables_initializer().run() - x_val = sess.run(x) - if num_gpus < 1: - self.assertEqual(x_val, [13.0, 25.0]) - else: - x_expect = [10.0 + 3 * num_gpus, 20.0 + 5 * num_gpus] - self.assertEqual(x_val, x_expect) - - def _test_device_assignment_local(self, - d, - compute_device='CPU', - variable_device='CPU', - num_gpus=0): - with ops.Graph().as_default(), \ - self.cached_session(target=self._default_target, - config=self._sess_config) as sess, \ - d.scope(): - - def model_fn(): - if 'CPU' in compute_device: - replica_compute_device = '/device:CPU:0' - else: - replica_id = _get_replica_id_integer() - replica_compute_device = ('/device:GPU:%d' % replica_id) - replica_compute_device = device_util.canonicalize( - replica_compute_device) - - if 'CPU' in variable_device: - replica_variable_device = '/device:CPU:0' - else: - replica_id = _get_replica_id_integer() - replica_variable_device = ('/device:GPU:%d' % replica_id) - replica_variable_device = device_util.canonicalize( - replica_variable_device) - - a = constant_op.constant(1.0) - b = constant_op.constant(2.0) - c = a + b - self.assertEqual(a.device, replica_compute_device) - self.assertEqual(b.device, replica_compute_device) - self.assertEqual(c.device, replica_compute_device) - - # The device scope is ignored for variables but not for normal ops. - with ops.device('/device:GPU:2'): - x = variable_scope.get_variable( - 'x', initializer=10.0, - aggregation=variable_scope.VariableAggregation.SUM) - x_add = x.assign_add(c) - e = a + c - self.assertEqual( - device_util.canonicalize(x.device), replica_variable_device) - self.assertEqual(x_add.device, x.device) - self.assertEqual(e.device, device_util.canonicalize('/device:GPU:2')) - - # The colocate_vars_with can override the distribution's device. - with d.extended.colocate_vars_with(x): - y = variable_scope.get_variable( - 'y', initializer=20.0, - aggregation=variable_scope.VariableAggregation.SUM) - # We add an identity here to avoid complaints about summing - # non-distributed values. - y_add = y.assign_add(array_ops.identity(x_add)) - self.assertEqual( - device_util.canonicalize(y.device), replica_variable_device) - self.assertEqual(y_add.device, y.device) - self.assertEqual(y.device, x.device) - - z = variable_scope.get_variable( - 'z', initializer=10.0, - aggregation=variable_scope.VariableAggregation.SUM) - self.assertEqual( - device_util.canonicalize(z.device), replica_variable_device) - - with ops.control_dependencies([y_add]): - # We add an identity here to avoid complaints about summing - # non-distributed values. - z_add = z.assign_add(array_ops.identity(y)) - with ops.control_dependencies([z_add]): - f = z + c - self.assertEqual(f.device, replica_compute_device) - - # The device scope would merge with the default worker device. - with ops.device('/CPU:1'): - g = e + 1.0 - self.assertEqual(g.device, device_util.canonicalize('/device:CPU:1')) - - # Ths ops.colocate_with will be ignored when defining a variale but not - # for a normal tensor. - with ops.colocate_with(x): - u = variable_scope.get_variable('u', initializer=30.0) - h = f + 1.0 - self.assertEqual( - device_util.canonicalize(u.device), replica_variable_device) - self.assertEqual( - device_util.canonicalize(x.device), - device_util.canonicalize(h.device)) - return y_add, z_add, f - - y, z, f = d.extended.call_for_each_replica(model_fn) - self.assertNotEqual(y, None) - self.assertNotEqual(z, None) - self.assertNotEqual(f, None) - - if context.num_gpus() >= 1 and num_gpus <= 1: - variables.global_variables_initializer().run() - y_val, z_val, f_val = sess.run([y, z, f]) - self.assertEqual(y_val, 33.0) - self.assertEqual(z_val, 43.0) - self.assertEqual(f_val, 46.0) - - def _test_simple_increment(self, task_type, task_id, num_gpus): - d, master_target, sess_config = self._get_test_objects( - task_type, task_id, num_gpus) - if d.extended._cluster_spec: - num_workers = len(d.extended._cluster_spec.as_dict().get(WORKER)) - if 'chief' in d.extended._cluster_spec.as_dict(): - num_workers += 1 - else: - num_workers = 1 - with ops.Graph().as_default(), \ - self.cached_session(target=master_target, - config=sess_config) as sess, \ - d.scope(): - - def model_fn(): - x = variable_scope.get_variable( - 'x', initializer=10.0, - aggregation=variable_scope.VariableAggregation.SUM) - y = variable_scope.get_variable( - 'y', initializer=20.0, - aggregation=variable_scope.VariableAggregation.SUM) - z = variable_scope.get_variable( - 'z', initializer=30.0, - aggregation=variable_scope.VariableAggregation.ONLY_FIRST_REPLICA) - - # We explicitly make a constant tensor here to avoid complaints about - # summing non-distributed values. - one = constant_op.constant(1.0) - x_add = x.assign_add(one, use_locking=True) - y_add = y.assign_add(one, use_locking=True) - z_add = z.assign_add(one, use_locking=True) - - train_op = control_flow_ops.group(x_add, y_add, z_add) - return x, y, z, train_op - - x, y, z, train_op = d.extended.call_for_each_replica(model_fn) - train_op = d.group(train_op) - - if context.num_gpus() < sum( - 1 for d in d.extended.worker_devices if 'GPU' in d.upper()): - return True - - if task_id == 0: - variables.global_variables_initializer().run() - - # Workers waiting for chief worker's initializing variables. - self._init_condition.acquire() - self._init_reached += 1 - while self._init_reached != num_workers: - self._init_condition.wait() - self._init_condition.notify_all() - self._init_condition.release() - - sess.run(train_op) - - # Wait for other workers to finish training. - self._finish_condition.acquire() - self._finish_reached += 1 - while self._finish_reached != num_workers: - self._finish_condition.wait() - self._finish_condition.notify_all() - self._finish_condition.release() - - x_val, y_val, z_val = sess.run([x, y, z]) - self.assertEqual(x_val, 10.0 + 1.0 * num_workers * d.num_replicas_in_sync) - self.assertEqual(y_val, 20.0 + 1.0 * num_workers * d.num_replicas_in_sync) - self.assertEqual(z_val, 30.0 + 1.0 * num_workers) - - def _test_minimize_loss_graph(self, task_type, task_id, num_gpus): - d, master_target, sess_config = self._get_test_objects( - task_type, task_id, num_gpus) - if task_type: - # Multi-worker - assert hasattr(d.extended, '_cluster_spec') and d.extended._cluster_spec - num_workers = len(d.extended._cluster_spec.as_dict().get(WORKER)) - if CHIEF in d.extended._cluster_spec.as_dict(): - num_workers += 1 - else: - # local - num_workers = 1 - - with ops.Graph().as_default(), \ - self.cached_session(target=master_target, - config=sess_config) as sess, \ - d.scope(): - l = core.Dense(1, use_bias=False) - - def loss_fn(x): - y = array_ops.reshape(l(x), []) - constant_op.constant(1.) - return y * y - - # TODO(yuefengz, apassos): eager.backprop.implicit_grad is not safe for - # multiple graphs (b/111216820). - def grad_fn(x): - loss = loss_fn(x) - var_list = ( - variables.trainable_variables() + ops.get_collection( - ops.GraphKeys.TRAINABLE_RESOURCE_VARIABLES)) - grads = gradients.gradients(loss, var_list) - ret = list(zip(grads, var_list)) - return ret - - def update(v, g): - return v.assign_sub(0.05 * g, use_locking=True) - - one = constant_op.constant([[1.]]) - - def step(): - """Perform one optimization step.""" - # Run forward & backward to get gradients, variables list. - g_v = d.extended.call_for_each_replica(grad_fn, args=(one,)) - # Update the variables using the gradients and the update() function. - before_list = [] - after_list = [] - for g, v in g_v: - fetched = d.extended.read_var(v) - before_list.append(fetched) - with ops.control_dependencies([fetched]): - # TODO(yuefengz): support non-Mirrored variable as destinations. - g = d.extended.reduce_to( - reduce_util.ReduceOp.SUM, g, destinations=v) - with ops.control_dependencies( - d.extended.update(v, update, args=(g,), group=False)): - after_list.append(d.extended.read_var(v)) - return before_list, after_list - - before_out, after_out = step() - - if context.num_gpus() < sum( - 1 for d in d.extended.worker_devices if 'GPU' in d.upper()): - return True - - if (not task_type or - multi_worker_util.is_chief( - d.extended._cluster_spec, task_type, task_id)): - variables.global_variables_initializer().run() - - # Workers waiting for chief worker's initializing variables. - self._init_condition.acquire() - self._init_reached += 1 - while self._init_reached != num_workers: - self._init_condition.wait() - self._init_condition.notify_all() - self._init_condition.release() - - for i in range(10): - b, a = sess.run((before_out, after_out)) - if i == 0: - before, = b - after, = a - - error_before = abs(before - 1) - error_after = abs(after - 1) - # Error should go down - self.assertLess(error_after, error_before) - - def _test_input_fn_iterator(self, - task_type, - task_id, - num_gpus, - input_fn, - expected_values, - test_reinitialize=True, - ignore_order=False): - distribution, master_target, config = self._get_test_objects( - task_type, task_id, num_gpus) - devices = distribution.extended.worker_devices - - with ops.Graph().as_default(), \ - self.cached_session(config=config, - target=master_target) as sess: - iterator = distribution.make_input_fn_iterator(input_fn) - sess.run(iterator.initialize()) - - for expected_value in expected_values: - next_element = iterator.get_next() - computed_value = sess.run([values.select_replica(r, next_element) - for r in range(len(devices))]) - if ignore_order: - self.assertCountEqual(expected_value, computed_value) - else: - self.assertEqual(expected_value, computed_value) - - with self.assertRaises(errors.OutOfRangeError): - next_element = iterator.get_next() - sess.run([values.select_replica(r, next_element) - for r in range(len(devices))]) - - # After re-initializing the iterator, should be able to iterate again. - if test_reinitialize: - sess.run(iterator.initialize()) - - for expected_value in expected_values: - next_element = iterator.get_next() - computed_value = sess.run([values.select_replica(r, next_element) - for r in range(len(devices))]) - if ignore_order: - self.assertCountEqual(expected_value, computed_value) - else: - self.assertEqual(expected_value, computed_value) - - -class ParameterServerStrategyTest( - ParameterServerStrategyTestBase, - strategy_test_lib.DistributionTestBase, - strategy_test_lib.TwoDeviceDistributionTestBase, - parameterized.TestCase): - - @classmethod - def setUpClass(cls): - cls._cluster_spec = multi_worker_test_base.create_in_process_cluster( - num_workers=3, num_ps=2) - cls._default_target = 'grpc://' + cls._cluster_spec[WORKER][0] - - @combinations.generate(combinations.combine(mode=['graph'])) - def test_num_replicas_in_sync(self): - strategy, _, _ = create_test_objects(num_gpus=2) - # All the devices on a given worker are in sync which in this case is the - # number of gpus on each worker. - self.assertEqual(2, strategy.num_replicas_in_sync) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testDeviceAssignmentLocalCPU(self): - strategy, _, _ = create_test_objects(num_gpus=0) - self._test_device_assignment_local( - strategy, compute_device='CPU', variable_device='CPU', num_gpus=0) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testDeviceAssignmentLocalOneGPU(self): - strategy, _, _ = create_test_objects(num_gpus=1) - self._test_device_assignment_local( - strategy, compute_device='GPU', variable_device='GPU', num_gpus=1) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testDeviceAssignmentLocalTwoGPUs(self): - strategy, _, _ = create_test_objects(num_gpus=2) - self._test_device_assignment_local( - strategy, compute_device='GPU', variable_device='CPU', num_gpus=2) - - @combinations.generate( - combinations.combine(mode=['graph'], num_gpus=[0, 1, 2])) - def testDeviceAssignmentDistributed(self, num_gpus): - self._test_device_assignment_distributed('worker', 1, num_gpus) - - @combinations.generate( - combinations.combine(mode=['graph'], num_gpus=[0, 1, 2])) - def testDeviceAssignmentDistributedEnablePartitioner(self, num_gpus): - self._test_device_assignment_distributed_enable_partitioner( - 'worker', 1, num_gpus) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testSimpleBetweenGraph(self): - self._run_between_graph_clients(self._test_simple_increment, - self._cluster_spec, context.num_gpus()) - - @combinations.generate( - combinations.combine(mode=['graph'], num_gpus=[0, 1, 2])) - def testLocalSimpleIncrement(self, num_gpus): - self._test_simple_increment(None, 0, num_gpus) - - @combinations.generate( - combinations.combine(mode=['graph'], num_gpus=[0, 1, 2])) - def testMinimizeLossGraphDistributed(self, num_gpus): - self._run_between_graph_clients(self._test_minimize_loss_graph, - self._cluster_spec, num_gpus) - - @combinations.generate( - combinations.combine(mode=['graph'], num_gpus=[0, 1, 2])) - def testMinimizeLossGraphLocal(self, num_gpus): - self._test_minimize_loss_graph(None, None, num_gpus) - - # TODO(priyag): Refactor this and other multi worker tests. - @combinations.generate( - combinations.combine( - mode=['graph'], - num_gpus=[1, 2], - required_gpus=1, - use_dataset=[True, False])) - def testMakeInputFnIteratorDistributed(self, num_gpus, use_dataset): - if context.num_gpus() < num_gpus: - self.skipTest('Not enough GPUs') - if use_dataset: - fn = lambda: dataset_ops.Dataset.range(100) - else: - def fn(): - dataset = dataset_ops.Dataset.range(100) - it = dataset.make_one_shot_iterator() - return it.get_next - expected_values = [[i+j for j in range(num_gpus)] - for i in range(0, 100, num_gpus)] - - input_fn = self._input_fn_to_test_input_context( - fn, - expected_num_replicas_in_sync=num_gpus, - expected_num_input_pipelines=3, - expected_input_pipeline_id=1) # because task_id = 1 - self._test_input_fn_iterator( - 'worker', - 1, - num_gpus, - input_fn, - expected_values, - test_reinitialize=use_dataset, - ignore_order=not use_dataset) - - @combinations.generate( - combinations.combine( - mode=['graph'], - num_gpus=[1, 2], - required_gpus=1, - use_dataset=[True, False])) - def testMakeInputFnIteratorLocal(self, num_gpus, use_dataset): - if context.num_gpus() < num_gpus: - self.skipTest('Not enough GPUs') - if use_dataset: - fn = lambda: dataset_ops.Dataset.range(100) - else: - def fn(): - dataset = dataset_ops.Dataset.range(100) - it = dataset.make_one_shot_iterator() - return it.get_next - expected_values = [[i+j for j in range(num_gpus)] - for i in range(0, 100, num_gpus)] - - input_fn = self._input_fn_to_test_input_context( - fn, - expected_num_replicas_in_sync=num_gpus, - expected_num_input_pipelines=1, - expected_input_pipeline_id=0) # only one worker and pipeline for local. - self._test_input_fn_iterator( - None, - None, - num_gpus, - input_fn, - expected_values, - test_reinitialize=use_dataset, - ignore_order=not use_dataset) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testGlobalStepUpdate(self): - strategy, _, _ = create_test_objects() - self._test_global_step_update(strategy) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testUpdateConfigProtoMultiWorker(self): - strategy, _, _ = create_test_objects( - cluster_spec=self._cluster_spec, - task_type='worker', - task_id=1, - num_gpus=2) - - config_proto = config_pb2.ConfigProto(device_filters=['to_be_overridden']) - - new_config = strategy.update_config_proto(config_proto) - - # Verify device filters. - self.assertEqual(['/job:worker/task:1', '/job:ps'], - new_config.device_filters) - - # Verify isolate_session_state - self.assertFalse(new_config.isolate_session_state) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testUpdateConfigProtoLocal(self): - strategy, _, _ = create_test_objects(num_gpus=2) - - config_proto = config_pb2.ConfigProto() - new_config = strategy.update_config_proto(config_proto) - - # Verify isolate_session_state - self.assertTrue(new_config.isolate_session_state) - - @combinations.generate(combinations.combine(required_gpus=[2])) - def testAllReduceSum(self): - distribution = parameter_server_strategy.ParameterServerStrategy( - num_gpus_per_worker=2) - self._test_all_reduce_sum(distribution) - - @combinations.generate(combinations.combine(required_gpus=[2])) - def testAllReduceSumGradients(self): - distribution = parameter_server_strategy.ParameterServerStrategy( - num_gpus_per_worker=2) - self._test_all_reduce_sum_gradients(distribution) - - @combinations.generate(combinations.combine(required_gpus=[2])) - def testAllReduceSumGradientTape(self): - distribution = parameter_server_strategy.ParameterServerStrategy( - num_gpus_per_worker=2) - self._test_all_reduce_sum_gradient_tape(distribution) - - @combinations.generate(combinations.combine(required_gpus=[2])) - def testAllReduceMean(self): - distribution = parameter_server_strategy.ParameterServerStrategy( - num_gpus_per_worker=2) - self._test_all_reduce_mean(distribution) - - @combinations.generate(combinations.combine(required_gpus=[2])) - def testAllReduceMeanGradients(self): - distribution = parameter_server_strategy.ParameterServerStrategy( - num_gpus_per_worker=2) - self._test_all_reduce_mean_gradients(distribution) - - @combinations.generate(combinations.combine(required_gpus=[2])) - def testAllReduceMeanGradientTape(self): - distribution = parameter_server_strategy.ParameterServerStrategy( - num_gpus_per_worker=2) - self._test_all_reduce_mean_gradient_tape(distribution) - - def testTrainableVariables(self): - distribution = parameter_server_strategy.ParameterServerStrategy() - self._test_trainable_variable(distribution) - - -class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase, - parameterized.TestCase): - - @classmethod - def setUpClass(cls): - cls._cluster_spec = multi_worker_test_base.create_in_process_cluster( - num_workers=3, num_ps=2, has_chief=True) - cls._default_target = 'grpc://' + cls._cluster_spec[CHIEF][0] - - @combinations.generate(combinations.combine(mode=['graph'])) - def testSimpleBetweenGraph(self): - self._run_between_graph_clients(self._test_simple_increment, - self._cluster_spec, context.num_gpus()) - - @combinations.generate( - combinations.combine(mode=['graph'], num_gpus=[0, 1, 2])) - def testMinimizeLossGraph(self, num_gpus): - self._run_between_graph_clients(self._test_minimize_loss_graph, - self._cluster_spec, num_gpus) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testGlobalStepIsWrappedOnTwoGPUs(self): - strategy, _, _ = create_test_objects(num_gpus=2) - with ops.Graph().as_default(), strategy.scope(): - created_step = training_util.create_global_step() - get_step = training_util.get_global_step() - self.assertEqual(created_step, get_step, - msg=('created_step %s type %s vs. get_step %s type %s' % - (id(created_step), created_step.__class__.__name__, - id(get_step), get_step.__class__.__name__))) - self.assertIs(values.AggregatingVariable, type(created_step)) - self.assertIs(values.AggregatingVariable, type(get_step)) - self.assertIs(strategy, created_step.distribute_strategy) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testGlobalStepIsNotWrappedOnOneGPU(self): - strategy, _, _ = create_test_objects(num_gpus=1) - with ops.Graph().as_default(), strategy.scope(): - created_step = training_util.create_global_step() - get_step = training_util.get_global_step() - self.assertEqual(created_step, get_step, - msg=('created_step %s type %s vs. get_step %s type %s' % - (id(created_step), created_step.__class__.__name__, - id(get_step), get_step.__class__.__name__))) - self.assertIs(resource_variable_ops.ResourceVariable, type(created_step)) - self.assertIs(resource_variable_ops.ResourceVariable, type(get_step)) - # All variables have an _distribute_strategy parameter. Only variable - # subclasses in distribution strategy expose it publicly. - self.assertFalse(hasattr(strategy, 'distribute_strategy')) - self.assertIs(strategy, created_step._distribute_strategy) - - @combinations.generate(combinations.combine(mode=['graph'])) - def testValueContainer(self): - strategy, _, _ = create_test_objects(num_gpus=2) - with ops.Graph().as_default(), strategy.scope(): - - def f(): - with backprop.GradientTape() as tape: - v = variable_scope.get_variable('v', initializer=10.0) - _ = v * v - v, = tape.watched_variables() - w = strategy.extended.value_container(v) - self.assertIs(values.AggregatingVariable, type(w)) - - strategy.extended.call_for_each_replica(f) - - -class CentralStorageStrategyTest(strategy_test_lib.DistributionTestBase, - parameterized.TestCase): - - @combinations.generate(combinations.combine(mode=['graph', 'eager'], - required_gpus=2)) - def testNumpyDataset(self): - strategy, _, _ = create_test_objects(num_gpus=2) - self._test_numpy_dataset(strategy) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py deleted file mode 100644 index 88f97dd9226..00000000000 --- a/tensorflow/contrib/distribute/python/tpu_strategy.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""TPU Distribution Strategy. - -This is experimental. It's not ready for general use. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.python.distribute.tpu_strategy import TPUStrategyV1 as TPUStrategy -from tensorflow.python.tpu.tpu_strategy_util import initialize_tpu_system diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD deleted file mode 100644 index c6ebd8594e9..00000000000 --- a/tensorflow/contrib/distributions/BUILD +++ /dev/null @@ -1,1339 +0,0 @@ -# Description: -# Contains ops for statistical distributions (with pdf, cdf, sample, etc...). -# APIs here are meant to evolve over time. - -load("//tensorflow:tensorflow.bzl", "cuda_py_test") - -package( - default_visibility = [ - "//learning/brain/contrib/bayesflow:__subpackages__", - "//tensorflow:__subpackages__", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "bijectors_py", - srcs = glob(["python/ops/bijectors/*.py"]), - deprecation = ("TensorFlow Distributions has migrated to " + - "TensorFlow Probability " + - "(https://github.com/tensorflow/probability). " + - "Deprecated copies remaining in tf.contrib.distributions " + - "are unmaintained, unsupported, and will be removed by " + - "late 2018. You should update all usage of " + - "`tf.contrib.distributions` to `tfp.distributions`."), - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:clip_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:layers", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:template", - "//tensorflow/python:tensor_util", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python/ops/distributions", - "//tensorflow/python/ops/linalg", - "//third_party/py/numpy", - ], -) - -py_library( - name = "distributions_py", - srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - deprecation = ("TensorFlow Distributions has migrated to " + - "TensorFlow Probability " + - "(https://github.com/tensorflow/probability). " + - "Deprecated copies remaining in tf.contrib.distributions " + - "are unmaintained, unsupported, and will be removed by " + - "late 2018. You should update all usage of " + - "`tf.contrib.distributions` to `tfp.distributions`."), - srcs_version = "PY2AND3", - deps = [ - ":bijectors_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:histogram_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:nn_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:tensor_util", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/ops/distributions", - "//tensorflow/python/ops/linalg", - "//tensorflow/python/ops/signal", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -cuda_py_test( - name = "estimator_test", - size = "small", - srcs = ["python/kernel_tests/estimator_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/contrib/learn", - "//tensorflow/contrib/learn:head_test_lib", - "//tensorflow/python/ops/distributions", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:session", - ], - tags = ["no_pip"], # contrib/learn:head_test is not available in pip. -) - -cuda_py_test( - name = "distribution_test", - size = "medium", - srcs = ["python/kernel_tests/distribution_test.py"], - additional_deps = [ - ":distributions_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - ], -) - -cuda_py_test( - name = "conditional_distribution_test", - size = "medium", - srcs = [ - "python/kernel_tests/conditional_distribution_test.py", - "python/kernel_tests/distribution_test.py", - ], - additional_deps = [ - ":distributions_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - ], - tags = ["no_pip"], -) - -cuda_py_test( - name = "autoregressive_test", - size = "small", - srcs = ["python/kernel_tests/autoregressive_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform_test", - ], - tags = [ - "no_oss", # b/134605801 - ], -) - -cuda_py_test( - name = "binomial_test", - size = "small", - srcs = ["python/kernel_tests/binomial_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform_test", - ], - tags = ["notap"], -) - -cuda_py_test( - name = "cauchy_test", - size = "medium", - srcs = ["python/kernel_tests/cauchy_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:platform_test", - "//tensorflow/python:variables", - ], -) - -cuda_py_test( - name = "chi2_test", - srcs = ["python/kernel_tests/chi2_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "deterministic_test", - size = "small", - srcs = ["python/kernel_tests/deterministic_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "geometric_test", - size = "small", - srcs = ["python/kernel_tests/geometric_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "half_normal_test", - size = "medium", - srcs = ["python/kernel_tests/half_normal_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:variables", - ], -) - -cuda_py_test( - name = "inverse_gamma_test", - srcs = ["python/kernel_tests/inverse_gamma_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "kumaraswamy_test", - srcs = ["python/kernel_tests/kumaraswamy_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], - tags = [ - # TODO(b/134605801): Re-enable this test. - "no_oss", - ], -) - -cuda_py_test( - name = "moving_stats_test", - size = "small", - srcs = ["python/kernel_tests/moving_stats_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], - tags = [ - "nomsan", # disable to avoid false positives from scipy. - "notap", # TODO(b/130421237) - ], -) - -cuda_py_test( - name = "mvn_diag_test", - size = "medium", - srcs = ["python/kernel_tests/mvn_diag_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "mvn_diag_plus_low_rank_test", - size = "medium", - srcs = ["python/kernel_tests/mvn_diag_plus_low_rank_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "mvn_full_covariance_test", - size = "small", - srcs = ["python/kernel_tests/mvn_full_covariance_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "mvn_tril_test", - size = "medium", - srcs = ["python/kernel_tests/mvn_tril_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], - tags = ["nomsan"], -) - -cuda_py_test( - name = "mixture_test", - size = "medium", - srcs = ["python/kernel_tests/mixture_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - "//tensorflow/python:variables", - ], - shard_count = 4, -) - -cuda_py_test( - name = "mixture_same_family_test", - size = "small", - srcs = ["python/kernel_tests/mixture_same_family_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:client_testlib", - ], -) - -cuda_py_test( - name = "negative_binomial_test", - size = "small", - srcs = ["python/kernel_tests/negative_binomial_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "poisson_test", - size = "small", - srcs = ["python/kernel_tests/poisson_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "poisson_lognormal_test", - size = "medium", - srcs = ["python/kernel_tests/poisson_lognormal_test.py"], - additional_deps = [ - ":distributions_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "sinh_arcsinh_test", - size = "medium", - srcs = ["python/kernel_tests/sinh_arcsinh_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "independent_test", - size = "small", - srcs = ["python/kernel_tests/independent_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], - tags = ["no_windows"], # TODO: needs investigation on Windows -) - -cuda_py_test( - name = "batch_reshape_test", - size = "medium", - srcs = ["python/kernel_tests/batch_reshape_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "sample_stats_test", - size = "medium", - srcs = ["python/kernel_tests/sample_stats_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], - tags = ["nomsan"], # disable to avoid false positives from scipy. - xla_enable_strict_auto_jit = False, -) - -cuda_py_test( - name = "seed_stream_test", - size = "small", - srcs = ["python/kernel_tests/seed_stream_test.py"], - additional_deps = [ - ":distributions_py", - "//tensorflow/python:client_testlib", - ], - tags = [ - "no_oss", # b/134605801 - ], -) - -cuda_py_test( - name = "statistical_testing_test", - size = "medium", - srcs = [ - "python/kernel_tests/statistical_testing_test.py", - ], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - ], - shard_count = 4, -) - -cuda_py_test( - name = "vector_sinh_arcsinh_diag_test", - size = "medium", - srcs = ["python/kernel_tests/vector_sinh_arcsinh_diag_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "vector_exponential_diag_test", - size = "medium", - srcs = ["python/kernel_tests/vector_exponential_diag_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "vector_laplace_diag_test", - size = "medium", - srcs = ["python/kernel_tests/vector_laplace_diag_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "vector_student_t_test", - size = "medium", - srcs = ["python/kernel_tests/vector_student_t_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "wishart_test", - size = "medium", - srcs = ["python/kernel_tests/wishart_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "normal_conjugate_posteriors_test", - size = "small", - srcs = ["python/kernel_tests/normal_conjugate_posteriors_test.py"], - additional_deps = [ - ":distributions_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "quantized_distribution_test", - size = "medium", - srcs = ["python/kernel_tests/quantized_distribution_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:gradients", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:variables", - ], -) - -cuda_py_test( - name = "logistic_test", - size = "small", - srcs = ["python/kernel_tests/logistic_test.py"], - additional_deps = [ - ":distributions_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "relaxed_bernoulli_test", - size = "small", - srcs = ["python/kernel_tests/relaxed_bernoulli_test.py"], - additional_deps = [ - ":distributions_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "onehot_categorical_test", - size = "small", - srcs = ["python/kernel_tests/onehot_categorical_test.py"], - additional_deps = [ - ":distributions_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "relaxed_onehot_categorical_test", - size = "small", - srcs = ["python/kernel_tests/relaxed_onehot_categorical_test.py"], - additional_deps = [ - ":distributions_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "transformed_distribution_test", - size = "medium", - srcs = ["python/kernel_tests/transformed_distribution_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "vector_diffeomixture_test", - size = "medium", - srcs = ["python/kernel_tests/vector_diffeomixture_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python/ops/linalg", - ], - shard_count = 4, - tags = ["noasan"], # times out, http://b/78588814 -) - -cuda_py_test( - name = "conditional_transformed_distribution_test", - size = "medium", - srcs = [ - "python/kernel_tests/conditional_transformed_distribution_test.py", - "python/kernel_tests/transformed_distribution_test.py", - ], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], - tags = ["no_pip"], -) - -cuda_py_test( - name = "distribution_util_test", - size = "small", - srcs = ["python/kernel_tests/distribution_util_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "shape_test", - size = "medium", - srcs = ["python/kernel_tests/shape_test.py"], - additional_deps = [ - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -# === Bijector Tests ========================================================== - -cuda_py_test( - name = "conditional_bijector_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/conditional_bijector_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "absolute_value_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/absolute_value_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "affine_test", - size = "medium", - srcs = ["python/kernel_tests/bijectors/affine_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], - shard_count = 10, - tags = ["noasan"], # times out b/63678675 -) - -cuda_py_test( - name = "affine_scalar_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/affine_scalar_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "affine_linear_operator_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/affine_linear_operator_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "batch_normalization_test", - size = "medium", - srcs = ["python/kernel_tests/bijectors/batch_normalization_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], - tags = ["optonly"], -) - -cuda_py_test( - name = "chain_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/chain_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "cholesky_outer_product_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/cholesky_outer_product_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "exp_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/exp_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "fill_triangular_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/fill_triangular_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "gumbel_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/gumbel_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "inline_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/inline_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "invert_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/invert_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "kumaraswamy_bijector_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/kumaraswamy_bijector_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "masked_autoregressive_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/masked_autoregressive_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "matrix_inverse_tril_test", - size = "medium", - srcs = ["python/kernel_tests/bijectors/matrix_inverse_tril_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "real_nvp_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/real_nvp_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "permute_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/permute_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "power_transform_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/power_transform_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "reshape_test", - size = "medium", - srcs = ["python/kernel_tests/bijectors/reshape_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "scale_tril_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/scale_tril_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "sigmoid_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/sigmoid_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -# Tests for SinhArcSinh bijector. The file name has the extra "_bijector" to -# avoid BUILD rule name conflicts with the distribution by the same name. -cuda_py_test( - name = "sinh_arcsinh_bijector_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/sinh_arcsinh_bijector_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], - tags = ["no_windows"], # TODO: needs investigation on Windows -) - -cuda_py_test( - name = "softmax_centered_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/softmax_centered_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "softplus_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/softplus_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "softsign_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/softsign_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "square_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/square_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "transform_diagonal_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/transform_diagonal_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "weibull_test", - size = "small", - srcs = ["python/kernel_tests/bijectors/weibull_test.py"], - additional_deps = [ - ":bijectors_py", - ":distributions_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python/ops/linalg", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) diff --git a/tensorflow/contrib/distributions/__init__.py b/tensorflow/contrib/distributions/__init__.py deleted file mode 100644 index 5f6b7fe3099..00000000000 --- a/tensorflow/contrib/distributions/__init__.py +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Classes representing statistical distributions and ops for working with them. - -Use [tfp.distributions](/probability/api_docs/python/tfp/distributions) instead. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.util import deprecation - - -# pylint: disable=unused-import,wildcard-import,line-too-long,g-importing-member,g-import-not-at-top - -with deprecation.silence(): - from tensorflow.contrib.distributions.python.ops import bijectors - from tensorflow.contrib.distributions.python.ops.autoregressive import * - from tensorflow.contrib.distributions.python.ops.batch_reshape import * - from tensorflow.contrib.distributions.python.ops.binomial import * - from tensorflow.contrib.distributions.python.ops.cauchy import * - from tensorflow.contrib.distributions.python.ops.chi2 import * - from tensorflow.contrib.distributions.python.ops.conditional_distribution import * - from tensorflow.contrib.distributions.python.ops.conditional_transformed_distribution import * - from tensorflow.contrib.distributions.python.ops.deterministic import * - from tensorflow.contrib.distributions.python.ops.distribution_util import fill_triangular - from tensorflow.contrib.distributions.python.ops.distribution_util import fill_triangular_inverse - from tensorflow.contrib.distributions.python.ops.distribution_util import matrix_diag_transform - from tensorflow.contrib.distributions.python.ops.distribution_util import reduce_weighted_logsumexp - from tensorflow.contrib.distributions.python.ops.distribution_util import softplus_inverse - from tensorflow.contrib.distributions.python.ops.distribution_util import tridiag - from tensorflow.contrib.distributions.python.ops.estimator import * - from tensorflow.contrib.distributions.python.ops.geometric import * - from tensorflow.contrib.distributions.python.ops.half_normal import * - from tensorflow.contrib.distributions.python.ops.independent import * - from tensorflow.contrib.distributions.python.ops.inverse_gamma import * - from tensorflow.contrib.distributions.python.ops.kumaraswamy import * - from tensorflow.contrib.distributions.python.ops.logistic import * - from tensorflow.contrib.distributions.python.ops.mixture import * - from tensorflow.contrib.distributions.python.ops.mixture_same_family import * - from tensorflow.contrib.distributions.python.ops.moving_stats import * - from tensorflow.contrib.distributions.python.ops.mvn_diag import * - from tensorflow.contrib.distributions.python.ops.mvn_diag_plus_low_rank import * - from tensorflow.contrib.distributions.python.ops.mvn_full_covariance import * - from tensorflow.contrib.distributions.python.ops.mvn_tril import * - from tensorflow.contrib.distributions.python.ops.negative_binomial import * - from tensorflow.contrib.distributions.python.ops.normal_conjugate_posteriors import * - from tensorflow.contrib.distributions.python.ops.onehot_categorical import * - from tensorflow.contrib.distributions.python.ops.poisson import * - from tensorflow.contrib.distributions.python.ops.poisson_lognormal import * - from tensorflow.contrib.distributions.python.ops.quantized_distribution import * - from tensorflow.contrib.distributions.python.ops.relaxed_bernoulli import * - from tensorflow.contrib.distributions.python.ops.relaxed_onehot_categorical import * - from tensorflow.contrib.distributions.python.ops.sample_stats import * - from tensorflow.contrib.distributions.python.ops.seed_stream import * - from tensorflow.contrib.distributions.python.ops.sinh_arcsinh import * - from tensorflow.contrib.distributions.python.ops.test_util import * - from tensorflow.contrib.distributions.python.ops.vector_diffeomixture import * - from tensorflow.contrib.distributions.python.ops.vector_exponential_diag import * - from tensorflow.contrib.distributions.python.ops.vector_laplace_diag import * - from tensorflow.contrib.distributions.python.ops.vector_sinh_arcsinh_diag import * - from tensorflow.contrib.distributions.python.ops.wishart import * - from tensorflow.python.ops.distributions.bernoulli import * - from tensorflow.python.ops.distributions.beta import * - from tensorflow.python.ops.distributions.categorical import * - from tensorflow.python.ops.distributions.dirichlet import * - from tensorflow.python.ops.distributions.dirichlet_multinomial import * - from tensorflow.python.ops.distributions.distribution import * - from tensorflow.python.ops.distributions.exponential import * - from tensorflow.python.ops.distributions.gamma import * - from tensorflow.python.ops.distributions.kullback_leibler import * - from tensorflow.python.ops.distributions.laplace import * - from tensorflow.python.ops.distributions.multinomial import * - from tensorflow.python.ops.distributions.normal import * - from tensorflow.python.ops.distributions.student_t import * - from tensorflow.python.ops.distributions.transformed_distribution import * - from tensorflow.python.ops.distributions.uniform import * - -# pylint: enable=unused-import,wildcard-import,line-too-long,g-importing-member - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'auto_correlation', - 'bijectors', - 'Cauchy', - 'ConditionalDistribution', - 'ConditionalTransformedDistribution', - 'FULLY_REPARAMETERIZED', - 'NOT_REPARAMETERIZED', - 'ReparameterizationType', - 'Distribution', - 'Autoregressive', - 'BatchReshape', - 'Bernoulli', - 'Beta', - 'Binomial', - 'BetaWithSoftplusConcentration', - 'Categorical', - 'Chi2', - 'Chi2WithAbsDf', - 'Deterministic', - 'VectorDeterministic', - 'Exponential', - 'ExponentialWithSoftplusRate', - 'VectorExponentialDiag', - 'Gamma', - 'GammaWithSoftplusConcentrationRate', - 'Geometric', - 'HalfNormal', - 'Independent', - 'InverseGamma', - 'InverseGammaWithSoftplusConcentrationRate', - 'Kumaraswamy', - 'Laplace', - 'LaplaceWithSoftplusScale', - 'Logistic', - 'NegativeBinomial', - 'Normal', - 'NormalWithSoftplusScale', - 'Poisson', - 'PoissonLogNormalQuadratureCompound', - 'SeedStream', - 'SinhArcsinh', - 'StudentT', - 'StudentTWithAbsDfSoftplusScale', - 'Uniform', - 'MultivariateNormalDiag', - 'MultivariateNormalFullCovariance', - 'MultivariateNormalTriL', - 'MultivariateNormalDiagPlusLowRank', - 'MultivariateNormalDiagWithSoftplusScale', - 'Dirichlet', - 'DirichletMultinomial', - 'Multinomial', - 'VectorDiffeomixture', - 'VectorLaplaceDiag', - 'VectorSinhArcsinhDiag', - 'WishartCholesky', - 'WishartFull', - 'TransformedDistribution', - 'QuantizedDistribution', - 'Mixture', - 'MixtureSameFamily', - 'ExpRelaxedOneHotCategorical', - 'OneHotCategorical', - 'RelaxedBernoulli', - 'RelaxedOneHotCategorical', - 'kl_divergence', - 'RegisterKL', - 'fill_triangular', - 'fill_triangular_inverse', - 'matrix_diag_transform', - 'reduce_weighted_logsumexp', - 'softplus_inverse', - 'tridiag', - 'normal_conjugates_known_scale_posterior', - 'normal_conjugates_known_scale_predictive', - 'percentile', - 'assign_moving_mean_variance', - 'assign_log_moving_mean_exp', - 'moving_mean_variance', - 'estimator_head_distribution_regression', - 'quadrature_scheme_softmaxnormal_gauss_hermite', - 'quadrature_scheme_softmaxnormal_quantiles', - 'quadrature_scheme_lognormal_gauss_hermite', - 'quadrature_scheme_lognormal_quantiles', -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/distributions/python/__init__.py b/tensorflow/contrib/distributions/python/__init__.py deleted file mode 100644 index c5ca3a623fb..00000000000 --- a/tensorflow/contrib/distributions/python/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ops module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/distributions/python/kernel_tests/autoregressive_test.py b/tensorflow/contrib/distributions/python/kernel_tests/autoregressive_test.py deleted file mode 100644 index a22d4d825b8..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/autoregressive_test.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import autoregressive as autoregressive_lib -from tensorflow.contrib.distributions.python.ops import independent as independent_lib -from tensorflow.contrib.distributions.python.ops import test_util -from tensorflow.contrib.distributions.python.ops.bijectors.affine import Affine -from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import MaskedAutoregressiveFlow -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import normal as normal_lib -from tensorflow.python.ops.distributions import transformed_distribution as transformed_distribution_lib -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.platform import test - - -class AutogressiveTest(test_util.VectorDistributionTestHelpers, test.TestCase): - """Tests the Autoregressive distribution.""" - - def setUp(self): - self._rng = np.random.RandomState(42) - - def _random_scale_tril(self, event_size): - n = np.int32(event_size * (event_size + 1) // 2) - p = 2. * self._rng.random_sample(n).astype(np.float32) - 1. - return distribution_util.fill_triangular(0.25 * p) - - def _normal_fn(self, affine_bijector): - def _fn(samples): - scale = math_ops.exp(affine_bijector.forward(samples)) - return independent_lib.Independent( - normal_lib.Normal(loc=0., scale=scale, validate_args=True), - reinterpreted_batch_ndims=1) - return _fn - - def testSampleAndLogProbConsistency(self): - batch_shape = [] - event_size = 2 - with self.cached_session() as sess: - batch_event_shape = np.concatenate([batch_shape, [event_size]], axis=0) - sample0 = array_ops.zeros(batch_event_shape) - affine = Affine(scale_tril=self._random_scale_tril(event_size)) - ar = autoregressive_lib.Autoregressive( - self._normal_fn(affine), sample0, validate_args=True) - self.run_test_sample_consistent_log_prob( - sess.run, ar, radius=1., center=0., rtol=0.01) - - def testCompareToBijector(self): - """Demonstrates equivalence between TD, Bijector approach and AR dist.""" - sample_shape = np.int32([4, 5]) - batch_shape = np.int32([]) - event_size = np.int32(2) - with self.cached_session() as sess: - batch_event_shape = np.concatenate([batch_shape, [event_size]], axis=0) - sample0 = array_ops.zeros(batch_event_shape) - affine = Affine(scale_tril=self._random_scale_tril(event_size)) - ar = autoregressive_lib.Autoregressive( - self._normal_fn(affine), sample0, validate_args=True) - ar_flow = MaskedAutoregressiveFlow( - is_constant_jacobian=True, - shift_and_log_scale_fn=lambda x: [None, affine.forward(x)], - validate_args=True) - td = transformed_distribution_lib.TransformedDistribution( - distribution=normal_lib.Normal(loc=0., scale=1.), - bijector=ar_flow, - event_shape=[event_size], - batch_shape=batch_shape, - validate_args=True) - x_shape = np.concatenate( - [sample_shape, batch_shape, [event_size]], axis=0) - x = 2. * self._rng.random_sample(x_shape).astype(np.float32) - 1. - td_log_prob_, ar_log_prob_ = sess.run([td.log_prob(x), ar.log_prob(x)]) - self.assertAllClose(td_log_prob_, ar_log_prob_, atol=0., rtol=1e-6) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/batch_reshape_test.py b/tensorflow/contrib/distributions/python/kernel_tests/batch_reshape_test.py deleted file mode 100644 index 62623deccd5..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/batch_reshape_test.py +++ /dev/null @@ -1,572 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for BatchReshape.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import batch_reshape as batch_reshape_lib -from tensorflow.contrib.distributions.python.ops import mvn_diag as mvn_lib -from tensorflow.contrib.distributions.python.ops import poisson as poisson_lib -from tensorflow.contrib.distributions.python.ops import wishart as wishart_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import array_ops -from tensorflow.python.ops.distributions import normal as normal_lib -from tensorflow.python.platform import test - - -class _BatchReshapeTest(object): - - def make_wishart(self, dims, new_batch_shape, old_batch_shape): - new_batch_shape_ph = ( - constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape - else array_ops.placeholder_with_default( - np.int32(new_batch_shape), shape=None)) - - scale = self.dtype([ - [[1., 0.5], - [0.5, 1.]], - [[0.5, 0.25], - [0.25, 0.75]], - ]) - scale = np.reshape(np.concatenate([scale, scale], axis=0), - old_batch_shape + [dims, dims]) - scale_ph = array_ops.placeholder_with_default( - scale, shape=scale.shape if self.is_static_shape else None) - wishart = wishart_lib.WishartFull(df=5, scale=scale_ph) - reshape_wishart = batch_reshape_lib.BatchReshape( - distribution=wishart, - batch_shape=new_batch_shape_ph, - validate_args=True) - - return wishart, reshape_wishart - - def test_matrix_variate_sample_and_log_prob(self): - dims = 2 - new_batch_shape = [4] - old_batch_shape = [2, 2] - wishart, reshape_wishart = self.make_wishart( - dims, new_batch_shape, old_batch_shape) - - batch_shape = reshape_wishart.batch_shape_tensor() - event_shape = reshape_wishart.event_shape_tensor() - - expected_sample_shape = [3, 1] + new_batch_shape + [dims, dims] - x = wishart.sample([3, 1], seed=42) - expected_sample = array_ops.reshape(x, expected_sample_shape) - actual_sample = reshape_wishart.sample([3, 1], seed=42) - - expected_log_prob_shape = [3, 1] + new_batch_shape - expected_log_prob = array_ops.reshape( - wishart.log_prob(x), expected_log_prob_shape) - actual_log_prob = reshape_wishart.log_prob(expected_sample) - - with self.cached_session() as sess: - [ - batch_shape_, - event_shape_, - expected_sample_, actual_sample_, - expected_log_prob_, actual_log_prob_, - ] = sess.run([ - batch_shape, - event_shape, - expected_sample, actual_sample, - expected_log_prob, actual_log_prob, - ]) - - self.assertAllEqual(new_batch_shape, batch_shape_) - self.assertAllEqual([dims, dims], event_shape_) - self.assertAllClose(expected_sample_, actual_sample_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_log_prob_, actual_log_prob_, - atol=0., rtol=1e-6) - if not self.is_static_shape: - return - self.assertAllEqual(new_batch_shape, reshape_wishart.batch_shape) - self.assertAllEqual([dims, dims], reshape_wishart.event_shape) - self.assertAllEqual(expected_sample_shape, actual_sample.shape) - self.assertAllEqual(expected_log_prob_shape, actual_log_prob.shape) - - def test_matrix_variate_stats(self): - dims = 2 - new_batch_shape = [4] - old_batch_shape = [2, 2] - wishart, reshape_wishart = self.make_wishart( - dims, new_batch_shape, old_batch_shape) - - expected_scalar_stat_shape = new_batch_shape - expected_matrix_stat_shape = new_batch_shape + [dims, dims] - - expected_entropy = array_ops.reshape( - wishart.entropy(), expected_scalar_stat_shape) - actual_entropy = reshape_wishart.entropy() - - expected_mean = array_ops.reshape( - wishart.mean(), expected_matrix_stat_shape) - actual_mean = reshape_wishart.mean() - - expected_mode = array_ops.reshape( - wishart.mode(), expected_matrix_stat_shape) - actual_mode = reshape_wishart.mode() - - expected_stddev = array_ops.reshape( - wishart.stddev(), expected_matrix_stat_shape) - actual_stddev = reshape_wishart.stddev() - - expected_variance = array_ops.reshape( - wishart.variance(), expected_matrix_stat_shape) - actual_variance = reshape_wishart.variance() - - with self.cached_session() as sess: - [ - expected_entropy_, actual_entropy_, - expected_mean_, actual_mean_, - expected_mode_, actual_mode_, - expected_stddev_, actual_stddev_, - expected_variance_, actual_variance_, - ] = sess.run([ - expected_entropy, actual_entropy, - expected_mean, actual_mean, - expected_mode, actual_mode, - expected_stddev, actual_stddev, - expected_variance, actual_variance, - ]) - - self.assertAllClose(expected_entropy_, actual_entropy_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_mean_, actual_mean_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_mode_, actual_mode_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_stddev_, actual_stddev_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_variance_, actual_variance_, - atol=0., rtol=1e-6) - if not self.is_static_shape: - return - self.assertAllEqual(expected_scalar_stat_shape, actual_entropy.shape) - self.assertAllEqual(expected_matrix_stat_shape, actual_mean.shape) - self.assertAllEqual(expected_matrix_stat_shape, actual_mode.shape) - self.assertAllEqual(expected_matrix_stat_shape, actual_stddev.shape) - self.assertAllEqual(expected_matrix_stat_shape, actual_variance.shape) - - def make_normal(self, new_batch_shape, old_batch_shape): - new_batch_shape_ph = ( - constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape - else array_ops.placeholder_with_default( - np.int32(new_batch_shape), shape=None)) - - scale = self.dtype(0.5 + np.arange( - np.prod(old_batch_shape)).reshape(old_batch_shape)) - scale_ph = array_ops.placeholder_with_default( - scale, shape=scale.shape if self.is_static_shape else None) - normal = normal_lib.Normal(loc=self.dtype(0), scale=scale_ph) - reshape_normal = batch_reshape_lib.BatchReshape( - distribution=normal, - batch_shape=new_batch_shape_ph, - validate_args=True) - return normal, reshape_normal - - def test_scalar_variate_sample_and_log_prob(self): - new_batch_shape = [2, 2] - old_batch_shape = [4] - - normal, reshape_normal = self.make_normal( - new_batch_shape, old_batch_shape) - - batch_shape = reshape_normal.batch_shape_tensor() - event_shape = reshape_normal.event_shape_tensor() - - expected_sample_shape = new_batch_shape - x = normal.sample(seed=52) - expected_sample = array_ops.reshape(x, expected_sample_shape) - actual_sample = reshape_normal.sample(seed=52) - - expected_log_prob_shape = new_batch_shape - expected_log_prob = array_ops.reshape( - normal.log_prob(x), expected_log_prob_shape) - actual_log_prob = reshape_normal.log_prob(expected_sample) - - with self.cached_session() as sess: - [ - batch_shape_, - event_shape_, - expected_sample_, actual_sample_, - expected_log_prob_, actual_log_prob_, - ] = sess.run([ - batch_shape, - event_shape, - expected_sample, actual_sample, - expected_log_prob, actual_log_prob, - ]) - self.assertAllEqual(new_batch_shape, batch_shape_) - self.assertAllEqual([], event_shape_) - self.assertAllClose(expected_sample_, actual_sample_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_log_prob_, actual_log_prob_, - atol=0., rtol=1e-6) - if not self.is_static_shape: - return - self.assertAllEqual(new_batch_shape, reshape_normal.batch_shape) - self.assertAllEqual([], reshape_normal.event_shape) - self.assertAllEqual(expected_sample_shape, actual_sample.shape) - self.assertAllEqual(expected_log_prob_shape, actual_log_prob.shape) - - def test_scalar_variate_stats(self): - new_batch_shape = [2, 2] - old_batch_shape = [4] - - normal, reshape_normal = self.make_normal(new_batch_shape, old_batch_shape) - - expected_scalar_stat_shape = new_batch_shape - - expected_entropy = array_ops.reshape( - normal.entropy(), expected_scalar_stat_shape) - actual_entropy = reshape_normal.entropy() - - expected_mean = array_ops.reshape( - normal.mean(), expected_scalar_stat_shape) - actual_mean = reshape_normal.mean() - - expected_mode = array_ops.reshape( - normal.mode(), expected_scalar_stat_shape) - actual_mode = reshape_normal.mode() - - expected_stddev = array_ops.reshape( - normal.stddev(), expected_scalar_stat_shape) - actual_stddev = reshape_normal.stddev() - - expected_variance = array_ops.reshape( - normal.variance(), expected_scalar_stat_shape) - actual_variance = reshape_normal.variance() - - with self.cached_session() as sess: - [ - expected_entropy_, actual_entropy_, - expected_mean_, actual_mean_, - expected_mode_, actual_mode_, - expected_stddev_, actual_stddev_, - expected_variance_, actual_variance_, - ] = sess.run([ - expected_entropy, actual_entropy, - expected_mean, actual_mean, - expected_mode, actual_mode, - expected_stddev, actual_stddev, - expected_variance, actual_variance, - ]) - self.assertAllClose(expected_entropy_, actual_entropy_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_mean_, actual_mean_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_mode_, actual_mode_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_stddev_, actual_stddev_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_variance_, actual_variance_, - atol=0., rtol=1e-6) - if not self.is_static_shape: - return - self.assertAllEqual(expected_scalar_stat_shape, actual_entropy.shape) - self.assertAllEqual(expected_scalar_stat_shape, actual_mean.shape) - self.assertAllEqual(expected_scalar_stat_shape, actual_mode.shape) - self.assertAllEqual(expected_scalar_stat_shape, actual_stddev.shape) - self.assertAllEqual(expected_scalar_stat_shape, actual_variance.shape) - - def make_mvn(self, dims, new_batch_shape, old_batch_shape): - new_batch_shape_ph = ( - constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape - else array_ops.placeholder_with_default( - np.int32(new_batch_shape), shape=None)) - - scale = np.ones(old_batch_shape + [dims], self.dtype) - scale_ph = array_ops.placeholder_with_default( - scale, shape=scale.shape if self.is_static_shape else None) - mvn = mvn_lib.MultivariateNormalDiag(scale_diag=scale_ph) - reshape_mvn = batch_reshape_lib.BatchReshape( - distribution=mvn, - batch_shape=new_batch_shape_ph, - validate_args=True) - return mvn, reshape_mvn - - def test_vector_variate_sample_and_log_prob(self): - dims = 3 - new_batch_shape = [2, 1] - old_batch_shape = [2] - mvn, reshape_mvn = self.make_mvn( - dims, new_batch_shape, old_batch_shape) - - batch_shape = reshape_mvn.batch_shape_tensor() - event_shape = reshape_mvn.event_shape_tensor() - - expected_sample_shape = [3] + new_batch_shape + [dims] - x = mvn.sample(3, seed=62) - expected_sample = array_ops.reshape(x, expected_sample_shape) - actual_sample = reshape_mvn.sample(3, seed=62) - - expected_log_prob_shape = [3] + new_batch_shape - expected_log_prob = array_ops.reshape( - mvn.log_prob(x), expected_log_prob_shape) - actual_log_prob = reshape_mvn.log_prob(expected_sample) - - with self.cached_session() as sess: - [ - batch_shape_, - event_shape_, - expected_sample_, actual_sample_, - expected_log_prob_, actual_log_prob_, - ] = sess.run([ - batch_shape, - event_shape, - expected_sample, actual_sample, - expected_log_prob, actual_log_prob, - ]) - self.assertAllEqual(new_batch_shape, batch_shape_) - self.assertAllEqual([dims], event_shape_) - self.assertAllClose(expected_sample_, actual_sample_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_log_prob_, actual_log_prob_, - atol=0., rtol=1e-6) - if not self.is_static_shape: - return - self.assertAllEqual(new_batch_shape, reshape_mvn.batch_shape) - self.assertAllEqual([dims], reshape_mvn.event_shape) - self.assertAllEqual(expected_sample_shape, actual_sample.shape) - self.assertAllEqual(expected_log_prob_shape, actual_log_prob.shape) - - def test_vector_variate_stats(self): - dims = 3 - new_batch_shape = [2, 1] - old_batch_shape = [2] - mvn, reshape_mvn = self.make_mvn( - dims, new_batch_shape, old_batch_shape) - - expected_scalar_stat_shape = new_batch_shape - - expected_entropy = array_ops.reshape( - mvn.entropy(), expected_scalar_stat_shape) - actual_entropy = reshape_mvn.entropy() - - expected_vector_stat_shape = new_batch_shape + [dims] - - expected_mean = array_ops.reshape( - mvn.mean(), expected_vector_stat_shape) - actual_mean = reshape_mvn.mean() - - expected_mode = array_ops.reshape( - mvn.mode(), expected_vector_stat_shape) - actual_mode = reshape_mvn.mode() - - expected_stddev = array_ops.reshape( - mvn.stddev(), expected_vector_stat_shape) - actual_stddev = reshape_mvn.stddev() - - expected_variance = array_ops.reshape( - mvn.variance(), expected_vector_stat_shape) - actual_variance = reshape_mvn.variance() - - expected_matrix_stat_shape = new_batch_shape + [dims, dims] - - expected_covariance = array_ops.reshape( - mvn.covariance(), expected_matrix_stat_shape) - actual_covariance = reshape_mvn.covariance() - - with self.cached_session() as sess: - [ - expected_entropy_, actual_entropy_, - expected_mean_, actual_mean_, - expected_mode_, actual_mode_, - expected_stddev_, actual_stddev_, - expected_variance_, actual_variance_, - expected_covariance_, actual_covariance_, - ] = sess.run([ - expected_entropy, actual_entropy, - expected_mean, actual_mean, - expected_mode, actual_mode, - expected_stddev, actual_stddev, - expected_variance, actual_variance, - expected_covariance, actual_covariance, - ]) - self.assertAllClose(expected_entropy_, actual_entropy_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_mean_, actual_mean_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_mode_, actual_mode_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_stddev_, actual_stddev_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_variance_, actual_variance_, - atol=0., rtol=1e-6) - self.assertAllClose(expected_covariance_, actual_covariance_, - atol=0., rtol=1e-6) - if not self.is_static_shape: - return - self.assertAllEqual(expected_scalar_stat_shape, actual_entropy.shape) - self.assertAllEqual(expected_vector_stat_shape, actual_mean.shape) - self.assertAllEqual(expected_vector_stat_shape, actual_mode.shape) - self.assertAllEqual(expected_vector_stat_shape, actual_stddev.shape) - self.assertAllEqual(expected_vector_stat_shape, actual_variance.shape) - self.assertAllEqual(expected_matrix_stat_shape, actual_covariance.shape) - - def test_bad_reshape_size(self): - dims = 2 - new_batch_shape = [2, 3] - old_batch_shape = [2] # 2 != 2*3 - - new_batch_shape_ph = ( - constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape - else array_ops.placeholder_with_default( - np.int32(new_batch_shape), shape=None)) - - scale = np.ones(old_batch_shape + [dims], self.dtype) - scale_ph = array_ops.placeholder_with_default( - scale, shape=scale.shape if self.is_static_shape else None) - mvn = mvn_lib.MultivariateNormalDiag(scale_diag=scale_ph) - - if self.is_static_shape: - with self.assertRaisesRegexp( - ValueError, (r"`batch_shape` size \(6\) must match " - r"`distribution\.batch_shape` size \(2\)")): - batch_reshape_lib.BatchReshape( - distribution=mvn, - batch_shape=new_batch_shape_ph, - validate_args=True) - - else: - with self.cached_session(): - with self.assertRaisesOpError(r"Shape sizes do not match."): - batch_reshape_lib.BatchReshape( - distribution=mvn, - batch_shape=new_batch_shape_ph, - validate_args=True).sample().eval() - - def test_non_positive_shape(self): - dims = 2 - old_batch_shape = [4] - if self.is_static_shape: - # Unknown first dimension does not trigger size check. Note that - # any dimension < 0 is treated statically as unknown. - new_batch_shape = [-1, 0] - else: - new_batch_shape = [-2, -2] # -2 * -2 = 4, same size as the old shape. - - new_batch_shape_ph = ( - constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape - else array_ops.placeholder_with_default( - np.int32(new_batch_shape), shape=None)) - - scale = np.ones(old_batch_shape + [dims], self.dtype) - scale_ph = array_ops.placeholder_with_default( - scale, shape=scale.shape if self.is_static_shape else None) - mvn = mvn_lib.MultivariateNormalDiag(scale_diag=scale_ph) - - if self.is_static_shape: - with self.assertRaisesRegexp(ValueError, r".*must be >=-1.*"): - batch_reshape_lib.BatchReshape( - distribution=mvn, - batch_shape=new_batch_shape_ph, - validate_args=True) - - else: - with self.cached_session(): - with self.assertRaisesOpError(r".*must be >=-1.*"): - batch_reshape_lib.BatchReshape( - distribution=mvn, - batch_shape=new_batch_shape_ph, - validate_args=True).sample().eval() - - def test_non_vector_shape(self): - dims = 2 - new_batch_shape = 2 - old_batch_shape = [2] - - new_batch_shape_ph = ( - constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape - else array_ops.placeholder_with_default( - np.int32(new_batch_shape), shape=None)) - - scale = np.ones(old_batch_shape + [dims], self.dtype) - scale_ph = array_ops.placeholder_with_default( - scale, shape=scale.shape if self.is_static_shape else None) - mvn = mvn_lib.MultivariateNormalDiag(scale_diag=scale_ph) - - if self.is_static_shape: - with self.assertRaisesRegexp(ValueError, r".*must be a vector.*"): - batch_reshape_lib.BatchReshape( - distribution=mvn, - batch_shape=new_batch_shape_ph, - validate_args=True) - - else: - with self.cached_session(): - with self.assertRaisesOpError(r".*must be a vector.*"): - batch_reshape_lib.BatchReshape( - distribution=mvn, - batch_shape=new_batch_shape_ph, - validate_args=True).sample().eval() - - def test_broadcasting_explicitly_unsupported(self): - old_batch_shape = [4] - new_batch_shape = [1, 4, 1] - rate_ = self.dtype([1, 10, 2, 20]) - - rate = array_ops.placeholder_with_default( - rate_, - shape=old_batch_shape if self.is_static_shape else None) - poisson_4 = poisson_lib.Poisson(rate) - new_batch_shape_ph = ( - constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape - else array_ops.placeholder_with_default( - np.int32(new_batch_shape), shape=None)) - poisson_141_reshaped = batch_reshape_lib.BatchReshape( - poisson_4, new_batch_shape_ph, validate_args=True) - - x_4 = self.dtype([2, 12, 3, 23]) - x_114 = self.dtype([2, 12, 3, 23]).reshape(1, 1, 4) - - if self.is_static_shape: - with self.assertRaisesRegexp(NotImplementedError, - "too few batch and event dims"): - poisson_141_reshaped.log_prob(x_4) - with self.assertRaisesRegexp(NotImplementedError, - "unexpected batch and event shape"): - poisson_141_reshaped.log_prob(x_114) - return - - with self.assertRaisesOpError("too few batch and event dims"): - with self.cached_session(): - poisson_141_reshaped.log_prob(x_4).eval() - - with self.assertRaisesOpError("unexpected batch and event shape"): - with self.cached_session(): - poisson_141_reshaped.log_prob(x_114).eval() - - -class BatchReshapeStaticTest(_BatchReshapeTest, test.TestCase): - - dtype = np.float32 - is_static_shape = True - - -class BatchReshapeDynamicTest(_BatchReshapeTest, test.TestCase): - - dtype = np.float64 - is_static_shape = False - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/absolute_value_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/absolute_value_test.py deleted file mode 100644 index 372b7e37b74..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/absolute_value_test.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for AbsoluteValue Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=g-importing-member -from tensorflow.contrib.distributions.python.ops.bijectors.absolute_value import AbsoluteValue -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - -# pylint: enable=g-importing-member - - -class AbsoluteValueTest(test.TestCase): - """Tests correctness of the absolute value bijector.""" - - def testBijectorVersusNumpyRewriteOfBasicFunctionsEventNdims0(self): - with self.cached_session() as sess: - bijector = AbsoluteValue(validate_args=True) - self.assertEqual("absolute_value", bijector.name) - x = array_ops.constant([[0., 1., -1], [0., -5., 3.]]) # Shape [2, 3] - y = math_ops.abs(x) - - y_ = y.eval() - - self.assertAllClose(y_, bijector.forward(x).eval()) - self.assertAllClose((-y_, y_), sess.run(bijector.inverse(y))) - self.assertAllClose((0., 0.), - sess.run(bijector.inverse_log_det_jacobian( - y, event_ndims=0))) - - # Run things twice to make sure there are no issues in caching the tuples - # returned by .inverse* - self.assertAllClose(y_, bijector.forward(x).eval()) - self.assertAllClose((-y_, y_), sess.run(bijector.inverse(y))) - self.assertAllClose((0., 0.), - sess.run(bijector.inverse_log_det_jacobian( - y, event_ndims=0))) - - def testNegativeYRaisesForInverseIfValidateArgs(self): - with self.cached_session() as sess: - bijector = AbsoluteValue(validate_args=True) - with self.assertRaisesOpError("y was negative"): - sess.run(bijector.inverse(-1.)) - - def testNegativeYRaisesForILDJIfValidateArgs(self): - with self.cached_session() as sess: - bijector = AbsoluteValue(validate_args=True) - with self.assertRaisesOpError("y was negative"): - sess.run(bijector.inverse_log_det_jacobian(-1., event_ndims=0)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_linear_operator_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_linear_operator_test.py deleted file mode 100644 index 1e36b7ff9be..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_linear_operator_test.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""AffineLinearOperator Tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.affine_linear_operator import AffineLinearOperator -from tensorflow.python.ops.linalg import linalg -from tensorflow.python.platform import test - - -class AffineLinearOperatorTest(test.TestCase): - - def testIdentity(self): - with self.cached_session(): - affine = AffineLinearOperator( - validate_args=True) - x = np.array([[1, 0, -1], [2, 3, 4]], dtype=np.float32) - y = x - ildj = 0. - - self.assertEqual(affine.name, "affine_linear_operator") - self.assertAllClose(y, affine.forward(x).eval()) - self.assertAllClose(x, affine.inverse(y).eval()) - self.assertAllClose(ildj, affine.inverse_log_det_jacobian( - y, event_ndims=2).eval()) - self.assertAllClose( - -affine.inverse_log_det_jacobian(y, event_ndims=2).eval(), - affine.forward_log_det_jacobian(x, event_ndims=2).eval()) - - def testDiag(self): - with self.cached_session(): - shift = np.array([-1, 0, 1], dtype=np.float32) - diag = np.array([[1, 2, 3], - [2, 5, 6]], dtype=np.float32) - scale = linalg.LinearOperatorDiag(diag, is_non_singular=True) - affine = AffineLinearOperator( - shift=shift, scale=scale, validate_args=True) - - x = np.array([[1, 0, -1], [2, 3, 4]], dtype=np.float32) - y = diag * x + shift - ildj = -np.sum(np.log(np.abs(diag)), axis=-1) - - self.assertEqual(affine.name, "affine_linear_operator") - self.assertAllClose(y, affine.forward(x).eval()) - self.assertAllClose(x, affine.inverse(y).eval()) - self.assertAllClose( - ildj, affine.inverse_log_det_jacobian(y, event_ndims=1).eval()) - self.assertAllClose( - -affine.inverse_log_det_jacobian(y, event_ndims=1).eval(), - affine.forward_log_det_jacobian(x, event_ndims=1).eval()) - - def testTriL(self): - with self.cached_session(): - shift = np.array([-1, 0, 1], dtype=np.float32) - tril = np.array([[[3, 0, 0], - [2, -1, 0], - [3, 2, 1]], - [[2, 0, 0], - [3, -2, 0], - [4, 3, 2]]], - dtype=np.float32) - scale = linalg.LinearOperatorLowerTriangular(tril, is_non_singular=True) - affine = AffineLinearOperator( - shift=shift, scale=scale, validate_args=True) - - x = np.array([[[1, 0, -1], - [2, 3, 4]], - [[4, 1, -7], - [6, 9, 8]]], - dtype=np.float32) - # If we made the bijector do x*A+b then this would be simplified to: - # y = np.matmul(x, tril) + shift. - y = np.squeeze(np.matmul(tril, np.expand_dims(x, -1)), -1) + shift - ildj = -np.sum(np.log(np.abs(np.diagonal( - tril, axis1=-2, axis2=-1)))) - - self.assertEqual(affine.name, "affine_linear_operator") - self.assertAllClose(y, affine.forward(x).eval()) - self.assertAllClose(x, affine.inverse(y).eval()) - self.assertAllClose( - ildj, affine.inverse_log_det_jacobian( - y, event_ndims=2).eval()) - self.assertAllClose( - -affine.inverse_log_det_jacobian(y, event_ndims=2).eval(), - affine.forward_log_det_jacobian(x, event_ndims=2).eval()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_scalar_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_scalar_test.py deleted file mode 100644 index bc6752a69df..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_scalar_test.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Affine Scalar Tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.affine_scalar import AffineScalar -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency -from tensorflow.python.platform import test - - -class AffineScalarBijectorTest(test.TestCase): - """Tests correctness of the Y = scale @ x + shift transformation.""" - - def testProperties(self): - with self.cached_session(): - mu = -1. - # scale corresponds to 1. - bijector = AffineScalar(shift=mu) - self.assertEqual("affine_scalar", bijector.name) - - def testNoBatchScalar(self): - with self.cached_session() as sess: - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x, **kwargs), feed_dict={x: x_value}) - - for run in (static_run, dynamic_run): - mu = -1. - # Corresponds to scale = 2 - bijector = AffineScalar(shift=mu, scale=2.) - x = [1., 2, 3] # Three scalar samples (no batches). - self.assertAllClose([1., 3, 5], run(bijector.forward, x)) - self.assertAllClose([1., 1.5, 2.], run(bijector.inverse, x)) - self.assertAllClose( - -np.log(2.), - run(bijector.inverse_log_det_jacobian, x, event_ndims=0)) - - def testOneBatchScalarViaIdentityIn64BitUserProvidesShiftOnly(self): - with self.cached_session() as sess: - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value).astype(np.float64) - x = array_ops.placeholder(dtypes.float64, name="x") - return sess.run(fun(x, **kwargs), feed_dict={x: x_value}) - - for run in (static_run, dynamic_run): - mu = np.float64([1.]) - # One batch, scalar. - # Corresponds to scale = 1. - bijector = AffineScalar(shift=mu) - x = np.float64([1.]) # One sample from one batches. - self.assertAllClose([2.], run(bijector.forward, x)) - self.assertAllClose([0.], run(bijector.inverse, x)) - self.assertAllClose( - 0., - run(bijector.inverse_log_det_jacobian, x, event_ndims=0)) - - def testOneBatchScalarViaIdentityIn64BitUserProvidesScaleOnly(self): - with self.cached_session() as sess: - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value).astype(np.float64) - x = array_ops.placeholder(dtypes.float64, name="x") - return sess.run(fun(x, **kwargs), feed_dict={x: x_value}) - - for run in (static_run, dynamic_run): - multiplier = np.float64([2.]) - # One batch, scalar. - # Corresponds to scale = 2, shift = 0. - bijector = AffineScalar(scale=multiplier) - x = np.float64([1.]) # One sample from one batches. - self.assertAllClose([2.], run(bijector.forward, x)) - self.assertAllClose([0.5], run(bijector.inverse, x)) - self.assertAllClose( - [np.log(0.5)], - run(bijector.inverse_log_det_jacobian, x, event_ndims=0)) - - def testTwoBatchScalarIdentityViaIdentity(self): - with self.cached_session() as sess: - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value).astype(np.float32) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x, **kwargs), feed_dict={x: x_value}) - - for run in (static_run, dynamic_run): - mu = [1., -1] - # Univariate, two batches. - # Corresponds to scale = 1. - bijector = AffineScalar(shift=mu) - x = [1., 1] # One sample from each of two batches. - self.assertAllClose([2., 0], run(bijector.forward, x)) - self.assertAllClose([0., 2], run(bijector.inverse, x)) - self.assertAllClose( - 0., - run(bijector.inverse_log_det_jacobian, x, event_ndims=0)) - - def testTwoBatchScalarIdentityViaScale(self): - with self.cached_session() as sess: - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value).astype(np.float32) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x, **kwargs), feed_dict={x: x_value}) - - for run in (static_run, dynamic_run): - mu = [1., -1] - # Univariate, two batches. - # Corresponds to scale = 1. - bijector = AffineScalar(shift=mu, scale=[2., 1]) - x = [1., 1] # One sample from each of two batches. - self.assertAllClose([3., 0], run(bijector.forward, x)) - self.assertAllClose([0., 2], run(bijector.inverse, x)) - self.assertAllClose( - [-np.log(2), 0.], - run(bijector.inverse_log_det_jacobian, x, event_ndims=0)) - - def testScalarCongruency(self): - with self.cached_session(): - bijector = AffineScalar(shift=3.6, scale=0.42) - assert_scalar_congruency(bijector, lower_x=-2., upper_x=2.) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_test.py deleted file mode 100644 index 8b61d4be63c..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_test.py +++ /dev/null @@ -1,625 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Affine Tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.affine import Affine -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class AffineBijectorTest(test.TestCase): - """Tests correctness of the Y = scale @ x + shift transformation.""" - - def testProperties(self): - with self.cached_session(): - mu = -1. - # scale corresponds to 1. - bijector = Affine(shift=mu) - self.assertEqual("affine", bijector.name) - - def testNoBatchMultivariateIdentity(self): - with self.cached_session() as sess: - placeholder = array_ops.placeholder(dtypes.float32, name="x") - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value) - return sess.run( - fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) - - for run in (static_run, dynamic_run): - mu = [1., -1] - # Multivariate - # Corresponds to scale = [[1., 0], [0, 1.]] - bijector = Affine(shift=mu) - x = [1., 1] - # matmul(sigma, x) + shift - # = [-1, -1] + [1, -1] - self.assertAllClose([2., 0], run(bijector.forward, x)) - self.assertAllClose([0., 2], run(bijector.inverse, x)) - - # x is a 2-batch of 2-vectors. - # The first vector is [1, 1], the second is [-1, -1]. - # Each undergoes matmul(sigma, x) + shift. - x = [[1., 1], [-1., -1]] - self.assertAllClose([[2., 0], [0., -2]], run(bijector.forward, x)) - self.assertAllClose([[0., 2], [-2., 0]], run(bijector.inverse, x)) - self.assertAllClose( - 0., run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) - - def testNoBatchMultivariateDiag(self): - with self.cached_session() as sess: - placeholder = array_ops.placeholder(dtypes.float32, name="x") - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value) - return sess.run( - fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) - - for run in (static_run, dynamic_run): - mu = [1., -1] - # Multivariate - # Corresponds to scale = [[2., 0], [0, 1.]] - bijector = Affine(shift=mu, scale_diag=[2., 1]) - x = [1., 1] - # matmul(sigma, x) + shift - # = [-1, -1] + [1, -1] - self.assertAllClose([3., 0], run(bijector.forward, x)) - self.assertAllClose([0., 2], run(bijector.inverse, x)) - self.assertAllClose( - -np.log(2.), - run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) - - # Reset bijector. - bijector = Affine(shift=mu, scale_diag=[2., 1]) - # x is a 2-batch of 2-vectors. - # The first vector is [1, 1], the second is [-1, -1]. - # Each undergoes matmul(sigma, x) + shift. - x = [[1., 1], - [-1., -1]] - self.assertAllClose([[3., 0], - [-1., -2]], - run(bijector.forward, x)) - self.assertAllClose([[0., 2], - [-1., 0]], - run(bijector.inverse, x)) - self.assertAllClose( - -np.log(2.), - run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) - - def testNoBatchMultivariateFullDynamic(self): - with self.cached_session() as sess: - x = array_ops.placeholder(dtypes.float32, name="x") - mu = array_ops.placeholder(dtypes.float32, name="mu") - scale_diag = array_ops.placeholder(dtypes.float32, name="scale_diag") - - x_value = np.array([[1., 1]], dtype=np.float32) - mu_value = np.array([1., -1], dtype=np.float32) - scale_diag_value = np.array([2., 2], dtype=np.float32) - feed_dict = { - x: x_value, - mu: mu_value, - scale_diag: scale_diag_value, - } - - bijector = Affine(shift=mu, scale_diag=scale_diag) - self.assertAllClose([[3., 1]], sess.run(bijector.forward(x), feed_dict)) - self.assertAllClose([[0., 1]], sess.run(bijector.inverse(x), feed_dict)) - self.assertAllClose( - -np.log(4), - sess.run(bijector.inverse_log_det_jacobian(x, event_ndims=1), - feed_dict)) - - def testBatchMultivariateIdentity(self): - with self.cached_session() as sess: - placeholder = array_ops.placeholder(dtypes.float32, name="x") - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value) - return sess.run( - fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) - - for run in (static_run, dynamic_run): - mu = [[1., -1]] - # Corresponds to 1 2x2 matrix, with twos on the diagonal. - scale = 2. - bijector = Affine(shift=mu, scale_identity_multiplier=scale) - x = [[[1., 1]]] - self.assertAllClose([[[3., 1]]], run(bijector.forward, x)) - self.assertAllClose([[[0., 1]]], run(bijector.inverse, x)) - self.assertAllClose( - -np.log(4), - run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) - - def testBatchMultivariateDiag(self): - with self.cached_session() as sess: - placeholder = array_ops.placeholder(dtypes.float32, name="x") - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value) - return sess.run( - fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) - - for run in (static_run, dynamic_run): - mu = [[1., -1]] - # Corresponds to 1 2x2 matrix, with twos on the diagonal. - scale_diag = [[2., 2]] - bijector = Affine(shift=mu, scale_diag=scale_diag) - x = [[[1., 1]]] - self.assertAllClose([[[3., 1]]], run(bijector.forward, x)) - self.assertAllClose([[[0., 1]]], run(bijector.inverse, x)) - self.assertAllClose( - [-np.log(4)], - run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) - - def testBatchMultivariateFullDynamic(self): - with self.cached_session() as sess: - x = array_ops.placeholder(dtypes.float32, name="x") - mu = array_ops.placeholder(dtypes.float32, name="mu") - scale_diag = array_ops.placeholder(dtypes.float32, name="scale_diag") - - x_value = np.array([[[1., 1]]], dtype=np.float32) - mu_value = np.array([[1., -1]], dtype=np.float32) - scale_diag_value = np.array([[2., 2]], dtype=np.float32) - - feed_dict = { - x: x_value, - mu: mu_value, - scale_diag: scale_diag_value, - } - - bijector = Affine(shift=mu, scale_diag=scale_diag) - self.assertAllClose([[[3., 1]]], sess.run(bijector.forward(x), feed_dict)) - self.assertAllClose([[[0., 1]]], sess.run(bijector.inverse(x), feed_dict)) - self.assertAllClose( - [-np.log(4)], - sess.run(bijector.inverse_log_det_jacobian( - x, event_ndims=1), feed_dict)) - - def testIdentityWithDiagUpdate(self): - with self.cached_session() as sess: - placeholder = array_ops.placeholder(dtypes.float32, name="x") - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value) - return sess.run( - fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) - - for run in (static_run, dynamic_run): - mu = -1. - # Corresponds to scale = 2 - bijector = Affine( - shift=mu, - scale_identity_multiplier=1., - scale_diag=[1., 1., 1.]) - x = [1., 2, 3] # Three scalar samples (no batches). - self.assertAllClose([1., 3, 5], run(bijector.forward, x)) - self.assertAllClose([1., 1.5, 2.], run(bijector.inverse, x)) - self.assertAllClose( - -np.log(2.**3), - run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) - - def testIdentityWithTriL(self): - with self.cached_session() as sess: - placeholder = array_ops.placeholder(dtypes.float32, name="x") - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value) - return sess.run( - fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) - - for run in (static_run, dynamic_run): - mu = -1. - # scale = [[2., 0], [2, 2]] - bijector = Affine( - shift=mu, - scale_identity_multiplier=1., - scale_tril=[[1., 0], [2., 1]]) - x = [[1., 2]] # One multivariate sample. - self.assertAllClose([[1., 5]], run(bijector.forward, x)) - self.assertAllClose([[1., 0.5]], run(bijector.inverse, x)) - self.assertAllClose( - -np.log(4.), - run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) - - def testDiagWithTriL(self): - with self.cached_session() as sess: - placeholder = array_ops.placeholder(dtypes.float32, name="x") - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value) - return sess.run( - fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) - - for run in (static_run, dynamic_run): - mu = -1. - # scale = [[2., 0], [2, 3]] - bijector = Affine( - shift=mu, scale_diag=[1., 2.], scale_tril=[[1., 0], [2., 1]]) - x = [[1., 2]] # One multivariate sample. - self.assertAllClose([[1., 7]], run(bijector.forward, x)) - self.assertAllClose([[1., 1 / 3.]], run(bijector.inverse, x)) - self.assertAllClose( - -np.log(6.), - run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) - - def testIdentityAndDiagWithTriL(self): - with self.cached_session() as sess: - placeholder = array_ops.placeholder(dtypes.float32, name="x") - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value) - return sess.run( - fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) - - for run in (static_run, dynamic_run): - mu = -1. - # scale = [[3., 0], [2, 4]] - bijector = Affine( - shift=mu, - scale_identity_multiplier=1.0, - scale_diag=[1., 2.], - scale_tril=[[1., 0], [2., 1]]) - x = [[1., 2]] # One multivariate sample. - self.assertAllClose([[2., 9]], run(bijector.forward, x)) - self.assertAllClose([[2 / 3., 5 / 12.]], run(bijector.inverse, x)) - self.assertAllClose( - -np.log(12.), - run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) - - def testIdentityWithVDVTUpdate(self): - with self.cached_session() as sess: - placeholder = array_ops.placeholder(dtypes.float32, name="x") - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value) - return sess.run( - fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) - - for run in (static_run, dynamic_run): - mu = -1. - # Corresponds to scale = [[10, 0, 0], [0, 2, 0], [0, 0, 3]] - bijector = Affine( - shift=mu, - scale_identity_multiplier=2., - scale_perturb_diag=[2., 1], - scale_perturb_factor=[[2., 0], [0., 0], [0, 1]]) - bijector_ref = Affine(shift=mu, scale_diag=[10., 2, 3]) - - x = [1., 2, 3] # Vector. - self.assertAllClose([9., 3, 8], run(bijector.forward, x)) - self.assertAllClose( - run(bijector_ref.forward, x), run(bijector.forward, x)) - - self.assertAllClose([0.2, 1.5, 4 / 3.], run(bijector.inverse, x)) - self.assertAllClose( - run(bijector_ref.inverse, x), run(bijector.inverse, x)) - self.assertAllClose( - -np.log(60.), - run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) - self.assertAllClose( - run(bijector.inverse_log_det_jacobian, x, event_ndims=1), - run(bijector_ref.inverse_log_det_jacobian, x, event_ndims=1)) - - def testDiagWithVDVTUpdate(self): - with self.cached_session() as sess: - placeholder = array_ops.placeholder(dtypes.float32, name="x") - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value) - return sess.run( - fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) - - for run in (static_run, dynamic_run): - mu = -1. - # Corresponds to scale = [[10, 0, 0], [0, 3, 0], [0, 0, 5]] - bijector = Affine( - shift=mu, - scale_diag=[2., 3, 4], - scale_perturb_diag=[2., 1], - scale_perturb_factor=[[2., 0], [0., 0], [0, 1]]) - bijector_ref = Affine(shift=mu, scale_diag=[10., 3, 5]) - - x = [1., 2, 3] # Vector. - self.assertAllClose([9., 5, 14], run(bijector.forward, x)) - self.assertAllClose( - run(bijector_ref.forward, x), run(bijector.forward, x)) - self.assertAllClose([0.2, 1., 0.8], run(bijector.inverse, x)) - self.assertAllClose( - run(bijector_ref.inverse, x), run(bijector.inverse, x)) - self.assertAllClose( - -np.log(150.), - run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) - self.assertAllClose( - run(bijector.inverse_log_det_jacobian, x, event_ndims=1), - run(bijector_ref.inverse_log_det_jacobian, x, event_ndims=1)) - - def testTriLWithVDVTUpdate(self): - with self.cached_session() as sess: - placeholder = array_ops.placeholder(dtypes.float32, name="x") - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value) - return sess.run( - fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) - - for run in (static_run, dynamic_run): - mu = -1. - # Corresponds to scale = [[10, 0, 0], [1, 3, 0], [2, 3, 5]] - bijector = Affine( - shift=mu, - scale_tril=[[2., 0, 0], [1, 3, 0], [2, 3, 4]], - scale_perturb_diag=[2., 1], - scale_perturb_factor=[[2., 0], [0., 0], [0, 1]]) - bijector_ref = Affine( - shift=mu, scale_tril=[[10., 0, 0], [1, 3, 0], [2, 3, 5]]) - - x = [1., 2, 3] # Vector. - self.assertAllClose([9., 6, 22], run(bijector.forward, x)) - self.assertAllClose( - run(bijector_ref.forward, x), run(bijector.forward, x)) - self.assertAllClose([0.2, 14 / 15., 4 / 25.], run(bijector.inverse, x)) - self.assertAllClose( - run(bijector_ref.inverse, x), run(bijector.inverse, x)) - self.assertAllClose( - -np.log(150.), - run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) - self.assertAllClose( - run(bijector.inverse_log_det_jacobian, x, event_ndims=1), - run(bijector_ref.inverse_log_det_jacobian, x, event_ndims=1)) - - def testTriLWithVDVTUpdateNoDiagonal(self): - with self.cached_session() as sess: - placeholder = array_ops.placeholder(dtypes.float32, name="x") - - def static_run(fun, x, **kwargs): - return fun(x, **kwargs).eval() - - def dynamic_run(fun, x_value, **kwargs): - x_value = np.array(x_value) - return sess.run( - fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) - - for run in (static_run, dynamic_run): - mu = -1. - # Corresponds to scale = [[6, 0, 0], [1, 3, 0], [2, 3, 5]] - bijector = Affine( - shift=mu, - scale_tril=[[2., 0, 0], [1, 3, 0], [2, 3, 4]], - scale_perturb_diag=None, - scale_perturb_factor=[[2., 0], [0., 0], [0, 1]]) - bijector_ref = Affine( - shift=mu, scale_tril=[[6., 0, 0], [1, 3, 0], [2, 3, 5]]) - - x = [1., 2, 3] # Vector. - self.assertAllClose([5., 6, 22], run(bijector.forward, x)) - self.assertAllClose( - run(bijector_ref.forward, x), run(bijector.forward, x)) - self.assertAllClose([1 / 3., 8 / 9., 4 / 30.], run(bijector.inverse, x)) - self.assertAllClose( - run(bijector_ref.inverse, x), run(bijector.inverse, x)) - self.assertAllClose( - -np.log(90.), - run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) - self.assertAllClose( - run(bijector.inverse_log_det_jacobian, x, event_ndims=1), - run(bijector_ref.inverse_log_det_jacobian, x, event_ndims=1)) - - def testNoBatchMultivariateRaisesWhenSingular(self): - with self.cached_session(): - mu = [1., -1] - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "diagonal part must be non-zero"): - _ = Affine( - shift=mu, - # Has zero on the diagonal. - scale_diag=[0., 1], - validate_args=True) - # Error detected statically; don't need to run the op. - - def _makeScale(self, - x, - scale_identity_multiplier=None, - scale_diag=None, - scale_tril=None, - scale_perturb_factor=None, - scale_perturb_diag=None): - """Create a scale matrix. Return None if it can not be created.""" - c = scale_identity_multiplier - d1 = scale_diag - tril = scale_tril - v = scale_perturb_factor - d2 = scale_perturb_diag - - # Ambiguous low rank update. - if v is None and d2 is not None: - return None - - if c is None and d1 is None and tril is None: - # Special case when no scale args are passed in. This means use an - # identity matrix. - c = 1. - - matrix = np.float32(0.) - if c is not None: - # Infer the dimension from x. - matrix += c * self._matrix_diag(np.ones_like(x)) - if d1 is not None: - matrix += self._matrix_diag(np.array(d1, dtype=np.float32)) - if tril is not None: - matrix += np.array(tril, dtype=np.float32) - if v is not None: - v = np.array(v, dtype=np.float32) - if v.ndim < 2: - vt = v.T - else: - vt = np.swapaxes(v, axis1=v.ndim - 2, axis2=v.ndim - 1) - if d2 is not None: - d2 = self._matrix_diag(np.array(d2, dtype=np.float32)) - right = np.matmul(d2, vt) - else: - right = vt - matrix += np.matmul(v, right) - return matrix - - def _matrix_diag(self, d): - """Batch version of np.diag.""" - orig_shape = d.shape - d = np.reshape(d, (int(np.prod(d.shape[:-1])), d.shape[-1])) - diag_list = [] - for i in range(d.shape[0]): - diag_list.append(np.diag(d[i, ...])) - return np.reshape(diag_list, orig_shape + (d.shape[-1],)) - - def _testLegalInputs(self, shift=None, scale_params=None, x=None): - - def _powerset(x): - s = list(x) - return itertools.chain.from_iterable( - itertools.combinations(s, r) for r in range(len(s) + 1)) - - for args in _powerset(scale_params.items()): - with self.cached_session(): - args = dict(args) - - scale_args = dict({"x": x}, **args) - scale = self._makeScale(**scale_args) - - # We haven't specified enough information for the scale. - if scale is None: - with self.assertRaisesRegexp(ValueError, ("must be specified.")): - bijector = Affine(shift=shift, **args) - else: - bijector = Affine(shift=shift, **args) - np_x = x - # For the case a vector is passed in, we need to make the shape - # match the matrix for matmul to work. - if x.ndim == scale.ndim - 1: - np_x = np.expand_dims(x, axis=-1) - - forward = np.matmul(scale, np_x) + shift - if x.ndim == scale.ndim - 1: - forward = np.squeeze(forward, axis=-1) - self.assertAllClose(forward, bijector.forward(x).eval()) - - backward = np.linalg.solve(scale, np_x - shift) - if x.ndim == scale.ndim - 1: - backward = np.squeeze(backward, axis=-1) - self.assertAllClose(backward, bijector.inverse(x).eval()) - - scale *= np.ones(shape=x.shape[:-1], dtype=scale.dtype) - ildj = -np.log(np.abs(np.linalg.det(scale))) - # TODO(jvdillon): We need to make it so the scale_identity_multiplier - # case does not deviate in expected shape. Fixing this will get rid of - # these special cases. - if (ildj.ndim > 0 and (len(scale_args) == 1 or ( - len(scale_args) == 2 and - scale_args.get("scale_identity_multiplier", None) is not None))): - ildj = np.squeeze(ildj[0]) - elif ildj.ndim < scale.ndim - 2: - ildj = np.reshape(ildj, scale.shape[0:-2]) - self.assertAllClose( - ildj, bijector.inverse_log_det_jacobian(x, event_ndims=1).eval()) - - def testLegalInputs(self): - self._testLegalInputs( - shift=np.float32(-1), - scale_params={ - "scale_identity_multiplier": 2., - "scale_diag": [2., 3.], - "scale_tril": [[1., 0.], - [-3., 3.]], - "scale_perturb_factor": [[1., 0], - [1.5, 3.]], - "scale_perturb_diag": [3., 1.] - }, - x=np.array( - [1., 2], dtype=np.float32)) - - def testLegalInputsWithBatch(self): - # Shape of scale is [2, 1, 2, 2] - self._testLegalInputs( - shift=np.float32(-1), - scale_params={ - "scale_identity_multiplier": 2., - "scale_diag": [[[2., 3.]], [[1., 2]]], - "scale_tril": [[[[1., 0.], [-3., 3.]]], [[[0.5, 0.], [1., 1.]]]], - "scale_perturb_factor": [[[[1., 0], [1.5, 3.]]], - [[[1., 0], [1., 1.]]]], - "scale_perturb_diag": [[[3., 1.]], [[0.5, 1.]]] - }, - x=np.array( - [[[1., 2]], [[3., 4]]], dtype=np.float32)) - - def testNegativeDetTrilPlusVDVT(self): - # scale = [[3.7, 2.7], - # [-0.3, -1.3]] - # inv(scale) = [[0.325, 0.675], - # [-0.075, -0.925]] - # eig(scale) = [3.5324, -1.1324] - self._testLegalInputs( - shift=np.float32(-1), - scale_params={ - "scale_tril": [[1., 0], [-3, -4]], - "scale_perturb_factor": [[0.1, 0], [0.5, 0.3]], - "scale_perturb_diag": [3., 1] - }, - x=np.array( - [1., 2], dtype=np.float32)) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/batch_normalization_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/batch_normalization_test.py deleted file mode 100644 index bf61e9f2fe3..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/batch_normalization_test.py +++ /dev/null @@ -1,237 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for BatchNorm Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib import distributions -from tensorflow.contrib.distributions.python.ops import test_util -from tensorflow.contrib.distributions.python.ops.bijectors.batch_normalization import BatchNormalization -from tensorflow.contrib.distributions.python.ops.bijectors.invert import Invert -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.layers import normalization -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables -from tensorflow.python.ops.distributions import normal as normal_lib -from tensorflow.python.ops.distributions import transformed_distribution as transformed_distribution_lib -from tensorflow.python.platform import test -from tensorflow.python.training import adam - - -class BatchNormTest(test_util.VectorDistributionTestHelpers, - test.TestCase): - - def _reduction_axes(self, input_shape, event_dims): - if isinstance(event_dims, int): - event_dims = [event_dims] - ndims = len(input_shape) - # Convert event_dims to non-negative indexing. - event_dims = list(event_dims) - for idx, x in enumerate(event_dims): - if x < 0: - event_dims[idx] = ndims + x - return tuple(i for i in range(ndims) if i not in event_dims) - - def testForwardInverse(self): - """Tests forward and backward passes with different event shapes. - - input_shape: Tuple of shapes for input tensor. - event_dims: Tuple of dimension indices that will be normalized. - training: Boolean of whether bijector runs in training or inference mode. - """ - params = [ - ((5*2, 4), [-1], False), - ((5, 2, 4), [-1], False), - ((5, 2, 4), [1, 2], False), - ((5, 2, 4), [0, 1], False), - ((5*2, 4), [-1], True), - ((5, 2, 4), [-1], True), - ((5, 2, 4), [1, 2], True), - ((5, 2, 4), [0, 1], True) - ] - for input_shape, event_dims, training in params: - x_ = np.arange(5 * 4 * 2).astype(np.float32).reshape(input_shape) - with self.cached_session() as sess: - x = constant_op.constant(x_) - # When training, memorize the exact mean of the last - # minibatch that it normalized (instead of moving average assignment). - layer = normalization.BatchNormalization( - axis=event_dims, momentum=0., epsilon=0.) - batch_norm = BatchNormalization( - batchnorm_layer=layer, training=training) - # Minibatch statistics are saved only after norm_x has been computed. - norm_x = batch_norm.inverse(x) - with ops.control_dependencies(batch_norm.batchnorm.updates): - moving_mean = array_ops.identity(batch_norm.batchnorm.moving_mean) - moving_var = array_ops.identity(batch_norm.batchnorm.moving_variance) - denorm_x = batch_norm.forward(array_ops.identity(norm_x)) - fldj = batch_norm.forward_log_det_jacobian( - x, event_ndims=len(event_dims)) - # Use identity to invalidate cache. - ildj = batch_norm.inverse_log_det_jacobian( - array_ops.identity(denorm_x), event_ndims=len(event_dims)) - variables.global_variables_initializer().run() - # Update variables. - norm_x_ = sess.run(norm_x) - [ - norm_x_, - moving_mean_, - moving_var_, - denorm_x_, - ildj_, - fldj_, - ] = sess.run([ - norm_x, - moving_mean, - moving_var, - denorm_x, - ildj, - fldj, - ]) - self.assertEqual("batch_normalization", batch_norm.name) - - reduction_axes = self._reduction_axes(input_shape, event_dims) - keepdims = len(event_dims) > 1 - - expected_batch_mean = np.mean( - x_, axis=reduction_axes, keepdims=keepdims) - expected_batch_var = np.var(x_, axis=reduction_axes, keepdims=keepdims) - - if training: - # When training=True, values become normalized across batch dim and - # original values are recovered after de-normalizing. - zeros = np.zeros_like(norm_x_) - self.assertAllClose(np.mean(zeros, axis=reduction_axes), - np.mean(norm_x_, axis=reduction_axes)) - - self.assertAllClose(expected_batch_mean, moving_mean_) - self.assertAllClose(expected_batch_var, moving_var_) - self.assertAllClose(x_, denorm_x_, atol=1e-5) - # Since moving statistics are set to batch statistics after - # normalization, ildj and -fldj should match. - self.assertAllClose(ildj_, -fldj_) - # ildj is computed with minibatch statistics. - expected_ildj = np.sum(np.log(1.) - .5 * np.log( - expected_batch_var + batch_norm.batchnorm.epsilon)) - self.assertAllClose(expected_ildj, ildj_) - else: - # When training=False, moving_mean, moving_var remain at their - # initialized values (0., 1.), resulting in no scale/shift (a small - # shift occurs if epsilon > 0.) - self.assertAllClose(x_, norm_x_) - self.assertAllClose(x_, denorm_x_, atol=1e-5) - # ildj is computed with saved statistics. - expected_ildj = np.sum( - np.log(1.) - .5 * np.log(1. + batch_norm.batchnorm.epsilon)) - self.assertAllClose(expected_ildj, ildj_) - - def testMaximumLikelihoodTraining(self): - # Test Maximum Likelihood training with default bijector. - with self.cached_session() as sess: - base_dist = distributions.MultivariateNormalDiag(loc=[0., 0.]) - batch_norm = BatchNormalization(training=True) - dist = transformed_distribution_lib.TransformedDistribution( - distribution=base_dist, - bijector=batch_norm) - target_dist = distributions.MultivariateNormalDiag(loc=[1., 2.]) - target_samples = target_dist.sample(100) - dist_samples = dist.sample(3000) - loss = -math_ops.reduce_mean(dist.log_prob(target_samples)) - with ops.control_dependencies(batch_norm.batchnorm.updates): - train_op = adam.AdamOptimizer(1e-2).minimize(loss) - moving_mean = array_ops.identity(batch_norm.batchnorm.moving_mean) - moving_var = array_ops.identity(batch_norm.batchnorm.moving_variance) - variables.global_variables_initializer().run() - for _ in range(3000): - sess.run(train_op) - [ - dist_samples_, - moving_mean_, - moving_var_ - ] = sess.run([ - dist_samples, - moving_mean, - moving_var - ]) - self.assertAllClose([1., 2.], np.mean(dist_samples_, axis=0), atol=5e-2) - self.assertAllClose([1., 2.], moving_mean_, atol=5e-2) - self.assertAllClose([1., 1.], moving_var_, atol=5e-2) - - def testLogProb(self): - with self.cached_session() as sess: - layer = normalization.BatchNormalization(epsilon=0.) - batch_norm = BatchNormalization(batchnorm_layer=layer, training=False) - base_dist = distributions.MultivariateNormalDiag(loc=[0., 0.]) - dist = transformed_distribution_lib.TransformedDistribution( - distribution=base_dist, - bijector=batch_norm, - validate_args=True) - samples = dist.sample(int(1e5)) - # No volume distortion since training=False, bijector is initialized - # to the identity transformation. - base_log_prob = base_dist.log_prob(samples) - dist_log_prob = dist.log_prob(samples) - variables.global_variables_initializer().run() - base_log_prob_, dist_log_prob_ = sess.run([base_log_prob, dist_log_prob]) - self.assertAllClose(base_log_prob_, dist_log_prob_) - - def testMutuallyConsistent(self): - # BatchNorm bijector is only mutually consistent when training=False. - dims = 4 - with self.cached_session() as sess: - layer = normalization.BatchNormalization(epsilon=0.) - batch_norm = BatchNormalization(batchnorm_layer=layer, training=False) - dist = transformed_distribution_lib.TransformedDistribution( - distribution=normal_lib.Normal(loc=0., scale=1.), - bijector=batch_norm, - event_shape=[dims], - validate_args=True) - self.run_test_sample_consistent_log_prob( - sess_run_fn=sess.run, - dist=dist, - num_samples=int(1e5), - radius=2., - center=0., - rtol=0.02) - - def testInvertMutuallyConsistent(self): - # BatchNorm bijector is only mutually consistent when training=False. - dims = 4 - with self.cached_session() as sess: - layer = normalization.BatchNormalization(epsilon=0.) - batch_norm = Invert( - BatchNormalization(batchnorm_layer=layer, training=False)) - dist = transformed_distribution_lib.TransformedDistribution( - distribution=normal_lib.Normal(loc=0., scale=1.), - bijector=batch_norm, - event_shape=[dims], - validate_args=True) - self.run_test_sample_consistent_log_prob( - sess_run_fn=sess.run, - dist=dist, - num_samples=int(1e5), - radius=2., - center=0., - rtol=0.02) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/chain_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/chain_test.py deleted file mode 100644 index ada99ec9c6e..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/chain_test.py +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Chain Tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.affine import Affine -from tensorflow.contrib.distributions.python.ops.bijectors.chain import Chain -from tensorflow.contrib.distributions.python.ops.bijectors.exp import Exp -from tensorflow.contrib.distributions.python.ops.bijectors.softmax_centered import SoftmaxCentered -from tensorflow.contrib.distributions.python.ops.bijectors.softplus import Softplus -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency -from tensorflow.python.platform import test - - -class ShapeChanging(bijector.Bijector): - """Only used for op_ndims manipulation.""" - - def __init__(self, forward_min_event_ndims=0, inverse_min_event_ndims=3): - super(ShapeChanging, self).__init__( - forward_min_event_ndims=forward_min_event_ndims, - inverse_min_event_ndims=inverse_min_event_ndims, - validate_args=False, name="shape_changer") - - -class ChainBijectorTest(test.TestCase): - """Tests the correctness of the Y = Chain(bij1, bij2, bij3) transformation.""" - - def testBijector(self): - with self.cached_session(): - chain = Chain((Exp(), Softplus())) - self.assertEqual("chain_of_exp_of_softplus", chain.name) - x = np.asarray([[[1., 2.], - [2., 3.]]]) - self.assertAllClose(1. + np.exp(x), chain.forward(x).eval()) - self.assertAllClose(np.log(x - 1.), chain.inverse(x).eval()) - self.assertAllClose( - -np.sum(np.log(x - 1.), axis=2), - chain.inverse_log_det_jacobian(x, event_ndims=1).eval()) - self.assertAllClose( - np.sum(x, axis=2), - chain.forward_log_det_jacobian(x, event_ndims=1).eval()) - - def testBijectorIdentity(self): - with self.cached_session(): - chain = Chain() - self.assertEqual("identity", chain.name) - x = np.asarray([[[1., 2.], - [2., 3.]]]) - self.assertAllClose(x, chain.forward(x).eval()) - self.assertAllClose(x, chain.inverse(x).eval()) - self.assertAllClose( - 0., chain.inverse_log_det_jacobian(x, event_ndims=1).eval()) - self.assertAllClose( - 0., chain.forward_log_det_jacobian(x, event_ndims=1).eval()) - - def testScalarCongruency(self): - with self.cached_session(): - chain = Chain((Exp(), Softplus())) - assert_scalar_congruency( - chain, lower_x=1e-3, upper_x=1.5, rtol=0.05) - - def testShapeGetters(self): - with self.cached_session(): - chain = Chain([ - SoftmaxCentered(validate_args=True), - SoftmaxCentered(validate_args=True), - ]) - x = tensor_shape.TensorShape([1]) - y = tensor_shape.TensorShape([2 + 1]) - self.assertAllEqual(y, chain.forward_event_shape(x)) - self.assertAllEqual( - y.as_list(), - chain.forward_event_shape_tensor(x.as_list()).eval()) - self.assertAllEqual(x, chain.inverse_event_shape(y)) - self.assertAllEqual( - x.as_list(), - chain.inverse_event_shape_tensor(y.as_list()).eval()) - - def testMinEventNdimsChain(self): - chain = Chain([Exp(), Exp(), Exp()]) - self.assertEqual(0, chain.forward_min_event_ndims) - self.assertEqual(0, chain.inverse_min_event_ndims) - - chain = Chain([Affine(), Affine(), Affine()]) - self.assertEqual(1, chain.forward_min_event_ndims) - self.assertEqual(1, chain.inverse_min_event_ndims) - - chain = Chain([Exp(), Affine()]) - self.assertEqual(1, chain.forward_min_event_ndims) - self.assertEqual(1, chain.inverse_min_event_ndims) - - chain = Chain([Affine(), Exp()]) - self.assertEqual(1, chain.forward_min_event_ndims) - self.assertEqual(1, chain.inverse_min_event_ndims) - - chain = Chain([Affine(), Exp(), Softplus(), Affine()]) - self.assertEqual(1, chain.forward_min_event_ndims) - self.assertEqual(1, chain.inverse_min_event_ndims) - - def testMinEventNdimsShapeChangingAddDims(self): - chain = Chain([ShapeChanging()]) - self.assertEqual(0, chain.forward_min_event_ndims) - self.assertEqual(3, chain.inverse_min_event_ndims) - - chain = Chain([ShapeChanging(), Affine()]) - self.assertEqual(1, chain.forward_min_event_ndims) - self.assertEqual(4, chain.inverse_min_event_ndims) - - chain = Chain([Affine(), ShapeChanging()]) - self.assertEqual(0, chain.forward_min_event_ndims) - self.assertEqual(3, chain.inverse_min_event_ndims) - - chain = Chain([ShapeChanging(), ShapeChanging()]) - self.assertEqual(0, chain.forward_min_event_ndims) - self.assertEqual(6, chain.inverse_min_event_ndims) - - def testMinEventNdimsShapeChangingRemoveDims(self): - chain = Chain([ShapeChanging(3, 0)]) - self.assertEqual(3, chain.forward_min_event_ndims) - self.assertEqual(0, chain.inverse_min_event_ndims) - - chain = Chain([ShapeChanging(3, 0), Affine()]) - self.assertEqual(3, chain.forward_min_event_ndims) - self.assertEqual(0, chain.inverse_min_event_ndims) - - chain = Chain([Affine(), ShapeChanging(3, 0)]) - self.assertEqual(4, chain.forward_min_event_ndims) - self.assertEqual(1, chain.inverse_min_event_ndims) - - chain = Chain([ShapeChanging(3, 0), ShapeChanging(3, 0)]) - self.assertEqual(6, chain.forward_min_event_ndims) - self.assertEqual(0, chain.inverse_min_event_ndims) - - def testMinEventNdimsShapeChangingAddRemoveDims(self): - chain = Chain([ - ShapeChanging(2, 1), - ShapeChanging(3, 0), - ShapeChanging(1, 2)]) - self.assertEqual(4, chain.forward_min_event_ndims) - self.assertEqual(1, chain.inverse_min_event_ndims) - - def testChainExpAffine(self): - scale_diag = np.array([1., 2., 3.], dtype=np.float32) - chain = Chain([Exp(), Affine(scale_diag=scale_diag)]) - x = [0., np.log(2., dtype=np.float32), np.log(3., dtype=np.float32)] - y = [1., 4., 27.] - self.assertAllClose(y, self.evaluate(chain.forward(x))) - self.assertAllClose(x, self.evaluate(chain.inverse(y))) - self.assertAllClose( - np.log(6, dtype=np.float32) + np.sum(scale_diag * x), - self.evaluate(chain.forward_log_det_jacobian(x, event_ndims=1))) - - self.assertAllClose( - -np.log(6, dtype=np.float32) - np.sum(scale_diag * x), - self.evaluate(chain.inverse_log_det_jacobian(y, event_ndims=1))) - - def testChainAffineExp(self): - scale_diag = np.array([1., 2., 3.], dtype=np.float32) - chain = Chain([Affine(scale_diag=scale_diag), Exp()]) - x = [0., np.log(2., dtype=np.float32), np.log(3., dtype=np.float32)] - y = [1., 4., 9.] - self.assertAllClose(y, self.evaluate(chain.forward(x))) - self.assertAllClose(x, self.evaluate(chain.inverse(y))) - self.assertAllClose( - np.log(6, dtype=np.float32) + np.sum(x), - self.evaluate(chain.forward_log_det_jacobian(x, event_ndims=1))) - - self.assertAllClose( - -np.log(6, dtype=np.float32) - np.sum(x), - self.evaluate(chain.inverse_log_det_jacobian(y, event_ndims=1))) - - def testChainIldjWithPlaceholder(self): - chain = Chain((Exp(), Exp())) - samples = array_ops.placeholder( - dtype=np.float32, shape=[None, 10], name="samples") - ildj = chain.inverse_log_det_jacobian(samples, event_ndims=0) - self.assertTrue(ildj is not None) - with self.cached_session(): - ildj.eval({samples: np.zeros([2, 10], np.float32)}) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/cholesky_outer_product_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/cholesky_outer_product_test.py deleted file mode 100644 index 9681b64cedf..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/cholesky_outer_product_test.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class CholeskyOuterProductBijectorTest(test.TestCase): - """Tests the correctness of the Y = X @ X.T transformation.""" - - def testBijectorMatrix(self): - with self.cached_session(): - bijector = bijectors.CholeskyOuterProduct(validate_args=True) - self.assertEqual("cholesky_outer_product", bijector.name) - x = [[[1., 0], [2, 1]], [[np.sqrt(2.), 0], [np.sqrt(8.), 1]]] - y = np.matmul(x, np.transpose(x, axes=(0, 2, 1))) - # Fairly easy to compute differentials since we have 2x2. - dx_dy = [[[2. * 1, 0, 0], - [2, 1, 0], - [0, 2 * 2, 2 * 1]], - [[2 * np.sqrt(2.), 0, 0], - [np.sqrt(8.), np.sqrt(2.), 0], - [0, 2 * np.sqrt(8.), 2 * 1]]] - ildj = -np.sum( - np.log(np.asarray(dx_dy).diagonal( - offset=0, axis1=1, axis2=2)), - axis=1) - self.assertAllEqual((2, 2, 2), bijector.forward(x).get_shape()) - self.assertAllEqual((2, 2, 2), bijector.inverse(y).get_shape()) - self.assertAllClose(y, bijector.forward(x).eval()) - self.assertAllClose(x, bijector.inverse(y).eval()) - self.assertAllClose( - ildj, bijector.inverse_log_det_jacobian( - y, event_ndims=2).eval(), atol=0., rtol=1e-7) - self.assertAllClose( - -bijector.inverse_log_det_jacobian( - y, event_ndims=2).eval(), - bijector.forward_log_det_jacobian( - x, event_ndims=2).eval(), - atol=0., - rtol=1e-7) - - def testNoBatchStaticJacobian(self): - x = np.eye(2) - bijector = bijectors.CholeskyOuterProduct() - - # The Jacobian matrix is 2 * tf.eye(2), which has jacobian determinant 4. - self.assertAllClose( - np.log(4), - self.evaluate(bijector.forward_log_det_jacobian(x, event_ndims=2))) - - def testNoBatchDynamicJacobian(self): - x = np.eye(2) - bijector = bijectors.CholeskyOuterProduct() - x_pl = array_ops.placeholder(dtypes.float32) - - with self.cached_session(): - log_det_jacobian = bijector.forward_log_det_jacobian(x_pl, event_ndims=2) - - # The Jacobian matrix is 2 * tf.eye(2), which has jacobian determinant 4. - self.assertAllClose( - np.log(4), - log_det_jacobian.eval({x_pl: x})) - - def testNoBatchStatic(self): - x = np.array([[1., 0], [2, 1]]) # np.linalg.cholesky(y) - y = np.array([[1., 2], [2, 5]]) # np.matmul(x, x.T) - with self.cached_session() as sess: - y_actual = bijectors.CholeskyOuterProduct().forward(x=x) - x_actual = bijectors.CholeskyOuterProduct().inverse(y=y) - [y_actual_, x_actual_] = sess.run([y_actual, x_actual]) - self.assertAllEqual([2, 2], y_actual.get_shape()) - self.assertAllEqual([2, 2], x_actual.get_shape()) - self.assertAllClose(y, y_actual_) - self.assertAllClose(x, x_actual_) - - def testNoBatchDeferred(self): - x = np.array([[1., 0], [2, 1]]) # np.linalg.cholesky(y) - y = np.array([[1., 2], [2, 5]]) # np.matmul(x, x.T) - with self.cached_session() as sess: - x_pl = array_ops.placeholder(dtypes.float32) - y_pl = array_ops.placeholder(dtypes.float32) - y_actual = bijectors.CholeskyOuterProduct().forward(x=x_pl) - x_actual = bijectors.CholeskyOuterProduct().inverse(y=y_pl) - [y_actual_, x_actual_] = sess.run([y_actual, x_actual], - feed_dict={x_pl: x, y_pl: y}) - self.assertEqual(None, y_actual.get_shape()) - self.assertEqual(None, x_actual.get_shape()) - self.assertAllClose(y, y_actual_) - self.assertAllClose(x, x_actual_) - - def testBatchStatic(self): - x = np.array([[[1., 0], - [2, 1]], - [[3., 0], - [1, 2]]]) # np.linalg.cholesky(y) - y = np.array([[[1., 2], - [2, 5]], - [[9., 3], - [3, 5]]]) # np.matmul(x, x.T) - with self.cached_session() as sess: - y_actual = bijectors.CholeskyOuterProduct().forward(x=x) - x_actual = bijectors.CholeskyOuterProduct().inverse(y=y) - [y_actual_, x_actual_] = sess.run([y_actual, x_actual]) - self.assertEqual([2, 2, 2], y_actual.get_shape()) - self.assertEqual([2, 2, 2], x_actual.get_shape()) - self.assertAllClose(y, y_actual_) - self.assertAllClose(x, x_actual_) - - def testBatchDeferred(self): - x = np.array([[[1., 0], - [2, 1]], - [[3., 0], - [1, 2]]]) # np.linalg.cholesky(y) - y = np.array([[[1., 2], - [2, 5]], - [[9., 3], - [3, 5]]]) # np.matmul(x, x.T) - with self.cached_session() as sess: - x_pl = array_ops.placeholder(dtypes.float32) - y_pl = array_ops.placeholder(dtypes.float32) - y_actual = bijectors.CholeskyOuterProduct().forward(x=x_pl) - x_actual = bijectors.CholeskyOuterProduct().inverse(y=y_pl) - [y_actual_, x_actual_] = sess.run([y_actual, x_actual], - feed_dict={x_pl: x, y_pl: y}) - self.assertEqual(None, y_actual.get_shape()) - self.assertEqual(None, x_actual.get_shape()) - self.assertAllClose(y, y_actual_) - self.assertAllClose(x, x_actual_) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/conditional_bijector_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/conditional_bijector_test.py deleted file mode 100644 index f8a52615b0f..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/conditional_bijector_test.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ConditionalBijector Tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops.bijectors.conditional_bijector import ConditionalBijector -from tensorflow.python.framework import dtypes -from tensorflow.python.platform import test - - -class _TestBijector(ConditionalBijector): - - def __init__(self): - super(_TestBijector, self).__init__( - forward_min_event_ndims=0, - graph_parents=[], - is_constant_jacobian=True, - validate_args=False, - dtype=dtypes.float32, - name="test_bijector") - - def _forward(self, _, arg1, arg2): - raise ValueError("forward", arg1, arg2) - - def _inverse(self, _, arg1, arg2): - raise ValueError("inverse", arg1, arg2) - - def _inverse_log_det_jacobian(self, _, arg1, arg2): - raise ValueError("inverse_log_det_jacobian", arg1, arg2) - - def _forward_log_det_jacobian(self, _, arg1, arg2): - raise ValueError("forward_log_det_jacobian", arg1, arg2) - - -class ConditionalBijectorTest(test.TestCase): - - def testConditionalBijector(self): - b = _TestBijector() - for name in ["forward", "inverse"]: - method = getattr(b, name) - with self.assertRaisesRegexp(ValueError, name + ".*b1.*b2"): - method(1., arg1="b1", arg2="b2") - - for name in ["inverse_log_det_jacobian", "forward_log_det_jacobian"]: - method = getattr(b, name) - with self.assertRaisesRegexp(ValueError, name + ".*b1.*b2"): - method(1., event_ndims=0, arg1="b1", arg2="b2") - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/exp_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/exp_test.py deleted file mode 100644 index d2c00865e7a..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/exp_test.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Exp Tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.exp import Exp -from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite -from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency -from tensorflow.python.platform import test - - -class ExpBijectorTest(test.TestCase): - """Tests correctness of the Y = g(X) = exp(X) transformation.""" - - def testBijector(self): - with self.cached_session(): - bijector = Exp() - self.assertEqual("exp", bijector.name) - x = [[[1.], [2.]]] - y = np.exp(x) - self.assertAllClose(y, bijector.forward(x).eval()) - self.assertAllClose(x, bijector.inverse(y).eval()) - self.assertAllClose( - -np.squeeze(np.log(y), axis=-1), - bijector.inverse_log_det_jacobian( - y, event_ndims=1).eval()) - self.assertAllClose( - -bijector.inverse_log_det_jacobian( - np.exp(x), event_ndims=1).eval(), - bijector.forward_log_det_jacobian( - x, event_ndims=1).eval()) - - def testScalarCongruency(self): - with self.cached_session(): - bijector = Exp() - assert_scalar_congruency( - bijector, lower_x=-2., upper_x=1.5, rtol=0.05) - - def testBijectiveAndFinite(self): - with self.cached_session(): - bijector = Exp() - x = np.linspace(-10, 10, num=10).astype(np.float32) - y = np.logspace(-10, 10, num=10).astype(np.float32) - assert_bijective_and_finite(bijector, x, y, event_ndims=0) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/fill_triangular_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/fill_triangular_test.py deleted file mode 100644 index 3530e142e4d..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/fill_triangular_test.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for FillTriangular bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class FillTriangularBijectorTest(test.TestCase): - """Tests the correctness of the FillTriangular bijector.""" - - @test_util.run_in_graph_and_eager_modes - def testBijector(self): - x = np.float32(np.array([1., 2., 3.])) - y = np.float32(np.array([[3., 0.], - [2., 1.]])) - - b = bijectors.FillTriangular() - - y_ = self.evaluate(b.forward(x)) - self.assertAllClose(y, y_) - - x_ = self.evaluate(b.inverse(y)) - self.assertAllClose(x, x_) - - fldj = self.evaluate(b.forward_log_det_jacobian(x, event_ndims=1)) - self.assertAllClose(fldj, 0.) - - ildj = self.evaluate(b.inverse_log_det_jacobian(y, event_ndims=2)) - self.assertAllClose(ildj, 0.) - - @test_util.run_in_graph_and_eager_modes - def testShape(self): - x_shape = tensor_shape.TensorShape([5, 4, 6]) - y_shape = tensor_shape.TensorShape([5, 4, 3, 3]) - - b = bijectors.FillTriangular(validate_args=True) - - x = array_ops.ones(shape=x_shape, dtype=dtypes.float32) - y_ = b.forward(x) - self.assertAllEqual(y_.shape.as_list(), y_shape.as_list()) - x_ = b.inverse(y_) - self.assertAllEqual(x_.shape.as_list(), x_shape.as_list()) - - y_shape_ = b.forward_event_shape(x_shape) - self.assertAllEqual(y_shape_.as_list(), y_shape.as_list()) - x_shape_ = b.inverse_event_shape(y_shape) - self.assertAllEqual(x_shape_.as_list(), x_shape.as_list()) - - y_shape_tensor = self.evaluate( - b.forward_event_shape_tensor(x_shape.as_list())) - self.assertAllEqual(y_shape_tensor, y_shape.as_list()) - x_shape_tensor = self.evaluate( - b.inverse_event_shape_tensor(y_shape.as_list())) - self.assertAllEqual(x_shape_tensor, x_shape.as_list()) - - @test_util.run_in_graph_and_eager_modes - def testShapeError(self): - - b = bijectors.FillTriangular(validate_args=True) - - x_shape_bad = tensor_shape.TensorShape([5, 4, 7]) - with self.assertRaisesRegexp(ValueError, "is not a triangular number"): - b.forward_event_shape(x_shape_bad) - with self.assertRaisesOpError("is not a triangular number"): - self.evaluate(b.forward_event_shape_tensor(x_shape_bad.as_list())) - - y_shape_bad = tensor_shape.TensorShape([5, 4, 3, 2]) - with self.assertRaisesRegexp(ValueError, "Matrix must be square"): - b.inverse_event_shape(y_shape_bad) - with self.assertRaisesOpError("Matrix must be square"): - self.evaluate(b.inverse_event_shape_tensor(y_shape_bad.as_list())) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py deleted file mode 100644 index b9cdbfb823d..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import stats - -from tensorflow.contrib.distributions.python.ops.bijectors.gumbel import Gumbel -from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite -from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency -from tensorflow.python.platform import test - - -class GumbelBijectorTest(test.TestCase): - """Tests correctness of the Gumbel bijector.""" - - def testBijector(self): - with self.cached_session(): - loc = 0.3 - scale = 5. - bijector = Gumbel(loc=loc, scale=scale, validate_args=True) - self.assertEqual("gumbel", bijector.name) - x = np.array([[[-3.], [0.], [0.5], [4.2], [12.]]], dtype=np.float32) - # Gumbel distribution - gumbel_dist = stats.gumbel_r(loc=loc, scale=scale) - y = gumbel_dist.cdf(x).astype(np.float32) - self.assertAllClose(y, bijector.forward(x).eval()) - self.assertAllClose(x, bijector.inverse(y).eval()) - self.assertAllClose( - np.squeeze(gumbel_dist.logpdf(x), axis=-1), - bijector.forward_log_det_jacobian(x, event_ndims=1).eval()) - self.assertAllClose( - -bijector.inverse_log_det_jacobian(y, event_ndims=1).eval(), - bijector.forward_log_det_jacobian(x, event_ndims=1).eval(), - rtol=1e-4, - atol=0.) - - def testScalarCongruency(self): - with self.cached_session(): - assert_scalar_congruency( - Gumbel(loc=0.3, scale=20.), lower_x=1., upper_x=100., rtol=0.02) - - def testBijectiveAndFinite(self): - with self.cached_session(): - bijector = Gumbel(loc=0., scale=3.0, validate_args=True) - x = np.linspace(-10., 10., num=10).astype(np.float32) - y = np.linspace(0.01, 0.99, num=10).astype(np.float32) - assert_bijective_and_finite(bijector, x, y, event_ndims=0, rtol=1e-3) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/inline_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/inline_test.py deleted file mode 100644 index c9bccb36fcc..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/inline_test.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.exp import Exp -from tensorflow.contrib.distributions.python.ops.bijectors.inline import Inline -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class InlineBijectorTest(test.TestCase): - """Tests correctness of the inline constructed bijector.""" - - def testBijector(self): - with self.cached_session(): - exp = Exp() - inline = Inline( - forward_fn=math_ops.exp, - inverse_fn=math_ops.log, - inverse_log_det_jacobian_fn=lambda y: -math_ops.log(y), - forward_log_det_jacobian_fn=lambda x: x, - forward_min_event_ndims=0, - name="exp") - - self.assertEqual(exp.name, inline.name) - x = [[[1., 2.], [3., 4.], [5., 6.]]] - y = np.exp(x) - self.assertAllClose(y, inline.forward(x).eval()) - self.assertAllClose(x, inline.inverse(y).eval()) - self.assertAllClose( - -np.sum(np.log(y), axis=-1), - inline.inverse_log_det_jacobian(y, event_ndims=1).eval()) - self.assertAllClose( - -inline.inverse_log_det_jacobian(y, event_ndims=1).eval(), - inline.forward_log_det_jacobian(x, event_ndims=1).eval()) - - def testShapeGetters(self): - with self.cached_session(): - bijector = Inline( - forward_event_shape_tensor_fn=lambda x: array_ops.concat((x, [1]), 0), - forward_event_shape_fn=lambda x: x.as_list() + [1], - inverse_event_shape_tensor_fn=lambda x: x[:-1], - inverse_event_shape_fn=lambda x: x[:-1], - forward_min_event_ndims=0, - name="shape_only") - x = tensor_shape.TensorShape([1, 2, 3]) - y = tensor_shape.TensorShape([1, 2, 3, 1]) - self.assertAllEqual(y, bijector.forward_event_shape(x)) - self.assertAllEqual( - y.as_list(), - bijector.forward_event_shape_tensor(x.as_list()).eval()) - self.assertAllEqual(x, bijector.inverse_event_shape(y)) - self.assertAllEqual( - x.as_list(), - bijector.inverse_event_shape_tensor(y.as_list()).eval()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/invert_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/invert_test.py deleted file mode 100644 index 7e3340aeb0e..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/invert_test.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops.distributions import gamma as gamma_lib -from tensorflow.python.ops.distributions import transformed_distribution as transformed_distribution_lib -from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency -from tensorflow.python.platform import test - - -class InvertBijectorTest(test.TestCase): - """Tests the correctness of the Y = Invert(bij) transformation.""" - - def testBijector(self): - with self.cached_session(): - for fwd in [ - bijectors.Identity(), - bijectors.Exp(), - bijectors.Affine(shift=[0., 1.], scale_diag=[2., 3.]), - bijectors.Softplus(), - bijectors.SoftmaxCentered(), - ]: - rev = bijectors.Invert(fwd) - self.assertEqual("_".join(["invert", fwd.name]), rev.name) - x = [[[1., 2.], - [2., 3.]]] - self.assertAllClose(fwd.inverse(x).eval(), rev.forward(x).eval()) - self.assertAllClose(fwd.forward(x).eval(), rev.inverse(x).eval()) - self.assertAllClose( - fwd.forward_log_det_jacobian(x, event_ndims=1).eval(), - rev.inverse_log_det_jacobian(x, event_ndims=1).eval()) - self.assertAllClose( - fwd.inverse_log_det_jacobian(x, event_ndims=1).eval(), - rev.forward_log_det_jacobian(x, event_ndims=1).eval()) - - def testScalarCongruency(self): - with self.cached_session(): - bijector = bijectors.Invert(bijectors.Exp()) - assert_scalar_congruency( - bijector, lower_x=1e-3, upper_x=1.5, rtol=0.05) - - def testShapeGetters(self): - with self.cached_session(): - bijector = bijectors.Invert(bijectors.SoftmaxCentered(validate_args=True)) - x = tensor_shape.TensorShape([2]) - y = tensor_shape.TensorShape([1]) - self.assertAllEqual(y, bijector.forward_event_shape(x)) - self.assertAllEqual( - y.as_list(), - bijector.forward_event_shape_tensor(x.as_list()).eval()) - self.assertAllEqual(x, bijector.inverse_event_shape(y)) - self.assertAllEqual( - x.as_list(), - bijector.inverse_event_shape_tensor(y.as_list()).eval()) - - def testDocstringExample(self): - with self.cached_session(): - exp_gamma_distribution = ( - transformed_distribution_lib.TransformedDistribution( - distribution=gamma_lib.Gamma(concentration=1., rate=2.), - bijector=bijectors.Invert(bijectors.Exp()))) - self.assertAllEqual( - [], array_ops.shape(exp_gamma_distribution.sample()).eval()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py deleted file mode 100644 index b3fb50005e5..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Kumaraswamy Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.kumaraswamy import Kumaraswamy -from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite -from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency -from tensorflow.python.platform import test - - -class KumaraswamyBijectorTest(test.TestCase): - """Tests correctness of the Kumaraswamy bijector.""" - - def testBijector(self): - with self.cached_session(): - a = 2. - b = 0.3 - bijector = Kumaraswamy( - concentration1=a, concentration0=b, validate_args=True) - self.assertEqual("kumaraswamy", bijector.name) - x = np.array([[[0.1], [0.2], [0.3], [0.4], [0.5]]], dtype=np.float32) - # Kumaraswamy cdf. This is the same as inverse(x). - y = 1. - (1. - x ** a) ** b - self.assertAllClose(y, bijector.inverse(x).eval()) - self.assertAllClose(x, bijector.forward(y).eval()) - kumaraswamy_log_pdf = (np.log(a) + np.log(b) + (a - 1) * np.log(x) + - (b - 1) * np.log1p(-x ** a)) - - self.assertAllClose( - np.squeeze(kumaraswamy_log_pdf, axis=-1), - bijector.inverse_log_det_jacobian(x, event_ndims=1).eval()) - self.assertAllClose( - -bijector.inverse_log_det_jacobian(x, event_ndims=1).eval(), - bijector.forward_log_det_jacobian(y, event_ndims=1).eval(), - rtol=1e-4, - atol=0.) - - def testScalarCongruency(self): - with self.cached_session(): - assert_scalar_congruency( - Kumaraswamy(concentration1=0.5, concentration0=1.1), - lower_x=0., upper_x=1., n=int(10e3), rtol=0.02) - - def testBijectiveAndFinite(self): - with self.cached_session(): - concentration1 = 1.2 - concentration0 = 2. - bijector = Kumaraswamy( - concentration1=concentration1, - concentration0=concentration0, validate_args=True) - # Omitting the endpoints 0 and 1, since idlj will be infinity at these - # endpoints. - y = np.linspace(.01, 0.99, num=10).astype(np.float32) - x = 1 - (1 - y ** concentration1) ** concentration0 - assert_bijective_and_finite(bijector, x, y, event_ndims=0, rtol=1e-3) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py deleted file mode 100644 index ad4329d4259..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for MaskedAutoregressiveFlow.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import test_util -from tensorflow.contrib.distributions.python.ops.bijectors.invert import Invert -from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import _gen_mask -from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import masked_autoregressive_default_template -from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import MaskedAutoregressiveFlow -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variables -from tensorflow.python.ops.distributions import normal as normal_lib -from tensorflow.python.ops.distributions import transformed_distribution as transformed_distribution_lib -from tensorflow.python.platform import test - - -class GenMaskTest(test.TestCase): - - def test346Exclusive(self): - expected_mask = np.array( - [[0, 0, 0, 0], - [0, 0, 0, 0], - [1, 0, 0, 0], - [1, 0, 0, 0], - [1, 1, 0, 0], - [1, 1, 0, 0]]) - mask = _gen_mask(num_blocks=3, n_in=4, n_out=6, mask_type="exclusive") - self.assertAllEqual(expected_mask, mask) - - def test346Inclusive(self): - expected_mask = np.array( - [[1, 0, 0, 0], - [1, 0, 0, 0], - [1, 1, 0, 0], - [1, 1, 0, 0], - [1, 1, 1, 0], - [1, 1, 1, 0]]) - mask = _gen_mask(num_blocks=3, n_in=4, n_out=6, mask_type="inclusive") - self.assertAllEqual(expected_mask, mask) - - -class MaskedAutoregressiveFlowTest(test_util.VectorDistributionTestHelpers, - test.TestCase): - - @property - def _autoregressive_flow_kwargs(self): - return { - "shift_and_log_scale_fn": masked_autoregressive_default_template( - hidden_layers=[2], shift_only=False), - "is_constant_jacobian": False, - } - - def testBijector(self): - x_ = np.arange(3 * 4 * 2).astype(np.float32).reshape(3, 4, 2) - with self.cached_session() as sess: - ma = MaskedAutoregressiveFlow( - validate_args=True, - **self._autoregressive_flow_kwargs) - x = constant_op.constant(x_) - forward_x = ma.forward(x) - # Use identity to invalidate cache. - inverse_y = ma.inverse(array_ops.identity(forward_x)) - fldj = ma.forward_log_det_jacobian(x, event_ndims=1) - # Use identity to invalidate cache. - ildj = ma.inverse_log_det_jacobian( - array_ops.identity(forward_x), event_ndims=1) - variables.global_variables_initializer().run() - [ - forward_x_, - inverse_y_, - ildj_, - fldj_, - ] = sess.run([ - forward_x, - inverse_y, - ildj, - fldj, - ]) - self.assertEqual("masked_autoregressive_flow", ma.name) - self.assertAllClose(forward_x_, forward_x_, rtol=1e-6, atol=0.) - self.assertAllClose(x_, inverse_y_, rtol=1e-5, atol=0.) - self.assertAllClose(ildj_, -fldj_, rtol=1e-6, atol=0.) - - def testMutuallyConsistent(self): - dims = 4 - with self.cached_session() as sess: - ma = MaskedAutoregressiveFlow( - validate_args=True, - **self._autoregressive_flow_kwargs) - dist = transformed_distribution_lib.TransformedDistribution( - distribution=normal_lib.Normal(loc=0., scale=1.), - bijector=ma, - event_shape=[dims], - validate_args=True) - self.run_test_sample_consistent_log_prob( - sess_run_fn=sess.run, - dist=dist, - num_samples=int(1e5), - radius=1., - center=0., - rtol=0.02) - - def testInvertMutuallyConsistent(self): - dims = 4 - with self.cached_session() as sess: - ma = Invert(MaskedAutoregressiveFlow( - validate_args=True, - **self._autoregressive_flow_kwargs)) - dist = transformed_distribution_lib.TransformedDistribution( - distribution=normal_lib.Normal(loc=0., scale=1.), - bijector=ma, - event_shape=[dims], - validate_args=True) - self.run_test_sample_consistent_log_prob( - sess_run_fn=sess.run, - dist=dist, - num_samples=int(1e5), - radius=1., - center=0., - rtol=0.02) - - -class MaskedAutoregressiveFlowShiftOnlyTest(MaskedAutoregressiveFlowTest): - - @property - def _autoregressive_flow_kwargs(self): - return { - "shift_and_log_scale_fn": masked_autoregressive_default_template( - hidden_layers=[2], shift_only=True), - "is_constant_jacobian": True, - } - - -class MaskedAutoregressiveFlowUnrollLoopTest(MaskedAutoregressiveFlowTest): - - @property - def _autoregressive_flow_kwargs(self): - return { - "shift_and_log_scale_fn": masked_autoregressive_default_template( - hidden_layers=[2], shift_only=False), - "is_constant_jacobian": False, - "unroll_loop": True, - } - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/matrix_inverse_tril_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/matrix_inverse_tril_test.py deleted file mode 100644 index 31ee36f024e..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/matrix_inverse_tril_test.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for MatrixInverseTriL bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.python.framework import errors -from tensorflow.python.framework import test_util -from tensorflow.python.platform import test - - -@test_util.run_all_in_graph_and_eager_modes -class MatrixInverseTriLBijectorTest(test.TestCase): - """Tests the correctness of the Y = inv(tril) transformation.""" - - #The inverse of 0 is undefined, as the numbers above the main - #diagonal must be zero, we zero out these numbers after running inverse. - #See: https://github.com/numpy/numpy/issues/11445 - def _inv(self, x): - y = np.linalg.inv(x) - #triu_indices only works on 2d arrays - #need to iterate over all the 2d arrays in a x-dimensional array. - for idx in np.ndindex(y.shape[0:-2]): - y[idx][np.triu_indices(y[idx].shape[-1], 1)] = 0 - return y - - def testComputesCorrectValues(self): - inv = bijectors.MatrixInverseTriL(validate_args=True) - self.assertEqual("matrix_inverse_tril", inv.name) - x_ = np.array([[0.7, 0., 0.], - [0.1, -1., 0.], - [0.3, 0.25, 0.5]], dtype=np.float32) - x_inv_ = np.linalg.inv(x_) - expected_fldj_ = -6. * np.sum(np.log(np.abs(np.diag(x_)))) - - y = inv.forward(x_) - x_back = inv.inverse(x_inv_) - fldj = inv.forward_log_det_jacobian(x_, event_ndims=2) - ildj = inv.inverse_log_det_jacobian(x_inv_, event_ndims=2) - - y_, x_back_, fldj_, ildj_ = self.evaluate([y, x_back, fldj, ildj]) - - self.assertAllClose(x_inv_, y_, atol=0., rtol=1e-5) - self.assertAllClose(x_, x_back_, atol=0., rtol=1e-5) - self.assertNear(expected_fldj_, fldj_, err=1e-3) - self.assertNear(-expected_fldj_, ildj_, err=1e-3) - - def testOneByOneMatrix(self): - inv = bijectors.MatrixInverseTriL(validate_args=True) - x_ = np.array([[5.]], dtype=np.float32) - x_inv_ = np.array([[0.2]], dtype=np.float32) - expected_fldj_ = np.log(0.04) - - y = inv.forward(x_) - x_back = inv.inverse(x_inv_) - fldj = inv.forward_log_det_jacobian(x_, event_ndims=2) - ildj = inv.inverse_log_det_jacobian(x_inv_, event_ndims=2) - - y_, x_back_, fldj_, ildj_ = self.evaluate([y, x_back, fldj, ildj]) - - self.assertAllClose(x_inv_, y_, atol=0., rtol=1e-5) - self.assertAllClose(x_, x_back_, atol=0., rtol=1e-5) - self.assertNear(expected_fldj_, fldj_, err=1e-3) - self.assertNear(-expected_fldj_, ildj_, err=1e-3) - - def testZeroByZeroMatrix(self): - inv = bijectors.MatrixInverseTriL(validate_args=True) - x_ = np.eye(0, dtype=np.float32) - x_inv_ = np.eye(0, dtype=np.float32) - expected_fldj_ = 0. - - y = inv.forward(x_) - x_back = inv.inverse(x_inv_) - fldj = inv.forward_log_det_jacobian(x_, event_ndims=2) - ildj = inv.inverse_log_det_jacobian(x_inv_, event_ndims=2) - - y_, x_back_, fldj_, ildj_ = self.evaluate([y, x_back, fldj, ildj]) - - self.assertAllClose(x_inv_, y_, atol=0., rtol=1e-5) - self.assertAllClose(x_, x_back_, atol=0., rtol=1e-5) - self.assertNear(expected_fldj_, fldj_, err=1e-3) - self.assertNear(-expected_fldj_, ildj_, err=1e-3) - - def testBatch(self): - # Test batch computation with input shape (2, 1, 2, 2), i.e. batch shape - # (2, 1). - inv = bijectors.MatrixInverseTriL(validate_args=True) - x_ = np.array([[[[1., 0.], - [2., 3.]]], - [[[4., 0.], - [5., -6.]]]], dtype=np.float32) - x_inv_ = self._inv(x_) - expected_fldj_ = -4. * np.sum( - np.log(np.abs(np.diagonal(x_, axis1=-2, axis2=-1))), axis=-1) - - y = inv.forward(x_) - x_back = inv.inverse(x_inv_) - fldj = inv.forward_log_det_jacobian(x_, event_ndims=2) - ildj = inv.inverse_log_det_jacobian(x_inv_, event_ndims=2) - - y_, x_back_, fldj_, ildj_ = self.evaluate([y, x_back, fldj, ildj]) - - self.assertAllClose(x_inv_, y_, atol=0., rtol=1e-5) - self.assertAllClose(x_, x_back_, atol=0., rtol=1e-5) - self.assertAllClose(expected_fldj_, fldj_, atol=0., rtol=1e-3) - self.assertAllClose(-expected_fldj_, ildj_, atol=0., rtol=1e-3) - - def testErrorOnInputRankTooLow(self): - inv = bijectors.MatrixInverseTriL(validate_args=True) - x_ = np.array([0.1], dtype=np.float32) - rank_error_msg = "must have rank at least 2" - with self.assertRaisesWithPredicateMatch(ValueError, rank_error_msg): - self.evaluate(inv.forward(x_)) - with self.assertRaisesWithPredicateMatch(ValueError, rank_error_msg): - self.evaluate(inv.inverse(x_)) - with self.assertRaisesWithPredicateMatch(ValueError, rank_error_msg): - self.evaluate(inv.forward_log_det_jacobian(x_, event_ndims=2)) - with self.assertRaisesWithPredicateMatch(ValueError, rank_error_msg): - self.evaluate(inv.inverse_log_det_jacobian(x_, event_ndims=2)) - - # TODO(b/80481923): Figure out why these assertions fail, and fix them. - ## def testErrorOnInputNonSquare(self): - ## inv = bijectors.MatrixInverseTriL(validate_args=True) - ## x_ = np.array([[1., 2., 3.], - ## [4., 5., 6.]], dtype=np.float32) - ## square_error_msg = "must be a square matrix" - ## with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - ## square_error_msg): - ## self.evaluate(inv.forward(x_)) - ## with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - ## square_error_msg): - ## self.evaluate(inv.inverse(x_)) - ## with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - ## square_error_msg): - ## self.evaluate(inv.forward_log_det_jacobian(x_, event_ndims=2)) - ## with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - ## square_error_msg): - ## self.evaluate(inv.inverse_log_det_jacobian(x_, event_ndims=2)) - - def testErrorOnInputNotLowerTriangular(self): - inv = bijectors.MatrixInverseTriL(validate_args=True) - x_ = np.array([[1., 2.], - [3., 4.]], dtype=np.float32) - triangular_error_msg = "must be lower triangular" - with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - triangular_error_msg): - self.evaluate(inv.forward(x_)) - with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - triangular_error_msg): - self.evaluate(inv.inverse(x_)) - with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - triangular_error_msg): - self.evaluate(inv.forward_log_det_jacobian(x_, event_ndims=2)) - with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - triangular_error_msg): - self.evaluate(inv.inverse_log_det_jacobian(x_, event_ndims=2)) - - def testErrorOnInputSingular(self): - inv = bijectors.MatrixInverseTriL(validate_args=True) - x_ = np.array([[1., 0.], - [0., 0.]], dtype=np.float32) - nonsingular_error_msg = "must have all diagonal entries nonzero" - with self.assertRaisesOpError(nonsingular_error_msg): - self.evaluate(inv.forward(x_)) - with self.assertRaisesOpError(nonsingular_error_msg): - self.evaluate(inv.inverse(x_)) - with self.assertRaisesOpError(nonsingular_error_msg): - self.evaluate(inv.forward_log_det_jacobian(x_, event_ndims=2)) - with self.assertRaisesOpError(nonsingular_error_msg): - self.evaluate(inv.inverse_log_det_jacobian(x_, event_ndims=2)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/ordered_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/ordered_test.py deleted file mode 100644 index 9a88f8f1bc9..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/ordered_test.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.ordered import Ordered -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite -from tensorflow.python.platform import test - - - -class OrderedBijectorTest(test.TestCase): - """Tests correctness of the ordered transformation.""" - - def setUp(self): - self._rng = np.random.RandomState(42) - - @test_util.run_in_graph_and_eager_modes - def testBijectorVector(self): - ordered = Ordered() - self.assertEqual("ordered", ordered.name) - x = np.asarray([[2., 3, 4], [4., 8, 13]]) - y = [[2., 0, 0], [4., np.log(4.), np.log(5.)]] - self.assertAllClose(y, self.evaluate(ordered.forward(x))) - self.assertAllClose(x, self.evaluate(ordered.inverse(y))) - self.assertAllClose( - np.sum(np.asarray(y)[..., 1:], axis=-1), - self.evaluate(ordered.inverse_log_det_jacobian(y, event_ndims=1)), - atol=0., - rtol=1e-7) - self.assertAllClose( - self.evaluate(-ordered.inverse_log_det_jacobian(y, event_ndims=1)), - self.evaluate(ordered.forward_log_det_jacobian(x, event_ndims=1)), - atol=0., - rtol=1e-7) - - def testBijectorUnknownShape(self): - with self.cached_session(): - ordered = Ordered() - self.assertEqual("ordered", ordered.name) - x = array_ops.placeholder(shape=[2, None], dtype=dtypes.float32) - real_x = np.asarray([[2., 3, 4], [4., 8, 13]]) - y = array_ops.placeholder(shape=[2, None], dtype=dtypes.float32) - real_y = [[2., 0, 0], [4., np.log(4.), np.log(5.)]] - self.assertAllClose(real_y, ordered.forward(x).eval( - feed_dict={x: real_x})) - self.assertAllClose(real_x, ordered.inverse(y).eval( - feed_dict={y: real_y})) - self.assertAllClose( - np.sum(np.asarray(real_y)[..., 1:], axis=-1), - ordered.inverse_log_det_jacobian(y, event_ndims=1).eval( - feed_dict={y: real_y}), - atol=0., - rtol=1e-7) - self.assertAllClose( - -ordered.inverse_log_det_jacobian(y, event_ndims=1).eval( - feed_dict={y: real_y}), - ordered.forward_log_det_jacobian(x, event_ndims=1).eval( - feed_dict={x: real_x}), - atol=0., - rtol=1e-7) - - @test_util.run_in_graph_and_eager_modes - def testShapeGetters(self): - x = tensor_shape.TensorShape([4]) - y = tensor_shape.TensorShape([4]) - bijector = Ordered(validate_args=True) - self.assertAllEqual(y, bijector.forward_event_shape(x)) - self.assertAllEqual(y.as_list(), - self.evaluate(bijector.forward_event_shape_tensor( - x.as_list()))) - self.assertAllEqual(x, bijector.inverse_event_shape(y)) - self.assertAllEqual(x.as_list(), - self.evaluate(bijector.inverse_event_shape_tensor( - y.as_list()))) - - def testBijectiveAndFinite(self): - with self.cached_session(): - ordered = Ordered() - x = np.sort(self._rng.randn(3, 10), axis=-1).astype(np.float32) - y = (self._rng.randn(3, 10)).astype(np.float32) - assert_bijective_and_finite(ordered, x, y, event_ndims=1) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/permute_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/permute_test.py deleted file mode 100644 index e2062ed55d5..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/permute_test.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Permute bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.permute import Permute -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite -from tensorflow.python.platform import test - - -class PermuteBijectorTest(test.TestCase): - """Tests correctness of the Permute bijector.""" - - def setUp(self): - self._rng = np.random.RandomState(42) - - def testBijector(self): - expected_permutation = np.int32([2, 0, 1]) - expected_x = np.random.randn(4, 2, 3) - expected_y = expected_x[..., expected_permutation] - - with self.cached_session() as sess: - permutation_ph = array_ops.placeholder(dtype=dtypes.int32) - bijector = Permute( - permutation=permutation_ph, - validate_args=True) - [ - permutation_, - x_, - y_, - fldj, - ildj, - ] = sess.run([ - bijector.permutation, - bijector.inverse(expected_y), - bijector.forward(expected_x), - bijector.forward_log_det_jacobian(expected_x, event_ndims=1), - bijector.inverse_log_det_jacobian(expected_y, event_ndims=1), - ], feed_dict={permutation_ph: expected_permutation}) - self.assertEqual("permute", bijector.name) - self.assertAllEqual(expected_permutation, permutation_) - self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) - self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) - self.assertAllClose(0., fldj, rtol=1e-6, atol=0) - self.assertAllClose(0., ildj, rtol=1e-6, atol=0) - - def testRaisesOpError(self): - with self.cached_session() as sess: - with self.assertRaisesOpError("Permutation over `d` must contain"): - permutation_ph = array_ops.placeholder(dtype=dtypes.int32) - bijector = Permute( - permutation=permutation_ph, - validate_args=True) - sess.run(bijector.inverse([1.]), - feed_dict={permutation_ph: [1, 2]}) - - def testBijectiveAndFinite(self): - permutation = np.int32([2, 0, 1]) - x = np.random.randn(4, 2, 3) - y = x[..., permutation] - with self.cached_session(): - bijector = Permute(permutation=permutation, validate_args=True) - assert_bijective_and_finite( - bijector, x, y, event_ndims=1, rtol=1e-6, atol=0) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/power_transform_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/power_transform_test.py deleted file mode 100644 index ef303ab664c..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/power_transform_test.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.power_transform import PowerTransform -from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite -from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency -from tensorflow.python.platform import test - - -class PowerTransformBijectorTest(test.TestCase): - """Tests correctness of the power transformation.""" - - def testBijector(self): - with self.cached_session(): - c = 0.2 - bijector = PowerTransform(power=c, validate_args=True) - self.assertEqual("power_transform", bijector.name) - x = np.array([[[-1.], [2.], [-5. + 1e-4]]]) - y = (1. + x * c)**(1. / c) - self.assertAllClose(y, bijector.forward(x).eval()) - self.assertAllClose(x, bijector.inverse(y).eval()) - self.assertAllClose( - (c - 1.) * np.sum(np.log(y), axis=-1), - bijector.inverse_log_det_jacobian(y, event_ndims=1).eval()) - self.assertAllClose( - -bijector.inverse_log_det_jacobian(y, event_ndims=1).eval(), - bijector.forward_log_det_jacobian(x, event_ndims=1).eval(), - rtol=1e-4, - atol=0.) - - def testScalarCongruency(self): - with self.cached_session(): - bijector = PowerTransform(power=0.2, validate_args=True) - assert_scalar_congruency( - bijector, lower_x=-2., upper_x=1.5, rtol=0.05) - - def testBijectiveAndFinite(self): - with self.cached_session(): - bijector = PowerTransform(power=0.2, validate_args=True) - x = np.linspace(-4.999, 10, num=10).astype(np.float32) - y = np.logspace(0.001, 10, num=10).astype(np.float32) - assert_bijective_and_finite(bijector, x, y, event_ndims=0, rtol=1e-3) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/real_nvp_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/real_nvp_test.py deleted file mode 100644 index b3b7b8535e1..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/real_nvp_test.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for MaskedAutoregressiveFlow.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib.distributions.python.ops import test_util -from tensorflow.contrib.distributions.python.ops.bijectors.invert import Invert -from tensorflow.contrib.distributions.python.ops.bijectors.real_nvp import real_nvp_default_template -from tensorflow.contrib.distributions.python.ops.bijectors.real_nvp import RealNVP -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variables -from tensorflow.python.ops.distributions import normal as normal_lib -from tensorflow.python.ops.distributions import transformed_distribution as transformed_distribution_lib -from tensorflow.python.platform import test - - -class RealNVPTest(test_util.VectorDistributionTestHelpers, test.TestCase): - - @property - def _real_nvp_kwargs(self): - return { - "shift_and_log_scale_fn": real_nvp_default_template( - hidden_layers=[3], shift_only=False), - "is_constant_jacobian": False, - } - - def testBijector(self): - x_ = np.arange(3 * 4 * 2).astype(np.float32).reshape(3, 4 * 2) - with self.cached_session() as sess: - nvp = RealNVP( - num_masked=4, - validate_args=True, - **self._real_nvp_kwargs) - x = constant_op.constant(x_) - forward_x = nvp.forward(x) - # Use identity to invalidate cache. - inverse_y = nvp.inverse(array_ops.identity(forward_x)) - forward_inverse_y = nvp.forward(inverse_y) - fldj = nvp.forward_log_det_jacobian(x, event_ndims=1) - # Use identity to invalidate cache. - ildj = nvp.inverse_log_det_jacobian( - array_ops.identity(forward_x), event_ndims=1) - variables.global_variables_initializer().run() - [ - forward_x_, - inverse_y_, - forward_inverse_y_, - ildj_, - fldj_, - ] = sess.run([ - forward_x, - inverse_y, - forward_inverse_y, - ildj, - fldj, - ]) - self.assertEqual("real_nvp", nvp.name) - self.assertAllClose(forward_x_, forward_inverse_y_, rtol=1e-1, atol=0.) - self.assertAllClose(x_, inverse_y_, rtol=1e-1, atol=0.) - self.assertAllClose(ildj_, -fldj_, rtol=1e-6, atol=0.) - - def testMutuallyConsistent(self): - dims = 4 - with self.cached_session() as sess: - nvp = RealNVP( - num_masked=3, - validate_args=True, - **self._real_nvp_kwargs) - dist = transformed_distribution_lib.TransformedDistribution( - distribution=normal_lib.Normal(loc=0., scale=1.), - bijector=nvp, - event_shape=[dims], - validate_args=True) - self.run_test_sample_consistent_log_prob( - sess_run_fn=sess.run, - dist=dist, - num_samples=int(1e5), - radius=1., - center=0., - rtol=0.02) - - def testInvertMutuallyConsistent(self): - dims = 4 - with self.cached_session() as sess: - nvp = Invert(RealNVP( - num_masked=3, - validate_args=True, - **self._real_nvp_kwargs)) - dist = transformed_distribution_lib.TransformedDistribution( - distribution=normal_lib.Normal(loc=0., scale=1.), - bijector=nvp, - event_shape=[dims], - validate_args=True) - self.run_test_sample_consistent_log_prob( - sess_run_fn=sess.run, - dist=dist, - num_samples=int(1e5), - radius=1., - center=0., - rtol=0.02) - - -class NICETest(RealNVPTest): - - @property - def _real_nvp_kwargs(self): - return { - "shift_and_log_scale_fn": real_nvp_default_template( - hidden_layers=[2], shift_only=True), - "is_constant_jacobian": True, - } - - -class RealNVPConstantShiftScaleTest(RealNVPTest): - - @property - def _real_nvp_kwargs(self): - - def constant_shift_log_scale_fn(x0, output_units): - del x0, output_units - shift = constant_op.constant([0.1]) - log_scale = constant_op.constant([0.5]) - return shift, log_scale - - return { - "shift_and_log_scale_fn": constant_shift_log_scale_fn, - "is_constant_jacobian": True, - } - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py deleted file mode 100644 index f3d63da373a..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py +++ /dev/null @@ -1,374 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Reshape Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.reshape import Reshape -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite -from tensorflow.python.platform import test - - -class _ReshapeBijectorTest(object): - """Base class for testing the reshape transformation. - - Methods defined in this class call a method self.build_shapes() that - is implemented by subclasses defined below, returning respectively - ReshapeBijectorTestStatic: static shapes, - ReshapeBijectorTestDynamic: shape placeholders of known ndims, and - ReshapeBijectorTestDynamicNdims: shape placeholders of unspecified ndims, - so that each test in this base class is automatically run over all - three cases. The subclasses also implement assertRaisesError to test - for either Python exceptions (in the case of static shapes) or - TensorFlow op errors (dynamic shapes). - """ - - def setUp(self): - self._rng = np.random.RandomState(42) - - def testBijector(self): - """Do a basic sanity check of forward, inverse, jacobian.""" - expected_x = np.random.randn(4, 3, 2) - expected_y = np.reshape(expected_x, [4, 6]) - - with self.cached_session() as sess: - shape_in, shape_out, feed_dict = self.build_shapes([3, 2], [6,]) - bijector = Reshape( - event_shape_out=shape_out, - event_shape_in=shape_in, - validate_args=True) - (x_, - y_, - fldj_, - ildj_) = sess.run(( - bijector.inverse(expected_y), - bijector.forward(expected_x), - bijector.forward_log_det_jacobian(expected_x, event_ndims=2), - bijector.inverse_log_det_jacobian(expected_y, event_ndims=2), - ), feed_dict=feed_dict) - self.assertEqual("reshape", bijector.name) - self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) - self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) - self.assertAllClose(0., fldj_, rtol=1e-6, atol=0) - self.assertAllClose(0., ildj_, rtol=1e-6, atol=0) - - def testEventShapeTensor(self): - """Test event_shape_tensor methods when even ndims may be dynamic.""" - - shape_in_static = [2, 3] - shape_out_static = [6,] - shape_in, shape_out, feed_dict = self.build_shapes(shape_in_static, - shape_out_static) - bijector = Reshape( - event_shape_out=shape_out, - event_shape_in=shape_in, validate_args=True) - - # using the _tensor methods, we should always get a fully-specified - # result since these are evaluated at graph runtime. - with self.cached_session() as sess: - (shape_out_, - shape_in_) = sess.run(( - bijector.forward_event_shape_tensor(shape_in), - bijector.inverse_event_shape_tensor(shape_out), - ), feed_dict=feed_dict) - self.assertAllEqual(shape_out_static, shape_out_) - self.assertAllEqual(shape_in_static, shape_in_) - - def testScalarReshape(self): - """Test reshaping to and from a scalar shape ().""" - - expected_x = np.random.randn(4, 3, 1) - expected_y = np.reshape(expected_x, [4, 3]) - - expected_x_scalar = np.random.randn(1,) - expected_y_scalar = expected_x_scalar[0] - - shape_in, shape_out, feed_dict = self.build_shapes([], [1,]) - with self.cached_session() as sess: - bijector = Reshape( - event_shape_out=shape_in, - event_shape_in=shape_out, validate_args=True) - (x_, - y_, - x_scalar_, - y_scalar_ - ) = sess.run(( - bijector.inverse(expected_y), - bijector.forward(expected_x), - bijector.inverse(expected_y_scalar), - bijector.forward(expected_x_scalar), - ), feed_dict=feed_dict) - self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) - self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) - self.assertAllClose(expected_y_scalar, y_scalar_, rtol=1e-6, atol=0) - self.assertAllClose(expected_x_scalar, x_scalar_, rtol=1e-6, atol=0) - - def testMultipleUnspecifiedDimensionsOpError(self): - - with self.cached_session() as sess: - shape_in, shape_out, feed_dict = self.build_shapes([2, 3], [4, -1, -1,]) - bijector = Reshape( - event_shape_out=shape_out, - event_shape_in=shape_in, - validate_args=True) - - with self.assertRaisesError( - "elements must have at most one `-1`."): - sess.run(bijector.forward_event_shape_tensor(shape_in), - feed_dict=feed_dict) - - # pylint: disable=invalid-name - def _testInvalidDimensionsOpError(self, expected_error_message): - - with self.cached_session() as sess: - - shape_in, shape_out, feed_dict = self.build_shapes([2, 3], [1, 2, -2,]) - bijector = Reshape( - event_shape_out=shape_out, - event_shape_in=shape_in, - validate_args=True) - - with self.assertRaisesError(expected_error_message): - sess.run(bijector.forward_event_shape_tensor(shape_in), - feed_dict=feed_dict) - - def _testInvalidDimensionsStatic(self, expected_error_message): - """Version of _testInvalidDimensionsOpError for errors detected statically. - - Statically means at graph construction time. - - Args: - expected_error_message: String that should be present in the error - message that `Reshape` raises for invalid shapes. - """ - shape_in, shape_out, _ = self.build_shapes([2, 3], [ - 1, - 2, - -2, - ]) - with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - expected_error_message): - _ = Reshape( - event_shape_out=shape_out, - event_shape_in=shape_in, - validate_args=True) - # pylint: enable=invalid-name - - def testValidButNonMatchingInputOpError(self): - x = np.random.randn(4, 3, 2) - - with self.cached_session() as sess: - shape_in, shape_out, feed_dict = self.build_shapes([2, 3], [1, 6, 1,]) - bijector = Reshape( - event_shape_out=shape_out, - event_shape_in=shape_in, - validate_args=True) - - # Here we pass in a tensor (x) whose shape is compatible with - # the output shape, so tf.reshape will throw no error, but - # doesn't match the expected input shape. - with self.assertRaisesError( - "Input `event_shape` does not match `event_shape_in`."): - sess.run(bijector.forward(x), - feed_dict=feed_dict) - - def testValidButNonMatchingInputPartiallySpecifiedOpError(self): - x = np.random.randn(4, 3, 2) - - with self.cached_session() as sess: - shape_in, shape_out, feed_dict = self.build_shapes([2, -1], [1, 6, 1,]) - bijector = Reshape( - event_shape_out=shape_out, - event_shape_in=shape_in, - validate_args=True) - - with self.assertRaisesError( - "Input `event_shape` does not match `event_shape_in`."): - sess.run(bijector.forward(x), - feed_dict=feed_dict) - - # pylint: disable=invalid-name - def _testInputOutputMismatchOpError(self, expected_error_message): - x1 = np.random.randn(4, 2, 3) - x2 = np.random.randn(4, 1, 1, 5) - - with self.cached_session() as sess: - shape_in, shape_out, fd_mismatched = self.build_shapes([2, 3], - [1, 1, 5]) - bijector = Reshape( - event_shape_out=shape_out, - event_shape_in=shape_in, - validate_args=True) - - with self.assertRaisesError(expected_error_message): - sess.run(bijector.forward(x1), feed_dict=fd_mismatched) - with self.assertRaisesError(expected_error_message): - sess.run(bijector.inverse(x2), feed_dict=fd_mismatched) - # pylint: enable=invalid-name - - def testOneShapePartiallySpecified(self): - expected_x = np.random.randn(4, 6) - expected_y = np.reshape(expected_x, [4, 2, 3]) - - with self.cached_session() as sess: - # one of input/output shapes is partially specified - shape_in, shape_out, feed_dict = self.build_shapes([-1,], [2, 3]) - bijector = Reshape( - event_shape_out=shape_out, - event_shape_in=shape_in, - validate_args=True) - (x_, - y_, - ) = sess.run(( - bijector.inverse(expected_y), - bijector.forward(expected_x), - ), feed_dict=feed_dict) - self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) - self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) - - def testBothShapesPartiallySpecified(self): - expected_x = np.random.randn(4, 2, 3) - expected_y = np.reshape(expected_x, [4, 3, 2]) - with self.cached_session() as sess: - shape_in, shape_out, feed_dict = self.build_shapes([-1, 3], [-1, 2]) - bijector = Reshape( - event_shape_out=shape_out, - event_shape_in=shape_in, - validate_args=True) - (x_, - y_, - ) = sess.run(( - bijector.inverse(expected_y), - bijector.forward(expected_x), - ), feed_dict=feed_dict) - self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) - self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) - - def testDefaultVectorShape(self): - expected_x = np.random.randn(4, 4) - expected_y = np.reshape(expected_x, [4, 2, 2]) - with self.cached_session() as sess: - _, shape_out, feed_dict = self.build_shapes([-1,], [-1, 2]) - bijector = Reshape(shape_out, - validate_args=True) - (x_, - y_, - ) = sess.run(( - bijector.inverse(expected_y), - bijector.forward(expected_x), - ), feed_dict=feed_dict) - self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) - self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) - - def build_shapes(self, *args, **kwargs): - raise NotImplementedError("Subclass failed to implement `build_shapes`.") - - -class ReshapeBijectorTestStatic(test.TestCase, _ReshapeBijectorTest): - - def build_shapes(self, shape_in, shape_out): - shape_in_static = shape_in - shape_out_static = shape_out - feed_dict = {} - return shape_in_static, shape_out_static, feed_dict - - def assertRaisesError(self, msg): - return self.assertRaisesRegexp(Exception, msg) - - def testEventShape(self): - shape_in_static = tensor_shape.TensorShape([2, 3]) - shape_out_static = tensor_shape.TensorShape([6,]) - bijector = Reshape( - event_shape_out=shape_out_static, - event_shape_in=shape_in_static, validate_args=True) - - # test that forward_ and inverse_event_shape do sensible things - # when shapes are statically known. - self.assertEqual( - bijector.forward_event_shape(shape_in_static), - shape_out_static) - self.assertEqual( - bijector.inverse_event_shape(shape_out_static), - shape_in_static) - - def testBijectiveAndFinite(self): - x = np.random.randn(4, 2, 3) - y = np.reshape(x, [4, 1, 2, 3]) - with self.cached_session(): - bijector = Reshape( - event_shape_in=[2, 3], - event_shape_out=[1, 2, 3], - validate_args=True) - assert_bijective_and_finite( - bijector, x, y, event_ndims=2, rtol=1e-6, atol=0) - - def testInvalidDimensionsStatic(self): - self._testInvalidDimensionsStatic( - "elements must be either positive integers or `-1`") - - def testInputOutputMismatchOpError(self): - self._testInputOutputMismatchOpError("Cannot reshape a tensor with") - - -class ReshapeBijectorTestDynamic(test.TestCase, _ReshapeBijectorTest): - - def build_shapes(self, shape_in, shape_out): - shape_in_ph = array_ops.placeholder(shape=(len(shape_in),), - dtype=dtypes.int32) - shape_out_ph = array_ops.placeholder(shape=(len(shape_out),), - dtype=dtypes.int32) - feed_dict = {shape_in_ph: shape_in, shape_out_ph: shape_out} - return shape_in_ph, shape_out_ph, feed_dict - - def assertRaisesError(self, msg): - return self.assertRaisesOpError(msg) - - def testInvalidDimensionsOpError(self): - self._testInvalidDimensionsOpError( - "elements must be either positive integers or `-1`.") - - def testInputOutputMismatchOpError(self): - self._testInputOutputMismatchOpError("Input to reshape is a tensor with") - - -class ReshapeBijectorTestDynamicNdims(test.TestCase, _ReshapeBijectorTest): - - def build_shapes(self, shape_in, shape_out): - shape_in_ph = array_ops.placeholder(shape=None, dtype=dtypes.int32) - shape_out_ph = array_ops.placeholder(shape=None, dtype=dtypes.int32) - feed_dict = {shape_in_ph: shape_in, shape_out_ph: shape_out} - return shape_in_ph, shape_out_ph, feed_dict - - def assertRaisesError(self, msg): - return self.assertRaisesOpError(msg) - - def testInvalidDimensionsOpError(self): - self._testInvalidDimensionsOpError( - "elements must be either positive integers or `-1`.") - - def testInputOutputMismatchOpError(self): - self._testInputOutputMismatchOpError("Input to reshape is a tensor with") - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/scale_tril_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/scale_tril_test.py deleted file mode 100644 index 1b88c1d130a..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/scale_tril_test.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for ScaleTriL bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.python.framework import test_util -from tensorflow.python.platform import test - - -class ScaleTriLBijectorTest(test.TestCase): - """Tests the correctness of the ScaleTriL bijector.""" - - def setUp(self): - self._rng = np.random.RandomState(42) - - def testComputesCorrectValues(self): - shift = 1.61803398875 - x = np.float32(np.array([-1, .5, 2])) - y = np.float32(np.array([[np.exp(2) + shift, 0.], - [.5, np.exp(-1) + shift]])) - - b = bijectors.ScaleTriL(diag_bijector=bijectors.Exp(), - diag_shift=shift) - - y_ = self.evaluate(b.forward(x)) - self.assertAllClose(y, y_, rtol=1e-4) - - x_ = self.evaluate(b.inverse(y)) - self.assertAllClose(x, x_, rtol=1e-4) - - @test_util.run_in_graph_and_eager_modes - def testInvertible(self): - - # Generate random inputs from an unconstrained space, with - # event size 6 to specify 3x3 triangular matrices. - batch_shape = [2, 1] - x = np.float32(self._rng.randn(*(batch_shape + [6]))) - b = bijectors.ScaleTriL(diag_bijector=bijectors.Softplus(), - diag_shift=3.14159) - y = self.evaluate(b.forward(x)) - self.assertAllEqual(y.shape, batch_shape + [3, 3]) - - x_ = self.evaluate(b.inverse(y)) - self.assertAllClose(x, x_, rtol=1e-4) - - fldj = self.evaluate(b.forward_log_det_jacobian(x, event_ndims=1)) - ildj = self.evaluate(b.inverse_log_det_jacobian(y, event_ndims=2)) - self.assertAllClose(fldj, -ildj, rtol=1e-4) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_test.py deleted file mode 100644 index a6d432753db..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_test.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Sigmoid Tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import special - -from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid import Sigmoid -from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite -from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency -from tensorflow.python.platform import test - - -class SigmoidBijectorTest(test.TestCase): - """Tests correctness of the Y = g(X) = (1 + exp(-X))^-1 transformation.""" - - def testBijector(self): - with self.cached_session(): - self.assertEqual("sigmoid", Sigmoid().name) - x = np.linspace(-10., 10., 100).reshape([2, 5, 10]).astype(np.float32) - y = special.expit(x) - ildj = -np.log(y) - np.log1p(-y) - bijector = Sigmoid() - self.assertAllClose(y, bijector.forward(x).eval(), atol=0., rtol=1e-2) - self.assertAllClose(x, bijector.inverse(y).eval(), atol=0., rtol=1e-4) - self.assertAllClose(ildj, bijector.inverse_log_det_jacobian( - y, event_ndims=0).eval(), atol=0., rtol=1e-6) - self.assertAllClose(-ildj, bijector.forward_log_det_jacobian( - x, event_ndims=0).eval(), atol=0., rtol=1e-4) - - def testScalarCongruency(self): - with self.cached_session(): - assert_scalar_congruency(Sigmoid(), lower_x=-7., upper_x=7.) - - def testBijectiveAndFinite(self): - with self.cached_session(): - x = np.linspace(-7., 7., 100).astype(np.float32) - eps = 1e-3 - y = np.linspace(eps, 1. - eps, 100).astype(np.float32) - assert_bijective_and_finite( - Sigmoid(), x, y, event_ndims=0, atol=0., rtol=1e-4) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_bijector_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_bijector_test.py deleted file mode 100644 index 282619a73b2..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_bijector_test.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for SinhArcsinh Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -# pylint: disable=g-importing-member -from tensorflow.contrib.distributions.python.ops.bijectors.sinh_arcsinh import SinhArcsinh -from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite -from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency -from tensorflow.python.platform import test - -# pylint: enable=g-importing-member - - -class SinhArcsinhBijectorTest(test.TestCase): - """Tests correctness of the power transformation.""" - - def testBijectorVersusNumpyRewriteOfBasicFunctions(self): - with self.cached_session(): - skewness = 0.2 - tailweight = 2.0 - bijector = SinhArcsinh( - skewness=skewness, - tailweight=tailweight, - validate_args=True) - self.assertEqual("SinhArcsinh", bijector.name) - x = np.array([[[-2.01], [2.], [1e-4]]]).astype(np.float32) - y = np.sinh((np.arcsinh(x) + skewness) * tailweight) - self.assertAllClose(y, bijector.forward(x).eval()) - self.assertAllClose(x, bijector.inverse(y).eval()) - self.assertAllClose( - np.sum( - np.log(np.cosh(np.arcsinh(y) / tailweight - skewness)) - - np.log(tailweight) - np.log(np.sqrt(y**2 + 1)), - axis=-1), - bijector.inverse_log_det_jacobian(y, event_ndims=1).eval()) - self.assertAllClose( - -bijector.inverse_log_det_jacobian(y, event_ndims=1).eval(), - bijector.forward_log_det_jacobian(x, event_ndims=1).eval(), - rtol=1e-4, - atol=0.) - - def testLargerTailWeightPutsMoreWeightInTails(self): - with self.cached_session(): - # Will broadcast together to shape [3, 2]. - x = [-1., 1.] - tailweight = [[0.5], [1.0], [2.0]] - bijector = SinhArcsinh(tailweight=tailweight, validate_args=True) - y = bijector.forward(x).eval() - - # x = -1, 1 should be mapped to points symmetric about 0 - self.assertAllClose(y[:, 0], -1. * y[:, 1]) - - # forward(1) should increase as tailweight increases, since higher - # tailweight should map 1 to a larger number. - forward_1 = y[:, 1] # The positive values of y. - self.assertLess(forward_1[0], forward_1[1]) - self.assertLess(forward_1[1], forward_1[2]) - - def testSkew(self): - with self.cached_session(): - # Will broadcast together to shape [3, 2]. - x = [-1., 1.] - skewness = [[-1.], [0.], [1.]] - bijector = SinhArcsinh(skewness=skewness, validate_args=True) - y = bijector.forward(x).eval() - - # For skew < 0, |forward(-1)| > |forward(1)| - self.assertGreater(np.abs(y[0, 0]), np.abs(y[0, 1])) - - # For skew = 0, |forward(-1)| = |forward(1)| - self.assertAllClose(np.abs(y[1, 0]), np.abs(y[1, 1])) - - # For skew > 0, |forward(-1)| < |forward(1)| - self.assertLess(np.abs(y[2, 0]), np.abs(y[2, 1])) - - def testScalarCongruencySkewness1Tailweight0p5(self): - with self.cached_session(): - bijector = SinhArcsinh(skewness=1.0, tailweight=0.5, validate_args=True) - assert_scalar_congruency(bijector, lower_x=-2., upper_x=2.0, rtol=0.05) - - def testScalarCongruencySkewnessNeg1Tailweight1p5(self): - with self.cached_session(): - bijector = SinhArcsinh(skewness=-1.0, tailweight=1.5, validate_args=True) - assert_scalar_congruency(bijector, lower_x=-2., upper_x=2.0, rtol=0.05) - - def testBijectiveAndFiniteSkewnessNeg1Tailweight0p5(self): - with self.cached_session(): - bijector = SinhArcsinh(skewness=-1., tailweight=0.5, validate_args=True) - x = np.concatenate((-np.logspace(-2, 10, 1000), [0], np.logspace( - -2, 10, 1000))).astype(np.float32) - assert_bijective_and_finite(bijector, x, x, event_ndims=0, rtol=1e-3) - - def testBijectiveAndFiniteSkewness1Tailweight3(self): - with self.cached_session(): - bijector = SinhArcsinh(skewness=1., tailweight=3., validate_args=True) - x = np.concatenate((-np.logspace(-2, 5, 1000), [0], np.logspace( - -2, 5, 1000))).astype(np.float32) - assert_bijective_and_finite( - bijector, x, x, event_ndims=0, rtol=1e-3) - - def testBijectorEndpoints(self): - with self.cached_session(): - for dtype in (np.float32, np.float64): - bijector = SinhArcsinh( - skewness=dtype(0.), tailweight=dtype(1.), validate_args=True) - bounds = np.array( - [np.finfo(dtype).min, np.finfo(dtype).max], dtype=dtype) - # Note that the above bijector is the identity bijector. Hence, the - # log_det_jacobian will be 0. Because of this we use atol. - assert_bijective_and_finite( - bijector, bounds, bounds, event_ndims=0, atol=2e-6) - - def testBijectorOverRange(self): - with self.cached_session(): - for dtype in (np.float32, np.float64): - skewness = np.array([1.2, 5.], dtype=dtype) - tailweight = np.array([2., 10.], dtype=dtype) - # The inverse will be defined up to where sinh is valid, which is - # arcsinh(np.finfo(dtype).max). - log_boundary = np.log( - np.sinh(np.arcsinh(np.finfo(dtype).max) / tailweight - skewness)) - x = np.array([ - np.logspace(-2, log_boundary[0], base=np.e, num=1000), - np.logspace(-2, log_boundary[1], base=np.e, num=1000) - ], dtype=dtype) - # Ensure broadcasting works. - x = np.swapaxes(x, 0, 1) - - y = np.sinh((np.arcsinh(x) + skewness) * tailweight) - bijector = SinhArcsinh( - skewness=skewness, tailweight=tailweight, validate_args=True) - - self.assertAllClose(y, bijector.forward(x).eval(), rtol=1e-4, atol=0.) - self.assertAllClose(x, bijector.inverse(y).eval(), rtol=1e-4, atol=0.) - - # On IBM PPC systems, longdouble (np.float128) is same as double except that it can have more precision. - # Type double being of 8 bytes, can't hold square of max of float64 (which is also 8 bytes) and - # below test fails due to overflow error giving inf. So this check avoids that error by skipping square - # calculation and corresponding assert. - - if np.amax(y) <= np.sqrt(np.finfo(np.float128).max) and \ - np.fabs(np.amin(y)) <= np.sqrt(np.fabs(np.finfo(np.float128).min)): - - # Do the numpy calculation in float128 to avoid inf/nan. - y_float128 = np.float128(y) - self.assertAllClose( - np.log(np.cosh( - np.arcsinh(y_float128) / tailweight - skewness) / np.sqrt( - y_float128**2 + 1)) - - np.log(tailweight), - bijector.inverse_log_det_jacobian(y, event_ndims=0).eval(), - rtol=1e-4, - atol=0.) - self.assertAllClose( - -bijector.inverse_log_det_jacobian(y, event_ndims=0).eval(), - bijector.forward_log_det_jacobian(x, event_ndims=0).eval(), - rtol=1e-4, - atol=0.) - - def testZeroTailweightRaises(self): - with self.cached_session(): - with self.assertRaisesOpError("not positive"): - SinhArcsinh(tailweight=0., validate_args=True).forward(1.0).eval() - - def testDefaultDtypeIsFloat32(self): - with self.cached_session(): - bijector = SinhArcsinh() - self.assertEqual(bijector.tailweight.dtype, np.float32) - self.assertEqual(bijector.skewness.dtype, np.float32) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softmax_centered_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softmax_centered_test.py deleted file mode 100644 index 8d18400487d..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softmax_centered_test.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.softmax_centered import SoftmaxCentered -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite -from tensorflow.python.platform import test - - -rng = np.random.RandomState(42) - - -class SoftmaxCenteredBijectorTest(test.TestCase): - """Tests correctness of the Y = g(X) = exp(X) / sum(exp(X)) transformation.""" - - def testBijectorVector(self): - with self.cached_session(): - softmax = SoftmaxCentered() - self.assertEqual("softmax_centered", softmax.name) - x = np.log([[2., 3, 4], [4., 8, 12]]) - y = [[0.2, 0.3, 0.4, 0.1], [0.16, 0.32, 0.48, 0.04]] - self.assertAllClose(y, softmax.forward(x).eval()) - self.assertAllClose(x, softmax.inverse(y).eval()) - self.assertAllClose( - -np.sum(np.log(y), axis=1), - softmax.inverse_log_det_jacobian(y, event_ndims=1).eval(), - atol=0., - rtol=1e-7) - self.assertAllClose( - -softmax.inverse_log_det_jacobian(y, event_ndims=1).eval(), - softmax.forward_log_det_jacobian(x, event_ndims=1).eval(), - atol=0., - rtol=1e-7) - - def testBijectorUnknownShape(self): - with self.cached_session(): - softmax = SoftmaxCentered() - self.assertEqual("softmax_centered", softmax.name) - x = array_ops.placeholder(shape=[2, None], dtype=dtypes.float32) - real_x = np.log([[2., 3, 4], [4., 8, 12]]) - y = array_ops.placeholder(shape=[2, None], dtype=dtypes.float32) - real_y = [[0.2, 0.3, 0.4, 0.1], [0.16, 0.32, 0.48, 0.04]] - self.assertAllClose(real_y, softmax.forward(x).eval( - feed_dict={x: real_x})) - self.assertAllClose(real_x, softmax.inverse(y).eval( - feed_dict={y: real_y})) - self.assertAllClose( - -np.sum(np.log(real_y), axis=1), - softmax.inverse_log_det_jacobian(y, event_ndims=1).eval( - feed_dict={y: real_y}), - atol=0., - rtol=1e-7) - self.assertAllClose( - -softmax.inverse_log_det_jacobian(y, event_ndims=1).eval( - feed_dict={y: real_y}), - softmax.forward_log_det_jacobian(x, event_ndims=1).eval( - feed_dict={x: real_x}), - atol=0., - rtol=1e-7) - - def testShapeGetters(self): - with self.cached_session(): - x = tensor_shape.TensorShape([4]) - y = tensor_shape.TensorShape([5]) - bijector = SoftmaxCentered(validate_args=True) - self.assertAllEqual(y, bijector.forward_event_shape(x)) - self.assertAllEqual(y.as_list(), - bijector.forward_event_shape_tensor( - x.as_list()).eval()) - self.assertAllEqual(x, bijector.inverse_event_shape(y)) - self.assertAllEqual(x.as_list(), - bijector.inverse_event_shape_tensor( - y.as_list()).eval()) - - def testBijectiveAndFinite(self): - with self.cached_session(): - softmax = SoftmaxCentered() - x = np.linspace(-50, 50, num=10).reshape(5, 2).astype(np.float32) - # Make y values on the simplex with a wide range. - y_0 = np.ones(5).astype(np.float32) - y_1 = (1e-5 * rng.rand(5)).astype(np.float32) - y_2 = (1e1 * rng.rand(5)).astype(np.float32) - y = np.array([y_0, y_1, y_2]) - y /= y.sum(axis=0) - y = y.T # y.shape = [5, 3] - assert_bijective_and_finite(softmax, x, y, event_ndims=1) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softplus_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softplus_test.py deleted file mode 100644 index 2e7ab3ecfd2..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softplus_test.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.softplus import Softplus -from tensorflow.python.framework import errors -from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite -from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency -from tensorflow.python.platform import test - -rng = np.random.RandomState(42) - - -class SoftplusBijectorTest(test.TestCase): - """Tests the correctness of the Y = g(X) = Log[1 + exp(X)] transformation.""" - - def _softplus(self, x): - return np.log(1 + np.exp(x)) - - def _softplus_inverse(self, y): - return np.log(np.exp(y) - 1) - - def _softplus_ildj_before_reduction(self, y): - """Inverse log det jacobian, before being reduced.""" - return -np.log(1 - np.exp(-y)) - - def testHingeSoftnessZeroRaises(self): - with self.cached_session(): - with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - "must be non-zero"): - _ = Softplus(hinge_softness=0., validate_args=True) - # Error detected statically; don't need to run op. - - def testBijectorForwardInverseEventDimsZero(self): - with self.cached_session(): - bijector = Softplus() - self.assertEqual("softplus", bijector.name) - x = 2 * rng.randn(2, 10) - y = self._softplus(x) - - self.assertAllClose(y, bijector.forward(x).eval()) - self.assertAllClose(x, bijector.inverse(y).eval()) - - def testBijectorForwardInverseWithHingeSoftnessEventDimsZero(self): - with self.cached_session(): - bijector = Softplus(hinge_softness=1.5) - x = 2 * rng.randn(2, 10) - y = 1.5 * self._softplus(x / 1.5) - - self.assertAllClose(y, bijector.forward(x).eval()) - self.assertAllClose(x, bijector.inverse(y).eval()) - - def testBijectorLogDetJacobianEventDimsZero(self): - with self.cached_session(): - bijector = Softplus() - y = 2 * rng.rand(2, 10) - # No reduction needed if event_dims = 0. - ildj = self._softplus_ildj_before_reduction(y) - - self.assertAllClose(ildj, bijector.inverse_log_det_jacobian( - y, event_ndims=0).eval()) - - def testBijectorForwardInverseEventDimsOne(self): - with self.cached_session(): - bijector = Softplus() - self.assertEqual("softplus", bijector.name) - x = 2 * rng.randn(2, 10) - y = self._softplus(x) - - self.assertAllClose(y, bijector.forward(x).eval()) - self.assertAllClose(x, bijector.inverse(y).eval()) - - def testBijectorLogDetJacobianEventDimsOne(self): - with self.cached_session(): - bijector = Softplus() - y = 2 * rng.rand(2, 10) - ildj_before = self._softplus_ildj_before_reduction(y) - ildj = np.sum(ildj_before, axis=1) - - self.assertAllClose(ildj, bijector.inverse_log_det_jacobian( - y, event_ndims=1).eval()) - - def testScalarCongruency(self): - with self.cached_session(): - bijector = Softplus() - assert_scalar_congruency( - bijector, lower_x=-2., upper_x=2.) - - def testScalarCongruencyWithPositiveHingeSoftness(self): - with self.cached_session(): - bijector = Softplus(hinge_softness=1.3) - assert_scalar_congruency( - bijector, lower_x=-2., upper_x=2.) - - def testScalarCongruencyWithNegativeHingeSoftness(self): - with self.cached_session(): - bijector = Softplus(hinge_softness=-1.3) - assert_scalar_congruency( - bijector, lower_x=-2., upper_x=2.) - - def testBijectiveAndFinite32bit(self): - with self.cached_session(): - bijector = Softplus() - x = np.linspace(-20., 20., 100).astype(np.float32) - y = np.logspace(-10, 10, 100).astype(np.float32) - assert_bijective_and_finite( - bijector, x, y, event_ndims=0, rtol=1e-2, atol=1e-2) - - def testBijectiveAndFiniteWithPositiveHingeSoftness32Bit(self): - with self.cached_session(): - bijector = Softplus(hinge_softness=1.23) - x = np.linspace(-20., 20., 100).astype(np.float32) - y = np.logspace(-10, 10, 100).astype(np.float32) - assert_bijective_and_finite( - bijector, x, y, event_ndims=0, rtol=1e-2, atol=1e-2) - - def testBijectiveAndFiniteWithNegativeHingeSoftness32Bit(self): - with self.cached_session(): - bijector = Softplus(hinge_softness=-0.7) - x = np.linspace(-20., 20., 100).astype(np.float32) - y = -np.logspace(-10, 10, 100).astype(np.float32) - assert_bijective_and_finite( - bijector, x, y, event_ndims=0, rtol=1e-2, atol=1e-2) - - def testBijectiveAndFinite16bit(self): - with self.cached_session(): - bijector = Softplus() - # softplus(-20) is zero, so we can't use such a large range as in 32bit. - x = np.linspace(-10., 20., 100).astype(np.float16) - # Note that float16 is only in the open set (0, inf) for a smaller - # logspace range. The actual range was (-7, 4), so use something smaller - # for the test. - y = np.logspace(-6, 3, 100).astype(np.float16) - assert_bijective_and_finite( - bijector, x, y, event_ndims=0, rtol=1e-1, atol=1e-3) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softsign_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softsign_test.py deleted file mode 100644 index c32ea9ade73..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softsign_test.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.bijectors.softsign import Softsign -from tensorflow.python.framework import test_util -from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite -from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency -from tensorflow.python.platform import test - - -class SoftsignBijectorTest(test.TestCase): - """Tests the correctness of the Y = g(X) = X / (1 + |X|) transformation.""" - - def _softsign(self, x): - return x / (1. + np.abs(x)) - - def _softsign_ildj_before_reduction(self, y): - """Inverse log det jacobian, before being reduced.""" - return -2. * np.log1p(-np.abs(y)) - - def setUp(self): - self._rng = np.random.RandomState(42) - - @test_util.run_in_graph_and_eager_modes - def testBijectorBounds(self): - bijector = Softsign(validate_args=True) - with self.assertRaisesOpError("greater than -1"): - self.evaluate(bijector.inverse(-3.)) - with self.assertRaisesOpError("greater than -1"): - self.evaluate(bijector.inverse_log_det_jacobian(-3., event_ndims=0)) - - with self.assertRaisesOpError("less than 1"): - self.evaluate(bijector.inverse(3.)) - with self.assertRaisesOpError("less than 1"): - self.evaluate(bijector.inverse_log_det_jacobian(3., event_ndims=0)) - - @test_util.run_in_graph_and_eager_modes - def testBijectorForwardInverse(self): - bijector = Softsign(validate_args=True) - self.assertEqual("softsign", bijector.name) - x = 2. * self._rng.randn(2, 10) - y = self._softsign(x) - - self.assertAllClose(y, self.evaluate(bijector.forward(x))) - self.assertAllClose(x, self.evaluate(bijector.inverse(y))) - - @test_util.run_in_graph_and_eager_modes - def testBijectorLogDetJacobianEventDimsZero(self): - bijector = Softsign(validate_args=True) - y = self._rng.rand(2, 10) - # No reduction needed if event_dims = 0. - ildj = self._softsign_ildj_before_reduction(y) - - self.assertAllClose(ildj, self.evaluate( - bijector.inverse_log_det_jacobian(y, event_ndims=0))) - - @test_util.run_in_graph_and_eager_modes - def testBijectorForwardInverseEventDimsOne(self): - bijector = Softsign(validate_args=True) - self.assertEqual("softsign", bijector.name) - x = 2. * self._rng.randn(2, 10) - y = self._softsign(x) - self.assertAllClose(y, self.evaluate(bijector.forward(x))) - self.assertAllClose(x, self.evaluate(bijector.inverse(y))) - - @test_util.run_in_graph_and_eager_modes - def testBijectorLogDetJacobianEventDimsOne(self): - bijector = Softsign(validate_args=True) - y = self._rng.rand(2, 10) - ildj_before = self._softsign_ildj_before_reduction(y) - ildj = np.sum(ildj_before, axis=1) - self.assertAllClose( - ildj, self.evaluate( - bijector.inverse_log_det_jacobian(y, event_ndims=1))) - - def testScalarCongruency(self): - with self.cached_session(): - bijector = Softsign(validate_args=True) - assert_scalar_congruency(bijector, lower_x=-20., upper_x=20.) - - def testBijectiveAndFinite(self): - with self.cached_session(): - bijector = Softsign(validate_args=True) - x = np.linspace(-20., 20., 100).astype(np.float32) - y = np.linspace(-0.99, 0.99, 100).astype(np.float32) - assert_bijective_and_finite( - bijector, x, y, event_ndims=0, rtol=1e-3, atol=1e-3) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/square_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/square_test.py deleted file mode 100644 index e5550cc8303..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/square_test.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency -from tensorflow.python.platform import test - - -class SquareBijectorTest(test.TestCase): - """Tests the correctness of the Y = X ** 2 transformation.""" - - def testBijectorScalar(self): - with self.cached_session(): - bijector = bijectors.Square(validate_args=True) - self.assertEqual("square", bijector.name) - x = [[[1., 5], - [2, 1]], - [[np.sqrt(2.), 3], - [np.sqrt(8.), 1]]] - y = np.square(x) - ildj = -np.log(2.) - np.log(x) - self.assertAllClose(y, bijector.forward(x).eval()) - self.assertAllClose(x, bijector.inverse(y).eval()) - self.assertAllClose( - ildj, bijector.inverse_log_det_jacobian( - y, event_ndims=0).eval(), atol=0., rtol=1e-7) - self.assertAllClose( - -bijector.inverse_log_det_jacobian(y, event_ndims=0).eval(), - bijector.forward_log_det_jacobian(x, event_ndims=0).eval(), - atol=0., - rtol=1e-7) - - def testScalarCongruency(self): - with self.cached_session(): - bijector = bijectors.Square(validate_args=True) - assert_scalar_congruency(bijector, lower_x=1e-3, upper_x=1.5, rtol=0.05) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/transform_diagonal_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/transform_diagonal_test.py deleted file mode 100644 index efc9f266d1f..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/transform_diagonal_test.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for TransformDiagonal bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.python.framework import test_util -from tensorflow.python.platform import test - - -class TransformDiagonalBijectorTest(test.TestCase): - """Tests correctness of the TransformDiagonal bijector.""" - - def setUp(self): - self._rng = np.random.RandomState(42) - - @test_util.run_in_graph_and_eager_modes - def testBijector(self): - x = np.float32(np.random.randn(3, 4, 4)) - - y = x.copy() - for i in range(x.shape[0]): - np.fill_diagonal(y[i, :, :], np.exp(np.diag(x[i, :, :]))) - - exp = bijectors.Exp() - b = bijectors.TransformDiagonal(diag_bijector=exp) - - y_ = self.evaluate(b.forward(x)) - self.assertAllClose(y, y_) - - x_ = self.evaluate(b.inverse(y)) - self.assertAllClose(x, x_) - - fldj = self.evaluate(b.forward_log_det_jacobian(x, event_ndims=2)) - ildj = self.evaluate(b.inverse_log_det_jacobian(y, event_ndims=2)) - self.assertAllEqual( - fldj, - self.evaluate(exp.forward_log_det_jacobian( - np.array([np.diag(x_mat) for x_mat in x]), - event_ndims=1))) - self.assertAllEqual( - ildj, - self.evaluate(exp.inverse_log_det_jacobian( - np.array([np.diag(y_mat) for y_mat in y]), - event_ndims=1))) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/weibull_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/weibull_test.py deleted file mode 100644 index 424eb58fa06..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/weibull_test.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import stats - -from tensorflow.contrib.distributions.python.ops.bijectors.weibull import Weibull -from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite -from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency -from tensorflow.python.platform import test - - -class WeibullBijectorTest(test.TestCase): - """Tests correctness of the weibull bijector.""" - - def testBijector(self): - with self.cached_session(): - scale = 5. - concentration = 0.3 - bijector = Weibull( - scale=scale, concentration=concentration, - validate_args=True) - self.assertEqual("weibull", bijector.name) - x = np.array([[[0.], [1.], [14.], [20.], [100.]]], dtype=np.float32) - # Weibull distribution - weibull_dist = stats.frechet_r(c=concentration, scale=scale) - y = weibull_dist.cdf(x).astype(np.float32) - self.assertAllClose(y, bijector.forward(x).eval()) - self.assertAllClose(x, bijector.inverse(y).eval()) - self.assertAllClose( - weibull_dist.logpdf(x), - bijector.forward_log_det_jacobian(x, event_ndims=0).eval()) - self.assertAllClose( - -bijector.inverse_log_det_jacobian(y, event_ndims=0).eval(), - bijector.forward_log_det_jacobian(x, event_ndims=0).eval(), - rtol=1e-4, - atol=0.) - - def testScalarCongruency(self): - with self.cached_session(): - assert_scalar_congruency( - Weibull(scale=20., concentration=0.3), - lower_x=1., upper_x=100., rtol=0.02) - - def testBijectiveAndFinite(self): - with self.cached_session(): - bijector = Weibull( - scale=20., concentration=2., validate_args=True) - x = np.linspace(1., 8., num=10).astype(np.float32) - y = np.linspace( - -np.expm1(-1 / 400.), - -np.expm1(-16), num=10).astype(np.float32) - assert_bijective_and_finite(bijector, x, y, event_ndims=0, rtol=1e-3) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/binomial_test.py b/tensorflow/contrib/distributions/python/kernel_tests/binomial_test.py deleted file mode 100644 index c317393fbcb..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/binomial_test.py +++ /dev/null @@ -1,209 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import stats -from tensorflow.contrib.distributions.python.ops import binomial -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class BinomialTest(test.TestCase): - - def testSimpleShapes(self): - with self.cached_session(): - p = np.float32(np.random.beta(1, 1)) - binom = binomial.Binomial(total_count=1., probs=p) - self.assertAllEqual([], binom.event_shape_tensor().eval()) - self.assertAllEqual([], binom.batch_shape_tensor().eval()) - self.assertEqual(tensor_shape.TensorShape([]), binom.event_shape) - self.assertEqual(tensor_shape.TensorShape([]), binom.batch_shape) - - def testComplexShapes(self): - with self.cached_session(): - p = np.random.beta(1, 1, size=(3, 2)).astype(np.float32) - n = [[3., 2], [4, 5], [6, 7]] - binom = binomial.Binomial(total_count=n, probs=p) - self.assertAllEqual([], binom.event_shape_tensor().eval()) - self.assertAllEqual([3, 2], binom.batch_shape_tensor().eval()) - self.assertEqual(tensor_shape.TensorShape([]), binom.event_shape) - self.assertEqual( - tensor_shape.TensorShape([3, 2]), binom.batch_shape) - - def testNProperty(self): - p = [[0.1, 0.2, 0.7], [0.2, 0.3, 0.5]] - n = [[3.], [4]] - with self.cached_session(): - binom = binomial.Binomial(total_count=n, probs=p) - self.assertEqual((2, 1), binom.total_count.get_shape()) - self.assertAllClose(n, binom.total_count.eval()) - - def testPProperty(self): - p = [[0.1, 0.2, 0.7]] - with self.cached_session(): - binom = binomial.Binomial(total_count=3., probs=p) - self.assertEqual((1, 3), binom.probs.get_shape()) - self.assertEqual((1, 3), binom.logits.get_shape()) - self.assertAllClose(p, binom.probs.eval()) - - def testLogitsProperty(self): - logits = [[0., 9., -0.5]] - with self.cached_session(): - binom = binomial.Binomial(total_count=3., logits=logits) - self.assertEqual((1, 3), binom.probs.get_shape()) - self.assertEqual((1, 3), binom.logits.get_shape()) - self.assertAllClose(logits, binom.logits.eval()) - - def testPmfAndCdfNandCountsAgree(self): - p = [[0.1, 0.2, 0.7]] - n = [[5.]] - with self.cached_session(): - binom = binomial.Binomial(total_count=n, probs=p, validate_args=True) - binom.prob([2., 3, 2]).eval() - binom.prob([3., 1, 2]).eval() - binom.cdf([2., 3, 2]).eval() - binom.cdf([3., 1, 2]).eval() - with self.assertRaisesOpError("Condition x >= 0.*"): - binom.prob([-1., 4, 2]).eval() - with self.assertRaisesOpError("Condition x <= y.*"): - binom.prob([7., 3, 0]).eval() - with self.assertRaisesOpError("Condition x >= 0.*"): - binom.cdf([-1., 4, 2]).eval() - with self.assertRaisesOpError("Condition x <= y.*"): - binom.cdf([7., 3, 0]).eval() - - def testPmfAndCdfNonIntegerCounts(self): - p = [[0.1, 0.2, 0.7]] - n = [[5.]] - with self.cached_session(): - # No errors with integer n. - binom = binomial.Binomial(total_count=n, probs=p, validate_args=True) - binom.prob([2., 3, 2]).eval() - binom.prob([3., 1, 2]).eval() - binom.cdf([2., 3, 2]).eval() - binom.cdf([3., 1, 2]).eval() - placeholder = array_ops.placeholder(dtypes.float32) - # Both equality and integer checking fail. - with self.assertRaisesOpError( - "cannot contain fractional components."): - binom.prob(placeholder).eval(feed_dict={placeholder: [1.0, 2.5, 1.5]}) - with self.assertRaisesOpError( - "cannot contain fractional components."): - binom.cdf(placeholder).eval(feed_dict={placeholder: [1.0, 2.5, 1.5]}) - - binom = binomial.Binomial(total_count=n, probs=p, validate_args=False) - binom.prob([1., 2., 3.]).eval() - binom.cdf([1., 2., 3.]).eval() - # Non-integer arguments work. - binom.prob([1.0, 2.5, 1.5]).eval() - binom.cdf([1.0, 2.5, 1.5]).eval() - - def testPmfAndCdfBothZeroBatches(self): - with self.cached_session(): - # Both zero-batches. No broadcast - p = 0.5 - counts = 1. - binom = binomial.Binomial(total_count=1., probs=p) - pmf = binom.prob(counts) - cdf = binom.cdf(counts) - self.assertAllClose(0.5, pmf.eval()) - self.assertAllClose(stats.binom.cdf(counts, n=1, p=p), cdf.eval()) - self.assertEqual((), pmf.get_shape()) - self.assertEqual((), cdf.get_shape()) - - def testPmfAndCdfBothZeroBatchesNontrivialN(self): - with self.cached_session(): - # Both zero-batches. No broadcast - p = 0.1 - counts = 3. - binom = binomial.Binomial(total_count=5., probs=p) - pmf = binom.prob(counts) - cdf = binom.cdf(counts) - self.assertAllClose(stats.binom.pmf(counts, n=5., p=p), pmf.eval()) - self.assertAllClose(stats.binom.cdf(counts, n=5., p=p), cdf.eval()) - self.assertEqual((), pmf.get_shape()) - self.assertEqual((), cdf.get_shape()) - - def testPmfAndCdfPStretchedInBroadcastWhenSameRank(self): - with self.cached_session(): - p = [[0.1, 0.9]] - counts = [[1., 2.]] - binom = binomial.Binomial(total_count=3., probs=p) - pmf = binom.prob(counts) - cdf = binom.cdf(counts) - self.assertAllClose(stats.binom.pmf(counts, n=3., p=p), pmf.eval()) - self.assertAllClose(stats.binom.cdf(counts, n=3., p=p), cdf.eval()) - self.assertEqual((1, 2), pmf.get_shape()) - self.assertEqual((1, 2), cdf.get_shape()) - - def testPmfAndCdfPStretchedInBroadcastWhenLowerRank(self): - with self.cached_session(): - p = [0.1, 0.4] - counts = [[1.], [0.]] - binom = binomial.Binomial(total_count=1., probs=p) - pmf = binom.prob(counts) - cdf = binom.cdf(counts) - self.assertAllClose([[0.1, 0.4], [0.9, 0.6]], pmf.eval()) - self.assertAllClose([[1.0, 1.0], [0.9, 0.6]], cdf.eval()) - self.assertEqual((2, 2), pmf.get_shape()) - self.assertEqual((2, 2), cdf.get_shape()) - - def testBinomialMean(self): - with self.cached_session(): - n = 5. - p = [0.1, 0.2, 0.7] - binom = binomial.Binomial(total_count=n, probs=p) - expected_means = stats.binom.mean(n, p) - self.assertEqual((3,), binom.mean().get_shape()) - self.assertAllClose(expected_means, binom.mean().eval()) - - def testBinomialVariance(self): - with self.cached_session(): - n = 5. - p = [0.1, 0.2, 0.7] - binom = binomial.Binomial(total_count=n, probs=p) - expected_variances = stats.binom.var(n, p) - self.assertEqual((3,), binom.variance().get_shape()) - self.assertAllClose(expected_variances, binom.variance().eval()) - - def testBinomialMode(self): - with self.cached_session(): - n = 5. - p = [0.1, 0.2, 0.7] - binom = binomial.Binomial(total_count=n, probs=p) - expected_modes = [0., 1, 4] - self.assertEqual((3,), binom.mode().get_shape()) - self.assertAllClose(expected_modes, binom.mode().eval()) - - def testBinomialMultipleMode(self): - with self.cached_session(): - n = 9. - p = [0.1, 0.2, 0.7] - binom = binomial.Binomial(total_count=n, probs=p) - # For the case where (n + 1) * p is an integer, the modes are: - # (n + 1) * p and (n + 1) * p - 1. In this case, we get back - # the larger of the two modes. - expected_modes = [1., 2, 7] - self.assertEqual((3,), binom.mode().get_shape()) - self.assertAllClose(expected_modes, binom.mode().eval()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/cauchy_test.py b/tensorflow/contrib/distributions/python/kernel_tests/cauchy_test.py deleted file mode 100644 index f5d6944d166..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/cauchy_test.py +++ /dev/null @@ -1,440 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Cauchy.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import importlib -import numpy as np - -from tensorflow.contrib.distributions.python.ops import cauchy as cauchy_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging - - -def try_import(name): # pylint: disable=invalid-name - module = None - try: - module = importlib.import_module(name) - except ImportError as e: - tf_logging.warning("Could not import %s: %s" % (name, str(e))) - return module - - -stats = try_import("scipy.stats") - - -class CauchyTest(test.TestCase): - - def setUp(self): - self._rng = np.random.RandomState(123) - - def assertAllFinite(self, tensor): - is_finite = np.isfinite(tensor.eval()) - all_true = np.ones_like(is_finite, dtype=np.bool) - self.assertAllEqual(all_true, is_finite) - - def _testParamShapes(self, sample_shape, expected): - with self.cached_session(): - param_shapes = cauchy_lib.Cauchy.param_shapes(sample_shape) - loc_shape, scale_shape = param_shapes["loc"], param_shapes["scale"] - self.assertAllEqual(expected, loc_shape.eval()) - self.assertAllEqual(expected, scale_shape.eval()) - loc = array_ops.zeros(loc_shape) - scale = array_ops.ones(scale_shape) - self.assertAllEqual(expected, - array_ops.shape( - cauchy_lib.Cauchy(loc, scale).sample()).eval()) - - def _testParamStaticShapes(self, sample_shape, expected): - param_shapes = cauchy_lib.Cauchy.param_static_shapes(sample_shape) - loc_shape, scale_shape = param_shapes["loc"], param_shapes["scale"] - self.assertEqual(expected, loc_shape) - self.assertEqual(expected, scale_shape) - - def testParamShapes(self): - sample_shape = [10, 3, 4] - self._testParamShapes(sample_shape, sample_shape) - self._testParamShapes(constant_op.constant(sample_shape), sample_shape) - - def testParamStaticShapes(self): - sample_shape = [10, 3, 4] - self._testParamStaticShapes(sample_shape, sample_shape) - self._testParamStaticShapes( - tensor_shape.TensorShape(sample_shape), sample_shape) - - def testCauchyLogPDF(self): - with self.cached_session(): - batch_size = 6 - loc = constant_op.constant([3.0] * batch_size) - scale = constant_op.constant([np.sqrt(10.0)] * batch_size) - x = np.array([-2.5, 2.5, 4.0, 0.0, -1.0, 2.0], dtype=np.float32) - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - - log_pdf = cauchy.log_prob(x) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), log_pdf.shape) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), - log_pdf.eval().shape) - self.assertAllEqual(cauchy.batch_shape, log_pdf.shape) - self.assertAllEqual(cauchy.batch_shape, log_pdf.eval().shape) - - pdf = cauchy.prob(x) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), pdf.shape) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), pdf.eval().shape) - self.assertAllEqual(cauchy.batch_shape, pdf.shape) - self.assertAllEqual(cauchy.batch_shape, pdf.eval().shape) - - if not stats: - return - expected_log_pdf = stats.cauchy(loc.eval(), scale.eval()).logpdf(x) - self.assertAllClose(expected_log_pdf, log_pdf.eval()) - self.assertAllClose(np.exp(expected_log_pdf), pdf.eval()) - - def testCauchyLogPDFMultidimensional(self): - with self.cached_session(): - batch_size = 6 - loc = constant_op.constant([[3.0, -3.0]] * batch_size) - scale = constant_op.constant( - [[np.sqrt(10.0), np.sqrt(15.0)]] * batch_size) - x = np.array([[-2.5, 2.5, 4.0, 0.0, -1.0, 2.0]], dtype=np.float32).T - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - - log_pdf = cauchy.log_prob(x) - log_pdf_values = log_pdf.eval() - self.assertEqual(log_pdf.shape, (6, 2)) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), log_pdf.shape) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), - log_pdf.eval().shape) - self.assertAllEqual(cauchy.batch_shape, log_pdf.shape) - self.assertAllEqual(cauchy.batch_shape, log_pdf.eval().shape) - - pdf = cauchy.prob(x) - pdf_values = pdf.eval() - self.assertEqual(pdf.shape, (6, 2)) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), pdf.shape) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), pdf_values.shape) - self.assertAllEqual(cauchy.batch_shape, pdf.shape) - self.assertAllEqual(cauchy.batch_shape, pdf_values.shape) - - if not stats: - return - expected_log_pdf = stats.cauchy(loc.eval(), scale.eval()).logpdf(x) - self.assertAllClose(expected_log_pdf, log_pdf_values) - self.assertAllClose(np.exp(expected_log_pdf), pdf_values) - - def testCauchyCDF(self): - with self.cached_session(): - batch_size = 50 - loc = self._rng.randn(batch_size) - scale = self._rng.rand(batch_size) + 1.0 - x = np.linspace(-8.0, 8.0, batch_size).astype(np.float64) - - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - cdf = cauchy.cdf(x) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), cdf.shape) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), cdf.eval().shape) - self.assertAllEqual(cauchy.batch_shape, cdf.shape) - self.assertAllEqual(cauchy.batch_shape, cdf.eval().shape) - if not stats: - return - expected_cdf = stats.cauchy(loc, scale).cdf(x) - self.assertAllClose(expected_cdf, cdf.eval(), atol=0) - - def testCauchySurvivalFunction(self): - with self.cached_session(): - batch_size = 50 - loc = self._rng.randn(batch_size) - scale = self._rng.rand(batch_size) + 1.0 - x = np.linspace(-8.0, 8.0, batch_size).astype(np.float64) - - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - - sf = cauchy.survival_function(x) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), sf.shape) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), sf.eval().shape) - self.assertAllEqual(cauchy.batch_shape, sf.shape) - self.assertAllEqual(cauchy.batch_shape, sf.eval().shape) - if not stats: - return - expected_sf = stats.cauchy(loc, scale).sf(x) - self.assertAllClose(expected_sf, sf.eval(), atol=0) - - def testCauchyLogCDF(self): - with self.cached_session(): - batch_size = 50 - loc = self._rng.randn(batch_size) - scale = self._rng.rand(batch_size) + 1.0 - x = np.linspace(-100.0, 10.0, batch_size).astype(np.float64) - - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - - cdf = cauchy.log_cdf(x) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), cdf.shape) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), cdf.eval().shape) - self.assertAllEqual(cauchy.batch_shape, cdf.shape) - self.assertAllEqual(cauchy.batch_shape, cdf.eval().shape) - - if not stats: - return - expected_cdf = stats.cauchy(loc, scale).logcdf(x) - self.assertAllClose(expected_cdf, cdf.eval(), atol=0, rtol=1e-5) - - def testFiniteGradientAtDifficultPoints(self): - for dtype in [np.float32, np.float64]: - g = ops.Graph() - with g.as_default(): - loc = variables.Variable(dtype(0.0)) - scale = variables.Variable(dtype(1.0)) - dist = cauchy_lib.Cauchy(loc=loc, scale=scale) - x = np.array([-100., -20., -5., 0., 5., 20., 100.]).astype(dtype) - for func in [ - dist.cdf, dist.log_cdf, dist.survival_function, - dist.log_survival_function, dist.log_prob, dist.prob - ]: - value = func(x) - grads = gradients_impl.gradients(value, [loc, scale]) - with self.session(graph=g): - variables.global_variables_initializer().run() - self.assertAllFinite(value) - self.assertAllFinite(grads[0]) - self.assertAllFinite(grads[1]) - - def testCauchyLogSurvivalFunction(self): - with self.cached_session(): - batch_size = 50 - loc = self._rng.randn(batch_size) - scale = self._rng.rand(batch_size) + 1.0 - x = np.linspace(-10.0, 100.0, batch_size).astype(np.float64) - - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - - sf = cauchy.log_survival_function(x) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), sf.shape) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), sf.eval().shape) - self.assertAllEqual(cauchy.batch_shape, sf.shape) - self.assertAllEqual(cauchy.batch_shape, sf.eval().shape) - - if not stats: - return - expected_sf = stats.cauchy(loc, scale).logsf(x) - self.assertAllClose(expected_sf, sf.eval(), atol=0, rtol=1e-5) - - def testCauchyEntropy(self): - with self.cached_session(): - loc = np.array([1.0, 1.0, 1.0]) - scale = np.array([[1.0, 2.0, 3.0]]) - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - - entropy = cauchy.entropy() - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), entropy.shape) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), - entropy.eval().shape) - self.assertAllEqual(cauchy.batch_shape, entropy.shape) - self.assertAllEqual(cauchy.batch_shape, entropy.eval().shape) - - if not stats: - return - expected_entropy = stats.cauchy(loc, scale[0]).entropy().reshape((1, 3)) - self.assertAllClose(expected_entropy, entropy.eval()) - - def testCauchyMode(self): - with self.cached_session(): - # Mu will be broadcast to [7, 7, 7]. - loc = [7.] - scale = [11., 12., 13.] - - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - - self.assertAllEqual((3,), cauchy.mode().shape) - self.assertAllEqual([7., 7, 7], cauchy.mode().eval()) - - def testCauchyMean(self): - with self.cached_session(): - loc = [1., 2., 3.] - scale = [7.] - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - - self.assertAllEqual((3,), cauchy.mean().shape) - self.assertAllEqual([np.nan] * 3, cauchy.mean().eval()) - - def testCauchyNanMean(self): - with self.cached_session(): - loc = [1., 2., 3.] - scale = [7.] - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale, allow_nan_stats=False) - - with self.assertRaises(ValueError): - cauchy.mean().eval() - - def testCauchyQuantile(self): - with self.cached_session(): - batch_size = 50 - loc = self._rng.randn(batch_size) - scale = self._rng.rand(batch_size) + 1.0 - p = np.linspace(0.000001, 0.999999, batch_size).astype(np.float64) - - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - x = cauchy.quantile(p) - - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), x.shape) - self.assertAllEqual(cauchy.batch_shape_tensor().eval(), x.eval().shape) - self.assertAllEqual(cauchy.batch_shape, x.shape) - self.assertAllEqual(cauchy.batch_shape, x.eval().shape) - - if not stats: - return - expected_x = stats.cauchy(loc, scale).ppf(p) - self.assertAllClose(expected_x, x.eval(), atol=0.) - - def testCauchyVariance(self): - with self.cached_session(): - # scale will be broadcast to [7, 7, 7] - loc = [1., 2., 3.] - scale = [7.] - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - - self.assertAllEqual((3,), cauchy.variance().shape) - self.assertAllEqual([np.nan] * 3, cauchy.variance().eval()) - - def testCauchyNanVariance(self): - with self.cached_session(): - # scale will be broadcast to [7, 7, 7] - loc = [1., 2., 3.] - scale = [7.] - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale, allow_nan_stats=False) - - with self.assertRaises(ValueError): - cauchy.variance().eval() - - def testCauchyStandardDeviation(self): - with self.cached_session(): - # scale will be broadcast to [7, 7, 7] - loc = [1., 2., 3.] - scale = [7.] - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - - self.assertAllEqual((3,), cauchy.stddev().shape) - self.assertAllEqual([np.nan] * 3, cauchy.stddev().eval()) - - def testCauchyNanStandardDeviation(self): - with self.cached_session(): - # scale will be broadcast to [7, 7, 7] - loc = [1., 2., 3.] - scale = [7.] - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale, allow_nan_stats=False) - - with self.assertRaises(ValueError): - cauchy.stddev().eval() - - def testCauchySample(self): - with self.cached_session(): - loc = constant_op.constant(3.0) - scale = constant_op.constant(1.0) - loc_v = 3.0 - n = constant_op.constant(100000) - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - samples = cauchy.sample(n) - sample_values = samples.eval() - - self.assertEqual(sample_values.shape, (100000,)) - self.assertAllClose(np.median(sample_values), loc_v, atol=1e-1) - - expected_shape = tensor_shape.TensorShape([n.eval()]).concatenate( - tensor_shape.TensorShape(cauchy.batch_shape_tensor().eval())) - - self.assertAllEqual(expected_shape, samples.shape) - self.assertAllEqual(expected_shape, sample_values.shape) - - expected_shape = ( - tensor_shape.TensorShape([n.eval()]).concatenate(cauchy.batch_shape)) - - self.assertAllEqual(expected_shape, samples.shape) - self.assertAllEqual(expected_shape, sample_values.shape) - - def testCauchySampleMultiDimensional(self): - with self.cached_session(): - batch_size = 2 - loc = constant_op.constant([[3.0, -3.0]] * batch_size) - scale = constant_op.constant([[0.5, 1.0]] * batch_size) - loc_v = [3.0, -3.0] - n = constant_op.constant(100000) - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - samples = cauchy.sample(n) - sample_values = samples.eval() - self.assertEqual(samples.shape, (100000, batch_size, 2)) - self.assertAllClose( - np.median(sample_values[:, 0, 0]), loc_v[0], atol=1e-1) - self.assertAllClose( - np.median(sample_values[:, 0, 1]), loc_v[1], atol=1e-1) - - expected_shape = tensor_shape.TensorShape([n.eval()]).concatenate( - tensor_shape.TensorShape(cauchy.batch_shape_tensor().eval())) - self.assertAllEqual(expected_shape, samples.shape) - self.assertAllEqual(expected_shape, sample_values.shape) - - expected_shape = ( - tensor_shape.TensorShape([n.eval()]).concatenate(cauchy.batch_shape)) - self.assertAllEqual(expected_shape, samples.shape) - self.assertAllEqual(expected_shape, sample_values.shape) - - def testCauchyNegativeLocFails(self): - with self.cached_session(): - with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - "Condition x > 0 did not hold"): - _ = cauchy_lib.Cauchy(loc=[1.], scale=[-5.], validate_args=True) - # Error detected statically; no need for _.mode().eval() - - def testCauchyShape(self): - with self.cached_session(): - loc = constant_op.constant([-3.0] * 5) - scale = constant_op.constant(11.0) - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - - self.assertEqual(cauchy.batch_shape_tensor().eval(), [5]) - self.assertEqual(cauchy.batch_shape, tensor_shape.TensorShape([5])) - self.assertAllEqual(cauchy.event_shape_tensor().eval(), []) - self.assertEqual(cauchy.event_shape, tensor_shape.TensorShape([])) - - def testCauchyShapeWithPlaceholders(self): - loc = array_ops.placeholder(dtype=dtypes.float32) - scale = array_ops.placeholder(dtype=dtypes.float32) - cauchy = cauchy_lib.Cauchy(loc=loc, scale=scale) - - with self.cached_session() as sess: - # get_batch_shape should return an "" tensor. - self.assertEqual(cauchy.batch_shape, tensor_shape.TensorShape(None)) - self.assertEqual(cauchy.event_shape, ()) - self.assertAllEqual(cauchy.event_shape_tensor().eval(), []) - self.assertAllEqual( - sess.run( - cauchy.batch_shape_tensor(), - feed_dict={ - loc: 5.0, - scale: [1.0, 2.0] - }), [2]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/chi2_test.py b/tensorflow/contrib/distributions/python/kernel_tests/chi2_test.py deleted file mode 100644 index 3b5a6aa90c1..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/chi2_test.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for initializers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import stats -from tensorflow.contrib.distributions.python.ops import chi2 as chi2_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class Chi2Test(test.TestCase): - - def testChi2LogPDF(self): - with self.cached_session(): - batch_size = 6 - df = constant_op.constant([2.0] * batch_size, dtype=np.float64) - df_v = 2.0 - x = np.array([2.5, 2.5, 4.0, 0.1, 1.0, 2.0], dtype=np.float64) - chi2 = chi2_lib.Chi2(df=df) - expected_log_pdf = stats.chi2.logpdf(x, df_v) - - log_pdf = chi2.log_prob(x) - self.assertEqual(log_pdf.get_shape(), (6,)) - self.assertAllClose(log_pdf.eval(), expected_log_pdf) - - pdf = chi2.prob(x) - self.assertEqual(pdf.get_shape(), (6,)) - self.assertAllClose(pdf.eval(), np.exp(expected_log_pdf)) - - def testChi2CDF(self): - with self.cached_session(): - batch_size = 6 - df = constant_op.constant([2.0] * batch_size, dtype=np.float64) - df_v = 2.0 - x = np.array([2.5, 2.5, 4.0, 0.1, 1.0, 2.0], dtype=np.float64) - - chi2 = chi2_lib.Chi2(df=df) - expected_cdf = stats.chi2.cdf(x, df_v) - - cdf = chi2.cdf(x) - self.assertEqual(cdf.get_shape(), (6,)) - self.assertAllClose(cdf.eval(), expected_cdf) - - def testChi2Mean(self): - with self.cached_session(): - df_v = np.array([1., 3, 5], dtype=np.float64) - expected_mean = stats.chi2.mean(df_v) - chi2 = chi2_lib.Chi2(df=df_v) - self.assertEqual(chi2.mean().get_shape(), (3,)) - self.assertAllClose(chi2.mean().eval(), expected_mean) - - def testChi2Variance(self): - with self.cached_session(): - df_v = np.array([1., 3, 5], np.float64) - expected_variances = stats.chi2.var(df_v) - chi2 = chi2_lib.Chi2(df=df_v) - self.assertEqual(chi2.variance().get_shape(), (3,)) - self.assertAllClose(chi2.variance().eval(), expected_variances) - - def testChi2Entropy(self): - with self.cached_session(): - df_v = np.array([1., 3, 5], dtype=np.float64) - expected_entropy = stats.chi2.entropy(df_v) - chi2 = chi2_lib.Chi2(df=df_v) - self.assertEqual(chi2.entropy().get_shape(), (3,)) - self.assertAllClose(chi2.entropy().eval(), expected_entropy) - - def testChi2WithAbsDf(self): - with self.cached_session(): - df_v = np.array([-1.3, -3.2, 5], dtype=np.float64) - chi2 = chi2_lib.Chi2WithAbsDf(df=df_v) - self.assertAllClose( - math_ops.floor(math_ops.abs(df_v)).eval(), - chi2.df.eval()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/conditional_distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/conditional_distribution_test.py deleted file mode 100644 index 982aa9c95b0..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/conditional_distribution_test.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the Bernoulli distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import distributions -from tensorflow.contrib.distributions.python.kernel_tests import distribution_test -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.platform import test - - -class ConditionalDistributionTest(distribution_test.DistributionTest): - - def _GetFakeDistribution(self): - class _FakeDistribution(distributions.ConditionalDistribution): - """Fake Distribution for testing _set_sample_static_shape.""" - - def __init__(self, batch_shape=None, event_shape=None): - self._static_batch_shape = tensor_shape.TensorShape(batch_shape) - self._static_event_shape = tensor_shape.TensorShape(event_shape) - super(_FakeDistribution, self).__init__( - dtype=dtypes.float32, - reparameterization_type=distributions.NOT_REPARAMETERIZED, - validate_args=True, - allow_nan_stats=True, - name="DummyDistribution") - - def _batch_shape(self): - return self._static_batch_shape - - def _event_shape(self): - return self._static_event_shape - - def _sample_n(self, unused_shape, unused_seed, arg1, arg2): - raise ValueError(arg1, arg2) - - def _log_prob(self, _, arg1, arg2): - raise ValueError(arg1, arg2) - - def _prob(self, _, arg1, arg2): - raise ValueError(arg1, arg2) - - def _cdf(self, _, arg1, arg2): - raise ValueError(arg1, arg2) - - def _log_cdf(self, _, arg1, arg2): - raise ValueError(arg1, arg2) - - def _log_survival_function(self, _, arg1, arg2): - raise ValueError(arg1, arg2) - - def _survival_function(self, _, arg1, arg2): - raise ValueError(arg1, arg2) - - return _FakeDistribution - - def testNotImplemented(self): - d = self._GetFakeDistribution()(batch_shape=[], event_shape=[]) - for name in ["sample", "log_prob", "prob", "log_cdf", "cdf", - "log_survival_function", "survival_function"]: - method = getattr(d, name) - with self.assertRaisesRegexp(ValueError, "b1.*b2"): - method([] if name == "sample" else 1.0, arg1="b1", arg2="b2") - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/conditional_transformed_distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/conditional_transformed_distribution_test.py deleted file mode 100644 index 7e63b5ca5f8..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/conditional_transformed_distribution_test.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for ConditionalTransformedDistribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib import distributions -from tensorflow.contrib.distributions.python.kernel_tests import transformed_distribution_test -from tensorflow.contrib.distributions.python.ops.bijectors.conditional_bijector import ConditionalBijector -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - -ds = distributions - - -class _ChooseLocation(ConditionalBijector): - """A Bijector which chooses between one of two location parameters.""" - - def __init__(self, loc, name="ChooseLocation"): - self._graph_parents = [] - self._name = name - with self._name_scope("init", values=[loc]): - self._loc = ops.convert_to_tensor(loc, name="loc") - super(_ChooseLocation, self).__init__( - graph_parents=[self._loc], - is_constant_jacobian=True, - validate_args=False, - forward_min_event_ndims=0, - name=name) - - def _forward(self, x, z): - return x + self._gather_loc(z) - - def _inverse(self, x, z): - return x - self._gather_loc(z) - - def _inverse_log_det_jacobian(self, x, event_ndims, z=None): - return 0. - - def _gather_loc(self, z): - z = ops.convert_to_tensor(z) - z = math_ops.cast((1 + z) / 2, dtypes.int32) - return array_ops.gather(self._loc, z) - - -class ConditionalTransformedDistributionTest( - transformed_distribution_test.TransformedDistributionTest): - - def _cls(self): - return ds.ConditionalTransformedDistribution - - def testConditioning(self): - with self.cached_session(): - conditional_normal = ds.ConditionalTransformedDistribution( - distribution=ds.Normal(loc=0., scale=1.), - bijector=_ChooseLocation(loc=[-100., 100.])) - z = [-1, +1, -1, -1, +1] - self.assertAllClose( - np.sign(conditional_normal.sample( - 5, bijector_kwargs={"z": z}).eval()), z) - - -class ConditionalScalarToMultiTest( - transformed_distribution_test.ScalarToMultiTest): - - def _cls(self): - return ds.ConditionalTransformedDistribution - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/deterministic_test.py b/tensorflow/contrib/distributions/python/kernel_tests/deterministic_test.py deleted file mode 100644 index bdcf6f39445..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/deterministic_test.py +++ /dev/null @@ -1,309 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib.distributions.python.ops import deterministic as deterministic_lib -from tensorflow.python.framework import errors -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - -rng = np.random.RandomState(0) - - -class DeterministicTest(test.TestCase): - - def testShape(self): - with self.cached_session(): - loc = rng.rand(2, 3, 4) - deterministic = deterministic_lib.Deterministic(loc) - - self.assertAllEqual(deterministic.batch_shape_tensor().eval(), (2, 3, 4)) - self.assertAllEqual(deterministic.batch_shape, (2, 3, 4)) - self.assertAllEqual(deterministic.event_shape_tensor().eval(), []) - self.assertEqual(deterministic.event_shape, tensor_shape.TensorShape([])) - - def testInvalidTolRaises(self): - loc = rng.rand(2, 3, 4).astype(np.float32) - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "Condition x >= 0"): - _ = deterministic_lib.Deterministic(loc, atol=-1, validate_args=True) - # Error detected statically; no need for _.prob(0.).eval() - - def testProbWithNoBatchDimsIntegerType(self): - deterministic = deterministic_lib.Deterministic(0) - with self.cached_session(): - self.assertAllClose(1, deterministic.prob(0).eval()) - self.assertAllClose(0, deterministic.prob(2).eval()) - self.assertAllClose([1, 0], deterministic.prob([0, 2]).eval()) - - def testProbWithNoBatchDims(self): - deterministic = deterministic_lib.Deterministic(0.) - with self.cached_session(): - self.assertAllClose(1., deterministic.prob(0.).eval()) - self.assertAllClose(0., deterministic.prob(2.).eval()) - self.assertAllClose([1., 0.], deterministic.prob([0., 2.]).eval()) - - def testProbWithDefaultTol(self): - loc = [[0., 1.], [2., 3.]] - x = [[0., 1.1], [1.99, 3.]] - deterministic = deterministic_lib.Deterministic(loc) - expected_prob = [[1., 0.], [0., 1.]] - with self.cached_session(): - prob = deterministic.prob(x) - self.assertAllEqual((2, 2), prob.get_shape()) - self.assertAllEqual(expected_prob, prob.eval()) - - def testProbWithNonzeroATol(self): - loc = [[0., 1.], [2., 3.]] - x = [[0., 1.1], [1.99, 3.]] - deterministic = deterministic_lib.Deterministic(loc, atol=0.05) - expected_prob = [[1., 0.], [1., 1.]] - with self.cached_session(): - prob = deterministic.prob(x) - self.assertAllEqual((2, 2), prob.get_shape()) - self.assertAllEqual(expected_prob, prob.eval()) - - def testProbWithNonzeroATolIntegerType(self): - loc = [[0, 1], [2, 3]] - x = [[0, 2], [4, 2]] - deterministic = deterministic_lib.Deterministic(loc, atol=1) - expected_prob = [[1, 1], [0, 1]] - with self.cached_session(): - prob = deterministic.prob(x) - self.assertAllEqual((2, 2), prob.get_shape()) - self.assertAllEqual(expected_prob, prob.eval()) - - def testProbWithNonzeroRTol(self): - loc = [[0., 1.], [100., 100.]] - x = [[0., 1.1], [100.1, 103.]] - deterministic = deterministic_lib.Deterministic(loc, rtol=0.01) - expected_prob = [[1., 0.], [1., 0.]] - with self.cached_session(): - prob = deterministic.prob(x) - self.assertAllEqual((2, 2), prob.get_shape()) - self.assertAllEqual(expected_prob, prob.eval()) - - def testProbWithNonzeroRTolIntegerType(self): - loc = [[10, 10, 10], [10, 10, 10]] - x = [[10, 20, 30], [10, 20, 30]] - # Batch 0 will have rtol = 0 - # Batch 1 will have rtol = 1 (100% slack allowed) - deterministic = deterministic_lib.Deterministic(loc, rtol=[[0], [1]]) - expected_prob = [[1, 0, 0], [1, 1, 0]] - with self.cached_session(): - prob = deterministic.prob(x) - self.assertAllEqual((2, 3), prob.get_shape()) - self.assertAllEqual(expected_prob, prob.eval()) - - def testCdfWithDefaultTol(self): - loc = [[0., 0.], [0., 0.]] - x = [[-1., -0.1], [-0.01, 1.000001]] - deterministic = deterministic_lib.Deterministic(loc) - expected_cdf = [[0., 0.], [0., 1.]] - with self.cached_session(): - cdf = deterministic.cdf(x) - self.assertAllEqual((2, 2), cdf.get_shape()) - self.assertAllEqual(expected_cdf, cdf.eval()) - - def testCdfWithNonzeroATol(self): - loc = [[0., 0.], [0., 0.]] - x = [[-1., -0.1], [-0.01, 1.000001]] - deterministic = deterministic_lib.Deterministic(loc, atol=0.05) - expected_cdf = [[0., 0.], [1., 1.]] - with self.cached_session(): - cdf = deterministic.cdf(x) - self.assertAllEqual((2, 2), cdf.get_shape()) - self.assertAllEqual(expected_cdf, cdf.eval()) - - def testCdfWithNonzeroRTol(self): - loc = [[1., 1.], [100., 100.]] - x = [[0.9, 1.], [99.9, 97]] - deterministic = deterministic_lib.Deterministic(loc, rtol=0.01) - expected_cdf = [[0., 1.], [1., 0.]] - with self.cached_session(): - cdf = deterministic.cdf(x) - self.assertAllEqual((2, 2), cdf.get_shape()) - self.assertAllEqual(expected_cdf, cdf.eval()) - - def testSampleNoBatchDims(self): - deterministic = deterministic_lib.Deterministic(0.) - for sample_shape in [(), (4,)]: - with self.cached_session(): - sample = deterministic.sample(sample_shape) - self.assertAllEqual(sample_shape, sample.get_shape()) - self.assertAllClose( - np.zeros(sample_shape).astype(np.float32), sample.eval()) - - def testSampleWithBatchDims(self): - deterministic = deterministic_lib.Deterministic([0., 0.]) - for sample_shape in [(), (4,)]: - with self.cached_session(): - sample = deterministic.sample(sample_shape) - self.assertAllEqual(sample_shape + (2,), sample.get_shape()) - self.assertAllClose( - np.zeros(sample_shape + (2,)).astype(np.float32), sample.eval()) - - def testSampleDynamicWithBatchDims(self): - loc = array_ops.placeholder(np.float32) - sample_shape = array_ops.placeholder(np.int32) - - deterministic = deterministic_lib.Deterministic(loc) - for sample_shape_ in [(), (4,)]: - with self.cached_session(): - sample_ = deterministic.sample(sample_shape).eval( - feed_dict={loc: [0., 0.], - sample_shape: sample_shape_}) - self.assertAllClose( - np.zeros(sample_shape_ + (2,)).astype(np.float32), sample_) - - def testEntropy(self): - loc = np.array([-0.1, -3.2, 7.]) - deterministic = deterministic_lib.Deterministic(loc=loc) - with self.cached_session() as sess: - entropy_ = sess.run(deterministic.entropy()) - self.assertAllEqual(np.zeros(3), entropy_) - - -class VectorDeterministicTest(test.TestCase): - - def testShape(self): - with self.cached_session(): - loc = rng.rand(2, 3, 4) - deterministic = deterministic_lib.VectorDeterministic(loc) - - self.assertAllEqual(deterministic.batch_shape_tensor().eval(), (2, 3)) - self.assertAllEqual(deterministic.batch_shape, (2, 3)) - self.assertAllEqual(deterministic.event_shape_tensor().eval(), [4]) - self.assertEqual(deterministic.event_shape, tensor_shape.TensorShape([4])) - - def testInvalidTolRaises(self): - loc = rng.rand(2, 3, 4).astype(np.float32) - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "Condition x >= 0"): - _ = deterministic_lib.VectorDeterministic( - loc, atol=-1, validate_args=True) - # Error detected statically; no need for _.prob(loc).eval() - - def testInvalidXRaises(self): - loc = rng.rand(2, 3, 4).astype(np.float32) - deterministic = deterministic_lib.VectorDeterministic( - loc, atol=None, validate_args=True) - with self.cached_session(): - with self.assertRaisesRegexp(ValueError, "must have rank at least 1"): - deterministic.prob(0.).eval() - - def testProbVectorDeterministicWithNoBatchDims(self): - # 0 batch of deterministics on R^1. - deterministic = deterministic_lib.VectorDeterministic([0.]) - with self.cached_session(): - self.assertAllClose(1., deterministic.prob([0.]).eval()) - self.assertAllClose(0., deterministic.prob([2.]).eval()) - self.assertAllClose([1., 0.], deterministic.prob([[0.], [2.]]).eval()) - - def testProbWithDefaultTol(self): - # 3 batch of deterministics on R^2. - loc = [[0., 1.], [2., 3.], [4., 5.]] - x = [[0., 1.], [1.9, 3.], [3.99, 5.]] - deterministic = deterministic_lib.VectorDeterministic(loc) - expected_prob = [1., 0., 0.] - with self.cached_session(): - prob = deterministic.prob(x) - self.assertAllEqual((3,), prob.get_shape()) - self.assertAllEqual(expected_prob, prob.eval()) - - def testProbWithNonzeroATol(self): - # 3 batch of deterministics on R^2. - loc = [[0., 1.], [2., 3.], [4., 5.]] - x = [[0., 1.], [1.9, 3.], [3.99, 5.]] - deterministic = deterministic_lib.VectorDeterministic(loc, atol=0.05) - expected_prob = [1., 0., 1.] - with self.cached_session(): - prob = deterministic.prob(x) - self.assertAllEqual((3,), prob.get_shape()) - self.assertAllEqual(expected_prob, prob.eval()) - - def testProbWithNonzeroRTol(self): - # 3 batch of deterministics on R^2. - loc = [[0., 1.], [1., 1.], [100., 100.]] - x = [[0., 1.], [0.9, 1.], [99.9, 100.1]] - deterministic = deterministic_lib.VectorDeterministic(loc, rtol=0.01) - expected_prob = [1., 0., 1.] - with self.cached_session(): - prob = deterministic.prob(x) - self.assertAllEqual((3,), prob.get_shape()) - self.assertAllEqual(expected_prob, prob.eval()) - - def testProbVectorDeterministicWithNoBatchDimsOnRZero(self): - # 0 batch of deterministics on R^0. - deterministic = deterministic_lib.VectorDeterministic( - [], validate_args=True) - with self.cached_session(): - self.assertAllClose(1., deterministic.prob([]).eval()) - - def testProbVectorDeterministicWithNoBatchDimsOnRZeroRaisesIfXNotInSameRk( - self): - # 0 batch of deterministics on R^0. - deterministic = deterministic_lib.VectorDeterministic( - [], validate_args=True) - with self.cached_session(): - with self.assertRaisesOpError("not defined in the same space"): - deterministic.prob([1.]).eval() - - def testSampleNoBatchDims(self): - deterministic = deterministic_lib.VectorDeterministic([0.]) - for sample_shape in [(), (4,)]: - with self.cached_session(): - sample = deterministic.sample(sample_shape) - self.assertAllEqual(sample_shape + (1,), sample.get_shape()) - self.assertAllClose( - np.zeros(sample_shape + (1,)).astype(np.float32), sample.eval()) - - def testSampleWithBatchDims(self): - deterministic = deterministic_lib.VectorDeterministic([[0.], [0.]]) - for sample_shape in [(), (4,)]: - with self.cached_session(): - sample = deterministic.sample(sample_shape) - self.assertAllEqual(sample_shape + (2, 1), sample.get_shape()) - self.assertAllClose( - np.zeros(sample_shape + (2, 1)).astype(np.float32), sample.eval()) - - def testSampleDynamicWithBatchDims(self): - loc = array_ops.placeholder(np.float32) - sample_shape = array_ops.placeholder(np.int32) - - deterministic = deterministic_lib.VectorDeterministic(loc) - for sample_shape_ in [(), (4,)]: - with self.cached_session(): - sample_ = deterministic.sample(sample_shape).eval( - feed_dict={loc: [[0.], [0.]], - sample_shape: sample_shape_}) - self.assertAllClose( - np.zeros(sample_shape_ + (2, 1)).astype(np.float32), sample_) - - def testEntropy(self): - loc = np.array([[8.3, 1.2, 3.3], [-0.1, -3.2, 7.]]) - deterministic = deterministic_lib.VectorDeterministic(loc=loc) - with self.cached_session() as sess: - entropy_ = sess.run(deterministic.entropy()) - self.assertAllEqual(np.zeros(2), entropy_) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py deleted file mode 100644 index 99cb105d668..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py +++ /dev/null @@ -1,331 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib import distributions -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import test - -tfd = distributions - - -class DistributionTest(test.TestCase): - - def testParamShapesAndFromParams(self): - classes = [ - tfd.Normal, - tfd.Bernoulli, - tfd.Beta, - tfd.Chi2, - tfd.Exponential, - tfd.Gamma, - tfd.InverseGamma, - tfd.Laplace, - tfd.StudentT, - tfd.Uniform, - ] - - sample_shapes = [(), (10,), (10, 20, 30)] - with self.cached_session(): - for cls in classes: - for sample_shape in sample_shapes: - param_shapes = cls.param_shapes(sample_shape) - params = dict([(name, random_ops.random_normal(shape)) - for name, shape in param_shapes.items()]) - dist = cls(**params) - self.assertAllEqual(sample_shape, - array_ops.shape(dist.sample()).eval()) - dist_copy = dist.copy() - self.assertAllEqual(sample_shape, - array_ops.shape(dist_copy.sample()).eval()) - self.assertEqual(dist.parameters, dist_copy.parameters) - - def testCopyExtraArgs(self): - with self.cached_session(): - # Note: we cannot easily test all distributions since each requires - # different initialization arguments. We therefore spot test a few. - normal = tfd.Normal(loc=1., scale=2., validate_args=True) - self.assertEqual(normal.parameters, normal.copy().parameters) - wishart = tfd.WishartFull(df=2, scale=[[1., 2], [2, 5]], - validate_args=True) - self.assertEqual(wishart.parameters, wishart.copy().parameters) - - def testCopyOverride(self): - with self.cached_session(): - normal = tfd.Normal(loc=1., scale=2., validate_args=True) - unused_normal_copy = normal.copy(validate_args=False) - base_params = normal.parameters.copy() - copy_params = normal.copy(validate_args=False).parameters.copy() - self.assertNotEqual( - base_params.pop("validate_args"), copy_params.pop("validate_args")) - self.assertEqual(base_params, copy_params) - - def testIsScalar(self): - with self.cached_session(): - mu = 1. - sigma = 2. - - normal = tfd.Normal(mu, sigma, validate_args=True) - self.assertTrue(tensor_util.constant_value(normal.is_scalar_event())) - self.assertTrue(tensor_util.constant_value(normal.is_scalar_batch())) - - normal = tfd.Normal([mu], [sigma], validate_args=True) - self.assertTrue(tensor_util.constant_value(normal.is_scalar_event())) - self.assertFalse(tensor_util.constant_value(normal.is_scalar_batch())) - - mvn = tfd.MultivariateNormalDiag([mu], [sigma], validate_args=True) - self.assertFalse(tensor_util.constant_value(mvn.is_scalar_event())) - self.assertTrue(tensor_util.constant_value(mvn.is_scalar_batch())) - - mvn = tfd.MultivariateNormalDiag([[mu]], [[sigma]], validate_args=True) - self.assertFalse(tensor_util.constant_value(mvn.is_scalar_event())) - self.assertFalse(tensor_util.constant_value(mvn.is_scalar_batch())) - - # We now test every codepath within the underlying is_scalar_helper - # function. - - # Test case 1, 2. - x = array_ops.placeholder(dtype=dtypes.int32, shape=[]) - # None would fire an exception were it actually executed. - self.assertTrue(normal._is_scalar_helper(x.get_shape(), lambda: None)) - self.assertTrue( - normal._is_scalar_helper(tensor_shape.TensorShape(None), - lambda: array_ops.shape(x))) - - x = array_ops.placeholder(dtype=dtypes.int32, shape=[1]) - # None would fire an exception were it actually executed. - self.assertFalse(normal._is_scalar_helper(x.get_shape(), lambda: None)) - self.assertFalse( - normal._is_scalar_helper(tensor_shape.TensorShape(None), - lambda: array_ops.shape(x))) - - # Test case 3. - x = array_ops.placeholder(dtype=dtypes.int32) - is_scalar = normal._is_scalar_helper(x.get_shape(), - lambda: array_ops.shape(x)) - self.assertTrue(is_scalar.eval(feed_dict={x: 1})) - self.assertFalse(is_scalar.eval(feed_dict={x: [1]})) - - def _GetFakeDistribution(self): - class FakeDistribution(tfd.Distribution): - """Fake Distribution for testing _set_sample_static_shape.""" - - def __init__(self, batch_shape=None, event_shape=None): - self._static_batch_shape = tensor_shape.TensorShape(batch_shape) - self._static_event_shape = tensor_shape.TensorShape(event_shape) - super(FakeDistribution, self).__init__( - dtype=dtypes.float32, - reparameterization_type=distributions.NOT_REPARAMETERIZED, - validate_args=True, - allow_nan_stats=True, - name="DummyDistribution") - - def _batch_shape(self): - return self._static_batch_shape - - def _event_shape(self): - return self._static_event_shape - - return FakeDistribution - - def testSampleShapeHints(self): - fake_distribution = self._GetFakeDistribution() - - with self.cached_session(): - # Make a new session since we're playing with static shapes. [And below.] - x = array_ops.placeholder(dtype=dtypes.float32) - dist = fake_distribution(batch_shape=[2, 3], event_shape=[5]) - sample_shape = ops.convert_to_tensor([6, 7], dtype=dtypes.int32) - y = dist._set_sample_static_shape(x, sample_shape) - # We use as_list since TensorShape comparison does not work correctly for - # unknown values, ie, Dimension(None). - self.assertAllEqual([6, 7, 2, 3, 5], y.get_shape().as_list()) - - with self.cached_session(): - x = array_ops.placeholder(dtype=dtypes.float32) - dist = fake_distribution(batch_shape=[None, 3], event_shape=[5]) - sample_shape = ops.convert_to_tensor([6, 7], dtype=dtypes.int32) - y = dist._set_sample_static_shape(x, sample_shape) - self.assertAllEqual([6, 7, None, 3, 5], y.get_shape().as_list()) - - with self.cached_session(): - x = array_ops.placeholder(dtype=dtypes.float32) - dist = fake_distribution(batch_shape=[None, 3], event_shape=[None]) - sample_shape = ops.convert_to_tensor([6, 7], dtype=dtypes.int32) - y = dist._set_sample_static_shape(x, sample_shape) - self.assertAllEqual([6, 7, None, 3, None], y.get_shape().as_list()) - - with self.cached_session(): - x = array_ops.placeholder(dtype=dtypes.float32) - dist = fake_distribution(batch_shape=None, event_shape=None) - sample_shape = ops.convert_to_tensor([6, 7], dtype=dtypes.int32) - y = dist._set_sample_static_shape(x, sample_shape) - self.assertTrue(y.get_shape().ndims is None) - - with self.cached_session(): - x = array_ops.placeholder(dtype=dtypes.float32) - dist = fake_distribution(batch_shape=[None, 3], event_shape=None) - sample_shape = ops.convert_to_tensor([6, 7], dtype=dtypes.int32) - y = dist._set_sample_static_shape(x, sample_shape) - self.assertTrue(y.get_shape().ndims is None) - - def testNameScopeWorksCorrectly(self): - x = tfd.Normal(loc=0., scale=1., name="x") - x_duplicate = tfd.Normal(loc=0., scale=1., name="x") - with ops.name_scope("y") as name: - y = tfd.Bernoulli(logits=0., name=name) - x_sample = x.sample(name="custom_sample") - x_sample_duplicate = x.sample(name="custom_sample") - x_log_prob = x.log_prob(0., name="custom_log_prob") - x_duplicate_sample = x_duplicate.sample(name="custom_sample") - - self.assertEqual(x.name, "x/") - self.assertEqual(x_duplicate.name, "x_1/") - self.assertEqual(y.name, "y/") - self.assertTrue(x_sample.name.startswith("x/custom_sample")) - self.assertTrue(x_sample_duplicate.name.startswith("x/custom_sample_1")) - self.assertTrue(x_log_prob.name.startswith("x/custom_log_prob")) - self.assertTrue(x_duplicate_sample.name.startswith( - "x_1/custom_sample")) - - def testStrWorksCorrectlyScalar(self): - normal = tfd.Normal(loc=np.float16(0), scale=np.float16(1)) - self.assertEqual( - ("tfp.distributions.Normal(" - "\"Normal/\", " - "batch_shape=(), " - "event_shape=(), " - "dtype=float16)"), # Got the dtype right. - str(normal)) - - chi2 = tfd.Chi2(df=np.float32([1., 2.]), name="silly") - self.assertEqual( - ("tfp.distributions.Chi2(" - "\"silly/\", " # What a silly name that is! - "batch_shape=(2,), " - "event_shape=(), " - "dtype=float32)"), - str(chi2)) - - exp = tfd.Exponential(rate=array_ops.placeholder(dtype=dtypes.float32)) - self.assertEqual( - ("tfp.distributions.Exponential(\"Exponential/\", " - # No batch shape. - "event_shape=(), " - "dtype=float32)"), - str(exp)) - - def testStrWorksCorrectlyMultivariate(self): - mvn_static = tfd.MultivariateNormalDiag( - loc=np.zeros([2, 2]), name="MVN") - self.assertEqual( - ("tfp.distributions.MultivariateNormalDiag(" - "\"MVN/\", " - "batch_shape=(2,), " - "event_shape=(2,), " - "dtype=float64)"), - str(mvn_static)) - - mvn_dynamic = tfd.MultivariateNormalDiag( - loc=array_ops.placeholder(shape=[None, 3], dtype=dtypes.float32), - name="MVN2") - if mvn_dynamic.batch_shape._v2_behavior: - self.assertEqual( - ("tfp.distributions.MultivariateNormalDiag(" - "\"MVN2/\", " - "batch_shape=(None,), " # Partially known. - "event_shape=(3,), " - "dtype=float32)"), - str(mvn_dynamic)) - else: - self.assertEqual( - ("tfp.distributions.MultivariateNormalDiag(" - "\"MVN2/\", " - "batch_shape=(?,), " # Partially known. - "event_shape=(3,), " - "dtype=float32)"), - str(mvn_dynamic)) - - def testReprWorksCorrectlyScalar(self): - normal = tfd.Normal(loc=np.float16(0), scale=np.float16(1)) - self.assertEqual( - (""), # Got the dtype right. - repr(normal)) - - chi2 = tfd.Chi2(df=np.float32([1., 2.]), name="silly") - self.assertEqual( - (""), - repr(chi2)) - - exp = tfd.Exponential(rate=array_ops.placeholder(dtype=dtypes.float32)) - self.assertEqual( - ("" - " event_shape=()" - " dtype=float32>"), - repr(exp)) - - def testReprWorksCorrectlyMultivariate(self): - mvn_static = tfd.MultivariateNormalDiag( - loc=np.zeros([2, 2]), name="MVN") - self.assertEqual( - (""), - repr(mvn_static)) - - mvn_dynamic = tfd.MultivariateNormalDiag( - loc=array_ops.placeholder(shape=[None, 3], dtype=dtypes.float32), - name="MVN2") - if mvn_dynamic.batch_shape._v2_behavior: - self.assertEqual( - (""), - repr(mvn_dynamic)) - else: - self.assertEqual( - (""), - repr(mvn_dynamic)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/distribution_util_test.py b/tensorflow/contrib/distributions/python/kernel_tests/distribution_util_test.py deleted file mode 100644 index 05f5d306664..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/distribution_util_test.py +++ /dev/null @@ -1,591 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for utility functions.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.contrib.distributions.python.ops import mixture -from tensorflow.contrib.distributions.python.ops import mixture_same_family -from tensorflow.contrib.distributions.python.ops import mvn_diag -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import categorical -from tensorflow.python.ops.distributions import normal -from tensorflow.python.ops.linalg import linear_operator_diag -import tensorflow.python.ops.nn_grad # pylint: disable=unused-import -from tensorflow.python.platform import test - - -def _powerset(x): - s = list(x) - return itertools.chain.from_iterable( - itertools.combinations(s, r) for r in range(len(s) + 1)) - - -def _matrix_diag(d): - """Batch version of np.diag.""" - orig_shape = d.shape - d = np.reshape(d, (int(np.prod(d.shape[:-1])), d.shape[-1])) - diag_list = [] - for i in range(d.shape[0]): - diag_list.append(np.diag(d[i, ...])) - return np.reshape(diag_list, orig_shape + (d.shape[-1],)) - - -def _make_tril_scale( - loc=None, - scale_tril=None, - scale_diag=None, - scale_identity_multiplier=None, - shape_hint=None): - if scale_tril is not None: - scale_tril = np.tril(scale_tril) - if scale_diag is not None: - scale_tril += _matrix_diag(np.array(scale_diag, dtype=np.float32)) - if scale_identity_multiplier is not None: - scale_tril += ( - scale_identity_multiplier * _matrix_diag(np.ones( - [scale_tril.shape[-1]], dtype=np.float32))) - return scale_tril - return _make_diag_scale( - loc, scale_diag, scale_identity_multiplier, shape_hint) - - -def _make_diag_scale( - loc=None, - scale_diag=None, - scale_identity_multiplier=None, - shape_hint=None): - if scale_diag is not None: - scale_diag = np.asarray(scale_diag) - if scale_identity_multiplier is not None: - scale_diag += scale_identity_multiplier - return _matrix_diag(scale_diag) - - if loc is None and shape_hint is None: - return None - - if shape_hint is None: - shape_hint = loc.shape[-1] - if scale_identity_multiplier is None: - scale_identity_multiplier = 1. - return scale_identity_multiplier * np.diag(np.ones(shape_hint)) - - -class MakeTrilScaleTest(test.TestCase): - - def _testLegalInputs( - self, loc=None, shape_hint=None, scale_params=None): - for args in _powerset(scale_params.items()): - with self.cached_session(): - args = dict(args) - - scale_args = dict({ - "loc": loc, - "shape_hint": shape_hint}, **args) - expected_scale = _make_tril_scale(**scale_args) - if expected_scale is None: - # Not enough shape information was specified. - with self.assertRaisesRegexp(ValueError, ("is specified.")): - scale = distribution_util.make_tril_scale(**scale_args) - scale.to_dense().eval() - else: - scale = distribution_util.make_tril_scale(**scale_args) - self.assertAllClose(expected_scale, scale.to_dense().eval()) - - def testLegalInputs(self): - self._testLegalInputs( - loc=np.array([-1., -1.], dtype=np.float32), - shape_hint=2, - scale_params={ - "scale_identity_multiplier": 2., - "scale_diag": [2., 3.], - "scale_tril": [[1., 0.], - [-3., 3.]], - }) - - def testLegalInputsMultidimensional(self): - self._testLegalInputs( - loc=np.array([[[-1., -1., 2.], [-2., -3., 4.]]], dtype=np.float32), - shape_hint=3, - scale_params={ - "scale_identity_multiplier": 2., - "scale_diag": [[[2., 3., 4.], [3., 4., 5.]]], - "scale_tril": [[[[1., 0., 0.], - [-3., 3., 0.], - [1., -2., 1.]], - [[2., 1., 0.], - [-4., 7., 0.], - [1., -1., 1.]]]] - }) - - def testZeroTriU(self): - with self.cached_session(): - scale = distribution_util.make_tril_scale(scale_tril=[[1., 1], [1., 1.]]) - self.assertAllClose([[1., 0], [1., 1.]], scale.to_dense().eval()) - - def testValidateArgs(self): - with self.cached_session(): - with self.assertRaisesOpError("diagonal part must be non-zero"): - scale = distribution_util.make_tril_scale( - scale_tril=[[0., 1], [1., 1.]], validate_args=True) - scale.to_dense().eval() - - def testAssertPositive(self): - with self.cached_session(): - with self.assertRaisesOpError("diagonal part must be positive"): - scale = distribution_util.make_tril_scale( - scale_tril=[[-1., 1], [1., 1.]], - validate_args=True, - assert_positive=True) - scale.to_dense().eval() - - -class MakeDiagScaleTest(test.TestCase): - - def _testLegalInputs( - self, loc=None, shape_hint=None, scale_params=None): - for args in _powerset(scale_params.items()): - with self.cached_session(): - args = dict(args) - - scale_args = dict({ - "loc": loc, - "shape_hint": shape_hint}, **args) - expected_scale = _make_diag_scale(**scale_args) - if expected_scale is None: - # Not enough shape information was specified. - with self.assertRaisesRegexp(ValueError, ("is specified.")): - scale = distribution_util.make_diag_scale(**scale_args) - scale.to_dense().eval() - else: - scale = distribution_util.make_diag_scale(**scale_args) - self.assertAllClose(expected_scale, scale.to_dense().eval()) - - def testLegalInputs(self): - self._testLegalInputs( - loc=np.array([-1., -1.], dtype=np.float32), - shape_hint=2, - scale_params={ - "scale_identity_multiplier": 2., - "scale_diag": [2., 3.] - }) - - def testLegalInputsMultidimensional(self): - self._testLegalInputs( - loc=np.array([[[-1., -1., 2.], [-2., -3., 4.]]], dtype=np.float32), - shape_hint=3, - scale_params={ - "scale_identity_multiplier": 2., - "scale_diag": [[[2., 3., 4.], [3., 4., 5.]]] - }) - - def testValidateArgs(self): - with self.cached_session(): - with self.assertRaisesOpError("diagonal part must be non-zero"): - scale = distribution_util.make_diag_scale( - scale_diag=[[0., 1], [1., 1.]], validate_args=True) - scale.to_dense().eval() - - def testAssertPositive(self): - with self.cached_session(): - with self.assertRaisesOpError("diagonal part must be positive"): - scale = distribution_util.make_diag_scale( - scale_diag=[[-1., 1], [1., 1.]], - validate_args=True, - assert_positive=True) - scale.to_dense().eval() - - -class ShapesFromLocAndScaleTest(test.TestCase): - - def test_static_loc_static_scale_non_matching_event_size_raises(self): - loc = constant_op.constant(np.zeros((2, 4))) - scale = linear_operator_diag.LinearOperatorDiag(np.ones((5, 1, 3))) - with self.assertRaisesRegexp(ValueError, "could not be broadcast"): - distribution_util.shapes_from_loc_and_scale(loc, scale) - - def test_static_loc_static_scale(self): - loc = constant_op.constant(np.zeros((2, 3))) - scale = linear_operator_diag.LinearOperatorDiag(np.ones((5, 1, 3))) - batch_shape, event_shape = distribution_util.shapes_from_loc_and_scale( - loc, scale) - - self.assertEqual(tensor_shape.TensorShape([5, 2]), batch_shape) - self.assertEqual(tensor_shape.TensorShape([3]), event_shape) - - def test_static_loc_dynamic_scale(self): - loc = constant_op.constant(np.zeros((2, 3))) - diag = array_ops.placeholder(dtypes.float64) - scale = linear_operator_diag.LinearOperatorDiag(diag) - with self.cached_session() as sess: - batch_shape, event_shape = sess.run( - distribution_util.shapes_from_loc_and_scale(loc, scale), - feed_dict={diag: np.ones((5, 1, 3))}) - self.assertAllEqual([5, 2], batch_shape) - self.assertAllEqual([3], event_shape) - - def test_dynamic_loc_static_scale(self): - loc = array_ops.placeholder(dtypes.float64) - diag = constant_op.constant(np.ones((5, 2, 3))) - scale = linear_operator_diag.LinearOperatorDiag(diag) - with self.cached_session(): - batch_shape, event_shape = distribution_util.shapes_from_loc_and_scale( - loc, scale) - # batch_shape depends on both args, and so is dynamic. Since loc did not - # have static shape, we inferred event shape entirely from scale, and this - # is available statically. - self.assertAllEqual( - [5, 2], batch_shape.eval(feed_dict={loc: np.zeros((2, 3))})) - self.assertAllEqual([3], event_shape) - - def test_dynamic_loc_dynamic_scale(self): - loc = array_ops.placeholder(dtypes.float64) - diag = array_ops.placeholder(dtypes.float64) - scale = linear_operator_diag.LinearOperatorDiag(diag) - with self.cached_session() as sess: - batch_shape, event_shape = sess.run( - distribution_util.shapes_from_loc_and_scale(loc, scale), - feed_dict={diag: np.ones((5, 2, 3)), loc: np.zeros((2, 3))}) - self.assertAllEqual([5, 2], batch_shape) - self.assertAllEqual([3], event_shape) - - def test_none_loc_static_scale(self): - loc = None - scale = linear_operator_diag.LinearOperatorDiag(np.ones((5, 1, 3))) - batch_shape, event_shape = distribution_util.shapes_from_loc_and_scale( - loc, scale) - - self.assertEqual(tensor_shape.TensorShape([5, 1]), batch_shape) - self.assertEqual(tensor_shape.TensorShape([3]), event_shape) - - def test_none_loc_dynamic_scale(self): - loc = None - diag = array_ops.placeholder(dtypes.float64) - scale = linear_operator_diag.LinearOperatorDiag(diag) - with self.cached_session() as sess: - batch_shape, event_shape = sess.run( - distribution_util.shapes_from_loc_and_scale(loc, scale), - feed_dict={diag: np.ones((5, 1, 3))}) - self.assertAllEqual([5, 1], batch_shape) - self.assertAllEqual([3], event_shape) - - -class GetBroadcastShapeTest(test.TestCase): - - def test_all_static_shapes_work(self): - x = array_ops.ones((2, 1, 3)) - y = array_ops.ones((1, 5, 3)) - z = array_ops.ones(()) - self.assertAllEqual([2, 5, 3], - distribution_util.get_broadcast_shape(x, y, z)) - - def test_with_some_dynamic_shapes_works(self): - x = array_ops.ones((2, 1, 3)) - y = array_ops.placeholder(x.dtype) - z = array_ops.ones(()) - with self.cached_session() as sess: - bcast_shape = sess.run( - distribution_util.get_broadcast_shape(x, y, z), - feed_dict={y: np.ones((1, 5, 3)).astype(np.float32)}) - self.assertAllEqual([2, 5, 3], bcast_shape) - - -class TridiagTest(test.TestCase): - - def testWorksCorrectlyNoBatches(self): - with self.cached_session(): - self.assertAllEqual( - [[4., 8., 0., 0.], - [1., 5., 9., 0.], - [0., 2., 6., 10.], - [0., 0., 3, 7.]], - distribution_util.tridiag( - [1., 2., 3.], - [4., 5., 6., 7.], - [8., 9., 10.]).eval()) - - def testWorksCorrectlyBatches(self): - with self.cached_session(): - self.assertAllClose( - [[[4., 8., 0., 0.], - [1., 5., 9., 0.], - [0., 2., 6., 10.], - [0., 0., 3, 7.]], - [[0.7, 0.1, 0.0, 0.0], - [0.8, 0.6, 0.2, 0.0], - [0.0, 0.9, 0.5, 0.3], - [0.0, 0.0, 1.0, 0.4]]], - distribution_util.tridiag( - [[1., 2., 3.], - [0.8, 0.9, 1.]], - [[4., 5., 6., 7.], - [0.7, 0.6, 0.5, 0.4]], - [[8., 9., 10.], - [0.1, 0.2, 0.3]]).eval(), - rtol=1e-5, atol=0.) - - def testHandlesNone(self): - with self.cached_session(): - self.assertAllClose( - [[[4., 0., 0., 0.], - [0., 5., 0., 0.], - [0., 0., 6., 0.], - [0., 0., 0, 7.]], - [[0.7, 0.0, 0.0, 0.0], - [0.0, 0.6, 0.0, 0.0], - [0.0, 0.0, 0.5, 0.0], - [0.0, 0.0, 0.0, 0.4]]], - distribution_util.tridiag( - diag=[[4., 5., 6., 7.], - [0.7, 0.6, 0.5, 0.4]]).eval(), - rtol=1e-5, atol=0.) - - -class MixtureStddevTest(test.TestCase): - - def test_mixture_dev(self): - mixture_weights = np.array([ - [1.0/3, 1.0/3, 1.0/3], - [0.750, 0.250, 0.000] - ]) - component_means = np.array([ - [1.0, 1.0, 1.0], - [-5, 0, 1.25] - ]) - component_devs = np.array([ - [1.0, 1.0, 1.0], - [0.01, 2.0, 0.1] - ]) - - # The first case should trivially have a standard deviation of 1.0 because - # all components are identical and have that standard deviation. - # The second case was computed by hand. - expected_devs = np.array([ - 1.0, - 2.3848637277 - ]) - - weights_tf = array_ops.constant(mixture_weights) - means_tf = array_ops.constant(component_means) - sigmas_tf = array_ops.constant(component_devs) - mix_dev = distribution_util.mixture_stddev(weights_tf, - means_tf, - sigmas_tf) - - with self.cached_session() as sess: - actual_devs = sess.run(mix_dev) - - self.assertAllClose(actual_devs, expected_devs) - - -class PadMixtureDimensionsTest(test.TestCase): - - def test_pad_mixture_dimensions_mixture(self): - with self.cached_session() as sess: - gm = mixture.Mixture( - cat=categorical.Categorical(probs=[[0.3, 0.7]]), - components=[ - normal.Normal(loc=[-1.0], scale=[1.0]), - normal.Normal(loc=[1.0], scale=[0.5]) - ]) - - x = array_ops.constant([[1.0, 2.0], [3.0, 4.0]]) - x_pad = distribution_util.pad_mixture_dimensions( - x, gm, gm.cat, gm.event_shape.ndims) - x_out, x_pad_out = sess.run([x, x_pad]) - - self.assertAllEqual(x_pad_out.shape, [2, 2]) - self.assertAllEqual(x_out.reshape([-1]), x_pad_out.reshape([-1])) - - def test_pad_mixture_dimensions_mixture_same_family(self): - with self.cached_session() as sess: - gm = mixture_same_family.MixtureSameFamily( - mixture_distribution=categorical.Categorical(probs=[0.3, 0.7]), - components_distribution=mvn_diag.MultivariateNormalDiag( - loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1.0, 0.5])) - - x = array_ops.constant([[1.0, 2.0], [3.0, 4.0]]) - x_pad = distribution_util.pad_mixture_dimensions( - x, gm, gm.mixture_distribution, gm.event_shape.ndims) - x_out, x_pad_out = sess.run([x, x_pad]) - - self.assertAllEqual(x_pad_out.shape, [2, 2, 1]) - self.assertAllEqual(x_out.reshape([-1]), x_pad_out.reshape([-1])) - - -class _PadTest(object): - - def testNegAxisCorrectness(self): - x_ = np.float32([[1., 2, 3], - [4, 5, 6]]) - value_ = np.float32(0.25) - count_ = np.int32(2) - with self.cached_session() as sess: - x = array_ops.placeholder_with_default( - x_, shape=x_.shape if self.is_static_shape else None) - value = (constant_op.constant(value_) if self.is_static_shape - else array_ops.placeholder_with_default(value_, shape=None)) - count = (constant_op.constant(count_) if self.is_static_shape - else array_ops.placeholder_with_default(count_, shape=None)) - - x0_front = distribution_util.pad( - x, axis=-2, value=value, count=count, front=True) - x0_back = distribution_util.pad( - x, axis=-2, count=count, back=True) - x0_both = distribution_util.pad( - x, axis=-2, value=value, front=True, back=True) - - if self.is_static_shape: - self.assertAllEqual([4, 3], x0_front.shape) - self.assertAllEqual([4, 3], x0_back.shape) - self.assertAllEqual([4, 3], x0_both.shape) - - [x0_front_, x0_back_, x0_both_] = sess.run([ - x0_front, x0_back, x0_both]) - - self.assertAllClose( - np.float32([[value_]*3, - [value_]*3, - [1, 2, 3], - [4, 5, 6]]), - x0_front_, atol=0., rtol=1e-6) - self.assertAllClose( - np.float32([[1, 2, 3], - [4, 5, 6], - [0.]*3, - [0.]*3]), - x0_back_, atol=0., rtol=1e-6) - self.assertAllClose( - np.float32([[value_]*3, - [1, 2, 3], - [4, 5, 6], - [value_]*3]), - x0_both_, atol=0., rtol=1e-6) - - def testPosAxisCorrectness(self): - x_ = np.float32([[1., 2, 3], - [4, 5, 6]]) - value_ = np.float32(0.25) - count_ = np.int32(2) - with self.cached_session() as sess: - x = array_ops.placeholder_with_default( - x_, shape=x_.shape if self.is_static_shape else None) - value = (constant_op.constant(value_) if self.is_static_shape - else array_ops.placeholder_with_default(value_, shape=None)) - count = (constant_op.constant(count_) if self.is_static_shape - else array_ops.placeholder_with_default(count_, shape=None)) - - x1_front = distribution_util.pad( - x, axis=1, value=value, count=count, front=True) - x1_back = distribution_util.pad( - x, axis=1, count=count, back=True) - x1_both = distribution_util.pad( - x, axis=1, value=value, front=True, back=True) - - if self.is_static_shape: - self.assertAllEqual([2, 5], x1_front.shape) - self.assertAllEqual([2, 5], x1_back.shape) - self.assertAllEqual([2, 5], x1_both.shape) - - [x1_front_, x1_back_, x1_both_] = sess.run([ - x1_front, x1_back, x1_both]) - - self.assertAllClose( - np.float32([[value_]*2 + [1, 2, 3], - [value_]*2 + [4, 5, 6]]), - x1_front_, atol=0., rtol=1e-6) - self.assertAllClose( - np.float32([[1, 2, 3] + [0.]*2, - [4, 5, 6] + [0.]*2]), - x1_back_, atol=0., rtol=1e-6) - self.assertAllClose( - np.float32([[value_, 1, 2, 3, value_], - [value_, 4, 5, 6, value_]]), - x1_both_, atol=0., rtol=1e-6) - - -class PadStaticTest(_PadTest, test.TestCase): - - @property - def is_static_shape(self): - return True - - -class PadDynamicTest(_PadTest, test.TestCase): - - @property - def is_static_shape(self): - return False - - -@test_util.run_all_in_graph_and_eager_modes -class TestMoveDimension(test.TestCase): - - def test_move_dimension_static_shape(self): - - x = random_ops.random_normal(shape=[200, 30, 4, 1, 6]) - - x_perm = distribution_util.move_dimension(x, 1, 1) - self.assertAllEqual(x_perm.shape.as_list(), [200, 30, 4, 1, 6]) - - x_perm = distribution_util.move_dimension(x, 0, 3) - self.assertAllEqual(x_perm.shape.as_list(), [30, 4, 1, 200, 6]) - - x_perm = distribution_util.move_dimension(x, 0, -2) - self.assertAllEqual(x_perm.shape.as_list(), [30, 4, 1, 200, 6]) - - x_perm = distribution_util.move_dimension(x, 4, 2) - self.assertAllEqual(x_perm.shape.as_list(), [200, 30, 6, 4, 1]) - - def test_move_dimension_dynamic_shape(self): - - x_ = random_ops.random_normal(shape=[200, 30, 4, 1, 6]) - x = array_ops.placeholder_with_default(input=x_, shape=None) - - x_perm = distribution_util.move_dimension(x, 1, 1) - self.assertAllEqual(self.evaluate(array_ops.shape(x_perm)), - [200, 30, 4, 1, 6]) - - x_perm = distribution_util.move_dimension(x, 0, 3) - self.assertAllEqual(self.evaluate(array_ops.shape(x_perm)), - [30, 4, 1, 200, 6]) - - x_perm = distribution_util.move_dimension(x, 0, -2) - self.assertAllEqual(self.evaluate(array_ops.shape(x_perm)), - [30, 4, 1, 200, 6]) - - x_perm = distribution_util.move_dimension(x, 4, 2) - self.assertAllEqual(self.evaluate(array_ops.shape(x_perm)), - [200, 30, 6, 4, 1]) - - x_perm = distribution_util.move_dimension(x, -1, 2) - self.assertAllEqual(self.evaluate(array_ops.shape(x_perm)), - [200, 30, 6, 4, 1]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/estimator_test.py b/tensorflow/contrib/distributions/python/kernel_tests/estimator_test.py deleted file mode 100644 index 5ff0544c977..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/estimator_test.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for estimator.py.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import six - -from tensorflow.contrib.distributions.python.ops import estimator as estimator_lib -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.estimators.head_test import _assert_metrics -from tensorflow.contrib.learn.python.learn.estimators.head_test import _assert_no_variables -from tensorflow.contrib.learn.python.learn.estimators.head_test import _assert_summary_tags -from tensorflow.python.client import session -from tensorflow.python.framework import ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops.distributions import normal as normal_lib -from tensorflow.python.platform import test - - -class EstimatorHeadDistributionRegressionTest(test.TestCase): - - def _assert_output_alternatives(self, model_fn_ops): - self.assertEquals({ - None: constants.ProblemType.LINEAR_REGRESSION - }, { - k: v[0] for k, v in six.iteritems(model_fn_ops.output_alternatives) - }) - - def testNormalLocScaleLogits(self): - # We will bias logits[..., 1] so that: logits[..., 1]=0 implies scale=1. - scale_bias = np.log(np.expm1(1.)) - - def softplus(x): - return np.log1p(np.exp(x)) - - def actual_loss(logits, labels): - mu = actual_mean(logits) - sigma = actual_stddev(logits) - labels = np.squeeze(labels, -1) - z = (labels - mu) / sigma - loss = 0.5 * (z**2. + np.log(2. * np.pi)) + np.log(sigma) - return loss.mean() - - def actual_mean(logits): - return logits[..., 0] - - def actual_stddev(logits): - return softplus(logits[..., 1] + scale_bias) - - def make_distribution_fn(logits): - return normal_lib.Normal( - loc=logits[..., 0], - scale=nn_ops.softplus(logits[..., 1] + scale_bias)) - - head = estimator_lib.estimator_head_distribution_regression( - make_distribution_fn, - logits_dimension=2) - labels = np.float32([[-1.], - [0.], - [1.]]) - logits = np.float32([[0., -1], - [1, 0.5], - [-1, 1]]) - with ops.Graph().as_default(), session.Session(): - # Convert to tensor so we can index into head.distributions. - tflogits = ops.convert_to_tensor(logits, name="logits") - model_fn_ops = head.create_model_fn_ops( - {}, - labels=labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=tflogits) - self._assert_output_alternatives(model_fn_ops) - _assert_summary_tags(self, ["loss"]) - _assert_no_variables(self) - loss = actual_loss(logits, labels) - _assert_metrics(self, loss, {"loss": loss}, model_fn_ops) - - # Now we verify the underlying distribution was correctly constructed. - expected_mean = logits[..., 0] - self.assertAllClose( - expected_mean, - head.distribution(tflogits).mean().eval(), - rtol=1e-6, atol=0.) - - expected_stddev = softplus(logits[..., 1] + scale_bias) - self.assertAllClose( - expected_stddev, - head.distribution(tflogits).stddev().eval(), - rtol=1e-6, atol=0.) - # Should have created only one distribution. - self.assertEqual(1, len(head.distributions)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/geometric_test.py b/tensorflow/contrib/distributions/python/kernel_tests/geometric_test.py deleted file mode 100644 index a627d85229d..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/geometric_test.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the Geometric distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import stats -from tensorflow.contrib.distributions.python.ops import geometric -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -# In all tests that follow, we use scipy.stats.geom, which -# represents the "Shifted" Geometric distribution. Hence, loc=-1 is passed -# in to each scipy function for testing. -class GeometricTest(test.TestCase): - - def testGeometricShape(self): - with self.cached_session(): - probs = constant_op.constant([.1] * 5) - geom = geometric.Geometric(probs=probs) - - self.assertEqual([5,], geom.batch_shape_tensor().eval()) - self.assertAllEqual([], geom.event_shape_tensor().eval()) - self.assertEqual(tensor_shape.TensorShape([5]), geom.batch_shape) - self.assertEqual(tensor_shape.TensorShape([]), geom.event_shape) - - def testInvalidP(self): - invalid_ps = [-.01, -0.01, -2.] - with self.cached_session(): - with self.assertRaisesOpError("Condition x >= 0"): - geom = geometric.Geometric(probs=invalid_ps, validate_args=True) - geom.probs.eval() - - invalid_ps = [1.1, 3., 5.] - with self.cached_session(): - with self.assertRaisesOpError("Condition x <= y"): - geom = geometric.Geometric(probs=invalid_ps, validate_args=True) - geom.probs.eval() - - def testGeomLogPmf(self): - with self.cached_session(): - batch_size = 6 - probs = constant_op.constant([.2] * batch_size) - probs_v = .2 - x = np.array([2., 3., 4., 5., 6., 7.], dtype=np.float32) - geom = geometric.Geometric(probs=probs) - expected_log_prob = stats.geom.logpmf(x, probs_v, loc=-1) - log_prob = geom.log_prob(x) - self.assertEqual([6,], log_prob.get_shape()) - self.assertAllClose(expected_log_prob, log_prob.eval()) - - pmf = geom.prob(x) - self.assertEqual([6,], pmf.get_shape()) - self.assertAllClose(np.exp(expected_log_prob), pmf.eval()) - - def testGeometricLogPmf_validate_args(self): - with self.cached_session(): - batch_size = 6 - probs = constant_op.constant([.9] * batch_size) - x = array_ops.placeholder(dtypes.float32, shape=[6]) - feed_dict = {x: [2.5, 3.2, 4.3, 5.1, 6., 7.]} - geom = geometric.Geometric(probs=probs, validate_args=True) - - with self.assertRaisesOpError("Condition x == y"): - log_prob = geom.log_prob(x) - log_prob.eval(feed_dict=feed_dict) - - with self.assertRaisesOpError("Condition x >= 0"): - log_prob = geom.log_prob(np.array([-1.], dtype=np.float32)) - log_prob.eval() - - geom = geometric.Geometric(probs=probs) - log_prob = geom.log_prob(x) - self.assertEqual([6,], log_prob.get_shape()) - pmf = geom.prob(x) - self.assertEqual([6,], pmf.get_shape()) - - def testGeometricLogPmfMultidimensional(self): - with self.cached_session(): - batch_size = 6 - probs = constant_op.constant([[.2, .3, .5]] * batch_size) - probs_v = np.array([.2, .3, .5]) - x = np.array([[2., 3., 4., 5., 6., 7.]], dtype=np.float32).T - geom = geometric.Geometric(probs=probs) - expected_log_prob = stats.geom.logpmf(x, probs_v, loc=-1) - log_prob = geom.log_prob(x) - log_prob_values = log_prob.eval() - self.assertEqual([6, 3], log_prob.get_shape()) - self.assertAllClose(expected_log_prob, log_prob_values) - - pmf = geom.prob(x) - pmf_values = pmf.eval() - self.assertEqual([6, 3], pmf.get_shape()) - self.assertAllClose(np.exp(expected_log_prob), pmf_values) - - def testGeometricCDF(self): - with self.cached_session(): - batch_size = 6 - probs = constant_op.constant([[.2, .4, .5]] * batch_size) - probs_v = np.array([.2, .4, .5]) - x = np.array([[2., 3., 4., 5.5, 6., 7.]], dtype=np.float32).T - - geom = geometric.Geometric(probs=probs) - expected_cdf = stats.geom.cdf(x, probs_v, loc=-1) - - cdf = geom.cdf(x) - self.assertEqual([6, 3], cdf.get_shape()) - self.assertAllClose(expected_cdf, cdf.eval()) - - def testGeometricEntropy(self): - with self.cached_session(): - probs_v = np.array([.1, .3, .25], dtype=np.float32) - geom = geometric.Geometric(probs=probs_v) - expected_entropy = stats.geom.entropy(probs_v, loc=-1) - self.assertEqual([3], geom.entropy().get_shape()) - self.assertAllClose(expected_entropy, geom.entropy().eval()) - - def testGeometricMean(self): - with self.cached_session(): - probs_v = np.array([.1, .3, .25]) - geom = geometric.Geometric(probs=probs_v) - expected_means = stats.geom.mean(probs_v, loc=-1) - self.assertEqual([3], geom.mean().get_shape()) - self.assertAllClose(expected_means, geom.mean().eval()) - - def testGeometricVariance(self): - with self.cached_session(): - probs_v = np.array([.1, .3, .25]) - geom = geometric.Geometric(probs=probs_v) - expected_vars = stats.geom.var(probs_v, loc=-1) - self.assertEqual([3], geom.variance().get_shape()) - self.assertAllClose(expected_vars, geom.variance().eval()) - - def testGeometricStddev(self): - with self.cached_session(): - probs_v = np.array([.1, .3, .25]) - geom = geometric.Geometric(probs=probs_v) - expected_stddevs = stats.geom.std(probs_v, loc=-1) - self.assertEqual([3], geom.stddev().get_shape()) - self.assertAllClose(geom.stddev().eval(), expected_stddevs) - - def testGeometricMode(self): - with self.cached_session(): - probs_v = np.array([.1, .3, .25]) - geom = geometric.Geometric(probs=probs_v) - self.assertEqual([3,], geom.mode().get_shape()) - self.assertAllClose([0.] * 3, geom.mode().eval()) - - def testGeometricSample(self): - with self.cached_session(): - probs_v = [.3, .9] - probs = constant_op.constant(probs_v) - n = constant_op.constant(100000) - geom = geometric.Geometric(probs=probs) - - samples = geom.sample(n, seed=12345) - self.assertEqual([100000, 2], samples.get_shape()) - - sample_values = samples.eval() - self.assertFalse(np.any(sample_values < 0.0)) - for i in range(2): - self.assertAllClose(sample_values[:, i].mean(), - stats.geom.mean(probs_v[i], loc=-1), - rtol=.02) - self.assertAllClose(sample_values[:, i].var(), - stats.geom.var(probs_v[i], loc=-1), - rtol=.02) - - def testGeometricSampleMultiDimensional(self): - with self.cached_session(): - batch_size = 2 - probs_v = [.3, .9] - probs = constant_op.constant([probs_v] * batch_size) - - geom = geometric.Geometric(probs=probs) - - n = 400000 - samples = geom.sample(n, seed=12345) - self.assertEqual([n, batch_size, 2], samples.get_shape()) - - sample_values = samples.eval() - - self.assertFalse(np.any(sample_values < 0.0)) - for i in range(2): - self.assertAllClose(sample_values[:, 0, i].mean(), - stats.geom.mean(probs_v[i], loc=-1), - rtol=.02) - self.assertAllClose(sample_values[:, 0, i].var(), - stats.geom.var(probs_v[i], loc=-1), - rtol=.02) - self.assertAllClose(sample_values[:, 1, i].mean(), - stats.geom.mean(probs_v[i], loc=-1), - rtol=.02) - self.assertAllClose(sample_values[:, 1, i].var(), - stats.geom.var(probs_v[i], loc=-1), - rtol=.02) - - def testGeometricAtBoundary(self): - with self.cached_session(): - geom = geometric.Geometric(probs=1., validate_args=True) - - x = np.array([0., 2., 3., 4., 5., 6., 7.], dtype=np.float32) - expected_log_prob = stats.geom.logpmf(x, [1.], loc=-1) - # Scipy incorrectly returns nan. - expected_log_prob[np.isnan(expected_log_prob)] = 0. - - log_prob = geom.log_prob(x) - self.assertEqual([7,], log_prob.get_shape()) - self.assertAllClose(expected_log_prob, log_prob.eval()) - - pmf = geom.prob(x) - self.assertEqual([7,], pmf.get_shape()) - self.assertAllClose(np.exp(expected_log_prob), pmf.eval()) - - expected_log_cdf = stats.geom.logcdf(x, 1., loc=-1) - - log_cdf = geom.log_cdf(x) - self.assertEqual([7,], log_cdf.get_shape()) - self.assertAllClose(expected_log_cdf, log_cdf.eval()) - - cdf = geom.cdf(x) - self.assertEqual([7,], cdf.get_shape()) - self.assertAllClose(np.exp(expected_log_cdf), cdf.eval()) - - expected_mean = stats.geom.mean(1., loc=-1) - self.assertEqual([], geom.mean().get_shape()) - self.assertAllClose(expected_mean, geom.mean().eval()) - - expected_variance = stats.geom.var(1., loc=-1) - self.assertEqual([], geom.variance().get_shape()) - self.assertAllClose(expected_variance, geom.variance().eval()) - - with self.assertRaisesOpError("Entropy is undefined"): - geom.entropy().eval() - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/half_normal_test.py b/tensorflow/contrib/distributions/python/kernel_tests/half_normal_test.py deleted file mode 100644 index 3ed96e6fdb8..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/half_normal_test.py +++ /dev/null @@ -1,323 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for initializers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import importlib -import numpy as np - -from tensorflow.contrib.distributions.python.ops import half_normal as hn_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging - - -def try_import(name): # pylint: disable=invalid-name - module = None - try: - module = importlib.import_module(name) - except ImportError as e: - tf_logging.warning("Could not import %s: %s" % (name, str(e))) - return module - - -stats = try_import("scipy.stats") - - -class HalfNormalTest(test.TestCase): - - def setUp(self): - self._rng = np.random.RandomState(123) - - def assertAllFinite(self, tensor): - is_finite = np.isfinite(tensor.eval()) - all_true = np.ones_like(is_finite, dtype=np.bool) - self.assertAllEqual(all_true, is_finite) - - def _testParamShapes(self, sample_shape, expected): - with self.cached_session(): - param_shapes = hn_lib.HalfNormal.param_shapes(sample_shape) - scale_shape = param_shapes["scale"] - self.assertAllEqual(expected, scale_shape.eval()) - scale = array_ops.ones(scale_shape) - self.assertAllEqual( - expected, - array_ops.shape(hn_lib.HalfNormal(scale).sample()).eval()) - - def _testParamStaticShapes(self, sample_shape, expected): - param_shapes = hn_lib.HalfNormal.param_static_shapes(sample_shape) - scale_shape = param_shapes["scale"] - self.assertEqual(expected, scale_shape) - - def _testBatchShapes(self, dist, tensor): - self.assertAllEqual(dist.batch_shape_tensor().eval(), tensor.shape) - self.assertAllEqual(dist.batch_shape_tensor().eval(), tensor.eval().shape) - self.assertAllEqual(dist.batch_shape, tensor.shape) - self.assertAllEqual(dist.batch_shape, tensor.eval().shape) - - def testParamShapes(self): - sample_shape = [10, 3, 4] - self._testParamShapes(sample_shape, sample_shape) - self._testParamShapes(constant_op.constant(sample_shape), sample_shape) - - def testParamStaticShapes(self): - sample_shape = [10, 3, 4] - self._testParamStaticShapes(sample_shape, sample_shape) - self._testParamStaticShapes( - tensor_shape.TensorShape(sample_shape), sample_shape) - - def testHalfNormalLogPDF(self): - with self.cached_session(): - batch_size = 6 - scale = constant_op.constant([3.0] * batch_size) - x = np.array([-2.5, 2.5, 4.0, 0.0, -1.0, 2.0], dtype=np.float32) - halfnorm = hn_lib.HalfNormal(scale=scale) - - log_pdf = halfnorm.log_prob(x) - self._testBatchShapes(halfnorm, log_pdf) - - pdf = halfnorm.prob(x) - self._testBatchShapes(halfnorm, pdf) - - if not stats: - return - expected_log_pdf = stats.halfnorm(scale=scale.eval()).logpdf(x) - self.assertAllClose(expected_log_pdf, log_pdf.eval()) - self.assertAllClose(np.exp(expected_log_pdf), pdf.eval()) - - def testHalfNormalLogPDFMultidimensional(self): - with self.cached_session(): - batch_size = 6 - scale = constant_op.constant([[3.0, 1.0]] * batch_size) - x = np.array([[-2.5, 2.5, 4.0, 0.0, -1.0, 2.0]], dtype=np.float32).T - halfnorm = hn_lib.HalfNormal(scale=scale) - - log_pdf = halfnorm.log_prob(x) - self._testBatchShapes(halfnorm, log_pdf) - - pdf = halfnorm.prob(x) - self._testBatchShapes(halfnorm, pdf) - - if not stats: - return - expected_log_pdf = stats.halfnorm(scale=scale.eval()).logpdf(x) - self.assertAllClose(expected_log_pdf, log_pdf.eval()) - self.assertAllClose(np.exp(expected_log_pdf), pdf.eval()) - - def testHalfNormalCDF(self): - with self.cached_session(): - batch_size = 50 - scale = self._rng.rand(batch_size) + 1.0 - x = np.linspace(-8.0, 8.0, batch_size).astype(np.float64) - halfnorm = hn_lib.HalfNormal(scale=scale) - - cdf = halfnorm.cdf(x) - self._testBatchShapes(halfnorm, cdf) - - log_cdf = halfnorm.log_cdf(x) - self._testBatchShapes(halfnorm, log_cdf) - - if not stats: - return - expected_logcdf = stats.halfnorm(scale=scale).logcdf(x) - self.assertAllClose(expected_logcdf, log_cdf.eval(), atol=0) - self.assertAllClose(np.exp(expected_logcdf), cdf.eval(), atol=0) - - def testHalfNormalSurvivalFunction(self): - with self.cached_session(): - batch_size = 50 - scale = self._rng.rand(batch_size) + 1.0 - x = np.linspace(-8.0, 8.0, batch_size).astype(np.float64) - halfnorm = hn_lib.HalfNormal(scale=scale) - - sf = halfnorm.survival_function(x) - self._testBatchShapes(halfnorm, sf) - - log_sf = halfnorm.log_survival_function(x) - self._testBatchShapes(halfnorm, log_sf) - - if not stats: - return - expected_logsf = stats.halfnorm(scale=scale).logsf(x) - self.assertAllClose(expected_logsf, log_sf.eval(), atol=0) - self.assertAllClose(np.exp(expected_logsf), sf.eval(), atol=0) - - def testHalfNormalQuantile(self): - with self.cached_session(): - batch_size = 50 - scale = self._rng.rand(batch_size) + 1.0 - p = np.linspace(0., 1.0, batch_size).astype(np.float64) - - halfnorm = hn_lib.HalfNormal(scale=scale) - x = halfnorm.quantile(p) - self._testBatchShapes(halfnorm, x) - - if not stats: - return - expected_x = stats.halfnorm(scale=scale).ppf(p) - self.assertAllClose(expected_x, x.eval(), atol=0) - - def testFiniteGradients(self): - for dtype in [np.float32, np.float64]: - g = ops.Graph() - with g.as_default(): - scale = variables.Variable(dtype(3.0)) - dist = hn_lib.HalfNormal(scale=scale) - x = np.array([0.01, 0.1, 1., 5., 10.]).astype(dtype) - for func in [ - dist.cdf, dist.log_cdf, dist.survival_function, - dist.log_prob, dist.prob, dist.log_survival_function, - ]: - print(func.__name__) - value = func(x) - grads = gradients_impl.gradients(value, [scale]) - with self.session(graph=g): - variables.global_variables_initializer().run() - self.assertAllFinite(value) - self.assertAllFinite(grads[0]) - - def testHalfNormalEntropy(self): - with self.cached_session(): - scale = np.array([[1.0, 2.0, 3.0]]) - halfnorm = hn_lib.HalfNormal(scale=scale) - - # See https://en.wikipedia.org/wiki/Half-normal_distribution for the - # entropy formula used here. - expected_entropy = 0.5 * np.log(np.pi * scale ** 2.0 / 2.0) + 0.5 - - entropy = halfnorm.entropy() - self._testBatchShapes(halfnorm, entropy) - self.assertAllClose(expected_entropy, entropy.eval()) - - def testHalfNormalMeanAndMode(self): - with self.cached_session(): - scale = np.array([11., 12., 13.]) - - halfnorm = hn_lib.HalfNormal(scale=scale) - expected_mean = scale * np.sqrt(2.0) / np.sqrt(np.pi) - - self.assertAllEqual((3,), halfnorm.mean().eval().shape) - self.assertAllEqual(expected_mean, halfnorm.mean().eval()) - - self.assertAllEqual((3,), halfnorm.mode().eval().shape) - self.assertAllEqual([0., 0., 0.], halfnorm.mode().eval()) - - def testHalfNormalVariance(self): - with self.cached_session(): - scale = np.array([7., 7., 7.]) - halfnorm = hn_lib.HalfNormal(scale=scale) - expected_variance = scale ** 2.0 * (1.0 - 2.0 / np.pi) - - self.assertAllEqual((3,), halfnorm.variance().eval().shape) - self.assertAllEqual(expected_variance, halfnorm.variance().eval()) - - def testHalfNormalStandardDeviation(self): - with self.cached_session(): - scale = np.array([7., 7., 7.]) - halfnorm = hn_lib.HalfNormal(scale=scale) - expected_variance = scale ** 2.0 * (1.0 - 2.0 / np.pi) - - self.assertAllEqual((3,), halfnorm.stddev().shape) - self.assertAllEqual(np.sqrt(expected_variance), halfnorm.stddev().eval()) - - def testHalfNormalSample(self): - with self.cached_session(): - scale = constant_op.constant(3.0) - n = constant_op.constant(100000) - halfnorm = hn_lib.HalfNormal(scale=scale) - - sample = halfnorm.sample(n) - - self.assertEqual(sample.eval().shape, (100000,)) - self.assertAllClose(sample.eval().mean(), - 3.0 * np.sqrt(2.0) / np.sqrt(np.pi), atol=1e-1) - - expected_shape = tensor_shape.TensorShape([n.eval()]).concatenate( - tensor_shape.TensorShape(halfnorm.batch_shape_tensor().eval())) - self.assertAllEqual(expected_shape, sample.shape) - self.assertAllEqual(expected_shape, sample.eval().shape) - - expected_shape_static = (tensor_shape.TensorShape( - [n.eval()]).concatenate(halfnorm.batch_shape)) - self.assertAllEqual(expected_shape_static, sample.shape) - self.assertAllEqual(expected_shape_static, sample.eval().shape) - - def testHalfNormalSampleMultiDimensional(self): - with self.cached_session(): - batch_size = 2 - scale = constant_op.constant([[2.0, 3.0]] * batch_size) - n = constant_op.constant(100000) - halfnorm = hn_lib.HalfNormal(scale=scale) - - sample = halfnorm.sample(n) - self.assertEqual(sample.shape, (100000, batch_size, 2)) - self.assertAllClose(sample.eval()[:, 0, 0].mean(), - 2.0 * np.sqrt(2.0) / np.sqrt(np.pi), atol=1e-1) - self.assertAllClose(sample.eval()[:, 0, 1].mean(), - 3.0 * np.sqrt(2.0) / np.sqrt(np.pi), atol=1e-1) - - expected_shape = tensor_shape.TensorShape([n.eval()]).concatenate( - tensor_shape.TensorShape(halfnorm.batch_shape_tensor().eval())) - self.assertAllEqual(expected_shape, sample.shape) - self.assertAllEqual(expected_shape, sample.eval().shape) - - expected_shape_static = (tensor_shape.TensorShape( - [n.eval()]).concatenate(halfnorm.batch_shape)) - self.assertAllEqual(expected_shape_static, sample.shape) - self.assertAllEqual(expected_shape_static, sample.eval().shape) - - def testNegativeSigmaFails(self): - with self.cached_session(): - with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - "Condition x > 0 did not hold"): - _ = hn_lib.HalfNormal(scale=[-5.], validate_args=True, name="G") - # Error detected statically; no need for _.mean().eval() - - def testHalfNormalShape(self): - with self.cached_session(): - scale = constant_op.constant([6.0] * 5) - halfnorm = hn_lib.HalfNormal(scale=scale) - - self.assertEqual(halfnorm.batch_shape_tensor().eval(), [5]) - self.assertEqual(halfnorm.batch_shape, tensor_shape.TensorShape([5])) - self.assertAllEqual(halfnorm.event_shape_tensor().eval(), []) - self.assertEqual(halfnorm.event_shape, tensor_shape.TensorShape([])) - - def testHalfNormalShapeWithPlaceholders(self): - scale = array_ops.placeholder(dtype=dtypes.float32) - halfnorm = hn_lib.HalfNormal(scale=scale) - - with self.cached_session() as sess: - # get_batch_shape should return an "" tensor. - self.assertEqual(halfnorm.batch_shape, tensor_shape.TensorShape(None)) - self.assertEqual(halfnorm.event_shape, ()) - self.assertAllEqual(halfnorm.event_shape_tensor().eval(), []) - self.assertAllEqual( - sess.run(halfnorm.batch_shape_tensor(), - feed_dict={scale: [1.0, 2.0]}), [2]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py b/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py deleted file mode 100644 index d8f629945e0..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py +++ /dev/null @@ -1,277 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the Independent distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import importlib -import numpy as np - -from tensorflow.contrib.distributions.python.ops import independent as independent_lib -from tensorflow.contrib.distributions.python.ops import mvn_diag as mvn_diag_lib -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bernoulli as bernoulli_lib -from tensorflow.python.ops.distributions import kullback_leibler -from tensorflow.python.ops.distributions import normal as normal_lib -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging - - -def try_import(name): # pylint: disable=invalid-name - module = None - try: - module = importlib.import_module(name) - except ImportError as e: - tf_logging.warning("Could not import %s: %s" % (name, str(e))) - return module - -stats = try_import("scipy.stats") - - -class ProductDistributionTest(test.TestCase): - - def setUp(self): - self._rng = np.random.RandomState(42) - - def testSampleAndLogProbUnivariate(self): - loc = np.float32([-1., 1]) - scale = np.float32([0.1, 0.5]) - with self.cached_session() as sess: - ind = independent_lib.Independent( - distribution=normal_lib.Normal(loc=loc, scale=scale), - reinterpreted_batch_ndims=1) - - x = ind.sample([4, 5], seed=42) - log_prob_x = ind.log_prob(x) - x_, actual_log_prob_x = sess.run([x, log_prob_x]) - - self.assertEqual([], ind.batch_shape) - self.assertEqual([2], ind.event_shape) - self.assertEqual([4, 5, 2], x.shape) - self.assertEqual([4, 5], log_prob_x.shape) - - expected_log_prob_x = stats.norm(loc, scale).logpdf(x_).sum(-1) - self.assertAllCloseAccordingToType(expected_log_prob_x, actual_log_prob_x) - - def testSampleAndLogProbMultivariate(self): - loc = np.float32([[-1., 1], [1, -1]]) - scale = np.float32([1., 0.5]) - with self.cached_session() as sess: - ind = independent_lib.Independent( - distribution=mvn_diag_lib.MultivariateNormalDiag( - loc=loc, - scale_identity_multiplier=scale), - reinterpreted_batch_ndims=1) - - x = ind.sample([4, 5], seed=42) - log_prob_x = ind.log_prob(x) - x_, actual_log_prob_x = sess.run([x, log_prob_x]) - - self.assertEqual([], ind.batch_shape) - self.assertEqual([2, 2], ind.event_shape) - self.assertEqual([4, 5, 2, 2], x.shape) - self.assertEqual([4, 5], log_prob_x.shape) - - expected_log_prob_x = stats.norm(loc, scale[:, None]).logpdf( - x_).sum(-1).sum(-1) - self.assertAllCloseAccordingToType(expected_log_prob_x, actual_log_prob_x) - - def testSampleConsistentStats(self): - loc = np.float32([[-1., 1], [1, -1]]) - scale = np.float32([1., 0.5]) - n_samp = 1e4 - with self.cached_session() as sess: - ind = independent_lib.Independent( - distribution=mvn_diag_lib.MultivariateNormalDiag( - loc=loc, - scale_identity_multiplier=scale), - reinterpreted_batch_ndims=1) - - x = ind.sample(int(n_samp), seed=42) - sample_mean = math_ops.reduce_mean(x, axis=0) - sample_var = math_ops.reduce_mean( - math_ops.squared_difference(x, sample_mean), axis=0) - sample_std = math_ops.sqrt(sample_var) - sample_entropy = -math_ops.reduce_mean(ind.log_prob(x), axis=0) - - [ - sample_mean_, sample_var_, sample_std_, sample_entropy_, - actual_mean_, actual_var_, actual_std_, actual_entropy_, - actual_mode_, - ] = sess.run([ - sample_mean, sample_var, sample_std, sample_entropy, - ind.mean(), ind.variance(), ind.stddev(), ind.entropy(), ind.mode(), - ]) - - self.assertAllCloseAccordingToType(sample_mean_, actual_mean_, rtol=0.02) - self.assertAllCloseAccordingToType(sample_var_, actual_var_, rtol=0.04) - self.assertAllCloseAccordingToType(sample_std_, actual_std_, rtol=0.02) - self.assertAllCloseAccordingToType( - sample_entropy_, actual_entropy_, rtol=0.01) - self.assertAllCloseAccordingToType(loc, actual_mode_, rtol=1e-6) - - def testKLRaises(self): - ind1 = independent_lib.Independent( - distribution=normal_lib.Normal( - loc=np.float32([-1., 1]), - scale=np.float32([0.1, 0.5])), - reinterpreted_batch_ndims=1) - ind2 = independent_lib.Independent( - distribution=normal_lib.Normal( - loc=np.float32(-1), - scale=np.float32(0.5)), - reinterpreted_batch_ndims=0) - - with self.assertRaisesRegexp( - ValueError, "Event shapes do not match"): - kullback_leibler.kl_divergence(ind1, ind2) - - ind1 = independent_lib.Independent( - distribution=normal_lib.Normal( - loc=np.float32([-1., 1]), - scale=np.float32([0.1, 0.5])), - reinterpreted_batch_ndims=1) - ind2 = independent_lib.Independent( - distribution=mvn_diag_lib.MultivariateNormalDiag( - loc=np.float32([-1., 1]), - scale_diag=np.float32([0.1, 0.5])), - reinterpreted_batch_ndims=0) - - with self.assertRaisesRegexp( - NotImplementedError, "different event shapes"): - kullback_leibler.kl_divergence(ind1, ind2) - - def testKLScalarToMultivariate(self): - normal1 = normal_lib.Normal( - loc=np.float32([-1., 1]), - scale=np.float32([0.1, 0.5])) - ind1 = independent_lib.Independent( - distribution=normal1, reinterpreted_batch_ndims=1) - - normal2 = normal_lib.Normal( - loc=np.float32([-3., 3]), - scale=np.float32([0.3, 0.3])) - ind2 = independent_lib.Independent( - distribution=normal2, reinterpreted_batch_ndims=1) - - normal_kl = kullback_leibler.kl_divergence(normal1, normal2) - ind_kl = kullback_leibler.kl_divergence(ind1, ind2) - self.assertAllCloseAccordingToType( - self.evaluate(math_ops.reduce_sum(normal_kl, axis=-1)), - self.evaluate(ind_kl)) - - def testKLIdentity(self): - normal1 = normal_lib.Normal( - loc=np.float32([-1., 1]), - scale=np.float32([0.1, 0.5])) - # This is functionally just a wrapper around normal1, - # and doesn't change any outputs. - ind1 = independent_lib.Independent( - distribution=normal1, reinterpreted_batch_ndims=0) - - normal2 = normal_lib.Normal( - loc=np.float32([-3., 3]), - scale=np.float32([0.3, 0.3])) - # This is functionally just a wrapper around normal2, - # and doesn't change any outputs. - ind2 = independent_lib.Independent( - distribution=normal2, reinterpreted_batch_ndims=0) - - normal_kl = kullback_leibler.kl_divergence(normal1, normal2) - ind_kl = kullback_leibler.kl_divergence(ind1, ind2) - self.assertAllCloseAccordingToType( - self.evaluate(normal_kl), self.evaluate(ind_kl)) - - def testKLMultivariateToMultivariate(self): - # (1, 1, 2) batch of MVNDiag - mvn1 = mvn_diag_lib.MultivariateNormalDiag( - loc=np.float32([[[[-1., 1, 3.], [2., 4., 3.]]]]), - scale_diag=np.float32([[[0.2, 0.1, 5.], [2., 3., 4.]]])) - ind1 = independent_lib.Independent( - distribution=mvn1, reinterpreted_batch_ndims=2) - - # (1, 1, 2) batch of MVNDiag - mvn2 = mvn_diag_lib.MultivariateNormalDiag( - loc=np.float32([[[[-2., 3, 2.], [1., 3., 2.]]]]), - scale_diag=np.float32([[[0.1, 0.5, 3.], [1., 2., 1.]]])) - - ind2 = independent_lib.Independent( - distribution=mvn2, reinterpreted_batch_ndims=2) - - mvn_kl = kullback_leibler.kl_divergence(mvn1, mvn2) - ind_kl = kullback_leibler.kl_divergence(ind1, ind2) - self.assertAllCloseAccordingToType( - self.evaluate(math_ops.reduce_sum(mvn_kl, axis=[-1, -2])), - self.evaluate(ind_kl)) - - def _testMnistLike(self, static_shape): - sample_shape = [4, 5] - batch_shape = [10] - image_shape = [28, 28, 1] - logits = 3 * self._rng.random_sample( - batch_shape + image_shape).astype(np.float32) - 1 - - def expected_log_prob(x, logits): - return (x * logits - np.log1p(np.exp(logits))).sum(-1).sum(-1).sum(-1) - - with self.cached_session() as sess: - logits_ph = array_ops.placeholder( - dtypes.float32, shape=logits.shape if static_shape else None) - ind = independent_lib.Independent( - distribution=bernoulli_lib.Bernoulli(logits=logits_ph)) - x = ind.sample(sample_shape, seed=42) - log_prob_x = ind.log_prob(x) - [ - x_, - actual_log_prob_x, - ind_batch_shape, - ind_event_shape, - x_shape, - log_prob_x_shape, - ] = sess.run([ - x, - log_prob_x, - ind.batch_shape_tensor(), - ind.event_shape_tensor(), - array_ops.shape(x), - array_ops.shape(log_prob_x), - ], feed_dict={logits_ph: logits}) - - if static_shape: - ind_batch_shape = ind.batch_shape - ind_event_shape = ind.event_shape - x_shape = x.shape - log_prob_x_shape = log_prob_x.shape - - self.assertAllEqual(batch_shape, ind_batch_shape) - self.assertAllEqual(image_shape, ind_event_shape) - self.assertAllEqual(sample_shape + batch_shape + image_shape, x_shape) - self.assertAllEqual(sample_shape + batch_shape, log_prob_x_shape) - self.assertAllCloseAccordingToType( - expected_log_prob(x_, logits), actual_log_prob_x) - - def testMnistLikeStaticShape(self): - self._testMnistLike(static_shape=True) - - def testMnistLikeDynamicShape(self): - self._testMnistLike(static_shape=False) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/inverse_gamma_test.py b/tensorflow/contrib/distributions/python/kernel_tests/inverse_gamma_test.py deleted file mode 100644 index 7c46674cc04..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/inverse_gamma_test.py +++ /dev/null @@ -1,326 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import stats -from tensorflow.contrib.distributions.python.ops import inverse_gamma -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import errors -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import nn_ops -from tensorflow.python.platform import test - - -class InverseGammaTest(test.TestCase): - - def testInverseGammaShape(self): - with self.cached_session(): - alpha = constant_op.constant([3.0] * 5) - beta = constant_op.constant(11.0) - inv_gamma = inverse_gamma.InverseGamma(concentration=alpha, rate=beta) - - self.assertEqual(inv_gamma.batch_shape_tensor().eval(), (5,)) - self.assertEqual(inv_gamma.batch_shape, - tensor_shape.TensorShape([5])) - self.assertAllEqual(inv_gamma.event_shape_tensor().eval(), []) - self.assertEqual(inv_gamma.event_shape, tensor_shape.TensorShape( - [])) - - def testInverseGammaLogPDF(self): - with self.cached_session(): - batch_size = 6 - alpha = constant_op.constant([2.0] * batch_size) - beta = constant_op.constant([3.0] * batch_size) - alpha_v = 2.0 - beta_v = 3.0 - x = np.array([2.5, 2.5, 4.0, 0.1, 1.0, 2.0], dtype=np.float32) - inv_gamma = inverse_gamma.InverseGamma(concentration=alpha, rate=beta) - expected_log_pdf = stats.invgamma.logpdf(x, alpha_v, scale=beta_v) - log_pdf = inv_gamma.log_prob(x) - self.assertEqual(log_pdf.get_shape(), (6,)) - self.assertAllClose(log_pdf.eval(), expected_log_pdf) - - pdf = inv_gamma.prob(x) - self.assertEqual(pdf.get_shape(), (6,)) - self.assertAllClose(pdf.eval(), np.exp(expected_log_pdf)) - - def testInverseGammaLogPDFMultidimensional(self): - with self.cached_session(): - batch_size = 6 - alpha = constant_op.constant([[2.0, 4.0]] * batch_size) - beta = constant_op.constant([[3.0, 4.0]] * batch_size) - alpha_v = np.array([2.0, 4.0]) - beta_v = np.array([3.0, 4.0]) - x = np.array([[2.5, 2.5, 4.0, 0.1, 1.0, 2.0]], dtype=np.float32).T - inv_gamma = inverse_gamma.InverseGamma(concentration=alpha, rate=beta) - expected_log_pdf = stats.invgamma.logpdf(x, alpha_v, scale=beta_v) - log_pdf = inv_gamma.log_prob(x) - log_pdf_values = log_pdf.eval() - self.assertEqual(log_pdf.get_shape(), (6, 2)) - self.assertAllClose(log_pdf_values, expected_log_pdf) - - pdf = inv_gamma.prob(x) - pdf_values = pdf.eval() - self.assertEqual(pdf.get_shape(), (6, 2)) - self.assertAllClose(pdf_values, np.exp(expected_log_pdf)) - - def testInverseGammaLogPDFMultidimensionalBroadcasting(self): - with self.cached_session(): - batch_size = 6 - alpha = constant_op.constant([[2.0, 4.0]] * batch_size) - beta = constant_op.constant(3.0) - alpha_v = np.array([2.0, 4.0]) - beta_v = 3.0 - x = np.array([[2.5, 2.5, 4.0, 0.1, 1.0, 2.0]], dtype=np.float32).T - inv_gamma = inverse_gamma.InverseGamma(concentration=alpha, rate=beta) - expected_log_pdf = stats.invgamma.logpdf(x, alpha_v, scale=beta_v) - log_pdf = inv_gamma.log_prob(x) - log_pdf_values = log_pdf.eval() - self.assertEqual(log_pdf.get_shape(), (6, 2)) - self.assertAllClose(log_pdf_values, expected_log_pdf) - - pdf = inv_gamma.prob(x) - pdf_values = pdf.eval() - self.assertEqual(pdf.get_shape(), (6, 2)) - self.assertAllClose(pdf_values, np.exp(expected_log_pdf)) - - def testInverseGammaCDF(self): - with self.cached_session(): - batch_size = 6 - alpha_v = 2.0 - beta_v = 3.0 - alpha = constant_op.constant([alpha_v] * batch_size) - beta = constant_op.constant([beta_v] * batch_size) - x = np.array([2.5, 2.5, 4.0, 0.1, 1.0, 2.0], dtype=np.float32) - - inv_gamma = inverse_gamma.InverseGamma(concentration=alpha, rate=beta) - expected_cdf = stats.invgamma.cdf(x, alpha_v, scale=beta_v) - - cdf = inv_gamma.cdf(x) - self.assertEqual(cdf.get_shape(), (batch_size,)) - self.assertAllClose(cdf.eval(), expected_cdf) - - def testInverseGammaMode(self): - with self.cached_session(): - alpha_v = np.array([5.5, 3.0, 2.5]) - beta_v = np.array([1.0, 4.0, 5.0]) - inv_gamma = inverse_gamma.InverseGamma(concentration=alpha_v, rate=beta_v) - expected_modes = beta_v / (alpha_v + 1) - self.assertEqual(inv_gamma.mode().get_shape(), (3,)) - self.assertAllClose(inv_gamma.mode().eval(), expected_modes) - - def testInverseGammaMeanAllDefined(self): - with self.cached_session(): - alpha_v = np.array([5.5, 3.0, 2.5]) - beta_v = np.array([1.0, 4.0, 5.0]) - inv_gamma = inverse_gamma.InverseGamma(concentration=alpha_v, rate=beta_v) - expected_means = stats.invgamma.mean(alpha_v, scale=beta_v) - self.assertEqual(inv_gamma.mean().get_shape(), (3,)) - self.assertAllClose(inv_gamma.mean().eval(), expected_means) - - def testInverseGammaMeanAllowNanStats(self): - with self.cached_session(): - # Mean will not be defined for the first entry. - alpha_v = np.array([1.0, 3.0, 2.5]) - beta_v = np.array([1.0, 4.0, 5.0]) - inv_gamma = inverse_gamma.InverseGamma( - concentration=alpha_v, rate=beta_v, allow_nan_stats=False) - with self.assertRaisesOpError("x < y"): - inv_gamma.mean().eval() - - def testInverseGammaMeanNanStats(self): - with self.cached_session(): - # Mode will not be defined for the first two entries. - alpha_v = np.array([0.5, 1.0, 3.0, 2.5]) - beta_v = np.array([1.0, 2.0, 4.0, 5.0]) - inv_gamma = inverse_gamma.InverseGamma( - concentration=alpha_v, rate=beta_v, allow_nan_stats=True) - expected_means = beta_v / (alpha_v - 1) - expected_means[0] = np.nan - expected_means[1] = np.nan - self.assertEqual(inv_gamma.mean().get_shape(), (4,)) - self.assertAllClose(inv_gamma.mean().eval(), expected_means) - - def testInverseGammaVarianceAllDefined(self): - with self.cached_session(): - alpha_v = np.array([7.0, 3.0, 2.5]) - beta_v = np.array([1.0, 4.0, 5.0]) - inv_gamma = inverse_gamma.InverseGamma(concentration=alpha_v, rate=beta_v) - expected_variances = stats.invgamma.var(alpha_v, scale=beta_v) - self.assertEqual(inv_gamma.variance().get_shape(), (3,)) - self.assertAllClose(inv_gamma.variance().eval(), expected_variances) - - def testInverseGammaVarianceAllowNanStats(self): - with self.cached_session(): - alpha_v = np.array([1.5, 3.0, 2.5]) - beta_v = np.array([1.0, 4.0, 5.0]) - inv_gamma = inverse_gamma.InverseGamma( - concentration=alpha_v, rate=beta_v, allow_nan_stats=False) - with self.assertRaisesOpError("x < y"): - inv_gamma.variance().eval() - - def testInverseGammaVarianceNanStats(self): - with self.cached_session(): - alpha_v = np.array([1.5, 3.0, 2.5]) - beta_v = np.array([1.0, 4.0, 5.0]) - inv_gamma = inverse_gamma.InverseGamma( - concentration=alpha_v, rate=beta_v, allow_nan_stats=True) - expected_variances = stats.invgamma.var(alpha_v, scale=beta_v) - expected_variances[0] = np.nan - self.assertEqual(inv_gamma.variance().get_shape(), (3,)) - self.assertAllClose(inv_gamma.variance().eval(), expected_variances) - - def testInverseGammaEntropy(self): - with self.cached_session(): - alpha_v = np.array([1.0, 3.0, 2.5]) - beta_v = np.array([1.0, 4.0, 5.0]) - expected_entropy = stats.invgamma.entropy(alpha_v, scale=beta_v) - inv_gamma = inverse_gamma.InverseGamma(concentration=alpha_v, rate=beta_v) - self.assertEqual(inv_gamma.entropy().get_shape(), (3,)) - self.assertAllClose(inv_gamma.entropy().eval(), expected_entropy) - - def testInverseGammaSample(self): - with session.Session(): - alpha_v = 4.0 - beta_v = 3.0 - alpha = constant_op.constant(alpha_v) - beta = constant_op.constant(beta_v) - n = 100000 - inv_gamma = inverse_gamma.InverseGamma(concentration=alpha, rate=beta) - samples = inv_gamma.sample(n, seed=137) - sample_values = samples.eval() - self.assertEqual(samples.get_shape(), (n,)) - self.assertEqual(sample_values.shape, (n,)) - self.assertAllClose( - sample_values.mean(), - stats.invgamma.mean( - alpha_v, scale=beta_v), - atol=.0025) - self.assertAllClose( - sample_values.var(), - stats.invgamma.var(alpha_v, scale=beta_v), - atol=.15) - self.assertTrue(self._kstest(alpha_v, beta_v, sample_values)) - - def testInverseGammaSampleMultiDimensional(self): - with session.Session(): - alpha_v = np.array([np.arange(3, 103, dtype=np.float32)]) # 1 x 100 - beta_v = np.array([np.arange(1, 11, dtype=np.float32)]).T # 10 x 1 - inv_gamma = inverse_gamma.InverseGamma(concentration=alpha_v, rate=beta_v) - n = 10000 - samples = inv_gamma.sample(n, seed=137) - sample_values = samples.eval() - self.assertEqual(samples.get_shape(), (n, 10, 100)) - self.assertEqual(sample_values.shape, (n, 10, 100)) - zeros = np.zeros_like(alpha_v + beta_v) # 10 x 100 - alpha_bc = alpha_v + zeros - beta_bc = beta_v + zeros - self.assertAllClose( - sample_values.mean(axis=0), - stats.invgamma.mean( - alpha_bc, scale=beta_bc), - atol=.25) - self.assertAllClose( - sample_values.var(axis=0), - stats.invgamma.var(alpha_bc, scale=beta_bc), - atol=4.5) - fails = 0 - trials = 0 - for ai, a in enumerate(np.reshape(alpha_v, [-1])): - for bi, b in enumerate(np.reshape(beta_v, [-1])): - s = sample_values[:, bi, ai] - trials += 1 - fails += 0 if self._kstest(a, b, s) else 1 - self.assertLess(fails, trials * 0.03) - - @staticmethod - def _kstest(alpha, beta, samples): - # Uses the Kolmogorov-Smirnov test for goodness of fit. - ks, _ = stats.kstest(samples, stats.invgamma(alpha, scale=beta).cdf) - # Return True when the test passes. - return ks < 0.02 - - def testInverseGammaPdfOfSampleMultiDims(self): - with session.Session() as sess: - inv_gamma = inverse_gamma.InverseGamma( - concentration=[7., 11.], - rate=[[5.], [6.]]) - num = 50000 - samples = inv_gamma.sample(num, seed=137) - pdfs = inv_gamma.prob(samples) - sample_vals, pdf_vals = sess.run([samples, pdfs]) - self.assertEqual(samples.get_shape(), (num, 2, 2)) - self.assertEqual(pdfs.get_shape(), (num, 2, 2)) - self.assertAllClose( - stats.invgamma.mean( - [[7., 11.], [7., 11.]], scale=np.array([[5., 5.], [6., 6.]])), - sample_vals.mean(axis=0), - atol=.1) - self.assertAllClose( - stats.invgamma.var([[7., 11.], [7., 11.]], - scale=np.array([[5., 5.], [6., 6.]])), - sample_vals.var(axis=0), - atol=.1) - self._assertIntegral(sample_vals[:, 0, 0], pdf_vals[:, 0, 0], err=0.02) - self._assertIntegral(sample_vals[:, 0, 1], pdf_vals[:, 0, 1], err=0.02) - self._assertIntegral(sample_vals[:, 1, 0], pdf_vals[:, 1, 0], err=0.02) - self._assertIntegral(sample_vals[:, 1, 1], pdf_vals[:, 1, 1], err=0.02) - - def _assertIntegral(self, sample_vals, pdf_vals, err=1e-3): - s_p = zip(sample_vals, pdf_vals) - prev = (0, 0) - total = 0 - for k in sorted(s_p, key=lambda x: x[0]): - pair_pdf = (k[1] + prev[1]) / 2 - total += (k[0] - prev[0]) * pair_pdf - prev = k - self.assertNear(1., total, err=err) - - def testInverseGammaNonPositiveInitializationParamsRaises(self): - with self.cached_session(): - alpha_v = constant_op.constant(0.0, name="alpha") - beta_v = constant_op.constant(1.0, name="beta") - with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - "alpha"): - _ = inverse_gamma.InverseGamma( - concentration=alpha_v, rate=beta_v, validate_args=True) - # Error detected statically; no need for _.mean().eval() - alpha_v = constant_op.constant(1.0, name="alpha") - beta_v = constant_op.constant(0.0, name="beta") - with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - "beta"): - _ = inverse_gamma.InverseGamma( - concentration=alpha_v, rate=beta_v, validate_args=True) - # Error detected statically; no need for _.mean().eval() - - def testInverseGammaWithSoftplusConcentrationRate(self): - with self.cached_session(): - alpha = constant_op.constant([-0.1, -2.9], name="alpha") - beta = constant_op.constant([1.0, -4.8], name="beta") - inv_gamma = inverse_gamma.InverseGammaWithSoftplusConcentrationRate( - concentration=alpha, rate=beta, validate_args=True) - self.assertAllClose(nn_ops.softplus(alpha).eval(), - inv_gamma.concentration.eval()) - self.assertAllClose(nn_ops.softplus(beta).eval(), - inv_gamma.rate.eval()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/kumaraswamy_test.py b/tensorflow/contrib/distributions/python/kernel_tests/kumaraswamy_test.py deleted file mode 100644 index e39db51728d..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/kumaraswamy_test.py +++ /dev/null @@ -1,386 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import importlib - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import kumaraswamy as kumaraswamy_lib -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging - - -def try_import(name): # pylint: disable=invalid-name - module = None - try: - module = importlib.import_module(name) - except ImportError as e: - tf_logging.warning("Could not import %s: %s" % (name, str(e))) - return module - - -special = try_import("scipy.special") -stats = try_import("scipy.stats") - - -def _kumaraswamy_mode(a, b): - a = np.asarray(a) - b = np.asarray(b) - return ((a - 1) / (a * b - 1))**(1 / a) - - -def _kumaraswamy_moment(a, b, n): - a = np.asarray(a) - b = np.asarray(b) - return b * special.beta(1.0 + n / a, b) - - -def _harmonic_number(b): - b = np.asarray(b) - return special.psi(b + 1) - special.psi(1) - - -def _kumaraswamy_cdf(a, b, x): - a = np.asarray(a) - b = np.asarray(b) - x = np.asarray(x) - return 1 - (1 - x**a)**b - - -def _kumaraswamy_pdf(a, b, x): - a = np.asarray(a) - b = np.asarray(b) - x = np.asarray(x) - return a * b * x ** (a - 1) * (1 - x ** a) ** (b - 1) - - -class KumaraswamyTest(test.TestCase): - - def testSimpleShapes(self): - with self.cached_session(): - a = np.random.rand(3) - b = np.random.rand(3) - dist = kumaraswamy_lib.Kumaraswamy(a, b) - self.assertAllEqual([], dist.event_shape_tensor().eval()) - self.assertAllEqual([3], dist.batch_shape_tensor().eval()) - self.assertEqual(tensor_shape.TensorShape([]), dist.event_shape) - self.assertEqual(tensor_shape.TensorShape([3]), dist.batch_shape) - - def testComplexShapes(self): - with self.cached_session(): - a = np.random.rand(3, 2, 2) - b = np.random.rand(3, 2, 2) - dist = kumaraswamy_lib.Kumaraswamy(a, b) - self.assertAllEqual([], dist.event_shape_tensor().eval()) - self.assertAllEqual([3, 2, 2], dist.batch_shape_tensor().eval()) - self.assertEqual(tensor_shape.TensorShape([]), dist.event_shape) - self.assertEqual(tensor_shape.TensorShape([3, 2, 2]), dist.batch_shape) - - def testComplexShapesBroadcast(self): - with self.cached_session(): - a = np.random.rand(3, 2, 2) - b = np.random.rand(2, 2) - dist = kumaraswamy_lib.Kumaraswamy(a, b) - self.assertAllEqual([], dist.event_shape_tensor().eval()) - self.assertAllEqual([3, 2, 2], dist.batch_shape_tensor().eval()) - self.assertEqual(tensor_shape.TensorShape([]), dist.event_shape) - self.assertEqual(tensor_shape.TensorShape([3, 2, 2]), dist.batch_shape) - - def testAProperty(self): - a = [[1., 2, 3]] - b = [[2., 4, 3]] - with self.cached_session(): - dist = kumaraswamy_lib.Kumaraswamy(a, b) - self.assertEqual([1, 3], dist.concentration1.get_shape()) - self.assertAllClose(a, dist.concentration1.eval()) - - def testBProperty(self): - a = [[1., 2, 3]] - b = [[2., 4, 3]] - with self.cached_session(): - dist = kumaraswamy_lib.Kumaraswamy(a, b) - self.assertEqual([1, 3], dist.concentration0.get_shape()) - self.assertAllClose(b, dist.concentration0.eval()) - - def testPdfXProper(self): - a = [[1., 2, 3]] - b = [[2., 4, 3]] - with self.cached_session(): - dist = kumaraswamy_lib.Kumaraswamy(a, b, validate_args=True) - dist.prob([.1, .3, .6]).eval() - dist.prob([.2, .3, .5]).eval() - # Either condition can trigger. - with self.assertRaisesOpError("sample must be non-negative"): - dist.prob([-1., 0.1, 0.5]).eval() - with self.assertRaisesOpError("sample must be no larger than `1`"): - dist.prob([.1, .2, 1.2]).eval() - - def testPdfTwoBatches(self): - with self.cached_session(): - a = [1., 2] - b = [1., 2] - x = [.5, .5] - dist = kumaraswamy_lib.Kumaraswamy(a, b) - pdf = dist.prob(x) - expected_pdf = _kumaraswamy_pdf(a, b, x) - self.assertAllClose(expected_pdf, pdf.eval()) - self.assertEqual((2,), pdf.get_shape()) - - def testPdfTwoBatchesNontrivialX(self): - with self.cached_session(): - a = [1., 2] - b = [1., 2] - x = [.3, .7] - dist = kumaraswamy_lib.Kumaraswamy(a, b) - pdf = dist.prob(x) - expected_pdf = _kumaraswamy_pdf(a, b, x) - self.assertAllClose(expected_pdf, pdf.eval()) - self.assertEqual((2,), pdf.get_shape()) - - def testPdfUniformZeroBatch(self): - with self.cached_session(): - # This is equivalent to a uniform distribution - a = 1. - b = 1. - x = np.array([.1, .2, .3, .5, .8], dtype=np.float32) - dist = kumaraswamy_lib.Kumaraswamy(a, b) - pdf = dist.prob(x) - expected_pdf = _kumaraswamy_pdf(a, b, x) - self.assertAllClose(expected_pdf, pdf.eval()) - self.assertEqual((5,), pdf.get_shape()) - - def testPdfAStretchedInBroadcastWhenSameRank(self): - with self.cached_session(): - a = [[1., 2]] - b = [[1., 2]] - x = [[.5, .5], [.3, .7]] - dist = kumaraswamy_lib.Kumaraswamy(a, b) - pdf = dist.prob(x) - expected_pdf = _kumaraswamy_pdf(a, b, x) - self.assertAllClose(expected_pdf, pdf.eval()) - self.assertEqual((2, 2), pdf.get_shape()) - - def testPdfAStretchedInBroadcastWhenLowerRank(self): - with self.cached_session(): - a = [1., 2] - b = [1., 2] - x = [[.5, .5], [.2, .8]] - pdf = kumaraswamy_lib.Kumaraswamy(a, b).prob(x) - expected_pdf = _kumaraswamy_pdf(a, b, x) - self.assertAllClose(expected_pdf, pdf.eval()) - self.assertEqual((2, 2), pdf.get_shape()) - - def testPdfXStretchedInBroadcastWhenSameRank(self): - with self.cached_session(): - a = [[1., 2], [2., 3]] - b = [[1., 2], [2., 3]] - x = [[.5, .5]] - pdf = kumaraswamy_lib.Kumaraswamy(a, b).prob(x) - expected_pdf = _kumaraswamy_pdf(a, b, x) - self.assertAllClose(expected_pdf, pdf.eval()) - self.assertEqual((2, 2), pdf.get_shape()) - - def testPdfXStretchedInBroadcastWhenLowerRank(self): - with self.cached_session(): - a = [[1., 2], [2., 3]] - b = [[1., 2], [2., 3]] - x = [.5, .5] - pdf = kumaraswamy_lib.Kumaraswamy(a, b).prob(x) - expected_pdf = _kumaraswamy_pdf(a, b, x) - self.assertAllClose(expected_pdf, pdf.eval()) - self.assertEqual((2, 2), pdf.get_shape()) - - def testKumaraswamyMean(self): - with session.Session(): - a = [1., 2, 3] - b = [2., 4, 1.2] - dist = kumaraswamy_lib.Kumaraswamy(a, b) - self.assertEqual(dist.mean().get_shape(), (3,)) - if not stats: - return - expected_mean = _kumaraswamy_moment(a, b, 1) - self.assertAllClose(expected_mean, dist.mean().eval()) - - def testKumaraswamyVariance(self): - with session.Session(): - a = [1., 2, 3] - b = [2., 4, 1.2] - dist = kumaraswamy_lib.Kumaraswamy(a, b) - self.assertEqual(dist.variance().get_shape(), (3,)) - if not stats: - return - expected_variance = _kumaraswamy_moment(a, b, 2) - _kumaraswamy_moment( - a, b, 1)**2 - self.assertAllClose(expected_variance, dist.variance().eval()) - - def testKumaraswamyMode(self): - with session.Session(): - a = np.array([1.1, 2, 3]) - b = np.array([2., 4, 1.2]) - expected_mode = _kumaraswamy_mode(a, b) - dist = kumaraswamy_lib.Kumaraswamy(a, b) - self.assertEqual(dist.mode().get_shape(), (3,)) - self.assertAllClose(expected_mode, dist.mode().eval()) - - def testKumaraswamyModeInvalid(self): - with session.Session(): - a = np.array([1., 2, 3]) - b = np.array([2., 4, 1.2]) - dist = kumaraswamy_lib.Kumaraswamy(a, b, allow_nan_stats=False) - with self.assertRaisesOpError("Mode undefined for concentration1 <= 1."): - dist.mode().eval() - - a = np.array([2., 2, 3]) - b = np.array([1., 4, 1.2]) - dist = kumaraswamy_lib.Kumaraswamy(a, b, allow_nan_stats=False) - with self.assertRaisesOpError("Mode undefined for concentration0 <= 1."): - dist.mode().eval() - - def testKumaraswamyModeEnableAllowNanStats(self): - with session.Session(): - a = np.array([1., 2, 3]) - b = np.array([2., 4, 1.2]) - dist = kumaraswamy_lib.Kumaraswamy(a, b, allow_nan_stats=True) - - expected_mode = _kumaraswamy_mode(a, b) - expected_mode[0] = np.nan - self.assertEqual((3,), dist.mode().get_shape()) - self.assertAllClose(expected_mode, dist.mode().eval()) - - a = np.array([2., 2, 3]) - b = np.array([1., 4, 1.2]) - dist = kumaraswamy_lib.Kumaraswamy(a, b, allow_nan_stats=True) - - expected_mode = _kumaraswamy_mode(a, b) - expected_mode[0] = np.nan - self.assertEqual((3,), dist.mode().get_shape()) - self.assertAllClose(expected_mode, dist.mode().eval()) - - def testKumaraswamyEntropy(self): - with session.Session(): - a = np.array([1., 2, 3]) - b = np.array([2., 4, 1.2]) - dist = kumaraswamy_lib.Kumaraswamy(a, b) - self.assertEqual(dist.entropy().get_shape(), (3,)) - if not stats: - return - expected_entropy = (1 - 1. / a) + ( - 1 - 1. / b) * _harmonic_number(b) + np.log(a * b) - self.assertAllClose(expected_entropy, dist.entropy().eval()) - - def testKumaraswamySample(self): - with self.cached_session(): - a = 1. - b = 2. - kumaraswamy = kumaraswamy_lib.Kumaraswamy(a, b) - n = constant_op.constant(100000) - samples = kumaraswamy.sample(n) - sample_values = samples.eval() - self.assertEqual(sample_values.shape, (100000,)) - self.assertFalse(np.any(sample_values < 0.0)) - if not stats: - return - self.assertLess( - stats.kstest( - # Kumaraswamy is a univariate distribution. - sample_values, - lambda x: _kumaraswamy_cdf(1., 2., x))[0], - 0.01) - # The standard error of the sample mean is 1 / (sqrt(18 * n)) - expected_mean = _kumaraswamy_moment(a, b, 1) - self.assertAllClose(sample_values.mean(axis=0), expected_mean, atol=1e-2) - expected_variance = _kumaraswamy_moment(a, b, 2) - _kumaraswamy_moment( - a, b, 1)**2 - self.assertAllClose( - np.cov(sample_values, rowvar=0), expected_variance, atol=1e-1) - - # Test that sampling with the same seed twice gives the same results. - def testKumaraswamySampleMultipleTimes(self): - with self.cached_session(): - a_val = 1. - b_val = 2. - n_val = 100 - - random_seed.set_random_seed(654321) - kumaraswamy1 = kumaraswamy_lib.Kumaraswamy( - concentration1=a_val, concentration0=b_val, name="kumaraswamy1") - samples1 = kumaraswamy1.sample(n_val, seed=123456).eval() - - random_seed.set_random_seed(654321) - kumaraswamy2 = kumaraswamy_lib.Kumaraswamy( - concentration1=a_val, concentration0=b_val, name="kumaraswamy2") - samples2 = kumaraswamy2.sample(n_val, seed=123456).eval() - - self.assertAllClose(samples1, samples2) - - def testKumaraswamySampleMultidimensional(self): - with self.cached_session(): - a = np.random.rand(3, 2, 2).astype(np.float32) - b = np.random.rand(3, 2, 2).astype(np.float32) - kumaraswamy = kumaraswamy_lib.Kumaraswamy(a, b) - n = constant_op.constant(100000) - samples = kumaraswamy.sample(n) - sample_values = samples.eval() - self.assertEqual(sample_values.shape, (100000, 3, 2, 2)) - self.assertFalse(np.any(sample_values < 0.0)) - if not stats: - return - self.assertAllClose( - sample_values[:, 1, :].mean(axis=0), - _kumaraswamy_moment(a, b, 1)[1, :], - atol=1e-1) - - def testKumaraswamyCdf(self): - with self.cached_session(): - shape = (30, 40, 50) - for dt in (np.float32, np.float64): - a = 10. * np.random.random(shape).astype(dt) - b = 10. * np.random.random(shape).astype(dt) - x = np.random.random(shape).astype(dt) - actual = kumaraswamy_lib.Kumaraswamy(a, b).cdf(x).eval() - self.assertAllEqual(np.ones(shape, dtype=np.bool), 0. <= x) - self.assertAllEqual(np.ones(shape, dtype=np.bool), 1. >= x) - if not stats: - return - self.assertAllClose( - _kumaraswamy_cdf(a, b, x), actual, rtol=1e-4, atol=0) - - def testKumaraswamyLogCdf(self): - with self.cached_session(): - shape = (30, 40, 50) - for dt in (np.float32, np.float64): - a = 10. * np.random.random(shape).astype(dt) - b = 10. * np.random.random(shape).astype(dt) - x = np.random.random(shape).astype(dt) - actual = math_ops.exp(kumaraswamy_lib.Kumaraswamy(a, - b).log_cdf(x)).eval() - self.assertAllEqual(np.ones(shape, dtype=np.bool), 0. <= x) - self.assertAllEqual(np.ones(shape, dtype=np.bool), 1. >= x) - if not stats: - return - self.assertAllClose( - _kumaraswamy_cdf(a, b, x), actual, rtol=1e-4, atol=0) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/logistic_test.py b/tensorflow/contrib/distributions/python/kernel_tests/logistic_test.py deleted file mode 100644 index 12a2d4f8ec9..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/logistic_test.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for initializers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import stats -from tensorflow.contrib.distributions.python.ops import logistic -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.platform import test - - -class LogisticTest(test.TestCase): - - def testReparameterizable(self): - batch_size = 6 - np_loc = np.array([2.0] * batch_size, dtype=np.float32) - loc = constant_op.constant(np_loc) - scale = 1.5 - dist = logistic.Logistic(loc, scale) - self.assertTrue( - dist.reparameterization_type == distribution.FULLY_REPARAMETERIZED) - - def testLogisticLogProb(self): - with self.cached_session(): - batch_size = 6 - np_loc = np.array([2.0] * batch_size, dtype=np.float32) - loc = constant_op.constant(np_loc) - scale = 1.5 - x = np.array([2.5, 2.5, 4.0, 0.1, 1.0, 2.0], dtype=np.float32) - dist = logistic.Logistic(loc, scale) - expected_log_prob = stats.logistic.logpdf(x, np_loc, scale) - - log_prob = dist.log_prob(x) - self.assertEqual(log_prob.get_shape(), (6,)) - self.assertAllClose(log_prob.eval(), expected_log_prob) - - prob = dist.prob(x) - self.assertEqual(prob.get_shape(), (6,)) - self.assertAllClose(prob.eval(), np.exp(expected_log_prob)) - - def testLogisticCDF(self): - with self.cached_session(): - batch_size = 6 - np_loc = np.array([2.0] * batch_size, dtype=np.float32) - loc = constant_op.constant(np_loc) - scale = 1.5 - - dist = logistic.Logistic(loc, scale) - x = np.array([2.5, 2.5, 4.0, 0.1, 1.0, 2.0], dtype=np.float32) - cdf = dist.cdf(x) - expected_cdf = stats.logistic.cdf(x, np_loc, scale) - - self.assertEqual(cdf.get_shape(), (6,)) - self.assertAllClose(cdf.eval(), expected_cdf) - - def testLogisticLogCDF(self): - with self.cached_session(): - batch_size = 6 - np_loc = np.array([2.0] * batch_size, dtype=np.float32) - loc = constant_op.constant(np_loc) - scale = 1.5 - - dist = logistic.Logistic(loc, scale) - x = np.array([2.5, 2.5, 4.0, 0.1, 1.0, 2.0], dtype=np.float32) - logcdf = dist.log_cdf(x) - expected_logcdf = stats.logistic.logcdf(x, np_loc, scale) - - self.assertEqual(logcdf.get_shape(), (6,)) - self.assertAllClose(logcdf.eval(), expected_logcdf) - - def testLogisticSurvivalFunction(self): - with self.cached_session(): - batch_size = 6 - np_loc = np.array([2.0] * batch_size, dtype=np.float32) - loc = constant_op.constant(np_loc) - scale = 1.5 - - dist = logistic.Logistic(loc, scale) - x = np.array([2.5, 2.5, 4.0, 0.1, 1.0, 2.0], dtype=np.float32) - survival_function = dist.survival_function(x) - expected_survival_function = stats.logistic.sf(x, np_loc, scale) - - self.assertEqual(survival_function.get_shape(), (6,)) - self.assertAllClose(survival_function.eval(), expected_survival_function) - - def testLogisticLogSurvivalFunction(self): - with self.cached_session(): - batch_size = 6 - np_loc = np.array([2.0] * batch_size, dtype=np.float32) - loc = constant_op.constant(np_loc) - scale = 1.5 - - dist = logistic.Logistic(loc, scale) - x = np.array([2.5, 2.5, 4.0, 0.1, 1.0, 2.0], dtype=np.float32) - logsurvival_function = dist.log_survival_function(x) - expected_logsurvival_function = stats.logistic.logsf(x, np_loc, scale) - - self.assertEqual(logsurvival_function.get_shape(), (6,)) - self.assertAllClose(logsurvival_function.eval(), - expected_logsurvival_function) - - def testLogisticMean(self): - with self.cached_session(): - loc = [2.0, 1.5, 1.0] - scale = 1.5 - expected_mean = stats.logistic.mean(loc, scale) - dist = logistic.Logistic(loc, scale) - self.assertAllClose(dist.mean().eval(), expected_mean) - - def testLogisticVariance(self): - with self.cached_session(): - loc = [2.0, 1.5, 1.0] - scale = 1.5 - expected_variance = stats.logistic.var(loc, scale) - dist = logistic.Logistic(loc, scale) - self.assertAllClose(dist.variance().eval(), expected_variance) - - def testLogisticEntropy(self): - with self.cached_session(): - batch_size = 3 - np_loc = np.array([2.0] * batch_size, dtype=np.float32) - loc = constant_op.constant(np_loc) - scale = 1.5 - expected_entropy = stats.logistic.entropy(np_loc, scale) - dist = logistic.Logistic(loc, scale) - self.assertAllClose(dist.entropy().eval(), expected_entropy) - - def testLogisticSample(self): - with self.cached_session(): - loc = [3.0, 4.0, 2.0] - scale = 1.0 - dist = logistic.Logistic(loc, scale) - sample = dist.sample(seed=100) - self.assertEqual(sample.get_shape(), (3,)) - self.assertAllClose(sample.eval(), [6.22460556, 3.79602098, 2.05084133]) - - def testDtype(self): - loc = constant_op.constant([0.1, 0.4], dtype=dtypes.float32) - scale = constant_op.constant(1.0, dtype=dtypes.float32) - dist = logistic.Logistic(loc, scale) - self.assertEqual(dist.dtype, dtypes.float32) - self.assertEqual(dist.loc.dtype, dist.scale.dtype) - self.assertEqual(dist.dtype, dist.sample(5).dtype) - self.assertEqual(dist.dtype, dist.mode().dtype) - self.assertEqual(dist.loc.dtype, dist.mean().dtype) - self.assertEqual(dist.loc.dtype, dist.variance().dtype) - self.assertEqual(dist.loc.dtype, dist.stddev().dtype) - self.assertEqual(dist.loc.dtype, dist.entropy().dtype) - self.assertEqual(dist.loc.dtype, dist.prob(0.2).dtype) - self.assertEqual(dist.loc.dtype, dist.log_prob(0.2).dtype) - - loc = constant_op.constant([0.1, 0.4], dtype=dtypes.float64) - scale = constant_op.constant(1.0, dtype=dtypes.float64) - dist64 = logistic.Logistic(loc, scale) - self.assertEqual(dist64.dtype, dtypes.float64) - self.assertEqual(dist64.dtype, dist64.sample(5).dtype) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py deleted file mode 100644 index faff42d2432..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for MixtureSameFamily distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import mixture_same_family as mixture_same_family_lib -from tensorflow.contrib.distributions.python.ops import mvn_diag as mvn_diag_lib -from tensorflow.contrib.distributions.python.ops import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bernoulli as bernoulli_lib -from tensorflow.python.ops.distributions import categorical as categorical_lib -from tensorflow.python.ops.distributions import normal as normal_lib -from tensorflow.python.platform import test - - -class MixtureSameFamilyTest(test_util.VectorDistributionTestHelpers, - test.TestCase): - - def testSampleAndLogProbUnivariateShapes(self): - with self.cached_session(): - gm = mixture_same_family_lib.MixtureSameFamily( - mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]), - components_distribution=normal_lib.Normal( - loc=[-1., 1], scale=[0.1, 0.5])) - x = gm.sample([4, 5], seed=42) - log_prob_x = gm.log_prob(x) - self.assertEqual([4, 5], x.shape) - self.assertEqual([4, 5], log_prob_x.shape) - - def testSampleAndLogProbBatch(self): - with self.cached_session(): - gm = mixture_same_family_lib.MixtureSameFamily( - mixture_distribution=categorical_lib.Categorical(probs=[[0.3, 0.7]]), - components_distribution=normal_lib.Normal( - loc=[[-1., 1]], scale=[[0.1, 0.5]])) - x = gm.sample([4, 5], seed=42) - log_prob_x = gm.log_prob(x) - self.assertEqual([4, 5, 1], x.shape) - self.assertEqual([4, 5, 1], log_prob_x.shape) - - def testSampleAndLogProbShapesBroadcastMix(self): - mix_probs = np.float32([.3, .7]) - bern_probs = np.float32([[.4, .6], [.25, .75]]) - with self.cached_session(): - bm = mixture_same_family_lib.MixtureSameFamily( - mixture_distribution=categorical_lib.Categorical(probs=mix_probs), - components_distribution=bernoulli_lib.Bernoulli(probs=bern_probs)) - x = bm.sample([4, 5], seed=42) - log_prob_x = bm.log_prob(x) - x_ = x.eval() - self.assertEqual([4, 5, 2], x.shape) - self.assertEqual([4, 5, 2], log_prob_x.shape) - self.assertAllEqual( - np.ones_like(x_, dtype=np.bool), np.logical_or(x_ == 0., x_ == 1.)) - - def testSampleAndLogProbMultivariateShapes(self): - with self.cached_session(): - gm = mixture_same_family_lib.MixtureSameFamily( - mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]), - components_distribution=mvn_diag_lib.MultivariateNormalDiag( - loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5])) - x = gm.sample([4, 5], seed=42) - log_prob_x = gm.log_prob(x) - self.assertEqual([4, 5, 2], x.shape) - self.assertEqual([4, 5], log_prob_x.shape) - - def testSampleAndLogProbBatchMultivariateShapes(self): - with self.cached_session(): - gm = mixture_same_family_lib.MixtureSameFamily( - mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]), - components_distribution=mvn_diag_lib.MultivariateNormalDiag( - loc=[[[-1., 1], - [1, -1]], - [[0., 1], - [1, 0]]], - scale_identity_multiplier=[1., 0.5])) - x = gm.sample([4, 5], seed=42) - log_prob_x = gm.log_prob(x) - self.assertEqual([4, 5, 2, 2], x.shape) - self.assertEqual([4, 5, 2], log_prob_x.shape) - - def testSampleConsistentLogProb(self): - with self.cached_session() as sess: - gm = mixture_same_family_lib.MixtureSameFamily( - mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]), - components_distribution=mvn_diag_lib.MultivariateNormalDiag( - loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5])) - # Ball centered at component0's mean. - self.run_test_sample_consistent_log_prob( - sess.run, gm, radius=1., center=[-1., 1], rtol=0.02) - # Larger ball centered at component1's mean. - self.run_test_sample_consistent_log_prob( - sess.run, gm, radius=1., center=[1., -1], rtol=0.02) - - def testLogCdf(self): - with self.cached_session() as sess: - gm = mixture_same_family_lib.MixtureSameFamily( - mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]), - components_distribution=normal_lib.Normal( - loc=[-1., 1], scale=[0.1, 0.5])) - x = gm.sample(10, seed=42) - actual_log_cdf = gm.log_cdf(x) - expected_log_cdf = math_ops.reduce_logsumexp( - (gm.mixture_distribution.logits + - gm.components_distribution.log_cdf(x[..., array_ops.newaxis])), - axis=1) - actual_log_cdf_, expected_log_cdf_ = sess.run([ - actual_log_cdf, expected_log_cdf]) - self.assertAllClose(actual_log_cdf_, expected_log_cdf_, - rtol=1e-6, atol=0.0) - - def testSampleConsistentMeanCovariance(self): - with self.cached_session() as sess: - gm = mixture_same_family_lib.MixtureSameFamily( - mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]), - components_distribution=mvn_diag_lib.MultivariateNormalDiag( - loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5])) - self.run_test_sample_consistent_mean_covariance(sess.run, gm) - - def testVarianceConsistentCovariance(self): - with self.cached_session() as sess: - gm = mixture_same_family_lib.MixtureSameFamily( - mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]), - components_distribution=mvn_diag_lib.MultivariateNormalDiag( - loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5])) - cov_, var_ = sess.run([gm.covariance(), gm.variance()]) - self.assertAllClose(cov_.diagonal(), var_, atol=0.) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mixture_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mixture_test.py deleted file mode 100644 index f8dbd34d02a..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/mixture_test.py +++ /dev/null @@ -1,932 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Mixture distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import contextlib - -import numpy as np -from scipy import stats - -from tensorflow.contrib import distributions -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging as logging - -ds = distributions - - -def _swap_first_last_axes(array): - rank = len(array.shape) - transpose = [rank - 1] + list(range(0, rank - 1)) - return array.transpose(transpose) - - -def _mixture_stddev_np(pi_vector, mu_vector, sigma_vector): - """Computes the standard deviation of a univariate mixture distribution. - - Acts upon `np.array`s (not `tf.Tensor`s). - - Args: - pi_vector: A `np.array` of mixture weights. Shape `[batch, components]`. - mu_vector: A `np.array` of means. Shape `[batch, components]` - sigma_vector: A `np.array` of stddevs. Shape `[batch, components]`. - - Returns: - A `np.array` containing the batch of standard deviations. - """ - pi_vector = np.expand_dims(pi_vector, axis=1) - mean_wa = np.matmul(pi_vector, np.expand_dims(mu_vector, axis=2)) - var_wa = np.matmul(pi_vector, np.expand_dims(sigma_vector**2, axis=2)) - mid_term = np.matmul(pi_vector, np.expand_dims(mu_vector**2, axis=2)) - mixture_variance = ( - np.squeeze(var_wa) + np.squeeze(mid_term) - np.squeeze(mean_wa**2)) - return np.sqrt(mixture_variance) - - -@contextlib.contextmanager -def _test_capture_mvndiag_sample_outputs(): - """Use monkey-patching to capture the output of an MVNDiag _call_sample_n.""" - data_container = [] - true_mvndiag_call_sample_n = ( - ds.MultivariateNormalDiag._call_sample_n) - - def _capturing_mvndiag_call_sample_n( - self, sample_shape, seed, name, **kwargs): - samples = true_mvndiag_call_sample_n( - self, sample_shape, seed, name, **kwargs) - data_container.append(samples) - return samples - - ds.MultivariateNormalDiag._call_sample_n = ( - _capturing_mvndiag_call_sample_n) - yield data_container - ds.MultivariateNormalDiag._call_sample_n = ( - true_mvndiag_call_sample_n) - - -@contextlib.contextmanager -def _test_capture_normal_sample_outputs(): - """Use monkey-patching to capture the output of an Normal _call_sample_n.""" - data_container = [] - true_normal_call_sample_n = ds.Normal._call_sample_n - - def _capturing_normal_call_sample_n(self, sample_shape, seed, name, **kwargs): - samples = true_normal_call_sample_n( - self, sample_shape, seed, name, **kwargs) - data_container.append(samples) - return samples - - ds.Normal._call_sample_n = _capturing_normal_call_sample_n - yield data_container - ds.Normal._call_sample_n = true_normal_call_sample_n - - -def make_univariate_mixture(batch_shape, num_components, use_static_graph): - batch_shape = ops.convert_to_tensor(batch_shape, dtypes.int32) - logits = random_ops.random_uniform( - array_ops.concat((batch_shape, [num_components]), axis=0), - -1, 1, dtype=dtypes.float32) - 50. - components = [ - ds.Normal( - loc=random_ops.random_normal(batch_shape), - scale=10 * random_ops.random_uniform(batch_shape)) - for _ in range(num_components) - ] - cat = ds.Categorical(logits, dtype=dtypes.int32) - return ds.Mixture(cat, components, use_static_graph=use_static_graph) - - -def make_multivariate_mixture(batch_shape, num_components, event_shape, - use_static_graph, batch_shape_tensor=None): - if batch_shape_tensor is None: - batch_shape_tensor = batch_shape - batch_shape_tensor = ops.convert_to_tensor(batch_shape_tensor, dtypes.int32) - logits = random_ops.random_uniform( - array_ops.concat((batch_shape_tensor, [num_components]), 0), - -1, 1, dtype=dtypes.float32) - 50. - logits.set_shape( - tensor_shape.TensorShape(batch_shape).concatenate(num_components)) - static_batch_and_event_shape = ( - tensor_shape.TensorShape(batch_shape).concatenate(event_shape)) - event_shape = ops.convert_to_tensor(event_shape, dtypes.int32) - batch_and_event_shape = array_ops.concat((batch_shape_tensor, event_shape), 0) - def create_component(): - loc = random_ops.random_normal(batch_and_event_shape) - scale_diag = 10 * random_ops.random_uniform(batch_and_event_shape) - loc.set_shape(static_batch_and_event_shape) - scale_diag.set_shape(static_batch_and_event_shape) - return ds.MultivariateNormalDiag( - loc=loc, scale_diag=scale_diag) - components = [create_component() for _ in range(num_components)] - cat = ds.Categorical(logits, dtype=dtypes.int32) - return ds.Mixture(cat, components, use_static_graph=use_static_graph) - - -class MixtureTest(test.TestCase): - use_static_graph = False - - def testShapes(self): - with self.cached_session(): - for batch_shape in ([], [1], [2, 3, 4]): - dist = make_univariate_mixture(batch_shape, num_components=10, - use_static_graph=self.use_static_graph) - self.assertAllEqual(batch_shape, dist.batch_shape) - self.assertAllEqual(batch_shape, dist.batch_shape_tensor().eval()) - self.assertAllEqual([], dist.event_shape) - self.assertAllEqual([], dist.event_shape_tensor().eval()) - - for event_shape in ([1], [2]): - dist = make_multivariate_mixture( - batch_shape, num_components=10, event_shape=event_shape, - use_static_graph=self.use_static_graph) - self.assertAllEqual(batch_shape, dist.batch_shape) - self.assertAllEqual(batch_shape, dist.batch_shape_tensor().eval()) - self.assertAllEqual(event_shape, dist.event_shape) - self.assertAllEqual(event_shape, dist.event_shape_tensor().eval()) - - def testBrokenShapesStatic(self): - with self.assertRaisesWithPredicateMatch(ValueError, - r"cat.num_classes != len"): - ds.Mixture( - ds.Categorical([0.1, 0.5]), # 2 classes - [ds.Normal(loc=1.0, scale=2.0)], - use_static_graph=self.use_static_graph) - with self.assertRaisesWithPredicateMatch( - ValueError, r"\(\) and \(2,\) are not compatible"): - # The value error is raised because the batch shapes of the - # Normals are not equal. One is a scalar, the other is a - # vector of size (2,). - ds.Mixture( - ds.Categorical([-0.5, 0.5]), # scalar batch - [ - ds.Normal( - loc=1.0, scale=2.0), # scalar dist - ds.Normal( - loc=[1.0, 1.0], scale=[2.0, 2.0]) - ], - use_static_graph=self.use_static_graph) - with self.assertRaisesWithPredicateMatch(ValueError, r"Could not infer"): - cat_logits = array_ops.placeholder(shape=[1, None], dtype=dtypes.float32) - ds.Mixture( - ds.Categorical(cat_logits), - [ds.Normal( - loc=[1.0], scale=[2.0])], - use_static_graph=self.use_static_graph) - - def testBrokenShapesDynamic(self): - with self.cached_session(): - d0_param = array_ops.placeholder(dtype=dtypes.float32) - d1_param = array_ops.placeholder(dtype=dtypes.float32) - d = ds.Mixture( - ds.Categorical([0.1, 0.2]), [ - ds.Normal( - loc=d0_param, scale=d0_param), ds.Normal( - loc=d1_param, scale=d1_param) - ], - validate_args=True, - use_static_graph=self.use_static_graph) - - if self.use_static_graph: - error_string = r"Shapes of all inputs must match" - else: - error_string = r"batch shape must match" - - with self.assertRaisesOpError(error_string): - d.sample().eval(feed_dict={d0_param: [2.0, 3.0], d1_param: [1.0]}) - with self.assertRaisesOpError(error_string): - d.sample().eval(feed_dict={d0_param: [2.0, 3.0], d1_param: 1.0}) - - def testBrokenTypes(self): - with self.assertRaisesWithPredicateMatch(TypeError, "Categorical"): - ds.Mixture(None, [], use_static_graph=self.use_static_graph) - cat = ds.Categorical([0.3, 0.2]) - # components must be a list of distributions - with self.assertRaisesWithPredicateMatch( - TypeError, "all .* must be Distribution instances"): - ds.Mixture(cat, [None], use_static_graph=self.use_static_graph) - with self.assertRaisesWithPredicateMatch(TypeError, "same dtype"): - ds.Mixture( - cat, [ - ds.Normal(loc=[1.0], scale=[2.0]), - ds.Normal(loc=[np.float16(1.0)], - scale=[np.float16(2.0)]), - ], use_static_graph=self.use_static_graph) - with self.assertRaisesWithPredicateMatch(ValueError, "non-empty list"): - ds.Mixture(ds.Categorical([0.3, 0.2]), None, - use_static_graph=self.use_static_graph) - - # TODO(ebrevdo): once distribution Domains have been added, add a - # test to ensure that the domains of the distributions in a - # mixture are checked for equivalence. - - def testMeanUnivariate(self): - with self.cached_session() as sess: - for batch_shape in ((), (2,), (2, 3)): - dist = make_univariate_mixture( - batch_shape=batch_shape, num_components=2, - use_static_graph=self.use_static_graph) - mean = dist.mean() - self.assertEqual(batch_shape, mean.get_shape()) - - cat_probs = nn_ops.softmax(dist.cat.logits) - dist_means = [d.mean() for d in dist.components] - - mean_value, cat_probs_value, dist_means_value = sess.run( - [mean, cat_probs, dist_means]) - self.assertEqual(batch_shape, mean_value.shape) - - cat_probs_value = _swap_first_last_axes(cat_probs_value) - true_mean = sum( - [c_p * m for (c_p, m) in zip(cat_probs_value, dist_means_value)]) - - self.assertAllClose(true_mean, mean_value) - - def testMeanMultivariate(self): - with self.cached_session() as sess: - for batch_shape in ((), (2,), (2, 3)): - dist = make_multivariate_mixture( - batch_shape=batch_shape, num_components=2, event_shape=(4,), - use_static_graph=self.use_static_graph) - mean = dist.mean() - self.assertEqual(batch_shape + (4,), mean.get_shape()) - - cat_probs = nn_ops.softmax(dist.cat.logits) - dist_means = [d.mean() for d in dist.components] - - mean_value, cat_probs_value, dist_means_value = sess.run( - [mean, cat_probs, dist_means]) - self.assertEqual(batch_shape + (4,), mean_value.shape) - - cat_probs_value = _swap_first_last_axes(cat_probs_value) - - # Add a new innermost dimension for broadcasting to mvn vector shape - cat_probs_value = [np.expand_dims(c_p, -1) for c_p in cat_probs_value] - - true_mean = sum( - [c_p * m for (c_p, m) in zip(cat_probs_value, dist_means_value)]) - - self.assertAllClose(true_mean, mean_value) - - def testStddevShapeUnivariate(self): - num_components = 2 - # This is the same shape test which is done in 'testMeanUnivariate'. - with self.cached_session() as sess: - for batch_shape in ((), (2,), (2, 3)): - dist = make_univariate_mixture( - batch_shape=batch_shape, num_components=num_components, - use_static_graph=self.use_static_graph) - dev = dist.stddev() - self.assertEqual(batch_shape, dev.get_shape()) - - cat_probs = nn_ops.softmax(dist.cat.logits) - dist_devs = [d.stddev() for d in dist.components] - dist_means = [d.mean() for d in dist.components] - - res = sess.run([dev, cat_probs, dist_devs, dist_means]) - dev_value, cat_probs_values, dist_devs_values, dist_means_values = res - # Manual computation of stddev. - batch_shape_res = cat_probs_values.shape[:-1] - event_shape_res = dist_devs_values[0].shape[len(batch_shape_res):] - stacked_mean_res = np.stack(dist_means_values, -1) - stacked_dev_res = np.stack(dist_devs_values, -1) - - # Broadcast cat probs over event dimensions. - for _ in range(len(event_shape_res)): - cat_probs_values = np.expand_dims(cat_probs_values, len(batch_shape)) - cat_probs_values = cat_probs_values + np.zeros_like(stacked_dev_res) # pylint: disable=g-no-augmented-assignment - - # Perform stddev computation on a flattened batch. - flat_batch_manual_dev = _mixture_stddev_np( - np.reshape(cat_probs_values, [-1, num_components]), - np.reshape(stacked_mean_res, [-1, num_components]), - np.reshape(stacked_dev_res, [-1, num_components])) - - # Reshape to full shape. - full_shape_res = list(batch_shape_res) + list(event_shape_res) - manual_dev = np.reshape(flat_batch_manual_dev, full_shape_res) - self.assertEqual(batch_shape, dev_value.shape) - self.assertAllClose(manual_dev, dev_value) - - def testStddevShapeMultivariate(self): - num_components = 2 - - # This is the same shape test which is done in 'testMeanMultivariate'. - with self.cached_session() as sess: - for batch_shape in ((), (2,), (2, 3)): - dist = make_multivariate_mixture( - batch_shape=batch_shape, - num_components=num_components, - event_shape=(4,), - use_static_graph=self.use_static_graph) - dev = dist.stddev() - self.assertEqual(batch_shape + (4,), dev.get_shape()) - - cat_probs = nn_ops.softmax(dist.cat.logits) - dist_devs = [d.stddev() for d in dist.components] - dist_means = [d.mean() for d in dist.components] - - res = sess.run([dev, cat_probs, dist_devs, dist_means]) - dev_value, cat_probs_values, dist_devs_values, dist_means_values = res - # Manual computation of stddev. - batch_shape_res = cat_probs_values.shape[:-1] - event_shape_res = dist_devs_values[0].shape[len(batch_shape_res):] - stacked_mean_res = np.stack(dist_means_values, -1) - stacked_dev_res = np.stack(dist_devs_values, -1) - - # Broadcast cat probs over event dimensions. - for _ in range(len(event_shape_res)): - cat_probs_values = np.expand_dims(cat_probs_values, len(batch_shape)) - cat_probs_values = cat_probs_values + np.zeros_like(stacked_dev_res) # pylint: disable=g-no-augmented-assignment - - # Perform stddev computation on a flattened batch. - flat_batch_manual_dev = _mixture_stddev_np( - np.reshape(cat_probs_values, [-1, num_components]), - np.reshape(stacked_mean_res, [-1, num_components]), - np.reshape(stacked_dev_res, [-1, num_components])) - - # Reshape to full shape. - full_shape_res = list(batch_shape_res) + list(event_shape_res) - manual_dev = np.reshape(flat_batch_manual_dev, full_shape_res) - self.assertEqual(tuple(full_shape_res), dev_value.shape) - self.assertAllClose(manual_dev, dev_value) - - def testSpecificStddevValue(self): - cat_probs = np.array([0.5, 0.5]) - component_means = np.array([-10, 0.1]) - component_devs = np.array([0.05, 2.33]) - ground_truth_stddev = 5.3120805 - - mixture_dist = ds.Mixture( - cat=ds.Categorical(probs=cat_probs), - components=[ - ds.Normal(loc=component_means[0], - scale=component_devs[0]), - ds.Normal(loc=component_means[1], - scale=component_devs[1]), - ], - use_static_graph=self.use_static_graph) - mix_dev = mixture_dist.stddev() - with self.cached_session() as sess: - actual_stddev = sess.run(mix_dev) - self.assertAllClose(actual_stddev, ground_truth_stddev) - - def testProbScalarUnivariate(self): - with self.cached_session() as sess: - dist = make_univariate_mixture(batch_shape=[], num_components=2, - use_static_graph=self.use_static_graph) - for x in [ - np.array( - [1.0, 2.0], dtype=np.float32), np.array( - 1.0, dtype=np.float32), - np.random.randn(3, 4).astype(np.float32) - ]: - p_x = dist.prob(x) - - self.assertEqual(x.shape, p_x.get_shape()) - cat_probs = nn_ops.softmax([dist.cat.logits])[0] - dist_probs = [d.prob(x) for d in dist.components] - - p_x_value, cat_probs_value, dist_probs_value = sess.run( - [p_x, cat_probs, dist_probs]) - self.assertEqual(x.shape, p_x_value.shape) - - total_prob = sum(c_p_value * d_p_value - for (c_p_value, d_p_value - ) in zip(cat_probs_value, dist_probs_value)) - - self.assertAllClose(total_prob, p_x_value) - - def testProbScalarMultivariate(self): - with self.cached_session() as sess: - dist = make_multivariate_mixture( - batch_shape=[], num_components=2, event_shape=[3], - use_static_graph=self.use_static_graph) - for x in [ - np.array( - [[-1.0, 0.0, 1.0], [0.5, 1.0, -0.3]], dtype=np.float32), np.array( - [-1.0, 0.0, 1.0], dtype=np.float32), - np.random.randn(2, 2, 3).astype(np.float32) - ]: - p_x = dist.prob(x) - - self.assertEqual(x.shape[:-1], p_x.get_shape()) - - cat_probs = nn_ops.softmax([dist.cat.logits])[0] - dist_probs = [d.prob(x) for d in dist.components] - - p_x_value, cat_probs_value, dist_probs_value = sess.run( - [p_x, cat_probs, dist_probs]) - - self.assertEqual(x.shape[:-1], p_x_value.shape) - - total_prob = sum(c_p_value * d_p_value - for (c_p_value, d_p_value - ) in zip(cat_probs_value, dist_probs_value)) - - self.assertAllClose(total_prob, p_x_value) - - def testProbBatchUnivariate(self): - with self.cached_session() as sess: - dist = make_univariate_mixture(batch_shape=[2, 3], num_components=2, - use_static_graph=self.use_static_graph) - - for x in [ - np.random.randn(2, 3).astype(np.float32), - np.random.randn(4, 2, 3).astype(np.float32) - ]: - p_x = dist.prob(x) - self.assertEqual(x.shape, p_x.get_shape()) - - cat_probs = nn_ops.softmax(dist.cat.logits) - dist_probs = [d.prob(x) for d in dist.components] - - p_x_value, cat_probs_value, dist_probs_value = sess.run( - [p_x, cat_probs, dist_probs]) - self.assertEqual(x.shape, p_x_value.shape) - - cat_probs_value = _swap_first_last_axes(cat_probs_value) - - total_prob = sum(c_p_value * d_p_value - for (c_p_value, d_p_value - ) in zip(cat_probs_value, dist_probs_value)) - - self.assertAllClose(total_prob, p_x_value) - - def testProbBatchMultivariate(self): - with self.cached_session() as sess: - dist = make_multivariate_mixture( - batch_shape=[2, 3], num_components=2, event_shape=[4], - use_static_graph=self.use_static_graph) - - for x in [ - np.random.randn(2, 3, 4).astype(np.float32), - np.random.randn(4, 2, 3, 4).astype(np.float32) - ]: - p_x = dist.prob(x) - self.assertEqual(x.shape[:-1], p_x.get_shape()) - - cat_probs = nn_ops.softmax(dist.cat.logits) - dist_probs = [d.prob(x) for d in dist.components] - - p_x_value, cat_probs_value, dist_probs_value = sess.run( - [p_x, cat_probs, dist_probs]) - self.assertEqual(x.shape[:-1], p_x_value.shape) - - cat_probs_value = _swap_first_last_axes(cat_probs_value) - total_prob = sum(c_p_value * d_p_value - for (c_p_value, d_p_value - ) in zip(cat_probs_value, dist_probs_value)) - - self.assertAllClose(total_prob, p_x_value) - - def testSampleScalarBatchUnivariate(self): - with self.cached_session() as sess: - num_components = 3 - batch_shape = [] - dist = make_univariate_mixture( - batch_shape=batch_shape, num_components=num_components, - use_static_graph=self.use_static_graph) - n = 4 - with _test_capture_normal_sample_outputs() as component_samples: - samples = dist.sample(n, seed=123) - self.assertEqual(samples.dtype, dtypes.float32) - self.assertEqual((4,), samples.get_shape()) - cat_samples = dist.cat.sample(n, seed=123) - sample_values, cat_sample_values, dist_sample_values = sess.run( - [samples, cat_samples, component_samples]) - self.assertEqual((4,), sample_values.shape) - - for c in range(num_components): - which_c = np.where(cat_sample_values == c)[0] - size_c = which_c.size - # Scalar Batch univariate case: batch_size == 1, rank 1 - if self.use_static_graph: - which_dist_samples = dist_sample_values[c][which_c] - else: - which_dist_samples = dist_sample_values[c][:size_c] - self.assertAllClose(which_dist_samples, sample_values[which_c]) - - # Test that sampling with the same seed twice gives the same results. - def testSampleMultipleTimes(self): - # 5 component mixture. - logits = [-10.0, -5.0, 0.0, 5.0, 10.0] - mus = [-5.0, 0.0, 5.0, 4.0, 20.0] - sigmas = [0.1, 5.0, 3.0, 0.2, 4.0] - - with self.cached_session(): - n = 100 - - random_seed.set_random_seed(654321) - components = [ - ds.Normal( - loc=mu, scale=sigma) for mu, sigma in zip(mus, sigmas) - ] - cat = ds.Categorical( - logits, dtype=dtypes.int32, name="cat1") - dist1 = ds.Mixture(cat, components, name="mixture1", - use_static_graph=self.use_static_graph) - samples1 = dist1.sample(n, seed=123456).eval() - - random_seed.set_random_seed(654321) - components2 = [ - ds.Normal( - loc=mu, scale=sigma) for mu, sigma in zip(mus, sigmas) - ] - cat2 = ds.Categorical( - logits, dtype=dtypes.int32, name="cat2") - dist2 = ds.Mixture(cat2, components2, name="mixture2", - use_static_graph=self.use_static_graph) - samples2 = dist2.sample(n, seed=123456).eval() - - self.assertAllClose(samples1, samples2) - - def testSampleScalarBatchMultivariate(self): - with self.cached_session() as sess: - num_components = 3 - dist = make_multivariate_mixture( - batch_shape=[], num_components=num_components, event_shape=[2], - use_static_graph=self.use_static_graph) - n = 4 - with _test_capture_mvndiag_sample_outputs() as component_samples: - samples = dist.sample(n, seed=123) - self.assertEqual(samples.dtype, dtypes.float32) - self.assertEqual((4, 2), samples.get_shape()) - cat_samples = dist.cat.sample(n, seed=123) - sample_values, cat_sample_values, dist_sample_values = sess.run( - [samples, cat_samples, component_samples]) - self.assertEqual((4, 2), sample_values.shape) - for c in range(num_components): - which_c = np.where(cat_sample_values == c)[0] - size_c = which_c.size - # Scalar Batch multivariate case: batch_size == 1, rank 2 - if self.use_static_graph: - which_dist_samples = dist_sample_values[c][which_c, :] - else: - which_dist_samples = dist_sample_values[c][:size_c, :] - self.assertAllClose(which_dist_samples, sample_values[which_c, :]) - - def testSampleBatchUnivariate(self): - with self.cached_session() as sess: - num_components = 3 - dist = make_univariate_mixture( - batch_shape=[2, 3], num_components=num_components, - use_static_graph=self.use_static_graph) - n = 4 - with _test_capture_normal_sample_outputs() as component_samples: - samples = dist.sample(n, seed=123) - self.assertEqual(samples.dtype, dtypes.float32) - self.assertEqual((4, 2, 3), samples.get_shape()) - cat_samples = dist.cat.sample(n, seed=123) - sample_values, cat_sample_values, dist_sample_values = sess.run( - [samples, cat_samples, component_samples]) - self.assertEqual((4, 2, 3), sample_values.shape) - for c in range(num_components): - which_c_s, which_c_b0, which_c_b1 = np.where(cat_sample_values == c) - size_c = which_c_s.size - # Batch univariate case: batch_size == [2, 3], rank 3 - if self.use_static_graph: - which_dist_samples = dist_sample_values[c][which_c_s, which_c_b0, - which_c_b1] - else: - which_dist_samples = dist_sample_values[c][range(size_c), which_c_b0, - which_c_b1] - self.assertAllClose(which_dist_samples, - sample_values[which_c_s, which_c_b0, which_c_b1]) - - def _testSampleBatchMultivariate(self, fully_known_batch_shape): - with self.cached_session() as sess: - num_components = 3 - if fully_known_batch_shape: - batch_shape = [2, 3] - batch_shape_tensor = [2, 3] - else: - batch_shape = [None, 3] - batch_shape_tensor = array_ops.placeholder(dtype=dtypes.int32) - - dist = make_multivariate_mixture( - batch_shape=batch_shape, - num_components=num_components, event_shape=[4], - batch_shape_tensor=batch_shape_tensor, - use_static_graph=self.use_static_graph) - n = 5 - with _test_capture_mvndiag_sample_outputs() as component_samples: - samples = dist.sample(n, seed=123) - self.assertEqual(samples.dtype, dtypes.float32) - if fully_known_batch_shape: - self.assertEqual((5, 2, 3, 4), samples.get_shape()) - else: - self.assertEqual([5, None, 3, 4], samples.get_shape().as_list()) - cat_samples = dist.cat.sample(n, seed=123) - if fully_known_batch_shape: - feed_dict = {} - else: - feed_dict = {batch_shape_tensor: [2, 3]} - sample_values, cat_sample_values, dist_sample_values = sess.run( - [samples, cat_samples, component_samples], - feed_dict=feed_dict) - self.assertEqual((5, 2, 3, 4), sample_values.shape) - - for c in range(num_components): - which_c_s, which_c_b0, which_c_b1 = np.where(cat_sample_values == c) - size_c = which_c_s.size - # Batch univariate case: batch_size == [2, 3], rank 4 (multivariate) - if self.use_static_graph: - which_dist_samples = dist_sample_values[c][which_c_s, which_c_b0, - which_c_b1, :] - else: - which_dist_samples = dist_sample_values[c][range(size_c), which_c_b0, - which_c_b1, :] - self.assertAllClose(which_dist_samples, - sample_values[which_c_s, which_c_b0, which_c_b1, :]) - - def testSampleBatchMultivariateFullyKnownBatchShape(self): - self._testSampleBatchMultivariate(fully_known_batch_shape=True) - - def testSampleBatchMultivariateNotFullyKnownBatchShape(self): - self._testSampleBatchMultivariate(fully_known_batch_shape=False) - - def testEntropyLowerBoundMultivariate(self): - with self.cached_session() as sess: - for batch_shape in ((), (2,), (2, 3)): - dist = make_multivariate_mixture( - batch_shape=batch_shape, num_components=2, event_shape=(4,), - use_static_graph=self.use_static_graph) - entropy_lower_bound = dist.entropy_lower_bound() - self.assertEqual(batch_shape, entropy_lower_bound.get_shape()) - - cat_probs = nn_ops.softmax(dist.cat.logits) - dist_entropy = [d.entropy() for d in dist.components] - - entropy_lower_bound_value, cat_probs_value, dist_entropy_value = ( - sess.run([entropy_lower_bound, cat_probs, dist_entropy])) - self.assertEqual(batch_shape, entropy_lower_bound_value.shape) - - cat_probs_value = _swap_first_last_axes(cat_probs_value) - - # entropy_lower_bound = sum_i pi_i entropy_i - # for i in num_components, batchwise. - true_entropy_lower_bound = sum( - [c_p * m for (c_p, m) in zip(cat_probs_value, dist_entropy_value)]) - - self.assertAllClose(true_entropy_lower_bound, entropy_lower_bound_value) - - def testCdfScalarUnivariate(self): - """Tests CDF against scipy for a mixture of seven gaussians.""" - # Construct a mixture of gaussians with seven components. - n_components = 7 - - # pre-softmax mixture probabilities. - mixture_weight_logits = np.random.uniform( - low=-1, high=1, size=(n_components,)).astype(np.float32) - - def _scalar_univariate_softmax(x): - e_x = np.exp(x - np.max(x)) - return e_x / e_x.sum() - - # Construct the ds.Mixture object. - mixture_weights = _scalar_univariate_softmax(mixture_weight_logits) - means = [np.random.uniform(low=-10, high=10, size=()).astype(np.float32) - for _ in range(n_components)] - sigmas = [np.ones(shape=(), dtype=np.float32) for _ in range(n_components)] - cat_tf = ds.Categorical(probs=mixture_weights) - components_tf = [ds.Normal(loc=mu, scale=sigma) - for (mu, sigma) in zip(means, sigmas)] - mixture_tf = ds.Mixture(cat=cat_tf, components=components_tf, - use_static_graph=self.use_static_graph) - - x_tensor = array_ops.placeholder(shape=(), dtype=dtypes.float32) - - # These are two test cases to verify. - xs_to_check = [ - np.array(1.0, dtype=np.float32), - np.array(np.random.randn()).astype(np.float32) - ] - - # Carry out the test for both d.cdf and exp(d.log_cdf). - x_cdf_tf = mixture_tf.cdf(x_tensor) - x_log_cdf_tf = mixture_tf.log_cdf(x_tensor) - - with self.cached_session() as sess: - for x_feed in xs_to_check: - x_cdf_tf_result, x_log_cdf_tf_result = sess.run( - [x_cdf_tf, x_log_cdf_tf], feed_dict={x_tensor: x_feed}) - - # Compute the cdf with scipy. - scipy_component_cdfs = [stats.norm.cdf(x=x_feed, loc=mu, scale=sigma) - for (mu, sigma) in zip(means, sigmas)] - scipy_cdf_result = np.dot(mixture_weights, - np.array(scipy_component_cdfs)) - self.assertAllClose(x_cdf_tf_result, scipy_cdf_result) - self.assertAllClose(np.exp(x_log_cdf_tf_result), scipy_cdf_result) - - def testCdfBatchUnivariate(self): - """Tests against scipy for a (batch of) mixture(s) of seven gaussians.""" - n_components = 7 - batch_size = 5 - mixture_weight_logits = np.random.uniform( - low=-1, high=1, size=(batch_size, n_components)).astype(np.float32) - - def _batch_univariate_softmax(x): - e_x = np.exp(x) - e_x_sum = np.expand_dims(np.sum(e_x, axis=1), axis=1) - return e_x / np.tile(e_x_sum, reps=[1, x.shape[1]]) - - psize = (batch_size,) - mixture_weights = _batch_univariate_softmax(mixture_weight_logits) - means = [np.random.uniform(low=-10, high=10, size=psize).astype(np.float32) - for _ in range(n_components)] - sigmas = [np.ones(shape=psize, dtype=np.float32) - for _ in range(n_components)] - cat_tf = ds.Categorical(probs=mixture_weights) - components_tf = [ds.Normal(loc=mu, scale=sigma) - for (mu, sigma) in zip(means, sigmas)] - mixture_tf = ds.Mixture(cat=cat_tf, components=components_tf, - use_static_graph=self.use_static_graph) - - x_tensor = array_ops.placeholder(shape=psize, dtype=dtypes.float32) - xs_to_check = [ - np.array([1.0, 5.9, -3, 0.0, 0.0], dtype=np.float32), - np.random.randn(batch_size).astype(np.float32) - ] - - x_cdf_tf = mixture_tf.cdf(x_tensor) - x_log_cdf_tf = mixture_tf.log_cdf(x_tensor) - - with self.cached_session() as sess: - for x_feed in xs_to_check: - x_cdf_tf_result, x_log_cdf_tf_result = sess.run( - [x_cdf_tf, x_log_cdf_tf], - feed_dict={x_tensor: x_feed}) - - # Compute the cdf with scipy. - scipy_component_cdfs = [stats.norm.cdf(x=x_feed, loc=mu, scale=sigma) - for (mu, sigma) in zip(means, sigmas)] - weights_and_cdfs = zip(np.transpose(mixture_weights, axes=[1, 0]), - scipy_component_cdfs) - final_cdf_probs_per_component = [ - np.multiply(c_p_value, d_cdf_value) - for (c_p_value, d_cdf_value) in weights_and_cdfs] - scipy_cdf_result = np.sum(final_cdf_probs_per_component, axis=0) - self.assertAllClose(x_cdf_tf_result, scipy_cdf_result) - self.assertAllClose(np.exp(x_log_cdf_tf_result), scipy_cdf_result) - - def testSampleBimixGamma(self): - """Tests a bug in the underlying tf.Gamma op. - - Mixture's use of dynamic partition requires `random_gamma` correctly returns - an empty `Tensor`. - """ - with self.cached_session(): - gm = ds.Mixture( - cat=ds.Categorical(probs=[.3, .7]), - components=[ds.Gamma(1., 2.), - ds.Gamma(2., 1.)], - use_static_graph=self.use_static_graph) - x_ = gm.sample().eval() - self.assertAllEqual([], x_.shape) - - -class MixtureStaticSampleTest(MixtureTest): - use_static_graph = True - - -class MixtureBenchmark(test.Benchmark): - use_static_graph = False - - def _runSamplingBenchmark(self, name, create_distribution, use_gpu, - num_components, batch_size, num_features, - sample_size): - config = config_pb2.ConfigProto() - config.allow_soft_placement = True - np.random.seed(127) - with session.Session(config=config, graph=ops.Graph()) as sess: - random_seed.set_random_seed(0) - with ops.device("/device:GPU:0" if use_gpu else "/cpu:0"): - mixture = create_distribution( - num_components=num_components, - batch_size=batch_size, - num_features=num_features) - sample_op = mixture.sample(sample_size).op - sess.run(variables.global_variables_initializer()) - reported = self.run_op_benchmark( - sess, - sample_op, - min_iters=10, - name=("%s_%s_components_%d_batch_%d_features_%d_sample_%d" % - (name, use_gpu, num_components, batch_size, num_features, - sample_size))) - logging.vlog(2, "\t".join(["%s", "%d", "%d", "%d", "%d", "%g"]) % ( - use_gpu, num_components, batch_size, num_features, sample_size, - reported["wall_time"])) - - def benchmarkSamplingMVNDiag(self): - logging.vlog( - 2, "mvn_diag\tuse_gpu\tcomponents\tbatch\tfeatures\tsample\twall_time") - - def create_distribution(batch_size, num_components, num_features): - cat = ds.Categorical( - logits=np.random.randn(batch_size, num_components)) - mus = [ - variables.Variable(np.random.randn(batch_size, num_features)) - for _ in range(num_components) - ] - sigmas = [ - variables.Variable(np.random.rand(batch_size, num_features)) - for _ in range(num_components) - ] - components = list( - ds.MultivariateNormalDiag( - loc=mu, scale_diag=sigma) for (mu, sigma) in zip(mus, sigmas)) - return ds.Mixture(cat, components, use_static_graph=self.use_static_graph) - - for use_gpu in False, True: - if use_gpu and not test.is_gpu_available(): - continue - for num_components in 1, 8, 16: - for batch_size in 1, 32: - for num_features in 1, 64, 512: - for sample_size in 1, 32, 128: - self._runSamplingBenchmark( - "mvn_diag", - create_distribution=create_distribution, - use_gpu=use_gpu, - num_components=num_components, - batch_size=batch_size, - num_features=num_features, - sample_size=sample_size) - - def benchmarkSamplingMVNFull(self): - logging.vlog( - 2, "mvn_full\tuse_gpu\tcomponents\tbatch\tfeatures\tsample\twall_time") - - def psd(x): - """Construct batch-wise PSD matrices.""" - return np.stack([np.dot(np.transpose(z), z) for z in x]) - - def create_distribution(batch_size, num_components, num_features): - cat = ds.Categorical( - logits=np.random.randn(batch_size, num_components)) - mus = [ - variables.Variable(np.random.randn(batch_size, num_features)) - for _ in range(num_components) - ] - sigmas = [ - variables.Variable( - psd(np.random.rand(batch_size, num_features, num_features))) - for _ in range(num_components) - ] - components = list( - ds.MultivariateNormalTriL( - loc=mu, scale_tril=linalg_ops.cholesky(sigma)) - for (mu, sigma) in zip(mus, sigmas)) - return ds.Mixture(cat, components, use_static_graph=self.use_static_graph) - - for use_gpu in False, True: - if use_gpu and not test.is_gpu_available(): - continue - for num_components in 1, 8, 16: - for batch_size in 1, 32: - for num_features in 1, 64, 512: - for sample_size in 1, 32, 128: - self._runSamplingBenchmark( - "mvn_full", - create_distribution=create_distribution, - use_gpu=use_gpu, - num_components=num_components, - batch_size=batch_size, - num_features=num_features, - sample_size=sample_size) - - -class MixtureStaticSampleBenchmark(MixtureBenchmark): - use_static_graph = True - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/moving_stats_test.py b/tensorflow/contrib/distributions/python/kernel_tests/moving_stats_test.py deleted file mode 100644 index be7c756bea5..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/moving_stats_test.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for computing moving-average statistics.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import moving_stats -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - -rng = np.random.RandomState(0) - - -class MovingReduceMeanVarianceTest(test.TestCase): - - def test_assign_moving_mean_variance(self): - shape = [1, 2] - true_mean = np.array([[0., 3.]]) - true_stddev = np.array([[1.1, 0.5]]) - with self.cached_session() as sess: - # Start "x" out with this mean. - mean_var = variables.VariableV1(array_ops.zeros_like(true_mean)) - variance_var = variables.VariableV1(array_ops.ones_like(true_stddev)) - x = random_ops.random_normal(shape, dtype=np.float64, seed=0) - x = true_stddev * x + true_mean - ema, emv = moving_stats.assign_moving_mean_variance( - mean_var, variance_var, x, decay=0.99) - - self.assertEqual(ema.dtype.base_dtype, dtypes.float64) - self.assertEqual(emv.dtype.base_dtype, dtypes.float64) - - # Run 1000 updates; moving averages should be near the true values. - variables.global_variables_initializer().run() - for _ in range(2000): - sess.run([ema, emv]) - - [mean_var_, variance_var_, ema_, emv_] = sess.run([ - mean_var, variance_var, ema, emv]) - # Test that variables are passed-through. - self.assertAllEqual(mean_var_, ema_) - self.assertAllEqual(variance_var_, emv_) - # Test that values are as expected. - self.assertAllClose(true_mean, ema_, rtol=0.005, atol=0.015) - self.assertAllClose(true_stddev**2., emv_, rtol=0.06, atol=0.) - - # Change the mean, var then update some more. Moving averages should - # re-converge. - sess.run([ - mean_var.assign(np.array([[-1., 2.]])), - variance_var.assign(np.array([[2., 1.]])), - ]) - for _ in range(2000): - sess.run([ema, emv]) - - [mean_var_, variance_var_, ema_, emv_] = sess.run([ - mean_var, variance_var, ema, emv]) - # Test that variables are passed-through. - self.assertAllEqual(mean_var_, ema_) - self.assertAllEqual(variance_var_, emv_) - # Test that values are as expected. - self.assertAllClose(true_mean, ema_, rtol=0.005, atol=0.015) - self.assertAllClose(true_stddev**2., emv_, rtol=0.1, atol=0.) - - def test_moving_mean_variance(self): - shape = [1, 2] - true_mean = np.array([[0., 3.]]) - true_stddev = np.array([[1.1, 0.5]]) - with self.cached_session() as sess: - # Start "x" out with this mean. - x = random_ops.random_normal(shape, dtype=np.float64, seed=0) - x = true_stddev * x + true_mean - ema, emv = moving_stats.moving_mean_variance( - x, decay=0.99) - - self.assertEqual(ema.dtype.base_dtype, dtypes.float64) - self.assertEqual(emv.dtype.base_dtype, dtypes.float64) - - # Run 1000 updates; moving averages should be near the true values. - variables.global_variables_initializer().run() - for _ in range(2000): - sess.run([ema, emv]) - - [ema_, emv_] = sess.run([ema, emv]) - self.assertAllClose(true_mean, ema_, rtol=0.005, atol=0.015) - self.assertAllClose(true_stddev**2., emv_, rtol=0.06, atol=0.) - - -class MovingLogExponentialMovingMeanExpTest(test.TestCase): - - def test_assign_log_moving_mean_exp(self): - shape = [1, 2] - true_mean = np.array([[0., 3.]]) - true_stddev = np.array([[1.1, 0.5]]) - decay = 0.99 - with self.cached_session() as sess: - # Start "x" out with this mean. - x = random_ops.random_normal(shape, dtype=np.float64, seed=0) - x = true_stddev * x + true_mean - log_mean_exp_var = variables.VariableV1(array_ops.zeros_like(true_mean)) - variables.global_variables_initializer().run() - log_mean_exp = moving_stats.assign_log_moving_mean_exp( - log_mean_exp_var, x, decay=decay) - expected_ = np.zeros_like(true_mean) - for _ in range(2000): - x_, log_mean_exp_ = sess.run([x, log_mean_exp]) - expected_ = np.log(decay * np.exp(expected_) + (1 - decay) * np.exp(x_)) - self.assertAllClose(expected_, log_mean_exp_, rtol=1e-6, atol=1e-9) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_plus_low_rank_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_plus_low_rank_test.py deleted file mode 100644 index 88d0d346a41..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_plus_low_rank_test.py +++ /dev/null @@ -1,392 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for MultivariateNormal.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib import distributions -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging as logging - - -ds = distributions - - -class MultivariateNormalDiagPlusLowRankTest(test.TestCase): - """Well tested because this is a simple override of the base class.""" - - def setUp(self): - self._rng = np.random.RandomState(42) - - def testDiagBroadcastBothBatchAndEvent(self): - # batch_shape: [3], event_shape: [2] - diag = np.array([[1., 2], [3, 4], [5, 6]]) - # batch_shape: [1], event_shape: [] - identity_multiplier = np.array([5.]) - with self.cached_session(): - dist = ds.MultivariateNormalDiagPlusLowRank( - scale_diag=diag, - scale_identity_multiplier=identity_multiplier, - validate_args=True) - self.assertAllClose( - np.array([[[1. + 5, 0], - [0, 2 + 5]], - [[3 + 5, 0], - [0, 4 + 5]], - [[5 + 5, 0], - [0, 6 + 5]]]), - dist.scale.to_dense().eval()) - - def testDiagBroadcastBothBatchAndEvent2(self): - # This test differs from `testDiagBroadcastBothBatchAndEvent` in that it - # broadcasts batch_shape's from both the `scale_diag` and - # `scale_identity_multiplier` args. - # batch_shape: [3], event_shape: [2] - diag = np.array([[1., 2], [3, 4], [5, 6]]) - # batch_shape: [3, 1], event_shape: [] - identity_multiplier = np.array([[5.], [4], [3]]) - with self.cached_session(): - dist = ds.MultivariateNormalDiagPlusLowRank( - scale_diag=diag, - scale_identity_multiplier=identity_multiplier, - validate_args=True) - self.assertAllEqual( - [3, 3, 2, 2], - dist.scale.to_dense().get_shape()) - - def testDiagBroadcastOnlyEvent(self): - # batch_shape: [3], event_shape: [2] - diag = np.array([[1., 2], [3, 4], [5, 6]]) - # batch_shape: [3], event_shape: [] - identity_multiplier = np.array([5., 4, 3]) - with self.cached_session(): - dist = ds.MultivariateNormalDiagPlusLowRank( - scale_diag=diag, - scale_identity_multiplier=identity_multiplier, - validate_args=True) - self.assertAllClose( - np.array([[[1. + 5, 0], - [0, 2 + 5]], - [[3 + 4, 0], - [0, 4 + 4]], - [[5 + 3, 0], - [0, 6 + 3]]]), # shape: [3, 2, 2] - dist.scale.to_dense().eval()) - - def testDiagBroadcastMultiplierAndLoc(self): - # batch_shape: [], event_shape: [3] - loc = np.array([1., 0, -1]) - # batch_shape: [3], event_shape: [] - identity_multiplier = np.array([5., 4, 3]) - with self.cached_session(): - dist = ds.MultivariateNormalDiagPlusLowRank( - loc=loc, - scale_identity_multiplier=identity_multiplier, - validate_args=True) - self.assertAllClose( - np.array([[[5, 0, 0], - [0, 5, 0], - [0, 0, 5]], - [[4, 0, 0], - [0, 4, 0], - [0, 0, 4]], - [[3, 0, 0], - [0, 3, 0], - [0, 0, 3]]]), - dist.scale.to_dense().eval()) - - def testMean(self): - mu = [-1.0, 1.0] - diag_large = [1.0, 5.0] - v = [[2.0], [3.0]] - diag_small = [3.0] - with self.cached_session(): - dist = ds.MultivariateNormalDiagPlusLowRank( - loc=mu, - scale_diag=diag_large, - scale_perturb_factor=v, - scale_perturb_diag=diag_small, - validate_args=True) - self.assertAllEqual(mu, dist.mean().eval()) - - def testSample(self): - # TODO(jvdillon): This test should be the basis of a new test fixture which - # is applied to every distribution. When we make this fixture, we'll also - # separate the analytical- and sample-based tests as well as for each - # function tested. For now, we group things so we can recycle one batch of - # samples (thus saving resources). - - mu = np.array([-1., 1, 0.5], dtype=np.float32) - diag_large = np.array([1., 0.5, 0.75], dtype=np.float32) - diag_small = np.array([-1.1, 1.2], dtype=np.float32) - v = np.array([[0.7, 0.8], - [0.9, 1], - [0.5, 0.6]], dtype=np.float32) # shape: [k, r] = [3, 2] - - true_mean = mu - true_scale = np.diag(diag_large) + np.matmul(np.matmul( - v, np.diag(diag_small)), v.T) - true_covariance = np.matmul(true_scale, true_scale.T) - true_variance = np.diag(true_covariance) - true_stddev = np.sqrt(true_variance) - - with self.cached_session() as sess: - dist = ds.MultivariateNormalDiagPlusLowRank( - loc=mu, - scale_diag=diag_large, - scale_perturb_factor=v, - scale_perturb_diag=diag_small, - validate_args=True) - - # The following distributions will test the KL divergence calculation. - mvn_identity = ds.MultivariateNormalDiag( - loc=np.array([1., 2, 0.25], dtype=np.float32), - validate_args=True) - mvn_scaled = ds.MultivariateNormalDiag( - loc=mvn_identity.loc, - scale_identity_multiplier=2.2, - validate_args=True) - mvn_diag = ds.MultivariateNormalDiag( - loc=mvn_identity.loc, - scale_diag=np.array([0.5, 1.5, 1.], dtype=np.float32), - validate_args=True) - mvn_chol = ds.MultivariateNormalTriL( - loc=np.array([1., 2, -1], dtype=np.float32), - scale_tril=np.array([[6., 0, 0], - [2, 5, 0], - [1, 3, 4]], dtype=np.float32) / 10., - validate_args=True) - - scale = dist.scale.to_dense() - - n = int(30e3) - samps = dist.sample(n, seed=0) - sample_mean = math_ops.reduce_mean(samps, 0) - x = samps - sample_mean - sample_covariance = math_ops.matmul(x, x, transpose_a=True) / n - - sample_kl_identity = math_ops.reduce_mean( - dist.log_prob(samps) - mvn_identity.log_prob(samps), 0) - analytical_kl_identity = ds.kl_divergence(dist, mvn_identity) - - sample_kl_scaled = math_ops.reduce_mean( - dist.log_prob(samps) - mvn_scaled.log_prob(samps), 0) - analytical_kl_scaled = ds.kl_divergence(dist, mvn_scaled) - - sample_kl_diag = math_ops.reduce_mean( - dist.log_prob(samps) - mvn_diag.log_prob(samps), 0) - analytical_kl_diag = ds.kl_divergence(dist, mvn_diag) - - sample_kl_chol = math_ops.reduce_mean( - dist.log_prob(samps) - mvn_chol.log_prob(samps), 0) - analytical_kl_chol = ds.kl_divergence(dist, mvn_chol) - - n = int(10e3) - baseline = ds.MultivariateNormalDiag( - loc=np.array([-1., 0.25, 1.25], dtype=np.float32), - scale_diag=np.array([1.5, 0.5, 1.], dtype=np.float32), - validate_args=True) - samps = baseline.sample(n, seed=0) - - sample_kl_identity_diag_baseline = math_ops.reduce_mean( - baseline.log_prob(samps) - mvn_identity.log_prob(samps), 0) - analytical_kl_identity_diag_baseline = ds.kl_divergence( - baseline, mvn_identity) - - sample_kl_scaled_diag_baseline = math_ops.reduce_mean( - baseline.log_prob(samps) - mvn_scaled.log_prob(samps), 0) - analytical_kl_scaled_diag_baseline = ds.kl_divergence( - baseline, mvn_scaled) - - sample_kl_diag_diag_baseline = math_ops.reduce_mean( - baseline.log_prob(samps) - mvn_diag.log_prob(samps), 0) - analytical_kl_diag_diag_baseline = ds.kl_divergence(baseline, mvn_diag) - - sample_kl_chol_diag_baseline = math_ops.reduce_mean( - baseline.log_prob(samps) - mvn_chol.log_prob(samps), 0) - analytical_kl_chol_diag_baseline = ds.kl_divergence(baseline, mvn_chol) - - [ - sample_mean_, - analytical_mean_, - sample_covariance_, - analytical_covariance_, - analytical_variance_, - analytical_stddev_, - scale_, - sample_kl_identity_, analytical_kl_identity_, - sample_kl_scaled_, analytical_kl_scaled_, - sample_kl_diag_, analytical_kl_diag_, - sample_kl_chol_, analytical_kl_chol_, - sample_kl_identity_diag_baseline_, - analytical_kl_identity_diag_baseline_, - sample_kl_scaled_diag_baseline_, analytical_kl_scaled_diag_baseline_, - sample_kl_diag_diag_baseline_, analytical_kl_diag_diag_baseline_, - sample_kl_chol_diag_baseline_, analytical_kl_chol_diag_baseline_, - ] = sess.run([ - sample_mean, - dist.mean(), - sample_covariance, - dist.covariance(), - dist.variance(), - dist.stddev(), - scale, - sample_kl_identity, analytical_kl_identity, - sample_kl_scaled, analytical_kl_scaled, - sample_kl_diag, analytical_kl_diag, - sample_kl_chol, analytical_kl_chol, - sample_kl_identity_diag_baseline, - analytical_kl_identity_diag_baseline, - sample_kl_scaled_diag_baseline, analytical_kl_scaled_diag_baseline, - sample_kl_diag_diag_baseline, analytical_kl_diag_diag_baseline, - sample_kl_chol_diag_baseline, analytical_kl_chol_diag_baseline, - ]) - - sample_variance_ = np.diag(sample_covariance_) - sample_stddev_ = np.sqrt(sample_variance_) - - logging.vlog(2, "true_mean:\n{} ".format(true_mean)) - logging.vlog(2, "sample_mean:\n{}".format(sample_mean_)) - logging.vlog(2, "analytical_mean:\n{}".format(analytical_mean_)) - - logging.vlog(2, "true_covariance:\n{}".format(true_covariance)) - logging.vlog(2, "sample_covariance:\n{}".format(sample_covariance_)) - logging.vlog(2, "analytical_covariance:\n{}".format( - analytical_covariance_)) - - logging.vlog(2, "true_variance:\n{}".format(true_variance)) - logging.vlog(2, "sample_variance:\n{}".format(sample_variance_)) - logging.vlog(2, "analytical_variance:\n{}".format(analytical_variance_)) - - logging.vlog(2, "true_stddev:\n{}".format(true_stddev)) - logging.vlog(2, "sample_stddev:\n{}".format(sample_stddev_)) - logging.vlog(2, "analytical_stddev:\n{}".format(analytical_stddev_)) - - logging.vlog(2, "true_scale:\n{}".format(true_scale)) - logging.vlog(2, "scale:\n{}".format(scale_)) - - logging.vlog(2, "kl_identity: analytical:{} sample:{}".format( - analytical_kl_identity_, sample_kl_identity_)) - - logging.vlog(2, "kl_scaled: analytical:{} sample:{}".format( - analytical_kl_scaled_, sample_kl_scaled_)) - - logging.vlog(2, "kl_diag: analytical:{} sample:{}".format( - analytical_kl_diag_, sample_kl_diag_)) - - logging.vlog(2, "kl_chol: analytical:{} sample:{}".format( - analytical_kl_chol_, sample_kl_chol_)) - - logging.vlog( - 2, "kl_identity_diag_baseline: analytical:{} sample:{}".format( - analytical_kl_identity_diag_baseline_, - sample_kl_identity_diag_baseline_)) - - logging.vlog( - 2, "kl_scaled_diag_baseline: analytical:{} sample:{}".format( - analytical_kl_scaled_diag_baseline_, - sample_kl_scaled_diag_baseline_)) - - logging.vlog(2, "kl_diag_diag_baseline: analytical:{} sample:{}".format( - analytical_kl_diag_diag_baseline_, - sample_kl_diag_diag_baseline_)) - - logging.vlog(2, "kl_chol_diag_baseline: analytical:{} sample:{}".format( - analytical_kl_chol_diag_baseline_, - sample_kl_chol_diag_baseline_)) - - self.assertAllClose(true_mean, sample_mean_, - atol=0., rtol=0.02) - self.assertAllClose(true_mean, analytical_mean_, - atol=0., rtol=1e-6) - - self.assertAllClose(true_covariance, sample_covariance_, - atol=0., rtol=0.02) - self.assertAllClose(true_covariance, analytical_covariance_, - atol=0., rtol=1e-6) - - self.assertAllClose(true_variance, sample_variance_, - atol=0., rtol=0.02) - self.assertAllClose(true_variance, analytical_variance_, - atol=0., rtol=1e-6) - - self.assertAllClose(true_stddev, sample_stddev_, - atol=0., rtol=0.02) - self.assertAllClose(true_stddev, analytical_stddev_, - atol=0., rtol=1e-6) - - self.assertAllClose(true_scale, scale_, - atol=0., rtol=1e-6) - - self.assertAllClose(sample_kl_identity_, analytical_kl_identity_, - atol=0., rtol=0.02) - self.assertAllClose(sample_kl_scaled_, analytical_kl_scaled_, - atol=0., rtol=0.02) - self.assertAllClose(sample_kl_diag_, analytical_kl_diag_, - atol=0., rtol=0.02) - self.assertAllClose(sample_kl_chol_, analytical_kl_chol_, - atol=0., rtol=0.02) - - self.assertAllClose( - sample_kl_identity_diag_baseline_, - analytical_kl_identity_diag_baseline_, - atol=0., rtol=0.02) - self.assertAllClose( - sample_kl_scaled_diag_baseline_, - analytical_kl_scaled_diag_baseline_, - atol=0., rtol=0.02) - self.assertAllClose( - sample_kl_diag_diag_baseline_, - analytical_kl_diag_diag_baseline_, - atol=0., rtol=0.04) - self.assertAllClose( - sample_kl_chol_diag_baseline_, - analytical_kl_chol_diag_baseline_, - atol=0., rtol=0.02) - - def testImplicitLargeDiag(self): - mu = np.array([[1., 2, 3], - [11, 22, 33]]) # shape: [b, k] = [2, 3] - u = np.array([[[1., 2], - [3, 4], - [5, 6]], - [[0.5, 0.75], - [1, 0.25], - [1.5, 1.25]]]) # shape: [b, k, r] = [2, 3, 2] - m = np.array([[0.1, 0.2], - [0.4, 0.5]]) # shape: [b, r] = [2, 2] - scale = np.stack([ - np.eye(3) + np.matmul(np.matmul(u[0], np.diag(m[0])), - np.transpose(u[0])), - np.eye(3) + np.matmul(np.matmul(u[1], np.diag(m[1])), - np.transpose(u[1])), - ]) - cov = np.stack([np.matmul(scale[0], scale[0].T), - np.matmul(scale[1], scale[1].T)]) - logging.vlog(2, "expected_cov:\n{}".format(cov)) - with self.cached_session(): - mvn = ds.MultivariateNormalDiagPlusLowRank( - loc=mu, - scale_perturb_factor=u, - scale_perturb_diag=m) - self.assertAllClose(cov, mvn.covariance().eval(), atol=0., rtol=1e-6) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py deleted file mode 100644 index 21fb54d1dc0..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py +++ /dev/null @@ -1,280 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for MultivariateNormal.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import stats -from tensorflow.contrib import distributions -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - -ds = distributions - - -class MultivariateNormalDiagTest(test.TestCase): - """Well tested because this is a simple override of the base class.""" - - def setUp(self): - self._rng = np.random.RandomState(42) - - def testScalarParams(self): - mu = -1. - diag = -5. - with self.cached_session(): - with self.assertRaisesRegexp(ValueError, "at least 1 dimension"): - ds.MultivariateNormalDiag(mu, diag) - - def testVectorParams(self): - mu = [-1.] - diag = [-5.] - with self.cached_session(): - dist = ds.MultivariateNormalDiag(mu, diag, validate_args=True) - self.assertAllEqual([3, 1], dist.sample(3).get_shape()) - - def testDistWithBatchShapeOneThenTransformedThroughSoftplus(self): - # This complex combination of events resulted in a loss of static shape - # information when tensor_util.constant_value(self._needs_rotation) was - # being used incorrectly (resulting in always rotating). - # Batch shape = [1], event shape = [3] - mu = array_ops.zeros((1, 3)) - diag = array_ops.ones((1, 3)) - with self.cached_session(): - base_dist = ds.MultivariateNormalDiag(mu, diag, validate_args=True) - dist = ds.TransformedDistribution( - base_dist, validate_args=True, bijector=bijectors.Softplus()) - samps = dist.sample(5) # Shape [5, 1, 3]. - self.assertAllEqual([5, 1], dist.log_prob(samps).get_shape()) - - def testMean(self): - mu = [-1., 1] - diag = [1., -5] - with self.cached_session(): - dist = ds.MultivariateNormalDiag(mu, diag, validate_args=True) - self.assertAllEqual(mu, dist.mean().eval()) - - def testMeanWithBroadcastLoc(self): - mu = [-1.] - diag = [1., -5] - with self.cached_session(): - dist = ds.MultivariateNormalDiag(mu, diag, validate_args=True) - self.assertAllEqual([-1., -1.], dist.mean().eval()) - - def testEntropy(self): - mu = [-1., 1] - diag = [-1., 5] - diag_mat = np.diag(diag) - scipy_mvn = stats.multivariate_normal(mean=mu, cov=diag_mat**2) - with self.cached_session(): - dist = ds.MultivariateNormalDiag(mu, diag, validate_args=True) - self.assertAllClose(scipy_mvn.entropy(), dist.entropy().eval(), atol=1e-4) - - def testSample(self): - mu = [-1., 1] - diag = [1., -2] - with self.cached_session(): - dist = ds.MultivariateNormalDiag(mu, diag, validate_args=True) - samps = dist.sample(int(1e3), seed=0).eval() - cov_mat = array_ops.matrix_diag(diag).eval()**2 - - self.assertAllClose(mu, samps.mean(axis=0), atol=0., rtol=0.05) - self.assertAllClose(cov_mat, np.cov(samps.T), atol=0.05, rtol=0.05) - - def testSingularScaleRaises(self): - mu = [-1., 1] - diag = [1., 0] - with self.cached_session(): - dist = ds.MultivariateNormalDiag(mu, diag, validate_args=True) - with self.assertRaisesOpError("Singular"): - dist.sample().eval() - - def testSampleWithBroadcastScale(self): - # mu corresponds to a 2-batch of 3-variate normals - mu = np.zeros([2, 3]) - - # diag corresponds to no batches of 3-variate normals - diag = np.ones([3]) - - with self.cached_session(): - dist = ds.MultivariateNormalDiag(mu, diag, validate_args=True) - - mean = dist.mean() - self.assertAllEqual([2, 3], mean.get_shape()) - self.assertAllClose(mu, mean.eval()) - - n = int(1e3) - samps = dist.sample(n, seed=0).eval() - cov_mat = array_ops.matrix_diag(diag).eval()**2 - sample_cov = np.matmul( - samps.transpose([1, 2, 0]), samps.transpose([1, 0, 2])) / n - - self.assertAllClose(mu, samps.mean(axis=0), atol=0.10, rtol=0.05) - self.assertAllClose([cov_mat, cov_mat], sample_cov, atol=0.10, rtol=0.05) - - def testCovariance(self): - with self.cached_session(): - mvn = ds.MultivariateNormalDiag( - loc=array_ops.zeros([2, 3], dtype=dtypes.float32)) - self.assertAllClose( - np.diag(np.ones([3], dtype=np.float32)), - mvn.covariance().eval()) - - mvn = ds.MultivariateNormalDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_identity_multiplier=[3., 2.]) - self.assertAllEqual([2], mvn.batch_shape) - self.assertAllEqual([3], mvn.event_shape) - self.assertAllClose( - np.array([[[3., 0, 0], [0, 3, 0], [0, 0, 3]], - [[2, 0, 0], [0, 2, 0], [0, 0, 2]]])**2., - mvn.covariance().eval()) - - mvn = ds.MultivariateNormalDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_diag=[[3., 2, 1], [4, 5, 6]]) - self.assertAllEqual([2], mvn.batch_shape) - self.assertAllEqual([3], mvn.event_shape) - self.assertAllClose( - np.array([[[3., 0, 0], [0, 2, 0], [0, 0, 1]], - [[4, 0, 0], [0, 5, 0], [0, 0, 6]]])**2., - mvn.covariance().eval()) - - def testVariance(self): - with self.cached_session(): - mvn = ds.MultivariateNormalDiag( - loc=array_ops.zeros([2, 3], dtype=dtypes.float32)) - self.assertAllClose(np.ones([3], dtype=np.float32), mvn.variance().eval()) - - mvn = ds.MultivariateNormalDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_identity_multiplier=[3., 2.]) - self.assertAllClose( - np.array([[3., 3, 3], [2, 2, 2]])**2., - mvn.variance().eval()) - - mvn = ds.MultivariateNormalDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_diag=[[3., 2, 1], [4, 5, 6]]) - self.assertAllClose( - np.array([[3., 2, 1], [4, 5, 6]])**2., - mvn.variance().eval()) - - def testStddev(self): - with self.cached_session(): - mvn = ds.MultivariateNormalDiag( - loc=array_ops.zeros([2, 3], dtype=dtypes.float32)) - self.assertAllClose(np.ones([3], dtype=np.float32), mvn.stddev().eval()) - - mvn = ds.MultivariateNormalDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_identity_multiplier=[3., 2.]) - self.assertAllClose( - np.array([[3., 3, 3], [2, 2, 2]]), - mvn.stddev().eval()) - - mvn = ds.MultivariateNormalDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_diag=[[3., 2, 1], [4, 5, 6]]) - self.assertAllClose( - np.array([[3., 2, 1], [4, 5, 6]]), - mvn.stddev().eval()) - - def testMultivariateNormalDiagWithSoftplusScale(self): - mu = [-1.0, 1.0] - diag = [-1.0, -2.0] - with self.cached_session(): - dist = ds.MultivariateNormalDiagWithSoftplusScale( - mu, diag, validate_args=True) - samps = dist.sample(1000, seed=0).eval() - cov_mat = array_ops.matrix_diag(nn_ops.softplus(diag)).eval()**2 - - self.assertAllClose(mu, samps.mean(axis=0), atol=0.1) - self.assertAllClose(cov_mat, np.cov(samps.T), atol=0.1) - - def testMultivariateNormalDiagNegLogLikelihood(self): - num_draws = 50 - dims = 3 - with self.cached_session() as sess: - x_pl = array_ops.placeholder( - dtype=dtypes.float32, shape=[None, dims], name="x") - mu_var = variable_scope.get_variable( - name="mu", - shape=[dims], - dtype=dtypes.float32, - initializer=init_ops.constant_initializer(1.)) - sess.run([variables.global_variables_initializer()]) - - mvn = ds.MultivariateNormalDiag( - loc=mu_var, - scale_diag=array_ops.ones(shape=[dims], dtype=dtypes.float32)) - - # Typically you'd use `mvn.log_prob(x_pl)` which is always at least as - # numerically stable as `tf.math.log(mvn.prob(x_pl))`. However in this - # test we're testing a bug specific to `prob` and not `log_prob`; - # http://stackoverflow.com/q/45109305. (The underlying issue was not - # related to `Distributions` but that `reduce_prod` didn't correctly - # handle negative indexes.) - neg_log_likelihood = -math_ops.reduce_sum(math_ops.log(mvn.prob(x_pl))) - grad_neg_log_likelihood = gradients_impl.gradients( - neg_log_likelihood, variables.trainable_variables()) - - x = np.zeros([num_draws, dims], dtype=np.float32) - grad_neg_log_likelihood_ = sess.run( - grad_neg_log_likelihood, feed_dict={x_pl: x}) - self.assertEqual(1, len(grad_neg_log_likelihood_)) - self.assertAllClose( - grad_neg_log_likelihood_[0], - np.tile(num_draws, dims), - rtol=1e-6, - atol=0.) - - def testDynamicBatchShape(self): - mvn = ds.MultivariateNormalDiag( - loc=array_ops.placeholder(dtypes.float32, shape=[None, None, 2]), - scale_diag=array_ops.placeholder(dtypes.float32, shape=[None, None, 2])) - self.assertListEqual(mvn.batch_shape.as_list(), [None, None]) - self.assertListEqual(mvn.event_shape.as_list(), [2]) - - def testDynamicEventShape(self): - mvn = ds.MultivariateNormalDiag( - loc=array_ops.placeholder(dtypes.float32, shape=[2, 3, None]), - scale_diag=array_ops.placeholder(dtypes.float32, shape=[2, 3, None])) - self.assertListEqual(mvn.batch_shape.as_list(), [2, 3]) - self.assertListEqual(mvn.event_shape.as_list(), [None]) - - def testKLDivIdenticalGradientDefined(self): - dims = 3 - with self.cached_session() as sess: - loc = array_ops.zeros([dims], dtype=dtypes.float32) - mvn = ds.MultivariateNormalDiag( - loc=loc, scale_diag=np.ones([dims], dtype=np.float32)) - g = gradients_impl.gradients(ds.kl_divergence(mvn, mvn), loc) - g_ = sess.run(g) - self.assertAllEqual(np.ones_like(g_, dtype=np.bool), np.isfinite(g_)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mvn_full_covariance_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mvn_full_covariance_test.py deleted file mode 100644 index bbf803f0455..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/mvn_full_covariance_test.py +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for MultivariateNormalFullCovariance.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import stats -from tensorflow.contrib import distributions -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.platform import test - - -ds = distributions -rng = np.random.RandomState(42) - - -class MultivariateNormalFullCovarianceTest(test.TestCase): - - def _random_pd_matrix(self, *shape): - mat = rng.rand(*shape) - chol = ds.matrix_diag_transform(mat, transform=nn_ops.softplus) - chol = array_ops.matrix_band_part(chol, -1, 0) - return math_ops.matmul(chol, chol, adjoint_b=True).eval() - - def testRaisesIfInitializedWithNonSymmetricMatrix(self): - with self.cached_session(): - mu = [1., 2.] - sigma = [[1., 0.], [1., 1.]] # Nonsingular, but not symmetric - mvn = ds.MultivariateNormalFullCovariance(mu, sigma, validate_args=True) - with self.assertRaisesOpError("not symmetric"): - mvn.covariance().eval() - - def testNamePropertyIsSetByInitArg(self): - with self.cached_session(): - mu = [1., 2.] - sigma = [[1., 0.], [0., 1.]] - mvn = ds.MultivariateNormalFullCovariance(mu, sigma, name="Billy") - self.assertEqual(mvn.name, "Billy/") - - def testDoesNotRaiseIfInitializedWithSymmetricMatrix(self): - with self.cached_session(): - mu = rng.rand(10) - sigma = self._random_pd_matrix(10, 10) - mvn = ds.MultivariateNormalFullCovariance(mu, sigma, validate_args=True) - # Should not raise - mvn.covariance().eval() - - def testLogPDFScalarBatch(self): - with self.cached_session(): - mu = rng.rand(2) - sigma = self._random_pd_matrix(2, 2) - mvn = ds.MultivariateNormalFullCovariance(mu, sigma, validate_args=True) - x = rng.rand(2) - - log_pdf = mvn.log_prob(x) - pdf = mvn.prob(x) - - scipy_mvn = stats.multivariate_normal(mean=mu, cov=sigma) - - expected_log_pdf = scipy_mvn.logpdf(x) - expected_pdf = scipy_mvn.pdf(x) - self.assertEqual((), log_pdf.get_shape()) - self.assertEqual((), pdf.get_shape()) - self.assertAllClose(expected_log_pdf, log_pdf.eval()) - self.assertAllClose(expected_pdf, pdf.eval()) - - def testLogPDFScalarBatchCovarianceNotProvided(self): - with self.cached_session(): - mu = rng.rand(2) - mvn = ds.MultivariateNormalFullCovariance( - mu, covariance_matrix=None, validate_args=True) - x = rng.rand(2) - - log_pdf = mvn.log_prob(x) - pdf = mvn.prob(x) - - # Initialize a scipy_mvn with the default covariance. - scipy_mvn = stats.multivariate_normal(mean=mu, cov=np.eye(2)) - - expected_log_pdf = scipy_mvn.logpdf(x) - expected_pdf = scipy_mvn.pdf(x) - self.assertEqual((), log_pdf.get_shape()) - self.assertEqual((), pdf.get_shape()) - self.assertAllClose(expected_log_pdf, log_pdf.eval()) - self.assertAllClose(expected_pdf, pdf.eval()) - - def testShapes(self): - with self.cached_session(): - mu = rng.rand(3, 5, 2) - covariance = self._random_pd_matrix(3, 5, 2, 2) - - mvn = ds.MultivariateNormalFullCovariance( - mu, covariance, validate_args=True) - - # Shapes known at graph construction time. - self.assertEqual((2,), tuple(mvn.event_shape.as_list())) - self.assertEqual((3, 5), tuple(mvn.batch_shape.as_list())) - - # Shapes known at runtime. - self.assertEqual((2,), tuple(mvn.event_shape_tensor().eval())) - self.assertEqual((3, 5), tuple(mvn.batch_shape_tensor().eval())) - - def _random_mu_and_sigma(self, batch_shape, event_shape): - # This ensures sigma is positive def. - mat_shape = batch_shape + event_shape + event_shape - mat = rng.randn(*mat_shape) - perm = np.arange(mat.ndim) - perm[-2:] = [perm[-1], perm[-2]] - sigma = np.matmul(mat, np.transpose(mat, perm)) - - mu_shape = batch_shape + event_shape - mu = rng.randn(*mu_shape) - - return mu, sigma - - def testKLBatch(self): - batch_shape = [2] - event_shape = [3] - with self.cached_session(): - mu_a, sigma_a = self._random_mu_and_sigma(batch_shape, event_shape) - mu_b, sigma_b = self._random_mu_and_sigma(batch_shape, event_shape) - mvn_a = ds.MultivariateNormalFullCovariance( - loc=mu_a, - covariance_matrix=sigma_a, - validate_args=True) - mvn_b = ds.MultivariateNormalFullCovariance( - loc=mu_b, - covariance_matrix=sigma_b, - validate_args=True) - - kl = ds.kl_divergence(mvn_a, mvn_b) - self.assertEqual(batch_shape, kl.get_shape()) - - kl_v = kl.eval() - expected_kl_0 = _compute_non_batch_kl(mu_a[0, :], sigma_a[0, :, :], - mu_b[0, :], sigma_b[0, :]) - expected_kl_1 = _compute_non_batch_kl(mu_a[1, :], sigma_a[1, :, :], - mu_b[1, :], sigma_b[1, :]) - self.assertAllClose(expected_kl_0, kl_v[0]) - self.assertAllClose(expected_kl_1, kl_v[1]) - - def testKLBatchBroadcast(self): - batch_shape = [2] - event_shape = [3] - with self.cached_session(): - mu_a, sigma_a = self._random_mu_and_sigma(batch_shape, event_shape) - # No batch shape. - mu_b, sigma_b = self._random_mu_and_sigma([], event_shape) - mvn_a = ds.MultivariateNormalFullCovariance( - loc=mu_a, - covariance_matrix=sigma_a, - validate_args=True) - mvn_b = ds.MultivariateNormalFullCovariance( - loc=mu_b, - covariance_matrix=sigma_b, - validate_args=True) - - kl = ds.kl_divergence(mvn_a, mvn_b) - self.assertEqual(batch_shape, kl.get_shape()) - - kl_v = kl.eval() - expected_kl_0 = _compute_non_batch_kl(mu_a[0, :], sigma_a[0, :, :], - mu_b, sigma_b) - expected_kl_1 = _compute_non_batch_kl(mu_a[1, :], sigma_a[1, :, :], - mu_b, sigma_b) - self.assertAllClose(expected_kl_0, kl_v[0]) - self.assertAllClose(expected_kl_1, kl_v[1]) - - -def _compute_non_batch_kl(mu_a, sigma_a, mu_b, sigma_b): - """Non-batch KL for N(mu_a, sigma_a), N(mu_b, sigma_b).""" - # Check using numpy operations - # This mostly repeats the tensorflow code _kl_mvn_mvn(), but in numpy. - # So it is important to also check that KL(mvn, mvn) = 0. - sigma_b_inv = np.linalg.inv(sigma_b) - - t = np.trace(sigma_b_inv.dot(sigma_a)) - q = (mu_b - mu_a).dot(sigma_b_inv).dot(mu_b - mu_a) - k = mu_a.shape[0] - l = np.log(np.linalg.det(sigma_b) / np.linalg.det(sigma_a)) - - return 0.5 * (t + q - k + l) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mvn_tril_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mvn_tril_test.py deleted file mode 100644 index 776fc2ca9da..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/mvn_tril_test.py +++ /dev/null @@ -1,452 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for MultivariateNormal.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import stats -from tensorflow.contrib import distributions -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging as logging - - -ds = distributions - - -class MultivariateNormalTriLTest(test.TestCase): - - def setUp(self): - self._rng = np.random.RandomState(42) - - def _random_chol(self, *shape): - mat = self._rng.rand(*shape) - chol = ds.matrix_diag_transform(mat, transform=nn_ops.softplus) - chol = array_ops.matrix_band_part(chol, -1, 0) - sigma = math_ops.matmul(chol, chol, adjoint_b=True) - return chol.eval(), sigma.eval() - - def testLogPDFScalarBatch(self): - with self.cached_session(): - mu = self._rng.rand(2) - chol, sigma = self._random_chol(2, 2) - chol[1, 1] = -chol[1, 1] - mvn = ds.MultivariateNormalTriL(mu, chol, validate_args=True) - x = self._rng.rand(2) - - log_pdf = mvn.log_prob(x) - pdf = mvn.prob(x) - - scipy_mvn = stats.multivariate_normal(mean=mu, cov=sigma) - - expected_log_pdf = scipy_mvn.logpdf(x) - expected_pdf = scipy_mvn.pdf(x) - self.assertEqual((), log_pdf.get_shape()) - self.assertEqual((), pdf.get_shape()) - self.assertAllClose(expected_log_pdf, log_pdf.eval()) - self.assertAllClose(expected_pdf, pdf.eval()) - - def testLogPDFXIsHigherRank(self): - with self.cached_session(): - mu = self._rng.rand(2) - chol, sigma = self._random_chol(2, 2) - chol[0, 0] = -chol[0, 0] - mvn = ds.MultivariateNormalTriL(mu, chol, validate_args=True) - x = self._rng.rand(3, 2) - - log_pdf = mvn.log_prob(x) - pdf = mvn.prob(x) - - scipy_mvn = stats.multivariate_normal(mean=mu, cov=sigma) - - expected_log_pdf = scipy_mvn.logpdf(x) - expected_pdf = scipy_mvn.pdf(x) - self.assertEqual((3,), log_pdf.get_shape()) - self.assertEqual((3,), pdf.get_shape()) - self.assertAllClose(expected_log_pdf, log_pdf.eval(), atol=0., rtol=0.02) - self.assertAllClose(expected_pdf, pdf.eval(), atol=0., rtol=0.03) - - def testLogPDFXLowerDimension(self): - with self.cached_session(): - mu = self._rng.rand(3, 2) - chol, sigma = self._random_chol(3, 2, 2) - chol[0, 0, 0] = -chol[0, 0, 0] - chol[2, 1, 1] = -chol[2, 1, 1] - mvn = ds.MultivariateNormalTriL(mu, chol, validate_args=True) - x = self._rng.rand(2) - - log_pdf = mvn.log_prob(x) - pdf = mvn.prob(x) - - self.assertEqual((3,), log_pdf.get_shape()) - self.assertEqual((3,), pdf.get_shape()) - - # scipy can't do batches, so just test one of them. - scipy_mvn = stats.multivariate_normal(mean=mu[1, :], cov=sigma[1, :, :]) - expected_log_pdf = scipy_mvn.logpdf(x) - expected_pdf = scipy_mvn.pdf(x) - - self.assertAllClose(expected_log_pdf, log_pdf.eval()[1]) - self.assertAllClose(expected_pdf, pdf.eval()[1]) - - def testEntropy(self): - with self.cached_session(): - mu = self._rng.rand(2) - chol, sigma = self._random_chol(2, 2) - chol[0, 0] = -chol[0, 0] - mvn = ds.MultivariateNormalTriL(mu, chol, validate_args=True) - entropy = mvn.entropy() - - scipy_mvn = stats.multivariate_normal(mean=mu, cov=sigma) - expected_entropy = scipy_mvn.entropy() - self.assertEqual(entropy.get_shape(), ()) - self.assertAllClose(expected_entropy, entropy.eval()) - - def testEntropyMultidimensional(self): - with self.cached_session(): - mu = self._rng.rand(3, 5, 2) - chol, sigma = self._random_chol(3, 5, 2, 2) - chol[1, 0, 0, 0] = -chol[1, 0, 0, 0] - chol[2, 3, 1, 1] = -chol[2, 3, 1, 1] - mvn = ds.MultivariateNormalTriL(mu, chol, validate_args=True) - entropy = mvn.entropy() - - # Scipy doesn't do batches, so test one of them. - expected_entropy = stats.multivariate_normal( - mean=mu[1, 1, :], cov=sigma[1, 1, :, :]).entropy() - self.assertEqual(entropy.get_shape(), (3, 5)) - self.assertAllClose(expected_entropy, entropy.eval()[1, 1]) - - def testSample(self): - with self.cached_session(): - mu = self._rng.rand(2) - chol, sigma = self._random_chol(2, 2) - chol[0, 0] = -chol[0, 0] - sigma[0, 1] = -sigma[0, 1] - sigma[1, 0] = -sigma[1, 0] - - n = constant_op.constant(100000) - mvn = ds.MultivariateNormalTriL(mu, chol, validate_args=True) - samples = mvn.sample(n, seed=137) - sample_values = samples.eval() - self.assertEqual(samples.get_shape(), [int(100e3), 2]) - self.assertAllClose(sample_values.mean(axis=0), mu, atol=1e-2) - self.assertAllClose(np.cov(sample_values, rowvar=0), sigma, atol=0.06) - - def testSingularScaleRaises(self): - with self.cached_session(): - mu = None - chol = [[1., 0.], [0., 0.]] - mvn = ds.MultivariateNormalTriL(mu, chol, validate_args=True) - with self.assertRaisesOpError("Singular operator"): - mvn.sample().eval() - - def testSampleWithSampleShape(self): - with self.cached_session(): - mu = self._rng.rand(3, 5, 2) - chol, sigma = self._random_chol(3, 5, 2, 2) - chol[1, 0, 0, 0] = -chol[1, 0, 0, 0] - chol[2, 3, 1, 1] = -chol[2, 3, 1, 1] - - mvn = ds.MultivariateNormalTriL(mu, chol, validate_args=True) - samples_val = mvn.sample((10, 11, 12), seed=137).eval() - - # Check sample shape - self.assertEqual((10, 11, 12, 3, 5, 2), samples_val.shape) - - # Check sample means - x = samples_val[:, :, :, 1, 1, :] - self.assertAllClose( - x.reshape(10 * 11 * 12, 2).mean(axis=0), mu[1, 1], atol=0.05) - - # Check that log_prob(samples) works - log_prob_val = mvn.log_prob(samples_val).eval() - x_log_pdf = log_prob_val[:, :, :, 1, 1] - expected_log_pdf = stats.multivariate_normal( - mean=mu[1, 1, :], cov=sigma[1, 1, :, :]).logpdf(x) - self.assertAllClose(expected_log_pdf, x_log_pdf) - - def testSampleMultiDimensional(self): - with self.cached_session(): - mu = self._rng.rand(3, 5, 2) - chol, sigma = self._random_chol(3, 5, 2, 2) - chol[1, 0, 0, 0] = -chol[1, 0, 0, 0] - chol[2, 3, 1, 1] = -chol[2, 3, 1, 1] - - mvn = ds.MultivariateNormalTriL(mu, chol, validate_args=True) - n = constant_op.constant(100000) - samples = mvn.sample(n, seed=137) - sample_values = samples.eval() - - self.assertEqual(samples.get_shape(), (100000, 3, 5, 2)) - self.assertAllClose( - sample_values[:, 1, 1, :].mean(axis=0), mu[1, 1, :], atol=0.05) - self.assertAllClose( - np.cov(sample_values[:, 1, 1, :], rowvar=0), - sigma[1, 1, :, :], - atol=1e-1) - - def testShapes(self): - with self.cached_session(): - mu = self._rng.rand(3, 5, 2) - chol, _ = self._random_chol(3, 5, 2, 2) - chol[1, 0, 0, 0] = -chol[1, 0, 0, 0] - chol[2, 3, 1, 1] = -chol[2, 3, 1, 1] - - mvn = ds.MultivariateNormalTriL(mu, chol, validate_args=True) - - # Shapes known at graph construction time. - self.assertEqual((2,), tuple(mvn.event_shape.as_list())) - self.assertEqual((3, 5), tuple(mvn.batch_shape.as_list())) - - # Shapes known at runtime. - self.assertEqual((2,), tuple(mvn.event_shape_tensor().eval())) - self.assertEqual((3, 5), tuple(mvn.batch_shape_tensor().eval())) - - def _random_mu_and_sigma(self, batch_shape, event_shape): - # This ensures sigma is positive def. - mat_shape = batch_shape + event_shape + event_shape - mat = self._rng.randn(*mat_shape) - perm = np.arange(mat.ndim) - perm[-2:] = [perm[-1], perm[-2]] - sigma = np.matmul(mat, np.transpose(mat, perm)) - - mu_shape = batch_shape + event_shape - mu = self._rng.randn(*mu_shape) - - return mu, sigma - - def testKLNonBatch(self): - batch_shape = [] - event_shape = [2] - with self.cached_session(): - mu_a, sigma_a = self._random_mu_and_sigma(batch_shape, event_shape) - mu_b, sigma_b = self._random_mu_and_sigma(batch_shape, event_shape) - mvn_a = ds.MultivariateNormalTriL( - loc=mu_a, - scale_tril=np.linalg.cholesky(sigma_a), - validate_args=True) - mvn_b = ds.MultivariateNormalTriL( - loc=mu_b, - scale_tril=np.linalg.cholesky(sigma_b), - validate_args=True) - - kl = ds.kl_divergence(mvn_a, mvn_b) - self.assertEqual(batch_shape, kl.get_shape()) - - kl_v = kl.eval() - expected_kl = _compute_non_batch_kl(mu_a, sigma_a, mu_b, sigma_b) - self.assertAllClose(expected_kl, kl_v) - - def testKLBatch(self): - batch_shape = [2] - event_shape = [3] - with self.cached_session(): - mu_a, sigma_a = self._random_mu_and_sigma(batch_shape, event_shape) - mu_b, sigma_b = self._random_mu_and_sigma(batch_shape, event_shape) - mvn_a = ds.MultivariateNormalTriL( - loc=mu_a, - scale_tril=np.linalg.cholesky(sigma_a), - validate_args=True) - mvn_b = ds.MultivariateNormalTriL( - loc=mu_b, - scale_tril=np.linalg.cholesky(sigma_b), - validate_args=True) - - kl = ds.kl_divergence(mvn_a, mvn_b) - self.assertEqual(batch_shape, kl.get_shape()) - - kl_v = kl.eval() - expected_kl_0 = _compute_non_batch_kl(mu_a[0, :], sigma_a[0, :, :], - mu_b[0, :], sigma_b[0, :]) - expected_kl_1 = _compute_non_batch_kl(mu_a[1, :], sigma_a[1, :, :], - mu_b[1, :], sigma_b[1, :]) - self.assertAllClose(expected_kl_0, kl_v[0]) - self.assertAllClose(expected_kl_1, kl_v[1]) - - def testKLBatchBroadcast(self): - batch_shape = [2] - event_shape = [3] - with self.cached_session(): - mu_a, sigma_a = self._random_mu_and_sigma(batch_shape, event_shape) - # No batch shape. - mu_b, sigma_b = self._random_mu_and_sigma([], event_shape) - mvn_a = ds.MultivariateNormalTriL( - loc=mu_a, - scale_tril=np.linalg.cholesky(sigma_a), - validate_args=True) - mvn_b = ds.MultivariateNormalTriL( - loc=mu_b, - scale_tril=np.linalg.cholesky(sigma_b), - validate_args=True) - - kl = ds.kl_divergence(mvn_a, mvn_b) - self.assertEqual(batch_shape, kl.get_shape()) - - kl_v = kl.eval() - expected_kl_0 = _compute_non_batch_kl(mu_a[0, :], sigma_a[0, :, :], - mu_b, sigma_b) - expected_kl_1 = _compute_non_batch_kl(mu_a[1, :], sigma_a[1, :, :], - mu_b, sigma_b) - self.assertAllClose(expected_kl_0, kl_v[0]) - self.assertAllClose(expected_kl_1, kl_v[1]) - - def testKLTwoIdenticalDistributionsIsZero(self): - batch_shape = [2] - event_shape = [3] - with self.cached_session(): - mu_a, sigma_a = self._random_mu_and_sigma(batch_shape, event_shape) - mvn_a = ds.MultivariateNormalTriL( - loc=mu_a, - scale_tril=np.linalg.cholesky(sigma_a), - validate_args=True) - - # Should be zero since KL(p || p) = =. - kl = ds.kl_divergence(mvn_a, mvn_a) - self.assertEqual(batch_shape, kl.get_shape()) - - kl_v = kl.eval() - self.assertAllClose(np.zeros(*batch_shape), kl_v) - - def testSampleLarge(self): - mu = np.array([-1., 1], dtype=np.float32) - scale_tril = np.array([[3., 0], [1, -2]], dtype=np.float32) / 3. - - true_mean = mu - true_scale = scale_tril - true_covariance = np.matmul(true_scale, true_scale.T) - true_variance = np.diag(true_covariance) - true_stddev = np.sqrt(true_variance) - - with self.cached_session() as sess: - dist = ds.MultivariateNormalTriL( - loc=mu, - scale_tril=scale_tril, - validate_args=True) - - # The following distributions will test the KL divergence calculation. - mvn_chol = ds.MultivariateNormalTriL( - loc=np.array([0.5, 1.2], dtype=np.float32), - scale_tril=np.array([[3., 0], [1, 2]], dtype=np.float32), - validate_args=True) - - n = int(10e3) - samps = dist.sample(n, seed=0) - sample_mean = math_ops.reduce_mean(samps, 0) - x = samps - sample_mean - sample_covariance = math_ops.matmul(x, x, transpose_a=True) / n - - sample_kl_chol = math_ops.reduce_mean( - dist.log_prob(samps) - mvn_chol.log_prob(samps), 0) - analytical_kl_chol = ds.kl_divergence(dist, mvn_chol) - - scale = dist.scale.to_dense() - - [ - sample_mean_, - analytical_mean_, - sample_covariance_, - analytical_covariance_, - analytical_variance_, - analytical_stddev_, - sample_kl_chol_, analytical_kl_chol_, - scale_, - ] = sess.run([ - sample_mean, - dist.mean(), - sample_covariance, - dist.covariance(), - dist.variance(), - dist.stddev(), - sample_kl_chol, analytical_kl_chol, - scale, - ]) - - sample_variance_ = np.diag(sample_covariance_) - sample_stddev_ = np.sqrt(sample_variance_) - - logging.vlog(2, "true_mean:\n{} ".format(true_mean)) - logging.vlog(2, "sample_mean:\n{}".format(sample_mean_)) - logging.vlog(2, "analytical_mean:\n{}".format(analytical_mean_)) - - logging.vlog(2, "true_covariance:\n{}".format(true_covariance)) - logging.vlog(2, "sample_covariance:\n{}".format(sample_covariance_)) - logging.vlog( - 2, "analytical_covariance:\n{}".format(analytical_covariance_)) - - logging.vlog(2, "true_variance:\n{}".format(true_variance)) - logging.vlog(2, "sample_variance:\n{}".format(sample_variance_)) - logging.vlog(2, "analytical_variance:\n{}".format(analytical_variance_)) - - logging.vlog(2, "true_stddev:\n{}".format(true_stddev)) - logging.vlog(2, "sample_stddev:\n{}".format(sample_stddev_)) - logging.vlog(2, "analytical_stddev:\n{}".format(analytical_stddev_)) - - logging.vlog(2, "true_scale:\n{}".format(true_scale)) - logging.vlog(2, "scale:\n{}".format(scale_)) - - logging.vlog(2, "kl_chol: analytical:{} sample:{}".format( - analytical_kl_chol_, sample_kl_chol_)) - - self.assertAllClose(true_mean, sample_mean_, - atol=0., rtol=0.03) - self.assertAllClose(true_mean, analytical_mean_, - atol=0., rtol=1e-6) - - self.assertAllClose(true_covariance, sample_covariance_, - atol=0., rtol=0.03) - self.assertAllClose(true_covariance, analytical_covariance_, - atol=0., rtol=1e-6) - - self.assertAllClose(true_variance, sample_variance_, - atol=0., rtol=0.02) - self.assertAllClose(true_variance, analytical_variance_, - atol=0., rtol=1e-6) - - self.assertAllClose(true_stddev, sample_stddev_, - atol=0., rtol=0.01) - self.assertAllClose(true_stddev, analytical_stddev_, - atol=0., rtol=1e-6) - - self.assertAllClose(true_scale, scale_, - atol=0., rtol=1e-6) - - self.assertAllClose(sample_kl_chol_, analytical_kl_chol_, - atol=0., rtol=0.02) - - -def _compute_non_batch_kl(mu_a, sigma_a, mu_b, sigma_b): - """Non-batch KL for N(mu_a, sigma_a), N(mu_b, sigma_b).""" - # Check using numpy operations - # This mostly repeats the tensorflow code _kl_mvn_mvn(), but in numpy. - # So it is important to also check that KL(mvn, mvn) = 0. - sigma_b_inv = np.linalg.inv(sigma_b) - - t = np.trace(sigma_b_inv.dot(sigma_a)) - q = (mu_b - mu_a).dot(sigma_b_inv).dot(mu_b - mu_a) - k = mu_a.shape[0] - l = np.log(np.linalg.det(sigma_b) / np.linalg.det(sigma_a)) - - return 0.5 * (t + q - k + l) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py b/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py deleted file mode 100644 index a46b81af358..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py +++ /dev/null @@ -1,268 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import stats -from tensorflow.contrib.distributions.python.ops import negative_binomial -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -# In all tests that follow, we use scipy.stats.nbinom, which -# represents a Negative Binomial distribution, with success and failure -# probabilities flipped. -class NegativeBinomialTest(test.TestCase): - - def testNegativeBinomialShape(self): - with self.cached_session(): - probs = [.1] * 5 - total_count = [2.0] * 5 - negbinom = negative_binomial.NegativeBinomial( - total_count=total_count, probs=probs) - - self.assertEqual([5], negbinom.batch_shape_tensor().eval()) - self.assertEqual(tensor_shape.TensorShape([5]), negbinom.batch_shape) - self.assertAllEqual([], negbinom.event_shape_tensor().eval()) - self.assertEqual(tensor_shape.TensorShape([]), negbinom.event_shape) - - def testNegativeBinomialShapeBroadcast(self): - with self.cached_session(): - probs = [[.1, .2, .3]] * 5 - total_count = [[2.]] * 5 - negbinom = negative_binomial.NegativeBinomial( - total_count=total_count, probs=probs) - - self.assertAllEqual([5, 3], negbinom.batch_shape_tensor().eval()) - self.assertAllEqual( - tensor_shape.TensorShape([5, 3]), negbinom.batch_shape) - self.assertAllEqual([], negbinom.event_shape_tensor().eval()) - self.assertAllEqual(tensor_shape.TensorShape([]), negbinom.event_shape) - - def testLogits(self): - logits = [[0., 9., -0.5]] - with self.cached_session(): - negbinom = negative_binomial.NegativeBinomial( - total_count=3., logits=logits) - self.assertEqual([1, 3], negbinom.probs.get_shape()) - self.assertEqual([1, 3], negbinom.logits.get_shape()) - self.assertAllClose(logits, negbinom.logits.eval()) - - def testInvalidP(self): - invalid_ps = [-.01, 0., -2.,] - with self.cached_session(): - with self.assertRaisesOpError("Condition x >= 0"): - negbinom = negative_binomial.NegativeBinomial( - 5., probs=invalid_ps, validate_args=True) - negbinom.probs.eval() - - invalid_ps = [1.01, 2., 1.001,] - with self.cached_session(): - with self.assertRaisesOpError("probs has components greater than 1."): - negbinom = negative_binomial.NegativeBinomial( - 5., probs=invalid_ps, validate_args=True) - negbinom.probs.eval() - - def testInvalidNegativeCount(self): - invalid_rs = [-.01, 0., -2.,] - with self.cached_session(): - with self.assertRaisesOpError("Condition x > 0"): - negbinom = negative_binomial.NegativeBinomial( - total_count=invalid_rs, probs=0.1, validate_args=True) - negbinom.total_count.eval() - - def testNegativeBinomialLogCdf(self): - with self.cached_session(): - batch_size = 6 - probs = [.2] * batch_size - probs_v = .2 - total_count = 5. - x = np.array([2., 3., 4., 5., 6., 7.], dtype=np.float32) - negbinom = negative_binomial.NegativeBinomial( - total_count=total_count, probs=probs) - expected_log_cdf = stats.nbinom.logcdf(x, n=total_count, p=1 - probs_v) - log_cdf = negbinom.log_cdf(x) - self.assertEqual([6], log_cdf.get_shape()) - self.assertAllClose(expected_log_cdf, log_cdf.eval()) - - cdf = negbinom.cdf(x) - self.assertEqual([6], cdf.get_shape()) - self.assertAllClose(np.exp(expected_log_cdf), cdf.eval()) - - def testNegativeBinomialLogCdfValidateArgs(self): - with self.cached_session(): - batch_size = 6 - probs = [.9] * batch_size - total_count = 5. - with self.assertRaisesOpError("Condition x >= 0"): - negbinom = negative_binomial.NegativeBinomial( - total_count=total_count, probs=probs, validate_args=True) - negbinom.log_cdf(-1.).eval() - - def testNegativeBinomialLogPmf(self): - with self.cached_session(): - batch_size = 6 - probs = [.2] * batch_size - probs_v = .2 - total_count = 5. - x = np.array([2., 3., 4., 5., 6., 7.], dtype=np.float32) - negbinom = negative_binomial.NegativeBinomial( - total_count=total_count, probs=probs) - expected_log_pmf = stats.nbinom.logpmf(x, n=total_count, p=1 - probs_v) - log_pmf = negbinom.log_prob(x) - self.assertEqual([6], log_pmf.get_shape()) - self.assertAllClose(expected_log_pmf, log_pmf.eval()) - - pmf = negbinom.prob(x) - self.assertEqual([6], pmf.get_shape()) - self.assertAllClose(np.exp(expected_log_pmf), pmf.eval()) - - def testNegativeBinomialLogPmfValidateArgs(self): - with self.cached_session(): - batch_size = 6 - probs = [.9] * batch_size - total_count = 5. - x = array_ops.placeholder(dtypes.float32, shape=[6]) - feed_dict = {x: [2.5, 3.2, 4.3, 5.1, 6., 7.]} - negbinom = negative_binomial.NegativeBinomial( - total_count=total_count, probs=probs, validate_args=True) - - with self.assertRaisesOpError("Condition x == y"): - log_pmf = negbinom.log_prob(x) - log_pmf.eval(feed_dict=feed_dict) - - with self.assertRaisesOpError("Condition x >= 0"): - log_pmf = negbinom.log_prob([-1.]) - log_pmf.eval(feed_dict=feed_dict) - - negbinom = negative_binomial.NegativeBinomial( - total_count=total_count, probs=probs, validate_args=False) - log_pmf = negbinom.log_prob(x) - self.assertEqual([6], log_pmf.get_shape()) - pmf = negbinom.prob(x) - self.assertEqual([6], pmf.get_shape()) - - def testNegativeBinomialLogPmfMultidimensional(self): - with self.cached_session(): - batch_size = 6 - probs = constant_op.constant([[.2, .3, .5]] * batch_size) - probs_v = np.array([.2, .3, .5]) - total_count = 5. - x = np.array([[2., 3., 4., 5., 6., 7.]], dtype=np.float32).T - negbinom = negative_binomial.NegativeBinomial( - total_count=total_count, probs=probs) - expected_log_pmf = stats.nbinom.logpmf( - x, n=total_count, p=1 - probs_v) - log_pmf = negbinom.log_prob(x) - log_pmf_values = log_pmf.eval() - self.assertEqual([6, 3], log_pmf.get_shape()) - self.assertAllClose(expected_log_pmf, log_pmf_values) - - pmf = negbinom.prob(x) - pmf_values = pmf.eval() - self.assertEqual([6, 3], pmf.get_shape()) - self.assertAllClose(np.exp(expected_log_pmf), pmf_values) - - def testNegativeBinomialMean(self): - with self.cached_session(): - total_count = 5. - probs = np.array([.1, .3, .25], dtype=np.float32) - negbinom = negative_binomial.NegativeBinomial( - total_count=total_count, probs=probs) - expected_means = stats.nbinom.mean(n=total_count, p=1 - probs) - self.assertEqual([3], negbinom.mean().get_shape()) - self.assertAllClose(expected_means, negbinom.mean().eval()) - - def testNegativeBinomialVariance(self): - with self.cached_session(): - total_count = 5. - probs = np.array([.1, .3, .25], dtype=np.float32) - negbinom = negative_binomial.NegativeBinomial( - total_count=total_count, probs=probs) - expected_vars = stats.nbinom.var(n=total_count, p=1 - probs) - self.assertEqual([3], negbinom.variance().get_shape()) - self.assertAllClose(expected_vars, negbinom.variance().eval()) - - def testNegativeBinomialStddev(self): - with self.cached_session(): - total_count = 5. - probs = np.array([.1, .3, .25], dtype=np.float32) - negbinom = negative_binomial.NegativeBinomial( - total_count=total_count, probs=probs) - expected_stds = stats.nbinom.std(n=total_count, p=1 - probs) - self.assertEqual([3], negbinom.stddev().get_shape()) - self.assertAllClose(expected_stds, negbinom.stddev().eval()) - - def testNegativeBinomialSample(self): - with self.cached_session() as sess: - probs = [.3, .9] - total_count = [4., 11.] - n = int(100e3) - negbinom = negative_binomial.NegativeBinomial( - total_count=total_count, probs=probs) - - samples = negbinom.sample(n, seed=12345) - self.assertEqual([n, 2], samples.get_shape()) - - sample_mean = math_ops.reduce_mean(samples, axis=0) - sample_var = math_ops.reduce_mean( - (samples - sample_mean[array_ops.newaxis, ...])**2., axis=0) - sample_min = math_ops.reduce_min(samples) - [sample_mean_, sample_var_, sample_min_] = sess.run([ - sample_mean, sample_var, sample_min]) - self.assertAllEqual(np.ones(sample_min_.shape, dtype=np.bool), - sample_min_ >= 0.0) - for i in range(2): - self.assertAllClose(sample_mean_[i], - stats.nbinom.mean(total_count[i], 1 - probs[i]), - atol=0., - rtol=.02) - self.assertAllClose(sample_var_[i], - stats.nbinom.var(total_count[i], 1 - probs[i]), - atol=0., - rtol=.02) - - def testLogProbOverflow(self): - with self.cached_session() as sess: - logits = np.float32([20., 30., 40.]) - total_count = np.float32(1.) - x = np.float32(0.) - nb = negative_binomial.NegativeBinomial( - total_count=total_count, logits=logits) - log_prob_ = sess.run(nb.log_prob(x)) - self.assertAllEqual(np.ones_like(log_prob_, dtype=np.bool), - np.isfinite(log_prob_)) - - def testLogProbUnderflow(self): - with self.cached_session() as sess: - logits = np.float32([-90, -100, -110]) - total_count = np.float32(1.) - x = np.float32(0.) - nb = negative_binomial.NegativeBinomial( - total_count=total_count, logits=logits) - log_prob_ = sess.run(nb.log_prob(x)) - self.assertAllEqual(np.ones_like(log_prob_, dtype=np.bool), - np.isfinite(log_prob_)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/normal_conjugate_posteriors_test.py b/tensorflow/contrib/distributions/python/kernel_tests/normal_conjugate_posteriors_test.py deleted file mode 100644 index ab3c07172a6..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/normal_conjugate_posteriors_test.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for initializers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -from tensorflow.contrib import distributions as distributions_lib -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - -distributions = distributions_lib - - -class NormalTest(test.TestCase): - - def testNormalConjugateKnownSigmaPosterior(self): - with session.Session(): - mu0 = constant_op.constant([3.0]) - sigma0 = constant_op.constant([math.sqrt(10.0)]) - sigma = constant_op.constant([math.sqrt(2.0)]) - x = constant_op.constant([-2.5, 2.5, 4.0, 0.0, -1.0, 2.0]) - s = math_ops.reduce_sum(x) - n = array_ops.size(x) - prior = distributions.Normal(loc=mu0, scale=sigma0) - posterior = distributions.normal_conjugates_known_scale_posterior( - prior=prior, scale=sigma, s=s, n=n) - - # Smoke test - self.assertTrue(isinstance(posterior, distributions.Normal)) - posterior_log_pdf = posterior.log_prob(x).eval() - self.assertEqual(posterior_log_pdf.shape, (6,)) - - def testNormalConjugateKnownSigmaPosteriorND(self): - with session.Session(): - batch_size = 6 - mu0 = constant_op.constant([[3.0, -3.0]] * batch_size) - sigma0 = constant_op.constant([[math.sqrt(10.0), math.sqrt(15.0)]] * - batch_size) - sigma = constant_op.constant([[math.sqrt(2.0)]] * batch_size) - x = array_ops.transpose( - constant_op.constant( - [[-2.5, 2.5, 4.0, 0.0, -1.0, 2.0]], dtype=dtypes.float32)) - s = math_ops.reduce_sum(x) - n = array_ops.size(x) - prior = distributions.Normal(loc=mu0, scale=sigma0) - posterior = distributions.normal_conjugates_known_scale_posterior( - prior=prior, scale=sigma, s=s, n=n) - - # Smoke test - self.assertTrue(isinstance(posterior, distributions.Normal)) - posterior_log_pdf = posterior.log_prob(x).eval() - self.assertEqual(posterior_log_pdf.shape, (6, 2)) - - def testNormalConjugateKnownSigmaNDPosteriorND(self): - with session.Session(): - batch_size = 6 - mu0 = constant_op.constant([[3.0, -3.0]] * batch_size) - sigma0 = constant_op.constant([[math.sqrt(10.0), math.sqrt(15.0)]] * - batch_size) - sigma = constant_op.constant([[math.sqrt(2.0), math.sqrt(4.0)]] * - batch_size) - x = constant_op.constant( - [[-2.5, 2.5, 4.0, 0.0, -1.0, 2.0], [2.5, -2.5, -4.0, 0.0, 1.0, -2.0]], - dtype=dtypes.float32) - s = math_ops.reduce_sum(x, axis=[1]) - x = array_ops.transpose(x) # Reshape to shape (6, 2) - n = constant_op.constant([6] * 2) - prior = distributions.Normal(loc=mu0, scale=sigma0) - posterior = distributions.normal_conjugates_known_scale_posterior( - prior=prior, scale=sigma, s=s, n=n) - - # Smoke test - self.assertTrue(isinstance(posterior, distributions.Normal)) - - # Calculate log_pdf under the 2 models - posterior_log_pdf = posterior.log_prob(x) - self.assertEqual(posterior_log_pdf.get_shape(), (6, 2)) - self.assertEqual(posterior_log_pdf.eval().shape, (6, 2)) - - def testNormalConjugateKnownSigmaPredictive(self): - with session.Session(): - batch_size = 6 - mu0 = constant_op.constant([3.0] * batch_size) - sigma0 = constant_op.constant([math.sqrt(10.0)] * batch_size) - sigma = constant_op.constant([math.sqrt(2.0)] * batch_size) - x = constant_op.constant([-2.5, 2.5, 4.0, 0.0, -1.0, 2.0]) - s = math_ops.reduce_sum(x) - n = array_ops.size(x) - prior = distributions.Normal(loc=mu0, scale=sigma0) - predictive = distributions.normal_conjugates_known_scale_predictive( - prior=prior, scale=sigma, s=s, n=n) - - # Smoke test - self.assertTrue(isinstance(predictive, distributions.Normal)) - predictive_log_pdf = predictive.log_prob(x).eval() - self.assertEqual(predictive_log_pdf.shape, (6,)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/onehot_categorical_test.py b/tensorflow/contrib/distributions/python/kernel_tests/onehot_categorical_test.py deleted file mode 100644 index 84ee19123c5..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/onehot_categorical_test.py +++ /dev/null @@ -1,252 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for OneHotCategorical distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib.distributions.python.ops import onehot_categorical -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import kullback_leibler -from tensorflow.python.platform import test - - -def make_onehot_categorical(batch_shape, num_classes, dtype=dtypes.int32): - logits = random_ops.random_uniform( - list(batch_shape) + [num_classes], -10, 10, dtype=dtypes.float32) - 50. - return onehot_categorical.OneHotCategorical(logits, dtype=dtype) - - -class OneHotCategoricalTest(test.TestCase): - - def setUp(self): - self._rng = np.random.RandomState(42) - - def testP(self): - p = [0.2, 0.8] - dist = onehot_categorical.OneHotCategorical(probs=p) - with self.cached_session(): - self.assertAllClose(p, dist.probs.eval()) - self.assertAllEqual([2], dist.logits.get_shape()) - - def testLogits(self): - p = np.array([0.2, 0.8], dtype=np.float32) - logits = np.log(p) - 50. - dist = onehot_categorical.OneHotCategorical(logits=logits) - with self.cached_session(): - self.assertAllEqual([2], dist.probs.get_shape()) - self.assertAllEqual([2], dist.logits.get_shape()) - self.assertAllClose(dist.probs.eval(), p) - self.assertAllClose(dist.logits.eval(), logits) - - def testShapes(self): - with self.cached_session(): - for batch_shape in ([], [1], [2, 3, 4]): - dist = make_onehot_categorical(batch_shape, 10) - self.assertAllEqual(batch_shape, dist.batch_shape.as_list()) - self.assertAllEqual(batch_shape, dist.batch_shape_tensor().eval()) - self.assertAllEqual([10], dist.event_shape.as_list()) - self.assertAllEqual([10], dist.event_shape_tensor().eval()) - # event_shape is available as a constant because the shape is - # known at graph build time. - self.assertEqual(10, - tensor_util.constant_value(dist.event_shape_tensor())) - - for batch_shape in ([], [1], [2, 3, 4]): - dist = make_onehot_categorical( - batch_shape, constant_op.constant(10, dtype=dtypes.int32)) - self.assertAllEqual(len(batch_shape), dist.batch_shape.ndims) - self.assertAllEqual(batch_shape, dist.batch_shape_tensor().eval()) - self.assertAllEqual([10], dist.event_shape.as_list()) - self.assertEqual(10, dist.event_shape_tensor().eval()) - - def testDtype(self): - dist = make_onehot_categorical([], 5, dtype=dtypes.int32) - self.assertEqual(dist.dtype, dtypes.int32) - self.assertEqual(dist.dtype, dist.sample(5).dtype) - self.assertEqual(dist.dtype, dist.mode().dtype) - dist = make_onehot_categorical([], 5, dtype=dtypes.int64) - self.assertEqual(dist.dtype, dtypes.int64) - self.assertEqual(dist.dtype, dist.sample(5).dtype) - self.assertEqual(dist.dtype, dist.mode().dtype) - self.assertEqual(dist.probs.dtype, dtypes.float32) - self.assertEqual(dist.logits.dtype, dtypes.float32) - self.assertEqual(dist.logits.dtype, dist.entropy().dtype) - self.assertEqual(dist.logits.dtype, dist.prob( - np.array([1]+[0]*4, dtype=np.int64)).dtype) - self.assertEqual(dist.logits.dtype, dist.log_prob( - np.array([1]+[0]*4, dtype=np.int64)).dtype) - - def testUnknownShape(self): - with self.cached_session(): - logits = array_ops.placeholder(dtype=dtypes.float32) - dist = onehot_categorical.OneHotCategorical(logits) - sample = dist.sample() - # Will sample class 1. - sample_value = sample.eval(feed_dict={logits: [-1000.0, 1000.0]}) - self.assertAllEqual([0, 1], sample_value) - # Batch entry 0 will sample class 1, batch entry 1 will sample class 0. - sample_value_batch = sample.eval( - feed_dict={logits: [[-1000.0, 1000.0], [1000.0, -1000.0]]}) - self.assertAllEqual([[0, 1], [1, 0]], sample_value_batch) - - def testEntropyNoBatch(self): - logits = np.log([0.2, 0.8]) - 50. - dist = onehot_categorical.OneHotCategorical(logits) - with self.cached_session(): - self.assertAllClose( - dist.entropy().eval(), - -(0.2 * np.log(0.2) + 0.8 * np.log(0.8))) - - def testEntropyWithBatch(self): - logits = np.log([[0.2, 0.8], [0.6, 0.4]]) - 50. - dist = onehot_categorical.OneHotCategorical(logits) - with self.cached_session(): - self.assertAllClose(dist.entropy().eval(), [ - -(0.2 * np.log(0.2) + 0.8 * np.log(0.8)), - -(0.6 * np.log(0.6) + 0.4 * np.log(0.4)) - ]) - - def testPmf(self): - # check that probability of samples correspond to their class probabilities - with self.cached_session(): - logits = self._rng.random_sample(size=(8, 2, 10)) - prob = np.exp(logits)/np.sum(np.exp(logits), axis=-1, keepdims=True) - dist = onehot_categorical.OneHotCategorical(logits=logits) - np_sample = dist.sample().eval() - np_prob = dist.prob(np_sample).eval() - expected_prob = prob[np_sample.astype(np.bool)] - self.assertAllClose(expected_prob, np_prob.flatten()) - - def testSample(self): - with self.cached_session(): - probs = [[[0.2, 0.8], [0.4, 0.6]]] - dist = onehot_categorical.OneHotCategorical(math_ops.log(probs) - 50.) - n = 100 - samples = dist.sample(n, seed=123) - self.assertEqual(samples.dtype, dtypes.int32) - sample_values = samples.eval() - self.assertAllEqual([n, 1, 2, 2], sample_values.shape) - self.assertFalse(np.any(sample_values < 0)) - self.assertFalse(np.any(sample_values > 1)) - - def testSampleWithSampleShape(self): - with self.cached_session(): - probs = [[[0.2, 0.8], [0.4, 0.6]]] - dist = onehot_categorical.OneHotCategorical(math_ops.log(probs) - 50.) - samples = dist.sample((100, 100), seed=123) - prob = dist.prob(samples) - prob_val = prob.eval() - self.assertAllClose([0.2**2 + 0.8**2], [prob_val[:, :, :, 0].mean()], - atol=1e-2) - self.assertAllClose([0.4**2 + 0.6**2], [prob_val[:, :, :, 1].mean()], - atol=1e-2) - - def testCategoricalCategoricalKL(self): - def np_softmax(logits): - exp_logits = np.exp(logits) - return exp_logits / exp_logits.sum(axis=-1, keepdims=True) - - with self.cached_session() as sess: - for categories in [2, 10]: - for batch_size in [1, 2]: - p_logits = self._rng.random_sample((batch_size, categories)) - q_logits = self._rng.random_sample((batch_size, categories)) - p = onehot_categorical.OneHotCategorical(logits=p_logits) - q = onehot_categorical.OneHotCategorical(logits=q_logits) - prob_p = np_softmax(p_logits) - prob_q = np_softmax(q_logits) - kl_expected = np.sum( - prob_p * (np.log(prob_p) - np.log(prob_q)), axis=-1) - - kl_actual = kullback_leibler.kl_divergence(p, q) - kl_same = kullback_leibler.kl_divergence(p, p) - x = p.sample(int(2e4), seed=0) - x = math_ops.cast(x, dtype=dtypes.float32) - # Compute empirical KL(p||q). - kl_sample = math_ops.reduce_mean(p.log_prob(x) - q.log_prob(x), 0) - - [kl_sample_, kl_actual_, kl_same_] = sess.run([kl_sample, kl_actual, - kl_same]) - self.assertEqual(kl_actual.get_shape(), (batch_size,)) - self.assertAllClose(kl_same_, np.zeros_like(kl_expected)) - self.assertAllClose(kl_actual_, kl_expected, atol=0., rtol=1e-6) - self.assertAllClose(kl_sample_, kl_expected, atol=1e-2, rtol=0.) - - def testSampleUnbiasedNonScalarBatch(self): - with self.cached_session() as sess: - logits = self._rng.rand(4, 3, 2).astype(np.float32) - dist = onehot_categorical.OneHotCategorical(logits=logits) - n = int(3e3) - x = dist.sample(n, seed=0) - x = math_ops.cast(x, dtype=dtypes.float32) - sample_mean = math_ops.reduce_mean(x, 0) - x_centered = array_ops.transpose(x - sample_mean, [1, 2, 3, 0]) - sample_covariance = math_ops.matmul( - x_centered, x_centered, adjoint_b=True) / n - [ - sample_mean_, - sample_covariance_, - actual_mean_, - actual_covariance_, - ] = sess.run([ - sample_mean, - sample_covariance, - dist.probs, - dist.covariance(), - ]) - self.assertAllEqual([4, 3, 2], sample_mean.get_shape()) - self.assertAllClose(actual_mean_, sample_mean_, atol=0., rtol=0.07) - self.assertAllEqual([4, 3, 2, 2], sample_covariance.get_shape()) - self.assertAllClose( - actual_covariance_, sample_covariance_, atol=0., rtol=0.10) - - def testSampleUnbiasedScalarBatch(self): - with self.cached_session() as sess: - logits = self._rng.rand(3).astype(np.float32) - dist = onehot_categorical.OneHotCategorical(logits=logits) - n = int(1e4) - x = dist.sample(n, seed=0) - x = math_ops.cast(x, dtype=dtypes.float32) - sample_mean = math_ops.reduce_mean(x, 0) # elementwise mean - x_centered = x - sample_mean - sample_covariance = math_ops.matmul( - x_centered, x_centered, adjoint_a=True) / n - [ - sample_mean_, - sample_covariance_, - actual_mean_, - actual_covariance_, - ] = sess.run([ - sample_mean, - sample_covariance, - dist.probs, - dist.covariance(), - ]) - self.assertAllEqual([3], sample_mean.get_shape()) - self.assertAllClose(actual_mean_, sample_mean_, atol=0., rtol=0.1) - self.assertAllEqual([3, 3], sample_covariance.get_shape()) - self.assertAllClose( - actual_covariance_, sample_covariance_, atol=0., rtol=0.1) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py deleted file mode 100644 index e2d04c9c274..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for PoissonLogNormalQuadratureCompoundTest.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import poisson_lognormal -from tensorflow.contrib.distributions.python.ops import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class _PoissonLogNormalQuadratureCompoundTest( - test_util.DiscreteScalarDistributionTestHelpers): - """Tests the PoissonLogNormalQuadratureCompoundTest distribution.""" - - def testSampleProbConsistent(self): - with self.cached_session() as sess: - pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( - loc=array_ops.placeholder_with_default( - -2., - shape=[] if self.static_shape else None), - scale=array_ops.placeholder_with_default( - 1.1, - shape=[] if self.static_shape else None), - quadrature_size=10, - validate_args=True) - self.run_test_sample_consistent_log_prob( - sess.run, pln, batch_size=1, rtol=0.1) - - def testMeanVariance(self): - with self.cached_session() as sess: - pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( - loc=array_ops.placeholder_with_default( - 0., - shape=[] if self.static_shape else None), - scale=array_ops.placeholder_with_default( - 1., - shape=[] if self.static_shape else None), - quadrature_size=10, - validate_args=True) - self.run_test_sample_consistent_mean_variance( - sess.run, pln, rtol=0.02) - - def testSampleProbConsistentBroadcastScalar(self): - with self.cached_session() as sess: - pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( - loc=array_ops.placeholder_with_default( - [0., -0.5], - shape=[2] if self.static_shape else None), - scale=array_ops.placeholder_with_default( - 1., - shape=[] if self.static_shape else None), - quadrature_size=10, - validate_args=True) - self.run_test_sample_consistent_log_prob( - sess.run, pln, batch_size=2, rtol=0.1, atol=0.01) - - def testMeanVarianceBroadcastScalar(self): - with self.cached_session() as sess: - pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( - loc=array_ops.placeholder_with_default( - [0., -0.5], - shape=[2] if self.static_shape else None), - scale=array_ops.placeholder_with_default( - 1., - shape=[] if self.static_shape else None), - quadrature_size=10, - validate_args=True) - self.run_test_sample_consistent_mean_variance( - sess.run, pln, rtol=0.1, atol=0.01) - - def testSampleProbConsistentBroadcastBoth(self): - with self.cached_session() as sess: - pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( - loc=array_ops.placeholder_with_default( - [[0.], [-0.5]], - shape=[2, 1] if self.static_shape else None), - scale=array_ops.placeholder_with_default( - [[1., 0.9]], - shape=[1, 2] if self.static_shape else None), - quadrature_size=10, - validate_args=True) - self.run_test_sample_consistent_log_prob( - sess.run, pln, batch_size=4, rtol=0.1, atol=0.08) - - def testMeanVarianceBroadcastBoth(self): - with self.cached_session() as sess: - pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( - loc=array_ops.placeholder_with_default( - [[0.], [-0.5]], - shape=[2, 1] if self.static_shape else None), - scale=array_ops.placeholder_with_default( - [[1., 0.9]], - shape=[1, 2] if self.static_shape else None), - quadrature_size=10, - validate_args=True) - self.run_test_sample_consistent_mean_variance( - sess.run, pln, rtol=0.1, atol=0.01) - - -class PoissonLogNormalQuadratureCompoundStaticShapeTest( - _PoissonLogNormalQuadratureCompoundTest, test.TestCase): - - @property - def static_shape(self): - return True - - -class PoissonLogNormalQuadratureCompoundDynamicShapeTest( - _PoissonLogNormalQuadratureCompoundTest, test.TestCase): - - @property - def static_shape(self): - return False - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/poisson_test.py b/tensorflow/contrib/distributions/python/kernel_tests/poisson_test.py deleted file mode 100644 index 29eba5afcaa..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/poisson_test.py +++ /dev/null @@ -1,265 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import special -from scipy import stats -from tensorflow.contrib.distributions.python.ops import poisson as poisson_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class PoissonTest(test.TestCase): - - def _make_poisson(self, rate, validate_args=False): - return poisson_lib.Poisson(rate=rate, validate_args=validate_args) - - def testPoissonShape(self): - with self.cached_session(): - lam = constant_op.constant([3.0] * 5) - poisson = self._make_poisson(rate=lam) - - self.assertEqual(poisson.batch_shape_tensor().eval(), (5,)) - self.assertEqual(poisson.batch_shape, tensor_shape.TensorShape([5])) - self.assertAllEqual(poisson.event_shape_tensor().eval(), []) - self.assertEqual(poisson.event_shape, tensor_shape.TensorShape([])) - - def testInvalidLam(self): - invalid_lams = [-.01, 0., -2.] - for lam in invalid_lams: - with self.cached_session(): - with self.assertRaisesOpError("Condition x > 0"): - poisson = self._make_poisson(rate=lam, validate_args=True) - poisson.rate.eval() - - def testPoissonLogPmf(self): - with self.cached_session(): - batch_size = 6 - lam = constant_op.constant([3.0] * batch_size) - lam_v = 3.0 - x = [2., 3., 4., 5., 6., 7.] - poisson = self._make_poisson(rate=lam) - log_pmf = poisson.log_prob(x) - self.assertEqual(log_pmf.get_shape(), (6,)) - self.assertAllClose(log_pmf.eval(), stats.poisson.logpmf(x, lam_v)) - - pmf = poisson.prob(x) - self.assertEqual(pmf.get_shape(), (6,)) - self.assertAllClose(pmf.eval(), stats.poisson.pmf(x, lam_v)) - - def testPoissonLogPmfValidateArgs(self): - with self.cached_session(): - batch_size = 6 - lam = constant_op.constant([3.0] * batch_size) - x = array_ops.placeholder(dtypes.float32, shape=[6]) - feed_dict = {x: [2.5, 3.2, 4.3, 5.1, 6., 7.]} - poisson = self._make_poisson(rate=lam, validate_args=True) - - # Non-integer - with self.assertRaisesOpError("cannot contain fractional components"): - log_pmf = poisson.log_prob(x) - log_pmf.eval(feed_dict=feed_dict) - - with self.assertRaisesOpError("Condition x >= 0"): - log_pmf = poisson.log_prob([-1.]) - log_pmf.eval(feed_dict=feed_dict) - - poisson = self._make_poisson(rate=lam, validate_args=False) - log_pmf = poisson.log_prob(x) - self.assertEqual(log_pmf.get_shape(), (6,)) - pmf = poisson.prob(x) - self.assertEqual(pmf.get_shape(), (6,)) - - def testPoissonLogPmfMultidimensional(self): - with self.cached_session(): - batch_size = 6 - lam = constant_op.constant([[2.0, 4.0, 5.0]] * batch_size) - lam_v = [2.0, 4.0, 5.0] - x = np.array([[2., 3., 4., 5., 6., 7.]], dtype=np.float32).T - - poisson = self._make_poisson(rate=lam) - log_pmf = poisson.log_prob(x) - self.assertEqual(log_pmf.get_shape(), (6, 3)) - self.assertAllClose(log_pmf.eval(), stats.poisson.logpmf(x, lam_v)) - - pmf = poisson.prob(x) - self.assertEqual(pmf.get_shape(), (6, 3)) - self.assertAllClose(pmf.eval(), stats.poisson.pmf(x, lam_v)) - - def testPoissonCDF(self): - with self.cached_session(): - batch_size = 6 - lam = constant_op.constant([3.0] * batch_size) - lam_v = 3.0 - x = [2., 3., 4., 5., 6., 7.] - - poisson = self._make_poisson(rate=lam) - log_cdf = poisson.log_cdf(x) - self.assertEqual(log_cdf.get_shape(), (6,)) - self.assertAllClose(log_cdf.eval(), stats.poisson.logcdf(x, lam_v)) - - cdf = poisson.cdf(x) - self.assertEqual(cdf.get_shape(), (6,)) - self.assertAllClose(cdf.eval(), stats.poisson.cdf(x, lam_v)) - - def testPoissonCDFNonIntegerValues(self): - with self.cached_session(): - batch_size = 6 - lam = constant_op.constant([3.0] * batch_size) - lam_v = 3.0 - x = np.array([2.2, 3.1, 4., 5.5, 6., 7.], dtype=np.float32) - - poisson = self._make_poisson(rate=lam) - cdf = poisson.cdf(x) - self.assertEqual(cdf.get_shape(), (6,)) - - # The Poisson CDF should be valid on these non-integer values, and - # equal to igammac(1 + x, rate). - self.assertAllClose(cdf.eval(), special.gammaincc(1. + x, lam_v)) - - with self.assertRaisesOpError("cannot contain fractional components"): - poisson_validate = self._make_poisson(rate=lam, validate_args=True) - poisson_validate.cdf(x).eval() - - def testPoissonCdfMultidimensional(self): - with self.cached_session(): - batch_size = 6 - lam = constant_op.constant([[2.0, 4.0, 5.0]] * batch_size) - lam_v = [2.0, 4.0, 5.0] - x = np.array([[2., 3., 4., 5., 6., 7.]], dtype=np.float32).T - - poisson = self._make_poisson(rate=lam) - log_cdf = poisson.log_cdf(x) - self.assertEqual(log_cdf.get_shape(), (6, 3)) - self.assertAllClose(log_cdf.eval(), stats.poisson.logcdf(x, lam_v)) - - cdf = poisson.cdf(x) - self.assertEqual(cdf.get_shape(), (6, 3)) - self.assertAllClose(cdf.eval(), stats.poisson.cdf(x, lam_v)) - - def testPoissonMean(self): - with self.cached_session(): - lam_v = [1.0, 3.0, 2.5] - poisson = self._make_poisson(rate=lam_v) - self.assertEqual(poisson.mean().get_shape(), (3,)) - self.assertAllClose(poisson.mean().eval(), stats.poisson.mean(lam_v)) - self.assertAllClose(poisson.mean().eval(), lam_v) - - def testPoissonVariance(self): - with self.cached_session(): - lam_v = [1.0, 3.0, 2.5] - poisson = self._make_poisson(rate=lam_v) - self.assertEqual(poisson.variance().get_shape(), (3,)) - self.assertAllClose(poisson.variance().eval(), stats.poisson.var(lam_v)) - self.assertAllClose(poisson.variance().eval(), lam_v) - - def testPoissonStd(self): - with self.cached_session(): - lam_v = [1.0, 3.0, 2.5] - poisson = self._make_poisson(rate=lam_v) - self.assertEqual(poisson.stddev().get_shape(), (3,)) - self.assertAllClose(poisson.stddev().eval(), stats.poisson.std(lam_v)) - self.assertAllClose(poisson.stddev().eval(), np.sqrt(lam_v)) - - def testPoissonMode(self): - with self.cached_session(): - lam_v = [1.0, 3.0, 2.5, 3.2, 1.1, 0.05] - poisson = self._make_poisson(rate=lam_v) - self.assertEqual(poisson.mode().get_shape(), (6,)) - self.assertAllClose(poisson.mode().eval(), np.floor(lam_v)) - - def testPoissonMultipleMode(self): - with self.cached_session(): - lam_v = [1.0, 3.0, 2.0, 4.0, 5.0, 10.0] - poisson = self._make_poisson(rate=lam_v) - # For the case where lam is an integer, the modes are: lam and lam - 1. - # In this case, we get back the larger of the two modes. - self.assertEqual((6,), poisson.mode().get_shape()) - self.assertAllClose(lam_v, poisson.mode().eval()) - - def testPoissonSample(self): - with self.cached_session(): - lam_v = 4.0 - lam = constant_op.constant(lam_v) - # Choosing `n >= (k/rtol)**2, roughly ensures our sample mean should be - # within `k` std. deviations of actual up to rtol precision. - n = int(100e3) - poisson = self._make_poisson(rate=lam) - samples = poisson.sample(n, seed=123456) - sample_values = samples.eval() - self.assertEqual(samples.get_shape(), (n,)) - self.assertEqual(sample_values.shape, (n,)) - self.assertAllClose( - sample_values.mean(), stats.poisson.mean(lam_v), rtol=.01) - self.assertAllClose( - sample_values.var(), stats.poisson.var(lam_v), rtol=.01) - - def testPoissonSampleMultidimensionalMean(self): - with self.cached_session(): - lam_v = np.array([np.arange(1, 51, dtype=np.float32)]) # 1 x 50 - poisson = self._make_poisson(rate=lam_v) - # Choosing `n >= (k/rtol)**2, roughly ensures our sample mean should be - # within `k` std. deviations of actual up to rtol precision. - n = int(100e3) - samples = poisson.sample(n, seed=123456) - sample_values = samples.eval() - self.assertEqual(samples.get_shape(), (n, 1, 50)) - self.assertEqual(sample_values.shape, (n, 1, 50)) - self.assertAllClose( - sample_values.mean(axis=0), - stats.poisson.mean(lam_v), - rtol=.01, - atol=0) - - def testPoissonSampleMultidimensionalVariance(self): - with self.cached_session(): - lam_v = np.array([np.arange(5, 15, dtype=np.float32)]) # 1 x 10 - poisson = self._make_poisson(rate=lam_v) - # Choosing `n >= 2 * lam * (k/rtol)**2, roughly ensures our sample - # variance should be within `k` std. deviations of actual up to rtol - # precision. - n = int(300e3) - samples = poisson.sample(n, seed=123456) - sample_values = samples.eval() - self.assertEqual(samples.get_shape(), (n, 1, 10)) - self.assertEqual(sample_values.shape, (n, 1, 10)) - - self.assertAllClose( - sample_values.var(axis=0), stats.poisson.var(lam_v), rtol=.03, atol=0) - - -class PoissonLogRateTest(PoissonTest): - - def _make_poisson(self, rate, validate_args=False): - return poisson_lib.Poisson( - log_rate=math_ops.log(rate), - validate_args=validate_args) - - def testInvalidLam(self): - # No need to worry about the non-negativity of `rate` when using the - # `log_rate` parameterization. - pass - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/quantized_distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/quantized_distribution_test.py deleted file mode 100644 index 82257e136ba..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/quantized_distribution_test.py +++ /dev/null @@ -1,424 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import stats -from tensorflow.contrib import distributions as distributions_lib -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - -distributions = distributions_lib -rng = np.random.RandomState(123) - - -class QuantizedDistributionTest(test.TestCase): - - def _assert_all_finite(self, array): - self.assertTrue(np.isfinite(array).all()) - - def testQuantizationOfUniformWithCutoffsHavingNoEffect(self): - with self.cached_session() as sess: - # The Quantized uniform with cutoffs == None divides the real line into: - # R = ...(-1, 0](0, 1](1, 2](2, 3](3, 4]... - # j = ... 0 1 2 3 4 ... - # Since this uniform (below) is supported on [0, 3], - # it places 1/3 of its mass in the intervals j = 1, 2, 3. - # Adding a cutoff at y = 0 changes the picture to - # R = ...(-inf, 0](0, 1](1, 2](2, 3](3, 4]... - # j = ... 0 1 2 3 4 ... - # So the QUniform still places 1/3 of its mass in the intervals - # j = 1, 2, 3. - # Adding a cutoff at y = 3 changes the picture to - # R = ...(-1, 0](0, 1](1, 2](2, inf) - # j = ... 0 1 2 3 - # and the QUniform still places 1/3 of its mass in the intervals - # j = 1, 2, 3. - for lcut, ucut in [(None, None), (0.0, None), (None, 3.0), (0.0, 3.0), - (-10., 10.)]: - qdist = distributions.QuantizedDistribution( - distribution=distributions.Uniform(low=0.0, high=3.0), - low=lcut, - high=ucut) - - # pmf - pmf_n1, pmf_0, pmf_1, pmf_2, pmf_3, pmf_4, pmf_5 = sess.run( - qdist.prob([-1., 0., 1., 2., 3., 4., 5.])) - # uniform had no mass below -1. - self.assertAllClose(0., pmf_n1) - # uniform had no mass below 0. - self.assertAllClose(0., pmf_0) - # uniform put 1/3 of its mass in each of (0, 1], (1, 2], (2, 3], - # which are the intervals j = 1, 2, 3. - self.assertAllClose(1 / 3, pmf_1) - self.assertAllClose(1 / 3, pmf_2) - self.assertAllClose(1 / 3, pmf_3) - # uniform had no mass in (3, 4] or (4, 5], which are j = 4, 5. - self.assertAllClose(0 / 3, pmf_4) - self.assertAllClose(0 / 3, pmf_5) - - # cdf - cdf_n1, cdf_0, cdf_1, cdf_2, cdf_2p5, cdf_3, cdf_4, cdf_5 = sess.run( - qdist.cdf([-1., 0., 1., 2., 2.5, 3., 4., 5.])) - self.assertAllClose(0., cdf_n1) - self.assertAllClose(0., cdf_0) - self.assertAllClose(1 / 3, cdf_1) - self.assertAllClose(2 / 3, cdf_2) - # Note fractional values allowed for cdfs of discrete distributions. - # And adding 0.5 makes no difference because the quantized dist has - # mass only on the integers, never in between. - self.assertAllClose(2 / 3, cdf_2p5) - self.assertAllClose(3 / 3, cdf_3) - self.assertAllClose(3 / 3, cdf_4) - self.assertAllClose(3 / 3, cdf_5) - - def testQuantizationOfUniformWithCutoffsInTheMiddle(self): - with self.cached_session() as sess: - # The uniform is supported on [-3, 3] - # Consider partitions the real line in intervals - # ...(-3, -2](-2, -1](-1, 0](0, 1](1, 2](2, 3] ... - # Before cutoffs, the uniform puts a mass of 1/6 in each interval written - # above. Because of cutoffs, the qdist considers intervals and indices - # ...(-infty, -1](-1, 0](0, infty) ... - # -1 0 1 - qdist = distributions.QuantizedDistribution( - distribution=distributions.Uniform(low=-3., high=3.), - low=-1.0, - high=1.0) - - # pmf - cdf_n3, cdf_n2, cdf_n1, cdf_0, cdf_0p5, cdf_1, cdf_10 = sess.run( - qdist.cdf([-3., -2., -1., 0., 0.5, 1.0, 10.0])) - # Uniform had no mass on (-4, -3] or (-3, -2] - self.assertAllClose(0., cdf_n3) - self.assertAllClose(0., cdf_n2) - # Uniform had 1/6 of its mass in each of (-3, -2], and (-2, -1], which - # were collapsed into (-infty, -1], which is now the "-1" interval. - self.assertAllClose(1 / 3, cdf_n1) - # The j=0 interval contained mass from (-3, 0], which is 1/2 of the - # uniform's mass. - self.assertAllClose(1 / 2, cdf_0) - # Adding 0.5 makes no difference because the quantized dist has mass on - # the integers, not in between them. - self.assertAllClose(1 / 2, cdf_0p5) - # After applying the cutoff, all mass was either in the interval - # (0, infty), or below. (0, infty) is the interval indexed by j=1, - # so pmf(1) should equal 1. - self.assertAllClose(1., cdf_1) - # Since no mass of qdist is above 1, - # pmf(10) = P[Y <= 10] = P[Y <= 1] = pmf(1). - self.assertAllClose(1., cdf_10) - - def testQuantizationOfBatchOfUniforms(self): - batch_shape = (5, 5) - with self.cached_session(): - # The uniforms are supported on [0, 10]. The qdist considers the - # intervals - # ... (0, 1](1, 2]...(9, 10]... - # with the intervals displayed above each holding 1 / 10 of the mass. - # The qdist will be defined with no cutoffs, - uniform = distributions.Uniform( - low=array_ops.zeros(batch_shape, dtype=dtypes.float32), - high=10 * array_ops.ones(batch_shape, dtype=dtypes.float32)) - qdist = distributions.QuantizedDistribution( - distribution=uniform, low=None, high=None) - - # x is random integers in {-3,...,12}. - x = rng.randint(-3, 13, size=batch_shape).astype(np.float32) - - # pmf - # qdist.prob(j) = 1 / 10 for j in {1,...,10}, and 0 otherwise, - expected_pmf = (1 / 10) * np.ones(batch_shape) - expected_pmf[x < 1] = 0. - expected_pmf[x > 10] = 0. - self.assertAllClose(expected_pmf, qdist.prob(x).eval()) - - # cdf - # qdist.cdf(j) - # = 0 for j < 1 - # = j / 10, for j in {1,...,10}, - # = 1, for j > 10. - expected_cdf = x.copy() / 10 - expected_cdf[x < 1] = 0. - expected_cdf[x > 10] = 1. - self.assertAllClose(expected_cdf, qdist.cdf(x).eval()) - - def testSamplingFromBatchOfNormals(self): - batch_shape = (2,) - with self.cached_session(): - normal = distributions.Normal( - loc=array_ops.zeros( - batch_shape, dtype=dtypes.float32), - scale=array_ops.ones( - batch_shape, dtype=dtypes.float32)) - - qdist = distributions.QuantizedDistribution( - distribution=normal, low=0., high=None) - - samps = qdist.sample(5000, seed=42) - samps_v = samps.eval() - - # With low = 0, the interval j=0 is (-infty, 0], which holds 1/2 - # of the mass of the normals. - # rtol chosen to be 2x as large as necessary to pass. - self.assertAllClose([0.5, 0.5], (samps_v == 0).mean(axis=0), rtol=0.03) - - # The interval j=1 is (0, 1], which is from the mean to one standard - # deviation out. This should contain 0.6827 / 2 of the mass. - self.assertAllClose( - [0.6827 / 2, 0.6827 / 2], (samps_v == 1).mean(axis=0), rtol=0.03) - - def testSamplesAgreeWithCdfForSamplesOverLargeRange(self): - # Consider the cdf for distribution X, F(x). - # If U ~ Uniform[0, 1], then Y := F^{-1}(U) is distributed like X since - # P[Y <= y] = P[F^{-1}(U) <= y] = P[U <= F(y)] = F(y). - # If F is a bijection, we also have Z = F(X) is Uniform. - # - # Make an exponential with large mean (= 100). This ensures we will get - # quantized values over a large range. This large range allows us to - # pretend that the cdf F is a bijection, and hence F(X) is uniform. - # Note that F cannot be bijection since it is constant between the - # integers. Hence, F(X) (see below) will not be uniform exactly. - with self.cached_session(): - qdist = distributions.QuantizedDistribution( - distribution=distributions.Exponential(rate=0.01)) - # X ~ QuantizedExponential - x = qdist.sample(10000, seed=42) - # Z = F(X), should be Uniform. - z = qdist.cdf(x) - # Compare the CDF of Z to that of a Uniform. - # dist = maximum distance between P[Z <= a] and P[U <= a]. - # We ignore pvalue, since of course this distribution is not exactly, and - # with so many sample points we would get a false fail. - dist, _ = stats.kstest(z.eval(), "uniform") - - # Since the distribution take values (approximately) in [0, 100], the - # cdf should have jumps (approximately) every 1/100 of the way up. - # Assert that the jumps are not more than 2/100. - self.assertLess(dist, 0.02) - - def testSamplesAgreeWithPdfForSamplesOverSmallRange(self): - # Testing that samples and pdf agree for a small range is important because - # it makes sure the bin edges are consistent. - - # Make an exponential with mean 5. - with self.cached_session(): - qdist = distributions.QuantizedDistribution( - distribution=distributions.Exponential(rate=0.2)) - # Standard error should be less than 1 / (2 * sqrt(n_samples)) - n_samples = 10000 - stddev_err_bound = 1 / (2 * np.sqrt(n_samples)) - samps = qdist.sample((n_samples,), seed=42).eval() - # The smallest value the samples can take on is 1, which corresponds to - # the interval (0, 1]. Recall we use ceiling in the sampling definition. - self.assertLess(0.5, samps.min()) - x_vals = np.arange(1, 11).astype(np.float32) - pmf_vals = qdist.prob(x_vals).eval() - for ii in range(10): - self.assertAllClose( - pmf_vals[ii], (samps == x_vals[ii]).mean(), atol=stddev_err_bound) - - def testNormalCdfAndSurvivalFunction(self): - # At integer values, the result should be the same as the standard normal. - batch_shape = (3, 3) - mu = rng.randn(*batch_shape) - sigma = rng.rand(*batch_shape) + 1.0 - with self.cached_session(): - qdist = distributions.QuantizedDistribution( - distribution=distributions.Normal( - loc=mu, scale=sigma)) - sp_normal = stats.norm(mu, sigma) - - x = rng.randint(-5, 5, size=batch_shape).astype(np.float64) - - self.assertAllClose(sp_normal.cdf(x), qdist.cdf(x).eval()) - - self.assertAllClose(sp_normal.sf(x), qdist.survival_function(x).eval()) - - def testNormalLogCdfAndLogSurvivalFunction(self): - # At integer values, the result should be the same as the standard normal. - batch_shape = (3, 3) - mu = rng.randn(*batch_shape) - sigma = rng.rand(*batch_shape) + 1.0 - with self.cached_session(): - qdist = distributions.QuantizedDistribution( - distribution=distributions.Normal( - loc=mu, scale=sigma)) - sp_normal = stats.norm(mu, sigma) - - x = rng.randint(-10, 10, size=batch_shape).astype(np.float64) - - self.assertAllClose(sp_normal.logcdf(x), qdist.log_cdf(x).eval()) - - self.assertAllClose( - sp_normal.logsf(x), qdist.log_survival_function(x).eval()) - - def testNormalProbWithCutoffs(self): - # At integer values, the result should be the same as the standard normal. - with self.cached_session(): - qdist = distributions.QuantizedDistribution( - distribution=distributions.Normal(loc=0., scale=1.), - low=-2., - high=2.) - sm_normal = stats.norm(0., 1.) - # These cutoffs create partitions of the real line, and indices: - # (-inf, -2](-2, -1](-1, 0](0, 1](1, inf) - # -2 -1 0 1 2 - # Test interval (-inf, -2], <--> index -2. - self.assertAllClose(sm_normal.cdf(-2), qdist.prob(-2.).eval(), atol=0) - # Test interval (-2, -1], <--> index -1. - self.assertAllClose( - sm_normal.cdf(-1) - sm_normal.cdf(-2), qdist.prob(-1.).eval(), atol=0) - # Test interval (-1, 0], <--> index 0. - self.assertAllClose( - sm_normal.cdf(0) - sm_normal.cdf(-1), qdist.prob(0.).eval(), atol=0) - # Test interval (1, inf), <--> index 2. - self.assertAllClose(1. - sm_normal.cdf(1), qdist.prob(2.).eval(), atol=0) - - def testNormalLogProbWithCutoffs(self): - # At integer values, the result should be the same as the standard normal. - with self.cached_session(): - qdist = distributions.QuantizedDistribution( - distribution=distributions.Normal(loc=0., scale=1.), - low=-2., - high=2.) - sm_normal = stats.norm(0., 1.) - # These cutoffs create partitions of the real line, and indices: - # (-inf, -2](-2, -1](-1, 0](0, 1](1, inf) - # -2 -1 0 1 2 - # Test interval (-inf, -2], <--> index -2. - self.assertAllClose( - np.log(sm_normal.cdf(-2)), qdist.log_prob(-2.).eval(), atol=0) - # Test interval (-2, -1], <--> index -1. - self.assertAllClose( - np.log(sm_normal.cdf(-1) - sm_normal.cdf(-2)), - qdist.log_prob(-1.).eval(), - atol=0) - # Test interval (-1, 0], <--> index 0. - self.assertAllClose( - np.log(sm_normal.cdf(0) - sm_normal.cdf(-1)), - qdist.log_prob(0.).eval(), - atol=0) - # Test interval (1, inf), <--> index 2. - self.assertAllClose( - np.log(1. - sm_normal.cdf(1)), qdist.log_prob(2.).eval(), atol=0) - - def testLogProbAndGradGivesFiniteResults(self): - for dtype in [np.float32, np.float64]: - g = ops.Graph() - with g.as_default(): - mu = variables.Variable(0., name="mu", dtype=dtype) - sigma = variables.Variable(1., name="sigma", dtype=dtype) - qdist = distributions.QuantizedDistribution( - distribution=distributions.Normal( - loc=mu, scale=sigma)) - x = np.arange(-100, 100, 2).astype(dtype) - proba = qdist.log_prob(x) - grads = gradients_impl.gradients(proba, [mu, sigma]) - with self.session(graph=g): - variables.global_variables_initializer().run() - self._assert_all_finite(proba.eval()) - self._assert_all_finite(grads[0].eval()) - self._assert_all_finite(grads[1].eval()) - - def testProbAndGradGivesFiniteResultsForCommonEvents(self): - with self.cached_session(): - mu = variables.Variable(0.0, name="mu") - sigma = variables.Variable(1.0, name="sigma") - qdist = distributions.QuantizedDistribution( - distribution=distributions.Normal( - loc=mu, scale=sigma)) - x = math_ops.ceil(4 * rng.rand(100).astype(np.float32) - 2) - - variables.global_variables_initializer().run() - - proba = qdist.prob(x) - self._assert_all_finite(proba.eval()) - - grads = gradients_impl.gradients(proba, [mu, sigma]) - self._assert_all_finite(grads[0].eval()) - self._assert_all_finite(grads[1].eval()) - - def testLowerCutoffMustBeBelowUpperCutoffOrWeRaise(self): - with self.cached_session(): - with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, - "must be strictly less"): - _ = distributions.QuantizedDistribution( - distribution=distributions.Normal(loc=0., scale=1.), - low=1., # not strictly less than high. - high=1., - validate_args=True) - # Error detected statically; no need for _.sample().eval() - - def testCutoffsMustBeIntegerValuedIfValidateArgsTrue(self): - with self.cached_session(): - low = array_ops.placeholder(dtypes.float32) - qdist = distributions.QuantizedDistribution( - distribution=distributions.Normal(loc=0., scale=1.), - low=low, - high=10., - validate_args=True) - - self.assertTrue(qdist.validate_args) # Default is True. - with self.assertRaisesOpError("has non-integer components"): - qdist.sample().eval(feed_dict={low: 1.5}) - - def testCutoffsCanBeFloatValuedIfValidateArgsFalse(self): - with self.cached_session(): - qdist = distributions.QuantizedDistribution( - distribution=distributions.Normal( - loc=0., scale=1., validate_args=False), - low=1.5, - high=10.11) - - self.assertFalse(qdist.validate_args) # Default is True. - - # Should not raise - qdist.sample().eval() - - def testDtypeAndShapeInheritedFromBaseDist(self): - batch_shape = (2, 3) - with self.cached_session(): - qdist = distributions.QuantizedDistribution( - distribution=distributions.Normal( - loc=array_ops.zeros(batch_shape), - scale=array_ops.zeros(batch_shape)), - low=1.0, - high=10.0) - - self.assertEqual(batch_shape, qdist.batch_shape) - self.assertAllEqual(batch_shape, qdist.batch_shape_tensor().eval()) - self.assertEqual((), qdist.event_shape) - self.assertAllEqual((), qdist.event_shape_tensor().eval()) - - samps = qdist.sample(10, seed=42) - self.assertEqual((10,) + batch_shape, samps.get_shape()) - self.assertAllEqual((10,) + batch_shape, samps.eval().shape) - - y = rng.randint(0, 5, size=batch_shape).astype(np.float32) - self.assertEqual(batch_shape, qdist.prob(y).get_shape()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/relaxed_bernoulli_test.py b/tensorflow/contrib/distributions/python/kernel_tests/relaxed_bernoulli_test.py deleted file mode 100644 index d4d1a4a9ebd..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/relaxed_bernoulli_test.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the RelaxedBernoulli distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import scipy.special -from tensorflow.contrib.distributions.python.ops import relaxed_bernoulli -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import test_util -from tensorflow.python.platform import test - - -class RelaxedBernoulliTest(test.TestCase): - - def testP(self): - """Tests that parameter P is set correctly. Note that dist.p != dist.pdf.""" - temperature = 1.0 - p = [0.1, 0.4] - dist = relaxed_bernoulli.RelaxedBernoulli(temperature, probs=p) - with self.cached_session(): - self.assertAllClose(p, dist.probs.eval()) - - def testLogits(self): - temperature = 2.0 - logits = [-42., 42.] - dist = relaxed_bernoulli.RelaxedBernoulli(temperature, logits=logits) - with self.cached_session(): - self.assertAllClose(logits, dist.logits.eval()) - - with self.cached_session(): - self.assertAllClose(scipy.special.expit(logits), dist.probs.eval()) - - p = [0.01, 0.99, 0.42] - dist = relaxed_bernoulli.RelaxedBernoulli(temperature, probs=p) - with self.cached_session(): - self.assertAllClose(scipy.special.logit(p), dist.logits.eval()) - - def testInvalidP(self): - temperature = 1.0 - invalid_ps = [1.01, 2.] - for p in invalid_ps: - with self.cached_session(): - with self.assertRaisesOpError("probs has components greater than 1"): - dist = relaxed_bernoulli.RelaxedBernoulli(temperature, - probs=p, - validate_args=True) - dist.probs.eval() - - invalid_ps = [-0.01, -3.] - for p in invalid_ps: - with self.cached_session(): - with self.assertRaisesOpError("Condition x >= 0"): - dist = relaxed_bernoulli.RelaxedBernoulli(temperature, - probs=p, - validate_args=True) - dist.probs.eval() - - valid_ps = [0.0, 0.5, 1.0] - for p in valid_ps: - with self.cached_session(): - dist = relaxed_bernoulli.RelaxedBernoulli(temperature, - probs=p) - self.assertEqual(p, dist.probs.eval()) - - def testShapes(self): - with self.cached_session(): - for batch_shape in ([], [1], [2, 3, 4]): - temperature = 1.0 - p = np.random.random(batch_shape).astype(np.float32) - dist = relaxed_bernoulli.RelaxedBernoulli(temperature, probs=p) - self.assertAllEqual(batch_shape, dist.batch_shape.as_list()) - self.assertAllEqual(batch_shape, dist.batch_shape_tensor().eval()) - self.assertAllEqual([], dist.event_shape.as_list()) - self.assertAllEqual([], dist.event_shape_tensor().eval()) - - def testZeroTemperature(self): - """If validate_args, raises InvalidArgumentError when temperature is 0.""" - temperature = constant_op.constant(0.0) - p = constant_op.constant([0.1, 0.4]) - with self.assertRaisesWithPredicateMatch(errors_impl.InvalidArgumentError, - "x > 0 did not hold"): - _ = relaxed_bernoulli.RelaxedBernoulli( - temperature, probs=p, validate_args=True) - # Error detected statically; no need to run the op. - - def testDtype(self): - temperature = constant_op.constant(1.0, dtype=dtypes.float32) - p = constant_op.constant([0.1, 0.4], dtype=dtypes.float32) - dist = relaxed_bernoulli.RelaxedBernoulli(temperature, probs=p) - self.assertEqual(dist.dtype, dtypes.float32) - self.assertEqual(dist.dtype, dist.sample(5).dtype) - self.assertEqual(dist.probs.dtype, dist.prob([0.0]).dtype) - self.assertEqual(dist.probs.dtype, dist.log_prob([0.0]).dtype) - - temperature = constant_op.constant(1.0, dtype=dtypes.float64) - p = constant_op.constant([0.1, 0.4], dtype=dtypes.float64) - dist64 = relaxed_bernoulli.RelaxedBernoulli(temperature, probs=p) - self.assertEqual(dist64.dtype, dtypes.float64) - self.assertEqual(dist64.dtype, dist64.sample(5).dtype) - - def testLogProb(self): - with self.cached_session(): - t = np.array(1.0, dtype=np.float64) - p = np.array(0.1, dtype=np.float64) # P(x=1) - dist = relaxed_bernoulli.RelaxedBernoulli(t, probs=p) - xs = np.array([0.1, 0.3, 0.5, 0.9], dtype=np.float64) - # analytical density from Maddison et al. 2016 - alpha = np.array(p/(1-p), dtype=np.float64) - expected_log_pdf = (np.log(t) + np.log(alpha) + - (-t-1)*(np.log(xs)+np.log(1-xs)) - - 2*np.log(alpha*np.power(xs, -t) + np.power(1-xs, -t))) - log_pdf = dist.log_prob(xs).eval() - self.assertAllClose(expected_log_pdf, log_pdf) - - def testBoundaryConditions(self): - with self.cached_session(): - temperature = 1e-2 - dist = relaxed_bernoulli.RelaxedBernoulli(temperature, probs=1.0) - self.assertAllClose(np.nan, dist.log_prob(0.0).eval()) - self.assertAllClose([np.nan], [dist.log_prob(1.0).eval()]) - - @test_util.disable_xla( - "TODO(b/141092326): prevent negative values in Sigmoid on XLA:CPU") - def testSampleN(self): - """mean of quantized samples still approximates the Bernoulli mean.""" - with self.cached_session(): - temperature = 1e-2 - p = [0.2, 0.6, 0.5] - dist = relaxed_bernoulli.RelaxedBernoulli(temperature, probs=p) - n = 10000 - samples = dist.sample(n) - self.assertEqual(samples.dtype, dtypes.float32) - sample_values = samples.eval() - self.assertTrue(np.all(sample_values >= 0)) - self.assertTrue(np.all(sample_values <= 1)) - - frac_ones_like = np.sum(sample_values >= 0.5, axis=0)/n - self.assertAllClose(p, frac_ones_like, atol=1e-2) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py b/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py deleted file mode 100644 index ff13c2decc5..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py +++ /dev/null @@ -1,177 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Relaxed One-Hot Categorical distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy.special import gamma - -from tensorflow.contrib.distributions.python.ops import relaxed_onehot_categorical -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import test - - -def make_relaxed_categorical(batch_shape, num_classes, dtype=dtypes.float32): - logits = random_ops.random_uniform( - list(batch_shape) + [num_classes], -10, 10, dtype=dtype) - 50. - temperatures = random_ops.random_uniform( - list(batch_shape), 0.1, 10, dtype=dtypes.float32) - return relaxed_onehot_categorical.RelaxedOneHotCategorical( - temperatures, logits, dtype=dtype) - - -class ExpRelaxedOneHotCategoricalTest(test.TestCase): - - def testP(self): - temperature = 1.0 - logits = [2.0, 3.0, -4.0] - dist = relaxed_onehot_categorical.ExpRelaxedOneHotCategorical(temperature, - logits) - expected_p = np.exp(logits)/np.sum(np.exp(logits)) - with self.cached_session(): - self.assertAllClose(expected_p, dist.probs.eval()) - self.assertAllEqual([3], dist.probs.get_shape()) - - def testPdf(self): - temperature = .4 - logits = [.3, .1, .4] - k = len(logits) - p = np.exp(logits)/np.sum(np.exp(logits)) - dist = relaxed_onehot_categorical.ExpRelaxedOneHotCategorical(temperature, - logits) - with self.cached_session(): - x = dist.sample().eval() - # analytical ExpConcrete density presented in Maddison et al. 2016 - prod_term = p*np.exp(-temperature * x) - expected_pdf = (gamma(k) * np.power(temperature, k-1) * - np.prod(prod_term/np.sum(prod_term))) - pdf = dist.prob(x).eval() - self.assertAllClose(expected_pdf, pdf) - - -class RelaxedOneHotCategoricalTest(test.TestCase): - - def testLogits(self): - temperature = 1.0 - logits = [2.0, 3.0, -4.0] - dist = relaxed_onehot_categorical.RelaxedOneHotCategorical(temperature, - logits) - with self.cached_session(): - # check p for ExpRelaxed base distribution - self.assertAllClose(logits, dist._distribution.logits.eval()) - self.assertAllEqual([3], dist._distribution.logits.get_shape()) - - def testSample(self): - temperature = 1.4 - with self.cached_session(): - # single logit - logits = [.3, .1, .4] - dist = relaxed_onehot_categorical.RelaxedOneHotCategorical(temperature, - logits) - self.assertAllEqual([3], dist.sample().eval().shape) - self.assertAllEqual([5, 3], dist.sample(5).eval().shape) - # multiple distributions - logits = [[2.0, 3.0, -4.0], [.3, .1, .4]] - dist = relaxed_onehot_categorical.RelaxedOneHotCategorical(temperature, - logits) - self.assertAllEqual([2, 3], dist.sample().eval().shape) - self.assertAllEqual([5, 2, 3], dist.sample(5).eval().shape) - # multiple distributions - logits = np.random.uniform(size=(4, 1, 3)).astype(np.float32) - dist = relaxed_onehot_categorical.RelaxedOneHotCategorical(temperature, - logits) - self.assertAllEqual([4, 1, 3], dist.sample().eval().shape) - self.assertAllEqual([5, 4, 1, 3], dist.sample(5).eval().shape) - - def testPdf(self): - def analytical_pdf(x, temperature, logits): - # analytical density of RelaxedOneHotCategorical - temperature = np.reshape(temperature, (-1, 1)) - if len(x.shape) == 1: - x = np.expand_dims(x, 0) - k = logits.shape[1] - p = np.exp(logits)/np.sum(np.exp(logits), axis=1, keepdims=True) - term1 = gamma(k)*np.power(temperature, k-1) - term2 = np.sum(p/(np.power(x, temperature)), axis=1, keepdims=True) - term3 = np.prod(p/(np.power(x, temperature+1)), axis=1, keepdims=True) - expected_pdf = term1*np.power(term2, -k)*term3 - return expected_pdf - - with self.cached_session(): - temperature = .4 - logits = np.array([[.3, .1, .4]]).astype(np.float32) - dist = relaxed_onehot_categorical.RelaxedOneHotCategorical(temperature, - logits) - x = dist.sample().eval() - pdf = dist.prob(x).eval() - expected_pdf = analytical_pdf(x, temperature, logits) - self.assertAllClose(expected_pdf.flatten(), pdf, rtol=1e-4) - - # variable batch size - logits = np.array([[.3, .1, .4], [.6, -.1, 2.]]).astype(np.float32) - temperatures = np.array([0.4, 2.3]).astype(np.float32) - dist = relaxed_onehot_categorical.RelaxedOneHotCategorical(temperatures, - logits) - x = dist.sample().eval() - pdf = dist.prob(x).eval() - expected_pdf = analytical_pdf(x, temperatures, logits) - self.assertAllClose(expected_pdf.flatten(), pdf, rtol=1e-4) - - def testShapes(self): - with self.cached_session(): - for batch_shape in ([], [1], [2, 3, 4]): - dist = make_relaxed_categorical(batch_shape, 10) - self.assertAllEqual(batch_shape, dist.batch_shape.as_list()) - self.assertAllEqual(batch_shape, dist.batch_shape_tensor().eval()) - self.assertAllEqual([10], dist.event_shape_tensor().eval()) - self.assertAllEqual([10], dist.event_shape_tensor().eval()) - - for batch_shape in ([], [1], [2, 3, 4]): - dist = make_relaxed_categorical( - batch_shape, constant_op.constant(10, dtype=dtypes.int32)) - self.assertAllEqual(len(batch_shape), dist.batch_shape.ndims) - self.assertAllEqual(batch_shape, dist.batch_shape_tensor().eval()) - self.assertAllEqual([10], dist.event_shape_tensor().eval()) - self.assertAllEqual([10], dist.event_shape_tensor().eval()) - - def testUnknownShape(self): - with self.cached_session(): - logits_pl = array_ops.placeholder(dtypes.float32) - temperature = 1.0 - dist = relaxed_onehot_categorical.ExpRelaxedOneHotCategorical(temperature, - logits_pl) - with self.cached_session(): - feed_dict = {logits_pl: [.3, .1, .4]} - self.assertAllEqual([3], dist.sample().eval(feed_dict=feed_dict).shape) - self.assertAllEqual([5, 3], - dist.sample(5).eval(feed_dict=feed_dict).shape) - - def testDTypes(self): - # check that sampling and log_prob work for a range of dtypes - with self.cached_session(): - for dtype in (dtypes.float16, dtypes.float32, dtypes.float64): - logits = random_ops.random_uniform(shape=[3, 3], dtype=dtype) - dist = relaxed_onehot_categorical.RelaxedOneHotCategorical( - temperature=0.5, logits=logits) - dist.log_prob(dist.sample()) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/sample_stats_test.py b/tensorflow/contrib/distributions/python/kernel_tests/sample_stats_test.py deleted file mode 100644 index f2be3bdb656..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/sample_stats_test.py +++ /dev/null @@ -1,462 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Sample Stats Ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import sample_stats -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - -rng = np.random.RandomState(0) - - -class _AutoCorrelationTest(object): - - @property - def use_static_shape(self): - raise NotImplementedError("Subclass failed to implement `use_static_shape`") - - @property - def dtype(self): - raise NotImplementedError("Subclass failed to implement `dtype`.") - - def test_constant_sequence_axis_0_max_lags_none_center_false(self): - x_ = np.array([[0., 0., 0.], - [1., 1., 1.]]).astype(self.dtype) - x_ph = array_ops.placeholder_with_default( - input=x_, - shape=x_.shape if self.use_static_shape else None) - with self.cached_session() as sess: - # Setting normalize = True means we divide by zero. - auto_corr = sample_stats.auto_correlation( - x_ph, axis=1, center=False, normalize=False) - if self.use_static_shape: - self.assertEqual((2, 3), auto_corr.shape) - auto_corr_ = sess.run(auto_corr) - self.assertAllClose( - [[0., 0., 0.], - [1., 1., 1.]], auto_corr_) - - def test_constant_sequence_axis_0_max_lags_none_center_true(self): - x_ = np.array([[0., 0., 0.], - [1., 1., 1.]]).astype(self.dtype) - x_ph = array_ops.placeholder_with_default( - input=x_, - shape=x_.shape if self.use_static_shape else None) - with self.cached_session() as sess: - # Setting normalize = True means we divide by zero. - auto_corr = sample_stats.auto_correlation( - x_ph, axis=1, normalize=False, center=True) - if self.use_static_shape: - self.assertEqual((2, 3), auto_corr.shape) - auto_corr_ = sess.run(auto_corr) - self.assertAllClose( - [[0., 0., 0.], - [0., 0., 0.]], auto_corr_) - - def check_results_versus_brute_force( - self, x, axis, max_lags, center, normalize): - """Compute auto-correlation by brute force, then compare to tf result.""" - # Brute for auto-corr -- avoiding fft and transpositions. - axis_len = x.shape[axis] - if max_lags is None: - max_lags = axis_len - 1 - else: - max_lags = min(axis_len - 1, max_lags) - auto_corr_at_lag = [] - if center: - x -= x.mean(axis=axis, keepdims=True) - for m in range(max_lags + 1): - auto_corr_at_lag.append(( - np.take(x, indices=range(0, axis_len - m), axis=axis) * - np.conj(np.take(x, indices=range(m, axis_len), axis=axis)) - ).mean(axis=axis, keepdims=True)) - rxx = np.concatenate(auto_corr_at_lag, axis=axis) - if normalize: - rxx /= np.take(rxx, [0], axis=axis) - - x_ph = array_ops.placeholder_with_default( - x, shape=x.shape if self.use_static_shape else None) - with self.cached_session(): - auto_corr = sample_stats.auto_correlation( - x_ph, axis=axis, max_lags=max_lags, center=center, - normalize=normalize) - if self.use_static_shape: - output_shape = list(x.shape) - output_shape[axis] = max_lags + 1 - self.assertAllEqual(output_shape, auto_corr.shape) - self.assertAllClose(rxx, auto_corr.eval(), rtol=1e-5, atol=1e-5) - - def test_axis_n1_center_false_max_lags_none(self): - x = rng.randn(2, 3, 4).astype(self.dtype) - if self.dtype in [np.complex64]: - x = 1j * rng.randn(2, 3, 4).astype(self.dtype) - self.check_results_versus_brute_force( - x, axis=-1, max_lags=None, center=False, normalize=False) - - def test_axis_n2_center_false_max_lags_none(self): - x = rng.randn(3, 4, 5).astype(self.dtype) - if self.dtype in [np.complex64]: - x = 1j * rng.randn(3, 4, 5).astype(self.dtype) - self.check_results_versus_brute_force( - x, axis=-2, max_lags=None, center=False, normalize=False) - - def test_axis_n1_center_false_max_lags_none_normalize_true(self): - x = rng.randn(2, 3, 4).astype(self.dtype) - if self.dtype in [np.complex64]: - x = 1j * rng.randn(2, 3, 4).astype(self.dtype) - self.check_results_versus_brute_force( - x, axis=-1, max_lags=None, center=False, normalize=True) - - def test_axis_n2_center_false_max_lags_none_normalize_true(self): - x = rng.randn(3, 4, 5).astype(self.dtype) - if self.dtype in [np.complex64]: - x = 1j * rng.randn(3, 4, 5).astype(self.dtype) - self.check_results_versus_brute_force( - x, axis=-2, max_lags=None, center=False, normalize=True) - - def test_axis_0_center_true_max_lags_none(self): - x = rng.randn(3, 4, 5).astype(self.dtype) - if self.dtype in [np.complex64]: - x = 1j * rng.randn(3, 4, 5).astype(self.dtype) - self.check_results_versus_brute_force( - x, axis=0, max_lags=None, center=True, normalize=False) - - def test_axis_2_center_true_max_lags_1(self): - x = rng.randn(3, 4, 5).astype(self.dtype) - if self.dtype in [np.complex64]: - x = 1j * rng.randn(3, 4, 5).astype(self.dtype) - self.check_results_versus_brute_force( - x, axis=2, max_lags=1, center=True, normalize=False) - - def test_axis_2_center_true_max_lags_100(self): - # There are less than 100 elements in axis 2, so expect we get back an array - # the same size as x, despite having asked for 100 lags. - x = rng.randn(3, 4, 5).astype(self.dtype) - if self.dtype in [np.complex64]: - x = 1j * rng.randn(3, 4, 5).astype(self.dtype) - self.check_results_versus_brute_force( - x, axis=2, max_lags=100, center=True, normalize=False) - - def test_long_orthonormal_sequence_has_corr_length_0(self): - l = 10000 - x = rng.randn(l).astype(self.dtype) - x_ph = array_ops.placeholder_with_default( - x, shape=(l,) if self.use_static_shape else None) - with self.cached_session(): - rxx = sample_stats.auto_correlation( - x_ph, max_lags=l // 2, center=True, normalize=False) - if self.use_static_shape: - self.assertAllEqual((l // 2 + 1,), rxx.shape) - rxx_ = rxx.eval() - # OSS CPU FFT has some accuracy issues, so this tolerance is a bit bad. - self.assertAllClose(1., rxx_[0], rtol=0.05) - # The maximal error in the rest of the sequence is not great. - self.assertAllClose(np.zeros(l // 2), rxx_[1:], atol=0.1) - # The mean error in the rest is ok, actually 0.008 when I tested it. - self.assertLess(np.abs(rxx_[1:]).mean(), 0.02) - - def test_step_function_sequence(self): - # x jumps to new random value every 10 steps. So correlation length = 10. - x = (rng.randint(-10, 10, size=(1000, 1)) - * np.ones((1, 10))).ravel().astype(self.dtype) - x_ph = array_ops.placeholder_with_default( - x, shape=(1000 * 10,) if self.use_static_shape else None) - with self.cached_session(): - rxx = sample_stats.auto_correlation( - x_ph, max_lags=1000 * 10 // 2, center=True, normalize=False) - if self.use_static_shape: - self.assertAllEqual((1000 * 10 // 2 + 1,), rxx.shape) - rxx_ = rxx.eval() - rxx_ /= rxx_[0] - # Expect positive correlation for the first 10 lags, then significantly - # smaller negative. - self.assertGreater(rxx_[:10].min(), 0) - self.assertGreater(rxx_[9], 5 * rxx_[10:20].mean()) - # RXX should be decreasing for the first 10 lags. - diff = np.diff(rxx_) - self.assertLess(diff[:10].max(), 0) - - def test_normalization(self): - l = 10000 - x = 3 * rng.randn(l).astype(self.dtype) - x_ph = array_ops.placeholder_with_default( - x, shape=(l,) if self.use_static_shape else None) - with self.cached_session(): - rxx = sample_stats.auto_correlation( - x_ph, max_lags=l // 2, center=True, normalize=True) - if self.use_static_shape: - self.assertAllEqual((l // 2 + 1,), rxx.shape) - rxx_ = rxx.eval() - # Note that RXX[0] = 1, despite the fact that E[X^2] = 9, and this is - # due to normalize=True. - # OSS CPU FFT has some accuracy issues, so this tolerance is a bit bad. - self.assertAllClose(1., rxx_[0], rtol=0.05) - # The maximal error in the rest of the sequence is not great. - self.assertAllClose(np.zeros(l // 2), rxx_[1:], atol=0.1) - # The mean error in the rest is ok, actually 0.008 when I tested it. - self.assertLess(np.abs(rxx_[1:]).mean(), 0.02) - - -class AutoCorrelationTestStaticShapeFloat32(test.TestCase, - _AutoCorrelationTest): - - @property - def dtype(self): - return np.float32 - - @property - def use_static_shape(self): - return True - - -class AutoCorrelationTestStaticShapeComplex64(test.TestCase, - _AutoCorrelationTest): - - @property - def dtype(self): - return np.complex64 - - @property - def use_static_shape(self): - return True - - -class AutoCorrelationTestDynamicShapeFloat32(test.TestCase, - _AutoCorrelationTest): - - @property - def dtype(self): - return np.float32 - - @property - def use_static_shape(self): - return False - - -class PercentileTestWithLowerInterpolation(test.TestCase): - - _interpolation = "lower" - - def test_one_dim_odd_input(self): - x = [1., 5., 3., 2., 4.] - for q in [0, 10, 25, 49.9, 50, 50.01, 90, 95, 100]: - expected_percentile = np.percentile( - x, q=q, interpolation=self._interpolation, axis=0) - with self.cached_session(): - pct = sample_stats.percentile( - x, q=q, interpolation=self._interpolation, axis=[0]) - self.assertAllEqual((), pct.get_shape()) - self.assertAllClose(expected_percentile, pct.eval()) - - def test_one_dim_even_input(self): - x = [1., 5., 3., 2., 4., 5.] - for q in [0, 10, 25, 49.9, 50, 50.01, 90, 95, 100]: - expected_percentile = np.percentile( - x, q=q, interpolation=self._interpolation) - with self.cached_session(): - pct = sample_stats.percentile(x, q=q, interpolation=self._interpolation) - self.assertAllEqual((), pct.get_shape()) - self.assertAllClose(expected_percentile, pct.eval()) - - def test_two_dim_odd_input_axis_0(self): - x = np.array([[-1., 50., -3.5, 2., -1], [0., 0., 3., 2., 4.]]).T - for q in [0, 10, 25, 49.9, 50, 50.01, 90, 95, 100]: - expected_percentile = np.percentile( - x, q=q, interpolation=self._interpolation, axis=0) - with self.cached_session(): - # Get dim 1 with negative and positive indices. - pct_neg_index = sample_stats.percentile( - x, q=q, interpolation=self._interpolation, axis=[0]) - pct_pos_index = sample_stats.percentile( - x, q=q, interpolation=self._interpolation, axis=[0]) - self.assertAllEqual((2,), pct_neg_index.get_shape()) - self.assertAllEqual((2,), pct_pos_index.get_shape()) - self.assertAllClose(expected_percentile, pct_neg_index.eval()) - self.assertAllClose(expected_percentile, pct_pos_index.eval()) - - def test_two_dim_even_axis_0(self): - x = np.array([[1., 2., 4., 50.], [1., 2., -4., 5.]]).T - for q in [0, 10, 25, 49.9, 50, 50.01, 90, 95, 100]: - expected_percentile = np.percentile( - x, q=q, interpolation=self._interpolation, axis=0) - with self.cached_session(): - pct = sample_stats.percentile( - x, q=q, interpolation=self._interpolation, axis=[0]) - self.assertAllEqual((2,), pct.get_shape()) - self.assertAllClose(expected_percentile, pct.eval()) - - def test_two_dim_even_input_and_keep_dims_true(self): - x = np.array([[1., 2., 4., 50.], [1., 2., -4., 5.]]).T - for q in [0, 10, 25, 49.9, 50, 50.01, 90, 95, 100]: - expected_percentile = np.percentile( - x, q=q, interpolation=self._interpolation, keepdims=True, axis=0) - with self.cached_session(): - pct = sample_stats.percentile( - x, - q=q, - interpolation=self._interpolation, - keep_dims=True, - axis=[0]) - self.assertAllEqual((1, 2), pct.get_shape()) - self.assertAllClose(expected_percentile, pct.eval()) - - def test_four_dimensional_input(self): - x = rng.rand(2, 3, 4, 5) - for axis in [None, 0, 1, -2, (0,), (-1,), (-1, 1), (3, 1), (-3, 0)]: - expected_percentile = np.percentile( - x, q=0.77, interpolation=self._interpolation, axis=axis) - with self.cached_session(): - pct = sample_stats.percentile( - x, - q=0.77, - interpolation=self._interpolation, - axis=axis) - self.assertAllEqual(expected_percentile.shape, pct.get_shape()) - self.assertAllClose(expected_percentile, pct.eval()) - - def test_four_dimensional_input_and_keepdims(self): - x = rng.rand(2, 3, 4, 5) - for axis in [None, 0, 1, -2, (0,), (-1,), (-1, 1), (3, 1), (-3, 0)]: - expected_percentile = np.percentile( - x, - q=0.77, - interpolation=self._interpolation, - axis=axis, - keepdims=True) - with self.cached_session(): - pct = sample_stats.percentile( - x, - q=0.77, - interpolation=self._interpolation, - axis=axis, - keep_dims=True) - self.assertAllEqual(expected_percentile.shape, pct.get_shape()) - self.assertAllClose(expected_percentile, pct.eval()) - - def test_four_dimensional_input_x_static_ndims_but_dynamic_sizes(self): - x = rng.rand(2, 3, 4, 5) - x_ph = array_ops.placeholder(dtypes.float64, shape=[None, None, None, None]) - for axis in [None, 0, 1, -2, (0,), (-1,), (-1, 1), (3, 1), (-3, 0)]: - expected_percentile = np.percentile( - x, q=0.77, interpolation=self._interpolation, axis=axis) - with self.cached_session(): - pct = sample_stats.percentile( - x_ph, - q=0.77, - interpolation=self._interpolation, - axis=axis) - self.assertAllClose(expected_percentile, pct.eval(feed_dict={x_ph: x})) - - def test_four_dimensional_input_and_keepdims_x_static_ndims_dynamic_sz(self): - x = rng.rand(2, 3, 4, 5) - x_ph = array_ops.placeholder(dtypes.float64, shape=[None, None, None, None]) - for axis in [None, 0, 1, -2, (0,), (-1,), (-1, 1), (3, 1), (-3, 0)]: - expected_percentile = np.percentile( - x, - q=0.77, - interpolation=self._interpolation, - axis=axis, - keepdims=True) - with self.cached_session(): - pct = sample_stats.percentile( - x_ph, - q=0.77, - interpolation=self._interpolation, - axis=axis, - keep_dims=True) - self.assertAllClose(expected_percentile, pct.eval(feed_dict={x_ph: x})) - - def test_with_integer_dtype(self): - x = [1, 5, 3, 2, 4] - for q in [0, 10, 25, 49.9, 50, 50.01, 90, 95, 100]: - expected_percentile = np.percentile( - x, q=q, interpolation=self._interpolation) - with self.cached_session(): - pct = sample_stats.percentile(x, q=q, interpolation=self._interpolation) - self.assertEqual(dtypes.int32, pct.dtype) - self.assertAllEqual((), pct.get_shape()) - self.assertAllClose(expected_percentile, pct.eval()) - - -class PercentileTestWithHigherInterpolation( - PercentileTestWithLowerInterpolation): - - _interpolation = "higher" - - -class PercentileTestWithNearestInterpolation(test.TestCase): - """Test separately because np.round and tf.round make different choices.""" - - _interpolation = "nearest" - - def test_one_dim_odd_input(self): - x = [1., 5., 3., 2., 4.] - for q in [0, 10.1, 25.1, 49.9, 50.1, 50.01, 89, 100]: - expected_percentile = np.percentile( - x, q=q, interpolation=self._interpolation) - with self.cached_session(): - pct = sample_stats.percentile(x, q=q, interpolation=self._interpolation) - self.assertAllEqual((), pct.get_shape()) - self.assertAllClose(expected_percentile, pct.eval()) - - def test_one_dim_even_input(self): - x = [1., 5., 3., 2., 4., 5.] - for q in [0, 10.1, 25.1, 49.9, 50.1, 50.01, 89, 100]: - expected_percentile = np.percentile( - x, q=q, interpolation=self._interpolation) - with self.cached_session(): - pct = sample_stats.percentile(x, q=q, interpolation=self._interpolation) - self.assertAllEqual((), pct.get_shape()) - self.assertAllClose(expected_percentile, pct.eval()) - - def test_invalid_interpolation_raises(self): - x = [1., 5., 3., 2., 4.] - with self.assertRaisesRegexp(ValueError, "interpolation"): - sample_stats.percentile(x, q=0.5, interpolation="bad") - - def test_vector_q_raises_static(self): - x = [1., 5., 3., 2., 4.] - with self.assertRaisesRegexp(ValueError, "Expected.*ndims"): - sample_stats.percentile(x, q=[0.5]) - - def test_vector_q_raises_dynamic(self): - x = [1., 5., 3., 2., 4.] - q_ph = array_ops.placeholder(dtypes.float32) - pct = sample_stats.percentile(x, q=q_ph, validate_args=True) - with self.cached_session(): - with self.assertRaisesOpError("rank"): - pct.eval(feed_dict={q_ph: [0.5]}) - - def test_finds_max_of_long_array(self): - # d - 1 == d in float32 and d = 3e7. - # So this test only passes if we use double for the percentile indices. - # If float is used, it fails with InvalidArgumentError about an index out of - # bounds. - x = math_ops.linspace(0., 3e7, num=int(3e7)) - with self.cached_session(): - minval = sample_stats.percentile(x, q=0, validate_args=True) - self.assertAllEqual(0, minval.eval()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/seed_stream_test.py b/tensorflow/contrib/distributions/python/kernel_tests/seed_stream_test.py deleted file mode 100644 index b91a610acf1..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/seed_stream_test.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the SeedStream class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import seed_stream -from tensorflow.python.platform import test - - -class SeedStreamTest(test.TestCase): - - def assertAllUnique(self, items): - self.assertEqual(len(items), len(set(items))) - - def testNonRepetition(self): - # The probability of repetitions in a short stream from a correct - # PRNG is negligible; this test catches bugs that prevent state - # updates. - strm = seed_stream.SeedStream(seed=4, salt="salt") - output = [strm() for _ in range(50)] - self.assertEqual(sorted(output), sorted(list(set(output)))) - - def testReproducibility(self): - strm1 = seed_stream.SeedStream(seed=4, salt="salt") - strm2 = seed_stream.SeedStream(seed=4, salt="salt") - strm3 = seed_stream.SeedStream(seed=4, salt="salt") - outputs = [strm1() for _ in range(50)] - self.assertEqual(outputs, [strm2() for _ in range(50)]) - self.assertEqual(outputs, [strm3() for _ in range(50)]) - - def testSeededDistinctness(self): - strm1 = seed_stream.SeedStream(seed=4, salt="salt") - strm2 = seed_stream.SeedStream(seed=5, salt="salt") - self.assertAllUnique( - [strm1() for _ in range(50)] + [strm2() for _ in range(50)]) - - def testSaltedDistinctness(self): - strm1 = seed_stream.SeedStream(seed=4, salt="salt") - strm2 = seed_stream.SeedStream(seed=4, salt="another salt") - self.assertAllUnique( - [strm1() for _ in range(50)] + [strm2() for _ in range(50)]) - - def testNestingRobustness(self): - # SeedStreams started from generated seeds should not collide with - # the master or with each other, even if the salts are the same. - strm1 = seed_stream.SeedStream(seed=4, salt="salt") - strm2 = seed_stream.SeedStream(strm1(), salt="salt") - strm3 = seed_stream.SeedStream(strm1(), salt="salt") - outputs = [strm1() for _ in range(50)] - self.assertAllUnique( - outputs + [strm2() for _ in range(50)] + [strm3() for _ in range(50)]) - - def testInitFromOtherSeedStream(self): - strm1 = seed_stream.SeedStream(seed=4, salt="salt") - strm2 = seed_stream.SeedStream(strm1, salt="salt") - strm3 = seed_stream.SeedStream(strm1, salt="another salt") - out1 = [strm1() for _ in range(50)] - out2 = [strm2() for _ in range(50)] - out3 = [strm3() for _ in range(50)] - self.assertAllEqual(out1, out2) - self.assertAllUnique(out1 + out3) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/shape_test.py b/tensorflow/contrib/distributions/python/kernel_tests/shape_test.py deleted file mode 100644 index a4d2aa381cc..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/shape_test.py +++ /dev/null @@ -1,694 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for ShapeUtil.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops.shape import _DistributionShape -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - -_empty_shape = np.array([], dtype=np.int32) - - -def _eval(x): - if hasattr(x, "__iter__"): - return [x.eval() for x in x] - return x.eval() - - -def _constant(x): - if hasattr(x, "__iter__"): - return [tensor_util.constant_value(x) for x in x] - return tensor_util.constant_value(x) - - -class MakeBatchReadyTest(test.TestCase): - - def setUp(self): - self._rng = np.random.RandomState(42) - - def _random_sample(self, sample_shape, dtype=np.float32): - return self._rng.random_sample(sample_shape).astype(dtype) - - def _get_expected(self, x, batch_ndims, event_ndims, expand_batch_dim): - # Cast as int32 array explicitly, since an empty x.shape defaults - # to float64, and we can't index as float64 in numpy 1.12+. - x_shape = np.array(x.shape, dtype=np.int32) - n = x.ndim - batch_ndims - event_ndims - sample_shape = x_shape[:n] - y = np.reshape(x, np.concatenate([[-1], x_shape[n:]], 0)) - y = np.transpose(y, np.roll(np.arange(y.ndim), -1)) - if event_ndims == 0: - y = y[..., np.newaxis, :] - if batch_ndims == 0 and expand_batch_dim: - y = y[np.newaxis, ...] - return y, sample_shape - - def _build_graph(self, x, batch_ndims, event_ndims, expand_batch_dim): - shaper = _DistributionShape(batch_ndims=batch_ndims, - event_ndims=event_ndims) - y, sample_shape = shaper.make_batch_of_event_sample_matrices( - x, expand_batch_dim=expand_batch_dim) - should_be_x_value = shaper.undo_make_batch_of_event_sample_matrices( - y, sample_shape, expand_batch_dim=expand_batch_dim) - return y, sample_shape, should_be_x_value - - def _test_dynamic(self, x, batch_ndims, event_ndims, expand_batch_dim=True): - with self.cached_session() as sess: - x_pl = array_ops.placeholder(x.dtype) - batch_ndims_pl = array_ops.placeholder(dtypes.int32) - event_ndims_pl = array_ops.placeholder(dtypes.int32) - [y_, sample_shape_, should_be_x_value_] = sess.run( - self._build_graph( - x_pl, batch_ndims_pl, event_ndims_pl, expand_batch_dim), - feed_dict={ - x_pl: x, - batch_ndims_pl: batch_ndims, - event_ndims_pl: event_ndims}) - expected_y, expected_sample_shape = self._get_expected( - x, batch_ndims, event_ndims, expand_batch_dim) - self.assertAllEqual(expected_sample_shape, sample_shape_) - self.assertAllEqual(expected_y, y_) - self.assertAllEqual(x, should_be_x_value_) - - def _test_static(self, x, batch_ndims, event_ndims, expand_batch_dim): - with self.cached_session() as sess: - [y_, sample_shape_, should_be_x_value_] = sess.run( - self._build_graph(x, batch_ndims, event_ndims, expand_batch_dim)) - expected_y, expected_sample_shape = self._get_expected( - x, batch_ndims, event_ndims, expand_batch_dim) - self.assertAllEqual(expected_sample_shape, sample_shape_) - self.assertAllEqual(expected_y, y_) - self.assertAllEqual(x, should_be_x_value_) - - # Group 1a: Static scalar input. - - def testStaticScalarNdims00ExpandNo(self): - self._test_static(x=self._random_sample([]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=False) - - def testStaticScalarNdims00ExpandYes(self): - self._test_static(x=self._random_sample([]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=True) - - def testStaticScalarNdims01ExpandNo(self): - with self.assertRaises(ValueError): - self._test_static(x=self._random_sample([]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=False) - - def testStaticScalarNdims01ExpandYes(self): - with self.assertRaises(ValueError): - self._test_static(x=self._random_sample([]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=True) - - def testStaticScalarNdims10ExpandNo(self): - with self.assertRaises(ValueError): - self._test_static(x=self._random_sample([]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=False) - - def testStaticScalarNdims10ExpandYes(self): - with self.assertRaises(ValueError): - self._test_static(x=self._random_sample([]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=True) - - def testStaticScalarNdims11ExpandNo(self): - with self.assertRaises(ValueError): - self._test_static(x=self._random_sample([]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=False) - - def testStaticScalarNdims11ExpandYes(self): - with self.assertRaises(ValueError): - self._test_static(x=self._random_sample([]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=True) - - # Group 1b: Dynamic scalar input. - def testDynamicScalar3Ndims00ExpandNo(self): - self._test_dynamic(x=self._random_sample([]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=False) - - def testDynamicScalar3Ndims00ExpandYes(self): - self._test_dynamic(x=self._random_sample([]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=True) - - def testDynamicScalarNdims01ExpandNo(self): - with self.assertRaisesOpError(""): - self._test_dynamic(x=self._random_sample([]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=False) - - def testDynamicScalarNdims01ExpandYes(self): - with self.assertRaisesOpError(""): - self._test_dynamic(x=self._random_sample([]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=True) - - def testDynamicScalarNdims10ExpandNo(self): - with self.assertRaisesOpError(""): - self._test_dynamic(x=self._random_sample([]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=False) - - def testDynamicScalarNdims10ExpandYes(self): - with self.assertRaisesOpError(""): - self._test_dynamic(x=self._random_sample([]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=True) - - def testDynamicScalarNdims11ExpandNo(self): - with self.assertRaisesOpError(""): - self._test_dynamic(x=self._random_sample([]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=False) - - def testDynamicScalarNdims11ExpandYes(self): - with self.assertRaisesOpError(""): - self._test_dynamic(x=self._random_sample([]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=True) - - # Group 2a: Static vector input. - - def testStaticVectorNdims00ExpandNo(self): - self._test_static(x=self._random_sample([3]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=False) - - def testStaticVectorNdims00ExpandYes(self): - self._test_static(x=self._random_sample([3]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=True) - - def testStaticVectorNdims01ExpandNo(self): - self._test_static(x=self._random_sample([3]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=False) - - def testStaticVectorNdims01ExpandYes(self): - self._test_static(x=self._random_sample([3]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=True) - - def testStaticVectorNdims10ExpandNo(self): - self._test_static(x=self._random_sample([3]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=False) - - def testStaticVectorNdims10ExpandYes(self): - self._test_static(x=self._random_sample([3]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=True) - - def testStaticVectorNdims11ExpandNo(self): - with self.assertRaises(ValueError): - self._test_static(x=self._random_sample([3]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=False) - - def testStaticVectorNdims11ExpandYes(self): - with self.assertRaises(ValueError): - self._test_static(x=self._random_sample([3]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=True) - - # Group 2b: Dynamic vector input. - - def testDynamicVectorNdims00ExpandNo(self): - self._test_dynamic(x=self._random_sample([3]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=False) - - def testDynamicVectorNdims00ExpandYes(self): - self._test_dynamic(x=self._random_sample([3]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=True) - - def testDynamicVectorNdims01ExpandNo(self): - self._test_dynamic(x=self._random_sample([3]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=False) - - def testDynamicVectorNdims01ExpandYes(self): - self._test_dynamic(x=self._random_sample([3]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=True) - - def testDynamicVectorNdims10ExpandNo(self): - self._test_dynamic(x=self._random_sample([3]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=False) - - def testDynamicVectorNdims10ExpandYes(self): - self._test_dynamic(x=self._random_sample([3]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=True) - - def testDynamicVectorNdims11ExpandNo(self): - with self.assertRaisesOpError(""): - self._test_dynamic(x=self._random_sample([3]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=False) - - def testDynamicVectorNdims11ExpandYes(self): - with self.assertRaisesOpError(""): - self._test_dynamic(x=self._random_sample([3]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=True) - - # Group 3a: Static matrix input. - - def testStaticMatrixNdims00ExpandNo(self): - self._test_static(x=self._random_sample([2, 3]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=False) - - def testStaticMatrixNdims00ExpandYes(self): - self._test_static(x=self._random_sample([2, 3]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=True) - - def testStaticMatrixNdims01ExpandNo(self): - self._test_static(x=self._random_sample([2, 3]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=False) - - def testStaticMatrixNdims01ExpandYes(self): - self._test_static(x=self._random_sample([2, 3]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=True) - - def testStaticMatrixNdims10ExpandNo(self): - self._test_static(x=self._random_sample([2, 3]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=False) - - def testStaticMatrixNdims10ExpandYes(self): - self._test_static(x=self._random_sample([2, 3]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=True) - - def testStaticMatrixNdims11ExpandNo(self): - self._test_static(x=self._random_sample([2, 3]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=False) - - def testStaticMatrixNdims11ExpandYes(self): - self._test_static(x=self._random_sample([2, 3]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=True) - - # Group 3b: Dynamic matrix input. - - def testDynamicMatrixNdims00ExpandNo(self): - self._test_dynamic(x=self._random_sample([2, 3]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=False) - - def testDynamicMatrixNdims00ExpandYes(self): - self._test_dynamic(x=self._random_sample([2, 3]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=True) - - def testDynamicMatrixNdims01ExpandNo(self): - self._test_dynamic(x=self._random_sample([2, 3]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=False) - - def testDynamicMatrixNdims01ExpandYes(self): - self._test_dynamic(x=self._random_sample([2, 3]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=True) - - def testDynamicMatrixNdims10ExpandNo(self): - self._test_dynamic(x=self._random_sample([2, 3]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=False) - - def testDynamicMatrixNdims10ExpandYes(self): - self._test_dynamic(x=self._random_sample([2, 3]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=True) - - def testDynamicMatrixNdims11ExpandNo(self): - self._test_dynamic(x=self._random_sample([2, 3]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=False) - - def testDynamicMatrixNdims11ExpandYes(self): - self._test_dynamic(x=self._random_sample([2, 3]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=True) - - # Group 4a: Static tensor input. - - def testStaticTensorNdims00ExpandNo(self): - self._test_static(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=False) - - def testStaticTensorNdims00ExpandYes(self): - self._test_static(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=True) - - def testStaticTensorNdims01ExpandNo(self): - self._test_static(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=False) - - def testStaticTensorNdims01ExpandYes(self): - self._test_static(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=True) - - def testStaticTensorNdims10ExpandNo(self): - self._test_static(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=False) - - def testStaticTensorNdims10ExpandYes(self): - self._test_static(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=True) - - def testStaticTensorNdims11ExpandNo(self): - self._test_static(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=False) - - def testStaticTensorNdims11ExpandYes(self): - self._test_static(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=True) - - # Group 4b: Dynamic tensor input. - - def testDynamicTensorNdims00ExpandNo(self): - self._test_dynamic(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=False) - - def testDynamicTensorNdims00ExpandYes(self): - self._test_dynamic(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=0, - event_ndims=0, - expand_batch_dim=True) - - def testDynamicTensorNdims01ExpandNo(self): - self._test_dynamic(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=False) - - def testDynamicTensorNdims01ExpandYes(self): - self._test_dynamic(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=0, - event_ndims=1, - expand_batch_dim=True) - - def testDynamicTensorNdims10ExpandNo(self): - self._test_dynamic(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=False) - - def testDynamicTensorNdims10ExpandYes(self): - self._test_dynamic(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=1, - event_ndims=0, - expand_batch_dim=True) - - def testDynamicTensorNdims11ExpandNo(self): - self._test_dynamic(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=False) - - def testDynamicTensorNdims11ExpandYes(self): - self._test_dynamic(x=self._random_sample([4, 1, 2, 3]), - batch_ndims=1, - event_ndims=1, - expand_batch_dim=True) - - -class DistributionShapeTest(test.TestCase): - - def setUp(self): - self._rng = np.random.RandomState(42) - - def _random_sample(self, sample_shape, dtype=dtypes.float64): - return self._rng.random_sample(sample_shape).astype(dtype.as_numpy_dtype()) - - def _assertNdArrayEqual(self, expected, actual): - """Helper which properly compares two np.ndarray-like objects. - - This function checks for exact equality so is probably only suitable for - integers or powers of 2. - - Args: - expected: np.ndarray. Ground-truth value. - actual: np.ndarray. Observed value. - """ - expected = np.asarray(expected) - actual = np.asarray(actual) - self.assertEqual(expected.shape, actual.shape, - "Shape mismatch: expected %s, got %s." % - (expected.shape, actual.shape)) - actual_item = actual.flat - for expected_item in expected.flat: - self.assertAllEqual(expected_item, next(actual_item)) - - def testDistributionShapeGetNdimsStatic(self): - with self.cached_session(): - shaper = _DistributionShape(batch_ndims=0, event_ndims=0) - x = 1 - self.assertEqual(0, shaper.get_sample_ndims(x).eval()) - self.assertEqual(0, shaper.batch_ndims.eval()) - self.assertEqual(0, shaper.event_ndims.eval()) - - shaper = _DistributionShape(batch_ndims=1, event_ndims=1) - x = self._random_sample((1, 2, 3)) - self.assertAllEqual(3, shaper.get_ndims(x).eval()) - self.assertEqual(1, shaper.get_sample_ndims(x).eval()) - self.assertEqual(1, shaper.batch_ndims.eval()) - self.assertEqual(1, shaper.event_ndims.eval()) - - x += self._random_sample((1, 2, 3)) - self.assertAllEqual(3, shaper.get_ndims(x).eval()) - self.assertEqual(1, shaper.get_sample_ndims(x).eval()) - self.assertEqual(1, shaper.batch_ndims.eval()) - self.assertEqual(1, shaper.event_ndims.eval()) - - # Test ndims functions work, even despite unfed Tensors. - y = array_ops.placeholder(dtypes.float32, shape=(1024, None, 1024)) - self.assertEqual(3, shaper.get_ndims(y).eval()) - self.assertEqual(1, shaper.get_sample_ndims(y).eval()) - self.assertEqual(1, shaper.batch_ndims.eval()) - self.assertEqual(1, shaper.event_ndims.eval()) - - def testDistributionShapeGetNdimsDynamic(self): - with self.cached_session() as sess: - batch_ndims = array_ops.placeholder(dtypes.int32) - event_ndims = array_ops.placeholder(dtypes.int32) - shaper = _DistributionShape( - batch_ndims=batch_ndims, event_ndims=event_ndims) - y = array_ops.placeholder(dtypes.float32) - y_value = np.ones((4, 2), dtype=y.dtype.as_numpy_dtype()) - feed_dict = {y: y_value, batch_ndims: 1, event_ndims: 1} - self.assertEqual(2, sess.run(shaper.get_ndims(y), feed_dict=feed_dict)) - - def testDistributionShapeGetDimsStatic(self): - with self.cached_session(): - shaper = _DistributionShape(batch_ndims=0, event_ndims=0) - x = 1 - self.assertAllEqual((_empty_shape, _empty_shape, _empty_shape), - _constant(shaper.get_dims(x))) - shaper = _DistributionShape(batch_ndims=1, event_ndims=2) - x += self._random_sample((1, 1, 2, 2)) - self._assertNdArrayEqual(([0], [1], [2, 3]), - _constant(shaper.get_dims(x))) - x += x - self._assertNdArrayEqual(([0], [1], [2, 3]), - _constant(shaper.get_dims(x))) - - def testDistributionShapeGetDimsDynamic(self): - with self.cached_session() as sess: - # Works for static {batch,event}_ndims despite unfed input. - shaper = _DistributionShape(batch_ndims=1, event_ndims=2) - y = array_ops.placeholder(dtypes.float32, shape=(10, None, 5, 5)) - self._assertNdArrayEqual([[0], [1], [2, 3]], _eval(shaper.get_dims(y))) - - # Works for deferred {batch,event}_ndims. - batch_ndims = array_ops.placeholder(dtypes.int32) - event_ndims = array_ops.placeholder(dtypes.int32) - shaper = _DistributionShape( - batch_ndims=batch_ndims, event_ndims=event_ndims) - y = array_ops.placeholder(dtypes.float32) - y_value = self._random_sample((10, 3, 5, 5), dtype=y.dtype) - feed_dict = {y: y_value, batch_ndims: 1, event_ndims: 2} - self._assertNdArrayEqual( - ([0], [1], [2, 3]), sess.run(shaper.get_dims(y), feed_dict=feed_dict)) - - def testDistributionShapeGetShapeStatic(self): - with self.cached_session(): - shaper = _DistributionShape(batch_ndims=0, event_ndims=0) - self.assertAllEqual((_empty_shape, _empty_shape, _empty_shape), - _constant(shaper.get_shape(1.))) - self._assertNdArrayEqual(([1], _empty_shape, _empty_shape), - _constant(shaper.get_shape(np.ones(1)))) - self._assertNdArrayEqual(([2, 2], _empty_shape, _empty_shape), - _constant(shaper.get_shape(np.ones((2, 2))))) - self._assertNdArrayEqual(([3, 2, 1], _empty_shape, _empty_shape), - _constant(shaper.get_shape(np.ones((3, 2, 1))))) - - shaper = _DistributionShape(batch_ndims=0, event_ndims=1) - with self.assertRaisesRegexp(ValueError, "expected .* <= ndims"): - shaper.get_shape(1.) - self._assertNdArrayEqual((_empty_shape, _empty_shape, [1]), - _constant(shaper.get_shape(np.ones(1)))) - self._assertNdArrayEqual(([2], _empty_shape, [2]), - _constant(shaper.get_shape(np.ones((2, 2))))) - self._assertNdArrayEqual(([3, 2], _empty_shape, [1]), - _constant(shaper.get_shape(np.ones((3, 2, 1))))) - - shaper = _DistributionShape(batch_ndims=1, event_ndims=0) - with self.assertRaisesRegexp(ValueError, "expected .* <= ndims"): - shaper.get_shape(1.) - self._assertNdArrayEqual((_empty_shape, [1], _empty_shape), - _constant(shaper.get_shape(np.ones(1)))) - self._assertNdArrayEqual(([2], [2], _empty_shape), - _constant(shaper.get_shape(np.ones((2, 2))))) - self._assertNdArrayEqual(([3, 2], [1], _empty_shape), - _constant(shaper.get_shape(np.ones((3, 2, 1))))) - - shaper = _DistributionShape(batch_ndims=1, event_ndims=1) - with self.assertRaisesRegexp(ValueError, "expected .* <= ndims"): - shaper.get_shape(1.) - with self.assertRaisesRegexp(ValueError, "expected .* <= ndims"): - shaper.get_shape(np.ones(1)) - self._assertNdArrayEqual((_empty_shape, [2], [2]), - _constant(shaper.get_shape(np.ones((2, 2))))) - self._assertNdArrayEqual(([3], [2], [1]), - _constant(shaper.get_shape(np.ones((3, 2, 1))))) - - def testDistributionShapeGetShapeDynamic(self): - with self.cached_session() as sess: - # Works for static ndims despite unknown static shape. - shaper = _DistributionShape(batch_ndims=1, event_ndims=1) - y = array_ops.placeholder(dtypes.int32, shape=(None, None, 2)) - y_value = np.ones((3, 4, 2), dtype=y.dtype.as_numpy_dtype()) - self._assertNdArrayEqual( - ([3], [4], [2]), - sess.run(shaper.get_shape(y), feed_dict={y: y_value})) - - shaper = _DistributionShape(batch_ndims=0, event_ndims=1) - y = array_ops.placeholder(dtypes.int32, shape=(None, None)) - y_value = np.ones((3, 2), dtype=y.dtype.as_numpy_dtype()) - self._assertNdArrayEqual( - ([3], _empty_shape, [2]), - sess.run(shaper.get_shape(y), feed_dict={y: y_value})) - - # Works for deferred {batch,event}_ndims. - batch_ndims = array_ops.placeholder(dtypes.int32) - event_ndims = array_ops.placeholder(dtypes.int32) - shaper = _DistributionShape( - batch_ndims=batch_ndims, event_ndims=event_ndims) - y = array_ops.placeholder(dtypes.float32) - y_value = self._random_sample((3, 4, 2), dtype=y.dtype) - feed_dict = {y: y_value, batch_ndims: 1, event_ndims: 1} - self._assertNdArrayEqual( - ([3], [4], [2]), sess.run(shaper.get_shape(y), feed_dict=feed_dict)) - - y_value = self._random_sample((3, 2), dtype=y.dtype) - feed_dict = {y: y_value, batch_ndims: 0, event_ndims: 1} - self._assertNdArrayEqual( - ([3], _empty_shape, [2]), - sess.run(shaper.get_shape(y), feed_dict=feed_dict)) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/sinh_arcsinh_test.py b/tensorflow/contrib/distributions/python/kernel_tests/sinh_arcsinh_test.py deleted file mode 100644 index 1811d85b7e0..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/sinh_arcsinh_test.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for SinhArcsinh.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib import distributions -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - -ds = distributions -rng = np.random.RandomState(123) - - -class SinhArcsinhTest(test.TestCase): - - def test_default_is_same_as_normal(self): - b = 10 - scale = rng.rand(b) + 0.5 - loc = rng.randn(b) - with self.cached_session() as sess: - norm = ds.Normal( - loc=loc, - scale=scale, - validate_args=True) - sasnorm = ds.SinhArcsinh( - loc=loc, - scale=scale, - validate_args=True) - - x = rng.randn(5, b) - norm_pdf, sasnorm_pdf = sess.run([norm.prob(x), sasnorm.prob(x)]) - self.assertAllClose(norm_pdf, sasnorm_pdf) - - norm_samps, sasnorm_samps = sess.run( - [norm.sample(10000, seed=0), - sasnorm.sample(10000, seed=0)]) - self.assertAllClose(loc, sasnorm_samps.mean(axis=0), atol=0.1) - self.assertAllClose( - norm_samps.mean(axis=0), sasnorm_samps.mean(axis=0), atol=0.1) - self.assertAllClose( - norm_samps.std(axis=0), sasnorm_samps.std(axis=0), atol=0.1) - - def test_broadcast_params_dynamic(self): - with self.cached_session() as sess: - loc = array_ops.placeholder(dtypes.float64) - scale = array_ops.placeholder(dtypes.float64) - skewness = array_ops.placeholder(dtypes.float64) - sasnorm = ds.SinhArcsinh( - loc=loc, - scale=scale, - skewness=skewness, - validate_args=True) - - samp = sess.run(sasnorm.sample(), - feed_dict={loc: rng.rand(5), - scale: np.float64(rng.rand()), # Scalar - skewness: rng.rand(5)}) - self.assertAllEqual((5,), samp.shape) - - def test_passing_in_laplace_plus_defaults_is_same_as_laplace(self): - b = 10 - scale = rng.rand(b) + 0.5 - loc = rng.randn(b) - with self.cached_session() as sess: - lap = ds.Laplace( - loc=loc, - scale=scale, - validate_args=True) - saslap = ds.SinhArcsinh( - loc=loc, - scale=scale, - distribution=ds.Laplace(np.float64(0), np.float64(1)), - validate_args=True) - - x = rng.randn(5, b) - lap_pdf, saslap_pdf = sess.run([lap.prob(x), saslap.prob(x)]) - self.assertAllClose(lap_pdf, saslap_pdf) - - lap_samps, saslap_samps = sess.run( - [lap.sample(10000, seed=0), - saslap.sample(10000, seed=0)]) - self.assertAllClose(loc, saslap_samps.mean(axis=0), atol=0.1) - self.assertAllClose( - lap_samps.mean(axis=0), saslap_samps.mean(axis=0), atol=0.1) - self.assertAllClose( - lap_samps.std(axis=0), saslap_samps.std(axis=0), atol=0.1) - - def test_tailweight_small_gives_fewer_outliers_than_normal(self): - batch_size = 10 - scale = rng.rand(batch_size) + 0.5 - loc = 0.1 * rng.randn(batch_size) - with self.cached_session() as sess: - norm = ds.Normal( - loc=loc, - scale=scale, - validate_args=True) - sasnorm = ds.SinhArcsinh( - loc=loc, - scale=scale, - tailweight=0.1, - validate_args=True) - - # sasnorm.pdf(x) is smaller on outliers (+-10 are outliers) - x = np.float64([[-10] * batch_size, [10] * batch_size]) # Shape [2, 10] - norm_lp, sasnorm_lp = sess.run([norm.log_prob(x), sasnorm.log_prob(x)]) - np.testing.assert_array_less(sasnorm_lp, norm_lp) - - # 0.1% quantile and 99.9% quantile are outliers, and should be more - # extreme in the normal. The 97.772% quantiles should be the same. - norm_samps, sasnorm_samps = sess.run( - [norm.sample(int(5e5), seed=1), - sasnorm.sample(int(5e5), seed=1)]) - np.testing.assert_array_less( - np.percentile(norm_samps, 0.1, axis=0), - np.percentile(sasnorm_samps, 0.1, axis=0)) - np.testing.assert_array_less( - np.percentile(sasnorm_samps, 99.9, axis=0), - np.percentile(norm_samps, 99.9, axis=0)) - # 100. * sp.stats.norm.cdf(2.) - q = 100 * 0.97724986805182079 - self.assertAllClose( - np.percentile(sasnorm_samps, q, axis=0), - np.percentile(norm_samps, q, axis=0), - rtol=0.03) - self.assertAllClose( - np.percentile(sasnorm_samps, 100 - q, axis=0), - np.percentile(norm_samps, 100 - q, axis=0), - rtol=0.03) - - def test_tailweight_large_gives_more_outliers_than_normal(self): - batch_size = 10 - scale = rng.rand(batch_size) + 0.5 - loc = np.float64(0.) - with self.cached_session() as sess: - norm = ds.Normal( - loc=loc, - scale=scale, - validate_args=True) - sasnorm = ds.SinhArcsinh( - loc=loc, - scale=scale, - tailweight=3., - validate_args=True) - - # norm.pdf(x) is smaller on outliers (+-10 are outliers) - x = np.float64([[-10] * batch_size, [10] * batch_size]) # Shape [2, 10] - norm_lp, sasnorm_lp = sess.run([norm.log_prob(x), sasnorm.log_prob(x)]) - np.testing.assert_array_less(norm_lp, sasnorm_lp) - - # 0.1% quantile and 99.9% quantile are outliers, and should be more - # extreme in the sasnormal. The 97.772% quantiles should be the same. - norm_samps, sasnorm_samps = sess.run( - [norm.sample(int(5e5), seed=2), - sasnorm.sample(int(5e5), seed=2)]) - np.testing.assert_array_less( - np.percentile(sasnorm_samps, 0.1, axis=0), - np.percentile(norm_samps, 0.1, axis=0)) - np.testing.assert_array_less( - np.percentile(norm_samps, 99.9, axis=0), - np.percentile(sasnorm_samps, 99.9, axis=0)) - # 100. * sp.stats.norm.cdf(2.) - q = 100 * 0.97724986805182079 - self.assertAllClose( - np.percentile(sasnorm_samps, q, axis=0), - np.percentile(norm_samps, q, axis=0), - rtol=0.03) - self.assertAllClose( - np.percentile(sasnorm_samps, 100 - q, axis=0), - np.percentile(norm_samps, 100 - q, axis=0), - rtol=0.03) - - def test_positive_skewness_moves_mean_to_the_right(self): - batch_size = 10 - scale = rng.rand(batch_size) + 0.5 - loc = rng.randn(batch_size) - with self.cached_session() as sess: - sasnorm = ds.SinhArcsinh( - loc=loc, - scale=scale, - skewness=3.0, - validate_args=True) - - sasnorm_samps = sess.run(sasnorm.sample(10000, seed=4)) - np.testing.assert_array_less(loc, sasnorm_samps.mean(axis=0)) - - def test_pdf_reflected_for_negative_skewness(self): - with self.cached_session() as sess: - sas_pos_skew = ds.SinhArcsinh( - loc=0., - scale=1., - skewness=2., - validate_args=True) - sas_neg_skew = ds.SinhArcsinh( - loc=0., - scale=1., - skewness=-2., - validate_args=True) - x = np.linspace(-2, 2, num=5).astype(np.float32) - self.assertAllClose( - *sess.run([sas_pos_skew.prob(x), sas_neg_skew.prob(x[::-1])])) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py b/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py deleted file mode 100644 index 9c4dfed8363..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the statistical testing library.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import statistical_testing as st -from tensorflow.python.framework import ops -from tensorflow.python.platform import test - - -class StatisticalTestingTest(test.TestCase): - - def test_dkwm_design_mean_one_sample_soundness(self): - thresholds = [1e-5, 1e-2, 1.1e-1, 0.9, 1., 1.02, 2., 10., 1e2, 1e5, 1e10] - rates = [1e-6, 1e-3, 1e-2, 1.1e-1, 0.2, 0.5, 0.7, 1.] - false_fail_rates, false_pass_rates = np.meshgrid(rates, rates) - false_fail_rates = false_fail_rates.flatten().astype(np.float32) - false_pass_rates = false_pass_rates.flatten().astype(np.float32) - - detectable_discrepancies = [] - for false_pass_rate, false_fail_rate in zip( - false_pass_rates, false_fail_rates): - sufficient_n = st.min_num_samples_for_dkwm_mean_test( - thresholds, low=0., high=1., false_fail_rate=false_fail_rate, - false_pass_rate=false_pass_rate) - detectable_discrepancies.append( - st.min_discrepancy_of_true_means_detectable_by_dkwm( - sufficient_n, low=0., high=1., false_fail_rate=false_fail_rate, - false_pass_rate=false_pass_rate)) - - detectable_discrepancies_ = self.evaluate(detectable_discrepancies) - for discrepancies, false_pass_rate, false_fail_rate in zip( - detectable_discrepancies_, false_pass_rates, false_fail_rates): - below_threshold = discrepancies <= thresholds - self.assertAllEqual( - np.ones_like(below_threshold, np.bool), below_threshold, - msg='false_pass_rate({}), false_fail_rate({})'.format( - false_pass_rate, false_fail_rate)) - - def test_dkwm_design_mean_two_sample_soundness(self): - thresholds = [1e-5, 1e-2, 1.1e-1, 0.9, 1., 1.02, 2., 10., 1e2, 1e5, 1e10] - rates = [1e-6, 1e-3, 1e-2, 1.1e-1, 0.2, 0.5, 0.7, 1.] - false_fail_rates, false_pass_rates = np.meshgrid(rates, rates) - false_fail_rates = false_fail_rates.flatten().astype(np.float32) - false_pass_rates = false_pass_rates.flatten().astype(np.float32) - - detectable_discrepancies = [] - for false_pass_rate, false_fail_rate in zip( - false_pass_rates, false_fail_rates): - [ - sufficient_n1, - sufficient_n2 - ] = st.min_num_samples_for_dkwm_mean_two_sample_test( - thresholds, low1=0., high1=1., low2=0., high2=1., - false_fail_rate=false_fail_rate, - false_pass_rate=false_pass_rate) - - detectable_discrepancies.append( - st.min_discrepancy_of_true_means_detectable_by_dkwm_two_sample( - n1=sufficient_n1, - low1=0., - high1=1., - n2=sufficient_n2, - low2=0., - high2=1., - false_fail_rate=false_fail_rate, - false_pass_rate=false_pass_rate)) - - detectable_discrepancies_ = self.evaluate(detectable_discrepancies) - for discrepancies, false_pass_rate, false_fail_rate in zip( - detectable_discrepancies_, false_pass_rates, false_fail_rates): - below_threshold = discrepancies <= thresholds - self.assertAllEqual( - np.ones_like(below_threshold, np.bool), below_threshold, - msg='false_pass_rate({}), false_fail_rate({})'.format( - false_pass_rate, false_fail_rate)) - - def test_true_mean_confidence_interval_by_dkwm_one_sample(self): - rng = np.random.RandomState(seed=0) - - num_samples = 5000 - # 5000 samples is chosen to be enough to find discrepancies of - # size 0.1 or more with assurance 1e-6, as confirmed here: - d = st.min_discrepancy_of_true_means_detectable_by_dkwm( - num_samples, 0., 1., false_fail_rate=1e-6, false_pass_rate=1e-6) - d = self.evaluate(d) - self.assertLess(d, 0.1) - - # Test that the confidence interval computed for the mean includes - # 0.5 and excludes 0.4 and 0.6. - samples = rng.uniform(size=num_samples).astype(np.float32) - (low, high) = st.true_mean_confidence_interval_by_dkwm( - samples, 0., 1., error_rate=1e-6) - low, high = self.evaluate([low, high]) - self.assertGreater(low, 0.4) - self.assertLess(low, 0.5) - self.assertGreater(high, 0.5) - self.assertLess(high, 0.6) - - def test_dkwm_mean_one_sample_assertion(self): - rng = np.random.RandomState(seed=0) - num_samples = 5000 - - # Test that the test assertion agrees that the mean of the standard - # uniform distribution is 0.5. - samples = rng.uniform(size=num_samples).astype(np.float32) - self.evaluate(st.assert_true_mean_equal_by_dkwm( - samples, 0., 1., 0.5, false_fail_rate=1e-6)) - - # Test that the test assertion confirms that the mean of the - # standard uniform distribution is not 0.4. - with self.assertRaisesOpError("true mean greater than expected"): - self.evaluate(st.assert_true_mean_equal_by_dkwm( - samples, 0., 1., 0.4, false_fail_rate=1e-6)) - - # Test that the test assertion confirms that the mean of the - # standard uniform distribution is not 0.6. - with self.assertRaisesOpError("true mean smaller than expected"): - self.evaluate(st.assert_true_mean_equal_by_dkwm( - samples, 0., 1., 0.6, false_fail_rate=1e-6)) - - def test_dkwm_mean_in_interval_one_sample_assertion(self): - rng = np.random.RandomState(seed=0) - num_samples = 5000 - - # Test that the test assertion agrees that the mean of the standard - # uniform distribution is between 0.4 and 0.6. - samples = rng.uniform(size=num_samples).astype(np.float32) - self.evaluate(st.assert_true_mean_in_interval_by_dkwm( - samples, 0., 1., - expected_low=0.4, expected_high=0.6, false_fail_rate=1e-6)) - - # Test that the test assertion confirms that the mean of the - # standard uniform distribution is not between 0.2 and 0.4. - with self.assertRaisesOpError("true mean greater than expected"): - self.evaluate(st.assert_true_mean_in_interval_by_dkwm( - samples, 0., 1., - expected_low=0.2, expected_high=0.4, false_fail_rate=1e-6)) - - # Test that the test assertion confirms that the mean of the - # standard uniform distribution is not between 0.6 and 0.8. - with self.assertRaisesOpError("true mean smaller than expected"): - self.evaluate(st.assert_true_mean_in_interval_by_dkwm( - samples, 0., 1., - expected_low=0.6, expected_high=0.8, false_fail_rate=1e-6)) - - def test_dkwm_mean_two_sample_assertion(self): - rng = np.random.RandomState(seed=0) - num_samples = 4000 - - # 4000 samples is chosen to be enough to find discrepancies of - # size 0.2 or more with assurance 1e-6, as confirmed here: - d = st.min_discrepancy_of_true_means_detectable_by_dkwm_two_sample( - num_samples, 0., 1., num_samples, 0., 1., - false_fail_rate=1e-6, false_pass_rate=1e-6) - d = self.evaluate(d) - self.assertLess(d, 0.2) - - # Test that the test assertion agrees that the standard - # uniform distribution has the same mean as itself. - samples1 = rng.uniform(size=num_samples).astype(np.float32) - samples2 = rng.uniform(size=num_samples).astype(np.float32) - self.evaluate(st.assert_true_mean_equal_by_dkwm_two_sample( - samples1, 0., 1., samples2, 0., 1., false_fail_rate=1e-6)) - - def test_dkwm_mean_two_sample_assertion_beta_2_1_false(self): - rng = np.random.RandomState(seed=0) - num_samples = 4000 - samples1 = rng.uniform(size=num_samples).astype(np.float32) - - # As established above, 4000 samples is enough to find discrepancies - # of size 0.2 or more with assurance 1e-6. - - # Test that the test assertion confirms that the mean of the - # standard uniform distribution is different from the mean of beta(2, 1). - beta_high_samples = rng.beta(2, 1, size=num_samples).astype(np.float32) - with self.assertRaisesOpError("true mean smaller than expected"): - self.evaluate(st.assert_true_mean_equal_by_dkwm_two_sample( - samples1, 0., 1., - beta_high_samples, 0., 1., - false_fail_rate=1e-6)) - - def test_dkwm_mean_two_sample_assertion_beta_1_2_false(self): - rng = np.random.RandomState(seed=0) - num_samples = 4000 - samples1 = rng.uniform(size=num_samples).astype(np.float32) - - # As established above, 4000 samples is enough to find discrepancies - # of size 0.2 or more with assurance 1e-6. - - # Test that the test assertion confirms that the mean of the - # standard uniform distribution is different from the mean of beta(1, 2). - beta_low_samples = rng.beta(1, 2, size=num_samples).astype(np.float32) - with self.assertRaisesOpError("true mean greater than expected"): - self.evaluate(st.assert_true_mean_equal_by_dkwm_two_sample( - samples1, 0., 1., - beta_low_samples, 0., 1., - false_fail_rate=1e-6)) - - def test_dkwm_argument_validity_checking(self): - rng = np.random.RandomState(seed=0) - samples = rng.uniform( - low=[0., 1.], high=[1., 2.], size=(2500, 1, 2)).astype(np.float32) - - # Test that the test library complains if the given samples fall - # outside the purported bounds. - with self.assertRaisesOpError("maximum value exceeds expectations"): - self.evaluate(st.true_mean_confidence_interval_by_dkwm( - samples, [[0., 1.]], [[0.5, 1.5]], error_rate=0.5)) - with self.assertRaisesOpError("minimum value falls below expectations"): - self.evaluate(st.true_mean_confidence_interval_by_dkwm( - samples, [[0.5, 1.5]], [[1., 2.]], error_rate=0.5)) - - # But doesn't complain if they don't. - op = st.true_mean_confidence_interval_by_dkwm( - samples, [[0., 1.]], [[1., 2.]], error_rate=0.5) - _ = self.evaluate(op) - - def test_do_maximum_mean(self): - n = 117 - envelope = 0.02 # > 2 / n, but < 3 / n - rng = np.random.RandomState(seed=8) - samples = rng.uniform(size=n).astype(np.float32) - - # Compute the answer in TF using the code under test - envelope_t = ops.convert_to_tensor(envelope) - max_mean = st._do_maximum_mean(samples, envelope_t, 1) - max_mean = self.evaluate(max_mean) - - # Compute the correct answer for this case in numpy. In this - # example, `n` and `envelope` are such that `samples[2]` is the - # element that should be taken partially, regardless of the - # content of the `samples` array (see algorithm description in - # `../ops/statistical_testing.py`). - samples = sorted(samples) - weight = 1. / n - (envelope - 2. / n) - answer = samples[2] * weight + sum(samples[3:]) / n + envelope * 1. - self.assertAllClose(max_mean, answer, rtol=1e-9) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py deleted file mode 100644 index 13370497ce7..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py +++ /dev/null @@ -1,549 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for TransformedDistribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import stats - -from tensorflow.contrib import distributions -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.linalg import linalg -from tensorflow.python.platform import test - -bs = bijectors -ds = distributions -la = linalg - - -class DummyMatrixTransform(bs.Bijector): - """Tractable matrix transformation. - - This is a non-sensical bijector that has forward/inverse_min_event_ndims=2. - The main use is to check that transformed distribution calculations are done - appropriately. - """ - - def __init__(self): - super(DummyMatrixTransform, self).__init__( - forward_min_event_ndims=2, - is_constant_jacobian=False, - validate_args=False, - name="dummy") - - def _forward(self, x): - return x - - def _inverse(self, y): - return y - - # Note: These jacobians don't make sense. - def _forward_log_det_jacobian(self, x): - return -linalg_ops.matrix_determinant(x) - - def _inverse_log_det_jacobian(self, x): - return linalg_ops.matrix_determinant(x) - - -class TransformedDistributionTest(test.TestCase): - - def _cls(self): - return ds.TransformedDistribution - - def _make_unimplemented(self, name): - def _unimplemented(self, *args): # pylint: disable=unused-argument - raise NotImplementedError("{} not implemented".format(name)) - return _unimplemented - - def testTransformedDistribution(self): - g = ops.Graph() - with g.as_default(): - mu = 3.0 - sigma = 2.0 - # Note: the Jacobian callable only works for this example; more generally - # you may or may not need a reduce_sum. - log_normal = self._cls()( - distribution=ds.Normal(loc=mu, scale=sigma), - bijector=bs.Exp()) - sp_dist = stats.lognorm(s=sigma, scale=np.exp(mu)) - - # sample - sample = log_normal.sample(100000, seed=235) - self.assertAllEqual([], log_normal.event_shape) - with self.session(graph=g): - self.assertAllEqual([], log_normal.event_shape_tensor().eval()) - self.assertAllClose( - sp_dist.mean(), np.mean(sample.eval()), atol=0.0, rtol=0.05) - - # pdf, log_pdf, cdf, etc... - # The mean of the lognormal is around 148. - test_vals = np.linspace(0.1, 1000., num=20).astype(np.float32) - for func in [[log_normal.log_prob, sp_dist.logpdf], - [log_normal.prob, sp_dist.pdf], - [log_normal.log_cdf, sp_dist.logcdf], - [log_normal.cdf, sp_dist.cdf], - [log_normal.survival_function, sp_dist.sf], - [log_normal.log_survival_function, sp_dist.logsf]]: - actual = func[0](test_vals) - expected = func[1](test_vals) - with self.session(graph=g): - self.assertAllClose(expected, actual.eval(), atol=0, rtol=0.01) - - def testNonInjectiveTransformedDistribution(self): - g = ops.Graph() - with g.as_default(): - mu = 1. - sigma = 2.0 - abs_normal = self._cls()( - distribution=ds.Normal(loc=mu, scale=sigma), - bijector=bs.AbsoluteValue()) - sp_normal = stats.norm(mu, sigma) - - # sample - sample = abs_normal.sample(100000, seed=235) - self.assertAllEqual([], abs_normal.event_shape) - with self.session(graph=g): - sample_ = sample.eval() - self.assertAllEqual([], abs_normal.event_shape_tensor().eval()) - - # Abs > 0, duh! - np.testing.assert_array_less(0, sample_) - - # Let X ~ Normal(mu, sigma), Y := |X|, then - # P[Y < 0.77] = P[-0.77 < X < 0.77] - self.assertAllClose( - sp_normal.cdf(0.77) - sp_normal.cdf(-0.77), - (sample_ < 0.77).mean(), rtol=0.01) - - # p_Y(y) = p_X(-y) + p_X(y), - self.assertAllClose( - sp_normal.pdf(1.13) + sp_normal.pdf(-1.13), - abs_normal.prob(1.13).eval()) - - # Log[p_Y(y)] = Log[p_X(-y) + p_X(y)] - self.assertAllClose( - np.log(sp_normal.pdf(2.13) + sp_normal.pdf(-2.13)), - abs_normal.log_prob(2.13).eval()) - - def testQuantile(self): - with self.cached_session() as sess: - logit_normal = self._cls()( - distribution=ds.Normal(loc=0., scale=1.), - bijector=bs.Sigmoid(), - validate_args=True) - grid = [0., 0.25, 0.5, 0.75, 1.] - q = logit_normal.quantile(grid) - cdf = logit_normal.cdf(q) - cdf_ = sess.run(cdf) - self.assertAllClose(grid, cdf_, rtol=1e-6, atol=0.) - - def testCachedSamples(self): - exp_forward_only = bs.Exp() - exp_forward_only._inverse = self._make_unimplemented( - "inverse") - exp_forward_only._inverse_event_shape_tensor = self._make_unimplemented( - "inverse_event_shape_tensor ") - exp_forward_only._inverse_event_shape = self._make_unimplemented( - "inverse_event_shape ") - exp_forward_only._inverse_log_det_jacobian = self._make_unimplemented( - "inverse_log_det_jacobian ") - - with self.cached_session() as sess: - mu = 3.0 - sigma = 0.02 - log_normal = self._cls()( - distribution=ds.Normal(loc=mu, scale=sigma), - bijector=exp_forward_only) - - sample = log_normal.sample([2, 3], seed=42) - sample_val, log_pdf_val = sess.run([sample, log_normal.log_prob(sample)]) - expected_log_pdf = stats.lognorm.logpdf( - sample_val, s=sigma, scale=np.exp(mu)) - self.assertAllClose(expected_log_pdf, log_pdf_val, rtol=1e-4, atol=0.) - - def testCachedSamplesInvert(self): - exp_inverse_only = bs.Exp() - exp_inverse_only._forward = self._make_unimplemented( - "forward") - exp_inverse_only._forward_event_shape_tensor = self._make_unimplemented( - "forward_event_shape_tensor ") - exp_inverse_only._forward_event_shape = self._make_unimplemented( - "forward_event_shape ") - exp_inverse_only._forward_log_det_jacobian = self._make_unimplemented( - "forward_log_det_jacobian ") - - log_forward_only = bs.Invert(exp_inverse_only) - - with self.cached_session() as sess: - # The log bijector isn't defined over the whole real line, so we make - # sigma sufficiently small so that the draws are positive. - mu = 2. - sigma = 1e-2 - exp_normal = self._cls()( - distribution=ds.Normal(loc=mu, scale=sigma), - bijector=log_forward_only) - - sample = exp_normal.sample([2, 3], seed=42) - sample_val, log_pdf_val = sess.run([sample, exp_normal.log_prob(sample)]) - expected_log_pdf = sample_val + stats.norm.logpdf( - np.exp(sample_val), loc=mu, scale=sigma) - self.assertAllClose(expected_log_pdf, log_pdf_val, atol=0.) - - def testShapeChangingBijector(self): - with self.cached_session(): - softmax = bs.SoftmaxCentered() - standard_normal = ds.Normal(loc=0., scale=1.) - multi_logit_normal = self._cls()( - distribution=standard_normal, - bijector=softmax, - event_shape=[1]) - x = [[[-np.log(3.)], [0.]], - [[np.log(3)], [np.log(5)]]] - y = softmax.forward(x).eval() - expected_log_pdf = ( - np.squeeze(stats.norm(loc=0., scale=1.).logpdf(x)) - - np.sum(np.log(y), axis=-1)) - self.assertAllClose(expected_log_pdf, - multi_logit_normal.log_prob(y).eval()) - self.assertAllClose( - [1, 2, 3, 2], - array_ops.shape(multi_logit_normal.sample([1, 2, 3])).eval()) - self.assertAllEqual([2], multi_logit_normal.event_shape) - self.assertAllEqual([2], multi_logit_normal.event_shape_tensor().eval()) - - def testCastLogDetJacobian(self): - """Test log_prob when Jacobian and log_prob dtypes do not match.""" - - with self.cached_session(): - # Create an identity bijector whose jacobians have dtype int32 - int_identity = bs.Inline( - forward_fn=array_ops.identity, - inverse_fn=array_ops.identity, - inverse_log_det_jacobian_fn=( - lambda y: math_ops.cast(0, dtypes.int32)), - forward_log_det_jacobian_fn=( - lambda x: math_ops.cast(0, dtypes.int32)), - forward_min_event_ndims=0, - is_constant_jacobian=True) - normal = self._cls()( - distribution=ds.Normal(loc=0., scale=1.), - bijector=int_identity, - validate_args=True) - - y = normal.sample() - normal.log_prob(y).eval() - normal.prob(y).eval() - normal.entropy().eval() - - def testEntropy(self): - with self.cached_session(): - shift = np.array([[-1, 0, 1], [-1, -2, -3]], dtype=np.float32) - diag = np.array([[1, 2, 3], [2, 3, 2]], dtype=np.float32) - actual_mvn_entropy = np.concatenate([ - [stats.multivariate_normal(shift[i], np.diag(diag[i]**2)).entropy()] - for i in range(len(diag))]) - fake_mvn = self._cls()( - ds.MultivariateNormalDiag( - loc=array_ops.zeros_like(shift), - scale_diag=array_ops.ones_like(diag), - validate_args=True), - bs.AffineLinearOperator( - shift, - scale=la.LinearOperatorDiag(diag, is_non_singular=True), - validate_args=True), - validate_args=True) - self.assertAllClose(actual_mvn_entropy, - fake_mvn.entropy().eval()) - - def testScalarBatchScalarEventIdentityScale(self): - with self.cached_session() as sess: - exp2 = self._cls()( - ds.Exponential(rate=0.25), - bijector=ds.bijectors.AffineScalar(scale=2.) - ) - log_prob = exp2.log_prob(1.) - log_prob_ = sess.run(log_prob) - base_log_prob = -0.5 * 0.25 + np.log(0.25) - ildj = np.log(2.) - self.assertAllClose(base_log_prob - ildj, log_prob_, rtol=1e-6, atol=0.) - - -class ScalarToMultiTest(test.TestCase): - - def _cls(self): - return ds.TransformedDistribution - - def setUp(self): - self._shift = np.array([-1, 0, 1], dtype=np.float32) - self._tril = np.array([[[1., 0, 0], - [2, 1, 0], - [3, 2, 1]], - [[2, 0, 0], - [3, 2, 0], - [4, 3, 2]]], - dtype=np.float32) - - def _testMVN(self, - base_distribution_class, - base_distribution_kwargs, - batch_shape=(), - event_shape=(), - not_implemented_message=None): - with self.cached_session() as sess: - # Overriding shapes must be compatible w/bijector; most bijectors are - # batch_shape agnostic and only care about event_ndims. - # In the case of `Affine`, if we got it wrong then it would fire an - # exception due to incompatible dimensions. - batch_shape_pl = array_ops.placeholder( - dtypes.int32, name="dynamic_batch_shape") - event_shape_pl = array_ops.placeholder( - dtypes.int32, name="dynamic_event_shape") - feed_dict = {batch_shape_pl: np.array(batch_shape, dtype=np.int32), - event_shape_pl: np.array(event_shape, dtype=np.int32)} - fake_mvn_dynamic = self._cls()( - distribution=base_distribution_class(validate_args=True, - **base_distribution_kwargs), - bijector=bs.Affine(shift=self._shift, scale_tril=self._tril), - batch_shape=batch_shape_pl, - event_shape=event_shape_pl, - validate_args=True) - - fake_mvn_static = self._cls()( - distribution=base_distribution_class(validate_args=True, - **base_distribution_kwargs), - bijector=bs.Affine(shift=self._shift, scale_tril=self._tril), - batch_shape=batch_shape, - event_shape=event_shape, - validate_args=True) - - actual_mean = np.tile(self._shift, [2, 1]) # Affine elided this tile. - actual_cov = np.matmul(self._tril, np.transpose(self._tril, [0, 2, 1])) - - def actual_mvn_log_prob(x): - return np.concatenate([ - [stats.multivariate_normal( - actual_mean[i], actual_cov[i]).logpdf(x[:, i, :])] - for i in range(len(actual_cov))]).T - - actual_mvn_entropy = np.concatenate([ - [stats.multivariate_normal( - actual_mean[i], actual_cov[i]).entropy()] - for i in range(len(actual_cov))]) - - self.assertAllEqual([3], fake_mvn_static.event_shape) - self.assertAllEqual([2], fake_mvn_static.batch_shape) - - self.assertAllEqual(tensor_shape.TensorShape(None), - fake_mvn_dynamic.event_shape) - self.assertAllEqual(tensor_shape.TensorShape(None), - fake_mvn_dynamic.batch_shape) - - x = fake_mvn_static.sample(5, seed=0).eval() - for unsupported_fn in (fake_mvn_static.log_cdf, - fake_mvn_static.cdf, - fake_mvn_static.survival_function, - fake_mvn_static.log_survival_function): - with self.assertRaisesRegexp(NotImplementedError, - not_implemented_message): - unsupported_fn(x) - - num_samples = 5e3 - for fake_mvn, feed_dict in ((fake_mvn_static, {}), - (fake_mvn_dynamic, feed_dict)): - # Ensure sample works by checking first, second moments. - y = fake_mvn.sample(int(num_samples), seed=0) - x = y[0:5, ...] - sample_mean = math_ops.reduce_mean(y, 0) - centered_y = array_ops.transpose(y - sample_mean, [1, 2, 0]) - sample_cov = math_ops.matmul( - centered_y, centered_y, transpose_b=True) / num_samples - [ - sample_mean_, - sample_cov_, - x_, - fake_event_shape_, - fake_batch_shape_, - fake_log_prob_, - fake_prob_, - fake_entropy_, - ] = sess.run([ - sample_mean, - sample_cov, - x, - fake_mvn.event_shape_tensor(), - fake_mvn.batch_shape_tensor(), - fake_mvn.log_prob(x), - fake_mvn.prob(x), - fake_mvn.entropy(), - ], feed_dict=feed_dict) - - self.assertAllClose(actual_mean, sample_mean_, atol=0.1, rtol=0.1) - self.assertAllClose(actual_cov, sample_cov_, atol=0., rtol=0.1) - - # Ensure all other functions work as intended. - self.assertAllEqual([5, 2, 3], x_.shape) - self.assertAllEqual([3], fake_event_shape_) - self.assertAllEqual([2], fake_batch_shape_) - self.assertAllClose(actual_mvn_log_prob(x_), fake_log_prob_, - atol=0., rtol=1e-6) - self.assertAllClose(np.exp(actual_mvn_log_prob(x_)), fake_prob_, - atol=0., rtol=1e-5) - self.assertAllClose(actual_mvn_entropy, fake_entropy_, - atol=0., rtol=1e-6) - - def testScalarBatchScalarEvent(self): - self._testMVN( - base_distribution_class=ds.Normal, - base_distribution_kwargs={"loc": 0., "scale": 1.}, - batch_shape=[2], - event_shape=[3], - not_implemented_message="not implemented when overriding event_shape") - - def testScalarBatchNonScalarEvent(self): - self._testMVN( - base_distribution_class=ds.MultivariateNormalDiag, - base_distribution_kwargs={"loc": [0., 0., 0.], - "scale_diag": [1., 1, 1]}, - batch_shape=[2], - not_implemented_message="not implemented") - - with self.cached_session(): - # Can't override event_shape for scalar batch, non-scalar event. - with self.assertRaisesRegexp(ValueError, "base distribution not scalar"): - self._cls()( - distribution=ds.MultivariateNormalDiag(loc=[0.], scale_diag=[1.]), - bijector=bs.Affine(shift=self._shift, scale_tril=self._tril), - batch_shape=[2], - event_shape=[3], - validate_args=True) - - def testNonScalarBatchScalarEvent(self): - self._testMVN( - base_distribution_class=ds.Normal, - base_distribution_kwargs={"loc": [0., 0], "scale": [1., 1]}, - event_shape=[3], - not_implemented_message="not implemented when overriding event_shape") - - with self.cached_session(): - # Can't override batch_shape for non-scalar batch, scalar event. - with self.assertRaisesRegexp(ValueError, "base distribution not scalar"): - self._cls()( - distribution=ds.Normal(loc=[0.], scale=[1.]), - bijector=bs.Affine(shift=self._shift, scale_tril=self._tril), - batch_shape=[2], - event_shape=[3], - validate_args=True) - - def testNonScalarBatchNonScalarEvent(self): - with self.cached_session(): - # Can't override event_shape and/or batch_shape for non_scalar batch, - # non-scalar event. - with self.assertRaisesRegexp(ValueError, "base distribution not scalar"): - self._cls()( - distribution=ds.MultivariateNormalDiag(loc=[[0.]], - scale_diag=[[1.]]), - bijector=bs.Affine(shift=self._shift, scale_tril=self._tril), - batch_shape=[2], - event_shape=[3], - validate_args=True) - - def testMatrixEvent(self): - with self.cached_session() as sess: - batch_shape = [2] - event_shape = [2, 3, 3] - batch_shape_pl = array_ops.placeholder( - dtypes.int32, name="dynamic_batch_shape") - event_shape_pl = array_ops.placeholder( - dtypes.int32, name="dynamic_event_shape") - feed_dict = {batch_shape_pl: np.array(batch_shape, dtype=np.int32), - event_shape_pl: np.array(event_shape, dtype=np.int32)} - - scale = 2. - loc = 0. - fake_mvn_dynamic = self._cls()( - distribution=ds.Normal( - loc=loc, - scale=scale), - bijector=DummyMatrixTransform(), - batch_shape=batch_shape_pl, - event_shape=event_shape_pl, - validate_args=True) - - fake_mvn_static = self._cls()( - distribution=ds.Normal( - loc=loc, - scale=scale), - bijector=DummyMatrixTransform(), - batch_shape=batch_shape, - event_shape=event_shape, - validate_args=True) - - def actual_mvn_log_prob(x): - # This distribution is the normal PDF, reduced over the - # last 3 dimensions + a jacobian term which corresponds - # to the determinant of x. - return (np.sum( - stats.norm(loc, scale).logpdf(x), axis=(-1, -2, -3)) + - np.sum(np.linalg.det(x), axis=-1)) - - self.assertAllEqual([2, 3, 3], fake_mvn_static.event_shape) - self.assertAllEqual([2], fake_mvn_static.batch_shape) - - self.assertAllEqual(tensor_shape.TensorShape(None), - fake_mvn_dynamic.event_shape) - self.assertAllEqual(tensor_shape.TensorShape(None), - fake_mvn_dynamic.batch_shape) - - num_samples = 5e3 - for fake_mvn, feed_dict in ((fake_mvn_static, {}), - (fake_mvn_dynamic, feed_dict)): - # Ensure sample works by checking first, second moments. - y = fake_mvn.sample(int(num_samples), seed=0) - x = y[0:5, ...] - [ - x_, - fake_event_shape_, - fake_batch_shape_, - fake_log_prob_, - fake_prob_, - ] = sess.run([ - x, - fake_mvn.event_shape_tensor(), - fake_mvn.batch_shape_tensor(), - fake_mvn.log_prob(x), - fake_mvn.prob(x), - ], feed_dict=feed_dict) - - # Ensure all other functions work as intended. - self.assertAllEqual([5, 2, 2, 3, 3], x_.shape) - self.assertAllEqual([2, 3, 3], fake_event_shape_) - self.assertAllEqual([2], fake_batch_shape_) - self.assertAllClose(actual_mvn_log_prob(x_), fake_log_prob_, - atol=0., rtol=1e-6) - self.assertAllClose(np.exp(actual_mvn_log_prob(x_)), fake_prob_, - atol=0., rtol=1e-5) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py deleted file mode 100644 index 856579da329..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py +++ /dev/null @@ -1,345 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for VectorDiffeomixture.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import test_util -from tensorflow.contrib.distributions.python.ops import vector_diffeomixture as vdm_lib -from tensorflow.python.ops.distributions import normal as normal_lib -from tensorflow.python.ops.linalg import linear_operator_diag as linop_diag_lib -from tensorflow.python.ops.linalg import linear_operator_identity as linop_identity_lib -from tensorflow.python.platform import test - -rng = np.random.RandomState(0) - - -class VectorDiffeomixtureTest( - test_util.VectorDistributionTestHelpers, test.TestCase): - """Tests the VectorDiffeomixture distribution.""" - - def testSampleProbConsistentBroadcastMixNoBatch(self): - with self.cached_session() as sess: - dims = 4 - vdm = vdm_lib.VectorDiffeomixture( - mix_loc=[[0.], [1.]], - temperature=[1.], - distribution=normal_lib.Normal(0., 1.), - loc=[ - None, - np.float32([2.]*dims), - ], - scale=[ - linop_identity_lib.LinearOperatorScaledIdentity( - num_rows=dims, - multiplier=np.float32(1.1), - is_positive_definite=True), - linop_diag_lib.LinearOperatorDiag( - diag=np.linspace(2.5, 3.5, dims, dtype=np.float32), - is_positive_definite=True), - ], - quadrature_size=8, - validate_args=True) - # Ball centered at component0's mean. - self.run_test_sample_consistent_log_prob( - sess.run, vdm, radius=2., center=0., rtol=0.015) - # Larger ball centered at component1's mean. - self.run_test_sample_consistent_log_prob( - sess.run, vdm, radius=4., center=2., rtol=0.015) - - def testSampleProbConsistentBroadcastMixNonStandardBase(self): - with self.cached_session() as sess: - dims = 4 - vdm = vdm_lib.VectorDiffeomixture( - mix_loc=[[0.], [1.]], - temperature=[1.], - distribution=normal_lib.Normal(1., 1.5), - loc=[ - None, - np.float32([2.]*dims), - ], - scale=[ - linop_identity_lib.LinearOperatorScaledIdentity( - num_rows=dims, - multiplier=np.float32(1.1), - is_positive_definite=True), - linop_diag_lib.LinearOperatorDiag( - diag=np.linspace(2.5, 3.5, dims, dtype=np.float32), - is_positive_definite=True), - ], - quadrature_size=8, - validate_args=True) - # Ball centered at component0's mean. - self.run_test_sample_consistent_log_prob( - sess.run, vdm, radius=2., center=1., rtol=0.015) - # Larger ball centered at component1's mean. - self.run_test_sample_consistent_log_prob( - sess.run, vdm, radius=4., center=3., rtol=0.01) - - def testSampleProbConsistentBroadcastMixBatch(self): - with self.cached_session() as sess: - dims = 4 - vdm = vdm_lib.VectorDiffeomixture( - mix_loc=[[0.], [1.]], - temperature=[1.], - distribution=normal_lib.Normal(0., 1.), - loc=[ - None, - np.float32([2.]*dims), - ], - scale=[ - linop_identity_lib.LinearOperatorScaledIdentity( - num_rows=dims, - multiplier=[np.float32(1.1)], - is_positive_definite=True), - linop_diag_lib.LinearOperatorDiag( - diag=np.stack([ - np.linspace(2.5, 3.5, dims, dtype=np.float32), - np.linspace(2.75, 3.25, dims, dtype=np.float32), - ]), - is_positive_definite=True), - ], - quadrature_size=8, - validate_args=True) - # Ball centered at component0's mean. - self.run_test_sample_consistent_log_prob( - sess.run, vdm, radius=2., center=0., rtol=0.01) - # Larger ball centered at component1's mean. - self.run_test_sample_consistent_log_prob( - sess.run, vdm, radius=4., center=2., rtol=0.01) - - def testSampleProbConsistentBroadcastMixTwoBatchDims(self): - dims = 4 - loc_1 = rng.randn(2, 3, dims).astype(np.float32) - - with self.cached_session() as sess: - vdm = vdm_lib.VectorDiffeomixture( - mix_loc=(rng.rand(2, 3, 1) - 0.5).astype(np.float32), - temperature=[1.], - distribution=normal_lib.Normal(0., 1.), - loc=[ - None, - loc_1, - ], - scale=[ - linop_identity_lib.LinearOperatorScaledIdentity( - num_rows=dims, - multiplier=[np.float32(1.1)], - is_positive_definite=True), - ] * 2, - validate_args=True) - # Ball centered at component0's mean. - self.run_test_sample_consistent_log_prob( - sess.run, vdm, radius=2., center=0., rtol=0.01) - # Larger ball centered at component1's mean. - self.run_test_sample_consistent_log_prob( - sess.run, vdm, radius=3., center=loc_1, rtol=0.02) - - def testMeanCovarianceNoBatch(self): - with self.cached_session() as sess: - dims = 3 - vdm = vdm_lib.VectorDiffeomixture( - mix_loc=[[0.], [4.]], - temperature=[1 / 10.], - distribution=normal_lib.Normal(0., 1.), - loc=[ - np.float32([-2.]), - None, - ], - scale=[ - linop_identity_lib.LinearOperatorScaledIdentity( - num_rows=dims, - multiplier=np.float32(1.5), - is_positive_definite=True), - linop_diag_lib.LinearOperatorDiag( - diag=np.linspace(2.5, 3.5, dims, dtype=np.float32), - is_positive_definite=True), - ], - quadrature_size=8, - validate_args=True) - self.run_test_sample_consistent_mean_covariance( - sess.run, vdm, rtol=0.02, cov_rtol=0.08) - - def testTemperatureControlsHowMuchThisLooksLikeDiscreteMixture(self): - # As temperature decreases, this should approach a mixture of normals, with - # components at -2, 2. - with self.cached_session() as sess: - dims = 1 - vdm = vdm_lib.VectorDiffeomixture( - mix_loc=[0.], - temperature=[[2.], [1.], [0.2]], - distribution=normal_lib.Normal(0., 1.), - loc=[ - np.float32([-2.]), - np.float32([2.]), - ], - scale=[ - linop_identity_lib.LinearOperatorScaledIdentity( - num_rows=dims, - multiplier=np.float32(0.5), - is_positive_definite=True), - ] * 2, # Use the same scale for each component. - quadrature_size=8, - validate_args=True) - - samps = vdm.sample(10000) - self.assertAllEqual((10000, 3, 1), samps.shape) - samps_ = sess.run(samps).reshape(10000, 3) # Make scalar event shape. - - # One characteristic of a discrete mixture (as opposed to a "smear") is - # that more weight is put near the component centers at -2, 2, and thus - # less weight is put near the origin. - prob_of_being_near_origin = (np.abs(samps_) < 1).mean(axis=0) - self.assertGreater( - prob_of_being_near_origin[0], prob_of_being_near_origin[1]) - self.assertGreater( - prob_of_being_near_origin[1], prob_of_being_near_origin[2]) - - # Run this test as well, just because we can. - self.run_test_sample_consistent_mean_covariance( - sess.run, vdm, rtol=0.02, cov_rtol=0.08) - - def testConcentrationLocControlsHowMuchWeightIsOnEachComponent(self): - with self.cached_session() as sess: - dims = 1 - vdm = vdm_lib.VectorDiffeomixture( - mix_loc=[[-1.], [0.], [1.]], - temperature=[0.5], - distribution=normal_lib.Normal(0., 1.), - loc=[ - np.float32([-2.]), - np.float32([2.]), - ], - scale=[ - linop_identity_lib.LinearOperatorScaledIdentity( - num_rows=dims, - multiplier=np.float32(0.5), - is_positive_definite=True), - ] * 2, # Use the same scale for each component. - quadrature_size=8, - validate_args=True) - - samps = vdm.sample(10000) - self.assertAllEqual((10000, 3, 1), samps.shape) - samps_ = sess.run(samps).reshape(10000, 3) # Make scalar event shape. - - # One characteristic of putting more weight on a component is that the - # mean is closer to that component's mean. - # Get the mean for each batch member, the names signify the value of - # concentration for that batch member. - mean_neg1, mean_0, mean_1 = samps_.mean(axis=0) - - # Since concentration is the concentration for component 0, - # concentration = -1 ==> more weight on component 1, which has mean = 2 - # concentration = 0 ==> equal weight - # concentration = 1 ==> more weight on component 0, which has mean = -2 - self.assertLess(-2, mean_1) - self.assertLess(mean_1, mean_0) - self.assertLess(mean_0, mean_neg1) - self.assertLess(mean_neg1, 2) - - # Run this test as well, just because we can. - self.run_test_sample_consistent_mean_covariance( - sess.run, vdm, rtol=0.02, cov_rtol=0.08) - - def testMeanCovarianceNoBatchUncenteredNonStandardBase(self): - with self.cached_session() as sess: - dims = 3 - vdm = vdm_lib.VectorDiffeomixture( - mix_loc=[[0.], [4.]], - temperature=[0.1], - distribution=normal_lib.Normal(-1., 1.5), - loc=[ - np.float32([-2.]), - np.float32([0.]), - ], - scale=[ - linop_identity_lib.LinearOperatorScaledIdentity( - num_rows=dims, - multiplier=np.float32(1.5), - is_positive_definite=True), - linop_diag_lib.LinearOperatorDiag( - diag=np.linspace(2.5, 3.5, dims, dtype=np.float32), - is_positive_definite=True), - ], - quadrature_size=8, - validate_args=True) - self.run_test_sample_consistent_mean_covariance( - sess.run, vdm, num_samples=int(1e6), rtol=0.01, cov_atol=0.025) - - def testMeanCovarianceBatch(self): - with self.cached_session() as sess: - dims = 3 - vdm = vdm_lib.VectorDiffeomixture( - mix_loc=[[0.], [4.]], - temperature=[0.1], - distribution=normal_lib.Normal(0., 1.), - loc=[ - np.float32([[-2.]]), - None, - ], - scale=[ - linop_identity_lib.LinearOperatorScaledIdentity( - num_rows=dims, - multiplier=[np.float32(1.5)], - is_positive_definite=True), - linop_diag_lib.LinearOperatorDiag( - diag=np.stack([ - np.linspace(2.5, 3.5, dims, dtype=np.float32), - np.linspace(0.5, 1.5, dims, dtype=np.float32), - ]), - is_positive_definite=True), - ], - quadrature_size=8, - validate_args=True) - self.run_test_sample_consistent_mean_covariance( - sess.run, vdm, rtol=0.02, cov_rtol=0.07) - - def testSampleProbConsistentQuadrature(self): - with self.cached_session() as sess: - dims = 4 - vdm = vdm_lib.VectorDiffeomixture( - mix_loc=[0.], - temperature=[0.1], - distribution=normal_lib.Normal(0., 1.), - loc=[ - None, - np.float32([2.]*dims), - ], - scale=[ - linop_identity_lib.LinearOperatorScaledIdentity( - num_rows=dims, - multiplier=np.float32(1.1), - is_positive_definite=True), - linop_diag_lib.LinearOperatorDiag( - diag=np.linspace(2.5, 3.5, dims, dtype=np.float32), - is_positive_definite=True), - ], - quadrature_size=3, - validate_args=True) - # Ball centered at component0's mean. - self.run_test_sample_consistent_log_prob( - sess.run, vdm, radius=2., center=0., rtol=0.015) - # Larger ball centered at component1's mean. - self.run_test_sample_consistent_log_prob( - sess.run, vdm, radius=4., center=2., rtol=0.005) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_exponential_diag_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_exponential_diag_test.py deleted file mode 100644 index db8186b79a1..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/vector_exponential_diag_test.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for VectorExponentialLinearOperator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib import distributions -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -ds = distributions - - -class VectorExponentialDiagTest(test.TestCase): - """Well tested because this is a simple override of the base class.""" - - def setUp(self): - self._rng = np.random.RandomState(42) - - def testScalarParams(self): - mu = -1. - diag = -5. - with self.cached_session(): - with self.assertRaisesRegexp(ValueError, "at least 1 dimension"): - ds.VectorExponentialDiag(mu, diag) - - def testVectorParams(self): - mu = [-1.] - diag = [-5.] - with self.cached_session(): - dist = ds.VectorExponentialDiag(mu, diag, validate_args=True) - self.assertAllEqual([3, 1], dist.sample(3).get_shape()) - - def testMean(self): - mu = [-1., 1] - diag = [1., -5] - with self.cached_session(): - dist = ds.VectorExponentialDiag(mu, diag, validate_args=True) - self.assertAllEqual([-1. + 1., 1. - 5.], dist.mean().eval()) - - def testMode(self): - mu = [-1.] - diag = [1., -5] - with self.cached_session(): - dist = ds.VectorExponentialDiag(mu, diag, validate_args=True) - self.assertAllEqual([-1., -1.], dist.mode().eval()) - - def testMeanWithBroadcastLoc(self): - mu = [-1.] - diag = [1., -5] - with self.cached_session(): - dist = ds.VectorExponentialDiag(mu, diag, validate_args=True) - self.assertAllEqual([-1. + 1, -1. - 5], dist.mean().eval()) - - def testSample(self): - mu = [-2., 1] - diag = [1., -2] - with self.cached_session(): - dist = ds.VectorExponentialDiag(mu, diag, validate_args=True) - samps = dist.sample(int(1e4), seed=0).eval() - cov_mat = array_ops.matrix_diag(diag).eval()**2 - - self.assertAllClose([-2 + 1, 1. - 2], samps.mean(axis=0), - atol=0., rtol=0.05) - self.assertAllClose(cov_mat, np.cov(samps.T), - atol=0.05, rtol=0.05) - - def testSingularScaleRaises(self): - mu = [-1., 1] - diag = [1., 0] - with self.cached_session(): - dist = ds.VectorExponentialDiag(mu, diag, validate_args=True) - with self.assertRaisesOpError("Singular"): - dist.sample().eval() - - def testSampleWithBroadcastScale(self): - # mu corresponds to a 2-batch of 3-variate normals - mu = np.zeros([2, 3]) - - # diag corresponds to no batches of 3-variate normals - diag = np.ones([3]) - - with self.cached_session(): - dist = ds.VectorExponentialDiag(mu, diag, validate_args=True) - - mean = dist.mean() - self.assertAllEqual([2, 3], mean.get_shape()) - self.assertAllClose(mu + diag, mean.eval()) - - n = int(1e4) - samps = dist.sample(n, seed=0).eval() - samps_centered = samps - samps.mean(axis=0) - cov_mat = array_ops.matrix_diag(diag).eval()**2 - sample_cov = np.matmul(samps_centered.transpose([1, 2, 0]), - samps_centered.transpose([1, 0, 2])) / n - - self.assertAllClose(mu + diag, samps.mean(axis=0), - atol=0.10, rtol=0.05) - self.assertAllClose([cov_mat, cov_mat], sample_cov, - atol=0.10, rtol=0.05) - - def testCovariance(self): - with self.cached_session(): - vex = ds.VectorExponentialDiag( - loc=array_ops.ones([2, 3], dtype=dtypes.float32)) - self.assertAllClose( - np.diag(np.ones([3], dtype=np.float32)), - vex.covariance().eval()) - - vex = ds.VectorExponentialDiag( - loc=array_ops.ones([3], dtype=dtypes.float32), - scale_identity_multiplier=[3., 2.]) - self.assertAllEqual([2], vex.batch_shape) - self.assertAllEqual([3], vex.event_shape) - self.assertAllClose( - np.array([[[3., 0, 0], - [0, 3, 0], - [0, 0, 3]], - [[2, 0, 0], - [0, 2, 0], - [0, 0, 2]]])**2., - vex.covariance().eval()) - - vex = ds.VectorExponentialDiag( - loc=array_ops.ones([3], dtype=dtypes.float32), - scale_diag=[[3., 2, 1], [4, 5, 6]]) - self.assertAllEqual([2], vex.batch_shape) - self.assertAllEqual([3], vex.event_shape) - self.assertAllClose( - np.array([[[3., 0, 0], - [0, 2, 0], - [0, 0, 1]], - [[4, 0, 0], - [0, 5, 0], - [0, 0, 6]]])**2., - vex.covariance().eval()) - - def testVariance(self): - with self.cached_session(): - vex = ds.VectorExponentialDiag( - loc=array_ops.zeros([2, 3], dtype=dtypes.float32)) - self.assertAllClose( - np.ones([3], dtype=np.float32), - vex.variance().eval()) - - vex = ds.VectorExponentialDiag( - loc=array_ops.ones([3], dtype=dtypes.float32), - scale_identity_multiplier=[3., 2.]) - self.assertAllClose( - np.array([[3., 3, 3], - [2., 2, 2]])**2., - vex.variance().eval()) - - vex = ds.VectorExponentialDiag( - loc=array_ops.ones([3], dtype=dtypes.float32), - scale_diag=[[3., 2, 1], - [4., 5, 6]]) - self.assertAllClose( - np.array([[3., 2, 1], - [4., 5, 6]])**2., - vex.variance().eval()) - - def testStddev(self): - with self.cached_session(): - vex = ds.VectorExponentialDiag( - loc=array_ops.zeros([2, 3], dtype=dtypes.float32)) - self.assertAllClose( - np.ones([3], dtype=np.float32), - vex.stddev().eval()) - - vex = ds.VectorExponentialDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_identity_multiplier=[3., 2.]) - self.assertAllClose( - np.array([[3., 3, 3], - [2., 2, 2]]), - vex.stddev().eval()) - - vex = ds.VectorExponentialDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_diag=[[3., 2, 1], [4, 5, 6]]) - self.assertAllClose( - np.array([[3., 2, 1], - [4., 5, 6]]), - vex.stddev().eval()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_laplace_diag_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_laplace_diag_test.py deleted file mode 100644 index 9ee19b7e933..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/vector_laplace_diag_test.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for VectorLaplaceLinearOperator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib import distributions -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -ds = distributions - - -class VectorLaplaceDiagTest(test.TestCase): - """Well tested because this is a simple override of the base class.""" - - def setUp(self): - self._rng = np.random.RandomState(42) - - def testScalarParams(self): - mu = -1. - diag = -5. - with self.cached_session(): - with self.assertRaisesRegexp(ValueError, "at least 1 dimension"): - ds.VectorLaplaceDiag(mu, diag) - - def testVectorParams(self): - mu = [-1.] - diag = [-5.] - with self.cached_session(): - dist = ds.VectorLaplaceDiag(mu, diag, validate_args=True) - self.assertAllEqual([3, 1], dist.sample(3).get_shape()) - - def testDistWithBatchShapeOneThenTransformedThroughSoftplus(self): - # This complex combination of events resulted in a loss of static shape - # information when tensor_util.constant_value(self._needs_rotation) was - # being used incorrectly (resulting in always rotating). - # Batch shape = [1], event shape = [3] - mu = array_ops.zeros((1, 3)) - diag = array_ops.ones((1, 3)) - with self.cached_session(): - base_dist = ds.VectorLaplaceDiag(mu, diag, validate_args=True) - dist = ds.TransformedDistribution( - base_dist, - validate_args=True, - bijector=bijectors.Softplus()) - samps = dist.sample(5) # Shape [5, 1, 3]. - self.assertAllEqual([5, 1], dist.log_prob(samps).get_shape()) - - def testMean(self): - mu = [-1., 1] - diag = [1., -5] - with self.cached_session(): - dist = ds.VectorLaplaceDiag(mu, diag, validate_args=True) - self.assertAllEqual(mu, dist.mean().eval()) - - def testMeanWithBroadcastLoc(self): - mu = [-1.] - diag = [1., -5] - with self.cached_session(): - dist = ds.VectorLaplaceDiag(mu, diag, validate_args=True) - self.assertAllEqual([-1., -1.], dist.mean().eval()) - - def testSample(self): - mu = [-1., 1] - diag = [1., -2] - with self.cached_session(): - dist = ds.VectorLaplaceDiag(mu, diag, validate_args=True) - samps = dist.sample(int(1e4), seed=0).eval() - cov_mat = 2. * array_ops.matrix_diag(diag).eval()**2 - - self.assertAllClose(mu, samps.mean(axis=0), - atol=0., rtol=0.05) - self.assertAllClose(cov_mat, np.cov(samps.T), - atol=0.05, rtol=0.05) - - def testSingularScaleRaises(self): - mu = [-1., 1] - diag = [1., 0] - with self.cached_session(): - dist = ds.VectorLaplaceDiag(mu, diag, validate_args=True) - with self.assertRaisesOpError("Singular"): - dist.sample().eval() - - def testSampleWithBroadcastScale(self): - # mu corresponds to a 2-batch of 3-variate normals - mu = np.zeros([2, 3]) - - # diag corresponds to no batches of 3-variate normals - diag = np.ones([3]) - - with self.cached_session(): - dist = ds.VectorLaplaceDiag(mu, diag, validate_args=True) - - mean = dist.mean() - self.assertAllEqual([2, 3], mean.get_shape()) - self.assertAllClose(mu, mean.eval()) - - n = int(1e4) - samps = dist.sample(n, seed=0).eval() - cov_mat = 2. * array_ops.matrix_diag(diag).eval()**2 - sample_cov = np.matmul(samps.transpose([1, 2, 0]), - samps.transpose([1, 0, 2])) / n - - self.assertAllClose(mu, samps.mean(axis=0), - atol=0.10, rtol=0.05) - self.assertAllClose([cov_mat, cov_mat], sample_cov, - atol=0.10, rtol=0.05) - - def testCovariance(self): - with self.cached_session(): - vla = ds.VectorLaplaceDiag( - loc=array_ops.zeros([2, 3], dtype=dtypes.float32)) - self.assertAllClose( - 2. * np.diag(np.ones([3], dtype=np.float32)), - vla.covariance().eval()) - - vla = ds.VectorLaplaceDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_identity_multiplier=[3., 2.]) - self.assertAllEqual([2], vla.batch_shape) - self.assertAllEqual([3], vla.event_shape) - self.assertAllClose( - 2. * np.array([[[3., 0, 0], - [0, 3, 0], - [0, 0, 3]], - [[2, 0, 0], - [0, 2, 0], - [0, 0, 2]]])**2., - vla.covariance().eval()) - - vla = ds.VectorLaplaceDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_diag=[[3., 2, 1], [4, 5, 6]]) - self.assertAllEqual([2], vla.batch_shape) - self.assertAllEqual([3], vla.event_shape) - self.assertAllClose( - 2. * np.array([[[3., 0, 0], - [0, 2, 0], - [0, 0, 1]], - [[4, 0, 0], - [0, 5, 0], - [0, 0, 6]]])**2., - vla.covariance().eval()) - - def testVariance(self): - with self.cached_session(): - vla = ds.VectorLaplaceDiag( - loc=array_ops.zeros([2, 3], dtype=dtypes.float32)) - self.assertAllClose( - 2. * np.ones([3], dtype=np.float32), - vla.variance().eval()) - - vla = ds.VectorLaplaceDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_identity_multiplier=[3., 2.]) - self.assertAllClose( - 2. * np.array([[3., 3, 3], - [2, 2, 2]])**2., - vla.variance().eval()) - - vla = ds.VectorLaplaceDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_diag=[[3., 2, 1], - [4, 5, 6]]) - self.assertAllClose( - 2. * np.array([[3., 2, 1], - [4, 5, 6]])**2., - vla.variance().eval()) - - def testStddev(self): - with self.cached_session(): - vla = ds.VectorLaplaceDiag( - loc=array_ops.zeros([2, 3], dtype=dtypes.float32)) - self.assertAllClose( - np.sqrt(2) * np.ones([3], dtype=np.float32), - vla.stddev().eval()) - - vla = ds.VectorLaplaceDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_identity_multiplier=[3., 2.]) - self.assertAllClose( - np.sqrt(2) * np.array([[3., 3, 3], - [2, 2, 2]]), - vla.stddev().eval()) - - vla = ds.VectorLaplaceDiag( - loc=array_ops.zeros([3], dtype=dtypes.float32), - scale_diag=[[3., 2, 1], [4, 5, 6]]) - self.assertAllClose( - np.sqrt(2) * np.array([[3., 2, 1], - [4, 5, 6]]), - vla.stddev().eval()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py deleted file mode 100644 index 0dd7d23eb04..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py +++ /dev/null @@ -1,272 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for VectorSinhArcsinhDiag.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib import distributions -from tensorflow.contrib.distributions.python.ops import test_util -from tensorflow.python.platform import test - -ds = distributions -rng = np.random.RandomState(123) - - -class VectorSinhArcsinhDiagTest(test_util.VectorDistributionTestHelpers, - test.TestCase): - - def test_default_is_same_as_normal(self): - d = 10 - scale_diag = rng.rand(d) - scale_identity_multiplier = np.float64(1.0) - loc = rng.randn(d) - with self.cached_session() as sess: - norm = ds.MultivariateNormalDiag( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - validate_args=True) - sasnorm = ds.VectorSinhArcsinhDiag( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - validate_args=True) - - x = rng.randn(5, d) - norm_pdf, sasnorm_pdf = sess.run([norm.prob(x), sasnorm.prob(x)]) - self.assertAllClose(norm_pdf, sasnorm_pdf) - - norm_samps, sasnorm_samps = sess.run( - [norm.sample(10000, seed=0), - sasnorm.sample(10000, seed=0)]) - self.assertAllClose(loc, sasnorm_samps.mean(axis=0), atol=0.1) - self.assertAllClose( - norm_samps.mean(axis=0), sasnorm_samps.mean(axis=0), atol=0.1) - self.assertAllClose( - norm_samps.std(axis=0), sasnorm_samps.std(axis=0), atol=0.1) - - def test_passing_in_laplace_plus_defaults_is_same_as_laplace(self): - d = 10 - scale_diag = rng.rand(d) - scale_identity_multiplier = np.float64(1.2) - loc = rng.randn(d) - with self.cached_session() as sess: - vlap = ds.VectorLaplaceDiag( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - validate_args=True) - sasvlap = ds.VectorSinhArcsinhDiag( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - distribution=ds.Laplace(np.float64(0.), np.float64(1.)), - validate_args=True) - - x = rng.randn(5, d) - vlap_pdf, sasvlap_pdf = sess.run([vlap.prob(x), sasvlap.prob(x)]) - self.assertAllClose(vlap_pdf, sasvlap_pdf) - - vlap_samps, sasvlap_samps = sess.run( - [vlap.sample(10000, seed=0), - sasvlap.sample(10000, seed=0)]) - self.assertAllClose(loc, sasvlap_samps.mean(axis=0), atol=0.1) - self.assertAllClose( - vlap_samps.mean(axis=0), sasvlap_samps.mean(axis=0), atol=0.1) - self.assertAllClose( - vlap_samps.std(axis=0), sasvlap_samps.std(axis=0), atol=0.1) - - def test_tailweight_small_gives_fewer_outliers_than_normal(self): - d = 10 - scale_diag = rng.rand(d) - scale_identity_multiplier = np.float64(0.9) - loc = rng.randn(d) - with self.cached_session() as sess: - norm = ds.MultivariateNormalDiag( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - validate_args=True) - sasnorm = ds.VectorSinhArcsinhDiag( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - tailweight=0.1, - validate_args=True) - - # sasnorm.pdf(x) is smaller on outliers (+-10 are outliers) - x = np.float64([[-10] * d, [10] * d]) # Shape [2, 10] - norm_lp, sasnorm_lp = sess.run([norm.log_prob(x), sasnorm.log_prob(x)]) - np.testing.assert_array_less(sasnorm_lp, norm_lp) - - # 0.1% quantile and 99.9% quantile are outliers, and should be more - # extreme in the normal. The 97.772% quantiles should be the same. - norm_samps, sasnorm_samps = sess.run( - [norm.sample(int(5e5), seed=1), - sasnorm.sample(int(5e5), seed=1)]) - np.testing.assert_array_less( - np.percentile(norm_samps, 0.1, axis=0), - np.percentile(sasnorm_samps, 0.1, axis=0)) - np.testing.assert_array_less( - np.percentile(sasnorm_samps, 99.9, axis=0), - np.percentile(norm_samps, 99.9, axis=0)) - # 100. * sp.stats.norm.cdf(2.) - q = 100 * 0.97724986805182079 - self.assertAllClose( - np.percentile(sasnorm_samps, q, axis=0), - np.percentile(norm_samps, q, axis=0), - rtol=0.03) - self.assertAllClose( - np.percentile(sasnorm_samps, 100 - q, axis=0), - np.percentile(norm_samps, 100 - q, axis=0), - rtol=0.03) - - def test_tailweight_large_gives_more_outliers_than_normal(self): - d = 10 - scale_diag = rng.rand(d) - scale_identity_multiplier = np.float64(1.0) - loc = rng.randn(d) - with self.cached_session() as sess: - norm = ds.MultivariateNormalDiag( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - validate_args=True) - sasnorm = ds.VectorSinhArcsinhDiag( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - tailweight=3., - validate_args=True) - - # norm.pdf(x) is smaller on outliers (+-10 are outliers) - x = np.float64([[-10] * d, [10] * d]) # Shape [2, 10] - norm_lp, sasnorm_lp = sess.run([norm.log_prob(x), sasnorm.log_prob(x)]) - np.testing.assert_array_less(norm_lp, sasnorm_lp) - - # 0.1% quantile and 99.9% quantile are outliers, and should be more - # extreme in the sasnormal. The 97.772% quantiles should be the same. - norm_samps, sasnorm_samps = sess.run( - [norm.sample(int(5e5), seed=2), - sasnorm.sample(int(5e5), seed=2)]) - np.testing.assert_array_less( - np.percentile(sasnorm_samps, 0.1, axis=0), - np.percentile(norm_samps, 0.1, axis=0)) - np.testing.assert_array_less( - np.percentile(norm_samps, 99.9, axis=0), - np.percentile(sasnorm_samps, 99.9, axis=0)) - # 100. * sp.stats.norm.cdf(2.) - q = 100 * 0.97724986805182079 - self.assertAllClose( - np.percentile(sasnorm_samps, q, axis=0), - np.percentile(norm_samps, q, axis=0), - rtol=0.03) - self.assertAllClose( - np.percentile(sasnorm_samps, 100 - q, axis=0), - np.percentile(norm_samps, 100 - q, axis=0), - rtol=0.03) - - def test_positive_skewness_moves_mean_to_the_right(self): - d = 10 - scale_diag = rng.rand(d) - scale_identity_multiplier = np.float64(1.0) - loc = rng.randn(d) - with self.cached_session() as sess: - sasnorm = ds.VectorSinhArcsinhDiag( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - skewness=3.0, - validate_args=True) - - sasnorm_samps = sess.run(sasnorm.sample(10000, seed=4)) - np.testing.assert_array_less(loc, sasnorm_samps.mean(axis=0)) - - def test_consistency_random_parameters_with_batch_dim(self): - b, d = 5, 2 - scale_diag = rng.rand(b, d) - scale_identity_multiplier = np.float64(1.1) - with self.cached_session() as sess: - sasnorm = ds.VectorSinhArcsinhDiag( - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - skewness=rng.randn(d) * 0.5, - tailweight=rng.rand(b, d) + 0.7, - validate_args=True) - - self.run_test_sample_consistent_log_prob( - sess.run, sasnorm, radius=1.0, center=0., rtol=0.1) - self.run_test_sample_consistent_log_prob( - sess.run, - sasnorm, - radius=1.0, - center=-0.15, - rtol=0.1) - self.run_test_sample_consistent_log_prob( - sess.run, - sasnorm, - radius=1.0, - center=0.15, - rtol=0.1) - - def test_consistency_random_parameters_no_batch_dims(self): - d = 3 - scale_diag = rng.rand(d) - scale_identity_multiplier = np.float64(1.1) - with self.cached_session() as sess: - sasnorm = ds.VectorSinhArcsinhDiag( - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - skewness=rng.randn(d) * 0.5, - tailweight=rng.rand(d) + 0.7, - validate_args=True) - - self.run_test_sample_consistent_log_prob( - sess.run, sasnorm, radius=1.0, center=0., rtol=0.1) - self.run_test_sample_consistent_log_prob( - sess.run, - sasnorm, - radius=1.0, - center=-0.15, - rtol=0.1) - self.run_test_sample_consistent_log_prob( - sess.run, - sasnorm, - radius=1.0, - center=0.15, - rtol=0.1) - - def test_pdf_reflected_for_negative_skewness(self): - with self.cached_session() as sess: - sas_pos_skew = ds.VectorSinhArcsinhDiag( - loc=[0.], - scale_identity_multiplier=1., - skewness=2., - validate_args=True) - sas_neg_skew = ds.VectorSinhArcsinhDiag( - loc=[0.], - scale_identity_multiplier=1., - skewness=-2., - validate_args=True) - x = np.linspace(-2, 2, num=5).astype(np.float32).reshape(5, 1) - self.assertAllClose( - *sess.run([sas_pos_skew.prob(x), sas_neg_skew.prob(x[::-1])])) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_student_t_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_student_t_test.py deleted file mode 100644 index aaec1f09d94..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/vector_student_t_test.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for MultivariateStudentsT Distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import linalg -from scipy import special - -from tensorflow.contrib.distributions.python.ops.vector_student_t import _VectorStudentT -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class _FakeVectorStudentT(object): - """Fake scipy implementation for Multivariate Student's t-distribution. - - Technically we don't need to test the `Vector Student's t-distribution` since - its composed of only unit-tested parts. However this _FakeVectorStudentT - serves as something like an end-to-end test of the - `TransformedDistribution + Affine` API. - - Other `Vector*` implementations need only test new code. That we don't need - to test every Vector* distribution is good because there aren't SciPy - analogs and reimplementing everything in NumPy sort of defeats the point of - having the `TransformedDistribution + Affine` API. - """ - - def __init__(self, df, loc, scale_tril): - self._df = np.asarray(df) - self._loc = np.asarray(loc) - self._scale_tril = np.asarray(scale_tril) - - def log_prob(self, x): - def _compute(df, loc, scale_tril, x): - k = scale_tril.shape[-1] - ildj = np.sum(np.log(np.abs(np.diag(scale_tril))), axis=-1) - logz = ildj + k * (0.5 * np.log(df) + - 0.5 * np.log(np.pi) + - special.gammaln(0.5 * df) - - special.gammaln(0.5 * (df + 1.))) - y = linalg.solve_triangular(scale_tril, np.matrix(x - loc).T, - lower=True, overwrite_b=True) - logs = -0.5 * (df + 1.) * np.sum(np.log1p(y**2. / df), axis=-2) - return logs - logz - if not self._df.shape: - return _compute(self._df, self._loc, self._scale_tril, x) - return np.concatenate([ - [_compute(self._df[i], self._loc[i], self._scale_tril[i], x[:, i, :])] - for i in range(len(self._df))]).T - - def prob(self, x): - return np.exp(self.log_prob(x)) - - -class VectorStudentTTest(test.TestCase): - - def setUp(self): - self._rng = np.random.RandomState(42) - - def testProbStaticScalar(self): - with self.cached_session(): - # Scalar batch_shape. - df = np.asarray(3., dtype=np.float32) - # Scalar batch_shape. - loc = np.asarray([1], dtype=np.float32) - scale_diag = np.asarray([2.], dtype=np.float32) - scale_tril = np.diag(scale_diag) - - expected_mst = _FakeVectorStudentT( - df=df, loc=loc, scale_tril=scale_tril) - - actual_mst = _VectorStudentT(df=df, loc=loc, scale_diag=scale_diag, - validate_args=True) - x = 2. * self._rng.rand(4, 1).astype(np.float32) - 1. - - self.assertAllClose(expected_mst.log_prob(x), - actual_mst.log_prob(x).eval(), - rtol=0., atol=1e-5) - self.assertAllClose(expected_mst.prob(x), - actual_mst.prob(x).eval(), - rtol=0., atol=1e-5) - - def testProbStatic(self): - # Non-scalar batch_shape. - df = np.asarray([1., 2, 3], dtype=np.float32) - # Non-scalar batch_shape. - loc = np.asarray([[0., 0, 0], - [1, 2, 3], - [1, 0, 1]], - dtype=np.float32) - scale_diag = np.asarray([[1., 2, 3], - [2, 3, 4], - [4, 5, 6]], - dtype=np.float32) - scale_tril = np.concatenate([[np.diag(scale_diag[i])] - for i in range(len(scale_diag))]) - x = 2. * self._rng.rand(4, 3, 3).astype(np.float32) - 1. - - expected_mst = _FakeVectorStudentT( - df=df, loc=loc, scale_tril=scale_tril) - - with self.cached_session(): - actual_mst = _VectorStudentT(df=df, loc=loc, scale_diag=scale_diag, - validate_args=True) - self.assertAllClose(expected_mst.log_prob(x), - actual_mst.log_prob(x).eval(), - rtol=0., atol=1e-5) - self.assertAllClose(expected_mst.prob(x), - actual_mst.prob(x).eval(), - rtol=0., atol=1e-5) - - def testProbDynamic(self): - # Non-scalar batch_shape. - df = np.asarray([1., 2, 3], dtype=np.float32) - # Non-scalar batch_shape. - loc = np.asarray([[0., 0, 0], - [1, 2, 3], - [1, 0, 1]], - dtype=np.float32) - scale_diag = np.asarray([[1., 2, 3], - [2, 3, 4], - [4, 5, 6]], - dtype=np.float32) - scale_tril = np.concatenate([[np.diag(scale_diag[i])] - for i in range(len(scale_diag))]) - x = 2. * self._rng.rand(4, 3, 3).astype(np.float32) - 1. - - expected_mst = _FakeVectorStudentT( - df=df, loc=loc, scale_tril=scale_tril) - - with self.cached_session(): - df_pl = array_ops.placeholder(dtypes.float32, name="df") - loc_pl = array_ops.placeholder(dtypes.float32, name="loc") - scale_diag_pl = array_ops.placeholder(dtypes.float32, name="scale_diag") - feed_dict = {df_pl: df, loc_pl: loc, scale_diag_pl: scale_diag} - actual_mst = _VectorStudentT(df=df, loc=loc, scale_diag=scale_diag, - validate_args=True) - self.assertAllClose(expected_mst.log_prob(x), - actual_mst.log_prob(x).eval(feed_dict=feed_dict), - rtol=0., atol=1e-5) - self.assertAllClose(expected_mst.prob(x), - actual_mst.prob(x).eval(feed_dict=feed_dict), - rtol=0., atol=1e-5) - - def testProbScalarBaseDistributionNonScalarTransform(self): - # Scalar batch_shape. - df = np.asarray(2., dtype=np.float32) - # Non-scalar batch_shape. - loc = np.asarray([[0., 0, 0], - [1, 2, 3], - [1, 0, 1]], - dtype=np.float32) - scale_diag = np.asarray([[1., 2, 3], - [2, 3, 4], - [4, 5, 6]], - dtype=np.float32) - scale_tril = np.concatenate([[np.diag(scale_diag[i])] - for i in range(len(scale_diag))]) - x = 2. * self._rng.rand(4, 3, 3).astype(np.float32) - 1. - - expected_mst = _FakeVectorStudentT( - df=np.tile(df, reps=len(scale_diag)), - loc=loc, - scale_tril=scale_tril) - - with self.cached_session(): - actual_mst = _VectorStudentT(df=df, loc=loc, scale_diag=scale_diag, - validate_args=True) - self.assertAllClose(expected_mst.log_prob(x), - actual_mst.log_prob(x).eval(), - rtol=0., atol=1e-5) - self.assertAllClose(expected_mst.prob(x), - actual_mst.prob(x).eval(), - rtol=0., atol=1e-5) - - def testProbScalarBaseDistributionNonScalarTransformDynamic(self): - # Scalar batch_shape. - df = np.asarray(2., dtype=np.float32) - # Non-scalar batch_shape. - loc = np.asarray([[0., 0, 0], - [1, 2, 3], - [1, 0, 1]], - dtype=np.float32) - scale_diag = np.asarray([[1., 2, 3], - [2, 3, 4], - [4, 5, 6]], - dtype=np.float32) - scale_tril = np.concatenate([[np.diag(scale_diag[i])] - for i in range(len(scale_diag))]) - x = 2. * self._rng.rand(4, 3, 3).astype(np.float32) - 1. - - expected_mst = _FakeVectorStudentT( - df=np.tile(df, reps=len(scale_diag)), - loc=loc, - scale_tril=scale_tril) - - with self.cached_session(): - df_pl = array_ops.placeholder(dtypes.float32, name="df") - loc_pl = array_ops.placeholder(dtypes.float32, name="loc") - scale_diag_pl = array_ops.placeholder(dtypes.float32, name="scale_diag") - feed_dict = {df_pl: df, loc_pl: loc, scale_diag_pl: scale_diag} - actual_mst = _VectorStudentT(df=df, loc=loc, scale_diag=scale_diag, - validate_args=True) - self.assertAllClose(expected_mst.log_prob(x), - actual_mst.log_prob(x).eval(feed_dict=feed_dict), - rtol=0., atol=1e-5) - self.assertAllClose(expected_mst.prob(x), - actual_mst.prob(x).eval(feed_dict=feed_dict), - rtol=0., atol=1e-5) - - def testProbNonScalarBaseDistributionScalarTransform(self): - # Non-scalar batch_shape. - df = np.asarray([1., 2., 3.], dtype=np.float32) - # Scalar batch_shape. - loc = np.asarray([1, 2, 3], dtype=np.float32) - scale_diag = np.asarray([2, 3, 4], dtype=np.float32) - scale_tril = np.diag(scale_diag) - x = 2. * self._rng.rand(4, 3, 3).astype(np.float32) - 1. - - expected_mst = _FakeVectorStudentT( - df=df, - loc=np.tile(loc[array_ops.newaxis, :], reps=[len(df), 1]), - scale_tril=np.tile(scale_tril[array_ops.newaxis, :, :], - reps=[len(df), 1, 1])) - - with self.cached_session(): - actual_mst = _VectorStudentT(df=df, loc=loc, scale_diag=scale_diag, - validate_args=True) - self.assertAllClose(expected_mst.log_prob(x), - actual_mst.log_prob(x).eval(), - rtol=0., atol=1e-5) - self.assertAllClose(expected_mst.prob(x), - actual_mst.prob(x).eval(), - rtol=0., atol=1e-5) - - def testProbNonScalarBaseDistributionScalarTransformDynamic(self): - # Non-scalar batch_shape. - df = np.asarray([1., 2., 3.], dtype=np.float32) - # Scalar batch_shape. - loc = np.asarray([1, 2, 3], dtype=np.float32) - scale_diag = np.asarray([2, 3, 4], dtype=np.float32) - scale_tril = np.diag(scale_diag) - - x = 2. * self._rng.rand(4, 3, 3).astype(np.float32) - 1. - - expected_mst = _FakeVectorStudentT( - df=df, - loc=np.tile(loc[array_ops.newaxis, :], reps=[len(df), 1]), - scale_tril=np.tile(scale_tril[array_ops.newaxis, :, :], - reps=[len(df), 1, 1])) - - with self.cached_session(): - df_pl = array_ops.placeholder(dtypes.float32, name="df") - loc_pl = array_ops.placeholder(dtypes.float32, name="loc") - scale_diag_pl = array_ops.placeholder(dtypes.float32, name="scale_diag") - feed_dict = {df_pl: df, loc_pl: loc, scale_diag_pl: scale_diag} - actual_mst = _VectorStudentT(df=df, loc=loc, scale_diag=scale_diag, - validate_args=True) - self.assertAllClose(expected_mst.log_prob(x), - actual_mst.log_prob(x).eval(feed_dict=feed_dict), - rtol=0., atol=1e-5) - self.assertAllClose(expected_mst.prob(x), - actual_mst.prob(x).eval(feed_dict=feed_dict), - rtol=0., atol=1e-5) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py b/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py deleted file mode 100644 index 1a0407b86ad..00000000000 --- a/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py +++ /dev/null @@ -1,420 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Wishart.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import linalg -from tensorflow.contrib import distributions as distributions_lib -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - -distributions = distributions_lib - - -def make_pd(start, n): - """Deterministically create a positive definite matrix.""" - x = np.tril(linalg.circulant(np.arange(start, start + n))) - return np.dot(x, x.T) - - -def chol(x): - """Compute Cholesky factorization.""" - return linalg.cholesky(x).T - - -def wishart_var(df, x): - """Compute Wishart variance for numpy scale matrix.""" - x = np.sqrt(df) * np.asarray(x) - d = np.expand_dims(np.diag(x), -1) - return x**2 + np.dot(d, d.T) - - -class WishartCholeskyTest(test.TestCase): - - def testEntropy(self): - with self.cached_session(): - scale = make_pd(1., 2) - df = 4 - w = distributions.WishartCholesky(df, chol(scale)) - # sp.stats.wishart(df=4, scale=make_pd(1., 2)).entropy() - self.assertAllClose(6.301387092430769, w.entropy().eval()) - - w = distributions.WishartCholesky(df=1, scale=[[1.]]) - # sp.stats.wishart(df=1,scale=1).entropy() - self.assertAllClose(0.78375711047393404, w.entropy().eval()) - - def testMeanLogDetAndLogNormalizingConstant(self): - with self.cached_session(): - - def entropy_alt(w): - return ( - w.log_normalization() - - 0.5 * (w.df - w.dimension - 1.) * w.mean_log_det() - + 0.5 * w.df * w.dimension).eval() - - w = distributions.WishartCholesky(df=4, - scale=chol(make_pd(1., 2))) - self.assertAllClose(w.entropy().eval(), entropy_alt(w)) - - w = distributions.WishartCholesky(df=5, scale=[[1.]]) - self.assertAllClose(w.entropy().eval(), entropy_alt(w)) - - def testMean(self): - with self.cached_session(): - scale = make_pd(1., 2) - df = 4 - w = distributions.WishartCholesky(df, chol(scale)) - self.assertAllEqual(df * scale, w.mean().eval()) - - def testMode(self): - with self.cached_session(): - scale = make_pd(1., 2) - df = 4 - w = distributions.WishartCholesky(df, chol(scale)) - self.assertAllEqual((df - 2. - 1.) * scale, w.mode().eval()) - - def testStd(self): - with self.cached_session(): - scale = make_pd(1., 2) - df = 4 - w = distributions.WishartCholesky(df, chol(scale)) - self.assertAllCloseAccordingToType( - chol(wishart_var(df, scale)), - w.stddev().eval()) - - def testVariance(self): - with self.cached_session(): - scale = make_pd(1., 2) - df = 4 - w = distributions.WishartCholesky(df, chol(scale)) - self.assertAllCloseAccordingToType( - wishart_var(df, scale), - w.variance().eval()) - - def testSample(self): - with self.cached_session(): - scale = make_pd(1., 2) - df = 4 - - chol_w = distributions.WishartCholesky( - df, chol(scale), cholesky_input_output_matrices=False) - - x = chol_w.sample(1, seed=42).eval() - chol_x = [chol(x[0])] - - full_w = distributions.WishartFull( - df, scale, cholesky_input_output_matrices=False) - self.assertAllClose(x, full_w.sample(1, seed=42).eval()) - - chol_w_chol = distributions.WishartCholesky( - df, chol(scale), cholesky_input_output_matrices=True) - self.assertAllClose(chol_x, chol_w_chol.sample(1, seed=42).eval()) - eigen_values = array_ops.matrix_diag_part( - chol_w_chol.sample( - 1000, seed=42)) - np.testing.assert_array_less(0., eigen_values.eval()) - - full_w_chol = distributions.WishartFull( - df, scale, cholesky_input_output_matrices=True) - self.assertAllClose(chol_x, full_w_chol.sample(1, seed=42).eval()) - eigen_values = array_ops.matrix_diag_part( - full_w_chol.sample( - 1000, seed=42)) - np.testing.assert_array_less(0., eigen_values.eval()) - - # Check first and second moments. - df = 4. - chol_w = distributions.WishartCholesky( - df=df, - scale=chol(make_pd(1., 3)), - cholesky_input_output_matrices=False) - x = chol_w.sample(10000, seed=42) - self.assertAllEqual((10000, 3, 3), x.get_shape()) - - moment1_estimate = math_ops.reduce_mean(x, axis=[0]).eval() - self.assertAllClose(chol_w.mean().eval(), moment1_estimate, rtol=0.05) - - # The Variance estimate uses the squares rather than outer-products - # because Wishart.Variance is the diagonal of the Wishart covariance - # matrix. - variance_estimate = (math_ops.reduce_mean(math_ops.square(x), axis=[0]) - - math_ops.square(moment1_estimate)).eval() - self.assertAllClose( - chol_w.variance().eval(), variance_estimate, rtol=0.05) - - # Test that sampling with the same seed twice gives the same results. - def testSampleMultipleTimes(self): - with self.cached_session(): - df = 4. - n_val = 100 - - random_seed.set_random_seed(654321) - chol_w1 = distributions.WishartCholesky( - df=df, - scale=chol(make_pd(1., 3)), - cholesky_input_output_matrices=False, - name="wishart1") - samples1 = chol_w1.sample(n_val, seed=123456).eval() - - random_seed.set_random_seed(654321) - chol_w2 = distributions.WishartCholesky( - df=df, - scale=chol(make_pd(1., 3)), - cholesky_input_output_matrices=False, - name="wishart2") - samples2 = chol_w2.sample(n_val, seed=123456).eval() - - self.assertAllClose(samples1, samples2) - - def testProb(self): - with self.cached_session(): - # Generate some positive definite (pd) matrices and their Cholesky - # factorizations. - x = np.array( - [make_pd(1., 2), make_pd(2., 2), make_pd(3., 2), make_pd(4., 2)]) - chol_x = np.array([chol(x[0]), chol(x[1]), chol(x[2]), chol(x[3])]) - - # Since Wishart wasn"t added to SciPy until 0.16, we'll spot check some - # pdfs with hard-coded results from upstream SciPy. - - log_prob_df_seq = np.array([ - # math.log(stats.wishart.pdf(x[0], df=2+0, scale=x[0])) - -3.5310242469692907, - # math.log(stats.wishart.pdf(x[1], df=2+1, scale=x[1])) - -7.689907330328961, - # math.log(stats.wishart.pdf(x[2], df=2+2, scale=x[2])) - -10.815845159537895, - # math.log(stats.wishart.pdf(x[3], df=2+3, scale=x[3])) - -13.640549882916691, - ]) - - # This test checks that batches don't interfere with correctness. - w = distributions.WishartCholesky( - df=[2, 3, 4, 5], - scale=chol_x, - cholesky_input_output_matrices=True) - self.assertAllClose(log_prob_df_seq, w.log_prob(chol_x).eval()) - - # Now we test various constructions of Wishart with different sample - # shape. - - log_prob = np.array([ - # math.log(stats.wishart.pdf(x[0], df=4, scale=x[0])) - -4.224171427529236, - # math.log(stats.wishart.pdf(x[1], df=4, scale=x[0])) - -6.3378770664093453, - # math.log(stats.wishart.pdf(x[2], df=4, scale=x[0])) - -12.026946850193017, - # math.log(stats.wishart.pdf(x[3], df=4, scale=x[0])) - -20.951582705289454, - ]) - - for w in ( - distributions.WishartCholesky( - df=4, - scale=chol_x[0], - cholesky_input_output_matrices=False), - distributions.WishartFull( - df=4, - scale=x[0], - cholesky_input_output_matrices=False)): - self.assertAllEqual((2, 2), w.event_shape_tensor().eval()) - self.assertEqual(2, w.dimension.eval()) - self.assertAllClose(log_prob[0], w.log_prob(x[0]).eval()) - self.assertAllClose(log_prob[0:2], w.log_prob(x[0:2]).eval()) - self.assertAllClose( - np.reshape(log_prob, (2, 2)), - w.log_prob(np.reshape(x, (2, 2, 2, 2))).eval()) - self.assertAllClose( - np.reshape(np.exp(log_prob), (2, 2)), - w.prob(np.reshape(x, (2, 2, 2, 2))).eval()) - self.assertAllEqual((2, 2), - w.log_prob(np.reshape(x, (2, 2, 2, 2))).get_shape()) - - for w in ( - distributions.WishartCholesky( - df=4, - scale=chol_x[0], - cholesky_input_output_matrices=True), - distributions.WishartFull( - df=4, - scale=x[0], - cholesky_input_output_matrices=True)): - self.assertAllEqual((2, 2), w.event_shape_tensor().eval()) - self.assertEqual(2, w.dimension.eval()) - self.assertAllClose(log_prob[0], w.log_prob(chol_x[0]).eval()) - self.assertAllClose(log_prob[0:2], w.log_prob(chol_x[0:2]).eval()) - self.assertAllClose( - np.reshape(log_prob, (2, 2)), - w.log_prob(np.reshape(chol_x, (2, 2, 2, 2))).eval()) - self.assertAllClose( - np.reshape(np.exp(log_prob), (2, 2)), - w.prob(np.reshape(chol_x, (2, 2, 2, 2))).eval()) - self.assertAllEqual((2, 2), - w.log_prob(np.reshape(x, (2, 2, 2, 2))).get_shape()) - - def testBatchShape(self): - with self.cached_session() as sess: - scale = make_pd(1., 2) - chol_scale = chol(scale) - - w = distributions.WishartCholesky(df=4, scale=chol_scale) - self.assertAllEqual([], w.batch_shape) - self.assertAllEqual([], w.batch_shape_tensor().eval()) - - w = distributions.WishartCholesky( - df=[4., 4], scale=np.array([chol_scale, chol_scale])) - self.assertAllEqual([2], w.batch_shape) - self.assertAllEqual([2], w.batch_shape_tensor().eval()) - - scale_deferred = array_ops.placeholder(dtypes.float32) - w = distributions.WishartCholesky(df=4, scale=scale_deferred) - self.assertAllEqual( - [], sess.run(w.batch_shape_tensor(), - feed_dict={scale_deferred: chol_scale})) - self.assertAllEqual( - [2], - sess.run(w.batch_shape_tensor(), - feed_dict={scale_deferred: [chol_scale, chol_scale]})) - - def testEventShape(self): - with self.cached_session() as sess: - scale = make_pd(1., 2) - chol_scale = chol(scale) - - w = distributions.WishartCholesky(df=4, scale=chol_scale) - self.assertAllEqual([2, 2], w.event_shape) - self.assertAllEqual([2, 2], w.event_shape_tensor().eval()) - - w = distributions.WishartCholesky( - df=[4., 4], scale=np.array([chol_scale, chol_scale])) - self.assertAllEqual([2, 2], w.event_shape) - self.assertAllEqual([2, 2], w.event_shape_tensor().eval()) - - scale_deferred = array_ops.placeholder(dtypes.float32) - w = distributions.WishartCholesky(df=4, scale=scale_deferred) - self.assertAllEqual( - [2, 2], - sess.run(w.event_shape_tensor(), - feed_dict={scale_deferred: chol_scale})) - self.assertAllEqual( - [2, 2], - sess.run(w.event_shape_tensor(), - feed_dict={scale_deferred: [chol_scale, chol_scale]})) - - @test_util.disable_xla("XLA cannot assert inside of an op.") - def testValidateArgs(self): - with self.cached_session() as sess: - df_deferred = array_ops.placeholder(dtypes.float32) - chol_scale_deferred = array_ops.placeholder(dtypes.float32) - x = make_pd(1., 3) - chol_scale = chol(x) - - # Check expensive, deferred assertions. - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - "cannot be less than"): - chol_w = distributions.WishartCholesky( - df=df_deferred, - scale=chol_scale_deferred, - validate_args=True) - sess.run(chol_w.log_prob(np.asarray( - x, dtype=np.float32)), - feed_dict={df_deferred: 2., - chol_scale_deferred: chol_scale}) - - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - "Cholesky decomposition was not successful"): - chol_w = distributions.WishartFull( - df=df_deferred, scale=chol_scale_deferred) - # np.ones((3, 3)) is not positive, definite. - sess.run(chol_w.log_prob(np.asarray( - x, dtype=np.float32)), - feed_dict={ - df_deferred: 4., - chol_scale_deferred: np.ones( - (3, 3), dtype=np.float32) - }) - - with self.assertRaisesOpError("scale must be square"): - chol_w = distributions.WishartCholesky( - df=4., - scale=np.array([[2., 3., 4.], [1., 2., 3.]], dtype=np.float32), - validate_args=True) - sess.run(chol_w.scale().eval()) - - # Ensure no assertions. - chol_w = distributions.WishartCholesky( - df=df_deferred, - scale=chol_scale_deferred, - validate_args=False) - sess.run(chol_w.log_prob(np.asarray( - x, dtype=np.float32)), - feed_dict={df_deferred: 4, - chol_scale_deferred: chol_scale}) - # Bogus log_prob, but since we have no checks running... c"est la vie. - sess.run(chol_w.log_prob(np.asarray( - x, dtype=np.float32)), - feed_dict={df_deferred: 4, - chol_scale_deferred: np.ones((3, 3))}) - - def testStaticAsserts(self): - with self.cached_session(): - x = make_pd(1., 3) - chol_scale = chol(x) - - # Still has these assertions because they're resolveable at graph - # construction - with self.assertRaisesRegexp(ValueError, "cannot be less than"): - distributions.WishartCholesky( - df=2, scale=chol_scale, validate_args=False) - with self.assertRaisesRegexp(TypeError, "."): - distributions.WishartCholesky( - df=4., - scale=np.asarray( - chol_scale, dtype=np.int32), - validate_args=False) - - def testSampleBroadcasts(self): - dims = 2 - batch_shape = [2, 3] - sample_shape = [2, 1] - scale = np.float32([ - [[1., 0.5], - [0.5, 1.]], - [[0.5, 0.25], - [0.25, 0.75]], - ]) - scale = np.reshape(np.concatenate([scale, scale, scale], axis=0), - batch_shape + [dims, dims]) - wishart = distributions.WishartFull(df=5, scale=scale) - x = wishart.sample(sample_shape, seed=42) - with self.cached_session() as sess: - x_ = sess.run(x) - expected_shape = sample_shape + batch_shape + [dims, dims] - self.assertAllEqual(expected_shape, x.shape) - self.assertAllEqual(expected_shape, x_.shape) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distributions/python/ops/autoregressive.py b/tensorflow/contrib/distributions/python/ops/autoregressive.py deleted file mode 100644 index 3ba1c3a6651..00000000000 --- a/tensorflow/contrib/distributions/python/ops/autoregressive.py +++ /dev/null @@ -1,222 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Autoregressive distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import ops -from tensorflow.python.ops.distributions import distribution as distribution_lib -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - - -class Autoregressive(distribution_lib.Distribution): - """Autoregressive distributions. - - The Autoregressive distribution enables learning (often) richer multivariate - distributions by repeatedly applying a [diffeomorphic]( - https://en.wikipedia.org/wiki/Diffeomorphism) transformation (such as - implemented by `Bijector`s). Regarding terminology, - - "Autoregressive models decompose the joint density as a product of - conditionals, and model each conditional in turn. Normalizing flows - transform a base density (e.g. a standard Gaussian) into the target density - by an invertible transformation with tractable Jacobian." [(Papamakarios et - al., 2016)][1] - - In other words, the "autoregressive property" is equivalent to the - decomposition, `p(x) = prod{ p(x[i] | x[0:i]) : i=0, ..., d }`. The provided - `shift_and_log_scale_fn`, `masked_autoregressive_default_template`, achieves - this property by zeroing out weights in its `masked_dense` layers. - - Practically speaking the autoregressive property means that there exists a - permutation of the event coordinates such that each coordinate is a - diffeomorphic function of only preceding coordinates - [(van den Oord et al., 2016)][2]. - - #### Mathematical Details - - The probability function is - - ```none - prob(x; fn, n) = fn(x).prob(x) - ``` - - And a sample is generated by - - ```none - x = fn(...fn(fn(x0).sample()).sample()).sample() - ``` - - where the ellipses (`...`) represent `n-2` composed calls to `fn`, `fn` - constructs a `tfp.distributions.Distribution`-like instance, and `x0` is a - fixed initializing `Tensor`. - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - def normal_fn(self, event_size): - n = event_size * (event_size + 1) / 2 - p = tf.Variable(tfd.Normal(loc=0., scale=1.).sample(n)) - affine = tfd.bijectors.Affine( - scale_tril=tfd.fill_triangular(0.25 * p)) - def _fn(samples): - scale = math_ops.exp(affine.forward(samples)).eval() - return independent_lib.Independent( - normal_lib.Normal(loc=0., scale=scale, validate_args=True), - reinterpreted_batch_ndims=1) - return _fn - - batch_and_event_shape = [3, 2, 4] - sample0 = array_ops.zeros(batch_and_event_shape) - ar = autoregressive_lib.Autoregressive( - self._normal_fn(batch_and_event_shape[-1]), sample0) - x = ar.sample([6, 5]) - # ==> x.shape = [6, 5, 3, 2, 4] - prob_x = ar.prob(x) - # ==> x.shape = [6, 5, 3, 2] - - ``` - - #### References - - [1]: George Papamakarios, Theo Pavlakou, and Iain Murray. Masked - Autoregressive Flow for Density Estimation. In _Neural Information - Processing Systems_, 2017. https://arxiv.org/abs/1705.07057 - - [2]: Aaron van den Oord, Nal Kalchbrenner, Oriol Vinyals, Lasse Espeholt, - Alex Graves, and Koray Kavukcuoglu. Conditional Image Generation with - PixelCNN Decoders. In _Neural Information Processing Systems_, 2016. - https://arxiv.org/abs/1606.05328 - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - distribution_fn, - sample0=None, - num_steps=None, - validate_args=False, - allow_nan_stats=True, - name="Autoregressive"): - """Construct an `Autoregressive` distribution. - - Args: - distribution_fn: Python `callable` which constructs a - `tfp.distributions.Distribution`-like instance from a `Tensor` (e.g., - `sample0`). The function must respect the "autoregressive property", - i.e., there exists a permutation of event such that each coordinate is a - diffeomorphic function of on preceding coordinates. - sample0: Initial input to `distribution_fn`; used to - build the distribution in `__init__` which in turn specifies this - distribution's properties, e.g., `event_shape`, `batch_shape`, `dtype`. - If unspecified, then `distribution_fn` should be default constructable. - num_steps: Number of times `distribution_fn` is composed from samples, - e.g., `num_steps=2` implies - `distribution_fn(distribution_fn(sample0).sample(n)).sample()`. - validate_args: Python `bool`. Whether to validate input with asserts. - If `validate_args` is `False`, and the inputs are invalid, - correct behavior is not guaranteed. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - Default value: "Autoregressive". - - Raises: - ValueError: if `num_steps` and - `distribution_fn(sample0).event_shape.num_elements()` are both `None`. - ValueError: if `num_steps < 1`. - """ - parameters = dict(locals()) - with ops.name_scope(name) as name: - self._distribution_fn = distribution_fn - self._sample0 = sample0 - self._distribution0 = (distribution_fn() if sample0 is None - else distribution_fn(sample0)) - if num_steps is None: - num_steps = self._distribution0.event_shape.num_elements() - if num_steps is None: - raise ValueError("distribution_fn must generate a distribution " - "with fully known `event_shape`.") - if num_steps < 1: - raise ValueError("num_steps ({}) must be at least 1.".format(num_steps)) - self._num_steps = num_steps - super(Autoregressive, self).__init__( - dtype=self._distribution0.dtype, - reparameterization_type=self._distribution0.reparameterization_type, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=self._distribution0._graph_parents, # pylint: disable=protected-access - name=name) - - @property - def distribution_fn(self): - return self._distribution_fn - - @property - def sample0(self): - return self._sample0 - - @property - def num_steps(self): - return self._num_steps - - @property - def distribution0(self): - return self._distribution0 - - def _batch_shape(self): - return self.distribution0.batch_shape - - def _batch_shape_tensor(self): - return self.distribution0.batch_shape_tensor() - - def _event_shape(self): - return self.distribution0.event_shape - - def _event_shape_tensor(self): - return self.distribution0.event_shape_tensor() - - def _sample_n(self, n, seed=None): - if seed is None: - seed = distribution_util.gen_new_seed( - seed=np.random.randint(2**32 - 1), - salt="autoregressive") - samples = self.distribution0.sample(n, seed=seed) - for _ in range(self._num_steps): - samples = self.distribution_fn(samples).sample(seed=seed) - return samples - - def _log_prob(self, value): - return self.distribution_fn(value).log_prob(value) - - def _prob(self, value): - return self.distribution_fn(value).prob(value) diff --git a/tensorflow/contrib/distributions/python/ops/batch_reshape.py b/tensorflow/contrib/distributions/python/ops/batch_reshape.py deleted file mode 100644 index e174596defd..00000000000 --- a/tensorflow/contrib/distributions/python/ops/batch_reshape.py +++ /dev/null @@ -1,432 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The BatchReshape distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import distribution as distribution_lib -from tensorflow.python.util import deprecation - - -__all__ = [ - "BatchReshape", -] - - -class BatchReshape(distribution_lib.Distribution): - """The Batch-Reshaping distribution. - - This "meta-distribution" reshapes the batch dimensions of another - distribution. - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - dtype = np.float32 - dims = 2 - new_batch_shape = [1, 2, -1] - old_batch_shape = [6] - - scale = np.ones(old_batch_shape + [dims], dtype) - mvn = tfd.MultivariateNormalDiag(scale_diag=scale) - reshape_mvn = tfd.BatchReshape( - distribution=mvn, - batch_shape=new_batch_shape, - validate_args=True) - - reshape_mvn.batch_shape - # ==> [1, 2, 3] - - x = reshape_mvn.sample(sample_shape=[4, 5]) - x.shape - # ==> [4, 5, 1, 2, 3, 2] == sample_shape + new_batch_shape + [dims] - - reshape_mvn.log_prob(x).shape - # ==> [4, 5, 1, 2, 3] == sample_shape + new_batch_shape - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - distribution, - batch_shape, - validate_args=False, - allow_nan_stats=True, - name=None): - """Construct BatchReshape distribution. - - Args: - distribution: The base distribution instance to reshape. Typically an - instance of `Distribution`. - batch_shape: Positive `int`-like vector-shaped `Tensor` representing - the new shape of the batch dimensions. Up to one dimension may contain - `-1`, meaning the remainder of the batch size. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: The name to give Ops created by the initializer. - Default value: `"BatchReshape" + distribution.name`. - - Raises: - ValueError: if `batch_shape` is not a vector. - ValueError: if `batch_shape` has non-positive elements. - ValueError: if `batch_shape` size is not the same as a - `distribution.batch_shape` size. - """ - parameters = dict(locals()) - name = name or "BatchReshape" + distribution.name - with ops.name_scope(name, values=[batch_shape]) as name: - # The unexpanded batch shape may contain up to one dimension of -1. - self._batch_shape_unexpanded = ops.convert_to_tensor( - batch_shape, dtype=dtypes.int32, name="batch_shape") - validate_init_args_statically(distribution, self._batch_shape_unexpanded) - batch_shape, batch_shape_static, runtime_assertions = calculate_reshape( - distribution.batch_shape_tensor(), self._batch_shape_unexpanded, - validate_args) - self._distribution = distribution - self._batch_shape_ = batch_shape - self._batch_shape_static = batch_shape_static - self._runtime_assertions = runtime_assertions - super(BatchReshape, self).__init__( - dtype=distribution.dtype, - reparameterization_type=distribution.reparameterization_type, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=( - [self._batch_shape_unexpanded] + distribution._graph_parents), # pylint: disable=protected-access - name=name) - - @property - def distribution(self): - return self._distribution - - def _batch_shape_tensor(self): - with ops.control_dependencies(self._runtime_assertions): - return array_ops.identity(self._batch_shape_) - - def _batch_shape(self): - return self._batch_shape_static - - def _event_shape_tensor(self): - with ops.control_dependencies(self._runtime_assertions): - return array_ops.identity(self.distribution.event_shape_tensor()) - - def _event_shape(self): - return self.distribution.event_shape - - def _sample_n(self, n, seed=None): - with ops.control_dependencies(self._runtime_assertions): - x = self.distribution.sample(sample_shape=n, seed=seed) - new_shape = array_ops.concat( - [ - [n], - self._batch_shape_unexpanded, - self.event_shape_tensor(), - ], - axis=0) - return array_ops.reshape(x, new_shape) - - def _log_prob(self, x): - return self._call_reshape_input_output( - self.distribution.log_prob, x) - - def _prob(self, x): - return self._call_reshape_input_output( - self.distribution.prob, x) - - def _log_cdf(self, x): - return self._call_reshape_input_output( - self.distribution.log_cdf, x) - - def _cdf(self, x): - return self._call_reshape_input_output( - self.distribution.cdf, x) - - def _log_survival_function(self, x): - return self._call_reshape_input_output( - self.distribution.log_survival_function, x) - - def _survival_function(self, x): - return self._call_reshape_input_output( - self.distribution.survival_function, x) - - def _entropy(self): - return self._call_and_reshape_output(self.distribution.entropy, [], - [tensor_shape.TensorShape([])]) - - def _mean(self): - return self._call_and_reshape_output(self.distribution.mean) - - def _mode(self): - return self._call_and_reshape_output(self.distribution.mode) - - def _stddev(self): - return self._call_and_reshape_output(self.distribution.stddev) - - def _variance(self): - return self._call_and_reshape_output(self.distribution.variance) - - def _covariance(self): - return self._call_and_reshape_output( - self.distribution.covariance, - [self.event_shape_tensor()]*2, - [self.event_shape]*2) - - def _sample_shape(self, x): - """Computes graph and static `sample_shape`.""" - x_ndims = (array_ops.rank(x) if x.shape.ndims is None else x.shape.ndims) - event_ndims = (array_ops.size(self.event_shape_tensor()) - if self.event_shape.ndims is None - else self.event_shape.ndims) - batch_ndims = ( - array_ops.size(self._batch_shape_unexpanded) - if self.batch_shape.ndims is None else self.batch_shape.ndims) - sample_ndims = x_ndims - batch_ndims - event_ndims - if isinstance(sample_ndims, int): - static_sample_shape = x.shape[:sample_ndims] - else: - static_sample_shape = tensor_shape.TensorShape(None) - if static_sample_shape.is_fully_defined(): - sample_shape = np.int32(static_sample_shape.as_list()) - else: - sample_shape = array_ops.shape(x)[:sample_ndims] - return sample_shape, static_sample_shape - - def _call_reshape_input_output(self, fn, x): - """Calls `fn`, appropriately reshaping its input `x` and output.""" - with ops.control_dependencies( - self._runtime_assertions + self._validate_sample_arg(x)): - sample_shape, static_sample_shape = self._sample_shape(x) - old_shape = array_ops.concat([ - sample_shape, - self.distribution.batch_shape_tensor(), - self.event_shape_tensor(), - ], axis=0) - result = fn(array_ops.reshape(x, old_shape)) - new_shape = array_ops.concat( - [ - sample_shape, - self._batch_shape_unexpanded, - ], axis=0) - result = array_ops.reshape(result, new_shape) - if (static_sample_shape.ndims is not None and - self.batch_shape.ndims is not None): - new_shape = static_sample_shape.concatenate(self.batch_shape) - result.set_shape(result.shape.merge_with(new_shape)) - return result - - def _call_and_reshape_output( - self, - fn, - event_shape_list=None, - static_event_shape_list=None): - """Calls `fn` and appropriately reshapes its output.""" - with ops.control_dependencies(self._runtime_assertions): - if event_shape_list is None: - event_shape_list = [self._event_shape_tensor()] - if static_event_shape_list is None: - static_event_shape_list = [self.event_shape] - new_shape = array_ops.concat( - [self._batch_shape_unexpanded] + event_shape_list, axis=0) - result = array_ops.reshape(fn(), new_shape) - if (self.batch_shape.ndims is not None and - self.event_shape.ndims is not None): - event_shape = tensor_shape.TensorShape([]) - for rss in static_event_shape_list: - event_shape = event_shape.concatenate(rss) - static_shape = result.shape.merge_with( - self.batch_shape.concatenate(event_shape)) - result.set_shape(static_shape) - return result - - def _validate_sample_arg(self, x): - """Helper which validates sample arg, e.g., input to `log_prob`.""" - with ops.name_scope(name="validate_sample_arg", values=[x]): - x_ndims = (array_ops.rank(x) if x.shape.ndims is None else x.shape.ndims) - event_ndims = (array_ops.size(self.event_shape_tensor()) - if self.event_shape.ndims is None - else self.event_shape.ndims) - batch_ndims = ( - array_ops.size(self._batch_shape_unexpanded) - if self.batch_shape.ndims is None else self.batch_shape.ndims) - expected_batch_event_ndims = batch_ndims + event_ndims - - if (isinstance(x_ndims, int) and - isinstance(expected_batch_event_ndims, int)): - if x_ndims < expected_batch_event_ndims: - raise NotImplementedError( - "Broadcasting is not supported; too few batch and event dims " - "(expected at least {}, saw {}).".format( - expected_batch_event_ndims, x_ndims)) - ndims_assertion = [] - elif self.validate_args: - ndims_assertion = [ - check_ops.assert_greater_equal( - x_ndims, - expected_batch_event_ndims, - message=("Broadcasting is not supported; too few " - "batch and event dims."), - name="assert_batch_and_event_ndims_large_enough"), - ] - - if (self.batch_shape.is_fully_defined() and - self.event_shape.is_fully_defined()): - expected_batch_event_shape = np.int32(self.batch_shape.concatenate( - self.event_shape).as_list()) - else: - expected_batch_event_shape = array_ops.concat([ - self.batch_shape_tensor(), - self.event_shape_tensor(), - ], axis=0) - - sample_ndims = x_ndims - expected_batch_event_ndims - if isinstance(sample_ndims, int): - sample_ndims = max(sample_ndims, 0) - if (isinstance(sample_ndims, int) and - x.shape[sample_ndims:].is_fully_defined()): - actual_batch_event_shape = np.int32(x.shape[sample_ndims:].as_list()) - else: - sample_ndims = math_ops.maximum(sample_ndims, 0) - actual_batch_event_shape = array_ops.shape(x)[sample_ndims:] - - if (isinstance(expected_batch_event_shape, np.ndarray) and - isinstance(actual_batch_event_shape, np.ndarray)): - if any(expected_batch_event_shape != actual_batch_event_shape): - raise NotImplementedError("Broadcasting is not supported; " - "unexpected batch and event shape " - "(expected {}, saw {}).".format( - expected_batch_event_shape, - actual_batch_event_shape)) - # We need to set the final runtime-assertions to `ndims_assertion` since - # its possible this assertion was created. We could add a condition to - # only do so if `self.validate_args == True`, however this is redundant - # as `ndims_assertion` already encodes this information. - runtime_assertions = ndims_assertion - elif self.validate_args: - # We need to make the `ndims_assertion` a control dep because otherwise - # TF itself might raise an exception owing to this assertion being - # ill-defined, ie, one cannot even compare different rank Tensors. - with ops.control_dependencies(ndims_assertion): - shape_assertion = check_ops.assert_equal( - expected_batch_event_shape, - actual_batch_event_shape, - message=("Broadcasting is not supported; " - "unexpected batch and event shape."), - name="assert_batch_and_event_shape_same") - runtime_assertions = [shape_assertion] - else: - runtime_assertions = [] - - return runtime_assertions - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def calculate_reshape(original_shape, new_shape, validate=False, name=None): - """Calculates the reshaped dimensions (replacing up to one -1 in reshape).""" - batch_shape_static = tensor_util.constant_value_as_shape(new_shape) - if batch_shape_static.is_fully_defined(): - return np.int32(batch_shape_static.as_list()), batch_shape_static, [] - with ops.name_scope(name, "calculate_reshape", [original_shape, new_shape]): - original_size = math_ops.reduce_prod(original_shape) - implicit_dim = math_ops.equal(new_shape, -1) - size_implicit_dim = ( - original_size // math_ops.maximum(1, -math_ops.reduce_prod(new_shape))) - new_ndims = array_ops.shape(new_shape) - expanded_new_shape = array_ops.where_v2( # Assumes exactly one `-1`. - implicit_dim, array_ops.fill(new_ndims, size_implicit_dim), new_shape) - validations = [] if not validate else [ - check_ops.assert_rank( - original_shape, 1, message="Original shape must be a vector."), - check_ops.assert_rank( - new_shape, 1, message="New shape must be a vector."), - check_ops.assert_less_equal( - math_ops.count_nonzero(implicit_dim, dtype=dtypes.int32), - 1, - message="At most one dimension can be unknown."), - check_ops.assert_positive( - expanded_new_shape, message="Shape elements must be >=-1."), - check_ops.assert_equal( - math_ops.reduce_prod(expanded_new_shape), - original_size, - message="Shape sizes do not match."), - ] - return expanded_new_shape, batch_shape_static, validations - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def validate_init_args_statically(distribution, batch_shape): - """Helper to __init__ which makes or raises assertions.""" - if batch_shape.shape.ndims is not None: - if batch_shape.shape.ndims != 1: - raise ValueError("`batch_shape` must be a vector " - "(saw rank: {}).".format(batch_shape.shape.ndims)) - - batch_shape_static = tensor_util.constant_value_as_shape(batch_shape) - batch_size_static = batch_shape_static.num_elements() - dist_batch_size_static = distribution.batch_shape.num_elements() - - if batch_size_static is not None and dist_batch_size_static is not None: - if batch_size_static != dist_batch_size_static: - raise ValueError("`batch_shape` size ({}) must match " - "`distribution.batch_shape` size ({}).".format( - batch_size_static, dist_batch_size_static)) - - if batch_shape_static.dims is not None: - if any( - dim.value is not None and - dim.value < 1 for dim in batch_shape_static.dims): - raise ValueError("`batch_shape` elements must be >=-1.") diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py deleted file mode 100644 index 3b17de9b8a9..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Bijector Ops. - -Use [tfp.bijectors](/probability/api_docs/python/tfp/bijectors) instead. - -@@AbsoluteValue -@@Affine -@@AffineLinearOperator -@@AffineScalar -@@Bijector -@@BatchNormalization -@@Chain -@@CholeskyOuterProduct -@@ConditionalBijector -@@Exp -@@FillTriangular -@@Gumbel -@@Identity -@@Inline -@@Invert -@@Kumaraswamy -@@MaskedAutoregressiveFlow -@@MatrixInverseTriL -@@Ordered -@@Permute -@@PowerTransform -@@RealNVP -@@Reshape -@@ScaleTriL -@@Sigmoid -@@SinhArcsinh -@@SoftmaxCentered -@@Softplus -@@Softsign -@@Square -@@TransformDiagonal -@@Weibull - -@@masked_autoregressive_default_template -@@masked_dense -@@real_nvp_default_template -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import,line-too-long,g-importing-member - -from tensorflow.contrib.distributions.python.ops.bijectors.absolute_value import * -from tensorflow.contrib.distributions.python.ops.bijectors.affine import * -from tensorflow.contrib.distributions.python.ops.bijectors.affine_linear_operator import * -from tensorflow.contrib.distributions.python.ops.bijectors.affine_scalar import * -from tensorflow.contrib.distributions.python.ops.bijectors.batch_normalization import * -from tensorflow.contrib.distributions.python.ops.bijectors.chain import * -from tensorflow.contrib.distributions.python.ops.bijectors.cholesky_outer_product import * -from tensorflow.contrib.distributions.python.ops.bijectors.conditional_bijector import * -from tensorflow.contrib.distributions.python.ops.bijectors.exp import * -from tensorflow.contrib.distributions.python.ops.bijectors.fill_triangular import * -from tensorflow.contrib.distributions.python.ops.bijectors.gumbel import * -from tensorflow.contrib.distributions.python.ops.bijectors.inline import * -from tensorflow.contrib.distributions.python.ops.bijectors.invert import * -from tensorflow.contrib.distributions.python.ops.bijectors.kumaraswamy import * -from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import * -from tensorflow.contrib.distributions.python.ops.bijectors.matrix_inverse_tril import * -from tensorflow.contrib.distributions.python.ops.bijectors.ordered import * -from tensorflow.contrib.distributions.python.ops.bijectors.permute import * -from tensorflow.contrib.distributions.python.ops.bijectors.power_transform import * -from tensorflow.contrib.distributions.python.ops.bijectors.real_nvp import * -from tensorflow.contrib.distributions.python.ops.bijectors.reshape import * -from tensorflow.contrib.distributions.python.ops.bijectors.scale_tril import * -from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid import * -from tensorflow.contrib.distributions.python.ops.bijectors.sinh_arcsinh import * -from tensorflow.contrib.distributions.python.ops.bijectors.softmax_centered import * -from tensorflow.contrib.distributions.python.ops.bijectors.softplus import * -from tensorflow.contrib.distributions.python.ops.bijectors.softsign import * -from tensorflow.contrib.distributions.python.ops.bijectors.square import * -from tensorflow.contrib.distributions.python.ops.bijectors.transform_diagonal import * -from tensorflow.python.ops.distributions.bijector import * -from tensorflow.python.ops.distributions.identity_bijector import Identity - -# pylint: enable=unused-import,wildcard-import,line-too-long,g-importing-member - -from tensorflow.python.util.all_util import remove_undocumented - -remove_undocumented(__name__) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value.py b/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value.py deleted file mode 100644 index 4d6a46e7358..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""AbsoluteValue bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - -__all__ = [ - "AbsoluteValue", -] - - -class AbsoluteValue(bijector.Bijector): - """Computes `Y = g(X) = Abs(X)`, element-wise. - - This non-injective bijector allows for transformations of scalar distributions - with the absolute value function, which maps `(-inf, inf)` to `[0, inf)`. - - * For `y in (0, inf)`, `AbsoluteValue.inverse(y)` returns the set inverse - `{x in (-inf, inf) : |x| = y}` as a tuple, `-y, y`. - * `AbsoluteValue.inverse(0)` returns `0, 0`, which is not the set inverse - (the set inverse is the singleton `{0}`), but "works" in conjunction with - `TransformedDistribution` to produce a left semi-continuous pdf. - * For `y < 0`, `AbsoluteValue.inverse(y)` happily returns the - wrong thing, `-y, y`. This is done for efficiency. If - `validate_args == True`, `y < 0` will raise an exception. - - - ```python - tfd = tf.contrib.distributions - - abs = tfd.bijectors.AbsoluteValue() - - abs.forward([-1., 0., 1.]) - ==> [1., 0., 1.] - - abs.inverse(1.) - ==> [-1., 1.] - - # The |dX/dY| is constant, == 1. So Log|dX/dY| == 0. - abs.inverse_log_det_jacobian(1.) - ==> [0., 0.] - - # Special case handling of 0. - abs.inverse(0.) - ==> [0., 0.] - - abs.inverse_log_det_jacobian(0.) - ==> [0., 0.] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, validate_args=False, name="absolute_value"): - """Instantiates the `AbsoluteValue` bijector. - - Args: - validate_args: Python `bool` indicating whether arguments should be - checked for correctness, in particular whether inputs to `inverse` and - `inverse_log_det_jacobian` are non-negative. - name: Python `str` name given to ops managed by this object. - """ - self._graph_parents = [] - self._name = name - - with self._name_scope("init"): - super(AbsoluteValue, self).__init__( - forward_min_event_ndims=0, - is_constant_jacobian=True, - validate_args=validate_args, - name=name) - - def _forward(self, x): - return math_ops.abs(x) - - def _inverse(self, y): - if self.validate_args: - y = control_flow_ops.with_dependencies( - [check_ops.assert_non_negative(y, message="Argument y was negative")], - y) - return -y, y - - def _inverse_log_det_jacobian(self, y): - # If event_ndims = 2, - # F^{-1}(y) = (-y, y), so DF^{-1}(y) = (-1, 1), - # so Log|DF^{-1}(y)| = Log[1, 1] = [0, 0]. - zeros = constant_op.constant(0., dtype=y.dtype) - if self.validate_args: - zeros = control_flow_ops.with_dependencies( - [check_ops.assert_non_negative(y, message="Argument y was negative")], - zeros) - return zeros, zeros - - @property - def _is_injective(self): - return False diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/affine.py b/tensorflow/contrib/distributions/python/ops/bijectors/affine.py deleted file mode 100644 index 2e0fd592c6c..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/affine.py +++ /dev/null @@ -1,396 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Affine bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.contrib.distributions.python.ops.shape import _DistributionShape -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.ops.linalg import linalg -from tensorflow.python.util import deprecation - - -__all__ = [ - "Affine", -] - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _as_tensor(x, name): - """Convenience to convert to `Tensor` or leave as `None`.""" - return None if x is None else ops.convert_to_tensor(x, name=name) - - -class Affine(bijector.Bijector): - """Compute `Y = g(X; shift, scale) = scale @ X + shift`. - - Here `scale = c * I + diag(D1) + tril(L) + V @ diag(D2) @ V.T`. - - In TF parlance, the `scale` term is logically equivalent to: - - ```python - scale = ( - scale_identity_multiplier * tf.linalg.tensor_diag(tf.ones(d)) + - tf.linalg.tensor_diag(scale_diag) + - scale_tril + - scale_perturb_factor @ diag(scale_perturb_diag) @ - tf.transpose([scale_perturb_factor]) - ) - ``` - - The `scale` term is applied without necessarily materializing constituent - matrices, i.e., the matmul is [matrix-free]( - https://en.wikipedia.org/wiki/Matrix-free_methods) when possible. - - #### Examples - - ```python - # Y = X - b = Affine() - - # Y = X + shift - b = Affine(shift=[1., 2, 3]) - - # Y = 2 * I @ X.T + shift - b = Affine(shift=[1., 2, 3], - scale_identity_multiplier=2.) - - # Y = tf.linalg.tensor_diag(d1) @ X.T + shift - b = Affine(shift=[1., 2, 3], - scale_diag=[-1., 2, 1]) # Implicitly 3x3. - - # Y = (I + v * v.T) @ X.T + shift - b = Affine(shift=[1., 2, 3], - scale_perturb_factor=[[1., 0], - [0, 1], - [1, 1]]) - - # Y = (diag(d1) + v * diag(d2) * v.T) @ X.T + shift - b = Affine(shift=[1., 2, 3], - scale_diag=[1., 3, 3], # Implicitly 3x3. - scale_perturb_diag=[2., 1], # Implicitly 2x2. - scale_perturb_factor=[[1., 0], - [0, 1], - [1, 1]]) - - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - shift=None, - scale_identity_multiplier=None, - scale_diag=None, - scale_tril=None, - scale_perturb_factor=None, - scale_perturb_diag=None, - validate_args=False, - name="affine"): - """Instantiates the `Affine` bijector. - - This `Bijector` is initialized with `shift` `Tensor` and `scale` arguments, - giving the forward operation: - - ```none - Y = g(X) = scale @ X + shift - ``` - - where the `scale` term is logically equivalent to: - - ```python - scale = ( - scale_identity_multiplier * tf.linalg.tensor_diag(tf.ones(d)) + - tf.linalg.tensor_diag(scale_diag) + - scale_tril + - scale_perturb_factor @ diag(scale_perturb_diag) @ - tf.transpose([scale_perturb_factor]) - ) - ``` - - If none of `scale_identity_multiplier`, `scale_diag`, or `scale_tril` are - specified then `scale += IdentityMatrix`. Otherwise specifying a - `scale` argument has the semantics of `scale += Expand(arg)`, i.e., - `scale_diag != None` means `scale += tf.linalg.tensor_diag(scale_diag)`. - - Args: - shift: Floating-point `Tensor`. If this is set to `None`, no shift is - applied. - scale_identity_multiplier: floating point rank 0 `Tensor` representing a - scaling done to the identity matrix. - When `scale_identity_multiplier = scale_diag = scale_tril = None` then - `scale += IdentityMatrix`. Otherwise no scaled-identity-matrix is added - to `scale`. - scale_diag: Floating-point `Tensor` representing the diagonal matrix. - `scale_diag` has shape [N1, N2, ... k], which represents a k x k - diagonal matrix. - When `None` no diagonal term is added to `scale`. - scale_tril: Floating-point `Tensor` representing the diagonal matrix. - `scale_diag` has shape [N1, N2, ... k, k], which represents a k x k - lower triangular matrix. - When `None` no `scale_tril` term is added to `scale`. - The upper triangular elements above the diagonal are ignored. - scale_perturb_factor: Floating-point `Tensor` representing factor matrix - with last two dimensions of shape `(k, r)`. When `None`, no rank-r - update is added to `scale`. - scale_perturb_diag: Floating-point `Tensor` representing the diagonal - matrix. `scale_perturb_diag` has shape [N1, N2, ... r], which - represents an `r x r` diagonal matrix. When `None` low rank updates will - take the form `scale_perturb_factor * scale_perturb_factor.T`. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - - Raises: - ValueError: if `perturb_diag` is specified but not `perturb_factor`. - TypeError: if `shift` has different `dtype` from `scale` arguments. - """ - self._graph_parents = [] - self._name = name - self._validate_args = validate_args - - # Ambiguous definition of low rank update. - if scale_perturb_diag is not None and scale_perturb_factor is None: - raise ValueError("When scale_perturb_diag is specified, " - "scale_perturb_factor must be specified.") - - # Special case, only handling a scaled identity matrix. We don't know its - # dimensions, so this is special cased. - # We don't check identity_multiplier, since below we set it to 1. if all - # other scale args are None. - self._is_only_identity_multiplier = (scale_tril is None and - scale_diag is None and - scale_perturb_factor is None) - - with self._name_scope("init", values=[ - shift, scale_identity_multiplier, scale_diag, scale_tril, - scale_perturb_diag, scale_perturb_factor]): - - # In the absence of `loc` and `scale`, we'll assume `dtype` is `float32`. - dtype = dtypes.float32 - - if shift is not None: - shift = ops.convert_to_tensor(shift, name="shift") - dtype = shift.dtype.base_dtype - self._shift = shift - - # When no args are specified, pretend the scale matrix is the identity - # matrix. - if (self._is_only_identity_multiplier and - scale_identity_multiplier is None): - scale_identity_multiplier = ops.convert_to_tensor(1., dtype=dtype) - - # self._create_scale_operator returns a LinearOperator in all cases - # except if self._is_only_identity_multiplier; in which case it - # returns a scalar Tensor. - scale = self._create_scale_operator( - identity_multiplier=scale_identity_multiplier, - diag=scale_diag, - tril=scale_tril, - perturb_diag=scale_perturb_diag, - perturb_factor=scale_perturb_factor, - shift=shift, - validate_args=validate_args) - - if scale.dtype is not None: - dtype = scale.dtype.base_dtype - - if scale is not None and not self._is_only_identity_multiplier: - if (shift is not None and - shift.dtype.base_dtype != scale.dtype.base_dtype): - raise TypeError( - "shift.dtype({}) is incompatible with scale.dtype({}).".format( - shift.dtype, scale.dtype)) - - if scale.tensor_rank is not None: - batch_ndims = scale.tensor_rank - 2 - else: - batch_ndims = scale.tensor_rank_tensor() - 2 - else: - # We won't need shape inference when scale is None or when scale is a - # scalar. - batch_ndims = 0 - self._scale = scale - self._shaper = _DistributionShape( - batch_ndims=batch_ndims, - event_ndims=1, - validate_args=validate_args) - super(Affine, self).__init__( - forward_min_event_ndims=1, - graph_parents=( - [self._shift] if self._shift is not None else []), - is_constant_jacobian=True, - dtype=dtype, - validate_args=validate_args, - name=name) - - def _create_scale_operator(self, identity_multiplier, diag, tril, - perturb_diag, perturb_factor, shift, - validate_args): - """Construct `scale` from various components. - - Args: - identity_multiplier: floating point rank 0 `Tensor` representing a scaling - done to the identity matrix. - diag: Floating-point `Tensor` representing the diagonal matrix. - `scale_diag` has shape [N1, N2, ... k], which represents a k x k - diagonal matrix. - tril: Floating-point `Tensor` representing the diagonal matrix. - `scale_tril` has shape [N1, N2, ... k], which represents a k x k lower - triangular matrix. - perturb_diag: Floating-point `Tensor` representing the diagonal matrix of - the low rank update. - perturb_factor: Floating-point `Tensor` representing factor matrix. - shift: Floating-point `Tensor` representing `shift in `scale @ X + shift`. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - - Returns: - scale. In the case of scaling by a constant, scale is a - floating point `Tensor`. Otherwise, scale is a `LinearOperator`. - - Raises: - ValueError: if all of `tril`, `diag` and `identity_multiplier` are `None`. - """ - identity_multiplier = _as_tensor(identity_multiplier, "identity_multiplier") - diag = _as_tensor(diag, "diag") - tril = _as_tensor(tril, "tril") - perturb_diag = _as_tensor(perturb_diag, "perturb_diag") - perturb_factor = _as_tensor(perturb_factor, "perturb_factor") - - # If possible, use the low rank update to infer the shape of - # the identity matrix, when scale represents a scaled identity matrix - # with a low rank update. - shape_hint = None - if perturb_factor is not None: - shape_hint = distribution_util.dimension_size(perturb_factor, axis=-2) - - if self._is_only_identity_multiplier: - if validate_args: - return control_flow_ops.with_dependencies( - [check_ops.assert_none_equal( - identity_multiplier, - array_ops.zeros([], identity_multiplier.dtype), - ["identity_multiplier should be non-zero."])], - identity_multiplier) - return identity_multiplier - - scale = distribution_util.make_tril_scale( - loc=shift, - scale_tril=tril, - scale_diag=diag, - scale_identity_multiplier=identity_multiplier, - validate_args=validate_args, - assert_positive=False, - shape_hint=shape_hint) - - if perturb_factor is not None: - return linalg.LinearOperatorLowRankUpdate( - scale, - u=perturb_factor, - diag_update=perturb_diag, - is_diag_update_positive=perturb_diag is None, - is_non_singular=True, # Implied by is_positive_definite=True. - is_self_adjoint=True, - is_positive_definite=True, - is_square=True) - - return scale - - @property - def shift(self): - """The `shift` `Tensor` in `Y = scale @ X + shift`.""" - return self._shift - - @property - def scale(self): - """The `scale` `LinearOperator` in `Y = scale @ X + shift`.""" - return self._scale - - def _forward(self, x): - y = x - if self._is_only_identity_multiplier: - y *= self._scale - if self.shift is not None: - return y + self.shift - return y - y, sample_shape = self._shaper.make_batch_of_event_sample_matrices( - y, expand_batch_dim=False) - with ops.control_dependencies(self._maybe_check_scale() if - self.validate_args else []): - y = self.scale.matmul(y) - y = self._shaper.undo_make_batch_of_event_sample_matrices( - y, sample_shape, expand_batch_dim=False) - if self.shift is not None: - y += self.shift - return y - - def _inverse(self, y): - x = y - if self.shift is not None: - x -= self.shift - if self._is_only_identity_multiplier: - return x / self._scale - - x, sample_shape = self._shaper.make_batch_of_event_sample_matrices( - x, expand_batch_dim=False) - # Solve fails if the op is singular so we may safely skip this assertion. - x = self.scale.solve(x) - x = self._shaper.undo_make_batch_of_event_sample_matrices( - x, sample_shape, expand_batch_dim=False) - return x - - def _forward_log_det_jacobian(self, x): - # is_constant_jacobian = True for this bijector, hence the - # `log_det_jacobian` need only be specified for a single input, as this will - # be tiled to match `event_ndims`. - if self._is_only_identity_multiplier: - # We don't pad in this case and instead let the fldj be applied - # via broadcast. - event_size = array_ops.shape(x)[-1] - event_size = math_ops.cast(event_size, dtype=self._scale.dtype) - return math_ops.log(math_ops.abs(self._scale)) * event_size - - return self.scale.log_abs_determinant() - - def _maybe_check_scale(self): - try: - return [self.scale.assert_non_singular()] - except NotImplementedError: - pass - return [] diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/affine_linear_operator.py b/tensorflow/contrib/distributions/python/ops/bijectors/affine_linear_operator.py deleted file mode 100644 index 722d843f7f4..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/affine_linear_operator.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""AffineLinearOperator bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops.shape import _DistributionShape -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.ops.linalg import linear_operator -from tensorflow.python.util import deprecation - - -__all__ = [ - "AffineLinearOperator", -] - - -class AffineLinearOperator(bijector.Bijector): - """Compute `Y = g(X; shift, scale) = scale @ X + shift`. - - `shift` is a numeric `Tensor` and `scale` is a `LinearOperator`. - - If `X` is a scalar then the forward transformation is: `scale * X + shift` - where `*` denotes the scalar product. - - Note: we don't always simply transpose `X` (but write it this way for - brevity). Actually the input `X` undergoes the following transformation - before being premultiplied by `scale`: - - 1. If there are no sample dims, we call `X = tf.expand_dims(X, 0)`, i.e., - `new_sample_shape = [1]`. Otherwise do nothing. - 2. The sample shape is flattened to have one dimension, i.e., - `new_sample_shape = [n]` where `n = tf.reduce_prod(old_sample_shape)`. - 3. The sample dim is cyclically rotated left by 1, i.e., - `new_shape = [B1,...,Bb, k, n]` where `n` is as above, `k` is the - event_shape, and `B1,...,Bb` are the batch shapes for each of `b` batch - dimensions. - - (For more details see `shape.make_batch_of_event_sample_matrices`.) - - The result of the above transformation is that `X` can be regarded as a batch - of matrices where each column is a draw from the distribution. After - premultiplying by `scale`, we take the inverse of this procedure. The input - `Y` also undergoes the same transformation before/after premultiplying by - `inv(scale)`. - - Example Use: - - ```python - linalg = tf.linalg - - x = [1., 2, 3] - - shift = [-1., 0., 1] - diag = [1., 2, 3] - scale = linalg.LinearOperatorDiag(diag) - affine = AffineLinearOperator(shift, scale) - # In this case, `forward` is equivalent to: - # y = scale @ x + shift - y = affine.forward(x) # [0., 4, 10] - - shift = [2., 3, 1] - tril = [[1., 0, 0], - [2, 1, 0], - [3, 2, 1]] - scale = linalg.LinearOperatorLowerTriangular(tril) - affine = AffineLinearOperator(shift, scale) - # In this case, `forward` is equivalent to: - # np.squeeze(np.matmul(tril, np.expand_dims(x, -1)), -1) + shift - y = affine.forward(x) # [3., 7, 11] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - shift=None, - scale=None, - validate_args=False, - name="affine_linear_operator"): - """Instantiates the `AffineLinearOperator` bijector. - - Args: - shift: Floating-point `Tensor`. - scale: Subclass of `LinearOperator`. Represents the (batch) positive - definite matrix `M` in `R^{k x k}`. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - - Raises: - TypeError: if `scale` is not a `LinearOperator`. - TypeError: if `shift.dtype` does not match `scale.dtype`. - ValueError: if not `scale.is_non_singular`. - """ - self._graph_parents = [] - self._name = name - self._validate_args = validate_args - graph_parents = [] - with self._name_scope("init", values=[shift]): - # In the absence of `loc` and `scale`, we'll assume `dtype` is `float32`. - dtype = dtypes.float32 - - if shift is not None: - shift = ops.convert_to_tensor(shift, name="shift") - graph_parents += [shift] - dtype = shift.dtype.base_dtype - self._shift = shift - - if scale is not None: - if (shift is not None and - shift.dtype.base_dtype != scale.dtype.base_dtype): - raise TypeError( - "shift.dtype({}) is incompatible with scale.dtype({}).".format( - shift.dtype, scale.dtype)) - if not isinstance(scale, linear_operator.LinearOperator): - raise TypeError("scale is not an instance of tf.LinearOperator") - if validate_args and not scale.is_non_singular: - raise ValueError("Scale matrix must be non-singular.") - if scale.tensor_rank is not None: - batch_ndims = scale.tensor_rank - 2 - else: - batch_ndims = scale.tensor_rank_tensor() - 2 - graph_parents += [batch_ndims] - if scale.dtype is not None: - dtype = scale.dtype.base_dtype - else: - batch_ndims = 0 # We won't need shape inference when scale is None. - self._scale = scale - self._shaper = _DistributionShape( - batch_ndims=batch_ndims, - event_ndims=1, - validate_args=validate_args) - super(AffineLinearOperator, self).__init__( - forward_min_event_ndims=1, - graph_parents=graph_parents, - is_constant_jacobian=True, - dtype=dtype, - validate_args=validate_args, - name=name) - - @property - def shift(self): - """The `shift` `Tensor` in `Y = scale @ X + shift`.""" - return self._shift - - @property - def scale(self): - """The `scale` `LinearOperator` in `Y = scale @ X + shift`.""" - return self._scale - - def _forward(self, x): - y = x - if self.scale is not None: - y, sample_shape = self._shaper.make_batch_of_event_sample_matrices( - y, expand_batch_dim=False) - with ops.control_dependencies(self._maybe_collect_assertions() if - self.validate_args else []): - y = self.scale.matmul(y) - y = self._shaper.undo_make_batch_of_event_sample_matrices( - y, sample_shape, expand_batch_dim=False) - if self.shift is not None: - y += self.shift - return y - - def _inverse(self, y): - x = y - if self.shift is not None: - x -= self.shift - if self.scale is not None: - x, sample_shape = self._shaper.make_batch_of_event_sample_matrices( - x, expand_batch_dim=False) - # Solve fails if the op is singular so we may safely skip this assertion. - x = self.scale.solve(x) - x = self._shaper.undo_make_batch_of_event_sample_matrices( - x, sample_shape, expand_batch_dim=False) - return x - - def _forward_log_det_jacobian(self, x): - # is_constant_jacobian = True for this bijector, hence the - # `log_det_jacobian` need only be specified for a single input, as this will - # be tiled to match `event_ndims`. - if self.scale is None: - return constant_op.constant(0., dtype=x.dtype.base_dtype) - - with ops.control_dependencies(self._maybe_collect_assertions() if - self.validate_args else []): - return self.scale.log_abs_determinant() - - def _maybe_collect_assertions(self): - try: - return [self.scale.assert_non_singular()] - except NotImplementedError: - pass - return [] diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/affine_scalar.py b/tensorflow/contrib/distributions/python/ops/bijectors/affine_scalar.py deleted file mode 100644 index 460d906231b..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/affine_scalar.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Affine bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - - -__all__ = [ - "AffineScalar", -] - - -class AffineScalar(bijector.Bijector): - """Compute `Y = g(X; shift, scale) = scale * X + shift`. - - Examples: - - ```python - # Y = X - b = AffineScalar() - - # Y = X + shift - b = AffineScalar(shift=[1., 2, 3]) - - # Y = 2 * X + shift - b = AffineScalar( - shift=[1., 2, 3], - scale=2.) - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - shift=None, - scale=None, - validate_args=False, - name="affine_scalar"): - """Instantiates the `AffineScalar` bijector. - - This `Bijector` is initialized with `shift` `Tensor` and `scale` arguments, - giving the forward operation: - - ```none - Y = g(X) = scale * X + shift - ``` - - if `scale` is not specified, then the bijector has the semantics of - `scale = 1.`. Similarly, if `shift` is not specified, then the bijector - has the semantics of `shift = 0.`. - - Args: - shift: Floating-point `Tensor`. If this is set to `None`, no shift is - applied. - scale: Floating-point `Tensor`. If this is set to `None`, no scale is - applied. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - """ - self._graph_parents = [] - self._name = name - self._validate_args = validate_args - - with self._name_scope("init", values=[scale, shift]): - self._shift = shift - self._scale = scale - - if self._shift is not None: - self._shift = ops.convert_to_tensor(shift, name="shift") - - if self._scale is not None: - self._scale = ops.convert_to_tensor(self._scale, name="scale") - if validate_args: - self._scale = control_flow_ops.with_dependencies( - [check_ops.assert_none_equal( - self._scale, - array_ops.zeros([], dtype=self._scale.dtype))], - self._scale) - - super(AffineScalar, self).__init__( - forward_min_event_ndims=0, - is_constant_jacobian=True, - validate_args=validate_args, - name=name) - - @property - def shift(self): - """The `shift` `Tensor` in `Y = scale @ X + shift`.""" - return self._shift - - @property - def scale(self): - """The `scale` `LinearOperator` in `Y = scale @ X + shift`.""" - return self._scale - - def _forward(self, x): - y = array_ops.identity(x) - if self.scale is not None: - y *= self.scale - if self.shift is not None: - y += self.shift - return y - - def _inverse(self, y): - x = array_ops.identity(y) - if self.shift is not None: - x -= self.shift - if self.scale is not None: - x /= self.scale - return x - - def _forward_log_det_jacobian(self, x): - # is_constant_jacobian = True for this bijector, hence the - # `log_det_jacobian` need only be specified for a single input, as this will - # be tiled to match `event_ndims`. - if self.scale is None: - return constant_op.constant(0., dtype=x.dtype.base_dtype) - - return math_ops.log(math_ops.abs(self.scale)) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/batch_normalization.py b/tensorflow/contrib/distributions/python/ops/bijectors/batch_normalization.py deleted file mode 100644 index f891e418427..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/batch_normalization.py +++ /dev/null @@ -1,283 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Batch Norm bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import ops -from tensorflow.python.layers import normalization -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - -__all__ = [ - "BatchNormalization", -] - - -@deprecation.deprecated( - "2018-10-01", "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _undo_batch_normalization(x, - mean, - variance, - offset, - scale, - variance_epsilon, - name=None): - r"""Inverse of tf.nn.batch_normalization. - - Args: - x: Input `Tensor` of arbitrary dimensionality. - mean: A mean `Tensor`. - variance: A variance `Tensor`. - offset: An offset `Tensor`, often denoted `beta` in equations, or None. If - present, will be added to the normalized tensor. - scale: A scale `Tensor`, often denoted `gamma` in equations, or `None`. If - present, the scale is applied to the normalized tensor. - variance_epsilon: A small `float` added to the minibatch `variance` to - prevent dividing by zero. - name: A name for this operation (optional). - - Returns: - batch_unnormalized: The de-normalized, de-scaled, de-offset `Tensor`. - """ - with ops.name_scope(name, "undo_batchnorm", - [x, mean, variance, scale, offset]): - # inv = math_ops.rsqrt(variance + variance_epsilon) - # if scale is not None: - # inv *= scale - # return x * inv + ( - # offset - mean * inv if offset is not None else -mean * inv) - rescale = math_ops.sqrt(variance + variance_epsilon) - if scale is not None: - rescale /= scale - batch_unnormalized = x * rescale + ( - mean - offset * rescale if offset is not None else mean) - return batch_unnormalized - - -class BatchNormalization(bijector.Bijector): - """Compute `Y = g(X) s.t. - - X = g^-1(Y) = (Y - mean(Y)) / std(Y)`. - - Applies Batch Normalization [(Ioffe and Szegedy, 2015)][1] to samples from a - data distribution. This can be used to stabilize training of normalizing - flows ([Papamakarios et al., 2016][3]; [Dinh et al., 2017][2]) - - When training Deep Neural Networks (DNNs), it is common practice to - normalize or whiten features by shifting them to have zero mean and - scaling them to have unit variance. - - The `inverse()` method of the `BatchNormalization` bijector, which is used in - the log-likelihood computation of data samples, implements the normalization - procedure (shift-and-scale) using the mean and standard deviation of the - current minibatch. - - Conversely, the `forward()` method of the bijector de-normalizes samples (e.g. - `X*std(Y) + mean(Y)` with the running-average mean and standard deviation - computed at training-time. De-normalization is useful for sampling. - - ```python - - dist = tfd.TransformedDistribution( - distribution=tfd.Normal()), - bijector=tfb.BatchNorm()) - - y = tfd.MultivariateNormalDiag(loc=1., scale=2.).sample(100) # ~ N(1, 2) - x = dist.bijector.inverse(y) # ~ N(0, 1) - y = dist.sample() # ~ N(1, 2) - ``` - - During training time, `BatchNorm.inverse` and `BatchNorm.forward` are not - guaranteed to be inverses of each other because `inverse(y)` uses statistics - of the current minibatch, while `forward(x)` uses running-average statistics - accumulated from training. In other words, - `BatchNorm.inverse(BatchNorm.forward(...))` and - `BatchNorm.forward(BatchNorm.inverse(...))` will be identical when - `training=False` but may be different when `training=True`. - - #### References - - [1]: Sergey Ioffe and Christian Szegedy. Batch Normalization: Accelerating - Deep Network Training by Reducing Internal Covariate Shift. In - _International Conference on Machine Learning_, 2015. - https://arxiv.org/abs/1502.03167 - - [2]: Laurent Dinh, Jascha Sohl-Dickstein, and Samy Bengio. Density Estimation - using Real NVP. In _International Conference on Learning - Representations_, 2017. https://arxiv.org/abs/1605.08803 - - [3]: George Papamakarios, Theo Pavlakou, and Iain Murray. Masked - Autoregressive Flow for Density Estimation. In _Neural Information - Processing Systems_, 2017. https://arxiv.org/abs/1705.07057 - """ - - @deprecation.deprecated( - "2018-10-01", "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - batchnorm_layer=None, - training=True, - validate_args=False, - name="batch_normalization"): - """Instantiates the `BatchNorm` bijector. - - Args: - batchnorm_layer: `tf.compat.v1.layers.BatchNormalization` layer object. If - `None`, defaults to - `tf.compat.v1.layers.BatchNormalization(gamma_constraint=nn_ops.relu(x) - + 1e-6)`. This ensures positivity of the scale variable. - training: If True, updates running-average statistics during call to - `inverse()`. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - - Raises: - ValueError: If bn_layer is not an instance of - `tf.compat.v1.layers.BatchNormalization`, or if it is specified with - `renorm=True` - or a virtual batch size. - """ - # Scale must be positive. - g_constraint = lambda x: nn.relu(x) + 1e-6 - self.batchnorm = batchnorm_layer or normalization.BatchNormalization( - gamma_constraint=g_constraint) - self._validate_bn_layer(self.batchnorm) - self._training = training - if isinstance(self.batchnorm.axis, int): - forward_min_event_ndims = 1 - else: - forward_min_event_ndims = len(self.batchnorm.axis) - super(BatchNormalization, self).__init__( - forward_min_event_ndims=forward_min_event_ndims, - validate_args=validate_args, - name=name) - - def _validate_bn_layer(self, layer): - """Check for valid BatchNormalization layer. - - Args: - layer: Instance of `tf.compat.v1.layers.BatchNormalization`. - - Raises: - ValueError: If batchnorm_layer argument is not an instance of - `tf.compat.v1.layers.BatchNormalization`, or if - `batchnorm_layer.renorm=True` or - if `batchnorm_layer.virtual_batch_size` is specified. - """ - if not isinstance(layer, normalization.BatchNormalization): - raise ValueError( - "batchnorm_layer must be an instance of BatchNormalization layer.") - if layer.renorm: - raise ValueError("BatchNorm Bijector does not support renormalization.") - if layer.virtual_batch_size: - raise ValueError( - "BatchNorm Bijector does not support virtual batch sizes.") - - def _get_broadcast_fn(self, x): - # Compute shape to broadcast scale/shift parameters to. - if not x.shape.is_fully_defined(): - raise ValueError("Input must have shape known at graph construction.") - input_shape = np.int32(x.shape.as_list()) - - ndims = len(input_shape) - reduction_axes = [i for i in range(ndims) if i not in self.batchnorm.axis] - # Broadcasting only necessary for single-axis batch norm where the axis is - # not the last dimension - broadcast_shape = [1] * ndims - broadcast_shape[self.batchnorm.axis[0]] = ( - input_shape[self.batchnorm.axis[0]]) - - def _broadcast(v): - if (v is not None and len(v.get_shape()) != ndims and - reduction_axes != list(range(ndims - 1))): - return array_ops.reshape(v, broadcast_shape) - return v - - return _broadcast - - def _normalize(self, y): - return self.batchnorm.apply(y, training=self._training) - - def _de_normalize(self, x): - # Uses the saved statistics. - if not self.batchnorm.built: - input_shape = x.get_shape() - self.batchnorm.build(input_shape) - broadcast_fn = self._get_broadcast_fn(x) - mean = broadcast_fn(self.batchnorm.moving_mean) - variance = broadcast_fn(self.batchnorm.moving_variance) - beta = broadcast_fn(self.batchnorm.beta) if self.batchnorm.center else None - gamma = broadcast_fn(self.batchnorm.gamma) if self.batchnorm.scale else None - return _undo_batch_normalization(x, mean, variance, beta, gamma, - self.batchnorm.epsilon) - - def _forward(self, x): - return self._de_normalize(x) - - def _inverse(self, y): - return self._normalize(y) - - def _forward_log_det_jacobian(self, x): - # Uses saved statistics to compute volume distortion. - return -self._inverse_log_det_jacobian(x, use_saved_statistics=True) - - def _inverse_log_det_jacobian(self, y, use_saved_statistics=False): - if not y.shape.is_fully_defined(): - raise ValueError("Input must have shape known at graph construction.") - input_shape = np.int32(y.shape.as_list()) - - if not self.batchnorm.built: - # Create variables. - self.batchnorm.build(input_shape) - - event_dims = self.batchnorm.axis - reduction_axes = [i for i in range(len(input_shape)) if i not in event_dims] - - if use_saved_statistics or not self._training: - log_variance = math_ops.log(self.batchnorm.moving_variance + - self.batchnorm.epsilon) - else: - # At training-time, ildj is computed from the mean and log-variance across - # the current minibatch. - _, v = nn.moments(y, axes=reduction_axes, keepdims=True) - log_variance = math_ops.log(v + self.batchnorm.epsilon) - - # `gamma` and `log Var(y)` reductions over event_dims. - # Log(total change in area from gamma term). - log_total_gamma = math_ops.reduce_sum(math_ops.log(self.batchnorm.gamma)) - - # Log(total change in area from log-variance term). - log_total_variance = math_ops.reduce_sum(log_variance) - # The ildj is scalar, as it does not depend on the values of x and are - # constant across minibatch elements. - return log_total_gamma - 0.5 * log_total_variance diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/chain.py b/tensorflow/contrib/distributions/python/ops/bijectors/chain.py deleted file mode 100644 index 3c61b7eb232..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/chain.py +++ /dev/null @@ -1,324 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Chain bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - - -__all__ = [ - "Chain", -] - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _use_static_shape(input_tensor, ndims): - return input_tensor.shape.is_fully_defined() and isinstance(ndims, int) - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _compute_min_event_ndims(bijector_list, compute_forward=True): - """Computes the min_event_ndims associated with the give list of bijectors. - - Given a list `bijector_list` of bijectors, compute the min_event_ndims that is - associated with the composition of bijectors in that list. - - min_event_ndims is the # of right most dimensions for which the bijector has - done necessary computation on (i.e. the non-broadcastable part of the - computation). - - We can derive the min_event_ndims for a chain of bijectors as follows: - - In the case where there are no rank changing bijectors, this will simply be - `max(b.forward_min_event_ndims for b in bijector_list)`. This is because the - bijector with the most forward_min_event_ndims requires the most dimensions, - and hence the chain also requires operating on those dimensions. - - However in the case of rank changing, more care is needed in determining the - exact amount of dimensions. Padding dimensions causes subsequent bijectors to - operate on the padded dimensions, and Removing dimensions causes bijectors to - operate more left. - - Args: - bijector_list: List of bijectors to be composed by chain. - compute_forward: Boolean. If True, computes the min_event_ndims associated - with a forward call to Chain, and otherwise computes the min_event_ndims - associated with an inverse call to Chain. The latter is the same as the - min_event_ndims associated with a forward call to Invert(Chain(....)). - - Returns: - min_event_ndims - """ - min_event_ndims = 0 - # This is a mouthful, but what this encapsulates is that if not for rank - # changing bijectors, we'd only need to compute the largest of the min - # required ndims. Hence "max_min". Due to rank changing bijectors, we need to - # account for synthetic rank growth / synthetic rank decrease from a rank - # changing bijector. - rank_changed_adjusted_max_min_event_ndims = 0 - - if compute_forward: - bijector_list = reversed(bijector_list) - - for b in bijector_list: - if compute_forward: - current_min_event_ndims = b.forward_min_event_ndims - current_inverse_min_event_ndims = b.inverse_min_event_ndims - else: - current_min_event_ndims = b.inverse_min_event_ndims - current_inverse_min_event_ndims = b.forward_min_event_ndims - - # New dimensions were touched. - if rank_changed_adjusted_max_min_event_ndims < current_min_event_ndims: - min_event_ndims += ( - current_min_event_ndims - rank_changed_adjusted_max_min_event_ndims) - rank_changed_adjusted_max_min_event_ndims = max( - current_min_event_ndims, rank_changed_adjusted_max_min_event_ndims) - - # If the number of dimensions has increased via forward, then - # inverse_min_event_ndims > forward_min_event_ndims, and hence the - # dimensions we computed on, have moved left (so we have operated - # on additional dimensions). - # Conversely, if the number of dimensions has decreased via forward, - # then we have inverse_min_event_ndims < forward_min_event_ndims, - # and so we will have operated on fewer right most dimensions. - - number_of_changed_dimensions = ( - current_min_event_ndims - current_inverse_min_event_ndims) - rank_changed_adjusted_max_min_event_ndims -= number_of_changed_dimensions - return min_event_ndims - - -class Chain(bijector.Bijector): - """Bijector which applies a sequence of bijectors. - - Example Use: - - ```python - chain = Chain([Exp(), Softplus()], name="one_plus_exp") - ``` - - Results in: - - * Forward: - - ```python - exp = Exp() - softplus = Softplus() - Chain([exp, softplus]).forward(x) - = exp.forward(softplus.forward(x)) - = tf.exp(tf.math.log(1. + tf.exp(x))) - = 1. + tf.exp(x) - ``` - - * Inverse: - - ```python - exp = Exp() - softplus = Softplus() - Chain([exp, softplus]).inverse(y) - = softplus.inverse(exp.inverse(y)) - = tf.math.log(tf.exp(tf.math.log(y)) - 1.) - = tf.math.log(y - 1.) - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, bijectors=None, validate_args=False, name=None): - """Instantiates `Chain` bijector. - - Args: - bijectors: Python `list` of bijector instances. An empty list makes this - bijector equivalent to the `Identity` bijector. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str`, name given to ops managed by this object. Default: - E.g., `Chain([Exp(), Softplus()]).name == "chain_of_exp_of_softplus"`. - - Raises: - ValueError: if bijectors have different dtypes. - """ - if bijectors is None: - bijectors = () - self._bijectors = bijectors - - for a_bijector in bijectors: - if not a_bijector._is_injective: # pylint: disable=protected-access - raise NotImplementedError( - "Invert is not implemented for non-injective bijector ({})".format( - a_bijector.name)) - - dtype = list(set([b.dtype for b in bijectors])) - if len(dtype) > 2: - raise ValueError("incompatible dtypes: %s" % dtype) - elif len(dtype) == 2: - dtype = dtype[1] if dtype[0] is None else dtype[0] - elif len(dtype) == 1: - dtype = dtype[0] - else: - dtype = None - - inverse_min_event_ndims = _compute_min_event_ndims( - bijectors, compute_forward=False) - forward_min_event_ndims = _compute_min_event_ndims( - bijectors, compute_forward=True) - - super(Chain, self).__init__( - graph_parents=list(itertools.chain.from_iterable( - b.graph_parents for b in bijectors)), - forward_min_event_ndims=forward_min_event_ndims, - inverse_min_event_ndims=inverse_min_event_ndims, - is_constant_jacobian=all(b.is_constant_jacobian for b in bijectors), - validate_args=validate_args, - dtype=dtype, - name=name or ("identity" if not bijectors else - "_of_".join(["chain"] + [b.name for b in bijectors]))) - - @property - def bijectors(self): - return self._bijectors - - def _shape_helper(self, func_name, input_shape, reverse): - new_shape = input_shape - for b in reversed(self.bijectors) if reverse else self.bijectors: - func = getattr(b, func_name, None) - if func is None: - raise ValueError("unable to call %s on bijector %s (%s)" % - (func_name, b.name, func)) - new_shape = func(new_shape) - return new_shape - - def _forward_event_shape(self, input_shape): - return self._shape_helper("forward_event_shape", input_shape, - reverse=True) - - def _forward_event_shape_tensor(self, input_shape): - return self._shape_helper( - "forward_event_shape_tensor", input_shape, reverse=True) - - def _inverse_event_shape(self, output_shape): - return self._shape_helper("inverse_event_shape", output_shape, - reverse=False) - - def _inverse_event_shape_tensor(self, output_shape): - return self._shape_helper("inverse_event_shape_tensor", output_shape, - reverse=False) - - def _inverse(self, y, **kwargs): - for b in self.bijectors: - y = b.inverse(y, **kwargs.get(b.name, {})) - return y - - def _inverse_log_det_jacobian(self, y, **kwargs): - y = ops.convert_to_tensor(y, name="y") - ildj = math_ops.cast(0., dtype=y.dtype.base_dtype) - - if not self.bijectors: - return ildj - - event_ndims = self._maybe_get_static_event_ndims( - self.inverse_min_event_ndims) - - if _use_static_shape(y, event_ndims): - event_shape = y.shape[y.shape.ndims - event_ndims:] - else: - event_shape = array_ops.shape(y)[array_ops.rank(y) - event_ndims:] - - for b in self.bijectors: - ildj += b.inverse_log_det_jacobian( - y, event_ndims=event_ndims, **kwargs.get(b.name, {})) - - if _use_static_shape(y, event_ndims): - event_shape = b.inverse_event_shape(event_shape) - event_ndims = self._maybe_get_static_event_ndims( - event_shape.ndims) - else: - event_shape = b.inverse_event_shape_tensor(event_shape) - event_ndims = array_ops.size(event_shape) - event_ndims_ = self._maybe_get_static_event_ndims(event_ndims) - if event_ndims_ is not None: - event_ndims = event_ndims_ - - y = b.inverse(y, **kwargs.get(b.name, {})) - return ildj - - def _forward(self, x, **kwargs): - for b in reversed(self.bijectors): - x = b.forward(x, **kwargs.get(b.name, {})) - return x - - def _forward_log_det_jacobian(self, x, **kwargs): - x = ops.convert_to_tensor(x, name="x") - - fldj = math_ops.cast(0., dtype=x.dtype.base_dtype) - - if not self.bijectors: - return fldj - - event_ndims = self._maybe_get_static_event_ndims( - self.forward_min_event_ndims) - - if _use_static_shape(x, event_ndims): - event_shape = x.shape[x.shape.ndims - event_ndims:] - else: - event_shape = array_ops.shape(x)[array_ops.rank(x) - event_ndims:] - - for b in reversed(self.bijectors): - fldj += b.forward_log_det_jacobian( - x, event_ndims=event_ndims, **kwargs.get(b.name, {})) - if _use_static_shape(x, event_ndims): - event_shape = b.forward_event_shape(event_shape) - event_ndims = self._maybe_get_static_event_ndims(event_shape.ndims) - else: - event_shape = b.forward_event_shape_tensor(event_shape) - event_ndims = array_ops.size(event_shape) - event_ndims_ = self._maybe_get_static_event_ndims(event_ndims) - if event_ndims_ is not None: - event_ndims = event_ndims_ - - x = b.forward(x, **kwargs.get(b.name, {})) - - return fldj diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py b/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py deleted file mode 100644 index 2358ef5976b..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py +++ /dev/null @@ -1,228 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""CholeskyOuterProduct bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - - -__all__ = [ - "CholeskyOuterProduct", -] - - -class CholeskyOuterProduct(bijector.Bijector): - """Compute `g(X) = X @ X.T`; X is lower-triangular, positive-diagonal matrix. - - Note: the upper-triangular part of X is ignored (whether or not its zero). - - The surjectivity of g as a map from the set of n x n positive-diagonal - lower-triangular matrices to the set of SPD matrices follows immediately from - executing the Cholesky factorization algorithm on an SPD matrix A to produce a - positive-diagonal lower-triangular matrix L such that `A = L @ L.T`. - - To prove the injectivity of g, suppose that L_1 and L_2 are lower-triangular - with positive diagonals and satisfy `A = L_1 @ L_1.T = L_2 @ L_2.T`. Then - `inv(L_1) @ A @ inv(L_1).T = [inv(L_1) @ L_2] @ [inv(L_1) @ L_2].T = I`. - Setting `L_3 := inv(L_1) @ L_2`, that L_3 is a positive-diagonal - lower-triangular matrix follows from `inv(L_1)` being positive-diagonal - lower-triangular (which follows from the diagonal of a triangular matrix being - its spectrum), and that the product of two positive-diagonal lower-triangular - matrices is another positive-diagonal lower-triangular matrix. - - A simple inductive argument (proceeding one column of L_3 at a time) shows - that, if `I = L_3 @ L_3.T`, with L_3 being lower-triangular with positive- - diagonal, then `L_3 = I`. Thus, `L_1 = L_2`, proving injectivity of g. - - #### Examples - - ```python - bijector.CholeskyOuterProduct().forward(x=[[1., 0], [2, 1]]) - # Result: [[1., 2], [2, 5]], i.e., x @ x.T - - bijector.CholeskyOuterProduct().inverse(y=[[1., 2], [2, 5]]) - # Result: [[1., 0], [2, 1]], i.e., cholesky(y). - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, validate_args=False, name="cholesky_outer_product"): - """Instantiates the `CholeskyOuterProduct` bijector. - - Args: - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - """ - self._graph_parents = [] - self._name = name - super(CholeskyOuterProduct, self).__init__( - forward_min_event_ndims=2, - validate_args=validate_args, - name=name) - - def _forward(self, x): - if self.validate_args: - is_matrix = check_ops.assert_rank_at_least(x, 2) - shape = array_ops.shape(x) - is_square = check_ops.assert_equal(shape[-2], shape[-1]) - x = control_flow_ops.with_dependencies([is_matrix, is_square], x) - # For safety, explicitly zero-out the upper triangular part. - x = array_ops.matrix_band_part(x, -1, 0) - return math_ops.matmul(x, x, adjoint_b=True) - - def _inverse(self, y): - return linalg_ops.cholesky(y) - - def _forward_log_det_jacobian(self, x): - # Let Y be a symmetric, positive definite matrix and write: - # Y = X X.T - # where X is lower-triangular. - # - # Observe that, - # dY[i,j]/dX[a,b] - # = d/dX[a,b] { X[i,:] X[j,:] } - # = sum_{d=1}^p { I[i=a] I[d=b] X[j,d] + I[j=a] I[d=b] X[i,d] } - # - # To compute the Jacobian dX/dY we must represent X,Y as vectors. Since Y is - # symmetric and X is lower-triangular, we need vectors of dimension: - # d = p (p + 1) / 2 - # where X, Y are p x p matrices, p > 0. We use a row-major mapping, i.e., - # k = { i (i + 1) / 2 + j i>=j - # { undef ij thus i,j!=a. - # - # Since the Jacobian is lower-triangular, we need only compute the product - # of diagonal elements: - # d vec[Y] / d vec[X] @[k(i,j), k(i,j)] - # = X[j,j] + I[i=j] X[i,j] - # = 2 X[j,j]. - # Since there is a 2 X[j,j] term for every lower-triangular element of X we - # conclude: - # |Jac(d vec[Y]/d vec[X])| = 2^p prod_{j=0}^{p-1} X[j,j]^{p-j}. - diag = array_ops.matrix_diag_part(x) - - # We now ensure diag is columnar. Eg, if `diag = [1, 2, 3]` then the output - # is `[[1], [2], [3]]` and if `diag = [[1, 2, 3], [4, 5, 6]]` then the - # output is unchanged. - diag = self._make_columnar(diag) - - if self.validate_args: - is_matrix = check_ops.assert_rank_at_least( - x, 2, message="Input must be a (batch of) matrix.") - shape = array_ops.shape(x) - is_square = check_ops.assert_equal( - shape[-2], shape[-1], - message="Input must be a (batch of) square matrix.") - # Assuming lower-triangular means we only need check diag>0. - is_positive_definite = check_ops.assert_positive( - diag, message="Input must be positive definite.") - x = control_flow_ops.with_dependencies( - [is_matrix, is_square, is_positive_definite], x) - - # Create a vector equal to: [p, p-1, ..., 2, 1]. - if x.get_shape().ndims is None or x.get_shape().dims[-1].value is None: - p_int = array_ops.shape(x)[-1] - p_float = math_ops.cast(p_int, dtype=x.dtype) - else: - p_int = x.get_shape().dims[-1].value - p_float = np.array(p_int, dtype=x.dtype.as_numpy_dtype) - exponents = math_ops.linspace(p_float, 1., p_int) - - sum_weighted_log_diag = array_ops.squeeze( - math_ops.matmul(math_ops.log(diag), - exponents[..., array_ops.newaxis]), - axis=-1) - fldj = p_float * np.log(2.) + sum_weighted_log_diag - - # We finally need to undo adding an extra column in non-scalar cases - # where there is a single matrix as input. - if x.get_shape().ndims is not None: - if x.get_shape().ndims == 2: - fldj = array_ops.squeeze(fldj, axis=-1) - return fldj - - shape = array_ops.shape(fldj) - maybe_squeeze_shape = array_ops.concat([ - shape[:-1], - distribution_util.pick_vector( - math_ops.equal(array_ops.rank(x), 2), - np.array([], dtype=np.int32), shape[-1:])], 0) - return array_ops.reshape(fldj, maybe_squeeze_shape) - - def _make_columnar(self, x): - """Ensures non-scalar input has at least one column. - - Example: - If `x = [1, 2, 3]` then the output is `[[1], [2], [3]]`. - - If `x = [[1, 2, 3], [4, 5, 6]]` then the output is unchanged. - - If `x = 1` then the output is unchanged. - - Args: - x: `Tensor`. - - Returns: - columnar_x: `Tensor` with at least two dimensions. - """ - if x.get_shape().ndims is not None: - if x.get_shape().ndims == 1: - x = x[array_ops.newaxis, :] - return x - shape = array_ops.shape(x) - maybe_expanded_shape = array_ops.concat([ - shape[:-1], - distribution_util.pick_vector( - math_ops.equal(array_ops.rank(x), 1), - [1], np.array([], dtype=np.int32)), - shape[-1:], - ], 0) - return array_ops.reshape(x, maybe_expanded_shape) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/conditional_bijector.py b/tensorflow/contrib/distributions/python/ops/bijectors/conditional_bijector.py deleted file mode 100644 index e9e994f839a..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/conditional_bijector.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ConditionalBijector base.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.ops.distributions import util as distribution_util - - -__all__ = ["ConditionalBijector"] - - -class ConditionalBijector(bijector.Bijector): - """Conditional Bijector is a Bijector that allows intrinsic conditioning.""" - - @distribution_util.AppendDocstring(kwargs_dict={ - "**condition_kwargs": - "Named arguments forwarded to subclass implementation."}) - def forward(self, x, name="forward", **condition_kwargs): - return self._call_forward(x, name, **condition_kwargs) - - @distribution_util.AppendDocstring(kwargs_dict={ - "**condition_kwargs": - "Named arguments forwarded to subclass implementation."}) - def inverse(self, y, name="inverse", **condition_kwargs): - return self._call_inverse(y, name, **condition_kwargs) - - @distribution_util.AppendDocstring(kwargs_dict={ - "**condition_kwargs": - "Named arguments forwarded to subclass implementation."}) - def inverse_log_det_jacobian( - self, y, event_ndims, name="inverse_log_det_jacobian", - **condition_kwargs): - return self._call_inverse_log_det_jacobian( - y, event_ndims, name, **condition_kwargs) - - @distribution_util.AppendDocstring(kwargs_dict={ - "**condition_kwargs": - "Named arguments forwarded to subclass implementation."}) - def forward_log_det_jacobian( - self, x, event_ndims, name="forward_log_det_jacobian", - **condition_kwargs): - return self._call_forward_log_det_jacobian( - x, event_ndims, name, **condition_kwargs) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/exp.py b/tensorflow/contrib/distributions/python/ops/bijectors/exp.py deleted file mode 100644 index 07627e1e45e..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/exp.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Exp bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops.bijectors import power_transform -from tensorflow.python.util import deprecation - - -__all__ = [ - "Exp", -] - - -class Exp(power_transform.PowerTransform): - """Compute `Y = g(X) = exp(X)`. - - Example Use: - - ```python - # Create the Y=g(X)=exp(X) transform which works only on Tensors with 1 - # batch ndim 2. - exp = Exp() - x = [[[1., 2], - [3, 4]], - [[5, 6], - [7, 8]]] - exp(x) == exp.forward(x) - log(x) == exp.inverse(x) - ``` - - Note: the exp(.) is applied element-wise but the Jacobian is a reduction - over the event space. - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - validate_args=False, - name="exp"): - """Instantiates the `Exp` bijector. - - Args: - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - """ - # forward_min_event_ndims = 0. - # No forward_min_event_ndims specified as this is done in PowerTransform. - super(Exp, self).__init__( - validate_args=validate_args, - name=name) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/fill_triangular.py b/tensorflow/contrib/distributions/python/ops/bijectors/fill_triangular.py deleted file mode 100644 index daab24e4333..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/fill_triangular.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""FillTriangular bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.ops.distributions import util as dist_util -from tensorflow.python.util import deprecation - - -__all__ = [ - "FillTriangular", -] - - -class FillTriangular(bijector.Bijector): - """Transforms vectors to triangular. - - Triangular matrix elements are filled in a clockwise spiral. - - Given input with shape `batch_shape + [d]`, produces output with - shape `batch_shape + [n, n]`, where - `n = (-1 + sqrt(1 + 8 * d))/2`. - This follows by solving the quadratic equation - `d = 1 + 2 + ... + n = n * (n + 1)/2`. - - #### Example - - ```python - b = tfb.FillTriangular(upper=False) - b.forward([1, 2, 3, 4, 5, 6]) - # ==> [[4, 0, 0], - # [6, 5, 0], - # [3, 2, 1]] - - b = tfb.FillTriangular(upper=True) - b.forward([1, 2, 3, 4, 5, 6]) - # ==> [[1, 2, 3], - # [0, 5, 6], - # [0, 0, 4]] - - ``` - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - upper=False, - validate_args=False, - name="fill_triangular"): - """Instantiates the `FillTriangular` bijector. - - Args: - upper: Python `bool` representing whether output matrix should be upper - triangular (`True`) or lower triangular (`False`, default). - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - """ - self._upper = upper - super(FillTriangular, self).__init__( - forward_min_event_ndims=1, - inverse_min_event_ndims=2, - validate_args=validate_args, - name=name) - - def _forward(self, x): - return dist_util.fill_triangular(x, upper=self._upper) - - def _inverse(self, y): - return dist_util.fill_triangular_inverse(y, upper=self._upper) - - def _forward_log_det_jacobian(self, x): - return array_ops.zeros_like(x[..., 0]) - - def _inverse_log_det_jacobian(self, y): - return array_ops.zeros_like(y[..., 0, 0]) - - def _forward_event_shape(self, input_shape): - batch_shape, d = (input_shape[:-1], - tensor_shape.dimension_value(input_shape[-1])) - if d is None: - n = None - else: - n = vector_size_to_square_matrix_size(d, self.validate_args) - return batch_shape.concatenate([n, n]) - - def _inverse_event_shape(self, output_shape): - batch_shape, n1, n2 = (output_shape[:-2], - tensor_shape.dimension_value(output_shape[-2]), - tensor_shape.dimension_value(output_shape[-1])) - if n1 is None or n2 is None: - m = None - elif n1 != n2: - raise ValueError("Matrix must be square. (saw [{}, {}])".format(n1, n2)) - else: - m = n1 * (n1 + 1) / 2 - return batch_shape.concatenate([m]) - - def _forward_event_shape_tensor(self, input_shape_tensor): - batch_shape, d = input_shape_tensor[:-1], input_shape_tensor[-1] - n = vector_size_to_square_matrix_size(d, self.validate_args) - return array_ops.concat([batch_shape, [n, n]], axis=0) - - def _inverse_event_shape_tensor(self, output_shape_tensor): - batch_shape, n = output_shape_tensor[:-2], output_shape_tensor[-1] - if self.validate_args: - is_square_matrix = check_ops.assert_equal( - n, output_shape_tensor[-2], message="Matrix must be square.") - with ops.control_dependencies([is_square_matrix]): - n = array_ops.identity(n) - d = math_ops.cast(n * (n + 1) / 2, output_shape_tensor.dtype) - return array_ops.concat([batch_shape, [d]], axis=0) - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def vector_size_to_square_matrix_size(d, validate_args, name=None): - """Convert a vector size to a matrix size.""" - if isinstance(d, (float, int, np.generic, np.ndarray)): - n = (-1 + np.sqrt(1 + 8 * d)) / 2. - if float(int(n)) != n: - raise ValueError("Vector length is not a triangular number.") - return int(n) - else: - with ops.name_scope(name, "vector_size_to_square_matrix_size", [d]) as name: - n = (-1. + math_ops.sqrt(1 + 8. * math_ops.cast(d, dtypes.float32))) / 2. - if validate_args: - with ops.control_dependencies([ - check_ops.assert_equal( - math_ops.cast(math_ops.cast(n, dtypes.int32), dtypes.float32), - n, - message="Vector length is not a triangular number") - ]): - n = array_ops.identity(n) - return math_ops.cast(n, d.dtype) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py deleted file mode 100644 index 71e562a927a..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Gumbel bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - -__all__ = [ - "Gumbel", -] - - -class Gumbel(bijector.Bijector): - """Compute `Y = g(X) = exp(-exp(-(X - loc) / scale))`. - - This bijector maps inputs from `[-inf, inf]` to [0, 1]`. The inverse of the - bijector applied to a uniform random variable `X ~ U(0, 1) gives back a - random variable with the - [Gumbel distribution](https://en.wikipedia.org/wiki/Gumbel_distribution): - - ```none - Y ~ Gumbel(loc, scale) - pdf(y; loc, scale) = exp( - -( (y - loc) / scale + exp(- (y - loc) / scale) ) ) / scale - ``` - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc=0., - scale=1., - validate_args=False, - name="gumbel"): - """Instantiates the `Gumbel` bijector. - - Args: - loc: Float-like `Tensor` that is the same dtype and is - broadcastable with `scale`. - This is `loc` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`. - scale: Positive Float-like `Tensor` that is the same dtype and is - broadcastable with `loc`. - This is `scale` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - """ - self._graph_parents = [] - self._name = name - self._validate_args = validate_args - with self._name_scope("init", values=[loc, scale]): - self._loc = ops.convert_to_tensor(loc, name="loc") - self._scale = ops.convert_to_tensor(scale, name="scale") - check_ops.assert_same_float_dtype([self._loc, self._scale]) - if validate_args: - self._scale = control_flow_ops.with_dependencies([ - check_ops.assert_positive( - self._scale, message="Argument scale was not positive") - ], self._scale) - - super(Gumbel, self).__init__( - validate_args=validate_args, - forward_min_event_ndims=0, - name=name) - - @property - def loc(self): - """The `loc` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.""" - return self._loc - - @property - def scale(self): - """This is `scale` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.""" - return self._scale - - def _forward(self, x): - z = (x - self.loc) / self.scale - return math_ops.exp(-math_ops.exp(-z)) - - def _inverse(self, y): - y = self._maybe_assert_valid_y(y) - return self.loc - self.scale * math_ops.log(-math_ops.log(y)) - - def _inverse_log_det_jacobian(self, y): - y = self._maybe_assert_valid_y(y) - return math_ops.log(self.scale / (-math_ops.log(y) * y)) - - def _forward_log_det_jacobian(self, x): - z = (x - self.loc) / self.scale - return -z - math_ops.exp(-z) - math_ops.log(self.scale) - - def _maybe_assert_valid_y(self, y): - if not self.validate_args: - return y - is_positive = check_ops.assert_non_negative( - y, message="Inverse transformation input must be greater than 0.") - less_than_one = check_ops.assert_less_equal( - y, - constant_op.constant(1., y.dtype), - message="Inverse transformation input must be less than or equal to 1.") - return control_flow_ops.with_dependencies([is_positive, less_than_one], y) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/inline.py b/tensorflow/contrib/distributions/python/ops/bijectors/inline.py deleted file mode 100644 index 1f8ffc554a5..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/inline.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Inline bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - - -__all__ = [ - "Inline", -] - - -class Inline(bijector.Bijector): - """Bijector constructed from custom callables. - - Example Use: - - ```python - exp = Inline( - forward_fn=tf.exp, - inverse_fn=tf.math.log, - inverse_log_det_jacobian_fn=( - lambda y: -tf.reduce_sum(tf.math.log(y), axis=-1)), - name="exp") - ``` - - The above example is equivalent to the `Bijector` `Exp()`. - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - forward_fn=None, - inverse_fn=None, - inverse_log_det_jacobian_fn=None, - forward_log_det_jacobian_fn=None, - forward_event_shape_fn=None, - forward_event_shape_tensor_fn=None, - inverse_event_shape_fn=None, - inverse_event_shape_tensor_fn=None, - is_constant_jacobian=False, - validate_args=False, - forward_min_event_ndims=None, - inverse_min_event_ndims=None, - name="inline"): - """Creates a `Bijector` from callables. - - Args: - forward_fn: Python callable implementing the forward transformation. - inverse_fn: Python callable implementing the inverse transformation. - inverse_log_det_jacobian_fn: Python callable implementing the - log o det o jacobian of the inverse transformation. - forward_log_det_jacobian_fn: Python callable implementing the - log o det o jacobian of the forward transformation. - forward_event_shape_fn: Python callable implementing non-identical - static event shape changes. Default: shape is assumed unchanged. - forward_event_shape_tensor_fn: Python callable implementing non-identical - event shape changes. Default: shape is assumed unchanged. - inverse_event_shape_fn: Python callable implementing non-identical - static event shape changes. Default: shape is assumed unchanged. - inverse_event_shape_tensor_fn: Python callable implementing non-identical - event shape changes. Default: shape is assumed unchanged. - is_constant_jacobian: Python `bool` indicating that the Jacobian is - constant for all input arguments. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - forward_min_event_ndims: Python `int` indicating the minimal - dimensionality this bijector acts on. - inverse_min_event_ndims: Python `int` indicating the minimal - dimensionality this bijector acts on. - name: Python `str`, name given to ops managed by this object. - """ - super(Inline, self).__init__( - forward_min_event_ndims=forward_min_event_ndims, - inverse_min_event_ndims=inverse_min_event_ndims, - is_constant_jacobian=is_constant_jacobian, - validate_args=validate_args, - name=name) - self._forward_fn = forward_fn - self._inverse_fn = inverse_fn - self._inverse_log_det_jacobian_fn = inverse_log_det_jacobian_fn - self._forward_log_det_jacobian_fn = forward_log_det_jacobian_fn - self._forward_event_shape_fn = forward_event_shape_fn - self._forward_event_shape_tensor_fn = forward_event_shape_tensor_fn - self._inverse_event_shape_fn = inverse_event_shape_fn - self._inverse_event_shape_tensor_fn = inverse_event_shape_tensor_fn - - def _forward_event_shape(self, input_shape): - if self._forward_event_shape_fn is None: - # By default assume shape doesn't change. - return input_shape - return self._forward_event_shape_fn(input_shape) - - def _forward_event_shape_tensor(self, input_shape): - if self._forward_event_shape_tensor_fn is None: - # By default assume shape doesn't change. - return input_shape - return self._forward_event_shape_tensor_fn(input_shape) - - def _inverse_event_shape(self, output_shape): - if self._inverse_event_shape_fn is None: - # By default assume shape doesn't change. - return output_shape - return self._inverse_event_shape_fn(output_shape) - - def _inverse_event_shape_tensor(self, output_shape): - if self._inverse_event_shape_tensor_fn is None: - # By default assume shape doesn't change. - return output_shape - return self._inverse_event_shape_tensor_fn(output_shape) - - def _forward(self, x, **kwargs): - if not callable(self._forward_fn): - raise NotImplementedError( - "forward_fn is not a callable function.") - return self._forward_fn(x, **kwargs) - - def _inverse(self, y, **kwargs): - if not callable(self._inverse_fn): - raise NotImplementedError( - "inverse_fn is not a callable function.") - return self._inverse_fn(y, **kwargs) - - def _inverse_log_det_jacobian(self, y, **kwargs): - if not callable(self._inverse_log_det_jacobian_fn): - raise NotImplementedError( - "inverse_log_det_jacobian_fn is not a callable function.") - return self._inverse_log_det_jacobian_fn(y, **kwargs) - - def _forward_log_det_jacobian(self, x, **kwargs): - if not callable(self._forward_log_det_jacobian_fn): - raise NotImplementedError( - "forward_log_det_jacobian_fn is not a callable function.") - return self._forward_log_det_jacobian_fn(x, **kwargs) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/invert.py b/tensorflow/contrib/distributions/python/ops/bijectors/invert.py deleted file mode 100644 index a648676d4b1..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/invert.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Invert bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - -__all__ = [ - "Invert", -] - - -class Invert(bijector.Bijector): - """Bijector which inverts another Bijector. - - Example Use: [ExpGammaDistribution (see Background & Context)]( - https://reference.wolfram.com/language/ref/ExpGammaDistribution.html) - models `Y=log(X)` where `X ~ Gamma`. - - ```python - exp_gamma_distribution = TransformedDistribution( - distribution=Gamma(concentration=1., rate=2.), - bijector=bijector.Invert(bijector.Exp()) - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, bijector, validate_args=False, name=None): - """Creates a `Bijector` which swaps the meaning of `inverse` and `forward`. - - Note: An inverted bijector's `inverse_log_det_jacobian` is often more - efficient if the base bijector implements `_forward_log_det_jacobian`. If - `_forward_log_det_jacobian` is not implemented then the following code is - used: - - ```python - y = self.inverse(x, **kwargs) - return -self.inverse_log_det_jacobian(y, **kwargs) - ``` - - Args: - bijector: Bijector instance. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str`, name given to ops managed by this object. - """ - - if not bijector._is_injective: # pylint: disable=protected-access - raise NotImplementedError( - "Invert is not implemented for non-injective bijectors.") - - self._bijector = bijector - super(Invert, self).__init__( - graph_parents=bijector.graph_parents, - forward_min_event_ndims=bijector.inverse_min_event_ndims, - inverse_min_event_ndims=bijector.forward_min_event_ndims, - is_constant_jacobian=bijector.is_constant_jacobian, - validate_args=validate_args, - dtype=bijector.dtype, - name=name or "_".join(["invert", bijector.name])) - - def _forward_event_shape(self, input_shape): - return self.bijector._inverse_event_shape(input_shape) # pylint: disable=protected-access - - def _forward_event_shape_tensor(self, input_shape): - return self.bijector._inverse_event_shape_tensor(input_shape) # pylint: disable=protected-access - - def _inverse_event_shape(self, output_shape): - return self.bijector._forward_event_shape(output_shape) # pylint: disable=protected-access - - def _inverse_event_shape_tensor(self, output_shape): - return self.bijector._forward_event_shape_tensor(output_shape) # pylint: disable=protected-access - - @property - def bijector(self): - return self._bijector - - def _forward(self, x, **kwargs): - return self.bijector._inverse(x, **kwargs) # pylint: disable=protected-access - - def _inverse(self, y, **kwargs): - return self.bijector._forward(y, **kwargs) # pylint: disable=protected-access - - def _inverse_log_det_jacobian(self, y, **kwargs): - return self.bijector._forward_log_det_jacobian(y, **kwargs) # pylint: disable=protected-access - - def _forward_log_det_jacobian(self, x, **kwargs): - return self.bijector._inverse_log_det_jacobian(x, **kwargs) # pylint: disable=protected-access diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/kumaraswamy.py b/tensorflow/contrib/distributions/python/ops/bijectors/kumaraswamy.py deleted file mode 100644 index 33b75a04d34..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/kumaraswamy.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Kumaraswamy bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - -__all__ = [ - "Kumaraswamy", -] - - -class Kumaraswamy(bijector.Bijector): - """Compute `Y = g(X) = (1 - (1 - X)**(1 / b))**(1 / a), X in [0, 1]`. - - This bijector maps inputs from `[0, 1]` to [0, 1]`. The inverse of the - bijector applied to a uniform random variable `X ~ U(0, 1) gives back a - random variable with the [Kumaraswamy distribution]( - https://en.wikipedia.org/wiki/Kumaraswamy_distribution): - - ```none - Y ~ Kumaraswamy(a, b) - pdf(y; a, b, 0 <= y <= 1) = a * b * y ** (a - 1) * (1 - y**a) ** (b - 1) - ``` - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - concentration1=None, - concentration0=None, - validate_args=False, - name="kumaraswamy"): - """Instantiates the `Kumaraswamy` bijector. - - Args: - concentration1: Python `float` scalar indicating the transform power, - i.e., `Y = g(X) = (1 - (1 - X)**(1 / b))**(1 / a)` where `a` is - `concentration1`. - concentration0: Python `float` scalar indicating the transform power, - i.e., `Y = g(X) = (1 - (1 - X)**(1 / b))**(1 / a)` where `b` is - `concentration0`. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - """ - self._graph_parents = [] - self._name = name - self._validate_args = validate_args - - with self._name_scope("init", values=[concentration1, concentration0]): - concentration1 = self._maybe_assert_valid_concentration( - ops.convert_to_tensor(concentration1, name="concentration1"), - validate_args=validate_args) - concentration0 = self._maybe_assert_valid_concentration( - ops.convert_to_tensor(concentration0, name="concentration0"), - validate_args=validate_args) - - self._concentration1 = concentration1 - self._concentration0 = concentration0 - super(Kumaraswamy, self).__init__( - forward_min_event_ndims=0, - validate_args=validate_args, - name=name) - - @property - def concentration1(self): - """The `a` in: `Y = g(X) = (1 - (1 - X)**(1 / b))**(1 / a)`.""" - return self._concentration1 - - @property - def concentration0(self): - """The `b` in: `Y = g(X) = (1 - (1 - X)**(1 / b))**(1 / a)`.""" - return self._concentration0 - - def _forward(self, x): - x = self._maybe_assert_valid(x) - return math_ops.exp( - math_ops.log1p(-math_ops.exp(math_ops.log1p(-x) / self.concentration0)) - / self.concentration1) - - def _inverse(self, y): - y = self._maybe_assert_valid(y) - return math_ops.exp(math_ops.log1p( - -(1 - y**self.concentration1)**self.concentration0)) - - def _inverse_log_det_jacobian(self, y): - y = self._maybe_assert_valid(y) - return ( - math_ops.log(self.concentration1) + math_ops.log(self.concentration0) + - (self.concentration1 - 1) * math_ops.log(y) + - (self.concentration0 - 1) * math_ops.log1p(-y**self.concentration1)) - - def _maybe_assert_valid_concentration(self, concentration, validate_args): - """Checks the validity of a concentration parameter.""" - if not validate_args: - return concentration - return control_flow_ops.with_dependencies([ - check_ops.assert_positive( - concentration, - message="Concentration parameter must be positive."), - ], concentration) - - def _maybe_assert_valid(self, x): - if not self.validate_args: - return x - return control_flow_ops.with_dependencies([ - check_ops.assert_non_negative( - x, - message="sample must be non-negative"), - check_ops.assert_less_equal( - x, array_ops.ones([], self.concentration0.dtype), - message="sample must be no larger than `1`."), - ], x) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py deleted file mode 100644 index 88855b27fd3..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py +++ /dev/null @@ -1,578 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""MaskedAutoregressiveFlow bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.layers import core as layers -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import template as template_ops -from tensorflow.python.ops import variable_scope as variable_scope_lib -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - -__all__ = [ - "MaskedAutoregressiveFlow", - "masked_autoregressive_default_template", - "masked_dense", -] - - -class MaskedAutoregressiveFlow(bijector.Bijector): - """Affine MaskedAutoregressiveFlow bijector for vector-valued events. - - The affine autoregressive flow [(Papamakarios et al., 2016)][3] provides a - relatively simple framework for user-specified (deep) architectures to learn - a distribution over vector-valued events. Regarding terminology, - - "Autoregressive models decompose the joint density as a product of - conditionals, and model each conditional in turn. Normalizing flows - transform a base density (e.g. a standard Gaussian) into the target density - by an invertible transformation with tractable Jacobian." - [(Papamakarios et al., 2016)][3] - - In other words, the "autoregressive property" is equivalent to the - decomposition, `p(x) = prod{ p(x[i] | x[0:i]) : i=0, ..., d }`. The provided - `shift_and_log_scale_fn`, `masked_autoregressive_default_template`, achieves - this property by zeroing out weights in its `masked_dense` layers. - - In the `tfp` framework, a "normalizing flow" is implemented as a - `tfp.bijectors.Bijector`. The `forward` "autoregression" - is implemented using a `tf.while_loop` and a deep neural network (DNN) with - masked weights such that the autoregressive property is automatically met in - the `inverse`. - - A `TransformedDistribution` using `MaskedAutoregressiveFlow(...)` uses the - (expensive) forward-mode calculation to draw samples and the (cheap) - reverse-mode calculation to compute log-probabilities. Conversely, a - `TransformedDistribution` using `Invert(MaskedAutoregressiveFlow(...))` uses - the (expensive) forward-mode calculation to compute log-probabilities and the - (cheap) reverse-mode calculation to compute samples. See "Example Use" - [below] for more details. - - Given a `shift_and_log_scale_fn`, the forward and inverse transformations are - (a sequence of) affine transformations. A "valid" `shift_and_log_scale_fn` - must compute each `shift` (aka `loc` or "mu" in [Germain et al. (2015)][1]) - and `log(scale)` (aka "alpha" in [Germain et al. (2015)][1]) such that each - are broadcastable with the arguments to `forward` and `inverse`, i.e., such - that the calculations in `forward`, `inverse` [below] are possible. - - For convenience, `masked_autoregressive_default_template` is offered as a - possible `shift_and_log_scale_fn` function. It implements the MADE - architecture [(Germain et al., 2015)][1]. MADE is a feed-forward network that - computes a `shift` and `log(scale)` using `masked_dense` layers in a deep - neural network. Weights are masked to ensure the autoregressive property. It - is possible that this architecture is suboptimal for your task. To build - alternative networks, either change the arguments to - `masked_autoregressive_default_template`, use the `masked_dense` function to - roll-out your own, or use some other architecture, e.g., using `tf.layers`. - - Warning: no attempt is made to validate that the `shift_and_log_scale_fn` - enforces the "autoregressive property". - - Assuming `shift_and_log_scale_fn` has valid shape and autoregressive - semantics, the forward transformation is - - ```python - def forward(x): - y = zeros_like(x) - event_size = x.shape[-1] - for _ in range(event_size): - shift, log_scale = shift_and_log_scale_fn(y) - y = x * math_ops.exp(log_scale) + shift - return y - ``` - - and the inverse transformation is - - ```python - def inverse(y): - shift, log_scale = shift_and_log_scale_fn(y) - return (y - shift) / math_ops.exp(log_scale) - ``` - - Notice that the `inverse` does not need a for-loop. This is because in the - forward pass each calculation of `shift` and `log_scale` is based on the `y` - calculated so far (not `x`). In the `inverse`, the `y` is fully known, thus is - equivalent to the scaling used in `forward` after `event_size` passes, i.e., - the "last" `y` used to compute `shift`, `log_scale`. (Roughly speaking, this - also proves the transform is bijective.) - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - tfb = tfp.bijectors - - dims = 5 - - # A common choice for a normalizing flow is to use a Gaussian for the base - # distribution. (However, any continuous distribution would work.) E.g., - maf = tfd.TransformedDistribution( - distribution=tfd.Normal(loc=0., scale=1.), - bijector=tfb.MaskedAutoregressiveFlow( - shift_and_log_scale_fn=tfb.masked_autoregressive_default_template( - hidden_layers=[512, 512])), - event_shape=[dims]) - - x = maf.sample() # Expensive; uses `tf.while_loop`, no Bijector caching. - maf.log_prob(x) # Almost free; uses Bijector caching. - maf.log_prob(0.) # Cheap; no `tf.while_loop` despite no Bijector caching. - - # [Papamakarios et al. (2016)][3] also describe an Inverse Autoregressive - # Flow [(Kingma et al., 2016)][2]: - iaf = tfd.TransformedDistribution( - distribution=tfd.Normal(loc=0., scale=1.), - bijector=tfb.Invert(tfb.MaskedAutoregressiveFlow( - shift_and_log_scale_fn=tfb.masked_autoregressive_default_template( - hidden_layers=[512, 512]))), - event_shape=[dims]) - - x = iaf.sample() # Cheap; no `tf.while_loop` despite no Bijector caching. - iaf.log_prob(x) # Almost free; uses Bijector caching. - iaf.log_prob(0.) # Expensive; uses `tf.while_loop`, no Bijector caching. - - # In many (if not most) cases the default `shift_and_log_scale_fn` will be a - # poor choice. Here's an example of using a "shift only" version and with a - # different number/depth of hidden layers. - shift_only = True - maf_no_scale_hidden2 = tfd.TransformedDistribution( - distribution=tfd.Normal(loc=0., scale=1.), - bijector=tfb.MaskedAutoregressiveFlow( - tfb.masked_autoregressive_default_template( - hidden_layers=[32], - shift_only=shift_only), - is_constant_jacobian=shift_only), - event_shape=[dims]) - ``` - - #### References - - [1]: Mathieu Germain, Karol Gregor, Iain Murray, and Hugo Larochelle. MADE: - Masked Autoencoder for Distribution Estimation. In _International - Conference on Machine Learning_, 2015. https://arxiv.org/abs/1502.03509 - - [2]: Diederik P. Kingma, Tim Salimans, Rafal Jozefowicz, Xi Chen, Ilya - Sutskever, and Max Welling. Improving Variational Inference with Inverse - Autoregressive Flow. In _Neural Information Processing Systems_, 2016. - https://arxiv.org/abs/1606.04934 - - [3]: George Papamakarios, Theo Pavlakou, and Iain Murray. Masked - Autoregressive Flow for Density Estimation. In _Neural Information - Processing Systems_, 2017. https://arxiv.org/abs/1705.07057 - """ - - @deprecation.deprecated( - "2018-10-01", "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - shift_and_log_scale_fn, - is_constant_jacobian=False, - validate_args=False, - unroll_loop=False, - name=None): - """Creates the MaskedAutoregressiveFlow bijector. - - Args: - shift_and_log_scale_fn: Python `callable` which computes `shift` and - `log_scale` from both the forward domain (`x`) and the inverse domain - (`y`). Calculation must respect the "autoregressive property" (see class - docstring). Suggested default - `masked_autoregressive_default_template(hidden_layers=...)`. Typically - the function contains `tf.Variables` and is wrapped using - `tf.compat.v1.make_template`. Returning `None` for either (both) - `shift`, `log_scale` is equivalent to (but more efficient than) - returning zero. - is_constant_jacobian: Python `bool`. Default: `False`. When `True` the - implementation assumes `log_scale` does not depend on the forward domain - (`x`) or inverse domain (`y`) values. (No validation is made; - `is_constant_jacobian=False` is always safe but possibly computationally - inefficient.) - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - unroll_loop: Python `bool` indicating whether the `tf.while_loop` in - `_forward` should be replaced with a static for loop. Requires that the - final dimension of `x` be known at graph construction time. Defaults to - `False`. - name: Python `str`, name given to ops managed by this object. - """ - name = name or "masked_autoregressive_flow" - self._shift_and_log_scale_fn = shift_and_log_scale_fn - self._unroll_loop = unroll_loop - super(MaskedAutoregressiveFlow, self).__init__( - forward_min_event_ndims=1, - is_constant_jacobian=is_constant_jacobian, - validate_args=validate_args, - name=name) - - def _forward(self, x): - if self._unroll_loop: - event_size = tensor_shape.dimension_value( - x.shape.with_rank_at_least(1)[-1]) - if event_size is None: - raise ValueError( - "The final dimension of `x` must be known at graph construction " - "time if `unroll_loop=True`. `x.shape: %r`" % x.shape) - y = array_ops.zeros_like(x, name="y0") - - for _ in range(event_size): - shift, log_scale = self._shift_and_log_scale_fn(y) - # next_y = scale * x + shift - next_y = x - if log_scale is not None: - next_y *= math_ops.exp(log_scale) - if shift is not None: - next_y += shift - y = next_y - return y - - event_size = array_ops.shape(x)[-1] - # If the event size is available at graph construction time, we can inform - # the graph compiler of the maximum number of steps. If not, - # static_event_size will be None, and the maximum_iterations argument will - # have no effect. - static_event_size = tensor_shape.dimension_value( - x.shape.with_rank_at_least(1)[-1]) - y0 = array_ops.zeros_like(x, name="y0") - # call the template once to ensure creation - _ = self._shift_and_log_scale_fn(y0) - - def _loop_body(index, y0): - """While-loop body for autoregression calculation.""" - # Set caching device to avoid re-getting the tf.Variable for every while - # loop iteration. - with variable_scope_lib.variable_scope( - variable_scope_lib.get_variable_scope()) as vs: - if vs.caching_device is None: - vs.set_caching_device(lambda op: op.device) - shift, log_scale = self._shift_and_log_scale_fn(y0) - y = x - if log_scale is not None: - y *= math_ops.exp(log_scale) - if shift is not None: - y += shift - return index + 1, y - - _, y = control_flow_ops.while_loop( - cond=lambda index, _: index < event_size, - body=_loop_body, - loop_vars=(0, y0), - maximum_iterations=static_event_size) - return y - - def _inverse(self, y): - shift, log_scale = self._shift_and_log_scale_fn(y) - x = y - if shift is not None: - x -= shift - if log_scale is not None: - x *= math_ops.exp(-log_scale) - return x - - def _inverse_log_det_jacobian(self, y): - _, log_scale = self._shift_and_log_scale_fn(y) - if log_scale is None: - return constant_op.constant(0., dtype=y.dtype, name="ildj") - return -math_ops.reduce_sum(log_scale, axis=-1) - - -MASK_INCLUSIVE = "inclusive" -MASK_EXCLUSIVE = "exclusive" - - -@deprecation.deprecated( - "2018-10-01", "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _gen_slices(num_blocks, n_in, n_out, mask_type=MASK_EXCLUSIVE): - """Generate the slices for building an autoregressive mask.""" - # TODO(b/67594795): Better support of dynamic shape. - slices = [] - col = 0 - d_in = n_in // num_blocks - d_out = n_out // num_blocks - row = d_out if mask_type == MASK_EXCLUSIVE else 0 - for _ in range(num_blocks): - row_slice = slice(row, None) - col_slice = slice(col, col + d_in) - slices.append([row_slice, col_slice]) - col += d_in - row += d_out - return slices - - -@deprecation.deprecated( - "2018-10-01", "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _gen_mask(num_blocks, - n_in, - n_out, - mask_type=MASK_EXCLUSIVE, - dtype=dtypes.float32): - """Generate the mask for building an autoregressive dense layer.""" - # TODO(b/67594795): Better support of dynamic shape. - mask = np.zeros([n_out, n_in], dtype=dtype.as_numpy_dtype()) - slices = _gen_slices(num_blocks, n_in, n_out, mask_type=mask_type) - for [row_slice, col_slice] in slices: - mask[row_slice, col_slice] = 1 - return mask - - -@deprecation.deprecated( - "2018-10-01", "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def masked_dense(inputs, - units, - num_blocks=None, - exclusive=False, - kernel_initializer=None, - reuse=None, - name=None, - *args, - **kwargs): - """A autoregressively masked dense layer. - - Analogous to `tf.compat.v1.layers.dense`. - - See [Germain et al. (2015)][1] for detailed explanation. - - Arguments: - inputs: Tensor input. - units: Python `int` scalar representing the dimensionality of the output - space. - num_blocks: Python `int` scalar representing the number of blocks for the - MADE masks. - exclusive: Python `bool` scalar representing whether to zero the diagonal of - the mask, used for the first layer of a MADE. - kernel_initializer: Initializer function for the weight matrix. If `None` - (default), weights are initialized using the - `tf.glorot_random_initializer`. - reuse: Python `bool` scalar representing whether to reuse the weights of a - previous layer by the same name. - name: Python `str` used to describe ops managed by this function. - *args: `tf.compat.v1.layers.dense` arguments. - **kwargs: `tf.compat.v1.layers.dense` keyword arguments. - - Returns: - Output tensor. - - Raises: - NotImplementedError: if rightmost dimension of `inputs` is unknown prior to - graph execution. - - #### References - - [1]: Mathieu Germain, Karol Gregor, Iain Murray, and Hugo Larochelle. MADE: - Masked Autoencoder for Distribution Estimation. In _International - Conference on Machine Learning_, 2015. https://arxiv.org/abs/1502.03509 - """ - # TODO(b/67594795): Better support of dynamic shape. - input_depth = tensor_shape.dimension_value( - inputs.shape.with_rank_at_least(1)[-1]) - if input_depth is None: - raise NotImplementedError( - "Rightmost dimension must be known prior to graph execution.") - - mask = _gen_mask(num_blocks, input_depth, units, - MASK_EXCLUSIVE if exclusive else MASK_INCLUSIVE).T - - if kernel_initializer is None: - kernel_initializer = init_ops.glorot_normal_initializer() - - def masked_initializer(shape, dtype=None, partition_info=None): - return mask * kernel_initializer(shape, dtype, partition_info) - - with ops.name_scope(name, "masked_dense", [inputs, units, num_blocks]): - layer = layers.Dense( - units, - kernel_initializer=masked_initializer, - kernel_constraint=lambda x: mask * x, - name=name, - dtype=inputs.dtype.base_dtype, - _scope=name, - _reuse=reuse, - *args, - **kwargs) - return layer.apply(inputs) - - -@deprecation.deprecated( - "2018-10-01", "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def masked_autoregressive_default_template( - hidden_layers, - shift_only=False, - activation=nn_ops.relu, - log_scale_min_clip=-5., - log_scale_max_clip=3., - log_scale_clip_gradient=False, - name=None, - *args, - **kwargs): - """Build the Masked Autoregressive Density Estimator (Germain et al., 2015). - - This will be wrapped in a make_template to ensure the variables are only - created once. It takes the input and returns the `loc` ("mu" in [Germain et - al. (2015)][1]) and `log_scale` ("alpha" in [Germain et al. (2015)][1]) from - the MADE network. - - Warning: This function uses `masked_dense` to create randomly initialized - `tf.Variables`. It is presumed that these will be fit, just as you would any - other neural architecture which uses `tf.compat.v1.layers.dense`. - - #### About Hidden Layers - - Each element of `hidden_layers` should be greater than the `input_depth` - (i.e., `input_depth = tf.shape(input)[-1]` where `input` is the input to the - neural network). This is necessary to ensure the autoregressivity property. - - #### About Clipping - - This function also optionally clips the `log_scale` (but possibly not its - gradient). This is useful because if `log_scale` is too small/large it might - underflow/overflow making it impossible for the `MaskedAutoregressiveFlow` - bijector to implement a bijection. Additionally, the `log_scale_clip_gradient` - `bool` indicates whether the gradient should also be clipped. The default does - not clip the gradient; this is useful because it still provides gradient - information (for fitting) yet solves the numerical stability problem. I.e., - `log_scale_clip_gradient = False` means - `grad[exp(clip(x))] = grad[x] exp(clip(x))` rather than the usual - `grad[clip(x)] exp(clip(x))`. - - Args: - hidden_layers: Python `list`-like of non-negative integer, scalars - indicating the number of units in each hidden layer. Default: `[512, 512]. - shift_only: Python `bool` indicating if only the `shift` term shall be - computed. Default: `False`. - activation: Activation function (callable). Explicitly setting to `None` - implies a linear activation. - log_scale_min_clip: `float`-like scalar `Tensor`, or a `Tensor` with the - same shape as `log_scale`. The minimum value to clip by. Default: -5. - log_scale_max_clip: `float`-like scalar `Tensor`, or a `Tensor` with the - same shape as `log_scale`. The maximum value to clip by. Default: 3. - log_scale_clip_gradient: Python `bool` indicating that the gradient of - `tf.clip_by_value` should be preserved. Default: `False`. - name: A name for ops managed by this function. Default: - "masked_autoregressive_default_template". - *args: `tf.compat.v1.layers.dense` arguments. - **kwargs: `tf.compat.v1.layers.dense` keyword arguments. - - Returns: - shift: `Float`-like `Tensor` of shift terms (the "mu" in - [Germain et al. (2015)][1]). - log_scale: `Float`-like `Tensor` of log(scale) terms (the "alpha" in - [Germain et al. (2015)][1]). - - Raises: - NotImplementedError: if rightmost dimension of `inputs` is unknown prior to - graph execution. - - #### References - - [1]: Mathieu Germain, Karol Gregor, Iain Murray, and Hugo Larochelle. MADE: - Masked Autoencoder for Distribution Estimation. In _International - Conference on Machine Learning_, 2015. https://arxiv.org/abs/1502.03509 - """ - name = name or "masked_autoregressive_default_template" - with ops.name_scope(name, values=[log_scale_min_clip, log_scale_max_clip]): - - def _fn(x): - """MADE parameterized via `masked_autoregressive_default_template`.""" - # TODO(b/67594795): Better support of dynamic shape. - input_depth = tensor_shape.dimension_value( - x.shape.with_rank_at_least(1)[-1]) - if input_depth is None: - raise NotImplementedError( - "Rightmost dimension must be known prior to graph execution.") - input_shape = ( - np.int32(x.shape.as_list()) - if x.shape.is_fully_defined() else array_ops.shape(x)) - for i, units in enumerate(hidden_layers): - x = masked_dense( - inputs=x, - units=units, - num_blocks=input_depth, - exclusive=True if i == 0 else False, - activation=activation, - *args, - **kwargs) - x = masked_dense( - inputs=x, - units=(1 if shift_only else 2) * input_depth, - num_blocks=input_depth, - activation=None, - *args, - **kwargs) - if shift_only: - x = array_ops.reshape(x, shape=input_shape) - return x, None - x = array_ops.reshape( - x, shape=array_ops.concat([input_shape, [2]], axis=0)) - shift, log_scale = array_ops.unstack(x, num=2, axis=-1) - which_clip = ( - math_ops.clip_by_value - if log_scale_clip_gradient else _clip_by_value_preserve_grad) - log_scale = which_clip(log_scale, log_scale_min_clip, log_scale_max_clip) - return shift, log_scale - - return template_ops.make_template(name, _fn) - - -@deprecation.deprecated( - "2018-10-01", "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _clip_by_value_preserve_grad(x, clip_value_min, clip_value_max, name=None): - """Clips input while leaving gradient unaltered.""" - with ops.name_scope(name, "clip_by_value_preserve_grad", - [x, clip_value_min, clip_value_max]): - clip_x = clip_ops.clip_by_value(x, clip_value_min, clip_value_max) - return x + array_ops.stop_gradient(clip_x - x) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/matrix_inverse_tril.py b/tensorflow/contrib/distributions/python/ops/bijectors/matrix_inverse_tril.py deleted file mode 100644 index 49e6192f067..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/matrix_inverse_tril.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""MatrixInverseTriL bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - - -__all__ = [ - "MatrixInverseTriL", -] - - -class MatrixInverseTriL(bijector.Bijector): - """Computes `g(L) = inv(L)`, where `L` is a lower-triangular matrix. - - `L` must be nonsingular; equivalently, all diagonal entries of `L` must be - nonzero. - - The input must have `rank >= 2`. The input is treated as a batch of matrices - with batch shape `input.shape[:-2]`, where each matrix has dimensions - `input.shape[-2]` by `input.shape[-1]` (hence `input.shape[-2]` must equal - `input.shape[-1]`). - - #### Examples - - ```python - tfd.bijectors.MatrixInverseTriL().forward(x=[[1., 0], [2, 1]]) - # Result: [[1., 0], [-2, 1]], i.e., inv(x) - - tfd.bijectors.MatrixInverseTriL().inverse(y=[[1., 0], [-2, 1]]) - # Result: [[1., 0], [2, 1]], i.e., inv(y). - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, validate_args=False, name="matrix_inverse_tril"): - """Instantiates the `MatrixInverseTriL` bijector. - - Args: - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - """ - self._graph_parents = [] - self._name = name - super(MatrixInverseTriL, self).__init__( - forward_min_event_ndims=2, - validate_args=validate_args, - name=name) - - def _forward(self, x): - with ops.control_dependencies(self._assertions(x)): - shape = array_ops.shape(x) - return linalg_ops.matrix_triangular_solve( - x, linalg_ops.eye(shape[-1], batch_shape=shape[:-2]), lower=True) - - def _inverse(self, y): - return self._forward(y) - - def _forward_log_det_jacobian(self, x): - # Calculation of the Jacobian: - # - # Let X = (x_{ij}), 0 <= i,j < n, be a matrix of indeterminates. Let Z = - # X^{-1} where Z = (z_{ij}). Then - # - # dZ/dx_{ij} = (d/dt | t=0) Y(t)^{-1}, - # - # where Y(t) = X + t*E_{ij} and E_{ij} is the matrix with a 1 in the (i,j) - # entry and zeros elsewhere. By the product rule, - # - # 0 = d/dt [Identity matrix] - # = d/dt [Y Y^{-1}] - # = Y d/dt[Y^{-1}] + dY/dt Y^{-1} - # - # so - # - # d/dt[Y^{-1}] = -Y^{-1} dY/dt Y^{-1} - # = -Y^{-1} E_{ij} Y^{-1}. - # - # Evaluating at t=0, - # - # dZ/dx_{ij} = -Z E_{ij} Z. - # - # Taking the (r,s) entry of each side, - # - # dz_{rs}/dx_{ij} = -z_{ri}z_{sj}. - # - # Now, let J be the Jacobian dZ/dX, arranged as the n^2-by-n^2 matrix whose - # (r*n + s, i*n + j) entry is dz_{rs}/dx_{ij}. Considering J as an n-by-n - # block matrix with n-by-n blocks, the above expression for dz_{rs}/dx_{ij} - # shows that the block at position (r,i) is -z_{ri}Z. Hence - # - # J = -KroneckerProduct(Z, Z), - # det(J) = (-1)^(n^2) (det Z)^(2n) - # = (-1)^n (det X)^(-2n). - with ops.control_dependencies(self._assertions(x)): - return (-2. * math_ops.cast(array_ops.shape(x)[-1], x.dtype.base_dtype) * - math_ops.reduce_sum( - math_ops.log(math_ops.abs(array_ops.matrix_diag_part(x))), - axis=-1)) - - def _assertions(self, x): - if not self.validate_args: - return [] - shape = array_ops.shape(x) - is_matrix = check_ops.assert_rank_at_least( - x, 2, message="Input must have rank at least 2.") - is_square = check_ops.assert_equal( - shape[-2], shape[-1], message="Input must be a square matrix.") - above_diagonal = array_ops.matrix_band_part( - array_ops.matrix_set_diag( - x, array_ops.zeros(shape[:-1], dtype=dtypes.float32)), - 0, -1) - is_lower_triangular = check_ops.assert_equal( - above_diagonal, array_ops.zeros_like(above_diagonal), - message="Input must be lower triangular.") - # A lower triangular matrix is nonsingular iff all its diagonal entries are - # nonzero. - diag_part = array_ops.matrix_diag_part(x) - is_nonsingular = check_ops.assert_none_equal( - diag_part, array_ops.zeros_like(diag_part), - message="Input must have all diagonal entries nonzero.") - return [is_matrix, is_square, is_lower_triangular, is_nonsingular] diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/ordered.py b/tensorflow/contrib/distributions/python/ops/bijectors/ordered.py deleted file mode 100644 index fb393218b6b..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/ordered.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ordered bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - - -__all__ = [ - "Ordered", -] - - -class Ordered(bijector.Bijector): - """Bijector which maps a tensor x_k that has increasing elements in the last - dimension to an unconstrained tensor y_k. - - Both the domain and the codomain of the mapping is [-inf, inf], however, - the input of the forward mapping must be strictly increasing. - The inverse of the bijector applied to a normal random vector `y ~ N(0, 1)` - gives back a sorted random vector with the same distribution `x ~ N(0, 1)` - where `x = sort(y)` - - On the last dimension of the tensor, Ordered bijector performs: - `y[0] = x[0]` - `y[1:] = math_ops.log(x[1:] - x[:-1])` - - #### Example Use: - - ```python - bijector.Ordered().forward([2, 3, 4]) - # Result: [2., 0., 0.] - - bijector.Ordered().inverse([0.06428002, -1.07774478, -0.71530371]) - # Result: [0.06428002, 0.40464228, 0.8936858] - ``` - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, validate_args=False, name="ordered"): - super(Ordered, self).__init__( - forward_min_event_ndims=1, - validate_args=validate_args, - name=name) - - def _forward_event_shape(self, input_shape): - if input_shape.ndims is None or input_shape[-1] is None: - return input_shape - return tensor_shape.TensorShape([input_shape[-1]]) - - def _forward_event_shape_tensor(self, input_shape): - return (input_shape[-1])[..., array_ops.newaxis] - - def _inverse_event_shape(self, output_shape): - if output_shape.ndims is None or output_shape[-1] is None: - return output_shape - if output_shape[-1] <= 1: - raise ValueError("output_shape[-1] = %d <= 1" % output_shape[-1]) - return tensor_shape.TensorShape([output_shape[-1]]) - - def _inverse_event_shape_tensor(self, output_shape): - if self.validate_args: - is_greater_one = check_ops.assert_greater( - output_shape[-1], 1, message="Need last dimension greater than 1.") - output_shape = control_flow_ops.with_dependencies( - [is_greater_one], output_shape) - return (output_shape[-1])[..., array_ops.newaxis] - - def _forward(self, x): - x = self._maybe_assert_valid_x(x) - y0 = x[..., 0, array_ops.newaxis] - yk = math_ops.log(x[..., 1:] - x[..., :-1]) - y = array_ops.concat([y0, yk], axis=-1) - return y - - def _inverse(self, y): - x0 = y[..., 0, array_ops.newaxis] - xk = math_ops.exp(y[..., 1:]) - x = array_ops.concat([x0, xk], axis=-1) - return math_ops.cumsum(x, axis=-1) - - def _inverse_log_det_jacobian(self, y): - # The Jacobian of the inverse mapping is lower - # triangular, with the diagonal elements being: - # J[i,i] = 1 if i=1, and - # exp(y_i) if 1 [1., 0., -1] - - reverse.inverse([1., 0., -1]) - # ==> [-1., 0., 1.] - - reverse.forward_log_det_jacobian(any_value) - # ==> 0. - - reverse.inverse_log_det_jacobian(any_value) - # ==> 0. - ``` - - Warning: `tf.estimator` may repeatedly build the graph thus - `Permute(np.random.permutation(event_size)).astype("int32"))` is not a - reliable parameterization (nor would it be even if using `tf.constant`). A - safe alternative is to use `tf.compat.v1.get_variable` to achieve "init once" - behavior, - i.e., - - ```python - def init_once(x, name): - return tf.compat.v1.get_variable(name, initializer=x, trainable=False) - - Permute(permutation=init_once( - np.random.permutation(event_size).astype("int32"), - name="permutation")) - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, permutation, validate_args=False, name=None): - """Creates the `Permute` bijector. - - Args: - permutation: An `int`-like vector-shaped `Tensor` representing the - permutation to apply to the rightmost dimension of the transformed - `Tensor`. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str`, name given to ops managed by this object. - - Raises: - TypeError: if `not permutation.dtype.is_integer`. - ValueError: if `permutation` does not contain exactly one of each of - `{0, 1, ..., d}`. - """ - with ops.name_scope(name, "permute", values=[permutation]): - permutation = ops.convert_to_tensor(permutation, name="permutation") - if not permutation.dtype.is_integer: - raise TypeError("permutation.dtype ({}) should be `int`-like.".format( - permutation.dtype.name)) - p = tensor_util.constant_value(permutation) - if p is not None: - if set(p) != set(np.arange(p.size)): - raise ValueError("Permutation over `d` must contain exactly one of " - "each of `{0, 1, ..., d}`.") - elif validate_args: - p, _ = nn_ops.top_k( - -permutation, k=array_ops.shape(permutation)[-1], sorted=True) - permutation = control_flow_ops.with_dependencies([ - check_ops.assert_equal( - -p, - math_ops.range(array_ops.size(p)), - message=("Permutation over `d` must contain exactly one of " - "each of `{0, 1, ..., d}`.")), - ], permutation) - self._permutation = permutation - super(Permute, self).__init__( - forward_min_event_ndims=1, - is_constant_jacobian=True, - validate_args=validate_args, - name=name or "permute") - - @property - def permutation(self): - return self._permutation - - def _forward(self, x): - return array_ops.gather(x, self.permutation, axis=-1) - - def _inverse(self, y): - return array_ops.gather( - y, array_ops.invert_permutation(self.permutation), axis=-1) - - def _inverse_log_det_jacobian(self, y): - # is_constant_jacobian = True for this bijector, hence the - # `log_det_jacobian` need only be specified for a single input, as this will - # be tiled to match `event_ndims`. - return constant_op.constant(0., dtype=y.dtype.base_dtype) - - def _forward_log_det_jacobian(self, x): - return constant_op.constant(0., dtype=x.dtype.base_dtype) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/power_transform.py b/tensorflow/contrib/distributions/python/ops/bijectors/power_transform.py deleted file mode 100644 index 16264fe728a..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/power_transform.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""PowerTransform bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - - -__all__ = [ - "PowerTransform", -] - - -class PowerTransform(bijector.Bijector): - """Compute `Y = g(X) = (1 + X * c)**(1 / c), X >= -1 / c`. - - The [power transform](https://en.wikipedia.org/wiki/Power_transform) maps - inputs from `[0, inf]` to `[-1/c, inf]`; this is equivalent to the `inverse` - of this bijector. - - This bijector is equivalent to the `Exp` bijector when `c=0`. - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - power=0., - validate_args=False, - name="power_transform"): - """Instantiates the `PowerTransform` bijector. - - Args: - power: Python `float` scalar indicating the transform power, i.e., - `Y = g(X) = (1 + X * c)**(1 / c)` where `c` is the `power`. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - - Raises: - ValueError: if `power < 0` or is not known statically. - """ - self._graph_parents = [] - self._name = name - self._validate_args = validate_args - with self._name_scope("init", values=[power]): - power = tensor_util.constant_value( - ops.convert_to_tensor(power, name="power")) - if power is None or power < 0: - raise ValueError("`power` must be a non-negative TF constant.") - self._power = power - super(PowerTransform, self).__init__( - forward_min_event_ndims=0, - validate_args=validate_args, - name=name) - - @property - def power(self): - """The `c` in: `Y = g(X) = (1 + X * c)**(1 / c)`.""" - return self._power - - def _forward(self, x): - x = self._maybe_assert_valid_x(x) - if self.power == 0.: - return math_ops.exp(x) - # If large x accuracy is an issue, consider using: - # (1. + x * self.power)**(1. / self.power) when x >> 1. - return math_ops.exp(math_ops.log1p(x * self.power) / self.power) - - def _inverse(self, y): - y = self._maybe_assert_valid_y(y) - if self.power == 0.: - return math_ops.log(y) - # If large y accuracy is an issue, consider using: - # (y**self.power - 1.) / self.power when y >> 1. - return math_ops.expm1(math_ops.log(y) * self.power) / self.power - - def _inverse_log_det_jacobian(self, y): - y = self._maybe_assert_valid_y(y) - return (self.power - 1.) * math_ops.log(y) - - def _forward_log_det_jacobian(self, x): - x = self._maybe_assert_valid_x(x) - if self.power == 0.: - return x - return (1. / self.power - 1.) * math_ops.log1p(x * self.power) - - def _maybe_assert_valid_x(self, x): - if not self.validate_args or self.power == 0.: - return x - is_valid = check_ops.assert_non_negative( - 1. + self.power * x, - message="Forward transformation input must be at least {}.".format( - -1. / self.power)) - return control_flow_ops.with_dependencies([is_valid], x) - - def _maybe_assert_valid_y(self, y): - if not self.validate_args: - return y - is_valid = check_ops.assert_positive( - y, message="Inverse transformation input must be greater than 0.") - return control_flow_ops.with_dependencies([is_valid], y) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/real_nvp.py b/tensorflow/contrib/distributions/python/ops/bijectors/real_nvp.py deleted file mode 100644 index c0b20fd8637..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/real_nvp.py +++ /dev/null @@ -1,308 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Real NVP bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.layers import core as layers -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import template as template_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - -__all__ = ["RealNVP", "real_nvp_default_template"] - - -class RealNVP(bijector.Bijector): - """RealNVP "affine coupling layer" for vector-valued events. - - Real NVP models a normalizing flow on a `D`-dimensional distribution via a - single `D-d`-dimensional conditional distribution [(Dinh et al., 2017)][1]: - - `y[d:D] = y[d:D] * math_ops.exp(log_scale_fn(y[d:D])) + shift_fn(y[d:D])` - `y[0:d] = x[0:d]` - - The last `D-d` units are scaled and shifted based on the first `d` units only, - while the first `d` units are 'masked' and left unchanged. Real NVP's - `shift_and_log_scale_fn` computes vector-valued quantities. For - scale-and-shift transforms that do not depend on any masked units, i.e. - `d=0`, use the `tfb.Affine` bijector with learned parameters instead. - - Masking is currently only supported for base distributions with - `event_ndims=1`. For more sophisticated masking schemes like checkerboard or - channel-wise masking [(Papamakarios et al., 2016)[4], use the `tfb.Permute` - bijector to re-order desired masked units into the first `d` units. For base - distributions with `event_ndims > 1`, use the `tfb.Reshape` bijector to - flatten the event shape. - - Recall that the MAF bijector [(Papamakarios et al., 2016)][4] implements a - normalizing flow via an autoregressive transformation. MAF and IAF have - opposite computational tradeoffs - MAF can train all units in parallel but - must sample units sequentially, while IAF must train units sequentially but - can sample in parallel. In contrast, Real NVP can compute both forward and - inverse computations in parallel. However, the lack of an autoregressive - transformations makes it less expressive on a per-bijector basis. - - A "valid" `shift_and_log_scale_fn` must compute each `shift` (aka `loc` or - "mu" in [Papamakarios et al. (2016)][4]) and `log(scale)` (aka "alpha" in - [Papamakarios et al. (2016)][4]) such that each are broadcastable with the - arguments to `forward` and `inverse`, i.e., such that the calculations in - `forward`, `inverse` [below] are possible. For convenience, - `real_nvp_default_nvp` is offered as a possible `shift_and_log_scale_fn` - function. - - NICE [(Dinh et al., 2014)][2] is a special case of the Real NVP bijector - which discards the scale transformation, resulting in a constant-time - inverse-log-determinant-Jacobian. To use a NICE bijector instead of Real - NVP, `shift_and_log_scale_fn` should return `(shift, None)`, and - `is_constant_jacobian` should be set to `True` in the `RealNVP` constructor. - Calling `real_nvp_default_template` with `shift_only=True` returns one such - NICE-compatible `shift_and_log_scale_fn`. - - Caching: the scalar input depth `D` of the base distribution is not known at - construction time. The first call to any of `forward(x)`, `inverse(x)`, - `inverse_log_det_jacobian(x)`, or `forward_log_det_jacobian(x)` memoizes - `D`, which is re-used in subsequent calls. This shape must be known prior to - graph execution (which is the case if using tf.layers). - - #### Example Use - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - tfb = tfp.bijectors - - # A common choice for a normalizing flow is to use a Gaussian for the base - # distribution. (However, any continuous distribution would work.) E.g., - num_dims = 3 - num_samples = 1 - nvp = tfd.TransformedDistribution( - distribution=tfd.MultivariateNormalDiag(loc=np.zeros(num_dims)), - bijector=tfb.RealNVP( - num_masked=2, - shift_and_log_scale_fn=tfb.real_nvp_default_template( - hidden_layers=[512, 512]))) - - x = nvp.sample(num_samples) - nvp.log_prob(x) - nvp.log_prob(np.zeros([num_samples, num_dims])) - ``` - - For more examples, see [Jang (2018)][3]. - - #### References - - [1]: Laurent Dinh, Jascha Sohl-Dickstein, and Samy Bengio. Density Estimation - using Real NVP. In _International Conference on Learning - Representations_, 2017. https://arxiv.org/abs/1605.08803 - - [2]: Laurent Dinh, David Krueger, and Yoshua Bengio. NICE: Non-linear - Independent Components Estimation. _arXiv preprint arXiv:1410.8516_, - 2014. https://arxiv.org/abs/1410.8516 - - [3]: Eric Jang. Normalizing Flows Tutorial, Part 2: Modern Normalizing Flows. - _Technical Report_, 2018. http://blog.evjang.com/2018/01/nf2.html - - [4]: George Papamakarios, Theo Pavlakou, and Iain Murray. Masked - Autoregressive Flow for Density Estimation. In _Neural Information - Processing Systems_, 2017. https://arxiv.org/abs/1705.07057 - """ - - @deprecation.deprecated( - "2018-10-01", "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - num_masked, - shift_and_log_scale_fn, - is_constant_jacobian=False, - validate_args=False, - name=None): - """Creates the Real NVP or NICE bijector. - - Args: - num_masked: Python `int` indicating that the first `d` units of the event - should be masked. Must be in the closed interval `[1, D-1]`, where `D` - is the event size of the base distribution. - shift_and_log_scale_fn: Python `callable` which computes `shift` and - `log_scale` from both the forward domain (`x`) and the inverse domain - (`y`). Calculation must respect the "autoregressive property" (see class - docstring). Suggested default - `masked_autoregressive_default_template(hidden_layers=...)`. Typically - the function contains `tf.Variables` and is wrapped using - `tf.compat.v1.make_template`. Returning `None` for either (both) - `shift`, `log_scale` is equivalent to (but more efficient than) - returning zero. - is_constant_jacobian: Python `bool`. Default: `False`. When `True` the - implementation assumes `log_scale` does not depend on the forward domain - (`x`) or inverse domain (`y`) values. (No validation is made; - `is_constant_jacobian=False` is always safe but possibly computationally - inefficient.) - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str`, name given to ops managed by this object. - - Raises: - ValueError: If num_masked < 1. - """ - name = name or "real_nvp" - if num_masked <= 0: - raise ValueError("num_masked must be a positive integer.") - self._num_masked = num_masked - # At construction time, we don't know input_depth. - self._input_depth = None - self._shift_and_log_scale_fn = shift_and_log_scale_fn - super(RealNVP, self).__init__( - forward_min_event_ndims=1, - is_constant_jacobian=is_constant_jacobian, - validate_args=validate_args, - name=name) - - def _cache_input_depth(self, x): - if self._input_depth is None: - self._input_depth = tensor_shape.dimension_value( - x.shape.with_rank_at_least(1)[-1]) - if self._input_depth is None: - raise NotImplementedError( - "Rightmost dimension must be known prior to graph execution.") - if self._num_masked >= self._input_depth: - raise ValueError( - "Number of masked units must be smaller than the event size.") - - def _forward(self, x): - self._cache_input_depth(x) - # Performs scale and shift. - x0, x1 = x[:, :self._num_masked], x[:, self._num_masked:] - shift, log_scale = self._shift_and_log_scale_fn( - x0, self._input_depth - self._num_masked) - y1 = x1 - if log_scale is not None: - y1 *= math_ops.exp(log_scale) - if shift is not None: - y1 += shift - y = array_ops.concat([x0, y1], axis=-1) - return y - - def _inverse(self, y): - self._cache_input_depth(y) - # Performs un-shift and un-scale. - y0, y1 = y[:, :self._num_masked], y[:, self._num_masked:] - shift, log_scale = self._shift_and_log_scale_fn( - y0, self._input_depth - self._num_masked) - x1 = y1 - if shift is not None: - x1 -= shift - if log_scale is not None: - x1 *= math_ops.exp(-log_scale) - x = array_ops.concat([y0, x1], axis=-1) - return x - - def _inverse_log_det_jacobian(self, y): - self._cache_input_depth(y) - y0 = y[:, :self._num_masked] - _, log_scale = self._shift_and_log_scale_fn( - y0, self._input_depth - self._num_masked) - if log_scale is None: - return constant_op.constant(0., dtype=y.dtype, name="ildj") - return -math_ops.reduce_sum(log_scale, axis=-1) - - def _forward_log_det_jacobian(self, x): - self._cache_input_depth(x) - x0 = x[:, :self._num_masked] - _, log_scale = self._shift_and_log_scale_fn( - x0, self._input_depth - self._num_masked) - if log_scale is None: - return constant_op.constant(0., dtype=x.dtype, name="fldj") - return math_ops.reduce_sum(log_scale, axis=-1) - - -@deprecation.deprecated( - "2018-10-01", "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def real_nvp_default_template( - hidden_layers, - shift_only=False, - activation=nn_ops.relu, - name=None, - *args, - **kwargs): - """Build a scale-and-shift function using a multi-layer neural network. - - This will be wrapped in a make_template to ensure the variables are only - created once. It takes the `d`-dimensional input x[0:d] and returns the `D-d` - dimensional outputs `loc` ("mu") and `log_scale` ("alpha"). - - Arguments: - hidden_layers: Python `list`-like of non-negative integer, scalars - indicating the number of units in each hidden layer. Default: `[512, 512]. - shift_only: Python `bool` indicating if only the `shift` term shall be - computed (i.e. NICE bijector). Default: `False`. - activation: Activation function (callable). Explicitly setting to `None` - implies a linear activation. - name: A name for ops managed by this function. Default: - "real_nvp_default_template". - *args: `tf.compat.v1.layers.dense` arguments. - **kwargs: `tf.compat.v1.layers.dense` keyword arguments. - - Returns: - shift: `Float`-like `Tensor` of shift terms ("mu" in - [Papamakarios et al. (2016)][1]). - log_scale: `Float`-like `Tensor` of log(scale) terms ("alpha" in - [Papamakarios et al. (2016)][1]). - - Raises: - NotImplementedError: if rightmost dimension of `inputs` is unknown prior to - graph execution. - - #### References - - [1]: George Papamakarios, Theo Pavlakou, and Iain Murray. Masked - Autoregressive Flow for Density Estimation. In _Neural Information - Processing Systems_, 2017. https://arxiv.org/abs/1705.07057 - """ - - with ops.name_scope(name, "real_nvp_default_template"): - - def _fn(x, output_units): - """Fully connected MLP parameterized via `real_nvp_template`.""" - for units in hidden_layers: - x = layers.dense( - inputs=x, units=units, activation=activation, *args, **kwargs) - x = layers.dense( - inputs=x, - units=(1 if shift_only else 2) * output_units, - activation=None, - *args, - **kwargs) - if shift_only: - return x, None - shift, log_scale = array_ops.split(x, 2, axis=-1) - return shift, log_scale - - return template_ops.make_template("real_nvp_default_template", _fn) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py b/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py deleted file mode 100644 index ec203e17173..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py +++ /dev/null @@ -1,343 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Reshape bijectors.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - - -__all__ = [ - "Reshape", -] - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _static_ndims_from_shape(shape): - return tensor_shape.dimension_value(shape.shape.with_rank_at_least(1)[0]) - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _ndims_from_shape(shape): - return array_ops.shape(shape)[0] - - -class Reshape(bijector.Bijector): - """Reshapes the `event_shape` of a `Tensor`. - - The semantics generally follow that of `tf.reshape()`, with - a few differences: - - * The user must provide both the input and output shape, so that - the transformation can be inverted. If an input shape is not - specified, the default assumes a vector-shaped input, i.e., - event_shape_in = (-1,). - * The `Reshape` bijector automatically broadcasts over the leftmost - dimensions of its input (`sample_shape` and `batch_shape`); only - the rightmost `event_ndims_in` dimensions are reshaped. The - number of dimensions to reshape is inferred from the provided - `event_shape_in` (`event_ndims_in = len(event_shape_in)`). - - Example usage: - ```python - - import tensorflow_probability as tfp - tfb = tfp.bijectors - - r = tfb.Reshape(event_shape_out=[1, -1]) - - r.forward([3., 4.]) # shape [2] - # ==> [[3., 4.]] # shape [1, 2] - - r.forward([[1., 2.], [3., 4.]]) # shape [2, 2] - # ==> [[[1., 2.]], - # [[3., 4.]]] # shape [2, 1, 2] - - r.inverse([[3., 4.]]) # shape [1,2] - # ==> [3., 4.] # shape [2] - - r.forward_log_det_jacobian(any_value) - # ==> 0. - - r.inverse_log_det_jacobian(any_value) - # ==> 0. - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, event_shape_out, event_shape_in=(-1,), - validate_args=False, name=None): - """Creates a `Reshape` bijector. - - Args: - event_shape_out: An `int`-like vector-shaped `Tensor` - representing the event shape of the transformed output. - event_shape_in: An optional `int`-like vector-shape `Tensor` - representing the event shape of the input. This is required in - order to define inverse operations; the default of (-1,) - assumes a vector-shaped input. - validate_args: Python `bool` indicating whether arguments should - be checked for correctness. - name: Python `str`, name given to ops managed by this object. - - Raises: - TypeError: if either `event_shape_in` or `event_shape_out` has - non-integer `dtype`. - ValueError: if either of `event_shape_in` or `event_shape_out` - has non-vector shape (`rank > 1`), or if their sizes do not - match. - """ - with ops.name_scope(name, "reshape", - values=[event_shape_out, event_shape_in]): - - event_shape_out = ops.convert_to_tensor(event_shape_out, - name="event_shape_out", - preferred_dtype=dtypes.int32) - event_shape_in = ops.convert_to_tensor(event_shape_in, - name="event_shape_in", - preferred_dtype=dtypes.int32) - - assertions = [] - assertions.extend(self._maybe_check_valid_shape( - event_shape_out, validate_args)) - assertions.extend(self._maybe_check_valid_shape( - event_shape_in, validate_args)) - - self._assertions = assertions - self._event_shape_in = event_shape_in - self._event_shape_out = event_shape_out - - super(Reshape, self).__init__( - forward_min_event_ndims=0, - is_constant_jacobian=True, - validate_args=validate_args, - name=name or "reshape") - - def _maybe_check_valid_shape(self, shape, validate_args): - """Check that a shape Tensor is int-type and otherwise sane.""" - if not shape.dtype.is_integer: - raise TypeError("{} dtype ({}) should be `int`-like.".format( - shape, shape.dtype.name)) - - assertions = [] - - ndims = array_ops.rank(shape) - ndims_ = tensor_util.constant_value(ndims) - if ndims_ is not None and ndims_ > 1: - raise ValueError("`{}` rank ({}) should be <= 1.".format( - shape, ndims_)) - elif validate_args: - assertions.append(check_ops.assert_less_equal( - ndims, 1, message="`{}` rank should be <= 1.".format(shape))) - - shape_ = tensor_util.constant_value_as_shape(shape) - if shape_.is_fully_defined(): - es = np.int32(shape_.as_list()) - if sum(es == -1) > 1: - raise ValueError( - "`{}` must have at most one `-1` (given {})" - .format(shape, es)) - if np.any(es < -1): - raise ValueError( - "`{}` elements must be either positive integers or `-1`" - "(given {})." - .format(shape, es)) - elif validate_args: - assertions.extend([ - check_ops.assert_less_equal( - math_ops.reduce_sum( - math_ops.cast(math_ops.equal(shape, -1), dtypes.int32)), - 1, - message="`{}` elements must have at most one `-1`." - .format(shape)), - check_ops.assert_greater_equal( - shape, -1, - message="`{}` elements must be either positive integers or `-1`." - .format(shape)), - ]) - return assertions - - def _reshape_helper(self, x, event_shape_in, event_shape_out): - """Reshape only the event_shape of an input `Tensor`.""" - - event_ndims_in_ = _static_ndims_from_shape(event_shape_in) - event_ndims_in = _ndims_from_shape(event_shape_in) - x_ndims_, x_ndims = x.shape.ndims, array_ops.rank(x) - - assertions = [] - - # Ensure x.event_shape is compatible with event_shape_in. - if (event_ndims_in_ is not None - and x_ndims_ is not None - and x.shape.with_rank_at_least(event_ndims_in_)[ - x_ndims_-event_ndims_in_:].is_fully_defined()): - x_event_shape_, x_event_shape = [ # pylint: disable=unbalanced-tuple-unpacking - np.int32(x.shape[x_ndims_-event_ndims_in_:])]*2 - else: - x_event_shape_, x_event_shape = ( - None, array_ops.shape(x)[x_ndims-event_ndims_in:]) - - event_shape_in_ = tensor_util.constant_value(event_shape_in) - - if x_event_shape_ is not None and event_shape_in_ is not None: - # Compare the shape dimensions that are fully specified in the - # input (i.e., for which event_shape_in is not -1). If x_event_shape - # matches along all of these dimensions, it is compatible with - # the desired input shape and any further mismatches (i.e., - # imcompatibility with the desired *output* shape) will be - # caught inside of array_ops.reshape() below. - x_event_shape_specified_ = x_event_shape_[event_shape_in_ >= 0] - event_shape_in_specified_ = event_shape_in_[event_shape_in_ >= 0] - if not np.equal(x_event_shape_specified_, - event_shape_in_specified_).all(): - raise ValueError( - "Input `event_shape` does not match `event_shape_in` ({} vs {}).". - format(x_event_shape_, event_shape_in_)) - elif self.validate_args: - # Similarly to the static case, we compare the shape dimensions - # that are fully specified in the input. We extract these - # dimensions using boolean_mask(), which requires that the mask - # have known ndims. We can assume that shape Tensors always have - # ndims==1 (this assumption is verified inside of - # _maybe_check_valid_shape), so the reshape operation is just a - # no-op that formally encodes this fact to make boolean_mask() - # happy. - event_shape_mask = array_ops.reshape(event_shape_in >= 0, [-1]) - x_event_shape_specified = array_ops.boolean_mask(x_event_shape, - event_shape_mask) - event_shape_in_specified = array_ops.boolean_mask(event_shape_in, - event_shape_mask) - assertions.append(check_ops.assert_equal( - x_event_shape_specified, event_shape_in_specified, - message="Input `event_shape` does not match `event_shape_in`.")) - - if assertions: - x = control_flow_ops.with_dependencies(assertions, x) - - # get the parts of shape(x) that will not change - sample_and_batch_shape = array_ops.shape(x) - - ndims = (x.shape.ndims if x.shape.ndims is not None - else array_ops.rank(x)) - sample_and_batch_shape = sample_and_batch_shape[ - :(ndims - math_ops.abs(event_ndims_in))] - - if (event_ndims_in_ is not None - and x_ndims_ is not None - and event_ndims_in_ == x_ndims_): - # Hack to allow forward/inverse_event_shape to do shape - # inference by calling this helper method with a dummy Tensor of - # shape event_shape_in. In this special case, - # sample_and_batch_shape will be empty so we can preserve static - # shape information by avoiding the concat operation below - # (which would be a no-op). - new_shape = event_shape_out - else: - new_shape = array_ops.concat( - [sample_and_batch_shape, event_shape_out], axis=0) - - return array_ops.reshape(x, new_shape) - - def _forward(self, x): - with ops.control_dependencies(self._assertions): - return self._reshape_helper(x, - self._event_shape_in, - self._event_shape_out) - - def _inverse(self, y): - with ops.control_dependencies(self._assertions): - return self._reshape_helper(y, - self._event_shape_out, - self._event_shape_in) - - def _inverse_log_det_jacobian(self, y): - with ops.control_dependencies(self._assertions): - return constant_op.constant(0., dtype=y.dtype) - - def _forward_log_det_jacobian(self, x): - with ops.control_dependencies(self._assertions): - return constant_op.constant(0., dtype=x.dtype) - - def _forward_event_shape(self, input_shape): - # NOTE: this method and the other *_event_shape* methods - # compute shape by explicit transformation of a dummy - # variable. This approach is not generally recommended because it - # bloats the graph and could in general trigger side effects. - # - # In this particular case of the Reshape bijector, the - # forward and inverse transforms have no side effects, and we - # believe the reduction in code complexity from delegating the - # heavy lifting to tf.reshape() is worth the added graph ops. - # However, you should think hard before implementing this approach - # in other Bijectors; it is strongly preferred to compute - # shapes explicitly whenever it's feasible to do so. - with ops.control_dependencies(self._assertions): - dummy = array_ops.zeros(dtype=dtypes.float32, shape=input_shape) - dummy_reshaped = self.forward(dummy) - return dummy_reshaped.shape - - def _inverse_event_shape(self, output_shape): - with ops.control_dependencies(self._assertions): - dummy = array_ops.zeros(dtype=dtypes.float32, shape=output_shape) - dummy_reshaped = self.inverse(dummy) - return dummy_reshaped.shape - - def _forward_event_shape_tensor(self, input_shape): - with ops.control_dependencies(self._assertions): - dummy = array_ops.zeros(dtype=dtypes.float32, shape=input_shape) - dummy_reshaped = self.forward(dummy) - return array_ops.shape(dummy_reshaped) - - def _inverse_event_shape_tensor(self, output_shape): - with ops.control_dependencies(self._assertions): - dummy = array_ops.zeros(dtype=dtypes.float32, shape=output_shape) - dummy_reshaped = self.inverse(dummy) - return array_ops.shape(dummy_reshaped) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/scale_tril.py b/tensorflow/contrib/distributions/python/ops/bijectors/scale_tril.py deleted file mode 100644 index 0a6d690b65c..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/scale_tril.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ScaleTriL bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops.bijectors import affine_scalar -from tensorflow.contrib.distributions.python.ops.bijectors import chain -from tensorflow.contrib.distributions.python.ops.bijectors import fill_triangular -from tensorflow.contrib.distributions.python.ops.bijectors import softplus -from tensorflow.contrib.distributions.python.ops.bijectors import transform_diagonal -from tensorflow.python.util import deprecation - -__all__ = [ - "ScaleTriL", -] - - -class ScaleTriL(chain.Chain): - """Transforms unconstrained vectors to TriL matrices with positive diagonal. - - This is implemented as a simple `tfb.Chain` of `tfb.FillTriangular` - followed by `tfb.TransformDiagonal`, and provided mostly as a - convenience. The default setup is somewhat opinionated, using a - Softplus transformation followed by a small shift (`1e-5`) which - attempts to avoid numerical issues from zeros on the diagonal. - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - tfb = tfp.bijectors - - b = tfb.ScaleTriL( - diag_bijector=tfb.Exp(), - diag_shift=None) - b.forward(x=[0., 0., 0.]) - # Result: [[1., 0.], - # [0., 1.]] - b.inverse(y=[[1., 0], - [.5, 2]]) - # Result: [log(2), .5, log(1)] - - # Define a distribution over PSD matrices of shape `[3, 3]`, - # with `1 + 2 + 3 = 6` degrees of freedom. - dist = tfd.TransformedDistribution( - tfd.Normal(tf.zeros(6), tf.ones(6)), - tfb.Chain([tfb.CholeskyOuterProduct(), tfb.ScaleTriL()])) - - # Using an identity transformation, ScaleTriL is equivalent to - # tfb.FillTriangular. - b = tfb.ScaleTriL( - diag_bijector=tfb.Identity(), - diag_shift=None) - - # For greater control over initialization, one can manually encode - # pre- and post- shifts inside of `diag_bijector`. - b = tfb.ScaleTriL( - diag_bijector=tfb.Chain([ - tfb.AffineScalar(shift=1e-3), - tfb.Softplus(), - tfb.AffineScalar(shift=0.5413)]), # softplus_inverse(1.) - # = log(expm1(1.)) = 0.5413 - diag_shift=None) - ``` - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - diag_bijector=None, - diag_shift=1e-5, - validate_args=False, - name="scale_tril"): - """Instantiates the `ScaleTriL` bijector. - - Args: - diag_bijector: `Bijector` instance, used to transform the output diagonal - to be positive. - Default value: `None` (i.e., `tfb.Softplus()`). - diag_shift: Float value broadcastable and added to all diagonal entries - after applying the `diag_bijector`. Setting a positive - value forces the output diagonal entries to be positive, but - prevents inverting the transformation for matrices with - diagonal entries less than this value. - Default value: `1e-5` (i.e., no shift is applied). - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - Default value: `False` (i.e., arguments are not validated). - name: Python `str` name given to ops managed by this object. - Default value: `scale_tril`. - """ - - if diag_bijector is None: - diag_bijector = softplus.Softplus(validate_args=validate_args) - - if diag_shift is not None: - diag_bijector = chain.Chain([affine_scalar.AffineScalar(shift=diag_shift), - diag_bijector]) - - super(ScaleTriL, self).__init__( - [transform_diagonal.TransformDiagonal(diag_bijector=diag_bijector), - fill_triangular.FillTriangular()], - validate_args=validate_args, - name=name) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/sigmoid.py b/tensorflow/contrib/distributions/python/ops/bijectors/sigmoid.py deleted file mode 100644 index 194b318fce3..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/sigmoid.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Sigmoid bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - - -__all__ = [ - "Sigmoid", -] - - -class Sigmoid(bijector.Bijector): - """Bijector which computes `Y = g(X) = 1 / (1 + exp(-X))`.""" - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, validate_args=False, name="sigmoid"): - super(Sigmoid, self).__init__( - forward_min_event_ndims=0, - validate_args=validate_args, - name=name) - - def _forward(self, x): - return math_ops.sigmoid(x) - - def _inverse(self, y): - return math_ops.log(y) - math_ops.log1p(-y) - - def _inverse_log_det_jacobian(self, y): - return -math_ops.log(y) - math_ops.log1p(-y) - - def _forward_log_det_jacobian(self, x): - return -nn_ops.softplus(-x) - nn_ops.softplus(x) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/sinh_arcsinh.py b/tensorflow/contrib/distributions/python/ops/bijectors/sinh_arcsinh.py deleted file mode 100644 index aee3a603d2b..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/sinh_arcsinh.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""SinhArcsinh bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - -__all__ = [ - "SinhArcsinh", -] - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _sqrtx2p1(x): - """Implementation of `sqrt(1 + x**2)` which is stable despite large `x`.""" - return array_ops.where_v2( - math_ops.abs(x) * np.sqrt(np.finfo(x.dtype.as_numpy_dtype).eps) <= 1., - math_ops.sqrt(x**2. + 1.), - # For large x, calculating x**2 can overflow. This can be alleviated by - # considering: - # sqrt(1 + x**2) - # = exp(0.5 log(1 + x**2)) - # = exp(0.5 log(x**2 * (1 + x**-2))) - # = exp(log(x) + 0.5 * log(1 + x**-2)) - # = |x| * exp(0.5 log(1 + x**-2)) - # = |x| * sqrt(1 + x**-2) - # We omit the last term in this approximation. - # When |x| > 1 / sqrt(machineepsilon), the second term will be 1, - # due to sqrt(1 + x**-2) = 1. This is also true with the gradient term, - # and higher order gradients, since the first order derivative of - # sqrt(1 + x**-2) is -2 * x**-3 / (1 + x**-2) = -2 / (x**3 + x), - # and all nth-order derivatives will be O(x**-(n + 2)). This makes any - # gradient terms that contain any derivatives of sqrt(1 + x**-2) vanish. - math_ops.abs(x)) - - -class SinhArcsinh(bijector.Bijector): - """Compute `Y = g(X) = Sinh( (Arcsinh(X) + skewness) * tailweight )`. - - For `skewness in (-inf, inf)` and `tailweight in (0, inf)`, this - transformation is a - diffeomorphism of the real line `(-inf, inf)`. The inverse transform is - `X = g^{-1}(Y) = Sinh( ArcSinh(Y) / tailweight - skewness )`. - - The `SinhArcsinh` transformation of the Normal is described in - [Sinh-arcsinh distributions](https://www.jstor.org/stable/27798865) - This Bijector allows a similar transformation of any distribution supported on - `(-inf, inf)`. - - #### Meaning of the parameters - - * If `skewness = 0` and `tailweight = 1`, this transform is the identity. - * Positive (negative) `skewness` leads to positive (negative) skew. - * positive skew means, for unimodal `X` centered at zero, the mode of `Y` is - "tilted" to the right. - * positive skew means positive values of `Y` become more likely, and - negative values become less likely. - * Larger (smaller) `tailweight` leads to fatter (thinner) tails. - * Fatter tails mean larger values of `|Y|` become more likely. - * If `X` is a unit Normal, `tailweight < 1` leads to a distribution that is - "flat" around `Y = 0`, and a very steep drop-off in the tails. - * If `X` is a unit Normal, `tailweight > 1` leads to a distribution more - peaked at the mode with heavier tails. - - To see the argument about the tails, note that for `|X| >> 1` and - `|X| >> (|skewness| * tailweight)**tailweight`, we have - `Y approx 0.5 X**tailweight e**(sign(X) skewness * tailweight)`. - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - skewness=None, - tailweight=None, - validate_args=False, - name="SinhArcsinh"): - """Instantiates the `SinhArcsinh` bijector. - - Args: - skewness: Skewness parameter. Float-type `Tensor`. Default is `0` - of type `float32`. - tailweight: Tailweight parameter. Positive `Tensor` of same `dtype` as - `skewness` and broadcastable `shape`. Default is `1` of type `float32`. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - """ - self._graph_parents = [] - self._name = name - self._validate_args = validate_args - with self._name_scope("init", values=[skewness, tailweight]): - tailweight = 1. if tailweight is None else tailweight - skewness = 0. if skewness is None else skewness - self._skewness = ops.convert_to_tensor( - skewness, name="skewness") - self._tailweight = ops.convert_to_tensor( - tailweight, name="tailweight", dtype=self._skewness.dtype) - check_ops.assert_same_float_dtype([self._skewness, self._tailweight]) - if validate_args: - self._tailweight = control_flow_ops.with_dependencies([ - check_ops.assert_positive( - self._tailweight, - message="Argument tailweight was not positive") - ], self._tailweight) - super(SinhArcsinh, self).__init__( - forward_min_event_ndims=0, - validate_args=validate_args, - name=name) - - @property - def skewness(self): - """The `skewness` in: `Y = Sinh((Arcsinh(X) + skewness) * tailweight)`.""" - return self._skewness - - @property - def tailweight(self): - """The `tailweight` in: `Y = Sinh((Arcsinh(X) + skewness) * tailweight)`.""" - return self._tailweight - - def _forward(self, x): - return math_ops.sinh((math_ops.asinh(x) + self.skewness) * self.tailweight) - - def _inverse(self, y): - return math_ops.sinh(math_ops.asinh(y) / self.tailweight - self.skewness) - - def _inverse_log_det_jacobian(self, y): - # x = sinh(arcsinh(y) / tailweight - skewness) - # Using sinh' = cosh, arcsinh'(y) = 1 / sqrt(y**2 + 1), - # dx/dy - # = cosh(arcsinh(y) / tailweight - skewness) - # / (tailweight * sqrt(y**2 + 1)) - - # This is computed inside the log to avoid catastrophic cancellations - # from cosh((arcsinh(y) / tailweight) - skewness) and sqrt(x**2 + 1). - return ( - math_ops.log(math_ops.cosh( - math_ops.asinh(y) / self.tailweight - self.skewness) - # TODO(srvasude): Consider using cosh(arcsinh(x)) in cases - # where (arcsinh(x) / tailweight) - skewness ~= arcsinh(x). - / _sqrtx2p1(y)) - - math_ops.log(self.tailweight)) - - def _forward_log_det_jacobian(self, x): - # y = sinh((arcsinh(x) + skewness) * tailweight) - # Using sinh' = cosh, arcsinh'(x) = 1 / sqrt(x**2 + 1), - # dy/dx - # = cosh((arcsinh(x) + skewness) * tailweight) * tailweight / sqrt(x**2 + 1) - - # This is computed inside the log to avoid catastrophic cancellations - # from cosh((arcsinh(x) + skewness) * tailweight) and sqrt(x**2 + 1). - return ( - math_ops.log(math_ops.cosh( - (math_ops.asinh(x) + self.skewness) * self.tailweight) - # TODO(srvasude): Consider using cosh(arcsinh(x)) in cases - # where (arcsinh(x) + skewness) * tailweight ~= arcsinh(x). - / _sqrtx2p1(x)) - + math_ops.log(self.tailweight)) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py b/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py deleted file mode 100644 index 26d5407125c..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""SoftmaxCentered bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - - -__all__ = [ - "SoftmaxCentered", -] - - -class SoftmaxCentered(bijector.Bijector): - """Bijector which computes `Y = g(X) = exp([X 0]) / sum(exp([X 0]))`. - - To implement [softmax](https://en.wikipedia.org/wiki/Softmax_function) as a - bijection, the forward transformation appends a value to the input and the - inverse removes this coordinate. The appended coordinate represents a pivot, - e.g., `softmax(x) = exp(x-c) / sum(exp(x-c))` where `c` is the implicit last - coordinate. - - Example Use: - - ```python - bijector.SoftmaxCentered().forward(tf.math.log([2, 3, 4])) - # Result: [0.2, 0.3, 0.4, 0.1] - # Extra result: 0.1 - - bijector.SoftmaxCentered().inverse([0.2, 0.3, 0.4, 0.1]) - # Result: tf.math.log([2, 3, 4]) - # Extra coordinate removed. - ``` - - At first blush it may seem like the [Invariance of domain]( - https://en.wikipedia.org/wiki/Invariance_of_domain) theorem implies this - implementation is not a bijection. However, the appended dimension - makes the (forward) image non-open and the theorem does not directly apply. - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - validate_args=False, - name="softmax_centered"): - self._graph_parents = [] - self._name = name - super(SoftmaxCentered, self).__init__( - forward_min_event_ndims=1, - validate_args=validate_args, - name=name) - - def _forward_event_shape(self, input_shape): - if input_shape.ndims is None or input_shape[-1] is None: - return input_shape - return tensor_shape.TensorShape([input_shape[-1] + 1]) - - def _forward_event_shape_tensor(self, input_shape): - return (input_shape[-1] + 1)[..., array_ops.newaxis] - - def _inverse_event_shape(self, output_shape): - if output_shape.ndims is None or output_shape[-1] is None: - return output_shape - if output_shape[-1] <= 1: - raise ValueError("output_shape[-1] = %d <= 1" % output_shape[-1]) - return tensor_shape.TensorShape([output_shape[-1] - 1]) - - def _inverse_event_shape_tensor(self, output_shape): - if self.validate_args: - # It is not possible for a negative shape so we need only check <= 1. - is_greater_one = check_ops.assert_greater( - output_shape[-1], 1, message="Need last dimension greater than 1.") - output_shape = control_flow_ops.with_dependencies( - [is_greater_one], output_shape) - return (output_shape[-1] - 1)[..., array_ops.newaxis] - - def _forward(self, x): - # Pad the last dim with a zeros vector. We need this because it lets us - # infer the scale in the inverse function. - y = distribution_util.pad(x, axis=-1, back=True) - - # Set shape hints. - if x.shape.ndims is not None: - shape = x.shape[:-1].concatenate(x.shape.dims[-1] + 1) - y.shape.assert_is_compatible_with(shape) - y.set_shape(shape) - - return nn_ops.softmax(y) - - def _inverse(self, y): - # To derive the inverse mapping note that: - # y[i] = exp(x[i]) / normalization - # and - # y[end] = 1 / normalization. - # Thus: - # x[i] = log(exp(x[i])) - log(y[end]) - log(normalization) - # = log(exp(x[i])/normalization) - log(y[end]) - # = log(y[i]) - log(y[end]) - - # Do this first to make sure CSE catches that it'll happen again in - # _inverse_log_det_jacobian. - x = math_ops.log(y) - - log_normalization = (-x[..., -1])[..., array_ops.newaxis] - x = x[..., :-1] + log_normalization - - # Set shape hints. - if y.shape.ndims is not None: - shape = y.shape[:-1].concatenate(y.shape.dims[-1] - 1) - x.shape.assert_is_compatible_with(shape) - x.set_shape(shape) - - return x - - def _inverse_log_det_jacobian(self, y): - # WLOG, consider the vector case: - # x = log(y[:-1]) - log(y[-1]) - # where, - # y[-1] = 1 - sum(y[:-1]). - # We have: - # det{ dX/dY } = det{ diag(1 ./ y[:-1]) + 1 / y[-1] } - # = det{ inv{ diag(y[:-1]) - y[:-1]' y[:-1] } } (1) - # = 1 / det{ diag(y[:-1]) - y[:-1]' y[:-1] } - # = 1 / { (1 + y[:-1]' inv(diag(y[:-1])) y[:-1]) * - # det(diag(y[:-1])) } (2) - # = 1 / { y[-1] prod(y[:-1]) } - # = 1 / prod(y) - # (1) - https://en.wikipedia.org/wiki/Sherman%E2%80%93Morrison_formula - # or by noting that det{ dX/dY } = 1 / det{ dY/dX } from Bijector - # docstring "Tip". - # (2) - https://en.wikipedia.org/wiki/Matrix_determinant_lemma - return -math_ops.reduce_sum(math_ops.log(y), axis=-1) - - def _forward_log_det_jacobian(self, x): - # This code is similar to nn_ops.log_softmax but different because we have - # an implicit zero column to handle. I.e., instead of: - # reduce_sum(logits - reduce_sum(exp(logits), dim)) - # we must do: - # log_normalization = 1 + reduce_sum(exp(logits)) - # -log_normalization + reduce_sum(logits - log_normalization) - log_normalization = nn_ops.softplus( - math_ops.reduce_logsumexp(x, axis=-1, keepdims=True)) - return array_ops.squeeze( - (-log_normalization + math_ops.reduce_sum( - x - log_normalization, axis=-1, keepdims=True)), axis=-1) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/softplus.py b/tensorflow/contrib/distributions/python/ops/bijectors/softplus.py deleted file mode 100644 index 3df84ef8b04..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/softplus.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Softplus bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - - -__all__ = [ - "Softplus", -] - - -class Softplus(bijector.Bijector): - """Bijector which computes `Y = g(X) = Log[1 + exp(X)]`. - - The softplus `Bijector` has the following two useful properties: - - * The domain is the positive real numbers - * `softplus(x) approx x`, for large `x`, so it does not overflow as easily as - the `Exp` `Bijector`. - - The optional nonzero `hinge_softness` parameter changes the transition at - zero. With `hinge_softness = c`, the bijector is: - - ```f_c(x) := c * g(x / c) = c * Log[1 + exp(x / c)].``` - - For large `x >> 1`, `c * Log[1 + exp(x / c)] approx c * Log[exp(x / c)] = x`, - so the behavior for large `x` is the same as the standard softplus. - - As `c > 0` approaches 0 from the right, `f_c(x)` becomes less and less soft, - approaching `max(0, x)`. - - * `c = 1` is the default. - * `c > 0` but small means `f(x) approx ReLu(x) = max(0, x)`. - * `c < 0` flips sign and reflects around the `y-axis`: `f_{-c}(x) = -f_c(-x)`. - * `c = 0` results in a non-bijective transformation and triggers an exception. - - Example Use: - - ```python - # Create the Y=g(X)=softplus(X) transform which works only on Tensors with 1 - # batch ndim and 2 event ndims (i.e., vector of matrices). - softplus = Softplus() - x = [[[1., 2], - [3, 4]], - [[5, 6], - [7, 8]]] - log(1 + exp(x)) == softplus.forward(x) - log(exp(x) - 1) == softplus.inverse(x) - ``` - - Note: log(.) and exp(.) are applied element-wise but the Jacobian is a - reduction over the event space. - """ - - @distribution_util.AppendDocstring( - kwargs_dict={ - "hinge_softness": ( - "Nonzero floating point `Tensor`. Controls the softness of what " - "would otherwise be a kink at the origin. Default is 1.0")}) - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - hinge_softness=None, - validate_args=False, - name="softplus"): - with ops.name_scope(name, values=[hinge_softness]): - if hinge_softness is not None: - self._hinge_softness = ops.convert_to_tensor( - hinge_softness, name="hinge_softness") - else: - self._hinge_softness = None - if validate_args: - nonzero_check = check_ops.assert_none_equal( - ops.convert_to_tensor( - 0, dtype=self.hinge_softness.dtype), - self.hinge_softness, - message="hinge_softness must be non-zero") - self._hinge_softness = control_flow_ops.with_dependencies( - [nonzero_check], self.hinge_softness) - - super(Softplus, self).__init__( - forward_min_event_ndims=0, - validate_args=validate_args, - name=name) - - def _forward(self, x): - if self.hinge_softness is None: - return nn_ops.softplus(x) - hinge_softness = math_ops.cast(self.hinge_softness, x.dtype) - return hinge_softness * nn_ops.softplus(x / hinge_softness) - - def _inverse(self, y): - if self.hinge_softness is None: - return distribution_util.softplus_inverse(y) - hinge_softness = math_ops.cast(self.hinge_softness, y.dtype) - return hinge_softness * distribution_util.softplus_inverse( - y / hinge_softness) - - def _inverse_log_det_jacobian(self, y): - # Could also do: - # ildj = math_ops.reduce_sum(y - distribution_util.softplus_inverse(y), - # axis=event_dims) - # but the following is more numerically stable. Ie, - # Y = Log[1 + exp{X}] ==> X = Log[exp{Y} - 1] - # ==> dX/dY = exp{Y} / (exp{Y} - 1) - # = 1 / (1 - exp{-Y}), - # which is the most stable for large Y > 0. For small Y, we use - # 1 - exp{-Y} approx Y. - if self.hinge_softness is not None: - y /= math_ops.cast(self.hinge_softness, y.dtype) - return -math_ops.log(-math_ops.expm1(-y)) - - def _forward_log_det_jacobian(self, x): - if self.hinge_softness is not None: - x /= math_ops.cast(self.hinge_softness, x.dtype) - return -nn_ops.softplus(-x) - - @property - def hinge_softness(self): - return self._hinge_softness diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/softsign.py b/tensorflow/contrib/distributions/python/ops/bijectors/softsign.py deleted file mode 100644 index f96a4bb01de..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/softsign.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Softsign bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - - -__all__ = [ - "Softsign", -] - - -class Softsign(bijector.Bijector): - """Bijector which computes `Y = g(X) = X / (1 + |X|)`. - - The softsign `Bijector` has the following two useful properties: - - * The domain is all real numbers - * `softsign(x) approx sgn(x)`, for large `|x|`. - - #### Examples - - ```python - # Create the Y = softsign(X) transform. - softsign = Softsign() - x = [[[1., 2], - [3, 4]], - [[5, 6], - [7, 8]]] - x / (1 + abs(x)) == softsign.forward(x) - x / (1 - abs(x)) == softsign.inverse(x) - ``` - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, validate_args=False, name="softsign"): - super(Softsign, self).__init__( - forward_min_event_ndims=0, - validate_args=validate_args, - name=name) - - def _forward(self, x): - return x / (1. + math_ops.abs(x)) - - def _inverse(self, y): - y = self._maybe_assert_valid_y(y) - return y / (1. - math_ops.abs(y)) - - def _forward_log_det_jacobian(self, x): - return -2. * math_ops.log1p(math_ops.abs(x)) - - def _inverse_log_det_jacobian(self, y): - y = self._maybe_assert_valid_y(y) - return -2. * math_ops.log1p(-math_ops.abs(y)) - - def _maybe_assert_valid_y(self, y): - if not self.validate_args: - return y - is_valid = [ - check_ops.assert_greater( - y, math_ops.cast(-1., dtype=y.dtype.base_dtype), - message="Inverse transformation input must be greater than -1."), - check_ops.assert_less( - y, math_ops.cast(1., dtype=y.dtype.base_dtype), - message="Inverse transformation input must be less than 1.") - ] - - return control_flow_ops.with_dependencies(is_valid, y) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/square.py b/tensorflow/contrib/distributions/python/ops/bijectors/square.py deleted file mode 100644 index 294460a80f6..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/square.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Square bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - - -__all__ = [ - "Square", -] - - -class Square(bijector.Bijector): - """Compute `g(X) = X^2`; X is a positive real number. - - g is a bijection between the non-negative real numbers (R_+) and the - non-negative real numbers. - - #### Examples - - ```python - bijector.Square().forward(x=[[1., 0], [2, 1]]) - # Result: [[1., 0], [4, 1]], i.e., x^2 - - bijector.Square().inverse(y=[[1., 4], [9, 1]]) - # Result: [[1., 2], [3, 1]], i.e., sqrt(y). - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, validate_args=False, name="square"): - """Instantiates the `Square` bijector. - - Args: - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - """ - self._name = name - super(Square, self).__init__( - forward_min_event_ndims=0, - validate_args=validate_args, - name=name) - - def _forward(self, x): - x = self._maybe_assert_valid(x) - return math_ops.square(x) - - def _inverse(self, y): - y = self._maybe_assert_valid(y) - return math_ops.sqrt(y) - - def _forward_log_det_jacobian(self, x): - x = self._maybe_assert_valid(x) - return np.log(2.) + math_ops.log(x) - - def _maybe_assert_valid(self, t): - if not self.validate_args: - return t - is_valid = check_ops.assert_non_negative( - t, message="All elements must be non-negative.") - return control_flow_ops.with_dependencies([is_valid], t) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/transform_diagonal.py b/tensorflow/contrib/distributions/python/ops/bijectors/transform_diagonal.py deleted file mode 100644 index 9b7a3b026b8..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/transform_diagonal.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""TransformDiagonal bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import array_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - -__all__ = [ - "TransformDiagonal", -] - - -class TransformDiagonal(bijector.Bijector): - """Applies a Bijector to the diagonal of a matrix. - - #### Example - - ```python - b = tfb.TransformDiagonal(diag_bijector=tfb.Exp()) - - b.forward([[1., 0.], - [0., 1.]]) - # ==> [[2.718, 0.], - [0., 2.718]] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - diag_bijector, - validate_args=False, - name="transform_diagonal"): - """Instantiates the `TransformDiagonal` bijector. - - Args: - diag_bijector: `Bijector` instance used to transform the diagonal. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - """ - self._diag_bijector = diag_bijector - super(TransformDiagonal, self).__init__( - forward_min_event_ndims=2, - inverse_min_event_ndims=2, - validate_args=validate_args, - name=name) - - def _forward(self, x): - diag = self._diag_bijector.forward(array_ops.matrix_diag_part(x)) - return array_ops.matrix_set_diag(x, diag) - - def _inverse(self, y): - diag = self._diag_bijector.inverse(array_ops.matrix_diag_part(y)) - return array_ops.matrix_set_diag(y, diag) - - def _forward_log_det_jacobian(self, x): - # We formulate the Jacobian with respect to the flattened matrices - # `vec(x)` and `vec(y)`. Suppose for notational convenience that - # the first `n` entries of `vec(x)` are the diagonal of `x`, and - # the remaining `n**2-n` entries are the off-diagonals in - # arbitrary order. Then the Jacobian is a block-diagonal matrix, - # with the Jacobian of the diagonal bijector in the first block, - # and the identity Jacobian for the remaining entries (since this - # bijector acts as the identity on non-diagonal entries): - # - # J_vec(x) (vec(y)) = - # ------------------------------- - # | J_diag(x) (diag(y)) 0 | n entries - # | | - # | 0 I | n**2-n entries - # ------------------------------- - # n n**2-n - # - # Since the log-det of the second (identity) block is zero, the - # overall log-det-jacobian is just the log-det of first block, - # from the diagonal bijector. - # - # Note that for elementwise operations (exp, softplus, etc) the - # first block of the Jacobian will itself be a diagonal matrix, - # but our implementation does not require this to be true. - return self._diag_bijector.forward_log_det_jacobian( - array_ops.matrix_diag_part(x), event_ndims=1) - - def _inverse_log_det_jacobian(self, y): - return self._diag_bijector.inverse_log_det_jacobian( - array_ops.matrix_diag_part(y), event_ndims=1) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/weibull.py b/tensorflow/contrib/distributions/python/ops/bijectors/weibull.py deleted file mode 100644 index 8903a70d98a..00000000000 --- a/tensorflow/contrib/distributions/python/ops/bijectors/weibull.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Weibull bijector.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util import deprecation - - -__all__ = [ - "Weibull", -] - - -class Weibull(bijector.Bijector): - """Compute `Y = g(X) = 1 - exp((-X / scale) ** concentration), X >= 0`. - - This bijector maps inputs from `[0, inf]` to [0, 1]`. The inverse of the - bijector applied to a uniform random variable `X ~ U(0, 1) gives back a - random variable with the - [Weibull distribution](https://en.wikipedia.org/wiki/Weibull_distribution): - - ```none - Y ~ Weibull(scale, concentration) - pdf(y; scale, concentration, y >= 0) = (scale / concentration) * ( - scale / concentration) ** (concentration - 1) * exp( - -(y / scale) ** concentration) - ``` - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - scale=1., - concentration=1., - validate_args=False, - name="weibull"): - """Instantiates the `Weibull` bijector. - - Args: - scale: Positive Float-type `Tensor` that is the same dtype and is - broadcastable with `concentration`. - This is `l` in `Y = g(X) = 1 - exp((-x / l) ** k)`. - concentration: Positive Float-type `Tensor` that is the same dtype and is - broadcastable with `scale`. - This is `k` in `Y = g(X) = 1 - exp((-x / l) ** k)`. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - name: Python `str` name given to ops managed by this object. - """ - self._graph_parents = [] - self._name = name - self._validate_args = validate_args - with self._name_scope("init", values=[scale, concentration]): - self._scale = ops.convert_to_tensor(scale, name="scale") - self._concentration = ops.convert_to_tensor( - concentration, name="concentration") - check_ops.assert_same_float_dtype([self._scale, self._concentration]) - if validate_args: - self._scale = control_flow_ops.with_dependencies([ - check_ops.assert_positive( - self._scale, - message="Argument scale was not positive") - ], self._scale) - self._concentration = control_flow_ops.with_dependencies([ - check_ops.assert_positive( - self._concentration, - message="Argument concentration was not positive") - ], self._concentration) - - super(Weibull, self).__init__( - forward_min_event_ndims=0, - validate_args=validate_args, - name=name) - - @property - def scale(self): - """The `l` in `Y = g(X) = 1 - exp((-x / l) ** k)`.""" - return self._scale - - @property - def concentration(self): - """The `k` in `Y = g(X) = 1 - exp((-x / l) ** k)`.""" - return self._concentration - - def _forward(self, x): - x = self._maybe_assert_valid_x(x) - return -math_ops.expm1(-((x / self.scale) ** self.concentration)) - - def _inverse(self, y): - y = self._maybe_assert_valid_y(y) - return self.scale * (-math_ops.log1p(-y)) ** (1 / self.concentration) - - def _inverse_log_det_jacobian(self, y): - y = self._maybe_assert_valid_y(y) - return ( - -math_ops.log1p(-y) + - (1 / self.concentration - 1) * math_ops.log(-math_ops.log1p(-y)) + - math_ops.log(self.scale / self.concentration)) - - def _forward_log_det_jacobian(self, x): - x = self._maybe_assert_valid_x(x) - return ( - -(x / self.scale) ** self.concentration + - (self.concentration - 1) * math_ops.log(x) + - math_ops.log(self.concentration) + - -self.concentration * math_ops.log(self.scale)) - - def _maybe_assert_valid_x(self, x): - if not self.validate_args: - return x - is_valid = check_ops.assert_non_negative( - x, - message="Forward transformation input must be at least 0.") - return control_flow_ops.with_dependencies([is_valid], x) - - def _maybe_assert_valid_y(self, y): - if not self.validate_args: - return y - is_positive = check_ops.assert_non_negative( - y, message="Inverse transformation input must be greater than 0.") - less_than_one = check_ops.assert_less_equal( - y, constant_op.constant(1., y.dtype), - message="Inverse transformation input must be less than or equal to 1.") - return control_flow_ops.with_dependencies([is_positive, less_than_one], y) diff --git a/tensorflow/contrib/distributions/python/ops/binomial.py b/tensorflow/contrib/distributions/python/ops/binomial.py deleted file mode 100644 index 38505c172f6..00000000000 --- a/tensorflow/contrib/distributions/python/ops/binomial.py +++ /dev/null @@ -1,301 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Binomial distribution class.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - - -_binomial_sample_note = """ -For each batch member of counts `value`, `P[value]` is the probability that -after sampling `self.total_count` draws from this Binomial distribution, the -number of successes is `value`. Since different sequences of draws can result in -the same counts, the probability includes a combinatorial coefficient. - -Note: `value` must be a non-negative tensor with dtype `dtype` and whose shape -can be broadcast with `self.probs` and `self.total_count`. `value` is only legal -if it is less than or equal to `self.total_count` and its components are equal -to integer values. -""" - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _bdtr(k, n, p): - """The binomial cumulative distribution function. - - Args: - k: floating point `Tensor`. - n: floating point `Tensor`. - p: floating point `Tensor`. - - Returns: - `sum_{j=0}^k p^j (1 - p)^(n - j)`. - """ - # Trick for getting safe backprop/gradients into n, k when - # betainc(a = 0, ..) = nan - # Write: - # where(unsafe, safe_output, betainc(where(unsafe, safe_input, input))) - ones = array_ops.ones_like(n - k) - k_eq_n = math_ops.equal(k, n) - safe_dn = array_ops.where_v2(k_eq_n, ones, n - k) - dk = math_ops.betainc(a=safe_dn, b=k + 1, x=1 - p) - return array_ops.where_v2(k_eq_n, ones, dk) - - -class Binomial(distribution.Distribution): - """Binomial distribution. - - This distribution is parameterized by `probs`, a (batch of) probabilities for - drawing a `1` and `total_count`, the number of trials per draw from the - Binomial. - - #### Mathematical Details - - The Binomial is a distribution over the number of `1`'s in `total_count` - independent trials, with each trial having the same probability of `1`, i.e., - `probs`. - - The probability mass function (pmf) is, - - ```none - pmf(k; n, p) = p**k (1 - p)**(n - k) / Z - Z = k! (n - k)! / n! - ``` - - where: - * `total_count = n`, - * `probs = p`, - * `Z` is the normalizing constant, and, - * `n!` is the factorial of `n`. - - #### Examples - - Create a single distribution, corresponding to 5 coin flips. - - ```python - dist = Binomial(total_count=5., probs=.5) - ``` - - Create a single distribution (using logits), corresponding to 5 coin flips. - - ```python - dist = Binomial(total_count=5., logits=0.) - ``` - - Creates 3 distributions with the third distribution most likely to have - successes. - - ```python - p = [.2, .3, .8] - # n will be broadcast to [4., 4., 4.], to match p. - dist = Binomial(total_count=4., probs=p) - ``` - - The distribution functions can be evaluated on counts. - - ```python - # counts same shape as p. - counts = [1., 2, 3] - dist.prob(counts) # Shape [3] - - # p will be broadcast to [[.2, .3, .8], [.2, .3, .8]] to match counts. - counts = [[1., 2, 1], [2, 2, 4]] - dist.prob(counts) # Shape [2, 3] - - # p will be broadcast to shape [5, 7, 3] to match counts. - counts = [[...]] # Shape [5, 7, 3] - dist.prob(counts) # Shape [5, 7, 3] - ``` - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - total_count, - logits=None, - probs=None, - validate_args=False, - allow_nan_stats=True, - name="Binomial"): - """Initialize a batch of Binomial distributions. - - Args: - total_count: Non-negative floating point tensor with shape broadcastable - to `[N1,..., Nm]` with `m >= 0` and the same dtype as `probs` or - `logits`. Defines this as a batch of `N1 x ... x Nm` different Binomial - distributions. Its components should be equal to integer values. - logits: Floating point tensor representing the log-odds of a - positive event with shape broadcastable to `[N1,..., Nm]` `m >= 0`, and - the same dtype as `total_count`. Each entry represents logits for the - probability of success for independent Binomial distributions. Only one - of `logits` or `probs` should be passed in. - probs: Positive floating point tensor with shape broadcastable to - `[N1,..., Nm]` `m >= 0`, `probs in [0, 1]`. Each entry represents the - probability of success for independent Binomial distributions. Only one - of `logits` or `probs` should be passed in. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[total_count, logits, probs]) as name: - self._total_count = self._maybe_assert_valid_total_count( - ops.convert_to_tensor(total_count, name="total_count"), - validate_args) - self._logits, self._probs = distribution_util.get_logits_and_probs( - logits=logits, - probs=probs, - validate_args=validate_args, - name=name) - super(Binomial, self).__init__( - dtype=self._probs.dtype, - reparameterization_type=distribution.NOT_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=[self._total_count, - self._logits, - self._probs], - name=name) - - @property - def total_count(self): - """Number of trials.""" - return self._total_count - - @property - def logits(self): - """Log-odds of drawing a `1`.""" - return self._logits - - @property - def probs(self): - """Probability of drawing a `1`.""" - return self._probs - - def _batch_shape_tensor(self): - return array_ops.broadcast_dynamic_shape( - array_ops.shape(self.total_count), - array_ops.shape(self.probs)) - - def _batch_shape(self): - return array_ops.broadcast_static_shape( - self.total_count.get_shape(), - self.probs.get_shape()) - - def _event_shape_tensor(self): - return constant_op.constant([], dtype=dtypes.int32) - - def _event_shape(self): - return tensor_shape.TensorShape([]) - - @distribution_util.AppendDocstring(_binomial_sample_note) - def _log_prob(self, counts): - return self._log_unnormalized_prob(counts) - self._log_normalization(counts) - - @distribution_util.AppendDocstring(_binomial_sample_note) - def _prob(self, counts): - return math_ops.exp(self._log_prob(counts)) - - def _cdf(self, counts): - counts = self._maybe_assert_valid_sample(counts) - probs = self.probs - if not (counts.shape.is_fully_defined() - and self.probs.shape.is_fully_defined() - and counts.shape.is_compatible_with(self.probs.shape)): - # If both shapes are well defined and equal, we skip broadcasting. - probs += array_ops.zeros_like(counts) - counts += array_ops.zeros_like(self.probs) - - return _bdtr(k=counts, n=self.total_count, p=probs) - - def _log_unnormalized_prob(self, counts): - counts = self._maybe_assert_valid_sample(counts) - return (counts * math_ops.log(self.probs) + - (self.total_count - counts) * math_ops.log1p(-self.probs)) - - def _log_normalization(self, counts): - counts = self._maybe_assert_valid_sample(counts) - return (math_ops.lgamma(1. + self.total_count - counts) - + math_ops.lgamma(1. + counts) - - math_ops.lgamma(1. + self.total_count)) - - def _mean(self): - return self.total_count * self.probs - - def _variance(self): - return self._mean() * (1. - self.probs) - - @distribution_util.AppendDocstring( - """Note that when `(1 + total_count) * probs` is an integer, there are - actually two modes. Namely, `(1 + total_count) * probs` and - `(1 + total_count) * probs - 1` are both modes. Here we return only the - larger of the two modes.""") - def _mode(self): - return math_ops.floor((1. + self.total_count) * self.probs) - - def _maybe_assert_valid_total_count(self, total_count, validate_args): - if not validate_args: - return total_count - return control_flow_ops.with_dependencies([ - check_ops.assert_non_negative( - total_count, - message="total_count must be non-negative."), - distribution_util.assert_integer_form( - total_count, - message="total_count cannot contain fractional components."), - ], total_count) - - def _maybe_assert_valid_sample(self, counts): - """Check counts for proper shape, values, then return tensor version.""" - if not self.validate_args: - return counts - counts = distribution_util.embed_check_nonnegative_integer_form(counts) - return control_flow_ops.with_dependencies([ - check_ops.assert_less_equal( - counts, self.total_count, - message="counts are not less than or equal to n."), - ], counts) diff --git a/tensorflow/contrib/distributions/python/ops/cauchy.py b/tensorflow/contrib/distributions/python/ops/cauchy.py deleted file mode 100644 index 6b1a022a312..00000000000 --- a/tensorflow/contrib/distributions/python/ops/cauchy.py +++ /dev/null @@ -1,231 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Cauchy distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.util import deprecation - -__all__ = [ - "Cauchy", -] - - -class Cauchy(distribution.Distribution): - """The Cauchy distribution with location `loc` and scale `scale`. - - #### Mathematical details - - The probability density function (pdf) is, - - ```none - pdf(x; loc, scale) = 1 / (pi scale (1 + z**2)) - z = (x - loc) / scale - ``` - where `loc` is the location, and `scale` is the scale. - - The Cauchy distribution is a member of the [location-scale family]( - https://en.wikipedia.org/wiki/Location-scale_family), i.e. - `Y ~ Cauchy(loc, scale)` is equivalent to, - - ```none - X ~ Cauchy(loc=0, scale=1) - Y = loc + scale * X - ``` - - #### Examples - - Examples of initialization of one or a batch of distributions. - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Define a single scalar Cauchy distribution. - dist = tfd.Cauchy(loc=0., scale=3.) - - # Evaluate the cdf at 1, returning a scalar. - dist.cdf(1.) - - # Define a batch of two scalar valued Cauchy distributions. - dist = tfd.Cauchy(loc=[1, 2.], scale=[11, 22.]) - - # Evaluate the pdf of the first distribution on 0, and the second on 1.5, - # returning a length two tensor. - dist.prob([0, 1.5]) - - # Get 3 samples, returning a 3 x 2 tensor. - dist.sample([3]) - - # Arguments are broadcast when possible. - # Define a batch of two scalar valued Cauchy distributions. - # Both have median 1, but different scales. - dist = tfd.Cauchy(loc=1., scale=[11, 22.]) - - # Evaluate the pdf of both distributions on the same point, 3.0, - # returning a length 2 tensor. - dist.prob(3.) - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc, - scale, - validate_args=False, - allow_nan_stats=True, - name="Cauchy"): - """Construct Cauchy distributions. - - The parameters `loc` and `scale` must be shaped in a way that supports - broadcasting (e.g. `loc + scale` is a valid operation). - - Args: - loc: Floating point tensor; the modes of the distribution(s). - scale: Floating point tensor; the locations of the distribution(s). - Must contain only positive values. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, - statistics (e.g., mean, mode, variance) use the value "`NaN`" to - indicate the result is undefined. When `False`, an exception is raised - if one or more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - TypeError: if `loc` and `scale` have different `dtype`. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[loc, scale]) as name: - with ops.control_dependencies([check_ops.assert_positive(scale)] - if validate_args else []): - self._loc = array_ops.identity(loc, name="loc") - self._scale = array_ops.identity(scale, name="scale") - check_ops.assert_same_float_dtype([self._loc, self._scale]) - super(Cauchy, self).__init__( - dtype=self._scale.dtype, - reparameterization_type=distribution.FULLY_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=[self._loc, self._scale], - name=name) - - @staticmethod - def _param_shapes(sample_shape): - return dict( - zip(("loc", "scale"), - ([ops.convert_to_tensor(sample_shape, dtype=dtypes.int32)] * 2))) - - @property - def loc(self): - """Distribution parameter for the mean.""" - return self._loc - - @property - def scale(self): - """Distribution parameter for standard deviation.""" - return self._scale - - def _batch_shape_tensor(self): - return array_ops.broadcast_dynamic_shape( - array_ops.shape(self.loc), array_ops.shape(self.scale)) - - def _batch_shape(self): - return array_ops.broadcast_static_shape(self.loc.shape, self.scale.shape) - - def _event_shape_tensor(self): - return constant_op.constant([], dtype=dtypes.int32) - - def _event_shape(self): - return tensor_shape.TensorShape([]) - - def _sample_n(self, n, seed=None): - shape = array_ops.concat([[n], self.batch_shape_tensor()], 0) - probs = random_ops.random_uniform( - shape=shape, minval=0., maxval=1., dtype=self.dtype, seed=seed) - return self._quantile(probs) - - def _log_prob(self, x): - return self._log_unnormalized_prob(x) - self._log_normalization() - - def _cdf(self, x): - return math_ops.atan(self._z(x)) / np.pi + 0.5 - - def _log_cdf(self, x): - return math_ops.log1p(2 / np.pi * math_ops.atan(self._z(x))) - np.log(2) - - def _log_unnormalized_prob(self, x): - return -math_ops.log1p(math_ops.square(self._z(x))) - - def _log_normalization(self): - return np.log(np.pi) + math_ops.log(self.scale) - - def _entropy(self): - h = np.log(4 * np.pi) + math_ops.log(self.scale) - return h * array_ops.ones_like(self.loc) - - def _quantile(self, p): - return self.loc + self.scale * math_ops.tan(np.pi * (p - 0.5)) - - def _mode(self): - return self.loc * array_ops.ones_like(self.scale) - - def _z(self, x): - """Standardize input `x`.""" - with ops.name_scope("standardize", values=[x]): - return (x - self.loc) / self.scale - - def _inv_z(self, z): - """Reconstruct input `x` from a its normalized version.""" - with ops.name_scope("reconstruct", values=[z]): - return z * self.scale + self.loc - - def _mean(self): - if self.allow_nan_stats: - return array_ops.fill(self.batch_shape_tensor(), - self.dtype.as_numpy_dtype(np.nan)) - else: - raise ValueError("`mean` is undefined for Cauchy distribution.") - - def _stddev(self): - if self.allow_nan_stats: - return array_ops.fill(self.batch_shape_tensor(), - self.dtype.as_numpy_dtype(np.nan)) - else: - raise ValueError("`stddev` is undefined for Cauchy distribution.") diff --git a/tensorflow/contrib/distributions/python/ops/chi2.py b/tensorflow/contrib/distributions/python/ops/chi2.py deleted file mode 100644 index e9a7b39070f..00000000000 --- a/tensorflow/contrib/distributions/python/ops/chi2.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Chi2 distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import gamma -from tensorflow.python.util import deprecation - - -__all__ = [ - "Chi2", - "Chi2WithAbsDf", -] - - -class Chi2(gamma.Gamma): - """Chi2 distribution. - - The Chi2 distribution is defined over positive real numbers using a degrees of - freedom ("df") parameter. - - #### Mathematical Details - - The probability density function (pdf) is, - - ```none - pdf(x; df, x > 0) = x**(0.5 df - 1) exp(-0.5 x) / Z - Z = 2**(0.5 df) Gamma(0.5 df) - ``` - - where: - - * `df` denotes the degrees of freedom, - * `Z` is the normalization constant, and, - * `Gamma` is the [gamma function]( - https://en.wikipedia.org/wiki/Gamma_function). - - The Chi2 distribution is a special case of the Gamma distribution, i.e., - - ```python - Chi2(df) = Gamma(concentration=0.5 * df, rate=0.5) - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - df, - validate_args=False, - allow_nan_stats=True, - name="Chi2"): - """Construct Chi2 distributions with parameter `df`. - - Args: - df: Floating point tensor, the degrees of freedom of the - distribution(s). `df` must contain only positive values. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - parameters = dict(locals()) - # Even though all stats of chi2 are defined for valid parameters, this is - # not true in the parent class "gamma." therefore, passing - # allow_nan_stats=True - # through to the parent class results in unnecessary asserts. - with ops.name_scope(name, values=[df]) as name: - with ops.control_dependencies([ - check_ops.assert_positive(df), - ] if validate_args else []): - self._df = array_ops.identity(df, name="df") - - super(Chi2, self).__init__( - concentration=0.5 * self._df, - rate=constant_op.constant(0.5, dtype=self._df.dtype), - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - self._parameters = parameters - - @staticmethod - def _param_shapes(sample_shape): - return {"df": ops.convert_to_tensor(sample_shape, dtype=dtypes.int32)} - - @property - def df(self): - return self._df - - -class Chi2WithAbsDf(Chi2): - """Chi2 with parameter transform `df = floor(abs(df))`.""" - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - df, - validate_args=False, - allow_nan_stats=True, - name="Chi2WithAbsDf"): - parameters = dict(locals()) - with ops.name_scope(name, values=[df]) as name: - super(Chi2WithAbsDf, self).__init__( - df=math_ops.floor( - math_ops.abs(df, name="abs_df"), - name="floor_abs_df"), - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - self._parameters = parameters diff --git a/tensorflow/contrib/distributions/python/ops/conditional_distribution.py b/tensorflow/contrib/distributions/python/ops/conditional_distribution.py deleted file mode 100644 index ef25d4aedd6..00000000000 --- a/tensorflow/contrib/distributions/python/ops/conditional_distribution.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Conditional distribution base class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.ops.distributions import util as distribution_util - - -class ConditionalDistribution(distribution.Distribution): - """Distribution that supports intrinsic parameters (local latents). - - Subclasses of this distribution may have additional keyword arguments passed - to their sample-based methods (i.e. `sample`, `log_prob`, etc.). - """ - - @distribution_util.AppendDocstring(kwargs_dict={ - "**condition_kwargs": - "Named arguments forwarded to subclass implementation."}) - def sample(self, sample_shape=(), seed=None, name="sample", - **condition_kwargs): - return self._call_sample_n(sample_shape, seed, name, **condition_kwargs) - - @distribution_util.AppendDocstring(kwargs_dict={ - "**condition_kwargs": - "Named arguments forwarded to subclass implementation."}) - def log_prob(self, value, name="log_prob", **condition_kwargs): - return self._call_log_prob(value, name, **condition_kwargs) - - @distribution_util.AppendDocstring(kwargs_dict={ - "**condition_kwargs": - "Named arguments forwarded to subclass implementation."}) - def prob(self, value, name="prob", **condition_kwargs): - return self._call_prob(value, name, **condition_kwargs) - - @distribution_util.AppendDocstring(kwargs_dict={ - "**condition_kwargs": - "Named arguments forwarded to subclass implementation."}) - def log_cdf(self, value, name="log_cdf", **condition_kwargs): - return self._call_log_cdf(value, name, **condition_kwargs) - - @distribution_util.AppendDocstring(kwargs_dict={ - "**condition_kwargs": - "Named arguments forwarded to subclass implementation."}) - def cdf(self, value, name="cdf", **condition_kwargs): - return self._call_cdf(value, name, **condition_kwargs) - - @distribution_util.AppendDocstring(kwargs_dict={ - "**condition_kwargs": - "Named arguments forwarded to subclass implementation."}) - def log_survival_function(self, value, name="log_survival_function", - **condition_kwargs): - return self._call_log_survival_function(value, name, **condition_kwargs) - - @distribution_util.AppendDocstring(kwargs_dict={ - "**condition_kwargs": - "Named arguments forwarded to subclass implementation."}) - def survival_function(self, value, name="survival_function", - **condition_kwargs): - return self._call_survival_function(value, name, **condition_kwargs) diff --git a/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py b/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py deleted file mode 100644 index 3598c8d23ea..00000000000 --- a/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A Conditional Transformed Distribution class.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import conditional_distribution -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import transformed_distribution -from tensorflow.python.ops.distributions import util as distribution_util - - -# pylint: disable=protected-access -_concat_vectors = transformed_distribution._concat_vectors -# pylint: enable=protected-access - - -__all__ = [ - "ConditionalTransformedDistribution", -] - - -_condition_kwargs_dict = { - "bijector_kwargs": ("Python dictionary of arg names/values " - "forwarded to the bijector."), - "distribution_kwargs": ("Python dictionary of arg names/values " - "forwarded to the distribution."), -} - - -class ConditionalTransformedDistribution( - conditional_distribution.ConditionalDistribution, - transformed_distribution.TransformedDistribution): - """A TransformedDistribution that allows intrinsic conditioning.""" - - @distribution_util.AppendDocstring(kwargs_dict=_condition_kwargs_dict) - def _sample_n(self, n, seed=None, - bijector_kwargs=None, - distribution_kwargs=None): - sample_shape = _concat_vectors( - distribution_util.pick_vector(self._needs_rotation, self._empty, [n]), - self._override_batch_shape, - self._override_event_shape, - distribution_util.pick_vector(self._needs_rotation, [n], self._empty)) - distribution_kwargs = distribution_kwargs or {} - x = self.distribution.sample(sample_shape=sample_shape, - seed=seed, - **distribution_kwargs) - x = self._maybe_rotate_dims(x) - # We'll apply the bijector in the `_call_sample_n` function. - return x - - def _call_sample_n(self, sample_shape, seed, name, - bijector_kwargs=None, - distribution_kwargs=None): - # We override `_call_sample_n` rather than `_sample_n` so we can ensure that - # the result of `self.bijector.forward` is not modified (and thus caching - # works). - with self._name_scope(name, values=[sample_shape]): - sample_shape = ops.convert_to_tensor( - sample_shape, dtype=dtypes.int32, name="sample_shape") - sample_shape, n = self._expand_sample_shape_to_vector( - sample_shape, "sample_shape") - - # First, generate samples. We will possibly generate extra samples in the - # event that we need to reinterpret the samples as part of the - # event_shape. - x = self._sample_n(n, seed, bijector_kwargs, distribution_kwargs) - - # Next, we reshape `x` into its final form. We do this prior to the call - # to the bijector to ensure that the bijector caching works. - batch_event_shape = array_ops.shape(x)[1:] - final_shape = array_ops.concat([sample_shape, batch_event_shape], 0) - x = array_ops.reshape(x, final_shape) - - # Finally, we apply the bijector's forward transformation. For caching to - # work, it is imperative that this is the last modification to the - # returned result. - bijector_kwargs = bijector_kwargs or {} - y = self.bijector.forward(x, **bijector_kwargs) - y = self._set_sample_static_shape(y, sample_shape) - - return y - - @distribution_util.AppendDocstring(kwargs_dict=_condition_kwargs_dict) - def _log_prob(self, y, bijector_kwargs=None, distribution_kwargs=None): - # For caching to work, it is imperative that the bijector is the first to - # modify the input. - bijector_kwargs = bijector_kwargs or {} - distribution_kwargs = distribution_kwargs or {} - x = self.bijector.inverse(y, **bijector_kwargs) - event_ndims = self._maybe_get_static_event_ndims() - ildj = self.bijector.inverse_log_det_jacobian( - y, event_ndims=event_ndims, **bijector_kwargs) - if self.bijector._is_injective: # pylint: disable=protected-access - return self._finish_log_prob_for_one_fiber(y, x, ildj, - distribution_kwargs) - - lp_on_fibers = [ - self._finish_log_prob_for_one_fiber(y, x_i, ildj_i, distribution_kwargs) - for x_i, ildj_i in zip(x, ildj)] - return math_ops.reduce_logsumexp(array_ops.stack(lp_on_fibers), axis=0) - - def _finish_log_prob_for_one_fiber(self, y, x, ildj, distribution_kwargs): - """Finish computation of log_prob on one element of the inverse image.""" - x = self._maybe_rotate_dims(x, rotate_right=True) - log_prob = self.distribution.log_prob(x, **distribution_kwargs) - if self._is_maybe_event_override: - log_prob = math_ops.reduce_sum(log_prob, self._reduce_event_indices) - return math_ops.cast(ildj, log_prob.dtype) + log_prob - - @distribution_util.AppendDocstring(kwargs_dict=_condition_kwargs_dict) - def _prob(self, y, bijector_kwargs=None, distribution_kwargs=None): - bijector_kwargs = bijector_kwargs or {} - distribution_kwargs = distribution_kwargs or {} - x = self.bijector.inverse(y, **bijector_kwargs) - event_ndims = self._maybe_get_static_event_ndims() - ildj = self.bijector.inverse_log_det_jacobian( - y, event_ndims=event_ndims, **bijector_kwargs) - if self.bijector._is_injective: # pylint: disable=protected-access - return self._finish_prob_for_one_fiber(y, x, ildj, distribution_kwargs) - - prob_on_fibers = [ - self._finish_prob_for_one_fiber(y, x_i, ildj_i, distribution_kwargs) - for x_i, ildj_i in zip(x, ildj)] - return sum(prob_on_fibers) - - def _finish_prob_for_one_fiber(self, y, x, ildj, distribution_kwargs): - """Finish computation of prob on one element of the inverse image.""" - x = self._maybe_rotate_dims(x, rotate_right=True) - prob = self.distribution.prob(x, **distribution_kwargs) - if self._is_maybe_event_override: - prob = math_ops.reduce_prod(prob, self._reduce_event_indices) - return math_ops.exp(math_ops.cast(ildj, prob.dtype)) * prob - - @distribution_util.AppendDocstring(kwargs_dict=_condition_kwargs_dict) - def _log_cdf(self, y, bijector_kwargs=None, distribution_kwargs=None): - if self._is_maybe_event_override: - raise NotImplementedError("log_cdf is not implemented when overriding " - "event_shape") - if not self.bijector._is_injective: # pylint: disable=protected-access - raise NotImplementedError("log_cdf is not implemented when " - "bijector is not injective.") - bijector_kwargs = bijector_kwargs or {} - distribution_kwargs = distribution_kwargs or {} - x = self.bijector.inverse(y, **bijector_kwargs) - return self.distribution.log_cdf(x, **distribution_kwargs) - - @distribution_util.AppendDocstring(kwargs_dict=_condition_kwargs_dict) - def _cdf(self, y, bijector_kwargs=None, distribution_kwargs=None): - if self._is_maybe_event_override: - raise NotImplementedError("cdf is not implemented when overriding " - "event_shape") - if not self.bijector._is_injective: # pylint: disable=protected-access - raise NotImplementedError("cdf is not implemented when " - "bijector is not injective.") - bijector_kwargs = bijector_kwargs or {} - distribution_kwargs = distribution_kwargs or {} - x = self.bijector.inverse(y, **bijector_kwargs) - return self.distribution.cdf(x, **distribution_kwargs) - - @distribution_util.AppendDocstring(kwargs_dict=_condition_kwargs_dict) - def _log_survival_function(self, y, - bijector_kwargs=None, distribution_kwargs=None): - if self._is_maybe_event_override: - raise NotImplementedError("log_survival_function is not implemented when " - "overriding event_shape") - if not self.bijector._is_injective: # pylint: disable=protected-access - raise NotImplementedError("log_survival_function is not implemented when " - "bijector is not injective.") - bijector_kwargs = bijector_kwargs or {} - distribution_kwargs = distribution_kwargs or {} - x = self.bijector.inverse(y, **bijector_kwargs) - return self.distribution.log_survival_function(x, **distribution_kwargs) - - @distribution_util.AppendDocstring(kwargs_dict=_condition_kwargs_dict) - def _survival_function(self, y, - bijector_kwargs=None, distribution_kwargs=None): - if self._is_maybe_event_override: - raise NotImplementedError("survival_function is not implemented when " - "overriding event_shape") - if not self.bijector._is_injective: # pylint: disable=protected-access - raise NotImplementedError("survival_function is not implemented when " - "bijector is not injective.") - bijector_kwargs = bijector_kwargs or {} - distribution_kwargs = distribution_kwargs or {} - x = self.bijector.inverse(y, **bijector_kwargs) - return self.distribution.survival_function(x, **distribution_kwargs) - - @distribution_util.AppendDocstring(kwargs_dict=_condition_kwargs_dict) - def _quantile(self, value, bijector_kwargs=None, distribution_kwargs=None): - if self._is_maybe_event_override: - raise NotImplementedError("quantile is not implemented when overriding " - "event_shape") - if not self.bijector._is_injective: # pylint: disable=protected-access - raise NotImplementedError("quantile is not implemented when " - "bijector is not injective.") - bijector_kwargs = bijector_kwargs or {} - distribution_kwargs = distribution_kwargs or {} - # x_q is the "qth quantile" of X iff q = P[X <= x_q]. Now, since X = - # g^{-1}(Y), q = P[X <= x_q] = P[g^{-1}(Y) <= x_q] = P[Y <= g(x_q)], - # implies the qth quantile of Y is g(x_q). - inv_cdf = self.distribution.quantile(value, **distribution_kwargs) - return self.bijector.forward(inv_cdf, **bijector_kwargs) - - def _maybe_get_static_event_ndims(self): - if self.event_shape.ndims is not None: - return self.event_shape.ndims - - event_ndims = array_ops.size(self.event_shape_tensor()) - event_ndims_ = distribution_util.maybe_get_static_value(event_ndims) - - if event_ndims_ is not None: - return event_ndims_ - - return event_ndims diff --git a/tensorflow/contrib/distributions/python/ops/deterministic.py b/tensorflow/contrib/distributions/python/ops/deterministic.py deleted file mode 100644 index 0d57a2ddc60..00000000000 --- a/tensorflow/contrib/distributions/python/ops/deterministic.py +++ /dev/null @@ -1,417 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Deterministic distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.util import deprecation - -__all__ = [ - "Deterministic", - "VectorDeterministic", -] - - -@six.add_metaclass(abc.ABCMeta) -class _BaseDeterministic(distribution.Distribution): - """Base class for Deterministic distributions.""" - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc, - atol=None, - rtol=None, - is_vector=False, - validate_args=False, - allow_nan_stats=True, - name="_BaseDeterministic"): - """Initialize a batch of `_BaseDeterministic` distributions. - - The `atol` and `rtol` parameters allow for some slack in `pmf`, `cdf` - computations, e.g. due to floating-point error. - - ``` - pmf(x; loc) - = 1, if Abs(x - loc) <= atol + rtol * Abs(loc), - = 0, otherwise. - ``` - - Args: - loc: Numeric `Tensor`. The point (or batch of points) on which this - distribution is supported. - atol: Non-negative `Tensor` of same `dtype` as `loc` and broadcastable - shape. The absolute tolerance for comparing closeness to `loc`. - Default is `0`. - rtol: Non-negative `Tensor` of same `dtype` as `loc` and broadcastable - shape. The relative tolerance for comparing closeness to `loc`. - Default is `0`. - is_vector: Python `bool`. If `True`, this is for `VectorDeterministic`, - else `Deterministic`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - ValueError: If `loc` is a scalar. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[loc, atol, rtol]) as name: - loc = ops.convert_to_tensor(loc, name="loc") - if is_vector and validate_args: - msg = "Argument loc must be at least rank 1." - if loc.get_shape().ndims is not None: - if loc.get_shape().ndims < 1: - raise ValueError(msg) - else: - loc = control_flow_ops.with_dependencies( - [check_ops.assert_rank_at_least(loc, 1, message=msg)], loc) - self._loc = loc - - super(_BaseDeterministic, self).__init__( - dtype=self._loc.dtype, - reparameterization_type=distribution.NOT_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=[self._loc], - name=name) - - self._atol = self._get_tol(atol) - self._rtol = self._get_tol(rtol) - # Avoid using the large broadcast with self.loc if possible. - if rtol is None: - self._slack = self.atol - else: - self._slack = self.atol + self.rtol * math_ops.abs(self.loc) - - def _get_tol(self, tol): - if tol is None: - return ops.convert_to_tensor(0, dtype=self.loc.dtype) - - tol = ops.convert_to_tensor(tol, dtype=self.loc.dtype) - if self.validate_args: - tol = control_flow_ops.with_dependencies([ - check_ops.assert_non_negative( - tol, message="Argument 'tol' must be non-negative") - ], tol) - return tol - - @property - def loc(self): - """Point (or batch of points) at which this distribution is supported.""" - return self._loc - - @property - def atol(self): - """Absolute tolerance for comparing points to `self.loc`.""" - return self._atol - - @property - def rtol(self): - """Relative tolerance for comparing points to `self.loc`.""" - return self._rtol - - def _entropy(self): - return array_ops.zeros(self.batch_shape_tensor(), dtype=self.dtype) - - def _mean(self): - return array_ops.identity(self.loc) - - def _variance(self): - return array_ops.zeros_like(self.loc) - - def _mode(self): - return self.mean() - - def _sample_n(self, n, seed=None): # pylint: disable=unused-arg - n_static = tensor_util.constant_value(ops.convert_to_tensor(n)) - if n_static is not None and self.loc.get_shape().ndims is not None: - ones = [1] * self.loc.get_shape().ndims - multiples = [n_static] + ones - else: - ones = array_ops.ones_like(array_ops.shape(self.loc)) - multiples = array_ops.concat(([n], ones), axis=0) - - return array_ops.tile(self.loc[array_ops.newaxis, ...], multiples=multiples) - - -class Deterministic(_BaseDeterministic): - """Scalar `Deterministic` distribution on the real line. - - The scalar `Deterministic` distribution is parameterized by a [batch] point - `loc` on the real line. The distribution is supported at this point only, - and corresponds to a random variable that is constant, equal to `loc`. - - See [Degenerate rv](https://en.wikipedia.org/wiki/Degenerate_distribution). - - #### Mathematical Details - - The probability mass function (pmf) and cumulative distribution function (cdf) - are - - ```none - pmf(x; loc) = 1, if x == loc, else 0 - cdf(x; loc) = 1, if x >= loc, else 0 - ``` - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single Deterministic supported at zero. - constant = tfd.Deterministic(0.) - constant.prob(0.) - ==> 1. - constant.prob(2.) - ==> 0. - - # Initialize a [2, 2] batch of scalar constants. - loc = [[0., 1.], [2., 3.]] - x = [[0., 1.1], [1.99, 3.]] - constant = tfd.Deterministic(loc) - constant.prob(x) - ==> [[1., 0.], [0., 1.]] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc, - atol=None, - rtol=None, - validate_args=False, - allow_nan_stats=True, - name="Deterministic"): - """Initialize a scalar `Deterministic` distribution. - - The `atol` and `rtol` parameters allow for some slack in `pmf`, `cdf` - computations, e.g. due to floating-point error. - - ``` - pmf(x; loc) - = 1, if Abs(x - loc) <= atol + rtol * Abs(loc), - = 0, otherwise. - ``` - - Args: - loc: Numeric `Tensor` of shape `[B1, ..., Bb]`, with `b >= 0`. - The point (or batch of points) on which this distribution is supported. - atol: Non-negative `Tensor` of same `dtype` as `loc` and broadcastable - shape. The absolute tolerance for comparing closeness to `loc`. - Default is `0`. - rtol: Non-negative `Tensor` of same `dtype` as `loc` and broadcastable - shape. The relative tolerance for comparing closeness to `loc`. - Default is `0`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - super(Deterministic, self).__init__( - loc, - atol=atol, - rtol=rtol, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - - def _batch_shape_tensor(self): - return array_ops.shape(self.loc) - - def _batch_shape(self): - return self.loc.get_shape() - - def _event_shape_tensor(self): - return constant_op.constant([], dtype=dtypes.int32) - - def _event_shape(self): - return tensor_shape.TensorShape([]) - - def _prob(self, x): - return math_ops.cast( - math_ops.abs(x - self.loc) <= self._slack, dtype=self.dtype) - - def _cdf(self, x): - return math_ops.cast(x >= self.loc - self._slack, dtype=self.dtype) - - -class VectorDeterministic(_BaseDeterministic): - """Vector `Deterministic` distribution on `R^k`. - - The `VectorDeterministic` distribution is parameterized by a [batch] point - `loc in R^k`. The distribution is supported at this point only, - and corresponds to a random variable that is constant, equal to `loc`. - - See [Degenerate rv](https://en.wikipedia.org/wiki/Degenerate_distribution). - - #### Mathematical Details - - The probability mass function (pmf) is - - ```none - pmf(x; loc) - = 1, if All[Abs(x - loc) <= atol + rtol * Abs(loc)], - = 0, otherwise. - ``` - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single VectorDeterministic supported at [0., 2.] in R^2. - constant = tfd.Deterministic([0., 2.]) - constant.prob([0., 2.]) - ==> 1. - constant.prob([0., 3.]) - ==> 0. - - # Initialize a [3] batch of constants on R^2. - loc = [[0., 1.], [2., 3.], [4., 5.]] - constant = tfd.VectorDeterministic(loc) - constant.prob([[0., 1.], [1.9, 3.], [3.99, 5.]]) - ==> [1., 0., 0.] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc, - atol=None, - rtol=None, - validate_args=False, - allow_nan_stats=True, - name="VectorDeterministic"): - """Initialize a `VectorDeterministic` distribution on `R^k`, for `k >= 0`. - - Note that there is only one point in `R^0`, the "point" `[]`. So if `k = 0` - then `self.prob([]) == 1`. - - The `atol` and `rtol` parameters allow for some slack in `pmf` - computations, e.g. due to floating-point error. - - ``` - pmf(x; loc) - = 1, if All[Abs(x - loc) <= atol + rtol * Abs(loc)], - = 0, otherwise - ``` - - Args: - loc: Numeric `Tensor` of shape `[B1, ..., Bb, k]`, with `b >= 0`, `k >= 0` - The point (or batch of points) on which this distribution is supported. - atol: Non-negative `Tensor` of same `dtype` as `loc` and broadcastable - shape. The absolute tolerance for comparing closeness to `loc`. - Default is `0`. - rtol: Non-negative `Tensor` of same `dtype` as `loc` and broadcastable - shape. The relative tolerance for comparing closeness to `loc`. - Default is `0`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - super(VectorDeterministic, self).__init__( - loc, - atol=atol, - rtol=rtol, - is_vector=True, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - - def _batch_shape_tensor(self): - return array_ops.shape(self.loc)[:-1] - - def _batch_shape(self): - return self.loc.get_shape()[:-1] - - def _event_shape_tensor(self): - return array_ops.shape(self.loc)[-1] - - def _event_shape(self): - return self.loc.get_shape()[-1:] - - def _prob(self, x): - if self.validate_args: - is_vector_check = check_ops.assert_rank_at_least(x, 1) - right_vec_space_check = check_ops.assert_equal( - self.event_shape_tensor(), - array_ops.gather(array_ops.shape(x), array_ops.rank(x) - 1), - message= - "Argument 'x' not defined in the same space R^k as this distribution") - with ops.control_dependencies([is_vector_check]): - with ops.control_dependencies([right_vec_space_check]): - x = array_ops.identity(x) - return math_ops.cast( - math_ops.reduce_all(math_ops.abs(x - self.loc) <= self._slack, axis=-1), - dtype=self.dtype) diff --git a/tensorflow/contrib/distributions/python/ops/distribution_util.py b/tensorflow/contrib/distributions/python/ops/distribution_util.py deleted file mode 100644 index e6acae57a40..00000000000 --- a/tensorflow/contrib/distributions/python/ops/distribution_util.py +++ /dev/null @@ -1,565 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities for probability distributions.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import smart_cond -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.linalg import linalg -from tensorflow.python.ops.distributions import distribution as distribution_lib - -# The following two lines are redundant, in a sense. The first enables -# good coding practice *within* this file (`util.prefer_static_value` -# rather than `prefer_static_value`). The second ensures that users -# also get the core utils when they import this file. -from tensorflow.python.ops.distributions import util -from tensorflow.python.ops.distributions.util import * # pylint: disable=wildcard-import - - -def _convert_to_tensor(x, name): - return None if x is None else ops.convert_to_tensor(x, name=name) - - -def mixture_stddev(mixture_weight_vector, mean_vector, stddev_vector): - """Computes the standard deviation of a mixture distribution. - - This function works regardless of the component distribution, so long as - each component's mean and standard deviation can be provided. - - Args: - mixture_weight_vector: A 2D tensor with shape [batch_size, num_components] - mean_vector: A 2D tensor of mixture component means. Has shape - `[batch_size, num_components]`. - stddev_vector: A 2D tensor of mixture component standard deviations. Has - shape `[batch_size, num_components]`. - Returns: - A 1D tensor of shape `[batch_size]` representing the standard deviation of - the mixture distribution with given weights and component means and standard - deviations. - Raises: - ValueError: If the shapes of the input tensors are not as expected. - """ - mixture_weight_vector.shape.assert_has_rank(2) - if not mean_vector.shape.is_compatible_with(mixture_weight_vector.shape): - raise ValueError("Expecting means to have same shape as mixture weights.") - if not stddev_vector.shape.is_compatible_with(mixture_weight_vector.shape): - raise ValueError("Expecting stddevs to have same shape as mixture weights.") - - # Reshape the distribution parameters for batched vectorized dot products. - pi_for_dot_prod = array_ops.expand_dims(mixture_weight_vector, axis=1) - mu_for_dot_prod = array_ops.expand_dims(mean_vector, axis=2) - sigma_for_dot_prod = array_ops.expand_dims(stddev_vector, axis=2) - - # weighted average of component means under mixture distribution. - mean_wa = math_ops.matmul(pi_for_dot_prod, mu_for_dot_prod) - mean_wa = array_ops.reshape(mean_wa, (-1,)) - # weighted average of component variances under mixture distribution. - var_wa = math_ops.matmul(pi_for_dot_prod, - math_ops.square(sigma_for_dot_prod)) - var_wa = array_ops.reshape(var_wa, (-1,)) - # weighted average of component squared means under mixture distribution. - sq_mean_wa = math_ops.matmul(pi_for_dot_prod, - math_ops.square(mu_for_dot_prod)) - sq_mean_wa = array_ops.reshape(sq_mean_wa, (-1,)) - mixture_variance = var_wa + sq_mean_wa - math_ops.square(mean_wa) - return math_ops.sqrt(mixture_variance) - - -def make_tril_scale( - loc=None, - scale_tril=None, - scale_diag=None, - scale_identity_multiplier=None, - shape_hint=None, - validate_args=False, - assert_positive=False, - name=None): - """Creates a LinOp representing a lower triangular matrix. - - Args: - loc: Floating-point `Tensor`. This is used for inferring shape in the case - where only `scale_identity_multiplier` is set. - scale_tril: Floating-point `Tensor` representing the diagonal matrix. - `scale_diag` has shape [N1, N2, ... k, k], which represents a k x k - lower triangular matrix. - When `None` no `scale_tril` term is added to the LinOp. - The upper triangular elements above the diagonal are ignored. - scale_diag: Floating-point `Tensor` representing the diagonal matrix. - `scale_diag` has shape [N1, N2, ... k], which represents a k x k - diagonal matrix. - When `None` no diagonal term is added to the LinOp. - scale_identity_multiplier: floating point rank 0 `Tensor` representing a - scaling done to the identity matrix. - When `scale_identity_multiplier = scale_diag = scale_tril = None` then - `scale += IdentityMatrix`. Otherwise no scaled-identity-matrix is added - to `scale`. - shape_hint: scalar integer `Tensor` representing a hint at the dimension of - the identity matrix when only `scale_identity_multiplier` is set. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - assert_positive: Python `bool` indicating whether LinOp should be checked - for being positive definite. - name: Python `str` name given to ops managed by this object. - - Returns: - `LinearOperator` representing a lower triangular matrix. - - Raises: - ValueError: If only `scale_identity_multiplier` is set and `loc` and - `shape_hint` are both None. - """ - - def _maybe_attach_assertion(x): - if not validate_args: - return x - if assert_positive: - return control_flow_ops.with_dependencies([ - check_ops.assert_positive( - array_ops.matrix_diag_part(x), - message="diagonal part must be positive"), - ], x) - return control_flow_ops.with_dependencies([ - check_ops.assert_none_equal( - array_ops.matrix_diag_part(x), - array_ops.zeros([], x.dtype), - message="diagonal part must be non-zero"), - ], x) - - with ops.name_scope(name, "make_tril_scale", - values=[loc, scale_diag, scale_identity_multiplier]): - - loc = _convert_to_tensor(loc, name="loc") - scale_tril = _convert_to_tensor(scale_tril, name="scale_tril") - scale_diag = _convert_to_tensor(scale_diag, name="scale_diag") - scale_identity_multiplier = _convert_to_tensor( - scale_identity_multiplier, - name="scale_identity_multiplier") - - if scale_tril is not None: - scale_tril = array_ops.matrix_band_part(scale_tril, -1, 0) # Zero out TriU. - tril_diag = array_ops.matrix_diag_part(scale_tril) - if scale_diag is not None: - tril_diag += scale_diag - if scale_identity_multiplier is not None: - tril_diag += scale_identity_multiplier[..., array_ops.newaxis] - - scale_tril = array_ops.matrix_set_diag(scale_tril, tril_diag) - - return linalg.LinearOperatorLowerTriangular( - tril=_maybe_attach_assertion(scale_tril), - is_non_singular=True, - is_self_adjoint=False, - is_positive_definite=assert_positive) - - return make_diag_scale( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - shape_hint=shape_hint, - validate_args=validate_args, - assert_positive=assert_positive, - name=name) - - -def make_diag_scale( - loc=None, - scale_diag=None, - scale_identity_multiplier=None, - shape_hint=None, - validate_args=False, - assert_positive=False, - name=None): - """Creates a LinOp representing a diagonal matrix. - - Args: - loc: Floating-point `Tensor`. This is used for inferring shape in the case - where only `scale_identity_multiplier` is set. - scale_diag: Floating-point `Tensor` representing the diagonal matrix. - `scale_diag` has shape [N1, N2, ... k], which represents a k x k - diagonal matrix. - When `None` no diagonal term is added to the LinOp. - scale_identity_multiplier: floating point rank 0 `Tensor` representing a - scaling done to the identity matrix. - When `scale_identity_multiplier = scale_diag = scale_tril = None` then - `scale += IdentityMatrix`. Otherwise no scaled-identity-matrix is added - to `scale`. - shape_hint: scalar integer `Tensor` representing a hint at the dimension of - the identity matrix when only `scale_identity_multiplier` is set. - validate_args: Python `bool` indicating whether arguments should be - checked for correctness. - assert_positive: Python `bool` indicating whether LinOp should be checked - for being positive definite. - name: Python `str` name given to ops managed by this object. - - Returns: - `LinearOperator` representing a lower triangular matrix. - - Raises: - ValueError: If only `scale_identity_multiplier` is set and `loc` and - `shape_hint` are both None. - """ - - def _maybe_attach_assertion(x): - if not validate_args: - return x - if assert_positive: - return control_flow_ops.with_dependencies([ - check_ops.assert_positive( - x, message="diagonal part must be positive"), - ], x) - return control_flow_ops.with_dependencies([ - check_ops.assert_none_equal( - x, - array_ops.zeros([], x.dtype), - message="diagonal part must be non-zero")], x) - - with ops.name_scope(name, "make_diag_scale", - values=[loc, scale_diag, scale_identity_multiplier]): - loc = _convert_to_tensor(loc, name="loc") - scale_diag = _convert_to_tensor(scale_diag, name="scale_diag") - scale_identity_multiplier = _convert_to_tensor( - scale_identity_multiplier, - name="scale_identity_multiplier") - - if scale_diag is not None: - if scale_identity_multiplier is not None: - scale_diag += scale_identity_multiplier[..., array_ops.newaxis] - return linalg.LinearOperatorDiag( - diag=_maybe_attach_assertion(scale_diag), - is_non_singular=True, - is_self_adjoint=True, - is_positive_definite=assert_positive) - - if loc is None and shape_hint is None: - raise ValueError( - "Cannot infer `event_shape` unless `loc` or " - "`shape_hint` is specified.") - - if shape_hint is None: - shape_hint = loc.shape[-1] - - if scale_identity_multiplier is None: - return linalg.LinearOperatorIdentity( - num_rows=shape_hint, - dtype=loc.dtype.base_dtype, - is_self_adjoint=True, - is_positive_definite=True, - assert_proper_shapes=validate_args) - - return linalg.LinearOperatorScaledIdentity( - num_rows=shape_hint, - multiplier=_maybe_attach_assertion(scale_identity_multiplier), - is_non_singular=True, - is_self_adjoint=True, - is_positive_definite=assert_positive, - assert_proper_shapes=validate_args) - - -def shapes_from_loc_and_scale(loc, scale, name="shapes_from_loc_and_scale"): - """Infer distribution batch and event shapes from a location and scale. - - Location and scale family distributions determine their batch/event shape by - broadcasting the `loc` and `scale` args. This helper does that broadcast, - statically if possible. - - Batch shape broadcasts as per the normal rules. - We allow the `loc` event shape to broadcast up to that of `scale`. We do not - allow `scale`'s event shape to change. Therefore, the last dimension of `loc` - must either be size `1`, or the same as `scale.range_dimension`. - - See `MultivariateNormalLinearOperator` for a usage example. - - Args: - loc: `N-D` `Tensor` with `N >= 1` (already converted to tensor) or `None`. - If `None`, both batch and event shape are determined by `scale`. - scale: A `LinearOperator` instance. - name: A string name to prepend to created ops. - - Returns: - batch_shape: `TensorShape` (if broadcast is done statically), or `Tensor`. - event_shape: `TensorShape` (if broadcast is done statically), or `Tensor`. - - Raises: - ValueError: If the last dimension of `loc` is determined statically to be - different than the range of `scale`. - """ - with ops.name_scope(name, values=[loc]): - # Get event shape. - event_size = scale.range_dimension_tensor() - event_size_const = tensor_util.constant_value(event_size) - if event_size_const is not None: - event_shape = event_size_const.reshape([1]) - else: - event_shape = event_size[array_ops.newaxis] - - # Static check that event shapes match. - if loc is not None: - loc_event_size = tensor_shape.dimension_value(loc.get_shape()[-1]) - if loc_event_size is not None and event_size_const is not None: - if loc_event_size != 1 and loc_event_size != event_size_const: - raise ValueError( - "Event size of 'scale' (%d) could not be broadcast up to that of " - "'loc' (%d)." % (loc_event_size, event_size_const)) - - # Get batch shape. - batch_shape = scale.batch_shape_tensor() - if loc is None: - batch_shape_const = tensor_util.constant_value(batch_shape) - batch_shape = ( - batch_shape_const if batch_shape_const is not None else batch_shape) - else: - loc_batch_shape = loc.get_shape().with_rank_at_least(1)[:-1] - if (loc.get_shape().ndims is None or - not loc_batch_shape.is_fully_defined()): - loc_batch_shape = array_ops.shape(loc)[:-1] - else: - loc_batch_shape = ops.convert_to_tensor(loc_batch_shape, - name="loc_batch_shape") - # This is defined in the core util module. - # pylint: disable=undefined-variable - batch_shape = prefer_static_broadcast_shape(batch_shape, loc_batch_shape) - # pylint: enable=undefined-variable - - return batch_shape, event_shape - - -def get_broadcast_shape(*tensors): - """Get broadcast shape as a Python list of integers (preferred) or `Tensor`. - - Args: - *tensors: One or more `Tensor` objects (already converted!). - - Returns: - broadcast shape: Python list (if shapes determined statically), otherwise - an `int32` `Tensor`. - """ - # Try static. - s_shape = tensors[0].shape - for t in tensors[1:]: - s_shape = array_ops.broadcast_static_shape(s_shape, t.shape) - if s_shape.is_fully_defined(): - return s_shape.as_list() - - # Fallback on dynamic. - d_shape = array_ops.shape(tensors[0]) - for t in tensors[1:]: - d_shape = array_ops.broadcast_dynamic_shape(d_shape, array_ops.shape(t)) - return d_shape - - -def is_diagonal_scale(scale): - """Returns `True` if `scale` is a `LinearOperator` that is known to be diag. - - Args: - scale: `LinearOperator` instance. - - Returns: - Python `bool`. - - Raises: - TypeError: If `scale` is not a `LinearOperator`. - """ - if not isinstance(scale, linalg.LinearOperator): - raise TypeError("Expected argument 'scale' to be instance of LinearOperator" - ". Found: %s" % scale) - return (isinstance(scale, linalg.LinearOperatorIdentity) or - isinstance(scale, linalg.LinearOperatorScaledIdentity) or - isinstance(scale, linalg.LinearOperatorDiag)) - - -def maybe_check_scalar_distribution( - distribution, expected_base_dtype, validate_args): - """Helper which checks validity of a scalar `distribution` init arg. - - Valid here means: - - * `distribution` has scalar batch and event shapes. - * `distribution` is `FULLY_REPARAMETERIZED` - * `distribution` has expected dtype. - - Args: - distribution: `Distribution`-like object. - expected_base_dtype: `TensorFlow` `dtype`. - validate_args: Python `bool`. Whether to do additional checks: - (i) check that reparameterization_type is `FULLY_REPARAMETERIZED`. - (ii) add `tf.Assert` ops to the graph to enforce that distribution - is scalar in the event that this cannot be determined statically. - - Returns: - List of `tf.Assert` ops to run to enforce validity checks that could not - be statically determined. Empty if `not validate_args`. - - Raises: - ValueError: If validate_args and distribution is not FULLY_REPARAMETERIZED - ValueError: If distribution is statically determined to not have both - scalar batch and scalar event shapes. - """ - if distribution.dtype != expected_base_dtype: - raise TypeError("dtype mismatch; " - "distribution.dtype=\"{}\" is not \"{}\"".format( - distribution.dtype.name, expected_base_dtype.name)) - - # Although `reparameterization_type` is a static property, we guard it by - # `validate_args`. This allows users to use a `distribution` which is not - # reparameterized itself. However, we tacitly assume that although the - # distribution is not reparameterized, it only depends on non-trainable - # variables. - if validate_args and (distribution.reparameterization_type - != distribution_lib.FULLY_REPARAMETERIZED): - raise ValueError("Base distribution should be reparameterized or be " - "a function of non-trainable variables; " - "distribution.reparameterization_type = \"{}\" " - "!= \"FULLY_REPARAMETERIZED\".".format( - distribution.reparameterization_type)) - with ops.name_scope(name="check_distribution"): - assertions = [] - def check_is_scalar(is_scalar, name): - is_scalar_ = static_value(is_scalar) - if is_scalar_ is not None: - if not is_scalar_: - raise ValueError("distribution must be scalar; " - "distribution.{}=False is not True".format(name)) - elif validate_args: - assertions.append(check_ops.assert_equal( - is_scalar, True, - message=("distribution must be scalar; " - "distribution.{}=False is not True".format(name)))) - check_is_scalar(distribution.is_scalar_event(), "is_scalar_event") - check_is_scalar(distribution.is_scalar_batch(), "is_scalar_batch") - return assertions - - -def pad_mixture_dimensions(x, mixture_distribution, categorical_distribution, - event_ndims): - """Pad dimensions of event tensors for mixture distributions. - - See `Mixture._sample_n` and `MixtureSameFamily._sample_n` for usage examples. - - Args: - x: event tensor to pad. - mixture_distribution: Base distribution of the mixture. - categorical_distribution: `Categorical` distribution that mixes the base - distribution. - event_ndims: Integer specifying the number of event dimensions in the event - tensor. - - Returns: - A padded version of `x` that can broadcast with `categorical_distribution`. - """ - with ops.name_scope("pad_mix_dims", values=[x]): - def _get_ndims(d): - if d.batch_shape.ndims is not None: - return d.batch_shape.ndims - return array_ops.shape(d.batch_shape_tensor())[0] - dist_batch_ndims = _get_ndims(mixture_distribution) - cat_batch_ndims = _get_ndims(categorical_distribution) - pad_ndims = array_ops.where_v2(categorical_distribution.is_scalar_batch(), - dist_batch_ndims, - dist_batch_ndims - cat_batch_ndims) - s = array_ops.shape(x) - x = array_ops.reshape(x, shape=array_ops.concat([ - s[:-1], - array_ops.ones([pad_ndims], dtype=dtypes.int32), - s[-1:], - array_ops.ones([event_ndims], dtype=dtypes.int32), - ], axis=0)) - return x - - -def static_value(x): - """Returns the static value of a `Tensor` or `None`.""" - return tensor_util.constant_value(ops.convert_to_tensor(x)) - - -def move_dimension(x, source_idx, dest_idx): - """Move a single tensor dimension within its shape. - - This is a special case of `tf.transpose()`, which applies - arbitrary permutations to tensor dimensions. - - Args: - x: Tensor of rank `ndims`. - source_idx: Integer index into `x.shape` (negative indexing is - supported). - dest_idx: Integer index into `x.shape` (negative indexing is - supported). - - Returns: - x_perm: Tensor of rank `ndims`, in which the dimension at original - index `source_idx` has been moved to new index `dest_idx`, with - all other dimensions retained in their original order. - - Example: - - ```python - x = tf.compat.v1.placeholder(shape=[200, 30, 4, 1, 6]) - x_perm = _move_dimension(x, 1, 1) # no-op - x_perm = _move_dimension(x, 0, 3) # result shape [30, 4, 1, 200, 6] - x_perm = _move_dimension(x, 0, -2) # equivalent to previous - x_perm = _move_dimension(x, 4, 2) # result shape [200, 30, 6, 4, 1] - ``` - """ - ndims = util.prefer_static_rank(x) - if isinstance(source_idx, int): - dtype = dtypes.int32 - else: - dtype = dtypes.as_dtype(source_idx.dtype) - - # Handle negative indexing. Since ndims might be dynamic, this makes - # source_idx and dest_idx also possibly dynamic. - if source_idx < 0: - source_idx = ndims + source_idx - if dest_idx < 0: - dest_idx = ndims + dest_idx - - # Construct the appropriate permutation of dimensions, depending - # whether the source is before or after the destination. - def move_left_permutation(): - return util.prefer_static_value( - array_ops.concat([ - math_ops.range(0, dest_idx, dtype=dtype), - [source_idx], - math_ops.range(dest_idx, source_idx, dtype=dtype), - math_ops.range(source_idx+1, ndims, dtype=dtype)], axis=0)) - - def move_right_permutation(): - return util.prefer_static_value( - array_ops.concat([ - math_ops.range(0, source_idx, dtype=dtype), - math_ops.range(source_idx+1, dest_idx+1, dtype=dtype), - [source_idx], - math_ops.range(dest_idx+1, ndims, dtype=dtype)], axis=0)) - - def x_permuted(): - return array_ops.transpose( - x, perm=smart_cond.smart_cond(source_idx < dest_idx, - move_right_permutation, - move_left_permutation)) - - # One final conditional to handle the special case where source - # and destination indices are equal. - return smart_cond.smart_cond(math_ops.equal(source_idx, dest_idx), - lambda: x, - x_permuted) diff --git a/tensorflow/contrib/distributions/python/ops/estimator.py b/tensorflow/contrib/distributions/python/ops/estimator.py deleted file mode 100644 index bdec6527d53..00000000000 --- a/tensorflow/contrib/distributions/python/ops/estimator.py +++ /dev/null @@ -1,202 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functions to bridge `Distribution`s and `tf.contrib.learn.estimator` APIs.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.estimators.head import _compute_weighted_loss -from tensorflow.contrib.learn.python.learn.estimators.head import _RegressionHead -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.util import deprecation - - -__all__ = [ - "estimator_head_distribution_regression", -] - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def estimator_head_distribution_regression(make_distribution_fn, - label_dimension=1, - logits_dimension=None, - label_name=None, - weight_column_name=None, - enable_centered_bias=False, - head_name=None): - """Creates a `Head` for regression under a generic distribution. - - Args: - make_distribution_fn: Python `callable` which returns a `tf.Distribution` - instance created using only logits. - label_dimension: Number of regression labels per example. This is the size - of the last dimension of the labels `Tensor` (typically, this has shape - `[batch_size, label_dimension]`). - logits_dimension: Number of logits per example. This is the size of the last - dimension of the logits `Tensor` (typically, this has shape - `[batch_size, logits_dimension]`). - Default value: `label_dimension`. - label_name: Python `str`, name of the key in label `dict`. Can be `None` if - label is a `Tensor` (single headed models). - weight_column_name: Python `str` defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - enable_centered_bias: Python `bool`. If `True`, estimator will learn a - centered bias variable for each class. Rest of the model structure learns - the residual after centered bias. - head_name: Python `str`, name of the head. Predictions, summary and metrics - keys are suffixed by `"/" + head_name` and the default variable scope is - `head_name`. - - Returns: - An instance of `Head` for generic regression. - """ - return _DistributionRegressionHead( - make_distribution_fn=make_distribution_fn, - label_dimension=label_dimension, - logits_dimension=logits_dimension, - label_name=label_name, - weight_column_name=weight_column_name, - enable_centered_bias=enable_centered_bias, - head_name=head_name) - - -class _DistributionRegressionHead(_RegressionHead): - """Creates a _RegressionHead instance from an arbitrary `Distribution`.""" - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - make_distribution_fn, - label_dimension, - logits_dimension=None, - label_name=None, - weight_column_name=None, - enable_centered_bias=False, - head_name=None): - """`Head` for regression. - - Args: - make_distribution_fn: Python `callable` which returns a `tf.Distribution` - instance created using only logits. - label_dimension: Number of regression labels per example. This is the - size of the last dimension of the labels `Tensor` (typically, this has - shape `[batch_size, label_dimension]`). - logits_dimension: Number of logits per example. This is the size of the - last dimension of the logits `Tensor` (typically, this has shape - `[batch_size, logits_dimension]`). - Default value: `label_dimension`. - label_name: Python `str`, name of the key in label `dict`. Can be `None` - if label is a tensor (single headed models). - weight_column_name: Python `str` defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - enable_centered_bias: Python `bool`. If `True`, estimator will learn a - centered bias variable for each class. Rest of the model structure - learns the residual after centered bias. - head_name: Python `str`, name of the head. Predictions, summary and - metrics keys are suffixed by `"/" + head_name` and the default variable - scope is `head_name`. - - Raises: - TypeError: if `make_distribution_fn` is not `callable`. - """ - if not callable(make_distribution_fn): - raise TypeError("`make_distribution_fn` must be a callable function.") - - self._distributions = {} - self._make_distribution_fn = make_distribution_fn - - def static_value(x): - """Returns the static value of a `Tensor` or `None`.""" - return tensor_util.constant_value(ops.convert_to_tensor(x)) - - def concat_vectors(*args): - """Concatenates input vectors, statically if possible.""" - args_ = [static_value(x) for x in args] - if any(vec is None for vec in args_): - return array_ops.concat(args, axis=0) - return [val for vec in args_ for val in vec] - - def loss_fn(labels, logits, weights=None): - """Returns the loss of using `logits` to predict `labels`.""" - d = self.distribution(logits) - labels_batch_shape = labels.shape.with_rank_at_least(1)[:-1] - labels_batch_shape = ( - labels_batch_shape.as_list() if labels_batch_shape.is_fully_defined() - else array_ops.shape(labels)[:-1]) - labels = array_ops.reshape( - labels, - shape=concat_vectors(labels_batch_shape, d.event_shape_tensor())) - return _compute_weighted_loss( - loss_unweighted=-d.log_prob(labels), - weight=weights) - - def link_fn(logits): - """Returns the inverse link function at `logits`.""" - # Note: What the API calls a "link function" is really the inverse-link - # function, i.e., the "mean". - d = self.distribution(logits) - return d.mean() - - super(_DistributionRegressionHead, self).__init__( - label_dimension=label_dimension, - loss_fn=loss_fn, - link_fn=link_fn, - logits_dimension=logits_dimension, - label_name=label_name, - weight_column_name=weight_column_name, - enable_centered_bias=enable_centered_bias, - head_name=head_name) - - @property - def distributions(self): - """Returns all distributions created by `DistributionRegressionHead`.""" - return self._distributions - - def distribution(self, logits, name=None): - """Retrieves a distribution instance, parameterized by `logits`. - - Args: - logits: `float`-like `Tensor` representing the parameters of the - underlying distribution. - name: The Python `str` name to given to this op. - Default value: "distribution". - - Returns: - distribution: `tf.Distribution` instance parameterized by `logits`. - """ - with ops.name_scope(name, "distribution", [logits]): - d = self._distributions.get(logits, None) - if d is None: - d = self._make_distribution_fn(logits) - self._distributions[logits] = d - return d diff --git a/tensorflow/contrib/distributions/python/ops/geometric.py b/tensorflow/contrib/distributions/python/ops/geometric.py deleted file mode 100644 index 0b5c47056f3..00000000000 --- a/tensorflow/contrib/distributions/python/ops/geometric.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Geometric distribution class.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - - -class Geometric(distribution.Distribution): - """Geometric distribution. - - The Geometric distribution is parameterized by p, the probability of a - positive event. It represents the probability that in k + 1 Bernoulli trials, - the first k trials failed, before seeing a success. - - The pmf of this distribution is: - - #### Mathematical Details - - ```none - pmf(k; p) = (1 - p)**k * p - ``` - - where: - - * `p` is the success probability, `0 < p <= 1`, and, - * `k` is a non-negative integer. - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - logits=None, - probs=None, - validate_args=False, - allow_nan_stats=True, - name="Geometric"): - """Construct Geometric distributions. - - Args: - logits: Floating-point `Tensor` with shape `[B1, ..., Bb]` where `b >= 0` - indicates the number of batch dimensions. Each entry represents logits - for the probability of success for independent Geometric distributions - and must be in the range `(-inf, inf]`. Only one of `logits` or `probs` - should be specified. - probs: Positive floating-point `Tensor` with shape `[B1, ..., Bb]` - where `b >= 0` indicates the number of batch dimensions. Each entry - represents the probability of success for independent Geometric - distributions and must be in the range `(0, 1]`. Only one of `logits` - or `probs` should be specified. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - - parameters = dict(locals()) - with ops.name_scope(name, values=[logits, probs]) as name: - self._logits, self._probs = distribution_util.get_logits_and_probs( - logits, probs, validate_args=validate_args, name=name) - - with ops.control_dependencies( - [check_ops.assert_positive(self._probs)] if validate_args else []): - self._probs = array_ops.identity(self._probs, name="probs") - - super(Geometric, self).__init__( - dtype=self._probs.dtype, - reparameterization_type=distribution.NOT_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=[self._probs, self._logits], - name=name) - - @property - def logits(self): - """Log-odds of a `1` outcome (vs `0`).""" - return self._logits - - @property - def probs(self): - """Probability of a `1` outcome (vs `0`).""" - return self._probs - - def _batch_shape_tensor(self): - return array_ops.shape(self._probs) - - def _batch_shape(self): - return self.probs.get_shape() - - def _event_shape_tensor(self): - return array_ops.constant([], dtype=dtypes.int32) - - def _event_shape(self): - return tensor_shape.TensorShape([]) - - def _sample_n(self, n, seed=None): - # Uniform variates must be sampled from the open-interval `(0, 1)` rather - # than `[0, 1)`. To do so, we use `np.finfo(self.dtype.as_numpy_dtype).tiny` - # because it is the smallest, positive, "normal" number. A "normal" number - # is such that the mantissa has an implicit leading 1. Normal, positive - # numbers x, y have the reasonable property that, `x + y >= max(x, y)`. In - # this case, a subnormal number (i.e., np.nextafter) can cause us to sample - # 0. - sampled = random_ops.random_uniform( - array_ops.concat([[n], array_ops.shape(self._probs)], 0), - minval=np.finfo(self.dtype.as_numpy_dtype).tiny, - maxval=1., - seed=seed, - dtype=self.dtype) - - return math_ops.floor( - math_ops.log(sampled) / math_ops.log1p(-self.probs)) - - def _cdf(self, x): - if self.validate_args: - x = distribution_util.embed_check_nonnegative_integer_form(x) - else: - # Whether or not x is integer-form, the following is well-defined. - # However, scipy takes the floor, so we do too. - x = math_ops.floor(x) - x *= array_ops.ones_like(self.probs) - return array_ops.where( - x < 0., - array_ops.zeros_like(x), - -math_ops.expm1((1. + x) * math_ops.log1p(-self.probs))) - - def _log_prob(self, x): - if self.validate_args: - x = distribution_util.embed_check_nonnegative_integer_form(x) - else: - # For consistency with cdf, we take the floor. - x = math_ops.floor(x) - x *= array_ops.ones_like(self.probs) - probs = self.probs * array_ops.ones_like(x) - safe_domain = array_ops.where( - math_ops.equal(x, 0.), - array_ops.zeros_like(probs), - probs) - return x * math_ops.log1p(-safe_domain) + math_ops.log(probs) - - def _entropy(self): - probs = self._probs - if self.validate_args: - probs = control_flow_ops.with_dependencies( - [check_ops.assert_less( - probs, - constant_op.constant(1., probs.dtype), - message="Entropy is undefined when logits = inf or probs = 1.")], - probs) - # Claim: entropy(p) = softplus(s)/p - s - # where s=logits and p=probs. - # - # Proof: - # - # entropy(p) - # := -[(1-p)log(1-p) + plog(p)]/p - # = -[log(1-p) + plog(p/(1-p))]/p - # = -[-softplus(s) + ps]/p - # = softplus(s)/p - s - # - # since, - # log[1-sigmoid(s)] - # = log[1/(1+exp(s)] - # = -log[1+exp(s)] - # = -softplus(s) - # - # using the fact that, - # 1-sigmoid(s) = sigmoid(-s) = 1/(1+exp(s)) - return nn.softplus(self.logits) / probs - self.logits - - def _mean(self): - return math_ops.exp(-self.logits) - - def _variance(self): - return self._mean() / self.probs - - def _mode(self): - return array_ops.zeros(self.batch_shape_tensor(), dtype=self.dtype) diff --git a/tensorflow/contrib/distributions/python/ops/gumbel.py b/tensorflow/contrib/distributions/python/ops/gumbel.py deleted file mode 100644 index 341d63f573b..00000000000 --- a/tensorflow/contrib/distributions/python/ops/gumbel.py +++ /dev/null @@ -1,233 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Gumbel distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import numpy as np -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.util import deprecation - - -class _Gumbel(distribution.Distribution): - """The scalar Gumbel distribution with location `loc` and `scale` parameters. - - #### Mathematical details - - The probability density function (pdf) of this distribution is, - - ```none - pdf(x; mu, sigma) = exp(-(x - mu) / sigma - exp(-(x - mu) / sigma)) - ``` - - where `loc = mu` and `scale = sigma`. - - The cumulative density function of this distribution is, - - ```cdf(x; mu, sigma) = exp(-exp(-(x - mu) / sigma))``` - - The Gumbel distribution is a member of the [location-scale family]( - https://en.wikipedia.org/wiki/Location-scale_family), i.e., it can be - constructed as, - - ```none - X ~ Gumbel(loc=0, scale=1) - Y = loc + scale * X - ``` - - #### Examples - - Examples of initialization of one or a batch of distributions. - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Define a single scalar Gumbel distribution. - dist = tfd.Gumbel(loc=0., scale=3.) - - # Evaluate the cdf at 1, returning a scalar. - dist.cdf(1.) - - # Define a batch of two scalar valued Gumbels. - # The first has mean 1 and scale 11, the second 2 and 22. - dist = tfd.Gumbel(loc=[1, 2.], scale=[11, 22.]) - - # Evaluate the pdf of the first distribution on 0, and the second on 1.5, - # returning a length two tensor. - dist.prob([0, 1.5]) - - # Get 3 samples, returning a 3 x 2 tensor. - dist.sample([3]) - ``` - - Arguments are broadcast when possible. - - ```python - # Define a batch of two scalar valued Logistics. - # Both have mean 1, but different scales. - dist = tfd.Gumbel(loc=1., scale=[11, 22.]) - - # Evaluate the pdf of both distributions on the same point, 3.0, - # returning a length 2 tensor. - dist.prob(3.0) - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc, - scale, - validate_args=False, - allow_nan_stats=True, - name="Gumbel"): - """Construct Gumbel distributions with location and scale `loc` and `scale`. - - The parameters `loc` and `scale` must be shaped in a way that supports - broadcasting (e.g. `loc + scale` is a valid operation). - - Args: - loc: Floating point tensor, the means of the distribution(s). - scale: Floating point tensor, the scales of the distribution(s). - scale must contain only positive values. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, - statistics (e.g., mean, mode, variance) use the value "`NaN`" to - indicate the result is undefined. When `False`, an exception is raised - if one or more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - TypeError: if loc and scale are different dtypes. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[loc, scale]) as name: - with ops.control_dependencies([check_ops.assert_positive(scale)] if - validate_args else []): - self._loc = array_ops.identity(loc, name="loc") - self._scale = array_ops.identity(scale, name="scale") - check_ops.assert_same_float_dtype([self._loc, self._scale]) - super(_Gumbel, self).__init__( - dtype=self._scale.dtype, - reparameterization_type=distribution.FULLY_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=[self._loc, self._scale], - name=name) - - @staticmethod - def _param_shapes(sample_shape): - return dict( - zip(("loc", "scale"), ([ops.convert_to_tensor( - sample_shape, dtype=dtypes.int32)] * 2))) - - @property - def loc(self): - """Distribution parameter for the location.""" - return self._loc - - @property - def scale(self): - """Distribution parameter for scale.""" - return self._scale - - def _batch_shape_tensor(self): - return array_ops.broadcast_dynamic_shape( - array_ops.shape(self.loc), array_ops.shape(self.scale)) - - def _batch_shape(self): - return array_ops.broadcast_static_shape( - self.loc.get_shape(), self.scale.get_shape()) - - def _event_shape_tensor(self): - return constant_op.constant([], dtype=dtypes.int32) - - def _event_shape(self): - return tensor_shape.TensorShape([]) - - def _sample_n(self, n, seed=None): - # Uniform variates must be sampled from the open-interval `(0, 1)` rather - # than `[0, 1)`. To do so, we use `np.finfo(self.dtype.as_numpy_dtype).tiny` - # because it is the smallest, positive, "normal" number. A "normal" number - # is such that the mantissa has an implicit leading 1. Normal, positive - # numbers x, y have the reasonable property that, `x + y >= max(x, y)`. In - # this case, a subnormal number (i.e., np.nextafter) can cause us to sample - # 0. - uniform = random_ops.random_uniform( - shape=array_ops.concat([[n], self.batch_shape_tensor()], 0), - minval=np.finfo(self.dtype.as_numpy_dtype).tiny, - maxval=1., - dtype=self.dtype, - seed=seed) - sampled = -math_ops.log(-math_ops.log(uniform)) - return sampled * self.scale + self.loc - - def _log_prob(self, x): - return self._log_unnormalized_prob(x) - self._log_normalization() - - def _log_cdf(self, x): - return -math_ops.exp(-self._z(x)) - - def _cdf(self, x): - return math_ops.exp(-math_ops.exp(-self._z(x))) - - def _log_unnormalized_prob(self, x): - z = self._z(x) - return - z - math_ops.exp(-z) - - def _log_normalization(self): - return math_ops.log(self.scale) - - def _entropy(self): - # Use broadcasting rules to calculate the full broadcast sigma. - scale = self.scale * array_ops.ones_like(self.loc) - return 1 + math_ops.log(scale) + np.euler_gamma - - def _mean(self): - return self.loc + self.scale * np.euler_gamma - - def _stddev(self): - return self.scale * array_ops.ones_like(self.loc) * math.pi / math.sqrt(6) - - def _mode(self): - return self.loc * array_ops.ones_like(self.scale) - - def _z(self, x): - """Standardize input `x` to a unit logistic.""" - with ops.name_scope("standardize", values=[x]): - return (x - self.loc) / self.scale diff --git a/tensorflow/contrib/distributions/python/ops/half_normal.py b/tensorflow/contrib/distributions/python/ops/half_normal.py deleted file mode 100644 index 1f04090b3ac..00000000000 --- a/tensorflow/contrib/distributions/python/ops/half_normal.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Half Normal distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.ops.distributions import special_math -from tensorflow.python.util import deprecation - - -__all__ = [ - "HalfNormal", -] - - -class HalfNormal(distribution.Distribution): - """The Half Normal distribution with scale `scale`. - - #### Mathematical details - - The half normal is a transformation of a centered normal distribution. - If some random variable `X` has normal distribution, - ```none - X ~ Normal(0.0, scale) - Y = |X| - ``` - Then `Y` will have half normal distribution. The probability density - function (pdf) is: - - ```none - pdf(x; scale, x > 0) = sqrt(2) / (scale * sqrt(pi)) * - exp(- 1/2 * (x / scale) ** 2) - ) - ``` - Where `scale = sigma` is the standard deviation of the underlying normal - distribution. - - #### Examples - - Examples of initialization of one or a batch of distributions. - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Define a single scalar HalfNormal distribution. - dist = tfd.HalfNormal(scale=3.0) - - # Evaluate the cdf at 1, returning a scalar. - dist.cdf(1.) - - # Define a batch of two scalar valued HalfNormals. - # The first has scale 11.0, the second 22.0 - dist = tfd.HalfNormal(scale=[11.0, 22.0]) - - # Evaluate the pdf of the first distribution on 1.0, and the second on 1.5, - # returning a length two tensor. - dist.prob([1.0, 1.5]) - - # Get 3 samples, returning a 3 x 2 tensor. - dist.sample([3]) - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - scale, - validate_args=False, - allow_nan_stats=True, - name="HalfNormal"): - """Construct HalfNormals with scale `scale`. - - Args: - scale: Floating point tensor; the scales of the distribution(s). - Must contain only positive values. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, - statistics (e.g., mean, mode, variance) use the value "`NaN`" to - indicate the result is undefined. When `False`, an exception is raised - if one or more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[scale]) as name: - with ops.control_dependencies([check_ops.assert_positive(scale)] if - validate_args else []): - self._scale = array_ops.identity(scale, name="scale") - super(HalfNormal, self).__init__( - dtype=self._scale.dtype, - reparameterization_type=distribution.FULLY_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=[self._scale], - name=name) - - @staticmethod - def _param_shapes(sample_shape): - return {"scale": ops.convert_to_tensor(sample_shape, dtype=dtypes.int32)} - - @property - def scale(self): - """Distribution parameter for the scale.""" - return self._scale - - def _batch_shape_tensor(self): - return array_ops.shape(self.scale) - - def _batch_shape(self): - return self.scale.shape - - def _event_shape_tensor(self): - return constant_op.constant([], dtype=dtypes.int32) - - def _event_shape(self): - return tensor_shape.TensorShape([]) - - def _sample_n(self, n, seed=None): - shape = array_ops.concat([[n], self.batch_shape_tensor()], 0) - sampled = random_ops.random_normal( - shape=shape, mean=0., stddev=1., dtype=self.dtype, seed=seed) - return math_ops.abs(sampled * self.scale) - - def _prob(self, x): - coeff = np.sqrt(2) / self.scale / np.sqrt(np.pi) - pdf = coeff * math_ops.exp(- 0.5 * (x / self.scale) ** 2) - return pdf * math_ops.cast(x >= 0, self.dtype) - - def _cdf(self, x): - truncated_x = nn.relu(x) - return math_ops.erf(truncated_x / self.scale / np.sqrt(2.0)) - - def _entropy(self): - return 0.5 * math_ops.log(np.pi * self.scale ** 2.0 / 2.0) + 0.5 - - def _mean(self): - return self.scale * np.sqrt(2.0) / np.sqrt(np.pi) - - def _quantile(self, p): - return np.sqrt(2.0) * self.scale * special_math.erfinv(p) - - def _mode(self): - return array_ops.zeros(self.batch_shape_tensor()) - - def _variance(self): - return self.scale ** 2.0 * (1.0 - 2.0 / np.pi) diff --git a/tensorflow/contrib/distributions/python/ops/independent.py b/tensorflow/contrib/distributions/python/ops/independent.py deleted file mode 100644 index 7e1411ea89e..00000000000 --- a/tensorflow/contrib/distributions/python/ops/independent.py +++ /dev/null @@ -1,336 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Independent distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import distribution as distribution_lib -from tensorflow.python.ops.distributions import kullback_leibler -from tensorflow.python.util import deprecation - - -class Independent(distribution_lib.Distribution): - """Independent distribution from batch of distributions. - - This distribution is useful for regarding a collection of independent, - non-identical distributions as a single random variable. For example, the - `Independent` distribution composed of a collection of `Bernoulli` - distributions might define a distribution over an image (where each - `Bernoulli` is a distribution over each pixel). - - More precisely, a collection of `B` (independent) `E`-variate random variables - (rv) `{X_1, ..., X_B}`, can be regarded as a `[B, E]`-variate random variable - `(X_1, ..., X_B)` with probability - `p(x_1, ..., x_B) = p_1(x_1) * ... * p_B(x_B)` where `p_b(X_b)` is the - probability of the `b`-th rv. More generally `B, E` can be arbitrary shapes. - - Similarly, the `Independent` distribution specifies a distribution over `[B, - E]`-shaped events. It operates by reinterpreting the rightmost batch dims as - part of the event dimensions. The `reinterpreted_batch_ndims` parameter - controls the number of batch dims which are absorbed as event dims; - `reinterpreted_batch_ndims < len(batch_shape)`. For example, the `log_prob` - function entails a `reduce_sum` over the rightmost `reinterpreted_batch_ndims` - after calling the base distribution's `log_prob`. In other words, since the - batch dimension(s) index independent distributions, the resultant multivariate - will have independent components. - - #### Mathematical Details - - The probability function is, - - ```none - prob(x; reinterpreted_batch_ndims) = tf.reduce_prod( - dist.prob(x), - axis=-1-range(reinterpreted_batch_ndims)) - ``` - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Make independent distribution from a 2-batch Normal. - ind = tfd.Independent( - distribution=tfd.Normal(loc=[-1., 1], scale=[0.1, 0.5]), - reinterpreted_batch_ndims=1) - - # All batch dims have been "absorbed" into event dims. - ind.batch_shape # ==> [] - ind.event_shape # ==> [2] - - # Make independent distribution from a 2-batch bivariate Normal. - ind = tfd.Independent( - distribution=tfd.MultivariateNormalDiag( - loc=[[-1., 1], [1, -1]], - scale_identity_multiplier=[1., 0.5]), - reinterpreted_batch_ndims=1) - - # All batch dims have been "absorbed" into event dims. - ind.batch_shape # ==> [] - ind.event_shape # ==> [2, 2] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__( - self, distribution, reinterpreted_batch_ndims=None, - validate_args=False, name=None): - """Construct a `Independent` distribution. - - Args: - distribution: The base distribution instance to transform. Typically an - instance of `Distribution`. - reinterpreted_batch_ndims: Scalar, integer number of rightmost batch dims - which will be regarded as event dims. When `None` all but the first - batch axis (batch axis 0) will be transferred to event dimensions - (analogous to `tf.compat.v1.layers.flatten`). - validate_args: Python `bool`. Whether to validate input with asserts. - If `validate_args` is `False`, and the inputs are invalid, - correct behavior is not guaranteed. - name: The name for ops managed by the distribution. - Default value: `Independent + distribution.name`. - - Raises: - ValueError: if `reinterpreted_batch_ndims` exceeds - `distribution.batch_ndims` - """ - parameters = dict(locals()) - name = name or "Independent" + distribution.name - self._distribution = distribution - with ops.name_scope(name) as name: - if reinterpreted_batch_ndims is None: - reinterpreted_batch_ndims = self._get_default_reinterpreted_batch_ndims( - distribution) - reinterpreted_batch_ndims = ops.convert_to_tensor( - reinterpreted_batch_ndims, - dtype=dtypes.int32, - name="reinterpreted_batch_ndims") - self._reinterpreted_batch_ndims = reinterpreted_batch_ndims - self._static_reinterpreted_batch_ndims = tensor_util.constant_value( - reinterpreted_batch_ndims) - if self._static_reinterpreted_batch_ndims is not None: - self._reinterpreted_batch_ndims = self._static_reinterpreted_batch_ndims - super(Independent, self).__init__( - dtype=self._distribution.dtype, - reparameterization_type=self._distribution.reparameterization_type, - validate_args=validate_args, - allow_nan_stats=self._distribution.allow_nan_stats, - parameters=parameters, - graph_parents=( - [reinterpreted_batch_ndims] + - distribution._graph_parents), # pylint: disable=protected-access - name=name) - self._runtime_assertions = self._make_runtime_assertions( - distribution, reinterpreted_batch_ndims, validate_args) - - @property - def distribution(self): - return self._distribution - - @property - def reinterpreted_batch_ndims(self): - return self._reinterpreted_batch_ndims - - def _batch_shape_tensor(self): - with ops.control_dependencies(self._runtime_assertions): - batch_shape = self.distribution.batch_shape_tensor() - dim0 = tensor_shape.dimension_value( - batch_shape.shape.with_rank_at_least(1)[0]) - batch_ndims = (dim0 - if dim0 is not None - else array_ops.shape(batch_shape)[0]) - return batch_shape[:batch_ndims - self.reinterpreted_batch_ndims] - - def _batch_shape(self): - batch_shape = self.distribution.batch_shape - if (self._static_reinterpreted_batch_ndims is None - or batch_shape.ndims is None): - return tensor_shape.TensorShape(None) - d = batch_shape.ndims - self._static_reinterpreted_batch_ndims - return batch_shape[:d] - - def _event_shape_tensor(self): - with ops.control_dependencies(self._runtime_assertions): - batch_shape = self.distribution.batch_shape_tensor() - dim0 = tensor_shape.dimension_value( - batch_shape.shape.with_rank_at_least(1)[0]) - batch_ndims = (dim0 - if dim0 is not None - else array_ops.shape(batch_shape)[0]) - return array_ops.concat([ - batch_shape[batch_ndims - self.reinterpreted_batch_ndims:], - self.distribution.event_shape_tensor(), - ], axis=0) - - def _event_shape(self): - batch_shape = self.distribution.batch_shape - if (self._static_reinterpreted_batch_ndims is None - or batch_shape.ndims is None): - return tensor_shape.TensorShape(None) - d = batch_shape.ndims - self._static_reinterpreted_batch_ndims - return batch_shape[d:].concatenate(self.distribution.event_shape) - - def _sample_n(self, n, seed): - with ops.control_dependencies(self._runtime_assertions): - return self.distribution.sample(sample_shape=n, seed=seed) - - def _log_prob(self, x): - with ops.control_dependencies(self._runtime_assertions): - return self._reduce_sum(self.distribution.log_prob(x)) - - def _entropy(self): - with ops.control_dependencies(self._runtime_assertions): - return self._reduce_sum(self.distribution.entropy()) - - def _mean(self): - with ops.control_dependencies(self._runtime_assertions): - return self.distribution.mean() - - def _variance(self): - with ops.control_dependencies(self._runtime_assertions): - return self.distribution.variance() - - def _stddev(self): - with ops.control_dependencies(self._runtime_assertions): - return self.distribution.stddev() - - def _mode(self): - with ops.control_dependencies(self._runtime_assertions): - return self.distribution.mode() - - def _make_runtime_assertions( - self, distribution, reinterpreted_batch_ndims, validate_args): - assertions = [] - static_reinterpreted_batch_ndims = tensor_util.constant_value( - reinterpreted_batch_ndims) - batch_ndims = distribution.batch_shape.ndims - if batch_ndims is not None and static_reinterpreted_batch_ndims is not None: - if static_reinterpreted_batch_ndims > batch_ndims: - raise ValueError("reinterpreted_batch_ndims({}) cannot exceed " - "distribution.batch_ndims({})".format( - static_reinterpreted_batch_ndims, batch_ndims)) - elif validate_args: - batch_shape = distribution.batch_shape_tensor() - dim0 = tensor_shape.dimension_value( - batch_shape.shape.with_rank_at_least(1)[0]) - batch_ndims = ( - dim0 - if dim0 is not None - else array_ops.shape(batch_shape)[0]) - assertions.append(check_ops.assert_less_equal( - reinterpreted_batch_ndims, batch_ndims, - message=("reinterpreted_batch_ndims cannot exceed " - "distribution.batch_ndims"))) - return assertions - - def _reduce_sum(self, stat): - if self._static_reinterpreted_batch_ndims is None: - range_ = math_ops.range(self._reinterpreted_batch_ndims) - else: - range_ = np.arange(self._static_reinterpreted_batch_ndims) - return math_ops.reduce_sum(stat, axis=-1-range_) - - def _get_default_reinterpreted_batch_ndims(self, distribution): - """Computes the default value for reinterpreted_batch_ndim __init__ arg.""" - ndims = distribution.batch_shape.ndims - if ndims is None: - which_maximum = math_ops.maximum - ndims = array_ops.shape(distribution.batch_shape_tensor())[0] - else: - which_maximum = np.maximum - return which_maximum(0, ndims - 1) - - -@kullback_leibler.RegisterKL(Independent, Independent) -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _kl_independent(a, b, name="kl_independent"): - """Batched KL divergence `KL(a || b)` for Independent distributions. - - We can leverage the fact that - ``` - KL(Independent(a) || Independent(b)) = sum(KL(a || b)) - ``` - where the sum is over the `reinterpreted_batch_ndims`. - - Args: - a: Instance of `Independent`. - b: Instance of `Independent`. - name: (optional) name to use for created ops. Default "kl_independent". - - Returns: - Batchwise `KL(a || b)`. - - Raises: - ValueError: If the event space for `a` and `b`, or their underlying - distributions don't match. - """ - p = a.distribution - q = b.distribution - - # The KL between any two (non)-batched distributions is a scalar. - # Given that the KL between two factored distributions is the sum, i.e. - # KL(p1(x)p2(y) || q1(x)q2(y)) = KL(p1 || q1) + KL(q1 || q2), we compute - # KL(p || q) and do a `reduce_sum` on the reinterpreted batch dimensions. - if a.event_shape.is_fully_defined() and b.event_shape.is_fully_defined(): - if a.event_shape == b.event_shape: - if p.event_shape == q.event_shape: - num_reduce_dims = a.event_shape.ndims - p.event_shape.ndims - reduce_dims = [-i - 1 for i in range(0, num_reduce_dims)] - - return math_ops.reduce_sum( - kullback_leibler.kl_divergence(p, q, name=name), axis=reduce_dims) - else: - raise NotImplementedError("KL between Independents with different " - "event shapes not supported.") - else: - raise ValueError("Event shapes do not match.") - else: - with ops.control_dependencies([ - check_ops.assert_equal(a.event_shape_tensor(), b.event_shape_tensor()), - check_ops.assert_equal(p.event_shape_tensor(), q.event_shape_tensor()) - ]): - num_reduce_dims = ( - array_ops.shape(a.event_shape_tensor()[0]) - - array_ops.shape(p.event_shape_tensor()[0])) - reduce_dims = math_ops.range(-num_reduce_dims - 1, -1, 1) - return math_ops.reduce_sum( - kullback_leibler.kl_divergence(p, q, name=name), axis=reduce_dims) diff --git a/tensorflow/contrib/distributions/python/ops/inverse_gamma.py b/tensorflow/contrib/distributions/python/ops/inverse_gamma.py deleted file mode 100644 index e55b4a1457a..00000000000 --- a/tensorflow/contrib/distributions/python/ops/inverse_gamma.py +++ /dev/null @@ -1,311 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The InverseGamma distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - - -__all__ = [ - "InverseGamma", - "InverseGammaWithSoftplusConcentrationRate", -] - - -class InverseGamma(distribution.Distribution): - """InverseGamma distribution. - - The `InverseGamma` distribution is defined over positive real numbers using - parameters `concentration` (aka "alpha") and `rate` (aka "beta"). - - #### Mathematical Details - - The probability density function (pdf) is, - - ```none - pdf(x; alpha, beta, x > 0) = x**(-alpha - 1) exp(-beta / x) / Z - Z = Gamma(alpha) beta**-alpha - ``` - - where: - - * `concentration = alpha`, - * `rate = beta`, - * `Z` is the normalizing constant, and, - * `Gamma` is the [gamma function]( - https://en.wikipedia.org/wiki/Gamma_function). - - The cumulative density function (cdf) is, - - ```none - cdf(x; alpha, beta, x > 0) = GammaInc(alpha, beta / x) / Gamma(alpha) - ``` - - where `GammaInc` is the [upper incomplete Gamma function]( - https://en.wikipedia.org/wiki/Incomplete_gamma_function). - - The parameters can be intuited via their relationship to mean and stddev, - - ```none - concentration = alpha = (mean / stddev)**2 - rate = beta = mean / stddev**2 - ``` - - Distribution parameters are automatically broadcast in all functions; see - examples for details. - - WARNING: This distribution may draw 0-valued samples for small concentration - values. See note in `tf.random.gamma` docstring. - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - dist = tfd.InverseGamma(concentration=3.0, rate=2.0) - dist2 = tfd.InverseGamma(concentration=[3.0, 4.0], rate=[2.0, 3.0]) - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - concentration, - rate, - validate_args=False, - allow_nan_stats=True, - name="InverseGamma"): - """Construct InverseGamma with `concentration` and `rate` parameters. - - The parameters `concentration` and `rate` must be shaped in a way that - supports broadcasting (e.g. `concentration + rate` is a valid operation). - - Args: - concentration: Floating point tensor, the concentration params of the - distribution(s). Must contain only positive values. - rate: Floating point tensor, the inverse scale params of the - distribution(s). Must contain only positive values. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - - Raises: - TypeError: if `concentration` and `rate` are different dtypes. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[concentration, rate]) as name: - with ops.control_dependencies([ - check_ops.assert_positive(concentration), - check_ops.assert_positive(rate), - ] if validate_args else []): - self._concentration = array_ops.identity( - concentration, name="concentration") - self._rate = array_ops.identity(rate, name="rate") - check_ops.assert_same_float_dtype( - [self._concentration, self._rate]) - super(InverseGamma, self).__init__( - dtype=self._concentration.dtype, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - reparameterization_type=distribution.NOT_REPARAMETERIZED, - parameters=parameters, - graph_parents=[self._concentration, - self._rate], - name=name) - - @staticmethod - def _param_shapes(sample_shape): - return dict( - zip(("concentration", "rate"), ([ops.convert_to_tensor( - sample_shape, dtype=dtypes.int32)] * 2))) - - @property - def concentration(self): - """Concentration parameter.""" - return self._concentration - - @property - def rate(self): - """Rate parameter.""" - return self._rate - - def _batch_shape_tensor(self): - return array_ops.broadcast_dynamic_shape( - array_ops.shape(self.concentration), - array_ops.shape(self.rate)) - - def _batch_shape(self): - return array_ops.broadcast_static_shape( - self.concentration.get_shape(), - self.rate.get_shape()) - - def _event_shape_tensor(self): - return constant_op.constant([], dtype=dtypes.int32) - - def _event_shape(self): - return tensor_shape.TensorShape([]) - - @distribution_util.AppendDocstring( - """Note: See `tf.random.gamma` docstring for sampling details and - caveats.""") - def _sample_n(self, n, seed=None): - return 1. / random_ops.random_gamma( - shape=[n], - alpha=self.concentration, - beta=self.rate, - dtype=self.dtype, - seed=seed) - - def _log_prob(self, x): - return self._log_unnormalized_prob(x) - self._log_normalization() - - def _cdf(self, x): - x = self._maybe_assert_valid_sample(x) - # Note that igammac returns the upper regularized incomplete gamma - # function Q(a, x), which is what we want for the CDF. - return math_ops.igammac(self.concentration, self.rate / x) - - def _log_unnormalized_prob(self, x): - x = self._maybe_assert_valid_sample(x) - return -(1. + self.concentration) * math_ops.log(x) - self.rate / x - - def _log_normalization(self): - return (math_ops.lgamma(self.concentration) - - self.concentration * math_ops.log(self.rate)) - - def _entropy(self): - return (self.concentration - + math_ops.log(self.rate) - + math_ops.lgamma(self.concentration) - - ((1. + self.concentration) * - math_ops.digamma(self.concentration))) - - @distribution_util.AppendDocstring( - """The mean of an inverse gamma distribution is - `rate / (concentration - 1)`, when `concentration > 1`, and `NaN` - otherwise. If `self.allow_nan_stats` is `False`, an exception will be - raised rather than returning `NaN`""") - def _mean(self): - mean = self.rate / (self.concentration - 1.) - if self.allow_nan_stats: - nan = array_ops.fill( - self.batch_shape_tensor(), - np.array(np.nan, dtype=self.dtype.as_numpy_dtype()), - name="nan") - return array_ops.where_v2(self.concentration > 1., mean, nan) - else: - return control_flow_ops.with_dependencies([ - check_ops.assert_less( - array_ops.ones([], self.dtype), self.concentration, - message="mean undefined when any concentration <= 1"), - ], mean) - - @distribution_util.AppendDocstring( - """Variance for inverse gamma is defined only for `concentration > 2`. If - `self.allow_nan_stats` is `False`, an exception will be raised rather - than returning `NaN`.""") - def _variance(self): - var = ( - math_ops.square(self.rate) / math_ops.squared_difference( - self.concentration, 1.) / (self.concentration - 2.)) - if self.allow_nan_stats: - nan = array_ops.fill( - self.batch_shape_tensor(), - np.array(np.nan, dtype=self.dtype.as_numpy_dtype()), - name="nan") - return array_ops.where_v2(self.concentration > 2., var, nan) - else: - return control_flow_ops.with_dependencies([ - check_ops.assert_less( - constant_op.constant(2., dtype=self.dtype), - self.concentration, - message="variance undefined when any concentration <= 2"), - ], var) - - @distribution_util.AppendDocstring( - """The mode of an inverse gamma distribution is `rate / (concentration + - 1)`.""") - def _mode(self): - return self.rate / (1. + self.concentration) - - def _maybe_assert_valid_sample(self, x): - check_ops.assert_same_float_dtype( - tensors=[x], dtype=self.dtype) - if not self.validate_args: - return x - return control_flow_ops.with_dependencies([ - check_ops.assert_positive(x), - ], x) - - -class InverseGammaWithSoftplusConcentrationRate(InverseGamma): - """`InverseGamma` with softplus of `concentration` and `rate`.""" - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - concentration, - rate, - validate_args=False, - allow_nan_stats=True, - name="InverseGammaWithSoftplusConcentrationRate"): - parameters = dict(locals()) - with ops.name_scope(name, values=[concentration, rate]) as name: - super(InverseGammaWithSoftplusConcentrationRate, self).__init__( - concentration=nn.softplus(concentration, - name="softplus_concentration"), - rate=nn.softplus(rate, name="softplus_rate"), - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - self._parameters = parameters diff --git a/tensorflow/contrib/distributions/python/ops/kumaraswamy.py b/tensorflow/contrib/distributions/python/ops/kumaraswamy.py deleted file mode 100644 index 56f35c28b1b..00000000000 --- a/tensorflow/contrib/distributions/python/ops/kumaraswamy.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Kumaraswamy distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import special_math_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.ops.distributions import transformed_distribution -from tensorflow.python.ops.distributions import uniform -from tensorflow.python.util import deprecation - -__all__ = [ - "Kumaraswamy", -] - -_kumaraswamy_sample_note = """Note: `x` must have dtype `self.dtype` and be in -`[0, 1].` It must have a shape compatible with `self.batch_shape()`.""" - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _harmonic_number(x): - """Compute the harmonic number from its analytic continuation. - - Derivation from [here]( - https://en.wikipedia.org/wiki/Digamma_function#Relation_to_harmonic_numbers) - and [Euler's constant]( - https://en.wikipedia.org/wiki/Euler%E2%80%93Mascheroni_constant). - - Args: - x: input float. - - Returns: - z: The analytic continuation of the harmonic number for the input. - """ - one = array_ops.ones([], dtype=x.dtype) - return math_ops.digamma(x + one) - math_ops.digamma(one) - - -class Kumaraswamy(transformed_distribution.TransformedDistribution): - """Kumaraswamy distribution. - - The Kumaraswamy distribution is defined over the `(0, 1)` interval using - parameters - `concentration1` (aka "alpha") and `concentration0` (aka "beta"). It has a - shape similar to the Beta distribution, but is reparameterizeable. - - #### Mathematical Details - - The probability density function (pdf) is, - - ```none - pdf(x; alpha, beta) = alpha * beta * x**(alpha - 1) * (1 - x**alpha)**(beta - - 1) - ``` - - where: - - * `concentration1 = alpha`, - * `concentration0 = beta`, - - Distribution parameters are automatically broadcast in all functions; see - examples for details. - - #### Examples - - ```python - # Create a batch of three Kumaraswamy distributions. - alpha = [1, 2, 3] - beta = [1, 2, 3] - dist = Kumaraswamy(alpha, beta) - - dist.sample([4, 5]) # Shape [4, 5, 3] - - # `x` has three batch entries, each with two samples. - x = [[.1, .4, .5], - [.2, .3, .5]] - # Calculate the probability of each pair of samples under the corresponding - # distribution in `dist`. - dist.prob(x) # Shape [2, 3] - ``` - - ```python - # Create batch_shape=[2, 3] via parameter broadcast: - alpha = [[1.], [2]] # Shape [2, 1] - beta = [3., 4, 5] # Shape [3] - dist = Kumaraswamy(alpha, beta) - - # alpha broadcast as: [[1., 1, 1,], - # [2, 2, 2]] - # beta broadcast as: [[3., 4, 5], - # [3, 4, 5]] - # batch_Shape [2, 3] - dist.sample([4, 5]) # Shape [4, 5, 2, 3] - - x = [.2, .3, .5] - # x will be broadcast as [[.2, .3, .5], - # [.2, .3, .5]], - # thus matching batch_shape [2, 3]. - dist.prob(x) # Shape [2, 3] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - concentration1=None, - concentration0=None, - validate_args=False, - allow_nan_stats=True, - name="Kumaraswamy"): - """Initialize a batch of Kumaraswamy distributions. - - Args: - concentration1: Positive floating-point `Tensor` indicating mean - number of successes; aka "alpha". Implies `self.dtype` and - `self.batch_shape`, i.e., - `concentration1.shape = [N1, N2, ..., Nm] = self.batch_shape`. - concentration0: Positive floating-point `Tensor` indicating mean - number of failures; aka "beta". Otherwise has same semantics as - `concentration1`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - with ops.name_scope(name, values=[concentration1, concentration0]) as name: - concentration1 = ops.convert_to_tensor( - concentration1, name="concentration1") - concentration0 = ops.convert_to_tensor( - concentration0, name="concentration0") - super(Kumaraswamy, self).__init__( - distribution=uniform.Uniform( - low=array_ops.zeros([], dtype=concentration1.dtype), - high=array_ops.ones([], dtype=concentration1.dtype), - allow_nan_stats=allow_nan_stats), - bijector=bijectors.Kumaraswamy( - concentration1=concentration1, concentration0=concentration0, - validate_args=validate_args), - batch_shape=distribution_util.get_broadcast_shape( - concentration1, concentration0), - name=name) - self._reparameterization_type = distribution.FULLY_REPARAMETERIZED - - @property - def concentration1(self): - """Concentration parameter associated with a `1` outcome.""" - return self.bijector.concentration1 - - @property - def concentration0(self): - """Concentration parameter associated with a `0` outcome.""" - return self.bijector.concentration0 - - def _entropy(self): - a = self.concentration1 - b = self.concentration0 - return (1 - 1. / a) + ( - 1 - 1. / b) * _harmonic_number(b) + math_ops.log(a) + math_ops.log(b) - - def _moment(self, n): - """Compute the n'th (uncentered) moment.""" - total_concentration = self.concentration1 + self.concentration0 - expanded_concentration1 = array_ops.ones_like( - total_concentration, dtype=self.dtype) * self.concentration1 - expanded_concentration0 = array_ops.ones_like( - total_concentration, dtype=self.dtype) * self.concentration0 - beta_arg0 = 1 + n / expanded_concentration1 - beta_arg = array_ops.stack([beta_arg0, expanded_concentration0], -1) - log_moment = math_ops.log(expanded_concentration0) + special_math_ops.lbeta( - beta_arg) - return math_ops.exp(log_moment) - - def _mean(self): - return self._moment(1) - - def _variance(self): - # TODO(b/72696533): Investigate a more numerically stable version. - return self._moment(2) - math_ops.square(self._moment(1)) - - @distribution_util.AppendDocstring( - """Note: The mode is undefined when `concentration1 <= 1` or - `concentration0 <= 1`. If `self.allow_nan_stats` is `True`, `NaN` - is used for undefined modes. If `self.allow_nan_stats` is `False` an - exception is raised when one or more modes are undefined.""") - def _mode(self): - a = self.concentration1 - b = self.concentration0 - mode = ((a - 1) / (a * b - 1))**(1. / a) - if self.allow_nan_stats: - nan = array_ops.fill( - self.batch_shape_tensor(), - np.array(np.nan, dtype=self.dtype.as_numpy_dtype), - name="nan") - is_defined = (self.concentration1 > 1.) & (self.concentration0 > 1.) - return array_ops.where_v2(is_defined, mode, nan) - - return control_flow_ops.with_dependencies([ - check_ops.assert_less( - array_ops.ones([], dtype=self.concentration1.dtype), - self.concentration1, - message="Mode undefined for concentration1 <= 1."), - check_ops.assert_less( - array_ops.ones([], dtype=self.concentration0.dtype), - self.concentration0, - message="Mode undefined for concentration0 <= 1.") - ], mode) diff --git a/tensorflow/contrib/distributions/python/ops/logistic.py b/tensorflow/contrib/distributions/python/ops/logistic.py deleted file mode 100644 index 03c5ba2997a..00000000000 --- a/tensorflow/contrib/distributions/python/ops/logistic.py +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Logistic distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import numpy as np - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.util import deprecation - - -class Logistic(distribution.Distribution): - """The Logistic distribution with location `loc` and `scale` parameters. - - #### Mathematical details - - The cumulative density function of this distribution is: - - ```none - cdf(x; mu, sigma) = 1 / (1 + exp(-(x - mu) / sigma)) - ``` - - where `loc = mu` and `scale = sigma`. - - The Logistic distribution is a member of the [location-scale family]( - https://en.wikipedia.org/wiki/Location-scale_family), i.e., it can be - constructed as, - - ```none - X ~ Logistic(loc=0, scale=1) - Y = loc + scale * X - ``` - - #### Examples - - Examples of initialization of one or a batch of distributions. - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Define a single scalar Logistic distribution. - dist = tfd.Logistic(loc=0., scale=3.) - - # Evaluate the cdf at 1, returning a scalar. - dist.cdf(1.) - - # Define a batch of two scalar valued Logistics. - # The first has mean 1 and scale 11, the second 2 and 22. - dist = tfd.Logistic(loc=[1, 2.], scale=[11, 22.]) - - # Evaluate the pdf of the first distribution on 0, and the second on 1.5, - # returning a length two tensor. - dist.prob([0, 1.5]) - - # Get 3 samples, returning a 3 x 2 tensor. - dist.sample([3]) - - # Arguments are broadcast when possible. - # Define a batch of two scalar valued Logistics. - # Both have mean 1, but different scales. - dist = tfd.Logistic(loc=1., scale=[11, 22.]) - - # Evaluate the pdf of both distributions on the same point, 3.0, - # returning a length 2 tensor. - dist.prob(3.0) - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc, - scale, - validate_args=False, - allow_nan_stats=True, - name="Logistic"): - """Construct Logistic distributions with mean and scale `loc` and `scale`. - - The parameters `loc` and `scale` must be shaped in a way that supports - broadcasting (e.g. `loc + scale` is a valid operation). - - Args: - loc: Floating point tensor, the means of the distribution(s). - scale: Floating point tensor, the scales of the distribution(s). Must - contain only positive values. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: The name to give Ops created by the initializer. - - Raises: - TypeError: if loc and scale are different dtypes. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[loc, scale]) as name: - with ops.control_dependencies([check_ops.assert_positive(scale)] if - validate_args else []): - self._loc = array_ops.identity(loc, name="loc") - self._scale = array_ops.identity(scale, name="scale") - check_ops.assert_same_float_dtype([self._loc, self._scale]) - super(Logistic, self).__init__( - dtype=self._scale.dtype, - reparameterization_type=distribution.FULLY_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=[self._loc, self._scale], - name=name) - - @staticmethod - def _param_shapes(sample_shape): - return dict( - zip(("loc", "scale"), ([ops.convert_to_tensor( - sample_shape, dtype=dtypes.int32)] * 2))) - - @property - def loc(self): - """Distribution parameter for the location.""" - return self._loc - - @property - def scale(self): - """Distribution parameter for scale.""" - return self._scale - - def _batch_shape_tensor(self): - return array_ops.broadcast_dynamic_shape( - array_ops.shape(self.loc), array_ops.shape(self.scale)) - - def _batch_shape(self): - return array_ops.broadcast_static_shape( - self.loc.get_shape(), self.scale.get_shape()) - - def _event_shape_tensor(self): - return constant_op.constant([], dtype=dtypes.int32) - - def _event_shape(self): - return tensor_shape.TensorShape([]) - - def _sample_n(self, n, seed=None): - # Uniform variates must be sampled from the open-interval `(0, 1)` rather - # than `[0, 1)`. To do so, we use `np.finfo(self.dtype.as_numpy_dtype).tiny` - # because it is the smallest, positive, "normal" number. A "normal" number - # is such that the mantissa has an implicit leading 1. Normal, positive - # numbers x, y have the reasonable property that, `x + y >= max(x, y)`. In - # this case, a subnormal number (i.e., np.nextafter) can cause us to sample - # 0. - uniform = random_ops.random_uniform( - shape=array_ops.concat([[n], self.batch_shape_tensor()], 0), - minval=np.finfo(self.dtype.as_numpy_dtype).tiny, - maxval=1., - dtype=self.dtype, - seed=seed) - sampled = math_ops.log(uniform) - math_ops.log1p(-1. * uniform) - return sampled * self.scale + self.loc - - def _log_prob(self, x): - return self._log_unnormalized_prob(x) - self._log_normalization() - - def _log_cdf(self, x): - return -nn_ops.softplus(-self._z(x)) - - def _cdf(self, x): - return math_ops.sigmoid(self._z(x)) - - def _log_survival_function(self, x): - return -nn_ops.softplus(self._z(x)) - - def _survival_function(self, x): - return math_ops.sigmoid(-self._z(x)) - - def _log_unnormalized_prob(self, x): - z = self._z(x) - return - z - 2. * nn_ops.softplus(-z) - - def _log_normalization(self): - return math_ops.log(self.scale) - - def _entropy(self): - # Use broadcasting rules to calculate the full broadcast sigma. - scale = self.scale * array_ops.ones_like(self.loc) - return 2 + math_ops.log(scale) - - def _mean(self): - return self.loc * array_ops.ones_like(self.scale) - - def _stddev(self): - return self.scale * array_ops.ones_like(self.loc) * math.pi / math.sqrt(3) - - def _mode(self): - return self._mean() - - def _z(self, x): - """Standardize input `x` to a unit logistic.""" - with ops.name_scope("standardize", values=[x]): - return (x - self.loc) / self.scale diff --git a/tensorflow/contrib/distributions/python/ops/mixture.py b/tensorflow/contrib/distributions/python/ops/mixture.py deleted file mode 100644 index 52b67f2c54c..00000000000 --- a/tensorflow/contrib/distributions/python/ops/mixture.py +++ /dev/null @@ -1,503 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Mixture distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import distribution_util as distribution_utils -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops.distributions import categorical -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - - -class Mixture(distribution.Distribution): - """Mixture distribution. - - The `Mixture` object implements batched mixture distributions. - The mixture model is defined by a `Categorical` distribution (the mixture) - and a python list of `Distribution` objects. - - Methods supported include `log_prob`, `prob`, `mean`, `sample`, and - `entropy_lower_bound`. - - - #### Examples - - ```python - # Create a mixture of two Gaussians: - import tensorflow_probability as tfp - tfd = tfp.distributions - - mix = 0.3 - bimix_gauss = tfd.Mixture( - cat=tfd.Categorical(probs=[mix, 1.-mix]), - components=[ - tfd.Normal(loc=-1., scale=0.1), - tfd.Normal(loc=+1., scale=0.5), - ]) - - # Plot the PDF. - import matplotlib.pyplot as plt - x = tf.linspace(-2., 3., int(1e4)).eval() - plt.plot(x, bimix_gauss.prob(x).eval()); - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - cat, - components, - validate_args=False, - allow_nan_stats=True, - use_static_graph=False, - name="Mixture"): - """Initialize a Mixture distribution. - - A `Mixture` is defined by a `Categorical` (`cat`, representing the - mixture probabilities) and a list of `Distribution` objects - all having matching dtype, batch shape, event shape, and continuity - properties (the components). - - The `num_classes` of `cat` must be possible to infer at graph construction - time and match `len(components)`. - - Args: - cat: A `Categorical` distribution instance, representing the probabilities - of `distributions`. - components: A list or tuple of `Distribution` instances. - Each instance must have the same type, be defined on the same domain, - and have matching `event_shape` and `batch_shape`. - validate_args: Python `bool`, default `False`. If `True`, raise a runtime - error if batch or event ranks are inconsistent between cat and any of - the distributions. This is only checked if the ranks cannot be - determined statically at graph construction time. - allow_nan_stats: Boolean, default `True`. If `False`, raise an - exception if a statistic (e.g. mean/mode/etc...) is undefined for any - batch member. If `True`, batch members with valid parameters leading to - undefined statistics will return NaN for this statistic. - use_static_graph: Calls to `sample` will not rely on dynamic tensor - indexing, allowing for some static graph compilation optimizations, but - at the expense of sampling all underlying distributions in the mixture. - (Possibly useful when running on TPUs). - Default value: `False` (i.e., use dynamic indexing). - name: A name for this distribution (optional). - - Raises: - TypeError: If cat is not a `Categorical`, or `components` is not - a list or tuple, or the elements of `components` are not - instances of `Distribution`, or do not have matching `dtype`. - ValueError: If `components` is an empty list or tuple, or its - elements do not have a statically known event rank. - If `cat.num_classes` cannot be inferred at graph creation time, - or the constant value of `cat.num_classes` is not equal to - `len(components)`, or all `components` and `cat` do not have - matching static batch shapes, or all components do not - have matching static event shapes. - """ - parameters = dict(locals()) - if not isinstance(cat, categorical.Categorical): - raise TypeError("cat must be a Categorical distribution, but saw: %s" % - cat) - if not components: - raise ValueError("components must be a non-empty list or tuple") - if not isinstance(components, (list, tuple)): - raise TypeError("components must be a list or tuple, but saw: %s" % - components) - if not all(isinstance(c, distribution.Distribution) for c in components): - raise TypeError( - "all entries in components must be Distribution instances" - " but saw: %s" % components) - - dtype = components[0].dtype - if not all(d.dtype == dtype for d in components): - raise TypeError("All components must have the same dtype, but saw " - "dtypes: %s" % [(d.name, d.dtype) for d in components]) - static_event_shape = components[0].event_shape - static_batch_shape = cat.batch_shape - for d in components: - static_event_shape = static_event_shape.merge_with(d.event_shape) - static_batch_shape = static_batch_shape.merge_with(d.batch_shape) - if static_event_shape.ndims is None: - raise ValueError( - "Expected to know rank(event_shape) from components, but " - "none of the components provide a static number of ndims") - - # Ensure that all batch and event ndims are consistent. - with ops.name_scope(name, values=[cat.logits]) as name: - num_components = cat.event_size - static_num_components = tensor_util.constant_value(num_components) - if static_num_components is None: - raise ValueError( - "Could not infer number of classes from cat and unable " - "to compare this value to the number of components passed in.") - # Possibly convert from numpy 0-D array. - static_num_components = int(static_num_components) - if static_num_components != len(components): - raise ValueError("cat.num_classes != len(components): %d vs. %d" % - (static_num_components, len(components))) - - cat_batch_shape = cat.batch_shape_tensor() - cat_batch_rank = array_ops.size(cat_batch_shape) - if validate_args: - batch_shapes = [d.batch_shape_tensor() for d in components] - batch_ranks = [array_ops.size(bs) for bs in batch_shapes] - check_message = ("components[%d] batch shape must match cat " - "batch shape") - self._assertions = [ - check_ops.assert_equal( - cat_batch_rank, batch_ranks[di], message=check_message % di) - for di in range(len(components)) - ] - self._assertions += [ - check_ops.assert_equal( - cat_batch_shape, batch_shapes[di], message=check_message % di) - for di in range(len(components)) - ] - else: - self._assertions = [] - - self._cat = cat - self._components = list(components) - self._num_components = static_num_components - self._static_event_shape = static_event_shape - self._static_batch_shape = static_batch_shape - - self._use_static_graph = use_static_graph - if use_static_graph and static_num_components is None: - raise ValueError("Number of categories must be known statically when " - "`static_sample=True`.") - # We let the Mixture distribution access _graph_parents since its arguably - # more like a baseclass. - graph_parents = self._cat._graph_parents # pylint: disable=protected-access - for c in self._components: - graph_parents += c._graph_parents # pylint: disable=protected-access - - super(Mixture, self).__init__( - dtype=dtype, - reparameterization_type=distribution.NOT_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=graph_parents, - name=name) - - @property - def cat(self): - return self._cat - - @property - def components(self): - return self._components - - @property - def num_components(self): - return self._num_components - - def _batch_shape_tensor(self): - return self._cat.batch_shape_tensor() - - def _batch_shape(self): - return self._static_batch_shape - - def _event_shape_tensor(self): - return self._components[0].event_shape_tensor() - - def _event_shape(self): - return self._static_event_shape - - def _expand_to_event_rank(self, x): - """Expand the rank of x up to static_event_rank times for broadcasting. - - The static event rank was checked to not be None at construction time. - - Args: - x: A tensor to expand. - Returns: - The expanded tensor. - """ - expanded_x = x - for _ in range(self.event_shape.ndims): - expanded_x = array_ops.expand_dims(expanded_x, -1) - return expanded_x - - def _mean(self): - with ops.control_dependencies(self._assertions): - distribution_means = [d.mean() for d in self.components] - cat_probs = self._cat_probs(log_probs=False) - cat_probs = [self._expand_to_event_rank(c_p) for c_p in cat_probs] - partial_means = [ - c_p * m for (c_p, m) in zip(cat_probs, distribution_means) - ] - # These should all be the same shape by virtue of matching - # batch_shape and event_shape. - return math_ops.add_n(partial_means) - - def _stddev(self): - with ops.control_dependencies(self._assertions): - distribution_means = [d.mean() for d in self.components] - distribution_devs = [d.stddev() for d in self.components] - cat_probs = self._cat_probs(log_probs=False) - - stacked_means = array_ops.stack(distribution_means, axis=-1) - stacked_devs = array_ops.stack(distribution_devs, axis=-1) - cat_probs = [self._expand_to_event_rank(c_p) for c_p in cat_probs] - broadcasted_cat_probs = (array_ops.stack(cat_probs, axis=-1) * - array_ops.ones_like(stacked_means)) - - batched_dev = distribution_utils.mixture_stddev( - array_ops.reshape(broadcasted_cat_probs, [-1, len(self.components)]), - array_ops.reshape(stacked_means, [-1, len(self.components)]), - array_ops.reshape(stacked_devs, [-1, len(self.components)])) - - # I.e. re-shape to list(batch_shape) + list(event_shape). - return array_ops.reshape(batched_dev, - array_ops.shape(broadcasted_cat_probs)[:-1]) - - def _log_prob(self, x): - with ops.control_dependencies(self._assertions): - x = ops.convert_to_tensor(x, name="x") - distribution_log_probs = [d.log_prob(x) for d in self.components] - cat_log_probs = self._cat_probs(log_probs=True) - final_log_probs = [ - cat_lp + d_lp - for (cat_lp, d_lp) in zip(cat_log_probs, distribution_log_probs) - ] - concat_log_probs = array_ops.stack(final_log_probs, 0) - log_sum_exp = math_ops.reduce_logsumexp(concat_log_probs, [0]) - return log_sum_exp - - def _log_cdf(self, x): - with ops.control_dependencies(self._assertions): - x = ops.convert_to_tensor(x, name="x") - distribution_log_cdfs = [d.log_cdf(x) for d in self.components] - cat_log_probs = self._cat_probs(log_probs=True) - final_log_cdfs = [ - cat_lp + d_lcdf - for (cat_lp, d_lcdf) in zip(cat_log_probs, distribution_log_cdfs) - ] - concatted_log_cdfs = array_ops.stack(final_log_cdfs, axis=0) - mixture_log_cdf = math_ops.reduce_logsumexp(concatted_log_cdfs, [0]) - return mixture_log_cdf - - def _sample_n(self, n, seed=None): - if self._use_static_graph: - # This sampling approach is almost the same as the approach used by - # `MixtureSameFamily`. The differences are due to having a list of - # `Distribution` objects rather than a single object, and maintaining - # random seed management that is consistent with the non-static code path. - samples = [] - cat_samples = self.cat.sample(n, seed=seed) - for c in range(self.num_components): - seed = distribution_util.gen_new_seed(seed, "mixture") - samples.append(self.components[c].sample(n, seed=seed)) - x = array_ops.stack( - samples, -self._static_event_shape.ndims - 1) # [n, B, k, E] - npdt = x.dtype.as_numpy_dtype - mask = array_ops.one_hot( - indices=cat_samples, # [n, B] - depth=self._num_components, # == k - on_value=np.ones([], dtype=npdt), - off_value=np.zeros([], dtype=npdt)) # [n, B, k] - mask = distribution_utils.pad_mixture_dimensions( - mask, self, self._cat, - self._static_event_shape.ndims) # [n, B, k, [1]*e] - return math_ops.reduce_sum( - x * mask, - axis=-1 - self._static_event_shape.ndims) # [n, B, E] - - with ops.control_dependencies(self._assertions): - n = ops.convert_to_tensor(n, name="n") - static_n = tensor_util.constant_value(n) - n = int(static_n) if static_n is not None else n - cat_samples = self.cat.sample(n, seed=seed) - - static_samples_shape = cat_samples.get_shape() - if static_samples_shape.is_fully_defined(): - samples_shape = static_samples_shape.as_list() - samples_size = static_samples_shape.num_elements() - else: - samples_shape = array_ops.shape(cat_samples) - samples_size = array_ops.size(cat_samples) - static_batch_shape = self.batch_shape - if static_batch_shape.is_fully_defined(): - batch_shape = static_batch_shape.as_list() - batch_size = static_batch_shape.num_elements() - else: - batch_shape = self.batch_shape_tensor() - batch_size = math_ops.reduce_prod(batch_shape) - static_event_shape = self.event_shape - if static_event_shape.is_fully_defined(): - event_shape = np.array(static_event_shape.as_list(), dtype=np.int32) - else: - event_shape = self.event_shape_tensor() - - # Get indices into the raw cat sampling tensor. We will - # need these to stitch sample values back out after sampling - # within the component partitions. - samples_raw_indices = array_ops.reshape( - math_ops.range(0, samples_size), samples_shape) - - # Partition the raw indices so that we can use - # dynamic_stitch later to reconstruct the samples from the - # known partitions. - partitioned_samples_indices = data_flow_ops.dynamic_partition( - data=samples_raw_indices, - partitions=cat_samples, - num_partitions=self.num_components) - - # Copy the batch indices n times, as we will need to know - # these to pull out the appropriate rows within the - # component partitions. - batch_raw_indices = array_ops.reshape( - array_ops.tile(math_ops.range(0, batch_size), [n]), samples_shape) - - # Explanation of the dynamic partitioning below: - # batch indices are i.e., [0, 1, 0, 1, 0, 1] - # Suppose partitions are: - # [1 1 0 0 1 1] - # After partitioning, batch indices are cut as: - # [batch_indices[x] for x in 2, 3] - # [batch_indices[x] for x in 0, 1, 4, 5] - # i.e. - # [1 1] and [0 0 0 0] - # Now we sample n=2 from part 0 and n=4 from part 1. - # For part 0 we want samples from batch entries 1, 1 (samples 0, 1), - # and for part 1 we want samples from batch entries 0, 0, 0, 0 - # (samples 0, 1, 2, 3). - partitioned_batch_indices = data_flow_ops.dynamic_partition( - data=batch_raw_indices, - partitions=cat_samples, - num_partitions=self.num_components) - samples_class = [None for _ in range(self.num_components)] - - for c in range(self.num_components): - n_class = array_ops.size(partitioned_samples_indices[c]) - seed = distribution_util.gen_new_seed(seed, "mixture") - samples_class_c = self.components[c].sample(n_class, seed=seed) - - # Pull out the correct batch entries from each index. - # To do this, we may have to flatten the batch shape. - - # For sample s, batch element b of component c, we get the - # partitioned batch indices from - # partitioned_batch_indices[c]; and shift each element by - # the sample index. The final lookup can be thought of as - # a matrix gather along locations (s, b) in - # samples_class_c where the n_class rows correspond to - # samples within this component and the batch_size columns - # correspond to batch elements within the component. - # - # Thus the lookup index is - # lookup[c, i] = batch_size * s[i] + b[c, i] - # for i = 0 ... n_class[c] - 1. - lookup_partitioned_batch_indices = ( - batch_size * math_ops.range(n_class) + - partitioned_batch_indices[c]) - samples_class_c = array_ops.reshape( - samples_class_c, - array_ops.concat([[n_class * batch_size], event_shape], 0)) - samples_class_c = array_ops.gather( - samples_class_c, lookup_partitioned_batch_indices, - name="samples_class_c_gather") - samples_class[c] = samples_class_c - - # Stitch back together the samples across the components. - lhs_flat_ret = data_flow_ops.dynamic_stitch( - indices=partitioned_samples_indices, data=samples_class) - # Reshape back to proper sample, batch, and event shape. - ret = array_ops.reshape(lhs_flat_ret, - array_ops.concat([samples_shape, - self.event_shape_tensor()], 0)) - ret.set_shape( - tensor_shape.TensorShape(static_samples_shape).concatenate( - self.event_shape)) - return ret - - def entropy_lower_bound(self, name="entropy_lower_bound"): - r"""A lower bound on the entropy of this mixture model. - - The bound below is not always very tight, and its usefulness depends - on the mixture probabilities and the components in use. - - A lower bound is useful for ELBO when the `Mixture` is the variational - distribution: - - \\( - \log p(x) >= ELBO = \int q(z) \log p(x, z) dz + H[q] - \\) - - where \\( p \\) is the prior distribution, \\( q \\) is the variational, - and \\( H[q] \\) is the entropy of \\( q \\). If there is a lower bound - \\( G[q] \\) such that \\( H[q] \geq G[q] \\) then it can be used in - place of \\( H[q] \\). - - For a mixture of distributions \\( q(Z) = \sum_i c_i q_i(Z) \\) with - \\( \sum_i c_i = 1 \\), by the concavity of \\( f(x) = -x \log x \\), a - simple lower bound is: - - \\( - \begin{align} - H[q] & = - \int q(z) \log q(z) dz \\\ - & = - \int (\sum_i c_i q_i(z)) \log(\sum_i c_i q_i(z)) dz \\\ - & \geq - \sum_i c_i \int q_i(z) \log q_i(z) dz \\\ - & = \sum_i c_i H[q_i] - \end{align} - \\) - - This is the term we calculate below for \\( G[q] \\). - - Args: - name: A name for this operation (optional). - - Returns: - A lower bound on the Mixture's entropy. - """ - with self._name_scope(name, values=[self.cat.logits]): - with ops.control_dependencies(self._assertions): - distribution_entropies = [d.entropy() for d in self.components] - cat_probs = self._cat_probs(log_probs=False) - partial_entropies = [ - c_p * m for (c_p, m) in zip(cat_probs, distribution_entropies) - ] - # These are all the same shape by virtue of matching batch_shape - return math_ops.add_n(partial_entropies) - - def _cat_probs(self, log_probs): - """Get a list of num_components batchwise probabilities.""" - which_softmax = nn_ops.log_softmax if log_probs else nn_ops.softmax - cat_probs = which_softmax(self.cat.logits) - cat_probs = array_ops.unstack(cat_probs, num=self.num_components, axis=-1) - return cat_probs diff --git a/tensorflow/contrib/distributions/python/ops/mixture_same_family.py b/tensorflow/contrib/distributions/python/ops/mixture_same_family.py deleted file mode 100644 index f34317f5abf..00000000000 --- a/tensorflow/contrib/distributions/python/ops/mixture_same_family.py +++ /dev/null @@ -1,349 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The same-family Mixture distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import distribution_util as distribution_utils -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - - -class MixtureSameFamily(distribution.Distribution): - """Mixture (same-family) distribution. - - The `MixtureSameFamily` distribution implements a (batch of) mixture - distribution where all components are from different parameterizations of the - same distribution type. It is parameterized by a `Categorical` "selecting - distribution" (over `k` components) and a components distribution, i.e., a - `Distribution` with a rightmost batch shape (equal to `[k]`) which indexes - each (batch of) component. - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - ### Create a mixture of two scalar Gaussians: - - gm = tfd.MixtureSameFamily( - mixture_distribution=tfd.Categorical( - probs=[0.3, 0.7]), - components_distribution=tfd.Normal( - loc=[-1., 1], # One for each component. - scale=[0.1, 0.5])) # And same here. - - gm.mean() - # ==> 0.4 - - gm.variance() - # ==> 1.018 - - # Plot PDF. - x = np.linspace(-2., 3., int(1e4), dtype=np.float32) - import matplotlib.pyplot as plt - plt.plot(x, gm.prob(x).eval()); - - ### Create a mixture of two Bivariate Gaussians: - - gm = tfd.MixtureSameFamily( - mixture_distribution=tfd.Categorical( - probs=[0.3, 0.7]), - components_distribution=tfd.MultivariateNormalDiag( - loc=[[-1., 1], # component 1 - [1, -1]], # component 2 - scale_identity_multiplier=[.3, .6])) - - gm.mean() - # ==> array([ 0.4, -0.4], dtype=float32) - - gm.covariance() - # ==> array([[ 1.119, -0.84], - # [-0.84, 1.119]], dtype=float32) - - # Plot PDF contours. - def meshgrid(x, y=x): - [gx, gy] = np.meshgrid(x, y, indexing='ij') - gx, gy = np.float32(gx), np.float32(gy) - grid = np.concatenate([gx.ravel()[None, :], gy.ravel()[None, :]], axis=0) - return grid.T.reshape(x.size, y.size, 2) - grid = meshgrid(np.linspace(-2, 2, 100, dtype=np.float32)) - plt.contour(grid[..., 0], grid[..., 1], gm.prob(grid).eval()); - - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - mixture_distribution, - components_distribution, - validate_args=False, - allow_nan_stats=True, - name="MixtureSameFamily"): - """Construct a `MixtureSameFamily` distribution. - - Args: - mixture_distribution: `tfp.distributions.Categorical`-like instance. - Manages the probability of selecting components. The number of - categories must match the rightmost batch dimension of the - `components_distribution`. Must have either scalar `batch_shape` or - `batch_shape` matching `components_distribution.batch_shape[:-1]`. - components_distribution: `tfp.distributions.Distribution`-like instance. - Right-most batch dimension indexes components. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - ValueError: `if not mixture_distribution.dtype.is_integer`. - ValueError: if mixture_distribution does not have scalar `event_shape`. - ValueError: if `mixture_distribution.batch_shape` and - `components_distribution.batch_shape[:-1]` are both fully defined and - the former is neither scalar nor equal to the latter. - ValueError: if `mixture_distribution` categories does not equal - `components_distribution` rightmost batch shape. - """ - parameters = dict(locals()) - with ops.name_scope(name) as name: - self._mixture_distribution = mixture_distribution - self._components_distribution = components_distribution - self._runtime_assertions = [] - - s = components_distribution.event_shape_tensor() - s_dim0 = tensor_shape.dimension_value(s.shape[0]) - self._event_ndims = (s_dim0 - if s_dim0 is not None - else array_ops.shape(s)[0]) - - if not mixture_distribution.dtype.is_integer: - raise ValueError( - "`mixture_distribution.dtype` ({}) is not over integers".format( - mixture_distribution.dtype.name)) - - if (mixture_distribution.event_shape.ndims is not None - and mixture_distribution.event_shape.ndims != 0): - raise ValueError("`mixture_distribution` must have scalar `event_dim`s") - elif validate_args: - self._runtime_assertions += [ - control_flow_ops.assert_has_rank( - mixture_distribution.event_shape_tensor(), 0, - message="`mixture_distribution` must have scalar `event_dim`s"), - ] - - mdbs = mixture_distribution.batch_shape - cdbs = components_distribution.batch_shape.with_rank_at_least(1)[:-1] - if mdbs.is_fully_defined() and cdbs.is_fully_defined(): - if mdbs.ndims != 0 and mdbs != cdbs: - raise ValueError( - "`mixture_distribution.batch_shape` (`{}`) is not " - "compatible with `components_distribution.batch_shape` " - "(`{}`)".format(mdbs.as_list(), cdbs.as_list())) - elif validate_args: - mdbs = mixture_distribution.batch_shape_tensor() - cdbs = components_distribution.batch_shape_tensor()[:-1] - self._runtime_assertions += [ - control_flow_ops.assert_equal( - distribution_util.pick_vector( - mixture_distribution.is_scalar_batch(), cdbs, mdbs), - cdbs, - message=( - "`mixture_distribution.batch_shape` is not " - "compatible with `components_distribution.batch_shape`"))] - - km = tensor_shape.dimension_value( - mixture_distribution.logits.shape.with_rank_at_least(1)[-1]) - kc = tensor_shape.dimension_value( - components_distribution.batch_shape.with_rank_at_least(1)[-1]) - if km is not None and kc is not None and km != kc: - raise ValueError("`mixture_distribution components` ({}) does not " - "equal `components_distribution.batch_shape[-1]` " - "({})".format(km, kc)) - elif validate_args: - km = array_ops.shape(mixture_distribution.logits)[-1] - kc = components_distribution.batch_shape_tensor()[-1] - self._runtime_assertions += [ - control_flow_ops.assert_equal( - km, kc, - message=("`mixture_distribution components` does not equal " - "`components_distribution.batch_shape[-1:]`")), - ] - elif km is None: - km = array_ops.shape(mixture_distribution.logits)[-1] - - self._num_components = km - - super(MixtureSameFamily, self).__init__( - dtype=self._components_distribution.dtype, - reparameterization_type=distribution.NOT_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=( - self._mixture_distribution._graph_parents # pylint: disable=protected-access - + self._components_distribution._graph_parents), # pylint: disable=protected-access - name=name) - - @property - def mixture_distribution(self): - return self._mixture_distribution - - @property - def components_distribution(self): - return self._components_distribution - - def _batch_shape_tensor(self): - with ops.control_dependencies(self._runtime_assertions): - return self.components_distribution.batch_shape_tensor()[:-1] - - def _batch_shape(self): - return self.components_distribution.batch_shape.with_rank_at_least(1)[:-1] - - def _event_shape_tensor(self): - with ops.control_dependencies(self._runtime_assertions): - return self.components_distribution.event_shape_tensor() - - def _event_shape(self): - return self.components_distribution.event_shape - - def _sample_n(self, n, seed): - with ops.control_dependencies(self._runtime_assertions): - x = self.components_distribution.sample(n) # [n, B, k, E] - # TODO(jvdillon): Consider using tf.gather (by way of index unrolling). - npdt = x.dtype.as_numpy_dtype - mask = array_ops.one_hot( - indices=self.mixture_distribution.sample(n), # [n, B] - depth=self._num_components, # == k - on_value=np.ones([], dtype=npdt), - off_value=np.zeros([], dtype=npdt)) # [n, B, k] - mask = distribution_utils.pad_mixture_dimensions( - mask, self, self.mixture_distribution, - self._event_shape().ndims) # [n, B, k, [1]*e] - return math_ops.reduce_sum( - x * mask, axis=-1 - self._event_ndims) # [n, B, E] - - def _log_prob(self, x): - with ops.control_dependencies(self._runtime_assertions): - x = self._pad_sample_dims(x) - log_prob_x = self.components_distribution.log_prob(x) # [S, B, k] - log_mix_prob = nn_ops.log_softmax( - self.mixture_distribution.logits, axis=-1) # [B, k] - return math_ops.reduce_logsumexp( - log_prob_x + log_mix_prob, axis=-1) # [S, B] - - def _mean(self): - with ops.control_dependencies(self._runtime_assertions): - probs = distribution_utils.pad_mixture_dimensions( - self.mixture_distribution.probs, self, self.mixture_distribution, - self._event_shape().ndims) # [B, k, [1]*e] - return math_ops.reduce_sum( - probs * self.components_distribution.mean(), - axis=-1 - self._event_ndims) # [B, E] - - def _log_cdf(self, x): - x = self._pad_sample_dims(x) - log_cdf_x = self.components_distribution.log_cdf(x) # [S, B, k] - log_mix_prob = nn_ops.log_softmax( - self.mixture_distribution.logits, axis=-1) # [B, k] - return math_ops.reduce_logsumexp( - log_cdf_x + log_mix_prob, axis=-1) # [S, B] - - def _variance(self): - with ops.control_dependencies(self._runtime_assertions): - # Law of total variance: Var(Y) = E[Var(Y|X)] + Var(E[Y|X]) - probs = distribution_utils.pad_mixture_dimensions( - self.mixture_distribution.probs, self, self.mixture_distribution, - self._event_shape().ndims) # [B, k, [1]*e] - mean_cond_var = math_ops.reduce_sum( - probs * self.components_distribution.variance(), - axis=-1 - self._event_ndims) # [B, E] - var_cond_mean = math_ops.reduce_sum( - probs * math_ops.squared_difference( - self.components_distribution.mean(), - self._pad_sample_dims(self._mean())), - axis=-1 - self._event_ndims) # [B, E] - return mean_cond_var + var_cond_mean # [B, E] - - def _covariance(self): - static_event_ndims = self.event_shape.ndims - if static_event_ndims != 1: - # Covariance is defined only for vector distributions. - raise NotImplementedError("covariance is not implemented") - - with ops.control_dependencies(self._runtime_assertions): - # Law of total variance: Var(Y) = E[Var(Y|X)] + Var(E[Y|X]) - probs = distribution_utils.pad_mixture_dimensions( - distribution_utils.pad_mixture_dimensions( - self.mixture_distribution.probs, self, self.mixture_distribution, - self._event_shape().ndims), - self, self.mixture_distribution, - self._event_shape().ndims) # [B, k, 1, 1] - mean_cond_var = math_ops.reduce_sum( - probs * self.components_distribution.covariance(), - axis=-3) # [B, e, e] - var_cond_mean = math_ops.reduce_sum( - probs * _outer_squared_difference( - self.components_distribution.mean(), - self._pad_sample_dims(self._mean())), - axis=-3) # [B, e, e] - return mean_cond_var + var_cond_mean # [B, e, e] - - def _pad_sample_dims(self, x): - with ops.name_scope("pad_sample_dims", values=[x]): - ndims = x.shape.ndims if x.shape.ndims is not None else array_ops.rank(x) - shape = array_ops.shape(x) - d = ndims - self._event_ndims - x = array_ops.reshape(x, shape=array_ops.concat([ - shape[:d], [1], shape[d:]], axis=0)) - return x - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _outer_squared_difference(x, y): - """Convenience function analogous to tf.squared_difference.""" - z = x - y - return z[..., array_ops.newaxis, :] * z[..., array_ops.newaxis] diff --git a/tensorflow/contrib/distributions/python/ops/moving_stats.py b/tensorflow/contrib/distributions/python/ops/moving_stats.py deleted file mode 100644 index 87d40805a3c..00000000000 --- a/tensorflow/contrib/distributions/python/ops/moving_stats.py +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functions for computing moving statistics.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope - - -__all__ = [ - "assign_moving_mean_variance", - "assign_log_moving_mean_exp", - "moving_mean_variance", -] - - -def assign_moving_mean_variance( - mean_var, variance_var, value, decay, name=None): - """Compute exponentially weighted moving {mean,variance} of a streaming value. - - The `value` updated exponentially weighted moving `mean_var` and - `variance_var` are given by the following recurrence relations: - - ```python - variance_var = decay * (variance_var + (1-decay) * (value - mean_var)**2) - mean_var = decay * mean_var + (1 - decay) * value - ``` - - Note: `mean_var` is updated *after* `variance_var`, i.e., `variance_var` uses - the lag-1 mean. - - For derivation justification, see [Finch (2009; Eq. 143)][1]. - - Args: - mean_var: `float`-like `Variable` representing the exponentially weighted - moving mean. Same shape as `variance_var` and `value`. - variance_var: `float`-like `Variable` representing the - exponentially weighted moving variance. Same shape as `mean_var` and - `value`. - value: `float`-like `Tensor`. Same shape as `mean_var` and `variance_var`. - decay: A `float`-like `Tensor`. The moving mean decay. Typically close to - `1.`, e.g., `0.999`. - name: Optional name of the returned operation. - - Returns: - mean_var: `Variable` representing the `value`-updated exponentially weighted - moving mean. - variance_var: `Variable` representing the `value`-updated - exponentially weighted moving variance. - - Raises: - TypeError: if `mean_var` does not have float type `dtype`. - TypeError: if `mean_var`, `variance_var`, `value`, `decay` have different - `base_dtype`. - - #### References - - [1]: Tony Finch. Incremental calculation of weighted mean and variance. - _Technical Report_, 2009. - http://people.ds.cam.ac.uk/fanf2/hermes/doc/antiforgery/stats.pdf - """ - with ops.name_scope(name, "assign_moving_mean_variance", - [variance_var, mean_var, value, decay]): - with ops.colocate_with(variance_var): - with ops.colocate_with(mean_var): - base_dtype = mean_var.dtype.base_dtype - if not base_dtype.is_floating: - raise TypeError( - "mean_var.base_dtype({}) does not have float type " - "`dtype`.".format(base_dtype.name)) - if base_dtype != variance_var.dtype.base_dtype: - raise TypeError( - "mean_var.base_dtype({}) != variance_var.base_dtype({})".format( - base_dtype.name, - variance_var.dtype.base_dtype.name)) - value = ops.convert_to_tensor(value, dtype=base_dtype, name="value") - decay = ops.convert_to_tensor(decay, dtype=base_dtype, name="decay") - delta = value - mean_var - with ops.control_dependencies([delta]): - mean_var = state_ops.assign_add( - mean_var, - (1. - decay) * delta) - variance_var = state_ops.assign_sub( - variance_var, - (1. - decay) * (variance_var - decay * math_ops.square(delta))) - return mean_var, variance_var - - -def assign_log_moving_mean_exp( - log_mean_exp_var, log_value, decay, name=None): - """Compute the log of the exponentially weighted moving mean of the exp. - - If `log_value` is a draw from a stationary random variable, this function - approximates `log(E[exp(log_value)])`, i.e., a weighted log-sum-exp. More - precisely, a `tf.Variable`, `log_mean_exp_var`, is updated by `log_value` - using the following identity: - - ```none - log_mean_exp_var = - = log(decay exp(log_mean_exp_var) + (1 - decay) exp(log_value)) - = log(exp(log_mean_exp_var + log(decay)) + exp(log_value + log1p(-decay))) - = log_mean_exp_var - + log( exp(log_mean_exp_var - log_mean_exp_var + log(decay)) - + exp(log_value - log_mean_exp_var + log1p(-decay))) - = log_mean_exp_var - + log_sum_exp([log(decay), log_value - log_mean_exp_var + log1p(-decay)]). - ``` - - In addition to numerical stability, this formulation is advantageous because - `log_mean_exp_var` can be updated in a lock-free manner, i.e., using - `assign_add`. (Note: the updates are not thread-safe; it's just that the - update to the tf.Variable is presumed efficient due to being lock-free.) - - Args: - log_mean_exp_var: `float`-like `Variable` representing the log of the - exponentially weighted moving mean of the exp. Same shape as `log_value`. - log_value: `float`-like `Tensor` representing a new (streaming) observation. - Same shape as `log_mean_exp_var`. - decay: A `float`-like `Tensor`. The moving mean decay. Typically close to - `1.`, e.g., `0.999`. - name: Optional name of the returned operation. - - Returns: - log_mean_exp_var: A reference to the input 'Variable' tensor with the - `log_value`-updated log of the exponentially weighted moving mean of exp. - - Raises: - TypeError: if `log_mean_exp_var` does not have float type `dtype`. - TypeError: if `log_mean_exp_var`, `log_value`, `decay` have different - `base_dtype`. - """ - with ops.name_scope(name, "assign_log_moving_mean_exp", - [log_mean_exp_var, log_value, decay]): - # We want to update the variable in a numerically stable and lock-free way. - # To do this, observe that variable `x` updated by `v` is: - # x = log(w exp(x) + (1-w) exp(v)) - # = log(exp(x + log(w)) + exp(v + log1p(-w))) - # = x + log(exp(x - x + log(w)) + exp(v - x + log1p(-w))) - # = x + lse([log(w), v - x + log1p(-w)]) - with ops.colocate_with(log_mean_exp_var): - base_dtype = log_mean_exp_var.dtype.base_dtype - if not base_dtype.is_floating: - raise TypeError( - "log_mean_exp_var.base_dtype({}) does not have float type " - "`dtype`.".format(base_dtype.name)) - log_value = ops.convert_to_tensor(log_value, dtype=base_dtype, - name="log_value") - decay = ops.convert_to_tensor(decay, dtype=base_dtype, name="decay") - delta = (log_value - log_mean_exp_var)[array_ops.newaxis, ...] - x = array_ops.concat([ - math_ops.log(decay) * array_ops.ones_like(delta), - delta + math_ops.log1p(-decay) - ], axis=0) - x = math_ops.reduce_logsumexp(x, axis=0) - return log_mean_exp_var.assign_add(x) - - -def moving_mean_variance(value, decay, collections=None, name=None): - """Compute exponentially weighted moving {mean,variance} of a streaming value. - - The exponentially-weighting moving `mean_var` and `variance_var` are updated - by `value` according to the following recurrence: - - ```python - variance_var = decay * (variance_var + (1-decay) * (value - mean_var)**2) - mean_var = decay * mean_var + (1 - decay) * value - ``` - - Note: `mean_var` is updated *after* `variance_var`, i.e., `variance_var` uses - the lag-`1` mean. - - For derivation justification, see [Finch (2009; Eq. 143)][1]. - - Unlike `assign_moving_mean_variance`, this function handles - variable creation. - - Args: - value: `float`-like `Tensor`. Same shape as `mean_var` and `variance_var`. - decay: A `float`-like `Tensor`. The moving mean decay. Typically close to - `1.`, e.g., `0.999`. - collections: Python list of graph-collections keys to which the internal - variables `mean_var` and `variance_var` are added. - Default value is `[GraphKeys.GLOBAL_VARIABLES]`. - name: Optional name of the returned operation. - - Returns: - mean_var: `Variable` representing the `value`-updated exponentially weighted - moving mean. - variance_var: `Variable` representing the `value`-updated - exponentially weighted moving variance. - - Raises: - TypeError: if `value_var` does not have float type `dtype`. - TypeError: if `value`, `decay` have different `base_dtype`. - - #### References - - [1]: Tony Finch. Incremental calculation of weighted mean and variance. - _Technical Report_, 2009. - http://people.ds.cam.ac.uk/fanf2/hermes/doc/antiforgery/stats.pdf - """ - if collections is None: - collections = [ops.GraphKeys.GLOBAL_VARIABLES] - with variable_scope.variable_scope( - name, "moving_mean_variance", [value, decay]): - value = ops.convert_to_tensor(value, name="value") - base_dtype = value.dtype.base_dtype - if not base_dtype.is_floating: - raise TypeError( - "value.base_dtype({}) does not have float type `dtype`.".format( - base_dtype.name)) - decay = ops.convert_to_tensor(decay, dtype=base_dtype, name="decay") - variance_var = variable_scope.get_variable( - "moving_variance", - shape=value.shape, - dtype=value.dtype, - initializer=init_ops.zeros_initializer(), - trainable=False, - collections=collections) - mean_var = variable_scope.get_variable( - "moving_mean", - shape=value.shape, - dtype=value.dtype, - initializer=init_ops.zeros_initializer(), - trainable=False, - collections=collections) - return assign_moving_mean_variance( - mean_var, variance_var, value, decay) diff --git a/tensorflow/contrib/distributions/python/ops/mvn_diag.py b/tensorflow/contrib/distributions/python/ops/mvn_diag.py deleted file mode 100644 index 0b5b76be923..00000000000 --- a/tensorflow/contrib/distributions/python/ops/mvn_diag.py +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Multivariate Normal distribution classes.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.contrib.distributions.python.ops import mvn_linear_operator as mvn_linop -from tensorflow.python.framework import ops -from tensorflow.python.ops import nn -from tensorflow.python.util import deprecation - - -__all__ = [ - "MultivariateNormalDiag", - "MultivariateNormalDiagWithSoftplusScale", -] - - -class MultivariateNormalDiag( - mvn_linop.MultivariateNormalLinearOperator): - """The multivariate normal distribution on `R^k`. - - The Multivariate Normal distribution is defined over `R^k` and parameterized - by a (batch of) length-`k` `loc` vector (aka "mu") and a (batch of) `k x k` - `scale` matrix; `covariance = scale @ scale.T` where `@` denotes - matrix-multiplication. - - #### Mathematical Details - - The probability density function (pdf) is, - - ```none - pdf(x; loc, scale) = exp(-0.5 ||y||**2) / Z, - y = inv(scale) @ (x - loc), - Z = (2 pi)**(0.5 k) |det(scale)|, - ``` - - where: - - * `loc` is a vector in `R^k`, - * `scale` is a linear operator in `R^{k x k}`, `cov = scale @ scale.T`, - * `Z` denotes the normalization constant, and, - * `||y||**2` denotes the squared Euclidean norm of `y`. - - A (non-batch) `scale` matrix is: - - ```none - scale = diag(scale_diag + scale_identity_multiplier * ones(k)) - ``` - - where: - - * `scale_diag.shape = [k]`, and, - * `scale_identity_multiplier.shape = []`. - - Additional leading dimensions (if any) will index batches. - - If both `scale_diag` and `scale_identity_multiplier` are `None`, then - `scale` is the Identity matrix. - - The MultivariateNormal distribution is a member of the [location-scale - family](https://en.wikipedia.org/wiki/Location-scale_family), i.e., it can be - constructed as, - - ```none - X ~ MultivariateNormal(loc=0, scale=1) # Identity scale, zero shift. - Y = scale @ X + loc - ``` - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single 2-variate Gaussian. - mvn = tfd.MultivariateNormalDiag( - loc=[1., -1], - scale_diag=[1, 2.]) - - mvn.mean().eval() - # ==> [1., -1] - - mvn.stddev().eval() - # ==> [1., 2] - - # Evaluate this on an observation in `R^2`, returning a scalar. - mvn.prob([-1., 0]).eval() # shape: [] - - # Initialize a 3-batch, 2-variate scaled-identity Gaussian. - mvn = tfd.MultivariateNormalDiag( - loc=[1., -1], - scale_identity_multiplier=[1, 2., 3]) - - mvn.mean().eval() # shape: [3, 2] - # ==> [[1., -1] - # [1, -1], - # [1, -1]] - - mvn.stddev().eval() # shape: [3, 2] - # ==> [[1., 1], - # [2, 2], - # [3, 3]] - - # Evaluate this on an observation in `R^2`, returning a length-3 vector. - mvn.prob([-1., 0]).eval() # shape: [3] - - # Initialize a 2-batch of 3-variate Gaussians. - mvn = tfd.MultivariateNormalDiag( - loc=[[1., 2, 3], - [11, 22, 33]] # shape: [2, 3] - scale_diag=[[1., 2, 3], - [0.5, 1, 1.5]]) # shape: [2, 3] - - # Evaluate this on a two observations, each in `R^3`, returning a length-2 - # vector. - x = [[-1., 0, 1], - [-11, 0, 11.]] # shape: [2, 3]. - mvn.prob(x).eval() # shape: [2] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc=None, - scale_diag=None, - scale_identity_multiplier=None, - validate_args=False, - allow_nan_stats=True, - name="MultivariateNormalDiag"): - """Construct Multivariate Normal distribution on `R^k`. - - The `batch_shape` is the broadcast shape between `loc` and `scale` - arguments. - - The `event_shape` is given by last dimension of the matrix implied by - `scale`. The last dimension of `loc` (if provided) must broadcast with this. - - Recall that `covariance = scale @ scale.T`. A (non-batch) `scale` matrix is: - - ```none - scale = diag(scale_diag + scale_identity_multiplier * ones(k)) - ``` - - where: - - * `scale_diag.shape = [k]`, and, - * `scale_identity_multiplier.shape = []`. - - Additional leading dimensions (if any) will index batches. - - If both `scale_diag` and `scale_identity_multiplier` are `None`, then - `scale` is the Identity matrix. - - Args: - loc: Floating-point `Tensor`. If this is set to `None`, `loc` is - implicitly `0`. When specified, may have shape `[B1, ..., Bb, k]` where - `b >= 0` and `k` is the event size. - scale_diag: Non-zero, floating-point `Tensor` representing a diagonal - matrix added to `scale`. May have shape `[B1, ..., Bb, k]`, `b >= 0`, - and characterizes `b`-batches of `k x k` diagonal matrices added to - `scale`. When both `scale_identity_multiplier` and `scale_diag` are - `None` then `scale` is the `Identity`. - scale_identity_multiplier: Non-zero, floating-point `Tensor` representing - a scaled-identity-matrix added to `scale`. May have shape - `[B1, ..., Bb]`, `b >= 0`, and characterizes `b`-batches of scaled - `k x k` identity matrices added to `scale`. When both - `scale_identity_multiplier` and `scale_diag` are `None` then `scale` is - the `Identity`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, - statistics (e.g., mean, mode, variance) use the value "`NaN`" to - indicate the result is undefined. When `False`, an exception is raised - if one or more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - ValueError: if at most `scale_identity_multiplier` is specified. - """ - parameters = dict(locals()) - with ops.name_scope(name) as name: - with ops.name_scope("init", values=[ - loc, scale_diag, scale_identity_multiplier]): - # No need to validate_args while making diag_scale. The returned - # LinearOperatorDiag has an assert_non_singular method that is called by - # the Bijector. - scale = distribution_util.make_diag_scale( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - validate_args=False, - assert_positive=False) - super(MultivariateNormalDiag, self).__init__( - loc=loc, - scale=scale, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - self._parameters = parameters - - -class MultivariateNormalDiagWithSoftplusScale(MultivariateNormalDiag): - """MultivariateNormalDiag with `diag_stddev = softplus(diag_stddev)`.""" - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc, - scale_diag, - validate_args=False, - allow_nan_stats=True, - name="MultivariateNormalDiagWithSoftplusScale"): - parameters = dict(locals()) - with ops.name_scope(name, values=[scale_diag]) as name: - super(MultivariateNormalDiagWithSoftplusScale, self).__init__( - loc=loc, - scale_diag=nn.softplus(scale_diag), - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - self._parameters = parameters diff --git a/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py b/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py deleted file mode 100644 index 80546083d3f..00000000000 --- a/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py +++ /dev/null @@ -1,265 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Multivariate Normal distribution classes.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.contrib.distributions.python.ops import mvn_linear_operator as mvn_linop -from tensorflow.python.framework import ops -from tensorflow.python.ops.linalg import linalg -from tensorflow.python.util import deprecation - - -__all__ = [ - "MultivariateNormalDiagPlusLowRank", -] - - -class MultivariateNormalDiagPlusLowRank( - mvn_linop.MultivariateNormalLinearOperator): - """The multivariate normal distribution on `R^k`. - - The Multivariate Normal distribution is defined over `R^k` and parameterized - by a (batch of) length-`k` `loc` vector (aka "mu") and a (batch of) `k x k` - `scale` matrix; `covariance = scale @ scale.T` where `@` denotes - matrix-multiplication. - - #### Mathematical Details - - The probability density function (pdf) is, - - ```none - pdf(x; loc, scale) = exp(-0.5 ||y||**2) / Z, - y = inv(scale) @ (x - loc), - Z = (2 pi)**(0.5 k) |det(scale)|, - ``` - - where: - - * `loc` is a vector in `R^k`, - * `scale` is a linear operator in `R^{k x k}`, `cov = scale @ scale.T`, - * `Z` denotes the normalization constant, and, - * `||y||**2` denotes the squared Euclidean norm of `y`. - - A (non-batch) `scale` matrix is: - - ```none - scale = diag(scale_diag + scale_identity_multiplier ones(k)) + - scale_perturb_factor @ diag(scale_perturb_diag) @ scale_perturb_factor.T - ``` - - where: - - * `scale_diag.shape = [k]`, - * `scale_identity_multiplier.shape = []`, - * `scale_perturb_factor.shape = [k, r]`, typically `k >> r`, and, - * `scale_perturb_diag.shape = [r]`. - - Additional leading dimensions (if any) will index batches. - - If both `scale_diag` and `scale_identity_multiplier` are `None`, then - `scale` is the Identity matrix. - - The MultivariateNormal distribution is a member of the [location-scale - family](https://en.wikipedia.org/wiki/Location-scale_family), i.e., it can be - constructed as, - - ```none - X ~ MultivariateNormal(loc=0, scale=1) # Identity scale, zero shift. - Y = scale @ X + loc - ``` - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single 3-variate Gaussian with covariance `cov = S @ S.T`, - # `S = diag(d) + U @ diag(m) @ U.T`. The perturbation, `U @ diag(m) @ U.T`, is - # a rank-2 update. - mu = [-0.5., 0, 0.5] # shape: [3] - d = [1.5, 0.5, 2] # shape: [3] - U = [[1., 2], - [-1, 1], - [2, -0.5]] # shape: [3, 2] - m = [4., 5] # shape: [2] - mvn = tfd.MultivariateNormalDiagPlusLowRank( - loc=mu - scale_diag=d - scale_perturb_factor=U, - scale_perturb_diag=m) - - # Evaluate this on an observation in `R^3`, returning a scalar. - mvn.prob([-1, 0, 1]).eval() # shape: [] - - # Initialize a 2-batch of 3-variate Gaussians; `S = diag(d) + U @ U.T`. - mu = [[1., 2, 3], - [11, 22, 33]] # shape: [b, k] = [2, 3] - U = [[[1., 2], - [3, 4], - [5, 6]], - [[0.5, 0.75], - [1,0, 0.25], - [1.5, 1.25]]] # shape: [b, k, r] = [2, 3, 2] - m = [[0.1, 0.2], - [0.4, 0.5]] # shape: [b, r] = [2, 2] - - mvn = tfd.MultivariateNormalDiagPlusLowRank( - loc=mu, - scale_perturb_factor=U, - scale_perturb_diag=m) - - mvn.covariance().eval() # shape: [2, 3, 3] - # ==> [[[ 15.63 31.57 48.51] - # [ 31.57 69.31 105.05] - # [ 48.51 105.05 162.59]] - # - # [[ 2.59 1.41 3.35] - # [ 1.41 2.71 3.34] - # [ 3.35 3.34 8.35]]] - - # Compute the pdf of two `R^3` observations (one from each batch); - # return a length-2 vector. - x = [[-0.9, 0, 0.1], - [-10, 0, 9]] # shape: [2, 3] - mvn.prob(x).eval() # shape: [2] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc=None, - scale_diag=None, - scale_identity_multiplier=None, - scale_perturb_factor=None, - scale_perturb_diag=None, - validate_args=False, - allow_nan_stats=True, - name="MultivariateNormalDiagPlusLowRank"): - """Construct Multivariate Normal distribution on `R^k`. - - The `batch_shape` is the broadcast shape between `loc` and `scale` - arguments. - - The `event_shape` is given by last dimension of the matrix implied by - `scale`. The last dimension of `loc` (if provided) must broadcast with this. - - Recall that `covariance = scale @ scale.T`. A (non-batch) `scale` matrix is: - - ```none - scale = diag(scale_diag + scale_identity_multiplier ones(k)) + - scale_perturb_factor @ diag(scale_perturb_diag) @ scale_perturb_factor.T - ``` - - where: - - * `scale_diag.shape = [k]`, - * `scale_identity_multiplier.shape = []`, - * `scale_perturb_factor.shape = [k, r]`, typically `k >> r`, and, - * `scale_perturb_diag.shape = [r]`. - - Additional leading dimensions (if any) will index batches. - - If both `scale_diag` and `scale_identity_multiplier` are `None`, then - `scale` is the Identity matrix. - - Args: - loc: Floating-point `Tensor`. If this is set to `None`, `loc` is - implicitly `0`. When specified, may have shape `[B1, ..., Bb, k]` where - `b >= 0` and `k` is the event size. - scale_diag: Non-zero, floating-point `Tensor` representing a diagonal - matrix added to `scale`. May have shape `[B1, ..., Bb, k]`, `b >= 0`, - and characterizes `b`-batches of `k x k` diagonal matrices added to - `scale`. When both `scale_identity_multiplier` and `scale_diag` are - `None` then `scale` is the `Identity`. - scale_identity_multiplier: Non-zero, floating-point `Tensor` representing - a scaled-identity-matrix added to `scale`. May have shape - `[B1, ..., Bb]`, `b >= 0`, and characterizes `b`-batches of scaled - `k x k` identity matrices added to `scale`. When both - `scale_identity_multiplier` and `scale_diag` are `None` then `scale` is - the `Identity`. - scale_perturb_factor: Floating-point `Tensor` representing a rank-`r` - perturbation added to `scale`. May have shape `[B1, ..., Bb, k, r]`, - `b >= 0`, and characterizes `b`-batches of rank-`r` updates to `scale`. - When `None`, no rank-`r` update is added to `scale`. - scale_perturb_diag: Floating-point `Tensor` representing a diagonal matrix - inside the rank-`r` perturbation added to `scale`. May have shape - `[B1, ..., Bb, r]`, `b >= 0`, and characterizes `b`-batches of `r x r` - diagonal matrices inside the perturbation added to `scale`. When - `None`, an identity matrix is used inside the perturbation. Can only be - specified if `scale_perturb_factor` is also specified. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, - statistics (e.g., mean, mode, variance) use the value "`NaN`" to - indicate the result is undefined. When `False`, an exception is raised - if one or more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - ValueError: if at most `scale_identity_multiplier` is specified. - """ - parameters = dict(locals()) - def _convert_to_tensor(x, name): - return None if x is None else ops.convert_to_tensor(x, name=name) - with ops.name_scope(name) as name: - with ops.name_scope("init", values=[ - loc, scale_diag, scale_identity_multiplier, scale_perturb_factor, - scale_perturb_diag]): - has_low_rank = (scale_perturb_factor is not None or - scale_perturb_diag is not None) - scale = distribution_util.make_diag_scale( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - validate_args=validate_args, - assert_positive=has_low_rank) - scale_perturb_factor = _convert_to_tensor( - scale_perturb_factor, - name="scale_perturb_factor") - scale_perturb_diag = _convert_to_tensor( - scale_perturb_diag, - name="scale_perturb_diag") - if has_low_rank: - scale = linalg.LinearOperatorLowRankUpdate( - scale, - u=scale_perturb_factor, - diag_update=scale_perturb_diag, - is_diag_update_positive=scale_perturb_diag is None, - is_non_singular=True, # Implied by is_positive_definite=True. - is_self_adjoint=True, - is_positive_definite=True, - is_square=True) - super(MultivariateNormalDiagPlusLowRank, self).__init__( - loc=loc, - scale=scale, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - self._parameters = parameters diff --git a/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py b/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py deleted file mode 100644 index bcb49379800..00000000000 --- a/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Multivariate Normal distribution class initialized with a full covariance.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import mvn_tril -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.util import deprecation - - -__all__ = [ - "MultivariateNormalFullCovariance", -] - - -class MultivariateNormalFullCovariance(mvn_tril.MultivariateNormalTriL): - """The multivariate normal distribution on `R^k`. - - The Multivariate Normal distribution is defined over `R^k` and parameterized - by a (batch of) length-`k` `loc` vector (aka "mu") and a (batch of) `k x k` - `covariance_matrix` matrices that are the covariance. - This is different than the other multivariate normals, which are parameterized - by a matrix more akin to the standard deviation. - - #### Mathematical Details - - The probability density function (pdf) is, with `@` as matrix multiplication, - - ```none - pdf(x; loc, covariance_matrix) = exp(-0.5 y) / Z, - y = (x - loc)^T @ inv(covariance_matrix) @ (x - loc) - Z = (2 pi)**(0.5 k) |det(covariance_matrix)|**(0.5). - ``` - - where: - - * `loc` is a vector in `R^k`, - * `covariance_matrix` is an `R^{k x k}` symmetric positive definite matrix, - * `Z` denotes the normalization constant. - - Additional leading dimensions (if any) in `loc` and `covariance_matrix` allow - for batch dimensions. - - The MultivariateNormal distribution is a member of the [location-scale - family](https://en.wikipedia.org/wiki/Location-scale_family), i.e., it can be - constructed e.g. as, - - ```none - X ~ MultivariateNormal(loc=0, scale=1) # Identity scale, zero shift. - scale = Cholesky(covariance_matrix) - Y = scale @ X + loc - ``` - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single 3-variate Gaussian. - mu = [1., 2, 3] - cov = [[ 0.36, 0.12, 0.06], - [ 0.12, 0.29, -0.13], - [ 0.06, -0.13, 0.26]] - mvn = tfd.MultivariateNormalFullCovariance( - loc=mu, - covariance_matrix=cov) - - mvn.mean().eval() - # ==> [1., 2, 3] - - # Covariance agrees with covariance_matrix. - mvn.covariance().eval() - # ==> [[ 0.36, 0.12, 0.06], - # [ 0.12, 0.29, -0.13], - # [ 0.06, -0.13, 0.26]] - - # Compute the pdf of an observation in `R^3` ; return a scalar. - mvn.prob([-1., 0, 1]).eval() # shape: [] - - # Initialize a 2-batch of 3-variate Gaussians. - mu = [[1., 2, 3], - [11, 22, 33]] # shape: [2, 3] - covariance_matrix = ... # shape: [2, 3, 3], symmetric, positive definite. - mvn = tfd.MultivariateNormalFullCovariance( - loc=mu, - covariance=covariance_matrix) - - # Compute the pdf of two `R^3` observations; return a length-2 vector. - x = [[-0.9, 0, 0.1], - [-10, 0, 9]] # shape: [2, 3] - mvn.prob(x).eval() # shape: [2] - - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc=None, - covariance_matrix=None, - validate_args=False, - allow_nan_stats=True, - name="MultivariateNormalFullCovariance"): - """Construct Multivariate Normal distribution on `R^k`. - - The `batch_shape` is the broadcast shape between `loc` and - `covariance_matrix` arguments. - - The `event_shape` is given by last dimension of the matrix implied by - `covariance_matrix`. The last dimension of `loc` (if provided) must - broadcast with this. - - A non-batch `covariance_matrix` matrix is a `k x k` symmetric positive - definite matrix. In other words it is (real) symmetric with all eigenvalues - strictly positive. - - Additional leading dimensions (if any) will index batches. - - Args: - loc: Floating-point `Tensor`. If this is set to `None`, `loc` is - implicitly `0`. When specified, may have shape `[B1, ..., Bb, k]` where - `b >= 0` and `k` is the event size. - covariance_matrix: Floating-point, symmetric positive definite `Tensor` of - same `dtype` as `loc`. The strict upper triangle of `covariance_matrix` - is ignored, so if `covariance_matrix` is not symmetric no error will be - raised (unless `validate_args is True`). `covariance_matrix` has shape - `[B1, ..., Bb, k, k]` where `b >= 0` and `k` is the event size. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, - statistics (e.g., mean, mode, variance) use the value "`NaN`" to - indicate the result is undefined. When `False`, an exception is raised - if one or more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - ValueError: if neither `loc` nor `covariance_matrix` are specified. - """ - parameters = dict(locals()) - - # Convert the covariance_matrix up to a scale_tril and call MVNTriL. - with ops.name_scope(name) as name: - with ops.name_scope("init", values=[loc, covariance_matrix]): - if covariance_matrix is None: - scale_tril = None - else: - covariance_matrix = ops.convert_to_tensor( - covariance_matrix, name="covariance_matrix") - if validate_args: - covariance_matrix = control_flow_ops.with_dependencies([ - check_ops.assert_near( - covariance_matrix, - array_ops.matrix_transpose(covariance_matrix), - message="Matrix was not symmetric")], covariance_matrix) - # No need to validate that covariance_matrix is non-singular. - # LinearOperatorLowerTriangular has an assert_non_singular method that - # is called by the Bijector. - # However, cholesky() ignores the upper triangular part, so we do need - # to separately assert symmetric. - scale_tril = linalg_ops.cholesky(covariance_matrix) - super(MultivariateNormalFullCovariance, self).__init__( - loc=loc, - scale_tril=scale_tril, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - self._parameters = parameters diff --git a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py deleted file mode 100644 index f9b51cc5a62..00000000000 --- a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py +++ /dev/null @@ -1,360 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Multivariate Normal distribution classes.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.contrib.distributions.python.ops.bijectors import AffineLinearOperator -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import kullback_leibler -from tensorflow.python.ops.distributions import normal -from tensorflow.python.ops.distributions import transformed_distribution -from tensorflow.python.ops.linalg import linalg -from tensorflow.python.util import deprecation - - -__all__ = [ - "MultivariateNormalLinearOperator", -] - - -_mvn_sample_note = """ -`value` is a batch vector with compatible shape if `value` is a `Tensor` whose -shape can be broadcast up to either: - -```python -self.batch_shape + self.event_shape -``` - -or - -```python -[M1, ..., Mm] + self.batch_shape + self.event_shape -``` - -""" - - -# TODO(b/35290280): Import in `../../__init__.py` after adding unit-tests. -class MultivariateNormalLinearOperator( - transformed_distribution.TransformedDistribution): - """The multivariate normal distribution on `R^k`. - - The Multivariate Normal distribution is defined over `R^k` and parameterized - by a (batch of) length-`k` `loc` vector (aka "mu") and a (batch of) `k x k` - `scale` matrix; `covariance = scale @ scale.T`, where `@` denotes - matrix-multiplication. - - #### Mathematical Details - - The probability density function (pdf) is, - - ```none - pdf(x; loc, scale) = exp(-0.5 ||y||**2) / Z, - y = inv(scale) @ (x - loc), - Z = (2 pi)**(0.5 k) |det(scale)|, - ``` - - where: - - * `loc` is a vector in `R^k`, - * `scale` is a linear operator in `R^{k x k}`, `cov = scale @ scale.T`, - * `Z` denotes the normalization constant, and, - * `||y||**2` denotes the squared Euclidean norm of `y`. - - The MultivariateNormal distribution is a member of the [location-scale - family](https://en.wikipedia.org/wiki/Location-scale_family), i.e., it can be - constructed as, - - ```none - X ~ MultivariateNormal(loc=0, scale=1) # Identity scale, zero shift. - Y = scale @ X + loc - ``` - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single 3-variate Gaussian. - mu = [1., 2, 3] - cov = [[ 0.36, 0.12, 0.06], - [ 0.12, 0.29, -0.13], - [ 0.06, -0.13, 0.26]] - scale = tf.cholesky(cov) - # ==> [[ 0.6, 0. , 0. ], - # [ 0.2, 0.5, 0. ], - # [ 0.1, -0.3, 0.4]]) - - mvn = tfd.MultivariateNormalLinearOperator( - loc=mu, - scale=tf.linalg.LinearOperatorLowerTriangular(scale)) - - # Covariance agrees with cholesky(cov) parameterization. - mvn.covariance().eval() - # ==> [[ 0.36, 0.12, 0.06], - # [ 0.12, 0.29, -0.13], - # [ 0.06, -0.13, 0.26]] - - # Compute the pdf of an`R^3` observation; return a scalar. - mvn.prob([-1., 0, 1]).eval() # shape: [] - - # Initialize a 2-batch of 3-variate Gaussians. - mu = [[1., 2, 3], - [11, 22, 33]] # shape: [2, 3] - scale_diag = [[1., 2, 3], - [0.5, 1, 1.5]] # shape: [2, 3] - - mvn = tfd.MultivariateNormalLinearOperator( - loc=mu, - scale=tf.linalg.LinearOperatorDiag(scale_diag)) - - # Compute the pdf of two `R^3` observations; return a length-2 vector. - x = [[-0.9, 0, 0.1], - [-10, 0, 9]] # shape: [2, 3] - mvn.prob(x).eval() # shape: [2] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc=None, - scale=None, - validate_args=False, - allow_nan_stats=True, - name="MultivariateNormalLinearOperator"): - """Construct Multivariate Normal distribution on `R^k`. - - The `batch_shape` is the broadcast shape between `loc` and `scale` - arguments. - - The `event_shape` is given by last dimension of the matrix implied by - `scale`. The last dimension of `loc` (if provided) must broadcast with this. - - Recall that `covariance = scale @ scale.T`. - - Additional leading dimensions (if any) will index batches. - - Args: - loc: Floating-point `Tensor`. If this is set to `None`, `loc` is - implicitly `0`. When specified, may have shape `[B1, ..., Bb, k]` where - `b >= 0` and `k` is the event size. - scale: Instance of `LinearOperator` with same `dtype` as `loc` and shape - `[B1, ..., Bb, k, k]`. - validate_args: Python `bool`, default `False`. Whether to validate input - with asserts. If `validate_args` is `False`, and the inputs are - invalid, correct behavior is not guaranteed. - allow_nan_stats: Python `bool`, default `True`. If `False`, raise an - exception if a statistic (e.g. mean/mode/etc...) is undefined for any - batch member If `True`, batch members with valid parameters leading to - undefined statistics will return NaN for this statistic. - name: The name to give Ops created by the initializer. - - Raises: - ValueError: if `scale` is unspecified. - TypeError: if not `scale.dtype.is_floating` - """ - parameters = dict(locals()) - if scale is None: - raise ValueError("Missing required `scale` parameter.") - if not scale.dtype.is_floating: - raise TypeError("`scale` parameter must have floating-point dtype.") - - with ops.name_scope(name, values=[loc]) as name: - # Since expand_dims doesn't preserve constant-ness, we obtain the - # non-dynamic value if possible. - loc = ops.convert_to_tensor(loc, name="loc") if loc is not None else loc - batch_shape, event_shape = distribution_util.shapes_from_loc_and_scale( - loc, scale) - - super(MultivariateNormalLinearOperator, self).__init__( - distribution=normal.Normal( - loc=array_ops.zeros([], dtype=scale.dtype), - scale=array_ops.ones([], dtype=scale.dtype)), - bijector=AffineLinearOperator( - shift=loc, scale=scale, validate_args=validate_args), - batch_shape=batch_shape, - event_shape=event_shape, - validate_args=validate_args, - name=name) - self._parameters = parameters - - @property - def loc(self): - """The `loc` `Tensor` in `Y = scale @ X + loc`.""" - return self.bijector.shift - - @property - def scale(self): - """The `scale` `LinearOperator` in `Y = scale @ X + loc`.""" - return self.bijector.scale - - @distribution_util.AppendDocstring(_mvn_sample_note) - def _log_prob(self, x): - return super(MultivariateNormalLinearOperator, self)._log_prob(x) - - @distribution_util.AppendDocstring(_mvn_sample_note) - def _prob(self, x): - return super(MultivariateNormalLinearOperator, self)._prob(x) - - def _mean(self): - shape = self.batch_shape.concatenate(self.event_shape) - has_static_shape = shape.is_fully_defined() - if not has_static_shape: - shape = array_ops.concat([ - self.batch_shape_tensor(), - self.event_shape_tensor(), - ], 0) - - if self.loc is None: - return array_ops.zeros(shape, self.dtype) - - if has_static_shape and shape == self.loc.get_shape(): - return array_ops.identity(self.loc) - - # Add dummy tensor of zeros to broadcast. This is only necessary if shape - # != self.loc.shape, but we could not determine if this is the case. - return array_ops.identity(self.loc) + array_ops.zeros(shape, self.dtype) - - def _covariance(self): - if distribution_util.is_diagonal_scale(self.scale): - return array_ops.matrix_diag(math_ops.square(self.scale.diag_part())) - else: - return self.scale.matmul(self.scale.to_dense(), adjoint_arg=True) - - def _variance(self): - if distribution_util.is_diagonal_scale(self.scale): - return math_ops.square(self.scale.diag_part()) - elif (isinstance(self.scale, linalg.LinearOperatorLowRankUpdate) and - self.scale.is_self_adjoint): - return array_ops.matrix_diag_part( - self.scale.matmul(self.scale.to_dense())) - else: - return array_ops.matrix_diag_part( - self.scale.matmul(self.scale.to_dense(), adjoint_arg=True)) - - def _stddev(self): - if distribution_util.is_diagonal_scale(self.scale): - return math_ops.abs(self.scale.diag_part()) - elif (isinstance(self.scale, linalg.LinearOperatorLowRankUpdate) and - self.scale.is_self_adjoint): - return math_ops.sqrt(array_ops.matrix_diag_part( - self.scale.matmul(self.scale.to_dense()))) - else: - return math_ops.sqrt(array_ops.matrix_diag_part( - self.scale.matmul(self.scale.to_dense(), adjoint_arg=True))) - - def _mode(self): - return self._mean() - - -@kullback_leibler.RegisterKL(MultivariateNormalLinearOperator, - MultivariateNormalLinearOperator) -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _kl_brute_force(a, b, name=None): - """Batched KL divergence `KL(a || b)` for multivariate Normals. - - With `X`, `Y` both multivariate Normals in `R^k` with means `mu_a`, `mu_b` and - covariance `C_a`, `C_b` respectively, - - ``` - KL(a || b) = 0.5 * ( L - k + T + Q ), - L := Log[Det(C_b)] - Log[Det(C_a)] - T := trace(C_b^{-1} C_a), - Q := (mu_b - mu_a)^T C_b^{-1} (mu_b - mu_a), - ``` - - This `Op` computes the trace by solving `C_b^{-1} C_a`. Although efficient - methods for solving systems with `C_b` may be available, a dense version of - (the square root of) `C_a` is used, so performance is `O(B s k**2)` where `B` - is the batch size, and `s` is the cost of solving `C_b x = y` for vectors `x` - and `y`. - - Args: - a: Instance of `MultivariateNormalLinearOperator`. - b: Instance of `MultivariateNormalLinearOperator`. - name: (optional) name to use for created ops. Default "kl_mvn". - - Returns: - Batchwise `KL(a || b)`. - """ - - def squared_frobenius_norm(x): - """Helper to make KL calculation slightly more readable.""" - # http://mathworld.wolfram.com/FrobeniusNorm.html - # The gradient of KL[p,q] is not defined when p==q. The culprit is - # linalg_ops.norm, i.e., we cannot use the commented out code. - # return math_ops.square(linalg_ops.norm(x, ord="fro", axis=[-2, -1])) - return math_ops.reduce_sum(math_ops.square(x), axis=[-2, -1]) - - # TODO(b/35041439): See also b/35040945. Remove this function once LinOp - # supports something like: - # A.inverse().solve(B).norm(order='fro', axis=[-1, -2]) - def is_diagonal(x): - """Helper to identify if `LinearOperator` has only a diagonal component.""" - return (isinstance(x, linalg.LinearOperatorIdentity) or - isinstance(x, linalg.LinearOperatorScaledIdentity) or - isinstance(x, linalg.LinearOperatorDiag)) - - with ops.name_scope(name, "kl_mvn", values=[a.loc, b.loc]): - # Calculation is based on: - # http://stats.stackexchange.com/questions/60680/kl-divergence-between-two-multivariate-gaussians - # and, - # https://en.wikipedia.org/wiki/Matrix_norm#Frobenius_norm - # i.e., - # If Ca = AA', Cb = BB', then - # tr[inv(Cb) Ca] = tr[inv(B)' inv(B) A A'] - # = tr[inv(B) A A' inv(B)'] - # = tr[(inv(B) A) (inv(B) A)'] - # = sum_{ij} (inv(B) A)_{ij}**2 - # = ||inv(B) A||_F**2 - # where ||.||_F is the Frobenius norm and the second equality follows from - # the cyclic permutation property. - if is_diagonal(a.scale) and is_diagonal(b.scale): - # Using `stddev` because it handles expansion of Identity cases. - b_inv_a = (a.stddev() / b.stddev())[..., array_ops.newaxis] - else: - b_inv_a = b.scale.solve(a.scale.to_dense()) - kl_div = (b.scale.log_abs_determinant() - - a.scale.log_abs_determinant() - + 0.5 * ( - - math_ops.cast(a.scale.domain_dimension_tensor(), a.dtype) - + squared_frobenius_norm(b_inv_a) - + squared_frobenius_norm(b.scale.solve( - (b.mean() - a.mean())[..., array_ops.newaxis])))) - kl_div.set_shape(array_ops.broadcast_static_shape( - a.batch_shape, b.batch_shape)) - return kl_div diff --git a/tensorflow/contrib/distributions/python/ops/mvn_tril.py b/tensorflow/contrib/distributions/python/ops/mvn_tril.py deleted file mode 100644 index 2f09f49f8c6..00000000000 --- a/tensorflow/contrib/distributions/python/ops/mvn_tril.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Multivariate Normal distribution classes.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import mvn_linear_operator as mvn_linop -from tensorflow.python.framework import ops -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.ops.linalg import linalg -from tensorflow.python.util import deprecation - -__all__ = [ - "MultivariateNormalTriL", -] - - -class MultivariateNormalTriL(mvn_linop.MultivariateNormalLinearOperator): - """The multivariate normal distribution on `R^k`. - - The Multivariate Normal distribution is defined over `R^k` and parameterized - by a (batch of) length-`k` `loc` vector (aka "mu") and a (batch of) `k x k` - `scale` matrix; `covariance = scale @ scale.T` where `@` denotes - matrix-multiplication. - - #### Mathematical Details - - The probability density function (pdf) is, - - ```none - pdf(x; loc, scale) = exp(-0.5 ||y||**2) / Z, - y = inv(scale) @ (x - loc), - Z = (2 pi)**(0.5 k) |det(scale)|, - ``` - - where: - - * `loc` is a vector in `R^k`, - * `scale` is a matrix in `R^{k x k}`, `covariance = scale @ scale.T`, - * `Z` denotes the normalization constant, and, - * `||y||**2` denotes the squared Euclidean norm of `y`. - - A (non-batch) `scale` matrix is: - - ```none - scale = scale_tril - ``` - - where `scale_tril` is lower-triangular `k x k` matrix with non-zero diagonal, - i.e., `tf.linalg.tensor_diag_part(scale_tril) != 0`. - - Additional leading dimensions (if any) will index batches. - - The MultivariateNormal distribution is a member of the [location-scale - family](https://en.wikipedia.org/wiki/Location-scale_family), i.e., it can be - constructed as, - - ```none - X ~ MultivariateNormal(loc=0, scale=1) # Identity scale, zero shift. - Y = scale @ X + loc - ``` - - Trainable (batch) lower-triangular matrices can be created with - `tfp.distributions.matrix_diag_transform()` and/or - `tfp.distributions.fill_triangular()` - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single 3-variate Gaussian. - mu = [1., 2, 3] - cov = [[ 0.36, 0.12, 0.06], - [ 0.12, 0.29, -0.13], - [ 0.06, -0.13, 0.26]] - scale = tf.linalg.cholesky(cov) - # ==> [[ 0.6, 0. , 0. ], - # [ 0.2, 0.5, 0. ], - # [ 0.1, -0.3, 0.4]]) - mvn = tfd.MultivariateNormalTriL( - loc=mu, - scale_tril=scale) - - mvn.mean().eval() - # ==> [1., 2, 3] - - # Covariance agrees with cholesky(cov) parameterization. - mvn.covariance().eval() - # ==> [[ 0.36, 0.12, 0.06], - # [ 0.12, 0.29, -0.13], - # [ 0.06, -0.13, 0.26]] - - # Compute the pdf of an observation in `R^3` ; return a scalar. - mvn.prob([-1., 0, 1]).eval() # shape: [] - - # Initialize a 2-batch of 3-variate Gaussians. - mu = [[1., 2, 3], - [11, 22, 33]] # shape: [2, 3] - tril = ... # shape: [2, 3, 3], lower triangular, non-zero diagonal. - mvn = tfd.MultivariateNormalTriL( - loc=mu, - scale_tril=tril) - - # Compute the pdf of two `R^3` observations; return a length-2 vector. - x = [[-0.9, 0, 0.1], - [-10, 0, 9]] # shape: [2, 3] - mvn.prob(x).eval() # shape: [2] - - # Instantiate a "learnable" MVN. - dims = 4 - with tf.compat.v1.variable_scope("model"): - mvn = tfd.MultivariateNormalTriL( - loc=tf.compat.v1.get_variable(shape=[dims], dtype=tf.float32, - name="mu"), - scale_tril=tfd.fill_triangular( - tf.compat.v1.get_variable(shape=[dims * (dims + 1) / 2], - dtype=tf.float32, name="chol_Sigma"))) - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc=None, - scale_tril=None, - validate_args=False, - allow_nan_stats=True, - name="MultivariateNormalTriL"): - """Construct Multivariate Normal distribution on `R^k`. - - The `batch_shape` is the broadcast shape between `loc` and `scale` - arguments. - - The `event_shape` is given by last dimension of the matrix implied by - `scale`. The last dimension of `loc` (if provided) must broadcast with this. - - Recall that `covariance = scale @ scale.T`. A (non-batch) `scale` matrix is: - - ```none - scale = scale_tril - ``` - - where `scale_tril` is lower-triangular `k x k` matrix with non-zero - diagonal, i.e., `tf.linalg.tensor_diag_part(scale_tril) != 0`. - - Additional leading dimensions (if any) will index batches. - - Args: - loc: Floating-point `Tensor`. If this is set to `None`, `loc` is - implicitly `0`. When specified, may have shape `[B1, ..., Bb, k]` where - `b >= 0` and `k` is the event size. - scale_tril: Floating-point, lower-triangular `Tensor` with non-zero - diagonal elements. `scale_tril` has shape `[B1, ..., Bb, k, k]` where `b - >= 0` and `k` is the event size. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or more - of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - ValueError: if neither `loc` nor `scale_tril` are specified. - """ - parameters = dict(locals()) - - def _convert_to_tensor(x, name): - return None if x is None else ops.convert_to_tensor(x, name=name) - - if loc is None and scale_tril is None: - raise ValueError("Must specify one or both of `loc`, `scale_tril`.") - with ops.name_scope(name) as name: - with ops.name_scope("init", values=[loc, scale_tril]): - loc = _convert_to_tensor(loc, name="loc") - scale_tril = _convert_to_tensor(scale_tril, name="scale_tril") - if scale_tril is None: - scale = linalg.LinearOperatorIdentity( - num_rows=distribution_util.dimension_size(loc, -1), - dtype=loc.dtype, - is_self_adjoint=True, - is_positive_definite=True, - assert_proper_shapes=validate_args) - else: - # No need to validate that scale_tril is non-singular. - # LinearOperatorLowerTriangular has an assert_non_singular - # method that is called by the Bijector. - scale = linalg.LinearOperatorLowerTriangular( - scale_tril, - is_non_singular=True, - is_self_adjoint=False, - is_positive_definite=False) - super(MultivariateNormalTriL, self).__init__( - loc=loc, - scale=scale, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - self._parameters = parameters diff --git a/tensorflow/contrib/distributions/python/ops/negative_binomial.py b/tensorflow/contrib/distributions/python/ops/negative_binomial.py deleted file mode 100644 index faf9827c8bf..00000000000 --- a/tensorflow/contrib/distributions/python/ops/negative_binomial.py +++ /dev/null @@ -1,199 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Negative Binomial distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - - -class NegativeBinomial(distribution.Distribution): - """NegativeBinomial distribution. - - The NegativeBinomial distribution is related to the experiment of performing - Bernoulli trials in sequence. Given a Bernoulli trial with probability `p` of - success, the NegativeBinomial distribution represents the distribution over - the number of successes `s` that occur until we observe `f` failures. - - The probability mass function (pmf) is, - - ```none - pmf(s; f, p) = p**s (1 - p)**f / Z - Z = s! (f - 1)! / (s + f - 1)! - ``` - - where: - * `total_count = f`, - * `probs = p`, - * `Z` is the normalizaing constant, and, - * `n!` is the factorial of `n`. - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - total_count, - logits=None, - probs=None, - validate_args=False, - allow_nan_stats=True, - name="NegativeBinomial"): - """Construct NegativeBinomial distributions. - - Args: - total_count: Non-negative floating-point `Tensor` with shape - broadcastable to `[B1,..., Bb]` with `b >= 0` and the same dtype as - `probs` or `logits`. Defines this as a batch of `N1 x ... x Nm` - different Negative Binomial distributions. In practice, this represents - the number of negative Bernoulli trials to stop at (the `total_count` - of failures), but this is still a valid distribution when - `total_count` is a non-integer. - logits: Floating-point `Tensor` with shape broadcastable to - `[B1, ..., Bb]` where `b >= 0` indicates the number of batch dimensions. - Each entry represents logits for the probability of success for - independent Negative Binomial distributions and must be in the open - interval `(-inf, inf)`. Only one of `logits` or `probs` should be - specified. - probs: Positive floating-point `Tensor` with shape broadcastable to - `[B1, ..., Bb]` where `b >= 0` indicates the number of batch dimensions. - Each entry represents the probability of success for independent - Negative Binomial distributions and must be in the open interval - `(0, 1)`. Only one of `logits` or `probs` should be specified. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - - parameters = dict(locals()) - with ops.name_scope(name, values=[total_count, logits, probs]) as name: - self._logits, self._probs = distribution_util.get_logits_and_probs( - logits, probs, validate_args=validate_args, name=name) - with ops.control_dependencies( - [check_ops.assert_positive(total_count)] if validate_args else []): - self._total_count = array_ops.identity(total_count) - - super(NegativeBinomial, self).__init__( - dtype=self._probs.dtype, - reparameterization_type=distribution.NOT_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=[self._total_count, self._probs, self._logits], - name=name) - - @property - def total_count(self): - """Number of negative trials.""" - return self._total_count - - @property - def logits(self): - """Log-odds of a `1` outcome (vs `0`).""" - return self._logits - - @property - def probs(self): - """Probability of a `1` outcome (vs `0`).""" - return self._probs - - def _batch_shape_tensor(self): - return array_ops.broadcast_dynamic_shape( - array_ops.shape(self.total_count), - array_ops.shape(self.probs)) - - def _batch_shape(self): - return array_ops.broadcast_static_shape( - self.total_count.get_shape(), - self.probs.get_shape()) - - def _event_shape_tensor(self): - return array_ops.constant([], dtype=dtypes.int32) - - def _event_shape(self): - return tensor_shape.TensorShape([]) - - def _sample_n(self, n, seed=None): - # Here we use the fact that if: - # lam ~ Gamma(concentration=total_count, rate=(1-probs)/probs) - # then X ~ Poisson(lam) is Negative Binomially distributed. - rate = random_ops.random_gamma( - shape=[n], - alpha=self.total_count, - beta=math_ops.exp(-self.logits), - dtype=self.dtype, - seed=seed) - return random_ops.random_poisson( - rate, - shape=[], - dtype=self.dtype, - seed=distribution_util.gen_new_seed(seed, "negative_binom")) - - def _cdf(self, x): - if self.validate_args: - x = distribution_util.embed_check_nonnegative_integer_form(x) - return math_ops.betainc(self.total_count, 1. + x, - math_ops.sigmoid(-self.logits)) - - def _log_prob(self, x): - return (self._log_unnormalized_prob(x) - - self._log_normalization(x)) - - def _log_unnormalized_prob(self, x): - if self.validate_args: - x = distribution_util.embed_check_nonnegative_integer_form(x) - return (self.total_count * math_ops.log_sigmoid(-self.logits) - + x * math_ops.log_sigmoid(self.logits)) - - def _log_normalization(self, x): - if self.validate_args: - x = distribution_util.embed_check_nonnegative_integer_form(x) - return (-math_ops.lgamma(self.total_count + x) - + math_ops.lgamma(1. + x) - + math_ops.lgamma(self.total_count)) - - def _mean(self): - return self.total_count * math_ops.exp(self.logits) - - def _mode(self): - adjusted_count = array_ops.where_v2(1. < self.total_count, - self.total_count - 1., - array_ops.zeros_like(self.total_count)) - return math_ops.floor(adjusted_count * math_ops.exp(self.logits)) - - def _variance(self): - return self._mean() / math_ops.sigmoid(-self.logits) diff --git a/tensorflow/contrib/distributions/python/ops/normal_conjugate_posteriors.py b/tensorflow/contrib/distributions/python/ops/normal_conjugate_posteriors.py deleted file mode 100644 index 4025285780b..00000000000 --- a/tensorflow/contrib/distributions/python/ops/normal_conjugate_posteriors.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Normal distribution: conjugate posterior closed form calculations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import normal - - -def normal_conjugates_known_scale_posterior(prior, scale, s, n): - """Posterior Normal distribution with conjugate prior on the mean. - - This model assumes that `n` observations (with sum `s`) come from a - Normal with unknown mean `loc` (described by the Normal `prior`) - and known variance `scale**2`. The "known scale posterior" is - the distribution of the unknown `loc`. - - Accepts a prior Normal distribution object, having parameters - `loc0` and `scale0`, as well as known `scale` values of the predictive - distribution(s) (also assumed Normal), - and statistical estimates `s` (the sum(s) of the observations) and - `n` (the number(s) of observations). - - Returns a posterior (also Normal) distribution object, with parameters - `(loc', scale'**2)`, where: - - ``` - mu ~ N(mu', sigma'**2) - sigma'**2 = 1/(1/sigma0**2 + n/sigma**2), - mu' = (mu0/sigma0**2 + s/sigma**2) * sigma'**2. - ``` - - Distribution parameters from `prior`, as well as `scale`, `s`, and `n`. - will broadcast in the case of multidimensional sets of parameters. - - Args: - prior: `Normal` object of type `dtype`: - the prior distribution having parameters `(loc0, scale0)`. - scale: tensor of type `dtype`, taking values `scale > 0`. - The known stddev parameter(s). - s: Tensor of type `dtype`. The sum(s) of observations. - n: Tensor of type `int`. The number(s) of observations. - - Returns: - A new Normal posterior distribution object for the unknown observation - mean `loc`. - - Raises: - TypeError: if dtype of `s` does not match `dtype`, or `prior` is not a - Normal object. - """ - if not isinstance(prior, normal.Normal): - raise TypeError("Expected prior to be an instance of type Normal") - - if s.dtype != prior.dtype: - raise TypeError( - "Observation sum s.dtype does not match prior dtype: %s vs. %s" - % (s.dtype, prior.dtype)) - - n = math_ops.cast(n, prior.dtype) - scale0_2 = math_ops.square(prior.scale) - scale_2 = math_ops.square(scale) - scalep_2 = 1.0/(1/scale0_2 + n/scale_2) - return normal.Normal( - loc=(prior.loc/scale0_2 + s/scale_2) * scalep_2, - scale=math_ops.sqrt(scalep_2)) - - -def normal_conjugates_known_scale_predictive(prior, scale, s, n): - """Posterior predictive Normal distribution w. conjugate prior on the mean. - - This model assumes that `n` observations (with sum `s`) come from a - Normal with unknown mean `loc` (described by the Normal `prior`) - and known variance `scale**2`. The "known scale predictive" - is the distribution of new observations, conditioned on the existing - observations and our prior. - - Accepts a prior Normal distribution object, having parameters - `loc0` and `scale0`, as well as known `scale` values of the predictive - distribution(s) (also assumed Normal), - and statistical estimates `s` (the sum(s) of the observations) and - `n` (the number(s) of observations). - - Calculates the Normal distribution(s) `p(x | sigma**2)`: - - ``` - p(x | sigma**2) = int N(x | mu, sigma**2)N(mu | prior.loc, prior.scale**2) dmu - = N(x | prior.loc, 1 / (sigma**2 + prior.scale**2)) - ``` - - Returns the predictive posterior distribution object, with parameters - `(loc', scale'**2)`, where: - - ``` - sigma_n**2 = 1/(1/sigma0**2 + n/sigma**2), - mu' = (mu0/sigma0**2 + s/sigma**2) * sigma_n**2. - sigma'**2 = sigma_n**2 + sigma**2, - ``` - - Distribution parameters from `prior`, as well as `scale`, `s`, and `n`. - will broadcast in the case of multidimensional sets of parameters. - - Args: - prior: `Normal` object of type `dtype`: - the prior distribution having parameters `(loc0, scale0)`. - scale: tensor of type `dtype`, taking values `scale > 0`. - The known stddev parameter(s). - s: Tensor of type `dtype`. The sum(s) of observations. - n: Tensor of type `int`. The number(s) of observations. - - Returns: - A new Normal predictive distribution object. - - Raises: - TypeError: if dtype of `s` does not match `dtype`, or `prior` is not a - Normal object. - """ - if not isinstance(prior, normal.Normal): - raise TypeError("Expected prior to be an instance of type Normal") - - if s.dtype != prior.dtype: - raise TypeError( - "Observation sum s.dtype does not match prior dtype: %s vs. %s" - % (s.dtype, prior.dtype)) - - n = math_ops.cast(n, prior.dtype) - scale0_2 = math_ops.square(prior.scale) - scale_2 = math_ops.square(scale) - scalep_2 = 1.0/(1/scale0_2 + n/scale_2) - return normal.Normal( - loc=(prior.loc/scale0_2 + s/scale_2) * scalep_2, - scale=math_ops.sqrt(scalep_2 + scale_2)) diff --git a/tensorflow/contrib/distributions/python/ops/onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/onehot_categorical.py deleted file mode 100644 index 214c6dca4a7..00000000000 --- a/tensorflow/contrib/distributions/python/ops/onehot_categorical.py +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The OneHotCategorical distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.ops.distributions import kullback_leibler -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - - -class OneHotCategorical(distribution.Distribution): - """OneHotCategorical distribution. - - The categorical distribution is parameterized by the log-probabilities - of a set of classes. The difference between OneHotCategorical and Categorical - distributions is that OneHotCategorical is a discrete distribution over - one-hot bit vectors whereas Categorical is a discrete distribution over - positive integers. OneHotCategorical is equivalent to Categorical except - Categorical has event_dim=() while OneHotCategorical has event_dim=K, where - K is the number of classes. - - This class provides methods to create indexed batches of OneHotCategorical - distributions. If the provided `logits` or `probs` is rank 2 or higher, for - every fixed set of leading dimensions, the last dimension represents one - single OneHotCategorical distribution. When calling distribution - functions (e.g. `dist.prob(x)`), `logits` and `x` are broadcast to the - same shape (if possible). In all cases, the last dimension of `logits,x` - represents single OneHotCategorical distributions. - - #### Examples - - Creates a 3-class distribution, with the 2nd class, the most likely to be - drawn from. - - ```python - p = [0.1, 0.5, 0.4] - dist = OneHotCategorical(probs=p) - ``` - - Creates a 3-class distribution, with the 2nd class the most likely to be - drawn from, using logits. - - ```python - logits = [-2, 2, 0] - dist = OneHotCategorical(logits=logits) - ``` - - Creates a 3-class distribution, with the 3rd class is most likely to be drawn. - - ```python - # counts is a scalar. - p = [0.1, 0.4, 0.5] - dist = OneHotCategorical(probs=p) - dist.prob([0,1,0]) # Shape [] - - # p will be broadcast to [[0.1, 0.4, 0.5], [0.1, 0.4, 0.5]] to match. - samples = [[0,1,0], [1,0,0]] - dist.prob(samples) # Shape [2] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__( - self, - logits=None, - probs=None, - dtype=dtypes.int32, - validate_args=False, - allow_nan_stats=True, - name="OneHotCategorical"): - """Initialize OneHotCategorical distributions using class log-probabilities. - - Args: - logits: An N-D `Tensor`, `N >= 1`, representing the log probabilities of a - set of Categorical distributions. The first `N - 1` dimensions index - into a batch of independent distributions and the last dimension - represents a vector of logits for each class. Only one of `logits` or - `probs` should be passed in. - probs: An N-D `Tensor`, `N >= 1`, representing the probabilities of a set - of Categorical distributions. The first `N - 1` dimensions index into a - batch of independent distributions and the last dimension represents a - vector of probabilities for each class. Only one of `logits` or `probs` - should be passed in. - dtype: The type of the event samples (default: int32). - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[logits, probs]) as name: - self._logits, self._probs = distribution_util.get_logits_and_probs( - name=name, logits=logits, probs=probs, validate_args=validate_args, - multidimensional=True) - - logits_shape_static = self._logits.get_shape().with_rank_at_least(1) - if logits_shape_static.ndims is not None: - self._batch_rank = ops.convert_to_tensor( - logits_shape_static.ndims - 1, - dtype=dtypes.int32, - name="batch_rank") - else: - with ops.name_scope(name="batch_rank"): - self._batch_rank = array_ops.rank(self._logits) - 1 - - with ops.name_scope(name="event_size"): - self._event_size = array_ops.shape(self._logits)[-1] - - super(OneHotCategorical, self).__init__( - dtype=dtype, - reparameterization_type=distribution.NOT_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=[self._logits, - self._probs], - name=name) - - @property - def event_size(self): - """Scalar `int32` tensor: the number of classes.""" - return self._event_size - - @property - def logits(self): - """Vector of coordinatewise logits.""" - return self._logits - - @property - def probs(self): - """Vector of coordinatewise probabilities.""" - return self._probs - - def _batch_shape_tensor(self): - return array_ops.shape(self.logits)[:-1] - - def _batch_shape(self): - return self.logits.get_shape()[:-1] - - def _event_shape_tensor(self): - return array_ops.shape(self.logits)[-1:] - - def _event_shape(self): - return self.logits.get_shape().with_rank_at_least(1)[-1:] - - def _sample_n(self, n, seed=None): - sample_shape = array_ops.concat([[n], array_ops.shape(self.logits)], 0) - logits = self.logits - if logits.get_shape().ndims == 2: - logits_2d = logits - else: - logits_2d = array_ops.reshape(logits, [-1, self.event_size]) - samples = random_ops.multinomial(logits_2d, n, seed=seed) - samples = array_ops.transpose(samples) - samples = array_ops.one_hot(samples, self.event_size, dtype=self.dtype) - ret = array_ops.reshape(samples, sample_shape) - return ret - - def _log_prob(self, x): - x = self._assert_valid_sample(x) - # broadcast logits or x if need be. - logits = self.logits - if (not x.get_shape().is_fully_defined() or - not logits.get_shape().is_fully_defined() or - x.get_shape() != logits.get_shape()): - logits = array_ops.ones_like(x, dtype=logits.dtype) * logits - x = array_ops.ones_like(logits, dtype=x.dtype) * x - - logits_shape = array_ops.shape(math_ops.reduce_sum(logits, -1)) - logits_2d = array_ops.reshape(logits, [-1, self.event_size]) - x_2d = array_ops.reshape(x, [-1, self.event_size]) - ret = -nn_ops.softmax_cross_entropy_with_logits(labels=x_2d, - logits=logits_2d) - # Reshape back to user-supplied batch and sample dims prior to 2D reshape. - ret = array_ops.reshape(ret, logits_shape) - return ret - - def _entropy(self): - return -math_ops.reduce_sum( - nn_ops.log_softmax(self.logits) * self.probs, axis=-1) - - def _mode(self): - ret = math_ops.argmax(self.logits, axis=self._batch_rank) - ret = array_ops.one_hot(ret, self.event_size, dtype=self.dtype) - ret.set_shape(self.logits.get_shape()) - return ret - - def _covariance(self): - p = self.probs - ret = -math_ops.matmul(p[..., None], p[..., None, :]) - return array_ops.matrix_set_diag(ret, self._variance()) - - def _variance(self): - return self.probs * (1. - self.probs) - - def _assert_valid_sample(self, x): - if not self.validate_args: - return x - return control_flow_ops.with_dependencies([ - check_ops.assert_non_positive(x), - check_ops.assert_near( - array_ops.zeros([], dtype=self.dtype), - math_ops.reduce_logsumexp(x, axis=[-1])), - ], x) - - -@kullback_leibler.RegisterKL(OneHotCategorical, OneHotCategorical) -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _kl_categorical_categorical(a, b, name=None): - """Calculate the batched KL divergence KL(a || b) with a, b OneHotCategorical. - - Args: - a: instance of a OneHotCategorical distribution object. - b: instance of a OneHotCategorical distribution object. - name: (optional) Name to use for created operations. - default is "kl_categorical_categorical". - - Returns: - Batchwise KL(a || b) - """ - with ops.name_scope(name, "kl_categorical_categorical", values=[ - a.logits, b.logits]): - # sum(p ln(p / q)) - return math_ops.reduce_sum( - nn_ops.softmax(a.logits) * (nn_ops.log_softmax(a.logits) - - nn_ops.log_softmax(b.logits)), - axis=-1) diff --git a/tensorflow/contrib/distributions/python/ops/poisson.py b/tensorflow/contrib/distributions/python/ops/poisson.py deleted file mode 100644 index 64c41c57d79..00000000000 --- a/tensorflow/contrib/distributions/python/ops/poisson.py +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Poisson distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - -__all__ = [ - "Poisson", -] - - -_poisson_sample_note = """ -The Poisson distribution is technically only defined for non-negative integer -values. When `validate_args=False`, non-integral inputs trigger an assertion. - -When `validate_args=False` calculations are otherwise unchanged despite -integral or non-integral inputs. - -When `validate_args=False`, evaluating the pmf at non-integral values, -corresponds to evaluations of an unnormalized distribution, that does not -correspond to evaluations of the cdf. -""" - - -class Poisson(distribution.Distribution): - """Poisson distribution. - - The Poisson distribution is parameterized by an event `rate` parameter. - - #### Mathematical Details - - The probability mass function (pmf) is, - - ```none - pmf(k; lambda, k >= 0) = (lambda^k / k!) / Z - Z = exp(lambda). - ``` - - where `rate = lambda` and `Z` is the normalizing constant. - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - rate=None, - log_rate=None, - validate_args=False, - allow_nan_stats=True, - name="Poisson"): - """Initialize a batch of Poisson distributions. - - Args: - rate: Floating point tensor, the rate parameter. `rate` must be positive. - Must specify exactly one of `rate` and `log_rate`. - log_rate: Floating point tensor, the log of the rate parameter. - Must specify exactly one of `rate` and `log_rate`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - ValueError: if none or both of `rate`, `log_rate` are specified. - TypeError: if `rate` is not a float-type. - TypeError: if `log_rate` is not a float-type. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[rate]) as name: - if (rate is None) == (log_rate is None): - raise ValueError("Must specify exactly one of `rate` and `log_rate`.") - elif log_rate is None: - rate = ops.convert_to_tensor(rate, name="rate") - if not rate.dtype.is_floating: - raise TypeError("rate.dtype ({}) is a not a float-type.".format( - rate.dtype.name)) - with ops.control_dependencies([check_ops.assert_positive(rate)] if - validate_args else []): - self._rate = array_ops.identity(rate, name="rate") - self._log_rate = math_ops.log(rate, name="log_rate") - else: - log_rate = ops.convert_to_tensor(log_rate, name="log_rate") - if not log_rate.dtype.is_floating: - raise TypeError("log_rate.dtype ({}) is a not a float-type.".format( - log_rate.dtype.name)) - self._rate = math_ops.exp(log_rate, name="rate") - self._log_rate = ops.convert_to_tensor(log_rate, name="log_rate") - super(Poisson, self).__init__( - dtype=self._rate.dtype, - reparameterization_type=distribution.NOT_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=[self._rate], - name=name) - - @property - def rate(self): - """Rate parameter.""" - return self._rate - - @property - def log_rate(self): - """Log rate parameter.""" - return self._log_rate - - def _batch_shape_tensor(self): - return array_ops.shape(self.rate) - - def _batch_shape(self): - return self.rate.shape - - def _event_shape_tensor(self): - return constant_op.constant([], dtype=dtypes.int32) - - def _event_shape(self): - return tensor_shape.TensorShape([]) - - @distribution_util.AppendDocstring(_poisson_sample_note) - def _log_prob(self, x): - return self._log_unnormalized_prob(x) - self._log_normalization() - - @distribution_util.AppendDocstring(_poisson_sample_note) - def _log_cdf(self, x): - return math_ops.log(self.cdf(x)) - - @distribution_util.AppendDocstring(_poisson_sample_note) - def _cdf(self, x): - if self.validate_args: - x = distribution_util.embed_check_nonnegative_integer_form(x) - return math_ops.igammac(1. + x, self.rate) - - def _log_normalization(self): - return self.rate - - def _log_unnormalized_prob(self, x): - if self.validate_args: - x = distribution_util.embed_check_nonnegative_integer_form(x) - return x * self.log_rate - math_ops.lgamma(1. + x) - - def _mean(self): - return array_ops.identity(self.rate) - - def _variance(self): - return array_ops.identity(self.rate) - - @distribution_util.AppendDocstring( - """Note: when `rate` is an integer, there are actually two modes: `rate` - and `rate - 1`. In this case we return the larger, i.e., `rate`.""") - def _mode(self): - return math_ops.floor(self.rate) - - def _sample_n(self, n, seed=None): - return random_ops.random_poisson( - self.rate, [n], dtype=self.dtype, seed=seed) diff --git a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py deleted file mode 100644 index b23a3231d27..00000000000 --- a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py +++ /dev/null @@ -1,459 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The PoissonLogNormalQuadratureCompound distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.contrib.distributions.python.ops import poisson as poisson_lib -from tensorflow.contrib.distributions.python.ops.bijectors.exp import Exp -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import categorical as categorical_lib -from tensorflow.python.ops.distributions import distribution as distribution_lib -from tensorflow.python.ops.distributions import normal as normal_lib -from tensorflow.python.ops.distributions import transformed_distribution as transformed_lib -from tensorflow.python.util import deprecation - - -__all__ = [ - "PoissonLogNormalQuadratureCompound", - "quadrature_scheme_lognormal_gauss_hermite", - "quadrature_scheme_lognormal_quantiles", -] - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def quadrature_scheme_lognormal_gauss_hermite( - loc, scale, quadrature_size, - validate_args=False, name=None): # pylint: disable=unused-argument - """Use Gauss-Hermite quadrature to form quadrature on positive-reals. - - Note: for a given `quadrature_size`, this method is generally less accurate - than `quadrature_scheme_lognormal_quantiles`. - - Args: - loc: `float`-like (batch of) scalar `Tensor`; the location parameter of - the LogNormal prior. - scale: `float`-like (batch of) scalar `Tensor`; the scale parameter of - the LogNormal prior. - quadrature_size: Python `int` scalar representing the number of quadrature - points. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - name: Python `str` name prefixed to Ops created by this class. - - Returns: - grid: (Batch of) length-`quadrature_size` vectors representing the - `log_rate` parameters of a `Poisson`. - probs: (Batch of) length-`quadrature_size` vectors representing the - weight associate with each `grid` value. - """ - with ops.name_scope(name, "vector_diffeomixture_quadrature_gauss_hermite", - [loc, scale]): - grid, probs = np.polynomial.hermite.hermgauss(deg=quadrature_size) - grid = grid.astype(loc.dtype.as_numpy_dtype) - probs = probs.astype(loc.dtype.as_numpy_dtype) - probs /= np.linalg.norm(probs, ord=1, keepdims=True) - probs = ops.convert_to_tensor(probs, name="probs", dtype=loc.dtype) - # The following maps the broadcast of `loc` and `scale` to each grid - # point, i.e., we are creating several log-rates that correspond to the - # different Gauss-Hermite quadrature points and (possible) batches of - # `loc` and `scale`. - grid = (loc[..., array_ops.newaxis] - + np.sqrt(2.) * scale[..., array_ops.newaxis] * grid) - return grid, probs - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def quadrature_scheme_lognormal_quantiles( - loc, scale, quadrature_size, - validate_args=False, name=None): - """Use LogNormal quantiles to form quadrature on positive-reals. - - Args: - loc: `float`-like (batch of) scalar `Tensor`; the location parameter of - the LogNormal prior. - scale: `float`-like (batch of) scalar `Tensor`; the scale parameter of - the LogNormal prior. - quadrature_size: Python `int` scalar representing the number of quadrature - points. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - name: Python `str` name prefixed to Ops created by this class. - - Returns: - grid: (Batch of) length-`quadrature_size` vectors representing the - `log_rate` parameters of a `Poisson`. - probs: (Batch of) length-`quadrature_size` vectors representing the - weight associate with each `grid` value. - """ - with ops.name_scope(name, "quadrature_scheme_lognormal_quantiles", - [loc, scale]): - # Create a LogNormal distribution. - dist = transformed_lib.TransformedDistribution( - distribution=normal_lib.Normal(loc=loc, scale=scale), - bijector=Exp(), - validate_args=validate_args) - batch_ndims = dist.batch_shape.ndims - if batch_ndims is None: - batch_ndims = array_ops.shape(dist.batch_shape_tensor())[0] - - def _compute_quantiles(): - """Helper to build quantiles.""" - # Omit {0, 1} since they might lead to Inf/NaN. - zero = array_ops.zeros([], dtype=dist.dtype) - edges = math_ops.linspace(zero, 1., quadrature_size + 3)[1:-1] - # Expand edges so its broadcast across batch dims. - edges = array_ops.reshape(edges, shape=array_ops.concat([ - [-1], array_ops.ones([batch_ndims], dtype=dtypes.int32)], axis=0)) - quantiles = dist.quantile(edges) - # Cyclically permute left by one. - perm = array_ops.concat([ - math_ops.range(1, 1 + batch_ndims), [0]], axis=0) - quantiles = array_ops.transpose(quantiles, perm) - return quantiles - quantiles = _compute_quantiles() - - # Compute grid as quantile midpoints. - grid = (quantiles[..., :-1] + quantiles[..., 1:]) / 2. - # Set shape hints. - grid.set_shape(dist.batch_shape.concatenate([quadrature_size])) - - # By construction probs is constant, i.e., `1 / quadrature_size`. This is - # important, because non-constant probs leads to non-reparameterizable - # samples. - probs = array_ops.fill( - dims=[quadrature_size], - value=1. / math_ops.cast(quadrature_size, dist.dtype)) - - return grid, probs - - -class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): - """`PoissonLogNormalQuadratureCompound` distribution. - - The `PoissonLogNormalQuadratureCompound` is an approximation to a - Poisson-LogNormal [compound distribution]( - https://en.wikipedia.org/wiki/Compound_probability_distribution), i.e., - - ```none - p(k|loc, scale) - = int_{R_+} dl LogNormal(l | loc, scale) Poisson(k | l) - approx= sum{ prob[d] Poisson(k | lambda(grid[d])) : d=0, ..., deg-1 } - ``` - - By default, the `grid` is chosen as quantiles of the `LogNormal` distribution - parameterized by `loc`, `scale` and the `prob` vector is - `[1. / quadrature_size]*quadrature_size`. - - In the non-approximation case, a draw from the LogNormal prior represents the - Poisson rate parameter. Unfortunately, the non-approximate distribution lacks - an analytical probability density function (pdf). Therefore the - `PoissonLogNormalQuadratureCompound` class implements an approximation based - on [quadrature](https://en.wikipedia.org/wiki/Numerical_integration). - - Note: although the `PoissonLogNormalQuadratureCompound` is approximately the - Poisson-LogNormal compound distribution, it is itself a valid distribution. - Viz., it possesses a `sample`, `log_prob`, `mean`, `variance`, etc. which are - all mutually consistent. - - #### Mathematical Details - - The `PoissonLogNormalQuadratureCompound` approximates a Poisson-LogNormal - [compound distribution]( - https://en.wikipedia.org/wiki/Compound_probability_distribution). Using - variable-substitution and [numerical quadrature]( - https://en.wikipedia.org/wiki/Numerical_integration) (default: - based on `LogNormal` quantiles) we can redefine the distribution to be a - parameter-less convex combination of `deg` different Poisson samples. - - That is, defined over positive integers, this distribution is parameterized - by a (batch of) `loc` and `scale` scalars. - - The probability density function (pdf) is, - - ```none - pdf(k | loc, scale, deg) - = sum{ prob[d] Poisson(k | lambda=exp(grid[d])) - : d=0, ..., deg-1 } - ``` - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Create two batches of PoissonLogNormalQuadratureCompounds, one with - # prior `loc = 0.` and another with `loc = 1.` In both cases `scale = 1.` - pln = tfd.PoissonLogNormalQuadratureCompound( - loc=[0., -0.5], - scale=1., - quadrature_size=10, - validate_args=True) - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc, - scale, - quadrature_size=8, - quadrature_fn=quadrature_scheme_lognormal_quantiles, - validate_args=False, - allow_nan_stats=True, - name="PoissonLogNormalQuadratureCompound"): - """Constructs the PoissonLogNormalQuadratureCompound`. - - Note: `probs` returned by (optional) `quadrature_fn` are presumed to be - either a length-`quadrature_size` vector or a batch of vectors in 1-to-1 - correspondence with the returned `grid`. (I.e., broadcasting is only - partially supported.) - - Args: - loc: `float`-like (batch of) scalar `Tensor`; the location parameter of - the LogNormal prior. - scale: `float`-like (batch of) scalar `Tensor`; the scale parameter of - the LogNormal prior. - quadrature_size: Python `int` scalar representing the number of quadrature - points. - quadrature_fn: Python callable taking `loc`, `scale`, - `quadrature_size`, `validate_args` and returning `tuple(grid, probs)` - representing the LogNormal grid and corresponding normalized weight. - normalized) weight. - Default value: `quadrature_scheme_lognormal_quantiles`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, - statistics (e.g., mean, mode, variance) use the value "`NaN`" to - indicate the result is undefined. When `False`, an exception is raised - if one or more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - TypeError: if `quadrature_grid` and `quadrature_probs` have different base - `dtype`. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[loc, scale]) as name: - if loc is not None: - loc = ops.convert_to_tensor(loc, name="loc") - if scale is not None: - scale = ops.convert_to_tensor( - scale, dtype=None if loc is None else loc.dtype, name="scale") - self._quadrature_grid, self._quadrature_probs = tuple(quadrature_fn( - loc, scale, quadrature_size, validate_args)) - - dt = self._quadrature_grid.dtype - if dt.base_dtype != self._quadrature_probs.dtype.base_dtype: - raise TypeError("Quadrature grid dtype ({}) does not match quadrature " - "probs dtype ({}).".format( - dt.name, self._quadrature_probs.dtype.name)) - - self._distribution = poisson_lib.Poisson( - log_rate=self._quadrature_grid, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats) - - self._mixture_distribution = categorical_lib.Categorical( - logits=math_ops.log(self._quadrature_probs), - validate_args=validate_args, - allow_nan_stats=allow_nan_stats) - - self._loc = loc - self._scale = scale - self._quadrature_size = quadrature_size - - super(PoissonLogNormalQuadratureCompound, self).__init__( - dtype=dt, - reparameterization_type=distribution_lib.NOT_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=[loc, scale], - name=name) - - @property - def mixture_distribution(self): - """Distribution which randomly selects a Poisson with quadrature param.""" - return self._mixture_distribution - - @property - def distribution(self): - """Base Poisson parameterized by a quadrature grid.""" - return self._distribution - - @property - def loc(self): - """Location parameter of the LogNormal prior.""" - return self._loc - - @property - def scale(self): - """Scale parameter of the LogNormal prior.""" - return self._scale - - @property - def quadrature_size(self): - return self._quadrature_size - - def _batch_shape_tensor(self): - return array_ops.broadcast_dynamic_shape( - self.distribution.batch_shape_tensor(), - array_ops.shape(self.mixture_distribution.logits))[:-1] - - def _batch_shape(self): - return array_ops.broadcast_static_shape( - self.distribution.batch_shape, - self.mixture_distribution.logits.shape)[:-1] - - def _event_shape(self): - return tensor_shape.TensorShape([]) - - def _sample_n(self, n, seed=None): - # Get ids as a [n, batch_size]-shaped matrix, unless batch_shape=[] then get - # ids as a [n]-shaped vector. - batch_size = self.batch_shape.num_elements() - if batch_size is None: - batch_size = math_ops.reduce_prod(self.batch_shape_tensor()) - # We need to "sample extra" from the mixture distribution if it doesn't - # already specify a probs vector for each batch coordinate. - # We only support this kind of reduced broadcasting, i.e., there is exactly - # one probs vector for all batch dims or one for each. - ids = self._mixture_distribution.sample( - sample_shape=concat_vectors( - [n], - distribution_util.pick_vector( - self.mixture_distribution.is_scalar_batch(), - [batch_size], - np.int32([]))), - seed=distribution_util.gen_new_seed( - seed, "poisson_lognormal_quadrature_compound")) - # We need to flatten batch dims in case mixture_distribution has its own - # batch dims. - ids = array_ops.reshape(ids, shape=concat_vectors( - [n], - distribution_util.pick_vector( - self.is_scalar_batch(), - np.int32([]), - np.int32([-1])))) - - # Stride `quadrature_size` for `batch_size` number of times. - offset = math_ops.range(start=0, - limit=batch_size * self._quadrature_size, - delta=self._quadrature_size, - dtype=ids.dtype) - ids += offset - rate = array_ops.gather( - array_ops.reshape(self.distribution.rate, shape=[-1]), ids) - rate = array_ops.reshape( - rate, shape=concat_vectors([n], self.batch_shape_tensor())) - return random_ops.random_poisson( - lam=rate, shape=[], dtype=self.dtype, seed=seed) - - def _log_prob(self, x): - return math_ops.reduce_logsumexp( - (self.mixture_distribution.logits - + self.distribution.log_prob(x[..., array_ops.newaxis])), - axis=-1) - - def _mean(self): - return math_ops.exp( - math_ops.reduce_logsumexp( - self.mixture_distribution.logits + self.distribution.log_rate, - axis=-1)) - - def _variance(self): - return math_ops.exp(self._log_variance()) - - def _stddev(self): - return math_ops.exp(0.5 * self._log_variance()) - - def _log_variance(self): - # Following calculation is based on law of total variance: - # - # Var[Z] = E[Var[Z | V]] + Var[E[Z | V]] - # - # where, - # - # Z|v ~ interpolate_affine[v](distribution) - # V ~ mixture_distribution - # - # thus, - # - # E[Var[Z | V]] = sum{ prob[d] Var[d] : d=0, ..., deg-1 } - # Var[E[Z | V]] = sum{ prob[d] (Mean[d] - Mean)**2 : d=0, ..., deg-1 } - v = array_ops.stack([ - # log(self.distribution.variance()) = log(Var[d]) = log(rate[d]) - self.distribution.log_rate, - # log((Mean[d] - Mean)**2) - 2. * math_ops.log( - math_ops.abs(self.distribution.mean() - - self._mean()[..., array_ops.newaxis])), - ], axis=-1) - return math_ops.reduce_logsumexp( - self.mixture_distribution.logits[..., array_ops.newaxis] + v, - axis=[-2, -1]) - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def concat_vectors(*args): - """Concatenates input vectors, statically if possible.""" - args_ = [distribution_util.static_value(x) for x in args] - if any(vec is None for vec in args_): - return array_ops.concat(args, axis=0) - return [val for vec in args_ for val in vec] diff --git a/tensorflow/contrib/distributions/python/ops/quantized_distribution.py b/tensorflow/contrib/distributions/python/ops/quantized_distribution.py deleted file mode 100644 index 134658deabe..00000000000 --- a/tensorflow/contrib/distributions/python/ops/quantized_distribution.py +++ /dev/null @@ -1,587 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Quantized distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import distribution as distributions -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - -__all__ = ["QuantizedDistribution"] - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def _logsum_expbig_minus_expsmall(big, small): - """Stable evaluation of `Log[exp{big} - exp{small}]`. - - To work correctly, we should have the pointwise relation: `small <= big`. - - Args: - big: Floating-point `Tensor` - small: Floating-point `Tensor` with same `dtype` as `big` and broadcastable - shape. - - Returns: - `Tensor` of same `dtype` of `big` and broadcast shape. - """ - with ops.name_scope("logsum_expbig_minus_expsmall", values=[small, big]): - return math_ops.log(1. - math_ops.exp(small - big)) + big - - -_prob_base_note = """ -For whole numbers `y`, - -``` -P[Y = y] := P[X <= low], if y == low, - := P[X > high - 1], y == high, - := 0, if j < low or y > high, - := P[y - 1 < X <= y], all other y. -``` - -""" - -_prob_note = _prob_base_note + """ -The base distribution's `cdf` method must be defined on `y - 1`. If the -base distribution has a `survival_function` method, results will be more -accurate for large values of `y`, and in this case the `survival_function` must -also be defined on `y - 1`. -""" - -_log_prob_note = _prob_base_note + """ -The base distribution's `log_cdf` method must be defined on `y - 1`. If the -base distribution has a `log_survival_function` method results will be more -accurate for large values of `y`, and in this case the `log_survival_function` -must also be defined on `y - 1`. -""" - - -_cdf_base_note = """ - -For whole numbers `y`, - -``` -cdf(y) := P[Y <= y] - = 1, if y >= high, - = 0, if y < low, - = P[X <= y], otherwise. -``` - -Since `Y` only has mass at whole numbers, `P[Y <= y] = P[Y <= floor(y)]`. -This dictates that fractional `y` are first floored to a whole number, and -then above definition applies. -""" - -_cdf_note = _cdf_base_note + """ -The base distribution's `cdf` method must be defined on `y - 1`. -""" - -_log_cdf_note = _cdf_base_note + """ -The base distribution's `log_cdf` method must be defined on `y - 1`. -""" - - -_sf_base_note = """ - -For whole numbers `y`, - -``` -survival_function(y) := P[Y > y] - = 0, if y >= high, - = 1, if y < low, - = P[X <= y], otherwise. -``` - -Since `Y` only has mass at whole numbers, `P[Y <= y] = P[Y <= floor(y)]`. -This dictates that fractional `y` are first floored to a whole number, and -then above definition applies. -""" - -_sf_note = _sf_base_note + """ -The base distribution's `cdf` method must be defined on `y - 1`. -""" - -_log_sf_note = _sf_base_note + """ -The base distribution's `log_cdf` method must be defined on `y - 1`. -""" - - -class QuantizedDistribution(distributions.Distribution): - """Distribution representing the quantization `Y = ceiling(X)`. - - #### Definition in Terms of Sampling - - ``` - 1. Draw X - 2. Set Y <-- ceiling(X) - 3. If Y < low, reset Y <-- low - 4. If Y > high, reset Y <-- high - 5. Return Y - ``` - - #### Definition in Terms of the Probability Mass Function - - Given scalar random variable `X`, we define a discrete random variable `Y` - supported on the integers as follows: - - ``` - P[Y = j] := P[X <= low], if j == low, - := P[X > high - 1], j == high, - := 0, if j < low or j > high, - := P[j - 1 < X <= j], all other j. - ``` - - Conceptually, without cutoffs, the quantization process partitions the real - line `R` into half open intervals, and identifies an integer `j` with the - right endpoints: - - ``` - R = ... (-2, -1](-1, 0](0, 1](1, 2](2, 3](3, 4] ... - j = ... -1 0 1 2 3 4 ... - ``` - - `P[Y = j]` is the mass of `X` within the `jth` interval. - If `low = 0`, and `high = 2`, then the intervals are redrawn - and `j` is re-assigned: - - ``` - R = (-infty, 0](0, 1](1, infty) - j = 0 1 2 - ``` - - `P[Y = j]` is still the mass of `X` within the `jth` interval. - - #### Examples - - We illustrate a mixture of discretized logistic distributions - [(Salimans et al., 2017)][1]. This is used, for example, for capturing 16-bit - audio in WaveNet [(van den Oord et al., 2017)][2]. The values range in - a 1-D integer domain of `[0, 2**16-1]`, and the discretization captures - `P(x - 0.5 < X <= x + 0.5)` for all `x` in the domain excluding the endpoints. - The lowest value has probability `P(X <= 0.5)` and the highest value has - probability `P(2**16 - 1.5 < X)`. - - Below we assume a `wavenet` function. It takes as `input` right-shifted audio - samples of shape `[..., sequence_length]`. It returns a real-valued tensor of - shape `[..., num_mixtures * 3]`, i.e., each mixture component has a `loc` and - `scale` parameter belonging to the logistic distribution, and a `logits` - parameter determining the unnormalized probability of that component. - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - tfb = tfp.bijectors - - net = wavenet(inputs) - loc, unconstrained_scale, logits = tf.split(net, - num_or_size_splits=3, - axis=-1) - scale = tf.nn.softplus(unconstrained_scale) - - # Form mixture of discretized logistic distributions. Note we shift the - # logistic distribution by -0.5. This lets the quantization capture "rounding" - # intervals, `(x-0.5, x+0.5]`, and not "ceiling" intervals, `(x-1, x]`. - discretized_logistic_dist = tfd.QuantizedDistribution( - distribution=tfd.TransformedDistribution( - distribution=tfd.Logistic(loc=loc, scale=scale), - bijector=tfb.AffineScalar(shift=-0.5)), - low=0., - high=2**16 - 1.) - mixture_dist = tfd.MixtureSameFamily( - mixture_distribution=tfd.Categorical(logits=logits), - components_distribution=discretized_logistic_dist) - - neg_log_likelihood = -tf.reduce_sum(mixture_dist.log_prob(targets)) - train_op = tf.train.AdamOptimizer().minimize(neg_log_likelihood) - ``` - - After instantiating `mixture_dist`, we illustrate maximum likelihood by - calculating its log-probability of audio samples as `target` and optimizing. - - #### References - - [1]: Tim Salimans, Andrej Karpathy, Xi Chen, and Diederik P. Kingma. - PixelCNN++: Improving the PixelCNN with discretized logistic mixture - likelihood and other modifications. - _International Conference on Learning Representations_, 2017. - https://arxiv.org/abs/1701.05517 - [2]: Aaron van den Oord et al. Parallel WaveNet: Fast High-Fidelity Speech - Synthesis. _arXiv preprint arXiv:1711.10433_, 2017. - https://arxiv.org/abs/1711.10433 - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - distribution, - low=None, - high=None, - validate_args=False, - name="QuantizedDistribution"): - """Construct a Quantized Distribution representing `Y = ceiling(X)`. - - Some properties are inherited from the distribution defining `X`. Example: - `allow_nan_stats` is determined for this `QuantizedDistribution` by reading - the `distribution`. - - Args: - distribution: The base distribution class to transform. Typically an - instance of `Distribution`. - low: `Tensor` with same `dtype` as this distribution and shape - able to be added to samples. Should be a whole number. Default `None`. - If provided, base distribution's `prob` should be defined at - `low`. - high: `Tensor` with same `dtype` as this distribution and shape - able to be added to samples. Should be a whole number. Default `None`. - If provided, base distribution's `prob` should be defined at - `high - 1`. - `high` must be strictly greater than `low`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - TypeError: If `dist_cls` is not a subclass of - `Distribution` or continuous. - NotImplementedError: If the base distribution does not implement `cdf`. - """ - parameters = dict(locals()) - values = ( - list(distribution.parameters.values()) + - [low, high]) - with ops.name_scope(name, values=values) as name: - self._dist = distribution - - if low is not None: - low = ops.convert_to_tensor(low, name="low") - if high is not None: - high = ops.convert_to_tensor(high, name="high") - check_ops.assert_same_float_dtype( - tensors=[self.distribution, low, high]) - - # We let QuantizedDistribution access _graph_parents since this class is - # more like a baseclass. - graph_parents = self._dist._graph_parents # pylint: disable=protected-access - - checks = [] - if validate_args and low is not None and high is not None: - message = "low must be strictly less than high." - checks.append( - check_ops.assert_less( - low, high, message=message)) - self._validate_args = validate_args # self._check_integer uses this. - with ops.control_dependencies(checks if validate_args else []): - if low is not None: - self._low = self._check_integer(low) - graph_parents += [self._low] - else: - self._low = None - if high is not None: - self._high = self._check_integer(high) - graph_parents += [self._high] - else: - self._high = None - - super(QuantizedDistribution, self).__init__( - dtype=self._dist.dtype, - reparameterization_type=distributions.NOT_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=self._dist.allow_nan_stats, - parameters=parameters, - graph_parents=graph_parents, - name=name) - - @property - def distribution(self): - """Base distribution, p(x).""" - return self._dist - - @property - def low(self): - """Lowest value that quantization returns.""" - return self._low - - @property - def high(self): - """Highest value that quantization returns.""" - return self._high - - def _batch_shape_tensor(self): - return self.distribution.batch_shape_tensor() - - def _batch_shape(self): - return self.distribution.batch_shape - - def _event_shape_tensor(self): - return self.distribution.event_shape_tensor() - - def _event_shape(self): - return self.distribution.event_shape - - def _sample_n(self, n, seed=None): - low = self._low - high = self._high - with ops.name_scope("transform"): - n = ops.convert_to_tensor(n, name="n") - x_samps = self.distribution.sample(n, seed=seed) - ones = array_ops.ones_like(x_samps) - - # Snap values to the intervals (j - 1, j]. - result_so_far = math_ops.ceil(x_samps) - - if low is not None: - result_so_far = array_ops.where(result_so_far < low, - low * ones, result_so_far) - - if high is not None: - result_so_far = array_ops.where(result_so_far > high, - high * ones, result_so_far) - - return result_so_far - - @distribution_util.AppendDocstring(_log_prob_note) - def _log_prob(self, y): - if not hasattr(self.distribution, "_log_cdf"): - raise NotImplementedError( - "'log_prob' not implemented unless the base distribution implements " - "'log_cdf'") - y = self._check_integer(y) - try: - return self._log_prob_with_logsf_and_logcdf(y) - except NotImplementedError: - return self._log_prob_with_logcdf(y) - - def _log_prob_with_logcdf(self, y): - return _logsum_expbig_minus_expsmall(self.log_cdf(y), self.log_cdf(y - 1)) - - def _log_prob_with_logsf_and_logcdf(self, y): - """Compute log_prob(y) using log survival_function and cdf together.""" - # There are two options that would be equal if we had infinite precision: - # Log[ sf(y - 1) - sf(y) ] - # = Log[ exp{logsf(y - 1)} - exp{logsf(y)} ] - # Log[ cdf(y) - cdf(y - 1) ] - # = Log[ exp{logcdf(y)} - exp{logcdf(y - 1)} ] - logsf_y = self.log_survival_function(y) - logsf_y_minus_1 = self.log_survival_function(y - 1) - logcdf_y = self.log_cdf(y) - logcdf_y_minus_1 = self.log_cdf(y - 1) - - # Important: Here we use select in a way such that no input is inf, this - # prevents the troublesome case where the output of select can be finite, - # but the output of grad(select) will be NaN. - - # In either case, we are doing Log[ exp{big} - exp{small} ] - # We want to use the sf items precisely when we are on the right side of the - # median, which occurs when logsf_y < logcdf_y. - big = array_ops.where(logsf_y < logcdf_y, logsf_y_minus_1, logcdf_y) - small = array_ops.where(logsf_y < logcdf_y, logsf_y, logcdf_y_minus_1) - - return _logsum_expbig_minus_expsmall(big, small) - - @distribution_util.AppendDocstring(_prob_note) - def _prob(self, y): - if not hasattr(self.distribution, "_cdf"): - raise NotImplementedError( - "'prob' not implemented unless the base distribution implements " - "'cdf'") - y = self._check_integer(y) - try: - return self._prob_with_sf_and_cdf(y) - except NotImplementedError: - return self._prob_with_cdf(y) - - def _prob_with_cdf(self, y): - return self.cdf(y) - self.cdf(y - 1) - - def _prob_with_sf_and_cdf(self, y): - # There are two options that would be equal if we had infinite precision: - # sf(y - 1) - sf(y) - # cdf(y) - cdf(y - 1) - sf_y = self.survival_function(y) - sf_y_minus_1 = self.survival_function(y - 1) - cdf_y = self.cdf(y) - cdf_y_minus_1 = self.cdf(y - 1) - - # sf_prob has greater precision iff we're on the right side of the median. - return array_ops.where( - sf_y < cdf_y, # True iff we're on the right side of the median. - sf_y_minus_1 - sf_y, - cdf_y - cdf_y_minus_1) - - @distribution_util.AppendDocstring(_log_cdf_note) - def _log_cdf(self, y): - low = self._low - high = self._high - - # Recall the promise: - # cdf(y) := P[Y <= y] - # = 1, if y >= high, - # = 0, if y < low, - # = P[X <= y], otherwise. - - # P[Y <= j] = P[floor(Y) <= j] since mass is only at integers, not in - # between. - j = math_ops.floor(y) - - result_so_far = self.distribution.log_cdf(j) - - # Broadcast, because it's possible that this is a single distribution being - # evaluated on a number of samples, or something like that. - j += array_ops.zeros_like(result_so_far) - - # Re-define values at the cutoffs. - if low is not None: - neg_inf = -np.inf * array_ops.ones_like(result_so_far) - result_so_far = array_ops.where(j < low, neg_inf, result_so_far) - if high is not None: - result_so_far = array_ops.where(j >= high, - array_ops.zeros_like(result_so_far), - result_so_far) - - return result_so_far - - @distribution_util.AppendDocstring(_cdf_note) - def _cdf(self, y): - low = self._low - high = self._high - - # Recall the promise: - # cdf(y) := P[Y <= y] - # = 1, if y >= high, - # = 0, if y < low, - # = P[X <= y], otherwise. - - # P[Y <= j] = P[floor(Y) <= j] since mass is only at integers, not in - # between. - j = math_ops.floor(y) - - # P[X <= j], used when low < X < high. - result_so_far = self.distribution.cdf(j) - - # Broadcast, because it's possible that this is a single distribution being - # evaluated on a number of samples, or something like that. - j += array_ops.zeros_like(result_so_far) - - # Re-define values at the cutoffs. - if low is not None: - result_so_far = array_ops.where(j < low, - array_ops.zeros_like(result_so_far), - result_so_far) - if high is not None: - result_so_far = array_ops.where(j >= high, - array_ops.ones_like(result_so_far), - result_so_far) - - return result_so_far - - @distribution_util.AppendDocstring(_log_sf_note) - def _log_survival_function(self, y): - low = self._low - high = self._high - - # Recall the promise: - # survival_function(y) := P[Y > y] - # = 0, if y >= high, - # = 1, if y < low, - # = P[X > y], otherwise. - - # P[Y > j] = P[ceiling(Y) > j] since mass is only at integers, not in - # between. - j = math_ops.ceil(y) - - # P[X > j], used when low < X < high. - result_so_far = self.distribution.log_survival_function(j) - - # Broadcast, because it's possible that this is a single distribution being - # evaluated on a number of samples, or something like that. - j += array_ops.zeros_like(result_so_far) - - # Re-define values at the cutoffs. - if low is not None: - result_so_far = array_ops.where(j < low, - array_ops.zeros_like(result_so_far), - result_so_far) - if high is not None: - neg_inf = -np.inf * array_ops.ones_like(result_so_far) - result_so_far = array_ops.where(j >= high, neg_inf, result_so_far) - - return result_so_far - - @distribution_util.AppendDocstring(_sf_note) - def _survival_function(self, y): - low = self._low - high = self._high - - # Recall the promise: - # survival_function(y) := P[Y > y] - # = 0, if y >= high, - # = 1, if y < low, - # = P[X > y], otherwise. - - # P[Y > j] = P[ceiling(Y) > j] since mass is only at integers, not in - # between. - j = math_ops.ceil(y) - - # P[X > j], used when low < X < high. - result_so_far = self.distribution.survival_function(j) - - # Broadcast, because it's possible that this is a single distribution being - # evaluated on a number of samples, or something like that. - j += array_ops.zeros_like(result_so_far) - - # Re-define values at the cutoffs. - if low is not None: - result_so_far = array_ops.where(j < low, - array_ops.ones_like(result_so_far), - result_so_far) - if high is not None: - result_so_far = array_ops.where(j >= high, - array_ops.zeros_like(result_so_far), - result_so_far) - - return result_so_far - - def _check_integer(self, value): - with ops.name_scope("check_integer", values=[value]): - value = ops.convert_to_tensor(value, name="value") - if not self.validate_args: - return value - dependencies = [distribution_util.assert_integer_form( - value, message="value has non-integer components.")] - return control_flow_ops.with_dependencies(dependencies, value) diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py b/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py deleted file mode 100644 index 7e1f64dc425..00000000000 --- a/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The RelaxedBernoulli distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import logistic -from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid import Sigmoid -# Bijectors must be directly imported because `remove_undocumented` prevents -# individual file imports. -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops.distributions import transformed_distribution -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - - -class RelaxedBernoulli(transformed_distribution.TransformedDistribution): - """RelaxedBernoulli distribution with temperature and logits parameters. - - The RelaxedBernoulli is a distribution over the unit interval (0,1), which - continuously approximates a Bernoulli. The degree of approximation is - controlled by a temperature: as the temperature goes to 0 the - RelaxedBernoulli becomes discrete with a distribution described by the - `logits` or `probs` parameters, as the temperature goes to infinity the - RelaxedBernoulli becomes the constant distribution that is identically 0.5. - - The RelaxedBernoulli distribution is a reparameterized continuous - distribution that is the binary special case of the RelaxedOneHotCategorical - distribution (Maddison et al., 2016; Jang et al., 2016). For details on the - binary special case see the appendix of Maddison et al. (2016) where it is - referred to as BinConcrete. If you use this distribution, please cite both - papers. - - Some care needs to be taken for loss functions that depend on the - log-probability of RelaxedBernoullis, because computing log-probabilities of - the RelaxedBernoulli can suffer from underflow issues. In many case loss - functions such as these are invariant under invertible transformations of - the random variables. The KL divergence, found in the variational autoencoder - loss, is an example. Because RelaxedBernoullis are sampled by a Logistic - random variable followed by a `tf.sigmoid` op, one solution is to treat - the Logistic as the random variable and `tf.sigmoid` as downstream. The - KL divergences of two Logistics, which are always followed by a `tf.sigmoid` - op, is equivalent to evaluating KL divergences of RelaxedBernoulli samples. - See Maddison et al., 2016 for more details where this distribution is called - the BinConcrete. - - An alternative approach is to evaluate Bernoulli log probability or KL - directly on relaxed samples, as done in Jang et al., 2016. In this case, - guarantees on the loss are usually violated. For instance, using a Bernoulli - KL in a relaxed ELBO is no longer a lower bound on the log marginal - probability of the observation. Thus care and early stopping are important. - - #### Examples - - Creates three continuous distributions, which approximate 3 Bernoullis with - probabilities (0.1, 0.5, 0.4). Samples from these distributions will be in - the unit interval (0,1). - - ```python - temperature = 0.5 - p = [0.1, 0.5, 0.4] - dist = RelaxedBernoulli(temperature, probs=p) - ``` - - Creates three continuous distributions, which approximate 3 Bernoullis with - logits (-2, 2, 0). Samples from these distributions will be in - the unit interval (0,1). - - ```python - temperature = 0.5 - logits = [-2, 2, 0] - dist = RelaxedBernoulli(temperature, logits=logits) - ``` - - Creates three continuous distributions, whose sigmoid approximate 3 Bernoullis - with logits (-2, 2, 0). - - ```python - temperature = 0.5 - logits = [-2, 2, 0] - dist = Logistic(logits/temperature, 1./temperature) - samples = dist.sample() - sigmoid_samples = tf.sigmoid(samples) - # sigmoid_samples has the same distribution as samples from - # RelaxedBernoulli(temperature, logits=logits) - ``` - - Creates three continuous distributions, which approximate 3 Bernoullis with - logits (-2, 2, 0). Samples from these distributions will be in - the unit interval (0,1). Because the temperature is very low, samples from - these distributions are almost discrete, usually taking values very close to 0 - or 1. - - ```python - temperature = 1e-5 - logits = [-2, 2, 0] - dist = RelaxedBernoulli(temperature, logits=logits) - ``` - - Creates three continuous distributions, which approximate 3 Bernoullis with - logits (-2, 2, 0). Samples from these distributions will be in - the unit interval (0,1). Because the temperature is very high, samples from - these distributions are usually close to the (0.5, 0.5, 0.5) vector. - - ```python - temperature = 100 - logits = [-2, 2, 0] - dist = RelaxedBernoulli(temperature, logits=logits) - ``` - - Chris J. Maddison, Andriy Mnih, and Yee Whye Teh. The Concrete Distribution: - A Continuous Relaxation of Discrete Random Variables. 2016. - - Eric Jang, Shixiang Gu, and Ben Poole. Categorical Reparameterization with - Gumbel-Softmax. 2016. - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - temperature, - logits=None, - probs=None, - validate_args=False, - allow_nan_stats=True, - name="RelaxedBernoulli"): - """Construct RelaxedBernoulli distributions. - - Args: - temperature: An 0-D `Tensor`, representing the temperature - of a set of RelaxedBernoulli distributions. The temperature should be - positive. - logits: An N-D `Tensor` representing the log-odds - of a positive event. Each entry in the `Tensor` parametrizes - an independent RelaxedBernoulli distribution where the probability of an - event is sigmoid(logits). Only one of `logits` or `probs` should be - passed in. - probs: An N-D `Tensor` representing the probability of a positive event. - Each entry in the `Tensor` parameterizes an independent Bernoulli - distribution. Only one of `logits` or `probs` should be passed in. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - ValueError: If both `probs` and `logits` are passed, or if neither. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[logits, probs, temperature]) as name: - with ops.control_dependencies([check_ops.assert_positive(temperature)] - if validate_args else []): - self._temperature = array_ops.identity(temperature, name="temperature") - self._logits, self._probs = distribution_util.get_logits_and_probs( - logits=logits, probs=probs, validate_args=validate_args) - super(RelaxedBernoulli, self).__init__( - distribution=logistic.Logistic( - self._logits / self._temperature, - 1. / self._temperature, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name + "/Logistic"), - bijector=Sigmoid(validate_args=validate_args), - validate_args=validate_args, - name=name) - self._parameters = parameters - - @staticmethod - def _param_shapes(sample_shape): - return {"logits": ops.convert_to_tensor(sample_shape, dtype=dtypes.int32)} - - @property - def temperature(self): - """Distribution parameter for the location.""" - return self._temperature - - @property - def logits(self): - """Log-odds of `1`.""" - return self._logits - - @property - def probs(self): - """Probability of `1`.""" - return self._probs diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py deleted file mode 100644 index 25aaac379a7..00000000000 --- a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py +++ /dev/null @@ -1,430 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Relaxed OneHotCategorical distribution classes.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.ops.distributions import transformed_distribution -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - - -class ExpRelaxedOneHotCategorical(distribution.Distribution): - """ExpRelaxedOneHotCategorical distribution with temperature and logits. - - An ExpRelaxedOneHotCategorical distribution is a log-transformed - RelaxedOneHotCategorical distribution. The RelaxedOneHotCategorical is a - distribution over random probability vectors, vectors of positive real - values that sum to one, which continuously approximates a OneHotCategorical. - The degree of approximation is controlled by a temperature: as the temperature - goes to 0 the RelaxedOneHotCategorical becomes discrete with a distribution - described by the logits, as the temperature goes to infinity the - RelaxedOneHotCategorical becomes the constant distribution that is identically - the constant vector of (1/event_size, ..., 1/event_size). - - Because computing log-probabilities of the RelaxedOneHotCategorical can - suffer from underflow issues, this class is one solution for loss - functions that depend on log-probabilities, such as the KL Divergence found - in the variational autoencoder loss. The KL divergence between two - distributions is invariant under invertible transformations, so evaluating - KL divergences of ExpRelaxedOneHotCategorical samples, which are always - followed by a `tf.exp` op, is equivalent to evaluating KL divergences of - RelaxedOneHotCategorical samples. See the appendix of Maddison et al., 2016 - for more mathematical details, where this distribution is called the - ExpConcrete. - - #### Examples - - Creates a continuous distribution, whose exp approximates a 3-class one-hot - categorical distribution. The 2nd class is the most likely to be the - largest component in samples drawn from this distribution. If those samples - are followed by a `tf.exp` op, then they are distributed as a relaxed onehot - categorical. - - ```python - temperature = 0.5 - p = [0.1, 0.5, 0.4] - dist = ExpRelaxedOneHotCategorical(temperature, probs=p) - samples = dist.sample() - exp_samples = tf.exp(samples) - # exp_samples has the same distribution as samples from - # RelaxedOneHotCategorical(temperature, probs=p) - ``` - - Creates a continuous distribution, whose exp approximates a 3-class one-hot - categorical distribution. The 2nd class is the most likely to be the - largest component in samples drawn from this distribution. - - ```python - temperature = 0.5 - logits = [-2, 2, 0] - dist = ExpRelaxedOneHotCategorical(temperature, logits=logits) - samples = dist.sample() - exp_samples = tf.exp(samples) - # exp_samples has the same distribution as samples from - # RelaxedOneHotCategorical(temperature, probs=p) - ``` - - Creates a continuous distribution, whose exp approximates a 3-class one-hot - categorical distribution. Because the temperature is very low, samples from - this distribution are almost discrete, with one component almost 0 and the - others very negative. The 2nd class is the most likely to be the largest - component in samples drawn from this distribution. - - ```python - temperature = 1e-5 - logits = [-2, 2, 0] - dist = ExpRelaxedOneHotCategorical(temperature, logits=logits) - samples = dist.sample() - exp_samples = tf.exp(samples) - # exp_samples has the same distribution as samples from - # RelaxedOneHotCategorical(temperature, probs=p) - ``` - - Creates a continuous distribution, whose exp approximates a 3-class one-hot - categorical distribution. Because the temperature is very high, samples from - this distribution are usually close to the (-log(3), -log(3), -log(3)) vector. - The 2nd class is still the most likely to be the largest component - in samples drawn from this distribution. - - ```python - temperature = 10 - logits = [-2, 2, 0] - dist = ExpRelaxedOneHotCategorical(temperature, logits=logits) - samples = dist.sample() - exp_samples = tf.exp(samples) - # exp_samples has the same distribution as samples from - # RelaxedOneHotCategorical(temperature, probs=p) - ``` - - Chris J. Maddison, Andriy Mnih, and Yee Whye Teh. The Concrete Distribution: - A Continuous Relaxation of Discrete Random Variables. 2016. - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__( - self, - temperature, - logits=None, - probs=None, - dtype=None, - validate_args=False, - allow_nan_stats=True, - name="ExpRelaxedOneHotCategorical"): - """Initialize ExpRelaxedOneHotCategorical using class log-probabilities. - - Args: - temperature: An 0-D `Tensor`, representing the temperature - of a set of ExpRelaxedCategorical distributions. The temperature should - be positive. - logits: An N-D `Tensor`, `N >= 1`, representing the log probabilities - of a set of ExpRelaxedCategorical distributions. The first - `N - 1` dimensions index into a batch of independent distributions and - the last dimension represents a vector of logits for each class. Only - one of `logits` or `probs` should be passed in. - probs: An N-D `Tensor`, `N >= 1`, representing the probabilities - of a set of ExpRelaxedCategorical distributions. The first - `N - 1` dimensions index into a batch of independent distributions and - the last dimension represents a vector of probabilities for each - class. Only one of `logits` or `probs` should be passed in. - dtype: The type of the event samples (default: inferred from - logits/probs). - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[logits, probs, temperature]) as name: - - self._logits, self._probs = distribution_util.get_logits_and_probs( - name=name, logits=logits, probs=probs, validate_args=validate_args, - multidimensional=True) - - if dtype is None: - dtype = self._logits.dtype - if not validate_args: - temperature = math_ops.cast(temperature, dtype) - - with ops.control_dependencies([check_ops.assert_positive(temperature)] - if validate_args else []): - self._temperature = array_ops.identity(temperature, name="temperature") - self._temperature_2d = array_ops.reshape(temperature, [-1, 1], - name="temperature_2d") - - logits_shape_static = self._logits.get_shape().with_rank_at_least(1) - if logits_shape_static.ndims is not None: - self._batch_rank = ops.convert_to_tensor( - logits_shape_static.ndims - 1, - dtype=dtypes.int32, - name="batch_rank") - else: - with ops.name_scope(name="batch_rank"): - self._batch_rank = array_ops.rank(self._logits) - 1 - - with ops.name_scope(name="event_size"): - self._event_size = array_ops.shape(self._logits)[-1] - - super(ExpRelaxedOneHotCategorical, self).__init__( - dtype=dtype, - reparameterization_type=distribution.FULLY_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=[self._logits, - self._probs, - self._temperature], - name=name) - - @property - def event_size(self): - """Scalar `int32` tensor: the number of classes.""" - return self._event_size - - @property - def temperature(self): - """Batchwise temperature tensor of a RelaxedCategorical.""" - return self._temperature - - @property - def logits(self): - """Vector of coordinatewise logits.""" - return self._logits - - @property - def probs(self): - """Vector of probabilities summing to one.""" - return self._probs - - def _batch_shape_tensor(self): - return array_ops.shape(self._logits)[:-1] - - def _batch_shape(self): - return self.logits.get_shape()[:-1] - - def _event_shape_tensor(self): - return array_ops.shape(self.logits)[-1:] - - def _event_shape(self): - return self.logits.get_shape().with_rank_at_least(1)[-1:] - - def _sample_n(self, n, seed=None): - sample_shape = array_ops.concat([[n], array_ops.shape(self.logits)], 0) - logits = self.logits * array_ops.ones(sample_shape, dtype=self.dtype) - logits_2d = array_ops.reshape(logits, [-1, self.event_size]) - # Uniform variates must be sampled from the open-interval `(0, 1)` rather - # than `[0, 1)`. To do so, we use `np.finfo(self.dtype.as_numpy_dtype).tiny` - # because it is the smallest, positive, "normal" number. A "normal" number - # is such that the mantissa has an implicit leading 1. Normal, positive - # numbers x, y have the reasonable property that, `x + y >= max(x, y)`. In - # this case, a subnormal number (i.e., np.nextafter) can cause us to sample - # 0. - uniform = random_ops.random_uniform( - shape=array_ops.shape(logits_2d), - minval=np.finfo(self.dtype.as_numpy_dtype).tiny, - maxval=1., - dtype=self.dtype, - seed=seed) - gumbel = -math_ops.log(-math_ops.log(uniform)) - noisy_logits = math_ops.div(gumbel + logits_2d, self._temperature_2d) - samples = nn_ops.log_softmax(noisy_logits) - ret = array_ops.reshape(samples, sample_shape) - return ret - - def _log_prob(self, x): - x = self._assert_valid_sample(x) - # broadcast logits or x if need be. - logits = self.logits - if (not x.get_shape().is_fully_defined() or - not logits.get_shape().is_fully_defined() or - x.get_shape() != logits.get_shape()): - logits = array_ops.ones_like(x, dtype=logits.dtype) * logits - x = array_ops.ones_like(logits, dtype=x.dtype) * x - logits_shape = array_ops.shape(math_ops.reduce_sum(logits, axis=[-1])) - logits_2d = array_ops.reshape(logits, [-1, self.event_size]) - x_2d = array_ops.reshape(x, [-1, self.event_size]) - # compute the normalization constant - k = math_ops.cast(self.event_size, x.dtype) - log_norm_const = (math_ops.lgamma(k) - + (k - 1.) - * math_ops.log(self.temperature)) - # compute the unnormalized density - log_softmax = nn_ops.log_softmax(logits_2d - x_2d * self._temperature_2d) - log_unnorm_prob = math_ops.reduce_sum(log_softmax, [-1], keepdims=False) - # combine unnormalized density with normalization constant - log_prob = log_norm_const + log_unnorm_prob - # Reshapes log_prob to be consistent with shape of user-supplied logits - ret = array_ops.reshape(log_prob, logits_shape) - return ret - - def _assert_valid_sample(self, x): - if not self.validate_args: - return x - return control_flow_ops.with_dependencies([ - check_ops.assert_non_positive(x), - check_ops.assert_near( - array_ops.zeros([], dtype=self.dtype), - math_ops.reduce_logsumexp(x, axis=[-1])), - ], x) - - -class RelaxedOneHotCategorical( - transformed_distribution.TransformedDistribution): - """RelaxedOneHotCategorical distribution with temperature and logits. - - The RelaxedOneHotCategorical is a distribution over random probability - vectors, vectors of positive real values that sum to one, which continuously - approximates a OneHotCategorical. The degree of approximation is controlled by - a temperature: as the temperature goes to 0 the RelaxedOneHotCategorical - becomes discrete with a distribution described by the `logits` or `probs` - parameters, as the temperature goes to infinity the RelaxedOneHotCategorical - becomes the constant distribution that is identically the constant vector of - (1/event_size, ..., 1/event_size). - - The RelaxedOneHotCategorical distribution was concurrently introduced as the - Gumbel-Softmax (Jang et al., 2016) and Concrete (Maddison et al., 2016) - distributions for use as a reparameterized continuous approximation to the - `Categorical` one-hot distribution. If you use this distribution, please cite - both papers. - - #### Examples - - Creates a continuous distribution, which approximates a 3-class one-hot - categorical distribution. The 2nd class is the most likely to be the - largest component in samples drawn from this distribution. - - ```python - temperature = 0.5 - p = [0.1, 0.5, 0.4] - dist = RelaxedOneHotCategorical(temperature, probs=p) - ``` - - Creates a continuous distribution, which approximates a 3-class one-hot - categorical distribution. The 2nd class is the most likely to be the - largest component in samples drawn from this distribution. - - ```python - temperature = 0.5 - logits = [-2, 2, 0] - dist = RelaxedOneHotCategorical(temperature, logits=logits) - ``` - - Creates a continuous distribution, which approximates a 3-class one-hot - categorical distribution. Because the temperature is very low, samples from - this distribution are almost discrete, with one component almost 1 and the - others nearly 0. The 2nd class is the most likely to be the largest component - in samples drawn from this distribution. - - ```python - temperature = 1e-5 - logits = [-2, 2, 0] - dist = RelaxedOneHotCategorical(temperature, logits=logits) - ``` - - Creates a continuous distribution, which approximates a 3-class one-hot - categorical distribution. Because the temperature is very high, samples from - this distribution are usually close to the (1/3, 1/3, 1/3) vector. The 2nd - class is still the most likely to be the largest component - in samples drawn from this distribution. - - ```python - temperature = 10 - logits = [-2, 2, 0] - dist = RelaxedOneHotCategorical(temperature, logits=logits) - ``` - - Eric Jang, Shixiang Gu, and Ben Poole. Categorical Reparameterization with - Gumbel-Softmax. 2016. - - Chris J. Maddison, Andriy Mnih, and Yee Whye Teh. The Concrete Distribution: - A Continuous Relaxation of Discrete Random Variables. 2016. - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__( - self, - temperature, - logits=None, - probs=None, - dtype=None, - validate_args=False, - allow_nan_stats=True, - name="RelaxedOneHotCategorical"): - """Initialize RelaxedOneHotCategorical using class log-probabilities. - - Args: - temperature: An 0-D `Tensor`, representing the temperature - of a set of RelaxedOneHotCategorical distributions. The temperature - should be positive. - logits: An N-D `Tensor`, `N >= 1`, representing the log probabilities - of a set of RelaxedOneHotCategorical distributions. The first - `N - 1` dimensions index into a batch of independent distributions and - the last dimension represents a vector of logits for each class. Only - one of `logits` or `probs` should be passed in. - probs: An N-D `Tensor`, `N >= 1`, representing the probabilities - of a set of RelaxedOneHotCategorical distributions. The first `N - 1` - dimensions index into a batch of independent distributions and the last - dimension represents a vector of probabilities for each class. Only one - of `logits` or `probs` should be passed in. - dtype: The type of the event samples (default: inferred from - logits/probs). - validate_args: Unused in this distribution. - allow_nan_stats: Python `bool`, default `True`. If `False`, raise an - exception if a statistic (e.g. mean/mode/etc...) is undefined for any - batch member. If `True`, batch members with valid parameters leading to - undefined statistics will return NaN for this statistic. - name: A name for this distribution (optional). - """ - dist = ExpRelaxedOneHotCategorical(temperature, - logits=logits, - probs=probs, - dtype=dtype, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats) - super(RelaxedOneHotCategorical, self).__init__(dist, - bijectors.Exp(), - name=name) diff --git a/tensorflow/contrib/distributions/python/ops/sample_stats.py b/tensorflow/contrib/distributions/python/ops/sample_stats.py deleted file mode 100644 index ad0f2317c99..00000000000 --- a/tensorflow/contrib/distributions/python/ops/sample_stats.py +++ /dev/null @@ -1,532 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functions for computing statistics of samples.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops.distributions import util -from tensorflow.python.ops.signal import fft_ops - -__all__ = [ - "auto_correlation", - "percentile", -] - - -# TODO(langmore) Write separate versions of this for real/complex dtype, taking -# advantage of optimized real-fft ops. -def auto_correlation( - x, - axis=-1, - max_lags=None, - center=True, - normalize=True, - name="auto_correlation"): - """Auto correlation along one axis. - - Given a `1-D` wide sense stationary (WSS) sequence `X`, the auto correlation - `RXX` may be defined as (with `E` expectation and `Conj` complex conjugate) - - ``` - RXX[m] := E{ W[m] Conj(W[0]) } = E{ W[0] Conj(W[-m]) }, - W[n] := (X[n] - MU) / S, - MU := E{ X[0] }, - S**2 := E{ (X[0] - MU) Conj(X[0] - MU) }. - ``` - - This function takes the viewpoint that `x` is (along one axis) a finite - sub-sequence of a realization of (WSS) `X`, and then uses `x` to produce an - estimate of `RXX[m]` as follows: - - After extending `x` from length `L` to `inf` by zero padding, the auto - correlation estimate `rxx[m]` is computed for `m = 0, 1, ..., max_lags` as - - ``` - rxx[m] := (L - m)**-1 sum_n w[n + m] Conj(w[n]), - w[n] := (x[n] - mu) / s, - mu := L**-1 sum_n x[n], - s**2 := L**-1 sum_n (x[n] - mu) Conj(x[n] - mu) - ``` - - The error in this estimate is proportional to `1 / sqrt(len(x) - m)`, so users - often set `max_lags` small enough so that the entire output is meaningful. - - Note that since `mu` is an imperfect estimate of `E{ X[0] }`, and we divide by - `len(x) - m` rather than `len(x) - m - 1`, our estimate of auto correlation - contains a slight bias, which goes to zero as `len(x) - m --> infinity`. - - Args: - x: `float32` or `complex64` `Tensor`. - axis: Python `int`. The axis number along which to compute correlation. - Other dimensions index different batch members. - max_lags: Positive `int` tensor. The maximum value of `m` to consider - (in equation above). If `max_lags >= x.shape[axis]`, we effectively - re-set `max_lags` to `x.shape[axis] - 1`. - center: Python `bool`. If `False`, do not subtract the mean estimate `mu` - from `x[n]` when forming `w[n]`. - normalize: Python `bool`. If `False`, do not divide by the variance - estimate `s**2` when forming `w[n]`. - name: `String` name to prepend to created ops. - - Returns: - `rxx`: `Tensor` of same `dtype` as `x`. `rxx.shape[i] = x.shape[i]` for - `i != axis`, and `rxx.shape[axis] = max_lags + 1`. - - Raises: - TypeError: If `x` is not a supported type. - """ - # Implementation details: - # Extend length N / 2 1-D array x to length N by zero padding onto the end. - # Then, set - # F[x]_k := sum_n x_n exp{-i 2 pi k n / N }. - # It is not hard to see that - # F[x]_k Conj(F[x]_k) = F[R]_k, where - # R_m := sum_n x_n Conj(x_{(n - m) mod N}). - # One can also check that R_m / (N / 2 - m) is an unbiased estimate of RXX[m]. - - # Since F[x] is the DFT of x, this leads us to a zero-padding and FFT/IFFT - # based version of estimating RXX. - # Note that this is a special case of the Wiener-Khinchin Theorem. - with ops.name_scope(name, values=[x]): - x = ops.convert_to_tensor(x, name="x") - - # Rotate dimensions of x in order to put axis at the rightmost dim. - # FFT op requires this. - rank = util.prefer_static_rank(x) - if axis < 0: - axis = rank + axis - shift = rank - 1 - axis - # Suppose x.shape[axis] = T, so there are T "time" steps. - # ==> x_rotated.shape = B + [T], - # where B is x_rotated's batch shape. - x_rotated = util.rotate_transpose(x, shift) - - if center: - x_rotated -= math_ops.reduce_mean(x_rotated, axis=-1, keepdims=True) - - # x_len = N / 2 from above explanation. The length of x along axis. - # Get a value for x_len that works in all cases. - x_len = util.prefer_static_shape(x_rotated)[-1] - - # TODO(langmore) Investigate whether this zero padding helps or hurts. At - # the moment is necessary so that all FFT implementations work. - # Zero pad to the next power of 2 greater than 2 * x_len, which equals - # 2**(ceil(Log_2(2 * x_len))). Note: Log_2(X) = Log_e(X) / Log_e(2). - x_len_float64 = math_ops.cast(x_len, np.float64) - target_length = math_ops.pow( - np.float64(2.), - math_ops.ceil(math_ops.log(x_len_float64 * 2) / np.log(2.))) - pad_length = math_ops.cast(target_length - x_len_float64, np.int32) - - # We should have: - # x_rotated_pad.shape = x_rotated.shape[:-1] + [T + pad_length] - # = B + [T + pad_length] - x_rotated_pad = util.pad(x_rotated, axis=-1, back=True, count=pad_length) - - dtype = x.dtype - if not dtype.is_complex: - if not dtype.is_floating: - raise TypeError("Argument x must have either float or complex dtype" - " found: {}".format(dtype)) - x_rotated_pad = math_ops.complex(x_rotated_pad, - dtype.real_dtype.as_numpy_dtype(0.)) - - # Autocorrelation is IFFT of power-spectral density (up to some scaling). - fft_x_rotated_pad = fft_ops.fft(x_rotated_pad) - spectral_density = fft_x_rotated_pad * math_ops.conj(fft_x_rotated_pad) - # shifted_product is R[m] from above detailed explanation. - # It is the inner product sum_n X[n] * Conj(X[n - m]). - shifted_product = fft_ops.ifft(spectral_density) - - # Cast back to real-valued if x was real to begin with. - shifted_product = math_ops.cast(shifted_product, dtype) - - # Figure out if we can deduce the final static shape, and set max_lags. - # Use x_rotated as a reference, because it has the time dimension in the far - # right, and was created before we performed all sorts of crazy shape - # manipulations. - know_static_shape = True - if not x_rotated.shape.is_fully_defined(): - know_static_shape = False - if max_lags is None: - max_lags = x_len - 1 - else: - max_lags = ops.convert_to_tensor(max_lags, name="max_lags") - max_lags_ = tensor_util.constant_value(max_lags) - if max_lags_ is None or not know_static_shape: - know_static_shape = False - max_lags = math_ops.minimum(x_len - 1, max_lags) - else: - max_lags = min(x_len - 1, max_lags_) - - # Chop off the padding. - # We allow users to provide a huge max_lags, but cut it off here. - # shifted_product_chopped.shape = x_rotated.shape[:-1] + [max_lags] - shifted_product_chopped = shifted_product[..., :max_lags + 1] - - # If possible, set shape. - if know_static_shape: - chopped_shape = x_rotated.shape.as_list() - chopped_shape[-1] = min(x_len, max_lags + 1) - shifted_product_chopped.set_shape(chopped_shape) - - # Recall R[m] is a sum of N / 2 - m nonzero terms x[n] Conj(x[n - m]). The - # other terms were zeros arising only due to zero padding. - # `denominator = (N / 2 - m)` (defined below) is the proper term to - # divide by to make this an unbiased estimate of the expectation - # E[X[n] Conj(X[n - m])]. - x_len = math_ops.cast(x_len, dtype.real_dtype) - max_lags = math_ops.cast(max_lags, dtype.real_dtype) - denominator = x_len - math_ops.range(0., max_lags + 1.) - denominator = math_ops.cast(denominator, dtype) - shifted_product_rotated = shifted_product_chopped / denominator - - if normalize: - shifted_product_rotated /= shifted_product_rotated[..., :1] - - # Transpose dimensions back to those of x. - return util.rotate_transpose(shifted_product_rotated, -shift) - - -# TODO(langmore) To make equivalent to numpy.percentile: -# Make work with a sequence of floats or single float for 'q'. -# Make work with "linear", "midpoint" interpolation. (linear should be default) -def percentile(x, - q, - axis=None, - interpolation=None, - keep_dims=False, - validate_args=False, - name=None): - """Compute the `q`-th percentile of `x`. - - Given a vector `x`, the `q`-th percentile of `x` is the value `q / 100` of the - way from the minimum to the maximum in a sorted copy of `x`. - - The values and distances of the two nearest neighbors as well as the - `interpolation` parameter will determine the percentile if the normalized - ranking does not match the location of `q` exactly. - - This function is the same as the median if `q = 50`, the same as the minimum - if `q = 0` and the same as the maximum if `q = 100`. - - - ```python - # Get 30th percentile with default ('nearest') interpolation. - x = [1., 2., 3., 4.] - percentile(x, q=30.) - ==> 2.0 - - # Get 30th percentile with 'lower' interpolation - x = [1., 2., 3., 4.] - percentile(x, q=30., interpolation='lower') - ==> 1.0 - - # Get 100th percentile (maximum). By default, this is computed over every dim - x = [[1., 2.] - [3., 4.]] - percentile(x, q=100.) - ==> 4.0 - - # Treat the leading dim as indexing samples, and find the 100th quantile (max) - # over all such samples. - x = [[1., 2.] - [3., 4.]] - percentile(x, q=100., axis=[0]) - ==> [3., 4.] - ``` - - Compare to `numpy.percentile`. - - Args: - x: Floating point `N-D` `Tensor` with `N > 0`. If `axis` is not `None`, - `x` must have statically known number of dimensions. - q: Scalar `Tensor` in `[0, 100]`. The percentile. - axis: Optional `0-D` or `1-D` integer `Tensor` with constant values. - The axis that hold independent samples over which to return the desired - percentile. If `None` (the default), treat every dimension as a sample - dimension, returning a scalar. - interpolation : {"lower", "higher", "nearest"}. Default: "nearest" - This optional parameter specifies the interpolation method to - use when the desired quantile lies between two data points `i < j`: - * lower: `i`. - * higher: `j`. - * nearest: `i` or `j`, whichever is nearest. - keep_dims: Python `bool`. If `True`, the last dimension is kept with size 1 - If `False`, the last dimension is removed from the output shape. - validate_args: Whether to add runtime checks of argument validity. - If False, and arguments are incorrect, correct behavior is not guaranteed. - name: A Python string name to give this `Op`. Default is "percentile" - - Returns: - A `(N - len(axis))` dimensional `Tensor` of same dtype as `x`, or, if - `axis` is `None`, a scalar. - - Raises: - ValueError: If argument 'interpolation' is not an allowed type. - """ - name = name or "percentile" - allowed_interpolations = {"lower", "higher", "nearest"} - - if interpolation is None: - interpolation = "nearest" - else: - if interpolation not in allowed_interpolations: - raise ValueError("Argument 'interpolation' must be in %s. Found %s" % - (allowed_interpolations, interpolation)) - - with ops.name_scope(name, values=[x, q]): - x = ops.convert_to_tensor(x, name="x") - # Double is needed here and below, else we get the wrong index if the array - # is huge along axis. - q = math_ops.cast(q, dtypes.float64, name="q") - _get_static_ndims(q, expect_ndims=0) - - if validate_args: - q = control_flow_ops.with_dependencies([ - check_ops.assert_rank(q, 0), - check_ops.assert_greater_equal(q, math_ops.cast(0., dtypes.float64)), - check_ops.assert_less_equal(q, math_ops.cast(100., dtypes.float64)) - ], q) - - if axis is None: - y = array_ops.reshape(x, [-1]) - else: - axis = ops.convert_to_tensor(axis, name="axis") - check_ops.assert_integer(axis) - axis_ndims = _get_static_ndims( - axis, expect_static=True, expect_ndims_no_more_than=1) - axis_const = tensor_util.constant_value(axis) - if axis_const is None: - raise ValueError( - "Expected argument 'axis' to be statically available. Found: %s" % - axis) - axis = axis_const - if axis_ndims == 0: - axis = [axis] - axis = [int(a) for a in axis] - x_ndims = _get_static_ndims( - x, expect_static=True, expect_ndims_at_least=1) - axis = _make_static_axis_non_negative(axis, x_ndims) - y = _move_dims_to_flat_end(x, axis, x_ndims) - - frac_at_q_or_above = 1. - q / 100. - d = math_ops.cast(array_ops.shape(y)[-1], dtypes.float64) - - if interpolation == "lower": - index = math_ops.ceil((d - 1) * frac_at_q_or_above) - elif interpolation == "higher": - index = math_ops.floor((d - 1) * frac_at_q_or_above) - elif interpolation == "nearest": - index = math_ops.round((d - 1) * frac_at_q_or_above) - - # If d is gigantic, then we would have d == d - 1, even in double... So - # let's use max/min to avoid out of bounds errors. - d = array_ops.shape(y)[-1] - # d - 1 will be distinct from d in int32. - index = clip_ops.clip_by_value(math_ops.cast(index, dtypes.int32), 0, d - 1) - - # Sort everything, not just the top 'k' entries, which allows multiple calls - # to sort only once (under the hood) and use CSE. - sorted_y = _sort_tensor(y) - - # result.shape = B - result = sorted_y[..., index] - result.set_shape(y.get_shape()[:-1]) - - if keep_dims: - if axis is None: - # ones_vec = [1, 1,..., 1], total length = len(S) + len(B). - ones_vec = array_ops.ones( - shape=[_get_best_effort_ndims(x)], dtype=dtypes.int32) - result *= array_ops.ones(ones_vec, dtype=x.dtype) - else: - result = _insert_back_keep_dims(result, axis) - - return result - - -def _get_static_ndims(x, - expect_static=False, - expect_ndims=None, - expect_ndims_no_more_than=None, - expect_ndims_at_least=None): - """Get static number of dimensions and assert that some expectations are met. - - This function returns the number of dimensions "ndims" of x, as a Python int. - - The optional expect arguments are used to check the ndims of x, but this is - only done if the static ndims of x is not None. - - Args: - x: A Tensor. - expect_static: Expect `x` to have statically defined `ndims`. - expect_ndims: Optional Python integer. If provided, assert that x has - number of dimensions equal to this. - expect_ndims_no_more_than: Optional Python integer. If provided, assert - that x has no more than this many dimensions. - expect_ndims_at_least: Optional Python integer. If provided, assert that - x has at least this many dimensions. - - Returns: - ndims: A Python integer. - - Raises: - ValueError: If any of the expectations above are violated. - """ - ndims = x.get_shape().ndims - if ndims is None: - shape_const = tensor_util.constant_value(array_ops.shape(x)) - if shape_const is not None: - ndims = shape_const.ndim - - if ndims is None: - if expect_static: - raise ValueError( - "Expected argument 'x' to have statically defined 'ndims'. Found: " % - x) - return - - if expect_ndims is not None: - ndims_message = ("Expected argument 'x' to have ndims %s. Found tensor %s" - % (expect_ndims, x)) - if ndims != expect_ndims: - raise ValueError(ndims_message) - - if expect_ndims_at_least is not None: - ndims_at_least_message = ( - "Expected argument 'x' to have ndims >= %d. Found tensor %s" % ( - expect_ndims_at_least, x)) - if ndims < expect_ndims_at_least: - raise ValueError(ndims_at_least_message) - - if expect_ndims_no_more_than is not None: - ndims_no_more_than_message = ( - "Expected argument 'x' to have ndims <= %d. Found tensor %s" % ( - expect_ndims_no_more_than, x)) - if ndims > expect_ndims_no_more_than: - raise ValueError(ndims_no_more_than_message) - - return ndims - - -def _get_best_effort_ndims(x, - expect_ndims=None, - expect_ndims_at_least=None, - expect_ndims_no_more_than=None): - """Get static ndims if possible. Fallback on `tf.rank(x)`.""" - ndims_static = _get_static_ndims( - x, - expect_ndims=expect_ndims, - expect_ndims_at_least=expect_ndims_at_least, - expect_ndims_no_more_than=expect_ndims_no_more_than) - if ndims_static is not None: - return ndims_static - return array_ops.rank(x) - - -def _insert_back_keep_dims(x, axis): - """Insert the dims in `axis` back as singletons after being removed. - - Args: - x: `Tensor`. - axis: Python list of integers. - - Returns: - `Tensor` with same values as `x`, but additional singleton dimensions. - """ - for i in sorted(axis): - x = array_ops.expand_dims(x, axis=i) - return x - - -def _make_static_axis_non_negative(axis, ndims): - """Convert possibly negatively indexed axis to non-negative. - - Args: - axis: Iterable over Python integers. - ndims: Number of dimensions into which axis indexes. - - Returns: - A list of non-negative Python integers. - - Raises: - ValueError: If values in `axis` are too big/small to index into `ndims`. - """ - non_negative_axis = [] - for d in axis: - if d >= 0: - if d >= ndims: - raise ValueError("dim %d not in the interval [0, %d]." % (d, ndims - 1)) - non_negative_axis.append(d) - else: - if d < -1 * ndims: - raise ValueError( - "Negatively indexed dim %d not in the interval [-%d, -1]" % (d, - ndims)) - non_negative_axis.append(ndims + d) - return non_negative_axis - - -def _move_dims_to_flat_end(x, axis, x_ndims): - """Move dims corresponding to `axis` in `x` to the end, then flatten. - - Args: - x: `Tensor` with shape `[B0,B1,...,Bb]`. - axis: Python list of indices into dimensions of `x`. - x_ndims: Python integer holding number of dimensions in `x`. - - Returns: - `Tensor` with value from `x` and dims in `axis` moved to end into one single - dimension. - """ - # Suppose x.shape = [a, b, c, d] - # Suppose axis = [1, 3] - - # front_dims = [0, 2] in example above. - front_dims = sorted(set(range(x_ndims)).difference(axis)) - # x_permed.shape = [a, c, b, d] - x_permed = array_ops.transpose(x, perm=front_dims + list(axis)) - - if x.get_shape().is_fully_defined(): - x_shape = x.get_shape().as_list() - # front_shape = [a, c], end_shape = [b * d] - front_shape = [x_shape[i] for i in front_dims] - end_shape = [np.prod([x_shape[i] for i in axis])] - full_shape = front_shape + end_shape - else: - front_shape = array_ops.shape(x_permed)[:x_ndims - len(axis)] - end_shape = [-1] - full_shape = array_ops.concat([front_shape, end_shape], axis=0) - return array_ops.reshape(x_permed, shape=full_shape) - - -def _sort_tensor(tensor): - """Use `top_k` to sort a `Tensor` along the last dimension.""" - sorted_, _ = nn_ops.top_k(tensor, k=array_ops.shape(tensor)[-1]) - return sorted_ diff --git a/tensorflow/contrib/distributions/python/ops/seed_stream.py b/tensorflow/contrib/distributions/python/ops/seed_stream.py deleted file mode 100644 index 3d39e9ce507..00000000000 --- a/tensorflow/contrib/distributions/python/ops/seed_stream.py +++ /dev/null @@ -1,228 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Local PRNG for amplifying seed entropy into seeds for base operations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import hashlib - - -class SeedStream(object): - """Local PRNG for amplifying seed entropy into seeds for base operations. - - Writing sampling code which correctly sets the pseudo-random number - generator (PRNG) seed is surprisingly difficult. This class serves as - a helper for the TensorFlow Probability coding pattern designed to - avoid common mistakes. - - # Motivating Example - - A common first-cut implementation of a sampler for the beta - distribution is to compute the ratio of a gamma with itself plus - another gamma. This code snippet tries to do that, but contains a - surprisingly common error: - - ```python - def broken_beta(shape, alpha, beta, seed): - x = tf.random.gamma(shape, alpha, seed=seed) - y = tf.random.gamma(shape, beta, seed=seed) - return x / (x + y) - ``` - - The mistake is that the two gamma draws are seeded with the same - seed. This causes them to always produce the same results, which, - in turn, leads this code snippet to always return `0.5`. Because it - can happen across abstraction boundaries, this kind of error is - surprisingly easy to make when handling immutable seeds. - - # Goals - - TensorFlow Probability adopts a code style designed to eliminate the - above class of error, without exacerbating others. The goals of - this code style are: - - - Support reproducibility of results (by encouraging seeding of all - pseudo-random operations). - - - Avoid shared-write global state (by not relying on a global PRNG). - - - Prevent accidental seed reuse by TF Probability implementers. This - goal is served with the local pseudo-random seed generator provided - in this module. - - - Mitigate potential accidental seed reuse by TF Probability clients - (with a salting scheme). - - - Prevent accidental resonances with downstream PRNGs (by hashing the - output). - - ## Non-goals - - - Implementing a high-performance PRNG for generating large amounts of - entropy. That's the job of the underlying TensorFlow PRNG we are - seeding. - - - Avoiding random seed collisions, aka "birthday attacks". - - # Code pattern - - ```python - def random_beta(shape, alpha, beta, seed): # (a) - seed = SeedStream(seed, salt="random_beta") # (b) - x = tf.random.gamma(shape, alpha, seed=seed()) # (c) - y = tf.random.gamma(shape, beta, seed=seed()) # (c) - return x / (x + y) - ``` - - The elements of this pattern are: - - - Accept an explicit seed (line a) as an argument in all public - functions, and write the function to be deterministic (up to any - numerical issues) for fixed seed. - - - Rationale: This provides the client with the ability to reproduce - results. Accepting an immutable seed rather than a mutable PRNG - object reduces code coupling, permitting different sections to be - reproducible independently. - - - Use that seed only to initialize a local `SeedStream` instance (line b). - - - Rationale: Avoids accidental seed reuse. - - - Supply the name of the function being implemented as a salt to the - `SeedStream` instance (line b). This serves to keep the salts - unique; unique salts ensure that clients of TF Probability will see - different functions always produce independent results even if - called with the same seeds. - - - Seed each callee operation with the output of a unique call to the - `SeedStream` instance (lines c). This ensures reproducibility of - results while preventing seed reuse across callee invocations. - - # Why salt? - - Salting the `SeedStream` instances (with unique salts) is defensive - programming against a client accidentally committing a mistake - similar to our motivating example. Consider the following situation - that might arise without salting: - - ```python - def tfp_foo(seed): - seed = SeedStream(seed, salt="") - foo_stuff = tf.random.normal(seed=seed()) - ... - - def tfp_bar(seed): - seed = SeedStream(seed, salt="") - bar_stuff = tf.random.normal(seed=seed()) - ... - - def client_baz(seed): - foo = tfp_foo(seed=seed) - bar = tfp_bar(seed=seed) - ... - ``` - - The client should have used different seeds as inputs to `foo` and - `bar`. However, because they didn't, *and because `foo` and `bar` - both sample a Gaussian internally as their first action*, the - internal `foo_stuff` and `bar_stuff` will be the same, and the - returned `foo` and `bar` will not be independent, leading to subtly - incorrect answers from the client's simulation. This kind of bug is - particularly insidious for the client, because it depends on a - Distributions implementation detail, namely the order in which `foo` - and `bar` invoke the samplers they depend on. In particular, a - Bayesflow team member can introduce such a bug in previously - (accidentally) correct client code by performing an internal - refactoring that causes this operation order alignment. - - A salting discipline eliminates this problem by making sure that the - seeds seen by `foo`'s callees will differ from those seen by `bar`'s - callees, even if `foo` and `bar` are invoked with the same input - seed. - """ - - def __init__(self, seed, salt): - """Initializes a `SeedStream`. - - Args: - seed: Any Python object convertible to string, supplying the - initial entropy. If `None`, operations seeded with seeds - drawn from this `SeedStream` will follow TensorFlow semantics - for not being seeded. - salt: Any Python object convertible to string, supplying - auxiliary entropy. Must be unique across the Distributions - and TensorFlow Probability code base. See class docstring for - rationale. - """ - self._seed = seed.original_seed if isinstance(seed, SeedStream) else seed - self._salt = salt - self._counter = 0 - - def __call__(self): - """Returns a fresh integer usable as a seed in downstream operations. - - If this `SeedStream` was initialized with `seed=None`, returns - `None`. This has the effect that downstream operations (both - `SeedStream`s and primitive TensorFlow ops) will behave as though - they were unseeded. - - The returned integer is non-negative, and uniformly distributed in - the half-open interval `[0, 2**512)`. This is consistent with - TensorFlow, as TensorFlow operations internally use the residue of - the given seed modulo `2**31 - 1` (see - `tensorflow/python/framework/random_seed.py`). - - Returns: - seed: A fresh integer usable as a seed in downstream operations, - or `None`. - """ - self._counter += 1 - if self._seed is None: - return None - composite = str((self._seed, self._counter, self._salt)).encode("utf-8") - return int(hashlib.sha512(composite).hexdigest(), 16) - - @property - def original_seed(self): - return self._seed - - @property - def salt(self): - return self._salt - -# Design rationales for the SeedStream class -# -# - Salts are accepted for the reason given above to supply them. -# -# - A `None` seed propagates to downstream seeds, so they exhibit -# their "unseeded" behavior. -# -# - The return value is a Python int so it can be passed directly to -# TensorFlow operations as a seed. It is large to avoid losing seed -# space needlessly (TF will internally read only the last 31 bits). -# -# - The output is hashed with a crypto-grade hash function as a form -# of defensive programming: this reliably prevents all possible -# accidental resonances with all possible downstream PRNGs. The -# specific function used is not important; SHA512 was ready to hand. -# -# - The internal state update is a simple counter because (a) given -# that the output is hashed anyway, this is enough, and (b) letting -# it be this predictable permits a future "generate many seeds in -# parallel" operation whose results would agree with running -# sequentially. diff --git a/tensorflow/contrib/distributions/python/ops/shape.py b/tensorflow/contrib/distributions/python/ops/shape.py deleted file mode 100644 index 1be2dd1c719..00000000000 --- a/tensorflow/contrib/distributions/python/ops/shape.py +++ /dev/null @@ -1,506 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A helper class for inferring Distribution shape.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import contextlib - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import util as distribution_util -from tensorflow.python.util import deprecation - - -class _DistributionShape(object): - """Manage and manipulate `Distribution` shape. - - #### Terminology - - Recall that a `Tensor` has: - - `shape`: size of `Tensor` dimensions, - - `ndims`: size of `shape`; number of `Tensor` dimensions, - - `dims`: indexes into `shape`; useful for transpose, reduce. - - `Tensor`s sampled from a `Distribution` can be partitioned by `sample_dims`, - `batch_dims`, and `event_dims`. To understand the semantics of these - dimensions, consider when two of the three are fixed and the remaining - is varied: - - `sample_dims`: indexes independent draws from identical - parameterizations of the `Distribution`. - - `batch_dims`: indexes independent draws from non-identical - parameterizations of the `Distribution`. - - `event_dims`: indexes event coordinates from one sample. - - The `sample`, `batch`, and `event` dimensions constitute the entirety of a - `Distribution` `Tensor`'s shape. - - The dimensions are always in `sample`, `batch`, `event` order. - - #### Purpose - - This class partitions `Tensor` notions of `shape`, `ndims`, and `dims` into - `Distribution` notions of `sample,` `batch,` and `event` dimensions. That - is, it computes any of: - - ``` - sample_shape batch_shape event_shape - sample_dims batch_dims event_dims - sample_ndims batch_ndims event_ndims - ``` - - for a given `Tensor`, e.g., the result of - `Distribution.sample(sample_shape=...)`. - - For a given `Tensor`, this class computes the above table using minimal - information: `batch_ndims` and `event_ndims`. - - #### Examples - - We show examples of distribution shape semantics. - - - Sample dimensions: - Computing summary statistics, i.e., the average is a reduction over sample - dimensions. - - ```python - sample_dims = [0] - tf.reduce_mean(Normal(loc=1.3, scale=1.).sample_n(1000), - axis=sample_dims) # ~= 1.3 - ``` - - - Batch dimensions: - Monte Carlo estimation of a marginal probability: - Average over batch dimensions where batch dimensions are associated with - random draws from a prior. - E.g., suppose we want to find the Monte Carlo estimate of the marginal - distribution of a `Normal` with a random `Laplace` location: - - ``` - P(X=x) = integral P(X=x|y) P(Y=y) dy - ~= 1/n sum_{i=1}^n P(X=x|y_i), y_i ~iid Laplace(0,1) - = tf.reduce_mean(Normal(loc=Laplace(0., 1.).sample_n(n=1000), - scale=tf.ones(1000)).prob(x), - axis=batch_dims) - ``` - - The `Laplace` distribution generates a `Tensor` of shape `[1000]`. When - fed to a `Normal`, this is interpreted as 1000 different locations, i.e., - 1000 non-identical Normals. Therefore a single call to `prob(x)` yields - 1000 probabilities, one for every location. The average over this batch - yields the marginal. - - - Event dimensions: - Computing the determinant of the Jacobian of a function of a random - variable involves a reduction over event dimensions. - E.g., Jacobian of the transform `Y = g(X) = exp(X)`: - - ```python - tf.compat.v1.div(1., tf.reduce_prod(x, event_dims)) - ``` - - We show examples using this class. - - Write `S, B, E` for `sample_shape`, `batch_shape`, and `event_shape`. - - ```python - # 150 iid samples from one multivariate Normal with two degrees of freedom. - mu = [0., 0] - sigma = [[1., 0], - [0, 1]] - mvn = MultivariateNormal(mu, sigma) - rand_mvn = mvn.sample(sample_shape=[3, 50]) - shaper = DistributionShape(batch_ndims=0, event_ndims=1) - S, B, E = shaper.get_shape(rand_mvn) - # S = [3, 50] - # B = [] - # E = [2] - - # 12 iid samples from one Wishart with 2x2 events. - sigma = [[1., 0], - [2, 1]] - wishart = Wishart(df=5, scale=sigma) - rand_wishart = wishart.sample(sample_shape=[3, 4]) - shaper = DistributionShape(batch_ndims=0, event_ndims=2) - S, B, E = shaper.get_shape(rand_wishart) - # S = [3, 4] - # B = [] - # E = [2, 2] - - # 100 iid samples from two, non-identical trivariate Normal distributions. - mu = ... # shape(2, 3) - sigma = ... # shape(2, 3, 3) - X = MultivariateNormal(mu, sigma).sample(shape=[4, 25]) - # S = [4, 25] - # B = [2] - # E = [3] - ``` - - #### Argument Validation - - When `validate_args=False`, checks that cannot be done during - graph construction are performed at graph execution. This may result in a - performance degradation because data must be switched from GPU to CPU. - - For example, when `validate_args=False` and `event_ndims` is a - non-constant `Tensor`, it is checked to be a non-negative integer at graph - execution. (Same for `batch_ndims`). Constant `Tensor`s and non-`Tensor` - arguments are always checked for correctness since this can be done for - "free," i.e., during graph construction. - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - batch_ndims=None, - event_ndims=None, - validate_args=False, - name="DistributionShape"): - """Construct `DistributionShape` with fixed `batch_ndims`, `event_ndims`. - - `batch_ndims` and `event_ndims` are fixed throughout the lifetime of a - `Distribution`. They may only be known at graph execution. - - If both `batch_ndims` and `event_ndims` are python scalars (rather than - either being a `Tensor`), functions in this class automatically perform - sanity checks during graph construction. - - Args: - batch_ndims: `Tensor`. Number of `dims` (`rank`) of the batch portion of - indexes of a `Tensor`. A "batch" is a non-identical distribution, i.e, - Normal with different parameters. - event_ndims: `Tensor`. Number of `dims` (`rank`) of the event portion of - indexes of a `Tensor`. An "event" is what is sampled from a - distribution, i.e., a trivariate Normal has an event shape of [3] and a - 4 dimensional Wishart has an event shape of [4, 4]. - validate_args: Python `bool`, default `False`. When `True`, - non-`tf.constant` `Tensor` arguments are checked for correctness. - (`tf.constant` arguments are always checked.) - name: Python `str`. The name prepended to Ops created by this class. - - Raises: - ValueError: if either `batch_ndims` or `event_ndims` are: `None`, - negative, not `int32`. - """ - if batch_ndims is None: raise ValueError("batch_ndims cannot be None") - if event_ndims is None: raise ValueError("event_ndims cannot be None") - self._batch_ndims = batch_ndims - self._event_ndims = event_ndims - self._validate_args = validate_args - with ops.name_scope(name): - self._name = name - with ops.name_scope("init"): - self._batch_ndims = self._assert_non_negative_int32_scalar( - ops.convert_to_tensor( - batch_ndims, name="batch_ndims")) - self._batch_ndims_static, self._batch_ndims_is_0 = ( - self._introspect_ndims(self._batch_ndims)) - self._event_ndims = self._assert_non_negative_int32_scalar( - ops.convert_to_tensor( - event_ndims, name="event_ndims")) - self._event_ndims_static, self._event_ndims_is_0 = ( - self._introspect_ndims(self._event_ndims)) - - @property - def name(self): - """Name given to ops created by this class.""" - return self._name - - @property - def batch_ndims(self): - """Returns number of dimensions corresponding to non-identical draws.""" - return self._batch_ndims - - @property - def event_ndims(self): - """Returns number of dimensions needed to index a sample's coordinates.""" - return self._event_ndims - - @property - def validate_args(self): - """Returns True if graph-runtime `Tensor` checks are enabled.""" - return self._validate_args - - def get_ndims(self, x, name="get_ndims"): - """Get `Tensor` number of dimensions (rank). - - Args: - x: `Tensor`. - name: Python `str`. The name to give this op. - - Returns: - ndims: Scalar number of dimensions associated with a `Tensor`. - """ - with self._name_scope(name, values=[x]): - x = ops.convert_to_tensor(x, name="x") - ndims = x.get_shape().ndims - if ndims is None: - return array_ops.rank(x, name="ndims") - return ops.convert_to_tensor(ndims, dtype=dtypes.int32, name="ndims") - - def get_sample_ndims(self, x, name="get_sample_ndims"): - """Returns number of dimensions corresponding to iid draws ("sample"). - - Args: - x: `Tensor`. - name: Python `str`. The name to give this op. - - Returns: - sample_ndims: `Tensor` (0D, `int32`). - - Raises: - ValueError: if `sample_ndims` is calculated to be negative. - """ - with self._name_scope(name, values=[x]): - ndims = self.get_ndims(x, name=name) - if self._is_all_constant_helper(ndims, self.batch_ndims, - self.event_ndims): - ndims = tensor_util.constant_value(ndims) - sample_ndims = (ndims - self._batch_ndims_static - - self._event_ndims_static) - if sample_ndims < 0: - raise ValueError( - "expected batch_ndims(%d) + event_ndims(%d) <= ndims(%d)" % - (self._batch_ndims_static, self._event_ndims_static, ndims)) - return ops.convert_to_tensor(sample_ndims, name="sample_ndims") - else: - with ops.name_scope(name="sample_ndims"): - sample_ndims = ndims - self.batch_ndims - self.event_ndims - if self.validate_args: - sample_ndims = control_flow_ops.with_dependencies( - [check_ops.assert_non_negative(sample_ndims)], sample_ndims) - return sample_ndims - - def get_dims(self, x, name="get_dims"): - """Returns dimensions indexing `sample_shape`, `batch_shape`, `event_shape`. - - Example: - - ```python - x = ... # Tensor with shape [4, 3, 2, 1] - sample_dims, batch_dims, event_dims = _DistributionShape( - batch_ndims=2, event_ndims=1).get_dims(x) - # sample_dims == [0] - # batch_dims == [1, 2] - # event_dims == [3] - # Note that these are not the shape parts, but rather indexes into shape. - ``` - - Args: - x: `Tensor`. - name: Python `str`. The name to give this op. - - Returns: - sample_dims: `Tensor` (1D, `int32`). - batch_dims: `Tensor` (1D, `int32`). - event_dims: `Tensor` (1D, `int32`). - """ - with self._name_scope(name, values=[x]): - def make_dims(start_sum, size, name): - """Closure to make dims range.""" - start_sum = start_sum if start_sum else [ - array_ops.zeros([], dtype=dtypes.int32, name="zero")] - if self._is_all_constant_helper(size, *start_sum): - start = sum(tensor_util.constant_value(s) for s in start_sum) - stop = start + tensor_util.constant_value(size) - return ops.convert_to_tensor( - list(range(start, stop)), dtype=dtypes.int32, name=name) - else: - start = sum(start_sum) - return math_ops.range(start, start + size) - sample_ndims = self.get_sample_ndims(x, name=name) - return (make_dims([], sample_ndims, name="sample_dims"), - make_dims([sample_ndims], self.batch_ndims, name="batch_dims"), - make_dims([sample_ndims, self.batch_ndims], - self.event_ndims, name="event_dims")) - - def get_shape(self, x, name="get_shape"): - """Returns `Tensor`'s shape partitioned into `sample`, `batch`, `event`. - - Args: - x: `Tensor`. - name: Python `str`. The name to give this op. - - Returns: - sample_shape: `Tensor` (1D, `int32`). - batch_shape: `Tensor` (1D, `int32`). - event_shape: `Tensor` (1D, `int32`). - """ - with self._name_scope(name, values=[x]): - x = ops.convert_to_tensor(x, name="x") - def slice_shape(start_sum, size, name): - """Closure to slice out shape.""" - start_sum = start_sum if start_sum else [ - array_ops.zeros([], dtype=dtypes.int32, name="zero")] - if (x.get_shape().ndims is not None and - self._is_all_constant_helper(size, *start_sum)): - start = sum(tensor_util.constant_value(s) for s in start_sum) - stop = start + tensor_util.constant_value(size) - slice_ = x.get_shape()[start:stop].as_list() - if all(s is not None for s in slice_): - return ops.convert_to_tensor(slice_, dtype=dtypes.int32, name=name) - return array_ops.slice(array_ops.shape(x), [sum(start_sum)], [size]) - sample_ndims = self.get_sample_ndims(x, name=name) - return (slice_shape([], sample_ndims, - name="sample_shape"), - slice_shape([sample_ndims], self.batch_ndims, - name="batch_shape"), - slice_shape([sample_ndims, self.batch_ndims], self.event_ndims, - name="event_shape")) - - # TODO(jvdillon): Make remove expand_batch_dim and make expand_batch_dim=False - # the default behavior. - def make_batch_of_event_sample_matrices( - self, x, expand_batch_dim=True, - name="make_batch_of_event_sample_matrices"): - """Reshapes/transposes `Distribution` `Tensor` from S+B+E to B_+E_+S_. - - Where: - - `B_ = B if B or not expand_batch_dim else [1]`, - - `E_ = E if E else [1]`, - - `S_ = [tf.reduce_prod(S)]`. - - Args: - x: `Tensor`. - expand_batch_dim: Python `bool`. If `True` the batch dims will be expanded - such that `batch_ndims >= 1`. - name: Python `str`. The name to give this op. - - Returns: - x: `Tensor`. Input transposed/reshaped to `B_+E_+S_`. - sample_shape: `Tensor` (1D, `int32`). - """ - with self._name_scope(name, values=[x]): - x = ops.convert_to_tensor(x, name="x") - # x.shape: S+B+E - sample_shape, batch_shape, event_shape = self.get_shape(x) - event_shape = distribution_util.pick_vector( - self._event_ndims_is_0, [1], event_shape) - if expand_batch_dim: - batch_shape = distribution_util.pick_vector( - self._batch_ndims_is_0, [1], batch_shape) - new_shape = array_ops.concat([[-1], batch_shape, event_shape], 0) - x = array_ops.reshape(x, shape=new_shape) - # x.shape: [prod(S)]+B_+E_ - x = distribution_util.rotate_transpose(x, shift=-1) - # x.shape: B_+E_+[prod(S)] - return x, sample_shape - - # TODO(jvdillon): Make remove expand_batch_dim and make expand_batch_dim=False - # the default behavior. - def undo_make_batch_of_event_sample_matrices( - self, x, sample_shape, expand_batch_dim=True, - name="undo_make_batch_of_event_sample_matrices"): - """Reshapes/transposes `Distribution` `Tensor` from B_+E_+S_ to S+B+E. - - Where: - - `B_ = B if B or not expand_batch_dim else [1]`, - - `E_ = E if E else [1]`, - - `S_ = [tf.reduce_prod(S)]`. - - This function "reverses" `make_batch_of_event_sample_matrices`. - - Args: - x: `Tensor` of shape `B_+E_+S_`. - sample_shape: `Tensor` (1D, `int32`). - expand_batch_dim: Python `bool`. If `True` the batch dims will be expanded - such that `batch_ndims>=1`. - name: Python `str`. The name to give this op. - - Returns: - x: `Tensor`. Input transposed/reshaped to `S+B+E`. - """ - with self._name_scope(name, values=[x, sample_shape]): - x = ops.convert_to_tensor(x, name="x") - # x.shape: _B+_E+[prod(S)] - sample_shape = ops.convert_to_tensor(sample_shape, name="sample_shape") - x = distribution_util.rotate_transpose(x, shift=1) - # x.shape: [prod(S)]+_B+_E - if self._is_all_constant_helper(self.batch_ndims, self.event_ndims): - if self._batch_ndims_is_0 or self._event_ndims_is_0: - squeeze_dims = [] - if self._event_ndims_is_0: - squeeze_dims += [-1] - if self._batch_ndims_is_0 and expand_batch_dim: - squeeze_dims += [1] - if squeeze_dims: - x = array_ops.squeeze(x, axis=squeeze_dims) - # x.shape: [prod(S)]+B+E - _, batch_shape, event_shape = self.get_shape(x) - else: - s = (x.get_shape().as_list() if x.get_shape().is_fully_defined() - else array_ops.shape(x)) - batch_shape = s[1:1+self.batch_ndims] - # Since sample_dims=1 and is left-most, we add 1 to the number of - # batch_ndims to get the event start dim. - event_start = array_ops.where_v2( - math_ops.logical_and(expand_batch_dim, self._batch_ndims_is_0), 2, - 1 + self.batch_ndims) - event_shape = s[event_start:event_start+self.event_ndims] - new_shape = array_ops.concat([sample_shape, batch_shape, event_shape], 0) - x = array_ops.reshape(x, shape=new_shape) - # x.shape: S+B+E - return x - - @contextlib.contextmanager - def _name_scope(self, name=None, values=None): - """Helper function to standardize op scope.""" - with ops.name_scope(self.name): - with ops.name_scope(name, values=( - (values or []) + [self.batch_ndims, self.event_ndims])) as scope: - yield scope - - def _is_all_constant_helper(self, *args): - """Helper which returns True if all inputs are constant_value.""" - return all(tensor_util.constant_value(x) is not None for x in args) - - def _assert_non_negative_int32_scalar(self, x): - """Helper which ensures that input is a non-negative, int32, scalar.""" - x = ops.convert_to_tensor(x, name="x") - if x.dtype.base_dtype != dtypes.int32.base_dtype: - raise TypeError("%s.dtype=%s is not %s" % (x.name, x.dtype, dtypes.int32)) - x_value_static = tensor_util.constant_value(x) - if x.get_shape().ndims is not None and x_value_static is not None: - if x.get_shape().ndims != 0: - raise ValueError("%s.ndims=%d is not 0 (scalar)" % - (x.name, x.get_shape().ndims)) - if x_value_static < 0: - raise ValueError("%s.value=%d cannot be negative" % - (x.name, x_value_static)) - return x - if self.validate_args: - x = control_flow_ops.with_dependencies([ - check_ops.assert_rank(x, 0), - check_ops.assert_non_negative(x)], x) - return x - - def _introspect_ndims(self, ndims): - """Helper to establish some properties of input ndims args.""" - if self._is_all_constant_helper(ndims): - return (tensor_util.constant_value(ndims), - tensor_util.constant_value(ndims) == 0) - return None, math_ops.equal(ndims, 0) diff --git a/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py b/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py deleted file mode 100644 index 4b520b912e7..00000000000 --- a/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""SinhArcsinh transformation of a distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops.distributions import normal -from tensorflow.python.ops.distributions import transformed_distribution -from tensorflow.python.util import deprecation - -__all__ = [ - "SinhArcsinh", -] - - -class SinhArcsinh(transformed_distribution.TransformedDistribution): - """The SinhArcsinh transformation of a distribution on `(-inf, inf)`. - - This distribution models a random variable, making use of - a `SinhArcsinh` transformation (which has adjustable tailweight and skew), - a rescaling, and a shift. - - The `SinhArcsinh` transformation of the Normal is described in great depth in - [Sinh-arcsinh distributions](https://www.jstor.org/stable/27798865). - Here we use a slightly different parameterization, in terms of `tailweight` - and `skewness`. Additionally we allow for distributions other than Normal, - and control over `scale` as well as a "shift" parameter `loc`. - - #### Mathematical Details - - Given random variable `Z`, we define the SinhArcsinh - transformation of `Z`, `Y`, parameterized by - `(loc, scale, skewness, tailweight)`, via the relation: - - ``` - Y := loc + scale * F(Z) * (2 / F_0(2)) - F(Z) := Sinh( (Arcsinh(Z) + skewness) * tailweight ) - F_0(Z) := Sinh( Arcsinh(Z) * tailweight ) - ``` - - This distribution is similar to the location-scale transformation - `L(Z) := loc + scale * Z` in the following ways: - - * If `skewness = 0` and `tailweight = 1` (the defaults), `F(Z) = Z`, and then - `Y = L(Z)` exactly. - * `loc` is used in both to shift the result by a constant factor. - * The multiplication of `scale` by `2 / F_0(2)` ensures that if `skewness = 0` - `P[Y - loc <= 2 * scale] = P[L(Z) - loc <= 2 * scale]`. - Thus it can be said that the weights in the tails of `Y` and `L(Z)` beyond - `loc + 2 * scale` are the same. - - This distribution is different than `loc + scale * Z` due to the - reshaping done by `F`: - - * Positive (negative) `skewness` leads to positive (negative) skew. - * positive skew means, the mode of `F(Z)` is "tilted" to the right. - * positive skew means positive values of `F(Z)` become more likely, and - negative values become less likely. - * Larger (smaller) `tailweight` leads to fatter (thinner) tails. - * Fatter tails mean larger values of `|F(Z)|` become more likely. - * `tailweight < 1` leads to a distribution that is "flat" around `Y = loc`, - and a very steep drop-off in the tails. - * `tailweight > 1` leads to a distribution more peaked at the mode with - heavier tails. - - To see the argument about the tails, note that for `|Z| >> 1` and - `|Z| >> (|skewness| * tailweight)**tailweight`, we have - `Y approx 0.5 Z**tailweight e**(sign(Z) skewness * tailweight)`. - - To see the argument regarding multiplying `scale` by `2 / F_0(2)`, - - ``` - P[(Y - loc) / scale <= 2] = P[F(Z) * (2 / F_0(2)) <= 2] - = P[F(Z) <= F_0(2)] - = P[Z <= 2] (if F = F_0). - ``` - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc, - scale, - skewness=None, - tailweight=None, - distribution=None, - validate_args=False, - allow_nan_stats=True, - name="SinhArcsinh"): - """Construct SinhArcsinh distribution on `(-inf, inf)`. - - Arguments `(loc, scale, skewness, tailweight)` must have broadcastable shape - (indexing batch dimensions). They must all have the same `dtype`. - - Args: - loc: Floating-point `Tensor`. - scale: `Tensor` of same `dtype` as `loc`. - skewness: Skewness parameter. Default is `0.0` (no skew). - tailweight: Tailweight parameter. Default is `1.0` (unchanged tailweight) - distribution: `tf.Distribution`-like instance. Distribution that is - transformed to produce this distribution. - Default is `tfp.distributions.Normal(0., 1.)`. - Must be a scalar-batch, scalar-event distribution. Typically - `distribution.reparameterization_type = FULLY_REPARAMETERIZED` or it is - a function of non-trainable parameters. WARNING: If you backprop through - a `SinhArcsinh` sample and `distribution` is not - `FULLY_REPARAMETERIZED` yet is a function of trainable variables, then - the gradient will be incorrect! - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, - statistics (e.g., mean, mode, variance) use the value "`NaN`" to - indicate the result is undefined. When `False`, an exception is raised - if one or more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - parameters = dict(locals()) - - with ops.name_scope(name, - values=[loc, scale, skewness, tailweight]) as name: - loc = ops.convert_to_tensor(loc, name="loc") - dtype = loc.dtype - scale = ops.convert_to_tensor(scale, name="scale", dtype=dtype) - tailweight = 1. if tailweight is None else tailweight - has_default_skewness = skewness is None - skewness = 0. if skewness is None else skewness - tailweight = ops.convert_to_tensor( - tailweight, name="tailweight", dtype=dtype) - skewness = ops.convert_to_tensor(skewness, name="skewness", dtype=dtype) - - batch_shape = distribution_util.get_broadcast_shape( - loc, scale, tailweight, skewness) - - # Recall, with Z a random variable, - # Y := loc + C * F(Z), - # F(Z) := Sinh( (Arcsinh(Z) + skewness) * tailweight ) - # F_0(Z) := Sinh( Arcsinh(Z) * tailweight ) - # C := 2 * scale / F_0(2) - if distribution is None: - distribution = normal.Normal( - loc=array_ops.zeros([], dtype=dtype), - scale=array_ops.ones([], dtype=dtype), - allow_nan_stats=allow_nan_stats) - else: - asserts = distribution_util.maybe_check_scalar_distribution( - distribution, dtype, validate_args) - if asserts: - loc = control_flow_ops.with_dependencies(asserts, loc) - - # Make the SAS bijector, 'F'. - f = bijectors.SinhArcsinh( - skewness=skewness, tailweight=tailweight) - if has_default_skewness: - f_noskew = f - else: - f_noskew = bijectors.SinhArcsinh( - skewness=skewness.dtype.as_numpy_dtype(0.), - tailweight=tailweight) - - # Make the AffineScalar bijector, Z --> loc + scale * Z (2 / F_0(2)) - c = 2 * scale / f_noskew.forward(ops.convert_to_tensor(2, dtype=dtype)) - affine = bijectors.AffineScalar( - shift=loc, - scale=c, - validate_args=validate_args) - - bijector = bijectors.Chain([affine, f]) - - super(SinhArcsinh, self).__init__( - distribution=distribution, - bijector=bijector, - batch_shape=batch_shape, - validate_args=validate_args, - name=name) - self._parameters = parameters - self._loc = loc - self._scale = scale - self._tailweight = tailweight - self._skewness = skewness - - @property - def loc(self): - """The `loc` in `Y := loc + scale @ F(Z) * (2 / F(2)).""" - return self._loc - - @property - def scale(self): - """The `LinearOperator` `scale` in `Y := loc + scale @ F(Z) * (2 / F(2)).""" - return self._scale - - @property - def tailweight(self): - """Controls the tail decay. `tailweight > 1` means faster than Normal.""" - return self._tailweight - - @property - def skewness(self): - """Controls the skewness. `Skewness > 0` means right skew.""" - return self._skewness diff --git a/tensorflow/contrib/distributions/python/ops/statistical_testing.py b/tensorflow/contrib/distributions/python/ops/statistical_testing.py deleted file mode 100644 index ed64c71a218..00000000000 --- a/tensorflow/contrib/distributions/python/ops/statistical_testing.py +++ /dev/null @@ -1,910 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Statistical test assertions calibrated for their error rates. - -Statistical tests have an inescapable probability of error: a correct -sampler can still fail a test by chance, and an incorrect sampler can -still pass a test by chance. This library is about bounding both of -those error rates. This requires admitting a task-specific notion of -"discrepancy": Correct code will fail rarely, code that misbehaves by -more than the discrepancy will pass rarely, and nothing reliable can -be said about code that misbehaves, but misbehaves by less than the -discrepancy. - -# Example - -Consider testing that the mean of a scalar probability distribution P -is some expected constant. Suppose the support of P is the interval -`[0, 1]`. Then you might do this: - -```python - from tensorflow_probability.python.distributions.internal import statistical_testing - - expected_mean = ... - num_samples = 5000 - samples = ... draw 5000 samples from P - - # Check that the mean looks right - check1 = statistical_testing.assert_true_mean_equal_by_dkwm( - samples, low=0., high=1., expected=expected_mean, - false_fail_rate=1e-6) - - # Check that the difference in means detectable with 5000 samples is - # small enough - check2 = tf.compat.v1.assert_less( - statistical_testing.min_discrepancy_of_true_means_detectable_by_dkwm( - num_samples, low=0., high=1.0, - false_fail_rate=1e-6, false_pass_rate=1e-6), - 0.01) - - # Be sure to execute both assertion ops - sess.run([check1, check2]) -``` - -The second assertion is an instance of experiment design. It's a -deterministic computation (independent of the code under test) that -checks that `5000` samples is enough to reliably resolve mean -differences of `0.01` or more. Here "reliably" means that if the code -under test is correct, the probability of drawing an unlucky sample -that causes this test to fail is at most 1e-6; and if the code under -test is incorrect enough that its true mean is 0.01 more or less than -expected, then the probability of drawing a "lucky" sample that causes -the test to false-pass is also at most 1e-6. - -# Overview - -Every function in this library can be characterized in terms of: - -- The property being tested, such as the full density of the - distribution under test, or just its true mean, or a single - Bernoulli probability, etc. - -- The relation being asserted, e.g., whether the mean is less, more, - or equal to the given expected value. - -- The stochastic bound being relied upon, such as the - [Dvoretzky-Kiefer-Wolfowitz-Massart inequality] - (https://en.wikipedia.org/wiki/CDF-based_nonparametric_confidence_interval) - or the CDF of the binomial distribution (for assertions about - Bernoulli probabilities). - -- The number of sample sets in the statistical test. For example, - testing equality of means has a one-sample variant, where the - expected mean is given exactly, and a two-sample variant, where the - expected mean is itself given by a set of samples (e.g., from an - alternative algorithm). - -- What operation(s) of the test are to be performed. Each test has - three of these: - - 1. `assert` executes the test. Specifically, it creates a TF op that - produces an error if it has enough evidence to prove that the - property under test is violated. These functions depend on the - desired false failure rate, because that determines the sizes of - appropriate confidence intervals, etc. - - 2. `min_discrepancy` computes the smallest difference reliably - detectable by that test, given the sample count and error rates. - What it's a difference of is test-specific. For example, a test - for equality of means would make detection guarantees about the - difference the true means. - - 3. `min_num_samples` computes the minimum number of samples needed - to reliably detect a given discrepancy with given error rates. - - The latter two are for experimental design, and are meant to be - usable either interactively or inline in the overall test method. - -This library follows a naming convention, to make room for every -combination of the above. A name mentions the operation first, then -the property, then the relation, then the bound, then, if the test -takes more than one set of samples, a token indicating this. For -example, `assert_true_mean_equal_by_dkwm` (which is implicitly -one-sample). Each name is a grammatically sound noun phrase (or verb -phrase, for the asserts). - -# Asymptotic properties - -The number of samples needed tends to scale as `O(1/discrepancy**2)` and -as `O(log(1/error_rate))`. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops - -__all__ = [ - "true_mean_confidence_interval_by_dkwm", - "assert_true_mean_equal_by_dkwm", - "min_discrepancy_of_true_means_detectable_by_dkwm", - "min_num_samples_for_dkwm_mean_test", - "assert_true_mean_in_interval_by_dkwm", - "assert_true_mean_equal_by_dkwm_two_sample", - "min_discrepancy_of_true_means_detectable_by_dkwm_two_sample", - "min_num_samples_for_dkwm_mean_two_sample_test", -] - - -def _batch_sort_vector(x, ascending=True, name=None): - with ops.name_scope(name, "_batch_sort_vector", [x]): - x = ops.convert_to_tensor(x, name="x") - n = array_ops.shape(x)[-1] - if ascending: - y, _ = nn_ops.top_k(-x, k=n, sorted=True) - y = -y - else: - y, _ = nn_ops.top_k(x, k=n, sorted=True) - y.set_shape(x.shape) - return y - - -def _do_maximum_mean(samples, envelope, high, name=None): - """Common code between maximum_mean and minimum_mean.""" - with ops.name_scope(name, "do_maximum_mean", [samples, envelope, high]): - n = array_ops.rank(samples) - # Move the batch dimension of `samples` to the rightmost position, - # where the _batch_sort_vector function wants it. - perm = array_ops.concat([math_ops.range(1, n), [0]], axis=0) - samples = array_ops.transpose(samples, perm) - - samples = _batch_sort_vector(samples) - - # The maximum mean is given by taking `envelope`-worth of - # probability from the smallest samples and moving it to the - # maximum value. This amounts to: - # - ignoring the smallest k samples, where `k/n < envelope` - # - taking a `1/n - (envelope - k/n)` part of the index k sample - # - taking all the other samples - # - and adding `envelope * high` at the end. - # The following is a vectorized and batched way of computing this. - # `max_mean_contrib` is a mask implementing the previous. - batch_size = array_ops.shape(samples)[-1] - batch_size = math_ops.cast(batch_size, dtype=samples.dtype.base_dtype) - step = 1. / batch_size - cum_steps = step * math_ops.range( - 1, batch_size + 1, dtype=samples.dtype.base_dtype) - max_mean_contrib = clip_ops.clip_by_value( - cum_steps - envelope[..., array_ops.newaxis], - clip_value_min=0., - clip_value_max=step) - return math_ops.reduce_sum( - samples * max_mean_contrib, axis=-1) + envelope * high - - -def _maximum_mean(samples, envelope, high, name=None): - """Returns a stochastic upper bound on the mean of a scalar distribution. - - The idea is that if the true CDF is within an `eps`-envelope of the - empirical CDF of the samples, and the support is bounded above, then - the mean is bounded above as well. In symbols, - - ```none - sup_x(|F_n(x) - F(x)|) < eps - ``` - - The 0th dimension of `samples` is interpreted as independent and - identically distributed samples. The remaining dimensions are - broadcast together with `envelope` and `high`, and operated on - separately. - - Args: - samples: Floating-point `Tensor` of samples from the distribution(s) - of interest. Entries are assumed IID across the 0th dimension. - The other dimensions must broadcast with `envelope` and `high`. - envelope: Floating-point `Tensor` of sizes of admissible CDF - envelopes (i.e., the `eps` above). - high: Floating-point `Tensor` of upper bounds on the distributions' - supports. `samples <= high`. - name: A name for this operation (optional). - - Returns: - bound: Floating-point `Tensor` of upper bounds on the true means. - - Raises: - InvalidArgumentError: If some `sample` is found to be larger than - the corresponding `high`. - """ - with ops.name_scope(name, "maximum_mean", [samples, envelope, high]): - samples = ops.convert_to_tensor(samples, name="samples") - envelope = ops.convert_to_tensor(envelope, name="envelope") - high = ops.convert_to_tensor(high, name="high") - - xmax = math_ops.reduce_max(samples, axis=[0]) - msg = "Given sample maximum value exceeds expectations" - check_op = check_ops.assert_less_equal(xmax, high, message=msg) - with ops.control_dependencies([check_op]): - return array_ops.identity(_do_maximum_mean(samples, envelope, high)) - - -def _minimum_mean(samples, envelope, low, name=None): - """Returns a stochastic lower bound on the mean of a scalar distribution. - - The idea is that if the true CDF is within an `eps`-envelope of the - empirical CDF of the samples, and the support is bounded below, then - the mean is bounded below as well. In symbols, - - ```none - sup_x(|F_n(x) - F(x)|) < eps - ``` - - The 0th dimension of `samples` is interpreted as independent and - identically distributed samples. The remaining dimensions are - broadcast together with `envelope` and `low`, and operated on - separately. - - Args: - samples: Floating-point `Tensor` of samples from the distribution(s) - of interest. Entries are assumed IID across the 0th dimension. - The other dimensions must broadcast with `envelope` and `low`. - envelope: Floating-point `Tensor` of sizes of admissible CDF - envelopes (i.e., the `eps` above). - low: Floating-point `Tensor` of lower bounds on the distributions' - supports. `samples >= low`. - name: A name for this operation (optional). - - Returns: - bound: Floating-point `Tensor` of lower bounds on the true means. - - Raises: - InvalidArgumentError: If some `sample` is found to be smaller than - the corresponding `low`. - """ - with ops.name_scope(name, "minimum_mean", [samples, envelope, low]): - samples = ops.convert_to_tensor(samples, name="samples") - envelope = ops.convert_to_tensor(envelope, name="envelope") - low = ops.convert_to_tensor(low, name="low") - - xmin = math_ops.reduce_min(samples, axis=[0]) - msg = "Given sample minimum value falls below expectations" - check_op = check_ops.assert_greater_equal(xmin, low, message=msg) - with ops.control_dependencies([check_op]): - return - _do_maximum_mean(-samples, envelope, -low) - - -def _dkwm_cdf_envelope(n, error_rate, name=None): - """Computes the CDF envelope that the DKWM inequality licenses. - - The [Dvoretzky-Kiefer-Wolfowitz-Massart inequality] - (https://en.wikipedia.org/wiki/CDF-based_nonparametric_confidence_interval) - gives a stochastic bound on the distance between the true cumulative - distribution function (CDF) of any distribution and its empirical - CDF. To wit, for `n` iid samples from any distribution with CDF F, - - ```none - P(sup_x |F_n(x) - F(x)| > eps) < 2exp(-2n eps^2) - ``` - - This function computes the envelope size `eps` as a function of the - number of samples `n` and the desired limit on the left-hand - probability above. - - Args: - n: `Tensor` of numbers of samples drawn. - error_rate: Floating-point `Tensor` of admissible rates of mistakes. - name: A name for this operation (optional). - - Returns: - eps: `Tensor` of maximum distances the true CDF can be from the - empirical CDF. This scales as `O(sqrt(-log(error_rate)))` and - as `O(1 / sqrt(n))`. The shape is the broadcast of `n` and - `error_rate`. - """ - with ops.name_scope(name, "dkwm_cdf_envelope", [n, error_rate]): - n = math_ops.cast(n, dtype=error_rate.dtype) - return math_ops.sqrt(-gen_math_ops.log(error_rate / 2.) / (2. * n)) - - -def _check_shape_dominates(samples, parameters): - """Check that broadcasting `samples` against `parameters` does not expand it. - - Why? Because I want to be very sure that the samples tensor is not - accidentally enlarged by broadcasting against tensors that are - supposed to be describing the distribution(s) sampled from, lest the - sample counts end up inflated. - - Args: - samples: A `Tensor` whose shape is to be protected against broadcasting. - parameters: A list of `Tensor`s who are parameters for the statistical test. - - Returns: - samples: Return original `samples` with control dependencies attached - to ensure no broadcasting. - """ - def check(t): - samples_batch_shape = array_ops.shape(samples)[1:] - broadcasted_batch_shape = array_ops.broadcast_dynamic_shape( - samples_batch_shape, array_ops.shape(t)) - # This rank check ensures that I don't get a wrong answer from the - # _shapes_ broadcasting against each other. - samples_batch_ndims = array_ops.size(samples_batch_shape) - ge = check_ops.assert_greater_equal( - samples_batch_ndims, array_ops.rank(t)) - eq = check_ops.assert_equal(samples_batch_shape, broadcasted_batch_shape) - return ge, eq - checks = list(itertools.chain(*[check(t) for t in parameters])) - with ops.control_dependencies(checks): - return array_ops.identity(samples) - - -def true_mean_confidence_interval_by_dkwm( - samples, low, high, error_rate=1e-6, name=None): - """Computes a confidence interval for the mean of a scalar distribution. - - In batch mode, computes confidence intervals for all distributions - in the batch (which need not be identically distributed). - - Relies on the [Dvoretzky-Kiefer-Wolfowitz-Massart inequality] - (https://en.wikipedia.org/wiki/CDF-based_nonparametric_confidence_interval). - - The probability (over the randomness of drawing the given samples) - that any true mean is outside the corresponding returned interval is - no more than the given `error_rate`. The size of the intervals - scale as - `O(1 / sqrt(#samples))`, as `O(high - low)`, and as `O(-log(error_rate))`. - - Note that `error_rate` is a total error rate for all the confidence - intervals in the batch. As such, if the batch is nontrivial, the - error rate is not broadcast but divided (evenly) among the batch - members. - - Args: - samples: Floating-point `Tensor` of samples from the distribution(s) - of interest. Entries are assumed IID across the 0th dimension. - The other dimensions must broadcast with `low` and `high`. - The support is bounded: `low <= samples <= high`. - low: Floating-point `Tensor` of lower bounds on the distributions' - supports. - high: Floating-point `Tensor` of upper bounds on the distributions' - supports. - error_rate: *Scalar* floating-point `Tensor` admissible total rate - of mistakes. - name: A name for this operation (optional). - - Returns: - low: A floating-point `Tensor` of stochastic lower bounds on the - true means. - high: A floating-point `Tensor` of stochastic upper bounds on the - true means. - """ - with ops.name_scope( - name, "true_mean_confidence_interval_by_dkwm", - [samples, low, high, error_rate]): - samples = ops.convert_to_tensor(samples, name="samples") - low = ops.convert_to_tensor(low, name="low") - high = ops.convert_to_tensor(high, name="high") - error_rate = ops.convert_to_tensor(error_rate, name="error_rate") - samples = _check_shape_dominates(samples, [low, high]) - check_ops.assert_scalar(error_rate) # Static shape - error_rate = _itemwise_error_rate(error_rate, [low, high], samples) - n = array_ops.shape(samples)[0] - envelope = _dkwm_cdf_envelope(n, error_rate) - min_mean = _minimum_mean(samples, envelope, low) - max_mean = _maximum_mean(samples, envelope, high) - return min_mean, max_mean - - -def _itemwise_error_rate( - total_error_rate, param_tensors, sample_tensor=None, name=None): - with ops.name_scope( - name, "itemwise_error_rate", - [total_error_rate, param_tensors, sample_tensor]): - result_shape = [1] - for p_tensor in param_tensors: - result_shape = array_ops.broadcast_dynamic_shape( - array_ops.shape(p_tensor), result_shape) - if sample_tensor is not None: - result_shape = array_ops.broadcast_dynamic_shape( - array_ops.shape(sample_tensor)[1:], result_shape) - num_items = math_ops.reduce_prod(result_shape) - return total_error_rate / math_ops.cast( - num_items, dtype=total_error_rate.dtype) - - -def assert_true_mean_equal_by_dkwm( - samples, low, high, expected, false_fail_rate=1e-6, name=None): - """Asserts the mean of the given distribution is as expected. - - More precisely, fails if there is enough evidence (using the - [Dvoretzky-Kiefer-Wolfowitz-Massart inequality] - (https://en.wikipedia.org/wiki/CDF-based_nonparametric_confidence_interval)) - that the true mean of some distribution from which the given samples are - drawn is _not_ the given expected mean with statistical significance - `false_fail_rate` or stronger, otherwise passes. If you also want to - check that you are gathering enough evidence that a pass is not - spurious, see `min_num_samples_for_dkwm_mean_test` and - `min_discrepancy_of_true_means_detectable_by_dkwm`. - - Note that `false_fail_rate` is a total false failure rate for all - the assertions in the batch. As such, if the batch is nontrivial, - the assertion will insist on stronger evidence to fail any one member. - - Args: - samples: Floating-point `Tensor` of samples from the distribution(s) - of interest. Entries are assumed IID across the 0th dimension. - The other dimensions must broadcast with `low` and `high`. - The support is bounded: `low <= samples <= high`. - low: Floating-point `Tensor` of lower bounds on the distributions' - supports. - high: Floating-point `Tensor` of upper bounds on the distributions' - supports. - expected: Floating-point `Tensor` of expected true means. - false_fail_rate: *Scalar* floating-point `Tensor` admissible total - rate of mistakes. - name: A name for this operation (optional). - - Returns: - check: Op that raises `InvalidArgumentError` if any expected mean is - outside the corresponding confidence interval. - """ - with ops.name_scope( - name, "assert_true_mean_equal_by_dkwm", - [samples, low, high, expected, false_fail_rate]): - return assert_true_mean_in_interval_by_dkwm( - samples, low, high, expected, expected, false_fail_rate) - - -def min_discrepancy_of_true_means_detectable_by_dkwm( - n, low, high, false_fail_rate, false_pass_rate, name=None): - """Returns the minimum mean discrepancy that a DKWM-based test can detect. - - DKWM is the [Dvoretzky-Kiefer-Wolfowitz-Massart inequality] - (https://en.wikipedia.org/wiki/CDF-based_nonparametric_confidence_interval). - - Note that `false_fail_rate` is a total false failure rate for all - the tests in the batch. As such, if the batch is nontrivial, each - member will demand more samples. The `false_pass_rate` is also - interpreted as a total, but is treated asymmetrically: If each test - in the batch detects its corresponding discrepancy with probability - at least `1 - false_pass_rate`, then running all those tests and - failing if any one fails will jointly detect all those discrepancies - with the same `false_pass_rate`. - - Args: - n: `Tensor` of numbers of samples to be drawn from the distributions - of interest. - low: Floating-point `Tensor` of lower bounds on the distributions' - supports. - high: Floating-point `Tensor` of upper bounds on the distributions' - supports. - false_fail_rate: *Scalar* floating-point `Tensor` admissible total - rate of false failures. - false_pass_rate: *Scalar* floating-point `Tensor` admissible rate - of false passes. - name: A name for this operation (optional). - - Returns: - discr: `Tensor` of lower bounds on the distances between true - means detectable by a DKWM-based test. - - For each batch member `i`, of `K` total, drawing `n[i]` samples from - some scalar distribution supported on `[low[i], high[i]]` is enough - to detect a difference in means of size `discr[i]` or more. - Specifically, we guarantee that (a) if the true mean is the expected - mean (resp. in the expected interval), then `assert_true_mean_equal_by_dkwm` - (resp. `assert_true_mean_in_interval_by_dkwm`) will fail with - probability at most `false_fail_rate / K` (which amounts to - `false_fail_rate` if applied to the whole batch at once), and (b) if - the true mean differs from the expected mean (resp. falls outside - the expected interval) by at least `discr[i]`, - `assert_true_mean_equal_by_dkwm` - (resp. `assert_true_mean_in_interval_by_dkwm`) will pass with - probability at most `false_pass_rate`. - - The detectable discrepancy scales as - - - `O(high[i] - low[i])`, - - `O(1 / sqrt(n[i]))`, - - `O(-log(false_fail_rate/K))`, and - - `O(-log(false_pass_rate))`. - """ - with ops.name_scope( - name, "min_discrepancy_of_true_means_detectable_by_dkwm", - [n, low, high, false_fail_rate, false_pass_rate]): - n = ops.convert_to_tensor(n, name="n") - low = ops.convert_to_tensor(low, name="low") - high = ops.convert_to_tensor(high, name="high") - false_fail_rate = ops.convert_to_tensor( - false_fail_rate, name="false_fail_rate") - false_pass_rate = ops.convert_to_tensor( - false_pass_rate, name="false_pass_rate") - # Algorithm: Assume a true CDF F. The DKWM inequality gives a - # stochastic bound on how far the observed empirical CDF F_n can be. - # Then, using the DKWM inequality again gives a stochastic bound on - # the farthest candidate true CDF F' that - # true_mean_confidence_interval_by_dkwm might consider. At worst, these - # errors may go in the same direction, so the distance between F and - # F' is bounded by the sum. - # On batching: false fail rates sum, so I need to reduce - # the input to account for the batching. False pass rates - # max, so I don't. - sampling_envelope = _dkwm_cdf_envelope(n, false_pass_rate) - false_fail_rate = _itemwise_error_rate(false_fail_rate, [n, low, high]) - analysis_envelope = _dkwm_cdf_envelope(n, false_fail_rate) - return (high - low) * (sampling_envelope + analysis_envelope) - - -def min_num_samples_for_dkwm_mean_test( - discrepancy, low, high, - false_fail_rate=1e-6, false_pass_rate=1e-6, name=None): - """Returns how many samples suffice for a one-sample DKWM mean test. - - To wit, returns an upper bound on the number of samples necessary to - guarantee detecting a mean difference of at least the given - `discrepancy`, with the given `false_fail_rate` and `false_pass_rate`, - using the [Dvoretzky-Kiefer-Wolfowitz-Massart inequality] - (https://en.wikipedia.org/wiki/CDF-based_nonparametric_confidence_interval) - on a scalar distribution supported on `[low, high]`. - - Args: - discrepancy: Floating-point `Tensor` of desired upper limits on mean - differences that may go undetected with probability higher than - `1 - false_pass_rate`. - low: `Tensor` of lower bounds on the distributions' support. - high: `Tensor` of upper bounds on the distributions' support. - false_fail_rate: *Scalar* floating-point `Tensor` admissible total - rate of false failures. - false_pass_rate: *Scalar* floating-point `Tensor` admissible rate - of false passes. - name: A name for this operation (optional). - - Returns: - n: `Tensor` of numbers of samples to be drawn from the distributions - of interest. - - The `discrepancy`, `low`, and `high` tensors must have - broadcast-compatible shapes. - - For each batch member `i`, of `K` total, drawing `n[i]` samples from - some scalar distribution supported on `[low[i], high[i]]` is enough - to detect a difference in means of size `discrepancy[i]` or more. - Specifically, we guarantee that (a) if the true mean is the expected - mean (resp. in the expected interval), then `assert_true_mean_equal_by_dkwm` - (resp. `assert_true_mean_in_interval_by_dkwm`) will fail with - probability at most `false_fail_rate / K` (which amounts to - `false_fail_rate` if applied to the whole batch at once), and (b) if - the true mean differs from the expected mean (resp. falls outside - the expected interval) by at least `discrepancy[i]`, - `assert_true_mean_equal_by_dkwm` - (resp. `assert_true_mean_in_interval_by_dkwm`) will pass with - probability at most `false_pass_rate`. - - The required number of samples scales - as `O((high[i] - low[i])**2)`, `O(-log(false_fail_rate/K))`, - `O(-log(false_pass_rate))`, and `O(1 / discrepancy[i]**2)`. - """ - with ops.name_scope( - name, "min_num_samples_for_dkwm_mean_test", - [low, high, false_fail_rate, false_pass_rate, discrepancy]): - discrepancy = ops.convert_to_tensor( - discrepancy, name="discrepancy") - low = ops.convert_to_tensor(low, name="low") - high = ops.convert_to_tensor(high, name="high") - false_fail_rate = ops.convert_to_tensor( - false_fail_rate, name="false_fail_rate") - false_pass_rate = ops.convert_to_tensor( - false_pass_rate, name="false_pass_rate") - # Could choose to cleverly allocate envelopes, but this is sound. - envelope1 = discrepancy / (2. * (high - low)) - envelope2 = envelope1 - false_fail_rate = _itemwise_error_rate( - false_fail_rate, [low, high, discrepancy]) - n1 = -math_ops.log(false_fail_rate / 2.) / (2. * envelope1**2) - n2 = -math_ops.log(false_pass_rate / 2.) / (2. * envelope2**2) - return math_ops.maximum(n1, n2) - - -def assert_true_mean_in_interval_by_dkwm( - samples, low, high, expected_low, expected_high, - false_fail_rate=1e-6, name=None): - """Asserts the mean of the given distribution is in the given interval. - - More precisely, fails if there is enough evidence (using the - [Dvoretzky-Kiefer-Wolfowitz-Massart inequality] - (https://en.wikipedia.org/wiki/CDF-based_nonparametric_confidence_interval)) - that the mean of the distribution from which the given samples are - drawn is _outside_ the given interval with statistical significance - `false_fail_rate` or stronger, otherwise passes. If you also want - to check that you are gathering enough evidence that a pass is not - spurious, see `min_num_samples_for_dkwm_mean_test` and - `min_discrepancy_of_true_means_detectable_by_dkwm`. - - Note that `false_fail_rate` is a total false failure rate for all - the assertions in the batch. As such, if the batch is nontrivial, - the assertion will insist on stronger evidence to fail any one member. - - Args: - samples: Floating-point `Tensor` of samples from the distribution(s) - of interest. Entries are assumed IID across the 0th dimension. - The other dimensions must broadcast with `low` and `high`. - The support is bounded: `low <= samples <= high`. - low: Floating-point `Tensor` of lower bounds on the distributions' - supports. - high: Floating-point `Tensor` of upper bounds on the distributions' - supports. - expected_low: Floating-point `Tensor` of lower bounds on the - expected true means. - expected_high: Floating-point `Tensor` of upper bounds on the - expected true means. - false_fail_rate: *Scalar* floating-point `Tensor` admissible total - rate of mistakes. - name: A name for this operation (optional). - - Returns: - check: Op that raises `InvalidArgumentError` if any expected mean - interval does not overlap with the corresponding confidence - interval. - """ - with ops.name_scope( - name, "assert_true_mean_in_interval_by_dkwm", - [samples, low, high, expected_low, expected_high, false_fail_rate]): - samples = ops.convert_to_tensor(samples, name="samples") - low = ops.convert_to_tensor(low, name="low") - high = ops.convert_to_tensor(high, name="high") - expected_low = ops.convert_to_tensor(expected_low, name="expected_low") - expected_high = ops.convert_to_tensor(expected_high, name="expected_high") - false_fail_rate = ops.convert_to_tensor( - false_fail_rate, name="false_fail_rate") - samples = _check_shape_dominates( - samples, [low, high, expected_low, expected_high]) - min_mean, max_mean = true_mean_confidence_interval_by_dkwm( - samples, low, high, false_fail_rate) - # Assert that the interval [min_mean, max_mean] intersects the - # interval [expected_low, expected_high]. This is true if - # max_mean >= expected_low and min_mean <= expected_high. - # By DeMorgan's law, that's also equivalent to - # not (max_mean < expected_low or min_mean > expected_high), - # which is a way of saying the two intervals are not disjoint. - check_confidence_interval_can_intersect = check_ops.assert_greater_equal( - max_mean, expected_low, message="Confidence interval does not " - "intersect: true mean smaller than expected") - with ops.control_dependencies([check_confidence_interval_can_intersect]): - return check_ops.assert_less_equal( - min_mean, expected_high, message="Confidence interval does not " - "intersect: true mean greater than expected") - - -def assert_true_mean_equal_by_dkwm_two_sample( - samples1, low1, high1, samples2, low2, high2, - false_fail_rate=1e-6, name=None): - """Asserts the means of the given distributions are equal. - - More precisely, fails if there is enough evidence (using the - [Dvoretzky-Kiefer-Wolfowitz-Massart inequality] - (https://en.wikipedia.org/wiki/CDF-based_nonparametric_confidence_interval)) - that the means of the distributions from which the given samples are - drawn are _not_ equal with statistical significance `false_fail_rate` - or stronger, otherwise passes. If you also want to check that you - are gathering enough evidence that a pass is not spurious, see - `min_num_samples_for_dkwm_mean_two_sample_test` and - `min_discrepancy_of_true_means_detectable_by_dkwm_two_sample`. - - Note that `false_fail_rate` is a total false failure rate for all - the assertions in the batch. As such, if the batch is nontrivial, - the assertion will insist on stronger evidence to fail any one member. - - Args: - samples1: Floating-point `Tensor` of samples from the - distribution(s) A. Entries are assumed IID across the 0th - dimension. The other dimensions must broadcast with `low1`, - `high1`, `low2`, and `high2`. - The support is bounded: `low1 <= samples1 <= high1`. - low1: Floating-point `Tensor` of lower bounds on the supports of the - distributions A. - high1: Floating-point `Tensor` of upper bounds on the supports of - the distributions A. - samples2: Floating-point `Tensor` of samples from the - distribution(s) B. Entries are assumed IID across the 0th - dimension. The other dimensions must broadcast with `low1`, - `high1`, `low2`, and `high2`. - The support is bounded: `low2 <= samples2 <= high2`. - low2: Floating-point `Tensor` of lower bounds on the supports of the - distributions B. - high2: Floating-point `Tensor` of upper bounds on the supports of - the distributions B. - false_fail_rate: *Scalar* floating-point `Tensor` admissible total - rate of mistakes. - name: A name for this operation (optional). - - Returns: - check: Op that raises `InvalidArgumentError` if any pair of confidence - intervals true for corresponding true means do not overlap. - """ - with ops.name_scope( - name, "assert_true_mean_equal_by_dkwm_two_sample", - [samples1, low1, high1, samples2, low2, high2, false_fail_rate]): - samples1 = ops.convert_to_tensor(samples1, name="samples1") - low1 = ops.convert_to_tensor(low1, name="low1") - high1 = ops.convert_to_tensor(high1, name="high1") - samples2 = ops.convert_to_tensor(samples2, name="samples2") - low2 = ops.convert_to_tensor(low2, name="low2") - high2 = ops.convert_to_tensor(high2, name="high2") - false_fail_rate = ops.convert_to_tensor( - false_fail_rate, name="false_fail_rate") - samples1 = _check_shape_dominates(samples1, [low1, high1]) - samples2 = _check_shape_dominates(samples2, [low2, high2]) - compatible_samples = check_ops.assert_equal( - array_ops.shape(samples1)[1:], array_ops.shape(samples2)[1:]) - with ops.control_dependencies([compatible_samples]): - # Could in principle play games with cleverly allocating - # significance instead of the even split below. It may be possible - # to get tighter intervals, in order to obtain a higher power test. - # Any allocation strategy that depends only on the support bounds - # and sample counts should be valid; however, because the intervals - # scale as O(-log(false_fail_rate)), there doesn't seem to be much - # room to win. - min_mean_2, max_mean_2 = true_mean_confidence_interval_by_dkwm( - samples2, low2, high2, false_fail_rate / 2.) - return assert_true_mean_in_interval_by_dkwm( - samples1, low1, high1, min_mean_2, max_mean_2, false_fail_rate / 2.) - - -def min_discrepancy_of_true_means_detectable_by_dkwm_two_sample( - n1, low1, high1, n2, low2, high2, - false_fail_rate, false_pass_rate, name=None): - """Returns the minimum mean discrepancy for a two-sample DKWM-based test. - - DKWM is the [Dvoretzky-Kiefer-Wolfowitz-Massart inequality] - (https://en.wikipedia.org/wiki/CDF-based_nonparametric_confidence_interval). - - Note that `false_fail_rate` is a total false failure rate for all - the tests in the batch. As such, if the batch is nontrivial, each - member will demand more samples. The `false_pass_rate` is also - interpreted as a total, but is treated asymmetrically: If each test - in the batch detects its corresponding discrepancy with probability - at least `1 - false_pass_rate`, then running all those tests and - failing if any one fails will jointly detect all those discrepancies - with the same `false_pass_rate`. - - Args: - n1: `Tensor` of numbers of samples to be drawn from the distributions A. - low1: Floating-point `Tensor` of lower bounds on the supports of the - distributions A. - high1: Floating-point `Tensor` of upper bounds on the supports of - the distributions A. - n2: `Tensor` of numbers of samples to be drawn from the distributions B. - low2: Floating-point `Tensor` of lower bounds on the supports of the - distributions B. - high2: Floating-point `Tensor` of upper bounds on the supports of - the distributions B. - false_fail_rate: *Scalar* floating-point `Tensor` admissible total - rate of false failures. - false_pass_rate: *Scalar* floating-point `Tensor` admissible rate - of false passes. - name: A name for this operation (optional). - - Returns: - discr: `Tensor` of lower bounds on the distances between true means - detectable by a two-sample DKWM-based test. - - For each batch member `i`, of `K` total, drawing `n1[i]` samples - from scalar distribution A supported on `[low1[i], high1[i]]` and `n2[i]` - samples from scalar distribution B supported on `[low2[i], high2[i]]` - is enough to detect a difference in their true means of size - `discr[i]` or more. Specifically, we guarantee that (a) if their - true means are equal, `assert_true_mean_equal_by_dkwm_two_sample` - will fail with probability at most `false_fail_rate/K` (which - amounts to `false_fail_rate` if applied to the whole batch at once), - and (b) if their true means differ by at least `discr[i]`, - `assert_true_mean_equal_by_dkwm_two_sample` will pass with - probability at most `false_pass_rate`. - - The detectable distribution scales as - - - `O(high1[i] - low1[i])`, `O(high2[i] - low2[i])`, - - `O(1 / sqrt(n1[i]))`, `O(1 / sqrt(n2[i]))`, - - `O(-log(false_fail_rate/K))`, and - - `O(-log(false_pass_rate))`. - """ - with ops.name_scope( - name, "min_discrepancy_of_true_means_detectable_by_dkwm_two_sample", - [n1, low1, high1, n2, low2, high2, false_fail_rate, false_pass_rate]): - n1 = ops.convert_to_tensor(n1, name="n1") - low1 = ops.convert_to_tensor(low1, name="low1") - high1 = ops.convert_to_tensor(high1, name="high1") - n2 = ops.convert_to_tensor(n2, name="n2") - low2 = ops.convert_to_tensor(low2, name="low2") - high2 = ops.convert_to_tensor(high2, name="high2") - false_fail_rate = ops.convert_to_tensor( - false_fail_rate, name="false_fail_rate") - false_pass_rate = ops.convert_to_tensor( - false_pass_rate, name="false_pass_rate") - det_disc1 = min_discrepancy_of_true_means_detectable_by_dkwm( - n1, low1, high1, false_fail_rate / 2., false_pass_rate / 2.) - det_disc2 = min_discrepancy_of_true_means_detectable_by_dkwm( - n2, low2, high2, false_fail_rate / 2., false_pass_rate / 2.) - return det_disc1 + det_disc2 - - -def min_num_samples_for_dkwm_mean_two_sample_test( - discrepancy, low1, high1, low2, high2, - false_fail_rate=1e-6, false_pass_rate=1e-6, name=None): - """Returns how many samples suffice for a two-sample DKWM mean test. - - DKWM is the [Dvoretzky-Kiefer-Wolfowitz-Massart inequality] - (https://en.wikipedia.org/wiki/CDF-based_nonparametric_confidence_interval). - - Args: - discrepancy: Floating-point `Tensor` of desired upper limits on mean - differences that may go undetected with probability higher than - `1 - false_pass_rate`. - low1: Floating-point `Tensor` of lower bounds on the supports of the - distributions A. - high1: Floating-point `Tensor` of upper bounds on the supports of - the distributions A. - low2: Floating-point `Tensor` of lower bounds on the supports of the - distributions B. - high2: Floating-point `Tensor` of upper bounds on the supports of - the distributions B. - false_fail_rate: *Scalar* floating-point `Tensor` admissible total - rate of false failures. - false_pass_rate: *Scalar* floating-point `Tensor` admissible rate - of false passes. - name: A name for this operation (optional). - - Returns: - n1: `Tensor` of numbers of samples to be drawn from the distributions A. - n2: `Tensor` of numbers of samples to be drawn from the distributions B. - - For each batch member `i`, of `K` total, drawing `n1[i]` samples - from scalar distribution A supported on `[low1[i], high1[i]]` and `n2[i]` - samples from scalar distribution B supported on `[low2[i], high2[i]]` - is enough to detect a difference in their true means of size - `discr[i]` or more. Specifically, we guarantee that (a) if their - true means are equal, `assert_true_mean_equal_by_dkwm_two_sample` - will fail with probability at most `false_fail_rate/K` (which - amounts to `false_fail_rate` if applied to the whole batch at once), - and (b) if their true means differ by at least `discr[i]`, - `assert_true_mean_equal_by_dkwm_two_sample` will pass with - probability at most `false_pass_rate`. - - The required number of samples scales as - - - `O((high1[i] - low1[i])**2)`, `O((high2[i] - low2[i])**2)`, - - `O(-log(false_fail_rate/K))`, - - `O(-log(false_pass_rate))`, and - - `O(1 / discrepancy[i]**2)`. - """ - with ops.name_scope( - name, "min_num_samples_for_dkwm_mean_two_sample_test", - [low1, high1, low2, high2, - false_fail_rate, false_pass_rate, discrepancy]): - discrepancy = ops.convert_to_tensor(discrepancy, name="discrepancy") - low1 = ops.convert_to_tensor(low1, name="low1") - high1 = ops.convert_to_tensor(high1, name="high1") - low2 = ops.convert_to_tensor(low2, name="low2") - high2 = ops.convert_to_tensor(high2, name="high2") - false_fail_rate = ops.convert_to_tensor( - false_fail_rate, name="false_fail_rate") - false_pass_rate = ops.convert_to_tensor( - false_pass_rate, name="false_pass_rate") - # Could choose to cleverly allocate discrepancy tolerances and - # failure probabilities, but this is sound. - n1 = min_num_samples_for_dkwm_mean_test( - discrepancy / 2., low1, high1, - false_fail_rate / 2., false_pass_rate / 2.) - n2 = min_num_samples_for_dkwm_mean_test( - discrepancy / 2., low2, high2, - false_fail_rate / 2., false_pass_rate / 2.) - return n1, n2 diff --git a/tensorflow/contrib/distributions/python/ops/test_util.py b/tensorflow/contrib/distributions/python/ops/test_util.py deleted file mode 100644 index 73ab3d818be..00000000000 --- a/tensorflow/contrib/distributions/python/ops/test_util.py +++ /dev/null @@ -1,388 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities for testing distributions and/or bijectors.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import histogram_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables as variables_ops - -__all__ = [ - "DiscreteScalarDistributionTestHelpers", - "VectorDistributionTestHelpers", -] - - -class DiscreteScalarDistributionTestHelpers(object): - """DiscreteScalarDistributionTestHelpers.""" - - def run_test_sample_consistent_log_prob(self, - sess_run_fn, - dist, - num_samples=int(1e5), - num_threshold=int(1e3), - seed=42, - batch_size=None, - rtol=1e-2, - atol=0.): - """Tests that sample/log_prob are consistent with each other. - - "Consistency" means that `sample` and `log_prob` correspond to the same - distribution. - - Note: this test only verifies a necessary condition for consistency--it does - does not verify sufficiency hence does not prove `sample`, `log_prob` truly - are consistent. - - Args: - sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and - returning a list of results after running one "step" of TensorFlow - computation, typically set to `sess.run`. - dist: Distribution instance or object which implements `sample`, - `log_prob`, `event_shape_tensor` and `batch_shape_tensor`. - num_samples: Python `int` scalar indicating the number of Monte-Carlo - samples to draw from `dist`. - num_threshold: Python `int` scalar indicating the number of samples a - bucket must contain before being compared to the probability. - Default value: 1e3; must be at least 1. Warning, set too high will cause - test to falsely pass but setting too low will cause the test to - falsely fail. - seed: Python `int` indicating the seed to use when sampling from `dist`. - In general it is not recommended to use `None` during a test as this - increases the likelihood of spurious test failure. - batch_size: Hint for unpacking result of samples. Default: `None` means - batch_size is inferred. - rtol: Python `float`-type indicating the admissible relative error between - analytical and sample statistics. - atol: Python `float`-type indicating the admissible absolute error between - analytical and sample statistics. - - Raises: - ValueError: if `num_threshold < 1`. - """ - if num_threshold < 1: - raise ValueError( - "num_threshold({}) must be at least 1.".format(num_threshold)) - # Histogram only supports vectors so we call it once per batch coordinate. - y = dist.sample(num_samples, seed=seed) - y = array_ops.reshape(y, shape=[num_samples, -1]) - if batch_size is None: - batch_size = math_ops.reduce_prod(dist.batch_shape_tensor()) - batch_dims = array_ops.shape(dist.batch_shape_tensor())[0] - edges_expanded_shape = 1 + array_ops.pad([-2], paddings=[[0, batch_dims]]) - for b, x in enumerate(array_ops.unstack(y, num=batch_size, axis=1)): - counts, edges = self.histogram(x) - edges = array_ops.reshape(edges, edges_expanded_shape) - probs = math_ops.exp(dist.log_prob(edges)) - probs = array_ops.reshape(probs, shape=[-1, batch_size])[:, b] - - [counts_, probs_] = sess_run_fn([counts, probs]) - valid = counts_ > num_threshold - probs_ = probs_[valid] - counts_ = counts_[valid] - self.assertAllClose(probs_, counts_ / num_samples, rtol=rtol, atol=atol) - - def run_test_sample_consistent_mean_variance(self, - sess_run_fn, - dist, - num_samples=int(1e5), - seed=24, - rtol=1e-2, - atol=0.): - """Tests that sample/mean/variance are consistent with each other. - - "Consistency" means that `sample`, `mean`, `variance`, etc all correspond - to the same distribution. - - Args: - sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and - returning a list of results after running one "step" of TensorFlow - computation, typically set to `sess.run`. - dist: Distribution instance or object which implements `sample`, - `log_prob`, `event_shape_tensor` and `batch_shape_tensor`. - num_samples: Python `int` scalar indicating the number of Monte-Carlo - samples to draw from `dist`. - seed: Python `int` indicating the seed to use when sampling from `dist`. - In general it is not recommended to use `None` during a test as this - increases the likelihood of spurious test failure. - rtol: Python `float`-type indicating the admissible relative error between - analytical and sample statistics. - atol: Python `float`-type indicating the admissible absolute error between - analytical and sample statistics. - """ - x = math_ops.cast(dist.sample(num_samples, seed=seed), dtypes.float32) - sample_mean = math_ops.reduce_mean(x, axis=0) - sample_variance = math_ops.reduce_mean( - math_ops.square(x - sample_mean), axis=0) - sample_stddev = math_ops.sqrt(sample_variance) - - [sample_mean_, sample_variance_, sample_stddev_, mean_, variance_, - stddev_] = sess_run_fn([ - sample_mean, - sample_variance, - sample_stddev, - dist.mean(), - dist.variance(), - dist.stddev(), - ]) - - self.assertAllClose(mean_, sample_mean_, rtol=rtol, atol=atol) - self.assertAllClose(variance_, sample_variance_, rtol=rtol, atol=atol) - self.assertAllClose(stddev_, sample_stddev_, rtol=rtol, atol=atol) - - def histogram(self, x, value_range=None, nbins=None, name=None): - """Return histogram of values. - - Given the tensor `values`, this operation returns a rank 1 histogram - counting the number of entries in `values` that fell into every bin. The - bins are equal width and determined by the arguments `value_range` and - `nbins`. - - Args: - x: 1D numeric `Tensor` of items to count. - value_range: Shape [2] `Tensor`. `new_values <= value_range[0]` will be - mapped to `hist[0]`, `values >= value_range[1]` will be mapped to - `hist[-1]`. Must be same dtype as `x`. - nbins: Scalar `int32 Tensor`. Number of histogram bins. - name: Python `str` name prefixed to Ops created by this class. - - Returns: - counts: 1D `Tensor` of counts, i.e., - `counts[i] = sum{ edges[i-1] <= values[j] < edges[i] : j }`. - edges: 1D `Tensor` characterizing intervals used for counting. - """ - with ops.name_scope(name, "histogram", [x]): - x = ops.convert_to_tensor(x, name="x") - if value_range is None: - value_range = [math_ops.reduce_min(x), 1 + math_ops.reduce_max(x)] - value_range = ops.convert_to_tensor(value_range, name="value_range") - lo = value_range[0] - hi = value_range[1] - if nbins is None: - nbins = math_ops.cast(hi - lo, dtypes.int32) - delta = (hi - lo) / math_ops.cast( - nbins, dtype=value_range.dtype.base_dtype) - edges = math_ops.range( - start=lo, limit=hi, delta=delta, dtype=x.dtype.base_dtype) - counts = histogram_ops.histogram_fixed_width( - x, value_range=value_range, nbins=nbins) - return counts, edges - - -class VectorDistributionTestHelpers(object): - """VectorDistributionTestHelpers helps test vector-event distributions.""" - - def run_test_sample_consistent_log_prob(self, - sess_run_fn, - dist, - num_samples=int(1e5), - radius=1., - center=0., - seed=42, - rtol=1e-2, - atol=0.): - """Tests that sample/log_prob are mutually consistent. - - "Consistency" means that `sample` and `log_prob` correspond to the same - distribution. - - The idea of this test is to compute the Monte-Carlo estimate of the volume - enclosed by a hypersphere, i.e., the volume of an `n`-ball. While we could - choose an arbitrary function to integrate, the hypersphere's volume is nice - because it is intuitive, has an easy analytical expression, and works for - `dimensions > 1`. - - Technical Details: - - Observe that: - - ```none - int_{R**d} dx [x in Ball(radius=r, center=c)] - = E_{p(X)}[ [X in Ball(r, c)] / p(X) ] - = lim_{m->infty} m**-1 sum_j^m [x[j] in Ball(r, c)] / p(x[j]), - where x[j] ~iid p(X) - ``` - - Thus, for fixed `m`, the above is approximately true when `sample` and - `log_prob` are mutually consistent. - - Furthermore, the above calculation has the analytical result: - `pi**(d/2) r**d / Gamma(1 + d/2)`. - - Note: this test only verifies a necessary condition for consistency--it does - does not verify sufficiency hence does not prove `sample`, `log_prob` truly - are consistent. For this reason we recommend testing several different - hyperspheres (assuming the hypersphere is supported by the distribution). - Furthermore, we gain additional trust in this test when also tested `sample` - against the first, second moments - (`run_test_sample_consistent_mean_covariance`); it is probably unlikely that - a "best-effort" implementation of `log_prob` would incorrectly pass both - tests and for different hyperspheres. - - For a discussion on the analytical result (second-line) see: - https://en.wikipedia.org/wiki/Volume_of_an_n-ball. - - For a discussion of importance sampling (fourth-line) see: - https://en.wikipedia.org/wiki/Importance_sampling. - - Args: - sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and - returning a list of results after running one "step" of TensorFlow - computation, typically set to `sess.run`. - dist: Distribution instance or object which implements `sample`, - `log_prob`, `event_shape_tensor` and `batch_shape_tensor`. The - distribution must have non-zero probability of sampling every point - enclosed by the hypersphere. - num_samples: Python `int` scalar indicating the number of Monte-Carlo - samples to draw from `dist`. - radius: Python `float`-type indicating the radius of the `n`-ball which - we're computing the volume. - center: Python floating-type vector (or scalar) indicating the center of - the `n`-ball which we're computing the volume. When scalar, the value is - broadcast to all event dims. - seed: Python `int` indicating the seed to use when sampling from `dist`. - In general it is not recommended to use `None` during a test as this - increases the likelihood of spurious test failure. - rtol: Python `float`-type indicating the admissible relative error between - actual- and approximate-volumes. - atol: Python `float`-type indicating the admissible absolute error between - actual- and approximate-volumes. In general this should be zero since a - typical radius implies a non-zero volume. - """ - - def actual_hypersphere_volume(dims, radius): - # https://en.wikipedia.org/wiki/Volume_of_an_n-ball - # Using tf.math.lgamma because we'd have to otherwise use SciPy which is - # not a required dependency of core. - radius = np.asarray(radius) - dims = math_ops.cast(dims, dtype=radius.dtype) - return math_ops.exp((dims / 2.) * np.log(np.pi) - - math_ops.lgamma(1. + dims / 2.) + - dims * math_ops.log(radius)) - - def is_in_ball(x, radius, center): - return math_ops.cast( - linalg_ops.norm(x - center, axis=-1) <= radius, dtype=x.dtype) - - def monte_carlo_hypersphere_volume(dist, num_samples, radius, center): - # https://en.wikipedia.org/wiki/Importance_sampling - x = dist.sample(num_samples, seed=seed) - x = array_ops.identity(x) # Invalidate bijector cacheing. - return math_ops.reduce_mean( - math_ops.exp(-dist.log_prob(x)) * is_in_ball(x, radius, center), - axis=0) - - # Build graph. - with ops.name_scope( - "run_test_sample_consistent_log_prob", - values=[num_samples, radius, center] + dist._graph_parents): # pylint: disable=protected-access - batch_shape = dist.batch_shape_tensor() - actual_volume = actual_hypersphere_volume( - dims=dist.event_shape_tensor()[0], radius=radius) - sample_volume = monte_carlo_hypersphere_volume( - dist, num_samples=num_samples, radius=radius, center=center) - init_op = variables_ops.global_variables_initializer() - - # Execute graph. - sess_run_fn(init_op) - [batch_shape_, actual_volume_, - sample_volume_] = sess_run_fn([batch_shape, actual_volume, sample_volume]) - - # Check results. - self.assertAllClose( - np.tile(actual_volume_, reps=batch_shape_), - sample_volume_, - rtol=rtol, - atol=atol) - - def run_test_sample_consistent_mean_covariance(self, - sess_run_fn, - dist, - num_samples=int(1e5), - seed=24, - rtol=1e-2, - atol=0.1, - cov_rtol=None, - cov_atol=None): - """Tests that sample/mean/covariance are consistent with each other. - - "Consistency" means that `sample`, `mean`, `covariance`, etc all correspond - to the same distribution. - - Args: - sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and - returning a list of results after running one "step" of TensorFlow - computation, typically set to `sess.run`. - dist: Distribution instance or object which implements `sample`, - `log_prob`, `event_shape_tensor` and `batch_shape_tensor`. - num_samples: Python `int` scalar indicating the number of Monte-Carlo - samples to draw from `dist`. - seed: Python `int` indicating the seed to use when sampling from `dist`. - In general it is not recommended to use `None` during a test as this - increases the likelihood of spurious test failure. - rtol: Python `float`-type indicating the admissible relative error between - analytical and sample statistics. - atol: Python `float`-type indicating the admissible absolute error between - analytical and sample statistics. - cov_rtol: Python `float`-type indicating the admissible relative error - between analytical and sample covariance. Default: rtol. - cov_atol: Python `float`-type indicating the admissible absolute error - between analytical and sample covariance. Default: atol. - """ - - x = dist.sample(num_samples, seed=seed) - sample_mean = math_ops.reduce_mean(x, axis=0) - sample_covariance = math_ops.reduce_mean( - _vec_outer_square(x - sample_mean), axis=0) - sample_variance = array_ops.matrix_diag_part(sample_covariance) - sample_stddev = math_ops.sqrt(sample_variance) - - [ - sample_mean_, sample_covariance_, sample_variance_, sample_stddev_, - mean_, covariance_, variance_, stddev_ - ] = sess_run_fn([ - sample_mean, - sample_covariance, - sample_variance, - sample_stddev, - dist.mean(), - dist.covariance(), - dist.variance(), - dist.stddev(), - ]) - - self.assertAllClose(mean_, sample_mean_, rtol=rtol, atol=atol) - self.assertAllClose( - covariance_, - sample_covariance_, - rtol=cov_rtol or rtol, - atol=cov_atol or atol) - self.assertAllClose(variance_, sample_variance_, rtol=rtol, atol=atol) - self.assertAllClose(stddev_, sample_stddev_, rtol=rtol, atol=atol) - - -def _vec_outer_square(x, name=None): - """Computes the outer-product of a vector, i.e., x.T x.""" - with ops.name_scope(name, "vec_osquare", [x]): - return x[..., :, array_ops.newaxis] * x[..., array_ops.newaxis, :] diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py deleted file mode 100644 index f17ac136406..00000000000 --- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py +++ /dev/null @@ -1,1063 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The VectorDiffeomixture distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.contrib.distributions.python.ops.bijectors.affine_linear_operator import AffineLinearOperator -from tensorflow.contrib.distributions.python.ops.bijectors.softmax_centered import SoftmaxCentered -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops.distributions import categorical as categorical_lib -from tensorflow.python.ops.distributions import distribution as distribution_lib -from tensorflow.python.ops.distributions import normal as normal_lib -from tensorflow.python.ops.linalg import linear_operator_addition as linop_add_lib -from tensorflow.python.ops.linalg import linear_operator_diag as linop_diag_lib -from tensorflow.python.ops.linalg import linear_operator_full_matrix as linop_full_lib -from tensorflow.python.ops.linalg import linear_operator_identity as linop_identity_lib -from tensorflow.python.ops.linalg import linear_operator_lower_triangular as linop_tril_lib -from tensorflow.python.util import deprecation - - -__all__ = [ - "VectorDiffeomixture", - "quadrature_scheme_softmaxnormal_gauss_hermite", - "quadrature_scheme_softmaxnormal_quantiles", -] - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def quadrature_scheme_softmaxnormal_gauss_hermite( - normal_loc, normal_scale, quadrature_size, - validate_args=False, name=None): - """Use Gauss-Hermite quadrature to form quadrature on `K - 1` simplex. - - A `SoftmaxNormal` random variable `Y` may be generated via - - ``` - Y = SoftmaxCentered(X), - X = Normal(normal_loc, normal_scale) - ``` - - Note: for a given `quadrature_size`, this method is generally less accurate - than `quadrature_scheme_softmaxnormal_quantiles`. - - Args: - normal_loc: `float`-like `Tensor` with shape `[b1, ..., bB, K-1]`, B>=0. - The location parameter of the Normal used to construct the SoftmaxNormal. - normal_scale: `float`-like `Tensor`. Broadcastable with `normal_loc`. - The scale parameter of the Normal used to construct the SoftmaxNormal. - quadrature_size: Python `int` scalar representing the number of quadrature - points. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - name: Python `str` name prefixed to Ops created by this class. - - Returns: - grid: Shape `[b1, ..., bB, K, quadrature_size]` `Tensor` representing the - convex combination of affine parameters for `K` components. - `grid[..., :, n]` is the `n`-th grid point, living in the `K - 1` simplex. - probs: Shape `[b1, ..., bB, K, quadrature_size]` `Tensor` representing the - associated with each grid point. - """ - with ops.name_scope(name, "quadrature_scheme_softmaxnormal_gauss_hermite", - [normal_loc, normal_scale]): - normal_loc = ops.convert_to_tensor(normal_loc, name="normal_loc") - dt = normal_loc.dtype.base_dtype - normal_scale = ops.convert_to_tensor( - normal_scale, dtype=dt, name="normal_scale") - - normal_scale = maybe_check_quadrature_param( - normal_scale, "normal_scale", validate_args) - - grid, probs = np.polynomial.hermite.hermgauss(deg=quadrature_size) - grid = grid.astype(dt.dtype.as_numpy_dtype) - probs = probs.astype(dt.dtype.as_numpy_dtype) - probs /= np.linalg.norm(probs, ord=1, keepdims=True) - probs = ops.convert_to_tensor(probs, name="probs", dtype=dt) - - grid = softmax( - -distribution_util.pad( - (normal_loc[..., array_ops.newaxis] + - np.sqrt(2.) * normal_scale[..., array_ops.newaxis] * grid), - axis=-2, - front=True), - axis=-2) # shape: [B, components, deg] - - return grid, probs - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def quadrature_scheme_softmaxnormal_quantiles( - normal_loc, normal_scale, quadrature_size, - validate_args=False, name=None): - """Use SoftmaxNormal quantiles to form quadrature on `K - 1` simplex. - - A `SoftmaxNormal` random variable `Y` may be generated via - - ``` - Y = SoftmaxCentered(X), - X = Normal(normal_loc, normal_scale) - ``` - - Args: - normal_loc: `float`-like `Tensor` with shape `[b1, ..., bB, K-1]`, B>=0. - The location parameter of the Normal used to construct the SoftmaxNormal. - normal_scale: `float`-like `Tensor`. Broadcastable with `normal_loc`. - The scale parameter of the Normal used to construct the SoftmaxNormal. - quadrature_size: Python `int` scalar representing the number of quadrature - points. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - name: Python `str` name prefixed to Ops created by this class. - - Returns: - grid: Shape `[b1, ..., bB, K, quadrature_size]` `Tensor` representing the - convex combination of affine parameters for `K` components. - `grid[..., :, n]` is the `n`-th grid point, living in the `K - 1` simplex. - probs: Shape `[b1, ..., bB, K, quadrature_size]` `Tensor` representing the - associated with each grid point. - """ - with ops.name_scope(name, "softmax_normal_grid_and_probs", - [normal_loc, normal_scale]): - normal_loc = ops.convert_to_tensor(normal_loc, name="normal_loc") - dt = normal_loc.dtype.base_dtype - normal_scale = ops.convert_to_tensor( - normal_scale, dtype=dt, name="normal_scale") - - normal_scale = maybe_check_quadrature_param( - normal_scale, "normal_scale", validate_args) - - dist = normal_lib.Normal(loc=normal_loc, scale=normal_scale) - - def _get_batch_ndims(): - """Helper to get dist.batch_shape.ndims, statically if possible.""" - ndims = dist.batch_shape.ndims - if ndims is None: - ndims = array_ops.shape(dist.batch_shape_tensor())[0] - return ndims - batch_ndims = _get_batch_ndims() - - def _get_final_shape(qs): - """Helper to build `TensorShape`.""" - bs = dist.batch_shape.with_rank_at_least(1) - num_components = tensor_shape.dimension_value(bs[-1]) - if num_components is not None: - num_components += 1 - tail = tensor_shape.TensorShape([num_components, qs]) - return bs[:-1].concatenate(tail) - - def _compute_quantiles(): - """Helper to build quantiles.""" - # Omit {0, 1} since they might lead to Inf/NaN. - zero = array_ops.zeros([], dtype=dist.dtype) - edges = math_ops.linspace(zero, 1., quadrature_size + 3)[1:-1] - # Expand edges so its broadcast across batch dims. - edges = array_ops.reshape(edges, shape=array_ops.concat([ - [-1], array_ops.ones([batch_ndims], dtype=dtypes.int32)], axis=0)) - quantiles = dist.quantile(edges) - quantiles = SoftmaxCentered().forward(quantiles) - # Cyclically permute left by one. - perm = array_ops.concat([ - math_ops.range(1, 1 + batch_ndims), [0]], axis=0) - quantiles = array_ops.transpose(quantiles, perm) - quantiles.set_shape(_get_final_shape(quadrature_size + 1)) - return quantiles - quantiles = _compute_quantiles() - - # Compute grid as quantile midpoints. - grid = (quantiles[..., :-1] + quantiles[..., 1:]) / 2. - # Set shape hints. - grid.set_shape(_get_final_shape(quadrature_size)) - - # By construction probs is constant, i.e., `1 / quadrature_size`. This is - # important, because non-constant probs leads to non-reparameterizable - # samples. - probs = array_ops.fill( - dims=[quadrature_size], - value=1. / math_ops.cast(quadrature_size, dist.dtype)) - - return grid, probs - - -class VectorDiffeomixture(distribution_lib.Distribution): - """VectorDiffeomixture distribution. - - A vector diffeomixture (VDM) is a distribution parameterized by a convex - combination of `K` component `loc` vectors, `loc[k], k = 0,...,K-1`, and `K` - `scale` matrices `scale[k], k = 0,..., K-1`. It approximates the following - [compound distribution] - (https://en.wikipedia.org/wiki/Compound_probability_distribution) - - ```none - p(x) = int p(x | z) p(z) dz, - where z is in the K-simplex, and - p(x | z) := p(x | loc=sum_k z[k] loc[k], scale=sum_k z[k] scale[k]) - ``` - - The integral `int p(x | z) p(z) dz` is approximated with a quadrature scheme - adapted to the mixture density `p(z)`. The `N` quadrature points `z_{N, n}` - and weights `w_{N, n}` (which are non-negative and sum to 1) are chosen - such that - - ```q_N(x) := sum_{n=1}^N w_{n, N} p(x | z_{N, n}) --> p(x)``` - - as `N --> infinity`. - - Since `q_N(x)` is in fact a mixture (of `N` points), we may sample from - `q_N` exactly. It is important to note that the VDM is *defined* as `q_N` - above, and *not* `p(x)`. Therefore, sampling and pdf may be implemented as - exact (up to floating point error) methods. - - A common choice for the conditional `p(x | z)` is a multivariate Normal. - - The implemented marginal `p(z)` is the `SoftmaxNormal`, which is a - `K-1` dimensional Normal transformed by a `SoftmaxCentered` bijector, making - it a density on the `K`-simplex. That is, - - ``` - Z = SoftmaxCentered(X), - X = Normal(mix_loc / temperature, 1 / temperature) - ``` - - The default quadrature scheme chooses `z_{N, n}` as `N` midpoints of - the quantiles of `p(z)` (generalized quantiles if `K > 2`). - - See [Dillon and Langmore (2018)][1] for more details. - - #### About `Vector` distributions in TensorFlow. - - The `VectorDiffeomixture` is a non-standard distribution that has properties - particularly useful in [variational Bayesian - methods](https://en.wikipedia.org/wiki/Variational_Bayesian_methods). - - Conditioned on a draw from the SoftmaxNormal, `X|z` is a vector whose - components are linear combinations of affine transformations, thus is itself - an affine transformation. - - Note: The marginals `X_1|v, ..., X_d|v` are *not* generally identical to some - parameterization of `distribution`. This is due to the fact that the sum of - draws from `distribution` are not generally itself the same `distribution`. - - #### About `Diffeomixture`s and reparameterization. - - The `VectorDiffeomixture` is designed to be reparameterized, i.e., its - parameters are only used to transform samples from a distribution which has no - trainable parameters. This property is important because backprop stops at - sources of stochasticity. That is, as long as the parameters are used *after* - the underlying source of stochasticity, the computed gradient is accurate. - - Reparametrization means that we can use gradient-descent (via backprop) to - optimize Monte-Carlo objectives. Such objectives are a finite-sample - approximation of an expectation and arise throughout scientific computing. - - WARNING: If you backprop through a VectorDiffeomixture sample and the "base" - distribution is both: not `FULLY_REPARAMETERIZED` and a function of trainable - variables, then the gradient is not guaranteed correct! - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Create two batches of VectorDiffeomixtures, one with mix_loc=[0.], - # another with mix_loc=[1]. In both cases, `K=2` and the affine - # transformations involve: - # k=0: loc=zeros(dims) scale=LinearOperatorScaledIdentity - # k=1: loc=[2.]*dims scale=LinOpDiag - dims = 5 - vdm = tfd.VectorDiffeomixture( - mix_loc=[[0.], [1]], - temperature=[1.], - distribution=tfd.Normal(loc=0., scale=1.), - loc=[ - None, # Equivalent to `np.zeros(dims, dtype=np.float32)`. - np.float32([2.]*dims), - ], - scale=[ - tf.linalg.LinearOperatorScaledIdentity( - num_rows=dims, - multiplier=np.float32(1.1), - is_positive_definite=True), - tf.linalg.LinearOperatorDiag( - diag=np.linspace(2.5, 3.5, dims, dtype=np.float32), - is_positive_definite=True), - ], - validate_args=True) - ``` - - #### References - - [1]: Joshua Dillon and Ian Langmore. Quadrature Compound: An approximating - family of distributions. _arXiv preprint arXiv:1801.03080_, 2018. - https://arxiv.org/abs/1801.03080 - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - mix_loc, - temperature, - distribution, - loc=None, - scale=None, - quadrature_size=8, - quadrature_fn=quadrature_scheme_softmaxnormal_quantiles, - validate_args=False, - allow_nan_stats=True, - name="VectorDiffeomixture"): - """Constructs the VectorDiffeomixture on `R^d`. - - The vector diffeomixture (VDM) approximates the compound distribution - - ```none - p(x) = int p(x | z) p(z) dz, - where z is in the K-simplex, and - p(x | z) := p(x | loc=sum_k z[k] loc[k], scale=sum_k z[k] scale[k]) - ``` - - Args: - mix_loc: `float`-like `Tensor` with shape `[b1, ..., bB, K-1]`. - In terms of samples, larger `mix_loc[..., k]` ==> - `Z` is more likely to put more weight on its `kth` component. - temperature: `float`-like `Tensor`. Broadcastable with `mix_loc`. - In terms of samples, smaller `temperature` means one component is more - likely to dominate. I.e., smaller `temperature` makes the VDM look more - like a standard mixture of `K` components. - distribution: `tf.Distribution`-like instance. Distribution from which `d` - iid samples are used as input to the selected affine transformation. - Must be a scalar-batch, scalar-event distribution. Typically - `distribution.reparameterization_type = FULLY_REPARAMETERIZED` or it is - a function of non-trainable parameters. WARNING: If you backprop through - a VectorDiffeomixture sample and the `distribution` is not - `FULLY_REPARAMETERIZED` yet is a function of trainable variables, then - the gradient will be incorrect! - loc: Length-`K` list of `float`-type `Tensor`s. The `k`-th element - represents the `shift` used for the `k`-th affine transformation. If - the `k`-th item is `None`, `loc` is implicitly `0`. When specified, - must have shape `[B1, ..., Bb, d]` where `b >= 0` and `d` is the event - size. - scale: Length-`K` list of `LinearOperator`s. Each should be - positive-definite and operate on a `d`-dimensional vector space. The - `k`-th element represents the `scale` used for the `k`-th affine - transformation. `LinearOperator`s must have shape `[B1, ..., Bb, d, d]`, - `b >= 0`, i.e., characterizes `b`-batches of `d x d` matrices - quadrature_size: Python `int` scalar representing number of - quadrature points. Larger `quadrature_size` means `q_N(x)` better - approximates `p(x)`. - quadrature_fn: Python callable taking `normal_loc`, `normal_scale`, - `quadrature_size`, `validate_args` and returning `tuple(grid, probs)` - representing the SoftmaxNormal grid and corresponding normalized weight. - normalized) weight. - Default value: `quadrature_scheme_softmaxnormal_quantiles`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, - statistics (e.g., mean, mode, variance) use the value "`NaN`" to - indicate the result is undefined. When `False`, an exception is raised - if one or more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - ValueError: if `not scale or len(scale) < 2`. - ValueError: if `len(loc) != len(scale)` - ValueError: if `quadrature_grid_and_probs is not None` and - `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])` - ValueError: if `validate_args` and any not scale.is_positive_definite. - TypeError: if any scale.dtype != scale[0].dtype. - TypeError: if any loc.dtype != scale[0].dtype. - NotImplementedError: if `len(scale) != 2`. - ValueError: if `not distribution.is_scalar_batch`. - ValueError: if `not distribution.is_scalar_event`. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[mix_loc, temperature]) as name: - if not scale or len(scale) < 2: - raise ValueError("Must specify list (or list-like object) of scale " - "LinearOperators, one for each component with " - "num_component >= 2.") - - if loc is None: - loc = [None]*len(scale) - - if len(loc) != len(scale): - raise ValueError("loc/scale must be same-length lists " - "(or same-length list-like objects).") - - dtype = scale[0].dtype.base_dtype - - loc = [ops.convert_to_tensor(loc_, dtype=dtype, name="loc{}".format(k)) - if loc_ is not None else None - for k, loc_ in enumerate(loc)] - - for k, scale_ in enumerate(scale): - if validate_args and not scale_.is_positive_definite: - raise ValueError("scale[{}].is_positive_definite = {} != True".format( - k, scale_.is_positive_definite)) - if scale_.dtype.base_dtype != dtype: - raise TypeError( - "dtype mismatch; scale[{}].base_dtype=\"{}\" != \"{}\"".format( - k, scale_.dtype.base_dtype.name, dtype.name)) - - self._endpoint_affine = [ - AffineLinearOperator(shift=loc_, - scale=scale_, - validate_args=validate_args, - name="endpoint_affine_{}".format(k)) - for k, (loc_, scale_) in enumerate(zip(loc, scale))] - - # TODO(jvdillon): Remove once we support k-mixtures. - # We make this assertion here because otherwise `grid` would need to be a - # vector not a scalar. - if len(scale) != 2: - raise NotImplementedError("Currently only bimixtures are supported; " - "len(scale)={} is not 2.".format(len(scale))) - - mix_loc = ops.convert_to_tensor( - mix_loc, dtype=dtype, name="mix_loc") - temperature = ops.convert_to_tensor( - temperature, dtype=dtype, name="temperature") - self._grid, probs = tuple(quadrature_fn( - mix_loc / temperature, - 1. / temperature, - quadrature_size, - validate_args)) - - # Note: by creating the logits as `log(prob)` we ensure that - # `self.mixture_distribution.logits` is equivalent to - # `math_ops.log(self.mixture_distribution.probs)`. - self._mixture_distribution = categorical_lib.Categorical( - logits=math_ops.log(probs), - validate_args=validate_args, - allow_nan_stats=allow_nan_stats) - - asserts = distribution_util.maybe_check_scalar_distribution( - distribution, dtype, validate_args) - if asserts: - self._grid = control_flow_ops.with_dependencies( - asserts, self._grid) - self._distribution = distribution - - self._interpolated_affine = [ - AffineLinearOperator(shift=loc_, - scale=scale_, - validate_args=validate_args, - name="interpolated_affine_{}".format(k)) - for k, (loc_, scale_) in enumerate(zip( - interpolate_loc(self._grid, loc), - interpolate_scale(self._grid, scale)))] - - [ - self._batch_shape_, - self._batch_shape_tensor_, - self._event_shape_, - self._event_shape_tensor_, - ] = determine_batch_event_shapes(self._grid, - self._endpoint_affine) - - super(VectorDiffeomixture, self).__init__( - dtype=dtype, - # We hard-code `FULLY_REPARAMETERIZED` because when - # `validate_args=True` we verify that indeed - # `distribution.reparameterization_type == FULLY_REPARAMETERIZED`. A - # distribution which is a function of only non-trainable parameters - # also implies we can use `FULLY_REPARAMETERIZED`. However, we cannot - # easily test for that possibility thus we use `validate_args=False` - # as a "back-door" to allow users a way to use non - # `FULLY_REPARAMETERIZED` distribution. In such cases IT IS THE USERS - # RESPONSIBILITY to verify that the base distribution is a function of - # non-trainable parameters. - reparameterization_type=distribution_lib.FULLY_REPARAMETERIZED, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - parameters=parameters, - graph_parents=( - distribution._graph_parents # pylint: disable=protected-access - + [loc_ for loc_ in loc if loc_ is not None]), - name=name) - - @property - def mixture_distribution(self): - """Distribution used to select a convex combination of affine transforms.""" - return self._mixture_distribution - - @property - def distribution(self): - """Base scalar-event, scalar-batch distribution.""" - return self._distribution - - @property - def grid(self): - """Grid of mixing probabilities, one for each grid point.""" - return self._grid - - @property - def endpoint_affine(self): - """Affine transformation for each of `K` components.""" - return self._endpoint_affine - - @property - def interpolated_affine(self): - """Affine transformation for each convex combination of `K` components.""" - return self._interpolated_affine - - def _batch_shape_tensor(self): - return self._batch_shape_tensor_ - - def _batch_shape(self): - return self._batch_shape_ - - def _event_shape_tensor(self): - return self._event_shape_tensor_ - - def _event_shape(self): - return self._event_shape_ - - def _sample_n(self, n, seed=None): - x = self.distribution.sample( - sample_shape=concat_vectors( - [n], - self.batch_shape_tensor(), - self.event_shape_tensor()), - seed=seed) # shape: [n, B, e] - x = [aff.forward(x) for aff in self.endpoint_affine] - - # Get ids as a [n, batch_size]-shaped matrix, unless batch_shape=[] then get - # ids as a [n]-shaped vector. - batch_size = self.batch_shape.num_elements() - if batch_size is None: - batch_size = array_ops.reduce_prod(self.batch_shape_tensor()) - mix_batch_size = self.mixture_distribution.batch_shape.num_elements() - if mix_batch_size is None: - mix_batch_size = math_ops.reduce_prod( - self.mixture_distribution.batch_shape_tensor()) - ids = self.mixture_distribution.sample( - sample_shape=concat_vectors( - [n], - distribution_util.pick_vector( - self.is_scalar_batch(), - np.int32([]), - [batch_size // mix_batch_size])), - seed=distribution_util.gen_new_seed( - seed, "vector_diffeomixture")) - # We need to flatten batch dims in case mixture_distribution has its own - # batch dims. - ids = array_ops.reshape(ids, shape=concat_vectors( - [n], - distribution_util.pick_vector( - self.is_scalar_batch(), - np.int32([]), - np.int32([-1])))) - - # Stride `components * quadrature_size` for `batch_size` number of times. - stride = self.grid.shape.with_rank_at_least( - 2)[-2:].num_elements() - if stride is None: - stride = array_ops.reduce_prod( - array_ops.shape(self.grid)[-2:]) - offset = math_ops.range(start=0, - limit=batch_size * stride, - delta=stride, - dtype=ids.dtype) - - weight = array_ops.gather( - array_ops.reshape(self.grid, shape=[-1]), - ids + offset) - # At this point, weight flattened all batch dims into one. - # We also need to append a singleton to broadcast with event dims. - if self.batch_shape.is_fully_defined(): - new_shape = [-1] + self.batch_shape.as_list() + [1] - else: - new_shape = array_ops.concat( - ([-1], self.batch_shape_tensor(), [1]), axis=0) - weight = array_ops.reshape(weight, shape=new_shape) - - if len(x) != 2: - # We actually should have already triggered this exception. However as a - # policy we're putting this exception wherever we exploit the bimixture - # assumption. - raise NotImplementedError("Currently only bimixtures are supported; " - "len(scale)={} is not 2.".format(len(x))) - - # Alternatively: - # x = weight * x[0] + (1. - weight) * x[1] - x = weight * (x[0] - x[1]) + x[1] - - return x - - def _log_prob(self, x): - # By convention, we always put the grid points right-most. - y = array_ops.stack( - [aff.inverse(x) for aff in self.interpolated_affine], - axis=-1) - log_prob = math_ops.reduce_sum(self.distribution.log_prob(y), axis=-2) - # Because the affine transformation has a constant Jacobian, it is the case - # that `affine.fldj(x) = -affine.ildj(x)`. This is not true in general. - fldj = array_ops.stack([ - aff.forward_log_det_jacobian( - x, - event_ndims=array_ops.rank(self.event_shape_tensor()) - ) for aff in self.interpolated_affine], axis=-1) - return math_ops.reduce_logsumexp( - self.mixture_distribution.logits - fldj + log_prob, axis=-1) - - def _mean(self): - p = self._expand_mix_distribution_probs() - m = self._expand_base_distribution_mean() - mean = None - for k, aff in enumerate(self.interpolated_affine): - # aff.forward is going to do this: - # y = array_ops.squeeze(aff.scale.matmul(m), axis=[-1]) - # if aff.shift is not None: - # y += aff.shift - mean = add(mean, p[..., k] * aff.forward(m)) - return mean - - def _covariance(self): - # Law of total variance: - # - # Cov[Z] = E[Cov[Z | V]] + Cov[E[Z | V]] - # - # where, - # - # E[Cov[Z | V]] = sum_i mix_prob[i] Scale[i] - # Cov[E[Z | V]] = sum_i mix_prob[i] osquare(loc[i]) - # - osquare(sum_i mix_prob[i] loc[i]) - # - # osquare(x) = x.transpose @ x - return add( - self._mean_of_covariance_given_quadrature_component(diag_only=False), - self._covariance_of_mean_given_quadrature_component(diag_only=False)) - - def _variance(self): - # Equivalent to: tf.linalg.tensor_diag_part(self._covariance()), - return add( - self._mean_of_covariance_given_quadrature_component(diag_only=True), - self._covariance_of_mean_given_quadrature_component(diag_only=True)) - - def _mean_of_covariance_given_quadrature_component(self, diag_only): - p = self.mixture_distribution.probs - - # To compute E[Cov(Z|V)], we'll add matrices within three categories: - # scaled-identity, diagonal, and full. Then we'll combine these at the end. - scale_identity_multiplier = None - diag = None - full = None - - for k, aff in enumerate(self.interpolated_affine): - s = aff.scale # Just in case aff.scale has side-effects, we'll call once. - if (s is None - or isinstance(s, linop_identity_lib.LinearOperatorIdentity)): - scale_identity_multiplier = add(scale_identity_multiplier, - p[..., k, array_ops.newaxis]) - elif isinstance(s, linop_identity_lib.LinearOperatorScaledIdentity): - scale_identity_multiplier = add( - scale_identity_multiplier, - (p[..., k, array_ops.newaxis] * math_ops.square(s.multiplier))) - elif isinstance(s, linop_diag_lib.LinearOperatorDiag): - diag = add(diag, (p[..., k, array_ops.newaxis] * - math_ops.square(s.diag_part()))) - else: - x = (p[..., k, array_ops.newaxis, array_ops.newaxis] * - s.matmul(s.to_dense(), adjoint_arg=True)) - if diag_only: - x = array_ops.matrix_diag_part(x) - full = add(full, x) - - # We must now account for the fact that the base distribution might have a - # non-unity variance. Recall that, since X ~ iid Law(X_0), - # `Cov(SX+m) = S Cov(X) S.T = S S.T Diag(Var(X_0))`. - # We can scale by `Var(X)` (vs `Cov(X)`) since X corresponds to `d` iid - # samples from a scalar-event distribution. - v = self.distribution.variance() - if scale_identity_multiplier is not None: - scale_identity_multiplier *= v - if diag is not None: - diag *= v[..., array_ops.newaxis] - if full is not None: - full *= v[..., array_ops.newaxis] - - if diag_only: - # Apparently we don't need the full matrix, just the diagonal. - r = add(diag, full) - if r is None and scale_identity_multiplier is not None: - ones = array_ops.ones(self.event_shape_tensor(), dtype=self.dtype) - return scale_identity_multiplier[..., array_ops.newaxis] * ones - return add(r, scale_identity_multiplier) - - # `None` indicates we don't know if the result is positive-definite. - is_positive_definite = (True if all(aff.scale.is_positive_definite - for aff in self.endpoint_affine) - else None) - - to_add = [] - if diag is not None: - to_add.append(linop_diag_lib.LinearOperatorDiag( - diag=diag, - is_positive_definite=is_positive_definite)) - if full is not None: - to_add.append(linop_full_lib.LinearOperatorFullMatrix( - matrix=full, - is_positive_definite=is_positive_definite)) - if scale_identity_multiplier is not None: - to_add.append(linop_identity_lib.LinearOperatorScaledIdentity( - num_rows=self.event_shape_tensor()[0], - multiplier=scale_identity_multiplier, - is_positive_definite=is_positive_definite)) - - return (linop_add_lib.add_operators(to_add)[0].to_dense() - if to_add else None) - - def _covariance_of_mean_given_quadrature_component(self, diag_only): - square = math_ops.square if diag_only else vec_osquare - - p = self._expand_mix_distribution_probs() - if not diag_only: - p = p[..., array_ops.newaxis, :] # Assuming event.ndims=1. - m = self._expand_base_distribution_mean() - - cov_e_z_given_v = None - e_z_given_v = self._mean() - for k, aff in enumerate(self.interpolated_affine): - y = aff.forward(m) - cov_e_z_given_v = add(cov_e_z_given_v, - p[..., k] * square(y - e_z_given_v)) - - return cov_e_z_given_v - - def _expand_base_distribution_mean(self): - """Ensures `self.distribution.mean()` has `[batch, event]` shape.""" - single_draw_shape = concat_vectors(self.batch_shape_tensor(), - self.event_shape_tensor()) - m = array_ops.reshape( - self.distribution.mean(), # A scalar. - shape=array_ops.ones_like(single_draw_shape, - dtype=dtypes.int32)) - m = array_ops.tile(m, multiples=single_draw_shape) - m.set_shape(self.batch_shape.concatenate(self.event_shape)) - return m - - def _expand_mix_distribution_probs(self): - p = self.mixture_distribution.probs # [B, deg] - deg = tensor_shape.dimension_value(p.shape.with_rank_at_least(1)[-1]) - if deg is None: - deg = array_ops.shape(p)[-1] - event_ndims = self.event_shape.ndims - if event_ndims is None: - event_ndims = array_ops.shape(self.event_shape_tensor())[0] - expand_shape = array_ops.concat([ - self.mixture_distribution.batch_shape_tensor(), - array_ops.ones([event_ndims], dtype=dtypes.int32), - [deg], - ], axis=0) - return array_ops.reshape(p, shape=expand_shape) - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def maybe_check_quadrature_param(param, name, validate_args): - """Helper which checks validity of `loc` and `scale` init args.""" - with ops.name_scope(name="check_" + name, values=[param]): - assertions = [] - if param.shape.ndims is not None: - if param.shape.ndims == 0: - raise ValueError("Mixing params must be a (batch of) vector; " - "{}.rank={} is not at least one.".format( - name, param.shape.ndims)) - elif validate_args: - assertions.append(check_ops.assert_rank_at_least( - param, 1, - message=("Mixing params must be a (batch of) vector; " - "{}.rank is not at least one.".format( - name)))) - - # TODO(jvdillon): Remove once we support k-mixtures. - if param.shape.with_rank_at_least(1)[-1] is not None: - if tensor_shape.dimension_value(param.shape[-1]) != 1: - raise NotImplementedError("Currently only bimixtures are supported; " - "{}.shape[-1]={} is not 1.".format( - name, - tensor_shape.dimension_value( - param.shape[-1]))) - elif validate_args: - assertions.append(check_ops.assert_equal( - array_ops.shape(param)[-1], 1, - message=("Currently only bimixtures are supported; " - "{}.shape[-1] is not 1.".format(name)))) - - if assertions: - return control_flow_ops.with_dependencies(assertions, param) - return param - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def determine_batch_event_shapes(grid, endpoint_affine): - """Helper to infer batch_shape and event_shape.""" - with ops.name_scope(name="determine_batch_event_shapes"): - # grid # shape: [B, k, q] - # endpoint_affine # len=k, shape: [B, d, d] - batch_shape = grid.shape[:-2] - batch_shape_tensor = array_ops.shape(grid)[:-2] - event_shape = None - event_shape_tensor = None - - def _set_event_shape(shape, shape_tensor): - if event_shape is None: - return shape, shape_tensor - return (array_ops.broadcast_static_shape(event_shape, shape), - array_ops.broadcast_dynamic_shape( - event_shape_tensor, shape_tensor)) - - for aff in endpoint_affine: - if aff.shift is not None: - batch_shape = array_ops.broadcast_static_shape( - batch_shape, aff.shift.shape[:-1]) - batch_shape_tensor = array_ops.broadcast_dynamic_shape( - batch_shape_tensor, array_ops.shape(aff.shift)[:-1]) - event_shape, event_shape_tensor = _set_event_shape( - aff.shift.shape[-1:], array_ops.shape(aff.shift)[-1:]) - - if aff.scale is not None: - batch_shape = array_ops.broadcast_static_shape( - batch_shape, aff.scale.batch_shape) - batch_shape_tensor = array_ops.broadcast_dynamic_shape( - batch_shape_tensor, aff.scale.batch_shape_tensor()) - event_shape, event_shape_tensor = _set_event_shape( - tensor_shape.TensorShape([aff.scale.range_dimension]), - aff.scale.range_dimension_tensor()[array_ops.newaxis]) - - return batch_shape, batch_shape_tensor, event_shape, event_shape_tensor - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def interpolate_loc(grid, loc): - """Helper which interpolates between two locs.""" - if len(loc) != 2: - raise NotImplementedError("Currently only bimixtures are supported; " - "len(scale)={} is not 2.".format(len(loc))) - deg = tensor_shape.dimension_value(grid.shape.with_rank_at_least(1)[-1]) - if deg is None: - raise ValueError("Num quadrature grid points must be known prior " - "to graph execution.") - with ops.name_scope("interpolate_loc", values=[grid, loc]): - if loc is None or loc[0] is None and loc[1] is None: - return [None]*deg - # shape: [B, 1, k, deg] - w = grid[..., array_ops.newaxis, :, :] - loc = [x[..., array_ops.newaxis] # shape: [B, e, 1] - if x is not None else None for x in loc] - if loc[0] is None: - x = w[..., 1, :] * loc[1] # shape: [B, e, deg] - elif loc[1] is None: - x = w[..., 0, :] * loc[0] # shape: [B, e, deg] - else: - delta = loc[0] - loc[1] - x = w[..., 0, :] * delta + loc[1] # shape: [B, e, deg] - return [x[..., k] for k in range(deg)] # list(shape:[B, e]) - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def interpolate_scale(grid, scale): - """Helper which interpolates between two scales.""" - if len(scale) != 2: - raise NotImplementedError("Currently only bimixtures are supported; " - "len(scale)={} is not 2.".format(len(scale))) - deg = tensor_shape.dimension_value(grid.shape.with_rank_at_least(1)[-1]) - if deg is None: - raise ValueError("Num quadrature grid points must be known prior " - "to graph execution.") - with ops.name_scope("interpolate_scale", values=[grid]): - return [linop_add_lib.add_operators([ - linop_scale(grid[..., k, q], s) - for k, s in enumerate(scale) - ])[0] for q in range(deg)] - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def linop_scale(w, op): - # We assume w > 0. (This assumption only relates to the is_* attributes.) - with ops.name_scope("linop_scale", values=[w]): - # TODO(b/35301104): LinearOperatorComposition doesn't combine operators, so - # special case combinations here. Once it does, this function can be - # replaced by: - # return linop_composition_lib.LinearOperatorComposition([ - # scaled_identity(w), op]) - def scaled_identity(w): - return linop_identity_lib.LinearOperatorScaledIdentity( - num_rows=op.range_dimension_tensor(), - multiplier=w, - is_non_singular=op.is_non_singular, - is_self_adjoint=op.is_self_adjoint, - is_positive_definite=op.is_positive_definite) - if isinstance(op, linop_identity_lib.LinearOperatorIdentity): - return scaled_identity(w) - if isinstance(op, linop_identity_lib.LinearOperatorScaledIdentity): - return scaled_identity(w * op.multiplier) - if isinstance(op, linop_diag_lib.LinearOperatorDiag): - return linop_diag_lib.LinearOperatorDiag( - diag=w[..., array_ops.newaxis] * op.diag_part(), - is_non_singular=op.is_non_singular, - is_self_adjoint=op.is_self_adjoint, - is_positive_definite=op.is_positive_definite) - if isinstance(op, linop_tril_lib.LinearOperatorLowerTriangular): - return linop_tril_lib.LinearOperatorLowerTriangular( - tril=w[..., array_ops.newaxis, array_ops.newaxis] * op.to_dense(), - is_non_singular=op.is_non_singular, - is_self_adjoint=op.is_self_adjoint, - is_positive_definite=op.is_positive_definite) - raise NotImplementedError( - "Unsupported Linop type ({})".format(type(op).__name__)) - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def concat_vectors(*args): - """Concatenates input vectors, statically if possible.""" - args_ = [distribution_util.static_value(x) for x in args] - if any(vec is None for vec in args_): - return array_ops.concat(args, axis=0) - return [val for vec in args_ for val in vec] - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def add(x, y): - """Adds inputs; interprets `None` as zero.""" - if x is None: - return y - if y is None: - return x - return x + y - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def vec_osquare(x): - """Computes the outer-product of a (batch of) vector, i.e., x.T x.""" - return x[..., :, array_ops.newaxis] * x[..., array_ops.newaxis, :] - - -@deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) -def softmax(x, axis, name=None): - """Equivalent to tf.nn.softmax but works around b/70297725.""" - with ops.name_scope(name, "softmax", [x, axis]): - x = ops.convert_to_tensor(x, name="x") - ndims = (x.shape.ndims if x.shape.ndims is not None - else array_ops.rank(x, name="ndims")) - axis = ops.convert_to_tensor(axis, dtype=dtypes.int32, name="axis") - axis_ = tensor_util.constant_value(axis) - if axis_ is not None: - axis = np.int(ndims + axis_ if axis_ < 0 else axis_) - else: - axis = array_ops.where_v2(axis < 0, ndims + axis, axis) - return nn_ops.softmax(x, axis=axis) diff --git a/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py b/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py deleted file mode 100644 index 36cbd71f8b3..00000000000 --- a/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Distribution of a vectorized Exponential, with uncorrelated components.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.contrib.distributions.python.ops import vector_exponential_linear_operator as vector_exponential_linop -from tensorflow.python.framework import ops -from tensorflow.python.util import deprecation - - -__all__ = [ - "VectorExponentialDiag", -] - - -class VectorExponentialDiag( - vector_exponential_linop.VectorExponentialLinearOperator): - """The vectorization of the Exponential distribution on `R^k`. - - The vector exponential distribution is defined over a subset of `R^k`, and - parameterized by a (batch of) length-`k` `loc` vector and a (batch of) `k x k` - `scale` matrix: `covariance = scale @ scale.T`, where `@` denotes - matrix-multiplication. - - #### Mathematical Details - - The probability density function (pdf) is defined over the image of the - `scale` matrix + `loc`, applied to the positive half-space: - `Supp = {loc + scale @ x : x in R^k, x_1 > 0, ..., x_k > 0}`. On this set, - - ```none - pdf(y; loc, scale) = exp(-||x||_1) / Z, for y in Supp - x = inv(scale) @ (y - loc), - Z = |det(scale)|, - ``` - - where: - - * `loc` is a vector in `R^k`, - * `scale` is a linear operator in `R^{k x k}`, `cov = scale @ scale.T`, - * `Z` denotes the normalization constant, and, - * `||x||_1` denotes the `l1` norm of `x`, `sum_i |x_i|`. - - The VectorExponential distribution is a member of the [location-scale - family](https://en.wikipedia.org/wiki/Location-scale_family), i.e., it can be - constructed as, - - ```none - X = (X_1, ..., X_k), each X_i ~ Exponential(rate=1) - Y = (Y_1, ...,Y_k) = scale @ X + loc - ``` - - #### About `VectorExponential` and `Vector` distributions in TensorFlow. - - The `VectorExponential` is a non-standard distribution that has useful - properties. - - The marginals `Y_1, ..., Y_k` are *not* Exponential random variables, due to - the fact that the sum of Exponential random variables is not Exponential. - - Instead, `Y` is a vector whose components are linear combinations of - Exponential random variables. Thus, `Y` lives in the vector space generated - by `vectors` of Exponential distributions. This allows the user to decide the - mean and covariance (by setting `loc` and `scale`), while preserving some - properties of the Exponential distribution. In particular, the tails of `Y_i` - will be (up to polynomial factors) exponentially decaying. - - To see this last statement, note that the pdf of `Y_i` is the convolution of - the pdf of `k` independent Exponential random variables. One can then show by - induction that distributions with exponential (up to polynomial factors) tails - are closed under convolution. - - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single 2-variate VectorExponential, supported on - # {(x, y) in R^2 : x > 0, y > 0}. - - # The first component has pdf exp{-x}, the second 0.5 exp{-x / 2} - vex = tfd.VectorExponentialDiag(scale_diag=[1., 2.]) - - # Compute the pdf of an`R^2` observation; return a scalar. - vex.prob([3., 4.]).eval() # shape: [] - - # Initialize a 2-batch of 3-variate Vector Exponential's. - loc = [[1., 2, 3], - [1., 0, 0]] # shape: [2, 3] - scale_diag = [[1., 2, 3], - [0.5, 1, 1.5]] # shape: [2, 3] - - vex = tfd.VectorExponentialDiag(loc, scale_diag) - - # Compute the pdf of two `R^3` observations; return a length-2 vector. - x = [[1.9, 2.2, 3.1], - [10., 1.0, 9.0]] # shape: [2, 3] - vex.prob(x).eval() # shape: [2] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc=None, - scale_diag=None, - scale_identity_multiplier=None, - validate_args=False, - allow_nan_stats=True, - name="VectorExponentialDiag"): - """Construct Vector Exponential distribution supported on a subset of `R^k`. - - The `batch_shape` is the broadcast shape between `loc` and `scale` - arguments. - - The `event_shape` is given by last dimension of the matrix implied by - `scale`. The last dimension of `loc` (if provided) must broadcast with this. - - Recall that `covariance = scale @ scale.T`. - - ```none - scale = diag(scale_diag + scale_identity_multiplier * ones(k)) - ``` - - where: - - * `scale_diag.shape = [k]`, and, - * `scale_identity_multiplier.shape = []`. - - Additional leading dimensions (if any) will index batches. - - If both `scale_diag` and `scale_identity_multiplier` are `None`, then - `scale` is the Identity matrix. - - Args: - loc: Floating-point `Tensor`. If this is set to `None`, `loc` is - implicitly `0`. When specified, may have shape `[B1, ..., Bb, k]` where - `b >= 0` and `k` is the event size. - scale_diag: Non-zero, floating-point `Tensor` representing a diagonal - matrix added to `scale`. May have shape `[B1, ..., Bb, k]`, `b >= 0`, - and characterizes `b`-batches of `k x k` diagonal matrices added to - `scale`. When both `scale_identity_multiplier` and `scale_diag` are - `None` then `scale` is the `Identity`. - scale_identity_multiplier: Non-zero, floating-point `Tensor` representing - a scaled-identity-matrix added to `scale`. May have shape - `[B1, ..., Bb]`, `b >= 0`, and characterizes `b`-batches of scaled - `k x k` identity matrices added to `scale`. When both - `scale_identity_multiplier` and `scale_diag` are `None` then `scale` is - the `Identity`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, - statistics (e.g., mean, mode, variance) use the value "`NaN`" to - indicate the result is undefined. When `False`, an exception is raised - if one or more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - ValueError: if at most `scale_identity_multiplier` is specified. - """ - parameters = dict(locals()) - with ops.name_scope(name) as name: - with ops.name_scope("init", values=[ - loc, scale_diag, scale_identity_multiplier]): - # No need to validate_args while making diag_scale. The returned - # LinearOperatorDiag has an assert_non_singular method that is called by - # the Bijector. - scale = distribution_util.make_diag_scale( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - validate_args=False, - assert_positive=False) - super(VectorExponentialDiag, self).__init__( - loc=loc, - scale=scale, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - self._parameters = parameters diff --git a/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py b/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py deleted file mode 100644 index 9dcd60dab5a..00000000000 --- a/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py +++ /dev/null @@ -1,297 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Vectorized Exponential distribution class, directly using LinearOperator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import exponential -from tensorflow.python.ops.distributions import transformed_distribution -from tensorflow.python.ops.linalg import linalg -from tensorflow.python.util import deprecation - -__all__ = ["VectorExponentialLinearOperator"] - -_mvn_sample_note = """ -`value` is a batch vector with compatible shape if `value` is a `Tensor` whose -shape can be broadcast up to either: - -```python -self.batch_shape + self.event_shape -``` - -or - -```python -[M1, ..., Mm] + self.batch_shape + self.event_shape -``` - -""" - - -class VectorExponentialLinearOperator( - transformed_distribution.TransformedDistribution): - """The vectorization of the Exponential distribution on `R^k`. - - The vector exponential distribution is defined over a subset of `R^k`, and - parameterized by a (batch of) length-`k` `loc` vector and a (batch of) `k x k` - `scale` matrix: `covariance = scale @ scale.T`, where `@` denotes - matrix-multiplication. - - #### Mathematical Details - - The probability density function (pdf) is - - ```none - pdf(y; loc, scale) = exp(-||x||_1) / Z, for y in S(loc, scale), - x = inv(scale) @ (y - loc), - Z = |det(scale)|, - ``` - - where: - - * `loc` is a vector in `R^k`, - * `scale` is a linear operator in `R^{k x k}`, `cov = scale @ scale.T`, - * `S = {loc + scale @ x : x in R^k, x_1 > 0, ..., x_k > 0}`, is an image of - the positive half-space, - * `||x||_1` denotes the `l1` norm of `x`, `sum_i |x_i|`, - * `Z` denotes the normalization constant. - - The VectorExponential distribution is a member of the [location-scale - family](https://en.wikipedia.org/wiki/Location-scale_family), i.e., it can be - constructed as, - - ```none - X = (X_1, ..., X_k), each X_i ~ Exponential(rate=1) - Y = (Y_1, ...,Y_k) = scale @ X + loc - ``` - - #### About `VectorExponential` and `Vector` distributions in TensorFlow. - - The `VectorExponential` is a non-standard distribution that has useful - properties. - - The marginals `Y_1, ..., Y_k` are *not* Exponential random variables, due to - the fact that the sum of Exponential random variables is not Exponential. - - Instead, `Y` is a vector whose components are linear combinations of - Exponential random variables. Thus, `Y` lives in the vector space generated - by `vectors` of Exponential distributions. This allows the user to decide the - mean and covariance (by setting `loc` and `scale`), while preserving some - properties of the Exponential distribution. In particular, the tails of `Y_i` - will be (up to polynomial factors) exponentially decaying. - - To see this last statement, note that the pdf of `Y_i` is the convolution of - the pdf of `k` independent Exponential random variables. One can then show by - induction that distributions with exponential (up to polynomial factors) tails - are closed under convolution. - - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single 2-variate VectorExponential, supported on - # {(x, y) in R^2 : x > 0, y > 0}. - mat = [[1.0, 0.1], - [0.1, 1.0]] - - vex = tfd.VectorExponentialLinearOperator( - scale=tf.linalg.LinearOperatorFullMatrix(mat)) - - # Compute the pdf of an`R^2` observation; return a scalar. - vex.prob([1., 2.]).eval() # shape: [] - - # Initialize a 2-batch of 3-variate Vector Exponential's. - mu = [[1., 2, 3], - [1., 0, 0]] # shape: [2, 3] - scale_diag = [[1., 2, 3], - [0.5, 1, 1.5]] # shape: [2, 3] - - vex = tfd.VectorExponentialLinearOperator( - loc=mu, - scale=tf.linalg.LinearOperatorDiag(scale_diag)) - - # Compute the pdf of two `R^3` observations; return a length-2 vector. - x = [[1.9, 2.2, 3.1], - [10., 1.0, 9.0]] # shape: [2, 3] - vex.prob(x).eval() # shape: [2] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc=None, - scale=None, - validate_args=False, - allow_nan_stats=True, - name="VectorExponentialLinearOperator"): - """Construct Vector Exponential distribution supported on a subset of `R^k`. - - The `batch_shape` is the broadcast shape between `loc` and `scale` - arguments. - - The `event_shape` is given by last dimension of the matrix implied by - `scale`. The last dimension of `loc` (if provided) must broadcast with this. - - Recall that `covariance = scale @ scale.T`. - - Additional leading dimensions (if any) will index batches. - - Args: - loc: Floating-point `Tensor`. If this is set to `None`, `loc` is - implicitly `0`. When specified, may have shape `[B1, ..., Bb, k]` where - `b >= 0` and `k` is the event size. - scale: Instance of `LinearOperator` with same `dtype` as `loc` and shape - `[B1, ..., Bb, k, k]`. - validate_args: Python `bool`, default `False`. Whether to validate input - with asserts. If `validate_args` is `False`, and the inputs are - invalid, correct behavior is not guaranteed. - allow_nan_stats: Python `bool`, default `True`. If `False`, raise an - exception if a statistic (e.g. mean/mode/etc...) is undefined for any - batch member If `True`, batch members with valid parameters leading to - undefined statistics will return NaN for this statistic. - name: The name to give Ops created by the initializer. - - Raises: - ValueError: if `scale` is unspecified. - TypeError: if not `scale.dtype.is_floating` - """ - parameters = dict(locals()) - if scale is None: - raise ValueError("Missing required `scale` parameter.") - if not scale.dtype.is_floating: - raise TypeError("`scale` parameter must have floating-point dtype.") - - with ops.name_scope(name, values=[loc]) as name: - # Since expand_dims doesn't preserve constant-ness, we obtain the - # non-dynamic value if possible. - loc = ops.convert_to_tensor(loc, name="loc") if loc is not None else loc - batch_shape, event_shape = distribution_util.shapes_from_loc_and_scale( - loc, scale) - - super(VectorExponentialLinearOperator, self).__init__( - distribution=exponential.Exponential(rate=array_ops.ones( - [], dtype=scale.dtype), allow_nan_stats=allow_nan_stats), - bijector=bijectors.AffineLinearOperator( - shift=loc, scale=scale, validate_args=validate_args), - batch_shape=batch_shape, - event_shape=event_shape, - validate_args=validate_args, - name=name) - self._parameters = parameters - - @property - def loc(self): - """The `loc` `Tensor` in `Y = scale @ X + loc`.""" - return self.bijector.shift - - @property - def scale(self): - """The `scale` `LinearOperator` in `Y = scale @ X + loc`.""" - return self.bijector.scale - - @distribution_util.AppendDocstring(_mvn_sample_note) - def _log_prob(self, x): - return super(VectorExponentialLinearOperator, self)._log_prob(x) - - @distribution_util.AppendDocstring(_mvn_sample_note) - def _prob(self, x): - return super(VectorExponentialLinearOperator, self)._prob(x) - - def _mean(self): - # Let - # W = (w1,...,wk), with wj ~ iid Exponential(0, 1). - # Then this distribution is - # X = loc + LW, - # and then E[X] = loc + L1, where 1 is the vector of ones. - scale_x_ones = self.bijector.scale.matvec( - array_ops.ones(self._mode_mean_shape(), self.dtype)) - - if self.loc is None: - return scale_x_ones - - return array_ops.identity(self.loc) + scale_x_ones - - def _covariance(self): - # Let - # W = (w1,...,wk), with wj ~ iid Exponential(0, 1). - # Then this distribution is - # X = loc + LW, - # and then since Cov(wi, wj) = 1 if i=j, and 0 otherwise, - # Cov(X) = L Cov(W W^T) L^T = L L^T. - if distribution_util.is_diagonal_scale(self.scale): - return array_ops.matrix_diag(math_ops.square(self.scale.diag_part())) - else: - return self.scale.matmul(self.scale.to_dense(), adjoint_arg=True) - - def _variance(self): - if distribution_util.is_diagonal_scale(self.scale): - return math_ops.square(self.scale.diag_part()) - elif (isinstance(self.scale, linalg.LinearOperatorLowRankUpdate) and - self.scale.is_self_adjoint): - return array_ops.matrix_diag_part( - self.scale.matmul(self.scale.to_dense())) - else: - return array_ops.matrix_diag_part( - self.scale.matmul(self.scale.to_dense(), adjoint_arg=True)) - - def _stddev(self): - if distribution_util.is_diagonal_scale(self.scale): - return math_ops.abs(self.scale.diag_part()) - elif (isinstance(self.scale, linalg.LinearOperatorLowRankUpdate) and - self.scale.is_self_adjoint): - return math_ops.sqrt( - array_ops.matrix_diag_part(self.scale.matmul(self.scale.to_dense()))) - else: - return math_ops.sqrt( - array_ops.matrix_diag_part( - self.scale.matmul(self.scale.to_dense(), adjoint_arg=True))) - - def _mode(self): - scale_x_zeros = self.bijector.scale.matvec( - array_ops.zeros(self._mode_mean_shape(), self.dtype)) - - if self.loc is None: - return scale_x_zeros - - return array_ops.identity(self.loc) + scale_x_zeros - - def _mode_mean_shape(self): - """Shape for the mode/mean Tensors.""" - shape = self.batch_shape.concatenate(self.event_shape) - has_static_shape = shape.is_fully_defined() - if not has_static_shape: - shape = array_ops.concat([ - self.batch_shape_tensor(), - self.event_shape_tensor(), - ], 0) - return shape diff --git a/tensorflow/contrib/distributions/python/ops/vector_laplace_diag.py b/tensorflow/contrib/distributions/python/ops/vector_laplace_diag.py deleted file mode 100644 index 8cd4e128c7a..00000000000 --- a/tensorflow/contrib/distributions/python/ops/vector_laplace_diag.py +++ /dev/null @@ -1,242 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Distribution of a vectorized Laplace, with uncorrelated components.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.contrib.distributions.python.ops import vector_laplace_linear_operator as vector_laplace_linop -from tensorflow.python.framework import ops -from tensorflow.python.util import deprecation - - -__all__ = [ - "VectorLaplaceDiag", -] - - -class VectorLaplaceDiag( - vector_laplace_linop.VectorLaplaceLinearOperator): - """The vectorization of the Laplace distribution on `R^k`. - - The vector laplace distribution is defined over `R^k`, and parameterized by - a (batch of) length-`k` `loc` vector (the means) and a (batch of) `k x k` - `scale` matrix: `covariance = 2 * scale @ scale.T`, where `@` denotes - matrix-multiplication. - - #### Mathematical Details - - The probability density function (pdf) is, - - ```none - pdf(x; loc, scale) = exp(-||y||_1) / Z, - y = inv(scale) @ (x - loc), - Z = 2**k |det(scale)|, - ``` - - where: - - * `loc` is a vector in `R^k`, - * `scale` is a linear operator in `R^{k x k}`, `cov = scale @ scale.T`, - * `Z` denotes the normalization constant, and, - * `||y||_1` denotes the `l1` norm of `y`, `sum_i |y_i|. - - A (non-batch) `scale` matrix is: - - ```none - scale = diag(scale_diag + scale_identity_multiplier * ones(k)) - ``` - - where: - - * `scale_diag.shape = [k]`, and, - * `scale_identity_multiplier.shape = []`. - - Additional leading dimensions (if any) will index batches. - - If both `scale_diag` and `scale_identity_multiplier` are `None`, then - `scale` is the Identity matrix. - - The VectorLaplace distribution is a member of the [location-scale - family](https://en.wikipedia.org/wiki/Location-scale_family), i.e., it can be - constructed as, - - ```none - X = (X_1, ..., X_k), each X_i ~ Laplace(loc=0, scale=1) - Y = (Y_1, ...,Y_k) = scale @ X + loc - ``` - - #### About `VectorLaplace` and `Vector` distributions in TensorFlow. - - The `VectorLaplace` is a non-standard distribution that has useful properties. - - The marginals `Y_1, ..., Y_k` are *not* Laplace random variables, due to - the fact that the sum of Laplace random variables is not Laplace. - - Instead, `Y` is a vector whose components are linear combinations of Laplace - random variables. Thus, `Y` lives in the vector space generated by `vectors` - of Laplace distributions. This allows the user to decide the mean and - covariance (by setting `loc` and `scale`), while preserving some properties of - the Laplace distribution. In particular, the tails of `Y_i` will be (up to - polynomial factors) exponentially decaying. - - To see this last statement, note that the pdf of `Y_i` is the convolution of - the pdf of `k` independent Laplace random variables. One can then show by - induction that distributions with exponential (up to polynomial factors) tails - are closed under convolution. - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single 2-variate VectorLaplace. - vla = tfd.VectorLaplaceDiag( - loc=[1., -1], - scale_diag=[1, 2.]) - - vla.mean().eval() - # ==> [1., -1] - - vla.stddev().eval() - # ==> [1., 2] * sqrt(2) - - # Evaluate this on an observation in `R^2`, returning a scalar. - vla.prob([-1., 0]).eval() # shape: [] - - # Initialize a 3-batch, 2-variate scaled-identity VectorLaplace. - vla = tfd.VectorLaplaceDiag( - loc=[1., -1], - scale_identity_multiplier=[1, 2., 3]) - - vla.mean().eval() # shape: [3, 2] - # ==> [[1., -1] - # [1, -1], - # [1, -1]] - - vla.stddev().eval() # shape: [3, 2] - # ==> sqrt(2) * [[1., 1], - # [2, 2], - # [3, 3]] - - # Evaluate this on an observation in `R^2`, returning a length-3 vector. - vla.prob([-1., 0]).eval() # shape: [3] - - # Initialize a 2-batch of 3-variate VectorLaplace's. - vla = tfd.VectorLaplaceDiag( - loc=[[1., 2, 3], - [11, 22, 33]] # shape: [2, 3] - scale_diag=[[1., 2, 3], - [0.5, 1, 1.5]]) # shape: [2, 3] - - # Evaluate this on a two observations, each in `R^3`, returning a length-2 - # vector. - x = [[-1., 0, 1], - [-11, 0, 11.]] # shape: [2, 3]. - vla.prob(x).eval() # shape: [2] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc=None, - scale_diag=None, - scale_identity_multiplier=None, - validate_args=False, - allow_nan_stats=True, - name="VectorLaplaceDiag"): - """Construct Vector Laplace distribution on `R^k`. - - The `batch_shape` is the broadcast shape between `loc` and `scale` - arguments. - - The `event_shape` is given by last dimension of the matrix implied by - `scale`. The last dimension of `loc` (if provided) must broadcast with this. - - Recall that `covariance = 2 * scale @ scale.T`. - - ```none - scale = diag(scale_diag + scale_identity_multiplier * ones(k)) - ``` - - where: - - * `scale_diag.shape = [k]`, and, - * `scale_identity_multiplier.shape = []`. - - Additional leading dimensions (if any) will index batches. - - If both `scale_diag` and `scale_identity_multiplier` are `None`, then - `scale` is the Identity matrix. - - Args: - loc: Floating-point `Tensor`. If this is set to `None`, `loc` is - implicitly `0`. When specified, may have shape `[B1, ..., Bb, k]` where - `b >= 0` and `k` is the event size. - scale_diag: Non-zero, floating-point `Tensor` representing a diagonal - matrix added to `scale`. May have shape `[B1, ..., Bb, k]`, `b >= 0`, - and characterizes `b`-batches of `k x k` diagonal matrices added to - `scale`. When both `scale_identity_multiplier` and `scale_diag` are - `None` then `scale` is the `Identity`. - scale_identity_multiplier: Non-zero, floating-point `Tensor` representing - a scaled-identity-matrix added to `scale`. May have shape - `[B1, ..., Bb]`, `b >= 0`, and characterizes `b`-batches of scaled - `k x k` identity matrices added to `scale`. When both - `scale_identity_multiplier` and `scale_diag` are `None` then `scale` is - the `Identity`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, - statistics (e.g., mean, mode, variance) use the value "`NaN`" to - indicate the result is undefined. When `False`, an exception is raised - if one or more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - ValueError: if at most `scale_identity_multiplier` is specified. - """ - parameters = dict(locals()) - with ops.name_scope(name): - with ops.name_scope("init", values=[ - loc, scale_diag, scale_identity_multiplier]): - # No need to validate_args while making diag_scale. The returned - # LinearOperatorDiag has an assert_non_singular method that is called by - # the Bijector. - scale = distribution_util.make_diag_scale( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - validate_args=False, - assert_positive=False) - super(VectorLaplaceDiag, self).__init__( - loc=loc, - scale=scale, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - self._parameters = parameters diff --git a/tensorflow/contrib/distributions/python/ops/vector_laplace_linear_operator.py b/tensorflow/contrib/distributions/python/ops/vector_laplace_linear_operator.py deleted file mode 100644 index 313046db9ba..00000000000 --- a/tensorflow/contrib/distributions/python/ops/vector_laplace_linear_operator.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Vectorized Laplace distribution class, directly using LinearOperator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.distributions import laplace -from tensorflow.python.ops.distributions import transformed_distribution -from tensorflow.python.ops.linalg import linalg -from tensorflow.python.util import deprecation - - -__all__ = [ - "VectorLaplaceLinearOperator" -] - -_mvn_sample_note = """ -`value` is a batch vector with compatible shape if `value` is a `Tensor` whose -shape can be broadcast up to either: - -```python -self.batch_shape + self.event_shape -``` - -or - -```python -[M1, ..., Mm] + self.batch_shape + self.event_shape -``` - -""" - - -class VectorLaplaceLinearOperator( - transformed_distribution.TransformedDistribution): - """The vectorization of the Laplace distribution on `R^k`. - - The vector laplace distribution is defined over `R^k`, and parameterized by - a (batch of) length-`k` `loc` vector (the means) and a (batch of) `k x k` - `scale` matrix: `covariance = 2 * scale @ scale.T`, where `@` denotes - matrix-multiplication. - - #### Mathematical Details - - The probability density function (pdf) is, - - ```none - pdf(x; loc, scale) = exp(-||y||_1) / Z, - y = inv(scale) @ (x - loc), - Z = 2**k |det(scale)|, - ``` - - where: - - * `loc` is a vector in `R^k`, - * `scale` is a linear operator in `R^{k x k}`, `cov = scale @ scale.T`, - * `Z` denotes the normalization constant, and, - * `||y||_1` denotes the `l1` norm of `y`, `sum_i |y_i|. - - The VectorLaplace distribution is a member of the [location-scale - family](https://en.wikipedia.org/wiki/Location-scale_family), i.e., it can be - constructed as, - - ```none - X = (X_1, ..., X_k), each X_i ~ Laplace(loc=0, scale=1) - Y = (Y_1, ...,Y_k) = scale @ X + loc - ``` - - #### About `VectorLaplace` and `Vector` distributions in TensorFlow. - - The `VectorLaplace` is a non-standard distribution that has useful properties. - - The marginals `Y_1, ..., Y_k` are *not* Laplace random variables, due to - the fact that the sum of Laplace random variables is not Laplace. - - Instead, `Y` is a vector whose components are linear combinations of Laplace - random variables. Thus, `Y` lives in the vector space generated by `vectors` - of Laplace distributions. This allows the user to decide the mean and - covariance (by setting `loc` and `scale`), while preserving some properties of - the Laplace distribution. In particular, the tails of `Y_i` will be (up to - polynomial factors) exponentially decaying. - - To see this last statement, note that the pdf of `Y_i` is the convolution of - the pdf of `k` independent Laplace random variables. One can then show by - induction that distributions with exponential (up to polynomial factors) tails - are closed under convolution. - - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single 3-variate VectorLaplace with some desired covariance. - mu = [1., 2, 3] - cov = [[ 0.36, 0.12, 0.06], - [ 0.12, 0.29, -0.13], - [ 0.06, -0.13, 0.26]] - - scale = tf.cholesky(cov) - # ==> [[ 0.6, 0. , 0. ], - # [ 0.2, 0.5, 0. ], - # [ 0.1, -0.3, 0.4]]) - - # Divide scale by sqrt(2) so that the final covariance will be what we want. - vla = tfd.VectorLaplaceLinearOperator( - loc=mu, - scale=tf.linalg.LinearOperatorLowerTriangular(scale / tf.sqrt(2.))) - - # Covariance agrees with cholesky(cov) parameterization. - vla.covariance().eval() - # ==> [[ 0.36, 0.12, 0.06], - # [ 0.12, 0.29, -0.13], - # [ 0.06, -0.13, 0.26]] - - # Compute the pdf of an`R^3` observation; return a scalar. - vla.prob([-1., 0, 1]).eval() # shape: [] - - # Initialize a 2-batch of 3-variate Vector Laplace's. - mu = [[1., 2, 3], - [11, 22, 33]] # shape: [2, 3] - scale_diag = [[1., 2, 3], - [0.5, 1, 1.5]] # shape: [2, 3] - - vla = tfd.VectorLaplaceLinearOperator( - loc=mu, - scale=tf.linalg.LinearOperatorDiag(scale_diag)) - - # Compute the pdf of two `R^3` observations; return a length-2 vector. - x = [[-0.9, 0, 0.1], - [-10, 0, 9]] # shape: [2, 3] - vla.prob(x).eval() # shape: [2] - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc=None, - scale=None, - validate_args=False, - allow_nan_stats=True, - name="VectorLaplaceLinearOperator"): - """Construct Vector Laplace distribution on `R^k`. - - The `batch_shape` is the broadcast shape between `loc` and `scale` - arguments. - - The `event_shape` is given by last dimension of the matrix implied by - `scale`. The last dimension of `loc` (if provided) must broadcast with this. - - Recall that `covariance = 2 * scale @ scale.T`. - - Additional leading dimensions (if any) will index batches. - - Args: - loc: Floating-point `Tensor`. If this is set to `None`, `loc` is - implicitly `0`. When specified, may have shape `[B1, ..., Bb, k]` where - `b >= 0` and `k` is the event size. - scale: Instance of `LinearOperator` with same `dtype` as `loc` and shape - `[B1, ..., Bb, k, k]`. - validate_args: Python `bool`, default `False`. Whether to validate input - with asserts. If `validate_args` is `False`, and the inputs are - invalid, correct behavior is not guaranteed. - allow_nan_stats: Python `bool`, default `True`. If `False`, raise an - exception if a statistic (e.g. mean/mode/etc...) is undefined for any - batch member If `True`, batch members with valid parameters leading to - undefined statistics will return NaN for this statistic. - name: The name to give Ops created by the initializer. - - Raises: - ValueError: if `scale` is unspecified. - TypeError: if not `scale.dtype.is_floating` - """ - parameters = dict(locals()) - if scale is None: - raise ValueError("Missing required `scale` parameter.") - if not scale.dtype.is_floating: - raise TypeError("`scale` parameter must have floating-point dtype.") - - with ops.name_scope(name, values=[loc]): - # Since expand_dims doesn't preserve constant-ness, we obtain the - # non-dynamic value if possible. - loc = ops.convert_to_tensor(loc, name="loc") if loc is not None else loc - batch_shape, event_shape = distribution_util.shapes_from_loc_and_scale( - loc, scale) - - super(VectorLaplaceLinearOperator, self).__init__( - distribution=laplace.Laplace( - loc=array_ops.zeros([], dtype=scale.dtype), - scale=array_ops.ones([], dtype=scale.dtype)), - bijector=bijectors.AffineLinearOperator( - shift=loc, scale=scale, validate_args=validate_args), - batch_shape=batch_shape, - event_shape=event_shape, - validate_args=validate_args, - name=name) - self._parameters = parameters - - @property - def loc(self): - """The `loc` `Tensor` in `Y = scale @ X + loc`.""" - return self.bijector.shift - - @property - def scale(self): - """The `scale` `LinearOperator` in `Y = scale @ X + loc`.""" - return self.bijector.scale - - @distribution_util.AppendDocstring(_mvn_sample_note) - def _log_prob(self, x): - return super(VectorLaplaceLinearOperator, self)._log_prob(x) - - @distribution_util.AppendDocstring(_mvn_sample_note) - def _prob(self, x): - return super(VectorLaplaceLinearOperator, self)._prob(x) - - def _mean(self): - shape = self.batch_shape.concatenate(self.event_shape) - has_static_shape = shape.is_fully_defined() - if not has_static_shape: - shape = array_ops.concat([ - self.batch_shape_tensor(), - self.event_shape_tensor(), - ], 0) - - if self.loc is None: - return array_ops.zeros(shape, self.dtype) - - if has_static_shape and shape == self.loc.get_shape(): - return array_ops.identity(self.loc) - - # Add dummy tensor of zeros to broadcast. This is only necessary if shape - # != self.loc.shape, but we could not determine if this is the case. - return array_ops.identity(self.loc) + array_ops.zeros(shape, self.dtype) - - def _covariance(self): - # Let - # W = (w1,...,wk), with wj ~ iid Laplace(0, 1). - # Then this distribution is - # X = loc + LW, - # and since E[X] = loc, - # Cov(X) = E[LW W^T L^T] = L E[W W^T] L^T. - # Since E[wi wj] = 0 if i != j, and 2 if i == j, we have - # Cov(X) = 2 LL^T - if distribution_util.is_diagonal_scale(self.scale): - return 2. * array_ops.matrix_diag(math_ops.square(self.scale.diag_part())) - else: - return 2. * self.scale.matmul(self.scale.to_dense(), adjoint_arg=True) - - def _variance(self): - if distribution_util.is_diagonal_scale(self.scale): - return 2. * math_ops.square(self.scale.diag_part()) - elif (isinstance(self.scale, linalg.LinearOperatorLowRankUpdate) and - self.scale.is_self_adjoint): - return array_ops.matrix_diag_part( - 2. * self.scale.matmul(self.scale.to_dense())) - else: - return 2. * array_ops.matrix_diag_part( - self.scale.matmul(self.scale.to_dense(), adjoint_arg=True)) - - def _stddev(self): - if distribution_util.is_diagonal_scale(self.scale): - return np.sqrt(2) * math_ops.abs(self.scale.diag_part()) - elif (isinstance(self.scale, linalg.LinearOperatorLowRankUpdate) and - self.scale.is_self_adjoint): - return np.sqrt(2) * math_ops.sqrt(array_ops.matrix_diag_part( - self.scale.matmul(self.scale.to_dense()))) - else: - return np.sqrt(2) * math_ops.sqrt(array_ops.matrix_diag_part( - self.scale.matmul(self.scale.to_dense(), adjoint_arg=True))) - - def _mode(self): - return self._mean() diff --git a/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py b/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py deleted file mode 100644 index da57d0cb55d..00000000000 --- a/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py +++ /dev/null @@ -1,274 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Multi-dimensional (Vector) SinhArcsinh transformation of a distribution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops.distributions import normal -from tensorflow.python.ops.distributions import transformed_distribution -from tensorflow.python.util import deprecation - -__all__ = [ - "VectorSinhArcsinhDiag", -] - - -class VectorSinhArcsinhDiag(transformed_distribution.TransformedDistribution): - """The (diagonal) SinhArcsinh transformation of a distribution on `R^k`. - - This distribution models a random vector `Y = (Y1,...,Yk)`, making use of - a `SinhArcsinh` transformation (which has adjustable tailweight and skew), - a rescaling, and a shift. - - The `SinhArcsinh` transformation of the Normal is described in great depth in - [Sinh-arcsinh distributions](https://www.jstor.org/stable/27798865). - Here we use a slightly different parameterization, in terms of `tailweight` - and `skewness`. Additionally we allow for distributions other than Normal, - and control over `scale` as well as a "shift" parameter `loc`. - - #### Mathematical Details - - Given iid random vector `Z = (Z1,...,Zk)`, we define the VectorSinhArcsinhDiag - transformation of `Z`, `Y`, parameterized by - `(loc, scale, skewness, tailweight)`, via the relation (with `@` denoting - matrix multiplication): - - ``` - Y := loc + scale @ F(Z) * (2 / F_0(2)) - F(Z) := Sinh( (Arcsinh(Z) + skewness) * tailweight ) - F_0(Z) := Sinh( Arcsinh(Z) * tailweight ) - ``` - - This distribution is similar to the location-scale transformation - `L(Z) := loc + scale @ Z` in the following ways: - - * If `skewness = 0` and `tailweight = 1` (the defaults), `F(Z) = Z`, and then - `Y = L(Z)` exactly. - * `loc` is used in both to shift the result by a constant factor. - * The multiplication of `scale` by `2 / F_0(2)` ensures that if `skewness = 0` - `P[Y - loc <= 2 * scale] = P[L(Z) - loc <= 2 * scale]`. - Thus it can be said that the weights in the tails of `Y` and `L(Z)` beyond - `loc + 2 * scale` are the same. - - This distribution is different than `loc + scale @ Z` due to the - reshaping done by `F`: - - * Positive (negative) `skewness` leads to positive (negative) skew. - * positive skew means, the mode of `F(Z)` is "tilted" to the right. - * positive skew means positive values of `F(Z)` become more likely, and - negative values become less likely. - * Larger (smaller) `tailweight` leads to fatter (thinner) tails. - * Fatter tails mean larger values of `|F(Z)|` become more likely. - * `tailweight < 1` leads to a distribution that is "flat" around `Y = loc`, - and a very steep drop-off in the tails. - * `tailweight > 1` leads to a distribution more peaked at the mode with - heavier tails. - - To see the argument about the tails, note that for `|Z| >> 1` and - `|Z| >> (|skewness| * tailweight)**tailweight`, we have - `Y approx 0.5 Z**tailweight e**(sign(Z) skewness * tailweight)`. - - To see the argument regarding multiplying `scale` by `2 / F_0(2)`, - - ``` - P[(Y - loc) / scale <= 2] = P[F(Z) * (2 / F_0(2)) <= 2] - = P[F(Z) <= F_0(2)] - = P[Z <= 2] (if F = F_0). - ``` - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - loc=None, - scale_diag=None, - scale_identity_multiplier=None, - skewness=None, - tailweight=None, - distribution=None, - validate_args=False, - allow_nan_stats=True, - name="MultivariateNormalLinearOperator"): - """Construct VectorSinhArcsinhDiag distribution on `R^k`. - - The arguments `scale_diag` and `scale_identity_multiplier` combine to - define the diagonal `scale` referred to in this class docstring: - - ```none - scale = diag(scale_diag + scale_identity_multiplier * ones(k)) - ``` - - The `batch_shape` is the broadcast shape between `loc` and `scale` - arguments. - - The `event_shape` is given by last dimension of the matrix implied by - `scale`. The last dimension of `loc` (if provided) must broadcast with this - - Additional leading dimensions (if any) will index batches. - - Args: - loc: Floating-point `Tensor`. If this is set to `None`, `loc` is - implicitly `0`. When specified, may have shape `[B1, ..., Bb, k]` where - `b >= 0` and `k` is the event size. - scale_diag: Non-zero, floating-point `Tensor` representing a diagonal - matrix added to `scale`. May have shape `[B1, ..., Bb, k]`, `b >= 0`, - and characterizes `b`-batches of `k x k` diagonal matrices added to - `scale`. When both `scale_identity_multiplier` and `scale_diag` are - `None` then `scale` is the `Identity`. - scale_identity_multiplier: Non-zero, floating-point `Tensor` representing - a scale-identity-matrix added to `scale`. May have shape - `[B1, ..., Bb]`, `b >= 0`, and characterizes `b`-batches of scale - `k x k` identity matrices added to `scale`. When both - `scale_identity_multiplier` and `scale_diag` are `None` then `scale` - is the `Identity`. - skewness: Skewness parameter. floating-point `Tensor` with shape - broadcastable with `event_shape`. - tailweight: Tailweight parameter. floating-point `Tensor` with shape - broadcastable with `event_shape`. - distribution: `tf.Distribution`-like instance. Distribution from which `k` - iid samples are used as input to transformation `F`. Default is - `tfp.distributions.Normal(loc=0., scale=1.)`. - Must be a scalar-batch, scalar-event distribution. Typically - `distribution.reparameterization_type = FULLY_REPARAMETERIZED` or it is - a function of non-trainable parameters. WARNING: If you backprop through - a VectorSinhArcsinhDiag sample and `distribution` is not - `FULLY_REPARAMETERIZED` yet is a function of trainable variables, then - the gradient will be incorrect! - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, - statistics (e.g., mean, mode, variance) use the value "`NaN`" to - indicate the result is undefined. When `False`, an exception is raised - if one or more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - ValueError: if at most `scale_identity_multiplier` is specified. - """ - parameters = dict(locals()) - - with ops.name_scope( - name, - values=[ - loc, scale_diag, scale_identity_multiplier, skewness, tailweight - ]) as name: - loc = ops.convert_to_tensor(loc, name="loc") if loc is not None else loc - tailweight = 1. if tailweight is None else tailweight - has_default_skewness = skewness is None - skewness = 0. if skewness is None else skewness - - # Recall, with Z a random variable, - # Y := loc + C * F(Z), - # F(Z) := Sinh( (Arcsinh(Z) + skewness) * tailweight ) - # F_0(Z) := Sinh( Arcsinh(Z) * tailweight ) - # C := 2 * scale / F_0(2) - - # Construct shapes and 'scale' out of the scale_* and loc kwargs. - # scale_linop is only an intermediary to: - # 1. get shapes from looking at loc and the two scale args. - # 2. combine scale_diag with scale_identity_multiplier, which gives us - # 'scale', which in turn gives us 'C'. - scale_linop = distribution_util.make_diag_scale( - loc=loc, - scale_diag=scale_diag, - scale_identity_multiplier=scale_identity_multiplier, - validate_args=False, - assert_positive=False) - batch_shape, event_shape = distribution_util.shapes_from_loc_and_scale( - loc, scale_linop) - # scale_linop.diag_part() is efficient since it is a diag type linop. - scale_diag_part = scale_linop.diag_part() - dtype = scale_diag_part.dtype - - if distribution is None: - distribution = normal.Normal( - loc=array_ops.zeros([], dtype=dtype), - scale=array_ops.ones([], dtype=dtype), - allow_nan_stats=allow_nan_stats) - else: - asserts = distribution_util.maybe_check_scalar_distribution( - distribution, dtype, validate_args) - if asserts: - scale_diag_part = control_flow_ops.with_dependencies( - asserts, scale_diag_part) - - # Make the SAS bijector, 'F'. - skewness = ops.convert_to_tensor(skewness, dtype=dtype, name="skewness") - tailweight = ops.convert_to_tensor( - tailweight, dtype=dtype, name="tailweight") - f = bijectors.SinhArcsinh( - skewness=skewness, tailweight=tailweight) - if has_default_skewness: - f_noskew = f - else: - f_noskew = bijectors.SinhArcsinh( - skewness=skewness.dtype.as_numpy_dtype(0.), - tailweight=tailweight) - - # Make the Affine bijector, Z --> loc + C * Z. - c = 2 * scale_diag_part / f_noskew.forward( - ops.convert_to_tensor(2, dtype=dtype)) - affine = bijectors.Affine( - shift=loc, scale_diag=c, validate_args=validate_args) - - bijector = bijectors.Chain([affine, f]) - - super(VectorSinhArcsinhDiag, self).__init__( - distribution=distribution, - bijector=bijector, - batch_shape=batch_shape, - event_shape=event_shape, - validate_args=validate_args, - name=name) - self._parameters = parameters - self._loc = loc - self._scale = scale_linop - self._tailweight = tailweight - self._skewness = skewness - - @property - def loc(self): - """The `loc` in `Y := loc + scale @ F(Z) * (2 / F(2)).""" - return self._loc - - @property - def scale(self): - """The `LinearOperator` `scale` in `Y := loc + scale @ F(Z) * (2 / F(2)).""" - return self._scale - - @property - def tailweight(self): - """Controls the tail decay. `tailweight > 1` means faster than Normal.""" - return self._tailweight - - @property - def skewness(self): - """Controls the skewness. `Skewness > 0` means right skew.""" - return self._skewness diff --git a/tensorflow/contrib/distributions/python/ops/vector_student_t.py b/tensorflow/contrib/distributions/python/ops/vector_student_t.py deleted file mode 100644 index bad91a08447..00000000000 --- a/tensorflow/contrib/distributions/python/ops/vector_student_t.py +++ /dev/null @@ -1,250 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Vector Student's t distribution classes.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distributions.python.ops import bijectors -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops.distributions import student_t -from tensorflow.python.ops.distributions import transformed_distribution -from tensorflow.python.util import deprecation - - -class _VectorStudentT(transformed_distribution.TransformedDistribution): - """A vector version of Student's t-distribution on `R^k`. - - #### Mathematical details - - The probability density function (pdf) is, - - ```none - pdf(x; df, mu, Sigma) = (1 + ||y||**2 / df)**(-0.5 (df + 1)) / Z - where, - y = inv(Sigma) (x - mu) - Z = abs(det(Sigma)) ( sqrt(df pi) Gamma(0.5 df) / Gamma(0.5 (df + 1)) )**k - ``` - - where: - * `loc = mu`; a vector in `R^k`, - * `scale = Sigma`; a lower-triangular matrix in `R^{k x k}`, - * `Z` denotes the normalization constant, and, - * `Gamma` is the [gamma function]( - https://en.wikipedia.org/wiki/Gamma_function), and, - * `||y||**2` denotes the [squared Euclidean norm]( - https://en.wikipedia.org/wiki/Norm_(mathematics)#Euclidean_norm) of `y`. - - The VectorStudentT distribution is a member of the [location-scale family]( - https://en.wikipedia.org/wiki/Location-scale_family), i.e., it can be - constructed as, - - ```none - X ~ StudentT(df, loc=0, scale=1) - Y = loc + scale * X - ``` - - Notice that the `scale` matrix has semantics closer to std. deviation than - covariance (but it is not std. deviation). - - This distribution is an Affine transformation of iid - [Student's t-distributions]( - https://en.wikipedia.org/wiki/Student%27s_t-distribution) - and should not be confused with the [Multivariate Student's t-distribution]( - https://en.wikipedia.org/wiki/Multivariate_t-distribution). The - traditional Multivariate Student's t-distribution is type of - [elliptical distribution]( - https://en.wikipedia.org/wiki/Elliptical_distribution); it has PDF: - - ```none - pdf(x; df, mu, Sigma) = (1 + ||y||**2 / df)**(-0.5 (df + k)) / Z - where, - y = inv(Sigma) (x - mu) - Z = abs(det(Sigma)) sqrt(df pi)**k Gamma(0.5 df) / Gamma(0.5 (df + k)) - ``` - - Notice that the Multivariate Student's t-distribution uses `k` where the - Vector Student's t-distribution has a `1`. Conversely the Vector version has a - broader application of the power-`k` in the normalization constant. - - #### Examples - - A single instance of a "Vector Student's t-distribution" is defined by a mean - vector of length `k` and a scale matrix of shape `k x k`. - - Extra leading dimensions, if provided, allow for batches. - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single 3-variate vector Student's t-distribution. - mu = [1., 2, 3] - chol = [[1., 0, 0.], - [1, 3, 0], - [1, 2, 3]] - vt = tfd.VectorStudentT(df=2, loc=mu, scale_tril=chol) - - # Evaluate this on an observation in R^3, returning a scalar. - vt.prob([-1., 0, 1]) - - # Initialize a batch of two 3-variate vector Student's t-distributions. - mu = [[1., 2, 3], - [11, 22, 33]] - chol = ... # shape 2 x 3 x 3, lower triangular, positive diagonal. - vt = tfd.VectorStudentT(loc=mu, scale_tril=chol) - - # Evaluate this on a two observations, each in R^3, returning a length two - # tensor. - x = [[-1, 0, 1], - [-11, 0, 11]] - vt.prob(x) - ``` - - For more examples of how to construct the `scale` matrix, see the - `tf.contrib.distributions.bijectors.Affine` docstring. - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - df, - loc=None, - scale_identity_multiplier=None, - scale_diag=None, - scale_tril=None, - scale_perturb_factor=None, - scale_perturb_diag=None, - validate_args=False, - allow_nan_stats=True, - name="VectorStudentT"): - """Instantiates the vector Student's t-distributions on `R^k`. - - The `batch_shape` is the broadcast between `df.batch_shape` and - `Affine.batch_shape` where `Affine` is constructed from `loc` and - `scale_*` arguments. - - The `event_shape` is the event shape of `Affine.event_shape`. - - Args: - df: Floating-point `Tensor`. The degrees of freedom of the - distribution(s). `df` must contain only positive values. Must be - scalar if `loc`, `scale_*` imply non-scalar batch_shape or must have the - same `batch_shape` implied by `loc`, `scale_*`. - loc: Floating-point `Tensor`. If this is set to `None`, no `loc` is - applied. - scale_identity_multiplier: floating point rank 0 `Tensor` representing a - scaling done to the identity matrix. When `scale_identity_multiplier = - scale_diag=scale_tril = None` then `scale += IdentityMatrix`. Otherwise - no scaled-identity-matrix is added to `scale`. - scale_diag: Floating-point `Tensor` representing the diagonal matrix. - `scale_diag` has shape [N1, N2, ..., k], which represents a k x k - diagonal matrix. When `None` no diagonal term is added to `scale`. - scale_tril: Floating-point `Tensor` representing the diagonal matrix. - `scale_diag` has shape [N1, N2, ..., k, k], which represents a k x k - lower triangular matrix. When `None` no `scale_tril` term is added to - `scale`. The upper triangular elements above the diagonal are ignored. - scale_perturb_factor: Floating-point `Tensor` representing factor matrix - with last two dimensions of shape `(k, r)`. When `None`, no rank-r - update is added to `scale`. - scale_perturb_diag: Floating-point `Tensor` representing the diagonal - matrix. `scale_perturb_diag` has shape [N1, N2, ..., r], which - represents an r x r Diagonal matrix. When `None` low rank updates will - take the form `scale_perturb_factor * scale_perturb_factor.T`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, - statistics (e.g., mean, mode, variance) use the value "`NaN`" to - indicate the result is undefined. When `False`, an exception is raised - if one or more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - parameters = dict(locals()) - graph_parents = [df, loc, scale_identity_multiplier, scale_diag, - scale_tril, scale_perturb_factor, scale_perturb_diag] - with ops.name_scope(name) as name: - with ops.name_scope("init", values=graph_parents): - # The shape of the _VectorStudentT distribution is governed by the - # relationship between df.batch_shape and affine.batch_shape. In - # pseudocode the basic procedure is: - # if df.batch_shape is scalar: - # if affine.batch_shape is not scalar: - # # broadcast distribution.sample so - # # it has affine.batch_shape. - # self.batch_shape = affine.batch_shape - # else: - # if affine.batch_shape is scalar: - # # let affine broadcasting do its thing. - # self.batch_shape = df.batch_shape - # All of the above magic is actually handled by TransformedDistribution. - # Here we really only need to collect the affine.batch_shape and decide - # what we're going to pass in to TransformedDistribution's - # (override) batch_shape arg. - affine = bijectors.Affine( - shift=loc, - scale_identity_multiplier=scale_identity_multiplier, - scale_diag=scale_diag, - scale_tril=scale_tril, - scale_perturb_factor=scale_perturb_factor, - scale_perturb_diag=scale_perturb_diag, - validate_args=validate_args) - distribution = student_t.StudentT( - df=df, - loc=array_ops.zeros([], dtype=affine.dtype), - scale=array_ops.ones([], dtype=affine.dtype)) - batch_shape, override_event_shape = ( - distribution_util.shapes_from_loc_and_scale( - affine.shift, affine.scale)) - override_batch_shape = distribution_util.pick_vector( - distribution.is_scalar_batch(), - batch_shape, - constant_op.constant([], dtype=dtypes.int32)) - super(_VectorStudentT, self).__init__( - distribution=distribution, - bijector=affine, - batch_shape=override_batch_shape, - event_shape=override_event_shape, - validate_args=validate_args, - name=name) - self._parameters = parameters - - @property - def df(self): - """Degrees of freedom in these Student's t distribution(s).""" - return self.distribution.df - - @property - def loc(self): - """Locations of these Student's t distribution(s).""" - return self.bijector.shift - - @property - def scale(self): - """Dense (batch) covariance matrix, if available.""" - return self.bijector.scale diff --git a/tensorflow/contrib/distributions/python/ops/wishart.py b/tensorflow/contrib/distributions/python/ops/wishart.py deleted file mode 100644 index 8b819053f92..00000000000 --- a/tensorflow/contrib/distributions/python/ops/wishart.py +++ /dev/null @@ -1,699 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The Wishart distribution class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import numpy as np - -from tensorflow.contrib.distributions.python.ops import distribution_util -from tensorflow.contrib.framework.python.framework import tensor_util as contrib_tensor_util -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops.distributions import distribution -from tensorflow.python.ops.linalg import linalg -from tensorflow.python.util import deprecation - -__all__ = [ - "WishartCholesky", - "WishartFull", -] - - -class _WishartLinearOperator(distribution.Distribution): - """The matrix Wishart distribution on positive definite matrices. - - This distribution is defined by a scalar number of degrees of freedom `df` and - an instance of `LinearOperator`, which provides matrix-free access to a - symmetric positive definite operator, which defines the scale matrix. - - #### Mathematical Details - - The probability density function (pdf) is, - - ```none - pdf(X; df, scale) = det(X)**(0.5 (df-k-1)) exp(-0.5 tr[inv(scale) X]) / Z - Z = 2**(0.5 df k) |det(scale)|**(0.5 df) Gamma_k(0.5 df) - ``` - - where: - - * `df >= k` denotes the degrees of freedom, - * `scale` is a symmetric, positive definite, `k x k` matrix, - * `Z` is the normalizing constant, and, - * `Gamma_k` is the [multivariate Gamma function]( - https://en.wikipedia.org/wiki/Multivariate_gamma_function). - - #### Examples - - See `WishartFull`, `WishartCholesky` for examples of initializing and using - this class. - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - df, - scale_operator, - cholesky_input_output_matrices=False, - validate_args=False, - allow_nan_stats=True, - name=None): - """Construct Wishart distributions. - - Args: - df: `float` or `double` tensor, the degrees of freedom of the - distribution(s). `df` must be greater than or equal to `k`. - scale_operator: `float` or `double` instance of `LinearOperator`. - cholesky_input_output_matrices: Python `bool`. Any function which whose - input or output is a matrix assumes the input is Cholesky and returns a - Cholesky factored matrix. Example `log_prob` input takes a Cholesky and - `sample_n` returns a Cholesky when - `cholesky_input_output_matrices=True`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - - Raises: - TypeError: if scale is not floating-type - TypeError: if scale.dtype != df.dtype - ValueError: if df < k, where scale operator event shape is - `(k, k)` - """ - parameters = dict(locals()) - self._cholesky_input_output_matrices = cholesky_input_output_matrices - with ops.name_scope(name) as name: - with ops.name_scope("init", values=[df, scale_operator]): - if not scale_operator.dtype.is_floating: - raise TypeError( - "scale_operator.dtype=%s is not a floating-point type" % - scale_operator.dtype) - if not scale_operator.is_square: - print(scale_operator.to_dense().eval()) - raise ValueError("scale_operator must be square.") - - self._scale_operator = scale_operator - self._df = ops.convert_to_tensor( - df, - dtype=scale_operator.dtype, - name="df") - contrib_tensor_util.assert_same_float_dtype( - (self._df, self._scale_operator)) - if (self._scale_operator.shape.ndims is None or - self._scale_operator.shape.dims[-1].value is None): - self._dimension = math_ops.cast( - self._scale_operator.domain_dimension_tensor(), - dtype=self._scale_operator.dtype, name="dimension") - else: - self._dimension = ops.convert_to_tensor( - self._scale_operator.shape.dims[-1].value, - dtype=self._scale_operator.dtype, name="dimension") - df_val = tensor_util.constant_value(self._df) - dim_val = tensor_util.constant_value(self._dimension) - if df_val is not None and dim_val is not None: - df_val = np.asarray(df_val) - if not df_val.shape: - df_val = [df_val] - if any(df_val < dim_val): - raise ValueError( - "Degrees of freedom (df = %s) cannot be less than " - "dimension of scale matrix (scale.dimension = %s)" - % (df_val, dim_val)) - elif validate_args: - assertions = check_ops.assert_less_equal( - self._dimension, self._df, - message=("Degrees of freedom (df = %s) cannot be " - "less than dimension of scale matrix " - "(scale.dimension = %s)" % - (self._dimension, self._df))) - self._df = control_flow_ops.with_dependencies( - [assertions], self._df) - super(_WishartLinearOperator, self).__init__( - dtype=self._scale_operator.dtype, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - reparameterization_type=distribution.FULLY_REPARAMETERIZED, - parameters=parameters, - graph_parents=[self._df, self._dimension], - name=name) - - @property - def df(self): - """Wishart distribution degree(s) of freedom.""" - return self._df - - def _square_scale_operator(self): - return self.scale_operator.matmul( - self.scale_operator.to_dense(), adjoint_arg=True) - - def scale(self): - """Wishart distribution scale matrix.""" - if self._cholesky_input_output_matrices: - return self.scale_operator.to_dense() - else: - return self._square_scale_operator() - - @property - def scale_operator(self): - """Wishart distribution scale matrix as an Linear Operator.""" - return self._scale_operator - - @property - def cholesky_input_output_matrices(self): - """Boolean indicating if `Tensor` input/outputs are Cholesky factorized.""" - return self._cholesky_input_output_matrices - - @property - def dimension(self): - """Dimension of underlying vector space. The `p` in `R^(p*p)`.""" - return self._dimension - - def _event_shape_tensor(self): - dimension = self.scale_operator.domain_dimension_tensor() - return array_ops.stack([dimension, dimension]) - - def _event_shape(self): - dimension = self.scale_operator.domain_dimension - return tensor_shape.TensorShape([dimension, dimension]) - - def _batch_shape_tensor(self): - return self.scale_operator.batch_shape_tensor() - - def _batch_shape(self): - return self.scale_operator.batch_shape - - def _sample_n(self, n, seed): - batch_shape = self.batch_shape_tensor() - event_shape = self.event_shape_tensor() - batch_ndims = array_ops.shape(batch_shape)[0] - - ndims = batch_ndims + 3 # sample_ndims=1, event_ndims=2 - shape = array_ops.concat([[n], batch_shape, event_shape], 0) - - # Complexity: O(nbk**2) - x = random_ops.random_normal(shape=shape, - mean=0., - stddev=1., - dtype=self.dtype, - seed=seed) - - # Complexity: O(nbk) - # This parametrization is equivalent to Chi2, i.e., - # ChiSquared(k) == Gamma(alpha=k/2, beta=1/2) - expanded_df = self.df * array_ops.ones( - self.scale_operator.batch_shape_tensor(), - dtype=self.df.dtype.base_dtype) - g = random_ops.random_gamma(shape=[n], - alpha=self._multi_gamma_sequence( - 0.5 * expanded_df, self.dimension), - beta=0.5, - dtype=self.dtype, - seed=distribution_util.gen_new_seed( - seed, "wishart")) - - # Complexity: O(nbk**2) - x = array_ops.matrix_band_part(x, -1, 0) # Tri-lower. - - # Complexity: O(nbk) - x = array_ops.matrix_set_diag(x, math_ops.sqrt(g)) - - # Make batch-op ready. - # Complexity: O(nbk**2) - perm = array_ops.concat([math_ops.range(1, ndims), [0]], 0) - x = array_ops.transpose(x, perm) - shape = array_ops.concat([batch_shape, [event_shape[0]], [-1]], 0) - x = array_ops.reshape(x, shape) - - # Complexity: O(nbM) where M is the complexity of the operator solving a - # vector system. E.g., for LinearOperatorDiag, each matmul is O(k**2), so - # this complexity is O(nbk**2). For LinearOperatorLowerTriangular, - # each matmul is O(k^3) so this step has complexity O(nbk^3). - x = self.scale_operator.matmul(x) - - # Undo make batch-op ready. - # Complexity: O(nbk**2) - shape = array_ops.concat([batch_shape, event_shape, [n]], 0) - x = array_ops.reshape(x, shape) - perm = array_ops.concat([[ndims - 1], math_ops.range(0, ndims - 1)], 0) - x = array_ops.transpose(x, perm) - - if not self.cholesky_input_output_matrices: - # Complexity: O(nbk^3) - x = math_ops.matmul(x, x, adjoint_b=True) - - return x - - def _log_prob(self, x): - if self.cholesky_input_output_matrices: - x_sqrt = x - else: - # Complexity: O(nbk^3) - x_sqrt = linalg_ops.cholesky(x) - - batch_shape = self.batch_shape_tensor() - event_shape = self.event_shape_tensor() - ndims = array_ops.rank(x_sqrt) - # sample_ndims = ndims - batch_ndims - event_ndims - sample_ndims = ndims - array_ops.shape(batch_shape)[0] - 2 - sample_shape = array_ops.strided_slice( - array_ops.shape(x_sqrt), [0], [sample_ndims]) - - # We need to be able to pre-multiply each matrix by its corresponding - # batch scale matrix. Since a Distribution Tensor supports multiple - # samples per batch, this means we need to reshape the input matrix `x` - # so that the first b dimensions are batch dimensions and the last two - # are of shape [dimension, dimensions*number_of_samples]. Doing these - # gymnastics allows us to do a batch_solve. - # - # After we're done with sqrt_solve (the batch operation) we need to undo - # this reshaping so what we're left with is a Tensor partitionable by - # sample, batch, event dimensions. - - # Complexity: O(nbk**2) since transpose must access every element. - scale_sqrt_inv_x_sqrt = x_sqrt - perm = array_ops.concat([math_ops.range(sample_ndims, ndims), - math_ops.range(0, sample_ndims)], 0) - scale_sqrt_inv_x_sqrt = array_ops.transpose(scale_sqrt_inv_x_sqrt, perm) - shape = array_ops.concat( - (batch_shape, (math_ops.cast( - self.dimension, dtype=dtypes.int32), -1)), - 0) - scale_sqrt_inv_x_sqrt = array_ops.reshape(scale_sqrt_inv_x_sqrt, shape) - - # Complexity: O(nbM*k) where M is the complexity of the operator solving - # a vector system. E.g., for LinearOperatorDiag, each solve is O(k), so - # this complexity is O(nbk**2). For LinearOperatorLowerTriangular, - # each solve is O(k**2) so this step has complexity O(nbk^3). - scale_sqrt_inv_x_sqrt = self.scale_operator.solve( - scale_sqrt_inv_x_sqrt) - - # Undo make batch-op ready. - # Complexity: O(nbk**2) - shape = array_ops.concat([batch_shape, event_shape, sample_shape], 0) - scale_sqrt_inv_x_sqrt = array_ops.reshape(scale_sqrt_inv_x_sqrt, shape) - perm = array_ops.concat([math_ops.range(ndims - sample_ndims, ndims), - math_ops.range(0, ndims - sample_ndims)], 0) - scale_sqrt_inv_x_sqrt = array_ops.transpose(scale_sqrt_inv_x_sqrt, perm) - - # Write V = SS', X = LL'. Then: - # tr[inv(V) X] = tr[inv(S)' inv(S) L L'] - # = tr[inv(S) L L' inv(S)'] - # = tr[(inv(S) L) (inv(S) L)'] - # = sum_{ik} (inv(S) L)_{ik}**2 - # The second equality follows from the cyclic permutation property. - # Complexity: O(nbk**2) - trace_scale_inv_x = math_ops.reduce_sum( - math_ops.square(scale_sqrt_inv_x_sqrt), - axis=[-2, -1]) - - # Complexity: O(nbk) - half_log_det_x = math_ops.reduce_sum( - math_ops.log(array_ops.matrix_diag_part(x_sqrt)), - axis=[-1]) - - # Complexity: O(nbk**2) - log_prob = ((self.df - self.dimension - 1.) * half_log_det_x - - 0.5 * trace_scale_inv_x - - self.log_normalization()) - - # Set shape hints. - # Try to merge what we know from the input then what we know from the - # parameters of this distribution. - if x.get_shape().ndims is not None: - log_prob.set_shape(x.get_shape()[:-2]) - if (log_prob.get_shape().ndims is not None and - self.batch_shape.ndims is not None and - self.batch_shape.ndims > 0): - log_prob.get_shape()[-self.batch_shape.ndims:].merge_with( - self.batch_shape) - - return log_prob - - def _prob(self, x): - return math_ops.exp(self._log_prob(x)) - - def _entropy(self): - half_dp1 = 0.5 * self.dimension + 0.5 - half_df = 0.5 * self.df - return (self.dimension * (half_df + half_dp1 * math.log(2.)) + - 2 * half_dp1 * self.scale_operator.log_abs_determinant() + - self._multi_lgamma(half_df, self.dimension) + - (half_dp1 - half_df) * self._multi_digamma(half_df, self.dimension)) - - def _mean(self): - if self.cholesky_input_output_matrices: - return (math_ops.sqrt(self.df) - * self.scale_operator.to_dense()) - return self.df * self._square_scale_operator() - - def _variance(self): - x = math_ops.sqrt(self.df) * self._square_scale_operator() - d = array_ops.expand_dims(array_ops.matrix_diag_part(x), -1) - v = math_ops.square(x) + math_ops.matmul(d, d, adjoint_b=True) - if self.cholesky_input_output_matrices: - return linalg_ops.cholesky(v) - return v - - def _stddev(self): - if self.cholesky_input_output_matrices: - raise ValueError( - "Computing std. dev. when is cholesky_input_output_matrices=True " - "does not make sense.") - return linalg_ops.cholesky(self.variance()) - - def _mode(self): - s = self.df - self.dimension - 1. - s = array_ops.where_v2( - math_ops.less(s, 0.), - constant_op.constant(float("NaN"), dtype=self.dtype, name="nan"), s) - if self.cholesky_input_output_matrices: - return math_ops.sqrt(s) * self.scale_operator.to_dense() - return s * self._square_scale_operator() - - def mean_log_det(self, name="mean_log_det"): - """Computes E[log(det(X))] under this Wishart distribution.""" - with self._name_scope(name): - return (self._multi_digamma(0.5 * self.df, self.dimension) + - self.dimension * math.log(2.) + - 2 * self.scale_operator.log_abs_determinant()) - - def log_normalization(self, name="log_normalization"): - """Computes the log normalizing constant, log(Z).""" - with self._name_scope(name): - return (self.df * self.scale_operator.log_abs_determinant() + - 0.5 * self.df * self.dimension * math.log(2.) + - self._multi_lgamma(0.5 * self.df, self.dimension)) - - def _multi_gamma_sequence(self, a, p, name="multi_gamma_sequence"): - """Creates sequence used in multivariate (di)gamma; shape = shape(a)+[p].""" - with self._name_scope(name, values=[a, p]): - # Linspace only takes scalars, so we'll add in the offset afterwards. - seq = math_ops.linspace( - constant_op.constant(0., dtype=self.dtype), - 0.5 - 0.5 * p, - math_ops.cast(p, dtypes.int32)) - return seq + array_ops.expand_dims(a, [-1]) - - def _multi_lgamma(self, a, p, name="multi_lgamma"): - """Computes the log multivariate gamma function; log(Gamma_p(a)).""" - with self._name_scope(name, values=[a, p]): - seq = self._multi_gamma_sequence(a, p) - return (0.25 * p * (p - 1.) * math.log(math.pi) + - math_ops.reduce_sum(math_ops.lgamma(seq), - axis=[-1])) - - def _multi_digamma(self, a, p, name="multi_digamma"): - """Computes the multivariate digamma function; Psi_p(a).""" - with self._name_scope(name, values=[a, p]): - seq = self._multi_gamma_sequence(a, p) - return math_ops.reduce_sum(math_ops.digamma(seq), - axis=[-1]) - - -class WishartCholesky(_WishartLinearOperator): - """The matrix Wishart distribution on positive definite matrices. - - This distribution is defined by a scalar degrees of freedom `df` and a - lower, triangular Cholesky factor which characterizes the scale matrix. - - Using WishartCholesky is a constant-time improvement over WishartFull. It - saves an O(nbk^3) operation, i.e., a matrix-product operation for sampling - and a Cholesky factorization in log_prob. For most use-cases it often saves - another O(nbk^3) operation since most uses of Wishart will also use the - Cholesky factorization. - - #### Mathematical Details - - The probability density function (pdf) is, - - ```none - pdf(X; df, scale) = det(X)**(0.5 (df-k-1)) exp(-0.5 tr[inv(scale) X]) / Z - Z = 2**(0.5 df k) |det(scale)|**(0.5 df) Gamma_k(0.5 df) - ``` - - where: - * `df >= k` denotes the degrees of freedom, - * `scale` is a symmetric, positive definite, `k x k` matrix, - * `Z` is the normalizing constant, and, - * `Gamma_k` is the [multivariate Gamma function]( - https://en.wikipedia.org/wiki/Multivariate_gamma_function). - - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single 3x3 Wishart with Cholesky factored scale matrix and 5 - # degrees-of-freedom.(*) - df = 5 - chol_scale = tf.linalg.cholesky(...) # Shape is [3, 3]. - dist = tfd.WishartCholesky(df=df, scale=chol_scale) - - # Evaluate this on an observation in R^3, returning a scalar. - x = ... # A 3x3 positive definite matrix. - dist.prob(x) # Shape is [], a scalar. - - # Evaluate this on a two observations, each in R^{3x3}, returning a length two - # Tensor. - x = [x0, x1] # Shape is [2, 3, 3]. - dist.prob(x) # Shape is [2]. - - # Initialize two 3x3 Wisharts with Cholesky factored scale matrices. - df = [5, 4] - chol_scale = tf.linalg.cholesky(...) # Shape is [2, 3, 3]. - dist = tfd.WishartCholesky(df=df, scale=chol_scale) - - # Evaluate this on four observations. - x = [[x0, x1], [x2, x3]] # Shape is [2, 2, 3, 3]. - dist.prob(x) # Shape is [2, 2]. - - # (*) - To efficiently create a trainable covariance matrix, see the example - # in tfp.distributions.matrix_diag_transform. - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - df, - scale, - cholesky_input_output_matrices=False, - validate_args=False, - allow_nan_stats=True, - name="WishartCholesky"): - """Construct Wishart distributions. - - Args: - df: `float` or `double` `Tensor`. Degrees of freedom, must be greater than - or equal to dimension of the scale matrix. - scale: `float` or `double` `Tensor`. The Cholesky factorization of - the symmetric positive definite scale matrix of the distribution. - cholesky_input_output_matrices: Python `bool`. Any function which whose - input or output is a matrix assumes the input is Cholesky and returns a - Cholesky factored matrix. Example `log_prob` input takes a Cholesky and - `sample_n` returns a Cholesky when - `cholesky_input_output_matrices=True`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - parameters = dict(locals()) - with ops.name_scope(name, values=[scale]) as name: - with ops.name_scope("init", values=[scale]): - scale = ops.convert_to_tensor(scale) - if validate_args: - scale = control_flow_ops.with_dependencies([ - check_ops.assert_positive( - array_ops.matrix_diag_part(scale), - message="scale must be positive definite"), - check_ops.assert_equal( - array_ops.shape(scale)[-1], - array_ops.shape(scale)[-2], - message="scale must be square") - ] if validate_args else [], scale) - - super(WishartCholesky, self).__init__( - df=df, - scale_operator=linalg.LinearOperatorLowerTriangular( - tril=scale, - is_non_singular=True, - is_positive_definite=True, - is_square=True), - cholesky_input_output_matrices=cholesky_input_output_matrices, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - self._parameters = parameters - - -class WishartFull(_WishartLinearOperator): - """The matrix Wishart distribution on positive definite matrices. - - This distribution is defined by a scalar degrees of freedom `df` and a - symmetric, positive definite scale matrix. - - Evaluation of the pdf, determinant, and sampling are all `O(k^3)` operations - where `(k, k)` is the event space shape. - - #### Mathematical Details - - The probability density function (pdf) is, - - ```none - pdf(X; df, scale) = det(X)**(0.5 (df-k-1)) exp(-0.5 tr[inv(scale) X]) / Z - Z = 2**(0.5 df k) |det(scale)|**(0.5 df) Gamma_k(0.5 df) - ``` - - where: - * `df >= k` denotes the degrees of freedom, - * `scale` is a symmetric, positive definite, `k x k` matrix, - * `Z` is the normalizing constant, and, - * `Gamma_k` is the [multivariate Gamma function]( - https://en.wikipedia.org/wiki/Multivariate_gamma_function). - - #### Examples - - ```python - import tensorflow_probability as tfp - tfd = tfp.distributions - - # Initialize a single 3x3 Wishart with Full factored scale matrix and 5 - # degrees-of-freedom.(*) - df = 5 - scale = ... # Shape is [3, 3]; positive definite. - dist = tfd.WishartFull(df=df, scale=scale) - - # Evaluate this on an observation in R^3, returning a scalar. - x = ... # A 3x3 positive definite matrix. - dist.prob(x) # Shape is [], a scalar. - - # Evaluate this on a two observations, each in R^{3x3}, returning a length two - # Tensor. - x = [x0, x1] # Shape is [2, 3, 3]. - dist.prob(x) # Shape is [2]. - - # Initialize two 3x3 Wisharts with Full factored scale matrices. - df = [5, 4] - scale = ... # Shape is [2, 3, 3]. - dist = tfd.WishartFull(df=df, scale=scale) - - # Evaluate this on four observations. - x = [[x0, x1], [x2, x3]] # Shape is [2, 2, 3, 3]; xi is positive definite. - dist.prob(x) # Shape is [2, 2]. - - # (*) - To efficiently create a trainable covariance matrix, see the example - # in tfd.matrix_diag_transform. - ``` - - """ - - @deprecation.deprecated( - "2018-10-01", - "The TensorFlow Distributions library has moved to " - "TensorFlow Probability " - "(https://github.com/tensorflow/probability). You " - "should update all references to use `tfp.distributions` " - "instead of `tf.contrib.distributions`.", - warn_once=True) - def __init__(self, - df, - scale, - cholesky_input_output_matrices=False, - validate_args=False, - allow_nan_stats=True, - name="WishartFull"): - """Construct Wishart distributions. - - Args: - df: `float` or `double` `Tensor`. Degrees of freedom, must be greater than - or equal to dimension of the scale matrix. - scale: `float` or `double` `Tensor`. The symmetric positive definite - scale matrix of the distribution. - cholesky_input_output_matrices: Python `bool`. Any function which whose - input or output is a matrix assumes the input is Cholesky and returns a - Cholesky factored matrix. Example `log_prob` input takes a Cholesky and - `sample_n` returns a Cholesky when - `cholesky_input_output_matrices=True`. - validate_args: Python `bool`, default `False`. When `True` distribution - parameters are checked for validity despite possibly degrading runtime - performance. When `False` invalid inputs may silently render incorrect - outputs. - allow_nan_stats: Python `bool`, default `True`. When `True`, statistics - (e.g., mean, mode, variance) use the value "`NaN`" to indicate the - result is undefined. When `False`, an exception is raised if one or - more of the statistic's batch members are undefined. - name: Python `str` name prefixed to Ops created by this class. - """ - parameters = dict(locals()) - with ops.name_scope(name) as name: - with ops.name_scope("init", values=[scale]): - scale = ops.convert_to_tensor(scale) - if validate_args: - scale = distribution_util.assert_symmetric(scale) - chol = linalg_ops.cholesky(scale) - chol = control_flow_ops.with_dependencies([ - check_ops.assert_positive(array_ops.matrix_diag_part(chol)) - ] if validate_args else [], chol) - super(WishartFull, self).__init__( - df=df, - scale_operator=linalg.LinearOperatorLowerTriangular( - tril=chol, - is_non_singular=True, - is_positive_definite=True, - is_square=True), - cholesky_input_output_matrices=cholesky_input_output_matrices, - validate_args=validate_args, - allow_nan_stats=allow_nan_stats, - name=name) - self._parameters = parameters diff --git a/tensorflow/contrib/eager/README.md b/tensorflow/contrib/eager/README.md deleted file mode 100644 index 4bd2769e879..00000000000 --- a/tensorflow/contrib/eager/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Eager Execution - -Eager execution provides an imperative interface to TensorFlow (similar to -[NumPy](http://www.numpy.org)). When you enable eager execution, TensorFlow -operations execute immediately; you do not execute a pre-constructed graph with -[`Session.run()`](https://www.tensorflow.org/api_docs/python/tf/Session). - -For example, consider a simple computation in TensorFlow: - -```python -x = tf.placeholder(tf.float32, shape=[1, 1]) -m = tf.matmul(x, x) - -with tf.Session() as sess: - print(sess.run(m, feed_dict={x: [[2.]]})) - -# Will print [[4.]] -``` - -Eager execution makes this much simpler: - -```python -x = [[2.]] -m = tf.matmul(x, x) - -print(m) -``` - -## Caveats - -This feature is in early stages and work remains to be done in terms of smooth -support for distributed and multi-GPU training and performance. - -- [Known issues](https://github.com/tensorflow/tensorflow/issues?q=is%3Aissue%20is%3Aopen%20label%3Acomp%3Aeager) -- Feedback is welcome, please consider - [filing an issue](https://github.com/tensorflow/tensorflow/issues/new) to provide it. - -## Installation - -For eager execution, we recommend using TensorFlow version 1.8 or newer. -Installation instructions at https://www.tensorflow.org/install/ - -## Documentation - -For an introduction to eager execution in TensorFlow, see: - -- [User Guide](https://www.tensorflow.org/guide/eager) ([source](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/eager/index.md)) -- Notebook: [Basic Usage](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/eager/eager_basics.ipynb) -- Notebook: [Automatic differentiation and gradient tape](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/eager/automatic_differentiation.ipynb) diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD deleted file mode 100644 index ef8c760ad96..00000000000 --- a/tensorflow/contrib/eager/python/BUILD +++ /dev/null @@ -1,257 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "py_test") -load("//tensorflow:tensorflow.bzl", "cuda_py_test") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "tfe", - srcs = ["tfe.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":datasets", - ":metrics", - ":network", - ":parameter_server", - ":saver", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:script_ops", - "//tensorflow/python:template", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python/eager:backprop", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:def_function", - "//tensorflow/python/eager:function", - "//tensorflow/python/eager:remote", - ], -) - -cuda_py_test( - name = "tfe_test", - srcs = ["tfe_test.py"], - additional_deps = [ - ":tfe", - "//tensorflow/python:array_ops", - "//tensorflow/python:metrics", - "//tensorflow/python:math_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:platform_test", - "//tensorflow/python:summary", - ], -) - -py_library( - name = "datasets", - srcs = ["datasets.py"], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/contrib/data/python/ops:prefetching_ops", - "//tensorflow/python:array_ops", - "//tensorflow/python:dataset_ops_gen", - "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python/data/ops:iterator_ops", - "//tensorflow/python/data/util:nest", - "//tensorflow/python/eager:context", - ], -) - -cuda_py_test( - name = "datasets_test", - srcs = ["datasets_test.py"], - additional_deps = [ - ":datasets", - "//tensorflow/contrib/data/python/ops:prefetching_ops", - "//tensorflow/contrib/data/python/ops:threadpool", - "//tensorflow/contrib/data/python/ops:unique", - "//tensorflow/contrib/lookup:lookup_py", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:script_ops", - "//tensorflow/python:training", - "//tensorflow/python/data", - "//tensorflow/python/eager:test", - ], - tags = ["noguitar"], -) - -py_library( - name = "saver", - srcs = ["saver.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:training", - "//tensorflow/python/eager:context", - ], -) - -py_library( - name = "parameter_server", - srcs = ["parameter_server.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:framework", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:variable_scope", - "//tensorflow/python/eager:context", - ], -) - -cuda_py_test( - name = "saver_test", - srcs = ["saver_test.py"], - additional_deps = [ - ":saver", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python/eager:test", - "//tensorflow/python:variables", - ], -) - -py_library( - name = "metrics", - srcs = [ - "metrics.py", - "metrics_impl.py", - ], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:summary_ops_v2", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:function", - "//tensorflow/python/training/tracking:base", - ], -) - -py_test( - name = "metrics_test", - srcs = ["metrics_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":metrics", - "//tensorflow/contrib/summary:summary_test_util", - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:summary_ops_v2", - "//tensorflow/python:training", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:test", - ], -) - -py_library( - name = "evaluator", - srcs = [ - "evaluator.py", - ], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:internal"], - deps = [ - ":datasets", - ":metrics", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", - "//tensorflow/python:summary_ops_v2", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:function", - "@six_archive//:six", - ], -) - -py_test( - name = "evaluator_test", - srcs = ["evaluator_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":evaluator", - ":metrics", - "//tensorflow/contrib/summary:summary_test_util", - "//tensorflow/python:framework_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:test", - ], -) - -py_library( - name = "network", - srcs = ["network.py"], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python:framework_ops", - "//tensorflow/python:layers_base", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python/eager:context", - "//tensorflow/python/estimator:estimator_py", - ], -) - -py_test( - name = "network_test", - srcs = ["network_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":network", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:constant_op", - "//tensorflow/python:errors", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:layers", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python/eager:function", - "//tensorflow/python/eager:test", - ], -) - -cuda_py_test( - name = "remote_test", - srcs = ["remote_test.py"], - additional_deps = [ - ":parameter_server", - "//tensorflow/python/eager:remote", - "//tensorflow/contrib/eager/python:tfe", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:framework", - "//tensorflow/python:math_ops", - "//tensorflow/python/eager:function", - ], -) diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py deleted file mode 100644 index 97bf02f6539..00000000000 --- a/tensorflow/contrib/eager/python/datasets.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Iteration over tf.data.Datasets when eager execution is enabled.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.experimental.ops import prefetching_ops -from tensorflow.python.data.ops import iterator_ops -from tensorflow.python.eager import context -from tensorflow.python.framework import ops - - -class Iterator(iterator_ops.IteratorV2): - """An iterator producing tf.Tensor objects from a tf.data.Dataset. - - NOTE: Unlike the iterator created by the - `tf.data.Dataset.make_one_shot_iterator` method, this class enables - additional experimental functionality, such as prefetching to the GPU. - """ - - def __init__(self, dataset): - """Creates a new iterator over the given dataset. - - For example: - ```python - dataset = tf.data.Dataset.range(4) - for x in Iterator(dataset): - print(x) - ``` - - Tensors produced will be placed on the device on which this iterator object - was created. - - Args: - dataset: A `tf.data.Dataset` object. - - Raises: - TypeError: If `dataset` is an unsupported type. - RuntimeError: When invoked without eager execution enabled. - """ - if not context.context().device_spec.device_type: - is_remote_device = False - else: - is_remote_device = context.context().device_spec.device_type != "CPU" - if is_remote_device: - with ops.device(None): - # Let the placer figure out where to place the various functions etc. - # created by the CopyToDeviceDataset. - dataset = dataset.apply(prefetching_ops.copy_to_device( - context.context().device_name)) - dataset = dataset.prefetch(1) - super(Iterator, self).__init__(dataset) - - def _next_internal(self): - """Returns a nested structure of `tf.Tensor`s containing the next element. - """ - # This runs in sync mode as iterators use an error status to communicate - # that there is no more data to iterate over. - # TODO(b/77291417): Fix - with context.execution_mode(context.SYNC): - return super(Iterator, self)._next_internal() diff --git a/tensorflow/contrib/eager/python/datasets_test.py b/tensorflow/contrib/eager/python/datasets_test.py deleted file mode 100644 index 0bbece7d6c3..00000000000 --- a/tensorflow/contrib/eager/python/datasets_test.py +++ /dev/null @@ -1,377 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -import threading -import time - -import numpy as np - -from tensorflow.contrib import lookup -from tensorflow.contrib.eager.python import datasets -from tensorflow.python.data.experimental.ops import threadpool -from tensorflow.python.data.experimental.ops import unique -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.eager import test -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import script_ops -from tensorflow.python.training import checkpoint_management -from tensorflow.python.training.tracking import util as trackable_utils - - -class IteratorTest(test.TestCase): - - def testBasic(self): - got = [] - for t in datasets.Iterator(dataset_ops.Dataset.range(4)): - got.append(t.numpy()) - self.assertAllEqual([0, 1, 2, 3], got) - - def testBasicOneShotIterator(self): - got = [] - for t in dataset_ops.Dataset.range(4).make_one_shot_iterator(): - got.append(t.numpy()) - self.assertAllEqual([0, 1, 2, 3], got) - - def testBasicImplicitIterator(self): - got = [] - for t in dataset_ops.Dataset.range(4): - got.append(t.numpy()) - self.assertAllEqual([0, 1, 2, 3], got) - - def testGetNext(self): - iterator = datasets.Iterator(dataset_ops.Dataset.range(4)) - self.assertEqual(0, iterator.get_next().numpy()) - self.assertEqual(1, iterator.get_next().numpy()) - self.assertEqual(2, iterator.get_next().numpy()) - self.assertEqual(3, iterator.get_next().numpy()) - with self.assertRaises(errors.OutOfRangeError): - iterator.get_next() - - def testGetNextOneShotIterator(self): - iterator = dataset_ops.Dataset.range(4).make_one_shot_iterator() - self.assertEqual(0, iterator.get_next().numpy()) - self.assertEqual(1, iterator.get_next().numpy()) - self.assertEqual(2, iterator.get_next().numpy()) - self.assertEqual(3, iterator.get_next().numpy()) - with self.assertRaises(errors.OutOfRangeError): - iterator.get_next() - - def testMultipleIteratorsOnTheSameDataset(self): - ds = dataset_ops.Dataset.range(4) - it1 = datasets.Iterator(ds) - it2 = datasets.Iterator(ds) - got = [x.numpy() for x in it1] - self.assertAllEqual([0, 1, 2, 3], got) - - got = [x.numpy() for x in it2] - self.assertAllEqual([0, 1, 2, 3], got) - - def testNestedOutputs(self): - ds = dataset_ops.Dataset.zip( - (dataset_ops.Dataset.range(4), - dataset_ops.Dataset.zip( - (dataset_ops.Dataset.range(4), dataset_ops.Dataset.range(4))))) - total = 0 - # The Iterator will return a nested structure of Tensor objects. - # Some funkiness to compare against simple integers. - for (i, x) in enumerate(datasets.Iterator(ds)): - want = (i, (i, i)) - got = (x[0].numpy(), (x[1][0].numpy(), x[1][1].numpy())) - self.assertEqual(got, want) - total += 1 - self.assertEqual(4, total) - - def testMapAndFilter(self): - - def even(x): - return math_ops.equal(math_ops.mod(x, 2), 0) - - it = datasets.Iterator( - dataset_ops.Dataset.range(8).map(math_ops.square).filter(even)) - got = [x.numpy() for x in it] - self.assertAllEqual([0, 4, 16, 36], got) - - def testMapCaptureLookupTable(self): - default_val = -1 - keys = constant_op.constant(['brain', 'salad', 'surgery']) - values = constant_op.constant([0, 1, 2], dtypes.int64) - table = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), default_val) - dataset = dataset_ops.Dataset.from_tensor_slices( - ['brain', 'salad', 'surgery']) - dataset = dataset.map(table.lookup) - it = datasets.Iterator(dataset) - got = [x.numpy() for x in it] - self.assertAllEqual([0, 1, 2], got) - - def testMultipleIteratorsOnADatasetThatUsesFunctions(self): - ds = dataset_ops.Dataset.from_tensor_slices([1, 2, 3, 4, 5, - 6]).map(math_ops.square) - - got1 = [x.numpy() for x in datasets.Iterator(ds)] - self.assertAllEqual([1, 4, 9, 16, 25, 36], got1) - got2 = [x.numpy() for x in datasets.Iterator(ds)] - self.assertAllEqual(got1, got2) - - def assertSparseValuesEqual(self, a, b): - self.assertAllEqual(a.indices, b.indices) - self.assertAllEqual(a.values, b.values) - self.assertAllEqual(a.dense_shape, b.dense_shape) - - def testSparseTensorElements(self): - components = (sparse_tensor.SparseTensorValue( - indices=np.array([[0, 0], [1, 0], [2, 0]]), - values=np.array([0, 0, 0]), - dense_shape=np.array([3, 1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[0, 0], [1, 1], [2, 2]]), - values=np.array([1, 2, 3]), - dense_shape=np.array([3, 3]))) - - expected = [ - (sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([0]), - dense_shape=np.array([1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([1]), - dense_shape=np.array([3]))), - (sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([0]), - dense_shape=np.array([1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[1]]), - values=np.array([2]), - dense_shape=np.array([3]))), - (sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([0]), - dense_shape=np.array([1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[2]]), - values=np.array([3]), - dense_shape=np.array([3]))), - ] - - for i, result in enumerate( - datasets.Iterator(dataset_ops.Dataset.from_tensor_slices(components))): - self.assertSparseValuesEqual(expected[i][0], result[0]) - self.assertSparseValuesEqual(expected[i][1], result[1]) - - def testPyFunc(self): - - def my_map(inp): - return [[x + 1 for x in inp]] - - ds = dataset_ops.Dataset.range(4).map( - lambda x: script_ops.py_func(my_map, [[x]], dtypes.int64)) - got = [x.numpy() for x in datasets.Iterator(ds)] - self.assertAllEqual([[1], [2], [3], [4]], got) - - def testTensorsPlacedOnDevice(self): - ds = dataset_ops.Dataset.from_tensors([0., 1.]) - with ops.device(test.gpu_device_name()): - x = datasets.Iterator(ds).next() - x = math_ops.add(x, x) - self.assertAllEqual([0., 2.], x.numpy()) - - def testGpuTensor(self): - ds = dataset_ops.Dataset.from_tensors([0., 1.]) - with ops.device(test.gpu_device_name()): - for x in ds: - y = math_ops.add(x, x) - self.assertAllEqual([0., 2.], y.numpy()) - - def testOverrideThreadPool(self): - - def get_thread_id(_): - # Python creates a dummy thread object to represent the current - # thread when called from an "alien" thread (such as a - # `PrivateThreadPool` thread in this case). It does not include - # the TensorFlow-given display name, but it has a unique - # identifier that maps one-to-one with the underlying OS thread. - return np.array(threading.current_thread().ident).astype(np.int64) - - for num_threads in [1, 2, 4, 8, 16]: - - dataset = ( - dataset_ops.Dataset.range(1000).map( - lambda x: script_ops.py_func(get_thread_id, [x], dtypes.int64), - num_parallel_calls=32).apply(unique.unique())) - - dataset = threadpool.override_threadpool( - dataset, - threadpool.PrivateThreadPool( - num_threads, display_name='private_thread_pool_%d' % num_threads)) - - thread_ids = [] - for next_element in datasets.Iterator(dataset): - thread_ids.append(next_element) - self.assertEqual(len(thread_ids), len(set(thread_ids))) - self.assertGreater(len(thread_ids), 0) - # NOTE(mrry): We don't control the thread pool scheduling, and - # so cannot guarantee that all of the threads in the pool will - # perform work. - self.assertLessEqual(len(thread_ids), num_threads) - - def testSaveRestore(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt') - dataset = dataset_ops.Dataset.from_tensor_slices( - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) - dataset = dataset.map(math_ops.square).batch(2) - # TODO(b/138399725): Re-enable default optimizations. - options = dataset_ops.Options() - options.experimental_optimization.apply_default_optimizations = False - dataset = dataset.with_options(options) - iterator = datasets.Iterator(dataset) - checkpoint = trackable_utils.Checkpoint(iterator=iterator) - self.assertAllEqual([1, 4], iterator.get_next().numpy()) - save_path = checkpoint.save(checkpoint_prefix) - self.assertAllEqual([9, 16], iterator.get_next().numpy()) - self.assertAllEqual([25, 36], iterator.get_next().numpy()) - checkpoint.restore(save_path) - self.assertAllEqual([9, 16], iterator.get_next().numpy()) - self.assertAllEqual([25, 36], iterator.get_next().numpy()) - - def testSaveRestoreMultipleIterator(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt') - dataset = dataset_ops.Dataset.from_tensor_slices( - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) - dataset = dataset.map(math_ops.square).batch(2) - # TODO(b/138399725): Re-enable default optimizations. - options = dataset_ops.Options() - options.experimental_optimization.apply_default_optimizations = False - dataset = dataset.with_options(options) - iterator_1 = datasets.Iterator(dataset) - iterator_2 = datasets.Iterator(dataset) - dataset_2 = dataset_ops.Dataset.range(10) - iterator_3 = datasets.Iterator(dataset_2) - - checkpoint = trackable_utils.Checkpoint( - iterator_1=iterator_1, iterator_2=iterator_2, iterator_3=iterator_3) - self.assertAllEqual([1, 4], iterator_1.get_next().numpy()) - self.assertEqual(0, iterator_3.get_next().numpy()) - self.assertEqual(1, iterator_3.get_next().numpy()) - self.assertEqual(2, iterator_3.get_next().numpy()) - - save_path = checkpoint.save(checkpoint_prefix) - self.assertAllEqual([1, 4], iterator_2.get_next().numpy()) - self.assertAllEqual([9, 16], iterator_2.get_next().numpy()) - self.assertEqual(3, iterator_3.get_next().numpy()) - checkpoint.restore(save_path) - self.assertAllEqual([9, 16], iterator_1.get_next().numpy()) - self.assertAllEqual([1, 4], iterator_2.get_next().numpy()) - self.assertEqual(3, iterator_3.get_next().numpy()) - - def testRestoreExhaustedIterator(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt') - dataset = dataset_ops.Dataset.range(3) - iterator = datasets.Iterator(dataset) - - checkpoint = trackable_utils.Checkpoint(iterator=iterator) - self.assertEqual(0, iterator.get_next().numpy()) - self.assertEqual(1, iterator.get_next().numpy()) - save_path = checkpoint.save(checkpoint_prefix) - self.assertEqual(2, iterator.get_next().numpy()) - checkpoint.restore(save_path) - self.assertEqual(2, iterator.get_next().numpy()) - - def testRestoreInReconstructedIterator(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt') - dataset = dataset_ops.Dataset.range(10) - for i in range(5): - iterator = datasets.Iterator(dataset) - checkpoint = trackable_utils.Checkpoint(iterator=iterator) - checkpoint.restore( - checkpoint_management.latest_checkpoint(checkpoint_directory)) - for j in range(2): - self.assertEqual(i * 2 + j, iterator.get_next().numpy()) - checkpoint.save(file_prefix=checkpoint_prefix) - - -class DatasetConstructorBenchmark(test.Benchmark): - - def benchmarkSliceRepeatBatchEager(self): - input_size = 10000 - batch_size = 100 - num_epochs = 100 - - input_data = np.random.randn(input_size) - - dataset = ( - dataset_ops.Dataset.from_tensor_slices(input_data).repeat( - num_epochs).batch(batch_size)) - iterator = datasets.Iterator(dataset) - - ends = [time.time()] - for _ in iterator: - ends.append(time.time()) - - deltas = np.ediff1d(ends) - median_wall_time = np.median(deltas) - print('Slice/repeat/batch eager input size: %d batch size: %d Median wall ' - 'time per element: %f' % (input_size, batch_size, median_wall_time)) - self.report_benchmark( - iters=len(deltas), - wall_time=median_wall_time, - name='benchmark_slice_repeat_batch_eager_input_%d_batch_%d' % - (input_size, batch_size)) - - def benchmarkSliceBatchCacheRepeatCallable(self): - input_size = 10000 - batch_size = 100 - num_epochs = 100 - - input_data = np.random.randn(input_size) - - dataset = ( - dataset_ops.Dataset.from_tensor_slices(input_data).batch( - batch_size).cache().repeat(num_epochs)) - iterator = datasets.Iterator(dataset) - - ends = [time.time()] - for _ in iterator: - ends.append(time.time()) - - deltas = np.ediff1d(ends) - median_wall_time = np.median(deltas) - print('Slice/batch/cache/repeat eager input size: %d batch size: %d Median ' - 'wall time per element: %f' % - (input_size, batch_size, median_wall_time)) - self.report_benchmark( - iters=len(deltas), - wall_time=median_wall_time, - name='benchmark_slice_batch_cache_repeat_eager_input_%d_batch_%d' % - (input_size, batch_size)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/eager/python/evaluator.py b/tensorflow/contrib/eager/python/evaluator.py deleted file mode 100644 index fa46d73241d..00000000000 --- a/tensorflow/contrib/eager/python/evaluator.py +++ /dev/null @@ -1,380 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Class Evaluator holds Metrics for the duration of an evaluation run.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import six - -from tensorflow.contrib.eager.python import datasets -from tensorflow.contrib.eager.python import metrics -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.eager import context -from tensorflow.python.eager import function -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import summary_ops_v2 as summary_ops - - -class Evaluator(object): - """This holds and updates Metrics for the duration of a single eval run. - - Usage: - evaluator = my_model.evaluator() # or MyEvaluator(my_model) - for example_batch in ...: - evaluator(example_batch) - results = evaluator.all_metric_results(optional_summary_logdir) - - Or, if you are getting your examples from a tf.data.Dataset, you can use - the evaluate_on_dataset() method. - - Implementers of Evaluators should - (a) Call `track_metric()` and/or `track_evaluator()` in __init__(). - (b) Override the `call()` method. It will be passed the output of the - model's `eval_data()` method, and should call its contained metrics - (treating them as callables) and any child Evaluators (using their - call() method to avoid calling eval_data() again). - - Args: - model: A `Model` object with an `eval_data()` method. - """ - - def __init__(self, model): - self._model = model - self._metrics = {} - self._evaluators = {} - if not context.executing_eagerly(): - self.call = function.defun(self.call) - - # ---- API for users ---- - def __call__(self, *args, **kwargs): - """Update metrics with a minibatch of input examples. - - Args: - *args: - **kwargs: Arguments representing an input mini-batch of examples to - pass to self.model.eval_data(). - - Returns: - The op to execute or None if executing eagerly. - """ - return self.call(self._model.eval_data(*args, **kwargs)) - - def init_variables(self): - """Return an op for initializing all contained uninitialized variables. - - Only for graph execution. Should be called after variables are created - in the first execution of __call__(). - - Returns: - An op. - - Raises: - RuntimeError: if eager execution is enabled. - - @compatibility(eager) - Only for graph execution. - @end_compatibility - """ - if context.executing_eagerly(): - raise RuntimeError("Evaluator.init_variables() not needed when " - "eager execution is enabled.") - return control_flow_ops.group([m.init_variables() for _, m in self.metrics]) - - def all_metric_results(self, summary_logdir=None): - """Computes results for all contained metrics. - - Args: - summary_logdir: An optional string. If specified, metric results - will be written as summaries to this directory. - - Returns: - A `dict` mapping string names to tensors. - """ - if summary_logdir is None: - with summary_ops.never_record_summaries(): - return self._all_metric_results() - else: - def f(): - with summary_ops.create_file_writer( - summary_logdir).as_default(), summary_ops.always_record_summaries(): - return self._all_metric_results() - - if context.executing_eagerly(): - return f() - else: - return function.defun(f)() - - def _all_metric_results(self): - """Implementation of `all_metric_results` in the summary context.""" - results = {} - for name, metric in six.iteritems(self._metrics): - results[name] = metric.result() - for prefix, evaluator in six.iteritems(self._evaluators): - for name, metric in six.iteritems(evaluator._metrics): # pylint: disable=protected-access - results[prefix + "/" + name] = metric.result() - return results - - def evaluate_on_dataset(self, dataset, *args, **kwargs): - """Convenience method for performing an eval on a Dataset. - - Args: - dataset: Dataset object with the input data to evaluate on. - *args: - **kwargs: Optional additional arguments to __call__(), except - `summary_logdir`: if specified, metrics will be written as summaries - to this directory. - - Returns: - @compatibility(eager) - When eager execution is enabled, this returns the result of performing - an evaluation as a dictionary. With graph execution, this returns a tuple - (init_op, call_op, results_op) which may be executed using this code: - ```python - sess.run(init_op) - try: - while True: - sess.run(call_op) - except tf.errors.OutOfRangeError: - pass - return sess.run(results_op) # A dictionary - - # equivalently: - return evaluator.run_evaluation(init_op, call_op, results_op, sess=sess) - ``` - @end_compatibility - """ - summary_logdir = kwargs.pop("summary_logdir", None) - if context.executing_eagerly(): - for example in datasets.Iterator(dataset): - self.__call__(example, *args, **kwargs) - return self.all_metric_results(summary_logdir) - # Graph construction - next_value = dataset_ops.make_one_shot_iterator(dataset).get_next() - # Function inlining destroys strict inputs semantics (function body might - # start execution before all inputs are ready). When iterator is exhausted - # and throws out of range error, function body might be partially executed. - # To prevent this we add an explicit control dependency from the 'get_next'. - with ops.control_dependencies([next_value]): - has_next_value = control_flow_ops.no_op(name="iterator_has_next") - with ops.control_dependencies([has_next_value]): - call_op = self.__call__(next_value, *args, **kwargs) - init_op = self.init_variables() - results_op = self.all_metric_results(summary_logdir) - return (init_op, call_op, results_op) - - @staticmethod - def run_evaluation(init_op, call_op, results_op, sess=None): - """Convenience method for running the ops returned by evaluate_on_dataset. - - Args: - init_op: An op that initializes/resets evaluation state. - call_op: An op that updates evaluation state on a mini-batch of examples. - Must generate an tf.errors.OutOfRangeError when done. - results_op: A dictionary of tensors that compute the final evaluation - results from the evaluation state. - sess: The Session to run the evaluation in. Defaults to the default - Session. - - Returns: - A dictionary of values, parallel to results_op. - - Raises: - RuntimeError: if eager execution is enabled. - - @compatibility(eager) - Only for graph execution. - @end_compatibility - """ - if context.executing_eagerly(): - raise RuntimeError("Evaluator.run_evaluation() not supported when " - "eager execution is enabled.") - sess = sess or ops.get_default_session() - sess.run(init_op) - try: - while True: - sess.run(call_op) - except errors_impl.OutOfRangeError: - pass - return sess.run(results_op) - - # ---- To be implemented by descendants --- - def call(self, eval_data): - """Update metrics using the output of self.model. - - Note: This function is executed as a graph function in graph mode. - This means: - a) Operations on the same resource are executed in textual order. - This should make it easier to do things like add the updated - value of a variable to another, for example. - b) You don't need to worry about collecting the update ops to execute. - All update ops added to the graph by this function will be executed. - As a result, code should generally work the same way with graph or - eager execution. - - Args: - eval_data: The output of self.model.eval_data() on a mini-batch of - examples. - """ - raise NotImplementedError("Evaluators must define a call member function.") - - # ---- For use by descendants --- - @property - def model(self): - return self._model - - def track_metric(self, metric): - """Add a Metric to be tracked. - - Metrics can only be tracked by one `Evaluator`. Metrics must be - tracked or they will not appear in `all_metric_results()`. - - Args: - metric: A `Metric` object. - - Returns: - The `metric` passed into this function. - - Raises: - RuntimeError: If called before __init__. - TypeError: If `metric` is not of the correct type. - ValueError: If there is a name collision between Metrics or `metric` - has already been added to another `Evaluator`. - """ - if not hasattr(self, "_metrics"): - raise RuntimeError( - "Need to call Evaluator.__init__ before adding metrics") - if not isinstance(metric, metrics.Metric): - raise TypeError( - "Evaluator.track_metric() passed type %s, not a tfe.metrics.Metric" % - (type(metric),)) - if metric.name in self._metrics: - if metric is self._metrics[metric.name]: - return metric - raise ValueError( - "Attempt to add two Metrics with the name '%s' to the same Evaluator " - "'%s'" % (metric.name, self.name)) - # pylint: disable=protected-access - if hasattr(metric, "_added_to_an_evaluator"): - raise ValueError("Metric %s already added to Evaluator %s" % - (metric.name, metric._added_to_an_evaluator)) - metric._added_to_an_evaluator = self.__class__.__name__ - # pylint: enable=protected-access - self._metrics[metric.name] = metric - return metric - - def track_evaluator(self, prefix, evaluator): - """Add a contained `Evaluator`. - - This is for delegating to another `Evaluator`, e.g. for when you have a - model with multiple heads. Users should manually invoke the child - `Evaluator`'s `call` method from their `call` method. - - Args: - prefix: A string. Metrics from `evaluator` are exported with this - prefix and a '/'. - evaluator: An `Evaluator` object. - - Returns: - The value of `evaluator` passed into this function. - - Raises: - RuntimeError: If called before __init__. - TypeError: If `evaluator` is not of the correct type. - ValueError: If an `Evaluator` has already been added with that `prefix`. - """ - if not hasattr(self, "_evaluators"): - raise RuntimeError( - "Need to call Evaluator.__init__ before adding evaluators") - if not isinstance(evaluator, Evaluator): - raise TypeError( - "Evaluator.track_evaluator() passed type %s, not a tfe.Evaluator." % - (type(evaluator),)) - if prefix in self._evaluators: - if evaluator is self._evaluators[prefix]: - return evaluator - raise RuntimeError( - "Attempt to add two Evaluators with the same prefix '%s'." % prefix) - self._evaluators[prefix] = evaluator - return evaluator - - @property - def metric_variables(self): - v = [] - for metric in six.itervalues(self._metrics): - v += metric.variables - for evaluator in six.itervalues(self._evaluators): - v += evaluator.metric_variables - return v - - @property - def metrics(self): - """Returns a list of (prefix, metric) pairs.""" - m = [] - for metric in six.itervalues(self._metrics): - m.append(("", metric)) - for prefix, evaluator in six.iteritems(self._evaluators): - m += [(prefix + "/" + p, m) for p, m in evaluator.metrics] - return m - - -class SparseSoftmaxEvaluator(Evaluator): - """Evaluator for a sparse softmax model. - - Computes a standard set of metrics for single-label, multi-class - models. - - Args: - model: A `SparseSoftmaxModel` object or a `Model` whose `eval_data()` - method produces a `dict` containing values for the loss, true - label, predicted class, and optional weights. - loss_key: Optional key for looking up the value of the loss in the - `eval_data()` dict. Defaults to "loss". - label_key: Optional key for looking up the value of the label in the - `eval_data()` dict. Defaults to "label". - predicted_class_key: Optional key for looking up the value of the - predicted class in the `eval_data()` dict. Defaults to "predicted_class". - weights_key: Optional key for looking up the value of the weights - in the `eval_data()` dict. Defaults to "weights". Note that weights - are optional, and default to 1 if not present in `eval_data`. - """ - - def __init__(self, model, loss_key="loss", label_key="label", - predicted_class_key="predicted_class", weights_key="weights"): - super(SparseSoftmaxEvaluator, self).__init__(model) - # TODO(josh11b): Expand this to include everything from the standard - # SparseSoftmax Head. - self.avg_loss = self.track_metric(metrics.Mean("Avg Loss")) - self.accuracy = self.track_metric(metrics.Accuracy()) - self.loss_key = loss_key - self.label_key = label_key - self.predicted_class_key = predicted_class_key - self.weights_key = weights_key - - def call(self, eval_data): - """Update metrics for `eval_data` dict (described above).""" - weights = eval_data.get(self.weights_key, None) - if weights is None: - self.avg_loss(eval_data[self.loss_key]) - self.accuracy(eval_data[self.label_key], - eval_data[self.predicted_class_key]) - else: - self.avg_loss(eval_data[self.loss_key], weights=weights) - self.accuracy(eval_data[self.label_key], - eval_data[self.predicted_class_key], - weights=weights) diff --git a/tensorflow/contrib/eager/python/evaluator_test.py b/tensorflow/contrib/eager/python/evaluator_test.py deleted file mode 100644 index 48d093e0754..00000000000 --- a/tensorflow/contrib/eager/python/evaluator_test.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for class Evaluator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tempfile - -from tensorflow.contrib.eager.python import evaluator - -from tensorflow.contrib.eager.python import metrics -from tensorflow.contrib.summary import summary_test_util -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.eager import context -from tensorflow.python.eager import test -from tensorflow.python.framework import ops -from tensorflow.python.ops import variables -from tensorflow.python.training import training_util - - -class IdentityModel(object): - - def eval_data(self, d): - return d - - -class PrefixLModel(object): - - def eval_data(self, d): - return {"l_" + key: d[key] for key in d} - - -class SimpleEvaluator(evaluator.Evaluator): - - def __init__(self, model): - super(SimpleEvaluator, self).__init__(model) - self.mean = self.track_metric(metrics.Mean("mean")) - - def call(self, eval_data): - self.mean(eval_data) - - -class DelegatingEvaluator(evaluator.Evaluator): - - def __init__(self, model): - super(DelegatingEvaluator, self).__init__(model) - self.sub = self.track_evaluator("inner", SimpleEvaluator(model)) - self.mean = self.track_metric(metrics.Mean("outer-mean")) - - def call(self, eval_data): - # Keys here come from PrefixLModel, which adds "l_". - self.mean(eval_data["l_outer"]) - self.sub.call(eval_data["l_inner"]) - - -# pylint: disable=not-callable -class EvaluatorTest(test.TestCase): - - def testSimple(self): - e = SimpleEvaluator(IdentityModel()) - e(3.0) - e([5.0, 7.0, 9.0]) - results = e.all_metric_results() - self.assertEqual(set(["mean"]), set(results.keys())) - self.assertEqual(6.0, results["mean"].numpy()) - - def testWriteSummaries(self): - e = SimpleEvaluator(IdentityModel()) - e(3.0) - e([5.0, 7.0, 9.0]) - training_util.get_or_create_global_step() - logdir = tempfile.mkdtemp() - - e.all_metric_results(logdir) - - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 2) - self.assertEqual(events[1].summary.value[0].simple_value, 6.0) - - def testComposition(self): - e = DelegatingEvaluator(PrefixLModel()) - e({"inner": 2.0, "outer": 100.0}) - e({"inner": 4.0, "outer": 1000.0}) - results = e.all_metric_results() - self.assertEqual(set(["inner/mean", "outer-mean"]), set(results.keys())) - self.assertEqual(3.0, results["inner/mean"].numpy()) - self.assertEqual(550.0, results["outer-mean"].numpy()) - - def testMetricVariables(self): - e = DelegatingEvaluator(PrefixLModel()) - e({"inner": 2.0, "outer": 100.0}) - prefix_count = {} - for v in e.metric_variables: - p = v.name.split("/")[0] - prefix_count[p] = prefix_count.get(p, 0) + 1 - self.assertEqual({"outer_mean": 2, "mean": 2}, prefix_count) - - def testDatasetEager(self): - e = SimpleEvaluator(IdentityModel()) - ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0]) - results = e.evaluate_on_dataset(ds) - self.assertEqual(set(["mean"]), set(results.keys())) - self.assertEqual(6.0, results["mean"].numpy()) - - def testDatasetGraph(self): - with context.graph_mode(), ops.Graph().as_default(), self.cached_session(): - e = SimpleEvaluator(IdentityModel()) - ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0]) - init_op, call_op, results_op = e.evaluate_on_dataset(ds) - results = e.run_evaluation(init_op, call_op, results_op) - self.assertEqual(set(["mean"]), set(results.keys())) - self.assertEqual(6.0, results["mean"]) - - def testWriteSummariesGraph(self): - with context.graph_mode(), ops.Graph().as_default(), self.cached_session(): - e = SimpleEvaluator(IdentityModel()) - ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0]) - training_util.get_or_create_global_step() - logdir = tempfile.mkdtemp() - init_op, call_op, results_op = e.evaluate_on_dataset( - ds, summary_logdir=logdir) - variables.global_variables_initializer().run() - e.run_evaluation(init_op, call_op, results_op) - - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 2) - self.assertEqual(events[1].summary.value[0].simple_value, 6.0) - - def testModelProperty(self): - m = IdentityModel() - e = SimpleEvaluator(m) - self.assertIs(m, e.model) - - def testMetricsProperty(self): - e = DelegatingEvaluator(PrefixLModel()) - names = set([(p, m.name) for p, m in e.metrics]) - self.assertEqual(set([("", "outer-mean"), ("inner/", "mean")]), names) - - def testSharedMetric(self): - - class MetricArgEvaluator(evaluator.Evaluator): - - def __init__(self, model, m): - super(MetricArgEvaluator, self).__init__(model) - self.m = self.track_metric(m) - - metric = metrics.Mean("mean") - model = IdentityModel() - e = MetricArgEvaluator(model, metric) - with self.assertRaisesRegexp(ValueError, "already added"): - MetricArgEvaluator(model, metric) - del e - - def testMetricTrackedTwice(self): - - class MetricTwiceEvaluator(evaluator.Evaluator): - - def __init__(self, model): - super(MetricTwiceEvaluator, self).__init__(model) - self.m = self.track_metric(metrics.Mean("mean")) - self.track_metric(self.m) # okay to track same metric again - - MetricTwiceEvaluator(IdentityModel()) - - -class SparseSoftmaxEvaluatorTest(test.TestCase): - - def testSimple(self): - e = evaluator.SparseSoftmaxEvaluator(IdentityModel()) - e({e.loss_key: 1.0, e.label_key: 5, e.predicted_class_key: 5}) - e({e.loss_key: [0.0, 3.0, 4.0], - e.label_key: [1, 2, 3], - e.predicted_class_key: [1, 1, 3]}) - results = e.all_metric_results() - self.assertEqual(set(["Avg Loss", "Accuracy"]), set(results.keys())) - self.assertEqual(2.0, results["Avg Loss"].numpy()) - self.assertEqual(0.75, results["Accuracy"].numpy()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/eager/python/examples/BUILD b/tensorflow/contrib/eager/python/examples/BUILD deleted file mode 100644 index 14e97a5138f..00000000000 --- a/tensorflow/contrib/eager/python/examples/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -# TensorFlow code for training gradient boosted trees. - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "examples_pip", - deps = [ - "//tensorflow/contrib/eager/python/examples/densenet:densenet_lib", - "//tensorflow/contrib/eager/python/examples/gan:mnist_lib", - "//tensorflow/contrib/eager/python/examples/l2hmc", - "//tensorflow/contrib/eager/python/examples/l2hmc:neural_nets", - "//tensorflow/contrib/eager/python/examples/linear_regression:linear_regression_lib", - "//tensorflow/contrib/eager/python/examples/resnet50", - "//tensorflow/contrib/eager/python/examples/revnet", - "//tensorflow/contrib/eager/python/examples/revnet:config", - "//tensorflow/contrib/eager/python/examples/rnn_colorbot:rnn_colorbot_lib", - "//tensorflow/contrib/eager/python/examples/rnn_ptb:rnn_ptb_lib", - "//tensorflow/contrib/eager/python/examples/spinn:data", - ], -) diff --git a/tensorflow/contrib/eager/python/examples/densenet/BUILD b/tensorflow/contrib/eager/python/examples/densenet/BUILD deleted file mode 100644 index b96f6b9027f..00000000000 --- a/tensorflow/contrib/eager/python/examples/densenet/BUILD +++ /dev/null @@ -1,62 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "py_binary") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -py_binary( - name = "densenet", - srcs = ["densenet.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [":densenet_lib"], -) - -py_library( - name = "densenet_lib", - srcs = ["densenet.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/eager/python:tfe", - ], -) - -cuda_py_test( - name = "densenet_test", - size = "medium", - srcs = ["densenet_test.py"], - additional_deps = [ - ":densenet", - "//tensorflow/contrib/eager/python:tfe", - "//tensorflow:tensorflow_py", - ], - shard_count = 4, - tags = [ - "no_pip", - "optonly", - "oss_serial", - ], -) - -cuda_py_test( - name = "densenet_graph_test", - size = "medium", - srcs = ["densenet_graph_test.py"], - additional_deps = [ - ":densenet", - "//third_party/py/numpy", - "//tensorflow:tensorflow_py", - ], - shard_count = 4, - tags = [ - "no_pip", - "noasan", - "nomsan", - "notsan", - "optonly", - "oss_serial", - ], -) diff --git a/tensorflow/contrib/eager/python/examples/densenet/densenet.py b/tensorflow/contrib/eager/python/examples/densenet/densenet.py deleted file mode 100644 index 6de4e694009..00000000000 --- a/tensorflow/contrib/eager/python/examples/densenet/densenet.py +++ /dev/null @@ -1,296 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Densely Connected Convolutional Networks. - -Reference [ -Densely Connected Convolutional Networks](https://arxiv.org/abs/1608.06993) - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf -l2 = tf.keras.regularizers.l2 - - -class ConvBlock(tf.keras.Model): - """Convolutional Block consisting of (batchnorm->relu->conv). - - Arguments: - num_filters: number of filters passed to a convolutional layer. - data_format: "channels_first" or "channels_last" - bottleneck: if True, then a 1x1 Conv is performed followed by 3x3 Conv. - weight_decay: weight decay - dropout_rate: dropout rate. - """ - - def __init__(self, num_filters, data_format, bottleneck, weight_decay=1e-4, - dropout_rate=0): - super(ConvBlock, self).__init__() - self.bottleneck = bottleneck - - axis = -1 if data_format == "channels_last" else 1 - inter_filter = num_filters * 4 - # don't forget to set use_bias=False when using batchnorm - self.conv2 = tf.keras.layers.Conv2D(num_filters, - (3, 3), - padding="same", - use_bias=False, - data_format=data_format, - kernel_initializer="he_normal", - kernel_regularizer=l2(weight_decay)) - self.batchnorm1 = tf.keras.layers.BatchNormalization(axis=axis) - self.dropout = tf.keras.layers.Dropout(dropout_rate) - - if self.bottleneck: - self.conv1 = tf.keras.layers.Conv2D(inter_filter, - (1, 1), - padding="same", - use_bias=False, - data_format=data_format, - kernel_initializer="he_normal", - kernel_regularizer=l2(weight_decay)) - self.batchnorm2 = tf.keras.layers.BatchNormalization(axis=axis) - - def call(self, x, training=True): - output = self.batchnorm1(x, training=training) - - if self.bottleneck: - output = self.conv1(tf.nn.relu(output)) - output = self.batchnorm2(output, training=training) - - output = self.conv2(tf.nn.relu(output)) - output = self.dropout(output, training=training) - - return output - - -class TransitionBlock(tf.keras.Model): - """Transition Block to reduce the number of features. - - Arguments: - num_filters: number of filters passed to a convolutional layer. - data_format: "channels_first" or "channels_last" - weight_decay: weight decay - dropout_rate: dropout rate. - """ - - def __init__(self, num_filters, data_format, - weight_decay=1e-4, dropout_rate=0): - super(TransitionBlock, self).__init__() - axis = -1 if data_format == "channels_last" else 1 - - self.batchnorm = tf.keras.layers.BatchNormalization(axis=axis) - self.conv = tf.keras.layers.Conv2D(num_filters, - (1, 1), - padding="same", - use_bias=False, - data_format=data_format, - kernel_initializer="he_normal", - kernel_regularizer=l2(weight_decay)) - self.avg_pool = tf.keras.layers.AveragePooling2D(data_format=data_format) - - def call(self, x, training=True): - output = self.batchnorm(x, training=training) - output = self.conv(tf.nn.relu(output)) - output = self.avg_pool(output) - return output - - -class DenseBlock(tf.keras.Model): - """Dense Block consisting of ConvBlocks where each block's - output is concatenated with its input. - - Arguments: - num_layers: Number of layers in each block. - growth_rate: number of filters to add per conv block. - data_format: "channels_first" or "channels_last" - bottleneck: boolean, that decides which part of ConvBlock to call. - weight_decay: weight decay - dropout_rate: dropout rate. - """ - - def __init__(self, num_layers, growth_rate, data_format, bottleneck, - weight_decay=1e-4, dropout_rate=0): - super(DenseBlock, self).__init__() - self.num_layers = num_layers - self.axis = -1 if data_format == "channels_last" else 1 - - self.blocks = [] - for _ in range(int(self.num_layers)): - self.blocks.append(ConvBlock(growth_rate, - data_format, - bottleneck, - weight_decay, - dropout_rate)) - - def call(self, x, training=True): - for i in range(int(self.num_layers)): - output = self.blocks[i](x, training=training) - x = tf.concat([x, output], axis=self.axis) - - return x - - -class DenseNet(tf.keras.Model): - """Creating the Densenet Architecture. - - Arguments: - depth_of_model: number of layers in the model. - growth_rate: number of filters to add per conv block. - num_of_blocks: number of dense blocks. - output_classes: number of output classes. - num_layers_in_each_block: number of layers in each block. - If -1, then we calculate this by (depth-3)/4. - If positive integer, then the it is used as the - number of layers per block. - If list or tuple, then this list is used directly. - data_format: "channels_first" or "channels_last" - bottleneck: boolean, to decide which part of conv block to call. - compression: reducing the number of inputs(filters) to the transition block. - weight_decay: weight decay - rate: dropout rate. - pool_initial: If True add a 7x7 conv with stride 2 followed by 3x3 maxpool - else, do a 3x3 conv with stride 1. - include_top: If true, GlobalAveragePooling Layer and Dense layer are - included. - """ - - def __init__(self, depth_of_model, growth_rate, num_of_blocks, - output_classes, num_layers_in_each_block, data_format, - bottleneck=True, compression=0.5, weight_decay=1e-4, - dropout_rate=0, pool_initial=False, include_top=True): - super(DenseNet, self).__init__() - self.depth_of_model = depth_of_model - self.growth_rate = growth_rate - self.num_of_blocks = num_of_blocks - self.output_classes = output_classes - self.num_layers_in_each_block = num_layers_in_each_block - self.data_format = data_format - self.bottleneck = bottleneck - self.compression = compression - self.weight_decay = weight_decay - self.dropout_rate = dropout_rate - self.pool_initial = pool_initial - self.include_top = include_top - - # deciding on number of layers in each block - if isinstance(self.num_layers_in_each_block, list) or isinstance( - self.num_layers_in_each_block, tuple): - self.num_layers_in_each_block = list(self.num_layers_in_each_block) - else: - if self.num_layers_in_each_block == -1: - if self.num_of_blocks != 3: - raise ValueError( - "Number of blocks must be 3 if num_layers_in_each_block is -1") - if (self.depth_of_model - 4) % 3 == 0: - num_layers = (self.depth_of_model - 4) / 3 - if self.bottleneck: - num_layers //= 2 - self.num_layers_in_each_block = [num_layers] * self.num_of_blocks - else: - raise ValueError("Depth must be 3N+4 if num_layer_in_each_block=-1") - else: - self.num_layers_in_each_block = [ - self.num_layers_in_each_block] * self.num_of_blocks - - axis = -1 if self.data_format == "channels_last" else 1 - - # setting the filters and stride of the initial covn layer. - if self.pool_initial: - init_filters = (7, 7) - stride = (2, 2) - else: - init_filters = (3, 3) - stride = (1, 1) - - self.num_filters = 2 * self.growth_rate - - # first conv and pool layer - self.conv1 = tf.keras.layers.Conv2D(self.num_filters, - init_filters, - strides=stride, - padding="same", - use_bias=False, - data_format=self.data_format, - kernel_initializer="he_normal", - kernel_regularizer=l2( - self.weight_decay)) - if self.pool_initial: - self.pool1 = tf.keras.layers.MaxPooling2D(pool_size=(3, 3), - strides=(2, 2), - padding="same", - data_format=self.data_format) - self.batchnorm1 = tf.keras.layers.BatchNormalization(axis=axis) - - self.batchnorm2 = tf.keras.layers.BatchNormalization(axis=axis) - - # last pooling and fc layer - if self.include_top: - self.last_pool = tf.keras.layers.GlobalAveragePooling2D( - data_format=self.data_format) - self.classifier = tf.keras.layers.Dense(self.output_classes) - - # calculating the number of filters after each block - num_filters_after_each_block = [self.num_filters] - for i in range(1, self.num_of_blocks): - temp_num_filters = num_filters_after_each_block[i-1] + ( - self.growth_rate * self.num_layers_in_each_block[i-1]) - # using compression to reduce the number of inputs to the - # transition block - temp_num_filters = int(temp_num_filters * compression) - num_filters_after_each_block.append(temp_num_filters) - - # dense block initialization - self.dense_blocks = [] - self.transition_blocks = [] - for i in range(self.num_of_blocks): - self.dense_blocks.append(DenseBlock(self.num_layers_in_each_block[i], - self.growth_rate, - self.data_format, - self.bottleneck, - self.weight_decay, - self.dropout_rate)) - if i+1 < self.num_of_blocks: - self.transition_blocks.append( - TransitionBlock(num_filters_after_each_block[i+1], - self.data_format, - self.weight_decay, - self.dropout_rate)) - - def call(self, x, training=True): - output = self.conv1(x) - - if self.pool_initial: - output = self.batchnorm1(output, training=training) - output = tf.nn.relu(output) - output = self.pool1(output) - - for i in range(self.num_of_blocks - 1): - output = self.dense_blocks[i](output, training=training) - output = self.transition_blocks[i](output, training=training) - - output = self.dense_blocks[ - self.num_of_blocks - 1](output, training=training) - output = self.batchnorm2(output, training=training) - output = tf.nn.relu(output) - - if self.include_top: - output = self.last_pool(output) - output = self.classifier(output) - - return output diff --git a/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py b/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py deleted file mode 100644 index 24f6b007b52..00000000000 --- a/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests and Benchmarks for Densenet model under graph execution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time -import numpy as np -import tensorflow as tf - -from tensorflow.contrib.eager.python.examples.densenet import densenet - - -def data_format(): - return 'channels_first' if tf.test.is_gpu_available() else 'channels_last' - - -def image_shape(batch_size): - if data_format() == 'channels_first': - return [batch_size, 3, 224, 224] - return [batch_size, 224, 224, 3] - - -def random_batch(batch_size): - images = np.random.rand(*image_shape(batch_size)).astype(np.float32) - num_classes = 1000 - labels = np.random.randint( - low=0, high=num_classes, size=[batch_size]).astype(np.int32) - one_hot = np.zeros((batch_size, num_classes)).astype(np.float32) - one_hot[np.arange(batch_size), labels] = 1. - return images, one_hot - - -class DensenetGraphTest(tf.test.TestCase): - - def testApply(self): - depth = 7 - growth_rate = 2 - num_blocks = 3 - output_classes = 10 - num_layers_in_each_block = -1 - batch_size = 1 - with tf.Graph().as_default(): - images = tf.placeholder(tf.float32, image_shape(None)) - model = densenet.DenseNet(depth, growth_rate, num_blocks, - output_classes, num_layers_in_each_block, - data_format(), bottleneck=True, compression=0.5, - weight_decay=1e-4, dropout_rate=0, - pool_initial=False, include_top=True) - predictions = model(images, training=False) - - init = tf.global_variables_initializer() - - with tf.Session() as sess: - sess.run(init) - np_images, _ = random_batch(batch_size) - out = sess.run(predictions, feed_dict={images: np_images}) - self.assertAllEqual([batch_size, output_classes], out.shape) - - -class DensenetBenchmark(tf.test.Benchmark): - - def __init__(self): - self.depth = 121 - self.growth_rate = 32 - self.num_blocks = 4 - self.output_classes = 1000 - self.num_layers_in_each_block = [6, 12, 24, 16] - - def _report(self, label, start, num_iters, batch_size): - avg_time = (time.time() - start) / num_iters - dev = 'gpu' if tf.test.is_gpu_available() else 'cpu' - name = 'graph_%s_%s_batch_%d_%s' % (label, dev, batch_size, data_format()) - extras = {'examples_per_sec': batch_size / avg_time} - self.report_benchmark( - iters=num_iters, wall_time=avg_time, name=name, extras=extras) - - def benchmark_graph_apply(self): - with tf.Graph().as_default(): - images = tf.placeholder(tf.float32, image_shape(None)) - model = densenet.DenseNet(self.depth, self.growth_rate, self.num_blocks, - self.output_classes, - self.num_layers_in_each_block, data_format(), - bottleneck=True, compression=0.5, - weight_decay=1e-4, dropout_rate=0, - pool_initial=True, include_top=True) - predictions = model(images, training=False) - - init = tf.global_variables_initializer() - - batch_size = 64 - with tf.Session() as sess: - sess.run(init) - np_images, _ = random_batch(batch_size) - num_burn, num_iters = (3, 30) - for _ in range(num_burn): - sess.run(predictions, feed_dict={images: np_images}) - start = time.time() - for _ in range(num_iters): - sess.run(predictions, feed_dict={images: np_images}) - self._report('apply', start, num_iters, batch_size) - - def benchmark_graph_train(self): - for batch_size in [16, 32, 64]: - with tf.Graph().as_default(): - np_images, np_labels = random_batch(batch_size) - dataset = tf.data.Dataset.from_tensors((np_images, np_labels)).repeat() - (images, labels) = tf.compat.v1.data.make_one_shot_iterator( - dataset).get_next() - - model = densenet.DenseNet(self.depth, self.growth_rate, self.num_blocks, - self.output_classes, - self.num_layers_in_each_block, data_format(), - bottleneck=True, compression=0.5, - weight_decay=1e-4, dropout_rate=0, - pool_initial=True, include_top=True) - logits = model(images, training=True) - cross_ent = tf.losses.softmax_cross_entropy( - logits=logits, onehot_labels=labels) - regularization = tf.add_n(model.losses) - loss = cross_ent + regularization - optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) - train_op = optimizer.minimize(loss) - - init = tf.global_variables_initializer() - with tf.Session() as sess: - sess.run(init) - (num_burn, num_iters) = (5, 10) - for _ in range(num_burn): - sess.run(train_op) - start = time.time() - for _ in range(num_iters): - sess.run(train_op) - self._report('train', start, num_iters, batch_size) - - -if __name__ == '__main__': - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/densenet/densenet_test.py b/tensorflow/contrib/eager/python/examples/densenet/densenet_test.py deleted file mode 100644 index a9fb0035d29..00000000000 --- a/tensorflow/contrib/eager/python/examples/densenet/densenet_test.py +++ /dev/null @@ -1,350 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests and Benchmarks for Densenet model.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import gc -import time -import tensorflow as tf -import tensorflow.contrib.eager as tfe - -from tensorflow.contrib.eager.python.examples.densenet import densenet -from tensorflow.python.client import device_lib - - -class DensenetTest(tf.test.TestCase): - - def test_bottleneck_true(self): - depth = 7 - growth_rate = 2 - num_blocks = 3 - output_classes = 10 - num_layers_in_each_block = -1 - batch_size = 1 - data_format = ('channels_first') if tf.test.is_gpu_available() else ( - 'channels_last') - - model = densenet.DenseNet(depth, growth_rate, num_blocks, - output_classes, num_layers_in_each_block, - data_format, bottleneck=True, compression=0.5, - weight_decay=1e-4, dropout_rate=0, - pool_initial=False, include_top=True) - - if data_format == 'channels_last': - rand_input = tf.random_uniform((batch_size, 32, 32, 3)) - else: - rand_input = tf.random_uniform((batch_size, 3, 32, 32)) - output_shape = model(rand_input).shape - self.assertEqual(output_shape, (batch_size, output_classes)) - - def test_bottleneck_false(self): - depth = 7 - growth_rate = 2 - num_blocks = 3 - output_classes = 10 - num_layers_in_each_block = -1 - batch_size = 1 - data_format = ('channels_first') if tf.test.is_gpu_available() else ( - 'channels_last') - - model = densenet.DenseNet(depth, growth_rate, num_blocks, - output_classes, num_layers_in_each_block, - data_format, bottleneck=False, compression=0.5, - weight_decay=1e-4, dropout_rate=0, - pool_initial=False, include_top=True) - - if data_format == 'channels_last': - rand_input = tf.random_uniform((batch_size, 32, 32, 3)) - else: - rand_input = tf.random_uniform((batch_size, 3, 32, 32)) - output_shape = model(rand_input).shape - self.assertEqual(output_shape, (batch_size, output_classes)) - - def test_pool_initial_true(self): - depth = 7 - growth_rate = 2 - num_blocks = 4 - output_classes = 10 - num_layers_in_each_block = [1, 2, 2, 1] - batch_size = 1 - data_format = ('channels_first') if tf.test.is_gpu_available() else ( - 'channels_last') - - model = densenet.DenseNet(depth, growth_rate, num_blocks, - output_classes, num_layers_in_each_block, - data_format, bottleneck=True, compression=0.5, - weight_decay=1e-4, dropout_rate=0, - pool_initial=True, include_top=True) - - if data_format == 'channels_last': - rand_input = tf.random_uniform((batch_size, 32, 32, 3)) - else: - rand_input = tf.random_uniform((batch_size, 3, 32, 32)) - output_shape = model(rand_input).shape - self.assertEqual(output_shape, (batch_size, output_classes)) - - def test_regularization(self): - if tf.test.is_gpu_available(): - rand_input = tf.random_uniform((10, 3, 32, 32)) - data_format = 'channels_first' - else: - rand_input = tf.random_uniform((10, 32, 32, 3)) - data_format = 'channels_last' - weight_decay = 1e-4 - - conv = tf.keras.layers.Conv2D( - 3, (3, 3), - padding='same', - use_bias=False, - data_format=data_format, - kernel_regularizer=tf.keras.regularizers.l2(weight_decay)) - optimizer = tf.train.GradientDescentOptimizer(0.1) - conv(rand_input) # Initialize the variables in the layer - - def compute_true_l2(vs, wd): - return tf.reduce_sum(tf.square(vs)) * wd - - true_l2 = compute_true_l2(conv.variables, weight_decay) - keras_l2 = tf.add_n(conv.losses) - self.assertAllClose(true_l2, keras_l2) - - with tf.GradientTape() as tape_true, tf.GradientTape() as tape_keras: - loss = tf.reduce_sum(conv(rand_input)) - loss_with_true_l2 = loss + compute_true_l2(conv.variables, weight_decay) - loss_with_keras_l2 = loss + tf.add_n(conv.losses) - - true_grads = tape_true.gradient(loss_with_true_l2, conv.variables) - keras_grads = tape_keras.gradient(loss_with_keras_l2, conv.variables) - self.assertAllClose(true_grads, keras_grads) - - optimizer.apply_gradients(zip(keras_grads, conv.variables)) - keras_l2_after_update = tf.add_n(conv.losses) - self.assertNotAllClose(keras_l2, keras_l2_after_update) - - -def compute_gradients(model, images, labels): - with tf.GradientTape() as tape: - logits = model(images, training=True) - cross_ent = tf.losses.softmax_cross_entropy( - logits=logits, onehot_labels=labels) - regularization = tf.add_n(model.losses) - loss = cross_ent + regularization - tf.contrib.summary.scalar(name='loss', tensor=loss) - return tape.gradient(loss, model.variables) - - -def apply_gradients(model, optimizer, gradients): - optimizer.apply_gradients(zip(gradients, model.variables)) - - -def device_and_data_format(): - return ('/gpu:0', - 'channels_first') if tf.test.is_gpu_available() else ('/cpu:0', - 'channels_last') - - -def random_batch(batch_size, data_format): - shape = (3, 224, 224) if data_format == 'channels_first' else (224, 224, 3) - shape = (batch_size,) + shape - - num_classes = 1000 - images = tf.random_uniform(shape) - labels = tf.random_uniform( - [batch_size], minval=0, maxval=num_classes, dtype=tf.int32) - one_hot = tf.one_hot(labels, num_classes) - - return images, one_hot - - -class MockIterator(object): - - def __init__(self, tensors): - self._tensors = [tf.identity(x) for x in tensors] - - def next(self): - return self._tensors - - -class DensenetBenchmark(tf.test.Benchmark): - - def __init__(self): - self.depth = 121 - self.growth_rate = 32 - self.num_blocks = 4 - self.output_classes = 1000 - self.num_layers_in_each_block = [6, 12, 24, 16] - - def _train_batch_sizes(self): - """Choose batch sizes based on GPU capability.""" - for device in device_lib.list_local_devices(): - if tf.DeviceSpec.from_string(device.name).device_type == 'GPU': - if 'K20' in device.physical_device_desc: - return (16,) - if 'P100' in device.physical_device_desc: - return (16, 32, 64) - - if tf.DeviceSpec.from_string(device.name).device_type == 'TPU': - return (32,) - return (16, 32) - - def _report(self, label, start, num_iters, device, batch_size, data_format): - avg_time = (time.time() - start) / num_iters - dev = tf.DeviceSpec.from_string(device).device_type.lower() - name = '%s_%s_batch_%d_%s' % (label, dev, batch_size, data_format) - extras = {'examples_per_sec': batch_size / avg_time} - self.report_benchmark( - iters=num_iters, wall_time=avg_time, name=name, extras=extras) - - def _force_device_sync(self): - # If this function is called in the context of a non-CPU device - # (e.g., inside a 'with tf.device("/gpu:0")' block) - # then this will force a copy from CPU->NON_CPU_DEVICE->CPU, - # which forces a sync. This is a roundabout way, yes. - tf.constant(1.).cpu() - - def _benchmark_eager_apply(self, label, device_and_format, defun=False, - execution_mode=None): - with tfe.execution_mode(execution_mode): - device, data_format = device_and_format - model = densenet.DenseNet(self.depth, self.growth_rate, self.num_blocks, - self.output_classes, - self.num_layers_in_each_block, data_format, - bottleneck=True, compression=0.5, - weight_decay=1e-4, dropout_rate=0, - pool_initial=True, include_top=True) - if defun: - # TODO(apassos) enable tfe.function here - model.call = tfe.defun(model.call) - batch_size = 64 - num_burn = 5 - num_iters = 30 - with tf.device(device): - images, _ = random_batch(batch_size, data_format) - for _ in xrange(num_burn): - model(images, training=False).cpu() - if execution_mode: - tfe.async_wait() - gc.collect() - start = time.time() - for _ in xrange(num_iters): - model(images, training=False).cpu() - if execution_mode: - tfe.async_wait() - self._report(label, start, num_iters, device, batch_size, data_format) - - def benchmark_eager_apply_sync(self): - self._benchmark_eager_apply('eager_apply', device_and_data_format(), - defun=False) - - def benchmark_eager_apply_async(self): - self._benchmark_eager_apply( - 'eager_apply_async', device_and_data_format(), defun=False, - execution_mode=tfe.ASYNC) - - def benchmark_eager_apply_with_defun(self): - self._benchmark_eager_apply('eager_apply_with_defun', - device_and_data_format(), defun=True) - - def _benchmark_eager_train(self, - label, - make_iterator, - device_and_format, - defun=False, - execution_mode=None): - with tfe.execution_mode(execution_mode): - device, data_format = device_and_format - for batch_size in self._train_batch_sizes(): - (images, labels) = random_batch(batch_size, data_format) - model = densenet.DenseNet(self.depth, self.growth_rate, self.num_blocks, - self.output_classes, - self.num_layers_in_each_block, data_format, - bottleneck=True, compression=0.5, - weight_decay=1e-4, dropout_rate=0, - pool_initial=True, include_top=True) - optimizer = tf.train.GradientDescentOptimizer(0.1) - apply_grads = apply_gradients - if defun: - model.call = tfe.defun(model.call) - apply_grads = tfe.defun(apply_gradients) - - num_burn = 3 - num_iters = 10 - with tf.device(device): - iterator = make_iterator((images, labels)) - for _ in xrange(num_burn): - (images, labels) = iterator.next() - apply_grads(model, optimizer, - compute_gradients(model, images, labels)) - if execution_mode: - tfe.async_wait() - self._force_device_sync() - gc.collect() - - start = time.time() - for _ in xrange(num_iters): - (images, labels) = iterator.next() - apply_grads(model, optimizer, - compute_gradients(model, images, labels)) - if execution_mode: - tfe.async_wait() - self._force_device_sync() - self._report(label, start, num_iters, device, batch_size, data_format) - - def benchmark_eager_train_sync(self): - self._benchmark_eager_train('eager_train', MockIterator, - device_and_data_format(), defun=False) - - def benchmark_eager_train_async(self): - self._benchmark_eager_train( - 'eager_train_async', - MockIterator, - device_and_data_format(), - defun=False, - execution_mode=tfe.ASYNC) - - def benchmark_eager_train_with_defun(self): - self._benchmark_eager_train( - 'eager_train_with_defun', MockIterator, - device_and_data_format(), defun=True) - - def benchmark_eager_train_datasets(self): - - def make_iterator(tensors): - with tf.device('/device:CPU:0'): - ds = tf.data.Dataset.from_tensors(tensors).repeat() - return tfe.Iterator(ds) - - self._benchmark_eager_train( - 'eager_train_dataset', make_iterator, - device_and_data_format(), defun=False) - - def benchmark_eager_train_datasets_with_defun(self): - - def make_iterator(tensors): - with tf.device('/device:CPU:0'): - ds = tf.data.Dataset.from_tensors(tensors).repeat() - return tfe.Iterator(ds) - - self._benchmark_eager_train( - 'eager_train_dataset_with_defun', make_iterator, - device_and_data_format(), defun=True) - - -if __name__ == '__main__': - tf.enable_eager_execution() - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/gan/BUILD b/tensorflow/contrib/eager/python/examples/gan/BUILD deleted file mode 100644 index a777374df83..00000000000 --- a/tensorflow/contrib/eager/python/examples/gan/BUILD +++ /dev/null @@ -1,46 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "py_binary") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -py_binary( - name = "mnist", - srcs = ["mnist.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [":mnist_lib"], -) - -py_library( - name = "mnist_lib", - srcs = ["mnist.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/eager/python:tfe", - "//tensorflow/examples/tutorials/mnist:input_data", - ], -) - -cuda_py_test( - name = "mnist_test", - srcs = ["mnist_test.py"], - additional_deps = [ - ":mnist_lib", - "//tensorflow/contrib/eager/python:tfe", - "//tensorflow:tensorflow_py", - ], -) - -cuda_py_test( - name = "mnist_graph_test", - srcs = ["mnist_graph_test.py"], - additional_deps = [ - ":mnist_lib", - "//third_party/py/numpy", - "//tensorflow:tensorflow_py", - ], -) diff --git a/tensorflow/contrib/eager/python/examples/gan/README.md b/tensorflow/contrib/eager/python/examples/gan/README.md deleted file mode 100644 index 208a64b05d4..00000000000 --- a/tensorflow/contrib/eager/python/examples/gan/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# GAN with TensorFlow eager execution - -A simple Generative Adversarial Network (GAN) example using eager execution. -The discriminator and generator networks each contain a few convolution and -fully connected layers. - -Other eager execution examples can be found under the parent directory. - -## Content - -- `mnist.py`: Model definitions and training routines. -- `mnist_test.py`: Benchmarks for training and using the models using eager -execution. -- `mnist_graph_test.py`: Benchmarks for training and using the models using -graph execution. The same model definitions and loss functions are used in -all benchmarks. - - -## To run - -- Make sure you have installed TensorFlow 1.5+ or the latest `tf-nightly` -or `tf-nightly-gpu` pip package in order to access the eager execution feature. - -- Train model. E.g., - - ```bash - python mnist.py - ``` - - Use `--output_dir=` to direct the script to save TensorBoard summaries - during training. Disabled by default. - - Use `--checkpoint_dir=` to direct the script to save checkpoints to - `` during training. DIR defaults to /tmp/tensorflow/mnist/checkpoints/. - The script will load the latest saved checkpoint from this directory if - one exists. - - Use `-h` for other options. diff --git a/tensorflow/contrib/eager/python/examples/gan/mnist.py b/tensorflow/contrib/eager/python/examples/gan/mnist.py deleted file mode 100644 index 697ff73d006..00000000000 --- a/tensorflow/contrib/eager/python/examples/gan/mnist.py +++ /dev/null @@ -1,366 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A deep MNIST classifier using convolutional layers. - -Sample usage: - python mnist.py --help -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import os -import sys -import time - -import tensorflow as tf - -from tensorflow.examples.tutorials.mnist import input_data - -layers = tf.keras.layers -FLAGS = None - - -class Discriminator(tf.keras.Model): - """GAN Discriminator. - - A network to differentiate between generated and real handwritten digits. - """ - - def __init__(self, data_format): - """Creates a model for discriminating between real and generated digits. - - Args: - data_format: Either 'channels_first' or 'channels_last'. - 'channels_first' is typically faster on GPUs while 'channels_last' is - typically faster on CPUs. See - https://www.tensorflow.org/performance/performance_guide#data_formats - """ - super(Discriminator, self).__init__(name='') - if data_format == 'channels_first': - self._input_shape = [-1, 1, 28, 28] - else: - assert data_format == 'channels_last' - self._input_shape = [-1, 28, 28, 1] - self.conv1 = layers.Conv2D( - 64, 5, padding='SAME', data_format=data_format, activation=tf.tanh) - self.pool1 = layers.AveragePooling2D(2, 2, data_format=data_format) - self.conv2 = layers.Conv2D( - 128, 5, data_format=data_format, activation=tf.tanh) - self.pool2 = layers.AveragePooling2D(2, 2, data_format=data_format) - self.flatten = layers.Flatten() - self.fc1 = layers.Dense(1024, activation=tf.tanh) - self.fc2 = layers.Dense(1, activation=None) - - def call(self, inputs): - """Return two logits per image estimating input authenticity. - - Users should invoke __call__ to run the network, which delegates to this - method (and not call this method directly). - - Args: - inputs: A batch of images as a Tensor with shape [batch_size, 28, 28, 1] - or [batch_size, 1, 28, 28] - - Returns: - A Tensor with shape [batch_size] containing logits estimating - the probability that corresponding digit is real. - """ - x = tf.reshape(inputs, self._input_shape) - x = self.conv1(x) - x = self.pool1(x) - x = self.conv2(x) - x = self.pool2(x) - x = self.flatten(x) - x = self.fc1(x) - x = self.fc2(x) - return x - - -class Generator(tf.keras.Model): - """Generator of handwritten digits similar to the ones in the MNIST dataset. - """ - - def __init__(self, data_format): - """Creates a model for discriminating between real and generated digits. - - Args: - data_format: Either 'channels_first' or 'channels_last'. - 'channels_first' is typically faster on GPUs while 'channels_last' is - typically faster on CPUs. See - https://www.tensorflow.org/performance/performance_guide#data_formats - """ - super(Generator, self).__init__(name='') - self.data_format = data_format - # We are using 128 6x6 channels as input to the first deconvolution layer - if data_format == 'channels_first': - self._pre_conv_shape = [-1, 128, 6, 6] - else: - assert data_format == 'channels_last' - self._pre_conv_shape = [-1, 6, 6, 128] - self.fc1 = layers.Dense(6 * 6 * 128, activation=tf.tanh) - - # In call(), we reshape the output of fc1 to _pre_conv_shape - - # Deconvolution layer. Resulting image shape: (batch, 14, 14, 64) - self.conv1 = layers.Conv2DTranspose( - 64, 4, strides=2, activation=None, data_format=data_format) - - # Deconvolution layer. Resulting image shape: (batch, 28, 28, 1) - self.conv2 = layers.Conv2DTranspose( - 1, 2, strides=2, activation=tf.nn.sigmoid, data_format=data_format) - - def call(self, inputs): - """Return a batch of generated images. - - Users should invoke __call__ to run the network, which delegates to this - method (and not call this method directly). - - Args: - inputs: A batch of noise vectors as a Tensor with shape - [batch_size, length of noise vectors]. - - Returns: - A Tensor containing generated images. If data_format is 'channels_last', - the shape of returned images is [batch_size, 28, 28, 1], else - [batch_size, 1, 28, 28] - """ - - x = self.fc1(inputs) - x = tf.reshape(x, shape=self._pre_conv_shape) - x = self.conv1(x) - x = self.conv2(x) - return x - - -def discriminator_loss(discriminator_real_outputs, discriminator_gen_outputs): - """Original discriminator loss for GANs, with label smoothing. - - See `Generative Adversarial Nets` (https://arxiv.org/abs/1406.2661) for more - details. - - Args: - discriminator_real_outputs: Discriminator output on real data. - discriminator_gen_outputs: Discriminator output on generated data. Expected - to be in the range of (-inf, inf). - - Returns: - A scalar loss Tensor. - """ - - loss_on_real = tf.losses.sigmoid_cross_entropy( - tf.ones_like(discriminator_real_outputs), - discriminator_real_outputs, - label_smoothing=0.25) - loss_on_generated = tf.losses.sigmoid_cross_entropy( - tf.zeros_like(discriminator_gen_outputs), discriminator_gen_outputs) - loss = loss_on_real + loss_on_generated - tf.contrib.summary.scalar('discriminator_loss', loss) - return loss - - -def generator_loss(discriminator_gen_outputs): - """Original generator loss for GANs. - - L = -log(sigmoid(D(G(z)))) - - See `Generative Adversarial Nets` (https://arxiv.org/abs/1406.2661) - for more details. - - Args: - discriminator_gen_outputs: Discriminator output on generated data. Expected - to be in the range of (-inf, inf). - - Returns: - A scalar loss Tensor. - """ - loss = tf.losses.sigmoid_cross_entropy( - tf.ones_like(discriminator_gen_outputs), discriminator_gen_outputs) - tf.contrib.summary.scalar('generator_loss', loss) - return loss - - -def train_one_epoch(generator, discriminator, generator_optimizer, - discriminator_optimizer, dataset, step_counter, - log_interval, noise_dim): - """Trains `generator` and `discriminator` models on `dataset`. - - Args: - generator: Generator model. - discriminator: Discriminator model. - generator_optimizer: Optimizer to use for generator. - discriminator_optimizer: Optimizer to use for discriminator. - dataset: Dataset of images to train on. - step_counter: An integer variable, used to write summaries regularly. - log_interval: How many steps to wait between logging and collecting - summaries. - noise_dim: Dimension of noise vector to use. - """ - - total_generator_loss = 0.0 - total_discriminator_loss = 0.0 - for (batch_index, images) in enumerate(dataset): - with tf.device('/cpu:0'): - tf.assign_add(step_counter, 1) - - with tf.contrib.summary.record_summaries_every_n_global_steps( - log_interval, global_step=step_counter): - current_batch_size = images.shape[0] - noise = tf.random_uniform( - shape=[current_batch_size, noise_dim], - minval=-1., - maxval=1., - seed=batch_index) - - # we can use 2 tapes or a single persistent tape. - # Using two tapes is memory efficient since intermediate tensors can be - # released between the two .gradient() calls below - with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape: - generated_images = generator(noise) - tf.contrib.summary.image( - 'generated_images', - tf.reshape(generated_images, [-1, 28, 28, 1]), - max_images=10) - - discriminator_gen_outputs = discriminator(generated_images) - discriminator_real_outputs = discriminator(images) - discriminator_loss_val = discriminator_loss(discriminator_real_outputs, - discriminator_gen_outputs) - total_discriminator_loss += discriminator_loss_val - - generator_loss_val = generator_loss(discriminator_gen_outputs) - total_generator_loss += generator_loss_val - - generator_grad = gen_tape.gradient(generator_loss_val, - generator.variables) - discriminator_grad = disc_tape.gradient(discriminator_loss_val, - discriminator.variables) - - generator_optimizer.apply_gradients( - zip(generator_grad, generator.variables)) - discriminator_optimizer.apply_gradients( - zip(discriminator_grad, discriminator.variables)) - - if log_interval and batch_index > 0 and batch_index % log_interval == 0: - print('Batch #%d\tAverage Generator Loss: %.6f\t' - 'Average Discriminator Loss: %.6f' % - (batch_index, total_generator_loss / batch_index, - total_discriminator_loss / batch_index)) - - -def main(_): - (device, data_format) = ('/gpu:0', 'channels_first') - if FLAGS.no_gpu or tf.contrib.eager.num_gpus() <= 0: - (device, data_format) = ('/cpu:0', 'channels_last') - print('Using device %s, and data format %s.' % (device, data_format)) - - # Load the datasets - data = input_data.read_data_sets(FLAGS.data_dir) - dataset = ( - tf.data.Dataset.from_tensor_slices(data.train.images).shuffle(60000) - .batch(FLAGS.batch_size)) - - # Create the models and optimizers. - model_objects = { - 'generator': Generator(data_format), - 'discriminator': Discriminator(data_format), - 'generator_optimizer': tf.compat.v1.train.AdamOptimizer(FLAGS.lr), - 'discriminator_optimizer': tf.compat.v1.train.AdamOptimizer(FLAGS.lr), - 'step_counter': tf.train.get_or_create_global_step(), - } - - # Prepare summary writer and checkpoint info - summary_writer = tf.contrib.summary.create_summary_file_writer( - FLAGS.output_dir, flush_millis=1000) - checkpoint_prefix = os.path.join(FLAGS.checkpoint_dir, 'ckpt') - latest_cpkt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) - if latest_cpkt: - print('Using latest checkpoint at ' + latest_cpkt) - checkpoint = tf.train.Checkpoint(**model_objects) - # Restore variables on creation if a checkpoint exists. - checkpoint.restore(latest_cpkt) - - with tf.device(device): - for _ in range(100): - start = time.time() - with summary_writer.as_default(): - train_one_epoch(dataset=dataset, log_interval=FLAGS.log_interval, - noise_dim=FLAGS.noise, **model_objects) - end = time.time() - checkpoint.save(checkpoint_prefix) - print('\nTrain time for epoch #%d (step %d): %f' % - (checkpoint.save_counter.numpy(), - checkpoint.step_counter.numpy(), - end - start)) - - -if __name__ == '__main__': - tf.enable_eager_execution() - - parser = argparse.ArgumentParser() - parser.add_argument( - '--data-dir', - type=str, - default='/tmp/tensorflow/mnist/input_data', - help=('Directory for storing input data (default ' - '/tmp/tensorflow/mnist/input_data)')) - parser.add_argument( - '--batch-size', - type=int, - default=128, - metavar='N', - help='input batch size for training (default: 128)') - parser.add_argument( - '--log-interval', - type=int, - default=100, - metavar='N', - help=('number of batches between logging and writing summaries ' - '(default: 100)')) - parser.add_argument( - '--output_dir', - type=str, - default=None, - metavar='DIR', - help='Directory to write TensorBoard summaries (defaults to none)') - parser.add_argument( - '--checkpoint_dir', - type=str, - default='/tmp/tensorflow/mnist/checkpoints/', - metavar='DIR', - help=('Directory to save checkpoints in (once per epoch) (default ' - '/tmp/tensorflow/mnist/checkpoints/)')) - parser.add_argument( - '--lr', - type=float, - default=0.001, - metavar='LR', - help='learning rate (default: 0.001)') - parser.add_argument( - '--noise', - type=int, - default=100, - metavar='N', - help='Length of noise vector for generator input (default: 100)') - parser.add_argument( - '--no-gpu', - action='store_true', - default=False, - help='disables GPU usage even if a GPU is available') - - FLAGS, unparsed = parser.parse_known_args() - tf.compat.v1.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/gan/mnist_graph_test.py b/tensorflow/contrib/eager/python/examples/gan/mnist_graph_test.py deleted file mode 100644 index d9ea11b0c0f..00000000000 --- a/tensorflow/contrib/eager/python/examples/gan/mnist_graph_test.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tempfile -import time - -import numpy as np -import tensorflow as tf - -from tensorflow.contrib.eager.python.examples.gan import mnist - -NOISE_DIM = 100 -# Big enough so that summaries are never recorded. -# Lower this value if would like to benchmark with some summaries. -SUMMARY_INTERVAL = 10000 -SUMMARY_FLUSH_MS = 100 # Flush summaries every 100ms - - -def data_format(): - return 'channels_first' if tf.test.is_gpu_available() else 'channels_last' - - -class MnistGraphGanBenchmark(tf.test.Benchmark): - - def _create_graph(self, batch_size): - # Generate some random data. - images_data = np.random.randn(batch_size, 784).astype(np.float32) - dataset = tf.data.Dataset.from_tensors(images_data) - images = tf.compat.v1.data.make_one_shot_iterator( - dataset.repeat()).get_next() - - # Create the models and optimizers - generator = mnist.Generator(data_format()) - discriminator = mnist.Discriminator(data_format()) - with tf.variable_scope('generator'): - generator_optimizer = tf.compat.v1.train.AdamOptimizer(0.001) - with tf.variable_scope('discriminator'): - discriminator_optimizer = tf.compat.v1.train.AdamOptimizer(0.001) - - # Run models and compute loss - noise_placeholder = tf.placeholder(tf.float32, - shape=[batch_size, NOISE_DIM]) - generated_images = generator(noise_placeholder) - tf.contrib.summary.image('generated_images', - tf.reshape(generated_images, [-1, 28, 28, 1]), - max_images=10) - discriminator_gen_outputs = discriminator(generated_images) - discriminator_real_outputs = discriminator(images) - generator_loss = mnist.generator_loss(discriminator_gen_outputs) - discriminator_loss = mnist.discriminator_loss(discriminator_real_outputs, - discriminator_gen_outputs) - # Get train ops - with tf.variable_scope('generator'): - generator_train = generator_optimizer.minimize( - generator_loss, var_list=generator.variables) - with tf.variable_scope('discriminator'): - discriminator_train = discriminator_optimizer.minimize( - discriminator_loss, var_list=discriminator.variables) - - return (generator_train, discriminator_train, noise_placeholder) - - def _report(self, test_name, start, num_iters, batch_size): - avg_time = (time.time() - start) / num_iters - dev = 'gpu' if tf.test.is_gpu_available() else 'cpu' - name = 'graph_%s_%s_batch_%d_%s' % (test_name, dev, batch_size, - data_format()) - extras = {'examples_per_sec': batch_size / avg_time} - self.report_benchmark( - iters=num_iters, wall_time=avg_time, name=name, extras=extras) - - def benchmark_train(self): - for batch_size in [64, 128, 256]: - with tf.Graph().as_default(): - global_step = tf.train.get_or_create_global_step() - increment_global_step = tf.assign_add(global_step, 1) - with tf.contrib.summary.create_file_writer( - tempfile.mkdtemp(), flush_millis=SUMMARY_FLUSH_MS).as_default(), ( - tf.contrib.summary.record_summaries_every_n_global_steps( - SUMMARY_INTERVAL)): - (generator_train, discriminator_train, noise_placeholder - ) = self._create_graph(batch_size) - - with tf.compat.v1.Session() as sess: - tf.contrib.summary.initialize(graph=tf.get_default_graph(), - session=sess) - - sess.run(tf.global_variables_initializer()) - - num_burn, num_iters = (3, 100) - for _ in range(num_burn): - noise = np.random.uniform(-1.0, 1.0, size=[batch_size, NOISE_DIM]) - # Increment global step before evaluating summary ops to avoid - # race condition. - sess.run(increment_global_step) - sess.run([generator_train, discriminator_train, - tf.contrib.summary.all_summary_ops()], - feed_dict={noise_placeholder: noise}) - - # Run and benchmark 2 epochs - start = time.time() - for _ in range(num_iters): - noise = np.random.uniform(-1.0, 1.0, size=[batch_size, NOISE_DIM]) - sess.run(increment_global_step) - sess.run([generator_train, discriminator_train, - tf.contrib.summary.all_summary_ops()], - feed_dict={noise_placeholder: noise}) - self._report('train', start, num_iters, batch_size) - - def benchmark_generate(self): - for batch_size in [64, 128, 256]: - with tf.Graph().as_default(): - # Using random weights. This will generate garbage. - generator = mnist.Generator(data_format()) - noise_placeholder = tf.placeholder(tf.float32, - shape=[batch_size, NOISE_DIM]) - generated_images = generator(noise_placeholder) - - init = tf.global_variables_initializer() - with tf.compat.v1.Session() as sess: - sess.run(init) - noise = np.random.uniform(-1.0, 1.0, size=[batch_size, NOISE_DIM]) - num_burn, num_iters = (30, 1000) - for _ in range(num_burn): - sess.run(generated_images, feed_dict={noise_placeholder: noise}) - - start = time.time() - for _ in range(num_iters): - # Comparison with the eager execution benchmark in mnist_test.py - # isn't entirely fair as the time here includes the cost of copying - # the feeds from CPU memory to GPU. - sess.run(generated_images, feed_dict={noise_placeholder: noise}) - self._report('generate', start, num_iters, batch_size) - - -if __name__ == '__main__': - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/gan/mnist_test.py b/tensorflow/contrib/eager/python/examples/gan/mnist_test.py deleted file mode 100644 index 7611ff13ab7..00000000000 --- a/tensorflow/contrib/eager/python/examples/gan/mnist_test.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tempfile -import time - -import tensorflow as tf - -from tensorflow.contrib.eager.python.examples.gan import mnist - -NOISE_DIM = 100 -# Big enough so that summaries are never recorded. -# Lower this value if would like to benchmark with some summaries. -SUMMARY_INTERVAL = 10000 -SUMMARY_FLUSH_MS = 100 # Flush summaries every 100ms - - -def data_format(): - return 'channels_first' if tf.test.is_gpu_available() else 'channels_last' - - -def device(): - return '/gpu:0' if tf.test.is_gpu_available() else '/cpu:0' - - -class MnistEagerGanBenchmark(tf.test.Benchmark): - - def _report(self, test_name, start, num_iters, batch_size): - avg_time = (time.time() - start) / num_iters - dev = 'gpu' if tf.test.is_gpu_available() else 'cpu' - name = 'eager_%s_%s_batch_%d_%s' % (test_name, dev, batch_size, - data_format()) - extras = {'examples_per_sec': batch_size / avg_time} - self.report_benchmark( - iters=num_iters, wall_time=avg_time, name=name, extras=extras) - - def benchmark_train(self): - for batch_size in [64, 128, 256]: - # Generate some random data. - burn_batches, measure_batches = (3, 100) - burn_images = [tf.random_normal([batch_size, 784]) - for _ in range(burn_batches)] - burn_dataset = tf.data.Dataset.from_tensor_slices(burn_images) - measure_images = [tf.random_normal([batch_size, 784]) - for _ in range(measure_batches)] - measure_dataset = tf.data.Dataset.from_tensor_slices(measure_images) - - step_counter = tf.train.get_or_create_global_step() - with tf.device(device()): - # Create the models and optimizers - generator = mnist.Generator(data_format()) - discriminator = mnist.Discriminator(data_format()) - with tf.variable_scope('generator'): - generator_optimizer = tf.compat.v1.train.AdamOptimizer(0.001) - with tf.variable_scope('discriminator'): - discriminator_optimizer = tf.compat.v1.train.AdamOptimizer(0.001) - - with tf.contrib.summary.create_file_writer( - tempfile.mkdtemp(), flush_millis=SUMMARY_FLUSH_MS).as_default(): - - # warm up - mnist.train_one_epoch(generator, discriminator, generator_optimizer, - discriminator_optimizer, - burn_dataset, step_counter, - log_interval=SUMMARY_INTERVAL, - noise_dim=NOISE_DIM) - # measure - start = time.time() - mnist.train_one_epoch(generator, discriminator, generator_optimizer, - discriminator_optimizer, - measure_dataset, step_counter, - log_interval=SUMMARY_INTERVAL, - noise_dim=NOISE_DIM) - self._report('train', start, measure_batches, batch_size) - - def benchmark_generate(self): - for batch_size in [64, 128, 256]: - with tf.device(device()): - # Using random weights. This will generate garbage. - generator = mnist.Generator(data_format()) - - num_burn, num_iters = (30, 1000) - for _ in range(num_burn): - noise = tf.random_uniform(shape=[batch_size, NOISE_DIM], - minval=-1., maxval=1.) - generator(noise) - - start = time.time() - for _ in range(num_iters): - noise = tf.random_uniform(shape=[batch_size, NOISE_DIM], - minval=-1., maxval=1.) - generator(noise) - self._report('generate', start, num_iters, batch_size) - - -if __name__ == '__main__': - tf.enable_eager_execution() - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb deleted file mode 100644 index e1a02db76f7..00000000000 --- a/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb +++ /dev/null @@ -1,60 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0TD5ZrvEMbhZ" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\").\n", - "\n", - "# Convolutional VAE: An example with tf.keras and eager\n", - "\n", - "This example has moved:\n", - "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/r2/tutorials/generative/cvae.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e \n", - "\u003c/td\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/r2/tutorials/generative/cvae.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ITZuApL56Mny" - }, - "source": [ - "![evolution of output during training](https://tensorflow.org/images/autoencoders/cvae.gif)\n", - "\n" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "cvae.ipynb", - "private_outputs": true, - "provenance": [ - { - "file_id": "1eb0NOTQapkYs3X0v-zL1x5_LFKgDISnp", - "timestamp": 1527173385672 - } - ], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb deleted file mode 100644 index 00171eb35d6..00000000000 --- a/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb +++ /dev/null @@ -1,51 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0TD5ZrvEMbhZ" - }, - "source": [ - "**Copyright 2018 The TensorFlow Authors**.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\").\n", - "\n", - "# Generating Handwritten Digits with DCGAN\n", - "\n", - "This example has moved.\n", - "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/r2/tutorials/generative/dcgan.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e \n", - "\u003c/td\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://github.com/tensorflow/blob/master/site/en/r2/tutorials/generative/dcgan.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2MbKJY38Puy9" - }, - "source": [ - "![sample output](https://tensorflow.org/images/gan/dcgan.gif)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "dcgan.ipynb", - "provenance": [], - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb deleted file mode 100644 index 979772acd3f..00000000000 --- a/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb +++ /dev/null @@ -1,71 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "K2s1A9eLRPEj" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\").\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cffg2i257iMS" - }, - "source": [ - "# Image Captioning with Attention\n", - "\n", - "This example has moved:\n", - "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/r2/tutorials/generative/image_captioning.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e \n", - "\u003c/td\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/r2/tutorials/generative/image_captioning.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QASbY_HGo4Lq" - }, - "source": [ - "![Man Surfing](https://tensorflow.org/images/surf.jpg) \n", - "\n", - "[Image Source](https://commons.wikimedia.org/wiki/Surfing#/media/File:Surfing_in_Hawaii.jpg), License: Public Domain\n", - "\n", - "![Prediction](https://tensorflow.org/images/imcap_prediction.png)\n" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "image_captioning_with_attention.ipynb", - "private_outputs": true, - "provenance": [ - { - "file_id": "1HI8OK2sMjcx9CTWVn0122QAHOuXaOaMg", - "timestamp": 1530222436922 - } - ], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb deleted file mode 100644 index c945c753b3b..00000000000 --- a/tensorflow/contrib/eager/python/examples/generative_examples/text_generation.ipynb +++ /dev/null @@ -1,44 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hcD2nPQvPOFM" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\").\n", - "\n", - "# Text Generation using a RNN\n", - "\n", - "This example has moved.\n", - "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/sequences/text_generation.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e \n", - "\u003c/td\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/sequences/text_generation.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "text_generation.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/BUILD b/tensorflow/contrib/eager/python/examples/l2hmc/BUILD deleted file mode 100644 index 98b8f1483ea..00000000000 --- a/tensorflow/contrib/eager/python/examples/l2hmc/BUILD +++ /dev/null @@ -1,44 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "cuda_py_test") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "neural_nets", - srcs = ["neural_nets.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/eager/python:tfe", - ], -) - -py_library( - name = "l2hmc", - srcs = ["l2hmc.py"], - srcs_version = "PY2AND3", - deps = [ - ":neural_nets", - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/eager/python:tfe", - "//third_party/py/numpy", - ], -) - -cuda_py_test( - name = "l2hmc_test", - size = "medium", - srcs = ["l2hmc_test.py"], - additional_deps = [ - ":l2hmc", - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/eager/python:tfe", - "//third_party/py/numpy", - ], - shard_count = 4, - tags = [ - "oss_serial", - ], -) diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/README.md b/tensorflow/contrib/eager/python/examples/l2hmc/README.md deleted file mode 100644 index f171806e379..00000000000 --- a/tensorflow/contrib/eager/python/examples/l2hmc/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# L2HMC with TensorFlow eager execution - -This folder contains an implementation of [L2HMC](https://arxiv.org/pdf/1711.09268.pdf) adapted from the released implementation by the authors. The presented implementation runs in both eager and graph mode. -With eager execution enabled, longer sample chains can be handled compared to graph mode, since no graph is explicitly stored. Moreover, with eager execution enabled, there is no need to use a `tf.while_loop`. - -## What is L2HMC? -L2HMC is an adaptive Markov Chain Monte Carlo (MCMC) algorithm that learns a non-volume preserving transformation -for a Hamiltonian Monte Carlo (HMC) sampling algorithm. More specifically, the non-volume preserving -transformation is learned with neural nets instantiated within Normalizing Flows -(real-NVPs). - -## Content - -- `l2hmc.py`: Dynamics definitions and example energy functions, -including the 2D strongly correlated Gaussian and the rough well energy function, -- `l2hmc_test.py`: Unit tests and benchmarks for training a sampler on the energy functions in both eager and graph mode. -- `neural_nets.py`: The neural net for learning the kernel on the 2D strongly correlated example. -- `main.py`: Run to train a samplers on 2D energy landscapes. - -## To run -- Make sure you have installed TensorFlow 1.9+ or the latest `tf-nightly` or `tf-nightly-gpu` pip package. -- Execute the command - -```bash -python main.py --train_dir ${PWD}/dump --use_defun -``` - -Specifying the optional argument `train_dir` will store event files for -tensorboard and a plot of sampled chain from the trained sampler. - -Specifying the optional argument `use_defun` will let the program use compiled -graphs when running specific sections and improve the overall speed. - -## Boosting Performance with `tfe.defun` -Currently, some models may experience increased overhead with eager execution enabled. -To improve performance, we could wrap certain functions with the decorator `@tfe.defun`. -For example, we could wrap the function that does the sampling step: - -```python -@tfe.defun -def apply_transition(old_sample): - new_sample = ... - return new_sample -``` - -We could also explicitly wrap the desired function with `tfe.defun`: - -```python -apply_transition = tfe.defun(apply_transition) -``` - -## Reference -Generalizing Hamiltonian Monte Carlo with Neural Networks. Levy, Daniel, Hoffman, Matthew D, and Sohl-Dickstein, Jascha. International Conference on Learning Representations (ICLR), 2018. diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc.py b/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc.py deleted file mode 100644 index 14b8324e488..00000000000 --- a/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc.py +++ /dev/null @@ -1,351 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""L2HMC compatible with TensorFlow's eager execution. - -Reference [Generalizing Hamiltonian Monte Carlo with Neural -Networks](https://arxiv.org/pdf/1711.09268.pdf) - -Code adapted from the released TensorFlow graph implementation by original -authors https://github.com/brain-research/l2hmc. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import numpy.random as npr -import tensorflow as tf -import tensorflow.contrib.eager as tfe -from tensorflow.contrib.eager.python.examples.l2hmc import neural_nets - - -class Dynamics(tf.keras.Model): - """Dynamics engine of naive L2HMC sampler.""" - - def __init__(self, - x_dim, - minus_loglikelihood_fn, - n_steps=25, - eps=.1, - np_seed=1): - """Initialization. - - Args: - x_dim: dimensionality of observed data - minus_loglikelihood_fn: log-likelihood function of conditional probability - n_steps: number of leapfrog steps within each transition - eps: initial value learnable scale of step size - np_seed: Random seed for numpy; used to control sampled masks. - """ - super(Dynamics, self).__init__() - - npr.seed(np_seed) - self.x_dim = x_dim - self.potential = minus_loglikelihood_fn - self.n_steps = n_steps - - self._construct_time() - self._construct_masks() - - self.position_fn = neural_nets.GenericNet(x_dim, factor=2.) - self.momentum_fn = neural_nets.GenericNet(x_dim, factor=1.) - - self.eps = tf.Variable( - initial_value=eps, name="eps", dtype=tf.float32, trainable=True) - - def apply_transition(self, position): - """Propose a new state and perform the accept or reject step.""" - - # Simulate dynamics both forward and backward; - # Use sampled Bernoulli masks to compute the actual solutions - position_f, momentum_f, accept_prob_f = self.transition_kernel( - position, forward=True) - position_b, momentum_b, accept_prob_b = self.transition_kernel( - position, forward=False) - - # Decide direction uniformly - batch_size = tf.shape(position)[0] - forward_mask = tf.cast(tf.random_uniform((batch_size,)) > .5, tf.float32) - backward_mask = 1. - forward_mask - - # Obtain proposed states - position_post = ( - forward_mask[:, None] * position_f + - backward_mask[:, None] * position_b) - momentum_post = ( - forward_mask[:, None] * momentum_f + - backward_mask[:, None] * momentum_b) - - # Probability of accepting the proposed states - accept_prob = forward_mask * accept_prob_f + backward_mask * accept_prob_b - - # Accept or reject step - accept_mask = tf.cast( - accept_prob > tf.random_uniform(tf.shape(accept_prob)), tf.float32) - reject_mask = 1. - accept_mask - - # Samples after accept/reject step - position_out = ( - accept_mask[:, None] * position_post + reject_mask[:, None] * position) - - return position_post, momentum_post, accept_prob, position_out - - def transition_kernel(self, position, forward=True): - """Transition kernel of augmented leapfrog integrator.""" - - lf_fn = self._forward_lf if forward else self._backward_lf - - # Resample momentum - momentum = tf.random_normal(tf.shape(position)) - position_post, momentum_post = position, momentum - sumlogdet = 0. - # Apply augmented leapfrog steps - for i in range(self.n_steps): - position_post, momentum_post, logdet = lf_fn(position_post, momentum_post, - i) - sumlogdet += logdet - accept_prob = self._compute_accept_prob(position, momentum, position_post, - momentum_post, sumlogdet) - - return position_post, momentum_post, accept_prob - - def _forward_lf(self, position, momentum, i): - """One forward augmented leapfrog step. See eq (5-6) in paper.""" - - t = self._get_time(i) - mask, mask_inv = self._get_mask(i) - sumlogdet = 0. - - momentum, logdet = self._update_momentum_forward(position, momentum, t) - sumlogdet += logdet - - position, logdet = self._update_position_forward(position, momentum, t, - mask, mask_inv) - sumlogdet += logdet - - position, logdet = self._update_position_forward(position, momentum, t, - mask_inv, mask) - sumlogdet += logdet - - momentum, logdet = self._update_momentum_forward(position, momentum, t) - sumlogdet += logdet - - return position, momentum, sumlogdet - - def _backward_lf(self, position, momentum, i): - """One backward augmented leapfrog step. See Appendix A in paper.""" - - # Reversed index/sinusoidal time - t = self._get_time(self.n_steps - i - 1) - mask, mask_inv = self._get_mask(self.n_steps - i - 1) - sumlogdet = 0. - - momentum, logdet = self._update_momentum_backward(position, momentum, t) - sumlogdet += logdet - - position, logdet = self._update_position_backward(position, momentum, t, - mask_inv, mask) - sumlogdet += logdet - - position, logdet = self._update_position_backward(position, momentum, t, - mask, mask_inv) - sumlogdet += logdet - - momentum, logdet = self._update_momentum_backward(position, momentum, t) - sumlogdet += logdet - - return position, momentum, sumlogdet - - def _update_momentum_forward(self, position, momentum, t): - """Update v in the forward leapfrog step.""" - - grad = self.grad_potential(position) - scale, translation, transformed = self.momentum_fn([position, grad, t]) - scale *= .5 * self.eps - transformed *= self.eps - momentum = ( - momentum * tf.exp(scale) - - .5 * self.eps * (tf.exp(transformed) * grad - translation)) - - return momentum, tf.reduce_sum(scale, axis=1) - - def _update_position_forward(self, position, momentum, t, mask, mask_inv): - """Update x in the forward leapfrog step.""" - - scale, translation, transformed = self.position_fn( - [momentum, mask * position, t]) - scale *= self.eps - transformed *= self.eps - position = ( - mask * position + - mask_inv * (position * tf.exp(scale) + self.eps * - (tf.exp(transformed) * momentum + translation))) - return position, tf.reduce_sum(mask_inv * scale, axis=1) - - def _update_momentum_backward(self, position, momentum, t): - """Update v in the backward leapfrog step. Inverting the forward update.""" - - grad = self.grad_potential(position) - scale, translation, transformed = self.momentum_fn([position, grad, t]) - scale *= -.5 * self.eps - transformed *= self.eps - momentum = ( - tf.exp(scale) * (momentum + .5 * self.eps * - (tf.exp(transformed) * grad - translation))) - - return momentum, tf.reduce_sum(scale, axis=1) - - def _update_position_backward(self, position, momentum, t, mask, mask_inv): - """Update x in the backward leapfrog step. Inverting the forward update.""" - - scale, translation, transformed = self.position_fn( - [momentum, mask * position, t]) - scale *= -self.eps - transformed *= self.eps - position = ( - mask * position + mask_inv * tf.exp(scale) * - (position - self.eps * (tf.exp(transformed) * momentum + translation))) - - return position, tf.reduce_sum(mask_inv * scale, axis=1) - - def _compute_accept_prob(self, position, momentum, position_post, - momentum_post, sumlogdet): - """Compute the prob of accepting the proposed state given old state.""" - - old_hamil = self.hamiltonian(position, momentum) - new_hamil = self.hamiltonian(position_post, momentum_post) - prob = tf.exp(tf.minimum(old_hamil - new_hamil + sumlogdet, 0.)) - - # Ensure numerical stability as well as correct gradients - return tf.where(tf.is_finite(prob), prob, tf.zeros_like(prob)) - - def _construct_time(self): - """Convert leapfrog step index into sinusoidal time.""" - - self.ts = [] - for i in range(self.n_steps): - t = tf.constant( - [ - np.cos(2 * np.pi * i / self.n_steps), - np.sin(2 * np.pi * i / self.n_steps) - ], - dtype=tf.float32) - self.ts.append(t[None, :]) - - def _get_time(self, i): - """Get sinusoidal time for i-th augmented leapfrog step.""" - - return self.ts[i] - - def _construct_masks(self): - """Construct different binary masks for different time steps.""" - - self.masks = [] - for _ in range(self.n_steps): - # Need to use npr here because tf would generated different random - # values across different `sess.run` - idx = npr.permutation(np.arange(self.x_dim))[:self.x_dim // 2] - mask = np.zeros((self.x_dim,)) - mask[idx] = 1. - mask = tf.constant(mask, dtype=tf.float32) - self.masks.append(mask[None, :]) - - def _get_mask(self, i): - """Get binary masks for i-th augmented leapfrog step.""" - - m = self.masks[i] - return m, 1. - m - - def kinetic(self, v): - """Compute the kinetic energy.""" - - return .5 * tf.reduce_sum(v**2, axis=1) - - def hamiltonian(self, position, momentum): - """Compute the overall Hamiltonian.""" - - return self.potential(position) + self.kinetic(momentum) - - def grad_potential(self, position, check_numerics=True): - """Get gradient of potential function at current location.""" - - if tf.executing_eagerly(): - grad = tfe.gradients_function(self.potential)(position)[0] - else: - grad = tf.gradients(self.potential(position), position)[0] - - return grad - - -# Examples of unnormalized log densities -def get_scg_energy_fn(): - """Get energy function for 2d strongly correlated Gaussian.""" - - # Avoid recreating tf constants on each invocation of gradients - mu = tf.constant([0., 0.]) - sigma = tf.constant([[50.05, -49.95], [-49.95, 50.05]]) - sigma_inv = tf.matrix_inverse(sigma) - - def energy(x): - """Unnormalized minus log density of 2d strongly correlated Gaussian.""" - - xmmu = x - mu - return .5 * tf.diag_part( - tf.matmul(tf.matmul(xmmu, sigma_inv), tf.transpose(xmmu))) - - return energy, mu, sigma - - -def get_rw_energy_fn(): - """Get energy function for rough well distribution.""" - # For small eta, the density underlying the rough-well energy is very close to - # a unit Gaussian; however, the gradient is greatly affected by the small - # cosine perturbations - eta = 1e-2 - mu = tf.constant([0., 0.]) - sigma = tf.constant([[1., 0.], [0., 1.]]) - - def energy(x): - ip = tf.reduce_sum(x**2., axis=1) - return .5 * ip + eta * tf.reduce_sum(tf.cos(x / eta), axis=1) - - return energy, mu, sigma - - -# Loss function -def compute_loss(dynamics, x, scale=.1, eps=1e-4): - """Compute loss defined in equation (8).""" - - z = tf.random_normal(tf.shape(x)) # Auxiliary variable - x_, _, x_accept_prob, x_out = dynamics.apply_transition(x) - z_, _, z_accept_prob, _ = dynamics.apply_transition(z) - - # Add eps for numerical stability; following released impl - x_loss = tf.reduce_sum((x - x_)**2, axis=1) * x_accept_prob + eps - z_loss = tf.reduce_sum((z - z_)**2, axis=1) * z_accept_prob + eps - - loss = tf.reduce_mean( - (1. / x_loss + 1. / z_loss) * scale - (x_loss + z_loss) / scale, axis=0) - - return loss, x_out, x_accept_prob - - -def loss_and_grads(dynamics, x, loss_fn=compute_loss): - """Obtain loss value and gradients.""" - with tf.GradientTape() as tape: - loss_val, out, accept_prob = loss_fn(dynamics, x) - grads = tape.gradient(loss_val, dynamics.trainable_variables) - - return loss_val, grads, out, accept_prob diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc_test.py b/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc_test.py deleted file mode 100644 index 1c925e455b9..00000000000 --- a/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc_test.py +++ /dev/null @@ -1,245 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests l2hmc fit to 2D strongly correlated Gaussian executed eagerly.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time - -import numpy.random as npr -import tensorflow as tf -import tensorflow.contrib.eager as tfe -from tensorflow.contrib.eager.python.examples.l2hmc import l2hmc - - -def get_default_hparams(): - return tf.contrib.training.HParams( - x_dim=2, - n_samples=200, - n_steps=10, - eps=.1, - n_iters=10, - learning_rate=.0003, - n_warmup_iters=3) - - -def step(dynamics, optimizer, samples): - loss, grads, samples, _ = l2hmc.loss_and_grads( - dynamics, samples, loss_fn=l2hmc.compute_loss) - optimizer.apply_gradients(zip(grads, dynamics.variables)) - - return loss, samples - - -# To be defunnable, the function cannot return an Operation, so the above -# function is used for defun or eager, and this function is used in graph to be -# able to run the gradient updates. -def graph_step(dynamics, optimizer, samples): - loss, grads, samples, _ = l2hmc.loss_and_grads( - dynamics, samples, loss_fn=l2hmc.compute_loss) - train_op = optimizer.apply_gradients(zip(grads, dynamics.variables)) - - return train_op, loss, samples - - -def warmup(dynamics, - optimizer, - n_iters=1, - n_samples=200, - step_fn=step): - """Warmup optimization to reduce overhead.""" - - samples = tf.random_normal( - shape=[n_samples, dynamics.x_dim], dtype=tf.float32) - - for _ in range(n_iters): - _, samples = step_fn(dynamics, optimizer, samples) - - -def fit(dynamics, - samples, - optimizer, - step_fn=step, - n_iters=5000, - verbose=True, - logdir=None): - """Fit L2HMC sampler with given log-likelihood function.""" - - if logdir: - summary_writer = tf.contrib.summary.create_file_writer(logdir) - - for i in range(n_iters): - loss, samples = step_fn(dynamics, optimizer, samples) - if verbose: - print("Iteration %d: loss %.4f" % (i, loss)) - - if logdir: - with summary_writer.as_default(): - with tf.contrib.summary.always_record_summaries(): - tf.contrib.summary.scalar("loss", loss) - - -class L2hmcTest(tf.test.TestCase): - """Unit tests for l2hmc in both eager and graph mode.""" - - def test_apply_transition(self): - """Testing function `Dynamics.apply_transition` in graph and eager mode.""" - - # Eager mode testing - hparams = get_default_hparams() - energy_fn, _, _ = l2hmc.get_scg_energy_fn() - dynamics = l2hmc.Dynamics( - x_dim=hparams.x_dim, - minus_loglikelihood_fn=energy_fn, - n_steps=hparams.n_steps, - eps=hparams.eps) - samples = tf.random_normal(shape=[hparams.n_samples, hparams.x_dim]) - x_, v_, x_accept_prob, x_out = dynamics.apply_transition(samples) - - self.assertEqual(x_.shape, v_.shape) - self.assertEqual(x_out.shape, samples.shape) - self.assertEqual(x_.shape, x_out.shape) - self.assertEqual(x_accept_prob.shape, (hparams.n_samples,)) - - # Graph mode testing - with tf.Graph().as_default(): - energy_fn, _, _ = l2hmc.get_scg_energy_fn() - dynamics = l2hmc.Dynamics( - x_dim=hparams.x_dim, - minus_loglikelihood_fn=energy_fn, - n_steps=hparams.n_steps, - eps=hparams.eps) - x = tf.placeholder(tf.float32, shape=[None, hparams.x_dim]) - x_, v_, x_accept_prob, x_out = dynamics.apply_transition(x) - samples = npr.normal(size=[hparams.n_samples, hparams.x_dim]) - - with tf.Session() as sess: - sess.run(tf.global_variables_initializer()) - np_x_, np_v_, np_x_accept_prob, np_x_out = sess.run( - [x_, v_, x_accept_prob, x_out], feed_dict={x: samples}) - - self.assertEqual(np_x_.shape, np_v_.shape) - self.assertEqual(samples.shape, np_x_out.shape) - self.assertEqual(np_x_.shape, np_x_out.shape) - self.assertEqual(np_x_accept_prob.shape, (hparams.n_samples,)) - - -class L2hmcBenchmark(tf.test.Benchmark): - """Eager and graph benchmarks for l2hmc.""" - - def benchmark_graph(self): - """Benchmark Graph performance.""" - - hparams = get_default_hparams() - tf.enable_resource_variables() - for sample_size in [10, 25, 50, 100, 200]: - hparams.n_samples = sample_size - tf.reset_default_graph() - with tf.Graph().as_default(): - energy_fn, _, _ = l2hmc.get_scg_energy_fn() - x = tf.random_normal([hparams.n_samples, hparams.x_dim], - dtype=tf.float32) - dynamics = l2hmc.Dynamics( - x_dim=hparams.x_dim, - minus_loglikelihood_fn=energy_fn, - n_steps=hparams.n_steps, - eps=hparams.eps) - loss, _, _ = l2hmc.compute_loss(dynamics, x) - - optimizer = tf.train.AdamOptimizer(learning_rate=hparams.learning_rate) - train_op, loss, _ = graph_step(dynamics, optimizer, x) - - # Single thread; fairer comparison against eager - session_conf = tf.ConfigProto(inter_op_parallelism_threads=1) - - with tf.Session(config=session_conf) as sess: - sess.run(tf.global_variables_initializer()) - - # Warmup to reduce initialization effect when timing - for _ in range(hparams.n_warmup_iters): - _, _ = sess.run([train_op, loss]) - - # Training - start_time = time.time() - for i in range(hparams.n_iters): - _, loss_np = sess.run([train_op, loss]) - print("Iteration %d: loss %.4f" % (i, loss_np)) - wall_time = (time.time() - start_time) / hparams.n_iters - examples_per_sec = hparams.n_samples / wall_time - - self.report_benchmark( - name="graph_train_%s_%d" % - ("gpu" if tf.test.is_gpu_available() else "cpu", sample_size), - iters=hparams.n_iters, - extras={"examples_per_sec": examples_per_sec}, - wall_time=wall_time) - - def benchmark_eager(self): - self._benchmark_eager() - - def benchmark_eager_defun(self): - self._benchmark_eager(defun=True) - - def _benchmark_eager(self, defun=False): - """Benchmark Eager performance.""" - - hparams = get_default_hparams() - for sample_size in [10, 25, 50, 100, 200]: - hparams.n_samples = sample_size - energy_fn, _, _ = l2hmc.get_scg_energy_fn() - dynamics = l2hmc.Dynamics( - x_dim=hparams.x_dim, - minus_loglikelihood_fn=energy_fn, - n_steps=hparams.n_steps, - eps=hparams.eps) - optimizer = tf.train.AdamOptimizer(learning_rate=hparams.learning_rate) - step_fn = tfe.defun(step) if defun else step - - # Warmup to reduce initialization effect when timing - warmup( - dynamics, - optimizer, - n_iters=hparams.n_warmup_iters, - n_samples=hparams.n_samples, - step_fn=step_fn) - - # Training - samples = tf.random_normal( - shape=[hparams.n_samples, hparams.x_dim], dtype=tf.float32) - start_time = time.time() - fit(dynamics, - samples, - optimizer, - step_fn=step_fn, - n_iters=hparams.n_iters) - wall_time = (time.time() - start_time) / hparams.n_iters - examples_per_sec = hparams.n_samples / wall_time - - self.report_benchmark( - name="eager_train_%s%s_%d" % - ("gpu" if tf.test.is_gpu_available() else "cpu", - "_defun" if defun else "", sample_size), - iters=hparams.n_iters, - extras={"examples_per_sec": examples_per_sec}, - wall_time=wall_time) - - del dynamics - - -if __name__ == "__main__": - tf.enable_eager_execution() - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/main.py b/tensorflow/contrib/eager/python/examples/l2hmc/main.py deleted file mode 100644 index 98fcb2ba10a..00000000000 --- a/tensorflow/contrib/eager/python/examples/l2hmc/main.py +++ /dev/null @@ -1,235 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""L2HMC on simple Gaussian mixture model with TensorFlow eager.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import sys - -from absl import flags -import numpy as np -import tensorflow as tf -from tensorflow.contrib.eager.python.examples.l2hmc import l2hmc -try: - import matplotlib.pyplot as plt # pylint: disable=g-import-not-at-top - HAS_MATPLOTLIB = True -except ImportError: - HAS_MATPLOTLIB = False -tfe = tf.contrib.eager - - -def main(_): - tf.enable_eager_execution() - global_step = tf.train.get_or_create_global_step() - global_step.assign(1) - - energy_fn, mean, covar = { - "scg": l2hmc.get_scg_energy_fn(), - "rw": l2hmc.get_rw_energy_fn() - }[FLAGS.energy_fn] - - x_dim = 2 - train_iters = 5000 - eval_iters = 2000 - eps = 0.1 - n_steps = 10 # Chain length - n_samples = 200 - record_loss_every = 100 - - dynamics = l2hmc.Dynamics( - x_dim=x_dim, minus_loglikelihood_fn=energy_fn, n_steps=n_steps, eps=eps) - learning_rate = tf.train.exponential_decay( - 1e-3, global_step, 1000, 0.96, staircase=True) - optimizer = tf.train.AdamOptimizer(learning_rate) - checkpointer = tf.train.Checkpoint( - optimizer=optimizer, dynamics=dynamics, global_step=global_step) - - if FLAGS.train_dir: - summary_writer = tf.contrib.summary.create_file_writer(FLAGS.train_dir) - if FLAGS.restore: - latest_path = tf.train.latest_checkpoint(FLAGS.train_dir) - checkpointer.restore(latest_path) - print("Restored latest checkpoint at path:\"{}\" ".format(latest_path)) - sys.stdout.flush() - - if not FLAGS.restore: - # Training - if FLAGS.use_defun: - # Use `tfe.deun` to boost performance when there are lots of small ops - loss_fn = tfe.function(l2hmc.compute_loss) - else: - loss_fn = l2hmc.compute_loss - - samples = tf.random_normal(shape=[n_samples, x_dim]) - for i in range(1, train_iters + 1): - loss, samples, accept_prob = train_one_iter( - dynamics, - samples, - optimizer, - loss_fn=loss_fn, - global_step=global_step) - - if i % record_loss_every == 0: - print("Iteration {}, loss {:.4f}, x_accept_prob {:.4f}".format( - i, loss.numpy(), - accept_prob.numpy().mean())) - if FLAGS.train_dir: - with summary_writer.as_default(): - with tf.contrib.summary.always_record_summaries(): - tf.contrib.summary.scalar("Training loss", loss, step=global_step) - print("Training complete.") - sys.stdout.flush() - - if FLAGS.train_dir: - saved_path = checkpointer.save( - file_prefix=os.path.join(FLAGS.train_dir, "ckpt")) - print("Saved checkpoint at path: \"{}\" ".format(saved_path)) - sys.stdout.flush() - - # Evaluation - if FLAGS.use_defun: - # Use tfe.deun to boost performance when there are lots of small ops - apply_transition = tfe.function(dynamics.apply_transition) - else: - apply_transition = dynamics.apply_transition - - samples = tf.random_normal(shape=[n_samples, x_dim]) - samples_history = [] - for i in range(eval_iters): - samples_history.append(samples.numpy()) - _, _, _, samples = apply_transition(samples) - samples_history = np.array(samples_history) - print("Sampling complete.") - sys.stdout.flush() - - # Mean and covariance of target distribution - mean = mean.numpy() - covar = covar.numpy() - ac_spectrum = compute_ac_spectrum(samples_history, mean, covar) - print("First 25 entries of the auto-correlation spectrum: {}".format( - ac_spectrum[:25])) - ess = compute_ess(ac_spectrum) - print("Effective sample size per Metropolis-Hastings step: {}".format(ess)) - sys.stdout.flush() - - if FLAGS.train_dir: - # Plot autocorrelation spectrum in tensorboard - plot_step = tfe.Variable(1, trainable=False, dtype=tf.int64) - - for ac in ac_spectrum: - with summary_writer.as_default(): - with tf.contrib.summary.always_record_summaries(): - tf.contrib.summary.scalar("Autocorrelation", ac, step=plot_step) - plot_step.assign(plot_step + n_steps) - - if HAS_MATPLOTLIB: - # Choose a single chain and plot the trajectory - single_chain = samples_history[:, 0, :] - xs = single_chain[:100, 0] - ys = single_chain[:100, 1] - plt.figure() - plt.plot(xs, ys, color="orange", marker="o", alpha=0.6) # Trained chain - plt.savefig(os.path.join(FLAGS.train_dir, "single_chain.png")) - - -def train_one_iter(dynamics, - x, - optimizer, - loss_fn=l2hmc.compute_loss, - global_step=None): - """Train the sampler for one iteration.""" - loss, grads, out, accept_prob = l2hmc.loss_and_grads( - dynamics, x, loss_fn=loss_fn) - optimizer.apply_gradients( - zip(grads, dynamics.trainable_variables), global_step=global_step) - - return loss, out, accept_prob - - -def compute_ac_spectrum(samples_history, target_mean, target_covar): - """Compute autocorrelation spectrum. - - Follows equation 15 from the L2HMC paper. - - Args: - samples_history: Numpy array of shape [T, B, D], where T is the total - number of time steps, B is the batch size, and D is the dimensionality - of sample space. - target_mean: 1D Numpy array of the mean of target(true) distribution. - target_covar: 2D Numpy array representing a symmetric matrix for variance. - Returns: - Autocorrelation spectrum, Numpy array of shape [T-1]. - """ - - # Using numpy here since eager is a bit slow due to the loop - time_steps = samples_history.shape[0] - trace = np.trace(target_covar) - - rhos = [] - for t in range(time_steps - 1): - rho_t = 0. - for tau in range(time_steps - t): - v_tau = samples_history[tau, :, :] - target_mean - v_tau_plus_t = samples_history[tau + t, :, :] - target_mean - # Take dot product over observation dims and take mean over batch dims - rho_t += np.mean(np.sum(v_tau * v_tau_plus_t, axis=1)) - - rho_t /= trace * (time_steps - t) - rhos.append(rho_t) - - return np.array(rhos) - - -def compute_ess(ac_spectrum): - """Compute the effective sample size based on autocorrelation spectrum. - - This follows equation 16 from the L2HMC paper. - - Args: - ac_spectrum: Autocorrelation spectrum - Returns: - The effective sample size - """ - # Cutoff from the first value less than 0.05 - cutoff = np.argmax(ac_spectrum[1:] < .05) - if cutoff == 0: - cutoff = len(ac_spectrum) - ess = 1. / (1. + 2. * np.sum(ac_spectrum[1:cutoff])) - return ess - - -if __name__ == "__main__": - flags.DEFINE_string( - "train_dir", - default=None, - help="[Optional] Directory to store the training information") - flags.DEFINE_boolean( - "restore", - default=False, - help="[Optional] Restore the latest checkpoint from `train_dir` if True") - flags.DEFINE_boolean( - "use_defun", - default=False, - help="[Optional] Use `tfe.defun` to boost performance") - flags.DEFINE_string( - "energy_fn", - default="scg", - help="[Optional] The energy function used for experimentation" - "Other options include `rw`") - FLAGS = flags.FLAGS - tf.app.run(main) diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/neural_nets.py b/tensorflow/contrib/eager/python/examples/l2hmc/neural_nets.py deleted file mode 100644 index 68e0bc31239..00000000000 --- a/tensorflow/contrib/eager/python/examples/l2hmc/neural_nets.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Neural nets utility for L2HMC compatible with TensorFlow's eager execution. - -Reference [Generalizing Hamiltonian Monte Carlo with Neural -Networks](https://arxiv.org/pdf/1711.09268.pdf) - -Code adapted from the released TensorFlow graph implementation by original -authors https://github.com/brain-research/l2hmc. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - - -class GenericNet(tf.keras.Model): - """Generic neural net with different initialization scale based on input. - - Args: - x_dim: dimensionality of observed data - factor: factor of variance scaling initializer - n_hidden: number of hidden units - """ - - def __init__(self, x_dim, factor, n_hidden=10): - super(GenericNet, self).__init__() - - self.v_layer = _custom_dense(n_hidden, 1. / 3.) - self.x_layer = _custom_dense(n_hidden, factor / 3.) - self.t_layer = _custom_dense(n_hidden, 1. / 3.) - self.h_layer = _custom_dense(n_hidden) - - # Scale - self.scale_layer = _custom_dense(x_dim, .001) - self.coeff_scale = tf.Variable( - initial_value=tf.zeros([1, x_dim]), name='coeff_scale', trainable=True) - # Translation - self.translation_layer = _custom_dense(x_dim, factor=.001) - # Transformation - self.transformation_layer = _custom_dense(x_dim, .001) - self.coeff_transformation = tf.Variable( - initial_value=tf.zeros([1, x_dim]), - name='coeff_transformation', - trainable=True) - - def call(self, inputs): - v, x, t = inputs - h = self.v_layer(v) + self.x_layer(x) + self.t_layer(t) - h = tf.nn.relu(h) - h = self.h_layer(h) - h = tf.nn.relu(h) - scale = tf.nn.tanh(self.scale_layer(h)) * tf.exp(self.coeff_scale) - translation = self.translation_layer(h) - transformation = ( - tf.nn.tanh(self.transformation_layer(h)) * tf.exp( - self.coeff_transformation)) - - return scale, translation, transformation - - -def _custom_dense(units, factor=1.): - """Custom dense layer with specified weight initialization.""" - - return tf.keras.layers.Dense( - units=units, - use_bias=True, - kernel_initializer=tf.contrib.layers.variance_scaling_initializer( - factor=factor * 2., mode='FAN_IN', uniform=False), - bias_initializer=tf.constant_initializer(0., dtype=tf.float32)) diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD deleted file mode 100644 index 6c831a0d916..00000000000 --- a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD +++ /dev/null @@ -1,49 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "py_binary") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -py_binary( - name = "linear_regression", - srcs = ["linear_regression.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [":linear_regression_lib"], -) - -py_library( - name = "linear_regression_lib", - srcs = ["linear_regression.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/eager/python:tfe", - ], -) - -cuda_py_test( - name = "linear_regression_test", - size = "small", - srcs = ["linear_regression_test.py"], - additional_deps = [ - ":linear_regression_lib", - "//tensorflow:tensorflow_py", - ], - tags = [ - "no_windows", # TODO: needs investigation on Windows - "oss_serial", - ], -) - -cuda_py_test( - name = "linear_regression_graph_test", - size = "small", - srcs = ["linear_regression_graph_test.py"], - additional_deps = [ - ":linear_regression_lib", - "//tensorflow:tensorflow_py", - ], -) diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py deleted file mode 100644 index 206ef9409df..00000000000 --- a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -r"""TensorFlow Eager Execution Example: Linear Regression. - -This example shows how to use TensorFlow Eager Execution to fit a simple linear -regression model using some synthesized data. Specifically, it illustrates how -to define the forward path of the linear model and the loss function, as well -as how to obtain the gradients of the loss function with respect to the -variables and update the variables with the gradients. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import sys - -import tensorflow as tf - -import tensorflow.contrib.eager as tfe - -layers = tf.keras.layers - - -class LinearModel(tf.keras.Model): - """A TensorFlow linear regression model.""" - - def __init__(self): - """Constructs a LinearModel object.""" - super(LinearModel, self).__init__() - self._hidden_layer = layers.Dense(1) - - def call(self, xs): - """Invoke the linear model. - - Args: - xs: input features, as a tensor of size [batch_size, ndims]. - - Returns: - ys: the predictions of the linear mode, as a tensor of size [batch_size] - """ - return self._hidden_layer(xs) - - -def mean_square_loss(model, xs, ys): - return tf.reduce_mean(tf.squared_difference(model(xs), ys)) - - -def fit(model, dataset, optimizer, verbose=False, logdir=None): - """Fit the linear-regression model. - - Args: - model: The LinearModel to fit. - dataset: The tf.data.Dataset to use for training data. - optimizer: The TensorFlow Optimizer object to be used. - verbose: If true, will print out loss values at every iteration. - logdir: The directory in which summaries will be written for TensorBoard - (optional). - """ - - # The loss function to optimize. - mse = lambda xs, ys: mean_square_loss(model, xs, ys) - loss_and_grads = tfe.implicit_value_and_gradients(mse) - - if logdir: - # Support for TensorBoard summaries. Once training has started, use: - # tensorboard --logdir= - summary_writer = tf.contrib.summary.create_file_writer(logdir) - - # Training loop. - for i, (xs, ys) in enumerate(tfe.Iterator(dataset)): - loss, grads = loss_and_grads(xs, ys) - if verbose: - print("Iteration %d: loss = %s" % (i, loss.numpy())) - - optimizer.apply_gradients(grads) - - if logdir: - with summary_writer.as_default(): - with tf.contrib.summary.always_record_summaries(): - tf.contrib.summary.scalar("loss", loss, step=i) - tf.contrib.summary.scalar("step", i, step=i) - - -def synthetic_dataset(w, b, noise_level, batch_size, num_batches): - """tf.data.Dataset that yields synthetic data for linear regression.""" - return synthetic_dataset_helper(w, b, - tf.shape(w)[0], noise_level, batch_size, - num_batches) - - -def synthetic_dataset_helper(w, b, num_features, noise_level, batch_size, - num_batches): - # w is a matrix with shape [N, M] - # b is a vector with shape [M] - # So: - # - Generate x's as vectors with shape [batch_size N] - # - y = tf.matmul(x, W) + b + noise - def batch(_): - x = tf.random_normal([batch_size, num_features]) - y = tf.matmul(x, w) + b + noise_level * tf.random_normal([]) - return x, y - - with tf.device("/device:CPU:0"): - return tf.data.Dataset.range(num_batches).map(batch) - - -def main(_): - tf.enable_eager_execution() - # Ground-truth constants. - true_w = [[-2.0], [4.0], [1.0]] - true_b = [0.5] - noise_level = 0.01 - - # Training constants. - batch_size = 64 - learning_rate = 0.1 - - print("True w: %s" % true_w) - print("True b: %s\n" % true_b) - - model = LinearModel() - dataset = synthetic_dataset(true_w, true_b, noise_level, batch_size, 20) - - device = "gpu:0" if tfe.num_gpus() else "cpu:0" - print("Using device: %s" % device) - with tf.device(device): - optimizer = tf.train.GradientDescentOptimizer(learning_rate) - fit(model, dataset, optimizer, verbose=True, logdir=FLAGS.logdir) - - print("\nAfter training: w = %s" % model.variables[0].numpy()) - print("\nAfter training: b = %s" % model.variables[1].numpy()) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--logdir", - type=str, - default=None, - help="logdir in which TensorBoard summaries will be written (optional).") - FLAGS, unparsed = parser.parse_known_args() - - tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_graph_test.py b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_graph_test.py deleted file mode 100644 index d412b25b368..00000000000 --- a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_graph_test.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Graph benchmark for linear regression, to contrast with eager execution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time - -import tensorflow as tf -from tensorflow.contrib.eager.python.examples.linear_regression import linear_regression - - -class GraphLinearRegressionBenchmark(tf.test.Benchmark): - - def benchmarkGraphLinearRegression(self): - num_epochs = 10 - num_batches = 200 - batch_size = 64 - dataset = linear_regression.synthetic_dataset_helper( - w=tf.random_uniform([3, 1]), - b=tf.random_uniform([1]), - num_features=3, - noise_level=0.01, - batch_size=batch_size, - num_batches=num_batches) - iterator = tf.compat.v1.data.make_initializable_iterator(dataset) - x, y = iterator.get_next() - - model = linear_regression.LinearModel() - - if tf.test.is_gpu_available(): - use_gpu = True - device = "/device:GPU:0" - else: - use_gpu = False - device = "/device:CPU:0" - - with tf.device(device): - loss = linear_regression.mean_square_loss(model, x, y) - optimization_step = tf.train.GradientDescentOptimizer( - learning_rate=0.1).minimize(loss) - - with tf.Session() as sess: - sess.run(tf.global_variables_initializer()) - - def train(num_epochs): - for _ in range(num_epochs): - sess.run(iterator.initializer) - try: - while True: - _, _ = sess.run([optimization_step, loss]) - except tf.errors.OutOfRangeError: - pass - - # Warmup: a single epoch. - train(1) - - start_time = time.time() - train(num_epochs) - wall_time = time.time() - start_time - - examples_per_sec = num_epochs * num_batches * batch_size / wall_time - self.report_benchmark( - name="graph_train_%s" % - ("gpu" if use_gpu else "cpu"), - iters=num_epochs * num_batches, - extras={"examples_per_sec": examples_per_sec}, - wall_time=wall_time) - - -if __name__ == "__main__": - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_test.py b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_test.py deleted file mode 100644 index 2bc2fc2aa91..00000000000 --- a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_test.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Unit tests for linear regression example under TensorFlow eager execution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import glob -import os -import shutil -import tempfile -import time - -import tensorflow as tf - -import tensorflow.contrib.eager as tfe -from tensorflow.contrib.eager.python.examples.linear_regression import linear_regression - - -def device(): - return "/device:GPU:0" if tfe.num_gpus() > 0 else "/device:CPU:0" - - -class LinearRegressionTest(tf.test.TestCase): - - def setUp(self): - super(LinearRegressionTest, self).setUp() - self._tmp_logdir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self._tmp_logdir) - super(LinearRegressionTest, self).tearDown() - - def testSyntheticDataset(self): - true_w = tf.random_uniform([3, 1]) - true_b = [1.0] - batch_size = 10 - num_batches = 2 - noise_level = 0. - dataset = linear_regression.synthetic_dataset(true_w, true_b, noise_level, - batch_size, num_batches) - - it = tfe.Iterator(dataset) - for _ in range(2): - (xs, ys) = it.next() - self.assertEqual((batch_size, 3), xs.shape) - self.assertEqual((batch_size, 1), ys.shape) - self.assertEqual(tf.float32, xs.dtype) - self.assertEqual(tf.float32, ys.dtype) - with self.assertRaises(StopIteration): - it.next() - - def testLinearRegression(self): - true_w = [[1.0], [-0.5], [2.0]] - true_b = [1.0] - - model = linear_regression.LinearModel() - dataset = linear_regression.synthetic_dataset( - true_w, true_b, noise_level=0., batch_size=64, num_batches=40) - - with tf.device(device()): - optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1) - linear_regression.fit(model, dataset, optimizer, logdir=self._tmp_logdir) - - self.assertAllClose(true_w, model.variables[0].numpy(), rtol=1e-2) - self.assertAllClose(true_b, model.variables[1].numpy(), rtol=1e-2) - self.assertTrue(glob.glob(os.path.join(self._tmp_logdir, "events.out.*"))) - - -class EagerLinearRegressionBenchmark(tf.test.Benchmark): - - def benchmarkEagerLinearRegression(self): - num_epochs = 10 - num_batches = 200 - batch_size = 64 - dataset = linear_regression.synthetic_dataset( - w=tf.random_uniform([3, 1]), - b=tf.random_uniform([1]), - noise_level=0.01, - batch_size=batch_size, - num_batches=num_batches) - burn_in_dataset = dataset.take(10) - - model = linear_regression.LinearModel() - - with tf.device(device()): - optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1) - - # Perform burn-in. - linear_regression.fit(model, burn_in_dataset, optimizer) - - start_time = time.time() - for _ in range(num_epochs): - linear_regression.fit(model, dataset, optimizer) - wall_time = time.time() - start_time - - examples_per_sec = num_epochs * num_batches * batch_size / wall_time - self.report_benchmark( - name="eager_train_%s" % - ("gpu" if tfe.num_gpus() > 0 else "cpu"), - iters=num_epochs * num_batches, - extras={"examples_per_sec": examples_per_sec}, - wall_time=wall_time) - - -if __name__ == "__main__": - tf.enable_eager_execution() - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/mnist/README.md b/tensorflow/contrib/eager/python/examples/mnist/README.md deleted file mode 100644 index d1c079ff6b5..00000000000 --- a/tensorflow/contrib/eager/python/examples/mnist/README.md +++ /dev/null @@ -1 +0,0 @@ -See https://github.com/tensorflow/models/tree/master/official/mnist/mnist_eager.py diff --git a/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb b/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb deleted file mode 100644 index cabc71c98e1..00000000000 --- a/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb +++ /dev/null @@ -1,845 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "nmt_with_attention.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "metadata": { - "colab_type": "text", - "id": "AOpGoE2T-YXS" - }, - "cell_type": "markdown", - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\").\n", - "\n", - "# Neural Machine Translation with Attention\n", - "\n", - "
\n", - "\n", - " Run in Google Colab \n", - "\n", - "View source on GitHub
" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "CiwtNgENbx2g" - }, - "cell_type": "markdown", - "source": [ - "This notebook trains a sequence to sequence (seq2seq) model for Spanish to English translation using [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager). This is an advanced example that assumes some knowledge of sequence to sequence models.\n", - "\n", - "After training the model in this notebook, you will be able to input a Spanish sentence, such as *\"¿todavia estan en casa?\"*, and return the English translation: *\"are you still at home?\"*\n", - "\n", - "The translation quality is reasonable for a toy example, but the generated attention plot is perhaps more interesting. This shows which parts of the input sentence has the model's attention while translating:\n", - "\n", - "\"spanish-english\n", - "\n", - "Note: This example takes approximately 10 mintues to run on a single P100 GPU." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "tnxXKDjq3jEL", - "colab": {} - }, - "cell_type": "code", - "source": [ - "from __future__ import absolute_import, division, print_function\n", - "\n", - "# Import TensorFlow >= 1.10 and enable eager execution\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()\n", - "\n", - "import matplotlib.pyplot as plt\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "import unicodedata\n", - "import re\n", - "import numpy as np\n", - "import os\n", - "import time\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "wfodePkj3jEa" - }, - "cell_type": "markdown", - "source": [ - "## Download and prepare the dataset\n", - "\n", - "We'll use a language dataset provided by http://www.manythings.org/anki/. This dataset contains language translation pairs in the format:\n", - "\n", - "```\n", - "May I borrow this book?\t¿Puedo tomar prestado este libro?\n", - "```\n", - "\n", - "There are a variety of languages available, but we'll use the English-Spanish dataset. For convenience, we've hosted a copy of this dataset on Google Cloud, but you can also download your own copy. After downloading the dataset, here are the steps we'll take to prepare the data:\n", - "\n", - "1. Add a *start* and *end* token to each sentence.\n", - "2. Clean the sentences by removing special characters.\n", - "3. Create a word index and reverse word index (dictionaries mapping from word → id and id → word).\n", - "4. Pad each sentence to a maximum length." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "kRVATYOgJs1b", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# Download the file\n", - "path_to_zip = tf.keras.utils.get_file(\n", - " 'spa-eng.zip', origin='https://storage.googleapis.com/download.tensorflow.org/data/spa-eng.zip', \n", - " extract=True)\n", - "\n", - "path_to_file = os.path.dirname(path_to_zip)+\"/spa-eng/spa.txt\"" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "rd0jw-eC3jEh", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# Converts the unicode file to ascii\n", - "def unicode_to_ascii(s):\n", - " return ''.join(c for c in unicodedata.normalize('NFD', s)\n", - " if unicodedata.category(c) != 'Mn')\n", - "\n", - "\n", - "def preprocess_sentence(w):\n", - " w = unicode_to_ascii(w.lower().strip())\n", - " \n", - " # creating a space between a word and the punctuation following it\n", - " # eg: \"he is a boy.\" => \"he is a boy .\" \n", - " # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n", - " w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n", - " w = re.sub(r'[\" \"]+', \" \", w)\n", - " \n", - " # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\")\n", - " w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n", - " \n", - " w = w.rstrip().strip()\n", - " \n", - " # adding a start and an end token to the sentence\n", - " # so that the model know when to start and stop predicting.\n", - " w = ' ' + w + ' '\n", - " return w" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "OHn4Dct23jEm", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# 1. Remove the accents\n", - "# 2. Clean the sentences\n", - "# 3. Return word pairs in the format: [ENGLISH, SPANISH]\n", - "def create_dataset(path, num_examples):\n", - " lines = open(path, encoding='UTF-8').read().strip().split('\\n')\n", - " \n", - " word_pairs = [[preprocess_sentence(w) for w in l.split('\\t')] for l in lines[:num_examples]]\n", - " \n", - " return word_pairs" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "9xbqO7Iie9bb", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# This class creates a word -> index mapping (e.g,. \"dad\" -> 5) and vice-versa \n", - "# (e.g., 5 -> \"dad\") for each language,\n", - "class LanguageIndex():\n", - " def __init__(self, lang):\n", - " self.lang = lang\n", - " self.word2idx = {}\n", - " self.idx2word = {}\n", - " self.vocab = set()\n", - " \n", - " self.create_index()\n", - " \n", - " def create_index(self):\n", - " for phrase in self.lang:\n", - " self.vocab.update(phrase.split(' '))\n", - " \n", - " self.vocab = sorted(self.vocab)\n", - " \n", - " self.word2idx[''] = 0\n", - " for index, word in enumerate(self.vocab):\n", - " self.word2idx[word] = index + 1\n", - " \n", - " for word, index in self.word2idx.items():\n", - " self.idx2word[index] = word" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "eAY9k49G3jE_", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def max_length(tensor):\n", - " return max(len(t) for t in tensor)\n", - "\n", - "\n", - "def load_dataset(path, num_examples):\n", - " # creating cleaned input, output pairs\n", - " pairs = create_dataset(path, num_examples)\n", - "\n", - " # index language using the class defined above \n", - " inp_lang = LanguageIndex(sp for en, sp in pairs)\n", - " targ_lang = LanguageIndex(en for en, sp in pairs)\n", - " \n", - " # Vectorize the input and target languages\n", - " \n", - " # Spanish sentences\n", - " input_tensor = [[inp_lang.word2idx[s] for s in sp.split(' ')] for en, sp in pairs]\n", - " \n", - " # English sentences\n", - " target_tensor = [[targ_lang.word2idx[s] for s in en.split(' ')] for en, sp in pairs]\n", - " \n", - " # Calculate max_length of input and output tensor\n", - " # Here, we'll set those to the longest sentence in the dataset\n", - " max_length_inp, max_length_tar = max_length(input_tensor), max_length(target_tensor)\n", - " \n", - " # Padding the input and output tensor to the maximum length\n", - " input_tensor = tf.keras.preprocessing.sequence.pad_sequences(input_tensor, \n", - " maxlen=max_length_inp,\n", - " padding='post')\n", - " \n", - " target_tensor = tf.keras.preprocessing.sequence.pad_sequences(target_tensor, \n", - " maxlen=max_length_tar, \n", - " padding='post')\n", - " \n", - " return input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_tar" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "GOi42V79Ydlr" - }, - "cell_type": "markdown", - "source": [ - "### Limit the size of the dataset to experiment faster (optional)\n", - "\n", - "Training on the complete dataset of >100,000 sentences will take a long time. To train faster, we can limit the size of the dataset to 30,000 sentences (of course, translation quality degrades with less data):" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "cnxC7q-j3jFD", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# Try experimenting with the size of that dataset\n", - "num_examples = 30000\n", - "input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_targ = load_dataset(path_to_file, num_examples)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "4QILQkOs3jFG", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# Creating training and validation sets using an 80-20 split\n", - "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n", - "\n", - "# Show length\n", - "len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "rgCLkfv5uO3d" - }, - "cell_type": "markdown", - "source": [ - "### Create a tf.data dataset" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "TqHsArVZ3jFS", - "colab": {} - }, - "cell_type": "code", - "source": [ - "BUFFER_SIZE = len(input_tensor_train)\n", - "BATCH_SIZE = 64\n", - "N_BATCH = BUFFER_SIZE//BATCH_SIZE\n", - "embedding_dim = 256\n", - "units = 1024\n", - "vocab_inp_size = len(inp_lang.word2idx)\n", - "vocab_tar_size = len(targ_lang.word2idx)\n", - "\n", - "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n", - "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "TNfHIF71ulLu" - }, - "cell_type": "markdown", - "source": [ - "## Write the encoder and decoder model\n", - "\n", - "Here, we'll implement an encoder-decoder model with attention which you can read about in the TensorFlow [Neural Machine Translation (seq2seq) tutorial](https://github.com/tensorflow/nmt). This example uses a more recent set of APIs. This notebook implements the [attention equations](https://github.com/tensorflow/nmt#background-on-the-attention-mechanism) from the seq2seq tutorial. The following diagram shows that each input word is assigned a weight by the attention mechanism which is then used by the decoder to predict the next word in the sentence.\n", - "\n", - "\"attention\n", - "\n", - "The input is put through an encoder model which gives us the encoder output of shape *(batch_size, max_length, hidden_size)* and the encoder hidden state of shape *(batch_size, hidden_size)*. \n", - "\n", - "Here are the equations that are implemented:\n", - "\n", - "\"attention\n", - "\"attention\n", - "\n", - "We're using *Bahdanau attention*. Lets decide on notation before writing the simplified form:\n", - "\n", - "* FC = Fully connected (dense) layer\n", - "* EO = Encoder output\n", - "* H = hidden state\n", - "* X = input to the decoder\n", - "\n", - "And the pseudo-code:\n", - "\n", - "* `score = FC(tanh(FC(EO) + FC(H)))`\n", - "* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, 1)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n", - "* `context vector = sum(attention weights * EO, axis = 1)`. Same reason as above for choosing axis as 1.\n", - "* `embedding output` = The input to the decoder X is passed through an embedding layer.\n", - "* `merged vector = concat(embedding output, context vector)`\n", - "* This merged vector is then given to the GRU\n", - " \n", - "The shapes of all the vectors at each step have been specified in the comments in the code:" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "avyJ_4VIUoHb", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def gru(units):\n", - " # If you have a GPU, we recommend using CuDNNGRU(provides a 3x speedup than GRU)\n", - " # the code automatically does that.\n", - " if tf.test.is_gpu_available():\n", - " return tf.keras.layers.CuDNNGRU(units, \n", - " return_sequences=True, \n", - " return_state=True, \n", - " recurrent_initializer='glorot_uniform')\n", - " else:\n", - " return tf.keras.layers.GRU(units, \n", - " return_sequences=True, \n", - " return_state=True, \n", - " recurrent_activation='sigmoid', \n", - " recurrent_initializer='glorot_uniform')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "nZ2rI24i3jFg", - "colab": {} - }, - "cell_type": "code", - "source": [ - "class Encoder(tf.keras.Model):\n", - " def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n", - " super(Encoder, self).__init__()\n", - " self.batch_sz = batch_sz\n", - " self.enc_units = enc_units\n", - " self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n", - " self.gru = gru(self.enc_units)\n", - " \n", - " def call(self, x, hidden):\n", - " x = self.embedding(x)\n", - " output, state = self.gru(x, initial_state = hidden) \n", - " return output, state\n", - " \n", - " def initialize_hidden_state(self):\n", - " return tf.zeros((self.batch_sz, self.enc_units))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "yJ_B3mhW3jFk", - "colab": {} - }, - "cell_type": "code", - "source": [ - "class Decoder(tf.keras.Model):\n", - " def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n", - " super(Decoder, self).__init__()\n", - " self.batch_sz = batch_sz\n", - " self.dec_units = dec_units\n", - " self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n", - " self.gru = gru(self.dec_units)\n", - " self.fc = tf.keras.layers.Dense(vocab_size)\n", - " \n", - " # used for attention\n", - " self.W1 = tf.keras.layers.Dense(self.dec_units)\n", - " self.W2 = tf.keras.layers.Dense(self.dec_units)\n", - " self.V = tf.keras.layers.Dense(1)\n", - " \n", - " def call(self, x, hidden, enc_output):\n", - " # enc_output shape == (batch_size, max_length, hidden_size)\n", - " \n", - " # hidden shape == (batch_size, hidden size)\n", - " # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n", - " # we are doing this to perform addition to calculate the score\n", - " hidden_with_time_axis = tf.expand_dims(hidden, 1)\n", - " \n", - " # score shape == (batch_size, max_length, 1)\n", - " # we get 1 at the last axis because we are applying tanh(FC(EO) + FC(H)) to self.V\n", - " score = self.V(tf.nn.tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis)))\n", - " \n", - " # attention_weights shape == (batch_size, max_length, 1)\n", - " attention_weights = tf.nn.softmax(score, axis=1)\n", - " \n", - " # context_vector shape after sum == (batch_size, hidden_size)\n", - " context_vector = attention_weights * enc_output\n", - " context_vector = tf.reduce_sum(context_vector, axis=1)\n", - " \n", - " # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n", - " x = self.embedding(x)\n", - " \n", - " # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n", - " x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n", - " \n", - " # passing the concatenated vector to the GRU\n", - " output, state = self.gru(x)\n", - " \n", - " # output shape == (batch_size * 1, hidden_size)\n", - " output = tf.reshape(output, (-1, output.shape[2]))\n", - " \n", - " # output shape == (batch_size * 1, vocab)\n", - " x = self.fc(output)\n", - " \n", - " return x, state, attention_weights\n", - " \n", - " def initialize_hidden_state(self):\n", - " return tf.zeros((self.batch_sz, self.dec_units))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "P5UY8wko3jFp", - "colab": {} - }, - "cell_type": "code", - "source": [ - "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n", - "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "_ch_71VbIRfK" - }, - "cell_type": "markdown", - "source": [ - "## Define the optimizer and the loss function" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "WmTHr5iV3jFr", - "colab": {} - }, - "cell_type": "code", - "source": [ - "optimizer = tf.train.AdamOptimizer()\n", - "\n", - "\n", - "def loss_function(real, pred):\n", - " mask = 1 - np.equal(real, 0)\n", - " loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n", - " return tf.reduce_mean(loss_)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "DMVWzzsfNl4e" - }, - "cell_type": "markdown", - "source": [ - "## Checkpoints (Object-based saving)" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "Zj8bXQTgNwrF", - "colab": {} - }, - "cell_type": "code", - "source": [ - "checkpoint_dir = './training_checkpoints'\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n", - "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n", - " encoder=encoder,\n", - " decoder=decoder)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "hpObfY22IddU" - }, - "cell_type": "markdown", - "source": [ - "## Training\n", - "\n", - "1. Pass the *input* through the *encoder* which return *encoder output* and the *encoder hidden state*.\n", - "2. The encoder output, encoder hidden state and the decoder input (which is the *start token*) is passed to the decoder.\n", - "3. The decoder returns the *predictions* and the *decoder hidden state*.\n", - "4. The decoder hidden state is then passed back into the model and the predictions are used to calculate the loss.\n", - "5. Use *teacher forcing* to decide the next input to the decoder.\n", - "6. *Teacher forcing* is the technique where the *target word* is passed as the *next input* to the decoder.\n", - "7. The final step is to calculate the gradients and apply it to the optimizer and backpropagate." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "ddefjBMa3jF0", - "colab": {} - }, - "cell_type": "code", - "source": [ - "EPOCHS = 10\n", - "\n", - "for epoch in range(EPOCHS):\n", - " start = time.time()\n", - " \n", - " hidden = encoder.initialize_hidden_state()\n", - " total_loss = 0\n", - " \n", - " for (batch, (inp, targ)) in enumerate(dataset):\n", - " loss = 0\n", - " \n", - " with tf.GradientTape() as tape:\n", - " enc_output, enc_hidden = encoder(inp, hidden)\n", - " \n", - " dec_hidden = enc_hidden\n", - " \n", - " dec_input = tf.expand_dims([targ_lang.word2idx['']] * BATCH_SIZE, 1) \n", - " \n", - " # Teacher forcing - feeding the target as the next input\n", - " for t in range(1, targ.shape[1]):\n", - " # passing enc_output to the decoder\n", - " predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n", - " \n", - " loss += loss_function(targ[:, t], predictions)\n", - " \n", - " # using teacher forcing\n", - " dec_input = tf.expand_dims(targ[:, t], 1)\n", - " \n", - " batch_loss = (loss / int(targ.shape[1]))\n", - " \n", - " total_loss += batch_loss\n", - " \n", - " variables = encoder.variables + decoder.variables\n", - " \n", - " gradients = tape.gradient(loss, variables)\n", - " \n", - " optimizer.apply_gradients(zip(gradients, variables))\n", - " \n", - " if batch % 100 == 0:\n", - " print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n", - " batch,\n", - " batch_loss.numpy()))\n", - " # saving (checkpoint) the model every 2 epochs\n", - " if (epoch + 1) % 2 == 0:\n", - " checkpoint.save(file_prefix = checkpoint_prefix)\n", - " \n", - " print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n", - " total_loss / N_BATCH))\n", - " print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "mU3Ce8M6I3rz" - }, - "cell_type": "markdown", - "source": [ - "## Translate\n", - "\n", - "* The evaluate function is similar to the training loop, except we don't use *teacher forcing* here. The input to the decoder at each time step is its previous predictions along with the hidden state and the encoder output.\n", - "* Stop predicting when the model predicts the *end token*.\n", - "* And store the *attention weights for every time step*.\n", - "\n", - "Note: The encoder output is calculated only once for one input." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "EbQpyYs13jF_", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n", - " attention_plot = np.zeros((max_length_targ, max_length_inp))\n", - " \n", - " sentence = preprocess_sentence(sentence)\n", - "\n", - " inputs = [inp_lang.word2idx[i] for i in sentence.split(' ')]\n", - " inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs], maxlen=max_length_inp, padding='post')\n", - " inputs = tf.convert_to_tensor(inputs)\n", - " \n", - " result = ''\n", - "\n", - " hidden = [tf.zeros((1, units))]\n", - " enc_out, enc_hidden = encoder(inputs, hidden)\n", - "\n", - " dec_hidden = enc_hidden\n", - " dec_input = tf.expand_dims([targ_lang.word2idx['']], 0)\n", - "\n", - " for t in range(max_length_targ):\n", - " predictions, dec_hidden, attention_weights = decoder(dec_input, dec_hidden, enc_out)\n", - " \n", - " # storing the attention weights to plot later on\n", - " attention_weights = tf.reshape(attention_weights, (-1, ))\n", - " attention_plot[t] = attention_weights.numpy()\n", - "\n", - " predicted_id = tf.argmax(predictions[0]).numpy()\n", - "\n", - " result += targ_lang.idx2word[predicted_id] + ' '\n", - "\n", - " if targ_lang.idx2word[predicted_id] == '':\n", - " return result, sentence, attention_plot\n", - " \n", - " # the predicted ID is fed back into the model\n", - " dec_input = tf.expand_dims([predicted_id], 0)\n", - "\n", - " return result, sentence, attention_plot" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "s5hQWlbN3jGF", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# function for plotting the attention weights\n", - "def plot_attention(attention, sentence, predicted_sentence):\n", - " fig = plt.figure(figsize=(10,10))\n", - " ax = fig.add_subplot(1, 1, 1)\n", - " ax.matshow(attention, cmap='viridis')\n", - " \n", - " fontdict = {'fontsize': 14}\n", - " \n", - " ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n", - " ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n", - "\n", - " plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "sl9zUHzg3jGI", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def translate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n", - " result, sentence, attention_plot = evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)\n", - " \n", - " print('Input: {}'.format(sentence))\n", - " print('Predicted translation: {}'.format(result))\n", - " \n", - " attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n", - " plot_attention(attention_plot, sentence.split(' '), result.split(' '))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "n250XbnjOaqP" - }, - "cell_type": "markdown", - "source": [ - "## Restore the latest checkpoint and test" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "UJpT9D5_OgP6", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# restoring the latest checkpoint in checkpoint_dir\n", - "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "WrAM0FDomq3E", - "colab": {} - }, - "cell_type": "code", - "source": [ - "translate(u'hace mucho frio aqui.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "zSx2iM36EZQZ", - "colab": {} - }, - "cell_type": "code", - "source": [ - "translate(u'esta es mi vida.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "A3LLCx3ZE0Ls", - "colab": {} - }, - "cell_type": "code", - "source": [ - "translate(u'todavia estan en casa?', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "DUQVLVqUE1YW", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# wrong translation\n", - "translate(u'trata de averiguarlo.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "RTe5P5ioMJwN" - }, - "cell_type": "markdown", - "source": [ - "## Next steps\n", - "\n", - "* [Download a different dataset](http://www.manythings.org/anki/) to experiment with translations, for example, English to German, or English to French.\n", - "* Experiment with training on a larger dataset, or using more epochs\n" - ] - } - ] -} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/README.md b/tensorflow/contrib/eager/python/examples/notebooks/README.md deleted file mode 100644 index 2778b228e93..00000000000 --- a/tensorflow/contrib/eager/python/examples/notebooks/README.md +++ /dev/null @@ -1,3 +0,0 @@ -The notebooks have been moved to the -[tensorflow/docs](https://github.com/tensorflow/docs/tree/master/site/en/tutorials/eager) -repository. diff --git a/tensorflow/contrib/eager/python/examples/notebooks/automatic_differentiation.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/automatic_differentiation.ipynb deleted file mode 100644 index 446e3401184..00000000000 --- a/tensorflow/contrib/eager/python/examples/notebooks/automatic_differentiation.ipynb +++ /dev/null @@ -1,88 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t09eeeR5prIJ" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "GCCk8_dHpuNf" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xh8WkEwWpnm7" - }, - "source": [ - "# Automatic differentiation and gradient tape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "clNGnJ3u8Rl6" - }, - "source": [ - "This file has moved." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "idv0bPeCp325" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/eager/automatic_differentiation.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - "\u003c/td\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/eager/automatic_differentiation.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "automatic_differentiation.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/custom_layers.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/custom_layers.ipynb deleted file mode 100644 index d89774c45ef..00000000000 --- a/tensorflow/contrib/eager/python/examples/notebooks/custom_layers.ipynb +++ /dev/null @@ -1,88 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tDnwEv8FtJm7" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "JlknJBWQtKkI" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "60RdWsg1tETW" - }, - "source": [ - "# Custom layers" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9sFn_RV_8zM-" - }, - "source": [ - "This file has moved." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BcJg7Enms86w" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/eager/custom_layers.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - "\u003c/td\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/eager/custom_layers.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "custom_layers.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb deleted file mode 100644 index 86dca0b423d..00000000000 --- a/tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb +++ /dev/null @@ -1,88 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5rmpybwysXGV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "m8y3rGtQsYP2" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hrXv0rU9sIma" - }, - "source": [ - "# Custom training: basics" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IGPZTmwn9IT4" - }, - "source": [ - "This file has moved." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7S0BwJ_8sLu7" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/eager/custom_training.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - "\u003c/td\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/eager/custom_training.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "Custom training: basics", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/eager_basics.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/eager_basics.ipynb deleted file mode 100644 index c6d1a566043..00000000000 --- a/tensorflow/contrib/eager/python/examples/notebooks/eager_basics.ipynb +++ /dev/null @@ -1,78 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iPpI7RaYoZuE" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "hro2InpHobKk" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U9i2Dsh-ziXr" - }, - "source": [ - "# Eager execution basics" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Hndw-YcxoOJK" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/eager/eager_basics.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - "\u003c/td\u003e\u003ctd\u003e\n", - "\u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/eager/eager_basics.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "eager_basics.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/pix2pix/pix2pix_eager.ipynb b/tensorflow/contrib/eager/python/examples/pix2pix/pix2pix_eager.ipynb deleted file mode 100644 index 57bd18d7529..00000000000 --- a/tensorflow/contrib/eager/python/examples/pix2pix/pix2pix_eager.ipynb +++ /dev/null @@ -1,51 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0TD5ZrvEMbhZ" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\").\n", - "\n", - "# Pix2Pix" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c7W3j96p219v" - }, - "source": [ - "This notebook has been moved to [https://github.com/tensorflow/docs/blob/master/site/en/r2/tutorials/generative/pix2pix.ipynb](https://github.com/tensorflow/docs/blob/master/site/en/r2/tutorials/generative/pix2pix.ipynb)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "pix2pix_eager.ipynb", - "private_outputs": true, - "provenance": [ - { - "file_id": "1eb0NOTQapkYs3X0v-zL1x5_LFKgDISnp", - "timestamp": 1527173385672 - } - ], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/resnet50/BUILD b/tensorflow/contrib/eager/python/examples/resnet50/BUILD deleted file mode 100644 index 0ebf17e7b36..00000000000 --- a/tensorflow/contrib/eager/python/examples/resnet50/BUILD +++ /dev/null @@ -1,64 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "cuda_py_test") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "resnet50", - srcs = ["resnet50.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/eager/python:tfe", - ], -) - -py_library( - name = "resnet50_test_lib", - srcs = ["resnet50_test.py"], - srcs_version = "PY2AND3", - deps = [ - ":resnet50", - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/eager/python:tfe", - ], -) - -cuda_py_test( - name = "resnet50_test", - size = "medium", - srcs = ["resnet50_test.py"], - additional_deps = [ - ":resnet50", - "//tensorflow/contrib/summary:summary_test_util", - "//tensorflow/contrib/eager/python:tfe", - "//tensorflow:tensorflow_py", - ], - shard_count = 4, - tags = [ - "optonly", - "oss_serial", - ], -) - -cuda_py_test( - name = "resnet50_graph_test", - size = "medium", - srcs = ["resnet50_graph_test.py"], - additional_deps = [ - ":resnet50", - "//tensorflow/contrib/summary:summary_test_util", - "//third_party/py/numpy", - "//tensorflow:tensorflow_py", - ], - shard_count = 4, - tags = [ - "noasan", - "nomsan", - "notsan", - "optonly", - "oss_serial", - ], -) diff --git a/tensorflow/contrib/eager/python/examples/resnet50/README.md b/tensorflow/contrib/eager/python/examples/resnet50/README.md deleted file mode 100644 index 79e46005294..00000000000 --- a/tensorflow/contrib/eager/python/examples/resnet50/README.md +++ /dev/null @@ -1,45 +0,0 @@ -Image classification using the ResNet50 model described in -[Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385). - -Contents: - -- `resnet50.py`: Model definition -- `resnet50_test.py`: Sanity unittests and benchmarks for using the model with - eager execution enabled. -- `resnet50_graph_test.py`: Sanity unittests and benchmarks when using the same - model code to construct a TensorFlow graph. - -# Benchmarks - -Using a synthetic data, run: - -``` -# Using eager execution -python resnet50_test.py --benchmarks=. - -# Using graph execution -python resnet50_graph_test.py --benchmarks=. -``` - -The above uses the model definition included with the TensorFlow pip -package. To build (and run benchmarks) from source: - -``` -# Using eager execution -bazel run -c opt --config=cuda :resnet50_test -- --benchmarks=. - -# Using graph execution -bazel run -c opt --config=cuda :resnet50_graph_test -- --benchmarks=. -``` - -(Or remove the `--config=cuda` flag for running on CPU instead of GPU). - -On October 31, 2017, the benchmarks demonstrated comparable performance -for eager and graph execution of this particular model when using -a single NVIDIA Titan X (Pascal) GPU on a host with an -Intel Xeon E5-1650 CPU @ 3.50GHz and a batch size of 32. - -| Benchmark name | batch size | images/second | -| --------------------------------------- | ------------- | ------------- | -| eager_train_gpu_batch_32_channels_first | 32 | 171 | -| graph_train_gpu_batch_32_channels_first | 32 | 172 | diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50.py deleted file mode 100644 index 9d090e84291..00000000000 --- a/tensorflow/contrib/eager/python/examples/resnet50/resnet50.py +++ /dev/null @@ -1,308 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ResNet50 model definition compatible with TensorFlow's eager execution. - -Reference [Deep Residual Learning for Image -Recognition](https://arxiv.org/abs/1512.03385) - -Adapted from tf.keras.applications.ResNet50. A notable difference is that the -model here outputs logits while the Keras model outputs probability. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -import tensorflow as tf - -layers = tf.keras.layers - - -class _IdentityBlock(tf.keras.Model): - """_IdentityBlock is the block that has no conv layer at shortcut. - - Args: - kernel_size: the kernel size of middle conv layer at main path - filters: list of integers, the filters of 3 conv layer at main path - stage: integer, current stage label, used for generating layer names - block: 'a','b'..., current block label, used for generating layer names - data_format: data_format for the input ('channels_first' or - 'channels_last'). - """ - - def __init__(self, kernel_size, filters, stage, block, data_format): - super(_IdentityBlock, self).__init__(name='') - filters1, filters2, filters3 = filters - - conv_name_base = 'res' + str(stage) + block + '_branch' - bn_name_base = 'bn' + str(stage) + block + '_branch' - bn_axis = 1 if data_format == 'channels_first' else 3 - - self.conv2a = layers.Conv2D( - filters1, (1, 1), name=conv_name_base + '2a', data_format=data_format) - self.bn2a = layers.BatchNormalization( - axis=bn_axis, name=bn_name_base + '2a') - - self.conv2b = layers.Conv2D( - filters2, - kernel_size, - padding='same', - data_format=data_format, - name=conv_name_base + '2b') - self.bn2b = layers.BatchNormalization( - axis=bn_axis, name=bn_name_base + '2b') - - self.conv2c = layers.Conv2D( - filters3, (1, 1), name=conv_name_base + '2c', data_format=data_format) - self.bn2c = layers.BatchNormalization( - axis=bn_axis, name=bn_name_base + '2c') - - def call(self, input_tensor, training=False): - x = self.conv2a(input_tensor) - x = self.bn2a(x, training=training) - x = tf.nn.relu(x) - - x = self.conv2b(x) - x = self.bn2b(x, training=training) - x = tf.nn.relu(x) - - x = self.conv2c(x) - x = self.bn2c(x, training=training) - - x += input_tensor - return tf.nn.relu(x) - - -class _ConvBlock(tf.keras.Model): - """_ConvBlock is the block that has a conv layer at shortcut. - - Args: - kernel_size: the kernel size of middle conv layer at main path - filters: list of integers, the filters of 3 conv layer at main path - stage: integer, current stage label, used for generating layer names - block: 'a','b'..., current block label, used for generating layer names - data_format: data_format for the input ('channels_first' or - 'channels_last'). - strides: strides for the convolution. Note that from stage 3, the first - conv layer at main path is with strides=(2,2), and the shortcut should - have strides=(2,2) as well. - """ - - def __init__(self, - kernel_size, - filters, - stage, - block, - data_format, - strides=(2, 2)): - super(_ConvBlock, self).__init__(name='') - filters1, filters2, filters3 = filters - - conv_name_base = 'res' + str(stage) + block + '_branch' - bn_name_base = 'bn' + str(stage) + block + '_branch' - bn_axis = 1 if data_format == 'channels_first' else 3 - - self.conv2a = layers.Conv2D( - filters1, (1, 1), - strides=strides, - name=conv_name_base + '2a', - data_format=data_format) - self.bn2a = layers.BatchNormalization( - axis=bn_axis, name=bn_name_base + '2a') - - self.conv2b = layers.Conv2D( - filters2, - kernel_size, - padding='same', - name=conv_name_base + '2b', - data_format=data_format) - self.bn2b = layers.BatchNormalization( - axis=bn_axis, name=bn_name_base + '2b') - - self.conv2c = layers.Conv2D( - filters3, (1, 1), name=conv_name_base + '2c', data_format=data_format) - self.bn2c = layers.BatchNormalization( - axis=bn_axis, name=bn_name_base + '2c') - - self.conv_shortcut = layers.Conv2D( - filters3, (1, 1), - strides=strides, - name=conv_name_base + '1', - data_format=data_format) - self.bn_shortcut = layers.BatchNormalization( - axis=bn_axis, name=bn_name_base + '1') - - def call(self, input_tensor, training=False): - x = self.conv2a(input_tensor) - x = self.bn2a(x, training=training) - x = tf.nn.relu(x) - - x = self.conv2b(x) - x = self.bn2b(x, training=training) - x = tf.nn.relu(x) - - x = self.conv2c(x) - x = self.bn2c(x, training=training) - - shortcut = self.conv_shortcut(input_tensor) - shortcut = self.bn_shortcut(shortcut, training=training) - - x += shortcut - return tf.nn.relu(x) - - -# pylint: disable=not-callable -class ResNet50(tf.keras.Model): - """Instantiates the ResNet50 architecture. - - Args: - data_format: format for the image. Either 'channels_first' or - 'channels_last'. 'channels_first' is typically faster on GPUs while - 'channels_last' is typically faster on CPUs. See - https://www.tensorflow.org/performance/performance_guide#data_formats - name: Prefix applied to names of variables created in the model. - trainable: Is the model trainable? If true, performs backward - and optimization after call() method. - include_top: whether to include the fully-connected layer at the top of the - network. - pooling: Optional pooling mode for feature extraction when `include_top` - is `False`. - - `None` means that the output of the model will be the 4D tensor - output of the last convolutional layer. - - `avg` means that global average pooling will be applied to the output of - the last convolutional layer, and thus the output of the model will be - a 2D tensor. - - `max` means that global max pooling will be applied. - classes: optional number of classes to classify images into, only to be - specified if `include_top` is True. - - Raises: - ValueError: in case of invalid argument for data_format. - """ - - def __init__(self, - data_format, - name='', - trainable=True, - include_top=True, - pooling=None, - classes=1000): - super(ResNet50, self).__init__(name=name) - - valid_channel_values = ('channels_first', 'channels_last') - if data_format not in valid_channel_values: - raise ValueError('Unknown data_format: %s. Valid values: %s' % - (data_format, valid_channel_values)) - self.include_top = include_top - - def conv_block(filters, stage, block, strides=(2, 2)): - return _ConvBlock( - 3, - filters, - stage=stage, - block=block, - data_format=data_format, - strides=strides) - - def id_block(filters, stage, block): - return _IdentityBlock( - 3, filters, stage=stage, block=block, data_format=data_format) - - self.conv1 = layers.Conv2D( - 64, (7, 7), - strides=(2, 2), - data_format=data_format, - padding='same', - name='conv1') - bn_axis = 1 if data_format == 'channels_first' else 3 - self.bn_conv1 = layers.BatchNormalization(axis=bn_axis, name='bn_conv1') - self.max_pool = layers.MaxPooling2D( - (3, 3), strides=(2, 2), data_format=data_format) - - self.l2a = conv_block([64, 64, 256], stage=2, block='a', strides=(1, 1)) - self.l2b = id_block([64, 64, 256], stage=2, block='b') - self.l2c = id_block([64, 64, 256], stage=2, block='c') - - self.l3a = conv_block([128, 128, 512], stage=3, block='a') - self.l3b = id_block([128, 128, 512], stage=3, block='b') - self.l3c = id_block([128, 128, 512], stage=3, block='c') - self.l3d = id_block([128, 128, 512], stage=3, block='d') - - self.l4a = conv_block([256, 256, 1024], stage=4, block='a') - self.l4b = id_block([256, 256, 1024], stage=4, block='b') - self.l4c = id_block([256, 256, 1024], stage=4, block='c') - self.l4d = id_block([256, 256, 1024], stage=4, block='d') - self.l4e = id_block([256, 256, 1024], stage=4, block='e') - self.l4f = id_block([256, 256, 1024], stage=4, block='f') - - self.l5a = conv_block([512, 512, 2048], stage=5, block='a') - self.l5b = id_block([512, 512, 2048], stage=5, block='b') - self.l5c = id_block([512, 512, 2048], stage=5, block='c') - - self.avg_pool = layers.AveragePooling2D( - (7, 7), strides=(7, 7), data_format=data_format) - - if self.include_top: - self.flatten = layers.Flatten() - self.fc1000 = layers.Dense(classes, name='fc1000') - else: - reduction_indices = [1, 2] if data_format == 'channels_last' else [2, 3] - reduction_indices = tf.constant(reduction_indices) - if pooling == 'avg': - self.global_pooling = functools.partial( - tf.reduce_mean, - reduction_indices=reduction_indices, - keep_dims=False) - elif pooling == 'max': - self.global_pooling = functools.partial( - tf.reduce_max, reduction_indices=reduction_indices, keep_dims=False) - else: - self.global_pooling = None - - def call(self, inputs, training=True): - x = self.conv1(inputs) - x = self.bn_conv1(x, training=training) - x = tf.nn.relu(x) - x = self.max_pool(x) - - x = self.l2a(x, training=training) - x = self.l2b(x, training=training) - x = self.l2c(x, training=training) - - x = self.l3a(x, training=training) - x = self.l3b(x, training=training) - x = self.l3c(x, training=training) - x = self.l3d(x, training=training) - - x = self.l4a(x, training=training) - x = self.l4b(x, training=training) - x = self.l4c(x, training=training) - x = self.l4d(x, training=training) - x = self.l4e(x, training=training) - x = self.l4f(x, training=training) - - x = self.l5a(x, training=training) - x = self.l5b(x, training=training) - x = self.l5c(x, training=training) - - x = self.avg_pool(x) - - if self.include_top: - return self.fc1000(self.flatten(x)) - elif self.global_pooling: - return self.global_pooling(x) - else: - return x diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py deleted file mode 100644 index fb7975d8fe8..00000000000 --- a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests and benchmarks for ResNet50 under graph execution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tempfile -import time - -import numpy as np -import tensorflow as tf - -from tensorflow.contrib.eager.python.examples.resnet50 import resnet50 -from tensorflow.contrib.summary import summary_test_util - - -def data_format(): - return 'channels_first' if tf.test.is_gpu_available() else 'channels_last' - - -def image_shape(batch_size): - if data_format() == 'channels_first': - return [batch_size, 3, 224, 224] - return [batch_size, 224, 224, 3] - - -def random_batch(batch_size): - images = np.random.rand(*image_shape(batch_size)).astype(np.float32) - num_classes = 1000 - labels = np.random.randint( - low=0, high=num_classes, size=[batch_size]).astype(np.int32) - one_hot = np.zeros((batch_size, num_classes)).astype(np.float32) - one_hot[np.arange(batch_size), labels] = 1. - return images, one_hot - - -class ResNet50GraphTest(tf.test.TestCase): - - def testApply(self): - # Use small batches for tests because the OSS version runs - # in constrained GPU environment with 1-2GB of memory. - batch_size = 8 - with tf.Graph().as_default(): - images = tf.placeholder(tf.float32, image_shape(None)) - model = resnet50.ResNet50(data_format()) - predictions = model(images, training=False) - - init = tf.global_variables_initializer() - - with tf.Session() as sess: - sess.run(init) - np_images, _ = random_batch(batch_size) - out = sess.run(predictions, feed_dict={images: np_images}) - self.assertAllEqual([batch_size, 1000], out.shape) - - def testTrainWithSummary(self): - with tf.Graph().as_default(): - images = tf.placeholder(tf.float32, image_shape(None), name='images') - labels = tf.placeholder(tf.float32, [None, 1000], name='labels') - - tf.train.get_or_create_global_step() - logdir = tempfile.mkdtemp() - with tf.contrib.summary.always_record_summaries(): - with tf.contrib.summary.create_file_writer( - logdir, max_queue=0, - name='t0').as_default(): - model = resnet50.ResNet50(data_format()) - logits = model(images, training=True) - loss = tf.losses.softmax_cross_entropy( - logits=logits, onehot_labels=labels) - tf.contrib.summary.scalar(name='loss', tensor=loss) - optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) - train_op = optimizer.minimize(loss) - - init = tf.global_variables_initializer() - self.assertEqual(321, len(tf.global_variables())) - - # Use small batches for tests because the OSS version runs - # in constrained GPU environment with 1-2GB of memory. - batch_size = 2 - with tf.Session() as sess: - sess.run(init) - sess.run(tf.contrib.summary.summary_writer_initializer_op()) - np_images, np_labels = random_batch(batch_size) - sess.run([train_op, tf.contrib.summary.all_summary_ops()], - feed_dict={images: np_images, labels: np_labels}) - - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 2) - self.assertEqual(events[1].summary.value[0].tag, 'loss') - - -class ResNet50Benchmarks(tf.test.Benchmark): - - def _report(self, label, start, num_iters, batch_size): - avg_time = (time.time() - start) / num_iters - dev = 'gpu' if tf.test.is_gpu_available() else 'cpu' - name = 'graph_%s_%s_batch_%d_%s' % (label, dev, batch_size, data_format()) - extras = {'examples_per_sec': batch_size / avg_time} - self.report_benchmark( - iters=num_iters, wall_time=avg_time, name=name, extras=extras) - - def benchmark_graph_apply(self): - with tf.Graph().as_default(): - images = tf.placeholder(tf.float32, image_shape(None)) - model = resnet50.ResNet50(data_format()) - predictions = model(images, training=False) - - init = tf.global_variables_initializer() - - batch_size = 64 - with tf.Session() as sess: - sess.run(init) - np_images, _ = random_batch(batch_size) - num_burn, num_iters = (3, 30) - for _ in range(num_burn): - sess.run(predictions, feed_dict={images: np_images}) - start = time.time() - for _ in range(num_iters): - # Comparison with the eager execution benchmark in resnet50_test.py - # isn't entirely fair as the time here includes the cost of copying - # the feeds from CPU memory to GPU. - sess.run(predictions, feed_dict={images: np_images}) - self._report('apply', start, num_iters, batch_size) - - def benchmark_graph_train(self): - for batch_size in [16, 32, 64]: - with tf.Graph().as_default(): - np_images, np_labels = random_batch(batch_size) - dataset = tf.data.Dataset.from_tensors((np_images, np_labels)).repeat() - images, labels = tf.compat.v1.data.make_one_shot_iterator( - dataset).get_next() - - model = resnet50.ResNet50(data_format()) - logits = model(images, training=True) - loss = tf.losses.softmax_cross_entropy( - logits=logits, onehot_labels=labels) - optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) - train_op = optimizer.minimize(loss) - - init = tf.global_variables_initializer() - with tf.Session() as sess: - sess.run(init) - (num_burn, num_iters) = (5, 10) - for _ in range(num_burn): - sess.run(train_op) - start = time.time() - for _ in range(num_iters): - sess.run(train_op) - self._report('train', start, num_iters, batch_size) - - -if __name__ == '__main__': - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py deleted file mode 100644 index fb81979d7bd..00000000000 --- a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py +++ /dev/null @@ -1,337 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests and benchmarks for the ResNet50 model, executed eagerly.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import gc -import tempfile -import time - -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow as tf - -import tensorflow.contrib.eager as tfe -from tensorflow.contrib.eager.python.examples.resnet50 import resnet50 -from tensorflow.contrib.summary import summary_test_util -from tensorflow.python.client import device_lib -from tensorflow.python.eager import tape - - -def device_and_data_format(): - return ('/gpu:0', 'channels_first') if tfe.num_gpus() else ('/cpu:0', - 'channels_last') - - -def random_batch(batch_size, data_format): - shape = (3, 224, 224) if data_format == 'channels_first' else (224, 224, 3) - shape = (batch_size,) + shape - - num_classes = 1000 - images = tf.random_uniform(shape) - labels = tf.random_uniform( - [batch_size], minval=0, maxval=num_classes, dtype=tf.int32) - one_hot = tf.one_hot(labels, num_classes) - - return images, one_hot - - -def compute_gradients(model, images, labels, num_replicas=1): - with tf.GradientTape() as grad_tape: - logits = model(images, training=True) - loss = tf.losses.softmax_cross_entropy( - logits=logits, onehot_labels=labels) - tf.contrib.summary.scalar(name='loss', tensor=loss) - if num_replicas != 1: - loss /= num_replicas - - # TODO(b/110991947): We can mistakenly trace the gradient call in - # multi-threaded environment. Explicitly disable recording until - # this is fixed. - with tape.stop_recording(): - grads = grad_tape.gradient(loss, model.variables) - return grads - - -def apply_gradients(model, optimizer, gradients): - optimizer.apply_gradients(zip(gradients, model.variables)) - - -class ResNet50Test(tf.test.TestCase): - - def _apply(self, defun=False, execution_mode=None): - device, data_format = device_and_data_format() - model = resnet50.ResNet50(data_format) - if defun: - model.call = tfe.function(model.call) - with tf.device(device), tfe.execution_mode(execution_mode): - images, _ = random_batch(2, data_format) - output = model(images, training=False) - tfe.async_wait() - self.assertEqual((2, 1000), output.shape) - - def test_apply(self): - self._apply(defun=False) - - def test_apply_async(self): - self._apply(defun=False, execution_mode=tfe.ASYNC) - - def test_apply_with_defun(self): - self._apply(defun=True) - - def test_apply_with_defun_async(self): - self._apply(defun=True, execution_mode=tfe.ASYNC) - - def test_apply_no_top(self): - device, data_format = device_and_data_format() - model = resnet50.ResNet50(data_format, include_top=False) - with tf.device(device): - images, _ = random_batch(2, data_format) - output = model(images, training=False) - output_shape = ((2, 2048, 1, 1) - if data_format == 'channels_first' else (2, 1, 1, 2048)) - self.assertEqual(output_shape, output.shape) - - def test_apply_with_pooling(self): - device, data_format = device_and_data_format() - model = resnet50.ResNet50(data_format, include_top=False, pooling='avg') - with tf.device(device): - images, _ = random_batch(2, data_format) - output = model(images, training=False) - self.assertEqual((2, 2048), output.shape) - - def _test_train(self, execution_mode=None): - device, data_format = device_and_data_format() - model = resnet50.ResNet50(data_format) - tf.train.get_or_create_global_step() - logdir = tempfile.mkdtemp() - with tf.contrib.summary.create_file_writer( - logdir, max_queue=0, - name='t0').as_default(), tf.contrib.summary.always_record_summaries(): - with tf.device(device), tfe.execution_mode(execution_mode): - optimizer = tf.train.GradientDescentOptimizer(0.1) - images, labels = random_batch(2, data_format) - apply_gradients(model, optimizer, - compute_gradients(model, images, labels)) - self.assertEqual(320, len(model.variables)) - tfe.async_wait() - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 2) - self.assertEqual(events[1].summary.value[0].tag, 'loss') - - def test_train(self): - self._test_train() - - def test_train_async(self): - self._test_train(execution_mode=tfe.ASYNC) - - def test_no_garbage(self): - device, data_format = device_and_data_format() - model = resnet50.ResNet50(data_format) - optimizer = tf.train.GradientDescentOptimizer(0.1) - with tf.device(device): - images, labels = random_batch(2, data_format) - gc.disable() - # Warm up. Note that this first run does create significant amounts of - # garbage to be collected. The hope is that this is a build-only effect, - # and a subsequent training loop will create nothing which needs to be - # collected. - apply_gradients(model, optimizer, - compute_gradients(model, images, labels)) - gc.collect() - previous_gc_debug_flags = gc.get_debug() - gc.set_debug(gc.DEBUG_SAVEALL) - for _ in range(2): - # Run twice to ensure that garbage that is created on the first - # iteration is no longer accessible. - apply_gradients(model, optimizer, - compute_gradients(model, images, labels)) - gc.collect() - # There should be no garbage requiring collection. - self.assertEqual(0, len(gc.garbage)) - gc.set_debug(previous_gc_debug_flags) - gc.enable() - - -class MockIterator(object): - - def __init__(self, tensors): - self._tensors = [tf.identity(x) for x in tensors] - - def next(self): - return self._tensors - - -class ResNet50Benchmarks(tf.test.Benchmark): - - def _train_batch_sizes(self): - """Choose batch sizes based on GPU capability.""" - for device in device_lib.list_local_devices(): - if tf.DeviceSpec.from_string(device.name).device_type == 'GPU': - # Avoid OOM errors with larger batch sizes, which seem to cause errors - # later on even if caught. - # - # TODO(allenl): Base this on device memory; memory limit information - # during the test seems to exclude the amount TensorFlow has allocated, - # which isn't useful. - if 'K20' in device.physical_device_desc: - return (16,) - if 'P100' in device.physical_device_desc: - return (16, 32, 64) - - if tf.DeviceSpec.from_string(device.name).device_type == 'TPU': - return (32,) - return (16, 32) - - def _report(self, label, start, num_iters, device, batch_size, data_format, - num_replicas=1): - avg_time = (time.time() - start) / num_iters - dev = tf.DeviceSpec.from_string(device).device_type.lower() - replica_str = '' if num_replicas == 1 else 'replicas_%d_' % num_replicas - name = '%s_%s_batch_%d_%s%s' % (label, dev, batch_size, - replica_str, data_format) - extras = {'examples_per_sec': (num_replicas * batch_size) / avg_time} - self.report_benchmark( - iters=num_iters, wall_time=avg_time, name=name, extras=extras) - - def _force_device_sync(self): - # If this function is called in the context of a non-CPU device - # (e.g., inside a 'with tf.device("/gpu:0")' block) - # then this will force a copy from CPU->NON_CPU_DEVICE->CPU, - # which forces a sync. This is a roundabout way, yes. - tf.constant(1.).cpu() - - def _benchmark_eager_apply(self, label, device_and_format, defun=False, - execution_mode=None): - with tfe.execution_mode(execution_mode): - device, data_format = device_and_format - model = resnet50.ResNet50(data_format) - if defun: - model.call = tfe.function(model.call) - batch_size = 64 - num_burn = 5 - num_iters = 30 - with tf.device(device): - images, _ = random_batch(batch_size, data_format) - for _ in xrange(num_burn): - model(images, training=False).cpu() - if execution_mode: - tfe.async_wait() - gc.collect() - start = time.time() - for _ in xrange(num_iters): - model(images, training=False).cpu() - if execution_mode: - tfe.async_wait() - self._report(label, start, num_iters, device, batch_size, data_format) - - def benchmark_eager_apply_sync(self): - self._benchmark_eager_apply('eager_apply', device_and_data_format(), - defun=False) - - def benchmark_eager_apply_async(self): - self._benchmark_eager_apply( - 'eager_apply_async', device_and_data_format(), defun=False, - execution_mode=tfe.ASYNC) - - def benchmark_eager_apply_with_defun(self): - self._benchmark_eager_apply('eager_apply_with_defun', - device_and_data_format(), defun=True) - - def _benchmark_eager_train(self, - label, - make_iterator, - device_and_format, - defun=False, - execution_mode=None): - with tfe.execution_mode(execution_mode): - device, data_format = device_and_format - for batch_size in self._train_batch_sizes(): - (images, labels) = random_batch(batch_size, data_format) - model = resnet50.ResNet50(data_format) - optimizer = tf.train.GradientDescentOptimizer(0.1) - apply_grads = apply_gradients - if defun: - model.call = tfe.function(model.call) - apply_grads = tfe.function(apply_gradients) - - num_burn = 3 - num_iters = 10 - with tf.device(device): - iterator = make_iterator((images, labels)) - for _ in xrange(num_burn): - (images, labels) = iterator.next() - apply_grads(model, optimizer, - compute_gradients(model, images, labels)) - if execution_mode: - tfe.async_wait() - self._force_device_sync() - gc.collect() - - start = time.time() - for _ in xrange(num_iters): - (images, labels) = iterator.next() - apply_grads(model, optimizer, - compute_gradients(model, images, labels)) - if execution_mode: - tfe.async_wait() - self._force_device_sync() - self._report(label, start, num_iters, device, batch_size, data_format) - - def benchmark_eager_train_sync(self): - self._benchmark_eager_train('eager_train', MockIterator, - device_and_data_format(), defun=False) - - def benchmark_eager_train_async(self): - self._benchmark_eager_train( - 'eager_train_async', - MockIterator, - device_and_data_format(), - defun=False, - execution_mode=tfe.ASYNC) - - def benchmark_eager_train_with_defun(self): - self._benchmark_eager_train( - 'eager_train_with_defun', MockIterator, - device_and_data_format(), defun=True) - - def benchmark_eager_train_datasets(self): - - def make_iterator(tensors): - with tf.device('/device:CPU:0'): - ds = tf.data.Dataset.from_tensors(tensors).repeat() - return tfe.Iterator(ds) - - self._benchmark_eager_train( - 'eager_train_dataset', make_iterator, - device_and_data_format(), defun=False) - - def benchmark_eager_train_datasets_with_defun(self): - - def make_iterator(tensors): - with tf.device('/device:CPU:0'): - ds = tf.data.Dataset.from_tensors(tensors).repeat() - return tfe.Iterator(ds) - - self._benchmark_eager_train( - 'eager_train_dataset_with_defun', make_iterator, - device_and_data_format(), defun=True) - - -if __name__ == '__main__': - tfe.enable_eager_execution() - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/revnet/BUILD b/tensorflow/contrib/eager/python/examples/revnet/BUILD deleted file mode 100644 index 7f0e3567651..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/BUILD +++ /dev/null @@ -1,201 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "cuda_py_test") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -# Model -py_library( - name = "ops", - srcs = ["ops.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - ], -) - -py_library( - name = "config", - srcs = ["config.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - ], -) - -py_library( - name = "blocks", - srcs = ["blocks.py"], - srcs_version = "PY2AND3", - deps = [ - ":ops", - "//tensorflow:tensorflow_py", - ], -) - -py_library( - name = "revnet", - srcs = ["revnet.py"], - srcs_version = "PY2AND3", - deps = [ - ":blocks", - "//tensorflow:tensorflow_py", - ], -) - -py_library( - name = "resnet_preprocessing", - srcs = ["resnet_preprocessing.py"], - srcs_version = "PY2AND3", - tags = ["local"], - deps = [ - "//tensorflow:tensorflow_py", - ], -) - -py_library( - name = "imagenet_input", - srcs = ["imagenet_input.py"], - srcs_version = "PY2AND3", - tags = ["local"], - deps = [ - ":resnet_preprocessing", - "//tensorflow:tensorflow_py", - ], -) - -# Tests -cuda_py_test( - name = "ops_test", - size = "medium", - srcs = ["ops_test.py"], - additional_deps = [ - ":ops", - "//tensorflow:tensorflow_py", - ], - shard_count = 4, - tags = [ - "oss_serial", - ], -) - -cuda_py_test( - name = "blocks_test", - size = "medium", - srcs = ["blocks_test.py"], - additional_deps = [":blocks_test_main_lib"], - shard_count = 4, - tags = [ - "no_oss", # TODO(b/132387200): Segfaulting - "optonly", - ], -) - -py_library( - name = "blocks_test_main_lib", - testonly = True, - srcs = ["blocks_test.py"], - tags = [ - "optonly", - ], - deps = [ - ":blocks", - "//tensorflow:tensorflow_py", - ], -) - -cuda_py_test( - name = "revnet_test", - size = "medium", - srcs = ["revnet_test.py"], - additional_deps = [ - ":blocks_test", - ":config", - ":revnet", - "//tensorflow:tensorflow_py", - ], - shard_count = 4, - tags = [ - "no_pip", # depends on blocks_test, which is not available in pip package - "optonly", - "oss_serial", - ], -) - -# Training -py_library( - name = "cifar_input", - srcs = ["cifar_input.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - ], -) - -py_binary( - name = "cifar_tfrecords", - srcs = ["cifar_tfrecords.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - ], -) - -py_binary( - name = "main", - srcs = ["main.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [":main_lib"], -) - -py_library( - name = "main_lib", - srcs = ["main.py"], - srcs_version = "PY2AND3", - deps = [ - ":cifar_input", - ":config", - ":revnet", - "//tensorflow:tensorflow_py", - ], -) - -py_binary( - name = "main_estimator", - srcs = ["main_estimator.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":cifar_input", - ":main_lib", - ":revnet", - "//tensorflow:tensorflow_py", - ], -) - -py_library( - name = "main_estimator_lib", - srcs = ["main_estimator.py"], - srcs_version = "PY2AND3", - deps = [ - ":cifar_input", - ":main_lib", - ":revnet", - "//tensorflow:tensorflow_py", - ], -) - -py_library( - name = "main_estimator_tpu_lib", - srcs = ["main_estimator_tpu.py"], - srcs_version = "PY2AND3", - deps = [ - ":cifar_input", - ":main_lib", - ":revnet", - "//tensorflow:tensorflow_py", - ], -) diff --git a/tensorflow/contrib/eager/python/examples/revnet/README.md b/tensorflow/contrib/eager/python/examples/revnet/README.md deleted file mode 100644 index 822d86e9c7a..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# RevNet with TensorFlow eager execution - -This folder contains a TensorFlow eager implementation of the [Reversible Residual Network](https://arxiv.org/pdf/1707.04585.pdf) adapted from the released implementation by the authors. The presented implementation can be ran with both eager and graph execution. The code is considerably simplified with `tf.GradientTape`. Moreover, we reduce the a redundant forward pass in the implementation by the authors. This saves us from using `tf.stop_gradient` and makes the model run faster. - -## Content - -- `revnet.py`: The RevNet model. -- `blocks.py`: The relevant reversible blocks. -- `ops.py`: Auxiliary downsampling operation. -- `cifar_tfrecords.py`: Script to generate the TFRecords for both CIFAR-10 and CIFAR-100. -- `cifar_input.py`: Script to read from TFRecords and generate dataset objects with the `tf.data` API. -- `config.py`: Configuration file for network architectures and training hyperparameters. -- `main.py`: Main training and evaluation script. -- `main_estimator.py`: Script to train RevNet models on CIFAR-10 and CIFAR-100 with the `tf.estimator` API. -- `main_estimator_tpu.py`: Script to train RevNet models on ImageNet with TPU estimators on Cloud TPUs. -- `resnet_preprocessing.py`, `imagenet_input.py`: Boilerplate to read ImageNet data from TFRecords. - -## Train on CIFAR-10/CIFAR-100 -- Make sure you have installed TensorFlow 1.10+ or the latest `tf-nightly` -or `tf-nightly-gpu` pip package in order to access the eager execution feature. - -- First run - -```bash -python cifar_tfrecords.py --data_dir ${PWD}/cifar -``` -to download the cifar dataset and convert them -to TFRecords. This produces TFRecord files for both CIFAR-10 and CIFAR-100. - -- To train a model, run - -```bash -python main.py --data_dir ${PWD}/cifar -``` - -- Optional arguments for `main.py` include - - `train_dir`: Directory to store eventfiles and checkpoints. - - `restore`: Restore the latest checkpoint. - - `validate`: Use validation set for training monitoring. - - `dataset`: Use either `cifar-10` or `cifar-100`. - - `config`: RevNet configuration. - - `use_defun`: Use `tfe.defun` to boost performance. - -- To train a model with estimators in graph execution, run - -```bash -python main_estimator.py --data_dir ${PWD}/cifar -``` -To ensure our code works properly when using the Keras model in an estimator, -`tf-nightly` or `tf-nightly-gpu` is highly recommended as of August 2018. - -- Optional arguments for `main.py` include - - `model_dir`: Directory to store eventfiles and checkpoints. - - `dataset`: Use either `cifar-10` or `cifar-100`. - - `config`: RevNet configuration. - - `export`: Export the model for serving if True. - -## Speed up with `tfe.defun` -To ensure that `tf.contrib.eager.defun` in our code works properly with all -part of the model during training, the latest `tf-nightly` or `tf-nightly-gpu` -is highly recommended as of August 2018. - -Even though the speed difference between pure eager execution and graph execution is noticeable, -the difference between fully "defunned" model training and graph -training is negligible. - -## Train on ImageNet with Cloud TPUs -The standard way to train models on Cloud TPUs is via TPU estimators and graph -execution. Models built with the `tf.keras` API are fully compatible with TPU estimators. -To ensure our code works properly in this setting, -`tf-nightly` or `tf-nightly-gpu` is highly recommended as of August 2018. - -### Setup a Google Cloud project - -Follow the instructions at the [Quickstart Guide](https://cloud.google.com/tpu/docs/quickstart) -to get a GCE VM with access to Cloud TPU. - -To run this model, you will need: - -* A GCE VM instance with an associated Cloud TPU resource -* A GCS bucket to store your training checkpoints -* (Optional): The ImageNet training and validation data preprocessed into - TFRecord format, and stored in GCS. - -### Format the data - -The data is expected to be formatted in TFRecord format, as generated by [this -script](https://github.com/tensorflow/tpu/blob/master/tools/datasets/imagenet_to_gcs.py). - -If you do not have ImageNet dataset prepared, you can use a randomly generated -fake dataset to test the model. It is located at -`gs://cloud-tpu-test-datasets/fake_imagenet`. - -### Start training - -Train the model by executing the following command (substituting the appropriate -values): - -```bash -python main_estimator_tpu.py \ - --tpu=$TPU_NAME \ - --data_dir=$DATA_DIR \ - --model_dir=$MODEL_DIR -``` - -## Performance -- RevNet-38 achieves >92% and >71% accuracy on CIFAR-10 and CIFAR-100 respectively. -- RevNet-56 achieves <26% top-1 error rate on ImageNet. - -## Reference -The Reversible Residual Network: Backpropagation Without Storing Activations. -Aidan N. Gomez, Mengye Ren, Raquel Urtasun, Roger B. Grosse. Neural Information Processing Systems (NIPS), 2017. diff --git a/tensorflow/contrib/eager/python/examples/revnet/blocks.py b/tensorflow/contrib/eager/python/examples/revnet/blocks.py deleted file mode 100644 index 221b0766225..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/blocks.py +++ /dev/null @@ -1,506 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Reversible residual network compatible with eager execution. - -Building blocks with manual backward gradient computation. - -Reference [The Reversible Residual Network: Backpropagation -Without Storing Activations](https://arxiv.org/pdf/1707.04585.pdf) -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import operator - -import tensorflow as tf -from tensorflow.contrib.eager.python.examples.revnet import ops - - -class RevBlock(tf.keras.Model): - """Single reversible block containing several `_Residual` blocks. - - Each `_Residual` block in turn contains two _ResidualInner blocks, - corresponding to the `F`/`G` functions in the paper. - """ - - def __init__(self, - n_res, - filters, - strides, - input_shape, - batch_norm_first=False, - data_format="channels_first", - bottleneck=False, - fused=True, - dtype=tf.float32): - """Initialization. - - Args: - n_res: number of residual blocks - filters: list/tuple of integers for output filter sizes of each residual - strides: length 2 list/tuple of integers for height and width strides - input_shape: length 3 list/tuple of integers - batch_norm_first: whether to apply activation and batch norm before conv - data_format: tensor data format, "NCHW"/"NHWC" - bottleneck: use bottleneck residual if True - fused: use fused batch normalization if True - dtype: float16, float32, or float64 - """ - super(RevBlock, self).__init__(dtype=dtype) - self.blocks = tf.contrib.checkpoint.List() - for i in range(n_res): - curr_batch_norm_first = batch_norm_first and i == 0 - curr_strides = strides if i == 0 else (1, 1) - block = _Residual( - filters, - curr_strides, - input_shape, - batch_norm_first=curr_batch_norm_first, - data_format=data_format, - bottleneck=bottleneck, - fused=fused, - dtype=dtype) - self.blocks.append(block) - - if data_format == "channels_first": - input_shape = (filters, input_shape[1] // curr_strides[0], - input_shape[2] // curr_strides[1]) - else: - input_shape = (input_shape[0] // curr_strides[0], - input_shape[1] // curr_strides[1], filters) - - def call(self, h, training=True): - """Apply reversible block to inputs.""" - - for block in self.blocks: - h = block(h, training=training) - return h - - def backward_grads(self, x, y, dy, training=True): - """Apply reversible block backward to outputs.""" - - grads_all = [] - for i in reversed(range(len(self.blocks))): - block = self.blocks[i] - if i == 0: - # First block usually contains downsampling that can't be reversed - dy, grads = block.backward_grads_with_downsample( - x, y, dy, training=True) - else: - y, dy, grads = block.backward_grads(y, dy, training=training) - grads_all = grads + grads_all - - return dy, grads_all - - -class _Residual(tf.keras.Model): - """Single residual block contained in a _RevBlock. Each `_Residual` object has - two _ResidualInner objects, corresponding to the `F` and `G` functions in the - paper. - """ - - def __init__(self, - filters, - strides, - input_shape, - batch_norm_first=True, - data_format="channels_first", - bottleneck=False, - fused=True, - dtype=tf.float32): - """Initialization. - - Args: - filters: output filter size - strides: length 2 list/tuple of integers for height and width strides - input_shape: length 3 list/tuple of integers - batch_norm_first: whether to apply activation and batch norm before conv - data_format: tensor data format, "NCHW"/"NHWC", - bottleneck: use bottleneck residual if True - fused: use fused batch normalization if True - dtype: float16, float32, or float64 - """ - super(_Residual, self).__init__(dtype=dtype) - - self.filters = filters - self.strides = strides - self.axis = 1 if data_format == "channels_first" else 3 - if data_format == "channels_first": - f_input_shape = (input_shape[0] // 2,) + input_shape[1:] - g_input_shape = (filters // 2, input_shape[1] // strides[0], - input_shape[2] // strides[1]) - else: - f_input_shape = input_shape[:2] + (input_shape[2] // 2,) - g_input_shape = (input_shape[0] // strides[0], - input_shape[1] // strides[1], filters // 2) - - factory = _BottleneckResidualInner if bottleneck else _ResidualInner - self.f = factory( - filters=filters // 2, - strides=strides, - input_shape=f_input_shape, - batch_norm_first=batch_norm_first, - data_format=data_format, - fused=fused, - dtype=dtype) - self.g = factory( - filters=filters // 2, - strides=(1, 1), - input_shape=g_input_shape, - batch_norm_first=batch_norm_first, - data_format=data_format, - fused=fused, - dtype=dtype) - - def call(self, x, training=True): - """Apply residual block to inputs.""" - x1, x2 = x - f_x2 = self.f(x2, training=training) - x1_down = ops.downsample( - x1, self.filters // 2, self.strides, axis=self.axis) - x2_down = ops.downsample( - x2, self.filters // 2, self.strides, axis=self.axis) - y1 = f_x2 + x1_down - g_y1 = self.g(y1, training=training) - y2 = g_y1 + x2_down - - return y1, y2 - - def backward_grads(self, y, dy, training=True): - """Manually compute backward gradients given input and output grads.""" - dy1, dy2 = dy - y1, y2 = y - - with tf.GradientTape() as gtape: - gtape.watch(y1) - gy1 = self.g(y1, training=training) - grads_combined = gtape.gradient( - gy1, [y1] + self.g.trainable_variables, output_gradients=dy2) - dg = grads_combined[1:] - dx1 = dy1 + grads_combined[0] - # This doesn't affect eager execution, but improves memory efficiency with - # graphs - with tf.control_dependencies(dg + [dx1]): - x2 = y2 - gy1 - - with tf.GradientTape() as ftape: - ftape.watch(x2) - fx2 = self.f(x2, training=training) - grads_combined = ftape.gradient( - fx2, [x2] + self.f.trainable_variables, output_gradients=dx1) - df = grads_combined[1:] - dx2 = dy2 + grads_combined[0] - # Same behavior as above - with tf.control_dependencies(df + [dx2]): - x1 = y1 - fx2 - - x = x1, x2 - dx = dx1, dx2 - grads = df + dg - - return x, dx, grads - - def backward_grads_with_downsample(self, x, y, dy, training=True): - """Manually compute backward gradients given input and output grads.""" - # Splitting this from `backward_grads` for better readability - x1, x2 = x - y1, _ = y - dy1, dy2 = dy - - with tf.GradientTape() as gtape: - gtape.watch(y1) - gy1 = self.g(y1, training=training) - grads_combined = gtape.gradient( - gy1, [y1] + self.g.trainable_variables, output_gradients=dy2) - dg = grads_combined[1:] - dz1 = dy1 + grads_combined[0] - - # dx1 need one more step to backprop through downsample - with tf.GradientTape() as x1tape: - x1tape.watch(x1) - z1 = ops.downsample(x1, self.filters // 2, self.strides, axis=self.axis) - dx1 = x1tape.gradient(z1, x1, output_gradients=dz1) - - with tf.GradientTape() as ftape: - ftape.watch(x2) - fx2 = self.f(x2, training=training) - grads_combined = ftape.gradient( - fx2, [x2] + self.f.trainable_variables, output_gradients=dz1) - dx2, df = grads_combined[0], grads_combined[1:] - - # dx2 need one more step to backprop through downsample - with tf.GradientTape() as x2tape: - x2tape.watch(x2) - z2 = ops.downsample(x2, self.filters // 2, self.strides, axis=self.axis) - dx2 += x2tape.gradient(z2, x2, output_gradients=dy2) - - dx = dx1, dx2 - grads = df + dg - - return dx, grads - - -# Ideally, the following should be wrapped in `tf.keras.Sequential`, however -# there are subtle issues with its placeholder insertion policy and batch norm -class _BottleneckResidualInner(tf.keras.Model): - """Single bottleneck residual inner function contained in _Resdual. - - Corresponds to the `F`/`G` functions in the paper. - Suitable for training on ImageNet dataset. - """ - - def __init__(self, - filters, - strides, - input_shape, - batch_norm_first=True, - data_format="channels_first", - fused=True, - dtype=tf.float32): - """Initialization. - - Args: - filters: output filter size - strides: length 2 list/tuple of integers for height and width strides - input_shape: length 3 list/tuple of integers - batch_norm_first: whether to apply activation and batch norm before conv - data_format: tensor data format, "NCHW"/"NHWC" - fused: use fused batch normalization if True - dtype: float16, float32, or float64 - """ - super(_BottleneckResidualInner, self).__init__(dtype=dtype) - axis = 1 if data_format == "channels_first" else 3 - if batch_norm_first: - self.batch_norm_0 = tf.keras.layers.BatchNormalization( - axis=axis, input_shape=input_shape, fused=fused, dtype=dtype) - self.conv2d_1 = tf.keras.layers.Conv2D( - filters=filters // 4, - kernel_size=1, - strides=strides, - input_shape=input_shape, - data_format=data_format, - use_bias=False, - padding="SAME", - dtype=dtype) - - self.batch_norm_1 = tf.keras.layers.BatchNormalization( - axis=axis, fused=fused, dtype=dtype) - self.conv2d_2 = tf.keras.layers.Conv2D( - filters=filters // 4, - kernel_size=3, - strides=(1, 1), - data_format=data_format, - use_bias=False, - padding="SAME", - dtype=dtype) - - self.batch_norm_2 = tf.keras.layers.BatchNormalization( - axis=axis, fused=fused, dtype=dtype) - self.conv2d_3 = tf.keras.layers.Conv2D( - filters=filters, - kernel_size=1, - strides=(1, 1), - data_format=data_format, - use_bias=False, - padding="SAME", - dtype=dtype) - - self.batch_norm_first = batch_norm_first - - def call(self, x, training=True): - net = x - if self.batch_norm_first: - net = self.batch_norm_0(net, training=training) - net = tf.nn.relu(net) - net = self.conv2d_1(net) - - net = self.batch_norm_1(net, training=training) - net = tf.nn.relu(net) - net = self.conv2d_2(net) - - net = self.batch_norm_2(net, training=training) - net = tf.nn.relu(net) - net = self.conv2d_3(net) - - return net - - -class _ResidualInner(tf.keras.Model): - """Single residual inner function contained in _ResdualBlock. - - Corresponds to the `F`/`G` functions in the paper. - """ - - def __init__(self, - filters, - strides, - input_shape, - batch_norm_first=True, - data_format="channels_first", - fused=True, - dtype=tf.float32): - """Initialization. - - Args: - filters: output filter size - strides: length 2 list/tuple of integers for height and width strides - input_shape: length 3 list/tuple of integers - batch_norm_first: whether to apply activation and batch norm before conv - data_format: tensor data format, "NCHW"/"NHWC" - fused: use fused batch normalization if True - dtype: float16, float32, or float64 - """ - super(_ResidualInner, self).__init__(dtype=dtype) - axis = 1 if data_format == "channels_first" else 3 - if batch_norm_first: - self.batch_norm_0 = tf.keras.layers.BatchNormalization( - axis=axis, input_shape=input_shape, fused=fused, dtype=dtype) - self.conv2d_1 = tf.keras.layers.Conv2D( - filters=filters, - kernel_size=3, - strides=strides, - input_shape=input_shape, - data_format=data_format, - use_bias=False, - padding="SAME", - dtype=dtype) - - self.batch_norm_1 = tf.keras.layers.BatchNormalization( - axis=axis, fused=fused, dtype=dtype) - self.conv2d_2 = tf.keras.layers.Conv2D( - filters=filters, - kernel_size=3, - strides=(1, 1), - data_format=data_format, - use_bias=False, - padding="SAME", - dtype=dtype) - - self.batch_norm_first = batch_norm_first - - def call(self, x, training=True): - net = x - if self.batch_norm_first: - net = self.batch_norm_0(net, training=training) - net = tf.nn.relu(net) - net = self.conv2d_1(net) - - net = self.batch_norm_1(net, training=training) - net = tf.nn.relu(net) - net = self.conv2d_2(net) - - return net - - -class InitBlock(tf.keras.Model): - """Initial block of RevNet.""" - - def __init__(self, config): - """Initialization. - - Args: - config: tf.contrib.training.HParams object; specifies hyperparameters - """ - super(InitBlock, self).__init__(config.dtype) - self.config = config - self.axis = 1 if self.config.data_format == "channels_first" else 3 - self.conv2d = tf.keras.layers.Conv2D( - filters=self.config.init_filters, - kernel_size=self.config.init_kernel, - strides=(self.config.init_stride, self.config.init_stride), - data_format=self.config.data_format, - use_bias=False, - padding="SAME", - input_shape=self.config.input_shape, - dtype=self.config.dtype) - self.batch_norm = tf.keras.layers.BatchNormalization( - axis=self.axis, fused=self.config.fused, dtype=self.config.dtype) - self.activation = tf.keras.layers.Activation("relu", - dtype=self.config.dtype) - - if self.config.init_max_pool: - self.max_pool = tf.keras.layers.MaxPooling2D( - pool_size=(3, 3), - strides=(2, 2), - padding="SAME", - data_format=self.config.data_format, - dtype=self.config.dtype) - - def call(self, x, training=True): - net = x - net = self.conv2d(net) - net = self.batch_norm(net, training=training) - net = self.activation(net) - - if self.config.init_max_pool: - net = self.max_pool(net) - - return tf.split(net, num_or_size_splits=2, axis=self.axis) - - -class FinalBlock(tf.keras.Model): - """Final block of RevNet.""" - - def __init__(self, config): - """Initialization. - - Args: - config: tf.contrib.training.HParams object; specifies hyperparameters - - Raises: - ValueError: Unsupported data format - """ - super(FinalBlock, self).__init__(dtype=config.dtype) - self.config = config - self.axis = 1 if self.config.data_format == "channels_first" else 3 - - f = self.config.filters[-1] # Number of filters - r = functools.reduce(operator.mul, self.config.strides, 1) # Reduce ratio - r *= self.config.init_stride - if self.config.init_max_pool: - r *= 2 - - if self.config.data_format == "channels_first": - w, h = self.config.input_shape[1], self.config.input_shape[2] - input_shape = (f, w // r, h // r) - elif self.config.data_format == "channels_last": - w, h = self.config.input_shape[0], self.config.input_shape[1] - input_shape = (w // r, h // r, f) - else: - raise ValueError("Data format should be either `channels_first`" - " or `channels_last`") - self.batch_norm = tf.keras.layers.BatchNormalization( - axis=self.axis, - input_shape=input_shape, - fused=self.config.fused, - dtype=self.config.dtype) - self.activation = tf.keras.layers.Activation("relu", - dtype=self.config.dtype) - self.global_avg_pool = tf.keras.layers.GlobalAveragePooling2D( - data_format=self.config.data_format, dtype=self.config.dtype) - self.dense = tf.keras.layers.Dense( - self.config.n_classes, dtype=self.config.dtype) - - def call(self, x, training=True): - net = tf.concat(x, axis=self.axis) - net = self.batch_norm(net, training=training) - net = self.activation(net) - net = self.global_avg_pool(net) - net = self.dense(net) - - return net diff --git a/tensorflow/contrib/eager/python/examples/revnet/blocks_test.py b/tensorflow/contrib/eager/python/examples/revnet/blocks_test.py deleted file mode 100644 index 9ff6b605b91..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/blocks_test.py +++ /dev/null @@ -1,288 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for basic building blocks used in eager mode RevNet.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf -from tensorflow.contrib.eager.python.examples.revnet import blocks - - -def compute_degree(g1, g2, eps=1e-7): - """Compute the degree between two vectors using their usual inner product.""" - - def _dot(u, v): - return tf.reduce_sum(u * v) - - g1_norm = tf.sqrt(_dot(g1, g1)) - g2_norm = tf.sqrt(_dot(g2, g2)) - if g1_norm.numpy() == 0 and g2_norm.numpy() == 0: - cosine = 1. - eps - else: - g1_norm = 1. if g1_norm.numpy() == 0 else g1_norm - g2_norm = 1. if g2_norm.numpy() == 0 else g2_norm - cosine = _dot(g1, g2) / g1_norm / g2_norm - # Restrict to arccos range - cosine = tf.minimum(tf.maximum(cosine, eps - 1.), 1. - eps) - degree = tf.acos(cosine) * 180. / 3.141592653589793 - - return degree - - -def _validate_block_call_channels_last(block_factory, test): - """Generic testing function for `channels_last` data format. - - Completes a set of tests varying data format, stride, and batch normalization - configured train vs test time. - Args: - block_factory: constructor of one of blocks.InitBlock, blocks.FinalBlock, - blocks._ResidualInner - test: tf.test.TestCase object - """ - with tf.device("/cpu:0"): # NHWC format - input_shape = (8, 8, 128) - data_shape = (16,) + input_shape - x = tf.random_normal(shape=data_shape) - - # Stride 1 - block = block_factory( - filters=128, - strides=(1, 1), - input_shape=input_shape, - data_format="channels_last") - y_tr, y_ev = block(x, training=True), block(x, training=False) - test.assertEqual(y_tr.shape, y_ev.shape) - test.assertEqual(y_ev.shape, (16, 8, 8, 128)) - test.assertNotAllClose(y_tr, y_ev) - - # Stride of 2 - block = block_factory( - filters=128, - strides=(2, 2), - input_shape=input_shape, - data_format="channels_last") - y_tr, y_ev = block(x, training=True), block(x, training=False) - test.assertEqual(y_tr.shape, y_ev.shape) - test.assertEqual(y_ev.shape, (16, 4, 4, 128)) - test.assertNotAllClose(y_tr, y_ev) - - -def _validate_block_call_channels_first(block_factory, test): - """Generic testing function for `channels_first` data format. - - Completes a set of tests varying data format, stride, and batch normalization - configured train vs test time. - Args: - block_factory: constructor of one of blocks.InitBlock, blocks.FinalBlock, - blocks._ResidualInner - test: tf.test.TestCase object - """ - if not tf.test.is_gpu_available(): - test.skipTest("GPU not available") - - with tf.device("/gpu:0"): # Default NCHW format - input_shape = (128, 8, 8) - data_shape = (16,) + input_shape - x = tf.random_normal(shape=data_shape) - - # Stride of 1 - block = block_factory(filters=128, strides=(1, 1), input_shape=input_shape) - y_tr, y_ev = block(x, training=True), block(x, training=False) - test.assertEqual(y_tr.shape, y_ev.shape) - test.assertEqual(y_ev.shape, (16, 128, 8, 8)) - test.assertNotAllClose(y_tr, y_ev) - - # Stride of 2 - block = block_factory(filters=128, strides=(2, 2), input_shape=input_shape) - y_tr, y_ev = block(x, training=True), block(x, training=False) - test.assertEqual(y_tr.shape, y_ev.shape) - test.assertEqual(y_ev.shape, (16, 128, 4, 4)) - test.assertNotAllClose(y_tr, y_ev) - - -class RevBlockTest(tf.test.TestCase): - - def _check_grad_angle(self, grads, grads_true, atol=1e0): - """Check the angle between two list of vectors are all close.""" - for g1, g2 in zip(grads, grads_true): - degree = compute_degree(g1, g2) - self.assertLessEqual(degree, atol) - - def test_backward_grads_channels_first(self): - """Test `backward` function with `channels_first` data format.""" - if not tf.test.is_gpu_available(): - self.skipTest("GPU not available") - - with tf.device("/gpu:0"): # Default NCHW format - # Stride 1 - input_shape = (128, 8, 8) - data_shape = (16,) + input_shape - x = tf.random_normal(shape=data_shape, dtype=tf.float64) - dy = tf.random_normal(shape=data_shape, dtype=tf.float64) - dy1, dy2 = tf.split(dy, num_or_size_splits=2, axis=1) - block = blocks.RevBlock( - n_res=3, - filters=128, - strides=(1, 1), - input_shape=input_shape, - fused=False, - dtype=tf.float64) - with tf.GradientTape() as tape: - tape.watch(x) - x1, x2 = tf.split(x, num_or_size_splits=2, axis=1) - y1, y2 = block((x1, x2), training=True) - y = tf.concat((y1, y2), axis=1) - # Compute grads from reconstruction - (dx1, dx2), dw = block.backward_grads( - x=(x1, x2), y=(y1, y2), dy=(dy1, dy2), training=True) - dx = tf.concat((dx1, dx2), axis=1) - vars_ = block.trainable_variables - # Compute true grads - grads = tape.gradient(y, [x] + vars_, output_gradients=dy) - dx_true, dw_true = grads[0], grads[1:] - self.assertAllClose(dx_true, dx) - self.assertAllClose(dw_true, dw) - self._check_grad_angle(dx_true, dx) - self._check_grad_angle(dw_true, dw) - - # Stride 2 - x = tf.random_normal(shape=data_shape, dtype=tf.float64) - dy = tf.random_normal(shape=(16, 128, 4, 4), dtype=tf.float64) - dy1, dy2 = tf.split(dy, num_or_size_splits=2, axis=1) - block = blocks.RevBlock( - n_res=3, - filters=128, - strides=(2, 2), - input_shape=input_shape, - fused=False, - dtype=tf.float64) - with tf.GradientTape() as tape: - tape.watch(x) - x1, x2 = tf.split(x, num_or_size_splits=2, axis=1) - y1, y2 = block((x1, x2), training=True) - y = tf.concat((y1, y2), axis=1) - # Compute grads from reconstruction - (dx1, dx2), dw = block.backward_grads( - x=(x1, x2), y=(y1, y2), dy=(dy1, dy2), training=True) - dx = tf.concat((dx1, dx2), axis=1) - vars_ = block.trainable_variables - # Compute true grads - grads = tape.gradient(y, [x] + vars_, output_gradients=dy) - dx_true, dw_true = grads[0], grads[1:] - self.assertAllClose(dx_true, dx) - self.assertAllClose(dw_true, dw) - self._check_grad_angle(dx_true, dx) - self._check_grad_angle(dw_true, dw) - - def test_backward_grads_with_nativepy(self): - if not tf.test.is_gpu_available(): - self.skipTest("GPU not available") - - input_shape = (128, 8, 8) - data_shape = (16,) + input_shape - x = tf.random_normal(shape=data_shape, dtype=tf.float64) - dy = tf.random_normal(shape=data_shape, dtype=tf.float64) - dy1, dy2 = tf.split(dy, num_or_size_splits=2, axis=1) - block = blocks.RevBlock( - n_res=3, - filters=128, - strides=(1, 1), - input_shape=input_shape, - fused=False, - dtype=tf.float64) - with tf.GradientTape() as tape: - tape.watch(x) - x1, x2 = tf.split(x, num_or_size_splits=2, axis=1) - y1, y2 = block((x1, x2), training=True) - y = tf.concat((y1, y2), axis=1) - - # Compute true grads - dx_true = tape.gradient(y, x, output_gradients=dy) - - # Compute grads from reconstruction - (dx1, dx2), _ = block.backward_grads( - x=(x1, x2), y=(y1, y2), dy=(dy1, dy2), training=True) - dx = tf.concat((dx1, dx2), axis=1) - - thres = 1e-5 - diff_abs = tf.reshape(abs(dx - dx_true), [-1]) - assert all(diff_abs < thres) - - -class _ResidualTest(tf.test.TestCase): - - def test_backward_grads_channels_first(self): - """Test `backward_grads` function with `channels_first` data format.""" - if not tf.test.is_gpu_available(): - self.skipTest("GPU not available") - - with tf.device("/gpu:0"): # Default NCHW format - input_shape = (128, 8, 8) - data_shape = (16,) + input_shape - # Use double precision for testing - x_true = tf.random_normal(shape=data_shape, dtype=tf.float64) - dy = tf.random_normal(shape=data_shape, dtype=tf.float64) - dy1, dy2 = tf.split(dy, num_or_size_splits=2, axis=1) - residual = blocks._Residual( - filters=128, - strides=(1, 1), - input_shape=input_shape, - fused=False, - dtype=tf.float64) - - with tf.GradientTape() as tape: - tape.watch(x_true) - x1_true, x2_true = tf.split(x_true, num_or_size_splits=2, axis=1) - y1, y2 = residual((x1_true, x2_true), training=True) - y = tf.concat((y1, y2), axis=1) - - # Gradients computed due to reversibility - (x1, x2), (dx1, dx2), dw = residual.backward_grads( - y=(y1, y2), dy=(dy1, dy2), training=True) - x = tf.concat((x1, x2), axis=1) - dx = tf.concat((dx1, dx2), axis=1) - # True gradients computed by the tape - grads = tape.gradient( - y, [x_true] + residual.trainable_variables, output_gradients=dy) - dx_true, dw_true = grads[0], grads[1:] - - self.assertAllClose(x_true, x) - self.assertAllClose(dx_true, dx) - self.assertAllClose(dw_true, dw) - - -class _ResidualInnerTest(tf.test.TestCase): - - def test_call(self): - """Test `call` function.""" - - _validate_block_call_channels_first(blocks._ResidualInner, self) - _validate_block_call_channels_last(blocks._ResidualInner, self) - - -class _BottleneckResidualInner(tf.test.TestCase): - - def test_call(self): - """Test `call` function.""" - - _validate_block_call_channels_first(blocks._BottleneckResidualInner, self) - _validate_block_call_channels_last(blocks._BottleneckResidualInner, self) - - -if __name__ == "__main__": - tf.enable_eager_execution() - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py b/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py deleted file mode 100644 index e9672f13e15..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Script for reading and loading CIFAR-10.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -import tensorflow as tf - -# Global constants describing the CIFAR data set. -IMAGE_HEIGHT = 32 -IMAGE_WIDTH = 32 -NUM_CHANNEL = 3 - - -def get_ds_from_tfrecords(data_dir, - split, - data_aug=True, - batch_size=100, - epochs=None, - shuffle=True, - data_format="channels_first", - num_parallel_calls=12, - prefetch=0, - div255=True, - dtype=tf.float32): - """Returns a tf.train.Dataset object from reading tfrecords. - - Args: - data_dir: Directory of tfrecords - split: "train", "validation", or "test" - data_aug: Apply data augmentation if True - batch_size: Batch size of dataset object - epochs: Number of epochs to repeat the dataset; default `None` means - repeating indefinitely - shuffle: Shuffle the dataset if True - data_format: `channels_first` or `channels_last` - num_parallel_calls: Number of threads for dataset preprocess - prefetch: Buffer size for prefetch - div255: Divide the images by 255 if True - dtype: Data type of images - Returns: - A tf.train.Dataset object - - Raises: - ValueError: Unknown split - """ - - if split not in ["train", "validation", "test", "train_all"]: - raise ValueError("Unknown split {}".format(split)) - - def _parser(serialized_example): - """Parses a single tf.Example into image and label tensors.""" - features = tf.parse_single_example( - serialized_example, - features={ - "image": tf.FixedLenFeature([], tf.string), - "label": tf.FixedLenFeature([], tf.int64), - }) - image = tf.decode_raw(features["image"], tf.uint8) - # Initially reshaping to [H, W, C] does not work - image = tf.reshape(image, [NUM_CHANNEL, IMAGE_HEIGHT, IMAGE_WIDTH]) - # This is needed for `tf.image.resize_image_with_crop_or_pad` - image = tf.transpose(image, [1, 2, 0]) - - image = tf.cast(image, dtype) - label = tf.cast(features["label"], tf.int32) - - if data_aug: - image = tf.image.resize_image_with_crop_or_pad(image, IMAGE_HEIGHT + 4, - IMAGE_WIDTH + 4) - image = tf.random_crop(image, [IMAGE_HEIGHT, IMAGE_WIDTH, NUM_CHANNEL]) - image = tf.image.random_flip_left_right(image) - - if data_format == "channels_first": - image = tf.transpose(image, [2, 0, 1]) - - if div255: - image /= 255. - - return image, label - - filename = os.path.join(data_dir, split + ".tfrecords") - dataset = tf.data.TFRecordDataset(filename) - dataset = dataset.repeat(epochs) - dataset = dataset.map(_parser, num_parallel_calls=num_parallel_calls) - dataset = dataset.prefetch(prefetch) - - if shuffle: - # Find the right size according to the split - size = { - "train": 40000, - "validation": 10000, - "test": 10000, - "train_all": 50000 - }[split] - dataset = dataset.shuffle(size) - - dataset = dataset.batch(batch_size, drop_remainder=True) - - return dataset diff --git a/tensorflow/contrib/eager/python/examples/revnet/cifar_tfrecords.py b/tensorflow/contrib/eager/python/examples/revnet/cifar_tfrecords.py deleted file mode 100644 index 377844ad8fb..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/cifar_tfrecords.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Read CIFAR data from pickled numpy arrays and writes TFRecords. - -Generates tf.train.Example protos and writes them to TFRecord files from the -python version of the CIFAR dataset downloaded from -https://www.cs.toronto.edu/~kriz/cifar.html. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import sys -import tarfile - -from absl import flags -from six.moves import cPickle as pickle -from six.moves import urllib -import tensorflow as tf - -BASE_URL = 'https://www.cs.toronto.edu/~kriz/' -CIFAR_FILE_NAMES = ['cifar-10-python.tar.gz', 'cifar-100-python.tar.gz'] -CIFAR_DOWNLOAD_URLS = [BASE_URL + name for name in CIFAR_FILE_NAMES] -CIFAR_LOCAL_FOLDERS = ['cifar-10', 'cifar-100'] -EXTRACT_FOLDERS = ['cifar-10-batches-py', 'cifar-100-python'] - - -def download_and_extract(data_dir, file_name, url): - """Download CIFAR if not already downloaded.""" - filepath = os.path.join(data_dir, file_name) - if tf.gfile.Exists(filepath): - return filepath - if not tf.gfile.Exists(data_dir): - tf.gfile.MakeDirs(data_dir) - - urllib.request.urlretrieve(url, filepath) - tarfile.open(os.path.join(filepath), 'r:gz').extractall(data_dir) - return filepath - - -def _int64_feature(value): - return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) - - -def _bytes_feature(value): - return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) - - -def _get_file_names(folder): - """Returns the file names expected to exist in the input_dir.""" - assert folder in ['cifar-10', 'cifar-100'] - - file_names = {} - if folder == 'cifar-10': - file_names['train'] = ['data_batch_%d' % i for i in range(1, 5)] - file_names['validation'] = ['data_batch_5'] - file_names['train_all'] = ['data_batch_%d' % i for i in range(1, 6)] - file_names['test'] = ['test_batch'] - else: - file_names['train_all'] = ['train'] - file_names['test'] = ['test'] - # Split in `convert_to_tfrecord` function - file_names['train'] = ['train'] - file_names['validation'] = ['train'] - return file_names - - -def read_pickle_from_file(filename): - with tf.gfile.Open(filename, 'rb') as f: - if sys.version_info >= (3, 0): - data_dict = pickle.load(f, encoding='bytes') - else: - data_dict = pickle.load(f) - return data_dict - - -def convert_to_tfrecord(input_files, output_file, folder): - """Converts files with pickled data to TFRecords.""" - assert folder in ['cifar-10', 'cifar-100'] - - print('Generating %s' % output_file) - with tf.python_io.TFRecordWriter(output_file) as record_writer: - for input_file in input_files: - data_dict = read_pickle_from_file(input_file) - data = data_dict[b'data'] - try: - labels = data_dict[b'labels'] - except KeyError: - labels = data_dict[b'fine_labels'] - - if folder == 'cifar-100' and input_file.endswith('train.tfrecords'): - data = data[:40000] - labels = labels[:40000] - elif folder == 'cifar-100' and input_file.endswith( - 'validation.tfrecords'): - data = data[40000:] - labels = labels[40000:] - - num_entries_in_batch = len(labels) - - for i in range(num_entries_in_batch): - example = tf.train.Example( - features=tf.train.Features( - feature={ - 'image': _bytes_feature(data[i].tobytes()), - 'label': _int64_feature(labels[i]) - })) - record_writer.write(example.SerializeToString()) - - -def main(_): - for file_name, url, folder, extract_folder in zip( - CIFAR_FILE_NAMES, CIFAR_DOWNLOAD_URLS, CIFAR_LOCAL_FOLDERS, - EXTRACT_FOLDERS): - print('Download from {} and extract.'.format(url)) - data_dir = os.path.join(FLAGS.data_dir, folder) - download_and_extract(data_dir, file_name, url) - file_names = _get_file_names(folder) - input_dir = os.path.join(data_dir, extract_folder) - - for mode, files in file_names.items(): - input_files = [os.path.join(input_dir, f) for f in files] - output_file = os.path.join(data_dir, mode + '.tfrecords') - try: - os.remove(output_file) - except OSError: - pass - convert_to_tfrecord(input_files, output_file, folder) - - print('Done!') - - -if __name__ == '__main__': - FLAGS = flags.FLAGS - flags.DEFINE_string( - 'data_dir', - default=None, - help='Directory to download, extract and store TFRecords.') - - tf.app.run(main) diff --git a/tensorflow/contrib/eager/python/examples/revnet/config.py b/tensorflow/contrib/eager/python/examples/revnet/config.py deleted file mode 100644 index 29f1db0e036..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/config.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Reversible residual network compatible with eager execution. - -Configuration in format of tf.contrib.training.HParams. -Supports CIFAR-10, CIFAR-100, and ImageNet datasets. - -Reference [The Reversible Residual Network: Backpropagation -Without Storing Activations](https://arxiv.org/pdf/1707.04585.pdf) - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - - -def get_hparams_cifar_38(): - """RevNet-38 configurations for CIFAR-10/CIFAR-100.""" - - config = tf.contrib.training.HParams() - config.add_hparam("num_train_images", 50000) - config.add_hparam("num_eval_images", 10000) - config.add_hparam("init_filters", 32) - config.add_hparam("init_kernel", 3) - config.add_hparam("init_stride", 1) - config.add_hparam("n_rev_blocks", 3) - config.add_hparam("n_res", [3, 3, 3]) - config.add_hparam("filters", [32, 64, 112]) - config.add_hparam("strides", [1, 2, 2]) - config.add_hparam("batch_size", 100) - config.add_hparam("bottleneck", False) - config.add_hparam("fused", True) - config.add_hparam("init_max_pool", False) - if tf.test.is_gpu_available(): - config.add_hparam("input_shape", (3, 32, 32)) - config.add_hparam("data_format", "channels_first") - else: - config.add_hparam("input_shape", (32, 32, 3)) - config.add_hparam("data_format", "channels_last") - - # Training details - config.add_hparam("weight_decay", 2e-4) - config.add_hparam("momentum", .9) - config.add_hparam("lr_decay_steps", [40000, 60000]) - config.add_hparam("lr_list", [1e-1, 1e-2, 1e-3]) - config.add_hparam("max_train_iter", 80000) - config.add_hparam("seed", 1234) - config.add_hparam("shuffle", True) - config.add_hparam("log_every", 500) - config.add_hparam("save_every", 500) - config.add_hparam("dtype", tf.float32) - config.add_hparam("eval_batch_size", 1000) - config.add_hparam("div255", True) - # This is imprecise, when training with validation set, - # we only have 40k images in training data - config.add_hparam("iters_per_epoch", - config.num_train_images // config.batch_size) - config.add_hparam("epochs", config.max_train_iter // config.iters_per_epoch) - - # Customized TPU hyperparameters due to differing batch size caused by - # TPU architecture specifics - # Suggested batch sizes to reduce overhead from excessive tensor padding - # https://cloud.google.com/tpu/docs/troubleshooting - config.add_hparam("tpu_batch_size", 1024) - config.add_hparam("tpu_eval_batch_size", 1024) - config.add_hparam("tpu_iters_per_epoch", - config.num_train_images // config.tpu_batch_size) - config.add_hparam("tpu_epochs", - config.max_train_iter // config.tpu_iters_per_epoch) - config.add_hparam("tpu_eval_steps", - config.num_eval_images // config.tpu_eval_batch_size) - return config - - -def get_hparams_cifar_110(): - config = get_hparams_cifar_38() - config.filters = [32, 64, 128] - config.n_res = [9, 9, 9] - - return config - - -def get_hparams_cifar_164(): - config = get_hparams_cifar_38() - config.filters = [32, 64, 128] - config.n_res = [9, 9, 9] - config.use_bottleneck = True - # Due to bottleneck residual blocks - filters = [f * 4 for f in config.filters] - config.filters = filters - - return config - - -def get_hparams_imagenet_56(): - """RevNet-56 configurations for ImageNet.""" - - config = tf.contrib.training.HParams() - config.add_hparam("n_classes", 1000) - config.add_hparam("dataset", "ImageNet") - config.add_hparam("num_train_images", 1281167) - config.add_hparam("num_eval_images", 50000) - config.add_hparam("init_filters", 128) - config.add_hparam("init_kernel", 7) - config.add_hparam("init_stride", 2) - config.add_hparam("n_rev_blocks", 4) - config.add_hparam("n_res", [2, 2, 2, 2]) - config.add_hparam("filters", [128, 256, 512, 832]) - config.add_hparam("strides", [1, 2, 2, 2]) - config.add_hparam("batch_size", 256) - config.add_hparam("bottleneck", True) - config.add_hparam("fused", True) - config.add_hparam("init_max_pool", True) - if tf.test.is_gpu_available(): - config.add_hparam("input_shape", (3, 224, 224)) - config.add_hparam("data_format", "channels_first") - else: - config.add_hparam("input_shape", (224, 224, 3)) - config.add_hparam("data_format", "channels_last") - # Due to bottleneck residual blocks - filters = [f * 4 for f in config.filters] - config.filters = filters - - # Training details - config.add_hparam("weight_decay", 1e-4) - config.add_hparam("momentum", .9) - config.add_hparam("lr_decay_steps", [160000, 320000, 480000]) - config.add_hparam("lr_list", [1e-1, 1e-2, 1e-3, 1e-4]) - config.add_hparam("max_train_iter", 600000) - config.add_hparam("seed", 1234) - config.add_hparam("shuffle", True) - config.add_hparam("log_every", 500) - config.add_hparam("save_every", 500) - config.add_hparam("dtype", tf.float32) - config.add_hparam("eval_batch_size", 256) - config.add_hparam("div255", True) - config.add_hparam("iters_per_epoch", - config.num_train_images // config.batch_size) - config.add_hparam("epochs", config.max_train_iter // config.iters_per_epoch) - - # Customized TPU hyperparameters due to differing batch size caused by - # TPU architecture specifics - # Suggested batch sizes to reduce overhead from excessive tensor padding - # https://cloud.google.com/tpu/docs/troubleshooting - config.add_hparam("tpu_batch_size", 1024) - config.add_hparam("tpu_eval_batch_size", 1024) - config.add_hparam("tpu_iters_per_epoch", - config.num_train_images // config.tpu_batch_size) - config.add_hparam("tpu_epochs", - config.max_train_iter // config.tpu_iters_per_epoch) - config.add_hparam("tpu_eval_steps", - config.num_eval_images // config.tpu_eval_batch_size) - return config - - -def get_hparams_imagenet_104(): - config = get_hparams_imagenet_56() - config.n_res = [2, 2, 11, 2] - - return config diff --git a/tensorflow/contrib/eager/python/examples/revnet/imagenet_input.py b/tensorflow/contrib/eager/python/examples/revnet/imagenet_input.py deleted file mode 100644 index d85188de030..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/imagenet_input.py +++ /dev/null @@ -1,231 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Efficient ImageNet input pipeline using tf.data.Dataset.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import os - -import tensorflow as tf - -from tensorflow.contrib.eager.python.examples.revnet import resnet_preprocessing - - -def image_serving_input_fn(): - """Serving input fn for raw images.""" - - def _preprocess_image(image_bytes): - """Preprocess a single raw image.""" - image = resnet_preprocessing.preprocess_image( - image_bytes=image_bytes, is_training=False) - return image - - image_bytes_list = tf.placeholder( - shape=[None], - dtype=tf.string, - ) - images = tf.map_fn( - _preprocess_image, image_bytes_list, back_prop=False, dtype=tf.float32) - return tf.estimator.export.ServingInputReceiver( - images, {'image_bytes': image_bytes_list}) - - -class ImageNetInput(object): - """Generates ImageNet input_fn for training or evaluation. - - The training data is assumed to be in TFRecord format with keys as specified - in the dataset_parser below, sharded across 1024 files, named sequentially: - train-00000-of-01024 - train-00001-of-01024 - ... - train-01023-of-01024 - - The validation data is in the same format but sharded in 128 files. - - The format of the data required is created by the script at: - https://github.com/tensorflow/tpu/blob/master/tools/datasets/imagenet_to_gcs.py - - Args: - is_training: `bool` for whether the input is for training - data_dir: `str` for the directory of the training and validation data; - if 'null' (the literal string 'null', not None), then construct a null - pipeline, consisting of empty images. - use_bfloat16: If True, use bfloat16 precision; else use float32. - transpose_input: 'bool' for whether to use the double transpose trick - num_cores: `int` for the number of TPU cores - """ - - def __init__(self, is_training, - use_bfloat16, - data_dir, - num_cores=8, - num_parallel_calls=64, - image_size=224, - transpose_input=False, - cache=False): - self.image_preprocessing_fn = resnet_preprocessing.preprocess_image - self.is_training = is_training - self.use_bfloat16 = use_bfloat16 - self.data_dir = data_dir - self.num_cores = num_cores - self.num_parallel_calls = num_parallel_calls - if self.data_dir == 'null' or self.data_dir == '': - self.data_dir = None - self.transpose_input = transpose_input - self.image_size = image_size - self.cache = cache - - def set_shapes(self, batch_size, images, labels): - """Statically set the batch_size dimension.""" - if self.transpose_input: - images.set_shape(images.get_shape().merge_with( - tf.TensorShape([None, None, None, batch_size]))) - labels.set_shape(labels.get_shape().merge_with( - tf.TensorShape([batch_size]))) - else: - images.set_shape(images.get_shape().merge_with( - tf.TensorShape([batch_size, None, None, None]))) - labels.set_shape(labels.get_shape().merge_with( - tf.TensorShape([batch_size]))) - - return images, labels - - def dataset_parser(self, value): - """Parse an ImageNet record from a serialized string Tensor.""" - keys_to_features = { - 'image/encoded': tf.FixedLenFeature((), tf.string, ''), - 'image/format': tf.FixedLenFeature((), tf.string, 'jpeg'), - 'image/class/label': tf.FixedLenFeature([], tf.int64, -1), - 'image/class/text': tf.FixedLenFeature([], tf.string, ''), - 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), - 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), - 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), - 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), - 'image/object/class/label': tf.VarLenFeature(dtype=tf.int64), - } - - parsed = tf.parse_single_example(value, keys_to_features) - image_bytes = tf.reshape(parsed['image/encoded'], shape=[]) - - image = self.image_preprocessing_fn( - image_bytes=image_bytes, - is_training=self.is_training, - image_size=self.image_size, - use_bfloat16=self.use_bfloat16) - - # Subtract one so that labels are in [0, 1000). - label = tf.cast( - tf.reshape(parsed['image/class/label'], shape=[]), dtype=tf.int32) - 1 - - return image, label - - def input_fn(self, params): - """Input function which provides a single batch for train or eval. - - Args: - params: `dict` of parameters passed from the `TPUEstimator`. - `params['batch_size']` is always provided and should be used as the - effective batch size. - - Returns: - A `tf.data.Dataset` object. - """ - if self.data_dir is None: - tf.logging.info('Using fake input.') - return self.input_fn_null(params) - - # Retrieves the batch size for the current shard. The # of shards is - # computed according to the input pipeline deployment. See - # tf.contrib.tpu.RunConfig for details. - batch_size = params['batch_size'] - - # Shuffle the filenames to ensure better randomization. - file_pattern = os.path.join( - self.data_dir, 'train-*' if self.is_training else 'validation-*') - dataset = tf.data.Dataset.list_files(file_pattern, shuffle=self.is_training) - - if self.is_training and not self.cache: - dataset = dataset.repeat() - - def fetch_dataset(filename): - buffer_size = 8 * 1024 * 1024 # 8 MiB per file - dataset = tf.data.TFRecordDataset(filename, buffer_size=buffer_size) - return dataset - - # Read the data from disk in parallel - dataset = dataset.apply( - tf.data.experimental.parallel_interleave( - fetch_dataset, cycle_length=self.num_parallel_calls, sloppy=True)) - if self.cache: - dataset = dataset.cache().apply( - tf.data.experimental.shuffle_and_repeat(1024 * 16)) - else: - dataset = dataset.shuffle(1024) - - # Use the fused map-and-batch operation. - # - # For XLA, we must used fixed shapes. Because we repeat the source training - # dataset indefinitely, we can use `drop_remainder=True` to get fixed-size - # batches without dropping any training examples. - # - # When evaluating, `drop_remainder=True` prevents accidentally evaluating - # the same image twice by dropping the final batch if it is less than a full - # batch size. As long as this validation is done with consistent batch size, - # exactly the same images will be used. - dataset = dataset.apply( - tf.data.experimental.map_and_batch( - self.dataset_parser, - batch_size=batch_size, - num_parallel_batches=self.num_cores, - drop_remainder=True)) - - # Transpose for performance on TPU - if self.transpose_input: - dataset = dataset.map( - lambda images, labels: (tf.transpose(images, [1, 2, 3, 0]), labels), - num_parallel_calls=self.num_cores) - - # Assign static batch size dimension - dataset = dataset.map(functools.partial(self.set_shapes, batch_size)) - - # Prefetch overlaps in-feed with training - dataset = dataset.prefetch(tf.contrib.data.AUTOTUNE) - return dataset - - def input_fn_null(self, params): - """Input function which provides null (black) images.""" - batch_size = params['batch_size'] - dataset = tf.data.Dataset.range(1).repeat().map(self._get_null_input) - dataset = dataset.prefetch(batch_size) - - dataset = dataset.batch(batch_size, drop_remainder=True) - if self.transpose_input: - dataset = dataset.map( - lambda images, labels: (tf.transpose(images, [1, 2, 3, 0]), labels), - num_parallel_calls=8) - - dataset = dataset.map(functools.partial(self.set_shapes, batch_size)) - - dataset = dataset.prefetch(32) # Prefetch overlaps in-feed with training - tf.logging.info('Input dataset: %s', str(dataset)) - return dataset - - def _get_null_input(self, _): - null_image = tf.zeros([224, 224, 3], tf.bfloat16 - if self.use_bfloat16 else tf.float32) - return (null_image, tf.constant(0, tf.int32)) diff --git a/tensorflow/contrib/eager/python/examples/revnet/main.py b/tensorflow/contrib/eager/python/examples/revnet/main.py deleted file mode 100644 index 9585f3565f8..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/main.py +++ /dev/null @@ -1,262 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Eager execution workflow with RevNet train on CIFAR-10.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import sys - -from absl import flags -import tensorflow as tf -from tensorflow.contrib.eager.python.examples.revnet import cifar_input -from tensorflow.contrib.eager.python.examples.revnet import config as config_ -from tensorflow.contrib.eager.python.examples.revnet import revnet -tfe = tf.contrib.eager - - -def apply_gradients(optimizer, grads, vars_, global_step=None): - """Functional style apply_grads for `tfe.defun`.""" - optimizer.apply_gradients(zip(grads, vars_), global_step=global_step) - - -def main(_): - """Eager execution workflow with RevNet trained on CIFAR-10.""" - tf.enable_eager_execution() - - config = get_config(config_name=FLAGS.config, dataset=FLAGS.dataset) - ds_train, ds_train_one_shot, ds_validation, ds_test = get_datasets( - data_dir=FLAGS.data_dir, config=config) - model = revnet.RevNet(config=config) - global_step = tf.train.get_or_create_global_step() # Ensure correct summary - global_step.assign(1) - learning_rate = tf.train.piecewise_constant( - global_step, config.lr_decay_steps, config.lr_list) - optimizer = tf.train.MomentumOptimizer( - learning_rate, momentum=config.momentum) - checkpointer = tf.train.Checkpoint( - optimizer=optimizer, model=model, optimizer_step=global_step) - - if FLAGS.use_defun: - model.call = tfe.defun(model.call) - model.compute_gradients = tfe.defun(model.compute_gradients) - model.get_moving_stats = tfe.defun(model.get_moving_stats) - model.restore_moving_stats = tfe.defun(model.restore_moving_stats) - global apply_gradients # pylint:disable=global-variable-undefined - apply_gradients = tfe.defun(apply_gradients) - - if FLAGS.train_dir: - summary_writer = tf.contrib.summary.create_file_writer(FLAGS.train_dir) - if FLAGS.restore: - latest_path = tf.train.latest_checkpoint(FLAGS.train_dir) - checkpointer.restore(latest_path) - print("Restored latest checkpoint at path:\"{}\" " - "with global_step: {}".format(latest_path, global_step.numpy())) - sys.stdout.flush() - - for x, y in ds_train: - train_one_iter(model, x, y, optimizer, global_step=global_step) - - if global_step.numpy() % config.log_every == 0: - acc_test, loss_test = evaluate(model, ds_test) - - if FLAGS.validate: - acc_train, loss_train = evaluate(model, ds_train_one_shot) - acc_validation, loss_validation = evaluate(model, ds_validation) - print("Iter {}, " - "training set accuracy {:.4f}, loss {:.4f}; " - "validation set accuracy {:.4f}, loss {:.4f}; " - "test accuracy {:.4f}, loss {:.4f}".format( - global_step.numpy(), acc_train, loss_train, acc_validation, - loss_validation, acc_test, loss_test)) - else: - print("Iter {}, test accuracy {:.4f}, loss {:.4f}".format( - global_step.numpy(), acc_test, loss_test)) - sys.stdout.flush() - - if FLAGS.train_dir: - with summary_writer.as_default(): - with tf.contrib.summary.always_record_summaries(): - tf.contrib.summary.scalar("Test accuracy", acc_test) - tf.contrib.summary.scalar("Test loss", loss_test) - if FLAGS.validate: - tf.contrib.summary.scalar("Training accuracy", acc_train) - tf.contrib.summary.scalar("Training loss", loss_train) - tf.contrib.summary.scalar("Validation accuracy", acc_validation) - tf.contrib.summary.scalar("Validation loss", loss_validation) - - if global_step.numpy() % config.save_every == 0 and FLAGS.train_dir: - saved_path = checkpointer.save( - file_prefix=os.path.join(FLAGS.train_dir, "ckpt")) - print("Saved checkpoint at path: \"{}\" " - "with global_step: {}".format(saved_path, global_step.numpy())) - sys.stdout.flush() - - -def get_config(config_name="revnet-38", dataset="cifar-10"): - """Return configuration.""" - print("Config: {}".format(config_name)) - sys.stdout.flush() - config = { - "revnet-38": config_.get_hparams_cifar_38(), - "revnet-110": config_.get_hparams_cifar_110(), - "revnet-164": config_.get_hparams_cifar_164(), - }[config_name] - - if dataset == "cifar-10": - config.add_hparam("n_classes", 10) - config.add_hparam("dataset", "cifar-10") - else: - config.add_hparam("n_classes", 100) - config.add_hparam("dataset", "cifar-100") - - return config - - -def get_datasets(data_dir, config): - """Return dataset.""" - if data_dir is None: - raise ValueError("No supplied data directory") - if not os.path.exists(data_dir): - raise ValueError("Data directory {} does not exist".format(data_dir)) - if config.dataset not in ["cifar-10", "cifar-100"]: - raise ValueError("Unknown dataset {}".format(config.dataset)) - - print("Training on {} dataset.".format(config.dataset)) - sys.stdout.flush() - data_dir = os.path.join(data_dir, config.dataset) - if FLAGS.validate: - # 40k Training set - ds_train = cifar_input.get_ds_from_tfrecords( - data_dir=data_dir, - split="train", - data_aug=True, - batch_size=config.batch_size, - epochs=config.epochs, - shuffle=config.shuffle, - data_format=config.data_format, - dtype=config.dtype, - prefetch=config.batch_size) - # 10k Training set - ds_validation = cifar_input.get_ds_from_tfrecords( - data_dir=data_dir, - split="validation", - data_aug=False, - batch_size=config.eval_batch_size, - epochs=1, - shuffle=False, - data_format=config.data_format, - dtype=config.dtype, - prefetch=config.eval_batch_size) - else: - # 50k Training set - ds_train = cifar_input.get_ds_from_tfrecords( - data_dir=data_dir, - split="train_all", - data_aug=True, - batch_size=config.batch_size, - epochs=config.epochs, - shuffle=config.shuffle, - data_format=config.data_format, - dtype=config.dtype, - prefetch=config.batch_size) - ds_validation = None - - # Always compute loss and accuracy on whole test set - ds_train_one_shot = cifar_input.get_ds_from_tfrecords( - data_dir=data_dir, - split="train_all", - data_aug=False, - batch_size=config.eval_batch_size, - epochs=1, - shuffle=False, - data_format=config.data_format, - dtype=config.dtype, - prefetch=config.eval_batch_size) - - ds_test = cifar_input.get_ds_from_tfrecords( - data_dir=data_dir, - split="test", - data_aug=False, - batch_size=config.eval_batch_size, - epochs=1, - shuffle=False, - data_format=config.data_format, - dtype=config.dtype, - prefetch=config.eval_batch_size) - - return ds_train, ds_train_one_shot, ds_validation, ds_test - - -def train_one_iter(model, inputs, labels, optimizer, global_step=None): - """Train for one iteration.""" - logits, saved_hiddens = model(inputs, training=True) - values = model.get_moving_stats() - grads, loss = model.compute_gradients(saved_hiddens, labels) - # Restore moving averages when executing eagerly to avoid updating twice - model.restore_moving_stats(values) - apply_gradients( - optimizer, grads, model.trainable_variables, global_step=global_step) - - return logits, loss - - -def evaluate(model, dataset): - """Compute accuracy with the given dataset iterator.""" - mean_loss = tfe.metrics.Mean() - accuracy = tfe.metrics.Accuracy() - for x, y in dataset: - logits, _ = model(x, training=False) - loss = model.compute_loss(logits=logits, labels=y) - accuracy( - labels=tf.cast(y, tf.int64), - predictions=tf.argmax(logits, axis=1, output_type=tf.int64)) - mean_loss(loss) - - return accuracy.result().numpy(), mean_loss.result().numpy() - - -if __name__ == "__main__": - flags.DEFINE_string( - "data_dir", default=None, help="Directory to load tfrecords") - flags.DEFINE_string( - "train_dir", - default=None, - help="[Optional] Directory to store the training information") - flags.DEFINE_boolean( - "restore", - default=False, - help="[Optional] Restore the latest checkpoint from `train_dir` if True") - flags.DEFINE_boolean( - "validate", - default=False, - help="[Optional] Use the validation set or not for hyperparameter search") - flags.DEFINE_string( - "dataset", - default="cifar-10", - help="[Optional] The dataset used; either `cifar-10` or `cifar-100`") - flags.DEFINE_string( - "config", - default="revnet-38", - help="[Optional] Architecture of network. " - "Other options include `revnet-110` and `revnet-164`") - flags.DEFINE_boolean( - "use_defun", - default=False, - help="[Optional] Use `tfe.defun` to boost performance.") - FLAGS = flags.FLAGS - tf.app.run(main) diff --git a/tensorflow/contrib/eager/python/examples/revnet/main_estimator.py b/tensorflow/contrib/eager/python/examples/revnet/main_estimator.py deleted file mode 100644 index 125adbb9de6..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/main_estimator.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Estimator workflow with RevNet train on CIFAR-10.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from absl import flags -import tensorflow as tf -from tensorflow.contrib.eager.python.examples.revnet import cifar_input -from tensorflow.contrib.eager.python.examples.revnet import main as main_ -from tensorflow.contrib.eager.python.examples.revnet import revnet - - -def model_fn(features, labels, mode, params): - """Function specifying the model that is required by the `tf.estimator` API. - - Args: - features: Input images - labels: Labels of images - mode: One of `ModeKeys.TRAIN`, `ModeKeys.EVAL` or 'ModeKeys.PREDICT' - params: A dictionary of extra parameter that might be passed - - Returns: - An instance of `tf.estimator.EstimatorSpec` - """ - - inputs = features - if isinstance(inputs, dict): - inputs = features["image"] - - config = params["config"] - model = revnet.RevNet(config=config) - - if mode == tf.estimator.ModeKeys.TRAIN: - global_step = tf.train.get_or_create_global_step() - learning_rate = tf.train.piecewise_constant( - global_step, config.lr_decay_steps, config.lr_list) - optimizer = tf.train.MomentumOptimizer( - learning_rate, momentum=config.momentum) - logits, saved_hidden = model(inputs, training=True) - grads, loss = model.compute_gradients(saved_hidden, labels, training=True) - with tf.control_dependencies(model.get_updates_for(inputs)): - train_op = optimizer.apply_gradients( - zip(grads, model.trainable_variables), global_step=global_step) - - return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) - else: - logits, _ = model(inputs, training=False) - predictions = tf.argmax(logits, axis=1) - probabilities = tf.nn.softmax(logits) - - if mode == tf.estimator.ModeKeys.EVAL: - loss = model.compute_loss(labels=labels, logits=logits) - return tf.estimator.EstimatorSpec( - mode=mode, - loss=loss, - eval_metric_ops={ - "accuracy": - tf.metrics.accuracy(labels=labels, predictions=predictions) - }) - - else: # mode == tf.estimator.ModeKeys.PREDICT - result = { - "classes": predictions, - "probabilities": probabilities, - } - - return tf.estimator.EstimatorSpec( - mode=mode, - predictions=predictions, - export_outputs={ - "classify": tf.estimator.export.PredictOutput(result) - }) - - -def get_input_fn(config, data_dir, split): - """Get the input function that is required by the `tf.estimator` API. - - Args: - config: Customized hyperparameters - data_dir: Directory where the data is stored - split: One of `train`, `validation`, `train_all`, and `test` - - Returns: - Input function required by the `tf.estimator` API - """ - - data_dir = os.path.join(data_dir, config.dataset) - # Fix split-dependent hyperparameters - if split == "train_all" or split == "train": - data_aug = True - batch_size = config.batch_size - epochs = config.epochs - shuffle = True - prefetch = config.batch_size - else: - data_aug = False - batch_size = config.eval_batch_size - epochs = 1 - shuffle = False - prefetch = config.eval_batch_size - - def input_fn(): - """Input function required by the `tf.estimator.Estimator` API.""" - return cifar_input.get_ds_from_tfrecords( - data_dir=data_dir, - split=split, - data_aug=data_aug, - batch_size=batch_size, - epochs=epochs, - shuffle=shuffle, - prefetch=prefetch, - data_format=config.data_format) - - return input_fn - - -def main(_): - tf.logging.set_verbosity(tf.logging.INFO) - - # RevNet specific configuration - config = main_.get_config(config_name=FLAGS.config, dataset=FLAGS.dataset) - - # Estimator specific configuration - run_config = tf.estimator.RunConfig( - model_dir=FLAGS.model_dir, # Directory for storing checkpoints - tf_random_seed=config.seed, - save_summary_steps=config.log_every, - save_checkpoints_steps=config.log_every, - session_config=None, # Using default - keep_checkpoint_max=100, - keep_checkpoint_every_n_hours=10000, # Using default - log_step_count_steps=config.log_every, - train_distribute=None # Default not use distribution strategy - ) - - # Construct estimator - revnet_estimator = tf.estimator.Estimator( - model_fn=model_fn, - model_dir=FLAGS.model_dir, - config=run_config, - params={"config": config}) - - # Construct input functions - train_input_fn = get_input_fn( - config=config, data_dir=FLAGS.data_dir, split="train_all") - eval_input_fn = get_input_fn( - config=config, data_dir=FLAGS.data_dir, split="test") - - # Train and evaluate estimator - revnet_estimator.train(input_fn=train_input_fn) - revnet_estimator.evaluate(input_fn=eval_input_fn) - - if FLAGS.export: - input_shape = (None,) + config.input_shape - inputs = tf.placeholder(tf.float32, shape=input_shape) - input_fn = tf.estimator.export.build_raw_serving_input_receiver_fn({ - "image": inputs - }) - revnet_estimator.export_saved_model(FLAGS.model_dir, input_fn) - - -if __name__ == "__main__": - flags.DEFINE_string( - "data_dir", default=None, help="Directory to load tfrecords") - flags.DEFINE_string( - "model_dir", - default=None, - help="[Optional] Directory to store the training information") - flags.DEFINE_string( - "dataset", - default="cifar-10", - help="[Optional] The dataset used; either `cifar-10` or `cifar-100`") - flags.DEFINE_boolean( - "export", - default=False, - help="[Optional] Export the model for serving if True") - flags.DEFINE_string( - "config", - default="revnet-38", - help="[Optional] Architecture of network. " - "Other options include `revnet-110` and `revnet-164`") - FLAGS = flags.FLAGS - tf.app.run() diff --git a/tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py b/tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py deleted file mode 100644 index b0676916a8d..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py +++ /dev/null @@ -1,394 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Cloud TPU Estimator workflow with RevNet train on ImageNet.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time - -from absl import flags -import tensorflow as tf -from tensorflow.contrib import summary -from tensorflow.contrib.eager.python.examples.revnet import config as config_ -from tensorflow.contrib.eager.python.examples.revnet import imagenet_input -from tensorflow.contrib.eager.python.examples.revnet import revnet -from tensorflow.contrib.training.python.training import evaluation -from tensorflow.python.estimator import estimator - -MEAN_RGB = [0.485, 0.456, 0.406] -STDDEV_RGB = [0.229, 0.224, 0.225] - - -def _host_call_fn(gs, loss, lr): - """Training host call. - - Creates scalar summaries for training metrics. - - This function is executed on the CPU and should not directly reference - any Tensors in the rest of the `model_fn`. To pass Tensors from the - model to the `metric_fn`, provide as part of the `host_call`. See - https://www.tensorflow.org/api_docs/python/tf/contrib/tpu/TPUEstimatorSpec - for more information. - - Arguments should match the list of `Tensor` objects passed as the second - element in the tuple passed to `host_call`. - - Args: - gs: `Tensor with shape `[batch]` for the global_step - loss: `Tensor` with shape `[batch]` for the training loss. - lr: `Tensor` with shape `[batch]` for the learning_rate. - - Returns: - List of summary ops to run on the CPU host. - """ - # Host call fns are executed FLAGS.iterations_per_loop times after one - # TPU loop is finished, setting max_queue value to the same as number of - # iterations will make the summary writer only flush the data to storage - # once per loop. - gs = gs[0] - with summary.create_file_writer( - FLAGS.model_dir, max_queue=FLAGS.iterations_per_loop).as_default(): - with summary.always_record_summaries(): - summary.scalar("loss", loss[0], step=gs) - summary.scalar("learning_rate", lr[0], step=gs) - return summary.all_summary_ops() - - -def _metric_fn(labels, logits): - """Evaluation metric function. Evaluates accuracy. - - This function is executed on the CPU and should not directly reference - any Tensors in the rest of the `model_fn`. To pass Tensors from the model - to the `metric_fn`, provide as part of the `eval_metrics`. See - https://www.tensorflow.org/api_docs/python/tf/contrib/tpu/TPUEstimatorSpec - for more information. - - Arguments should match the list of `Tensor` objects passed as the second - element in the tuple passed to `eval_metrics`. - - Args: - labels: `Tensor` with shape `[batch]`. - logits: `Tensor` with shape `[batch, num_classes]`. - - Returns: - A dict of the metrics to return from evaluation. - """ - predictions = tf.argmax(logits, axis=1) - top_1_accuracy = tf.metrics.accuracy(labels, predictions) - in_top_5 = tf.cast(tf.nn.in_top_k(logits, labels, 5), tf.float32) - top_5_accuracy = tf.metrics.mean(in_top_5) - - return { - "top_1_accuracy": top_1_accuracy, - "top_5_accuracy": top_5_accuracy, - } - - -def model_fn(features, labels, mode, params): - """Model function required by the `tf.contrib.tpu.TPUEstimator` API. - - Args: - features: Input images - labels: Labels of images - mode: One of `ModeKeys.TRAIN`, `ModeKeys.EVAL` or 'ModeKeys.PREDICT' - params: A dictionary of extra parameter that might be passed - - Returns: - An instance of `tf.contrib.tpu.TPUEstimatorSpec` - """ - revnet_config = params["revnet_config"] - model = revnet.RevNet(config=revnet_config) - - inputs = features - if isinstance(inputs, dict): - inputs = features["image"] - - if revnet_config.data_format == "channels_first": - assert not FLAGS.transpose_input # channels_first only for GPU - inputs = tf.transpose(inputs, [0, 3, 1, 2]) - - if FLAGS.transpose_input and mode != tf.estimator.ModeKeys.PREDICT: - inputs = tf.transpose(inputs, [3, 0, 1, 2]) # HWCN to NHWC - - # Normalize the image to zero mean and unit variance. - inputs -= tf.constant(MEAN_RGB, shape=[1, 1, 3], dtype=inputs.dtype) - inputs /= tf.constant(STDDEV_RGB, shape=[1, 1, 3], dtype=inputs.dtype) - - if mode == tf.estimator.ModeKeys.TRAIN: - global_step = tf.train.get_or_create_global_step() - learning_rate = tf.train.piecewise_constant( - global_step, revnet_config.lr_decay_steps, revnet_config.lr_list) - optimizer = tf.train.MomentumOptimizer(learning_rate, - revnet_config.momentum) - if FLAGS.use_tpu: - optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) - - logits, saved_hidden = model(inputs, training=True) - grads, loss = model.compute_gradients(saved_hidden, labels, training=True) - with tf.control_dependencies(model.get_updates_for(inputs)): - train_op = optimizer.apply_gradients( - zip(grads, model.trainable_variables), global_step=global_step) - if not FLAGS.skip_host_call: - # To log the loss, current learning rate, and epoch for Tensorboard, the - # summary op needs to be run on the host CPU via host_call. host_call - # expects [batch_size, ...] Tensors, thus reshape to introduce a batch - # dimension. These Tensors are implicitly concatenated to - # [params['batch_size']]. - gs_t = tf.reshape(global_step, [1]) - loss_t = tf.reshape(loss, [1]) - lr_t = tf.reshape(learning_rate, [1]) - host_call = (_host_call_fn, [gs_t, loss_t, lr_t]) - - return tf.contrib.tpu.TPUEstimatorSpec( - mode=mode, loss=loss, train_op=train_op, host_call=host_call) - - elif mode == tf.estimator.ModeKeys.EVAL: - logits, _ = model(inputs, training=False) - loss = model.compute_loss(labels=labels, logits=logits) - - return tf.contrib.tpu.TPUEstimatorSpec( - mode=mode, loss=loss, eval_metrics=(_metric_fn, [labels, logits])) - - else: # Predict or export - logits, _ = model(inputs, training=False) - predictions = { - "classes": tf.argmax(logits, axis=1), - "probabilities": tf.nn.softmax(logits), - } - - return tf.contrib.tpu.TPUEstimatorSpec( - mode=mode, - predictions=predictions, - export_outputs={ - "classify": tf.estimator.export.PredictOutput(predictions) - }) - - -def main(_): - tf.logging.set_verbosity(tf.logging.INFO) - - # RevNet specific configuration - revnet_config = { - "revnet-56": config_.get_hparams_imagenet_56(), - "revnet-104": config_.get_hparams_imagenet_104() - }[FLAGS.revnet_config] - - if FLAGS.use_tpu: - revnet_config.data_format = "channels_last" - - tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( - FLAGS.tpu, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) - - # Estimator specific configuration - config = tf.contrib.tpu.RunConfig( - cluster=tpu_cluster_resolver, - model_dir=FLAGS.model_dir, - session_config=tf.ConfigProto( - allow_soft_placement=True, log_device_placement=True), - tpu_config=tf.contrib.tpu.TPUConfig( - iterations_per_loop=FLAGS.iterations_per_loop, - num_shards=FLAGS.num_shards, - per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig. - PER_HOST_V2), - ) - - # Input pipelines are slightly different (with regards to shuffling and - # preprocessing) between training and evaluation. - imagenet_train, imagenet_eval = [ - imagenet_input.ImageNetInput( - is_training=is_training, - data_dir=FLAGS.data_dir, - transpose_input=FLAGS.transpose_input, - use_bfloat16=False) for is_training in [True, False] - ] - - revnet_classifier = tf.contrib.tpu.TPUEstimator( - model_fn=model_fn, - use_tpu=FLAGS.use_tpu, - train_batch_size=revnet_config.tpu_batch_size, - eval_batch_size=revnet_config.tpu_eval_batch_size, - config=config, - export_to_tpu=False, - params={"revnet_config": revnet_config}) - - steps_per_epoch = revnet_config.tpu_iters_per_epoch - eval_steps = revnet_config.tpu_eval_steps - - # pylint: disable=protected-access - if FLAGS.mode == "eval": - # Run evaluation when there's a new checkpoint - for ckpt in evaluation.checkpoints_iterator( - FLAGS.model_dir, timeout=FLAGS.eval_timeout): - tf.logging.info("Starting to evaluate.") - try: - start_timestamp = time.time() # This time will include compilation time - eval_results = revnet_classifier.evaluate( - input_fn=imagenet_eval.input_fn, - steps=eval_steps, - checkpoint_path=ckpt) - elapsed_time = int(time.time() - start_timestamp) - tf.logging.info("Eval results: %s. Elapsed seconds: %d" % - (eval_results, elapsed_time)) - - # Terminate eval job when final checkpoint is reached - current_step = int(os.path.basename(ckpt).split("-")[1]) - if current_step >= revnet_config.max_train_iter: - tf.logging.info( - "Evaluation finished after training step %d" % current_step) - break - - except tf.errors.NotFoundError: - # Since the coordinator is on a different job than the TPU worker, - # sometimes the TPU worker does not finish initializing until long after - # the CPU job tells it to start evaluating. In this case, the checkpoint - # file could have been deleted already. - tf.logging.info( - "Checkpoint %s no longer exists, skipping checkpoint" % ckpt) - - else: # FLAGS.mode == 'train' or FLAGS.mode == 'train_and_eval' - current_step = estimator._load_global_step_from_checkpoint_dir( - FLAGS.model_dir) - - tf.logging.info( - "Training for %d steps (%.2f epochs in total). Current" - " step %d." % (revnet_config.max_train_iter, - revnet_config.max_train_iter / steps_per_epoch, - current_step)) - - start_timestamp = time.time() # This time will include compilation time - - if FLAGS.mode == "train": - revnet_classifier.train( - input_fn=imagenet_train.input_fn, - max_steps=revnet_config.max_train_iter) - - else: - assert FLAGS.mode == "train_and_eval" - while current_step < revnet_config.max_train_iter: - # Train for up to steps_per_eval number of steps. - # At the end of training, a checkpoint will be written to --model_dir. - next_checkpoint = min(current_step + FLAGS.steps_per_eval, - revnet_config.max_train_iter) - revnet_classifier.train( - input_fn=imagenet_train.input_fn, max_steps=next_checkpoint) - current_step = next_checkpoint - - tf.logging.info("Finished training up to step %d. Elapsed seconds %d." % - (next_checkpoint, int(time.time() - start_timestamp))) - - # Evaluate the model on the most recent model in --model_dir. - # Since evaluation happens in batches of --eval_batch_size, some images - # may be excluded modulo the batch size. As long as the batch size is - # consistent, the evaluated images are also consistent. - tf.logging.info("Starting to evaluate.") - eval_results = revnet_classifier.evaluate( - input_fn=imagenet_eval.input_fn, steps=eval_steps) - tf.logging.info("Eval results: %s" % eval_results) - - elapsed_time = int(time.time() - start_timestamp) - tf.logging.info("Finished training up to step %d. Elapsed seconds %d." % - (revnet_config.max_train_iter, elapsed_time)) - - if FLAGS.export_dir is not None: - # The guide to serve an exported TensorFlow model is at: - # https://www.tensorflow.org/serving/serving_basic - tf.logging.info("Starting to export model.") - revnet_classifier.export_saved_model( - export_dir_base=FLAGS.export_dir, - serving_input_receiver_fn=imagenet_input.image_serving_input_fn) - - -if __name__ == "__main__": - # Cloud TPU Cluster Resolver flags - flags.DEFINE_string( - "tpu", - default=None, - help="The Cloud TPU to use for training. This should be either the name " - "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 " - "url.") - flags.DEFINE_string( - "tpu_zone", - default=None, - help="[Optional] GCE zone where the Cloud TPU is located in. If not " - "specified, we will attempt to automatically detect the GCE project from " - "metadata.") - flags.DEFINE_string( - "gcp_project", - default=None, - help="[Optional] Project name for the Cloud TPU-enabled project. If not " - "specified, we will attempt to automatically detect the GCE project from " - "metadata.") - - # Model specific parameters - flags.DEFINE_string( - "data_dir", default=None, help="Directory to load tfrecords") - flags.DEFINE_string( - "model_dir", - default=None, - help="[Optional] Directory to store the model information") - flags.DEFINE_string( - "revnet_config", - default="revnet-56", - help="[Optional] Architecture of network. " - "Other options include `revnet-104`") - flags.DEFINE_boolean( - "use_tpu", default=True, help="[Optional] Whether to use TPU") - flags.DEFINE_integer( - "num_shards", default=8, help="Number of shards (TPU chips).") - flags.DEFINE_integer( - "iterations_per_loop", - default=100, - help=( - "Number of steps to run on TPU before feeding metrics to the CPU." - " If the number of iterations in the loop would exceed the number of" - " train steps, the loop will exit before reaching" - " --iterations_per_loop. The larger this value is, the higher the" - " utilization on the TPU.")) - flags.DEFINE_integer( - "eval_timeout", - default=None, - help="Maximum seconds between checkpoints before evaluation terminates.") - flags.DEFINE_integer( - "steps_per_eval", - default=5000, - help=( - "Controls how often evaluation is performed. Since evaluation is" - " fairly expensive, it is advised to evaluate as infrequently as" - " possible (i.e. up to --train_steps, which evaluates the model only" - " after finishing the entire training regime).")) - flags.DEFINE_bool( - "transpose_input", - default=True, - help="Use TPU double transpose optimization") - flags.DEFINE_string( - "export_dir", - default=None, - help=("The directory where the exported SavedModel will be stored.")) - flags.DEFINE_bool( - "skip_host_call", - default=False, - help=("Skip the host_call which is executed every training step. This is" - " generally used for generating training summaries (train loss," - " learning rate, etc...). When --skip_host_call=false, there could" - " be a performance drop if host_call function is slow and cannot" - " keep up with the TPU-side computation.")) - flags.DEFINE_string( - "mode", - default="train_and_eval", - help='One of {"train_and_eval", "train", "eval"}.') - FLAGS = flags.FLAGS - tf.app.run() diff --git a/tensorflow/contrib/eager/python/examples/revnet/ops.py b/tensorflow/contrib/eager/python/examples/revnet/ops.py deleted file mode 100644 index af17a22b4d1..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/ops.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Reversible residual network compatible with eager execution. - -Customized basic operations. - -Reference [The Reversible Residual Network: Backpropagation -Without Storing Activations](https://arxiv.org/pdf/1707.04585.pdf) -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - - -def downsample(x, filters, strides, axis=1): - """Downsample feature map with avg pooling, if filter size doesn't match.""" - - def pad_strides(strides, axis=1): - """Convert length 2 to length 4 strides. - - Needed since `tf.compat.v1.layers.Conv2D` uses length 2 strides, whereas - operations - such as `tf.nn.avg_pool2d` use length 4 strides. - - Args: - strides: length 2 list/tuple strides for height and width - axis: integer specifying feature dimension according to data format - - Returns: - length 4 strides padded with 1 on batch and channel dimension - """ - - assert len(strides) == 2 - - if axis == 1: - return [1, 1, strides[0], strides[1]] - return [1, strides[0], strides[1], 1] - - assert len(x.shape) == 4 and (axis == 1 or axis == 3) - - data_format = "NCHW" if axis == 1 else "NHWC" - strides_ = pad_strides(strides, axis=axis) - - if strides[0] > 1: - x = tf.nn.avg_pool( - x, strides_, strides_, padding="VALID", data_format=data_format) - - in_filter = x.shape[axis] - out_filter = filters - - if in_filter < out_filter: - pad_size = [(out_filter - in_filter) // 2, (out_filter - in_filter) // 2] - if axis == 1: - x = tf.pad(x, [[0, 0], pad_size, [0, 0], [0, 0]]) - else: - x = tf.pad(x, [[0, 0], [0, 0], [0, 0], pad_size]) - # In case `tape.gradient(x, [x])` produces a list of `None` - return x + 0. diff --git a/tensorflow/contrib/eager/python/examples/revnet/ops_test.py b/tensorflow/contrib/eager/python/examples/revnet/ops_test.py deleted file mode 100644 index e92c0fd9258..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/ops_test.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for basic ops used in eager mode RevNet.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf -from tensorflow.contrib.eager.python.examples.revnet import ops -tfe = tf.contrib.eager - - -class OpsTest(tf.test.TestCase): - - def test_downsample(self): - """Test `possible_down_sample` function with mock object.""" - - batch_size = 100 - # NHWC format - x = tf.random_normal(shape=[batch_size, 32, 32, 3]) - # HW doesn't change but number of features increased - y = ops.downsample(x, filters=5, strides=(1, 1), axis=3) - self.assertEqual(y.shape, [batch_size, 32, 32, 5]) - # Feature map doesn't change but HW reduced - y = ops.downsample(x, filters=3, strides=(2, 2), axis=3) - self.assertEqual(y.shape, [batch_size, 16, 16, 3]) - # Number of feature increased and HW reduced - y = ops.downsample(x, filters=5, strides=(2, 2), axis=3) - self.assertEqual(y.shape, [batch_size, 16, 16, 5]) - - # Test gradient flow - x = tf.random_normal(shape=[batch_size, 32, 32, 3]) - with tfe.GradientTape() as tape: - tape.watch(x) - y = ops.downsample(x, filters=3, strides=(1, 1)) - self.assertEqual(y.shape, x.shape) - dy = tf.random_normal(shape=[batch_size, 32, 32, 3]) - grad, = tape.gradient(y, [x], output_gradients=[dy]) - self.assertEqual(grad.shape, x.shape) - - # Default NCHW format - if tf.test.is_gpu_available(): - x = tf.random_normal(shape=[batch_size, 3, 32, 32]) - # HW doesn't change but feature map reduced - y = ops.downsample(x, filters=5, strides=(1, 1)) - self.assertEqual(y.shape, [batch_size, 5, 32, 32]) - # Feature map doesn't change but HW reduced - y = ops.downsample(x, filters=3, strides=(2, 2)) - self.assertEqual(y.shape, [batch_size, 3, 16, 16]) - # Both feature map and HW reduced - y = ops.downsample(x, filters=5, strides=(2, 2)) - self.assertEqual(y.shape, [batch_size, 5, 16, 16]) - - # Test gradient flow - x = tf.random_normal(shape=[batch_size, 3, 32, 32]) - with tfe.GradientTape() as tape: - tape.watch(x) - y = ops.downsample(x, filters=3, strides=(1, 1)) - self.assertEqual(y.shape, x.shape) - dy = tf.random_normal(shape=[batch_size, 3, 32, 32]) - grad, = tape.gradient(y, [x], output_gradients=[dy]) - self.assertEqual(grad.shape, x.shape) - - -if __name__ == '__main__': - tf.enable_eager_execution() - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/revnet/resnet_preprocessing.py b/tensorflow/contrib/eager/python/examples/revnet/resnet_preprocessing.py deleted file mode 100644 index 21a1ab85d46..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/resnet_preprocessing.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ImageNet preprocessing for ResNet.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -IMAGE_SIZE = 224 -CROP_PADDING = 32 - - -def distorted_bounding_box_crop(image_bytes, - bbox, - min_object_covered=0.1, - aspect_ratio_range=(0.75, 1.33), - area_range=(0.05, 1.0), - max_attempts=100, - scope=None): - """Generates cropped_image using one of the bboxes randomly distorted. - - See `tf.image.sample_distorted_bounding_box` for more documentation. - - Args: - image_bytes: `Tensor` of binary image data. - bbox: `Tensor` of bounding boxes arranged `[1, num_boxes, coords]` - where each coordinate is [0, 1) and the coordinates are arranged - as `[ymin, xmin, ymax, xmax]`. If num_boxes is 0 then use the whole - image. - min_object_covered: An optional `float`. Defaults to `0.1`. The cropped - area of the image must contain at least this fraction of any bounding - box supplied. - aspect_ratio_range: An optional list of `float`s. The cropped area of the - image must have an aspect ratio = width / height within this range. - area_range: An optional list of `float`s. The cropped area of the image - must contain a fraction of the supplied image within in this range. - max_attempts: An optional `int`. Number of attempts at generating a cropped - region of the image of the specified constraints. After `max_attempts` - failures, return the entire image. - scope: Optional `str` for name scope. - Returns: - cropped image `Tensor` - """ - with tf.name_scope(scope, 'distorted_bounding_box_crop', [image_bytes, bbox]): - shape = tf.image.extract_jpeg_shape(image_bytes) - sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box( - shape, - bounding_boxes=bbox, - min_object_covered=min_object_covered, - aspect_ratio_range=aspect_ratio_range, - area_range=area_range, - max_attempts=max_attempts, - use_image_if_no_bounding_boxes=True) - bbox_begin, bbox_size, _ = sample_distorted_bounding_box - - # Crop the image to the specified bounding box. - offset_y, offset_x, _ = tf.unstack(bbox_begin) - target_height, target_width, _ = tf.unstack(bbox_size) - crop_window = tf.stack([offset_y, offset_x, target_height, target_width]) - image = tf.image.decode_and_crop_jpeg(image_bytes, crop_window, channels=3) - - return image - - -def _at_least_x_are_equal(a, b, x): - """At least `x` of `a` and `b` `Tensors` are equal.""" - match = tf.equal(a, b) - match = tf.cast(match, tf.int32) - return tf.greater_equal(tf.reduce_sum(match), x) - - -def _decode_and_random_crop(image_bytes, image_size): - """Make a random crop of image_size.""" - bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4]) - image = distorted_bounding_box_crop( - image_bytes, - bbox, - min_object_covered=0.1, - aspect_ratio_range=(3. / 4, 4. / 3.), - area_range=(0.08, 1.0), - max_attempts=10, - scope=None) - original_shape = tf.image.extract_jpeg_shape(image_bytes) - bad = _at_least_x_are_equal(original_shape, tf.shape(image), 3) - - image = tf.cond( - bad, - lambda: _decode_and_center_crop(image_bytes, image_size), - lambda: tf.image.resize_bicubic([image], # pylint: disable=g-long-lambda - [image_size, image_size])[0]) - - return image - - -def _decode_and_center_crop(image_bytes, image_size): - """Crops to center of image with padding then scales image_size.""" - shape = tf.image.extract_jpeg_shape(image_bytes) - image_height = shape[0] - image_width = shape[1] - - padded_center_crop_size = tf.cast( - ((image_size / (image_size + CROP_PADDING)) * - tf.cast(tf.minimum(image_height, image_width), tf.float32)), - tf.int32) - - offset_height = ((image_height - padded_center_crop_size) + 1) // 2 - offset_width = ((image_width - padded_center_crop_size) + 1) // 2 - crop_window = tf.stack([offset_height, offset_width, - padded_center_crop_size, padded_center_crop_size]) - image = tf.image.decode_and_crop_jpeg(image_bytes, crop_window, channels=3) - image = tf.image.resize_bicubic([image], [image_size, image_size])[0] - - return image - - -def _flip(image): - """Random horizontal image flip.""" - image = tf.image.random_flip_left_right(image) - return image - - -def preprocess_for_train(image_bytes, use_bfloat16, image_size=IMAGE_SIZE): - """Preprocesses the given image for evaluation. - - Args: - image_bytes: `Tensor` representing an image binary of arbitrary size. - use_bfloat16: `bool` for whether to use bfloat16. - image_size: image size. - - Returns: - A preprocessed image `Tensor`. - """ - image = _decode_and_random_crop(image_bytes, image_size) - image = _flip(image) - image = tf.reshape(image, [image_size, image_size, 3]) - image = tf.image.convert_image_dtype( - image, dtype=tf.bfloat16 if use_bfloat16 else tf.float32) - return image - - -def preprocess_for_eval(image_bytes, use_bfloat16, image_size=IMAGE_SIZE): - """Preprocesses the given image for evaluation. - - Args: - image_bytes: `Tensor` representing an image binary of arbitrary size. - use_bfloat16: `bool` for whether to use bfloat16. - image_size: image size. - - Returns: - A preprocessed image `Tensor`. - """ - image = _decode_and_center_crop(image_bytes, image_size) - image = tf.reshape(image, [image_size, image_size, 3]) - image = tf.image.convert_image_dtype( - image, dtype=tf.bfloat16 if use_bfloat16 else tf.float32) - return image - - -def preprocess_image(image_bytes, - is_training=False, - use_bfloat16=False, - image_size=IMAGE_SIZE): - """Preprocesses the given image. - - Args: - image_bytes: `Tensor` representing an image binary of arbitrary size. - is_training: `bool` for whether the preprocessing is for training. - use_bfloat16: `bool` for whether to use bfloat16. - image_size: image size. - - Returns: - A preprocessed image `Tensor`. - """ - if is_training: - return preprocess_for_train(image_bytes, use_bfloat16, image_size) - else: - return preprocess_for_eval(image_bytes, use_bfloat16, image_size) diff --git a/tensorflow/contrib/eager/python/examples/revnet/revnet.py b/tensorflow/contrib/eager/python/examples/revnet/revnet.py deleted file mode 100644 index 08f2d8d6f17..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/revnet.py +++ /dev/null @@ -1,218 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Reversible residual network compatible with eager execution. - -Code for main model. - -Reference [The Reversible Residual Network: Backpropagation -Without Storing Activations](https://arxiv.org/pdf/1707.04585.pdf) -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf -from tensorflow.contrib.eager.python.examples.revnet import blocks - - -class RevNet(tf.keras.Model): - """RevNet that depends on all the blocks.""" - - def __init__(self, config): - """Initialize RevNet with building blocks. - - Args: - config: tf.contrib.training.HParams object; specifies hyperparameters - """ - super(RevNet, self).__init__(dtype=config.dtype) - self.axis = 1 if config.data_format == "channels_first" else 3 - self.config = config - - self._init_block = blocks.InitBlock(config=self.config) - self._final_block = blocks.FinalBlock(config=self.config) - self._block_list = self._construct_intermediate_blocks() - self._moving_average_variables = [] - - def _construct_intermediate_blocks(self): - # Precompute input shape after initial block - stride = self.config.init_stride - if self.config.init_max_pool: - stride *= 2 - if self.config.data_format == "channels_first": - w, h = self.config.input_shape[1], self.config.input_shape[2] - input_shape = (self.config.init_filters, w // stride, h // stride) - else: - w, h = self.config.input_shape[0], self.config.input_shape[1] - input_shape = (w // stride, h // stride, self.config.init_filters) - - # Aggregate intermediate blocks - block_list = tf.contrib.checkpoint.List() - for i in range(self.config.n_rev_blocks): - # RevBlock configurations - n_res = self.config.n_res[i] - filters = self.config.filters[i] - if filters % 2 != 0: - raise ValueError("Number of output filters must be even to ensure" - "correct partitioning of channels") - stride = self.config.strides[i] - strides = (self.config.strides[i], self.config.strides[i]) - - # Add block - rev_block = blocks.RevBlock( - n_res, - filters, - strides, - input_shape, - batch_norm_first=(i != 0), # Only skip on first block - data_format=self.config.data_format, - bottleneck=self.config.bottleneck, - fused=self.config.fused, - dtype=self.config.dtype) - block_list.append(rev_block) - - # Precompute input shape for the next block - if self.config.data_format == "channels_first": - w, h = input_shape[1], input_shape[2] - input_shape = (filters, w // stride, h // stride) - else: - w, h = input_shape[0], input_shape[1] - input_shape = (w // stride, h // stride, filters) - - return block_list - - def call(self, inputs, training=True): - """Forward pass.""" - - saved_hidden = None - if training: - saved_hidden = [inputs] - - h = self._init_block(inputs, training=training) - if training: - saved_hidden.append(h) - - for block in self._block_list: - h = block(h, training=training) - if training: - saved_hidden.append(h) - - logits = self._final_block(h, training=training) - - return (logits, saved_hidden) if training else (logits, None) - - def compute_loss(self, logits, labels): - """Compute cross entropy loss.""" - - if self.config.dtype == tf.float32 or self.config.dtype == tf.float16: - cross_ent = tf.nn.sparse_softmax_cross_entropy_with_logits( - logits=logits, labels=labels) - else: - # `sparse_softmax_cross_entropy_with_logits` does not have a GPU kernel - # for float64, int32 pairs - labels = tf.one_hot( - labels, depth=self.config.n_classes, axis=1, dtype=self.config.dtype) - cross_ent = tf.nn.softmax_cross_entropy_with_logits( - logits=logits, labels=labels) - - return tf.reduce_mean(cross_ent) - - def compute_gradients(self, saved_hidden, labels, training=True, l2_reg=True): - """Manually computes gradients. - - This method silently updates the running averages of batch normalization. - - Args: - saved_hidden: List of hidden states Tensors - labels: One-hot labels for classification - training: Use the mini-batch stats in batch norm if set to True - l2_reg: Apply l2 regularization - - Returns: - A tuple with the first entry being a list of all gradients and the second - being the loss - """ - - def _defunable_pop(l): - """Functional style list pop that works with `tfe.defun`.""" - t, l = l[-1], l[:-1] - return t, l - - # Backprop through last block - x = saved_hidden[-1] - with tf.GradientTape() as tape: - tape.watch(x) - logits = self._final_block(x, training=training) - loss = self.compute_loss(logits, labels) - grads_combined = tape.gradient(loss, - [x] + self._final_block.trainable_variables) - dy, final_grads = grads_combined[0], grads_combined[1:] - - # Backprop through intermediate blocks - intermediate_grads = [] - for block in reversed(self._block_list): - y, saved_hidden = _defunable_pop(saved_hidden) - x = saved_hidden[-1] - dy, grads = block.backward_grads(x, y, dy, training=training) - intermediate_grads = grads + intermediate_grads - - # Backprop through first block - _, saved_hidden = _defunable_pop(saved_hidden) - x, saved_hidden = _defunable_pop(saved_hidden) - assert not saved_hidden - with tf.GradientTape() as tape: - y = self._init_block(x, training=training) - init_grads = tape.gradient( - y, self._init_block.trainable_variables, output_gradients=dy) - - # Ordering match up with `model.trainable_variables` - grads_all = init_grads + final_grads + intermediate_grads - if l2_reg: - grads_all = self._apply_weight_decay(grads_all) - - return grads_all, loss - - def _apply_weight_decay(self, grads): - """Update gradients to reflect weight decay.""" - return [ - g + self.config.weight_decay * v if v.name.endswith("kernel:0") else g - for g, v in zip(grads, self.trainable_variables) - ] - - def get_moving_stats(self): - """Get moving averages of batch normalization.""" - device = "/gpu:0" if tf.test.is_gpu_available() else "/cpu:0" - with tf.device(device): - return [v.read_value() for v in self.moving_average_variables] - - def restore_moving_stats(self, values): - """Restore moving averages of batch normalization.""" - device = "/gpu:0" if tf.test.is_gpu_available() else "/cpu:0" - with tf.device(device): - for var_, val in zip(self.moving_average_variables, values): - var_.assign(val) - - @property - def moving_average_variables(self): - """Get all variables that are batch norm moving averages.""" - - def _is_moving_avg(v): - n = v.name - return n.endswith("moving_mean:0") or n.endswith("moving_variance:0") - - if not self._moving_average_variables: - self._moving_average_variables = filter(_is_moving_avg, self.variables) - - return self._moving_average_variables diff --git a/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py b/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py deleted file mode 100644 index 971aa44f303..00000000000 --- a/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py +++ /dev/null @@ -1,343 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for basic building blocks used in eager mode RevNet.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import gc -import time - -import tensorflow as tf -from tensorflow.contrib.eager.python.examples.revnet import blocks_test -from tensorflow.contrib.eager.python.examples.revnet import config as config_ -from tensorflow.contrib.eager.python.examples.revnet import revnet -from tensorflow.python.client import device_lib -tfe = tf.contrib.eager - - -def train_one_iter(model, inputs, labels, optimizer, global_step=None): - """Train for one iteration.""" - logits, saved_hidden = model(inputs) - grads, loss = model.compute_gradients( - saved_hidden=saved_hidden, labels=labels) - optimizer.apply_gradients( - zip(grads, model.trainable_variables), global_step=global_step) - - return logits, loss - - -class RevNetTest(tf.test.TestCase): - - def setUp(self): - super(RevNetTest, self).setUp() - config = config_.get_hparams_cifar_38() - config.add_hparam("n_classes", 10) - config.add_hparam("dataset", "cifar-10") - # Reconstruction could cause numerical error, use double precision for tests - config.dtype = tf.float64 - config.fused = False # Fused batch norm does not support tf.float64 - # Reduce the batch size for tests because the OSS version runs - # in constrained GPU environment with 1-2GB of memory. - config.batch_size = 2 - shape = (config.batch_size,) + config.input_shape - self.model = revnet.RevNet(config=config) - self.x = tf.random_normal(shape=shape, dtype=tf.float64) - self.t = tf.random_uniform( - shape=[config.batch_size], - minval=0, - maxval=config.n_classes, - dtype=tf.int64) - self.config = config - - def tearDown(self): - del self.model - del self.x - del self.t - del self.config - super(RevNetTest, self).tearDown() - - def test_call(self): - """Test `call` function.""" - - y, _ = self.model(self.x, training=False) - self.assertEqual(y.shape, [self.config.batch_size, self.config.n_classes]) - - def _check_grad_angle_combined(self, grads, grads_true): - """Verify that the reconstructed gradients has correct direction. - - Due to numerical imprecision, the magnitude may be slightly different. - Yet according to the paper, the angle should be roughly the same. - - Args: - grads: list of gradients from reconstruction - grads_true: list of true gradients - """ - - def _combine(gs): - return [tf.reshape(g, [-1]) for g in gs] - - g1_all = tf.concat(_combine(grads), axis=0) - g2_all = tf.concat(_combine(grads_true), axis=0) - - self.assertEqual(len(g1_all.shape), 1) - self.assertEqual(len(g2_all.shape), 1) - - degree = blocks_test.compute_degree(g1_all, g2_all) - self.assertLessEqual(degree, 1e0) - - def test_compute_gradients(self): - """Test `compute_gradients` function.""" - _, saved_hidden = self.model(self.x) # Initialize model - grads, loss = self.model.compute_gradients( - saved_hidden=saved_hidden, labels=self.t) - vars_ = self.model.trainable_variables - self.assertTrue(isinstance(grads, list)) - self.assertTrue(isinstance(vars_, list)) - self.assertEqual(len(grads), len(vars_)) - for grad, var in zip(grads, vars_): - self.assertEqual(grad.shape, var.shape) - - # Compare against the true gradient computed by the tape - with tf.GradientTape() as tape: - logits, _ = self.model(self.x) - loss_true = self.model.compute_loss(logits=logits, labels=self.t) - grads_true = tape.gradient(loss_true, vars_) - self.assertAllClose(loss, loss_true) - self.assertAllClose(grads, grads_true, rtol=1e-4, atol=1e-4) - self._check_grad_angle_combined(grads, grads_true) - - def test_call_defun(self): - """Test `call` function with defun.""" - y, _ = tfe.defun(self.model.call)(self.x, training=False) - self.assertEqual(y.shape, [self.config.batch_size, self.config.n_classes]) - - def test_compute_gradients_defun(self): - """Test `compute_gradients` function with defun.""" - # TODO(apassos): make cond support returning None to let this happen with - # tf.function. - compute_gradients = tfe.defun(self.model.compute_gradients) - _, saved_hidden = self.model(self.x) - grads, _ = compute_gradients(saved_hidden=saved_hidden, labels=self.t) - vars_ = self.model.trainable_variables - self.assertTrue(isinstance(grads, list)) - self.assertTrue(isinstance(vars_, list)) - self.assertEqual(len(grads), len(vars_)) - for grad, var in zip(grads, vars_): - if grad is not None: - self.assertEqual(grad.shape, var.shape) - - def test_training_graph(self): - """Test model training in graph mode.""" - with tf.Graph().as_default(): - config = config_.get_hparams_cifar_38() - config.add_hparam("n_classes", 10) - config.add_hparam("dataset", "cifar-10") - - x = tf.random_normal( - shape=(self.config.batch_size,) + self.config.input_shape) - t = tf.random_uniform( - shape=(self.config.batch_size,), - minval=0, - maxval=self.config.n_classes, - dtype=tf.int32) - global_step = tf.Variable(0., trainable=False) - model = revnet.RevNet(config=config) - _, saved_hidden = model(x) - grads, _ = model.compute_gradients(saved_hidden=saved_hidden, labels=t) - optimizer = tf.train.AdamOptimizer(learning_rate=1e-3) - train_op = optimizer.apply_gradients( - zip(grads, model.trainable_variables), global_step=global_step) - - with tf.Session() as sess: - sess.run(tf.global_variables_initializer()) - for _ in range(1): - sess.run(train_op) - - -# Benchmark related -def device_and_data_format(): - return ("/gpu:0", - "channels_first") if tf.test.is_gpu_available() else ("/cpu:0", - "channels_last") - - -def random_batch(batch_size, config): - shape = (batch_size,) + config.input_shape - images = tf.random_uniform(shape) - labels = tf.random_uniform( - [batch_size], minval=0, maxval=config.n_classes, dtype=tf.int32) - - return images, labels - - -class MockIterator(object): - - def __init__(self, tensors): - self._tensors = [tf.identity(x) for x in tensors] - - def next(self): - return self._tensors - - -class RevNetBenchmark(tf.test.Benchmark): - """Eager and graph benchmarks for RevNet.""" - - def _train_batch_sizes(self): - """Shamelessly copied from `resnet50_test.py`. - - Note: This is targeted towards ImageNet. CIFAR-10 should allow more - aggressive batch sizes. - - Returns: - A tuple of possible batch sizes - """ - for device in device_lib.list_local_devices(): - if tf.DeviceSpec.from_string(device.name).device_type == "GPU": - if "K20" in device.physical_device_desc: - return (16,) - if "P100" in device.physical_device_desc: - return (16, 32, 64) - if tf.DeviceSpec.from_string(device.name).device_type == "TPU": - return (32,) - return (16, 32) - - def _force_device_sync(self): - """Shamelessly copied from `resnet50_test.py`.""" - tf.constant(1.).cpu() - - def _report(self, label, start, num_iters, device, batch_size, data_format): - avg_time = (time.time() - start) / num_iters - dev = tf.DeviceSpec.from_string(device).device_type.lower() - name = "%s_%s_batch_%d_%s" % (label, dev, batch_size, data_format) - extras = {"examples_per_sec": batch_size / avg_time} - self.report_benchmark( - iters=num_iters, wall_time=avg_time, name=name, extras=extras) - - def _benchmark_eager_apply(self, - label, - device_and_format, - defun=False, - execution_mode=None): - config = config_.get_hparams_imagenet_56() - with tfe.execution_mode(execution_mode): - device, data_format = device_and_format - model = revnet.RevNet(config=config) - if defun: - # TODO(apassos): reenable after cond lets you return None - model.call = tfe.defun(model.call) - batch_size = 64 - num_burn = 5 - num_iters = 10 - with tf.device(device): - images, _ = random_batch(batch_size, config) - for _ in range(num_burn): - model(images, training=False) - if execution_mode: - tfe.async_wait() - gc.collect() - start = time.time() - for _ in range(num_iters): - model(images, training=False) - if execution_mode: - tfe.async_wait() - self._report(label, start, num_iters, device, batch_size, data_format) - - def benchmark_eager_apply_sync(self): - self._benchmark_eager_apply( - "eager_apply_sync", device_and_data_format(), defun=False) - - def benchmark_eager_apply_async(self): - self._benchmark_eager_apply( - "eager_apply_async", - device_and_data_format(), - defun=False, - execution_mode=tfe.ASYNC) - - def benchmark_eager_call_defun(self): - self._benchmark_eager_apply( - "eager_apply_with_defun", device_and_data_format(), defun=True) - - def _benchmark_eager_train(self, - label, - make_iterator, - device_and_format, - defun=False, - execution_mode=None): - config = config_.get_hparams_imagenet_56() - with tfe.execution_mode(execution_mode): - device, data_format = device_and_format - for batch_size in self._train_batch_sizes(): - (images, labels) = random_batch(batch_size, config) - model = revnet.RevNet(config=config) - optimizer = tf.train.GradientDescentOptimizer(0.1) - if defun: - model.call = tfe.function(model.call) - - num_burn = 3 - num_iters = 10 - with tf.device(device): - iterator = make_iterator((images, labels)) - for _ in range(num_burn): - (images, labels) = iterator.next() - train_one_iter(model, images, labels, optimizer) - if execution_mode: - tfe.async_wait() - self._force_device_sync() - gc.collect() - - start = time.time() - for _ in range(num_iters): - (images, labels) = iterator.next() - train_one_iter(model, images, labels, optimizer) - if execution_mode: - tfe.async_wait() - self._force_device_sync() - self._report(label, start, num_iters, device, batch_size, data_format) - - def benchmark_eager_train_sync(self): - self._benchmark_eager_train( - "eager_train_sync", MockIterator, device_and_data_format(), defun=False) - - def benchmark_eager_train_async(self): - self._benchmark_eager_train( - "eager_train_async", - MockIterator, - device_and_data_format(), - defun=False, - execution_mode=tfe.ASYNC) - - def benchmark_eager_train_defun(self): - self._benchmark_eager_train( - "eager_train", MockIterator, device_and_data_format(), defun=False) - - def benchmark_eager_train_datasets_with_defun(self): - - def make_iterator(tensors): - with tf.device("/device:CPU:0"): - ds = tf.data.Dataset.from_tensors(tensors).repeat() - return tfe.Iterator(ds) - - self._benchmark_eager_train( - "eager_train_dataset_with_defun", - make_iterator, - device_and_data_format(), - defun=True) - - -if __name__ == "__main__": - tf.enable_eager_execution() - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD deleted file mode 100644 index 82f8dc56289..00000000000 --- a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD +++ /dev/null @@ -1,40 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "py_binary") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -py_binary( - name = "rnn_colorbot", - srcs = ["rnn_colorbot.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [":rnn_colorbot_lib"], -) - -py_library( - name = "rnn_colorbot_lib", - srcs = ["rnn_colorbot.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/eager/python:tfe", - "//tensorflow/python/eager:context", - "@six_archive//:six", - ], -) - -cuda_py_test( - name = "rnn_colorbot_test", - srcs = ["rnn_colorbot_test.py"], - additional_deps = [ - ":rnn_colorbot_lib", - "//tensorflow/contrib/eager/python:tfe", - "//tensorflow:tensorflow_py", - ], - tags = [ - "oss_serial", - ], -) diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/README.md b/tensorflow/contrib/eager/python/examples/rnn_colorbot/README.md deleted file mode 100644 index 750bbc66f35..00000000000 --- a/tensorflow/contrib/eager/python/examples/rnn_colorbot/README.md +++ /dev/null @@ -1,26 +0,0 @@ -RNN Colorbot: An RNN that predicts colors using eager execution. - -To train and generate colors, run: - -``` -python rnn_colorbot.py -``` - -This example shows how to: - 1. read, process, (one-hot) encode, and pad text data via the - Datasets API; - 2. build a trainable model; - 3. implement a multi-layer RNN using Python control flow - constructs (e.g., a for loop); - 4. train a model using an iterative gradient-based method; and - 5. log training and evaluation loss for consumption by TensorBoard - (to view summaries, use: tensorboard --log_dir=/summaries). - -The data used in this example is licensed under the Creative Commons -Attribution-ShareAlike License and is available at - https://en.wikipedia.org/wiki/List_of_colors:_A-F - https://en.wikipedia.org/wiki/List_of_colors:_G-M - https://en.wikipedia.org/wiki/List_of_colors:_N-Z - -This example was adapted from - https://github.com/random-forests/tensorflow-workshop/tree/master/archive/extras/colorbot diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py deleted file mode 100644 index 1c718a5ce3d..00000000000 --- a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py +++ /dev/null @@ -1,360 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -r"""TensorFlow Eager Execution Example: RNN Colorbot. - -This example builds, trains, and evaluates a multi-layer RNN that can be -run with eager execution enabled. The RNN is trained to map color names to -their RGB values: it takes as input a one-hot encoded character sequence and -outputs a three-tuple (R, G, B) (scaled by 1/255). - -For example, say we'd like the RNN Colorbot to generate the RGB values for the -color white. To represent our query in a form that the Colorbot could -understand, we would create a sequence of five 256-long vectors encoding the -ASCII values of the characters in "white". The first vector in our sequence -would be 0 everywhere except for the ord("w")-th position, where it would be -1, the second vector would be 0 everywhere except for the -ord("h")-th position, where it would be 1, and similarly for the remaining three -vectors. We refer to such indicator vectors as "one-hot encodings" of -characters. After consuming these vectors, a well-trained Colorbot would output -the three tuple (1, 1, 1), since the RGB values for white are (255, 255, 255). -We are of course free to ask the colorbot to generate colors for any string we'd -like, such as "steel gray," "tensorflow orange," or "green apple," though -your mileage may vary as your queries increase in creativity. - -This example shows how to: - 1. read, process, (one-hot) encode, and pad text data via the - Datasets API; - 2. build a trainable model; - 3. implement a multi-layer RNN using Python control flow - constructs (e.g., a for loop); - 4. train a model using an iterative gradient-based method; and - -The data used in this example is licensed under the Creative Commons -Attribution-ShareAlike License and is available at - https://en.wikipedia.org/wiki/List_of_colors:_A-F - https://en.wikipedia.org/wiki/List_of_colors:_G-M - https://en.wikipedia.org/wiki/List_of_colors:_N-Z - -This example was adapted from - https://github.com/random-forests/tensorflow-workshop/tree/master/extras/colorbot -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import functools -import os -import sys -import time -import urllib - -import six -import tensorflow as tf - -from tensorflow.contrib.eager.python import tfe - -try: - import matplotlib.pyplot as plt # pylint: disable=g-import-not-at-top - HAS_MATPLOTLIB = True -except ImportError: - HAS_MATPLOTLIB = False - -layers = tf.keras.layers - - -def parse(line): - """Parse a line from the colors dataset.""" - - # Each line of the dataset is comma-separated and formatted as - # color_name, r, g, b - # so `items` is a list [color_name, r, g, b]. - items = tf.string_split([line], ",").values - rgb = tf.string_to_number(items[1:], out_type=tf.float32) / 255. - # Represent the color name as a one-hot encoded character sequence. - color_name = items[0] - chars = tf.one_hot(tf.decode_raw(color_name, tf.uint8), depth=256) - # The sequence length is needed by our RNN. - length = tf.cast(tf.shape(chars)[0], dtype=tf.int64) - return rgb, chars, length - - -def maybe_download(filename, work_directory, source_url): - """Download the data from source url, unless it's already here. - - Args: - filename: string, name of the file in the directory. - work_directory: string, path to working directory. - source_url: url to download from if file doesn't exist. - - Returns: - Path to resulting file. - """ - if not tf.gfile.Exists(work_directory): - tf.gfile.MakeDirs(work_directory) - filepath = os.path.join(work_directory, filename) - if not tf.gfile.Exists(filepath): - temp_file_name, _ = urllib.request.urlretrieve(source_url) - tf.gfile.Copy(temp_file_name, filepath) - with tf.gfile.GFile(filepath) as f: - size = f.size() - print("Successfully downloaded", filename, size, "bytes.") - return filepath - - -def load_dataset(data_dir, url, batch_size): - """Loads the colors data at path into a PaddedDataset.""" - - # Downloads data at url into data_dir/basename(url). The dataset has a header - # row (color_name, r, g, b) followed by comma-separated lines. - path = maybe_download(os.path.basename(url), data_dir, url) - - # This chain of commands loads our data by: - # 1. skipping the header; (.skip(1)) - # 2. parsing the subsequent lines; (.map(parse)) - # 3. shuffling the data; (.shuffle(...)) - # 3. grouping the data into padded batches (.padded_batch(...)). - dataset = tf.data.TextLineDataset(path).skip(1).map(parse).shuffle( - buffer_size=10000).padded_batch( - batch_size, padded_shapes=([None], [None, None], [])) - return dataset - - -# pylint: disable=not-callable -class RNNColorbot(tf.keras.Model): - """Multi-layer (LSTM) RNN that regresses on real-valued vector labels. - """ - - def __init__(self, rnn_cell_sizes, label_dimension, keep_prob): - """Constructs an RNNColorbot. - - Args: - rnn_cell_sizes: list of integers denoting the size of each LSTM cell in - the RNN; rnn_cell_sizes[i] is the size of the i-th layer cell - label_dimension: the length of the labels on which to regress - keep_prob: (1 - dropout probability); dropout is applied to the outputs of - each LSTM layer - """ - super(RNNColorbot, self).__init__(name="") - self.label_dimension = label_dimension - self.keep_prob = keep_prob - - self.cells = tf.contrib.checkpoint.List( - [tf.nn.rnn_cell.BasicLSTMCell(size) for size in rnn_cell_sizes]) - self.relu = layers.Dense( - label_dimension, activation=tf.nn.relu, name="relu") - - def call(self, inputs, training=False): - """Implements the RNN logic and prediction generation. - - Args: - inputs: A tuple (chars, sequence_length), where chars is a batch of - one-hot encoded color names represented as a Tensor with dimensions - [batch_size, time_steps, 256] and sequence_length holds the length - of each character sequence (color name) as a Tensor with dimension - [batch_size]. - training: whether the invocation is happening during training - - Returns: - A tensor of dimension [batch_size, label_dimension] that is produced by - passing chars through a multi-layer RNN and applying a ReLU to the final - hidden state. - """ - (chars, sequence_length) = inputs - # Transpose the first and second dimensions so that chars is of shape - # [time_steps, batch_size, dimension]. - chars = tf.transpose(chars, [1, 0, 2]) - # The outer loop cycles through the layers of the RNN; the inner loop - # executes the time steps for a particular layer. - batch_size = int(chars.shape[1]) - for l in range(len(self.cells)): - cell = self.cells[l] - outputs = [] - state = cell.zero_state(batch_size, tf.float32) - # Unstack the inputs to obtain a list of batches, one for each time step. - chars = tf.unstack(chars, axis=0) - for ch in chars: - output, state = cell(ch, state) - outputs.append(output) - # The outputs of this layer are the inputs of the subsequent layer. - chars = tf.stack(outputs, axis=0) - if training: - chars = tf.nn.dropout(chars, self.keep_prob) - # Extract the correct output (i.e., hidden state) for each example. All the - # character sequences in this batch were padded to the same fixed length so - # that they could be easily fed through the above RNN loop. The - # `sequence_length` vector tells us the true lengths of the character - # sequences, letting us obtain for each sequence the hidden state that was - # generated by its non-padding characters. - batch_range = [i for i in range(batch_size)] - indices = tf.stack([sequence_length - 1, batch_range], axis=1) - hidden_states = tf.gather_nd(chars, indices) - return self.relu(hidden_states) - - -def loss(labels, predictions): - """Computes mean squared loss.""" - return tf.reduce_mean(tf.squared_difference(predictions, labels)) - - -def test(model, eval_data): - """Computes the average loss on eval_data, which should be a Dataset.""" - avg_loss = tfe.metrics.Mean("loss") - for (labels, chars, sequence_length) in tfe.Iterator(eval_data): - predictions = model((chars, sequence_length), training=False) - avg_loss(loss(labels, predictions)) - print("eval/loss: %.6f\n" % avg_loss.result()) - with tf.contrib.summary.always_record_summaries(): - tf.contrib.summary.scalar("loss", avg_loss.result()) - - -def train_one_epoch(model, optimizer, train_data, log_interval=10): - """Trains model on train_data using optimizer.""" - - tf.train.get_or_create_global_step() - - def model_loss(labels, chars, sequence_length): - predictions = model((chars, sequence_length), training=True) - loss_value = loss(labels, predictions) - tf.contrib.summary.scalar("loss", loss_value) - return loss_value - - for (batch, (labels, chars, sequence_length)) in enumerate( - tfe.Iterator(train_data)): - with tf.contrib.summary.record_summaries_every_n_global_steps(log_interval): - batch_model_loss = functools.partial(model_loss, labels, chars, - sequence_length) - optimizer.minimize( - batch_model_loss, global_step=tf.train.get_global_step()) - if log_interval and batch % log_interval == 0: - print("train/batch #%d\tloss: %.6f" % (batch, batch_model_loss())) - - -SOURCE_TRAIN_URL = "https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/archive/extras/colorbot/data/train.csv" -SOURCE_TEST_URL = "https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/archive/extras/colorbot/data/test.csv" - - -def main(_): - data_dir = os.path.join(FLAGS.dir, "data") - train_data = load_dataset( - data_dir=data_dir, url=SOURCE_TRAIN_URL, batch_size=FLAGS.batch_size) - eval_data = load_dataset( - data_dir=data_dir, url=SOURCE_TEST_URL, batch_size=FLAGS.batch_size) - - model = RNNColorbot( - rnn_cell_sizes=FLAGS.rnn_cell_sizes, - label_dimension=3, - keep_prob=FLAGS.keep_probability) - optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate) - - if FLAGS.no_gpu or tfe.num_gpus() <= 0: - print(tfe.num_gpus()) - device = "/cpu:0" - else: - device = "/gpu:0" - print("Using device %s." % device) - - log_dir = os.path.join(FLAGS.dir, "summaries") - tf.gfile.MakeDirs(log_dir) - train_summary_writer = tf.contrib.summary.create_file_writer( - os.path.join(log_dir, "train"), flush_millis=10000) - test_summary_writer = tf.contrib.summary.create_file_writer( - os.path.join(log_dir, "eval"), flush_millis=10000, name="eval") - - with tf.device(device): - for epoch in range(FLAGS.num_epochs): - start = time.time() - with train_summary_writer.as_default(): - train_one_epoch(model, optimizer, train_data, FLAGS.log_interval) - end = time.time() - print("train/time for epoch #%d: %.2f" % (epoch, end - start)) - with test_summary_writer.as_default(): - test(model, eval_data) - - print("Colorbot is ready to generate colors!") - while True: - try: - color_name = six.moves.input( - "Give me a color name (or press enter to exit): ") - except EOFError: - return - - if not color_name: - return - - _, chars, length = parse(color_name) - with tf.device(device): - (chars, length) = (tf.identity(chars), tf.identity(length)) - chars = tf.expand_dims(chars, 0) - length = tf.expand_dims(length, 0) - preds = tf.unstack(model((chars, length), training=False)[0]) - - # Predictions cannot be negative, as they are generated by a ReLU layer; - # they may, however, be greater than 1. - clipped_preds = tuple(min(float(p), 1.0) for p in preds) - rgb = tuple(int(p * 255) for p in clipped_preds) - print("rgb:", rgb) - data = [[clipped_preds]] - if HAS_MATPLOTLIB: - plt.imshow(data) - plt.title(color_name) - plt.show() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--dir", - type=str, - default="/tmp/rnn_colorbot/", - help="Directory to download data files and save logs.") - parser.add_argument( - "--log_interval", - type=int, - default=10, - metavar="N", - help="Log training loss every log_interval batches.") - parser.add_argument( - "--num_epochs", type=int, default=20, help="Number of epochs to train.") - parser.add_argument( - "--rnn_cell_sizes", - type=int, - nargs="+", - default=[256, 128], - help="List of sizes for each layer of the RNN.") - parser.add_argument( - "--batch_size", - type=int, - default=64, - help="Batch size for training and eval.") - parser.add_argument( - "--keep_probability", - type=float, - default=0.5, - help="Keep probability for dropout between layers.") - parser.add_argument( - "--learning_rate", - type=float, - default=0.01, - help="Learning rate to be used during training.") - parser.add_argument( - "--no_gpu", - action="store_true", - default=False, - help="Disables GPU usage even if a GPU is available.") - - FLAGS, unparsed = parser.parse_known_args() - tfe.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot_test.py b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot_test.py deleted file mode 100644 index 2955b94037f..00000000000 --- a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot_test.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -from tensorflow.contrib.eager.python.examples.rnn_colorbot import rnn_colorbot -from tensorflow.python.framework import test_util - - -LABEL_DIMENSION = 5 - - -def random_dataset(): - batch_size = 64 - time_steps = 10 - alphabet = 50 - chars = tf.one_hot( - tf.random_uniform( - [batch_size, time_steps], minval=0, maxval=alphabet, dtype=tf.int32), - alphabet) - sequence_length = tf.constant( - [time_steps for _ in range(batch_size)], dtype=tf.int64) - labels = tf.random_normal([batch_size, LABEL_DIMENSION]) - return tf.data.Dataset.from_tensors((labels, chars, sequence_length)) - - -class RNNColorbotTest(tf.test.TestCase): - - def testTrainOneEpoch(self): - model = rnn_colorbot.RNNColorbot( - rnn_cell_sizes=[256, 128, 64], - label_dimension=LABEL_DIMENSION, - keep_prob=1.0) - optimizer = tf.train.AdamOptimizer(learning_rate=.01) - dataset = random_dataset() - with test_util.use_gpu(): - rnn_colorbot.train_one_epoch(model, optimizer, dataset) - - def testTest(self): - model = rnn_colorbot.RNNColorbot( - rnn_cell_sizes=[256], - label_dimension=LABEL_DIMENSION, - keep_prob=1.0) - dataset = random_dataset() - with test_util.use_gpu(): - rnn_colorbot.test(model, dataset) - - -if __name__ == "__main__": - tf.enable_eager_execution() - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD deleted file mode 100644 index 362dacff9fe..00000000000 --- a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD +++ /dev/null @@ -1,50 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "py_binary") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -py_binary( - name = "rnn_ptb", - srcs = ["rnn_ptb.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [":rnn_ptb_lib"], -) - -py_library( - name = "rnn_ptb_lib", - srcs = ["rnn_ptb.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/cudnn_rnn:cudnn_rnn_py", - "//tensorflow/contrib/eager/python:tfe", - "//third_party/py/numpy", - ], -) - -cuda_py_test( - name = "rnn_ptb_test", - srcs = ["rnn_ptb_test.py"], - additional_deps = [ - ":rnn_ptb_lib", - "//tensorflow/contrib/eager/python:tfe", - "//tensorflow:tensorflow_py", - ], -) - -cuda_py_test( - name = "rnn_ptb_graph_test", - srcs = ["rnn_ptb_graph_test.py"], - additional_deps = [ - ":rnn_ptb_lib", - "//third_party/py/numpy", - "//tensorflow:tensorflow_py", - ], - tags = [ - "oss_serial", - ], -) diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/README.md b/tensorflow/contrib/eager/python/examples/rnn_ptb/README.md deleted file mode 100644 index 966177e91c2..00000000000 --- a/tensorflow/contrib/eager/python/examples/rnn_ptb/README.md +++ /dev/null @@ -1,54 +0,0 @@ -Recurrent Neural Network model. - -Implements a language modeling network described in -https://www.tensorflow.org/tutorials/recurrent -that is compatible with (and idiomatic for) eager execution. - -To run: - -- Download and extract the Penn Treebank dataset from - http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz - - ```sh - tar xvzf simple-examples.tgz -C /tmp - ``` - -- Run: `python rnn_ptb.py --data-dir=/tmp/simple-examples/data` - - -Benchmarks (using synthetic data): - -``` -# Using eager execution -python rnn_ptb_test.py --benchmarks=. - -# Using graph execution -python rnn_ptb_graph_test.py --benchmarks=. -``` - -The above uses the model definition included with the TensorFlow pip -package. To build (and run benchmarks) from source: - - -``` -# Using eager execution -bazel run -c opt --config=cuda :rnn_ptb_test -- --benchmarks=. - -# Using graph execution -bazel run -c opt --config=cuda :rnn_ptb_graph_test -- --benchmarks=. -``` - -(Or remove the `--config=cuda` flag for running on CPU instead of GPU). - -On October 31, 2017, the benchmarks demonstrated slightly better performance -(3-6%) for graph execution over eager execution for this particular model when -using a single NVIDIA Titan X (Pascal) GPU on a host with an Intel Xeon E5-1650 -CPU @ 3.50GHz and a batch size of 32. - -| Benchmark name | examples/second | -| ------------------------------------ | --------------- | -| eager_cudnn_train_large_gpu_batch_20 | 938 | -| graph_cudnn_train_large_gpu_batch_20 | 971 | -| eager_cudnn_train_small_gpu_batch_20 | 2433 | -| graph_cudnn_train_small_gpu_batch_20 | 2585 | - diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py deleted file mode 100644 index 56aeb534230..00000000000 --- a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py +++ /dev/null @@ -1,374 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Penn Treebank RNN model definition compatible with eager execution. - -Model similar to -https://github.com/tensorflow/models/tree/master/tutorials/rnn/ptb - -Usage: python ./rnn_ptb.py --data-path= - -Penn Treebank (PTB) dataset from: -http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import os -import sys -import time - -import numpy as np -import tensorflow as tf - -from tensorflow.contrib.cudnn_rnn.python.layers import cudnn_rnn -from tensorflow.contrib.eager.python import tfe - -layers = tf.keras.layers - - -class RNN(tf.keras.Model): - """A static RNN. - - Similar to tf.compat.v1.nn.static_rnn, implemented as a class. - """ - - def __init__(self, hidden_dim, num_layers, keep_ratio): - super(RNN, self).__init__() - self.keep_ratio = keep_ratio - self.cells = tf.contrib.checkpoint.List([ - tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_dim) - for _ in range(num_layers) - ]) - - def call(self, input_seq, training): - batch_size = int(input_seq.shape[1]) - for c in self.cells: - state = c.zero_state(batch_size, tf.float32) - outputs = [] - input_seq = tf.unstack(input_seq, num=int(input_seq.shape[0]), axis=0) - for inp in input_seq: - output, state = c(inp, state) - outputs.append(output) - - input_seq = tf.stack(outputs, axis=0) - if training: - input_seq = tf.nn.dropout(input_seq, self.keep_ratio) - # Returning a list instead of a single tensor so that the line: - # y = self.rnn(y, ...)[0] - # in PTBModel.call works for both this RNN and CudnnLSTM (which returns a - # tuple (output, output_states). - return [input_seq] - - -class Embedding(layers.Layer): - """An Embedding layer.""" - - def __init__(self, vocab_size, embedding_dim, **kwargs): - super(Embedding, self).__init__(**kwargs) - self.vocab_size = vocab_size - self.embedding_dim = embedding_dim - - def build(self, _): - self.embedding = self.add_variable( - "embedding_kernel", - shape=[self.vocab_size, self.embedding_dim], - dtype=tf.float32, - initializer=tf.random_uniform_initializer(-0.1, 0.1), - trainable=True) - - def call(self, x): - return tf.nn.embedding_lookup(self.embedding, x) - - -# pylint: disable=not-callable -class PTBModel(tf.keras.Model): - """LSTM for word language modeling. - - Model described in: - (Zaremba, et. al.) Recurrent Neural Network Regularization - http://arxiv.org/abs/1409.2329 - - See also: - https://github.com/tensorflow/models/tree/master/tutorials/rnn/ptb - """ - - def __init__(self, - vocab_size, - embedding_dim, - hidden_dim, - num_layers, - dropout_ratio, - use_cudnn_rnn=True): - super(PTBModel, self).__init__() - - self.keep_ratio = 1 - dropout_ratio - self.use_cudnn_rnn = use_cudnn_rnn - self.embedding = Embedding(vocab_size, embedding_dim) - - if self.use_cudnn_rnn: - self.rnn = cudnn_rnn.CudnnLSTM( - num_layers, hidden_dim, dropout=dropout_ratio) - else: - self.rnn = RNN(hidden_dim, num_layers, self.keep_ratio) - - self.linear = layers.Dense( - vocab_size, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1)) - self._output_shape = [-1, hidden_dim] - - def call(self, input_seq, training): - """Run the forward pass of PTBModel. - - Args: - input_seq: [length, batch] shape int64 tensor. - training: Is this a training call. - Returns: - outputs tensors of inference. - """ - y = self.embedding(input_seq) - if training: - y = tf.nn.dropout(y, self.keep_ratio) - y = self.rnn(y, training=training)[0] - return self.linear(tf.reshape(y, self._output_shape)) - - -def clip_gradients(grads_and_vars, clip_ratio): - gradients, variables = zip(*grads_and_vars) - clipped, _ = tf.clip_by_global_norm(gradients, clip_ratio) - return zip(clipped, variables) - - -def loss_fn(model, inputs, targets, training): - labels = tf.reshape(targets, [-1]) - outputs = model(inputs, training=training) - return tf.reduce_mean( - tf.nn.sparse_softmax_cross_entropy_with_logits( - labels=labels, logits=outputs)) - - -def _divide_into_batches(data, batch_size): - """Convert a sequence to a batch of sequences.""" - nbatch = data.shape[0] // batch_size - data = data[:nbatch * batch_size] - data = data.reshape(batch_size, -1).transpose() - return data - - -def _get_batch(data, i, seq_len): - slen = min(seq_len, data.shape[0] - 1 - i) - inputs = data[i:i + slen, :] - target = data[i + 1:i + 1 + slen, :] - return tf.constant(inputs), tf.constant(target) - - -def evaluate(model, data): - """evaluate an epoch.""" - total_loss = 0.0 - total_batches = 0 - start = time.time() - for _, i in enumerate(range(0, data.shape[0] - 1, FLAGS.seq_len)): - inp, target = _get_batch(data, i, FLAGS.seq_len) - loss = loss_fn(model, inp, target, training=False) - total_loss += loss.numpy() - total_batches += 1 - time_in_ms = (time.time() - start) * 1000 - sys.stderr.write("eval loss %.2f (eval took %d ms)\n" % - (total_loss / total_batches, time_in_ms)) - return total_loss - - -def train(model, optimizer, train_data, sequence_length, clip_ratio): - """training an epoch.""" - - def model_loss(inputs, targets): - return loss_fn(model, inputs, targets, training=True) - - grads = tfe.implicit_gradients(model_loss) - - total_time = 0 - for batch, i in enumerate(range(0, train_data.shape[0] - 1, sequence_length)): - train_seq, train_target = _get_batch(train_data, i, sequence_length) - start = time.time() - optimizer.apply_gradients( - clip_gradients(grads(train_seq, train_target), clip_ratio)) - total_time += (time.time() - start) - if batch % 10 == 0: - time_in_ms = (total_time * 1000) / (batch + 1) - sys.stderr.write("batch %d: training loss %.2f, avg step time %d ms\n" % - (batch, model_loss(train_seq, train_target).numpy(), - time_in_ms)) - - -class Datasets(object): - """Processed form of the Penn Treebank dataset.""" - - def __init__(self, path): - """Load the Penn Treebank dataset. - - Args: - path: Path to the data/ directory of the dataset from Tomas Mikolov's - webpage - http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz - """ - - self.word2idx = {} # string -> integer id - self.idx2word = [] # integer id -> word string - # Files represented as a list of integer ids (as opposed to list of string - # words). - self.train = self.tokenize(os.path.join(path, "ptb.train.txt")) - self.valid = self.tokenize(os.path.join(path, "ptb.valid.txt")) - - def vocab_size(self): - return len(self.idx2word) - - def add(self, word): - if word not in self.word2idx: - self.idx2word.append(word) - self.word2idx[word] = len(self.idx2word) - 1 - - def tokenize(self, path): - """Read text file in path and return a list of integer token ids.""" - tokens = 0 - with tf.gfile.Open(path, "r") as f: - for line in f: - words = line.split() + [""] - tokens += len(words) - for word in words: - self.add(word) - - # Tokenize file content - with tf.gfile.Open(path, "r") as f: - ids = np.zeros(tokens).astype(np.int64) - token = 0 - for line in f: - words = line.split() + [""] - for word in words: - ids[token] = self.word2idx[word] - token += 1 - - return ids - - -def small_model(use_cudnn_rnn): - """Returns a PTBModel with a 'small' configuration.""" - return PTBModel( - vocab_size=10000, - embedding_dim=200, - hidden_dim=200, - num_layers=2, - dropout_ratio=0., - use_cudnn_rnn=use_cudnn_rnn) - - -def large_model(use_cudnn_rnn): - """Returns a PTBModel with a 'large' configuration.""" - return PTBModel( - vocab_size=10000, - embedding_dim=650, - hidden_dim=650, - num_layers=2, - dropout_ratio=0.5, - use_cudnn_rnn=use_cudnn_rnn) - - -def test_model(use_cudnn_rnn): - """Returns a tiny PTBModel for unit tests.""" - return PTBModel( - vocab_size=100, - embedding_dim=20, - hidden_dim=20, - num_layers=2, - dropout_ratio=0., - use_cudnn_rnn=use_cudnn_rnn) - - -def main(_): - tf.enable_eager_execution() - - if not FLAGS.data_path: - raise ValueError("Must specify --data-path") - corpus = Datasets(FLAGS.data_path) - train_data = _divide_into_batches(corpus.train, FLAGS.batch_size) - eval_data = _divide_into_batches(corpus.valid, 10) - - have_gpu = tfe.num_gpus() > 0 - use_cudnn_rnn = not FLAGS.no_use_cudnn_rnn and have_gpu - - with tf.device("/device:GPU:0" if have_gpu else None): - # Make learning_rate a Variable so it can be included in the checkpoint - # and we can resume training with the last saved learning_rate. - learning_rate = tf.Variable(20.0, name="learning_rate") - model = PTBModel(corpus.vocab_size(), FLAGS.embedding_dim, - FLAGS.hidden_dim, FLAGS.num_layers, FLAGS.dropout, - use_cudnn_rnn) - optimizer = tf.train.GradientDescentOptimizer(learning_rate) - checkpoint = tf.train.Checkpoint( - learning_rate=learning_rate, model=model, - # GradientDescentOptimizer has no state to checkpoint, but noting it - # here lets us swap in an optimizer that does. - optimizer=optimizer) - # Restore existing variables now (learning_rate), and restore new variables - # on creation if a checkpoint exists. - checkpoint.restore(tf.train.latest_checkpoint(FLAGS.logdir)) - sys.stderr.write("learning_rate=%f\n" % learning_rate.numpy()) - - best_loss = None - for _ in range(FLAGS.epoch): - train(model, optimizer, train_data, FLAGS.seq_len, FLAGS.clip) - eval_loss = evaluate(model, eval_data) - if not best_loss or eval_loss < best_loss: - if FLAGS.logdir: - checkpoint.save(os.path.join(FLAGS.logdir, "ckpt")) - best_loss = eval_loss - else: - learning_rate.assign(learning_rate / 4.0) - sys.stderr.write("eval_loss did not reduce in this epoch, " - "changing learning rate to %f for the next epoch\n" % - learning_rate.numpy()) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--data-path", - type=str, - default="", - help="Data directory of the Penn Treebank dataset from " - "http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz") - parser.add_argument( - "--logdir", type=str, default="", help="Directory for checkpoint.") - parser.add_argument("--epoch", type=int, default=20, help="Number of epochs.") - parser.add_argument("--batch-size", type=int, default=20, help="Batch size.") - parser.add_argument( - "--seq-len", type=int, default=35, help="Sequence length.") - parser.add_argument( - "--embedding-dim", type=int, default=200, help="Embedding dimension.") - parser.add_argument( - "--hidden-dim", type=int, default=200, help="Hidden layer dimension.") - parser.add_argument( - "--num-layers", type=int, default=2, help="Number of RNN layers.") - parser.add_argument( - "--dropout", type=float, default=0.2, help="Drop out ratio.") - parser.add_argument( - "--clip", type=float, default=0.25, help="Gradient clipping ratio.") - parser.add_argument( - "--no-use-cudnn-rnn", - action="store_true", - default=False, - help="Disable the fast CuDNN RNN (when no gpu)") - - FLAGS, unparsed = parser.parse_known_args() - tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py deleted file mode 100644 index 770484abed9..00000000000 --- a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for PTBModel used for graph construction.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import gc -import time - -import numpy as np -import tensorflow as tf - -from tensorflow.contrib.eager.python.examples.rnn_ptb import rnn_ptb - - -class PTBTest(tf.test.TestCase): - - def testTrain(self): - batch_size = 20 - sequence_length = 35 - with tf.Graph().as_default(), tf.device(tf.test.gpu_device_name()): - inputs_ph = tf.placeholder(tf.int64, [sequence_length, batch_size], - "inputs") - labels_ph = tf.placeholder(tf.int64, [sequence_length, batch_size], - "labels") - - inputs = np.ones(inputs_ph.shape.as_list(), dtype=np.int64) - labels = np.ones(labels_ph.shape.as_list(), dtype=np.int64) - - model = rnn_ptb.test_model(tf.test.is_gpu_available()) - optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) - loss = rnn_ptb.loss_fn(model, inputs_ph, labels_ph, training=True) - grads = rnn_ptb.clip_gradients(optimizer.compute_gradients(loss), 0.25) - train_op = optimizer.apply_gradients(grads) - - with tf.Session() as sess: - sess.run(tf.global_variables_initializer()) - sess.run(train_op, feed_dict={inputs_ph: inputs, labels_ph: labels}) - sess.run( - [train_op, loss], feed_dict={ - inputs_ph: inputs, - labels_ph: labels - }) - - -class PTBBenchmark(tf.test.Benchmark): - - BATCH_SIZE = 20 - SEQ_LEN = 35 - - def _report(self, label, start, num_iters, device, batch_size): - wall_time = (time.time() - start) / num_iters - dev = "cpu" if "cpu" in device.lower() else "gpu" - name = "%s_%s_batch_%d" % (label, dev, batch_size) - examples_per_sec = batch_size / wall_time - self.report_benchmark( - iters=num_iters, - wall_time=wall_time, - name=name, - extras={ - "examples_per_sec": examples_per_sec - }) - - def _benchmark_apply(self, label, model): - num_iters = 100 - num_warmup = 10 - dataset = tf.data.Dataset.from_tensors( - tf.ones( - [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE], - dtype=tf.int64)).repeat(num_iters + num_warmup) - inputs = tf.compat.v1.data.make_one_shot_iterator(dataset).get_next() - - with tf.device(tf.test.gpu_device_name()): - outputs = model(inputs, training=True) - - with tf.Session() as sess: - sess.run(tf.global_variables_initializer()) - for _ in range(num_warmup): - sess.run(outputs) - gc.collect() - - start = time.time() - for _ in range(num_iters): - sess.run(outputs) - self._report(label, start, num_iters, - tf.test.gpu_device_name(), PTBBenchmark.BATCH_SIZE) - - def benchmark_apply_small(self): - self._benchmark_apply("graph_apply_small", rnn_ptb.small_model(False)) - - def benchmark_apply_large(self): - self._benchmark_apply("graph_apply_large", rnn_ptb.large_model(False)) - - def benchmark_cudnn_apply_small(self): - if not tf.test.is_gpu_available(): - return - self._benchmark_apply("graph_cudnn_apply_small", rnn_ptb.small_model(True)) - - def benchmark_cudnn_apply_large(self): - if not tf.test.is_gpu_available(): - return - self._benchmark_apply("graph_cudnn_apply_large", rnn_ptb.large_model(True)) - - def _benchmark_train(self, label, model): - num_iters = 100 - num_warmup = 10 - dataset = tf.data.Dataset.from_tensors( - tf.ones( - [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE], - dtype=tf.int64)).repeat(num_iters + num_warmup) - # inputs and labels have the same shape - dataset = tf.data.Dataset.zip((dataset, dataset)) - (inputs, labels) = tf.compat.v1.data.make_one_shot_iterator( - dataset).get_next() - - with tf.device(tf.test.gpu_device_name()): - optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) - loss = rnn_ptb.loss_fn(model, inputs, labels, training=True) - grads = rnn_ptb.clip_gradients(optimizer.compute_gradients(loss), 0.25) - train_op = optimizer.apply_gradients(grads) - - with tf.Session() as sess: - sess.run(tf.global_variables_initializer()) - for _ in range(num_warmup): - sess.run(train_op) - gc.collect() - start = time.time() - for _ in range(num_iters): - sess.run(train_op) - self._report(label, start, num_iters, - tf.test.gpu_device_name(), PTBBenchmark.BATCH_SIZE) - - def benchmark_train_small(self): - self._benchmark_train("graph_train_small", rnn_ptb.small_model(False)) - - def benchmark_train_large(self): - self._benchmark_train("graph_train_large", rnn_ptb.large_model(False)) - - def benchmark_cudnn_train_small(self): - if not tf.test.is_gpu_available(): - return - self._benchmark_train("graph_cudnn_train_small", rnn_ptb.small_model(True)) - - def benchmark_cudnn_train_large(self): - if not tf.test.is_gpu_available(): - return - self._benchmark_train("graph_cudnn_train_large", rnn_ptb.large_model(True)) - - -if __name__ == "__main__": - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py deleted file mode 100644 index b279bc4a7c3..00000000000 --- a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for PTBModel with eager execution enabled.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import gc -import time - -import numpy as np -import tensorflow as tf - -from tensorflow.contrib.eager.python import tfe -from tensorflow.contrib.eager.python.examples.rnn_ptb import rnn_ptb - - -def device(): - return "/device:GPU:0" if tfe.num_gpus() else "/device:CPU:0" - - -class PTBTest(tf.test.TestCase): - - def testTrain(self): - model = rnn_ptb.test_model(tfe.num_gpus() > 0) - sequence_length = 35 - data = np.ones([4 * sequence_length, 20], dtype=np.int64) - with tf.device(device()): - optimizer = tf.train.GradientDescentOptimizer(1.0) - # Train two epochs - rnn_ptb.train(model, optimizer, data, sequence_length, 0.25) - rnn_ptb.train(model, optimizer, data, sequence_length, 0.25) - - def testApply(self): - model = rnn_ptb.test_model(tfe.num_gpus() > 0) - with tf.device(device()): - model(tf.ones([35, 20], dtype=tf.int64), training=False) - - -def force_gpu_sync(): - if tfe.num_gpus(): - tf.constant(1).gpu().cpu() - - -class PTBBenchmark(tf.test.Benchmark): - - BATCH_SIZE = 20 - SEQ_LEN = 35 - - def _report(self, label, start, num_iters, dev, batch_size): - wall_time = (time.time() - start) / num_iters - dev = "cpu" if "cpu" in dev.lower() else "gpu" - name = "%s_%s_batch_%d" % (label, dev, batch_size) - examples_per_sec = batch_size / wall_time - self.report_benchmark( - iters=num_iters, - wall_time=wall_time, - name=name, - extras={ - "examples_per_sec": examples_per_sec - }) - - def _benchmark_apply(self, label, model): - with tf.device(device()): - sequence_batch = tf.ones( - [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE], dtype=tf.int64) - - for _ in range(10): # Warmup - model(sequence_batch, training=False).cpu() - gc.collect() - - start = time.time() - iters = 100 - for _ in range(iters): - model(sequence_batch, training=False).cpu() - self._report(label, start, iters, device(), int(sequence_batch.shape[1])) - - def benchmark_apply_small(self): - self._benchmark_apply("eager_apply_small", rnn_ptb.small_model(False)) - - def benchmark_apply_large(self): - self._benchmark_apply("eager_apply_large", rnn_ptb.large_model(False)) - - def benchmark_cudnn_apply_small(self): - if not tfe.num_gpus(): - return - self._benchmark_apply("eager_cudnn_apply_small", rnn_ptb.small_model(True)) - - def benchmark_cudnn_apply_large(self): - if not tfe.num_gpus(): - return - self._benchmark_apply("eager_cudnn_apply_large", rnn_ptb.large_model(True)) - - def _benchmark_train(self, label, model): - with tf.device(device()): - optimizer = tf.train.GradientDescentOptimizer(1.) - - def model_loss(inputs, targets): - return rnn_ptb.loss_fn(model, inputs, targets, training=True) - - grads = tfe.implicit_gradients(model_loss) - - sequence_batch = tf.ones( - [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE], dtype=tf.int64) - - def step(): - optimizer.apply_gradients( - rnn_ptb.clip_gradients(grads(sequence_batch, sequence_batch), 0.25)) - - for _ in range(10): # Warmup - step() - force_gpu_sync() - gc.collect() - - start = time.time() - iters = 100 - for _ in range(iters): - step() - force_gpu_sync() - self._report(label, start, iters, device(), int(sequence_batch.shape[1])) - - def benchmark_train_small(self): - self._benchmark_train("eager_train_small", rnn_ptb.small_model(False)) - - def benchmark_train_large(self): - self._benchmark_train("eager_train_large", rnn_ptb.large_model(False)) - - def benchmark_cudnn_train_small(self): - if not tfe.num_gpus(): - return - self._benchmark_train("eager_cudnn_train_small", rnn_ptb.small_model(True)) - - def benchmark_cudnn_train_large(self): - if not tfe.num_gpus(): - return - self._benchmark_train("eager_cudnn_train_large", rnn_ptb.large_model(True)) - - -if __name__ == "__main__": - tfe.enable_eager_execution() - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/spinn/BUILD b/tensorflow/contrib/eager/python/examples/spinn/BUILD deleted file mode 100644 index a285c87ca50..00000000000 --- a/tensorflow/contrib/eager/python/examples/spinn/BUILD +++ /dev/null @@ -1,49 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "data", - srcs = ["data.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = ["//third_party/py/numpy"], -) - -py_test( - name = "data_test", - size = "small", - srcs = ["data_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":data", - "//tensorflow:tensorflow_py", - ], -) - -cuda_py_test( - name = "spinn_test", - size = "medium", - srcs = ["spinn_test.py"], - additional_deps = [ - ":data", - "//third_party/examples/eager/spinn", - "//third_party/py/numpy", - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/summary:summary_test_util", - "//tensorflow/python/eager:test", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - ], - tags = [ - "no-internal-py3", # flaky - "no_cuda_on_cpu_tap", - "no_pip", # because spinn.py is under third_party/. - "oss_serial", - ], -) diff --git a/tensorflow/contrib/eager/python/examples/spinn/README.md b/tensorflow/contrib/eager/python/examples/spinn/README.md deleted file mode 100644 index eb0637df473..00000000000 --- a/tensorflow/contrib/eager/python/examples/spinn/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# SPINN: Dynamic neural network with TensorFlow eager execution - -This directory contains files supporting the -[spinn.py model in third_party/examples/eager/spinn/](../../../../../../third_party/examples/eager/spinn/spinn.py), -including - -- `data.py`: Utility library for loading and preprocessing the SNLI and GloVe - data. -- `data_test.py` and `spinn_test.py`: Unit tests for the data and model modules. - -See the [README.md in third_party/examples/eager/spinn/](../../../../../../third_party/examples/eager/spinn/README.md) -for detailed background, license and usage information regarding the SPINN code. - diff --git a/tensorflow/contrib/eager/python/examples/spinn/data.py b/tensorflow/contrib/eager/python/examples/spinn/data.py deleted file mode 100644 index 72d23630cd9..00000000000 --- a/tensorflow/contrib/eager/python/examples/spinn/data.py +++ /dev/null @@ -1,373 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities of SNLI data and GloVe word vectors for SPINN model. - -See more details about the SNLI data set at: - https://nlp.stanford.edu/projects/snli/ - -See more details about the GloVe pretrained word embeddings at: - https://nlp.stanford.edu/projects/glove/ -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import glob -import math -import os -import random - -import numpy as np - -POSSIBLE_LABELS = ("entailment", "contradiction", "neutral") - -UNK_CODE = 0 # Code for unknown word tokens. -PAD_CODE = 1 # Code for padding tokens. - -SHIFT_CODE = 3 -REDUCE_CODE = 2 - -WORD_VECTOR_LEN = 300 # Embedding dimensions. - -LEFT_PAREN = "(" -RIGHT_PAREN = ")" -PARENTHESES = (LEFT_PAREN, RIGHT_PAREN) - - -def get_non_parenthesis_words(items): - """Get the non-parenthesis items from a SNLI parsed sentence. - - Args: - items: Data items from a parsed SNLI sentence, with parentheses. E.g., - ["(", "Man", "(", "(", "(", "(", "(", "wearing", "pass", ")", ... - - Returns: - A list of non-parentheses word items, all converted to lower case. E.g., - ["man", "wearing", "pass", ... - """ - return [x.lower() for x in items if x not in PARENTHESES and x] - - -def get_shift_reduce(items): - """Obtain shift-reduce vector from a list of items from the SNLI data. - - Args: - items: Data items as a list of str, e.g., - ["(", "Man", "(", "(", "(", "(", "(", "wearing", "pass", ")", ... - - Returns: - A list of shift-reduce transitions, encoded as `SHIFT_CODE` for shift and - `REDUCE_CODE` for reduce. See code above for the values of `SHIFT_CODE` - and `REDUCE_CODE`. - """ - trans = [] - for item in items: - if item == LEFT_PAREN: - continue - elif item == RIGHT_PAREN: - trans.append(REDUCE_CODE) - else: - trans.append(SHIFT_CODE) - return trans - - -def pad_and_reverse_word_ids(sentences): - """Pad a list of sentences to the common maximum length + 1. - - Args: - sentences: A list of sentences as a list of list of integers. Each integer - is a word ID. Each list of integer corresponds to one sentence. - - Returns: - A numpy.ndarray of shape (num_sentences, max_length + 1), wherein max_length - is the maximum sentence length (in # of words). Each sentence is reversed - and then padded with an extra one at head, as required by the model. - """ - max_len = max(len(sent) for sent in sentences) - for sent in sentences: - if len(sent) < max_len: - sent.extend([PAD_CODE] * (max_len - len(sent))) - # Reverse in time order and pad an extra one. - sentences = np.fliplr(np.array(sentences, dtype=np.int64)) - sentences = np.concatenate( - [np.ones([sentences.shape[0], 1], dtype=np.int64), sentences], axis=1) - return sentences - - -def pad_transitions(sentences_transitions): - """Pad a list of shift-reduce transitions to the maximum length.""" - max_len = max(len(transitions) for transitions in sentences_transitions) - for transitions in sentences_transitions: - if len(transitions) < max_len: - transitions.extend([PAD_CODE] * (max_len - len(transitions))) - return np.array(sentences_transitions, dtype=np.int64) - - -def load_vocabulary(data_root): - """Load vocabulary from SNLI data files. - - Args: - data_root: Root directory of the data. It is assumed that the SNLI data - files have been downloaded and extracted to the "snli/snli_1.0" - subdirectory of it. - - Returns: - Vocabulary as a set of strings. - - Raises: - ValueError: If SNLI data files cannot be found. - """ - snli_path = os.path.join(data_root, "snli") - snli_glob_pattern = os.path.join(snli_path, "snli_1.0/snli_1.0_*.txt") - file_names = glob.glob(snli_glob_pattern) - if not file_names: - raise ValueError( - "Cannot find SNLI data files at %s. " - "Please download and extract SNLI data first." % snli_glob_pattern) - - print("Loading vocabulary...") - vocab = set() - for file_name in file_names: - with open(os.path.join(snli_path, file_name), "rt") as f: - for i, line in enumerate(f): - if i == 0: - continue - items = line.split("\t") - premise_words = get_non_parenthesis_words(items[1].split(" ")) - hypothesis_words = get_non_parenthesis_words(items[2].split(" ")) - vocab.update(premise_words) - vocab.update(hypothesis_words) - return vocab - - -def load_word_vectors(data_root, vocab): - """Load GloVe word vectors for words present in the vocabulary. - - Args: - data_root: Data root directory. It is assumed that the GloVe file - has been downloaded and extracted at the "glove/" subdirectory of it. - vocab: A `set` of words, representing the vocabulary. - - Returns: - 1. word2index: A dict from lower-case word to row index in the embedding - matrix, i.e, `embed` below. - 2. embed: The embedding matrix as a float32 numpy array. Its shape is - [vocabulary_size, WORD_VECTOR_LEN]. vocabulary_size is len(vocab). - WORD_VECTOR_LEN is the embedding dimension (300). - - Raises: - ValueError: If GloVe embedding file cannot be found. - """ - glove_path = os.path.join(data_root, "glove/glove.42B.300d.txt") - if not os.path.isfile(glove_path): - raise ValueError( - "Cannot find GloVe embedding file at %s. " - "Please download and extract GloVe embeddings first." % glove_path) - - print("Loading word vectors...") - - word2index = {} - embed = [] - - embed.append([0] * WORD_VECTOR_LEN) # - embed.append([0] * WORD_VECTOR_LEN) # - word2index[""] = UNK_CODE - word2index[""] = PAD_CODE - - with open(glove_path, "rt") as f: - for line in f: - items = line.split(" ") - word = items[0] - if word in vocab and word not in word2index: - word2index[word] = len(embed) - vector = np.array([float(item) for item in items[1:]]) - assert (WORD_VECTOR_LEN,) == vector.shape - embed.append(vector) - embed = np.array(embed, dtype=np.float32) - return word2index, embed - - -def calculate_bins(length2count, min_bin_size): - """Calculate bin boundaries given a histogram of lengths and minimum bin size. - - Args: - length2count: A `dict` mapping length to sentence count. - min_bin_size: Minimum bin size in terms of total number of sentence pairs - in the bin. - - Returns: - A `list` representing the right bin boundaries, starting from the inclusive - right boundary of the first bin. For example, if the output is - [10, 20, 35], - it means there are three bins: [1, 10], [11, 20] and [21, 35]. - """ - bounds = [] - lengths = sorted(length2count.keys()) - cum_count = 0 - for length in lengths: - cum_count += length2count[length] - if cum_count >= min_bin_size: - bounds.append(length) - cum_count = 0 - if bounds[-1] != lengths[-1]: - bounds.append(lengths[-1]) - return bounds - - -def encode_sentence(sentence, word2index): - """Encode a single sentence as word indices and shift-reduce code. - - Args: - sentence: The sentence with added binary parse information, represented as - a string, with all the word items and parentheses separated by spaces. - E.g., '( ( The dog ) ( ( is ( playing toys ) ) . ) )'. - word2index: A `dict` mapping words to their word indices. - - Returns: - 1. Word indices as a numpy array, with shape `(sequence_len, 1)`. - 2. Shift-reduce sequence as a numpy array, with shape - `(sequence_len * 2 - 3, 1)`. - """ - items = [w for w in sentence.split(" ") if w] - words = get_non_parenthesis_words(items) - shift_reduce = get_shift_reduce(items) - word_indices = pad_and_reverse_word_ids( - [[word2index.get(word, UNK_CODE) for word in words]]).T - return (word_indices, - np.expand_dims(np.array(shift_reduce, dtype=np.int64), -1)) - - -class SnliData(object): - """A split of SNLI data.""" - - def __init__(self, data_file, word2index, sentence_len_limit=-1): - """SnliData constructor. - - Args: - data_file: Full path to the data file, e.g., - "/tmp/spinn-data/snli/snli_1.0/snli_1.0.train.txt" - word2index: A dict from lower-case word to row index in the embedding - matrix (see `load_word_vectors()` for details). - sentence_len_limit: Maximum allowed sentence length (# of words). - A value of <= 0 means unlimited. Sentences longer than this limit - are currently discarded, not truncated. - """ - - self._labels = [] - self._premises = [] - self._premise_transitions = [] - self._hypotheses = [] - self._hypothesis_transitions = [] - - with open(data_file, "rt") as f: - for i, line in enumerate(f): - if i == 0: - # Skip header line. - continue - items = line.split("\t") - if items[0] not in POSSIBLE_LABELS: - continue - - premise_items = items[1].split(" ") - hypothesis_items = items[2].split(" ") - premise_words = get_non_parenthesis_words(premise_items) - hypothesis_words = get_non_parenthesis_words(hypothesis_items) - - if (sentence_len_limit > 0 and - (len(premise_words) > sentence_len_limit or - len(hypothesis_words) > sentence_len_limit)): - # TODO(cais): Maybe truncate; do not discard. - continue - - premise_ids = [ - word2index.get(word, UNK_CODE) for word in premise_words] - hypothesis_ids = [ - word2index.get(word, UNK_CODE) for word in hypothesis_words] - - self._premises.append(premise_ids) - self._hypotheses.append(hypothesis_ids) - self._premise_transitions.append(get_shift_reduce(premise_items)) - self._hypothesis_transitions.append(get_shift_reduce(hypothesis_items)) - assert (len(self._premise_transitions[-1]) == - 2 * len(premise_words) - 1) - assert (len(self._hypothesis_transitions[-1]) == - 2 * len(hypothesis_words) - 1) - - self._labels.append(POSSIBLE_LABELS.index(items[0]) + 1) - - assert len(self._labels) == len(self._premises) - assert len(self._labels) == len(self._hypotheses) - assert len(self._labels) == len(self._premise_transitions) - assert len(self._labels) == len(self._hypothesis_transitions) - - def num_batches(self, batch_size): - """Calculate number of batches given batch size.""" - return int(math.ceil(len(self._labels) / batch_size)) - - def get_generator(self, batch_size): - """Obtain a generator for batched data. - - All examples of this SnliData object are randomly shuffled, sorted - according to the maximum sentence length of the premise and hypothesis - sentences in the pair, and batched. - - Args: - batch_size: Desired batch size. - - Returns: - A generator for data batches. The generator yields a 5-tuple: - label: An array of the shape (batch_size,). - premise: An array of the shape (max_premise_len, batch_size), wherein - max_premise_len is the maximum length of the (padded) premise - sentence in the batch. - premise_transitions: An array of the shape (2 * max_premise_len -3, - batch_size). - hypothesis: Same as `premise`, but for hypothesis sentences. - hypothesis_transitions: Same as `premise_transitions`, but for - hypothesis sentences. - All the elements of the 5-tuple have dtype `int64`. - """ - # Randomly shuffle examples. - zipped = list(zip( - self._labels, self._premises, self._premise_transitions, - self._hypotheses, self._hypothesis_transitions)) - random.shuffle(zipped) - # Then sort the examples by maximum of the premise and hypothesis sentence - # lengths in the pair. During training, the batches are expected to be - # shuffled. So it is okay to leave them sorted by max length here. - (labels, premises, premise_transitions, hypotheses, - hypothesis_transitions) = zip( - *sorted(zipped, key=lambda x: max(len(x[1]), len(x[3])))) - - def _generator(): - begin = 0 - while begin < len(labels): - # The sorting above and the batching here makes sure that sentences of - # similar max lengths are batched together, minimizing the inefficiency - # due to uneven max lengths. The sentences are batched differently in - # each call to get_generator() due to the shuffling before sorting - # above. The pad_and_reverse_word_ids() and pad_transitions() functions - # take care of any remaining unevenness of the max sentence lengths. - end = min(begin + batch_size, len(labels)) - # Transpose, because the SPINN model requires time-major, instead of - # batch-major. - yield (labels[begin:end], - pad_and_reverse_word_ids(premises[begin:end]).T, - pad_transitions(premise_transitions[begin:end]).T, - pad_and_reverse_word_ids(hypotheses[begin:end]).T, - pad_transitions(hypothesis_transitions[begin:end]).T) - begin = end - return _generator diff --git a/tensorflow/contrib/eager/python/examples/spinn/data_test.py b/tensorflow/contrib/eager/python/examples/spinn/data_test.py deleted file mode 100644 index 54fef2c3fe4..00000000000 --- a/tensorflow/contrib/eager/python/examples/spinn/data_test.py +++ /dev/null @@ -1,270 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Unit tests for SPINN data module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import shutil -import tempfile - -import numpy as np -import tensorflow as tf - -from tensorflow.contrib.eager.python.examples.spinn import data - - -class DataTest(tf.test.TestCase): - - def setUp(self): - super(DataTest, self).setUp() - self._temp_data_dir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self._temp_data_dir) - super(DataTest, self).tearDown() - - def testGenNonParenthesisWords(self): - seq_with_parse = ( - "( Man ( ( ( ( ( wearing pass ) ( on ( a lanyard ) ) ) and " - ") ( standing ( in ( ( a crowd ) ( of people ) ) ) ) ) . ) )") - self.assertEqual( - ["man", "wearing", "pass", "on", "a", "lanyard", "and", "standing", - "in", "a", "crowd", "of", "people", "."], - data.get_non_parenthesis_words(seq_with_parse.split(" "))) - - def testGetShiftReduce(self): - seq_with_parse = ( - "( Man ( ( ( ( ( wearing pass ) ( on ( a lanyard ) ) ) and " - ") ( standing ( in ( ( a crowd ) ( of people ) ) ) ) ) . ) )") - self.assertEqual( - [3, 3, 3, 2, 3, 3, 3, 2, 2, 2, 3, 2, 3, 3, 3, 3, 2, 3, 3, 2, 2, 2, 2, 2, - 3, 2, 2], data.get_shift_reduce(seq_with_parse.split(" "))) - - def testPadAndReverseWordIds(self): - id_sequences = [[0, 2, 3, 4, 5], - [6, 7, 8], - [9, 10, 11, 12, 13, 14, 15, 16]] - self.assertAllClose( - [[1, 1, 1, 1, 5, 4, 3, 2, 0], - [1, 1, 1, 1, 1, 1, 8, 7, 6], - [1, 16, 15, 14, 13, 12, 11, 10, 9]], - data.pad_and_reverse_word_ids(id_sequences)) - - def testPadTransitions(self): - unpadded = [[3, 3, 3, 2, 2, 2, 2], - [3, 3, 2, 2, 2]] - self.assertAllClose( - [[3, 3, 3, 2, 2, 2, 2], - [3, 3, 2, 2, 2, 1, 1]], - data.pad_transitions(unpadded)) - - def testCalculateBins(self): - length2count = { - 1: 10, - 2: 15, - 3: 25, - 4: 40, - 5: 35, - 6: 10} - self.assertEqual([2, 3, 4, 5, 6], - data.calculate_bins(length2count, 20)) - self.assertEqual([3, 4, 6], data.calculate_bins(length2count, 40)) - self.assertEqual([4, 6], data.calculate_bins(length2count, 60)) - - def testLoadVoacbulary(self): - snli_1_0_dir = os.path.join(self._temp_data_dir, "snli/snli_1.0") - fake_train_file = os.path.join(snli_1_0_dir, "snli_1.0_train.txt") - fake_dev_file = os.path.join(snli_1_0_dir, "snli_1.0_dev.txt") - os.makedirs(snli_1_0_dir) - - with open(fake_train_file, "wt") as f: - f.write("gold_label\tsentence1_binary_parse\tsentence2_binary_parse\t" - "sentence1_parse\tsentence2_parse\tsentence1\tsentence2\t" - "captionID\tpairID\tlabel1\tlabel2\tlabel3\tlabel4\tlabel5\n") - f.write("neutral\t( ( Foo bar ) . )\t( ( foo baz ) . )\t" - "DummySentence1Parse\tDummySentence2Parse\t" - "Foo bar.\tfoo baz.\t" - "4705552913.jpg#2\t4705552913.jpg#2r1n\t" - "neutral\tentailment\tneutral\tneutral\tneutral\n") - with open(fake_dev_file, "wt") as f: - f.write("gold_label\tsentence1_binary_parse\tsentence2_binary_parse\t" - "sentence1_parse\tsentence2_parse\tsentence1\tsentence2\t" - "captionID\tpairID\tlabel1\tlabel2\tlabel3\tlabel4\tlabel5\n") - f.write("neutral\t( ( Quux quuz ) ? )\t( ( Corge grault ) ! )\t" - "DummySentence1Parse\tDummySentence2Parse\t" - "Quux quuz?\t.Corge grault!\t" - "4705552913.jpg#2\t4705552913.jpg#2r1n\t" - "neutral\tentailment\tneutral\tneutral\tneutral\n") - - vocab = data.load_vocabulary(self._temp_data_dir) - self.assertSetEqual( - {".", "?", "!", "foo", "bar", "baz", "quux", "quuz", "corge", "grault"}, - vocab) - - def testLoadVoacbularyWithoutFileRaisesError(self): - with self.assertRaisesRegexp(ValueError, "Cannot find SNLI data files at"): - data.load_vocabulary(self._temp_data_dir) - - os.makedirs(os.path.join(self._temp_data_dir, "snli")) - with self.assertRaisesRegexp(ValueError, "Cannot find SNLI data files at"): - data.load_vocabulary(self._temp_data_dir) - - os.makedirs(os.path.join(self._temp_data_dir, "snli/snli_1.0")) - with self.assertRaisesRegexp(ValueError, "Cannot find SNLI data files at"): - data.load_vocabulary(self._temp_data_dir) - - def testLoadWordVectors(self): - glove_dir = os.path.join(self._temp_data_dir, "glove") - os.makedirs(glove_dir) - glove_file = os.path.join(glove_dir, "glove.42B.300d.txt") - - words = [".", ",", "foo", "bar", "baz"] - with open(glove_file, "wt") as f: - for i, word in enumerate(words): - f.write("%s " % word) - for j in range(data.WORD_VECTOR_LEN): - f.write("%.5f" % (i * 0.1)) - if j < data.WORD_VECTOR_LEN - 1: - f.write(" ") - else: - f.write("\n") - - vocab = {"foo", "bar", "baz", "qux", "."} - # Notice that "qux" is not present in `words`. - word2index, embed = data.load_word_vectors(self._temp_data_dir, vocab) - - self.assertEqual(6, len(word2index)) - self.assertEqual(0, word2index[""]) - self.assertEqual(1, word2index[""]) - self.assertEqual(2, word2index["."]) - self.assertEqual(3, word2index["foo"]) - self.assertEqual(4, word2index["bar"]) - self.assertEqual(5, word2index["baz"]) - self.assertEqual((6, data.WORD_VECTOR_LEN), embed.shape) - self.assertAllClose([0.0] * data.WORD_VECTOR_LEN, embed[0, :]) - self.assertAllClose([0.0] * data.WORD_VECTOR_LEN, embed[1, :]) - self.assertAllClose([0.0] * data.WORD_VECTOR_LEN, embed[2, :]) - self.assertAllClose([0.2] * data.WORD_VECTOR_LEN, embed[3, :]) - self.assertAllClose([0.3] * data.WORD_VECTOR_LEN, embed[4, :]) - self.assertAllClose([0.4] * data.WORD_VECTOR_LEN, embed[5, :]) - - def testLoadWordVectorsWithoutFileRaisesError(self): - vocab = {"foo", "bar", "baz", "qux", "."} - with self.assertRaisesRegexp( - ValueError, "Cannot find GloVe embedding file at"): - data.load_word_vectors(self._temp_data_dir, vocab) - - os.makedirs(os.path.join(self._temp_data_dir, "glove")) - with self.assertRaisesRegexp( - ValueError, "Cannot find GloVe embedding file at"): - data.load_word_vectors(self._temp_data_dir, vocab) - - def _createFakeSnliData(self, fake_snli_file): - # Four sentences in total. - with open(fake_snli_file, "wt") as f: - f.write("gold_label\tsentence1_binary_parse\tsentence2_binary_parse\t" - "sentence1_parse\tsentence2_parse\tsentence1\tsentence2\t" - "captionID\tpairID\tlabel1\tlabel2\tlabel3\tlabel4\tlabel5\n") - f.write("neutral\t( ( Foo bar ) . )\t( ( foo . )\t" - "DummySentence1Parse\tDummySentence2Parse\t" - "Foo bar.\tfoo baz.\t" - "4705552913.jpg#2\t4705552913.jpg#2r1n\t" - "neutral\tentailment\tneutral\tneutral\tneutral\n") - f.write("contradiction\t( ( Bar foo ) . )\t( ( baz . )\t" - "DummySentence1Parse\tDummySentence2Parse\t" - "Foo bar.\tfoo baz.\t" - "4705552913.jpg#2\t4705552913.jpg#2r1n\t" - "neutral\tentailment\tneutral\tneutral\tneutral\n") - f.write("entailment\t( ( Quux quuz ) . )\t( ( grault . )\t" - "DummySentence1Parse\tDummySentence2Parse\t" - "Foo bar.\tfoo baz.\t" - "4705552913.jpg#2\t4705552913.jpg#2r1n\t" - "neutral\tentailment\tneutral\tneutral\tneutral\n") - f.write("entailment\t( ( Quuz quux ) . )\t( ( garply . )\t" - "DummySentence1Parse\tDummySentence2Parse\t" - "Foo bar.\tfoo baz.\t" - "4705552913.jpg#2\t4705552913.jpg#2r1n\t" - "neutral\tentailment\tneutral\tneutral\tneutral\n") - - def _createFakeGloveData(self, glove_file): - words = [".", "foo", "bar", "baz", "quux", "quuz", "grault", "garply"] - with open(glove_file, "wt") as f: - for i, word in enumerate(words): - f.write("%s " % word) - for j in range(data.WORD_VECTOR_LEN): - f.write("%.5f" % (i * 0.1)) - if j < data.WORD_VECTOR_LEN - 1: - f.write(" ") - else: - f.write("\n") - - def testEncodeSingleSentence(self): - snli_1_0_dir = os.path.join(self._temp_data_dir, "snli/snli_1.0") - fake_train_file = os.path.join(snli_1_0_dir, "snli_1.0_train.txt") - os.makedirs(snli_1_0_dir) - self._createFakeSnliData(fake_train_file) - vocab = data.load_vocabulary(self._temp_data_dir) - glove_dir = os.path.join(self._temp_data_dir, "glove") - os.makedirs(glove_dir) - glove_file = os.path.join(glove_dir, "glove.42B.300d.txt") - self._createFakeGloveData(glove_file) - word2index, _ = data.load_word_vectors(self._temp_data_dir, vocab) - - sentence_variants = [ - "( Foo ( ( bar baz ) . ) )", - " ( Foo ( ( bar baz ) . ) ) ", - "( Foo ( ( bar baz ) . ) )"] - for sentence in sentence_variants: - word_indices, shift_reduce = data.encode_sentence(sentence, word2index) - self.assertEqual(np.int64, word_indices.dtype) - self.assertEqual((5, 1), word_indices.shape) - self.assertAllClose( - np.array([[3, 3, 3, 2, 3, 2, 2]], dtype=np.int64).T, shift_reduce) - - def testSnliData(self): - snli_1_0_dir = os.path.join(self._temp_data_dir, "snli/snli_1.0") - fake_train_file = os.path.join(snli_1_0_dir, "snli_1.0_train.txt") - os.makedirs(snli_1_0_dir) - self._createFakeSnliData(fake_train_file) - - glove_dir = os.path.join(self._temp_data_dir, "glove") - os.makedirs(glove_dir) - glove_file = os.path.join(glove_dir, "glove.42B.300d.txt") - self._createFakeGloveData(glove_file) - - vocab = data.load_vocabulary(self._temp_data_dir) - word2index, _ = data.load_word_vectors(self._temp_data_dir, vocab) - - train_data = data.SnliData(fake_train_file, word2index) - self.assertEqual(4, train_data.num_batches(1)) - self.assertEqual(2, train_data.num_batches(2)) - self.assertEqual(2, train_data.num_batches(3)) - self.assertEqual(1, train_data.num_batches(4)) - - generator = train_data.get_generator(2)() - for _ in range(2): - label, prem, prem_trans, hypo, hypo_trans = next(generator) - self.assertEqual(2, len(label)) - self.assertEqual((4, 2), prem.shape) - self.assertEqual((5, 2), prem_trans.shape) - self.assertEqual((3, 2), hypo.shape) - self.assertEqual((3, 2), hypo_trans.shape) - - -if __name__ == "__main__": - tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py deleted file mode 100644 index 6e47cc57051..00000000000 --- a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py +++ /dev/null @@ -1,478 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import gc -import glob -import os -import shutil -import tempfile -import time - -import numpy as np -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow as tf - -# pylint: disable=g-bad-import-order -from tensorflow.contrib.eager.python.examples.spinn import data -from third_party.examples.eager.spinn import spinn -from tensorflow.contrib.summary import summary_test_util -from tensorflow.python.eager import test -from tensorflow.python.framework import test_util -from tensorflow.python.training import checkpoint_management -from tensorflow.python.training.tracking import util as trackable_utils -# pylint: enable=g-bad-import-order - - -def _generate_synthetic_snli_data_batch(sequence_length, - batch_size, - vocab_size): - """Generate a fake batch of SNLI data for testing.""" - with tf.device("cpu:0"): - labels = tf.random_uniform([batch_size], minval=1, maxval=4, dtype=tf.int64) - prem = tf.random_uniform( - (sequence_length, batch_size), maxval=vocab_size, dtype=tf.int64) - prem_trans = tf.constant(np.array( - [[3, 3, 2, 3, 3, 3, 2, 2, 2, 3, 3, 3, - 2, 3, 3, 2, 2, 3, 3, 3, 2, 2, 2, 2, - 3, 2, 2]] * batch_size, dtype=np.int64).T) - hypo = tf.random_uniform( - (sequence_length, batch_size), maxval=vocab_size, dtype=tf.int64) - hypo_trans = tf.constant(np.array( - [[3, 3, 2, 3, 3, 3, 2, 2, 2, 3, 3, 3, - 2, 3, 3, 2, 2, 3, 3, 3, 2, 2, 2, 2, - 3, 2, 2]] * batch_size, dtype=np.int64).T) - if test_util.is_gpu_available(): - labels = labels.gpu() - prem = prem.gpu() - prem_trans = prem_trans.gpu() - hypo = hypo.gpu() - hypo_trans = hypo_trans.gpu() - return labels, prem, prem_trans, hypo, hypo_trans - - -def _test_spinn_config(d_embed, d_out, logdir=None, inference_sentences=None): - """Generate a config tuple for testing. - - Args: - d_embed: Embedding dimensions. - d_out: Model output dimensions. - logdir: Optional logdir. - inference_sentences: A 2-tuple of strings representing the sentences (with - binary parsing result), e.g., - ("( ( The dog ) ( ( is running ) . ) )", "( ( The dog ) ( moves . ) )"). - - Returns: - A config tuple. - """ - config_tuple = collections.namedtuple( - "Config", ["d_hidden", "d_proj", "d_tracker", "predict", - "embed_dropout", "mlp_dropout", "n_mlp_layers", "d_mlp", - "d_out", "projection", "lr", "batch_size", "epochs", - "force_cpu", "logdir", "log_every", "dev_every", "save_every", - "lr_decay_every", "lr_decay_by", "inference_premise", - "inference_hypothesis"]) - - inference_premise = inference_sentences[0] if inference_sentences else None - inference_hypothesis = inference_sentences[1] if inference_sentences else None - return config_tuple( - d_hidden=d_embed, - d_proj=d_embed * 2, - d_tracker=8, - predict=False, - embed_dropout=0.1, - mlp_dropout=0.1, - n_mlp_layers=2, - d_mlp=32, - d_out=d_out, - projection=True, - lr=2e-2, - batch_size=2, - epochs=20, - force_cpu=False, - logdir=logdir, - log_every=1, - dev_every=2, - save_every=2, - lr_decay_every=1, - lr_decay_by=0.75, - inference_premise=inference_premise, - inference_hypothesis=inference_hypothesis) - - -class SpinnTest(test_util.TensorFlowTestCase): - - def setUp(self): - super(SpinnTest, self).setUp() - self._test_device = "gpu:0" if test_util.is_gpu_available() else "cpu:0" - self._temp_data_dir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self._temp_data_dir) - super(SpinnTest, self).tearDown() - - def testBundle(self): - with tf.device(self._test_device): - lstm_iter = [np.array([[0, 1], [2, 3]], dtype=np.float32), - np.array([[0, -1], [-2, -3]], dtype=np.float32), - np.array([[0, 2], [4, 6]], dtype=np.float32), - np.array([[0, -2], [-4, -6]], dtype=np.float32)] - out = spinn._bundle(lstm_iter) - - self.assertEqual(2, len(out)) - self.assertEqual(tf.float32, out[0].dtype) - self.assertEqual(tf.float32, out[1].dtype) - self.assertAllEqual(np.array([[0, 2, 0, -2, 0, 4, 0, -4]]).T, - out[0].numpy()) - self.assertAllEqual(np.array([[1, 3, -1, -3, 2, 6, -2, -6]]).T, - out[1].numpy()) - - def testUnbunbdle(self): - with tf.device(self._test_device): - state = [np.array([[0, 1, 2], [3, 4, 5]], dtype=np.float32), - np.array([[0, -1, -2], [-3, -4, -5]], dtype=np.float32)] - out = spinn._unbundle(state) - - self.assertEqual(2, len(out)) - self.assertEqual(tf.float32, out[0].dtype) - self.assertEqual(tf.float32, out[1].dtype) - self.assertAllEqual(np.array([[0, 1, 2, 0, -1, -2]]), - out[0].numpy()) - self.assertAllEqual(np.array([[3, 4, 5, -3, -4, -5]]), - out[1].numpy()) - - def testReducer(self): - with tf.device(self._test_device): - batch_size = 3 - size = 10 - tracker_size = 8 - reducer = spinn.Reducer(size, tracker_size=tracker_size) - - left_in = [] - right_in = [] - tracking = [] - for _ in range(batch_size): - left_in.append(tf.random_normal((1, size * 2))) - right_in.append(tf.random_normal((1, size * 2))) - tracking.append(tf.random_normal((1, tracker_size * 2))) - - out = reducer(left_in, right_in, tracking=tracking) - self.assertEqual(batch_size, len(out)) - self.assertEqual(tf.float32, out[0].dtype) - self.assertEqual((1, size * 2), out[0].shape) - - def testReduceTreeLSTM(self): - with tf.device(self._test_device): - size = 10 - tracker_size = 8 - reducer = spinn.Reducer(size, tracker_size=tracker_size) - - lstm_in = np.array([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]], - dtype=np.float32) - c1 = np.array([[0, 1], [2, 3]], dtype=np.float32) - c2 = np.array([[0, -1], [-2, -3]], dtype=np.float32) - - h, c = reducer._tree_lstm(c1, c2, lstm_in) - self.assertEqual(tf.float32, h.dtype) - self.assertEqual(tf.float32, c.dtype) - self.assertEqual((2, 2), h.shape) - self.assertEqual((2, 2), c.shape) - - def testTracker(self): - with tf.device(self._test_device): - batch_size = 2 - size = 10 - tracker_size = 8 - buffer_length = 18 - stack_size = 3 - - tracker = spinn.Tracker(tracker_size, False) - tracker.reset_state() - - # Create dummy inputs for testing. - bufs = [] - buf = [] - for _ in range(buffer_length): - buf.append(tf.random_normal((batch_size, size * 2))) - bufs.append(buf) - self.assertEqual(1, len(bufs)) - self.assertEqual(buffer_length, len(bufs[0])) - self.assertEqual((batch_size, size * 2), bufs[0][0].shape) - - stacks = [] - stack = [] - for _ in range(stack_size): - stack.append(tf.random_normal((batch_size, size * 2))) - stacks.append(stack) - self.assertEqual(1, len(stacks)) - self.assertEqual(3, len(stacks[0])) - self.assertEqual((batch_size, size * 2), stacks[0][0].shape) - - for _ in range(2): - out1, out2 = tracker(bufs, stacks) - self.assertIsNone(out2) - self.assertEqual(batch_size, len(out1)) - self.assertEqual(tf.float32, out1[0].dtype) - self.assertEqual((1, tracker_size * 2), out1[0].shape) - - self.assertEqual(tf.float32, tracker.state.c.dtype) - self.assertEqual((batch_size, tracker_size), tracker.state.c.shape) - self.assertEqual(tf.float32, tracker.state.h.dtype) - self.assertEqual((batch_size, tracker_size), tracker.state.h.shape) - - def testSPINN(self): - with tf.device(self._test_device): - embedding_dims = 10 - d_tracker = 8 - sequence_length = 15 - num_transitions = 27 - - config_tuple = collections.namedtuple( - "Config", ["d_hidden", "d_proj", "d_tracker", "predict"]) - config = config_tuple( - embedding_dims, embedding_dims * 2, d_tracker, False) - s = spinn.SPINN(config) - - # Create some fake data. - buffers = tf.random_normal((sequence_length, 1, config.d_proj)) - transitions = tf.constant( - [[3], [3], [2], [3], [3], [3], [2], [2], [2], [3], [3], [3], - [2], [3], [3], [2], [2], [3], [3], [3], [2], [2], [2], [2], - [3], [2], [2]], dtype=tf.int64) - self.assertEqual(tf.int64, transitions.dtype) - self.assertEqual((num_transitions, 1), transitions.shape) - - out = s(buffers, transitions, training=True) - self.assertEqual(tf.float32, out.dtype) - self.assertEqual((1, embedding_dims), out.shape) - - def testSNLIClassifierAndTrainer(self): - with tf.device(self._test_device): - vocab_size = 40 - batch_size = 2 - d_embed = 10 - sequence_length = 15 - d_out = 4 - - config = _test_spinn_config(d_embed, d_out) - - # Create fake embedding matrix. - embed = tf.random_normal((vocab_size, d_embed)) - - model = spinn.SNLIClassifier(config, embed) - trainer = spinn.SNLIClassifierTrainer(model, config.lr) - - (labels, prem, prem_trans, hypo, - hypo_trans) = _generate_synthetic_snli_data_batch(sequence_length, - batch_size, - vocab_size) - - # Invoke model under non-training mode. - logits = model(prem, prem_trans, hypo, hypo_trans, training=False) - self.assertEqual(tf.float32, logits.dtype) - self.assertEqual((batch_size, d_out), logits.shape) - - # Invoke model under training model. - logits = model(prem, prem_trans, hypo, hypo_trans, training=True) - self.assertEqual(tf.float32, logits.dtype) - self.assertEqual((batch_size, d_out), logits.shape) - - # Calculate loss. - loss1 = trainer.loss(labels, logits) - self.assertEqual(tf.float32, loss1.dtype) - self.assertEqual((), loss1.shape) - - loss2, logits = trainer.train_batch( - labels, prem, prem_trans, hypo, hypo_trans) - self.assertEqual(tf.float32, loss2.dtype) - self.assertEqual((), loss2.shape) - self.assertEqual(tf.float32, logits.dtype) - self.assertEqual((batch_size, d_out), logits.shape) - # Training on the batch should have led to a change in the loss value. - self.assertNotEqual(loss1.numpy(), loss2.numpy()) - - def _create_test_data(self, snli_1_0_dir): - fake_train_file = os.path.join(snli_1_0_dir, "snli_1.0_train.txt") - os.makedirs(snli_1_0_dir) - - # Four sentences in total. - with open(fake_train_file, "wt") as f: - f.write("gold_label\tsentence1_binary_parse\tsentence2_binary_parse\t" - "sentence1_parse\tsentence2_parse\tsentence1\tsentence2\t" - "captionID\tpairID\tlabel1\tlabel2\tlabel3\tlabel4\tlabel5\n") - f.write("neutral\t( ( Foo bar ) . )\t( ( foo . )\t" - "DummySentence1Parse\tDummySentence2Parse\t" - "Foo bar.\tfoo baz.\t" - "4705552913.jpg#2\t4705552913.jpg#2r1n\t" - "neutral\tentailment\tneutral\tneutral\tneutral\n") - f.write("contradiction\t( ( Bar foo ) . )\t( ( baz . )\t" - "DummySentence1Parse\tDummySentence2Parse\t" - "Foo bar.\tfoo baz.\t" - "4705552913.jpg#2\t4705552913.jpg#2r1n\t" - "neutral\tentailment\tneutral\tneutral\tneutral\n") - f.write("entailment\t( ( Quux quuz ) . )\t( ( grault . )\t" - "DummySentence1Parse\tDummySentence2Parse\t" - "Foo bar.\tfoo baz.\t" - "4705552913.jpg#2\t4705552913.jpg#2r1n\t" - "neutral\tentailment\tneutral\tneutral\tneutral\n") - f.write("entailment\t( ( Quuz quux ) . )\t( ( garply . )\t" - "DummySentence1Parse\tDummySentence2Parse\t" - "Foo bar.\tfoo baz.\t" - "4705552913.jpg#2\t4705552913.jpg#2r1n\t" - "neutral\tentailment\tneutral\tneutral\tneutral\n") - - glove_dir = os.path.join(self._temp_data_dir, "glove") - os.makedirs(glove_dir) - glove_file = os.path.join(glove_dir, "glove.42B.300d.txt") - - words = [".", "foo", "bar", "baz", "quux", "quuz", "grault", "garply"] - with open(glove_file, "wt") as f: - for i, word in enumerate(words): - f.write("%s " % word) - for j in range(data.WORD_VECTOR_LEN): - f.write("%.5f" % (i * 0.1)) - if j < data.WORD_VECTOR_LEN - 1: - f.write(" ") - else: - f.write("\n") - - return fake_train_file - - def testInferSpinnWorks(self): - """Test inference with the spinn model.""" - snli_1_0_dir = os.path.join(self._temp_data_dir, "snli/snli_1.0") - self._create_test_data(snli_1_0_dir) - - vocab = data.load_vocabulary(self._temp_data_dir) - word2index, embed = data.load_word_vectors(self._temp_data_dir, vocab) - - config = _test_spinn_config( - data.WORD_VECTOR_LEN, 4, - logdir=os.path.join(self._temp_data_dir, "logdir"), - inference_sentences=("( foo ( bar . ) )", "( bar ( foo . ) )")) - logits = spinn.train_or_infer_spinn( - embed, word2index, None, None, None, config) - self.assertEqual(tf.float32, logits.dtype) - self.assertEqual((3,), logits.shape) - - def testInferSpinnThrowsErrorIfOnlyOneSentenceIsSpecified(self): - snli_1_0_dir = os.path.join(self._temp_data_dir, "snli/snli_1.0") - self._create_test_data(snli_1_0_dir) - - vocab = data.load_vocabulary(self._temp_data_dir) - word2index, embed = data.load_word_vectors(self._temp_data_dir, vocab) - - config = _test_spinn_config( - data.WORD_VECTOR_LEN, 4, - logdir=os.path.join(self._temp_data_dir, "logdir"), - inference_sentences=("( foo ( bar . ) )", None)) - with self.assertRaises(ValueError): - spinn.train_or_infer_spinn(embed, word2index, None, None, None, config) - - def testTrainSpinn(self): - """Test with fake toy SNLI data and GloVe vectors.""" - - # 1. Create and load a fake SNLI data file and a fake GloVe embedding file. - snli_1_0_dir = os.path.join(self._temp_data_dir, "snli/snli_1.0") - fake_train_file = self._create_test_data(snli_1_0_dir) - - vocab = data.load_vocabulary(self._temp_data_dir) - word2index, embed = data.load_word_vectors(self._temp_data_dir, vocab) - - train_data = data.SnliData(fake_train_file, word2index) - dev_data = data.SnliData(fake_train_file, word2index) - test_data = data.SnliData(fake_train_file, word2index) - - # 2. Create a fake config. - config = _test_spinn_config( - data.WORD_VECTOR_LEN, 4, - logdir=os.path.join(self._temp_data_dir, "logdir")) - - # 3. Test training of a SPINN model. - trainer = spinn.train_or_infer_spinn( - embed, word2index, train_data, dev_data, test_data, config) - - # 4. Load train loss values from the summary files and verify that they - # decrease with training. - summary_file = glob.glob(os.path.join(config.logdir, "events.out.*"))[0] - events = summary_test_util.events_from_file(summary_file) - train_losses = [event.summary.value[0].simple_value for event in events - if event.summary.value - and event.summary.value[0].tag == "train/loss"] - self.assertEqual(config.epochs, len(train_losses)) - - # 5. Verify that checkpoints exist and contains all the expected variables. - self.assertTrue(glob.glob(os.path.join(config.logdir, "ckpt*"))) - object_graph = trackable_utils.object_metadata( - checkpoint_management.latest_checkpoint(config.logdir)) - ckpt_variable_names = set() - for node in object_graph.nodes: - for attribute in node.attributes: - ckpt_variable_names.add(attribute.full_name) - self.assertIn("global_step", ckpt_variable_names) - for v in trainer.variables: - variable_name = v.name[:v.name.index(":")] if ":" in v.name else v.name - self.assertIn(variable_name, ckpt_variable_names) - - -class EagerSpinnSNLIClassifierBenchmark(test.Benchmark): - - def benchmarkEagerSpinnSNLIClassifier(self): - test_device = "gpu:0" if test_util.is_gpu_available() else "cpu:0" - with tf.device(test_device): - burn_in_iterations = 2 - benchmark_iterations = 10 - - vocab_size = 1000 - batch_size = 128 - sequence_length = 15 - d_embed = 200 - d_out = 4 - - embed = tf.random_normal((vocab_size, d_embed)) - - config = _test_spinn_config(d_embed, d_out) - model = spinn.SNLIClassifier(config, embed) - trainer = spinn.SNLIClassifierTrainer(model, config.lr) - - (labels, prem, prem_trans, hypo, - hypo_trans) = _generate_synthetic_snli_data_batch(sequence_length, - batch_size, - vocab_size) - - for _ in range(burn_in_iterations): - trainer.train_batch(labels, prem, prem_trans, hypo, hypo_trans) - - gc.collect() - start_time = time.time() - for _ in xrange(benchmark_iterations): - trainer.train_batch(labels, prem, prem_trans, hypo, hypo_trans) - wall_time = time.time() - start_time - # Named "examples"_per_sec to conform with other benchmarks. - extras = {"examples_per_sec": benchmark_iterations / wall_time} - self.report_benchmark( - name="Eager_SPINN_SNLIClassifier_Benchmark", - iters=benchmark_iterations, - wall_time=wall_time, - extras=extras) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/eager/python/examples/workshop/1_basic.ipynb b/tensorflow/contrib/eager/python/examples/workshop/1_basic.ipynb deleted file mode 100644 index 17121cc21f8..00000000000 --- a/tensorflow/contrib/eager/python/examples/workshop/1_basic.ipynb +++ /dev/null @@ -1,282 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "TFE Workshop: control flow", - "version": "0.3.2", - "provenance": [], - "include_colab_link": true - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "view-in-github", - "colab_type": "text" - }, - "source": [ - "[View in Colaboratory](https://colab.research.google.com/gist/alextp/664b2f8700485ff6801f4d26293bd567/tfe-workshop-control-flow.ipynb)" - ] - }, - { - "metadata": { - "id": "9BpQzh9BvJlj", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 37 - }, - "outputId": "0b336886-8204-4815-89fa-5291a49d5784" - }, - "cell_type": "code", - "source": [ - "import tensorflow as tf\n", - "import numpy as np\n", - "tf.enable_eager_execution()" - ], - "execution_count": 1, - "outputs": [] - }, - { - "metadata": { - "id": "0roIB19GvOjI", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "# Eager execution basics\n", - "\n", - "When eager execution is enabled TensorFlow immediately executes operations, and Tensors are always available. " - ] - }, - { - "metadata": { - "id": "jeO8F-V-vN24", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 68 - }, - "outputId": "aeb3bdec-50b7-440d-93d8-5a171f091081" - }, - "cell_type": "code", - "source": [ - "t = tf.constant([[1, 2], [3, 4]])\n", - "t" - ], - "execution_count": 2, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 2 - } - ] - }, - { - "metadata": { - "id": "Y17RwSFxvlDL", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 68 - }, - "outputId": "cfcc10c7-707b-4997-99b3-a5f382c5166b" - }, - "cell_type": "code", - "source": [ - "tf.matmul(t, t)" - ], - "execution_count": 3, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 3 - } - ] - }, - { - "metadata": { - "id": "Dab1bS3TvmRE", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 - }, - "outputId": "8a624f3d-a658-4359-c586-1c5f6bf4c8b7" - }, - "cell_type": "code", - "source": [ - "# It's also possible to have Python control flow which depends on the value of tensors.\n", - "if t[0, 0] > 0.5:\n", - " print(\"T is bigger\")\n", - "else:\n", - " print(\"T is smaller\")" - ], - "execution_count": 4, - "outputs": [ - { - "output_type": "stream", - "text": [ - "T is bigger\n" - ], - "name": "stdout" - } - ] - }, - { - "metadata": { - "id": "dPgptJcGwIon", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 - }, - "outputId": "c4f27f2b-0848-4475-dde5-2534dac65a5c" - }, - "cell_type": "code", - "source": [ - "# Tensors are also usable as numpy arrays\n", - "np.prod(t)" - ], - "execution_count": 6, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "24" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 6 - } - ] - }, - { - "metadata": { - "id": "p3DTfQXnwXzj", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "# Exercise\n", - "\n", - "The algorithm for bisecting line search is a pretty simple way to find a zero of a continuous scalar function in an interval [a,b] where f(a) and f(b) have different signs. Simply evaluate f((a+b)/2), and narrow the interval by replacing either a or b with (a+b)/2 such that the function when applied on the boundary of the interval still has different signs.\n", - "\n", - "Implement a python function `bisecting_line_search(f, a, b, epsilon)` which returns a value such that `tf.abs(f(value)) < epsilon`.\n", - "\n", - "One thing to keep in mind: python's `==` operator is not overloaded on Tensors, so you need to use `tf.equal` to compare for equality." - ] - }, - { - "metadata": { - "id": "6eq0YuI6ykm5", - "colab_type": "code", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# Example test harness to get you going\n", - "\n", - "def test_f(x):\n", - " return x - 0.1234\n", - "def bisecting_line_search(f, a, b, epsilon):\n", - " # Return x such that f(x) <= epsilon.\n", - " pass\n", - "a = tf.constant(0.0)\n", - "b = tf.constant(1.0)\n", - "epsilon = tf.constant(0.001)\n", - "x = bisecting_line_search(test_f, a, b, epsilon)\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "LcMmEfd_xvej", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 170 - }, - "outputId": "f402aa50-8ce3-4416-f755-8bbcd1af7809" - }, - "cell_type": "code", - "source": [ - "#@title Double-click to see the solution\n", - "\n", - "def bisecting_line_search(f, a, b, epsilon):\n", - " f_a = f(a)\n", - " f_b = f(b)\n", - " probe = (a + b) / 2\n", - " f_probe = f(probe)\n", - " while tf.abs(f_probe) > epsilon:\n", - " if tf.equal(tf.sign(f_probe), tf.sign(f_a)):\n", - " a = probe\n", - " f_a = f_probe\n", - " else:\n", - " b = probe\n", - " f_b = f_probe\n", - " probe = (a + b) / 2\n", - " f_probe = f(probe)\n", - " print(\"new probe\", probe)\n", - " return probe\n", - "\n", - "bisecting_line_search(test_f, 0., 1., 0.001)" - ], - "execution_count": 8, - "outputs": [ - { - "output_type": "stream", - "text": [ - "('new probe', 0.25)\n", - "('new probe', 0.125)\n", - "('new probe', 0.0625)\n", - "('new probe', 0.09375)\n", - "('new probe', 0.109375)\n", - "('new probe', 0.1171875)\n", - "('new probe', 0.12109375)\n", - "('new probe', 0.123046875)\n" - ], - "name": "stdout" - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "0.123046875" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 8 - } - ] - } - ] -} diff --git a/tensorflow/contrib/eager/python/examples/workshop/2_models.ipynb b/tensorflow/contrib/eager/python/examples/workshop/2_models.ipynb deleted file mode 100644 index f3a65f5aab1..00000000000 --- a/tensorflow/contrib/eager/python/examples/workshop/2_models.ipynb +++ /dev/null @@ -1,1018 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "TFE Workshop: Models.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [], - "include_colab_link": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "view-in-github", - "colab_type": "text" - }, - "source": [ - "[View in Colaboratory](https://colab.research.google.com/gist/alextp/5cfcffd408bd5103f5ae747bc97ab0b5/tfe-workshop-models.ipynb)" - ] - }, - { - "metadata": { - "id": "BMxv1O6Q0SJL", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 17 - }, - "outputId": "8be9c556-ac7f-4142-e35e-19dc2b097121" - }, - "cell_type": "code", - "source": [ - "import tensorflow as tf\n", - "tf.enable_eager_execution()\n", - "tfe = tf.contrib.eager" - ], - "execution_count": 1, - "outputs": [] - }, - { - "metadata": { - "id": "lE1vJhxp0WR9", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "# Variables\n", - "\n", - "TensorFlow variables are useful to store the state in your program. They are integrated with other parts of the API (taking gradients, checkpointing, graph functions)." - ] - }, - { - "metadata": { - "id": "C4ztQNgc0VpW", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 - }, - "outputId": "8b63ae1f-2670-49c0-a31b-8cf7fc4194a1" - }, - "cell_type": "code", - "source": [ - "# Creating variables\n", - "v = tf.Variable(1.0)\n", - "v" - ], - "execution_count": 2, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 2 - } - ] - }, - { - "metadata": { - "id": "H0daItGg1IAp", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 - }, - "outputId": "e47d5aab-16a1-4e29-c27d-7fbc0b94b5d3" - }, - "cell_type": "code", - "source": [ - "v.assign_add(1.0)\n", - "v" - ], - "execution_count": 3, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 3 - } - ] - }, - { - "metadata": { - "id": "BJvBzcIG1hyK", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "# Layers: common sets of useful operations\n", - "\n", - "Most of the time when writing code for machine learning models you want to operate at a higher level of abstraction than individual operations and manipulation of individual variables.\n", - "\n", - "Many machine learning models are expressible as the composition and stacking of relatively simple layers, and TensorFlow provides both a set of many common layers as a well as easy ways for you to write your own application-specific layers either from scratch or as the composition of existing layers.\n", - "\n", - "TensorFlow includes the full [Keras](https://keras.io) API in the tf.keras package, and the Keras layers are very useful when building your own models.\n" - ] - }, - { - "metadata": { - "id": "iSQTS3QW1YQQ", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 17 - }, - "outputId": "c5d8aa10-dcad-44f7-f0eb-0faf5249fd7e" - }, - "cell_type": "code", - "source": [ - "# In the tf.keras.layers package, layers are objects. To construct a layer,\n", - "# simply construct the object. Most layers take as a first argument the number\n", - "# of output dimensions / channels.\n", - "layer = tf.keras.layers.Dense(100)\n", - "\n", - "# The number of input dimensions is often unnecessary, as it can be inferred\n", - "# the first time the layer is used, but it can be provided if you want to \n", - "# specify it manually, which is useful in some complex models.\n", - "layer = tf.keras.layers.Dense(10, input_shape=(None, 5))\n" - ], - "execution_count": 4, - "outputs": [] - }, - { - "metadata": { - "id": "nRuUogoS1liV", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 68 - }, - "outputId": "c352ce79-d519-45e4-a12e-1eaba76871a2" - }, - "cell_type": "code", - "source": [ - "layer(tf.zeros([2, 2]))" - ], - "execution_count": 5, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 5 - } - ] - }, - { - "metadata": { - "id": "JH4Kf4ka1mht", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 136 - }, - "outputId": "c34e2378-f83d-42c5-d30a-ebe55620368a" - }, - "cell_type": "code", - "source": [ - "layer.variables" - ], - "execution_count": 6, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "[,\n", - " ]" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 6 - } - ] - }, - { - "metadata": { - "id": "DSI4NF0_1vn-", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "The full list of pre-existing layers can be seen in [the documentation](https://www.tensorflow.org/api_docs/python/tf/keras/layers). It includes Dense (a fully-connected layer),\n", - "Conv2D, LSTM, BatchNormalization, Dropout, and many others." - ] - }, - { - "metadata": { - "id": "hMgDBftJ12Bp", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "# Models: composing layers\n", - "\n", - "Many interesting layer-like things in machine learning models are implemented by composing existing layers. For example, each residual block in a resnet is a composition of convolutions, batch normalizations, and a shortcut.\n", - "\n", - "The main class used when creating a layer-like thing which contains other layers is tf.keras.Model. Implementing one is done by inheriting from tf.keras.Model.\n" - ] - }, - { - "metadata": { - "id": "K3gVY6gj1nbe", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 190 - }, - "outputId": "6e9be0c4-960e-46c2-cdd9-7e94ad09d46b" - }, - "cell_type": "code", - "source": [ - "class ResnetIdentityBlock(tf.keras.Model):\n", - " def __init__(self, kernel_size, filters):\n", - " super(ResnetIdentityBlock, self).__init__(name='')\n", - " filters1, filters2, filters3 = filters\n", - "\n", - " self.conv2a = tf.keras.layers.Conv2D(filters1, (1, 1))\n", - " self.bn2a = tf.keras.layers.BatchNormalization()\n", - "\n", - " self.conv2b = tf.keras.layers.Conv2D(filters2, kernel_size, padding='same')\n", - " self.bn2b = tf.keras.layers.BatchNormalization()\n", - "\n", - " self.conv2c = tf.keras.layers.Conv2D(filters3, (1, 1))\n", - " self.bn2c = tf.keras.layers.BatchNormalization()\n", - "\n", - " def call(self, input_tensor, training=False):\n", - " x = self.conv2a(input_tensor)\n", - " x = self.bn2a(x, training=training)\n", - " x = tf.nn.relu(x)\n", - "\n", - " x = self.conv2b(x)\n", - " x = self.bn2b(x, training=training)\n", - " x = tf.nn.relu(x)\n", - "\n", - " x = self.conv2c(x)\n", - " x = self.bn2c(x, training=training)\n", - "\n", - " x += input_tensor\n", - " return tf.nn.relu(x)\n", - " \n", - "block = ResnetIdentityBlock(1, [1, 2, 3])\n", - "print(block(tf.zeros([1, 2, 3, 3])))\n", - "print([x.name for x in block.variables])" - ], - "execution_count": 7, - "outputs": [ - { - "output_type": "stream", - "text": [ - "tf.Tensor(\n", - "[[[[0. 0. 0.]\n", - " [0. 0. 0.]\n", - " [0. 0. 0.]]\n", - "\n", - " [[0. 0. 0.]\n", - " [0. 0. 0.]\n", - " [0. 0. 0.]]]], shape=(1, 2, 3, 3), dtype=float32)\n", - "['resnet_identity_block/conv2d/kernel:0', 'resnet_identity_block/conv2d/bias:0', 'resnet_identity_block/batch_normalization/gamma:0', 'resnet_identity_block/batch_normalization/beta:0', 'resnet_identity_block/conv2d_1/kernel:0', 'resnet_identity_block/conv2d_1/bias:0', 'resnet_identity_block/batch_normalization_1/gamma:0', 'resnet_identity_block/batch_normalization_1/beta:0', 'resnet_identity_block/conv2d_2/kernel:0', 'resnet_identity_block/conv2d_2/bias:0', 'resnet_identity_block/batch_normalization_2/gamma:0', 'resnet_identity_block/batch_normalization_2/beta:0', 'resnet_identity_block/batch_normalization/moving_mean:0', 'resnet_identity_block/batch_normalization/moving_variance:0', 'resnet_identity_block/batch_normalization_1/moving_mean:0', 'resnet_identity_block/batch_normalization_1/moving_variance:0', 'resnet_identity_block/batch_normalization_2/moving_mean:0', 'resnet_identity_block/batch_normalization_2/moving_variance:0']\n" - ], - "name": "stdout" - } - ] - }, - { - "metadata": { - "id": "LPXhHUIc1-sO", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "Much of the time, however, models which compose many layers simply call one layer after the other. This can be done in very little code using tf.keras.Sequential" - ] - }, - { - "metadata": { - "id": "5pXgzNAU17xk", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 173 - }, - "outputId": "03b7eaf8-9b35-482b-bcf0-a99af6c2c6a4" - }, - "cell_type": "code", - "source": [ - " my_seq = tf.keras.Sequential([tf.keras.layers.Conv2D(1, (1, 1)),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Conv2D(2, 1, \n", - " padding='same'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Conv2D(3, (1, 1)),\n", - " tf.keras.layers.BatchNormalization()])\n", - "my_seq(tf.zeros([1, 2, 3, 3]))\n" - ], - "execution_count": 8, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 8 - } - ] - }, - { - "metadata": { - "id": "MZrns6p22GEQ", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Exercise!\n", - "\n", - "Make a simple convolutional neural network model, useful for things such as MNIST which don't need too many parameters. A sequence of two or three convolutions with small output channels (say, 32 and 64) plus one or two fully connected layers is probably enough.\n", - "\n", - "The input shape should be [batch_size, 28, 28, 1]." - ] - }, - { - "metadata": { - "id": "8CAUa3KNN916", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 17 - }, - "outputId": "97c0ff3c-c962-4c13-eee8-406101465761" - }, - "cell_type": "code", - "source": [ - "# TODO: Implement a convolutional model as described above, and assign it to\n", - "# model.\n", - "model = tf.keras.Sequential([\n", - " \n", - "])" - ], - "execution_count": 9, - "outputs": [] - }, - { - "metadata": { - "id": "vLDDduR32E82", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 - }, - "outputId": "09bb1d43-b4c6-44b5-916e-0d2903d10cf4" - }, - "cell_type": "code", - "source": [ - "#@title Click to see the answer\n", - "\n", - "max_pool = tf.keras.layers.MaxPooling2D(\n", - " (2, 2), (2, 2), padding='same')\n", - " # The model consists of a sequential chain of layers, so tf.keras.Sequential\n", - " # (a subclass of tf.keras.Model) makes for a compact description.\n", - "model = tf.keras.Sequential(\n", - " [\n", - " tf.keras.layers.Conv2D(\n", - " 32,\n", - " 5,\n", - " padding='same',\n", - " activation=tf.nn.relu),\n", - " max_pool,\n", - " tf.keras.layers.Conv2D(\n", - " 64,\n", - " 5,\n", - " padding='same',\n", - " activation=tf.nn.relu),\n", - " max_pool,\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dense(1024, activation=tf.nn.relu),\n", - " tf.keras.layers.Dropout(0.4),\n", - " tf.keras.layers.Dense(10)\n", - " ])\n", - "\n", - "model(tf.zeros([1, 28, 28, 1]))" - ], - "execution_count": 10, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 10 - } - ] - }, - { - "metadata": { - "id": "H_CKVBroik4M", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "# Stop here for now" - ] - }, - { - "metadata": { - "id": "_yRwuE6MMmzC", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "# Training\n", - "\n", - "When eager execution is enabled, you can write Pythonic training loops. Simply\n", - "\n", - "1. load your data into a `tf.data.Dataset`, which lets you construct functional pipelines for processing, shuffling, and batching your data,\n", - "2. iterate over the dataset using a Python `for` loop, and\n", - "3. perform an optimization step in the body of your `for` loop.\n", - "\n", - "This workflow is exemplified in the following exercise." - ] - }, - { - "metadata": { - "id": "gj0-EkTc_Xt1", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "\n", - "\n", - "## Exercise!\n", - "\n", - "In this exercise, you'll train the convolutional model you implemented for the previous exericse on the MNIST dataset. " - ] - }, - { - "metadata": { - "id": "WOGm9HHn_byR", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 17 - }, - "outputId": "bbccc7ad-33cd-446e-bcda-f358c7547e1b" - }, - "cell_type": "code", - "source": [ - "#@title Utilities for downloading MNIST data (double-click to show code)\n", - "import gzip\n", - "import os\n", - "import tempfile\n", - "from six.moves import urllib\n", - "import shutil\n", - "\n", - "import numpy as np\n", - "\n", - "def read32(bytestream):\n", - " \"\"\"Read 4 bytes from bytestream as an unsigned 32-bit integer.\"\"\"\n", - " dt = np.dtype(np.uint32).newbyteorder('>')\n", - " return np.frombuffer(bytestream.read(4), dtype=dt)[0]\n", - "\n", - "\n", - "def check_image_file_header(filename):\n", - " \"\"\"Validate that filename corresponds to images for the MNIST dataset.\"\"\"\n", - " with tf.gfile.Open(filename, 'rb') as f:\n", - " magic = read32(f)\n", - " read32(f) # num_images, unused\n", - " rows = read32(f)\n", - " cols = read32(f)\n", - " if magic != 2051:\n", - " raise ValueError('Invalid magic number %d in MNIST file %s' % (magic,\n", - " f.name))\n", - " if rows != 28 or cols != 28:\n", - " raise ValueError(\n", - " 'Invalid MNIST file %s: Expected 28x28 images, found %dx%d' %\n", - " (f.name, rows, cols))\n", - "\n", - "\n", - "def check_labels_file_header(filename):\n", - " \"\"\"Validate that filename corresponds to labels for the MNIST dataset.\"\"\"\n", - " with tf.gfile.Open(filename, 'rb') as f:\n", - " magic = read32(f)\n", - " read32(f) # num_items, unused\n", - " if magic != 2049:\n", - " raise ValueError('Invalid magic number %d in MNIST file %s' % (magic,\n", - " f.name))\n", - " \n", - "def download(directory, filename):\n", - " \"\"\"Download (and unzip) a file from the MNIST dataset if not already done.\"\"\"\n", - " filepath = os.path.join(directory, filename)\n", - " if tf.gfile.Exists(filepath):\n", - " return filepath\n", - " if not tf.gfile.Exists(directory):\n", - " tf.gfile.MakeDirs(directory)\n", - " # CVDF mirror of http://yann.lecun.com/exdb/mnist/\n", - " url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'\n", - " _, zipped_filepath = tempfile.mkstemp(suffix='.gz')\n", - " print('Downloading %s to %s' % (url, zipped_filepath))\n", - " urllib.request.urlretrieve(url, zipped_filepath)\n", - " with gzip.open(zipped_filepath, 'rb') as f_in, \\\n", - " tf.gfile.Open(filepath, 'wb') as f_out:\n", - " shutil.copyfileobj(f_in, f_out)\n", - " os.remove(zipped_filepath)\n", - " return filepath\n", - "\n", - "\n", - "def dataset(directory, images_file, labels_file):\n", - " \"\"\"Download and parse MNIST dataset.\"\"\"\n", - "\n", - " images_file = download(directory, images_file)\n", - " labels_file = download(directory, labels_file)\n", - "\n", - " check_image_file_header(images_file)\n", - " check_labels_file_header(labels_file)\n", - "\n", - " def decode_image(image):\n", - " # Normalize from [0, 255] to [0.0, 1.0]\n", - " image = tf.decode_raw(image, tf.uint8)\n", - " image = tf.cast(image, tf.float32)\n", - " image = tf.reshape(image, [28, 28, 1])\n", - " return image / 255.0\n", - "\n", - " def decode_label(label):\n", - " label = tf.decode_raw(label, tf.uint8) # tf.string -> [tf.uint8]\n", - " label = tf.reshape(label, []) # label is a scalar\n", - " return tf.to_int32(label)\n", - "\n", - " images = tf.data.FixedLengthRecordDataset(\n", - " images_file, 28 * 28, header_bytes=16).map(decode_image)\n", - " labels = tf.data.FixedLengthRecordDataset(\n", - " labels_file, 1, header_bytes=8).map(decode_label)\n", - " return tf.data.Dataset.zip((images, labels))\n", - "\n", - "\n", - "def get_training_data(directory):\n", - " \"\"\"tf.data.Dataset object for MNIST training data.\"\"\"\n", - " return dataset(directory, 'train-images-idx3-ubyte',\n", - " 'train-labels-idx1-ubyte').take(1024)\n", - "\n", - "def get_test_data(directory):\n", - " \"\"\"tf.data.Dataset object for MNIST test data.\"\"\"\n", - " return dataset(directory, 't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte')" - ], - "execution_count": 11, - "outputs": [] - }, - { - "metadata": { - "id": "4ejmJ2dv_f0R", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 85 - }, - "outputId": "274c0381-e505-4e69-f910-3def6f8572a7" - }, - "cell_type": "code", - "source": [ - "# Don't forget to run the cell above!\n", - "training_data = get_training_data(\"/tmp/mnist/train\")\n", - "test_data = get_test_data(\"/tmp/mnist/test\")" - ], - "execution_count": 12, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/train-images-idx3-ubyte.gz to /tmp/tmp4ull1xwa.gz\n", - "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/train-labels-idx1-ubyte.gz to /tmp/tmp1eikhj1v.gz\n", - "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/t10k-images-idx3-ubyte.gz to /tmp/tmpcp8xah9c.gz\n", - "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/t10k-labels-idx1-ubyte.gz to /tmp/tmpqww_1e74.gz\n" - ], - "name": "stdout" - } - ] - }, - { - "metadata": { - "id": "TANpFS6GKLMC", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "Fill in the implementation of `train_one_epoch` below and run the cell to train your model. " - ] - }, - { - "metadata": { - "id": "btKL0Ss9_rmC", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 102 - }, - "outputId": "56858516-86fc-424a-f00d-6f088f98bf9b" - }, - "cell_type": "code", - "source": [ - "EPOCHS = 5\n", - "optimizer = tf.train.MomentumOptimizer(learning_rate=0.01, momentum=0.5)\n", - "\n", - "def loss_fn(logits, labels):\n", - " return tf.reduce_mean(\n", - " tf.nn.sparse_softmax_cross_entropy_with_logits(\n", - " logits=tf.squeeze(logits), labels=labels))\n", - "\n", - "def train_one_epoch(model, training_data, optimizer):\n", - " # TODO: Implement an optimization step and return the average loss.\n", - " #\n", - " # Hint: Use `tf.GradientTape` to compute the gradient of the loss, and use\n", - " # `optimizer.apply_gradients` to update the model's variables, which are\n", - " # accessible as `model.variables`\n", - " average_loss = tfe.metrics.Mean('loss')\n", - " for images, labels in training_data.shuffle(buffer_size=10000).batch(64):\n", - " pass\n", - " return average_loss.result()\n", - "\n", - "for epoch in range(EPOCHS):\n", - " loss = train_one_epoch(model, training_data, optimizer)\n", - " print(\"Average loss after epoch %d: %.4f\" % (epoch, loss))" - ], - "execution_count": 14, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Average loss after epoch 0: 2.2847\n", - "Average loss after epoch 1: 2.2305\n", - "Average loss after epoch 2: 2.1334\n", - "Average loss after epoch 3: 1.9115\n", - "Average loss after epoch 4: 1.4285\n" - ], - "name": "stdout" - } - ] - }, - { - "metadata": { - "id": "yAOFupJN_htg", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 102 - }, - "outputId": "67e711e4-76c9-4e3f-bb49-a14955dba03a" - }, - "cell_type": "code", - "source": [ - "#@title Double-click to see a solution.\n", - "EPOCHS = 5\n", - "optimizer = tf.train.MomentumOptimizer(learning_rate=0.01, momentum=0.5)\n", - "\n", - "def _loss_fn(logits, labels):\n", - " return tf.reduce_mean(\n", - " tf.nn.sparse_softmax_cross_entropy_with_logits(\n", - " logits=tf.squeeze(logits), labels=labels))\n", - "\n", - "def _train_one_epoch(model, training_data):\n", - " average_loss = tfe.metrics.Mean(\"loss\")\n", - " for images, labels in training_data.shuffle(buffer_size=10000).batch(64):\n", - " with tf.GradientTape() as tape:\n", - " logits = model(images, training=True)\n", - " loss = _loss_fn(logits, labels)\n", - " average_loss(loss)\n", - " gradients = tape.gradient(loss, model.variables)\n", - " optimizer.apply_gradients(zip(gradients, model.variables))\n", - " return average_loss.result()\n", - " \n", - "for epoch in range(EPOCHS):\n", - " loss = _train_one_epoch(model, training_data)\n", - " print(\"Average loss after epoch %d: %.4f\" % (epoch, loss))" - ], - "execution_count": 15, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Average loss after epoch 0: 1.0563\n", - "Average loss after epoch 1: 0.8013\n", - "Average loss after epoch 2: 0.6306\n", - "Average loss after epoch 3: 0.5543\n", - "Average loss after epoch 4: 0.5037\n" - ], - "name": "stdout" - } - ] - }, - { - "metadata": { - "id": "uDy1DrYA_2Jz", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "Run the below cell to qualitatively evaluate your model. Note how eager execution interoperates seamlessly with `matplotlib`." - ] - }, - { - "metadata": { - "id": "vR7rMtpu_3nB", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1752 - }, - "outputId": "b212aefa-f4b3-425c-f34d-2491429fa521" - }, - "cell_type": "code", - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "sampled_data = test_data.batch(1).shuffle(buffer_size=10000).take(5)\n", - "for image, label in sampled_data:\n", - " plt.figure()\n", - " plt.imshow(tf.reshape(image, (28, 28)))\n", - " plt.show()\n", - " logits = model(image, training=False)\n", - " prediction = tf.argmax(logits, axis=1, output_type=tf.int64)\n", - " print(\"Prediction: %d\" % prediction)" - ], - "execution_count": 16, - "outputs": [ - { - "output_type": "display_data", - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAFKCAYAAACU6307AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAEwpJREFUeJzt3X1Ilff/x/HXmScxV2GZOmLVohXK\nKmLQjbUsy+pbI7rbaEm1IFhRSU1aE+kO3LqxCGrBMlsNkq0zZIM2Cu1mUTg1itXQbVnBQqKZNtcN\n2d3J3x9ffpLrNN/ndM65jn6fj7/m5cfrvI9XPHedc7zOcTU3NzcLAPCvXnJ6AABoD4glABgQSwAw\nIJYAYEAsAcCAWAKAAbEEAANiCQAG7kB/cOPGjbpw4YJcLpdyc3M1ZMiQYM4FABEloFieOXNGV69e\nlcfj0ZUrV5SbmyuPxxPs2QAgYgT0MLy8vFwZGRmSpP79++vWrVu6e/duUAcDgEgSUCwbGhrUvXv3\nlq979Oih+vr6oA0FAJEmKC/w8F4cADq6gGKZmJiohoaGlq9v3LihhISEoA0FAJEmoFiOHj1aJSUl\nkqTq6molJiaqS5cuQR0MACJJQK+Gv/nmm3rjjTf03nvvyeVyaf369cGeCwAiios3/wWAtnEFDwAY\nEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkA\nBsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgC\ngAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMHA7\nPQAQiAcPHpjX3rlzx+f2nj17qqGhodW2kydPmvb566+/mm//xx9/NK+13r4kjRgx4pltFRUVGjly\nZKttP/30k3mfL73E+dPz8JsBAIOAziwrKyu1YsUKDRgwQJI0cOBArV27NqiDAUAkCfhh+PDhw7Vz\n585gzgIAEYuH4QBgEHAsL1++rCVLlmju3LkqKysL5kwAEHFczc3Nzf7+UF1dnc6dO6cpU6aotrZW\nCxYsUGlpqaKjo0MxIwA4LqDnLJOSkjR16lRJUp8+fdSzZ0/V1dWpd+/eQR0OeB7+dIg/HQq3gH4z\nhw4d0hdffCFJqq+v182bN5WUlBTUwQAgkgR0Zjl+/HitWrVKx48f16NHj7RhwwYeggPo0AKKZZcu\nXbR79+5gzwIAESugF3gAf1RVVZnXfvfdd6Z1hw8fNu/zzJkzPrd7vV5FRUWZ99Me+LpPDx8+NP98\nR/t9BBPP5gKAAbEEAANiCQAGxBIADIglABgQSwAwIJYAYEAsAcCAWAKAAbEEAAM+3RGtPO/qV5fL\n1ep7BQUF5n1mZWWZ1z558sS8NhRcLpdpnT9vZebPJYT9+vUzry0pKfG5/Y8//mj1NW+7Fhz8FgHA\ngFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgCt40MrBgwd9bp87d26r7y1btsy8z1de\necW89q233jKte//99837/Dfff/99q68TExNNP/fqq6+ab8Of+x8MvXv3Duvt/a/gzBIADIglABgQ\nSwAwIJYAYEAsAcCAWAKAAbEEAANiCQAGxBIADIglABi4mp/3CVXoMB49emRe+/rrr/vcfvXqVfXt\n27fl68zMTPM+P/74Y/PauLg481ognDizBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGx\nBAADYgkABny6YztVX19vXjthwgTz2oEDB5q+l5eXZ96n223/Z/b48WPTuuvXr5v3efz4cZ/bFy5c\nqC+//NK8n0CNHTvWvLZfv34hnAQvwnRmWVNTo4yMDBUVFUn67z/U+fPnKzMzUytWrNDDhw9DOiQA\nOK3NWN67d095eXlKTU1t2bZz505lZmbqq6++Ut++fVVcXBzSIQHAaW3GMjo6WoWFha0+fL6ysrLl\noV16errKy8tDNyEARIA2n0xyu93PPOfU1NSk6OhoSVJ8fLxfz58BQHv0wi/w8HaYzkhISDCv/eWX\nX4Jym0ePHg3Kfv6N9cWg3r17m/e5cOHCgL4HPC2gWMbGxur+/fuKiYlRXV1dq4foCI9QvRqelJTk\nc/vRo0c1ceLElq+PHDli3ievhvNqeEcQ0N9Zjho1SiUlJZKk0tJSjRkzJqhDAUCkafN/+VVVVdqy\nZYuuXbsmt9utkpISbdu2TTk5OfJ4POrVq5dmzJgRjlkBwDFtxnLQoEE6cODAM9v3798fkoEAIBLx\ngWXt1A8//GBeO3v2bPPa572Ik5aWplOnTrV8ff78efM+J02aZF5rnfX333837/N5vF6voqKiAvrZ\nd99917x20KBB5rWrVq0yr42JiTGvxYvj2nAAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAY\nEEsAMCCWAGDA5Y7tlD+X23377bcvfHv/vDTQn7cS8+ft1NLS0kzr/Ln/o0aN8rk9OTn5mcsmO3Xq\nZNrn7du3zbc/YsQI89q9e/ea1y5YsMC8Fi+OM0sAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyI\nJQAYEEsAMCCWAGDQ5kfhIjItXrzYvHb06NHmtRcvXnzu9z744IOW//bnUruhQ4ea11ovN3S7g/NP\nNzk5OaCfe/qTLtvi9XrNa/351E4udwwvziwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBg\nQCwBwIAreNqpjIyMkKz9N59//nlQ9tMRPHjwwOkREGacWQKAAbEEAANiCQAGxBIADIglABgQSwAw\nIJYAYEAsAcCAWAKAAbEEAANiCQAGxBIADEyxrKmpUUZGhoqKiiRJOTk5mjZtmubPn6/58+fr5MmT\noZwRABzX5rsO3bt3T3l5eUpNTW21PTs7W+np6SEbDAAiSZtnltHR0SosLFRiYmI45gGAiNTmmaXb\n7Zbb/eyyoqIi7d+/X/Hx8Vq7dq169OgRkgGBSDRx4kTzWq/XG8JJEC4Bvfnv9OnTFRcXp5SUFO3Z\ns0e7du3SunXrgj0bELGOHj1qXvuf//zHvHb27Nnmtd988415LV5cQK+Gp6amKiUlRZI0fvx41dTU\nBHUoAIg0AcUyKytLtbW1kqTKykoNGDAgqEMBQKRp82F4VVWVtmzZomvXrsntdqukpETz5s3TypUr\n1blzZ8XGxmrTpk3hmBUAHNNmLAcNGqQDBw48s33y5MkhGQgAIhGf7ggEgAsx/vdwuSMAGBBLADAg\nlgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADDgckcgAKdPnw7JfqdNmxaS/eLFcWYJAAbE\nEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAZcwQM85dSpU6Z1P//8s3mfL7/8snntuHHj\nzGsRXpxZAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAAy53RIf3999/+9we\nFxf3zPcyMjJM+/R6vebbP3jwoHlt7969zWsRXpxZAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBg\nQCwBwIBYAoABsQQAAy53DIMnT56Y1+bm5prWbdiwwbzPmJgY89r24u7du+a1b7/9ts/tZWVlz3zP\nehnjO++8Y7792bNnm9cicplimZ+fr3Pnzunx48davHixBg8erNWrV8vr9SohIUFbt25VdHR0qGcF\nAMe0GcuKigpdunRJHo9HjY2NmjlzplJTU5WZmakpU6Zo+/btKi4uVmZmZjjmBQBHtPmc5bBhw7Rj\nxw5JUrdu3dTU1KTKykpNmDBBkpSenq7y8vLQTgkADmszllFRUYqNjZUkFRcXKy0tTU1NTS0Pu+Pj\n41VfXx/aKQHAYeYXeI4dO6bi4mLt27dPkyZNatne3NwcksE6kpdesv/RwebNm0M4ScfRpUsX89qy\nsrKAvgc8zRTL06dPa/fu3dq7d6+6du2q2NhY3b9/XzExMaqrq1NiYmKo52zXeDU8+Px5NXzy5Mk+\nt5eVlWn06NGttlVUVJj26c+r4V9//bV5rT//Y0V4tXlk7ty5o/z8fBUUFCguLk6SNGrUKJWUlEiS\nSktLNWbMmNBOCQAOa/PM8vDhw2psbNTKlStbtm3evFlr1qyRx+NRr169NGPGjJAOCQBOazOWc+bM\n0Zw5c57Zvn///pAMBACRyNXMKzQh58+HW1n/uP/TTz817zM7Ozvotx8qv/32m2nd0qVLzfs8deqU\nz+1er1dRUVHm/TyturravDY5OTmg20Bk4dlkADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUA\nGBBLADAglgBgwOWOYeDP5Y4JCQmmdbdu3TLvc+LEiea148aN87k9Jycn4PfavH//vnntJ598Ylrn\nzz/bbt26+dze2Nio7t27t9p28eJF0z6tx0mSXC6XeS0iF2eWAGBALAHAgFgCgAGxBAADYgkABsQS\nAAyIJQAYEEsAMCCWAGBALAHAgMsdI0xxcbFp3bJly8z7bGhoCHScFi/ySYj++Oflh88zefJk8z4/\n+ugjn9uHDh2q8+fPP7MN8IUzSwAwIJYAYEAsAcCAWAKAAbEEAANiCQAGxBIADIglABgQSwAw4Aqe\ndqqmpsa8Njs727z2yJEjPre/yBU8q1evNq8dPHiwaV1mZmZAswCB4swSAAyIJQAYEEsAMCCWAGBA\nLAHAgFgCgAGxBAADYgkABsQSAAyIJQAYcLkjABi4LYvy8/N17tw5PX78WIsXL9aJEydUXV2tuLg4\nSdKiRYs0bty4UM4JAI5qM5YVFRW6dOmSPB6PGhsbNXPmTI0cOVLZ2dlKT08Px4wA4Lg2Yzls2DAN\nGTJEktStWzc1NTXJ6/WGfDAAiCR+PWfp8Xh09uxZRUVFqb6+Xo8ePVJ8fLzWrl2rHj16hHJOAHCU\nOZbHjh1TQUGB9u3bp6qqKsXFxSklJUV79uzRn3/+qXXr1oV6VgBwjOlPh06fPq3du3ersLBQXbt2\nVWpqqlJSUiRJ48eP9+uNaAGgPWozlnfu3FF+fr4KCgpaXv3OyspSbW2tJKmyslIDBgwI7ZQA4LA2\nX+A5fPiwGhsbtXLlypZts2bN0sqVK9W5c2fFxsZq06ZNIR0SAJzGH6UDgAGXOwKAAbEEAANiCQAG\nxBIADIglABgQSwAwIJYAYEAsAcCAWAKAAbEEAANiCQAGxBIADIglABgQSwAwIJYAYEAsAcCAWAKA\nAbEEAANiCQAGxBIADIglABgQSwAwIJYAYEAsAcCAWAKAAbEEAANiCQAGxBIADIglABi4nbjRjRs3\n6sKFC3K5XMrNzdWQIUOcGCOoKisrtWLFCg0YMECSNHDgQK1du9bhqQJXU1OjpUuXauHChZo3b56u\nX7+u1atXy+v1KiEhQVu3blV0dLTTY/rln/cpJydH1dXViouLkyQtWrRI48aNc3ZIP+Xn5+vcuXN6\n/PixFi9erMGDB7f74yQ9e79OnDjh+LEKeyzPnDmjq1evyuPx6MqVK8rNzZXH4wn3GCExfPhw7dy5\n0+kxXti9e/eUl5en1NTUlm07d+5UZmampkyZou3bt6u4uFiZmZkOTukfX/dJkrKzs5Wenu7QVC+m\noqJCly5dksfjUWNjo2bOnKnU1NR2fZwk3/dr5MiRjh+rsD8MLy8vV0ZGhiSpf//+unXrlu7evRvu\nMfAvoqOjVVhYqMTExJZtlZWVmjBhgiQpPT1d5eXlTo0XEF/3qb0bNmyYduzYIUnq1q2bmpqa2v1x\nknzfL6/X6/BUDsSyoaFB3bt3b/m6R48eqq+vD/cYIXH58mUtWbJEc+fOVVlZmdPjBMztdismJqbV\ntqamppaHc/Hx8e3umPm6T5JUVFSkBQsW6MMPP9Rff/3lwGSBi4qKUmxsrCSpuLhYaWlp7f44Sb7v\nV1RUlOPHypHnLJ/W3Nzs9AhB8dprr2n58uWaMmWKamtrtWDBApWWlrbL54va0lGO2fTp0xUXF6eU\nlBTt2bNHu3bt0rp165wey2/Hjh1TcXGx9u3bp0mTJrVsb+/H6en7VVVV5fixCvuZZWJiohoaGlq+\nvnHjhhISEsI9RtAlJSVp6tSpcrlc6tOnj3r27Km6ujqnxwqa2NhY3b9/X5JUV1fXIR7OpqamKiUl\nRZI0fvx41dTUODyR/06fPq3du3ersLBQXbt27TDH6Z/3KxKOVdhjOXr0aJWUlEiSqqurlZiYqC5d\nuoR7jKA7dOiQvvjiC0lSfX29bt68qaSkJIenCp5Ro0a1HLfS0lKNGTPG4YleXFZWlmprayX99znZ\n//9Lhvbizp07ys/PV0FBQcurxB3hOPm6X5FwrFzNDpyrb9u2TWfPnpXL5dL69euVnJwc7hGC7u7d\nu1q1apVu376tR48eafny5Ro7dqzTYwWkqqpKW7Zs0bVr1+R2u5WUlKRt27YpJydHDx48UK9evbRp\n0yZ16tTJ6VHNfN2nefPmac+ePercubNiY2O1adMmxcfHOz2qmcfj0WeffaZ+/fq1bNu8ebPWrFnT\nbo+T5Pt+zZo1S0VFRY4eK0diCQDtDVfwAIABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwOD/\nAKCzFeFbFn4BAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "Prediction: 5\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAFKCAYAAACU6307AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAEQ1JREFUeJzt3W9Ilff/x/HXSSd2VmKaRwiqjTBy\nq9gfap2iliaFQfRvsCXW1rpRRJGTCJG0MSHLIpbF8M9qN3L7cjZvNQiOVAQt7LQcBLqB1Y0QaXYs\naUa2mZ3fjS9ff7Vcvj2ec65jez7ueZ1P57wPlzy7Li8vjysUCoUEAHihcU4PAABjAbEEAANiCQAG\nxBIADIglABgQSwAwIJYAYEAsAcAgMdx/uH//fl27dk0ul0ulpaWaO3duJOcCgLgSViyvXLmiW7du\nyefz6ebNmyotLZXP54v0bAAQN8I6DW9ublZeXp4kacaMGbp//74ePHgQ0cEAIJ6EFcvu7m5NmjRp\n8Ou0tDQFg8GIDQUA8SYiF3j4WxwAXnZhxdLj8ai7u3vw6zt37igjIyNiQwFAvAkrlosWLZLf75ck\ntbW1yePxaMKECREdDADiSVhXw9955x29+eab+uijj+RyubRv375IzwUAccXFH/8FgOFxBw8AGBBL\nADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbE\nEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoAB\nsQQAA2IJAAbEEgAMEp0eAIgnP/30k2nd+vXrzc+Zl5dnXvvtt9+a1yK2OLIEAANiCQAGxBIADIgl\nABgQSwAwIJYAYEAsAcCAWAKAAbEEAAPu4AGecuzYMdO6YDBofk6XyxXuOIgjHFkCgEFYR5aBQEC7\ndu1SVlaWJGnmzJkqKyuL6GAAEE/CPg2fP3++qqurIzkLAMQtTsMBwCDsWN64cUPbtm3Thg0bdOnS\npUjOBABxxxUKhUIj/UddXV1qaWlRfn6+Ojo6tGnTJjU1NSkpKSkaMwKA48L6mWVmZqZWrlwpSZo2\nbZomT56srq4uTZ06NaLDAbH24Ycfmtb98MMP5ucsKCgwr21oaDCvRWyFdRp++vRpnThxQtJ/f9/s\n7t27yszMjOhgABBPwjqyzM3N1e7du3Xu3Dn19/fr888/5xQcwEstrFhOmDBBNTU1kZ4FAOIWtzsC\nT7lw4ULEn3PVqlURf07EHr9nCQAGxBIADIglABgQSwAwIJYAYEAsAcCAWAKAAbEEAANiCQAGxBIA\nDLjdES89v98/5PYVK1Y899hIPrXRqre3N+LPidjjyBIADIglABgQSwAwIJYAYEAsAcCAWAKAAbEE\nAANiCQAGxBIADLiDB2NSKBQyr21oaBhy+4oVK/7xsUh6++23o/4aiD6OLAHAgFgCgAGxBAADYgkA\nBsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgIErNJL7xoA40dnZaV47derUIbc/efJE48aFd7zw7rvv\nmtf+/PPPYb0G4gtHlgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBPd8SY\nVFlZ6ejrb9682dHXR+yZjizb29uVl5c3+LGht2/f1saNG1VQUKBdu3bpr7/+iuqQAOC0YWP58OFD\nVVRUyOv1Dm6rrq5WQUGBvvvuO02fPl2NjY1RHRIAnDZsLJOSklRfXy+PxzO4LRAIaNmyZZKknJwc\nNTc3R29CAIgDw/7MMjExUYmJzy7r6+tTUlKSJCk9PV3BYDA60wFAnBj1BR7+HCaccPz48YisffLk\nSSTGwb9AWLF0u9169OiRkpOT1dXV9cwpOhALO3bsMK/96quvhtw+mj/+O5JYb9++PazXQHwJ6ztl\n4cKF8vv9kqSmpiYtXrw4okMBQLwZ9siytbVVBw8eVGdnpxITE+X3+3X48GGVlJTI5/NpypQpWrNm\nTSxmBQDHDBvL2bNn69SpU89t/+abb6IyEADEI+7gQVyxXnCJ1oeAWX/+XlhYGJXXR/zi3nAAMCCW\nAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGDA7Y6IKxUVFaZ10brd8dVXXzWt6+3t\nNT9nSkpKuOMgjnBkCQAGxBIADIglABgQSwAwIJYAYEAsAcCAWAKAAbEEAANiCQAGxBIADLjdEXHl\nyy+/dPT1BwYGTOv8fr/5OT/99NNwx0Ec4cgSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkA\nBsQSAAy4gwdR99tvv5nXjuSDwKzcbrf5sV9++cX0nGlpaaOaCWMPR5YAYEAsAcCAWAKAAbEEAANi\nCQAGxBIADIglABgQSwAwIJYAYEAsAcCA2x0RFusHe0kj+xCyJ0+ehDPOC507d878GLcx4p9wZAkA\nBqZYtre3Ky8vTw0NDZKkkpISrVq1Shs3btTGjRt14cKFaM4IAI4b9jT84cOHqqiokNfrfWZ7cXGx\ncnJyojYYAMSTYY8sk5KSVF9fL4/HE4t5ACAuuUKhUMiy8NixY5o0aZIKCwtVUlKiYDCo/v5+paen\nq6ysjB+MA3iphXU1fPXq1UpNTVV2drbq6up0/PhxlZeXR3o2xLGRXA3fvn27eW19fX0447xQc3Pz\nkNvfe+89BQKB57YBQwnrarjX61V2drYkKTc3V+3t7REdCgDiTVix3Llzpzo6OiRJgUBAWVlZER0K\nAOLNsKfhra2tOnjwoDo7O5WYmCi/36/CwkIVFRVp/PjxcrvdqqysjMWsAOCYYWM5e/ZsnTp16rnt\nK1asiMpAABCPzFfDgafdu3fPvHby5MkRf/0PPvjAvPY///nPkNsTEhKeu1CVkJAwqrnw8uJ2RwAw\nIJYAYEAsAcCAWAKAAbEEAANiCQAGxBIADIglABgQSwAwIJYAYMCnO+IZ//TpiuPGjXvmsc2bN0fl\n9V0ul2ndF198YX7OF93CyO2NsOLIEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAM\nuIMHz/jf58H/3fTp05957Mcff4zK6xcWFprWzZo1KyqvD/wTjiwBwIBYAoABsQQAA2IJAAbEEgAM\niCUAGBBLADAglgBgQCwBwIBYAoABtzviGRcuXBhy+8cff/zMY6FQKCqvX15eHpXnBUaLI0sAMCCW\nAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGDgCkXrvjXEjV9//dW8ds6cOUNuHxgY\nUEJCwuDXI/m2Wb9+vXmtz+czrRs3jv/nEVume8OrqqrU0tKix48fa+vWrZozZ4727NmjgYEBZWRk\n6NChQ0pKSor2rADgmGFjefnyZV2/fl0+n089PT1au3atvF6vCgoKlJ+fryNHjqixsVEFBQWxmBcA\nHDHsucy8efN09OhRSVJKSor6+voUCAS0bNkySVJOTo6am5ujOyUAOGzYWCYkJMjtdkuSGhsbtWTJ\nEvX19Q2edqenpysYDEZ3SgBwmPnvWZ49e1aNjY06efKkli9fPrid60Px74033jCvHRgYCOsx4GVn\niuXFixdVU1Ojr7/+WhMnTpTb7dajR4+UnJysrq4ueTyeaM+JUeBqODB6w37H9fb2qqqqSrW1tUpN\nTZUkLVy4UH6/X5LU1NSkxYsXR3dKAHDYsEeWZ86cUU9Pj4qKiga3HThwQHv37pXP59OUKVO0Zs2a\nqA4JAE7jl9L/BTgNB0aPDyz7F7AGSHpxBJ9+LCUlxfycJ06cMK8lgohXfGcCgAGxBAADYgkABsQS\nAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADbnf8F7hx44Z5rfV2x+TkZPNzjuTWSCBecWQJAAbE\nEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMuN3xX6C4uNi89vvvv//HxxIT///b\n5a233hrVTMBYw5ElABgQSwAwIJYAYEAsAcCAWAKAAbEEAANiCQAGxBIADIglABi4Qi/6hCoAgCSO\nLAHAhFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAAD06c7VlVVqaWlRY8f\nP9bWrVt1/vx5tbW1KTU1VZK0ZcsWLV26NJpzAoCjho3l5cuXdf36dfl8PvX09Gjt2rVasGCBiouL\nlZOTE4sZAcBxw8Zy3rx5mjt3riQpJSVFfX19GhgYiPpgABBPRvQn2nw+n65evaqEhAQFg0H19/cr\nPT1dZWVlSktLi+acAOAocyzPnj2r2tpanTx5Uq2trUpNTVV2drbq6ur0+++/q7y8PNqzAoBjTFfD\nL168qJqaGtXX12vixInyer3Kzs6WJOXm5qq9vT2qQwKA04aNZW9vr6qqqlRbWzt49Xvnzp3q6OiQ\nJAUCAWVlZUV3SgBw2LAXeM6cOaOenh4VFRUNblu3bp2Kioo0fvx4ud1uVVZWRnVIAHAan8EDAAbc\nwQMABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHA\ngFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsA\nMCCWAGCQ6MSL7t+/X9euXZPL5VJpaanmzp3rxBgRFQgEtGvXLmVlZUmSZs6cqbKyMoenCl97e7u2\nb9+uTz75RIWFhbp9+7b27NmjgYEBZWRk6NChQ0pKSnJ6zBH5+3sqKSlRW1ubUlNTJUlbtmzR0qVL\nnR1yhKqqqtTS0qLHjx9r69atmjNnzpjfT9Lz7+v8+fOO76uYx/LKlSu6deuWfD6fbt68qdLSUvl8\nvliPERXz589XdXW102OM2sOHD1VRUSGv1zu4rbq6WgUFBcrPz9eRI0fU2NiogoICB6ccmaHekyQV\nFxcrJyfHoalG5/Lly7p+/bp8Pp96enq0du1aeb3eMb2fpKHf14IFCxzfVzE/DW9ublZeXp4kacaM\nGbp//74ePHgQ6zHwAklJSaqvr5fH4xncFggEtGzZMklSTk6OmpubnRovLEO9p7Fu3rx5Onr0qCQp\nJSVFfX19Y34/SUO/r4GBAYenciCW3d3dmjRp0uDXaWlpCgaDsR4jKm7cuKFt27Zpw4YNunTpktPj\nhC0xMVHJycnPbOvr6xs8nUtPTx9z+2yo9yRJDQ0N2rRpkz777DPdu3fPgcnCl5CQILfbLUlqbGzU\nkiVLxvx+koZ+XwkJCY7vK0d+Zvm0UCjk9AgR8dprr2nHjh3Kz89XR0eHNm3apKampjH586LhvCz7\nbPXq1UpNTVV2drbq6up0/PhxlZeXOz3WiJ09e1aNjY06efKkli9fPrh9rO+np99Xa2ur4/sq5keW\nHo9H3d3dg1/fuXNHGRkZsR4j4jIzM7Vy5Uq5XC5NmzZNkydPVldXl9NjRYzb7dajR48kSV1dXS/F\n6azX61V2drYkKTc3V+3t7Q5PNHIXL15UTU2N6uvrNXHixJdmP/39fcXDvop5LBctWiS/3y9Jamtr\nk8fj0YQJE2I9RsSdPn1aJ06ckCQFg0HdvXtXmZmZDk8VOQsXLhzcb01NTVq8eLHDE43ezp071dHR\nIem/P5P9328yjBW9vb2qqqpSbW3t4FXil2E/DfW+4mFfuUIOHKsfPnxYV69elcvl0r59+zRr1qxY\njxBxDx480O7du/XHH3+ov79fO3bs0Pvvv+/0WGFpbW3VwYMH1dnZqcTERGVmZurw4cMqKSnRn3/+\nqSlTpqiyslKvvPKK06OaDfWeCgsLVVdXp/Hjx8vtdquyslLp6elOj2rm8/l07Ngxvf7664PbDhw4\noL17947Z/SQN/b7WrVunhoYGR/eVI7EEgLGGO3gAwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAg\nlgBg8H/nb4OLnfGqVAAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "Prediction: 1\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAFKCAYAAACU6307AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAE1ZJREFUeJzt3X1olfX/x/HXccc1DyrLuY1GaRGL\nRqZSaE7zZmqKgnhDsVwqkYGRE29QW8tp4M102solNJ03fzSqgyPoBmFDIlg1Jw0xNsrZDbKGranD\nG5x3x33/+NF+rp153js751znrOfjv13n43Xex4NPrrPL61yujo6ODgEA7muA0wMAQCwglgBgQCwB\nwIBYAoABsQQAA2IJAAbEEgAMiCUAGLiD/YM7duzQ6dOn5XK5lJ+fr9GjR4dyLgCIKkHF8uTJkzp3\n7py8Xq9+++035efny+v1hno2AIgaQX0Mr6mp0cyZMyVJjz/+uC5fvqxr166FdDAAiCZBxfLChQt6\n8MEHO38eNmyYWltbQzYUAESbkJzg4bs4APR3QcUyJSVFFy5c6Pz577//VnJycsiGAoBoE1QsJ02a\npMrKSklSQ0ODUlJSNHjw4JAOBgDRJKiz4c8884yeeuopvfzyy3K5XNqyZUuo5wKAqOLiy38BIDCu\n4AEAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBg\nQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUA\nGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJ\nAAbEEgAMiCUAGLiD+UO1tbVavXq10tPTJUlPPPGECgoKQjoYAESToGIpSePHj1dJSUkoZwGAqMXH\ncAAwCDqWv/76q9544w0tXrxY33//fShnAoCo4+ro6Ojo7R9qaWlRXV2d5syZo6amJi1btkxVVVWK\nj48Px4wA4LigjixTU1M1d+5cuVwujRgxQsOHD1dLS0uoZwOAqBFULL/88ksdOnRIktTa2qqLFy8q\nNTU1pIMBQDQJ6mP4tWvXtH79el25ckW3b99Wbm6upk6dGo75ACAqBBVLAPivCfr/WQL90alTp0zr\nSktLzfssKysLdpz78nec09HRIZfL1WVbbm6ueZ+9+b/T/36e/o7/ZwkABsQSAAyIJQAYEEsAMCCW\nAGBALAHAgFgCgAGxBAADYgkABsQSAAy4Nhz93tmzZ/1uT09P7/bY4sWLTfu0XhYZaT6fT3FxcUH/\n+Vu3bpnX9uV5YhFHlgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgwA3LEHa9uUjs\nzJkzpnXz588377Opqcnv9uvXr2vMmDFdtt28edO8Xyu32/7PrKCgwLw2Pj7e7/bCwsIuPz/77LPm\nfQ4YwPFTT/ibAQADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABtywDEG5ffu2\nee1bb71lXrt3795gxgmKv5t7PfTQQ6Y/u3r1avPzLF++3Lz2yJEj5rW5ubndtj3wwAPdLtl84IEH\nzPtEzziyBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABtzdEV3cvXvX7/YB\nAwZ0eSwvL8+8z0hewujPokWLzI999NFHpn16PB7z8y9evNi89uuvvzavbW5u7ratuLhYb7/9drdt\n6DvTkWVjY6Nmzpyp8vJySdL58+e1dOlS5eTkaPXq1bp161ZYhwQApwWM5fXr17V161ZlZmZ2bisp\nKVFOTo4++eQTjRw5UhUVFWEdEgCcFjCW8fHxKisrU0pKSue22tpazZgxQ5KUlZWlmpqa8E0IAFEg\n4O8s3W633O6uy9rb2xUfHy9JSkpKUmtra3imA4Ao0ecTPHwdZv8yYEDPHzbufey9994z77M3ayPt\n6NGjYX+OL774IuzPcS9O6IRHULH0eDy6ceOGEhIS1NLS0uUjOmKb9Wz4hg0bzPv84IMP+jxXX/R0\nNvzo0aN66aWXumyLpbPh/r6AuLi4WOvWreu2DX0X1P+znDhxoiorKyVJVVVVmjx5ckiHAoBoE/DI\nsr6+Xrt27VJzc7PcbrcqKyu1Z88e5eXlyev1Ki0tTQsWLIjErADgmICxHDVqlD7++ONu23tzrxAA\niHVcwfMf8Ndff5nXzpo1y+/2n376SWPHju38uaGhoc9z+TN06FDTutLSUvM+X3zxxR4f++yzz7r8\nfL8TXPf69NNPzc/fm99D9kZaWlqvtqNvuDYcAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkA\nBsQSAAyIJQAYuDr4QsqYdPXqVfPaUaNGmdf++eeffrf7fD7FxcWZ93Ovf75V3+LQoUOmdY888khQ\nswRivUVKdnZ2WJ7/ny/Vtjh16lS3bU8++aR++eWXbtvQdxxZAoABsQQAA2IJAAbEEgAMiCUAGBBL\nADAglgBgQCwBwIBYAoABsQQAA+7uGKPKy8vNa3u6hLEvlixZYl67Z88e89rk5GTTupaWFvM+X3/9\ndb/bv/rqK82bN6/LtsrKSvN+w6E3d43s6TJGLm8MD44sAcCAWAKAAbEEAANiCQAGxBIADIglABgQ\nSwAwIJYAYEAsAcCAG5ZFmbt375rWvfDCC+Z9fvvtt+a1Pd0wq729XYMGDer8ubGx0bzPtLQ089qf\nf/7ZtG7Dhg3mfVZVVfnd3pebsIXLjRs3zGsHDhwYxknwbxxZAoABsQQAA2IJAAbEEgAMiCUAGBBL\nADAglgBgQCwBwIBYAoABsQQAA25YFmWsV5/25hLG3vD5fKbHiouLzfv8448/zGu/+uor89pYsWDB\nAvPaaLv8Ev+PI0sAMDDFsrGxUTNnzuy8/WpeXp7mzZunpUuXaunSpWE7ygGAaBHwY/j169e1detW\nZWZmdtm+bt06ZWVlhW0wAIgmAY8s4+PjVVZWppSUlEjMAwBRKeCRpdvtltvdfVl5ebmOHDmipKQk\nFRQUaNiwYWEZ8L/G+gv++52ICZdbt25F/DnDzYm/R8SmoM6Gz58/X4mJicrIyNCBAwe0b98+bd68\nOdSz/SdZ//H29CW9fdVTrG/dutXlOVeuXGneZ7SeDY/Ul//25mz40aNHzWsHDOD8bCQF9bedmZmp\njIwMSdL06dN79a3ZABCLgorlqlWr1NTUJEmqra1Venp6SIcCgGgT8GN4fX29du3apebmZrndblVW\nVmrJkiVas2aNBg0aJI/Ho8LCwkjMCgCOCRjLUaNG6eOPP+62ffbs2WEZCACiEZc7ogvr5Y4lJSWR\nGKdf6M0JHk7aRC/eGQAwIJYAYEAsAcCAWAKAAbEEAANiCQAGxBIADIglABgQSwAwIJYAYMDljlHG\nernbsWPHzPvszeV24fiC3958MfT69etN6/Lz84MdJyS2bdtmXvvKK6+EcRJECkeWAGBALAHAgFgC\ngAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGDAFTxRxuVymdb15u6ap06dMq+9dOlSj49VV1eb93Ov\nsWPHmtfW1dUF9RyhMmbMGNO6lStXmvfJTcj6B95FADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAM\niCUAGBBLADAglgBg4Oro6Ohwegj0b21tbea1kyZNMq07c+ZMsON08vl8iouL67Lthx9+MP3Z5557\nrs/Pj9jCkSUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADDg7o4Iu5MnT5rX\nhuIyxn/Ly8szPzZ+/PiQPz/6B1Msi4qKVFdXpzt37mjFihV6+umntXHjRvl8PiUnJ2v37t2Kj48P\n96wA4JiAsTxx4oTOnj0rr9ertrY2LVy4UJmZmcrJydGcOXNUXFysiooK5eTkRGJeAHBEwN9Zjhs3\nTnv37pUkDR06VO3t7aqtrdWMGTMkSVlZWaqpqQnvlADgsICxjIuLk8fjkSRVVFRoypQpam9v7/zY\nnZSUpNbW1vBOCQAOM5/gOX78uCoqKnT48GHNmjWrcztfh4lAZs+ebV7r8/nCOEl327dvj+jzIXaZ\nYlldXa3S0lIdPHhQQ4YMkcfj0Y0bN5SQkKCWlhalpKSEe07EsMrKSvPauXPnhvz5ezobvn37dr3z\nzjtdtm3bts20T5fL1ee5EFsCfgy/evWqioqKtH//fiUmJkqSJk6c2PkPoKqqSpMnTw7vlADgsIBH\nlseOHVNbW5vWrFnTuW3nzp3atGmTvF6v0tLStGDBgrAOCQBOCxjL7OxsZWdnd9t+5MiRsAwEANGI\nG5YhKL25CVlGRoZ5bTj+Z8Xvv//ud/vIkSN17ty5btsAf7g2HAAMiCUAGBBLADAglgBgQCwBwIBY\nAoABsQQAA2IJAAbEEgAMiCUAGHDDMgSlrKzMvDYclzDm5uaa16alpQX1GHAvjiwBwIBYAoABsQQA\nA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABlzuiizt37vjd7na7uzz2+eefh+X5V61aZVr3\n/vvvm/fpcrl6fGzgwIHm/eC/jSNLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADBw\ndXR0dDg9BKLHd99953f7888/3+WxqVOnmvf58MMPm9eeOXPGtC4hIcG8TyAUOLIEAANiCQAGxBIA\nDIglABgQSwAwIJYAYEAsAcCAWAKAAbEEAANiCQAG3LAMXQwZMiSox+5ny5Yt5rVcxohoZYplUVGR\n6urqdOfOHa1YsULffPONGhoalJiYKElavny5pk2bFs45AcBRAWN54sQJnT17Vl6vV21tbVq4cKEm\nTJigdevWKSsrKxIzAoDjAsZy3LhxGj16tCRp6NCham9vl8/nC/tgABBNAp7giYuLk8fjkSRVVFRo\nypQpiouLU3l5uZYtW6a1a9fq0qVLYR8UAJxk/j7L48ePa//+/Tp8+LDq6+uVmJiojIwMHThwQH/9\n9Zc2b94c7lkBwDGmEzzV1dUqLS3VwYMHNWTIEGVmZnY+Nn36dL377rvhmg8Rdvr0ab/bx4wZ0+Wx\nZ555xrzPsrIy89rXXnvNvBaIpIAfw69evaqioiLt37+/8+z3qlWr1NTUJEmqra1Venp6eKcEAIcF\nPLI8duyY2tratGbNms5tixYt0po1azRo0CB5PB4VFhaGdUgAcFrAWGZnZys7O7vb9oULF4ZlIACI\nRlzuCAAG3N0RAAw4sgQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAM\niCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQA\nA2IJAAbEEgAM3E486Y4dO3T69Gm5XC7l5+dr9OjRTowRUrW1tVq9erXS09MlSU888YQKCgocnip4\njY2NevPNN/Xqq69qyZIlOn/+vDZu3Cifz6fk5GTt3r1b8fHxTo/ZK/9+TXl5eWpoaFBiYqIkafny\n5Zo2bZqzQ/ZSUVGR6urqdOfOHa1YsUJPP/10zL9PUvfX9c033zj+XkU8lidPntS5c+fk9Xr122+/\nKT8/X16vN9JjhMX48eNVUlLi9Bh9dv36dW3dulWZmZmd20pKSpSTk6M5c+aouLhYFRUVysnJcXDK\n3vH3miRp3bp1ysrKcmiqvjlx4oTOnj0rr9ertrY2LVy4UJmZmTH9Pkn+X9eECRMcf68i/jG8pqZG\nM2fOlCQ9/vjjunz5sq5duxbpMXAf8fHxKisrU0pKSue22tpazZgxQ5KUlZWlmpoap8YLir/XFOvG\njRunvXv3SpKGDh2q9vb2mH+fJP+vy+fzOTyVA7G8cOGCHnzwwc6fhw0bptbW1kiPERa//vqr3njj\nDS1evFjff/+90+MEze12KyEhocu29vb2zo9zSUlJMfee+XtNklReXq5ly5Zp7dq1unTpkgOTBS8u\nLk4ej0eSVFFRoSlTpsT8+yT5f11xcXGOv1eO/M7yXh0dHU6PEBKPPvqocnNzNWfOHDU1NWnZsmWq\nqqqKyd8XBdJf3rP58+crMTFRGRkZOnDggPbt26fNmzc7PVavHT9+XBUVFTp8+LBmzZrVuT3W36d7\nX1d9fb3j71XEjyxTUlJ04cKFzp///vtvJScnR3qMkEtNTdXcuXPlcrk0YsQIDR8+XC0tLU6PFTIe\nj0c3btyQJLW0tPSLj7OZmZnKyMiQJE2fPl2NjY0OT9R71dXVKi0tVVlZmYYMGdJv3qd/v65oeK8i\nHstJkyapsrJSktTQ0KCUlBQNHjw40mOE3JdffqlDhw5JklpbW3Xx4kWlpqY6PFXoTJw4sfN9q6qq\n0uTJkx2eqO9WrVqlpqYmSf/3O9l//idDrLh69aqKioq0f//+zrPE/eF98ve6ouG9cnU4cKy+Z88e\n/fjjj3K5XNqyZYuefPLJSI8QcteuXdP69et15coV3b59W7m5uZo6darTYwWlvr5eu3btUnNzs9xu\nt1JTU7Vnzx7l5eXp5s2bSktLU2FhoQYOHOj0qGb+XtOSJUt04MABDRo0SB6PR4WFhUpKSnJ6VDOv\n16sPP/xQjz32WOe2nTt3atOmTTH7Pkn+X9eiRYtUXl7u6HvlSCwBINZwBQ8AGBBLADAglgBgQCwB\nwIBYAoABsQQAA2IJAAbEEgAM/gepgR0uaefKmwAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "Prediction: 4\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAFKCAYAAACU6307AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAEelJREFUeJzt3W9MlfX/x/HXEWJyhg5BIG1ZfR0u\nKr3hhopOE2Q23FxiN0xCdNmGa5pG6hhTtNn8g85NtI0/aS1Z29moG96wILM2dYDKDRu0hrpyzCkC\nkUocDeH8brQfk8R4czyH64DPx624+Hid99nFnl2H61wHl8/n8wkA8J/GOD0AAIwExBIADIglABgQ\nSwAwIJYAYEAsAcCAWAKAAbEEAINwf//h7t27denSJblcLhUUFGjGjBmBnAsAQopfsTx//ryuXbsm\nj8ejq1evqqCgQB6PJ9CzAUDI8OtleE1NjdLT0yVJU6dO1e3bt9XZ2RnQwQAglPgVy7a2Nk2YMKHv\n65iYGLW2tgZsKAAINQG5wMNncQAY7fyKZXx8vNra2vq+vnXrluLi4gI2FACEGr9iOW/ePFVVVUmS\nGhsbFR8fr6ioqIAOBgChxK+r4TNnztSrr76qt99+Wy6XSzt27Aj0XAAQUlx8+C8ADI47eADAgFgC\ngAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCW\nAGBALAHAgFgCgAGxBAADYgkABsQSAAz8+lO4AJz3yy+/PLLtlVdeeWT777//bt7ne++9Z147f/58\n0zqPx2PeZyjjzBIADIglABgQSwAwIJYAYEAsAcCAWAKAAbEEAANiCQAGxBIADIglABi4fD6fz+kh\ngNHsr7/+Mq+tr683r33rrbce2dba2qq4uLh+29rb2837XL16tXntp59+alrndrvN+wxlnFkCgAGx\nBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAF/sAzww/37981rMzMzzWtPnTplXvu4O2O8\nXm+/rysrK837XLJkiXnt2LFjzWtHA84sAcDArzPLuro6bdy4UYmJiZKkadOmafv27QEdDABCid8v\nw2fNmqXi4uJAzgIAIYuX4QBg4Hcsr1y5onXr1mnlypU6d+5cIGcCgJDj1+dZtrS0qL6+XhkZGWpu\nblZOTo6qq6sVERERjBkBwHF+/c4yISGh7y0GU6ZM0cSJE9XS0qLnn38+oMMBoWoobx1aunSpee2T\nvnWos7NTUVFR/bZ9+eWX5n3y1qHH8+tl+IkTJ3T06FFJ/3wyc3t7uxISEgI6GACEEr/OLNPS0rR5\n82b98MMP6u7u1s6dO3kJDmBU8yuWUVFRKikpCfQsABCyuN0ReIj1vcNbtmwx77O7u9u8dii/9//x\nxx8H3P7zzz/3+/p///ufeZ94PN5nCQAGxBIADIglABgQSwAwIJYAYEAsAcCAWAKAAbEEAANiCQAG\nxBIADPz6PEvAaT09Pea1x48fH3D7mjVr9MUXX/Tblpuba9pnb2+v+fE/+eQT89qcnBzz2kmTJpnX\n4slxZgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABtzBgxHpcXflDGT16tUDbu/t\n7dWYMf6dL+zcudO8trCw0K/HQGjhzBIADIglABgQSwAwIJYAYEAsAcCAWAKAAbEEAANiCQAGxBIA\nDIglABhwuyNCSnFxsWndRx99ZN7n4/642UC3O77zzjumff77D539l7CwMPNahC7OLAHAgFgCgAGx\nBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAG3OyLovF6vee2kSZNM6+7cuePvOH0Gut2x\npqbG9G9nz579xI+PkcV0ZtnU1KT09HRVVFRIkm7cuKFVq1YpKytLGzdu1N9//x3UIQHAaYPGsqur\nS7t27VJKSkrftuLiYmVlZemrr77SCy+8oMrKyqAOCQBOGzSWERERKi8vV3x8fN+2uro6LVq0SJKU\nmppqfukCACNV+KALwsMVHt5/mdfrVUREhCQpNjZWra2twZkOAELEoLEcDNeHMJjIyEjz2j///DOI\nkzyqt7d3WB8PI5dfsXS73bp3757Gjh2rlpaWfi/RgX/jajhGA7/eZzl37lxVVVVJkqqrqzV//vyA\nDgUAoWbQM8uGhgbt27dP169fV3h4uKqqqnTgwAHl5+fL4/Fo8uTJWrZs2XDMCgCO4U3pCDpehmM0\neOILPHg6ffvtt+a1hw4dMq8NRASfRElJiWkdsXz6cG84ABgQSwAwIJYAYEAsAcCAWAKAAbEEAANi\nCQAGxBIADIglABgQSwAw4HZH+MV6W6D0zydTWU2ZMsW07v79++Z9trS0mNcCj8OZJQAYEEsAMCCW\nAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMOB2R/Rz4cKFAbcnJyf3+15tbW1QHv/77783\nrRvKX4FMTk72dxygD2eWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGDAHTzoZ8GC\nBQNu93q9/b43lD8YNhTWP1jm9XqD8vjA43BmCQAGxBIADIglABgQSwAwIJYAYEAsAcCAWAKAAbEE\nAANiCQAGxBIADLjd8Slw5MgR89r/uo3R31scZ8yYYV7rcrn8eoxAuXnzpmldV1eXeZ9ut9vfcRBC\nOLMEAANTLJuampSenq6KigpJUn5+vpYuXapVq1Zp1apV+umnn4I5IwA4btCX4V1dXdq1a5dSUlL6\nbc/Ly1NqamrQBgOAUDLomWVERITKy8sVHx8/HPMAQEhy+Xw+n2Xh4cOHNWHCBGVnZys/P1+tra3q\n7u5WbGystm/frpiYmGDPCgCO8etq+Jtvvqno6GglJSWprKxMR44cUWFhYaBnQ4AM5Wr4Bx98MOD2\n3t5ejRnj3/XAoVwNP3/+vGndUK5GP+5/5AM9pzfeeMO0z6+//tr8+FwNHx38+ulPSUlRUlKSJCkt\nLU1NTU0BHQoAQo1fsdywYYOam5slSXV1dUpMTAzoUAAQagZ9Gd7Q0KB9+/bp+vXrCg8PV1VVlbKz\ns7Vp0yZFRkbK7XZrz549wzErADhm0Fi+9tprOn78+CPbrb/bAYDRgNsdnwLt7e2OPv6WLVvMayMi\nIkzrhnKBZyiqqqpM63799VfzPmfOnOnvOAgh3O4IAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoAB\nsQQAA2IJAAbEEgAMuN0RfomNjTWvTU5ODvjjnz17NuD7lNT30YODee6554Ly+AhdnFkCgAGxBAAD\nYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAF38MAv48ePN6999tlnA/74FRUVAd+nJM2aNcu0\nLiEhISiPj9DFmSUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADDgdkf45bff\nfjOv/eabb8xrs7OzTet6e3vN+/T5fH59D3gYZ5YAYEAsAcCAWAKAAbEEAANiCQAGxBIADIglABgQ\nSwAwIJYAYEAsAcCA2x0RdO+++25Q1lq5XC6/vgc8zBTLoqIi1dfX68GDB8rNzdX06dO1detW9fT0\nKC4uTvv371dERESwZwUAxwway9raWl2+fFkej0cdHR3KzMxUSkqKsrKylJGRoYMHD6qyslJZWVnD\nMS8AOGLQ31kmJyfr0KFDkqTx48fL6/Wqrq5OixYtkiSlpqaqpqYmuFMCgMMGjWVYWJjcbrckqbKy\nUgsWLJDX6+172R0bG6vW1tbgTgkADjNf4Dl16pQqKyt17NgxLV68uG87nwcY+nbs2BGQtUP5DMmR\nYjQ+JwSHKZZnzpxRSUmJPvvsM40bN05ut1v37t3T2LFj1dLSovj4+GDPiSfw8ccfP/Ha3t5ejRkz\nut5pNtBzWr16tenffv7558EYCSFs0J/+u3fvqqioSKWlpYqOjpYkzZ07V1VVVZKk6upqzZ8/P7hT\nAoDDBj2zPHnypDo6OrRp06a+bXv37tW2bdvk8Xg0efJkLVu2LKhDAoDTBo3lihUrtGLFike28zIE\nwNOEO3ieAnl5eea1Fy5ceOz3lixZ0vffZ8+eNe/zzp075rVAqBpdv7EHgCAhlgBgQCwBwIBYAoAB\nsQQAA2IJAAbEEgAMiCUAGBBLADAglgBg4PLxgZTww3fffWde+/Btkk543I+4z+d75A+W1dbWmvY5\ne/bsJ54LIwtnlgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIDbHQHAgDNL\nADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbE\nEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAg3DLoqKiItXX1+vBgwfKzc3V6dOn1djYqOjoaEnS\n2rVrtXDhwmDOCQCOGjSWtbW1unz5sjwejzo6OpSZmak5c+YoLy9PqampwzEjADhu0FgmJydrxowZ\nkqTx48fL6/Wqp6cn6IMBQChx+Xw+n3Wxx+PRxYsXFRYWptbWVnV3dys2Nlbbt29XTExMMOcEAEeZ\nY3nq1CmVlpbq2LFjamhoUHR0tJKSklRWVqabN2+qsLAw2LMCgGNMV8PPnDmjkpISlZeXa9y4cUpJ\nSVFSUpIkKS0tTU1NTUEdEgCcNmgs7969q6KiIpWWlvZd/d6wYYOam5slSXV1dUpMTAzulADgsEEv\n8Jw8eVIdHR3atGlT37bly5dr06ZNioyMlNvt1p49e4I6JAA4bUgXeADgacUdPABgQCwBwIBYAoAB\nsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBg\nQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbhTjzo7t27\ndenSJblcLhUUFGjGjBlOjBFQdXV12rhxoxITEyVJ06ZN0/bt2x2eyn9NTU16//33tWbNGmVnZ+vG\njRvaunWrenp6FBcXp/379ysiIsLpMYfk388pPz9fjY2Nio6OliStXbtWCxcudHbIISoqKlJ9fb0e\nPHig3NxcTZ8+fcQfJ+nR53X69GnHj9Wwx/L8+fO6du2aPB6Prl69qoKCAnk8nuEeIyhmzZql4uJi\np8d4Yl1dXdq1a5dSUlL6thUXFysrK0sZGRk6ePCgKisrlZWV5eCUQzPQc5KkvLw8paamOjTVk6mt\nrdXly5fl8XjU0dGhzMxMpaSkjOjjJA38vObMmeP4sRr2l+E1NTVKT0+XJE2dOlW3b99WZ2fncI+B\n/xAREaHy8nLFx8f3baurq9OiRYskSampqaqpqXFqPL8M9JxGuuTkZB06dEiSNH78eHm93hF/nKSB\nn1dPT4/DUzkQy7a2Nk2YMKHv65iYGLW2tg73GEFx5coVrVu3TitXrtS5c+ecHsdv4eHhGjt2bL9t\nXq+37+VcbGzsiDtmAz0nSaqoqFBOTo4+/PBD/fHHHw5M5r+wsDC53W5JUmVlpRYsWDDij5M08PMK\nCwtz/Fg58jvLh/l8PqdHCIgXX3xR69evV0ZGhpqbm5WTk6Pq6uoR+fuiwYyWY/bmm28qOjpaSUlJ\nKisr05EjR1RYWOj0WEN26tQpVVZW6tixY1q8eHHf9pF+nB5+Xg0NDY4fq2E/s4yPj1dbW1vf17du\n3VJcXNxwjxFwCQkJWrJkiVwul6ZMmaKJEyeqpaXF6bECxu126969e5KklpaWUfFyNiUlRUlJSZKk\ntLQ0NTU1OTzR0J05c0YlJSUqLy/XuHHjRs1x+vfzCoVjNeyxnDdvnqqqqiRJjY2Nio+PV1RU1HCP\nEXAnTpzQ0aNHJUmtra1qb29XQkKCw1MFzty5c/uOW3V1tebPn+/wRE9uw4YNam5ulvTP72T//50M\nI8Xdu3dVVFSk0tLSvqvEo+E4DfS8QuFYuXwOnKsfOHBAFy9elMvl0o4dO/Tyyy8P9wgB19nZqc2b\nN+vOnTvq7u7W+vXr9frrrzs9ll8aGhq0b98+Xb9+XeHh4UpISNCBAweUn5+v+/fva/LkydqzZ4+e\neeYZp0c1G+g5ZWdnq6ysTJGRkXK73dqzZ49iY2OdHtXM4/Ho8OHDeumll/q27d27V9u2bRuxx0ka\n+HktX75cFRUVjh4rR2IJACMNd/AAgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHA4P8ALqDX\nN3rmU3AAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "Prediction: 1\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAFKCAYAAACU6307AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAEqVJREFUeJzt3W9Ilff/x/HX+eWkpMQ0dQRrZdgm\nq24Miiz6Y0nrFKPVjZqiMgiW/SMX0ZxlDYJMiyALZrnqRlKc4a1u5B9cjIWZUbDA7ljWQqJMm1iR\nbSbne2P8/H7NY77P8Ryvoz0f97y8us777BpPrnMuP+e4vF6vVwCAd/o/pwcAgNGAWAKAAbEEAANi\nCQAGxBIADIglABgQSwAwIJYAYBAR6D88dOiQbt++LZfLpYKCAs2dOzeYcwFAWAkoljdu3NDDhw/l\n8XjU0tKigoICeTyeYM8GAGEjoJfhDQ0NSk9PlyTNnDlTXV1devnyZVAHA4BwElAsOzo6NHny5L6f\nY2Nj1d7eHrShACDcBOUGD5/FAWCsCyiWCQkJ6ujo6Pv56dOnio+PD9pQABBuAorlokWLVFNTI0m6\nc+eOEhISNHHixKAOBgDhJKC74Z9//rk+++wzff3113K5XDpw4ECw5wKAsOLiw38BYGis4AEAA2IJ\nAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBY\nAoABsQQAA2IJAAbEEgAMiCUAGBBLADAI6KtwgVC5ePGiab+9e/eaj/ngwQOf271er1wul/k4gWpp\naTHvm5SUFMJJMBxcWQKAAbEEAANiCQAGxBIADIglABgQSwAwIJYAYEAsAcCAWAKAAbEEAAOWOyIg\n9+/fD8lxMzMzTfutWrXKfMzBljv6MmPGjKAf88mTJ+Z9We4YvriyBAADYgkABsQSAAyIJQAYEEsA\nMCCWAGBALAHAgFgCgAGxBAADVvAgIOnp6eZ9/VntYrV06VLzvh6PZ9DfdXV19fs5OjradMwtW7aY\nH3/27NnmfRG+uLIEAIOAriwbGxu1c+dOJScnS5JmzZqlwsLCoA4GAOEk4Jfh8+fPV2lpaTBnAYCw\nxctwADAIOJb37t1Tbm6uMjIyVF9fH8yZACDsuLxer9fff9TW1qZbt27J7XartbVVOTk5qq2tVWRk\nZChmBADHBfSeZWJiolavXi1JmjZtmqZMmaK2tjZ99NFHQR0O4cufD6kNxZ8OFRUVmffdunWrz+3R\n0dF6/vz5gG0W/vzpUHFxsXlf6+Nj5AX0MvzSpUs6c+aMJKm9vV3Pnj1TYmJiUAcDgHAS0JXl8uXL\ntXv3bv3666/q6enRjz/+yEtwAGNaQLGcOHGiysrKgj0LAIStgG7wYHR5+325d9m4caPP7VVVVXK7\n3X0/V1dXD3suX6zvRebn54fk8YHB8HeWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCW\nAGBALAHAgOWO7wF/Pk5ssDX/Xq9XLpcroMf35+PUWMaIcMWVJQAYEEsAMCCWAGBALAHAgFgCgAGx\nBAADYgkABsQSAAyIJQAYsIJnlLp27Zp530WLFg378d5ewXPhwgXzv83IyBj24wNO48oSAAyIJQAY\nEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYRDg9APp7/vy5ab9gLGH0JTc31/Q7ljDi\nfcOVJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMODbHcOM2+027VddXW0+\n5qpVq8z7ejwen9ujo6P7LcWMjo42HxMYC0xXls3NzUpPT1dFRYUk6fHjx8rOzlZmZqZ27typf/75\nJ6RDAoDThozlq1evdPDgQaWmpvZtKy0tVWZmpi5cuKCPP/5YlZWVIR0SAJw2ZCwjIyNVXl6uhISE\nvm2NjY1asWKFJCktLU0NDQ2hmxAAwsCQH9EWERGhiIj+u3V3dysyMlKSFBcXp/b29tBMBwBhYtif\nZ8n9oeCqqqpyeoRBcVMH77OAYhkVFaXXr19r/Pjxamtr6/cSHcPD3XAgPAX0d5YLFy5UTU2NJKm2\ntlaLFy8O6lAAEG6GvLJsampScXGxHj16pIiICNXU1Ojo0aPKz8+Xx+PR1KlT9dVXX43ErADgmCFj\nOXv2bJ0/f37A9nPnzoVkIAAIR6zgGQH379837ztz5sygP35LS4t536SkpKA/PjAWsDYcAAyIJQAY\nEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYDPvzLDG0I0eOBP2Yubm55n1ZwggMH1eW\nAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgOWOI6Cmpibox8zOzg76Mceq\nwb5dMykpacDvrEtT//zzT/PjT58+3byvP/+vfPLJJwO2VVVVye1299uWk5NjPuaaNWvM+0ZHR5v3\nHQu4sgQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA5fX6/U6PcRY588Xhj148MC0\nX0tLS0ge32kXL1407bd3717zMQf7b+r1euVyuczHGQ2G+5xWrVpl3tfj8Zj2GysrfbiyBAADYgkA\nBsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABix3HAFbtmwx71tWVmbabzSdtlAs9wyG\n4SwN9GdZYHV1dUCPEYiRXMJpXXI7mpbbvgtXlgBgYIplc3Oz0tPTVVFRIUnKz8/Xl19+qezsbGVn\nZ+u3334L5YwA4Lghvzf81atXOnjwoFJTU/tt37Vrl9LS0kI2GACEkyGvLCMjI1VeXq6EhISRmAcA\nwpL5Bs+JEyc0efJkZWVlKT8/X+3t7erp6VFcXJwKCwsVGxsb6lkBwDFDvgz3Ze3atYqJiVFKSopO\nnz6tkydPav/+/cGebczgbjh3w0cKd8NDJ6C74ampqUpJSZEkLV++XM3NzUEdCgDCTUCx3LFjh1pb\nWyVJjY2NSk5ODupQABBuhnwZ3tTUpOLiYj169EgRERGqqalRVlaW8vLyNGHCBEVFRamoqGgkZgUA\nxwwZy9mzZ+v8+fMDtn/xxRchGQgAwlFAN3gAt9tt3jcUN238eTWzYcOGQX/39k2KKVOmBDzTYEL1\n7YbPnz/3ub2rq6vfz99//735mNYbjJK0bds2035VVVXmY4YzljsCgAGxBAADYgkABsQSAAyIJQAY\nEEsAMCCWAGBALAHAgFgCgAGxBAADljuOUteuXTPvu3DhwmEfd+HChf1+F6rPaKyvrzft589zepfR\n/FmLgy2jfHv7Tz/9ZD6mP8sd3zdcWQKAAbEEAANiCQAGxBIADIglABgQSwAwIJYAYEAsAcCAWAKA\nASt4RkBxcbF535qaGtN+WVlZ5mP+8ccf5n19fZOn9O+KmcF+NxR/vlwsWCtz8F/+rPbyR2FhYUiO\nG664sgQAA2IJAAbEEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAYur9frdXoI/Jd1adqi\nRYtCPEl/Xq9XLpcroH/b1dVl3newL+HCQBcvXhywLSMjY8D2zMxM8zEvXLhg3nfNmjWm/cbKOeXK\nEgAMiCUAGBBLADAglgBgQCwBwIBYAoABsQQAA2IJAAbEEgAMiCUAGLDccZTy5xv7grE0cjjLHf35\ndseHDx+a9svOzjYf88MPP/S5PSkpSffv3++37ZdffjEdc8mSJebH98fBgwfN+1ZXVw/YNpzzJEn1\n9fXmfd+3b+I0fRVuSUmJbt26pTdv3mjz5s2aM2eO9uzZo97eXsXHx+vIkSOKjIwM9awA4JghY3n9\n+nXdvXtXHo9HnZ2dWrdunVJTU5WZmSm3261jx46psrLSr8X6ADDaDPme5bx583T8+HFJ/356SHd3\ntxobG7VixQpJUlpamhoaGkI7JQA4bMhYjhs3TlFRUZKkyspKLVmyRN3d3X0vu+Pi4tTe3h7aKQHA\nYab3LCWprq5OlZWVOnv2rFauXNm3nftDzvDnzfVgnaOxeK6TkpL6/Zyfn+/QJP+qqqoa9jHG4nkK\nB6ZYXr16VWVlZfr55581adIkRUVF6fXr1xo/frza2tqUkJAQ6jnxFu6Gczecu+Eja8iX4S9evFBJ\nSYlOnTqlmJgYSf/+R6qpqZEk1dbWavHixaGdEgAcNuSV5eXLl9XZ2am8vLy+bYcPH9a+ffvk8Xg0\ndepUffXVVyEdEgCcNmQsN27cqI0bNw7Yfu7cuZAMBADhiBU874G335d7l23btvncXlVVJbfb3fez\nr/fLRpvhvr/ntBkzZgzYdv/+/QE3rerq6szHnDJlinnfsfJFZFasDQcAA2IJAAbEEgAMiCUAGBBL\nADAglgBgQCwBwIBYAoABsQQAA2IJAAYsd0RA/PmIuPPnz5v3tX702u+//24+5g8//OBzu6/ljr6W\nEPry7bffmh9/w4YN5n398fayRoQWV5YAYEAsAcCAWAKAAbEEAANiCQAGxBIADIglABgQSwAwIJYA\nYEAsAcCA5Y4AYMCVJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBA\nLAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgEGEZaeSkhLdunVLb9680ebN\nm3XlyhXduXNHMTExkqRNmzZp2bJloZwTABw1ZCyvX7+uu3fvyuPxqLOzU+vWrdOCBQu0a9cupaWl\njcSMAOC4IWM5b948zZ07V5IUHR2t7u5u9fb2hnwwAAgnLq/X67Xu7PF4dPPmTY0bN07t7e3q6elR\nXFycCgsLFRsbG8o5AcBR5ljW1dXp1KlTOnv2rJqamhQTE6OUlBSdPn1aT5480f79+0M9KwA4xnQ3\n/OrVqyorK1N5ebkmTZqk1NRUpaSkSJKWL1+u5ubmkA4JAE4bMpYvXrxQSUmJTp061Xf3e8eOHWpt\nbZUkNTY2Kjk5ObRTAoDDhrzBc/nyZXV2diovL69v2/r165WXl6cJEyYoKipKRUVFIR0SAJzm1w0e\nAHhfsYIHAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHA\ngFgCgAGxBAADYgkABsQSAAyIJQAYEEsAMCCWAGBALAHAgFgCgAGxBAADYgkABsQSAAyIJQAYEEsA\nMCCWAGBALAHAIMKJBz106JBu374tl8ulgoICzZ0714kxgqqxsVE7d+5UcnKyJGnWrFkqLCx0eKrA\nNTc3a+vWrfrmm2+UlZWlx48fa8+ePert7VV8fLyOHDmiyMhIp8f0y9vPKT8/X3fu3FFMTIwkadOm\nTVq2bJmzQ/qppKREt27d0ps3b7R582bNmTNn1J8naeDzunLliuPnasRjeePGDT18+FAej0ctLS0q\nKCiQx+MZ6TFCYv78+SotLXV6jGF79eqVDh48qNTU1L5tpaWlyszMlNvt1rFjx1RZWanMzEwHp/SP\nr+ckSbt27VJaWppDUw3P9evXdffuXXk8HnV2dmrdunVKTU0d1edJ8v28FixY4Pi5GvGX4Q0NDUpP\nT5ckzZw5U11dXXr58uVIj4F3iIyMVHl5uRISEvq2NTY2asWKFZKktLQ0NTQ0ODVeQHw9p9Fu3rx5\nOn78uCQpOjpa3d3do/48Sb6fV29vr8NTORDLjo4OTZ48ue/n2NhYtbe3j/QYIXHv3j3l5uYqIyND\n9fX1To8TsIiICI0fP77ftu7u7r6Xc3FxcaPunPl6TpJUUVGhnJwcfffdd/rrr78cmCxw48aNU1RU\nlCSpsrJSS5YsGfXnSfL9vMaNG+f4uXLkPcv/5fV6nR4hKKZPn67t27fL7XartbVVOTk5qq2tHZXv\nFw1lrJyztWvXKiYmRikpKTp9+rROnjyp/fv3Oz2W3+rq6lRZWamzZ89q5cqVfdtH+3n63+fV1NTk\n+Lka8SvLhIQEdXR09P389OlTxcfHj/QYQZeYmKjVq1fL5XJp2rRpmjJlitra2pweK2iioqL0+vVr\nSVJbW9uYeDmbmpqqlJQUSdLy5cvV3Nzs8ET+u3r1qsrKylReXq5JkyaNmfP09vMKh3M14rFctGiR\nampqJEl37txRQkKCJk6cONJjBN2lS5d05swZSVJ7e7uePXumxMREh6cKnoULF/adt9raWi1evNjh\niYZvx44dam1tlfTve7L//5cMo8WLFy9UUlKiU6dO9d0lHgvnydfzCodz5fI6cK1+9OhR3bx5Uy6X\nSwcOHNCnn3460iME3cuXL7V79249f/5cPT092r59u5YuXer0WAFpampScXGxHj16pIiICCUmJuro\n0aPKz8/X33//ralTp6qoqEgffPCB06Oa+XpOWVlZOn36tCZMmKCoqCgVFRUpLi7O6VHNPB6PTpw4\noRkzZvRtO3z4sPbt2zdqz5Pk+3mtX79eFRUVjp4rR2IJAKMNK3gAwIBYAoABsQQAA2IJAAbEEgAM\niCUAGBBLADAglgBg8B9OkjtgR8VvdgAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "Prediction: 6\n" - ], - "name": "stdout" - } - ] - }, - { - "metadata": { - "id": "4SJizeJtNaAs", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "# Profiling\n", - "\n", - "If you want to drill down into the performance characteristics of your code, you can use native Python profilers like [`cProfile`](https://docs.python.org/3/library/profile.html). In the next exercise, you'll do just that." - ] - }, - { - "metadata": { - "id": "_2v0QnG8__PJ", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Exercise!\n", - "\n", - "This exercise does not require coding. If you have not completed the training exercise, replace `train_one_epoch` below with `_train_one_epoch`.\n", - "\n", - "Run the below cell and inspect the printed profiles. What parts of the code appear to be hotspots or\n", - "bottlenecks? How does sorting the profile by total time compare to sorting it\n", - "by cumulative time?\n", - "\n" - ] - }, - { - "metadata": { - "id": "IFypaYbG_9fB", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 714 - }, - "outputId": "d9c3596b-a165-4edd-fc6b-53ccd0d01d19" - }, - "cell_type": "code", - "source": [ - "import cProfile\n", - "import pstats\n", - "\n", - "cProfile.run(\"train_one_epoch(model, training_data, optimizer)\", \"training_profile\")\n", - "\n", - "stats = pstats.Stats(\"training_profile\").strip_dirs().sort_stats(\"tottime\")\n", - "stats.print_stats(10)\n", - "\n", - "stats.sort_stats(\"cumtime\").print_stats(10)" - ], - "execution_count": 17, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Thu Jun 7 12:25:04 2018 training_profile\n", - "\n", - " 92209 function calls (91817 primitive calls) in 3.446 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 672 to 10 due to restriction <10>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1080 2.552 0.002 2.552 0.002 {built-in method _pywrap_tensorflow_internal.TFE_Py_FastPathExecute}\n", - " 83 0.753 0.009 0.753 0.009 {built-in method _pywrap_tensorflow_internal.TFE_Py_Execute}\n", - " 16 0.006 0.000 1.019 0.064 network.py:736(_run_internal_graph)\n", - " 16 0.005 0.000 2.253 0.141 {built-in method _pywrap_tensorflow_internal.TFE_Py_TapeGradient}\n", - " 2321 0.004 0.000 0.007 0.000 abc.py:178(__instancecheck__)\n", - " 288 0.004 0.000 0.009 0.000 inspect.py:2092(_signature_from_function)\n", - " 878 0.004 0.000 0.005 0.000 ops.py:5936(__enter__)\n", - " 288 0.004 0.000 0.016 0.000 inspect.py:1079(getfullargspec)\n", - " 11006 0.003 0.000 0.005 0.000 {built-in method builtins.isinstance}\n", - " 768 0.003 0.000 0.008 0.000 {built-in method _pywrap_tensorflow_internal.Flatten}\n", - "\n", - "\n", - "Thu Jun 7 12:25:04 2018 training_profile\n", - "\n", - " 92209 function calls (91817 primitive calls) in 3.446 seconds\n", - "\n", - " Ordered by: cumulative time\n", - " List reduced from 672 to 10 due to restriction <10>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.000 0.000 3.446 3.446 {built-in method builtins.exec}\n", - " 1 0.000 0.000 3.446 3.446 :1()\n", - " 1 0.001 0.001 3.446 3.446 :9(train_one_epoch)\n", - " 1080 2.552 0.002 2.552 0.002 {built-in method _pywrap_tensorflow_internal.TFE_Py_FastPathExecute}\n", - " 16 0.000 0.000 2.255 0.141 backprop.py:739(gradient)\n", - " 16 0.000 0.000 2.253 0.141 imperative_grad.py:31(imperative_grad)\n", - " 16 0.005 0.000 2.253 0.141 {built-in method _pywrap_tensorflow_internal.TFE_Py_TapeGradient}\n", - " 400 0.002 0.000 2.246 0.006 backprop.py:145(grad_fn)\n", - " 400 0.002 0.000 2.239 0.006 backprop.py:95(_magic_gradient_function)\n", - " 32 0.001 0.000 1.601 0.050 nn_grad.py:497(_Conv2DGrad)\n", - "\n", - "\n" - ], - "name": "stdout" - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 17 - } - ] - }, - { - "metadata": { - "id": "8ixpnyCNNTI4", - "colab_type": "code", - "colab": {} - }, - "cell_type": "code", - "source": [ - "" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/tensorflow/contrib/eager/python/examples/workshop/3_inspecting.ipynb b/tensorflow/contrib/eager/python/examples/workshop/3_inspecting.ipynb deleted file mode 100644 index 64d19ec5c9b..00000000000 --- a/tensorflow/contrib/eager/python/examples/workshop/3_inspecting.ipynb +++ /dev/null @@ -1,443 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "Debugging \"graph-first\" models with eager execution", - "version": "0.3.2", - "provenance": [], - "include_colab_link": true - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "view-in-github", - "colab_type": "text" - }, - "source": [ - "[View in Colaboratory](https://colab.research.google.com/gist/alextp/9568ab40f6ed6f9a3ba4736f6aef6127/debugging-graph-first-models-with-eager-execution.ipynb)" - ] - }, - { - "metadata": { - "id": "mm-t0GuIu1Dt", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "This colab uses eager execution and the Python debugger to modify the execution of a translation model. This combination lets you quickly explore counterfactuals when researching and designing modifications to a model.\n", - "\n", - "The model, Transformer from [Tensor2Tensor](https://github.com/tensorflow/tensor2tensor), was originally written with graph building in mind. Executing it eagerly can still be helpful!" - ] - }, - { - "metadata": { - "id": "gxb1DvIDg4sv", - "colab_type": "code", - "colab": {} - }, - "cell_type": "code", - "source": [ - "#@title License (double click to show)\n", - "# Copyright 2018 The TensorFlow Authors.\n", - "\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "Gx3HA9N1ui64", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 37 - }, - "outputId": "f6986f34-f3e1-44e1-c902-2eb33081acad" - }, - "cell_type": "code", - "source": [ - "import tensorflow as tf\n", - "import pdb\n", - "tfe = tf.contrib.eager\n", - "\n", - "tf.enable_eager_execution()" - ], - "execution_count": 1, - "outputs": [] - }, - { - "metadata": { - "id": "3LkOm2ct-Lmc", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 37 - }, - "outputId": "2edc74d9-6bc0-4e78-ab4e-83bf96099ef4" - }, - "cell_type": "code", - "source": [ - "!pip install -q -U tensor2tensor\n", - "from tensor2tensor.models import transformer" - ], - "execution_count": 2, - "outputs": [] - }, - { - "metadata": { - "id": "1Z3oMsqV0zB6", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 170 - }, - "outputId": "0a8186ee-c688-457f-c9f6-9a6c1477a93b" - }, - "cell_type": "code", - "source": [ - "#@title Create a tensor2tensor translation model, fetch a checkpoint (double click to show)\n", - "from tensor2tensor import problems\n", - "from tensor2tensor.utils import trainer_lib\n", - "from tensor2tensor.utils import registry\n", - "\n", - "import numpy as np\n", - "import os\n", - "\n", - "# Setup some directories\n", - "data_dir = os.path.expanduser(\"~/t2t/data\")\n", - "tmp_dir = os.path.expanduser(\"~/t2t/tmp\")\n", - "train_dir = os.path.expanduser(\"~/t2t/train\")\n", - "checkpoint_dir = os.path.expanduser(\"~/t2t/checkpoints\")\n", - "tf.gfile.MakeDirs(data_dir)\n", - "tf.gfile.MakeDirs(tmp_dir)\n", - "tf.gfile.MakeDirs(train_dir)\n", - "tf.gfile.MakeDirs(checkpoint_dir)\n", - "gs_data_dir = \"gs://tensor2tensor-data\"\n", - "gs_ckpt_dir = \"gs://tensor2tensor-checkpoints/\"\n", - "\n", - "# Fetch the problem\n", - "ende_problem = problems.problem(\"translate_ende_wmt32k\")\n", - "\n", - "# Copy the vocab file locally so we can encode inputs and decode model outputs\n", - "# All vocabs are stored on GCS\n", - "vocab_name = \"vocab.ende.32768\"\n", - "vocab_file = os.path.join(gs_data_dir, vocab_name)\n", - "!gsutil cp {vocab_file} {data_dir}\n", - "\n", - "# Get the encoders from the problem\n", - "encoders = ende_problem.feature_encoders(data_dir)\n", - "\n", - "# Setup helper functions for encoding and decoding\n", - "def encode(input_str, output_str=None):\n", - " \"\"\"Input str to features dict, ready for inference\"\"\"\n", - " inputs = encoders[\"inputs\"].encode(input_str) + [1] # add EOS id\n", - " batch_inputs = tf.reshape(inputs, [1, -1, 1]) # Make it 3D.\n", - " return {\"inputs\": batch_inputs}\n", - "\n", - "def decode(integers):\n", - " \"\"\"List of ints to str\"\"\"\n", - " integers = list(np.squeeze(integers))\n", - " if 1 in integers:\n", - " integers = integers[:integers.index(1)]\n", - " return encoders[\"inputs\"].decode(np.squeeze(integers))\n", - "\n", - "# Copy the pretrained checkpoint locally\n", - "ckpt_name = \"transformer_ende_test\"\n", - "gs_ckpt = os.path.join(gs_ckpt_dir, ckpt_name)\n", - "!gsutil -q cp -R {gs_ckpt} {checkpoint_dir}\n", - "checkpoint_path = tf.train.latest_checkpoint(\n", - " os.path.join(checkpoint_dir, ckpt_name))\n", - "\n", - "# Create hparams and the model\n", - "model_name = \"transformer\"\n", - "hparams_set = \"transformer_base\"\n", - "\n", - "hparams = trainer_lib.create_hparams(hparams_set, data_dir=data_dir, problem_name=\"translate_ende_wmt32k\")\n", - "\n", - "# NOTE: Only create the model once when restoring from a checkpoint; it's a\n", - "# Layer and so subsequent instantiations will have different variable scopes\n", - "# that will not match the checkpoint.\n", - "translate_model = registry.model(model_name)(hparams, tf.estimator.ModeKeys.EVAL)" - ], - "execution_count": 3, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Copying gs://tensor2tensor-data/vocab.ende.32768...\n", - "/ [1 files][316.4 KiB/316.4 KiB] \n", - "Operation completed over 1 objects/316.4 KiB. \n", - "INFO:tensorflow:Setting T2TModel mode to 'eval'\n", - "INFO:tensorflow:Setting hparams.layer_prepostprocess_dropout to 0.0\n", - "INFO:tensorflow:Setting hparams.symbol_dropout to 0.0\n", - "INFO:tensorflow:Setting hparams.attention_dropout to 0.0\n", - "INFO:tensorflow:Setting hparams.dropout to 0.0\n", - "INFO:tensorflow:Setting hparams.relu_dropout to 0.0\n" - ], - "name": "stdout" - } - ] - }, - { - "metadata": { - "id": "4IblPXLGjuCl", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "We've created a Transformer model and fetched an existing training checkpoint. It hasn't created variables yet, and we want to load them from the checkpoint before they're used (restore-on-create) so the first run of the model outputs the correct value. The `tfe.restore_variables_on_create` API looks up variables by name on creation and restores their values." - ] - }, - { - "metadata": { - "id": "o3MWxcAqJoqG", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 51 - }, - "outputId": "fbc1b1bf-ffbe-4621-b3cb-5eb855fec3a8" - }, - "cell_type": "code", - "source": [ - "with tfe.restore_variables_on_create(checkpoint_path):\n", - " model_output = translate_model.infer(encode(\"Eager execution\"))\n", - "print(decode(model_output[\"outputs\"]))" - ], - "execution_count": 4, - "outputs": [ - { - "output_type": "stream", - "text": [ - "INFO:tensorflow:Greedy Decoding\n", - "Hinrichtung\n" - ], - "name": "stdout" - } - ] - }, - { - "metadata": { - "id": "xk5HV9Hhu9zO", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "Using global variable names can get somewhat fragile, so for new code we recommend the object-based `tf.keras.Model.save_weights` or `tf.train.Checkpoint`. However, these require some small code changes to work with existing graph building code.\n", - "\n", - "The Transformer model translates \"Eager execution\" in English to \"Hinrichtung\" in German, which refers to capital punishment rather than getting things done. Transformer first encodes the English, then decodes to German. We'll add a debugging hook at the start of the decode phase (once the encodings have been finalized) and see if we can correct the translation." - ] - }, - { - "metadata": { - "id": "GUGwbYvXZ9-7", - "colab_type": "code", - "colab": {} - }, - "cell_type": "code", - "source": [ - "previous_fast_decode = transformer.fast_decode\n", - "def debug_fn(*args, **kwargs):\n", - " pdb.set_trace()\n", - " return previous_fast_decode(*args, **kwargs) # \"step\" in pdb to step in\n", - "transformer.fast_decode = debug_fn # Add our debugging hook to Transformer" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "f61HlvECxJn0", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "Now that we've \"monkey patched\" the model, we'll drop into a debugger just before decoding starts. In most cases it'd be simpler to add the `pdb.set_trace()` call to the code directly, but in this case we're working with prepackaged library code.\n", - "\n", - "First, let's find an encoding which represents the correct sense of \"execution\". Then we'll patch part of that encoding into the encoding of \"Eager execution\" to fix the translation. Feel free to poke around with the debugger (e.g. print a Tensor's value), but your main task is to save the encodings by assigning them to an attribute of the function:\n", - "\n", - "```\n", - "(running the next cell drops you into a pdb shell)\n", - "step\n", - "fast_decode.previous_encoding = encoder_output\n", - "continue\n", - "\n", - "```\n", - "\n", - "You can type `next` (or `n`) a few times before `continue` to watch the decoding ops run." - ] - }, - { - "metadata": { - "id": "dX4CPOGSpZrb", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 179 - }, - "outputId": "6de38c31-836f-40ef-b701-e42908172619" - }, - "cell_type": "code", - "source": [ - "model_output = translate_model.infer(encode(\"Immediate running\"))\n", - "print(decode(model_output[\"outputs\"]))" - ], - "execution_count": 7, - "outputs": [ - { - "output_type": "stream", - "text": [ - "> (4)debug_fn()\n", - "-> return previous_fast_decode(*args, **kwargs) # \"step\" in pdb to step in\n", - "(Pdb) step\n", - "--Call--\n", - "> /usr/local/lib/python2.7/dist-packages/tensor2tensor/models/transformer.py(427)fast_decode()\n", - "-> def fast_decode(encoder_output,\n", - "(Pdb) fast_decode.previous_encoding = encoder_output\n", - "(Pdb) continue\n", - "Sofortige Durchführung\n" - ], - "name": "stdout" - } - ] - }, - { - "metadata": { - "id": "-ZEZciV4FpLo", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "Now we have an encoding saved which gets the correct sense for \"execution\"." - ] - }, - { - "metadata": { - "id": "QeC_oDVqHD_v", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 179 - }, - "outputId": "253c9af1-003e-46bd-8bf5-db968cf6a8cf" - }, - "cell_type": "code", - "source": [ - "# Assumes you followed the pdb instructions above!\n", - "transformer.fast_decode.previous_encoding" - ], - "execution_count": 8, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 8 - } - ] - }, - { - "metadata": { - "id": "bC9JjeDcHEav", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "Let's replace part of the encoding for \"Eager execution\" with the encoding of \"Immediate running\".\n", - "\n", - "Again we'll drop into a pdb shell. This time we'll run some TensorFlow operations to patch the encodings while the model is running.\n", - "\n", - "```\n", - "(running the next cell again drops you into a pdb shell)\n", - "step\n", - "encoder_output = tf.concat([fast_decode.previous_encoding[:, :3], encoder_output[:, 3:]], axis=1)\n", - "continue\n", - "```" - ] - }, - { - "metadata": { - "id": "t2as_Kn1h65G", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 179 - }, - "outputId": "5b4e546e-3bb4-4761-c545-467b631e3ffe" - }, - "cell_type": "code", - "source": [ - "model_output = translate_model.infer(encode(\"Eager execution\"))\n", - "print(decode(model_output[\"outputs\"]))" - ], - "execution_count": 9, - "outputs": [ - { - "output_type": "stream", - "text": [ - "> (4)debug_fn()\n", - "-> return previous_fast_decode(*args, **kwargs) # \"step\" in pdb to step in\n", - "(Pdb) step\n", - "--Call--\n", - "> /usr/local/lib/python2.7/dist-packages/tensor2tensor/models/transformer.py(427)fast_decode()\n", - "-> def fast_decode(encoder_output,\n", - "(Pdb) encoder_output = tf.concat([fast_decode.previous_encoding[:, :3], encoder_output[:, 3:]], axis=1)\n", - "(Pdb) continue\n", - "sofortige Ausführung\n" - ], - "name": "stdout" - } - ] - }, - { - "metadata": { - "id": "rK6tYZ23I2cm", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "We get a different decoding, with the correct sense of \"execution\". Likely we're keeping just the encoding of \"tion\" from \"Eager execution\", so no great breakthrough in translation modeling.\n", - "\n", - "Similarly it's possible to modify attention vectors, or change words during decoding to help debug a beam search." - ] - }, - { - "metadata": { - "id": "Nb-4ipYNRWxA", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "This colab was adapted from the [Tensor2Tensor colab](https://colab.research.google.com/github/tensorflow/tensor2tensor/blob/master/tensor2tensor/notebooks/hello_t2t.ipynb). Credit to Ankur Taly for its concept." - ] - } - ] -} \ No newline at end of file diff --git a/tensorflow/contrib/eager/python/g3doc/guide.md b/tensorflow/contrib/eager/python/g3doc/guide.md deleted file mode 100644 index 23f33d0230b..00000000000 --- a/tensorflow/contrib/eager/python/g3doc/guide.md +++ /dev/null @@ -1,18 +0,0 @@ -# Eager execution - -Eager execution is a feature that makes TensorFlow execute operations -immediately: concrete values are returned, instead of creating a computational -graph that is executed later. - -A user guide is available: https://www.tensorflow.org/guide/eager -([source file](../../../../docs_src/guide/eager.md)) - -We welcome feedback through [GitHub issues](https://github.com/tensorflow/tensorflow/labels/comp:eager). - -Sample code is available, including benchmarks for some: - -- [Linear Regression](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/linear_regression) -- [MNIST handwritten digit classifier](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist) -- [ResNet50 image classification](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/resnet50) -- [RNN to generate colors](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_colorbot) -- [RNN language model](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_ptb) diff --git a/tensorflow/contrib/eager/python/metrics.py b/tensorflow/contrib/eager/python/metrics.py deleted file mode 100644 index 04b7b1165e1..00000000000 --- a/tensorflow/contrib/eager/python/metrics.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Metrics namespace.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint:disable=wildcard-import -from tensorflow.contrib.eager.python.metrics_impl import * -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = ['Accuracy', 'Mean', 'Metric', 'CategoricalAccuracy', - 'BinaryAccuracy', 'SparseAccuracy'] -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py deleted file mode 100644 index 7885eb84f04..00000000000 --- a/tensorflow/contrib/eager/python/metrics_impl.py +++ /dev/null @@ -1,547 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Metrics classes for computing the output of an evaluation.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re - -from tensorflow.python.eager import context -from tensorflow.python.eager import function -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import smart_cond -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import summary_ops_v2 as summary_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.training.tracking import base as trackable - -_to_replace = re.compile("[^A-Za-z0-9.]") - - -class Metric(trackable.Trackable): - """A metric holds state for aggregating statistics over an evaluation run. - - Example use with eager execution: - - ```python - m = SomeMetric(...) - for input in ...: - m(input) - print(m.result()) - ``` - - Example use with graph execution: - - ```python - m = SomeMetric(...) - inputs = ... # Some tensors to compute the metric on. - m_update = m(inputs) - # Variables defined in first call, so get the initialization op afterwards. - m_init = m.init_variables() # or tf.compat.v1.global_variables_initializer() - m_result = m.result() - with tf.compat.v1.Session() as sess: - sess.run(m_init) - for input in ...: - sess.run(m_update) - print(sess.run(m_result)) - ``` - Example use with graph execution with placeholders and feed_dict: - ```python - m = SomeMetric(...) - m_placeholder = tf.compat.v1.placeholder(...) - m_update = m(m_placeholder) - # Variables defined in first call, so get the initialization op afterwards. - m_init = m.init_variables() # or tf.compat.v1.global_variables_initializer() - m_result = m.result() - with tf.compat.v1.Session() as sess: - sess.run(m_init) - for input in ...: - sess.run(m_update, feed_dict={m_placeholder: input}) - print(sess.run(m_result)) - ``` - - Descendants will implement: - * `build()`: All variables should be created in this method, by calling - `self.add_variable()` as in: `self.var = self.add_variable(...)` - build() will be called in the first invocation of `__call__()`, with - the same arguments passed `call()`. - * `call()`: Has all updates to variables, as in: - self.var.assign_add(...) - * `result()`: Computes and returns a final value for the metric - from the variables in `self`. - - Descendants may override `aggregate()`, but usually won't need to. It - adds in the state from a list of metrics of the same type as `self`. - (Default is to sum all the variables.) Note that users should not call - `aggregate()`, it is for use by TensorFlow infrastructure. - """ - - def __init__(self, name=None, use_global_variables=False): - self._built = False - self._vars = [] - self._initial_values = {} - self._updates = [] - self._use_global_variables = use_global_variables - name = name or self.__class__.__name__ - # Replace things like spaces in name to create a valid scope name. - scope_name = _to_replace.sub("_", name) - # We create the variable scope now to get the unique name that will - # be used as a variable prefix when build() calls add_variable(). - with variable_scope.variable_scope( - scope_name, use_resource=True, reuse=False) as scope: - pos = scope.name.rfind(scope_name) - self._name = name + scope.name[pos + len(scope_name):] - self._scope = scope - - # Ensures that if the user calls build directly we still set self._built to - # True to prevent variables from being recreated. - self._build = self.build - - def actual_build(*args, **kwargs): - self._build(*args, **kwargs) - self._built = True - self.build = actual_build - self.build.__doc__ = self._build.__doc__ - - # Captures construction scope for proper initialization. - if context.executing_eagerly(): - self._construction_scope = context.eager_mode - else: - # We make self.call() into a graph callable here, so that we can - # return a single op that performs all of the variable updates. - self._construction_scope = ops.get_default_graph().as_default - self.call = function.defun(self.call) - - # ---- API for users ---- - def __call__(self, *args, **kwargs): - """Returns op to execute to update this metric for these inputs. - - Returns None if eager execution is enabled. - Returns a graph-mode function if graph execution is enabled. - - Args: - *args: - **kwargs: A mini-batch of inputs to the Metric, passed on to `call()`. - """ - if not self._built: - with variable_scope.variable_scope( - self._scope), self._construction_scope(): - self.build(*args, **kwargs) - self._built = True - return self.call(*args, **kwargs) - - @property - def name(self): - return self._name - - @property - def variables(self): - return self._vars - - def init_variables(self): - """Initializes this Metric's variables. - - Should be called after variables are created in the first execution - of `__call__()`. If using graph execution, the return value should be - `run()` in a session before running the op returned by `__call__()`. - (See example above.) - - Returns: - If using graph execution, this returns an op to perform the - initialization. Under eager execution, the variables are reset to their - initial values as a side effect and this function returns None. - """ - if context.executing_eagerly(): - for v in self._vars: - v.assign(self._initial_values[v]) - else: - return control_flow_ops.group([v.initializer for v in self._vars]) - - # ---- To be implemented by descendants --- - def build(self, *args, **kwargs): - """Method to create variables. - - Called by `__call__()` before `call()` for the first time. - - Args: - *args: - **kwargs: The arguments to the first invocation of `__call__()`. - `build()` may use the shape and/or dtype of these arguments - when deciding how to create variables. - """ - raise NotImplementedError("Metrics must define a build() member function") - - def call(self, *args, **kwargs): - """Accumulates statistics for the metric. Users should use __call__ instead. - - Note: This function is executed as a graph function in graph mode. - This means: - a) Operations on the same resource are executed in textual order. - This should make it easier to do things like add the updated - value of a variable to another, for example. - b) You don't need to worry about collecting the update ops to execute. - All update ops added to the graph by this function will be executed. - As a result, code should generally work the same way with graph or - eager execution. - - Args: - *args: - **kwargs: A mini-batch of inputs to the Metric, as passed to - `__call__()`. - """ - raise NotImplementedError("Metrics must define a call() member function") - - def result(self): # TODO(josh11b): Add an optional summary_writer parameter. - """Computes and returns a final value for the metric.""" - raise NotImplementedError("Metrics must define a result() member function") - - def value(self): - """In graph mode returns the result Tensor while in eager the callable.""" - if context.executing_eagerly(): - return self.result - else: - return self.result() - - # We can support two different strategies of for doing data-parallel - # distributed metric computations: - # * Put metric variables on the first device and rely on small - # bandwidth needed to do updates. (Doesn't require any particular - # code in Metric implementations.) - # * Ask each type of metric to define an aggregation method to run - # at the end of eval to merge across devices. Note: this is good - # for the use case where they want to record the metric's state - # for each example and then later decide which examples they want - # to aggregate over. (Recommended -- not too much harder and adds - # flexibility over previous option.) - # I'm going with the second strategy since we can define a default - # implementation of aggregate() that will work for most descendants. - def aggregate(self, metrics): - """Adds in the state from a list of metrics. - - Default implementation sums all the metric variables. - - Args: - metrics: A list of metrics with the same type as `self`. - - Raises: - ValueError: If metrics contains invalid data. - """ - for m in metrics: - if type(self) != type(m): # pylint: disable=unidiomatic-typecheck - raise TypeError("All metrics must be the same type, '%s' != '%s'." % - (type(self), type(m))) - # pylint: disable=protected-access - for i in range(len(self._vars)): - if any(m._vars[i].name != self._vars[i].name for m in metrics): - raise ValueError("All metrics must have variables in the same order.") - self._vars[i].assign_add(math_ops.add_n([m._vars[i] for m in metrics])) - # pylint: enable=protected-access - - # ---- For use by descendants --- - def add_variable(self, name, shape=None, dtype=None, initializer=None): - """***Only for use by descendants of Metric***.""" - if self._built: - raise RuntimeError("Can't call add_variable() except in build().") - if context.executing_eagerly(): - collections = None - else: - if self._use_global_variables: - collections = [ops.GraphKeys.GLOBAL_VARIABLES] - else: - collections = [ops.GraphKeys.LOCAL_VARIABLES] - collections += [ops.GraphKeys.METRIC_VARIABLES] - # Variables are Trackable dependencies of Metrics regardless of the - # global/local distinction. Users can avoid saving variables by not adding a - # dependency on the Metric. - v = self._add_variable_with_custom_getter( - name=name, - shape=shape, - dtype=dtype, - initializer=initializer, - trainable=False, - collections=collections, - use_resource=True, - getter=variable_scope.get_variable, - # Raise duplicate variable exceptions from get_variable rather than - # Trackable. - overwrite=True) - self._vars.append(v) - if context.executing_eagerly(): - self._initial_values[v] = v.value() - return v - - -class Mean(Metric): - """Computes the (weighted) mean of the given values.""" - - def __init__(self, name=None, dtype=dtypes.float64, - use_global_variables=False): - super(Mean, self).__init__(name=name, - use_global_variables=use_global_variables) - self.dtype = dtype - - def build(self, *args, **kwargs): - # build() does not use call's arguments, by using *args, **kwargs - # we make it easier to inherit from Mean(). - del args, kwargs - self.numer = self.add_variable(name="numer", shape=(), - dtype=self.dtype, - initializer=init_ops.zeros_initializer) - self.denom = self.add_variable(name="denom", shape=(), - dtype=self.dtype, - initializer=init_ops.zeros_initializer) - - def call(self, values, weights=None): - """Accumulate statistics for computing the mean. - - For example, if values is [1, 3, 5, 7] then the mean is 4. - If the weights were specified as [1, 1, 0, 0] then the mean would be 2. - - Args: - values: Tensor with the per-example value. - weights: Optional weighting of each example. Defaults to 1. - - Returns: - The arguments, for easy chaining. - """ - if weights is None: - self.denom.assign_add( - math_ops.cast(array_ops.identity(array_ops.size(values)), self.dtype)) - values = math_ops.reduce_sum(values) - self.numer.assign_add(math_ops.cast(values, self.dtype)) - else: - weights = math_ops.cast(weights, self.dtype) - self.denom.assign_add(math_ops.reduce_sum(weights)) - values = math_ops.cast(values, self.dtype) * weights - self.numer.assign_add(math_ops.reduce_sum(values)) - if weights is None: - return values - return values, weights - - def result(self, write_summary=True): - """Returns the result of the Metric. - - Args: - write_summary: bool indicating whether to feed the result to the summary - before returning. - Returns: - aggregated metric as float. - Raises: - ValueError: if the optional argument is not bool - """ - # Convert the boolean to tensor for tf.cond, if it is not. - if not isinstance(write_summary, ops.Tensor): - write_summary = ops.convert_to_tensor(write_summary) - t = self.numer / self.denom - def write_summary_f(): - summary_ops.scalar(name=self.name, tensor=t) - return t - smart_cond.smart_cond(write_summary, - write_summary_f, - lambda: t, - name="") - return t - - -class Accuracy(Mean): - """Calculates how often `predictions` matches `labels`. - Attributes: - name: name of the accuracy object - dtype: data type of the tensor - """ - - def __init__(self, name=None, dtype=dtypes.float64): - """Inits Accuracy class with name and dtype.""" - super(Accuracy, self).__init__(name=name, dtype=dtype) - - def call(self, labels, predictions, weights=None): - """Accumulate accuracy statistics. - - For example, if labels is [1, 2, 3, 4] and predictions is [0, 2, 3, 4] - then the accuracy is 3/4 or .75. If the weights were specified as - [1, 1, 0, 0] then the accuracy would be 1/2 or .5. - - `labels` and `predictions` should have the same shape and type. - - Args: - labels: Tensor with the true labels for each example. One example - per element of the Tensor. - predictions: Tensor with the predicted label for each example. - weights: Optional weighting of each example. Defaults to 1. - - Returns: - The arguments, for easy chaining. - """ - check_ops.assert_equal( - array_ops.shape(labels), array_ops.shape(predictions), - message="Shapes of labels and predictions are unequal") - matches = math_ops.equal(labels, predictions) - matches = math_ops.cast(matches, self.dtype) - super(Accuracy, self).call(matches, weights=weights) - if weights is None: - return labels, predictions - return labels, predictions, weights - - -class CategoricalAccuracy(Mean): - """Calculates how often `predictions` matches `labels`. - - This class is compatible with `tf.keras.losses.categorical_crossentropy`, - `tf.nn.softmax_cross_entropy_with_logits`, - `tf.compat.v1.losses.softmax_cross_entropy`. - - Attributes: - name: name of the accuracy object. - dtype: data type of tensor. - """ - - def __init__(self, name=None, dtype=dtypes.float64): - """Inits CategoricalAccuracy with name and dtype.""" - super(CategoricalAccuracy, self).__init__(name=name, dtype=dtype) - - def call(self, labels, predictions, weights=None): - """Accumulate accuracy statistics. - - `labels` and `predictions` should have the same shape. - As argmax is being done here, labels and predictions type - can be different. - - Args: - labels: One-hot Tensor. - predictions: Tensor with the logits or probabilities for each example. - weights: Optional weighting of each example. Defaults to 1. - - Returns: - The arguments, for easy chaining. - """ - check_ops.assert_equal( - array_ops.shape(labels), array_ops.shape(predictions), - message="Shapes of labels and predictions are unequal") - labels = math_ops.argmax(labels, axis=-1) - predictions = math_ops.argmax(predictions, axis=-1) - matches = math_ops.equal(labels, predictions) - matches = math_ops.cast(matches, self.dtype) - super(CategoricalAccuracy, self).call(matches, weights=weights) - if weights is None: - return labels, predictions - return labels, predictions, weights - - -class BinaryAccuracy(Mean): - """Calculates how often `predictions` matches `labels`. - - This class is compatible with `tf.keras.losses.binary_crossentropy`, - `tf.compat.v1.losses.sigmoid_cross_entropy`, - `tf.nn.sigmoid_cross_entropy_with_logits`. - If there is more than one label, this will become multi-label classification. - - Attributes: - name: name of the accuracy object. - threshold: Used for rounding off the predictions. - If the predictions are, - 1. probabilities then set the threshold to 0.5. - 2. logits then set the threshold to 0. - You can set the threshold appropriately, - to trade off with precision and recall. - dtype: data type of tensor. - """ - - def __init__(self, threshold, name=None, dtype=dtypes.float64): - """Inits BinaryAccuracy with name, threshold and dtype.""" - - super(BinaryAccuracy, self).__init__(name=name, dtype=dtype) - self.threshold = threshold - - def call(self, labels, predictions, weights=None): - """Accumulate accuracy statistics. - - `labels` and `predictions` should have the same shape and type. - - Args: - labels: Binary Tensor(containing 0 or 1). - predictions: Tensor with probabilities or logits. - weights: Optional weighting of each example. Defaults to 1. - - Returns: - The arguments, for easy chaining. - """ - check_ops.assert_equal( - array_ops.shape(labels), array_ops.shape(predictions), - message="Shapes of labels and predictions are unequal") - predictions = ops.convert_to_tensor(predictions) - predictions = predictions > self.threshold - # Convert labels to bool to match predictions. - labels = math_ops.cast(labels, dtypes.bool) - matches = math_ops.equal(labels, predictions) - matches = math_ops.cast(matches, self.dtype) - super(BinaryAccuracy, self).call(matches, weights=weights) - if weights is None: - return labels, predictions - return labels, predictions, weights - - -class SparseAccuracy(Mean): - """Calculates how often `predictions` matches `labels`. - - This class is compatible with - `tf.keras.losses.sparse_categorical_crossentropy`, - `tf.nn.sparse_softmax_cross_entropy_with_logits`, - `tf.compat.v1.losses.sparse_softmax_cross_entropy`. - - Attributes: - name: name of the accuracy object - dtype: data type of tensor. - """ - - def __init__(self, name=None, dtype=dtypes.float64): - """Inits SparseAccuracy with name and dtype.""" - - super(SparseAccuracy, self).__init__(name=name, dtype=dtype) - - def call(self, labels, predictions, weights=None): - """Accumulate accuracy statistics. - - `labels` and `predictions` should have the same shape except the - predictions must have one additional trailing dimension equal to the - number of classes(you want to predict). - - Type of labels and predictions can be different. - - Args: - labels: Tensor of shape (batch_size, ) containing integers - predictions: Tensor with the logits or probabilities for each example. - weights: Optional weighting of each example. Defaults to 1. - - Returns: - The arguments, for easy chaining. - """ - check_ops.assert_equal( - array_ops.shape(labels), array_ops.shape(predictions)[0], - message="First axis of labels and predictions is unequal") - predictions = math_ops.argmax(predictions, axis=-1) - labels = math_ops.cast(labels, dtypes.int64) - matches = math_ops.equal(labels, predictions) - matches = math_ops.cast(matches, self.dtype) - super(SparseAccuracy, self).call(matches, weights=weights) - if weights is None: - return labels, predictions - return labels, predictions, weights diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py deleted file mode 100644 index c56d1956fde..00000000000 --- a/tensorflow/contrib/eager/python/metrics_test.py +++ /dev/null @@ -1,339 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Metrics.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import tempfile - -from tensorflow.contrib.eager.python import metrics -from tensorflow.contrib.summary import summary_test_util -from tensorflow.python.eager import context -from tensorflow.python.eager import test -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import summary_ops_v2 as summary_ops -from tensorflow.python.training import training_util -from tensorflow.python.training.tracking import util as trackable_utils - - -class MetricsTest(test.TestCase): - - def testMean(self): - m = metrics.Mean() - m([1, 10, 100]) - m(1000) - m([10000.0, 100000.0]) - self.assertEqual(111111.0/6, m.result().numpy()) - self.assertEqual(dtypes.float64, m.dtype) - self.assertEqual(dtypes.float64, m.result().dtype) - - def testVariableCollections(self): - with context.graph_mode(), ops.Graph().as_default(): - m = metrics.Mean() - m(1000) - self.assertEqual( - set(m.variables), - set(ops.get_collection(ops.GraphKeys.LOCAL_VARIABLES))) - self.assertEqual(ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES), []) - self.assertEqual( - set(m.variables), - set(ops.get_collection(ops.GraphKeys.METRIC_VARIABLES))) - - def testUseGlobalVariablesCollections(self): - with context.graph_mode(), ops.Graph().as_default(): - m = metrics.Mean(use_global_variables=True) - m(1000) - self.assertEqual( - set(m.variables), - set(ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))) - self.assertEqual(ops.get_collection(ops.GraphKeys.LOCAL_VARIABLES), []) - self.assertEqual( - set(m.variables), - set(ops.get_collection(ops.GraphKeys.METRIC_VARIABLES))) - - def testInitVariables(self): - m = metrics.Mean() - m([1, 10, 100, 1000]) - m([10000.0, 100000.0]) - self.assertEqual(111111.0/6, m.result().numpy()) - m.init_variables() - m(7) - self.assertEqual(7.0, m.result().numpy()) - - def testWriteSummaries(self): - m = metrics.Mean() - m([1, 10, 100]) - training_util.get_or_create_global_step() - logdir = tempfile.mkdtemp() - with summary_ops.create_file_writer( - logdir, max_queue=0, - name="t0").as_default(), summary_ops.always_record_summaries(): - m.result() # As a side-effect will write summaries. - - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 2) - self.assertEqual(events[1].summary.value[0].simple_value, 37.0) - - # Get result without saving the summary. - logdir = tempfile.mkdtemp() - with summary_ops.create_file_writer( - logdir, max_queue=0, - name="t0").as_default(), summary_ops.always_record_summaries(): - m.result(write_summary=False) # As a side-effect will write summaries. - # events_from_logdir(_) asserts the directory exists. - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 1) - - def testWeightedMean(self): - m = metrics.Mean() - m([1, 100, 100000], weights=[1, 0.2, 0.3]) - m([500000, 5000, 500]) # weights of 1 each - self.assertNear(535521/4.5, m.result().numpy(), 0.001) - - def testMeanDtype(self): - # Can override default dtype of float64. - m = metrics.Mean(dtype=dtypes.float32) - m([0, 2]) - self.assertEqual(1, m.result().numpy()) - self.assertEqual(dtypes.float32, m.dtype) - self.assertEqual(dtypes.float32, m.result().dtype) - - def testAccuracy(self): - m = metrics.Accuracy() - m([0, 1, 2, 3], [0, 0, 0, 0]) # 1 correct - m([4], [4]) # 1 correct - m([5], [0]) # 0 correct - m([6], [6]) # 1 correct - m([7], [2]) # 0 correct - self.assertEqual(3.0/8, m.result().numpy()) - self.assertEqual(dtypes.float64, m.dtype) - self.assertEqual(dtypes.float64, m.result().dtype) - - def testCategoricalAccuracy(self): - m = metrics.CategoricalAccuracy() - m([[1, 0, 0, 0], [0, 1, 0, 0]], - [[0.6, 0.1, 0.25, 0.05], [0.4, 0.05, 0.45, 0.0]]) # 1/2 correct - m([[0, 0, 0, 1]], [[0.25, 0.95, 0.25, 0.0]]) # 0/1 correct - m([[1, 0, 0, 0], [0, 1, 0, 0]], - [[0.99, 0.01, 0.0, 0.0], [0.35, 0.35, 0.3, 0.0]]) # 1/2 correct - self.assertEqual(2.0/5, m.result().numpy()) - self.assertEqual(dtypes.float64, m.dtype) - self.assertEqual(dtypes.float64, m.result().dtype) - - def testBinaryAccuracy(self): - m = metrics.BinaryAccuracy(threshold=0) - # as threshold is 0 hence the predictions are logits - m([[0, 0, 0, 0]], - [[-4.2, 4.5, 1.2, -1.1]]) # 2/4 correct - m([[0, 1]], [[-5.3, 11.65]]) # 2/2 correct - m([[0, 1], [1, 1]], - [[-5.3, 11.65], [-10.32, 56.38]]) # 3/4 correct - self.assertEqual(7.0/10, m.result().numpy()) - self.assertEqual(dtypes.float64, m.dtype) - self.assertEqual(dtypes.float64, m.result().dtype) - - def testSparseAccuracy(self): - m = metrics.SparseAccuracy() - m([0, 2], - [[0.6, 0.1, 0.25, 0.05], [0.4, 0.05, 0.45, 0.0]]) # 2/2 correct - m([1], [[0.25, 0.95, 0.25, 0.0]]) # 1/1 correct - m([0, 3], [[0.99, 0.01, 0.0, 0.0], [0.35, 0.35, 0.3, 0.0]]) # 1/2 correct - self.assertEqual(4.0/5, m.result().numpy()) - self.assertEqual(dtypes.float64, m.dtype) - self.assertEqual(dtypes.float64, m.result().dtype) - - def testAccuracyDifferentShapes(self): - m = metrics.Accuracy() - with self.assertRaises(errors.InvalidArgumentError): - m([[0], [0]], [0, 1]) - - def testWeightedAccuracy(self): - m = metrics.Accuracy() - # 1 correct, total weight of 2 - m([0, 1, 2, 3], [0, 0, 0, 0], weights=[1, 1, 0, 0]) - m([4], [4], weights=[0.5]) # 1 correct with a weight of 0.5 - m([5], [0], weights=[0.5]) # 0 correct, weight 0.5 - m([6], [6]) # 1 correct, weight 1 - m([7], [2]) # 0 correct, weight 1 - self.assertEqual(2.5/5, m.result().numpy()) - - def testAccuracyDtype(self): - # Can override default dtype of float64. - m = metrics.Accuracy(dtype=dtypes.float32) - m([0, 0], [0, 1]) - self.assertEqual(0.5, m.result().numpy()) - self.assertEqual(dtypes.float32, m.dtype) - self.assertEqual(dtypes.float32, m.result().dtype) - - def testTwoMeans(self): - # Verify two metrics with the same class and name don't - # accidentally share state. - m1 = metrics.Mean() - m1(0) - m2 = metrics.Mean() - m2(2) - self.assertAllEqual(0.0, m1.result()) - self.assertAllEqual(2.0, m2.result()) - - def testNamesWithSpaces(self): - m1 = metrics.Mean("has space") - m1(0) - self.assertEqual(m1.name, "has space") - self.assertEqual(m1.numer.name, "has_space/numer:0") - - def testGraphWithPlaceholder(self): - with context.graph_mode(), self.cached_session() as sess: - m = metrics.Mean() - p = array_ops.placeholder(dtypes.float32) - accumulate = m(p) - init_op = m.init_variables() - init_op.run() - sess.run(accumulate, feed_dict={p: [1, 10, 100]}) - sess.run(accumulate, feed_dict={p: 1000}) - sess.run(accumulate, feed_dict={p: [10000, 100000]}) - self.assertAllEqual(m.result().eval(), 111111.0/6) - # Second init resets all the variables. - init_op.run() - sess.run(accumulate, feed_dict={p: 7}) - self.assertAllEqual(m.result().eval(), 7) - - @test_util.run_in_graph_and_eager_modes - def testGraphAndEagerTensor(self): - m = metrics.Mean() - inputs = ops.convert_to_tensor([1.0, 2.0]) - accumulate = m(inputs) - result = m.result() - self.evaluate(m.init_variables()) - self.evaluate(accumulate) - self.assertEqual(self.evaluate(result), 1.5) - # Second init resets all the variables. - self.evaluate(m.init_variables()) - inputs = ops.convert_to_tensor([2.0, 3.0]) - self.evaluate(m(inputs)) - value = m.value() - self.assertEqual(self.evaluate(value), 2.5) - - @test_util.run_in_graph_and_eager_modes - def testGraphAndEagerTensorGlobalVariables(self): - m = metrics.Mean(use_global_variables=True) - inputs = ops.convert_to_tensor([1.0, 2.0]) - accumulate = m(inputs) - result = m.result() - self.evaluate(m.init_variables()) - self.evaluate(accumulate) - self.assertEqual(self.evaluate(result), 1.5) - # Second init resets all the variables. - self.evaluate(m.init_variables()) - inputs = ops.convert_to_tensor([2.0, 3.0]) - self.evaluate(m(inputs)) - value = m.value() - self.assertEqual(self.evaluate(value), 2.5) - - @test_util.run_in_graph_and_eager_modes - def testGraphAndEagerTensorWhileLoopDoubleCall(self): - m = metrics.Mean() - init_value = constant_op.constant(1) - cond = lambda i: math_ops.less(i, 3) - def body(x): - with ops.control_dependencies([m(x)]): - return math_ops.add(x, 1) - accumulate = control_flow_ops.while_loop(cond, body, [init_value]) - - result = m.result() - self.evaluate(m.init_variables()) - self.evaluate(accumulate) - self.assertEqual(self.evaluate(result), 1.5) - # Second init resets all the variables. - self.evaluate(m.init_variables()) - inputs = ops.convert_to_tensor([2.0, 3.0]) - self.evaluate(m(inputs)) - if ops.context.executing_eagerly(): - self.evaluate(control_flow_ops.while_loop(cond, body, [init_value])) - else: - # Reuse the loop operators in graph mode - self.evaluate(accumulate) - value = m.value() - self.assertEqual(self.evaluate(value), 2.0) - - def testTwoMeansGraph(self): - # Verify two metrics with the same name in the same graph raises a - # ValueError. - with context.graph_mode(): - m1 = metrics.Mean() - m1(0) - with self.assertRaises(ValueError): - m2 = metrics.Mean() - m2(2) - - def testBuildMean(self): - # Verify that calling build() on Mean and then calling it won't recreate - # variables. - m = metrics.Mean() - m.build() - old_numer = m.numer - m(0.0) - self.assertTrue(old_numer is m.numer) - - def testMetricsChain(self): - with context.graph_mode(), self.cached_session(): - m1 = metrics.Mean() - m2 = metrics.Mean(name="m2") - update_m2 = m2(3.0) - update_m2_2 = m2(m1(1.0)) - m1.init_variables().run() - m2.init_variables().run() - update_m2.eval() - update_m2_2.eval() - self.assertAllEqual(m2.result().eval(), 2.0) - self.assertAllEqual(m1.result().eval(), 1.0) - - @test_util.run_in_graph_and_eager_modes - def testSaveRestore(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - mean = metrics.Mean() - checkpoint = trackable_utils.Checkpoint(mean=mean) - mean.build() - mean._built = True - self.evaluate(mean.init_variables()) - self.evaluate(mean(100.)) - self.evaluate(mean(200.)) - save_path = checkpoint.save(checkpoint_prefix) - self.evaluate(mean(1000.)) - checkpoint.restore(save_path).assert_consumed().run_restore_ops() - self.evaluate(mean(300.)) - self.assertAllEqual(200., self.evaluate(mean.value())) - - restore_mean = metrics.Mean() - restore_checkpoint = trackable_utils.Checkpoint(mean=restore_mean) - status = restore_checkpoint.restore(save_path) - restore_update = restore_mean(300.) - status.assert_consumed().run_restore_ops() - self.evaluate(restore_update) - self.assertAllEqual(200., self.evaluate(restore_mean.value())) - self.assertEqual(3, self.evaluate(restore_mean.denom)) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py deleted file mode 100644 index 363e2191c3d..00000000000 --- a/tensorflow/contrib/eager/python/network.py +++ /dev/null @@ -1,1027 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A Network is a composition of Layers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import os -import weakref - -from tensorflow.python.eager import context -from tensorflow.python.framework import ops -from tensorflow.python.keras import backend -from tensorflow.python.layers import base -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import checkpoint_utils -from tensorflow.python.training import saver as saver_lib -from tensorflow.python.training import training_util -from tensorflow.python.util import deprecation -from tensorflow.python.util import function_utils - -# pylint: disable=protected-access -# Explanation for protected-access disable: Network has lots of same-class and -# parent-class references across different objects, and some to private -# functions in base.py which should be reused. - - -def _network_name_scope_naming(current_variable_scope): - """Name scope naming to match operation names to variable names. - - Used in Networks and also applied to non-Network Layers which are added to - Networks before being built. - - Args: - current_variable_scope: A VariableScope object. - - Returns: - A name scope name. - """ - return current_variable_scope.name + "/" - - -_NETWORK_DEPRECATION_MESSAGE = ( - "Please inherit from `tf.keras.Model`, and see its documentation for " - "details. `tf.keras.Model` should be a drop-in replacement for " - "`tfe.Network` in most cases, but note that `track_layer` is no longer " - "necessary or supported. Instead, `Layer` instances are tracked on " - "attribute assignment (see the section of `tf.keras.Model`'s documentation " - "on subclassing). Since the output of `track_layer` is often assigned to " - "an attribute anyway, most code can be ported by simply removing the " - "`track_layer` calls.\n\n`tf.keras.Model` works with all TensorFlow " - "`Layer` instances, including those from `tf.layers`, but switching to " - "the `tf.keras.layers` versions along with the migration to " - "`tf.keras.Model` is recommended, since it will preserve variable names. " - "Feel free to import it with an alias to avoid excess typing :).") - - -class Network(base.Layer): - """Represents the composition of a set of Layers. - - *Deprecated*. Please inherit from `tf.keras.Model`, and see its documentation - for details. `tf.keras.Model` should be a drop-in replacement for - `tfe.Network` in most cases, but note that `track_layer` is no longer - necessary or supported. Instead, `Layer` instances are tracked on attribute - assignment (see the section of `tf.keras.Model`'s documentation on - subclassing). Since the output of `track_layer` is often assigned to an - attribute anyway, most code can be ported by simply removing the `track_layer` - calls. - - `tf.keras.Model` works with all TensorFlow `Layer` instances, including those - from `tf.layers`, but switching to the `tf.keras.layers` versions along with - the migration to `tf.keras.Model` is recommended, since it will preserve - variable names. Feel free to import it with an alias to avoid excess typing - :). - - `Network` implements the `Layer` interface and adds convenience methods for - managing sub-`Layer`s, such as listing variables. - - `Layer`s (including other `Network`s) should be added via `track_layer`. They - can then be used when overriding the `Network.call` method: - - ```python - class TwoLayerNetwork(tfe.Network): - - def __init__(self, name): - super(TwoLayerNetwork, self).__init__(name=name) - self.layer_one = self.track_layer(tf.compat.v1.layers.Dense(16, - input_shape=(8,))) - self.layer_two = self.track_layer(tf.compat.v1.layers.Dense(1, - input_shape=(16,))) - - def call(self, inputs): - return self.layer_two(self.layer_one(inputs)) - ``` - - After constructing an object and calling the `Network`, a list of variables - created by tracked `Layer`s is available via `Network.variables`: - - ```python - net = TwoLayerNetwork(name="net") - output = net(tf.ones([1, 8])) - print([v.name for v in net.variables]) - ``` - - This example prints variable names, one kernel and one bias per - `tf.compat.v1.layers.Dense` layer: - - ``` - ['net/dense/kernel:0', - 'net/dense/bias:0', - 'net/dense_1/kernel:0', - 'net/dense_1/bias:0'] - ``` - - These variables can be passed to a `Saver` (`tf.compat.v1.train.Saver`, or - `tf.contrib.eager.Saver` when executing eagerly) to save or restore the - `Network`, typically alongside a global step and - `tf.compat.v1.train.Optimizer` - variables when checkpointing during training. - - Note that the semantics of calling a `Network` with graph execution (i.e. not - executing eagerly) may change slightly in the future. Currently stateful ops - are pruned from the graph unless they or something that depends on them is - executed in a session, but this behavior is not consistent with eager - execution (where stateful ops are executed eagerly). `Layer`s from `tf.layers` - do not depend on this pruning and so will not be affected, but `Network`s - which rely on stateful ops being added to the graph but not executed (e.g. via - custom `Layer`s which manage stateful ops) may break with this change. - """ - # TODO(josh11b,ashankar,allenl): - # - Should 'trainable' be changeable on the Network object? - # - Do we allow add_variable in Network? - # - Detect layers used in __call__ that weren't registered with track_layer. - # - Convert inputs to __call__ to tensors. - - @deprecation.deprecated(date=None, instructions=_NETWORK_DEPRECATION_MESSAGE) - def __init__(self, name=None): - """Configure the `Network`. - - Args: - name: The name to use for this `Network`. If specified, it must be unique - in the context where this `Network` is first (1) added to another - `Network` (in which case it must not share a name with other `Layers` - added to that `Network`), or (2) built/called (in which case no other - 'top-level' `Network`s may share this name). If unspecified or None, the - `Network` will be named using its class name, with a number appended if - necessary for uniqueness (e.g. MyNetwork -> 'my_network_1'). - - Raises: - ValueError: If `name` is not valid. Note that some naming errors will - instead be raised when the `Network` is called. - """ - if context.executing_eagerly(): - logging.warning( - ("** tfe.Network is deprecated and will be removed in a future " - "version.\n\n%s"), _NETWORK_DEPRECATION_MESSAGE) - if isinstance(name, variable_scope.VariableScope): - raise ValueError("VariableScopes are not valid Network names.") - if name is not None and "/" in name: - raise ValueError( - "Forward slashes ('/') are not allowed in Network names.") - super(Network, self).__init__(name=name) - self._layers = [] - self._sub_layer_name_uids = collections.defaultdict(int) - # Initially None, but set to False for networks which are first built as - # top-level. - self._first_parent = None # A weak reference to our first parent. - self._non_network_sublayers = [] - self._owned_layers = {} - # The scope to use if we end up without a parent. - self._default_parent_variable_scope = variable_scope.get_variable_scope() - # Hold on to the variable scope counts from init to check whether a scope - # with the name we want was ever created in our parent scope. Without this - # check we might have name collisions if the parent scope on init gets - # closed before build is called. - self._variable_scope_counts_on_init = ( - variable_scope.get_variable_scope_store().variable_scopes_count) - - def _gather_saveables_for_checkpoint(self): - raise NotImplementedError( - "tfe.Network does not support object-based checkpointing.\n\n%s" % - _NETWORK_DEPRECATION_MESSAGE) - - def _name_scope_name(self, current_variable_scope): - """Overrides Layer op naming to match variable naming.""" - return _network_name_scope_naming( - current_variable_scope=current_variable_scope) - - def _init_set_name(self, name): - # Anonymous Networks (name=None) defer setting a final name until they are - # (1) added to another Network, or (2) built/called (where (2) is only used - # for a "top level" network). - # - # However, if we were provided an explicit name (name is not None), that - # will always be the final name of the Network; if it turns out not to be - # unique or if variable names can't be prefixed by it we will throw an - # error. - self._name = name - self._base_name = None - - def _finalize_name(self, parent_network): - if not self._name: - # Were were not passed a name explicitly (or it was blank), so this is an - # anonymous Network. We make up a unique name. - if parent_network: - avoid_names = parent_network._owned_layers - name_uid_map = parent_network._sub_layer_name_uids - else: - name_uid_map = backend.get_default_graph_uid_map() - # Figure out which names we have to avoid based on which variable scope - # we're nested in. - strip_name = self._default_parent_variable_scope.name - if strip_name: - strip_name += "/" - - def _strip_on_init_scope(name): - if name.startswith(strip_name): - return name[len(strip_name):] - else: - return None - - avoid_names = set( - _strip_on_init_scope(name) - for name in self._variable_scope_counts_on_init.keys() - if name) - self._name, self._base_name = self._make_unique_name( - name_uid_map=name_uid_map, - avoid_names=avoid_names, - namespace=self._default_parent_variable_scope.name, - zero_based=True) - if self._first_parent is None or (self._first_parent # False = no parent - and self._first_parent() is None): - # Save a pointer to the parent Network so that we can later check that the - # scope name we get is correct. - if not parent_network: - self._first_parent = parent_network - else: - self._first_parent = weakref.ref(parent_network) - - def _set_scope(self, scope=None): - if self._scope is None: - if not self._first_parent: - first_parent = self._first_parent - else: - first_parent = self._first_parent() - if first_parent is None: - # If we were never added to another Network, or that Network has beed - # garbage collected before being called, then we're a top-level Network. - self._finalize_name( - # Use False to make sure the value sticks and we don't inherit a - # parent if we're added to a network later. - parent_network=False) - if scope is not None: - raise ValueError("Networks may not be created with explicit scopes.") - if first_parent: - first_parent._set_scope() - parent_scope = first_parent._scope - else: - parent_scope = self._default_parent_variable_scope - with variable_scope.variable_scope(parent_scope) as parent_vs: - expected_scope_name = parent_vs.name + "/" + self._name - if expected_scope_name in self._variable_scope_counts_on_init: - raise ValueError( - ("A Network named '%s' already exists (or a variable_scope was " - "created with this name). Names must be unique.") % - (self._name,)) - # Make sure variables with this prefix will be unique. - with variable_scope.variable_scope( - None, use_resource=True, default_name=self._name) as scope: - self._scope = scope - scope_name = scope.name - suffix_start = scope_name.rfind("/") + 1 - # rfind is -1 if there is no slash in the string, in which case the - # suffix starts at the beginning of the string (there is no prefix). - scope_suffix = scope_name[suffix_start:] - scope_prefix = scope_name[:suffix_start] - if scope_suffix != self._name: - raise ValueError( - ("A Network named '%s' already exists (or a variable_scope was " - "created with this name). Names must be unique.") % - (self._name,)) - if (first_parent and scope_prefix[:-1] != first_parent.scope_name): - raise ValueError( - ("Network variable names must match a nesting of sub-Network " - "names. Expected prefix '%s' from parent network, but got " - "'%s' when attempting to create a variable_scope for Network " - "'%s'. Likely an explicit variable_scope was inserted into " - "the nesting.") % - (first_parent.scope_name, scope_prefix[:-1], self._name)) - elif not first_parent and scope_prefix: - # For the case when this Network is not nested inside any other - # Network, but is in a variable_scope. This Network's name takes on - # the full variable scope prefix. - self._name = scope_name - - for non_network_sublayer in self._non_network_sublayers: - self._set_scope_for_nonnetwork_sublayer(non_network_sublayer) - - def _set_scope_for_nonnetwork_sublayer(self, sublayer): - if sublayer._scope is None: - if sublayer._first_parent is None: - constituent_first_parent = None - else: - constituent_first_parent = sublayer._first_parent() - if constituent_first_parent: - constituent_first_parent._set_scope() - parent_scope = constituent_first_parent._scope - else: - self._finalize_name(False) - raise ValueError( - ("The parent of a Layer added to Network %s was garbage collected " - "before the Layer was built. If this limitation bothers you " - "please file a feature request.") % (self.name,)) - with variable_scope.variable_scope(parent_scope): - # Horrid hack to make Layer variable names which are direct - # sub-layers of Networks conform to the Network variable naming - # conventions. - with variable_scope.variable_scope( - None, use_resource=True, default_name=sublayer.name) as sub_scope: - sublayer._scope = sub_scope - # Also switch op naming for this Layer to match Network conventions, - # i.e. op naming matching variable naming. - sublayer._name_scope_name = _network_name_scope_naming - - @base.Layer.name.getter - def name(self): - if self._name is None: - raise ValueError( - "The network does not yet have a final name, but a name was " - "requested for it. Networks get a name when they are added to " - "another Network via track_layer, or when they are first " - "called/built.") - return self._name - - def track_layer(self, layer): - """Track a Layer in this Network. - - `Network` requires that all `Layer`s used in `call()` be tracked so that the - `Network` can export a complete list of variables. - - Args: - layer: A `tf.compat.v1.layers.Layer` object. - - Returns: - The passed in `layer`. - - Raises: - RuntimeError: If __init__ has not been called. - TypeError: If `layer` is the wrong type. - ValueError: If a `Layer` with the same name has already been added. - """ - if not hasattr(self, "_layers"): - raise RuntimeError("Need to call Network.__init__ before adding layers") - if not isinstance(layer, base.Layer): - raise TypeError( - "Network.track_layer() passed type %s, not a tf.layers.Layer" % - (type(layer),)) - # Always use `ResourceVariable` with legacy layers. - layer._use_resource_variables = True - if isinstance(layer, Network): - layer._finalize_name(parent_network=self) - else: - # `layer` is a non-Network, so it hasn't been named to follow Network - # conventions for contained Layers (i.e. the same conventions as for - # sub-Networks). This renaming is necessary to isolate Network variable - # naming from Layers constructed outside the Network and never added to it - # (because Layers are named globally). - if not layer.built: - if not hasattr(layer, "_first_parent"): - dereferenced_layer_first_parent = None - else: - dereferenced_layer_first_parent = layer._first_parent() - if dereferenced_layer_first_parent is None: - if layer._name != layer._base_name: - # If name and base_name do not match, then this Layer used anonymous - # naming and we have to rename it. Otherwise there's an explicit - # name, and we should respect it (subject to error checking). - layer._name, layer._base_name = layer._make_unique_name( - name_uid_map=self._sub_layer_name_uids, - avoid_names=self._owned_layers, - zero_based=True - # No namespace required, since we've specified our own UID map. - ) - layer._first_parent = weakref.ref(self) - self._non_network_sublayers.append(layer) - if (not layer.built and layer._first_parent and - self is layer._first_parent()): - if layer.name in self._owned_layers: - if self._owned_layers[layer.name] is layer: - return layer - raise ValueError( - "Attempt to add two Layers with the name '%s' to the same Network." - % (layer.name)) - self._owned_layers[layer.name] = layer - self._layers.append(layer) - return layer - - def get_layer(self, name=None, index=None): - """Get a contained `tf.compat.v1.layers.Layer` either by name or index. - - Args: - name: String matching one of the names of a contained `Layer`. Note that - the names of `Layer`s added to `Network`s may not be unique when doing - layer sharing (i.e. adding a `Layer` to this `Network` which was already - added to another `Network`). The lowest index `Layer` with a matching - name will be returned. - index: Integer in [0, number of layers). Layers are assigned an index by - the order they are added. - - Returns: - A `tf.compat.v1.layers.Layer` object. - - Raises: - ValueError: If neither or both of 'index' or 'name' is specified, or the - lookup failed. - """ - if index is not None: - if name is not None: - raise ValueError("Exactly one of 'index' or 'name' must be provided") - if len(self._layers) <= index: - raise ValueError("Was asked to retrieve layer at index " + str(index) + - " but model only has " + str(len(self._layers)) + - " layers.") - else: - return self._layers[index] - else: - if not name: - raise ValueError("Provide either a layer name or layer index.") - for layer in self._layers: - if layer.name == name: - return layer - raise ValueError("No such layer: " + name) - - # The following methods are for implementing the Layer interface. - - @property - def weights(self): - # TODO(josh11b): Should this return a set or perform de-duplication of - # variables in the case of shared layers/variables that appear in - # multiple places in the Network? - weights = [] - for layer in self._layers: - weights += layer.weights - return weights - - @property - def trainable_weights(self): - weights = [] - for layer in self._layers: - weights += layer.trainable_weights - return weights - - @property - def non_trainable_weights(self): - weights = [] - for layer in self._layers: - weights += layer.non_trainable_weights - return weights - - @property - def trainable(self): - return True - - @trainable.setter - def trainable(self, value): - if not value: - # We believe it better to decide which layers & networks are trainable - # at the Trainer level than here. Otherwise you can run into trouble if a - # layer/network is shared between two models, but is trainable in one - # but not the other (like with adversarial networks). - raise AttributeError("cannot mark Network as not trainable") - - @property - def layers(self): - return self._layers - - def add_variable(self, - name, - shape, - dtype=None, - initializer=None, - regularizer=None, - trainable=True, - constraint=None): - raise RuntimeError( - "add_variable not supported in Network class yet. Please file an issue " - "at https://github.com/tensorflow/tensorflow/issues/new if this is " - "important to you") - - def add_loss(self, losses, inputs=None): - raise RuntimeError( - "add_loss is not supported in Network class yet. Please file an issue " - "at https://github.com/tensorflow/tensorflow/issues/new if this is " - "important to you") - - @property - def losses(self): - """Gather losses from `Layer`s in the `Network`. - - Note that when executing eagerly, `Layer.losses` evaluates - regularizers. When using graph execution, variable regularization ops have - already been created and are simply returned here. - - Returns: - A list of tensors. - """ - layer_losses = [] - for layer in self.layers: - layer_losses.extend(layer.losses) - return layer_losses - - # TODO(allenl): Support other Layer methods needed for graph mode, such as for - # updates - - -class Sequential(Network): - """Represents a linear sequence of Layers or functions. - - The output of each layer/function is provided as the input to the next. - The inputs passed to `__call__` are passed to the inputs of the first - Layer, and it returns the outputs of the last Layer. - - Args: - layers_funcs: An optional sequence where each element is either a - tf.compat.v1.layers.Layer object or a callable. - name: An optional string name to use for this Network. - """ - - def __init__(self, layers_funcs=None, name=None): - super(Sequential, self).__init__(name=name) - self._layers_funcs = [] - if layers_funcs: - for l in layers_funcs: - self.add(l) - - def add(self, layer_func): - if isinstance(layer_func, base.Layer): - args = function_utils.fn_args(layer_func.call) - self.track_layer(layer_func) - elif callable(layer_func): - args = function_utils.fn_args(layer_func) - else: - raise TypeError( - "Sequential.add() takes only tf.layers.Layer objects or callables; " - "not '%s' of type '%s'." % (layer_func, type(layer_func))) - self._layers_funcs.append((("training" in args), layer_func)) - - def call(self, inputs, training=None): - """Call each Layer in the order they were added.""" - # TODO(josh11b): Support "mode" and maybe other arguments - if training is None: - for _, l in self._layers_funcs: - inputs = l(inputs) - else: - for has_training_arg, l in self._layers_funcs: - if has_training_arg: - inputs = l(inputs, training) - else: - inputs = l(inputs) - return inputs - - -_DeferredRestoration = collections.namedtuple( - "_DeferredRestoration", - [ - # The map_func to use (either user-specified or the default). - "map_func", - # Boolean, True if the user specified an explicit map_func, for error - # messages. - "map_func_is_user", - # A mapping from checkpoint names to initial values of not-yet-created - # variables which should be restored. These values come from parsing a - # checkpoint. - "checkpointed_variables_to_restore", - # A mapping from checkpoint name to variable objects of variables which - # have already been restored, for error checking. - "restored_variables", - # The session to restore with (if in graph mode). - "session", - # Names of the Network where the restore was requested, for error - # messages. - "network_name", - "network_scope_name" - ]) - - -def _default_naming_conflict_error_message(mapped_name, first_variable, - second_variable, network_name, - network_scope_name): - return ( - ("The default checkpoint variable name mapping strategy for Network " - "'%s' resulted in a naming conflict. We attempted to strip off the " - "variable prefix for the Network ('%s'), but this resulted in two " - "variables named '%s' (originally '%s' and '%s'). This should only " - "happen when using variable sharing (i.e. the Network contains Networks " - "or Layers which were first added to another Network, and therefore " - "have that Network's variable prefix). One solution is to pass " - "`map_func=lambda n: n` to save and restore to use fully qualified " - "variable names in the checkpoint, although this will require that the " - "variable prefix of the Network being restored into is also '%s'. You " - "may alternatively write an arbitrary mapping.") % - (network_name, network_scope_name, mapped_name, - first_variable._shared_name, second_variable._shared_name, - network_scope_name)) - - -def _restore_custom_map_func_error_message(mapped_name, first_variable, - second_variable, network_name, - network_scope_name): - return ( - ("The map_func passed to restore_network_checkpoint for the Network '%s' " - "resulted in two variables named '%s' (originally '%s' and '%s'). Since " - "this is also an error when saving, this Network was " - "probably not saved with this map_func. Note that map_func " - "always maps from full variable names to checkpoint names; " - "there is no need to specify an inverse mapping.\n\n" - "Try stripping less from the variable names, or renaming parts " - "of the Network. For reference, variables created by sub-Layers " - "of this Network are prefixed with '%s', but if they are " - "re-used after being added to another Network they will have " - "that Network's full variable prefix instead.") % - (network_name, mapped_name, first_variable._shared_name, - second_variable._shared_name, network_scope_name)) - - -def _make_custom_getter_for_deferred_restorations(): - """Returns a custom getter which searches `deferred_restorations`. - - Returns: A tuple of (_custom_getter, deferred_restorations) - _custom_getter: The getter which should be added to variable_scopes where - variables will be created. - deferred_restorations: A list for _DeferredRestoration objects. Typically - empty when the getter is set, and expanded as deferred restorations are - requested. All new deferred restorations should be appended to the end of - the list, where they will have priority over older deferred restorations. - """ - deferred_restorations = [] - - def _custom_getter(getter, - name, - shape=None, - dtype=None, - initializer=None, - *args, - **kwargs): - """A custom getter which processes deferred restorations.""" - # Iterate over restorations, newest first (newer restorations will take - # precedence over older restorations, just like with immediate restorations - # into existing variables). - delayed_restoration = None - found_value = False - value_to_restore = None - for delayed_restoration in reversed(deferred_restorations): - checkpoint_name = delayed_restoration.map_func(name) - if (checkpoint_name in - delayed_restoration.checkpointed_variables_to_restore): - found_value = True - value_to_restore = ( - delayed_restoration - .checkpointed_variables_to_restore[checkpoint_name]) - if found_value: - break - # value_to_restore may be False because this variable is not in any - # checkpoint we are restoring, or None because we have explicitly set it to - # None when it was previously fetched. In either case, we don't need to - # set an initializer. - if found_value and value_to_restore is not None: - initializer = value_to_restore - shape = None - variable = getter( - name, - shape=shape, - dtype=dtype, - initializer=initializer, - *args, - **kwargs) - if found_value and value_to_restore is not None: - # Mark as already restored from this checkpoint. - delayed_restoration.checkpointed_variables_to_restore[ - checkpoint_name] = None - if not context.executing_eagerly(): - delayed_restoration.session.run(variable.initializer) - if found_value: - # Error checking should run even if we've already restored a value. - if delayed_restoration.restored_variables.setdefault( - checkpoint_name, variable) is not variable: - # Naming conflict. We've tried to initialize two variables with the - # same value from the checkpoint. - if delayed_restoration.map_func_is_user: - raise ValueError( - _restore_custom_map_func_error_message( - mapped_name=checkpoint_name, - first_variable=delayed_restoration - .restored_variables[checkpoint_name], - second_variable=variable, - network_name=delayed_restoration.network_name, - network_scope_name=delayed_restoration.network_scope_name)) - else: - raise ValueError( - _default_naming_conflict_error_message( - mapped_name=checkpoint_name, - first_variable=delayed_restoration - .restored_variables[checkpoint_name], - second_variable=variable, - network_name=delayed_restoration.network_name, - network_scope_name=delayed_restoration.network_scope_name)) - return variable - - return _custom_getter, deferred_restorations - - -def _make_prefix_stripping_map_fn(scope_name): - """Closure for stripping the scope name of a Network. - - Implemented as a closure rather than a member function to avoid reference - cycles in deferred restorations (this function should not have a reference to - the Network which created it). - - Args: - scope_name: The Network.scope_name to strip from variables. - - Returns: - A scope_name-stripping default `map_fn` for the Network. - """ - - def _strip_variable_prefix(original_variable_name): - """The default map_func for saving or restoring variables. - - Strips the variable prefix for the Network on which save/restore was called, - and leaves other variable names fully qualified in the checkpoint. - - Args: - original_variable_name: The _shared_name of the variable (no :0 suffix) to - map. - - Returns: - The checkpoint name of the variable. - """ - scope_name_with_slash = scope_name + "/" - if original_variable_name.startswith(scope_name_with_slash): - return original_variable_name[len(scope_name_with_slash):] - else: - return original_variable_name - - return _strip_variable_prefix - - -@deprecation.deprecated( - date=None, - instructions=( - "Please inherit from tf.keras.Model instead of tfe.Network, and use " - "tf.keras.Model.save_weights.")) -def save_network_checkpoint(network, save_path, global_step=None, - map_func=None): - """Save variables from the Network to a checkpoint. - - Args: - network: A Network object to save. - save_path: Either a checkpoint prefix or the name of a directory to save the - checkpoint in (in which case the checkpoint will be named based on the - Network name). - global_step: The global step to use when naming the checkpoint. If None - (default), we will first try to get the default global step. If that fails - because no default global step exists, then the checkpoint is created - without a global step suffix. - map_func: A function mapping fully qualified variable names (e.g. - 'my_network_1/dense_1/kernel') to names in the checkpoint. By default (if - `map_func=None`), the variable prefix for the network being restored - (`Network.scope_name + '/'`, e.g. 'my_network_1/') is stripped and all - other variable names (shared with other Networks) are left unchanged. - - Returns: - The checkpoint prefix for the saved checkpoint, which may be passed to - `Network.restore`. - Raises: - ValueError: If the Network has not yet been called, or if map_func results - in a name collision. - """ - if not network.built: - raise ValueError( - "Attempt to save the Network before it was first called. This means " - "variables have not yet been created, so there is nothing to save.") - network._set_scope() # scope_name should be available to map_funcs - if global_step is None: - global_step = training_util.get_global_step() - if os.path.isdir(save_path): - # If we were passed a directory, default to naming based on the Network - # name. - save_path = os.path.join(save_path, network.name.replace("/", "_")) - user_map_func = map_func - if map_func is None: - map_func = _make_prefix_stripping_map_fn(network.scope_name) - variable_map = {} - for variable in network.variables: - mapped_name = map_func(variable._shared_name) - if variable_map.setdefault(mapped_name, variable) is not variable: - if user_map_func is None: - # Instead of erroring out, we could just re-try and silently use the - # full variable names in the checkpoint. This could be odd for deeply - # nested sub-Networks (since the full prefix from the nesting would - # get added), so for now we'll let the user deal with this case. - raise ValueError( - _default_naming_conflict_error_message( - mapped_name=mapped_name, - first_variable=variable_map[mapped_name], - second_variable=variable, - network_name=network.name, - network_scope_name=network.scope_name)) - else: - # The user passed their own problematic map_func. - raise ValueError( - ("The map_func passed to save_network_checkpoint for the Network " - "'%s' resulted in two variables named '%s' ('%s' and '%s'). Try " - "stripping less from the variable names, or renaming parts of " - "the Network. For reference, variables created by sub-Layers of " - "this Network are prefixed with '%s', but if they are re-used " - "after being added to another Network, they will have that " - "Network's full variable prefix instead.") % - (network.name, mapped_name, variable_map[mapped_name]._shared_name, - variable._shared_name, network.scope_name)) - if context.executing_eagerly(): - sess = None - else: - sess = ops.get_default_session() - return saver_lib.Saver(variable_map).save( - sess=sess, - save_path=save_path, - write_meta_graph=False, - global_step=global_step) - - -def _add_deferred_restoration(layer, deferred_restoration): - """Add a deferred restoration to this Layer and all children. - - Restorations which are requested later have higher priority, and the highest - priority matching restoration is applied to a variable when it is created. - - Args: - layer: The Layer (may not be a Network) to operate on. - deferred_restoration: A _DeferredRestoration object. - """ - # Networks don't create variables at the moment, so this append isn't strictly - # necessary. We could get by with only adding deferred restorations to - # non-Network Layers. - if isinstance(layer, Network): - layer._set_scope() - # Make sure this Layer has a deferred restoration queue and a custom getter, - # then add our request to it. - if not hasattr(layer, "_custom_getter"): - assert not hasattr(layer, "_deferred_restorations") - layer._custom_getter, layer._deferred_restorations = ( - _make_custom_getter_for_deferred_restorations()) - # We use set_custom_getter because it avoids recursively calling up the - # variable_scope tree. We've done the tree traversal ourselves and have added - # the request to each Layer which needs it. - layer._scope.set_custom_getter(layer._custom_getter) - layer._deferred_restorations.append(deferred_restoration) - if isinstance(layer, Network): - for sublayer in layer.layers: - if not isinstance(sublayer, Network): - layer._set_scope_for_nonnetwork_sublayer(sublayer) - _add_deferred_restoration(sublayer, deferred_restoration) - - -def _restore_existing_variables(network, save_path, map_func, user_map_func): - """Use a standard Saver to restore existing variables from a checkpoint. - - Args: - network: A Network object to restore. - save_path: The checkpoint prefix or directory to read from. - map_func: The function to use when mapping from variable names to checkpoint - names. - user_map_func: The original map_func passed by the user, for error checking. - - Returns: - A dictionary mapping from checkpoint names to variable objects which have - been restored (for bookkeeping to avoid deferred restorations on these - variables). - Raises: - ValueError: If there is a name collision. - """ - existing_variables_by_checkpoint_name = {} - for variable in network.variables: - checkpoint_name = map_func(variable._shared_name) - if existing_variables_by_checkpoint_name.setdefault( - checkpoint_name, variable) is not variable: - if user_map_func is None: - raise ValueError( - _default_naming_conflict_error_message( - mapped_name=checkpoint_name, - first_variable=existing_variables_by_checkpoint_name[ - checkpoint_name], - second_variable=variable, - network_name=network.name, - network_scope_name=network.scope_name)) - else: - raise ValueError( - _restore_custom_map_func_error_message( - mapped_name=checkpoint_name, - first_variable=existing_variables_by_checkpoint_name[ - checkpoint_name], - second_variable=variable, - network_name=network.name, - network_scope_name=network.scope_name)) - if existing_variables_by_checkpoint_name: - if context.executing_eagerly(): - sess = None - else: - sess = ops.get_default_session() - saver_lib.Saver(var_list=existing_variables_by_checkpoint_name).restore( - sess=sess, save_path=save_path) - return existing_variables_by_checkpoint_name - - -def _set_restore_on_create(network, save_path, map_func, user_map_func, - existing_variables_by_checkpoint_name): - """If necessary, request deferred restorations of variables.""" - checkpoint_reader = checkpoint_utils.load_checkpoint(save_path) - checkpointed_variables_to_restore = {} - for checkpoint_name, _ in checkpoint_utils.list_variables(save_path): - if checkpoint_name in existing_variables_by_checkpoint_name: - # This variable was already created and restored. - continue - # Save the variable for later restoration in a custom getter. - checkpointed_variables_to_restore[checkpoint_name] = ( - checkpoint_reader.get_tensor(checkpoint_name)) - # Only set a deferred restoration if there are checkpoint variables which - # have not been assigned to existing variables. Note that this loses out on - # some opportunity for error checking, but avoids creating - # _DeferredRestoration objects once a Network has been built (so that - # restoring in a loop does not take increasing amounts of memory). - if checkpointed_variables_to_restore: - if context.executing_eagerly(): - sess = None - else: - sess = ops.get_default_session() - # We need a name for error messages. If we haven't been added to another - # Network yet, we're top-level. - network._finalize_name(False) - network._set_scope() - # Save a record of this restoration for use in the custom getter. - deferred_restoration = _DeferredRestoration( - map_func=map_func, - map_func_is_user=(user_map_func is not None), - checkpointed_variables_to_restore=checkpointed_variables_to_restore, - restored_variables={}, - session=sess, - network_name=network.name, - network_scope_name=network.scope_name) - # Add the deferred registration to non-Network children, and request that - # Networks propagate the request to their children. - _add_deferred_restoration(network, deferred_restoration) - - -@deprecation.deprecated( - date=None, - instructions=( - "Please inherit from tf.keras.Model instead of tfe.Network, and use " - "tf.keras.Model.load_weights.")) -def restore_network_checkpoint(network, save_path, map_func=None): - """Restore the Network from a checkpoint. - - If variables have already been created (typically when some or all of the - `Network` is built), they are assigned values from the checkpoint immediately, - overwriting any existing values (in graph mode the default session is used for - the assignments). - - If there are checkpoint entries which do not correspond to any existing - variables in the `Network`, these values are saved for deferred restoration; - their initial values will be the checkpointed values once they are - created. Requests for multiple deferred restorations behave the same way as - immediate restorations, in that later requests will take priority over earlier - requests relevant to the same variable. - - If this `Network` shares `Layer`s with another network, those `Layer`s will - also have their variables restored from the checkpoint. - - Args: - network: A Network object to restore. - save_path: The return value of `tfe.save_network_checkpoint`, or a directory - to search for a checkpoint. - map_func: A function mapping fully qualified variable names (e.g. - 'my_network_1/dense_1/kernel') to names in the checkpoint. By default (if - `map_func=None`), the variable prefix for the network being restored - (`Network.scope_name + '/'`, e.g. 'my_network_1/') is stripped and all - other variable names (shared with other Networks) are left unchanged. Note - that this is the _same_ map_func as `tfe.save_network_checkpoint`, not an - inverse mapping. - """ - network._finalize_name(parent_network=False) - network._set_scope() # scope_name should be available to map_funcs - if os.path.isdir(save_path): - # If we don't have a name yet, set no parent. - save_path = os.path.join(save_path, network.name.replace("/", "_")) - user_map_func = map_func - if map_func is None: - map_func = _make_prefix_stripping_map_fn(network.scope_name) - # Step one is to restore any existing variables from the checkpoint. - existing_variables_by_checkpoint_name = _restore_existing_variables( - network=network, - save_path=save_path, - map_func=map_func, - user_map_func=user_map_func) - # Step two is to set a custom getter which restores variables on creation, - # for those variables which have not been added to sub-Layers yet. - _set_restore_on_create( - network=network, - save_path=save_path, - map_func=map_func, - user_map_func=user_map_func, - existing_variables_by_checkpoint_name=( - existing_variables_by_checkpoint_name)) diff --git a/tensorflow/contrib/eager/python/network_test.py b/tensorflow/contrib/eager/python/network_test.py deleted file mode 100644 index b3e8daddaf2..00000000000 --- a/tensorflow/contrib/eager/python/network_test.py +++ /dev/null @@ -1,1254 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import gc - -from tensorflow.contrib.eager.python import network -from tensorflow.contrib.layers.python.layers import regularizers -from tensorflow.python.eager import function -from tensorflow.python.eager import test -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import test_util -from tensorflow.python.layers import core -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.training import training_util -from tensorflow.python.training.tracking import util as trackable_utils - - -# pylint: disable=not-callable -class MyNetwork(network.Network): - - def __init__(self, name=None): - super(MyNetwork, self).__init__(name=name) - self.l1 = self.track_layer(core.Dense(1, use_bias=False)) - - def call(self, x): - return self.l1(x) - - -class RegularizedNetwork(network.Network): - - def __init__(self): - super(RegularizedNetwork, self).__init__() - self.l1 = self.track_layer(core.Dense( - 1, - bias_regularizer=regularizers.l1_regularizer(2.0), - kernel_regularizer=regularizers.l1_regularizer(2.0))) - self.l2 = self.track_layer(core.Dense( - 1, - bias_regularizer=regularizers.l1_regularizer(2.0))) - - def call(self, values): - return self.l2(self.l1(values)) - - -class NetworkTest(test.TestCase): - - def test_checkpointing_not_implemented(self): - checkpoint_directory = self.get_temp_dir() - checkpoint = trackable_utils.Checkpoint(net=MyNetwork()) - with self.assertRaises(NotImplementedError): - checkpoint.save(checkpoint_directory) - - def _save_modify_load_network_built(self, net, global_step=None): - checkpoint_directory = self.get_temp_dir() - checkpoint_path = network.save_network_checkpoint( - network=net, save_path=checkpoint_directory, global_step=global_step) - input_value = constant_op.constant([[42.0]]) - original_output = self.evaluate(net(input_value)) - for var in net.variables: - self.evaluate(var.assign(var + 1.)) - self.assertGreater( - self.evaluate(net(input_value)), - original_output) - # Either the returned explicit checkpoint path or the directory should work. - network.restore_network_checkpoint(net, save_path=checkpoint_directory) - self.assertAllEqual( - original_output, - self.evaluate(net(input_value))) - for var in net.variables: - self.evaluate(var.assign(var + 2.)) - network.restore_network_checkpoint(net, save_path=checkpoint_path) - self.assertAllEqual( - original_output, - self.evaluate(net(input_value))) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testTrainableAttribute(self): - net = network.Network() - self.assertTrue(net.trainable) - with self.assertRaises(AttributeError): - net.trainable = False - self.assertTrue(net.trainable) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testNetworkCall(self): - net = MyNetwork(name="abcd") - net(constant_op.constant([[2.0]])) # Force variables to be created. - self.assertEqual(1, len(net.trainable_variables)) - self.evaluate(net.trainable_variables[0].assign([[17.0]])) - # TODO(josh11b): Support passing Python values to networks. - result = net(constant_op.constant([[2.0]])) - self.assertEqual(34.0, self.evaluate(result)) - - def testReplacingNetworkCallWithDefun(self): - net = MyNetwork(name="abcd") - net.call = function.defun(net.call) - x = constant_op.constant([[2.0]]) - net(x) # Force variables to be created. - self.evaluate(net.trainable_variables[0].assign([[17.0]])) - - result = net(x) # Build and execute the TensorFlow function - self.assertEqual(34.0, self.evaluate(result)) - - # Force the creation of another TensorFlow function by changing input shape - y = constant_op.constant([[1.0], [2.0]]) - result = net(y) - self.assertAllEqual([[17.0], [34.0]], self.evaluate(result)) - - # TODO(allenl): This test creates garbage in some Python versions - @test_util.run_in_graph_and_eager_modes - def testNetworkSaveRestoreAlreadyBuilt(self): - net = MyNetwork(name="abcd") - with self.assertRaisesRegexp( - ValueError, "Attempt to save the Network before it was first called"): - network.save_network_checkpoint(net, self.get_temp_dir()) - net(constant_op.constant([[2.0]])) - self.evaluate(net.trainable_variables[0].assign([[17.0]])) - self._save_modify_load_network_built(net, global_step=None) - self._save_modify_load_network_built(net, global_step=10) - - # TODO(allenl): This test creates garbage in some Python versions - @test_util.run_in_graph_and_eager_modes - def testSaveRestoreDefaultGlobalStep(self): - net = MyNetwork(name="abcd") - net(constant_op.constant([[2.0]])) - self.evaluate(net.variables[0].assign([[3.]])) - default_global_step = training_util.get_or_create_global_step() - self.evaluate(default_global_step.assign(4242)) - save_path = network.save_network_checkpoint(net, self.get_temp_dir()) - self.assertIn("abcd-4242", save_path) - - # TODO(allenl): This test creates garbage in some Python versions - @test_util.run_in_graph_and_eager_modes - def testNetworkSaveAndRestoreIntoUnbuilt(self): - save_dir = self.get_temp_dir() - net1 = MyNetwork() - test_input = constant_op.constant([[2.0]]) - net1(test_input) - self.evaluate(net1.trainable_variables[0].assign([[17.0]])) - save_path = network.save_network_checkpoint(net1, save_dir) - # With a pre-build restore we should have the same value. - net2 = MyNetwork() - network.restore_network_checkpoint(net2, save_path) - self.assertAllEqual(self.evaluate(net1(test_input)), - self.evaluate(net2(test_input))) - self.assertIsNot(net1.variables[0], net2.variables[0]) - self.assertAllEqual(self.evaluate(net1.variables[0]), - self.evaluate(net2.variables[0])) - - @test_util.run_in_graph_and_eager_modes - def testNetworkMatchesLayerVariableNames(self): - zero = constant_op.constant([[0.]]) - layer_one = core.Dense(1, use_bias=False) - layer_one(zero) - layer_two = core.Dense(1, use_bias=False) - layer_two(zero) - - class TwoLayerNet(network.Network): - - def __init__(self, name=None): - super(TwoLayerNet, self).__init__(name=name) - self.first = self.track_layer(core.Dense( - 1, use_bias=False)) - self.second = self.track_layer(core.Dense( - 1, use_bias=False)) - - def call(self, x): - return self.second(self.first(x)) - - net = TwoLayerNet() - net(zero) - self.assertEqual("two_layer_net/" + layer_one.variables[0].name, - net.first.variables[0].name) - self.assertEqual("two_layer_net/" + layer_two.variables[0].name, - net.second.variables[0].name) - - @test_util.run_in_graph_and_eager_modes - def testLoadIntoUnbuiltSharedLayer(self): - - class Owner(network.Network): - - def __init__(self, name=None): - super(Owner, self).__init__(name=name) - self.first = self.track_layer(core.Dense( - 1, name="first_layer", use_bias=False)) - - def call(self, x): - return self.first(x) - - first_owner = Owner() - - class User(network.Network): - - def __init__(self, use_layer, name=None): - super(User, self).__init__(name=name) - self.first = self.track_layer(use_layer) - self.second = self.track_layer(core.Dense( - 1, name="second_layer", use_bias=False)) - - def call(self, x): - return self.second(self.first(x)) - - class LikeUserButNotSharing(network.Network): - - def __init__(self, name=None): - super(LikeUserButNotSharing, self).__init__(name=name) - self.first = self.track_layer(core.Dense( - 1, name="first_layer", use_bias=False)) - self.second = self.track_layer(core.Dense( - 1, name="second_layer", use_bias=False)) - - def call(self, x): - return self.second(self.first(x)) - - checkpoint_creator = LikeUserButNotSharing(name="checkpoint_creator") - one = constant_op.constant([[1.0]]) - checkpoint_creator(one) - self.assertEqual(2, len(checkpoint_creator.variables)) - self.evaluate(checkpoint_creator.variables[0].assign([[5.]])) - self.evaluate(checkpoint_creator.variables[1].assign([[6.]])) - # Re-map the variable names so that with default restore mapping we'll - # attempt to restore into the unbuilt Layer. - name_mapping = { - "checkpoint_creator/first_layer/kernel": "owner/first_layer/kernel", - "checkpoint_creator/second_layer/kernel": "second_layer/kernel", - } - save_path = network.save_network_checkpoint( - checkpoint_creator, - self.get_temp_dir(), - map_func=lambda full_name: name_mapping[full_name]) - load_into = User(use_layer=first_owner.first) - network.restore_network_checkpoint(load_into, save_path) - self.assertEqual(0, len(first_owner.variables)) - self.assertAllEqual(self.evaluate(checkpoint_creator(one)), - self.evaluate(load_into(one))) - self.assertEqual(1, len(first_owner.variables)) - self.assertAllEqual([[5.]], self.evaluate(load_into.variables[0])) - self.assertAllEqual([[6.]], self.evaluate(load_into.variables[1])) - first_owner(one) - self.assertAllEqual([[5.]], self.evaluate(first_owner.variables[0])) - - # Try again with a garbage collected parent. - first_owner = Owner() - load_into = User(use_layer=first_owner.first) - del first_owner - gc.collect() - def _restore_map_func(original_name): - if original_name.startswith("owner/"): - return original_name.replace("owner/", "owner_1/") - else: - return "user_1/" + original_name - with self.assertRaisesRegexp(ValueError, "garbage collected"): - network.restore_network_checkpoint( - load_into, save_path, map_func=_restore_map_func) - - @test_util.run_in_graph_and_eager_modes - def testRestoreIntoSubNetwork(self): - - class Parent(network.Network): - - def __init__(self, name=None): - super(Parent, self).__init__(name=name) - self.first = self.track_layer(MyNetwork()) - self.second = self.track_layer(MyNetwork()) - - def call(self, x): - return self.first(self.second(x)) - - one = constant_op.constant([[3.]]) - whole_model_saver = Parent() - whole_model_saver(one) - self.evaluate(whole_model_saver.variables[0].assign([[15.]])) - self.evaluate(whole_model_saver.variables[1].assign([[16.]])) - whole_model_checkpoint = network.save_network_checkpoint( - whole_model_saver, self.get_temp_dir()) - - save_from = MyNetwork() - save_from(one) - self.evaluate(save_from.variables[0].assign([[5.]])) - checkpoint = network.save_network_checkpoint(save_from, self.get_temp_dir()) - save_into_parent = Parent() - network.restore_network_checkpoint(save_into_parent, whole_model_checkpoint) - network.restore_network_checkpoint(save_into_parent.first, checkpoint) - # deferred loading multiple times is fine - network.restore_network_checkpoint(save_into_parent.first, checkpoint) - save_into_parent(one) # deferred loading - self.assertAllEqual([[5.]], self.evaluate(save_into_parent.variables[0])) - self.assertAllEqual([[16.]], self.evaluate(save_into_parent.variables[1])) - - # Try again with the opposite ordering, and we should get different results - # (deferred restoration should happen the same way non-deferred happens, - # with later restorations overwriting older ones). - save_into_parent = Parent() - # deferred loading multiple times is fine - network.restore_network_checkpoint(save_into_parent.first, checkpoint) - network.restore_network_checkpoint(save_into_parent, whole_model_checkpoint) - save_into_parent(one) # deferred loading - # We've overwritten the sub-Network restore. - self.assertAllEqual([[15.]], self.evaluate(save_into_parent.variables[0])) - self.assertAllEqual([[16.]], self.evaluate(save_into_parent.variables[1])) - - self.evaluate(save_into_parent.variables[0].assign([[3.]])) - self.evaluate(save_into_parent.variables[1].assign([[4.]])) - network.restore_network_checkpoint(save_into_parent.second, checkpoint) - self.assertAllEqual([[5.]], self.evaluate(save_into_parent.variables[1])) - with self.assertRaisesRegexp(errors_impl.NotFoundError, - "not found in checkpoint"): - # The checkpoint is incompatible. - network.restore_network_checkpoint(save_into_parent, checkpoint) - - @test_util.run_in_graph_and_eager_modes - def testCustomMapCollisionErrors(self): - - class Parent(network.Network): - - def __init__(self, name=None): - super(Parent, self).__init__(name=name) - self.first = self.track_layer(MyNetwork()) - self.second = self.track_layer(MyNetwork()) - - def call(self, x): - return self.first(self.second(x)) - - make_checkpoint = Parent() - one = constant_op.constant([[1.]]) - make_checkpoint(one) - self.evaluate(make_checkpoint.variables[0].assign([[2.]])) - self.evaluate(make_checkpoint.variables[1].assign([[3.]])) - with self.assertRaisesRegexp( - ValueError, - "The map_func passed to save_network_checkpoint for the Network " - "'parent' resulted in two variables named 'foo'"): - network.save_network_checkpoint( - make_checkpoint, self.get_temp_dir(), map_func=lambda n: "foo") - checkpoint = network.save_network_checkpoint( - network=make_checkpoint.first, - save_path=self.get_temp_dir(), - map_func=lambda n: "foo") - loader = Parent() - network.restore_network_checkpoint( - loader, checkpoint, map_func=lambda n: "foo") - with self.assertRaisesRegexp( - ValueError, - ("The map_func passed to restore_network_checkpoint for the Network" - " 'parent_1' resulted in two variables named 'foo'")): - loader(one) - loader = Parent() - loader(one) - with self.assertRaisesRegexp( - ValueError, - ("The map_func passed to restore_network_checkpoint for the Network" - " 'parent_2' resulted in two variables named 'foo'")): - network.restore_network_checkpoint( - loader, checkpoint, map_func=lambda n: "foo") - - @test_util.run_in_graph_and_eager_modes - def testDefaultMapCollisionErrors(self): - - one = constant_op.constant([[1.]]) - first = core.Dense(1, name="dense", use_bias=False) - first(one) - - class Parent(network.Network): - - def __init__(self, name=None): - super(Parent, self).__init__(name=name) - self.first = self.track_layer(first) - self.second = self.track_layer(core.Dense(1, use_bias=False)) - - def call(self, x): - return self.first(self.second(x)) - - make_checkpoint = Parent() - one = constant_op.constant([[1.]]) - make_checkpoint(one) - self.evaluate(make_checkpoint.variables[0].assign([[2.]])) - self.evaluate(make_checkpoint.variables[1].assign([[3.]])) - with self.assertRaisesRegexp( - ValueError, - ("The default checkpoint variable name mapping strategy for Network " - "'parent' resulted in a naming conflict.")): - network.save_network_checkpoint(make_checkpoint, self.get_temp_dir()) - - class Compatible(network.Network): - - def __init__(self, name=None): - super(Compatible, self).__init__(name=name) - self.first = self.track_layer(core.Dense(1, use_bias=False)) - - def call(self, x): - return self.first(x) - - successful_checkpoint = Compatible() - successful_checkpoint(one) - self.evaluate(successful_checkpoint.variables[0].assign([[-1.]])) - checkpoint_path = network.save_network_checkpoint( - successful_checkpoint, self.get_temp_dir()) - load_checkpoint = Parent() - load_checkpoint(one) - with self.assertRaisesRegexp( - ValueError, - ("The default checkpoint variable name mapping strategy for Network " - "'parent_1' resulted in a naming conflict.")): - network.restore_network_checkpoint(load_checkpoint, checkpoint_path) - - def testNoReferenceCyclesAfterCall(self): - - class ChildNetwork(network.Network): - - def __init__(self, name=None): - super(ChildNetwork, self).__init__(name=name) - - def call(self, x): - return x * 2. - - class ParentNetwork(network.Network): - - def __init__(self, name=None): - super(ParentNetwork, self).__init__(name=name) - self.l1 = self.track_layer(ChildNetwork()) - - def call(self, x): - return self.l1(x) - - one = constant_op.constant([[1.0]]) - gc.disable() - gc.collect() - previous_gc_debug_flags = gc.get_debug() - gc.set_debug(gc.DEBUG_SAVEALL) - preexisting = len(gc.garbage) - net = ParentNetwork() - net(one) - del net - gc.collect() - # There should be no additional garbage requiring collection. - self.assertEqual(preexisting, len(gc.garbage)) - gc.set_debug(previous_gc_debug_flags) - gc.enable() - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testAnonymousNoNameInitially(self): - net = MyNetwork() - with self.assertRaisesRegexp(ValueError, "does not yet have a final name"): - net.name # pylint: disable=pointless-statement - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testExplicitHasNameInitially(self): - net = MyNetwork(name="abcd") - self.assertEqual("abcd", net.name) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testUsingResourceVariables(self): - net = MyNetwork() - net(constant_op.constant([[0.]])) - self.assertIsInstance(net.trainable_weights[0], - resource_variable_ops.ResourceVariable) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testVariableRegularizers(self): - net = RegularizedNetwork() - net(constant_op.constant([[1.]])) - self.evaluate(net.variables[0].assign([[2.]])) - self.evaluate(net.variables[1].assign([3.])) - self.evaluate(net.variables[2].assign([[-2.]])) - self.evaluate(net.variables[3].assign([4.])) - self.assertAllEqual([4., 6., 8.], self.evaluate(net.losses)) - self.evaluate(net.variables[3].assign([5.])) - self.assertAllEqual([4., 6., 10.], self.evaluate(net.losses)) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testDuplicateNameError(self): - one = constant_op.constant([[1.]]) - net = MyNetwork(name="foo") - net(one) - with self.assertRaisesRegexp( - ValueError, "named 'foo' already exists"): - net1 = MyNetwork(name="foo") - net1(one) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testWrappingInVariableScope(self): - one = constant_op.constant([[1.]]) - # Naming happens in the order of first build rather than the order of - # construction, but for clarity they're the same here and construction is - # annotated. - outside_net_before = MyNetwork() # name=my_network - outside_net_before(one) - captured_scope = variable_scope.get_variable_scope() - with variable_scope.variable_scope("outside_scope"): - net1 = MyNetwork() # name=outside_scope/my_network - net1(one) - name_conflict1 = MyNetwork(name="name_conflict") # fine, unique so far - name_conflict2 = MyNetwork(name="name_conflict") # error on build - with variable_scope.variable_scope("inside_scope"): - # No issue here since the name is unique within its scope. - name_conflict3 = MyNetwork(name="name_conflict") - net2 = MyNetwork() # name=outside_scope/my_network_2 to avoid the - # variable_scope my_network_1 below. - vs_name_conflict = MyNetwork(name="vs_name_conflict") # conflict below - with variable_scope.variable_scope("intervening_scope"): - with variable_scope.variable_scope(captured_scope): - with variable_scope.variable_scope("outside_scope"): - name_conflict4 = MyNetwork(name="name_conflict") # error on build - with variable_scope.variable_scope("my_network_1"): - pass - with variable_scope.variable_scope("vs_name_conflict"): - pass - net3 = MyNetwork() # name=outside_scope/my_network_4 - name_conflict1(one) - with self.assertRaisesRegexp( - ValueError, "named 'name_conflict' already exists"): - name_conflict2(one) - name_conflict3(one) - net2(one) - with self.assertRaisesRegexp( - ValueError, "or a variable_scope was created with this name"): - vs_name_conflict(one) - with self.assertRaisesRegexp( - ValueError, "named 'name_conflict' already exists"): - name_conflict4(one) - self.assertEqual("outside_scope/name_conflict", - name_conflict1.name) - self.assertStartsWith( - expected_start="outside_scope/name_conflict/dense/", - actual=name_conflict1.variables[0].name) - self.assertEqual("outside_scope/inside_scope/name_conflict", - name_conflict3.name) - self.assertStartsWith( - expected_start="outside_scope/inside_scope/name_conflict/dense/", - actual=name_conflict3.variables[0].name) - self.assertEqual("outside_scope/my_network", net1.name) - self.assertStartsWith( - expected_start="outside_scope/my_network/dense/", - actual=net1.trainable_weights[0].name) - self.assertEqual("outside_scope/my_network_2", net2.name) - self.assertStartsWith( - expected_start="outside_scope/my_network_2/dense/", - actual=net2.trainable_weights[0].name) - net3(one) - self.assertEqual("outside_scope/my_network_3", net3.name) - self.assertStartsWith( - expected_start="outside_scope/my_network_3/dense/", - actual=net3.trainable_weights[0].name) - outside_net_after = MyNetwork() - outside_net_after(one) - self.assertEqual("my_network", outside_net_before.name) - self.assertStartsWith( - expected_start="my_network/dense/", - actual=outside_net_before.trainable_weights[0].name) - self.assertEqual("my_network_1", outside_net_after.name) - self.assertStartsWith( - expected_start="my_network_1/dense/", - actual=outside_net_after.trainable_weights[0].name) - - @test_util.run_in_graph_and_eager_modes - def testVariableScopeStripping(self): - with variable_scope.variable_scope("scope1"): - with variable_scope.variable_scope("scope2"): - net = MyNetwork() - net(constant_op.constant([[2.0]])) - self.evaluate(net.variables[0].assign([[42.]])) - self.assertEqual(net.name, "scope1/scope2/my_network") - self.assertStartsWith( - expected_start="scope1/scope2/my_network/dense/", - actual=net.trainable_weights[0].name) - save_path = network.save_network_checkpoint(net, self.get_temp_dir()) - self.assertIn("scope1_scope2_my_network", save_path) - restore_net = MyNetwork() - # Delayed restoration - network.restore_network_checkpoint(restore_net, save_path) - restore_net(constant_op.constant([[1.0]])) - self.assertAllEqual([[42.]], - self.evaluate(restore_net.variables[0])) - self.evaluate(restore_net.variables[0].assign([[-1.]])) - # Immediate restoration - network.restore_network_checkpoint(restore_net, save_path) - self.assertAllEqual([[42.]], - self.evaluate(restore_net.variables[0])) - - @test_util.run_in_graph_and_eager_modes - def testLayerNamesRespected(self): - class ParentNetwork(network.Network): - - def __init__(self): - super(ParentNetwork, self).__init__() - self.first = self.track_layer( - core.Dense(1, use_bias=False, name="explicit_name")) - - def call(self, x): - return self.first(x) - - one = constant_op.constant([[1.]]) - net = ParentNetwork() - net(one) - self.assertStartsWith(expected_start="parent_network/explicit_name/", - actual=net.trainable_weights[0].name) - self.assertEqual("explicit_name", net.first.name) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testWrappingInAnonymousVariableScope(self): - # Named outside variable_scopes are not supported at the moment. However, - # blank-named top level variable scopes do not change variable names, and so - # can be used to set the properties of Network variables. - was_called = [False] - def _custom_getter(getter, *args, **kwargs): - was_called[0] = True - return getter(*args, **kwargs) - with variable_scope.variable_scope("", custom_getter=_custom_getter): - net = MyNetwork() - one = constant_op.constant([[1.]]) - net(one) - self.assertTrue(was_called[0]) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testReasonableSlashError(self): - with self.assertRaisesRegexp( - ValueError, "not allowed in Network names"): - MyNetwork(name="slash/slash") - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testNoVariableScopeNames(self): - with self.assertRaisesRegexp( - ValueError, "VariableScopes are not valid Network names"): - with variable_scope.variable_scope("some_scope") as vs: - MyNetwork(name=vs) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testVariableScopeNameCollision(self): - with variable_scope.variable_scope("abcd"): - pass - with self.assertRaisesRegexp( - ValueError, "or a variable_scope was created with this name"): - net = MyNetwork(name="abcd") - one = constant_op.constant([[1.]]) - net(one) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testNetworkVariablesDoNotInterfere(self): - core.Dense(1, use_bias=True) # Should not interfere with naming. - net1 = MyNetwork() - net2 = MyNetwork() - one = constant_op.constant([[1.]]) - net1(one) - net2(one) - # Layer names typically are globally unique rather than being unique within - # the scope of their first use. However, within a Network they must be named - # locally so that previous Layer construction does not interfere with - # variable naming (e.g. add a Layer construction before the Network, - # suddenly your previously saved checkpoint is incompatible). - self.assertEqual("dense", net1.l1.name) - self.assertEqual("dense", net2.l1.name) - self.evaluate(net1.trainable_weights[0].assign([[1.]])) - self.evaluate(net2.trainable_weights[0].assign([[2.]])) - self.assertEqual(2., self.evaluate(net2.trainable_weights[0])) - self.assertEqual(1., self.evaluate(net1.trainable_weights[0])) - self.assertStartsWith(expected_start="my_network/dense/", - actual=net1.trainable_weights[0].name) - self.assertStartsWith(expected_start="my_network_1/dense/", - actual=net2.trainable_weights[0].name) - - @test_util.run_in_graph_and_eager_modes - def testNestableAnonymous(self): - - # The case where no explicit names are specified. We make up unique names, - # and these should match the variable names. - class ParentNetwork(network.Network): - - def __init__(self): - super(ParentNetwork, self).__init__() - self.first = self.track_layer(MyNetwork()) - self.second = self.track_layer(MyNetwork()) - - def call(self, x): - return self.second(self.first(x)) - - one = constant_op.constant([[1.]]) - net = ParentNetwork() - net(one) - self.assertStartsWith(expected_start="parent_network/my_network/dense", - actual=net.trainable_weights[0].name) - self.assertStartsWith(expected_start="parent_network/my_network/dense", - actual=net.first.trainable_weights[0].name) - self.assertStartsWith(expected_start="parent_network/my_network_1/dense", - actual=net.trainable_weights[1].name) - self.assertStartsWith(expected_start="parent_network/my_network_1/dense", - actual=net.second.trainable_weights[0].name) - self.assertEqual("parent_network", net.name) - self.assertEqual("my_network", net.first.name) - self.assertEqual("my_network_1", net.second.name) - - net2 = ParentNetwork() - net2(one) - self.assertStartsWith(expected_start="parent_network_1/my_network/dense", - actual=net2.trainable_weights[0].name) - self.assertStartsWith(expected_start="parent_network_1/my_network/dense", - actual=net2.first.trainable_weights[0].name) - self.assertStartsWith(expected_start="parent_network_1/my_network_1/dense", - actual=net2.trainable_weights[1].name) - self.assertStartsWith(expected_start="parent_network_1/my_network_1/dense", - actual=net2.second.trainable_weights[0].name) - self.assertEqual("parent_network_1", net2.name) - self.assertEqual("my_network", net2.first.name) - self.assertEqual("my_network_1", net2.second.name) - - @test_util.run_in_graph_and_eager_modes - def testNestableExplicit(self): - - # We have explicit network names and everything is globally unique. - class ParentNetwork(network.Network): - - def __init__(self): - super(ParentNetwork, self).__init__(name="unique_parent_name") - self.first = self.track_layer( - MyNetwork(name="first_unique_child_name")) - self.second = self.track_layer( - MyNetwork(name="second_unique_child_name")) - - def call(self, x): - return self.second(self.first(x)) - - one = constant_op.constant([[1.]]) - net = ParentNetwork() - net(one) - self.assertStartsWith( - expected_start="unique_parent_name/first_unique_child_name/dense", - actual=net.trainable_weights[0].name) - self.assertStartsWith( - expected_start="unique_parent_name/second_unique_child_name/dense", - actual=net.trainable_weights[1].name) - self.assertEqual("unique_parent_name", net.name) - self.assertEqual("first_unique_child_name", net.first.name) - self.assertEqual("second_unique_child_name", net.second.name) - - @test_util.run_in_graph_and_eager_modes - def testLayerNetworkNameInteractions(self): - - # Same base name as core.Dense; Networks and non-Network Layers with the - # same base name should use the same numbering system. - class Dense(network.Network): - - def __init__(self): - super(Dense, self).__init__() - self.first = self.track_layer(core.Dense(1, use_bias=False)) - - def call(self, x): - return self.first(x) - - class MixedLayerNetwork(network.Network): - - def __init__(self): - super(MixedLayerNetwork, self).__init__() - self.first = self.track_layer(core.Dense(1, use_bias=False)) - self.second = self.track_layer(core.Dense(1, use_bias=False)) - self.third = self.track_layer(Dense()) - self.fourth = self.track_layer(core.Dense(1, use_bias=False)) - self.fifth = self.track_layer(core.Dense(1, use_bias=False)) - - def call(self, x): - return self.fifth(self.fourth(self.third(self.second(self.first(x))))) - - one = constant_op.constant([[1.]]) - net = MixedLayerNetwork() - net(one) - self.assertEqual("dense", net.first.name) - self.assertEqual("dense_1", net.second.name) - self.assertEqual("dense_2", net.third.name) - self.assertEqual("dense_3", net.fourth.name) - self.assertEqual("dense_4", net.fifth.name) - # Note that this is _not_ the default naming behavior for Layers. Layers - # which are added to Networks follow Network variable naming conventions - # (i.e. variable names = network name unless variable sharing). Nested - # Layers revert to Layer behavior. - self.assertStartsWith(expected_start="mixed_layer_network/dense/", - actual=net.trainable_weights[0].name) - self.assertStartsWith(expected_start="mixed_layer_network/dense_1/", - actual=net.trainable_weights[1].name) - self.assertStartsWith(expected_start="mixed_layer_network/dense_2/", - actual=net.trainable_weights[2].name) - self.assertStartsWith(expected_start="mixed_layer_network/dense_3/", - actual=net.trainable_weights[3].name) - self.assertStartsWith(expected_start="mixed_layer_network/dense_4/", - actual=net.trainable_weights[4].name) - self.assertEqual("mixed_layer_network", net.name) - - @test_util.run_in_graph_and_eager_modes - def testNestableExplicitCollisions(self): - - # We have explicit network names and they are unique within the layer - # they're added to. - class ParentNetwork(network.Network): - - def __init__(self): - super(ParentNetwork, self).__init__(name="nonunique_name") - self.first = self.track_layer( - MyNetwork(name="nonunique_name")) - self.second = self.track_layer( - MyNetwork(name="second_unique_child_name")) - - def call(self, x): - return self.second(self.first(x)) - - one = constant_op.constant([[1.]]) - net = ParentNetwork() - net(one) - self.assertStartsWith( - expected_start="nonunique_name/nonunique_name/dense", - actual=net.trainable_weights[0].name) - self.assertStartsWith( - expected_start="nonunique_name/second_unique_child_name/dense", - actual=net.trainable_weights[1].name) - self.assertEqual("nonunique_name", net.name) - self.assertEqual("nonunique_name", net.first.name) - self.assertEqual("second_unique_child_name", net.second.name) - - @test_util.run_in_graph_and_eager_modes - def testNestableExplicitWithAnonymousParent(self): - - # A parent network is instantiated multiple times with explicitly named - # children. We shouldn't throw any name errors. - class ParentNetwork(network.Network): - - def __init__(self): - super(ParentNetwork, self).__init__() - self.first = self.track_layer( - MyNetwork(name="first_unique_child_name")) - self.second = self.track_layer( - MyNetwork(name="second_unique_child_name")) - - def call(self, x): - return self.second(self.first(x)) - - one = constant_op.constant([[1.]]) - net = ParentNetwork() - net(one) - self.assertStartsWith( - expected_start="parent_network/first_unique_child_name/dense/", - actual=net.trainable_weights[0].name) - self.assertStartsWith( - expected_start="parent_network/second_unique_child_name/dense/", - actual=net.trainable_weights[1].name) - self.assertEqual("parent_network", net.name) - self.assertEqual("first_unique_child_name", net.first.name) - self.assertEqual("second_unique_child_name", net.second.name) - - net2 = ParentNetwork() - net2(one) - self.assertStartsWith( - expected_start="parent_network_1/first_unique_child_name/dense", - actual=net2.trainable_weights[0].name) - self.assertStartsWith( - expected_start="parent_network_1/second_unique_child_name/dense", - actual=net2.trainable_weights[1].name) - self.assertEqual("parent_network_1", net2.name) - self.assertEqual("first_unique_child_name", net2.first.name) - self.assertEqual("second_unique_child_name", net2.second.name) - - @test_util.run_in_graph_and_eager_modes - def testNestableExplicitSameLayerCollisions(self): - - # We have explicit network names and they are _not_ unique within the layer - # they're added to. Error. - class ParentNetwork(network.Network): - - def __init__(self): - super(ParentNetwork, self).__init__(name="unique_parent_name") - self.first = self.track_layer(MyNetwork(name="nonunique_name")) - self.second = self.track_layer(MyNetwork(name="nonunique_name")) - - def call(self, x): - return self.second(self.first(x)) - - with self.assertRaisesRegexp(ValueError, "nonunique_name"): - ParentNetwork() - - @test_util.run_in_graph_and_eager_modes - def testAnonymousVariableSharing(self): - - # Two "owned" Networks - class FirstParentNetwork(network.Network): - - def __init__(self): - super(FirstParentNetwork, self).__init__() - self.first = self.track_layer(MyNetwork()) - self.second = self.track_layer(MyNetwork()) - - def call(self, x): - return self.second(self.first(x)) - - one = constant_op.constant([[1.]]) - net = FirstParentNetwork() - net(one) - - # One Network shared with FirstParentNetwork, one owned Network. Same name, - # but this is OK because only one is owned. This name collision is - # avoidable; we could have looked at the base_name of the non-owned Network - # and incremented our naming based on that. - class SecondParentNetwork(network.Network): - - def __init__(self): - super(SecondParentNetwork, self).__init__() - self.first = self.track_layer(net.first) - self.second = self.track_layer(MyNetwork()) - - def call(self, x): - return self.second(self.first(x)) - - net2 = SecondParentNetwork() - net2(one) - - self.assertStartsWith( - expected_start="first_parent_network/my_network/dense/", - actual=net2.trainable_weights[0].name) - self.assertStartsWith( - expected_start="second_parent_network/my_network/dense/", - actual=net2.trainable_weights[1].name) - self.assertEqual("second_parent_network", net2.name) - self.assertTrue(net2.first is net.first) - self.assertEqual("my_network", net2.first.name) - self.assertEqual("my_network", net2.second.name) - - # No name collision; the owned Network is added first and has a different - # name than the shared Network. - class ThirdParentNetwork(network.Network): - - def __init__(self): - super(ThirdParentNetwork, self).__init__() - self.first = self.track_layer(MyNetwork()) - self.second = self.track_layer(net.second) - - def call(self, x): - return self.second(self.first(x)) - - net3 = ThirdParentNetwork() - net3(one) - - self.assertStartsWith( - expected_start="third_parent_network/my_network/dense", - actual=net3.trainable_weights[0].name) - self.assertStartsWith( - expected_start="first_parent_network/my_network_1/dense", - actual=net3.trainable_weights[1].name) - self.assertEqual("third_parent_network", net3.name) - self.assertTrue(net3.second is net.second) - self.assertEqual("my_network", net3.first.name) - self.assertEqual("my_network_1", net3.second.name) - - # "Unavoidable" same-name Layer. The owned name is added first (fixed), then - # a shared Network is added with the same name. - class FourthParentNetwork(network.Network): - - def __init__(self): - super(FourthParentNetwork, self).__init__() - self.first = self.track_layer(MyNetwork()) - self.second = self.track_layer(net.first) - - def call(self, x): - return self.second(self.first(x)) - - net4 = FourthParentNetwork() - net4(one) - - self.assertStartsWith( - expected_start="fourth_parent_network/my_network/dense/", - actual=net4.trainable_weights[0].name) - self.assertStartsWith( - expected_start="first_parent_network/my_network/dense/", - actual=net4.trainable_weights[1].name) - self.assertEqual("fourth_parent_network", net4.name) - self.assertTrue(net4.second is net.first) - self.assertEqual("my_network", net4.first.name) - self.assertEqual("my_network", net4.second.name) - - @test_util.run_in_graph_and_eager_modes - def testRecursiveLayerRenaming(self): - core.Dense(1) # Under default Layer naming, would change subsequent names. - - class NetworkWithLayerChildren(network.Network): - - def __init__(self): - super(NetworkWithLayerChildren, self).__init__() - self.first = self.track_layer(core.Dense(1, use_bias=False)) - self.second = self.track_layer(core.Dense(1, use_bias=False)) - - def call(self, x): - return self.second(self.first(x)) - - class ParentNetwork(network.Network): - - def __init__(self): - super(ParentNetwork, self).__init__() - self.first = self.track_layer(NetworkWithLayerChildren()) - self.second = self.track_layer(NetworkWithLayerChildren()) - - def call(self, x): - return self.second(self.first(x)) - - net = ParentNetwork() - one = constant_op.constant([[1.]]) - net(one) - - self.assertStartsWith( - expected_start=("parent_network/network_with_layer_children/" - "dense/"), - actual=net.trainable_weights[0].name) - self.assertStartsWith( - expected_start=("parent_network/network_with_layer_children/" - "dense_1/"), - actual=net.trainable_weights[1].name) - self.assertStartsWith( - expected_start=("parent_network/network_with_layer_children_1/" - "dense/"), - actual=net.trainable_weights[2].name) - self.assertStartsWith( - expected_start=("parent_network/network_with_layer_children_1/" - "dense_1/"), - actual=net.trainable_weights[3].name) - self.assertEqual("parent_network", net.name) - self.assertEqual("network_with_layer_children", net.first.name) - self.assertEqual("network_with_layer_children_1", net.second.name) - self.assertEqual("dense", net.first.first.name) - self.assertEqual("dense_1", net.first.second.name) - self.assertEqual("dense", net.second.first.name) - self.assertEqual("dense_1", net.second.second.name) - - @test_util.run_in_graph_and_eager_modes - def testCallInDifferentOrderThanConstruct(self): - shared_network = MyNetwork() - - class FirstNetwork(network.Network): - - def __init__(self): - super(FirstNetwork, self).__init__() - self.first = self.track_layer(shared_network) - self.second = self.track_layer(MyNetwork()) - - def call(self, x): - return self.second(self.first(x)) - - class SecondNetwork(network.Network): - - def __init__(self): - super(SecondNetwork, self).__init__() - self.first = self.track_layer(shared_network) - self.second = self.track_layer(MyNetwork()) - - def call(self, x): - return self.second(self.first(x)) - - net1 = FirstNetwork() - net2 = SecondNetwork() - - one = constant_op.constant([[1.]]) - net2(one) - net1(one) - - self.assertStartsWith( - expected_start="first_network/my_network/dense/", - actual=net1.trainable_weights[0].name) - self.assertStartsWith( - expected_start="first_network/my_network_1/dense/", - actual=net1.trainable_weights[1].name) - self.assertStartsWith( - expected_start="first_network/my_network/dense/", - actual=net2.trainable_weights[0].name) - self.assertStartsWith( - expected_start="second_network/my_network/dense/", - actual=net2.trainable_weights[1].name) - self.assertTrue(net1.trainable_weights[0] is net2.trainable_weights[0]) - self.assertEqual("first_network", net1.name) - self.assertEqual("my_network", net1.first.name) - self.assertEqual("my_network_1", net1.second.name) - self.assertTrue(net2.first is net1.first) - self.assertEqual("my_network", net2.second.name) - - @test_util.run_in_graph_and_eager_modes - def testLayerCallInDifferentOrderThanConstruct(self): - # Same idea as testCallInDifferentOrderThanConstruct, but this time with a - # non-Network Layer shared between two Networks rather than a - # Network. Naming should follow the same rules. - shared_layer = core.Dense(1, use_bias=False) - - class FirstNetwork(network.Network): - - def __init__(self): - super(FirstNetwork, self).__init__() - self.first = self.track_layer(shared_layer) - self.second = self.track_layer(core.Dense(1, use_bias=False)) - - def call(self, x): - return self.second(self.first(x)) - - class SecondNetwork(network.Network): - - def __init__(self): - super(SecondNetwork, self).__init__() - self.first = self.track_layer(shared_layer) - self.second = self.track_layer(core.Dense(1, use_bias=False)) - - def call(self, x): - return self.second(self.first(x)) - - net1 = FirstNetwork() - net2 = SecondNetwork() - - one = constant_op.constant([[1.]]) - net2(one) - net1(one) - - self.assertStartsWith( - expected_start="first_network/dense/", - actual=net1.trainable_weights[0].name) - self.assertStartsWith( - expected_start="first_network/dense_1/", - actual=net1.trainable_weights[1].name) - self.assertStartsWith( - expected_start="first_network/dense/", - actual=net2.trainable_weights[0].name) - self.assertStartsWith( - expected_start="second_network/dense/", - actual=net2.trainable_weights[1].name) - self.assertTrue(net1.trainable_weights[0] is net2.trainable_weights[0]) - self.assertEqual("first_network", net1.name) - self.assertEqual("dense", net1.first.name) - self.assertEqual("dense_1", net1.second.name) - self.assertTrue(net2.first is net1.first) - self.assertEqual("dense", net2.second.name) - - @test_util.run_in_graph_and_eager_modes - def testLayerAlreadyBuilt(self): - one = constant_op.constant([[1.]]) - core.Dense(1, use_bias=False) # pre-built layers use global naming - one = constant_op.constant([[1.]]) - core.Dense(1, use_bias=False)(one) - shared_layer = core.Dense(1, use_bias=False) - shared_layer(one) - - class FirstNetwork(network.Network): - - def __init__(self): - super(FirstNetwork, self).__init__() - self.first = self.track_layer(shared_layer) - self.second = self.track_layer(core.Dense(1, use_bias=False)) - - def call(self, x): - return self.second(self.first(x)) - - net = FirstNetwork() - net(one) - - self.assertStartsWith( - expected_start="dense_1/", # Pre-built layers have variable names which - # do not match their layer names. - actual=net.trainable_weights[0].name) - self.assertStartsWith( - expected_start="first_network/dense/", - actual=net.trainable_weights[1].name) - self.assertTrue( - net.trainable_weights[0] is shared_layer.trainable_weights[0]) - self.assertEqual("first_network", net.name) - self.assertEqual("dense_3", net.first.name) - self.assertEqual("dense", net.second.name) - - -class SequentialTest(test.TestCase): - - @test_util.assert_no_garbage_created - def testTwoLayers(self): - # Create a sequential network with one layer. - net = network.Sequential([core.Dense(1, use_bias=False)]) - - # Set that layer's weights so it multiplies by 3 - l1 = net.get_layer(index=0) - net(constant_op.constant([[2.0]])) # Create l1's variables - self.assertEqual(1, len(l1.trainable_variables)) - l1.trainable_variables[0].assign([[3.0]]) - self.assertEqual(21.0, net(constant_op.constant([[7.0]])).numpy()) - - # Add a second layer to the network. - l2 = core.Dense(1, use_bias=False) - net.add(l2) - - # Set the second layer's weights so it multiplies by 11 - net(constant_op.constant([[2.0]])) # Create l2's variables - self.assertEqual(1, len(l2.trainable_variables)) - l2.trainable_variables[0].assign([[11.0]]) - self.assertEqual(231.0, net(constant_op.constant([[7.0]])).numpy()) - - @test_util.assert_no_garbage_created - def testFunctions(self): - # Create a sequential network with one function. - net = network.Sequential([nn_ops.relu]) - two = constant_op.constant(2.0) - self.assertEqual(2.0, net(two).numpy()) - self.assertEqual(0.0, net(-two).numpy()) - # Add a second function. - net.add(math_ops.negative) - self.assertEqual(-2.0, net(two).numpy()) - - @test_util.assert_no_garbage_created - def testTrainingLayer(self): - net = network.Sequential([core.Dropout(0.99999)]) - two = constant_op.constant(2.0) - self.assertEqual(2.0, net(two).numpy()) - self.assertEqual(2.0, net(two, training=False).numpy()) - for _ in range(20): - with_dropout = net(two, training=True).numpy() - self.assertIn(with_dropout, [0.0, 2.0]) - if with_dropout == 0.0: - return - # Should only fail spuriously 1 in 10^100 runs. - self.fail("Didn't see dropout happen after 20 tries.") - - @test_util.assert_no_garbage_created - def testTrainingFunction(self): - # Output depends on value of "training". - def add_training(input_value, training=None): - if training is None: - return input_value - elif training: - return input_value + 1 - return input_value - 1 - - # Passing a "training" argument to double would cause an error. - def double(input_value): - return 2 * input_value - - net = network.Sequential([add_training, double]) - two = constant_op.constant(2) - self.assertEqual(4, net(two).numpy()) - self.assertEqual(2, net(two, training=False).numpy()) - self.assertEqual(6, net(two, training=True).numpy()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/eager/python/parameter_server.py b/tensorflow/contrib/eager/python/parameter_server.py deleted file mode 100644 index c96a03dd999..00000000000 --- a/tensorflow/contrib/eager/python/parameter_server.py +++ /dev/null @@ -1,284 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""EXPERIMENTAL utilities for parameter server training with eager execution. - -Note: this should eventually be merged with the distribution strategy for -ParameterServer. -""" - - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import contextlib -import time - -from tensorflow.python.eager import context -from tensorflow.python.framework import ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.training.tracking import base as trackable - - -def _eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode): - """Creates a variable handle with information to do shape inference.""" - container = ops.get_default_graph()._container # pylint: disable=protected-access - if container is None: - container = "" - handle = resource_variable_ops.var_handle_op(shape=shape, dtype=dtype, - shared_name=shared_name, - name=name, - container=container) - if graph_mode: - return handle - - with context.graph_mode(), ops.Graph().as_default() as graph: - h = resource_variable_ops.var_handle_op(shape=shape, dtype=dtype, - shared_name=shared_name, - name=name, - container=container) - - # Tensor._handle_data contains information for the shape-inference code to - # know the shape and dtype of the variable pointed to by a handle. Since - # shape inference doesn't run in eager mode we copy this data here for when - # the handle is captured by an eager mode function. - # pylint: disable=protected-access - handle._handle_data = resource_variable_ops.get_resource_handle_data(h) - # pylint: enable=protected-access - # Clean up op->graph->op reference cycles. - ops.dismantle_graph(graph) - return handle - - -class SharedVariable(resource_variable_ops.BaseResourceVariable): - """Experimental Variable designed for parameter server training. - - A SharedVariable has a name and two instances of SharedVariable with the - same name will have the same value, even if they are in different Sessions, - as long as they are placed on the same device. - - The storage associated with SharedVariables is also not deleted when they go - out of scope. - """ - - def __init__(self, # pylint: disable=super-init-not-called - initial_value=None, - trainable=True, - name=None, - dtype=None, - constraint=None, - initialize=True, - **unused_kwargs): - """Creates a variable. - - Args: - initial_value: A `Tensor`, or Python object convertible to a `Tensor`, - which is the initial value for the Variable. The initial value must have - a shape specified unless `validate_shape` is set to False. Can also be a - callable with no argument that returns the initial value when called. - (Note that initializer functions from init_ops.py must first be bound - to a shape before being used here.) - trainable: If `True`, automatically watches this variable on GradientTape - whenever it's used. - name: Optional name for the variable. Defaults to `'Variable'` and gets - uniquified automatically. - dtype: If set, initial_value will be converted to the given type. - If None, either the datatype will be kept (if initial_value is - a Tensor) or float32 will be used (if it is a Python object convertible - to a Tensor). - constraint: An optional projection function to be applied to the variable - after being updated by an `Optimizer` (e.g. used to implement norm - constraints or value constraints for layer weights). The function must - take as input the unprojected Tensor representing the value of the - variable and return the Tensor for the projected value - (which must have the same shape). Constraints are not safe to - use when doing asynchronous distributed training. - initialize: if True, runs initialization in eager execution; leaves the - variable uninitialized otherwise. - - Raises: - ValueError: If the initial value is not specified, or does not have a - shape and `validate_shape` is `True`. - """ - if initial_value is None: - raise ValueError("initial_value must be specified.") - init_from_fn = callable(initial_value) - - if isinstance(initial_value, ops.Tensor) and hasattr( - initial_value, "graph") and initial_value.graph.building_function: - raise ValueError("Tensor-typed variable initializers must either be " - "wrapped in an init_scope or callable " - "(e.g., `tf.Variable(lambda : " - "tf.truncated_normal([10, 40]))`) when building " - "functions. Please file a feature request if this " - "restriction inconveniences you.") - - if constraint is not None and not callable(constraint): - raise ValueError("The `constraint` argument must be a callable.") - - if isinstance(initial_value, trackable.CheckpointInitialValue): - self._maybe_initialize_trackable() - self._update_uid = initial_value.checkpoint_position.restore_uid - initial_value = initial_value.wrapped_value - - self._trainable = trainable - self._save_slice_info = None - # Store the graph key so optimizers know how to only retrieve variables from - # this graph. - self._graph_key = ops.get_default_graph()._graph_key # pylint: disable=protected-access - with ops.init_scope(): - self._in_graph_mode = not context.executing_eagerly() - with ops.name_scope(name, "Variable", [] - if init_from_fn else [initial_value]) as name: - # pylint: disable=protected-access - handle_name = ops.name_from_scope_name(name) - shared_name = handle_name - if init_from_fn: - # Use attr_scope and device(None) to simulate the behavior of - # colocate_with when the variable we want to colocate with doesn't - # yet exist. - if self._in_graph_mode: - with ops.name_scope("Initializer"), ops.device(None): - initial_value = ops.convert_to_tensor( - initial_value(), name="initial_value", dtype=dtype) - self._handle = _eager_safe_variable_handle( - shape=initial_value.get_shape(), - dtype=initial_value.dtype.base_dtype, - shared_name=shared_name, - name=name, - graph_mode=self._in_graph_mode) - self._shape = initial_value.get_shape() - else: - initial_value = initial_value() - with ops.name_scope("Initializer"): - initial_value = ops.convert_to_tensor( - initial_value, name="initial_value", dtype=dtype) - self._handle = _eager_safe_variable_handle( - shape=initial_value.get_shape(), - dtype=initial_value.dtype.base_dtype, - shared_name=shared_name, - name=name, - graph_mode=False) - self._shape = initial_value.get_shape() - # pylint: enable=protected-access - - # Or get the initial value from a Tensor or Python object. - else: - with ops.name_scope("Initializer"): - initial_value = ops.convert_to_tensor( - initial_value, name="initial_value", dtype=dtype) - # pylint: disable=protected-access - if (self._in_graph_mode and initial_value is not None and - initial_value.op._get_control_flow_context() is not None): - raise ValueError( - "Initializer for variable %s is from inside a control-flow " - "construct, such as a loop or conditional. When creating a " - "variable inside a loop or conditional, use a lambda as the " - "initializer." % name) - # pylint: enable=protected-access - self._handle = _eager_safe_variable_handle( - shape=initial_value.get_shape(), - dtype=initial_value.dtype.base_dtype, - shared_name=shared_name, - name=name, - graph_mode=self._in_graph_mode) - self._shape = initial_value.get_shape() - - self._unique_id = shared_name - self._initial_value = initial_value if self._in_graph_mode else None - self._handle_name = handle_name + ":0" - self._dtype = initial_value.dtype.base_dtype - self._constraint = constraint - - if self._in_graph_mode: - with ops.name_scope("IsInitialized"): - self._is_initialized_op = ( - resource_variable_ops.var_is_initialized_op(self._handle)) - if initial_value is not None: - with ops.name_scope("Assign") as n, ops.colocate_with(self._handle): - self._initializer_op = ( - resource_variable_ops.assign_variable_op( - self._handle, - self._try_guard_against_uninitialized_dependencies( - initial_value), - name=n)) - with ops.name_scope("Read"), ops.colocate_with(self._handle): - # Manually assign reads to the handle's device to avoid log - # messages. - with ops.device(self._handle.device): - value = self._read_variable_op() - self._graph_element = value - self._cached_value = None - else: - if initialize: - resource_variable_ops.assign_variable_op(self._handle, - initial_value) - self._is_initialized_op = None - self._initializer_op = None - self._graph_element = None - self._cached_value = None - - self._handle_deleter = object() - self._cached_shape_as_list = None - - -@contextlib.contextmanager -def parameter_server_scope(is_chief, ps_job_name, num_ps_tasks): - """Strategy to use parameter servers in eager. - - Creates SharedVariable objects for variables created in this scope. These - SharedVariable objects will be placed round-robin on the parameter servers - specified by the ps_job_name and num_ps_tasks arguments. - - To use parameter servers you need only to wrap your model initialization in - this scope: - - ``` - with tf.contrib.eager.parameter_server_scope( - is_chief, ps_job_name, num_ps_tasks): - my_model = tf.keras.Sequential([...]) # Or - input = tf.keras.Input(...) - .... - my_model = tf.keras.Model(input, output) - my_model.compile(...) - # or other usages of the model. - ``` - - Args: - is_chief: Boolean. Whether this worker is responsible for initializing - variables. - ps_job_name: The name of the ps job in this cluster. - num_ps_tasks: The number of ps tasks to use. - - Yields: - a context manager. - """ - # Note: capturing in a list to allow assignment. - ps_index = [0] - - def variable_creator_scope(unused_next_creator, **kwargs): - kwargs["initialize"] = is_chief - with ops.device( - "/job:%s/task:%s" % (ps_job_name, ps_index[0] % num_ps_tasks)): - ps_index[0] += 1 - v = SharedVariable(**kwargs) - if not is_chief: - while not resource_variable_ops.var_is_initialized_op(v.handle): - time.sleep(10) - return v - - with variable_scope.variable_creator_scope(variable_creator_scope): - yield diff --git a/tensorflow/contrib/eager/python/remote_test.py b/tensorflow/contrib/eager/python/remote_test.py deleted file mode 100644 index e3329ca1c79..00000000000 --- a/tensorflow/contrib/eager/python/remote_test.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for remote eager execution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import os - -import numpy as np - -from tensorflow.contrib.eager.python import parameter_server -from tensorflow.core.protobuf import cluster_pb2 -from tensorflow.core.protobuf import tensorflow_server_pb2 -from tensorflow.python import pywrap_tensorflow -from tensorflow.python.eager import backprop -from tensorflow.python.eager import context -from tensorflow.python.eager import function -from tensorflow.python.eager import remote -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import server_lib - -JOB_NAME = "remote_device" -ALT_JOB_NAME = "alt_remote_device" - - -def run_sync_and_async(f): - """Execute all test methods in the given class in sync and async modes.""" - - @functools.wraps(f) - def decorator(self, *args, **kwargs): - with context.execution_mode(context.ASYNC): - f(self, *args, **kwargs) - - with context.execution_mode(context.SYNC): - f(self, *args, **kwargs) - - return decorator - - -def get_server_def(job_name, local_server_port, remote_server_addresses, - task_index): - """Returns a server def with a single job + multiple tasks.""" - cluster_def = cluster_pb2.ClusterDef() - job_def = cluster_def.job.add() - job_def.name = job_name - job_def.tasks[0] = "localhost:%d" % local_server_port - - for i, remote_server_address in enumerate(remote_server_addresses, start=1): - job_def.tasks[i] = remote_server_address - - server_def = tensorflow_server_pb2.ServerDef( - cluster=cluster_def, - job_name=job_name, - task_index=task_index, - protocol="grpc") - - return server_def - - -class RemoteExecutionTest(test.TestCase): - - def __init__(self, methodName="runTest"): # pylint: disable=invalid-name - super(RemoteExecutionTest, self).__init__(methodName) - self._cached_server1 = server_lib.Server.create_local_server() - self._cached_server2 = server_lib.Server.create_local_server() - - os.environ["TF_EAGER_REMOTE_USE_SEND_TENSOR_RPC"] = "1" - - self._cached_server1_target = self._cached_server1.target[len("grpc://"):] - self._cached_server2_target = self._cached_server2.target[len("grpc://"):] - - def setUp(self): - # Start the local server. - local_port = pywrap_tensorflow.TF_PickUnusedPortOrDie() - context.set_server_def( - server_def=get_server_def( - JOB_NAME, - local_server_port=local_port, - remote_server_addresses=[ - self._cached_server1_target, self._cached_server2_target - ], - task_index=0)) - - @test_util.run_gpu_only - @run_sync_and_async - def testGpuToRemoteCopy(self): - with ops.device("gpu:0"): - x = array_ops.ones([2, 2]) - with ops.device("job:%s/replica:0/task:1/device:CPU:0" % JOB_NAME): - y = math_ops.matmul(x, x) - - np.testing.assert_array_equal([[2, 2], [2, 2]], y.numpy()) - - @run_sync_and_async - def testDefunMatmul(self): - """Basic remote eager execution with defun.""" - - mm_defun = function.defun(math_ops.matmul) - with ops.device("job:%s/replica:0/task:1/device:CPU:0" % JOB_NAME): - x1 = array_ops.ones([2, 2]) - with ops.device("job:%s/replica:0/task:2/device:CPU:0" % JOB_NAME): - x2 = array_ops.ones([2, 2]) - y = mm_defun(x1, x2) - np.testing.assert_array_equal([[2, 2], [2, 2]], y.numpy()) - - @run_sync_and_async - def testSimpleMatmul(self): - """Basic remote eager execution.""" - - with ops.device("job:%s/replica:0/task:1/device:CPU:0" % JOB_NAME): - x1 = array_ops.ones([2, 2]) - with ops.device("job:%s/replica:0/task:2/device:CPU:0" % JOB_NAME): - x2 = array_ops.ones([2, 2]) - y = math_ops.matmul(x1, x2) - np.testing.assert_array_equal([[2, 2], [2, 2]], y.numpy()) - - def testParameterServer(self): - with parameter_server.parameter_server_scope( - is_chief=True, ps_job_name=JOB_NAME, num_ps_tasks=3): - v0 = variables.Variable([1.0], name="v0") - v1 = variables.Variable([2.0], name="v1") - v0.assign(v0 * v1) - self.assertAllEqual(v0.read_value(), [2.0]) - self.assertAllEqual(v0.device, - "/job:%s/replica:0/task:0/device:CPU:0" % JOB_NAME) - self.assertAllEqual(v1.device, - "/job:%s/replica:0/task:1/device:CPU:0" % JOB_NAME) - v1.assign_add(v1) - # Simulate aliasing another variable of the same name as v1 - with ops.device("/job:%s/replica:0/task:1/device:CPU:0" % JOB_NAME): - v1_replica = parameter_server.SharedVariable( - [1.0], name="v1", initialize=False) - self.assertAllEqual(v1_replica.read_value(), [4.0]) - - @run_sync_and_async - def testSimpleWeightRead(self): - """Basic remote eager weight read.""" - - with ops.device("job:%s/replica:0/task:1/device:CPU:0" % JOB_NAME): - w = resource_variable_ops.ResourceVariable([[2.0]]) - loss = w * w - np.testing.assert_array_equal([[4.0]], loss.numpy()) - - @run_sync_and_async - def testTapeWeightRead(self): - """Remote eager weight read in a tape.""" - - with ops.device("job:%s/replica:0/task:1/device:CPU:0" % JOB_NAME): - w = resource_variable_ops.ResourceVariable([[3.0]]) - with backprop.GradientTape() as tape: - loss = w * w - - grad = tape.gradient(loss, w) - np.testing.assert_array_equal([[9.0]], loss.numpy()) - np.testing.assert_array_equal([[6.0]], grad.numpy()) - - @run_sync_and_async - def testServerDefChanged(self): - """Update server def, and run ops on new cluster.""" - context.set_server_def( - server_def=get_server_def( - ALT_JOB_NAME, - local_server_port=0, - remote_server_addresses=[ - self._cached_server1_target, self._cached_server2_target - ], - task_index=0)) - - with ops.device("job:%s/replica:0/task:1/device:CPU:0" % ALT_JOB_NAME): - x1 = array_ops.ones([2, 2]) - y = math_ops.matmul(x1, x1) - np.testing.assert_array_equal([[2, 2], [2, 2]], y.numpy()) - - # Set the server def back to JOB_NAME - context.set_server_def( - server_def=get_server_def( - JOB_NAME, - local_server_port=0, - remote_server_addresses=[ - self._cached_server1_target, self._cached_server2_target - ], - task_index=0)) - - with ops.device("job:%s/replica:0/task:1/device:CPU:0" % JOB_NAME): - x1 = array_ops.ones([2, 2]) - y = math_ops.matmul(x1, x1) - np.testing.assert_array_equal([[2, 2], [2, 2]], y.numpy()) - - @run_sync_and_async - def testConnectToRemoteServer(self): - """Basic server connection.""" - remote.connect_to_remote_host(self._cached_server1_target) - - with ops.device("job:worker/replica:0/task:0/device:CPU:0"): - x1 = array_ops.ones([2, 2]) - x2 = array_ops.ones([2, 2]) - y = math_ops.matmul(x1, x2) - np.testing.assert_array_equal([[2, 2], [2, 2]], y.numpy()) - - @run_sync_and_async - def testContextDeviceUpdated(self): - """Tests that the context device is correctly updated.""" - - with ops.device("cpu:0"): - x1 = array_ops.ones([2, 2]) - x2 = array_ops.ones([2, 2]) - y = math_ops.matmul(x1, x2) - np.testing.assert_array_equal([[2, 2], [2, 2]], y.numpy()) - - # `y` is placed on the local CPU as expected. - self.assertEqual(y.device, - "/job:%s/replica:0/task:0/device:CPU:0" % JOB_NAME) - - @test_util.run_gpu_only - @run_sync_and_async - def testGPUToRemoteCopy(self): - """Tests that the remote copy happens satisfactorily.""" - x1 = array_ops.ones([2, 2]).gpu() - - with ops.device("/job:remote_device/replica:0/task:1/device:CPU:0"): - x2 = x1._copy() # pylint: disable=protected-access - - np.testing.assert_array_equal(x1.numpy(), x2.numpy()) - - -if __name__ == "__main__": - ops.enable_eager_execution() - test.main() diff --git a/tensorflow/contrib/eager/python/saver.py b/tensorflow/contrib/eager/python/saver.py deleted file mode 100644 index 8649e56556a..00000000000 --- a/tensorflow/contrib/eager/python/saver.py +++ /dev/null @@ -1,187 +0,0 @@ -"""Saver for eager mode TensorFlow.""" -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import contextlib - -from tensorflow.python.eager import context -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.training import checkpoint_utils -from tensorflow.python.training import saver as _saver - - -def _init_from_checkpoint(self, *args, **kwargs): - """Overrides default init by loading value from checkpoint.""" - # pylint: disable=protected-access - self._old_init(*args, **kwargs) - ckpt_name = self._map_func(self._shared_name) - if ckpt_name not in self._ckpt_var_cache: - raise errors.NotFoundError(None, None, - "%s not found in checkpoint" % ckpt_name) - - val = self._ckpt_var_cache.get(ckpt_name, None) - if val is not None: - self.assign(val) - # Avoid assigning for the second time. - self._ckpt_var_cache[ckpt_name] = None - # pylint: enable=protected-access - - -@contextlib.contextmanager -def restore_variables_on_create(save_path, map_func=None): - """ContextManager that restores variables on creation. - - When save_path is None (e.g. No checkpoint), does nothing. - Otherwise, it preloads all values from checkpoint. When the - corresponding variable is first created, it assigns the checkpoint - value to the variable. - - ```python - with restore_variables_on_create( - tf.train.latest_checkpoint(checkpoint_dir)): - ``` - - Args: - save_path: The checkpoint file prefix. - map_func: A function that given the variable name as argument - and returns a variable name in checkpoint for restore. If - None, use the variable with the same name in checkpoint to restore. - It's an error that the mapped variable name doesn't exist in - checkpoint. - - Yields: - Nothing. - - Raises: - NotFoundError: If the variable is not found in checkpoint. - ValueError: If not used in eager mode or map_func is not callable. - """ - if not context.executing_eagerly(): - raise ValueError( - "Currently, restore_variables_on_create can only be used with " - "eager execution enabled.") - if save_path: - if map_func is None: - map_func_wrapper = lambda self, x: x - else: - if not callable(map_func): - raise ValueError("map_func must be callable.") - map_func_wrapper = lambda self, x: map_func(x) - - ckpt_var_cache = {} - reader = checkpoint_utils.load_checkpoint(save_path) - for k, _ in checkpoint_utils.list_variables(save_path): - ckpt_var_cache[k] = reader.get_tensor(k) - - old_init = getattr(resource_variable_ops.ResourceVariable, - "_init_from_args", None) - assert old_init, "ResourceVariable misses _init_from_args method." - setattr(resource_variable_ops.ResourceVariable, "_init_from_args", - _init_from_checkpoint) - setattr(resource_variable_ops.ResourceVariable, "_old_init", old_init) - setattr(resource_variable_ops.ResourceVariable, "_map_func", - map_func_wrapper) - setattr(resource_variable_ops.ResourceVariable, "_ckpt_var_cache", - ckpt_var_cache) - try: - yield - except Exception as e: - raise e - finally: - if save_path: - setattr(resource_variable_ops.ResourceVariable, "_init_from_args", - old_init) - setattr(resource_variable_ops.ResourceVariable, "_old_init", None) - setattr(resource_variable_ops.ResourceVariable, "_map_func", None) - setattr(resource_variable_ops.ResourceVariable, "_ckpt_var_cache", None) - - -class Saver(object): - """A tf.compat.v1.train.Saver adapter for use when eager execution is enabled. - - `Saver`'s name-based checkpointing strategy is fragile. Please switch to - `tf.train.Checkpoint` or `tf.keras.Model.save_weights`, which perform a more - robust object-based saving. These APIs will load checkpoints written by - `Saver`. - """ - - def __init__(self, var_list): - """A tf.compat.v1.train.Saver adapter for use when eager execution is enabled. - - The API, and on-disk format, mimic tf.compat.v1.train.Saver except that no - Session is needed. - - Args: - var_list: The list of variables that will be saved and restored. Either a - list of `tf.Variable` objects, or a dictionary mapping names to - `tf.Variable` objects. - - Raises: - RuntimeError: if invoked when eager execution has not been enabled. - """ - if not context.executing_eagerly(): - raise RuntimeError("tfe.Saver can only be used when eager " - "execution is enabled. Use tf.train.Saver when " - "building graphs.") - self._saver = _saver.Saver(var_list=var_list) - - def save(self, file_prefix, global_step=None): - """Saves variables. - - Args: - file_prefix: Path prefix of files created for the checkpoint. - global_step: If provided the global step number is appended to file_prefix - to create the checkpoint filename. The optional argument can be a - Tensor, a Variable, or an integer. - - Returns: - A string: prefix of filenames created for the checkpoint. This may be - an extension of file_prefix that is suitable to pass as an argument - to a subsequent call to `restore()`. - """ - with ops.device("/device:CPU:0"): - return self._saver.save( - None, file_prefix, write_meta_graph=False, global_step=global_step) - - def restore(self, file_prefix): - """Restores previously saved variables. - - Args: - file_prefix: Path prefix where parameters were previously saved. - Typically obtained from a previous `save()` call, or from - `tf.train.latest_checkpoint`. - """ - with ops.device("/device:CPU:0"): - self._saver.restore(None, file_prefix) - - -def get_optimizer_variables(optimizer): - """Returns a list of variables for the given `tf.compat.v1.train.Optimizer`. - - Equivalent to `optimizer.variables()`. - - Args: - optimizer: An instance of `tf.compat.v1.train.Optimizer` which has created - variables (typically after a call to `Optimizer.minimize`). - - Returns: - A list of variables which have been created by the `Optimizer`. - """ - return optimizer.variables() diff --git a/tensorflow/contrib/eager/python/saver_test.py b/tensorflow/contrib/eager/python/saver_test.py deleted file mode 100644 index 91bc75213c7..00000000000 --- a/tensorflow/contrib/eager/python/saver_test.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Tests for eager mode Saver.""" -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from tensorflow.contrib.eager.python import saver as _saver -from tensorflow.python.eager import context -from tensorflow.python.eager import test -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.training import adam -from tensorflow.python.training import gradient_descent -from tensorflow.python.training import momentum -from tensorflow.python.training import rmsprop - - -class SaverTest(test.TestCase): - - def _dev(self): - return '/device:GPU:0' if context.num_gpus() else '/device:CPU:0' - - def testBasics(self): - with ops.device(self._dev()): - v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') - def model(): - return array_ops.constant(2.0) * v1 - - ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') - - _ = model() - saver = _saver.Saver([v1]) - saver.save(ckpt_prefix) - v1.assign(2.0) - self.assertEqual(v1.read_value().numpy(), 2.0) - - saver.restore(ckpt_prefix) - self.assertEqual(v1.read_value().numpy(), 1.0) - - def testSameNameNoClobbering(self): - with ops.device(self._dev()): - v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') - v2 = resource_variable_ops.ResourceVariable(2.0, name='v1') - saver = _saver.Saver([v1, v2]) - ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') - with self.assertRaisesRegexp(ValueError, 'v1'): - saver.save(ckpt_prefix) - - def testSameObjectOK(self): - with ops.device(self._dev()): - v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') - # While different objects with the same shared_name are not good, passing - # in the same object multiple times is fine. - saver = _saver.Saver([v1, v1]) - ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') - saver.save(ckpt_prefix) - - def testSaveByDict(self): - with ops.device(self._dev()): - v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') - v2 = resource_variable_ops.ResourceVariable(1.0, name='v2') - def model(): - return array_ops.constant(2.0) * v1 * v2 - - ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') - - # Save the variables under different names. - _ = model() - saver = _saver.Saver({'ckpt/v1': v1, 'ckpt/v2': v2}) - saver.save(ckpt_prefix) - v1.assign(2.0) - v2.assign(2.0) - self.assertEqual(v1.read_value().numpy(), 2.0) - self.assertEqual(v2.read_value().numpy(), 2.0) - # Can still restore it. - saver.restore(ckpt_prefix) - self.assertEqual(v1.read_value().numpy(), 1.0) - # However, cannot restore it with default name. - with self.assertRaisesOpError('not found in checkpoint'): - saver = _saver.Saver([v1, v2]).restore(ckpt_prefix) - - # Can specify which variable in ckpt to restore to which variable. - def map_func(x): - return {'v3': 'ckpt/v1', 'v4': 'ckpt/v2'}.get(x, x) - with _saver.restore_variables_on_create(ckpt_prefix, map_func): - v3 = resource_variable_ops.ResourceVariable(2.0, name='v3') - v4 = resource_variable_ops.ResourceVariable(2.0, name='v4') - self.assertEqual(v3.read_value().numpy(), 1.0) - self.assertEqual(v4.read_value().numpy(), 1.0) - - def testRestoreOnCreate(self): - with ops.device(self._dev()): - def model(init_val): - v1 = resource_variable_ops.ResourceVariable(init_val, name='v1') - return array_ops.constant(1.0) * v1, v1 - - ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') - _, v1 = model(1.0) - saver = _saver.Saver([v1]) - saver.save(ckpt_prefix) - - saver = _saver.Saver([v1]) - with _saver.restore_variables_on_create(ckpt_prefix): - # Value is from checkpoint, but not from argument. - ret, _ = model(2.0) - self.assertEqual(ret.numpy(), 1.0) - - def testRestoreNotFound(self): - with ops.device(self._dev()): - def model(v): - return array_ops.constant(1.0) * v - - ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') - v = resource_variable_ops.ResourceVariable(1.0, name='v1') - _ = model(v) - saver = _saver.Saver([v]) - saver.save(ckpt_prefix) - - with self.assertRaisesRegexp(errors.NotFoundError, - 'v2 not found in checkpoint'): - with _saver.restore_variables_on_create(ckpt_prefix): - _ = model(resource_variable_ops.ResourceVariable(1.0, name='v2')) - - -class GetOptimizerTests(test.TestCase): - - def _optimizer_test_template(self, optimizer): - """Checks save and restore. Returns the optimizer variables.""" - v = resource_variable_ops.ResourceVariable([[2., 3.]], name='v') - loss_fn = lambda: v[0, 0] ** 2 + v[0, 1] ** 2 - optimizer.minimize(loss_fn) - optimizer_variables = _saver.get_optimizer_variables(optimizer) - saver = _saver.Saver(optimizer_variables + [v]) - checkpoint_path = saver.save(self.get_temp_dir()) - optimizer.minimize(loss_fn) - after_first_minimize = v.numpy() - # After we restore, the next step should be exactly the same as the one we - # just did. - saver.restore(checkpoint_path) - optimizer.minimize(loss_fn) - self.assertAllEqual(after_first_minimize, v.numpy()) - return optimizer_variables - - def testAdam(self): - optimizer = adam.AdamOptimizer(0.1) - self._optimizer_test_template(optimizer) - - def testGradientDescent(self): - optimizer = gradient_descent.GradientDescentOptimizer(0.02) - self.assertEqual(0, len(self._optimizer_test_template(optimizer))) - - def testMomentum(self): - optimizer = momentum.MomentumOptimizer( - learning_rate=0.03, - momentum=0.5) - self._optimizer_test_template(optimizer) - - def testRMSProp(self): - optimizer = rmsprop.RMSPropOptimizer(0.01) - self._optimizer_test_template(optimizer) - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py deleted file mode 100644 index 7a7cf712543..00000000000 --- a/tensorflow/contrib/eager/python/tfe.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""TensorFlow Eager execution prototype. - -EXPERIMENTAL: APIs here are unstable and likely to change without notice. - -To use, at program startup, call `tf.compat.v1.enable_eager_execution()`. - -@@metrics - -@@list_devices -@@num_gpus - -@@py_func -@@defun -@@function -@@make_template -@@implicit_gradients -@@implicit_value_and_gradients -@@gradients_function -@@value_and_gradients_function -@@GradientTape - -@@run -@@enable_eager_execution -@@enable_remote_eager_execution - -@@custom_gradient - -@@Iterator -@@Saver -@@restore_variables_on_create -@@Variable -@@get_optimizer_variables -@@EagerVariableStore - -@@Network -@@Sequential -@@save_network_checkpoint -@@restore_network_checkpoint - -@@Checkpoint -@@Checkpointable - -@@executing_eagerly -@@in_eager_mode -@@set_execution_mode -@@execution_mode -@@async_wait -@@async_clear_error -@@set_server_def - -@@run_test_in_graph_and_eager_modes -@@run_all_tests_in_graph_and_eager_modes - -@@TensorSpec - -@@connect_to_remote_host - -@@DEVICE_PLACEMENT_EXPLICIT -@@DEVICE_PLACEMENT_WARN -@@DEVICE_PLACEMENT_SILENT -@@SYNC -@@ASYNC -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -# pylint:disable=g-bad-import-order,g-import-not-at-top,unused-import -# -from tensorflow.contrib.eager.python import metrics -from tensorflow.contrib.eager.python.datasets import Iterator -from tensorflow.contrib.eager.python.network import Network -from tensorflow.contrib.eager.python.network import Sequential -from tensorflow.contrib.eager.python.network import save_network_checkpoint -from tensorflow.contrib.eager.python.network import restore_network_checkpoint -from tensorflow.contrib.eager.python.saver import get_optimizer_variables -from tensorflow.contrib.eager.python.saver import restore_variables_on_create -from tensorflow.contrib.eager.python.saver import Saver -from tensorflow.python.eager import backprop -from tensorflow.python.eager import function as _function_lib -from tensorflow.python.eager.context import DEVICE_PLACEMENT_EXPLICIT -from tensorflow.python.eager.context import DEVICE_PLACEMENT_WARN -from tensorflow.python.eager.context import DEVICE_PLACEMENT_SILENT -from tensorflow.python.eager.context import executing_eagerly -from tensorflow.python.eager.context import list_devices -from tensorflow.python.eager.context import set_execution_mode -from tensorflow.python.eager.context import execution_mode -from tensorflow.python.eager.context import async_wait -from tensorflow.python.eager.context import async_clear_error -from tensorflow.python.eager.context import SYNC -from tensorflow.python.eager.context import ASYNC -from tensorflow.python.eager.context import num_gpus -from tensorflow.python.eager.context import set_server_def -from tensorflow.python.eager.def_function import function -from tensorflow.python.eager.remote import connect_to_remote_host -from tensorflow.python.framework.tensor_spec import TensorSpec -from tensorflow.python.framework.ops import enable_eager_execution -from tensorflow.python.framework.ops import enable_eager_execution_internal as enable_remote_eager_execution -from tensorflow.python.framework.ops import eager_run as run -from tensorflow.python.framework.test_util import run_in_graph_and_eager_modes as run_test_in_graph_and_eager_modes -from tensorflow.python.framework.test_util import run_all_in_graph_and_eager_modes as run_all_tests_in_graph_and_eager_modes -from tensorflow.python.ops.custom_gradient import custom_gradient -from tensorflow.python.ops.resource_variable_ops import ResourceVariable as Variable -from tensorflow.python.ops.variable_scope import EagerVariableStore -from tensorflow.python.ops import script_ops -from tensorflow.python.ops import template -from tensorflow.python.training.tracking.tracking import AutoTrackable as Checkpointable -from tensorflow.python.training.tracking.util import CheckpointV1 as Checkpoint -from tensorflow.python.util.all_util import remove_undocumented - -py_func = script_ops.eager_py_func -defun = _function_lib.defun -make_template = template.make_template_internal -implicit_gradients = backprop.implicit_grad -implicit_value_and_gradients = backprop.implicit_val_and_grad -gradients_function = backprop.gradients_function -value_and_gradients_function = backprop.val_and_grad_function -GradientTape = backprop.GradientTape # pylint: disable=invalid-name -in_eager_mode = executing_eagerly - -remove_undocumented(__name__) diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py deleted file mode 100644 index 2e44ff4096a..00000000000 --- a/tensorflow/contrib/eager/python/tfe_test.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""TensorFlow Eager Execution: Sanity tests.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tempfile - -from tensorflow.contrib.eager.python import tfe -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import numerics -from tensorflow.python.platform import test -from tensorflow.python.summary import summary -from tensorflow.python.summary.writer import writer - - -class TFETest(test_util.TensorFlowTestCase): - - def testMatmul(self): - x = [[2.]] - y = math_ops.matmul(x, x) # tf.matmul - self.assertAllEqual([[4.]], y.numpy()) - - def testInstantError(self): - if test_util.is_gpu_available(): - # TODO(nareshmodi): make this test better - self.skipTest("Gather doesn't do index checking on GPUs") - - with self.assertRaisesRegexp(errors.InvalidArgumentError, - r'indices = 7 is not in \[0, 3\)'): - array_ops.gather([0, 1, 2], 7) - - def testGradients(self): - - def square(x): - return math_ops.multiply(x, x) - - grad = tfe.gradients_function(square) - self.assertEquals([6], [x.numpy() for x in grad(3.)]) - - def testGradOfGrad(self): - - def square(x): - return math_ops.multiply(x, x) - - grad = tfe.gradients_function(square) - gradgrad = tfe.gradients_function(lambda x: grad(x)[0]) - self.assertEquals([2], [x.numpy() for x in gradgrad(3.)]) - - def testCustomGrad(self): - - @tfe.custom_gradient - def f(x): - y = math_ops.multiply(x, x) - - def grad_fn(_): - return [x + y] - - return y, grad_fn - - grad = tfe.gradients_function(f) - self.assertEquals([12], [x.numpy() for x in grad(3.)]) - - @test_util.run_gpu_only - def testGPU(self): - # tf.Tensor.as_gpu_device() moves a tensor to GPU. - x = constant_op.constant([[1., 2.], [3., 4.]]).gpu() - # Alternatively, tf.device() as a context manager places tensors and - # operations. - with ops.device('gpu:0'): - x += 1. - # Without a device context, heuristics are used to place ops. - # In this case, ops.reduce_mean runs on the GPU. - axis = range(x.shape.ndims) - m = math_ops.reduce_mean(x, axis) - # m is on GPU, bring it back to CPU and compare. - self.assertEqual(3.5, m.cpu().numpy()) - - def testListDevices(self): - # Expect at least one device. - self.assertTrue(tfe.list_devices()) - - def testAddCheckNumericsOpsRaisesError(self): - with self.assertRaisesRegexp( - RuntimeError, - r'add_check_numerics_ops\(\) is not compatible with eager execution'): - numerics.add_check_numerics_ops() - - def testClassicSummaryOpsErrorOut(self): - x = constant_op.constant(42) - x_summary = summary.scalar('x', x) - y = constant_op.constant([1, 3, 3, 7]) - y_summary = summary.histogram('hist', y) - - with self.assertRaisesRegexp( - RuntimeError, - r'Merging tf\.summary\.\* ops is not compatible with eager execution'): - summary.merge([x_summary, y_summary]) - - with self.assertRaisesRegexp( - RuntimeError, - r'Merging tf\.summary\.\* ops is not compatible with eager execution'): - summary.merge_all() - - def testClassicSummaryFileWriterErrorsOut(self): - with self.assertRaisesRegexp( - RuntimeError, - r'tf\.summary\.FileWriter is not compatible with eager execution'): - writer.FileWriter(tempfile.mkdtemp()) - - -if __name__ == '__main__': - tfe.enable_eager_execution() - test.main() diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD deleted file mode 100644 index 1b14eeca504..00000000000 --- a/tensorflow/contrib/estimator/BUILD +++ /dev/null @@ -1,224 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "py_test") -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -# PLACEHOLDER PIP REQUIREMENTS - -package( - default_visibility = [ - "//tensorflow:internal", - ], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "estimator_py", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - deps = [ - ":boosted_trees", - ":dnn_with_layer_annotations", - ":early_stopping", - ":expect_tensorflow_estimator_installed", - ":export", - ":exporter", - ":extenders", - ":head", - ":hooks", - ":logit_fns", - ":multi_head", - ":replicate_model_fn", - ":rnn", - ":saved_model_estimator", - "//tensorflow:tensorflow_py_no_contrib", - "//tensorflow/python/estimator:estimator_py", - ], -) - -py_library( - name = "boosted_trees", - srcs = ["python/estimator/boosted_trees.py"], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - "//tensorflow/python/estimator", - "//tensorflow/python/estimator:boosted_trees", - ], -) - -py_library( - name = "dnn_with_layer_annotations", - srcs = ["python/estimator/dnn_with_layer_annotations.py"], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - "//tensorflow:tensorflow_py_no_contrib", - "//tensorflow/python/estimator", - "//tensorflow/python/estimator:head", - "//tensorflow/python/estimator:model_fn", - "//tensorflow/python/estimator:optimizers", - ], -) - -py_library( - name = "extenders", - srcs = [ - "python/estimator/extenders.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - "//tensorflow:tensorflow_py_no_contrib", - "//tensorflow/python/estimator", - "//tensorflow/python/estimator:model_fn", - "//tensorflow/python/estimator:util", - "@six_archive//:six", - ], -) - -py_library( - name = "export", - srcs = [ - "python/estimator/export.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - "//tensorflow/python/estimator:model_fn", - ], -) - -py_library( - name = "exporter", - srcs = [ - "python/estimator/exporter.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - "//tensorflow:tensorflow_py_no_contrib", - "//tensorflow/python/estimator:exporter", - ], -) - -py_library( - name = "head", - srcs = [ - "python/estimator/head.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - "//tensorflow:tensorflow_py_no_contrib", - "//tensorflow/python/estimator:export_output", - "//tensorflow/python/estimator:head", - "//tensorflow/python/estimator:metric_keys", - "//tensorflow/python/estimator:model_fn", - "//tensorflow/python/estimator:prediction_keys", - ], -) - -py_library( - name = "hooks", - srcs = [ - "python/estimator/hooks.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - "//tensorflow:tensorflow_py_no_contrib", - "//tensorflow/python/estimator:estimator_py", - ], -) - -py_library( - name = "logit_fns", - srcs = [ - "python/estimator/logit_fns.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - "//tensorflow:tensorflow_py_no_contrib", - "//tensorflow/python/estimator:dnn", - "//tensorflow/python/estimator:linear", - ], -) - -py_library( - name = "multi_head", - srcs = [ - "python/estimator/multi_head.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - "//tensorflow:tensorflow_py_no_contrib", - "//tensorflow/python/estimator:export_output", - "//tensorflow/python/estimator:head", - "//tensorflow/python/estimator:metric_keys", - "//tensorflow/python/estimator:model_fn", - "@six_archive//:six", - ], -) - -py_library( - name = "replicate_model_fn", - srcs = [ - "python/estimator/replicate_model_fn.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - "//tensorflow:tensorflow_py_no_contrib", - "//tensorflow/python/estimator:export_output", - "//tensorflow/python/estimator:model_fn", - "//tensorflow/python/estimator:util", - "@six_archive//:six", - ], -) - -py_library( - name = "rnn", - srcs = ["python/estimator/rnn.py"], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - ":extenders", - "//tensorflow:tensorflow_py_no_contrib", - "//tensorflow/contrib/feature_column:feature_column_py", - "//tensorflow/python/estimator", - "//tensorflow/python/estimator:head", - "//tensorflow/python/estimator:optimizers", - "@six_archive//:six", - ], -) - -py_library( - name = "early_stopping", - srcs = ["python/estimator/early_stopping.py"], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - "//tensorflow:tensorflow_py_no_contrib", - "//tensorflow/python/estimator", - ], -) - -py_library( - name = "saved_model_estimator", - srcs = ["python/estimator/saved_model_estimator.py"], - deps = [ - ":expect_tensorflow_estimator_installed", - ":export", - "//tensorflow:tensorflow_py_no_contrib", - "//tensorflow/python/estimator", - "//tensorflow/python/estimator:export", - "//tensorflow/python/estimator:model_fn", - ], -) - -py_library( - name = "expect_tensorflow_estimator_installed", - # This is a dummy rule used as a dependency in open-source. - # We expect tensorflow_estimator to already be installed. - visibility = ["//visibility:public"], -) diff --git a/tensorflow/contrib/estimator/__init__.py b/tensorflow/contrib/estimator/__init__.py deleted file mode 100644 index 7d61247e7ef..00000000000 --- a/tensorflow/contrib/estimator/__init__.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""estimator python module. - -Importing from tensorflow.python.estimator -is unsupported and will soon break! -""" - -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# Importing from tensorflow.python.estimator -# is unsupported and will soon break! - -from tensorflow_estimator.contrib import estimator - -# Fixes remove_undocumented not working as intended. -# -# Problem is that when the below import happens (for first time, -# Python only imports things once), Python sets attribute named -# 'python' to this package. If this first import happens -# after the call to remove_undocumented, then the 'python' -# attribute won't be removed. -import tensorflow.contrib.estimator.python - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -estimator.__all__ = [s for s in dir(estimator) if not s.startswith('__')] - -from tensorflow_estimator.contrib.estimator import * -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'add_metrics', - 'binary_classification_head', - 'clip_gradients_by_norm', - 'forward_features', - 'InMemoryEvaluatorHook', - 'make_stop_at_checkpoint_step_hook', - 'logistic_regression_head', - 'multi_class_head', - 'multi_head', - 'multi_label_head', - 'poisson_regression_head', - 'regression_head', - 'boosted_trees_classifier_train_in_memory', - 'boosted_trees_regressor_train_in_memory', - 'call_logit_fn', - 'dnn_logit_fn_builder', - 'linear_logit_fn_builder', - 'replicate_model_fn', - 'TowerOptimizer', - 'RNNClassifier', - 'RNNEstimator', - 'export_saved_model_for_mode', - 'export_all_saved_models', - 'make_early_stopping_hook', - 'read_eval_metrics', - 'stop_if_lower_hook', - 'stop_if_higher_hook', - 'stop_if_no_increase_hook', - 'stop_if_no_decrease_hook', - 'build_raw_supervised_input_receiver_fn', - 'build_supervised_input_receiver_fn_from_input_fn', - 'SavedModelEstimator', - 'DNNClassifierWithLayerAnnotations', - 'DNNRegressorWithLayerAnnotations', -] - -remove_undocumented(__name__, allowed_exception_list=_allowed_symbols) diff --git a/tensorflow/contrib/estimator/python/estimator/boosted_trees.py b/tensorflow/contrib/estimator/python/estimator/boosted_trees.py deleted file mode 100644 index 4cb66883a50..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/boosted_trees.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""boosted_trees python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.contrib.estimator.python.estimator import boosted_trees - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -boosted_trees.__all__ = [ - s for s in dir(boosted_trees) if not s.startswith('__') -] - -from tensorflow_estimator.contrib.estimator.python.estimator.boosted_trees import * diff --git a/tensorflow/contrib/estimator/python/estimator/dnn_with_layer_annotations.py b/tensorflow/contrib/estimator/python/estimator/dnn_with_layer_annotations.py deleted file mode 100644 index 854d2e4011b..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/dnn_with_layer_annotations.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""dnn_with_layer_annotations python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.contrib.estimator.python.estimator import dnn_with_layer_annotations - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -dnn_with_layer_annotations.__all__ = [ - s for s in dir(dnn_with_layer_annotations) if not s.startswith('__') -] - -from tensorflow_estimator.contrib.estimator.python.estimator.dnn_with_layer_annotations import * diff --git a/tensorflow/contrib/estimator/python/estimator/early_stopping.py b/tensorflow/contrib/estimator/python/estimator/early_stopping.py deleted file mode 100644 index 47f568ed3d3..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/early_stopping.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""early_stopping python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.python.estimator import early_stopping - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -early_stopping.__all__ = [ - s for s in dir(early_stopping) if not s.startswith('__') -] - -from tensorflow_estimator.python.estimator.early_stopping import * diff --git a/tensorflow/contrib/estimator/python/estimator/export.py b/tensorflow/contrib/estimator/python/estimator/export.py deleted file mode 100644 index 738b343dfde..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/export.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""export python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.contrib.estimator.python.estimator import export - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -export.__all__ = [s for s in dir(export) if not s.startswith('__')] - -from tensorflow_estimator.contrib.estimator.python.estimator.export import * diff --git a/tensorflow/contrib/estimator/python/estimator/exporter.py b/tensorflow/contrib/estimator/python/estimator/exporter.py deleted file mode 100644 index 25ac78ca37d..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/exporter.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""exporter python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.contrib.estimator.python.estimator import exporter - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -exporter.__all__ = [s for s in dir(exporter) if not s.startswith('__')] - -from tensorflow_estimator.contrib.estimator.python.estimator.exporter import * diff --git a/tensorflow/contrib/estimator/python/estimator/extenders.py b/tensorflow/contrib/estimator/python/estimator/extenders.py deleted file mode 100644 index 9ab9bc7a8e7..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/extenders.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""extenders python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.contrib.estimator.python.estimator import extenders - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -extenders.__all__ = [s for s in dir(extenders) if not s.startswith('__')] - -from tensorflow_estimator.contrib.estimator.python.estimator.extenders import * diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py deleted file mode 100644 index 92144d394b5..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/head.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""head python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.contrib.estimator.python.estimator import head - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -head.__all__ = [s for s in dir(head) if not s.startswith('__')] - -from tensorflow_estimator.contrib.estimator.python.estimator.head import * diff --git a/tensorflow/contrib/estimator/python/estimator/hooks.py b/tensorflow/contrib/estimator/python/estimator/hooks.py deleted file mode 100644 index f7ff5adcc7c..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/hooks.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""hooks python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.contrib.estimator.python.estimator import hooks - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -hooks.__all__ = [s for s in dir(hooks) if not s.startswith('__')] - -from tensorflow_estimator.contrib.estimator.python.estimator.hooks import * diff --git a/tensorflow/contrib/estimator/python/estimator/logit_fns.py b/tensorflow/contrib/estimator/python/estimator/logit_fns.py deleted file mode 100644 index 5e418558663..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/logit_fns.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""logit_fns python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.contrib.estimator.python.estimator import logit_fns - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -logit_fns.__all__ = [s for s in dir(logit_fns) if not s.startswith('__')] - -from tensorflow_estimator.contrib.estimator.python.estimator.logit_fns import * diff --git a/tensorflow/contrib/estimator/python/estimator/multi_head.py b/tensorflow/contrib/estimator/python/estimator/multi_head.py deleted file mode 100644 index 6cf2917df3e..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/multi_head.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""multi_head python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.contrib.estimator.python.estimator import multi_head - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -multi_head.__all__ = [s for s in dir(multi_head) if not s.startswith('__')] - -from tensorflow_estimator.contrib.estimator.python.estimator.multi_head import * diff --git a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py deleted file mode 100644 index e6b0cf027ae..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""replicate_model_fn python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.contrib.estimator.python.estimator import replicate_model_fn - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -replicate_model_fn.__all__ = [ - s for s in dir(replicate_model_fn) if not s.startswith('__') -] - -from tensorflow_estimator.contrib.estimator.python.estimator.replicate_model_fn import * diff --git a/tensorflow/contrib/estimator/python/estimator/rnn.py b/tensorflow/contrib/estimator/python/estimator/rnn.py deleted file mode 100644 index c5ddd1b7359..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/rnn.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""rnn python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.contrib.estimator.python.estimator import rnn - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -rnn.__all__ = [s for s in dir(rnn) if not s.startswith('__')] - -from tensorflow_estimator.contrib.estimator.python.estimator.rnn import * diff --git a/tensorflow/contrib/estimator/python/estimator/saved_model_estimator.py b/tensorflow/contrib/estimator/python/estimator/saved_model_estimator.py deleted file mode 100644 index 031cae4071c..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/saved_model_estimator.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""saved_model_estimator python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.contrib.estimator.python.estimator import saved_model_estimator - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -saved_model_estimator.__all__ = [ - s for s in dir(saved_model_estimator) if not s.startswith('__') -] - -from tensorflow_estimator.contrib.estimator.python.estimator.saved_model_estimator import * diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD deleted file mode 100644 index ae9693619e8..00000000000 --- a/tensorflow/contrib/factorization/BUILD +++ /dev/null @@ -1,323 +0,0 @@ -# Description: -# Contains ops for factorization of data, including matrix factorization and -# clustering. - -load("//tensorflow:tensorflow.bzl", "py_test", "tf_custom_op_library", "tf_gen_op_libs", "tf_gen_op_wrapper_py", "tf_py_test") -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_custom_op_py_library( - name = "factorization_py", - srcs = [ - "__init__.py", - "python/ops/clustering_ops.py", - "python/ops/factorization_ops.py", - "python/ops/gmm.py", - "python/ops/gmm_ops.py", - "python/ops/kmeans.py", - "python/ops/wals.py", - ], - dso = [ - ":python/ops/_factorization_ops.so", - ], - kernels = [ - ":all_ops", - "//tensorflow/contrib/factorization/kernels:all_kernels", - ], - srcs_version = "PY2AND3", - deps = [ - ":factorization_ops_test_utils_py", - ":gen_factorization_ops", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:clustering_ops_gen", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:logging_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:metrics", - "//tensorflow/python:nn", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:state_ops", - "//tensorflow/python:summary", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/estimator:estimator_py", - "//tensorflow/python/feature_column:feature_column_py", - "//third_party/py/numpy", - ], -) - -py_library( - name = "factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", - deps = [ - "//tensorflow/contrib/learn", - ], -) - -tf_custom_op_library( - name = "python/ops/_factorization_ops.so", - srcs = [ - "ops/factorization_ops.cc", - ], - deps = [ - "//tensorflow/contrib/factorization/kernels:masked_matmul_ops", - "//tensorflow/contrib/factorization/kernels:wals_solver_ops", - ], -) - -tf_gen_op_libs([ - "factorization_ops", -]) - -cc_library( - name = "all_ops", - deps = [ - ":factorization_ops_op_lib", - ], -) - -tf_gen_op_wrapper_py( - name = "gen_factorization_ops", - out = "python/ops/gen_factorization_ops.py", - deps = [ - ":factorization_ops_op_lib", - ], -) - -# Ops tests -tf_py_test( - name = "gmm_test", - size = "medium", - srcs = [ - "python/ops/gmm_test.py", - ], - additional_deps = [ - ":factorization_py", - ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", - "//third_party/py/numpy", - "//tensorflow/contrib/learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:training", - ], - shard_count = 4, - tags = [ - "no_pip", # b/38283730 - "notsan", # Flaky: b/30756419 - ], -) - -tf_py_test( - name = "gmm_ops_test", - size = "large", - srcs = [ - "python/ops/gmm_ops_test.py", - ], - additional_deps = [ - ":factorization_py", - ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_seed", - "//tensorflow/python:variables", - ], - tags = ["notsan"], # b/62863147 -) - -py_library( - name = "factorization_ops_test_utils_py", - srcs = [ - "python/ops/factorization_ops_test_utils.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:math_ops", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:sparse_tensor", - "//third_party/py/numpy", - ], -) - -tf_py_test( - name = "factorization_ops_test", - srcs = ["python/ops/factorization_ops_test.py"], - additional_deps = [ - ":factorization_py", - ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", - ":factorization_ops_test_utils_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:sparse_tensor", - ], - shard_count = 4, -) - -# Estimators tests -py_test( - name = "kmeans_test", - size = "medium", - srcs = ["python/ops/kmeans_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - tags = ["notsan"], - deps = [ - ":factorization_py", - ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_benchmark", - "//tensorflow/python:random_ops", - "//tensorflow/python:training", - "//tensorflow/python/estimator:estimator_py", - "//tensorflow/python/feature_column:feature_column_py", - "//third_party/py/numpy", - ], -) - -tf_py_test( - name = "wals_test", - size = "medium", - srcs = ["python/ops/wals_test.py"], - additional_deps = [ - ":factorization_py", - ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", - ":factorization_ops_test_utils_py", - "//third_party/py/numpy", - "//tensorflow/contrib/learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_benchmark", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], - shard_count = 4, - tags = [ - "noasan", # times out b/63678675 - "nomsan", - ], -) - -# Kernel tests -tf_py_test( - name = "wals_solver_ops_test", - srcs = ["python/kernel_tests/wals_solver_ops_test.py"], - additional_deps = [ - ":factorization_py", - ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", - ":gen_factorization_ops", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:sparse_tensor", - ], -) - -tf_py_test( - name = "clustering_ops_test", - srcs = ["python/kernel_tests/clustering_ops_test.py"], - additional_deps = [ - ":factorization_py", - ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -tf_py_test( - name = "masked_matmul_ops_test", - srcs = ["python/kernel_tests/masked_matmul_ops_test.py"], - additional_deps = [ - ":gen_factorization_ops", - ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:sparse_tensor", - ], -) - -cuda_py_test( - name = "masked_matmul_benchmark", - srcs = ["python/kernel_tests/masked_matmul_benchmark.py"], - additional_deps = [ - ":gen_factorization_ops", - ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:random_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:variables", - ], - main = "python/kernel_tests/masked_matmul_benchmark.py", -) diff --git a/tensorflow/contrib/factorization/__init__.py b/tensorflow/contrib/factorization/__init__.py deleted file mode 100644 index 6112c9d8300..00000000000 --- a/tensorflow/contrib/factorization/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops and modules related to factorization.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.factorization.python.ops.clustering_ops import * -from tensorflow.contrib.factorization.python.ops.factorization_ops import * -from tensorflow.contrib.factorization.python.ops.gmm import * -from tensorflow.contrib.factorization.python.ops.gmm_ops import * -from tensorflow.contrib.factorization.python.ops.kmeans import * -from tensorflow.contrib.factorization.python.ops.wals import * -# pylint: enable=wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'COSINE_DISTANCE', - 'GMM', - 'gmm', - 'GmmAlgorithm', - 'KMeans', - 'KMEANS_PLUS_PLUS_INIT', - 'KMeansClustering', - 'RANDOM_INIT', - 'SQUARED_EUCLIDEAN_DISTANCE', - 'WALSMatrixFactorization', - 'WALSModel', -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/factorization/examples/BUILD b/tensorflow/contrib/factorization/examples/BUILD deleted file mode 100644 index fa9542b44f5..00000000000 --- a/tensorflow/contrib/factorization/examples/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -# Example TensorFlow models using factorization ops. - -load("//tensorflow:tensorflow.bzl", "tf_py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_py_test( - name = "mnist", - size = "medium", - srcs = [ - "mnist.py", - ], - additional_deps = [ - "//tensorflow:tensorflow_py", - "//tensorflow/examples/tutorials/mnist", - "//tensorflow/examples/tutorials/mnist:input_data", - ], - tags = ["notsan"], -) diff --git a/tensorflow/contrib/factorization/examples/mnist.py b/tensorflow/contrib/factorization/examples/mnist.py deleted file mode 100644 index 06a62db0049..00000000000 --- a/tensorflow/contrib/factorization/examples/mnist.py +++ /dev/null @@ -1,340 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Example mnist model with jointly computed k-means clustering. - -This is a toy example of how clustering can be embedded into larger tensorflow -graphs. In this case, we learn a clustering on-the-fly and transform the input -into the 'distance to clusters' space. These are then fed into hidden layers to -learn the supervised objective. - -To train this model on real mnist data, run this model as follows: - mnist --fake_data=False --max_steps=2000 -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import math -import sys -import tempfile -import time - -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow as tf - -from tensorflow.examples.tutorials.mnist import input_data -from tensorflow.examples.tutorials.mnist import mnist - -FLAGS = None - -# The MNIST dataset has 10 classes, representing the digits 0 through 9. -NUM_CLASSES = 10 - -# The MNIST images are always 28x28 pixels. -IMAGE_SIZE = 28 -IMAGE_PIXELS = IMAGE_SIZE * IMAGE_SIZE - - -def placeholder_inputs(): - """Generate placeholder variables to represent the input tensors. - - Returns: - images_placeholder: Images placeholder. - labels_placeholder: Labels placeholder. - """ - images_placeholder = tf.placeholder(tf.float32, shape=(None, - mnist.IMAGE_PIXELS)) - labels_placeholder = tf.placeholder(tf.int32, shape=(None)) - return images_placeholder, labels_placeholder - - -def fill_feed_dict(data_set, images_pl, labels_pl, batch_size): - """Fills the feed_dict for training the given step. - - Args: - data_set: The set of images and labels, from input_data.read_data_sets() - images_pl: The images placeholder, from placeholder_inputs(). - labels_pl: The labels placeholder, from placeholder_inputs(). - batch_size: Batch size of data to feed. - - Returns: - feed_dict: The feed dictionary mapping from placeholders to values. - """ - # Create the feed_dict for the placeholders filled with the next - # `batch size ` examples. - images_feed, labels_feed = data_set.next_batch(batch_size, FLAGS.fake_data) - feed_dict = { - images_pl: images_feed, - labels_pl: labels_feed, - } - return feed_dict - - -def do_eval(sess, - eval_correct, - images_placeholder, - labels_placeholder, - data_set): - """Runs one evaluation against the full epoch of data. - - Args: - sess: The session in which the model has been trained. - eval_correct: The Tensor that returns the number of correct predictions. - images_placeholder: The images placeholder. - labels_placeholder: The labels placeholder. - data_set: The set of images and labels to evaluate, from - input_data.read_data_sets(). - Returns: - Precision value on the dataset. - """ - # And run one epoch of eval. - true_count = 0 # Counts the number of correct predictions. - batch_size = min(FLAGS.batch_size, data_set.num_examples) - steps_per_epoch = data_set.num_examples // batch_size - num_examples = steps_per_epoch * batch_size - for _ in xrange(steps_per_epoch): - feed_dict = fill_feed_dict(data_set, - images_placeholder, - labels_placeholder, - batch_size) - true_count += sess.run(eval_correct, feed_dict=feed_dict) - precision = true_count / num_examples - print(' Num examples: %d Num correct: %d Precision @ 1: %0.04f' % - (num_examples, true_count, precision)) - return precision - - -def inference(inp, num_clusters, hidden1_units, hidden2_units): - """Build the MNIST model up to where it may be used for inference. - - Args: - inp: input data - num_clusters: number of clusters of input features to train. - hidden1_units: Size of the first hidden layer. - hidden2_units: Size of the second hidden layer. - - Returns: - logits: Output tensor with the computed logits. - clustering_loss: Clustering loss. - kmeans_training_op: An op to train the clustering. - """ - # Clustering - kmeans = tf.contrib.factorization.KMeans( - inp, - num_clusters, - distance_metric=tf.contrib.factorization.COSINE_DISTANCE, - # TODO(agarwal): kmeans++ is currently causing crash in dbg mode. - # Enable this after fixing. - # initial_clusters=tf.contrib.factorization.KMEANS_PLUS_PLUS_INIT, - use_mini_batch=True) - - (all_scores, _, clustering_scores, _, kmeans_init, - kmeans_training_op) = kmeans.training_graph() - # Some heuristics to approximately whiten this output. - all_scores = (all_scores[0] - 0.5) * 5 - # Here we avoid passing the gradients from the supervised objective back to - # the clusters by creating a stop_gradient node. - all_scores = tf.stop_gradient(all_scores) - clustering_loss = tf.reduce_sum(clustering_scores[0]) - # Hidden 1 - with tf.name_scope('hidden1'): - weights = tf.Variable( - tf.truncated_normal([num_clusters, hidden1_units], - stddev=1.0 / math.sqrt(float(IMAGE_PIXELS))), - name='weights') - biases = tf.Variable(tf.zeros([hidden1_units]), - name='biases') - hidden1 = tf.nn.relu(tf.matmul(all_scores, weights) + biases) - # Hidden 2 - with tf.name_scope('hidden2'): - weights = tf.Variable( - tf.truncated_normal([hidden1_units, hidden2_units], - stddev=1.0 / math.sqrt(float(hidden1_units))), - name='weights') - biases = tf.Variable(tf.zeros([hidden2_units]), - name='biases') - hidden2 = tf.nn.relu(tf.matmul(hidden1, weights) + biases) - # Linear - with tf.name_scope('softmax_linear'): - weights = tf.Variable( - tf.truncated_normal([hidden2_units, NUM_CLASSES], - stddev=1.0 / math.sqrt(float(hidden2_units))), - name='weights') - biases = tf.Variable(tf.zeros([NUM_CLASSES]), - name='biases') - logits = tf.matmul(hidden2, weights) + biases - return logits, clustering_loss, kmeans_init, kmeans_training_op - - -def run_training(): - """Train MNIST for a number of steps.""" - # Get the sets of images and labels for training, validation, and - # test on MNIST. - train_dir = tempfile.mkdtemp() - data_sets = input_data.read_data_sets(train_dir, FLAGS.fake_data) - - # Tell TensorFlow that the model will be built into the default Graph. - with tf.Graph().as_default(): - # Generate placeholders for the images and labels. - images_placeholder, labels_placeholder = placeholder_inputs() - - # Build a Graph that computes predictions from the inference model. - logits, clustering_loss, kmeans_init, kmeans_training_op = inference( - images_placeholder, - FLAGS.num_clusters, - FLAGS.hidden1, - FLAGS.hidden2) - - # Add to the Graph the Ops for loss calculation. - loss = mnist.loss(logits, labels_placeholder) - - # Add to the Graph the Ops that calculate and apply gradients. - train_op = tf.group(mnist.training(loss, FLAGS.learning_rate), - kmeans_training_op) - - # Add the Op to compare the logits to the labels during evaluation. - eval_correct = mnist.evaluation(logits, labels_placeholder) - - # Add the variable initializer Op. - init = tf.global_variables_initializer() - - # Create a session for running Ops on the Graph. - sess = tf.Session() - - # Run the Op to initialize the variables. - sess.run(init) - - feed_dict = fill_feed_dict(data_sets.train, - images_placeholder, - labels_placeholder, - batch_size=max(FLAGS.batch_size, 5000)) - # Run the Op to initialize the clusters. - sess.run(kmeans_init, feed_dict=feed_dict) - - # Start the training loop. - max_test_prec = 0 - for step in xrange(FLAGS.max_steps): - start_time = time.time() - - # Fill a feed dictionary with the actual set of images and labels - # for this particular training step. - feed_dict = fill_feed_dict(data_sets.train, - images_placeholder, - labels_placeholder, - FLAGS.batch_size) - - # Run one step of the model. - _, loss_value, clustering_loss_value = sess.run([train_op, - loss, - clustering_loss], - feed_dict=feed_dict) - - duration = time.time() - start_time - if step % 100 == 0: - # Print status to stdout. - print('Step %d: loss = %.2f, clustering_loss = %.2f (%.3f sec)' % ( - step, loss_value, clustering_loss_value, duration)) - - # Save a checkpoint and evaluate the model periodically. - if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps: - # Evaluate against the training set. - print('Training Data Eval:') - do_eval(sess, - eval_correct, - images_placeholder, - labels_placeholder, - data_sets.train) - # Evaluate against the validation set. - print('Validation Data Eval:') - do_eval(sess, - eval_correct, - images_placeholder, - labels_placeholder, - data_sets.validation) - # Evaluate against the test set. - print('Test Data Eval:') - test_prec = do_eval(sess, - eval_correct, - images_placeholder, - labels_placeholder, - data_sets.test) - max_test_prec = max(max_test_prec, test_prec) - return max_test_prec - - -class MnistTest(tf.test.TestCase): - - def test_train(self): - self.assertTrue(run_training() > 0.6) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Basic model parameters as external flags.' - ) - parser.register('type', 'bool', lambda v: v.lower() == 'true') - parser.add_argument( - '--learning_rate', - type=float, - default=0.3, - help='Initial learning rate.' - ) - parser.add_argument( - '--max_steps', - type=int, - default=200, - help='Number of steps to run trainer.' - ) - parser.add_argument( - '--num_clusters', - type=int, - default=384, - help='Number of input feature clusters' - ) - parser.add_argument( - '--hidden1', - type=int, - default=256, - help='Number of units in hidden layer 1.' - ) - parser.add_argument( - '--hidden2', - type=int, - default=32, - help='Number of units in hidden layer 2.' - ) - parser.add_argument( - '--batch_size', - type=int, - default=100, - help='Batch size. Must divide evenly into the dataset sizes.' - ) - parser.add_argument( - '--train_dir', - type=str, - default='data', - help='Directory to put the training data.' - ) - parser.add_argument( - '--fake_data', - type='bool', - default=True, - help='Use fake input data.' - ) - FLAGS, unparsed = parser.parse_known_args() - sys.argv = [sys.argv[0]] + unparsed - tf.test.main() diff --git a/tensorflow/contrib/factorization/g3doc/gmm.md b/tensorflow/contrib/factorization/g3doc/gmm.md deleted file mode 100644 index ec11329e9ff..00000000000 --- a/tensorflow/contrib/factorization/g3doc/gmm.md +++ /dev/null @@ -1,14 +0,0 @@ -# Gaussian Mixture Models - -Gaussian mixture models assume that the input is generated by a mixture of a -finite number of Gaussian distributions with unknown parameters. - -![gmm](gmm.png) - -Typically an EM (Expectation Maximization) algorithm is used to estimate these -parameters, which include the mean, covariance and mixture ratios of the -underlying components. - -We provide a distributed implementation for running the GMM algorithm. Options -are provided to let the user decide on which parameters to learn. The learned -covariance can be either full or diagonal. diff --git a/tensorflow/contrib/factorization/g3doc/gmm.png b/tensorflow/contrib/factorization/g3doc/gmm.png deleted file mode 100644 index 1d44e55cb3c..00000000000 Binary files a/tensorflow/contrib/factorization/g3doc/gmm.png and /dev/null differ diff --git a/tensorflow/contrib/factorization/g3doc/kmeans.md b/tensorflow/contrib/factorization/g3doc/kmeans.md deleted file mode 100644 index c1843f0bf07..00000000000 --- a/tensorflow/contrib/factorization/g3doc/kmeans.md +++ /dev/null @@ -1,34 +0,0 @@ -# K-Means Clustering - -Given a set of input $$x_i$$, K-means clustering finds a set C of cluster -centers that minimizes the inertia, defined as $$ \sum_{i=0}^{n} \min_{\mu_j \in -C} (||x_i - \mu_j||^2) $$. - -![kmeans](kmeans.png) - -The **K-means** algorithm, or **Lloyd's algorithm**, is an iterative approach -where each iteration involves assigning each input point to its closest cluster, -followed by recomputing the new value of the cluster center as the mean of the -points assigned to it. It runs full-batch, which means each iteration goes -through the full set of inputs to compute a new set of cluster centers. -Convergence is typically pretty fast, in terms of number of iterations, but -there can be local minima. - -**[Mini-batch K-means](https://www.eecs.tufts.edu/~dsculley/papers/fastkmeans.pdf)** -is a stochastic variant of Lloyd's algorithm which updates the cluster centers -based on a mini-batch of data. It converges much faster than the full-batch -version, especially for large data sets, and the quality is typically close to -the full-batch version. - -**[K-Means++](http://ilpubs.stanford.edu:8090/778/1/2006-13.pdf)** is an -approach for computing the initial cluster assignments that is expensive but is -typically less prone to getting stuck in bad local minima. - -**[k-MC2](https://www.aaai.org/ocs/index.php/AAAI/AAAI16/paper/view/12147/11759)** -provides a very fast seeding method that provides high quality centers -comparable to K-Means++ seeding. k-MC2 works particularly well if it is combined -with Mini-batch K-Means. - -We provide distributed implementations of both full-batch and mini-batch K-Means -algorithm. K-Means++, k-MC2 and random initialization are supported. The user -can also choose between **Cosine** and **Squared Euclidean** distance metrics. diff --git a/tensorflow/contrib/factorization/g3doc/kmeans.png b/tensorflow/contrib/factorization/g3doc/kmeans.png deleted file mode 100644 index b470f745ca0..00000000000 Binary files a/tensorflow/contrib/factorization/g3doc/kmeans.png and /dev/null differ diff --git a/tensorflow/contrib/factorization/g3doc/wals.md b/tensorflow/contrib/factorization/g3doc/wals.md deleted file mode 100644 index a428b393ba7..00000000000 --- a/tensorflow/contrib/factorization/g3doc/wals.md +++ /dev/null @@ -1,67 +0,0 @@ -# WALS Factorization - -$$ -% commands -\newcommand\bracket[2]{\left\langle #1, #2 \right\rangle} -\newcommand\trace{\text{trace}} -\newcommand\Rbb{\mathbb{R}} -$$ - -### Problem formulation - -WALS (Weighed Alternating Least Squares) is an algorithm for factorizing a -sparse matrix $$A \in \Rbb^{n \times m}$$ into low rank factors, $$U \in \Rbb^{n -\times k}$$ and $$V \in \Rbb^{m \times k}$$, such that the product $$UV^T$$ is a -"good" approximation of the full matrix. - -![wals](wals.png) - -Typically, it involves minimizing the following loss function: - -$$ min_{U,V} -(\|\sqrt{W} \odot (A- UV^T)\|_F^2 + \lambda (\|U\|_F^2 + \|V\|_F^2)), -$$ - -where - -- $$\lambda$$ is a regularization parameter, -- $$\odot$$ denotes the component-wise product, -- $$W$$ is a weight matrix of the form $$W_{i, j} = w_0 + 1_{A_{ij} \neq 0}R_i - C_j$$, where $$w_0$$ is the weight of unobserved entries, and $$R \in - \Rbb^n$$ and $$C \in \Rbb^m$$ are the row and column weights respectively. - This form of the weight matrix lends this problem to an efficient - implementation. - -### Solution method - -The WALS algorithm proceeds in phases, or "sweeps", where each sweep involves -fixing $$U$$ and solving for $$V$$, and then fixing $$V$$ and solving for $$U$$. -Note that each subproblem is an unconstrained quadratic minimization problem and -can be solved exactly. Convergence is typically pretty fast (10-20 sweeps). - -### Loss computation - -The product $$UV^T$$ is dense, and can be too large to compute. So we use the -following reformulation of the loss to be able to compute it efficiently. First, -we decompose the norm into two terms, corresponding to the sparsity pattern of -$$A$$. Let $$S = \{(i, j) : A_{i, j} \neq 0\}$$. - -$$ -\begin{align} -\|\sqrt W \odot (A - UV^T)\|_F^2 -&= \sum_{(i, j) \in S} (w_0 + R_i C_j) (A_{ij} - \bracket{u_i}{v_j})^2 + -\sum_{(i, j) \not\in S} w_0 (\bracket{u_i}{v_j})^2 \\ -&= \sum_{(i, j) \in S} \left( (w_0 + R_i C_j) (A_{ij} - \bracket{u_i}{v_j})^2 - -w_0 \bracket{u_i}{v_j}^2 \right) + w_0\|UV^T\|_F^2 -\end{align} -$$ - -The last term can be computed efficiently by observing that - -$$ -\|UV^T\|_F^2 = \trace(UV^TVU^T) = \trace(U^TUV^TV) -$$ - -Each of the Gramian matrices $$U^TU$$ and $$V^TV$$ are $$k\times k$$ and are -cheap to store. Additionally, $$\|U\|_F^2 = \trace(U^TU)$$ and similarly for -$$V$$, so we can use the trace of the individual Gramians to compute the norms. diff --git a/tensorflow/contrib/factorization/g3doc/wals.png b/tensorflow/contrib/factorization/g3doc/wals.png deleted file mode 100644 index eb90a888a10..00000000000 Binary files a/tensorflow/contrib/factorization/g3doc/wals.png and /dev/null differ diff --git a/tensorflow/contrib/factorization/kernels/BUILD b/tensorflow/contrib/factorization/kernels/BUILD deleted file mode 100644 index 60719db6570..00000000000 --- a/tensorflow/contrib/factorization/kernels/BUILD +++ /dev/null @@ -1,40 +0,0 @@ -# OpKernels for data factorization and clustering. - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -cc_library( - name = "all_kernels", - deps = [ - ":masked_matmul_ops", - ":wals_solver_ops", - "@com_google_protobuf//:protobuf_headers", - ], -) - -cc_library( - name = "wals_solver_ops", - srcs = ["wals_solver_ops.cc"], - deps = [ - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - "@com_google_protobuf//:protobuf_headers", - ], - alwayslink = 1, -) - -cc_library( - name = "masked_matmul_ops", - srcs = ["masked_matmul_ops.cc"], - deps = [ - "//tensorflow/core:framework_headers_lib", - "//tensorflow/core/kernels:bounds_check", - "//third_party/eigen3", - "@com_google_protobuf//:protobuf_headers", - ], - alwayslink = 1, -) diff --git a/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc b/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc deleted file mode 100644 index 68078ba8bbb..00000000000 --- a/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy -// of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. -// ============================================================================= - -// TensorFlow kernels and Ops for computing a masked matrix product. - -#include -#include -#include - -#include "tensorflow/core/framework/bounds_check.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/threadpool.h" - -using tensorflow::DEVICE_CPU; -using tensorflow::DT_BOOL; -using tensorflow::DT_FLOAT; -using tensorflow::DT_INT64; -using tensorflow::OpKernel; -using tensorflow::OpKernelConstruction; -using tensorflow::OpKernelContext; -using tensorflow::Tensor; -using tensorflow::TensorShape; -using tensorflow::TensorShapeUtils; -using tensorflow::errors::InvalidArgument; - -namespace tensorflow { - -typedef Eigen::Map< - Eigen::Matrix> - EigenMatInt64Map; -typedef Eigen::Map< - Eigen::Matrix> - EigenMatFloatMap; -typedef Eigen::Map< - const Eigen::Matrix> - ConstEigenMatInt64Map; -typedef Eigen::Map< - const Eigen::Matrix> - ConstEigenMatFloatMap; - -class MaskedMatmulOp : public OpKernel { - public: - explicit MaskedMatmulOp(OpKernelConstruction* context) : OpKernel(context) { - OP_REQUIRES_OK( - context, - context->MatchSignature( - {DT_FLOAT, DT_FLOAT, DT_INT64, DT_BOOL, DT_BOOL}, {DT_FLOAT})); - } - - void Compute(OpKernelContext* context) override { - // Computes the product a * b, but only for indices (i, j) in mask_indices. - // The result is stored in prod_values, a 1-tensor, such that for all i, - // prod_values[i] = (a * b)[mask_indices[i, 0], mask_indices[i, 1]]. - const Tensor& a = context->input(0); - const Tensor& b = context->input(1); - const Tensor& mask_indices = context->input(2); - const Tensor& transpose_a = context->input(3); - const Tensor& transpose_b = context->input(4); - - OP_REQUIRES(context, TensorShapeUtils::IsMatrix(a.shape()), - InvalidArgument("Input a should be a matrix.")); - OP_REQUIRES(context, TensorShapeUtils::IsMatrix(b.shape()), - InvalidArgument("Input b should be a matrix.")); - OP_REQUIRES(context, TensorShapeUtils::IsMatrix(mask_indices.shape()), - InvalidArgument("Input mask_indices should be a matrix.")); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(transpose_a.shape()), - InvalidArgument("Input transpose_a should be a scalar.")); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(transpose_b.shape()), - InvalidArgument("Input transpose_b should be a scalar.")); - - const bool adj_a = transpose_a.scalar()(); - const bool adj_b = transpose_b.scalar()(); - const int64 a_dim_0 = a.dim_size(adj_a ? 1 : 0); - const int64 a_dim_1 = a.dim_size(adj_a ? 0 : 1); - const int64 b_dim_0 = b.dim_size(adj_b ? 1 : 0); - const int64 b_dim_1 = b.dim_size(adj_b ? 0 : 1); - const int64 num_nonzero_elements = mask_indices.dim_size(0); - - OP_REQUIRES(context, a_dim_1 == b_dim_0, - InvalidArgument("Matrix shapes are incompatible: a has shape ", - a.shape().DebugString(), ", while b has shape ", - b.shape().DebugString(), ".")); - OP_REQUIRES(context, mask_indices.dim_size(1) == 2, - InvalidArgument("mask_indices should be a matrix of shape ", - "[nnz 2], where nnz is the number of non-zero ", - "elements.")); - - ConstEigenMatFloatMap a_mat(a.matrix().data(), a.dim_size(0), - a.dim_size(1)); - ConstEigenMatFloatMap b_mat(b.matrix().data(), b.dim_size(0), - b.dim_size(1)); - ConstEigenMatInt64Map indices_mat(mask_indices.matrix().data(), - num_nonzero_elements, 2); - - Tensor* prod_values_tensor; - OP_REQUIRES_OK(context, context->allocate_output( - 0, TensorShape({num_nonzero_elements}), - &prod_values_tensor)); - EigenMatFloatMap prod_values(prod_values_tensor->vec().data(), 1, - num_nonzero_elements); - - auto get_a_index = [&indices_mat, &a_dim_0](int64 i) { - int64 a_index = internal::SubtleMustCopy(indices_mat(i, 0)); - CHECK(FastBoundsCheck(a_index, a_dim_0)) - << "In mask_indices[" << i << ", :], the row index " << a_index - << " is out of bounds [0, " << a_dim_0 << ")."; - return a_index; - }; - auto get_b_index = [&indices_mat, &b_dim_1](int64 i) { - int64 b_index = internal::SubtleMustCopy(indices_mat(i, 1)); - CHECK(FastBoundsCheck(b_index, b_dim_1)) - << "In mask_indices[" << i << ", :], the column index " << b_index - << " is out of bounds [0, " << b_dim_1 << ")."; - return b_index; - }; - auto get_dot_product = [&adj_a, &adj_b, &a_mat, &b_mat](int64 i, int64 j) { - if (adj_a) { - if (adj_b) { - return a_mat.col(i).dot(b_mat.row(j)); - } else { - return a_mat.col(i).dot(b_mat.col(j)); - } - } else { - if (adj_b) { - return a_mat.row(i).dot(b_mat.row(j)); - } else { - return a_mat.row(i).dot(b_mat.col(j)); - } - } - }; - - std::vector perm(num_nonzero_elements); - std::iota(perm.begin(), perm.end(), 0); - // TODO(walidk): improve performance in the case adj_a and not adj_b - // TODO(walidk): benchmark smaller inputs, and potentially skip the sort - // when the input fits in L3 cache. - // Compute a permutation to sort either the a or b matrix, to take advantage - // of CPU caching. Since row access is efficient (given the RowMajor - // ordering), we prefer to - // sort according to a when a is transposed, - // sort according to b when b is not transpose. - auto compare_a_index = [&get_a_index](int64 i, int64 j) { - return get_a_index(i) < get_a_index(j); - }; - auto compare_b_index = [&get_b_index](int64 i, int64 j) { - return get_b_index(i) < get_b_index(j); - }; - if (adj_a) { - std::stable_sort(perm.begin(), perm.end(), compare_a_index); - } else if (!adj_b) { - std::stable_sort(perm.begin(), perm.end(), compare_b_index); - } - - auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads()); - // Based on benchmarks, the cost is on the order of 20 cycles per dimension - const int64 cost_per_unit = 20 * a_dim_1; - // Lambda encapsulating the per-shard computation. - auto work = [&](int64 begin, int64 end) { - for (int64 i = begin; i < end; ++i) { - const int64 p = perm[i]; - const int64 a_index = get_a_index(p); - const int64 b_index = get_b_index(p); - prod_values(p) = get_dot_product(a_index, b_index); - } - }; - // Shard the work. - worker_threads.workers->ParallelFor(num_nonzero_elements, cost_per_unit, - work); - } -}; -REGISTER_KERNEL_BUILDER(Name("MaskedMatmul").Device(DEVICE_CPU), - MaskedMatmulOp); - -} // namespace tensorflow diff --git a/tensorflow/contrib/factorization/kernels/wals_solver_ops.cc b/tensorflow/contrib/factorization/kernels/wals_solver_ops.cc deleted file mode 100644 index 7fcae5ad8e1..00000000000 --- a/tensorflow/contrib/factorization/kernels/wals_solver_ops.cc +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy -// of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. -// ============================================================================== - -// TensorFlow kernels and Ops for constructing WALS normal equations. -// TODO(agarwal,rmlarsen): Add security checks to the code. - -#include -#include -#include - -// This is only used for std::this_thread::get_id() -#include // NOLINT - -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/lib/core/blocking_counter.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/platform/mutex.h" - -using tensorflow::DEVICE_CPU; -using tensorflow::DT_BOOL; -using tensorflow::DT_FLOAT; -using tensorflow::DT_INT64; -using tensorflow::OpKernel; -using tensorflow::OpKernelConstruction; -using tensorflow::OpKernelContext; -using tensorflow::Tensor; -using tensorflow::TensorShape; -using tensorflow::TensorShapeUtils; -using tensorflow::errors::InvalidArgument; - -namespace tensorflow { - -// TODO(ataei): Consider using RowMajor maps. -typedef Eigen::Map< - Eigen::Matrix> - EigenMatrixFloatMap; -typedef Eigen::Map< - const Eigen::Matrix> - ConstEigenMatrixInt64Map; -typedef Eigen::Map< - const Eigen::Matrix> - ConstEigenMatrixFloatMap; - -class WALSComputePartialLhsAndRhsOp : public OpKernel { - public: - explicit WALSComputePartialLhsAndRhsOp(OpKernelConstruction* context) - : OpKernel(context) { - OP_REQUIRES_OK(context, - context->MatchSignature( - {DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_FLOAT, DT_INT64, - DT_FLOAT, DT_FLOAT, DT_INT64, DT_BOOL}, - {DT_FLOAT, DT_FLOAT})); - } - - void Compute(OpKernelContext* context) override { - const Tensor& factors = context->input(0); - const Tensor& factor_weights = context->input(1); - const Tensor& unobserved_weights = context->input(2); - const Tensor& input_weights = context->input(3); - const Tensor& input_indices = context->input(4); - const Tensor& input_values = context->input(5); - const Tensor& entry_weights = context->input(6); - const Tensor& input_block_size = context->input(7); - const Tensor& input_is_transpose = context->input(8); - - OP_REQUIRES(context, TensorShapeUtils::IsMatrix(factors.shape()), - InvalidArgument("Input factors should be a matrix.")); - OP_REQUIRES(context, TensorShapeUtils::IsVector(factor_weights.shape()), - InvalidArgument("Input factor_weights should be a vector.")); - OP_REQUIRES( - context, TensorShapeUtils::IsScalar(unobserved_weights.shape()), - InvalidArgument("Input unobserved_weights should be a scalar.")); - OP_REQUIRES(context, TensorShapeUtils::IsVector(input_weights.shape()), - InvalidArgument("Input input_weights should be a vector.")); - OP_REQUIRES(context, TensorShapeUtils::IsMatrix(input_indices.shape()), - InvalidArgument("Input input_indices should be a matrix.")); - OP_REQUIRES( - context, input_indices.dim_size(1) == 2, - InvalidArgument("Input input_indices should have shape (?, 2).")); - OP_REQUIRES(context, TensorShapeUtils::IsVector(input_values.shape()), - InvalidArgument("Input input_values should be a vector")); - OP_REQUIRES(context, TensorShapeUtils::IsVector(entry_weights.shape()), - InvalidArgument("Input entry_weights should be a vector")); - OP_REQUIRES(context, input_indices.dim_size(0) == input_values.dim_size(0), - InvalidArgument("Input input_values' length should match the " - "first dimension of Input input_indices ")); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(input_block_size.shape()), - InvalidArgument("Input input_block_size should be a scalar.")); - OP_REQUIRES( - context, TensorShapeUtils::IsScalar(input_is_transpose.shape()), - InvalidArgument("Input input_is_transpose should be a scalar.")); - OP_REQUIRES( - context, - ((input_weights.dim_size(0) > 0 && - factor_weights.dim_size(0) == factors.dim_size(0) && - entry_weights.dim_size(0) == 0) || - (input_weights.dim_size(0) == 0 && factor_weights.dim_size(0) == 0 && - entry_weights.dim_size(0) == input_indices.dim_size(0))), - InvalidArgument("To specify the weights for observed entries, either " - "(1) entry_weights must be set or (2) input_weights " - "and factor_weights must be set, but not both.")); - // TODO(yifanchen): Deprecate the support of input_weights and - // factor_weights. - - const int64 factor_dim = factors.dim_size(1); - const int64 factors_size = factors.dim_size(0); - const int64 num_nonzero_elements = input_indices.dim_size(0); - const int64 block_size = input_block_size.scalar()(); - const auto& factor_weights_vec = factor_weights.vec(); - const auto& input_weights_vec = input_weights.vec(); - const float w_0 = unobserved_weights.scalar()(); - const auto& input_values_vec = input_values.vec(); - const auto& entry_weights_vec = entry_weights.vec(); - - ConstEigenMatrixFloatMap factors_mat(factors.matrix().data(), - factor_dim, factors_size); - ConstEigenMatrixInt64Map indices_mat(input_indices.matrix().data(), - 2, num_nonzero_elements); - - Tensor* output_lhs_tensor; - OP_REQUIRES_OK(context, - context->allocate_output( - 0, TensorShape({block_size, factor_dim, factor_dim}), - &output_lhs_tensor)); - auto output_lhs_t = output_lhs_tensor->tensor(); - output_lhs_t.setZero(); - Tensor* output_rhs_tensor; - OP_REQUIRES_OK(context, context->allocate_output( - 1, TensorShape({block_size, factor_dim}), - &output_rhs_tensor)); - EigenMatrixFloatMap rhs_mat(output_rhs_tensor->matrix().data(), - factor_dim, block_size); - rhs_mat.setZero(); - const bool is_transpose = input_is_transpose.scalar()(); - - auto get_input_index = [is_transpose, &indices_mat](int64 i) { - return is_transpose ? indices_mat(1, i) : indices_mat(0, i); - }; - auto get_factor_index = [is_transpose, &indices_mat](int64 i) { - return is_transpose ? indices_mat(0, i) : indices_mat(1, i); - }; - - const bool use_entry_weights = entry_weights_vec.size() > 0; - - // TODO(rmlarsen): In principle, we should be using the SparseTensor class - // and machinery for iterating over groups, but the fact that class - // SparseTensor makes a complete copy of the matrix makes me reluctant to - // use it. - std::vector perm(num_nonzero_elements); - std::iota(perm.begin(), perm.end(), 0); - - typedef std::pair Shard; - std::vector shards; - auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads()); - int64 shard_total = 0; - // Compute a permutation such that get_input_index(perm[i]) is sorted, use - // stable_sort to preserve spatial locality. - std::stable_sort(perm.begin(), perm.end(), - [&get_input_index](int64 i, int64 j) { - return get_input_index(i) < get_input_index(j); - }); - - // Compute the start and end of runs with identical input_index. - // These are the shards of work that can be processed in parallel - // without locking. - int64 start = 0; - int64 end = 0; - while (end < num_nonzero_elements) { - start = end; - while (end < num_nonzero_elements && - get_input_index(perm[start]) == get_input_index(perm[end])) { - ++end; - } - shards.emplace_back(start, end); - shard_total += end - start; - } - CHECK_EQ(shard_total, num_nonzero_elements); - CHECK_LE(shards.size(), num_nonzero_elements); - CHECK_GT(shards.size(), 0); - - // Batch the rank-one updates into a rank-k update to lower memory traffic - const int kMaxBatchSize = 128; - - // Since we do not have an easy way of generating thread id's within the - // range [0,num_threads), we can instead call out to an std::unordered_map - // of matrices and initialize the matrix on the first call. - // However, this might have a performance penalty, as memory allocation can - // cause the OS kernel to enter a critical section and temporarily disable - // parallelism, and the unordered_map must be protected with a read/write - // mutex. - // - // TODO(jpoulson): Simplify after the thread rank can be queried - std::unordered_map factor_batch_map; - mutex map_mutex; - - BlockingCounter counter(shards.size()); - // Lambda encapsulating the per-shard computation. - auto work = [&](const Shard& shard) { - const std::thread::id thread_id = std::this_thread::get_id(); - const size_t id_hash = std::hash()(thread_id); - // If this thread's unique factors_mat.rows() x kMaxBatchSize - // batching matrix has not yet been created, then emplace it into the - // map using the hash of the thread id as the key. - // - // TODO(jpoulson): Switch to try_emplace once C++17 is supported - // TODO(b/72952120): Check whether the 3 lock-unlock pairs can be - // consolidated into just one. - map_mutex.lock(); - const auto key_count = factor_batch_map.count(id_hash); - map_mutex.unlock(); - if (!key_count) { - map_mutex.lock(); - factor_batch_map.emplace( - std::piecewise_construct, std::forward_as_tuple(id_hash), - std::forward_as_tuple(factors_mat.rows(), kMaxBatchSize)); - map_mutex.unlock(); - } - map_mutex.lock(); - auto& factor_batch = factor_batch_map[id_hash]; - map_mutex.unlock(); - - CHECK_GE(shard.first, 0); - CHECK_LE(shard.second, perm.size()); - CHECK_LE(shard.first, shard.second); - const int64 input_index = get_input_index(perm[shard.first]); - const float input_weight = - use_entry_weights ? 1.0 : input_weights_vec(input_index); - // Accumulate the rhs and lhs terms in the normal equations - // for the non-zero elements in the row or column of the sparse matrix - // corresponding to input_index. - int num_batched = 0; - EigenMatrixFloatMap lhs_mat(output_lhs_tensor->flat().data() + - input_index * factor_dim * factor_dim, - factor_dim, factor_dim); - auto lhs_symm = lhs_mat.selfadjointView(); - for (int64 p = shard.first; p < shard.second; ++p) { - const int64 i = perm[p]; - // Check that all entries in the shard have the same input index. - CHECK_EQ(input_index, get_input_index(i)); - const int64 factor_index = get_factor_index(i); - const float input_value = input_values_vec(i); - const float weight = - use_entry_weights ? entry_weights_vec(i) - : input_weight * factor_weights_vec(factor_index); - CHECK_GE(weight, 0); - factor_batch.col(num_batched) = - factors_mat.col(factor_index) * std::sqrt(weight); - ++num_batched; - if (num_batched == kMaxBatchSize) { - lhs_symm.rankUpdate(factor_batch); - num_batched = 0; - } - - rhs_mat.col(input_index) += - input_value * (w_0 + weight) * factors_mat.col(factor_index); - } - if (num_batched != 0) { - auto factor_block = - factor_batch.block(0, 0, factors_mat.rows(), num_batched); - lhs_symm.rankUpdate(factor_block); - } - // Copy lower triangular to upper triangular part of normal equation - // matrix. - lhs_mat = lhs_symm; - counter.DecrementCount(); - }; - for (size_t i = 1; i < shards.size(); ++i) { - worker_threads.workers->Schedule(std::bind(work, shards[i])); - } - // Inline execute the 1st shard. - work(shards[0]); - counter.Wait(); - } -}; - -REGISTER_KERNEL_BUILDER(Name("WALSComputePartialLhsAndRhs").Device(DEVICE_CPU), - WALSComputePartialLhsAndRhsOp); - -} // namespace tensorflow diff --git a/tensorflow/contrib/factorization/ops/factorization_ops.cc b/tensorflow/contrib/factorization/ops/factorization_ops.cc deleted file mode 100644 index 1d31bd38c82..00000000000 --- a/tensorflow/contrib/factorization/ops/factorization_ops.cc +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy -// of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. -// ============================================================================== - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" - -namespace tensorflow { - -REGISTER_OP("WALSComputePartialLhsAndRhs") - .Input("factors: float32") - .Input("factor_weights: float32") - .Input("unobserved_weights: float32") - .Input("input_weights: float32") - .Input("input_indices: int64") - .Input("input_values: float32") - .Input("entry_weights: float32") - .Input("input_block_size: int64") - .Input("input_is_transpose: bool") - .Output("partial_lhs: float32") - .Output("partial_rhs: float32") - .SetShapeFn(shape_inference::UnknownShape) - .Doc(R"( -Computes the partial left-hand side and right-hand side of WALS update. For -observed entry input_indices[i]=[m, n] with value input_values[i]=v, the weight -should be specified either through (1) entry_weights[i] or (2) through -input_weights[m] * factor_weights[n] (if input_is_transpose is false) or -input_weights[n] * factor_weights[m] (if input_is_transpose is true). Note it is -not allowed to have both (1) and (2) specified at the same time: when one -approach is used, the input tensors related to the other approach must be kept -completely empty. - -factors: Matrix of size m * k. -factor_weights: Vector of size m. Corresponds to column weights. Should be empty - if entry_weights is used. -unobserved_weights: Scalar. Weight for unobserved input entries. -input_weights: Vector of size n. Corresponds to row weights. Should be empty if - entry_weights is used. -input_indices: Indices for the input SparseTensor. -input_values: Values for the input SparseTensor. -entry_weights: If not empty, this must be same length as input_vaues and is used - as the per-entry non-zero weight. If this is used, input_weights and - factor_weights must be empty. -input_block_size: Scalar. Number of rows spanned by input. -input_is_transpose: If true, logically transposes the input for processing. -partial_lhs: 3-D tensor with size input_block_size x k x k. -partial_rhs: Matrix with size input_block_size x k. -)"); - -REGISTER_OP("MaskedMatmul") - .Input("a: float32") - .Input("b: float32") - .Input("mask_indices: int64") - .Input("transpose_a: bool") - .Input("transpose_b: bool") - .Output("prod_values: float32") - .SetShapeFn(shape_inference::UnknownShape) - .Doc(R"( -Computes the product a * b, but only for indices (i, j) in mask_indices. The -result is stored in prod_values, a rank 1 tensor, such that for all i, -prod_values[i] = (a * b)[mask_indices[i, 0], mask_indices[i, 1]]. -Note that the shapes of the input matrices a, b should be compatible (after -transposing as specified by the arguments transpose_a and transpose_b). - -Input arguments: -a: A rank 2 tensor of shape [m, n]. -b: A rank 2 tensor of shape [s, t]. The inner dimensions of a and b should match - after transposition. -mask_indices: A rank 2 tensor, of shape [nnz, 2] where nnz is the number of - non-zero elements in the output. The indices are not assumed to be in - lexicographic, or any particular order. - For all i, mask_indices[i, :] should represent a valid index of the product - matrix (a * b) (after transposition). That is: - mask_indices[i, 0] should be in [0, m) if !transpose_a, and in [0, n) - otherwise. - mask_indices[i, 1] should be in [0, t) if !transpose_b, and in [0, s) - otherwise. -transpose_a: A boolean, specifies whether to transpose the matrix a. -transpose_b: A boolean, specifies whether to transpose the matrix b. - -Output arguments: -prod_values: A rank 1 tensor of shape [nnz], representing the values of the - non-zero elements in the product, such that for all i, - prod_values[i] = (a * b)[mask_indices[i, 0], mask_indices[i, 1]]. -)"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/factorization/python/__init__.py b/tensorflow/contrib/factorization/python/__init__.py deleted file mode 100644 index 9eab8845453..00000000000 --- a/tensorflow/contrib/factorization/python/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The python module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py b/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py deleted file mode 100644 index db47073fcc5..00000000000 --- a/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py +++ /dev/null @@ -1,205 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# ============================================================================== -"""Tests for clustering_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.factorization.python.ops import clustering_ops -from tensorflow.python.platform import test - - -class KmeansPlusPlusInitializationTest(test.TestCase): - - # All but one input point are close to (101, 1). With uniform random sampling, - # it is highly improbable for (-1, -1) to be selected. - def setUp(self): - self._points = np.array([[100., 0.], - [101., 2.], - [102., 0.], - [100., 1.], - [100., 2.], - [101., 0.], - [101., 0.], - [101., 1.], - [102., 0.], - [-1., -1.]]).astype(np.float32) - - def runTestWithSeed(self, seed): - with self.cached_session(): - sampled_points = clustering_ops.kmeans_plus_plus_initialization( - self._points, 3, seed, (seed % 5) - 1) - self.assertAllClose( - sorted(sampled_points.eval().tolist()), [[-1., -1.], - [101., 1.], - [101., 1.]], - atol=1.0) - - def testBasic(self): - for seed in range(100): - self.runTestWithSeed(seed) - - -class KMC2InitializationTest(test.TestCase): - - def runTestWithSeed(self, seed): - with self.cached_session(): - distances = np.zeros(1000).astype(np.float32) - distances[6] = 10e7 - distances[4] = 10e3 - - sampled_point = clustering_ops.kmc2_chain_initialization(distances, seed) - self.assertEquals(sampled_point.eval(), 6) - distances[6] = 0.0 - sampled_point = clustering_ops.kmc2_chain_initialization(distances, seed) - self.assertEquals(sampled_point.eval(), 4) - - def testBasic(self): - for seed in range(100): - self.runTestWithSeed(seed) - - -class KMC2InitializationLargeTest(test.TestCase): - - def setUp(self): - self._distances = np.zeros(1001) - self._distances[500] = 100.0 - self._distances[1000] = 50.0 - - def testBasic(self): - with self.cached_session(): - counts = {} - seed = 0 - for i in range(50): - sample = clustering_ops.kmc2_chain_initialization( - self._distances, seed + i).eval() - counts[sample] = counts.get(sample, 0) + 1 - self.assertEquals(len(counts), 2) - self.assertTrue(500 in counts) - self.assertTrue(1000 in counts) - self.assertGreaterEqual(counts[500], 5) - self.assertGreaterEqual(counts[1000], 5) - - -class KMC2InitializationCornercaseTest(test.TestCase): - - def setUp(self): - self._distances = np.zeros(10) - - def runTestWithSeed(self, seed): - with self.cached_session(): - sampled_point = clustering_ops.kmc2_chain_initialization( - self._distances, seed) - self.assertEquals(sampled_point.eval(), 0) - - def testBasic(self): - for seed in range(100): - self.runTestWithSeed(seed) - - -# A simple test that can be verified by hand. -class NearestCentersTest(test.TestCase): - - def setUp(self): - self._points = np.array([[100., 0.], - [101., 2.], - [99., 2.], - [1., 1.]]).astype(np.float32) - - self._centers = np.array([[100., 0.], - [99., 1.], - [50., 50.], - [0., 0.], - [1., 1.]]).astype(np.float32) - - def testNearest1(self): - with self.cached_session(): - [indices, distances] = clustering_ops.nearest_neighbors(self._points, - self._centers, 1) - self.assertAllClose(indices.eval(), [[0], [0], [1], [4]]) - self.assertAllClose(distances.eval(), [[0.], [5.], [1.], [0.]]) - - def testNearest2(self): - with self.cached_session(): - [indices, distances] = clustering_ops.nearest_neighbors(self._points, - self._centers, 2) - self.assertAllClose(indices.eval(), [[0, 1], [0, 1], [1, 0], [4, 3]]) - self.assertAllClose(distances.eval(), - [[0., 2.], [5., 5.], [1., 5.], [0., 2.]]) - - -# A test with large inputs. -class NearestCentersLargeTest(test.TestCase): - - def setUp(self): - num_points = 1000 - num_centers = 2000 - num_dim = 100 - max_k = 5 - # Construct a small number of random points and later tile them. - points_per_tile = 10 - assert num_points % points_per_tile == 0 - points = np.random.standard_normal( - [points_per_tile, num_dim]).astype(np.float32) - # Construct random centers. - self._centers = np.random.standard_normal( - [num_centers, num_dim]).astype(np.float32) - - # Exhaustively compute expected nearest neighbors. - def squared_distance(x, y): - return np.linalg.norm(x - y, ord=2)**2 - - nearest_neighbors = [ - sorted([(squared_distance(point, self._centers[j]), j) - for j in range(num_centers)])[:max_k] for point in points - ] - expected_nearest_neighbor_indices = np.array( - [[i for _, i in nn] for nn in nearest_neighbors]) - expected_nearest_neighbor_squared_distances = np.array( - [[dist for dist, _ in nn] for nn in nearest_neighbors]) - # Tile points and expected results to reach requested size (num_points) - (self._points, self._expected_nearest_neighbor_indices, - self._expected_nearest_neighbor_squared_distances) = ( - np.tile(x, (int(num_points / points_per_tile), 1)) - for x in (points, expected_nearest_neighbor_indices, - expected_nearest_neighbor_squared_distances)) - - def testNearest1(self): - with self.cached_session(): - [indices, distances] = clustering_ops.nearest_neighbors(self._points, - self._centers, 1) - self.assertAllClose(indices.eval(), - self._expected_nearest_neighbor_indices[:, [0]]) - self.assertAllClose( - distances.eval(), - self._expected_nearest_neighbor_squared_distances[:, [0]]) - - def testNearest5(self): - with self.cached_session(): - [indices, distances] = clustering_ops.nearest_neighbors(self._points, - self._centers, 5) - self.assertAllClose(indices.eval(), - self._expected_nearest_neighbor_indices[:, 0:5]) - self.assertAllClose( - distances.eval(), - self._expected_nearest_neighbor_squared_distances[:, 0:5]) - - -if __name__ == "__main__": - np.random.seed(0) - test.main() diff --git a/tensorflow/contrib/factorization/python/kernel_tests/masked_matmul_benchmark.py b/tensorflow/contrib/factorization/python/kernel_tests/masked_matmul_benchmark.py deleted file mode 100644 index a5d2cbf2cc9..00000000000 --- a/tensorflow/contrib/factorization/python/kernel_tests/masked_matmul_benchmark.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Benchmark for masked_matmul_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=g-bad-todo, g-import-not-at-top -import time - -from tensorflow.contrib.factorization.python.ops import gen_factorization_ops -from tensorflow.python.client import session as session_lib -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class MaskedmatmulBenchmark(test.Benchmark): - """Benchmark masked_matmul_ops.""" - - def _make_sparse_mask(self, mask_shape, nnz, sort=False): - """Creates a sparse tensor to be used as a mask in masked_matmul. - - Args: - mask_shape: int list, the shape of the mask. - nnz: int, the number of non-zero elements in the mask. - sort: boolean, whether to sort the indices of the mask (in lexicographic - order). - Returns: - A sparse tensor, with nnz indices, drawn uniformly at random. - """ - num_rows = mask_shape[0] - num_cols = mask_shape[1] - row_idx = random_ops.random_uniform( - [nnz], minval=0, maxval=num_rows, dtype=dtypes.int64) - col_idx = random_ops.random_uniform( - [nnz], minval=0, maxval=num_cols, dtype=dtypes.int64) - indices = array_ops.stack([row_idx, col_idx], axis=1) - values = array_ops.ones([nnz]) - unordered_mask = sparse_tensor.SparseTensor(indices, values, mask_shape) - return sparse_ops.sparse_reorder(unordered_mask) if sort else unordered_mask - - def _run_graph(self, a_shape, b_shape, nnz, num_iters, sort=False, - transpose_a=False, transpose_b=False): - """Run the graph and return its average execution time. - - Args: - a_shape: int list, the shape of the a matrix. - b_shape: int list, the shape of the b matrix. - nnz: int, the number of non-zero elements in the mask. - num_iters: int, the number of iterations to run (the output is the average - execution time, over num_iters). - sort: Boolean, whether to sort the indices in the mask. - transpose_a: boolean, whether to transpose the a matrix. - transpose_b: boolean, whether to transpose the b matrix. - - Returns: - The average duration of the masked_matmul op in seconds. - """ - graph = ops.Graph() - - with graph.as_default(), session_lib.Session(graph=graph) as session: - mask_shape = [a_shape[0], b_shape[1]] - a_shape = a_shape if not transpose_a else [a_shape[1], a_shape[0]] - b_shape = b_shape if not transpose_b else [b_shape[1], b_shape[0]] - a_var = variables.Variable(random_ops.random_normal(a_shape)) - b_var = variables.Variable(random_ops.random_normal(b_shape)) - mask_indices_ph = array_ops.placeholder(dtypes.int64, shape=[nnz, 2]) - a_ph = array_ops.placeholder(dtypes.float32, shape=a_shape) - b_ph = array_ops.placeholder(dtypes.float32, shape=b_shape) - mask = self._make_sparse_mask(mask_shape, nnz, sort) - masked_prod = gen_factorization_ops.masked_matmul( - a_ph, b_ph, mask_indices_ph, transpose_a, transpose_b) - with ops.control_dependencies([masked_prod]): - result = control_flow_ops.no_op() - - variables.global_variables_initializer().run() - avg_wall_time = 0 - for _ in range(num_iters): - a, b, mask_indices = session.run([a_var, b_var, mask.indices]) - feed_dict = { - mask_indices_ph: mask_indices, - a_ph: a, - b_ph: b - } - start_time = time.time() - session.run(result, feed_dict=feed_dict) - avg_wall_time += (time.time() - start_time)/num_iters - - bench_name = ( - "cpu nnz:{nnz} a_shape:{a_shape} b_shape:{b_shape} tr_a:{tr_a} " - "tr_b:{tr_b} sort:{sort}" - ).format( - nnz=nnz, - a_shape=a_shape, - b_shape=b_shape, - tr_a=int(transpose_a), - tr_b=int(transpose_b), - sort=int(sort) - ) - print(bench_name + " - %f secs" % avg_wall_time) - name = bench_name.replace(", ", "_").replace(":", "_").replace(" ", "_") - self.report_benchmark( - name=name, - iters=num_iters, - wall_time=avg_wall_time) - - return avg_wall_time - - # TODO(walidk): compare benchmarks to using existing tf ops. - def benchmark_matmul(self): - num_iters = 10 - nnz = 100000 - for transpose_a in [False, True]: - for transpose_b in [False, True]: - for dim in [200, 400, 800]: - for sort in [False, True]: - a_shape = [10000, dim] - b_shape = [dim, 10000] - self._run_graph(a_shape, b_shape, nnz, num_iters, sort, transpose_a, - transpose_b) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/factorization/python/kernel_tests/masked_matmul_ops_test.py b/tensorflow/contrib/factorization/python/kernel_tests/masked_matmul_ops_test.py deleted file mode 100644 index dd115735d0f..00000000000 --- a/tensorflow/contrib/factorization/python/kernel_tests/masked_matmul_ops_test.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# ============================================================================== -"""Tests for masked_matmul_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=g-bad-todo, g-import-not-at-top -import numpy as np - -from tensorflow.contrib.factorization.python.ops import gen_factorization_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -def MakeMask(): - inds = [[0, 0], [0, 2], [1, 1], [2, 0], [2, 3]] * 100 - indices = np.array(inds).astype(np.int64) - shape = np.array([5, 4]).astype(np.int64) - return (indices, shape) - - -class MaskedProductOpsTest(test.TestCase): - - def setUp(self): - a = [ - [0.1, 0.2, 0.3], - [0.4, 0.5, 0.6], - [0.7, 0.8, 0.9], - [1.1, 1.2, 1.3], - [1.4, 1.5, 1.6], - ] - b = [ - [0.1, 0.4, 0.7, 1.1], - [0.2, 0.5, 0.8, 1.2], - [0.3, 0.6, 0.9, 1.3], - ] - self._dot_products = np.array([0.14, 0.5, 0.77, 0.5, 2.9] * 100) - self._a = np.array(a).astype(np.float32) - self._b = np.array(b).astype(np.float32) - self._mask_ind, self._mask_shape = MakeMask() - - def _runTestMaskedProduct(self, transpose_a, transpose_b): - with ops.Graph().as_default(), self.cached_session() as sess: - a = self._a if not transpose_a else array_ops.transpose(self._a) - b = self._b if not transpose_b else array_ops.transpose(self._b) - - def AssertClose(sp_x, sp_y): - x_inds, x_vals, y_inds, y_vals = sess.run( - [sp_x.indices, sp_x.values, - sp_y.indices, sp_y.values]) - self.assertAllClose(x_inds, y_inds) - self.assertAllClose(x_vals, y_vals) - - values = gen_factorization_ops.masked_matmul( - a, b, self._mask_ind, transpose_a, transpose_b) - result = sparse_tensor.SparseTensor( - self._mask_ind, values, self._mask_shape) - true_result = sparse_tensor.SparseTensor( - self._mask_ind, self._dot_products, self._mask_shape) - AssertClose(result, true_result) - - def _runTestEmptyMaskedProduct(self): - with ops.Graph().as_default(), self.cached_session() as sess: - empty_mask = constant_op.constant(0, shape=[0, 2], dtype=dtypes.int64) - values = gen_factorization_ops.masked_matmul( - self._a, self._b, empty_mask, False, False) - self.assertEqual(len(values.eval(session=sess)), 0) - - def testMaskedProduct(self): - self._runTestMaskedProduct(False, False) - - def testMaskedProductTransposeA(self): - self._runTestMaskedProduct(True, False) - - def testMaskedProductTransposeB(self): - self._runTestMaskedProduct(False, True) - - def testMaskedProductTransposeAAndB(self): - self._runTestMaskedProduct(True, True) - - def testEmptyMaskedProduct(self): - self._runTestEmptyMaskedProduct() - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/factorization/python/kernel_tests/wals_solver_ops_test.py b/tensorflow/contrib/factorization/python/kernel_tests/wals_solver_ops_test.py deleted file mode 100644 index 8a16e22663d..00000000000 --- a/tensorflow/contrib/factorization/python/kernel_tests/wals_solver_ops_test.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# ============================================================================== -"""Tests for wals_solver_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.factorization.python.ops import gen_factorization_ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.platform import test - - -def SparseBlock3x3(): - ind = np.array( - [[0, 0], [0, 2], [1, 1], [2, 0], [2, 1], [3, 2]]).astype(np.int64) - val = np.array([0.1, 0.2, 1.1, 2.0, 2.1, 3.2]).astype(np.float32) - shape = np.array([4, 3]).astype(np.int64) - return sparse_tensor.SparseTensor(ind, val, shape) - - -class WalsSolverOpsTest(test.TestCase): - - def setUp(self): - self._column_factors = np.array([ - [0.1, 0.2, 0.3], - [0.4, 0.5, 0.6], - [0.7, 0.8, 0.9], - ]).astype(np.float32) - self._row_factors = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], - [0.7, 0.8, 0.9], - [1.1, 1.2, 1.3]]).astype(np.float32) - self._column_weights = np.array([0.1, 0.2, 0.3]).astype(np.float32) - self._row_weights = np.array([0.1, 0.2, 0.3, 0.4]).astype(np.float32) - self._unobserved_weights = 0.1 - - def testWalsSolverLhs(self): - sparse_block = SparseBlock3x3() - with self.cached_session(): - [lhs_tensor, - rhs_matrix] = gen_factorization_ops.wals_compute_partial_lhs_and_rhs( - self._column_factors, self._column_weights, self._unobserved_weights, - self._row_weights, sparse_block.indices, sparse_block.values, - [], - input_block_size=sparse_block.dense_shape[0], - input_is_transpose=False) - self.assertAllClose(lhs_tensor.eval(), [[ - [0.014800, 0.017000, 0.019200], - [0.017000, 0.019600, 0.022200], - [0.019200, 0.022200, 0.025200], - ], [ - [0.0064000, 0.0080000, 0.0096000], - [0.0080000, 0.0100000, 0.0120000], - [0.0096000, 0.0120000, 0.0144000], - ], [ - [0.0099000, 0.0126000, 0.0153000], - [0.0126000, 0.0162000, 0.0198000], - [0.0153000, 0.0198000, 0.0243000], - ], [ - [0.058800, 0.067200, 0.075600], - [0.067200, 0.076800, 0.086400], - [0.075600, 0.086400, 0.097200], - ]]) - self.assertAllClose(rhs_matrix.eval(), [[0.019300, 0.023000, 0.026700], - [0.061600, 0.077000, 0.092400], - [0.160400, 0.220000, 0.279600], - [0.492800, 0.563200, 0.633600]]) - - def testWalsSolverLhsEntryWeights(self): - sparse_block = SparseBlock3x3() - with self.cached_session(): - [lhs_tensor, - rhs_matrix] = gen_factorization_ops.wals_compute_partial_lhs_and_rhs( - self._column_factors, [], self._unobserved_weights, - [], sparse_block.indices, sparse_block.values, - [0.01, 0.03, 0.04, 0.03, 0.06, 0.12], - input_block_size=sparse_block.dense_shape[0], - input_is_transpose=False) - self.assertAllClose(lhs_tensor.eval(), [[ - [0.014800, 0.017000, 0.019200], - [0.017000, 0.019600, 0.022200], - [0.019200, 0.022200, 0.025200], - ], [ - [0.0064000, 0.0080000, 0.0096000], - [0.0080000, 0.0100000, 0.0120000], - [0.0096000, 0.0120000, 0.0144000], - ], [ - [0.0099000, 0.0126000, 0.0153000], - [0.0126000, 0.0162000, 0.0198000], - [0.0153000, 0.0198000, 0.0243000], - ], [ - [0.058800, 0.067200, 0.075600], - [0.067200, 0.076800, 0.086400], - [0.075600, 0.086400, 0.097200], - ]]) - self.assertAllClose(rhs_matrix.eval(), [[0.019300, 0.023000, 0.026700], - [0.061600, 0.077000, 0.092400], - [0.160400, 0.220000, 0.279600], - [0.492800, 0.563200, 0.633600]]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/factorization/python/ops/clustering_ops.py b/tensorflow/contrib/factorization/python/ops/clustering_ops.py deleted file mode 100644 index 505d8d731fa..00000000000 --- a/tensorflow/contrib/factorization/python/ops/clustering_ops.py +++ /dev/null @@ -1,771 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Clustering Operations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_clustering_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops.embedding_ops import embedding_lookup -# go/tf-wildcard-import -# pylint: disable=wildcard-import -from tensorflow.python.ops.gen_clustering_ops import * -# pylint: enable=wildcard-import - -# Euclidean distance between vectors U and V is defined as \\(||U - V||_F\\) -# which is the square root of the sum of the absolute squares of the elements -# difference. -SQUARED_EUCLIDEAN_DISTANCE = 'squared_euclidean' -# Cosine distance between vectors U and V is defined as -# \\(1 - (U \dot V) / (||U||_F ||V||_F)\\) -COSINE_DISTANCE = 'cosine' - -RANDOM_INIT = 'random' -KMEANS_PLUS_PLUS_INIT = 'kmeans_plus_plus' -KMC2_INIT = 'kmc2' - -# The name of the variable holding the cluster centers. Used by the Estimator. -CLUSTERS_VAR_NAME = 'clusters' - - -class KMeans(object): - """Creates the graph for k-means clustering.""" - - def __init__(self, - inputs, - num_clusters, - initial_clusters=RANDOM_INIT, - distance_metric=SQUARED_EUCLIDEAN_DISTANCE, - use_mini_batch=False, - mini_batch_steps_per_iteration=1, - random_seed=0, - kmeans_plus_plus_num_retries=2, - kmc2_chain_length=200): - """Creates an object for generating KMeans clustering graph. - - This class implements the following variants of K-means algorithm: - - If use_mini_batch is False, it runs standard full batch K-means. Each step - runs a single iteration of K-Means. This step can be run sharded across - multiple workers by passing a list of sharded inputs to this class. Note - however that a single step needs to process the full input at once. - - If use_mini_batch is True, it runs a generalization of the mini-batch - K-means algorithm. It runs multiple iterations, where each iteration is - composed of mini_batch_steps_per_iteration steps. Two copies of cluster - centers are maintained: one that is updated at the end of each iteration, - and one that is updated every step. The first copy is used to compute - cluster allocations for each step, and for inference, while the second copy - is the one updated each step using the mini-batch update rule. After each - iteration is complete, this second copy is copied back the first copy. - - Note that for use_mini_batch=True, when mini_batch_steps_per_iteration=1, - the algorithm reduces to the standard mini-batch algorithm. Also by setting - mini_batch_steps_per_iteration = num_inputs / batch_size, the algorithm - becomes an asynchronous version of the full-batch algorithm. Note however - that there is no guarantee by this implementation that each input is seen - exactly once per iteration. Also, different updates are applied - asynchronously without locking. So this asynchronous version may not behave - exactly like a full-batch version. - - Args: - inputs: An input tensor or list of input tensors. It is assumed that the - data points have been previously randomly permuted. - num_clusters: An integer tensor specifying the number of clusters. This - argument is ignored if initial_clusters is a tensor or numpy array. - initial_clusters: Specifies the clusters used during initialization. One - of the following: - - a tensor or numpy array with the initial cluster centers. - - a function f(inputs, k) that returns up to k centers from `inputs`. - - "random": Choose centers randomly from `inputs`. - - "kmeans_plus_plus": Use kmeans++ to choose centers from `inputs`. - - "kmc2": Use the fast k-MC2 algorithm to choose centers from `inputs`. - In the last three cases, one batch of `inputs` may not yield - `num_clusters` centers, in which case initialization will require - multiple batches until enough centers are chosen. In the case of - "random" or "kmeans_plus_plus", if the input size is <= `num_clusters` - then the entire batch is chosen to be cluster centers. - distance_metric: Distance metric used for clustering. Supported options: - "squared_euclidean", "cosine". - use_mini_batch: If true, use the mini-batch k-means algorithm. Else assume - full batch. - mini_batch_steps_per_iteration: Number of steps after which the updated - cluster centers are synced back to a master copy. - random_seed: Seed for PRNG used to initialize seeds. - kmeans_plus_plus_num_retries: For each point that is sampled during - kmeans++ initialization, this parameter specifies the number of - additional points to draw from the current distribution before selecting - the best. If a negative value is specified, a heuristic is used to - sample O(log(num_to_sample)) additional points. - kmc2_chain_length: Determines how many candidate points are used by the - k-MC2 algorithm to produce one new cluster centers. If a (mini-)batch - contains less points, one new cluster center is generated from the - (mini-)batch. - - Raises: - ValueError: An invalid argument was passed to initial_clusters or - distance_metric. - """ - if isinstance(initial_clusters, str) and initial_clusters not in [ - RANDOM_INIT, KMEANS_PLUS_PLUS_INIT, KMC2_INIT - ]: - raise ValueError( - "Unsupported initialization algorithm '%s'" % initial_clusters) - if distance_metric not in [SQUARED_EUCLIDEAN_DISTANCE, COSINE_DISTANCE]: - raise ValueError("Unsupported distance metric '%s'" % distance_metric) - self._inputs = inputs if isinstance(inputs, list) else [inputs] - self._num_clusters = num_clusters - self._initial_clusters = initial_clusters - self._distance_metric = distance_metric - self._use_mini_batch = use_mini_batch - self._mini_batch_steps_per_iteration = int(mini_batch_steps_per_iteration) - self._random_seed = random_seed - self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries - self._kmc2_chain_length = kmc2_chain_length - - @classmethod - def _distance_graph(cls, inputs, clusters, distance_metric): - """Computes distance between each input and each cluster center. - - Args: - inputs: list of input Tensors. - clusters: cluster Tensor. - distance_metric: distance metric used for clustering - - Returns: - list of Tensors, where each element corresponds to each element in inputs. - The value is the distance of each row to all the cluster centers. - Currently only Euclidean distance and cosine distance are supported. - """ - assert isinstance(inputs, list) - if distance_metric == SQUARED_EUCLIDEAN_DISTANCE: - return cls._compute_euclidean_distance(inputs, clusters) - elif distance_metric == COSINE_DISTANCE: - return cls._compute_cosine_distance( - inputs, clusters, inputs_normalized=True) - else: - assert False, str(distance_metric) - - @classmethod - def _compute_euclidean_distance(cls, inputs, clusters): - """Computes Euclidean distance between each input and each cluster center. - - Args: - inputs: list of input Tensors. - clusters: cluster Tensor. - - Returns: - list of Tensors, where each element corresponds to each element in inputs. - The value is the distance of each row to all the cluster centers. - """ - output = [] - for inp in inputs: - with ops.colocate_with(inp, ignore_existing=True): - # Computes Euclidean distance. Note the first and third terms are - # broadcast additions. - squared_distance = ( - math_ops.reduce_sum(math_ops.square(inp), 1, keepdims=True) - - 2 * math_ops.matmul(inp, clusters, transpose_b=True) + - array_ops.transpose( - math_ops.reduce_sum( - math_ops.square(clusters), 1, keepdims=True))) - output.append(squared_distance) - - return output - - @classmethod - def _compute_cosine_distance(cls, inputs, clusters, inputs_normalized=True): - """Computes cosine distance between each input and each cluster center. - - Args: - inputs: list of input Tensor. - clusters: cluster Tensor - inputs_normalized: if True, it assumes that inp and clusters are - normalized and computes the dot product which is equivalent to the cosine - distance. Else it L2 normalizes the inputs first. - - Returns: - list of Tensors, where each element corresponds to each element in inp. - The value is the distance of each row to all the cluster centers. - """ - output = [] - if not inputs_normalized: - with ops.colocate_with(clusters, ignore_existing=True): - clusters = nn_impl.l2_normalize(clusters, dim=1) - for inp in inputs: - with ops.colocate_with(inp, ignore_existing=True): - if not inputs_normalized: - inp = nn_impl.l2_normalize(inp, dim=1) - output.append(1 - math_ops.matmul(inp, clusters, transpose_b=True)) - return output - - def _infer_graph(self, inputs, clusters): - """Maps input to closest cluster and the score. - - Args: - inputs: list of input Tensors. - clusters: Tensor of cluster centers. - - Returns: - List of tuple, where each value in tuple corresponds to a value in inp. - The tuple has following three elements: - all_scores: distance of each input to each cluster center. - score: distance of each input to closest cluster center. - cluster_idx: index of cluster center closest to the corresponding input. - """ - assert isinstance(inputs, list) - # Pairwise distances are used only by transform(). In all other cases, this - # sub-graph is not evaluated. - scores = self._distance_graph(inputs, clusters, self._distance_metric) - output = [] - if (self._distance_metric == COSINE_DISTANCE and - not self._clusters_l2_normalized()): - # The cosine distance between normalized vectors x and y is the same as - # 2 * squared_euclidean_distance. We are using this fact and reusing the - # nearest_neighbors op. - # TODO(ands): Support COSINE distance in nearest_neighbors and remove - # this. - with ops.colocate_with(clusters, ignore_existing=True): - clusters = nn_impl.l2_normalize(clusters, dim=1) - for inp, score in zip(inputs, scores): - with ops.colocate_with(inp, ignore_existing=True): - (indices, distances) = gen_clustering_ops.nearest_neighbors( - inp, clusters, 1) - if self._distance_metric == COSINE_DISTANCE: - distances *= 0.5 - output.append((score, array_ops.squeeze(distances, [-1]), - array_ops.squeeze(indices, [-1]))) - return zip(*output) - - def _clusters_l2_normalized(self): - """Returns True if clusters centers are kept normalized.""" - return (self._distance_metric == COSINE_DISTANCE and - (not self._use_mini_batch or - self._mini_batch_steps_per_iteration > 1)) - - def _create_variables(self, num_clusters): - """Creates variables. - - Args: - num_clusters: an integer Tensor providing the number of clusters. - - Returns: - Tuple with following elements: - - cluster_centers: a Tensor for storing cluster centers - - cluster_centers_initialized: bool Variable indicating whether clusters - are initialized. - - cluster_counts: a Tensor for storing counts of points assigned to this - cluster. This is used by mini-batch training. - - cluster_centers_updated: Tensor representing copy of cluster centers - that are updated every step. - - update_in_steps: numbers of steps left before we sync - cluster_centers_updated back to cluster_centers. - """ - init_value = array_ops.constant([], dtype=dtypes.float32) - cluster_centers = variable_scope.variable( - init_value, name=CLUSTERS_VAR_NAME, validate_shape=False) - cluster_centers_initialized = variable_scope.variable( - False, dtype=dtypes.bool, name='initialized') - - if self._use_mini_batch and self._mini_batch_steps_per_iteration > 1: - # Copy of cluster centers actively updated each step according to - # mini-batch update rule. - cluster_centers_updated = variable_scope.variable( - init_value, name='clusters_updated', validate_shape=False) - # How many steps till we copy the updated clusters to cluster_centers. - update_in_steps = variable_scope.variable( - self._mini_batch_steps_per_iteration, - dtype=dtypes.int64, - name='update_in_steps') - # Count of points assigned to cluster_centers_updated. - cluster_counts = variable_scope.variable( - array_ops.zeros([num_clusters], dtype=dtypes.int64)) - else: - cluster_centers_updated = cluster_centers - update_in_steps = None - cluster_counts = ( - variable_scope.variable( - array_ops.ones([num_clusters], dtype=dtypes.int64)) - if self._use_mini_batch else None) - return (cluster_centers, cluster_centers_initialized, cluster_counts, - cluster_centers_updated, update_in_steps) - - @classmethod - def _l2_normalize_data(cls, inputs): - """Normalized the input data.""" - output = [] - for inp in inputs: - with ops.colocate_with(inp, ignore_existing=True): - output.append(nn_impl.l2_normalize(inp, dim=1)) - return output - - def training_graph(self): - """Generate a training graph for kmeans algorithm. - - This returns, among other things, an op that chooses initial centers - (init_op), a boolean variable that is set to True when the initial centers - are chosen (cluster_centers_initialized), and an op to perform either an - entire Lloyd iteration or a mini-batch of a Lloyd iteration (training_op). - The caller should use these components as follows. A single worker should - execute init_op multiple times until cluster_centers_initialized becomes - True. Then multiple workers may execute training_op any number of times. - - Returns: - A tuple consisting of: - all_scores: A matrix (or list of matrices) of dimensions (num_input, - num_clusters) where the value is the distance of an input vector and a - cluster center. - cluster_idx: A vector (or list of vectors). Each element in the vector - corresponds to an input row in 'inp' and specifies the cluster id - corresponding to the input. - scores: Similar to cluster_idx but specifies the distance to the - assigned cluster instead. - cluster_centers_initialized: scalar indicating whether clusters have been - initialized. - init_op: an op to initialize the clusters. - training_op: an op that runs an iteration of training. - """ - # Implementation of kmeans. - if (isinstance(self._initial_clusters, str) or - callable(self._initial_clusters)): - initial_clusters = self._initial_clusters - num_clusters = ops.convert_to_tensor(self._num_clusters) - else: - initial_clusters = ops.convert_to_tensor(self._initial_clusters) - num_clusters = array_ops.shape(initial_clusters)[0] - - inputs = self._inputs - (cluster_centers_var, cluster_centers_initialized, total_counts, - cluster_centers_updated, - update_in_steps) = self._create_variables(num_clusters) - init_op = _InitializeClustersOpFactory( - self._inputs, num_clusters, initial_clusters, self._distance_metric, - self._random_seed, self._kmeans_plus_plus_num_retries, - self._kmc2_chain_length, cluster_centers_var, cluster_centers_updated, - cluster_centers_initialized).op() - cluster_centers = cluster_centers_var - - if self._distance_metric == COSINE_DISTANCE: - inputs = self._l2_normalize_data(inputs) - if not self._clusters_l2_normalized(): - cluster_centers = nn_impl.l2_normalize(cluster_centers, dim=1) - - all_scores, scores, cluster_idx = self._infer_graph(inputs, cluster_centers) - if self._use_mini_batch: - sync_updates_op = self._mini_batch_sync_updates_op( - update_in_steps, cluster_centers_var, cluster_centers_updated, - total_counts) - assert sync_updates_op is not None - with ops.control_dependencies([sync_updates_op]): - training_op = self._mini_batch_training_op( - inputs, cluster_idx, cluster_centers_updated, total_counts) - else: - assert cluster_centers == cluster_centers_var - training_op = self._full_batch_training_op( - inputs, num_clusters, cluster_idx, cluster_centers_var) - - return (all_scores, cluster_idx, scores, cluster_centers_initialized, - init_op, training_op) - - def _mini_batch_sync_updates_op(self, update_in_steps, cluster_centers_var, - cluster_centers_updated, total_counts): - if self._use_mini_batch and self._mini_batch_steps_per_iteration > 1: - assert update_in_steps is not None - with ops.colocate_with(update_in_steps, ignore_existing=True): - - def _f(): - # Note that there is a race condition here, so we do a best effort - # updates here. We reset update_in_steps first so that other workers - # don't duplicate the updates. Also we update cluster_center_vars - # before resetting total_counts to avoid large updates to - # cluster_centers_updated based on partially updated - # cluster_center_vars. - with ops.control_dependencies([ - state_ops.assign(update_in_steps, - self._mini_batch_steps_per_iteration - 1) - ]): - with ops.colocate_with( - cluster_centers_updated, ignore_existing=True): - if self._distance_metric == COSINE_DISTANCE: - cluster_centers = nn_impl.l2_normalize( - cluster_centers_updated, dim=1) - else: - cluster_centers = cluster_centers_updated - with ops.colocate_with(cluster_centers_var, ignore_existing=True): - with ops.control_dependencies( - [state_ops.assign(cluster_centers_var, cluster_centers)]): - with ops.colocate_with(None, ignore_existing=True): - with ops.control_dependencies([ - state_ops.assign(total_counts, - array_ops.zeros_like(total_counts)) - ]): - return array_ops.identity(update_in_steps) - - return control_flow_ops.cond( - update_in_steps <= 0, _f, - lambda: state_ops.assign_sub(update_in_steps, 1)) - else: - return control_flow_ops.no_op() - - def _mini_batch_training_op(self, inputs, cluster_idx_list, cluster_centers, - total_counts): - """Creates an op for training for mini batch case. - - Args: - inputs: list of input Tensors. - cluster_idx_list: A vector (or list of vectors). Each element in the - vector corresponds to an input row in 'inp' and specifies the cluster id - corresponding to the input. - cluster_centers: Tensor Ref of cluster centers. - total_counts: Tensor Ref of cluster counts. - - Returns: - An op for doing an update of mini-batch k-means. - """ - update_ops = [] - for inp, cluster_idx in zip(inputs, cluster_idx_list): - with ops.colocate_with(inp, ignore_existing=True): - assert total_counts is not None - cluster_idx = array_ops.reshape(cluster_idx, [-1]) - # Dedupe the unique ids of cluster_centers being updated so that updates - # can be locally aggregated. - unique_ids, unique_idx = array_ops.unique(cluster_idx) - num_unique_cluster_idx = array_ops.size(unique_ids) - # Fetch the old values of counts and cluster_centers. - with ops.colocate_with(total_counts, ignore_existing=True): - old_counts = array_ops.gather(total_counts, unique_ids) - # TODO(agarwal): This colocation seems to run into problems. Fix it. - with ops.colocate_with(cluster_centers, ignore_existing=True): - old_cluster_centers = array_ops.gather(cluster_centers, unique_ids) - # Locally aggregate the increment to counts. - count_updates = math_ops.unsorted_segment_sum( - array_ops.ones_like(unique_idx, dtype=total_counts.dtype), - unique_idx, num_unique_cluster_idx) - # Locally compute the sum of inputs mapped to each id. - # For a cluster with old cluster value x, old count n, and with data - # d_1,...d_k newly assigned to it, we recompute the new value as - # \\(x += (sum_i(d_i) - k * x) / (n + k)\\). - # Compute \\(sum_i(d_i)\\), see comment above. - cluster_center_updates = math_ops.unsorted_segment_sum( - inp, unique_idx, num_unique_cluster_idx) - # Shape to enable broadcasting count_updates and learning_rate to inp. - # It extends the shape with 1's to match the rank of inp. - broadcast_shape = array_ops.concat([ - array_ops.reshape(num_unique_cluster_idx, [1]), - array_ops.ones( - array_ops.reshape(array_ops.rank(inp) - 1, [1]), - dtype=dtypes.int32) - ], 0) - # Subtract k * x, see comment above. - cluster_center_updates -= math_ops.cast( - array_ops.reshape(count_updates, broadcast_shape), - inp.dtype) * old_cluster_centers - learning_rate = math_ops.reciprocal( - math_ops.cast(old_counts + count_updates, inp.dtype)) - learning_rate = array_ops.reshape(learning_rate, broadcast_shape) - # scale by 1 / (n + k), see comment above. - cluster_center_updates *= learning_rate - # Apply the updates. - update_counts = state_ops.scatter_add(total_counts, unique_ids, - count_updates) - update_cluster_centers = state_ops.scatter_add( - cluster_centers, unique_ids, cluster_center_updates) - update_ops.extend([update_counts, update_cluster_centers]) - return control_flow_ops.group(*update_ops) - - def _full_batch_training_op(self, inputs, num_clusters, cluster_idx_list, - cluster_centers): - """Creates an op for training for full batch case. - - Args: - inputs: list of input Tensors. - num_clusters: an integer Tensor providing the number of clusters. - cluster_idx_list: A vector (or list of vectors). Each element in the - vector corresponds to an input row in 'inp' and specifies the cluster id - corresponding to the input. - cluster_centers: Tensor Ref of cluster centers. - - Returns: - An op for doing an update of mini-batch k-means. - """ - cluster_sums = [] - cluster_counts = [] - epsilon = constant_op.constant(1e-6, dtype=inputs[0].dtype) - for inp, cluster_idx in zip(inputs, cluster_idx_list): - with ops.colocate_with(inp, ignore_existing=True): - cluster_sums.append( - math_ops.unsorted_segment_sum(inp, cluster_idx, num_clusters)) - cluster_counts.append( - math_ops.unsorted_segment_sum( - array_ops.reshape( - array_ops.ones( - array_ops.reshape(array_ops.shape(inp)[0], [-1])), - [-1, 1]), cluster_idx, num_clusters)) - with ops.colocate_with(cluster_centers, ignore_existing=True): - new_clusters_centers = math_ops.add_n(cluster_sums) / ( - math_ops.cast(math_ops.add_n(cluster_counts), cluster_sums[0].dtype) + - epsilon) - if self._clusters_l2_normalized(): - new_clusters_centers = nn_impl.l2_normalize(new_clusters_centers, dim=1) - return state_ops.assign(cluster_centers, new_clusters_centers) - - -class _InitializeClustersOpFactory(object): - """Internal class to create the op to initialize the clusters. - - The op performs this algorithm (see constructor args): - - num_remaining = num_clusters - length(cluster_centers) - if num_remaining == 0: - assert that cluster_centers_initialized is true - else: - assert that num_remaining > 0 - new_centers = choose up to num_remaining initial centers - l2-normalize new_centers if using cosine distance - all_centers = concat(cluster_centers, new_centers) - cluster_centers := all_centers - if there is a cluster_centers_updated variable: - cluster_centers_updated := cluster_centers - num_now_remaining = num_clusters - length(cluster_centers) - if num_now_remaining == 0: - cluster_centers_initialized := true - """ - - # TODO(ccolby): Refactor this class so that kmc2 isn't so much a special case. - - def __init__(self, inputs, num_clusters, initial_clusters, distance_metric, - random_seed, kmeans_plus_plus_num_retries, kmc2_chain_length, - cluster_centers, cluster_centers_updated, - cluster_centers_initialized): - """Creates an op factory. - - Args: - inputs: See KMeans constructor. - num_clusters: An integer Tensor providing the number of clusters. - initial_clusters: See KMeans constructor. - distance_metric: See KMeans constructor. - random_seed: See KMeans constructor. - kmeans_plus_plus_num_retries: See KMeans constructor. - kmc2_chain_length: See KMeans constructor. - cluster_centers: The TF variable holding the initial centers. It may - already contain some centers when the op is executed. - cluster_centers_updated: A second TF variable to hold a copy of the - initial centers, used for full-batch mode. In mini-batch mode, - cluster_centers_updated is the same variable as cluster_centers. - cluster_centers_initialized: A boolean TF variable that will be set - to true when all the initial centers have been chosen. - """ - # All of these instance variables are constants. - self._inputs = inputs - self._num_clusters = num_clusters - self._initial_clusters = initial_clusters - self._distance_metric = distance_metric - self._random_seed = random_seed - self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries - self._kmc2_chain_length = kmc2_chain_length - self._cluster_centers = cluster_centers - self._cluster_centers_updated = cluster_centers_updated - self._cluster_centers_initialized = cluster_centers_initialized - - self._num_selected = array_ops.shape(self._cluster_centers)[0] - self._num_remaining = self._num_clusters - self._num_selected - self._num_data = math_ops.add_n( - [array_ops.shape(i)[0] for i in self._inputs]) - - def _random(self): - indices = random_ops.random_uniform( - array_ops.reshape(self._num_remaining, [-1]), - minval=0, - maxval=math_ops.cast(self._num_data, dtypes.int64), - seed=self._random_seed, - dtype=dtypes.int64) - return embedding_lookup(self._inputs, indices, partition_strategy='div') - - def _kmeans_plus_plus(self): - # Points from only the first shard are used for initializing centers. - # TODO(ands): Use all points. - inp = self._inputs[0] - if self._distance_metric == COSINE_DISTANCE: - inp = nn_impl.l2_normalize(inp, dim=1) - return gen_clustering_ops.kmeans_plus_plus_initialization( - inp, - math_ops.cast(self._num_remaining, dtypes.int64), - self._random_seed, - self._kmeans_plus_plus_num_retries) - - def _kmc2_multiple_centers(self): - """Adds new initial cluster centers using the k-MC2 algorithm. - - In each call to the op, the provided batch is split into subsets based on - the specified `kmc2_chain_length`. On each subset, a single Markov chain of - the k-MC2 algorithm is used to add *one* new center cluster center. If there - are less than `kmc2_chain_length` points in the subset, a single center is - added using one Markov chain on the full input. It is assumed that the - provided batch has previously been randomly permuted. Otherwise, k-MC2 may - return suboptimal centers. - - Returns: - An op that adds new cluster centers. - """ - # The op only operates on the first shard of data. - first_shard = self._inputs[0] - # Number of points in the input that can be used. - batch_size = array_ops.shape(first_shard)[0] - # Maximum number of subsets such that the size of each subset is at least - # `kmc2_chain_length`. Final subsets may be larger. - max_to_sample = math_ops.cast( - batch_size / self._kmc2_chain_length, dtype=dtypes.int32) - # We sample at least one new center and at most all remaining centers. - num_to_sample = math_ops.maximum( - math_ops.minimum(self._num_remaining, max_to_sample), 1) - - def _cond(i, _): - """Stopping condition for the while loop.""" - return math_ops.less(i, num_to_sample) - - def _body(i, _): - """Body that adds a single new center based on a subset.""" - - def _sample_random(): - """Returns a random point as a cluster center.""" - # By assumption the batch is reshuffled and _sample_random is always - # called for i=0. Hence, we simply return the first point. - new_center = array_ops.reshape(first_shard[0], [1, -1]) - if self._distance_metric == COSINE_DISTANCE: - new_center = nn_impl.l2_normalize(new_center, dim=1) - return new_center - - def _sample_kmc2_chain(): - """Returns previous centers as well as a new center sampled using k-MC2. - """ - # Extract the subset from the underlying batch. - start = i * self._kmc2_chain_length - end = start + self._kmc2_chain_length - subset = first_shard[start:end] - # Compute the distances from points in the subset to previous centers. - _, distances = gen_clustering_ops.nearest_neighbors( - subset, self._cluster_centers, 1) - # Sample index of new center using k-MC2 Markov chain. - new_center_index = gen_clustering_ops.kmc2_chain_initialization( - array_ops.squeeze(distances), self._random_seed) - # Extract actual new center. - newly_sampled_center = array_ops.reshape(subset[new_center_index], - [1, -1]) - # Return concatenation with previously sampled centers. - if self._distance_metric == COSINE_DISTANCE: - newly_sampled_center = nn_impl.l2_normalize( - newly_sampled_center, dim=1) - return array_ops.concat([self._cluster_centers, newly_sampled_center], - 0) - - # Obtain a random point if there are no previously sampled centers. - # Otherwise, construct a k-MC2 Markov chain. - new_centers = control_flow_ops.cond( - math_ops.equal(self._num_selected, 0), _sample_random, - _sample_kmc2_chain) - # Assign new cluster centers to underlying variable. - assigned_centers = state_ops.assign( - self._cluster_centers, new_centers, validate_shape=False) - if self._cluster_centers_updated is not self._cluster_centers: - assigned_centers = state_ops.assign( - self._cluster_centers_updated, - assigned_centers, - validate_shape=False) - return i + 1, self._num_clusters - array_ops.shape(assigned_centers)[0] - - # Add num_to_sample new data points. - _, num_remaining = control_flow_ops.while_loop(_cond, _body, [0, 0]) - return num_remaining - - def _greedy_batch_sampler(self, sampler): - # If the input dataset size is smaller than the number of centers - # remaining, choose the entire input dataset as centers. This can happen - # with mini-batch. Otherwise, sample the batch according to the provided - # sampler. - return control_flow_ops.cond(self._num_data <= self._num_remaining, - lambda: array_ops.concat(self._inputs, 0), - sampler) - - def _single_batch_sampler(self, sampler): - # Enforce that there are at least as many data points as centers - # remaining. This gives the provided sampler the chance to select all - # remaining centers from a single batch. - with ops.control_dependencies( - [check_ops.assert_greater_equal(self._num_data, self._num_remaining)]): - return sampler() - - def _choose_initial_centers(self): - if isinstance(self._initial_clusters, str): - if self._initial_clusters == RANDOM_INIT: - return self._greedy_batch_sampler(self._random) - else: # self._initial_clusters == KMEANS_PLUS_PLUS_INIT - return self._single_batch_sampler(self._kmeans_plus_plus) - elif callable(self._initial_clusters): - return self._initial_clusters(self._inputs, self._num_remaining) - else: - with ops.control_dependencies([ - check_ops.assert_equal(self._num_remaining, - array_ops.shape(self._initial_clusters)[0]) - ]): - return self._initial_clusters - - def _add_new_centers(self): - """Adds some centers and returns the number of centers remaining.""" - new_centers = self._choose_initial_centers() - if self._distance_metric == COSINE_DISTANCE: - new_centers = nn_impl.l2_normalize(new_centers, dim=1) - # If cluster_centers is empty, it doesn't have the right shape for concat. - all_centers = control_flow_ops.cond( - math_ops.equal(self._num_selected, 0), lambda: new_centers, - lambda: array_ops.concat([self._cluster_centers, new_centers], 0)) - # TODO(ccolby): De-dupe all_centers? - a = state_ops.assign( - self._cluster_centers, all_centers, validate_shape=False) - if self._cluster_centers_updated is not self._cluster_centers: - a = state_ops.assign( - self._cluster_centers_updated, a, validate_shape=False) - return self._num_clusters - array_ops.shape(a)[0] - - def _initialize(self): - with ops.control_dependencies([ - check_ops.assert_positive(self._num_remaining), - ]): - if self._initial_clusters == KMC2_INIT: - num_now_remaining = self._kmc2_multiple_centers() - else: - num_now_remaining = self._add_new_centers() - return control_flow_ops.cond( - math_ops.equal(num_now_remaining, 0), - lambda: state_ops.assign(self._cluster_centers_initialized, True), - control_flow_ops.no_op) - - def op(self): - """Returns the cluster initializer op.""" - return control_flow_ops.cond( - math_ops.equal(self._num_remaining, 0), - lambda: check_ops.assert_equal(self._cluster_centers_initialized, True), - self._initialize) diff --git a/tensorflow/contrib/factorization/python/ops/factorization_ops.py b/tensorflow/contrib/factorization/python/ops/factorization_ops.py deleted file mode 100644 index e04de0579b1..00000000000 --- a/tensorflow/contrib/factorization/python/ops/factorization_ops.py +++ /dev/null @@ -1,1017 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for matrix factorization.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numbers - -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib.factorization.python.ops import gen_factorization_ops -from tensorflow.contrib.util import loader -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import resource_loader -from tensorflow.python.util.compat import collections_abc - -_factorization_ops = loader.load_op_library( - resource_loader.get_path_to_datafile("_factorization_ops.so")) - - -class WALSModel(object): - r"""A model for Weighted Alternating Least Squares matrix factorization. - - It minimizes the following loss function over U, V: - $$ - \|\sqrt W \odot (A - U V^T)\|_F^2 + \lambda (\|U\|_F^2 + \|V\|_F^2) - $$ - where, - A: input matrix, - W: weight matrix. Note that the (element-wise) square root of the weights - is used in the objective function. - U, V: row_factors and column_factors matrices, - \\(\lambda)\\: regularization. - Also we assume that W is of the following special form: - \\( W_{ij} = W_0 + R_i * C_j \\) if \\(A_{ij} \ne 0\\), - \\(W_{ij} = W_0\\) otherwise. - where, - \\(W_0\\): unobserved_weight, - \\(R_i\\): row_weights, - \\(C_j\\): col_weights. - - Note that the current implementation supports two operation modes: The default - mode is for the condition where row_factors and col_factors can individually - fit into the memory of each worker and these will be cached. When this - condition can't be met, setting use_factors_weights_cache to False allows the - larger problem sizes with slight performance penalty as this will avoid - creating the worker caches and instead the relevant weight and factor values - are looked up from parameter servers at each step. - - Loss computation: The loss can be computed efficiently by decomposing it into - a sparse term and a Gramian term, see wals.md. - The loss is returned by the update_{col, row}_factors(sp_input), and is - normalized as follows: - _, _, unregularized_loss, regularization, sum_weights = - update_row_factors(sp_input) - if sp_input contains the rows \\({A_i, i \in I}\\), and the input matrix A - has n total rows, then the minibatch loss = unregularized_loss + - regularization is - $$ - (\|\sqrt W_I \odot (A_I - U_I V^T)\|_F^2 + \lambda \|U_I\|_F^2) * n / |I| + - \lambda \|V\|_F^2 - $$ - The sum_weights tensor contains the normalized sum of weights - \\(sum(W_I) * n / |I|\\). - - A typical usage example (pseudocode): - - with tf.Graph().as_default(): - # Set up the model object. - model = tf.contrib.factorization.WALSModel(....) - - # To be run only once as part of session initialization. In distributed - # training setting, this should only be run by the chief trainer and all - # other trainers should block until this is done. - model_init_op = model.initialize_op - - # To be run once per worker after session is available, prior to - # the prep_gramian_op for row(column) can be run. - worker_init_op = model.worker_init - - # To be run once per iteration sweep before the row(column) update - # initialize ops can be run. Note that in the distributed training - # situations, this should only be run by the chief trainer. All other - # trainers need to block until this is done. - row_update_prep_gramian_op = model.row_update_prep_gramian_op - col_update_prep_gramian_op = model.col_update_prep_gramian_op - - # To be run once per worker per iteration sweep. Must be run before - # any actual update ops can be run. - init_row_update_op = model.initialize_row_update_op - init_col_update_op = model.initialize_col_update_op - - # Ops to update row(column). This can either take the entire sparse - # tensor or slices of sparse tensor. For distributed trainer, each - # trainer handles just part of the matrix. - _, row_update_op, unreg_row_loss, row_reg, _ = model.update_row_factors( - sp_input=matrix_slices_from_queue_for_worker_shard) - row_loss = unreg_row_loss + row_reg - _, col_update_op, unreg_col_loss, col_reg, _ = model.update_col_factors( - sp_input=transposed_matrix_slices_from_queue_for_worker_shard, - transpose_input=True) - col_loss = unreg_col_loss + col_reg - - ... - - # model_init_op is passed to Supervisor. Chief trainer runs it. Other - # trainers wait. - sv = tf.compat.v1.train.Supervisor(is_chief=is_chief, - ..., - init_op=tf.group(..., model_init_op, ...), ...) - ... - - with sv.managed_session(...) as sess: - # All workers/trainers run it after session becomes available. - worker_init_op.run(session=sess) - - ... - - while i in iterations: - - # All trainers need to sync up here. - while not_all_ready: - wait - - # Row update sweep. - if is_chief: - row_update_prep_gramian_op.run(session=sess) - else: - wait_for_chief - - # All workers run upate initialization. - init_row_update_op.run(session=sess) - - # Go through the matrix. - reset_matrix_slices_queue_for_worker_shard - while_matrix_slices: - row_update_op.run(session=sess) - - # All trainers need to sync up here. - while not_all_ready: - wait - - # Column update sweep. - if is_chief: - col_update_prep_gramian_op.run(session=sess) - else: - wait_for_chief - - # All workers run upate initialization. - init_col_update_op.run(session=sess) - - # Go through the matrix. - reset_transposed_matrix_slices_queue_for_worker_shard - while_transposed_matrix_slices: - col_update_op.run(session=sess) - """ - - def __init__(self, - input_rows, - input_cols, - n_components, - unobserved_weight=0.1, - regularization=None, - row_init="random", - col_init="random", - num_row_shards=1, - num_col_shards=1, - row_weights=1, - col_weights=1, - use_factors_weights_cache=True, - use_gramian_cache=True, - use_scoped_vars=False): - """Creates model for WALS matrix factorization. - - Args: - input_rows: total number of rows for input matrix. - input_cols: total number of cols for input matrix. - n_components: number of dimensions to use for the factors. - unobserved_weight: weight given to unobserved entries of matrix. - regularization: weight of L2 regularization term. If None, no - regularization is done. - row_init: initializer for row factor. Can be a tensor or numpy constant. - If set to "random", the value is initialized randomly. - col_init: initializer for column factor. See row_init for details. - num_row_shards: number of shards to use for row factors. - num_col_shards: number of shards to use for column factors. - row_weights: Must be in one of the following three formats: None, a list - of lists of non-negative real numbers (or equivalent iterables) or a - single non-negative real number. - - When set to None, w_ij = unobserved_weight, which simplifies to ALS. - Note that col_weights must also be set to "None" in this case. - - If it is a list of lists of non-negative real numbers, it needs to be - in the form of [[w_0, w_1, ...], [w_k, ... ], [...]], with the number of - inner lists matching the number of row factor shards and the elements in - each inner list are the weights for the rows of the corresponding row - factor shard. In this case, w_ij = unobserved_weight + - row_weights[i] * col_weights[j]. - - If this is a single non-negative real number, this value is used for - all row weights and \\(w_ij\\) = unobserved_weight + row_weights * - col_weights[j]. - Note that it is allowed to have row_weights as a list while col_weights - a single number or vice versa. - col_weights: See row_weights. - use_factors_weights_cache: When True, the factors and weights will be - cached on the workers before the updates start. Defaults to True. Note - that the weights cache is initialized through `worker_init`, and the - row/col factors cache is initialized through - `initialize_{col/row}_update_op`. In the case where the weights are - computed outside and set before the training iterations start, it is - important to ensure the `worker_init` op is run afterwards for the - weights cache to take effect. - use_gramian_cache: When True, the Gramians will be cached on the workers - before the updates start. Defaults to True. - use_scoped_vars: When True, the factor and weight vars will also be nested - in a tf.name_scope. - """ - self._input_rows = input_rows - self._input_cols = input_cols - self._num_row_shards = num_row_shards - self._num_col_shards = num_col_shards - self._n_components = n_components - self._unobserved_weight = unobserved_weight - self._regularization = regularization - self._regularization_matrix = ( - regularization * linalg_ops.eye(self._n_components) - if regularization is not None else None) - assert (row_weights is None) == (col_weights is None) - self._use_factors_weights_cache = use_factors_weights_cache - self._use_gramian_cache = use_gramian_cache - - if use_scoped_vars: - with ops.name_scope("row_weights"): - self._row_weights = WALSModel._create_weights( - row_weights, self._input_rows, self._num_row_shards, "row_weights") - with ops.name_scope("col_weights"): - self._col_weights = WALSModel._create_weights( - col_weights, self._input_cols, self._num_col_shards, "col_weights") - with ops.name_scope("row_factors"): - self._row_factors = self._create_factors( - self._input_rows, self._n_components, self._num_row_shards, - row_init, "row_factors") - with ops.name_scope("col_factors"): - self._col_factors = self._create_factors( - self._input_cols, self._n_components, self._num_col_shards, - col_init, "col_factors") - else: - self._row_weights = WALSModel._create_weights( - row_weights, self._input_rows, self._num_row_shards, "row_weights") - self._col_weights = WALSModel._create_weights( - col_weights, self._input_cols, self._num_col_shards, "col_weights") - self._row_factors = self._create_factors( - self._input_rows, self._n_components, self._num_row_shards, row_init, - "row_factors") - self._col_factors = self._create_factors( - self._input_cols, self._n_components, self._num_col_shards, col_init, - "col_factors") - - self._row_gramian = self._create_gramian(self._n_components, "row_gramian") - self._col_gramian = self._create_gramian(self._n_components, "col_gramian") - with ops.name_scope("row_prepare_gramian"): - self._row_update_prep_gramian = self._prepare_gramian( - self._col_factors, self._col_gramian) - with ops.name_scope("col_prepare_gramian"): - self._col_update_prep_gramian = self._prepare_gramian( - self._row_factors, self._row_gramian) - with ops.name_scope("transient_vars"): - self._create_transient_vars() - - @property - def row_factors(self): - """Returns a list of tensors corresponding to row factor shards.""" - return self._row_factors - - @property - def col_factors(self): - """Returns a list of tensors corresponding to column factor shards.""" - return self._col_factors - - @property - def row_weights(self): - """Returns a list of tensors corresponding to row weight shards.""" - return self._row_weights - - @property - def col_weights(self): - """Returns a list of tensors corresponding to col weight shards.""" - return self._col_weights - - @property - def initialize_op(self): - """Returns an op for initializing tensorflow variables.""" - all_vars = self._row_factors + self._col_factors - all_vars.extend([self._row_gramian, self._col_gramian]) - if self._row_weights is not None: - assert self._col_weights is not None - all_vars.extend(self._row_weights + self._col_weights) - return variables.variables_initializer(all_vars) - - @classmethod - def _shard_sizes(cls, dims, num_shards): - """Helper function to split dims values into num_shards.""" - shard_size, residual = divmod(dims, num_shards) - return [shard_size + 1] * residual + [shard_size] * (num_shards - residual) - - @classmethod - def _create_factors(cls, rows, cols, num_shards, init, name): - """Helper function to create row and column factors.""" - if callable(init): - init = init() - if isinstance(init, list): - assert len(init) == num_shards - elif isinstance(init, str) and init == "random": - pass - elif num_shards == 1: - init = [init] - sharded_matrix = [] - sizes = cls._shard_sizes(rows, num_shards) - assert len(sizes) == num_shards - - def make_initializer(i, size): - - def initializer(): - if init == "random": - return random_ops.random_normal([size, cols]) - else: - return init[i] - - return initializer - - for i, size in enumerate(sizes): - var_name = "%s_shard_%d" % (name, i) - var_init = make_initializer(i, size) - sharded_matrix.append( - variable_scope.variable( - var_init, dtype=dtypes.float32, name=var_name)) - - return sharded_matrix - - @classmethod - def _create_weights(cls, wt_init, num_wts, num_shards, name): - """Helper function to create sharded weight vector. - - Args: - wt_init: init value for the weight. If None, weights are not created. This - can be one of the None, a list of non-negative real numbers or a single - non-negative real number (or equivalent iterables). - num_wts: total size of all the weight shards - num_shards: number of shards for the weights - name: name for the new Variables. - - Returns: - A list of weight shard Tensors. - - Raises: - ValueError: If wt_init is not the right format. - """ - - if wt_init is None: - return None - - init_mode = "list" - if isinstance(wt_init, collections_abc.Iterable): - if num_shards == 1 and len(wt_init) == num_wts: - wt_init = [wt_init] - assert len(wt_init) == num_shards - elif isinstance(wt_init, numbers.Real) and wt_init >= 0: - init_mode = "scalar" - else: - raise ValueError( - "Invalid weight initialization argument. Must be one of these: " - "None, a real non-negative real number, or a list of lists of " - "non-negative real numbers (or equivalent iterables) corresponding " - "to sharded factors.") - - sizes = cls._shard_sizes(num_wts, num_shards) - assert len(sizes) == num_shards - - def make_wt_initializer(i, size): - - def initializer(): - if init_mode == "scalar": - return wt_init * array_ops.ones([size]) - else: - return wt_init[i] - - return initializer - - sharded_weight = [] - for i, size in enumerate(sizes): - var_name = "%s_shard_%d" % (name, i) - var_init = make_wt_initializer(i, size) - sharded_weight.append( - variable_scope.variable( - var_init, dtype=dtypes.float32, name=var_name)) - - return sharded_weight - - @staticmethod - def _create_gramian(n_components, name): - """Helper function to create the gramian variable. - - Args: - n_components: number of dimensions of the factors from which the gramian - will be calculated. - name: name for the new Variables. - - Returns: - A gramian Tensor with shape of [n_components, n_components]. - """ - return variable_scope.variable( - array_ops.zeros([n_components, n_components]), - dtype=dtypes.float32, - name=name) - - @staticmethod - def _transient_var(name): - """Helper function to create a Variable.""" - return variable_scope.variable( - 1.0, - trainable=False, - collections=[ops.GraphKeys.LOCAL_VARIABLES], - validate_shape=False, - name=name) - - def _prepare_gramian(self, factors, gramian): - """Helper function to create ops to prepare/calculate gramian. - - Args: - factors: Variable or list of Variable representing (sharded) factors. - Used to compute the updated corresponding gramian value. - gramian: Variable storing the gramian calculated from the factors. - - Returns: - An op that updates the gramian with the calculated value from the factors. - """ - partial_gramians = [] - for f in factors: - with ops.colocate_with(f): - partial_gramians.append(math_ops.matmul(f, f, transpose_a=True)) - - with ops.colocate_with(gramian): - prep_gramian = state_ops.assign(gramian, - math_ops.add_n(partial_gramians)).op - - return prep_gramian - - def _cached_copy(self, var, name, pass_through=False): - """Helper function to create a worker cached copy of a Variable. - - This assigns the var (either a single Variable or a list of Variables) to - local transient cache Variable(s). Note that if var is a list of Variables, - the assignment is done sequentially to minimize the memory overheads. - Also note that if pass_through is set to True, this does not create new - Variables but simply return the input back. - - Args: - var: A Variable or a list of Variables to cache. - name: name of cached Variable. - pass_through: when set to True, this simply pass through the var back - through identity operator and does not actually creates a cache. - - Returns: - Tuple consisting of following three entries: - cache: the new transient Variable or list of transient Variables - corresponding one-to-one with var. - cache_init: op to initialize the Variable or the list of Variables. - cache_reset: op to reset the Variable or the list of Variables to some - default value. - """ - if var is None: - return None, None, None - elif pass_through: - cache = var - cache_init = control_flow_ops.no_op() - cache_reset = control_flow_ops.no_op() - elif isinstance(var, variables.Variable): - cache = WALSModel._transient_var(name=name) - with ops.colocate_with(cache): - cache_init = state_ops.assign(cache, var, validate_shape=False) - cache_reset = state_ops.assign(cache, 1.0, validate_shape=False) - else: - assert isinstance(var, list) - assert var - cache = [ - WALSModel._transient_var(name="%s_shard_%d" % (name, i)) - for i in xrange(len(var)) - ] - reset_ops = [] - for i, c in enumerate(cache): - with ops.colocate_with(c): - if i == 0: - cache_init = state_ops.assign(c, var[i], validate_shape=False) - else: - with ops.control_dependencies([cache_init]): - cache_init = state_ops.assign(c, var[i], validate_shape=False) - reset_ops.append(state_ops.assign(c, 1.0, validate_shape=False)) - cache_reset = control_flow_ops.group(*reset_ops) - - return cache, cache_init, cache_reset - - def _create_transient_vars(self): - """Creates local cache of factors, weights and gramian for rows and columns. - - Note that currently the caching strategy is as follows: - When initiating a row (resp. column) update: - - The column (resp. row) gramian is computed. - - Optionally, if use_gramian_cache is True, the column (resp. row) Gramian - is cached, while the row (resp. column) gramian is reset. - - Optionally, if use_factors_weights_cache is True, the column (resp. row) - factors and weights are cached, while the row (resp. column) factors and - weights are reset. - """ - - (self._row_factors_cache, row_factors_cache_init, - row_factors_cache_reset) = self._cached_copy( - self._row_factors, - "row_factors_cache", - pass_through=not self._use_factors_weights_cache) - (self._col_factors_cache, col_factors_cache_init, - col_factors_cache_reset) = self._cached_copy( - self._col_factors, - "col_factors_cache", - pass_through=not self._use_factors_weights_cache) - (self._row_wt_cache, row_wt_cache_init, _) = self._cached_copy( - self._row_weights, - "row_wt_cache", - pass_through=not self._use_factors_weights_cache) - (self._col_wt_cache, col_wt_cache_init, _) = self._cached_copy( - self._col_weights, - "col_wt_cache", - pass_through=not self._use_factors_weights_cache) - (self._row_gramian_cache, row_gramian_cache_init, - row_gramian_cache_reset) = self._cached_copy( - self._row_gramian, - "row_gramian_cache", - pass_through=not self._use_gramian_cache) - (self._col_gramian_cache, col_gramian_cache_init, - col_gramian_cache_reset) = self._cached_copy( - self._col_gramian, - "col_gramian_cache", - pass_through=not self._use_gramian_cache) - - self._row_updates_init = control_flow_ops.group( - col_factors_cache_init, row_factors_cache_reset, col_gramian_cache_init, - row_gramian_cache_reset) - self._col_updates_init = control_flow_ops.group( - row_factors_cache_init, col_factors_cache_reset, row_gramian_cache_init, - col_gramian_cache_reset) - - if self._row_wt_cache is not None: - assert self._col_wt_cache is not None - self._worker_init = control_flow_ops.group( - row_wt_cache_init, col_wt_cache_init, name="worker_init") - else: - self._worker_init = control_flow_ops.no_op(name="worker_init") - - @property - def worker_init(self): - """Op to initialize worker state once before starting any updates. - - Note that specifically this initializes the cache of the row and column - weights on workers when `use_factors_weights_cache` is True. In this case, - if these weights are being calculated and reset after the object is created, - it is important to ensure this ops is run afterwards so the cache reflects - the correct values. - """ - return self._worker_init - - @property - def row_update_prep_gramian_op(self): - """Op to form the gramian before starting row updates. - - Must be run before initialize_row_update_op and should only be run by one - trainer (usually the chief) when doing distributed training. - - Returns: - Op to form the gramian. - """ - return self._row_update_prep_gramian - - @property - def col_update_prep_gramian_op(self): - """Op to form the gramian before starting col updates. - - Must be run before initialize_col_update_op and should only be run by one - trainer (usually the chief) when doing distributed training. - - Returns: - Op to form the gramian. - """ - return self._col_update_prep_gramian - - @property - def initialize_row_update_op(self): - """Op to initialize worker state before starting row updates.""" - return self._row_updates_init - - @property - def initialize_col_update_op(self): - """Op to initialize worker state before starting column updates.""" - return self._col_updates_init - - @staticmethod - def _get_sharding_func(size, num_shards): - """Create sharding function for scatter update.""" - - def func(ids): - if num_shards == 1: - return None, ids - else: - ids_per_shard = size // num_shards - extras = size % num_shards - assignments = math_ops.maximum(ids // (ids_per_shard + 1), - (ids - extras) // ids_per_shard) - new_ids = array_ops.where_v2(assignments < extras, - ids % (ids_per_shard + 1), - (ids - extras) % ids_per_shard) - return assignments, new_ids - - return func - - @classmethod - def scatter_update(cls, factor, indices, values, sharding_func, name=None): - """Helper function for doing sharded scatter update.""" - assert isinstance(factor, list) - if len(factor) == 1: - with ops.colocate_with(factor[0]): - # TODO(agarwal): assign instead of scatter update for full batch update. - return state_ops.scatter_update( - factor[0], indices, values, name=name).op - else: - num_shards = len(factor) - assignments, new_ids = sharding_func(indices) - assert assignments is not None - assignments = math_ops.cast(assignments, dtypes.int32) - sharded_ids = data_flow_ops.dynamic_partition(new_ids, assignments, - num_shards) - sharded_values = data_flow_ops.dynamic_partition(values, assignments, - num_shards) - updates = [] - for i in xrange(num_shards): - updates.append( - state_ops.scatter_update(factor[i], sharded_ids[i], sharded_values[ - i])) - return control_flow_ops.group(*updates, name=name) - - def update_row_factors(self, sp_input=None, transpose_input=False): - r"""Updates the row factors. - - Args: - sp_input: A SparseTensor representing a subset of rows of the full input - in any order. Please note that this SparseTensor must retain the - indexing as the original input. - transpose_input: If true, the input will be logically transposed and the - rows corresponding to the transposed input are updated. - - Returns: - A tuple consisting of the following elements: - new_values: New values for the row factors. - update_op: An op that assigns the newly computed values to the row - factors. - unregularized_loss: A tensor (scalar) that contains the normalized - minibatch loss corresponding to sp_input, without the regularization - term. If sp_input contains the rows \\({A_{i, :}, i \in I}\\), and the - input matrix A has n total rows, then the unregularized loss is: - \\(\|\sqrt W_I \odot (A_I - U_I V^T)\|_F^2 * n / |I|\\) - The total loss is unregularized_loss + regularization. - regularization: A tensor (scalar) that contains the normalized - regularization term for the minibatch loss corresponding to sp_input. - If sp_input contains the rows \\({A_{i, :}, i \in I}\\), and the input - matrix A has n total rows, then the regularization term is: - \\(\lambda \|U_I\|_F^2) * n / |I| + \lambda \|V\|_F^2\\). - sum_weights: The sum of the weights W_I corresponding to sp_input, - normalized by a factor of \\(n / |I|\\). The root weighted squared - error is: \sqrt(unregularized_loss / sum_weights). - """ - return self._process_input_helper( - True, sp_input=sp_input, transpose_input=transpose_input) - - def update_col_factors(self, sp_input=None, transpose_input=False): - r"""Updates the column factors. - - Args: - sp_input: A SparseTensor representing a subset of columns of the full - input. Please refer to comments for update_row_factors for - restrictions. - transpose_input: If true, the input will be logically transposed and the - columns corresponding to the transposed input are updated. - - Returns: - A tuple consisting of the following elements: - new_values: New values for the column factors. - update_op: An op that assigns the newly computed values to the column - factors. - unregularized_loss: A tensor (scalar) that contains the normalized - minibatch loss corresponding to sp_input, without the regularization - term. If sp_input contains the columns \\({A_{:, j}, j \in J}\\), and - the input matrix A has m total columns, then the unregularized loss is: - \\(\|\sqrt W_J \odot (A_J - U V_J^T)\|_F^2 * m / |I|\\) - The total loss is unregularized_loss + regularization. - regularization: A tensor (scalar) that contains the normalized - regularization term for the minibatch loss corresponding to sp_input. - If sp_input contains the columns \\({A_{:, j}, j \in J}\\), and the - input matrix A has m total columns, then the regularization term is: - \\(\lambda \|V_J\|_F^2) * m / |J| + \lambda \|U\|_F^2\\). - sum_weights: The sum of the weights W_J corresponding to sp_input, - normalized by a factor of \\(m / |J|\\). The root weighted squared - error is: \sqrt(unregularized_loss / sum_weights). - """ - return self._process_input_helper( - False, sp_input=sp_input, transpose_input=transpose_input) - - def project_row_factors(self, - sp_input=None, - transpose_input=False, - projection_weights=None): - """Projects the row factors. - - This computes the row embedding \\(u_i\\) for an observed row \\(a_i\\) by - solving one iteration of the update equations. - - Args: - sp_input: A SparseTensor representing a set of rows. Please note that the - column indices of this SparseTensor must match the model column feature - indexing while the row indices are ignored. The returned results will be - in the same ordering as the input rows. - transpose_input: If true, the input will be logically transposed and the - rows corresponding to the transposed input are projected. - projection_weights: The row weights to be used for the projection. If None - then 1.0 is used. This can be either a scaler or a rank-1 tensor with - the number of elements matching the number of rows to be projected. - Note that the column weights will be determined by the underlying WALS - model. - - Returns: - Projected row factors. - """ - if projection_weights is None: - projection_weights = 1 - return self._process_input_helper( - True, - sp_input=sp_input, - transpose_input=transpose_input, - row_weights=projection_weights)[0] - - def project_col_factors(self, - sp_input=None, - transpose_input=False, - projection_weights=None): - """Projects the column factors. - - This computes the column embedding \\(v_j\\) for an observed column - \\(a_j\\) by solving one iteration of the update equations. - - Args: - sp_input: A SparseTensor representing a set of columns. Please note that - the row indices of this SparseTensor must match the model row feature - indexing while the column indices are ignored. The returned results will - be in the same ordering as the input columns. - transpose_input: If true, the input will be logically transposed and the - columns corresponding to the transposed input are projected. - projection_weights: The column weights to be used for the projection. If - None then 1.0 is used. This can be either a scaler or a rank-1 tensor - with the number of elements matching the number of columns to be - projected. Note that the row weights will be determined by the - underlying WALS model. - - Returns: - Projected column factors. - """ - if projection_weights is None: - projection_weights = 1 - return self._process_input_helper( - False, - sp_input=sp_input, - transpose_input=transpose_input, - row_weights=projection_weights)[0] - - def _process_input_helper(self, - update_row_factors, - sp_input=None, - transpose_input=False, - row_weights=None): - """Creates the graph for processing a sparse slice of input. - - Args: - update_row_factors: if True, update or project the row_factors, else - update or project the column factors. - sp_input: Please refer to comments for update_row_factors, - update_col_factors, project_row_factors, and project_col_factors for - restrictions. - transpose_input: If True, the input is logically transposed and then the - corresponding rows/columns of the transposed input are updated. - row_weights: If not None, this is the row/column weights to be used for - the update or projection. If None, use the corresponding weights from - the model. Note that the feature (column/row) weights will be - determined by the model. When not None, it can either be a scalar or - a rank-1 tensor with the same number of elements as the number of rows - of columns to be updated/projected. - - Returns: - A tuple consisting of the following elements: - new_values: New values for the row/column factors. - update_op: An op that assigns the newly computed values to the row/column - factors. - unregularized_loss: A tensor (scalar) that contains the normalized - minibatch loss corresponding to sp_input, without the regularization - term. Add the regularization term below to yield the loss. - regularization: A tensor (scalar) that contains the normalized - regularization term for the minibatch loss corresponding to sp_input. - sum_weights: The sum of the weights corresponding to sp_input. This - can be used with unregularized loss to calculate the root weighted - squared error. - """ - assert isinstance(sp_input, sparse_tensor.SparseTensor) - - if update_row_factors: - left = self._row_factors - right_factors = self._col_factors_cache - row_wt = self._row_wt_cache - col_wt = self._col_wt_cache - total_rows = self._input_rows - total_cols = self._input_cols - sharding_func = WALSModel._get_sharding_func(self._input_rows, - self._num_row_shards) - gramian = self._col_gramian_cache - else: - left = self._col_factors - right_factors = self._row_factors_cache - row_wt = self._col_wt_cache - col_wt = self._row_wt_cache - total_rows = self._input_cols - total_cols = self._input_rows - sharding_func = WALSModel._get_sharding_func(self._input_cols, - self._num_col_shards) - gramian = self._row_gramian_cache - transpose_input = not transpose_input - - # Note that the row indices of sp_input are based on the original full input - # Here we reindex the rows and give them contiguous ids starting at 0. - # We use tf.unique to achieve this reindexing. Note that this is done so - # that the downstream kernel can assume that the input is "dense" along the - # row dimension. - row_ids, col_ids = array_ops.split( - value=sp_input.indices, num_or_size_splits=2, axis=1) - update_row_indices, all_row_ids = array_ops.unique(row_ids[:, 0]) - update_col_indices, all_col_ids = array_ops.unique(col_ids[:, 0]) - col_ids = array_ops.expand_dims(math_ops.cast(all_col_ids, dtypes.int64), 1) - row_ids = array_ops.expand_dims(math_ops.cast(all_row_ids, dtypes.int64), 1) - - if transpose_input: - update_indices = update_col_indices - row_shape = [ - math_ops.cast(array_ops.shape(update_row_indices)[0], dtypes.int64) - ] - gather_indices = update_row_indices - else: - update_indices = update_row_indices - row_shape = [ - math_ops.cast(array_ops.shape(update_col_indices)[0], dtypes.int64) - ] - gather_indices = update_col_indices - - num_rows = math_ops.cast(array_ops.shape(update_indices)[0], dtypes.int64) - col_shape = [num_rows] - right = embedding_ops.embedding_lookup( - right_factors, gather_indices, partition_strategy="div") - new_sp_indices = array_ops.concat([row_ids, col_ids], 1) - new_sp_shape = (array_ops.concat([row_shape, col_shape], 0) - if transpose_input else - array_ops.concat([col_shape, row_shape], 0)) - new_sp_input = sparse_tensor.SparseTensor( - indices=new_sp_indices, - values=sp_input.values, - dense_shape=new_sp_shape) - - # Compute lhs and rhs of the normal equations - total_lhs = (self._unobserved_weight * gramian) - if self._regularization_matrix is not None: - total_lhs += self._regularization_matrix - if self._row_weights is None: - # Special case of ALS. Use a much simpler update rule. - total_rhs = ( - self._unobserved_weight * sparse_ops.sparse_tensor_dense_matmul( - new_sp_input, right, adjoint_a=transpose_input)) - # TODO(rmlarsen): handle transposing in tf.linalg.solve instead of - # transposing explicitly. - # TODO(rmlarsen): multi-thread tf.matrix_solve. - new_left_values = array_ops.transpose( - linalg_ops.matrix_solve(total_lhs, array_ops.transpose(total_rhs))) - else: - if row_weights is None: - # TODO(yifanchen): Add special handling for single shard without using - # embedding_lookup and perform benchmarks for those cases. Same for - # col_weights lookup below. - row_weights_slice = embedding_ops.embedding_lookup( - row_wt, update_indices, partition_strategy="div") - else: - num_indices = array_ops.shape(update_indices)[0] - with ops.control_dependencies( - [check_ops.assert_less_equal(array_ops.rank(row_weights), 1)]): - row_weights_slice = control_flow_ops.cond( - math_ops.equal(array_ops.rank(row_weights), 0), - lambda: (array_ops.ones([num_indices]) * row_weights), - lambda: math_ops.cast(row_weights, dtypes.float32)) - - col_weights = embedding_ops.embedding_lookup( - col_wt, gather_indices, partition_strategy="div") - partial_lhs, total_rhs = ( - gen_factorization_ops.wals_compute_partial_lhs_and_rhs( - right, - col_weights, - self._unobserved_weight, - row_weights_slice, - new_sp_input.indices, - new_sp_input.values, - [], - num_rows, - transpose_input, - name="wals_compute_partial_lhs_rhs")) - total_lhs = array_ops.expand_dims(total_lhs, 0) + partial_lhs - total_rhs = array_ops.expand_dims(total_rhs, -1) - new_left_values = array_ops.squeeze( - linalg_ops.matrix_solve(total_lhs, total_rhs), [2]) - - update_op_name = "row_update" if update_row_factors else "col_update" - update_op = self.scatter_update( - left, - update_indices, - new_left_values, - sharding_func, - name=update_op_name) - - # Create the loss subgraph - loss_sp_input = (sparse_ops.sparse_transpose(new_sp_input) - if transpose_input else new_sp_input) - # sp_approx is the low rank estimate of the input matrix, formed by - # computing the product <\\(u_i, v_j\\)> for (i, j) in loss_sp_input.indices. - sp_approx_vals = gen_factorization_ops.masked_matmul( - new_left_values, - right, - loss_sp_input.indices, - transpose_a=False, - transpose_b=True) - sp_approx = sparse_tensor.SparseTensor( - loss_sp_input.indices, sp_approx_vals, loss_sp_input.dense_shape) - sp_approx_sq = math_ops.square(sp_approx) - sp_residual = sparse_ops.sparse_add(loss_sp_input, sp_approx * (-1)) - sp_residual_sq = math_ops.square(sp_residual) - row_wt_mat = (constant_op.constant(0.) - if self._row_weights is None else array_ops.expand_dims( - row_weights_slice, 1)) - col_wt_mat = (constant_op.constant(0.) - if self._col_weights is None else array_ops.expand_dims( - col_weights, 0)) - - # We return the normalized loss - partial_row_gramian = math_ops.matmul( - new_left_values, new_left_values, transpose_a=True) - normalization_factor = total_rows / math_ops.cast(num_rows, dtypes.float32) - - unregularized_loss = ( - self._unobserved_weight * ( # pyformat line break - sparse_ops.sparse_reduce_sum(sp_residual_sq) - # pyformat break - sparse_ops.sparse_reduce_sum(sp_approx_sq) + # pyformat break - math_ops.trace(math_ops.matmul(partial_row_gramian, gramian))) + - sparse_ops.sparse_reduce_sum(row_wt_mat * (sp_residual_sq * col_wt_mat)) - ) * normalization_factor - - if self._regularization is not None: - regularization = self._regularization * ( - math_ops.trace(partial_row_gramian) * normalization_factor + - math_ops.trace(gramian)) - else: - regularization = constant_op.constant(0.) - - sum_weights = self._unobserved_weight * math_ops.cast( - total_rows * total_cols, dtypes.float32) - if self._row_weights is not None and self._col_weights is not None: - ones = sparse_tensor.SparseTensor( - indices=loss_sp_input.indices, - values=array_ops.ones(array_ops.shape(loss_sp_input.values)), - dense_shape=loss_sp_input.dense_shape) - sum_weights += sparse_ops.sparse_reduce_sum(row_wt_mat * ( - ones * col_wt_mat)) * normalization_factor - - return (new_left_values, update_op, unregularized_loss, regularization, - sum_weights) diff --git a/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py b/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py deleted file mode 100644 index 6aa62fb82e8..00000000000 --- a/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py +++ /dev/null @@ -1,827 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for factorization_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib.factorization.python.ops import factorization_ops -from tensorflow.contrib.factorization.python.ops import factorization_ops_test_utils -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - -INPUT_MATRIX = factorization_ops_test_utils.INPUT_MATRIX -np_matrix_to_tf_sparse = factorization_ops_test_utils.np_matrix_to_tf_sparse - - -class WalsModelTest(test.TestCase): - - def sparse_input(self): - return np_matrix_to_tf_sparse(INPUT_MATRIX) - - def count_rows(self, sp_input): - return math_ops.cast( - array_ops.shape(array_ops.unique(sp_input.indices[:, 0])[0])[0], - dtypes.float32) - - def count_cols(self, sp_input): - return math_ops.cast( - array_ops.shape(array_ops.unique(sp_input.indices[:, 1])[0])[0], - dtypes.float32) - - def calculate_loss_from_wals_model(self, wals_model, sp_inputs): - current_rows = embedding_ops.embedding_lookup( - wals_model.row_factors, - math_ops.range(wals_model._input_rows), - partition_strategy="div") - current_cols = embedding_ops.embedding_lookup( - wals_model.col_factors, - math_ops.range(wals_model._input_cols), - partition_strategy="div") - row_wts = embedding_ops.embedding_lookup( - wals_model._row_weights, - math_ops.range(wals_model._input_rows), - partition_strategy="div") - col_wts = embedding_ops.embedding_lookup( - wals_model._col_weights, - math_ops.range(wals_model._input_cols), - partition_strategy="div") - return factorization_ops_test_utils.calculate_loss( - sp_inputs, current_rows, current_cols, wals_model._regularization, - wals_model._unobserved_weight, row_wts, col_wts) - - def setUp(self): - self.col_init = [ - # shard 0 - [ - [-0.36444709, -0.39077035, -0.32528427], # pyformat line break - [1.19056475, 0.07231052, 2.11834812], - [0.93468881, -0.71099287, 1.91826844] - ], - # shard 1 - [[1.18160152, 1.52490723, -0.50015002], - [1.82574749, -0.57515913, -1.32810032]], - # shard 2 - [[-0.15515432, -0.84675711, 0.13097958], - [-0.9246484, 0.69117504, 1.2036494]] - ] - - self.row_wts = [[0.1, 0.2, 0.3], [0.4, 0.5]] - self.col_wts = [[0.1, 0.2, 0.3], [0.4, 0.5], [0.6, 0.7]] - - # Values of factor shards after running one iteration of row and column - # updates. - self._row_factors_0 = [ - [0.097689, -0.219293, -0.020780], # pyformat line break - [0.50842, 0.64626, 0.22364], - [0.401159, -0.046558, -0.192854] - ] - self._row_factors_1 = [[1.20597, -0.48025, 0.35582], - [1.5564, 1.2528, 1.0528]] - self._col_factors_0 = [ - [2.4725, -1.2950, -1.9980], # pyformat line break - [0.44625, 1.50771, 1.27118], - [1.39801, -2.10134, 0.73572] - ] - self._col_factors_1 = [[3.36509, -0.66595, -3.51208], - [0.57191, 1.59407, 1.33020]] - self._col_factors_2 = [[3.3459, -1.3341, -3.3008], - [0.57366, 1.83729, 1.26798]] - - def _run_test_sum_weights(self, test_rows): - # test_rows: True to test row weights, False to test column weights. - - num_rows = 5 - num_cols = 5 - unobserved_weight = 0.1 - row_weights = [[8., 18., 28., 38., 48.]] - col_weights = [[90., 91., 92., 93., 94.]] - sparse_indices = [[0, 1], [2, 3], [4, 1]] - sparse_values = [666., 777., 888.] - - unobserved = unobserved_weight * num_rows * num_cols - observed = 8. * 91. + 28. * 93. + 48. * 91. - # sparse_indices has three unique rows and two unique columns - observed *= num_rows / 3. if test_rows else num_cols / 2. - want_weight_sum = unobserved + observed - - with ops.Graph().as_default(), self.cached_session() as sess: - wals_model = factorization_ops.WALSModel( - input_rows=num_rows, - input_cols=num_cols, - n_components=5, - unobserved_weight=unobserved_weight, - row_weights=row_weights, - col_weights=col_weights, - use_factors_weights_cache=False) - - wals_model.initialize_op.run() - wals_model.worker_init.run() - - update_factors = (wals_model.update_row_factors - if test_rows else wals_model.update_col_factors) - - (_, _, _, _, sum_weights) = update_factors( - sp_input=sparse_tensor.SparseTensor( - indices=sparse_indices, - values=sparse_values, - dense_shape=[num_rows, num_cols]), - transpose_input=False) - - got_weight_sum = sess.run(sum_weights) - - self.assertNear( - got_weight_sum, - want_weight_sum, - err=.001, - msg="got weight sum [{}], want weight sum [{}]".format( - got_weight_sum, want_weight_sum)) - - def _run_test_process_input(self, - use_factors_weights_cache, - compute_loss=False): - with ops.Graph().as_default(), self.cached_session() as sess: - self._wals_inputs = self.sparse_input() - sp_feeder = array_ops.sparse_placeholder(dtypes.float32) - num_rows = 5 - num_cols = 7 - factor_dim = 3 - wals_model = factorization_ops.WALSModel( - num_rows, - num_cols, - factor_dim, - num_row_shards=2, - num_col_shards=3, - regularization=0.01, - unobserved_weight=0.1, - col_init=self.col_init, - row_weights=self.row_wts, - col_weights=self.col_wts, - use_factors_weights_cache=use_factors_weights_cache) - - wals_model.initialize_op.run() - wals_model.worker_init.run() - - # Split input into multiple sparse tensors with scattered rows. Note that - # this split can be different than the factor sharding and the inputs can - # consist of non-consecutive rows. Each row needs to include all non-zero - # elements in that row. - sp_r0 = np_matrix_to_tf_sparse(INPUT_MATRIX, [0, 2]).eval() - sp_r1 = np_matrix_to_tf_sparse(INPUT_MATRIX, [1, 4], shuffle=True).eval() - sp_r2 = np_matrix_to_tf_sparse(INPUT_MATRIX, [3], shuffle=True).eval() - input_scattered_rows = [sp_r0, sp_r1, sp_r2] - - # Test updating row factors. - # Here we feed in scattered rows of the input. - wals_model.row_update_prep_gramian_op.run() - wals_model.initialize_row_update_op.run() - (_, process_input_op, unregularized_loss, regularization, - _) = wals_model.update_row_factors( - sp_input=sp_feeder, transpose_input=False) - factor_loss = unregularized_loss + regularization - for inp in input_scattered_rows: - feed_dict = {sp_feeder: inp} - process_input_op.run(feed_dict=feed_dict) - row_factors = [x.eval() for x in wals_model.row_factors] - - self.assertAllClose(row_factors[0], self._row_factors_0, atol=1e-3) - self.assertAllClose(row_factors[1], self._row_factors_1, atol=1e-3) - - # Test row projection. - # Using the specified projection weights for the 2 row feature vectors. - # This is expected to reproduce the same row factors in the model as the - # weights and feature vectors are identical to that used in model - # training. - projected_rows = wals_model.project_row_factors( - sp_input=sp_feeder, - transpose_input=False, - projection_weights=[0.2, 0.5]) - # Don't specify the projection weight, so 1.0 will be used. The feature - # weights will be those specified in model. - projected_rows_no_weights = wals_model.project_row_factors( - sp_input=sp_feeder, transpose_input=False) - feed_dict = { - sp_feeder: - np_matrix_to_tf_sparse(INPUT_MATRIX, [1, 4], shuffle=False) - .eval() - } - self.assertAllClose( - projected_rows.eval(feed_dict=feed_dict), - [self._row_factors_0[1], self._row_factors_1[1]], - atol=1e-3) - self.assertAllClose( - projected_rows_no_weights.eval(feed_dict=feed_dict), - [[0.569082, 0.715088, 0.31777], [1.915879, 1.992677, 1.109057]], - atol=1e-3) - - if compute_loss: - # Test loss computation after the row update - loss = sum( - sess.run( - factor_loss * self.count_rows(inp) / num_rows, - feed_dict={sp_feeder: inp}) for inp in input_scattered_rows) - true_loss = self.calculate_loss_from_wals_model(wals_model, - self._wals_inputs) - self.assertNear( - loss, - true_loss, - err=.001, - msg="After row update, computed loss [{}] does not match" - " true loss [{}]".format(loss, true_loss)) - - # Split input into multiple sparse tensors with scattered columns. Note - # that here the elements in the sparse tensors are not ordered and also - # do not need to consist of consecutive columns. However, each column - # needs to include all non-zero elements in that column. - sp_c0 = np_matrix_to_tf_sparse(INPUT_MATRIX, col_slices=[2, 0]).eval() - sp_c1 = np_matrix_to_tf_sparse( - INPUT_MATRIX, col_slices=[5, 3, 1], shuffle=True).eval() - sp_c2 = np_matrix_to_tf_sparse(INPUT_MATRIX, col_slices=[4, 6]).eval() - sp_c3 = np_matrix_to_tf_sparse( - INPUT_MATRIX, col_slices=[3, 6], shuffle=True).eval() - - input_scattered_cols = [sp_c0, sp_c1, sp_c2, sp_c3] - input_scattered_cols_non_duplicate = [sp_c0, sp_c1, sp_c2] - - # Test updating column factors. - # Here we feed in scattered columns of the input. - wals_model.col_update_prep_gramian_op.run() - wals_model.initialize_col_update_op.run() - (_, process_input_op, unregularized_loss, regularization, - _) = wals_model.update_col_factors( - sp_input=sp_feeder, transpose_input=False) - factor_loss = unregularized_loss + regularization - for inp in input_scattered_cols: - feed_dict = {sp_feeder: inp} - process_input_op.run(feed_dict=feed_dict) - col_factors = [x.eval() for x in wals_model.col_factors] - - self.assertAllClose(col_factors[0], self._col_factors_0, atol=1e-3) - self.assertAllClose(col_factors[1], self._col_factors_1, atol=1e-3) - self.assertAllClose(col_factors[2], self._col_factors_2, atol=1e-3) - - # Test column projection. - # Using the specified projection weights for the 3 column feature vectors. - # This is expected to reproduce the same column factors in the model as - # the weights and feature vectors are identical to that used in model - # training. - projected_cols = wals_model.project_col_factors( - sp_input=sp_feeder, - transpose_input=False, - projection_weights=[0.6, 0.4, 0.2]) - # Don't specify the projection weight, so 1.0 will be used. The feature - # weights will be those specified in model. - projected_cols_no_weights = wals_model.project_col_factors( - sp_input=sp_feeder, transpose_input=False) - feed_dict = { - sp_feeder: - np_matrix_to_tf_sparse( - INPUT_MATRIX, col_slices=[5, 3, 1], shuffle=False).eval() - } - self.assertAllClose( - projected_cols.eval(feed_dict=feed_dict), [ - self._col_factors_2[0], self._col_factors_1[0], - self._col_factors_0[1] - ], - atol=1e-3) - self.assertAllClose( - projected_cols_no_weights.eval(feed_dict=feed_dict), - [[3.471045, -1.250835, -3.598917], [3.585139, -0.487476, -3.852232], - [0.346433, 1.360644, 1.677121]], - atol=1e-3) - - if compute_loss: - # Test loss computation after the column update. - loss = sum( - sess.run( - factor_loss * self.count_cols(inp) / num_cols, - feed_dict={sp_feeder: inp}) - for inp in input_scattered_cols_non_duplicate) - true_loss = self.calculate_loss_from_wals_model(wals_model, - self._wals_inputs) - self.assertNear( - loss, - true_loss, - err=.001, - msg="After col update, computed loss [{}] does not match" - " true loss [{}]".format(loss, true_loss)) - - def _run_test_process_input_transposed(self, - use_factors_weights_cache, - compute_loss=False): - with ops.Graph().as_default(), self.cached_session() as sess: - self._wals_inputs = self.sparse_input() - sp_feeder = array_ops.sparse_placeholder(dtypes.float32) - num_rows = 5 - num_cols = 7 - factor_dim = 3 - wals_model = factorization_ops.WALSModel( - num_rows, - num_cols, - factor_dim, - num_row_shards=2, - num_col_shards=3, - regularization=0.01, - unobserved_weight=0.1, - col_init=self.col_init, - row_weights=self.row_wts, - col_weights=self.col_wts, - use_factors_weights_cache=use_factors_weights_cache) - - wals_model.initialize_op.run() - wals_model.worker_init.run() - - # Split input into multiple SparseTensors with scattered rows. - # Here the inputs are transposed. But the same constraints as described in - # the previous non-transposed test case apply to these inputs (before they - # are transposed). - sp_r0_t = np_matrix_to_tf_sparse( - INPUT_MATRIX, [0, 3], transpose=True).eval() - sp_r1_t = np_matrix_to_tf_sparse( - INPUT_MATRIX, [4, 1], shuffle=True, transpose=True).eval() - sp_r2_t = np_matrix_to_tf_sparse(INPUT_MATRIX, [2], transpose=True).eval() - sp_r3_t = sp_r1_t - input_scattered_rows = [sp_r0_t, sp_r1_t, sp_r2_t, sp_r3_t] - input_scattered_rows_non_duplicate = [sp_r0_t, sp_r1_t, sp_r2_t] - # Test updating row factors. - # Here we feed in scattered rows of the input. - # Note that the needed suffix of placeholder are in the order of test - # case name lexicographical order and then in the line order of where - # they appear. - wals_model.row_update_prep_gramian_op.run() - wals_model.initialize_row_update_op.run() - (_, process_input_op, unregularized_loss, regularization, - _) = wals_model.update_row_factors( - sp_input=sp_feeder, transpose_input=True) - factor_loss = unregularized_loss + regularization - for inp in input_scattered_rows: - feed_dict = {sp_feeder: inp} - process_input_op.run(feed_dict=feed_dict) - row_factors = [x.eval() for x in wals_model.row_factors] - - self.assertAllClose(row_factors[0], self._row_factors_0, atol=1e-3) - self.assertAllClose(row_factors[1], self._row_factors_1, atol=1e-3) - - # Test row projection. - # Using the specified projection weights for the 2 row feature vectors. - # This is expected to reproduce the same row factors in the model as the - # weights and feature vectors are identical to that used in model - # training. - projected_rows = wals_model.project_row_factors( - sp_input=sp_feeder, - transpose_input=True, - projection_weights=[0.5, 0.2]) - # Don't specify the projection weight, so 1.0 will be used. The feature - # weights will be those specified in model. - projected_rows_no_weights = wals_model.project_row_factors( - sp_input=sp_feeder, transpose_input=True) - feed_dict = { - sp_feeder: - np_matrix_to_tf_sparse( - INPUT_MATRIX, [4, 1], shuffle=False, transpose=True).eval() - } - self.assertAllClose( - projected_rows.eval(feed_dict=feed_dict), - [self._row_factors_1[1], self._row_factors_0[1]], - atol=1e-3) - self.assertAllClose( - projected_rows_no_weights.eval(feed_dict=feed_dict), - [[1.915879, 1.992677, 1.109057], [0.569082, 0.715088, 0.31777]], - atol=1e-3) - - if compute_loss: - # Test loss computation after the row update - loss = sum( - sess.run( - factor_loss * self.count_cols(inp) / num_rows, - feed_dict={sp_feeder: inp}) - for inp in input_scattered_rows_non_duplicate) - true_loss = self.calculate_loss_from_wals_model(wals_model, - self._wals_inputs) - self.assertNear( - loss, - true_loss, - err=.001, - msg="After row update, computed loss [{}] does not match" - " true loss [{}]".format(loss, true_loss)) - - # Split input into multiple SparseTensors with scattered columns. - # Here the inputs are transposed. But the same constraints as described in - # the previous non-transposed test case apply to these inputs (before they - # are transposed). - sp_c0_t = np_matrix_to_tf_sparse( - INPUT_MATRIX, col_slices=[0, 1], transpose=True).eval() - sp_c1_t = np_matrix_to_tf_sparse( - INPUT_MATRIX, col_slices=[4, 2], transpose=True).eval() - sp_c2_t = np_matrix_to_tf_sparse( - INPUT_MATRIX, col_slices=[5], transpose=True, shuffle=True).eval() - sp_c3_t = np_matrix_to_tf_sparse( - INPUT_MATRIX, col_slices=[3, 6], transpose=True).eval() - - sp_c4_t = sp_c2_t - input_scattered_cols = [sp_c0_t, sp_c1_t, sp_c2_t, sp_c3_t, sp_c4_t] - input_scattered_cols_non_duplicate = [sp_c0_t, sp_c1_t, sp_c2_t, sp_c3_t] - - # Test updating column factors. - # Here we feed in scattered columns of the input. - wals_model.col_update_prep_gramian_op.run() - wals_model.initialize_col_update_op.run() - (_, process_input_op, unregularized_loss, regularization, - _) = wals_model.update_col_factors( - sp_input=sp_feeder, transpose_input=True) - factor_loss = unregularized_loss + regularization - for inp in input_scattered_cols: - feed_dict = {sp_feeder: inp} - process_input_op.run(feed_dict=feed_dict) - col_factors = [x.eval() for x in wals_model.col_factors] - - self.assertAllClose(col_factors[0], self._col_factors_0, atol=1e-3) - self.assertAllClose(col_factors[1], self._col_factors_1, atol=1e-3) - self.assertAllClose(col_factors[2], self._col_factors_2, atol=1e-3) - - # Test column projection. - # Using the specified projection weights for the 2 column feature vectors. - # This is expected to reproduce the same column factors in the model as - # the weights and feature vectors are identical to that used in model - # training. - projected_cols = wals_model.project_col_factors( - sp_input=sp_feeder, - transpose_input=True, - projection_weights=[0.4, 0.7]) - # Don't specify the projection weight, so 1.0 will be used. The feature - # weights will be those specified in model. - projected_cols_no_weights = wals_model.project_col_factors( - sp_input=sp_feeder, transpose_input=True) - feed_dict = {sp_feeder: sp_c3_t} - self.assertAllClose( - projected_cols.eval(feed_dict=feed_dict), - [self._col_factors_1[0], self._col_factors_2[1]], - atol=1e-3) - self.assertAllClose( - projected_cols_no_weights.eval(feed_dict=feed_dict), - [[3.585139, -0.487476, -3.852232], [0.557937, 1.813907, 1.331171]], - atol=1e-3) - if compute_loss: - # Test loss computation after the col update - loss = sum( - sess.run( - factor_loss * self.count_rows(inp) / num_cols, - feed_dict={sp_feeder: inp}) - for inp in input_scattered_cols_non_duplicate) - true_loss = self.calculate_loss_from_wals_model(wals_model, - self._wals_inputs) - self.assertNear( - loss, - true_loss, - err=.001, - msg="After col update, computed loss [{}] does not match" - " true loss [{}]".format(loss, true_loss)) - - # Note that when row_weights and col_weights are 0, WALS gives identical - # results as ALS (Alternating Least Squares). However our implementation does - # not handle the case of zero weights differently. Instead, when row_weights - # and col_weights are set to None, we interpret that as the ALS case, and - # trigger the more efficient ALS updates. - # Here we test that those two give identical results. - def _run_test_als(self, use_factors_weights_cache): - with ops.Graph().as_default(), self.cached_session(): - self._wals_inputs = self.sparse_input() - col_init = np.random.rand(7, 3) - als_model = factorization_ops.WALSModel( - 5, - 7, - 3, - col_init=col_init, - row_weights=None, - col_weights=None, - use_factors_weights_cache=use_factors_weights_cache) - - als_model.initialize_op.run() - als_model.worker_init.run() - als_model.row_update_prep_gramian_op.run() - als_model.initialize_row_update_op.run() - process_input_op = als_model.update_row_factors(self._wals_inputs)[1] - process_input_op.run() - row_factors1 = [x.eval() for x in als_model.row_factors] - # Testing row projection. Projection weight doesn't matter in this case - # since the model is ALS special case. - als_projected_row_factors1 = als_model.project_row_factors( - self._wals_inputs).eval() - - wals_model = factorization_ops.WALSModel( - 5, - 7, - 3, - col_init=col_init, - row_weights=0, - col_weights=0, - use_factors_weights_cache=use_factors_weights_cache) - wals_model.initialize_op.run() - wals_model.worker_init.run() - wals_model.row_update_prep_gramian_op.run() - wals_model.initialize_row_update_op.run() - process_input_op = wals_model.update_row_factors(self._wals_inputs)[1] - process_input_op.run() - row_factors2 = [x.eval() for x in wals_model.row_factors] - - for r1, r2 in zip(row_factors1, row_factors2): - self.assertAllClose(r1, r2, atol=1e-3) - self.assertAllClose( - als_projected_row_factors1, - [row for shard in row_factors2 for row in shard], - atol=1e-3) - - # Here we test partial column updates. - sp_c = np_matrix_to_tf_sparse( - INPUT_MATRIX, col_slices=[2, 0], shuffle=True).eval() - - sp_feeder = array_ops.sparse_placeholder(dtypes.float32) - feed_dict = {sp_feeder: sp_c} - als_model.col_update_prep_gramian_op.run() - als_model.initialize_col_update_op.run() - process_input_op = als_model.update_col_factors(sp_input=sp_feeder)[1] - process_input_op.run(feed_dict=feed_dict) - col_factors1 = [x.eval() for x in als_model.col_factors] - # Testing column projection. Projection weight doesn't matter in this case - # since the model is ALS special case. - als_projected_col_factors1 = als_model.project_col_factors( - np_matrix_to_tf_sparse( - INPUT_MATRIX, col_slices=[2, 0], shuffle=False)).eval() - - feed_dict = {sp_feeder: sp_c} - wals_model.col_update_prep_gramian_op.run() - wals_model.initialize_col_update_op.run() - process_input_op = wals_model.update_col_factors(sp_input=sp_feeder)[1] - process_input_op.run(feed_dict=feed_dict) - col_factors2 = [x.eval() for x in wals_model.col_factors] - - for c1, c2 in zip(col_factors1, col_factors2): - self.assertAllClose(c1, c2, rtol=5e-3, atol=1e-2) - self.assertAllClose( - als_projected_col_factors1, [col_factors2[0][2], col_factors2[0][0]], - atol=1e-2) - - def _run_test_als_transposed(self, use_factors_weights_cache): - with ops.Graph().as_default(), self.cached_session(): - self._wals_inputs = self.sparse_input() - col_init = np.random.rand(7, 3) - als_model = factorization_ops.WALSModel( - 5, - 7, - 3, - col_init=col_init, - row_weights=None, - col_weights=None, - use_factors_weights_cache=use_factors_weights_cache) - - als_model.initialize_op.run() - als_model.worker_init.run() - - wals_model = factorization_ops.WALSModel( - 5, - 7, - 3, - col_init=col_init, - row_weights=[0] * 5, - col_weights=[0] * 7, - use_factors_weights_cache=use_factors_weights_cache) - wals_model.initialize_op.run() - wals_model.worker_init.run() - sp_feeder = array_ops.sparse_placeholder(dtypes.float32) - # Here test partial row update with identical inputs but with transposed - # input for als. - sp_r_t = np_matrix_to_tf_sparse( - INPUT_MATRIX, [3, 1], transpose=True).eval() - sp_r = np_matrix_to_tf_sparse(INPUT_MATRIX, [3, 1]).eval() - - feed_dict = {sp_feeder: sp_r_t} - als_model.row_update_prep_gramian_op.run() - als_model.initialize_row_update_op.run() - process_input_op = als_model.update_row_factors( - sp_input=sp_feeder, transpose_input=True)[1] - process_input_op.run(feed_dict=feed_dict) - # Only updated row 1 and row 3, so only compare these rows since others - # have randomly initialized values. - row_factors1 = [ - als_model.row_factors[0].eval()[1], als_model.row_factors[0].eval()[3] - ] - # Testing row projection. Projection weight doesn't matter in this case - # since the model is ALS special case. Note that the ordering of the - # returned results will be preserved as the input feature vectors - # ordering. - als_projected_row_factors1 = als_model.project_row_factors( - sp_input=sp_feeder, transpose_input=True).eval(feed_dict=feed_dict) - - feed_dict = {sp_feeder: sp_r} - wals_model.row_update_prep_gramian_op.run() - wals_model.initialize_row_update_op.run() - process_input_op = wals_model.update_row_factors(sp_input=sp_feeder)[1] - process_input_op.run(feed_dict=feed_dict) - # Only updated row 1 and row 3, so only compare these rows since others - # have randomly initialized values. - row_factors2 = [ - wals_model.row_factors[0].eval()[1], - wals_model.row_factors[0].eval()[3] - ] - for r1, r2 in zip(row_factors1, row_factors2): - self.assertAllClose(r1, r2, atol=1e-3) - # Note that the ordering of the returned projection results is preserved - # as the input feature vectors ordering. - self.assertAllClose( - als_projected_row_factors1, [row_factors2[1], row_factors2[0]], - atol=1e-3) - - def simple_train(self, model, inp, num_iterations): - """Helper function to train model on inp for num_iterations.""" - row_update_op = model.update_row_factors(sp_input=inp)[1] - col_update_op = model.update_col_factors(sp_input=inp)[1] - - model.initialize_op.run() - model.worker_init.run() - for _ in xrange(num_iterations): - model.row_update_prep_gramian_op.run() - model.initialize_row_update_op.run() - row_update_op.run() - model.col_update_prep_gramian_op.run() - model.initialize_col_update_op.run() - col_update_op.run() - - # Trains an ALS model for a low-rank matrix and make sure the product of - # factors is close to the original input. - def _run_test_train_full_low_rank_als(self, use_factors_weights_cache): - rows = 15 - cols = 11 - dims = 3 - with ops.Graph().as_default(), self.cached_session(): - data = np.dot(np.random.rand(rows, 3), np.random.rand( - 3, cols)).astype(np.float32) / 3.0 - indices = [[i, j] for i in xrange(rows) for j in xrange(cols)] - values = data.reshape(-1) - inp = sparse_tensor.SparseTensor(indices, values, [rows, cols]) - model = factorization_ops.WALSModel( - rows, - cols, - dims, - regularization=1e-5, - row_weights=None, - col_weights=None, - use_factors_weights_cache=use_factors_weights_cache) - self.simple_train(model, inp, 25) - row_factor = model.row_factors[0].eval() - col_factor = model.col_factors[0].eval() - self.assertAllClose( - data, - np.dot(row_factor, np.transpose(col_factor)), - rtol=0.01, - atol=0.01) - - # Trains a WALS model for a low-rank matrix and make sure the product of - # factors is close to the original input. - def _run_test_train_full_low_rank_wals(self, use_factors_weights_cache): - rows = 15 - cols = 11 - dims = 3 - - with ops.Graph().as_default(), self.cached_session(): - data = np.dot(np.random.rand(rows, 3), np.random.rand( - 3, cols)).astype(np.float32) / 3.0 - indices = [[i, j] for i in xrange(rows) for j in xrange(cols)] - values = data.reshape(-1) - inp = sparse_tensor.SparseTensor(indices, values, [rows, cols]) - model = factorization_ops.WALSModel( - rows, - cols, - dims, - regularization=1e-5, - row_weights=0, - col_weights=[0] * cols, - use_factors_weights_cache=use_factors_weights_cache) - self.simple_train(model, inp, 25) - row_factor = model.row_factors[0].eval() - col_factor = model.col_factors[0].eval() - self.assertAllClose( - data, - np.dot(row_factor, np.transpose(col_factor)), - rtol=0.01, - atol=0.01) - - # Trains a WALS model for a partially observed low-rank matrix and makes - # sure the product of factors is reasonably close to the original input. - def _run_test_train_matrix_completion_wals(self, use_factors_weights_cache): - rows = 11 - cols = 9 - dims = 4 - - def keep_index(x): - return not (x[0] + x[1]) % 4 - - with ops.Graph().as_default(), self.cached_session(): - row_wts = 0.1 + np.random.rand(rows) - col_wts = 0.1 + np.random.rand(cols) - data = np.dot(np.random.rand(rows, 3), np.random.rand( - 3, cols)).astype(np.float32) / 3.0 - indices = np.array( - list( - filter(keep_index, - [[i, j] for i in xrange(rows) for j in xrange(cols)]))) - values = data[indices[:, 0], indices[:, 1]] - inp = sparse_tensor.SparseTensor(indices, values, [rows, cols]) - model = factorization_ops.WALSModel( - rows, - cols, - dims, - unobserved_weight=0.01, - regularization=0.001, - row_weights=row_wts, - col_weights=col_wts, - use_factors_weights_cache=use_factors_weights_cache) - self.simple_train(model, inp, 25) - row_factor = model.row_factors[0].eval() - col_factor = model.col_factors[0].eval() - out = np.dot(row_factor, np.transpose(col_factor)) - for i in xrange(rows): - for j in xrange(cols): - if keep_index([i, j]): - self.assertNear( - data[i][j], out[i][j], err=0.4, msg="%d, %d" % (i, j)) - else: - self.assertNear(0, out[i][j], err=0.5, msg="%d, %d" % (i, j)) - - def test_process_input_with_cache(self): - self._run_test_process_input(True) - - def test_process_input_without_cache(self): - self._run_test_process_input(False) - - def test_process_input_transposed_with_cache(self): - self._run_test_process_input_transposed(True) - - def test_process_input_transposed_without_cache(self): - self._run_test_process_input_transposed(False) - - def test_als_with_cache(self): - self._run_test_als(True) - - def test_als_without_cache(self): - self._run_test_als(False) - - def test_als_transposed_with_cache(self): - self._run_test_als_transposed(True) - - def test_als_transposed_without_cache(self): - self._run_test_als_transposed(False) - - def test_train_full_low_rank_wals_with_cache(self): - self._run_test_train_full_low_rank_wals(True) - - def test_train_full_low_rank_wals_without_cache(self): - self._run_test_train_full_low_rank_wals(False) - - def test_train_matrix_completion_wals_with_cache(self): - self._run_test_train_matrix_completion_wals(True) - - def test_train_matrix_completion_wals_without_cache(self): - self._run_test_train_matrix_completion_wals(False) - - def test_loss_transposed_with_cache(self): - self._run_test_process_input_transposed(True, compute_loss=True) - - def test_loss_transposed_without_cache(self): - self._run_test_process_input_transposed(False, compute_loss=True) - - def test_loss_with_cache(self): - self._run_test_process_input(True, compute_loss=True) - - def test_loss_without_cache(self): - self._run_test_process_input(False, compute_loss=True) - - def test_sum_row_weights(self): - self._run_test_sum_weights(True) - - def test_sum_col_weights(self): - self._run_test_sum_weights(False) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/factorization/python/ops/factorization_ops_test_utils.py b/tensorflow/contrib/factorization/python/ops/factorization_ops_test_utils.py deleted file mode 100644 index ead9474805c..00000000000 --- a/tensorflow/contrib/factorization/python/ops/factorization_ops_test_utils.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Test utils for factorization_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random - -import numpy as np - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import sparse_ops - - -INPUT_MATRIX = np.array( - [[0.1, 0.0, 0.2, 0.0, 0.4, 0.5, 0.0], - [0.0, 1.1, 0.0, 1.3, 1.4, 0.0, 1.6], - [2.0, 0.0, 0.0, 2.3, 0.0, 2.5, 0.0], - [3.0, 0.0, 3.2, 3.3, 0.0, 3.5, 0.0], - [0.0, 4.1, 0.0, 0.0, 4.4, 0.0, 4.6]]).astype(np.float32) - - -def remove_empty_rows_columns(np_matrix): - """Simple util to remove empty rows and columns of a matrix. - - Args: - np_matrix: A numpy array. - Returns: - A tuple consisting of: - mat: A numpy matrix obtained by removing empty rows and columns from - np_matrix. - nz_row_ids: A numpy array of the ids of non-empty rows, such that - nz_row_ids[i] is the old row index corresponding to new index i. - nz_col_ids: A numpy array of the ids of non-empty columns, such that - nz_col_ids[j] is the old column index corresponding to new index j. - """ - nz_row_ids = np.where(np.sum(np_matrix, axis=1) != 0)[0] - nz_col_ids = np.where(np.sum(np_matrix, axis=0) != 0)[0] - mat = np_matrix[np.ix_(nz_row_ids, nz_col_ids)] - return mat, nz_row_ids, nz_col_ids - - -def np_matrix_to_tf_sparse(np_matrix, - row_slices=None, - col_slices=None, - transpose=False, - shuffle=False): - """Simple util to slice non-zero np matrix elements as tf.SparseTensor.""" - indices = np.nonzero(np_matrix) - - # Only allow slices of whole rows or whole columns. - assert not (row_slices is not None and col_slices is not None) - - if row_slices is not None: - selected_ind = np.concatenate( - [np.where(indices[0] == r)[0] for r in row_slices], 0) - indices = (indices[0][selected_ind], indices[1][selected_ind]) - - if col_slices is not None: - selected_ind = np.concatenate( - [np.where(indices[1] == c)[0] for c in col_slices], 0) - indices = (indices[0][selected_ind], indices[1][selected_ind]) - - if shuffle: - shuffled_ind = [x for x in range(len(indices[0]))] - random.shuffle(shuffled_ind) - indices = (indices[0][shuffled_ind], indices[1][shuffled_ind]) - - ind = (np.concatenate((np.expand_dims(indices[1], 1), - np.expand_dims(indices[0], 1)), 1).astype(np.int64) if - transpose else np.concatenate((np.expand_dims(indices[0], 1), - np.expand_dims(indices[1], 1)), - 1).astype(np.int64)) - val = np_matrix[indices].astype(np.float32) - shape = (np.array([max(indices[1]) + 1, max(indices[0]) + 1]).astype(np.int64) - if transpose else np.array( - [max(indices[0]) + 1, max(indices[1]) + 1]).astype(np.int64)) - return sparse_tensor.SparseTensor(ind, val, shape) - - -def calculate_loss(input_mat, row_factors, col_factors, regularization=None, - w0=1., row_weights=None, col_weights=None): - """Calculates the loss of a given factorization. - - Using a non distributed method, different than the one implemented in the - WALS model. The weight of an observed entry (i, j) (i.e. such that - input_mat[i, j] is non zero) is (w0 + row_weights[i]col_weights[j]). - - Args: - input_mat: The input matrix, a SparseTensor of rank 2. - row_factors: The row factors, a dense Tensor of rank 2. - col_factors: The col factors, a dense Tensor of rank 2. - regularization: the regularization coefficient, a scalar. - w0: the weight of unobserved entries. A scalar. - row_weights: A dense tensor of rank 1. - col_weights: A dense tensor of rank 1. - - Returns: - The total loss. - """ - wr = (array_ops.expand_dims(row_weights, 1) if row_weights is not None - else constant_op.constant(1.)) - wc = (array_ops.expand_dims(col_weights, 0) if col_weights is not None - else constant_op.constant(1.)) - reg = (regularization if regularization is not None - else constant_op.constant(0.)) - - row_indices, col_indices = array_ops.split(input_mat.indices, - axis=1, - num_or_size_splits=2) - gathered_row_factors = array_ops.gather(row_factors, row_indices) - gathered_col_factors = array_ops.gather(col_factors, col_indices) - sp_approx_vals = array_ops.squeeze(math_ops.matmul( - gathered_row_factors, gathered_col_factors, adjoint_b=True)) - sp_approx = sparse_tensor.SparseTensor( - indices=input_mat.indices, - values=sp_approx_vals, - dense_shape=input_mat.dense_shape) - - sp_approx_sq = math_ops.square(sp_approx) - row_norm = math_ops.reduce_sum(math_ops.square(row_factors)) - col_norm = math_ops.reduce_sum(math_ops.square(col_factors)) - row_col_norm = math_ops.reduce_sum(math_ops.square(math_ops.matmul( - row_factors, col_factors, transpose_b=True))) - - resid = sparse_ops.sparse_add(input_mat, sp_approx * (-1)) - resid_sq = math_ops.square(resid) - loss = w0 * ( - sparse_ops.sparse_reduce_sum(resid_sq) - - sparse_ops.sparse_reduce_sum(sp_approx_sq) - ) - loss += (sparse_ops.sparse_reduce_sum(wr * (resid_sq * wc)) + - w0 * row_col_norm + reg * (row_norm + col_norm)) - return loss.eval() diff --git a/tensorflow/contrib/factorization/python/ops/gmm.py b/tensorflow/contrib/factorization/python/ops/gmm.py deleted file mode 100644 index b2dfe48b2db..00000000000 --- a/tensorflow/contrib/factorization/python/ops/gmm.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implementation of Gaussian mixture model (GMM) clustering using tf.Learn.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time -import numpy as np - -from tensorflow.contrib import framework -from tensorflow.contrib.factorization.python.ops import gmm_ops -from tensorflow.contrib.framework.python.framework import checkpoint_utils -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import logging_ops as logging -from tensorflow.python.ops import state_ops -from tensorflow.python.ops.control_flow_ops import with_dependencies -from tensorflow.python.training import session_run_hook -from tensorflow.python.training import training_util - - -def _streaming_sum(scalar_tensor): - """Create a sum metric and update op.""" - sum_metric = framework.local_variable(constant_op.constant(0.0)) - sum_update = sum_metric.assign_add(scalar_tensor) - return sum_metric, sum_update - - -class _InitializeClustersHook(session_run_hook.SessionRunHook): - """Initializes clusters or waits for cluster initialization.""" - - def __init__(self, init_op, is_initialized_op, is_chief): - self._init_op = init_op - self._is_chief = is_chief - self._is_initialized_op = is_initialized_op - - def after_create_session(self, session, _): - assert self._init_op.graph == ops.get_default_graph() - assert self._is_initialized_op.graph == self._init_op.graph - while True: - try: - if session.run(self._is_initialized_op): - break - elif self._is_chief: - session.run(self._init_op) - else: - time.sleep(1) - except RuntimeError as e: - logging.info(e) - - -class GMM(estimator.Estimator): - """An estimator for GMM clustering.""" - SCORES = 'scores' - LOG_LIKELIHOOD = 'loss' - ASSIGNMENTS = 'assignments' - - def __init__(self, - num_clusters, - model_dir=None, - random_seed=0, - params='wmc', - initial_clusters='random', - covariance_type='full', - config=None): - """Creates a model for running GMM training and inference. - - Args: - num_clusters: number of clusters to train. - model_dir: the directory to save the model results and log files. - random_seed: Python integer. Seed for PRNG used to initialize centers. - params: Controls which parameters are updated in the training process. - Can contain any combination of "w" for weights, "m" for means, - and "c" for covars. - initial_clusters: specifies how to initialize the clusters for training. - See gmm_ops.gmm for the possible values. - covariance_type: one of "full", "diag". - config: See Estimator - """ - self._num_clusters = num_clusters - self._params = params - self._training_initial_clusters = initial_clusters - self._covariance_type = covariance_type - self._training_graph = None - self._random_seed = random_seed - super(GMM, self).__init__( - model_fn=self._model_builder(), model_dir=model_dir, config=config) - - def predict_assignments(self, input_fn=None, batch_size=None, outputs=None): - """See BaseEstimator.predict.""" - results = self.predict(input_fn=input_fn, - batch_size=batch_size, - outputs=outputs) - for result in results: - yield result[GMM.ASSIGNMENTS] - - def score(self, input_fn=None, batch_size=None, steps=None): - """Predict total log-likelihood. - - Args: - input_fn: see predict. - batch_size: see predict. - steps: see predict. - - Returns: - Total log-likelihood. - """ - results = self.evaluate(input_fn=input_fn, batch_size=batch_size, - steps=steps) - return np.log(np.sum(np.exp(results[GMM.SCORES]))) - - def weights(self): - """Returns the cluster weights.""" - return checkpoint_utils.load_variable( - self.model_dir, gmm_ops.GmmAlgorithm.CLUSTERS_WEIGHT) - - def clusters(self): - """Returns cluster centers.""" - clusters = checkpoint_utils.load_variable( - self.model_dir, gmm_ops.GmmAlgorithm.CLUSTERS_VARIABLE) - return np.squeeze(clusters, 1) - - def covariances(self): - """Returns the covariances.""" - return checkpoint_utils.load_variable( - self.model_dir, gmm_ops.GmmAlgorithm.CLUSTERS_COVS_VARIABLE) - - def _parse_tensor_or_dict(self, features): - if isinstance(features, dict): - return array_ops.concat([features[k] for k in sorted(features.keys())], - 1) - return features - - def _model_builder(self): - """Creates a model function.""" - - def _model_fn(features, labels, mode, config): - """Model function.""" - assert labels is None, labels - (loss, - scores, - model_predictions, - training_op, - init_op, - is_initialized) = gmm_ops.gmm(self._parse_tensor_or_dict(features), - self._training_initial_clusters, - self._num_clusters, self._random_seed, - self._covariance_type, - self._params) - incr_step = state_ops.assign_add(training_util.get_global_step(), 1) - training_op = with_dependencies([training_op, incr_step], loss) - training_hooks = [_InitializeClustersHook( - init_op, is_initialized, config.is_chief)] - predictions = { - GMM.ASSIGNMENTS: model_predictions[0][0], - } - eval_metric_ops = { - GMM.SCORES: scores, - GMM.LOG_LIKELIHOOD: _streaming_sum(loss), - } - return model_fn_lib.ModelFnOps(mode=mode, predictions=predictions, - eval_metric_ops=eval_metric_ops, - loss=loss, train_op=training_op, - training_hooks=training_hooks) - - return _model_fn diff --git a/tensorflow/contrib/factorization/python/ops/gmm_ops.py b/tensorflow/contrib/factorization/python/ops/gmm_ops.py deleted file mode 100644 index 000b9832aa4..00000000000 --- a/tensorflow/contrib/factorization/python/ops/gmm_ops.py +++ /dev/null @@ -1,506 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Gaussian mixture models Operations.""" -# TODO(xavigonzalvo): Factor out covariance matrix operations to make -# code reusable for different types (e.g. diag). - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops.embedding_ops import embedding_lookup - -# Machine epsilon. -MEPS = np.finfo(float).eps -FULL_COVARIANCE = 'full' -DIAG_COVARIANCE = 'diag' - - -def _covariance(x, diag): - """Defines the covariance operation of a matrix. - - Args: - x: a matrix Tensor. Dimension 0 should contain the number of examples. - diag: if True, it computes the diagonal covariance. - - Returns: - A Tensor representing the covariance of x. In the case of - diagonal matrix just the diagonal is returned. - """ - num_points = math_ops.cast(array_ops.shape(x)[0], dtypes.float32) - x -= math_ops.reduce_mean(x, 0, keepdims=True) - if diag: - cov = math_ops.reduce_sum( - math_ops.square(x), 0, keepdims=True) / (num_points - 1) - else: - cov = math_ops.matmul(x, x, transpose_a=True) / (num_points - 1) - return cov - - -def _init_clusters_random(data, num_clusters, random_seed): - """Does random initialization of clusters. - - Args: - data: a list of Tensors with a matrix of data, each row is an example. - num_clusters: an integer with the number of clusters. - random_seed: Seed for PRNG used to initialize seeds. - - Returns: - A Tensor with num_clusters random rows of data. - """ - assert isinstance(data, list) - num_data = math_ops.add_n([array_ops.shape(inp)[0] for inp in data]) - with ops.control_dependencies( - [check_ops.assert_less_equal(num_clusters, num_data)]): - indices = random_ops.random_uniform( - [num_clusters], - minval=0, - maxval=math_ops.cast(num_data, dtypes.int64), - seed=random_seed, - dtype=dtypes.int64) - indices %= math_ops.cast(num_data, dtypes.int64) - clusters_init = embedding_lookup(data, indices, partition_strategy='div') - return clusters_init - - -class GmmAlgorithm(object): - """Tensorflow Gaussian mixture model clustering class.""" - CLUSTERS_WEIGHT = 'alphas' - CLUSTERS_VARIABLE = 'clusters' - CLUSTERS_COVS_VARIABLE = 'clusters_covs' - - def __init__(self, - data, - num_classes, - initial_means=None, - params='wmc', - covariance_type=FULL_COVARIANCE, - random_seed=0): - """Constructor. - - Args: - data: a list of Tensors with data, each row is a new example. - num_classes: number of clusters. - initial_means: a Tensor with a matrix of means. If None, means are - computed by sampling randomly. - params: Controls which parameters are updated in the training - process. Can contain any combination of "w" for weights, "m" for - means, and "c" for covariances. - covariance_type: one of "full", "diag". - random_seed: Seed for PRNG used to initialize seeds. - - Raises: - Exception if covariance type is unknown. - """ - self._params = params - self._random_seed = random_seed - self._covariance_type = covariance_type - if self._covariance_type not in [DIAG_COVARIANCE, FULL_COVARIANCE]: - raise Exception( # pylint: disable=g-doc-exception - 'programmer error: Invalid covariance type: %s' % - self._covariance_type) - # Create sharded variables for multiple shards. The following - # lists are indexed by shard. - # Probability per example in a class. - num_shards = len(data) - self._probs = [None] * num_shards - # Prior probability. - self._prior_probs = [None] * num_shards - # Membership weights w_{ik} where "i" is the i-th example and "k" - # is the k-th mixture. - self._w = [None] * num_shards - # Number of examples in a class. - self._points_in_k = [None] * num_shards - first_shard = data[0] - self._dimensions = array_ops.shape(first_shard)[1] - self._num_classes = num_classes - # Small value to guarantee that covariances are invertible. - self._min_var = array_ops.diag( - array_ops.ones(array_ops.stack([self._dimensions]))) * 1e-3 - self._create_variables() - self._initialize_variables(data, initial_means) - # Operations of partial statistics for the computation of the means. - self._w_mul_x = [] - # Operations of partial statistics for the computation of the covariances. - self._w_mul_x2 = [] - self._define_graph(data) - - def _create_variables(self): - """Initializes GMM algorithm.""" - init_value = array_ops.constant([], dtype=dtypes.float32) - self._means = variables.VariableV1(init_value, - name=self.CLUSTERS_VARIABLE, - validate_shape=False) - self._covs = variables.VariableV1( - init_value, name=self.CLUSTERS_COVS_VARIABLE, validate_shape=False) - # Mixture weights, representing the probability that a randomly - # selected unobservable data (in EM terms) was generated by component k. - self._alpha = variable_scope.variable( - array_ops.tile([1.0 / self._num_classes], [self._num_classes]), - name=self.CLUSTERS_WEIGHT, - validate_shape=False) - self._cluster_centers_initialized = variables.VariableV1(False, - dtype=dtypes.bool, - name='initialized') - - def _initialize_variables(self, data, initial_means=None): - """Initializes variables. - - Args: - data: a list of Tensors with data, each row is a new example. - initial_means: a Tensor with a matrix of means. - """ - first_shard = data[0] - # Initialize means: num_classes X 1 X dimensions. - if initial_means is not None: - means = array_ops.expand_dims(initial_means, 1) - else: - # Sample data randomly - means = array_ops.expand_dims( - _init_clusters_random(data, self._num_classes, self._random_seed), 1) - - # Initialize covariances. - if self._covariance_type == FULL_COVARIANCE: - cov = _covariance(first_shard, False) + self._min_var - # A matrix per class, num_classes X dimensions X dimensions - covs = array_ops.tile( - array_ops.expand_dims(cov, 0), [self._num_classes, 1, 1]) - elif self._covariance_type == DIAG_COVARIANCE: - cov = _covariance(first_shard, True) + self._min_var - # A diagonal per row, num_classes X dimensions. - covs = array_ops.tile( - array_ops.expand_dims(array_ops.diag_part(cov), 0), - [self._num_classes, 1]) - - with ops.colocate_with(self._cluster_centers_initialized): - initialized = control_flow_ops.with_dependencies( - [means, covs], - array_ops.identity(self._cluster_centers_initialized)) - self._init_ops = [] - with ops.colocate_with(self._means): - init_means = state_ops.assign(self._means, means, validate_shape=False) - init_means = control_flow_ops.with_dependencies( - [init_means], - state_ops.assign(self._cluster_centers_initialized, True)) - self._init_ops.append(control_flow_ops.cond(initialized, - control_flow_ops.no_op, - lambda: init_means).op) - with ops.colocate_with(self._covs): - init_covs = state_ops.assign(self._covs, covs, validate_shape=False) - init_covs = control_flow_ops.with_dependencies( - [init_covs], - state_ops.assign(self._cluster_centers_initialized, True)) - self._init_ops.append(control_flow_ops.cond(initialized, - control_flow_ops.no_op, - lambda: init_covs).op) - - def init_ops(self): - """Returns the initialization operation.""" - return control_flow_ops.group(*self._init_ops) - - def training_ops(self): - """Returns the training operation.""" - return control_flow_ops.group(*self._train_ops) - - def is_initialized(self): - """Returns a boolean operation for initialized variables.""" - return self._cluster_centers_initialized - - def alphas(self): - return self._alpha - - def clusters(self): - """Returns the clusters with dimensions num_classes X 1 X num_dimensions.""" - return self._means - - def covariances(self): - """Returns the covariances matrices.""" - return self._covs - - def assignments(self): - """Returns a list of Tensors with the matrix of assignments per shard.""" - ret = [] - for w in self._w: - ret.append(math_ops.argmax(w, 1)) - return ret - - def scores(self): - """Returns the per-sample likelihood fo the data. - - Returns: - Log probabilities of each data point. - """ - return self._scores - - def log_likelihood_op(self): - """Returns the log-likelihood operation.""" - return self._log_likelihood_op - - def _define_graph(self, data): - """Define graph for a single iteration. - - Args: - data: a list of Tensors defining the training data. - """ - for shard_id, shard in enumerate(data): - self._num_examples = array_ops.shape(shard)[0] - shard = array_ops.expand_dims(shard, 0) - self._define_log_prob_operation(shard_id, shard) - self._define_prior_log_prob_operation(shard_id) - self._define_expectation_operation(shard_id) - self._define_partial_maximization_operation(shard_id, shard) - self._define_maximization_operation(len(data)) - self._define_loglikelihood_operation() - self._define_score_samples() - - def _define_full_covariance_probs(self, shard_id, shard): - """Defines the full covariance probabilities per example in a class. - - Updates a matrix with dimension num_examples X num_classes. - - Args: - shard_id: id of the current shard. - shard: current data shard, 1 X num_examples X dimensions. - """ - diff = shard - self._means - cholesky = linalg_ops.cholesky(self._covs + self._min_var) - log_det_covs = 2.0 * math_ops.reduce_sum( - math_ops.log(array_ops.matrix_diag_part(cholesky)), 1) - x_mu_cov = math_ops.square( - linalg_ops.matrix_triangular_solve( - cholesky, array_ops.transpose( - diff, perm=[0, 2, 1]), lower=True)) - diag_m = array_ops.transpose(math_ops.reduce_sum(x_mu_cov, 1)) - self._probs[shard_id] = ( - -0.5 * (diag_m + math_ops.cast(self._dimensions, dtypes.float32) * - math_ops.log(2 * np.pi) + log_det_covs)) - - def _define_diag_covariance_probs(self, shard_id, shard): - """Defines the diagonal covariance probabilities per example in a class. - - Args: - shard_id: id of the current shard. - shard: current data shard, 1 X num_examples X dimensions. - - Returns a matrix num_examples * num_classes. - """ - # num_classes X 1 - # TODO(xavigonzalvo): look into alternatives to log for - # reparametrization of variance parameters. - det_expanded = math_ops.reduce_sum( - math_ops.log(self._covs + 1e-3), 1, keepdims=True) - x2 = math_ops.squared_difference(shard, self._means) - cov_expanded = array_ops.expand_dims(1.0 / (self._covs + 1e-3), 2) - # num_classes X num_examples - x2_cov = math_ops.matmul(x2, cov_expanded) - x2_cov = array_ops.transpose(array_ops.squeeze(x2_cov, [2])) - self._probs[shard_id] = -0.5 * ( - math_ops.cast(self._dimensions, dtypes.float32) * - math_ops.log(2.0 * np.pi) + - array_ops.transpose(det_expanded) + x2_cov) - - def _define_log_prob_operation(self, shard_id, shard): - """Probability per example in a class. - - Updates a matrix with dimension num_examples X num_classes. - - Args: - shard_id: id of the current shard. - shard: current data shard, 1 X num_examples X dimensions. - """ - # TODO(xavigonzalvo): Use the pdf defined in - # third_party/tensorflow/contrib/distributions/python/ops/gaussian.py - if self._covariance_type == FULL_COVARIANCE: - self._define_full_covariance_probs(shard_id, shard) - elif self._covariance_type == DIAG_COVARIANCE: - self._define_diag_covariance_probs(shard_id, shard) - self._probs[shard_id] += math_ops.log(self._alpha) - - def _define_prior_log_prob_operation(self, shard_id): - """Computes the prior probability of all samples. - - Updates a vector where each item is the prior probability of an - input example. - - Args: - shard_id: id of current shard_id. - """ - self._prior_probs[shard_id] = math_ops.reduce_logsumexp( - self._probs[shard_id], axis=1, keepdims=True) - - def _define_expectation_operation(self, shard_id): - # Shape broadcasting. - probs = array_ops.expand_dims(self._probs[shard_id], 0) - # Membership weights are computed as: - # $$w_{ik} = \frac{\alpha_k f(\mathbf{y_i}|\mathbf{\theta}_k)}$$ - # $$ {\sum_{m=1}^{K}\alpha_mf(\mathbf{y_i}|\mathbf{\theta}_m)}$$ - # where "i" is the i-th example, "k" is the k-th mixture, theta are - # the model parameters and y_i the observations. - # These are defined for each shard. - self._w[shard_id] = array_ops.reshape( - math_ops.exp(probs - self._prior_probs[shard_id]), - array_ops.stack([self._num_examples, self._num_classes])) - - def _define_partial_maximization_operation(self, shard_id, shard): - """Computes the partial statistics of the means and covariances. - - Args: - shard_id: current shard id. - shard: current data shard, 1 X num_examples X dimensions. - """ - # Soft assignment of each data point to each of the two clusters. - self._points_in_k[shard_id] = math_ops.reduce_sum( - self._w[shard_id], 0, keepdims=True) - # Partial means. - w_mul_x = array_ops.expand_dims( - math_ops.matmul( - self._w[shard_id], array_ops.squeeze(shard, [0]), transpose_a=True), - 1) - self._w_mul_x.append(w_mul_x) - # Partial covariances. - x = array_ops.concat([shard for _ in range(self._num_classes)], 0) - x_trans = array_ops.transpose(x, perm=[0, 2, 1]) - x_mul_w = array_ops.concat([ - array_ops.expand_dims(x_trans[k, :, :] * self._w[shard_id][:, k], 0) - for k in range(self._num_classes) - ], 0) - self._w_mul_x2.append(math_ops.matmul(x_mul_w, x)) - - def _define_maximization_operation(self, num_batches): - """Maximization operations.""" - # TODO(xavigonzalvo): some of these operations could be moved to C++. - # Compute the effective number of data points assigned to component k. - with ops.control_dependencies(self._w): - points_in_k = array_ops.squeeze( - math_ops.add_n(self._points_in_k), axis=[0]) - # Update alpha. - if 'w' in self._params: - final_points_in_k = points_in_k / num_batches - num_examples = math_ops.cast(math_ops.reduce_sum(final_points_in_k), - dtypes.float32) - self._alpha_op = self._alpha.assign(final_points_in_k / - (num_examples + MEPS)) - else: - self._alpha_op = control_flow_ops.no_op() - self._train_ops = [self._alpha_op] - - # Update means. - points_in_k_expanded = array_ops.reshape(points_in_k, - [self._num_classes, 1, 1]) - if 'm' in self._params: - self._means_op = self._means.assign( - math_ops.div( - math_ops.add_n(self._w_mul_x), points_in_k_expanded + MEPS)) - else: - self._means_op = control_flow_ops.no_op() - # means are (num_classes x 1 x dims) - - # Update covariances. - with ops.control_dependencies([self._means_op]): - b = math_ops.add_n(self._w_mul_x2) / (points_in_k_expanded + MEPS) - new_covs = [] - for k in range(self._num_classes): - mean = self._means.value()[k, :, :] - square_mean = math_ops.matmul(mean, mean, transpose_a=True) - new_cov = b[k, :, :] - square_mean + self._min_var - if self._covariance_type == FULL_COVARIANCE: - new_covs.append(array_ops.expand_dims(new_cov, 0)) - elif self._covariance_type == DIAG_COVARIANCE: - new_covs.append( - array_ops.expand_dims(array_ops.diag_part(new_cov), 0)) - new_covs = array_ops.concat(new_covs, 0) - if 'c' in self._params: - # Train operations don't need to take care of the means - # because covariances already depend on it. - with ops.control_dependencies([self._means_op, new_covs]): - self._train_ops.append( - state_ops.assign( - self._covs, new_covs, validate_shape=False)) - - def _define_loglikelihood_operation(self): - """Defines the total log-likelihood of current iteration.""" - op = [] - for prior_probs in self._prior_probs: - op.append(math_ops.reduce_logsumexp(prior_probs)) - self._log_likelihood_op = math_ops.reduce_logsumexp(op) - - def _define_score_samples(self): - """Defines the likelihood of each data sample.""" - op = [] - for shard_id, prior_probs in enumerate(self._prior_probs): - op.append(prior_probs + math_ops.log(self._w[shard_id])) - self._scores = array_ops.squeeze( - math_ops.reduce_logsumexp(op, axis=2, keepdims=True), axis=0) - - -def gmm(inp, - initial_clusters, - num_clusters, - random_seed, - covariance_type=FULL_COVARIANCE, - params='wmc'): - """Creates the graph for Gaussian mixture model (GMM) clustering. - - Args: - inp: An input tensor or list of input tensors - initial_clusters: Specifies the clusters used during - initialization. Can be a tensor or numpy array, or a function - that generates the clusters. Can also be "random" to specify - that clusters should be chosen randomly from input data. Note: type - is diverse to be consistent with skflow. - num_clusters: number of clusters. - random_seed: Python integer. Seed for PRNG used to initialize centers. - covariance_type: one of "diag", "full". - params: Controls which parameters are updated in the training - process. Can contain any combination of "w" for weights, "m" for - means, and "c" for covars. - - Returns: - Note: tuple of lists returned to be consistent with skflow - A tuple consisting of: - assignments: A vector (or list of vectors). Each element in the vector - corresponds to an input row in 'inp' and specifies the cluster id - corresponding to the input. - training_op: an op that runs an iteration of training. - init_op: an op that runs the initialization. - """ - initial_means = None - if initial_clusters != 'random' and not isinstance(initial_clusters, - ops.Tensor): - initial_means = constant_op.constant(initial_clusters, dtype=dtypes.float32) - - # Implementation of GMM. - inp = inp if isinstance(inp, list) else [inp] - gmm_tool = GmmAlgorithm(inp, num_clusters, initial_means, params, - covariance_type, random_seed) - assignments = gmm_tool.assignments() - scores = gmm_tool.scores() - loss = gmm_tool.log_likelihood_op() - return (loss, scores, [assignments], gmm_tool.training_ops(), - gmm_tool.init_ops(), gmm_tool.is_initialized()) diff --git a/tensorflow/contrib/factorization/python/ops/gmm_ops_test.py b/tensorflow/contrib/factorization/python/ops/gmm_ops_test.py deleted file mode 100644 index 112e4d289b0..00000000000 --- a/tensorflow/contrib/factorization/python/ops/gmm_ops_test.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for gmm_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time - -import numpy as np -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib.factorization.python.ops import gmm_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed as random_seed_lib -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging as logging - - -class GmmOpsTest(test.TestCase): - - def setUp(self): - self.num_examples = 1000 - self.iterations = 40 - self.seed = 4 - random_seed_lib.set_random_seed(self.seed) - np.random.seed(self.seed * 2) - self.data, self.true_assignments = self.make_data(self.num_examples) - # Generate more complicated data. - self.centers = [[1, 1], [-1, 0.5], [2, 1]] - self.more_data, self.more_true_assignments = self.make_data_from_centers( - self.num_examples, self.centers) - - @staticmethod - def make_data(num_vectors): - """Generates 2-dimensional data centered on (2,2), (-1,-1). - - Args: - num_vectors: number of training examples. - - Returns: - A tuple containing the data as a numpy array and the cluster ids. - """ - vectors = [] - classes = [] - for _ in xrange(num_vectors): - if np.random.random() > 0.5: - vectors.append([np.random.normal(2.0, 0.6), np.random.normal(2.0, 0.9)]) - classes.append(0) - else: - vectors.append( - [np.random.normal(-1.0, 0.4), np.random.normal(-1.0, 0.5)]) - classes.append(1) - return np.asarray(vectors), classes - - @staticmethod - def make_data_from_centers(num_vectors, centers): - """Generates 2-dimensional data with random centers. - - Args: - num_vectors: number of training examples. - centers: a list of random 2-dimensional centers. - - Returns: - A tuple containing the data as a numpy array and the cluster ids. - """ - vectors = [] - classes = [] - for _ in xrange(num_vectors): - current_class = np.random.random_integers(0, len(centers) - 1) - vectors.append([ - np.random.normal(centers[current_class][0], - np.random.random_sample()), - np.random.normal(centers[current_class][1], np.random.random_sample()) - ]) - classes.append(current_class) - return np.asarray(vectors), len(centers) - - def test_covariance(self): - start_time = time.time() - data = self.data.T - np_cov = np.cov(data) - logging.info('Numpy took %f', time.time() - start_time) - - start_time = time.time() - with self.cached_session() as sess: - op = gmm_ops._covariance( - constant_op.constant( - data.T, dtype=dtypes.float32), False) - op_diag = gmm_ops._covariance( - constant_op.constant( - data.T, dtype=dtypes.float32), True) - variables.global_variables_initializer().run() - tf_cov = sess.run(op) - np.testing.assert_array_almost_equal(np_cov, tf_cov) - logging.info('Tensorflow took %f', time.time() - start_time) - tf_cov = sess.run(op_diag) - np.testing.assert_array_almost_equal( - np.diag(np_cov), np.ravel(tf_cov), decimal=5) - - def test_simple_cluster(self): - """Tests that the clusters are correct.""" - num_classes = 2 - graph = ops.Graph() - with graph.as_default() as g: - g.seed = 5 - with self.cached_session() as sess: - data = constant_op.constant(self.data, dtype=dtypes.float32) - loss_op, scores, assignments, training_op, init_op, _ = gmm_ops.gmm( - data, 'random', num_classes, random_seed=self.seed) - - variables.global_variables_initializer().run() - sess.run(init_op) - first_loss = sess.run(loss_op) - for _ in xrange(self.iterations): - sess.run(training_op) - assignments = sess.run(assignments) - end_loss = sess.run(loss_op) - scores = sess.run(scores) - self.assertEqual((self.num_examples, 1), scores.shape) - accuracy = np.mean( - np.asarray(self.true_assignments) == np.squeeze(assignments)) - logging.info('Accuracy: %f', accuracy) - logging.info('First loss: %f, end loss: %f', first_loss, end_loss) - self.assertGreater(end_loss, first_loss) - self.assertGreater(accuracy, 0.98) - - def testParams(self): - """Tests that the params work as intended.""" - num_classes = 2 - with self.cached_session() as sess: - # Experiment 1. Update weights only. - data = constant_op.constant(self.data, dtype=dtypes.float32) - gmm_tool = gmm_ops.GmmAlgorithm([data], num_classes, - [[3.0, 3.0], [0.0, 0.0]], 'w') - training_ops = gmm_tool.training_ops() - variables.global_variables_initializer().run() - sess.run(gmm_tool.init_ops()) - for _ in xrange(self.iterations): - sess.run(training_ops) - - # Only the probability to each class is updated. - alphas = sess.run(gmm_tool.alphas()) - self.assertGreater(alphas[1], 0.6) - means = sess.run(gmm_tool.clusters()) - np.testing.assert_almost_equal( - np.expand_dims([[3.0, 3.0], [0.0, 0.0]], 1), means) - covs = sess.run(gmm_tool.covariances()) - np.testing.assert_almost_equal(covs[0], covs[1]) - - # Experiment 2. Update means and covariances. - gmm_tool = gmm_ops.GmmAlgorithm([data], num_classes, - [[3.0, 3.0], [0.0, 0.0]], 'mc') - training_ops = gmm_tool.training_ops() - variables.global_variables_initializer().run() - sess.run(gmm_tool.init_ops()) - for _ in xrange(self.iterations): - sess.run(training_ops) - alphas = sess.run(gmm_tool.alphas()) - self.assertAlmostEqual(alphas[0], alphas[1]) - means = sess.run(gmm_tool.clusters()) - np.testing.assert_almost_equal( - np.expand_dims([[2.0, 2.0], [-1.0, -1.0]], 1), means, decimal=1) - covs = sess.run(gmm_tool.covariances()) - np.testing.assert_almost_equal( - [[0.371111, -0.0050774], [-0.0050774, 0.8651744]], covs[0], decimal=4) - np.testing.assert_almost_equal( - [[0.146976, 0.0259463], [0.0259463, 0.2543971]], covs[1], decimal=4) - - # Experiment 3. Update covariances only. - gmm_tool = gmm_ops.GmmAlgorithm([data], num_classes, - [[-1.0, -1.0], [1.0, 1.0]], 'c') - training_ops = gmm_tool.training_ops() - variables.global_variables_initializer().run() - sess.run(gmm_tool.init_ops()) - for _ in xrange(self.iterations): - sess.run(training_ops) - alphas = sess.run(gmm_tool.alphas()) - self.assertAlmostEqual(alphas[0], alphas[1]) - means = sess.run(gmm_tool.clusters()) - np.testing.assert_almost_equal( - np.expand_dims([[-1.0, -1.0], [1.0, 1.0]], 1), means) - covs = sess.run(gmm_tool.covariances()) - np.testing.assert_almost_equal( - [[0.1299582, 0.0435872], [0.0435872, 0.2558578]], covs[0], decimal=5) - np.testing.assert_almost_equal( - [[3.195385, 2.6989155], [2.6989155, 3.3881593]], covs[1], decimal=5) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/factorization/python/ops/gmm_test.py b/tensorflow/contrib/factorization/python/ops/gmm_test.py deleted file mode 100644 index 4fc9c96e9d0..00000000000 --- a/tensorflow/contrib/factorization/python/ops/gmm_test.py +++ /dev/null @@ -1,222 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for ops.gmm.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.factorization.python.ops import gmm as gmm_lib -from tensorflow.contrib.learn.python.learn.estimators import kmeans -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import random_seed as random_seed_lib -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import test -from tensorflow.python.training import queue_runner - - -class GMMTest(test.TestCase): - - def input_fn(self, batch_size=None, points=None): - batch_size = batch_size or self.batch_size - points = points if points is not None else self.points - num_points = points.shape[0] - - def _fn(): - x = constant_op.constant(points) - if batch_size == num_points: - return x, None - indices = random_ops.random_uniform(constant_op.constant([batch_size]), - minval=0, maxval=num_points-1, - dtype=dtypes.int32, - seed=10) - return array_ops.gather(x, indices), None - return _fn - - def setUp(self): - np.random.seed(3) - random_seed_lib.set_random_seed(2) - self.num_centers = 2 - self.num_dims = 2 - self.num_points = 4000 - self.batch_size = self.num_points - self.true_centers = self.make_random_centers(self.num_centers, - self.num_dims) - self.points, self.assignments = self.make_random_points( - self.true_centers, self.num_points) - - # Use initial means from kmeans (just like scikit-learn does). - clusterer = kmeans.KMeansClustering(num_clusters=self.num_centers) - clusterer.fit(input_fn=lambda: (constant_op.constant(self.points), None), - steps=30) - self.initial_means = clusterer.clusters() - - @staticmethod - def make_random_centers(num_centers, num_dims): - return np.round( - np.random.rand(num_centers, num_dims).astype(np.float32) * 500) - - @staticmethod - def make_random_points(centers, num_points): - num_centers, num_dims = centers.shape - assignments = np.random.choice(num_centers, num_points) - offsets = np.round( - np.random.randn(num_points, num_dims).astype(np.float32) * 20) - points = centers[assignments] + offsets - return (points, assignments) - - def test_weights(self): - """Tests the shape of the weights.""" - gmm = gmm_lib.GMM(self.num_centers, - initial_clusters=self.initial_means, - random_seed=4, - config=run_config.RunConfig(tf_random_seed=2)) - gmm.fit(input_fn=self.input_fn(), steps=0) - weights = gmm.weights() - self.assertAllEqual(list(weights.shape), [self.num_centers]) - - def test_clusters(self): - """Tests the shape of the clusters.""" - gmm = gmm_lib.GMM(self.num_centers, - initial_clusters=self.initial_means, - random_seed=4, - config=run_config.RunConfig(tf_random_seed=2)) - gmm.fit(input_fn=self.input_fn(), steps=0) - clusters = gmm.clusters() - self.assertAllEqual(list(clusters.shape), [self.num_centers, self.num_dims]) - - def test_fit(self): - gmm = gmm_lib.GMM(self.num_centers, - initial_clusters='random', - random_seed=4, - config=run_config.RunConfig(tf_random_seed=2)) - gmm.fit(input_fn=self.input_fn(), steps=1) - score1 = gmm.score(input_fn=self.input_fn(batch_size=self.num_points), - steps=1) - gmm.fit(input_fn=self.input_fn(), steps=10) - score2 = gmm.score(input_fn=self.input_fn(batch_size=self.num_points), - steps=1) - self.assertLess(score1, score2) - - def test_infer(self): - gmm = gmm_lib.GMM(self.num_centers, - initial_clusters=self.initial_means, - random_seed=4, - config=run_config.RunConfig(tf_random_seed=2)) - gmm.fit(input_fn=self.input_fn(), steps=60) - clusters = gmm.clusters() - - # Make a small test set - num_points = 40 - points, true_assignments = self.make_random_points(clusters, num_points) - - assignments = [] - for item in gmm.predict_assignments( - input_fn=self.input_fn(points=points, batch_size=num_points)): - assignments.append(item) - assignments = np.ravel(assignments) - self.assertAllEqual(true_assignments, assignments) - - def _compare_with_sklearn(self, cov_type): - # sklearn version. - iterations = 40 - np.random.seed(5) - sklearn_assignments = np.asarray([0, 0, 1, 0, 0, 0, 1, 0, 0, 1]) - sklearn_means = np.asarray([[144.83417719, 254.20130341], - [274.38754816, 353.16074346]]) - sklearn_covs = np.asarray([[[395.0081194, -4.50389512], - [-4.50389512, 408.27543989]], - [[385.17484203, -31.27834935], - [-31.27834935, 391.74249925]]]) - - # skflow version. - gmm = gmm_lib.GMM(self.num_centers, - initial_clusters=self.initial_means, - covariance_type=cov_type, - config=run_config.RunConfig(tf_random_seed=2)) - gmm.fit(input_fn=self.input_fn(), steps=iterations) - points = self.points[:10, :] - skflow_assignments = [] - for item in gmm.predict_assignments( - input_fn=self.input_fn(points=points, batch_size=10)): - skflow_assignments.append(item) - self.assertAllClose(sklearn_assignments, - np.ravel(skflow_assignments).astype(int)) - self.assertAllClose(sklearn_means, gmm.clusters()) - if cov_type == 'full': - self.assertAllClose(sklearn_covs, gmm.covariances(), rtol=0.01) - else: - for d in [0, 1]: - self.assertAllClose( - np.diag(sklearn_covs[d]), gmm.covariances()[d, :], rtol=0.01) - - def test_compare_full(self): - self._compare_with_sklearn('full') - - def test_compare_diag(self): - self._compare_with_sklearn('diag') - - def test_random_input_large(self): - # sklearn version. - iterations = 5 # that should be enough to know whether this diverges - np.random.seed(5) - num_classes = 20 - x = np.array([[np.random.random() for _ in range(100)] - for _ in range(num_classes)], dtype=np.float32) - - # skflow version. - gmm = gmm_lib.GMM(num_classes, - covariance_type='full', - config=run_config.RunConfig(tf_random_seed=2)) - - def get_input_fn(x): - def input_fn(): - return constant_op.constant(x.astype(np.float32)), None - return input_fn - - gmm.fit(input_fn=get_input_fn(x), steps=iterations) - self.assertFalse(np.isnan(gmm.clusters()).any()) - - -class GMMTestQueues(test.TestCase): - - def input_fn(self): - def _fn(): - queue = data_flow_ops.FIFOQueue(capacity=10, - dtypes=dtypes.float32, - shapes=[10, 3]) - enqueue_op = queue.enqueue(array_ops.zeros([10, 3], dtype=dtypes.float32)) - queue_runner.add_queue_runner(queue_runner.QueueRunner(queue, - [enqueue_op])) - return queue.dequeue(), None - return _fn - - # This test makes sure that there are no deadlocks when using a QueueRunner. - # Note that since cluster initialization is dependent on inputs, if input - # is generated using a QueueRunner, one has to make sure that these runners - # are started before the initialization. - def test_queues(self): - gmm = gmm_lib.GMM(2, covariance_type='diag') - gmm.fit(input_fn=self.input_fn(), steps=1) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/factorization/python/ops/kmeans.py b/tensorflow/contrib/factorization/python/ops/kmeans.py deleted file mode 100644 index 260ea5bf127..00000000000 --- a/tensorflow/contrib/factorization/python/ops/kmeans.py +++ /dev/null @@ -1,478 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A canned Estimator for k-means clustering.""" - -# TODO(ccolby): Move clustering_ops.py into this file and streamline the code. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time - -from tensorflow.contrib.factorization.python.ops import clustering_ops -from tensorflow.python.estimator import estimator -from tensorflow.python.estimator import model_fn as model_fn_lib -from tensorflow.python.estimator.export import export_output -from tensorflow.python.feature_column import feature_column_lib as fc -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import metrics -from tensorflow.python.ops import state_ops -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.saved_model import signature_constants -from tensorflow.python.summary import summary -from tensorflow.python.training import session_run_hook -from tensorflow.python.training import training_util - - -class _LossRelativeChangeHook(session_run_hook.SessionRunHook): - """Stops when the change in loss goes below a tolerance.""" - - def __init__(self, loss_tensor, tolerance): - """Creates a _LossRelativeChangeHook. - - Args: - loss_tensor: A scalar tensor of the loss value. - tolerance: A relative tolerance of loss change between iterations. - """ - self._loss_tensor = loss_tensor - self._tolerance = tolerance - self._prev_loss = None - - def before_run(self, run_context): - del run_context # unused - return session_run_hook.SessionRunArgs(self._loss_tensor) - - def after_run(self, run_context, run_values): - loss = run_values.results - assert loss is not None - if self._prev_loss: - relative_change = ( - abs(loss - self._prev_loss) / (1 + abs(self._prev_loss))) - if relative_change < self._tolerance: - run_context.request_stop() - self._prev_loss = loss - - -class _InitializeClustersHook(session_run_hook.SessionRunHook): - """Initializes the cluster centers. - - The chief repeatedly invokes an initialization op until all cluster centers - are initialized. The workers wait for the initialization phase to complete. - """ - - def __init__(self, init_op, is_initialized_var, is_chief): - """Creates an _InitializeClustersHook. - - Args: - init_op: An op that, when run, will choose some initial cluster centers. - This op may need to be run multiple times to choose all the centers. - is_initialized_var: A boolean variable reporting whether all initial - centers have been chosen. - is_chief: A boolean specifying whether this task is the chief. - """ - self._init_op = init_op - self._is_initialized_var = is_initialized_var - self._is_chief = is_chief - - def after_create_session(self, session, coord): - del coord # unused - assert self._init_op.graph is ops.get_default_graph() - assert self._is_initialized_var.graph is self._init_op.graph - while True: - try: - if session.run(self._is_initialized_var): - break - elif self._is_chief: - session.run(self._init_op) - else: - time.sleep(1) - except RuntimeError as e: - logging.info(e) - - -def _parse_features_if_necessary(features, feature_columns): - """Helper function to convert the input points into a usable format. - - Args: - features: The input features. - feature_columns: An optionable iterable containing all the feature columns - used by the model. All items in the set should be feature column instances - that can be passed to `tf.compat.v1.feature_column.input_layer`. If this - is None, all features will be used. - - Returns: - If `features` is a dict of `k` features (optionally filtered by - `feature_columns`), each of which is a vector of `n` scalars, the return - value is a Tensor of shape `(n, k)` representing `n` input points, where the - items in the `k` dimension are sorted lexicographically by `features` key. - If `features` is not a dict, it is returned unmodified. - """ - if not isinstance(features, dict): - return features - - if feature_columns: - return fc.input_layer(features, feature_columns) - - keys = sorted(features.keys()) - with ops.colocate_with(features[keys[0]]): - return array_ops.concat([features[k] for k in keys], axis=1) - - -class _ModelFn(object): - """Model function for the estimator.""" - - def __init__(self, num_clusters, initial_clusters, distance_metric, - random_seed, use_mini_batch, mini_batch_steps_per_iteration, - kmeans_plus_plus_num_retries, relative_tolerance, - feature_columns): - self._num_clusters = num_clusters - self._initial_clusters = initial_clusters - self._distance_metric = distance_metric - self._random_seed = random_seed - self._use_mini_batch = use_mini_batch - self._mini_batch_steps_per_iteration = mini_batch_steps_per_iteration - self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries - self._relative_tolerance = relative_tolerance - self._feature_columns = feature_columns - - def model_fn(self, features, mode, config): - """Model function for the estimator. - - Note that this does not take a `labels` arg. This works, but `input_fn` must - return either `features` or, equivalently, `(features, None)`. - - Args: - features: The input points. See `tf.estimator.Estimator`. - mode: See `tf.estimator.Estimator`. - config: See `tf.estimator.Estimator`. - - Returns: - A `tf.estimator.EstimatorSpec` (see `tf.estimator.Estimator`) specifying - this behavior: - * `train_op`: Execute one mini-batch or full-batch run of Lloyd's - algorithm. - * `loss`: The sum of the squared distances from each input point to its - closest center. - * `eval_metric_ops`: Maps `SCORE` to `loss`. - * `predictions`: Maps `ALL_DISTANCES` to the distance from each input - point to each cluster center; maps `CLUSTER_INDEX` to the index of - the closest cluster center for each input point. - """ - # input_points is a single Tensor. Therefore, the sharding functionality - # in clustering_ops is unused, and some of the values below are lists of a - # single item. - input_points = _parse_features_if_necessary(features, self._feature_columns) - - # Let N = the number of input_points. - # all_distances: A list of one matrix of shape (N, num_clusters). Each value - # is the distance from an input point to a cluster center. - # model_predictions: A list of one vector of shape (N). Each value is the - # cluster id of an input point. - # losses: Similar to cluster_idx but provides the distance to the cluster - # center. - # is_initialized: scalar indicating whether the initial cluster centers - # have been chosen; see init_op. - # init_op: an op to choose the initial cluster centers. A single worker - # repeatedly executes init_op until is_initialized becomes True. - # training_op: an op that runs an iteration of training, either an entire - # Lloyd iteration or a mini-batch of a Lloyd iteration. Multiple workers - # may execute this op, but only after is_initialized becomes True. - (all_distances, model_predictions, losses, is_initialized, init_op, - training_op) = clustering_ops.KMeans( - inputs=input_points, - num_clusters=self._num_clusters, - initial_clusters=self._initial_clusters, - distance_metric=self._distance_metric, - use_mini_batch=self._use_mini_batch, - mini_batch_steps_per_iteration=self._mini_batch_steps_per_iteration, - random_seed=self._random_seed, - kmeans_plus_plus_num_retries=self._kmeans_plus_plus_num_retries - ).training_graph() - - loss = math_ops.reduce_sum(losses) - summary.scalar('loss/raw', loss) - - incr_step = state_ops.assign_add(training_util.get_global_step(), 1) - training_op = control_flow_ops.with_dependencies([training_op, incr_step], - loss) - - training_hooks = [ - _InitializeClustersHook(init_op, is_initialized, config.is_chief) - ] - if self._relative_tolerance is not None: - training_hooks.append( - _LossRelativeChangeHook(loss, self._relative_tolerance)) - - export_outputs = { - KMeansClustering.ALL_DISTANCES: - export_output.PredictOutput(all_distances[0]), - KMeansClustering.CLUSTER_INDEX: - export_output.PredictOutput(model_predictions[0]), - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: - export_output.PredictOutput(model_predictions[0]) - } - - return model_fn_lib.EstimatorSpec( - mode=mode, - predictions={ - KMeansClustering.ALL_DISTANCES: all_distances[0], - KMeansClustering.CLUSTER_INDEX: model_predictions[0], - }, - loss=loss, - train_op=training_op, - eval_metric_ops={KMeansClustering.SCORE: metrics.mean(loss)}, - training_hooks=training_hooks, - export_outputs=export_outputs) - - -# TODO(agarwal,ands): support sharded input. -class KMeansClustering(estimator.Estimator): - """An Estimator for K-Means clustering. - - Example: - ``` - import numpy as np - import tensorflow as tf - - num_points = 100 - dimensions = 2 - points = np.random.uniform(0, 1000, [num_points, dimensions]) - - def input_fn(): - return tf.compat.v1.train.limit_epochs( - tf.convert_to_tensor(points, dtype=tf.float32), num_epochs=1) - - num_clusters = 5 - kmeans = tf.contrib.factorization.KMeansClustering( - num_clusters=num_clusters, use_mini_batch=False) - - # train - num_iterations = 10 - previous_centers = None - for _ in xrange(num_iterations): - kmeans.train(input_fn) - cluster_centers = kmeans.cluster_centers() - if previous_centers is not None: - print 'delta:', cluster_centers - previous_centers - previous_centers = cluster_centers - print 'score:', kmeans.score(input_fn) - print 'cluster centers:', cluster_centers - - # map the input points to their clusters - cluster_indices = list(kmeans.predict_cluster_index(input_fn)) - for i, point in enumerate(points): - cluster_index = cluster_indices[i] - center = cluster_centers[cluster_index] - print 'point:', point, 'is in cluster', cluster_index, 'centered at', center - ``` - - The `SavedModel` saved by the `export_savedmodel` method does not include the - cluster centers. However, the cluster centers may be retrieved by the - latest checkpoint saved during training. Specifically, - ``` - kmeans.cluster_centers() - ``` - is equivalent to - ``` - tf.train.load_variable( - kmeans.model_dir, KMeansClustering.CLUSTER_CENTERS_VAR_NAME) - ``` - """ - - # Valid values for the distance_metric constructor argument. - SQUARED_EUCLIDEAN_DISTANCE = clustering_ops.SQUARED_EUCLIDEAN_DISTANCE - COSINE_DISTANCE = clustering_ops.COSINE_DISTANCE - - # Values for initial_clusters constructor argument. - RANDOM_INIT = clustering_ops.RANDOM_INIT - KMEANS_PLUS_PLUS_INIT = clustering_ops.KMEANS_PLUS_PLUS_INIT - - # Metric returned by evaluate(): The sum of the squared distances from each - # input point to its closest center. - SCORE = 'score' - - # Keys returned by predict(). - # ALL_DISTANCES: The distance from each input point to each cluster center. - # CLUSTER_INDEX: The index of the closest cluster center for each input point. - CLUSTER_INDEX = 'cluster_index' - ALL_DISTANCES = 'all_distances' - - # Variable name used by cluster_centers(). - CLUSTER_CENTERS_VAR_NAME = clustering_ops.CLUSTERS_VAR_NAME - - def __init__(self, - num_clusters, - model_dir=None, - initial_clusters=RANDOM_INIT, - distance_metric=SQUARED_EUCLIDEAN_DISTANCE, - random_seed=0, - use_mini_batch=True, - mini_batch_steps_per_iteration=1, - kmeans_plus_plus_num_retries=2, - relative_tolerance=None, - config=None, - feature_columns=None): - """Creates an Estimator for running KMeans training and inference. - - This Estimator implements the following variants of the K-means algorithm: - - If `use_mini_batch` is False, it runs standard full batch K-means. Each - training step runs a single iteration of K-Means and must process the full - input at once. To run in this mode, the `input_fn` passed to `train` must - return the entire input dataset. - - If `use_mini_batch` is True, it runs a generalization of the mini-batch - K-means algorithm. It runs multiple iterations, where each iteration is - composed of `mini_batch_steps_per_iteration` steps. Each training step - accumulates the contribution from one mini-batch into temporary storage. - Every `mini_batch_steps_per_iteration` steps, the cluster centers are - updated and the temporary storage cleared for the next iteration. Note - that: - * If `mini_batch_steps_per_iteration=1`, the algorithm reduces to the - standard K-means mini-batch algorithm. - * If `mini_batch_steps_per_iteration = num_inputs / batch_size`, the - algorithm becomes an asynchronous version of the full-batch algorithm. - However, there is no guarantee by this implementation that each input - is seen exactly once per iteration. Also, different updates are applied - asynchronously without locking. So this asynchronous version may not - behave exactly like a full-batch version. - - Args: - num_clusters: An integer tensor specifying the number of clusters. This - argument is ignored if `initial_clusters` is a tensor or numpy array. - model_dir: The directory to save the model results and log files. - initial_clusters: Specifies how the initial cluster centers are chosen. - One of the following: * a tensor or numpy array with the initial cluster - centers. * a callable `f(inputs, k)` that selects and returns up to - `k` centers from an input batch. `f` is free to return any number of - centers from `0` to `k`. It will be invoked on successive input - batches as necessary until all `num_clusters` centers are chosen. - * `KMeansClustering.RANDOM_INIT`: Choose centers randomly from an input - batch. If the batch size is less than `num_clusters` then the entire - batch is chosen to be initial cluster centers and the remaining - centers are chosen from successive input batches. - * `KMeansClustering.KMEANS_PLUS_PLUS_INIT`: Use kmeans++ to choose - centers from the first input batch. If the batch size is less than - `num_clusters`, a TensorFlow runtime error occurs. - distance_metric: The distance metric used for clustering. One of: - * `KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE`: Euclidean distance - between vectors `u` and `v` is defined as \\(||u - v||_2\\) which is - the square root of the sum of the absolute squares of the elements' - difference. - * `KMeansClustering.COSINE_DISTANCE`: Cosine distance between vectors - `u` and `v` is defined as \\(1 - (u . v) / (||u||_2 ||v||_2)\\). - random_seed: Python integer. Seed for PRNG used to initialize centers. - use_mini_batch: A boolean specifying whether to use the mini-batch k-means - algorithm. See explanation above. - mini_batch_steps_per_iteration: The number of steps after which the - updated cluster centers are synced back to a master copy. Used only if - `use_mini_batch=True`. See explanation above. - kmeans_plus_plus_num_retries: For each point that is sampled during - kmeans++ initialization, this parameter specifies the number of - additional points to draw from the current distribution before selecting - the best. If a negative value is specified, a heuristic is used to - sample `O(log(num_to_sample))` additional points. Used only if - `initial_clusters=KMeansClustering.KMEANS_PLUS_PLUS_INIT`. - relative_tolerance: A relative tolerance of change in the loss between - iterations. Stops learning if the loss changes less than this amount. - This may not work correctly if `use_mini_batch=True`. - config: See `tf.estimator.Estimator`. - feature_columns: An optionable iterable containing all the feature columns - used by the model. All items in the set should be feature column - instances that can be passed to - `tf.compat.v1.feature_column.input_layer`. If this is None, all features - will be used. - - Raises: - ValueError: An invalid argument was passed to `initial_clusters` or - `distance_metric`. - """ - if isinstance(initial_clusters, str) and initial_clusters not in [ - KMeansClustering.RANDOM_INIT, KMeansClustering.KMEANS_PLUS_PLUS_INIT - ]: - raise ValueError("Unsupported initialization algorithm '%s'" % - initial_clusters) - if distance_metric not in [ - KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE, - KMeansClustering.COSINE_DISTANCE - ]: - raise ValueError("Unsupported distance metric '%s'" % distance_metric) - super(KMeansClustering, self).__init__( - model_fn=_ModelFn(num_clusters, initial_clusters, distance_metric, - random_seed, use_mini_batch, - mini_batch_steps_per_iteration, - kmeans_plus_plus_num_retries, relative_tolerance, - feature_columns).model_fn, - model_dir=model_dir, - config=config) - - def _predict_one_key(self, input_fn, predict_key): - for result in self.predict(input_fn=input_fn, predict_keys=[predict_key]): - yield result[predict_key] - - def predict_cluster_index(self, input_fn): - """Finds the index of the closest cluster center to each input point. - - Args: - input_fn: Input points. See `tf.estimator.Estimator.predict`. - - Yields: - The index of the closest cluster center for each input point. - """ - for index in self._predict_one_key(input_fn, - KMeansClustering.CLUSTER_INDEX): - yield index - - def score(self, input_fn): - """Returns the sum of squared distances to nearest clusters. - - Note that this function is different from the corresponding one in sklearn - which returns the negative sum. - - Args: - input_fn: Input points. See `tf.estimator.Estimator.evaluate`. Only one - batch is retrieved. - - Returns: - The sum of the squared distance from each point in the first batch of - inputs to its nearest cluster center. - """ - return self.evaluate(input_fn=input_fn, steps=1)[KMeansClustering.SCORE] - - def transform(self, input_fn): - """Transforms each input point to its distances to all cluster centers. - - Note that if `distance_metric=KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE`, - this - function returns the squared Euclidean distance while the corresponding - sklearn function returns the Euclidean distance. - - Args: - input_fn: Input points. See `tf.estimator.Estimator.predict`. - - Yields: - The distances from each input point to each cluster center. - """ - for distances in self._predict_one_key(input_fn, - KMeansClustering.ALL_DISTANCES): - yield distances - - def cluster_centers(self): - """Returns the cluster centers.""" - return self.get_variable_value(KMeansClustering.CLUSTER_CENTERS_VAR_NAME) diff --git a/tensorflow/contrib/factorization/python/ops/kmeans_test.py b/tensorflow/contrib/factorization/python/ops/kmeans_test.py deleted file mode 100644 index 2f7cd131d3e..00000000000 --- a/tensorflow/contrib/factorization/python/ops/kmeans_test.py +++ /dev/null @@ -1,616 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for KMeans.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import time - -import numpy as np -from sklearn.cluster import KMeans as SklearnKMeans - -# pylint: disable=g-import-not-at-top -from tensorflow.contrib.factorization.python.ops import kmeans as kmeans_lib -from tensorflow.python.estimator import run_config -from tensorflow.python.feature_column import feature_column_lib as fc -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import benchmark -from tensorflow.python.platform import flags -from tensorflow.python.platform import test -from tensorflow.python.training import input as input_lib -from tensorflow.python.training import queue_runner - -FLAGS = flags.FLAGS - - -def normalize(x): - return x / np.sqrt(np.sum(x * x, axis=-1, keepdims=True)) - - -def cosine_similarity(x, y): - return np.dot(normalize(x), np.transpose(normalize(y))) - - -def make_random_centers(num_centers, num_dims, center_norm=500): - return np.round( - np.random.rand(num_centers, num_dims).astype(np.float32) * center_norm) - - -def make_random_points(centers, num_points, max_offset=20): - num_centers, num_dims = centers.shape - assignments = np.random.choice(num_centers, num_points) - offsets = np.round( - np.random.randn(num_points, num_dims).astype(np.float32) * max_offset) - return (centers[assignments] + offsets, assignments, np.add.reduce( - offsets * offsets, 1)) - - -class KMeansTestBase(test.TestCase): - - def input_fn(self, - batch_size=None, - points=None, - randomize=None, - num_epochs=None): - """Returns an input_fn that randomly selects batches from given points.""" - batch_size = batch_size or self.batch_size - points = points if points is not None else self.points - num_points = points.shape[0] - if randomize is None: - randomize = (self.use_mini_batch and - self.mini_batch_steps_per_iteration <= 1) - - def _fn(): - x = constant_op.constant(points) - if batch_size == num_points: - return input_lib.limit_epochs(x, num_epochs=num_epochs), None - if randomize: - indices = random_ops.random_uniform( - constant_op.constant([batch_size]), - minval=0, - maxval=num_points - 1, - dtype=dtypes.int32, - seed=10) - else: - # We need to cycle through the indices sequentially. We create a queue - # to maintain the list of indices. - q = data_flow_ops.FIFOQueue(num_points, dtypes.int32, ()) - - # Conditionally initialize the Queue. - def _init_q(): - with ops.control_dependencies( - [q.enqueue_many(math_ops.range(num_points))]): - return control_flow_ops.no_op() - - init_q = control_flow_ops.cond(q.size() <= 0, _init_q, - control_flow_ops.no_op) - with ops.control_dependencies([init_q]): - offsets = q.dequeue_many(batch_size) - with ops.control_dependencies([q.enqueue_many(offsets)]): - indices = array_ops.identity(offsets) - batch = array_ops.gather(x, indices) - return (input_lib.limit_epochs(batch, num_epochs=num_epochs), None) - - return _fn - - @staticmethod - def config(tf_random_seed): - return run_config.RunConfig().replace(tf_random_seed=tf_random_seed) - - @property - def initial_clusters(self): - return kmeans_lib.KMeansClustering.KMEANS_PLUS_PLUS_INIT - - @property - def batch_size(self): - return self.num_points - - @property - def use_mini_batch(self): - return False - - @property - def mini_batch_steps_per_iteration(self): - return 1 - - -class KMeansTest(KMeansTestBase): - - def setUp(self): - np.random.seed(3) - self.num_centers = 5 - self.num_dims = 2 - self.num_points = 1000 - self.true_centers = make_random_centers(self.num_centers, self.num_dims) - self.points, _, self.scores = make_random_points(self.true_centers, - self.num_points) - self.true_score = np.add.reduce(self.scores) - - def _kmeans(self, relative_tolerance=None): - return kmeans_lib.KMeansClustering( - self.num_centers, - initial_clusters=self.initial_clusters, - distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE, - use_mini_batch=self.use_mini_batch, - mini_batch_steps_per_iteration=self.mini_batch_steps_per_iteration, - random_seed=24, - relative_tolerance=relative_tolerance) - - def test_clusters(self): - kmeans = self._kmeans() - kmeans.train(input_fn=self.input_fn(), steps=1) - clusters = kmeans.cluster_centers() - self.assertAllEqual(list(clusters.shape), [self.num_centers, self.num_dims]) - - def test_fit(self): - kmeans = self._kmeans() - kmeans.train(input_fn=self.input_fn(), steps=1) - score1 = kmeans.score(input_fn=self.input_fn(batch_size=self.num_points)) - steps = 10 * self.num_points // self.batch_size - kmeans.train(input_fn=self.input_fn(), steps=steps) - score2 = kmeans.score(input_fn=self.input_fn(batch_size=self.num_points)) - self.assertTrue(score1 > score2) - self.assertNear(self.true_score, score2, self.true_score * 0.05) - - def test_monitor(self): - if self.use_mini_batch: - # We don't test for use_mini_batch case since the loss value can be noisy. - return - kmeans = kmeans_lib.KMeansClustering( - self.num_centers, - initial_clusters=self.initial_clusters, - distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE, - use_mini_batch=self.use_mini_batch, - mini_batch_steps_per_iteration=self.mini_batch_steps_per_iteration, - config=self.config(14), - random_seed=12, - relative_tolerance=1e-4) - - kmeans.train( - input_fn=self.input_fn(), - # Force it to train until the relative tolerance monitor stops it. - steps=None) - score = kmeans.score(input_fn=self.input_fn(batch_size=self.num_points)) - self.assertNear(self.true_score, score, self.true_score * 0.01) - - def _infer_helper(self, kmeans, clusters, num_points): - points, true_assignments, true_offsets = make_random_points( - clusters, num_points) - input_fn = self.input_fn(batch_size=num_points, points=points, num_epochs=1) - # Test predict - assignments = list(kmeans.predict_cluster_index(input_fn)) - self.assertAllEqual(assignments, true_assignments) - - # Test score - score = kmeans.score(input_fn=lambda: (constant_op.constant(points), None)) - self.assertNear(score, np.sum(true_offsets), 0.01 * score) - - # Test transform - transform = list(kmeans.transform(input_fn)) - true_transform = np.maximum( - 0, - np.sum(np.square(points), axis=1, keepdims=True) - - 2 * np.dot(points, np.transpose(clusters)) + np.transpose( - np.sum(np.square(clusters), axis=1, keepdims=True))) - self.assertAllClose(transform, true_transform, rtol=0.05, atol=10) - - def test_infer(self): - kmeans = self._kmeans() - # Make a call to fit to initialize the cluster centers. - max_steps = 1 - kmeans.train(input_fn=self.input_fn(), max_steps=max_steps) - clusters = kmeans.cluster_centers() - - # Run inference on small datasets. - self._infer_helper(kmeans, clusters, 10) - self._infer_helper(kmeans, clusters, 1) - - def _parse_feature_dict_helper(self, features, parsed_feature_dict): - # Perform a sanity check. - self.assertEqual(features.shape, parsed_feature_dict.shape) - self.assertEqual(features.dtype, parsed_feature_dict.dtype) - # Then check that running the tensor yields the original list of points. - with self.cached_session() as sess: - parsed_points = sess.run(parsed_feature_dict) - self.assertAllEqual(self.points, parsed_points) - - def test_parse_features(self): - """Tests the various behaviours of kmeans._parse_features_if_necessary.""" - - # No-op if a tensor is passed in. - features = constant_op.constant(self.points) - parsed_features = kmeans_lib._parse_features_if_necessary(features, None) - self.assertAllEqual(features, parsed_features) - - # All values from a feature dict are transformed into a tensor. - feature_dict = { - 'x': [[point[0]] for point in self.points], - 'y': [[point[1]] for point in self.points] - } - parsed_feature_dict = kmeans_lib._parse_features_if_necessary( - feature_dict, None) - self._parse_feature_dict_helper(features, parsed_feature_dict) - - # Only the feature_columns of a feature dict are transformed into a tensor. - feature_dict_with_extras = { - 'foo': 'bar', - 'x': [[point[0]] for point in self.points], - 'baz': {'fizz': 'buzz'}, - 'y': [[point[1]] for point in self.points] - } - feature_columns = [fc.numeric_column(key='x'), fc.numeric_column(key='y')] - parsed_feature_dict = kmeans_lib._parse_features_if_necessary( - feature_dict_with_extras, feature_columns) - self._parse_feature_dict_helper(features, parsed_feature_dict) - - -class KMeansTestMultiStageInit(KMeansTestBase): - - def test_random(self): - points = np.array( - [[1, 2], [3, 4], [5, 6], [7, 8], [9, 0]], dtype=np.float32) - kmeans = kmeans_lib.KMeansClustering( - num_clusters=points.shape[0], - initial_clusters=kmeans_lib.KMeansClustering.RANDOM_INIT, - distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE, - use_mini_batch=True, - mini_batch_steps_per_iteration=100, - random_seed=24, - relative_tolerance=None) - kmeans.train( - input_fn=self.input_fn(batch_size=1, points=points, randomize=False), - steps=1) - clusters = kmeans.cluster_centers() - self.assertAllEqual(points, clusters) - - def test_kmeans_plus_plus_batch_just_right(self): - points = np.array([[1, 2]], dtype=np.float32) - kmeans = kmeans_lib.KMeansClustering( - num_clusters=points.shape[0], - initial_clusters=kmeans_lib.KMeansClustering.KMEANS_PLUS_PLUS_INIT, - distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE, - use_mini_batch=True, - mini_batch_steps_per_iteration=100, - random_seed=24, - relative_tolerance=None) - kmeans.train( - input_fn=self.input_fn(batch_size=1, points=points, randomize=False), - steps=1) - clusters = kmeans.cluster_centers() - self.assertAllEqual(points, clusters) - - def test_kmeans_plus_plus_batch_too_small(self): - points = np.array( - [[1, 2], [3, 4], [5, 6], [7, 8], [9, 0]], dtype=np.float32) - kmeans = kmeans_lib.KMeansClustering( - num_clusters=points.shape[0], - initial_clusters=kmeans_lib.KMeansClustering.KMEANS_PLUS_PLUS_INIT, - distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE, - use_mini_batch=True, - mini_batch_steps_per_iteration=100, - random_seed=24, - relative_tolerance=None) - with self.assertRaisesOpError(AssertionError): - kmeans.train( - input_fn=self.input_fn(batch_size=4, points=points, randomize=False), - steps=1) - - -class MiniBatchKMeansTest(KMeansTest): - - @property - def batch_size(self): - return 50 - - @property - def use_mini_batch(self): - return True - - -class FullBatchAsyncKMeansTest(KMeansTest): - - @property - def batch_size(self): - return 50 - - @property - def use_mini_batch(self): - return True - - @property - def mini_batch_steps_per_iteration(self): - return self.num_points // self.batch_size - - -class KMeansCosineDistanceTest(KMeansTestBase): - - def setUp(self): - self.points = np.array( - [[2.5, 0.1], [2, 0.2], [3, 0.1], [4, 0.2], [0.1, 2.5], [0.2, 2], - [0.1, 3], [0.2, 4]], - dtype=np.float32) - self.num_points = self.points.shape[0] - self.true_centers = np.array( - [ - normalize( - np.mean(normalize(self.points)[0:4, :], axis=0, - keepdims=True))[0], - normalize( - np.mean(normalize(self.points)[4:, :], axis=0, - keepdims=True))[0] - ], - dtype=np.float32) - self.true_assignments = np.array([0] * 4 + [1] * 4) - self.true_score = len(self.points) - np.tensordot( - normalize(self.points), self.true_centers[self.true_assignments]) - - self.num_centers = 2 - self.kmeans = kmeans_lib.KMeansClustering( - self.num_centers, - initial_clusters=kmeans_lib.KMeansClustering.RANDOM_INIT, - distance_metric=kmeans_lib.KMeansClustering.COSINE_DISTANCE, - use_mini_batch=self.use_mini_batch, - mini_batch_steps_per_iteration=self.mini_batch_steps_per_iteration, - config=self.config(3)) - - def test_fit(self): - max_steps = 10 * self.num_points // self.batch_size - self.kmeans.train(input_fn=self.input_fn(), max_steps=max_steps) - centers = normalize(self.kmeans.cluster_centers()) - centers = centers[centers[:, 0].argsort()] - true_centers = self.true_centers[self.true_centers[:, 0].argsort()] - self.assertAllClose(centers, true_centers, atol=0.04) - - def test_transform(self): - self.kmeans.train(input_fn=self.input_fn(), steps=10) - centers = normalize(self.kmeans.cluster_centers()) - true_transform = 1 - cosine_similarity(self.points, centers) - transform = list( - self.kmeans.transform( - input_fn=self.input_fn(batch_size=self.num_points, num_epochs=1))) - self.assertAllClose(transform, true_transform, atol=1e-3) - - def test_predict(self): - max_steps = 10 * self.num_points // self.batch_size - self.kmeans.train(input_fn=self.input_fn(), max_steps=max_steps) - centers = normalize(self.kmeans.cluster_centers()) - - assignments = list( - self.kmeans.predict_cluster_index( - input_fn=self.input_fn(num_epochs=1, batch_size=self.num_points))) - self.assertAllClose( - centers[assignments], - self.true_centers[self.true_assignments], - atol=1e-2) - - centers = centers[centers[:, 0].argsort()] - true_centers = self.true_centers[self.true_centers[:, 0].argsort()] - self.assertAllClose(centers, true_centers, atol=0.04) - score = self.kmeans.score( - input_fn=self.input_fn(batch_size=self.num_points)) - self.assertAllClose(score, self.true_score, atol=1e-2) - - def test_predict_kmeans_plus_plus(self): - # Most points are concentrated near one center. KMeans++ is likely to find - # the less populated centers. - points = np.array( - [[2.5, 3.5], [2.5, 3.5], [-2, 3], [-2, 3], [-3, -3], [-3.1, -3.2], - [-2.8, -3.], [-2.9, -3.1], [-3., -3.1], [-3., -3.1], [-3.2, -3.], - [-3., -3.]], - dtype=np.float32) - true_centers = np.array( - [ - normalize( - np.mean(normalize(points)[0:2, :], axis=0, keepdims=True))[0], - normalize( - np.mean(normalize(points)[2:4, :], axis=0, keepdims=True))[0], - normalize(np.mean(normalize(points)[4:, :], axis=0, - keepdims=True))[0] - ], - dtype=np.float32) - true_assignments = [0] * 2 + [1] * 2 + [2] * 8 - true_score = len(points) - np.tensordot( - normalize(points), true_centers[true_assignments]) - kmeans = kmeans_lib.KMeansClustering( - 3, - initial_clusters=self.initial_clusters, - distance_metric=kmeans_lib.KMeansClustering.COSINE_DISTANCE, - use_mini_batch=self.use_mini_batch, - mini_batch_steps_per_iteration=self.mini_batch_steps_per_iteration, - config=self.config(3)) - kmeans.train( - input_fn=lambda: (constant_op.constant(points), None), steps=30) - - centers = normalize(kmeans.cluster_centers()) - self.assertAllClose( - sorted(centers.tolist()), sorted(true_centers.tolist()), atol=1e-2) - - def _input_fn(): - return (input_lib.limit_epochs( - constant_op.constant(points), num_epochs=1), None) - - assignments = list(kmeans.predict_cluster_index(input_fn=_input_fn)) - self.assertAllClose( - centers[assignments], true_centers[true_assignments], atol=1e-2) - - score = kmeans.score(input_fn=lambda: (constant_op.constant(points), None)) - self.assertAllClose(score, true_score, atol=1e-2) - - -class MiniBatchKMeansCosineTest(KMeansCosineDistanceTest): - - @property - def batch_size(self): - return 2 - - @property - def use_mini_batch(self): - return True - - -class FullBatchAsyncKMeansCosineTest(KMeansCosineDistanceTest): - - @property - def batch_size(self): - return 2 - - @property - def use_mini_batch(self): - return True - - @property - def mini_batch_steps_per_iteration(self): - return self.num_points // self.batch_size - - -class KMeansBenchmark(benchmark.Benchmark): - """Base class for benchmarks.""" - - def SetUp(self, - dimension=50, - num_clusters=50, - points_per_cluster=10000, - center_norm=500, - cluster_width=20): - np.random.seed(123456) - self.num_clusters = num_clusters - self.num_points = num_clusters * points_per_cluster - self.centers = make_random_centers( - self.num_clusters, dimension, center_norm=center_norm) - self.points, _, scores = make_random_points( - self.centers, self.num_points, max_offset=cluster_width) - self.score = float(np.sum(scores)) - - def _report(self, num_iters, start, end, scores): - print(scores) - self.report_benchmark( - iters=num_iters, - wall_time=(end - start) / num_iters, - extras={'true_sum_squared_distances': self.score, - 'fit_scores': scores}) - - def _fit(self, num_iters=10): - pass - - def benchmark_01_2dim_5center_500point(self): - self.SetUp(dimension=2, num_clusters=5, points_per_cluster=100) - self._fit() - - def benchmark_02_20dim_20center_10kpoint(self): - self.SetUp(dimension=20, num_clusters=20, points_per_cluster=500) - self._fit() - - def benchmark_03_100dim_50center_50kpoint(self): - self.SetUp(dimension=100, num_clusters=50, points_per_cluster=1000) - self._fit() - - def benchmark_03_100dim_50center_50kpoint_unseparated(self): - self.SetUp( - dimension=100, - num_clusters=50, - points_per_cluster=1000, - cluster_width=250) - self._fit() - - def benchmark_04_100dim_500center_500kpoint(self): - self.SetUp(dimension=100, num_clusters=500, points_per_cluster=1000) - self._fit(num_iters=4) - - def benchmark_05_100dim_500center_500kpoint_unseparated(self): - self.SetUp( - dimension=100, - num_clusters=500, - points_per_cluster=1000, - cluster_width=250) - self._fit(num_iters=4) - - -class TensorflowKMeansBenchmark(KMeansBenchmark): - - def _fit(self, num_iters=10): - scores = [] - start = time.time() - for i in range(num_iters): - print('Starting tensorflow KMeans: %d' % i) - tf_kmeans = kmeans_lib.KMeansClustering( - self.num_clusters, - initial_clusters=kmeans_lib.KMeansClustering.KMEANS_PLUS_PLUS_INIT, - kmeans_plus_plus_num_retries=int(math.log(self.num_clusters) + 2), - random_seed=i * 42, - relative_tolerance=1e-6, - config=self.config(3)) - tf_kmeans.train( - input_fn=lambda: (constant_op.constant(self.points), None), steps=50) - _ = tf_kmeans.cluster_centers() - scores.append( - tf_kmeans.score( - input_fn=lambda: (constant_op.constant(self.points), None))) - self._report(num_iters, start, time.time(), scores) - - -class SklearnKMeansBenchmark(KMeansBenchmark): - - def _fit(self, num_iters=10): - scores = [] - start = time.time() - for i in range(num_iters): - print('Starting sklearn KMeans: %d' % i) - sklearn_kmeans = SklearnKMeans( - n_clusters=self.num_clusters, - init='k-means++', - max_iter=50, - n_init=1, - tol=1e-4, - random_state=i * 42) - sklearn_kmeans.train(self.points) - scores.append(sklearn_kmeans.inertia_) - self._report(num_iters, start, time.time(), scores) - - -class KMeansTestQueues(test.TestCase): - - def input_fn(self): - - def _fn(): - queue = data_flow_ops.FIFOQueue( - capacity=10, dtypes=dtypes.float32, shapes=[10, 3]) - enqueue_op = queue.enqueue(array_ops.zeros([10, 3], dtype=dtypes.float32)) - queue_runner.add_queue_runner( - queue_runner.QueueRunner(queue, [enqueue_op])) - return queue.dequeue(), None - - return _fn - - # This test makes sure that there are no deadlocks when using a QueueRunner. - # Note that since cluster initialization is dependent on inputs, if input - # is generated using a QueueRunner, one has to make sure that these runners - # are started before the initialization. - def test_queues(self): - kmeans = kmeans_lib.KMeansClustering(5) - kmeans.train(input_fn=self.input_fn(), steps=1) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/factorization/python/ops/wals.py b/tensorflow/contrib/factorization/python/ops/wals.py deleted file mode 100644 index b82bf1188f0..00000000000 --- a/tensorflow/contrib/factorization/python/ops/wals.py +++ /dev/null @@ -1,609 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Weighted Alternating Least Squares (WALS) on the tf.learn API.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.factorization.python.ops import factorization_ops -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.summary import summary -from tensorflow.python.training import session_run_hook -from tensorflow.python.training import training_util - - -class _SweepHook(session_run_hook.SessionRunHook): - """Keeps track of row/col sweeps, and runs prep ops before each sweep.""" - - def __init__(self, is_row_sweep_var, is_sweep_done_var, init_op, - row_prep_ops, col_prep_ops, row_train_op, col_train_op, - switch_op): - """Initializes SweepHook. - - Args: - is_row_sweep_var: A Boolean tf.Variable, determines whether we are - currently doing a row or column sweep. It is updated by the hook. - is_sweep_done_var: A Boolean tf.Variable, determines whether we are - starting a new sweep (this is used to determine when to run the prep ops - below). - init_op: op to be run once before training. This is typically a local - initialization op (such as cache initialization). - row_prep_ops: A list of TensorFlow ops, to be run before the beginning of - each row sweep (and during initialization), in the given order. - col_prep_ops: A list of TensorFlow ops, to be run before the beginning of - each column sweep (and during initialization), in the given order. - row_train_op: A TensorFlow op to be run during row sweeps. - col_train_op: A TensorFlow op to be run during column sweeps. - switch_op: A TensorFlow op to be run before each sweep. - """ - self._is_row_sweep_var = is_row_sweep_var - self._is_sweep_done_var = is_sweep_done_var - self._init_op = init_op - self._row_prep_ops = row_prep_ops - self._col_prep_ops = col_prep_ops - self._row_train_op = row_train_op - self._col_train_op = col_train_op - self._switch_op = switch_op - # Boolean variable that determines whether the init_op has been run. - self._is_initialized = False - - def before_run(self, run_context): - """Runs the appropriate prep ops, and requests running update ops.""" - sess = run_context.session - is_sweep_done = sess.run(self._is_sweep_done_var) - if not self._is_initialized: - logging.info("SweepHook running init op.") - sess.run(self._init_op) - if is_sweep_done: - logging.info("SweepHook starting the next sweep.") - sess.run(self._switch_op) - is_row_sweep = sess.run(self._is_row_sweep_var) - if is_sweep_done or not self._is_initialized: - logging.info("SweepHook running prep ops for the {} sweep.".format( - "row" if is_row_sweep else "col")) - prep_ops = self._row_prep_ops if is_row_sweep else self._col_prep_ops - for prep_op in prep_ops: - sess.run(prep_op) - self._is_initialized = True - logging.info("Next fit step starting.") - return session_run_hook.SessionRunArgs( - fetches=[self._row_train_op if is_row_sweep else self._col_train_op]) - - -class _IncrementGlobalStepHook(session_run_hook.SessionRunHook): - """Hook that increments the global step.""" - - def __init__(self): - global_step = training_util.get_global_step() - if global_step: - self._global_step_incr_op = state_ops.assign_add( - global_step, 1, name="global_step_incr").op - else: - self._global_step_incr_op = None - - def before_run(self, run_context): - if self._global_step_incr_op: - run_context.session.run(self._global_step_incr_op) - - -class _StopAtSweepHook(session_run_hook.SessionRunHook): - """Hook that requests stop at a given sweep.""" - - def __init__(self, last_sweep): - """Initializes a `StopAtSweepHook`. - - This hook requests stop at a given sweep. Relies on the tensor named - COMPLETED_SWEEPS in the default graph. - - Args: - last_sweep: Integer, number of the last sweep to run. - """ - self._last_sweep = last_sweep - - def begin(self): - try: - self._completed_sweeps_var = ops.get_default_graph().get_tensor_by_name( - WALSMatrixFactorization.COMPLETED_SWEEPS + ":0") - except KeyError: - raise RuntimeError(WALSMatrixFactorization.COMPLETED_SWEEPS + - " counter should be created to use StopAtSweepHook.") - - def before_run(self, run_context): - return session_run_hook.SessionRunArgs(self._completed_sweeps_var) - - def after_run(self, run_context, run_values): - completed_sweeps = run_values.results - if completed_sweeps >= self._last_sweep: - run_context.request_stop() - - -def _wals_factorization_model_function(features, labels, mode, params): - """Model function for the WALSFactorization estimator. - - Args: - features: Dictionary of features. See WALSMatrixFactorization. - labels: Must be None. - mode: A model_fn.ModeKeys object. - params: Dictionary of parameters containing arguments passed to the - WALSMatrixFactorization constructor. - - Returns: - A ModelFnOps object. - - Raises: - ValueError: If `mode` is not recognized. - """ - assert labels is None - use_factors_weights_cache = (params["use_factors_weights_cache_for_training"] - and mode == model_fn.ModeKeys.TRAIN) - use_gramian_cache = (params["use_gramian_cache_for_training"] and - mode == model_fn.ModeKeys.TRAIN) - max_sweeps = params["max_sweeps"] - model = factorization_ops.WALSModel( - params["num_rows"], - params["num_cols"], - params["embedding_dimension"], - unobserved_weight=params["unobserved_weight"], - regularization=params["regularization_coeff"], - row_init=params["row_init"], - col_init=params["col_init"], - num_row_shards=params["num_row_shards"], - num_col_shards=params["num_col_shards"], - row_weights=params["row_weights"], - col_weights=params["col_weights"], - use_factors_weights_cache=use_factors_weights_cache, - use_gramian_cache=use_gramian_cache) - - # Get input rows and cols. We either update rows or columns depending on - # the value of row_sweep, which is maintained using a session hook. - input_rows = features[WALSMatrixFactorization.INPUT_ROWS] - input_cols = features[WALSMatrixFactorization.INPUT_COLS] - - # TRAIN mode: - if mode == model_fn.ModeKeys.TRAIN: - # Training consists of the following ops (controlled using a SweepHook). - # Before a row sweep: - # row_update_prep_gramian_op - # initialize_row_update_op - # During a row sweep: - # update_row_factors_op - # Before a col sweep: - # col_update_prep_gramian_op - # initialize_col_update_op - # During a col sweep: - # update_col_factors_op - - is_row_sweep_var = variable_scope.variable( - True, - trainable=False, - name="is_row_sweep", - collections=[ops.GraphKeys.GLOBAL_VARIABLES]) - is_sweep_done_var = variable_scope.variable( - False, - trainable=False, - name="is_sweep_done", - collections=[ops.GraphKeys.GLOBAL_VARIABLES]) - completed_sweeps_var = variable_scope.variable( - 0, - trainable=False, - name=WALSMatrixFactorization.COMPLETED_SWEEPS, - collections=[ops.GraphKeys.GLOBAL_VARIABLES]) - loss_var = variable_scope.variable( - 0., - trainable=False, - name=WALSMatrixFactorization.LOSS, - collections=[ops.GraphKeys.GLOBAL_VARIABLES]) - # The root weighted squared error = - # \\(\sqrt( \sum_{i,j} w_ij * (a_ij - r_ij)^2 / \sum_{i,j} w_ij )\\) - rwse_var = variable_scope.variable( - 0., - trainable=False, - name=WALSMatrixFactorization.RWSE, - collections=[ops.GraphKeys.GLOBAL_VARIABLES]) - - summary.scalar("loss", loss_var) - summary.scalar("root_weighted_squared_error", rwse_var) - summary.scalar("completed_sweeps", completed_sweeps_var) - - def create_axis_ops(sp_input, num_items, update_fn, axis_name): - """Creates book-keeping and training ops for a given axis. - - Args: - sp_input: A SparseTensor corresponding to the row or column batch. - num_items: An integer, the total number of items of this axis. - update_fn: A function that takes one argument (`sp_input`), and that - returns a tuple of - * new_factors: A float Tensor of the factor values after update. - * update_op: a TensorFlow op which updates the factors. - * loss: A float Tensor, the unregularized loss. - * reg_loss: A float Tensor, the regularization loss. - * sum_weights: A float Tensor, the sum of factor weights. - axis_name: A string that specifies the name of the axis. - - Returns: - A tuple consisting of: - * reset_processed_items_op: A TensorFlow op, to be run before the - beginning of any sweep. It marks all items as not-processed. - * axis_train_op: A Tensorflow op, to be run during this axis' sweeps. - """ - processed_items_init = array_ops.fill(dims=[num_items], value=False) - with ops.colocate_with(processed_items_init): - processed_items = variable_scope.variable( - processed_items_init, - collections=[ops.GraphKeys.GLOBAL_VARIABLES], - trainable=False, - name="processed_" + axis_name) - _, update_op, loss, reg, sum_weights = update_fn(sp_input) - input_indices = sp_input.indices[:, 0] - with ops.control_dependencies([ - update_op, - state_ops.assign(loss_var, loss + reg), - state_ops.assign(rwse_var, math_ops.sqrt(loss / sum_weights))]): - with ops.colocate_with(processed_items): - update_processed_items = state_ops.scatter_update( - processed_items, - input_indices, - array_ops.ones_like(input_indices, dtype=dtypes.bool), - name="update_processed_{}_indices".format(axis_name)) - with ops.control_dependencies([update_processed_items]): - is_sweep_done = math_ops.reduce_all(processed_items) - axis_train_op = control_flow_ops.group( - state_ops.assign(is_sweep_done_var, is_sweep_done), - state_ops.assign_add( - completed_sweeps_var, - math_ops.cast(is_sweep_done, dtypes.int32)), - name="{}_sweep_train_op".format(axis_name)) - return processed_items.initializer, axis_train_op - - reset_processed_rows_op, row_train_op = create_axis_ops( - input_rows, - params["num_rows"], - lambda x: model.update_row_factors(sp_input=x, transpose_input=False), - "rows") - reset_processed_cols_op, col_train_op = create_axis_ops( - input_cols, - params["num_cols"], - lambda x: model.update_col_factors(sp_input=x, transpose_input=True), - "cols") - switch_op = control_flow_ops.group( - state_ops.assign( - is_row_sweep_var, math_ops.logical_not(is_row_sweep_var)), - reset_processed_rows_op, - reset_processed_cols_op, - name="sweep_switch_op") - row_prep_ops = [ - model.row_update_prep_gramian_op, model.initialize_row_update_op] - col_prep_ops = [ - model.col_update_prep_gramian_op, model.initialize_col_update_op] - init_op = model.worker_init - sweep_hook = _SweepHook( - is_row_sweep_var, is_sweep_done_var, init_op, - row_prep_ops, col_prep_ops, row_train_op, col_train_op, switch_op) - global_step_hook = _IncrementGlobalStepHook() - training_hooks = [sweep_hook, global_step_hook] - if max_sweeps is not None: - training_hooks.append(_StopAtSweepHook(max_sweeps)) - - return model_fn.ModelFnOps( - mode=model_fn.ModeKeys.TRAIN, - predictions={}, - loss=loss_var, - eval_metric_ops={}, - train_op=control_flow_ops.no_op(), - training_hooks=training_hooks) - - # INFER mode - elif mode == model_fn.ModeKeys.INFER: - projection_weights = features.get( - WALSMatrixFactorization.PROJECTION_WEIGHTS) - - def get_row_projection(): - return model.project_row_factors( - sp_input=input_rows, - projection_weights=projection_weights, - transpose_input=False) - - def get_col_projection(): - return model.project_col_factors( - sp_input=input_cols, - projection_weights=projection_weights, - transpose_input=True) - - predictions = { - WALSMatrixFactorization.PROJECTION_RESULT: control_flow_ops.cond( - features[WALSMatrixFactorization.PROJECT_ROW], - get_row_projection, - get_col_projection) - } - - return model_fn.ModelFnOps( - mode=model_fn.ModeKeys.INFER, - predictions=predictions, - loss=None, - eval_metric_ops={}, - train_op=control_flow_ops.no_op(), - training_hooks=[]) - - # EVAL mode - elif mode == model_fn.ModeKeys.EVAL: - def get_row_loss(): - _, _, loss, reg, _ = model.update_row_factors( - sp_input=input_rows, transpose_input=False) - return loss + reg - def get_col_loss(): - _, _, loss, reg, _ = model.update_col_factors( - sp_input=input_cols, transpose_input=True) - return loss + reg - loss = control_flow_ops.cond( - features[WALSMatrixFactorization.PROJECT_ROW], - get_row_loss, - get_col_loss) - return model_fn.ModelFnOps( - mode=model_fn.ModeKeys.EVAL, - predictions={}, - loss=loss, - eval_metric_ops={}, - train_op=control_flow_ops.no_op(), - training_hooks=[]) - - else: - raise ValueError("mode=%s is not recognized." % str(mode)) - - -class WALSMatrixFactorization(estimator.Estimator): - """An Estimator for Weighted Matrix Factorization, using the WALS method. - - WALS (Weighted Alternating Least Squares) is an algorithm for weighted matrix - factorization. It computes a low-rank approximation of a given sparse (n x m) - matrix `A`, by a product of two matrices, `U * V^T`, where `U` is a (n x k) - matrix and `V` is a (m x k) matrix. Here k is the rank of the approximation, - also called the embedding dimension. We refer to `U` as the row factors, and - `V` as the column factors. - See tensorflow/contrib/factorization/g3doc/wals.md for the precise problem - formulation. - - The training proceeds in sweeps: during a row_sweep, we fix `V` and solve for - `U`. During a column sweep, we fix `U` and solve for `V`. Each one of these - problems is an unconstrained quadratic minimization problem and can be solved - exactly (it can also be solved in mini-batches, since the solution decouples - across rows of each matrix). - The alternating between sweeps is achieved by using a hook during training, - which is responsible for keeping track of the sweeps and running preparation - ops at the beginning of each sweep. It also updates the global_step variable, - which keeps track of the number of batches processed since the beginning of - training. - The current implementation assumes that the training is run on a single - machine, and will fail if `config.num_worker_replicas` is not equal to one. - Training is done by calling `self.fit(input_fn=input_fn)`, where `input_fn` - provides two tensors: one for rows of the input matrix, and one for rows of - the transposed input matrix (i.e. columns of the original matrix). Note that - during a row sweep, only row batches are processed (ignoring column batches) - and vice-versa. - Also note that every row (respectively every column) of the input matrix - must be processed at least once for the sweep to be considered complete. In - particular, training will not make progress if some rows are not generated by - the `input_fn`. - - For prediction, given a new set of input rows `A'`, we compute a corresponding - set of row factors `U'`, such that `U' * V^T` is a good approximation of `A'`. - We call this operation a row projection. A similar operation is defined for - columns. Projection is done by calling - `self.get_projections(input_fn=input_fn)`, where `input_fn` satisfies the - constraints given below. - - The input functions must satisfy the following constraints: Calling `input_fn` - must return a tuple `(features, labels)` where `labels` is None, and - `features` is a dict containing the following keys: - - TRAIN: - * `WALSMatrixFactorization.INPUT_ROWS`: float32 SparseTensor (matrix). - Rows of the input matrix to process (or to project). - * `WALSMatrixFactorization.INPUT_COLS`: float32 SparseTensor (matrix). - Columns of the input matrix to process (or to project), transposed. - - INFER: - * `WALSMatrixFactorization.INPUT_ROWS`: float32 SparseTensor (matrix). - Rows to project. - * `WALSMatrixFactorization.INPUT_COLS`: float32 SparseTensor (matrix). - Columns to project. - * `WALSMatrixFactorization.PROJECT_ROW`: Boolean Tensor. Whether to project - the rows or columns. - * `WALSMatrixFactorization.PROJECTION_WEIGHTS` (Optional): float32 Tensor - (vector). The weights to use in the projection. - - EVAL: - * `WALSMatrixFactorization.INPUT_ROWS`: float32 SparseTensor (matrix). - Rows to project. - * `WALSMatrixFactorization.INPUT_COLS`: float32 SparseTensor (matrix). - Columns to project. - * `WALSMatrixFactorization.PROJECT_ROW`: Boolean Tensor. Whether to project - the rows or columns. - """ - # Keys to be used in model_fn - # Features keys - INPUT_ROWS = "input_rows" - INPUT_COLS = "input_cols" - PROJECT_ROW = "project_row" - PROJECTION_WEIGHTS = "projection_weights" - # Predictions key - PROJECTION_RESULT = "projection" - # Name of the completed_sweeps variable - COMPLETED_SWEEPS = "completed_sweeps" - # Name of the loss variable - LOSS = "WALS_loss" - # Name of the Root Weighted Squared Error variable - RWSE = "WALS_RWSE" - - def __init__(self, - num_rows, - num_cols, - embedding_dimension, - unobserved_weight=0.1, - regularization_coeff=None, - row_init="random", - col_init="random", - num_row_shards=1, - num_col_shards=1, - row_weights=1, - col_weights=1, - use_factors_weights_cache_for_training=True, - use_gramian_cache_for_training=True, - max_sweeps=None, - model_dir=None, - config=None): - r"""Creates a model for matrix factorization using the WALS method. - - Args: - num_rows: Total number of rows for input matrix. - num_cols: Total number of cols for input matrix. - embedding_dimension: Dimension to use for the factors. - unobserved_weight: Weight of the unobserved entries of matrix. - regularization_coeff: Weight of the L2 regularization term. Defaults to - None, in which case the problem is not regularized. - row_init: Initializer for row factor. Must be either: - - A tensor: The row factor matrix is initialized to this tensor, - - A numpy constant, - - "random": The rows are initialized using a normal distribution. - col_init: Initializer for column factor. See row_init. - num_row_shards: Number of shards to use for the row factors. - num_col_shards: Number of shards to use for the column factors. - row_weights: Must be in one of the following three formats: - - None: In this case, the weight of every entry is the unobserved_weight - and the problem simplifies to ALS. Note that, in this case, - col_weights must also be set to "None". - - List of lists of non-negative scalars, of the form - \\([[w_0, w_1, ...], [w_k, ... ], [...]]\\), - where the number of inner lists equal to the number of row factor - shards and the elements in each inner list are the weights for the - rows of that shard. In this case, - \\(w_ij = unonbserved_weight + row_weights[i] * col_weights[j]\\). - - A non-negative scalar: This value is used for all row weights. - Note that it is allowed to have row_weights as a list and col_weights - as a scalar, or vice-versa. - col_weights: See row_weights. - use_factors_weights_cache_for_training: Boolean, whether the factors and - weights will be cached on the workers before the updates start, during - training. Defaults to True. - Note that caching is disabled during prediction. - use_gramian_cache_for_training: Boolean, whether the Gramians will be - cached on the workers before the updates start, during training. - Defaults to True. Note that caching is disabled during prediction. - max_sweeps: integer, optional. Specifies the number of sweeps for which - to train the model, where a sweep is defined as a full update of all the - row factors (resp. column factors). - If `steps` or `max_steps` is also specified in model.fit(), training - stops when either of the steps condition or sweeps condition is met. - model_dir: The directory to save the model results and log files. - config: A Configuration object. See Estimator. - - Raises: - ValueError: If config.num_worker_replicas is strictly greater than one. - The current implementation only supports running on a single worker. - """ - # TODO(walidk): Support power-law based weight computation. - # TODO(walidk): Add factor lookup by indices, with caching. - # TODO(walidk): Support caching during prediction. - # TODO(walidk): Provide input pipelines that handle missing rows. - - params = { - "num_rows": - num_rows, - "num_cols": - num_cols, - "embedding_dimension": - embedding_dimension, - "unobserved_weight": - unobserved_weight, - "regularization_coeff": - regularization_coeff, - "row_init": - row_init, - "col_init": - col_init, - "num_row_shards": - num_row_shards, - "num_col_shards": - num_col_shards, - "row_weights": - row_weights, - "col_weights": - col_weights, - "max_sweeps": - max_sweeps, - "use_factors_weights_cache_for_training": - use_factors_weights_cache_for_training, - "use_gramian_cache_for_training": - use_gramian_cache_for_training - } - self._row_factors_names = [ - "row_factors_shard_%d" % i for i in range(num_row_shards) - ] - self._col_factors_names = [ - "col_factors_shard_%d" % i for i in range(num_col_shards) - ] - - super(WALSMatrixFactorization, self).__init__( - model_fn=_wals_factorization_model_function, - params=params, - model_dir=model_dir, - config=config) - - if self._config is not None and self._config.num_worker_replicas > 1: - raise ValueError("WALSMatrixFactorization must be run on a single worker " - "replica.") - - def get_row_factors(self): - """Returns the row factors of the model, loading them from checkpoint. - - Should only be run after training. - - Returns: - A list of the row factors of the model. - """ - return [self.get_variable_value(name) for name in self._row_factors_names] - - def get_col_factors(self): - """Returns the column factors of the model, loading them from checkpoint. - - Should only be run after training. - - Returns: - A list of the column factors of the model. - """ - return [self.get_variable_value(name) for name in self._col_factors_names] - - def get_projections(self, input_fn): - """Computes the projections of the rows or columns given in input_fn. - - Runs predict() with the given input_fn, and returns the results. Should only - be run after training. - - Args: - input_fn: Input function which specifies the rows or columns to project. - Returns: - A generator of the projected factors. - """ - return (result[WALSMatrixFactorization.PROJECTION_RESULT] - for result in self.predict(input_fn=input_fn)) diff --git a/tensorflow/contrib/factorization/python/ops/wals_test.py b/tensorflow/contrib/factorization/python/ops/wals_test.py deleted file mode 100644 index 75d577f4295..00000000000 --- a/tensorflow/contrib/factorization/python/ops/wals_test.py +++ /dev/null @@ -1,506 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for WALSMatrixFactorization.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools -import json -import numpy as np - -from tensorflow.contrib.factorization.python.ops import factorization_ops_test_utils -from tensorflow.contrib.factorization.python.ops import wals as wals_lib -from tensorflow.contrib.learn.python.learn import run_config -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.estimators import run_config as run_config_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import input as input_lib -from tensorflow.python.training import monitored_session - - -class WALSMatrixFactorizationTest(test.TestCase): - INPUT_MATRIX = factorization_ops_test_utils.INPUT_MATRIX - - def np_array_to_sparse(self, np_array): - """Transforms an np.array to a tf.SparseTensor.""" - return factorization_ops_test_utils.np_matrix_to_tf_sparse(np_array) - - def calculate_loss(self): - """Calculates the loss of the current (trained) model.""" - current_rows = embedding_ops.embedding_lookup( - self._model.get_row_factors(), math_ops.range(self._num_rows), - partition_strategy='div') - current_cols = embedding_ops.embedding_lookup( - self._model.get_col_factors(), math_ops.range(self._num_cols), - partition_strategy='div') - row_wts = embedding_ops.embedding_lookup( - self._row_weights, math_ops.range(self._num_rows), - partition_strategy='div') - col_wts = embedding_ops.embedding_lookup( - self._col_weights, math_ops.range(self._num_cols), - partition_strategy='div') - sp_inputs = self.np_array_to_sparse(self.INPUT_MATRIX) - return factorization_ops_test_utils.calculate_loss( - sp_inputs, current_rows, current_cols, self._regularization_coeff, - self._unobserved_weight, row_wts, col_wts) - - # TODO(walidk): Replace with input_reader_utils functions once open sourced. - def remap_sparse_tensor_rows(self, sp_x, row_ids, shape): - """Remaps the row ids of a tf.SparseTensor.""" - old_row_ids, old_col_ids = array_ops.split( - value=sp_x.indices, num_or_size_splits=2, axis=1) - new_row_ids = array_ops.gather(row_ids, old_row_ids) - new_indices = array_ops.concat([new_row_ids, old_col_ids], 1) - return sparse_tensor.SparseTensor( - indices=new_indices, values=sp_x.values, dense_shape=shape) - - # TODO(walidk): Add an option to shuffle inputs. - def input_fn(self, np_matrix, batch_size, mode, - project_row=None, projection_weights=None, - remove_empty_rows_columns=False): - """Returns an input_fn that selects row and col batches from np_matrix. - - This simple utility creates an input function from a numpy_array. The - following transformations are performed: - * The empty rows and columns in np_matrix are removed (if - remove_empty_rows_columns is true) - * np_matrix is converted to a SparseTensor. - * The rows of the sparse matrix (and the rows of its transpose) are batched. - * A features dictionary is created, which contains the row / column batches. - - In TRAIN mode, one only needs to specify the np_matrix and the batch_size. - In INFER and EVAL modes, one must also provide project_row, a boolean which - specifies whether we are projecting rows or columns. - - Args: - np_matrix: A numpy array. The input matrix to use. - batch_size: Integer. - mode: Can be one of model_fn.ModeKeys.{TRAIN, INFER, EVAL}. - project_row: A boolean. Used in INFER and EVAL modes. Specifies whether - to project rows or columns. - projection_weights: A float numpy array. Used in INFER mode. Specifies - the weights to use in the projection (the weights are optional, and - default to 1.). - remove_empty_rows_columns: A boolean. When true, this will remove empty - rows and columns in the np_matrix. Note that this will result in - modifying the indices of the input matrix. The mapping from new indices - to old indices is returned in the form of two numpy arrays. - - Returns: - A tuple consisting of: - _fn: A callable. Calling _fn returns a features dict. - nz_row_ids: A numpy array of the ids of non-empty rows, such that - nz_row_ids[i] is the old row index corresponding to new index i. - nz_col_ids: A numpy array of the ids of non-empty columns, such that - nz_col_ids[j] is the old column index corresponding to new index j. - """ - if remove_empty_rows_columns: - np_matrix, nz_row_ids, nz_col_ids = ( - factorization_ops_test_utils.remove_empty_rows_columns(np_matrix)) - else: - nz_row_ids = np.arange(np.shape(np_matrix)[0]) - nz_col_ids = np.arange(np.shape(np_matrix)[1]) - - def extract_features(row_batch, col_batch, num_rows, num_cols): - row_ids = row_batch[0] - col_ids = col_batch[0] - rows = self.remap_sparse_tensor_rows( - row_batch[1], row_ids, shape=[num_rows, num_cols]) - cols = self.remap_sparse_tensor_rows( - col_batch[1], col_ids, shape=[num_cols, num_rows]) - features = { - wals_lib.WALSMatrixFactorization.INPUT_ROWS: rows, - wals_lib.WALSMatrixFactorization.INPUT_COLS: cols, - } - return features - - def _fn(): - num_rows = np.shape(np_matrix)[0] - num_cols = np.shape(np_matrix)[1] - row_ids = math_ops.range(num_rows, dtype=dtypes.int64) - col_ids = math_ops.range(num_cols, dtype=dtypes.int64) - sp_mat = self.np_array_to_sparse(np_matrix) - sp_mat_t = sparse_ops.sparse_transpose(sp_mat) - row_batch = input_lib.batch( - [row_ids, sp_mat], - batch_size=min(batch_size, num_rows), - capacity=10, - enqueue_many=True) - col_batch = input_lib.batch( - [col_ids, sp_mat_t], - batch_size=min(batch_size, num_cols), - capacity=10, - enqueue_many=True) - - features = extract_features(row_batch, col_batch, num_rows, num_cols) - - if mode == model_fn.ModeKeys.INFER or mode == model_fn.ModeKeys.EVAL: - self.assertTrue( - project_row is not None, - msg='project_row must be specified in INFER or EVAL mode.') - features[wals_lib.WALSMatrixFactorization.PROJECT_ROW] = ( - constant_op.constant(project_row)) - - if mode == model_fn.ModeKeys.INFER and projection_weights is not None: - weights_batch = input_lib.batch( - projection_weights, - batch_size=batch_size, - capacity=10, - enqueue_many=True) - features[wals_lib.WALSMatrixFactorization.PROJECTION_WEIGHTS] = ( - weights_batch) - - labels = None - return features, labels - - return _fn, nz_row_ids, nz_col_ids - - @property - def input_matrix(self): - return self.INPUT_MATRIX - - @property - def row_steps(self): - return np.ceil(self._num_rows / self.batch_size) - - @property - def col_steps(self): - return np.ceil(self._num_cols / self.batch_size) - - @property - def batch_size(self): - return 5 - - @property - def use_cache(self): - return False - - @property - def max_sweeps(self): - return None - - def setUp(self): - self._num_rows = 5 - self._num_cols = 7 - self._embedding_dimension = 3 - self._unobserved_weight = 0.1 - self._num_row_shards = 2 - self._num_col_shards = 3 - self._regularization_coeff = 0.01 - self._col_init = [ - # Shard 0. - [[-0.36444709, -0.39077035, -0.32528427], - [1.19056475, 0.07231052, 2.11834812], - [0.93468881, -0.71099287, 1.91826844]], - # Shard 1. - [[1.18160152, 1.52490723, -0.50015002], - [1.82574749, -0.57515913, -1.32810032]], - # Shard 2. - [[-0.15515432, -0.84675711, 0.13097958], - [-0.9246484, 0.69117504, 1.2036494]], - ] - self._row_weights = [[0.1, 0.2, 0.3], [0.4, 0.5]] - self._col_weights = [[0.1, 0.2, 0.3], [0.4, 0.5], [0.6, 0.7]] - - # Values of row and column factors after running one iteration or factor - # updates. - self._row_factors_0 = [[0.097689, -0.219293, -0.020780], - [0.50842, 0.64626, 0.22364], - [0.401159, -0.046558, -0.192854]] - self._row_factors_1 = [[1.20597, -0.48025, 0.35582], - [1.5564, 1.2528, 1.0528]] - self._col_factors_0 = [[2.4725, -1.2950, -1.9980], - [0.44625, 1.50771, 1.27118], - [1.39801, -2.10134, 0.73572]] - self._col_factors_1 = [[3.36509, -0.66595, -3.51208], - [0.57191, 1.59407, 1.33020]] - self._col_factors_2 = [[3.3459, -1.3341, -3.3008], - [0.57366, 1.83729, 1.26798]] - self._model = wals_lib.WALSMatrixFactorization( - self._num_rows, - self._num_cols, - self._embedding_dimension, - self._unobserved_weight, - col_init=self._col_init, - regularization_coeff=self._regularization_coeff, - num_row_shards=self._num_row_shards, - num_col_shards=self._num_col_shards, - row_weights=self._row_weights, - col_weights=self._col_weights, - max_sweeps=self.max_sweeps, - use_factors_weights_cache_for_training=self.use_cache, - use_gramian_cache_for_training=self.use_cache) - - def test_fit(self): - # Row sweep. - input_fn = self.input_fn(np_matrix=self.input_matrix, - batch_size=self.batch_size, - mode=model_fn.ModeKeys.TRAIN, - remove_empty_rows_columns=True)[0] - self._model.fit(input_fn=input_fn, steps=self.row_steps) - row_factors = self._model.get_row_factors() - self.assertAllClose(row_factors[0], self._row_factors_0, atol=1e-3) - self.assertAllClose(row_factors[1], self._row_factors_1, atol=1e-3) - - # Col sweep. - # Running fit a second time will resume training from the checkpoint. - input_fn = self.input_fn(np_matrix=self.input_matrix, - batch_size=self.batch_size, - mode=model_fn.ModeKeys.TRAIN, - remove_empty_rows_columns=True)[0] - self._model.fit(input_fn=input_fn, steps=self.col_steps) - col_factors = self._model.get_col_factors() - self.assertAllClose(col_factors[0], self._col_factors_0, atol=1e-3) - self.assertAllClose(col_factors[1], self._col_factors_1, atol=1e-3) - self.assertAllClose(col_factors[2], self._col_factors_2, atol=1e-3) - - def test_predict(self): - input_fn = self.input_fn(np_matrix=self.input_matrix, - batch_size=self.batch_size, - mode=model_fn.ModeKeys.TRAIN, - remove_empty_rows_columns=True, - )[0] - # Project rows 1 and 4 from the input matrix. - proj_input_fn = self.input_fn( - np_matrix=self.INPUT_MATRIX[[1, 4], :], - batch_size=2, - mode=model_fn.ModeKeys.INFER, - project_row=True, - projection_weights=[[0.2, 0.5]])[0] - - self._model.fit(input_fn=input_fn, steps=self.row_steps) - projections = self._model.get_projections(proj_input_fn) - projected_rows = list(itertools.islice(projections, 2)) - - self.assertAllClose( - projected_rows, - [self._row_factors_0[1], self._row_factors_1[1]], - atol=1e-3) - - # Project columns 5, 3, 1 from the input matrix. - proj_input_fn = self.input_fn( - np_matrix=self.INPUT_MATRIX[:, [5, 3, 1]], - batch_size=3, - mode=model_fn.ModeKeys.INFER, - project_row=False, - projection_weights=[[0.6, 0.4, 0.2]])[0] - - self._model.fit(input_fn=input_fn, steps=self.col_steps) - projections = self._model.get_projections(proj_input_fn) - projected_cols = list(itertools.islice(projections, 3)) - self.assertAllClose( - projected_cols, - [self._col_factors_2[0], self._col_factors_1[0], - self._col_factors_0[1]], - atol=1e-3) - - def test_eval(self): - # Do a row sweep then evaluate the model on row inputs. - # The evaluate function returns the loss of the projected rows, but since - # projection is idempotent, the eval loss must match the model loss. - input_fn = self.input_fn(np_matrix=self.input_matrix, - batch_size=self.batch_size, - mode=model_fn.ModeKeys.TRAIN, - remove_empty_rows_columns=True, - )[0] - self._model.fit(input_fn=input_fn, steps=self.row_steps) - eval_input_fn_row = self.input_fn(np_matrix=self.input_matrix, - batch_size=1, - mode=model_fn.ModeKeys.EVAL, - project_row=True, - remove_empty_rows_columns=True)[0] - loss = self._model.evaluate( - input_fn=eval_input_fn_row, steps=self._num_rows)['loss'] - - with self.cached_session(): - true_loss = self.calculate_loss() - - self.assertNear( - loss, true_loss, err=.001, - msg="""After row update, eval loss = {}, does not match the true - loss = {}.""".format(loss, true_loss)) - - # Do a col sweep then evaluate the model on col inputs. - self._model.fit(input_fn=input_fn, steps=self.col_steps) - eval_input_fn_col = self.input_fn(np_matrix=self.input_matrix, - batch_size=1, - mode=model_fn.ModeKeys.EVAL, - project_row=False, - remove_empty_rows_columns=True)[0] - loss = self._model.evaluate( - input_fn=eval_input_fn_col, steps=self._num_cols)['loss'] - - with self.cached_session(): - true_loss = self.calculate_loss() - - self.assertNear( - loss, true_loss, err=.001, - msg="""After col update, eval loss = {}, does not match the true - loss = {}.""".format(loss, true_loss)) - - -class WALSMatrixFactorizationTestSweeps(WALSMatrixFactorizationTest): - - @property - def max_sweeps(self): - return 2 - - # We set the column steps to None so that we rely only on max_sweeps to stop - # training. - @property - def col_steps(self): - return None - - -class WALSMatrixFactorizationTestCached(WALSMatrixFactorizationTest): - - @property - def use_cache(self): - return True - - -class WALSMatrixFactorizaiontTestPaddedInput(WALSMatrixFactorizationTest): - PADDED_INPUT_MATRIX = np.pad( - WALSMatrixFactorizationTest.INPUT_MATRIX, - [(1, 0), (1, 0)], mode='constant') - - @property - def input_matrix(self): - return self.PADDED_INPUT_MATRIX - - -class WALSMatrixFactorizationUnsupportedTest(test.TestCase): - - def setUp(self): - pass - - def testDistributedWALSUnsupported(self): - tf_config = { - 'cluster': { - run_config_lib.TaskType.PS: ['host1:1', 'host2:2'], - run_config_lib.TaskType.WORKER: ['host3:3', 'host4:4'] - }, - 'task': { - 'type': run_config_lib.TaskType.WORKER, - 'index': 1 - } - } - with test.mock.patch.dict('os.environ', - {'TF_CONFIG': json.dumps(tf_config)}): - config = run_config.RunConfig() - self.assertEqual(config.num_worker_replicas, 2) - with self.assertRaises(ValueError): - self._model = wals_lib.WALSMatrixFactorization(1, 1, 1, config=config) - - -class SweepHookTest(test.TestCase): - - def test_sweeps(self): - is_row_sweep_var = variables.VariableV1(True) - is_sweep_done_var = variables.VariableV1(False) - init_done = variables.VariableV1(False) - row_prep_done = variables.VariableV1(False) - col_prep_done = variables.VariableV1(False) - row_train_done = variables.VariableV1(False) - col_train_done = variables.VariableV1(False) - - init_op = state_ops.assign(init_done, True) - row_prep_op = state_ops.assign(row_prep_done, True) - col_prep_op = state_ops.assign(col_prep_done, True) - row_train_op = state_ops.assign(row_train_done, True) - col_train_op = state_ops.assign(col_train_done, True) - train_op = control_flow_ops.no_op() - switch_op = control_flow_ops.group( - state_ops.assign(is_sweep_done_var, False), - state_ops.assign(is_row_sweep_var, - math_ops.logical_not(is_row_sweep_var))) - mark_sweep_done = state_ops.assign(is_sweep_done_var, True) - - with self.cached_session() as sess: - sweep_hook = wals_lib._SweepHook( - is_row_sweep_var, - is_sweep_done_var, - init_op, - [row_prep_op], - [col_prep_op], - row_train_op, - col_train_op, - switch_op) - mon_sess = monitored_session._HookedSession(sess, [sweep_hook]) - sess.run([variables.global_variables_initializer()]) - - # Row sweep. - mon_sess.run(train_op) - self.assertTrue(sess.run(init_done), - msg='init op not run by the Sweephook') - self.assertTrue(sess.run(row_prep_done), - msg='row_prep_op not run by the SweepHook') - self.assertTrue(sess.run(row_train_done), - msg='row_train_op not run by the SweepHook') - self.assertTrue( - sess.run(is_row_sweep_var), - msg='Row sweep is not complete but is_row_sweep_var is False.') - # Col sweep. - mon_sess.run(mark_sweep_done) - mon_sess.run(train_op) - self.assertTrue(sess.run(col_prep_done), - msg='col_prep_op not run by the SweepHook') - self.assertTrue(sess.run(col_train_done), - msg='col_train_op not run by the SweepHook') - self.assertFalse( - sess.run(is_row_sweep_var), - msg='Col sweep is not complete but is_row_sweep_var is True.') - # Row sweep. - mon_sess.run(mark_sweep_done) - mon_sess.run(train_op) - self.assertTrue( - sess.run(is_row_sweep_var), - msg='Col sweep is complete but is_row_sweep_var is False.') - - -class StopAtSweepHookTest(test.TestCase): - - def test_stop(self): - hook = wals_lib._StopAtSweepHook(last_sweep=10) - completed_sweeps = variables.VariableV1( - 8, name=wals_lib.WALSMatrixFactorization.COMPLETED_SWEEPS) - train_op = state_ops.assign_add(completed_sweeps, 1) - hook.begin() - - with self.cached_session() as sess: - sess.run([variables.global_variables_initializer()]) - mon_sess = monitored_session._HookedSession(sess, [hook]) - mon_sess.run(train_op) - # completed_sweeps is 9 after running train_op. - self.assertFalse(mon_sess.should_stop()) - mon_sess.run(train_op) - # completed_sweeps is 10 after running train_op. - self.assertTrue(mon_sess.should_stop()) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/feature_column/BUILD b/tensorflow/contrib/feature_column/BUILD deleted file mode 100644 index 9b1156faa33..00000000000 --- a/tensorflow/contrib/feature_column/BUILD +++ /dev/null @@ -1,73 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "tf_py_test") - -package( - default_visibility = [ - "//tensorflow:internal", - ], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "feature_column_py", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - deps = [ - ":sequence_feature_column", - "//tensorflow/python:util", - ], -) - -py_library( - name = "sequence_feature_column", - srcs = ["python/feature_column/sequence_feature_column.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:variable_scope", - "//tensorflow/python/feature_column:feature_column_py", - "//tensorflow/python/feature_column:utils", - ], -) - -tf_py_test( - name = "sequence_feature_column_test", - srcs = ["python/feature_column/sequence_feature_column_test.py"], - additional_deps = [ - ":sequence_feature_column", - "@absl_py//absl/testing:parameterized", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:training", - "//tensorflow/python/feature_column:feature_column_py", - ], - tags = ["no_pip"], -) - -tf_py_test( - name = "sequence_feature_column_integration_test", - srcs = ["python/feature_column/sequence_feature_column_integration_test.py"], - additional_deps = [ - ":sequence_feature_column", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python/feature_column:feature_column_py", - "//tensorflow/python/keras:layers", - ], - tags = ["no_pip"], -) diff --git a/tensorflow/contrib/feature_column/__init__.py b/tensorflow/contrib/feature_column/__init__.py deleted file mode 100644 index baa8c1567a5..00000000000 --- a/tensorflow/contrib/feature_column/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Experimental utilities for tf.feature_column.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,line-too-long,wildcard-import -from tensorflow.contrib.feature_column.python.feature_column.sequence_feature_column import * - -from tensorflow.python.util.all_util import remove_undocumented -# pylint: enable=unused-import,line-too-long,wildcard-import - -_allowed_symbols = [ - 'sequence_categorical_column_with_hash_bucket', - 'sequence_categorical_column_with_identity', - 'sequence_categorical_column_with_vocabulary_list', - 'sequence_categorical_column_with_vocabulary_file', - 'sequence_input_layer', - 'sequence_numeric_column', -] - -remove_undocumented(__name__, allowed_exception_list=_allowed_symbols) diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py deleted file mode 100644 index 456304ac61c..00000000000 --- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py +++ /dev/null @@ -1,516 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Experimental methods for tf.feature_column sequence input.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -import collections - - -from tensorflow.python.feature_column import feature_column as fc -from tensorflow.python.feature_column import utils as fc_utils -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import variable_scope - -# pylint: disable=protected-access - - -def sequence_input_layer( - features, - feature_columns, - weight_collections=None, - trainable=True): - """"Builds input layer for sequence input. - - All `feature_columns` must be sequence dense columns with the same - `sequence_length`. The output of this method can be fed into sequence - networks, such as RNN. - - The output of this method is a 3D `Tensor` of shape `[batch_size, T, D]`. - `T` is the maximum sequence length for this batch, which could differ from - batch to batch. - - If multiple `feature_columns` are given with `Di` `num_elements` each, their - outputs are concatenated. So, the final `Tensor` has shape - `[batch_size, T, D0 + D1 + ... + Dn]`. - - Example: - - ```python - rating = sequence_numeric_column('rating') - watches = sequence_categorical_column_with_identity( - 'watches', num_buckets=1000) - watches_embedding = embedding_column(watches, dimension=10) - columns = [rating, watches] - - features = tf.io.parse_example(..., features=make_parse_example_spec(columns)) - input_layer, sequence_length = sequence_input_layer(features, columns) - - rnn_cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(hidden_size) - outputs, state = tf.compat.v1.nn.dynamic_rnn( - rnn_cell, inputs=input_layer, sequence_length=sequence_length) - ``` - - Args: - features: A dict mapping keys to tensors. - feature_columns: An iterable of dense sequence columns. Valid columns are - - `embedding_column` that wraps a `sequence_categorical_column_with_*` - - `sequence_numeric_column`. - weight_collections: A list of collection names to which the Variable will be - added. Note that variables will also be added to collections - `tf.GraphKeys.GLOBAL_VARIABLES` and `ops.GraphKeys.MODEL_VARIABLES`. - trainable: If `True` also add the variable to the graph collection - `GraphKeys.TRAINABLE_VARIABLES`. - - Returns: - An `(input_layer, sequence_length)` tuple where: - - input_layer: A float `Tensor` of shape `[batch_size, T, D]`. - `T` is the maximum sequence length for this batch, which could differ - from batch to batch. `D` is the sum of `num_elements` for all - `feature_columns`. - - sequence_length: An int `Tensor` of shape `[batch_size]`. The sequence - length for each example. - - Raises: - ValueError: If any of the `feature_columns` is the wrong type. - """ - feature_columns = fc._normalize_feature_columns(feature_columns) - for c in feature_columns: - if not isinstance(c, fc._SequenceDenseColumn): - raise ValueError( - 'All feature_columns must be of type _SequenceDenseColumn. ' - 'You can wrap a sequence_categorical_column with an embedding_column ' - 'or indicator_column. ' - 'Given (type {}): {}'.format(type(c), c)) - - with variable_scope.variable_scope( - None, default_name='sequence_input_layer', values=features.values()): - builder = fc._LazyBuilder(features) - output_tensors = [] - sequence_lengths = [] - ordered_columns = [] - - for column in sorted(feature_columns, key=lambda x: x.name): - ordered_columns.append(column) - with variable_scope.variable_scope( - None, default_name=column._var_scope_name): - dense_tensor, sequence_length = column._get_sequence_dense_tensor( - builder, - weight_collections=weight_collections, - trainable=trainable) - # Flattens the final dimension to produce a 3D Tensor. - num_elements = column._variable_shape.num_elements() - shape = array_ops.shape(dense_tensor) - target_shape = [shape[0], shape[1], num_elements] - output_tensors.append( - array_ops.reshape(dense_tensor, shape=target_shape)) - sequence_lengths.append(sequence_length) - - fc._verify_static_batch_size_equality(output_tensors, ordered_columns) - fc._verify_static_batch_size_equality(sequence_lengths, ordered_columns) - sequence_length = _assert_all_equal_and_return(sequence_lengths) - - return array_ops.concat(output_tensors, -1), sequence_length - - -def concatenate_context_input(context_input, sequence_input): - """Replicates `context_input` across all timesteps of `sequence_input`. - - Expands dimension 1 of `context_input` then tiles it `sequence_length` times. - This value is appended to `sequence_input` on dimension 2 and the result is - returned. - - Args: - context_input: A `Tensor` of dtype `float32` and shape `[batch_size, d1]`. - sequence_input: A `Tensor` of dtype `float32` and shape `[batch_size, - padded_length, d0]`. - - Returns: - A `Tensor` of dtype `float32` and shape `[batch_size, padded_length, - d0 + d1]`. - - Raises: - ValueError: If `sequence_input` does not have rank 3 or `context_input` does - not have rank 2. - """ - seq_rank_check = check_ops.assert_rank( - sequence_input, - 3, - message='sequence_input must have rank 3', - data=[array_ops.shape(sequence_input)]) - seq_type_check = check_ops.assert_type( - sequence_input, - dtypes.float32, - message='sequence_input must have dtype float32; got {}.'.format( - sequence_input.dtype)) - ctx_rank_check = check_ops.assert_rank( - context_input, - 2, - message='context_input must have rank 2', - data=[array_ops.shape(context_input)]) - ctx_type_check = check_ops.assert_type( - context_input, - dtypes.float32, - message='context_input must have dtype float32; got {}.'.format( - context_input.dtype)) - with ops.control_dependencies( - [seq_rank_check, seq_type_check, ctx_rank_check, ctx_type_check]): - padded_length = array_ops.shape(sequence_input)[1] - tiled_context_input = array_ops.tile( - array_ops.expand_dims(context_input, 1), - array_ops.concat([[1], [padded_length], [1]], 0)) - return array_ops.concat([sequence_input, tiled_context_input], 2) - - -def sequence_categorical_column_with_identity( - key, num_buckets, default_value=None): - """Returns a feature column that represents sequences of integers. - - Pass this to `embedding_column` or `indicator_column` to convert sequence - categorical data into dense representation for input to sequence NN, such as - RNN. - - Example: - - ```python - watches = sequence_categorical_column_with_identity( - 'watches', num_buckets=1000) - watches_embedding = embedding_column(watches, dimension=10) - columns = [watches_embedding] - - features = tf.io.parse_example(..., features=make_parse_example_spec(columns)) - input_layer, sequence_length = sequence_input_layer(features, columns) - - rnn_cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(hidden_size) - outputs, state = tf.compat.v1.nn.dynamic_rnn( - rnn_cell, inputs=input_layer, sequence_length=sequence_length) - ``` - - Args: - key: A unique string identifying the input feature. - num_buckets: Range of inputs. Namely, inputs are expected to be in the - range `[0, num_buckets)`. - default_value: If `None`, this column's graph operations will fail for - out-of-range inputs. Otherwise, this value must be in the range - `[0, num_buckets)`, and will replace out-of-range inputs. - - Returns: - A `_SequenceCategoricalColumn`. - - Raises: - ValueError: if `num_buckets` is less than one. - ValueError: if `default_value` is not in range `[0, num_buckets)`. - """ - return fc._SequenceCategoricalColumn( - fc._categorical_column_with_identity( - key=key, num_buckets=num_buckets, default_value=default_value)) - - -def sequence_categorical_column_with_hash_bucket( - key, hash_bucket_size, dtype=dtypes.string): - """A sequence of categorical terms where ids are set by hashing. - - Pass this to `embedding_column` or `indicator_column` to convert sequence - categorical data into dense representation for input to sequence NN, such as - RNN. - - Example: - - ```python - tokens = sequence_categorical_column_with_hash_bucket( - 'tokens', hash_bucket_size=1000) - tokens_embedding = embedding_column(tokens, dimension=10) - columns = [tokens_embedding] - - features = tf.io.parse_example(..., features=make_parse_example_spec(columns)) - input_layer, sequence_length = sequence_input_layer(features, columns) - - rnn_cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(hidden_size) - outputs, state = tf.compat.v1.nn.dynamic_rnn( - rnn_cell, inputs=input_layer, sequence_length=sequence_length) - ``` - - Args: - key: A unique string identifying the input feature. - hash_bucket_size: An int > 1. The number of buckets. - dtype: The type of features. Only string and integer types are supported. - - Returns: - A `_SequenceCategoricalColumn`. - - Raises: - ValueError: `hash_bucket_size` is not greater than 1. - ValueError: `dtype` is neither string nor integer. - """ - return fc._SequenceCategoricalColumn( - fc._categorical_column_with_hash_bucket( - key=key, hash_bucket_size=hash_bucket_size, dtype=dtype)) - - -def sequence_categorical_column_with_vocabulary_file( - key, vocabulary_file, vocabulary_size=None, num_oov_buckets=0, - default_value=None, dtype=dtypes.string): - """A sequence of categorical terms where ids use a vocabulary file. - - Pass this to `embedding_column` or `indicator_column` to convert sequence - categorical data into dense representation for input to sequence NN, such as - RNN. - - Example: - - ```python - states = sequence_categorical_column_with_vocabulary_file( - key='states', vocabulary_file='/us/states.txt', vocabulary_size=50, - num_oov_buckets=5) - states_embedding = embedding_column(states, dimension=10) - columns = [states_embedding] - - features = tf.io.parse_example(..., features=make_parse_example_spec(columns)) - input_layer, sequence_length = sequence_input_layer(features, columns) - - rnn_cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(hidden_size) - outputs, state = tf.compat.v1.nn.dynamic_rnn( - rnn_cell, inputs=input_layer, sequence_length=sequence_length) - ``` - - Args: - key: A unique string identifying the input feature. - vocabulary_file: The vocabulary file name. - vocabulary_size: Number of the elements in the vocabulary. This must be no - greater than length of `vocabulary_file`, if less than length, later - values are ignored. If None, it is set to the length of `vocabulary_file`. - num_oov_buckets: Non-negative integer, the number of out-of-vocabulary - buckets. All out-of-vocabulary inputs will be assigned IDs in the range - `[vocabulary_size, vocabulary_size+num_oov_buckets)` based on a hash of - the input value. A positive `num_oov_buckets` can not be specified with - `default_value`. - default_value: The integer ID value to return for out-of-vocabulary feature - values, defaults to `-1`. This can not be specified with a positive - `num_oov_buckets`. - dtype: The type of features. Only string and integer types are supported. - - Returns: - A `_SequenceCategoricalColumn`. - - Raises: - ValueError: `vocabulary_file` is missing or cannot be opened. - ValueError: `vocabulary_size` is missing or < 1. - ValueError: `num_oov_buckets` is a negative integer. - ValueError: `num_oov_buckets` and `default_value` are both specified. - ValueError: `dtype` is neither string nor integer. - """ - return fc._SequenceCategoricalColumn( - fc._categorical_column_with_vocabulary_file( - key=key, - vocabulary_file=vocabulary_file, - vocabulary_size=vocabulary_size, - num_oov_buckets=num_oov_buckets, - default_value=default_value, - dtype=dtype)) - - -def sequence_categorical_column_with_vocabulary_list( - key, vocabulary_list, dtype=None, default_value=-1, num_oov_buckets=0): - """A sequence of categorical terms where ids use an in-memory list. - - Pass this to `embedding_column` or `indicator_column` to convert sequence - categorical data into dense representation for input to sequence NN, such as - RNN. - - Example: - - ```python - colors = sequence_categorical_column_with_vocabulary_list( - key='colors', vocabulary_list=('R', 'G', 'B', 'Y'), - num_oov_buckets=2) - colors_embedding = embedding_column(colors, dimension=3) - columns = [colors_embedding] - - features = tf.io.parse_example(..., features=make_parse_example_spec(columns)) - input_layer, sequence_length = sequence_input_layer(features, columns) - - rnn_cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(hidden_size) - outputs, state = tf.compat.v1.nn.dynamic_rnn( - rnn_cell, inputs=input_layer, sequence_length=sequence_length) - ``` - - Args: - key: A unique string identifying the input feature. - vocabulary_list: An ordered iterable defining the vocabulary. Each feature - is mapped to the index of its value (if present) in `vocabulary_list`. - Must be castable to `dtype`. - dtype: The type of features. Only string and integer types are supported. - If `None`, it will be inferred from `vocabulary_list`. - default_value: The integer ID value to return for out-of-vocabulary feature - values, defaults to `-1`. This can not be specified with a positive - `num_oov_buckets`. - num_oov_buckets: Non-negative integer, the number of out-of-vocabulary - buckets. All out-of-vocabulary inputs will be assigned IDs in the range - `[len(vocabulary_list), len(vocabulary_list)+num_oov_buckets)` based on a - hash of the input value. A positive `num_oov_buckets` can not be specified - with `default_value`. - - Returns: - A `_SequenceCategoricalColumn`. - - Raises: - ValueError: if `vocabulary_list` is empty, or contains duplicate keys. - ValueError: `num_oov_buckets` is a negative integer. - ValueError: `num_oov_buckets` and `default_value` are both specified. - ValueError: if `dtype` is not integer or string. - """ - return fc._SequenceCategoricalColumn( - fc._categorical_column_with_vocabulary_list( - key=key, - vocabulary_list=vocabulary_list, - dtype=dtype, - default_value=default_value, - num_oov_buckets=num_oov_buckets)) - - -def sequence_numeric_column( - key, - shape=(1,), - default_value=0., - dtype=dtypes.float32, - normalizer_fn=None): - """Returns a feature column that represents sequences of numeric data. - - Example: - - ```python - temperature = sequence_numeric_column('temperature') - columns = [temperature] - - features = tf.io.parse_example(..., features=make_parse_example_spec(columns)) - input_layer, sequence_length = sequence_input_layer(features, columns) - - rnn_cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(hidden_size) - outputs, state = tf.compat.v1.nn.dynamic_rnn( - rnn_cell, inputs=input_layer, sequence_length=sequence_length) - ``` - - Args: - key: A unique string identifying the input features. - shape: The shape of the input data per sequence id. E.g. if `shape=(2,)`, - each example must contain `2 * sequence_length` values. - default_value: A single value compatible with `dtype` that is used for - padding the sparse data into a dense `Tensor`. - dtype: The type of values. - normalizer_fn: If not `None`, a function that can be used to normalize the - value of the tensor after `default_value` is applied for parsing. - Normalizer function takes the input `Tensor` as its argument, and returns - the output `Tensor`. (e.g. lambda x: (x - 3.0) / 4.2). Please note that - even though the most common use case of this function is normalization, it - can be used for any kind of Tensorflow transformations. - - Returns: - A `_SequenceNumericColumn`. - - Raises: - TypeError: if any dimension in shape is not an int. - ValueError: if any dimension in shape is not a positive integer. - ValueError: if `dtype` is not convertible to `tf.float32`. - """ - shape = fc._check_shape(shape=shape, key=key) - if not (dtype.is_integer or dtype.is_floating): - raise ValueError('dtype must be convertible to float. ' - 'dtype: {}, key: {}'.format(dtype, key)) - if normalizer_fn is not None and not callable(normalizer_fn): - raise TypeError( - 'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn)) - - return _SequenceNumericColumn( - key, - shape=shape, - default_value=default_value, - dtype=dtype, - normalizer_fn=normalizer_fn) - - -def _assert_all_equal_and_return(tensors, name=None): - """Asserts that all tensors are equal and returns the first one.""" - with ops.name_scope(name, 'assert_all_equal', values=tensors): - if len(tensors) == 1: - return tensors[0] - assert_equal_ops = [] - for t in tensors[1:]: - assert_equal_ops.append(check_ops.assert_equal(tensors[0], t)) - with ops.control_dependencies(assert_equal_ops): - return array_ops.identity(tensors[0]) - - -class _SequenceNumericColumn( - fc._SequenceDenseColumn, - collections.namedtuple( - '_SequenceNumericColumn', - ['key', 'shape', 'default_value', 'dtype', 'normalizer_fn'])): - """Represents sequences of numeric data.""" - - @property - def name(self): - return self.key - - @property - def _parse_example_spec(self): - return {self.key: parsing_ops.VarLenFeature(self.dtype)} - - def _transform_feature(self, inputs): - input_tensor = inputs.get(self.key) - if self.normalizer_fn is not None: - input_tensor = self.normalizer_fn(input_tensor) - return input_tensor - - @property - def _variable_shape(self): - return tensor_shape.TensorShape(self.shape) - - def _get_sequence_dense_tensor( - self, inputs, weight_collections=None, trainable=None): - # Do nothing with weight_collections and trainable since no variables are - # created in this function. - del weight_collections - del trainable - sp_tensor = inputs.get(self) - dense_tensor = sparse_ops.sparse_tensor_to_dense( - sp_tensor, default_value=self.default_value) - # Reshape into [batch_size, T, variable_shape]. - dense_shape = array_ops.concat( - [array_ops.shape(dense_tensor)[:1], [-1], self._variable_shape], - axis=0) - dense_tensor = array_ops.reshape(dense_tensor, shape=dense_shape) - - # Get the number of timesteps per example - # For the 2D case, the raw values are grouped according to num_elements; - # for the 3D case, the grouping happens in the third dimension, and - # sequence length is not affected. - num_elements = (self._variable_shape.num_elements() - if sp_tensor.shape.ndims == 2 else 1) - seq_length = fc_utils.sequence_length_from_sparse_tensor( - sp_tensor, num_elements=num_elements) - - return fc._SequenceDenseColumn.TensorSequenceLengthPair( - dense_tensor=dense_tensor, sequence_length=seq_length) - -# pylint: enable=protected-access diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_integration_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_integration_test.py deleted file mode 100644 index d4f4d657975..00000000000 --- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_integration_test.py +++ /dev/null @@ -1,281 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Integration test for sequence feature columns with SequenceExamples.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import string -import tempfile - -from google.protobuf import text_format - -from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column as sfc -from tensorflow.core.example import example_pb2 -from tensorflow.core.example import feature_pb2 -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.feature_column import feature_column as fc -from tensorflow.python.keras.layers import recurrent -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.util import compat - - -class SequenceFeatureColumnIntegrationTest(test.TestCase): - - def _make_sequence_example(self): - example = example_pb2.SequenceExample() - example.context.feature['int_ctx'].int64_list.value.extend([5]) - example.context.feature['float_ctx'].float_list.value.extend([123.6]) - for val in range(0, 10, 2): - feat = feature_pb2.Feature() - feat.int64_list.value.extend([val] * val) - example.feature_lists.feature_list['int_list'].feature.extend([feat]) - for val in range(1, 11, 2): - feat = feature_pb2.Feature() - feat.bytes_list.value.extend([compat.as_bytes(str(val))] * val) - example.feature_lists.feature_list['str_list'].feature.extend([feat]) - - return example - - def _build_feature_columns(self): - col = fc._categorical_column_with_identity('int_ctx', num_buckets=100) - ctx_cols = [ - fc._embedding_column(col, dimension=10), - fc._numeric_column('float_ctx') - ] - - identity_col = sfc.sequence_categorical_column_with_identity( - 'int_list', num_buckets=10) - bucket_col = sfc.sequence_categorical_column_with_hash_bucket( - 'bytes_list', hash_bucket_size=100) - seq_cols = [ - fc._embedding_column(identity_col, dimension=10), - fc._embedding_column(bucket_col, dimension=20) - ] - - return ctx_cols, seq_cols - - def test_sequence_example_into_input_layer(self): - examples = [_make_sequence_example().SerializeToString()] * 100 - ctx_cols, seq_cols = self._build_feature_columns() - - def _parse_example(example): - ctx, seq = parsing_ops.parse_single_sequence_example( - example, - context_features=fc.make_parse_example_spec(ctx_cols), - sequence_features=fc.make_parse_example_spec(seq_cols)) - ctx.update(seq) - return ctx - - ds = dataset_ops.Dataset.from_tensor_slices(examples) - ds = ds.map(_parse_example) - ds = ds.batch(20) - - # Test on a single batch - features = dataset_ops.make_one_shot_iterator(ds).get_next() - - # Tile the context features across the sequence features - seq_layer, _ = sfc.sequence_input_layer(features, seq_cols) - ctx_layer = fc.input_layer(features, ctx_cols) - input_layer = sfc.concatenate_context_input(ctx_layer, seq_layer) - - rnn_layer = recurrent.RNN(recurrent.SimpleRNNCell(10)) - output = rnn_layer(input_layer) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - features_r = sess.run(features) - self.assertAllEqual(features_r['int_list'].dense_shape, [20, 3, 6]) - - output_r = sess.run(output) - self.assertAllEqual(output_r.shape, [20, 10]) - - -class SequenceExampleParsingTest(test.TestCase): - - def test_seq_ex_in_sequence_categorical_column_with_identity(self): - self._test_parsed_sequence_example( - 'int_list', sfc.sequence_categorical_column_with_identity, - 10, [3, 6], [2, 4, 6]) - - def test_seq_ex_in_sequence_categorical_column_with_hash_bucket(self): - self._test_parsed_sequence_example( - 'bytes_list', sfc.sequence_categorical_column_with_hash_bucket, - 10, [3, 4], [compat.as_bytes(x) for x in 'acg']) - - def test_seq_ex_in_sequence_categorical_column_with_vocabulary_list(self): - self._test_parsed_sequence_example( - 'bytes_list', sfc.sequence_categorical_column_with_vocabulary_list, - list(string.ascii_lowercase), [3, 4], - [compat.as_bytes(x) for x in 'acg']) - - def test_seq_ex_in_sequence_categorical_column_with_vocabulary_file(self): - _, fname = tempfile.mkstemp() - with open(fname, 'w') as f: - f.write(string.ascii_lowercase) - self._test_parsed_sequence_example( - 'bytes_list', sfc.sequence_categorical_column_with_vocabulary_file, - fname, [3, 4], [compat.as_bytes(x) for x in 'acg']) - - def _test_parsed_sequence_example( - self, col_name, col_fn, col_arg, shape, values): - """Helper function to check that each FeatureColumn parses correctly. - - Args: - col_name: string, name to give to the feature column. Should match - the name that the column will parse out of the features dict. - col_fn: function used to create the feature column. For example, - sequence_numeric_column. - col_arg: second arg that the target feature column is expecting. - shape: the expected dense_shape of the feature after parsing into - a SparseTensor. - values: the expected values at index [0, 2, 6] of the feature - after parsing into a SparseTensor. - """ - example = _make_sequence_example() - columns = [ - fc._categorical_column_with_identity('int_ctx', num_buckets=100), - fc._numeric_column('float_ctx'), - col_fn(col_name, col_arg) - ] - context, seq_features = parsing_ops.parse_single_sequence_example( - example.SerializeToString(), - context_features=fc.make_parse_example_spec(columns[:2]), - sequence_features=fc.make_parse_example_spec(columns[2:])) - - with self.cached_session() as sess: - ctx_result, seq_result = sess.run([context, seq_features]) - self.assertEqual(list(seq_result[col_name].dense_shape), shape) - self.assertEqual( - list(seq_result[col_name].values[[0, 2, 6]]), values) - self.assertEqual(list(ctx_result['int_ctx'].dense_shape), [1]) - self.assertEqual(ctx_result['int_ctx'].values[0], 5) - self.assertEqual(list(ctx_result['float_ctx'].shape), [1]) - self.assertAlmostEqual(ctx_result['float_ctx'][0], 123.6, places=1) - - -_SEQ_EX_PROTO = """ -context { - feature { - key: "float_ctx" - value { - float_list { - value: 123.6 - } - } - } - feature { - key: "int_ctx" - value { - int64_list { - value: 5 - } - } - } -} -feature_lists { - feature_list { - key: "bytes_list" - value { - feature { - bytes_list { - value: "a" - } - } - feature { - bytes_list { - value: "b" - value: "c" - } - } - feature { - bytes_list { - value: "d" - value: "e" - value: "f" - value: "g" - } - } - } - } - feature_list { - key: "float_list" - value { - feature { - float_list { - value: 1.0 - } - } - feature { - float_list { - value: 3.0 - value: 3.0 - value: 3.0 - } - } - feature { - float_list { - value: 5.0 - value: 5.0 - value: 5.0 - value: 5.0 - value: 5.0 - } - } - } - } - feature_list { - key: "int_list" - value { - feature { - int64_list { - value: 2 - value: 2 - } - } - feature { - int64_list { - value: 4 - value: 4 - value: 4 - value: 4 - } - } - feature { - int64_list { - value: 6 - value: 6 - value: 6 - value: 6 - value: 6 - value: 6 - } - } - } - } -} -""" - - -def _make_sequence_example(): - example = example_pb2.SequenceExample() - return text_format.Parse(_SEQ_EX_PROTO, example) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py deleted file mode 100644 index d5f74028298..00000000000 --- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py +++ /dev/null @@ -1,1502 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for sequential_feature_column.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -from absl.testing import parameterized -import numpy as np - -from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column as sfc -from tensorflow.python.feature_column import feature_column as fc -from tensorflow.python.feature_column import feature_column_lib as fc_lib -from tensorflow.python.feature_column.feature_column import _LazyBuilder -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.platform import test -from tensorflow.python.training import monitored_session - - -class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'sparse_input_args_a': { - # example 0, ids [2] - # example 1, ids [0, 1] - 'indices': ((0, 0), (1, 0), (1, 1)), - 'values': (2, 0, 1), - 'dense_shape': (2, 2)}, - 'sparse_input_args_b': { - # example 0, ids [1] - # example 1, ids [2, 0] - 'indices': ((0, 0), (1, 0), (1, 1)), - 'values': (1, 2, 0), - 'dense_shape': (2, 2)}, - 'expected_input_layer': [ - # example 0, ids_a [2], ids_b [1] - [[5., 6., 14., 15., 16.], [0., 0., 0., 0., 0.]], - # example 1, ids_a [0, 1], ids_b [2, 0] - [[1., 2., 17., 18., 19.], [3., 4., 11., 12., 13.]],], - 'expected_sequence_length': [1, 2]}, - {'testcase_name': '3D', - 'sparse_input_args_a': { - # feature 0, ids [[2], [0, 1]] - # feature 1, ids [[0, 0], [1]] - 'indices': ( - (0, 0, 0), (0, 1, 0), (0, 1, 1), - (1, 0, 0), (1, 0, 1), (1, 1, 0)), - 'values': (2, 0, 1, 0, 0, 1), - 'dense_shape': (2, 2, 2)}, - 'sparse_input_args_b': { - # feature 0, ids [[1, 1], [1]] - # feature 1, ids [[2], [0]] - 'indices': ((0, 0, 0), (0, 0, 1), (0, 1, 0), (1, 0, 0), (1, 1, 0)), - 'values': (1, 1, 1, 2, 0), - 'dense_shape': (2, 2, 2)}, - 'expected_input_layer': [ - # feature 0, [a: 2, -, b: 1, 1], [a: 0, 1, b: 1, -] - [[5., 6., 14., 15., 16.], [2., 3., 14., 15., 16.]], - # feature 1, [a: 0, 0, b: 2, -], [a: 1, -, b: 0, -] - [[1., 2., 17., 18., 19.], [3., 4., 11., 12., 13.]]], - 'expected_sequence_length': [2, 2]}, - ) - def test_embedding_column( - self, sparse_input_args_a, sparse_input_args_b, expected_input_layer, - expected_sequence_length): - - sparse_input_a = sparse_tensor.SparseTensorValue(**sparse_input_args_a) - sparse_input_b = sparse_tensor.SparseTensorValue(**sparse_input_args_b) - vocabulary_size = 3 - embedding_dimension_a = 2 - embedding_values_a = ( - (1., 2.), # id 0 - (3., 4.), # id 1 - (5., 6.) # id 2 - ) - embedding_dimension_b = 3 - embedding_values_b = ( - (11., 12., 13.), # id 0 - (14., 15., 16.), # id 1 - (17., 18., 19.) # id 2 - ) - def _get_initializer(embedding_dimension, embedding_values): - def _initializer(shape, dtype, partition_info): - self.assertAllEqual((vocabulary_size, embedding_dimension), shape) - self.assertEqual(dtypes.float32, dtype) - self.assertIsNone(partition_info) - return embedding_values - return _initializer - - categorical_column_a = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - embedding_column_a = fc._embedding_column( - categorical_column_a, - dimension=embedding_dimension_a, - initializer=_get_initializer(embedding_dimension_a, embedding_values_a)) - categorical_column_b = sfc.sequence_categorical_column_with_identity( - key='bbb', num_buckets=vocabulary_size) - embedding_column_b = fc._embedding_column( - categorical_column_b, - dimension=embedding_dimension_b, - initializer=_get_initializer(embedding_dimension_b, embedding_values_b)) - - input_layer, sequence_length = sfc.sequence_input_layer( - features={ - 'aaa': sparse_input_a, - 'bbb': sparse_input_b, - }, - # Test that columns are reordered alphabetically. - feature_columns=[embedding_column_b, embedding_column_a]) - - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertItemsEqual( - ('sequence_input_layer/aaa_embedding/embedding_weights:0', - 'sequence_input_layer/bbb_embedding/embedding_weights:0'), - tuple([v.name for v in global_vars])) - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual(embedding_values_a, global_vars[0].eval(session=sess)) - self.assertAllEqual(embedding_values_b, global_vars[1].eval(session=sess)) - self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess)) - self.assertAllEqual( - expected_sequence_length, sequence_length.eval(session=sess)) - - def test_embedding_column_with_non_sequence_categorical(self): - """Tests that error is raised for non-sequence embedding column.""" - vocabulary_size = 3 - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - indices=((0, 0), (1, 0), (1, 1)), - values=(2, 0, 1), - dense_shape=(2, 2)) - - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - embedding_column_a = fc._embedding_column(categorical_column_a, dimension=2) - - with self.assertRaisesRegexp( - ValueError, - r'In embedding_column: aaa_embedding\. categorical_column must be of ' - r'type _SequenceCategoricalColumn to use sequence_input_layer\.'): - _, _ = sfc.sequence_input_layer( - features={'aaa': sparse_input}, - feature_columns=[embedding_column_a]) - - def test_shared_embedding_column(self): - vocabulary_size = 3 - sparse_input_a = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - indices=((0, 0), (1, 0), (1, 1)), - values=(2, 0, 1), - dense_shape=(2, 2)) - sparse_input_b = sparse_tensor.SparseTensorValue( - # example 0, ids [1] - # example 1, ids [2, 0] - indices=((0, 0), (1, 0), (1, 1)), - values=(1, 2, 0), - dense_shape=(2, 2)) - - embedding_dimension = 2 - embedding_values = ( - (1., 2.), # id 0 - (3., 4.), # id 1 - (5., 6.) # id 2 - ) - - def _get_initializer(embedding_dimension, embedding_values): - - def _initializer(shape, dtype, partition_info): - self.assertAllEqual((vocabulary_size, embedding_dimension), shape) - self.assertEqual(dtypes.float32, dtype) - self.assertIsNone(partition_info) - return embedding_values - - return _initializer - - expected_input_layer = [ - # example 0, ids_a [2], ids_b [1] - [[5., 6., 3., 4.], [0., 0., 0., 0.]], - # example 1, ids_a [0, 1], ids_b [2, 0] - [[1., 2., 5., 6.], [3., 4., 1., 2.]], - ] - expected_sequence_length = [1, 2] - - categorical_column_a = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - categorical_column_b = sfc.sequence_categorical_column_with_identity( - key='bbb', num_buckets=vocabulary_size) - # Test that columns are reordered alphabetically. - shared_embedding_columns = fc_lib.shared_embedding_columns( - [categorical_column_b, categorical_column_a], - dimension=embedding_dimension, - initializer=_get_initializer(embedding_dimension, embedding_values)) - - input_layer, sequence_length = sfc.sequence_input_layer( - features={ - 'aaa': sparse_input_a, - 'bbb': sparse_input_b, - }, - feature_columns=shared_embedding_columns) - - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertItemsEqual( - ('sequence_input_layer/aaa_bbb_shared_embedding/embedding_weights:0',), - tuple([v.name for v in global_vars])) - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual(embedding_values, global_vars[0].eval(session=sess)) - self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess)) - self.assertAllEqual( - expected_sequence_length, sequence_length.eval(session=sess)) - - def test_shared_embedding_column_with_non_sequence_categorical(self): - """Tests that error is raised for non-sequence shared embedding column.""" - vocabulary_size = 3 - sparse_input_a = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - indices=((0, 0), (1, 0), (1, 1)), - values=(2, 0, 1), - dense_shape=(2, 2)) - sparse_input_b = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - indices=((0, 0), (1, 0), (1, 1)), - values=(2, 0, 1), - dense_shape=(2, 2)) - - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc._categorical_column_with_identity( - key='bbb', num_buckets=vocabulary_size) - shared_embedding_columns = fc_lib.shared_embedding_columns( - [categorical_column_a, categorical_column_b], dimension=2) - - with self.assertRaisesRegexp( - ValueError, - r'In embedding_column: aaa_shared_embedding\. categorical_column must ' - r'be of type _SequenceCategoricalColumn to use sequence_input_layer\.'): - _, _ = sfc.sequence_input_layer( - features={ - 'aaa': sparse_input_a, - 'bbb': sparse_input_b - }, - feature_columns=shared_embedding_columns) - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'sparse_input_args_a': { - # example 0, ids [2] - # example 1, ids [0, 1] - 'indices': ((0, 0), (1, 0), (1, 1)), - 'values': (2, 0, 1), - 'dense_shape': (2, 2)}, - 'sparse_input_args_b': { - # example 0, ids [1] - # example 1, ids [1, 0] - 'indices': ((0, 0), (1, 0), (1, 1)), - 'values': (1, 1, 0), - 'dense_shape': (2, 2)}, - 'expected_input_layer': [ - # example 0, ids_a [2], ids_b [1] - [[0., 0., 1., 0., 1.], [0., 0., 0., 0., 0.]], - # example 1, ids_a [0, 1], ids_b [1, 0] - [[1., 0., 0., 0., 1.], [0., 1., 0., 1., 0.]]], - 'expected_sequence_length': [1, 2]}, - {'testcase_name': '3D', - 'sparse_input_args_a': { - # feature 0, ids [[2], [0, 1]] - # feature 1, ids [[0, 0], [1]] - 'indices': ( - (0, 0, 0), (0, 1, 0), (0, 1, 1), - (1, 0, 0), (1, 0, 1), (1, 1, 0)), - 'values': (2, 0, 1, 0, 0, 1), - 'dense_shape': (2, 2, 2)}, - 'sparse_input_args_b': { - # feature 0, ids [[1, 1], [1]] - # feature 1, ids [[1], [0]] - 'indices': ((0, 0, 0), (0, 0, 1), (0, 1, 0), (1, 0, 0), (1, 1, 0)), - 'values': (1, 1, 1, 1, 0), - 'dense_shape': (2, 2, 2)}, - 'expected_input_layer': [ - # feature 0, [a: 2, -, b: 1, 1], [a: 0, 1, b: 1, -] - [[0., 0., 1., 0., 2.], [1., 1., 0., 0., 1.]], - # feature 1, [a: 0, 0, b: 1, -], [a: 1, -, b: 0, -] - [[2., 0., 0., 0., 1.], [0., 1., 0., 1., 0.]]], - 'expected_sequence_length': [2, 2]}, - ) - def test_indicator_column( - self, sparse_input_args_a, sparse_input_args_b, expected_input_layer, - expected_sequence_length): - sparse_input_a = sparse_tensor.SparseTensorValue(**sparse_input_args_a) - sparse_input_b = sparse_tensor.SparseTensorValue(**sparse_input_args_b) - - vocabulary_size_a = 3 - vocabulary_size_b = 2 - - categorical_column_a = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size_a) - indicator_column_a = fc._indicator_column(categorical_column_a) - categorical_column_b = sfc.sequence_categorical_column_with_identity( - key='bbb', num_buckets=vocabulary_size_b) - indicator_column_b = fc._indicator_column(categorical_column_b) - input_layer, sequence_length = sfc.sequence_input_layer( - features={ - 'aaa': sparse_input_a, - 'bbb': sparse_input_b, - }, - # Test that columns are reordered alphabetically. - feature_columns=[indicator_column_b, indicator_column_a]) - - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess)) - self.assertAllEqual( - expected_sequence_length, sequence_length.eval(session=sess)) - - def test_indicator_column_with_non_sequence_categorical(self): - """Tests that error is raised for non-sequence categorical column.""" - vocabulary_size = 3 - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - indices=((0, 0), (1, 0), (1, 1)), - values=(2, 0, 1), - dense_shape=(2, 2)) - - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - indicator_column_a = fc._indicator_column(categorical_column_a) - - with self.assertRaisesRegexp( - ValueError, - r'In indicator_column: aaa_indicator\. categorical_column must be of ' - r'type _SequenceCategoricalColumn to use sequence_input_layer\.'): - _, _ = sfc.sequence_input_layer( - features={'aaa': sparse_input}, - feature_columns=[indicator_column_a]) - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'sparse_input_args': { - # example 0, values [0., 1] - # example 1, [10.] - 'indices': ((0, 0), (0, 1), (1, 0)), - 'values': (0., 1., 10.), - 'dense_shape': (2, 2)}, - 'expected_input_layer': [ - [[0.], [1.]], - [[10.], [0.]]], - 'expected_sequence_length': [2, 1]}, - {'testcase_name': '3D', - 'sparse_input_args': { - # feature 0, ids [[20, 3], [5]] - # feature 1, ids [[3], [8]] - 'indices': ((0, 0, 0), (0, 0, 1), (0, 1, 0), (1, 0, 0), (1, 1, 0)), - 'values': (20, 3, 5., 3., 8.), - 'dense_shape': (2, 2, 2)}, - 'expected_input_layer': [ - [[20.], [3.], [5.], [0.]], - [[3.], [0.], [8.], [0.]]], - 'expected_sequence_length': [2, 2]}, - ) - def test_numeric_column( - self, sparse_input_args, expected_input_layer, expected_sequence_length): - sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args) - - numeric_column = sfc.sequence_numeric_column('aaa') - - input_layer, sequence_length = sfc.sequence_input_layer( - features={'aaa': sparse_input}, - feature_columns=[numeric_column]) - - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess)) - self.assertAllEqual( - expected_sequence_length, sequence_length.eval(session=sess)) - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'sparse_input_args': { - # example 0, values [0., 1., 2., 3., 4., 5., 6., 7.] - # example 1, [10., 11., 12., 13.] - 'indices': ((0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5), (0, 6), - (0, 7), (1, 0), (1, 1), (1, 2), (1, 3)), - 'values': (0., 1., 2., 3., 4., 5., 6., 7., 10., 11., 12., 13.), - 'dense_shape': (2, 8)}, - 'expected_input_layer': [ - # The output of numeric_column._get_dense_tensor should be flattened. - [[0., 1., 2., 3.], [4., 5., 6., 7.]], - [[10., 11., 12., 13.], [0., 0., 0., 0.]]], - 'expected_sequence_length': [2, 1]}, - {'testcase_name': '3D', - 'sparse_input_args': { - # example 0, values [[0., 1., 2., 3.]], [[4., 5., 6., 7.]] - # example 1, [[10., 11., 12., 13.], []] - 'indices': ((0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), - (0, 1, 0), (0, 1, 1), (0, 1, 2), (0, 1, 3), - (1, 0, 0), (1, 0, 1), (1, 0, 2), (1, 0, 3)), - 'values': (0., 1., 2., 3., 4., 5., 6., 7., 10., 11., 12., 13.), - 'dense_shape': (2, 2, 4)}, - 'expected_input_layer': [ - # The output of numeric_column._get_dense_tensor should be flattened. - [[0., 1., 2., 3.], [4., 5., 6., 7.]], - [[10., 11., 12., 13.], [0., 0., 0., 0.]]], - 'expected_sequence_length': [2, 1]}, - ) - def test_numeric_column_multi_dim( - self, sparse_input_args, expected_input_layer, expected_sequence_length): - """Tests sequence_input_layer for multi-dimensional numeric_column.""" - sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args) - - numeric_column = sfc.sequence_numeric_column('aaa', shape=(2, 2)) - - input_layer, sequence_length = sfc.sequence_input_layer( - features={'aaa': sparse_input}, - feature_columns=[numeric_column]) - - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess)) - self.assertAllEqual( - expected_sequence_length, sequence_length.eval(session=sess)) - - def test_sequence_length_not_equal(self): - """Tests that an error is raised when sequence lengths are not equal.""" - # Input a with sequence_length = [2, 1] - sparse_input_a = sparse_tensor.SparseTensorValue( - indices=((0, 0), (0, 1), (1, 0)), - values=(0., 1., 10.), - dense_shape=(2, 2)) - # Input b with sequence_length = [1, 1] - sparse_input_b = sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0)), - values=(1., 10.), - dense_shape=(2, 2)) - numeric_column_a = sfc.sequence_numeric_column('aaa') - numeric_column_b = sfc.sequence_numeric_column('bbb') - - _, sequence_length = sfc.sequence_input_layer( - features={ - 'aaa': sparse_input_a, - 'bbb': sparse_input_b, - }, - feature_columns=[numeric_column_a, numeric_column_b]) - - with monitored_session.MonitoredSession() as sess: - with self.assertRaisesRegexp( - errors.InvalidArgumentError, - r'\[Condition x == y did not hold element-wise:\] ' - r'\[x \(sequence_input_layer/aaa/sequence_length:0\) = \] \[2 1\] ' - r'\[y \(sequence_input_layer/bbb/sequence_length:0\) = \] \[1 1\]'): - sess.run(sequence_length) - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'sparse_input_args': { - # example 0, values [[[0., 1.], [2., 3.]], [[4., 5.], [6., 7.]]] - # example 1, [[[10., 11.], [12., 13.]]] - 'indices': ((0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5), (0, 6), - (0, 7), (1, 0), (1, 1), (1, 2), (1, 3)), - 'values': (0., 1., 2., 3., 4., 5., 6., 7., 10., 11., 12., 13.), - 'dense_shape': (2, 8)}, - 'expected_shape': [2, 2, 4]}, - {'testcase_name': '3D', - 'sparse_input_args': { - # example 0, values [[0., 1., 2., 3.]], [[4., 5., 6., 7.]] - # example 1, [[10., 11., 12., 13.], []] - 'indices': ((0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), - (0, 1, 0), (0, 1, 1), (0, 1, 2), (0, 1, 2), - (1, 0, 0), (1, 0, 1), (1, 0, 2), (1, 0, 3)), - 'values': (0., 1., 2., 3., 4., 5., 6., 7., 10., 11., 12., 13.), - 'dense_shape': (2, 2, 4)}, - 'expected_shape': [2, 2, 4]}, - ) - def test_static_shape_from_tensors_numeric( - self, sparse_input_args, expected_shape): - """Tests that we return a known static shape when we have one.""" - sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args) - numeric_column = sfc.sequence_numeric_column('aaa', shape=(2, 2)) - - input_layer, _ = sfc.sequence_input_layer( - features={'aaa': sparse_input}, - feature_columns=[numeric_column]) - shape = input_layer.get_shape() - self.assertEqual(shape, expected_shape) - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'sparse_input_args': { - # example 0, ids [2] - # example 1, ids [0, 1] - # example 2, ids [] - # example 3, ids [1] - 'indices': ((0, 0), (1, 0), (1, 1), (3, 0)), - 'values': (2, 0, 1, 1), - 'dense_shape': (4, 2)}, - 'expected_shape': [4, 2, 3]}, - {'testcase_name': '3D', - 'sparse_input_args': { - # example 0, ids [[2]] - # example 1, ids [[0, 1], [2]] - # example 2, ids [] - # example 3, ids [[1], [0, 2]] - 'indices': ((0, 0, 0), (1, 0, 0), (1, 0, 1), (1, 1, 0), - (3, 0, 0), (3, 1, 0), (3, 1, 1)), - 'values': (2, 0, 1, 2, 1, 0, 2), - 'dense_shape': (4, 2, 2)}, - 'expected_shape': [4, 2, 3]} - ) - def test_static_shape_from_tensors_indicator( - self, sparse_input_args, expected_shape): - """Tests that we return a known static shape when we have one.""" - sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args) - categorical_column = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=3) - indicator_column = fc._indicator_column(categorical_column) - - input_layer, _ = sfc.sequence_input_layer( - features={'aaa': sparse_input}, feature_columns=[indicator_column]) - shape = input_layer.get_shape() - self.assertEqual(shape, expected_shape) - - -class ConcatenateContextInputTest(test.TestCase, parameterized.TestCase): - """Tests the utility fn concatenate_context_input.""" - - def test_concatenate_context_input(self): - seq_input = ops.convert_to_tensor(np.arange(12).reshape(2, 3, 2)) - context_input = ops.convert_to_tensor(np.arange(10).reshape(2, 5)) - seq_input = math_ops.cast(seq_input, dtype=dtypes.float32) - context_input = math_ops.cast(context_input, dtype=dtypes.float32) - input_layer = sfc.concatenate_context_input(context_input, seq_input) - - expected = np.array([ - [[0, 1, 0, 1, 2, 3, 4], [2, 3, 0, 1, 2, 3, 4], [4, 5, 0, 1, 2, 3, 4]], - [[6, 7, 5, 6, 7, 8, 9], [8, 9, 5, 6, 7, 8, 9], [10, 11, 5, 6, 7, 8, 9]] - ], dtype=np.float32) - with monitored_session.MonitoredSession() as sess: - output = sess.run(input_layer) - self.assertAllEqual(expected, output) - - @parameterized.named_parameters( - {'testcase_name': 'rank_lt_3', - 'seq_input_arg': np.arange(100).reshape(10, 10)}, - {'testcase_name': 'rank_gt_3', - 'seq_input_arg': np.arange(100).reshape(5, 5, 2, 2)} - ) - def test_sequence_input_throws_error(self, seq_input_arg): - seq_input = ops.convert_to_tensor(seq_input_arg) - context_input = ops.convert_to_tensor(np.arange(100).reshape(10, 10)) - seq_input = math_ops.cast(seq_input, dtype=dtypes.float32) - context_input = math_ops.cast(context_input, dtype=dtypes.float32) - with self.assertRaisesRegexp(ValueError, 'sequence_input must have rank 3'): - sfc.concatenate_context_input(context_input, seq_input) - - @parameterized.named_parameters( - {'testcase_name': 'rank_lt_2', - 'context_input_arg': np.arange(100)}, - {'testcase_name': 'rank_gt_2', - 'context_input_arg': np.arange(100).reshape(5, 5, 4)} - ) - def test_context_input_throws_error(self, context_input_arg): - context_input = ops.convert_to_tensor(context_input_arg) - seq_input = ops.convert_to_tensor(np.arange(100).reshape(5, 5, 4)) - seq_input = math_ops.cast(seq_input, dtype=dtypes.float32) - context_input = math_ops.cast(context_input, dtype=dtypes.float32) - with self.assertRaisesRegexp(ValueError, 'context_input must have rank 2'): - sfc.concatenate_context_input(context_input, seq_input) - - def test_integer_seq_input_throws_error(self): - seq_input = ops.convert_to_tensor(np.arange(100).reshape(5, 5, 4)) - context_input = ops.convert_to_tensor(np.arange(100).reshape(10, 10)) - context_input = math_ops.cast(context_input, dtype=dtypes.float32) - with self.assertRaisesRegexp( - TypeError, 'sequence_input must have dtype float32'): - sfc.concatenate_context_input(context_input, seq_input) - - def test_integer_context_input_throws_error(self): - seq_input = ops.convert_to_tensor(np.arange(100).reshape(5, 5, 4)) - context_input = ops.convert_to_tensor(np.arange(100).reshape(10, 10)) - seq_input = math_ops.cast(seq_input, dtype=dtypes.float32) - with self.assertRaisesRegexp( - TypeError, 'context_input must have dtype float32'): - sfc.concatenate_context_input(context_input, seq_input) - - -class InputLayerTest(test.TestCase): - """Tests input_layer with sequence feature columns.""" - - def test_embedding_column(self): - """Tests that error is raised for sequence embedding column.""" - vocabulary_size = 3 - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - indices=((0, 0), (1, 0), (1, 1)), - values=(2, 0, 1), - dense_shape=(2, 2)) - - categorical_column_a = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - embedding_column_a = fc._embedding_column(categorical_column_a, dimension=2) - - with self.assertRaisesRegexp( - ValueError, - r'In embedding_column: aaa_embedding\. categorical_column must not be ' - r'of type _SequenceCategoricalColumn\.'): - _ = fc.input_layer( - features={'aaa': sparse_input}, - feature_columns=[embedding_column_a]) - - def test_indicator_column(self): - """Tests that error is raised for sequence indicator column.""" - vocabulary_size = 3 - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - indices=((0, 0), (1, 0), (1, 1)), - values=(2, 0, 1), - dense_shape=(2, 2)) - - categorical_column_a = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - indicator_column_a = fc._indicator_column(categorical_column_a) - - with self.assertRaisesRegexp( - ValueError, - r'In indicator_column: aaa_indicator\. categorical_column must not be ' - r'of type _SequenceCategoricalColumn\.'): - _ = fc.input_layer( - features={'aaa': sparse_input}, - feature_columns=[indicator_column_a]) - - -def _assert_sparse_tensor_value(test_case, expected, actual): - _assert_sparse_tensor_indices_shape(test_case, expected, actual) - - test_case.assertEqual( - np.array(expected.values).dtype, np.array(actual.values).dtype) - test_case.assertAllEqual(expected.values, actual.values) - - -def _assert_sparse_tensor_indices_shape(test_case, expected, actual): - test_case.assertEqual(np.int64, np.array(actual.indices).dtype) - test_case.assertAllEqual(expected.indices, actual.indices) - - test_case.assertEqual(np.int64, np.array(actual.dense_shape).dtype) - test_case.assertAllEqual(expected.dense_shape, actual.dense_shape) - - -class SequenceCategoricalColumnWithIdentityTest( - test.TestCase, parameterized.TestCase): - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'inputs_args': { - 'indices': ((0, 0), (1, 0), (1, 1)), - 'values': (1, 2, 0), - 'dense_shape': (2, 2)}, - 'expected_args': { - 'indices': ((0, 0, 0), (1, 0, 0), (1, 1, 0)), - 'values': np.array((1, 2, 0), dtype=np.int64), - 'dense_shape': (2, 2, 1)}}, - {'testcase_name': '3D', - 'inputs_args': { - 'indices': ((0, 0, 2), (1, 0, 0), (1, 2, 0)), - 'values': (6, 7, 8), - 'dense_shape': (2, 2, 2)}, - 'expected_args': { - 'indices': ((0, 0, 2), (1, 0, 0), (1, 2, 0)), - 'values': (6, 7, 8), - 'dense_shape': (2, 2, 2)}} - ) - def test_get_sparse_tensors(self, inputs_args, expected_args): - inputs = sparse_tensor.SparseTensorValue(**inputs_args) - expected = sparse_tensor.SparseTensorValue(**expected_args) - column = sfc.sequence_categorical_column_with_identity('aaa', num_buckets=9) - - id_weight_pair = column._get_sparse_tensors(_LazyBuilder({'aaa': inputs})) - - self.assertIsNone(id_weight_pair.weight_tensor) - with monitored_session.MonitoredSession() as sess: - _assert_sparse_tensor_value( - self, expected, id_weight_pair.id_tensor.eval(session=sess)) - - -class SequenceCategoricalColumnWithHashBucketTest( - test.TestCase, parameterized.TestCase): - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'inputs_args': { - 'indices': ((0, 0), (1, 0), (1, 1)), - 'values': ('omar', 'stringer', 'marlo'), - 'dense_shape': (2, 2)}, - 'expected_args': { - 'indices': ((0, 0, 0), (1, 0, 0), (1, 1, 0)), - # Ignored to avoid hash dependence in test. - 'values': np.array((0, 0, 0), dtype=np.int64), - 'dense_shape': (2, 2, 1)}}, - {'testcase_name': '3D', - 'inputs_args': { - 'indices': ((0, 0, 2), (1, 0, 0), (1, 2, 0)), - 'values': ('omar', 'stringer', 'marlo'), - 'dense_shape': (2, 2, 2)}, - 'expected_args': { - 'indices': ((0, 0, 2), (1, 0, 0), (1, 2, 0)), - # Ignored to avoid hash dependence in test. - 'values': np.array((0, 0, 0), dtype=np.int64), - 'dense_shape': (2, 2, 2)}} - ) - def test_get_sparse_tensors(self, inputs_args, expected_args): - inputs = sparse_tensor.SparseTensorValue(**inputs_args) - expected = sparse_tensor.SparseTensorValue(**expected_args) - column = sfc.sequence_categorical_column_with_hash_bucket( - 'aaa', hash_bucket_size=10) - - id_weight_pair = column._get_sparse_tensors(_LazyBuilder({'aaa': inputs})) - - self.assertIsNone(id_weight_pair.weight_tensor) - with monitored_session.MonitoredSession() as sess: - _assert_sparse_tensor_indices_shape( - self, expected, id_weight_pair.id_tensor.eval(session=sess)) - - -class SequenceCategoricalColumnWithVocabularyFileTest( - test.TestCase, parameterized.TestCase): - - def _write_vocab(self, vocab_strings, file_name): - vocab_file = os.path.join(self.get_temp_dir(), file_name) - with open(vocab_file, 'w') as f: - f.write('\n'.join(vocab_strings)) - return vocab_file - - def setUp(self): - super(SequenceCategoricalColumnWithVocabularyFileTest, self).setUp() - - vocab_strings = ['omar', 'stringer', 'marlo'] - self._wire_vocabulary_file_name = self._write_vocab(vocab_strings, - 'wire_vocabulary.txt') - self._wire_vocabulary_size = 3 - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'inputs_args': { - 'indices': ((0, 0), (1, 0), (1, 1)), - 'values': ('marlo', 'skywalker', 'omar'), - 'dense_shape': (2, 2)}, - 'expected_args': { - 'indices': ((0, 0, 0), (1, 0, 0), (1, 1, 0)), - 'values': np.array((2, -1, 0), dtype=np.int64), - 'dense_shape': (2, 2, 1)}}, - {'testcase_name': '3D', - 'inputs_args': { - 'indices': ((0, 0, 2), (1, 0, 0), (1, 2, 0)), - 'values': ('omar', 'skywalker', 'marlo'), - 'dense_shape': (2, 2, 2)}, - 'expected_args': { - 'indices': ((0, 0, 2), (1, 0, 0), (1, 2, 0)), - 'values': np.array((0, -1, 2), dtype=np.int64), - 'dense_shape': (2, 2, 2)}} - ) - def test_get_sparse_tensors(self, inputs_args, expected_args): - inputs = sparse_tensor.SparseTensorValue(**inputs_args) - expected = sparse_tensor.SparseTensorValue(**expected_args) - column = sfc.sequence_categorical_column_with_vocabulary_file( - key='aaa', - vocabulary_file=self._wire_vocabulary_file_name, - vocabulary_size=self._wire_vocabulary_size) - - id_weight_pair = column._get_sparse_tensors(_LazyBuilder({'aaa': inputs})) - - self.assertIsNone(id_weight_pair.weight_tensor) - with monitored_session.MonitoredSession() as sess: - _assert_sparse_tensor_value( - self, expected, id_weight_pair.id_tensor.eval(session=sess)) - - def test_get_sparse_tensors_dynamic_zero_length(self): - """Tests _get_sparse_tensors with a dynamic sequence length.""" - inputs = sparse_tensor.SparseTensorValue( - indices=np.zeros((0, 2)), values=[], dense_shape=(2, 0)) - expected = sparse_tensor.SparseTensorValue( - indices=np.zeros((0, 3)), - values=np.array((), dtype=np.int64), - dense_shape=(2, 0, 1)) - column = sfc.sequence_categorical_column_with_vocabulary_file( - key='aaa', - vocabulary_file=self._wire_vocabulary_file_name, - vocabulary_size=self._wire_vocabulary_size) - input_placeholder_shape = list(inputs.dense_shape) - # Make second dimension (sequence length) dynamic. - input_placeholder_shape[1] = None - input_placeholder = array_ops.sparse_placeholder( - dtypes.string, shape=input_placeholder_shape) - id_weight_pair = column._get_sparse_tensors( - _LazyBuilder({'aaa': input_placeholder})) - - self.assertIsNone(id_weight_pair.weight_tensor) - with monitored_session.MonitoredSession() as sess: - result = id_weight_pair.id_tensor.eval( - session=sess, feed_dict={input_placeholder: inputs}) - _assert_sparse_tensor_value( - self, expected, result) - - -class SequenceCategoricalColumnWithVocabularyListTest( - test.TestCase, parameterized.TestCase): - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'inputs_args': { - 'indices': ((0, 0), (1, 0), (1, 1)), - 'values': ('marlo', 'skywalker', 'omar'), - 'dense_shape': (2, 2)}, - 'expected_args': { - 'indices': ((0, 0, 0), (1, 0, 0), (1, 1, 0)), - 'values': np.array((2, -1, 0), dtype=np.int64), - 'dense_shape': (2, 2, 1)}}, - {'testcase_name': '3D', - 'inputs_args': { - 'indices': ((0, 0, 2), (1, 0, 0), (1, 2, 0)), - 'values': ('omar', 'skywalker', 'marlo'), - 'dense_shape': (2, 2, 2)}, - 'expected_args': { - 'indices': ((0, 0, 2), (1, 0, 0), (1, 2, 0)), - 'values': np.array((0, -1, 2), dtype=np.int64), - 'dense_shape': (2, 2, 2)}} - ) - def test_get_sparse_tensors(self, inputs_args, expected_args): - inputs = sparse_tensor.SparseTensorValue(**inputs_args) - expected = sparse_tensor.SparseTensorValue(**expected_args) - column = sfc.sequence_categorical_column_with_vocabulary_list( - key='aaa', - vocabulary_list=('omar', 'stringer', 'marlo')) - - id_weight_pair = column._get_sparse_tensors(_LazyBuilder({'aaa': inputs})) - - self.assertIsNone(id_weight_pair.weight_tensor) - with monitored_session.MonitoredSession() as sess: - _assert_sparse_tensor_value( - self, expected, id_weight_pair.id_tensor.eval(session=sess)) - - -class SequenceEmbeddingColumnTest( - test.TestCase, parameterized.TestCase): - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'inputs_args': { - # example 0, ids [2] - # example 1, ids [0, 1] - # example 2, ids [] - # example 3, ids [1] - 'indices': ((0, 0), (1, 0), (1, 1), (3, 0)), - 'values': (2, 0, 1, 1), - 'dense_shape': (4, 2)}, - 'expected': [ - # example 0, ids [2] - [[7., 11.], [0., 0.]], - # example 1, ids [0, 1] - [[1., 2.], [3., 5.]], - # example 2, ids [] - [[0., 0.], [0., 0.]], - # example 3, ids [1] - [[3., 5.], [0., 0.]]]}, - {'testcase_name': '3D', - 'inputs_args': { - # example 0, ids [[2]] - # example 1, ids [[0, 1], [2]] - # example 2, ids [] - # example 3, ids [[1], [0, 2]] - 'indices': ((0, 0, 0), (1, 0, 0), (1, 0, 1), (1, 1, 0), - (3, 0, 0), (3, 1, 0), (3, 1, 1)), - 'values': (2, 0, 1, 2, 1, 0, 2), - 'dense_shape': (4, 2, 2)}, - 'expected': [ - # example 0, ids [[2]] - [[7., 11.], [0., 0.]], - # example 1, ids [[0, 1], [2]] - [[2, 3.5], [7., 11.]], - # example 2, ids [] - [[0., 0.], [0., 0.]], - # example 3, ids [[1], [0, 2]] - [[3., 5.], [4., 6.5]]]} - ) - def test_get_sequence_dense_tensor(self, inputs_args, expected): - inputs = sparse_tensor.SparseTensorValue(**inputs_args) - vocabulary_size = 3 - embedding_dimension = 2 - embedding_values = ( - (1., 2.), # id 0 - (3., 5.), # id 1 - (7., 11.) # id 2 - ) - def _initializer(shape, dtype, partition_info): - self.assertAllEqual((vocabulary_size, embedding_dimension), shape) - self.assertEqual(dtypes.float32, dtype) - self.assertIsNone(partition_info) - return embedding_values - - categorical_column = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - embedding_column = fc._embedding_column( - categorical_column, - dimension=embedding_dimension, - initializer=_initializer) - - embedding_lookup, _ = embedding_column._get_sequence_dense_tensor( - _LazyBuilder({'aaa': inputs})) - - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertItemsEqual( - ('embedding_weights:0',), tuple([v.name for v in global_vars])) - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual(embedding_values, global_vars[0].eval(session=sess)) - self.assertAllEqual(expected, embedding_lookup.eval(session=sess)) - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'inputs_args': { - # example 0, ids [2] - # example 1, ids [0, 1] - 'indices': ((0, 0), (1, 0), (1, 1)), - 'values': (2, 0, 1), - 'dense_shape': (2, 2)}, - 'expected_sequence_length': [1, 2]}, - {'testcase_name': '3D', - 'inputs_args': { - # example 0, ids [[2]] - # example 1, ids [[0, 1], [2]] - 'indices': ((0, 0, 0), (1, 0, 0), (1, 0, 1), (1, 1, 0)), - 'values': (2, 0, 1, 2), - 'dense_shape': (2, 2, 2)}, - 'expected_sequence_length': [1, 2]} - ) - def test_sequence_length(self, inputs_args, expected_sequence_length): - inputs = sparse_tensor.SparseTensorValue(**inputs_args) - vocabulary_size = 3 - - categorical_column = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - embedding_column = fc._embedding_column(categorical_column, dimension=2) - - _, sequence_length = embedding_column._get_sequence_dense_tensor( - _LazyBuilder({'aaa': inputs})) - - with monitored_session.MonitoredSession() as sess: - sequence_length = sess.run(sequence_length) - self.assertAllEqual(expected_sequence_length, sequence_length) - self.assertEqual(np.int64, sequence_length.dtype) - - def test_sequence_length_with_empty_rows(self): - """Tests _sequence_length when some examples do not have ids.""" - vocabulary_size = 3 - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, ids [] - # example 1, ids [2] - # example 2, ids [0, 1] - # example 3, ids [] - # example 4, ids [1] - # example 5, ids [] - indices=((1, 0), (2, 0), (2, 1), (4, 0)), - values=(2, 0, 1, 1), - dense_shape=(6, 2)) - expected_sequence_length = [0, 1, 2, 0, 1, 0] - - categorical_column = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - embedding_column = fc._embedding_column(categorical_column, dimension=2) - - _, sequence_length = embedding_column._get_sequence_dense_tensor( - _LazyBuilder({'aaa': sparse_input})) - - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual( - expected_sequence_length, sequence_length.eval(session=sess)) - - -class SequenceSharedEmbeddingColumnTest(test.TestCase): - - def test_get_sequence_dense_tensor(self): - vocabulary_size = 3 - embedding_dimension = 2 - embedding_values = ( - (1., 2.), # id 0 - (3., 5.), # id 1 - (7., 11.) # id 2 - ) - - def _initializer(shape, dtype, partition_info): - self.assertAllEqual((vocabulary_size, embedding_dimension), shape) - self.assertEqual(dtypes.float32, dtype) - self.assertIsNone(partition_info) - return embedding_values - - sparse_input_a = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - # example 2, ids [] - # example 3, ids [1] - indices=((0, 0), (1, 0), (1, 1), (3, 0)), - values=(2, 0, 1, 1), - dense_shape=(4, 2)) - sparse_input_b = sparse_tensor.SparseTensorValue( - # example 0, ids [1] - # example 1, ids [0, 2] - # example 2, ids [0] - # example 3, ids [] - indices=((0, 0), (1, 0), (1, 1), (2, 0)), - values=(1, 0, 2, 0), - dense_shape=(4, 2)) - - expected_lookups_a = [ - # example 0, ids [2] - [[7., 11.], [0., 0.]], - # example 1, ids [0, 1] - [[1., 2.], [3., 5.]], - # example 2, ids [] - [[0., 0.], [0., 0.]], - # example 3, ids [1] - [[3., 5.], [0., 0.]], - ] - - expected_lookups_b = [ - # example 0, ids [1] - [[3., 5.], [0., 0.]], - # example 1, ids [0, 2] - [[1., 2.], [7., 11.]], - # example 2, ids [0] - [[1., 2.], [0., 0.]], - # example 3, ids [] - [[0., 0.], [0., 0.]], - ] - - categorical_column_a = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - categorical_column_b = sfc.sequence_categorical_column_with_identity( - key='bbb', num_buckets=vocabulary_size) - shared_embedding_columns = fc_lib.shared_embedding_columns( - [categorical_column_a, categorical_column_b], - dimension=embedding_dimension, - initializer=_initializer) - - embedding_lookup_a = shared_embedding_columns[0]._get_sequence_dense_tensor( - _LazyBuilder({ - 'aaa': sparse_input_a - }))[0] - embedding_lookup_b = shared_embedding_columns[1]._get_sequence_dense_tensor( - _LazyBuilder({ - 'bbb': sparse_input_b - }))[0] - - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertItemsEqual(('embedding_weights:0',), - tuple([v.name for v in global_vars])) - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual(embedding_values, global_vars[0].eval(session=sess)) - self.assertAllEqual( - expected_lookups_a, embedding_lookup_a.eval(session=sess)) - self.assertAllEqual( - expected_lookups_b, embedding_lookup_b.eval(session=sess)) - - def test_sequence_length(self): - vocabulary_size = 3 - - sparse_input_a = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - indices=((0, 0), (1, 0), (1, 1)), - values=(2, 0, 1), - dense_shape=(2, 2)) - expected_sequence_length_a = [1, 2] - categorical_column_a = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - - sparse_input_b = sparse_tensor.SparseTensorValue( - # example 0, ids [0, 2] - # example 1, ids [1] - indices=((0, 0), (0, 1), (1, 0)), - values=(0, 2, 1), - dense_shape=(2, 2)) - expected_sequence_length_b = [2, 1] - categorical_column_b = sfc.sequence_categorical_column_with_identity( - key='bbb', num_buckets=vocabulary_size) - shared_embedding_columns = fc_lib.shared_embedding_columns( - [categorical_column_a, categorical_column_b], dimension=2) - - sequence_length_a = shared_embedding_columns[0]._get_sequence_dense_tensor( - _LazyBuilder({ - 'aaa': sparse_input_a - }))[1] - sequence_length_b = shared_embedding_columns[1]._get_sequence_dense_tensor( - _LazyBuilder({ - 'bbb': sparse_input_b - }))[1] - - with monitored_session.MonitoredSession() as sess: - sequence_length_a = sess.run(sequence_length_a) - self.assertAllEqual(expected_sequence_length_a, sequence_length_a) - self.assertEqual(np.int64, sequence_length_a.dtype) - sequence_length_b = sess.run(sequence_length_b) - self.assertAllEqual(expected_sequence_length_b, sequence_length_b) - self.assertEqual(np.int64, sequence_length_b.dtype) - - def test_sequence_length_with_empty_rows(self): - """Tests _sequence_length when some examples do not have ids.""" - vocabulary_size = 3 - sparse_input_a = sparse_tensor.SparseTensorValue( - # example 0, ids [] - # example 1, ids [2] - # example 2, ids [0, 1] - # example 3, ids [] - # example 4, ids [1] - # example 5, ids [] - indices=((1, 0), (2, 0), (2, 1), (4, 0)), - values=(2, 0, 1, 1), - dense_shape=(6, 2)) - expected_sequence_length_a = [0, 1, 2, 0, 1, 0] - categorical_column_a = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - - sparse_input_b = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [] - # example 2, ids [] - # example 3, ids [] - # example 4, ids [1] - # example 5, ids [0, 1] - indices=((0, 0), (4, 0), (5, 0), (5, 1)), - values=(2, 1, 0, 1), - dense_shape=(6, 2)) - expected_sequence_length_b = [1, 0, 0, 0, 1, 2] - categorical_column_b = sfc.sequence_categorical_column_with_identity( - key='bbb', num_buckets=vocabulary_size) - - shared_embedding_columns = fc_lib.shared_embedding_columns( - [categorical_column_a, categorical_column_b], dimension=2) - - sequence_length_a = shared_embedding_columns[0]._get_sequence_dense_tensor( - _LazyBuilder({ - 'aaa': sparse_input_a - }))[1] - sequence_length_b = shared_embedding_columns[1]._get_sequence_dense_tensor( - _LazyBuilder({ - 'bbb': sparse_input_b - }))[1] - - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual( - expected_sequence_length_a, sequence_length_a.eval(session=sess)) - self.assertAllEqual( - expected_sequence_length_b, sequence_length_b.eval(session=sess)) - - -class SequenceIndicatorColumnTest(test.TestCase, parameterized.TestCase): - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'inputs_args': { - # example 0, ids [2] - # example 1, ids [0, 1] - # example 2, ids [] - # example 3, ids [1] - 'indices': ((0, 0), (1, 0), (1, 1), (3, 0)), - 'values': (2, 0, 1, 1), - 'dense_shape': (4, 2)}, - 'expected': [ - # example 0, ids [2] - [[0., 0., 1.], [0., 0., 0.]], - # example 1, ids [0, 1] - [[1., 0., 0.], [0., 1., 0.]], - # example 2, ids [] - [[0., 0., 0.], [0., 0., 0.]], - # example 3, ids [1] - [[0., 1., 0.], [0., 0., 0.]]]}, - {'testcase_name': '3D', - 'inputs_args': { - # example 0, ids [[2]] - # example 1, ids [[0, 1], [2]] - # example 2, ids [] - # example 3, ids [[1], [2, 2]] - 'indices': ((0, 0, 0), (1, 0, 0), (1, 0, 1), (1, 1, 0), - (3, 0, 0), (3, 1, 0), (3, 1, 1)), - 'values': (2, 0, 1, 2, 1, 2, 2), - 'dense_shape': (4, 2, 2)}, - 'expected': [ - # example 0, ids [[2]] - [[0., 0., 1.], [0., 0., 0.]], - # example 1, ids [[0, 1], [2]] - [[1., 1., 0.], [0., 0., 1.]], - # example 2, ids [] - [[0., 0., 0.], [0., 0., 0.]], - # example 3, ids [[1], [2, 2]] - [[0., 1., 0.], [0., 0., 2.]]]} - ) - def test_get_sequence_dense_tensor(self, inputs_args, expected): - inputs = sparse_tensor.SparseTensorValue(**inputs_args) - vocabulary_size = 3 - - categorical_column = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - indicator_column = fc._indicator_column(categorical_column) - - indicator_tensor, _ = indicator_column._get_sequence_dense_tensor( - _LazyBuilder({'aaa': inputs})) - - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual(expected, indicator_tensor.eval(session=sess)) - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'inputs_args': { - # example 0, ids [2] - # example 1, ids [0, 1] - 'indices': ((0, 0), (1, 0), (1, 1)), - 'values': (2, 0, 1), - 'dense_shape': (2, 2)}, - 'expected_sequence_length': [1, 2]}, - {'testcase_name': '3D', - 'inputs_args': { - # example 0, ids [[2]] - # example 1, ids [[0, 1], [2]] - 'indices': ((0, 0, 0), (1, 0, 0), (1, 0, 1), (1, 1, 0)), - 'values': (2, 0, 1, 2), - 'dense_shape': (2, 2, 2)}, - 'expected_sequence_length': [1, 2]} - ) - def test_sequence_length(self, inputs_args, expected_sequence_length): - inputs = sparse_tensor.SparseTensorValue(**inputs_args) - vocabulary_size = 3 - - categorical_column = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - indicator_column = fc._indicator_column(categorical_column) - - _, sequence_length = indicator_column._get_sequence_dense_tensor( - _LazyBuilder({'aaa': inputs})) - - with monitored_session.MonitoredSession() as sess: - sequence_length = sess.run(sequence_length) - self.assertAllEqual(expected_sequence_length, sequence_length) - self.assertEqual(np.int64, sequence_length.dtype) - - def test_sequence_length_with_empty_rows(self): - """Tests _sequence_length when some examples do not have ids.""" - vocabulary_size = 3 - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, ids [] - # example 1, ids [2] - # example 2, ids [0, 1] - # example 3, ids [] - # example 4, ids [1] - # example 5, ids [] - indices=((1, 0), (2, 0), (2, 1), (4, 0)), - values=(2, 0, 1, 1), - dense_shape=(6, 2)) - expected_sequence_length = [0, 1, 2, 0, 1, 0] - - categorical_column = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - indicator_column = fc._indicator_column(categorical_column) - - _, sequence_length = indicator_column._get_sequence_dense_tensor( - _LazyBuilder({'aaa': sparse_input})) - - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual( - expected_sequence_length, sequence_length.eval(session=sess)) - - -class SequenceNumericColumnTest(test.TestCase, parameterized.TestCase): - - def test_defaults(self): - a = sfc.sequence_numeric_column('aaa') - self.assertEqual('aaa', a.key) - self.assertEqual('aaa', a.name) - self.assertEqual('aaa', a._var_scope_name) - self.assertEqual((1,), a.shape) - self.assertEqual(0., a.default_value) - self.assertEqual(dtypes.float32, a.dtype) - self.assertIsNone(a.normalizer_fn) - - def test_shape_saved_as_tuple(self): - a = sfc.sequence_numeric_column('aaa', shape=[1, 2]) - self.assertEqual((1, 2), a.shape) - - def test_shape_must_be_positive_integer(self): - with self.assertRaisesRegexp(TypeError, 'shape dimensions must be integer'): - sfc.sequence_numeric_column('aaa', shape=[1.0]) - - with self.assertRaisesRegexp( - ValueError, 'shape dimensions must be greater than 0'): - sfc.sequence_numeric_column('aaa', shape=[0]) - - def test_dtype_is_convertible_to_float(self): - with self.assertRaisesRegexp( - ValueError, 'dtype must be convertible to float'): - sfc.sequence_numeric_column('aaa', dtype=dtypes.string) - - def test_normalizer_fn_must_be_callable(self): - with self.assertRaisesRegexp(TypeError, 'must be a callable'): - sfc.sequence_numeric_column('aaa', normalizer_fn='NotACallable') - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'inputs_args': { - # example 0, values [0., 1] - # example 1, [10.] - 'indices': ((0, 0), (0, 1), (1, 0)), - 'values': (0., 1., 10.), - 'dense_shape': (2, 2)}, - 'expected': [ - [[0.], [1.]], - [[10.], [0.]]]}, - {'testcase_name': '3D', - 'inputs_args': { - # feature 0, ids [[20, 3], [5]] - # feature 1, ids [[3], [8]] - 'indices': ((0, 0, 0), (0, 0, 1), (0, 1, 0), (1, 0, 0), (1, 1, 0)), - 'values': (20, 3, 5., 3., 8.), - 'dense_shape': (2, 2, 2)}, - 'expected': [ - [[20.], [3.], [5.], [0.]], - [[3.], [0.], [8.], [0.]]]}, - ) - def test_get_sequence_dense_tensor(self, inputs_args, expected): - inputs = sparse_tensor.SparseTensorValue(**inputs_args) - numeric_column = sfc.sequence_numeric_column('aaa') - - dense_tensor, _ = numeric_column._get_sequence_dense_tensor( - _LazyBuilder({'aaa': inputs})) - - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual(expected, dense_tensor.eval(session=sess)) - - def test_get_sequence_dense_tensor_with_normalizer_fn(self): - - def _increment_two(input_sparse_tensor): - return sparse_ops.sparse_add( - input_sparse_tensor, - sparse_tensor.SparseTensor(((0, 0), (1, 1)), (2.0, 2.0), (2, 2)) - ) - - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, values [[0.], [1]] - # example 1, [[10.]] - indices=((0, 0), (0, 1), (1, 0)), - values=(0., 1., 10.), - dense_shape=(2, 2)) - - # Before _increment_two: - # [[0.], [1.]], - # [[10.], [0.]], - # After _increment_two: - # [[2.], [1.]], - # [[10.], [2.]], - expected_dense_tensor = [ - [[2.], [1.]], - [[10.], [2.]], - ] - numeric_column = sfc.sequence_numeric_column( - 'aaa', normalizer_fn=_increment_two) - - dense_tensor, _ = numeric_column._get_sequence_dense_tensor( - _LazyBuilder({'aaa': sparse_input})) - - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual( - expected_dense_tensor, dense_tensor.eval(session=sess)) - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'sparse_input_args': { - # example 0, values [[[0., 1.], [2., 3.]], [[4., 5.], [6., 7.]]] - # example 1, [[[10., 11.], [12., 13.]]] - 'indices': ((0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5), (0, 6), - (0, 7), (1, 0), (1, 1), (1, 2), (1, 3)), - 'values': (0., 1., 2., 3., 4., 5., 6., 7., 10., 11., 12., 13.), - 'dense_shape': (2, 8)}, - 'expected_dense_tensor': [ - [[[0., 1.], [2., 3.]], [[4., 5.], [6., 7.]]], - [[[10., 11.], [12., 13.]], [[0., 0.], [0., 0.]]]]}, - {'testcase_name': '3D', - 'sparse_input_args': { - 'indices': ((0, 0, 0), (0, 0, 2), (0, 0, 4), (0, 0, 6), - (0, 1, 0), (0, 1, 2), (0, 1, 4), (0, 1, 6), - (1, 0, 0), (1, 0, 2), (1, 0, 4), (1, 0, 6)), - 'values': (0., 1., 2., 3., 4., 5., 6., 7., 10., 11., 12., 13.), - 'dense_shape': (2, 2, 8)}, - 'expected_dense_tensor': [ - [[[0., 0.], [1., 0.]], [[2., 0.], [3., 0.]], - [[4., 0.], [5., 0.]], [[6., 0.], [7., 0.]]], - [[[10., 0.], [11., 0.]], [[12., 0.], [13., 0.]], - [[0., 0.], [0., 0.]], [[0., 0.], [0., 0.]]]]}, - ) - def test_get_dense_tensor_multi_dim( - self, sparse_input_args, expected_dense_tensor): - """Tests get_sequence_dense_tensor for multi-dim numeric_column.""" - sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args) - numeric_column = sfc.sequence_numeric_column('aaa', shape=(2, 2)) - - dense_tensor, _ = numeric_column._get_sequence_dense_tensor( - _LazyBuilder({'aaa': sparse_input})) - - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual( - expected_dense_tensor, dense_tensor.eval(session=sess)) - - @parameterized.named_parameters( - {'testcase_name': '2D', - 'inputs_args': { - # example 0, ids [2] - # example 1, ids [0, 1] - 'indices': ((0, 0), (1, 0), (1, 1)), - 'values': (2., 0., 1.), - 'dense_shape': (2, 2)}, - 'expected_sequence_length': [1, 2], - 'shape': (1,)}, - {'testcase_name': '3D', - 'inputs_args': { - # example 0, ids [[2]] - # example 1, ids [[0, 1], [2]] - 'indices': ((0, 0, 0), (1, 0, 0), (1, 0, 1), (1, 1, 0)), - 'values': (2., 0., 1., 2.), - 'dense_shape': (2, 2, 2)}, - 'expected_sequence_length': [1, 2], - 'shape': (1,)}, - {'testcase_name': '2D_with_shape', - 'inputs_args': { - # example 0, ids [2] - # example 1, ids [0, 1] - 'indices': ((0, 0), (1, 0), (1, 1)), - 'values': (2., 0., 1.), - 'dense_shape': (2, 2)}, - 'expected_sequence_length': [1, 1], - 'shape': (2,)}, - {'testcase_name': '3D_with_shape', - 'inputs_args': { - # example 0, ids [[2]] - # example 1, ids [[0, 1], [2]] - 'indices': ((0, 0, 0), (1, 0, 0), (1, 0, 1), (1, 1, 0)), - 'values': (2., 0., 1., 2.), - 'dense_shape': (2, 2, 2)}, - 'expected_sequence_length': [1, 2], - 'shape': (2,)}, - ) - def test_sequence_length(self, inputs_args, expected_sequence_length, shape): - inputs = sparse_tensor.SparseTensorValue(**inputs_args) - numeric_column = sfc.sequence_numeric_column('aaa', shape=shape) - - _, sequence_length = numeric_column._get_sequence_dense_tensor( - _LazyBuilder({'aaa': inputs})) - - with monitored_session.MonitoredSession() as sess: - sequence_length = sess.run(sequence_length) - self.assertAllEqual(expected_sequence_length, sequence_length) - self.assertEqual(np.int64, sequence_length.dtype) - - def test_sequence_length_with_empty_rows(self): - """Tests _sequence_length when some examples do not have ids.""" - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, values [] - # example 1, values [[0.], [1.]] - # example 2, [[2.]] - # example 3, values [] - # example 4, [[3.]] - # example 5, values [] - indices=((1, 0), (1, 1), (2, 0), (4, 0)), - values=(0., 1., 2., 3.), - dense_shape=(6, 2)) - expected_sequence_length = [0, 2, 1, 0, 1, 0] - numeric_column = sfc.sequence_numeric_column('aaa') - - _, sequence_length = numeric_column._get_sequence_dense_tensor( - _LazyBuilder({'aaa': sparse_input})) - - with monitored_session.MonitoredSession() as sess: - self.assertAllEqual( - expected_sequence_length, sequence_length.eval(session=sess)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/ffmpeg/BUILD b/tensorflow/contrib/ffmpeg/BUILD deleted file mode 100644 index 526fb350733..00000000000 --- a/tensorflow/contrib/ffmpeg/BUILD +++ /dev/null @@ -1,177 +0,0 @@ -# Ops that process audio and/or video files using FFmpeg. -# (https://www.ffmpeg.org/) - -load("//tensorflow:tensorflow.bzl", "tf_copts", "tf_custom_op_library", "tf_gen_op_wrapper_py", "tf_py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -filegroup( - name = "test_data", - srcs = glob(["testdata/*"]), -) - -exports_files(["ffmpeg_lib.h"]) - -cc_library( - name = "decode_audio_op_cc", - srcs = ["decode_audio_op.cc"], - copts = tf_copts(), - linkstatic = 1, - visibility = ["//visibility:private"], - deps = [ - "//tensorflow/contrib/ffmpeg/default:ffmpeg_lib", - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - ], - alwayslink = 1, -) - -cc_library( - name = "encode_audio_op_cc", - srcs = ["encode_audio_op.cc"], - copts = tf_copts(), - linkstatic = 1, - visibility = ["//visibility:private"], - deps = [ - "//tensorflow/contrib/ffmpeg/default:ffmpeg_lib", - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - ], - alwayslink = 1, -) - -cc_library( - name = "decode_video_op_cc", - srcs = ["decode_video_op.cc"], - copts = tf_copts(), - linkstatic = 1, - visibility = ["//visibility:private"], - deps = [ - "//tensorflow/contrib/ffmpeg/default:ffmpeg_lib", - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - ], - alwayslink = 1, -) - -tf_custom_op_library( - name = "ffmpeg.so", - deps = [ - ":decode_audio_op_cc", - ":decode_video_op_cc", - ":encode_audio_op_cc", - ], -) - -cc_library( - name = "ffmpeg_op_lib", - deps = [ - ":decode_audio_op_cc", - ":decode_video_op_cc", - ":encode_audio_op_cc", - ], -) - -tf_gen_op_wrapper_py( - name = "decode_audio_op_py", - visibility = ["//visibility:private"], - deps = [ - ":decode_audio_op_cc", - ], -) - -tf_gen_op_wrapper_py( - name = "encode_audio_op_py", - visibility = ["//visibility:private"], - deps = [ - ":encode_audio_op_cc", - ], -) - -tf_gen_op_wrapper_py( - name = "decode_video_op_py", - visibility = ["//visibility:private"], - deps = [ - ":decode_video_op_cc", - ], -) - -tf_py_test( - name = "decode_audio_op_test", - srcs = ["decode_audio_op_test.py"], - additional_deps = [ - ":ffmpeg_ops_py", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:platform", - ], - data = [ - ":test_data", - ], - tags = ["manual"], -) - -tf_py_test( - name = "encode_audio_op_test", - srcs = ["encode_audio_op_test.py"], - additional_deps = [ - ":ffmpeg_ops_py", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:platform", - ], - data = [ - ":test_data", - ], - tags = ["manual"], -) - -tf_py_test( - name = "decode_video_op_test", - size = "small", - srcs = ["decode_video_op_test.py"], - additional_deps = [ - ":ffmpeg_ops_py", - "@six_archive//:six", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:platform", - "//tensorflow/python:image_ops", - ], - data = [ - ":test_data", - ], - tags = [ - "manual", - "nomsan", # http://b/134927401 - ], -) - -py_library( - name = "ffmpeg_ops_py", - srcs = [ - "__init__.py", - "ffmpeg_ops.py", - ], - data = [":ffmpeg.so"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":decode_audio_op_py", - ":decode_video_op_py", - ":encode_audio_op_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:util", - ], -) diff --git a/tensorflow/contrib/ffmpeg/README.md b/tensorflow/contrib/ffmpeg/README.md deleted file mode 100644 index b3e6af0e387..00000000000 --- a/tensorflow/contrib/ffmpeg/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# FFmpeg TensorFlow integration - -Decoding audio files can be done using a new op that uses -[FFmpeg](http://www.ffmpeg.org) to convert various audio file formats into -tensors. - -tf.audio.decode_audio accepts MP3, WAV, and OGG file formats. - -FFmpeg must be installed before these ops can be used. The ops will look for the -ffmpeg binary somewhere in `$PATH`. When the binary is unavailable, the error -`FFmpeg must be installed to run this op. FFmpeg can be found at -http://www.ffmpeg.org.` will be returned. - -## Testing - -In addition to the regular tests, the integration tests should also be -run on this code. First, install `docker`. Then run the integration tests: - -```shell -export TF_BUILD_CONTAINER_TYPE=CPU # or GPU -export TF_BUILD_PYTHON_VERSION=PYTHON2 # or PYTHON3 -export TF_BUILD_IS_OPT=OPT -export TF_BUILD_IS_PIP=PIP -export TF_BUILD_INTEGRATION_TESTS=1 -tensorflow/tools/ci_build/ci_parameterized_build.sh -``` diff --git a/tensorflow/contrib/ffmpeg/__init__.py b/tensorflow/contrib/ffmpeg/__init__.py deleted file mode 100644 index 3a756da932b..00000000000 --- a/tensorflow/contrib/ffmpeg/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# pylint: disable=g-short-docstring-punctuation -"""Working with audio using FFmpeg. - -See the [FFMPEG](https://tensorflow.org/api_guides/python/contrib.ffmpeg) guide. - -@@decode_audio -@@encode_audio -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.ffmpeg.ffmpeg_ops import decode_audio -from tensorflow.contrib.ffmpeg.ffmpeg_ops import decode_video -from tensorflow.contrib.ffmpeg.ffmpeg_ops import encode_audio - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = ['decode_audio', 'encode_audio', 'decode_video'] -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/ffmpeg/decode_audio_op.cc b/tensorflow/contrib/ffmpeg/decode_audio_op.cc deleted file mode 100644 index 32e62a6725f..00000000000 --- a/tensorflow/contrib/ffmpeg/decode_audio_op.cc +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include - -#include -#include - -#include "tensorflow/contrib/ffmpeg/ffmpeg_lib.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/logging.h" - -namespace tensorflow { -namespace ffmpeg { -namespace { - -// The complete set of audio file formats that are supported by the op. These -// strings are defined by FFmpeg and documented here: -// https://www.ffmpeg.org/ffmpeg-formats.html -const char* kValidFileFormats[] = {"mp3", "mp4", "ogg", "wav"}; - -/* - * Decoding implementation, shared across V1 and V2 ops. Creates a new - * output in the context. - */ -void Decode(OpKernelContext* context, - const tensorflow::StringPiece& file_contents, - const string& file_format, const int32 samples_per_second, - const int32 channel_count, const string& stream) { - // Write the input data to a temp file. - const string temp_filename = io::GetTempFilename(file_format); - OP_REQUIRES_OK(context, WriteFile(temp_filename, file_contents)); - FileDeleter deleter(temp_filename); - - // Run FFmpeg on the data and verify results. - std::vector output_samples; - Status result = - ffmpeg::ReadAudioFile(temp_filename, file_format, samples_per_second, - channel_count, stream, &output_samples); - if (result.code() == error::Code::NOT_FOUND) { - OP_REQUIRES( - context, result.ok(), - errors::Unavailable("FFmpeg must be installed to run this op. FFmpeg " - "can be found at http://www.ffmpeg.org.")); - } else if (result.code() == error::UNKNOWN) { - LOG(ERROR) << "Ffmpeg failed with error '" << result.error_message() - << "'. Returning empty tensor."; - Tensor* output = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(0, TensorShape({0, 0}), &output)); - return; - } else { - OP_REQUIRES_OK(context, result); - } - OP_REQUIRES(context, !output_samples.empty(), - errors::Unknown("No output created by FFmpeg.")); - OP_REQUIRES( - context, output_samples.size() % channel_count == 0, - errors::Unknown("FFmpeg created non-integer number of audio frames.")); - - // Copy the output data to the output Tensor. - Tensor* output = nullptr; - const int64 frame_count = output_samples.size() / channel_count; - OP_REQUIRES_OK(context, - context->allocate_output( - 0, TensorShape({frame_count, channel_count}), &output)); - auto matrix = output->tensor(); - for (int32 frame = 0; frame < frame_count; ++frame) { - for (int32 channel = 0; channel < channel_count; ++channel) { - matrix(frame, channel) = output_samples[frame * channel_count + channel]; - } - } -} - -} // namespace - -/* - * Supersedes `DecodeAudioOp`. Allows all parameters to be inputs - * instead of attributes, so that they can be given as tensors rather - * than constants only. - */ -class DecodeAudioOpV2 : public OpKernel { - public: - explicit DecodeAudioOpV2(OpKernelConstruction* context) : OpKernel(context) { - string stream; - if (context->GetAttr("stream", &stream).ok()) { - stream_ = stream; - } - } - - void Compute(OpKernelContext* context) override { - OP_REQUIRES( - context, context->num_inputs() == 4, - errors::InvalidArgument("DecodeAudio requires exactly four inputs.")); - - const Tensor& contents_tensor = context->input(0); - const Tensor& file_format_tensor = context->input(1); - const Tensor& samples_per_second_tensor = context->input(2); - const Tensor& channel_count_tensor = context->input(3); - - OP_REQUIRES(context, TensorShapeUtils::IsScalar(contents_tensor.shape()), - errors::InvalidArgument( - "contents must be a rank-0 tensor but got shape ", - contents_tensor.shape().DebugString())); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(file_format_tensor.shape()), - errors::InvalidArgument( - "file_format must be a rank-0 tensor but got shape ", - file_format_tensor.shape().DebugString())); - OP_REQUIRES(context, - TensorShapeUtils::IsScalar(samples_per_second_tensor.shape()), - errors::InvalidArgument( - "samples_per_second must be a rank-0 tensor but got shape ", - samples_per_second_tensor.shape().DebugString())); - OP_REQUIRES(context, - TensorShapeUtils::IsScalar(channel_count_tensor.shape()), - errors::InvalidArgument( - "channel_count must be a rank-0 tensor but got shape ", - channel_count_tensor.shape().DebugString())); - - const tensorflow::StringPiece contents = - contents_tensor.scalar()(); - const string file_format = - absl::AsciiStrToLower(file_format_tensor.scalar()()); - const int32 samples_per_second = - samples_per_second_tensor.scalar()(); - const int32 channel_count = channel_count_tensor.scalar()(); - - const std::set valid_file_formats( - kValidFileFormats, kValidFileFormats + TF_ARRAYSIZE(kValidFileFormats)); - OP_REQUIRES(context, valid_file_formats.count(file_format) == 1, - errors::InvalidArgument("file_format must be one of {", - absl::StrJoin(valid_file_formats, ", "), - "}, but was: \"", file_format, "\"")); - OP_REQUIRES(context, samples_per_second > 0, - errors::InvalidArgument( - "samples_per_second must be positive, but got: ", - samples_per_second)); - OP_REQUIRES( - context, channel_count > 0, - errors::InvalidArgument("channel_count must be positive, but got: ", - channel_count)); - - Decode(context, contents, file_format, samples_per_second, channel_count, - stream_); - } - - private: - string stream_; -}; - -REGISTER_KERNEL_BUILDER(Name("DecodeAudioV2").Device(DEVICE_CPU), - DecodeAudioOpV2); - -REGISTER_OP("DecodeAudioV2") - .Input("contents: string") - .Input("file_format: string") - .Input("samples_per_second: int32") - .Input("channel_count: int32") - .Output("sampled_audio: float") - .Attr("stream: string = ''") - .SetShapeFn([](shape_inference::InferenceContext* c) { - const Tensor* channels_tensor = c->input_tensor(3); - if (channels_tensor == nullptr) { - c->set_output(0, c->Matrix(c->UnknownDim(), c->UnknownDim())); - return Status::OK(); - } - const int32 channels = channels_tensor->scalar()(); - if (channels <= 0) { - return errors::InvalidArgument( - "channel_count must be positive, but got: ", channels); - } - c->set_output(0, c->Matrix(c->UnknownDim(), channels)); - return Status::OK(); - }) - .Doc(R"doc( -Processes the contents of an audio file into a tensor using FFmpeg to decode -the file. - -One row of the tensor is created for each channel in the audio file. Each -channel contains audio samples starting at the beginning of the audio and -having `1/samples_per_second` time between them. If the `channel_count` is -different from the contents of the file, channels will be merged or created. - -contents: The binary audio file contents, as a string or rank-0 string - tensor. -file_format: A string or rank-0 string tensor describing the audio file - format. This must be one of: "mp3", "mp4", "ogg", "wav". -samples_per_second: The number of samples per second that the audio - should have, as an `int` or rank-0 `int32` tensor. This value must - be positive. -channel_count: The number of channels of audio to read, as an int rank-0 - int32 tensor. Must be a positive integer. -sampled_audio: A rank-2 tensor containing all tracks of the audio. - Dimension 0 is time and dimension 1 is the channel. If ffmpeg fails - to decode the audio then an empty tensor will be returned. -)doc"); - -/* - * Deprecated in favor of DecodeAudioOpV2. - */ -class DecodeAudioOp : public OpKernel { - public: - explicit DecodeAudioOp(OpKernelConstruction* context) : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr("file_format", &file_format_)); - file_format_ = absl::AsciiStrToLower(file_format_); - const std::set valid_file_formats( - kValidFileFormats, kValidFileFormats + TF_ARRAYSIZE(kValidFileFormats)); - OP_REQUIRES(context, valid_file_formats.count(file_format_) == 1, - errors::InvalidArgument("file_format must be one of {", - absl::StrJoin(valid_file_formats, ", "), - "}, but was: \"", file_format_, "\"")); - - OP_REQUIRES_OK(context, context->GetAttr("channel_count", &channel_count_)); - OP_REQUIRES(context, channel_count_ > 0, - errors::InvalidArgument("channel_count must be > 0.")); - } - - void Compute(OpKernelContext* context) override { - // Get and verify the input data. - OP_REQUIRES( - context, context->num_inputs() == 1, - errors::InvalidArgument("DecodeAudio requires exactly one input.")); - const Tensor& contents = context->input(0); - OP_REQUIRES( - context, TensorShapeUtils::IsScalar(contents.shape()), - errors::InvalidArgument("contents must be scalar but got shape ", - contents.shape().DebugString())); - - const tensorflow::StringPiece file_contents = contents.scalar()(); - Decode(context, file_contents, file_format_, samples_per_second_, - channel_count_, ""); - } - - private: - string file_format_; - int32 samples_per_second_; - int32 channel_count_; -}; - -REGISTER_KERNEL_BUILDER(Name("DecodeAudio").Device(DEVICE_CPU), DecodeAudioOp); - -REGISTER_OP("DecodeAudio") - .Input("contents: string") - .Output("sampled_audio: float") - .Attr("file_format: string") - .Attr("samples_per_second: int") - .Attr("channel_count: int") - .SetShapeFn([](shape_inference::InferenceContext* c) { - int64 channels; - if (c->GetAttr("channel_count", &channels).ok()) { - c->set_output(0, c->Matrix(c->UnknownDim(), channels)); - } else { - c->set_output(0, c->Matrix(c->UnknownDim(), c->UnknownDim())); - } - return Status::OK(); - }) - .Doc(R"doc( -Processes the contents of an audio file into a tensor using FFmpeg to decode -the file. - -One row of the tensor is created for each channel in the audio file. Each -channel contains audio samples starting at the beginning of the audio and -having `1/samples_per_second` time between them. If the `channel_count` is -different from the contents of the file, channels will be merged or created. - -contents: The binary audio file contents. -sampled_audio: A rank 2 tensor containing all tracks of the audio. Dimension 0 - is time and dimension 1 is the channel. If ffmpeg fails to decode the audio - then an empty tensor will be returned. -file_format: A string describing the audio file format. This can be "mp3", "mp4", "ogg", or "wav". -samples_per_second: The number of samples per second that the audio should have. -channel_count: The number of channels of audio to read. -)doc"); - -} // namespace ffmpeg -} // namespace tensorflow diff --git a/tensorflow/contrib/ffmpeg/decode_audio_op_test.py b/tensorflow/contrib/ffmpeg/decode_audio_op_test.py deleted file mode 100644 index 784da1c432f..00000000000 --- a/tensorflow/contrib/ffmpeg/decode_audio_op_test.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Tests for third_party.tensorflow.contrib.ffmpeg.decode_audio_op.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os.path - -import six - -from tensorflow.contrib import ffmpeg -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import resource_loader -from tensorflow.python.platform import test - - -class DecodeAudioOpTest(test.TestCase): - - def _loadFileAndTest(self, filename, file_format, duration_sec, - samples_per_second, channel_count, - samples_per_second_tensor=None, feed_dict=None, - stream=None): - """Loads an audio file and validates the output tensor. - - Args: - filename: The filename of the input file. - file_format: The format of the input file. - duration_sec: The duration of the audio contained in the file in seconds. - samples_per_second: The desired sample rate in the output tensor. - channel_count: The desired channel count in the output tensor. - samples_per_second_tensor: The value to pass to the corresponding - parameter in the instantiated `decode_audio` op. If not - provided, will default to a constant value of - `samples_per_second`. Useful for providing a placeholder. - feed_dict: Used when evaluating the `decode_audio` op. If not - provided, will be empty. Useful when providing a placeholder for - `samples_per_second_tensor`. - stream: A string specifying which stream from the content file - should be decoded. The default value is '' which leaves the - decision to ffmpeg. - """ - if samples_per_second_tensor is None: - samples_per_second_tensor = samples_per_second - with self.cached_session(): - path = os.path.join(resource_loader.get_data_files_path(), 'testdata', - filename) - with open(path, 'rb') as f: - contents = f.read() - - audio_op = ffmpeg.decode_audio( - contents, - file_format=file_format, - samples_per_second=samples_per_second_tensor, - channel_count=channel_count, stream=stream) - audio = audio_op.eval(feed_dict=feed_dict or {}) - self.assertEqual(len(audio.shape), 2) - self.assertNear( - duration_sec * samples_per_second, - audio.shape[0], - # Duration should be specified within 10%: - 0.1 * audio.shape[0]) - self.assertEqual(audio.shape[1], channel_count) - - def testStreamIdentifier(self): - # mono_16khz_mp3_32khz_aac.mp4 was generated from: - # ffmpeg -i tensorflow/contrib/ffmpeg/testdata/mono_16khz_mp3.mp4 \ - # -i tensorflow/contrib/ffmpeg/testdata/mono_32khz_aac.mp4 \ - # -strict -2 -map 0:a -map 1:a \ - # tensorflow/contrib/ffmpeg/testdata/mono_16khz_mp3_32khz_aac.mp4 - self._loadFileAndTest('mono_16khz_mp3_32khz_aac.mp4', 'mp4', 2.77, 20000, - 1, stream='0') - self._loadFileAndTest('mono_16khz_mp3_32khz_aac.mp4', 'mp4', 2.77, 20000, - 1, stream='1') - - def testMonoMp3(self): - self._loadFileAndTest('mono_16khz.mp3', 'mp3', 0.57, 20000, 1) - self._loadFileAndTest('mono_16khz.mp3', 'mp3', 0.57, 20000, 2) - - def testMonoMp4Mp3Codec(self): - # mp3 compressed audio streams in mp4 container. - self._loadFileAndTest('mono_16khz_mp3.mp4', 'mp4', 2.77, 20000, 1) - self._loadFileAndTest('mono_16khz_mp3.mp4', 'mp4', 2.77, 20000, 2) - - def testMonoMp4AacCodec(self): - # aac compressed audio streams in mp4 container. - self._loadFileAndTest('mono_32khz_aac.mp4', 'mp4', 2.77, 20000, 1) - self._loadFileAndTest('mono_32khz_aac.mp4', 'mp4', 2.77, 20000, 2) - - def testStereoMp3(self): - self._loadFileAndTest('stereo_48khz.mp3', 'mp3', 0.79, 50000, 1) - self._loadFileAndTest('stereo_48khz.mp3', 'mp3', 0.79, 20000, 2) - - def testStereoMp4Mp3Codec(self): - # mp3 compressed audio streams in mp4 container. - self._loadFileAndTest('stereo_48khz_mp3.mp4', 'mp4', 0.79, 50000, 1) - self._loadFileAndTest('stereo_48khz_mp3.mp4', 'mp4', 0.79, 20000, 2) - - def testStereoMp4AacCodec(self): - # aac compressed audio streams in mp4 container. - self._loadFileAndTest('stereo_48khz_aac.mp4', 'mp4', 0.79, 50000, 1) - self._loadFileAndTest('stereo_48khz_aac.mp4', 'mp4', 0.79, 20000, 2) - - def testMonoWav(self): - self._loadFileAndTest('mono_10khz.wav', 'wav', 0.57, 5000, 1) - self._loadFileAndTest('mono_10khz.wav', 'wav', 0.57, 10000, 4) - - def testOgg(self): - self._loadFileAndTest('mono_10khz.ogg', 'ogg', 0.57, 10000, 1) - - def testInvalidFile(self): - with self.cached_session(): - contents = 'invalid file' - audio_op = ffmpeg.decode_audio( - contents, - file_format='wav', - samples_per_second=10000, - channel_count=2) - audio = audio_op.eval() - self.assertEqual(audio.shape, (0, 0)) - - def testSampleRatePlaceholder(self): - placeholder = array_ops.placeholder(dtypes.int32) - self._loadFileAndTest('mono_16khz.mp3', 'mp3', 0.57, 20000, 1, - samples_per_second_tensor=placeholder, - feed_dict={placeholder: 20000}) - - def testSampleRateBadType(self): - placeholder = array_ops.placeholder(dtypes.float32) - with self.assertRaises(TypeError): - self._loadFileAndTest('mono_16khz.mp3', 'mp3', 0.57, 20000.0, 1, - samples_per_second_tensor=placeholder, - feed_dict={placeholder: 20000.0}) - - def testSampleRateBadValue_Zero(self): - placeholder = array_ops.placeholder(dtypes.int32) - with six.assertRaisesRegex(self, Exception, - r'samples_per_second must be positive'): - self._loadFileAndTest('mono_16khz.mp3', 'mp3', 0.57, 20000.0, 1, - samples_per_second_tensor=placeholder, - feed_dict={placeholder: 0}) - - def testSampleRateBadValue_Negative(self): - placeholder = array_ops.placeholder(dtypes.int32) - with six.assertRaisesRegex(self, Exception, - r'samples_per_second must be positive'): - self._loadFileAndTest('mono_16khz.mp3', 'mp3', 0.57, 20000.0, 1, - samples_per_second_tensor=placeholder, - feed_dict={placeholder: -2}) - - def testInvalidFileFormat(self): - with six.assertRaisesRegex(self, Exception, - r'file_format must be one of'): - self._loadFileAndTest('mono_16khz.mp3', 'docx', 0.57, 20000, 1) - - def testStaticShapeInference_ConstantChannelCount(self): - with self.cached_session(): - audio_op = ffmpeg.decode_audio(b'~~~ wave ~~~', - file_format='wav', - samples_per_second=44100, - channel_count=2) - self.assertEqual([None, 2], audio_op.shape.as_list()) - - def testStaticShapeInference_NonConstantChannelCount(self): - with self.cached_session(): - channel_count = array_ops.placeholder(dtypes.int32) - audio_op = ffmpeg.decode_audio(b'~~~ wave ~~~', - file_format='wav', - samples_per_second=44100, - channel_count=channel_count) - self.assertEqual([None, None], audio_op.shape.as_list()) - - def testStaticShapeInference_ZeroChannelCountInvalid(self): - with self.cached_session(): - with six.assertRaisesRegex(self, Exception, - r'channel_count must be positive'): - ffmpeg.decode_audio(b'~~~ wave ~~~', - file_format='wav', - samples_per_second=44100, - channel_count=0) - - def testStaticShapeInference_NegativeChannelCountInvalid(self): - with self.cached_session(): - with six.assertRaisesRegex(self, Exception, - r'channel_count must be positive'): - ffmpeg.decode_audio(b'~~~ wave ~~~', - file_format='wav', - samples_per_second=44100, - channel_count=-2) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/ffmpeg/decode_video_op.cc b/tensorflow/contrib/ffmpeg/decode_video_op.cc deleted file mode 100644 index 0bfdc2781aa..00000000000 --- a/tensorflow/contrib/ffmpeg/decode_video_op.cc +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include - -#include -#include - -#include "tensorflow/contrib/ffmpeg/ffmpeg_lib.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/logging.h" - -namespace tensorflow { -namespace ffmpeg { - -class DecodeVideoOp : public OpKernel { - public: - explicit DecodeVideoOp(OpKernelConstruction* context) : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - OP_REQUIRES( - context, context->num_inputs() == 1, - errors::InvalidArgument("DecodeVideo requires exactly 1 input.")); - const Tensor& contents_tensor = context->input(0); - - OP_REQUIRES(context, TensorShapeUtils::IsScalar(contents_tensor.shape()), - errors::InvalidArgument( - "contents must be a rank-0 tensor but got shape ", - contents_tensor.shape().DebugString())); - const tensorflow::StringPiece contents = - contents_tensor.scalar()(); - - // Write the input data to a temp file. - string extension; - const string temp_filename = io::GetTempFilename(extension); - OP_REQUIRES_OK(context, WriteFile(temp_filename, contents)); - FileDeleter deleter(temp_filename); - - uint32 width = 0; - uint32 height = 0; - uint32 frames = 0; - - // Run FFmpeg on the data and verify results. - std::vector output_data; - const Status result = ffmpeg::ReadVideoFile(temp_filename, &output_data, - &width, &height, &frames); - if (result.code() == error::Code::NOT_FOUND) { - OP_REQUIRES( - context, result.ok(), - errors::Unavailable("FFmpeg must be installed to run this op. FFmpeg " - "can be found at http://www.ffmpeg.org.")); - } else if (result.code() == error::UNKNOWN) { - LOG(ERROR) << "Ffmpeg failed with error '" << result.error_message() - << "'. Returning empty tensor."; - Tensor* output = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(0, TensorShape({0, 0}), &output)); - return; - } else { - OP_REQUIRES_OK(context, result); - } - OP_REQUIRES(context, !output_data.empty(), - errors::Unknown("No output created by FFmpeg.")); - OP_REQUIRES( - context, output_data.size() == (frames * height * width * 3), - errors::Unknown("Output created by FFmpeg [", output_data.size(), - "] does not match description [", frames, ", ", height, - ", ", width, ", 3]")); - Tensor* output = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output( - 0, TensorShape({frames, height, width, 3}), &output)); - auto output_flat = output->flat(); - std::copy_n(output_data.begin(), output_data.size(), &output_flat(0)); - } -}; - -REGISTER_KERNEL_BUILDER(Name("DecodeVideo").Device(DEVICE_CPU), DecodeVideoOp); - -REGISTER_OP("DecodeVideo") - .Input("contents: string") - .Output("output: uint8") - .SetShapeFn([](shape_inference::InferenceContext* c) { - c->set_output(0, c->UnknownShapeOfRank(4)); - return Status::OK(); - }) - .Doc(R"doc( -Processes the contents of an video file into a tensor using FFmpeg to decode -the file. - -contents: The binary contents of the video file to decode. This is a - scalar. -output: A rank-4 `Tensor` that has `[frames, height, width, 3]` RGB as output. -)doc"); - -} // namespace ffmpeg -} // namespace tensorflow diff --git a/tensorflow/contrib/ffmpeg/decode_video_op_test.py b/tensorflow/contrib/ffmpeg/decode_video_op_test.py deleted file mode 100644 index 478b927fe0e..00000000000 --- a/tensorflow/contrib/ffmpeg/decode_video_op_test.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Tests for third_party.tensorflow.contrib.ffmpeg.decode_video_op.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os.path - -import six # pylint: disable=unused-import - -from tensorflow.contrib import ffmpeg -from tensorflow.python.ops import image_ops -from tensorflow.python.platform import resource_loader -from tensorflow.python.platform import test - - -class DecodeVideoOpTest(test.TestCase): - - def _loadFileAndTest(self, filename, width, height, frames, bmp_filename, - index): - """Loads an video file and validates the output tensor. - - Args: - filename: The filename of the input file. - width: The width of the video. - height: The height of the video. - frames: The frames of the video. - bmp_filename: The filename for the bmp file. - index: Index location inside the video. - """ - with self.cached_session(): - path = os.path.join(resource_loader.get_data_files_path(), 'testdata', - filename) - with open(path, 'rb') as f: - contents = f.read() - - bmp_path = os.path.join(resource_loader.get_data_files_path(), 'testdata', - bmp_filename) - with open(bmp_path, 'rb') as f: - bmp_contents = f.read() - - image_op = image_ops.decode_bmp(bmp_contents) - image = image_op.eval() - self.assertEqual(image.shape, (height, width, 3)) - video_op = ffmpeg.decode_video(contents) - video = video_op.eval() - self.assertEqual(video.shape, (frames, height, width, 3)) - # ffmpeg produces results where channels can be off 1. - self.assertAllClose(video[index, :, :, :], image, atol=1) - - def testMp4(self): - self._loadFileAndTest('small.mp4', 560, 320, 166, 'small_100.bmp', 99) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/ffmpeg/default/BUILD b/tensorflow/contrib/ffmpeg/default/BUILD deleted file mode 100644 index 8a28a4068a0..00000000000 --- a/tensorflow/contrib/ffmpeg/default/BUILD +++ /dev/null @@ -1,77 +0,0 @@ -# Description: -# Libraries and kernels for manipulating audio and video using FFmpeg. -# (https://www.ffmpeg.org) - -load("//tensorflow:tensorflow.bzl", "tf_cc_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -cc_library( - name = "ffmpeg_lib", - srcs = ["ffmpeg_lib.cc"], - hdrs = [ - # Header is shared between implementations. - "//tensorflow/contrib/ffmpeg:ffmpeg_lib.h", - ], - deps = [ - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - "@com_google_protobuf//:protobuf_headers", - ], -) - -tf_cc_test( - name = "ffmpeg_lib_utility_test", - srcs = ["ffmpeg_lib_utility_test.cc"], - deps = [ - ":ffmpeg_lib", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -tf_cc_test( - name = "ffmpeg_lib_installed_test", - srcs = ["ffmpeg_lib_test.cc"], - args = [ - "--should_ffmpeg_be_installed=true", - ], - data = [ - "//tensorflow/contrib/ffmpeg:test_data", - ], - tags = [ - "local", - "manual", - ], - deps = [ - ":ffmpeg_lib", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/core:test", - ], -) - -tf_cc_test( - name = "ffmpeg_lib_uninstalled_test", - srcs = ["ffmpeg_lib_test.cc"], - args = [ - "--should_ffmpeg_be_installed=false", - ], - tags = [ - "local", - "manual", - ], - deps = [ - ":ffmpeg_lib", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/core:test", - ], -) diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc deleted file mode 100644 index cca1a054193..00000000000 --- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc +++ /dev/null @@ -1,422 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include "tensorflow/contrib/ffmpeg/ffmpeg_lib.h" - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/lib/strings/numbers.h" -#include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/platform/byte_order.h" -#include "tensorflow/core/platform/env.h" - -using tensorflow::strings::StrCat; - -namespace tensorflow { -namespace ffmpeg { -namespace { - -const char kFfmpegExecutable[] = "ffmpeg"; -const int32 kDefaultProbeSize = 5000000; // 5MB - -std::vector FfmpegAudioCommandLine(const string& input_filename, - const string& output_filename, - const string& input_format_id, - int32 samples_per_second, - int32 channel_count, - const string& stream) { - std::vector command({ - "-nostats", // No additional progress display. - "-nostdin", // No interactive commands accepted. - "-f", input_format_id, // eg: "mp3" - "-probesize", StrCat(kDefaultProbeSize), "-i", input_filename, - "-loglevel", "error", // Print errors only. - "-hide_banner", // Skip printing build options, version, etc. - "-map_metadata", "-1", // Copy global metadata from input to output. - "-vn", // No video recording. - "-ac:a:0", StrCat(channel_count), "-ar:a:0", StrCat(samples_per_second), - // Output set (in several ways) to signed 16-bit little-endian ints. - "-codec:a:0", "pcm_s16le", "-sample_fmt", "s16", "-f", "s16le", - "-sn", // No subtitle recording. - "-y" // Overwrite output file. - }); - if (!stream.empty()) { - command.emplace_back("-map"); - command.emplace_back(StrCat("0:", stream)); - } - command.emplace_back(StrCat(output_filename)); - - return command; -} - -std::vector FfmpegVideoCommandLine(const string& input_filename, - const string& output_filename) { - return {"-nostats", // No additional progress display. - "-nostdin", // No interactive commands accepted. - "-i", input_filename, "-f", "image2pipe", "-probesize", - StrCat(kDefaultProbeSize), "-loglevel", - // Info is needed to get the information about stream, etc. - // It is generated to a separate file, not stdout/stderr. - "info", - "-hide_banner", // Skip printing build options, version, etc. - "-vcodec", "rawvideo", "-pix_fmt", "rgb24", - "-y", // Overwrite output file. - StrCat(output_filename)}; -} - -// Is a named binary installed and executable by the current process? -// Note that this is harder than it seems like it should be... -bool IsBinaryInstalled(const string& binary_name) { - string path = ::getenv("PATH"); - for (const string& dir : str_util::Split(path, ':')) { - const string binary_path = io::JoinPath(dir, binary_name); - char absolute_path[PATH_MAX + 1]; - if (::realpath(binary_path.c_str(), absolute_path) == nullptr) { - continue; - } - struct stat statinfo; - int result = ::stat(absolute_path, &statinfo); - if (result < 0) { - continue; - } - if (!S_ISREG(statinfo.st_mode)) { - continue; - } - - // Is the current user able to execute the file? - if (statinfo.st_uid == ::geteuid() && statinfo.st_mode & S_IXUSR) { - return true; - } - // Is the current group able to execute the file? - if (statinfo.st_uid == ::getegid() && statinfo.st_mode & S_IXGRP) { - return true; - } - // Is anyone able to execute the file? - if (statinfo.st_mode & S_IXOTH) { - return true; - } - } - return false; -} - -[[noreturn]] int ExecuteFfmpeg(const std::vector& args) { - std::vector args_chars; - std::transform(args.begin(), args.end(), std::back_inserter(args_chars), - [](const string& s) { return const_cast(s.c_str()); }); - args_chars.push_back(nullptr); - ::execvp(kFfmpegExecutable, args_chars.data()); - // exec only returns on error. - const int error = errno; - LOG(ERROR) << "FFmpeg could not be executed: " << strerror(error); - ::_exit(error); -} - -// Reads a PCM file using signed little endian 16-bit encoding (s16le). -std::vector ReadPcmFile(const string& filename) { - string raw_data; - TF_QCHECK_OK(ReadFileToString(Env::Default(), filename, &raw_data)) - << "Could not read FFmpeg output file: " << filename; - - std::vector samples; - const int32 sample_count = raw_data.size() / sizeof(int16); - samples.reserve(sample_count); - - for (int32 i = 0; i < sample_count; ++i) { - // Most of this is jumping through hoops in the standard to convert some - // bits into the right format. I hope that an optimizing compiler will - // remove almost all of this code. - char raw[2] = {raw_data[i * 2], raw_data[i * 2 + 1]}; - if (!port::kLittleEndian) { - std::swap(raw[0], raw[1]); - } - int16 host_order; - ::memcpy(&host_order, raw, sizeof(host_order)); - const double normalized = - static_cast(host_order) / std::numeric_limits::max(); - samples.push_back(normalized); - } - return samples; -} - -template -string LittleEndianData(UInt data) { - static_assert(std::is_unsigned::value, "UInt must be unsigned"); - string str; - for (size_t i = 0; i < sizeof(UInt); ++i) { - const unsigned char bits = static_cast(data & 0xFFU); - char ch; - ::memcpy(&ch, &bits, sizeof(bits)); - str.push_back(ch); - data >>= 8; - } - return str; -} - -string LittleEndianDataInt(uint32 data) { - return LittleEndianData(data); -} - -string LittleEndianDataShort(uint16 data) { - return LittleEndianData(data); -} - -string WavHeader(int32 samples_per_second, int32 channel_count, - const std::vector& samples) { - string header = "RIFF"; - header += LittleEndianDataInt(36U + samples.size() * sizeof(int16)); - header += "WAVEfmt "; - header += LittleEndianDataInt(16); - header += LittleEndianDataShort(1); - header += LittleEndianDataShort(channel_count); - header += LittleEndianDataInt(samples_per_second); - header += - LittleEndianDataInt(samples_per_second * channel_count * sizeof(int16)); - header += LittleEndianDataShort(channel_count * sizeof(int16)); - header += LittleEndianDataShort(16); - header += "data"; - header += LittleEndianDataInt(samples.size() * sizeof(int16)); - CHECK_EQ(header.size(), 44); - return header; -} - -// Creates the contents of a .wav file using pcm_s16le format (signed 16 bit -// little endian integers). -string BuildWavFile(int32 samples_per_second, int32 channel_count, - const std::vector& samples) { - string data = WavHeader(samples_per_second, channel_count, samples); - data.reserve(data.size() + samples.size() * sizeof(int16)); - for (float value : samples) { - const int16 quantized = - static_cast(value * std::numeric_limits::max()); - char raw[2]; - ::memcpy(raw, &quantized, sizeof(int16)); - if (!port::kLittleEndian) { - std::swap(raw[0], raw[1]); - } - data.push_back(raw[0]); - data.push_back(raw[1]); - } - return data; -} - -Status ReadInfoFile(const string& filename, uint32* width, uint32* height, - uint32* frames) { - string data; - TF_QCHECK_OK(ReadFileToString(Env::Default(), filename, &data)) - << "Could not read FFmpeg file: " << filename; - bool in_output = false; - bool in_mapping = false; - uint32 frames_value = 0; - uint32 height_value = 0; - uint32 width_value = 0; - for (const string& line : str_util::Split(data, '\n')) { - // Output starts with the first line of `Output #..`. - // Further processing output region starts next line so we could continue - // the loop. - if (!in_output && line.find("Output #") == 0) { - in_output = true; - in_mapping = false; - continue; - } - // Stream mapping starts with the first line of `Stream mapping`, it also - // signals the end of Output section. - // Further processing of stream mapping region starts next line so we could - // continue the loop. - if (!in_mapping && line.find("Stream mapping:") == 0) { - in_output = false; - in_mapping = true; - continue; - } - if (in_output) { - // We only look for the first stream in output `Stream #0`. - // Once processed we will not further process output section. - if (line.find(" Stream #") == 0) { - size_t p = line.find(", rgb24, ", 24); - if (p != std::string::npos) { - string rgb24 = line.substr(p + 9, line.find(" ", p + 9)); - rgb24 = rgb24.substr(0, rgb24.find(",")); - // Strip anything after " ", in case the format is - // `640x360 [SAR 1:1 DAR 16:9]` - rgb24 = rgb24.substr(0, rgb24.find(" ")); - string rgb24_width = rgb24.substr(0, rgb24.find("x")); - string rgb24_height = rgb24.substr(rgb24_width.length() + 1); - if (strings::safe_strtou32(rgb24_width, &width_value) && - strings::safe_strtou32(rgb24_height, &height_value)) { - in_output = false; - } - } - } - continue; - } - if (in_mapping) { - // We only look for the first stream mapping to have the number of the - // frames. - // Once processed we will not further process stream mapping section. - if (line.find("frame=") == 0) { - // The format might be `frame= 166 ` or `frame=12488 ` - string number = line.substr(6); - number = number.substr(number.find_first_not_of(" ")); - number = number.substr(0, number.find(" ")); - if (strings::safe_strtou32(number, &frames_value)) { - in_mapping = false; - } - } - continue; - } - } - if (frames_value == 0 || height_value == 0 || width_value == 0) { - return errors::Unknown("Not enough video info returned by FFmpeg [", - frames_value, ", ", height_value, ", ", width_value, - ", 3]"); - } - *width = width_value; - *height = height_value; - *frames = frames_value; - return Status::OK(); -} - -} // namespace - -FileDeleter::~FileDeleter() { - Env& env = *Env::Default(); - env.DeleteFile(filename_).IgnoreError(); -} - -Status WriteFile(const string& filename, StringPiece contents) { - Env& env = *Env::Default(); - std::unique_ptr file; - TF_RETURN_IF_ERROR(env.NewWritableFile(filename, &file)); - TF_RETURN_IF_ERROR(file->Append(contents)); - TF_RETURN_IF_ERROR(file->Close()); - return Status::OK(); -} - -Status ReadAudioFile(const string& filename, const string& audio_format_id, - int32 samples_per_second, int32 channel_count, - const string& stream, std::vector* output_samples) { - // Create an argument list. - string output_filename = io::GetTempFilename("raw"); - const std::vector args = - FfmpegAudioCommandLine(filename, output_filename, audio_format_id, - samples_per_second, channel_count, stream); - // Unfortunately, it's impossible to differentiate an exec failure due to the - // binary being missing and an error from the binary's execution. Therefore, - // check to see if the binary *should* be available. If not, return an error - // that will be converted into a helpful error message by the TensorFlow op. - if (!IsBinaryInstalled(kFfmpegExecutable)) { - return Status(error::Code::NOT_FOUND, StrCat("FFmpeg could not be found.")); - } - - // Execute ffmpeg and report errors. - pid_t child_pid = ::fork(); - if (child_pid < 0) { - return Status(error::Code::UNKNOWN, - StrCat("fork failed: ", strerror(errno))); - } - if (child_pid == 0) { - ExecuteFfmpeg(args); - } else { - int status_code; - ::waitpid(child_pid, &status_code, 0); - if (status_code) { - return Status(error::Code::UNKNOWN, - StrCat("FFmpeg execution failed: ", status_code)); - } - *output_samples = ReadPcmFile(output_filename); - TF_QCHECK_OK(Env::Default()->DeleteFile(output_filename)) - << output_filename; - return Status::OK(); - } -} - -Status CreateAudioFile(const string& audio_format_id, int32 bits_per_second, - int32 samples_per_second, int32 channel_count, - const std::vector& samples, string* output_data) { - if (audio_format_id != "wav") { - return Status(error::Code::INVALID_ARGUMENT, - "CreateAudioFile only supports the 'wav' audio format."); - } - *output_data = BuildWavFile(samples_per_second, channel_count, samples); - return Status::OK(); -} - -Status ReadVideoFile(const string& filename, std::vector* output_data, - uint32* width, uint32* height, uint32* frames) { - if (!IsBinaryInstalled(kFfmpegExecutable)) { - return Status(error::Code::NOT_FOUND, StrCat("FFmpeg could not be found.")); - } - - string output_filename = io::GetTempFilename("raw"); - string stderr_filename = io::GetTempFilename("err"); - - // Create an argument list. - const std::vector args = - FfmpegVideoCommandLine(filename, output_filename); - // Execute ffmpeg and report errors. - pid_t child_pid = ::fork(); - if (child_pid < 0) { - return Status(error::Code::UNKNOWN, - StrCat("fork failed: ", strerror(errno))); - } - if (child_pid == 0) { - const int fd = - open(stderr_filename.c_str(), O_RDWR | O_CREAT | O_APPEND, 0600); - if (fd < 0) { - const int error = errno; - LOG(ERROR) << "FFmpeg stderr file could not be created: " - << strerror(error); - ::_exit(error); - } - close(STDERR_FILENO); - dup2(fd, STDERR_FILENO); - ExecuteFfmpeg(args); - } else { - int status_code; - if (::waitpid(child_pid, &status_code, 0) < 0) { - return Status(error::Code::UNKNOWN, - StrCat("waitpid failed: ", strerror(errno))); - } - if (status_code) { - return Status(error::Code::UNKNOWN, - StrCat("FFmpeg execution failed: ", status_code)); - } - - TF_QCHECK_OK(ReadInfoFile(stderr_filename, width, height, frames)) - << "Could not read FFmpeg stderr file: " << stderr_filename; - - string raw_data; - TF_QCHECK_OK(ReadFileToString(Env::Default(), output_filename, &raw_data)) - << "Could not read FFmpeg output file: " << output_filename; - output_data->resize(raw_data.size()); - std::copy_n(raw_data.data(), raw_data.size(), output_data->begin()); - - TF_QCHECK_OK(Env::Default()->DeleteFile(output_filename)) - << output_filename; - TF_QCHECK_OK(Env::Default()->DeleteFile(stderr_filename)) - << stderr_filename; - return Status::OK(); - } -} -} // namespace ffmpeg -} // namespace tensorflow diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_test.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_test.cc deleted file mode 100644 index 05728b3d375..00000000000 --- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_test.cc +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include "tensorflow/contrib/ffmpeg/ffmpeg_lib.h" - -#include -#include - -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/platform/thread_annotations.h" -#include "tensorflow/core/util/command_line_flags.h" - -using tensorflow::testing::TensorFlowSrcRoot; - -namespace tensorflow { -namespace ffmpeg { -namespace { - -const char kTestWavFilename[] = "contrib/ffmpeg/testdata/mono_10khz.wav"; -const char kTestMp3Filename[] = "contrib/ffmpeg/testdata/test_sound1.mp3"; - -// Set to true via a command line flag iff the test is expected to have FFmpeg -// installed. -mutex mu(LINKER_INITIALIZED); -bool should_ffmpeg_be_installed GUARDED_BY(mu) = false; - -string ParseTestFlags(int* argc, char** argv) { - mutex_lock l(mu); - std::vector flag_list = { - Flag("should_ffmpeg_be_installed", &should_ffmpeg_be_installed, - "indicates that ffmpeg should be installed")}; - string usage = Flags::Usage(argv[0], flag_list); - if (!Flags::Parse(argc, argv, flag_list)) { - LOG(ERROR) << "\n" << usage; - exit(2); - } - return usage; -} - -TEST(FfmpegLibTest, TestUninstalled) { - { - mutex_lock l(mu); - if (should_ffmpeg_be_installed) { - return; - } - LOG(INFO) << "Assuming FFmpeg is uninstalled."; - } - - string filename = io::JoinPath(TensorFlowSrcRoot(), kTestMp3Filename); - std::vector output_samples; - Status status = ReadAudioFile(filename, "mp3", 5000, 1, &output_samples); - ASSERT_EQ(status.code(), error::Code::NOT_FOUND); -} - -TEST(FfmpegLibTest, TestInstalled) { - { - mutex_lock l(mu); - if (!should_ffmpeg_be_installed) { - return; - } - LOG(INFO) << "Assuming FFmpeg is installed."; - } - - string filename = io::JoinPath(TensorFlowSrcRoot(), kTestMp3Filename); - std::vector output_samples; - Status status = ReadAudioFile(filename, "mp3", 5000, 1, &output_samples); - ASSERT_TRUE(status.ok()); -} - -TEST(FfmpegLibTest, TestRoundTripGeneratedWav) { - { - mutex_lock l(mu); - if (!should_ffmpeg_be_installed) { - return; - } - } - - std::vector sine_wave; - sine_wave.reserve(20000); - for (int i = 0; i < 20000; ++i) { - sine_wave.push_back(std::sin(6.28 * 440.0 * i / 20000.0)); - } - string content; - ASSERT_TRUE(CreateAudioFile("wav", 0, 20000, 1, sine_wave, &content).ok()); - string temp_filename = GetTempFilename("wav"); - ASSERT_TRUE(WriteStringToFile(Env::Default(), temp_filename, content).ok()); - std::vector roundtrip_data; - ASSERT_TRUE( - ReadAudioFile(temp_filename, "wav", 20000, 1, &roundtrip_data).ok()); - EXPECT_EQ(sine_wave.size(), roundtrip_data.size()); - size_t min_size = std::min(sine_wave.size(), roundtrip_data.size()); - for (size_t i = 0; i < min_size; ++i) { - EXPECT_NEAR(sine_wave[i], roundtrip_data[i], 0.01); - EXPECT_LE(roundtrip_data[i], 1.0); - EXPECT_LE(-1.0, roundtrip_data[i]); - } -} - -TEST(FfmpegLibTest, TestRoundTripWav) { - { - mutex_lock l(mu); - if (!should_ffmpeg_be_installed) { - return; - } - } - - string filename = io::JoinPath(TensorFlowSrcRoot(), kTestWavFilename); - std::vector output_samples; - ASSERT_TRUE(ReadAudioFile(filename, "wav", 10000, 1, &output_samples).ok()); - string original_audio; - ASSERT_TRUE(ReadFileToString(Env::Default(), filename, &original_audio).ok()); - - string written_audio; - ASSERT_TRUE( - CreateAudioFile("wav", 0, 10000, 1, output_samples, &written_audio).ok()); - - EXPECT_EQ(original_audio, written_audio); -} - -} // namespace -} // namespace ffmpeg -} // namespace tensorflow - -int main(int argc, char** argv) { - tensorflow::string usage = tensorflow::ffmpeg::ParseTestFlags(&argc, argv); - testing::InitGoogleTest(&argc, argv); - if (argc != 1) { - LOG(ERROR) << usage; - return 2; - } - return RUN_ALL_TESTS(); -} diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc deleted file mode 100644 index d6c885a3242..00000000000 --- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include "tensorflow/contrib/ffmpeg/ffmpeg_lib.h" - -#include -#include -#include -#include - -#include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace ffmpeg { -namespace { - -TEST(FfmpegLibTest, TestTempDirectoryThreading) { - // Testing a fix for a bug that allowed different threads to create - // conflicting temp files. - // See github.com/tensorflow/tensorflow/issues/5804 for details. - const int32 kNumThreads = 10; - const int32 kNumWorkItems = 10000; - static constexpr size_t kStringsPerItem = 100; - Env* environment = Env::Default(); - thread::ThreadPool pool(environment, "test", kNumThreads); - - mutex mu; - std::vector temp_filenames; - temp_filenames.reserve(kNumWorkItems * kStringsPerItem); - - // Queue a large number of work items for the threads to process. Each work - // item creates a temp file and then deletes it. - for (int i = 0; i < kNumWorkItems; ++i) { - pool.Schedule([&mu, &temp_filenames, environment]() { - std::array buffer; - for (int32 j = 0; j < kStringsPerItem; ++j) { - buffer[j] = io::GetTempFilename("mp3"); - TF_QCHECK_OK(environment->DeleteFile(buffer[j])); - } - mutex_lock l(mu); - for (const auto& fn : buffer) { - temp_filenames.push_back(fn); - } - }); - } - - // Wait until all work items are complete. - while (true) { - mutex_lock l(mu); - if (temp_filenames.size() == kNumWorkItems * kStringsPerItem) { - break; - } - } - - // Check that no duplicates are created. - std::set unique_filenames; - mutex_lock l(mu); - for (const auto& fn : temp_filenames) { - ASSERT_TRUE(unique_filenames.insert(fn).second); - } -} - -} // namespace -} // namespace ffmpeg -} // namespace tensorflow diff --git a/tensorflow/contrib/ffmpeg/encode_audio_op.cc b/tensorflow/contrib/ffmpeg/encode_audio_op.cc deleted file mode 100644 index ee418fb9020..00000000000 --- a/tensorflow/contrib/ffmpeg/encode_audio_op.cc +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include - -#include "tensorflow/contrib/ffmpeg/ffmpeg_lib.h" -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" - -namespace tensorflow { -namespace ffmpeg { -namespace { - -/* - * Encoding implementation, shared across V1 and V2 ops. Creates a new - * output in the context. - */ -void Encode(OpKernelContext* context, const Tensor& contents, - const string& file_format, const int32 bits_per_second, - const int32 samples_per_second) { - std::vector samples; - samples.reserve(contents.NumElements()); - for (int32 i = 0; i < contents.NumElements(); ++i) { - samples.push_back(contents.flat()(i)); - } - const int32 channel_count = contents.dim_size(1); - string encoded_audio; - OP_REQUIRES_OK( - context, CreateAudioFile(file_format, bits_per_second, samples_per_second, - channel_count, samples, &encoded_audio)); - - // Copy the encoded audio file to the output tensor. - Tensor* output = nullptr; - OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape(), &output)); - output->scalar()() = encoded_audio; -} - -} // namespace - -/* - * Supersedes `EncodeAudioOp`. Allows all parameters to be inputs - * instead of attributes, so that the sample rate (and, probably less - * usefully, the output file format) can be given as tensors rather than - * constants only. - */ -class EncodeAudioOpV2 : public OpKernel { - public: - explicit EncodeAudioOpV2(OpKernelConstruction* context) : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - OP_REQUIRES( - context, context->num_inputs() == 4, - errors::InvalidArgument("EncodeAudio requires exactly four inputs.")); - - const Tensor& contents = context->input(0); - const Tensor& file_format_tensor = context->input(1); - const Tensor& samples_per_second_tensor = context->input(2); - const Tensor& bits_per_second_tensor = context->input(3); - - OP_REQUIRES(context, TensorShapeUtils::IsMatrix(contents.shape()), - errors::InvalidArgument( - "sampled_audio must be a rank-2 tensor but got shape ", - contents.shape().DebugString())); - OP_REQUIRES( - context, contents.NumElements() <= std::numeric_limits::max(), - errors::InvalidArgument( - "sampled_audio cannot have more than 2^31 entries. Shape = ", - contents.shape().DebugString())); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(file_format_tensor.shape()), - errors::InvalidArgument( - "file_format must be a rank-0 tensor but got shape ", - file_format_tensor.shape().DebugString())); - OP_REQUIRES(context, - TensorShapeUtils::IsScalar(samples_per_second_tensor.shape()), - errors::InvalidArgument( - "samples_per_second must be a rank-0 tensor but got shape ", - samples_per_second_tensor.shape().DebugString())); - OP_REQUIRES(context, - TensorShapeUtils::IsScalar(bits_per_second_tensor.shape()), - errors::InvalidArgument( - "bits_per_second must be a rank-0 tensor but got shape ", - bits_per_second_tensor.shape().DebugString())); - - const string file_format = - absl::AsciiStrToLower(file_format_tensor.scalar()()); - const int32 samples_per_second = - samples_per_second_tensor.scalar()(); - const int32 bits_per_second = bits_per_second_tensor.scalar()(); - - OP_REQUIRES(context, file_format == "wav", - errors::InvalidArgument( - "file_format must be \"wav\", but got: ", file_format)); - OP_REQUIRES(context, samples_per_second > 0, - errors::InvalidArgument( - "samples_per_second must be positive, but got: ", - samples_per_second)); - OP_REQUIRES( - context, bits_per_second > 0, - errors::InvalidArgument("bits_per_second must be positive, but got: ", - bits_per_second)); - - Encode(context, contents, file_format, bits_per_second, samples_per_second); - } -}; - -REGISTER_KERNEL_BUILDER(Name("EncodeAudioV2").Device(DEVICE_CPU), - EncodeAudioOpV2); - -REGISTER_OP("EncodeAudioV2") - .Input("sampled_audio: float") - .Input("file_format: string") - .Input("samples_per_second: int32") - .Input("bits_per_second: int32") - .Output("contents: string") - .SetShapeFn(shape_inference::ScalarShape) - .Doc(R"doc( -Processes a `Tensor` containing sampled audio with the number of channels -and length of the audio specified by the dimensions of the `Tensor`. The -audio is converted into a string that, when saved to disk, will be equivalent -to the audio in the specified audio format. - -The input audio has one row of the tensor for each channel in the audio file. -Each channel contains audio samples starting at the beginning of the audio and -having `1/samples_per_second` time between them. The output file will contain -all of the audio channels contained in the tensor. - -sampled_audio: A rank-2 float tensor containing all tracks of the audio. - Dimension 0 is time and dimension 1 is the channel. -file_format: A string or rank-0 string tensor describing the audio file - format. This value must be `"wav"`. -samples_per_second: The number of samples per second that the audio should - have, as an int or rank-0 `int32` tensor. This value must be - positive. -bits_per_second: The approximate bitrate of the encoded audio file, as - an int or rank-0 `int32` tensor. This is ignored by the "wav" file - format. -contents: The binary audio file contents, as a rank-0 string tensor. -)doc"); - -/* - * Deprecated in favor of EncodeAudioOpV2. - */ -class EncodeAudioOp : public OpKernel { - public: - explicit EncodeAudioOp(OpKernelConstruction* context) : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr("file_format", &file_format_)); - file_format_ = absl::AsciiStrToLower(file_format_); - OP_REQUIRES(context, file_format_ == "wav", - errors::InvalidArgument("file_format arg must be \"wav\".")); - - OP_REQUIRES_OK( - context, context->GetAttr("samples_per_second", &samples_per_second_)); - OP_REQUIRES(context, samples_per_second_ > 0, - errors::InvalidArgument("samples_per_second must be > 0.")); - OP_REQUIRES_OK(context, - context->GetAttr("bits_per_second", &bits_per_second_)); - } - - void Compute(OpKernelContext* context) override { - // Get and verify the input data. - OP_REQUIRES( - context, context->num_inputs() == 1, - errors::InvalidArgument("EncodeAudio requires exactly one input.")); - const Tensor& contents = context->input(0); - OP_REQUIRES(context, TensorShapeUtils::IsMatrix(contents.shape()), - errors::InvalidArgument( - "sampled_audio must be a rank 2 tensor but got shape ", - contents.shape().DebugString())); - OP_REQUIRES( - context, contents.NumElements() <= std::numeric_limits::max(), - errors::InvalidArgument( - "sampled_audio cannot have more than 2^31 entries. Shape = ", - contents.shape().DebugString())); - - Encode(context, contents, file_format_, bits_per_second_, - samples_per_second_); - } - - private: - string file_format_; - int32 samples_per_second_; - int32 bits_per_second_; -}; - -REGISTER_KERNEL_BUILDER(Name("EncodeAudio").Device(DEVICE_CPU), EncodeAudioOp); - -REGISTER_OP("EncodeAudio") - .Input("sampled_audio: float") - .Output("contents: string") - .Attr("file_format: string") - .Attr("samples_per_second: int") - .Attr("bits_per_second: int = 192000") - .SetShapeFn(shape_inference::ScalarShape) - .Doc(R"doc( -Processes a `Tensor` containing sampled audio with the number of channels -and length of the audio specified by the dimensions of the `Tensor`. The -audio is converted into a string that, when saved to disk, will be equivalent -to the audio in the specified audio format. - -The input audio has one row of the tensor for each channel in the audio file. -Each channel contains audio samples starting at the beginning of the audio and -having `1/samples_per_second` time between them. The output file will contain -all of the audio channels contained in the tensor. - -sampled_audio: A rank 2 tensor containing all tracks of the audio. Dimension 0 - is time and dimension 1 is the channel. -contents: The binary audio file contents. -file_format: A string describing the audio file format. This must be "wav". -samples_per_second: The number of samples per second that the audio should have. -bits_per_second: The approximate bitrate of the encoded audio file. This is - ignored by the "wav" file format. -)doc"); - -} // namespace ffmpeg -} // namespace tensorflow diff --git a/tensorflow/contrib/ffmpeg/encode_audio_op_test.py b/tensorflow/contrib/ffmpeg/encode_audio_op_test.py deleted file mode 100644 index eb4325da82b..00000000000 --- a/tensorflow/contrib/ffmpeg/encode_audio_op_test.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Tests for third_party.tensorflow.contrib.ffmpeg.encode_audio_op.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os.path - -import six - -from tensorflow.contrib import ffmpeg -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import resource_loader -from tensorflow.python.platform import test - - -class EncodeAudioOpTest(test.TestCase): - - def setUp(self): - super(EncodeAudioOpTest, self).setUp() - path = os.path.join(resource_loader.get_data_files_path(), - 'testdata/mono_10khz.wav') - with open(path, 'rb') as f: - self._contents = f.read() - - def _compareWavFiles(self, original, encoded): - """Compares the important bits of two WAV files. - - Some encoders will create a slightly different header to the WAV file. - This compares only the important bits of the header as well as the contents. - - Args: - original: Contents of the original .wav file. - encoded: Contents of the new, encoded .wav file. - """ - self.assertLess(44, len(original)) - self.assertLess(44, len(encoded)) - self.assertEqual(original[:4], encoded[:4]) - # Skip file size - self.assertEqual(original[8:16], encoded[8:16]) - # Skip header size - self.assertEqual(original[20:36], encoded[20:36]) - # Skip extra bits inserted by ffmpeg. - self.assertEqual(original[original.find(b'data'):], - encoded[encoded.find(b'data'):]) - - def testRoundTrip(self): - """Reads a wav file, writes it, and compares them.""" - with self.cached_session(): - audio_op = ffmpeg.decode_audio( - self._contents, - file_format='wav', - samples_per_second=10000, - channel_count=1) - encode_op = ffmpeg.encode_audio( - audio_op, file_format='wav', samples_per_second=10000) - encoded_contents = encode_op.eval() - self._compareWavFiles(self._contents, encoded_contents) - - def testRoundTripWithPlaceholderSampleRate(self): - with self.cached_session(): - placeholder = array_ops.placeholder(dtypes.int32) - audio_op = ffmpeg.decode_audio( - self._contents, - file_format='wav', - samples_per_second=placeholder, - channel_count=1) - encode_op = ffmpeg.encode_audio( - audio_op, file_format='wav', samples_per_second=placeholder) - encoded_contents = encode_op.eval(feed_dict={placeholder: 10000}) - self._compareWavFiles(self._contents, encoded_contents) - - def testFloatingPointSampleRateInvalid(self): - with self.cached_session(): - with self.assertRaises(TypeError): - ffmpeg.encode_audio( - [[0.0], [1.0]], - file_format='wav', - samples_per_second=12345.678) - - def testZeroSampleRateInvalid(self): - with self.cached_session() as sess: - encode_op = ffmpeg.encode_audio( - [[0.0], [1.0]], - file_format='wav', - samples_per_second=0) - with six.assertRaisesRegex(self, Exception, 'must be positive'): - sess.run(encode_op) - - def testNegativeSampleRateInvalid(self): - with self.cached_session() as sess: - encode_op = ffmpeg.encode_audio( - [[0.0], [1.0]], - file_format='wav', - samples_per_second=-2) - with six.assertRaisesRegex(self, Exception, 'must be positive'): - sess.run(encode_op) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/ffmpeg/ffmpeg_lib.h b/tensorflow/contrib/ffmpeg/ffmpeg_lib.h deleted file mode 100644 index bf2aa755458..00000000000 --- a/tensorflow/contrib/ffmpeg/ffmpeg_lib.h +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#ifndef TENSORFLOW_CONTRIB_FFMPEG_FFMPEG_LIB_H_ -#define TENSORFLOW_CONTRIB_FFMPEG_FFMPEG_LIB_H_ - -#include -#include - -#include "tensorflow/core/lib/core/status.h" - -namespace tensorflow { -namespace ffmpeg { - -// Cleans up a file on destruction. -class FileDeleter { - public: - explicit FileDeleter(const string& filename) : filename_(filename) {} - ~FileDeleter(); - - private: - const string filename_; -}; - -// Writes binary data to a file. -Status WriteFile(const string& filename, tensorflow::StringPiece contents); - -// Reads an audio file using ffmpeg and converts it into an array of samples in -// [-1.0, 1.0]. If there are multiple channels in the audio then each frame will -// contain a separate sample for each channel. Frames are ordered by time. -Status ReadAudioFile(const string& filename, const string& audio_format_id, - int32 samples_per_second, int32 channel_count, - const string& stream, std::vector* output_samples); - -// Creates an audio file using ffmpeg in a specific format. The samples are in -// [-1.0, 1.0]. If there are multiple channels in the audio then each frame will -// contain a separate sample for each channel. Frames are ordered by time. -// Currently, the implementation only supports wav files, and ffmpeg is not used -// to create them. -Status CreateAudioFile(const string& audio_format_id, int32 bits_per_second, - int32 samples_per_second, int32 channel_count, - const std::vector& samples, string* output_data); - -// Reads an video file using ffmpeg and converts it into a RGB24 in uint8 -// [frames, height, width, 3]. The w, h, and frames are obtained from ffmpeg. -Status ReadVideoFile(const string& filename, std::vector* output_data, - uint32* width, uint32* height, uint32* frames); - -} // namespace ffmpeg -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_FFMPEG_DEFAULT_FFMPEG_LIB_H_ diff --git a/tensorflow/contrib/ffmpeg/ffmpeg_ops.py b/tensorflow/contrib/ffmpeg/ffmpeg_ops.py deleted file mode 100644 index 42e212b0c11..00000000000 --- a/tensorflow/contrib/ffmpeg/ffmpeg_ops.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Encoding and decoding audio using FFmpeg.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.ffmpeg.ops import gen_decode_audio_op_py -from tensorflow.contrib.ffmpeg.ops import gen_decode_video_op_py -from tensorflow.contrib.ffmpeg.ops import gen_encode_audio_op_py -from tensorflow.contrib.util import loader -from tensorflow.python.framework import ops -from tensorflow.python.platform import resource_loader -from tensorflow.python.util.deprecation import deprecated - -_ffmpeg_so = loader.load_op_library( - resource_loader.get_path_to_datafile('ffmpeg.so')) - - -@deprecated('2018-09-04', - 'tf.contrib.ffmpeg will be removed in 2.0, the support for video ' - 'and audio will continue to be provided in tensorflow-io: ' - 'https://github.com/tensorflow/io') -def decode_audio(contents, file_format=None, samples_per_second=None, - channel_count=None, stream=None): - """Create an op that decodes the contents of an audio file. - - Note that ffmpeg is free to select the "best" audio track from an mp4. - https://trac.ffmpeg.org/wiki/Map - - Args: - contents: The binary contents of the audio file to decode. This is a - scalar. - file_format: A string or scalar string tensor specifying which - format the contents will conform to. This can be mp3, mp4, ogg, - or wav. - samples_per_second: The number of samples per second that is - assumed, as an `int` or scalar `int32` tensor. In some cases, - resampling will occur to generate the correct sample rate. - channel_count: The number of channels that should be created from the - audio contents, as an `int` or scalar `int32` tensor. If the - `contents` have more than this number, then some channels will - be merged or dropped. If `contents` has fewer than this, then - additional channels will be created from the existing ones. - stream: A string specifying which stream from the content file - should be decoded, e.g., '0' means the 0-th stream. - The default value is '' which leaves the decision to ffmpeg. - - Returns: - A rank-2 tensor that has time along dimension 0 and channels along - dimension 1. Dimension 0 will be `samples_per_second * - length_in_seconds` wide, and dimension 1 will be `channel_count` - wide. If ffmpeg fails to decode the audio then an empty tensor will - be returned. - """ - return gen_decode_audio_op_py.decode_audio_v2( - contents, file_format=file_format, samples_per_second=samples_per_second, - channel_count=channel_count, stream=stream) - - -ops.NotDifferentiable('DecodeAudio') - - -@deprecated('2018-09-04', - 'tf.contrib.ffmpeg will be removed in 2.0, the support for video ' - 'and audio will continue to be provided in tensorflow-io: ' - 'https://github.com/tensorflow/io') -def encode_audio(audio, file_format=None, samples_per_second=None): - """Creates an op that encodes an audio file using sampled audio from a tensor. - - Args: - audio: A rank-2 `Tensor` that has time along dimension 0 and - channels along dimension 1. Dimension 0 is `samples_per_second * - length_in_seconds` long. - file_format: The type of file to encode, as a string or rank-0 - string tensor. "wav" is the only supported format. - samples_per_second: The number of samples in the audio tensor per - second of audio, as an `int` or rank-0 `int32` tensor. - - Returns: - A scalar tensor that contains the encoded audio in the specified file - format. - """ - return gen_encode_audio_op_py.encode_audio_v2( - audio, - file_format=file_format, - samples_per_second=samples_per_second, - bits_per_second=192000) # not used by WAV - - -ops.NotDifferentiable('EncodeAudio') - - -@deprecated('2018-09-04', - 'tf.contrib.ffmpeg will be removed in 2.0, the support for video ' - 'and audio will continue to be provided in tensorflow-io: ' - 'https://github.com/tensorflow/io') -def decode_video(contents): - """Create an op that decodes the contents of a video file. - - Args: - contents: The binary contents of the video file to decode. This is a scalar. - - Returns: - A rank-4 `Tensor` that has `[frames, height, width, 3]` RGB as output. - """ - return gen_decode_video_op_py.decode_video(contents) - - -ops.NotDifferentiable('DecodeVideo') diff --git a/tensorflow/contrib/ffmpeg/testdata/frame100.bin b/tensorflow/contrib/ffmpeg/testdata/frame100.bin deleted file mode 100644 index 219c29a5ea9..00000000000 --- a/tensorflow/contrib/ffmpeg/testdata/frame100.bin +++ /dev/null @@ -1,2408 +0,0 @@ -I*I*I*I*I*I*I*I*I*I*I*I*I*I*I*I*H*H*H*H*H*H*H*H*I*I*I*I*I*I*I*I*K)K)K)K)K)K)K)K)K)K)K)K)K)K)K)K)J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+L-K-K-K-I+I+I+I+I+I+I+I+I+I+I+I+I+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+L-L,L,L,J*J*J*J*J*J*J*J*J*J*J*J*J*J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+I*I*I*I*I*I*I*I*I*I*I*I*I*I*J+H)H)H)H)H)H)H)H)I)I)I)I)I)I)I)I)H*G)G(G(G(G(G(G(G)G)G)G)G)G)G)G)G(G(G(G(F'F'F'F'F'F'F'F'F'F'F'F'F( F( F( F( F( F( E' E' E&D%D%D%D%D%D%D%D% D% D% D% D% D% D% D% D%D%D%D%D%D%D%D%D% D% D% B# B# B# B# B# B# B# B# B# @$ @$ @$ @$ >$ >$ @$ @$ @$ @$ @$ @$ @$ @$ ?# ?# ?# ?# A"A"?#?#?#?#>" >" >" >" >" >" >">">" >" >" >" >" >" >">">" >" >" =! =! =! =!=!=!=!=! =! =! =! =!=!=!=!=! =! < < ; -< < < : : : 9 < < < < < ; -; -; -; -; -; -; -; ; ; ; 9 9 9 9 9 9 7 -7 -7 -7 -7 7 7 -7 -7 -7 -7 -7 -7 -7 -6 6 6 6 4 4 4 4 6 6 777755555 -5 -5 -4 4 4 4 4 2 2 2 2 222 2 2 2 2 11 -1 -1 -1 -1 -1 -1 -1 -1 -1 -00000000..,,..........++------- - - - + + + + + + + + ****))))))))))**))))(())))((&&'(('''''''''''&&&&I*I*I*I*I*I*I*I*I*I*I*I*I*I*I*I*H*H*H*H*H*H*H*H*I*I*I*I*I*I*I*I*K)K)K)K)K)K)K)K)K)K)K)K)K)K)K)K)J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-K-K-K-K-K-K-K-K-K-K-K-K-K-K-K-K-J+L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-J+L-L-L-L-L-L-L-L-L-L-L-J+J+J+L-L,L,L,J*J*J*J*J*J*J*J*J*J*J*J*J*J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+I*I*I*I*I*I*I*I*I*I*I*I*I*I*J+H)H)H)H)H)H)H)H)I)I)I)I)I)I)I)I)H*G)G(G(G(G(G(G(G)G)G)G)G)G)G)G)G(G(G(G(F'F'F'F'F'F'F'F'F'F'F'F'F( F( F( F( F( E' E' E' D%D%D%D%D%D%E&E&D% D% E&E&E&E&E&E&E&D%D%D%D%D%D%D%D% D% D% D% D% D% D% D% D% D% B# B# @$ @$ @$ @$ >$ >$ @$ @$ @$ @$ @$ @$ @$ @$ ?# ?# ?# ?# A"A"?#?#?#?#>" >" >" >" >" ?#?#>">" >" >" >" >" >" >">">" >" >" =! =! =! =!=!=!=!=! =! =! =! =!=!=!=!=! =! =! < ; -< =! =! : : : 9 < < < < < ; -; -; -; -; -; -; -; ; ; ; 9 9 9 9 9 9 7 -7 -7 -7 -7 7 7 -7 -7 -7 -7 -7 -7 -7 -6 6 6 6 4 4 4 4 6 6 777755555 -5 -5 -4 4 4 4 4 2 2 2 2 222 2 2 2 2 11 -1 -1 -1 -1 -1 -1 -1 -1 -1 -00000000..,,..........++------- - - - + + + + + + + + ****))))))))))**))))(())))((&&'(('''''''''''&&&&I*I*I*I*I*I*I*I*I*I*I*I*I*I*I*I*I+I+I+I+I+I+I+I+J+J+J+J+J+J+J+J+K)K)K)K)K)K)K)K)K)K)K)K)K)K)K)K)J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L.L.L.K-K-K-K-K-K-K-K-K-K-K-K-K-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L,L,L,J*J*J*J*J*J*J*L,L,J*J*J*J*J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+I*I*I*H)H)H)H)H)I)I)I)I)I)I)I)I)H*G)G(G(G(G(G(G(G)G)G)G)G)G)G)G)G(G(G(G(F'F'F'F'F'F'F'F'F'F'F'F'F( F( F( F( F( F( E' E' E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&D%E&E&D%D%D% D% D% D% D% D% D% D% D% D% D% B# @$ @$ @$ @$ >$ >$ @$ @$ @$ @$ @$ @$ @$ @$ ?# ?# ?# ?# A"A"?#?#?#?#?#?#?#?#?#?#?#?#?#?#?#>" >" >" >">">" >" >" =! =! =! =!=!=!=!=! =! =! =! =!=!=!=!=! =! =! =! =! =! =! =! ;! : 9 9 < < < < < ; -; -; -; -; -; -; -; ; ; ; : 9 9 9 9 9 9 7 -7 -9 7 99 7 -7 -7 -7 -7 -7 -7 -6 6 6 6 4 4 4 4 6 6 777755555 -5 -5 -5 -5 -4 4 4 2 2 2 3 -222 2 2 2 2 1111 -1 -1 -1 -1 -1 -1 -1 -11100000....0 0 ........++------- - - - + + + + + + + + ****))))))))))**))))(())))((&&'(('''''''''''&&&&I*I*I*I*I*I*I*I*I*I*I*I*I*I*I*I*I+I+I+I+I+I+I+I+J+J+J+J+J+J+J+J+L*L*L*L*L*L*L*L*L*L*L*L*L*L*L*L*L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-M.L.L.L.K-K-K-K-K-K-K-K-K-K-K-K-K-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-M.L,L,L,L,L,L,L,L,L,L,L,L,L,L,L,L,L-J+J+J+J+J+J+J+L-J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+I*I*I*I*I*I*I*H)I)I)I)I)I)I)I)I)G)G)G(G(G(G(G(G(G)G)G)G)G)G)G)G)G(G(F'G(G(G(G(F'G(G(F'G(G(G(G(G(F( F( F( F( F( E' F( F( E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&D% D% D% D% D% D% D% D% D% B&B&B&B&>$ >$ @$ @$ @$ @$ @$ @$ @$ ?# ?# ?# ?# ?# A"A"?#?#?#?#?#?#?#?#?#?#?#?#?#?#?#?#>" >" >">">" >" >" =! =! =! =!=!=!=!=! =! =! =! =!=!=!=!=! =! =! < =! =! =! =! ;! ;! : : < < < < < ; -; -; -; -; -; -; -; ; ; < : : : : : 9 9 7 -7 -9 999 9 7 -7 -7 -7 -7 -7 -6 6 6 6 4 4 4 4 6 6 8 8 7755555 -5 -4 5 -5 -5 -5 -5 -3 -3 -3 -3 -332 2 2 2 2 1111 -2 2 2 1 -1 -1 -1 -11110000....0 0 0 .......++------- - - - + + + + + + + + ****))))))))))**))))))))))((&&'(('''''''''''''''I*I*I*I*I*I*I*I*I*I*I*I*I*I*I*I*I+I+I+I+I+I+I+I+J+J+J+J+J+J+J+J+L*L*L*L*L*L*L*L*L*L*L*L*L*L*L*L*L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-M.M.M.M.M.M.M.M.M.M.M.M.M.M.L.L.L.L.L.L.L.L.L.L.L.L.L.L.L.L.L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-M.M-M-M-L,L,L,L,L,L,L,L,L,L,L,L,L,L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-J+J+J+J+J+J+J+J+J+J+J+J+J+J+L-I*I*I*I*I*I*I*I*I)I)I)I)I)I)I)J*H*H*H)G(H)H)G(G(G)G)G)G)G)G)G)G)H)H)G(G(G(G(G(G(G(G(G(G(G(G(G(G(G)G)G)G)G)F( F( F( F'E&E&E&E&E&F'F'E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&D% D% B&B&B&B&@&@&B&B&B&B&B&B&@$ @$ @$ ?# @$ @$ A"A"?#?#?#?#?#?#?#?#?#?#?#?#?#?#?#?#?#?#>">">" >" >" >" >" >" >">">">">" =! =! =! =! =! =! >" >" =! =! < < =! =! =! ;! ;! ;!;!=!=!=! < ; -< < < < < < < < < < < : : : : : : 9 9 9 9 999 9 9 9 9 9 7 -7 -7 -6 6 6 4 4 4 4 5 -6 8 8 8 8 6 5555 -5 -5 -5 -5 -5 -5 -5 -3 -3 -3 -3 -333 -3 -2 2 201112 2 2 1 -1 -1 -1 -11111111....0 0 0 0 ......,,....--- - - - + + + + + + + + ****))))))))))**))))))*)))((&&((('''''''''''''''J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+I+I+I+I+I+I+I+I+L-L-L-L-L-L-L-L-L*L*L*L*L*L*L*L*L*L*L*L*L*L*L*L*L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-M.M.M.M.M.M.M.M.M.M.M.M.M.M.M/M/M/L.L.L.L.L.L.L.L.L.L.L.L.L.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.L-M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M-M-M-L,L,L,L,L,L,L,M-M-L,L,L,L,L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-I*K,K,I*I*I*I*I*J*J*J*J*J*J*J*J*H*H*H)H)H)H)H)H)H*H*H*H*H*H*G)G)H)H)H)H)G(G(G(G(G(G(G(G(G(G(G(G(G)G)G)G)G)G)G)F( F'E&E&E&E&E&F'F'E&E&E&F'F'F'F'F'F'F'F'E&F'F'E&E&E&E&E&E&E&E&E&E&E&E&E&D% B&B&B&B&@&@&B&B&B&B&B&B&B&@$ @$ @$ @$ @$ B#B#@$@$@$@$?#?#?#?#?#@$@$?#?#?#?#?#?#?#?#?#?#>" >" >" >" >" >">">">">" =! =! >" >" >" >" >" >" >" >" =! =! >" =! =! ;! ;! ;!: < =!=! =! < < < < < < < < < < < < : : : : : : : 9 9 9 999 9 9 9 9 9 9 9 7 -6 6 6 4 4 4 4 5 -6 8 8 8 8 6 6 6 6 5 -5 -5 -6 6 5 -5 -5 -3 -3 -3 -3 -333 -3 -2 2 1112 2 2 2 2 1 -2 2 1 -11111111/...0 0 0 0 0 0 .-..,,....... -. -- - + + + + + + + + ***)))))))))))**))))))*)))((&&((((''''''''''''''J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+K-K-K-K-K-K-K-K-L-L-L-L-L-L-L-L-N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-M.M.M.M.M.M.M.M.M.M.L.L.L.L.L.L.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.N/M/M/N/M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M-M-M-M-M-M-M-M-M-M-M-M-M-M-M-M-M.L-L-L-L-L-L-L-M.L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-K,K,K,K,K,K,L,J*J*J*J*J*J*J*J*J*I*I*I)I)I)I)I)I)I*I*I*I*I*I*I*H)H)H)H)H)H)H)H)G(H)H)G(H)H)H)H)H)G)G)G)G)G)G)G)G)G(G(G(G(G(G(F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&E&C'C'C'C'@&@&B&B&B&B&B&B&B&B&@$ @$ @$ @$ B# B# C# C# B# B# B# B# B# B# @$ @$ @$ @$ @$ @$ @$?#?#?#?#?#?#?#>" >" >" >" >" >" >">">">">" >" >" >" >" >" >" >" >" =! >" >" >" >" <" <" ;!: < < =! =! =! < < < < < < < < < < < < < : : : : : 9 9 9 9 9 9 9 9 -9 -9 9 9 9 7 -7 -7 -7 -5 -5 -5 5 5 -6 9 -9 -8 8 6 6 6 6 6 6 7 -8 8 7 -5 -5 -3 -3 -33333 -3 -3 -3 -2112 2 2 2 2 2 2 2 2 2 12 2 1111/../1 -0 0 0 0 0 ....,,....... -. -. -. -, -+ + + + + + + ********))))))**))))))***)((&&((((''''''''''''''J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+K-K-K-K-K-K-K-K-L-L-L-L-L-L-L-L-N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.L.L.L.L.L.L.M.M.N/N/N/N/N/N/N/N/N/N/N/N/N/N/M/M/N/N/N/N/N/N/N/N/N/N/N/N/N/N/M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.N/N/M.M.M.M.M.M.M.N/N/N/N/N/M-M-M-M-M-M-M-M-M-M-M-M-M-M-M-M-M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.L-L-L-L-L-L-L-L-L-L-L-L-L-L-M.L-K,K,K,K,K,L,L,J*J*J*J*J*J*J*L,J+J+J*I)J*J*I)I)I*I*I*I*I*I*I*I*I*I*H)H)H)H)H)H)H)H)H)H)H)H)H)H)H*H*H*H*H*G)H*G)G(G(G(G(G(G(G(G(F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'F'E&E&C'C'C'C'A'A'C'C'C'C'C'C'B&B&B&@$ B&B&B# B# C# C# B# B# B# B# B# B# @$ @$ @$ @$ @$ @$ @$@$?#?#?#?#?#?#?#?#?#?#?#?#?#?#?#>">" >" >" =! >" >" >" >" >" >" >" >" >" >" <" <" <";!=!=!=! =! =! =! =! =! =! =! =! =! =!=!=!=!=!< : : : : : 9 9 : 9 : : 9 9 -9 -9 9 9 9 9 7 -7 -7 -5 -5 -5 5 6 7 9 -9 -9 -9 -7 -6 6 6 6 6 8 8 8 8 6 6 4 3 -34 4 4 4 4 3 -3 -2212 2 3 -2 2 2 2 2 2 2 12 2 1111////1 -1 -0 0 0 0 0 .0 0 . . 0 0 ..... -. -. -. -, -, -+ + + + + + **********))))**))))))****((&&(('(((((('''''''''J+J+J+J+J+J+J+J+J+J+J+J+J+J+J+L-L-L-L-L-L-L-L-L-O-O-O-O-O-O-O-O-M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.L.L.L.L.L.L.M.M.M.M.N/N/N/N/N/N/N/N/N/N/N/N/N/N/O0O0O0N/N/N/N/N/N/N/N/N/N/N/N/N/N/M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.N/N/N/N/M.M.M.M.M.M.N/N/N/N/N/N/N/N/N/M.M.M.M.M.M.M.N/N/M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.L-L-M.L-L-L-L-L-L-N,N,L-L-L-L-L-L-L-L-J+J+J*J*J*J*J*J*J+J+J+J+J+J+I*I*I*I*I*I*H)H)H)H)H)H)H)H)H)H)H)H)H*H*H*H*H* H* H* H* G(G(G(G(G(H)G(G(F'F'F'G(G(G(G(G(G(G(G(F'G(G(F'F'F'F'F'F'F'F'F'F'F'F'F'E&C'C'E&E&E&E&F&F&E&E&E&E&E&D%D%D%D%D%D%D%D% D% B& B& B# B# B# B# B# D% D%B# @$ @$ @$@$@$ @$ ?# -?# -?# -?# -?# -?# -?# -?# -?# -?# -?# -?# -?# ->" >" >" ?#?#?#>" <" <" <" <" ;! <" <" <" <" <" >" =! =! =! >" >" ? ? ? ? ? ? =! =! =!=!=! =! =! =! ;! ;! : : : 9 -9 -: 9 -: : 9 -9 -9 -9 9 9 9 9 7 -7 -7 -5 -5 -5 5 7 7 9 -9 -9 -9 -7 -7 -6 6 8 8 8 8 8 8 6 6 4 4 4 4 4 4 4 4 3 -3 -222 3 -3 -3 -3 -3 -1 -2 2 2 2 2 2 2 2 2 2 1///// -/ -1 -1 -1 -0 0 0 0 0 . . 0 0 00... -. -. -. -, -, -, -, -+ + + + + + ++++****('''))))))))****((&&(((())))((''((((((L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-M.M.M.M.M.M.O-O-O-O-O-O-O-O-M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.N/N/N/N/N/N/N/N/M/M/M/M/M/M/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/O0O0O0O0N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/O0O0N/N/N/N/M.N/N/N/O0O0N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/M.M.M.M.M.M.M.N/M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.O-N,L-L-L-L-L-L-L-L-J+J+J*J*J*J*J*J*J+J+J+J+J+J+J+J+I*I*I*I*H)H)H)H)H)H)H)H)H)H)H)H)H*H*H*H*H* H* H* H* H)H)H)H)H)H)H)G(G(G(G(G(G(G(G(G(G(G(G(G(G(G(G(G(G(G(G(F'F'F'F'F'F'F'F'F'D(D(F'F'E&E&F&F&E&E&E&E&E&E&D%D%D%D%D%D%D% D% B& B& D% D% D% D% D% D% D%D%B&B&B&@$@$ @$ @$ @$ @$ ?# -?# -?# -?# -?# -?# -?# -?# -?# -?# -?# -?#?#?#?#?#?#=#=#<" =#<" <" <" <" <" <" >" >" >" >" >" >" @! ? ? ? ? ? =! =! =!=!=! =! =! =! ;! ;! : : : 9 -: : : : : : : : : : 9 9 9 9 9 9 7 7 7 7 7 7 ; ; 9 -9 -7 -7 -7 -6 8 8 8 9 9 8 6 6 4 4 4 4 4 4 4 4 4 3 -334 3 -3 -3 -3 -3 -2 3 3 -3 -3 -2 2 2 2 2 2 1///// -/ -1 -1 -1 -1 -.0 0 0 . . 0 0 000 0 0 0 . -. -, -, -, -, -, -, -, -, -+ + ++++*+**(((())))))))****))&&(((())))((((((((((N,N,N,N,N,N,N,N,N,N,N,N,N,N,L-L-O-O-M.M.M.M.M.M.O-O-O-O-O-O-O-O-M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/O0O0O0O0O0O0O0O0O0O0O0O0O0O0Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0P.Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/O0O0O0P1P1O0O0O0P1P1O0O0O0O0Q/Q/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.O-O-O-O-L-L-L-L-L-L-L-M.L-L-L,J*L,L,J+J+J+J+J+J+J+J+J+J+L-L-J+J+J+J+J+J+J*J*J*J*J*J*J*J*I*I*I*I*H* H* H* H* H)H)H)H)H)H)H)H)H)H)H(H(H(H(G(G(G(G(G(G(G(G(G(G(G)G)G)G)G)G)G)G)G)G)F( F( F'F'F'F'G'G'G'G'F'F'F'F'E&E&E&D%E&E&D%D%B& B& B& B& D% D% D% D% D% D% D%D%D%D%D%D%B# B# B# B# B# B# B# B# @$ @$ @$ @$ @$ @$ @$ ?# -A" A" B# A" ?# ?# ?#?#?#?#?#?#?# ?# ?# ?# ?# >" >" >" @! @! @! @! @! @! @! @! @! @! >">"? ? =! =! ;! ;! ;! ;! ;! : : : : : : : : : < < < < ; ; ; ; 9 9 9 9 6 9 ; ; ; ; ; 9 -7 -7 -9 -9 -9 -9 -9 -9 -7 -7 -5 -4 4 5 -4 4 4 4 6 6 6 6 6 5 -5 -6 3 -3 -3 -3 -3 -3 -3 -2 3 -2 2 2 2 1///// -/ -/ -/ -/ -/ -/ -. 0 0 0 0 2 2 220000..,,,,,,,,,++,,,,,+*((******))********)))((())))))((((((((N,N,N,N,N,N,N,N,N,N,N,N,N,N,L-M.O-O-M.M.M.M.M.M.O-O-O-O-O-O-P.P.N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/P1O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/P1P1P1P1P1P1P1P1P1P1O0O0O0O0Q/Q/O0O0O0N/N/N/N/N/N/N/O0O0N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/M.M.M.M.M.M.M.M.M.M.M.M.M.N/M.M.M.M.O-O-O-O-L-L-L-L-M.M.M.M.L-L-L,L,L,L,L-L-L-L-L-L-L-L-L-J+L-L-L-L-J+J+J+J+J*J*J*J*J*J*J*J*J+J+J+J+I+I+I+I+I*I*I*I*I*I*H)H)I*H)H(H(H(H(G(G(G(H)H)G(G(G(G(G(G)G)G)G)G)G)G)G)G)G)G)F( F'F'F'F'G'G'G'G'F'F'F'F'E&E&E&D%E&E&D%D%B& B& B& B& D% D% D% D% D% D% D%D%D%D%D%D%D%B# B# B# B# B# B# B# @$ @$ @$ @$ @$ @$ @$ ?# -A" A" B# B# ?# ?# ?#?#?#?#?#?#?# ?# ?# ?# ?# ?# ?#>" @! @! @! @! @! @! @! @! @! @! >">"@! ? =! =! ;! ;! ;! ;! ;! : : ;! : ;! ;! : : : < < < < ; ; ; ; 9 9 9 9 7 -9 ; ; ; ; ; ; 7 -7 -9 -9 -9 -9 -9 -9 -7 -7 -5 -5 -5 -5 -5 -5 -5 -5 -6 6 6 6 6 6 6 6 4 4 3 -3 -3 -3 -3 -3 -3 -3 -2 2 2 2 0 0 0 0 0 0 / -/ -/ -/ -/ -/ -1 -1 -1 -1 -3 -3 -220000.....,,,,,,,,,,,,,++((*******)+*****)*))))))*)*)))((((((((O-O-O-O-O-O-O-O-O-O-O-O-O-O-M.M.O-O-N/N/N/N/N/N/P.P.P.P.P.P.P.P.N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0P1P1P1P1P1P1R0R0R0R0R0R0R0R0R0R0Q/R0R0Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/P1O0O0O0O0O0O0O0O0O0O0O0O0O0P1P1R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0Q/R0R0R0T/T/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/O0O0O0O0Q/Q/Q/P.P.P.P.P.P.P.O0N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/M-M-P-P-P-P-N+N+M-M-M-M-M-M-M.M.L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-L-M.M.M.L-L-L-L-L-L,L,L,L,L,L,L,L,J+J+J+J+J+J+I+I+I*I*J*J*J*J*J*I)I*H)I)J*J*J*J*J*I*H)I*H)H)I*H)H)H*H*H*G)G(G(G(G(G(G(G(G(F'F'G(G(G'G'G'G'G'G'G'G'G'G'G'F&F'F'E&E&C'C'E&E&E&E&E&E&E&E&E&E&E&E&E&D%D% D% D% D% D& D& D& D& D& D& D% D% D% D% D% B# B# A" B# B# @$ @$ @$ @$ ?# @$ @$ ?# ?# ?# ?# ?# ?# ?# A" A" A"@! @! @! @! @! @! @! @! @! @!@!@! @! >" >" >" >" ;! ;! ;! : ;! ;! =! =! =! =! < < < < : : 9 9 9 9 9 9 9 9 9 9 < < ; ; ; ; 9 7 -9 -9 -9 -; ; 9 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -6 7 -6 6 6 6 6 4 4 3 -4 4 3 -3 -3 -3 -3 -3 -3 -3 -2 2 2 2 2 0 0 0 / -/ -/ -/ -/ -1 -1 -3 -3 -3 -3 -330000......,,,,,,....,,,,++**++++++++----************))))))))))O-O-O-O-O-O-O-O-O-O-O-O-O-O-N/N/P.O-N/N/N/N/N/N/P.P.P.P.P.P.Q/Q/O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0P1Q2Q2Q2Q2Q2Q2Q2Q2Q2Q2Q2Q2Q2P1P1R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0T/T/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/O0O0O0O0Q/Q/Q/P.P.P.P.P.Q/Q/O0O0O0O0O0O0O0N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/N.N.P-P-P-P-O,O,N.N.N.N.N.N.M.N/M.M.M.L-M.M.L-L-L-L-L-L-L-L-L-M.M.M.M.M.M.M.M.M.M-M-M-L,L,L,L,L,L-L-L-L-J+J+I+I+I*I*J*J*J*J*J*J*J+J+J*J*J*J*J*J*I*I*I*I*I*I*I*H)H*H*H*G)G(G(G(G(G(G(G(G(H)H)G(G(G'G'G'G'G'G'G'G'G'G'G'G'F'F'F'E&C'C'E&E&E&E&E&E&E&E&E&E&E&E&E&D%D% D% D% D% D& D& D& D& D& D& D% D% D% D% D% B# B# D%D%B# B&@$ @$ @$ @$ @$ @$ @$ @$ @$ ?# ?# ?# ?# A" A" A"A"A"A"A"A"A"A"A"A"A"A"@! @! >" >" >" >" <" <" <" ;! ;! ;! =! =! =! =! =! =! =! =! : : : : : : : : : : 9 : < < < < < ; 9 9 ; ; ; ; ; ; 9 9 9 7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -6 5 -6 6 4 4 4 4 4 3 -3 -2 3 -3 -3 -3 -3 -2 2 2 2 2 0 0 0 0 0 0 / -/ -1 -1 -3 -3 -3 -3 -331111.......,,,,+.0 0 0 . . . ,+++*+,,+++++----+++************)))))))P.P.P.P.P.P.P.P.P.P.P.P.P.P.N/N/P.Q/N/N/N/N/N/N/P.P.P.P.P.P.Q/Q/O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1U3S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1U0T/U0U0U0U0T/S.Q/R0R0R0Q/Q/R0R0R0R0Q/Q/Q/Q/R0R0R0R0R0R0R0Q/O0O0O0O0O0O0O0O0O0N/N/N/N/N/N/N/N/N/N/N/N/O0O0O0Q/Q/Q/P.P.Q/Q/Q/O0N/N/N/N/N/N/N/M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.M.N/N/M.M.M.M.M.M.M.M.M-M-M-M-M-M-L-L-L-L-L-M.M.M.M-L,N+N+N+N+L)N+N+N+J*J*L,J*L,J*J*L,I*I*J*J*I*I*I+H*I+H*G(G(G(H)H)H)H)H)H)H)H)G(H(H(G'G'G'G'G'G'G'G'F'F'F'F'F'F'E' F( F( F( G'G'F&F&E&E&F&F&F&F&E&E&E' E' F&F&D& D& B& B& D% D% D% D% D% D% D% D% B# D%D%D%D%B# B# B# B# B# B# B# B# B# B# B# B# B# B# A" A"A"A"A"A"A"A"A"A"A"A"A"A"@! @! @! >" >" >" >" >" =! =! >" =! =! =! =! =! =! =!=!;!;!: : : : : : : : ; < > > < < < < < < ; ; ; ; < ; ; ; 9 9 ; 9 -9 -9 -9 -9 -9 -9 -7 -7 -7 6 7 6 6 6 6 6 6 6 4 3 -5 -5 -5 -5 -5 -5 -5 -4 4 5 -3 -2 2 2 0 0 2 2 4 3 -3 -3 -3 -3 -11111111///......,000 0 0 0 . . ,. ,,,,,,++,+----+++++++++**********)))P.P.P.P.P.P.P.P.P.P.P.P.P.P.N/N/P.Q/O0O0O0O0O0O0Q/Q/Q/Q/Q/Q/Q/Q/O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1P1R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1U3U3S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1U0U0U0U0U0U0T/T/R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0Q/O0O0O0O0O0O0O0O0O0O0O0O0N/N/N/O0O0O0O0O0O0O0O0O0Q/Q/Q/Q/Q/Q/Q/Q/O0O0N/N/N/N/N/N/N/N/N/N/N/N/M.M.M.M.M.M.M.M.M.M.N/N/M.N/M.N/N/N/N/N/N.M-M-M-N.M-L-M.M.L-M.M.M.M.M-M-N+N+N+N+O,O,N+N+L,L,L,L,L,L,L,L,K,I*L,L,K,I*I+I+I+I+H)H)H)H)H)I*H)H)H)H)H)H)I)H(I)I)I)I)I)H(H(H(G(G(G(G(G(G(G)G)G)F( G'G'G'G'F'F'G'G'G'G'F'F'F( F( F&F&E' D& C'C'E&E&D% D% D% D% D% D% D%D%D%D%D%D%D%D%D%D%D%B# B# B# B# B# B# B# B# B# B#A"A"A"A"A"A"A"A"A"A"A"A"A"A"@! >" >" >" >" >" =! >" >" >" >" >" >" =! =! =!=!;!;!: : : : : : : : < < > > < < < < < < ; ; ; ; < ; ; ; 9 9 ; ; 9 -9 -9 -9 -9 -9 -7 -7 -7 6 7 7 6 6 6 6 6 6 4 3 -6 6 6 6 6 5 -5 -4 4 5 -3 -3 -2 2 0 0 2 2 4 4 3 -4 4 3 -11111111////.....,000 0 0 0 . . . . ,,. ,,,,,,,....+++++++++++***********P.P.P.P.P.P.P.P.P.P.P.P.P.P.P.P.O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0P1R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1U3U3S1S1S1S1S1S1S1S1S1S1S1S1S1U3U3U3U3U3S1S1S1S1S1S1S1S1S1U0U0S1S1S1S1S1S1S1S1S1S1U0U0U0U0U0U0S1R0R0S1S1S1R0R0S1S1Q2Q2P1P1R0R0S1S1S1S1S1R0T/T/R0R0R0R0R0R0R0R0R0Q/Q/Q/P.P.P.Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/P.P.P.P.P.P.P.P.P.P.P.P.P.N/N/M.N/M/L.M/L.L.L.M.M.M.M.M.L-M.M.M.N/M.M.M.M.M.M.M.M.N,N,L-L-M.M.L,L,L,L,L-L-L-L-L-L-L-L-L-J+J*L,J+J+I+I+I*I*J*I)I)I)H)H)H)I*I*I*H*H*H*H*I+I+I+H*I*I*H*H*H)H)H)H)H(H(H)H)D(E)G(F'G( G( F&G'F'F'F'F'G( G( E&E&E&E&E' E' E&E&D% E&E&D% D% D% D%D%D%D%D%D%D%D%D%D%D%D%D%B# B# B# B# B# B# B# B#B#B#A"A"A"A"A"A"A"A"A"A"A"A"A"?#?#>">">">">">">">">" >" >" >" >" >" =! =! =! =! =! =! =! =! ;! ;! =! < > > > > > > > > > > > > > > > > ; ; 9 9 ; ; ; ; 9 9 9 7 -7 -7 -7 7 5 5 5 4 4 4 4 4 4 4 4 4 4 4 6 5544 5 -3 -3 -3 -3 -5 -5 -4 4 4 4 2 2 2 2 222211/////.......00000 .. . 0 0 00,,,,......+++++++ + ++++**********P.P.P.P.P.P.Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/O0O0O0O0O0O0O0O0O0O0O0O0O0O0O0P1R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1U3U3S1S1S1S1S1S1S1S1S1S1S1U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3W2W2U3U3U3U3U3U3U3U3U3U3U0U0U0U0U0U0S1S1S1S1S1S1S1S1S1S1Q2Q2P1P1R0S1S1S1S1S1S1R0T/T/R0R0R0R0R0R0R0R0R0Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/R0R0R0R0R0Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/P.P.P.P.P.P.P.N/N/N/N/M/K-M/L.L.L.M.M.M.M.M.M.M.M.N/N/N/M.M.M.M.M.M.M.O-O-M.M.M.M.L,L,L,J*L-L-J+J+L-L-L-L-L-J+J*L,J+J+I+I+I*I*J*J*J*I)H)H)H)I*I*I*H*H*H*H*I+I+I+G)I*I*G)G)G(G(G(G(I)I)I*I*F*F*F'F'G( G( F&G'F'F'G(G(H)H)G(F'G(G(F( F( F'F'E&E&E&E&D% E&D%D%D%D%D%D%D%D%D%D%D%D%D%B# B# B# D%B# B# B# B#B#B#B#B#B#B#B#A"A"A"A"A"A"A"A"?#?#?#>">">">">">">">" >" >" >" >" >" >" =! =! =! =! =! =! =! ;! ;! =! =! ? ? ? ? ? > > > > > > > > > > > < < : 9 ; ; ; ; 9 9 9 7 -7 -7 -7 7 5 5 5 5 5 -5 -4 4 4 4 4 4 4 4 6 6 6 55 -5 -3 -3 -3 -3 -5 -5 -5 -5 -4 4 2 2 2 2 222221///////.....00000 0 . . 0 0 00,,,,......,++++++ + ++++**********Q/Q/Q/Q/Q/Q/R0R0R0R0R0R0R0R0R0R0Q/R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3V4V4U3U3U3U3U3U3U3U3U3U3U3U3U3V4V4U3U3U3U3U3U3U3U3U3U3U3U3W2W2U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3S1U3U3S1S1S1U3U3S1S1S1S1S1S1S1S1S1U3S1S1S1S1U0U0U0U0S1S1S1R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/P.P.N/N/N/N/M/L.M/M/M/M/P.P.P.P.O-O-O-P.P.P.N/N/N/M.M.L-L-M.M.M.M.M.M.M.M.M.L-L-L-L-L-L-M.L-N,O-N,N,L-L-L-J+J+J+J+J+J*J*J*J*I*I*H)I*I*K,I*I*I*K,K,K,F+F+M(M(H* G) H)H)H*G)H)H)I* I* K) J( -L'K& I' I' G( G( G)G)F( F( H)H)F( F( F+ F+ G( G( F'E&E&E&E&E&E&E&D% E&E&D%E&E&D%D%D%D%D%D%D%D%D%D%D%D%D%B# B# B# B# B# B# B# B# B# B# B# B#B#B#B#A"A"?#?#?#?#?#?#?#?#?#>" >" >" >" >" >" >" >" >" >" >" =! -=! -=! -=! -=! -=! -=! -=! -? ? ? ? ? > > > > > > > > > > > < < : 9 9 9 9 9 9 9 9 9 7 5 -5 -5 -5 -5 -5 -5 -5 -5 -5 -4 4 4 4 4 4 4 4 4 4 4 4 4 6 6 3333333 -2 2 2 22222222111111110000000000000000.,,,,,,,,,,,,+++++++++++++******Q/Q/Q/Q/Q/Q/R0R0R0R0R0R0R0R0R0R0Q/R0R0R0R0R0R0R0R0R0R0R0R0R0R0S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1U3U3S1S1S1S1S1S1U3U3U3U3U3U3U3U3S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3V4V4U3U3U3U3U3U3U3U3U3U3U3U3U3U3V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4U3U3V4V4V4V4V4U3U3U3U3U3U3U3U3U3W2W2U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3S1S1S1U3U3U3U3U3U3U3U3S1U0U0U0U0S1S1S1S1S1U3U3R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0S1R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0Q/Q/R0Q/R0Q/Q/Q/Q/Q/Q/Q/O0O0N/O0N0N0M/M/M/M/P.P.P.P.P.O-P.P.Q/Q/O0N/N/N/M.N/N/N/N/N/N/M.M.M.M.M.L-L-M.M.M.M.M.M.O-O-O-N,L-L-L-L-L-L-L-J+J*J*J*J*I*I*I*I*I*I*I*I*I*I*I*K,I.G,N)N)I+H* H)H)H*H*H)H)I* I* J( -J( -L'K& I' J( -H)I*H*G)E'E'H)H)G)G)G,F+ H) -G( E&F'F'E&F'F'F'F'E&E&E&E&E&E&E&E&E&E&E&E&E&D%D%D%D%D%D%B# B# B# B# B# B# B# B# B# B# B# B#B#B#B#B#B#@$@$?#?#?#?#?#?#?#?#?#?#>" >" >" >" >" >" >" >" =! -=! -=! -=! -=! -=! -=! -=! -? ? ? ? ? ? ? ? ? ? ? ? ? > > > < < : 9 9 9 9 9 9 9 9 9 7 5 -5 -5 -5 -5 -5 -5 -5 -5 -4 4 4 4 4 5 -5 -5 -5 -5 -4 4 4 4 6 6 4 4 33333 -3 -3 -2 22222222111111111001111000000000...,,,,,,,,,,+++,,+++++++++*****Q/Q/Q/Q/Q/Q/R0R0R0R0R0R0R0R0R0R0R0R0S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1U3U3U3S1S1S1S1S1S1U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4X3W2W2W2W2X3X3X3X3X3X3X3V4V4V4V4V4V4V4V4V4V4U3U3U3U3U3U3U3V4W2W2W2W2V4U3U3U3S1S1U3U3U3U3U3U3U3U3U3S1U0U0W2W2U0U0S1S1S1U3S1S1S1S1S1S1R0R0S1S1S1S1S1S1S1S1S1S1S1S1S1S1R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0R0Q/Q/Q/R0Q/Q/O0O0O0O0M/N0N0M/N0N0Q/P.P.P.P.P.P.P.Q/Q/O0N/N/N/N/N/N/N/N/N/N/N/N/N/M.N/M.M.M.M.M.M.M.L-N,O-N,N,N,O-L-L-N,N,L-L-L,L,J*L,K,I*I*I*L,L,J+J+J*J*K,K,K,K,P+N)J+I* I+I+D*D*F+F+F+ F+ @)A*8*%7)$+,3+,3%-7%-7',7&+6,+0,+00+(1,);, ;, @+@+D+ C* D) D) F( F( F( F( F'F'E&F'E&E&E&E&E&E&E&E&E&E&E&D%D%D%E&D%D%D%D%D%D%D%D%D%D%D%B# B# B# B# B# B# B#B#@$@$@$?#?# ?# ?# ?# ?# ?# ?#?#>" >" >" >" >" >" >" >" >" >" >" >" >" >" >" >" @! @! ? ? ? ? ? ? ? ? ? ? ? ? ? ? < < : : : : : : 9 9 9 9 7 7 7 7 5 -5 -5 -5 -5 -5 -5 -4 4 5 -5 -5 -5 -5 -5 -4 34 4 4 6 6 6 6 4 334 3333333332222222211111111111111.0000........,,,,,,,,,,,,++++++++****R0R0R0R0R0R0S1S1S1S1S1S1S1S1S1S1R0R0S1S1S1S1S1S1S1S1S1S1S1S1S1U3U3U3U3U3U3U3S1S1S1S1U3U3U3U3U3U3U3U3U3U3U3U3U3U3V4V4V4V4V4V4V4V4U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4W5W5V4V4V4V4V4V4V4V4V4V4V4V4V4W5W5W5V4X3W2W2W2W2X3X3X3X3X3X3X3V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4X3X3X3X3V4V4V4V4V4V4V4V4V4V4V4V4V4U3U3U3W2W2W2W2W2W2U3S1U3U3U3U3U3V4V4S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1R0R0S1S1S1S1S1R0R0R0R0R0R0R0R0R0R0R0R0R0R0Q/Q/O0O0O0P1N0N0N0N0N0N0Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/O0O0N/O0O0N/N/N/N/N/N/N/N/N/M.N/M.M.M.M.M.M.M.L-N,O-O-O-O-O-L-L-N,N,L-L-L,L,L,L,K,K,K,K,L,L,L-L-J*L,K,K,K,K,P+P+L-J+I+I+D*D*F+E*E* C( -?(<%0"3% )*1*+2#+5!)3#(3"'2'&+&%**%"+&8)=(>)C* C* E*E*F( F( F( F( F'F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&E&E&E&D%D%D%D%D%D%D%D%D%D%D%D%D%D%D%B#B#@$@$@$?#@$ @$ ?# ?# ?# ?# ?#?#?# ?# ?# ?# ?# ?# ?# >" >" >" >" >" >" >" >" >" @! @! ? ? ? ? ? ? ? ? ? ? ? ? ? ? =!=!;!: : : : : : : : : 7 7 7 7 7 7 7 7 7 5 -5 -4 4 5 -5 -5 -5 -5 -5 -5 -5 -4 4 4 7 6 6 6 4 4 4 4 4 33333333333222222221111111111111000............,,,,,,,,,+++++++****R0R0R0R0R0R0S1S1S1S1S1S1S1S1S1S1R0R0S1S1S1S1S1S1S1S1S1S1S1U3U3U3U3U3U3U3U3U3S1S1S1S1U3U3U3U3U3V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3Y4Y4Y4Y4Y4Y4Y4Y4Y4V4V4V4V4V4V4V4V4V4V4V4W5W5W5W5W5W5W5W5W5W5W5V4V4V4V4V4V4V4V4V4V4Y4Y4Y4Y4Y4X3X3X3X3X3X3X3X3X3Y4Y4X3X3X3X3X3X3X3X3X3X3V4V4V4V4V4V4V4V4X3X3X3X3X3X3X3X3X3X3X3X3V4V4X3X3V4V4V4V4X3X3X3W2W2W2W2W2W2X3V4V4V4V4V4V4U3U3S1S1U3U3U3U3U3U3U3U3S1U3U3S1S1S1U3S1S1S1S1S1S1S1S1S1S1S1S1S1S1R0R0R0R0R0Q/Q/O0O0R0R0P1P1P1P1O0O0Q/Q/Q/Q/Q/Q/Q/Q/R0R0P1O0O0O0O0O0O0O0O0O0N/N/N/O0N/N/M.N/N/N/N/N/M.L-M.M.M.M.M.M.M.M.M.M.L,L,J+L-L-J+L-L-K-K-L-L-L-L-L,M-J.J.H/G.N,N,L* N,L-J+C,B+F+ F+ A*@)%$'%E6Vr@u@u -9q3k+\%V > < 4 !6&+",1?*@+E)E)G(F'G)G)F( G)F( F( F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&D%D%D%D%D%D%D%D%D%D%D%D%D%D%D%D%D%D%D%B#B#B# B# B# B# @$ @$ @$@$?# ?# ?# ?# ?# -?# -?# ->" >" >" >" >" >" >" >" >" @! @! @! @! ? ? ? ? ? ? ? ? ? ? ? ? =!=!;!: : : : : : : : : 7 7 7 7 7 7 7 7 7 7 7 7 5 -5 -5 -5 -5 -5 -5 -5 -3 -3 -5 -5 -5 -5 -7 -7 -6 6 6 6 6 5 -5 -5 -3333333322222222211111111111111110...........,,,,,,,,,,,,+++++++R0R0R0R0R0R0S1S1S1S1S1S1S1S1U3S1S1S1U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3S1S1S1V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4X3Y4Y4Y4Y4Y4Y4Y4Y4V4V4V4V4V4V4V4W5Y4X3X3X3X3X3X3X3X3X3Y4Y4W5W5Y4X3V4V4V4V4X3X3X3X3W2W2W2W2X3X3V4V4V4V4V4V4U3U3S1S1U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1S1Q2Q2R0R0P1P1P1P1P1P1R0Q/R0R0R0R0R0R0R0R0P1P1P1P1O0O0O0O0O0O0O0O0O0O0O0N/M.N/N/N/N/N/M.M.N/N/N/N/N/N/M.M.M.M.M-L,L-L-L-M.M.M.L.L.L-L-M.M.M-L,J.J.H/H/N,N,N,N,J+J+C,B+E* I.B+?("B.N;oAuCxAv>v7o /`'X31)*#(-=(?*E)E)F'G(H*G)G)G)F( F( G(G(G(F'F'F'F'F'F'F'F'F'F'F'F'E&E&E&E&E&E&E&D%D%D%D%D%D%D%D%D%D%D%D%D%D%D%D%B# B# B# B# @$ @$ @$@$?# ?# ?# ?# ?# -?# -?# -?# ->" >" >" >" >" >" >" >" @! @! @! @! @! @! @! @! @! @! @! @! @! @! @! @! =!=!;!: : : : : : : : : 8! 7 7 7 7 7 7 7 7 7 7 7 5 -5 -5 -5 -5 -5 -5 -5 -3 -3 -5 -5 -5 -5 -7 -7 -6 6 6 6 6 6 6 6 4 4 4 3333322222222211222111111111111...........,,,,,,,,,,,,,++++++S1S1S1S1S1S1U3U3U3U3U3U3U3U3U3U3S1U3U3U3U3U3U3U3U3U3U3U3U3U3U3V4V4V4V4V4V4V4V4V4V5V5V5V5V5V5V4V4Y4Y4Y4Y4Y4X3X3X3X3X3X3Y4Y4Y4Y4Y4X3X3X3X3X3X3X3X3V4V4V4V4V4V4V4V4X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Z5Z5Z5Z5Z5Y4Y4W5W5W5W5W5W5W5W5W5W5W5W5W5W5Y4Z5Z5Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4X3X3X3Y4Y4Y4Y4Y4Y4Y4Y4[4[4Y4Y4W5V4V4V4X3X3X3X3X3X3X3W2X3X3V4V4V4V4V4V4V4V4S1U3V4V4V4U3U3U3V4U3U3U3U3U3U3U3U3U3U3U3U3U3U3U3U0W2W2W2W2U0U0U0U0U0U0U0U0U0U0U0S1S1S1S1S1S1S1S1R0R0R0R0P1Q2R0S1P1P1P1P1P1O0P1O0O0O0O0O0O0O0O0O0O0O0N/N/O0O0N/O0O0N/N/N/N/N/N/N/N/N/N/N/L-L-K-L.M.M.M.M.M.M.M-M-K-L.J/J/J/K0K0I.I-I-K-K-K-H*H)H)F*I-@-<)  -B G-V.W2a2a2b+[%YR VT'_+c)M -/S<*!@.%H* F( H) I* K(J'I*I*H)H)G(G(G(G(F'F'F'F'F'F'F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&D%D%D%D%D%D%D%D%D%B# B# B# @$@$@$ @$ @$ @$ @$ ?# -?# -?# -?# -?# -?# -?# -?# -?# -?# -?# -A" A" A" A" @! @! @! @! @! @! @! @! @! @! @! @! >" >" <" ;! ;! ;! ;! ;! ;! ;! ;! : : : : : 9 9 9 9 9 9 9 -9 -9 -779 -7 5 -5 -7 5 -4 5 -5 -5 -5 -5 -5 -7 -9 7 -6 7 6 6 6 4 4 6 6 6 6 6 5555533322222222222211111////........,.,,. . . ,,,,,,,,+++++S1S1S1S1S1S1U3U3U3U3U3U3U3U3U3U3S1U3U3U3U3U3V4U3U3U3V4V4V4V4V4V4V4V4V4V4V4V4V4V4V5V5V5V5V5V5V4W5Y4Y4Y4Y4Y4X3X3X3X3X3X3Y4Y4Y4Y4Y4X3X3X3X3X3X3X3X3V4V4V4V4V4V4V4V4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Z5Z5Z5Z5Z5Z5Z5Z5Z5Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Z5Z5Z5Z5Z5Z5Z5X6X6X6W5W5W5W5W5W5W5W5W5W5W5Y4Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Y4Z5Z5Z5Z5Z5Z5Z5Z5Y4Y4X3Y4Y4Y4Y4Y4Y4Y4Y4Y4X3X3Y4Z5Z5Y4Y4Y4Y4Y4Y4[4[4Y4Y4W5V4V4W5Y4Y4Y4Y4X3X3W2W2X3X3W5W5W5W5V4V4V4V4V4V4V4V4V4V4V4V4V4V4U3V4V4V4V4V4V4U3U3U3U3U3U3U3U0W2W2W2W2U0U0U0U0W2U0W2U0U0U0U0S1U3S1S1S1S1S1S1S1S1S1S1Q2Q2S1S1P1P1Q2Q2P1O0P1P1P1P1O0O0O0P1P1P1P1O0O0O0O0O0O0O0O0O0O0N/N/N/N/N/N/N/N/N/N/M.L.L.M.M.M.M.L-M.M-M-L.L.J/J/J/J/J/J/I-J.K-G) I+K-L-L-K/J.>+;(  @E-V0Y9h?nn"XŒS‡G}@vC{Bz:^8\@.%@.%G) G) H) I* K(K(I*I*I*I*G(G(G(G(G(G(G(F'F'F'F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&E&D%D%D%D%D%D%D%D%B# D%D%B&@$@$ @$ @$ @$ @$ @$ @$ ?# -?# -?# -?# -?# -?# -?# -?# -?# -A" A" A" A" @! @! @! @! @! @! @! @! @! @! @! @! >" >" <" ;! ;! ;! ;! ;! ;! ;! ;! ;! : : : : : : : : : : : 9 -9 -9 -9 -9 -7 7 7 7 7 7 5 -5 -7 5 -5 -5 -9 : 7 -7 -7 7 6 6 4 4 6 6 6 6 6 6 555533332222222222221111///////////...... . . . . ,,,,,,+++++S1S1S1S1S1S1U3U3U3U3U3U3U3V4V4V4V4V4V4V4V4V4V4U3U3U3V4V4V4V4V4V4V4V4V4V4V4V4W5V4V5V5W6W6W6W6W6W6Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Z5Z5Y4Y4Y4Y4Y4Y4Y4Y4W5W5W5W5W5W5W5W5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Y4Y4Z5Z5Z5Z5Z5Z5Z5Z5Z5Y4Z5Z5Z5Z5Z5Z5Z5Z5Z5Z6Z6Z6Z6Z5Z5X6X6Z5Z5Z5Z5Z5Z5Z5Y4Y4Z5Z5Y4Y4Y4Z6Z6Z6Z6Z6Z6Z5Z5Y4Z5Z5Z5Z5Z5Z5Z5Z5Y4Y4Y4W5W5Y4Y4Y4Y4Y4Y4Y4X3Y4Y4Y4Y4Y4Y4Y4X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3W2W2X3X3W2W2W2W2X3X3W2U0U0U0U0W2W2W2W2W2W2X3U3U3S4S4Q2S4S4Q2W0W0W0W0U3S1S4S4P2P2P2P2S1R0R0R0R0R0P1P1P1P1P1P1P1P1O/O/O/O/O/N.N/O0O0O0O0O0O0N/N/N/N/N/N/M.M.M.M.M.M.M.M.M.M.M.N/N/M.N/M/L.C0B/610610(0<%-91L0K.M*I#.E!,C!'/!)  7</[6b?tG|HLƒ1u®$h¡ PŽ -M‹LŽD†?m )?*F'G(M'M'I)J*I)I)H)G(G(G(G(G(G(G(G(G(G(F'F'F'H(G'G'G'G'G'G'G'F&F&F&F&F&F&F&F&F&F&E&E&E&E&E&E&E&D% D% D% D% D% D%D%@$ @$ @$ @$ @$ @$ @$ @$ ?# -?# -?# -?# -?# -?# -?# -?# -A" A" A" A" A" @! @! @! @! @! @! @! @! @! @! @! >" >" >" =! -=! -=! -=! =! =! =! ;! ;! ;! : < < < < : : : : 8! 8! : 9 -9 -9 -9 9 7 7 7 7 7 7 9 -9 -9 -9 -79 -777 -7 -7 -7 -44666666555555555445332222222 2 2 2 0///.....//...,,,,. . . . ,,,,,,,,,,U3U3U3U3U3U3V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4V4W5V4V4V4W5W5W5W5W5W5W5W5W5W5W5W5W5W5W6W6W6W6W6W6W6X7Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5X6X6X6X6X6X6X6X6Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6Z5Z5Z5Z5Z5Z5Z5Z5[6[6[6[6Z5Z5Z5Z5Z5Z5Z6Z6Z6Z6Z5Z5X6W5Z5Z5Z5[6[6Z5Z5Z5Z5Z5Z5Z5Z5[6Z6Z6Z6Z6Z6Z6[6[6[6[6[6Z5Z5Z5[6[6Z5Z5Z5Z5X6X6Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4X3X3Y4Y4Y4X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3X3W2W2W2W2W2X3U3U3S4S4S4Q2S4S4Y2Y2Y2Y2U3U3S4S4P2P2P2P2S1S1R0S1R0R0P1P1P1P1P1P1P1P1P0P0P0P0P0P0O0O0O0O0O0O0O0O0O0O0O0N/N/N/N/N/N/N/N/L-M.M.M.M.N/N/N/N/L.L.@-=*.)(,'&$,8(0<"6Q$8S 8W 8W#.E)@"* 5"A.Z3_=rCx@wE|@yD}EƒEƒHŠFˆ?m8f-*2-*2>)>)F'G(N(N(J*J*I)J*H)H)H)H)H)H)H)G(G(G(G(G(G(G(H(H(H(G'G'G'G'G'G'G'G'G'G'F&F&F&F&F&E&E&E&E&E&E&E&E&E&E&D% D% D%D%B&B&B& @$ @$ @$ @$ @$ @$ ?# -?# -?# -?# -?# -?# -?# -B# B# A" A" A" A" A" A" A" A" A" A" A" A" A" A" >" >" >" >" >" =! -=! =! =! =! ;! ;! ;! ;! =! < < < : : : : 8! 8! : : : : : : 7 7 8! 7 7 7 9 -9 -9 -79 -7777 -7 -7 -7 -55766666655555555555333332222 2 2 2 00//..........,,. . . ,. . . . . . ,,,,,,W2W2W2W2W2W2X3X3X3X3X3X3X3X3X3X3V4V4V4V4V4V4W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W5W6W6X7W6W6X7X7X7Z6Z6Z6Z6Z6Z6Z6Z6Z6Z6Z6Z6Z6Z6Z6Z6Z5Z5Z5Z5Z5Z5Z5Z5X6X6X6X6X6X6X6X6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6\7\7[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[7[7[7[7[7[7[7[7[7[7[7[7[7[7]6]6]7]7[6[6[6[6[6[6[6[6[7[7[7[7Z5[6Y7X6[6[6[6[6[6[6Z5Z5[6[6[6[6[6[6]7\6\5\5]6]6]6]6]6]6]6\5]6]6]6]6[6Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Y4Y4Z5Z5Z5Z5Z5Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Z5Y4Y4Y4Y4Y4Y4X3X3X3X3X3X3X3X3Y4X3X3X3X3X3X3X3W2W2X3X3U3U3U3U3S4T5S4Q2Q2Q2U2S0S0U2W2W2W2X3W2W2W2W2S1S1S1S1S1S1R0R0S1S1S1S1S1R0R/R/P0P0P0O/O0P1P1P1O0O0O0O0O0O0O0O0O0N/N/N/N/N/N/M.N/N/N/N/L.M/M/K-O0T57*")7 (FDxN‚Mˆ I„ Ax @w:w1n,XI=8@D,U4]:qAxD}D}D~ J„GƒF‚EB~ ->m7f+D'@%/(27*"9,$I*I*I*K,I*I*I+I+I)I)I)I)I)H(H(H(H(H(H(H(H(H(H(H(H(H(H(H(G'G'G'G'G'G'G'G'G'G'G'F&E&E&E&E&E&E&E&E&E&E&E&D%B& B& D% D% D% D% D% B# @$ @$ @% @% @% @% @$ @$ B# B# B# B# B# A" A" A" A" -A" -A" -A" -A" -A" -A" -A" -?# ?# ?# >" >" >" >" >" >" >" <" <" ;! ;! =! =! ;! ;! : : : ;! 8! 8! : : : : : : : : 8! 7 5 -5 -9 -9 -9 -9 -9 9 9 7 -7 -7 -7 -7 -55777777666666665555555555544 4 2 2 222 1111111//....0 0 1 -0 0 0 0 0 0 0 . ,,,,,W2W2W2W2W2W2X3X3X3X3X3X3X3Y4Y4Y4W5W5W5W5W5W5W5W5W5W5W5W5W5W5X6X6W5W5W5W5W5W5X6X6X7X7X7W6X7X7X7Y8[7[7[7[7[7[7[7[7[7[7[7[7[7[7[7[7[6[6[6[6[6[6[6[6Y7Y7Y7Y7Y7Y7Y7Y7[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\8\8[7[7[7[7[7[7[7[7[7[7[7[7]6]6^8^8\7[6[6[6[6[6[6[6[7[7[7[7[6[6Y7Y7[6[6[6[6[6[6[6[6[6[6[6[6[6[6]7]7]6]6]6]6]6]6]6]6]6]6]6]6]6]6[6[6[6[6[6[6Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Y4Y4Z5Y4Y4Z5Z5Z5Z5Z5Z5Z5Y4Y4Y4Y4Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Y4Y4Y4Y4Y4Y4Y4X3Y4Y4X3X3X3Y4Y4X3Y4Y4X3X3S1U3V4W5Q2Q2Q2Q2P1P1S0S0U2V3X3X3W2X3W2W2W2W2U3S1S1S1S1S1S1S1S1S1S1S1U3U3S0R/Q1Q1P0Q1Q2P1P1P1P1P1O0O0O0O0O0O0O0N/O0O0O0O0O0N/N/N/N/N/L.M/M/L.I* O0/"!? )G4h:n>y=x9p" >" >" >" >" >" >" <" <" <" ;! =! =! ;! ;! ;! ;! ;! ;! 9" 9" ;! ;! ;! : : : : : 8! 8! 8! 8! : : 9 -9 -9 9 9 7 -7 -7 -7 -7 -55777777666666666 6 56 6 6 5555555 -5 -3 -3 -222 2 111111////..1 -1 -1 -0 1 -1 -0 0 0 0 . . . . . . X3X3X3X3X3X3Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Z5Z5Z5Z5Z6Z6Z6Z6Z5Z5Z6Z6Z5Z5Z5Y4Z5Z5Z5Z5[6[6Z5[6[7[7[7[7[7[7[7[7[7[7[7[7[7[7[7[7\8\8[6[6[6[6[6[6[6[6Y7Y7Y7Y7Y7Y7[6[6^7^7^7^7^7^7^7^7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7^8^8^8^8^8^8^8^8]6]6]6]6]6]6]6]6^8^8\8\8\8\8\8\8\8\8\8\8\8\8\8\8[6\7\7\7\7\7\7\7[6[6[6[6\7\7\7\7]7]7]6]6^7^7^7^7^7^7]6]6]6]6]6]6[6[6[6[6[6[6Z5[6[6[6[6[6[6[6Z5[6[6[6Z5Z5[6Z5Z5Z5Z5Z5Z5[6Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Y4Y4Y4Z5Y4Y4Z5Z5\5[4Y4Z5Y4Y4V4V4R6!R6!08N-5K3S3S 5W!6X%6Y#4W*1I+2J40=73@F87F87M6$J3!S3S3S1U3W2U0U0U0U3U3U0U0U0W2S1U3S1S1P1P1P1P1P1P1P1O0O1O1P1P1R0Q/O0P1O0O0O0N/O0O0O0O0N/O0O0N/N/N/L.M/M/L.F/I2#! ;#A%S'U)d,g0k3n'e[Q$X Y*c5w6x7h6g:l CuCzG~KKKƒIH‚H‚@u5j +R%L#J$K&-$-4G-G-K-K-K-K-L-J+I)I)I)I)I)I)I)I)I)I)I)I)I)I)I)H(H(H(H(H(H(H(H(H(H(G'G'G'G'G'G'G'F'F'F'F'F'F'F'F'F'F'E&E&D% D% D& D& D& D& D& D& D& B$ @% @% @% @% @$ @$ D%D%D%D%B# B# B# B# B# B# B# B# B# B# B# B# ?# -?# -?# ?# ?# >" >" >" >" >" >" >" <" <" <" <" 9" :# ;! ;! =! =! ;!;!;!;!;!: : : : : 8!8!8!8!: : : : 9 9 9 9 9 9 9 9 79 -; -97777766666666 6 6 6 6 6 6 6 6 6 555 -5 -3 -3 -33224433111111111111111000. . . . . . Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Y4Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z6Z6Z6Z6Z5Z5Z6Z6Z5Z5Z5[6Z5[6[6[6[6[6[6[6[7[7[7[7[7[7[7[7[7[7[7[7[7[7[7\8\8\8\7\7\7\7\7\7\7\7Z8Z8Z8Z8Z8Z8\7\7^7^7^7^7^7^7^7^7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7^9^9^9^9^9^9^9^9^9^8^8^8^8^8^8^8^8^7^7^7^7^7^7^7^7`:`:^:\8\8\8\8\8\8\8\8\8\8\8\8\8\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7^8^8]6^7^7^7^7^7^7^7^7]6^7^7^7^7\7\7\7[6[6[6\7[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6Z5[6[6Z5Z5Z5Z5Z5[6Z5[6[6[6Z5Z5[6Z5Z5Z5Z5Z5Y4Y4Y4Z5Z5Z5Y4Z5\5[4Y4Z5Y4Z5Y7X6I-F*&.D%-C"7W'<\&;]&;]&7Z#4W$+C '?'#0)%27)(9+*H1I2 S3S3S1U3X3X3X3X3V4V4X3U0X3W2U3V4U3S1R3R3P1P1O0O0O0R3R4O1P1P1S1R0P1P1P1O0O0O0O0P1O0O0O0O0O0P1O0O0M/M/N0O1I2I2! 3=&T2`:uB} -Hƒ I„ Jˆ/lª$XŒJ~R‹P‰JŒC…;l" >" @! @! >" >" >" =! =! =! =! =! : : ;!: : : : : : : : : : : : 9 -; -; -; ; 9 9 9 9 9 7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -6 6 6 6 6 6 6 6 6 6 6 5553 -3 -3 -2 3 -3 -2 2 2 2 2 2 2 2 2 2 2 2 1111111111/ -/ -/ -/ -Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5[6[6[6[6[6[6[6[6[7[7[6[6[6[6[6[6[7[7[7[7[7\8\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\8\8\8\8\8\8\8\8\;\;\;\;\;Z9Z9Z9\7\7\7^9^9^9^9^9\7\7\7\7\7^9^9^9`:`:`:`:`:`:`:`:`:`:`:`:`:`:`9`9`9`9`9`9`9`9`9`9`9`9`9`9`9`9`9`9^:^:^:^:^:^:^:^:^:^:^:^:^:^:^9^9\;\;^9^9^9^9^9^9^:^:^:^:^:^:^9^9`:`:`:`:`:`:`:`:^8^8^8^8^8^8^7^7^7^7^7^7^7^7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7[6[6[6[6[6[6[6[6\7\7[6[6[6[6[6[6[6[6[6[6[6[6[6[6Z5Z5Z5Z5Z5[6[6Y4Y4Y4Z9V5U9"=! -" -*#I'M1f7l7p:s;t1j"YLEB>"L&" >" @! @! >" >" >" >" =! =! =! =! ;! ;! ;!;!;!: : : : : : : : : : : ; -; -; ; 9 9 9 9 9 9 7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -6 6 6 6 6 6 6 6 6 6 6 4 4 4 3 -3 -3 -3 -3 -3 -2 2 2 2 2 2 2 2 2 2 2 11111111/ -/ -/ -/ -Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6\7\7[7[7[7[7[7[7[7[7[7[7\8\8\8\8\7\7^7^7^7^7^7^7^7^7\7\7\7\7\7^9^9^9\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7^:^:^:^:^:^:^:^:\;\;\;\;\;\;\;\;^9^9^9^9^9^9^9^9\7\7\7^9^9^9^9^9`:`:`:`:`:`:`:`:`:`:`:`:`:`:`9`9`9`9`9`9`9`9`9`9`9`9`9`9`9`9`9`9_;_;_;^:^:^:^:^:^:^:^:^:^:^:^:^:^9^9^9^9^9^9^9^9^:^:^:^:^:^:^9^9`:`:`:`:`:`:`:`:`:`:`:^8`:`:`9`9`9`9`9`9`9`9`9^7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7[6[6[6Z5[6[6[6Z5[6[6Z5Y4Z5\:\:V?->'&(!G*P1c1c5n8q:s9rEƒA/m;yD|GFr -" >" >" >" @! @! >" >" >" >" >" >" =! -=! -=! -=! -=!=!=!=!=!=!: : : : : : : : < < ; ; 9 9 9 9 9 9 9 7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -6 6 6 6 6 6 6 6 6 6 6 4 4 4 3 -3333333322222 2 2 2 211111111111Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5[6[6[6[6[6[6[6[6[6[6[6[6[6[6\7\7\7\7\7\7\7\7[7[7[7[7[7\8\8\8\8\8\8\8\8\8\7^9^7^7^7^7^7^7^7^7\7\7\7\7^9^9^9^9^9^9^9^9^9^9^9^9^9^9^9^9^9^9^9^9^:^:^:^:^:^:^:^:\;\;\;\;\;\;\;\;^9^9^9^9^9^9^9_:^9^9^9^9^9^9_:_:a;a;a;a;a;a;a;a;a;a;a;a;a;a;a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:_;_;_;_;_;_;_;_;_;_;_;_;_;_;_;_;^9^9^9^9^9^9^9^9^:^:^:^:^:^:^9^9`:`:`:`:`:`:`:`:`:`:`:^8`:`:`9`9`9`9`9`9`9`9`9^7^9^9^9^9^9^9^9^9\7^9^9^9^9^9^9^9\7\7\7\7\7\7\7^9\7\7\7\7\7\7\7\7\7\7\7\7\7\7[6\7[6[6[6[6[6[6[6[6[6\7\:X6Q:(9" (D&L6hBtG€L…M†VišØ^ÍZ˜X–^– UIu@l<2,>4.I2H1I2L5P5S8[<Z;E3(' - ' -4$^-g/h/h1p/n-h3n&N€ -2d2e8kA|F Ac;]R4V8W5S1X3X3W2W2Y2Y2W2W2W2Q,C(@%" ,$. / 8*B4&  < F -)Y/_6q" >" >" >" >" >" >" >">"=!=!=!=!;!: : : : : : : < < < < : 9 9 9 9 9 9 7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -6 6 6 6 4 4 4 4 4 4 33333333333 -2 2 2 222222111111[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6[6\7\7\7\7\7\7\7\7\8\8\8\8\8\8\8\8\8\8^:^:\8^:^:^:`:`:`:`:`:`:`:`:^:^:^:^:^:^:`:`:^9^9^9^9^9^9^9^9^9^9^9^9^9^9`9`9`:`:^:^:^:^:^:^:]<]<]<]<]<\;]<]<_;_;_:_:_:_:_:_:^9^9^9_:_:_:_:_:a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:_;_;_;_;_;_;_;_;_;_;_;_;_;_;_;_;_:_:_:_:_:_:_:_:_;_;_;_;_;_;_;_;a;a;a;a;a;a;a;a;`:`:`:`:`:`:`:`:a:a:`9`9`9`9`9a:^9^9^9^9^9^9^9^9^9^9^9^9^9^9^9^9\7\7\7\7\7\7\7\7^9^9\7^9\7^9\7\7^:\8\8\8\8\8\8\8[7[7]7]7]6^7[<V7L8-D0%-#    ">%H1a6fC{ -KƒK†N‰OŒMŠRPŽPŒM‰Bz?w!A<:< 6 "9 $   -  &!O,Z7o >v -C„MŽGˆ1h©ÒÿF{¶ X  Y¡XšS• An:g0:?0:?2<5.811603825:47<66;55:4466355$5?*4 >8CCCEABCDJ!M"P &T /b2e5r -=z C‚H‡O‹LˆP‰M†O†P‡GzFyAu;o4i5j8o:q.3>N/N/L.L.M.M.M.M.M.L-L-L-L-L-L-L-L-L-L-L-N,N,L-L-L-L-K-I+L*L*J+J+J+J+J*J*I+H*H*H*H*H*H)H)H*H*H*G)G)G)G)G)G)G)G(G(E)E)C)C)F'F'F'F'G'G'G'G'F'E&D% D% E&E&E&D% D% D% E% E% E% E% D%D%D%D%E%C# C# C# C# C# B# B# B# B# B# A" A" A" A"A"A"A"A" A" ?# ?# ?# ?# >" >" >" >" >" >" >" >" >" =! =!=!=!=!=! -=! -=! -=! -< < : : : : : : : 9 9 9 9 9 9 9 7 -7 -7 -7 -9 9 9 9 7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -6 6 6 6 6 4 4 33333333333 -2 2 2 222222211111[6[6[6[6[6[6[6[6[6[6[6\7\7\7\7\7[6[6[6\7\7\7\7\7\7\7\7\7\7\7\7\7\8\8\8\8\8\8\8\8\8\8^:^:^:^:^:^:`:`:`:`:`:`:`:`:^:^:^:^:_;_;a;a;^9^9^9^9^9^9^9^9^9^9^9^9^9^9`9`9`:`:^:^:^:^:^:^:]<]<]<]<]<]<]<]<_;_;_:_:_:_:_:_:^9^9^9_:_:_:_:_:a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:`<`<`<_;_;_;_;_;_;_;_;_;_;_;_;_;_:_:_:_:_:_:_:_:_;_;_;_;_;_;_;_;a;a;a;a;a;a;a;a;a;a;a;`:a;a;a;a;a:a:a:a:a:a:a:a:_:_:^9^9^9^9_:_:^9^9^9^9^9^9^9^9\7^9^9^9^9^9^9^9^9^9^9^9^9^9^9^9^:^:^:^:^:^:\8\8[7[7]7^8]6]6\= V7F2'+  =%H1a6f@xGK†MˆE‚G„NŒPŽTNŠG@x$D;7830 ")L*X4l8p@IŠ C„Gˆ$Y”MˆQ™Q™MI‹ Bo3` % % #""""""  &"88@DBCBABC%Q --Y/].\7j.3>N/N/L.L.M.M.M.M.M.L-L-L-L-L-L-L-L-L-L-L-N,N,L-L-L-L-K-I+L*L*J+J+J+J+J*J*I+I+I+I+I+I+H)H)H*H*H*H*H*H*G)G)G)G)G(G(E)E)C)C)F'F'F'F'G'G'G'G'F'E&E&F'E&E&E&E&E&E&F&F&F&F&E&E&D%D%E%E%C# E% E% E% D% D% B# B# B# B# B# B# A"A"A"A"A" A" ?# ?# ?# ?# ?# >" ?# ?# >" >" >" >" >" >" >">"=!=!=! -=! -=! -=! -=! =! ;! : : : : : : : : 9 9 9 9 9 9 9 9 9 9 9 9 9 7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -6 6 6 6 6 4 4 4 3333333333 -3 -2 2 222222221111\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7^9^9^9^9^9^9^9^9^9^:^:^:^:^:^:\8\8\8\8^:^:^:_;_;_;`:`:`:`:`:`:`:`:`:`:`:`:a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;c:c:a;a;a;a;a;a;a;b<_;_;_;_;_;_;_;_;_;_;a;b<b<b<b<b<a;a;a;a;a;a;b<b<b<b<b<b<b<b<b<b<b<b<b<b<b<b<b<b<b;b;b;b;b;b;b;b;b;b;b;b;b;b;b;b;a=a=a=`<`<`<`<`<`<`<`<`<`<`<`<`<`;`;`;`;`;`;`;`;`<`<`<`<`<`<`<`<b<b<b<b<b<b<a;a;a;a;a;a;`:a;a;a;a:a:a:a:a:a:a:a:_:_:_:^9_:_:_:_:^9_:_:_:_:_:_:_:^9^9^9^9^9^9^9_:^9^9^9^9^9^9^9^9^9^9^:^:^:^:\8\8]6]6_5`6b9_6];X6".*6, -%@ -#L.b -5i>x D~ Kˆ MŠ RU’ R RQ LˆK‚Cz ,P&J$I%J*K&G:/ (1"I*Q1l5p@€ HˆIŒD‡?‡EU—P’M‘LAr._G?@@CB@??BHJE K!E:03-054,O&Dg$Ky(O}>o -._ 2d 3e1h4k8q6o2f ->r -@op>p.3>N/M.M/M/M.M.M.M.M.M.M.M.M.M.M.M.M.M.L-L-N,N,L-L-L-L-L-L-N,N,L-L-L-J+J+J+J+J+I+I+I+I+I+I+H*H*H*H*H*H*I*H)G)G)G(G(E)E)C)B(G(G(G(G(H(H(H(G'F'F'F'F'E&E&E&E&E&E&F&F&F&F&F&F&E&E&E% E% C# E% E% E% E% E% E% E% B# B# B# B# B#B#B#B#A" A" ?# ?# ?# ?# ?# ?# ?# ?# >" >" >" >" >" >" >">"=!=!=! =! =! =! =! =! ;! ;! ;!;!: : : : : : 9 9 9 9 9 9 9 9 9 9 9 9 9 7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 7 6 6 4 4 4 4 4 4 4 4 4 4 4 33 -3 -2 2 222222222111\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7^9^9^9^9^9^9^9^9^9^9^9^9^9^:^:^:^:^:^:_;_;_;_;_;_;_;_;_;_;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;c:c:b<b<b<b<b<b<b<b<`<`<`<`<`<`<`<`<`<`<b<b<b<b<b<c=b<b<b<b<b<b<b<b<c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c<c<c<c<c<c<c<c<c<c<c<c<c<c<c<c<a=a=a=a=a=a=a=a=a=a=a=a=a=a=a=a=`;`;`;`;`;`;`;`;`<`<`<`<`<`<`<`<b<b<b<b<b<b<b<b<b<a;a;a;b<b<b<b<b;b;b;b;b;b;b;b;_:_:_:^9_:_:_:_:_:_:_:_:_:_:_:_:^9^9^9^9^9^9_:_:_:_:_:_:_:_:_:^9^9^9^:^:^:^:^:^:^7^7`6_5b9^5];X6!-$&=%< 3,+*C "K.b -5i@z D~ -J‡ MŠ S S T‘ T‘SQM„D{ 0T ,P +P -*O -N)J(C *E0  '3 G)P/j5p>~ -G‡D‡KŽN– TœP’S•Q•Q•Gx3d$M!J"J"J GFF GEEM#R.Y0[<*% ( # # % #6,OFt Gu 0a!R$V$V/f3j/h9r"VŠG{2a2a3b3b3a6dqCy@v=u>v?q?q.3>N/N/M/M/N/N/N/N/N/N/N/N/N/M.M.M.M.M.M.M.O-O-M.M.M.M.M.L-N,N,L-L-L-L-L-L-L-L-K-K-K-I+I+I+I+I+I+H*H*H*I*I*H*H*H)H)F*F*D*C)G(G(G(G(H(H(H(H(G(F'F'F'F'F'F'F'E&E&F&F&F&F&F&F&E&E&F&E% E% E% E% E% E% E% E% E% D% D% D%B# B#B#B#B#B# B# @$ @$ @$ @$ ?# ?# ?# ?# ?# ?# ?#?#?#>" >">">">"=! =! =! =! =! =! ;! ;! ;!;!;!: : : : : 9 9 9 9 9 9 9 9 : : 9 9 9 9 9 9 9 9 9 9 9 7 -7 -7 -7 7 7 7 5 -5 -4 4 4 4 4 4 4 4 4 4 4 3 -3 -3 -333333222222^7^7^7^7^7^7^7^7^7^7`9^7^7^7^7^7\8\8\8^:^:^:^:^:^:^:^:^:^:^:^:^:`:`:`:`:`:a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;a;b<b<b<b<b<b<b<b<b<b<b<a;a;a;a;a;a;a;a;a;a;a;a;a;a;c:c:b<b<b<b<b<b<b<c=`<`<`<`<`<`<`<`<`<`<b<c=c=c=c=c=b<b<b<b<b<b<c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=e<e<e<e<e<e<e<e<e<e<e<e<e<e<e<e<e;e;e;d:d:d:d:d:e<e<d;e<e<d;e<e<c=c=c=c=c=c=b<b<b<b<b<b<b<b<b<b<b<b<b;b;b;b;b;b;b;b;b;b;b;b;b;b;c9c9c9c9c9c9c9c9b8c9c9c9c9c9c9c9a:a:a:a:a:a:a:a:a;`:a:a:`9`9`9`9^7^7`6_5a;\6^9Z5"#5 ";&J(L(P&N'S,X0b:l @wCzG„KˆQSR‘QQO‹IBz6^ 1Y.S +P (N -'M$J#I&H(J C<;>#R(W3jrCwB| Z”Hƒ´+f— B}G‚7o+c*U.Y.X/Y6l:p=t=t?u?u@q@q?o?o?r?r">">">" >" >" >" >" >" >" =! =!=!=!=!;!;!;!: : : : : : : : : : : : : : 9 9 9 9 9 9 9 9 9 9 9 7 7 7 7 5 -5 -5 -4 4 4 4 4 4 4 4 4 4 4 3 -3 -333333322222`9`9`9`9`9`9`9`9`9`9`9`9`9`9`9`9^:^:^:^:^:^:^:^:^:^:^:^:_;_;_;_;a;a;a;a;a;a;a;a;a;a;a;a;a;b<b<b<a;a;a;a;a;b<b<b<b<b<b<b<b<b<c=c=b<b<b<b<b<b<b<b<b<b<b<b<b<b<d;d;b<c=c=c=c=c=c=c=a=`<`<`<`<`<a=a=`<`<b<c=c=c=c=c=b<b<b<b<b<b<c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=f=f=f=e<e<e<e<e<e<e<e<e<e<e<e<e<e;e;e;e;e;e;e;e;e<e<e<e<e<e<e<e<c=c=c=c=c=c=c=c=c=b<b<b<b<c=c=c=b<c=c<b;c<c<b;b;b;b;b;b;b;b;b;b;d:d:d:d:d:d:c9c9b8c9c9c9c9c9c9c9b;b;b;b;a:a:a:a:a;`:a:a:`9`9`9`9`9^7`6b8c=]7_:X3 ! / 8&J)M/W2Z3_6b:l=o @w AxFƒMŠQO‹J‰Iˆ E>z :r4l*R&N#H"G (N -'M$J#I%G#E )L.Q*R*R -.] -.]3j;rH… -J‡KˆU’Z˜T’U‘SL„J‚@q:k-X &Q#KGEEDCCFDC33 -2=K&T @tKO‰K… Fw DuC~D :r+cFBB!K3in@s@s=r>s -=f -=f271271M2M2P0P0P.P.P.P.P.P.P.P.N/N/N/N/N/N/M.M.O-O-M.M.O-N,M.M.O-O-M.M.L-L-L-L-L-L-K-K-K-K-K-K-I+I+I+I+I+I+I*J+I+H*H)H)I*I*F*E)I)I)I)I)K(K(K(K(H(H(H(H(G'G'G'G'G'G'G'G'G'G'G'G'F'F'F&F&F&F&F&F&F&F&F&E% D% D% D%D%D%D%D%D%B# B# @$ @$ @$ @$ @$ @$ @$ @$ ?# ?# ?#?#?#?#?#?#>">">" >" >" >" >" >" >" >" >">"=!=!;!;!;!;!: : : : : : : : : : : : : 9 9 9 9 9 9 9 9 9 9 9 7 7 7 7 5 -5 -5 -5 -4 4 4 4 4 4 4 4 4 4 3 -3 -333333332222`9`9`9`9`9`9`9`9`9`9`9a:a:a:a:a:_;_;_;_;_;_;_;_;_;_;_;_;_;_;_;_;a;a;a;a;a;a;b<b<b<b<b<b<b<b<b<b<a;a;a;a;a;a;a;a;a;a;a;a;b<c=c=c=b<b<b<b<b<b<b<b<b<b<b<b<b<b<b<b<c=c=c=c=c=c=c=d>a=a=a=a=a=a=a=a=c=c=c=c=c=c=c=d>c=c=c=c=c=c=c=c=d>d>d>d>d>d>d>d>d>d>d>d>d>d>f=f=d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=e<e<e<e<e<e<e<e<e<e<e<e<e<e<e<e<d>d>d>d>d>c=c=c=b<c=c=c=c=c=c=c=c=c=c=b<c=c=c=a;c=c=b<b<c=c=b<b<d;d;d;d;d;d;d;c:c:c:d;d;c:d;d;d;b<b<b<b<b<b<a;b<b<`:a:a:`9`9`9a:a:a:a:`9`?\;a:Z3$ - '/#G(L2_9f=oBtDzDzFHGƒ -M‰R‘ MŒ -J‰ -J‰F…B?z:u0e-b)_']'_&^'^#Z#X!V"W&[,c3j3l7p8q;tBDBT“$_ŸLŒFˆFˆDA|6k-b%U#S!OKLNOMHKHF E*O2!%:/:O  %B#K/d2g:z:z>y?z6q+f*c(aRO%X0c7k;o>p?q@q@q=r=r>u>u=n>o;^<_,99,99E4!E4!Q2Q2P1P1Q/Q/Q/Q/Q/Q/Q/Q/O0N/N/N/N/N/N/N/P.O-O-O-O-O-M.M.O-O-M.M.L-L-L-L-M.M.M.M.M.L-L-L-K-K-K-K-I+I+I*J+J+J+J+J+I*I*G+G+I)I)I)I)K(K(K(K(I)H(H(H(H(H(H(H(G'H(H)H)H)H)H(H(F'F'G'F&F&G'F&F&F&F&F&F&E&E&E&D% D%D%D%D%D% D% B& B& B& @$ @$ @$ @$ @$ @$ @$ @$@$@$?#?#?#?#?#>" >" >" >" >" >" >" >" >" >" >" =! =! =! =! =! =! < < < < < < < < < < < < < ; -; -; -; -; -; -; -; -; ; 9997 5 -5 -5 -5 -7 -7 -7 -7 -7 -7 -7 -6 6 6 4 3 -333333333222a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:a:_;_;_;_;_;_;_;_;_;_;_;_;`<`<`<`<b<b<b<b<b<b<b<b<b<b<b<b<b<b<c=b<b<b<b<b<b<b<b<c=c=c=c=c=b<c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=d>c=d>d>d>d>b>a=a=a=a=a=a=b>d>d>d>d>d>d>d>d>c=d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>f=f=d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>g>g>g>f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=d>d>d>d>c=d>d>c=d>d>c=c=c=c=c=c=c=c=d>c=b<d>c=c=c=c=c=c=c=c=c=c=e<e<e<e<e<e<e<e<d;d;d;d;d;d;d;d;b<b<b<b<b<b<a;b<b<a;a:b;a:a:b;b;a:b;b;b;_>]<a:Z3" &+!E%I0]6c=oBtE{I -N‡ QŠ Lˆ PŒOŽ L‹PP MŒ MŒL‡MˆM‚LJ€IH€H€K‚JJ I~ H} -F{AxD{C|B{D} E~H…H…DƒV•"]I‰@‚@‚ -=x:u4i+`&V!QMLLLNNHEFFC E81(& ! +0,+  - !; H+`1f7w:zu>u?p?p<_?b/<<,99E4!G6#Q2Q2Q2Q2R0R0R0Q/Q/Q/Q/Q/O0O0O0O0O0O0O0O0Q/P.P.P.O-O-M.M.O-O-M.M.M.M.M.L-M.M.M.M.M.M.M.M.K-K-K-K-K-K-L-J+J+J+J+J+I*I*G+G+I)J*J*I)K(K(K(K(I)I)I)H(H(H(H(H(H(H(H)H)H)H)H(H(G(G(G'G'G'G'G'G'G'G'F&F&E&E&E&E&D%E&D%D%D% D% B& B& B& B& B& B& B& @$ @$ @$ @$@$@$@$@$@$?#?#?# ?# ?# >" >" >" >" >" >" >" >" =! =! =! =! =! =! =! =! < < < < < =! =! =! =! < < < < < < < < < ; -; ; 99997 7 7 5 -7 -7 -7 -7 -7 -7 -7 -7 -7 7 4 4 4 4 4 4 4 4 333333a:a:a:a:a:a:a:a:a:a:a:b;b;b;b;b;`<`<`<`<`<`<b<b<`<`<`<`<`<`<`<`<b<b<b<b<b<b<c=c=c=c=c=c=c=c=c=c=b<b<b<b<b<c=c=c=c=c=c=c=b<c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=d>d>d>d>d>d>d>d>e?b>a=a=a=a=b>c?c?f=f=f=f=f=f=f=g>f=f=f=f=f=f=f=f=g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=f=d>d>d>e?e?d>d>d>d>d>d>c=d>d>d>d>c=d>d>d>d>d>d>d>c=c=c=c=c=c=c=c=e<e<e<e<e<e<e<e<d;d;d;d;d;d;e<e<c=c=c=c=b<b<b<b<b<b<b;b;b;b;b;b;b;b;`; `; _=];_=#U3! "'B%I.[4a;q@vD} I‚ MŠPQ‘ NŽNNMMMNO‘Q“PŽPŽP‘T•T— U˜U˜T—V˜V˜ ZœW™T”W— Z—V“PŽS‘V“W”T•S” R” R”Q” P“N’N’ W˜ -V— V•QN‹ R T‘ RFE€HG€ Aw2h'U$RJICDJK?: " ' 5'V -,[7w<|LKU:{µ>¹ a› -V”R K„ K„DtAqCr @o =o As;r;r9k9k&8R(:TC6.C6.P4P4S4S4W0W0S1S1S1R0R0R0R0R0R0R0P1O0O0O0O0O0O0O0Q/P.P.P.P.P.N/N/M-M-M-M-M.M.M.M.M.M.M.M.M.M.M.M.L-L-L-L-L-L-L-L-L-L-L-J+K-K-G,G,J+J+J*J*L)L)L)L)I)I)I)I)H)H)I*I*H)H)H)H)H) H) H)H)G)G)G( G( G( G( G( G( G'G'G'F&F&F&F&F&F&F&E%E%D% D% B& B& B& B& B& B& B& B& B# B# B# B# B# B# B#B#?#?#?# ?# ?# ?# ?# ?# ?# ?# ?#>" >" >" >" >" >" >" =! =! =! =! =! =! =! =! =! =! =! =! =! =! < < < < < < < < < < ;;999 9 7 7 7 -7 -7 -7 -7 -7 -7 -7 -7 7 6 6 4 4 4 4 4 4 4 33333a:a:a:a:a:a:a:a:a:a:a:b;b;b;b;b;`<`<`<`<`<`<b<b<`<`<`<`<a=a=a=a=c=c=b<b<b<b<c=c=c=c=c=c=c=c=c=d>c=c=c=c=c=c=c=d>d>d>d>d>d>d>d>d>c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=d>e?e?e?d>e?e?e?e?c?c?c?c?c?c?c?c?g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>g>i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@g>g>g>f=f=f=f=f=g>g>g>g>g>g>g>g>e?e?e?e?e?e?e?d>d>d>d>d>d>d>d>e?d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>f=f=f=f=e<f=f=f=e<e<e<e<e<e<e<e<c=c=c=c=c=c=c=c=c=c=c<c<b;b;c<b;b;b;`; `; _=_=_=#U3  $'B$H-Z4a" >" >" >" >" >" >" >" >" =! =! =! =! =! =! =! =! =! =! =! < < < < < < < < < < < < : 99 9 7 7 9 9 9 9 9 9 9 7 -7 7 7 6 4 4 4 4 4 4 4 4 3333b;b;b;b;b;b;b;b;b<b<b<b<b<b<b<b<b<b<b<b<b<b<d;d;b<b<b<c=c=c=c=c=c=c=d>d>d>d>c=c=c=c=c=c=c=c=d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>e?e?e?d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?g>g>g>g>g>g>g>i@g>g>g>g>g>g>g>g>i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@g>g>i@i@i@i@i@i@i@g>g>g>g>g>g>g>e?e?e?e?e?e?e?e?e?e?e?d>e?e?e?e?e?e?e?e?d>e?e?d>d>d>d>d>d>d>d>d>f=f=f=f=f=f=f=f=e<e<e<e<e<e<e<e<d>d>d>c=c=c=c=c=c=c=c<c<c<c<c<d=c<c<c<c<a<a<\>#R4 &)C$H,Z3a=p BuD|E}J†M‰ NŽM NŽMLŒMMMNMMŒL‹MN‘M‘N’NOOOOOPŽONNONOPONŽMMOOOON‘MKŒKŒLKŒK‰LŠM‹NŒL†L†Fx>p'L:28;<< >#D A0%'4#R +Z5r;xBƒBƒB‰EŒIŒGŠH‡J‰D{Cz :g.[0b5g;u=w" >" >" >" >" >" >" >" >" =! =! =! =! >" >" >" >" >" =! =! =! =! =! =! =! =! < < < < < : : : : 7 7 9 9 9 9 9 9 9 9 9 9 7 7 7 -7 -7 -7 -7 -6 6 6 6 6 6 6 b;b;b;b;b;b;b;b;b<b<b<c=c=c=c=c=c=c=c=c=c=c=e<e<c=c=c=c=c=c=d>d>c=c=d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?gAgAgAe?gAgAgAgAgAgAgAgAe?e?gAgAi@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@g>i@i@i@i@i@i@i@i@i@i@i@g>i@i@gAgAgAgAgAgAe?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?f=f=f=f=f=f=f=f=f=f=f=f=e<e<f=f=d>d>d>d>d>d>c=d>d>c=c<d=d=c<c<d=d=c<c<c<b=a<\>#R4 &*C$H-[3a;nAtE}F~J†M‰MMMLŒLŒLŒMMMMMŒL‹MN‘LM‘NOONOOOONNOP‘PPPPN‘N‘OPPON‘MMŽMŽMŽMŽM‹M‹NŒOO‰O‰ K}?q'L:,.35:=$E$E =:& -*4"Q +Z5r -=zD…BƒEŒN•MR•OŽH‡Cz8o,Y+X7iEwIƒ E G EAz ?x7^.U%71/ +8)4C.G2 U3S1S3T4W2U0S1S1S1S1S1R0P1Q2Q2Q2Q2P1P1P1R0R0R0R0S.S.Q/Q/O0O0N/N/N/N/N/N/N/N/P.P.N/N/N/N/N/M.M.M.M.M.M.M.M.M.M.L-L.L.J/I.L-L-L,L,N+N+L)L)J*J*J*I)J+J+J+I*I*I*I*I*I*I*I*I*H)H)H(H(H)H)H)H)G( G( G'G'G'G'G'G'G'G'F&F&F&F&E&E&E&E&E&E&E&E&D%D%D%D%D%D%D%D%@$@$@$ @$ @$ @$ @$ @$ @$ @$ ?#?#?#?#?#?#?#?#>" >" >" >" >" >" >" >" >" >" >" >" >" =! =! =! =! =! =! =! =! < < < < < : : : : 8! 7 9 9 9 9 9 9 9 9 9 9 7 7 7 -7 -7 -7 -7 -6 6 6 6 6 6 6 b<b<b<b<b<b<b<b<c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=d>d>d>d>d>d>d>d>d>d>d>d>d>d>d>e?e?g>g>g>g>g>g>g>g>c?c?c?c?c?eAeAeAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAi@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@jAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAi@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@gAgAi@i@i@i@i@i@i@i@i@i@i@i@i@jAi@i@jAjAi@i@i@i@g>i@i@g>g>g>gAgAe?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?g>g>g>g>g>g>f=f=f=f=f=f=f=f=f=f=d>d>d>d>f=f=d>d>b>b>b>b>b>b>d>d>d=d=e<f=b>b>U>&A* $("C&G,Y3`:r>vDFƒJŠ LŒMMNŠNŠNŠO‹O‹NŠNŒONMMŽMŽN‘N‘MŽNONŒP‹P‹P‹P‹OOP‘P‘O’O’OONOOP‘P‘ONNMŽNNŒNŒNNNŠO‹OŠOŠIz?p%J9+*,-/058!E(L )R(Q"IC<=%S/]7k =qDJ…L†SQ‰L„QˆQˆ@p+[!O(V9m AuF~ -Kƒ I„C~ :t4n -5d2aNIR$W ,9,8ET5S4V4V4Y2Y2U3U3U3U3S1S1S1S1S1S1S1S1S1S1R0R0R0R0R0R0R0Q/Q/Q/Q/Q/O0O0Q/Q/O0O0Q/Q/O0O0O0N/N/N/N/M.M.M.M.M.O-O-O-O-M.M.M.M.L-L-L-J+L,L,J*J*J*J*J*J*J*J*J*J*I*I*I*I*I*I*I*I*I)I)I)H(H)H)H)H)H)H)G'G'G'G'G'G'G'G'F'F'F'F'F'E&E&E&E&E&E&E&E&D%D% E&D% D% D%D%D%D%B# B# B# B# @$ @$ @$@$@$@$?#?#?#?#?#?#?#?#?#?#>" >" >" >" >" >" >" >" >" =! =! =! =! =! =! =! =! =! =! < < < : : : : 8! 8! : : : : : 9 9 9 9 9 9 7 -7 7 7 7 7 7 7 6 6 6 6 6 b<b<b<b<b<b<b<b<c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=c=d>d>d>d>d>e?e?e?e?e?e?e?d>d>d>e?e?g>g>g>g>g>g>g>g>c?c?c?eAeAeAeAeAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAhBhBjAjAjAjAi@jAjAjAi@jAjAjAjAjAjAjAjAjAjAjAjAjAjAjAkBkBjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAgAgAjAjAjAjAjAjAjAjAjAi@i@i@i@jAjAjAjAjAjAjAi@i@g>i@i@i@g>g>gAgAgAgAgAgAe?e?gAe?e?e?e?e?e?e?e?gAg>g>g>g>g>g>g>g>g>g>f=f=f=f=f=f=e?d>d>d>f=f=d>d>b>b>b>b>b>b>d>d>d=d=f=f=b>c?V?'B+ $( A'H-Z3`:r>vDH…K‹ LŒ N‘ N‘PŒPŒPŒO‹PŒO‹OONMNNN‘N‘NOOOP‹P‹P‹P‹OOP‘P‘P“P“OONOP‘P‘P‘OOONOONŒNNNŠO‹OŠN‰IzAr%J;+'))+,02?C#L(Q -*Q ,S+S -'O+Y 3a;o =qDL‡O‰SY‘Y‘WŽT‹=m#SH#Q/c0d8p" >" >" >" >" >" >" >" >" >" >" =! =! =! =! =! =! =! =! =! < < : : : : 7 8! : : : : : 9 9 9 9 9 9 7 -7 7 7 7 7 7 7 6 6 6 6 6 e;e;e;e;e;e;c<c<f<f<d=d=d=d=d=d=d=d=d=d=d>d>d>d>d>d>f=f=d>d>d>d>e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?g>g>g>g>g>g>g>i@eAeAeAeAeAeAgAgAi@i@i@i@gAgAgAgAgAgAgAgAgAgAgAgAhBhBjAi@jAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAjAi@jAjAjAjAjAjAjAjAjAi@jAjAi@gAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAi@i@g>g>g>g>g>g>g>g>g>g>f=f=f=g>e?e?e?e?g>g>f=g>e?e?c?c?c?c?e?e?f=g>f=g>e?d>\>!P2#  < -$C.[3`:sB{H…J‡LŠOR•R•QOŽPŽPŽPŽOOPŽOOOOONNOOOO‹O‹O‹O‹O‹O‹P‘P‘P‘P‘PŽPŽPŽOPŽPŽOŽOŽOŽOŽOŒOŒOŒN‹N‰N‰N‰OŠPˆO‡J{Ar%J:*' % % # # # #&*/2#;*B 0V 0V 0b6h :k6;C+_1e8q>wC~E€X”?yµI·"ZMˆ U R’I‰"=U"=UM6$N7%V4W5[4Z3X3X3V4U3U3U3U3U3U3U3U3U3U3U3U3S1S1S1S1S1S1S1S1R0R0R0R0R0R0R0R0R0R0Q/Q/Q/O0O0O0N/N/N/N/N/N/N/P.O-O-O-N/N/N/M.M.M.M.M.M.M.J+J+J+L-L-J+L-J+J+J+J+J+J+J+J+I*I*I*I)I)I*I*I*I*H)H)H)H)H)H)G(G(G(G(F'F'F'F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&D%D%D%D%D%D%D%D%D%D%D%B# B# B# B# B# B# A" A" A" A" A" A" A" A" @! @! @! >" >" >" >" >" >" >" >" =! -=! -=! -=! -=! -=! -=! -=! -< < < < : : : : : : : : : 9 9 9 9 9 9 7 -7 -7 -7 -7 -7 -6 6 6 6 6 6 6 e;e;e;e;e;e;c<c<f<f<d=d=d=d=d=d=d=d=d=d=d>d>d>d>d>d>f=f=d>e?e?e?e?e?e?e?e?e?e?e?e?gAgAgAgAgAgAgAi@i@i@i@i@i@i@i@eAeAeAeAeAfBhBhBjAjAjAjAhBhBhBhBhBhBhBhBhBhBhBhBhBhBjAjAjAjAjAkBkBkBkBkBkBkBkBkBjAjAjAkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBjAjAkBkBkBkBkBkBkBkBkBjAjAjAjAjAkBkBkBkBkBkBjAjAjAjAi@jAjAjAhBhBgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAi@i@i@i@i@i@i@g>g>g>g>g>g>g>g>g>e?e?e?e?g>g>g>g>e?e?c?c?c?c?e?e?g>g>g>g>e?e?]?"Z<93)*$% $ ? &E.[ -5bVN7%L5#U3V4[4Z3Y4X3V4V4U3V4V4V4V4V4U3U3U3U3U3U3U3U3U3S1S1S1S1S1R0R0R0R0R0R0R0R0R0R0R0R0P1P1O0O0O0O0O0O0N/N/P.P.P.P.N/N/N/N/N/M.M.M.M.M.M.L-L-L-L-L-L-L-J+J+J+J+J+J+J+J+J+J+J*J*I*I*I*I*I*I*I*I*H)H)G(G(G(G(G(G(G(G(G(F'F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&D%D%D%D%D%D%D%D%B# B# B# B# B# B# B# B# B# B# A" A" A" A" A" @! A" ?# ?# ?# >" >" >" >" >" =! -=! -=! -=! -=! -=! -=! -=! -< < < =!;!: : : : : : : : : : 9 9 9 9 9 9 7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -7 -f=f=f=f=f=f=d>d>f=f=d>d>d>d>d>d>d>d>d>d>d>e?e?e?e?d>e?e?e?e?e?e?gAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAi@i@i@i@i@i@i@i@eAeAfBfBfBfBhBhBjAjAjAjAjAjAjAjAhBhBhBhBhBhBhBhBhBhBjAjAjAjAkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBlClCkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBjAjAkBkBkBkBkBkBkBjAjAjAi@jAjAjAhBhBgAgAhBgAhBhBgAgAgAgAgAgAgAgAgAhBi@i@i@i@i@i@i@i@i@i@i@i@g>g>g>g>e?e?e?e?g>g>g>g>e?e?c?eAc?c?g>g>g>f=h>h>g>g>eAeAe@'^9 I. B'1,+1,+ .@&4F8^ =cBpEsGxL}M…M…M†M†O‰NˆNˆL†LˆLˆM„Lƒ M‡ M‡ L‰ L‰ M‡ Nˆ Nˆ M‡ M… M… M… M… L‰ MŠ MŠ MŠNNN“L‘L‘N“N“O” P NŽ NŽ NŽ NŽ NŽO“O“P“P“O“Q• -QŽOŒJ| -@r$J9((      - !+ -. A -$E+T.W3_6b -7h 9j,UD. ) & -' ,4#O+W6o -?xD€GƒH…A~EƒI‡KŽL -J‰F…;a8^).=,1@C4(F7+T5U6Y4X3V4V4V4V4V4V4V4V4T5S4U3U3U3U3U3U3U3U3S1S1S1S1S1S1S1S1R0R0R0R0R0R0R0R0R0R0O0O0O0O0O0O0O0O0P.P.P.P.N/N/N/N/N/N/N/M.M.M.M.M.M.M.M.L-L-L-L-L-L-L-L-L-J+J+J+J+J+J+J+J+I*I*I*I*I*I*I*I*I*H)H)H)G)G)G(G(G(G(G(F'F'F'F'F'F'F'E&F'E&E&E&E&E&E&E&E&E&E&D%D%D%D%D% D% D% D% B$ B$ B# B# B# B# B# B# A" A" A" A" A" A" ?# ?# ?# ?# ?# >" >" >" >" >" >" >" >" =! -=! -=! -=! =! =! =! =! =! < < < < : : : : : 9 : : : 9 9 9 9 9 7 -7 -7 -7 -7 -7 -7 -7 -f=f=f=f=f=f=d>d>g>g>e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?d>e?e?e?e?e?gAgAgAgAgAgAgAgAgAgAhBhBhBhBhBhBhBjAjAjAjAjAjAjAjAfBfBfBfBfBfBhBiCjAjAjAjAjAjAjAjAhBhBhBhBhBhBhBiCiCiCkBkBjAjAkBkBkBkBkBlClClClClCkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBlClCkBkBkBkBkBkBkBkBkBkBkBkBkBkBlClClClClClClClClClClClClClClClCkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBlClClClClClClClCkBkBkBkBkBkBkBlClCkBkBkBkBkBjAkBkBkBkBiC hBhBgAhBhBhBhBiChBgAhBhBhBhBhBhBhBjAjAi@i@i@i@i@i@i@i@i@i@g>g>g>g>e?e?gAgAi@i@i@i@gAgAeAeAeAeAi@g>g>f=h>h>g>g>fBfBe@'fA(Z?1W<.D?>A<;-;M-;M =c"?eEsFtIz M~M…N†OˆOˆO‰O‰O‰NˆLˆLˆM„Lƒ M‡ M‡ L‰ L‰ M‡ Nˆ Nˆ Nˆ N† N† M… N†OŒ MŠOŒOŒO‘P’P•O”O”O”O”O”Q‘ P P NŽ NŽ PQ•O“P“P“Q•Q• R RJ| -@r$J8 '(     ((7< I&O,X1]4e 9j 4]'P:1 -'" -+9"N)U6o;t@|GƒJ‡J‡K‰NŒN‘N‘ MŒH‡;a7]!&5!&5;, A2&S4U6X3Y4W5W5W5W5W5W5V4V4T5T5V4U3U3U3U3U3U3U3U3U3U3S1S1S1S1S1S1S1S1S1S1S1S1S1R0R0P1P1P1O0O0O0O0O0Q/P.P.Q/O0O0O0O0N/N/N/N/N/N/M.M.M.M.M.M.M.L-L-L-L-L-L-L-L-J+J+J+J+J+J+J+J+J+I*J+I*I*I*I*I*I*I*H)G)G)G(G(G(G(G(G(G(F'F'F'F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&E&D% D% D% D% D& D& D% B# B# B# B# B# B# B# A" A" A" A" ?# ?# ?# ?# ?# ?# ?# ?# >" >" >" >" >" >" >" >" =! =! =! =! =! =! =! =! =! =! ;! ;! ;! : : : : : : 9 9 9 9 9 7 -7 -7 -7 -7 -7 -7 -7 -g>g>g>g>g>g>e?e?g>g>e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?gAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAhBhBhBhBhBhBhBkAkAkAkAkAkAkAkAhBhBiCiCiCiCiCiCkBkBkBkBkBkBkBkBjAkBiCiCiCiCiCiCiCiCkBkBkBkBkBlClClClClClClClClClClClClClClClClClClClClClClClClCmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDlClClClClClClClClClClClClClClClClClClClClClClClClClClClClClClClCkBkBlClClClClClClClClClCkBkBkBkBkBlClClClClClCkBkBjAkBkBkBkBkB kB kBkBkBjAkBkBkBkBkBkBkBkBkBjAjAkBjAjAjAjAjAjAjAjAi@i@i@i@i@i@i@i@hBgAgAgAi@i@i@i@gAgAeAeAeAeAi@g>g>g>g>g>g>i@i@i@eAeAfBfBe@ b=`=&a>'e>#e>#`=(a>)`A/bC1^C8^C8`C8`C8WC:XD;XD;WC:V@6V@6HBGGAFBBNAAMAAOAAOAAOBBPBBPBBPBBWBBW@CW@CW=D\=D\k $E2 %(   - -   - - ( -@ -$E /[8d:h8f3]%O;5;D#Q+Y9q;sAzK„ R‹UŽU’ -QŽN’ -Q•NŠI…?n7f,Z)W*H-KG3#K7'W5W5Y5Y5X6X6W5W5W5W5U6T5W5V4U3U3V4V4U3U3U3U3U3U3U3U3S1S1S1S1S1S1S1S1S1S1S1S1R0R0R0R0R0R0Q/Q/Q/Q/Q/Q/O0O0O0O0O0O0N/N/N/N/N/N/N/N/M.M.M.M.L-L-L-L-L-L-L-L-L-L-L-L-J+J+J+J+J+J+J+J+I*I*I*I*I*I*I*I*I)I)H(H(H(H(H(H(H(H(G'G'G'G'G'G'F'F'F'E&E&E&E&E&E&E&E&E&D% D% D& D& D& D& D% D% D% D% B# A" -B# B# B# A" A" A" ?# -?# -?# ?# ?# ?# ?# ?# >" >" >" >" >" >" >" >" =! =! =! =! =! =! =! =! =! =! ;! ;! ;! ;! ;! : : : : 9 9 9 9 9 9 9 9 9 9 9 9 7 -g>g>g>g>g>g>e?e?g>g>e?e?e?e?e?e?e?e?gAgAgAgAgAgAgAgAgAgAgAgAgAgAhBhBhBhBhBhBhBhBhBhBhBhBhBiCiC iC kAkAkAkAkAkAkAkAhBhBiCiCiCiCiCiCkBkBkBkBkBkBkBkBjAkBiCiCiCiCiCiCjDjDlClClClClClClClClCmD mD mD mD mD lClClClClClClCmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmD mD mD mD mD mD mD mD mD mD mD mD mD mD mD mD lClClClClClClClClClClClClClClClCkBlClClClClClClCkBkBkBlClCkBlC!kB kBkBkBjAjAkBkBkBkBkBkBkBjAjAkBkBkBkBkBjAjAjAjAjAjAjAjAjAjAjAjAi@hBhBhBgAi@i@i@i@hBhBfBfBeAeAi@g>g>g>g>g>g>i@i@i@eAeAeAeAe@ c>`=&a>'hA&hA&dA,c@+aB0bC1^C8]B7^A6^A6WC:UA8WC:UA8V@6XB8HBGGAFBBNAAM??M??M@@NAAOAAO@@N??T??T>AU>AU9@X;BZ9D]9D]6C`6C`7Cc7Cc4Cd5De/Gk/Gk*Gm,Io&Im(Ko&Ko&KoK{IyK|K|LM€O‚WŠ(U‚Cp(I5((  - - - -  - &)77J(T/] 3a -6` 5_ (NBCD#Q-[9q;sG€K„M†UŽV“U’U™ S—LˆK‡Cr9h 3a-['E*HI5%L8(X6X6Z6Y5W5W5W5W5W5W5T5U6W5W5W5V4V4V4V4V4V4V4V4U3U3U3U3U3U3U3U3U3U3S1S1S1S1S1S1S1S1R0R0R0R0R0Q/Q/Q/Q/O0O0O0O0O0O0O0O0N/N/N/N/N/N/N/M.M.M.M.M.M.M.M.M.M.L-L-L-L-L-L-L-J+J+J+J+J+J+J+J+J+I*I*I*I*I*I)I)I)I)I)H(H(H(H(H(H(H(G'H(G'F&F'F'F'F'F'E&E&E&E&E&E&E&E&E&D& E' D& D& D% D% D% D% D% D% B# B# B# B# B# B# @$ @$ @$ ?# ?# ?# ?# ?# >" >" >" >" >" >" >" >" =! =! =! >" >" >" =! =! =! =! ;! ;! ;! ;! ;! : : : ;!: : : : 9 9 9 9 9 9 9 9 9 g>g>g>g>g>g>e?e?g>g>e?e?e?e?e?e?e?e?gAgAgAgAgAgAgAgAgAhBjAjAhBhBjAjAhBhBhBhBhBhBhBiCiCiCiCiCiCiCkAlBlBlBlBlBlBlBiCiCiCiCiCiCiCjDlClClClClClClClCkBkBlClClClCjDjDlClCmCmCmCmCmCnDnDnDnDnDnDnDnDnDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDnDnDnDnDnDnDnDnDnDnDnDnDnDnDnDnDnD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD mD mD lClCmD mD mD mD mD mD mD mD lClClCmD mD mD mD mD mD mD lClClClClClClClClCkBkBkBkBkBkBkBlCkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBjAjAjAjAjAjAjAjAhBhBhBhBjAjAhBhBjAjAjAjAjAjAjAjAgAgAg>g>i@i@i@i@hBhBjAjAi@i@gAgAi@i@h>h>jAkB jAjAeAeAe@ c>eAeAgAgAj@g=f<g=g=f<g=g=g=g=f<g=e;e;c<c<`> _=_=_=`>"_=!\>!\>![?&Z>%\>#[="\>#\>#[="[="\>#\>#]>!]>!\>#\>#SB/P?, -  - -    - -   - -  #&6 ;#I )O,[*Y-a/c0h4l" >" >" >" >" >" >" >" =! =! =! =! =! =! =! =! =! ;! ;! ;!: : : : 9 9 9 9 9 9 9 9 9 g>g>g>g>g>g>e?e?g>g>e?e?e?e?e?e?gAgAgAgAgAhBhBhBhBhBhBhBjAjAhBhBkBkBiChBhBhBhBhBhBiCiCiCiCiCiCiClBlBlBlBlBlBlBlBiCiCjDjDjDjDjDjDlClClClClClClClClClClClClClCjDjDlClCmCnDnDnDnDnDnDnDnDnDoEoEoEoEmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDoEoEoEoEoEoEoEoEoEoEoEoEoEoEoEoEnD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD mD mD mD mD mD mD mD mD mD mD mD mD lClClCmD mD mD mD mD mD mD mD mD lClClClClClClClClClCkBkBlClClClClClClClClCkBkBkBkBkBkBkBkBkBkBkBjAjAjAjAjAjAjAjAiCiChBhBjAjAhBhBjAjAjAjAjAjAjAjAhBhBjAjAjAi@i@i@hBhBjAjAjAi@hBiC kB jAkAj@jAjAjAkB fBfBfA!fA!eAeAgAgAh>j@h>h>j@j@j@j@j@j@j@h>g=g=e>e>cA#a?!a?!cA#a?#a?#]?"]?"\@'\@']?$]?$]?$]?$\>#\>#\>#[="\= \= [="[="UD1K:' -    - - -       $$03C &L,[-\3g5i7o7o" >" >" >" >" >" >" >" >" >" >" >" >" =! =! =! ;! ;! ;!: : : : : 9 9 9 9 9 9 9 9 i@i@i@i@i@i@gAgAi@i@gAgAgAgAgAgAhBhBhBhBhBhBhBhBhBhBhBhBkBkBjAkBkBkBkBkBkBkBkBkBkBkBlClClClClClClBlBlBlBlBlBlBlBiCiCjDjDjDjDjDjDlClClClClClClClClClClClClClCjDkEmDmDnDnDnDnDnDnDnDnDnDoEoEoEoEoEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEoEoEoEoEoEoEoEoEoEoEoEoEoEoEoEoEnD nD nD nD nD nD nD nD nD nD nD nD nD nD nD nD oEoEoEoEoEoEoEoEoEoEoEoEoEoEoEoEnDnDnEnEnEnEnEnEnEnEmDmDlClCmDnEmDmDnEnEmDmDmDmDmDmDmDmDmD mD mD mD mD lCkBkBlClClClClClClClClCkBlClClClClCkBkBkBkBkBkBkBkBkBkBkBjAjAkBkBkBkBkBkBiCiCkBkBkBkBkBkBjAjAhBhBhBhBhBhBhBhBjAjAjAkBkBjAhBiC kB jAhBiC iC iC iC iC hBhBhBhBgAgAgAhBi@i@i@i@i@i@i@i@i@i@i@g>g>f=g>g>i@i@i@i@g>g>g>f=g>g>g>g>g>f=f<f<f<e;g:g:g:f9cA#X6 -    -         - -  -&&: -?&T-[6i8k" >" >" ?# ?#?#>" >" >" >" >" >" >" >" >" =! =! =! =! =! : : : : : : : : : : : : jAjAjAjAjAjAhBhBjAjAhBhBhBhBhBhBhBhBhBhBhBiCiCiCiCiCiCiCjAkBkBkBkBlClClClClClClClClClClClClClClCmCmCmCmCmCmCmCmCjDjDkEkEkEkEkEkEmDmDmDmDmDmDmDmDlClClClClClCjDkEmDmDnDnDnDnDoEoEoEoEoEoEoEoEoEoEnEnEnEnEnEnEnEnEnEnEnEpG!pG!pG!pG!pG!oEoEoEoEoEoEoEoEoEoEoEoEoEoEoEoEqG#qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#oEoEoEoEoEoEoEoEoEoEoEoEoEoEoEoEoEoEnEnEnEnEnEnEnEnEnEnEmDmDmDnEnEnEnEnEnEnEnEmDlCmDmDmDmD mD mD mD mD mD mD lClClClClCmD lClClClClClCmD lClClClClClClClClClClCkBkBkBkBkBlClCkBkBkBkBiCiCkBkBkBkBkBkBjAjAhBhBhBhBhBhBhBhBjAjAjAkBlClCjD!jD!kB kB iC iC iC iC iC iC iC hBhBhBiCgAhBhBjAi@i@i@i@i@i@i@i@i@i@i@g>g>g>g>i@i@i@i@i@g>g>g>g>g>g>g>f=f=f<f<f<f<g:g:h;f9cA#V4   - - -  -     -    & '9<%S-[7j" >" >" >" >" >" >" >" =! =! =! >" =! ;!;!;!: : : : : : : : : jAjAjAjAjAjAhBhBjAjAhBhBhBhBhBhBhBhBhBhBhBiCiCiCiCiCiCiCkBkBkBkBlClClClClClClClClClClClClClCmDmDnD nD nD nD nD nD nDnDkE kE kE kE kE kE kE kE mDmDmDmDmDmDmDmDnEnEnEmDlCmDmDnEoEoEoEoEoEoEqG!qG!qG!qG!qG!qG!qG!qG!qG!qG!pGpGpGpGpGpGpGpGpGpGpGpGpGpGpGpGrH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!oEoEoEoEoEoEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEmDmDnE!nE!mD nE!nE!mD mD mD lCmD mD lClClClCmD mD mD mD mD mD lClClClClClClClClClClClCkBlClCkBlCkBkBjDiCkBkBkBkBkBjAkBlCjDiCiChBhBhBiChBjAkBkBkBlClCjDjDlC!kB iC iC iC jD!iC iC iC iC iC iC iC iC hBhBfBeAeAeAeAeAeAeAe@ e@ e@ e@ g@ e>e>e>i@i@i@i@i@i@h>h>g>g>g>g>c=c=e<f=g=f<e<e<f<d:c>#W2 -    -   -     !!48'T-Z6i;n=vE~N‹L‰LN‘P•T™S˜S˜UšUšV—W˜V’RŽN‰J…AqAqE8.H;1Z8Z8\8\8\7[6]6]6]6]6]6\5\5\5\5\5Z5Z5W5W5W5W5W5W5W5W5W5V4V4V4V4W5W5W5U3V4V4V4V4V4V4U3U3U3U3U3U0U0U0U0S1S1U0U0U0U0T/T/R0R0R0R0R0R0P1O0N/N/N/N/N/N/N/N/N/N/N/N/N/N/N/M.N/M.M.M.M.M.M.M.M.L-L-L-L-L-J*J*J*J*J*J*J*J*J*I)I)I)H(I)I)H(H(I)H(H(H(H(H(H(H(H(H(H(G'G'G( G( F( F( F( F( E&E&E&E&E&E&E&B# D% D% B&B&B&B&B&B&B&B&@$@$@$@$@$@$@$@$?# ?# ?# ?# ?# ?# ?#?#?#?#?#?#?#>" >" >" >" >" >" =! =! =! =! =! < < < < < < < < kBkBkBkBkBkBiCiCkBkBiCiCiCiCiCiCiCiCiCiCiCjDjDjDjDjDjDjDlClClClCmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDmDnD nD nD nD nD nD nDnDkE kE kE kE kE kE kE lF!nEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEqG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!rH"rH"rH"rH"rH"qH qH qH qH qH qH qH qH qH qH qH qH qH qH qH qH rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"qG!qG!qG!qG!qG!qG!pG!pG!pG!pG!pG!pG!nEnEnEnEpG!pG!pG!pG!pG!pG!nEnEnEnEnEnEpG!pG!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!mD mD mD mD mD nE!mD mD mD mD mD mD mD mD mD mD mD lClClClClCmD mD mD lClClCjDjDlCmDmDmDlClClClCjDjDjDjDjDjDiCiClClClClClClCjDiCkB kB jD!jD!jD!jD!jD!iC iC iC iC iC iC iC hBiC fBfBeAeAeAeAeAeAe@ e@ e@ e@ g@ e>e>e>i@i@i@i@i@i@h>h>g>g>g>g>d>d>f=f=g=f<e<e<g=d:e@%U0 -    - -   - -  -   " "48'T-Z5h;n>wD}KˆJ‡KŽLO”S˜S˜S˜T™UšV—V—U‘RŽOŠL‡AqAqF9/H;1Z8Z8\8\8\7[6]6]6]6]6]6\5]6]6\5\5Z5Z5X6X6X6X6W5W5W5W5W5W5W5W5V4W5W5W5W5W5V4V4V4V4V4V4V4V4U3U3W2U0U0W2U3U3W2U0U0U0U0U0S1S1S1S1R0R0P1O0O0P1O0O0O0O0O0O0O0N/N/N/N/N/N/N/N/N/N/M.M.M.M.M.M.M.M.L-L-L-J*L,L,L,L,J*J*J*J*J*J*I)I)I)I)I)I)I)I)I)I)H(H(H(H(H(H(H(H(G'G( G( F( F( F( F( F'F'F'F'E&E&E&E&D% D% B&B&B&B&B&B&B&B&@$@$@$@$@$@$@$@$?# ?# ?# @$ @$ ?# ?#?#?#?#?#?#?#>" >" >" >" >" >" =! =! =! =! =! =! < < < < < < < kBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBlClCjDjDjDjDjDjDjDjDmD mD lCmD mDnEnEmDmDmDmDmDnEnEnEnEnEnEnEnEoE!oE!oE!oE!oE!oE!oEoElF!lF!lF!lF!lF!lF!lF!lF!oEoEoEoEoEoEoEoEoEoEoEoEqG!oEoEoEqG!qG!qG!qG!qG!qG!qG!rH"rH"rH"rH"rH"rH"rH"rH"rH"qH qH qH qH qH qH qH qH qH qH qH qH qH qH qH qH rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!pG#nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!mD mD mD mD mD mD mD mD mD mD lClCmD"mD"mD"mD"mD mD mDmDmDmDmDmDmDmDlClCjDjDiCjDjDjDjDkElClClClClClCjDjDlCkBjD!jD!jD!jD!jD!jD!jD!iC jD!jD!jD!iC iCiCjAjAjAi@jAjAhBgAgAgAgAgAi@g>g>i@i@i@i@i@i@i@j@h>g>g>i@i@d>d>f=g>f=f=f=f=f=d;dB(Q/    -""& )58(R.X4g;n>uD{J…J…L‰L‰N R‘S”T•U˜U˜ -S— -S—T‘QŽNŠK‡@m@mH9-J;/X9X9`:^8\7\7^7^7^7^7]6]6_5_5]6]6[6[6Z5Z5Z5Z5Z5Z5Y4Z5Z5Y4Z5Z5X6W5W5W5W5W5W5W5W5W5V4V4V4V4U3V4W2W2W2W2U3U3W2W2W2U0U0U0S1S1S1S1S1R0R0R0P1P1O0O0Q/Q/Q/Q/Q/Q/Q/Q/P.Q/N/N/N/N/N/N/M.M.M.M.M.M.M.M.M.M.L-L-L-L-L,L,L,J*J*J*J*J*I)J*J*I)I)I)I)I)I)I)I)I)H(H(H(H(H(H(H)H)G)G)F( F( F'F'F'F'F'E&E&E&E&E&E&E&E&D%D%D%D%D%D%B#B#B#B#B#B#B#B# B# B# B# @$ @$ @$ ?# ?#?#?#?#?#?#?#>" >" >" ?#>" >" >" >" =! =! =! =! =! =! =! =! =! kBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBkBlClCjDjDjDjDjDjDjDkEmD mD mD mD nEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEoE!oE!oE!oE!oE!oE!oEoElF!lF!lF!lF!lF!lF!lF!lF!oEoEoEoEoEoEoEoEqG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!rH"rH"rH"rH"rH"rH"rH"rH"rH"qH qH qH qH qH qH qH qH qH qH qH qH qH qH qH qH sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$sI%sI%sI%sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#rH"rH"rH"rH"rH"rH"rH"rH"rH"qG!qG!qG!qG!qG!qG!qG!rH"rH"qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!pG#pG#nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!mD mD mD mD mD mD mD mD mD mD"mD"mD"mD"mD mD mDmDmDmDmDmDmDmDmD mD kEkEkEjDjDjDjDjDmDmDmDmDmDlCjDjDlClCjD!jD!jD!jD!jD!jD!jD!jD!jD!jD!jD!jD!iCiCkBkBkBkBjAjAhBhBgAgAhBhBjAjAjAi@i@i@i@i@i@i@j@j@g>g>i@i@e?gAg>g>g>f=f=f=f=e<fD*N,&$%642,*(   - "#"' -*6 :*T.X4g;n?vE|K†K†KˆMŠN R‘T•T•U˜V™ T˜ T˜U’QŽNŠK‡@m@mH9-J;/X9Z;`:^8^9^9`9`9^7^7^7^7`6`6]6]6[6[6[6Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5X6X6X6X6X6W5W5W5W5W5W5W5W5V4W5V4X3W2W2W2U3U3W2W2W2W2W2U0S1S1S1S1S1S1S1R0P1P1P1P1Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/N/N/N/N/N/N/N/N/N/N/N/M.M.M.M.M.M.L-L-M.L,L,L,L,L,L,J*J*I)J*J*I)I)J*I)I)I)I)I)I)I)I)I)I)H(H(H)H)G)G)G)G)F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&E&D%D%D%D%D%D%D%D%B# B# B# B# @$ @$ @$ @$ @$?#@$@$?#?#?#?#?#?#?#>" >" >" >" >" =! =! =! =! =! =! =! =! kBkBkBkBkBkBkBkBkBkBkBkBlClCmCmCmCmCmCmClClCmD lClClCmDmDnDnDnDnDoEoEoEoEoEoEoEoEoEoEoEoEoEoEoEoEnE!nE!nE!nE!nE!nE!nEnEnE!nE!nE!nE!nE!nE!nE!nE!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qGqGqGqGqGqGrH rH rH rH rH rH rH rH rH rH sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!pG!pG!pG!pG!pG!pG!nEnEnEpG!pG!nEnE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!mD nE!mD mD nD nD mD mD nD nD nD nD nD mCmD mD mD mD mDmDmD lCjDjDmDmDmCnDmDmDmD lClClClClClClClCkBlClClClClClClCkBkBkBkBkBkBjAjAjAjAjAjAjAhBhBhBhBi@i@eAeAgAgAgAgAg>g>gAgAe>g@ e?d>e?e?d>d>d>c=dE(M.>/!XC+Q<$C0;(+%!! -(,< @(R-W2d7i=tByI…J†L…N‡QŽ RT’U“V”U“V˜U—T”Q‘MˆJ…?dAfL:/L:/X9Z;`:`:^9^9`:`:`9`9^9\7\7\7\7\7\7\7\7[6[6[6]6\5]6]6[6[6[6[6Z5Z5Z5Z5Z5Z5W5W5W5W5W5W5W5W5Y4X3V4U3V4V4V4V4V4V4W2W2W2W2W2W2U3U3S1S1S1S1S1S1R0R0Q/R0R0R0R0Q/R0Q/Q/Q/R0R0P.P.Q/P.P.P.N/N/N/N/N/N/O-O-O-O-O-N,N,N,N,N,N,L*L*L*L*L*J*J*J*J*J+J+J+I*I*I*I*I*I*I*I*H)H)H)H)H)G)G)H(G'H(G'F'F'F'F'E&E&E&E&E&E&E&E&E&E&D%D%D%D%D%D%D%D%B# B# B# B# @$ @$ @$ @$ @$ @$ @$@$@$?#?#?#?#?#?#>" >" >" >" >" >" =! =! =! =! =! =! =! kBkBkBkBkBkBkBkBkBkBkBkBlClCmCnDmCmCnD mClClCmD mD mDmDmDmDoEoEnDoEoEoEoEoEoEoEoEoEoEoEoEqG!qG!qG!qG!qG!nE!nE!nE!nE!nE!nE!nEnEpG#pG#pG#pG#pG#pG#pG#pG#qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!rH"rH"rH"rH"rH"rH rH rH rH rH rH rH rH rH rH rH sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#rH"sI#sI#sI#sI#sI#sI#sI#sI#rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"qG!qG!qG!qG!qG!qG!qG!qG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG#pG#pG#nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!oE!oE!nE!nE!oE!nD nD nD nD nD mD mD mD mD mDmDmD mD kE kE mDlCmCnDmDmDmD lCmD mD lClClClClClCmD mD lClClClClCkBkBkBkBkBkBjAjAjAjAjAjAjAhBhBhBhBjAjAfBfBhBhBhBgAg>g>gAhBhA!hA!e?e?e?e?e?e?c=c=eF)L- >/!VA)U@(TA,Q>)D7/>1)(%- % -(,: -?*T-W3e9k>uByGƒK‡N‡N‡OŒ -SU“U“V”V”V˜U—T”Q‘MˆJ…@eBgL:/N<1[<[<a;`:^9_:a;`:`9`9^9^9^9^9\7\7\7\7\7\7\7\7^7]6]6]6[6[6[6[6Z5[6[6[6Z5Z5X6X6X6X6X6W5W5W5Y4Y4V4W5V4V4V4V4V4V4X3X3X3W2W2W2U3U3U3U3S1S1S1S1R0R0S1R0R0R0R0R0R0R0R0R0R0R0R0R0Q/Q/Q/Q/O0O0N/N/N/N/O-P.P.O-O-N,N,N,N,O-N,L*L*N,L*N,J*J*J*J*J+J+J+J+J+J+I*I*I*I*I*I*H)I*I*H)G)G)H(H(H(H(F'F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&E&D%D%D%D%D%D%B# @$ @$ @$ @$ @$ @$ @$@$@$?#?#?#?#?#?#?#?#?#?#>" >" >" >" >" >" >" >" >" iCiCiCiCjDjDjDjDjDjDlClCmDmDmDmDmDmDmDlClClCmDnEnEnEoEoEoEoEoEoEqG!qG!qG!oEoEoEoEoEqG!qG!qG!qG!qG!qG!qG!qG!pG#pG#pG#pG#pG#pG#pG!pG!pG!pG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!rH"rH"rH"rH"rH"rH"rH rH rH rH rH rH rH sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$uK%uK%uK%uK%uK%uK%uK%tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"qG!qG!qG!qG!qG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!nEnEnEnEnEnE!nE!oE!oE!oE!oE!oE!oE!oE!oE!oE!oE!nD nD nD nD mD mD mD mD mD"mD"mD"mD"nD nD nDnDnDnDmCmCnEnEmD mD lClCmDmDnE!mD mD mD lClClClClClClCkBkBkBhBiCiCiCiCiCiChBhBiC hBhBhBhBhBhBjAjAkAkAj@lBmCmCjAi@g>i@i@g>c=e?eF+G( F0!_@#^?"dB!dB!`B']?$P8)G/ ;)"0%% ."0(O ,S5j:o@| -D€ IŠ KŒ Oˆ QŠ -S U’ -V” -V” V“ V“V—U– RŽ PŒJG~,=^->_Q9*S;,[<[<c:c:a:a:a:a:c9c9`9`9^9^9^9^9\7\7\7\7\7\7^7^7^7^7^7^7]6]6Z5[6]6]6[6[6Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5W5W5V4V4V4W5V4V4V4V4X3W2W2W2U3U3U3U3S1S1S1S1R0S1S1R0S1R0R0R0T/T/T/T/T/S.R0R0Q/Q/Q/Q/Q/Q/O0N/N/O0P.P.P.O-O-O-O-N,N,O-O-L*N,N,N,N,L-L-L-L-J+J+J+J+J+J+L*L*K)K)I*I*H)I*I*H)G)G)H)H)H)H)H(H(F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&E&D%D%D%D%D%D%B&B&B&@$ @$ @$ @$ @$ @$@$@$@$?#?#?# ?# ?# ?# ?# ?# >" >" >" >" >" >" >" >" jDjDjDjDkEkEkEkEkEkEmDmDnEnEnEnEnEnEnEnEnEnEnEnEnEnEoEoEoEoEoEoEqG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!pG#pG#pG#pG#pG#pG#pG!pG!qH"qH"qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!rH"rH"rH"rH"rH"rH"sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$uK%tJ$tJ$uK%uK%uK%uK%uK%uK%uK%uK%uK%uK%uK%tJ$tJ$uK%tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#tJ$sI#tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI#rH"rH"sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#rH"rH"rH"rH"rH"rH"rH"rH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"pG!pG!pG!pG!pG!pG!pG!pG!pG!nE!pG#qG#qG#qG#qG#oE!oE!oE!oE!oE!oE!oE!oE!oE!oE!nE!nE!nE!nE!nE#nE#nE#mD"nD nD nDnDnDnDnD nD nEnEmD mD mD mD nEnEnE!nE!nE!mD mD mD mD mD lCmD lClClCkBiCiCiCiCiCiCiCiCiC iC iCiCiCiCiCiCkBkBkAkAlBlBmCmCkBi@i@i@jAjAd>hBfG,B#F0!_@#_@#a?`>^@%`B'ZB3X@1P>7I705+.-#&%3&4 ,S0W9n =r C E -J‹ L Oˆ QŠ -S U’ W• W• V“X• W˜V—S QJG~/@a/@aS;,S;,[<[<c:c:a:`9a:a:c9c9a:a:^9^9^9^9^9^9\7^9^9\7^7^7^7^7^7^7^7^7[6[6]6]6[6[6[6[6[6[6Z5Z5Z5Z5Z5Z5W5W5X6X6X6W5V4W5W5W5Y4Y4Y4Y4U3U3S1V4V4S1U3U3U3S1S1U3S1S1S1R0U0U0T/U0U0T/R0R0R0R0Q/Q/Q/Q/O0O0O0O0Q/P.P.P.P.P.O-O-O-O-O-N,O-N,N,N,L-L-L-L-L-L-L-J+J+J+L*L*L*L*J+I*I*I*I*I*H*H*I*H)H)H)H(H(G(F'G(F'F'F'F'F'F'E&F'F'E&E&E&E&E&E&E&E&D%D%D%D%B&B&B&@$ @$ @$ @$ @$ @$@$@$@$?#@$@$ ?# ?# ?# ?# ?# ?# >" >" >" >" >" >" >" jDjDjDjDkEkEkEkEkEkEmDmDnEnEnEpG!nEnEpG!nEnEnEnEpG!pG!pG!qG!oEqGqGoEqGqG!qG!qG!qG!qG!qG!qG!qG!qG!qG!qG!rH"rH"rH"rH"rH"pG#pG#pG#pG#pG#pG#qH$qH$qH qH rH rH rH rH rH rH sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI!sI!sI!sI!sI!sI!sI!tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"uK#uK#tJ"vL&vL&vL&vL&vL&vL&vL&uK%uK%uK%uK%tJ$tJ$uK%uK#uK#uK#uK#tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ$tJ$tJ$tJ$tJ$tJ$tJ$sI#tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#rH"rH"rH"rH"rH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"pG!pG!pG!pG!pG!pG#pG#qG#qG#qG#qG#qG#qG#qG#oE!oE!oE!oE!oE!oE!oE!nE!nE!nE!nE!nE#nE#nE#nE#oE!oE!nDnDoEoEnDnDnEnEnEmDmDmDnEnEnEnEnEmDmDmDmDmDmDmDlClCmCmClClCkBkBkBkBiCiCkB lC!iCiCiCiCiCiCkBkBkBkBkBkBlBlBkBi@j@j@nAm@c=iC hI.@!O5"c>!b= d>d>d>d>d= c<ciC hI.<S9&b= b= d>d>e?e?d= d= d="e>#b<&`:$R<0R<0B?CA>B6BQ6BQ/C\/C\)Ff*Gg"Jm$Lo!Ou"Pv!Ry"Sz$Rz&T|1Nt/Lr6Ig5Hf>CN3_=#_=#_=_=e<d;b;b;b;b;d:d:b;b;a:a:_;_;_:_:_:^9^9^9`9^7`9`9`9`9`9^7\7\7^7^7^7^7[6[6[6[6[6[6[6[6[6[6Z5Z5Z5Z5Z5[6Z5Z5Z5Y4Y4Y4Y4Y4W5W5W5W5V4V4U3U3U3V4U3U3U3S1S1S1U0U0U0U0U0U0U0U0S1R0R0R0Q/R0R0R0T/T/Q/Q/Q/Q/Q/P.P.O-O-P.P.O-O-O-P.O-M.M.M.M.L-L-L-L-L-L-N,N,N,L*L*L*I*J+J+I*I*I*I*I*I*I*I)I)G(G(G(G(G(G(G(F'F( F( F( F( F( F( F'F'F'F'E&E&E&E&E&E&C'B&B&B&B&B&B&B&B&B&B&B&@$ @$ @$ @$ @$ @$ @$ @$ ?# ?# ?# ?# ?# ?# ?# ?# kE kE kE kE lFlFlFlFlFlFnEnEpG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!qG!qG!qGqGqGqGrH"rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"sI#sI#sI#sI#sI#qH"qH"qH"qH"qH"qH"rI#rI#rI!rI!sI!sI!sI!sI!sI!sI!sI!sI!sI#sI#sI#sI#tJ$tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI#tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#vL$vL$vL$uK#vL$vL$xN&vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$xK$xK$xK$xK$xK$xK$xK$xK$xK$xK$xK$xK$wJ#wJ#wJ#wJ#uK%uK%uK%tJ$uK%uK%tJ$uK%tJ$uK%uK%uK%uK%uK%uK%uK%tJ$tJ$tJ$sI#sI#tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI#sI#sI#sI#sI#sI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#qH"qH"qH"qH"qH"qH"qH"qH"qH"qG#rH$sI%sI%rH$rH$qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#pG#pG#pG#pG#pG%nE#nE#nE#qG#oE!oEoEoEqG!qG!oEpG!pG!nEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEnEmDmDmDmDmDkEkEkEkEkEkElClCmD mD mD mD mD mD mD lClBlBlClCnD nD mCkAkBkBlBlBd>jD!fF.: [?.iB"e>h>h>i@i@h>g=g=h>i<i<f< f< e<f=d>b<^<_=Z>%[?&Y@$ZA%XA'ZC)XA)XA)\A%\A%^@!]? _>_>^<^<b<c=c=c=a<a<c<c<c<c<b;b;b;b;b;b;b;b;c9c9c9c9b8c9a:`9^9^9`9`9`9`9`9`9\7\7^7^7^8^8\8\8\8\8[7[7[7[7[6[6Z5[6[6[6[7[7[7Z6Z5Z5Z5Z5Z5Z5Y4Y4Y4Y4Y4X3X3X3X3X3X3X3W2U0W2W2W2W2W2W2W2W2W2U0U0U0U0T/S1S1R0R0R0R0T/S.S.S.Q/Q/Q/O-O-P.R-R-R-R-O-O-L-M.M.M.M.M.M.M.M.L-N,N,N,N,N,L*L*L*J+J+J+J+J+J+J+I*I*I*H*H*H)G(G(G(G)G)G) F( G) G) F( F( F'F'F'F'F'F'F'F'F'E&E&D% D% D% D% D% D% D% D% D% D%D%B# B# B# B# @$ @$ @$ @$ ?# ?# ?# ?# ?# ?# ?# ?# lF!lF!lF!lF!nH!nH!nH!nH!nH!nH!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!qH"qH"qH"rH"rH"rH rH rH rH rH"rH"rH"rH"rH"rH"rH"sI#sI#sI#sI#sI#sI#sI#sI#sI#rI#rI#rI#rI#rI#rI#rI#rI#rI!rI!sI!sI!sI!sI!sI!sI!tJ"tJ"tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$uK%uK%uK%uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$xN&vL$vL$xN&xN&xN&xN&xN&xN&xN&vL$vL$vL$vL$vL$vL$xN&xK$xK$xK$xK$xK$xK$xK$xK$xK$xK$xK$xK$xK$xK$xK$xK$vL&vL&vL&vL&vL&uK%uK%uK%vL&vL&vL&vL&uK%vL&uK%uK%uK%tJ$tJ$tJ$sI#tJ$uK%tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$sJ$rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#qH"qH"qH"qH"sI%sI%sI%sI%rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$qG#rH$qH$pG#pG#pG#pG%qH&pG%pG%qG#qG#qG!qG!qG!qG!oEoEpG!pG!pG!nEnEnEnEpG!pG!pG!pG!pG!pG!nEnEpG!nEnEnEnEnEmDmDmDkEkEkEkEkEkEmD mD mD mD mD mD mD mD mD mD mCmClClCmCnD nD lBkBkBlBlBgAjD!fF.9"Z>-iB"e>j@j@i@g>h>h>g=g=i<i<f< g=!g>f=d>d>_=_=[?&[?&Y@$Y@$V?%V?%V?'V?'Z?#X=!\>\>^=^=];^<c=c=d>d>b=b=c<d=d= c<c<c<c<c<c<c<b;b;d:d:c9c9b8c9a:a:_:^9`9a:`9`9`9`9^9^9`9`9`:`:\8\8\8\8\8\8\8[7\7\7[6[6[6[6[7[7[7[7Z5Z5Z5Z5Z5Z5Z5Z5Y4Y4Y4Y4Y4X3X3X3X3X3X3X3X3X3W2W2W2W2W2W2W2W2U0U0U0T/S1S1R0R0R0S1T/T/T/T/Q/Q/Q/Q/P.Q/R-R-R-R-P.P.N/M.M.M.M.M.M.M.M.M.O-N,N,N,N,N,L*N,L-J+J+J+J+J+J+J+I*I*H*H*H)H)H)H)H*G)G) G) G) G) G)G)G(F'F'F'F'F'F'F'F'E&E&E&E&E&D% D% D% D% D% D% D%D%D%D%D%D%B&B&B&@$ @$ @$ @$ @$ @$ @$ @$ ?# nE!nE!nE!nE!pG!pG!pG!pG!pG!pG!qG!qG!qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"rH"rH"rH rH rH rH rH sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!rI#rI#rI#rI#rI#rI#sJ$sJ$sJ"sJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$uK%uK%uK%uK#uK#uK#uK#uK#uK#uK#uK#vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$xN&xN&xN&vL$vL$xN$yO%xN$xN$xN$xN$xN$vL"vL"vL"vL"vL"vL"xN$xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&vL$vL$vL$vL$vL$vL$vL&vL&vL&uK%uK%vL&vL&vL&vL&vL&vL&vL&vL&vL&uK%tJ$sI#tJ$tJ$tJ$uK%uK%uK%uK%uK%uK%uK%uK%uK%uK%tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ$tJ$tJ$tJ$tJ&tJ&tJ&tJ&tJ$sI#sI#sI#sI#sI#sI#sI#sI#sI#sI%sI%sI%sI%sI%sI%rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$qH$qH$rH$rH$rH&rH&qH&qH&tG"tG"qG!qG!qG!qG!qG!rH"qH"pG!pG!nEpG!pG!nEnEnEpG!pG!pG!pG!pG!pG!pG!pG!pG!nEnEnEnEnEmDkEkEkEkEkEkEkEkEkE kE kE kE mD mD mD mD lCmD mD mD mDlClCkBmCmClBlBg@ jC#eF06"W=*iB"g@ j@j@g>g>e?d>d>e?f=f=f=g>e?e?e?e?b>b>b>b>b>b>b>b>a=a=a=a=a<`;`;`;a<a<a=b>d>d>b>b>b>b>d=d=d>c=c<c<c<c<e;d:d;d;d:d:d:d:c9c9a:a:a:a:_:_:_:^9^9^9`:`:`:`:^9\7\7\7\7\7\7\7\7\7[6[6[6\7\8\8[7[7]6]6]6]6Z5Z5Z5Z5Z5Z5Y4Y4Y4X3Y4Y4X3X3X3X3X3X3W2W2W2W2W2W2W2W2U0U0U0U0S1S1S1S1S1S1T/T/T/T/T/T/Q/Q/S.S.S.R-R-R-P.P.N/N/N/N/N/N/M.M.O-O-O-O-O-N,L-L-J+L-L-J+J+J+J+J+J+J+J+J+H*H*H)H)H)H)H*G)G) G) G) G) G)G)G)G)G(G(F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&E&D%D%D%D%D%D%D%D%B# B# B# B# B# B# B# B# nE!nE!nE!nE!pG!pG!pG!pG!pG!pG!qG!qG!qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qH"qG!rH"rH rH rH rH rH sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!tJ"tJ"tJ"tJ"tJ"rI#rI#rI#rI#rI#sJ$sJ$sJ$sJ"sJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ$tJ$tJ$tJ$uK%uK%uK%uK%uK%uK%uK%uK%uK%uK%uK#uK#uK#uK#uK#uK#uK#uK#vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$xN&xN&xN&yO'yO'yO%yO%yO%yO%yO%yO%yO%yO%yO%yO%yO%yO%yO%xN$xN&yO'yO'yO'xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&vL$xN&vL&vL&vL&vL&vL&xN(vL&vL&xN(vL&vL&vL&vL&vL&vL&vL&vL&uK%uK%uK%uK%uK%uK%uK%uK%uK%uK%uK%uK%uK%uK#tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ$tJ$tJ$tJ$tJ&tJ&tJ&tJ&tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI#sI#tJ&tJ&tJ&sI%sI%sI%rH$rH$sI%sI%sI%sI%sI%sI%rH$sI%rI%qH$rH$rH$rH&rH&qH&qH&tG"sF!qG!rH"rH"rH"qG!qG!qH"qH"pG!qH"qH"pG!pGnEnEpG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!pG!nElFlFlFlFlFkEkElFkE kE kE kE mD mD mD mD lCmD mD mD mDlClClCmCmCmCmChA!iB"dE/4& X>+iB"g@ j@j@i@i@gAe?e?gAg>g>g>g>d>d>e?d>b>b>b>b>c?c?c?c?b>b>b>b>a<a<a<a<b=b=b>b>d>d>b>b>b>b>d=d=d>d>c<d=d=c<e;e;e<d;d:d:d:d:c9d:b;a:a:a:_:_:_:_:^9_:a;a;a;`:^9^9^9\7\7\7\7\7\7\7\7\7\7\7\8\8\8\8]6]6]6]6[6[6[6Z5Z5Z5Z5Z5Z5Y4Y4Y4X3X3X3X3Y4Y4X3X3X3X3X3X3X3W2X3X3W2W2U3U3S1S1S1S1T/T/T/U0T/T/R0Q/S.S.T/S.R-S.P.P.N/N/N/N/N/N/N/N/P.P.O-O-O-O-M.L-L-L-L-L-L-L-L-L-L-J+J+J+I+I+I*I*I*H)G)H*H* G) H* H* G)G)G)G)G(G(G(G(F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&E&D%D%D%D%D%D%D%D%B# B# B# B# B# B# B# B# qG#qG#qG#qG#rH"rH"rH"rH"rH"rH"rH"rH"qH"qH"qH"rI#qH"qH"rI#qH"qH"qH"qH"rI#rI#rI#rH"sI#sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!sI!tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ$tJ$tJ$tJ$tJ$tJ$tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"uK#uK#uK#uK#uK#uK#uK#vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$xN&xN&xN&xN&xN&xN&xN&xN$xN$xN$xN$xN$xN$xN$xN$xN$xN$xN$xN$xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&yO'yO'yO%yO%yO%yO%yO%yO%yO%yO%yO%yO%yO%yO%yO%zP&yO'yO'yO'yO'yO'yO'xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&yO'yO'xN&xN&xN&xN&xN&vL$xN&xN&xN&xN&xN&xN&vL$vL$vL&uK%vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&uK%vL&uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#tJ"tJ"tJ"tJ"tJ$tJ$tJ$tJ$tJ&tJ&tJ$tJ$tJ"tJ"tJ"tJ"tJ$tJ$sI#sI#sI#sI#tJ&tJ&tJ&tJ&tJ&tJ&sI%sI%sI%sI%sI%sI%sI%sI%tG$uH%sI%rH$sI%sI%sI%sI%sI%sI%uH#uH#tG"tG"tG"tG"rH"rH"rH rH qH"qH"qH"qH"pG!pG!pG!pG!qG!qG!rH"rH"qG!rH"qG!qG!qG!qG!pG!pG!pG!nElFlFlFlFlFlFlF!lF!lF!lF!lF!lF!nE!nE!nE!nE!lCmD mD mD nDmCoBoBnDnDnDmCgB"hC#bF/0(Y?,jD!hBl?l?i@i@gAgAeAeAgAgAgAe?e?e?f=g>g>g>g>g>e?e?e?e?e?e?e?d>d>d>d>d>d>d>d>e?e?e?c?c?c?c?e?d>d>d>d>d>d>d>f=e<e;e;d:d:e;e;d:d:b;b;a:a:a:a:a:a:a:a:a;a;_;_;^9^9^9^9^9^9^9^9`9`9\7\7\7\7\7\7\7\7[6[6]6]6]6]6[6[6[6[6Z5Z5Z5Y4Y4Y4X3Y4Z5Y4Y4Y4X3X3X3X3X3X3X3X3X3X3W2W2U3U3U3S1S1U3Q2P1S1S1U0T/T/T/T/T/T/T/S.T/Q/Q/O0O0P.P.P.P.P.P.P.P.P.O-O-O-M.M.K-L.L.K-L-L-L-L-L-L-J+J+I+I+I*I*J*J*J*I)H)G(H)H)H)H)H*G)G)G)G)G)G(G(G(F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&E&D% D% D% D% D% D% B# B# B# B# B# B# B# rH$rH$rH$rH$sI#sI#sI#sI#sI#sI#sI#sI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#rI#sI#sI#sI!sI!sI!sI!tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ$tJ$tJ$tJ$tJ$tJ$uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&yO'yO'xN&xN&xN$xN$xN$xN$xN$xN$xN$xN$xN$xN$xN$xN$xN&yO'yO'zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(yO'yO'yO'yO'yO'yO%{Q'yO%yO%zP&zP&zP&zP&zP&zP&zP&zP&zP&zP&zP(zP(zP(yO'yO'yO'xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&yO'yO'xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&vL$uK%vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK%uK%uK%uK%uK'uK'uK%uK%uK#uK#uK#uK#tJ$tJ$tJ$tJ$tJ$tJ$uK'uK'tJ&tJ&tJ&tJ&tJ&tJ&tJ&sI%sI%sI%sI%sI%uH%uH%sI%tJ&sI%sI%sI%sI%sI%sI%uH#uH#uH#uH#uH#uH#rH"rH"sI!rH qH"qH"qH"qH"qH"qH"qH"qH"rH"rH"rH"rH"rH"rH"rH"rH"rH"qG!qH"qH"pG!pG!lFlFlFlFlFlFlF!nH#lF!lF!lF!lF!nE!nE!nE!nE!nE!nE!nE!nE!oEnDpCoBnDnDnDmCgB"iD$aE.-)Z@-jD!iC nAnAjAjAhBgAeAfBhBhBhBhBgAd>g>i@i@i@i@g>gAgAe?e?e?e?e?d>d>d>d>d>e?e?e?e?e?e?c?c?c?c?e?d>d>d>d>d>d>d>f=f=e;e;e;e;e;e;e;e;c<b;b;b;b;a:b;b;a:a:a;a;_;_;^9_:^9^9^9^9^9^9`9`9\7\7\7\7\7\7\7[6\7\7^7^7^7]6[6[6[6[6[6Z5Z5Z5Z5Z5Z5Z5Z5Y4Y4Y4Y4Y4Y4X3X3X3X3X3X3X3W2W2U3U3U3U3U3U3Q2Q2S1S1U0U0T/T/T/T/S.T/U0S.Q/Q/O0O0Q/Q/Q/Q/Q/Q/Q/P.P.P.P.O-M.M.K-L.L.K-L-L-L-L-L-L-L-L-I+I+I*I*J*J*J*J*I*I*H)H)H)H)H*H*H*H*G)G)G(G(G(F'F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&E&E&E&E&E&D% D% D% D% D% D% D% D% rH$rH$rH$rH$sI#sI#sI#sI#sI#sI#sI#sI#rI#rI#rI#rI#rI!rI!sI!sI!sI#sI#sI#tJ$tJ$tJ$sI#tJ$tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"tJ"uK#tJ"uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL"vL"vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$xN&vL$vL$xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&yO'yO'yO'yO'yO'xN&xN&xN$xN$xN$xN$xN$xN$xN$xN$xN$xN$xN$xN$xN&yO'{N'|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O({Q){Q){Q){Q){Q){Q)zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP({Q){Q)zP(zP(zP(yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'xN&yO'yO'yO'xN&xN&yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'xN&vL$xN&xN&xN&vL$xN&xN&vL$vL$vL$xN&xN&xN&xN&xN&vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$uK#uK#uK#uK#uK%uK%uK%uK%uK'uK'uK%uK%uK#uK#uK#uK#uK%uK%uK%uK%uK%uK%tJ&uK'vL(vL(uK'uK'tJ&tJ&tJ&tJ&tJ&tJ&tJ&tJ&vI&vI&tJ&tJ&tJ&sI%sI%sI%sI%sI%uH#uH#uH#uH#uH#uH#uH#uH#sI#sI#rI#rI#rI#rI#qH"qH"qH"qH"sI#sI#rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"qH"pG!pG!pG!pG!pG!lFlFnE!nE!nH#nH#lF!lF!nE!nE!nE!nE!nE!nE!nE!nE!oE!oE!pCpCnDnDnDmCgB"lG'aE0+0^B-kE"iC nAnAlBlBiChBfBfBhBhBhBhBjAi@i@i@i@i@g>g>i@g>g>g>g>i@g>f=f=f=f=f=g>g>e?e?e?e?e?e?e?e?e?e?e?e?e?e?d>d>d>d>f=e<e<e<e<e<e<e<c=c=c=c=c<b;b;b;a;a;a;a;a;a;a:a:`9`9`9`9`9`9`9`9`9`9`9`9^9\7\7[6\7\7^7^7^7^7\7[6[6[6[6[6[6Z5Z5Z5Z5Z5Z5Z5Z5Z5Y4Y4Y4Y4Y4Y4Y4Y4Y4X3W2W2U3V4V4U3U3V4W2U0W2W2U0U0U0U0T/T/T/T/T/T/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/Q/P.P.P.P.N/M.M.M.L-M.L.L.M.M.L-L-L-L-L-L-L,L,L,J*J*J*I*I*I*I*H)H)H)H)H*H*H*H*G(G(G(G(G(G(G(G(F'F'F'F'F'F'F'F'F'F'F'E&E&E&E&E&D% D% D% D% D% D% D% D% sI%sI%sI%sI%tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$sJ$sJ$sJ$sJ$sJ"sJ"tJ"tJ"tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ"uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#vL$vL$uK#uK#uK#uK#uK#vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL"vL"vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$xN&yO'yO'xN&xN&xN&xN&yO'yO'yO'yO'yO'yO'yO'yO'yO'zP(zP(zP(yO'yO'yO%yO%yO%yO%yO%yO%yO%yO%yO%yO%yO%yO%yO'zP(|O(|O(|O(}P)|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O({Q){Q){Q){Q){Q){Q)zP(zP(zP(zP(zP(zP(zP(zP(zP({Q){Q){Q){Q){Q){Q){Q)zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'xN&xN&xN&xN&vL$xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL&vL&vL&vL&vL(vL(vL&vL&vL$vL$vL$vL$uK%uK%uK%uK%uK%uK%uK'uK'vL(vL(uK'uK'uK'uK'uK'tJ&tJ&tJ&tJ&tJ&vI&vI&tJ&tJ&sI%sI%uK'uK'tJ&tJ&vI$vI$uH#uH#vI$vI$uH#vI$tJ$sI#rI#rI#rI#rI#rI#rI#rI#rI#sI#sI#sI#sI#sI#sI#sI#sI#rH"rH"rH"rH"qH"qH"qH"qH"qH"qH"oI"oI"pG#pG#oI$oI$oI$oI$qH$qH$qH$qH$nE!pG#pG#pG#qG#oE!qDqDnDoEoEoEiD$mH(aE0+6aE0lF#iC nAnAlBlBiCiCgCgCiC iC iCiCjAjAjAjAjAjAjAjAi@jAjAjAjAjAi@i@i@i@i@i@i@i@gAgAgAgAgAe?e?e?e?e?e?e?e?e?e?e?e?d>f=f=f=f=f=f=f=f=d>d>c=c=c<c<c<c<b<b<b<a;a;b<b;a:`9`9`9`9`9a:a:a:`9`9`9`9^9^9\7[6\7\7^7^7^7^7\7\7\7\7[6[6[6Z5Z5Z5Z5[6[6Z5Z5Z5Y4Y4Y4Y4Y4Y4Y4Y4Y4X3Y4Y4W5V4V4U3V4V4W2W2W2W2W2U0U0U0U0U0U0U0T/U0R0R0R0R0Q/R0R0Q/Q/Q/Q/Q/Q/P.P.P.N/N/M.N/N/M.L.L.M.M.M.M.L-L-L-L-L,L,L,L,L,J*I*H)I*I*I*I*I*H)H*H*H*H*G(G(G(G(G(G(G(G(G(G(F'F'F'F'F'F'F'F'F'E&E&E&E&E&E&D% D% D% D% D% D% D% sI%sI%sI%sI%tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$uK%tJ$tJ$uK%tJ$tJ$tJ$tJ$uK%uK%uK%uK%uK%uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#vL$uK#uK#uK#uK#vL$vL$uK#vL$vL$vL$vL$vL$vL$uK#vL"vL"xN&xN&xN&xN&xN&xN&xN&xN&yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'zP(zP(zP(zP(zP(zP(zP(zP&zP&zP&zP&zP&zP&|O&|O&|O&|O&|O&|O&|O&}P'}P)}P)}P)}P)}P)}P)}P)}P)}P)}P)}P+}P+}P+}P+}P)}P)}P)}P)}P)~Q*}P)}P)~Q*}P)}P)}P)}P)}P)}P)}P)}P)}P)}P)}P)}P+}P+}P+}P+|O*|O*|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(zP(zP(zP(zP(yO'zP(yO'yO'yO'yO'{N'{N'|O(|O({N'{N'yO'xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&vL$vL$vL$vL&xN(vL&vL&vL&vL&zM(xK&xK$xK$xK$xK$xK&xK&vL&vL&vL&vL&vL(vL(vL(vL(vL(uK'uK'uK'uK'uK'uK'uK'wJ%wJ%wJ%wJ%tJ&tJ&uK'uK'uK'uK'uK'uK'wJ%wJ%vI$vI$vI&vI&vI&vI&tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI%sI%sI%sI%sI#sI#sI%sI%sI#sI#sI#sI#sI#rH"qH"qH"qH"qH"qH"qH"qH"qH"qH$qH$qH$qH$qH$qH$qH$qH$qH$qH$pG#qH$qH$qH$oE!qG#qG!oEoEqGnE#nE#iD'oJ-bF3*: cI6nG'iB"oBnAmCmCjDjDjDjDjDiCkBkBkBjAjAjAjAjAjAjAkBkBjAjAjAjAi@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@i@g>g>g>e?e?e?e?e?e?e?d>d>d>d>d>d>d>d>d>d>d>d>d>c=c=c=c=c=c=b<b<b<a;a;a;a;b<b<a;a:a:`9`9`9`9`:`:^8`:\7^9`9`9`9^7\7\7\7\7\7\7[6[6[6[6[6[6[6Z5\5\5\5\5\5\5Z5Z5Z5Z5Y4Y4Y4Y4W5W5W5V4V4V4W2W2W2X3W2W2W2U0U0U0U0U0U0U0S1R0R0S1S1R0R0R0R0R0Q/Q/Q/Q/Q/P.P.P.N/N/N/N/N.M-M-M-M-M-M-M-M-M-L,L,L,L,L,L,I*I*I*I*I*I*I*H)H*H*H*H*H)H)H)G(G(G(G(G(G(G(G(F'F'F'F'F'F'F'F'F'F'F'F'F'E&E&E&E&E&E&E&E&sI%tJ&tJ&tJ&tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$uK%tJ$tJ$uK%tJ$uK%uK%uK%uK%uK%uK%uK%uK%uK#uK#uK#uK#uK#uK#uK#vL$uK#uK#uK#vL$vL$vL$vL$xN&xN&xN&xN&xN&vL$vL$vL$xN&xN&xN&xN&xN&xN$xN$xN&xN&yO'yO'yO'yO'xN&xN&yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP&zP&zP&zP&zP&zP&|O&|O&|O&|O&|O&|O&|O&}P'}P)}P)}P)}P)}P)}P)}P)}P)}P)}P)}P+}P+}P+}P+}P)}P)}P)}P)~Q*~Q*~Q*~Q*~Q*}P)}P)}P)}P)}P)}P)}P)}P)}P)~Q*~Q*~Q,~Q,}P+}P+}P+}P+}P)}P)}P)}P)}P)}P)}P)}P)}P)}P){Q){Q){Q){Q){Q)zP(zP(zP(zP(zP(|O(|O(|O(|O({N'{N'yO'yO'yO'zP(yO'yO'yO'yO'yO'yO'yO'yO'yO'xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN(xN(xN(xN(xN(xN(zM(zM(zM&zM&zM&zM&xK&xK&vL&vL&vL&vL&vL(vL(vL(vL(vL(vL(vL(vL(vL(vL(uK'uK'wJ%wJ%wJ%wJ%uK'uK'uK'uK'uK'uK'uK'uK'wJ%wJ%wJ%wJ%wJ'vI&wJ'vI&tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI%sI%sI%sI%sI#sI#sI%sI%sI#sI#sI#sI#sI#sI#qH"qH"qH"qH"qH"qH"qH"qH"qH$qH$qH$qH$qH$qH$qH$qH$qH$qH$qH$qH$qH$qH$rH$qG#qG!qG!rH qGnE#mD"hC&nI,`D1( =#eK8nG'jC#oBoBmCmCjDjDjDjDjDjDlClCkBkBkBkBkBkBkBkBkBkBjAjAjAjAjAjAjAjAi@i@i@i@i@i@jAjAjAjAjAjAjAi@i@i@e?e?e?e?e?e?e?e?d>d>d>d>d>d>d>d>d>d>d>d>c=c=c=c=c=c=c=c=c=c=c=c=b<b<b<a;a:a:a:a:a:a:a;a;a;a;_:^9`9`9`9^7\7\7\7\7\7\7\7[6[6[6[6[6[6[6\5\5\5\5\5\5Z5Z5Z5Z5Z5Y4Y4Y4W5W5W5V4V4W5X3X3X3X3W2W2W2W2U0U0W2U0U0U0S1R0R0S1S1R0R0R0R0R0R0R0Q/Q/Q/Q/Q/P.N/N/N/N/N.N.N.N.M-M-M-M-M-M-M-M-M-M-L,L,K,I*K,K,I*I*I*I*I+I+I+H*H)H)H)H)H)H)H)H)G(G(G(G(G(G(G(G(G(G(G(F'F'F'F'F'E&E&E&E&E&E&E&E&tJ$uK%uK%uK%uK%uK%uK#uK#uK#uK#uK#uK#uK#uK#uK#uK#uK%uK%uK%uK%uK%uK%uK%uK%uK#uK#uK#uK#vL$vL$vL$vL$vL&vL&vL&vL&vL&vL&vL&vL&vL"xN$xN$xN$xN$xN$xN$xN$xN$xN$xN$xN$xN&xN&yO'yO'yO'yO'yO'yO'yO'yO'yO%yO%zM$zM${N%{N%{N%{N%{N%{N%{N%{N%{N%|O&{N%{N%{N%{N%|O({N'{N'|O(|O(|O(|O(|O(|O*|O*|O*|O*|O*}P+}P+}P+}P)}P)}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q*~Q*~Q*~Q*~Q*~Q*}P)}P)~Q(~Q(~Q(~Q(~Q(}P'}P'}P'}P'}P'}P'}P'}P)}P)~Q*~Q*~Q*~Q*}P)}P)}P)}P)}P'}P'}P'}P'}P'}P'}P'}P'}P)}P)}P)}P){Q'{Q'{Q'zP&zP&zP&|O(|O(|O(|O(|O(|O(zP(zP(zP(zP(zP(zP(|O*|O*|O*|O*|O*|O*|O*|O*|O*{N){N'{N'{N'{N'{N'{N'{N'{N'yO'yO'xN&xN&xN&xN&yO'yO'zM&zM&zM(zM(zM&zM&xN&xN&xN&xN&xN&xN&xN&xN&vL$vL$xN(xN(xN(xN(xN(xN(xN(xN(xN(vL&vL&vL&xK&xK&xK&xK&uK%uK%vL&vL&uK%uK%uK%uK%uK%uK%uK%uK%uK%uK%uK%uK%uK#uK#uK#tJ"tJ"tJ"tJ$tJ$tJ$tJ$tJ$tJ$tJ&tJ&tJ&sI%rI#sJ$tJ&sI%qH$rI%sI%sI%sI%sI%rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH"qG!rH$rH$qH qH vO/zS3oO=eE3P>32 S8-kPEjD!jD!lBmCmCmClClClClClClClClCkBlClClClCkBkBlClClCkBkBjAkBkBkBkB kB jAi@i@i@i@jAjAjAjAjAjAjAi@i@i@i@i@i@g>g>g>g>e?e?e?e?e?e?e?e?e?e?e?e?e?d>d>d>d>d>d>d>c=c=c=c=c=c=b<c=c=b<b;b;_:_:a:b;`<`<`<_;_;_;_;_;`9`9`9`9`9`9`9`9^7^7^7^7^7^7^7]6]6]6]6]6\5\5Z5Z5Z5Z5Z5Y4Z5Y4W5W5W5V4V4W5W5V4Y4W2W2W2W2W2W2W2U3U3U3S1S1S1S1S1R0S1S1R0T/T/T/T/R0Q/Q/Q/Q/Q/O0N/O0N/N/N/N/N/N/N/N/M.M.M.M.M.M.M.M.L-L,L,L,L,J*J*J+J+J+J+J+J+H*H*H)H)H)H)H)H)H)H)G(G(G(G(G(G(G)G)H(G'G'G'G'G'G'F&F&F&F&F&F' F' uK%uK%uK%uK%uK%uK%uK#vL$vL$vL$vL$vL$vL$vL$vL$vL$uK%uK%uK%uK%uK%uK%uK%vL&vL$vL$vL$vL$vL$vL$vL$vL$vL&vL&vL&xN(xN(vL&vL&vL&xN$xN$xN$xN$xN$xN$yO%yO%xN$xN$xN$xN$xN&xN&yO'yO'yO'yO'yO'yO'yO'yO'yO%yO%|O&|O&{N%{N%{N%{N%{N%{N%{N%{N%{N%|O&}P'}P'}P'}P'|O(|O(|O(|O(|O(|O(|O(}P)}P+}P+}P+}P+}P+}P+}P+}P+}P)}P)}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*}P)~Q(~Q(~Q(}P'}P'}P'}P'~Q(}P)}P)}P)}P){Q'{Q'{Q'zP&zP&zP&|O(|O(|O(|O(}P)}P){Q)zP(zP(zP(zP(zP(|O*|O*|O*|O*|O*|O*|O*|O*|O*|O*{N'{N'{N'{N'{N'{N'{N'{N'yO'yO'yO'yO'yO'yO'yO'yO'{N'{N'zM(zM(zM&zM&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN(xN(xN(xN(xN(xN(xN(xN(xN(xN(vL&vL&xK&xK&xK&zM(vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&uK%vL&vL&vL&vL&vL&vL$uK#uK#tJ"uK#uK#tJ$tJ$tJ$tJ$tJ$tJ$tJ&tJ&uK'uK'tK%tK%tJ&sI%rI%rI%sI%tJ&sI%sI%rH$rH$rH$sI%sI%rH$rH$rH$rH$sI%rH$rH$sI%sI%rH$rH$sI%rH$rH"rH"sI%sI%sJ"tK#lE%[4; & 3 _D9oI&jD!nDmCmCmCmD mD mD lCmD mD mD mD lCmD mD mD mD lClClClClCkBkBkBkBkBkBkB kB kB kB jAjAjAjAjAjAi@i@i@i@i@i@i@i@i@i@i@i@g>g>e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?e?d>d>d>d>c=d>c=c=c=c=c=c=c=b<b;b;`;`;b;b;`<`<`<`<`<_;_;_;a:`9`9`9`9`9`9`9`9^7^7^7^7^7^7^7]6]6]6]6]6]6Z5Z5[6[6Z5Z5Z5Z5W5X6X6W5W5W5W5W5X3X3X3W2W2W2W2W2U3U3U3U3S1S1U3U3U3S1S1S1U0T/T/T/R0R0R0Q/Q/Q/O0N/O0O0O0O0N/N/N/N/N/N/N/N/M.M.M.M.M.M.L,L,L,L,L,L,L-J+J+J+J+J+I+I+I*H)H)H)H)H)H)H)H)G(G(G(G(G(G)G)H(G'G'G'G'G'G'G'G'G'F&F&F' F' uK#uK#uK#uK#uK#uK#uK#vL$vL$vL$vL"vL"vL"vL"vL$vL$vL&vL&vL&vL&vL&vL&vL&vL&vL$vL$vL$vL$vL$vL$xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xO"xO"xO"yP#yP#yP#yP#yP#yO%yO%yO%yO%xN$xN$yO'yO'yO'yO'zP(zP(zP(zP(|O(|O(|O&}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'|O(|O(|O(|O(}P)}P)}P)}P)}P)}P)}P)}P)}P)~Q*~Q*~Q*~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q*~Q*~Q*R+~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q*~Q*~Q*~Q*|R({Q'{Q'{Q'{Q'{Q'}P'}P'}P)}P)}P)}P){Q){Q){Q){Q){Q){Q)}P+}P+}P+}P+}P+}P+}P+}P+|O*|O*|O(|O(|O(|O(|O({N'|O(|O(|O(|O({N'{N'{N'{N'{N'|O(|O({N'{N'{N'{N'{N'yO'yO'yO'yO'yO'xN&xN&xN&xN&xN&yO)yO)yO)yO)yO)yO)yO)yO)yO)yO){N)zM(zM(zM(zM(zM(xN&vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL&vL&vL&vL&uK%uK%tJ$uK%tK%tK%tK%sJ$tJ&tJ&uK%uK%uK%uK%tJ&tJ&tJ&uK'uK'uK'tJ&sI%tJ&tJ&tJ&sI%sI%sI%rH$sI%sI%sI%rH$rH$sI%sI%rH$rH$sI%sI%sI%sI%rI#sJ$uO$jDC$  %rJ:uM=uG,qC(nD nD nE#nE#mD"mD"mD mD mD"mD"mD"mD"mD mD mD mD lClClClClClClClClCkBkB kB kB kB kB kB kB kB kB jAjAjAjAjAjAjAjAjAjAjAjAjAi@i@gAgAgAgAgAgAgAgAgAgAe?e?e?e?d>e?e?e?e?e?d>d>d>d>d>d>d>d>c=c=c=b<b<b<b<b<`<`<`<`<`;`;`;`;a:`9a:`9`9`9`9`9`9^7^7`9`9`9^7^7]6]6]6]6]6]6]6]6]6]6[6Z5Z5Z5Z5Z5X6X6W5W5W5W5Y4Y4X3X3X3X3X3W2W2U0W2W2W2W2W2W2W2W2S1S1S1S1S1S1R0R0R0R0R0Q/Q/Q/Q/Q/O0O0O0O0N/N/N/N/N/N/N/N/M.M.M.M.K-L.L.L.K-K-L-L-L-L-L-J+J+J+J+J+I+I+H)H)I)I)I)I)H(H(H(H(H)H)H)H)G)G)G)F( F( F( F( F( F'E&E&E&vL$vL$vL$vL$vL$vL$vL$vL$vL$vL$vL"vL"vL"vL"vL$vL$vL&vL&vL&vL&vL&vL&vL&xN(xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&yP#yP#yP#yP#yP#yP#zQ$zQ$yO%yO%yO%yO%zP&zP&zP(zP(zP(zP(zP(zP(zP(zP(|O(|O(}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P)}P)}P)}P)}P)}P)~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q(~Q(~Q(R)R)R)R)R)R)R)R)R)R)R)R)R)R+R+R+R+R+R+R+R+R+R+R+R+R+R+R+R+~Q(~Q(R)R)R)R)R)~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(R+R+R+R+R+R+~Q*~Q*~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q*~Q*~Q*~Q*|R(|R(|R({Q'{Q'{Q'}P'}P'}P)}P)}P)}P){Q){Q){Q){Q){Q){Q)}P+}P+}P+}P+}P+}P+}P+}P+}P+}P+}P)|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(|O(zP(yO'yO'yO'yO'yO'yO'yO'yO'yO'yO)yO)yO)yO)yO)yO)yO)yO)yO)yO){N){N){N){N){N){N)xN&xN&xN&xN&xN&xN&vL$xN&xN&xN&xN&xN&xN&xN&xN&xN&xN(xN(xN(vL&vL&vL&vL&vL&uL&uL&uL&uL&uK'uK'uK%vL&vL&uK%vL(uK'uK'uK'uK'uK'uK'uK'tJ&tJ&tJ&tJ&sI%tJ&tJ&tJ&tJ&sI%sI%sI%sI%sI%rH$rH$sI%sI%sI%tJ&tK%tK%uO$nHP18(  I!sK;vH-uG,rH$nD nE#nE#nE#nE#nE!nE!mD"nE#nE#nE#nE!nE!nE!nE!mD mD lClClClClClClCkBlC!lC!lC!lC!lC!lC!kB lC!kB jAjAjAjAjAjAjAjAjAjAjAjAi@jAjAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAgAe?e?e?e?e?e?e?d>d>d>d>d>d>c=c=c=c=c=b<b<`<a=a=`<`;`;`;`;a:b;a:a:a:a:a:a:a:`9`9`9`9`9`9`9^7]6^7^7]6]6]6]6]6]6[6[6Z5[6Z5Z5X6X6W5W5X6X6Y4Y4Y4Y4X3X3X3W2W2W2W2W2W2W2W2X3W2W2U3S1S1S1S1S1S1R0R0R0R0R0R0R0R0Q/O0O0O0O0O0N/N/N/N/N/N/N/M.M.M.M.M/L.L.L.K-K-L-L-L-L-L-L-J+J+J+J+I+I+I*H)I)I)I)I)I)I)H(H(H)H)G( H)G)G)G)G)G)G)F( F( F'E&E&E&vL$vL$vL$vL$vL$vL$vL$xN&xN&xN&xN$xN$xN$xN$xN&xN&xN(xN(xN(xN(xN(xN(xN(xN(xN&xN&xN&xN&xN&xN&yO'yO'yO'yO'yO'yO'yO'yO'xN&yO'yP#yP#yP#zQ$zQ$zQ$zQ$zQ$zP&zP&yO%zP&zP&zP&zP&zP&zP&zP&zP&zP&{Q'{Q'}P)}P)}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'~Q(}P'}P'}P'}P'}P)}P)~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*R+R+R+R)R)R)R)R)R)R)R)R)R)R)R)R)R)R)R)R+R+R+R+R+R+R+R+R+R+R+R+R+R+R+R+R)R)R)R)R)R)R)R)R)R)R)R)R)R)R)R)R+R+R+R+R+R+R+R+R)R)R)~Q(~Q(~Q(~Q(R)R+R+R+R+|R(|R({Q'|R(|R(|R(~Q(~Q(}P)~Q*~Q*~Q*|R*{Q){Q){Q){Q){Q)}P)~Q*~Q*~Q*~Q*~Q*~Q*}P)}P)}P)}P)}P)}P)}P)|O(}P)}P)}P)}P)}P)}P)}P)}P)|O(|O(|O(|O(|O(|O(|O(|O(|O(zP(yO'yO'yO'yO'yO'yO'zP(zP(yO'yO)yO)yO)zP*zP*yO)yO)yO)yO)yO){N){N){N){N){N){N)yO'yO'yO'yO'xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN&xN(xN(xN(vL&vL&uK%vL&vL&vL&vL&vL&vL&vL&vL&xN(xN(vL&vL&vL(vL(vL(vL(uK'uK'uK'uK'uK'tJ&tJ&uK'uK'uK'tJ&tJ&tJ&tJ&tJ&tJ&sI%sI%sI%sI%tJ&tJ&sI%tJ&tJ&uK'tK#qH \7L' -;45& ;"]D&qG%qG%nE#nE#nE#nE#nE#nE#nE#nE#nE!nE!nE!nE!nE!nE!nE!mD mD mD mD mD mD mD mD lClC!lC!lC!lC!lC!lC!lC!lC!lC!kB kB kB kB kB kB kB kB kB jAjAjAjAjAjAhBhBgAhBhBhBhBhBgAgAgAgAgAgAgAgAe?e?e?e?e?e?e?e?e?e?d>d>d=c<e<f=c=c=c=d>a=a=a=`<b<b<b;b;b;b;a:a:a:a:a:a:a:a:a:`9`9`9`9`9`9`9`9`9^7^7^7]6]6^7]6\5[6[6[6Z5X6X6X6X6Z5Z5Z5Z5\5[4[4[4X3Y4X3X3X3X3Z3Z3X3X3X3W2U3U3U3S1S1S1S1S1S1R0R0R0R0R0R0R0P1P1Q/Q/Q/P.N/O0O0O0N/N/M.N/N/N/N/M.M.M.M.M.M.L-L-L-N,N,N,N,L-L-I+I+I*I*J*J*J*I)I)I)I)I)I*I*I*H)G)G)G)G)G)G)F( F( F'F'E&E&vL$vL$vL$vL$vL$vL$vL$xN&xN&xN&xN$xN$xN$xN$xN&xN&xN(xN(xN(xN(xN(xN(xN(xN(xN&xN&xN&xN&xN&xN&yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yP#zQ$zQ$zQ$zQ$zQ$zQ$zQ$zP&zP&zP&zP&zP&zP&zP&{Q'{Q'{Q'{Q'{Q'{Q'{Q'}P)}P)}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'}P'~Q(R)R)R)R)~Q*}P)~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*R+R+R+R+R)R)R)R)R)R)R)R)R)R)R)R)R)R)R)R)R+R+R+R+R+R+R+R+R+R+R+R+R+R+R+R+R)R)€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*R+R+R+R+R+R+R+R+R)R)R)~Q(~Q(~Q(R)R)R+R+R+R+}S)}S)}S)}S)}S)}S)~Q(R)R+~Q*~Q*~Q*|R*|R*|R*|R*|R*|R*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*}P)}P)}P)}P)|O(}P)}P)}P)}P)}P)}P)}P)}P)}P)}P)}P)}P)}P)}P)}P)|O(|O(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP*zP*zP*zP*zP*zP*zP*zP*zP*zP*|O*|O*|O*|O*|O*|O*yO'yO'yO'yO'yO'yO'yO'yO'xN&xN&xN&xN&xN&yO'yO'yO'yO)yO)xN(xN(xN(vL&vL&xN(xN(xN(xN(vL&vL&vL&vL&xN(vL&vL&vL(vL(vL(vL(vL(vL(uK'uK'uK'uK'uK'uK'uK'uK'uK'tJ&uK'uK'uK'uK'uK'uK'uK'uK'uK'tJ&tJ&sI%sI%tJ&sJ"qH W2M( =V4‚hO|bIgR>=( - 0^E'uK)oE#pG%pG%pG%pG%nE#nE#nE#nE#nE!nE!nE!nE!nE!nE!nE!nE!nE!mD mD mD mD mD mD mD mD"mD"lC!lC!lC!lC!lC!lC!lC!lC!lC!lC!kB kB kB kB kB kB kB kB kB kB kB kB hBiChBhBhBhBhBhBgAgAgAgAhBhBgAgAgAgAgAgAgAgAgAe?e?e?e?e?e>e>g>f=c=c=d>d>b>b>b>b>b<c=d=c<b;c<b;b;b;a:a:a:a:a:a:a:a:a:a:`9`9`9`9`9^7^7^7^7^7^7^7^7[6[6[6[6X6Y7X6X6Z5Z5[6[6\5\5\5[4Y4Y4Y4Y4Y4X3Z3Z3X3X3X3X3V4U3U3U3U3U3S1S1S1S1S1S1S1S1R0R0P1P1R0R0Q/Q/O0O0O0O0O0O0O0N/N/N/N/N/N/N/M.M.M.M.M.M.N,N,N,N,L-L-I+I+I*I*J*J*J*J*I)I)I)I)I*I*I*H)G)G)G)G)G)G)G)G)G(F'F'F'xN&xN&xN&xN&xN&xN&xN&xN&yO'yO'yO%yO%yO%yO%yO'yO'yO)yO)yO)yO)yO)yO)yO)yO)yO'yO'yO'yO'yO'yO'yO'yO'yO'zP(zP(zP(yO'yO'yO'zP(zQ$zQ$zQ$zQ$zQ${R%{R%{R%{Q'{Q'zP&{Q'{Q'{Q'zP&{Q'{Q'{Q'{Q'{Q'{Q'{Q'}P)}P)R)R)R)R)R)R)R)R)R)R)R)~Q(R)R)R)R)R+~Q*~Q*R+R+R+R+R+R+R+R+R+R+R+€S,€S,€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S,€S,R)R)€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S,€S,€S,€S,€S,R+€S*€S*€S*€S*€S*€S*R)€S*R)R)R)R)R)R)R)R)R)R)R)R)R+R+R+R+}S+}S+}S+}S+}S+}S+R-R-R-R-R-R-R-~Q,~Q,~Q,~Q*~Q*~Q*~Q*~Q*}P)~Q*~Q*~Q*~Q*~Q*~Q*}P)}P)}P)}P)}P)}P)}P)}P)}P)}P){Q){Q){Q){Q){Q)zP(zP({Q){Q)zP(zP*zP*zP*zP*zP*zP*zP*zP*zP*zP*zP*zP*zP*zP*zP*zP*zP(yO'zP(zP(yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'{N'{N'{N'{N'zM(zM(vL&xN(xN(xN(xN(xN(xN(xN(xN(vL&xK&zM(xN(vL&vL&vL&vL&vL&vL&vL&vL&vL&uK%uK%uK%uK%vL&vL&uK%uK%uK%uK%vL&uK%vL&vL&uK%uK%uK%uK%tJ$tJ$tJ$tJ$uLsJa4U(T'xK&b4¨zL²„T¡sC€W1`7;% 0 3 C%dF!pG%pG%pG#pG#pG#pG#pG%nE#pG#pG#qG#qG#qG#qG#qG#qG#oE!oE!nE!nE!oE!oE!oE!oE!oE!nD mD mD mD mD mD mD mD mD lC!lC!lC!lC!lC!lC!lC!lC!lC!lC!kB kB kB kB kB kB kBkBkBkBkBjAjAkBkBkBjAjAjAjAjAjAgAgAgAgAgAgAgAgAe?e?e?e?e>e>e>e>g>g>f=f=d>d>d>d>d>d>d=c<c<c<b;b;b;b;b;b;b;b;b;a:a:a:a:a:a:a:a:a:`9`9^7^7^7^7^7^7^7]6\7[6[6[6[6Z5Z5[6[6Z5\5\5\5\5Z5Y4Y4Y4[4[4Z3Z3X3Y4X3X3V4V4U3U3U3U3U3S1S1S1S1S1S1S1S1R0P1P1R0R0R0R0P1P1O0O0O0O0O0N/N/O0N/N/N/N/N/M.M.M.M.M.O-O-N,N,L-L-K-K-I*I*J*J*J*J*J*J*H*H*I*I*I*I*H*H*H*H*H*G)G)G)G)F( F'F'xN&xN&xN&xN&xN&xN&xN&yO'yO'yO'yO%yO%yO%yO%yO'yO'yO)yO)yO)yO)yO)yO)yO)zP*zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(zQ$zQ${R%{R%{R%{R%{R%{R%{Q'{Q'{Q'{Q'{Q'{Q'|R(|R(|R({Q'{Q'{Q'}S)}S)R+R+R)R)R)R)R)R)R)R)R)R)R)R)R)R)R)R)R+R+R+R+R+R+R+R+R+€S,€S,€S,€S,€S,€S,€S,€S*€S*€S*€S*€S*€S*€S*‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U.‚U.€S*‚U,‚U,€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*‚U,‚U,€S,€S,€S,€S,€S,€S,€S*€S*€S*€S*€S*€S*R)€S*€S*€S*€S*€S*€S*R)R)R)R)R)R)R)R+R+R+R+}S+}S+}S+}S+}S+}S+R-R-R-R-R-R-R-R-~Q,~Q,~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*}P)~Q*~Q*~Q*~Q*~Q*~Q*}P)}P){Q){Q){Q){Q){Q){Q){Q){Q){Q){Q)zP*{Q+zP*{Q+{Q+{Q+{Q+{Q+{Q+{Q+zP*{Q+{Q+zP*{Q+{Q+zP(zP(zP(zP(zP(zP(zP(zP(yO'yO'yO'yO'yO'yO'{N'{N'{N'{N'{N){N)yO)yO)yO)xN(yO)yO)xN(xN(xN(vL&zM(zM(xN(vL&xN(xN(xN(vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&uK%vL&uK%vL&vL&vL&vL&vL&uK%vL&uK%uK%uK%tJ$tJ$uK%vMtKm@[. ^1 wJ%„V(b4”f6™k;f@V0oO7T41       -(-#8 7H*hJ%qH&rI'rI%qH$qH$qH$pG%pG%pG#pG#qG#qG#qG#qG#qG#qG#qG#qG#nE!nE!oE!oE!oE!oE!oE!nD mD mD mD mD mD mD mD mD mD"lC!lC!lC!lC!lC!lC!lC!lC!lC!kB kB lC!lC!lC!lC!kBkBkBkBkBkBkBkBkBkBkBkBkBjAjAjAgAgAgAhBhBgAgAgAgAgAgAgAe>e>e>e>g>g>g>g>e?e?d>d>d>d>d=d=d=c<c<b;b;b;b;b;b;b;b;b;a:a:b;a:a:a:a:a:`9`9`9`9^7`9^7^7^7^7\7\7[6\7[6[6[6[6[6[6]6]6\5\5Z5Z5Z5Z5[4[4[4[4Y4Y4Y4Y4V4V4V4V4U3U3U3U3U3U3S1S1S1S1S1S1Q2P1R0R0R0R0P1P1O0P1O0O0O0O0O0O0O0O0O0N/N/N/N/N/N/M.O-O-N,O-M.L-K-K-K,K,L,L,J*J*J*J*I+H*J+J+H)I*H*H*H*H*H*G)G)G)G)G)F'F'xN&xN&xN&xN&xN&xN&xN&xN&yO'yO'{N%|O&|O&|O&|O&|O&|O(|O(|O(|O(|O(|O(|O(|O(|O&|O&|O&|O&|O&|O&|O&|O&|O(|O(}P)}P)}P)}P)}P)}P)}Q%}Q%}Q%}Q%}Q%}Q%~R&~R&}P'}P'}P'}P'~Q(~Q(~Q(~Q(~Q(R)R)R)R)R)R)R)S'S'S'S'S'S'S'S'S'S'S'S'S'S'R)R)S'S'S'€T(€T(€T(€T(€T(€T(€T(€T(€T(€T(€T(€T(€T(‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚V*‚V*‚V*‚V*‚V*€T(€T(€T(€T(€T(€T(€T(€T(€T(‚U,‚U,‚U,‚U,‚U,‚U,‚U,€S*€S*€S*€S*€S*€S*€S*€S*€S*€T(€T(€T(€T(S'S'€T(€T(€T(€T(€T(€T(€T(€T(€T(€T(R)R)R)R)€S*€S*€S*R)R)R)R)R)€S*R)R+R+R+~Q*~Q*~Q*R+R+R+R+R+R+~Q*~Q*~Q*}P)~Q(~Q(~Q(~Q(~Q*~Q*}P)}P)}P)}P)}P)}P)}P+}P+}P+~Q,~Q*}P){Q){Q){Q){Q){Q){Q){Q){Q){Q){Q){Q){Q){Q){Q)}P)}P){Q)zP({Q){Q)zP(zP(zP(zP(zP(yO'zP(zP(|O(|O(|O(|O(|O(|O(|O({N'{N){N)yO)xN(yO)yO)xN(xN(zM({N)zM({N){N)zM(zM(zM(zM(zM(yO)xN(vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&uK%uK%uK%vL"uK!wJ!wJ!{Q'uK!kAqG!‰_9™oI™lE”g@c:‡Z1sFg:V,EADC H%I#G! G E @<4<e<pG#rInEsI%rH$rH$rH$qG#qG#qG#qG#qG#qG#qG!qG!qG!qG!qG#qG#qG#qG#qG#qG#oE!oE!oE!oE!nE!nE!nE!nE!nE!nE!nE!nE!mD mD mD mD mD lClClClClClClClClClClClClClClClClCkBkBkBkBkBkBkBkBkBkBjAjAjAjAjAjAjAi@i@i@i@i@i@i@gAe?e?e?e?e?e?e?e?e?d>e?e>e>d=d=c<c<c<b;b;b;b;b;b;b;b;b;b;b;c9c9d:c9c9b8`9`9`9`9`9^7`9^7\7\7\7\7\7[6[6\7\7[6]6]6]6]6]6\5\5\5\5\5[4[4[4[4Y4Y4W5V4V4V4V4V4U3U3U3U3U3U3U3U3S1S1Q2Q2S1S1R0R0P1P1P1P1P1P1O0O0O0O0O0O0O0N/N/N/N/N/N/N/O-O-O-O-M.M.K-K-K,K,L,L,L,L,J+J+J+J+J+J+J+I*H*H*H*H*H*H*G)G)G)G)G)G)yO'yO'yO'yO'yO'yO'yO'yO'zP(zP(|O&|O&|O&|O&|O&|O&|O(|O(|O(|O(|O(|O(|O(|O(|O&|O&|O&|O&|O&|O&|O&|O&}P)}P)}P)}P)}P)}P)}P)}P)}Q%}Q%}Q%~R&~R&~R&~R&~R&~Q(~Q(}P'~Q(~Q(~Q(~Q(R)R)R)R)R)R)R)R)R)S'S'S'S'S'S'S'S'S'S'S'€T(€T(€T(€S*€S*€T(S'S'€T(€T(€T(€T(€T(€T(€T(€T(€T(€T(€T(€T(€T(‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚V*‚V*‚V*‚V*‚V*€T(€T(€T(€T(€T(€T(€T(€T(‚V*‚U,‚U,‚U,‚U,‚U,‚U,‚U,€S*€S*€S*€S*€S*€S*€S*€S*€S*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*€T(€T(€T(€T(€T(€T(R)€S*€S*‚U,€S*€S*€S*€S*€S*€S*€S*€S*€S*R)R+R+R+~Q*~Q*R+R+R+R+R+R+R+R+R+R+R+R)R)R)R)R+~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q,~Q,~Q,~Q,~Q*~Q*|R*{Q){Q){Q)|R*|R*{Q){Q){Q){Q){Q){Q)zP({Q)}P)~Q*{Q)zP({Q){Q){Q){Q){Q){Q)zP(zP(zP(zP(|O(|O(|O(|O(|O(|O({N'|O(|O*|O*zP*zP*yO)yO)yO)xN(|O*|O*|O*|O*{N){N)zM(zM(zM({N)yO)xN(xN(xN(vL&vL&vL&vL&xN(xN(vL&vL&vL&vL&xN(vL&vL&xN(vL&vL&vL&vL&vL&vL&vL"uK!xK"zM$zP&vL"j@c9h>{Q+’e>šmF›nE˜kB‘d=…X1tJ&]3DBA E"I#L&P(R*P+N)W2^9nE!pG#rIrIrH$rH$rH$rH$rH$rH$rH$rH$rH$rH$rH"rH"rH"rH"rH$rH$qG#qG#qG#qG#qG#qG#qG#qG#pG#pG#nE!nE!nE!nE!nE!nE!nE!mD mD mD mD mD mD mD mD mD mD mD mD lClClClClClClClClClClClClClClClCkBkBkBkBkBkBkBjAjAjAjAjAjAjAi@i@i@gAgAgAgAe?e?e?d>e?e?e?e?e>d=d=d=c<c<c<c<c<c<c<c<c<c<b;b;b;b;d:d:d:d:c9c9a:`9a:`9a:`9`9`9^9\7\7\7\7\7\7\7\7\7]6]6]6]6]6]6\5\5\5\5[4\5\5\5Z5Y4W5W5W5V4V4V4V4V4V4U3U3U3U3U3U3S1Q2Q2S1S1S1S1P1P1P1P1P1P1P1O0P1O0O0O0O0O0N/N/N/N/N/N/P.P.O-O-M.M.L.L.K,K,L,L,L,L,L-J+J+J+J+J+J+I*I+I+H*H*H*H*H*H*H*G)G)G)yO'yO'yO'yO'yO'yO'yO'zP(|O&|O&|O&|O&|O&|O&|O&|O&|O(|O(|O(}P)}P)}P)}P)}P)}P'}P'}P'}P'}P'}P'}P'}P'}P)}P)}P)}P)~Q*~Q*~Q*~Q*}Q%}Q%~R&~R&~R&~R&~R&~R&~Q(~Q(}P'~Q(R)R)R)R)R)R)R)R)R)R)R)R)S'S'S'S'S'S'S'S'S'S'S'€T(€T(€T(€S*€S*€T(€T(€T(€T(€T(€T(‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚U,ƒV-…V/„U.„U.„U.„U.„U.‚V*‚V*‚V*‚V*‚V*‚V*€T(€T(‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*€S*‚U,‚U,‚U,€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*R)€S*€S*€S*€S*€S*€S*R+R+R)R)R)R)R+R+R+R+R+R+R+R+~Q*~Q*~Q*~Q*~Q*~Q*|R*|R*|R*}S+}S+|R*|R*|R*|R*|R*|R*|R*~Q*}P)~Q*~Q*~Q*}P)~Q*~Q*}P)}P)}P)|O({Q'{Q'{Q'{Q'}P'}P'|O&|O&|O(|O(|O(|O(}P)}P)zP(zP(zP(zP(zP(zP(|O(|O(|O(|O({N)|O*|O*{N){N)|O*{N){N)yO)yO)xN(xN(xN(xN(xN(xN(vL&xN(vL&xN(xN(xN(zM(zM(zM(xK&vL&vL&vL&vL&vL&vL&xN&xN&yO)vL&g=U+ LM N!U(a4h;nBvJ~O&vGi9e5a6e: ]3Y/W,U*S-e?+wS.sO*tK!qHrI'tK)tJ&sI%sI%sI%sI%sI%sI%sI%sI%sI%sI#sI#rH"rH"rH$rH$rH$rH$rH$rH$qG#qG#qG#qG#pG!pG!pG!pG!pG!pG!pG#pG#nE!nE!nE!nE!nE!mD mD mD mD mD mD mD mD mD mD mD lCmD mD lClClClClClClClClClClClClCkBkBkBkBkBkBkBjAjAjAjAjAjAjAgAgAgAgAgAgAgAd>e?e?e?e?e?e?g>f=f=f=f=e<e<e<e<e<c=c=b<c=c=b<d:d:d:d:c9c9c9b8a:a:a:a:`9`9`9`9\7^9\7\7\7\7\8\8^8]7]6]6]6]6]6]6]6\5\5\5\5\5Z5Z5Y4Y4Y4Y4X3X3V4V4V4V4U3V4V4V4U3U3U3S1S1S1S1S1Q2Q2P1P1P1P1R0R0R0R0R0Q/Q/Q/Q/Q/Q/P.P.P.P.P.O-P.P.O-M.M.M.M.M-M-L-L-L-L-J+J+J+J+J+J+J+J+I+I+I+H*H*H*H*G)G)G)zP(zP(zP(zP(zP(zP(zP(zP(}P'}P'}P'}P'}P'}P'}P'}P'}P)}P)}P)}P)}P)}P)}P)}P)}P'}P'}P'~Q(~Q(~Q(~Q(~Q(~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~R&~R&~R&~R&S'S'S'S'R)R)R)R)R)R)R)€S*€S*R)R)R)‚U,‚U,‚U,‚U,‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*€T(€T(€T(€S*€S*€T(‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*ƒW+ƒW+ƒW+ƒW+ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-„W.„W.„W.„W.„W.ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒV-ƒV-…V/…V/…V/…V/…V/…V/ƒW+ƒW+‚V*‚V*‚V*‚V*‚V*‚V*ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,€S*‚U,€S*€S*€S*€S*€S*R)€S*€S*€S*€S*€S*€S*€S,€S,€S*€S*€S*€S*€S,R+R+R+R+R+R+R+R+R+R+R+R+R+}S+}S+}S+}S+}S+}S+|R*|R*|R*|R*|R*|R*~Q*~Q*~Q*~Q*}P)}P)~Q*~Q*~Q*~Q*~Q*~Q*{Q'{Q'{Q'{Q'}P'}P'}P'}P'}P)}P)}P)}P)}P)}P){Q)zP(zP(zP(zP(zP(|O(|O(|O(|O(|O*|O*|O*|O*|O*|O*|O*|O*yO)yO)yO)yO)yO)yO)xN(xN(yO)yO)xN(xN(xN(xN(zM(zM(zM(zM(xN(vL&vL&vL&vL&vL&vL$xN&zP*vL&lB X. Q$Q$P#KM P#R&S'W(\-^.W' N# L!N$ J G=0 -R,oK&oK&sJ sJ sJ(sJ(tJ&sI%sI%sI%sI%sI%sI%sI%sI%sI%sI#sI#sI#sI#sI%sI%rH$rH$rH$rH$rH$rH$rH$rH$qH"qH"pG!pG!pG!pG!pG#pG#pG#nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!mD mD mD mD mD mD mD mD mD mD lCmD mD mD mD mD lClClClClClClClClCkBkBkBkBkBkBjAjAjAjAhBhBhBgAgAgAgAgAgAgAgAgAgAgAg>g>g>g>g>f=e<e<e<e<c=d>d>d>c=c=d:e;e;d:d:d:d:c9b;a:a:a:a:`9a:`9^9^9^9^9^9^9^:\8^8^8^7^7^7]6]6]6]6]6\5\5\5\5Z5Z5Z5Y4Y4Y4Y4Y4W5V4V4V4V4V4V4V4U3U3U3U3U3U3S1S1Q2Q2Q2Q2Q2Q2R0R0R0R0R0R0R0Q/Q/Q/Q/Q/Q/Q/P.P.P.P.P.P.M.M.M.M.M-M-M.L-L-L-L-L-L-L-J+J+J+J+I+I+I+I+H*H*H*H*H*G)zP&zP&zP&zP&zP&zP&zP&zP&}P'}P'}P'}P'}P'}P'}P'}P'}P)}P)}P)~Q*~Q*~Q*~Q*~Q*~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q*~Q*~Q*~Q*~Q*~Q*R+R+S'S'S'S'S'S'S'S'R)R)R)R)€S*R)R)€S*€S*‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*ƒW+ƒW+ƒW+ƒV-ƒV-ƒW+‚V*‚V*‚V*‚V*‚V*ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„X,„X,„X,ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒV-ƒV-ƒW+ƒW+†X,†X,…W+…W+…W+…W+…W+…W+…W+…W+…W+…W+…W+…W+„W.„W.…V-…V-…V-…V-…V-…V-ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+‚V*‚V*ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+‚V*‚V*‚V*ƒW+ƒW+‚U,ƒV-ƒV-ƒV-‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,€S*‚U,‚U,‚U,‚U,‚U,€S*‚U,‚U,€S*€S*€S*€S*€S*€S*€S,€S,€S*€S*€S*€S*€S,€S,€S,€S,€S,€S,€S,€S,€S,R+R+€S,€S,R+}S+}S+~T,~T,~T,~T,}S+}S+}S+|R*|R*|R*~Q*~Q*~Q*R+~Q*~Q*}P)~Q*~Q*~Q*}P'~Q(~Q(~Q(~Q(~Q(}P'~Q(~Q(~Q(~Q*~Q*~Q*}P)}P)}P){Q)zP({Q){Q)|O(|O(|O(|O(}P)}P)}P)|O(|O(|O(|O(|O(|O(|O(zP(yO'yO'yO'yO'yO'{N'{N'yO'yO'yO'yO'zM&zM&zM&zM&zM&{N'{N'zM&vL$vL$xN&vL$uK#vL$yO+xN*j@P&S&h;vI$~Q,ƒW+†Z.Œ^+‰[(„V(xJe8[.S&R%@=7$(L)uQ,nJ%tK!sJ sJ(sJ(tJ&tJ&tJ&tJ&tJ&tJ&tJ&sI%tJ&tJ&tJ$tJ$sI#sI#sI%sI%sI%sI%sI%sI%rH$rH$rH$rH$rH$rH$rH$rH$rH$rH$qG#rH$pG!pG!pG!pG!pG#nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!mD mD mD mD mD mD mD mD mD mD mD mD mD"mD"lC!mD"lClClClClCkBlCkBkBkBkBkBjAjAhBhBhBhBhBhBhBhBgAgAgAgAi@i@i@g>g>g>g>g>g>g>e<e<e<f=d>d>d>d>f<e;e;d:d:d:d:d:d;d;b<a;a:a:a:a:^9\7^9^9`:`:`:`:^7^7^7^7]6^7^7]6]6]6]6]6]6]6]6\5Z5Z5Z5Y4Y4Y4W5W5W5V4V4V4V4V4X3X3W2W2W2W2W2W2S1S1S1S1S1S1S1S1S1R0R0R0R0R0Q/Q/Q/Q/Q/Q/Q/Q/P.P.P.P.N/N/M.M.O-O-O-O-O-N,L-L-L-L-L-J+J+J+J+J+J+J+J+I* I*I*I*I*zP&zP&zP&zP&zP&zP&zP&zP&}P'~Q(~Q(}P'}P'}P'~Q(~Q(~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q(R)R)R)R)R)R)R)R+R+R+R+R+R+€S,€S,€T(€T(€T(‚V*‚V*€T(€T(€T(R)R)R)R)€S*‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*ƒW+ƒW+ƒW+ƒV-ƒV-ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+„X,„X,„X,„X,„X,„X,„X,„X,„X,„W.„W.„W.„W.„W.„W.…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…Y-…Y-…Y-„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„W.„W.„X,„X,†X,†X,†X,†X,†X,†X,†X,†X,†X,†X,†X,†X,†X,†X,„W.„W.†W.†W.†W.†W.†W.†W.„X,„X,„X,ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+„X,„X,„X,„X,ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒV-ƒV-ƒV-ƒV-ƒV-‚U,‚U,‚U,‚U,‚U,‚U,‚U,ƒV-ƒV-‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U.‚U.‚U,‚U,‚U,‚U,‚U.€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,~T,~T,~T,~T,~T,~T,~T,~T,~T,}S+}S+}S+R+R+R+R+R+~Q*R+R+R+R+R)R)R)~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q*~Q*~Q*~Q*~Q*~Q*{Q){Q){Q){Q)}P)}P)|O(}P)}P)}P)}P)}P)|O(}P)|O(|O(|O(|O(zP(zP(zP(zP(zP(yO'|O(|O(yO'zP(yO'yO'|O(|O({N'{N'{N'{N'{N'{N'yO'yO'yO'xN&xN&yO'zP,xN*f<R(Z-sF!…X3Ža<•i=šnB¡s@£uB¡sE–h:”g>Ža8ƒV-{N%jDb<W8:. Q.tP+pL'tK!sJ sJ(sJ(uK'uK'uK'uK'uK'tJ&tJ&tJ&tJ&tJ&tJ$tJ$sI#tJ$tJ&sI%sI%sI%sI%sI%sI%rH$sI%sI%sI%sI%rH$rH$rH$rH$rH$rH$qH"pG!pG!pG!pG#pG#pG#pG#pG#pG#pG#nE!nE!nE!nE!nE!mD nE!nE!mD mD nE!nE!nE!nE!nE!nE!mD mD"mD"mD"mD"mD mD mD lClClClClClCkBkBkBkB kB hBhBhBhBhBhBhBhBgAgAgAhBjAi@i@i@g>g>g>g>g>g>g>g>g>f=d>e?d>d>f<e;f<f<e;e;d:c9d;d;b<b<a:b;a:a:_:_:^9\7^8`:`:^8^7^7^7^7^7^7^7^7^7]6]6]6]6]6]6]6Z5Z5Z5Z5Z5Z5W5W5W5W5V4W5W5V4X3X3X3X3W2W2W2W2U3U3S1S1S1S1S1S1S1S1S1R0R0R0R0R0R0Q/Q/Q/Q/Q/P.Q/Q/P.N/N/N/N/P.P.O-O-O-O-L-L-L-L-L-L-L-L-L-L-L-J+J+J+J+I*I*I*}P'}P'}P'}P'}P'}P'~Q(~Q(|R(|R(~Q(R)R)R)R)R)~Q*~Q*~Q*R+R+R+R+R+R)R)R)R)R)R)€S*€S*€S,€S,€S,€S,€S,€S,€S,‚U.‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*€T(‚V*‚V*‚V*‚V*‚V*‚V*‚V*ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+…W+…W+ƒW+ƒW+ƒW+„X,„X,„X,„X,„X,„X,„X,„X,„X,…Y-…Y-…Y-…Y-…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡X/‡X/‡Y-‡Y-‡Y+‡Y+†X*†X*†X*†X*†X*†X*†X*†X*†X*†X*†X,†X,‡X/‡X/†W.†W.†W.†W.†W.†W.„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„W.„W.„W.„W.„W.„W.„X,„X,ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒV-ƒV-…V-…V-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-‚U,‚U,‚U,‚U,ƒV-ƒV-‚U,‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.€S,€S,€S,€S,€S,€S,€S,€S*€S*~T*€V,€V,€V,€V.€V.€V.~T,€V.€V.€V.€V.‚U.‚U.€S,R+R+R+€S,€S,R+R+R+R+R+R+R+~Q*R)R)R)R)R+R+R+R+~Q*~Q*~Q*~Q*}P)}P)}P)}P)}P)~Q*~Q*~Q*}P)~Q*}P)}P)}P)|O(}P+|O*{Q+zP*zP(zP(zP(zP(zP(zP(zP(zP(zP(zP(|O(|O(|O(|O(|O(|O({N'{N'yO'yO'yO'yO'xO'xO'zP(yO'f<S)_2 uH!ˆ[!”g-Ÿo4©y>®|A´‚G°~E®|C¶„I³F°|@§s7¢u=šm5f>Y0J( -fD&xN*vL(zM*xK(uK#uK#uK'uK'uK'vL(vL(uK'uK'tJ&tJ$tJ$tJ$tJ$tJ&tJ&tJ&tJ&sI%sI%sI%sI%sI%sI%sI%sI%sI%sI%sI%rH$rH$rH$rH"sI#rH"rH"rH$rH$qG#qG#qG#qG#qG#qG#pG#nE!nE!nE!nE!nE!oE!oE!oE!oE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!mD nE!mD mD mD mD lC!lC!lC!lC!lClClCkBkB kB jAkB iChBhBiCkBjAkBkBjAjAjAjAjAi@jAjAjAi@g>g>i@g>g>g>g>g>f=f=f=f=f<f<e;e;e<e<e<d;d;d;b<b<b;a:a:a:a;`:a;a;`:`:`9`9`9`9`9^7^7^7^7]6]6]6^7]6]6]6[6[6Z5Z5Z5Z5X6W5W5W5W5W5Y4Y4X3X3X3X3X3X3W2W2W2W2U3U3U3U3S1S1S1S1S1S1S1S1R0R0R0Q/R0R0Q/Q/Q/Q/Q/Q/N/N/N/N/P.P.P.O-O-O-M.M.M.M.L-L-L-L-L-L-L-L-J+J+J+J+J+I*}P'}P'}P'}P'}P'}P'~Q(~Q(|R(|R(~Q(R)R)R)R)R)R+R+R+R+R+R+R+R+R)R)R)€S*€S*€S*€S*€S*€S,€S,€S,€S,€S,€S,€S,‚U.‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*ƒW+ƒW+ƒW+ƒW+ƒW+‚V*‚V*‚V*‚V*ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+†X,†X,ƒW+ƒW+„X,„X,„X,„X,„X,„X,„X,„X,…Y-…Y-…Y-…Y-…Y-…Y-…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡X/‡X/‡Y-‡Y-‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y-‡Y-‡X/‡X/‡X/‡X/‡X/‡X/†W.†W.„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„W.„W.„W.„W.„W.„W.„X,„X,„X,„X,„X,ƒW+ƒW+ƒW+„W.„W.†W.†W.„W.ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV/ƒV/ƒV/ƒV/ƒV/‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.€S,‚U.‚U.‚U.‚U,‚U,€V,~T*€V,€V,€V.€V.€V.€V.~T,€V.€V.€V.‚U.€S,‚U.‚U.€S,€S,€S,€S,€S,€S,R+€S,€S,€S,€S,R+R)R)R)R)R+R+R+R+R+R+~Q*~Q*}P)~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*}P)}P)}P)}P+}P+{Q+{Q+{Q){Q)zP(zP(zP({Q)zP(zP(zP(zP(|O(|O(|O(|O(|O(|O(|O(|O(yO'zP(zP({Q)yP(xO'zP(yO'f<V,b5xK$b(œo5ªz?´„I¹‡LÀŽSÁVÀŽUÅ“XÂUÄTÃS¸‹S²…M´‹ckB`> uS5xN*yO+zM*xK(uK#uK#vL(vL(vL(vL(vL(uK'uK'uK'uK%tJ$uK%uK%uK'tJ&tJ&tJ&tJ&tJ&tJ&tJ&sI%sI%sI%sI%sI%sI%sI%sI%sI%sI%sI#rH"rH"rH"rH$rH$rH$rH$rH$rH$qG#qG#pG#pG#nE!nE!nE!nE!oE!oE!oE!oE!nE!nE!nE!nE!nE!nE!nE!nE!nE!nE!mD nE!nE!mD mD mD nE#lC!mD"lC!lClClClCkB lC!lC!kB iCiCiCiCkBjAkBkBjAjAjAjAjAjAjAjAjAi@i@i@i@g>g>g>g>g>g>g>f=f=f<f<f<f<e<e<e<e<d;d;b<b<b;b;a:a:b<a;a;a;a;a;a:`9`9`9`9`9`9`9^7^7^7^7^7^7^7^7[6[6[6[6Z5Z5X6X6X6W5W5W5Y4Y4Y4Y4X3X3X3X3X3W2W2W2U3U3U3U3U3S1U3S1S1S1S1S1S1R0R0R0R0R0R0R0Q/Q/Q/Q/O0O0N/N/P.P.P.P.P.O-M.M.M.M.M.L-L-L-L-L-L-L-L-J+J+J+J+J+}P'}P'}P'}P'}P'}P'~Q(~Q(|R(|R(R)R)R)R)R)R)R)R)R)R)R)R)R)R)R)€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*‚U,‚U,‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*…W)…W)…W)…W)…W)…W)…W)…W)…W)…W)…W)…W)…W)…W)…W)…W)„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„X,„X,„X,„X,„X,„X,…Y-…Y-…Y-…Y-…Y-…Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‡Y-‡Y-‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-„X,„X,„X,„X,†X,†X,†X,†X,„X,„X,„X,„X,„X,„X,ƒV-„W.†W.†W.†W.†W.„W.„W.„W.„W.„W.„W.„W.„W.ƒV-ƒV-ƒV-„W.ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV/‚U.‚U.‚U.‚U.‚U.‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,€S,€S,‚U.€S,€S,€S,€S,€S,~T,~T,€S,R+R)R)€S,€S,€S,€S,‚S,‚S,‚S,R+R-R-~Q(~Q(|R(|R(|R(|R(|R*|R*~Q(~Q(~Q*~Q*~Q*~Q*~Q(~Q(}P'}P'}P)}P){Q){Q){Q){Q){Q){Q){Q)zP(zP(zP(|O(|O(|O(|O(|O(|O(zP(zP(zP({Q)zP&zP&{Q'yO%d;V- d5 yJ!‘`,žm9¬z?µƒHºˆKÁRÅ‘UÈ”XÉ•YË—[Ñœ]Ò^Ðœ`È”XÀYO€N&’`8|M*}N+zM(xK&xK&xK&vL&vL&vL&vL&vL&uK%uK%uK%uK%tJ$uK%uK%uK'tJ&tJ&tJ&tJ$tJ$tJ&tJ&tJ&sI%tJ&tJ&sI#sI#tJ&tJ&sI%sI%sI#sI#rH"sI#rH"rH"rH"rH"rH$rH$rH$rH$rH$qG#qG#qG#qG#qG#qG#qG#qG#qG#qG!qG!qG!qG!pG#pG#nE!pG#pG#nE!nE!nE!mD nE!mD nE!nE!mD mD mD mD lClClClClClClCkBkBkBkBkBkBkBkBkBjAkBjAjAjAjAjAjAi@i@i@i@i@i@g>g>i@i@i@g>f=f<f<f=f=f=f=e<f=e<e<f=e<b<b<b<b<b<b<b<a;a;a;a:a:a:`9`9`9`9`9^7^7^7^7^7^7^7^7\7[6[6[6[6[6Z5Z5Z5Z5Z5Z5Z5Y4[4[4Y4Y4X3X3X3W2X3X3U3U3U3U3U3U3U3U3U3S1S1S1S1S1S1S1R0R0R0R0Q/Q/Q/Q/Q/Q/Q/Q/Q/P.P.P.P.P.P.O-M.M.M.M.M.M.M.L-L-L-L-L-L-J+J*J*}P'}P'}P'}P'}P'}P'~Q(~Q(|R(}S)€S*R)R)R)R)R)R)R)R)€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚V*‚V*‚V*ƒW+ƒW+ƒW+ƒW+ƒW+…W)…W)…W)…W)…W)…W)…W)…W)…W)…W)…W)…W)…W)…W)†X*†X*„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„X,„X,„X,…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-‡Y-‡Y-‡Y-‡Y-ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ,ˆZ,‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y+‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-…Y-…Y-…Y-…Y-‡Y-‡Y-‡Y-‡Y-…Y-„X,„X,„X,„X,„X,„W.„W.‡X/‡X/‡X/‡X/…X/…X/…X/…X/„W.„W.„W.„W.„W.„W.„W.„W.„W.ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-„W.„W.ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-‚U,ƒV-ƒV-‚U,‚U,ƒV-‚U,‚U.‚U.‚U.‚U.€S,‚U.‚U.‚U.€V.~T,‚U.‚U.€S*€S*€S,€S,€S,€S,‚S,‚S,‚S,R+R-R-R)R)}S)}S)}S)}S)}S+}S+~Q(~Q(~Q*~Q*~Q*R+R)}P'}P'}P'~Q*~Q*{Q)|R*|R*{Q){Q){Q){Q){Q){Q){Q)|O(}P)}P)|O(|O(|O(zP(zP(zP({Q){Q'{Q'}S)yO%d;U,g8|M$^*l8¬z?µƒH»‰LÁRÆ’VÉ•Y̘\Ï›_Óž_Óž_Òžb̘\Ç—`†V›iAœjBzK(}N+{N)zM(zM(xK&vL&vL&vL&vL&vL&vL&vL&vL&uK%uK%uK%vL&vL(uK'uK'tJ&tJ$tJ$tJ&tJ&tJ&tJ&tJ&tJ&tJ$tJ$tJ&tJ&tJ&tJ&tJ$tJ$tJ$tJ$sI#sI#sI#sI#sI%sI%rH$rH$rH$rH$rH$rH$rH$qG#qG#qG#qG#qG#qG!qG!qG!qG!pG#pG#pG#pG#pG#pG#nE!nE!mD nE!nE!nE!nE!nE!mD mD mD mD mD mD mD lClClClClClClClCkBkBkBkBkBkBkBkBjAjAjAjAjAjAi@i@i@i@i@i@i@i@i@g>g>f<f<f=f=f=f=f=f=f=f=f=f=c=c=b<b<b<b<b<b<b<a;a:a:a:a:`9a:`9`9`9^7^7^7`9`9`9^7\7\7\7[6[6[6[6[6[6Z5Z5Z5Z5Z5\5\5Y4Y4Y4Y4Y4Y4X3X3V4U3V4V4V4U3U3U3U3U3U3U3S1S1S1S1S1S1R0R0S1S1R0R0Q/Q/Q/Q/Q/Q/Q/P.P.P.P.P.N/N/L-M.M.M.M.M.M.L-L-L-L-L-J*J*~Q(~Q(~Q(~Q(~Q(~Q(~Q(R)R)€S*€T(S'S'S'€T(€T(}S)}S)~T*~T*€T(€T(€S*€S*€S*€S*€S*€S*€S*€S*€S*€S*‚U,‚U,‚U,‚U,‚U,‚U,‚U,ƒV-‚V*‚V*ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+…W+…W+…W+…W+…W+…W+…W+…W+†X,†X,†X,†X,†X,†X,†X,†X,„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.…X/…X/…X/…X/…X/‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‡Y-‡Y-ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‡Y-‡Y-‡Y-…Y-…Y-…Y-…Y-‡Y-‡Y-‡Y-‡Y-…Y-…Y-…Y-…Y-…X/„W.„W.…X/‡X/‡X/‡X/‡X/…X/…X/…X/…X/…X/…X/…X/…X/„W.„W.„W.…X/„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,ƒW+„X,„X,„X,„X,„W.„W.„X,„X,„X,„X,„W.„W.ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-‚U,ƒV-ƒV-ƒV-ƒV-‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,€S*€S*€S,‚U.‚U.€S,‚S,‚S,‚S,R+R-‚S.€S,€S,~T,~T,~T,}S+}S+~T,€S*€S*€S,€S,R+R+R)~Q(R)R)R+R+~Q*~Q*R+~Q*~Q*~Q*}P)~Q*~Q*~Q*~Q*~Q*~Q*}P)}P)}P)}P)|O(zP&|R({R(zQ'}T'zQ$d:U+j<Q%‘`,l8®z@·ƒI¿‹OÄTÈ–[Ë™^Í™]Ðœ`Óž_Óž_Ñž]ÒŸ^ʘ]†TŸn?žm>zK"~O&zM&zM&zM(zM(vL$xN&xN&xN&xN&vL$xN&xN&vL&vL&vL&xN(xN(vL&vL(uK'uK'tJ&uK'tJ&tJ&tJ&tJ&tJ&tJ&tJ&tJ&tJ&tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI%sI%sI%sI%sI%rH$rH$rH$rH$rH$rH$rH$rH$rH$rH"rH"qG!qG!rH$qG#qG#qG#qG#qG#qG#qG#nE!nE!nE!nE!nE!nE!mD mD mD mD nE!nE!mD lClClClClClClClClClClCkBkBkBkBkBkBkBkBjAjAjAi@i@i@i@i@i@i@g>g>g>g>g>g>g>g>g>f=f=f=f=f=f=f=f=e<e<e<d;d;b<b<`<_;_:`;`;_:b8c9a:`9`9`9^7`9^9^9^9^9\7\7\7\7\7[6[6[6[6[6]6]6Z5Z5Z5Z5Z5Y4Y4Y4W5W5Y4X3V4W5X3X3X3W2V4V4U3U3U3U3U3S1Q2Q2Q2Q2S1S1S1S1R0R0R0Q/Q/Q/Q/Q/Q/Q/P.P.P.P.P.P.L-M.M.M.M.M.M.M.M.L-L-L-L-L-~Q(~Q(~Q(~Q(~Q(~Q(~Q(~Q(R)€S*€T(€T(€T(€T(€T(€T(~T*~T*~T*~T*€T(€T(€S*€S*‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,ƒV-ƒV-ƒV-ƒV-ƒV-ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+…W+…W+…W+…W+…W+…W+…W+†X,†X,†X,†X,†X,†X,†X,†X,†X,…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.†Z.†Z.†Z.†Z.‡Y-‡Y-‡Y-‡Y-…Y-…Y-…Y-…Y-…X/…X/…X/†Y0‡X/‡X/‡X/‡X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…Y-„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„W.„W.„X,„X,„X,„X,„W.„W.„W.„W.ƒV-„W.„W.ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-„W.ƒV-ƒV-„W.ƒV-ƒV-ƒV-‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,€S*€S*‚U.ƒV/ƒV/‚U.„U.„U.„U.„U.‚S.‚S.€S,€S,~T,~T,~T,~T,~T,~T,‚U,‚U,‚U.‚U.‚U.‚U.€S*€S*€S*€S*€S,€S,€S,€S,€S,R+R+R+~Q*R+R+R+R+R+R+~Q*~Q*~Q*}P)}P){Q'}S){R(zQ'~U({R%c9U+j<‚T(–e1Ÿn:®z@º†LÁQÅ‘UÉ—\Ìš_Í™]ÑaÓž_ÔŸ`Ñž]Ô¡`É—\ˆV£rCœkg>g>f=f=f=f=f=f=f=f=e<e<e<e<e<c=b<`<_;`;`;`;_:c9c9a:`9`9a:a:`9^9^9^9^9^9\7\7\7\7\7\7\7[6[6]6]6[6[6Z5Z5Z5Z5Y4Y4W5W5Y4Y4W5W5Y4X3X3X3V4V4V4U3U3U3U3U3S4S4S4Q2S1S1S1S1R0R0R0R0R0R0R0Q/Q/Q/Q/Q/Q/P.P.P.N/N/M.N/M.M.M.M.M.M.M.L-L-L-R)R)R)R)R)R)R)R)€S*€S*€T(€T(€T(€T(€T(€T(~U(~U(~U(€W*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚U,‚U,‚U,‚U,ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒW+ƒW+ƒW+„X,„X,„X,„X,„X,†X,†X,†X,†X,†X,†X,†X,†X,†X,†X,†X,†X,‡Y-‡Y-‡Y-‡Y-…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/†Y0†Y0†Y0†Y0†Y0ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‰[/‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.†Z.†Z.ˆZ.ˆZ.‡Y-ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‡Y-ˆZ.ˆZ.ˆZ.ˆZ.‡Y-‡Y-ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆY0ˆY0‡X/‡X/‡X/ˆY0…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-„X,„X,„X,‡Y-‡Y-†X,†X,†X,†X,†X,†X,†W.†W.†W.†W.…V-†W.†W.†W.†W.†W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-‚U,ƒV-ƒV-ƒV-ƒV-ƒV-‚U,ƒV/ƒV/ƒV/ƒV/…V/…V/‚S,‚S,„U0„U0‚U.‚U.€V.€V.€V.€V.~T,€V.‚V*‚V*‚U,‚U,‚U,‚U,‚U,‚U,‚U.€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,R+R+R+R+~Q*~Q*~Q*~Q(~Q(}S)|R(~U({R%c9U+e6 P'”c/¢q=¯{A¸„JÂŽRÆ’VË—]Í™_Í™_ÐœbÑž_Ñž_Ñž]Ó _Ë™^‹Y¥tB˜g5|K P%~O(~O(|O*|O*yO'yO'yO'yO'yO'xN&xN&xN&xN&xN&xN(xN(xN(xN(xN(xN(vL&vL&vL&vL&vL&vL&vL&vL&vL&tJ$uK%uK%uK%uK%uK%uK%uK%uK%uK#uK#tJ"tJ"tJ$tJ$tJ$sI#sI%sI%sI%sI%rH$sI%sI%sI%sI#sI#sI#sI#rH"rH"rH$rH$rH$rH$rH$rH$rH$qG#qG!qG!qG!qG!qG#qG#qG#oE!oE!oE!oE!oE!oE!oE!oE!oE!nDoEmDmDmDlClClClClClClClCkBkBkBkBkBkBkBkBkBkBjAjAjAjAjAjAjAi@i@i@i@i@g>g>g>g>g>g>g>f=f=f=f=e<e<c=c=a=`<`;a<`;a<d:d:d:c9c9c9a:a:_:^9^9^9^9[6\7^9\7\7^8^8^7^7\7[6[6[6[6Z5Z5Z5Z5Z5Z5Z5Z5Y4Y4Y4Y4Y4Y4X3V4V4V4V4V4V4U3U3S4S4S4S4U3S1U3S1S1S1R0R0R0R0R0R0R0Q/Q/Q/Q/Q/Q/Q/N/N/N/N/N/N/M.M.M.M.M.M.M.L-€S*€S*€S*€S*€S*€S*€S*€S*‚U,‚U,‚V*‚V*‚V*‚V*‚V*‚V*€W*€W*€W*€W*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚V*‚U,‚U,‚U,‚U,ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-„W.ƒW+ƒW+„X,„X,„X,„X,„X,„X,†X,†X,†X,†X,†X,†X,†X,†X,‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/…X/†Y0†Y0†Y0†Y0†Y0ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‹]1‹]1‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.†Z.†Z.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0†Z.…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-„X,…Y-…Y-…Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡X/‡X/‡X/‡X/‡X/‡X/‡X/†W.†W.†W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.ƒV-ƒV-ƒV-ƒV-‚U,ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV/ƒV/ƒV/ƒV/…V/…V/…V/„U.„U0„U0ƒV/‚U.€V.€V.€V.~T,~T,W/ƒW+‚V*‚U,‚U,‚U,‚U,‚U,‚U,‚U.€S,‚U.‚U.€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,R+R+R+R+R)R)}S)|R(}T'{R%a7U+i:P'”c/¢q=°|Bº†LÂŽRÆ’VË—]̘^̘^ÐœbÐ^Ð^Ð\Ð\Ç•ZˆV¦uC•d2~M"‚Q&P)~O(|O*|O*zP(yO'yO'yO'yO'yO'xN&yO'yO'xN&xN(xN(xN(xN(xN(xN(xN(xN(xN(vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&uK%uK%uK%uK%uK%uK#uK#tJ"tJ"tJ$tJ$tJ$tJ$tJ&tJ&tJ&tJ&tJ&sI%sI%sI%sI#sI#rH"sI#sI#sI#sI%rH$rH$rH$rH$sI%sI%rH$qG!rH"qG!rH"rH$qG#qG#qG#oE!oE!qG#qG#oE!oE!oE!oE!oEoEmDmDmDmDmDmDlClClClClClClClCkBkBkBkBkBkBkBjAjAjAjAjAjAjAi@jAi@i@i@g>g>g>g>g>g>g>g>f=f=f=f=f=c=d>b>a=a<a<a<a<d:d:d:c9c9c9a:a:_:_:_:_:_:^9\7^9^9\7^8^8^7^7\7\7\7[6[6[6[6Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5Y4Y4Y4W5W5V4V4V4V4V4U3S4S4S4S4U3U3U3U3S1S1R0R0R0R0R0R0R0R0Q/Q/Q/Q/Q/Q/N/N/N/N/N/N/N/N/N/M.M.M.M.L-‚S,‚S,‚S,‚S,‚S,‚S,‚S,‚S,„U.„U.„U,„U,„U,„U,„U,„U,‚U,‚U,‚U,‚U,‚V*‚V*‚V*‚V*ƒW+ƒW+ƒW+ƒW+ƒV-ƒV-ƒV-ƒV-ƒV-„W.„W.„W.„W.„W.„W.„W.ƒV-ƒV-„W.„W.„X,„X,„X,„X,†X,†X,†X,†X,†X,†X,†X,‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-†Y0†Y0†Y0†Y0†Y0†Y0†Y0†Y0†Y0†Y0†Y0†Y0†Y0†Y0†Y0†Y0ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]/‹]/‹]/‹]/‹]/‹]/‹]/‰[-‰[/‰[/‰[/‰[/‰[/‹]1‹]/‹]/‹]/‹]/‹]/‰[-‰[-‰[-‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.ˆZ.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.…Y-…Y-…Y-…Y-…Y-…Y-†Z.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆY0ˆY0‡X/‡X/‡X/‡X/‡X/‡X/‡X/†W.†W.†W.†W.†W.†W.†W.†W.†W.†W.‡X/‡X/†W.…X/…X/„X,„X,„X,„X,„X,„X,„W.ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-…V/…V/…V/…V/…V/…V/…V/„U.„U.…V/ƒW+ƒW+X+€W*€V,€V,‚U,ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV/‚U.‚U.‚U.‚U.‚U.‚U.‚U.€S,€S,‚U.‚U.€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,€S,R+R+R+}S+|R*zQ+xO)`6V,k<‚S*•d5¢qB°|D¼ˆPÄVÆ’Xʘ]Ë™^ʘ]ÏbÑaÑaÎ]Ò¡aÑXƒQªyE’a-}O!‚T&}P'}P'}P)|O(|O({N'{N'{N'{N'{N'{N'{N'yO'yO'yO)yO)yO)yO)yO)xN(xN(xN(xN(xN(xN(vL&vL&vL&vL&vL&vL&vL&vL&vL&uK%uK%uK%uK%uK#uK#uK#tJ"tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI%sI%sI%sI%sI%sI%sI%rH$rH"rH"rH"rH"rH$rH$qG#qG#qG#qG#qG#qG#qG#qG#qG#qG#oEoEqG!oEnEmDmDmDlClClCmDmDlClClClClClCkBkBkBjAkBkBkBkBjAjAjAjAjAjAjAi@i@i@i@i@i@g>g>f=f=e<f=f=f=d>d>b>a=a=a=a=a=e<e<d;d;d;d;d:c9a:a:_:_:_:_:_:^9`:^8`:`:`9`9\7\7\7\7\7[6[6[6[6[6[6[6Z5Z5Z5Z5\5[4Y4Y4Y4Y4W5V4U3V4V4V4V4V4T5T5U3U3U3U3U3S1S1S1R0R0R0R0R0R0Q/R0Q/Q/Q/Q/O0O0N/N/N/M.N/N/N/N/N/M.M.M.„U.„U.„U.„U.„U.„U.„U.„U.„U.„U.„U,„U,„U,„U,…V-…V-‚U,‚U,ƒV-ƒV-ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.„W.…X/…Y-…Y-…Y-…Y-†X,‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-†X,‡Y-‡Y-‡Y-‡Y-‡Y-ˆZ.ˆZ.†Y0†Y0†Y0†Y0†Y0†Y0†Y0‡Z1‡Z1‡Z1‡Z1‡Z1‡Z1‡Z1‡Z1‡Z1‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‹]1‹]1‹]1‹]1‹]1‹]1‰[/‰[/‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]1‹]1‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]1‹]1‹]1‹]1‹]1‹]1‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/…X/…X/…Y-…Y-„X,„X,„X,„X,„W.„W.„W.„W.„W.„W.„W.ƒV-†W0†W0†W0†W0†W0†W0†W0…V/…V/…V/„X,„X,€W*€W*W-€V,‚U,ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV/‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.€S,€S,€S,€S,€S,€S,R+}S+W/€W1|S-a7U+h9€Q(–e6£rC°|D¸„LÁSÆ’Xʘ]Ë™^Ë™^ÏbÑaÑaÏž^Ò¡aÁV‰W°K^*Q#‚T&}P'}P'}P)}P)|O(|O(|O(|O(|O(|O(|O({N'yO'yO'yO)yO)yO)yO)yO)xN(yO)yO)yO)xN(xN(xN(xN(xN(xN(xN(xN(xN(xN(vL&vL&vL&vL&vL&vL$uK#uK#tJ"uK%uK%tJ$tJ$tJ$tJ$tJ$tJ$sI#tJ$tJ$tJ$tJ$tJ$tJ$tJ$sI#tJ$tJ&sI%sI%sI%sI%sI%sI%sI%rH"rH"rH"rH"rH$sI%sI%rH$rH$rH$rH$rH$qG#qG#qG#qG#qG!qG!qG!oEnEnEnEnEnEnEnEmDmDmDmDmDmD mD mD lClClClClCkBkBkBkBkBkBjAjAjAjAjAi@i@i@i@i@i@i@i@g>g>g>e<f=d>d>b>b>b>b>b>b>e<e<e<e<d;d;d:d:b;b;_:_:^9_:_:_:a;a;`:`:`9`9^9^9^9\7\7\7[6[6[6[6[6[6[6[6Z5Z5\5\5Z5Y4Y4Y4W5W5W5W5V4V4V4V4T5T5V4V4U3U3U3S1S1S1S1S1S1S1S1R0S1R0R0R0R0Q/O0O0O0O0O0N/N/N/N/N/N/N/N/N/„U.„U.„U.„U.„U.„U.„U.„U.…V/…V/…V-…V-…V-…V-…V-…V-ƒV-ƒV-ƒV-ƒV-ƒW+ƒW+ƒW+ƒW+„X,„X,„X,„X,„X,„X,„W.„W.„X,„X,„X,…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-‡Y+‡Y+‡Y+‡Y+ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‹]/‹]/‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]1‹]1‹]1‹]1‹]1‹]1Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]/‹]/‹]1‹]1‹]1‹]1‹]1‰[/‰[/‹]1‰[/‰[/‰[/‰[/‰[/‰[/‹]1‹]1‹]1‹]1‹]1‰[/‰[/‹]1‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‡[/‡[/†Z.†Z.†Z.†Z.†Z.†Z.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆY0ˆY0ˆZ.ˆZ.ˆY0ˆY0ˆY0‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡Y-‡Y-‡Y-†X,†X,‡Y-‡X/‡X/†W.†W.„W.„W.„W.„W.†W0†W0†W0†W0ˆV0ˆV0†W0†W0ƒV/„W0‡V)†U(‡V)‡V)„V*…W+…V-…V-‚V*ƒW+ƒV-„W.ƒV-‚U,ƒV-ƒV-ƒV-ƒV-ƒV-‚U,‚U,‚U,‚U,‚U,‚V*‚V*‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,€S*€S*R)~T,}S+~T,ƒY1ˆb=}W2\7P+ e?[4˜e3¥r@¯{A¼ˆNÃSÈ”Xʘ]Ë™^Ë™^ÏbÑaÒžbÎ]Ó¢b¾ŒSˆV°KŠY%€R$‚T&~Q(~Q(}P)}P)|O(|O(|O(|O(|O(|O(|O(|O(|O({N'|O(|O({N){N)yO)xN(yO)yO)yO)xN(xN(xN(xN(xN(xN(xN(xN(xN(xN(xN(vL&vL&vL&vL&vL$vL$vL$vL$uK%uK%tJ$tJ$uK%tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#tJ$tJ$tJ$tJ$sI%sI%sI#sI#sI#sI#sI#sI#sI#rH"sI%sI%sI%rH$rH$rH$rH$rH$rH$qG#qG#qG#qG%qG%qG%oE#oE!oE!oE!oE!oE!oE!oE!oE!nD nD nD nD mD mD mD lClClClClClClCkBkBkBkBkBkBjAjAjAi@i@i@i@i@i@i@i@g>g>g>g>g>e?e?d>d>b>b>b>b>f=e<e<e<e<e<e<d;b;b;b;b;`<_;a:a:a:a:a;a;`9`9^9^9^9\7\7[6\7\7[6[6[6[6[6[6[6[6]6\5Z5Z5Z5Z5Y4Y4W5W5W5V4W5V4V4U3V4V4V4V4U3U3S1S1S1S1S1S1S1S1S1R0R0R0R0R0P1O0O0O0O0O0O0O0O0N/N/N/N/N/„U.„U.„U.„U.„U.„U.…V/…V/…V/…V/…V-…V-…V-…V-…V-…V-ƒV-ƒV-ƒV-ƒV-ƒW+ƒW+ƒW+ƒW+„X,„X,„X,„X,„X,„X,„W.„W.„X,„X,„X,…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-†Z.†Z.†Z.†Z.†Z.‡Y+‡Y+‡Y+ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‹]/‹]/‹]/‹]/‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1Œ^2Œ^2Œ^2Œ^2Œ^2_3_3Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]1‹]1‹]1‹]1‹]1Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]/‹]/‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‰[/‰[/‹]1‹]1‰[/‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‰[/‰[/‹]1‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‡[/‡[/†Z.†Z.†Z.‡[/‡[/‡[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆY0ˆY0ˆZ.ˆZ.ˆY0ˆY0ˆY0ˆY0‡X/ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡X/‡X/‡X/‡X/…X/…X/…X/…X/†W0†W0†W0†W0ˆV0ˆV0†W0†W0„W0„W0ˆW*‰X+ˆW*‡V)†X,…W+…V-…V-„X,„X,ƒV-„W.ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-‚U,‚U,‚U,ƒV-ƒV-ƒW+‚V*‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,‚X0€V.€V.uK#K%3 =L'c=wQ*”a/¥r@¯{A»‡MÂŽRÈ”Xʘ]Ë™^Ìš_ÏbÒžbÒžbÒ¡aÒ¡a»‰P”b)´ƒO‡V"‚T&S%~Q(~Q(}P)}P)|O(|O(|O(}P)}P)|O({N'|O(|O(|O(|O(|O(|O*{N)yO)yO)yO)yO)yO)yO)yO)xN(xN(xN(xN(xN(yO)yO)xN(xN(xN(xN(xN(xN(xN&xN&xN&vL$vL&vL&vL&vL&vL&vL&vL&vL&uK%uK%uK%uK%tJ$tJ$uK%uK%uK%uK%tJ$tJ$tJ&sI%tJ$tJ$sI#sI#sI#sI#sI#sI#sI%sI%sI%rH$sI%sI%rH$rH$rH$rH$rH$rH$qG%rH&qG%qG%qG#oE!oE!oE!oE!oE!oE!oE!oE!nD nD nD mD mD mD mD mD mD lClClClClCkBkBkBkBkBjAjAjAjAjAjAjAjAjAi@i@i@i@g>g>g>e?e?e?e?c?b>b>b>e<f=f=e<e<e<e<e<c<b;b;b;`<`<b;a:a:a:a;a;a:a:_:^9^9^9^9\7\7\7\7\7\7[6[6[6[6[6]6\5Z5Z5Z5Z5Z5Y4W5W5W5W5W5W5W5V4V4V4V4V4V4U3U3U3U3S1S1S1S1S1S1S1S1R0R0R0P1O0P1O0O0O0O0O0O0N/O0O0O0O0…V-…V-…V-…V-…V-…V-…V-…V-…V-…V-…W+…W+†X,†X,†X,†X,ƒV-„W.„W.„W.„X,„X,„X,„X,„X,„X,„X,„X,…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/Œ^0Œ^0Œ^2Œ^2Œ^0Œ^0‹]/‹]/Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1_3_3_3_3_3_3_3_3_1_1_1_1Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]/‹]/Œ^0Œ^0Œ^0Œ^0Œ^0‹]/‹]/‹]/Œ^0‹]/‹]1‹]1‹]1‹]1Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]1Œ^2Œ^2‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‰[/‰[/‰[/‰[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‡Y-ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆY0ˆY0ˆZ.ˆZ.‡Y-‡Y-‡Y-‡Y-ˆY0ˆY0‡X/‡X/‡X/‡X/‡X/‡X/‡X1‡X1‡X1‡X1ˆV0ˆV0ŠV0ŠV0ŠV.ŠV.‚Y*‚Y*„X(„X(X'X'‚Y,‚Y,„W.„W.†W0†W0†W0†W0†W0†W0„W.ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒV-ƒV-ƒV-‚U,‚U,‚U,‚U,‚U,€V.‚X0xO'b9J!C=G\0xL’_+¤q=¯{A¼ˆNÂŽRÈ”Xʘ[Ë™\Îœ_ОaОaÑŸbОaÑŸb¼ŠSµƒLÇ–b‚QwI†X*|O&}P'~Q(R)~Q*~Q*~Q*~Q*~Q*}P)}P)}P)}P)|O(|O(|O({N'{N'{N'|O({N'{N'{N'{N'{N'{N'zM&zM&xN(xN(yO)yO)xN(xN(xN(xN(xN&xN&xN&xN&xN&xN&vL&vL&vL&vL&vL&vL&vL&uK%vL&vL&vL&vL&vL&vL&vL&vL&uK%uK%uK%uK%uK%tJ$tJ$uK%uK%sI#sI#sI#sI#sI#sI%sI%sI#sI#rH"sI#sI#sI#sI#rH"rH"rH"rH$rH$rH$qG#qG!oEoEoEqG!qG!qG#qG#oE!nD nD nD oEoEoEnDnD nD nD nD mCmClBmCmClBlBlBjAjAjAjAjAjAjAjAjAjAjAjAi@i@gAgAgAgAe?e?e?e?e?e?g=h>h>f<g=f<e;e;c<c<b;b;b<a;b;b;b<a;a;a;a:a:a:a:_:^9^9^9^:^:^9\7\7\7\7\7\7[6]6]6]6]6Z5Z5Z5Z5Z5Y4W5W5V4W5W5W5W5W5V4V4V4U3U3U3U3U3U3U3S1S1S1S1S1R0S1S1S1Q/P1P1P1O0P1O0O0N/O0O0O0O0…V-…V-…V-…V-…V-…V-…V-†W.†W.†W.†X,†X,†X,†X,†X,‡Y-„W.„W.…X/…X/…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,ˆZ,‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^2Œ^2_1Œ^0Œ^0Œ^0_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1_3_3_3_3_3_3_3_3_1_1_1_1_1_1_1_1_1_1_3_3_3_3_3_3_1_1_1_1_1_1_1_1_3_3_3_3_3_3_3Œ^2Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‰]1‰]1‰]1‰]1‰]1‰]1‰]1‰]1‹]1‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/ˆZ.‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆY0ˆY0ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆY0ˆY0‡X/‡X/‡X/ˆY0‡X/‡X/ˆY2ˆY2‡X1‡X1‰W1ŠX2Y3‹W1‰U-ˆT,€W(X)ƒW'„X(‚Y(‚Y(ƒZ-ƒZ-„W.…X/†W0‡X1‡X1‡X1‡X1‡X1…X/„W.„W.„W.„W.„W.ƒV-ƒV-„X,„X,„X,ƒW+ƒW+ƒW+ƒV-ƒV-ƒV-ƒV-‚U,‚U,‚U,‚U,€V.‚X0tK#mD†]7”kE’hB‰_9~R$S%\(›h4§s9³EÄTÈ”Xʘ[Ìš]Îœ_ОaÏ`Ï`Ï`Ë™\ÁXÂY¼‹W¢q=b4S%}P'R)R)€S*R+R+R+~Q*~Q*~Q*~Q*}P)}P)}P)}P)}P)}P)}P)}P)|O(|O(|O({N'{N'{N'{N'{N'{N'yO)yO)yO)yO)yO)xN(xN(xN(xN&xN&xN&xN&xN&xN&xN(xN(xN(vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&uK%uK%vL&vL&vL&uK%uK%uK%uK%uK%tJ$tJ$tJ$tJ$sI#sI%sI%sI#sI#sI#sI#sI#sI#sI#sI#rH"rH"rH$rH$rH$rH$rH"rH"rH"rH"qG!qG!qG#qG#qG#qG#qG#qG#oEoEoEoEoE!oE!nD nD nD nD nD nD mCmCmCmCkB kB kB jAjAjAjAkB kB jAjAjAi@i@gAgAgAgAgAe?e?e?e?e?h>j@h>g=g=g=e;e;c<c<c<c<b<a;b;b;b<a;b<b<b;a:a:a:_:_:^9^9^:^:^9\7\7\7\7\7[6\7^7]6]6]6[6[6Y4Z5Z5Y4X6X6W5W5W5W5W5W5W5W5V4V4V4V4U3U3U3U3U3U3U3U3S1U3S1S1S1S1P1P1P1P1P1P1P1P1O0O0O0O0†W.†W.†W.†W.†W.†W.†W.†W.†W.†W.†W.†W.†W.†W.‡X/‡X/„W.„W.…X/…X/…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.‡[/‡[/‡[/‡[/‡[/‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1_1Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`2Ž`2Ž`2Ž`2_1_1_1_1_1_1_3_3_3_3_3_3_1_1_1_1_1_1_1_1_3_3_3_3_3_3_3_3Œ^0Œ^0_1_1^1^1_1_1_1_1_1_1_3_3^1^1Œ^0_1_1_1_1_1_1_1_3_3_3_3_3_3Œ^2Œ^2Š^0Š^0Š^0Š^0‰]/‰]/‰]/‰]/‰]/‰]/‰]/‰]/‰]/‰]/‰]1‰]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‰[/‰[/‰[/‰[/‰[/‰[/‹]1‹]1‹]1‹]1‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‰[/ˆY0‡X/ˆY0ˆY0‰Z1ˆY0‡Z3†Y2„Z2„Z2…X1‡Z3…X1€S,„U.‚S,R-P+„P,†R.ƒQ-‡U1†W2R-…X5…X5…V3‡X5‡X3‡X3‡X1‡X1…X/„W.„W.„W.„W.„W.„W.„W.„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,ƒV-ƒV-‚U,ƒV-W/ƒY1uK!Y/sJ—nA©xI®}NµƒL´‚K±}CµGµ‚C¶ƒDÁŒMÈ“TÊ—XÍš[Ð^Ì™ZÍš[Ì™ZÐ^˘YÄ’YÂW¹‡P²€I«}Mo?S'~R&R)€S*R+R+R+~Q*~Q*~Q*~Q(~Q(}P)}P)}P)}P)}P)}P)}P)|O(|O(|O(|O(|O(|O(|O(zP&zP&zP(yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'xN&xN(xN(xN(xN(xN(xN(vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&vL&uK%uK%uK%uK%uK%uK%tJ"uK#uK%tJ$tJ$uK%sI#sI#tJ$tJ$tJ$tJ$sI#sI#sI#sI#sI%sI%sI%rH$rH"rH"rH"rH"rH"rH"rH$rH$qG#qG#qG#qG#qG!qG!qG!oEoEoEoEoEnDnDnD nD nD nD mCnD lClClCkBkBkBlClCjAjAkBjAjAjAjAjAhBgAgAgAgAgAgAe?h>h>h>h>h>g=g<g<e;e;c<c<c=c=c=c=b<c=b<b<b;b;b;a:_:_:_:_:^9^9^9^9^9^9^9\7^7^7^7^7^7]6]6]6]6\5Z5Y4Z5Z5W5X6X6W5W5W5W5W5W5V4V4V4V4U3V4V4U3U3U3U3U3U3S1S1S1S1Q2Q2P1P1P1P1P1P1P1P1P1P1†W.†W.†W.†W.†W.†W.†W.†W.‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/…X/…X/…X/…X/…Y-…Y-…Y-…Y-…Y-…Y-…Y-†Z.†Z.†Z.†Z.†Z.‡[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‰[-‰[-‰[-‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/Œ^0Œ^0‹]/‹]/‹]/Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0_1_1_1_1_1_1_1_1_1_1_1Ž`2Ž`2_1_1_1Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`4Ž`4Ž`4Ž`4Ž`4_3_3_3_1_1Ž`2Ž`2_2_2Ž`2_1_1_1_1_1_3_3^1^1Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2_1_1_3_3_3_3_3_3_3_3‹_1‹_1‹_1‹_1Š^0Š^0Š^0Š^0Š^0Š^0Š^0Š^0Š^0Š^0Š^2Š^2Œ^2Œ^2Œ^2‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1Œ^2Œ^2Œ^2Œ^2‹]1‹]1‹]1‹]1‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‹]1‹]1‰Z1‰Z1‰Z1‰Z1ˆY0‹\3‰\5‡Z3„Z2€V.~Q*zM&~Q*`9”e>“d=’c>”e@˜d@iE™gC™gC“d?ˆY4†Y6ƒV3†W4‰Z7‡X3‡X3‡X1†W0„W.„W.„W.„W.„W.„W.„W.„W.„X,„X,„X,„X,„X,„X,„X,„X,„X,„X,„W.„W.„W.„W.W/‚X0zP&W-N%h?”c4©xI²€IºˆQÅ‘WÆ’XÅ’SÂPÅQÊ•VÉ–WÍš[Ñž_Ù¦gÖ£dÉ–WÐ^Ê—XÄ’YÑXÈ–_ÂYÁ“c¬~N‡[/S'€S*€S*R+R+R+~Q*~Q*~Q*~Q(~Q(}P)}P)}P)}P)}P)}P)}P)}P)}P)}P)|O(|O(|O(|O(zP&zP&zP(zP(zP(zP(yO'yO'yO'yO'yO'yO'yO'yO'yO'yO'yO)yO)xN(xN(xN(xN(vL&xN(xN(xN(xN(xN(xN(xN(xN(vL&vL&vL&vL&vL&vL&vL&uK%vL&vL&vL&uK#uK#uK%uK%uK%uK%uK%tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI#sI%sI%sI%sI%sI#rH"rH"rH"rH"rH"rH$rH$rH$rH$rH$rH$qG!rH"qG!qG!qG!qG!oEoEoEoEnD nD nD nD nD nD mD mD mD lClClClClClClCkBkBkBjAjAjAhBhBhBgAgAgAgAgAj@j@j@j@h>h>h=h=f<e;c<c<c=d>c=c=c=c=c=c=c<b;b;b;`;`;_:_:_:_:_:^9^9^9^9^9^7^7^7^7^7^7^7^7]6]6[6[6Z5Z5X6X6X6X6X6X6X6W5W5V4V4V4V4V4V4V4U3U3U3U3U3U3S1U3U3S1Q2Q2Q2Q2P1P1Q2P1P1P1P1P1†W.†W.†W.†W.†W.†W.‡X/‡X/‡X/‡X/‡X/‡X/‡X/ˆY0ˆY0ˆY0†Z.…Y-†Z.†Z.†Z.†Z.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‹]1‹]1‹]1‹]1‹]1‹]/‹]/‹]/‹]/‹]/Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^._/_/_/_/_/_3_3_1_1_1_1_1_1_1Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2a3a3a3a3a3a3a3a3a3a3a5a5a5a5a5a5a5a5a3a3a3a3a3a3a3a3a3a3a3a3Ž`2Ž`2Ž`2a3a3a3a3a3a3a3a3a3Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4^1^1_2_2_2_2_2_2Ž`2Ž`2^1^1^1^1^3_4_2_2Ž`2a3a3Ž`2Ž`2Ž`2Ž`2_1_3_3_3_3_3_3‹_1‹_1‹_1‹_1‹_1‹_1‹_1‹_1_3_3_3_3_3_3_3_3_3Œ^2Œ^2Œ^2Œ^2Œ^2Œ^0Œ^0Œ^2Œ^2Œ^2‹]1‹]1‹]1Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]1‹]1‰[/‰[/‰Z1‰Z1‰[/‰[/‹]1‹]1‰]1‰]1‹]1‹]1‹\3‹\3‹\3‹\3‹\3‹\3‹\5‰Z3†Y2‡Z3‰Z1R)~O&~O&ƒZ2£zR¯‘j¢„]y`DjQ5u`J~iS€mZ‚o\wfQo^ItaFlQŠeE}X8}S/ƒY5†Y2„W0ƒV-ƒV-…X/…X/„W0„W0†W0†W0‡X/‡X/…Y-…Y-…Y-…Y-…Y-„X,„X,„X,„W.„W.„W0„W0†W0†W0„W0…X1‰Z1xI d1d1yB’[(©n9ºJµƒH¼ŠOÉ‘SÏ—YИZÑ™[Öž`ןaݨiè³tä¬nÕ_ߧiÚ¢dÖ fØ¢hÕ¢nÑžjÃ’e¤sF†X,S'R)€S*R+R+R+~Q*~Q*~Q*~Q(~Q(~Q,~Q,}P+}P+}P+}P+~Q,}P+}P)}P)}P)}P)}P)}P)|O(|O(|O(|O(|O*|O*{N)|O*|O*|O*|O({N'{N'{N'yO)yO)yO)yO)yO)xN(xN(xN(xN(xN(zM(zM(zM(zM(zM(zM(zM(zM(xN(xN(zM(zM(zM(zM(xK$xK$vL$vL$uK%uK%uK%uK%uK%uK%wJ%wJ%uK%tJ$tJ$tJ$tJ$tJ$tJ$tJ$tJ&tJ&tJ&sI%sI#sI#sI#sI#rH"rH"sI%rH$rH$rH$rH$rH$rH"rH"rH$qG#qG#qG#oE!oE!oE!oE!oE!nD nD oE!nD nD nD nD nD mCmCmCmCmCmCmClBlBkBjAjAjAjAjAhBhBhBgAgAgAj@j@j@j@h>h>h>h>g=g=g=f<d>d>d>d>c=c=c=c=c<c<b;b;`;`;`;`;_:_:_:^9_:_:a:^7^7`9`9^7`6`6`6`6^7]6]6]6[6[6Z5Z5X6X6X6X6X6X6W5W5W5W5W5V4V4V4U3V4V4V4V4V4V4U3U3U3S1U3S1S1Q2Q2Q2Q2Q2P1P1P1‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/ˆY0ˆY0ˆY0ˆY0†Z.†Z.†Z.†Z.†Z.†Z.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‹]1‹]1‹]1‹]1‹]1‹]1‹]/‹]/‹]/‹]/‹]/Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^._/_/_/_/_/_/_/_/_/Ž`4Ž`4Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a5a5a5a5a5a5a5a5a3a3a3a3a3a3a3a3a3a3a3a3Ž`2Ž`2Ž`2a3a3a3a3a3a3a3a3a3Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4_2_2‘`3‘`3‘`3‘`3‘`3‘`3a3Ž`2_2_2_2_2_4_4‘`3‘`3a3a3a3Ž`2Ž`2Ž`2Ž`2Ž`2Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4‹_1‹_1‹_1‹_1‹_1‹_1‹_1‹_1_3_3_3_3_3_3_3_3_3_3_3_3Œ^2Œ^2Œ^0Œ^0Œ^2Œ^2Œ^2Œ^2‹]1Œ^2Œ^2Œ^2_3_3_3_3Œ^2Œ^2Œ^2Œ^2‹]1Œ^2Œ]4Œ]4Œ^2Œ^2Œ^2Œ^2‰]1Š^2Œ^2‹]1‹\3‹\3‹\3‹\3‹\3Œ]4‹\5‹\5‹^7†Y2}N%zK"yJ!ˆY0¤{S´‹c{]67  /uP0xS3qG#vL(…X1…X1„W.„W.„W.…X/„W0„W0‡X1‡X1‡X/‡X/…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-…X/„W.„W0„W0†W0‡X1…X1†Y2ˆY0‹\3|IzGzC„Mžc.¯t?¯}B´‚GÀˆJÉ‘SÏ—YÏ—YÕ_Õ_ä¯pè³tà¨jÛ£eÛ£eÛ£eØ¢hØ¢hÐiÊ—c²T”c6‚T(‚T(R)€S*€S,€S,R+R+R+R+R)R)~Q,~Q,~Q,~Q,~Q,~Q,~Q,~Q,~Q*~Q*~Q*~Q*~Q*~Q*}P)}P)}P)}P)}P+|O*|O*|O*|O*|O*|O(|O(|O({N'yO)yO)yO)yO)yO)yO)yO)yO)yO)yO){N){N)zM(zM(zM(zM(zM(zM(xN(xN(zM(zM(zM(zM(zM&zM&xN&vL$vL&vL&vL&uK%vL&vL&wJ%wJ%uK%uK%tJ$tJ$tJ$tJ$tJ$tJ$tJ&tJ&tJ&tJ&tJ$sI#sI#sI#sI#sI#sI%sI%rH$sI%rH$rH$rH"rH"rH$qG#qG#qG#qG#qG#oE!oE!oE!oE!oE!oE!oE!oE!nD nD nD nD nD nD nD nD mCmCmCmClCkBkB kB kB kB iC hBhBhBhBhBj@j@j@j@j@j@j@h>g=g=g=g=e?e?d>d>d>d>d>d>d=c<c<c<`;`;`;`;`;`;`;_:_:_:a:a:a:a:`9`9b8b8`6`6^7^7^7]6[6[6[6Z5Y7Y7X6X6X6X6X6W5W5W5W5V4W5W5W5V4V4V4V4V4V4V4V4U3U3U3S1S1Q2Q2Q2Q2Q2Q2Q2Q2…Y-…Y-…Y-…Y-…Y-…Y-…Y-…Y-†Z.†Z.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‡Y-ˆZ.ˆZ.‰[/‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‹]/‹]/‰[+‰[+‰[+‹]-‹]-‹]-‹]-‹]-‹]1‹]1‹]1‹]1‹]1‹]1Œ^2Œ^2Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0_1_1_1_1_1Œ^2Œ^2Œ^2_3_3_3_3_3_3_3_3_3Ž`4Ž`4Ž`4Ž`4Ž`2Ž`2Ž`2Ž`2Ž`2a3a3a3a3a3a3a3a3a3a3a3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`5‘`5‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3a5a5a5a5Ž`2Ž`2a1a1‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3a3a3‘`3_2_2_2’_4“`5‘`3‘`3‘`3‘`3_2_2_2_2_2_2_4_4_4_4_4_4Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2_3_3_3_3_1_1_1_1_3_3_3Œ^2_3_3_3_3_1_1Œ^0Œ^0Œ^2Œ^2Š]4Š]4Š]4Š]4Š^2Š^2Œ^2Œ^2Š^2Š^2Š^2Š^2Š^2Š^2‡]5‡]5ˆ^6…[3ƒZ2„[3‹^5|O&|N"Q%}Q%Œ`4›rHžuK]? -o\G¸šoœ~Sƒ[,W(ƒW)…Y+…Y-…Y-ƒY/ƒY/…X1…X1†Y0†Y0ˆZ.ˆZ.ˆZ.ˆZ.‡Y+‡Y+‡Y-‡Y-…Y-…Y-‡X/†W.…X/…X/ˆY0ˆY0ŠX0ŠX0ŒW)‹V(yJ%d5mB&ŽcG¦rL²~XÉ^Ì’gÓ™iØžnØžnÜ¢rݦsݦsã©wå«yá¥uå©yÛ¡qÍ“c¾„T¥k;‰X-†U*‚U,R)‚U.‚U.€V,€V,€S*€S*€S*€S*€S,R+}S+|R*~Q*~Q*~Q*~Q*R+~Q*~Q(~Q(~Q(~Q(~Q(~Q(~Q(}P'}P+}P+}P+}P+}P+|O*}P+|O*zP({Q)zP(zP(zP(zP(zP*zP*zP*zP*yO)yO)yO)yO){N){N){N){N)zM({N){N){N)yO)yO)zM(zM(zM(zM(zM(zM(zM&zM&xN&vL$vL$vL$xK$xK$xK&wJ%wJ#xK$uK#uK#uK%uK%uK%uK%uK%tJ$uK%tJ$tJ"tJ"sI#tJ$sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#rH"rH$rH$rH$qG#qG#qG#qG#qG#oE!oE!qG#oE!oE!nD nD nD oE!oE!nD nD mCnD mCmCmCmCmClBkBkBkB kB jAjAjAjAjAjAjAi@i@i@i@i@i@g>g=g=g>g>g>g>f=f=f=f=f=f=d>d>c=b<b;c<b;b;b;b;b;b;b;a:b;a:a:`9b8b8`9`9^7^7^7^7^7]6[6Z5[6[6[6[6X6X6X6W5W5W5W5X6W5W5W5V4W5W5W5V4X3X3X3W2U3U3U3U3U3S1U3U3S1S1S1S1…Y-…Y-…Y-…Y-…Y-…Y-…Y-†Z.†Z.†Z.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‰[/‰[/‰[/‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‹]/‹]/‹]/‹]-‹]-‹]-‹]-‹]-‹]-‹]-‹]-‹]1‹]1‹]1Œ^2Œ^2Œ^2Œ^2Œ^2Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0_1_1_1_1_1_1_1_1_1_3_3_3_3_3Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a6’a6’a4’a4’a4’a4’a4’a4’a4‘`3a5a5a5a5a3a3b2b2’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4‘`3‘`3’a4’a4’a4’a4’a4’a4b4b4’a4‘`3‘`3‘`3’_4“`5‘`3‘`3‘`3‘`3_2‘`3‘`3_2_2_2_4_4_4_4_4_4Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`4Ž`4Ž`4Ž`4_1_1_1_1_3_3_3_3_3_3_3_3_1_1_1Œ^0_3_3‹^5Š]4‹^5‹^5‹_3Š^2_3_3‹_3‹_3Š^2Š^2Š^2Š^2ˆ^6ˆ^6ˆ^6‰_7‰`8„[3ƒV-tGyKˆZ.‡[/†Z.†]3¢yOµ—uV8 *. <)­d˜zO‰a2‚Z+Š^0’f8d8†Z.„Z0„Z0†Y2†Y2†Y0†Y0ˆZ.ˆZ.ˆZ.ˆZ.ˆZ,ˆZ,ˆZ.ˆZ.…Y-…Y-‡X/†W.…X/…X/ˆY0ˆY0ŠX0‹Y1ŒW)X*ˆY4R#0>V"f2 €F‘W, f6©o?°vF³yI¹‚O¸N·}K¹M°tD d4’X(‹Q!…K…KP%„S(€S*€S*‚U.‚U.€V,€V,‚U,€S*€S*€S*€S,€S,~T,}S+R+R+R+R+R+R+R)R)R)R)R)R)R)~Q(~Q,~Q,~Q,~Q,~Q,}P+}P+}P+{Q)|R*{Q)zP(zP(zP({Q+{Q+zP*zP*zP*zP*zP*yO){N){N){N){N){N){N){N){N)yO)yO){N){N){N){N){N){N){N'{N'xN&xN&xN&vL$xK$wJ#xK&wJ%wJ#wJ#uK#uK#uK%uK%uK%uK%uK%uK%uK%uK%tJ"tJ"tJ$tJ$tJ$sI#tJ$tJ$sI#sI#sI#sI#sI#sI#sI#rH"rH$rH$rH$rH$qG#rH$rH$qG#qG#qG#qG#qG#qG#qG#qG#oE!oE!oE!oE!oE!nD nD nD mCmCmCmCmClCkBlC!kB kB kB kB kB jAjAjAjAjAi@i@i@i@i@i? i? i@g>i@g>g>g>g>f=f=f=e?d>d>c=c<c<c<c<c<b;b;b;b;b;b;b;a:a:c9c9`9`9`9`9`9^7^7^7\7\7\7[6[6[6X6Y7Y7Y7Y7Y7X6X6X6X6X6X6W5W5W5W5Y4X3X3X3V4V4U3U3U3U3U3U3U3U3S1S1†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.†Z.ˆZ.ˆZ.ˆZ,‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‰[-‹]/‹]/‹]/‹]/‹]/‹]-‹]-‹]-‹]-‹]-‹]-‹]-Œ^.Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2_1_1_1_1_1_1_1_1_1_1_1Ž`2Ž`2Ž`2Ž`2Ž`2^1_2_2_2_2_2_2_2_2_2‘`3‘`3‘`3‘`3‘`3‘`3a3a3a3a3a3a3a3a3a3a3a3a3b4b4b4b4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a6’a6’a4’a4’a4’a4’a4’a4’a4’a4b4b4b4b4b4b4b2b2’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4‘`3‘`3’a4’a4’a4’a4’a4’a4’a4’a4’a4‘`3‘`3‘`3’a4’a4’a4’a4’a4’a4‘`3‘`3‘`3_2‘`3‘`3‘`3‘`5‘`5‘`3‘`3a3a3a3a3Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2a3a3Ž`2Ž`2Ž`2Ž`4Ž`4Ž_6Ž_6Ž`4Ž`4_2^1Ž`4Ž`4_3_3_3_3Ž`4Ž`4Ž`2Ž`2_1Œ^0_3_3‹_3‹_3Œ`4Œ`4Œ`2Œ`2‹_3‹_3‰`6‰`6ˆ^6‰_7_3_3‘^,\*’]*ŽY&„V*yKmCd:i7zH ”a2¡n?œk7‘`,’f1‹_*S#xLlCf= f: |P"‡Z1~Q(zM&nAqCsEP^*™f2 m9³‚S´ƒT—i9ˆZ*‡[/Š^2‰\3†Y0‡Z1‡Z1‰[-‰[-ˆZ,‰[-‰[-‰[-‹Z/‹Z/‡[/†Z.ˆY0ˆY0…X/…X/ˆZ.ˆZ.†Y0†Y0†X,‡Y-~\>< - - 6lG,„U,‡X/…V/R+‚U0‚U0‚U0‚U0‚U.‚U.‚U,‚U,‚U,‚U,‚U.€S,~T,~T,€S,€S,€S,€S,€S*R)S'S'R)R)R)R)R)R)~Q,~Q,~Q,~Q,~Q,~Q,~Q,~Q,|R*|R*|R*|R*|R*|R*|R,{Q+{Q+zP*zP*zP*zP*zP*{N){N){N){N){N){N){N)|O*zP*yO){N){N){N){N){N){N){N'{N'{N'{N'{N'{N'zM&zM&|M&|M&xK$xK$xK&wJ%uK%vL&wJ%wJ%uK%uK%uK%uK%uK#uK#uK%tJ$tJ$tJ$tJ$tJ$tJ$tJ$vI$vI$sI#tJ$tJ$sI#qG!rH"rH"rH"rH"rH"rH"rH"rH"rH"qG!qG!qG!qG!qG!qG!oEoEoEoEoEnDnDmCmCnDnDnDmCnD mC!lB kB kB kB kB kB kB jAjAjAjAjAi@jAi@i@i@i@i@i@i@g>g>g>g>g>g>g>g>d>e?e>d=c<c<c<b;b;b;c<a:b;b;a:a:a:a:`9`9`9`9`9^7^7^7^7^7^7]6]6]6Z5[6[6[6[6[6Z5Z5Z5Z5Z5Z5Z5Y4Y4Y4Y4Y4Y4X3V4V4V4V4V4U3U3U3U3U3U3U3†Z.†Z.†Z.†Z.†Z.†Z.†Z.‡[/‡[/‡[/‰[/‰[/‰[-‰[-‰[-‰[-‰[-‰[-‰[-‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/‹]/Œ^0Œ^0Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^2Œ^2Œ^2_3_3_3_3_3_1_1_1_1_1_1_1Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2_2_2_2_2_2‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4b4’a4’a4’a4”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c8”c8”c6”c6”c6”c6”c6”c6”c6’a4b4b4b4b4b4b4’d4b2’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4”c6’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4‘`3’a4’a4’a4’a4’a4’a4‘`5‘`5‘`3‘`3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a5a5`7`7Ž`4Ž`4_2_2Ž`4Ž`4Ž`4Ž`4Ž`4a5Ž`4Ž`4Ž`2Ž`2Ž`2_1Ž`4Ž`4‹_3‹_3Œ`4Œ`4Œ`2Œ`2‹_3‹_3Ša7Ša7‹a9Š`8Œ^2_3’_-’_-[(ŠU"xJ`2R(X. -c1 i7€M˜e6­|H¹ˆT±…PºŽY¿“c–fÀ—f¿–e¼b´ˆZ°ƒZ·Ša¿’kÖoÕiÇ™mÆ•aÅ”`ÒŸkÖ£oÍœm´ƒT–h8‹]-‹_3Œ`4Š]4‡Z1‡Z1‡Z1‹]/‹]/‰[-‰[-‰[-‰[-‹Z/‹Z/‡[/‡[/‰Z1‰Z1‡Z1†Y0‰[/‰[/‡Z1†Y0†X,‡Y-€^@:   >|W<‡X/†W.…V/†W0„W2ƒV1‚U0‚U0€S,€S,€S*‚U,‚U,‚U,‚U.€S,~T,~T,€S,€S,€S,€S,€S*€S*€T(€T(€S*€S*R)R)R)R)R-R-R-R-R-~Q,~Q,R-|R*|R*}S+}S+}S+}S+|R,{Q+{Q+{Q+{Q+{Q+{Q+zP*}P+|O*|O*|O*|O*}P+|O*|O*zP*zP*{N){N){N){N)|O*{N){N'{N'{N'{N'{N'{N'zM&{N'}N'|M&zM&xK$xK&xK&vL&vL&xK&xK&vL&vL&vL&uK%uK#uK#uK%uK%tJ$tJ$tJ$tJ$uK%tJ$vI$vI$tJ$tJ$tJ$sI#sI#sI#sI#rH"rH"rH"rH"rH"rH"rH"rH"rH"rH"qG!qG!oEqG!qG!qG!oEoEoEoEoEnDnDnDnDnD nD mC!nD"lC!kB kB kB kB kB kB kB kB jAjAjAjAjAjAi@i@i@i@i@i@g>g>g>g>g>g>g>e?d>e>e>c<d=d=d=c<c<c<b;a:a:a:a:a:a:b;b;a:`9`9^7^7^7^7^7`9`9`9]6\7\7\7\7\7[6[6[6[6[6Z5Z5Z5Z5Z5Z5Y4Y4Y4Y4V4V4V4V4V4U3V4V4U3U3U3U3†Z.†Z.†Z.†Z.†Z.†Z.†Z.‡[/‡[/‡[/‰[/‰[/‰[-‰[-‰[-‰[-‹]-‹]-‹]-‹]-‹]-‹]-‹]-‹]-‹]-‹]-‹]-Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^._/_3_3_3_3_3_3_3_3_1_1_1_1_1_1_1Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2_2‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3’a4’a4’a4’a4’a4’a4b4b4b4b4b4b4b4b4b4b4b4b4’d6b4b4b4’a4’a4’a4”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c8”c8”c6”c6”c6”c6”c6”c6”c6”c6’d6’d6’d6’d6’d6’d6’d4’d4”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6’a4’a4”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6’a4’a4’a4”c6”c6’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4b4b4b4b4a3a3a3a3a3a3a3a3a3a3a1a1a1a1a3a3a5a5a3a3‘`1_0a3a3a5a5a5a5Ž`4Ž`4Ž`4Ž`4_1_1Ž`4Ž`4Œ`2Œ`2a3Œ`2‘`3_2“`3“`3’a4’a4˜_(˜_(ža¡d¦b \™T‹Fp1YAK`+o: vC…R%•`/¥p?¹€KÁˆSÅWΘ`Ù£kݧoâ¬rå¯uè´xí¹}îº~îº~é¶uå²qä¯iâ­gߧgÑ™YÃS²|B™d3‘\+_4_4Œ]4‰Z1‰\3‰\3‹]/‹]/‹]1‹]1‹]1‰[/\1\1‰]/‰]/‹]1‹]1‰[/‰[/‰[/Œ^2Œ_8†Y2„W.‰\3}_B3 ?(w`H…X3ƒV1†W2…V1…V1†W2„W0ƒV/ƒV-ƒV-‚U.‚U.‚U.‚U.‚U.‚U.€V.€V.€S,€S,€S,€S,€S*€S*€T(€T(€T(€T(€T(€T(S'S'R+R+R-R-R-R-~Q,~Q,}S+}S+}S+}S+}S+}S+|R*|R*{Q){Q){Q+{Q+{Q+{Q+~Q,~Q,~Q*~Q*~Q*}P)}P+}P+{Q)zP(}P)}P)}P+}P+}P+|O*|O(|O(|O(|O({N'{N'{N'{N'}N'}N'zM&zM&zM(xK&zM(zM(xK&zM(xK$xK$xK$xK$uK#vL$vL&uK%uK%vL&uK#uK#uK#uK#wJ%vI$tJ$tJ$tJ$tJ$tJ$sI#sI#sI#sI#sI#sI#sI#sI#rH"rH"rH"rH"qG!qG!qG!qG!qG!qG!qG!oEoEoEoEoEoEoEoEnD nD nD"nD"mC!mC!mD"lC!lC!kB kB kB kB kB jAjAi@jAjAjAjAi@i@i@j@j@j@j@j@h>f=g>g>f=f=f=c=d>d>d>d>d>c=c=c<b;b;b;c<c<b;b;b;b;b;a:a:a:a:`9`9`9`9`9^9\7\7\7\7\7]6]6]6]6]6]6]6\5Z5Z5Z5Y4Y4Y4V4W5W5V4V4V4V4V4V4V4U3U3‡[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‡[/‰[/‰[/‹]/‹]/‹]/‹]/‹]-‹]-‹]-‹]-‹]-‹]-Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^._/_3_3_3_3_3_3_3_3Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2a3a3a3a3a3a3a3a3_2‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3’a4’a4’a4’a4’a4’a4’d6’d6’d6’d6’d6b4b4b4b4b4b4b4’d6“e7“e7“e7”c6”c6”c6”c6”c6”c6”c6•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c8”c8”c6•d7”c6”c6”c6”c6”c6”c6’d6’d6’d6’d6’d6“e7“e5“e5•d7•d7•d7”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6’a4”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6’a4’a4’a4’a4’a4’a4’a4’a4’a4b4b4b4b4a3a3a3a3a3a3a3a3a3a3a1a1a1a1a3a3a5a5a3a3‘`1‘`1a3a3a5a5a5a5a5a5a5a5Ž`2_1a5a5Œ`2Œ`2a3Œ`2‘`3_2“`3”a4’a4_2˜_(Ÿf/ªm$²u,¸t ±m [ŠEn/\I@D[& {H‰V)še4¦q@®u@µ|Gº„L¾ˆPÅWÌ–^Θ^Θ^Ðœ`Ô dÙ¥iؤhØ¥dפcØ£]ÏšTÛ£cÇO½‡M©s9”_.‘\+_4^3Œ]4Œ]4Š]4Š]4Œ^0Œ^0‹]1‹]1‹]1‹]1\1\1‰]/‰]/‹]1‹]1‹]1‰[/‹]1ˆZ.ƒV/†Y2„W.…X/~`C1;$ u^F„W2„W2†W2†W2‡X3‡X3„W0„W0ƒV-ƒV-ƒV/ƒV/ƒV/ƒV/‚U.‚U.€V.€V.‚U.‚U.‚U.‚U.‚U,‚U,‚V*‚V*€T(€T(€T(€T(€T(€T(€S,R+R-R-R-R-~Q,~Q,}S+}S+}S+}S+}S+}S+}S+}S+}S+|R*|R,|R,|R,|R,~Q,~Q,~Q*~Q*~Q*}P)|O*}P+{Q)|R*~Q*}P)}P+}P+}P+|O*|O(|O(|O(|O(|O(|O(|O({N'~O(~O(zM&zM&zM(xK&zM(zM(xK&zM(zM&xK$xK$xK$vL$vL$vL&vL&vL&vL&vL$vL$uK#uK#xK&wJ%tJ$tJ$tJ$tJ$tJ$tJ$sI#sI#sI#sI#sI#sI#sI#sI#rH"sI#rH"rH"qG!rH"qG!oEqG!qG!oEoEoEoEoEoEoEoEoE!oE!nD"nD"nD"nD"mD"lC!lC!lC!lC!lC!kB kB jAkBjAjAjAjAjAjAi@i@j@j@j@j@j@j@i@i@i@g>g>g>e?d>d>d>d>d>d>d>d=c<c<b;c<c<b;b;b;b;c<c<a:a:a:`9a:`9`9`9^9^9^9^9\7\7]6^7^7]6]6]6]6\5Z5Z5Z5Z5Z5Y4W5W5W5V4W5W5V4V4V4V4V4V4‰[/‰[/‰[/‰[/‰[/‰[/‰Z1‹\3‹]1‹]1‹]/‹]/‹]/‹]/‹]/Œ^0‹]-‹]-‹]-Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^._/_/_/_/_/_/_/_/_/_/_/_/Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`2Ž`2Ž`2a3a3a3a3a3a3a3a3a3a3a3b4b4‘`3‘`3’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4’a4”c6”c6”c6’d4’d4’d4’d4’d4“e5“e5“e5“e5“e5“e5“e5“e5“e5“e5“e5•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d9•d9•d7–e8•d7•d7•d7•d7•d7•d7“e7“e7“e7“e7“e7“e7•d5•d5•d5•d5•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7”c6”c6”c6”c6”c6”c6•d7”c6”c6”c6”c6”c6”c6”c6”c6•d7•d7”c6”c6”c6”c6”c6’a4”c6”c6”c6”c6”c6”c6”c6”c6”c6b4’d6’d8’d8’d8’d8’d6’d6’d8’d8’d6’d6’d6’d6’d6’d6’d4’d4’d6’d6b6b6a3b4’a2’a2’a4’a4b6b6’a6’a6”c8’a6‘`5‘`5’a4_2a3a3Œ`2Œ`2Œ`0Œ`0_-’a/š`š`©g±oÂyÍ„Ö‡ÜÞŠÒ~³[—?v(eQ H=F ^, k9|J$‘_9j;¨uFµ‚NÀYÌ•bΗdÚ›cà¡iݤiÞ¥jÚ§dÛ¨eÞ¦dÚ¢`ÝŸ`Û^דVÄ€C¯q2_ ]+’_-_2^1Š]4Š]4Š]4Š]4Œ^0Œ^0‰]/Š^0Œ^0‹]/Ž]0Ž]0‰]1‰]1‰]1‰]1‰\3‰\3‰\5‰\5Œ]8Œ]8‚Y3‡^8`H/  E&}^FŒV1ŒV1‹W/‹W/…X1…X1„W0„W0†W.†W.†W.…V-…V-…V-ƒV-ƒV-ƒV-ƒV-‚U,‚U,‚U,‚U,‚U,‚U,€S*‚U,‚U,€S*€S*€S*€S*€S*€S,€S,€S,€S,€S,R+R+€S,}S+}S+}S+}S+}S+}S+}S+}S+}S+}S+}S+}S+}S+}S+R+~Q*~Q*~Q*~Q*~Q*~Q*~Q*~Q*}P){Q)zP({Q){Q){Q){Q){Q)zP(zP(zP(zP(zP(yO'zP(zP(yO'{N'{N'{N){N){N){N){N){N){N'zM&zM&zM&xK$zM&xK&xK&xN(vL&vL$vL$uK#uK#vL$uK#uK%uK%uK%tJ$tJ$tJ$tJ$tJ$sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#rH"sI#rH"rH"qG!qG!qG!oEoEoEoEoEoEoE!oE!oE!nD oE!nD nD nD mD"lC!lClClClClCkBkBkBkBkBlBkAkAlBkAkAkAj@j@j@i@i@i@i@g>g>e?e?e?e?e?e?d>d>d>c=c=b<c<c<c<c<a<`;a<a<`;_:a:`9a:a:a:a:^9^9^9^9`9`9^7^7^7^7]6^7]6\5]6]6Z5Z5Z5Z5X6X6W5W5W5W5W5W5W5W5V4V4‹]1‹]1‹]1‹]1‹]1‹]1‹\3‹\3‹]1‹]1‹]/‹]/‹]/Œ^0Œ^0Œ^0Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^.Œ^._/_/_/_/_/_/_/_/_/Ž`0Ž`0Ž`0Ž`0Ž`0Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4a5a5a3a3a3a3a3a3a3a3b4b4b4b4b4b4b4b4’a4’a4’a4’a4’a4’a4’a4’a4”c6”c6”c6”c6”c6”c6”c6”c6“e5“e5“e5“e5“e5“e5“e5“e5“e5“e5“e5“e5“e5“e5“e5“e5•d7•d7•d7•d7•d7•d7•d7–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e:–e:–e8–e8–e8–e8–e8–e8–e8–e8”f8”f8“e7“e7“e7”f8–e6•d5–e6–e6–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8•d7”c6”c6•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7”c6•d7•d7•d7•d7’d6“e7“e9“e9“e9“e9“e7“e7“e9“e9“e7“e7“e7“e7“e7“e7“e5“e5“e7“e7’d8’d8’d6b4’a2”c4”c6’a4b6b6’a6’a6”c8’a6’a6‘`5_2_2a3a3d6Œ`2a1a1_-‘`.š`¨n)ŃØ–/ì£$ö­.ú«!÷¨ñÚ†´\–>s%^N P G -F PY']+`.i6r?}JˆU!–_,žg4±r:¹zB¸D¸D´>±~;­u3§o-¥g(›]”PÄ€CĆG¨j+‘^,“`._2^1‹^5‹^5‹^5Š]4Œ^0Œ^0Š^0Š^0Œ^0Œ^0Ž]0Ž]0Š^2‰]1‰]1‰]1‰\3‰\3‰\5Š]6^9a<‘hB’iCvW?,  -H)}^F‹U0‹U0‹W/‹W/…X1†Y2†Y2„W0†W.†W.†W.†W.†W.†W.ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-‚U,‚U,‚U,‚U,‚U,‚U,€S,€S,€S,€S,€S,€S,€S,€S,~T,~T,~T,}S+}S+~T,~T,~T,~T,}S+}S+}S+}S+}S+R+~Q*~Q*R+R+R+R+~Q*~Q*~Q*|R*|R*|R*{Q){Q){Q){Q)zP(zP(zP(zP(zP(zP(zP(yO'zP(|O(|O(|O*|O*|O*{N){N){N){N'{N'{N'{N'zM&zM&xK&xK&xN(xN(vL$vL$vL$vL$vL$vL$vL&vL&uK%uK%uK%uK%uK%sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#sI#rH"sI#sI#rH"rH"rH"rH"rH"qG!qG!qG!qG!qG!qG#qG#oE!oE!oE!oE!nD nD mD"mD"mD mD lClClClCkBkBkBkBlBlBlBlBlBlBlBkAkAkAi@jAg>i@i@i@e?e?e?e?e?e?e?e?d>d>d>d>d= d= c<c<a<a<a<a<`;`;a:b;b;a:a:a:_:_:^9^9`9`9^7`9`9^7^7^7^7^7]6]6[6Z5Z5Z5X6X6X6X6X6X6W5W5W5W5W5V4‹]1‹]1‹]1‹]1‹\3‹\3‹\3Œ]4Œ^2Œ^2Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^.Œ^.Œ^.Œ^.Œ^.Œ^._/_/_/_/_/_/_/_/_/_/Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2a1a1a1a1a1a1a1a1a3a3a3a3a3a3a3a3b4b4b4b4b4b4b4b4’a4’a4”c6”c6”c6”c6”c6”c6”c6”c6•d7•d7•d7•d7•d7•d7•d5•d5•d5•d5•d5•d5•d5•d5•d5•d5•d5•d5–e6•d5•d5•d5•d5•d5•d5•d5•d5•d5•d5–e6–e6–e6–e6–e6–e6–e6–e8–e8–e8–e8–e6–e6–e6–e6–e6–e6–e6–e6–e8–e8–e8–e8–e8–e8–e:–e:–e8—f9–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e6–e6–e6–e6–e6–e6–e6–e6–e6–e6–e6–e6–e6–e6–e6–e6–e8–e8–e8–e8•d7–e8–e8•d7•d7•d7•d7•d7•d7•d7•d7•d7—d7—d7—d7—d7—d7—d7—d7—d7—d7—d7–c6—d7—d7—d7•d9•d9•d9•d9“e9“e9“e9“e9“e9“e9•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7”c4”c4’d6’d6’d8’d8’d6’d6a8’c:–b<”`:’a4’a4a1Žb2’d6”f8›a8™_6 `¦f·oÀxàŽó¡ÿ­ÿ´ÿºÿ¼ÿ¿ÿ´ú› à²S“4i[KD &+7@I$W*[.c2f5n= xG{K€P‡WŽ^%—e,™g.žj0¢n4¡k1—a'd)ºFÊ”Z¬v<’_+“`,_1_1^3^3_3_3’]1‘\0]2]2Ž]0Ž]0Š^.Š^.Ž\6[5‰[/‹]1‰\3‰\3‰\5Š]6Œ]:†W4c>!N) + - -  - Q6}bD…[3‚X0…X/…X/†Y0†Y0†Y0…X/„W.„W.„W.„W.„W0„W0…V/…V/…V/†W0„W0ƒV/ƒV-ƒV-ƒV-ƒV-ƒW+ƒW+‚U,‚U,‚U,‚U,‚V*‚V*‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.~T*~T*~T*}S)}S)~T*~T,~T,~T,}S+}S+~T,~T,~T,R+R+R+R+R+R+R+R+R+€Q*~Q*~Q*~Q*}P){Q){Q){Q){Q)zP({Q)zP(zP(zP&zP&zP(zP(|O(|O(|O*|O*|O*|O*|O*|O*{N'{N'{N'{N'zM&{N'zM(zM({N){N)zM&zM&zM&zM&xK$xK$xK$xK$xK$wJ#wJ#wJ#wJ%wJ%wJ%wJ%vI$uH#uH#uH#sI#sI#sI#sI#uH#uH#sI#sI#sI#rH"rH"rH"rH"rH"rH rH qG!qG!qG!qG!qG#oE!oE!oE!nD nD oE!nD mD mD mD mD lClCmCmCmCmClBlBlBlBlBlBlBkAkAkAjAjAjAi@i@i@gAgAgAe?e?e?e?e?e?d>d>d>d=d=d=d=b=a<a<`;a=`<_;`<`<`<b;b;a:a:a:a:a:a:^7`9`9^7^7^7^7^7^7]6]6]6[6Z5X6X6X6X6X6X6X6W5W5W5W5W5‹]1‹]1‹]1‹]1‹\3‹\3‹\3Œ]4Œ^2Œ^2Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^.Œ^._/_/_/_/_/_/_/_/_/Ž`0Ž`0Ž`0Ž`0Ž`0Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2a1a1a1a1a1a1a1a1b4b4b4b4b4b4b4b4b4b4b4’d6’d6’d6’d6’d6’a4’a4”c6”c6”c6”c6”c6”c6”c6”c6•d7•d7•d7•d7•d7•d7–e6–e6–e6–e6–e6•d5•d5•d5•d5•d5•d5•d5–e6—f7—f7—f7–e6–e6–e6–e6–e6–e6–e6–e6–e6–e6–e6—f7—f7—f7—f9—f9—f9—f9–e6–e6–e6–e6–e6–e6–e6–e6–e8–e8–e8–e8–e8–e8–e:–e:–e8—f9–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8—f9—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7–e8–e8–e8–e8•d7–e8–e8•d7•d7•d7–e8–e8–e8–e8–e8–e8—d7˜e8˜e8—d7—d7—d7—d7—d7—d7—d7—d7—d7—d7—d7•d9•d9”c8•d9“e9“e9“e9“e9“e9“e9•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d5•d5“e7“e7“e9’d8’d6’d6’c:a8–b<–b<•d7•d7d4‘e5“e7”f8›a8 f=²r(Ã9Þ–í¥.ÿµÿÀ"ÿÂÿ¿ÿºÿºÿ¹ ÿ²ü Þ§H‰*d^N= '3>N) b=wJ!‚U,_-˜g5¦uA°K³ƒL²‚K³ƒJ¼ŒSË™`Ë™`Í™_Í™_È’XÅUºF¤k0¢l2¤n4”a-“`,_1_1Ž]2^3Ž`4Ž`4’]1’]1‘^3]2^1^1‹_/‹_/]7[5‹]1‹]1‰\3‰\3‰\5Š]6†W4l=7 - - X=w\>„Z2ƒY1…X/…X/†Y0†Y0†Y0…X/„W.„W.„W.„W.„W0„W0…V/‡X1†W0†W0„W0„W0ƒV-ƒV-„W.ƒV-ƒW+ƒW+ƒV-ƒV-ƒV-ƒV-ƒW+ƒW+ƒV/ƒV/ƒV/ƒV/ƒV/‚U.‚U.‚U.€V,~T*~T*~T*~T*€V,~T,~T,€V.€V.€V.~T,~T,~T,€S,R+R+R+R+R+R+R+R+€Q*~Q*~Q*~Q*}P)|R*|R*|R*{Q){Q)|R*|R*{Q){Q'{Q'{Q){Q)}P)}P)|O*|O*|O*|O*|O*|O*|O(|O(|O({N'{N'|O(|O*{N){N){N){N'zM&zM&zM&zM&zM&xK$xK$xK$xK$xK$xK$xK&wJ%wJ%wJ%wJ%wJ%wJ%wJ%uK%tJ$sI#sI#uH#uH#sI#sI#sI#sI#sI#sI#sI#rH"rH rH rH"rH"qG!rH"qG#oE!oE!oE!oE!oE!oE!nD nE!nE!mD mD mD mD mCmCmCmCmCmCmCmClBlBlBlBlBkAjAi@jAjAjAi@gAgAgAgAgAgAgAe?e?e?d>d>d=d=d=d=b=b=b=a<a=a=a=a=a=`<b;b;b;a:a:a:a:a:`9`9a:`9`9^7`9^7^7^7]6]6[6[6X6X6X6Y7X6X6X6X6X6X6W5W5Œ^2Œ^2Œ^2Œ^2Œ]4Œ]4Œ]4Œ]4Œ^2Œ^2Œ^0Œ^0_1_1_1_1_/_/_/_/_/_/_/_/_/Ž`0Ž`0_/Ž`0Ž`0Ž`0Ž`0a3a3a3a3a3a3a3a3a1a1a1b2b2b2b2b2b4b4b4b4b4b4b4’d6’d6’d6’d6’d6’d6’d6“e7“e7”c6”c6”c6”c6”c6•d7•d7•d7•d7•d7•d7•d7•d7–e8–e8–e8–e6–e6–e6–e6–e6—f7—f7—f7—f7—f7—f7—f7–e6—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f9—f9—f9—f9—f9—f9—f;—f;—f9˜g:—f9—f9—f9—f9—f9—f9—f9—f9—f9—f9—f9—f9˜g8˜g8˜g8—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7–e8—f9—f9–e8–e8–e8–e8–e8–e8–e8–e8–e8˜e8˜e8˜e8—d7˜e8˜e8˜e8˜e8˜e8˜e8—d7˜e8˜e8—d7–e8–e8•d9•d9“e9“e9“e9“e9“e9“e9•d7•d7•d7•d7•d7•d7•d7–e8–e8–e8–e:–e:–e:–e:–e8–e8”f8”f8”f:“e9“e9“e9a8’c:f2f2‘e5‘e5“e5b2œd" h&¹oÒˆò¡ÿ± ÿ»ÿ¿ÿÂÿÂÿÀ ÿ¹ÿ°ÿ«ÿ©ÿ° ÿ¨Øy’5uYS T_"d:uK%|R*‚X0Ša2’i:žr?§{Hª~K­NµŒXÀ—cË¢qܳ‚羊ìÃëŒḂתrÒ¥mÍbÅ•Z¾ŒS³H£o9˜d.—e.“a*Ž].^/‰`3‰`3‰a4‰a4Ž`-Ž`-_,Ž`-Ž`-_,‹_,‹_,‹^5‹^5‰`3‡^1Œ^2Œ^2_3ˆZ.zP&~T*yU,uQ(G)  - -    [={]8…\/ƒZ-†X,‡Y-ˆZ.ˆZ.†Y0†Y0…X/…X/…X/…X/…X1„W0„W0…X1…X1„W0…X1„W0„W.„W.„W.„W.„X,„X,„X,„X,ƒW+ƒW+ƒW+ƒW+ƒV/ƒV/ƒV/ƒV/ƒV/‚U.‚U.‚U.W-W-€V,€V,€V,€V,€V.€V.‚U.‚U.‚U.€S,€V.€V.€S,€S,€S,€S,€S,€S,€S,€S,‚S,R+R+R+~Q*~Q*}S+}S+|R*|R*|R*|R*|R*|R*|R(|R(|R*{Q)}P)}P)}P+}P+}P)|O(P)~O(|O(|O(|O(|O(|O(|O(|O*|O*{N){N){N'{N'{N'zM&{N'{N'zM&zM&zM&zM&zM&zM&xK&xK&xK&xK&wJ%wJ%wJ%wJ%wJ%wJ%wJ%wJ%uK%tJ$sI#sI#sI#tJ$sI#sI#sI!sI!rH rH rH rH rH"rH"rH$rH$rH"qG!qG!qG!qG!oEoEoEnE!mD mD mD mD lCmD lCmCmCmCmCmCmClBlBkBkBkBkBkBjAjAjAjAi@i@i@gAgAgAgAe?e?e?e?e>e>e>d=d=d=d=d=d>c=a=a=a=a=c<b;b;b;b;b;a:a:a:a:a:`9`9`9`9^7^7^7^7^7]6]6[6[6Z5[6[6[6Z5Z5Z5Z5Z5Z5Œ^2Œ^2Œ^2Œ^2Œ]4Œ]4Œ]4^5_3_3_1_1_1_1_1_1Œ^._/_/Ž`0Ž`0_/Ž`0Ž`0Ž`0Ž`0Ž`0Ž`0a1a1a1a1a3a3a3a3a3a3a3b4b2b2b2b2b2b2b2b2’d6’d6’d6’d6’d6’d6’d6’d6’d6’d6’d6“e7“e7“e7“e7“e7•d7•d7•d7•d7•d7•d7•d7•d7–e8–e8–e8–e8–e8–e8–e8–e8–e6–e6–e6–e6–e6—f7—f7—f7—f7—f7—f7—f7˜g8˜g8˜g8˜g8—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7—f7˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g:˜g:˜g:˜g:˜g:˜g:˜g<˜g<˜g:˜g:˜g:˜g:˜g:˜g:˜g:˜g:˜g:˜g:˜g:—f9—f9—f9˜g8˜g8˜g8˜g8—f7˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8—f7—f7—f7—f7—f9—f9—f9—f9—f9—f9—f9—f9—f9–e8–e8—f9™f9™f9™f9™f9™f9™f9™f9™f9˜e8˜e8˜e8™f9™f9™f9—f9—f9–e:—f;•g;•g;•g;•g;•g;•g;—f9—f9—f9—f9—f9—f9—f9—f9—f9—f9—f;—f;—f;–e:–e8–e8”f8”f8”f:”f:”f:”f:’c:“d;‘h4g3’f6d4’d4Ž`0˜`ªr0Ü’!ý³BÿÂÿÆ"ÿÆ ÿÂÿ½ÿÁÿÂÿ´ÿŸü’þ–ÿ§ÿ¨Ðqƒ&oXP [p3vL&‹a;špH¡wO¤{L¥|M¬€M±…R¶ŠW¼]¼“_ÚfÉ o弋úÑîő羈ݴ~ΡiËžfÌœaÍbÇ•\ÁV¾ŠT³I¢p9—e.’a2’a2Œc6‹b5Šb5‰a4a.a.a.a.a.Ž`-‹_,‹_,‹^5‹^5‰`3ˆ_2_3Ž`4Ž`4‡Y-xN$„Z0„`7Šf=‹mKrT2<%[=‡iD‹b5ˆ_2a5Œ^2‹]1ˆZ.†Y0†Y0†Y0†Y0…X/…X/…X1…X1…X1…X1†Y2†Y2…X1…X1„W.„W.„W.…X/„X,„X,„X,„X,„X,ƒW+ƒW+ƒW+ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/W-W-€V,€V,W-€V,€V.€V.‚U.‚U.‚U.‚U.€V.€V.€S,€S,€S,€S,‚U.‚U.‚U.‚U.‚S,‚S,€S,€S,€S,R+}S+}S+}S+}S+}S+}S+}S+}S+}S)|R(|R*|R*~Q*~Q*~Q,~Q,~Q*}P)P)P)}P)}P)}P)|O(|O(|O(|O*|O*|O*|O*{N'{N'|O({N'{N'{N'{N'zM&{N'zM&zM&zM&zM(xK&xK&xK&xK&xK&xK&xK&wJ%wJ%wJ%wJ%uK%uK%uK%uK%tJ$tJ$tJ$tJ$sI!sI!sI!rH rH rH sI#sI#rH$rH$rH"rH"qG!qG!qG!oEqG!oEnE!nE!mD mD mD mD mD mD nD nD nD nD nD mCmCmClCkBkBkBkBkBjAjAjAjAjAjAhBhBgAgAgAgAe?e?e>e>e>e>e>e>d=d=d>d>b>b>a=a=c<c<c<b;b;b;b;b;a:a:a:a:`9`9`9`9`9^7^7^7^7]6\7\7[6[6[6[6[6Z5[6[6Z5Z5Œ^2Œ^2Œ^2Œ^2Œ]4Œ]4Œ]4^5_3_3_1_1_1_1_1_1_/Ž`0_/Ž`0Ž`0_/Ž`0Ž`0Ž`0a1a1a1a1a1a1a1b4b4b4b4b4b4b4b4b2b2b2’d4’d4’d4’d4’d4•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7–e8–e8–e8–e8•d7•d7•d7•d7•d7–e8–e8–e8–e8–e8–e8–e8–e8—f9—f9—f9˜g8˜g8˜g8˜g8˜g8—f7—f7—f7—f7—f7—f7—f7˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h;™h;™h;™h;™h;™h;™h=™h=™h;™h;™h;™h;™h;™h;˜g:˜g:˜g:˜g:˜g:˜g:˜g:™h;™h9™h9™h9˜g8˜g8™h9™h9™h9™h9™h9™h9™h9™h9˜g8˜g8˜g8˜g8˜g8˜g8˜g8—f7˜g8˜g8—f7—f7—f7—f7—f7—f7—f7—f7—f7˜e6™f7™f7™f7™f7™f7™f7˜e6™f7™f7˜e6™f7™f9™f9—f9—f9—f7—f7•g9•g9•g9•g9•g9•g9—f9—f9—f9—f9—f9—f9—f9—f9—f9—f9—f;—f;—f;—f;—f9—f9—f9—f9•g;•g;”f8•g9•f=•f=‘i0h/”c6”c6›^$“V¥\Їÿ°ÿÁÿÂÿÀÿÆ ÿÂÿÁÿÅ ÿÃÿ·ÿšò‡ùŽÿ›ÿ§Ûw+hUP ^u1 o:J—d0§t@²€G¹‡N½‰QÂŽVÇ•\È–]ʘ]Ìš_Í`Êš]Å•XÁ‘T“SÉšZÕ¤dצfÕ aÒ^×_Õ›]×—OƆ>®l¢`¤`¥aa/žb0”^9“]8—\6˜]7™_6™_6]3]3¤Z1¤Z1ž^4ž^4š^1™]0’]1’]1‘`1„S$}?‹M¡k5£m7žm9¥t@žrD‚V(iAM%7/ '(:\;O- >4 -8Lj=Ž`2šl>™k8§yF©xF‘`.‰[+‹]-‹]1‰[/†Y0†Y0†Z.†Z.†Y0…X/…X/†Y0ˆY0ˆY0ˆY0‡X/†Y0†Y0†Y0…X/‡Y-‡Y-‡Y-‡Y-†X,†X,„X,„X,„W0„W0„W0„W0„W0ƒV/ƒV/ƒV/ƒV-ƒV-‚U,ƒV-ƒV-ƒV-ƒV/ƒV/ƒV/‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚S,„U.‚U.‚U.‚U.€S,~T,~T,€S,€S,€S,R+R)R)R)R)R+R+R+R+R+R+~Q*~Q*~Q*~Q*€Q*P)€Q*P)}P)|O(|O*|O*|O(|O(~O(}N'~O(~O({N'{N'{N'{N'{N'{N'zM&xK$zM(zM(xK&zM(zM(xK&xK&xK&xK&xK&wJ%xK&wJ%wJ%wJ%wJ%wJ%wJ%tJ$tJ$tJ$sI#tJ$sI#rH"rH"sI#sI#sI#rH"rH"rH"rH"rH"rH"qG!qG!qG!qG#oE!nEnEnEmDmDmDmD mD mD mD mD mD mD lClClCkBkBjAkBkB kB jAjAjAjAjAjAjAi@gAgAgAgAe>e>e>e>e>e>e>d=d>d>d>d>d>d>d=c<c<b;c<b;d:d:d:d:d:d:d:`6b8b8b8b8`9^7^7]6\7\7[6[6[6[6[6[6[6[6[6Z5_3_3_3_3^5^5^5^5_3Ž`4Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2Ž`0Ž`0a1a1a1a1a1a1a1a1a1a1b2b2b2b2b4’d6’d6’d6’d6’d6’d6’d6’d4’d4’d4’d4’d4“e5“e5“e5•d7•d7•d7•d7•d7•d7•d7•d7•d7–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8—f9—f9—f9—f9—f9—f9—f9—f9˜g:˜g:˜g8™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9˜g8˜g8˜g8˜g8™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h;™h;™h;™h;™h;™h;™h=™h=™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8™f7šg8šg8šg8šg8šg8šg8šg8šg8šg8˜e6˜e6šg:šg:˜g:˜g:˜g8—f7•g9•g9•g9•g9•g9•g9—f9—f9—f9—f9—f9—f9—f9˜g:˜g:˜g:˜g<˜g<˜g<˜g<˜g:˜g:˜g:˜g:•g;•g;”f8•g9•f=•f=h/h/_2ŠY,ŠM…H³jì£5ÿÁÿÀÿ¼ÿ¹ÿ¶ÿ¸ÿ¼ÿÀÿÀÿ¾ ÿª ÿ•ûÿ™ÿ¥脇1dUP \l(_*o:…Ršg3«y@·…L¾ŠRÄXʘ_Ë™`ÎœaÑŸdÕ¥hÙ©lâ²uðÀƒñ‚ë¼|ä³sß®nâ­nâ­ná§iÜ¢dÕ•M¿7®l§ežZ—S’V$˜\*•_:“]8œa;•Z4‘W.’X/–V,–V,œR)›Q(“S)•U+“W*“W*X,‘\0’a2„S$w9t6ƒMg1ªyE°Kµ‰[·‹]¹‘d°ˆ[¦‚W˜tIoE“rHnJœ{WŽlK’pOœrVžtX©|W³†a³…W³…W¼Ž[Ç™f·†T’a/ˆZ*‹]-‹]1‹]1‡Z1‡Z1‡[/‡[/†Y0†Y0†Y0‡Z1ˆY0ˆY0‰Z1ˆY0†Y0†Y0†Y0†Y0‡Y-‡Y-ˆZ.‡Y-‡Y-‡Y-…Y-…Y-…X1…X1…X1„W0…X1„W0„W0„W0ƒV-ƒV-„W.ƒV-ƒV-ƒV-ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/‚U.‚U.‚U.ƒV/ƒV/‚U.‚U.„U.„U.‚U.‚U.‚U.‚U.~T,~T,€S,€S,€S,€S,€S*€S*€S*€S*€S,R+R+R+R+R+~Q*R+R+~Q*€Q*R+€Q*€Q*}P)}P)~Q,}P+|O(}P)P)P)~O(~O(}P)|O(|O(|O({N'|O({N'{N'{N)zM(zM(xK&zM(zM(zM(zM(zM(xK&xK&xK&xK&xK&wJ%wJ%wJ%wJ%uK%tJ$tJ$uK%uK%tJ$sI#sI#sI#tJ$sI#sI#sI#sI#sI#rH"rH"rH"rH"qG!qG#qG#nEnEnEnEnEnEnE!mD mD mD mD mD mD mD lClClClClClCkB kB kBkBkBkBjAjAjAjAhBgAgAgAg@ g@ g@ g@ e>e>e>e>e?e?e?d>d>d>d=d=d=d=c<b;d:e;d:d:d:d:d:c9b8b8b8b8`9`9`9]6^9^9\7\7\7\7\7\7[6[6[6[6^3^3^3^3^3^3^3_4_4_4_2_2_2_2_2_2Ž`0a1a1a1a1a1a1a1a1a1a1b2b2b2b2b2’d6’d6’d6’d6’d6’d6’d6’d6’d4’d4’d4“e5“e5“e5“e5“e5•d7•d7•d7•d7•d7•d7•d7–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8—f9—f9—f9—f9—f9—f7—f7—f7—f7—f7—f7˜g8˜g8˜g:˜g:˜g:™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;›j=›j=›j=›j=›j=™h9›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=™h;™h;™h;›j;›j;›j;›j;›j;›j;›j;›j;›j=›j=›j=›j=›j;›j;›j;›j;™h9™h9˜g8˜g8˜g8™h9˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8™h9›h9šg8šg8šg8šg8šg8šg8šg8šg8šg8šg8šg8šg:šg:šg8šg8˜g5˜g5˜g8˜g8˜g:˜g:˜g:˜g:šg8šg8šg:šg:šg:šg:šg:šg:šg:šg:›f8›f8˜g:˜g:šg:šg:˜g:˜g:—f;—f;–e8—f9še9™d8™b/–_,’O!‚?†4˜FÛÿ¯'ÿ¹"ÿ±ÿž(ì‰è€öŽ"ÿžÿ¦$ÿ²ÿº!ÿ¸"ÿ¤ÿ•ý‘ÿ£ú‘œKjOOj z0k%r,…AšV­l)¹x5¸|?¿ƒFÅŠOΓXÑ—WÖœ\Þ¦hâªlæ®põ½ôºxí³qð´nç«eë¨_ë¨_î£Nã˜CÝã–ú©ûªåÂl¥K“9Œ5‡0Ž68™7•3¡,¡,¾,À.Ø#Ú%ß*à+Ð-Ó0½<¿>¬B¡7 †6™Iœ_'®q9ºHÇŽUÌ“ZÓšaÚ¤hÛ¥iá­qܨlܨpà¬tá­wã¯yå®}Ü¥tÞ¤rߥsݧoâ¬tܦnΘ`´M–c/\-Ž].\1\1‰Z1‰Z1‰Z1‰Z1‰[/‰[/‰Z1‰Z1‰[/‰[/‰Z1‰Z1†Y0‡Z1ˆY0‰Z1‰Z1ˆY0ˆZ.ˆZ.ˆY0ˆY0…Y-†Z.‰Z3‹\5‰\5„W0‡X1ˆY2‡X1‡X1„W0„W0„W0ƒV/…V/†W0„W.„W.ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/ƒV-ƒV-W-W-ƒV-ƒV-„U,…V-ƒV-ƒV-ƒV-‚U,‚U,‚U,„U,„U,„U,‚S*‚S*‚S*€S,€S,€S*€S*‚S*‚S*‚S*R)R)R)R+R+R+R+R+~Q*R+R+~Q*~Q*~Q*}P)€Q*P)P)P)P)~O(|O(|O(|O(|O(|O({N'{N'zM&{N){N){N){N){N)zM(zM(xK&xK$xK$xK&xK&xK&wJ%wJ%xK&xK&wJ%wJ%xK&uK%uK%uK%tJ$tJ$tJ$tJ$sI#sI!sI!sI#sI#rH"rH"rH"rH"rH$qG#qG!qG!qG!oEnEnEnEnEnEnEoEnDmCnD mD lClCmD mD lClCkBkBjAkBkBjAjAjAjAjAjAjAi@i? i? i? i? i? i? i@g>e?e?e?e?e>d=d=d=d=d=c<d=f<e;e;d:d:d:d:d:b8c9c9b8c9c9a:^7`9`9`9^7\7\7\7\7\7\7\7[6_4_4_4_4_4_4_4_4‘`5‘`5‘`3‘`3‘`3‘`3‘`3‘`3a1a1a1a1a1a1b2a1a1’d4’d4b2b2b2b2’d4’d6’d6’d6’d6’d6’d6’d6“e7“e5“e5’d4“e5“e5“e5“e5“e5–e8–e8–e8–e8–e8–e8–e8–e8–e8—f9—f9—f9—f9—f9—f9—f9–e8—f9—f9—f9—f9—f9—f7—f7˜g8˜g8˜g8˜g8˜g8˜g8˜g:™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;›j=›j=›j=›j=›j=›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=œk<œk<›j;›j;›j;›j;›j;›j;›j=›j=›j=›j=›j;›j;›j;›j;›j;›j;›j;›j;™h9™h9˜g8˜g8™h9™h9™h9™h9™h9™h9™h9™h9›h9›h9›h9›h9›h9›h9›h9›h9›h9šg8›h9›h9šg:›h;›h9›h9˜g5˜g5˜g8˜g8˜g:˜g:˜g:˜g:™f7šg8šg:›h;›h;›h;›h;›h;›h;›h;œg9›f8™h;™h;›h;›h;˜g:˜g:˜g<™h=™h;˜g:œg;še9‹T!€I{8 -u2Ž<ªX çÿ©!ýç‡Æc©F°HÁYÔkÞué‰ù™ÿ¡ ÿ  -ÿ˜ùÿœÿž°_s"N Or(’HŠDŠD…AK ˜W¤c ­q4·{>ÅŠOЕZÑ—WÚ `ߧià¨jã«mä¬ná§eâ¨fè¬fã§aç¤[Þ›RÞ“>׌7ß’ÿ³8ÿØ2ÿÒ,ÿË.ÿ¼õ›&Âh‹4qffgl -ˆŠª² ÍÔ Ù$Ö! Á¾¤#©(3 ¢8‘AœL’U¡d,©p7¸F¹€GÅŒSË•YΘ\×£gß«oå±y츀ôÀŠöÂŒö¿Žè±€ã©wÚ nݧoÓeÏ™aÇ‘Y¬yE‘^*Ž].Ž].Ž]2Ž]2‹\3‹\3‹\3‹\3‹]1‹]1‰Z1‹\3‰[/‹]1‰Z1‹\3‡Z1‡Z1‰Z1‰Z1‰Z1‰Z1‰[/‰[/ˆY0‰Z1†Z.†Z.‰Z3ˆY2†Y2…X1‡X1ˆY2ˆY2‰Z3†Y2…X1„W0ƒV/†W0†W0…X/…X/„W0ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/ƒV-ƒV-W-W-ƒV-ƒV-„U,…V-ƒV-ƒV-ƒV-‚U,‚U,‚U,„U,„U,„U,„U,„U,‚S*€S,€S,€S*€S*‚S*‚S*‚S*‚S*€S*€S*R+€S,R+R+R+R+R+R+R+~Q*~Q*R+€Q*€Q*€Q*P)P)P)}P)|O(|O(|O(|O({N'{N'|O({N){N){N){N){N){N){N){N)zM&zM&{N)xK&xK&zM(xK&xK&xK&xK&xK&xK&vL&vL&uK%uK%tJ$uK%tJ$tJ$sI!sI!sI#sI#sI#sI#sI#rH"rH$rH$qG!qG!qG!qG!pG!pG!pG!nEnEnEoEoEoE!nD mD mD mD mD mD mD lCkBkBlCkBlClCkBjAjAkB i@jAjAj@!j@!j@!i? i? i? i@i@gAe?e?e?e>e>e>d=d=d=d=d=f<d:e;e;e;e;e;d:d:d:d:d:c9c9a:`9`9`9`9`9^9\7\7\7\7\7\7[6_2_2_2_2_4_4‘`5‘`5‘`5‘`5‘`3‘`3‘`3‘`3‘`3‘`3b4b4b4b4Žb2Žb2d6d6’d6’d6’d6’d6’d6’d6’d6’d6”c4•d5•d5•d5•d5•d5•d5•d5•d5•d5•d5•d5•d5–e6–e8–e8–e6–e6–e8–e8–e6–e6–e6–e6–e6—f7—f7—f7—f7—f7—f9—f9—f7—f7—f7—f7—f7—f7—f7˜g8˜g8˜g8˜g8˜g8˜g8™h9™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;›j=›j=›j=›j=›j=›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;j;j;j=j=j=j=j=j=j=j=j=j=j=j=j=j=j;j;j;j;j;j;j;j;j;j;j;j;j;j;j;j;œk9œk9›j8›j8›j8›j8›j8›j8›j=›j=›j=›j=›j;›j;›j=›j=›j8›j8›j;›j;j;›h9›h9›h9›h9›h9›h6›h6›h6›h6›h6›h6™h9™h9™h9™h9™h9™h9™h9™h9™h9˜g8™h9™h9˜g8™h9™h6™h6™h6˜g5šg5šg5šg8šg8˜g8˜g8—f4˜g5˜g:™h;™h9™h9›i2›i2™h=™h=—i;—i;–h8–h8f:f:’i<’i<¡e3¢f4²b0°`.®^*M=.‡,˜=¨HºZÖuÛz½fŸHr)`W" -b-w3y5“7¨L×fðõ‡÷‰ø”ÿ§ Õ~8QP n'‘JªUÄoÎuÃjÀf¼b¹b¿hÄoÐ{Ü‹ä“ êœ,ï¡1ô§3ý°<õ¨*êæ—ä•â’Û‹ä‘ô¡ÿ±ÿÂÿÈÿºÿµÿ´ÿ³ÿ®ÿ¤ó—ç…Â` 8‰!wvƒ£ ©&© © » ¶·´ © ¯–+ ž3”T,£c;³sIÀ€V¾~OÅ…VÊYÏ’^Ó–^Ö™aÙš`Ùš`Øœ_Ù`Ý¡dá¥hܤfܤfߤiÜ¡fÖ›d½‚Kf5”],^1^1Ž\4Ž\4\-Ž].Œ^0Œ^0Œ]4‹\3[5[5‰]/‰]/‰]1‰]1‹]1‹]1‰Z1‰Z1‰Z3‰Z3‰Z3‰Z3„Z0…[1ˆY0ˆY0†Y0„W.X.X.X4X4‚Y7|S1ƒZ)‚Y(…W+…W+ƒV-„W.ƒY1ƒY1„W.„W.W-W-ƒV-ƒV-…V-…V-„W0„W0„W0„W0†W0†W0…V-…V-ƒV/ƒV/ƒV/ƒV/…V/…V/…V/…V/ƒV-‚U,‚U,‚U,‚U,‚U,„U,„U,‚U,€S*€S*€S*‚S*‚S*€S*€S*€S*R)R+R+R+R+R+~Q*R+R+€Q*€Q*€Q*P)P)P)P'P'}P'|O&|O&|O&|O(|O(|O*|O*|O*|O*{N){N)|O({N'zM&{N'{N)zM(zM({N)xK$xK$xK$xK$xK$xK$vL&uK%uK%vL&uK%uK%uK%sI#tJ$uK%tJ$tJ$sI#sI#sI#sI#sI#rH"rH"qG!qH"pG!pG!pG!pG!pG!pG!pG!nEnEnEnEmDmDkEkEmDmDmD mD lClCmD lClClClClCkB kB kB jAjAjAjAjAi@i@i@i@gAgAgAe?e?e?e>e>e>d=d>d>f=f=e<e<e;e;e;e;d:d:d:d:d:c9c9c9c9c9b8b8`9`9`9`9^7^7^7^7_2_2_2_2_4‘`5‘`5‘`5‘`5‘`5‘`3‘`3‘`3‘`3’a4’a4b4b4b4b4Žb2d4d6d6’d6’d6’d6’d6’d6’d6“e7“e7•d5•d5•d5•d5•d5•d5•d5•d5•d5•d5•d5–e6–e6–e6–e8–e8–e6–e6–e8–e8–e6–e6–e6—f7—f7—f7—f7—f7—f7—f7—f9—f9—f7˜g8˜g8˜g8˜g8˜g8˜g8˜g8™h9™h9™h9™h9™h9™h9™h;™h;™h;™h;™h;™h;›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j=›j;›j;›j;›j;›j;›j;›j;›j;›j;œk<œk<œk<œk<œk<œk<œk<žk<žk<žk>žk>žk>žk>žk>žk>žk>žk>žk>žk>žk>žk>žk>žk>j;žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žkœk>œk>œk>œk<œk<œk>›j=›j8›j8›j;›j;j;j;j;j;›h9›h9›h6›h6›h6›h6j8j8›j;›j;›j;›j;›j;›j;›j;›j;™h9™h9™h9›j;›j;™h9›j8›j8™h6˜g5šg5šg5šg8šg8˜g8™h9™h6™h6™h;™h;›j;›j;›i2k4›j?™h=—i;™k=™k;™k; i= i=“j=•l? d2žb0ªZ(¢R •E„4‡4Š7B¦K«K²R»Z´S9w V F08M Rl{²AÓbé{ð‚ï‹ÿ¢í–¬U^P o(ŠCžI¸c Ý„óšÿ¦"ÿ¨$÷  óœäÒ}ÊyÅtÆxÍÌ Í€ ׊ á”ëœó¤ÿ² ÿÄ2ÿÏ+ÿÐ,ÿÄÿµÿ­ÿ®ÿ¼ÿÉÿÐÿÔÿÕ)ÿÕ)ÿÏ3ÿ±öŽÓk³K 5•#˜–’ ¥¨± -·¯¯‹ “( -g'r2 -ˆH™Y/¤d5²rC¿‚NƉUÏ’ZÖ™aÜcá¢hߣfÛŸbÛŸbØœ_ИZÌ”VÈM³x=¤i2—\%“\+“\+^1Ž]0Ž\4]5^/Ž].Œ^0Œ^0Œ]4Œ]4[5Ž\6Š^0‰]/‰]1‰]1‹]1‹]1‰Z1‰Z1Œ]6‹\5Œ]6‹\5„Z0„Z0†W.‚S*ƒV-}P'tK!wN$xO+zQ-xO-tK)xOyP~P$S'~Q(ƒV-„Z2ƒY1„W.…X/‚X.‚X.„W.„W.†W.†W.„W0„W0…X1…X1‡X1†W0†W.†W.„W0„W0„W0„W0†W0…V/…V/…V/ƒV-ƒV-ƒV-‚U,‚U,‚U,„U,„U,‚U,€S*€S*€S*‚S*‚S*€S*€S*€S*€S*€S,€S,‚S,‚S,R+R+R+R+R+€Q*€Q*€Q*P)P)€Q(€Q(}P'}P'}P'}P'}P)|O(|O*|O*|O*|O*|O*|O*|O(|O(|O(|O(|O*|O*{N){N){N'{N'{N'zM&zM&xK$vL&uK%vL&vL&vL&uK%uK%uK%uK%uK%tJ$tJ$tJ$tJ$tJ$sI#sI#sI#rH"sI#qH"qH"pG!pG!qH"pG!pG!pG!pG!nEpG!nEnEmDkEkEnEnEmD mD mD mD mD mD lClClClClC!lC!kB kB kB kB jAjAjAi@i@i@gAgAgAgAe?e?e>e>e>e>d>d>f=f=f=f=e;e;e;e;d:d:d:d:d:d:c9c9c9c9c9c9`9`9`9`9`9^7^7^7‘`3‘`3‘`3‘`3‘`5‘`5‘`5’a6’a6’a6’a4’a4’a4’a4’a4’a4b4b4’d6’d6d6d6’d6’d6’d6’d6’d6“e7“e5“e5“e5“e5•d5•d5•d5•d5•d5–e6–e6–e6–e6–e6–e6–e6–e6–e6—f9—f9—f7—f7—f7—f7—f7—f7—f7—f7—f7˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g5˜g5™h6™h6™h6™h6™h6™h6™h9™h9›h9›h9›h9j;j;j;j;j;j;j;j;j;j;žk<žk<žk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<Ÿl:Ÿl:Ÿl=Ÿl=žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žk<žkœk>›j8›j8j8j8j;j;j;j;j;j;j8j8›h6›h6j8j8›j;›j;›j;›j;›j;›j;›j;›j;›j;™h9›j;›j;›j;›j;›j;›j;™h9˜g8›h9›h9›h6šg5™h6›j8™k8—i6—i;—i;™k=™k=™h9›j;•l;•l;–m>–m>™m=™m=£i<£i<¿a2¼^/æMáHë; Þ.°.­+¨>°F³M´N®G¨A>“4a P.! Li0¸TÒnìxíyê{ùŠþÚyˆ7a{%™C­JÁ^á}ê†þšÿ£ ÿªÿ²ÿ¹ ÿ¿ÿÀÿ¹ ÿ¶ÿ¸ -ÿ¶ -ÿ³ÿ´ ÿ¼ÿÇÿÑ(ÿÔ-ÿÔ-ÿÎ'ÿ¾ÿ«ÿ¥ÿ¹ÿÇÿÎÿÔÿÚÿÖÿÓÿÓÿÐÿÂÿ®ü–áo -¹GŽ-„#y*…6‡<‚7x$gJJ9 : -=R(V, -N$QR [(b/r8 ~D„HŠN”W!™\&–[ ™^#–[ “XŽXWŒQ„I‰K”V%¤^6§a9 \8Ÿ[7š]7š]7“^2’]1^/^/]5]5Ž\4]5‰`1ˆ_0Š^0Š^0Œ^0Œ^0\1\1^7Œ]6Ž_8‹\5‹Y1‹Y1‰[/†X,Š^2•i=£uI¯Uºˆ`¦tLŸrK®Zµˆc®\§xSœmH”f:b6‰[-ˆZ,…X/†Y0„Z2„Z2…X/…X/‡X/†W.‡X/‡X/‡Y-‡Y-‰X-‰X-‰W/ˆV.†W0†W0„W0„W0†W0…V/…V/…V/ƒV-ƒV-ƒV-ƒV-ƒV-ƒV-…V-…V-…V-„U,„U,„U,„U,„U,„U,„U,€S,€S,€S,€S,€S,€S,€S,€S,€S,R+R+R+R+R+€Q*€Q*€Q(€Q(€Q(€Q(}Q%~R&~R&~R&}P)|O(|O(|O(|O(|O(|O*|O*{N'|O(|O(|O(|O*{N){N){N){N'zM&xN&vL$xN(xN(vL&vL&vL&vL&vL&uK%uK%vL&uK%uK%tJ$tJ$tJ$tJ$tJ$sI#rH"sI#rI#qH"rI#rI#qH"qH"qH"pG!qH"qH"pG!pG!pG!pG!pG!pG!oEoEnE!nE!mD mD mD mD mD mD lClClC!lC!lC!lC!kB kB kB jAjAjAjAi@i@i@i@i@i@i@g>g>g=g=g>g>f=f=f=f=f=e<e<e<c<c<c<b;d:d:c9c9c9c9c9c9`9`9`9`9`9^7^7^7‘`3‘`3‘`3‘`3‘`5‘`5’a6’a6’a6’a6’a4’a4’a4’a4’a4’a4’d6’d6’d6’d6d6d6’d6’d6’d6“e7“e7“e7“e5“e5“e5“e5•d5•d5–e6–e6–e6–e6–e6–e6–e6–e6–e6—f7—f7—f7—f9—f9—f7—f7˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8™h9™h9™h9™h9˜g8˜g8™h9™h9™h9™h9™h6™h6™h6™h6™h6™h6™h6›j8›j;›j;›h9›h9j;j;j;j;j;j;j;j;j;žk<žk<žk<žk<žk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œkœk>œk9œk9žk9žk9žk<žk<žk<žkšn>˜l<Ÿe8Ÿe8¶X)²T%ÜCÛBç7ß/¹7Â@ºP»Q¸R³M«D 9Ž/‚#RD( ;h/Á] Ùu!àlàlÞoçxó’Xp{%˜B±NÄaß{îŠú–ý™û›þžÿ£ÿ§ÿ«ÿ¯ÿ³ÿµÿ¶ -ÿ¸ ÿ·ÿ³ -ÿ°ÿ­ÿ©ÿ² ÿ² ÿ¥ú™õ”ÿµ ÿÕ+ÿÖÿÎÿÏ -ÿÏ -ÿÏ -ÿÏ -ÿÎ ÿÍ ÿ·ÿ™Ùg«9…$‚!p!w(4ˆ=Š6+`J*)(2CK!Y'c1 n;o<w=~D‰MŠNP‘T›`%§l1°u:µz?µC¼†JÀ…NºH›],„F’L$›U-Y5žZ6š]7š]7”_3”_3‘`1_0]5]5^6^6‰`1‰`1‹_1Š^0Œ^0Œ^0Ž]2\1^7`9^7ˆY2ŠX0[3•g;®€TÉqج€ä¶Šâ´ˆÀŽf´‚ZУ|湒ת…ÉœwǘsÄ•p¾d®€T›m?Ž`2„W.„W.ƒY1„Z2…X/…X/‡X/‡X/ˆY0ˆY0ˆZ.‡Y-‰X-‰X-‰W/‰W/‡X1‡X1…X1…X1†W0†W0†W0†W0„W.„W.„W.„W.„W.ƒV-…V-…V-…V-…V-…V-…V-„U,„U,„U,„U,‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.‚U.€S,€S,R+€S,€S,R+R+R)R)€Q(€Q(}Q%~R&~R&~R&}P)}P)}P)|O(}P)}P)}P+}P+}P)|O(|O(|O(|O*|O*|O*{N){N'{N'yO'yO'xN(xN(xN(xN(xN(vL&vL&uK%vL&vL&vL&uK%uK%tJ$tJ$uK%uK%tJ$tJ$tJ$rI#rI#rI#rI#rI#rI#qH"qH"qH"qH"qH"pG!pG!pG!pG!pG!qG!oEnE!nE!nE!nE!nE!mD mD mD mD mD lC!lC!lC!lC!lC!kB kB kB jAjAjAjAjAjAi@i@i@i@i@g>g=g=g>g>g>g>f=f=f=e<e<e<c<c<c<c<e;e;d:d:d:c9c9c9a:a:a:a:`9`9`9`9’a4’a4’a4’a4’a4’a4’a6”c8”c6”c6”c6”c6”c6”c6”c6”c6’d6’d6’d6’d6d6d6’d6’d6’d6“e7“e7“e7“e5”f6”f6“e5•d5•d5–e6–e6–e6–e6–e6–e6–e6–e6–e6—f7—f7—f7—f7—f7—f7˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8™h9™h9™h9™h9™h6™h6™h6™h6™h6›j8›j8›j8›j8›j8›j8›j8›j8›j8›j;›j;j8j8j8žk9žk9žk9žk9žk9žk9žk9žk9žk9žk9žk9Ÿl:Ÿl:Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl=Ÿl=Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=l:l:l:l:l:l:l:l:l=l=l:l:l:l:Ÿl=Ÿl=žk7žk7žk9žk9žk<žk<žk<žkœk>žk<›h9›j8œk9›j8›j8›j=™h;œk9l:žk¨l?¥i<Å`3Â]0çN&ÝDÿ6ÿ5 ÿ-ÿ+ÿ2ÿ1ÐIØQÎZÏ[ÆX¸J©@œ3†*wJ;h,‘U*¿YÂ\Í_×iâ}î‰ Ûy -¦D“,¦?¼PÍaå{ôŠþ•ÿ™ÿœÿ ÿ¡ÿ¡ÿ ÿ£ÿ£ÿ¥ÿ¨ÿ«ÿ¬ÿ¬ÿ¬ÿªÿ©ÿ¨ÿ¨ÿ¨ÿ¤ö•ïÿ©ÿÅÿÌ#ÿÑÿÐÿÔÿÓÿÑ&ÿÜ1ÿÂ2öŒÎT­3•)#†#rxvz"g'SDK"Q-Z6oE}S)‹]1–h<¥tEªyJ°}K¼‰WÈ•aÐiÖ¢jÙ¥mÚ¨oÝ«rá¯tå³xÕ¨lÒ¥iÍbÂ’W¾|GœZ%¢. ž*Â-Ä/¾2Ç;ÁG!ÆL&¹T'¹T'­X.­X.¡Z7¡Z7•^2”]1Œ^2_3‰`3‰`3_1_1‰[/Ž`4‹]1‹]1ˆW*‹Z-–e:­|Q¯S©{M}T2b9-6[E;\Fg>g>g>g>f=f=f=f=f=c<c<c<c<e;e;d:d:d:d:d:d:a:a:a:a:a:`9`9`9’a4’a4’a4’a4’a4’a4”c8”c8”c6”c6”c6”c6”c6”c6”c6”c6’d6’d6“e7“e7‘e7‘e7“e7”f8”f8“e7”f8”f8”f6”f6”f6•g7—f7–e6–e6–e6–e6—f7—f7—f7—f7—f7—f7—f7—f7˜g8˜g8˜g8˜g8˜g8˜g8˜g8™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9›j;›j;™h6›j8›j8›j8›j8›j8›j8›j8›j8›j8›j8›j8›j8œk9œk<œk m; m; m; m;Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl= m>l:l:l:l:l:l:l:l:l=l=l:l:l:l:Ÿl=Ÿl=Ÿl8Ÿl8Ÿl:žk9žk<Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:l=œkl?l?œk>žk<žk<›j8l:l:l:l?l?œk9œk9¡n?žk<ŸiBŸiB—n?˜o@¦j=¡e8½X+·R%Ö=Ô;ü/ÿ7ÿ5ÿ5ÿ8ÿ=ßXã\ÖbÖbÈZºL§>š1…)vI ;Bm1©C³M½OÈZÑlã~ç…Ík©B¨A»OÎbäzòˆþ•ÿ›ÿ ÿ¢ÿ¤ÿ£ÿ£ÿ£ÿ£ÿ¤ÿ¤ÿ¦ÿ¤ÿ¤ÿ¦ÿ¨ÿ¨ÿ¨ÿªÿ¬ÿ¬ -ÿ« ÿ¢ú›÷šÿ¤ÿ¼ÿËÿÓÿÕÿÔ)ÿË ÿ Øn¿E®4›/”(’/ˆ%wghnP^zQ'‰`6‘mFœxQ£yO¥{Q¬~R°‚Vº‰Z»Š[ÁŽ\Å’`Ó læ³üÈÿÏ—ûÉð¾…å³xÝ«pËžbÈ›_Å•Z¾ŽSÇ…PÆ„O»G"ˆ©ª¨³' -«1 ±7¥@©DœGŸJ ˜Q.V3•^2™b6’d8’d8Že8Œc6a3a3Ž`4Ž`4„V*‡Y-_2”c6Ž]2‘`5’d6’d69- -e<œsQŸsG˜l@“g;‘e9Žb6a5_3‰[/‰[/‰[/‰[/‰[/‹Z/ŠY.ŒX0ŒX0ˆY2ˆY2†Y2†Y2‡X1‡X1‡X1‡X1…X/…X/…X/…X/„W.„W.†W.†W.†W.…V-…V-…V-…V-…V-…V/…V/‚U.‚U.ƒV/ƒV/‚U.‚U.‚U.‚U.€V,€V,€V,€V,€S,€S,‚S,‚S,‚S*‚S*‚S*R)S'S'S'S'R+R+~Q*~Q*~Q*~Q*~Q,~Q,}P)}P)}P)}P)}P+}P+}P+|O*|O(|O({N'{N'yO)yO)yO)yO)yO)xN(xN(xN(xN(xN(xN(vL&vL&vL&vL&vL&uK%uK%uK%tJ$sJ$sJ$sJ$sJ$rI#rI#rI#rI#rI#rI#qH"qH"qH"qH"qH$qH$qG#qG#qG#qG#pG#pG#nE!nE!nE!nE!nE!mD mD mD mD"mD"lClCkBkBkBkBkB jAjAjAjAjAjAjAi@i@i@i@i@i@g>g>g>f=g>f=f=f=c<c<c<c<e;e;d:d:d:d:d:d:a:a:a:a:a:`9`9`9”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6”c6•d7•d7”c6”c6”c6“e7“e7“e7“e7‘e7’f8”f8”f8”f8”f8”f8”f8”f6•g7•g7•g7—f7—f7—f7—f7—f7—f7—f7˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8™h9™h9™h9™h9™h9™h9™h9™h9›j;›j;›j;›j;›j;›j;›j;›j;›j;›j8›j8›j8›j8›j8›j8›j8›j8›j8›j8œk9œk9œk9œk9œk9œk9žk9žk9žk9žk9žk9žk9žk9žk9žk9žk9žk9Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl: m; m; m; m; m; m; m; m; m;¡n< m; m; m; m; m; m; m; m; m; m; m; m; m;Ÿl:Ÿl: m> m> m> m> m> m> m> m> m> m> m> m> m> m>Ÿn<Ÿn< m;¡n< m; m; m; m; m> m> m> m> m>Ÿl=Ÿl=Ÿl=Ÿl8Ÿl8Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl=Ÿl=l=œkÿF ï]óaâiàgÒ[ÃL²?§4Ž+€VF3s5 ~@; ‘=±OÈfèë„ -¿P®?½KÓa -äzó‰þ•ÿ›ÿŸÿ ÿ ÿ ÿ ÿ¡ÿ¡ÿ¢ÿ¢ÿ£ÿ£ÿ£ÿ¢ÿ¡ÿ ÿ¡ÿ¢ÿ¤ÿ¦ÿ©ÿ©ÿ¨ÿ£ÿ ü™ø•ÿÿ¦ÿ¦ ÿ˜ëÑgÄ\¿W¼XÀ\Òmå€Û‡Àl(j[x! ¡Q£S e,³x?¶Nº…R¾‡TÃŒYÁW¾ŠTÂ’YÇ—^Å•\ÏŸfÙ¥kܨnß«oÞªnÚ¤jÖ fÖ£dÓ aËšVÅ”PÇ’L͘RÉ`$}µÀèó*ý$õêæÖÖÅË$ À1Ä5»@ÁF#ºO(¾S,°X/³[2©Z/¦W,„F…G—`-¢k8›h6šg5›f3¦q>|R(X.@6( -( -*69( -1Z+“d;o<•g4”f3–h5 tD tD“e7‹]/‰\3‰\3‰]1‡[/‹\3‰Z1‰Z1‰Z1ˆY2ˆY2ˆY2ˆY2‡X1‡X1‡X1‡X1…X/…X/…X/…X/…X/…X/‡X/‡X/†W.†W.†W.†W.†W.†W.†W0†W0ƒV/ƒV/ƒV/ƒV/ƒV/ƒV/‚U.ƒV/W-€V,€V,€V,‚U.‚U.‚U.‚U.„U,‚S*‚S*‚S*S'‚T(‚T(‚T(‚S,R+R+R+R+€Q*~Q*~Q*~Q(~Q(~Q*}P)}P)}P)}P+}P+|O(|O(|O({N'{N){N){N){N){N){N)zM(zM(zM(zM(zM(xK&zM(zM(xK&xK&xK&vI$wJ#wJ#tJ"tJ"sJ"sJ"sJ"sJ"sJ"rI!rI!rI!rI#rI#qH"qH"qH"qH"rH"rH"rH$qG#qG#qG#pG#pG#nE!nE!nE!nE!mD mD mD mD mD mD mD lClCkBkBkBkBkBjAjAjAjAjAi@i@i@i@i@g>g>g>i@g>g>g>f=e>e>d=d=e;e;e;e;d:d:d:d:b;a:a:a:a:a:a:`9”c6”c6”c6”c6”c6”c6•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7“e7“e7”f8”f8’f8’f8”f8•g9”f8”f8”f8”f8•g7•g7•g7•g7–e6—f7—f7—f7—f7˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8™h9™h9™h9™h9™h9™h9™h9›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;œk<œk<›j8›j8œk9œk9œk9œk9œk9œk9œk9œk9œk9œk9œk9œk9œk9l:žk9žk9žk9žk9Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl: m; m; m> m> m> m> m> m> m> m> m; m; m; m; m; m;¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n< m;¡n<¡n?¡n?¡n?¡n?¡n?¡n?¡n?¡n?¡n?¡n?¡n?¡n?¡n?¡n?Ÿn<Ÿn<¡n<¡n< m; m; m; m; m> m> m> m> m>¡n?¡n? m> m9 m9 m; m; m; m; m; m; m; m; m; m; m;Ÿl:Ÿl= m>žm>žm>žm>l=žm;žm;žm;l:žm;žm;Ÿl: m;žm;l:Ÿn< o= m>Ÿl=žk9žk9j;Ÿl=¢k:¡j9œf.œf.¥l ©p$°s$®q"¬d!£[¶L¬Bä7ã6ÿ-ÿ/ -ÿ0ÿ0ÿ/ ÿ. ÿ+ ÿ1ÿ>ÿIõcùgçnäkÙbÌUºG«8“0†#YG - Za#lj-¸Vàyð‰¿P©:½KÓa -äzó‰þ•ÿ›ÿŸÿŸÿ ÿŸÿ ÿ ÿ ÿŸÿ ÿ ÿ¢ÿ¡ÿ¡ÿ¢ÿ¡ÿ¡ÿ ÿ ÿ ÿ ÿœÿ ÿ¢ÿ¢ÿ£ÿ£ÿ¢ÿ¡ÿœÿ›ÿž ÿž ÿ£ÿ›ö’ü˜ÿŸÿµ$ÿÆ=ÿµ, Id Ww Œ<Œ<ˆMžc*¯zG¼‡TÅŽ[Í–cÑgÍ™cÊšaΞeÊšaÌœcÐœbÑcÚ¦jÛ§kܦlܦlÙ¦gÚ§hÚ©eÚ©eÙ¤^ÑœVÏf*–-ÁÇèð'þ%öíéÑÊ -¶·§©œ!Ÿ$Ÿ4 §<žF§O&žO$N#ƒE}?„M›d1©vD­zH®yF³~K®„Z±‡]¡€\œ{WŽpU†hM„jDŽtNsL}`9d@—nJœmD¡rI¨zGª|I®€MºŒYÁ•e±…U–h:Œ^0‰\3‰\3Š^2Š^2‹\3‹\3‰Z1‰Z1‰Z3ˆY2ˆY2ˆY2ˆY2ˆY2ˆY2ˆY2†Y0†Y0†Y0†Y0…X/…X/‡X/‡X/‡X/†W.†W.†W.†W.†W.†W0†W0„W0„W0„W0ƒV/„W0„W0„W0„W0W-W-W-€V,‚U.‚U.‚U.‚U.„U,„U,„U,‚S*‚T(‚T(‚T(‚T(‚S,‚S,‚S,‚S,R+R+R+~Q*~Q(~Q(~Q*~Q*~Q*}P)}P+}P+}P)|O(|O(|O({N)|O*{N){N){N){N){N)zM({N){N){N)xK&zM(zM(zM(xK&xK&xK&wJ#wJ#uK#uK#sJ"sJ"sJ"sJ"sJ"sJ"rI!rI!rI#rI#rI#rI#rI#qH"rH"rH"rH$rH$qG#qG#pG#pG#pG#pG#nE!nE!nE!nE!mD mD mD mD mD lClCkBkBkBkBkBjAjAjAjAjAjAjAjAjAi@jAjAi@i@g>g>g>g>e>e>d=d=f<f<e;e;e;e;e;e;b;b;b;b;b;a:a:a:•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7•d7–e8–e8•d7•d7•d7˜e8˜e8˜e8˜e8˜e8™f9™f9™f9™f9™f9™f9™f9™f7˜e6™f7™f7—f7˜g8˜g8—f7˜g8˜g8˜g8—f7™h9™h9™h9™h9™h9™h9™h9™h9™h9›j;›j;™h9›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;œk<œk<œk9œk9œk9œk9œk9œk9œk9œk9œk9œk9l:l:l:l:l:l:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl: m; m; m; m; m; m; m; m; m; m> m> m> m> m> m> m> m>¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n?¡n?¡n?¡n?¡n?¡n? m> m> m> m> m; m; m; m; m;¡n<¡n< m; m; m; m; m; m> m> m> m> m> m> m> m> m; m; m> m>œn>œn>œn@œn@œn>œn>œn>œn>žm;žm; m>Ÿl=žm>žm>l?žm@ŸnAŸnA£lB¥nD©n3žc(¤b®lÇ|Ý’û­ÿ»ÿ¼ÿ·ú©Ü‹ËN°3û$ÿ5ÿ.ÿ0ÿ0ÿ1ÿ+ÿ!õý#ÿ9ÿKùjýnësêrälÙaÈN»A©5)zf5*+/i•K"Úrä|»E®8¹MÎbâzòŠþ—ÿœÿŸÿŸÿ ÿ¡ÿ ÿ ÿ ÿ¡ÿŸÿŸÿ ÿ ÿ¢ÿ¢ÿ¢ÿ¡ÿ¢ÿ¢ÿ¡ÿ ÿ ÿŸÿŸÿ ÿ¡ÿ ÿ¤ÿªÿªÿ«ÿ® ÿ¯ -ÿ¯ ÿ±ÿ® ÿ§ÿœôñŽñŽÖu´S‹1{!{'ƒ/Š=•H¦_'´m5¹z@ȉOÊ’TÓ›]Õ aÛ¦gá®oå²sð½~úLjûȇõÂï¹}ëµyæ­rä«pæªiæªií£Eà–8à’ëí–Û„ÔgË^ÓDÂ3²«º¸ÐÖÝãîíîæ Ó×ÃÈ ­1 ³7ƒ/•A¥\+·n=Ä…OÌWЗ^ÕœcÒœbÔždÒ gÕ£jÕ£jÖ¤kܬqä´yá´xתnÕ¥jקlÙ§lÚ¨mÝ«rÚ¨oÌ›i²O™k=Œ^0Š^.Š^.Š^.Š^.‰]/‰]/‹]/‰[-‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.ˆY0ˆY0ˆY0ˆY0‡X/‡X/‡X/‡X/‡X/†W.†W.†W.†W.†W.†W.‡X/†W.†W.†W0†W0„W0„W0„W.„W.„W.„W.„W0ƒV/‚U.‚U.‚U.‚U.‚U,‚U,‚U,‚U,‚T(„V*„V*‚T(‚S,‚S,‚S,‚S,‚S,R+R+R+€Q(€Q(€Q(€Q(~Q*~Q*~Q*~Q*}P)}P)}P)|O(}P)|O({N)|O*|O*{N){N){N){N){N){N){N){N){N)zM(zM(xK$xK$xK$xK$uK#uK#uK#uK#tK#tK#uK#tJ"tJ"tJ"sJ"rI!rI#rI#rI#rI#sI#sI#sI#rH"rH$rH$pG#pG#pG#pG#pG#nE!nE!nE!nE!mD mD mD mD"mD"kB kB kB lC!lC!kB jAjAjAkB jAjAjAjAjAjAjAjAi@i@i@i@gAe?e>e>e>d=f<f<f<f<e;e;e;e;e;d:d:d:d:d:d:c9•d7•d7•d7•d7•d7•d7•d7–e8–e8–e8–e8–e8–e8–e8–e8–e8˜e8˜e8˜e8˜e8˜e8™f9™f9™f9™f9™f9™f9™f9™f7šg8™f7™f7˜g8™h9™h9™h9˜g8˜g8™h9™h9™h9™h9™h9™h9™h9™h9™h9›j;›j;›j;›j;›j;œk<œk<œk<œk<œk<œk<œk<œk<œk<œk m> m> m> m> m> m> m>¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n?¡n?¢o@¢o@¢o@¢o@¡n?¡n?¡n?¡n?¡n<¡n<¡n<¡n<¡n<¢o=¢o= m; m; m; m; m; m> m> m> m> m> m> m> m> m; m; m> m>œn>›m=œn@œn@œn>œn>œn>œn>žm;žm; m> m>žm> o@ oB oB oB oB i?™b8žc(¢g,Ãàž7ÿº%ÿÌ7ÿÎ)ÿÐ+ÿÑÿÐÿÍ)ÿË'ÿ¥+ìoþ'÷ ÿ'ÿ*ÿ1ÿ2ÿ$ïä -éí%ÿ@ùjÿrîvìtälÝeÒXÃI±=¦2„%qB -5*/Z‡=ÒjÛs ¸B®8¹MÍaßwòŠÿ˜ÿœÿŸÿŸÿ ÿ ÿ¡ÿ ÿ ÿ ÿžÿŸÿŸÿ ÿ¡ÿ¡ÿ¢ÿ¢ÿ¢ÿ¢ÿ¡ÿ ÿ ÿŸÿŸÿœÿžÿŸÿ ÿ ÿ¡ÿ£ÿ¡ÿ£ÿ¥ÿ¨ÿ©ÿ©ÿ©ÿ¬ÿ¬ ÿ© ÿ°"ÿ«û¡.Þ„³_ -‘=€3B˜Q«d,²s9·x>·AŠLÆ‘RÊ•VÖ£dÚ§h߬mê·xê·vé¶uè²vã­qãªoߦkÝ¡`Úž]Ý“5Þ”6ÿ¾(ÿàJÿÙ.ÿÊÿ¼-ÿ«ÿ‘ ÿþhéSá<Õ0ØÜìî!ý%ÿ'ÿ% -÷ØÔ· ´›¬0‘= ”@—N¨_.¯p:¾I‰PÈVΘ^Ö fÚ¨oá¯vá¯væ´{ñÁ†õÅŠï†å¸|à°uÞ®sÞ¬qÚ¨mئmÑŸfÅ”b¬{Ib4Œ^0Š^.Š^.Š^.Š^.Š^0‰]/‹]/‹]/‹]1‰[/‰[/‰[/‰[/‰[/‰[/‰[/ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0‡X/‡X/‡X/‡X/‡X/‡X/‡X/‡X/†W.†W.‡X1‡X1…X1…X1…X/…X/„W.„W.„W0„W0„W0ƒV/‚U.ƒV/ƒV-‚U,‚U,‚U,„V*„V*„V*„V*„U.„U.„U.‚S,‚S,‚S,R+R+R)R)R)R)~Q*~Q*~Q*~Q*~Q*}P)}P)|O(}P)}P)}P+|O*|O*|O*|O*{N){N){N){N){N){N){N)zM(zM(zM&xK$xK$xK$vL$uK#uK#uK#tK#tK#uK#uK#tJ"tJ"sJ"sJ"rI#rI#rI#rI#sI#sI#sI#sI#rH$rH$qH$qH$pG#pG#pG#pG#nE!nE!nE!nE!nE!nE!nE#mD"mD"mD"mD"lC!lC!lC!lC!lC!lC!kB kB kB kB kB jAjAjAjAjAjAi@i@gAgAg@ g@ e>e>f<f<f<f<e;e;e;e;e;d:d:d:d:d:d:c9–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8˜e8˜e8™f9™f9™f7šg8šg8šg8šg8šg8šg8šg8›h9›h9›h9›h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9›j;›j;›j;›j;›j;›j;œk<œk<›j;œk9œk9œk9œk9œk9œk9œk9l:l=l=Ÿl=Ÿl=œk9œk9l:l:l:l:l:l:l:l:l:l:l:žm;žm;žm;Ÿl: m;Ÿl: m; m; m; m;¡n<¡n<¡n<¡n<¡n<¡n<¡n< m;¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¢o=¢o=¢o=¢o=¢o=¢o=¢o=¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n?¢o@¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¢o=žp@o?o?o?of<f<f<f<f<e;e;e;e;e;d:d:d:d:d:d:–e8–e8–e8–e8–e8–e8–e8–e8–e8–e8—f9—f9–e8–e8–e8—f9™f9™f9™f9™f9™f7šg8šg8šg8šg8šg8šg8šg8›h9›h9›h9›h9™h9›j;›j;™h9™h9™h9™h9›j;›j;›j;›j;›j;›j;›j;›j;œk<œk<œk<œk<œkžm> m> m>l:l:l:l:l:žm;žm;žm;žm;žm;žm;žm;žm;žm;žm;Ÿn< m; m; m; m;¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¡n<¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o@¢o@¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=žp@žp@žp@ rB r? r?žr?œp=žrDžrD™pA™pAœn@œn@£n= k:˜`¦n'Ìï°=ÿÊ,ÿÑ3ÿËÿÉÿÍ ÿÇÿÆÿÉÿÁ ÿ¶ÿ½ÿÈ ÿÉ ÿÎÿÊ4ÿ¢ ÿ<ô&ÿ&ÿ%ÿ.ÿ+ ý ðàô'îZÿpöyöyðxèpÞdÐV½D¬3’)†bW 7-  #B_$·Zß‚!ÅS°>¶LÆ\Útîˆü•ÿšÿÿÿ ÿ¡ÿ¡ÿ ÿœÿœÿžÿŸÿŸÿŸÿ ÿ ÿ ÿ ÿŸÿŸÿŸÿžÿœÿžÿŸÿŸÿŸÿŸÿžÿŸÿŸÿŸÿ ÿ ÿ ÿ ÿ¡ÿ¡ÿ ÿ ÿ ÿ¡ÿŸÿ¢ÿ¥ÿ£ÿ©ÿ¯ÿ´*ÿ«!ö¡(é”؇Ú‰Ô…&Ð"É}!É}!ØŽ2Ý“7äš<å›=ê£Båž=å›9äš8åš,åš,îž!úª-ÿ»ÿÂÿ»ÿ°ÿ­ÿ°ÿ·ÿ½ÿÇÿÊ ÿÏÿÖÿÔÿÒÿÌÿÇÿÀ=ÿª'ún(Â6±¨ £¡¤ ²Ã¾¦®…5Ž>›T"¦_-¯p8½~FÉPÒ–YÙŸaÞ¤fá¬mä¯pá®oà­nß®nÞ­mÝ«nÖ¤gÏ›cÄXº…R¦q>™f7“`1‘`3_2_1_1_1_1Œ^0Œ^0Ž]0Ž]0Œ^2‹]1‹]1‹]1‹]1‹]1‹\3‹\3‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0‡Y-‡Y-‡X/‡X/‡X/‡X/…X/„W.…X/…X/„W0„W0„W0„W0„W0„W0„W.„W.ƒV-ƒV-…W+…W+ƒW+ƒW+ƒV/ƒV/‚U.‚U.„U.‚S,‚S,‚S,‚S*‚S*‚S*R)R+R+R+R+R+~Q*~Q*~Q*~Q*}P)}P)}P)}P)|O(}P)}P)}P)}P)|O({N'|O(|O(|O(zM&zM&zM&xK$zM&xN&vL$vL$vL$vL$vL$vL$vL$vL$uK#uK#tJ"tJ"tJ"sJ"sJ"tJ"tJ"tJ$tJ$sI%sI%rI%qH$qH$qH$qH$pG#pG#pG#pG#nE!nE#nE#nE#nE#nE#mD"mD"mD"mD"lC!lC!lC!lC!lC!kB kB kB kB kB kB kB kB jAjAhBhBhBgAgAgAg@ e>g=g=g=g=f<f<f<f<f<e;e;e;e;d:d:d:–e8–e8–e8–e8–e8–e8—f9—f9—f9—f9—f9—f9—f9—f9—f9—f9™f9™f9™f9™f9™f7™f7šg8›h9›h9›h9›h9›h9›h9›h9j;j;›j;›j;œk<œk<›j;›j;›j;›j;œk<œk<œk<œk<œk<œk<œk<œk m> m> m>žm;žm;žm;žm;žm;žm;žm;žm;žm;žm;žm;Ÿn<Ÿn<Ÿn<Ÿn<Ÿn<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¤q?¤q?¤q?¤q=¤q=¤q?¤q?¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o@¤qB¤q?¤q?¥p?¥p?¥p?¥p?¤q?¤q?¥r@¤q?¤q?¤q?¤q?¤q?¤q?¤q?¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=žp;žp; r= r= r?¡s@ŸsEŸsE¤sD¤sD©s;©s;®q"¥h¨e±nã’ÿµÿÍÿÕÿÌ ÿÊ ÿÉ ÿÄÿÄÿÌÿÂûœï~õ„#ÿ¦ ÿ½"ÿÇ ÿÃÿÑ ÿ×ÿ§Íaò'÷,ÿ)ÿ-ÿ+ÿÞÝÍKðn#ôyø}ù|ðsãfÐS½A¯3˜&’ ‚ xXF)'"%4B”BÝ‹EÊZ¦6½IÑ]ãtô… ýÿ•ÿ˜ÿšÿŸÿ¡ÿŸÿŸÿ ÿ ÿŸÿ ÿ ÿ ÿŸÿ ÿ ÿŸÿŸÿŸÿŸÿœÿžÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿ ÿ ÿ ÿ ÿ ÿ ÿŸÿ ÿ¢ÿ¡ÿ¡ÿ¡ÿ¡ÿžÿ›ÿœÿŸÿ ÿ£ ÿ¤ -ÿ©ÿ°ÿ´ÿ¶ÿ±ÿªÿ¨ ÿ¨ ÿª ÿ©ÿ©ÿª ÿ´ÿÀ$ÿÊ(ÿÒ0ÿÕ-ÿÏ'ÿÄÿ­ÿ§ÿ²ÿÈÿÍÿÕÿØÿÜÿÙÿØÿÕÿÎÿÌÿÈÿ½ÿ£$ð¾R  4—6Œ+}"q`] ktou% ^&]%a#i+t8~BŒO“V"b+¥j3®u<´{B¶‚F±}A¯}B®|AªzA¥u<šh1’`)ŠW(…R#\/‘^1^3_4_1_1^3^3^3Ž]2Ž]0Ž]0Œ^2Œ^2Œ^2Œ^2‹]1‹]1‹\3‹\3‹\3‹\3‹\3‹\3‹\3‹\3‹\3‹\3‰Z1‰Z1‰Z1‰Z1‰Z1ˆY0ˆY0ˆY0ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.†Y0…X/…X/…X/‡X/‡X/†W0†W0„W.„W.„W.„W.ƒW+ƒW+ƒW)ƒW)ƒV-ƒV-ƒV-‚U,„U,„U,„U.‚S,‚S*‚S*‚S*‚S*R+R+R+R+R+R+~Q,~Q,~Q*~Q*~Q*~Q*~Q*~Q*}P)}P)}P)}P)}P)}P)|O(|O(|O(|O({N'{N'xK&zM(xN(vL&vL&vL&xN(xN(vL&vL&vL&uK%vL$uK#tJ"tJ"tJ"tJ"vI"wJ#tJ$tJ$tJ$tJ$tJ&sI%qH$qH$qH$qH$qH$qH$pG#qH$pG#nE!pG#nE!nE!nE!nE#mD"mD"lC!mD"mD"mD"lC!kB kB kB lC!lC!kB kB kB kB kB jAjAjAjAhBhBg@ g@ g=g=g=g=g=f<f<f<f<f<e;e;e;e;e;e;—f9—f9—f9—f9—f9—f9—f9—f9—f9—f9˜g:˜g:—f9—f9—f9—f9šg:šg:šg:šg:šg8›h9›h9j;j;j;j;j;j;j;j;j;›j;›j;œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk m> m>¡n?žm;žm;žm;žm;žm;Ÿn<Ÿn<Ÿn<Ÿn<Ÿn<Ÿn<Ÿn<Ÿn<Ÿn<Ÿn<Ÿn<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¤q?¤q?¢o=¢o=¢o=¢o=¢o=¢o=¢o=¤q?¤q?¤q?¥r>¥r>¥r@¥r@¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¤q?¤q?¢o=¤qB¤qB¤q?¤q?¥p?¥p?¥p?¥p?¤q?¥r@¥r@¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q? r= r= r= r=žp= r?žrDœpB¢qB£rC¤n6œf.¡d¯r#Ü™!ÿÂJÿÐ*ÿÔ.ÿÏÿÊ ÿÈÿÍ ÿÍÿÉ ÿÆÿÍÿÂÅfŸ.ápÿ¢ÿµÿË ÿÇ ÿËÿÔ ÿÇ'ÿ¤ÿI -ù.ÿ'ÿ(ÿ1ÿ-áÅ®,ædõz÷|õxïrÝ`ÈK¶:«/˜&—%Œ*†$bO1-& #%36~,Ìz4¦6‚£/ÊVîÿÿš ÿ ÿ ÿœÿœÿ¢ÿŸÿœÿœÿÿ ÿ¡ÿ¡ÿ¡ÿ ÿ ÿŸÿŸÿŸÿœÿžÿžÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿžÿŸÿŸÿŸÿ ÿ ÿ ÿŸÿ ÿ ÿ ÿ ÿ ÿŸÿŸÿŸÿžÿ›ÿ™ý™ý™ýœÿžÿ¡ÿ¦ ÿ¯ÿ²ÿµÿ¸ÿ¼ÿ¾ÿ¾ÿ¼ÿ¹ÿ¿#ÿ¿ÿ ÿÆÿ½ÿ©ÿ£ÿ§ÿÎ!ÿÝ&ÿÔÿÑÿÑÿÏ ÿÏ ÿÏ ÿÏ ÿÍÿÈ -ÿµ ñ•Ôe¶G£7 4š9£BžC“8y(] SPHRKLW`"h,s7D‰LU”Y"˜_&›b)›g+Ÿk/ n3¤r7¤t;§w>®|E­{D¢o@—d5”a4“`3‘`5‘`5Ž`2Ž`2_4_4^3^3Ž]0^1_3Œ^2Œ^2Œ^2Œ^2Œ^2Œ]4Œ]4‹\3‹\3‹\3‹\3‹\3‹\3‹\3‹\3‹\3‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.†Y0†Y0†Y0…X/‡X/‡X/‡X1‡X1…X/„W.„W.„W.ƒW+ƒW+„X*„X*„W.ƒV-ƒV-ƒV-„U,„U,„U.‚S,‚S*‚S*‚S*„U,€S,R+R+R+€S,R+R-~Q,R+~Q*~Q*~Q*~Q*~Q*~Q*}P)}P)}P)}P)}P)}P)|O(|O(|O({N'{N'{N)zM(yO)yO)yO)vL&vL&xN(xN(vL&vL&vL&vL$vL$vL$vL$vL$vL$vI"wJ#uK%uK%tJ$tJ$tJ&tJ&qH$qH$qH$qH$qH$qH$pG#qH$qH$qH$pG#pG#nE!nE!nE#nE#nE#nE#mD"mD"mD"mD"mD"mD"mD"lC!lC!lC!lC!lC!kB kB kB kB kB jAhBhBhA!g@ i? i? g=g=g=g=g=g=g=f<f<f<f<e;e;e;—f7—f7—f7—f7—f7—f7—f7˜g8˜g8˜g8˜g8˜g8˜g:˜g:˜g:˜g:šg:šg:›h;›h;›h9j;j;j;j;j;j;j;j;j;žk<žk<œk<œk<œk<œk<œk<œk<žk<žk<žk<žk<žk<žkžm>žm>žm;žm;žm;žm;žm>Ÿn?Ÿn?Ÿn?¡n?¡n?¡n?¡n?žm>Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn? o@ o@ o@¡n<¡n<¢o=¢o=¢o=¢o=¢o=¢o=¤q?¤q?¢o=¢o=¢o=¢o=¢o=¢o=¥p?¥p?¥p?¥p?¦q@¦q@¥p?¥p?¥p?£n=£n=¥p?¥p?¦q@¦q@¦q@¥r@¥r@¥r@¥r@¥r@¤q?¤q?¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¤q?¤q?¥r>¤q=¤q?¤q?¤q?¤q?¢o=¤q?¤q?¤q?¢o=¤q?¥p?¦q@¤q?¤q?¤q?¤q?¤q?¥r@¤q?¥r@¥r@¥r@¥r@¥r@¤q?¥r@¤q?¥r@¤qB¤qB¤qB¤qB¤q?¤q?¤qB¤qB¤qB¤qB¤qB¤qB¢q=¢q=£r@£r@£r@¢q?¢q?¢q?£n;¦q>­u7®v8­n¥f³kÇï¢ÿÄ%ÿÔÿØÿÌÿÇÿÆÿËÿÃÿ½ÿÄÿÌÿËÿÈÿÐåX[•[+Öœlÿ¿@ÿÆGÿÀÿ»ÿÉÿÎÿ×Uÿ(ÿ#ÿ' ÿ3æ. µx -ºL#õxõxörïkÜYÆC±3§)›%›%+™' oK -A84%#!#&)54e˜H')} Ž¡Ò2ìLÿd!ÿs0ÿ1ÿ‹;ÿ˜2ÿ7ÿ¢#ÿ›ÿ™ÿ— ÿ˜ÿ™ÿ›ÿœÿžÿ ÿ ÿŸÿŸÿŸÿ ÿ ÿ ÿžÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿ ÿŸÿŸÿ ÿŸÿŸÿŸÿŸÿŸÿŸÿÿ›ÿ˜ -ÿ˜ -ÿš ÿ™ÿšÿšÿšÿ›ÿŸÿŸÿŸÿ¡ÿ¦ÿ¨ÿ¬ÿ¬ÿ©ÿ¨ÿ§ÿ§ÿ£ -ÿ ÿŸÿ›óÿŸÿÆÿÒ*ÿÔÿÔÿÖÿÕÿÒÿØÿÕ/ÿÀÿŒÔ]¼6¯)¤)Ÿ$•*›0•6–7ˆ:w)KC?N(c5j<|I‰V+—b4¢m?±zG¸N¿‰SÇ‘[É•_Ï›eÒ iÕ£lÓ£lÏŸhÌŸgËžfÊšaÅ•\·†T¦uCl=–e6’a6‘`5Œ`2Œ`2Ž`4Ž`4_3_3_3_3_3Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ]4Œ]4Œ]4Œ]4‹\3‹\3‹]1‹]1‹\3‹\3‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1‰[/‰[/ˆZ.ˆZ.ˆZ.ˆZ.ˆY0ˆY0ˆY0ˆY0‡X/‡X/‡X/‡X/‡X/‡X/†X,†X,…Y-…Y-…Y-…Y-„W0„W0„W.ƒV-…V/…V/…V/„U.„U,…V-…V-„U,„U.„U.€S,R+€S,€S,R-R-R+~Q*R+R+~Q*~Q*~Q*}P)~Q*~Q*|O(}P)}P)}P)|O(}P)|O({N'{N){N)yO)yO)yO)yO)xN(xN(xN(xN(xN(vL&vL&vL&vL&vL&vL&vL&xK&xK&wJ%wJ%uK'uK'uK'tJ&tJ&sI%rI%rI%qH$rI%qH$qH$qH$qH$pG#pG#pG#pG#pG#nE!nE!nE!nE!mD mD mD mD mD mD mD lC!lC!lC!lC!kB kB kB kB kB jAhBhBhA!g@ i? i? i? i? i? g=g=g=g=f<f<f<f<f<f<f<˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8™h9™h9˜g:˜g:˜g:˜g:šg:›h;›h;›h;j;j;j;j;j;j;j;j;žk<žk<žk<žk<œk<œk<œk<œk<œk<œk<žk<žk<žk<žk<Ÿl=Ÿl=l=l=l=l=l=žm>žm>žm>žm;žm;žm;žm;žm>Ÿn?Ÿn?Ÿn?¡n?¡n?¡n?¡n?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn? o@ o@ o@ o@ o@ o@ o@ o@ o@ o@¢o=¢o=¢o=¢o=¢o=¢o=¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¥p?¥p?¥p?¦q@¦q@¦q@¦q@¥p?¦q@¥p?¥p?¦q@¥p?¦q@¦q@¦q@¦sA¦sA¥r@¦sA¦sA¥r@¥r@¦sA¦sA¦sA¦sA¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r>¥r>¥r@¥r@¥r@¥r@¤q?¥r@¤q?¤q?¥r@¥r@¦q@§rA¥r@¦sA¦sA¥r@¥r@¥r@¥r@¦sA¥r@¥r@¥r@¥r@¥r@¦sA¥r@¥r@¥rC¥rC¥rC¥rC¥r@¥r@¥rC¥rC¥rC¥rC¤qB¤qB o;¢q=£r@£r@¤sA¤sA¢q? o=¨s@¬wD§o1žf(¬mȉ6ð¨1ÿÆOÿØ9ÿÐ1ÿÊ ÿÑÿÍÿÏ ÿÈÿ®þ”ÿ™ÿ·ÿÉÿÈÿÉÿËÿÊ™O@:k1Ålê‘üÿ¦ÿ¾ ÿÉÿ¯#úxÿ0 ÿ&ÿ!ÿ0ÿR-Ý%i–(æi÷zñmåaÓPÀ=ª,¤&›%›%›)˜&ƒ$|WL ?;+($&**7<] w'†¡®³ÁÉ$Ñ6ÝBßVõlÿÿœÿ¥ÿ¥ÿ§ÿ£ ÿ ÿ ÿŸÿœÿ›ÿŸÿŸÿ¡ÿ¡ÿ¡ÿŸÿŸÿŸÿžÿœÿžÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿŸÿžÿÿ›ÿ™ ÿ˜ -ÿš ÿ™ÿšÿ›ÿžÿ ÿ ÿ¢ÿ¡ÿ¡ÿ£ÿ£ÿ£ÿ£ÿ£ÿ£ÿ¢ÿ£ÿ§ÿ¦ ÿ£ ÿŸÿ›òŽó™ÿ®ÿÀ ÿÎÿÚÿÝ ÿÙ ÿÜ#ÿÏ)þ¤áj½F¾8·1¨-Ÿ$…‚~‚#v(t&j8†T.‰c<“mF¡sE¡sE«xMµ‚W¼‡Y¾‰[ÅŽ[Ë”aÑ›eÚ¤nñ½‡ÿКÿÙ¢ÿЙæ¶ܬuÔ§oÌŸgΞeȘ_Á^·†T­|M£rC•d9‘`5a3Œ`2Ž`4Ž`4Ž`4_3_3_3_3_3_3Œ^2Œ^2Œ^2Œ^2Œ^2Œ]4Œ]4Œ]4Œ]4Œ]4Œ]4Œ^2Œ^2‹\3‹\3‹\3‹\3‹\3‹\3‹\3‹\3‰Z1‰Z1‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0‡X/‡X/‡X/‡X/‡Y-‡Y-…Y-…Y-…Y-…Y-…X1„W0„W.„W.†W0…V/†W0†W0…V-…V-…V-…V-„U.…V/‚U.‚U.‚U.‚U.€S.€S.€S,€S,R+R+R+R+~Q*~Q*~Q*~Q*~Q*~Q*}P)}P)}P)}P)}P)}P)|O*{N)yO)zP*yO)zP*yO)yO)xN(xN(yO)xN(xN(vL&xN(xN(vL&vL&xK&xK&xK&wJ%uK'uK'uK'uK'uK'tJ&sJ&sJ&sJ&rI%rI%rI%rI%qH$qH$qH$pG#pG#pG#nE!nE!nE!nE!nE!nE!mD mD mD mD mD mD"lC!lC!lC!lC!kB kB kB kB jAhBhBhA!hA!i? i? i? i? i? g=g=g=g=g=f<f<f<f<f<f<˜g8˜g8˜g8˜g8˜g8˜g8˜g8˜g8™h9™h9™h9™h9™h9™h9™h9™h9™h9™h9›j;›j;›j8›j8›j;œk<œk<œk<œk<œk<œk<œk¤q=¤q=¥r>¥r>¥r>¥r>¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@£r@£r@£r@£r@£r@£r@£r@£r@¦sA¦sA¥r@§tB§tB¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¥r@¥r>¥r>¥r>¦s?¦s?¦s?¦s?¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r@¦sA¦sA¦sA¦sA¦sA¥r@¦sA¦sA¦sA¥r@¦sA¦sA¦sA¦sA¦sA¥r@¦sA¥s<¥s<¥r>¥r>¥r>¥r>¥r>¥r>¦qE§rF§t@©vB¡n:¢o;¨qC¨qC¤sA¥tB°q9©j2§f­l -Éyæ–ÿ¶ÿÉÿÌÿÈÿ¼ÿ¿ÿÇ ÿÉÿÌÿÑÿÀñ‘ü|ÿŽÿªÿ¼ ÿÇ ÿÊÿÁÿÍ ×–2X<& ·YæˆAÿ¤ÿÆÿÓÿµÿUî<ÿ(ÿ' ÿK-ÿ?!‰ª<ásHõkæ\ÓGÃ7°&©š$š$•%–&‹#‰!~!s[UGA;<?AH -K W \†®¶Úà#ðô#øöêÞ -ÞèÝ6õN'ïi/ÿ~Dÿ™Lÿ¤Wÿ£Aÿ¤Bÿ¨!ÿŸþœþœÿŸÿ ÿ ÿŸÿŸÿ ÿ ÿ ÿŸÿ ÿŸÿŸÿŸÿžÿŸÿžÿŸÿŸÿ ÿ ÿ ÿ ÿŸÿŸÿÿÿœÿœÿœÿœÿ›ÿ›ÿÿÿÿ¡ÿ¡ÿ¡ÿ¢ÿ¢ÿ¢ÿ¡ÿ ÿžÿŸÿŸÿ ÿ ÿžÿÿŸÿ¡ÿ¢ÿ¡ÿœÿ”ý•ÿ ÿ¢ÿ¨ ÿ´ÿ±ÿ”ãwÍ[½K·D²?²G¾S¸S¨C.}mm‘BN!©g4ºxE¼€NÁ…SºƒR¾‡V¾‰XÂ\Ç”`˘dÒžhؤnß­tæ´{çµzä²wÚ¨kئi×¥jÕ£hÒ eÏbÍ™aÉ•]Ç•Z²€E•c;‘_7a3a3a5a5a5Ž`4Ž`4Ž`4Ž`4_3_3_3_3_3Œ]4^5Œ]4Œ]4_3Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Ž]2\1‹]1‹]1‹\3‹\3‹]1‹]1‹]1‹]1‰[/‰[/‰[/ˆZ.‰[/‰[/ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0‡X/‡X/‡X/‡X/‡X/‡X/‡X/…X/…X/…X/„W.†W.…V-†W0†W0…V/…V/…V/…V/…V/…V/ƒV/‚U.‚U.‚U.‚U0€S.€S,€S,€S,R+R+R+R+R+~Q*~Q*~Q*~Q*~Q*}P)}P)|O(}P)}P)}P+|O*{N'|O(|O(|O(|O(|O({N'{N'{N'{N'{N'zM&zM(zM(wJ%xK&zM(xK&xK&xK&vL&vL&uK%uK%uK%uK%tJ&uK'tJ&tJ&tJ&sI%sI%rH$rH$rH$rH$qG#qG#qG#qG#qG#oE!oE!nE!nE!nE!nE!nE!mD nD nD lC!lC!lC!lC!lC!kB kB kB kB kB kB jAj@!j@!j@!j@!hA!g@ g@ g@ g@ e>e>e>e>e>e>d=™h9™h9™h9™h9™h9™h9™h9™h9›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;œk9œk9œk<œk<œk<œk<œk¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¦sA¦sA¥r@¥r@¥r@¦sA¦sA¦sA¤sA¤sA¤sA¤sA¤sA¤sA¤sA¤sA¥r@¥r@¥r@¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦s?¦s?§t@§t@§t@§t@§t@¨uA¨uA¨uA¨uA¨uA§t@§t@§t@§t@§tB§tB§tB§tB§tB§tB§tB¦sA¦sA¦sA¥r@¦sA¦sA¥r@¥r@¥r@¥r@¥r@¥s<¥s<¥r>¥r>¥r>¦s?¥r>¥r>¦qE¦qE¤q=¤q=¨uA¤q=¨qCªsE¤sAœk9¢c+¡b*¶uÜ›9ÿº+ÿÌ=ÿÓÿÅ ÿ·ÿ¯ÿªÿ«ÿ­ÿ»ÿÅ ÿÐÿÇþžÿ‡ÿ•ÿ£ÿ°ÿÅ ÿÈÿÉÿÛÓ’.D mÅg ÿ–ÿÀÿÎÿÍÿ~ÿMÿ5ÿ"ÿ( -ÿ7´'}x -ºL!é_ÚPÌ@½1¬"¥—!– ‘!”$‹#‰!"zfbQ KFFFI K K V W ƒ Š¨ ²ØÞ!ò!ø'ÿ+ÿ.ÿ4ÿ1ë!Õ «¤œ¯6ÄKÒ]íxý—ÿ¢ÿ«ÿ¬ÿ© -ÿ£ÿ ÿœÿ›ÿšÿšÿ›ÿ›ÿœÿœÿœÿžÿžÿžÿžÿŸÿŸÿ ÿ ÿ ÿ ÿŸÿŸÿžÿžÿžÿœÿœÿœÿÿ›ÿÿŸÿ ÿ¡ÿ¡ÿ¡ÿ¢ÿ¡ÿŸÿžÿÿžÿžÿÿÿœÿ™ÿ˜ÿ™ÿšÿšÿÿ ÿŸÿŸÿšþ–þ–þ—û”ÿ›þ’úˆ -õƒízìyï„ÿ™ÿ¹DÿºEÕv#„%g q‰: Š;ŠHž\)­q?¹}K¾‡VÆ^É”cË–eÉ–bΛgÓŸiÖ¢lÕ£jÏdÉ—\Ë™^Õ£fÙ§jÛ©nÛ©nÙ§lÙ§l×£kÖ¢jÎœa¯}B”b:’`8b4a3a5a5a5a5Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4_3_3Œ]4^5Œ]4Œ]4_3Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Ž]2Ž]2Œ^2‹]1‹\3‹\3‹]1‹]1‹]1‹]1‹]1‰[/‰[/‰[/‰[/‰[/‰Z1‰Z1‰Z1‰Z1‰Z1ˆY0ˆY0ˆY0ˆY0ˆY0‡X/‡X/‡X/‡X/…X/…X/…X/…X/†W.†W.†W0†W0†W0†W0†W0†W0†W0…V/ƒV/ƒV/ƒV/‚U.‚U0‚U0‚U.€S,€S,€S,€S,€S,R+R+R+R+R+~Q*~Q*~Q*}P)}P)}P)}P)}P+}P+|O(|O(|O(|O(|O(}P)|O({N'{N'{N'{N'{N'zM(zM(xK&zM(zM(zM(zM(xK&vL&vL&vL&vL&uK%uK%uK'uK'uK'tJ&tJ&sI%sI%sI%rH$rH$rH$rH$rH$qG#qG#qG#qG#qG#nE!nE!nE!nE!nE!nE!nD nD mD"mD"lC!lC!lC!lC!kB kB kB kB kB jAj@!j@!j@!j@!hA!g@ g@ g@ g@ e>e>e>e>e>e>e>›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;›j;œk<œk<œk<œk<œk9œk9œkžm>žm>žm>žm>žm> m;Ÿl: m; m; m; m; m; m;žm9žm9Ÿn:Ÿn:Ÿn<Ÿn<Ÿn<Ÿn<Ÿn<Ÿn<Ÿn< o=¢o=¢o=¢o=¢o=¢o=¢o=¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¥r@¥r@¥r@¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¦s?¦s?¦t=¦t=¦t=¦t=¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¤sA¥tB¥tB¥tB¥tB¥tB¥tB¥tB¦s?¦s?¦s?¦s?¦s?¦s?§t@§t@§t@§t@§t@§t@§t@¦s?§t@§t@§u>§u>§u>§u>¨uA¨uA¨uA¨uA¨uA¨uA¨uA§t@§t@§t@§u>§u>§tB¨uC¨uC§tB§tB§tB§tB¥r@¥r@¦sA¦sA¤q?¥r@¥r@¤q?¥r@¤r;¤r;¢q=¢q= o= o= o= o=ŸnA oB£nD¥pF«q?«q?­m©i©a¯g܆ÿ­ÿÆÿÄÿ½ÿà ÿÇ ÿÀÿÈÿÎÿÐÿËÿ¼ÿµÿ³ÿ¾ÿÐÿÅ ÿ þ”ÿ™ÿ¨ÿÇ&ÿÎ-ÿÌkõ°OnI,Q7$æ„ÿ¢1ÿÁÿÈÿ­èsÿ:ÿ,ÿÿ$ÿ-Ë‘´Ò<Ó=Å: ³(  ›Š Š ‰!ˆ € ~tqe`YXWVRQ V W ƒ Šª ¶Úß#óú&ÿ(ÿ- -ÿ1ÿ1ÿ6ÿ0ùῼ˜“„‡Œ -«)ÏMík<ÿ†:ÿ”HÿŸ5ÿ¦<ÿ§(ÿ !ÿžÿ ÿÿÿÿÿÿÿœÿÿœÿžÿŸÿŸÿ ÿ ÿŸÿŸÿžÿÿÿÿ›ÿÿžÿžÿžÿŸÿ ÿ¡ÿ¡ÿ¡ÿ ÿ ÿŸÿÿœÿœÿ›ÿ™ÿ™ÿ˜ÿ–ÿ–ÿ—ÿ–ÿ•ÿ•ÿ“ÿ—ÿœÿšÿœÿÿŸÿ¡ÿ¨ ÿ¯ÿ±ÿ°ÿªÿ¦ÿŸý•ýšÿ¨䃦E py‹0Œ6žH §Y)´f6Â|AЊOÒ”WÖ˜[Ö›`Ü¡fÞ¥là§nê±xóºÿÉþÈŽðº~ì¶zì´vç¯qç¬sæ«rã§fÛŸ^ÉH®t-šc7™b6”c8’a6’a6’a6’a6‘`5‘`5‘`5a5a5a5Ž`4Ž`4Ž`4_3Ž`4_3_3Ž`4_3_3_3_3_3_1_1^3Ž]2Œ^2Œ^2Œ^2Œ^2‹]1‹]1‹]1‹]1‹]/‰[-‰[-‰[-‰[/‰[/‰[/‰[/‰[/‰[/‰[/ˆZ.†Z.†Z.ˆZ.ˆZ.‡X/ˆY0ˆY0‡X/‡X/‡X/‡X/†W.…X/…X/†W.†W.†W.†W.†W0†W0†W.†W.†W0…V/…V/…V/…V/„U.„U.„U.„U.‚S,‚S,‚S,‚S,R+R)R)R)R)~Q(~Q(~Q(}P'P'€Q(P)P)~Q*~Q*}P)}P)}P'}P'}P'|O&{N'{N'{N'{N'{N'|O({N'zM&zM&zM&zM&zM&zM(xK&vL&vL&uK%uK%uK%uK%tJ&uK'tJ&sI%sI%sI%sI%rH$rH$rH$rH$rH$rH$rH$qG#qG#nE#nE#nE#nE#nE#nE#mD mD mD"mD"lC!lC!lC!lC!kB kB kB kB kB kB jAjAjAjAhBhBhBhBgAgAgAgAgAe?e?e?›j;›j;›j;›j;›j;›j;›j;›j;›j;œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œkžm>žm>žm>žm>žm>žm>žm>žm> m; m; m; m; m;¡n<¡n<¡n<Ÿn:Ÿn:Ÿn:Ÿn:Ÿn<Ÿn<Ÿn< o= o= o= o= o=¢o=¢o=¢o=¤q?¤q?¤q?¤q?¤q?¤q?¤q?¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¦s?¦s?¦t=¦t=¦t=§u>¦sA¦sA¦sA¦sA¦sA§tB§tB§tB¥tB¥tB¥tB¥tB¥tB¥tB¥tB¥tB§t@§t@§t@§t@§t@§t@§t@§t@¨uA¨uA¨uA¨uA¨uA§t@§t@¨uA§u>§u>¨v?¨v?¨uA¨uA¨uA¨uA¨uA¨uA¨uA§t@§t@§t@¨v?¨v?§tB¨uC¨uC§tB§tB§tB§tB¦sA¦sA¦sA¦sA¥r@¥r@¥r@¢o=¤q?¢p9¡o8Ÿn:Ÿn:žm;žm;žm;Ÿn<ŸnA£rE¨sI£nD¦l:žd2ž^³s#ß— ýµ>ÿÏ6ÿÏ6ÿÑÿËÿµÿµÿÇ ÿÎÿÕÿÖ ÿ×$ÿâ/ÿÛÿËÿºÿ°ÿ»ÿÕÿÆÿ›ÿÿ¨ÿºÿ¨¬gV -"ºX÷•$ÿ®ÿÂÿÂ$ü‡ÿ2ÿ0ÿ% ÿ#ÿ1é« ­¹#Â,·,ª™™‰ˆ‡‡~|srge^]ZWTQ T V „ ‹« ·ÙÞ"ô û'ÿ+ÿ/ ÿ0ÿ1ÿ3ÿ6ÿ1ùÏŤ -›Œ‹{„•›¯3ÆMÜcë|üÿ™ ÿ ÿ¡ ÿ¡ ÿ£ÿ£ÿ£ÿ£ÿ¡ÿœÿ›ÿœÿžÿŸÿŸÿ ÿ ÿŸÿÿÿÿÿÿÿžÿžÿžÿ ÿ ÿ¡ÿ¡ÿ¡ÿ ÿ ÿžÿÿœÿšÿ™ÿ›ÿ™ÿ™ÿ—ÿ–ÿ—ÿ–ÿ•ÿ–ÿ•ÿ“ÿ•ÿ–ÿ–ÿ˜ÿšÿÿžÿ ÿ¡ÿ¢ÿ£ÿ¦ÿªÿ«ÿ¦þ›ü›ûšè‡*Ëj  EŽ3…/9—I©[+²l1Àz?ĆIÈŠMЕZÙžcÛ¢iÛ¢iÛ¢iå¬sê´zê´zé³wè²vå­oã«mä©pá¦mÝ¡`ДSÉB¯u.f:™b6”c8’a6’a6’a6’a6’a6’a6’a6a5a5a5a5a5Ž`4Ž`4Ž`4Ž`4Ž`4_3_3Ž`4_3_3_3_1_1^3^3_3_3Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]/‹]/‹]/‹]/‹]1‹]1‹]1‰[/‰[/ˆZ.ˆZ.‰[/‡[/‡[/‰[/‰[/ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0‡X/…X/…X/‡X/‡X/†W.‡X/†W0†W0†W.†W.†W0…V/…V/…V/…V/„U.„U.„U.„U.‚S,‚S,‚S,‚S,‚S,R)R)R)R)~Q(~Q(~Q(~Q(R)R)€Q*€Q*~Q*~Q*~Q*~Q*~Q(~Q(}P'}P'}P){N'|O(|O({N'|O(|O(|O({N'{N'zM&zM&zM(zM(xN(xN(xN(uK%uK%uK%uK'uK'tJ&tJ&tJ&tJ&sI%rH$rH$rH$rH$rH$rH$rH$qG#qG#nE#nE#nE#nE#nE#nE#nE!nE!lC!lC!mD"mD"mD"mD"kB lC!lC!lC!lC!kB jAkB kB kB iC hBhBhBhBhBgAgAgAgAgAe?œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œk<œkžm>žm>žm>žm>žm>žm>žm>Ÿn?¡n< m;¡n<¡n<¡n<¡n<¡n<¡n<Ÿn:Ÿn: o; o; o= o= o= o= o= o= o=¢q?¤q?¤q?¤q?¤q?¤q?¤q?¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¦sA¦sA¦sA¦s?¦s?¦s?¦s?¦s?¦s?¦s?¦s?¦s?¦s?§t@§t@§u>§u>§u>§u>§tB§tB¦sA§tB§tB§tB§tB§tB¥tB¥tB¥tB¥tB¥tB¥tB¥tB¥tB§t@§t@§t@§t@§t@§t@¨uA¨uA¨uA¨uA¨uA¨uA¨uA§t@§t@¨uA©w@©w@©w@¨v?¨uA©vB¨uA¨uA¨uA¨uA©vB©vB©vB©vB©vB©vB©vD¨uC¨uC¨uC¨uC¨uC§tB¦sA¦sA¥r@£r@ o= o= o=žm;Ÿn<žm9žm9žm9žm9žm9Ÿn:l:œk9©o?¥k;¤dœ\ ¢^¾zê£ÿÀÿÓ ÿÓ ÿÓÿÍÿÍÿÒÿÅÿ¾ ÿÇ"ÿÈ#ÿ¢ ÿ˜ÿ¥ ÿ³ÿÍÿß.ÿÜÿÂÿ«ÿ¹ÿËÿÅ ÿ‘èz›P0\ 10!p)Æ5ÿŸÿ¾ÿÍÿ¤ÿ?ÿ:ÿ'ÿ$ÿ-ÿ& âѧ« œŠ‹ƒƒ}~}|vshf[YWWVUT S TX ‡Œ­ ¹ÙÝ!ò!ø'ÿ(ÿ- ÿ0ÿ2ÿ3ÿ3ÿ3ÿ/ ÿ"é Á¸™š• -¯)²,«"¦¥¦¨!·0ÌKÝ\éo ùÿ&ÿ›4ÿ¢3ÿ£4ÿ¤*ÿ &ÿžÿÿ  -ÿÿÿœÿœÿœÿÿ›ÿžÿŸÿŸÿŸÿœÿžÿžÿŸÿ ÿ ÿ¡ÿ¡ÿ ÿ ÿžÿžÿÿÿœÿœÿ™ÿ™ÿ™ÿ–ÿ–ÿ–ÿ•ÿ•ÿ•ÿ•ÿ—ÿ•ÿ”ÿ•ÿ–ÿ–ÿ™ÿ–ÿ˜ÿšÿ—ÿ™ÿ ÿ¦ -ÿ¥ÿ¡ÿªÿµ ÿ· -ÿ¶ ÿ§ôŽÌj¶T§K«O¶\ÁgÇtÕ‚&ÚŽ2â–:äœIè Mé¡Nê¢Oé£Më¥Oë¥OðªTì§Jê¥Hè¢3äž/Þ–Øâž1È„¢e/Ÿb,”c8”c8’a4’a4’a4’a4’a6’a6’a6’a6’a6‘`5‘`5‘`5‘`5‘`5‘`5‘`5‘`5_4_4_4_4_4_2_2^3^3Ž]2Ž]2_3Œ^2Œ^2Œ^2Œ^2Œ^2Œ^0Œ^0Ž]0Ž]0Œ^0Œ^0Œ^0‹]/‹]/‹]/‹]/‹]/‰]1‡[/‰[/‰[/‰Z1‰Z1‰Z1‰Z1‰Z1ˆY0ˆZ.ˆZ.ˆZ.ˆZ.‡X/‡X/†W.‡X/‡X/‡X/‡X/‡X/‡X/‡X/†W.†W.†W.†W.…V-…V-…V-…V-…V-„U,„U,„U,€S*€S*‚U,€S*€S*€S*€S*€S*R)R)R+€Q*€Q(€Q(€Q(€Q(~Q(~Q(~Q(}P'}P'}P'|O&|O&{N'|O(|O(|O(|O(|O(|O({N'zM&zM&xN&xN&zM&xK$xK$zM&xK&wJ%vI$vI$tJ$tJ$sI%sI%rH$rH$rH$rH$rH$rH$sI%rH$qH&qH&qH&nE#nE#nE#nE!nE!mD mD mD"mD"mD"mD"mD"lC!lC!lC!lC!lC!lC!kB kA"kA"iB"iB"iB"hA!hA!hA!g@ g@ g@ g@ g@ g@ œk<œk<œk<œk<œk<œk<œk<œk<œkžm>žm>žm>žm>žm>žm>Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?¡n<¡n<¡n<¡n<¡n<¡n<¢o=¢o= o; o; o; o; o= o= o= o= o= o= o=¢q?¤q?¤q?¤q?¤q?¤q?¤q?¥r@¥r@¥r@¥r@¦sA¦sA¦sA¦sA¦sA¥r@¥r@¥r@¥r@¥r@¥r@¦sA¦sA¦sA¦s?¦s?¦s?§t@§t@§t@§t@§t@§t@§t@§t@§t@¨v?¨v?¨v?¨v?§tB§tB§tB¨uC¨uC¨uC¨uC¨uC¦uC¦uC¦uC¦uC¦uC¦uC¦uC¦uC§t@§t@§t@§t@§t@§t@¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA©w@©w@©w@©w@©vB©vB¨uA¨uA©vB©vB©vB©vB©vB©vB©vB©vB©vD©vD©vD¨uC¨uC¨uC§tB¥r@¥r@¥r@¢q?Ÿn<Ÿn<žm;œk9œk9›j6›j6›j6›j6›j6œk7™h6˜g5›a1S#£cÁ1ì¨$ÿÅAÿ×&ÿÓ"ÿÍÿÌÿÏÿÓÿÏÿÌ ÿÍÿÊÿÁô•ârñöþ‰ÿ£ÿ¾ ÿÚÿÚÿÃÿ³ÿ¾ÿÖÿ­"£5B9$jJ8¬{M1 aÁz0ÿ©ÿ¾ÿÉÿºÿIÿ:ÿ-ÿ ÿ# ÿ)ãÌ  ‘ „„|}zzyytpd_WVP P Q RVT TX ‡Œ®¹ÙÝ!ô#ø'ÿ(ÿ- ÿ2ÿ6ÿ7ÿ7ÿ3ÿ3ÿ2ÿ$Íà -Ÿ—‘¶+¶0­'­$±(±&´)¶/»4º9À?ÀFÅKÃMÁKÇSÛgézüÿ ÿ¦ÿ¡ ÿ¡ ÿ¥ÿ§ÿ£ÿ ÿžÿžÿŸÿžÿžÿžÿœÿžÿŸÿ ÿ ÿ ÿ ÿ ÿŸÿŸÿŸÿžÿÿÿœÿœÿ™ÿ™ÿ™ÿ—ÿ˜ÿ–ÿ•ÿ–ÿ”ÿ”ÿ–ÿ–ÿ–ÿ–ÿ—ÿ—ÿ—ÿ–ÿ—ÿ–ÿ”ÿ—ÿ˜ÿ–ÿ—ÿœÿœÿžÿ£ÿ§ÿ© -ÿ¯ÿ«)ÿžç‹Ù} ÐvÈn¼i ÁnÅyÉ}!Ì„1Ó‹8Û“@Ü”AÛ•?Ý—AÝ—AÝ—AÙ”7Õ3Ý—(äž/è )õ­6ú¶IÏ‹¡d.Ÿb,•d9”c8”c6”c6”c6”c6”c8”c8’a6’a6’a6’a6’a6‘`5‘`5‘`5‘`5‘`5‘`5‘`5_4_4_4_4_2_2_4_4_4_4_3_3Œ^2_3_3Œ^2Œ^0Œ^0Ž]0Ž]0Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0‹]/‹]/‰]1‰]1‹]1‹]1‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1ˆZ.ˆZ.ˆZ.ˆZ.ˆY0ˆY0ˆY0ˆY0‡X/‡X/‡X/‡X/‡X/‡X/‡X/†W.†W.†W.†W.…V-…V-…V-…V-…V-„U,„U,‚U,‚U,‚U,‚U,‚U,‚U,‚U,€S*R)R)R+R+R)R)R)R)R)~Q(R)~Q(~Q(~Q(}P'}P'}P)}P)|O(|O(|O(|O(|O(|O(|O({N'yO'xN&zM&xK$xK$zM&zM(wJ%vI$vI$tJ$uK%uK'uK'tJ&tJ&sI%tJ&tJ&tJ&tJ&rH$rI'rI'qH&pG%pG%nE#nE!nE!mD mD mD"mD"nE#mD"mD"lC!lC!lC!mD"lC!lC!lC!lB#lB#iB"iB"iB"iB"hA!hA!hA!hA!hA!g@ g@ g@ œk<œk<œk<œk+غ•–xS/(ŒAî£9ÿËÿÈÿÈÿÉÿbí:ÿ+ÿ ÿ ÿ)îÒŸ …giimijhhecVRL J -CCK NXXW -Y ‰ ± -¾Þãø"ü&ÿ( ÿ, ÿ0ÿ5ÿ8ÿ6ÿ5ÿ3 ÿ2 ÿ0 ÿ%ç §¨âsÿžîÞ}ÕhÅX·Iª<§2¬7¸=º?¹=°4¦(¡#˜”™#¡+§>È_(Ò…5ÙŒ<á’3ï Aøž1ü¢5ÿ¡ ÿ¡ ÿ¡ÿŸ ÿÿÿÿÿ ÿ¡ÿ¡ÿ¢ÿ¢ÿ¢ÿ¡ÿ ÿŸÿŸÿÿÿœÿÿÿÿÿšÿšÿ˜ÿ–ÿ–ÿ•ÿ•ÿ•ÿ•ÿ—ÿ—ÿ—ÿ˜ÿ—ÿ–ÿ“ÿ‘ÿŽÿÿÿŽÿÿ“ÿ–ÿ•ÿ•ÿ”ÿ“ÿ—ÿšÿžÿ ÿ¡ÿ¢ÿ¢ÿ¦ÿ¢þ¥þ¥÷¤ú§ì›ïží¡ï£ð§ñ¨û¯ÿ· -ÿÂÿË ÿÌÿÊÿ½4Óˆ¡d.¡d.˜e6—d5”c6”c6”c6”c6”c6”c6”c6”c6”c8”c8’a6’a6’a6’a6‘`5’a6_4_4‘`5_4_4_4_2_2_4_4‘`5_4_4_4_3Œ^2_3_3Ž]0^1^1Ž]0Œ^0Œ^0‹]/Œ^0Œ^0‹]/‹]/‹]/‹]1‹]1‹]1‹]1‰[/‰[/‹]1‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0‡X/‡X/‡X/‡X/†W.†W.†W.†W.†W.†W.…V-…V-„U,…V-‚V*‚V*ƒW+‚V*‚V*‚V*‚U,‚U,‚S*‚S*R)R)R)R)R)R)R)R)S'S'~Q(~Q(~Q(}P'}P)}P)}P)}P)|O(|O(|O(|O(|O(|O(zP(xN&zM&{N'xK$zM&zM&xK$wJ%wJ%xK&wJ%wJ%xK&vI&wJ'wJ'wJ'uK'tJ&tJ&tJ&rI%rI%rI'rI'qH$qH$qH$qH$pG#pG#pG#pG#nE!nE!nE!mD mD mD mD"lC!mD"lC!lC!lC!jC#iB"iB"iB"iB"iB"hA!hA!hA!hA!hA!g@ œk<œk<œk<œk›j;›j;˜g<–e:–e3’a/_/_/Ž]+Ž]+ŒY'ŒY'â™ÿÐOÿÙ'ÿÓ!ÿÈÿËÿÌ ÿÌ ÿÐ -ÿÕÿ¾ü˜ÿ™ÿ¨ÿ´ÿÇÿŸóÿ„ÿ‚ÿ†ÿ†ÿÿ—ÿ§ÿºÿØÿÑ -ÿ³ÿ«ÿÎ:ÿÅ1a- /! ]?jL'#)#& -¾s ÿ¼RÿÒÿËÿÅÿÏÿyê7ÿ*ÿ"ÿ ÿ+öÝ¥¤…{[]_b^ a ``]\Q O GEACMR]]\\Š ± -¾Þä ø"ü&ÿ( ÿ, ÿ/ÿ5ÿ6ÿ6ÿ6ÿ6ÿ4ÿ4ÿ5ù« -à? ÿµ4ÿÀ?ÿ¾ÿÀÿ¹*ÿ°!ÿ¥þíxØcÈMº?°4°4¯1«-¥*ž#š$– &­`·j¶gÆwØ~ß…ï‹ -ý™ÿ¡ÿ§ÿ§ÿ¥ÿ¡ÿÿœÿœÿœÿžÿ¡ÿ¢ÿ¢ÿ¢ÿ ÿ ÿŸÿžÿÿžÿžÿžÿÿšÿšÿ˜ÿ—ÿ—ÿ•ÿ”ÿ•ÿ•ÿ–ÿ–ÿ—ÿ—ÿ–ÿ—ÿ”ÿ”ÿÿÿÿŽÿÿ‘ÿ•ÿ’ÿ‘ÿ‘ÿ’ÿ’ÿ”ÿ“ÿ”ÿ™ÿ˜ÿ¤ ÿ´ ÿ¾ÿÉÿÌÿÐ,ÿÒ.ÿÓ+ÿ×/ÿÙ.ÿÙ.ÿÛ-ÿÜ.ÿÚ-ÿÞ1ÿ×ÿÙÿÓÿÉÿ¹0Í‚¡d.¥h2˜e6—d5•d7•d7•d7•d7”c6”c6”c6”c6”c8”c8”c8‘`5’a6’a6‘`5’a6‘`5‘`5‘`5‘`5‘`5‘`5‘`3‘`3‘`5‘`5‘`5‘`5‘`5‘`5Ž`4Ž`4_3Ž`4_2^1^1^1Œ^0_1_1Œ^0Œ^0Œ^0‹]/‹]/Œ^2‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‰[/‰[/ˆZ.‰[/‰[/ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0‡X/‡X/‡X/‡X/†W.†W.†W.†W.…V-…V-…V-ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+‚U,‚U,‚S*‚S*‚S*R)R)R)R)R)‚S*R)S'S'R)~Q(~Q(~Q(~Q*}P)}P)}P)}P)}P)}P)|O(|O({N'zP(yO'{N'|O({N'{N'{N'{N'zM(zM(zM(xK&zM(xK&xK(wJ'wJ'wJ'uK'tJ&tJ&tJ&rI%rI%rI'rI'qH$rI%qH$qH$qH$qH$qH$pG#nE!nE!nE!mD mD mD nE#nE#mD"lC!lC!lC!jC#jC#jC#iB"iB"iB"hA!hA!hA!hA!hA!hA!l=l=l=l=l=l=l=l=l=žm>žm>l=žm;žm;žm;žm;žm;žm;žm;žm;Ÿn:Ÿn:Ÿn<Ÿn<Ÿn<Ÿn<Ÿn< o= o= o= o= o= o= o= o= o=¢o;¤q=¤q=¤q=¤q=¤q=¤q=¤q=¤q=¤q=¤q=¤q=¥r>¥r>¥r>¥r>¥r>¥r>¥r>¦s?¦s?¦s?¦s?¦s?¦s?¦s?¦s?¦s?¦sA¦sA§tB§tB§tB§tB§tB§tB§tB§tB§tB§tB§tB¨uC¨uC¨uC¨v?¨v?¨v?¨v?¨v?¨v?¨v?¨v?¨v?¨v?©w@©w@¨v?¨v?¨v?¨v?©vB©vB©vB©vB©vB©vB©vB©vB§vB©xD©xD©xD©xD©xD©xD©xD©w@©w@©w@©w@©w@©w@«yB«yB«xD«xD«xD©vB©vB«xD«xD«xD«yB«yB«xD«xD«yB«yB«yB¬zC«yB«yB«yB«yB©vB¨uA¨uC¨uC§t@§t@§t@§t@§t@¦s?¥rC¢o@Ÿnÿ,ÿ' ÿ!ÿ,ÿ* ÷ÜÛ¬ šXSLOJMI J -G G D A???@HS\!W]] ‘³ ½ãéøþ%ÿ(ÿ, ÿ0ÿ5ÿ6ÿ6ÿ5ÿ5ÿ5 ÿ6 -ÿ7#ü"«ÿ†*ÿÂÿ¶ÿ¹ÿÀÿÃÿÃÿÄ ÿÇÿÆÿÀ ÿ±ÿ­ó²L¯2°3§-¡'š#¡*˜&—%¢Z-¸pCœnB™k?ŸnAœk>¡k3©s;´s3À?ЋB×’IߢWð³hÿ±Jÿ®Gÿ¥-ÿ¤,ÿ¢ÿšÿŸÿ¡ÿ¡ÿ ÿžÿžÿÿÿÿÿ›ÿ™ÿ™ÿ˜ÿ—ÿ˜ÿ–ÿ•ÿ•ÿ•ÿ–ÿ–ÿ–ÿ—ÿ–ÿ–ÿ–ÿ•ÿ”ÿ‘ÿŽÿÿ‘ÿ“ÿ“ÿ‘ÿÿÿÿ”ÿ”ÿ”ÿ“ÿ“ÿ–ÿžÿ©ÿ°ÿ¿ÿÃÿÌ ÿÍ ÿÏ ÿÓÿÕÿÔ ÿÕ ÿÕ ÿÖÿÖÿÓÿÏ -ÿÌÿËÿ¹.΃›d3šc2—d7—d7–e8–e8–e8”c6”c8”c8”c6”c6”c6”c6”c4”c4”c6”c6”c6”c6”c6”c6’a6’a6’a6’a6‘`3‘`3‘`5‘`5‘`5‘`5‘`5‘`5‘`3‘`3_2‘`3‘`3_2_2_2_2^1Ž`2Ž`2Ž`2_1Œ^0_1Œ^2Œ^2Ž]2Ž]2Œ^2‹]1‹]1‹]1‹]1‹]1‹]1‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‡Y-‡Y-‡Y-‡Y-†X,†X,†X,†X,…W+…W+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+ƒW+‚V*„V*„V*„V*„V*‚S*‚S*R)R)‚S*‚S*S'S'R)R)R)~Q(~Q*~Q*~Q*~Q*}P)}P)}P)}P)}P)|O(zP(zP(|O(|O(|O({N'{N'{N'{N'zM&zM&zM&zM(zM(xK&xK&xK(wJ'wJ'wJ'uK'tJ&tJ&tJ&tJ&sI%sI%sI%rH$rH$rH$rH$rH$qG#rH$oE!oE!nD nE!nE!nE!nE!nE#mD"mD"mD"lC!lC!jD!jD!iC iC kB kB kB jAjAjAl=l=l=l=žm>žm>žm>žm>žm>žm>žm>žm>žm;žm;žm;žm;žm;žm;Ÿn<Ÿn<Ÿn:Ÿn:Ÿn< o= o= o= o= o=¢q?¢q?¢q?¢q?¢q?¢q?¢q?¢q?¤q=¤q=¤q=¤q=¤q=¤q=¤q=¤q=¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¦s?¦s?¦s?¦s?¦s?¦s?¦s?¦s?§t@§tB§tB¨uC¨uC¨uC¨uC¨uC§tB§tB§tB§tB§tB§tB¨uC¨uC¨uC¨v?¨v?¨v?©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©vB©vB«xD«xD«xD«xD«xD«xD©xD©xD©xDªyEªyEªyEªyEªyE«yB«yB«yB«yB«yB«yB«yB¬zC«xD«xD«xD©vB«xD«xD«xD«xD«yB¬zC¬yE«xD«yB«yB«yB¬zC¬zC«yB«yB©w@©vB¨uA¨uC§tB§t@§t@§t@¦s?¦s?¥r>¢o@žk<œk9™h6“g7’f6˜c5—b4‘^/Z+\/ˆW*ŠX4]9‚<i#¯\ÿÀIÿÒÿË ÿÒÿØÿÑÿÎÿÎ ÿÍÿÊÿÐ ÿÏÿ¼ÿšÿšÿ ÿÀ ÿ—1àdÿ‡ÿ‘ÿÿ‘ÿ—ÿžÿ ÿ­ÿËÿÛ*ÿÇÿ·ÿÐ ÿÔ\Ž@%C ¯së¯VÿÒ8ÿ×=ÿÐBý·)ŠM)/…4é˜Gÿ¼ÿËÿÅÿÐÿŠè@ÿ*ÿ' ÿ" ÿ,ÿ.ÿ+ þ*ô »¢_XKLHKEFDC@=;>;@M\#c(\!`^Œ ‘³ »âéú!ÿ&ÿ(ÿ, ÿ0ÿ3ÿ4ÿ4ÿ4ÿ4ÿ3ÿ7 ÿ/âÍ<ÿªNÿ¶ÿ³ÿµÿ¼ÿ»ÿ¼ÿ¼ÿÀÿ¿ ÿÁ ÿÃÿÒÿÈIÂ\ª-·:¯5§-š#&Ÿ-”"’J¸pC¢tHœnB£rE oB£m5¡k3¦e%¬k+¯j!p+Nk.©RÕ~ü•ÿ¢*ÿ¯ÿ¶"ÿ´ÿ®ÿ§ÿ£ÿšÿ›ÿœÿšÿšÿ™ÿ›ÿ™ÿœÿšÿ™ÿ˜ÿ—ÿ–ÿ–ÿ–ÿ•ÿ–ÿ–ÿ˜ÿ—ÿ–ÿ–ÿ—ÿ–ÿ”ÿÿÿ‘ÿ‘ÿ‘ÿÿŽÿÿ‘ÿ’ÿ’ÿ”ÿ–ÿ˜ÿ™ÿ¢ÿ«ÿ°ÿºÿ¿ÿÆÿÉ ÿÍ ÿÐ ÿÓ ÿÑ -ÿÒÿÔ -ÿÒ -ÿÔ ÿÑ ÿÎ ÿËÿËÿ¶+Ìœe4f5™f9™f9—f9—f9—f9–e8–e:–e:•d7•d7•d7•d7”c4”c4”c6”c6”c6”c6”c6”c6’a6’a6’a6’a6‘`3’a4’a6’a6’a6’a6’a6‘`5‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3a3a3a3Ž`2Ž`2Ž`2_3Œ^2Ž]2Ž]2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]1‹]1‹]1‹]1‹]1‰[/‰[/‰[/‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.ˆZ.‡Y-‡Y-‡Y-‡Y-‡Y-†X,†X,…W+†X,„X,„X,„X,„X,ƒW+ƒW+ƒW+ƒW+…W+„V*„V*„V*„U,„U,„U,„U,„U,‚S*‚T(‚T(‚S*R)R)R)R+~Q*~Q*~Q*~Q*~Q*}P)}P)}P)}P){Q)zP(|O(|O(|O(|O(|O(|O({N'{N'zM&zM&{N)zM(xK&xK&xK(xK(xK(wJ'uK'uK'tJ&tJ&tJ&tJ&sI%sI%sI%sI%sI%sI%sI%rH$rH$rH$rH$qG#pG#pG#pG#nE!nE#mD"mD"mD"mD"lC!jD!jD!jD!jD!kB kB kB kB kB jAžm>žm>žm>žm>žm>žm>žm>žm>žm>žm>žm>žm>Ÿn<Ÿn<Ÿn<Ÿn<Ÿn<Ÿn<Ÿn< o= o; o; o= o=¢q?¢q?¢q?¢q?¢q?¢q?¢q?¢q?¢q?¢q?¢q?¢q?¤q=¤q=¤r;¤r;¤q=¥r>¥r>¥r>¥s<¥s<¥s<¥s<¦s?¦s?¦s?¦s?¦s?¦s?¦s?§t@§t@§t@§t@§t@§t@§t@¨uA¨uA¨uC¨uC¨uC¨uC¨uC¨uC©vD©vD¨uC¨uC¨uC¨uC¨uC¨uC©vD©vD©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@«yB«yB«y@«y@©w@©w@«xD«xD«xD«xD«xD«xD«xD«xD©xD©xDªyEªyEªyEªyEªyEªyE«xD¬yE¬yE¬yE¬yE¬yE¬yE¬yE«xD«xD«xF¬yG«xF«xF¬yE¬yE¬yE¬yE¬yE¬yE«yB¬zC¬yE¬yE«xD«xD«yB©w@©vB¨uA¨uC¦sA¦s?¦s?¤s?¢q=¢q?¢q?l:™h6—f7—f7’f:d8’a6^3„[.‚Y,‚U,‚U,ŒIq.€#–9õ”ÿË)ÿÊ -ÿÌ ÿËÿ½ÿªÿ­ÿÅÿÐ ÿÒÿÐ ÿÏÿÐÿµÿ¥ÿ™ÿ·ÿÂ8-™æ_3ÿ‡ÿ‹ÿÿ—ÿÿ¡ÿ§ÿ·ÿÑÿÔÿ¿ÿ·ÿØ(䆒!ñ€ÿËÿÖÿÎÿË ÿÅÿÚÎxL*Þ{ÿ¤ÿÇ ÿÇÿÎÿƒæ>ÿ+ÿ( ÿ&ÿ/ ÿ1ÿ6ÿ4ÿ)ؾtlJIDFGGFE=;===AM\#c$`!c^~¢ªØâ ù#ý'ÿ%ÿ+ÿ-ÿ1 -ÿ1ÿ0ÿ/ ÿ/ ÿ2ÿ. å Éèqÿ·<ÿ³ÿ·ÿ³ÿ¹ÿ¹ÿ¹ÿ¹ÿ¹ÿ¸ÿ·ÿÂÿØ%ÿ£.²?¤+µ<±=«7š%š%¢0–$Ž; -¿l;¥rE¢oB¢tH¢tH rD¢tFŸt<¢w?—vP<$2_5xN4¬l@Ñ‘eù¤]ÿ¬eÿ°=ÿ¯<ÿ§#ÿ¢ÿ  ÿž ÿÿ›ÿ—ÿ•ÿ™ÿšÿ˜ÿ—ÿ—ÿ—ÿ—ÿ–ÿ—ÿ˜ÿ˜ÿ™ÿ˜ÿ˜ÿ˜ÿ—ÿ•ÿ”ÿ”ÿ”ÿ”ÿ’ÿÿÿÿÿÿ“ÿ•ÿ˜ÿ™ÿ¡ÿ«ÿ²ÿ»ÿÁÿÉ ÿËÿÎ ÿÑÿÔÿÓÿÕÿÕÿÔÿÖÿÓÿÏ -ÿÊ ÿÉ ÿµ.É}f:žg;šg8šg8˜g:—f9—f9—f9—f;—f;—f9—f9–e8–e8•d5•d5•d5•d5•d5•d5•d7”c6”c8”c8”c8”c8”c6’a4’a6”c8”c8’a6‘`5‘`5‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3_2_2_2Ž`4_3^3^3Ž]2Ž]2Ž]2Ž]2Œ^2‹]1‹]1‰[/‹]1‹]1‹]1‹]1‰[/‹]1‹]1‹]1‹]1‰[/‰[/ˆZ.‰[/ˆZ.ˆZ.‡Y-‡Y-ˆZ.‡Y-‡Y-‡Y-‡Y-‡X/†W.„W0„W0„W0„W0„W0„W0„W0ƒV/…V/…V/„U,…V-„U,„U,…W+„V*„V*„V*‚T(‚T(‚S*‚S*R)R)R+R+R+R+~Q(R)~Q(~Q(}P)}P){Q){Q)}P)}P)|O(|O(|O(|O({N'{N'{N'{N'{N){N)zM(zM(xK&xK&xK(xK(uK%uK%tJ$tJ$tJ$tJ$tJ$tJ$sI!sI!sI#sI#sI#rH"rH$sI%rH$rH$qG#qG#qG#qG#oE!oE!nE!mD mD mD kE jDlClClCkBkBkBkBkBžm>žm>žm>žm>Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn<Ÿn<Ÿn<Ÿn<Ÿn<Ÿn< o= o= o; o; o=¢q?¢q?¢q?¢q?¢q?£r@£r@£r@£r@£r@£r@£r@£r@¤q=¥r>¥s<¥s<¥r>¥r>¥r>¥r>¦t=¦t=¦t=¦t=¦s?¦s?¦s?§t@§t@§t@§t@§t@§t@§t@§t@¨uA¨uA¨uA¨uA¨uA©vD©vD©vD©vD©vD©vD©vD©vD©vD©vD©vD©vD©vD©vD©vD©vD©w@©w@©w@«yB«yB«yB«yB«yB«yB«yB¬zC¬zC«y@«y@«yB«yB«xD«xD¬yE¬yE¬yE¬yE¬yE¬yEªyEªyEªyEªyEªyEªyE«zF«zF¬yE¬yE¬yE¬yE¬yE¬yE­zF­zF¬yE¬yE¬yG­zH¬yG¬yG¬yE¬yE¬yE¬yE¬yE«xD¬zC¬zC«xD¬yE¬yE¬yE«yB«yB©vB¨uA§tB¦sA¥r>¦s?£r> o;Ÿn<žm;›j8—f4–e6–e6d8‹_3‹Z/ˆW,}T'|S&~Q(i<d!i&™<¹\ÿ« ÿË)ÿÍ ÿÎÿÁù˜ÿ“ÿ¤ÿ¼ÿÌ ÿØÿÒÿÍ ÿÓÿÑ!ÿ¼ ÿŸÿ¨ÿÎDÇdx«$Úaÿ†ÿ‘ÿ–ÿœÿŸÿ¤ÿ§ÿ¾ÿÚÿÇÿ¹ÿÍÿ¿ÿÿªÿ¶ÿà ÿÌÿË ÿÌÿÌÿ¼H’<Ÿ<Ýzÿ¦ÿÈÿÈ ÿÌ ÿä<ÿ.ÿ* ÿ)ÿ/ ÿ5ÿ1ÿ. ÿ- á$Ê { tQ -OI J K J -FE><@A?AF PY^ca~}Ÿ ¨×áú$ÿ* ÿ)ÿ.ÿ0 ÿ1 -ÿ0ÿ0ÿ/ ÿ0 ÿ4ÿ"Ï -Ï -ÿÿ­2ÿ·ÿ´ÿ³ÿ·ÿ¸ÿ¹ÿ¹ÿ¼ÿ¾ÿ¸ÿÅÿÕ"Öc*²9²9¯;®:(š%¤2˜&‰6½j9¥rE¤qD rF rF¡sE¡sEžs;§|Dœ{UC"  3GkŒ7ªWÊw÷Ž -ÿ™ÿ¡ ÿ§ÿ§ÿ£ÿ¡ÿÿ™ÿ˜ÿ–ÿ•ÿ—ÿ—ÿ—ÿ—ÿ—ÿ—ÿ—ÿ™ÿ™ÿ˜ÿ˜ÿ˜ÿ—ÿ•ÿ–ÿ•ÿ•ÿ”ÿ“ÿÿÿÿ‘ÿ‘ÿ•ÿ™ÿšÿ¢ÿ­ÿ²ÿ»ÿÁÿÉ ÿËÿÎ ÿÑÿÓÿÓÿÕÿÔÿÕÿÕÿÓÿÐ ÿÉ ÿÉ ÿ¶/Ì€f: i=›h9›h9˜g:˜g:˜g:—f9—f;—f;—f9—f9—f9—f9—f7–e6–e6–e6–e6–e6–e8–e8•d9•d9•d9•d9•d7•d7•d9•d9•d9”c8”c8’a6’a4‘`3’a4’a4’a4’a4‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3‘`3Ž`4_3^3^3^3^3^3Ž]2Œ^2‹]1Œ^2Œ^2‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.ˆZ.‡Y-‡X/‡X/…X1…X1…X1„W0„W0„W0„W0„W0…V/…V/…V-…V-„U,…V-…W+„V*„V*„V*„V*„V*‚S*‚S*‚S*‚S*R+€S,R+R+~Q(R)R)~Q(~Q*~Q*|R*|R*}P)}P)}P)}P)|O(|O(}P)|O({N'|O({N){N)zM(zM(xK&xK&zM*xK(uK%vL&vL&uK%uK%uK%tJ$tJ$tJ"tJ"sI#sI#sI#sI#sI%sI%rH$rH$rH$rH$qG#qG#qG#oE!nE!nE!mD mD kE kE lClClClClClCkBkBŸn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn<Ÿn< o= o= o= o= o= o=¢q?¢q?¢q=¢q=¢q?¢q?£r@£r@£r@£r@£r@¤sA¤sA¤sA¤sA¤sA¤s?¤s?¦t=¥s<¦t=¦t=¦t=¦t=¦t=¦t=§u>§u>§u>§u>§t@§t@§t@§t@§t@§t@§t@¨uA¨uA¨uA¨uA¨uA¨uA¨uA©vB©vB©vB©vB©vB©vB©vB©vB©vB©vB©vB©vB©vB©vB©vB«xD«xD«xD«yB«yB«yB«yB«yB¬zC¬zC¬zC¬zC¬zC¬zC¬zC¬zA¬zA¬zA¬zA«xD«xD­xE­xE­xE­xE­xE­xE¬yE¬yE¬yE¬yE­zF­zF­zF­zF¬yE­zF­zF­zF­zF­zF­zF­zF­zF¬yE­zF­zF¬yG¬yG¬yG¬yG­xE­xE¬wD­xE¬yE«xD¬yE¬yE¬zC«yB©vB©vB¨uC¦sA¤q?¤q?¢q=Ÿn:l:œk9™h6—f4“e5’d4Ž`0‹]-‡Z3ƒV/ƒW)}Q#ƒN{Fk)d"‡)™;ÁQÝmÿ®ÿÌÿÑÿÎÿÀ ÿ¦ÿ¥ÿ§ÿªÿ¾ÿÐÿÑÿÒÿÍÿÇÿ· ÿ ÿžÿÂÿ¬Š‚Ã/ùe0ÿŽÿšÿ–ÿ˜ÿ™ÿ—ÿ³ÿÎÿÎÿ»ÿ½ÿÌ ÿ ÿÿ—ÿ¬ÿ ÿÉÿËÿÆ -ÿÑÿ¼ÿ°ÿÁÿÉÿÆÿÄÿ½ÿbù3ÿ.ÿ, ÿ/ ÿ3ÿ7ÿ3é<ê=¸A¤-n"bQ P O Q MI -DB@?DFH I M N X^\[_[_gŠ ”*Ã+Î6%ê7&í:)ô?)õ@*ù@(ø?'ù?ýC#æ:'È –Ã5ÿªÿ®ÿ±ÿµÿ´ÿ· ÿ·ÿ·ÿ·ÿ¸ÿ¼ÿ½ÿÚJÿ­»<µ6º<³5±:®7Ÿ*›&¤/š%‚2»k7¤q?¥r@£rC£rC¥r>¦s?§m=±wG¡wSA  - 6 -N"˜@ÀhBõ8ÿDÿ4ÿ©@ÿ¨/ÿ¨/ÿ ÿ— ÿ˜ÿ˜ÿ—ÿ–ÿ–ÿ–ÿ˜ÿšÿ˜ÿ—ÿ—ÿ–ÿ”ÿ•ÿ–ÿ–ÿ”ÿ”ÿ”ÿ‘ÿÿÿÿ’ÿ”ÿ—ÿ˜ÿ¡ÿ­ÿ²ÿºÿÀ ÿÆ ÿÊÿÌ ÿÐÿÒÿÒÿÓÿÓÿÓÿÓÿÒÿÏÿÉÿÉÿ´3Î~¢f;¤h=j=›h;™h;™h;˜g<˜g<˜g<˜g<˜g:˜g:—f9—f9—f9—f9—f7—f7—f7—f7—f9—f9–e:–e:–e:–e:•d7•d7•d9•d9•d9•d9•d7•d7”c6”c6”c6”c6”c6’a4‘`5‘`5‘`5‘`5‘`3_2‘`3‘`3‘`3‘`3_4_4_4_4_4^3^3Ž]2Ž]2^3Ž]2Ž]2Ž]2Ž]2Œ^2Œ^2‹]1Œ^2Œ^2Œ^2Œ^2Œ^2‹]1‹]1‹]1‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.ˆZ.ˆZ.‡Y-‡X/‡X/‡X/‡X/…X/…X/„W.„W.†W.†W.…V-†W.…W+…W+…V-…V-…W+…W+„V*„V*„U,‚S*‚S*‚S*‚S*‚S*‚S,‚S,R)R)R)R)~Q*~Q*|R*|R*~Q*}P)}P)}P)}P)}P)}P)|O({N'|O(|O*{N){N){N)xK&zM(zM(xK&xN(vL&vL&uK%uK%uK%uK%tJ$tJ"tJ"tJ$tJ$tJ$sI#sI%sI%sI%sI%rH$rH$rH$rH$qG#oE!oE!oE!nEmDmD mD lClClClClClClCkBŸn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn?Ÿn? o@ o= o= o= o= o= o= o= o=¢q?¢q?¢q=£r>£r@£r@£r@£r@£r@¤sA¤sA¤sA¤sA¤sA¤sA¤sA¤s?¥t@§u>¦t=§u>§u>§u>§u>§u>§u>§u>§u>¨v?¨v?§t@§t@¨uA¨uA¨uA¨uA¨uA¨uA©vB©vB©vB©vB©vB©vB«xD«xD©vB«xD«xD«xD«xD«xD«xD«xD©vB©vB©vB©vB©vB©vB¬yE¬yE«yB«yB«yB«yB¬zC¬zC¬zC¬zC¬zC¬zC­{D­{D­{B­{B­{B­{B¬yE¬yE®yF®yF®yF®yF®yF®yF­zF­zF¬yE¬yE­zF­zF®{G®{G­zF­zF­zF­zF­zF­zF­zF®{G­zF­zF­zF®{G­zH¬yG¬yG¬yG®yF®yF®yF­xE¬yE¬yE¬yE¬yE«yB¨v?¨uA¨uA¤q?¡n< m;Ÿl:l8œk7™h6—f4–e3’a/Ž`0Ž`0‹]-†X(€S,}P)wKuIs> a,f$|:©K®P ÊZèxÿ³ÿÉÿÑÿÎÿÍÿ½ÿ£ÿ¦ÿªÿ®ÿÁ ÿÉÿÉ ÿ¸ÿ£ÿœÿšÿÿ´ÿÍ×]{¥ÚFÿ…ÿ•ÿ’ÿ’ÿ’ÿ”ÿªÿÉ ÿÕÿ½ÿºÿÇÿÿŠÿ–ÿ¢ÿ¶ÿÄ ÿÌÿÌÿÍÿÕÿÓ%ÿÐ"ÿËÿÌÿË&ÿ§ÿ<ö0ÿ3ÿ* -ÿ2ÿ3ÿ5ÿ9öI$ùL'ÂK ©2m!\RURP K I -CCAB I MQSWY]_^]aZY RdjœªÉÎ -Ô Û&ä+æ-æ, ê0Ò&´ˆÚLÿ´ÿªÿ°ÿ³ÿ±ÿ²ÿ²ÿ±ÿ´ÿ¸ÿ»ÿÇÿØH܃´5ÄEÅG¿A¼E·@ +š%£.š%‚2¹i5¦sA¦sA¤sD¥tE§t@§t@©o?®tD vR= - -Kgœ(«7ÇSÐ\álú… ÿ– ÿ¡ÿ¦ÿ©ÿ  -ÿ˜ÿ“ÿ•ÿ–ÿ”ÿ•ÿ•ÿ•ÿ•ÿ•ÿ–ÿ—ÿ—ÿ”ÿ•ÿ“ÿ‘ÿ‘ÿ‘ÿÿ’ÿ•ÿ•ÿ˜ÿ¡ÿ«ÿ±ÿ¹ÿ¾ -ÿÆ ÿÊÿÌ ÿÎÿÏÿÏÿÑ ÿÓÿÔÿÓÿÐÿÎÿËÿÉþ®-Åu¡e:¥i>j=j=›j=›j=›j?™h=™h=™h=™h;˜g:˜g:˜g:˜g:—f9—f7—f7—f7—f7–e8–e8•d9•d9–e:–e:•d7–e8–e:•d9•d9•d9•d7•d7•d7•d7•d7•d7•d7”c6”c8”c8”c8”c8’a4’a4’a4’a4’a4’a4’a6‘`5‘`5‘`5‘`5‘`5‘`5_4_4_4_4^3^3^3_3Œ^2‹]1Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]1‹]1‹]1‰[/‰[/‰[/‰[/‰[/ˆZ.‰[/‰[/ˆZ.ˆZ.‡X/ˆY0‡X/‡X/…X/…X/…X/…X/†W.†W.†W.†W.†X,†X,†W.…V-…W+…W+„V*…W+…V-„U,„U,„U,„U,‚S*‚S,‚S,€S*€S*€S*€S*R+~Q*}S+|R*}P)}P)}P)}P)}P)}P)}P)}P)}P)|O(|O*|O*{N){N)zM(zM(zM({N)xN(vL&vL&uK%vL&vL&uK%uK%uK#uK#tJ$tJ$tJ$sI#sI%sI%sI%sI%rH$rH$rH$rH$qG#oE!oE!oE!nEnEnE!nE!nE!nE!mD mD mD lClClC¡n<¡n<¡n<¡n<¡n<¡n<¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¤q?¤q?¤q?¥r>¥r>¥r>¥r>¥r@¥r@¥r@¦sA¦sA¦sA§tB§tB§u>§u>§u>§u>§u>§u>§u>§u>§u>§u>§u>¨v?¨v?¨v?¨v?¨v?¨uA¨uA¨uA¨uA¨uA¨uA¨uA©vB©vB©vB©vB©vB«xD©vB«xD«xD«xD«xD«xD«xD«xD«xD«xD«xD©vB©vB©vB«xD«xD«xD¬yE¬yE«yB«yB«yB¬zC¬zC¬zC¬zC­{D­{D­{D­{D­{D­{D­{D­{D­{D­zF¬yE®yF®yF®yF®yF®yF®yF­zF­zF¬yE¬yE­zF­zF®{G®{G®{G®{G®{G®{G®{G®{G®{G¯|H®{G®{G®{G®{G®{G­zF¬yG­zH¯zG®yF®zD­yC­yC­yC­yC¬xB¨v?§u>§t@¦s?¢o= m;žk9j8›j8›j8–h8“e5b2Œ^.‰Z1Œ]4…V/P)wN,sJ(qHlCw5s1ŸC¨L¼NÈZÛið~ÿ´ÿÉÿÐÿÏÿÕÿÉ ÿ©ÿ¦ÿ¢ÿžÿ—ÿ•ÿ‘ÿÿÿ“ÿ—ÿÿ¥ÿÇ ÿ¿1.ŸÂ ìcÿÿÿÿÿÿœÿ» -ÿ×ÿÀÿ²ÿÄÿ» ô–þ‹ÿ–ÿ©ÿ¿ÿËÿÌÿÏÿÌÿÍÿÌÿÅÿÏ$ÿ§Aÿ`ÿ'ÿ-ÿ4ÿ.ä/Ú%±1ÈH-­a:£W0El2^Vaac^P M B -D H KRUW[^a!g!ddeg`PF)$ 4¬s<ÿ´ÿ« ÿ©ÿªÿ©ÿ¬ÿ­ÿ®ÿ² ÿ¶ ÿ¼ -ÿÉÿ¶?¿U¶1¾9¾8À:¿B¼?¨*Ÿ! +š%…2»h3©vD§tB¡uE¡uE£uB£uB¡s>§yD–uK3  @T ˆˆ‡~v{ ˆ¢3ÇQ$Ûe8ðyÿŽ3ÿš'ÿš'ÿ™ÿ›ÿ“ÿŽ -ÿ”ÿÿÿŽÿŽÿÿ‘ÿ‘ÿ‘ÿŽÿ‘ÿ‘ÿ‘ÿ’ÿ”ÿÿ–ÿžÿ¨ÿ¯ÿ·ÿ½ -ÿà ÿÈÿË ÿÍ ÿÎ ÿÎ ÿÑ ÿÔÿÔÿÒÿÐÿÎÿÉÿÆÿ§,Çn¬`2²f8›j=›j=j=j=›j=™h;™h;™h;›h;šg:šg:šg:šg:šg:šg:šg:šg:šg:˜g<—f;—f;—f;—f;—f;—f9—f9—f9–e8–e6–e6–e6–e6•d7•d7•d7•d7•d7•d7•d9•d9•d9•d9”c6”c6’a4”c6”c6’a4’`8’`8’`8‘_7‘_7‘_7‘_7‘_7_4_4_4_4_4_4Ž]2Ž]2^3^3^3^3^3Ž]2Ž]2Ž]2Ž]2Ž]2Ž]2Ž]2\1\1\1\1‹Z/‹Z/‹Z/‹Z/‰[/ˆZ.ˆZ.ˆZ.…Y-…Y-…Y-…Y-‡Y-†X,†W.†W.†W0†W0†W.†W.†X,…W+†X,…W+…V-…V-…V-…V-„U,„U,„U,„U,‚T(‚T(‚S*‚S*€S,€S,R+R+R+R+R+~Q*~Q*~Q*~Q*}P)}P)}P)|O*|O*{N)zM({N){N){N){N)xN(xN(xN(xN(vL&vL&vL&uK%uK#uK#uK%tJ$tJ$uK%sI%sI%sI%sI%rH$rH$rH$rH$rH$qG#rH$rH$nE!pG#pG#nE!pG#pG#mD mD mD lClClC¡n<¡n<¡n<¡n<¡n<¡n<¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¤q?¥r@¥r>¦s?¦s?¦s?¦sA¦sA¦sA¦sA¦sA§tB§tB§tB§u>§u>§u>§u>§u>§u>§u>¨v?¨v?¨v?¨v?¨v?¨v?¨v?¨v?¨v?¨uA¨uA¨uA©vB©vB©vB©vB©vB©vB¨uA©vB©vB«xD©vB©vB©vB©vB©vB©vB¨uA©vB¨uA¨uA¨uA¨uA¨uA¨uA©vB©vB©vB©vB¬yE©w@¬zC«yB«yB­{D¬zC¬zC¬zC­{D­{D®|E®|E­{D­{D­{D­{D­zF­zF®yF®yF®yF®yF®yF®yF­zF­zF­zF­zF­zF­zF­zF­zF®{G®{G®{G®{G®{G®{G®{G¯|H®{G®{G®{G®{G®{G®{G¬yG­zH¯zG®yF­yC­yC­yC­yC­yCªv@§u>¦t=¥r>¤q= m;žk9šg5˜e3–e3”c1a1_/‹]-†X(R)P'~O(zK$qH&mD"h?lC†D ŠH°T»_Ê\Ðbætô‚ÿ±ÿÊÿÖÿÏÿÔÿÖÿ´ÿŸÿŽÿ‚úƒÿŠÿ’ÿ–ÿ—ÿ›ÿœÿžÿ¥ÿ±ÿØJ늩­ ÍDúq ÿ‡ÿ‹ÿ‹ÿˆÿ—ÿ¶ÿ×ÿÄÿ°ÿ½ ÿÐ ÿ±ÿÿ“ÿ²ÿÅ ÿÊÿÉÿÎÿÐÿÎ ÿÑ ÿÍ"ÿ· ÿ`Ü5ÿ+ÿ/ÿ4ÿ+Ò¹ ¦& ‰=†:k1k1q1o/p&j ieUM FHKPVY[]a!c#g!i#h"i#l eUK -'# -  5¾…Nÿ¯ ÿ¦ÿ©ÿ¯ ÿ¶ ÿµ -ÿµ ÿ´ ÿ¼ÿ·ÿ¿ ÿÒ ÿ¥.¥;©$±,²,µ/´7´7¤&œ(˜#…2½j5«xF©vD¢vF¢vF¤vC¤vC¡s>§yDoE.  -?S …€wqlrpŸ(ÆOÞbæjû~ÿ‡ ÿˆÿ‘ ÿ–ÿ˜ ÿ”ÿ•ÿ–ÿ’ÿÿ‹ÿÿÿŽÿÿŽÿÿÿÿ“ÿ›ÿ§ÿ¯ÿ¸ÿºÿà ÿÈÿË ÿÌ ÿÎ ÿÏÿÓÿÔÿÓÿÒÿÐÿÏÿÉÿÅö"¼cª^0²f8œk>›j=j=j=›j=›j=™h;™h;›h;›h;›h;›h;›h;›h;›h;›h;šg:›h;˜g<˜g<˜g<˜g<˜g<˜g<˜g:˜g:˜g:—f9—f7—f7—f7–e6–e8–e8–e8–e8•d7•d7•d9•d9•d9–e:”c6•d7•d7”c6”c6”c6”b:”b:’`8’`8’`8‘_7‘_7‘_7‘`5‘`5‘`5‘`5_4_4_4_4^3^3^3^3^3^3^3Ž]2Ž]2Ž]2Ž]2Ž]2Ž]2\1\1\1\1‹Z/‹Z/‹Z/‰[/‰[/ˆZ.ˆZ.†Z.†Z.†Z.…Y-‡Y-‡Y-‡X/‡X/‡X1†W0†W.†W.†X,…W+†X,†X,…V-…V-…V-…V-…V-„U,„U,„U,„V*„V*‚S*‚S*€S,€S,€S,R+R+R+R+R+~Q*~Q*~Q*~Q*~Q*}P)|O*|O*{N){N){N){N){N){N)yO)xN(xN(xN(xN(vL&vL&uK%vL$vL$uK%uK%vL&vL&uK'tJ&uK'uK'tJ&tJ&sI%rH$rH$sI%rH$sI%pG#pG#pG#pG#pG#pG#mD mD mD nE!mD lC¢o=¢o=¢o=¢o=¢o=¢o=¢o=¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¥r@¥r>¦s?¦s?¦s?¦s?¦s?¦s?§t@§t@¦s?§t@§t@§t@¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA©vB©vB©vB©vD©vD©vD©vD©vD©vD¨uC¨uC¨uC©vD§t@¨uA¨uA§t@¦v?¦v?¦uA¦uA¥t@¥t@¥t@¥t@¤s?¥t@¦uA¦uA¥t@¥t@¦uA¦uA¦uA¥t@§t@¨uA¨uA©vB¨uA¨uAªuBªuB©vB«xD«xD«xD­xE®yF®yH­xG­xG­xG­xG­xG­xG­xG­xG­xG­xG­xG­xG­xG®yH®yH®yH®yH®{G®{G­zH®{I®{G®{G®{I®{I®{I­zH­zH®{I®{I®{I­zF¬yE¬yE¬yE©vB¨uA§t@¦s?¦sH¢oDžlDœjB›j=–e8Œ`4Œ`4Ž`2ˆZ,‡Y+…W)~T*xN$iL#eHoI&jD!dE&]>s8‚G­L¸WÅaÉeÑbÙjê|öˆÿ©ÿÈÿÔÿÐ -ÿÇÿÀÿŸñƒï~ì{ÿ‚ÿ–ÿ ÿ¦ÿžÿžÿ£ÿ£ÿ¡ÿ¡ÿ¹ÿÒÉ]}»(ôa!ÿÿƒÿ‡ÿ…ÿ‘ÿ±ÿÖÿÆÿ²ÿÁÿËÿÈÿÀ -ÿ¾ÿÊ ÿÏÿÍÿÓÿÐÿÌÿÃ9ÿ´*ÿ{üGæ.ó;òVû_í^Î?–Y`by5„H‹O‰OE€:u/u&o `VL PSVY[[]e"f#g'i)l&k%i#eXR7 , -  D ‡:ÿ ÿ¢ÿŸ,ÿ”!ÿ{ÿrÿgÿ] ÿW -ÿV ÿZÿ]ÿL ÿ<ÿ?ÿ<ÿ:ÿ=ô<ë3Ó(Ç·!° 5Óh-ÉvA¿l7´r?³q>©rAªsBžuF¥|M‘lL.    CTmmmplk h g jv¯.µ4°(°(·(¼-Å3ÓA -âTí_ÿnÿ0ÿŠ,ÿ‹-ÿ‹%ÿˆ"ÿ†ÿ„ÿ…ÿ…ÿ†ÿˆÿŒÿ‘ÿÿ˜ÿ¤ÿ¬ÿ¸ÿºÿÂÿÇ ÿÊ -ÿË ÿÍ ÿÍ ÿÏÿÐÿÐÿÐÿÐÿÏÿÍÿÆÿ”(ÍS¼Y0Æc:œk<›j;›j?›j?•l;•l;™k;™k;œg9œg9›h9›h9™h9™h9™h;™h;™h=™h=˜g:˜g:™h=™h=™h=˜g<˜g:˜g:˜g8—f7—f7—f7—f7–e6—f7—f7—f7–e6•d7•d7–e8–e8–e8•d7•d7•d7•d7•d7”c8•d9’d8’d8’d8’d8b6b6b6b6a5b6b6a5a5a5a5Ž`4^3^3_4_4_4^3^3^3^3^3^3Ž]2Ž]2Ž]2Ž]2\1\/\/\/‹Z-‰[/‰[/‰[/ˆZ.†Z.†Z.†Z.†Z.‡Y-‡Y-‡X/‡X/‡X/‡X/†W.†W.†W.†W.†W.†W.†X,†X,†W.…V-…V-…V-…W+…W+„V*„V*‚S*‚S*€S,€S,€S,R+R+R+R+R+~Q*~Q*~Q*~Q*~Q*|O(|O*|O*}P+|O*|O*{N)|O*{N){N){N){N)zM(xN(xN(vL&xN(vL$vL$uK%vL&vL&vL&vL(vL(uK'uK'uK'tJ&uK'tJ&tJ&tJ&tJ&sI%rI%qH$pG#pG#pG#pG#pG#pG#pG#nE!pG#mD ¢o=¢o=¢o=¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¤q?¥r@¥r@¥r@¥r@¥r@¥r@¥r@¦sA¦s?¦s?§t@§t@§t@§t@§t@§t@§t@§t@¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA¨uA©vB©vB©vB©vD©vD©vD©vD©vD©vD¨uC§tB¦sA¦sA¥r>¥r>¥r>¥r>¤t=¤t=¤s?¢q=£r>¢q=£r> o;¢q=¢q=£r>¤s?£r>£r>¤s?¤s?£r>¢q=¤q=¥r>¥r>¦s?¥r>¥r>§r?§r?¦s?§t@§t@¨uAªuB¬wDªuD©tC©tC©tC©tC©tC©tC©tC¬wF¬wF¬wF¬wF¬wF¬wF¬wF¬wF¬wF¬wF©vB©vB«xF«xF«xD«xD«xF«xF«xF«xF«xF«xF«xF«xF©vB©vB¨uA§t@§t@¥r>¤q=¢o;¡nCšg<•c;”b:”c6^1†Z.‚V*…W)‚T&€R$|N uK!rHcFaDhBc=]>T5f+T½\À_ÌhÏkÛlàqì~öˆÿ¥ÿÅÿÐ -ÿÔÿ¼ûœõ‡ÿœÿ¡ö…ÿ“ÿšÿžÿ¢ÿ¦ÿ¤ÿ£ÿ›ÿ–ÿ—ÿ¥ÿÂÿÀIÁU—ÝJ -ÿÿ€ÿ€ÿ‚÷‰ÿ¯ÿÔÿÃÿ±ÿÂÿÄÿ»ÿÉÿÒÿÎÿÎÿÍÿÑÿÓ!ÿÊÿêhæ1ç2ÿIÿfÿ‰ÿ“#ÿw àQ¤+d [g#‘M)¨l:®r@§m=—]-‘K„>€1v'g_PQUWYZ[^d!g$h(i)l&i#k%i#_"X;- - IÌ‘Dÿ˜ëyí^ÞOóEñCþ8ú4ø/ú1þ4ÿ<ÿAÿHÿE ÿ>ÿ:ÿ=ø@ø@ë@ê?Ö@Ê4®CÐe*Æs>Âo:¸vC·uB¯xG°yHŸvG´‹\“nN+ - - - M Xf jpoh -fi f -o ªDÓRÈGÂ:º2¶'°!² ² ±#±#º&Ä0Ó7âFòUÿfÿwÿ€ ÿŠ ÿÿ’ÿÿŒÿŒÿ‘ÿ•ÿ¤ÿ©ÿ´ÿ¸ÿÀÿÄÿÇÿÊ -ÿË -ÿÌ ÿÌ ÿÍ ÿÏÿÐÿÒÿÈÿÅÿÅÿ’&ÄJ·T+Â_6žm>œk<œk@œk@•l;•l;™k;™k;ži;ži;j;›h9™h9™h9™h;™h;™h=™h=™h;™h;™h=™h=™h=˜g<˜g:˜g:˜g8—f7˜g8—f7—f7–e6–e6—f7–e6–e6–e8–e8–e8—f9–e8•d7•d7•d7•d7•d7•d9•d9“e9“e9’d8’d8’d8’d8’d8b6b6b6b6b6b6a5a5a5‘`5‘`5_4_4_4_4^3^3^3^3^3^3Ž]2Ž]2Ž]2Ž]2Ž]0\/\/\/‹]1‰[/‰[/‰[/‡[/†Z.‡[/†Z.ˆZ.ˆZ.ˆY0‡X/‡X/‡X/‡X/‡X/†W.†W.†W.†W.†X,†X,†W.†W.†W.†W.…W+…W+„V*‚T(„U,„U,€S,€S,€S,€S,R+R+R+R+R+~Q*~Q*~Q*}P)~Q*~Q,~Q,~Q,|O*}P+}P+|O*|O*|O*{N){N){N)xN(xN(yO)yO)xN&vL$vL&vL&vL&vL&vL(vL(uK'uK'uK'tJ&uK'uK'uK'tJ&tJ&sI%rI%qH$pG#pG#pG#pG#pG#pG#pG#nE!pG#pG#¤q?¤q?¤q?¤q?¤q?¤q?¤q?¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¦sA¦sA¦s?§t@§t@§t@§t@§t@¨uA¨uA¨uA¨uA¨uA¨uA¨v?¨v?¨v?¨v?¨v?¨v?¨v?©w@©w@©w@©w@©w@©w@©w@©vB©vB©vB«xD«xF«xF©vD©vD©vD¨uC§tB¦sA¥r@¥r@¥r@¡n<¢o=¤q?Ÿn: o; o=žm;žm>žm>žm>l=l:l:Ÿn< o=Ÿn<Ÿn<Ÿn? o@Ÿn<žm; m; m;¢o=¤q?¢o=¢o=£n=¥p?¤q?¤q?¥r@¥r@§rA¨sB§rA¦q@¦q@¦q@¦q@¦q@¦q@¦q@§rA§rA§rA§rA§rA§rA§rD§rD§rD§rD¦sD¨uF¨uF¨uF¨uF¨uF¨uF¨uF¨uF¨uF¨uF¨uF¨uF¨uF§tB§tB¥r>¥r>¤q?¢o=Ÿn!`;\= T5W+Hb¥\%ËfÊeØoÝtävèzï‚ø‹ÿ£ÿÀÿÓÿÖÿ¾ð›ÿ­ÿÍÿÐ ú¦ÿšÿ¡ÿ ÿ¡ÿ¡ÿ ÿÿ¥ÿµÿ­ÿŸÿ¡ÿÇÿȽX£>äqù†évæsîˆÿ¼ÿÕÿ¹ÿ±ÿÉÿÊÿ¼ÿ¸ÿÅÿÊ!ÿÆÿÃ$ÿ·ÿš-ÿ}óGà4îSÿrÿÿŠÿ–ÿ’ÿ’îËZ¥4sci%œX8²{Jµ~M´{D¥l5•V‡H„7y,j bUTWY\\\_be!e$h'g&b!g(j+d)^#C.   WÓ‹Qý_ âDùBÿJ ÿB ÿ? ÿ8ÿ4 -ÿ&ÿ( -ÿ.ÿ5ÿ7ÿ4ÿ0 ÿ*ÿ" ÿñý$ÿC ÿJÿKÿJÿCÿJÿSÿVÿ]ÿa#ùg*ýk.æh6ÖX&™0 \06#" h' -N1‹, vsu w -o²Mÿ¯7ÿºÿ¨ü—ì‡ÚpÉ_ÇN¾EÉ1È0Ý Ý ðêðô÷ÿ%ÿ8ÿGÿZ!ÿi0ÿ}4ÿ…<ÿ“#ÿ›+ÿ¢ÿ¨#ÿ®ÿ±ÿ½ÿÁÿÅÿÈÿÌÿÎÿÍÿÏÿÏÿÏÿË#ÿÇÿÃEÿ¶8ÿe$ò0ò<üF#¯i:­g8Ÿl=žk<™k;šl<˜l<˜l<ži=ži=j=j=›j=›j=™h=™h=™h;™h;—i;—i;™h=™h=™h=™h=™h;˜g:˜g:˜g:˜g:—f9—f7—f7—f7—f7—f7–e6—f7—f7—f9—f9–e6–e6•d7•d7–e8•d7–e:•d9“e9“e9“e9“e9“e9“e9’d6’d6’d6’d6’d6’d6b4b4b4b4‘`3‘`3‘`3‘`3a5Ž`4Ž`4_3Ž`4Ž`4Ž`4Ž`4_3_3Œ^0Œ^0Ž]0\/\/\/‹]/‹]/‰[-‰[-‰[-ˆZ,‰[-‰[-ˆZ,ˆZ,ˆZ,ˆZ,ˆZ.‡Y-‡Y-‡Y-‡X/‡X/‡X/‡X/†W.†W.†X,†X,†X,†X,…W+…W+…V-„U,„U,„U,‚U.‚U.‚U.€S,€S,€S,€S,R+R+R+~Q*R+~Q,~Q,~Q,~Q,~Q*}P)}P)}P)|O(|O(|O(|O(zP(yO'yO)yO)xO)xO)yO+yO+vL(vL(vL(uK'uK'uK'vL(vL(vL*vL*vL*uK)uK)tJ(tJ&tJ&tJ&sI%sI%sI%sI%sI%rH$rH$nE!pG#nE!pG#¤q?¤q?¤q?¤q?¤q?¤q?¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¦sA§t@§t@§t@§t@§t@¨uA¨uA¨uA¨uA¨uA¨uA©vB©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©vB«xD«xD«xD«xF«xF«xF«xF¨uC¨uC¥r@¤q?¢o=¥r@¥r@ m; m; m;›j6›j6›j8™h6™h9™h9™h9›j;›j8™h6›j8›j8›j8›j8›j;œk<›j8›j8j8j8žk9žk9žk9žk9Ÿj9 k: m;Ÿl:Ÿl:žk9Ÿj9 k:¡l; k: k: k: k: k: k: k:£n=£n=£n=£n=£n=£n=£n@£n@£n@£n@¤qB¥rC¤qB¤qB¤qB¤qB¤qB¤qB¤qB¤qB¤qB¤qB¥rC¥rC¥r@¥r@¢o; m9¢o= m;™h6—f4—f4’a/_2ŠY,†X,S'yO%tJ qH"pG!nDj@e<e<`;X3F' -56 -,]±h1ÙtÔoÝtä{é{î€ö‰üÿŸÿ· -ÿÌÿÔÿÍ þ©ÿ¸ÿÌÿÔ$ÿ¿ÿ˜ÿžÿ£ÿ§ÿ¥ÿ¦ÿ³ÿÆÿÐÿà ÿŸÿžÿ«ÿÉÿËAäÞkÞkÔaâoÿ¥ÿÍ(ÿÎÿ²ÿ»ÿË -ÿÌÿÉÿ» -ÿ¼ ÿºÿ² ÿ éƒáKÐ:ùMÿrÿ‘ÿˆÿ‚ýuöÿœÿ³ÿªï~¹H|c\L,®wF®wF®u>ªq:¡b*ŽOˆ;y,k!eZUWY\^]_`ba b!a \c$o0 p5j/ L2  - a¾v<Ý?æHÿOÿOÿIÿIÿ@ÿ4 -ÿ"ÿ$ÿ' -ÿ.ÿ4ÿ/ ÿ+ÿ)ÿ$ ýêõÿ:ÿE ÿMÿPÿJÿIÿM ÿL ñLïJ âPÞLÇI®0„s -DA($ \†E(Š+zs o qv -w -„ø“ÿÈPÿÄ ÿÇ ÿÈÿ¿ÿ¿-ÿ¸&ÿ«5ÿ›%ÿ{(ô\ ÿF ÿG -ÿFÿ?ÿ8ÿ0 õä -ÒÒÃËÃ/×Cäfú| ÿ–ÿª%ÿ­ÿ²ÿÀÿÆÿËÿÏÿÖÿ×ÿÖÿÖÿÒ!ÿÏÿÄÿ·ÿ–él÷5ö4ï9úD!«e6¯i: m> m>šl<šl<˜l<˜l<Ÿj>Ÿj>žk>žk>›j=›j=›j?›j?›j=™h;—i;—i;›j?›j?›j?›j?›j=™h;˜g:˜g:™h;˜g:˜g8˜g8™h9˜g8—f7—f7—f7—f7—f9—f9—f7—f7–e8–e8–e8–e8–e:•d9”f:“e9“e9“e9“e9“e9“e7“e7“e7“e7“e7b4b4b4b4b4’a4’a4’a4‘`3a5a5Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4Ž`4_3_1_1^1Ž]0Ž]0Ž]0Œ^0‹]/‹]/‰[-‹]/‹]/‹]/‹]/‰[-‰[-‰[-‰[-ˆZ.ˆZ.‡Y-‡Y-‡X/‡X/‡X/‡X/‡X/†W.†X,†X,†X,†X,†X,†X,…V-„U,„U,„U,‚U.‚U.‚U.€S,€S,€S,€S,€S,R+R+R+R+R-R-R-~Q,~Q*}P)}P)}P)~Q*}P)|O(|O(zP(yO'yO)yO)xO)yP*yO+yO+vL(vL(vL(vL(xN*xN*vL(vL(vL*vL*vL*vL*vL*vL*uK'tJ&uK'uK'uK'tJ&tJ&sI%sI%rH$nE!pG#pG#nE!¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥r@¦sA¦sA¦sA¦sA¦sA¦sA¦sA§tB§tB§t@§t@¨uA¨uA¨uA¨uA¨uA©vB©vB©vB©vB©vB©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©w@©vB«xD«xD«xD¬yG¬yG¬yG©vD¨uF¦sD¥rC¢o@ m;¡n< m@Ÿl?j;šg8•d7”c6”c6’a4b6’d8b4’d6’d6b4b4b4“e7“e7‘e9‘e9“e7”f8–e8•d7•d7•d7–e8–e8—f9—f9™h;˜g:–e8•d7—d7˜e8—d5—d5—d5—d5—d5—d5–c4–c4™f7™f7™f7™f7™f7™f7šg:šg:šg:šg:›j=žm@›m?›m?›m?›m?›m?›m?›m?›m?›m?›m?›m?›m?œk<œk<˜g8–e6™h;–e8`7^5Œ^2ˆZ.‚U,R)yO+tJ&oE#j@lC!g>n;l9h4]) G 60.JGl¾eÚvÛwçzîñƒó…û‹ÿ’ÿšÿ¬ÿÅ ÿÑÿØÿ¾ÿ¯ÿÊ -ÿÓÿÓÿ«ÿ¢ÿ¦ÿœÿ›ÿ» ÿÓÿÍÿÍ -ÿÍ -ÿ§ÿÿšÿÿÆ ÿÔÿ¶ñ˜óšÿ³ÿÎÿËÿ½ÿ¿ÿÈÿÏÿÌ,ÿ¿ÿ¤ÿ—ÿŠîuâVÇ;ØJÿ{ÿ &ÿ“ûówê€ü’ÿªÿ¿ÿÎÿÆÿž -Òe—(z Zƒ:¥pD®yM°w@³zC©p7–]$‰D u0o%ia\X[[]_b\[ZYVT[p3|?v9b* E   c¹oAÐ#Û.÷/ ÷/ ÷*ý0ÿ0ÿ*ÿ"ÿ$ ÿ.ÿ6ÿ:ÿ7ÿ1ÿ)ò'ßÓ× æ÷#ÿ/ -ÿ5ÿ6 ÿ<ÿ<ÿ9 ÿ2ÿ)ÿ'ÿ#ÿ-ÿ= ÿEÿ= ù0ì#ÕÉ£˜wqKK,05D kly uxz -t -°Fÿ¼ÿ½ÿ¹ÿ½ÿ¸ -ÿÁÿ¿'ÿº"ÿ¤4ÿŒÿj'ÿRÿIÿLÿP!ÿT%ÿJ"ÿ@ÿ7 ÿ(ÿÿÿ -÷̶uw¤Æ:ÿDÿV&ÿ`"ÿi+ÿz3ÿ~7ÿƒ7ÿ„8ÿ{2ÿt+ÿf$ÿ[ÿK ÿ>ÿ+ÿ$ÿ'ÿ)ÿ.ÿ4 ÆX-Î`5žm9l8œk9l:›m:›m:Ÿl?žk>œk>œk>™k=™k=šl>šl>—k;˜l<šl>™k=›j=›j=›j=›j=›j=›j=™h;™h;™h;™h;™h9˜g8™h9™h9˜g8—f7—f7—f7—f7˜g8šg8™f7˜e8˜e8—f9–e8–e8–e8“e9“e9”f:”f:”f8“e7“e7“e7“e7“e7“e7’d6’d6’d6’d6’d6’a4’a4’a4’a4a3b4b6a5a5a5a5Ž`4Ž`4Ž`4_1_1^1^1Ž]0Ž]0Œ^0Œ^0Œ^0Œ^0‹]/‹]/‹]/‹]/‹]/‰[-‰[-‰[-‰[/‰[/‰[/‰[/ˆZ.‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-‡Y-†W.†W.†W.†W.…V-„U,…V-…V-…V-…V-„U,„U,„U,„U,€S*€S*€S*€S*€S,R+R-R-R-R-~Q*R+R+R+~Q,~Q,}P+}P+}P+}P+zP*zP*yP*yP*zP,zP,yO+xN*vL(vL(xN*xN*vL(xN*xN,vL*vL*vL*vL*vL*uK'uK'uK'uK'uK'sI%tJ&tJ&sI%rH$qG#qG#sI%qG#¥r@¥r@¥r@¥r@¥r@¥r@¥r@¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA§tB§tB§t@¨uA¨uA¨uA¨uA¨uA©vB©vB©vB©vB©vB©vB©w@©w@©w@©w@©w@©w@©w@«yB«yB«yB«yB«yB«yB«yB«xD«xD¬yE¬yE­zH­zH¬yG©vD§tE¦sD¤qB¡n?Ÿl:›h6™f9˜e8˜e6–c4_2^1^1_2_3‰[/‡Y+‡Y+‡Y+‡Y+‡Y+‰[-‹]/‹]/‡[/‡[/‰[-‰[-‹Z-‹Z-‹Z-\/^1^1Ž]0Ž]0Ž]0Ž]0\/Ž]0]0‘^1‘^/‘^/‘^/‘^/‘^/‘^/].].‘^/‘^/‘^/‘^/‘^/‘^/‘^1‘^1‘^1‘^1‘`3’a4b4b4b4b4b4b4b4b4b4b4b4b4”c4•d5‘`1_0^1‹Z-ˆY0†W.…W+S'{N%xK"qG#kAkAf<f=\3e2f3T A /-9 -Cci%†-¿fçƒãï‚ò…÷‰øŠþŽÿ•ÿŸÿ¨ÿÃÿ×ÿÛÿÍ -ÿÃÿÑÿÒÿÕÿÊÿ¦ÿÿ…ÿžÿ¿ÿÐÿÊ ÿÌ ÿÑÿ±ÿ–ÿ”ÿ‘ÿªÿÉÿÕÿÓÿÐÿÊÿÉÿÀÿÄÿÒÿÒ ÿ½ ÿè…îxæpÓZ¾EÕIûoÿ”1ÿ•2ÿ|íiïsÿÿªÿ·ÿ½ ÿ¿ÿÄÿÈÿ®é|¬=†V q( ™d8¯zN²yB´{D²y@¢i0“N}8r(ib]XZ\\\^XTRQKJ Tl/ €C‚Es;U(   -c¹oAÇËæçìí ö÷ÿ ÿ!ÿ% ÿ.ÿ8ÿ;ÿ=ÿ;ÿ9ö+êåðï÷ü#ÿ&ÿ1ÿ7 -ÿ6 ÿ4ÿ1ÿ0ÿ3ÿ5ÿ;ÿHÿQÿY0ÿO&ÿI*ÿD%å>'Ù2¯( pd;7#16_k|~†…†æ|6ÿÂÿ¼ÿ¿ÿÅÿÀÿ· ÿ˜ýyÿYñJÿ?ÿCÿFÿHÿIÿMÿDÿ?ÿ<%ÿ:#ÿ5 ÿ5 ÿ6"ÿ6"õ,áˆq}”Øáèç×ØÞ&à(ê&ï+ö(ü.ÿ6ÿ8ÿ4 ÿ7ÿ1ÿ+ -ÿ.ÿ1ÄV+Í_4Ÿn:žm9Ÿn<žm;œn;œn;Ÿl?Ÿl?l?l?šl>šl>šl>šl>˜l<˜l<šl>™k=›j=›j=›j=›j=›j=™h;›j=›j=™h;™h;™h9™h9™h9™h9™h9˜g8™h9™h9™h9˜g8šg8™f7™f9™f9—f9—f9–e8—f9”f:”f:”f:”f:”f8“e7”f8”f8“e7“e7“e7“e7“e7“e7’d6’d6’a4’a4’a4’a4b4b4b6b6a5a5a5a5a5Ž`4Ž`2Ž`2^1^1^1^1_1Œ^0Œ^0Œ^0Œ^0Œ^0Œ^0‹]/‹]/‹]/‹]/‹]/‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆZ.‡Y-‡Y-‡Y-‡Y-‡X/‡X/†W.†W.†W.…V-…V-…V-…V-…V-…V-…V-…V-„U,‚U,‚U,‚U,€S*€S,€S,€S.R-R-R-€S,€S,R+R+~Q,~Q,~Q,~Q,}P+}P+{Q+{Q+yP*zQ+zP,zP,zP,yO+yO+yO+xN*xN*yO+xN*xN,xN,xN,xN,vL*vL*uK'vL(vL(vL(uK'uK'tJ&tJ&sI%tJ&sI%sI%sI%sI%¥r@¥r@¥r@¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA§tB§tB§tB§tB§tB§tB§tB¨uC¨uA¨uA¨uA¨uA¨uA©vB©vB©vB©vB©vB«xD«xD«yB«yB«yB«yB«yB«yB«yB«yB«yB«yB«yB¬zC¬zC¬zC¬yE¬yE¬yE¬yE­zH­zH¬yE©vB§tB¦sA¤q? m;žk<›h9–e8”c6’a4_2‰[/ˆZ.‡Y-‡Y-†X,‚T(~Q*|O(|R*{Q)yP({R*}S+|R*|R,|R,|R*{Q)R+R+|R*~T,‚U.€S,€S.R-‚U0‚U0R-~Q,R-€S.€S,€S,€S,€S,€S,€S,€S,€S,‡X/‡X/†W.†W.†W.†W.†W0†W0†W0†W0†Y0‡Z1‡Z1‡Z1‡Z1‡Z1‡Z1‡Z1‡Z1‡Z1‡Z1‡Z1‰Z3‰Z3‰\3‰\3†Y0…X/ƒV-€S*R+|O(zP*tJ$sI'mC!l:k9p3o2|2ƒ9¤I£H%dVXY ^¢A -ô„û‹ç}ì‚÷‹÷‹ùŒÿ’ÿ—ÿ›ÿ¡ÿ¨ÿ¸ÿÎÿÚÿÒÿÐÿÐÿÌÿËÿÇð”ÿyÿˆÿ£ÿ¹ÿÏÿÍÿÍ -ÿÓÿÊû¦ìŒúšÿ»ÿË ÿÍ ÿÓÿÑÿÍÿÎÿÐÿÈ&ÿ°ò‚Î^ÉW×eÝbÌQÎOêkþƒ'ö{ÞeÔ[Øqû”ÿ´ÿÄÿÆÿÂÿ¼ÿ»ÿÆÿÊ -ÿ¹ô“ÉRš#_ nX1±yR´zHµ{I²{Hªs@›\$„E {3k#c_XZYXWVNM J H -DBG _${>‡J$@l+D/ c¸q?»¼Üâ þ%ÿ* ÿ$ÿ"ÿ#ÿ"ÿ!ÿ"ÿ)ÿ0 -ÿ9 ÿDÿLÿPÿOÿOÿBÿ5ÿ$ ÿÿ!ÿ-ÿ7ÿ:ÿ:ÿ=ÿ>ÿ>ÿAÿ>ÿ<ÿ=ÿ=ÿ;ÿ6ÿ6ÿ5ÿ2ÿ- ÿ* -ÿ1ÿ"ëáµ ª‡zmo|‹“™˜¯>ÿ¥;ÿº$ÿº$ÿ­1ÿ›ÿz!ÿ^ÿ8ÿ0ÿ3ÿ:ÿ;#ÿ:"ÿ;%ÿ;%ÿ<#ÿ:!ÿ7"ÿ8#ÿ4ÿ0ÿ.ÿ-ÿ- ÿ- ÿ0ÿ/ÿ1ÿ) ñ!ñ!ÿ3ÿ7ÿ5ÿ<ÿJÿSÿW$ÿS ÿM"ÿM"ÿE!ÿ?ÿ=ÿ<ÿ:ÿ:ÿ3ÿ, ÿ*ÿ+ßG$éQ.¡l9¢m:Ÿn<Ÿn<¡l;¡l; m> m>žm@l?˜l>˜l>˜l>˜l>˜l<˜l<šl>šl>›j=œk>žk>žk>žk<žk ÿ<ÿ6ÿ(ÿ ÿ"ÿ+ ÿ7ÿ=ÿ=ÿ=ÿ=ÿ>ÿ?ÿ?ÿ>ÿ<ÿ9ÿ6ÿ4ÿ5ÿ2ÿ* ÿ(ÿ(ÿ-ÿ' ü/ú- Ü4á9É<$¾1¨+ -¬/°+­(¨$«'¦'œÐ_ÿ´Jÿ§ÿ”ÿmò[ÿFÿGÿ;ÿ> -ÿ8ÿ9ÿ;#ÿ:"ÿ<&ÿ?)ÿH/ÿG.ÿ=(ÿ5 ÿ1ÿ1ÿ.ÿ+ ÿ(ÿ'ÿ#ÿú$ÿ) ÿ6 ÿDÿIÿOÿTÿ[ÿd,ÿf.ÿ[(ÿOÿBÿ@ÿ>ÿ<ÿ<ÿ:ÿ:ÿ6ÿ/ ÿ, ÿ)ÿ*ÛC åM* k8£n; o= o=¢m<¡l; m> m>žm@l?™m?™m?˜l>™m?˜l<˜l<›m?›m?œk>l?Ÿl?žk>žk<žk<žkc=c<c<a]>\=]>\>\>\>]?_@cD"cD"cD"cD"cD"cD"cD"cD"cD"cD"cD"eC"fD#kD'lE(nEnEjD!hBdFaCgAgAg3['j%w2Ž8¤NÌgâ}÷ÿ™ÿžÿ¡å‚œ9y{Ž!’%·7ôtÿ‘þ‹ùŽÿ—ÿÿœÿŸÿ£ÿ¨ÿ«ÿ°ÿ¶ÿÉ ÿÓÿÑÿÍ ÿÎ -ÿÌÿÓ ÿÆÿÿ—ÿ ÿ¥ÿÀÿÌ ÿÓ ÿÓ ÿÓÿÕÿ×ÿÔÿÏÿÓÿÐÿÏÿà ÿ¹ÿ¨ÿ›ÿ‘!û} ×\ÌQ»J¯>¶F·G°9£,±>Ôaþ’ÿ²ÿÈÿÉÿÇ ÿÉ ÿÊÿÇ ÿÁÿ¿ÿÂÿÆ ÿÆÿ¶ð|¹E{q t5©jE¶N´L¶€J·K²w@š_(†Dp.eZYZXT M IFFEEAB@D Yn3r5i,a VB 7(&..))1!z(Àn2¯«ÓØô#ù( ÿ( -ÿ+ ÿ,ÿ* ÿ+ ÿ* ÿ+ÿ- -ÿ+ÿ(ÿ(ÿ%ÿ%ÿ'ÿ)ÿ)ÿ&ÿ#ÿ$ÿ*ÿ2 ÿ8ÿ9ÿ:ÿ<ÿ=ÿ<"ÿ>$ÿ@'ÿG.ÿI,ÿ@#ÿ6ÿ4ÿ3ÿ-ÿ)ÿ'ÿ%ÿýüüÿ ÿ8 ÿGÿQÿPÿIÿ=ù:÷8ß.×&ÿQÿr'ÿWÿBÿ:ÿ;ÿ=ÿ?ÿ>ÿ>ÿ>ÿ>ÿ=ÿ=ÿ> ÿF(ÿS8ÿY>ÿI/ÿ=#ÿ:!ÿ7ÿ5ÿ.ÿ' ÿ$ÿýï åêÿ< -ÿZÿ\ÿ^"ÿ[ÿJÿCÿ8ÿ7 ÿ@ÿ=ÿ:ÿ:ÿ:ÿ9ÿ9ÿ7ÿ0ÿ+ ÿ) ÿ) ä?ïJ%¤j8ªp>£rE¢qD¥n@¤m?Ÿn<Ÿn< m;Ÿl: m; m;Ÿl=Ÿl=l?l?l=l=Ÿl=Ÿl=žk<Ÿl=žk<žk<žk<žk<žk<žkf=e;d:_6\3\6Z4Y2V/S.Q,O-L*I' J( N,O-P.Q/P1P1P1P1R0R0Q/R0P1P1P1P1S1S1S4T5S4T5S5S5S5T6V7X9X9X9X9X9X9X9X9X9X9X9\:\:a:b;d;d;^8[5V8Q3 O)K%a-q=–Q®iÐzåü— -ÿ›ÿ¢ÿ¨ÿ«ÿ¶ÿ®Òoš2‘)ˆ(„$–) 3º:ØXÿÿ’ÿ—ÿœÿ£ÿ¡ÿ¡ÿ§ÿ¬ÿ®ÿ¯ÿ´ÿÂÿÌÿÓÿÓÿÐ ÿÐ ÿÐ -ÿÓ ÿ¯ÿÿŠÿœÿ¾ÿÎÿÓ ÿÓ ÿÕÿÔÿÏÿÔÿ×ÿÕÿÉÿÁÿ¹ÿ³ÿ þ‰ékÛ]ÔYÁF©8¦5 0–&Ÿ(»DâoÿŽÿ© ÿ´ÿ ÿÆÿÉ ÿÉ ÿÊÿÉÿà -ÿÀÿÁÿÄÿÇÿ½ ÿ Ð\Š"ne&œ]8µ€Mµ€M¶€J¶€J¹~G©n7”R|:i#[YXVS K HEEEDBA?>H \!f)f)`YMD 3055/.5'€.Àn2° ­Ô×ñ ö% -ÿ&ÿ* ÿ* ÿ* ÿ) -ÿ) -ÿ, ÿ/ ÿ1 ÿ0 ÿ. -ÿ- ÿ,ÿ,ÿ*ÿ*ÿ)ÿ)ÿ(ÿ)ÿ,ÿ-ÿ0 -ÿ5ÿ:ÿ>ÿ=#ÿ=#ÿA(ÿL3ÿM0ÿF)ÿ8ÿ8ÿ5ÿ/ÿ-ÿ+ÿ'ÿÿöëî -ÿ%ÿ?ÿ? ÿ<ÿ: ÿ6 ÿA ÿA óBôC÷GñAý>ÿAÿ=ÿBÿ=ÿ,ÿ#ÿ,ÿ8ÿ=ÿ?ÿ@ÿ@"ÿ@"ÿA&ÿE*ÿ>$ÿ;!ÿ8ÿ7ÿ5ÿ-ÿ( -ÿ%ÿ#ÿ#ýúÿ+ÿEÿSÿPÿI ÿCÿ> ÿ> ÿ<ÿ>ÿ>ÿ=ÿ<ÿ<ÿ<ÿ:ÿ9ÿ6ÿ0ÿ, ÿ) ÿ*ä?ïJ%£i7©o=£rE oB¤m?¤m?Ÿn<Ÿn<¡n< m; m; m;Ÿl= m>l?l?l=l=Ÿl=Ÿl=žk<Ÿl=žk<žk<žk<žk<žk<žk<žk<žk<žkd;^:\8]6\5W2T/R0O-I+E' >" ?#;! 735779 -9 -8! 8! 8! 8! : : : 9 -8! 8! 8! 8! ;! ;! :# ;$ :# ;$ ;$ ;$ ;$ >'C)H.H.H.H.H.H.H.H.H.H.H.J.J.L-L-K/J.I-F*V'Io&›RÛ‚ú¡ÿ± ÿ¶ÿ½ÿºÿ¸ÿ²ÿ¶ÿ¹ÿ¸ÿ¼ÿÇÿ­Í]¬<¨+¤'¬3·>ÆRètÿ¨ÿ¥ÿ›ÿ ÿ¤ÿ¥ÿ§ÿ©ÿ¬ÿ® ÿ¯ÿ³ÿ¸ÿ¼ÿËÿÒÿÒÿÒÿÑ ÿÕÿË ÿºÿ½ÿËÿÕÿÎÿÓ -ÿÙÿ×ÿÔÿÑÿÑÿÁÿ¹ÿ¶#ÿ¹&ÿ¬Eÿ(ßYÉCÅGácÇW-,”3’1•4¸KØk÷†ÿ˜ -ÿ¨ÿ² ÿ¿ÿÄ ÿËÿÌ ÿËÿÌÿÇ ÿÂÿÁÿÄÿÈ ÿÆ ÿ¨âv¥7z b–K-²{M·€R³I´€J·K°zD d2„Hp-^[YT S L -HDDAB??=;=KZ`#gaXQ DADD@@H:†-Äk7« ¦ÍÑî ó% ÿ$ÿ) ÿ*ÿ*ÿ* ÿ* ÿ* ÿ* ÿ+ÿ,ÿ+ÿ*ÿ+ÿ.ÿ-ÿ+ ÿ+ ÿ, -ÿ*ÿ)ÿ+ÿ+ÿ,ÿ/ ÿ3ÿ:ÿ=ÿ=ÿ>$ÿ?%ÿ>"ÿ< ÿ;ÿ<ÿ8ÿ2ÿ0 ÿ- -ÿ*ÿ'ÿ%ÿ%ÿ!ÿ&ÿ1 ÿ9ÿ6ÿ2ÿ,ÿ'ÿ/ÿ5 ÿ= ÿFÿA ÿ= ÿFÿCÿBÿAÿ7ÿ'üÿÿ$ÿ+ÿ3 ÿ7ÿ9ÿ<ÿ<ÿ<ÿ<ÿ;ÿ8ÿ5ÿ2 -ÿ.ÿ.ÿ+ÿ0ÿ5 ÿ9ÿ@ÿIÿL"ÿ@ÿ:ÿ:ÿ<ÿ=ÿ@ÿ>ÿ=ÿ=ÿ=ÿ=ÿ<ÿ<ÿ:ÿ:ÿ8ÿ3ÿ0ÿ+ÿ+è>óI%¨j9®p?£rE oB¥n@¥n@Ÿn< o=¢o= m;Ÿl: m;Ÿl:Ÿl:Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl= m>Ÿl=Ÿl= m>Ÿl=žk<žk>žk>žk>žk>žk>žk>žk<žk<žkžk>j;j;j=j=›h;›h;—i;—i;—i9–h8–h8–h8•g7•g7•g7–h8˜g8—f7—f9–e8–e8–e8—d7—d7•d7•d7“e7“e7•d7•d7’d6’d6’d8’d8’d8b6’a6’a6”a4”a4“`3“`3‘`1‘`1_0_0_0_0_0_0^/^/^/^/^1Ž]0Ž]0Ž]0Œ^2‹]1‹]1‹]1‹]1‰[/‰[/‰[/‰[/‰[/ˆZ.ˆZ.ˆY0ˆY0‡X1‡X1‡X/‡X/‡X/‡X/‡X/†W.†W.†W.„W.ƒV-ƒV-ƒV-ƒV-‚U,…V/„U.†T.†T.†T.„R,„R,†T.„U.R+‚S,‚S,R+R+R-R-€Q*€Q*~Q*}P)}P)}P){Q){Q)zP(zP(zP(zP(zP(zP(yO'yO'yO)yO)yO)yO)xN(xN(xN(xN(xN(xN(xN(xN(uK%vL&§tB§tB§tB¨uC¨uC¨uC¨uC¨uC¨uC¨uC¨uC©vD©vD©vD©vD©vD©vD©vD©vB«xD«xD«xD«xD¬yE¬yE¬yE¬yE¬yE¬yE¬yE­zF­zF­{D­{D­{D­{D­{D­{D­{D­{D­{D­{D­{D­{D­{D­{D­{D®|E®|E®|E®|E®|E¬zC­{D­zF¥r>¢o;žk7›h9˜e6‘`3‹Z-…V-R)zP&xN$sI!nDg>c:[7Y5Y2W0R- M(L*I' @":6842/.----++++-.--++++-/-------15: : : : : : : : : : : < ;==< =!>" 6Gj;¼sãš.ÿ¬ÿ«ÿ¯ ÿ³ÿ²ÿºÿÃÿÉÿÅ ÿà ÿÄÿÃÿ ÿà ë{ºJ°3°3³:¾Eéuÿ­/ÿÏ%ÿ½ÿŸÿ£ÿ¨ÿªÿªÿ¬ ÿ¬ÿ® ÿ¯ÿ±ÿµÿ¸ÿÁÿËÿÕÿÕÿØÿÔÿÎÿÖÿÝ"ÿÛ ÿÕÿÑÿÕ ÿØÿÔÿÍÿÄÿ½ÿ´ÿ­ÿ¦ô„ÈK¬/½7ÜVòtý Ül¡1|Œ+•4¥DÉ\átø‡ÿ•ÿ¥ÿ¯ÿ½ÿÄ ÿËÿÌ ÿÌÿÍÿÇ ÿÂÿÂÿÃÿÉÿËÿ¶õ‰³E\‡<ªsE·€R³IµK¸‚LµI¬p>”X&z7 b\ZVS N K EDA@>><;9AP[i f[RF -BFK J -GP AŠ1Âi5ª¥ËÑíò$ÿ$ÿ* ÿ*ÿ*ÿ* ÿ* ÿ* ÿ* ÿ*ÿ*ÿ)ÿ*ÿ) ÿ* ÿ* ÿ* ÿ+ ÿ+ ÿ+ ÿ+ ÿ+ÿ+ÿ+ÿ. -ÿ/ ÿ2ÿ4ÿ5ÿ6ÿ7ÿ6ÿ6ÿ6ÿ7ÿ8ÿ8ÿ8ÿ6ÿ3ÿ4ÿ6ÿ9ÿ<ÿ:ÿ7ÿ3 ÿ7ÿ6ÿ.ÿ)ÿ,ÿ0ÿ: ÿ@ÿA ÿHÿHÿHÿHÿHÿ?ÿ8ÿ8ÿ6ÿ1 -ÿ0 ÿ.ÿ.ÿ/ -ÿ2 ÿ2ÿ4ÿ4ÿ4ÿ4ÿ4ÿ5 ÿ9ÿ<ÿ=ÿBÿCÿHÿDÿ>ÿ?ÿ?ÿ>ÿ>ÿ?ÿ>ÿ?ÿ>ÿ>ÿ<ÿ<ÿ<ÿ<ÿ<ÿ9ÿ:ÿ8ÿ3ÿ1ÿ-ÿ-è>ñG#ªl;°rA¥tG£rE§pB¥n@ o= o=¢o=¢o=¡n<¡n<¡n< m; m> m>Ÿl=Ÿl=Ÿl= m> m> m> m> m>Ÿl=Ÿl=Ÿl?žk>Ÿl?Ÿl?žk>žk>žk<žk<Ÿl=žk<žk<žk<žk>žk>žkj=j=j=™k=—i;—i9—i9—i9—i9—i9—i9–h8–h8˜g8˜g8˜g:˜g:—f9—f9™f9™f9–e8•d7“e7“e7•d7•d7“e7’d6’d8’d8’d8b6’a6”c8–c6”a4”a4”a4’a2‘`1_0‘`1‘`1_0_0_0_0_0_0_0^1^1^1^1Œ^2Œ^2Œ^2‹]1‹]1‹]1‰[/‰[/‰[/‰[/‰[/ˆZ.ˆY0ˆY0ˆY2ˆY2‡X/‡X/‡X/‡X/‡X/‡X/†W.†W.„W.„W.ƒV-ƒV-ƒV-ƒV-…V/…V/‡U/†T.†T.†T.†T.†T.„U.„U.‚S,‚S,R+‚S,R-R-R+R+~Q*~Q*~Q*}P){Q){Q){Q){Q){Q)zP(zP(zP(zP(zP(zP*yO)yO)yO)yO)yO)yO)yO)xN(xN(xN(xN(xN(vL&¨uC¨uC¨uC¨uC¨uC¨uC¨uC¨uC¨uC¨uC©vD©vD©vD©vD©vD©vD©vD«xF«xD«xD«xD¬yE¬yE¬yE¬yE¬yE­zF­zF­zF­zF­zF­zF­{D­{D­{D­{D­{D­{D­{D­{D­{D­{D®|E®|E®|E®|E®|E®|E®|E¯}F¯}F¯}F­{D­{D¬zC§u>¦s?¢o;Ÿl:šg5”c4Ž].‡Y-„V*|R(yO%sJ"mDgAb<Y5U1R0O- H) E& B$ @"72%))$! !        -          - - -     #''''''''''&&&'')<;ŠBΆÿ ÿ¢ýœú™ÿ¢ÿ©ÿ¥ÿªÿ¶ÿÄ ÿÍ -ÿÎ ÿÎÿÈ -ÿÇÿÑÿ¥Ä\À:Ä>ÄDÉIårÿ¹+ÿÝÿÔÿ±ý©ÿ®ÿ«ÿ¬ÿ¯ ÿ® ÿ¬ÿ­ÿ°ÿ²ÿ¶ÿ¶ÿ»ÿÉ ÿÑÿÙÿÒÿÑÿÓÿÐÿÔÿÑÿÏÿÒÿËÿÁ ÿÀÿ¼(ÿ¶"ÿ–%ãkºB (£0ÂOÝqò†ø–ÿ¦ÿ¸÷–œ=„%›8ªGÈ]ßtü†ÿ”ÿ¢ÿ­ÿ»ÿÆ ÿË ÿË ÿÍÿÍÿÉÿÄ ÿÂÿÃÿÉ ÿÌ ÿ¼ûœÊU”h}$œb7¸~S³~Mµ€O¶ƒO´M¯{Ežj4ˆJj,_XVT N K -EDA@==;8=CQZh h c]N IV bvwv_ :Ëe&ª¥ÉÏ íò$ÿ$ ÿ)ÿ+ ÿ+ ÿ* ÿ* ÿ* ÿ* ÿ*ÿ)ÿ*ÿ+ÿ+ ÿ+ ÿ+ ÿ+ ÿ, ÿ, ÿ+ ÿ+ ÿ* -ÿ* -ÿ+ÿ- -ÿ+ ÿ*ÿ)ÿ+ ÿ. ÿ/ ÿ/ÿ0ÿ2ÿ4ÿ7ÿ:ÿ@ÿBÿCÿCÿAÿAÿ?ÿ>ÿ@ÿ>ÿ=ÿ@ÿ;ÿ0ÿ+ÿ0 ÿ)ÿ*ÿ4ÿ<ÿ?ÿBÿ@ÿ<ÿ7ÿ2ÿ/ÿ1ÿ9ÿ9ÿ;ÿ;ÿ;ÿ7ÿ4ÿ4ÿ4ÿ5ÿ4ÿ7ÿ< ÿAÿAÿEÿEÿEÿBÿ@ÿ>ÿ?ÿ>ÿ=ÿ>ÿ<ÿ=ÿ=ÿ=ÿ=ÿ;ÿ:ÿ<ÿ<ÿ<ÿ9ÿ9ÿ7ÿ3ÿ1ÿ.ÿ/ä?ïJ%©k:°rA¦uF¤sD§p?¨q@¥r@¢o=¢o=¢o=¢o=¢o=¢o=¢o=¡n?¡n? m>¡n?¡n? m> m>¡n? m> m> m>Ÿl= m@Ÿl?Ÿl?Ÿl? m@Ÿl?žk>Ÿl?Ÿl=Ÿl=žk<žk<žk>žk>žk>žk>žk>žk>j;j;›j;™h9›j;›j;™h9™h9™h9™h9™h9™h9™h9™h9˜g8˜g8˜g:˜g:™f9™f9™f9˜e8–e8—f9—f9–e8•d7•d7”f8“e7”c8”c8•d7”c6–c4–c4–c4”a2’a2’a2’a2‘`1‘`1‘`1‘`1‘`1_0_0_0_0_2^1^1^1_3Œ^2Œ^2Œ^2‹]1‹]1‹]1‹]1‹]1‰[/‹]1ˆZ.ˆZ.ˆZ.ˆY0ˆY0ˆY0ˆY0ˆY0ˆY0‡X/‡X/‡X/‡X/„W.„W.„W.ƒV-…V-…V-…V/…V/…V/…V/…V/„U.„U.„U.„U.„U.„U.‚S,‚S,‚S,‚S,R+R+R+R)~Q(~Q*~Q*~Q*}P)}P)}P){Q){Q){Q)zP(zP(zP(zP*zP*zP*yO)yO)yO)yO)yO)yO)yO)yO)yO)xN(vL&¨uC¨uC¨uC¨uC¨uC¨uC©vD©vD©vD©vD©vD©vD©vD©vD©vD©vD©vD«xF«xD«xD¬yE¬yE¬yE¬yE¬yE­zF­zF­zF­zF­zF­zF­zF­{D­{D®|E®|E®|E®|E®|E®|E®|E®|E®|E®|E®|E®|E®|E¯}F¯}F¯}F¯}F¯}F®|E®|E­{D¬zC«xD§t@¤q?Ÿl:˜g8”c4Œ^2ˆZ.€V,{Q'wN&qH jDd>^:W3R0P. J+G( B$ ?!72%%!     3j? ΆçŸ.ù™üœÿ£ÿ©ÿ¯ ÿ¨ÿÿ©ÿ¨ÿ¯ÿÃÿÏ ÿÕÿÒÿÊÿÑÿ¿ëƒÑKÍGËKÐPÖcÿ•ÿÓÿÜÿÉÿ­ÿ¬ÿ°ÿ¯ ÿ­ÿ¯ -ÿ° ÿ°ÿ°ÿ²ÿ±ÿ³ÿ´ÿ»ÿÃÿÎ ÿÓÿ×ÿÚÿÜÿÝÿØÿÑÿÆÿÄÿÅ ÿÅ ÿ¢æzÌT¹AºBÍUévÿŽÿ¦ÿ¶ÿÁÿÇÿÌ&ÿÍ'õ–&§H•2©FÉ^Üqû…ÿ”ÿ¡ÿ«ÿºÿÄ ÿË ÿË ÿÎÿÎÿËÿÆ ÿÀÿÀÿÇÿË ÿ ÿªÜgž)f p‘W,·}R´N¶P¶ƒO¶ƒO¶‚L©u?˜Z+z< h"ZT VN K -FFB@==;<CJTZh h gcUN ^n!‹052w¬FÄ^§¦ÈÎíò$ÿ#ÿ' ÿ* ÿ* ÿ) ÿ* ÿ* ÿ* ÿ*ÿ+ÿ+ÿ+ÿ+ ÿ+ ÿ+ ÿ+ ÿ, ÿ, ÿ+ ÿ* -ÿ* -ÿ+ ÿ, ÿ+ÿ)ÿ)ÿ)ÿ)ÿ)ÿ*ÿ) -ÿ) -ÿ* ÿ+ ÿ- -ÿ1ÿ8ÿ>ÿ@ÿ@ÿ@ÿ@ÿ?ÿ?ÿBÿEÿJÿBÿ8ÿ3ÿ0 ÿ/ÿ(ÿ+ÿ0 ÿ6ÿ=ÿ>ÿ9ÿ5ÿ-ÿ' -ÿ%ÿ#ÿ/ÿ8ÿA ÿB ÿBÿ?ÿ<ÿ:ÿ8ÿ9ÿ9 ÿ< ÿAÿAÿ?ÿ@ÿ@ÿ@ÿ?ÿ>ÿ?ÿ@ÿ>ÿ=ÿ=ÿ<ÿ=ÿ=ÿ=ÿ=ÿ:ÿ:ÿ<ÿ:ÿ9ÿ8ÿ9ÿ7ÿ3ÿ1ÿ/ÿ0æAñL'©k:®p?¦uF¥tE§p?§p?¥r@¥r@¤q?¢o=¢o=¤q?¤q?¤q?¢o@¢o@¡n?¢o@¢o@¡n?¡n?¡n?¡n?¢o@¡n? m> m@ m@Ÿl?Ÿl? m@Ÿl?Ÿl?Ÿl?Ÿl= m>Ÿl=Ÿl=Ÿl?Ÿl?žk>Ÿl?žk>žk>žkånÃL¿HÇXÚkç„ÿ¡ÿÂÿÐÿÖÿÖÿÓÿÎÿÆÿÊ ÿÐõš¯J¥@ÈZÞp÷€ÿÿŸÿ¨ÿ¸ÿÀ -ÿÇÿÊÿÏÿÏÿËÿÉÿÂÿÁÿÈÿËÿÊÿ¼÷€¸Awp „FµwO·‚Oµ€M´ƒOµ„P·ƒM±}G f4‚Hn.[WWVR M K -H -E@@BDM SVXdfec`[v Œ6´NÂ\ÁT¨;ÃWÇ[§©ÉÎíò$ÿ" ÿ& ÿ' ÿ' ÿ' ÿ' ÿ* ÿ* ÿ)ÿ+ÿ, ÿ, ÿ+ ÿ+ ÿ+ ÿ+ ÿ+ ÿ+ ÿ* ÿ* ÿ+ ÿ* ÿ+ ÿ* ÿ) ÿ( -ÿ)ÿ)ÿ*ÿ+ÿ)ÿ)ÿ+ÿ+ÿ*ÿ*ÿ+ ÿ/ÿ0ÿ2ÿ6ÿ8ÿ8ÿ8ÿ<ÿ>ÿ=ÿ/ý#ÿ(ÿ) ÿ+ ÿ* ÿ- ÿ-ÿ4ÿ:ÿ:ÿ:ÿ:ÿ1ÿ' ÿ!ÿÿ!ÿ/ -ÿ=ÿB ÿCÿDÿBÿAÿ= ÿ= ÿ>ÿ?ÿAÿAÿ?ÿ?ÿ@ÿ@ÿAÿ@ÿ@ÿ@ÿ?ÿ>ÿ>ÿ>ÿ=ÿ<ÿ=ÿ=ÿ9ÿ8ÿ:ÿ:ÿ9ÿ7ÿ8ÿ7ÿ5ÿ3ÿ1ÿ2áBîO+§m;¬r@§tB§tB«qDªpC¥r@¥r@¥r@¥r@¥r@¤q?¤qB¤qB¤qB¤qB¤qB¢o@¢o=¢o=¤q?¡n<¢o=¢o=¡n? m>¢o@¡n? m@ m@¡nAŸl?Ÿl? m@ m> m>Ÿl=Ÿl=¡nA m@Ÿl?Ÿl?Ÿl=žk<žka;[6S.N,J( B" -;74.+%#      –XØš=õ•û›öžÎvk.:5‰Fû¢þ¥ÿ¤ÿ± ÿ²ÿ®ÿ¾ÿÎÿØÿÔÿÔÿÔÿÑ[ÖQÛVÜ[âaþžÿÌÿÜ#ÿÚ!ÿ ÿ°ÿ¬ÿ´ÿ² ÿ¯ÿ²ÿ±ÿ¯ÿ¯ÿ­ÿ­ÿ¬ÿ®ÿ¯ÿ©ÿ™ôŠé~÷Œÿÿù…ÄP§0¯8ÂKÖ_åvÿ‘ ÿ«ÿ½ÿÐÿÚ&ÿÖÿÎÿËÿÊ ÿÇ ÿÃÿÊÿÌóŽ­HµGÞpøÿŒÿ›ÿ¦ÿ´ÿ¼ÿÇÿËÿÏÿÏÿÌÿÊÿÄÿÂÿÇÿÉÿË ÿÅÿ•Ò[Š'n t6¬nF¸ƒPµ€M³‚N·†R¸„N·ƒM®tB’X&|<e%[XWWPM J I DCFI PVWV`abccc~(™C -ÊdÚt -âu ÒeÒf ÃW« ªÉÍíò$ÿ" ÿ$ ÿ& ÿ& ÿ' ÿ& ÿ'ÿ) -ÿ( ÿ)ÿ* ÿ, ÿ* ÿ+ ÿ+ ÿ+ ÿ+ ÿ+ ÿ* ÿ* ÿ* ÿ) ÿ* ÿ* ÿ) ÿ( -ÿ( ÿ)ÿ+ÿ,ÿ*ÿ*ÿ+ÿ+ÿ+ÿ*ÿ* ÿ* ÿ,ÿ.ÿ/ÿ0ÿ0ÿ1ÿ5ÿ4ÿ6ÿ3 ÿ+ ÿ+ ÿ) ÿ(ÿ'ÿ'ÿ-ÿ3ÿ6ÿ5ÿ6ÿ4ÿ/ÿ' ÿ!ÿÿÿ(ÿ6ÿ@ÿBÿBÿBÿBÿ> -ÿ> -ÿ>ÿ?ÿ@ÿAÿ@ÿ@ÿ@ÿAÿBÿAÿ@ÿ@ÿ>ÿ>ÿ?ÿ?ÿ=ÿ<ÿ=ÿ<ÿ9ÿ8ÿ:ÿ:ÿ8ÿ8ÿ8ÿ7ÿ4ÿ3ÿ1ÿ1ãD ïP,«q?­sA§tB¦sA«qD¬rE¦sA¥r@¥r@¥r@¥r@¥r@¥rC¥rC¤qB¥rC¤qB¤qB¤q?¤q?¤q?¤q?¢o=¤q?¢o@¢o@¢o@¢o@¡nA¡nA¡nA¡nA m@ m@ m> m> m> m> m@ m@ m@Ÿl?Ÿl=Ÿl=žk<žk<›j;›j;œk<œk<›j;œk<œk<›j;›j;›j;›j;›j;›j;›j;›j;™h9›h9›h9›h9›h9˜g8˜g8—f9—f9—f9—f9–h:•g9—f9—f9—f9—f9˜e6˜e6—d7˜e8–e8•d7•d5•d5”c6”c6”c6”c6”c4’a2’a2’a2’a4’a4‘`3‘`3_2_2^1^1Ž]2\1\1\1\1\1‹]1‹]1‹]1‹]1‹\3‹\3‹]1‹]1‰[/‰[/‰Z1ˆY0ˆY0‡X/…X/†Y0‡X/ˆY0‡X/‡X/‰W/‰W/‰W1ˆV0†W0†W0†W0†W0…V/…V/…V/…V/„U.„U.„U.„U.„U.„U.R+R+R+R+~Q*R+~Q,~Q,~Q,}P+|R,|R,{Q+{Q+{Q+{Q+{Q+{Q+zP*zP*zP*zP*zP*zP*zP*zP*yO)xN(©vB©vB©vB©vB©vB©vB©vB«xD«xD«xD«xD«xD«xD«xD«yB«yB¬yE¬yE¬zC¬zC­{D­{D­{D­{D­{D­{D®|E®|E®|E®|E®|E®|E®|E®|E¯{E¯{E¯{E¯{E¯{E°|F°|F°|F°|F°|F°|F±}G±}G±}G°~G°~G³I³I±}G±}G±}G°|F±}G±}G°{H®yFªuD¦q@ m>›h9˜g8”c4a3Œ^0ƒW+~R&xN&tJ"oElBg>b9Z3Y2T/Q,H&C!<:761-++*' % % $ -$ -$ -"""""""$ -""""" #!)S%Ñƒêœ -ûžúÂr#] 8 k4è¶æ¦ZÊŠ>ò§B÷¬Gÿµÿ¾ÿÙÿÛÿ×ÿáÿ»Û{ßVáXÚXÔRïrÿ«/ÿÎ)ÿÏ*ÿÒ$ÿÁÿ±ÿ²ÿ°ÿ°ÿ±ÿ²ÿ²ÿ²ÿ°ÿ¨ÿ§ÿªÿŸ ÿó~ó~ÿŽ1ÿŒ/äl¾F›"”¢7¿TÒzí•þ±ÿÃÿÈ ÿÆ ÿÇÿÎÿÔÿÏÿÌ -ÿË ÿËÿÊÿÂÿË ÿÆ剶MÐgñzÿ ÿ™ÿ¢ÿ²ÿºÿÆÿÌ ÿÐÿÎÿËÿÊÿÇ ÿÆÿÈÿÈÿÈ ÿÇ ÿ§âv¦9ƒy$¯Z:»X¾„[¸‡Xµ„U·„U¼‰Zº€P§m=‹Mp2^TXT Q P P O M L -M PUY[[[[\bmq…+CÊhä‚óŠò‰ïyÄN§¦ÄÆë ï$ -þ%ÿ'þ% ÿ(ÿ' ÿ' ÿ% ÿ( ÿ)ÿ)ÿ)ÿ*ÿ* ÿ* ÿ+ ÿ+ ÿ*ÿ*ÿ)ÿ)ÿ)ÿ)ÿ*ÿ*ÿ*ÿ*ÿ* ÿ* ÿ+ ÿ+ ÿ+ ÿ+ ÿ+ ÿ+ ÿ+ -ÿ+ -ÿ, ÿ, ÿ, ÿ/ÿ/ÿ.ÿ/ ÿ0 ÿ1ÿ0ÿ1ÿ1ÿ2ÿ, ÿ+ ÿ.ÿ-ÿ,ÿ.ÿ3ÿ5ÿ0ÿ) ÿ'ÿ) ÿ-ÿ,ÿ&ÿ+ÿ1 ÿ5ÿ8ÿ>ÿ?ÿ>ÿ=ÿ=ÿ?ÿ?ÿ?ÿ?ÿ@ÿ?ÿ?ÿ?ÿ?ÿ?ÿ?ÿ?ÿ>ÿ?ÿ?ÿ?ÿ>ÿ>ÿ>ÿ=ÿ<ÿ8ÿ8ÿ:ÿ:ÿ8ÿ6ÿ7ÿ6ÿ3ÿ2ÿ1ÿ1ßCíQ(«r9­t;¢vA¡u@§t@§t@¦sA¥r@¥r>¦s?¦s?¦s?¦q@¦q@¥rC¥rC¥r@¥r@¥r@¥r@¥r>¤q=¤q=¤q=¤q?¢o=¤q?¢o=¢o@¡n?¡n?¡n?¡n? m> m; m; m; m; m; m; m>Ÿl=Ÿl:Ÿl:žk9žk9žk<žk<žk<žk<œk<œk<œk<œk<œk9œk9›j8›j8›j8›j8›j8›j8j;›h9j;›h9™h9™h9™h9™h9˜g8˜g8˜g:˜g:˜g:˜g:—f7—f7—f9–e8–e6–e6–e6–e6•d7•d7•d5•d5”c4”c4”c6”c6”c6’a4’a2’a2’a2’a2‘`3‘`3‘`3_2_4_4^3^3^3^3Ž]2^3Ž]2\1Œ^2Œ^2Œ^2‹]1‹]1‹]1‰[/‰[/ˆZ.ˆZ.ˆY0ˆY0ˆY0ˆY0ˆY0‡X/‡X/‡X/‡X/‡X/‡X1†W0†W0†W0…V/…V/…V-…V-…V-…V-‚U.‚U.‚U.‚U.€S*€S*€S*R)R+R+R+R+~Q*~Q*|R*|R*|R*{Q){Q+{Q+{Q+{Q+{Q+{Q+{Q+{Q+{Q+zP*zP*zP*zP*xN(«xD«xD«xD«xD«xD«xD«xD«xD«xD«xD«xD«xD¬yE¬yE¬zC¬zC¬yE¬yE¬zC¬zC­{D­{D­{D­{D­{D­{D®|E®|E®|E®|E®|E®|E¯}F¯}F°|F¯{E°|F°|F°|F°|F±}G±}G±}G±}G±}G±}G±}G±}G²€I²€I³I´€J´€J´€J´€J³I³I³I³~K¯zG®yH¬wF§tE¢o@l=™h9”f8a3Š^2†Z.€V.zP(xN(tJ$pG%jAe>!c<^9[6R0N,F' E& A" @! =! -8666530///.............,+&&&&''''''''''-' 1}O#íŸ ï¡ÿ¤ÿ¸ºjH(hKF#4vD w7J -‰>ï¤?ÿ­ÿ´ÿÓÿàÿ×ÿÝÿÎ!ýé`àWÙWØV×Zöyþ–ý•ÿÿ½ÿÅÿ±ÿ°ÿ¬ÿ°ÿ±ÿ¯ÿ®ÿ«ÿªÿ¨ÿŸþ‰ð{ó~ìwÍV°9¥-§/°7ÈOÙnÿ—(ÿ¿$ÿË0ÿÕ%ÿÕ%ÿÏÿÊ ÿÇÿÌ ÿÓÿÔÿÓÿÎ ÿÊÿÌ ÿÇÿÂÿÊÿÄë‚Ç^èqÿˆÿ•ÿ¡ÿ±ÿ¹ÿÅÿÌ ÿÐÿÍÿËÿËÿÇ ÿÅÿÅÿÈÿÉÿÇ ÿ¯ õ‰¼O#rŸJ*¸~UÀ†]¹ˆY·†W¹†W»ˆY»Q°vFœ^-ƒEm']YVRRRRRTTWY[[WXV [`lqˆ.œBÄbãÿ–ÿšåo®8£ª"ÊÌ!êð% þ%ÿ'ÿ& þ% ÿ$ -ÿ$ -ÿ% ÿ( ÿ)ÿ)ÿ( ÿ*ÿ* ÿ+ ÿ+ ÿ* ÿ)ÿ)ÿ)ÿ)ÿ)ÿ)ÿ*ÿ*ÿ+ÿ*ÿ* ÿ+ ÿ* ÿ+ ÿ+ ÿ+ ÿ+ ÿ+ ÿ+ -ÿ+ -ÿ, ÿ, ÿ+ ÿ, ÿ.ÿ.ÿ/ ÿ0 ÿ1ÿ0ÿ0ÿ0ÿ1ÿ/ÿ.ÿ1ÿ3ÿ1ÿ3ÿ6ÿ7ÿ4ÿ5ÿ5ÿ:ÿ=ÿ<ÿ8ÿ:ÿ<ÿ<ÿ=ÿ?ÿ?ÿ?ÿ@ÿAÿCÿAÿ@ÿ?ÿ@ÿ?ÿ?ÿ?ÿ?ÿ?ÿ>ÿ?ÿ>ÿ?ÿ?ÿ=ÿ>ÿ>ÿ>ÿ>ÿ<ÿ9ÿ9ÿ¥r>¤q=¤q=¤q?¤q?¤q?¤q?¢o@¢o@¢o@¢o@¡n?¡n?¡n<¡n<¡n< m; m; m; m> m>Ÿl:Ÿl:Ÿl:Ÿl:Ÿl=Ÿl=žk<žk<œk<œk<œk<œk<œk9œk9œk9œk9›j8›j8›j8›j8j;j;j;j;™h9™h9™h9™h9™h9˜g8˜g:˜g:˜g:˜g:—f7—f7˜g:˜g:—f7—f7—f7–e6•d7–e8•d5•d5•d5•d5•d7”c6”c6”c6”c4”c4”c4’a2’a4’a4’a4‘`3‘`5_4_4^3_4_4^3^3^3Ž]2_3Œ^2Œ^2Œ^2‹]1‹]1‹]1‹]1‹]1‰[/‰Z1‰Z1‰Z1‰Z1ˆY0‡X/ˆY0ˆY0‡X/‡X/‡X1‡X1†W0†W0†W0†W0†W.…V-…V-…V-ƒV/‚U.ƒV/ƒV/€S*€S*€S*€S*€S,R+R+R+R+R+|R*|R*|R*|R*|R,|R,{Q+{Q+{Q+{Q+{Q+{Q+{Q+zP*zP*zP*zP*zP*«xD«xD«xD«xD«xD«xD«xD«xD¬yE¬yE¬yE¬yE¬yE¬yE¬yE¬yE­zF­zF­{D­{D¬zC®|E®|E®|E®|E®|E®|E¯}F¯}F¯}F¯}F¯}F¯}F¯}F°|F°|F±}G±}G±}G±}G±}G±}G±}G±}G³I³I³I´€J²€I²€I´€J´€J´€J´€J´€J´€J³I´€J´L³~K³~K¯zG­xG¬wF§tB¤q?l=™h9˜g:–e8_4\1‹Y1ˆV.R+}N'wJ%tG"nD mCj@d:_6^5[5W1S.R- P+ P+ P+ M(L'K&J%I$E#C!C!C!C!C!C!C!C!C!C!C!C!D"D"C!:;;<<<<<<<<<<<@:K£k&øò—ÿ® ÿÖ5·jJ -4$  - pB$ÿ©!ÿª"ÿºÿÑÿß#ÿ×ÿ×ÿÈï‡È`ßUáWÔJÚPè_é`åaÿƒ ÿ¾(ÿÅ/ÿ¹ÿ³ÿ¸ ÿ±ÿ°ÿ±ÿ°ÿ°ÿŸ(÷„ çuÓa¿L«8§1±;ºF½IÌ^õ‡ÿµÿÊÿÝÿÜÿØÿÖÿÒÿÏÿÌ ÿÊ ÿÊ -ÿÑÿÕÿÓÿÌÿÌÿÌÿÈÿÇÿÏ ÿ½ æˆÞqîÿ•ÿ ÿ¬ÿ¶ÿÃÿÊ ÿÎÿÍÿÍÿÌÿÇ ÿÅ ÿÃÿÄÿË ÿÊ ÿ¼ÿ¦áw­Cƒ' D·xBÄ…Oº…T»†U·„R¶ƒQ¼…R¶L§m;‘W%|< d$YVWWUUVVWWZ[XUT S V ]kq‰,›>ºZ×wø’ÿ¥&ãs©9•!” ±¼#Þ"å)ü,ý-ÿ(ÿ(ÿ% ÿ# ÿ'ÿ(ÿ)ÿ*ÿ' -ÿ) ÿ)ÿ)ÿ*ÿ)ÿ)ÿ)ÿ)ÿ)ÿ(ÿ(ÿ*ÿ+ÿ,ÿ+ÿ*ÿ*ÿ* ÿ+ ÿ* -ÿ* -ÿ* -ÿ* -ÿ* -ÿ* -ÿ+ ÿ+ ÿ+ ÿ.ÿ.ÿ.ÿ.ÿ.ÿ.ÿ/ÿ/ÿ/ÿ.ÿ.ÿ/ÿ1ÿ5ÿ8ÿ=ÿ>ÿ?ÿ@ÿ@ÿ@ÿG%ÿK)ÿG%ÿE#ÿC$ÿB#ÿD!ÿBÿ>ÿ>ÿ?ÿCÿC!ÿC!ÿAÿ@ÿ>ÿ?ÿ?ÿ?ÿ@ÿ@ÿ?ÿ=ÿ>ÿ=ÿ>ÿ>ÿ=ÿ>ÿ>ÿ?ÿ>ÿ<ÿ>ÿ@ÿ@ÿ;ÿ5ÿ0ÿ3ÿ3ÿ4ÿ3ÿ0ÿ2ÝHëV-¬v>¬v>¢vC¢vC¨uF§tE¨sB¨sB¨s@§r?§r?§r?§rA§rA§rA§rA¦sA¥r@¥r@¥r@¥r>¥r>¥r>¤q=¤q?¤q?¤q?¤q?¤q?¢o=¢o=¢o=¢o=¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<Ÿl:Ÿl:Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=žk<žkb=a<_:`;^9\7\7Z5V4U3U3U3U3U3U3U3U3U3U3U3U3V4V4U3M.M.N/N/N/N/N/N/N/N/N/N/N/N/R0L* `(»ƒ>øùžÿ¬ ÿ³¨[ag9‹]?Óyÿ©!ÿ´ÿÄÿæ*ÿÙÿÒ ÿÜÿ²àxàVáWÚPÙOßVæ]ã_ä`ÿ”ÿË5ÿ¸ ÿ¥ÿºÿÀÿ¹ÿ¹ÿ¹ÿ êwÁN°>²@©6¯<¿IÇQÊVÜhþÿ¬ÿÇÿÒ"ÿÕÿÔÿÖÿÕÿÓÿÓÿÏÿÊ ÿÆÿÊ -ÿÓÿÖÿÒÿÍÿÊÿËÿÊÿÇÿÊÿ¹ø‹ë~ÿ’ÿœÿ«ÿ³ÿ¿ÿÇ -ÿÎÿÎÿÎÿÎÿÊÿÆ -ÿÃÿÄÿÉ -ÿË ÿà ÿ¹ÿ›Öl›?ŸC¯p:Á‚L»†U»†U·„R·„R¾‡TºƒP°vDžd2ŠJr2`WYZVVVVVVXXURR S U -\ipˆ+–9²RÍmì†ÿ£$ð€ÈX¸Dª6¶´ÓÙîô$ÿ ÿ# ÿ# ÿ% ÿ'ÿ(ÿ*ÿ+ÿ( ÿ) ÿ)ÿ)ÿ)ÿ)ÿ)ÿ)ÿ)ÿ)ÿ(ÿ(ÿ*ÿ*ÿ+ÿ*ÿ*ÿ*ÿ* ÿ* ÿ* -ÿ* -ÿ* -ÿ* -ÿ* -ÿ* -ÿ* -ÿ+ ÿ, ÿ, ÿ,ÿ,ÿ.ÿ.ÿ.ÿ.ÿ,ÿ,ÿ-ÿ.ÿ.ÿ0ÿ5ÿ9ÿ=ÿAÿE!ÿD ÿC"ÿD#ÿF$ÿG%ÿF$ÿD"ÿC$ÿC$ÿD!ÿBÿ?ÿ?ÿ@ÿCÿC!ÿAÿAÿ@ÿ?ÿ?ÿ?ÿ?ÿ@ÿ@ÿ?ÿ>ÿ=ÿ=ÿ>ÿ>ÿ=ÿ<ÿ<ÿ<ÿ<ÿ<ÿ@ÿ>ÿ2 ÿ)ÿ"ÿ* ÿ2ÿ3ÿ4ÿ2ÿ1ÿ2ÝHìW.¬v>¬v>¢vC£wD¨uF¨uF¨sB¨sB¨s@¨s@§r?§r?¨sB¨sB§rA¨sB¦sA¦sA¥r@¥r@¥r>¥r>¥r>¥r>¤q?¥r@¤q?¤q?¤q?¢o=¢o=¢o=¢o=¢o=¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n< m; m> m> m> m> m>Ÿl=Ÿl=Ÿl=žk<Ÿl=Ÿl=žk<žk9žk9žk9žk9žk<žk<žk9žk9žk9j8žk9j8j;j;j;›h9›h9›h9™h9™h9™h9™h9˜g:™h;˜g8˜g8˜g8—f7—f9—f9–e6–e6–e6–e6•d7•d7•d7•d7•d5•d5”c6”c6”c4”c4’a2’a2’a6’a6b4a3‘`3‘`3_4_4_4^3_3Ž`4Ž`4Œ^2‹]1Œ^2Œ^2Œ^2‹]1‹]1‹]1‹]1‰[/‰[/‰Z1‰Z1‰Z1‰Z1ˆY0‰Z1ˆY0ˆY0†Y0†Y0…X/…X/…X/„W.…X/„W.ƒV-ƒV-ƒV/ƒV/ƒV-ƒV-ƒV-‚U,‚U,‚U,€S,€S,€S*€S*R)R)R)R)R+R+~Q,~Q,~Q,~Q,~Q,}P+}P+}P+}P+}P+}P+}P+¬yE¬yE¬yE¬yE¬yE¬yE¬yE¬yE¬yE¬yE¬yE¬yE­zF­zF­zF­zF­zF­zF­{D®|E®|E®|E®|E®|E¯}F¯}F¯}F¯}F¯}F¯}F¯}F¯}F°~G°~G±}G°|F±}G±}G±}G±}G³I³I³I³I³I´€J´€J´€J³J³JµKµK´€JµKµKµK¶‚L¶‚Lµ€M³~K³~K±|I±|I±|I²K°}I®{G¬yE©vD¨uC¦sD m>šg8–c4”c4’a2a3Œ^0\1\1\1‰X-…V-‚S*~Q(|O&xN$uK!tJ"sI!rH sI!uH!tG qDqDnD lBlBlBlBlBlBlBlBlBlBlBlBmCmClBf=g>g>g>i@i@i@i@i@i@i@i@i@i@kAh>{BÊ8øùžÿŸ ÿ¡ ³hk   -ZEA¸£Ÿ‚7ò§Uÿ°ÿ´ÿ×ÿÛ#ÿÙÿØÿÙþ´ãtÔeÙ\ÓVÙWÜZç_âZß^ÿÿ•ímöuÿŒÿœ4ÿš2ö‡5ÉZ¦<›1¢3©:²B·GÁPÏ^çvÿ’ÿ«ÿ¸ ÿ¾ÿÅÿÐ ÿÕÿÖÿÕÿÔ ÿÔ ÿÒÿÍÿÆÿÅÿÉ ÿÐÿÖÿÒÿÍÿÌÿÌÿÉÿÆÿÌÿ´û”ýÿŸÿ©ÿ±ÿ¹ÿÁÿÌÿÌÿÍÿÎÿÌÿÇ ÿÃÿÄÿËÿÑ ÿÌÿÅÿ¶üÔuº[®fÆ~-¾ˆR½‡Qµ„Rµ„R»†Sº…R»‚M®u@—Y(}?k+\XXWVVVTUVVOON OVY`i~(Š4¡Dº] Ü}ð‘%ãÎlÐk -ØsÞmËZ¿F¸?²6®2Ð*Ð*á(ç.ø-ù.ÿ-ÿ.ÿ,ÿ,ÿ* -ÿ* -ÿ* ÿ* ÿ* -ÿ* -ÿ)ÿ)ÿ)ÿ)ÿ)ÿ)ÿ)ÿ)ÿ* ÿ) ÿ* ÿ* ÿ* ÿ* ÿ* ÿ* ÿ) ÿ) ÿ* ÿ* ÿ,ÿ,ÿ,ÿ,ÿ,ÿ.ÿ.ÿ+ÿ*ÿ,ÿ-ÿ-ÿ/ÿ1ÿ2ÿ7ÿ:ÿ> ÿD"ÿD"ÿD%ÿC$ÿB#ÿB#ÿB#ÿ@!ÿB#ÿB#ÿC#ÿA!ÿ?ÿ?ÿ@ÿAÿAÿAÿ@ÿ?ÿ@ÿ@ÿ?ÿ@ÿAÿ@ÿ?ÿ>ÿ>ÿ>ÿ>ÿ>ÿ=ÿ<ÿ?ÿ?ÿH.ø?%¼?#¬/Ó×þÿ' ÿ.ÿ1ÿ3ÿ2ÿ1ÿ3ÞI ìW.¬x@¬x@£wD£wD©vI¨uH©tF©tF©tA©tA©tA©tA¨sB¨sB¨sB¨sB¨sB¨sB§tB¦sA¦s?¥r>¦s?¥r>¥r>¦s?¥r>¥r>¥r@¥r@¤q?¤q?¤q?¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¡n<¡n< m> m> m> m>Ÿl= m> m> m>žk<Ÿl=Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl:Ÿl=žk<žk>žk>žk<žk<žk<žkœk<šl>—i;˜g<˜g<˜g<—f;a8^5Š]4‡Z1ƒY/W-W/€V.€V.€V.€S,€S,R+R+|R.{Q-zP,zP,zP,zP,zP,zP,zP,zP,zP,zP,{Q-{Q-|R.{Q-uL*uL*uL*wN,wN,wN,wN,wN,wN,wN,wN,wN,wN,wN,yO'xN&ŒSÉ>øøþ ÿ«Àu e 4V ׌:ÿ´ÿ¬ÿà ÿà(ÿß!ÿÔÿà ÿ×ÿª ézádägà^ÛYßWßWÔSÜ[êjâbÓRÒQÂH®4¡2 1£9¨>¥6¬=µE»KÎ]çv ÿ‘ÿ¡ÿ«ÿ±ÿ¼ÿÁÿÍ ÿÕÿ×ÿÖÿÕ!ÿÕ!ÿÕÿÓÿÍ ÿÉ ÿÆÿÉ ÿÒÿØÿÒ ÿÌÿÍÿÉÿÅÿÉÿÆÿ¶ÿ˜ÿ—ÿ¤ÿ¯ÿ¹ÿÀÿÊÿÌÿÍÿÍÿÌÿÉ ÿÄÿÄÿËÿÎ ÿÊ ÿÊ ÿ ÿ¹ÿ¡Ùz³kÂz)¾ˆR½‡Q·†T·†Tº…Rº…R¿†QºL§i8‹Mu5 b"[XYUTRRQ R R N N ORZ[diz$„.˜;­PÈiÝ~à~ Äb¾YØsö…ÿ’*ÿš.ÿ”(ÿ‹ù}ÿjûUø?õ<ÿ7ÿ6ÿ4 -ÿ3 ÿ.ÿ-ÿ+ ÿ+ ÿ* ÿ* ÿ* -ÿ* -ÿ)ÿ)ÿ)ÿ)ÿ)ÿ(ÿ)ÿ)ÿ* ÿ* ÿ* ÿ* ÿ) ÿ) ÿ) ÿ) ÿ) ÿ* ÿ* ÿ* ÿ,ÿ.ÿ.ÿ.ÿ,ÿ,ÿ,ÿ)ÿ*ÿ,ÿ+ÿ-ÿ/ÿ1ÿ2ÿ8ÿ<ÿ> ÿD"ÿE#ÿD%ÿC$ÿ@!ÿ@!ÿ? ÿ? ÿ@!ÿ@!ÿA!ÿ@ ÿ@ÿAÿAÿAÿAÿAÿ@ÿ@ÿ@ÿAÿ?ÿ?ÿDÿ@ÿ?ÿ=ÿ>ÿ?ÿ@ÿ@ÿ?ÿ?ÿ;ÿ0 á(Ç}zÀÕÿÿ& ÿ.ÿ0ÿ2ÿ3ÿ2ÿ4ßJ!ìW.¬x@¬x@£wD£wD©vI¨uHªuGªuGªuBªuBªuB©tA©tC©tC©tC¨sB¨sB¨sB§tB§tB§t@§t@¦s?¦s?¦s?¦s?¦s?¦s?¦sA¦sA¥r@¥r@¤q?¤q?¥r@¥r@¥r@¥r@¥r@¤q?¢o=¢o=¢o=¡n< m> m> m> m> m> m> m>Ÿl=Ÿl= m>Ÿl: m; m;Ÿl:Ÿl:Ÿl:Ÿl=Ÿl=žk>žk>žk<žk<žk<žk<žk<žk‘:‹4”8žB«J¸WØkí€ýÿÿ¦ÿ® ÿºÿ¼ÿÂÿÊÿÕÿØÿÖÿÖÿÖ"ÿÔ ÿÏÿËÿÊÿÆÿÉÿÕ ÿÕÿÐÿÏÿÍÿÈÿÆÿÊ -ÿË ÿ´ÿ ÿ ÿ®ÿ¸ÿ½ÿÉÿÌÿÍÿÍÿÍÿËÿÅÿÃÿÉÿÊ ÿÊ -ÿË ÿÇÿÄÿ»øœÉzÊ{ÇHÇH»‡Q¼ˆR»‰PºˆO¿†O¿†OµyG˜\*|>g)[TVRP O O O O O L MNRWYbds {(4B¬OÉlÜyÃ`»XÇdÚvô ÿ¤ ÿ±ÿÂÿÈÿÃÿ´ÿŸðŠïlã`îUèOòEîAö=ö=ü9ø5ÿ3ÿ3ÿ.ÿ-ÿ,ÿ,ÿ,ÿ,ÿ,ÿ,ÿ-ÿ,ÿ+ÿ, ÿ* -ÿ) ÿ) ÿ) ÿ) ÿ* ÿ* -ÿ+ ÿ+ ÿ,ÿ-ÿ-ÿ+ÿ)ÿ)ÿ(ÿ' ÿ*ÿ+ÿ+ÿ-ÿ0ÿ4ÿ8ÿ=ÿ@!ÿD$ÿE%ÿD%ÿC$ÿC#ÿA!ÿ@!ÿ@!ÿB#ÿB#ÿ?!ÿ?!ÿ?ÿ?ÿ@ÿ@ÿ@ÿ@ÿ@ÿ@ÿ@ÿC!ÿ@ÿ@ÿCÿ@ÿ?ÿ?ÿA#ÿC%ÿI1ÿA)ËB+¸/w!_ ,#$ ·Å!ÿÿ'ÿ/ÿ0ÿ3ÿ5ÿ3ÿ6ÜL"éY/¯yA®x@©xF©xF«xK©vIªuGªuGªuBªuBªuB©tAªuDªuD©tCªuD©tC¨sB§tB§tB§t@¨uA¨s@¨s@¨uA§t@¦s?¦s?¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¥r@¥r@¥r@¥r@¢o=¢o=¢o=¢o=¡n<¡n<¡n?¡n?¡n?¡n?¡n? m> m; m; m; m; m; m;Ÿl: m; m>Ÿl=žk>žk>žk>žk>Ÿl=žk<žk<žk<œk<œk<›j=›j=›j=›j=›j=›j=™h;›j=›j;™h9™h9™h9™h9˜g8˜g8˜g8—f7—f7—f7–e6–e8–e8–e6–e6–e8–e8–e6•d5•d5•d5”c8”c8’d8’d8’a6’a6’a6‘`5‘`5‘`5Ž`4Ž`4Ž`2_1_1_1_1_1_3Œ^2Œ^2Œ^2‹]1‹]1‹]1Œ^2‹]1‹]1‰[/‰[/‹]1‹]1‰[/ˆZ.‰[/‰[/‰[/ˆZ.‡X/‡X/…X/„W.…X1…X1„W.„W.„W.„W.ƒV-ƒV-ƒV-‚U,‚U,‚U,‚V*‚V*‚V*‚V*€S*€S*€S,€S,R+R+R+R+R+R+~Q*~Q*~Q*~Q*­zF­zF­zF­zF­zF­zF­zF­zF­zF®{G®{G®{G®{G®{G®|E®|E®{G®{G¯}F¯}F¯}F¯}F¯}F¯}F°~G°~G°~G°~G°~G°~G²€I²€I²€I²€I³I³I´€J´€J´€J´€J´€JµKµKµKµKµKµKµK´‚KµƒL¶‚L¶‚L¶‚L¶‚L¶‚L¶‚L¶‚L¶‚L·ƒM·ƒM¸„N¸„N·‚O·‚O¸„Nº†P¸„N·ƒM·‚O¶Nµ€Mµ€Mµ€M´L²M°}K¯|J­zH¬yG«xF¨uA¨uA§tB¦sA¥r@¥r@¢q?¢q? o@¢qB¢qBŸn?l=l=›j;›j;™k?—i=—i=—i=—i=—i=—i=—i=—i=—i=—i=—i=—i=™k?™k?™k?”e<•f=•f=•f=–g>–g>–g>–g>–g>–g>–g>–g>–g>–g>˜g:•d7¢iÅŒ>ù›ÿª ÿ¶-º]M ?0&7! -Z¾w1ÿ¤ÿªÿ³ÿÁÿë;ÿâ2ÿÔÿØÿÙ -ÿßÿÀý–ìe×PåLêQäOâMßMßMØXädûˆçt–0Ž(’;“<¢F§K°O¸WÔgâuñƒüŽÿŸÿ©ÿµÿ¹ÿ¾ÿÂÿÎ ÿÖÿ×ÿÖÿÔ ÿÓÿÑÿÌÿÌ ÿÊÿÇÿÍÿÕÿÕÿÐ ÿÏÿÌ ÿÊÿÇÿÈÿÄ ÿ³ÿ¤ÿ«ÿµÿºÿÆ ÿËÿÍÿÎÿÎÿÌÿÇ ÿÂÿÅÿË -ÿË ÿÌ ÿÊÿÅÿÄÿ²ÛŒ#Í~Á…FʼnJ¾ŠT¾ŠT½‹R»‰P¿†O¿†O¼€N¥i7ŠL t6 -b"TRP O N -M N -O P L NRTW[bfqx%‡,Ž3˜;º] -Ôq À]»XÏlá}í‰þžÿ¦ ÿ±ÿ» ÿÃÿÉÿÄ#ÿ¸ÿ—ÿ}ücô[ÿTýPÿHÿFÿBÿ?ÿ<ÿ9ÿ2ÿ1ÿ0ÿ-ÿ+ÿ+ÿ-ÿ,ÿ-ÿ-ÿ, ÿ, ÿ+ ÿ* -ÿ* ÿ) ÿ( -ÿ) ÿ* -ÿ+ ÿ+ ÿ+ ÿ* ÿ* ÿ*ÿ)ÿ(ÿ(ÿ' ÿ*ÿ+ÿ+ÿ-ÿ/ÿ4ÿ7ÿ<ÿ@!ÿD$ÿE%ÿD%ÿD%ÿD$ÿA!ÿ@!ÿB#ÿB#ÿB#ÿA#ÿ?!ÿ?ÿ?ÿ?ÿ?ÿ?ÿ@ÿ@ÿ@ÿC!ÿC!ÿ?ÿ@ÿ@ÿAÿ?ÿCÿA#ÿ7ò)Ú” G?!&·Æ"ÿÿ&ÿ/ÿ1ÿ4ÿ5ÿ4ÿ7ÝM#í]3±{C¯yAªyGªyG«xK«xK¬wI¬wI¬wD¬wDªuBªuB¬wF¬wFªuD¬wFªuD©tC©vD©vD©vB©vB©tA©tA¨uA¨uA¨uA¨uA§tB§tB§tB¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¥r@¥r@¥r@¤q?¤q?¤q?¤q?¤qB¢o@¢o@¢o@¢o@¡n?¡n<¡n<¡n< m; m; m; m; m; m> m> m@ m@Ÿl?Ÿl?Ÿl=Ÿl=Ÿl=žk<œk<œk<œk>›j=›j=›j=›j=›j=›j=›j=›j;›j;™h9™h9™h9™h9˜g8˜g8˜g8—f7—f7—f7—f9—f9—f7–e6–e8–e8–e6–e6–e6–e6•d9•d9“e9’d8’a6’a6’a6’a6‘`5‘`5Ž`4Ž`4Ž`2Ž`2Ž`2_1Ž`2Ž`2_3_3_3Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]1Œ^2‹]1‹]1‰[/‹]1‰[/‰[/‰[/ˆZ.‰Z1ˆY0…X/…X/…X1„W0…X/…X/„W.„W.„W.„W.ƒV-ƒV-ƒV-ƒV-ƒW+‚V*‚V*‚V*‚U,€S*€S,€S,R+R+R+R+R+R+~Q*~Q*~Q*~Q*­zF­zF­zF­zF­zF­zF­zF®{G®{G®{G®{G®{G®{G®{G®{G®{G¯}F¯}F¯}F°~G°~G°~G°~G°~G°~G°~G²€I²€I²€I²€I²€I²€I²€I²€I´€J´€J³I´€J´€J´€J´€JµKµKµKµKµKµKµKµƒLµƒL¶‚L¶‚L¶‚L¶‚L¶‚L¶‚L·ƒM·ƒM¶‚J·ƒK¸„N¸„N·ƒM·ƒM·ƒM·ƒM¸„N¸„N¸„Nº†Pº†P¸„N¶‚L¶‚Lµ€M´L´L³~K±|I°{H°|D°|D¯{E¯{E°|F¯{E­yA¬x@¬xB­yC­yCªv@ªv@©u?§s=§s=¦s?¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¥r>¦s?§t@¦s?¢q?¢q?¢q?£r@¤sA¤sA¤sA¤sA¤sA¤sA¤sA¤sA¤sA¤sA¦t=¢p9¥l"Ê@ò›ÿ¬ÿ¾5²UD8 -" 6Êxè–ÿŸÿ©ÿ«ÿµÿÐ#ÿé<ÿÙÿÖÿØ ÿßÿÛÿÉ -ýžØyÌ`ÅYÉVÈUÌ[Þmä‹ù ÿÆÿÙÌrŽ4‹EŽHyG!uC¢S§X½ZÉfçxø‰ÿ‘ÿžÿªÿ°ÿ¶ÿ¹ÿÃÿÊ ÿÓÿÕÿÕÿÔÿÕÿÓÿÏÿÌÿÆÿÆÿÌ ÿÒÿÒÿÎ ÿÌ ÿÌ ÿÉ ÿÉ ÿÄ -ÿÄ -ÿ»ÿ±ÿ´ÿ»ÿÇ ÿËÿÎÿÑÿÎÿËÿÃÿÁÿÀÿÆ ÿËÿÊÿÈ -ÿÇ ÿÈÿ½õ¡Ö‚Ä€<ΊF¾ŠT¿‹U»‹NºŠM»‰PºˆO¾ˆR±{E—]-z@e)TNK I -GGJ K MMPRTYZ`blp!w€(’3 A¹W»Y½YÐlàzñ‹ÿ ÿ©ÿ¶ÿ¼ÿÅ ÿÆ ÿÌÿÑÿÄÿ©òƒÞoädãcß]ß]å\ä[àWßVëQêPóJòIøDó?õ9ó7ÿ4ÿ4ÿ.ÿ.ÿ* -ÿ* -ÿ* ÿ* ÿ( -ÿ( -ÿ) ÿ* ÿ+ ÿ+ ÿ+ ÿ+ ÿ+ÿ*ÿ)ÿ)ÿ( -ÿ) ÿ+ÿ.ÿ.ÿ/ÿ1ÿ6ÿ<ÿ?!ÿB#ÿC$ÿE'ÿE'ÿC%ÿA#ÿC%ÿD&ÿA!ÿA!ÿB#ÿ? ÿ>ÿ? ÿ? ÿ? ÿ@ÿ?ÿ?ÿ@ÿD!ÿBÿ@ÿAÿE(ÿ?"óE)è: 4‰B+ '·Æ#ÿÿ# ÿ/ÿ2ÿ4ÿ4ÿ3ÿ5ÛM#ë]3¯{C­yA«zHªyG¬yJ¬yJ­xG­xG­xE­xE¬wD¬wD¬wI¬wI¬wI¬wI¬wK¬wK¬wIªuGªuGªuGªuGªuG©tF©tF¨uF¨uF¨uF§tE§tE§tE§tE§tE§tE§tE§tE§tE¦sD¦sD¥rC¥rC¥rE¤qD¤qB¤qB¤qB¢o@¢o=¢o=¢o=¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n< m> m>Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=žk<žk<œk>›j=›j=›j=œk>œk>›j=œk>œk<›j;›j;›j;™h9™h9™h9˜g8˜g8˜g8—f7—f7—f7—f7—f7–e6–e6–e6–e6–e6–e6–e6–e:•d9“e9’d8”c8’a6”c6”c6’a4’a4b4a3a3a3a3a3Ž`2Ž`2_4^3^3^3Ž]2Ž]2Œ^2Œ^2Œ^2Œ^2Œ^2Œ^2‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‰Z1‰Z1ˆY0‡X/ˆY2ˆY2‡X/‡X/‡X/†W.„W.„W.ƒV-ƒV-ƒV-ƒV-ƒW+ƒW+ƒW+‚V*‚U,‚U,‚U.‚U.€S,€S,€S,€S,R+R+R+R+R+R+®{G®{G®{G®{G®{G®{G®{G®{G®{G®{G®{G®{G¯|H¯|H¯|H¯|H¯}F¯}F¯}F°~G°~G°~G°~G°~G²€I²€I²€I²€I²€I²€I²€I²€I³J³J´€J´€J´€JµKµKµKµKµKµKµK¶‚L¶‚L¶‚L¶‚LµƒL¶„M·ƒM¸„N¸„N¸„N¸„N¸„N¸„N·ƒM¸„L¸„L¸„N¸„N¸„N¸„N·ƒMº†Pº†P»‡Q»‡Q»‡Q»‡Q»‡Q»‡Q»‡Q¸ƒP¸ƒP¸ƒP¶N¶Nµ€MµI¶‚J¶‚L¶‚LµK´€J±}E°|D³I´€JµK´€J³I±}G°|F°|F®{G®{G­zF­zF­zF­zF­zF­zF­zF­zF­zF­zF®{G¯|H°}I¯|H«zH¬{I¬{I­|J­|J­|J­|J­|J­|J­|J­|J­|J­|J­|J¯}F­{D¯v,Ê@óœÿªÿ¬#³VY> & a=ó¡$÷¥(ÿ¢ÿ¨ÿªÿ¯ÿÀÿá4ÿá'ÿ×ÿÙÿÙÿÚÿÛÿÎ%ÿÀÿ£ý‘ôù†ÿ“ÿ­ÿÊÿÌÿÕÿâ ÿ¾M™?t.o)IB…6¨Y¶SÁ^Ýnê{û‰ÿ‘ÿ ÿ©ÿ±ÿ·ÿ½ÿÂÿÍÿÓÿÖÿÖÿÙ"ÿ× ÿÒÿÍÿÉ ÿÅÿÅÿË ÿÒÿÑÿÍ ÿÌ ÿÉ ÿÊ ÿÅ ÿÅ ÿÄ ÿ¹ÿ´ÿ»ÿÅ ÿÊÿÎÿÎÿÎÿÒÿÍÿÈ ÿÆ ÿÂÿà ÿà ÿÃÿÃÿÉÿÉÿµ*äÄ€<̈D¾ŠT¿‹U»‹N»‹N½‹R»‰PÁ‹U½‡Q§m=ŠP s7 ZQK J GI -J L MNPRSVY_aimv}%‰*‰*¥CºX¾ZÍiâ|õ ÿ¡ÿ®ÿº ÿ¿ÿÄ -ÿÇ ÿÍÿÓÿÑ ÿÇÿ© ø‰ëkädãaãaç^æ]å\ä[ñWñWýTÿ[ÿaÿZ ÿNÿEÿ>ÿ:ÿ1ÿ.ÿ* -ÿ* -ÿ* ÿ+ ÿ+ ÿ) ÿ* ÿ+ ÿ+ ÿ+ ÿ+ ÿ+ ÿ+ÿ*ÿ)ÿ( ÿ( -ÿ* ÿ,ÿ,ÿ-ÿ/ÿ1ÿ6ÿ=ÿ?!ÿB#ÿC$ÿD&ÿC%ÿA#ÿC%ÿC%ÿC%ÿC#ÿC#ÿB#ÿ? ÿ? ÿ@!ÿ@!ÿ@!ÿ@ÿ@ÿ@ÿC!ÿD!ÿAÿ?ÿ9ÿ3ÿ,ʶl_( '¸È%ÿÿ&ÿ0ÿ2ÿ4ÿ3ÿ4ÿ5ÛM#ì^4¯{C­yA«zHªyG¬yJ¬yJ­xG­xG­xE­xE­xE­xE­xJ­xJ­xJ¬wI­xL¬wK¬wI¬wIªuG¬wIªuGªuG¬wIªuG¨uF¨uF§tE¨uF§tE§tE§tE§tE§tE§tE§tE§tE§tE§tE¥rC¥rC¥rE¥rE¤qB¤qB¤qB¤qB¢o=¢o=¢o=¢o=¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n<¡n? m> m>Ÿl= m> m>Ÿl= m> m> m>žm@l?l?l?œk>œk>œk>œk>œk<›j;›j;›j;›j;™h9™h9™h9˜g8˜g8˜g8—f7—f7˜g8—f7—f7—f7—f7–e6—f7—f7–e6–e:–e:”f:”f:•d9•d9•d7•d7”c6”c6b4b4b4b4a3a3a3a3‘`5_4_4_4^3^3_3Œ^2_3_3Œ^2Œ^2‹]1‹]1‹]1‹]1‹]1‹]1‹]1‹]1‰Z1‰Z1‰Z1ˆY0ˆY2‡X1ˆY0ˆY0‡X/‡X/…X/…X/…X/„W.„W.„W.„X,ƒW+ƒW+ƒW+ƒV-ƒV-‚U.‚U.‚U.‚U.€S,€S,€S,€S,€S,€S,R+R+®{G®{G®{G®{G®{G®{G®{G®{G®{G¯|H¯|H¯|H¯|H¯|H¯|H¯|H°~G°~G°~G²€I²€I²€I²€I²€I°~G²€I²€I²€I³J³J³J³J³J³JµKµKµKµKµKµK¶‚L¶‚L¶‚L¶‚L·ƒM·ƒM·ƒM·ƒM¶„M¶„M¸„N¸„N¸„N¸„N¸„N¸„N¸„N¸„N¸„L¸„L¸„N¸„N¸„N¸„N¼†P¼†P¼†P½‡Q½‡Q½‡Q¾ˆR¾ˆR¾ˆR½‡Q»‡Q¼ˆRº†Pº†P¸„N¸„Nº†N»‡O»‡O¸„L·ƒK·ƒKº„Lº„Lº„Lº„Lº„Lº„L¸„L¸„L¸„L·ƒK·ƒM·ƒM·ƒM·ƒM·ƒM·ƒM·ƒM·ƒM·ƒM·ƒM·ƒM·ƒM·ƒM·ƒM·ƒM¶‚LµK¶‚L¶‚L¶‚L¶‚L¶‚L¶‚L¶‚L¶‚L¶‚L¶‚L¶‚L¶‚L¶‚L·ƒK¶‚J·}?ˆJç•ô¢!ÿªÔvt"\ -- % -E.+Ù¿`, -a- ÿŸ$ÿ %ÿ£ÿ«ÿ­ÿ³ÿºÿËÿçEÿÜ:ÿÜÿÛÿØ ÿ×ÿÚ ÿäÿàÿÙ ÿÒÿÕ ÿØÿÝÿØÿ×ÿÖ ÿÐÿá4Áu@;(cJ.¨S ­XÊ_Ùnð~úˆÿ”ÿžÿªÿ²ÿ¹ÿ½ÿÅÿË ÿÕÿØÿ×ÿ×ÿÔÿÑÿÌÿÈ ÿÅÿÅÿÍ ÿÓÿÒÿÎ ÿÍÿÌ ÿÉ ÿÇ ÿÇÿÆÿ¾ÿ½ÿÁ -ÿËÿÐ)ÿÑ*ÿÓ2ÿÓ2ÿÈ8ÿµ%ÿœ8ÿŠ&ð~î|ï‚ûŽ#û—&ÿŸ.ó¤9ß%Å„HȇK¿ŠWÁŒY¾ŒS¾ŒS¾ŒQ½‹PÁ‹SÁ‹Sµ{Ižd2G`&RJ I I J KLLNPQQRT[]firy$!{¤B½[¾WÎgæ{ùŽ ÿ¡ÿ®ÿ¸ ÿ¼ÿà ÿÇ ÿÎÿÒÿÕÿÙÿÒÿ¶ú‰åtìeébàaàaá_Ý[Ý[ÜZÞXä^òhõköiñdæ]àWáHØ?è=ê?ó<ó<õ4õ4ÿ3ÿ1ÿ-ÿ-ÿ) ÿ) ÿ* -ÿ+ ÿ* ÿ( -ÿ(ÿ+ÿ.ÿ,ÿ-ÿ0ÿ3ÿ7ÿ=ÿ?!ÿB#ÿC$ÿD&ÿA#ÿ@&ÿB(ÿB%ÿB%ÿB#ÿB#ÿC#ÿ?ÿ@ÿ@ÿ@ÿ@ÿ?ÿ@ÿ@ÿAÿEÿ@ïIèB ¹!& -   - -+@ÎÜ ÿÿ(ÿ1ÿ2ÿ4ÿ4ÿ3ÿ3ÛQ&ê`5¯{C¬x@­zF­zF«zK«zK®yH®yH­xE®yF®yF­xE­xG­xG­xG­xG­xJ­xJ¬wI¬wI¬wI¬wI¬wF¬wF¬wFªuDªuD©tC©tFªuG¨uC§tB§tB§tB§tB§tB§tB§tB§tE§tE§tE§tE¦sD¦sD¦sD¦sD¥rC¤qB¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¢o=¡n<¡n<¡n<¡n?¡n?¡n?¡n? m> m> m> m> m> m> m> m>Ÿl?Ÿl?Ÿl?žk>žk>žk>žk<žk m> m> m> m> m> m> m@ m@Ÿl?Ÿl?Ÿl?Ÿl?Ÿl=žk<žk<žk¨)—˜›" '¤3¤3–3,~7™R ·}KÈŽ\ÁŽZÃ\ÂYÂYÁTÀŽSÃSÄŽTÁ‹SµGc1Ee)SOMLLKKL M PRRRWV`ku rqwªG¿\½VÎgå|÷Ž -ÿ¢ÿ¯ÿ¸ÿ¼ ÿÄÿÈ -ÿÏ ÿÓÿØÿ×ÿÜÿÛÿÊÿ®ü‡èsäiâgå`â]ÞYÜWÞXã]ÞZÝYÜZÛYÌYÅR±Q¯O C€4i%]cj!€"‡)  ) ¦/½1Â6ç0è1ò.ô0ÿ3ü.ÿ-ÿ-ÿ/ÿ1ÿ5ÿ9ÿ@ÿAÿAÿCÿCÿCÿB!ÿ@ÿBÿBÿ@ÿ@ÿBÿ=ÿ@ÿ@ÿCÿB ÿGÿAÚCÎ7¦?¦?§=Íc!å* Ä pe',8 387++FO•ªíòÿÿ'ÿ1ÿ2ÿ3ÿ1ÿ/ÿ0ÙU(çc6®|A®|A®{G®{G­|M­|M®yH®yH¯zG¯zG¯zG®yF¯zI¯zI®yH®yH¯zL¯zL¯zI®yH­xG­xG®yH­xG­xG­xG­xG­xG¬wFªuDªuDªuD©vD©vD©vD¨uC¨uC¨uC§tB§tB§tE¦sD§tE¦sD¦sD¦sD¦sA¦sA¦sA¦sA¦sA¦sA¦sA¥r@¥r@¥r@¥r@¥r@¥r@¥r@¥rC¤qB¢o=¢o=¢o=¡n<¡n?¡n?¡n? m> m> m> m> m> m> m> m@Ÿl?Ÿl?Ÿl?Ÿl=Ÿl=žk<žk<žk<žkœRˆ>S7)+/$hÒ‚.ÿ¬ÿ±ÿ´ðísÿÿ¾ÿá7ÿí1ÿÛÿØ ÿÙ -ÿÛÿÛÿØ ÿ×ÿÚ ÿÚ ÿØÿÙÿÙ ÿØÿÕ ÿÔÿÑÿÛ -ÿÇXo+ F"ªN¶ZÅTÑ`èqò{ÿÿ™ÿ¥ÿ¯ÿ·ÿºÿÃÿÉÿÏÿÔÿÖÿÕÿÔÿÔÿÐÿÊÿÆÿÅÿÊÿÓÿÔÿÑÿÍ ÿÎ ÿËÿÆÿÂÿÉ -ÿ² Ók¢,– š!—šœ™!ž%£*¡0¡0–3”1€9•N´zHÈŽ\ÁŽZÃ\ÂYÂYÂUÁTÄŽTÅUÅW¾ˆP¬r@‘W%y=`$RMLKKJ L M PRRRUP ckpmow¨E¿\¾WÐiæ}÷Ž -ÿ£ÿ® ÿ·ÿ» -ÿÄÿÈ -ÿÏ ÿÓÿØÿ×ÿØÿÙÿÔÿÊÿ¡ õ€ækãhæaã^ÝXÜWßYàZá]â^â`à^ÍZÆS¢BŠ*bTIKOQcex— ŸÊÏÞé% ý/ÿH,ÿ>!ÿ2ÿ.ÿ1ÿ4ÿ8ÿ?ÿ@ÿAÿCÿDÿDÿC"ÿB!ÿC ÿC ÿBÿ@ÿBÿ=ÿ?ÿ>ÿ@ ÿ=ÿDÿBÛDÐ9¤=¦?¥;»Qä) Ø’ -ƒ;112:>7 7 Q^¥¯ëòÿÿ'ÿ1ÿ2ÿ2ÿ0ÿ+ ÿ/ÙU(æb5­{@®|A®{G®{G­|M­|M°{J°{J¯zG¯zG¯zG¯zG°{J°{J¯zI°{J°{M¯zL°{J¯zI®yH¯zI®yH®yH®yH­xG­xG­xG­xG¬wF¬wF¬wF«xF«xF©vD©vD©vD©vD¨uC¨uC¨uF§tE¨uF¨uF¨uF¨uF¨uC§tB§tB§tB§tB§tB¦sA¦sA¦sA¥r@¦sA¦sA¥r@¥r@¥rC¥rC¤q?¤q?¢o=¢o=¢o@¤qB¤qB¢o@¡n?¡n?¢o@¢o@ m> m> m@ m@Ÿl?Ÿl? m>Ÿl=Ÿl=žk<žk<žk<žk m>¡n? m>Ÿl= m>Ÿl=žk<žk<žk<žk<žk<žk<Ÿl=j;j;j;j8j8j;j;j8›h6›h6›h6šg:šg:šg:™f9™f9™f9˜e8˜e8˜e8˜e8•d7•d7–e8•d7•d7•d7”c6•d7•d7”c6’a6’a6’a6”c8”c8’a6’a6’a6‘`5‘`5‘`5‘`5_4_4Ž`4_3^5^5^5Œ]4Œ]4‹\3Œ]4Œ]4[3[3\1\1‰Z1‰Z1‰Z1‰Z1ˆY0ˆY0ˆZ.ˆZ.ˆZ,‡Y+‡Y-‡Y-†W.†W.†W.†W.†W0†W0…V/…V/…V/…V/…V/…V/°{J°{J°{J°{J°{J°{J°{J±|K±|K±|K°}I°}I°}K°}K°}K°}K²K³€L³J³J³J³J³J´‚K³J´‚K´‚K´‚K´‚KµƒLµƒLµƒLµƒL¶„M·ƒM·ƒM·ƒM·ƒM¸„N¸„N¸„N¸„N¸„N¸„N¸„N¸„N¸„N¸„N¹‡P¹‡Pº†P»‡Q»‡Q»‡Q»‡Q»‡Q»‡Q»‡QºˆQºˆQ¼ˆR¼ˆR»‡Q»‡Q»‡O»‡O½‰Q½‰Q½‰S¾ŠT¾ŠT¾ŠT¾ŠT¿‹U¿‹U¾ŠT¿‹U¿‹U¿‹U¿‹U¿‹S¾ŠR¿‹S¿‹SÁUÁU¿‹SÁUÁUÁU¿‹SÁUÃUÃUÃUÁ‹S¿‹U¿‹U¿‹UÁWÁWÁWÁWÁWÁŒYÁŒYÁŒYÁŒYÁŒYÁŒYÁŒYÁŒY¿‹S¾ŠR¿‹U¿‹UÁUÁUÂŽVÂŽVÃUÃUÅŒUÆVÅŒWÅŒWÃWÄŽXÄŽXÄŽX΋DɆ?åŽöŸÿ£ÿ«ÿªÿªÿ§ÿ»ÿÐGÿÉ@Ý”/ŸV‡&/œ#$ïˆÿ·=ìv ¶@±$½0ñ—ÿÀÿã4ÿë<ÿÞÿÜÿÝÿÝÿÙÿØÿÙÿÙÿÙ ÿÙ ÿÚÿÙ ÿÕÿÔÿÓÿÐÿÏÿÇd*##U.²T ¶XÅTÖeçuô‚ý‘ÿ›ÿ¤ÿ¯ÿ·ÿ¹ÿÂÿË ÿÓÿÓÿÔÿÕÿÔÿÓÿÏÿÌ ÿÇÿÆÿÊ ÿÎÿÏÿÒÿÏ ÿË ÿÅÿÄÿÁ ü”¸;¨+²6¸<«6¥0ª2¬4­9±=±F¨=”:6?["·‚TÅbÂWÑXÄZÄZÑXÂWÄŽVÅWÃUÄŽVÀŠT±{E›a1z@`&UJG G G I K NQRO QXk eddgo§G¿_Ã\Òkç~÷Ž ÿ ÿ« ÿµÿ¹ÿÈÿÌÿÑ ÿÕÿÕÿÖÿØÿÚÿÛÿÛÿÓ!ÿ½ ÿçpåiàdØYÛ\âeæiÝhÏZ§>‡YO==2 -3 2 -2 -1 1 - -, / -//44>=PQœ3âyTÿƒVúxKíLÝ<ÿ@ÿB ÿA ÿC ÿCÿBÿBÿCÿD ÿFÿJÿJÿJ ü=à5ä9ØKßRËZÇV¿[À\³Z£J•C;»ºßÛ»Ã$¤;¹P0º]:¶Y6ÀE$³8Õ%Ò" ôòòøÿ!ÿ+ÿ/ÿ0ÿ-ÿ*ÿ+ÿ+ØEèU*³t<¸yA³|I³|I°}K°}K±|K±|K°{J±|K±|I±|I³|K³|K°{J°{J°{J±|K°{J°{J¯zI¯zI°{H¯zG®yF®yF®yH®yH®yH®yH­xG­xG­xG­xG¬yG¬yG«xF«xF«xF«xF«xF«xF©vG©vG©vD©vD©vD©vD¨uA¨uA¨uA§t@§t@§t@§t@§t@§t@§t@§t@§t@¦s?¦s?¦sA¥r@¥r@¥r@¥r@¥r@¥rC¤qB¤qB¤qB¤qB¤qB¢o@¢o@¢o@¢o@¢o@¢o@¡n?¡n?¡n? m> m>Ÿl=Ÿl= m> m>Ÿl=Ÿl=Ÿl=žkÿ§ÿ¦ÿ¥ÿ£ÿ£ÿ§ÿÆÿê:ÿóRÿÇ&Öc¯<À*¾(¾2ÑE«(¦#ª* ¤$ÅTÿ©HÿÊÿâ/ÿâ-ÿÞ)ÿÝÿÞÿÜÿÙÿÙ ÿÙ ÿÙÿÙÿØÿØÿÖ ÿÕ ÿÕ ÿÑ ÿÔÿà؈Kq;“]:ºXºXÒbãsìƒ÷Žÿ˜ÿ¢ÿ«ÿ²ÿºÿÁÿË ÿÐÿÔÿÕÿÕÿÕÿÓÿÎÿÉ ÿÆÿÆÿÊÿÑÿÒÿÑÿÍ ÿÄÿÃÿª áp¶B·CÉWÜjÒjÉaÇcÄ`ÀaÄe¶^¤LŠ;†7›Y"¹w@ÆVÊ‘ZÂ’YÂ’YÄ’YÄ’YÃWÄXÃWÃWÆXÇŽYÈŽ^¿…U§m;ŠPl2WMH H -I I LJJK -I]glg``bi˜<³WÀ]Ïl ç€÷ÿž ÿªÿ· -ÿ¼ÿà ÿÉÿÒÿÕÿØÿÙÿÙÿÙÿÚÿØÿÛÿÔÿ«àƒàkÝhÛbÝd²b, P|5bF D A ? -; 9 45 66333220./.,++135Q;Ô™sÊi¹[(›= -ÏFÐGÚFÝIÞIÝHßIßIãJèOãTáRÁN¾KµF·HÈVÏ]Æ]ÁXÀTÄXº^«O™B‘:….….§'ª*Ÿ%¬2©U ÊvAĈVÇU¿wQ­e?²F¢6 Û×îòÿ ÿ*ÿ/ÿ-ÿ* ÿ' ÿ( ÿ( ÿ' ÿ/éV+÷d9ÁxCÄ{F°K°K·}M·}M·}M·}M¬~K¬~K®}K®}K°}K°}K°}K¯|J¯|J¯|J¯|J®{I®{G®{G®{G®{G®{G­zF­zF­zF®yH®yH­xG­xG­xG­xG­xG­xG­xG­xG¬wF¬wF¬wFªuD¬wF¬wFªuDªuDªuD©tC©tC©tC¨uC§tB§tB§tB§tB§tB§tB§tB§tB§tB¦sA¦sA¥rC¥rC¤q?¥r@¥r@¤q?¥rC¤qB¤qB¢o@¢o@¢o@¢o@¢o@¢o=¢o=¢o=¡n<¡n<¡n< m; m; m; m; m> m>Ÿl=Ÿl=žk<žk<žk<žk<žk<žku?»Y ·UÅUÕeãzí„þÿ˜ÿ¢ÿ®ÿ¸ÿ¼ÿÆÿÌ ÿÑÿÔÿÕÿÕÿÓÿÏÿË ÿÆÿÆÿÈÿÍÿÒÿÖÿÎ ÿÇ ÿ¿ÿŸì{ètëwñÿÿ§ÿ¡ÿŸÿžö—'Þ³[šB•F§XºxAÌŠSÆVÆV¾ŽU¿VÁVÂWÃWÃWÃWÃWÇŽYÇŽYÊ`ÆŒ\µ{Iœb0z@^$OH FFH -H -DDFM gkfb]^bhŽ2«O À]Òoïˆÿž$ÿªÿ´"ÿº ÿ¹ ÿÁ -ÿÈÿÐÿÔÿÕÿÖÿ×ÿØÿÚÿÜÿÜÿßÿÉ&ÿ¥÷‚ åpÌS´;oREQ -MIDC= : -445523300//0/...4; -Ap2¤iC—\62‘3Ç>Ã:Ï;Ò>Ñ<Ò=Ó=Ô>ÜCáHÚKØIÁNÁN½N¿PÃQÆTÆ]ÁX¹M»O¶Z®RšC9‰2„-¬,âbÝc"¶<¦RÈt?½O»MÉ[Å}WÓg>¹M$ä' -Úïòÿ ÿ+ÿ+ÿ) ÿ( -ÿ&ÿ%ÿ%ÿ#ÿ( ×DèU*¹p;ÀwB®}I³‚N¸~N¸~N¸~N¸~N¯N¬~K®}K°M°}K°}K°}K¯|J¯|J¯|J¯|J¯|J®{G®{G®{G®{G­zF­zF­zF­zF®yH®yH®yH®yH®yH®yH®yH®yH­xG­xG­xG­xG¬wF¬wF¬wF¬wF¬wF¬wFªuDªuDªuDªuD¨uC¨uC¨uC¨uC¨uC¨uC§tB§tB§tB§tB§tB¦sA¦sD¦sD¥r@¦sA¥r@¦sA¥rC¤qB¤qB¤qB¤qB¥rC¤qB¢o@¢o=¢o=¢o=¢o=¡n<¡n<¡n<¡n< m; m; m> m> m>Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=žk<žk<žk<žk<žkÿèLÿ×Sû’àMÎ;Â3¼-©/˜ž !!åiÿ®ÿ¿ÿÜ0ÿà4ÿØ ÿÚ"ÿÙÿØ ÿØ ÿØ ÿØÿ×ÿ× ÿÖ ÿÖ ÿÕ ÿÕ ÿÖ -ÿÔ ÿÕ ÿá!æš8%$|@›_/¹U½YÜkéxïüŽÿ˜ÿ£ÿ¯ÿ³ÿ¿ÿÃÿÌ ÿÒÿÕÿÕÿÕÿÓÿÎÿËÿËÿËÿÉ ÿÎÿÔ -ÿÐÿÉÿ¿ÿ·ÿ·ÿ¶ÿ® ÿ¢ ÿ­ÿ½/ÿ¾0ÿ¶Nó¡9Ά5«cŒH K¢g2ºJº…W¶S³‚P¸‡U¹ˆVº‰W¾ŒUÀŽWÂŽXÂŽXÅWÅWÆZÆZÉ[É[À‡R­t?T'n2QD CCEEDCQ[kh[ZYZdj‘2£Dœ>¡C¼VÖpø’&ÿ¥9ÿ¹3ÿÄ>ÿÆ2ÿÎ:ÿÙ:ÿÙ:ÿÛ1ÿÛ1ÿà5ÿà5ÿÞ6ÿÚ2ÿ×4ÿ×4ÿÑGÿ¨çrÈSŽ#k9=18<;;:@ ? = -< ;;887767<;<<;;8= -EHC/XŽ7 µ<«2°3´7µ8¶9º:»;ÃBÆEÃKÆNµO·Q¹O¸N¿M¾LÉTËVºL¸JºT·Q›D‘:‰7ƒ1¡Nõ¢+ÿË(ÿ° ãÔÂz"Ã{#Â}KÇ‚PÀ„T°tDÆL ®4çèÿ ÿ+ÿ*ÿ( ÿ%ÿ"ÿ$ÿ&ÿ#ÿ%ÿ$ÿ/óF!ÿT/ÑqA×wG¹€K¹€K´Lµ€M¼~O»}N¶Nµ~M°}K°}K°}K¯|J°}K°}K°}K°}K¯|H¯|H¯|H®{G®{G®{G®yF¯zG¯zI¯zI°yH±zI¯zI¯zI®yH®yH¯zI¯zI­xG­xG­xG­xG­xG­xG­xG¬wF¬wF¬wF¬wFªuDªuDªuDªuB©tA©tC©tC¨uC¨uC¨uC¨uC§tB§tB¦sA¦sA¥r@¦sA¦sA¦sA¦sA¥r@¥rC¦sD¦sD¥rC¥rC¥rC¤q?¤q?¢o=¢o=¢o=¢o=¡n<¡n<¡n<¡n<¡n?¡n? m> m> m> m>Ÿl=Ÿl=Ÿl= m>Ÿl=Ÿl=žk<žk­B ƒ;=2+/6 -5 :ABAA@ A> > ; -98: ? -A B B ? -? -< < B9 -2k•>®5©0­0¯2¯2±4·7¸8»:½<»CÂJ·Q»U¾T¸N¾L¾LÄOÊU»M·IºT¸R›D”=Ž<Œ:»hÿ²;ÿà ÿÄ!ÿÇ9ÿ¯!Ý•=Ã{#À{IÂ}KÁ…UÀ„TÛa5»Aççÿÿ(ÿ) ÿ' ÿ$ÿ"ÿ&ÿ(ÿ&ÿ&ÿ'ÿ$ä7òE Äd4Ïo?µ|G·~Iµ€Mµ€M¼~O¼~O¶N¶N²M²M²M°}K²M²M²M²M²K°}I°}I¯|H¯|H®{G°{H°{H°{J¯zI²{J²{J¯zI¯zI¯zI¯zI¯zI¯zI¯zI¯zI®yH®yH®yH®yH­xG­xG­xG­xG¬wF¬wF¬wF¬wFªuBªuBªuDªuD©vD¨uC¨uC¨uC¨uC§tB¦sA¦sA¥r@¦sA¦sA¦sA¦sA¦sA¦sD¦sD¦sD¦sD¦sD¦sD¥r@¤q?¤q?¤q?¤q?¤q?¢o=¢o=¢o=¢o=¢o@¡n?¡n?¡n? m> m>Ÿl= m>¡n? m> m> m>Ÿl=Ÿl=žk<žk¥r>¥r>¤q=¤q?¤q?¤q?¢o=¢o=¢o=¡n<¡n<¡n< m;¡n?¡n?¡n?¡n? m> m> m>Ÿl=Ÿl=Ÿl=žk<žk] JDD AS -`ie^]YZVWdk @®N ·XÙz -ú–ÿ³2ÿ¾7ÿÂ;ÿÂ:ÿ¹1ÿ¶'ÿ³$ÿ¸'ÿº)ÿ»(ÿ¿,ÿÂ(ÿÀ&ÿÂÿÈÿË%ÿÔ.ÿÓ8ÿÐ5ÿ­)í{É`Š!/6-,*)++,++-034476463= MK? 4+*+'Ec#œ,Ÿ/°-°-®,®,²,´.¹0½4º2º2µ:ÁFÄZ¾T²C³D½EÁIÄJÅKÃKÁI½J»H°J®HÄ\ Ñiî‹ÿ±$ÿÅÿ¾ ÿ»ÿ¾ÿ¿ÿÁÿÃ;÷¨ Ì8¾*ÌyJ¾k<ç9Ô&öøÿÿ!ÿ' -ÿ& ÿ%ÿ&ÿ%ÿ%ÿ&ÿ( ÿ' -ÿ#ò1ù8ÇY*×i:Á|FÃ~H½O»M·€O¶N´N´N³€N³€N³€N³€N³€N²M²K²K°}I°}I°}I¯|H±|I±|I±|K°{J³|K³|K±|K±|K±|K°{J°{J°{J°{J°{J°{J°{J¯zI¯zI¯zI®yH®yH®yH®yH­xG­xG­xG­xG¬wF¬wF¬wF«xF«xF«xF«xF©vD©vD¨uC§tB§tB¨uC¦sA¦sA¨sB§rA¦q@§rA¨sB§rA§rA§rA¦s?¦s?¦s?¥r>¥r>¥r>¥r@¥r@¤q?¤q?¢o=¢o=¢o=¡n<¡n< m;¡n?¡n?¡n?¡n?¡n?¡n? m> m> m>Ÿl=Ÿl=Ÿl=Ÿl?žk>žk<žk<žk r=£u@§vD¬{I¬yL™f9{E"[%ID CG `ghd[Z\]Z[bi¢F¨L¼VÐjç~ú‘ ÿ¡ÿ³ ÿÂÿËÿÕÿÛÿå"ÿè%ÿç%ÿå#ÿæ#ÿæ#ÿæÿåÿä"ÿáÿÛÿÑÿ¸î‘Ðc ‰.6*).24 2/-**(())+)*+,6 @</(-/<Bš0®+«(²,²,µ(µ(µ+¹/À2Ã5º4À:½IÒ^Úr¾V©:°AÂJÅMÄOÃNÁLÁLÄLÃK®C®C¹GÁOÛ^õx ÿÿµ)ÿ¿ÿ¹ÿ¸ÿ¿ ÿÁ ÿÆÿ»ÿªë• Èr¿7±)êìûýÿÿ" ÿ&ÿ$ÿ$ÿ%ÿ& ÿ' -ÿ& -ÿ% ÿ!ÿ!ÿ3ÿA#ä`1ìh9ÇyEÉ{G¹O¹O´Nµ€O´O³€N³€N³€N´Q´Q´L³~K³~K³~K±|I±|I±|I³~K³|K³|K³|K³|K±|K±|K±|K±|K±|K±|K±|K°{J°{J°{J¯zI¯zI¯zI¯zI¯zI®yH®yH®yH®yH­xG­xG­xG­xG­xG¬yG«xF«xF«xF©vD©vD©vD¨uC¨uA¨uA¨uC¨uC¨sB¨sB¨sB¨sB¨sB§rA§rA¨sB§r?§r?¦s?¦s?¥r@¥r@¥r@¥r@¤q?¤q?¤q?¤q?¢o=¢o=¢o=¢o=¢o@¢o@¡n?¡n?¢o=¡n<¡n? m> m> m>Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=žk<žk<žk<žk_2ƒV-}P'wJ'm@e;[1N,H&55B/§Vÿ¶EÿÍÿÚ)ÿáÿØÿÜÿê)ÿÃ#ð…ÿ• -ÿ©ÿží†ï‚úÿ‹ÿ†û~õxðzÿ“ÿ°ÿ·ÿÒ%ÿë>ÿàÿÖÿÖÿÕÿÔ ÿÔ ÿÖ -ÿÖ -ÿÖ ÿÖ ÿÖ ÿÖ ÿÕ ÿÔ ÿÒ ÿÏ ÿÎ ÿ㾄=7r.Y¹X²QÊ[Úkìyù†ÿ•ÿžÿ¨ÿ°ÿºÿÂÿÊ ÿÍÿÍÿÉ ÿÍÿÒÿ´+žS2 - -  %,37 <#A(F+ L1X4\8i@pG#uK!zP&S#…Y)‰[&Ž`+—f4l:Ÿl?–c6I&c- -OF G Pedfc[Y\][\dn£G¬P¼VÓmç~õŒÿ¡ÿªÿ·ÿÁÿÇÿÎ ÿØÿØÿÙÿØÿØÿ×ÿ× ÿ× ÿÖÿÖÿØÿÔÿÃÿ£Ôg‚,1)(.28 6 6 -4331/0/-,+--36 -4025 7 G X—-ž4«(«(®(°*¶)¶)µ+¸.À2Æ8ÊDÜVânÿ‹"ý•+ÁY§8°A¾FÃKÇRÎYÏZÓ^ÝeçoÐeÈ]ÄR½KÆIÈKËcëƒÿ¦ÿ·ÿÂÿÀÿ¼ÿ»ÿ¾ÿÂÿÃ:ú¤æ^µ-Õàûýÿÿÿ"ÿ"ÿ!ÿ%ÿ' -ÿ( ÿ' ÿ' ÿ$ ÿõ( -û.ÑMà\-ÃuAÈzF¸~N¹O´Nµ€O´O´O´O´Oµ€Rµ€R´L´L´L´L´L´L³~K³~K¶N¶Nµ~Mµ~M³~M±|K³~M³~M±|K±|K±|K±|K±|K°{J°{J°{J°{J°{J¯zI¯zI¯zI¯zI®yH®yH®yH®yH®yH®yH¬yG¬yG¬yG¬yG«xF«xF©vD¨uC©vB§t@¨uC¨uC¨sB©tC¨sB©tC©tC©tC©tC¨sB¨s@§r?¦s?¦s?¦sA¦sA¥r@¥r@¥r@¥r@¥r@¤q?¤q?¤q?¢o=¢o=¢o@¢o@¢o@¢o@¢o=¡n<¡n?¡n? m> m> m> m> m>Ÿl=Ÿl=Ÿl=Ÿl=žk<žk<žk<žk>j=j=j=™h;™h;™h;™h;˜g:˜g:˜g:˜g:˜g:˜g:˜g:—f9˜g:—f9—f9—f9–e8–e8•d9•d9•d9•d9”c6”c6’d6b4b2b2a3a3a5a5`7`7Ž_6Ž_6Ž_6Ž_6_3_3_3_3_3_3_1Œ^0Œ^0Œ^0‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1ˆY0µ€Mµ€Mµ€Mµ€Mµ€M¶N¶N¶N¶N¶Nµ‚Nµ‚N¶„M¶„M¶„M¶„M¶„M¶„M·ƒM·ƒM·ƒM¸„N¸„N¸„N¸„N¸„N¸„N¸„N¸„N¸„Nº†Pº†Pº†P»‡Q»‡Q»‡Q»‡Q»‡Q»‡O¼ˆP¼ˆP¼ˆP¼ˆP¼ˆP¼ˆP¼ˆP½‰Q½‰Q¾ŠR¾ŠR¾ŠR¾ŠR½‰Q¾ŠR¿‹S¿‹S¾ŠR¾ŠR¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁUÁUÁUÁUÁUÂŽVÂŽVÁUÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÄXÄXÄXÄXÄXÄXÄXÄXÄXÄXÅ‘YÅ‘YÅ‘YÄ’YÄ’YÄ’YÄ’YÄ’YÄ’YÅ‘WÆ’XÊ‘VÈTÈTÈTÆ‘RÅQÅŒSÅŒSÁU½‰Q»†S·‚O±|I®yF¨uF¤qBškB“d;ƒZ6wN*lG'e@ Z; Q2J.B&5,  ;ºlÿÀÿÓ(ÿÛÿÕÿÉÿÆÿ©ÿžÿ²ÿ¶ÿ¥ÿžÿšÿŽõmä\ÔIÊ?Â:ÑIît ÿ›3ÿ¼ÿÖ(ÿë8ÿØ%ÿÕ ÿÕ ÿØÿØÿÖ -ÿÕ ÿÖ ÿÖ ÿÖ ÿÕ ÿÔ ÿÕÿ×ÿÔÿÏ ÿÝÿÓ2ƒ?!#"S9"ŸYœV³OÀ\Ýlêy ÷Šÿ•ÿŸÿ§ÿ°ÿ· ÿÁÿÉÿÎÿÏÿÑKÿÁ;‹a7& -  -  ' .2 7&<+F3K8N:UA$aF&cH(mK-tR4{U0{U0tG$^1PF\dd^WVRQ\]\]is£@°M¿WÕmíüŽ ÿŸÿ©ÿµÿ½ÿÇÿÎÿÖÿÙÿÛÿÛÿÙÿØÿÕÿÖÿ×ÿØÿÙÿ×ÿÊ ÿ¶ßt†47%%-0&)/014 6 6 687 5 3222264 5 -98oŠ+£+¢*­&®'³%´&º(»)Æ-É0Å@ØSå~ÿ™ÿ«ÿÀÿ¬'¾X¼JÁOÇPÑZÙgëyýÿ¡ÿ¢ÿ°ÿ¢ô•ÿ˜ç€ÔfÅWÃNÏZânú† -ÿ§ÿº%ÿÀÿ¼ÿ» ÿ½ ÿÃÿÉÿÁþŸÓ`©6¿Äëó% ÿ ÿ ÿ"ÿ&ÿ&ÿ&ÿ& ÿ& ÿ% ÿ" ÿ"ÿ# ÿ) ÿ2ñU,ýa8Éu@ÌxC¹€K¹€K³‚S³‚Sµ‚Wµ‚W¹T¹T´M´M´M´M´M´M´L´L¶L¶L¶Lµ~K³~K±|I³~K³~K³~K³~K³~K±|I±|I±|I°{H°{H°{J°{J°{J°{J°{J¯zI¯zI¯zI¯zI®yH®yH®yH®yH®yH®yH­xG¬wF¬wF¬wF©tCªuBªuB©tA©tAªuDªuD¨sB©tC©tC©tC©tC©tC©tA¨s@§t@§t@§t@¦s?¦s?¦s?¦sA¦sA¥r@¥r@¤q?¤q?¤q?¢o=¤q?¤q?¤q?¢o=¢o=¢o=¢o=¡n<¡n<¡n<¡n< m; m; m; m> m>Ÿl=Ÿl=Ÿl=žk<žk>žk>j=j=›j=™h;™h;™h;™h;˜g:™h;˜g:˜g:˜g:˜g:—f9˜g:˜g:—f9—f9—f9–e8–e:–e:•d9•d9•d9”c8’d8b6b6b6a8a8a8`7a3a3Ž`2Ž`2_1Ž`2Ž`2_1_1Œ^0_1_1_1Œ^0Œ^0Œ^0‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1‰Z1ˆY0µ€Mµ€Mµ€M¶N¶N¶N¶N¶N·‚O·‚O¶ƒO¶ƒO¶„M¶„M¶„M¶„M¶„M¶„M¸„N¸„N¸„N¸„Nº†Pº†Pº†Pº†P¸„Nº†Pº†Pº†Pº†Pº†Pº†P»‡Q»‡Q»‡Q»‡Q»‡Q¼ˆP¼ˆP¼ˆP¼ˆP¼ˆP¼ˆP¼ˆP½‰Q½‰Q½‰Q½‰Q¾ŠR½‰Q¾ŠR¾ŠR¾ŠR¿‹S¿‹S¿‹S¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁUÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÁUÁUÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÃWÃWÄXÄXÄXÄXÄXÄXÄXÄXÅ‘YÄXÅ‘YÅ‘YÅ‘YÅ‘YÄ’YÄ’YÄ’YÄ’YÅ“ZÅ“ZÆ’XÆ’XÊ‘VÉUÈTÈTÆ‘RÅQÅŒSÀ‡N»‡O¶‚J±|I¬wD¦q>¡l9šg8’_0†W.P'pG#e<[6T/G( >6.#  g=ÿµ -ÿÅÿÝÿÞÿ» -îŽøŠÿ§ÿ¿ -ÿ»ÿ½ ÿ½ ÿž ÷|ëcÛSÐEÉ>½5¹1²8×]ÿ¬ÿÈÿê7ÿå2ÿÛÿ× ÿ×ÿØÿ× ÿÖ -ÿÖ ÿÕ ÿÕ ÿÖ ÿÕÿÕÿÖÿÕÿÖÿÐ -ÿâAç£2"  % ~8 Z¸T´PÎ]Ýlê}÷Šü–ÿ¡ÿªÿ±ÿ¶ÿ¼ÿÈ ÿÙÿÄ>³d& -  $)27#:& E* -I.S1Z8d>jDl?Y, MJdd`[UURP\]\^jw#£@¯L¿WÕmíüŽ ÿžÿ¨ÿµÿ½ÿÆÿÐÿÖÿÚÿÝÿÝÿÛÿÙÿÙÿÚÿÜÿÜÿÛÿÛÿÎÿ¿ñ†";>+&+)#$&(***,*+0/.--+--+*,: „%’3§/¤,®'¯(³%´&¹'½+Ì3ÜCðkÿ’ÿ¶ÿ¹ÿÀÿÉ ÿ¨#Ïiætð~öü…ÿ“ÿ¦ÿÈÿÁ ÿº ÿÂÿºÿ± ÿ¾%ÿ¶ÿœñƒÝhÐ[Ð\Ó_ßoþŽÿªÿºÿ¿ ÿ½ ÿ¼ ÿºÿÂÿÌ'ÿÂ;ÿžøQ -Ë$Ù âþÿÿÿ%ÿ&ÿ&ÿ& ÿ& ÿ% ÿ$ ÿ% ÿ# þ%þ%Ø<éM$¿k6Ær=¹€K»‚M³‚S´ƒTµ‚Wµ‚Wº€Uº€Uµ‚Nµ‚Nµ‚Nµ‚N´Mµ‚Nµ€Mµ€M·€M·€M¶L¶L´L´L´L´L´L³~K³~K³~K±|I±|I±|I±|I±|K±|K°{J°{J°{J°{J°{J¯zI¯zI¯zI¯zI¯zI¯zI®yH®yH®yH­xG­xG¬wF¬wF¬wDªuBªuB©tAªuDªuDªuD©tCªuDªuD©tC©tC©tA©tA¨uA§t@§t@§t@§t@§t@§tB¦sA¦sA¦sA¥r@¥r@¥r@¤q?¤q?¤q?¤q?¤q?¢o=¢o=¢o=¢o=¡n<¡n<¡n<¡n<¡n< m; m> m> m>Ÿl=Ÿl=Ÿl=Ÿl?žk>žk>žk>™h;™h;™h;™h;™h;™h;™h;™h;™h;˜g:˜g:™h;˜g:˜g:˜g:˜g:—f9—f9—f;–e:–e:–e:•d9•d9’d8’d8’d8’d8a8a8a8a8b4b4a3a3a3a3Ž`2Ž`2Ž`2Ž`2Ž`2_1_1Œ^0Œ^0Œ^0‹\3‹\3‹\3‹\3‰Z1‰Z1‰Z1‰Z1µ€Mµ€M¶N¶N¶N·‚O·‚O·‚O·‚O·‚O¶ƒO¶ƒO¶„M¶„M¶„M¶„M¶„M·…N¸„N¸„N¸„Nº†Pº†Pº†Pº†P»‡Qº†Pº†Pº†P»‡Q»‡Q»‡Q¼ˆR»‡Q»‡Q»‡Q¼ˆP¼ˆP¼ˆP¼ˆP¼ˆP½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q¾ŠR¿‹S¾ŠR¿‹S¿‹S¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁUÁUÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÃWÃWÃWÃWÄXÄXÃWÃUÄVÄVÄVÄVÄVÄXÄXÅ‘YÅ‘YÅ‘YÆ’ZÆ’ZÅ‘YÅ‘YÅ‘YÄ’YÄ’YÄ’YÄ’YÅ‘YÆ’ZÆ’ZÆ’ZÊ‘VÉUÅUÄŽT¾ŠP½‰Oº…R´L¯|J¬yG¥tIžmB•f=^5W3xN*kD'e>!V7M.B(;! - $  - ¾yÿ¿dÿÏÿÙ&ÿ²ãƒû—ÿºÿËÿÇÿÃÿºÿîlæSÝJÓBÍ<Í=É9·/²*Ö\ÿ¢FÿÚ7ÿéFÿÝÿ×ÿÖÿÖÿ×ÿÖÿÖ -ÿÕ ÿÕ ÿÖ ÿÖ ÿÖ ÿÕ ÿÖ -ÿÖÿÓ ÿÖÿà—V6 -*cG.¦]žU¹PÆ]Ùnæ{õ‹ÿ˜ÿ£ÿ« -ÿªÿ®ÿÃAÿ¹7™pC' -   $ ,1"5&;,H(B"XWjgdc\\UUZ[]aky% DªNÄVÙkð~ÿ ÿžÿ¨ÿ´ÿ¼ÿÇ ÿÎÿÖÿÚÿÜÿÜÿØÿØÿØÿØÿÙÿÙÿÛÿÙÿÏÿÉÿ—˜/DG -7/440,*--,+*#$$#'&'(-*Rx#¬7§2®*­)´'µ(»(¼)½4ÐGÞdÿ†ÿ²ÿËÿÆÿÄ ÿÇÿÍÿ§ðÿ¥ÿ­ÿ¯ÿ²ÿ¶ÿÁ -ÿÔÿÊ ÿÅ ÿÊÿ¼ÿ°ÿºÿÃÿÅÿ¹ÿ£òˆÜkËZËSÕ]êsÿŒÿ¤ÿ´ÿ½ÿ¸ÿ¸ÿ»ÿ¿ ÿÌÿÂõ›ÖQ³.ßè'þ ÿ!ÿ"ÿ"ÿ$ÿ$ÿ& ÿ& ÿ"ÿ ÿ!ÿ!ÿ!ÿ( ðCÿT-Ûo@áuFÁL€M·‚Q¶P´Tµ‚U³ƒL³ƒL´Mµ‚Nµ‚Nµ‚Nµ‚Nµ‚N¶Nµ€Mµ€Mµ€M´L´L´L´L´L´L´L³~K³~K³~K±|I±|I±|I±|I±|I±|I±|K°{J°{J°{J°{J¯zI¯zI¯zI¯zI®yH®yH®yH®yH­xG­xG¬wF¬wD¬wDªuB¬wDªuBªuB¬wD¬wDªuDªuDªuDªuD©tA©tA¨s@©tA¨uA§t@§t@§t@§t@¦s?¦sA¦sA¦sA¥r@¥r@¥r@¥r@¤q?¤q?¢o=¢o=¢o=¢o=¢o=¢o=¢o=¡n<¡n<¡n<¡n<¡n? m> m> m>Ÿl=Ÿl=Ÿl?Ÿl?žk>žk>›j=™h;›j=›j=™h;™h;™h;™h;™h;™h;™h;™h;™h;™h;˜g:˜g:˜g:—f9—f9—f9–e:–e:–e:–e:“e9’d8’d8’d8’c:’c:a8a8b4b4b4a3a3a3a3Ž`2Ž`2Ž`2Ž`2Ž`2Ž`2_1_1Œ^0Œ]4Œ]4‹\3‹\3‹\3‹\3‹\3‹\3µ€Mµ€M¶N¶N¶N·‚O·‚O·‚O·‚O·‚O¶ƒO·„P·…N·…N·…N·…N·…N·…Nº†Pº†Pº†Pº†P»‡Q»‡Q»‡Q»‡Qº†P»‡Q»‡Q»‡Q»‡Q»‡Q¼ˆR¼ˆR¼ˆR¼ˆR¼ˆP¼ˆP¼ˆP¼ˆP½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q¾ŠR¾ŠR¿‹S¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁU¿‹SÁUÁUÁUÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÃWÃWÃWÃWÃWÃWÃWÄXÄXÄVÄVÄVÅ‘WÅ‘WÅ‘WÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÅ“ZÄ’YÄ’YÄ’YÄXÅ‘YÄXÂŽVÆRÊO¾ˆNº„J¶‚H³E­xE¨s@¤q?Ÿl:•d9‹Z/‚S*yJ!mCe;[4U.E&:/&  i$ë¦KÿÃÿÇÿ°ÿ£ÿ´ÿÌÿÒ -ÿËÿÆ ÿ¸ÿˆçeâOÛHÒAÐ?Ì<É9Æ>¹1¤*ße ÿÇ$ÿã@ÿà"ÿÖÿÖÿÖÿÖÿÖÿÖ -ÿÕ ÿÕ ÿÕ ÿÕ ÿÖ ÿÖ -ÿÕ ÿÕÿÖÿÎÿßó²NY -1ŽE£Z¸OºQÍbÙnè~õ‹ü–ÿ£ÿªÿ­ÿŸµU/ -  ".8XXhffe__WVZ[`bm})£GªNÅWÙkñÿŽ ÿžÿ¨ÿµÿ¼ÿÆÿÍÿÖÿÛÿÝÿÜÿØÿÖÿ×ÿ×ÿØÿÙÿÙÿØÿÏÿÐÿ¦(˜/DM>7= <9724642/#"$$$&+,,//2r“>©4 +¬(®*´'µ(Â/Ð=ßVÿwÿ¡ÿºÿËÿÍÿÍÿÉÿÁÿÂÿ¬ÿ­ÿºÿ¾ÿ¾ÿ¾ÿÁ -ÿÈÿÒÿÇ ÿËÿÇ ÿ¼ÿ¹ÿ¿ ÿÀÿÀÿÃÿÄÿ¹ÿ™ ëzàhÕ]Ô]Üeêzÿ‘ÿ¨ÿµÿ¼ÿ»ÿ´ÿ±ÿºÿÀÿ¨?ÿ‹"ÿBá û þ ÿ ÿ ÿ$ÿ%ÿ& ÿ% ÿ"ÿ ÿ#ÿ#ÿ%ÿ!Ý0 ïBË_0Ún?ÁL€M·‚Q¶Pµ‚Uµ‚U³ƒL³ƒLµ‚Nµ‚Nµ‚Nµ‚N´Mµ‚N¶N¶N¶N¶Nµ€Mµ€Mµ€M´L´L´L´L´L´L³~K³~K³~K³~K³~K±|I±|I±|K±|K±|K±|K°{J°{J°{J°{J¯zI¯zI¯zI¯zI®yH®yH­xG­xG­xE¬wDªuB¬wD­xE¬wD¬wD¬wD¬wF¬wFªuDªuDªuBªuB©tA©tA¨uA¨uA¨uA¨uA§t@§t@§tB¦sA¦sA¥r@¥r@¤q?¥r@¥r@¥r@¥r@¤q?¤q?¢o=¢o=¢o=¢o=¢o=¡n<¡n<¡n<¡n?¡n?¡n? m> m> m>Ÿl? m@Ÿl?žk>l?œk>œk>œk>œk>›j=›j=›j=™h;™h;™h;™h;™h;™h;˜g:˜g:˜g:—f9—f9—f9—f;—f;–e:–e:”f:“e9“e9’d8’c:a8’c:’c:b4b4b4a3a3a3a3a3a3a3Ž`2Ž`2Ž`2_1_1_1Œ]4Œ]4Œ]4Œ]4‹\3‹\3‹\3‹\3·‚O·‚O¶N·‚O¸ƒP·‚O¸ƒP¸ƒP¸ƒP¸ƒP·…N·…N·…N·…N·…N·…N·…N¹‡Pº†Pº†Pº†P»‡Q»‡Q»‡Q»‡Q¼ˆR»‡Q¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR½‰S¼ˆR¼ˆP¼ˆP½‰Q¼ˆP¼ˆP¼ˆP½‰Q½‰Q¼ˆP½‰Q½‰Q½‰Q¾ŠR¿‹S¿‹S¿‹S¿‹SÁU¿‹SÁUÁUÁU¿‹SÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÃWÄXÄXÄXÄXÄXÄXÄXÄXÄXÄXÄXÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÆ’ZÆ’ZÅ‘YÆ’ZÆ’ZÅ‘YÅ‘YÅ‘YÅ‘YÄXÃWÃWÂŽV¾ŠR»‡Oº„N·K´L±|I­zH¨uCoC–h<Žd<…[3{R0rI'gB'_:U5N.@$8 ) -ŠgPÿ³)ÿ¯%ÿ´ÿ·ÿÅÿÔÿÕÿÎÿÍ -ÿÃÿ†å`åNÞGÔ>Ð:Ì<È8À9Å>³/îÿÐTÿä0ÿÚ&ÿÖÿÕÿ× ÿÖ -ÿÖ -ÿÕ ÿÕ ÿÖ ÿÕ ÿÖ ÿÖÿÖÿÔÿÕÿÔ ÿÏÿà6Áy%/ -AvP-žT V¸] ¿dÛsè€îˆý—êŸHÅz#W7%6I ƒƒ¡ ¥ ³´³±   -š}~qrr‚)©D±LÄVÚlð~ÿŽ ÿžÿ©ÿ·ÿ¼ÿÆÿÎÿ×ÿÚÿÜÿÜÿÙÿ×ÿÕÿÕÿ×ÿÙÿÙÿ×ÿÎÿÕ ÿ³1‡--7- 0FEADBBDCCC==A@AB? ? DCSg±3¸:´+´+µ+´*·0Á:ÑWóyÿœÿ±ÿÅÿÑÿÒÿÏÿÅ ÿ·ÿ¯ÿ½ÿ¸ÿ¾ -ÿÁÿÁÿÁÿÁÿÌÿÛÿÕ%ÿÆÿÉÿÆÿºÿÄ ÿÊÿÄÿÀÿ½ ÿ¼ÿÀÿÀÿ³ÿ›îƒáoÛiàcâeúxÿ‹ÿ¡ÿ­ÿ±ÿ«ÿ«ÿ´å¨[¸{.v?:Laiš©*÷%ý+ÿ#ÿ$ÿ'ÿ&ÿ$ÿ$ÿ%ÿ#ÿ!ÿ(ô8ÿI%áj=ëtGÇ€NÅ~Lº€Nº€N¶‚L¶‚L¶ƒO·„PµƒL¶„M¶„MµƒL¶Nµ€M¶N¶Nµ€Mµ€Mµ€Mµ€Mµ€Mµ€Mµ€M´L´L´L´L´L³~K³~K³~K³~K³~M³~M³~M±|K±|K±|K±|K±|K±|K°{J°{J°{J¯zI®yH®yH®yH­xE­xE­xE­xE­xE¬wD¬wD¬wD¬wF¬wF¬wF¬wFªuBªuB©tA©tA©tA©tA¨uA¨uA¨uA§t@§t@§t@§tB¦sA¦sA¦sA¦sA¥r@¥r@¦sA¥r@¥r@¥r@¢o=¢o=¢o=¤q?¢o=¢o=¢o=¡n?¡n?¡n?¡n? m> m> m@ m@ m@ m@Ÿl?Ÿl?žk>žk>žk>žk>›j=›j=›j=›j=›j=›j=j=›h;›h;›h;›h;šg:˜g:˜g:—f;—f;–e:–e:–e:•d9•d9•d9•c;•c;”b:”b:”c6’a4‘`3‘`3‘`3‘`3a3a3a3a3a5Ž`4Ž`2_1_1_1Œ]4Œ]4Œ]4Œ]4‹\3‹\3‹\3‹\3·‚O·‚O¶N·‚O¸ƒP·‚O¸ƒP¸ƒP¸ƒP¸ƒP·…N·…N·…N·…N·…N¹‡P¹‡P¹‡P»‡Q»‡Q»‡Q»‡Q»‡Q¼ˆR¼ˆR¼ˆR»‡Q¼ˆR¼ˆR¼ˆR½‰S½‰S½‰S½‰S½‰Q½‰Q½‰Q½‰Q½‰Q¾ŠR¾ŠR¾ŠR¾ŠR¾ŠR¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁUÁU¿‹SÁUÁUÁUÂŽVÂŽVÂŽVÁUÂŽVÂŽVÂŽVÃWÂŽVÂŽVÂŽVÃWÃWÃWÄXÄXÄXÄXÄXÄXÄXÄXÄXÄXÄXÅ‘YÅ‘YÄXÅ‘YÅ‘YÅ‘YÅ‘YÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÅ‘YÅ‘YÅ‘YÃWÄXÂŽVÁU¼ˆP·ƒK³GµI°zD¨s@£n;Ÿl:˜e3_3„V*zP(qGi@_6T/M( B" <1*  >ï• ÿ¶,ÿ¬ÿ»ÿÊ -ÿÔÿÖÿÐÿÑÿÈÿ‰á\äMÞGÔ>Ñ;Ì<Ë;»4Â;À<’¥Gÿ¾Bÿã/ÿá-ÿÓÿÕÿ× ÿÖ -ÿÕ ÿÕ ÿÕ ÿÖ ÿÕ ÿÖ ÿÖÿÖÿÖÿÔÿÕ -ÿÍÿÝ3ÿÁ`+&!K%‘G ¢X±V­RËcÛsá{×qžSa ;T‹²º"Ï*Ö1Ö1 Ò-»%±Švvq€'©D®IÅWÚlñÿŽ ÿžÿ©ÿ·ÿ¼ÿÆÿÎÿÖÿÙÿÛÿÛÿÙÿÖÿÔÿÕÿ×ÿØÿÙÿØÿÎÿÒÿ³1…+'+$+ >A >?>?BACCDB G I JJCCDA dƒ4¼>µ7·.·.²(¼2ÕNòk ÿÿ¤ÿ¸ÿÃÿÏÿÒÿÏÿÌ ÿ»ÿ±ÿ´ÿÅ ÿ¿ ÿ¿ ÿÁÿÂÿÂÿÃÿÏÿÝ ÿç7ÿÑ!ÿÒÿÓÿ½ÿÅ ÿÈÿÃÿÁÿ¿ ÿ»ÿ»ÿ¾ÿÀÿ»ÿ°ÿ¡ -ÿ‘ù|åhß]êhü‚ÿ›ÿ®ÿ°ÿ®Úƒk.2%$9=là÷%ÿ ÿ" ÿ'ÿ&ÿ$ÿ$ÿ%ÿ&ÿ$ ÿ# -é- ð4ÉR%Ýf9Â{IÅ~Lº€Nº€N¸„N·ƒM·„P·„P·…N¶„M·…N¶„M·‚O·‚O¶N¶N¶N¶N¶N¶N¶Nµ€Mµ€Mµ€Mµ€Mµ€M´L´L´L´L´L´L´N³~M³~M³~M³~M±|K±|K±|K±|K°{J°{J°{J¯zI¯zI¯zI®yH­xE­xE­xE­xE­xE­xE­xE®yF­xG¬wF­xG¬wF¬wDªuBªuBªuB©tA©tA§t@¨uA¨uA§t@§t@§t@§tB§tB§tB¦sA¦sA¥r@¦sA¦sA¥r@¥r@¦sA¥r@¤q?¤q?¤q?¢o=¢o=¢o=¢o@¢o@¡n?¡n?¡n? m> m@ m@ m@ m@ m@žk>žk>žk>Ÿl?žk>œk>œk>›j=›j=›j=›j=j=j=›h;›h;›h;šg:˜g:˜g:—f;—f;—f;—f;—f;–e:–e:•d9•c;”b:•c;•c;”c6”c6”c6”c6’a4‘`3a3a3a3a3a5Ž`4Ž`2Ž`2_1_1^5^5^5Œ]4Œ]4Œ]4Œ]4Œ]4·‚O·‚O·‚O·‚O¸ƒP¸ƒP¸ƒP¸ƒPº…Rº…Rº†Pº†Pº†Pº†Pº†Pº†Pº†P»‡Q»‡Q»‡Q»‡Q»‡Q»‡Q¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR½‰S½‰S¾ŠR¾ŠR¾ŠR¾ŠR¾ŠR¾ŠR¾ŠR¾ŠR¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁUÁU¿‹SÁUÁUÁUÂŽVÁUÂŽVÂŽVÂŽVÂŽVÂŽVÁUÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÃWÄXÄXÄXÄXÄXÄXÄXÄXÅ‘YÅ‘YÅ‘[Å‘[Ä’YÄ’YÄ’WÄ’WÅ‘YÆ’ZÆ’ZÆ’ZÆ’XÅ‘WÅ‘WÅ‘WÆ’XÆ’XÈ’XÈ’XÄ’YÁV½‹T»‰R¹†T´O®}N§vG§tE m>“e9Œ^2‚X0{Q)qK(hB_=#X6O0E&; 4 (!  ŒKð¯lÿ¯ÿ·ÿÏ ÿÙÿ× ÿÐÿÖÿÐ ÿ—ßdçJáDØ=Ö;Ï;Ï;Ä9À5ÅD´3‘ÿ7ÿÕ%ÿá1ÿÛ'ÿÖ"ÿØÿÕÿÔ ÿÔ ÿÕ ÿÕ ÿÕ ÿÕ ÿÔ ÿÕ ÿÕÿÕÿÓ ÿÔÿÔÿâ×B <%[D0ƒL ‹T(–`&“]#oH+I" 68cs¶ Äæîÿ(ÿ2 ÿ7ÿ8ÿ5ÿ/ ÿ) ÿ"ó笤zŠ#«D¯HÅXÝpï€þ ÿžÿ© ÿ¶ÿ½ ÿÇ ÿÎÿÖÿÙÿÛÿÛÿØÿÖÿÕÿÕÿ×ÿØÿ×ÿ×ÿÍÿÒ ÿ±5}(!# $340 -50&'"#$9 -; B B :/.3Œ¯>È<¼0¼0¾2ÈKåhÿ‹ÿ¦ÿ¸ÿ»ÿà ÿËÿÑÿÎÿÈÿÁ -ÿ³ÿ°ÿºÿÈÿ¿ÿ½ÿÁÿÂÿÅÿËÿÐÿ×ÿÜ&ÿÑÿÕÿÏÿ¿ÿÈ ÿÇ ÿÃÿÄÿÂÿ¿ÿ½ÿ¼ÿºÿ¼ÿÄ ÿË ÿÅÿ²ö˜íƒöŒÿŸÿªÿ· ÿ¸ Àz;d gŠ'ý!ÿ$ÿ"ÿ"ÿ$ÿ$ÿ" ÿ% ÿ$ ÿ$ ÿ$ ÿ#ì1û@#Ù^;énKÃ}NÄ~O¹ƒM¼†P·…N·…N¸„N¸„N¸„N¸„N¹‚OºƒP·‚O·‚O·‚O¶N¸N¸N¸N¸N¸N·€M·€M·€Mµ€M´L´L´L´L´L´N´N´N´N´N³~M³~M³~M±|K±|K³|K³|K°{J°{J±zI°yH±zG°yF°yF±zG¯zG®yF®yF®yF­xG­xG­xE¬wDªuBªuBªuDªuDªuDªuD©vD¨uC¨uC¨uC¨uC¨uC¨uC§tB§tB§tB§tB§tB¦sD¦sD¦sA¦sA¦sA¦sA¦sA¦sA¥r@¤q?¤q?¢o=¢o@¢o@¢o@¢o@¢o@¡n?¡n? m> m>¡n? m> m> m> m>Ÿl=Ÿl=žk<žk<žk>žk>j=j=j=j=j=›h;›h;›h;šg:šg:šg<šg<šg<™f;™f;™f;™f;˜e:˜d<˜d<—c;—c;—d7—d7—d7–c6”c6”c6’a4’a4b6b6`7`7a3Ž`2Ž`2Ž`2^5^5^5^5^5^5Œ]4Œ]4·‚O·‚O·‚O·‚O¸ƒP¸ƒP¸ƒP¸ƒPº…Rº…Rº†Pº†Pº†Pº†Pº†P»‡Q»‡Q»‡Q»‡Q»‡Q»‡Q»‡Q»‡Q¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR½‰S½‰S¾ŠR¾ŠR¾ŠR¾ŠR¾ŠR¾ŠR¾ŠR¿‹S¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁUÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÃWÃWÄXÄXÃWÃWÃWÄXÄXÄXÄXÄXÄXÄXÅ‘YÅ‘YÅ‘YÅ‘YÅ‘[Å‘[Ä’YÄ’YÅ“XÄ’WÆ’ZÆ’ZÆ’ZÆ’ZÆ’XÅ‘WÅ‘WÆ’XÆ’XÆ’XÈ’XÅU½‹RºˆO´‚K³J°}K«xF¥tEl=˜e6].‚T(zL oEh>^8W1O-G% = -6' " -AɈEÿ¼ÿ´ÿÎ -ÿÚÿ× ÿÑÿÖÿÕÿ©ëpæIáDÜA×<Ð<Ð<È=È=ÂAÈG–áiÿÆÿÜ,ÿæ2ÿÕ!ÿÖÿ×ÿÖ ÿÔ ÿÔ ÿÔ ÿÔ -ÿÕ ÿÔ ÿÕ ÿÕÿÖÿÓ ÿÕÿÓÿÛ ÿÕdƒ;  GZ#h2R+  -DL ‚˜%Ï$Ò'û*ÿ9ÿ;ÿ?ÿ8ÿ7ÿ5ÿ3ÿ0ÿ0ÿ2ý' À*®{&©B¯HÆYÞqðþ ÿŸÿ© ÿ´ÿ¾ÿÉ ÿÐÿÖÿÙÿÛÿÛÿ×ÿÕÿÔÿÕÿÖÿØÿ×ÿ×ÿÍÿÓ ÿ°4y$$(!#2054,( 58 B @ 9 ')G§6´CÁ5¼0Å9ØLüÿ™ÿ® -ÿµÿÀ -ÿ ÿÉÿÌÿÏÿËÿÄ ÿ»ÿ®ÿ²ÿ¼ÿÅÿ¿ÿ¼ÿÀÿÁÿÆÿÌÿÒÿÓÿËÿÒÿÎÿÀÿÂÿÈ ÿÇ ÿÃÿÂÿÂÿÀÿ¿ÿ½ÿ»ÿºÿ¼ÿÆÿÑÿÊÿÀÿµ ÿÀÿÄÿÃÿÌ!÷•j$>T| -ùÿ#ÿ"ÿ& -ÿ' ÿ( -ÿ% ÿ& ÿ$ ÿ$ ÿ' ÿ' ç,æ+ÀE"Ø]:ÅPЊ[¿‰S½‡Q¹‡P¹‡P¸„N¸„N¸„N¸„NºƒPºƒP¸ƒP·‚O·‚O·‚O¸N¸N¸N¸N¸N¸N¸N·€M¶Nµ€Mµ€M´L´L´L´N´N´N´N´N³~M³~M³~M³~M³~Mµ~M³|K±|K±|K³|K±zI±zG²{H±zG±zG¯zG¯zG®yF®yF®yH®yH®yF­xE¬wD¬wD¬wF¬wF¬wF¬wF©vD©vD©vD¨uC¨uC¨uC¨uC§tB§tB§tB§tB§tB§tE§tE¦sA¦sA¦sA¦sA¦sA¦sA¦sA¥r@¥r@¤q?¤qB¢o@¢o@¢o@¢o@¢o@¢o@¢o@¡n?¡n?¡n? m> m> m> m>Ÿl=Ÿl=žk<žk>žk>žk>j=j=j=j=›h;›h;›h;šg:šg:šg<šg<šg<šg<™f;™f;™f;™f;˜d<˜d<˜d<˜d<—d7—d7—d7–c6”c6”c6’a4’a4b6b6a8`7a3a3Ž`2Ž`2Ž_6Ž_6Ž_6^5^5^5^5^5·‚O·‚O·‚O·‚O·‚O¸ƒPº…R¸ƒP¸ƒPº…Rº†Pº†Pº†Pº†Pº†Pº†P»‡Q»‡Q»‡Q»‡Q»‡Q¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR½‰S½‰S½‰S½‰S½‰S½‰S¾ŠR¾ŠR¾ŠR¾ŠR¾ŠR¾ŠR¾ŠR¿‹S¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁUÁUÁSÂŽTÂŽTÂŽTÂŽTÂŽTÂŽTÂŽTÂŽTÃUÃUÃUÄVÄVÄVÄVÑXÑXÑVÑVÅ‘WÅ‘WÅ‘WÅ‘WÄ’YÄ’YÆ’ZÆ’ZÅ‘YÆ’ZÆ’ZÆ’ZÅ‘WÅ‘WÆ’ZÆ’ZÈ’XÈ’XÇ‘WÇ‘WÅ‘WÅ‘WÅ‘WÄVÄVÄV¿‹S»‡O¸ƒP·‚O²P­zK¦tLŸmE’hB‰_9~X5uO,jE(c>!T5N/C);!5 *   M8&ÿ±*ÿ·0ÿÎÿáÿÓÿÎÿÖ ÿÛÿÇø“ëSßGÖ@Ó=Ò<Ò<Æ>Å=Ì<Î>¿9ã]ÿ³ ÿÅÿêAÿÝ4ÿÔÿÕÿÖÿÕÿÔÿÔÿÔ ÿÕ ÿÔ ÿÔ ÿÕ ÿÕ ÿÔÿÖÿÒÿÎÿÛó­8'  *1‚’ÅÓúÿ%ÿ&ÿ*ÿ1ÿ8ÿ8ÿ9ÿ8ÿ6ÿ2ÿ4ÿ6ÿ6ÿ8ÿ3ÿ-úµÃµA¹EÆYÝpïþÿžÿ¨ ÿ´ÿ½ ÿÉÿÐÿ×ÿÙÿÛÿÙÿ×ÿÕÿÕ ÿÕ ÿÕÿÕÿÖÿÕÿÌ ÿÑÿª6z$,0 ), 44/ &%!/4 = -@ - !r˜.Å?ÇAÊM×Zçzÿ”ÿ¢ÿµÿ¿ ÿ¿ ÿÂÿÇ ÿÌ ÿÏÿÏÿÌÿÉÿÀÿªÿ¯ÿ¸ÿÀÿ¾ -ÿºÿ¼ÿÀÿÅÿÍ ÿÓÿÒÿÓ ÿÒ ÿÉ ÿÃÿÆÿÇÿÈ ÿÅÿÃÿÁÿ¿ÿ½ÿ¼ÿ¼ÿÀÿ¾ÿÄÿÍÿÎÿË ÿÅ ÿÌÿÇÿËÿºRG-.Wh%}#‰/¾1"Ì?0î6*ì4(ò1#ñ0"·6"Ÿ -R A0 @[0†[F¼ˆbÁgº†P¸„N¼‚P¼‚P¼†L¼†Lº†P¸„NºƒPºƒP¹ƒM¹ƒM·ƒM·ƒM¹‚O¹‚O¹‚O¸N¸‚L¸‚L·K¸‚L·€M·€M·€M·€Mµ€Mµ€Mµ€M´L´L´L´L´L´L³~K³~K³~K³|I³|I³|I²{H²{H²{H²{H²{H°{H¯zG¯zG¯zG¯zG¯zG¯zG­xE­yC¬xB­xE­xE­xE­xE¬wD¬wDªuBªuB©tAªuBªuB©tA¨uA¨uA¨uC§tB§tB§tB§tB§tB§tB¦sA¦sA¦sA¦sA¦sA¥r@¥r@¥rC¤qB¤qB¤qB¢o@¢o@¢o@¢o@¢o@¡n?¡n?¡n?¡n? m> m> m>Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=žk<žk<žk[6T/D% >3+# -ÊtÿÃ<ÿÐÿÜ ÿÛÿÏÿÒÿÙÿÕ"ÿ¯ýeàHØBÕ?Ô>Ó=Æ>Ã;É9Ï?ÉCæ`ÿ¥ÿ¸ÿÚ1ÿç>ÿÚÿÔÿÖÿÖÿÕÿÔÿÕ ÿÕ ÿÔ ÿÔ ÿÔ ÿÔ ÿÔÿÕÿÒÿÓÿÑÿÞ!X%# 6F ±$ á!ë+ÿ+ ÿ, ÿ,ÿ1 ÿ1ÿ4ÿ5ÿ6ÿ6ÿ7ÿ5ÿ5ÿ7ÿ:ÿ;"ÿ:!ÿ;ÿ, ÄÈ"¶BºFÆYÚmð‚þÿžÿ¨ ÿ´ÿ¾ÿÉÿÐÿ×ÿÚÿÛÿÙÿ×ÿÖÿÕ ÿÕ ÿÔÿÕÿÖÿÕÿË ÿÑú¤0v -4 (, 42 .'#!/4 = -< '& -“)¯EÐJßY÷zÿ ÿ¢ ÿ®ÿ¶ÿ¼ÿ¾ÿÀ -ÿÅ ÿÊÿÍ ÿÎ ÿÍÿÍÿÏÿÄ ÿ«ÿªÿ²ÿ¿ ÿÁ ÿ¹ÿ»ÿÂÿË -ÿÒÿÓÿÒÿÔÿÐ -ÿÄÿÄÿÉÿÊÿÉ -ÿÊ ÿÈ ÿÄÿ¿ÿ½ÿ»ÿ»ÿÀÿÂÿÃÿÆ ÿÌ ÿÍ ÿÊÿÈÿÊÿÖÆ€Q -1 -3;MTŒ˜ ËÏ á â!’k','o;®zT¿‹U¸„N¿…S¿…S¼†L¼†Lº†Pº†PºƒP¼…R¼†P¼†Pº†P¸„N¹‚O¹‚O¹‚O¸N¸‚L¸‚L·K¸‚L¸N¸N¸N·€Mµ€Mµ€Mµ€M´L´L´L´L´L´L´L³~K³~Kµ~Kµ~K³|I³|I³|I²{H²{H²{H°{H°{H°{H¯zG¯zG¯zG®yF®yF®zD®zD­xE­xE­xE­xE¬wD­xE¬wDªuBªuBªuBªuBªuB©vB¨uA¨uC¨uC§tB§tB§tB§tB§tB§tB¦sA¦sA¦sA¥r@¥r@¥r@¥rC¤qB¤qB¤qB¤qB¤qB¤qB¢o@¢o@¢o@¢o@¢o@¢o@ m> m> m> m> m> m>Ÿl=Ÿl=Ÿl=žk<žk<žkÿÐ,ÿÕ ÿ× ÿ×ÿÖÿÕÿÕÿÔ ÿÔ ÿÓ -ÿÓ -ÿÓ ÿÔ ÿÓÿÕ -ÿÑ -ÿÞü»2T +˜¯ðú&ÿ$ÿ&ÿ) ÿ+ü/þ1ÿ3 ÿ7ÿ5ÿ7ÿ7ÿ8ÿ;ÿ;ÿ<ÿ=ÿ< ÿ< ÿ;ÿ9ú-õ(Ð>ÓAÏYâlð‚ÿ‘ ÿžÿ¨ÿ·ÿ¾ÿÆ ÿÍÿ×ÿÚÿÙÿÙÿÖÿÕÿÕ ÿÖ ÿÔÿÔÿÖÿÕÿÊ ÿÐï1m$/% -( 7 6 -1,%%$#04 9 9 2)EpËTãlò~ÿ•ÿ¨ÿ²ÿ¶ÿ¸ÿ¼ÿ½ÿÁÿÄÿË ÿÎÿÍÿÍÿÊ ÿÊ ÿÕÿË ÿ¬ÿ©ÿªÿ¹ÿÈÿà -ÿÈÿÒÿÕ ÿÔ ÿÓÿÕÿÒÿÊ -ÿÁÿÄÿÉÿËÿÍÿÌÿÉÿÆÿÇÿÅÿÅÿÈÿÃÿ¿ ÿ¾ÿÄ ÿÊÿÎÿÊÿÅ ÿÔ,ÿÃn:5+  -  ^?)ÁWÁWÃNŃP¸‡S·†R¹‡PºˆQ½†S½†S½†S¼…Rº…Rº…R·‚O¸ƒP¸ƒP¸ƒP¹ƒM¹ƒM¹ƒM¸‚L¸‚L·K¸‚L·Kµ€Mµ€Mµ€Mµ€Mµ€Mµ€M´L´L´L´L³~K³~Kµ~K³|Iµ~Kµ~K³|I²{H°{H±|I°{H°{H¯zG¯zG°{H¯zG¯zG®yF®yF¯zG­xE­xE­xE­xE­xE­xE­xE­xE¬wDªuBªuB¬wDªuDªuD©tCªuD©vD§tB¨uF¨uF§tE§tE§tE§tE§tB§tB¦sA¦sA¥r@¥r@¥r@¥r@¤q?¤q?¤qB¤qB¤qB¢o@¢oB¢oB¡nA¡nA¡nA¡nA m@¡nA m@ m@Ÿl=Ÿl=Ÿl=žk<žkj;“`1ŠY.†U*zK&qBd:]3N,G% =6,'  #™d%ÿÇÿÞ-ÿßÿÓÿÍÿÔÿÖÿÖÿ¡ -ÞláEßCÔ@Ó?Ð<Ð<Ð<Ð<ÌBå[ÿŸÿ´ÿ½ ÿÔ$ÿêFÿ×3ÿÕ ÿÖ -ÿÖÿÖÿÕÿÕÿÔ ÿÔ ÿÓ -ÿÓ -ÿÓ ÿÓ ÿÑÿÑÿÑ -ÿÎÿãZ°o - *B²Ä+ú&û' ÿ#ÿ#ý#þ$õ(ü/ÿ0ÿ2 -ÿ1ÿ6ÿ:ÿ:ÿ<ÿ= ÿ=ÿ>ÿ=!ÿ=!ÿ>!ÿC&ÿCÿ=ÙGÖDÐZänñƒÿ‘ ÿžÿ©ÿ·ÿ¾ÿÇ ÿÎÿ×ÿÛÿÛÿØÿÕÿÔÿÕ ÿÕ ÿÔÿÔÿÖÿÓÿÊ ÿÑìš.e&!& ;9 3-'&''299 ; 1)j²^+û„ÿ—ÿ­ÿµÿµ ÿ· ÿºÿºÿ½ÿ½ÿÄÿÉ ÿÐÿÐÿÎÿËÿÊ ÿÍÿÕÿÎÿ®ÿ©ÿ¦ÿ¯ÿà -ÿÅ ÿÏ ÿÙÿÖ ÿÐÿÓÿÕÿË ÿÅÿÄÿÉ ÿËÿÌ ÿÎÿÌÿËÿËÿÍ ÿÎÿÐÿÑÿÉÿÂÿ»ÿÀÿÍÿÐÿÊÿÎÿÞ6é˜D4'  - &/•U+¹yOŃPŃP¸‡S¸‡SºˆQºˆQ½†S½†S½†S½†Sº…Rº…Rº…R¸ƒP¸ƒP¸ƒPº„N¹ƒM¹ƒMº„N¹ƒM¹ƒM¸‚L¸‚L¶N¶N¶N¶N¶Nµ€Mµ€Mµ€Mµ€M´L´L´L¶Lµ~Kµ~Kµ~Kµ~Kµ~K³~K³~K°{H°{H±|I±|I°{H°{H°{H°{H°{H¯zG¯zG¯zG¯zG¯zG¯zG®yF­xE®yF­xE¬wD¬wD¬wD¬wFªuDªuD¬wF©vD©vD¨uF¨uF¨uF§tE§tE§tE§tB§tB¦sA¦sA¦sA¥r@¥r@¥r@¤q?¤q?¤qB¤qB¢o@¢o@¡nA¡nA¢oB¡nA¡nA¡nA m@¡nA¡nA¡nA m>Ÿl=Ÿl=Ÿl=žk<žk<žk<žk<žk<žk<žk<žk<žk>j=™h;™h;™h=˜g<˜g<˜g<˜g<˜g<—f;—f;—f;–e:”f:”f:“e9”f:•d9•d9•d9”c8”c8”c8”c8”c8’a6’a6’a6’a6’a6’a6’a6‘`5»‡Q»‡Q»‡Q»‡Q»‡Q»‡Q»‡Q¼ˆR¼ˆR¼ˆR¼ˆP¼ˆP¼ˆP¼ˆP¼ˆP¼ˆP¼ˆP¼ˆP½‰Q½‰Q½‰Q½‰Q½‰Q¾ŠR¾ŠR¾ŠR¾ŠR¿‹S¿‹S¾ŠR¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁUÁUÁUÂŽVÂŽVÂŽVÃWÃWÂŽVÂŽVÃWÃWÃUÃUÃUÃUÄVÄVÄVÄVÄVÄVÃUÄVÄVÄVÅ‘WÅ‘WÄ’YÄ’YÅ“ZÅ“ZÅ‘YÈ”\È”\È”\È”\È”\Ê”\Ê”\Ê”\Ê”\Ê”\È’ZÆ’\Æ’\È’\È’\È‘^ÅŽ[ÁŒ[¾‰X·‚Q³~M«xI¤qBšl@b6‚X0{Q)qH&lC!a æ¡)ÿÝeÿßÿ×ÿÎÿÙÿÔÿÔÿÌê˜äVÕGÙ?×=Ú;Û<Ó=Ó=ÇAæ`ÿœÿ¯ ÿ±ÿ¿ÿè>ÿæ<ÿ×ÿÖÿÖ ÿÖ ÿÕÿÕÿÔ ÿÔ ÿÔ ÿÔ ÿÔ ÿÔ ÿÓ -ÿÔ ÿÑ -ÿÔ ÿàÿÑ R- - ˆ§ -ÿÿ% ÿ#ÿ"ú"ý%Ý6ç@êjñqãWÍAû9þ<ÿ7ÿ=ÿ;ÿ> ÿ?ÿ?ÿ>!ÿ?"ÿ@!ÿB#ÿBÿ>úEõ@ÚXïmõ„ÿ‘ÿžÿ©ÿ·ÿ¾ ÿÇ ÿÐÿÙÿÚÿÛÿØÿÕÿÔÿÓÿÓÿÓÿÔÿÖÿÑ -ÿÉÿÓ åš5[ - ) $ -" &% & & 0 , Be Ûqÿ£ÿ¬ÿ¸ ÿÀÿ¿ÿ¼ÿºÿ¼ÿ½ÿÁÿÄÿÉ ÿÍ ÿÑÿÑÿÎÿÌÿÊ ÿÎÿÒÿÒÿºÿªÿ§ÿ¨ÿÀÿÄÿÊ -ÿÍ ÿÈÿÍ ÿÑÿÊÿÀÿÄÿÉ -ÿË ÿÎ ÿÎ ÿÏÿÏÿÏÿÓ -ÿÔÿÖÿ½ÿ£ÿŸÿ¬ÿ¸ÿ¸ÿÇ ÿÄ -ÿÆÿ×ÿÑY®`9*5& -     -# v!±\@ég8ôrCË„RʃQºˆOºˆO¸‡S¸‡S»†S»†Sº…Rº…R¹†R¹†R¸ƒP¸ƒPºƒPºƒPºƒPºƒP¹ƒM¹ƒMº„N¹ƒM¹ƒM¹ƒM·ƒM¶‚L¶‚L¶‚L¶‚L¶‚LµKµKµKµK´€J´€J¶€J¶€J³I³I³I³I³I±}G±|I±|I±|I±|I±|K°{J°{H°{H°{H°{H¯zG¯zG¯zG¯zG¯zG®yF®yF­xE­xE­xE¬wF¬wF¬wF¬wF©vD©vD©vG¨uF¨uF¨uF¨uF¨uF¨uC¨uC§tB¦sA§tB¦sA¥r@¥r@¥r@¥r@¥rC¥rC¤qB¤qB¢o@¤qB¤qB¢o@¢o@¢o@¢o@¢o@¢o@¡n?¡n? m> m>Ÿl=Ÿl=Ÿl=Ÿl=žk<žk<žk<žk<žk<žk>žk>›j=›j=›j?™h=™h=™h=™h=™h=˜g:˜g:˜g:—f9•g9•g9”f:”f:–e:•d9”c8”c8”c8”c8”c8”c8”c8”c8”c8’a6’a6’a6’a6’a6»‡Q»‡Q»‡Q»‡Q»‡Q»‡Q¼ˆR¼ˆR¼ˆR¼ˆR½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q¼ˆP½‰Q½‰Q½‰Q½‰Q¾ŠR¾ŠR¾ŠR¿‹S¿‹S¿‹S¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÃWÃWÃWÃUÃUÄVÄVÄVÄVÄVÄVÃUÄVÄVÄVÅ‘WÅ‘WÆ’XÆ’XÅ“ZÅ“ZÅ“ZÇ•\È”\Æ’ZÆ’ZÆ’ZÈ”\È”\È’ZÊ”\Ê”\Ê”\Ê”\Ê”\È”^Æ’\Ç‘[ÅYÀ‰V¼…R´N±|KªuD¢m<˜e6].…W+}O#oEh>^5V- J% -B4-"  - &"XÿÏWÿä ÿÜÿÒÿÕÿÔÿÓÿÚ%ÿ¾ ÿtÝOáGÞDÛ<×8Ô>Ò<ÊDìfÿ¡ÿ°ÿµÿ¼ÿÔ*ÿê@ÿÜÿØÿ× ÿ× ÿÕÿÕÿÕÿÕÿÔ ÿÔ ÿÔ ÿÔ ÿÓ -ÿÓ -ÿÓ ÿÔ ÿÒ ÿäÀ‡;/ -  — »-ÿ)ÿ+ÿ* ÿ) -ÿ*ÿ/èAÿ_ÿ¢)ÿ·>ÿ˜"ôhø6ó1ÿ2 ÿ9ÿ8ÿ> ÿ@ ÿ?ÿ?"ÿ?"ÿ@!ÿ@!ÿAÿDÿM þIÜZðn÷†ÿ’ÿŸÿ©ÿ¶ÿ¿ ÿÊÿÑÿÙÿÙÿÛÿØÿÕÿÔÿÓÿÓÿÓÿÔÿÖÿÑ -ÿÇÿÒ -à•0U - - - $ -2'R ²m;ÿ´&ÿ»-ÿº ÿ¾ÿ¿ÿ¼ÿ¼ÿ¼ÿ¿ÿÀÿÄÿÈÿÌ ÿÍ ÿÏÿÎÿÎÿÎÿÍÿÍÿÍ ÿÓÿÆ ÿ¥ÿ£ÿ´ÿÁÿÊ ÿÉ ÿÄÿÌ ÿÒÿÌÿÀÿ¾ÿÇÿË ÿÍÿÎ ÿÏ -ÿÒÿÒÿÑÿÕ ÿÕÿ¿÷”à}ârúŠÿ©ÿ±ÿ»ÿ¾ÿÆÿ×û­5Š<.7N?5      -(" ^ Ž9ÍKà^/Å~LÇ€N¹‡N¹‡N¸‡S¹ˆTº…R»†S¼‡Tº…R¹†R¹†Rº…Rº…R¼…RºƒPºƒPºƒPº„Nº„Nº„Nº„Nº„Nº„N·ƒM·ƒM·ƒM·ƒM·ƒM·ƒM¶‚L¶‚L¶‚LµKµKµK·K¶€J´€J´€J³I´€J´€J³I³~K±|I³~K³~K°{J°{J°{H°{H°{H°{H°{H¯zG¯zG¯zG¯zG®yF¯zG®yF­xE®yF­xG¬wF¬wF¬wF«xF«xF©vG©vG¨uF¨uF¨uF¨uF¨uC¨uC¨uC¦sA¦sA§tB¦sA¦sA¦sA¦sA¦sD¦sD¥rC¥rC¥rC¥rC¤qB¤qB¤qB¤qB¢o@¢o@¢o@¢o@¡n?¡n?¡n? m> m> m> m> m> m>Ÿl=Ÿl=Ÿl=Ÿl?Ÿl?›j=›j=›j?™h=™h=™h=™h=˜g<˜g:˜g:—f9–e8•g9•g9”f:•g;–e:•d9•d9•d9”c8”c8”c8”c8”c8”c8”c8”c8’a6’a6’a6”c8»‡Q»‡Q»‡Q»‡Q»‡Q»‡Q¼ˆR¼ˆR¼ˆR¼ˆR½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q¾ŠR¾ŠR¾ŠR¾ŠR¿‹S¿‹S¿‹S¿‹SÁUÁU¿‹SÁUÁUÁUÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÃWÃWÄXÄVÄVÄVÄVÄVÄVÄVÄVÅ‘WÅ‘WÅ‘WÅ‘WÆ’XÆ’XÆ’ZÆ’ZÇ•\Ç•\Å“XÇ•ZÇ•ZÇ•ZÈ”ZÉ•[Ë•]Ë•]È’ZÈ’ZÈ”\È”\Ê”\È’ZÄZÃY¾‰X¸ƒR²R­zM¦tLŸmE’c>ˆY4|R.sI%jC#a:Q2J+>" 9,!   (A;e+Ò˜SÿÛ5ÿÚ4ÿÚÿÒÿÎÿÓ -ÿÕÿÔÿ¨ßtçPáJßDÙ>Ó@ÙFÐWðwÿ­ ÿ­ ÿ¢ÿ¦ÿ»ÿÛ'ÿçEÿÖ4ÿÖ ÿØÿÕÿÕÿÕ ÿÕ ÿÔ ÿÔ ÿÔ ÿÓ ÿÓ ÿÓ ÿÔÿÔÿÑÿÖ ÿÙ;x7# - -  9MÞý/ÿ0ÿ/ÿ4ÿ7ÿ<ÿ8àQÿ‡ÿÃÿÃÿ¹ÿ¤åq¿Ké1ö>ÿ9ÿBÿD&ÿ@"ÿC%ÿA#ÿ@&ÿ@&ÿ@ÿAÿH ÿG ì]ûlùƒÿ‘ÿ¢ÿ«ÿ·ÿÀ -ÿÌÿÓÿÚÿÚÿÛÿØÿÕÿÔÿÓÿÓÿÓÿÔÿÕÿÐ ÿÆÿÒ -ÕŽ/H  &/«Sÿ±)ÿÅÿÀÿÁÿÁÿÁÿÀÿ¾ÿ¿ÿÃÿÃÿÆÿÊ ÿÎÿÍÿÍÿÎÿÑÿÔÿÒÿÍÿË ÿÌÿÏÿ±ÿˆÿ“ÿŸÿ¯#ÿÂ=ÿÍHÿÏ.ÿÅ$ÿºÿ¸ÿÃÿÉ ÿËÿÌ ÿÏÿÐÿÔ ÿÔ ÿÑÿ×ÿº$ðƒøÿ¥ÿ”õ†ÿšÿ©ÿ²ÿ¸ÿ¿ÿÍ#ψ:Y+.!  - "  X~,÷'ÿ4 êW,þk@΀JÏK¾ˆP¾ˆP¼ˆR¼ˆRÀ†TÀ†T½†S¼…R¼…R¼…R¼…RºƒPºƒPºƒPºƒPºƒPºƒPºƒPº„Nº„Nº„Nº„Nº„Nº„Nº„N¹ƒM·ƒM·ƒM¶‚L¶‚L¶‚L¶‚L¶‚L¶‚LµKµKµKµK´€J´€J³I³I³~K³~K±|I°{H°{H°{H±|I±|I°{H°{H°{H°{H¯zG°{H¯zG¯zG¯zG®yF®yH®yH®yH­xG¬yG«xF«xF«xF«xI©vG¨uF¨uF¨uC©vD¨uC¨uC§tB§tB§tB¦sA¦sA¦sA¦sD¦sD¦sD¥rC¥rC¥rC¥r@¤q?¤qB¤qB¤qB¤qB¢o@¢o@¢o@¡n?¡n<¢o=¡n? m> m> m> m> m> m>Ÿl=Ÿl=Ÿl=œk>œk>œk>›j=›j=›j=›j=™h;™h;™h;™h;˜g:–h:–h:•g;—i=˜g<—f;—f;—f;•d9•d9–e:–e:–e:•d9”c8•d9•d9•d9•d9•d9¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR½‰S½‰S½‰S½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q¾ŠR¾ŠR¾ŠR¿‹S¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁU¿‹SÁUÁUÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÃWÃWÃWÄXÄXÄXÄVÄVÄVÄVÄVÅ‘WÅ‘WÅ‘WÆ’XÆ’XÆ’XÆ’XÆ’XÆ’XÆ’ZÈ”\È–]È–]Ç•ZÈ–[È–[È–[É•[É•[Ë•]Ê”\È’ZÈ’ZÅ‘YÄXÄŽV¿‰Q¼ˆRµK¯zIªuD¤qDj=•c;‹Y1~O*vG"kAa7V/N'@!9*! '7& O-_=#z@Ÿe ÿ¾ÿÙ3ÿãÿÔÿÅÿÈÿÍ ÿÏ ÿÃÿŸüeãLÞCÝBÞKæSÕ\ÿŒÿ½ÿ¡þ~ÿÿ·ÿÉÿïMÿÝ;ÿ×ÿØÿÖÿÖÿÕ ÿÕ ÿÔ ÿÔ ÿÔ ÿÔ ÿÔ ÿÔ ÿÕÿÕÿÔ ÿÎÿçIÜ›%%   - - - >_ -ò$ ÿ;"ÿ6ÿ4ÿ;ÿ@ÿDÿ4ë\ÿ™&ÿÀ ÿ·ÿ´ÿ±ÿœåqð8ð8ÿ4 ÿBÿD&ÿD&ÿD&ÿC%ÿB(ÿ@&ÿ@ÿ@ÿG ÿH ï`ýnû…ÿ“ÿ¢ÿ«ÿ·ÿÀ -ÿÌÿÓÿÚÿÙÿÙÿØÿÔÿÓÿÓÿÓÿÓÿÕÿÕÿÏÿÆÿÒ -Љ*D - -`2ú¢ÿÄ<ÿ»ÿÀÿÁÿÂÿÁÿÁÿ¾ÿÀÿÄÿÅ ÿÈ ÿÊ ÿÌÿË ÿËÿÍÿÑÿÕÿÕÿÎÿË ÿÌÿÒÿÃÿöwìpáeäuó„÷‘ÿ£ÿ¶ÿ¿ÿÉ ÿÇ ÿÊÿËÿÌÿÒ -ÿÕ -ÿÔ ÿÙÿÉðƒÍ`ìuÿªÿ¼ÿžÿ˜ÿ¢ÿ³ÿ¶ÿÈÿÔ*«dA$$    " ' 1" -  X{)õ%ö&Ð=êW,Äv@΀J½‡O¾ˆP¼ˆR½‰SÁ‡UÀ†T¾‡T½†S¼…R½†S½†S½†S½†S¼…R¼…RºƒPºƒP¼…R¼†P¼†P¼†Pº„Nº„Nº„Nº„N¹ƒM·ƒM·ƒM·ƒM·ƒM·ƒM·ƒM¶‚L¶‚L¶‚LµKµKµKµK´€J´€J´€J³~K³~K³~K³~K³~K±|I±|I±|I±|I°{H°{H°{H°{H°{H°{H¯zG¯zG¯zG¯zI®yH®yH®yH¬yG¬yG«xF«xF«xI«xI«xI«xI«xF©vD©vD¨uC¨uC¨uC¨uC§tB§tB§tB§tE¦sD¦sD¦sD¥rC¥rC¥r@¤q?¤qB¤qB¤qB¤qB¤qB¤qB¤qB¢o@¢o=¢o=¡n?¡n?¡n?¡n? m>Ÿl=Ÿl=Ÿl=Ÿl=Ÿl=l?l?œk>œk>œk>œk>œk>›j=›j=›j=›j=™h;—i;–h:—i=—i=™h=™h=˜g<˜g<—f;–e:—f;—f;–e:–e:•d9•d9•d9•d9•d9•d9¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR½‰S½‰S½‰S½‰S½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q¾ŠR¾ŠR¾ŠR¿‹S¿‹S¿‹SÁUÁUÁUÁUÁUÁUÁUÁUÂŽVÂŽVÂŽVÃWÃWÃWÃWÃWÃWÃWÄXÄXÄXÄXÄXÄXÄVÄVÄVÄVÅ‘WÅ‘WÅ‘WÆ’XÈ’ZÈ’ZÈ’ZÈ’ZÈ’ZÊ”\Ê”ZÊ”ZÈ”ZÉ•[É•[É•[É•[È”ZÉ•[É•[É•YÉ•YÈ”ZÄVÂŽX¿‹U¹†R³€L­|M©xIžqH”g>ˆ_9}T.sO,iE"\="S4D*=#5+  !>" M1g@ xQ1†Y2‹^7ç•ÿÅHÿÜÿÚÿÎÿÃÿÂÿÇ ÿÇÿà -ÿ¢ò~âeâeîcßTÕnÿ­ÿÊ2ÿ”ÿ[ÿ^ÿ—ÿ·"ÿÜ-ÿë<ÿ× ÿÖÿÕ ÿÖ ÿÔÿÔÿÕÿÔÿÓÿÓÿÓÿÓÿÔ ÿÕÿÖ ÿÑÿÝÿÜ}@.  - - - -  - -   - g” ÿ0ÿ;ÿ:ÿ8ÿ@ÿEÿ<ÿ5øgÿŸÿ»ÿ·ÿ³ÿ³ÿº -ÿ°öw -¼=ÿ-ÿDÿJ*ÿG'ÿE(ÿC&ÿC(ÿA&ÿ?ÿ?ÿA ÿCý\ÿjú‡ÿ•ÿ¢ÿªÿ¸ÿÀÿÌÿÑÿ×ÿ×ÿÙÿØÿÖÿÕÿÓÿÓÿÓÿÔÿÔ ÿÎÿÇÿÓ É‚'M \ÓˆÿÈÿÁ ÿ½ÿ¿ÿ¿ÿÀÿÀÿÀÿ¿ÿÁÿà ÿÇ ÿÊÿÊÿÌÿÉÿÇ ÿÊÿÑÿÕÿÓÿÏÿÎÿÎÿÓÿÓÿ¾úèiÈIÀIËTÕtú™ÿ¿ÿË'ÿÐ0ÿÑ1ÿÒ5ÿÔ7ÿ×"ÿÕ ÿÒÿØÿÐ'ø£ÄLÎVàhü„ÿ» ÿ» ÿ¢ÿ¤ÿ¶#ð¯i:°j;G )     - aRH"   -Kg!ÿ ÿ$ÿ ÿ( öE!ÿW3×wGßO¼ŠQ»‰P½‰S¼ˆR¼‡T¼‡TÁ…SÁ…S‡R‡R»†S¼‡T½‡Q½‡Q½‡Q¼†P¼†P¼†P¼†P¼†P¼†Pº„Nº„Nº„N¹ƒM¹ƒM¹ƒM¹ƒM·ƒM·ƒM·ƒM·ƒM¶‚L¶‚L¶‚LµKµKµKµKµK´L´L´L³~K³~K³~K±|I±|I±|I±|I°{H°{H°{H°{H°{H°{H¯zG¯zG¯zI¯zI®yH®yH­zH¬yG¬yG¬yG¬yJ¬yJ¬yJ«xI«xF©vD©vD©vD¨uC©vD¨uC¨uC§tB§tB§tE§tE¦sD¦sD¦sD¥rC¥r@¥r@¤qB¤qB¤qB¥rC¤qB¤qB¤qB¤qB¤qB¢o@¢o@¢o@¢o@¢o@¡n?¡n?¡n? m> m> m>l?l?l?l?œk>œk>œk>œk>›j=›j=™k=–h:—i;—i;—i=—i=™h=™h=˜g<˜g<˜g<˜g<—f;—f;–e:•d9•d9•d9•d9•d9•d9•d9¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR¼ˆR½‰S½‰S½‰S½‰S½‰Q½‰Q½‰Q½‰Q½‰Q½‰Q¾ŠR¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÃWÄXÄXÄXÄXÄXÄXÄXÄXÄXÅ‘WÅ‘WÅ‘WÅ‘WÆ’XÆ’XÆ’XÆ’XÊ”\Ê”\È’ZÊ”\Ê”\Ê”\Ê”ZÊ”ZÈ”ZÈ”ZÈ”ZÉ•[É•[È”ZÈ”ZÈ”ZÆ’VÄTÁS¼ˆN·ƒM°|F«xD¥r>œk<–e6Š]4€S*uL&nEa=W3J+A"3,#  @$ R6lE%}V6‰\5™lEÎ|û©,ÿÊÿÔÿÕÿÑÿÊ ÿÄÿÀÿÁÿµÿ¨ÿšÿ ñfÙNü•ÿÇ0ÿ²ôÿYúOìsÿ©ÿÆÿç8ÿâ+ÿÖÿÕ ÿÖ ÿÔÿÔÿÕÿÔÿÓÿÓÿÓÿÓÿÔ ÿÔ ÿÖ ÿÒÿÕÿãã¦U=      - - - - - kœ(ÿ6ÿ>"ÿ;ÿ;ÿDÿHÿ9ÿ/ûjÿ¥ ÿ¼ÿºÿ·ÿ³ÿ¿ÿÖ&ÿ—*¶7òÿ:ÿA!ÿD$ÿC&ÿ@#ÿ@%ÿ@%ÿ?ÿ?ÿA ÿEÿ_ÿjú‡ÿ•ÿ£ÿ«ÿ¸ÿÁÿËÿÐÿÖÿ×ÿÙÿØÿÖÿÕÿÓÿÓÿÓÿÕÿÔ ÿÎÿÆÿÑ -Å~#@(¾s ÿÃYÿÂÿºÿ¿ÿ¿ÿ¿ÿ¿ÿÀÿÀÿÀÿÁÿÅ ÿÉÿÊÿÊÿÊÿÊÿÉÿËÿÏÿÑÿÏÿÏÿÏÿÐÿÔÿÙÿÑÿ¼ÿÿ“ÿ¦ÿ±&ÿÁÿÂÿÄ ÿÀÿ³ÿ¾ÿÄ'ÿÊ-ÿÛ&ÿÜ'ÿÖ ÿÕ ÿÅÌwÒZãkéq÷ÿ­ÿÂÿ«ÿ¤ÿ»(Óp32'      ,j[Q   Ieÿ ÿ$ÿ ÿÝ,í<Èh8ÙyI¼ŠQ¼ŠQ½‰S½‰S½ˆU»†SÁ…SÁ…S‡R‡R¼‡T¼‡T¾ˆR½‡Q¾ˆR½‡Q½‡Q½‡Q½‡Q¼†P¼†P¼†Pº„Nº„Nº„N¹ƒM¹ƒM¹ƒM·ƒM·ƒM·ƒM·ƒM¶‚L¶‚L¶‚LµKµKµKµKµK´L´L´L´L³~K³~K³~K³~K³~K±|I±|I±|I±|I±|I°{H°{H°{H¯zG¯zI¯zI¯zI¯zI­zH­zH¬yG¬yG¬yJ¬yJ¬yJ¬yJ«xF«xF«xF©vD©vD©vD©vD¨uC¨uC¨uC¨uF§tE¦sD§tE§tE¥rC¥r@¥r@¤qB¥rC¥rC¥rC¥rC¥rC¥rC¤qB¤qB¤qB¢o@¢o@¢o@¢o@¡n?¡n?¡n? m> m> m>žm@žm@l?l?l?l?œk>œk>œk>›j=™k=—i;™k=™k=—i=™k?™h=™h=™h=˜g<˜g<˜g<—f;—f;—f;•d9•d9•d9•d9•d9•d9•d9½‰S½‰S½‰S½‰S½‰S½‰S½‰S¾ŠT¾ŠT¾ŠT¾ŠR¾ŠR¾ŠR¾ŠR¿‹S¿‹S¾ŠR¿‹S¿‹S¿‹S¿‹SÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÃWÄXÄXÄXÄXÄXÄXÅ‘YÅ‘YÅ‘YÅ‘WÅ‘WÅ‘WÅ‘WÆ’XÆ’XÆ’XÈ”ZÊ”\Ê”\Ê”\Ê”\Ê”\Ê”\Ê”ZÊ”ZÊ”ZÊ”ZÊ”XÊ”XÊ”XÊ”XÇ‘WÅU¾ŠR¸„L¶N±|I§tG m@–g>Ž_6V0xO)kI+dB$X<#N2A*9" . $    . -@,Y=&fJ3{W4iF©q1Í•Uÿ²ÿÁ$ÿÑÿÚÿÕÿÊÿÀÿ½ÿ·ÿ³ÿ¦òƒÓfúÿÄ0ÿº&ÿuóUýTùPô]ÿˆÿ±ÿÏ&ÿîAÿÜ/ÿÕÿÖÿÕÿÕÿÕÿÔÿÔ ÿÔ ÿÕÿÕÿÔ ÿÔ ÿÓÿÔÿÖÿÕ ÿÚ=—S'+" '--/./,( $ ( " u§+ÿ9ÿ?%ÿ=&ÿ;$ÿE"ÿH%ÿ4ÿ%öhÿ§+ÿ¾ÿ¼ÿ»ÿ½ÿÌ&ÿÛ5æ)(¸Ö8 -ÿ:ÿC#ÿ?!ÿ?!ÿ@%ÿ@%ÿ=!ÿ?#ÿBÿEÿ^ÿkþˆÿ– ÿ¦ÿ®ÿºÿ ÿÊ ÿÐÿ×ÿ×ÿØÿØÿÔÿÔÿÓ ÿÓ ÿÓÿÕÿÓ ÿÍÿÆÿÑ -»s :;‘Nÿ¼ÿÈÿ½ÿ¿ÿÀÿÀÿ¿ÿ¿ÿ¿ÿÀÿÁÿÄÿÊ ÿÌÿÊÿÉÿÉÿÊÿËÿÌÿÍÿÍÿÎÿÎÿÓÿÓÿÔ ÿ×ÿ×ÿÕ ÿÒÿØ ÿ×ÿÜ ÿÛ1ÿÌ"þ­<å”#Õ’K×”MÕ‘OÏ‹IÚ™5ÿ¿[ÿàKÿæQÿ°Òrõzù~ýƒÿ‰ÿ¦ÿÂÿ·ÿ£ÿÀ,⃠-  -  -   -   Fdýÿ$ÿ ÿ ÿ"ÿ"ó4ÿK,àoAì{MÊQÁˆO¿‰Q¿‰Q½‡Q¾ˆR¾ˆR¾ˆR»‰R»‰R¾ˆR¾ˆR¼ˆR»‡Q½‡Q½‡Q½‡Q½‡Q½‡Q¼†P¼†Pº„Nº„N¹ƒM¹ƒMº„Nº„Nº„N·ƒM·ƒM·ƒM¶‚L¶‚L¶‚LµKµK´€J´€Jµ€M´L´L´L´€J³I³I³I³I³I³I±}G±|I±|I±|I±|I°{H°{H°{J¯zI¯zI¯zI¯zI®yH®yH®yH®yH­xG¬yJ¬yJ¬yG«xF«xF«xF«xF«xF©vG¨uF¨uF¨uF¨uC¨uC¨uF¨uF¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¦sA¥r@¥r@¤q?¤qB¤qB¢o@¢o@¢o@¢o@¢o@¢o@¡n?¡n?¡n?¡n?žm@žm@žm@l?l?l?l?l?œk>œk>šl>™k=™k=™k=™k=™k=›j?›j?›j?›j?˜g<™h=˜g<—f;—f;—f;—f;—f;—f;—f;—f;—f;½‰S½‰S½‰S½‰S½‰S¾ŠT¾ŠT¾ŠT¾ŠT¾ŠT¿‹S¿‹S¿‹S¿‹S¿‹S¿‹S¾ŠR¿‹S¿‹S¿‹SÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÃWÄXÄXÄXÄXÄXÄXÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÆ’XÆ’XÆ’XÆ’XÆ’XÈ”ZÈ”ZÈ”ZÊ”\Ê”\Ê”\Ê”\Ê”\Ê”\Ê”ZË•[Ë•[Ë•[Ë•YË•YÇ‘UÃQÀŠP¼†L´€H¬x@¥p= k8”a4ŒY,‚S*yJ!mDe<V4K) ?# -;,%    *@$ K/fBvR/’Z¢j*þžÿ¸ÿÀÿÍÿÕÿÓÿÇÿÂÿ¹ÿ±ÿ›ø‰ÿšÿÆ.ÿ´ ê}óUóUÿVýTêSÿhÿ©ÿÀÿé<ÿä7ÿÕÿÕÿÔ ÿÔ ÿÕÿÕÿÔ ÿÔ ÿÕÿÕÿÔ ÿÔ ÿÓÿÔÿ×ÿÐÿáDùµO) .5 <" A'D(C'D& A#83+& - D2%, ' - v°4#ÿ?%ÿ@&ÿ>'ÿ<%ÿE"ÿH%ÿ7ÿïaÿ¥)ÿ¿ÿ¾ÿÀÿÊ ÿØ2ÿÈ"߆"¢IÂ$Ð2ÿ6ÿ?ÿA#ÿ?!ÿ@%ÿ@%ÿ>"ÿ?#ÿDÿFÿ_ÿlÿ‰ÿ— -ÿ¦ÿ¯ÿ»ÿà -ÿË ÿÐÿÖÿÖÿØÿØÿÔÿÔÿÒ ÿÓ ÿÔÿÖÿÓ ÿÌÿÇÿÒ ²j:v3ì©ZÿÅÿÁÿ ÿÄ ÿÁÿ¿ ÿ¿ÿ¿ÿ¿ÿÁÿÃÿÅ ÿËÿÌÿÉÿÈÿÉÿÌÿÍÿÍÿÌÿËÿÌ ÿÎÿÓÿÓÿÓ ÿÑ -ÿÓ -ÿÔ ÿÕ ÿÕ ÿÖÿÙ -ÿÄòŸáíœ+æ£\ÝšSØ”RÒŽL¾}ÀÿË6ÿâMïé‰ÿ‰þƒÿÿ•ÿ¯ÿÄÿÃÿ«ÿÄ0íŽ   -   -    Fg!ÿ ÿ$ÿ ÿ ÿ% ÿ"ã$î/ÇV(ßn@‰P‰PÀŠR¿‰Q¿‰S¿‰SÀŠT¿‰S»‰R¼ŠS¾ˆR¾ˆR¼ˆR¼ˆR¾ˆR¾ˆR¾ˆR½‡Q½‡Q½‡Q½‡Q¼†P¼†P¼†P¼†P¼†P¼†Pº„N¸„N¸„N¸„N¸„N¸„N·ƒM·ƒM¶‚LµK¶‚L¶Nµ€Mµ€M´L´€J³I³I³I³I³I³I±}G±|I³~K±|I±|I°{H°{H°{J°{J°{J¯zI¯zI¯zI®yH®yH®yH®yH­zK­zK¬yG«xF«xF©vD«xF«xF©vG©vG©vG©vG©vD©vD©vG¨uF¨uC¨uC¨uC§tB§tB§tB¦sA¦sA¦sA¦sA¥r@¥r@¤qB¤qB¥rC¤qB¤qB¤qB¤qB¤qB¢o@¢o@¢o@¢o@ŸnAŸnAŸnAžm@žm@žm@žm@l?l?l?šl>™k=šl>šl>™k=™k=œk@œk@›j?›j?›j?™h=™h=™h=™h=˜g<˜g<˜g<—f;—f;—f;—f;½‰S½‰S½‰S½‰S½‰S¾ŠT¾ŠT¾ŠT¾ŠT¾ŠT¿‹S¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁUÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÃWÃWÃWÃWÂŽVÃWÃWÃWÃWÃWÄXÄXÄXÄXÅ‘YÅ‘YÅ‘YÇ‘YÇ‘YÈ’ZÈ’ZÈ’ZÈ’ZÈ’XÈ’XÈ’XÈ’XÈ’XÊ”ZÈ’XÊ”ZÊ”ZÈ’XÊ”\Ê”\Ê”\È’ZÊ”\Ë•]È”\È”\Æ’ZÄX¾ŠTµK®{I«xF oB–e8Œ_8†Y2{R0tK)gB%`;T6K-9"2&    - -!-@-O5"V<)»y%ò°\ÿ®ÿ¶ÿÀÿÆ ÿÅÿÁ -ÿ¾ÿ¸ ÿ­ÿ²ÿ·Dÿ“ ö]çNêVîZâXÜRÝRé^ÿ™ÿ´ ÿÒ%ÿîAÿÝ,ÿÕ$ÿÖÿÖÿÕ ÿÕ ÿÔ ÿÔ ÿÕ ÿÕ ÿÔ ÿÔ ÿÔÿÔÿÕÿÓ ÿÖÿä#ºq>$%%> I+Z6eAlBj@l?i<\.U' G @2- ""^L?$+  ˆ¾8ÿ<ÿE$ÿ:%ÿ<'ÿD)ÿF+ÿ<ÿ$÷Nÿ1ÿ¼ÿ¹ÿÃÿÝ1ÿÅQáœ(ÀŒ?Á@Ëd1©Bÿ2ÿB"ÿA#ÿA#ÿAÿAÿAÿD ÿIÿKÿ^ÿiÿˆÿ—ÿ¦ÿ¯ÿ»ÿÄ ÿÍÿÏÿ×ÿ×ÿ×ÿ×ÿÔÿÒÿÑ ÿÒ ÿÔÿÖÿÒÿÉÿÇÿÒ ªc7*Pç•ÿÌ.ÿÃÿÂÿÄÿÆÿÀ ÿ¾ -ÿ½ÿÀÿÂÿÄÿÇÿÊ -ÿÊ ÿÉ -ÿÊÿÉ ÿËÿÍÿÎÿÎÿÍÿÌÿÎ -ÿÏ ÿÑ ÿÔÿÖÿÔÿÓ ÿÔ ÿÕÿÖÿÙ%ÿÌòœ,ì–&ë¥cñ«iÞ¦aØ [Úœ_Øš]ÑŽEÂ6Њø²G߈ò›ÿ˜ÿ˜ÿ›ÿ¡ÿ´ÿÈÿÁÿ³ÿÈ ò1)  -'  Pu$ÿ! ÿ! ÿ ÿ ÿ"ÿ"ÿ ÿú& -ÿ9íf:ývJÖƒPÕ‚O¿‰QÀŠR¹ŒR¹ŒR½‹T½‹T¼ŠS»‰R½‰S½‰S¿‰S¿‰SÀ‡RÀ‡RÀ‡RÀ‡R¿†Q¿†Q½‡Q¼†P¼†P¼†P¼†P¼†P¼†P¼†P¸„N¸„N¸„N¸„N·ƒM·ƒM·ƒM·ƒM¶N¶N·€M·€M·K¶€J¶€J¶€J¶€J¶€J´€J³I³I³I³~K³~K±|I±|I±|K°{J°{H°{H°{H¯zG¯zI¯zI¯zI®yH®yH®yH­zH­zH¬yG©vD«xF«xF«xI«xI©vG©vG©vD©vD©vD¨uC¨uC¨uC¨uC¨uC§tB§tB§tB¦sA¦sA¦sA¦sA¦sA¥rC¥rC¥rC¥rC¥rC¤qB¤qB¤qB¢o@¢o@¢o@¢o@ o@Ÿn?ŸnAŸnA m@ m@žm@žm@žm@žm@l?l?l?œk>œk>l?œk>œk>œk>›j=›j=›j=™h;™h;™h;™h;˜g:˜g:˜g:—f9—f9—f9½‰S½‰S½‰S½‰S½‰S¾ŠT¾ŠT¾ŠT¾ŠT¾ŠT¿‹S¿‹S¿‹S¿‹S¿‹S¿‹SÁUÁUÁUÁUÁUÂŽVÂŽVÁUÃWÃWÃWÃWÃWÃWÃWÃWÄXÄXÄXÄXÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÈ’ZÈ’ZÈ’ZÈ’ZÈ’ZÈ’ZÈ’XÈ’XÈ’XÈ’XÈ’XÈ’XË•[Ë•[Ê”ZÊ”ZÊ”\Ê”\È’ZÈ’ZÈ’ZÇ‘YÄX¿‹S»‡OµI°|F¨t> m;˜e3\/„S&zM&qDe<]4P+G";6)"  - - -3/r0Ø–Bÿ²ÿ²ÿ±ÿµÿ·ÿ¸ÿµ -ÿ±ÿ§ÿ™ðlÏKàGçNæRêVïeønÿuÿvÿ—ÿ®ÿÂÿã6ÿð?ÿØ'ÿÚÿØÿÖ ÿÕ ÿÔ ÿÔ ÿÕ ÿÕ ÿÔ ÿÔ ÿÓÿÓÿÕÿ×ÿÑÿßÿ¾Tz1'4 " %,D& R4eAuQ.W/€V.R+{N'pB$g9W0M& A! ;( "9'$:+ “ ÌF*ÿ>ÿ>ÿ=(ÿ<'ÿA&ÿE*ÿ=ÿ"à7ÿ…&ÿ¼ÿà -ÿÑ%ÿÆݘ$ÓŽÏ›Nà¬_ò‹X³Lÿ, ÿB"ÿB$ÿB$ÿAÿC!ÿD ÿF"ÿKÿNÿ`ÿkÿ‰ÿ˜ ÿ§ÿ¯ÿ»ÿà -ÿÍÿÑÿ×ÿ×ÿ×ÿ×ÿÔÿÒÿÑ ÿÒ ÿÓÿÕÿÐÿÉÿÆÿÑ ¤]5:¨q>ÿÉ+ÿÅ'ÿÁÿÄÿÂÿÃÿÀ ÿ¾ -ÿ¿ÿÁÿÄÿÇ ÿË ÿË ÿÊ ÿÈ ÿÊÿÊÿÊÿÍÿÍÿÍÿÎÿÎÿÏ ÿÏ ÿÑ ÿÖÿÖÿÖÿÕÿÖÿÚÿÛÿÔ ö©ê”$ý§7ðªhë¥cÚ¢]ÔœWÎSÈŠM΋BÜ™Pß™.åŸ4ñš þ§ÿ¥ÿ¥ÿ¢ÿ©ÿºÿÊÿÃÿ¶ÿÉ!ñŽFb*Z3B& - -  9'  # -Rz)ÿ(ÿ&ÿ$ ÿ" ÿ%ÿ$ÿ"ÿ ö"ù% ÎGä]1ÍzGÕ‚OÀŠRÀŠR¹ŒR¹ŒR½‹T¾ŒU½‹T½‹T¾ŠT¾ŠT¿‰S¿‰SÁˆSÁˆSÁˆSÁˆSÀ‡RÀ‡R¾ˆR½‡Q½‡Q½‡Q½‡Q½‡Q¼†P¼†Pº†P¸„N¸„N¸„N·ƒM¸„N¸„N¸„N¶N·‚O¹‚O¸N¸‚L¸‚L¸‚L¸‚L·K·KµK´€J´€J´€J³~K³~K³~K±|I±|K±|K±|I°{H°{H°{H°{J¯zI¯zI¯zI¯zI®yH­zH­zH­zH¬yG¬yG¬yG«xI«xI«xI«xI«xF©vD©vD©vD©vD¨uC©vD¨uC¨uC¨uC¨uC¨uC§tB§tB§tB§tB§tE¥rC¥rC¥rC¥rC¥rC¥rC¥rC¤qB¤qB¤qB¤qB o@ o@ oB oB¡nA¡nAŸnAŸnAŸnAŸnAžm@žm@žm@l?l?l?l?l?œk>œk>œk>œk>›j=›j=™h;›j=›j=™h;™h;™h;™h;™h;¾ŠT¾ŠT¾ŠT¾ŠT¾ŠT¿‹U¿‹U¿‹U¿‹U¿‹UÁUÁUÁUÁUÁUÁUÁUÁUÁUÂŽVÂŽVÂŽVÂŽVÂŽVÃWÃWÄXÄXÄXÃWÄXÄXÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÆ’ZÆ’ZÆ’ZÈ’ZÊ”\È’ZÈ’ZÈ’XÈ’XÈ”ZÈ”ZÈ”XÆ’VÆ’VÆ’VÉ•YÉ•YÈ”ZÈ”ZÈ”\Æ’ZÅ“\ÑZÀY½ŠV¸‡U³‚P«zK¢qB™k?’d8„Z4{Q+oH(hA!\: R0F'? 3 -+   - - -qRBÿ¬&ÿ¶0ÿ³ÿ²ÿ¤ü’ÿˆü‚úmßRá;æ@ÓPåbÿŽÿ¦ -ÿ½ÿÀ!ÿ¶ÿ® ÿ«ÿ¯ÿ¶ÿÈÿîJÿÜ8ÿØÿØÿÖ ÿÕ ÿÔ ÿÔ ÿÕ ÿÕ ÿÔÿÔÿÔ ÿÔ ÿÖÿ×ÿÖÿÒ ÿä-Ø\[D -66>G%U3jA{R.‡]7ˆ^8Š]<ƒV5xN*oE!e: Y.K(C ) *1'³)æ\#ÿ=ÿ@ÿB*ÿ>&ÿA)ÿB*ÿ=ÿ) -ô!ÿ[ÿ· ÿÇ0ÿÒOçžÎŽBá¡UÕ­rÕ­ré—_©Wþ-ÿC$ÿC!ÿD"ÿEÿGÿJÿMÿPÿQÿdÿoÿ‹ÿ›ÿ¨ÿ±ÿ¼ÿÃÿÌÿÑÿÖÿÖÿ×ÿ×ÿÕÿÓÿÎ -ÿÐ ÿÓÿÔÿÐÿÉÿÆÿПZ4 Hÿ·7ÿËÿÄÿÁ ÿ -ÿÁÿÁÿÁÿÀÿÁÿÃÿÆ ÿÊÿÌ ÿË ÿËÿÌÿË ÿË ÿÌÿÊ ÿÈ ÿÊ ÿËÿÎÿÏ ÿÏ ÿÕÿØÿÖÿÖÿÔÿÒÿÚÿÝÿ¹.ë• -ã£Wë«_â¨fâ¨fã¥_Õ—QÕGØ“JÞ’Ià”Kã•Hã•HÙŽ9Ú:ã:æ’=ñ1ÿ«?ÿÂ=ÿ×RÿÓ-ÿÏ)ÿÐIñ¦·p*Ò‹EÕŸiÔžhΠp”f6;  ' %) -   gŽ,ÿ.ÿ* -ÿ' -ÿ& ÿ&ÿ%ÿ"ÿ ÿÿþ$ÿ+ çR%ÿl?ׄLׄL¼ŒS¾ŽUÁW¿‹U»ŠV»ŠVÁ‹UÀŠT‰T‰TÀŠT¿‰SÁˆSÁˆSÀ‡RÀ‡RÀ‡R¿†Q½‡Q½‡Q½‡Q½‡Q½‡Q½‡Q»†S»†S»†S»†Sº…Rº…RºƒPºƒP¹‚O¹‚O¹‚O¹‚O¹‚O¹‚O¸N¸N¸N·€M¸N·€M´Lµ€M´L³~K´L³~K³~K³~K³|I³|I±|I±|I°{J°{J¯zI°{J¯zI°{J°{J¯zI®yF®yF®yF­xE¬yG¬yG«xF«xF©vB©vB«xF©vD©vD©vD©vD©vD©vD¨uC¨uC¨uC¨uC¨uC¨uF¨uF©tF¨sE¨sE§rD§rD§rD§rD§rD¦qC¦qC¦qC¦qC¤qB¤qB¢o@¢o@¢o@¢o@¢o@¡n?Ÿn?Ÿn?Ÿn?žm>žm>žm>žm@l?Ÿl?Ÿl?žk>žk>œk>œk>œk>œk>›j=›j=›j=›j=›j=™h;™h;™h;¾ŠT¾ŠT¾ŠT¾ŠT¾ŠT¿‹U¿‹U¿‹U¿‹U¿‹UÁUÁUÁUÁUÁUÂŽVÂŽVÂŽVÃWÃWÃWÃWÃWÃWÄXÄXÄXÄXÄXÄXÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÆ’ZÆ’ZÆ’ZÈ’ZÊ”\Ê”\Ê”\Ê”ZÊ”ZÈ”ZÈ”ZÈ”XÈ”XÈ”XÉ•YÉ•YÉ•YÉ•[È”ZÅ‘YÃW¾ŒU»‰R¶ƒO²KªyG o=™h9‘`1†X,~P$qG!g=[4R+ G% A6/#  -   Ú€ÿ¹3ÿ½ ÿÂÿ«ñ‡÷}ÿˆÿzòeñKïIôqÿŸ#ÿ·ÿ´ÿµÿ³ÿ²ÿ±ÿ±ÿ² ÿµÿ¸ÿÜ8ÿéEÿÖÿØÿÖ ÿÕ ÿÕ -ÿÔ ÿÕ ÿÕ ÿÔÿÔÿÕ -ÿÕ -ÿÖÿÖÿ×ÿÖÿÚ#ÿÒ\q0^6$T,F!F!E#O-`7nE!{Q+€V0ƒV5€S2yO+oE!g<"^3P-C & ' 9 ç]$ôj1ÿ9 ÿDÿB*ÿ=%ÿA)ÿB*ÿB#ÿ8ü)ÿ@ÿªÿÓ<ø¯,Û’ÜœPé©]Ø°uØ°uè–^°^&ÿ.ÿB#ÿC!ÿE#ÿH ÿJ"ÿNÿP!ÿRÿRÿeÿpÿÿ›ÿ©ÿ²ÿ¿ÿÄÿË ÿÑÿ×ÿ×ÿØÿØÿÖÿÓÿÍ ÿÎ -ÿÓÿÔÿÐÿÉÿÆÿÑšU -2Q+õ  ÿËKÿÄÿÄÿ -ÿÁ ÿÀÿ¿ÿ¾ÿÀÿÂÿÄÿÉÿÌÿÌ ÿÊ -ÿËÿÍÿÍ ÿÍ ÿÌÿÊ ÿÈ ÿÈ ÿÍÿÏÿÏ ÿÏ ÿÕÿØÿ×ÿ×ÿÖÿÓÿÝÿÔ ë• -ôží­aî®bäªhäªhÝŸYÒ”N×’IÚ•LÚŽEÜGÚŒ?Ô†9ц1Ï„/Õ,Ö‚-ÕÙ…Ö‹åšø¯ ÿ¾ÿ·0åš·AψBÔžhè²|â´„Ü®~¸–xE#   % f.ÿ) ÿ(ÿ' -ÿ' -ÿ&ÿ&ÿ"ÿ ÿÿü"ùÍ8 æQ$Év>ÖƒK¾ŽUÀWÂŽXÁW¾Y¾YÁ‹UÀŠT‰T‰TÀŠTÀŠTÁˆSÁˆSÁˆSÀ‡RÀ‡RÀ‡R¾ˆR¾ˆR¾ˆR½‡Q¾ˆR¾ˆR¼‡T¼‡T¼‡T»†S»†S»†S¼…R¼…R¼…RºƒPºƒPºƒP¹‚O¹‚O¹‚O¸N¸N¸N¸N¸N¶Nµ€Mµ€Mµ€M´L´L³~K³~Kµ~K³|I±|I±|I±|K±|K±|K±|K°{J°{J°{J°{J¯zG®yF®yF®yF¬yG¬yG¬yG¬yG«xD¬yE¬yG«xF©vD«xF«xF«xF«xF¨uC¨uC¨uC¨uC¨uC¨uF¨uF©tF¨sE¨sE¨sE¨sE¨sE§rD§rD§rD§rD¦qC¦qC¤qB¤qB¤qB¢o@¢o@¢o@¡n?¡n?žm>Ÿn?Ÿn?žm>žm>žm>žm@l?Ÿl?Ÿl?žk>žk>œk>œk>œk>œk>œk>›j=›j=›j=›j=›j=›j=›j=¾‰V¾‰V¾‰V¾‰V¿ŠW¿ŠW¿ŠW¿ŠW¿ŠWÁŒYÁWÁWÁWÁWÂŽXÂŽXÂŽVÃWÃWÃWÃWÃWÃYÄZÄXÄXÄXÄXÄXÄXÄXÅ‘YÅ‘YÅ‘YÅ‘YÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÈ”\È”\È”\È”\È”ZÉ•[É•]È”\È”ZÈ”ZÈ”ZÉ•[É•]É•]È”XÄT¿‹S»‡O³‚N®}I¤vFo?‘h9ˆ_0~X1uO(fD#^<R4H*:#3 -'  )1DF!HGA=3. + $  \)à­lÿÅÿÙ%ÿÓÿÀ ÿÆ ÿÔÿÌ0ÿäiý‚ÿª ÿ®ÿ™ÿÿŒÿÿŽÿ‘ÿœÿ¬$ÿ·ÿ³ÿÅÿî>ÿâ.ÿÖ"ÿ×ÿÖÿÕÿÕÿÔ ÿÕÿÔ ÿÔ ÿÖÿÖÿÖ -ÿÖ -ÿÖ ÿÖ ÿ×ÿÜ÷¯)¨`”Q(ŠGv>g/Y'V$R(Y/ c8lA%mB&j?#e<\3Q,N) J'<! BqBÿ+ûiÿ8 ÿHÿC+ÿ=%ÿ?+ÿC/ÿN0ÿI+ÿA ÿHÿŠCÿ©bêFì’HØ©eâ³oà®qÝ«nï–b»b.ÿ- ÿA ÿEÿFÿHÿL ÿNÿOÿRÿQÿfÿsÿŽÿœÿªÿ´ÿ¿ÿÃÿÍ ÿÓÿØÿØÿØÿØÿÕÿÒÿÎÿÑÿÓÿÓÿÐÿÊ ÿËÿÆ“N1]Ð…ÿÊÿÄÿÂÿÂÿÀ ÿ¿ -ÿÀÿ¿ÿÀÿÀÿÃÿÅÿÊÿÌÿË ÿÇÿÌÿÐ -ÿÏ ÿÒ ÿÎ -ÿÎ -ÿÍ ÿÎ ÿÒÿÒÿÎÿÎÿÓÿØÿÖÿÙÿÚÿÕÿÞ5ÿ¸Ûˆø¥6ì²mé¯jߥ`â¨cÜ¢`ç­kç«nÝ¡dÝŸbÖ˜[דXÓTÕŽFÒ‹CʉKɈJljJÁƒD¼y2¹v/µq6À|A¿~;¹x5¼„DÌ”TÒ eÕ£hÊ—cµ‚NÆše­L!  - -  - - !      r¡?ö<õ;ø7õ4ý)ü(ÿ#ÿ!ÿÿ ÿ" ÿýÿ$ûC ÿX5ãKé…QÉŒTÈ‹SÁWÁW¿‹U¿‹UÁ‹UÁ‹UÁ‹UÁ‹U¿ˆUÀ‰VÀŠT¿‰S¿‰S¾ˆR¿‰S¿‰S¿‰S¾ˆR¾ˆR¾ˆR¾‡T¾‡T¾‡T½†S½†S½†S¼…R¼…R¼…RºƒPºƒPºƒPºƒP¹‚O¹‚O¹‚O¸N¸N¸N¸N¸N¸Nµ€Mµ€Mµ€Mµ€M´L³~Kµ~Kµ~K³|I³|I³|K³|K³|K³|K³|K²{J²{J±zI¯zG¯zG®yF®yF­xG®yH®yH®yH®yF®yF¬yG­zH­zH¬yG¬yG«xF«xF©vD©vD©vDªuG©tF©tF©tF©tF¨sE¨sE¨sE¨sE¨sE§rA§rA§rA§rA¦q@¦q@¦q@¥p?¥pB¥pB£n@£n@¥pB¥pB¤qB¢o@¢o@¢o@¡n?¡n? m> m> m@Ÿl?Ÿl?žk>Ÿl?Ÿl?žk>žk>žk>žk>j=j=j=j=j=j=¾‰V¾‰V¿ŠW¿ŠW¿ŠW¿ŠW¿ŠWÁŒYÁŒYÁŒYÁWÁWÂŽXÂŽXÂŽXÂŽXÂŽVÂŽVÂŽVÃWÃWÃWÄZÄZÃWÃWÄXÄXÅ‘YÅ‘YÄXÅ‘YÅ‘YÅ‘YÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÈ”\È”\È”\È”\È”\È”ZÉ•[É•]È”\È”ZÈ”ZÈ”ZÆ’XÅ‘YÂŽV¿‹O»‡K´€H­yA¤s?œk7b2ˆZ*}T%tKhBa;S1I'=6*"    %/E*N3fA$lG*nD%mC$hA!c<X6Q/J+B#5."  - 4¥r1ÿÅÿÑÿé5ÿÝ)ÿÞ#ÿå*ÿàDÿµÿŒÿ¢ ÿ¯ÿ—û‡ÿÿˆþ€òrêjêpÿ…ÿ¡ÿ²ÿÁÿØ(ÿå1ÿ×#ÿÖÿÖÿÕÿÕÿÔ ÿÕÿÔ ÿÔ ÿÖÿÖÿÖ -ÿÖ -ÿÖ ÿÖ ÿÔ ÿØÿÍG㛬i@¦c:W0I"k9_- P&N$Q& -S( U*S( O&I B@@6!F’c@ÿ›7ï]ÿ7 ÿEÿD,ÿ@(ÿ@,ÿF2ÿR4ÿW9ÿi4ÿm8ÿo(÷fà†<ï•Kâ³oâ³oÛ©l×¥hæY²Y%ÿ* ÿA ÿEÿG ÿIÿL ÿOÿOÿQÿQÿfÿsÿŽÿœÿªÿ´ÿ¿ÿÃÿÍ ÿÔÿÙÿÙÿØÿØÿÔÿÒÿÎÿÑÿÒÿÒÿÑÿÊ ÿÑ!ÿÂ’MA,±fÿÃUÿÉÿÂÿÃÿÁÿÀ ÿ¿ -ÿ½ÿ½ÿÀÿÁÿÅÿÆÿÊÿÊÿÊ ÿÈ ÿÎÿÒ ÿÐ -ÿÓ ÿÓÿÓÿÒÿÕÿ×ÿÒÿÐÿÎÿÐÿÑÿÒÿÕÿÖÿÝÿá8ã—å’#ýª;ù¿zë±lå«fà¦aÀ†D«q/µy<ÇJÜžaèªmó¯tîªoæŸWãœTÜ›]ÐQŇHËNˈA³p)ÃDΊOdžCÓ’OÌ”TÕ]Ó¡fÕ£hÊ—cÃ\³‡RÅ™dfM-   & -   1"$  wÉg9ÿaÿYÿSÿGÿ0ÿ,ÿ$ÿ"ÿ ÿ# ÿ# ÿ" ÿ!ÿ ã+ñ9Ïk7ä€LÊUÈ‹SÁWÁWÂŽXÁWÁ‹UÁ‹UÃWÁ‹UÀ‰VÀ‰VÀŠT¿‰SÀŠTÀŠTÀŠTÀŠT¿‰S¿‰S¾ˆR¾ˆR¾‡T¾‡T¾‡T¾‡T¾‡T½†S½†S½†S¼…R¼…R¼…RºƒPºƒPºƒP¹‚O¹‚O¹‚O¸N¹‚O¸N¸N¸N¶N¶Nµ€Mµ€M´Lµ€M¶L¶L¶Lµ~Kµ~M³|K³|K³|K³|K²{J²{J³|K°{H°{H°{H¯zG¯zI®yH¯zI®yH®yF®yF¬yG­zH­zH­zH¬yG¬yG¬yG«xF«xF©vDªuGªuG©tF©tF©tF©tF¨sE¨sE¨sE¨sE§rA§rA§rA§rA¦q@¦q@¦q@¦q@¦qC¦qC¦qC¦qC¥pB¥pB¤qB¤qB¢o@¢o@¡n?¡n?¡n?¡n? m@ m@ m@ m@ m@Ÿl?Ÿl?Ÿl?žk>žk>žk>žk>j=j=j=j=¿ŠW¿ŠW¿ŠW¿ŠW¿ŠWÁŒYÁŒYÁŒYÁŒYÂZÂŽXÂŽXÂŽXÂŽXÂŽXÂŽXÂŽVÂŽVÃWÃWÃWÄXÄXÄXÄXÄXÄVÄVÅ‘YÅ‘YÄXÅ‘YÅ‘YÅ‘YÆ’ZÆ’ZÆ’XÆ’XÆ’XÆ’XÆ’XÈ”ZÈ”ZÈ”ZÈ”ZÈ”ZÈ”ZÉ•[Ë•]Ë•]Ë•]Ë•]Ê”\ÅW½ˆU·‚O²€I©w@ o@˜g8Œ_6ƒV-vP-oI&aB%W8K1C)3 -* !*9AR)]4pC}P'“\0œe9¡j7 i6œg6še4•`2X*ŒU'„MtAi6 [,P!C</ -*  J&ö¤'ÿÅHÿÞ ÿ×ÿÏÿÕÿ¿ -ÿ¨ÿ¨ÿ¶ÿ®ÿ¤ÿ˜ÿýmî^ÞNÎ>Ã5Ç9ÖRÿ†1ÿµÿÄ%ÿá5ÿâ6ÿÖÿØÿÕ ÿÕ ÿÕÿÕÿÕ ÿÕ ÿÖ ÿÖ ÿÖ ÿÖ ÿÕ ÿÔ ÿÓÿÓÿÔÿÂ×+Ç}·qK§a;‹T&}Fj9 \+N!IA?94/-+)    -KºDÿœ+ðZÿ3 ÿCÿJ1ÿG.ÿD-ÿE.ÿO/ÿ]=ÿmCÿpFÿEó ïN!ÿ}Pÿ¶~þ¬tä©rÚŸhïg>Ã;ÿ* ÿ? ÿEÿI ÿNÿNÿOÿPÿQÿQÿiÿwÿŽÿœÿ«ÿµÿ¿ÿÂÿÎ ÿÕÿÚÿÚÿÙÿÙÿÔÿÒÿÑÿÑÿÑÿÒÿÑÿÎÿÇAÿµ/Ò‡0¶k h(›c#nB>B’Gÿ³ ÿÊ"ÿÄÿÄÿÄÿÂÿÁÿÀÿ¾ÿÀÿÄ ÿÆ ÿÈÿÉÿËÿÈ ÿÄÿË ÿÑÿÐÿÓ ÿÕÿÕÿÖÿÔ ÿÊÿ±ÿÿžÿ® ÿÂÿÉÿÏÿÔÿÖÿä ÿÆ1݈ë Iô©Rð¶xߥgÜ©cÊ—Q\13B!S2(yPE”k`®ƒgØ|Õ¤wÓ¢uÖbÔ›`²i4¡X#ȉOÒ“YÍ•WߧiËšZÈ—W¼”^¢zD”jKÈžͪ|Ì©{›…d -$^(I"‡O(u=^,J; +%'(+#% .B*1 - %iË6ÿ—ÿ“ÿ”ÿ‘õvçhðGä;ÿ'ÿ(ÿ#ÿ#ÿ"ÿ!ÿ ÿ"ÿ<ÿX3Ý‚Já†NÇŒWÈXÃYÂŽXÄŽXÄŽXÁXÀŽWÃŒ[ÁŠYÀŠT¿‰SÀŠTÀŠT‰T‰TÀŠTÀŠTÀŠT¿‰S¿‰S¾ˆR¾ˆR¾ˆR¾‡T¾‡T½†S½†S½†S½†S¼…R¼…RºƒP¼…R¼…RºƒPºƒPºƒP¹‚O¹‚O¸N¸N¸N¸N·€M¸N´Lµ€Mµ€M´Lµ€M´L¶L¶Lµ~Kµ~Kµ~Kµ~Kµ~K³|I±|I±|I°{J°{J°{H°{H¯zG¯zG¯zG®yF¯zG®yF­zF­zF­zF¬yE¬yE«xD«xF«xF©vD«xF«xI©vG©tCªuD©tC©tC©tC©tC§r?§r?§r?§r?§r?¦q>§r?¦q>¦q@¦q@¦qC¦qC¦qC¥pB¤qB¤qB¤qB¢o@¢o@¡n?¡n?¡n?¡n? m> m@ m@ m@ m@Ÿl?Ÿl?Ÿl?Ÿl?žk>žk>žk>žk>žk>j=¿ŠW¿ŠWÁŒYÁŒYÁŒYÁŒYÁŒYÂZÂZÂZÂŽXÂŽXÃYÃYÃYÃYÂŽVÂŽVÃWÃWÃWÄXÄXÄXÄXÅ‘YÅ‘WÅ‘WÅ‘YÅ‘YÅ‘YÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÈ”ZÈ”ZÈ”ZÈ”ZÈ”ZÈ”ZÈ”ZÈ”ZÈ”ZÈ”ZÉ•[É•[Ë•]Ë•]Ë•]Ì–^Ç‘Y¿‰Q¶N­xE¤r;šh1^/†U&{N%qDc=Z4J+A"6/!  - "8D(V4`>$tK)€W5`7˜kBªsG²{O¸N½†S¼‡V»†U¸ƒUµ€R·€R°yK¢oD˜e:ˆY4~O*lA"`5P+E 80% - - ·eÿ¿BÿÌÿÜÿÕÿ¸ü’ÿœÿ¸ÿÀ ÿ¾ÿ¾ÿ© ÿ†ôdéYÙIÑAÅ7¾0°,ÅAè€ÿ±ÿÚ.ÿä8ÿÙÿÖÿÖ ÿÔ ÿÔ ÿÕÿÖ -ÿÖ -ÿÖ ÿÖ ÿÖ ÿÖ ÿÔ ÿÔ ÿÔÿÔÿÔÿÑôªHÙ-Óg¾xRªsE›d6†U(uD`3R%A8. , -%"#) $   -  Mלaÿ–%íWÿ2ÿ?ÿG.ÿH/ÿC,ÿ>'ÿG'ÿQ1ÿQ'ÿEÿ2 -ô!Ã"ôS&áWê˜`í²{Ñ–_ÍE¶.ÿ0ÿ? ÿEÿI ÿOÿOÿOÿQÿRÿRÿjÿwÿŽÿÿ®ÿ¶ÿ¿ÿÂÿÎ ÿÖÿÚÿÚÿÜÿØÿÓÿÐÿÑÿÏÿÒÿÖ ÿÔ"ÿÊÿ³-ùª$û°Yÿ¼eú‚ùÁáµ…®‚RR. #>C…:åšJÿÈ ÿ½ÿÄÿÃÿÃÿÂÿÁÿ¾ÿ¾ÿÀÿÅ ÿÉÿÌÿÍÿËÿÆ ÿÃÿÎÿÔÿÐÿÔ ÿÔ ÿÖÿÛÿÍÿ­ù“îˆë€ü‘ÿ¹ ÿÉÿÎ ÿÑÿÖÿä ý¨âêŸHÿµ^ì²tâ¨já®h]$-)$&>f5‡V)¹€EÊO¢Y$̃NÍŽTÏVÖž`Ôœ^ËšZÈ—W¢zDK#[1‹h:°_ŠtS* gH2°zS·Z½…^¹Zµƒ_©wS›kMˆX:nC'^3F -DI*T5H.3)7 -<$+ "fσ:ÿŸÿ›ÿ£ÿ©ÿ£ÿ›ÿsÿVÿ/ÿ(ÿ$ÿ#ÿ# ÿ"ÿ' -ÿ%ò/ -ÿ>Ëp8á†NÈXÈXÃYÃYÅYÅYÁXÀŽWÃŒ[ÁŠYÁ‹UÀŠTÀŠTÀŠT‰TÊUÀŠTÀŠTÀŠT¿‰S¿‰S¿‰S¾ˆR¿‰S¿ˆU¾‡T¾‡T½†S¾‡T½†S½†S½†S½†S½†S½†S½†S¼…R¼…RºƒP¹‚OºƒP¹‚O¸N¸N¹‚O¸N¶Nµ€Mµ€M¶Nµ€Mµ€M¶L¶L¶L¶L¶Lµ~Kµ~Kµ~K±|I±|I±|K°{J°{H°{H°{H°{H¯zG¯zG¯zG¯zG®{G­zF­zF­zF¬yE¬yE¬yG¬yG¬yG«xF«xI«xIªuDªuD©tC©tC©tC©tC©tA©tA©tA¨s@¨s@¨s@¨s@§r?¦q@¦q@¦qC¦qC§rD¦qC¥rC¥rC¤qB¢o@¢o@¢o@¡n?¡n?¡n?¡n? m@¡nA m@ m@ m@ m@Ÿl?Ÿl?Ÿl?žk>žk>žk>žk>žk>ÁŒYÁŒYÁŒYÁŒYÁŒYÂZÂZÂZÂZÃŽ[ÃYÃYÃYÃYÃYÃYÃWÃWÄXÄXÄXÄXÄXÄXÅ‘YÅ‘YÄVÅ‘WÅ‘YÆ’ZÆ’ZÆ’ZÆ’ZÈ”\È”\È”\È”ZÈ”ZÈ”ZÈ”ZÈ”ZÉ•[É•[É•[É•YÉ•YÉ•YÉ•YÌ–^Ì–^Ì–^Ë•]ÆX¾ˆP²€I¥s<—d2Z(€O$uDf=[2H(?,% - -% /? L-h;tG"„S(^3™f7¢o@®xB³}Gº„LÀŠRÃSÈ”Xʘ]Ë™^Ìš_ÎœaÔ—_Ñ”\ÍVÉŒRÀ…LµzAªq< g2•[.‰O"wBl7 ^,V$G:+' JÕœeÿÆÿÖ,ÿÐÿ ÿÿ°ÿÆÿÆÿÈÿ¾ÿŸ÷zø\èLÙEÔ@Ç6É8Æ<·-­7ú„2ÿÂ-ÿÝHÿã3ÿÑ!ÿÔÿÔÿÔÿÕÿÖ ÿÖ ÿÖ ÿÖ ÿÖ ÿÖ ÿÖÿÖÿÓÿÔÿÔÿÛ ÿÌ,ë¢ã™[Ý“UʈS»yD¥l5—^'…P"u@^/N<5 -+'!( )"  -  -p,ë§SÿˆïUÿ1 ÿ9ÿ?'ÿD,ÿB+ÿ=&ÿ:"ÿ<$ÿ=!ÿ< ÿ= ÿ0øðÓð:þfEäL+ãæ"ÿ1ÿ>ÿDÿJ ÿOÿOÿPÿQÿSÿRþiÿwÿÿžÿ¯ÿ·ÿ¿ÿÂÿÎÿÖÿÛ ÿÛ ÿÚÿÖÿÓÿÐÿÐÿÐÿÒ#ÿÔ%ÿÇGÿ´4î¦UòªYíµnì´må«fÞ¤_å§aé«e½9œ^·oÂz$÷ÿÀÿÃÿ½ÿ ÿÁ ÿÂÿÁÿÁÿÁÿ¾ÿÂÿÆ ÿËÿÏÿÊÿÅÿÂÿÌÿÓÿÔÿÖÿÓÿÖ ÿÐ$ÿÒ&ÿ™ årÿ ÿ¼(ÿ§ÿ–ÿ£ÿ½ ÿÒ ÿËÿÐÿÛê -î‘ã¦[ð³hâ­ná¬mä¬g‘Y@ 5*'((/ * .P$´wAÔ—aÈŽNЖV×£g̘\»a¶Š\ŠjR+   *JŽT'À†YÇŒQÁ†K¼‡Hº…FÀIÀI¾K¼I¹€K±xC¤qB«xI¾[ÏžlºŽ`Žb4Q$JB3 ' -dÕŽBÿ¨ ÿ£ÿ­ ÿ¯ ÿ±ÿµÿ²ý›ÿ_äDÿ*ÿ*ÿ'ÿ'ÿ' -ÿ& ÿ'ÿ'îMÿn?Ò‰TÒ‰TÅYÅYÃYÃYÁXÀŽWÃŒYÃŒYÁ‹SÁ‹SÁ‹SÁ‹SÃUÁ‹S‰T‰TÊU‰T¿‰SÀŠTÀŠT¿‰S¿ˆU¿ˆU¿ˆU¿ˆU¿ˆU¾‡T¾‡T¾‡T½‡Q½‡Q½‡Q½‡Q¼†P¼†P¼…R¼…R¾„R¼‚PºƒPºƒP¹‚O¹‚O·‚O·‚O¸N¸N¶Nµ€M¶L¶L¶L¶L¶L¶Lµ~K¶L³~K±|I±|K±|K°{H°{H°{H°{H°{H¯zG¯zG¯zG¯zG¯zG­zF­zF­zF­zF¬yE¬yE¬yG¬yG«xI«xI©vD©vD¨uC¨uC¨uA¨uA©tA©tA©tA©tA¨uA¨uA¨s@¨s@¨sB¨sB§tE¦sD§rD§rD¥rC¥rC¤qB¥rC¤qB¢o@¢o@¢o@¢o@¢o@¢oB¡nAŸn?Ÿn?¡n? m> m>Ÿl= m>Ÿl=žk<žk<žk<žk<ÁŒYÁŒYÁŒYÁŒYÁŒYÂZÂZÂZÂZÃŽ[ÃYÃYÃYÃYÃYÃYÄXÄXÄXÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÆ’ZÆ’ZÅ‘WÆ’XÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÈ”\È”\È”\È”ZÈ”ZÈ”ZÈ”ZÈ”ZÉ•[É•[É•[É•YÉ•YÉ•YÊ–ZÍ—_Í—_Í—_Ë•]Ç‘YÁ‹S´‚K¦t=”a/ŠW%zIm<Y0K"80  - + -=# I/]>!iJ-‚U0Ža<œk@¥tI­zK³€Q¼†P¿‰SÅWÊ”\̘\ÑaÑŸdÓ¡fÕ£hÖ¤iߢjÞ¡iߢhߢhÚŸfÖ›bÏ–aÈZˆ[¹R¨sIœg=Ž\8„R.rH&a7S.I$81)!"-Tÿ¼ÿÄÿÁÿ­ÿ©ÿ¿ -ÿÑÿË ÿÈÿ¿ÿ›ðsóWàD×CÑ=Ç6Ç6Â8Ã9¢,Ö`ÿ¬ÿË6ÿí=ÿ×'ÿÓÿÔÿÔÿÔÿÕ ÿÕ ÿÖ ÿÖ ÿÖ ÿÖ ÿÕ ÿÕ ÿÓÿÔÿÔÿ× ÿÔ4ù°Ó‰Kâ˜ZÓ‘\Í‹V»‚K«r;še7‰T&qB!a2L!C6-%--&   …Añ­Yÿ íSÿ0ÿ4 ÿ7ÿ?'ÿA*ÿ=&ÿ6ÿ5ÿ;ÿA%ÿ?"ÿ7ÿ+öǺ¦§Þî*ÿ3ÿ=ÿDÿJ ÿPÿPÿQÿQÿRÿRÿjÿwÿÿžÿ¯ÿ·ÿ¿ÿÄÿÎÿÕÿÛ ÿÛ ÿÖÿÔÿÐÿÎ ÿÐÿÒÿÒ#ÿÊÿ²2ô§'ô¬[ù±`ê²kä¬eߥ`ÖœWÏ‘KÒ”NÝŸYã¥_è Jð¨Rÿºÿ ÿÂÿÁÿÁ ÿÀ ÿÁÿÁÿÂÿÂÿÂÿÄÿÈÿËÿÎÿÊÿË ÿË ÿÏ -ÿÑ ÿÔÿÓÿÑÿÚÿÐ$ÿ­Øe×dû†ÿ®ÿË*ÿ® ÿ›ÿ¶ÿÌÿÈÿÔÿÎÝ€ì ÚRì¯dá¬mã®oÌ”OÉ‘LÃŽ`°{M‚S.Z+>)$"#K -Ó–`Ñ”^Ë‘QÞ¤d̘\½‰Mi=T(>#   Y+£u^Ï•hÅ‹^ĉN‡L½ˆI½ˆIÄLÀI¼Iº}G·~I¸JÅ’cÕ¢sÎk»ŠXª~P¥yK‹^;uH%[1R( D) .i"è¡Uÿ» ÿ±ÿ¶ÿÁÿºÿ¸ÿ¹ -ÿµÿÿdÿ7ÿ1ÿ+ -ÿ)ÿ) ÿ) ÿ'ÿ'á@úY*φQÔ‹VÅYÅYÃYÄZÂYÂYÄZÄZÃUÃUÁ‹SÄŽVÅWÃUÊUÊUÊUÊUÁ‹UÁ‹UÀŠTÀŠTÀ‰VÀ‰V¿ˆU¿ˆU¿ˆU¿ˆU¾‡T¾‡T¾ˆR¾ˆR¾ˆR½‡Q½‡Q¼†P¼…R¼…R¾„R¾„RºƒPºƒPºƒP¹‚O·‚O·‚O¹‚O¸N¶N¶N¶L¶L¶L¶L¶Lµ~K¶L·€M´L±|I±|K±|K±|I°{H±|I°{H°{H¯zG¯zG°{H°{H¯zG®{G®{G­zF­zF­zF­zF­zH¬yG«xI«xI«xF©vD©vD©vD¨uA¨uA©tA©tA©tA©tA¨uA¨uA¨s@¨s@¨sB¨sB§tE§tE§rD§rD¦sD¦sD¥rC¥rC¤qB¢o@¢o@¢o@¢o@¢o@¢oB¢oB o@Ÿn?¡n?¡n?¡n?¡n? m> m> m>Ÿl=Ÿl=Ÿl=ÂZÂZÂZÂZÂZÃŽ[ÃŽ[ÃŽ[ÃŽ[Ä\ÄZÄZÄZÄZÄZÄZÄXÄXÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÆ’ZÆ’ZÅ‘YÆ’ZÆ’ZÆ’ZÆ’ZÈ”\È”\È”\È”\È”\È”ZÈ”ZÈ”ZÈ”ZÉ•[É•[É•[É•[É•YÊ–ZÊ–ZÊ–ZË—_Ë—_̘`Ë—_É•]Å‘Y¼ˆP°|D k8’]*ƒN"s>W- I3' -5E)a<nI)…X1c<Ÿl=§tE²|F¹ƒM»‡O¿‹SÆTÈ’VÑ–[לaØ£dØ£dÙ£gÚ¤hÚ¤hÛ¥iÚ¤hÚ¤hÛ¥iÛ¥iÚ¤hÚ¤hÜ£hÛ¢gÛ¢iמeÓšcÌ“\ÅŠS½‚K³xA¤i2—^'ŽU‚Hw= f2 -Z&IB.DÖŽÿºGÿ³ÿ´ÿ»ÿÉÿÕÿÏÿÍÿÈÿ¡êrîQáDÔ@Ð<Ê7Æ3Æ8É;¨+ÆIõ™*ÿ¯@ÿã;ÿâ:ÿØÿÔÿÔ ÿÔ ÿÕ ÿÕ ÿÖ -ÿÖ -ÿÖ ÿÖ ÿÕÿÕÿÕÿÖÿÕ ÿÓ -ÿÖÿÇ҉ɀʈQʈQ¹ƒM²|F¦sA–c1M'n<a2 V'G;. 3/) -  žUô«Dÿ|êNÿ, ÿ3ÿ4ÿ8"ÿ>'ÿ>'ÿ;%ÿ9#ÿ5 ÿ5 ÿ5ÿ4ÿ1ÿ)ÿ& öíëÿ!ÿ+ÿ5ÿ=ÿEÿM"ÿPÿPÿPÿPÿTÿUþlÿyÿŽÿÿ®ÿ¶ÿ¿ÿÄÿÍÿÔÿÚ'ÿÛ(ÿÔÿÍÿË -ÿË -ÿÍ ÿÑÿÊ<ÿ¸*ï¢Pò¥Sò³{ñ²zâ±oß®lߤiÑ–[ɉAÐHâœDè¢Jý¦ÿ±$ÿ¼ÿÁÿÀÿ¿ÿÀÿÀÿÀÿÁÿÁÿÂÿÄÿÅ ÿÈÿÑ#ÿÍ4ÿÂ)ÿÂ<ÿÑKÿÞIÿÙDÿÕÿÐÿÐ ÿÖÿ½+å{ËJæeïzÿœÿÌÿÅÿ¨ÿ¶ÿÊÿÇÿÑ"ÿÃÖwè‰מTé°fà­jÖ£`ʉDÐJߤ[ã¨_ì¬`ä¤XÑ“R¹{:›b+‹Ri-R6+,u?Ú™]ʉMÞ¦fð¸x¾`™k;* -  - -* -©p;ÕœgÓ™WÇKÈŽNÈŽNÇŒUÅŠSĉPĉPÈQ¼J·|GÈXØfÕšcÀ‡N¸Fµ|A´{@¼ƒLµ|E®xB«u?™h4†U!ŽM ÍŒLò§Pë IÚ—;àAÿÀ>ÿ¿=ÿÁÿÁÿ»ÿ¡ÿ`ëCÿ(ÿ)ÿ) ÿ) ÿ)ÿ)ÿ.ÿAÞ}Hê‰TÈVÇŽUÇŽYÈZÅ‘[Å‘[ÅYÅYÄŽVÃUÃUÄŽVÅWÄŽVÆVÆVÆVÅŒUÊUÊUÊUÊUÀŠTÀŠTÀŠTÀŠT¿ˆU¿ˆU¿‰S¾ˆR¾ˆR¾ˆR¾ˆR¾ˆR¼†P¼†P¼…R¼…R¾„R¾„R¼…RºƒPºƒPºƒP·‚O·‚O¹‚O¸N¸N¸N¸N¸N¸N¸N¸N·€M·€M·€Mµ€M³~K³~K±|I³I³I±|I±|I±|I±|I±|I°{H°{H°{H¯zG¯zG®{G­zF­zF­zF­zF­zF­zH¬yG«xF«xF©vD©vD©vB«xDªuBªuB©tAªuB¨uA¨uAªuB©tA©tC¨sB§tE¦sD§rD§rD¦sD¦sD¦sD¥rC¥rC¤qB¢o@¢o@¤qB¢o@ oB oB o@ o@¡n?¡n?¡n?¡n?¡n? m> m> m>Ÿl=Ÿl=ÂZÂZÂZÂZÂZÃŽ[ÃŽ[ÃŽ[ÃŽ[Ä\ÄZÄZÄZÄZÄZÄZÄXÄXÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÅ‘YÆ’ZÆ’ZÅ‘YÆ’ZÆ’ZÈ”\È”\È”\È”\É•]É•]É•]É•[É•[É•[É•[Ê–\Ê–\Ê–\Ê–\Ê–ZÊ–ZË—[Ë—[Ë—_Ë—_̘`Ï›c̘`Ê–^ÄX½‰Q±|I¥p=”_3ƒN"f<V,<.(B&U9"uP0„_?šmF£vO²P·„UÀŠTÅYÆ’ZÊ–^Ï™]Òœ`ÚŸdÝ¢gÙ¤eÙ¤eÙ£gÚ¤hÛ¥iÚ¤hÚ¤hÙ£gÙ£gÚ¤hÛ¥iÛ¥iݤiÞ¥jߦmߦmߦoߦoߤmÜ¡jÙžgΓ\ÊS¿†O·}Kªp>—c;‰U-yJ!l=W-A‘Iï§4ÿ³ÿ±ÿ¿ÿÍÿ×ÿÏÿÏÿÏÿ«ìtðSäGÓ?Ï;Ë8Ë8Ç9Ê<«.´7èŒÿ§8ÿÊ"ÿç?ÿÞ!ÿÕÿÔ ÿÔ ÿÕ ÿÕ ÿÖ -ÿÖ -ÿ× ÿÖ ÿÕÿÕÿÕÿÕÿÖ ÿÓ -ÿÐ ÿÜô«=«b«i2³q:«u?¬v@¢o=™f4„R,r@e6\-M#B22,%  -  «bî¥>ÿu èLÿ+ -ÿ2ÿ1ÿ1ÿ4ÿ:#ÿ='ÿ;%ÿ5 ÿ3ÿ0ÿ.ÿ,ÿ*ÿ)ý$ ù# ú$ -ÿ'ÿ/ÿ7ÿ>ÿGÿM"ÿPÿPÿOÿPÿUÿUþlÿyÿŽÿœÿ­ÿµÿÀÿÄÿÎÿÚÿã0ÿÛ(ÿÏÿÌÿË -ÿÈÿË -ÿÏÿ¿1ø¥êKù¬Zó´|ï°xÞ­kÚ©gÒ—\¼FLJ?Û›SãEì¦Nÿ¸+ÿÁ4ÿÂÿÁÿ¾ ÿ¿ÿ¿ÿ¿ÿÀÿÁÿÂÿÂÿÃÿÌÿÓ%ÿÃì–ÏyÀoÔƒë¡ ÿÁ,ÿÞ#ÿ×ÿØÿØÿ¥Òhäcîmò}ÿ‘ÿÁ ÿÍÿ¶ÿ¶ÿÉÿÇÿÕ&ÿµÐqå†Ü£Yæ­cå²oÊ—TÂ<ʉDÈ?ĉ@ÍAÑ‘EÝŸ^ß¡`à§pÜ£lÕ™iÊŽ^°~^ŠX8F‹U0×–ZdžJå­mòºz¿‘ak= &' @ ÆXÈZÇKÈŽLÉOÈŽNÈVÈVÈTÇŒSÅŠSÁ†O‡RÚŸjÞ£lËYÁˆO¾…L¼ƒHºFºJºJ¸‚L¸‚Lµ„P±€L´s3µt4Ë€)Ð….Â#µrß…ÿ¦$ÿÄÿÃÿÆÿÁÿŽ+ÿbÿ8ÿ*ÿ* ÿ* ÿ)ÿ)ÿ-ÿ6Õt?è‡RÉWÉWÉ[ÈZÆ’\Å‘[ÅYÅYÄŽVÃUÄŽVÅWÅWÄŽVÆVÇŽWÇŽWÆVÅŒWÅŒWÊUÊUÁ‹UÁ‹UÀŠTÀŠTÀ‰VÀ‰V¿‰S¿‰S¾ˆR¿‰S¿‰S¾ˆR¾ˆR¾ˆR½†S½†S¿…S¾„R¼…R¼…RºƒP¼…Rº…Rº…RºƒPºƒP¹‚O¹‚O¹‚O¹‚O¹‚O¹‚O¹‚O¸N¸N·€Mµ€Mµ€M´L´L³I³I³~K³~K±|I±|I±|I°{H°{H¯zG¯zG¯zG®{G®{G­zF®{G®{G­zF­zH­zH¬yG¬yG¬yG¬yG«xD«xD¬wD¬wD¬wDªuB©vB©vBªuBªuBªuD©tC¨uF¨uF©tF¨sE§tE§tE¦sD¥rC¥rC¥rC¥rC¥rC¤qB¤qB¢qD oB o@ o@¢o@¡n?¡n?¡n?¡n?¡n? m> m>Ÿl=Ÿl=ÂŽXÂŽXÃYÃYÃYÃYÄZÄZÄXÄXÅ‘WÅ‘WÅ‘WÅ‘WÅ‘WÅ‘WÅ‘YÅ‘YÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÈ”\È”\Æ’ZÈ”\È”\È”\É•]É•]É•]É•]É•]É•]É•[É•[É•[É•[Ê–\Ê–\Ê–\Ê–\Ê–ZÊ–ZË—[Ê–ZÊ–^Ê–^̘^̘^̘\Í™]Ë—[É•YÃS¸„H°zD¢l6‰X)yHa7 Q'=1 -$$>$S9&nL+~\;–lB vL·€RÀ‰[ËYÑ–_ј]ÖbÕŸcØ¢fØ¢fÙ£gÞ¤fÝ£eÙ¤eØ£dÛ¦gÛ¦gܤfܤfܤdÞ¦fÞ¦hÞ¦hÝ¥gÞ¦hߧiÞ¦hߦkߦkà¥jà¥jà¥jà¥jܦjÚ¤hÔ fÐœbË—]Æ’X¼ˆN±}C£i7U#‹DÃ|6ÿ¯ÿ´ÿÆÿÛ ÿÛÿÑÿÒÿ× ÿ¿ì…ïSãGÓ@Ï<Ð:Ð:È7Ë:À8­%ÓdÿžLÿµÿÕ9ÿâ'ÿ×ÿÔÿÔÿÕ ÿÕ ÿÖ ÿÖ ÿ× ÿÖ -ÿÖÿÖÿÕ ÿÕ ÿÖ ÿÔ ÿÎ ÿÖÿÎ$³k~6“KƒO'ŒX0‚S,R+{N)oBb5Y, J @62.-  - ²bë›:ÿuéLÿ+ ÿ3ÿ4ÿ-ÿ%ÿ)ÿ5ÿ9#ÿ9ÿ7ÿ/ÿ,ÿ( -ÿ&ÿ% ÿ# ÿ% ÿ( ÿ* ÿ1ÿ7ÿ>ÿH!ÿL%ÿO"ÿO"ÿOÿNÿVÿXùiÿxÿŽÿœÿ­ÿµ ÿÀÿà ÿÌÿâ+ÿæ6ÿÑ!ÿÊÿÈ ÿÈÿÇÿÉÿÄñ¥Eß“3â¤^ò´nëµ}ã­uâ­nÔŸ`º}E¬o7×”Mã Y÷¤-ÿ³<ÿÆÿÉÿÄÿÂÿÂÿÂÿÁ ÿÁ ÿÁÿÁÿÁ -ÿ¿ÿÄ ÿÐü«Z´cY" D 1.<U!ž`ü¾xÿâGÿ×<ú”ã}ÿˆÿÿ”ÿ£ÿ -ÿÑÿ¿ÿºÿÈÿÅ ÿÐ#óšÀkÛ†.ߢWçª_ݨ`Æ‘I·{<¾‚Cǃ?Â~:Ä}7Ì…?Ã~7Ç‚;ƈBÎJДPÔ˜TÛ¢k¸Hb%ªm9ÒŽQˇJà­nÚ§h¦„cD"  - - - +yG'ÆZ¾ˆRÅŠOÇŒQÇŒQÈRÈTÈTÇŒSÅŠQňPÂ…MÌ‘ZߤmÕšcÈVÊOÇŽSÆRÊOÁˆMÀ‡L¿‡Iº‚D±x?­t;¼HËWÌ‘ZÅŠSºH´yB¨l?©m@¹x8ÂAÙ“;ë¥Mþ©Oõ Fÿk6×CÒDÐBà9Þ7ÿ*ÿ-Ü^.÷yIÌ‘XËWÇ‘YÆXÇ‘YÆXÅWÅWÇŽUÆTÆVÈXÇŽWÇŽWÆXÇŽYÇŽYÆXÅŒWÅŒWÅŒWÅŒWÃWÁ‹UÁ‹UÁ‹UÁ‹UÁ‹UÀŠTÀŠTÀŠT¿‰S¿‰S¿‰S¾ˆR¾ˆR¾ˆR½‡Q¿…S¿…S½†S¼…R½†S½†S¼…R¼…R¼…R¼…RºƒP¹‚O»O»O»O»O¹‚O¸N¸N¸N·€M·€M·K·KµK´€J´L³~K³~K³~K±|I³~K±|I±|I¯zG¯zG®{G®{G­zF®{G®{G®{G­zH­zH¬yE¬yE¬yE¬yE¬yE«xD¬wD¬wD¬wD¬wD©vB«xDªuBªuBªuBªuB¨uC¨uC©tC©tC§tB§tB§tE§tE¦sD¦sD¦sD¥rC¥rC¥rC¢qB¢qB¢qB o@¢o@¢o@¢o@¢o@¢o@¡n?¡n?¡n?¡n?¡n?ÃYÃYÃYÃYÃYÄZÄZÄZÄXÅ‘YÅ‘WÅ‘WÅ‘WÅ‘WÅ‘WÅ‘WÅ‘YÅ‘YÆ’ZÆ’ZÆ’ZÈ”\È”\È”\È”\È”\É•]É•]É•]É•]É•]É•]É•]É•]É•]É•]É•[É•[É•[É•[Ê–\Ê–\Ê–\Ê–\Ê–ZÊ–ZË—[Ë—[Ê–^Ê–^Ë—]̘^̘\Í™]Í™]Í™]̘\É•YÇ‘[¿‰S¬{Lžm>‰_5xN$c<S,@ -2 ,>$X6jH'‚X.•kA®wI¼…WËYÒ—`Ô›`Ú¡fÙ£gÙ£gÙ£gÙ£gÞ¤fÞ¤fÙ¤eÙ¤eÙ¤eÚ¥fÝ¥gÝ¥gÝ¥eÞ¦fÞ¦hÞ¦hÞ¦hÝ¥gÝ¥gÝ¥gݤiݤiÞ£hߤiߤiߤiÛ¥iݧkÛ§mÛ§mÛ§mÛ§mÚ¦lÖ¢hÑ—eÈŽ\Ã|6Ä}7ù›ÿ³ÿÇÿÚ ÿÚÿÑÿÒÿÙ ÿËþ—ïSãGÔAÏ<Ð:Í7È7Ë:Â:·/¨9çx&ÿ­ÿÂ&ÿá&ÿÜ!ÿÔÿÔÿÖ -ÿÕ ÿÕ ÿÖ ÿÖ -ÿÖ -ÿÕ ÿÖÿÕ ÿÕ ÿÕ -ÿÕ -ÿÕÿÑ ÿÙ/î¦u-ee1 e1 c4 g8e8a4Z- Q$F=40 *'   - ±aä”3ÿo çJÿ+ ÿ4ÿ7ÿ*ÿýÿ! ÿ-ÿ5ÿ6ÿ5ÿ3ÿ1ÿ,ÿ)ÿ(ÿ,ÿ-ÿ- ÿ2ÿ8ÿ?ÿH!ÿK$ÿN!ÿM ÿMÿNÿVÿZýmÿyÿŽÿ›ÿªÿ´ÿ¾ÿÁÿËÿ× ÿÔ$ÿÅÿÄ ÿÄ ÿÄÿÅÿÊ ÿ¼å™9á•5ñ³mÿÁ{é³{â¬tÛ¦g¾‰J¡d,Á„LÞ›TážWü©2ÿ·@ÿÆÿÅÿÂÿÂÿÂÿÂÿÁ ÿÁ ÿÁÿÂÿÁ -ÿà ÿÏÿª˜GX82&"*/ŽP -ÿÔ9ÿÈ-óú”ÿ‘ÿ”ÿ˜ÿ§ÿ -ÿÒÿ¿ÿºÿÊÿÈÿÒ%ò™¼gÙ„,å¨]è«`ã®f¼‡?­q2¶z;À|8À|8Ày3Å~8É„=Ã~7¿;ŇAÁ…AʼnEÊS‰Pe(¿‚NˇJÐŒOÖ£dÑž_„bA' GœjJÆZ¾ˆRÈMÇŒQÅŠOÅŠOÅŠQÇŒSÅŠQĉPÂ…MÁ„LÇŒUÌ‘ZËYÇŒUÆRÈTÆRÅŒQÅŒQÊOŠL¿‡I·~E·~EÏ”[Ü¡hΓ\Á†O½‚KºHº~Q°tG¬k+­l,¯i²lÅpÐ{!úf1ød/ùkAóe;ûT+ëDÿ) ÿ#ÔV&òtDÌ‘XÈTÇ‘YÇ‘YÇ‘YÇ‘YÆXÆXÈVÈVÇŽWÈXÈXÈXÇŽYÇŽYÆXÆXÆXÅŒWÅŒWÅŒWÃWÃWÃWÁ‹UÁ‹UÁ‹UÁ‹UÁ‹UÀŠTÀŠT¿‰S¿‰S¾ˆR¾ˆR¾ˆR½‡QÀ†T¿…S½†S¼…R½†S½†S½†S½†S¼…R¼…R¼…R¹‚O»O»O»O»O¹‚O¹‚O¸N·€M·€M·€M·K·KµK´€J´L´L´L³~K³~K³~K±|I³~K±|I±|I°}I¯|H¯|H¯|H®{G®{G®{I®{I­zF¬yE¬yE¬yE«xD«xD¬wD¬wD­xE­xE¬yE«xD¬wDªuB¬wDªuB¨uC¨uCªuD©tC§tB¨uC§tE§tE§tE¦sD¦sD¦sD¥rC¥rC£rC¢qB¢qB¢qB¤qB¤qB¢o@¢o@¢o@¢o@¢o@¡n?¡n?¡n?ÃYÃYÃYÃYÃYÄZÄZÄZÄXÅ‘YÅ‘YÅ‘YÅ‘WÅ‘WÅ‘WÅ‘WÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÈ”\È”\È”\È”\É•]É•]É•]É•]É•]Ê–\Ê–\Ê–\Ê–\Ê–\Ê–\Ê–\É•[Ê–\Ê–\Ê–\Ê–\Ê–\Ê–\Ë—[Ë—[Ë—[̘\ʘ_ʘ_ʘ]Ìš_Ìš]Îœ_Ï›_Í™]Í™]Ï›_Ñ›aΘ^ÆZ¿‰S³~K¦q>”d-„TrD`2K!@. % ,=(V8kM0‡^<˜oM¨{Rµˆ_Ã’`ÍœjÑŸfÖ¤kؤhؤhؤh×£gÚ¤hÛ¥iݤiݤiÞ£hߤiߤiߤià£kà£kà£ià£iߥgߥgߥeߥeߥgߥgà¦hà¦hܧhܧhÚ§hÛ¨iÞ«lÞ«lÞ¨lݧkÞ¥jߦkâ¦iÙ`è›+ù¬<ÿÅÿÙÿÝÿÔÿÒÿØ ÿÒÿ²òf×KÛCÖ>Ó:Ò9Ë7Ì8Ë8È5ª.¿Cÿ–ÿ´"ÿÓÿâ&ÿÓÿÔÿÖ ÿÕ ÿÕ ÿÕ ÿÕÿÕÿÖ ÿÖ ÿÕÿÖÿÔ ÿÕ -ÿØÿÓ -ÿÖÿÚ°gQFA ?@ADD@:6/ . ( $  -  ›NÓ†4ÿa æEÿ+ÿ5ÿ9!ÿ)÷î ë óý"ÿ+ÿ0ÿ0ÿ0ÿ/ÿ,ÿ)ÿ*ÿ-ÿ-ÿ2ÿ7ÿ>ÿH!ÿK$ÿKÿKÿKÿMÿV ÿ]ûsÿ}ÿÿœÿ§ÿ°ÿ»ÿÀÿÃÿÂÿ¾ -ÿºÿ¾ÿ¾ÿÀÿÂÿÊ8þ®ÔCæŸUýȉõÀà®wóÁŠà£m£f0©d.ÕZß–Hë¢Tÿ²ÿº!ÿ -ÿà ÿÃÿÃÿÁÿÁÿÂÿÂÿÂÿÂÿÄÿÑÿ¸G¥T/$ -  -+8 Çz%ù¬Wüœÿ§#ÿ¤ÿ§ÿ¨ÿ¬ÿÇÿÔÿÀÿÀÿÊ%ÿÀÿ¼Vܤ`!ÅBÔ fÞªpç²s·‚C®p,±s/·y8·y8½y:»w8½y:ºv7º~?¸|=¼‡F·‚A³€Nl9g2µ€RÁ†MÖ›bП]Î[`B% -wBÂcÇŽWÁˆQ‰NÊOĉPÅŠQÅŠSÅŠSÇŒUÈQÁ†O‡PÈQ¾ƒLÅŠUÇŒWÃQÅSÅŒQÊOÊOÊOÊO¿†KºHÆTÖ hÒœdÃW¾ˆRº…T¸ƒR¶„G·…HµIº†N´zJ±wG³uF¯qB¹ƒKΘ`ЛhÁŒYÀK¨i3³U$¬NµU%ÑqAÉYÌ“\É•_Å‘[ÆZÆZÆXÆXÈXÊ‘ZÉ[ÈZÈZÈZÇŽYÈZÇŽYÇŽYÆXÆXÅŒWÅŒWÃWÄŽXÃWÃWÃWÃWÁ‹UÁ‹UÀŠTÁ‹UÁ‹U¿‰SÀŠT¿‰SÁˆSÁˆS¾ˆR¾ˆR»‡Q¼ˆR»†S»†S»†S»†Sº…Rº…R¼†P¼†P¾…P¾…P¼‚P¼‚PºƒPºƒPºƒP¸N¸N·€M¸‚L¸‚LµKµK´L´L´L´L³~K³~K³~K³~K³~K±|I±|I±|I°{H°{H°{H°{H¯zI¯zI¯zG¯zG¯zG®yF®yF®yF®yF­xE­xE­xE¬yE¬yE­xE­xE¬wD¬wD«xF«xFªuDªuD©vD¨uC¨uF¨uF§tE§tE§tE¦sD¦sD¥rC£rC£rC¢qB¢qB¤qB¤qB¤qB¢o@¢o@¢o@¢o@¢o@¡n?¡n?ÄZÄZÄZÄZÄZÅ‘[Å‘[Å‘[Å‘YÆ’ZÆ’ZÆ’ZÆ’XÆ’XÆ’XÆ’XÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÈ”\È”\È”\È”\É•]É•]É•]É•]Ê–^Ê–\Ê–\Ê–\Ê–\Ê–\Ê–\Ê–\Ë—]Ë—]Ë—]Ë—]Ë—]Ë—]Ë—]̘\̘\̘\̘\ÌšaË™`Ìš_Ìš_Ìš]Îœ_Ï›_Ï›_Ðœ`Ðœ`Ñ›aÒœbÏ™cÍ—aÉ”aÃŽ[´„M§w@šl>‰[-uK)f<N,>%  - /> W. kB R)Ža8©xF·†TÁVË™`Ô d×£gؤhÙ¥iÛ¥iÚ¤hÞ¥jÞ¥jߤiߤiߤiߤià£kà£kà£ià£iߥgߥgߥeߥeߥgߥgߥgߥgÛ¦gݨiÜ©jÜ©jÜ©jÜ©jݧkܦjߦkߦkç«næªmô§7ô§7ÿÁÿÚÿßÿÖÿÔÿØ ÿÓÿË ÿÞRÝE×?Ó:Ó:Î:Ë7Ê7Ë8µ9ª.ï…ÿ³!ÿÇ ÿà$ÿà&ÿÓÿÔ -ÿÕ ÿÕ ÿÖ ÿÖÿÖÿÖ ÿÖ ÿÕÿÖÿÔ ÿÔ ÿÓ -ÿÔ ÿÔ ÿÛý´Fg4= 2 0 1 -2 1 -0 /-((& " - - Œ?¿r ÷VâAÿ+ÿ5ÿ9!ÿ.ýëßâç îþþûüõðëóÿ) ÿ2ÿ7ÿ>ÿH!ÿI"ÿKÿKÿKÿKÿV ÿdÿxÿÿÿ˜ÿ§ÿ°ÿ¹ÿºÿ»ÿ¸ÿ·ÿºÿºÿºÿ¼ÿÄ -ÿÀ.êšÑŠ@ö¯eÿÏç²sí»„ÿÖŸÌYP½xBÓŽXåœNø¯aÿ¶ÿ»"ÿ¾ÿÁ ÿÃÿÃÿÁÿÁÿÁÿÂÿÂÿ¾ÿÊÿËÊya#(";H|/·jãƒúšÿ© ÿ±ÿ¸ ÿ¾ÿÒÿà ÿÏ -ÿÇÿ»Ý€·\–;€<L §s9Þªpêµv¶B£e!°r.µw6²t3ºv7¹u6·s4¹u6´x9·{<²}<¶@žk9Rb-‰T&­r9ÚŸfצdÇ–TF(  -Ÿj@Ê•kÊSÊSÊOÅŒQÇŒSÅŠQÅŠSÅŠSÇŒUÈQÈQĉRĉRÁ†O‡RĉTÁ‹OÃQÊOÊOÊOÊOÊO¿†K»‚IÊ‘XØ¢jÍ—_ÃWÀŠT¼‡V¼‡VºˆK»‰Lº†N»‡O¼‚R¹OÃ…VÉ‹\ÒœdÌ–^¾‰V·‚OÀK·xBÑsBÎp?Äd4Êj:µ|EºJ½‰S¿‹UÃWÄŽXÅWÈ’ZÉYÉYÉ[ÈZÈZÈZÈZÈZÇŽYÇŽYÇŽYÆXÆXÆXÄŽXÄŽXÄŽXÄŽXÃWÃWÃWÃWÃWÁ‹UÁ‹UÁ‹UÁ‹UÀŠTÁˆSÁˆS¿‰S¿‰S½‰S¼ˆR¼‡T»†S»†S»†S¼‡T¼‡T¾ˆR½‡Q¿†Q¾…P¾„R¾„RºƒPºƒPºƒP¹‚O¹‚O¹‚O¸‚L¸‚LµK´€J´L´L´Lµ€M³~K³~K´L³~K³~K³~K±|I±|I±|I±|I°{H°{H°{J¯zI¯zG¯zG¯zG¯zG®yF®yF®yF®yF®yF®yF­zF¬yE­xE­xE­xE­xE¬yG«xF¬wFªuD©vD©vD¨uF¨uF¨uF§tE§tE§tE¥rC¥rC£rC£rC£rC¢qB¤qB¤qB¤qB¢o@¢o@¢o@¢o@¢o@¡n?¢o@ÄZÄZÄZÄZÄZÅ‘[Å‘[Å‘[Å‘YÆ’ZÆ’ZÆ’ZÆ’XÆ’XÆ’XÆ’XÆ’ZÆ’ZÅ‘YÆ’ZÆ’ZÈ”\È”\È”\È”\É•]É•[É•[É•[Ê–\Ê–\Ê–\Ê–\Ê–\Ê–\Ê–\Ë—[Ë—[Ë—[Ë—[Ë—[Ë—[Ë—[Ë—[̘\̘\̘\Í™]ÌšaÌšaÌš_ÎœaÌš]Ìš]Íš[Ð^Ï›_Ðœ`ÓaÓaÓaÓaÕœcÔ›bИZÌ”VÆT¼ƒJªuBœg4ŠY,zIa5 Q%>/  -  -&:# I2fG({\=–pI¥Xµ‰[–hÏžjÓ¢nÕ£jÖ¤k×£gؤhÛ¥iܦjܦjܦjÞ¥jÞ¥jÞ¥jÞ¥jܧhܧhܧfÛ¦eÞ¦hÞ¦hߧiߧiݨiݨiÛ¨iÛ¨iÞ©jݨiߧiߧiݨiÞ©jÜ©jÞ«lé©aç§_ÿ¸ÿÑ6ÿãÿÙÿÓÿÚ ÿØ ÿØ ÿ® -ØmÛIÖDÔ;Ô;Ï;Ê6Ê8Ê8»;ÄDñƒÿ¯ÿ»ÿÎÿè:ÿÕ'ÿÏÿÔÿÕ ÿÖ ÿÖ -ÿÖ -ÿÕ ÿÕ ÿÕÿÕÿÔ -ÿÔ -ÿÒÿÔÿÐÿÑÿÜ4¾u8<2 1)%&&((%'$  i$§b0ñF -â7ÿ* ÿ4ÿ;ÿ7ÿ"ó êî î ð õ øøõ ê -æàéÿ# ÿ1ÿ7ÿ>ÿG ÿI"ÿKÿKÿLÿLÿXÿnÿ’ÿÿ‘ÿ˜ÿ§ÿ®ÿ²ÿ²ÿµÿ¶ÿ·ÿ¸ÿ¸ÿ·ÿ¹ ÿÃÿ³>݌ЌQý¹~ÿÌܧhÿЗÿΕ•V QË„LØ‘YôªLÿ»]ÿ¾ÿº ÿ¼ÿ¿ÿÀÿÀÿÀ ÿ¿ÿÀ ÿÁ -ÿÃÿ¿ÿÐ4ú°o72 - -% ;?IR i,z=”O¨c1Ç|*åšHÿµLÿÆ]ÿÂQð,š\0T#&* -&8eF4³‰_¡wM—]¦l,¨r8ªt:¨m2«p5¬o7¨k3ªq<¬s>¢n2©u9‹Y1Qg1 -ƒM&—a)Þ¨pà¯oɘXZ< 6 ´zHÇ[¿†MÅŒSÆRÆRÅŒSÅŒSÇŒUÅŠSÅŒUÁˆQ¿†OÊSÊS‰R‰RÁˆQ‰P‰PÊQÊQÊQÅŒSÊQ¼ƒJ»‚KÊSÈ’ZÈ’ZÅWÁ‹SÀŠTÃWÂP¾‹L½‰O¼ˆNÀŠP¸‚HÁQÑaÓ£jÆ–]»‡O·ƒK¹ˆT¸‡S²…M°ƒK¾I¶wA«u;¬v<­yC¯{E¹‚OÄZÃWÈ’\È’\Ç‘[ÈXÉYÉYÉYÉ[ÈZÈZÇŽYÆVÆVÆVÆVÆVÆVÆVÆVÆXÆXÆXÆXÆXÆXÆXÅŒWÁ‹UÁ‹UÊUÊU‰T‰TÀŠT¿‰S¾‡T¾‡T½ˆU½ˆU¼‡T¼‡T¾ˆR¾ˆR¼†P¼†P¼†P¼†PºƒPºƒPºƒPºƒP¹‚OºƒP¸N¹‚O·K·K¶N¶N¶Nµ€Mµ€Mµ€M´L³~K´L³~K±|I³~K´L´L±|I±|I°{J¯zI¯zG¯zG¯zG¯zG¯zG¯zG¯zG¯zG®yF®yF­zF­zF­xE­xE­xE­xE¬yG¬yG¬wF¬wF¬wFªuD©vG©vG¨uF¨uF¨uF§tE§tE§tE¥tB£r@¤sA£r@¤qB¤qB¢o@¢o@¢o@¢o@¢o@¤qB¤qB¢o@ÄZÄZÄZÄZÄZÅ‘[Å‘[Å‘[Å‘YÆ’ZÆ’ZÆ’ZÆ’XÆ’XÆ’XÆ’XÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÈ”\È”\È”\È”\É•]É•[É•[É•[É•[Ê–\Ë—]Ë—]Ë—]Ë—]Ë—]Ë—[Ë—[Ë—[Ë—[̘\̘\̘\̘\̘\Í™]Í™]Í™]ÎœcÎœcÎœaÎœaÎœ_Îœ_Ïœ]Ïœ]Ï›_Ðœ`ÓaÓaÓaÓaÕœcÖdÖž`ןaØŸfÕœcÊ•bÁŒY·†Y©xK‘e9€T(jD!U/ > 4!   # 8H) -a;vP)Š^0ŸsE³‚N¿ŽZË™`Ò gÖ¢fÙ¥iܦjݧkݧkݧkߦkߦkߦkߦkÞ©jÞ©jݨgݨgߧiߧià¨jà¨jݨiݨiÛ¨iÛ¨iݨiݨià¨jà¨jݨiÞ©jÜ©jÞ«lé©aæ¦^ÿ°ÿÊ/ÿãÿÞÿÑÿØ -ÿÙ ÿÛÿÇ#ü‘åSÙG×>Ô;Ì8Ì8È6Î<ËKÙYó…ÿ§ÿµÿÀÿã5ÿä6ÿÑÿÔÿ× ÿÖ ÿÖ -ÿ× ÿÖ -ÿÖ -ÿÕÿÖÿÖ ÿÕ ÿÓÿÓÿÔÿÍÿÜ4ÿÀl.4-)()'&''#$ T“Nç<á6ÿ+ -ÿ1ÿ;ÿ<ÿ/ÿ ÿ -ÿ ÿ%ÿ%ÿ$ÿ$ÿ$ÿ -ôèßæ ÿ! ÿ1ÿ7ÿ>ÿG ÿI"ÿJÿJÿLÿKÿWÿqÿ™ÿ•ÿ—ÿœÿ¥ÿ¨ÿ¬ÿ°ÿ±ÿ±ÿ²ÿ³ÿ²ÿ²ÿµÿ½ù¨3؇Õ‘Vö²wò½~ä¯pÿÏ–Ô›b|=¥f0ÑŠRÙ’Zÿ¶Xÿ¶Xÿ´ÿº ÿ½ÿ½ÿ½ÿ½ÿ¿ÿ¿ÿ¿ÿ¿ÿÃÿÊ ÿË/¾t?0 +1 ? 9==DKZl!’9§N±^Ž;@/""%X.}S)V‘WŽXY–[ ›`% c+¡d,¢i4¢i4œh,¥q5Ž\4W%i3 ‘[4™c+Û¥mà¯oاg‰kL E*¾„RÀ†T»‚I¿†M‰NÅŒQÅŒSÆTÉŽWÇŒUÁˆQ¾…NÉYמgΕ^ÈXÆVÆVÊQÊQ‰P‰P‰PÅŒS¾…L¾…L¾…NÀ‡P¾ˆPÁ‹SÀŠRÁ‹SÁ‹UÃWÁŽO¿ŒM¾ŠP¾ŠPÃS¿‰OÉ•YؤhȘ_Â’YÁU¿‹S»ŠV»ŠV·ŠRµˆPÁ‚LÀK¸‚H½‡M½‰SÂŽXÇ]ÅŽ[ÄŽXÅYÅYÈ’\Ê‘ZÌ“\Ê‘ZÉYÈZÈZÉ[ÈZÇŽWÆVÇŽWÇŽWÇŽWÇŽWÇŽWÇŽWÆXÆXÆXÆXÆXÆXÆXÆXÃWÃWÊUÊUÊU‰TÀŠTÀŠTÀ‰VÀ‰V¾‰V¾‰V½ˆU¼‡T¾ˆR¾ˆR¾ˆR¾ˆR½‡Q½‡Q½†S½†S½†SºƒPºƒPºƒPºƒP¹‚O¹ƒM¹ƒM¶N¶N¶N¶Nµ€Mµ€Mµ€Mµ€M´L´L´L³~K´L³~K³~K³~K±|K±|K°{H°{H°{H¯zG¯zG¯zG¯zG¯zG¯zG®yF­zF­zF®yF®yF®yF­xE¬yG¬yG­xG¬wF¬wF¬wF©vG©vG©vG¨uF¨uF¨uF¨uF§tE¥tB¥tB¥tB¤sA¦sD¦sD¥rC¥rC¥rC¥rC¥rC¤qB¤qB¤qBÄZÄZÅ‘[Å‘[Å‘[Å‘[Å‘[Å‘[Å‘YÆ’ZÆ’XÆ’XÈ”ZÈ”ZÈ”ZÈ”ZÈ”\È”\È”\È”\È”\É•]É•]É•]É•]Ê–^Ê–\Ê–\Ê–\Ë—]Ë—_Ë—_Ë—]Ë—]̘^̘^̘^̘^̘\̘\̘\̘\̘\̘\Í™]Í™]Í™]Ï›_ÐœdÐœdÐœbÏ›aÏ›_Ï›_Ï›_Ðœ`ÐœbÑcÓcÓcÓcÓcÔždÕŸeÓŸgÓŸgÖ¢jÖ¢jÒžfÑeÔždÍ—]ÀŠN·Eªt<–`(ˆNx> `-P8,  -%3K-]?$vQ1ŠeE¡wO²ˆ`ÕeÊœlÕ£hÙ§lܨlܨlݨiݨiÞ©jÞ©jÞ¨lÞ¨lÞ©jÞ©jà¨hà¨hâ¨já§iݨiÞ©jÞ¨lÞ¨lÞ¨nÞ¨nÛ©lÛ©lÙ©lÙ©lܨnܨnâ¨jâ¨jô«=ÿ¼NÿÞÿß ÿÒÿÖ -ÿ×ÿ×ÿÕÿ½ðvÍSÖDÒ@Ñ<Ò=ÐAØIä[âYñ„ÿ¦ÿ«ÿ°ÿÐ ÿâ2ÿØÿÑÿÙÿ×ÿÖÿ×ÿØ ÿØ ÿÙ ÿÙ ÿØÿØÿÕÿÕÿÔÿÏÿÍÿÝ ÇM/8/-)(&&#" -N‚4 â-è3ÿ* ÿ.ÿ4ÿ5ÿ5ÿ1ÿ0ÿ0ÿ3ÿ6ÿ5ÿ7ÿ9ÿ5ÿ'ÿò ó ÿÿ/ÿ7ÿ>ÿG ÿG ÿGÿGÿGÿFÿTÿlÿšÿ›ÿ ÿ¡ÿ ÿ£ÿ§ÿ¨ÿ©ÿªÿªÿ¬ÿ®ÿ¯ÿ²ÿ¶ì˜3Ø„Ú™]ó²vê²tî¶xü½…«l4|7´o;чMß•[ÿ²?ÿ°=ÿ²ÿºÿ¼ÿ¼ÿ¼ÿ¼ÿ¾ÿ¾ÿ¾ -ÿ¼ÿÄÿÑù´Ww2#  $$$# -) -L-n?qBzI|KƒN„O‹Q!‘W'“\0•^2\&™e/›iEc1 u9§kB©l6Þ¡kæ´wÝ«n‚cD  N2³zCºJµE¸‚H¼†J½‡KÀ‡N‰P‰R¿†O¸HÌ“\é³yì¶|Ë•[ÅUÈTÈTÇŒQÅŠOÈMÈM‡LÈM¹€G¼ƒJÁˆQ¾…N½‡QÁ‹U¿‰Q¿‰QÀŠRÃUÁSÁS¿‹S¿‹S¾ŠP¾ŠP̘`ÑeÇ•XÑTÆTÆTÆXÄŽVÅŒUÅŒUÃSÄŽTÆTÌ–ZΞcΞcÍfÁ‘ZÁ\¼‹W¾‹WÄ‘]Å‘[Æ’\È’\È’\Ê‘ZÊ‘ZÉYÈXÈXÈXÇŽWÇŽWÇŽWÇŽWÇŽWÇŽWÅYÅYÄŽVÄŽVÆVÆVÆVÆVÆXÅŒWÅŒWÊUÊUÊUÁ‹UÁ‹UˆVˆVÀ‰VÀ‰VÀ‰VÀ‰V½ˆU½ˆU¿‰S¾ˆR¾ˆR¾ˆR½†S½†S½†S½†SºƒP¼…R¼…RºƒP¹ƒMº„N¸ƒP¸ƒP¸„N·ƒM·‚O·‚O¶N¶Nµ€Oµ€Oµ€M³~K³~K³~K³~K³~K±|K±|K±|K°{J°{J°{J¯zI¯zI¯zI¯zI¯zI¯zI¯zI¯zI®yH®yH®yF®yF­zH¬yG­xG­xG¬wF¬wF©vG«xI©vG¨uF©vG¨uF¨uF¨uF¥tB¥tB¥tB¥tB¦sD¦sD¦sD¦sD¥rC¥rC¥rC¥rC¥rC¥rCÅ‘[Å‘[Å‘[Å‘[Æ’\Æ’\Æ’\Æ’\Æ’ZÆ’ZÆ’XÈ”ZÈ”ZÈ”ZÈ”ZÈ”ZÉ•]É•]È”\É•]É•]Ê–^Ê–^Ê–^Ê–^Ë—_Ë—]Ë—]Ë—]Ë—]Ë—_Ë—_Ë—]̘^̘^̘^̘^̘^̘\̘\̘\̘\̘\Í™]Í™]Ï›_Ï›_Ï›_ÐœdÐœdÐœbÏ›aÏ›_Ðœ`Ðœ`Ðœ`ÑcÑcÓcÓcÓcÔždÕŸeÕŸeÓŸgÓŸgÓŸgÓŸgÓŸgÖ¢jÙ£iÙ£iÔžbÑ›_Ì–^ÃU¹M©o=“`3‚O"b;"N'8+  - - - - - !+ @T/nDW/—i9©{KÀŽSÎœaÖ¢fÙ¥iÞ©jÞ©jÞ©jÞ©jÞ¨làªnà«là«lâªjâªjäªlâ¨jÞ©jà«làªnàªnàªpàªpÝ«nÝ«nÛ«nÛ«nÞªpÞªpäªläªlóª<ü³EÿÓÿá"ÿØ ÿÏÿÔÿ×ÿÔÿÒÿ¢ßeàNâPãNàKßPßPÐGáX÷Šÿ¨ÿ¯ÿ³ÿÁÿÚ*ÿá&ÿÔÿÖ ÿ×ÿÖÿ×ÿÚ ÿÚ ÿÜÿÜÿÛÿÙÿÙÿÖÿÕÿÓÿÎÿÕÿÂIy1(61,))((#!Bm Ü'è3ÿ+ ÿ.ÿ.ÿ.ÿ/ÿ/ÿ0ÿ,ÿ.ÿ/ÿ0ÿ2ÿ8ÿ;!ÿ4ÿ*ÿÿÿ#ÿ0ÿ8ÿ@!ÿFÿG ÿFÿGÿFÿEÿRÿhÿ—ÿšÿ ÿ¡ÿžÿ ÿ¥ÿ¥ÿ§ÿ¨ÿ¨ÿ¨ÿ©ÿ®ÿ±ÿ¯ ß‹&ÓÞað¯sã«mì´vá¢j‹Lƒ> -¹t@чMÞ”Zó¢/ý¬9ÿ¸ÿºÿ¼ÿ¼ÿ¾ÿ¾ÿ¼ÿ¾ÿ¾ -ÿ»ÿÉÿÕ#Ô2L6 - !$  -  -+ 4W(k<m<k: o: q< |B…K†O#‰R&‹W!Y#‘_;^,z>°tK­p:Ûžhæ´wÖ¤gP1 N2¬s<°w@°z@³}CµC¶€DºHºHºJ´{D®u>Óšcò¼‚å¯uÁ‹Q¿‰OĉPÅŠQÇŒQÇŒQÈRÇŒQĉNÈMÀ‡NÀ‡NÅŒUÈXÅYÃW¿‰Q¿‰QÀŠRÁ‹S¾ŠP¾ŠP¾ŠR¿‹S¿‹Q¸„J¾ŠRÃWÑTÇ•XÊ”XË•YÊ”\È’ZÉYÊ‘ZÄŽTÊ”ZÑ›_ÕŸcΞcÆ–[¼ŒU°€I°Kµ„P¾‹WÃ\Æ’\Å‘[È’\È’\Ì“\Ì“\Ê‘ZÉYÈXÈXÈXÈXÈXÈXÈXÈXÆZÅYÅWÄŽVÆVÆVÆVÆVÆXÅŒWÅŒWÅŒWÅŒWÊUÁ‹UÁ‹UÉWÉWÁŠWÀ‰VÀ‰VÀ‰V¾‰V½ˆU¿‰S¿‰S¾ˆR¾ˆR¾‡T½†S½†S½†S¼…R¼…R¼…R¼…Rº„Nº„N¸ƒP¸ƒP¸„N¸„N·‚O·‚O¶N¶N¶Pµ€Oµ€Mµ€Mµ€Mµ€M´L´L´N³~M±|K±|K°{J±|K±|K±|K°{J°{J¯zI¯zI¯zI¯zI¯zI¯zI®yF®yF­zH­zH®yH­xG­xG­xG¬yJ«xI«xI«xI©vG©vG©vG¨uF¦uC¦uC¦uC¥tB§tE§tE§tE¦sD¦sD¥rC¥rC¥rC¥rC¥rCÅ‘[Å‘[Å‘[Å‘[Æ’\Æ’\Æ’\Æ’\Æ’ZÆ’ZÆ’ZÈ”\È”\È”\È”\É•]É•[É•[È”ZÉ•[É•[Ê–\Ê–\Ê–\Ê–\Ë—]Ë—]Ë—]Ë—]Ë—]Ë—]̘^̘^̘^̘`̘`̘^̘^̘^̘^̘^̘^̘^Í™_Í™_Ï›aÏ›aÏ›aÐœbÐœbÐœbÐœbÐœbÐœbÑaÑaÒœbÓcÓcÓcÓaÔžbÔžbÕŸcÒžbÒžbÒžbÒžbÓŸcÔ dÖ fÖ fÙ£gØ¢fØ¢fÖ dÓcΘ^ÆX¹ƒK¥p?’],J m8U&F8,  - - - -  -.: U5iI1€Y<—pS®\¼jÏœoÙ¦yà§rà§rà§nà§nà«là«lâªlâªlâªjà¨hà¨jâªlâ©nâ©nâ©nâ©nÞªnÞªnÞªnÞªnâ©nâ©näªjäªjë¨]ë¨]ÿÀ$ÿÙ=ÿàÿÔ -ÿÈÿÒÿÔÿÙÿËÿ¥üyêgîZíYÖQÌGÍPàcÿ›ÿ®ÿ•ÿ™ÿ³ÿ½ÿæ5ÿã2ÿÐÿÔÿÔÿÕÿ×ÿ×ÿÜÿÝÿÝÿÜÿÛÿÙÿÖÿÖÿÒÿÏÿÝ3ÖƒGH8<--&'  8>u|°Êð"ü.ÿ-ÿ/ -ÿ. -ÿ. -ÿ,ÿ*ÿ+ÿ&ñ*ò+â2å5ê7õBÿ;ÿ7ÿ,ÿ'ÿ' ÿ,ÿ7ÿ?ÿEÿGÿFÿFÿD!ÿC ÿIÿXÿ„ÿÿ–ÿ—ÿœÿÿ ÿ¡ÿŸÿ ÿ¤ÿ¥ÿ§ÿ¦ÿ±ÿ¬Ö~!Ö~!Ø—[ñ°tç¯qÚ¢d€Iy7ˆC¹tBΆCÛ“Pûšÿ§*ÿ²ÿ¸ ÿºÿ¹ÿ¹ÿºÿ»ÿ»ÿ¾ÿ»ÿÆÿÇ –["39') - - -  -   " $" = -U6"h>e;a4 _2 c2j9s;w?pAwH!qDT'uB¬yNµzEלgæ´wË™\;  Q1!®u@¬s>§u<«y@®x>¬v<¬v>°zB²yB¬s<¥o7Í—_å±wÓŸeÀŠN¿‰MĉNÅŠOÅŠOĉNÈM‡L¾ƒH»€EÆTÖdîµ|Ô›bÆTÃQÃSÁ‹QÀŠPÁ‹Q½‰M½‰M½ŠK»ˆIÂŽRÁQÃQÄŽRÃQÁ‹OÃUÅWÇ‘YÈ’ZÇ‘UÊ”XÅ•ZÌœaبkÝ­pФo¼[¬ƒ]gAa:xQ1™sPª„a¿•qË¡}Ô¢~ʘtÊ—hÉ–gÍ”]Í”]ΓXËUÇ‘YÇ‘YÇ‘UÆTÆVÆVÆZÅYÅŽ[ÅŽ[ÇŽWÇŽWÆVÆVÄŽVÄŽVÆVÅŒUÅŒUÅŒUÅŒUÅŒUÅŒWÊUÊUÊUÊUÊUÀŠTÀŠTÀŠRÀŠR¿‰Q¿‰Q¿ˆU¾‡T¾‡T¾‡T½†S¼…R¼…R¼…Rº…R¸ƒP·„P¶ƒO·ƒM·ƒM·ƒM·ƒM·ƒM¶‚L¶N¶NµKµKµKµKµ€M´L´L³~K³~M³~M±|I±|I±|I±|I±|I°{H°{H°{H°{H¯zG¯zG¯zG®yH¯zI®{I®{I®yH®yH®yH®yH­zH¬yG«xF«xF«xF«xF«xI©vG©vG©vG¨uF¨uF¨uF§tE§tE§tE§tE¦sD¦sD¦sD¦sD¦sDÅ‘[Å‘[Å‘[Æ’\Æ’\Æ’\Æ’\Æ’\È”\È”\È”\È”\È”\É•]É•]É•]É•[É•[É•[É•[É•[Ê–\Ê–\Ê–\Ê–\Ë—]Ë—]Ë—]Ë—]̘^̘^̘^̘^̘^̘`̘`̘^̘^̘^̘^̘^̘^Í™_Í™_Í™_Í™_Í™_ÐœbÐœbÐœbÐœbÐœbÐœbÑcÑaÑaÒœbÒœbÔždÓcÓaÔžbÔžbÔžbÒžbÓŸcÓŸcÒžbÒžbÒžbÔždÕŸeÖ dÖ dÖ dÖ dÖ fÖ fÕŸgÒœdË–eÁŒ[³~T¢mC‹\9zK(`6J 5)  - - - * -.BV/tG"‹^9§tG»ˆ[Ï–aØŸjߦmä«rã®oá¬mã«mã«mâªjã«kã«mã«mãªoãªoãªoãªoß«oß«oß«oß«oâ©nâ©nå«kå«kë¨]ë¨]û±ÿÉ-ÿâÿÞÿÉÿÌÿÐ ÿÓÿÒÿÌÿ•çdæRèTØSÖQÜ_ënÿ±ÿ»"ÿúvÿÿ°ÿÛ*ÿì;ÿÔÿÓÿÕÿÕÿ×ÿÙÿÝÿÞÿàÿÞÿÜÿÝÿ×ÿÕÿÔÿÔÿØ.ÿȈ>F33))&&  -$M[ “¡ ÒÝ) ÷) þ0ÿ0 ÿ1 ÿ0 ÿ/ ÿ+ÿ' ÿ'ÿ,÷0ñ*Ù)ì<øEöCÿ4ÿ0ÿ.ÿ,ÿ,ÿ/ÿ6ÿ?ÿFÿGÿG ÿEÿC ÿBÿGÿOÿxÿ‰ ÿ’ÿ’ÿ—ÿ˜ÿœÿÿ›ÿœÿŸÿ ÿ¢ÿ¢ÿ«ÿ§ Òz×"Ü›_ñ°tæ®pÍ•W¥c,q/‘L»vD˃@ØMú™ÿ§*ÿ±ÿ´ÿ¶ÿ¶ÿ·ÿ·ÿ¹ÿºÿ¼ÿºÿÉ"ÿ° m2-,!!  -  # + ( " 1A"\2 _5a4 ^1],^-e-h0 b3 b3 `3N!yF¹†[ĉTà¥pä²uÒ cD"!#N.§n9ªq<¥s:¨v=«u;¨r8§q9ªt<ªq:¢i2žh0Θ`á­sÉ•[¿‰M¼†JÁ†K‡LÁ†KÁ†KÁ†KÀ…J»€E¶{@Ì“Zç®uóºÐ—^Á‹OÃQÅUÄŽTÄŽTÄŽTÂŽRÁQ½ŠK¿ŒM»‡K¾ŠNÁ‹OÁ‹OÁ‹OÁ‹OÃUÄŽVÅWÅWÆTÊ”XÌœaÔ¤ià°sبkºŽY´ˆSª[lC+6F M'vL(fB­{W¹‡c¾‹\Ç”eÔ›dÕœeÕš_Ñ–[Ë•]Ê”\È’VÇ‘UÆVÆVÇ‘[Ç‘[Æ\Æ\ÈXÈXÇŽWÇŽWÅWÅWÆVÆVÅŒUÆVÆVÆVÆXÅŒWÅŒWÅŒWÅŒWÊUÁ‹UÁ‹UÁ‹SÁ‹SÀŠRÀŠRÀ‰V¾‡T¾‡T¾‡T½†S½†S½†S¼…Rº…Rº…R·„P·„P¸„N·ƒM·ƒM·ƒM·ƒM¶‚L¶N¶NµKµK¶‚L¶‚Lµ€Mµ€Mµ€Mµ€M´N³~M³~K±|I±|I³~K±|I±|I°{H°{H°{H¯zG¯zG¯zG®yH¯zI®{I®{I®yH®yH®yH®yH­zH¬yG¬yG¬yG«xF«xF«xI©vG©vG©vG©vG©vG¨uF¨uF¨uF§tE§tE§tE¦sD¦sD¦sD¦sDÆ’\Æ’\Æ’\Æ’\Æ’\Æ’\È”^È”^Æ’ZÈ”\È”\È”\É•]É•]É•]Ê–^Ê–\Ê–\É•[Ê–\Ê–\Ê–\Ë—]Ë—]Ë—]Ë—]̘^̘^̘^̘^̘^Í™_̘^̘^̘`̘`̘^Í™_Í™_Í™_Í™_Í™_Ï›aÏ›aÏ›aÏ›aÐœbÐœbÐœbÐœbÐœbÑcÑcÑcÑcÑcÒœbÓcÒždÒždÔžbÕŸcÕŸcÕŸcÒžbÒžbÓŸcÓŸcÕŸcÕŸcÕŸcÕŸcÕŸcÔžbÕŸcÖ dÖ dØ¢fØ¢fÙ£gØ¥fפeÑcÉ•[Á‹U¶€J¡l;ŒW&uD`/G9*" - - - - - - - $ -0B$\;qP,Že=§~VÃcÐp×¢oÚ¥ràªná«oá¬kâ­lâ­jâ­jâ­lâ­lâ­nâ­nâ¬pâ¬pâ­nâ­nâ­nâ­nä¬nä¬nèªièªiï¦:ý´HÿÑÿß*ÿÕÿÉÿÃÿÇ ÿÅÿÃÿµÿü~ñsírâgÍcöŒÿ¿)ÿ§ÿhø]ÿmÿœÿÅÿå7ÿß0ÿØ)ÿÕ -ÿØ ÿÜÿÞÿÞÿßÿäÿàÿßÿßÿØÿÕÿÒ ÿÓÿÀÿ» ûÁU›;‚"]R?<1- ),_iœªÖãûÿ% ÿ* ÿ,ÿ+ ÿ/ÿ1ÿ2ÿ2ÿ/ÿ0 ÿ0 ÿ<ÿ@ÛE Ë5‘0ªI År8¾k1Î> Æ6ÿ1ÿ2ÿ1ÿ1ÿ4ÿ=ÿD ÿF"ÿG"ÿE ÿAÿAÿBÿGÿ^ ÿnÿƒÿ† ÿÿ‘ÿ”ÿ•ÿ•ÿ–ÿ™ÿšÿÿžÿ¦ÿŸ Ît×}$Ýœ^ñ°rä©nÒ—\œS o&›PÀu@Ì€7׋Bý—ÿ¦!ÿ°ÿ°ÿ­ÿ®ÿ°ÿ°ÿ³ÿ¶ÿ¶ÿ¸ÿÄ,ø–H % "!! # -)-EHU#]+ ^*^*b) d+[-Z,Z,Q# -vCÂdÒœ`ï¹}á¯r×¥h|S)%$! I*¡l9£n;¢q=£r>¦r<¥q;¥n;¤m: j4•_)g1ÕŸiá­uÄX¼†L¼†L¾…N¾…NÀ…JÀ…J¿†K¿†K´{@°w<Θ^è²xÔždÀŠPÄTÁQÂŽRÂŽRÃSÄŽTÅSÄŽR½‰MÂŽRÃSÆ’VÆVÃSÃSÃSÃUÄŽVÄŽVÃUÄVÉ•[Íš[Ú§hà°uÆ–[|W7vQ1kSK8  -  2#O5"gM:…fK—x]®‡g¸‘qŘsÉœwΛp˘mÌ•bË”aÊ”\Ê”\Ì”VÉ‘SÇ‘YÇ‘YÆZÆZÆXÆXÅWÅWÆTÇŽUÇŽUÇŽUÇŽUÆTÆVÆVÄŽVÃUÅŒUÅŒUÊSÊSÊS‰RÀŠTÀŠT¿ˆU¿ˆU¿ˆU¾‡T¾‡T¾‡T»†S»†Sº…Rº…Rº…Rº…R¸ƒP¸ƒP¸„N·ƒM·‚O·‚O¸N¸N·€M¸Nµ€Mµ€M¶Nµ€M´M´M´L´L´L´L³~K±|I±|I±|I°{H±|I°}I¯|H¯|J¯|J®{I®{I®{I­zH¬yG¬yG¬yG­zH¬yG¬yG«xF¬yG©vD©vD©vG©vG©vG©vG¨uF¨uF¨uF¨uF§tE§tE¦sD¦sD¦sD¦sDÆ’\Æ’\Æ’\Æ’\Æ’\Æ’\È”^È”^Æ’ZÈ”\È”\È”\É•]É•]É•]Ê–^Ê–\Ê–\Ê–\Ê–\Ê–\Ê–\Ë—]Ë—]Ë—]Ë—]̘^̘^̘^̘^̘^Í™_Í™_Í™_Í™aÍ™aÍ™_Í™_Í™_Í™_Í™_Ï›aÏ›aÏ›aÏ›aÏ›aÐœbÐœbÐœbÐœbÑcÑcÑcÑcÑcÑcÓcÔždÒždÒždÕŸcÕŸcÕŸcÕŸcÓŸcÒžbÓŸcÓŸcÕŸcÕŸcÖ dÖ dÕŸcÕŸcÖ dÖ dØ¢fØ¢fØ¢fÙ£gØ¥fפe×£i×£iÙ£mØ¢l͘g¿ŠY®}RlA~T0h>P+;( #   - - ". :O&f=…R%›h;¸ƒPÉ”aÓaܦjâ­lâ­lâ­jã®kä¯nä¯nä¯pä¯pä®rä®rä¯pä¯pä¯pã®oå­oå­oé«jê¬kô«?ô«?ÿ½ÿÓÿÜÿÓ ÿÂÿ¾ÿÄÿÀÿ¸ÿ¶ÿ©ÿ¢û€Ø]æ|ÿ´$ÿ»%ýˆù^òWéWÿ‡ -ÿ½ÿÍÿê;ÿÜ-ÿÖ ÿØ ÿÞÿàÿàÿãÿæÿåÿãÿàÿ×ÿÓ ÿÎ ÿÅÿ¯ý–íé}Ñq,µUŠ? x-\M=2  =G~ŒÆ Ñ(ó%ø*ÿ+ÿ,ÿ/ÿ0ÿ0ÿ3ÿ5ÿ7ÿ5ÿ4ÿ8ÿ=ÿHþ;Ð:Å/,†%¢O´a'À0¶&ú&ÿ1ÿ1ÿ2ÿ5ÿ=ÿD ÿF"ÿG"ÿE ÿC!ÿC!ÿCÿFÿUÿ] ÿvÿÿŠÿŒÿŽÿÿÿ‘ÿ“ÿ•ÿ—ÿ™ÿ¡ ÿ™ÈnÕ{"Ü›]ñ°rá¦kÏ”YFq(œQ¾s>Ì€7ÖŠAú”ÿ¦!ÿ«ÿ«ÿ©ÿ«ÿ­ÿ­ÿ°ÿ±ÿ±ÿ·ÿÅ-Ùw4%  4">,!(  - )-A>CIQV"["]$S% X*Z,Y+e2“`5½‡Kñ»å³vÒ c±ˆ^<#& - K,¡l9¢m: o; o;¥q;¥q;¥n;¥n;¡k5‹Užh2Ú¤nÛ§oÃW·G¸‚H¼ƒL¼ƒL½‚G¼F¼ƒHºFªq6±x=Ôždâ¬rÍ—]¾ˆN¾ŠN¸„H½‰M¾ŠNÁ‹QÁ‹QÀŠN¼†J»‡KÆ’VÛ§kÖ¢fÍ—]È’XÅUÄŽTÄŽVÅWÅWÅWÃUË—]Ð^Ú§hÔ¤i p57(#    &5O(hA!„W2™lG³€UÀbË”aÔjΘ`Ì–^Í•WÌ”VÊ”\È’ZÈ’\È’\Ê”\È’ZÆXÅWÇŽUÇŽUÈVÈVÇŽUÇŽUÇŽWÇŽWÄŽVÄŽVÆVÅŒUÅŒUÅŒUÊSÊSÁ‹UÀŠTÀ‰VÀ‰V¿ˆU¿ˆU¾‡T¾‡T»†S»†S»†Sº…R»†S»†Sº…Rº…Rº†Pº†P¸ƒP¸ƒPºƒP¹‚O¹‚O¸N¶N¶N¶N¶N´Mµ‚N¶N¶Nµ€Mµ€M´L´L´L³~K±|I³~K²K°}I°}K¯|J®{I®{I®{I­zH¬yG¬yG­zH­zH­zH­zH­zH¬yG¬yG¬yG«xI«xI©vG¨uF©vG©vG©vG©vG§tE§tE§tE¦sD§tE§tEÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÆ’ZÈ”\È”\Æ’ZÈ”\È”\È”\É•]É•]É•]Ê–^Ê–\Ê–\Ê–\Ê–\Ê–\Ê–\Ë—]Ë—]Ë—]Ë—]̘^̘^̘^Í™_Í™_̘^̘\Í™]Í™_Í™_Í™_Í™_Í™_Í™_Í™_Ï›aÏ›aÏ›aÏ›aÏ›aÐœbÐœbÑcÑcÑcÑcÑcÑcÒždÒždÔždÔždÓŸeÓŸeÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÖ dÖ dÖ dÖ dÖ dÖ dÖ dÖ dØ¢fØ¢fÖ£dÖ£dÖ¥eצfØ¥bØ¥bÚ¥dÚ¥dܤdÝ¥eÚ¥fÓž_ÁS®z@šf0„Pj8R <4)% !&* ;M& f<€V7šmF°ƒ\Ê™jÙ¨yÞ®w߯xâ®râ®rä¯nä¯nä¯nä¯nã®oä¯pã®oã®oã®oã®oå­oå­oë­ié«gÿ®ÿº*ÿÑÿÙÿÔÿÊÿÅÿÃÿ¹ÿ¸ÿªÿ˜ésïyÿ¨&ÿ¸6ÿŠïaôWðSêNþbÿ¤ÿ¶ÿè6ÿá/ÿØÿØÿÞÿàÿá ÿæ%ÿæ"ÿå!ÿàÿØÿÏ ÿÉÿ½ÿµÿžñ‚ÚbËSµ>­6›1˜./‰+o'd< -/# !PX‹™ÉØôû"ÿ$ÿ)ÿ0 ÿ1 ÿ0ÿ/ÿ+ ÿ-ÿ/ÿ4ÿ7ÿ8ÿ9ÿ=ÿBÿCûE å/'Ž(m7h2v9}@x/m$³Æ0ÿ. ÿ3ÿ5ÿ;ÿ@ÿE#ÿI!ÿM%ÿR!ÿT#ÿSÿNÿLÿKÿWÿcÿsÿz ÿƒÿ†ÿŒÿÿÿÿÿ‘ÿ™ÿ’ÊlÒtß™Wñ«iã§hÈŒM‘C z,ŸP¿p7Ï}1×…9ý -ÿžÿ¢ÿ¥ÿ¦ÿ§ÿ©ÿ©ÿ­ÿ®ÿ°ÿ¸ -ÿ¼1»Z)# M>0L=/)%! "    - +0EFKKQTTSMSVZ! ]%i1|E¹‚Oã¯s×£gá¯x—e.3 -<%    -W<. k?¡l@ o@¥tE¥o9¥o9£n;¥p=f3…Nœh2Ú¦pܨpÃW·I¶€H»‚K»‚K¹€G¹€G·G³}Cœf,¯y?Ú¤lÚ¤lÈ’X¿‰Oº†L»‡M»‡M¼ˆNÁˆO‰P¾ˆNµE¶€FÒœbð¾ƒÞ¬qË•[È’XÆVÅUÆRÈTÆTÇ‘UÃW̘`Ò^ã®o×isG 0A+#aD6wZL“s[¤„lµŽn¾—wÈ™tÊ›v͘nË–lË”cË”cÍ”]Ì“\È’XÈ’XÅ‘WÅ‘WÇ‘WÆVÆXÆXÈXÈXÇŽWÇŽWÆVÆVÆXÅŒWÅŒWÊUÉWÉWˆVˆV¿ˆU¾‡T¼ˆR¼ˆR½†S½†S½†S½†S½†S¼…R¾…P¾…P¼ƒN¼ƒN·ƒM¸„NºƒPºƒP¹‚O¸N¶N·‚O·ƒM¶‚L¶N¶Nµ€Mµ€Mµ€Mµ€M´M³€L²K²K²K°}I±|K°{J°}K®{I®{I­zH®{I®{I®{I­zH­zH­zH­zH¬yG­zH­zH¬yG¬yG«xI«xI©vG©vG©vG©vG©vG¨uF¨uF¨uF¨uF¨uFÈ”\È”\È”\È”\È”\È”\È”\È”\É•]É•]É•]É•]É•]Ê–^Ê–^Ê–^Ê–\Ê–\Ê–\Ë—]Ë—]Ë—]Ë—]̘^̘^̘^̘^̘^̘^Í™_Í™_Í™_Í™]Í™]Í™_Í™_Í™_Ï›aÏ›aÏ›aÏ›aÏ›aÐœbÐœbÐœbÐœbÐœbÑcÑcÑcÑcÒždÒždÒždÒždÒždÔždÕŸeÓŸeÓŸeÓŸcÔ dÔ dÔ dÔ dÔ dÔ dÓŸcÖ dÖ dÖ dÖ dÖ dÖ dØ¢fØ¢fÖ dØ¢fפeפeצfצfØ¥bØ¥bÚ¥dÚ¥dÞ¦fߧgݨiܧhؤjÖ¢hÍ™c¾ŠTªxP—e=|N0b4H!9, %      ""&&/0 6 CX+m@‹Z+©xIÀYÑ¡jܨlá­qç²qç²qç²qä¯nä¯pä¯pã®oã®oã®oã®oæ®pæ®pë­ië­iÿ«ÿ­ÿÀÿÊ ÿÒÿÖÿÐ ÿÆÿ»ÿµÿ§ÿ•ÿ“ÿ°#ÿ¶4ÿ„äVçYü_ø[ðTíQø„ÿ¯ ÿÓ!ÿä2ÿÜÿ×ÿÞÿá ÿä#ÿæ%ÿæ"ÿãÿÛÿÐÿÇÿÅÿ¼ÿ¶ÿ¦ÿ–êrÍU®7›$Œ"Œ"ƒ%‚$v.o'KH9 -5+($1<q‚· À%ê& ò.ÿ-ÿ.ÿ+ ÿ+ ÿ0 ÿ0 ÿ0ÿ, ÿ'ÿ) -ÿ/ÿ6ÿ=ÿ<ÿ<ÿDÿJÿAî8Õo _IQ[a$j!h¨¼&ÿ(ÿ0ÿ2 ÿ7ÿ>ÿC!ÿI!ÿS+ÿa0ÿb1ÿ^'ÿW ÿPÿIÿLÿTÿeÿnÿ{ÿÿ…ÿ‡ÿ‹ÿ‹ÿ‹ÿŽÿ—ÿÉkÎpÚ”Rî¨fà¤e¿ƒD’D}/N»l3Ìz.Ó5öˆÿ—ÿžÿ ÿ¡ÿ¢ÿ¥ÿ¦ÿ©ÿªÿ¬ÿ¸ -ÿ¸-®M$# - -& * ')() - .0FJSTTQPK DB>??]%o8zCÊ–ZÔ dÛ©rÕ£lsJ(mD"v\GeK63)%  --V;-Ÿj>¥pD¢qB¢qB¥o9¥o9£n;£n;—`-Jšf0Ö¢lܨpÂŽVµG¸‚JºJ¹€Iµ|Cµ|C²|B®x>“]#¯y?ܦnÚ¤lÄŽT¼†L¸„J¸„J¸„Jº†L¾…L¾…L¹ƒI°z@·GÖ fä²wОcÃSÃSÄŽTÄŽTÆRÈTÆTÇ‘UÈ”\Ðœdà«lç²sµ‰[T() @ `9vO/^9¡rM¶WÂcÈ‘`ϘgÓšcјaΘ^Ì–\É•[Æ’XÈ’XÈ’XÆXÆXÈXÈXÈXÈXÇŽWÇŽWÆXÆXÆXÅŒWÅ‹YÉWˆVˆVÀ‰V¿ˆU¼ˆR¼ˆR¾‡T¾‡T¾‡T½†S½†S½†S¿†Q¾…P¾…P¾…Pº†Pº†P¼…R¼…RºƒPºƒP¸ƒP·‚O·ƒM·ƒM¶N¶Nµ€Mµ€Mµ€Mµ€Mµ‚N³€L²K²K²K°}I³~M³~M°}K°}K¯|J¯|J°}K¯|J®{I¯|J­zH­zH®{I®{I¬yG¬yG¬yG¬yG¬yJ¬yJ«xI«xI«xI©vG«xI©vG¨uF¨uF¨uF¨uFÈ”\È”\È”\È”\È”\È”\È”\È”\É•]É•]É•]Ê–^Ê–^Ê–^Ê–^Ë—_Ë—]Ë—]Ë—]Ë—]Ë—]Ë—]̘^̘^̘^̘^Í™_Í™_Í™_Í™_Í™_Í™_Í™]Í™]Ï›aÏ›aÏ›aÏ›aÏ›aÏ›aÏ›aÐœbÐœbÐœbÐœbÐœbÐœbÑcÑcÑcÒždÒždÒždÒždÒždÒždÔždÕŸeÓŸeÓŸeÔ dÔ dÔ dÔ dÔ dÔ dÔ dÔ dÕŸcÖ dÖ dØ¢fØ¢fØ¢fØ¢fÙ£gÙ£gÙ£gÙ£gÚ¤hפeפeפeØ¥fÚ¥fÚ¥fؤhؤhØ¥fÙ¦gÛ§kß«oà­nÛ¨i×£iÏ›aÁŠW¬uB“^4zE\- -G4 --%"#& -) -) -) -) -) ) ,-/ / , + * -'- 5G \5zM(“fA°}PÅ’eÒžhÛ§qå±uè´xç³wç³wé´uè³té±sé±sç¯qç¯qç®uæ­tï¬Lï¬Lÿ± ÿ¶ÿ¿ÿÅÿÅ ÿÆÿÂÿ·ÿ±"ÿ°!ÿ®2ÿ¡%ï\ßLäPíY×TÖSÚSÖOòtÿ¥ÿ¹ÿ×"ÿæ;ÿÞ3ÿÜÿâ#ÿà"ÿÛÿØÿÓÿÍ ÿÇÿÁÿÀÿ¾ÿºÿ±ÿ¯ÿžù’âzËc·E§5“)#‚&#eac ` -uxœ¢ÂÏñý#ÿ*ÿ,ÿ. ÿ- ÿ- ÿ- ÿ-ÿ.ÿ.þ-ù,õ( ñ'ó) -ÿ/ÿ8ÿ?ÿ=ÿ<ÿC#ÿGÿ7æ%Í K1 -) Wk#Ü# ì3ÿ1ÿ5ÿ:ÿ>"ÿG"ÿT/ÿ`3ÿe8ÿe*ÿa&ÿT'ÿKÿDÿDÿLÿRÿaÿf ÿrÿuÿ~ÿÿ„ÿ†ÿÿ†Ìd ÑiÜKï£^Þš[½y:A €2šM´g.Íw)Ô~0ñ‚ÿ‘ÿ˜ÿ›ÿœÿÿ¡ÿ¤ÿ¥ÿ£ÿ¦ÿ¶ÿ¯2<""  "(/ -4 0%  69SWUXURROO -MCCBXo7o7žg6â«zÖ hâ¬tª€VsIYF3I6#4 c>œoJŸl:Ÿl:¤n6¤n6£o9¢n8—a+ƒM˜d,Ñeܨn¿‹Q®zB¶‚J·ƒK´€H±}C°|B¯{C¦r:ŒX ±}EÚ¦l×£i»‡M³E¶€D·E¹ƒG½‡K¿‰M¹ƒG³}E¨r:µGÛ¥mÙ§nÅ“Z¾ˆNÃS¿‰MÀŠNÅŒQÅŒQÄŽRÅSÈ”\Ñeå±yÞªr‡iL6 -& ?*XC-sZ<ˆoQž€Y¯‘j¸’iº”kÁ˜k™lƘhƘhÉ”aÉ”aÆ’ZÆ’ZÈ’ZÆXÈXÈXÇŽYÇŽYÆXÅŒWÆŒZÅ‹YÉWÉWÊU‰TÀŠT¿‰S¿ˆU¿ˆU¾‡T¾‡T½†S¾‡T¾ˆR¾ˆR¾ˆR¼†P¼†P¼†P¼†P¼†PºƒP¼…R¼…RºƒPºƒPºƒPºƒP¹‚O¹‚O¹‚O¸N¸N¶N¶N´L³~K³~K´L³~M³~M³~M´N³~M³~M°}K°}K±|K±|K¯|J¯|J®{I®{I®{I­zH­zH­zH¬yG¬yG¬yJ¬yJ«xI«xI«xI«xI©vG¨uF¨uF¨uFÈ”\È”\È”\È”\È”\È”\È”\È”\É•]É•]É•]Ê–^Ê–^Ê–^Ê–^Ë—_Ë—]Ë—]Ë—]Ë—]Ë—]Ë—]̘^̘^̘^̘^Í™_Í™_Í™_Ï›aÏ›aÍ™_Í™]Í™]Ï›aÏ›aÏ›aÏ›aÏ›aÏ›aÏ›aÐœbÐœbÐœbÐœbÐœbÑcÑcÒždÒždÒždÒždÒždÒždÓŸeÓŸeÕŸeÕŸeÔ fÔ fÔ dÔ dÔ dÔ dÔ dÔ dÔ dÔ dÕŸcÖ dÖ dØ¢fØ¢fØ¢fÙ£gÙ£gÙ£gÙ£gÙ£gÚ¤hפeפeפeØ¥fÚ¥fÙ¤e×£gؤhØ¥fÙ¦gÛ§kÛ§kÛ¨iÜ©jܨnܨnÞ§tÜ¥rÒsÂc¥vSŒ]:mC)U+:, " " # $ -$ -( ) + , , /++01563454633 9IT'n;‹X+©u?ÂŽXÓŸcß«oã¯sè´xé´uç²sç¯qé±sç¯qç¯që²yç®uï¬Lî«Kÿªÿ¯ ÿµÿµÿ·ÿ¾ÿ¹ÿ± ÿªÿ› ôpÏKÒ?ÚGâNï[ïløuÿ|õnÿ‹ÿŸ ÿµÿËÿîCÿê?ÿÛÿÛÿÓÿË ÿÆÿÅÿÇÿÆÿÁÿÀÿ¼ÿ¹ÿ³ÿ²ÿ±ÿ°ÿ¦ÿ—ìzÁO˜.Ž$‡+‚&edjm†Š³À" ä'è+ÿ)ÿ*ÿ,ÿ-ÿ- ÿ- ÿ- ÿ- ÿ.ÿ,ü+÷& í ëèë!ÿ* ÿ8ÿA!ÿ=ÿ=ÿC#ÿEÿ2 -ßÉM5 -5HÉå,ÿ0ÿ2ÿ5ÿ< ÿDÿK&ÿM ÿS&ÿb'ÿa&ÿO"ÿJÿEÿCÿGÿIÿTÿZÿgÿiýrÿvûzþ}ÿ…ú€ÉaÌd Õ‰DêžYדT¨d%‚4|.•H°c*És%Ìv(îýŽ ÿ”ÿ—ÿ—ÿ˜ÿÿ ÿ¡ÿ ÿ£ÿ´ÿ­0”3##  % -(+5 -/"  7;RUTURQRPQ Q L L J K[#q9~GÇ_è²zç±yšpF3 zM(žqL›h6žk9¢l4¤n6¥q;¡m7šd.‹U•a)Ï›cÙ¥k½‰O¬x@±}EµI´€H¯{A¯{Aªv>¡m5ŠV°|DÚ¦lÓŸe½‰O±}C±{?²|@·E¼†J¾ˆL·E®x@ j2·IܦnئmÄ’Y¼†L¾ˆN¼†J½‡KÊOÊOÃQÄŽRÉ•]ÓŸgà¬tÏ›cbD'  -  9O1 -iC€Z1™pC¨RºŒ\ÕeÈ“`É”aÉ•]È”\Ç‘YÆXÉYÉYÈZÈZÇŽYÆXÆŒZÆŒZÅ‹YÅ‹YÅŒWÊUÁ‹UÁ‹UÃŒYÁŠWÀ‰V¿ˆU¿ˆU¿ˆU¿‰S¿‰S¾ˆR¾ˆR½‡Q½‡Q¾ˆR½‡Q¼…R¼…R¼…R¼…RºƒPºƒPºƒPºƒP¹‚O¹‚O¹‚O¸N¶N¶Nµ€Mµ€Mµ€M´L´N´N´N´N³~M´N²M°}K±|K±|K°}K¯|J¯|J®{I¯|J®{I®{I®{I­zH­zH¬yJ¬yJ¬yJ¬yJ¬yJ«xI«xI«xI«xI¨uFÈ”\È”\È”\È”\È”\È”\È”\É•]É•]É•]É•]Ê–^Ê–^Ê–^Ë—_Ë—_Ë—]Ë—]Ë—]̘^̘^̘^̘^Í™_Í™_Í™_Í™_Í™_Í™_Ï›aÏ›aÍ™_Í™]Ï›_Ï›aÏ›aÏ›aÏ›aÏ›aÏ›aÏ›aÐœbÐœbÐœbÑcÑcÑcÒždÒždÒždÒždÒždÓŸeÓŸeÓŸeÓŸeÕŸeÖ fÔ fÔ fÔ dÔ dÖ¢fÖ¢fÔ dÔ dÔ dÔ dÕŸcÖ dØ¢fØ¢fØ¢fÙ£gÙ£gÙ£gÙ£gÙ£gÙ£gÙ£gפeפeפeØ¥fؤhؤhؤh×£gؤhÚ¦jÚ¦jÚ¦jÚ§hÛ¨iÞ¨lÞ¨làªpàªpãªsãªsߦqÔ›fÁ…U¦j:ŽT+t:X"B 20'&   !"' *--.134325577788688:<>CU'm?&‹\;¤uTÁeÕ¤yã®{æ±~é³wç±ué±oé±oã°mã°mê®oê®oó¦2ó¦2ÿµÿÂÿ° ïŽÿxÿwâhÐVØ?Ù@ÇCÛWö€ÿ¥ÿ¸ÿÀÿÁ"ÿ®ÿªÿªÿ¯ÿ¸ÿÚÿâ'ÿÒÿÊ ÿËÿÆ ÿÅ ÿÆ -ÿÇ ÿÂÿ¿ÿ¼ÿ»ÿ¹ÿ¶ÿ´ÿ·ÿ°ÿ¥ÿ™ì|ÙiÉK·9¶-© §¦ÏÒäëû!ÿ* -ÿ.ÿ.ÿ.ÿ, ÿ.ÿ0ÿ1ÿ/ÿ.û* ú$ö èæÙØÛá!ÿ% ÿ9!ÿ?#ÿ>"ÿ=$ÿC*ÿD'ÿ5öàaI µ3ÿ-ÿ0ÿ1ÿ5ÿ9ÿ>ÿAÿDÿEÿFÿGÿGÿDÿAÿ@ÿAÿEÿFÿLÿNÿ[ÿ^ôgøkïv -ërÎYÎYÎy0ç’IÌŽQŽPo(n'= ¯_+Çn!Ër%ézö‡ÿŒÿÿÿ“ÿ–ÿ˜ÿ™ÿœÿ ÿ´ÿž/‘/#    -% -%$( (   ==OOSTQPKKHHHE IKLY#X&uC!¯‹bº–mG0 8 y]Jžm>—f7˜f/¡o8£o9¢n8g1•_)•a+̘bؤl»‡O©u=ªv>¬x@­yA®zD­yC©vB m9‰T!¯zGÙ¥o×£m½‰S³IµG·I²|Dº„L½‡Q·K¥q;”`*¶‚LÚ¦pÒ gÂW¼†L¹ƒI»‚K»‚KÀ‡NÁˆO¿†MÅŒSÅ‘WÓŸe×¥jºˆM?*! 6"P5*cH=~^JŽnZ¡€\­Œhº”k¿™pÀ—hÀ—h¿“^½‘\½_½_¿ŽZ¿ŽZÁXÀŽW¿‹S¿‹SÁWÁWÃŒYÃŒYÁŠW½†S¸ƒPº…R»†S¼‡T¼‡T¼‡T¼‡T»†S»†U»†U»†S»†S¼…R¼…R¼…R¹‚O¸ƒP¸ƒP¸ƒP¸ƒP·‚O·‚O¹‚O¹‚O¸N·€M´O´O´Nµ€O´M³€L³€N³€N²K²K³~K³~K±|K±|K°{J°{J¯|J®{I®{I®{I®{I­zH¬yG¬yG¬yJ¬yJ¬yJ«xI©vG«xI«xI«xIÈ”\È”\È”\É•]É•]É•]É•]É•]Ê–^Ê–^Ê–^Ê–^Ê–^Ë—_Ë—_Ë—_̘^̘^̘^̘^̘^̘^Í™_Í™_Í™_Í™_Í™_Ï›aÏ›aÏ›aÏ›aÐœbÏ›_Ï›_ÐœbÐœbÐœbÐœbÐœbÐœbÐœbÑcÑcÑcÑcÑcÒždÒždÒždÒždÒždÓŸeÓŸeÓŸeÓŸeÓŸeÕŸeÖ fÔ fÔ fÔ dÖ¢fÖ¢fÖ¢fÖ¢fÖ¢fÖ¢fÖ¢fØ¢fØ¢fØ¢fÖ dØ¢fÙ£gÙ£gÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hØ¥fØ¥fØ¥fØ¥fÙ¥iÙ¥iÙ¥iÙ¥iÙ¥iÙ¥iÛ§kÚ¦jÚ§hÛ¨iÞ¨làªnàªpàªpãªsä«tå¬wå¬wç«{á¥uÔšq¿…\žhE„N+h8M6 /&% !  "$ -' ( , , .1452366888888::===<<>IY* xG—f;µ€MÊ•bݧké³wì´ríµsèµrç´qî²sî²sù¬8ö©5ÿ¶ÿÍÿÁô“ÿwÿÿ“÷}ïVäKæbÿ“ÿµ#ÿ¬ÿ¢ÿŸÿŸÿ ÿ§ÿ¯ ÿ±ÿ²ÿ¾ÿÇ ÿÊ ÿÅÿÇ -ÿÈ ÿÆ -ÿÆ -ÿÃÿÂÿ¿ÿ¼ÿ¼ÿºÿ·ÿ·ÿµÿ²ÿ¨ÿ¢ÿÿŸÿ‰'àbÄ;±(¸#º%å) è, ú, ü.ÿ.ÿ.ÿ,ÿ.ÿ0ÿ0ÿ0ÿ0ÿ,ÿ* ÷&ò!óïâàÕÕÙà ÿ$ ÿ7ÿ?#ÿA%ÿ>%ÿB)ÿC&ÿ8ÿ& êh I‚ª(ÿ)ÿ/ÿ0ÿ0ÿ5ÿ9ÿ=ÿ?ÿAÿBÿEÿCÿAÿ@ÿ@ÿ@ÿAÿAÿFÿFùPúQåXæYÚaÙ`ÈSÀK¹dÙ„;¾€CAad‚2¤T ¿fÆm âsí~ÿ„ÿ‡ÿŒÿÿ‘ÿ”ÿ•ÿ—ÿ›ÿ° ÿ¡2’0"# - -   % -" " &# @<MKOOMMH H E F F E CKFB = 8I%K'  -?#¢qBžm>Ÿm6Ÿm6 l6Ÿk5¡k5œf0•a+¾ŠTÏ›c¶‚J¨t<©u=©u=¦r:ªv@¬xB¤q=šg3ŠU"¬wDÙ¥o×£mÁW¯{E²|D¶€HµG¹ƒK¸‚L²|Fœh2\&¶‚LÞªtÖ¤kÁV·G³}Cµ|E¸H¹€GºH»‚IÀ‡NÂŽTÑc×¥j«y>.%5L+]<rL#†`7œsDªR±…P¶ŠU¼Ž^¾`‘]Ã’^ÂY¾ŒUÁUÁUÁWÁWÄZÁŠW¾‡T¹‚O´L¶N¶N¸ƒPº…R»†S»†Sº…Rº…T»†U»†S»†S½†S½†S¼…RºƒP¸ƒP¸ƒP¸ƒP¸ƒP¸ƒP·‚O¹‚O¹‚O¸N¸N¶ƒQ¶ƒQ·‚Qµ€Oµ‚N´M³€N³€N´M²K³~K³~K³~M³~M³~M±|K¯|J®{I®{I®{I®{I­zH­zH­zH­zK­zK¬yJ¬yJ¬yJ«xI«xI«xIÈ”\È”\È”\É•]É•]É•]É•]É•]Ê–^Ê–^Ê–^Ë—_Ë—_Ë—_Ë—_̘`̘^̘^̘^̘^Í™_Í™_Í™_Í™_Í™_Í™_Ï›aÏ›aÏ›aÏ›aÏ›aÐœbÐœ`Ðœ`ÐœbÐœbÐœbÐœbÐœbÐœbÐœbÑcÑcÑcÒždÒždÒždÒždÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÔ dÔ dÖ fÖ fÖ¢fÖ¢fÖ¢fÖ¢fÖ¢fÖ¢fÖ¢fÖ¢fÖ¢fÖ¢fØ¢fØ¢fØ¢fØ¢fØ¢fÙ£gÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÙ¦gÙ¦gÚ§hÚ§hÚ§hÚ§hÚ§hÚ§hÚ§hÛ¨iݨiݨiÛ¨iÜ©jÞ©jÞ©jà¨jà¨já§iâ¨jâ¨jäªlæªkæªkæ¬nè®pÞ¨nΘ^µKšf0~Ie0L= 21,,))&) ' ' * * + , - .333355555556667776768::CW-tJ$›mA¸Š^Òjà«xì³xñ¸}îµzîµzõ±_í©Wÿ°ÿÌ,ÿå"ÿ×ÿÊÿÑ ÿÞ-ÿÅð€ævÿ§ÿ­ ÿ—òô~ò|ðv òxø}!ÿŒ0ÿ£!ÿ®,ÿµÿ²ÿ»ÿÀÿ¿ÿÁ -ÿÂÿÂÿÂÿÃÿÃÿÃÿÅ -ÿÁÿÂÿÂÿ»ÿ ÿÂ+ÿ¿(ÿ°Bÿ¡3ÿm'ï<ó'î"þ$ÿ&ÿ, ÿ/ÿ.ÿ.ÿ, ÿ.ÿ/ÿ-ÿ-ÿ,ü*ù'õ$ ò!ï!ï!òòâßÏÏÓØú#ÿ7"ÿB(ÿB(ÿ>'ÿA*ÿD'ÿB%ÿ7ÿ"ª …%" x$ÿ+ÿ0ÿ0ÿ0ÿ1ÿ5ÿ7ÿ9ÿ?ÿ@ÿBÿAÿ?ÿ>ÿ?ÿ>ÿ>ÿ>ÿBÿAÿD ÿCúGöCÓKÔL´I£8—Dµb#ªh5q/GJk"G$¯Y·a#×iâtûxÿ|ÿ‚ÿ…ÿ‡ÿŠÿŽÿÿ•ÿ©ÿ¥D—2$$ - - # - " % -! " JJWS S S J J D B EE< ;9 -28? ??" -   - - ^B+…iR›v[žy^¡wOžtLžp=šl9—i9¨zJº‰U®}I§s9¦r8¦r8¦r8©s=«u?¥n;žg4ŽY&¯zGؤn×£m¿‹S®zB±{C³}EµG³}E²|D¬v>¡k3Y!µIÞªrÖ¤k¾ŒSµG±{C±{A³}Cµ|C¸F¹€G¼ƒJÀŠRË•]Ó¦l¯‚H5 ! 5 H-]>*qR>ƒdE‘rS£X¬ˆa²_²_¶^¸`¼Ž^¼Ž^ÁŒ[¼‡V´L®yF©tA§r?¨s@©tA«xD¯|H°{J´N´O·„R¶P·‚Q¸ƒR¸ƒR¸ƒPº…R¸ƒP¸ƒP¸ƒR¸ƒR¸ƒR·‚Q·‚Q·‚Q·‚Q·‚Q¶ƒQ¶ƒQ¶ƒO¶ƒOµ‚N´Mµ‚N´Mµ€M´L³~K³~K³~M³~M³~M´N²M²M°}K¯|J¯|J®{I¯|J¯|J®{I®{I®{I®{I­zH­zH­zH¬yGÉ•]É•]É•]É•]É•]É•]É•]Ê–^Ê–^Ê–^Ë—_Ë—_Ë—_Ë—_Ë—_̘`̘^̘^̘^Í™_Í™_Í™_Í™_Ï›aÏ›aÏ›aÏ›aÐœbÐœbÏ›aÏ›aÏ›aÐœ`Ðœ`ÐœbÐœbÐœbÐœbÐœbÐœbÐœbÑcÑcÑcÒždÒždÒždÓŸeÓŸcÓŸcÓŸcÓŸcÓŸcÔ dÔ dÔ dÖ fÖ fÖ¢fÖ¢fÖ¢fÖ¢f×£g×£gÖ¢fÖ¢fÖ¢fÖ¢fØ¢fØ¢fØ¢fÚ¤hÚ¤hÚ¤hÚ¤hÛ¥iÛ¥iÛ¥iܦjܦjÚ§hÚ§hÚ§hÚ§hÚ§hÚ§hÛ¨iÛ¨iÛ¨iÛ¨iݨiݨiÜ©jÜ©jÞ©jÞ©jà¨jà¨jâ¨jâ¨jâ¨jâ¨jå©jæªkå«mç­oç±wè²xç³}á­w͘jµ€R”`:t@V'E2.**'+ ( ( * * * + , , / / / / 1 1 1 1 1 1 1 2 2 4445589;;95 5 9I]/ƒN¨s@ÈTߦkê±ví´yû·eð¬Zÿ¯ÿÁ!ÿáÿä!ÿàÿç#ÿå4ÿÓ"ÿšÿ  ÿ®ÿ˜ï~ëzùƒïyÜbÌRÁFÉNßkÿ•ÿ¼ÿºÿ´ÿ¸ÿ»ÿ¿ÿÄÿÅÿÆÿÆÿÃÿÅÿÆ ÿÆ ÿÇ ÿÊÿÄÿÁ ÿ¹"ÿ§ÿrßNá.Ø%ò&ø,ÿ+ ÿ, ÿ- ÿ- ÿ, ÿ, ÿ.ÿ/ÿ+ ÿ) -û' ÷#ñïñ ô#ó% õ' û' ù% æ àÏÍÑ×ø! ÿ7"ÿ@&ÿ@&ÿ?(ÿ?(ÿB%ÿI,ÿEÿ,Ç*Á$c(H ( vš!ÿ,ÿ2ÿ/ÿ,ÿ*ÿ/ÿ1ÿ5ÿ:ÿ>ÿ?ÿ@ÿ>ÿ>ÿ>ÿ>ÿ?ÿ>ÿBÿCÿJÿF ùFð=Ç?Ç?©>›0ƒ0<ˆFf$EB [|3G ¥OÅWÖhóp÷tùwûyþ|ÿ€ÿ…ÿ†þŒÿŸò,”/% - - - -  # - -  ( '  %M]*i"`[WKF A@CB426=OW$ -=! -   /Q,jE*ƒY1“iA¤vCª|I§yI rB¥t@¢q=¦r8£o5¥q7§s9¨r<ªt>¤m:¡j7—b/ªuBÓŸiÒžh¼ˆP¯{C°zB°zB¯yA¯yA±{C®x@ j2Z"´€HÚ¦nÕ£j½‹R³}E²|D²|B²|B´{B´{B´{B·~EÀŠRÍ—_Ó¦l¾‘WE0 -$3@!W3 hD\.k=ŸvG§~O¬~Nµ‡W»†U·‚Q­xE§r?¢m:ži6ži6Ÿj7Ÿl8¡n:£n=§rA­zH³€N°{J³~M´N´N³~K±|I´Lµ€M·‚Q¶P¶P¶P·‚Q·‚Q·‚Q·‚Q¶ƒQµ‚P¶ƒO¶ƒOµ‚Nµ‚Nµ‚N´Mµ€M´L³~K³~K³~Mµ€Oµ€O´N³€N²M²M²M°}K°}K°}K°}K¯|J¯|J¯|J¯|J­zH­zH­zH­zHÉ•]É•]É•]Ê–^Ê–^Ê–^Ê–^Ë—_Ë—_Ë—_Ë—_Ë—_Ë—_̘`̘`̘`Í™_Í™_Í™_Í™_Í™_Í™_Ï›aÏ›aÏ›aÏ›aÏ›aÐœbÐœbÐœbÐœbÐœbÐœ`Ï›_ÐœbÐœbÐœbÑcÑcÑcÑcÑcÑcÑcÒždÒždÒždÓŸeÓŸcÓŸcÓŸcÓŸcÔ dÔ dÔ dÔ dØ¢hØ¢hÖ¢fÖ¢fÖ¢f×£g×£g×£g×£gؤhؤhؤhÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÛ¥iܦjܦjܦjÚ§hÚ§hÚ§hÚ§hÛ¨iÛ¨iÛ¨iÛ¨iÛ¨iÛ¨iݨiݨiÜ©jÜ©jÜ©jÞ«là«jÞ©hÞ©fÞ©fà«hÞ©fÞ«hÞ«hÞ­kà¯má®mâ¯nå°oå°oç²sç²sã­sÓc·‚O”_,o8 T< -95 3 1 -1 -, , * -+ + + . . / / 1 1 1 1 2 2 2 2 2 4444554442 2 1 1 4 -5 ?Gd0‚N.£qIÇ•má¤nî±{ý©=ÿ°DÿÌÿÙÿ×ÿÜÿÌÿ°ÿ©ÿ´ÿ´ÿ¨ÿ¥ÿ“ðgÖM¿5²(«(¤!œ&¹Cç~ÿ±Nÿ¶ ÿ²ÿµÿ² ÿ±ÿ·#ÿ²ÿ³ÿ´ÿµÿ®ÿ·ÿ¶*ÿ±%ÿ™5ÿ|ÿRé4ý%öÿ#ÿ*ÿ)ÿ+ÿ, ÿ-ÿ.ÿ-ÿ-ÿ)÷(õ&ò ðððòô ü& -ÿ+ÿ,ÿ/ÿ1ÿ*ç!ßÍÊÌÑò -ÿ4ÿ@%ÿC(ÿC*ÿC*ÿC%ÿH*ÿP'ÿ2 ñ3ÿf ÿ¢'ÿ™ê‡Ñn·S™5rc °ÿ.ÿ:%ÿ-ÿ ÿÿ#ÿ-ÿ2ÿ6ÿ:ÿ=ÿ?ÿ?ÿ>ÿ?ÿ?ÿAÿAÿBÿFÿMÿKÿAÿ.Ù$Ε#‘g#e![%RC =@ T q2 z;™?­SÖ` Üfæiæiëlïpôqøuòzþ† Öq ˆ###     $ : Z;+L2!,+ . Y!zBD€7q)bTPHF =68F [-e7 >$! -! " #8#H3!kT>u^HŠkI•vTœsF•l?¡n<¦sA oBžm@£l@£l@£l;¤m<ÅYÌ–`¸‚J¯yA¬x@ªv>¬v@¬v@®xB«u?œh0\$µGÚ¤lÓ¡h½‹RµG²|D²|B±{A´{D·~G¹€G»‚IÀŠRÏ™aÖ¦m´„KB+  -B*T<0{Q2fG¦uJªyN¬wDªuB©u?¨t>¥r>¤q=¢o=Ÿl:žk9šg5j8¡n<¨uC«xF®{I¬yG¤q?¢o=¨uC­zH³€Qµ‚S·„U¶ƒT¶ƒT¶ƒT¹†W·„Uµ‚S´Rµ‚Pµ‚P´Oµ‚Pµ‚N´M´L´Lµ€M´L´M´Mµ€Mµ€M´N´N³€N³€N²M²M°}K°}K°}K°}K°}K¯|J®{I®{I®{I®{IÉ•]É•]Ê–^Ê–^Ê–^Ë—_Ë—_Ë—_Ë—_Ë—_̘`̘`̘`̘`̘`Í™aÍ™_Í™_Í™_Ï›aÏ›aÏ›aÏ›aÏ›aÍ™_Í™_Ï›aÏ›aÏ›aÐœbÐœbÐœbÐœ`ÑaÑcÑcÑcÑcÑcÑcÑcÒždÒždÒždÓŸeÓŸeÓŸeÓŸeÓŸcÔ dÔ dÓŸcÔ dÔ dÔ dÔ dØ¢hØ¢hÖ¢fÖ¢fÖ¢f×£g×£g×£gؤhؤhؤhÙ¥iÛ¥iÛ¥iÛ¥iÛ¥iÚ¤hÚ¤hܦjܦjܦjݧkݧkÞ¨lÜ©jÜ©jÜ©jÜ©jÜ©jÜ©jÜ©jÜ©jÜ©jÜ©jÞ©jݨiÛ¨iÜ©jÞ«l߬mà«já¬ká¬iá¬iá¬iá¬i߬i߬iß®lß®là­l߬ká¬kã®mç²sè³tëµ{ï¹굂ߪwÍ–j²{O‹Y3g5G7 ,,, , * -* -+ + - -. . . 1 1 1 1 2 2 2 2 2 2 2 44442 2 2 2 1 2 488; 9 = ? Kb0”W!¸{Eîš.ý©=ÿÀÿÙÿÚÿÉÿ›ü–ÿ¬ÿ¼ÿ¿ÿÀÿ½ÿ“ÝTÇ>¹/µ+®+¦#Ž‡’)ãzÿ· -ÿ³ÿ´ÿ™ôvôvüÿÿ¢ÿªÿ¦ ÿÿ…òpíQÔ8Ý(ã.ÿ+ÿ-ÿ-ÿ0 ÿ/ ÿ- -ÿ, ÿ-ÿ,ÿ)ü%õ ëéííðõ$ ú& ÿ+ÿ/ÿ2ÿ1ÿ3ÿ3ÿ,è" ßÊÈÊÍëþ,ÿ="ÿF+ÿG.ÿF-ÿH*ÿJ,ÿGÿBû=ÿLÿ•ÿ³8ÿ½.ÿ¸)ÿº5ÿ¯*ù¢9ä$íZ(Ñ> ÿ0ÿ>)ÿ-ÿ÷üÿ% ÿ,ÿ3ÿ7ÿ:ÿ=ÿ?ÿ>ÿ>ÿ>ÿAÿDÿFÿHÿKÿNÿLÿ= -á,ËzrO P FG? >;CZe&‚(”:½GÈRÔWÔW×XÛ\â_æcàhåm½X%" - !" E&‰jZz`O4 01`(X5ªa.¤[(šR#ŒDz;j+OA8@ Z! -j1X*9  "&  - - - 3[<™pC¥|O§tB§tB¢qD¥tG¬uI¬uI«tC¤m<±{E¸‚L¶€Hªt<©u=¨t<©s=ªt>ªt>¨r<£o7—c+µGØ¢jÑŸf¾ŒS²|D³}E³}C²|Bµ|E·~G¹€G¼ƒJÁ‹SÏ™aÖ¦mªzA5   2Y/\1§vK¯zG®yF¯{E¯{E®{G­zF©vD§tB§tB¤q?¡n<¢o=¨uC®{I®{I¨uCj8šg5žk9¢o=¨uF­zK°}N¯|M°}N³€Q³€Q³€Q°}N²P´Oµ‚P´O³€N³€L´Mµ€M´Lµ€Mµ€M´M´Mµ€Mµ€Mµ€Oµ€O´O´O³€N³€N²M°}K°}K°}K°}K°}K¯|J®{I®{I®{IÌ–^Ì–^Ì–^Í—_Í—_Í—_Θ`Θ`Θ`Θ`Ï™aÏ™aÏ™aÏ™aÏ™aÏ™aÑ›aÑ›aÏ›aÏ›aÏ›aÏ›aÐœbÐœbÏ›aÍ™_Ï›aÐœbÐœbÐœbÐœbÐœbÑaÑaÑcÑcÓcÔždÔždÔždÒždÒždÒždÒždÓŸeÓŸeÓŸeÔ fÔ f×£iÖ¢fÖ¢fÖ¢fÖ¢fÙ£gØ¢fØ¢hÙ£i×£gؤh×£g×£gؤhؤhؤhÚ¦jÚ¦jÚ¦jܦjݧkݧkݧkܦjݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÛ¨iÜ©jÜ©jÜ©jÜ©jÜ©jÜ©jÜ©jÜ©jÜ©jà«là«lÞ«lÞ«l߬m߬mã«mä¬nä¬lä¬lä¬lä¬lâ­nâ­nà­nà­nâ¬pá«oâ­nâ­nã®oä¯pç¯qê²tî¶xñ¹{ò¹€ò¹€â©tÇŽY©rD†O!^*F9 -73333220 -0 -2 2 1 2 2 2 2 2 4 4 6 6 4 4 2 1 4 4 6 89:999 9 7 9 . +6N x9»|Wÿµ#ÿÈ6ÿÖ$ÿ» ÿÿ ÿÃÿÉ ÿÄÿÊÿ²ò}ÝJÏ<À.»)¤)Ÿ$!‡r…#êˆÿ²CÿÆó¡ÞRË?å7å7ÿW ÿp$ÿ^ÿIû/í!õþ$ÿ+ÿ/ÿ/ÿ/ÿ.ÿ.ÿ/ÿ, ÿ*ÿ(óìççëðô ÷# ü& -ÿ*ÿ*ÿ+ÿ,ÿ-ÿ-ÿ/ÿ0ÿ+í" -ãÌÊÈÊáò'ÿ:"ÿI1ÿM2ÿM2ÿL-ÿM.ÿT)ÿa6ÿX)ÿ=ÿKÿt)ÿŸÿ¤ÿ­ÿ­ÿ´ÿ¶ -ÿ’6îTÿ0ÿ@'ÿ0þðñ÷þ$ÿ-ÿ2ÿ6ÿ:ÿ=ÿ@ÿ?ÿ?ÿAÿEÿIÿJÿOÿMÿKÿKÿ@ÿ(Ó»ij78**, + 2;N[%‚2‘A§D«H³G·K½N¿PÂSÂSœFm*$ ##  . F2%E00 8 4e- -£kH³n6´o7µuD·wFšbAh0@ 8DU%K(7)    -L-wX=‚Y3uL&wH!„U.™b8¡j@°q=ªk7«j,«j,¬n-¨j)¯u7®t6­r7®s8®s8«p5¬q6¥j/´w=Ò•[Ï–]»‚I¯y?±{A¶{B¶{B²|DµG¸H»‚KÅŒSÓšaئk©w<8 -&P-‹Z/«zO¶N³~K±}G³I²€I°~G®{G­zF¬yE¬yE­zF¯|H´Mµ‚N®{G©vB¦s?j6›h6j8žk9Ÿl:¡n?¡n?¥rC¨uF­zK°}N«zK©xI¬yG®{I²M°}K°}N²Pµ‚Pµ‚Pµ‚P¶ƒQµ‚Pµ‚Pµ‚Pµ‚P´O´Oµ€Oµ€Oµ€O´N³€N³€N°}K°}K±|K±|K±|K°{J¯|J¯|JÌ–^Ì–^Í—_Í—_Í—_Θ`Θ`Θ`Θ`Θ`Ï™aÏ™aÏ™aÏ™aÏ™aÏ™aÑ›aÑ›aÏ›aÏ›aÐœbÐœbÐœbÐœbÏ›aÐœbÏ›aÐœbÐœbÑcÑcÐœbÑaÑaÑcÒždÔždÔždÔždÔždÓŸeÓŸeÓŸeÓŸeÓŸeÔ fÔ fÖ¢hÖ¢h×£iؤhؤhؤhؤhÛ¥iÚ¤hÛ¥kÛ¥kÙ¥iÚ¦jÙ¥iÚ¦jÚ¦jÚ¦jÚ¦jÛ§kÛ§kÛ§kÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lݧkݧkÞ¨lÞ¨lÞ¨lÜ©jÜ©jÜ©jÜ©jÜ©jÜ©jÜ©jÜ©jÞ«lÞ«là«là«lÞ«l߬m߬m߬mä¬nä¬nä¬lä¬lä¬lä¬lâ­nâ­nà­nà­nâ¬pâ¬pá¬mâ­nâ­nâ­nå­oæ®pç¯qê²té°wê±xê±|óº…ó¼Žáª|¾Šbšf>uF#U&;4 7 6 4 5 -1 2 42 1 2 2 2 2 2 64 6 6 4 4 2 2 4 4 99::99::::449 7 <[ï–ÿ»)ÿ½ ÿ¹ÿ¬ÿµÿÌÿË ÿÌ ÿËÿ¯ìwÔAÊ7½+¶$ž#šŠ‹zeœ:ÿŸ0ÿÓ"ÿÉÿ‡ ãWè:Ý/Ý3ä:é)å%ñ%ø,ÿ2ÿ0ÿ-ÿ-ÿ-ÿ-ÿ,ÿ* ÿ'ÿ"ú öðïì -í óø# ý)ÿ+ÿ+ÿ,ÿ,ÿ,ÿ,ÿ-ÿ-ÿ/ÿ0ÿ+í" -âÌÊÆÅÜì! ÿ2ÿF.ÿO4ÿM2ÿF'ÿI*ÿ]2ÿa6ÿ[,ÿDå(÷:ùÿ¡ÿ«ÿ©ÿ«ÿ£übÔ:ÿ3ÿA(ÿ,ýììóú -ÿ' ÿ+ÿ/ÿ3ÿ9ÿ=ÿ:ÿ=ÿ@ÿFÿJÿM!ÿMÿOÿMÿV%ÿF ÿ:ø4 -à…t68.0* -&(. <? _k„!*š. 4§8«<¬=«<Š4p,$  -#!  +0 9 -3l4¸€]Ø“[ßšbÑ‘` `/]%;9DEA( $ 3.   -    "A"J!HNT%e.v?…F“T Ÿ^ ¬k-¶x7¬n-©o1¥k-¤i.¢g,¥j/¨m2¬q6¦k0¯r8ÉŒRÈV¾…L¯y?ªt:³x?³x?°zB±{C´{D·~G‰PЗ^Ù§l¼ŠOG( '! #  :lI2lA®}R¶N´L´€J´€J³J³J¯|H¯|H®{G­zF¯|H²K¼‰U¶ƒO°}I­zF®{G¨uA¢o= m;Ÿl:j8j;žk<›h9Ÿl=©vG­zK§vG¢qB¢o=¤q?©vD­zH­zK®{L³€Nµ‚Pµ‚Pµ‚Pµ‚Pµ‚P´Oµ‚P´O´Oµ€Oµ€Oµ€O´N³€N³€N³€N²M³~M±|K³~M±|K¯|J¯|JÍ—_Í—_Í—_Í—_Θ`Θ`Θ`Θ`Ï™aÏ™aÏ™_Ï™_Ï™_Ï™_Ñ›aÑ›aÑ›aÒœbÐœ`Ðœ`Ðœ`Ðœ`Ðœ`Ðœ`Ðœ`Ðœ`Ðœ`ÑaÑaÑaÑaÑaÒžbÒžbÒžbÒžbÕŸcÕŸcÕŸcÕŸcÔ fÔ fÔ fÔ fÔ fÔ fÔ fÖ¢hÕ£fÖ¤g×¥h×¥h×¥h×¥hÙ¥iÙ¥iÚ¦jÚ¦jÚ§hÛ¨iÛ¨iÜ©jÛ¨iÛ¨iݨiݨiÞ¨lÞ¨làªpàªpÞ¨nÞ¨nÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ©jÞ©jÞ©jÞ©jÞ©jà«là«là«là«lÞ©jà«já¬ká¬ká¬kâ­nâ­nâ­lâ­lâ­lâ­lâ­lâ­lä¬nä¬nâ¬pâ¬pâ¬pâ¬pä¬nä¬næ®pæ®på­oç¯qç¯qç¯qç¯qé±sé±qê²rî¶vñ¹yòº|ñ¹{å¬sÉWŸe3u; T>7<9 ;8885 4 -5 99888888664 4 999::9;;<<;;<;=6‡Dî«dÿµÿµÿ»ÿÄ ÿÐ -ÿÎÿÌÿÌÿ²ç{Ø:Ð2­0£&™$• Š‚{d ½e'ÿÃ:ÿå\ÿµQí[û+ü,û' ù% ÿ%ÿ)ÿ, ÿ0ÿ- -ÿ- -ÿ0ÿ.ÿ+ÿ!ý÷ñôôôî ó% ø' ù( þ*ÿ+ÿ,ÿ,ÿ,ÿ,ÿ,ÿ,ÿ+ÿ+ÿ-ÿ-ÿ-ÿ+ò'æÒÏÊÊÊÔú,ÿB&ÿK2ÿK2ÿA&ÿ?$ÿD-ÿH1ÿD(ÿ6ÿ$ñâ'ÿR"ÿ}%ÿ8ÿ“5ÿ!þ: -ò.ÿ5ÿ?%ÿ.ýèëäéè" î(ÿ+ÿ1ÿ5ÿ7ÿ6ÿ;ÿAÿDÿFÿO%ÿMÿJÿP#ÿO"ÿBÿ@ÿ@ÿ8 ü$â -”†B B ) & !$ &%/5 -D LW^i%l( -r+ -t- p, -b*" "" -  ++//`=(®‹v²ˆn‚X>= 00(  ,$ -  !'.  4H)L-N/a.d1p1z;˜J¨ZÇs݉4Åx#±d«c±i®n²r"ªm ¨k¨j$¨j$¨j+³u6¶{@´y>­t=ªq:°s;°s;°w@³zC³x?´y@À…JΓXÞªpÆ’XoC;6@4-* $   -O)‚\5¥tB¬{I´€J´€J´L´L²€I²€I°}I¯|H®{G®{G´‚K¼ŠS¾‹W¶ƒO°}I°}I®}I­|H¬{G«zFªyE¦uA£r> o;¢q=¤s?ªyE­|H¦uCŸn<œk9žm;žm;¢q?¦sA«xFªyG«zH­|J®}K±€Q°P¯|M²P´Rµ‚S³€N³€N³€Q³€Q³€N³€N´L´L³~K±|I±|I±|I±|I±|IΘ`Θ`Θ`Θ`Ï™aÏ™aÏ™aÏ™aÑ›cÑ›cÑ›aÑ›aÑ›aÑ›aÒœbÒœbÒœbÒœbÐœ`Ðœ`Ðœ`Ðœ`Ðœ`ÑaÑaÑaÑaÒžbÒžbÒžbÒžbÒžbÒžbÒžbÓŸcÓŸcÕŸcÕŸcÕŸcÖ dÔ fÔ fÔ fÖ¢hÖ¢hÖ¢h×£i×£iÖ¤g×¥h×¥hئiئiئiÙ¥iÚ¦jÚ¦jÛ§kÛ¨iÜ©jÜ©jÜ©jÜ©jÜ©jݨiÞ©jÞ¨lÞ¨làªpàªpÞ¨nÞ¨nÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnà«lá¬má¬mà«là«là«là«là«là«lÞ©jà«já¬ká¬ká¬kâ­nâ­nâ­lâ­lâ­lâ­lâ­lâ­lå­oå­oã­qã­qã­qã­qå­oæ®pæ®pæ®pæ®pæ®pç¯qé±sé±sé±sé±qé±qê²rì´tî¶xî¶xóºø¿†ó¹‡ã©wÀ‰]—`4n:O8 8 9;;;;:<<;;;:::7667999::9;;==:9;;IHL Â8ÿµÿ¯ ÿ´ÿÉÿÓ ÿÎÿÉÿÌÿ¹ñ…Þ@Ò4­0¢%™$• Š†a …-ÿ³*ÿÉ@úhÌ:ù)ù)þ*ÿ,ÿ, -ÿ, -ÿ, ÿ+ ÿ+ÿ/ ÿ5ÿ5ÿ)üñ ï ñùú!ü# -ö( ú,þ-þ-ÿ,ÿ,ÿ-ÿ,ÿ*ÿ+ÿ*ÿ*ÿ*ÿ+ÿ,ÿ+ÿ+ÿ*ò'èÓÏÊÊÈÍî ÿ7ÿ@'ÿF-ÿC(ÿ:ÿ7 ÿ;$ÿ< ÿ< ÿ9ÿ, Ý"ÚÉ?ÝSäSÐ?ç#ö2ÿ8ÿ6ÿ#øêëâäáç!ù# ÿ+ÿ,ÿ1ÿ4ÿ7ÿ>ÿ?ÿBÿQ'ÿW&ÿR!ÿIÿ@ÿBÿDÿEÿEÿ;þ&±$— -D ?( ) & $ %) 8 8 C C J -K O Q VY\R*#! #$    -'(,.H%W49'&&$"&,,! $) .3!2#)  @!H)O0Q2l9 uB”U!«l8Ì~9Çy4ÁmÀlÕˆ3ð£Nð¨Uì¤Qû»kÿÆvõ¸kä§ZÎJÁƒ=°r3ªl-¦k0§l1ªq:°w@°s;°s;°w@±xA´y@´y@¹~CĉNؤj×£i¥yK\0<J% -B#> ;:8 4(&  % -e?iB¬{I®}K³I´€J´L³~K²€I°~G°}I¯|H¯|H¯|H²€IµƒL¹†R³€L²K³€L°K°K°K­|H«zF©xD§vB©xDªyE®}I±€L®}I«zH¦uCžm;›j8œk9œk9j8Ÿl:žm;Ÿn<£r@ªyG­|M¬{L¬yJ¨uF«xI®{L°}K°}K°}N²P³€N³€N´L´L´L±|I±|I±|I±|I±|IΘ`Θ`Θ`Θ`Ï™aÑ›cÑ›cÑ›cÑ›cÑ›cÑ›aÑ›aÒœbÒœbÒœbÒœbÒœbÒœbÑaÑaÑaÑaÑaÑaÑaÑaÑaÒžbÒžbÒžbÒžbÓŸcÒžbÒžbÓŸcÓŸcÕŸcÕŸcÕŸcÖ dÔ fÔ fÔ fÔ fÖ¢h×£iÙ£iÚ¤jؤhؤhئiئiاgاgÚ¦jÚ¦jÚ§hÛ¨iÛ¨iÜ©jÜ©jÜ©jÜ©jÜ©jÞ¨lÞ¨lÞ¨lÞ¨làªpàªpàªpàªpàªnàªnà«là«là«là«là«là«lá¬má¬má¬má¬má¬má¬má¬mà«là«lá¬má¬kâ­lâ­lâ­lâ­nâ­nâ­nã®oã®oã®oã®oã®oä¯pä¯pæ®pæ®pæ®pç¯qæ®næ®nç¯qç¯qç¯qç¯qé±sé±sé±qé±qé±qê²rê²rê²rê²rê²ré±qíµuö¼|úÀ€ù¿ó¹{â§nÁ†M•X$p3TI JQONLLHHDD@=<;9:<;<<<<;=>>===>@ D?k3ÿ¤'ÿ±4ÿ¶ÿÌÿÕÿÏ ÿÌÿÏ -ÿÊùœÛIÉ7°/§&›$— ’Ž„ˆs -yëR#Ö=äø'ÿ- ÿ*ÿ+ -ÿ+ -ú-û.ÿ, ÿ, ÿ, ÿ/ÿ6!ÿ=(ÿ7ÿ(ÿ ÿÿÿÿ$ ÿ)ÿ+ÿ+ÿ+ ÿ+ ÿ,ÿ,ÿ,þ* ÿ'ÿ'ÿ&ÿ% ÿ%ÿ'ÿ'ÿ'ÿ'ÿ'ÿ$ùÛÔÊÊÃÄÒå*ÿ8ÿE%ÿA$ÿ;ÿ6"ÿ7#ÿ9#ÿ9#ÿ:ÿ6ÿ7ÿ2ö-äÚÜùÿ5ÿ9ÿ/ ý" ðåæÛÛÖÙàç# õ& ü-ÿ1ÿ5ÿ8ÿ9ÿ>ÿE#ÿS,ÿZ3ÿI$ÿ@ÿCÿEÿFÿEÿ>ÿ6 ÿ. ÿ¯  VS;737@ B>=< < :7 5 6 -60  $"$"     &'$%&   $* 035!5! -& W1[5b9qH$˜Z+¥g8´o(«f«\¸iÝŽ1ÿº]ÿÕuÿÝ}ÿÝ}ÿÝ}ÿÝ~ÿÜ}ÿÛÿÛÿÙ}ÿÏsüºhí«YÖ–H¼|.¨m$¨m$¬k-­l.«l2®o5³r6·v:ºy=džJÒ˜Zà¦hÎœc—e,g4d1i5 t@wEq?b0 -V$NG=9/ -*&! >(X+ŸvI°~G²€I³I´€J³~K³~K±}G°|F°}I¯|H¯|H¯|H°~G²€I´‚K°~G²€I²€I°€I®~G°K­|H¬|EªzCªyE¬{G²Mµ„P²M®}I¬{G§vB¥tB£r@¢q?Ÿn rB«zK«zKªyJ¬{L²P°}N±}G´€J³I³I¶L¶L³}G³}GÑ›cÑ›cÑ›cÑ›cÑ›cÑ›cÑ›cÑ›cÑ›cÏ™aÒœbÒœbÒœbÒœbÒœbÒœbÒœbÓcÑaÑaÑaÑaÑaÒžbÒžbÒžbÒžbÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÕŸcÕŸcÖ dÖ dÔ fÔ fÔ fÔ f×£i×£iÙ£iÚ¤jؤhؤh×¥hئiاgاgÚ¦jÚ¦jÚ§hÚ§hÛ¨iÛ¨iÛ¨iÜ©jÜ©jÜ©jÞ¨lÞ¨lÞ¨lÞ¨làªpàªpàªpàªpàªnàªnà«là«là«là«là«là«lá¬má¬má¬má¬má¬má¬má¬má¬mâ­nâ­nâ­lâ­lâ­lâ­lâ­nâ­nâ­nã®oã®oã®oã®oã®oä¯pä¯pæ®pç¯qç¯qç¯qç¯oç¯oé±sé±sé±sé±sé±sé±sé±qê²rê²rê²rë³së³sê²rê²rë³sê²rí³sð¶vôº|ù¿ý‰ÿÇŽü¿‹ã¦rÀ„T’V&f,OFIMNJJFDBB@<:;==>>====?>=>>>BBFD Êoÿ¶9ÿ¼ÿËÿ×ÿÐ -ÿÍÿÑ ÿÒÿ´éWÈ6°/§&&™"”‘‰…{v ´¿&û*ý,ÿ*ÿ)þ* ý)ö) õ( -ÿ+ ÿ.ÿ- ÿ, ÿ6!ÿB-ÿB)ÿ<#ÿ8#ÿ1ÿ(ÿ$ÿ(ÿ,ÿ-ÿ+ÿ, ÿ, ÿ,ÿ,þ* þ* ÿ'ÿ'ÿ&ÿ&ÿ&ÿ&ÿ%ÿ%ÿ%ÿ%ÿ$ùÝÖÊÉ ÃÄÏØÿ* -ÿ;ÿ>!ÿ;ÿ:&ÿ7#ÿ5ÿ4ÿ4ÿ< ÿ@!ÿ2ö-ë"èéÿ0ÿÿ1Ñ.»eW<<9 9 ;A @===::9 7 81   $!## !  "& -! ./%    ! -# * )*)"5b<&vP:‹b>—nJŸa2‹M•P ®i"ÝŽ3ÿ¼aÿÖyÿÛ~ÿÚzÿ×wÿ×wÿÓsÿÓtÿÍnÿÏsÿÖzÿÕyÿÔxÿ×…ÿ×…ÿЂô´fÊF¶{2»z<ÀAÁ‚H¹z@µt8³r6°o3µt8¾„FÓ™[ÑŸf»‰P‘^1}J†R* lDÈ–pʘrµƒ]¤rLŒ]¢qB¤sD¥tE©vG­zK°|F±}G³I±}G¶L¶L³}GµIÑ›cÑ›cÑ›cÑ›cÑ›cÑ›cÑ›cÑ›cÑ›cÑ›cÒœbÒœbÒœbÒœbÓcÓcÓcÓcÑaÑaÑaÑaÑaÒžbÒžbÒžbÒžbÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÔ dÔ dÖ dÖ dÖ dÖ dÔ fÔ fÔ fÖ¢h×£i×£iÚ¤jÚ¤jؤhؤhئiئiاgاgÙ¦gÙ¦gÚ§hÚ§hÚ§hÛ¨iÛ¨iÚ§hÚ§hÚ§hݧkÞ¨lÞ¨lÞ¨lݧmàªpàªpàªpàªnàªnà«là«là«là«là«là«lá¬má¬má¬mâ­nâ­nâ­nâ­nâ­nâ­nâ­nâ­lâ­lâ­lâ­lâ­lâ­lã®oã®oã®oã®oã®oã®oæ­ræ­ræ®pç¯qç¯qç¯qç¯oç¯oé±sé±sé±sé±sé±sé±sê²rê²rê²rë³sê²rê²rë³së³sê²rì´tì·tì·tî¹vî¹vòºxó»y÷½}þÄ„ÿňñµxÙšb¶w?ˆKa$KE IQNLHEDDA@>@BBBBA@BBBBBDCBDB p)é¢\ÿ½ÿÅÿÚÿÓ ÿÐ ÿÔÿÕÿÎöƒ¹F¶*°$§$£ •“• ©ð"ý/ÿ. ÿ*ÿ*ÿ0é)Ö†%,Ä-Ï8"ÿ+ ÿ- ÿ2ÿ= ÿB%ÿB%ÿD)ÿA&ÿ5ÿ*ÿ(ÿ+ÿ1ÿ1ÿ/ÿ+ÿ*ÿ+ÿ+ÿ+ÿ)ÿ+ÿ-ÿ/ÿ.ÿ.ÿ-ÿ+ÿ)ÿ&ÿ" ÿðéÏÍ ½¿¾Áß" ï2ÿ7ÿ>&ÿ=ÿ:ÿ5ÿ1ÿ.ÿ-ÿ+ÿ(ÿ& ÿ#ÿ$ ÿ( ÿ3 ÿ9ÿ7ÿ0ÿ,ÿ& íçÝÚÐÐÓ×ÛÞì! ô)ÿ-ÿ4ÿ5ÿ8ÿ8ÿ<ÿ>ÿ?ÿAÿD ÿFÿEÿBÿ?ÿ<ÿ8ÿ2ÿ/ êCÞ7¨6™'yq\ Z Z -VOS L L J J =7 &"  $"$"!    "  - -  -     =zRD›kM’bDyF]*s1¢`-ߘNÿ¼rÿÖ{ÿÚÿÔyÿÏtÿÎvÿÐxÿÐxÿÎvÿÍuÿÌtÿÉqÿÇoÿÇkÿÇkÿÆnÿÉqÿÉrÿÉrÿÀnõ±_÷²kù´mù³qî¨fæž]ÕLɃDÄ~?¼~=Ã…DÍ•WÊ’T­t9 g,´y>Ñ–[åªqì±xî³zé®uåªuÜ¡lÆ\®wD¡j7’[(zEi4\*R ID?9- %!  -/bF/œp=­NµI´€H´€H³G´L´L¶‚LµK¯|H²K³€L°}I²€I²€I°~G¯}F®|E¯}F¯}F¯}F­zF®{G®|C¯}D®|E®|E¶„M¶„M±|I°{H¬|E­}F¬{GªyE«zF©xD§vD¥tB¤sA¥tB®{I³€N°}K¯|J¨uC¤q?œn>œn>™k;šlŸn?§vG®yF­xE®{G²K´M´M°~G®|EÑ›cÑ›cÑ›cÑ›cÒœdÒœdÒœdÒœdÒœdÒœdÒœbÒœbÓcÓcÓcÓcÓcÓcÒžbÒžbÒžbÒžbÒžbÒžbÒžbÒžbÒžbÓŸcÓŸcÓŸcÓŸcÓŸcÔ dÔ dÔ dÔ dØ¢fØ¢fØ¢fØ¢fÖ¢hÖ¢hÖ¢hÖ¢h×£i×£iÙ£iÚ¤jؤhؤhئiئiاgاgÚ§hÚ§hÚ§hÚ§hÚ§hÚ§hÚ§hÚ§hÛ¨iÛ¨iݧkݧkÞ¨lÞ¨lݧmÞ¨nàªpàªpàªnàªnà«là«là«là«là«là«lá¬má¬má¬mâ­nâ­nâ­nâ­nâ­nâ­nâ­nâ­lâ­lâ­lâ­lâ­lâ­lã®oã®oã®oã®oã®oã®oå¬qå¬qå­oæ®på­oæ®pç¯oç¯oç¯qç¯qç¯qç¯qç¯qé±sê²rê²rê²rë³sê²rê²rë³së³sì´tíµuë¶së¶së¶së¶síµsì´rïµuò¸xöº}üÀƒÿŠÿÄŒô·ƒÑ”`¢f9s7 -PA9@ -JJFFAA@@BBBBABCCDBCDDDFDR «dÿ½ÿÉ ÿÛÿÕÿÓÿÖÿÓ ÿÖÿªÑ^º.¶*ª'¥"— ”•”«¼- -û- þ0ÿ*ÿ*ÿ+ ÿ5ß³VZ™µÿ(ÿ, ÿ.ÿ6ÿ>!ÿC&ÿF+ÿG,ÿA*ÿ0ÿ þü" ÿ(ÿ(ÿ)ÿ*ÿ,ÿ.ÿ0ÿ/ÿ2ÿ4ÿ6ÿ5ÿ7ÿ9!ÿ6ÿ2ÿ-ÿ(ÿ -ôìÒÏ¿¿½¾ÔÞ!ÿ(ÿ4ÿ9ÿ=ÿ:ÿ5ÿ/ÿ,ÿ-ÿ*ÿ*ÿ+ÿ,ÿ.ÿ3 ÿ8ÿ4ÿ2ÿ.ÿ+õ&íÞÜÐÏÓÔÚÚåêþ$ ÿ-ÿ.ÿ3ÿ8ÿ<ÿ>ÿ>ÿ@ÿCÿEÿEÿBÿ>ÿ<ÿ=ÿ<ÿD!ÿp#ÿ~1ÿ‘5ò€$Çl³X ’C}.v&j]W L IDF<7 '"   -$%''%%"       -  - -        -)„\NgYl<@L~K Ñ\üº‡ÿÈ~ÿÆ|ÿÇlÿÃhÿÅjÿÍrÿËsÿËsÿËsÿËsÿËsÿÉqÿÈpÿÆnÿÇkÿÄhÿÀhÿÀhÿÁjÿ¿hÿÂpÿÁoÿÂ{ÿ½vÿ¹wÿ¾|ÿ¾}ýµtö°qðªkà¢aØšYÖž`ןaÍ”YΕZÚŸdä©nâ§nߤkæ«rè­tè­xí²}é²å®{ݦsÔjÆ‘cµ€R nH’`:{P1h=\4S+G'@ 8 5 -%   < qU>¢vC®‚OµI´€H´€H´€H´L´L´€J°|F¯|H´M»ˆT»ˆT·…N´‚K²€I¯}F¯}F¯}F®|E®|E­zF®{G¯}D®|C¬zC°~G²€I°~G°{H°{H¬|E¬|E¬{G«zF«zFªyE©xF©xFªyG®}K´Oµ‚P°}K­zH«xF©vD£uE¡sCžp@o?žm>Ÿn? o@¤sD¬wD®yF­zF­zF­zF®{G°~G²€IÒœdÒœdÒœdÒœdÒœdÒœdÒœdÒœdÒœdÓeÓcÓcÓcÓcÓcÓcÓcÓcÒžbÒžbÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÔ dÔ dÔ dÔ dÔ dÔ dÔ dÔ dÖ¢fØ¢fØ¢fÙ£gÙ£g×£i×£i×£i×£iÖ¢h×£i×£iؤjؤhؤhÙ¥iÙ¥iئiئiÚ§hÚ§hÚ§hÚ§hÚ¦jÚ¦jÚ¦jÚ¦jÛ§kÛ§kݧkݧkÞ¨lÞ¨lܦlݧmݧmÞ¨nÞ¨lÞ¨làªnàªnàªnàªnà«là«lá¬ká¬ká¬mâ­nâ­nâ­nã®oã®oã®oã®oã®mã®mã®mã®mã®mã®mã®oã®oä¯pä¯pä¯pä¯pä¯pä¯pæ®pæ®pæ®pç¯qæ®næ®nç¯qç¯qç¯qé±sç¯qé±sê²rê²rê²rë³sê²rê²rë³së³së³së³së³sì´tíµuð¸xð¸xð¸xïµuïµuïµuð¶vð¸vòºxõÀøÂúÁ†ì³x‡PUh*K @DLPIIIHGGFHHHIIGGHHIIIGF[%ñ™#ÿÈRÿ×ÿÙÿÌÿÒ ÿÕ ÿØÿÏç‘Ë?¾2¹.±&ž"›žÓâ, ÿ% ÿ)ÿ,ÿ*ÿ*ÿ9%¡'l;` ñ(ö-ÿ* ÿ0ÿ9ÿG&ÿS3ÿP0ÿG.ÿ9 ÿéÑÜõÿ(ÿ*ÿ*ÿ0ÿ2ÿ1ÿ3ÿ6ÿ8ÿ9!ÿ<$ÿ<#ÿ<#ÿ<ÿ7ÿ/ÿ& ÿþåãÃû»º¾ Ð Ü)ÿ1ÿ:ÿ:!ÿ<#ÿ7ÿ4ÿ3ÿ2 ÿ/ÿ0ÿ.ÿ0ÿ/ÿ2ÿ1ÿ1ÿ0ÿ,ÿ)ú# ì! èØØÍÍËÊÑÖãì(ÿ)ÿ/ÿ2ÿ6ÿ7ÿ<ÿ>ÿ?ÿ@ÿAÿAÿ>ÿ:ÿ;ÿ=ÿ>ÿC ÿX!ÿ† ÿ—1ÿ²ÿªÿ£ø’î‚àtÏfºQ¨C—2…%zg[HA)' " #&%( %%! -         - -# ) $  - ) [;)[(8Z$“]8Öœlñ·‡û¾ˆø»…ø·wÿ¾~ÿÀtÿÅyÿÄwÿÄwÿÅvÿÄuÿÆtÿÆtÿÆuÿÅtÿÅsÿÅsÿÅsÿÃqÿÁrÿÀqÿÀsÿÀsÿ»pý¸mþ¹n÷²gù°jÿ¶pø®kû±nþ¶uü´sõ·vð²qé±qå­må­oå­oé¯qæ¬nè®päªlá§ià¦hߥgà¦hä©nç¬qç­oç­oè®på«mç¬qåªoØŸjÆX·€M¥n;\/ƒP#rCc4 R(K!E;20$ H& -}[?¦v?®~G°|B³EµG¶€H´L³~K¯{E­yC¯|H·„PÄ‘]Ç”`¾ŒU·…N³J³J´€J³I°}I¯|H¯|H°}I¯}D®|C®|C²€G®|C¬zA­{D¯}F®{G­zF­zF®{G«zFªyEªyGªyG¬yE°}I´Mµ‚N°}I®{G¬yE«xD©xF©xF©xF§vD©xFªyG©{H©{H©xD¦uA¤vF£uE¤vF¤vFªyG®}KÒœdÒœdÒœdÒœdÒœdÒœdÒœdÒœdÒœdÓeÓcÓcÓcÓcÓcÓcÓcÔždÒžbÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÔ dÔ dÔ dÔ dÔ dÔ dÔ dÔ dÖ¢fÙ£gÙ£gÚ¤hÚ¤hؤjؤjؤjؤj×£iؤjؤjؤjؤhؤhÙ¥iؤh×¥hئiÙ¦gÙ¦gÙ¦gÙ¦gÙ¥iÙ¥iÚ¦jÚ¦jÚ¦jÛ§kݧkݧkݧkÞ¨lÞ¨nÞ¨nÞ¨nàªpàªnàªnàªnàªnàªnàªnà«lá¬má¬ká¬ká¬mâ­nâ­nâ­nã®oã®oã®oã®oã®mã®mã®mã®mã®mä¯nã®oã®oä¯pä¯pä¯pä¯på°qå°qç¯qç¯qé±sé±sé±qé±qé±sé±sé±sé±sê²tê²té±që³së³së³së³së³së³sì´tì´tì´tì´tì´tíµuíµuíµuì´tî´tî´tí³sí³së³që³qé´sé´sð·|û‡ÿÇþÃŒî°ŇVŽP&b$J B?D ILIJIIIIIIJJIIIIIIN9®VÿºDÿ×ÿÚÿÈÿÉÿÖ ÿ×ÿÖ&ÿ½ ïcÃ7µ*´) $!Ÿ£!Ú$è2ÿ*ÿ' ÿ*ÿ*ÿ)ÿ* e7Wî% ÷.ÿ'ÿ+ ÿ3ÿA ÿX8ÿW7ÿG.ÿ?&ÿ'êÄÎõÿ.ÿ+ÿ)ÿ+ÿ*ÿ)ÿ)ÿ)ÿ0ÿ4ÿ8 ÿ<#ÿ<#ÿ=ÿ=ÿ9ÿ/ÿ&ÿêäÄû¹¹¹ÂÈéó%ÿ'ÿ/ÿ0ÿ0ÿ0 ÿ0 ÿ.ÿ-ÿ-ÿ/ÿ1ÿ2ÿ1ÿ1ÿ3ÿ0ÿ-ÿ+ö+ï$ ßÙÏÍËÉÑÒÞãÿ#ÿ*ÿ-ÿ0ÿ3ÿ7ÿ=ÿ>ÿ@ÿ@ÿ@ÿ=ÿ;ÿ;ÿ9ÿ;ÿ<ÿ<ÿWÿp -ÿ™ÿ¬ÿ¶ÿ·ÿµ!ÿ³ÿ®*ÿ¦"ÿ +ôÞ~În­X ˜Cw.gF8" " !#            - ! # + d1¬y_ã­ˆøÂõ»‹í³ƒê­wê­w÷¶vûºzÿ¿sÿÁuÿ¿rÿ¿rÿÀqÿÀqÿÂpÿÅsÿÄsÿÂqÿÃqÿÅsÿÃqÿÃqÿÁrÿÀqÿÀsÿÀsÿ»pþ¹ný¸mý¸mþµoø¯iü²oú°m÷¯n÷¯nð²qî°oç¯oå­mä¬nä¬næ¬nà¦häªläªläªlâ¨jâ¨já§iߤiÞ£hÝ£eÞ¤fÞ¤fߥgâ§låªoç®yæ­xä­zã¬yÚ§zÏœoº‹d§xQ’hF…[9xQ1kD$Z8L*?# ;0 )! O-ƒaE¤t=ªzC®z@±}CµG²|D±|I°{H­yCªv@®{G·„P˘dÌ™e¾ŒU¶„M³J³JµK³I²K²K¯|H°}I²€G²€G³H²€G²€G­{B®|E¯}F¯|H­zF®{G¬yE©xD©xD§vD¬{I¨uA¬yE°}I²K¯|H­zF­zF­zFªyG¬{I­|J®}K®}K±€N±ƒP¯N¬{G¥t@šl<”f6•g7™k;§vD®}KÒœdÒœdÒœdÒœdÒœdÒœdÒœdÒœdÒœdÓeÓcÓcÓcÓcÔždÔždÔždÔždÒžbÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÔ dÔ dÔ dÔ dÔ dÖ¢fÖ¢f×£g×£gÙ£gÚ¤hÚ¤hÚ¤hؤjؤjؤjؤjÙ¥kÙ¥kؤjؤjؤhؤhؤhÙ¥iئiئiÙ¦gÙ¦gÚ§hÚ§hÚ¦jÚ¦jÚ¦jÛ§kÛ§kÛ§kݧkݧkݧkÞ¨lݧkݧkݧkÞ¨là«là«làªná«oá«oá«oá¬má¬má¬kâ­lâ­lâ­lâ­lâ­lâ­lâ­lâ­nä¯pä¯pä¯pä¯pä¯pæ®pæ®pä¯nä¯nä¯nå°oå°oå°oå°oå°oç¯oç¯oé±sé±sé±qé±qé±sé±sé±qê²rê²rê²rë³së³së³së³së³së³sì´tì´tì´tì´tî´tî´tî´tî´tî´tî´tí³qí³qíµsì´rì´oíµpíµpî¶qî¶tî¶tñ¹w÷¿}ÿÆÿˆþƒà¤e±t>|? Q=?KPPPMJIIJIIIIIIGISK [Ù“OÿÚ@ÿÜBÿÓÿÇÿÐ ÿ×ÿÕÿÕÿÁT¯+«'¢$£%«"°'î"û/ÿ0ÿ*Ö$ Ò š2!‚ $ - $7Å!Ü8'ÿ)ÿ*ÿ5ÿ9ÿP-ÿW4ÿC)ÿ@&ÿ1ú Õëÿ)ÿ.ÿ) -ÿ$ù$ö! â!ß Ó$ Ö'å*ì1ÿ3ÿ:#ÿ=ÿ=ÿ>ÿ9ÿ3ÿ'ÿúÚØŠ¹¶¿ -Â Ö Ø ìòÿÿ -ÿ ÿ ÿÿ ÿ'ÿ+ÿ0ÿ/ÿ2ÿ2ÿ3ÿ1ÿ1ÿ2ÿ.ÿ& ò!ëØÖÌËÍÍÑÔá!è(ÿ+ÿ1ÿ2ÿ7ÿ8ÿ;ÿ>ÿ@ÿ@ÿ?ÿ;ÿ;ÿ>ÿ>ÿ<ÿ:ÿ<ÿCÿf ÿ(ÿž ÿ¢ÿ¨ÿ«ÿ¯ÿ²ÿ´ -ÿ±ÿ¯ÿ«ÿ®ÿ¥þ—í†Ím¸X•?w!YO90! " " "       -  - -  "",~M¿Ž\óºƒúÁŠó¸è­tæ«rí²yî³zï´{îµzð·|ó»{÷¿û¿yþÂ|ú¿rýÂuÿÁnÿÃpÿÃoÿÃoÿÂnÿÃoÿÂpÿÂpÿÃpÿÂoÿÁmÿÀlÿ¾lÿ»iÿ¹oú³iüµoø±kû±nü²oü¶wø²sï±rë­næ®næ®nä¬lä¬lã«kä¬læ¬näªläªläªlå«mâ¨jâ¨jâ¨jÞ¨lÞ¨lߦkÞ¥jÝ¥gߧiÞ¦hߧiߦkÞ¥jåªoä©nâ©nä«pݧmÕŸeÆV½‡M¨t>“_)„QwD`2V(T'I>:-+ & Q, ‚]=¦sF¦sFªuG®yK®wF°yH°{J®yH©tA¦q>¬yE¹†RÉ–bÂ[µ‚N²K°€I®~G®~G®~G¯}D¯}D®|C­{B²Kº‡S¼‰U»ˆT´M°}I¯|H¯|H¬|E¬|E­zF¬yE«xD«xD©vB¬yE¬yE¬yE¯|H®{G­zF­zF­zF¬yE­xE­xE±zI¶N¶Nº…Rº‡Uµ‚P§€G–o6{Z0nM#mN/wX9ŒhEšvSÒœdÒœdÒœdÒœdÒœdÒœdÒœdÒœdÒœdÓeÓcÓcÔždÔždÔždÔždÔždÔždÒžbÓŸcÓŸcÓŸcÓŸcÓŸcÔ dÔ dÔ dÔ dÔ dÔ dÔ dÔ dÖ¢fÖ¢f×£g×£gÚ¤hÚ¤hÚ¤hÚ¤hؤjؤjؤjؤjؤjؤjÙ¥kؤjؤhÙ¥iÙ¥iÙ¥iئiئiÙ¦gÚ§hÚ§hÚ§hÚ¦jÚ¦jÛ§kÛ§kÛ§kÛ§kݧkݧkÞ¨lÞ¨làªnàªnàªná«oà«là«làªná«oá«oá«oá¬má¬má¬kâ­lâ­lâ­lã®mã®mã®mä¯nä¯pä¯på°qå°qå°qå°qç¯qç¯qå°oä¯nä¯nå°oå°oå°oå°oç²qé±qé±qé±sé±sé±qé±qé±sé±sé±qê²rê²rê²rë³së³së³së³së³sê²rê²rë³së³sì´tî´tî´tî´tî´tî´tî´tî´rïµsíµsíµsíµpíµpì´oíµpî¶tî¶tñ¹wòºxôºuõ»vüÀÿÉŠÿË•÷º„×›k©m=‚FW>:A IMKJKJJJJJJNJXUS ŽHÿÐ6ÿìRÿÜÿÎ ÿÉÿÖÿÓÿÒÿ¾&ø‹ÌH½9²4«-²)´+ò&ú.ÿ1ÿ( Á­f^)0± ×3"ÿ.ÿ)ÿ.ÿ2ÿ@ÿK(ÿD*ÿC)ÿ5 ÿìÿ/ÿ2ÿ*ÿ#ÿ!ó ëÍ Ê ¿ÀÏØÿ$ ÿ3ÿ;ÿ> ÿ? ÿ? ÿ<$ÿ0ÿ$ÿÞ -ÛÆ -ĺ·ÃÆßáññúÿÿÿ ÿÿ$ ÿ&ÿ+ÿ0ÿ0ÿ2ÿ2ÿ2ÿ0ÿ0ÿ2ÿ1ÿ* ÷& ò!ÛØÏËÌËÎÐÚà ö"þ*ÿ-ÿ2ÿ4ÿ6ÿ9ÿ>ÿAÿ?ÿ=ÿ=ÿ>ÿ>ÿ<ÿ<ÿ>ÿ<ÿLÿ_ÿ…ÿ—ÿ¦ÿ¦ÿ©ÿ¨ÿ§ÿ¨ÿ«ÿ¯ÿ®ÿ¯ÿ°ÿ¯ÿ­'ÿ¤ðš*Ùƒ´lŸWD -l/PA ) -# ##   -     - !!%') ) !  H2#rWÇ©ŽùÈ–õÄ’ì³|æ­vé®uë°wë°wí²yî³zî³zí´yí´yî¶vòºz÷»uø¼vú¿rú¿rÿÀmÿÂoÿÁmÿÁmÿÂnÿÂnÿÂpÿÁoÿÂoÿÁnÿÂnÿÀlÿ¾lÿ¼jý¶lú³iú³mø±kú°mú°mö°qõ¯pî°qì®oæ®nå­mä¬lä¬lä¬lä¬læ¬nå«mâ¨jâ¨jâ¨jâ¨jâ¨jâ¨jàªnàªnà§lߦkߧiߧiÝ¥gÝ¥gÜ£hݤiߤiÞ£hÞ¥jÞ¥jÞ¨nâ¬rã­sàªpÚ¦pÓŸiÈ•c¾‹Yª|PœnBŽa<|O*c<[4L-B#8 0-)  R- ‚]=¥rE¤qD¨sE©tF®wF¯xG­xGªuD¥p=¡l9«xD½ŠVÇ”`½ŠV²K­zF­}F­}F¬|E­}F®|C­{B­{B¬zAµ‚NÂ[Å’^ÁŽZµ‚N¯|H°}I¯|H¬|E¬|E­zF«xD«xD«xD©vB®{G­zF®{G®{G­zF¬yE­zF­zF«xD¬wD­xE²{J·€O¾‰V¾‰V¶ƒQ«xF™r9‰b)eDJ)H) -T5gC wS0ÒœdÒœdÒœdÒœdÒœdÒœdÒœdÒœdÒœdÓeÓcÓcÔždÔždÔždÔždÔždÔždÓŸcÔ dÔ dÔ dÔ dÔ dÔ dÔ dÔ dÖ¢fÖ¢fÖ¢fÖ¢fÖ¢fÖ¢fÖ¢f×£g×£gÙ£gÚ¤hÚ¤hÚ¤hؤjؤjؤjؤjؤjؤjؤjؤj×£gؤhÙ¥iÙ¥iئiئiÙ¦gÚ§hÛ¨iÛ¨iÛ§kÛ§kÛ§kÛ§kܨlܨlݧkݧkÞ¨lÞ¨lÞ¨làªnàªnàªnà«là«lá«oá«oá«oá«oâ­nâ­nâ­lâ­lâ­lã®mã®mã®mä¯nä¯nä¯på°qå°qå°qå°qå°qç®sç®så°qä¯pä¯nå°oã°oã°oå°oç²qç²sç²sç²sç²sç²qç²qç²sç²sç²qè³rê²rê²rë³së³së³së³së³sê²rê²rë³së³së³sïµuïµuïµuïµuïµuïµuïµuïµuïµuïµuíµsíµsíµsíµsî¶tð¸vð¸vð¸vò¸sò¸sô¸töºvû¿~ÿÆ…ÿÎŽÿÎŽú¿„Ö›`£j5q8K<BKPQRRRRPPVRWY]U Є"ÿÜzÿÜÿØÿÅÿÇÿÌ ÿÐÿÒ1ÿÃ"ÿ‚'ÖX¾B±5µ/·1÷*ý0ÿ1ÿ&– y!  -   O p*Ç4"Í:(ÿ-ÿ0ÿ8ÿF'ÿM8ÿ;&í$ßÿ#ÿ2ÿ1ÿ* ÿ/ÿ'ÝÎ ¡¡‘’“˜½ Î1ÿ6ÿA%ÿCÿE!ÿC%ÿ<ÿ-ÿ øóÝÚÖÖæëÿ! ÿ&ÿ(ÿ)ÿ,ÿ,ÿ,ÿ0ÿ4ÿ5ÿ0ÿ.ÿ.ÿ-ÿ-ÿ.ÿ2ÿ/ÿ0ÿ1ÿ0ÿ, ÿ* û%ê! ã×ÓÉÆÄÇÏÐÛá$ ö+û0ÿ3ÿ7ÿ6ÿ9ÿ>ÿ?ÿ>ÿ>ÿ>ÿ>ÿ=ÿ=ÿ=ÿ=ÿ<ÿ=ÿQ ÿo*ÿ’ÿœÿ©ÿ§ÿ¬ÿ¬ÿªÿ«ÿ©ÿ§ÿ¨ÿ¨ÿªÿ®ÿ±ÿ±ÿ«ÿ£þ›ñŽÙ~ÄiŸPz+S@ - - # &)(        -     *.5503/ / ) #" 2 mQ8¨ŒsÖµÞ½—è¿ç¾òÁñÀŽòÀ…î¼ì»yç¶tå³xâ°uä²uæ´wé´uë¶wò¸sôºuö½qùÀtýÀqþÁrÿÀpÿÀpÿÁqÿÁqÿÂqÿÂqþÁpþÁpÿÁoÿÁoÿ¾oÿ»lþ·qüµoû±pû±pü¯tú­rõ­sõ­sî°sì®qä¬nä¬nå­oä¬nä¬nä¬næ«påªoä©nâ§lâ§lâ§lâ¨jâ¨jÞ¨lÞ¨là§là§lߦkݤiݤiÞ¥jߦmߦmÞ¥nÞ¥nߤkߤkÞ¤fÜ¢dÚ `Ü¢bÝ¥gà¨jâªlã«má«oÙ£gÑ›cÅW²K¢o;’a2‚Q"rCi:]. T%D<;6) %!W-ˆ^8¦sF¤qD¦qC§rDªsBªsB¦q@¦q@ k8œg4©vB¾‹WÄ‘]»ˆT¯|H­zF¬|E«{D­{D­{D¬zC¬zC«y@«y@´MÃ\É—`ÁX²‚K®~G¬{G¬{G¬|E«{D­zF¬yE¬yE­zF®{G°}I¯|H¯|H°}I®{G®{G®{G¬yE¬yE¯xG°yH³|K¹‚Q¾‹W¼‰U¤{N’iÿ5 êÈÝÿ0ÿ6ÿ4ÿ0ÿ3ÿÏ Æ  ‘‘ŽŽª ¸ø$ÿ6ÿE!ÿI%ÿF(ÿA#ÿ5ÿ)ÿ øãàßå# ö(û-ÿ+ÿ2ÿ4ÿ8ÿ;ÿ=!ÿ>&ÿA)ÿ?&ÿA(ÿ;%ÿ4ÿ.ÿ+ÿ+ÿ-ÿ1ÿ/ÿ0ÿ0ÿ0ÿ1ÿ.ÿ* ñ(ë" -ÛØÉÆÂÅÉËÔÙì! ô)ÿ,ÿ0ÿ1ÿ6ÿ<ÿ>ÿ=ÿ=ÿ>ÿ>ÿ=ÿ=ÿ=ÿ=ÿ>ÿ=ÿ@ÿMÿtÿÿ§ÿªÿ¯ÿ¯ÿ®ÿ¯ÿ¯ÿ®ÿ¯ÿ«ÿ¨ÿ¥ÿ¨ÿ«ÿ®ÿ±ÿ·ÿºÿ¼ÿ·ý®(Þ ©[x*!## ),,&$$$$%&& -   ') - #* 1.42.-'("%E)N2`?sR,šqB³Š[Ã’`ɘfÚ¨mä²wï¾|í¼zìºçµzè¶yè¶yì·xì·xð¶qò¸sõ¼pùÀtýÀqÿÃtÿÀpÿÀpÿÁqÿÁqþÁpÿÂqÿÃrÿÃrÿÂpÿÁoÿ¾oÿºkÿ¸rý¶pü²qû±pý°uü¯t÷¯uõ­sî°sì®qä¬nä¬nå­oå­oä¬nä¬næ«påªoä©nä©nä©nä©näªlâ¨jÞ¨lÞ¨là§là§lߦkߦkߦkߦkÞ¥lߦmÞ¥nÞ¥nà¥là¥là¦hà¦hߥeߥeÚ¢dÖž`Ø bÚ¢dÚ¤hܦjÞ¨pÞ¨pÛ¨tÕ¢nÌ›lÃ’c¸‰b«|U•fA…V1uJ+g<]6R+E%:2- ( X.‰_9¦sF¤qD¦qC¨sEªsB©rA¥p?£n=œg4—b/©vB¾‹WÃ\º‡S®{G­zF«{DªzC«yB©w@©w@©w@§u<§u<°}IÀYÄ’[ºˆQ°€I¬|E«zF¬{G«{D«{D­zF¬yE¬yE®{Gµ‚Nº‡S·„P²K°}I®{G®{G®{G®{G®{G°yH±zI³|K¼…T¾‹W¶ƒO•l?xO"M2?$" ÒœdÒœdÒœdÒœdÒœdÒœdÒœdÒœdÒœdÓeÔždÔždÔždÔždÔždÔždÔždÔždÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÔ dÔ dÔ dÔ dÔ dÔ dÖ dØ¢fÖ¢fÖ¢fפeפeÙ£gÙ£gÙ£gÚ¤hؤjؤjؤjؤj×£iÙ¥kÙ¥kÙ¥kؤhÙ¥iÙ¥iÚ¦jÚ¦jÚ¦jÚ§hÚ§hÚ§hÚ§hÚ¦jÚ¦jÚ¦jÛ§kÛ§kÛ§kݧkݧkÞ¨lÞ¨làªnÞ¨lÞ¨lÞ¨là«là«lá«oá«oá«oá«oâ­nâ­nã®mã®mã®oä¯pä¯nä¯nä¯nä¯nä¯på°qå°qå°qå°qå°qç®sç®så°qä¯pä¯nå°oã°oå²qç²qè³rè³tè³tè³tè³tè³rè³rè³tè³tè³ré´së³së³së³së³së³sì´tì´të³sì´tíµuíµuî¶vð¶vð¶vð¶vð¶vð¶vð¶vð¶vð¶vð¶vð¶vî¶tî¶tð¸vð¸vð¸vð¸vð¸vð¸vò¸vò¸vô¸wô¸wô¸tõ¹uôºuôºuôºuôºuòºxùÁÿÊŠÿË‹ò¸zˆJŠM^!K FM VWWYZZ\^XZ ¡R%ÿÇ?ÿÒJÿØ/ÿÌ#ÿ®ÿšêq ÍT¹B³<ª6«7·:»>ò.ø4ÿ1 ÿ*› ‰72  '6__nt W;~´'ÿ5ÿ9ÿ4ÿ7ÿ1ÿÅ ¹“ “ ‰ Š€} €»×1ÿC%ÿR4ÿN*ÿF"ÿ< ÿ.ÿ$ ÿ ÿ ýú!ÿ(ÿ-ÿ0ÿ-ÿ-ÿ3ÿ4ÿ3ÿ9ÿ<"ÿ@&ÿB%ÿE(ÿF+ÿA&ÿ7!ÿ2ÿ+ÿ)ÿ-ÿ-ÿ/ÿ/ÿ0ÿ0ÿ0ÿ,ÿ*û% ðêáÞÖÓ Ð Ð ÒÔÚá é# -î(ÿ/ÿ5ÿ6ÿ9ÿ<ÿ=ÿ=ÿ=ÿ:ÿ:ÿ<ÿ<ÿ;#ÿ;#ÿ9ÿ6ÿCÿYÿ†ÿ˜#ÿ¬ÿ­ÿ¯ÿ®ÿ®ÿ­ÿ®ÿ¬ÿ®ÿ¬ÿ­ÿ¯ÿ°ÿ±ÿ¶ÿ¼ÿÃÿÉÿÏÿÏÿ¾ â’“DQ% & ,20/23323,+%"   %2 >(?)>&&#-144/-'&  -.: F&T4hF%~\;•qH¯‹bÅŸtҬ߶…ç¾êºƒë»„ñ¼}ì·xõ¾tõ¾tú½nýÀqÿ¿mÿÀnÿÀpÿÁqÿÁqÿÁqÿÃrÿÄsÿÂsÿÁrÿ¾rÿ¼pÿ¸uýµrþ±tþ±tü¯tü¯tö°uô®sð®uî¬sè­rè­rå­oå­oä«pä«pæ¬nå«mäªläªlå¨nå¨nå¨nã¦lÞ¨nÞ¨nà§nà§nߦmߦmߦmߦmÞ¥jߦkܦlܦlܦjÛ¥iÝ¥gÝ¥gܤfÝ¥gܤfÛ£eÛ£cÛ£cܤbܤbߣdߣdÝ£eÞ¤fݤkߦmâ§pÝ¢kמgÏ–_Ç[»O¢m?’]/P'qBm@_2 O%IGGk: ‘`3§rD¥pB¦qC¨sE«tF«tF¦qC£n@›f3—b/ªuB¿ŠWÃ\·„P­zF©vB©vB«xD«xD¨uA¨uA§t@¢p9¤r;³€N¾‹Y¼‰Uµ‚N«zFªyEªyEªyEªyEªyE«xD¬yE©vB®{G·„P¾‹W¾‹Wµ‚N¯|H®{G­zF­zF®{G¯|H¯zI±|K´€Fº†LµŠR¨}EyZ8I*ÒœdÒœdÒœdÒœdÒœdÒœdÒœdÒœdÓeÓeÔždÔždÔždÔždÔždÔždÔždÕŸeÓŸcÓŸcÓŸcÓŸcÓŸcÓŸcÔ dÔ dÔ dÔ dÔ dÔ dÖ dØ¢fÖ¢fÖ¢fפeפeÙ£gÙ£gÙ£gÙ£gؤjؤjؤjؤjؤjÙ¥kÙ¥kÙ¥kÙ¥iÙ¥iÙ¥iÚ¦jÚ¦jÚ¦jÚ§hÚ§hÚ§hÚ§hÛ§kÛ§kÛ§kÛ§kÛ§kܨlݧkݧkÞ¨lÞ¨làªnàªnàªnàªnà«lá¬má«oá«oá«oá«oâ­nâ­nã®mã®mã®oä¯pä¯nä¯nä¯nä¯nä¯på°qå°qå°qå°qå°qç®sç®så°qç²sç²qç²qå²qå²qç²qè³rè³tè³tè³tè³tè³rè³rè³tè³tè³ré´së³së³sì´tì´tíµuíµuíµuî¶vî¶vî¶vî¶vî¶vð¶vð¶vð¶vð¶vð¶vð¶vð¶vð¶vò¸xð¶vî¶tð¸vð¸vñ¹wñ¹wñ¹wñ¹wòºxôºxó¹wõ¹xõ¹xõ¹uõ¹uôºuôºuôºuôºuñ¹wñ¹wñ¹yó»{þĆÿÏ‘ÿÌ”í°xÀ€OŠJ\GFR[\\[`a]^ÅtÿÃ;ÿÄÿ³ -ùÕiÃJ·>³<·@µAµA¹<¹<ò.ø4ÿ2 ÿ-œ†: -5   -  &"95?=+"…Ä7(ÿ:ÿ:ÿ6ÿ7ÿ1ÿƹ‘‘† -† -~~| -yªÄÿ8ÿN0ÿO+ÿH$ÿ>"ÿ1ÿ% ÿ$ ÿ' ÿ*ÿ0ÿ0ÿ* ÿ) -þ% -øðò!ÿ%ÿ+ ÿ0ÿ7ÿ= ÿC&ÿG,ÿF+ÿ@*ÿ:$ÿ0ÿ*ÿ.ÿ-ÿ/ÿ/ÿ0ÿ-ÿ.ÿ.ÿ,ý' ô" îæá×Ó Ð Î Í ÒÔ×ßåû% ÿ.ÿ1ÿ4ÿ7ÿ;ÿ<ÿ<ÿ:ÿ:ÿ;ÿ;ÿ:"ÿ:"ÿ:ÿ9ÿ=ÿBÿcÿ -ÿ¥ÿ± ÿ²ÿ¯ÿ­ÿ­ÿ®ÿ®ÿ¯ÿ²ÿ¶ÿ¸ -ÿºÿ»ÿÀÿÁÿÄÿÆÿË ÿÏÿÓ"ÿÍï :¢S; &' .5447 -6 553.,'%  ! '. 7#E/ G1"8"# !& -+/12/,*$"   )2I%W1sM"–m<³ŠYÍfÛ«têµvò½~úÃyÿÉÿÈyÿÄuÿÃqÿÀnÿ¿oÿÁqÿÁqÿÁqÿÂqÿÂqÿÀqÿÀqÿ¿sÿ½qÿ¸uýµrþ±tþ±tü¯tü¯tõ¯tô®sð®uð®uè­ré®sæ®på­oå¬qä«pæ¬nå«må«mäªlæ©oå¨nå¨nå¨nàªpÞ¨nà§nà§nà§nߦmߦmߦmߦkߦkݧmܦlܦjܦjÝ¥gÝ¥gÝ¥gܤfܤfܤfܤdܤdܤbÛ£aÝ¡bÝ¡bÜ¢dÙŸaמeמeÚŸhÞ£lÜ£lÜ£lÜ¢pÜ¢pܧyÒoÔkµ†]¢uP—jEŠ`AzP1pG#lC{J”c6¦qC¥pB¦qC¦qC©rDªsE§rD£n@›f3™d1ªuBÁŒYÂ[µ‚N«xD«xD©vB«xD«xD«xD§t@¡n:žl5¢p9´O¼‰W¹†R°}I©xD¦uA§vB§vB§vB§vB¨uA¨uA¥r>­zF¹†R½ŠVº‡S´M®{G®{G­zF­zF®{G®{G¯zI±|K¶‚H»‡M°…MŸt€A WGM X^dbdX`´i4ê3ËqÇZ®A..•2•2—0™2¦6¬<å-ì4ÿ6ÿ3Ò¶up&    )' ªä;$ÿ=ÿ;ÿ8ÿ;ÿ4ÿô…†‚ ‚ z -w…•ó% ÿ>%ÿD'ÿC&ÿA(ÿ5ÿ" ÿ ÿ*ÿ/ÿ0ÿ)þ# ü! ã -ØÂÄÆ Ë% ß( ê3ÿ8ÿ@'ÿF+ÿG,ÿI*ÿE&ÿ;"ÿ3ÿ-ÿ+ÿ-ÿ-ÿ/ÿ-ÿ, ÿ, ÿ* ÿ( ÿ% þ!ÿÿÿùöò -æçÝ -ß ÕÚçð'þ,ÿ3ÿ6ÿ7ÿ8ÿ;"ÿ8"ÿ8"ÿ;ÿ;ÿ>ÿ=ÿ<ÿ=ÿ;ÿ;ÿ=ÿIÿtÿ.ÿ© ÿª -ÿ®ÿ®ÿ²ÿ³ÿ¹ÿ»ÿ¾ÿÂÿÆÿÈÿËÿÍÿÍÿÏÿÐÿÑÿÓ -ÿÖ ÿÒÿ±§V_-; -478 : = 7 7 422/''$ - &.>"F*G+J.B" + " ,5 >=9 5 33-)   -* > X3pK.‘g?«YÑœ]êµvÿÆzÿÉ}ÿÌyÿÈuÿÆtÿÃqÿÃqÿÂpÿÃpÿÂoÿÀqÿÁrÿÁtÿ¾qþ¸tü¶rü²tü²tú°vú°vó¯tò®sð¯sð¯së°ué®sæ®pæ®på­oå­oç­mæ¬lå«må«måªoä©nå¨nå¨nàªpÞ¨nߦmߦmà§nߦmߦmߦmߦkÞ¥jÞ¥jÞ¥jݤiݤiݤiÜ£hܤfܤfܤfܤfܤfܤfÜ£hÜ£hÚ¡fØŸdØŸdמcמcÖbØbÚŸdÙžcÙžcÙžcÚŸdØ¢jÛ¥mÛ¥mÚ¤lÛ¥oÖ jÍ™c¿‹U®zD¥q; k8£n;§p?¥n=§pB¥n@§pB§pB§pB¤m?še2˜c0©tAÁŒY¿ŽZ²M©xD¥t@¦s?¦s?§t@§t@¥p=ži6˜d.¡m7´O½ŠX·„P²K§vD¤sA¦uC¦uC¥t@¥t@§t@¥r>¢o;¬yEº‡S¾‹W¶ƒO®{G«xD¯|H¬zC¬zC¬xB®zD°yH³|Kµ„D»ŠJ©K‹c-K1ÓeÓeÓeÓeÓeÓeÓeÔžfÔžfÔžfÔždÔždÓŸeÓŸeÓŸeÓŸeÕŸeÕŸeÕŸcÕŸcÖ dÖ dÖ dÖ dÖ dÖ dÖ dØ¢fØ¢fØ¢fØ¢fØ¢f×£g×£gÙ£gÙ£gÙ£gÙ£gÙ£gÙ£gؤjؤjؤjؤjÙ¥kÙ¥kÙ¥kÚ¦lÚ¦jÚ¦jÚ¦jÚ¦jÚ¦jÛ§kÛ§kÛ§kÚ¦jÛ§kÛ§kÛ§kÛ§kÛ§kܨlܨlÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnàªná¬má¬má«oá«oá«oâ¬pâ¬pâ¬pã®oã®oã®oä¯pä¯nä¯nä¯nä¯nå°qå°qå°qå°qå°qå°qå¯så¯sé°ué°ué±qé±qç²qè³rè³tè³tè²vè²vè²vè²vè³rè³ré´ué´ué´ué´uë³së³síµuíµuì´tî¶vî¶vî¶vð¸zî¶xî¶vî¶vð¸xñ¹yñ¹yñ¹yñ¹yñ¹yñ¹yñ¹yòºzñ¹yòºxòºxòºxòºxòºxòºxòºxòºxõ»yõ»yöºyöºy÷»w÷»wö¼zö¼z÷½x÷½xö¼wö¼wö½qö½qõ¼nö½o÷¼o÷¼oú¼xþÀ|ÿÈ‹ÿÑ”ÿÈ”ã¤p«kCx8UHS ]`d[_œB¯U´G°C£4—(Œ)‰&‰"&Ÿ/©9å-ê2 ÿ0ÿ3é% -Ö€q% -"# " -  +- " -² íD-ÿ>ÿ;ÿ;ÿ>ÿ:!ÿij„ƒ~~ww„Š -äÿ4ÿ?"ÿD'ÿ>%ÿ0ÿ ÿ" ÿ.ÿ1ÿ/ÿ)ÿ'û -ÖÍ º»¸ºÊÔÿ)ÿ9 ÿA&ÿG,ÿL-ÿK,ÿB)ÿ7ÿ-ÿ+ÿ+ÿ-ÿ/ÿ-ÿ-ÿ, ÿ, ÿ+ ÿ,ÿ+ÿ+ÿ,ÿ,ÿ'ÿ%ÿôì Ü ÛÐÓÝæô" -þ,ÿ-ÿ1ÿ4ÿ6ÿ7!ÿ8"ÿ;ÿ;ÿ>ÿ=ÿ<ÿ=ÿ;ÿ;ÿ;ÿ9 ÿPÿi -ÿ•ÿ¤ÿ¬ÿ®ÿ³ÿ·ÿ½ÿ¾ÿÃÿÆÿÊÿËÿÎÿÏ ÿÏÿÐÿÓÿÔÿ×ÿ×ÿÒÿÔõ¤/ŸN?9 078 9 -: ; -7 6 20/.'(*((/ :#B+L0K/@$ 00- 70 -++.4/./1*'      "$&(5 Ep;•`!Å…9á¡Uø²_ÿÁnÿÊxÿÊxÿÆtÿÅsÿÆsÿÃpÿÁrÿÂsÿÁtÿ¾qþ¸tü¶rþ´vû±sú°vû±wó¯tô°uñ°tð¯së°uè­ræ®pæ®på­oå­oç­mæ¬læ¬nå«måªoä©nå¨nå¨nÞ¨nÞ¨nߦmߦmà§nߦmߦmߦmߦkÞ¥jÞ¥jÞ¥jÞ¥jݤiݤiݤiÝ¥gܤfܤfܤfܤfܤfÛ¢gÛ¢gÚ¡fÚ¡fÚ¡fØŸdÚ¡fÚ¡fÜ¡fÚŸdÜ¡fÚŸdÚŸdÚŸdÖ hÔžfÓeÖ hØ¢lØ¢lؤnÙ¥oÚ¦pÒžhÉ”a·‚O¬uD¥n=¥n@¥n@¥n@¤m?¥n@£l>še2˜c0©tA¾‰V¿ŽZ±€L¦uA¤s?¦s?¥r>¥r>¦s?¡l9›f3˜d.œh2µ‚P¾‹Y·„P®{G¦uC¤sA¥tB£r@¤s?¤s?¤q=¡n:Ÿl8«xD·„Pº‡S²K©vB§t@©vB¨v?¨v?©u?ªv@°yHµ~M¶…E¼‹K§I|T<" ÓeÓeÓeÓeÓeÓeÓeÔžfÔžfÔžfÔždÔždÓŸeÓŸeÓŸeÓŸeÓŸeÓŸeÕŸcÕŸcÖ dÖ dÖ dÖ dÖ dÖ dÖ dÖ dØ¢fØ¢fÙ£gÙ£gÙ£gÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iܦjܦjÚ¦jÚ¦jÚ¦jÚ¦jܦjݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnà«là«là«là«là«lá¬má¬má¬má¬má¬mâ­nâ­nâ­nâ­nã®oã®oã®oã®oä¯pä¯pä¯pä¯pä¯på°qå°qå°qå°qå°qå°qç²sé±sé±sé±sé±sê²tê²tè³tè³tè³tè³tè³tè³tè³tè³té´ué´uë³sì´tì´tíµuíµuíµuî¶vî¶vî¶vî¶vð¸xð¸xð¸zð¸zð¸zð¸zð¸xð¸xð¸xñ¹yñ¹yñ¹yñ¹wñ¹wñ¹wòºxòºxòºxòºuó»vó»vó»võ»võ»võ»võ»vö¼wö¼wö¼w÷½x÷½x÷½x÷½v÷½v÷½v÷½vö¼uö¼uø¼vø¼vø¼võ¹só·qöºtýÁ{ÿΈÿÔŽýÁ{à¢^°r.€<d Zai!bab}3…;‰; AŒ9 -‹8 ‰6‹8’9”; Ñ/Ù7ÿ,ÿ4ÿ0ÿ* âÊ]Q "MU‹““Œ‚†Óÿ:&ÿA#ÿ<ÿ:ÿ=ÿ;%ÿ)áÉŽ‰xxx{yyxyÀå7ÿ= ÿC&ÿ7#ÿ ÿÿ)ÿ5ÿ6ÿ4ÿ1ÿ&òý¥¦ŸÅÜ.ÿ:ÿH-ÿQ0ÿU4ÿM1ÿB&ÿ0ÿ,ÿ*ÿ*ÿ+ÿ,ÿ-ÿ-ÿ,ÿ0ÿ1ÿ2ÿ3ÿ4ÿ7ÿ7ÿ8ÿ1ÿ'ÿýóßàÓ×Ûâ!ï)ö0ÿ1ÿ4ÿ8ÿ;ÿ<ÿ<ÿ=ÿ=ÿ<ÿ<ÿ;ÿ;ÿ9ÿ8ÿ7 ÿ?ÿ^ÿy-ÿŸ ÿ©ÿº ÿ¿ÿÆ -ÿÆ -ÿÌÿÍ ÿÌ ÿÍ -ÿÒÿÒÿÒÿÓÿÕÿÕÿÓÿÕÿÖÿÙÿÕ$å~=K -+6 ;;;;5 -22101++49 -F&L,V/W0W)P" MIJ@ -9D i.y>g/ -L;2.*&*(+  " $ -  B `+‰D›VÃx&ã˜Fþ³aÿÃqÿÇtÿÇtÿÄqÿÀmÿ¾pÿ½oÿ¼rÿºpÿ¸uþ¶sýµyø°tø°vø°vô²yò°wë°ué®sã¯sâ®rã­sã­sã­qã­qâ¬pã­qà¬pß«oá«oá«oÞ¨nàªpà¨jà¨jà¨jà¨jߦkߦkݧkܦjݨiݨiܦjܦjܦlܦlÛ¥iÛ¥iÚ¤hÚ¤hÚ¤hÚ¤hÙ£gÙ£gÙ£iØ¢hÚ¡hÚ¡hØ¢fØ¢fÖ dÖ dÖ dØ¢fÖ fÖ fØ¢jØ¢jØŸjÚ¡lÖ jÔžhÖ dÖ dÓcÑ›aÏ™aÏ™aÏ™aÄŽV´€J¦r<¢m:¢m:¡l9¡l9¡l9 k8™f2˜e1¨uAº‡SÁŽZ²K¦s?¤q=¤q?¤q?¥p?¥p? k:še4˜c2¡l;¶ƒQ¾‹Y¶ƒQ¯|J¥t@¤s?¤s?£r>¢q?¢q? m;žk9j8«xF·„Rº‡U®{I§tB¦s?¦s?§t@§t@¨s@ªuB­xJ³~P¶„M·…N—tDmJ&ÓeÓeÓeÓeÓeÓeÔžfÔžfÔžfÕŸgÕŸeÕŸeÓŸeÓŸeÓŸeÓŸeÓŸeÔ fÖ dÖ dÖ dÖ dÖ dÖ dÖ dØ¢fØ¢fØ¢fØ¢fØ¢fÙ£gÙ£gÙ£gÙ£gÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iܦjܦjÚ¦jÚ¦jÚ¦jÚ¦jݧkݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnà«là«lá¬má¬má¬má¬má¬mâ­nâ­nâ­nâ­nâ­nã®oã®oã®oã®oã®oä¯pä¯pä¯pä¯pä¯på°qå°qå°qå°qå°qå°qå°qç²sé±sé±sé±sé±sê²tê²tè³tè³tè³tè³tè³té´ué´ué´ué´uêµvì´tì´tíµuíµuíµuíµuî¶vî¶vî¶vî¶vð¸xð¸xð¸zð¸zð¸zð¸zð¸xð¸xñ¹yñ¹yñ¹yòºzòºxòºxòºxòºxòºxòºxòºuó»vó»vó»võ»võ»võ»võ»vö¼wö¼wö¼w÷½x÷½x÷½x÷½v÷½v÷½v÷½vö¼u÷½vø¼v÷»u÷»u÷»u÷»u÷»u÷»u÷»uõ¹sÿÃ}ÿ̈ÿΊÿÆÃB‡B g"m%i!c\\]o!|.}*~+€-†3…,Š1Ì*Ù7ÿ.ÿ0ÿ-ÿ.ø4å!iU% [oªª¿Å" -¼²ž —Ýÿ8$ÿ?!ÿ<ÿ:ÿ=ÿ;%ÿ3ì! Ίwwx{yyyx¸ -Ü.ÿ9ÿC&ÿ0ÿÿÿ1ÿ>!ÿ;ÿ8ÿ5ÿ&ë ½¸¤ ¥œ›œš³Èÿ,ÿC(ÿY8ÿ[:ÿN2ÿD(ÿ6ÿ,ÿ)ÿ)ÿ+ÿ+ÿ-ÿ-ÿ,ÿ.ÿ-ÿ-ÿ/ÿ3ÿ6ÿ9!ÿ=ÿ<ÿ7!ÿ0ÿ!ÿå á Ð ÑÔÚæ í'ÿ)ÿ/ÿ4ÿ8ÿ;ÿ<ÿ=ÿ=ÿ<ÿ;ÿ;ÿ;ÿ9ÿ8ÿ:ÿ7 ÿDÿZÿŠÿ¤ÿ¾ ÿÅÿÊÿÊÿÐ ÿÒÿÎ ÿÏ ÿÒÿÔÿÔÿÕÿÖÿ×ÿÖÿ×ÿ×ÿ×ÿÛ*ÿ̼{])6 ;;::4 .1112,. 7@$M-M-P)L% MKMB5EzBº‚_î³÷¼–⪅Ëfµ^›eD‰R;v?(['K4   " $ -& ( ( G[&~9„?ŒAŸT»pÖ‹9í¥Rü´aÿÂoÿÈuÿÇyÿÅwÿÄzÿÅ{ÿ¿|ÿ¹vù±uõ­qø°v÷¯uð®uð®ué®sé®sã¯sâ®rã­sã­sã­qã­qã­qã­qá­qß«oá«oá«oàªpàªpâªlà¨jà¨jà¨jà§là§lÞ¨lݧkݨiݨiܦjܦjܦlܦlܦjÛ¥iÚ¤hÚ¤hÚ¤hÚ¤hÙ£gÙ£gÙ£iØ¢hÚ¡hÚ¡hÖ dÖ dÖ dÖ dÖ dÖ dÖ fÖ fØ¢jØ¢jØŸjÚ¡lØ¢lÕŸiÖ dÖ dÔždÕŸeÓeÒœdÑ›cÍ—_¿‹U¬xB¢m:¢m:¢m:¡l9 k8Ÿj7›h4™f2¦s?³€Lº‡S­zF¢o;¢o;¡n<¢o=£n=£n=ži8™d3”_.Ÿj9·„RÀ[·„R­zH£r>¢q=¢q=¢q=¢q? o=j8˜e3˜e3¨uC·„R·„R­zH¦sA¤q=¤q=¥r>¥r>¦q>¨s@¨sE®yKµƒL³JŒi9[8ÑeÑeÑeÒžfÒžfÒžfÒžfÒžfÓŸgÓŸgÓŸeÓŸeÓŸeÓŸeÓŸeÓŸeÓŸeÔ fÖ dÖ dÖ dÖ dÖ dÖ dÖ dØ¢fØ¢fØ¢fØ¢fÙ£gÙ£gÙ£gÚ¤hÚ¤hÚ¤hÚ¤hÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjܦjÚ¦jÚ¦jÚ¦jÚ¦jݧkݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnà«là«lá¬má¬má¬má¬má¬mâ­nâ­nâ­nâ­nâ­nã®oã®oä¯pä¯pä¯pä¯pä¯på°qå°qå°qå°qå°qå°qå°qå°qå°qå°qå°qç²sç²sç²sç²sè³tè³tè³tè³té´ué´ué´ué´ué´ué´uêµvêµvì´tíµuíµuíµuíµuíµuî¶vî¶vî¶vî¶vð¸xð¸xð¸zð¸zð¸zð¸zñ¹{ñ¹{ñ¹{ñ¹{òºzòºzòºxòºxòºxòºxó»yó»yó»yó»yõ»yõ»yõ»võ»võ»vö¼wö¼wö¼wö¼wö¼wö¼wö¼w÷½x÷½xö¼u÷½v÷½v÷½vø¼vú¾xú¾xø¼vú¾xú¾xú¾zû¿{üÀz÷»uøºtý¿yÿÆ}÷´k±k'o)]b[YQQMF -MMPUX]¥3ó. ö1 ÿ-ÿ,ÿ/ ÿ*¤†1 1 HP Õì1ÿ4ÿ3ÿ4ÿ7ÿ1ÿ+ îÔèÿ1ÿC%ÿ?!ÿ:ÿ;ÿ=ÿ:ÿ%ó¨œz{xz~~tt¯Ð)ÿ6ÿC(ÿ-ÿÿ" ÿ:!ÿ@!ÿ=ÿ;#ÿ6ÿ#î ¹ ´™ › –” ”œßù7ÿN/ÿU6ÿO4ÿI.ÿ8!ÿ.ÿ'ÿ&ÿ+ÿ-ÿ+ ÿ* ÿ+ÿ)÷(ö'ò,õ/ø2þ8ÿ8ÿ=ÿ? ÿ? ÿ4ÿ'ÿù áàÏÓÐ×"è& ï-ÿ/ÿ6ÿ6ÿ9ÿ;ÿ<ÿ9ÿ:ÿ9ÿ7ÿ8ÿ8ÿ8ÿ8ÿ4ÿ7ÿUÿv0ÿ©ÿ¸-ÿËÿËÿÔÿÕÿÓÿÒÿÖÿ×ÿ×ÿ×ÿ×ÿØÿÙÿÙÿÚÿÜÿ×ÿÜñ©‚:;D<<764/ +.4556BG! S%Q# V!W"UE =RˆL!Ñ•jÿÄ’þÂþÊþÊÿÅŒÿÆÿÈŠÿÈŠþÈõ¿‡â®xÔ jµ‚N—d0l>G/'""$&$$""     # %AW%…=G —F—F™IžNªZºjÏ.è˜Gô¦Yÿ±dÿ¹pÿÂyÿ„ÿ„ÿ¼…ÿ·€þ·ƒû´€ñ²zî¯wå¯uå¯uâ®rá­qã­sã­sã­sã­sã­uä®vä®vã­uâ¬ràªpàªnàªnâªjà¨hÞ©jÞ©jà¨jà¨jÞ¨lݧkݧkݧkܦjܦjÛ¥kܦlܦlÛ¥kÜ£jÜ£jÜ£jÛ¢iÛ¢iÛ¢iÚ¡hÚ¡hØ¢hÖ fÖ fÖ fÖ fÖ fÖ fØ¢hÔ fÔ fÖ hÖ hÖ hÖ hÕŸgÖ hÕŸeÕŸeÔžfÔžfÓeÓeÒœfÑ›eÌ–`º„N¯yC¥o9¡m7¡m7Ÿk5žj4žj4œh2¢o;«xD®{G©vB m9¡n: k:¡l;¡l;¡l;ži6™d1•`/Ÿj9·‚Q¾‰Xµ‚P«xF¤sA£r@£rC¢qB¤qB¤qB›h9–c4˜e6¨uF·„Uµ‚S©vD¤q?¢o=¡n< m> m>£n=¦q@¥rC©vG´‚K°~Gƒ_6M) -ÑeÑeÑeÒžfÒžfÒžfÒžfÓŸgÓŸgÓŸgÓŸeÓŸeÓŸeÓŸeÓŸeÓŸeÔ fÖ¢hÖ dÖ dÖ dÖ dÖ dÖ dÖ dØ¢fØ¢fØ¢fØ¢fÙ£gÙ£gÙ£gÚ¤hÚ¤hÚ¤hÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjܦjÚ¦jÚ¦jÚ¦jÛ§kݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªná¬má¬má¬má¬má¬má¬má¬mâ­nâ­nâ­nâ­nã®oã®oã®oä¯pä¯pä¯pä¯på°qå°qå°qå°qå°qç²sç²sç²sç²sç²sç²sç²sç²sç²sè³tè³tè³té´ué´ué´ué´ué´ué´uêµvêµvêµvêµvêµvíµuíµuíµuíµuíµuî¶vî¶vî¶vî¶vð¸xð¸xð¸xð¸zñ¹{ñ¹{ñ¹{ñ¹{ñ¹{òº|òº|òºzòºzó»yó»yó»yó»yó»yó»yó»yó»yõ»yõ»yõ»võ»võ»vö¼wö¼wö¼wö¼wö¼wö¼wö¼w÷½x÷½xö¼u÷½v÷½v÷½vø¼vú¾xú¾xû¿yüÀzüÀzüÀ|ýÁ}üÀzüÀzý¿yþÀzÿ¾uÿÁxñ«g¹s/p(Q W[RPCB@@EFJO u‘ì'õ0 ÿ-ÿ,ÿ)ÿ+ ¬Š2 2 I_ä)î3ÿ, ÿ0ÿ.ÿ4ÿ7ÿB#ÿ/Ôå ÿ/ÿD&ÿA#ÿ;ÿ:ÿA#ÿ?!ÿ0ÿ± Ÿzzwy~~ut¯Ì%ÿ1ÿA&ÿ*ÿÿ)ÿ>%ÿB#ÿ>ÿ<$ÿ7ÿ'𠸲— — ’ ’ ’– -• Î é' ÿ>ÿL-ÿN3ÿJ/ÿ;$ÿ.ÿ&ÿ&ÿ/ÿ1ÿ,ÿ) ÿ& ùìéâåè"î( ÿ* ÿ4ÿ>ÿC$ÿ='ÿ5ÿ+ÿè âÍ ÑÉÍÝå# ÿ( ÿ0ÿ1ÿ5ÿ:ÿ;ÿ9ÿ9ÿ7ÿ7ÿ8ÿ8ÿ8ÿ;ÿ8ÿ3ÿ=ÿNÿ†ÿ¨ÿÊÿÌÿÔÿÕÿÕÿÔÿ×ÿ×ÿ×ÿ×ÿÙÿÙÿÙÿÙÿÜÿÝÿÔÿÝÿÂ6­eBD@ <7640 --0 69 -9 -9 -D -G! N P" M=?[#˜[7Ý |ÿÇœÿË ÷»‰ò¶„í²yí²yò·~õºô¼~õ½õ¿‡ó½…üÈ’ÿË•úÇ“õŽ㵉Πt£vQ{N)\2B7- #!$( ) ' -$ $ " -" -# " -" " " ? W%…=‘I œKžMšJ’B?Ž>=’BœN®`¹m$Âv-ІHá—Yë¢kõ¬uþ·ƒÿ¹…ú»ƒùº‚ì¶|ç±wã¯sã¯sã­sâ¬rá«qâ¬râ¬tâ¬tâ¬tâ¬tâ¬rá«qàªnàªnâªjâªjÞ©jÞ©jà¨jà¨jÞ¨lÞ¨lÞ¨lݧkݧkݧkܦlܦlܦlÛ¥kݤkÜ£jÜ£jÜ£jÛ¢iÛ¢iÚ¡hØŸfÖ fÖ fÖ fÖ fÖ fØ¢hØ¢hÖ fÔ fÖ¢hØ¢jÖ hØ¢jØ¢jÖ hÖ hÖ fÕŸeÔžfÔžfÔžfÓeÓgÓgÔžhΘbÅY¶€J±}G¯{E¨t>¥q;§s=¡m7¡n:¦s?«xD§t@žk7žk7Ÿj9 k:ži8œg6›f3™d1”_.ži8·‚Q¾‰X³€N©vD o=Ÿn<Ÿn?žm> m> m>˜e6”a2˜e6©vG·„Uµ‚S§tB m;žk9žk9Ÿl=Ÿl=¢m<£n=¢o@¨uF³J¯}F‚^5I%ÒžfÒžfÒžfÒžfÒžfÒžfÒžfÓŸgÓŸgÓŸgÓŸeÓŸeÓŸeÓŸeÓŸeÔ fÖ fÖ fØ¢fØ¢fØ¢fØ¢fØ¢fØ¢fØ¢fÙ£gÙ£gÙ£gÙ£gÙ£gÙ£gÙ£gÚ¤hÚ¤hÚ¤hÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjܦjݧkݧkܦjݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªnàªnàªnàªnà«lá¬mà«là«lâ­nâ­nâ­nã®oã®oã®oã®oã®oã®oã®oä¯pä¯pä¯på°qå°qå°qå°qå°qç²sç²sç²sç²sç²sç²sç²sç²sç²sç²sè³tè³tè³té´ué´ué´ué´sé´sé´sêµtêµvêµvë¶wë¶wíµuíµuíµuíµuíµuî¶vî¶vî¶vî¶vî¶vî¶vð¸xð¸xð¸xñ¹yñ¹yñ¹yñ¹yòºzòºzòºxòºxó»yó»yó»yó»yó»yó»yó»yó»yõ»{õ»{õ»yõ»yõ»yõ»yõ»vö¼wö¼wö¼w÷½x÷½x÷½x÷½x÷½v÷½v÷½x÷½xø¼xú¾zú¾xú¾xú¾xú¾xù¿zúÀ{ûÁzúÀyù¿xúÀyþÁvÿÄyÿÊ‚ÿÆ~æ¡i§b*n"T] ] ^]T OMKLKHX½"Ó8ÿ.ÿ,ÿ,ÿ/ -ë͈€¢ú& ÿ-ÿ)ÿ. -ÿ* -ÿ1ÿ;ÿ= ô"ÒÅî1ÿC$ÿ? ÿ=#ÿ8ÿE*ÿE*ÿ>ÿ(äÍ‘wy{~z{±Ë" ÿ2ÿ>$ÿ&ÿÿ/ÿC&ÿB%ÿ>!ÿ=%ÿ9!ÿ*÷¼µ““Ž -• – ŽŽ­ Ä"ÿ3ÿD*ÿI/ÿD*ÿ7ÿ(ÿ%ÿ-ÿ0ÿ1ÿ.ÿ)ù$íÊÈ´¶¶¸Ñß-ÿ4ÿ?"ÿ>ÿ>ÿ:ÿ*ÿ -ó -Ó ÒÁÅÏÖþ%ÿ.ÿ/ÿ2ÿ6ÿ7ÿ9ÿ9ÿ;ÿ;ÿ8ÿ8ÿ7ÿ7ÿ5ÿ5ÿ4ÿ3ÿLÿp*ÿ®ÿ¿0ÿÓÿÕÿÖÿÖÿØÿØÿÚÿÚÿÚÿÜÿÝÿÝÿÜÿÛÿ×ÿßÿÕ%Þ‘`#R=!; 672 -/3> -A! K N#V# QP=@n-·tFî«}ÿÇ’ÿÈ“ÿ¿†ù·~ò´wóµxó¹yò¸xð¸zð¸zì¹zê·xê¶|ì¸~éµ{éµ{ï¹ó½ƒø¿†úÁˆüÁŠù¾‡î³~ߤoÓ™lˆ[šg:‰V)k9\*@2&%',24 -3 2 -- -- -, , ) -) -* * $ -% =T%ˆ=“H˜F˜FšIœK›I˜F”C‹:ƒ6‚5w.u,f m'y8ŽM(¨kG½€\ÈiØ yÞ­‚é¸ì¾Žë½ì»‰é¸†è¶å³|â®xß«uàªràªrÞ¨nÞ¨nÞ©jà«lÜ©jÞ«lÞ«lÞ«lÞ¨lÞ¨lÞ¨nÞ¨nÞ¨nݧmݧmܦlܦlÛ¥kܦnܦnݤkݤkݤkÜ£jÜ£jÜ£jÚ¡hÚ¡hÚ¡jÚ¡jØ¢hØ¢hØ¢hØ¢hÖ fØ¢h×£iÖ¢hØ¢jØ¢jØ¢hØ¢hÖ fÕŸeÕŸcÔžbÔždÔždÔžfÔžfÓeÓeÕŸgÕŸgÒœfÊ”^È’\ÅY»‡Q¶‚L´€J°|F©tA§r?¥p=¡l9œg6œg6 i6f3œg4ži6œg4›f3˜c0 k8±|K¸ƒR¯|J¥r@ m;Ÿl:Ÿl=žk<žk<Ÿl=šg8“`1˜e6¨uF¶ƒT´R¬yJ¤qBŸl?Ÿl?Ÿl=Ÿl=¡l;£n=¡nA§tG³J¯}Fƒ_6I%ÒžfÒžfÒžfÒžfÒžfÒžfÒžfÓŸgÓŸgÓŸgÓŸeÓŸeÓŸeÓŸeÔ fÔ fÖ fØ¢hØ¢fØ¢fØ¢fØ¢fØ¢fØ¢fØ¢fÙ£gÙ£gÙ£gÙ£gÙ£gÚ¤hÚ¤hÚ¤hÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjܦjܦjܦjܦjܦjݧkݧkݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªnàªná«oá«oá¬má¬mâ­nâ­nâ­nâ­nâ­nã®oã®oã®oã®oã®oä¯pä¯pä¯pä¯på°qå°qå°qå°qç²sç²sç²sç²sç²sç²sç²sç²sç²sç²sè³tè³tè³té´ué´ué´uêµvêµvêµtêµtêµtêµtêµvêµvë¶wë¶wíµuíµuì´tì´tíµuíµuî¶vî¶vî¶vî¶vî¶vî¶vî¶vð¸xð¸xð¸xñ¹yñ¹yñ¹yñ¹yòºxòºxòºxòºxòºxòºxó»yó»yó»yó»yõ»{õ»{õ»yõ»yõ»yö¼zö¼wö¼w÷½x÷½x÷½x÷½xù¿zù¿zù¿xù¿xù¿zù¿zø¼xú¾zú¾xú¾xú¾xú¾xù¿zù¿zúÀyúÀyù¿xúÀyýÀuü¿tÿÀxÿÆ~ÿË“ÿÅéq°d8€/ c[bed_YU S MK¦ É.ÿ1 -ÿ.ÿ,ÿ3ÿ0’®ú& ÿ-ÿ- ÿ. -ÿ.ÿ/ÿ9ÿ1æμà# ÿ>ÿC$ÿ>$ÿ=#ÿH-ÿN3ÿCÿ9ö$Ú˜uxy{z|³ -Í$ ÿ4ÿ<"ÿ$ ÿÿ/ÿC&ÿB%ÿ>!ÿ=%ÿ;#ÿ1 ý½´‘ -• – ‘©»ÿ)ÿ>$ÿF,ÿB(ÿ0ÿ ÿ$ ÿ3ÿ6ÿ5ÿ0ÿ,ù$æÁ ¿ -®®®­ÀÉú!ÿ2ÿ<ÿF'ÿH-ÿ:ÿ$úÔ -Í» ÂÏÖý$ÿ0ÿ0ÿ1ÿ0ÿ4ÿ7ÿ9ÿ;ÿ;ÿ8ÿ8ÿ9ÿ9ÿ7!ÿ8"ÿ:!ÿ7ÿ@ÿLÿ‹ÿ°!ÿÒÿ×ÿØÿØÿÙÿÙÿÜÿÜÿÝÿÜÿÝÿÝÿÜÿÛÿÙÿÛÿÙ)ù¬o2M=!=!7;! -5-,4 A! C#M" N#MCIq6·vQð¯ŠÿÉ›ÿÅ—ÿ¾‰ù·‚÷µ|÷µ|õ·zö¸{ò¸xò¸xñ¹{ñ¹{ì¹zì¹zì¸~ê¶|éµ{ê¶|ëµ{ëµ{í´{í´{ó¸÷¼…þÃŽÿÆ‘ûÁ”ÿÆ™÷Ä—óÀ“é·‘Þ¬†Â•r£vSzP1V, B1**(*, , - -. -///))?T%ˆ=‘FšHšHšI—F—E–D?>‹>‚5r)j!ZO DK -H SZ"k3 ~M"•d9®€PÄ–fÖ¥sã²€í»„í»„í¹ƒê¶€ê´|å¯wä®tâ¬rà«là«lÞ«lÞ«lÞ«lÜ©jàªnàªnàªpàªpÞ¨nݧmݧmݧmܦlݧmݧoݧoÞ¥lݤkݤkݤkÜ£jÜ£jÛ¢iÛ¢iÛ¢kÛ¢kÙ£iØ¢hØ¢hÙ£iÙ£iÙ£iÖ¢h×£iÙ£kØ¢jØ¢hØ¢hÖ fÖ fÖ dÕŸcÔždÔždÔžfÔžfÔžfÓeÕŸgÕŸgÕŸiÖ jÖ jØ¢lÒžh̘bË—aÆ’\½ˆU¸ƒP°{H¬wD¦q@ži8 i6¡j7Ÿj7 k8 k8ži6›f3¢m:¥p?¬wF¨uCŸl:žk9›h6›h9j;›h9šg8˜e6’_0˜e6§tE³€Q´R¨uF¡n?Ÿl? m@ m> m>¢m<¥p?¡nA¦sF²€I®|E€\3G# -ÒžfÒžfÒžfÓŸgÓŸgÓŸgÓŸgÓŸgÔ hÔ hÔ fÔ fÔ fÔ fÔ fÔ fÖ fØ¢hØ¢fØ¢fÖ¢fÖ¢fÖ¢fÖ¢fÖ¢f×£g×£g×£g×£gؤhؤhؤhÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjܦjܦjܦjܦjݧkݧkݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªnàªná«oá«oá«oá«oá«oá«oá«oá¬má¬mâ­nâ­nâ­nâ­nâ­nã®oã®oã®oã®oä¯pä¯pä¯på°qå°qå°qå°qç²sç²sç²sç²sç²sè³tè³tç²sè³tè³té´ué´ué´ué´ué´ué´uêµvêµvêµvêµvêµvêµvêµvë¶wë¶wë¶wíµwíµwíµuíµuî¶víµuíµuíµuî¶xî¶xî¶vî¶vð¸xð¸xð¸xð¸xñ¹yñ¹yñ¹yñ¹yñ¹yñ¹yòºxòºxòºxòºxòºxòºxòºxòºxòºxó»yõ»{õ»{õ»yõ»yõ»yö¼zö¼zö¼z÷½x÷½x÷½x÷½xù¿}ù¿}ù¿zù¿zù¿zù¿zø¼xú¾zú¾zú¾zú¾zû¿{û¿{û¿{úÀ{úÀ{ù¿zù¿zø¼}ø¼}þ¾vþ¾vÿ¿sÿÇ{ÿЇÿË‚ð¬j¾z8’E t'ronnoj YReÚ0ã9ÿ.ÿ2ÿ3ÿ0ÿ* ôêô ÿ* -ÿ, ÿ-ÿ/ÿ.ÿ0ÿ>ÿ5öÞ¼Ùÿ8ÿN0ÿM5ÿD,ÿ@0ÿD4ÿ> ÿ<ÿ+ÿǺˆ…wyƒˆ½ Ø$ ÿ6ÿ;ÿ ÿÿ0ÿC+ÿC%ÿ?!ÿ>'ÿA*ÿ8'ÿ Ï À—•Œ • – Ž– -Ÿô ÿ;!ÿ@(ÿ;#ÿ%ÿÿ*ÿ;$ÿ9ÿ7ÿ5ÿ/ö!ß -³ -³ -š›˜— •ÉÞ* ÿ:ÿT2ÿ_7ÿP(ÿ4!ÿß Ò¿É Ýçÿ*ÿ1 ÿ0ÿ1ÿ2ÿ3ÿ3ÿ7ÿ9ÿ;ÿ;ÿ;ÿ;ÿ:ÿ;ÿ<ÿ=#ÿ<"ÿ6ÿ2ÿU ÿ€8ÿ¾ÿÏ0ÿØÿØÿÝÿÝÿÝÿÝÿÝÿÝÿÜÿÜÿÝÿÝÿÙÿÖÿ×ÿÀG[@ A! =@! 8! ,2;K$ N'Y'U#c$@¾~M÷·†ÿИÿÉ‘þ½÷¶zö¸y÷¹zõ¹|õ¹|õ¸€ô·ó¸ô¹€ò¹€ò¹€í»€í»€í»‚í»‚ìºí»‚캃캃î¸~ñ»ð¼‚ê¶|é³{ëµ}îµ~ñ¸îº‚ﻃøŠÿÉ‘üÈ’úÆﺇܧtÁŒY¢m:ˆU(uBe3 T"E7651402 -/ 0I^&ŒA -–K›J›J•GBŽAŒ?†>…=y6q.d(ZTL=8 3.')&- -3E& a?!{Y;—qL¬†aÁ—oÏ¥}Ü°‚㷉鸉깊鶂æ³æ´}â°yß­rÝ«pß«qß«qß«oß«oàªnàªnàªpÞ¨nÞ¨nÞ¨nݧoݧoÞ¥lÞ¥lÞ¥lݤkݤkݤkÚ¤lÚ¤lÚ¤lÚ¤lÚ¤lÙ£k×£i×£i×£i×£iÙ£iÙ£iØ¢jÙ£kÚ¡jÚ¡jØŸfØŸfÚ¡hØŸfמeמeמeמeמeÖdÕŸeÕŸeÕŸgÕŸgÕŸiÖ jÔžhÕŸiÔžhÔžhÏ›eË—aÆ‘^ÁŒY´L©tAªt>«u?ªt>«u?©u?¦r<¡l9¥p=Ÿl8¢o;¤q?žk9›h4™f2še4›f5™d3™d3—d5“`1˜e3¦sA²M²M¦sDžk<Ÿl=Ÿl=¡l>¡l>¡l;¥p?£nB¦qE®|C®|C„`5I% -ÓŸgÓŸgÓŸgÓŸgÓŸgÓŸgÓŸgÔ hÔ hÔ hÔ fÔ fÔ fÔ fÔ fÔ fÖ fØ¢hØ¢fØ¢fÖ¢fÖ¢fÖ¢fÖ¢fÖ¢f×£g×£g×£g×£gؤhؤhؤhÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjܦjܦjݧkݧkݧkݧkݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªná«oá«oá«oá«oá«oá«oá«oá«oâ­nâ­nâ­nâ­nâ­nâ­nâ­nã®oã®oã®oã®oä¯pä¯pä¯på°qå°qå°qç²sç²sç²sç²sè³tè³tè³té´ué´ué´ué´ué´ué´ué´ué´ué´uêµvêµvêµvë¶wë¶wë¶wë¶wë¶wë¶wë¶wë¶wî¶xî¶xî¶vð¸xð¸xð¸xð¸xî¶vî¶xð¸zð¸xð¸xñ¹yð¸xð¸xñ¹yñ¹yñ¹yñ¹yñ¹yñ¹yñ¹yòºxòºxòºxòºxòºxòºxòºxòºxòºxó»yõ»{õ»{õ»yõ»yõ»yö¼zö¼zö¼z÷½x÷½x÷½xù¿zù¿}ù¿}ù¿zù¿zù¿zù¿zú¾zú¾zú¾zú¾zû¿{û¿{û¿{û¿{úÀ{úÀ{ûÁ|úÀ{ø¼}ýÁ‚ÿÁyÿÁyÿÂvÿÁuÿÅ|ÿ̃ÿˉÿÏÿÁˆæ™`­Q …)ppqp^]io ÃÝ3 ÿ0ÿ1ÿ/ÿ0ÿ/ÿ3ÿ.ý)ÿ, ÿ- ÿ-ÿ/ÿ,ÿ0ÿD%ÿD%ÿ'éÃ×ÿ/ÿQ3ÿU=ÿG/ÿ9)ÿ5%ÿ<ÿ8ÿ3#ÿ#Øňww‚‰Äß+ÿ9ÿ8ÿÿÿ/ÿD,ÿD&ÿ?!ÿ>'ÿB+ÿ?.ÿ(ÚÈš•‰Œ’ ” • žíÿ6ÿ@(ÿ5ÿ -ÿ! ÿ3ÿ>'ÿ=#ÿ;!ÿ9!ÿ0ï Ú±±˜– “ ’ ¼Ëÿ, -ÿM+ÿ`8ÿT,ÿ>+ÿ&æÕ¿Ìäñ(ÿ1 ÿ1 ÿ/ÿ/ÿ1ÿ-ÿ/ÿ3ÿ7ÿ9ÿ;ÿ<ÿ;ÿ:ÿ<ÿ=ÿ=#ÿ<"ÿ8ÿ8ÿFÿ_ÿ®ÿÍ.ÿÚÿÛÿÞÿÞÿÞÿÞÿÜÿÝÿÜÿÜÿÞÿÝÿØÿÖÿ×ÿË ¦]b@ A! @! A" 7 -7A! P)T-_- yG'¿€[û¼—ÿËšÿÁ÷¸€õ¶~ù¸|þ½ú¼}ú¼}÷»~÷»~ö¹õ¸€ô¹€ô¹€ò¹€ò¹€í»€í»€í»‚í»‚í»‚í»‚ìºƒé·€î¸~ì¶|ì¸~í¹ò¼„ô»„ñ¸îº‚éµ}é³{ëµ}츂ﻅöÁŽüÇ”üÇ”ûÆ“õ•ï¼ã±‹Ø¦€Ì|§xWˆU8uB%R$ E6-""?[#Š?D ”C?Œ>†8ƒ6„77~6t1m*d([ID< 8 4 4 1 .'"%4F Q'g=€T&˜l>³‚SÈ—hÙ¦râ¯{è¶캃ìºìºç³yâ®tà¬pß«oÞ¨làªnàªpàªpÞ¨nàªpàªrÞ¨pߦmÞ¥lݤkÞ¥lÞ¥lݤkÛ¥mÛ¥mÛ¥mÛ¥mÚ¤lÛ¥m×£i×£i×£i×£iÙ£iÚ¤jÙ£kÙ£kÛ¢kÚ¡jÚ¡hÚ¡hÚ¡hØŸfØŸfØŸfØŸfØŸfמeמeÔždÕŸeÕŸgÕŸgÕŸiÕŸiÕŸiÕŸiÕŸiÓgÏ›eÏ›eÒj͘eÆ‘^¼‡T½‡Q¿‰SÁ‹UÁ‹U·ƒM´€J¯zG¬wD¦s?¦s?¢o=šg5›h4šg3›f5še4›f5›f5›h9—d5—d2¢o=®{I®{I¥rCj;j;›h9ži;ži;ži8¥p?¥pD¥pD¬zA°~E†b7N*  ÓŸgÓŸgÓŸgÓŸgÓŸgÓŸgÓŸgÔ hÔ hÔ hÔ fÔ fÔ fÔ fÖ¢hÖ¢hØ¢hÙ£iÙ£gÙ£g×£g×£g×£g×£g×£gؤhؤhؤhؤhؤhÙ¥iÙ¥iÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjܦjܦjݧkݧkݧkݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªná«oá«oá«oá«oá«oá«oá«oá«oá«oá«oâ­nâ­nâ­nâ­nâ­nâ­nâ­nã®oã®oã®oã®oä¯pä¯pä¯på°qå°qå°qç²sç²sç²sç²sè³tè³rè³rè³té´ué´ué´ué´ué´uê´xê´xê´xê´xê´xëµyëµyëµyë¶wì·xì·xì·xì·xì·xî¶vî¶vð¸xð¸xð¸zð¸zð¸zñ¹{ñ¹{ñ¹{ñ¹{ñ¹{ñ¹{ñ¹{ñ¹{ñ¹{ñ¹{ñ¹{ñ¹yñ¹yñ¹yñ¹yñ¹yñ¹yñ¹yñ¹yñ¹yñ¹yñ¹yñ¹yñ¹yòºzó»{ó»{õ»yõ»yõ»yö¼z÷½{÷½{÷½{ù¿}ù¿}ù¿}ù¿}ù¿}ù¿}ù¿}ù¿zúÀ{úÀ~ûÁûÁûÁûÁûÁûÁûÁûÁûÁûÁúÀ~üÀzýÁ{ÿÆ}ÿÊÿÈ|ÿÆzÿÅvÿÅvÿÄyÿÄyÿÇ€ÿ͆ÿʆõ¯kЃF¤W€-lmlps‹˜Ô0á=%þ3û0ÿ- ÿ- ÿ,ÿ,ÿ+ÿ,ÿ.ÿ.ÿ-ÿ1 -ÿ7ÿ9ÿ0ÿúÿÿ4ÿ? ÿD&ÿA#ÿ7ÿ7ÿ;"ÿ=$ÿ7ÿ/ÿ" í ½¶š¢ªßø.ÿ6ÿ0ö íÿ(ÿC*ÿE*ÿ@%ÿ>&ÿ@(ÿA,ÿ5 ÿ 欧”Œ - • žàþ6ÿ?#ÿ3ÿÿ$ ÿ:ÿA%ÿ= ÿ= ÿ< ÿ.òߨ§– ” • • ’’ •Ý ü?"ÿL(ÿI%ÿ?%ÿ.ñßÚëÿ'ÿ-ÿ3ÿ1 ÿ/ÿ, ü+÷& é- ï3ÿ3ÿ6ÿ8ÿ=ÿ>ÿ?ÿ>ÿ?ÿ?!ÿ> ÿ;ÿ>ÿ<ÿ?ÿ…!ÿ¸TÿÚ(ÿÙ'ÿÝÿÜÿÛÿÛÿÜÿÝÿÛÿÛÿÞÿÞÿÚÿÚÿØÿÓ³mcF I#E%E%>" =! -K$R+a2 a2 s>®yMýÂÿÉ”û¾„ýÀ†þÀƒý¿‚÷»~÷»~ö»‚÷¼ƒö»€ö»€óºóºò¹€óºò¹‚óºƒò¹€ò¹€ðº€ðº€ï¹ï¹ñ¸ð·~ê¶|í¹ï¹ï¹ñ»ñ»ò¼‚ñ»ñ»ó½ƒñ½ƒñ½ƒð¼€îº~îº~í¹}ðº~ñ»õ¿…úÄŠýÇýÇþÈ’ÿË•úÄŽðº„à©vÈ‘^®wD”]*|Ck2i.o4ˆC ~9ƒ=†@‡A9|6p*c$`![YRNDA9 6 -2 -1 ) '&$! ,< L0dE(|]@˜mN±†gÄ•rÏ }ଆ沌쵋î·é´†æ±ƒç­{å«yâ©tߦqߤoߤoܦnܦnݤkݤkݤkݤkÛ¥mܦnܦnÛ¥mÙ¥kؤjÙ£iÚ¤jÚ¤jÚ¤jÚ¤jÙ£iØ¢hØ¢hÚ¡hÚ¡hØŸhÚ¡jÚ¡fÚ¡fÚ¡fØŸdØŸdØŸdמeמeÕŸeÔždÔždÔždÔžfÔžfÔžhÕŸiÔžhÓgÒœfÓgÓŸiÓŸiÑ›eÌ–`Ï™cÒœfÓgÓgÏ™cÌ–`Ë•_Á‹U¼ˆR½‰S°|F¢n8 k8Ÿj7Ÿk5Ÿk5Ÿj7œg4›f3˜c0•`/œg6¥p?¦q@¢m<œg6šg8šg8še7še7še4œg6¦l:©o=®yF°{H’e ÿ:"ÿ4ÿ6ÿ8ÿ6ÿ6ÿ/ÿ Ч« ® -¶ë!ÿ6ÿ6ÿ' -ë æÿ$ ÿ?&ÿF+ÿD)ÿ>&ÿB*ÿC.ÿ:%ÿ.õ¶ª‹ŠŒˆŒ -• œßú2ÿ>"ÿ3ÿÿ&ÿ;ÿB&ÿ>!ÿ?"ÿ< ÿ1û -㦥“ -” • • ’‘‹ -Ñì/ÿ>ÿBÿB(ÿ5õæ èù'ÿ/ -ÿ/ -ÿ2 ÿ3ÿ/ÿ'íâÓÛù% ÿ1ÿ5ÿ:ÿ>ÿ?ÿB!ÿC"ÿC%ÿ@"ÿ=ÿ>ÿD"ÿ=ÿfÿ›7ÿÑÿØ&ÿÜÿÜÿØÿÚÿÜÿÝÿÛÿÛÿÞÿÞÿÝÿÚÿØÿÖ¸reJ$L&H(J*I-J.W0^7i:n?X,×¢vú¿Šò·‚ø»÷º€ú¼ú¼÷»~÷»~÷¼ƒ÷¼ƒö»€ö»€ô»‚ô»‚óºóºô»„óºƒóºóºñ»ðº€ñ»ðº€ñ¸ð·~í¹ï»ñ»ðº€ò¼‚ò¼‚ò¼‚ñ»ñ»õ¿…ó¿…ó¿…ó¿ƒó¿ƒñ½îº~ï¹}ï¹}ðº€ðº€ðº‚ëµ}ëµõ¿‰öÀŠúÄŽþÇ”úÃÿÈ•ùÂð·‚æ­xâ§rà¥pæ¡iÙ”\ÕKØ’NÜ–RÙ“OЊM½w:¡b.…Ff*VE;6:45 2 -2 -+ + ( ( $$  " #4 =S$g8‰U/žjD¶UÌ•kØ£uâ­ë±í³í´ê±|ç¬wåªuàªrâ¬tæ­tä«rãªqâ©pÞ¨pܦnܦnܦnÙ¥kÖ¢hÚ¤jÚ¤jÚ¤jÛ¥kÛ¥kÙ£iÙ£iÙ£iÛ¢iÚ¡hÚ¡jÚ¡jÛ¢gÚ¡fÚ¡fÚ¡fÚ¡fÚ¡fØŸfמeÕŸeÕŸeÔždÔždÔžfÔžfÔžhÕŸiÔžhÔžhÕŸiÔžhÒžhÒžhÔžhÔžhÓgÔžhÔžhÕŸiÕŸiÔžhÓgΘbÏ›eÏ›eÃY¸„N°{H®yF­yC¬xB¬wDªuB¥p= k8 k: k: k: k:ži8še4™f7™f7˜c5—b4—b1še4Ÿe3¦l:ªuB­xEšmDvI A3&&$# ÓŸgÓŸgÓŸgÔ hÔ hÔ hÔ hÔ hÔ hÔ hÔ fÖ¢hÖ¢hÖ¢hÖ¢hÖ¢hØ¢hÙ£iÙ£gÙ£g×£g×£gؤhؤh×£gؤhؤhؤhؤhÙ¥iÙ¥iÙ¥iܦjܦjܦjܦjܦjݧkݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnÞªnß«oß«oß«oà¬pà¬pâ¬pâ¬pâ¬pâ¬pâ¬pâ¬pã®oã®oã®oã®oã®oã®oã®oä¯pä¯pä¯pä¯på°qå°qå°qç²sç²sç²sç²sç²sè³tè³tè³té´ué´ué³wê´xê´xê´xê´xê´xëµyëµyëµyëµyì¶zì¶zì¶zì¶zî¹zî¹zî¹zî¹zî¹zî¹zñ¹yñ¹yð¸xð¸xñ¹{ñ¹{ñ¹{ñ¹{ñ¹{ñ¹{ñ¹{ñ¹{ñ¹{ñ¹{ñ¹{òº|òº|òº|òº|òº|òº|ó»}ó»{ó»{ó»{ó»{ó»{ó»{ó»{ó»{ó»{ô¼|ô¼~ô¼~ö¼|ö¼|ö¼|÷½}÷½{÷½{ù¿}ù¿}ù¿}úÀ~úÀ€úÀ€úÀ~úÀ~ûÁýÃýÃÿƆÿƆÿƆÿƆÿƆÿLJÿLJÿņÿÄÿÅ„ÿÆ…ÿÈ‚ÿÊ„ÿÏ„ÿÒ‡ÿуÿÓ…ÿÐÿÏ€ÿÍ‚ÿÈ}ÿÆÿÂ{ÿÃÿÃÿƃÿɆÿÍŠÿÐÿćð¬oÇ}A¢XŽ5vz} — Ç$Ð-ê1ñ8 ý4þ5ÿ1ÿ1ÿ/ÿ0ÿ1ÿ1ÿ0ÿ0ÿ/ÿ+ ÿ-ÿ2ÿ6ÿ;"ÿ>(ÿ;%ÿ4ÿ3ÿ2ÿ4ÿ1ÿ,ÿ% øäâØÞÿ& ÿ;ÿ9ÿ"áÞùÿ8#ÿH,ÿH,ÿC*ÿE,ÿJ0ÿD*ÿ8ÿ!Û Ìžš‡ˆ†‰™ âþ4ÿ=%ÿ2ÿÿ(ÿ<"ÿ@&ÿ=!ÿ=!ÿ>%ÿ6ÿ# ó ®ª”“‘’„ ‡ ¸Ò*ÿ7ÿAÿ>&ÿ3øðú ÿ,ÿ0 ÿ/ ÿ2 ÿ4ÿ0ÿ!¬—\ iÅÙ-ÿ0ÿ9ÿ>ÿ@ÿH%ÿM*ÿE%ÿC#ÿ? ÿ@!ÿC(ÿC(ÿKÿl/ÿµÿË.ÿÝÿÜÿÜÿÜÿÝÿÝÿÜÿÛÿÜÿÜÿÞÿÚÿÙ"ÿÔ½v*k$S(R' T-Y2X3Z5f9nA{H—d7Ì“^ùÀ‹úÀ‚ù¿ø»ø»÷¼÷¼õ¼õ¼õ¼ƒõ¼ƒò¼‚ó½ƒó½‡ó½‡ó½…ò¼„ô»‚ô»‚ô»‚ô»‚ô¼~ô¼~ô»€óºóºóºóºô»€ô»€óºóºô»€ö»€ö»€ô»€õ¼ó½ò¼€ñ¾ñ¾ñ¾}ñ¾}ò½|ò½|ô¼~ó»}ó»}òº|ò¹~ñ¸}ò¹~ò¹~îµ|ð·~óºø¿„ø†ùÇÿÌ‘ÿЕÿÕ–ÿÏÿÉ…ÿÉ…ÿʆÿʆÿÉ„ÿȃö¼|è®nØdÀ…L¢h6…Kg0SC ;6502--))(($#"4H)fA(|W>œrS·nÊzÕ¨…Ü­†á²‹â´ˆãµ‰ä³„ீޭ{à¯}æ³å²€â¯}â¯}ä±}Þ«wÚ¨qئoئmÙ§nܦlÛ¥kÚ¤jÚ¤jÚ¤jÙ£iÛ¢kÛ¢kÛ¢gÛ¢gÛ¢gÛ¢gÛ¢gÛ¢gÚ¡fØŸdØŸdØŸdÕŸeÕŸeÕŸeÕŸeÕŸgÕŸgÔžhÔžhÔžhÔžhÒžhÑgÓeÔžfÓeÔžfÓeÓeÒœbÒœbÔ›dÔ›dÒœbÑ›aÒœbË•[ÄŽVÁ‹SÀŠRÀŠRÁ‹S¾ˆPº„L·I±{E®xBªt>¥o9 l6šf0še2˜c0™d1™d1™d1›f3 f4¤j8©s=¯yC©vGŒY*b5IFJO% U+O% K!A8Ô hÔ hÔ hÔ hÔ hÔ hÔ hÔ hÔ hÔ hÖ¢hÖ¢hÖ¢hÖ¢h×£i×£iÙ£iÙ£iÙ£gÙ£gؤhؤhؤhؤhؤhÙ¥iÙ¥iÙ¥iÙ¥iÙ¥iÚ¦jÚ¦jܦjܦjܦjܦjݧkݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnàªnÞªnß«oß«oß«oà¬pà¬pâ¬pâ¬pâ¬pâ¬pâ¬pâ¬pã®oã®oã®oã®oã®oã®oã®oä¯pä¯pä¯pä¯på°qå°qå°qç²sç²sç²sç²sè³tè³tè³tè³té´ué´ué³wê´xê´xê´xê´xê´xëµyëµyì¶zì¶zì¶zî¸|î¸|î¸|î¹zî¹zî¹zïº{ïº{ïº{ñ¹yñ¹yñ¹yñ¹yñ¹{ó»}ó»}ó»}ó»}ó»}ó»}ó»}ó»}ó»}ó»}ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼|ô¼|ô¼|ô¼|ô¼|ô¼|õ½}õ½}õ½}õ½}õ½õ½÷½}÷½}÷½}÷½}÷½{÷½{ù¿}ù¿}ù¿}úÀ~úÀ€úÀ€úÀ~úÀ~ûÁýÃþÄ„ÿLJÿLJÿƆÿLJÿLJÿLJÿLJÿņÿņÿÆ…ÿȇÿÊ„ÿ͇ÿÕŠÿ׌ÿÖˆÿÖˆÿÓ„ÿÒƒÿ΃ÿÌÿÈÿÇ€ÿÄ€ÿÄ€ÿÅ‚ÿÂÿÁ~ÿÂÿÍÿÓ–ÿÑ•ÿʼnÿ«yçŽ\Ùe@½I$¯)š° ® à -Íäë" -þ)ÿ0ÿ*ÿ-ÿ0ÿ/ÿ/ÿ0ÿ+ ÿ) -ÿ0ÿ4ÿ5ÿ:!ÿ='ÿ<&ÿ7ÿ3ÿ.ÿ-ÿ/ÿ0ÿ1ÿ-÷(îäé# ÿ-ÿ?"ÿ?!ÿ$ßÚñÿ1ÿH,ÿN2ÿK2ÿK2ÿL2ÿF,ÿA%ÿ0éÔ¥ž†‡„ˆ™¢äÿ6ÿ=%ÿ/ÿÿ*ÿ>$ÿ@&ÿ>"ÿ>"ÿ?&ÿ:!ÿ-ú±«ŽŒ„ „ ² -Ê"ÿ1ÿ?ÿ;#ÿ,öñý#ÿ*ÿ- -ÿ/ ÿ3ÿ4ÿ.ý‚DM®Ì ÿ* ÿ7ÿ:ÿ>ÿI&ÿS0ÿJ*ÿE%ÿC$ÿB#ÿD)ÿF+ÿNÿ[ÿ¡ÿÃ&ÿÝÿÝÿÜÿÜÿÝÿÝÿÜÿÜÿÝÿÜÿÞÿÚÿÙ"ÿͶo#n'X-Y.[4^7^9`;nAzM(–c6Å’eõ¼‡ùÀ‹ù¿÷½ø»ø»÷¼÷¼ô»€ô»€ô»‚ô»‚ó½ƒó½ƒó½‡ó½‡ó½…ò¼„ô»‚ô»‚ô»‚ô»‚ô¼~ô¼~ô»€ô»€ô»€óºóºô»€ô»€ô»€ô»€õ¼÷¼÷¼õ¼ô»€ó½ó½ñ¾ñ¾ñ¾}ð½|ò½|ò½|ô¼~ô¼~ó»}ô¼~óºò¹~óºò¹~ò¹€ò¹€ò¹~ð·|î¸|ï¹}óº÷¾ƒÿÄÿņÿÃþÂ~ýÁ}û¿{úÀ{üÂ}þÄ„ÿɉÿÊ‘ÿÈûÁôºˆáª|Ê“e¯yR“]6tB ]+ B2,(%'$$#$  #5 F^1tG$‰Z3žoH¯U²„X·†W¸‡X¹ˆVÀ]Å’`Ì™gÖ£qפrݪvæ³â°yà®wá¯vÝ«rܦlݧmÛ¥kÚ¤jØ¢hÛ¥kÛ¢kÛ¢kÛ¢gÛ¢gÛ¢gÚ¡fÚ¡fÚ¡fÚ¡fÚ¡fØŸdØŸdÖ fÖ fÕŸeÖ fÕŸgÖ hÕŸiÕŸiÔžhÔžhÒžhÑgÒœdÓeÓeÔžfÓeÒœdÑ›aÑ›aÓšcÓšcÒœbÒœbÒœbÓcÓeÒœdÒœdÒœdÑ›cÍ—_Ì–^Ì–^ÄŽXÀŠT¾ˆRµIªv@£o9ži6œg4œg4›f3›f3›f3Ÿe3 f4¥o9ªt>­zKŸl=}P-[. X*a3mC)yO5wM1nD(c<Y2Ô hÔ hÔ hÔ hÔ hÔ hÔ hÔ hÔ hÔ hÖ¢hÖ¢hÖ¢hÖ¢h×£i×£iÙ£iÚ¤jÚ¤jÚ¤jÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÛ¥iÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjܦjܦjݧkݧkݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªnß«oß«oß«oà¬pà¬pà¬pã­qã­qã­qã­qã­qã­qã®oã®oã®oã®oã®oã®oã®oä¯pä¯pä¯pä¯på°qå°qå°qç²sç²sç²sç²sç²sè³tè³tè³tè²vé³wé³wè²vé³wê´xê´xëµyëµyëµyì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¸|î¹zî¹zïº{ïº{ñ¹{ñ¹{òºzòºzòº|ô¼~õ½õ½õ½õ½õ½õ½õ½õ½õ½õ½õ½õ½õ½ô¼~ô¼~õ½õ½õ½õ½}õ½}õ½}õ½}õ½}õ½}õ½}õ½}õ½õ½÷½}÷½}÷½}÷½}÷½}÷½}ù¿}ù¿}ù¿ù¿úÀ€úÀ€ûÁûÁûÁýÃþÄ„þÄ„ÿƆÿƆÿƆÿLJÿLJÿƆÿÄÿņÿÆ…ÿdžÿ̆ÿ͇ÿÒ‡ÿÕŠÿÔ†ÿÔ†ÿÒƒÿÐÿÍ‚ÿË€ÿÈÿÆÿÅÿÅÿÆ‚ÿǃÿȃÿÉ„ÿÌ‹ÿÎÿÏÿÑÿÓ’ÿÖ•ÿÕ‘ÿʆÿ½|ð¨gÕDºd)©E,ƒ” »à Ù' â0ó1ø6ÿ5ÿ6ÿ3ÿ4ÿ5ÿ6ÿ:ÿ?ÿ=ÿ9ÿ4ÿ0ÿ.ÿ)ÿ( -ÿ&ÿ"ÿÿ'ÿ-ÿ3 ÿ?ÿG(ÿ1÷éóÿ*ÿD&ÿQ3ÿR7ÿJ/ÿ?%ÿ?%ÿ?ÿ9ÿ)þÏÅŸ”•£µóÿ6ÿ<&ÿ(ýÿ*ÿ>'ÿ?(ÿ?&ÿ=$ÿ?"ÿ>!ÿ4ÿÍÕ”„„Šˆˆ¨ -¹ÿ) -ÿ>ÿ;ÿ&çà Ù!à(í+ñ/ú/ú/Ö/ R={£*ÿ0ÿ7ÿ5ÿ8ÿC%ÿN0ÿO3ÿH,ÿE'ÿD&ÿE$ÿF%ÿH!ÿL%ÿ†ÿ²3ÿÜÿÝÿÜÿÝÿÜ%ÿÜ%ÿÜÿÜÿÛÿÜ ÿÜÿØÿÖ.ÿÙUl(^4c9j;k<n?pAzL’d6½ˆUå°}ë²wë²wô»€õ¼õ¼õ¼õ¼õ¼ô»€ô»€õ¼ƒõ¼ƒó½ƒò¼‚ò¼„ò¼„ò¼„ò¼„ô»‚ô»‚ô»€ô»€ô»€ô»€ô»€ô»€ô»€ô»€ô»€ô»€õ¼õ¼õ¼õ¼÷¼÷¼õ¼õ¼ó¾ó¾ñ¾}ñ¾}ñ¾}ñ¾}ò½~ò½~ô¼~õ½ô»€ô»€ô»€ô»€óºò¹~ò¹~óºòº|ó»}ó»}ó»}õ»{ö¼|÷½{ù¿}ù¿zö¼wòºxñ¹wïºtî¹sî¹vð»xô¼|÷¿ûÁƒü„ú„ú„û‰÷¾…ì³|מg¾‡V¤m<‚O l9 -W%B0-++)+&&%$! +6 @&G+L0[<$\=%aB'fG,bC(qR7‚`B“qSª†cº–sÆsЧ}Ô¬}Ö®Ù­}Ù­}Ú¬wÖ¨sاsاsÙ§lئk×£gÖ¢fØ£dØ£dÚ¡fÚ¡fÖ¢hÖ¢hÔ dÔ dÔ fÔ fÖ fÖ fÖ hÕŸgÓŸgÓŸgÓŸgÒžfÒždÑcÔždÕŸeÓcÑ›aÒœ`Ñ›_Ï™_Ñ›aÓšaÓšaÓšaÔ›bÔ›bÓšaј_ј_Ï™_Ï™_Í™_Í™_̘`Ë—_Ê–^Æ’Z¿‹S³Gªv@ªv@¯zG°{H¬xB§s=©p;ªq<ªq<®u@µI«u?”_1J~I‰T&—b4¡l>¢o=j8˜e6‹X)Ô hÔ hÔ hÔ hÔ hÔ hÔ hÔ hÔ hÔ hÖ¢hÖ¢hÖ¢hÖ¢h×£i×£iÙ£iÚ¤jÚ¤jÚ¤jÚ¤hÚ¤hÚ¤hÚ¤hÚ¤hÛ¥iÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjܦjܦjݧkݧkݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªnÞªnß«oß«oß«oà¬pà¬pã­qã­qã­qã­qã­qã­qã®oã®oã®oä¯pä¯pä¯pä¯pä¯på°qå°qå°qå°qå°qå°qç²sç²sç²sç²sç²sè³tè³tè³tè²vè²vè²vè²vè²vê´xê´xê´xëµyëµyëµyì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¹zî¹zî¹zî¹zð¸zñ¹{òºzó»{ó»}ô¼~õ½õ½õ½ô¼~ô¼~õ½õ½÷¿õ½õ½õ½õ½ô¼~õ½õ½õ½÷¿õ½õ½}õ½}õ½}õ½}õ½}õ½}õ½}õ½}õ½õ½÷½}÷½}÷½}÷½}÷½}÷½}ù¿}ù¿}ù¿ù¿úÀ€úÀ€ûÁûÁûÁü€ü‚ýÃÿƆÿƆÿƆÿLJÿLJÿƆþƒÿņÿÆ…ÿdžÿ͇ÿÑ‹ÿÕŠÿÕŠÿÓ…ÿÓ…ÿÐÿÎÿÌÿË€ÿÇ€ÿÅ~ÿÄ€ÿÃÿÅÿÅÿÇ‚ÿÉ„ÿɈÿʉÿÍ‹ÿÏÿÎÿÎÿÐŒÿÓÿÑÿÑÿЕÿÍ’ÿ¸‚ÿ§qÿ’aætCÜQ)Ç<Ä!¶¸µÌ -Ôåíøÿ( ÿ.ÿ1ÿ2ÿ7ÿ=ÿ?ÿ ÿM/ÿL1ÿA&ÿ;!ÿ<"ÿ<ÿ=ÿ6ÿ&àѦ¡•–©¼ú$ÿ:ÿ7!ÿ -ôÿ% ÿ<%ÿC,ÿ>%ÿ<#ÿ?"ÿ@#ÿ;ÿ' ÛƘ”ƒƒŠ‡ˆ¥¶ÿ'ÿ;ÿ7ÿ"à ÓÅ Ê×Üæ忬E;{$þ-ÿ3ÿ1ÿ6ÿ> ÿJ,ÿM1ÿH,ÿF(ÿE'ÿF%ÿF%ÿH!ÿI"ÿzÿª+ÿÙÿÜÿÝÿÝÿÜ%ÿÜ%ÿÜÿÜÿÛÿÜ ÿÜÿØÿÔ,ü³ ŽJu1e;!g=#o@rCwH#„U0œn@µ‡YЛhÞ©vä«pé°uñ¸}ô»€ô»€õ¼õ¼õ¼õ¼õ¼õ¼ƒõ¼ƒó½ƒó½ƒó½…ó½…ó½…ó½…õ¼ƒõ¼ƒõ¼õ¼õ¼õ¼õ¼ô»€õ¼õ¼õ¼õ¼õ¼õ¼õ¼õ¼÷¼ù¾ƒõ¼õ¼õÀõÀóÀóÀóÀñ¾}ó¾ó¾õ½÷¿÷¾ƒõ¼ô»€ô»€óºóºóºóºó»}ô¼~ô¼~ô¼~ö¼|÷½}÷½{ù¿}ù¿zù¿zõ½{õ½{ó¾xñ¼vð»xïºwñ¹yð¸xð¶xð¶xî¶xñ¹{óºõ¼ƒø¿ˆû‹üÅ”øÁ븉ޫ|Ìšr²€X’eD{N-[1K!>,"$#"* )/ BW3jA|S)’j;§PºŽ^Å™iÓ¥pשtß®zà¯{Þ¬qÚ¨mÙ¥i×£gÙ¤eÙ¤eÛ¢gÛ¢g×£iÖ¢hÖ¢fÖ¢fÔ fÔ fÖ fÖ fØ¢jÖ hÔ hÔ hÓŸgÒžfÒždÒždÕŸeÕŸeÔždÓcÓaÒœ`ÒœbÒœbÕœcÔ›bÔ›bÕœcÔ›bÔ›bÓšaÓšaÑ›aÑ›aÍ™_̘^̘`Í™aÏ›cÏ›c̘`Å‘YÃYÃYÂZÂZ¾ŠT»‡Q´{F³zE´{Fµ|GµI¹ƒM¨sEše7•`2¡l>¶SÁŒ^Ã^Â]¾‹\³€QÖ hÖ hÖ hÖ hÖ hÖ hÖ jÖ jØ¢jØ¢jØ¢hÙ£iÙ£iÙ£iÙ£kÙ£kÙ£iÚ¤jÚ¤jÚ¤jÚ¤hÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjܦjܦjܦjݧkݧkݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªnàªnàªnÞªnß«oá«oá«oâ¬pâ¬pã­qã­qã­qã­qã­qã­qã®oã®oã®oä¯pä¯pä¯pä¯pä¯på°qå°qå¯så¯så¯så¯sç²sç²sç²sç²sç²sè³tè³tç²sè³tè³tè³têµvêµvêµvé³wê´xëµyëµyëµyì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¹zî¹zî¹zî¹zñ¹{ñ¹{òº|òº|òº|ó»}ó»}ó»}ó»}ó»}ó»}ó»}ó»}ô¼~ô¼~ó»}ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~õ½ô¼~ô¼|ô¼|ô¼|ô¼|õ½}õ½}õ½}õ½}õ½}õ½}÷½{÷½{÷½{÷½{÷½{÷½{ù¿}ù¿}ù¿ù¿úÀ€úÀ€úÀ€ûÁûÁûÁü‚ü‚þÄ„ÿƆÿƆÿLJÿLJÿƆþÂ~ÿÅÿÆ‚ÿÈ„ÿІÿÔŠÿÔˆÿÔˆÿÔ„ÿÓƒÿЄÿÍÿÊ‚ÿÉÿÆÿÄÿÄÿÂÿ‚ÿ‚ÿÁÿÁÿÀ„ÿ†ÿÀ‚ÿÀ‚ÿćÿćÿÇÿʼnÿÅ‹ÿÆŒÿÉ‹ÿÌŽÿÒÿÔ’ÿÔ’ÿÔ’ÿÑ“ÿŇý·|ãbÎ{F³`+¥C‘/Œ¡¡¸ÅÞ# -å*û-ÿ6ÿ7ÿ9!ÿ7!ÿ6 ÿ7ÿ3ÿ2ÿ0ÿ- -ÿ/ ÿ1ÿ3ÿ4ÿ4ÿ4ÿ+ÿ)ÿ.ÿ5ÿ=#ÿ@#ÿ@#ÿ;"ÿ7ÿ4ÿ8#ÿ:ÿ3ÿ& ø×Ѽ »Æ Øÿ) ÿ:ÿ2úè ÿ ÿ9!ÿD,ÿ?'ÿ<$ÿ?&ÿA(ÿB%ÿ6ÿ -毨……Š‹Š‹¡±ÿ%ÿ;ÿ7ÿ% Ë´}‚ yvE8 Tn"Í*Ø5 ÿ2ÿ6ÿ:ÿD)ÿG.ÿF-ÿG)ÿG)ÿF&ÿG'ÿG'ÿF&ÿjÿ˜,ÿÔÿÚ ÿÜ#ÿÛ"ÿÛ)ÿÛ)ÿÛ%ÿÛ%ÿÚ'ÿÙ&ÿ×&ÿÔ#ÿËDä˜z> w; qDoB‚MŽY+¡l>¸ƒUÌ—dÖ¡nؤjÚ¦lã®oé´uð·|óºò¼‚ó½ƒó½ƒõ¿…õ¿ƒõ¿ƒ÷¾…õ¼ƒó½ƒó½ƒó½…ó½…õ¼…õ¼…õ¼ƒõ¼ƒ÷¼÷¼÷¼ƒ÷¼ƒ÷¼ƒù¾…÷¾…÷¾…õ¼÷¾ƒõ¼õ¼÷¾ƒ÷¾ƒù¾ƒù¾ƒ÷¾ƒ÷¾ƒõÀõÀõÀõÀõÀõÀõ¿ƒõ¿ƒ÷¾ƒ÷¾ƒ÷¾…õ¼ƒõ¼ƒõ¼ƒô»‚ô»‚ô»€ô»€ô»€ô»€ö¼~ö¼~ö¼~ö¼~÷»|÷»|ö¼zö¼zô¼|ô¼|ó¾}ó¾}ó¾ó¾õ½ô¼~òº|ñ¹{ð·|ñ¸}ðµ|ðµ|ï´yðµzñ¸}ô»€õ¿ƒø†öÀ„÷Á…÷Á‡ï¹á«sË•]±|I”_,vCa.R!A/)())(!"$#  - +5I-[?&lM2{\AŠhJœz\±‹bÃtϦwÕ¬}Û­xÚ¬wݧqÛ¥o×¥lÖ¤k×£m×£mÕ£lÕ£lÕ£jÕ£jÖ¤kÕ£jÕ£lÕ£lÕ£lÕ£lÕ£jÕ£jÔ hÔ hÒ eÒ eÓŸeÒždÔždÔždÔžbÓaÕœcÖdÕœcÕœcÔ›bÔ›bÑ›aÑ›aÌš_Ìš_Ìš_Ìš_Í™aÍ™a̘`Ï›cÏ›eÐœfÒœfÒœfÒœfΘbÈZÈZÅŒWÀ‡R¼…R¼…RµK±}G°|Bº†LÄ’WË™^Ä’YÅ“ZÃ’^À[Ö hÖ hÖ hÖ hÖ hÖ hÖ jÖ jØ¢jØ¢jØ¢hÙ£iÙ£iÙ£iÙ£kÙ£kÚ¤jÚ¤jÚ¤jÚ¤jÚ¤hÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjݧkݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªná«oá«oàªnß«oß«oá«oâ¬pâ¬pâ¬pã­qã­qã­qã­qã­qã­qã®oã®oä¯pä¯pä¯pä¯pä¯på°qå°qå°qå¯så¯så¯sç±uç²sç²sç²sç²sç²sè³tè³tè³tç²sé´ué´ué´uêµvêµvé³wê´xê´xëµyëµyëµyì¶zì¶zì¶zî¸|î¸|î¸|î¹zî¹zî¹zî¹zñ¹{ñ¹{ñ¹{ð¸zð¸zñ¹{ñ¹{òº|òº|òº|òº|òº|òº|òº|òº|òº|ó»}ó»}ó»}ô¼~ô¼~ô¼~ó»}ó»}ô¼|ô¼|ô¼|ô¼|õ½}õ½}õ½}õ½}õ½}õ½}÷½{÷½{÷½{÷½{ù¿}ù¿}ù¿}ù¿}ù¿úÀ€úÀ€úÀ€úÀ€ûÁûÁü€ü‚ü‚ýÃÿƆþÄ„ÿƆÿLJÿƆÿÅÿÆ‚ÿÈ„ÿʆÿÏ…ÿÓ‰ÿÔˆÿÓ‡ÿÓƒÿÏÿÍÿÊ~ÿÈ€ÿÈ€ÿÅ€ÿÄÿÄÿÄÿÄ„ÿÄ„ÿ‚ÿ‚ÿÀ„ÿ¿ƒÿ¿ÿ¿ÿÆÿÆÿ†ÿÇÿÄŠÿÄŠÿÃ…ÿÃ…ÿÁÿÃÿʈÿÏÿÎÿÍÿÒ—ÿÖ›ÿÑœÿÉ”ÿ·‰ÿª|ÿ’jízRî_<ÔE"Ð*¿Æ Æ Ù ãöÿ" -ÿ'ÿ+ÿ0ÿ2ÿ3ÿ3ÿ0 ÿ0 ÿ2ÿ1ÿ/ÿ1ÿ5ÿ1ÿ/ÿ.ÿ1ÿ5ÿ<ÿD'ÿ>%ÿ6ÿ2ÿ3ÿ6ÿ4ÿ2ÿ& ë áÍËÔâ'ÿ-ÿ;ÿ.óãôÿ4ÿD,ÿB*ÿ>&ÿA(ÿE,ÿF)ÿ>!ÿ)óµ ©„„Š‹‰‹£´ÿ& ÿ;ÿ8ÿ& -Ë´ywopba75 - M]»È%ó% þ0ÿ5ÿ>#ÿF-ÿF-ÿG)ÿG)ÿH(ÿG'ÿH(ÿG'ÿfÿ–*ÿÒÿÙÿÜ#ÿÛ"ÿÛ)ÿÛ)ÿÛ%ÿÛ%ÿÙ&ÿÖ#ÿÔ#ÿÒ!ÿ¼5»ov:E}P+Š]8¦qC¼‡YÑœnÞ©{Ú¥rÛ¦sß«qã¯uç²sêµvð·|óº÷Á‡öÀ†õ¿…õ¿…õ¿ƒõ¿ƒ÷¾…õ¼ƒõ¿…ó½ƒó½…ó½…õ¼…õ¼…õ¼ƒõ¼ƒ÷¼÷¼÷¼ƒ÷¼ƒ÷¼ƒù¾…÷¾…÷¾…÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒù¾ƒù¾ƒ÷¾ƒ÷¾ƒõÀõÀõÀõÀõÀõÀõ¿ƒöÀ„÷¾ƒø¿„÷¾…÷¾…õ¼ƒõ¼ƒõ¼ƒõ¼ƒõ¼õ¼õ¼ô»€ö¼~÷½ö¼~ö¼~öº{÷»|ö¼zö¼zô¼|ô¼|ñ¼{ñ¼{ñ¼}ñ¼}ó»}ó»}òº|òº|ñ¸}ð·|ò·~ò·~ðµzðµzîµzîµzëµyì¶zî¸|ðº~ñ»ò¼‚õ¿‡øŠ÷Âñ¼‰æ³„Ø¥vÅ”i«zO‹^9qD^1K6+'%!!"$. @I#d>„[,tEºŒWÌžiÛ¥oá«uß­tß­tܨrÛ§qئoئoئmئmئmئm×¥n×¥nÖ¤mÖ¤mÖ¤k×¥l×£kؤlÖ¤iÖ¤iÖ¢hÖ¢hÕŸeÕŸeÕŸcÕŸcמeÖdÖdÕœcÕœcÔ›bÒœbÒœbÎœaÎœaÎœaÎœa̘`̘`̘`̘`Í™cÍ™cÏ™cΘbΘbΘbÏ–aΕ`Ì“^É[Ê“`ÄZ¿‹UÂŽXÃUÅ‘WÄ’WÄ’WÄ’YÂW¿ŽZÀ[Ö fÖ fÖ fÖ fÖ fÖ fÖ hÖ hØ¢hØ¢hØ¢fÙ£gÙ£gÙ£gÙ£gÙ£gÚ¤hÚ¤hÚ¤jÚ¤jÚ¤hÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjÛ§kÛ§kݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnÞªnÞªnß«oß«oß«oß«oà¬pà¬pâ¬pâ¬pâ¬pâ¬pâ¬pã­qã­qã­qã­qã­qä®rä®rä®rä®rä®rå¯så°qå°qå°qå°qå°qç²sç²sç²sç²sç²sç²sç²sç²sè³tè³tè³té´ué´ué´ué´ué´uêµvê´xê´xê´xê´xëµyëµyëµyì¶zì·xì·xî¹zî¹zî¹zî¹zî¹zî¹zî¹zïº{ïº{î¹zî¹zî¹zî¹zð»|ð»|ïº{ð»|ð»|ð»|ð»|ð»zð»zð»|ð»|ó»}ó»}ó»}ó»}ó»}ó»}ó»{ô¼|ö¼|õ»{ö¼|ö¼|ö¼|ö¼|ö¼|ö¼|õ½}õ½}õ½}õ½}÷¿÷¿÷¿øÀ€øÀ‚øÀ‚øÀ€ùÁùÁùÁùÁú‚ú‚ûÃþÄ‚þÄ‚ÿÆ„ÿÆ„ÿÅ„ÿÅ„ÿÆ‚ÿÈ„ÿË‚ÿЇÿÓ‡ÿÓ‡ÿЃÿЃÿуÿÏÿÌ€ÿÉ}ÿÇÿÆ~ÿÄÿÄÿ‚ÿ‚ÿÁ‚ÿÁ‚ÿÁ‚ÿÁ‚ÿ¿ÿ¿ÿ¾€ÿ¿ÿÁ‚ÿÄÿ„ÿÃ…ÿ‡ÿ‡ÿÁ„ÿÁ„ÿ¿„ÿ¿„ÿÀ†ÿÀ†ÿÁ…ÿÁ…ÿÂ…ÿÁ„ÿÈÿÅŠÿË•ÿÑ›ÿÒ›ÿÒ›ÿÏ–ÿÊÿ¯vð˜_àJÆe0¶H 2¤$ž­±ÈÏ%ã)ë1ù1ü4ÿ4ÿ4ÿ3ÿ3ÿ2ÿ2ÿ1ÿ2ÿ2ÿ6ÿ;!ÿ?%ÿ>'ÿ:#ÿ8ÿ6ÿ3ÿ3ÿ0ÿ.ÿ,ÿ$ ñíö ÿ+ÿ7ÿ;!ÿ,ñãñÿ0ÿE+ÿM3ÿH.ÿE/ÿH2ÿE'ÿ?!ÿ6 ÿ -Ú É§¦¯Áü+ ÿ<ÿ9ÿ)͹Œˆ~|RO-0 $$% " - Q\¼Ñ1ÿ2ÿ;ÿE*ÿE*ÿG&ÿH'ÿI-ÿI-ÿJ"ÿJ"ÿlÿ™1ÿÐÿÙ$ÿÛ(ÿÙ&ÿÚ(ÿÚ(ÿÙ,ÿ×*ÿÓ)ÿÑ'ÿÎ7ÿÈ1è HŸW‹Rže,©xFÃ’`Ò gÛ©pÝ«rÝ«rÛ©pÛ©pÞªrà¬tå¯sé³wî¸~ï¹ò¼‚ó½ƒõ¿‡õ¿‡õ¿…õ¿…õ¿…õ¿…õ¼õ¼õ¼ƒ÷¾…õ¼ƒõ¼ƒõ¼õ¼÷¿÷¿÷¿÷¿÷¿÷¿÷¿÷¿÷¾…÷¾…÷¾…÷¾…÷¾ƒ÷¾ƒ÷¾…÷¾…õ¿ƒõ¿ƒ÷¾ƒ÷¾ƒõ¿ƒõ¿ƒõ¿ƒõ¿ƒ÷¾ƒ÷¾ƒöÀ„öÀ„ø¿†÷¾…÷¾…÷¾…÷¾ƒõ¼õ¼÷¾ƒ÷¾ƒõ¼õ½õ½ô¼~ô¼~ö¼~ö¼~ô¼|ô¼|ô¼|ô¼|ñ¼{ñ¼{ó»}ó»}ñ¼}ñ¼}òº|òº|ñ¹{ð¸zò·|ò·|ðµzðµzîµzîµzîµzîµzì¶|ëµ{é³yç±wè²zé³{ê´|ñ»ƒöÀˆöÀˆøˆøˆò¼€é³wÛ¦eÆ‘P«y@“a(tCW&B6.+'&$# - - - - - - - ( B+ZC/wU;nT¡|\µpÃxÅŸzÉ vÔ«Û­}Ú¬|߯xÙ©rÙ¥kؤjÔžbØ¢fئkئkئk×¥jÖ¤iÖ¤iÕ£fÕ£fÕŸcÖ dÖ jÖ jמiמiמiÖhÓgÓgÐœ`Ðœ`ÐœbÏ›aÐœbÏ›aÏ™aÏ™aÑ›cÏ™aÑ›eÑ›eÏ™cÍ—aÊ”^Ç‘[Ç‘YÊ”\É•_É•_É•_È”^Ä’[Å“\Å“ZÅ“ZÅ“ZÄ’YÁ‘ZÂ’[Â’YÀWÖ fÖ fÖ fÖ fÖ fÖ fÖ hØ¢jØ¢hØ¢hØ¢fÙ£gÙ£gÙ£gÙ£gÙ£gÚ¤hÚ¤hÚ¤jÛ¥kÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjܦjݧkÛ§kÛ§kݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnÞªnÞªnß«oß«oß«oß«oà¬pà¬pâ¬pâ¬pâ¬pâ¬pã­qã­qã­qã­qã­qã­qä®rä®rä®rä®rä®rå¯så°qå°qå°qå°qå°qç²sç²sç²sè³tè³tè³tè³tè³tè³tè³tè³té´ué´ué´ué´uêµvêµvê´xê´xê´xê´xëµyëµyëµyì¶zì·xì·xì·xî¹zî¹zî¹zî¹zî¹zî¹zî¹zî¹zî¹zî¹zî¹zî¹zð»|ð»|ïº{ð»|ð»|ð»|ð»|ð»zð»zð»|ð»|ó»}ó»}ó»}ô¼~ô¼~ô¼~ô¼|ô¼|ö¼|ö¼|ö¼|ö¼|ö¼|ö¼|ö¼|÷½}ô¼|ô¼|õ½}õ½}÷¿÷¿÷¿øÀ€øÀ‚øÀ‚øÀ€ùÁùÁùÁùÁú‚ú‚ûÃýÃýÃýÃýÃþÂÿÅ„ÿÉ…ÿ̈ÿÎ…ÿЇÿЄÿЄÿÏ‚ÿЃÿуÿÏÿÊ~ÿÈ|ÿÆ~ÿÅ}ÿÅ‚ÿÄÿ‚ÿ‚ÿÀÿÀÿÀÿÀÿÀ‚ÿÀ‚ÿÀ‚ÿÀ‚ÿÁ‚ÿÁ‚ÿ„ÿ„ÿÁ†ÿ¿„ÿ¿‚ÿ¿‚ÿ¾ƒÿ¿„ÿ¾„ÿ¾„ÿÀ„ÿÀ„ÿÁ„ÿÁ„ÿÁ†ÿÁ†ÿ¾ˆÿÁ‹ÿÃŒÿÅŽÿÉÿÊ‘ÿÉÿΕÿÌ—ÿÆ‘ÿ·ˆÿ§xÿ‘føxMïZ1ÙD×-Ê ÍÉÔ Ûçï!ÿ) ÿ-ÿ0ÿ1ÿ2ÿ3ÿ4ÿ7ÿ5ÿ;!ÿ<%ÿ<%ÿ<#ÿ:!ÿ4ÿ3ÿ0ÿ-ÿ*ÿ'ø$ -ö"ý' ÿ1ÿ8ÿ=#ÿ0öê -îÿ)ÿD*ÿW=ÿP6ÿF0ÿG1ÿA#ÿ?!ÿ='ÿ0ë -Ó­©ŽŒ² Æþ-ÿ=ÿ8ÿ)μ–”‹†VT1 -5))'$ <@§Ã# -ÿ-ÿ9ÿA&ÿD)ÿH'ÿL+ÿL0ÿN2ÿN&ÿN&ÿpÿ˜0ÿÏÿÙ$ÿÙ&ÿØ%ÿØ&ÿ×%ÿÖ)ÿÕ(ÿÏ%ÿÑ'ÿÆ/ÿ±Æ~&­e µ|CÈVÏžlݬzÞ¬sÞ¬sÝ«rß­tß­tá¯vâ®vã¯wå¯sðº~ùÉùÉøˆ÷Á‡öÀˆõ¿‡õ¿…õ¿…õ¿…öÀ†ø¿„÷¾ƒø¿†ø¿†÷¾…ø¿†ø¿„ø¿„øÀ‚÷¿øÀ‚øÀ‚øÀ‚øÀ‚øÀ‚øÀ‚ø¿†ø¿†÷¾…÷¾…÷¾ƒø¿„ø¿†÷¾…õ¿ƒõ¿ƒ÷¾ƒ÷¾ƒõ¿ƒõ¿ƒõ¿ƒõ¿ƒ÷¾ƒ÷¾ƒöÀ„öÀ„ø¿†÷¾…÷¾…÷¾…÷¾ƒõ¼õ¼÷¾ƒ÷¾ƒõ¼õ½õ½ô¼~ô¼~ö¼~ö¼~ô¼|ô¼|ô¼|ô¼|ñ¼{ñ¼{ó»}ó»}ñ¼}ñ¼}ó»}òº|ñ¹{ñ¹{ó¸}ò·|ò·|ò·|îµzîµzí´yîµzì¶|î¸~î¸~ëµ{é³{é³{ëµ}ì¶~ê´|è²zê´zì¶|ï¹}õ¿ƒõÀ÷ÂóÁˆî¼ƒâ±Пm·‰]oC†Y6j=T*D0( - - - - - - - &2BS.hBuO*†]3—nD®€PÄ–fРiÙ©râ®tæ²xâ¬pàªnÛ©nÛ©nÚ¨mÙ§l×¥jÖ¤iÖ¤gÖ¤gÙ£gÙ£gØ¢lÖ jØŸjמiמiמiÔžhÔžhÑaÑaÑcÒždÐœbÏ›aÑ›cÓeÒœdÑ›cÒœfÑ›eÏ™cΘbΘbÌ–`Ë•]È’ZÆ’\Å‘[Æ’\È”^Ç•^Ç•^Ç•\Ç•\Å“ZÅ“ZÓ\Â’[Â’YÁ‘XØ¢jØ¢jØ¢jØ¢jØ¢jØ¢jØ¢hÙ£iÙ£iÙ£iÙ£iÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÚ¤hÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjܦjݧkÛ§kÛ§kݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnß«oß«oß«oà¬pà¬pà¬pà¬pà¬pâ¬pâ¬pã­qã­qã­qä®rä®rä®rä®rã­qä®rä®rä®rä®rä®rå¯så°qå°qå°qå°qå°qç²sç²sç²sè³tè³tè³tè³tè³tè³té´ué´ué´ué´ué´ué´uêµvêµvê´xê´xêµvêµvë¶wë¶wë¶wì·xì·xì·xì·xî¹zî¹zî¹zî¹xî¹xî¹zî¹zî¹zî¹zî¹zî¹zïº{ïº{ïº{ïº{ð»|ð»|ð»|ð»|ð»zð»zð»zð»zó»}ó»}óºô»€ô»€ô»€ô»€ô»€ö¼~ö¼~ö¼~ö¼~ö¼~ö¼~ö¼~÷½ô¼~ô¼~õ½õ½÷¿÷¿÷¿øÀ‚øÀ‚øÀ‚ùÁƒùÁƒùÁùÁùÁú‚ü„ýÃ…ýÃþÄ‚þÄþÄÿÃÿǃÿÊ„ÿ͇ÿ̓ÿÏ…ÿÎÿЃÿЃÿÏ‚ÿÏ„ÿ΃ÿÉ‚ÿÆÿÄ€ÿÄ€ÿÄ‚ÿÁÿÀÿÀÿÀÿÀÿÀÿÁ‚ÿÀ‚ÿÀ‚ÿ„ÿ„ÿ„ÿÀ‚ÿÀ‚ÿÀ‚ÿÀ„ÿÀ„ÿÀ„ÿ†ÿ†ÿ†ÿÁ†ÿÁ†ÿÂ…ÿÂ…ÿÂ…ÿÁ„ÿ‡ÿ‡ÿÇÿÇÿ†ÿ†ÿ¿‚ÿÁ„ÿÆÿňÿÊÿÌ‘ÿÑ•ÿÔ˜ÿÕ›ÿÔšÿË“ÿ¾†ÿ¬vð•_ÜxDÂ^*´F¥7¨, $¯µÎ Ó%é( ð/þ1ÿ7ÿ2ÿ4ÿ;!ÿ@&ÿ=$ÿ<#ÿ8 ÿ4ÿ-ÿ)ÿ(ÿ& ÿ' ÿ)ÿ* -ÿ0ÿ7ÿ>ÿ:ÿ#ÿÿÿ&ÿ? ÿR4ÿO1ÿB*ÿ=%ÿ< ÿ=!ÿÿ?'ÿ<$ÿ7ÿ9ÿ;ÿ;ÿ5ÿ-ÿ# ûÚÔÍ Ïêû&ÿ1ÿ6ÿ-ýãëÿ* ÿ4ÿ/ÿ+ ÿ( ÿ!é× x k" -,,OT°ÿ( ÿ5ÿ<ÿE%ÿT'ÿX+ÿ\,ÿ^.ÿd#ÿi(ÿŒÿ®;ÿÏ2ÿÑ4ÿÑ;ÿÐ:ÿÐEÿÏDÿÊIÿÆEÿÃWÿ»O÷°bø±c÷¶xò±sè²zè²zçµzë¹~ｄô‹ÿΗÿÕ¡ÿÞªÿߦÿߦÿÙ¡ÿÔœÿÍ“ÿÊüÆŒùÉùÉùÉøˆøˆøˆøˆúÁ†úÁ†úÁˆúÁˆû‰û‰úÁ†û‡üĆüĆüĆüĆûÃ…üĆüĆüĆúÁ†úÁ†úÁˆúÁˆúÁˆùÀ‡úÁŠúÁŠùÀ‡úÁˆùÀ…ùÀ…ùÀ…ùÀ…ùÀ…ùÀ…ùÀ…ùÀ…ø¿„ø¿„ø¿†ø¿†ø¿†ø¿†ø¿„ø¿„÷¾ƒ÷¾ƒ÷¾…÷¾…÷¾…÷¾…÷¾ƒ÷¾ƒ÷¿÷¿÷¿÷¿÷¾…õ¼ƒõ¼ƒõ¼ƒõ¼ƒõ¼ƒô»€ô»€ô»€óºó»}ó»}óºò¹~ò¹~ñ¸}ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}î¸|î¸|ì¶zëµyëµyê´xê´xé³wê´xê´xì´vì´víµwíµwì´vë³uë³uê²té°uç®sê²tê²tì´vì´víµwòº|ðº€ï¹èµtØ¥dµ…L]$e9 G0 -&" 2 ;#R-a¡wS±‡cÄ•lÏ wÔ§oÕ¨pÖ¦kÕ¥jÓ¡dÕ£fÓ¢bÓ¢bÓŸgÓŸgÔžfÓeÔžfÓeÓeÓeÓcÓcÓcÓcÔ›bÓšaÓšaј_Θ`Θ`ÒœbÑ›aÍ—]Í—]Ì–\Ì–\Í—_Ì–^Ì–^Ì–^Ì–^Ë•]Ë•]Ë•]Ù£kÙ£kÙ£kÙ£kÙ£kÙ£kÙ£iÚ¤jÚ¤jÚ¤jÚ¤jÛ¥kÛ¥kÚ¤jÛ¥iÛ¥iÙ¥iÙ¥iÙ¥iÚ¦jÚ¦jÚ¦jÚ¦jÚ¦jÛ§kÛ§kÛ§kÛ§kÛ§kÛ§kÛ§kÛ§kܨlܨlÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªnàªnàªnàªnàªná«oá«oß«oß«oà¬pà¬pà¬pà¬pà¬pá­qã­qã­qã­qä®rä®rä®rä®rä®rä®rä®rä®rä®rä®rä®rå¯så¯så°qå°qç²sç²sç²sç²sç²sè³tè³té´ué´ué´ué´ué´ué´ué´ué´ué´uè³têµvêµvêµvêµvêµvêµvêµvêµvêµvêµvë¶wë¶wë¶wëµyì¶zì·xì·xì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¸|ï¹}ï¹}ï¹}ðº~ðº~ðº~ðº~ðº~ð»|ð»|ðº~ðº~óºóºóºô»€ô»€ô»€ô»€ô»€ö¼~ö¼~ö¼~ö¼~÷½÷½÷½÷½õ½õ½õ½÷¿÷¿÷¿øÀ‚øÀ‚øÀ‚ùÁƒùÁƒùÁƒùÁƒùÁƒü„ü„ûÁƒûÁƒü€þÄ‚ÿÇ‚ÿȃÿɃÿÊ„ÿÊ‚ÿÌ„ÿÍ„ÿÍ„ÿÌÿÍ€ÿÎÿÎÿÍ‚ÿË€ÿÈÿÆÿÄ€ÿÄ€ÿÄ‚ÿÃÿÄÿÄÿÆÿÆÿÆÿÆÿÁ„ÿÁ„ÿÁ‚ÿÁ‚ÿÁ‚ÿÁ‚ÿÁ‚ÿÁ‚ÿÁ„ÿÁ„ÿÆÿÆÿÆÿćÿÇÿÇÿÃ…ÿÃ…ÿÃ…ÿĆÿĈÿÇÿÇÿÇÿ†ÿĈÿÆÿÆÿÆÿÆÿˆÿÉÿÉÿÉÿćÿćÿĈÿĈÿćÿňÿňÿňÿÉÿËÿÍ‘ÿÎ’ÿΔÿÏ•ÿÍ•ÿÌ”ÿÇÿÄÿµÿ¢nÿ„VöoAïV,ÝDê9â1 é&æ#ï!ò$ý$ÿ) ÿ( ÿ) ÿ- ÿ2ÿ/ÿ-ÿ1ÿ1ÿ3ÿ2ÿ2ÿ5ÿ7ÿ=ÿ=%ÿ<$ÿ9ÿ9ÿ6ÿ5ÿ4ÿ.ÿ*ÿ$ é# -åÝÝø# ÿ1ÿ6ÿ6ÿ*ùá ìÿ-ÿ7ÿ5ÿ6ÿ6ÿ:ÿ:"ë |k+1Y_™µ# -ÿ* ÿ8ÿ<ÿH(ÿV)ÿZ-ÿ`0ÿb2ÿg&ÿr1ÿ•"ÿ²?ÿÐ3ÿÑ4ÿÏ9ÿÎ8ÿÎCÿÏDÿÌKÿÌKÿÍaÿÉ]ÿÃuÿÁsûº|ú¹{ðº‚î¼óÁ†ûÉÿÒ™ÿØ¡ÿݦÿÞªÿÞªÿݤÿØŸÿÑ™ÿÍ•ÿÊþÈŽýÇüÆŒüÆŒùÉùÉùÉøˆøˆúÁ†û‡û‰úÁˆû‰û‰üÈüÈüĆüĆüĆüĆûÃ…üĆüĆüĆû‡úÁ†úÁˆúÁˆúÁˆúÁˆúÁŠû‹úÁˆúÁˆúÁ†úÁ†ùÀ…úÁ†úÁ†ùÀ…ùÀ…ùÀ…ø¿„ø¿„ø¿†ø¿†ø¿†ø¿†ø¿„ø¿„÷¾ƒ÷¾ƒ÷¾…÷¾…÷¾…÷¾…÷¾ƒ÷¾ƒ÷¿÷¿÷¿÷¿÷¾…õ¼ƒõ¼ƒõ¼ƒõ¼ƒõ¼ƒô»€ô»€ô»€ô»€ô¼~ô¼~óºóºóºò¹~ðº~ðº~ï¹}ï¹}î¸|ï¹}ï¹}î¸|ì¶zì¶zëµyëµyëµyê´xëµyëµyì´vì´vì´vì´vë³uë³uë³uê²tê±vê±vê²tê²té±så­oä¬nä¬ná«qã­sæ³ré¶ué¹€â²yºŽb“g;iC K%3%  - - - - - !*4 CQ'rC’c: s;·ŠRÇ—\ÏŸd×¥hئiÖ¥eÕ¤dÔ hÓŸgÕŸgÖ hÖ hÕŸgÕŸgÔžfÔždÔždÔždÔždÕœcÓšaÓšaј_Ï™aÏ™aÑ›aÏ™_Í—]Í—]Í—]Í—]Í—_Ì–^Ì–^Ì–^Ì–^Ì–^Ì–^Ì–^Ù£iÙ£iÙ£iÙ£iÙ£iÙ£iÚ¤jÚ¤jÚ¤jÚ¤jÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÙ¥iÙ¥iÚ¦jÚ¦jÚ¦jÚ¦jÚ¦jÛ§kÛ§kÛ§kÛ§kÛ§kÛ§kÛ§kÛ§kܨlܨlܨlÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªná«oá«oá«oá«oá«oá«oà¬pà¬pà¬pá­qá­qá­qá­qá­qã­qã­qä®rä®rä®rä®rä®rä®rä®rä®rä®rä®rä®rä®rå¯så¯så°qå°qç²sç²sç²sç²sè³tè³té³wé³wé³wé³wé³wé³wé³wé³wê´xê´xè²vê´xê´xê´xê´xëµyêµvêµvë¶wë¶wë¶wë¶wë¶wë¶wëµyì¶zì·xì·xì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¸|ï¹}ï¹}ï¹}ï¹}ðº~ðº~ðº~ðº~ð»|ð»|ðº~ðº~ñ»ñ»ñ»ñ»ò¼€ò¼€ò¼€ò¼€ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô»€õ¼ó¾ó¾õ½÷¿÷¿÷¿øÀ‚øÀ‚øÀ‚ùÁƒùÁƒùÁƒú„ú„ü‚ýÃýÃýÃþÂ~ÿÃÿÈÿÊÿË‚ÿË‚ÿÌ„ÿÌ„ÿË‚ÿË‚ÿË€ÿÌÿÍ‚ÿÍ‚ÿÍ„ÿË‚ÿÈÿÆÿŃÿŃÿņÿÄ…ÿćÿćÿÆÿÆÿÆÿÆÿÆÿÆÿÁ„ÿÁ„ÿÁ„ÿÁ„ÿÁƒÿ„ÿ†ÿ†ÿĈÿĈÿÈÿĉÿĉÿĉÿćÿćÿćÿćÿÇÿÇÿÇÿÇÿÁ†ÿÈÿÆÿÆÿ†ÿ†ÿÆÿÆÿÈÿÈÿÈÿÈÿÊÿÊÿŠÿŠÿˆÿˆÿˆÿˆÿ‚ÿ‚ÿÇÿʼnÿÇ‹ÿÊŽÿÊŽÿÌÿÑ•ÿÓ—ÿÓ—ÿÎ’ÿÅ‹ÿ·}ÿ§oð•]ä€JÏk5ÄT!µE¸9 ±2¾'Á*Ó&Û. ñ.ó0þ1ÿ3ÿ1ÿ5ÿ5ÿ7ÿ5ÿ8ÿ9!ÿ<$ÿ= ÿ<ÿ8ÿ4ÿ3ÿ/ÿ-ÿ(ÿ) þ$ú$ü& -ÿ+ ÿ6ÿ:ÿ7ÿ&ÿì öÿ+ ÿ4ÿ6ÿ4ÿ:ÿCÿA#ÿ( -³—34bq ½Ä àõ$ÿ. -ÿ7ÿ@ÿL$ÿ\%ÿc,ÿj*ÿm-ÿv#ÿƒ0ÿ¦&ÿÁAÿÑCÿÒDÿÐRÿÐRÿÎ`ÿÑcÿÐiÿÓlÿÐwÿÍtÿÆ€ÿÁ{ý¿‚ÿÂ…õƒøņýËŽÿÒ•ÿÜŸÿá¤ÿà¦ÿÜ¢ÿÙ¡ÿןÿÕ›ÿÒ˜ÿЖÿÍ“ÿÉÿÇŽýÇ‹üÆŠùÉøˆùÉùÉùÉúÄŠüÈüÈüÈû‡û‰üÊüÈþÅŠþƈþƈþƈþƈþƈþƈüĆüĆüÈüÈüÊû‰û‰û‰û‹û‹û‰úÁˆúÁ†úÁ†ùÀ…úÁ†úÁ†úÁ†úÁˆùÀ‡ùÀ…ùÀ…ùÀ‡ø¿†ø¿†ø¿†ø¿„ø¿„ø¿„÷¾ƒ÷¾…÷¾…÷¾…÷¾…÷¾…÷¾…÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒõ¼ƒõ¼ƒô»‚õ¼ƒõ¼õ¼ô»€ô»€ô»€ô»€ñ»ò¼€ò¼€ñ»ñ»ðº~ðº~ðº~ï¹}ï¹}î¸|ï¹}ï¹}î¸|ì¶zì¶zì¶zëµyë¶wë¶wë¶wë¶wë¶wë¶wì³xì³xë²wë²wé³wè²vè²vè²vé°wê±xë°wë°wç®sç®sç¯qæ®på¯uç±wæ²xè´zæ´yå³x×¥l»‰P›k2„Tj< S%>0*(# - - - -& 1 -E0aE,w[BŒmN¢ƒd³hÁ›vÒ¥|Ï¢yÖ£qÖ£qÔ jÔ jÒžfÒžfÒždÒždÔždÓcÑ›_Ñ›_Ñœ]Ïš[Ï™_Ï™_Ñ›cÏ™aÍ—_Ì–^Ì–^Ì–^Ë—_Ê–^Ê–`Ë—aË—aË—aÊ–`Ë—aÙ£iÙ£iÙ£iÙ£iÚ¤jÙ£iÚ¤jÛ¥kÛ¥kÛ¥kÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÛ¥iÚ¦jÚ¦jÚ¦jÚ¦jÚ¦jÚ¦jÚ¦jÛ§kÛ§kÛ§kÛ§kÛ§kÛ§kÛ§kÛ§kܨlܨlܨlÞ¨lÞ¨làªnàªnàªnàªnàªnàªná«oá«oá«oá«oá«oâ¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qã­qã­qä®rä®rä®rä®rä®rä®rå¯så¯så¯så¯så¯så¯sç±uç±uç²sç²sç²sç²sç²sè³tè³tè³té³wé³wé³wé³wé³wé³wé³wê´xê´xê´xé³wê´xê´xê´xëµyëµyë¶wë¶wë¶wë¶wë¶wë¶wë¶wë¶wëµyì¶zì·xì·xì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¸|î¸|î¸|ï¹}ï¹}ï¹}ï¹}ðº~ðº~ð»|ð»|ðº~ðº~ñ»ñ»ñ»ò¼€ò¼€ò¼€ò¼€ò¼€ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~õ¼õ¼ó¾ó¾õ½÷¿÷¿÷¿øÀ‚øÀ‚øÀ‚ùÁƒùÁƒùÁƒú„ú„ü‚ýÃýÃÿƆÿÅÿÅÿÈÿÊÿË‚ÿË‚ÿÊ‚ÿÊ‚ÿÉ€ÿË‚ÿË€ÿË€ÿÌÿÌÿ̃ÿÉ€ÿÇ€ÿÆÿŃÿŃÿņÿÄ…ÿćÿćÿćÿÆÿÆÿÆÿÆÿÆÿÆÿÁ„ÿÁ„ÿÁ„ÿ„ÿÁƒÿ†ÿ†ÿ†ÿĈÿÈÿÈÿÈÿĉÿćÿćÿćÿćÿÇÿÇÿÇÿÇÿÁ†ÿÈÿÆÿÆÿ†ÿ†ÿÆÿÆÿÈÿÈÿÈÿÈÿ‰ÿÊÿŠÿŠÿˆÿˆÿˆÿˆÿÄ„ÿÄ„ÿʼnÿʼnÿĈÿĈÿÇÿĈÿÆŠÿÇ‹ÿÆŠÿËÿÏ•ÿЖÿИÿÓ›ÿÊ”ÿ¾ˆÿ¬yÿ›hÿ†XîoAò[3âK#ä7Û. å"æ#ñ$ó& -ø$ -ú& ÿ+ÿ.ÿ/ÿ1ÿ2ÿ8 ÿ9ÿ;ÿ<ÿ8ÿ4ÿ0ÿ/ÿ+ÿ,ÿ( ÿ*ÿ-ÿ0ÿ8ÿ:ÿ9ÿ+ÿõøÿ) ÿ2ÿ3ÿ2 ÿ5 ÿ>ÿ=ÿ* ºš5:v‹#Ú"á)ö%ÿ/ÿ3ÿ7ÿBÿO'ÿb+ÿi2ÿq1ÿu5ÿ‚/ÿ’?ÿ¶6ÿÉIÿÔFÿÔFÿÒTÿÔVÿÔfÿÓeÿÑjÿÐiÿÍtÿÉpÿÆ€ÿÅÿÇŠÿÍÿÑ’ÿטÿÞ¡ÿä§ÿã¦ÿÞ¡ÿØžÿÕ›ÿÔœÿÑ™ÿÏ•ÿΔÿΔÿÍ“ÿÉÿÇŽþÈŒúĈùÉùÉùÉùÉúÄŠúÄŠüÈüÈüÈüÈû‰û‰üÈüÈþƈþƈþƈüĆþƈþƈþƈþƈüÈüÈüÊüÊüÊüÊüÃŒû‹úÁˆû‰û‡û‡úÁ†úÁ†úÁ†úÁ†úÁˆùÀ‡ùÀ…ùÀ…ùÀ‡ø¿†ø¿†ø¿†ø¿„ø¿„ø¿„ø¿„÷¾…÷¾…÷¾…÷¾…÷¾…÷¾…÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒ÷¾…õ¼ƒõ¼ƒõ¼ƒõ¼ô»€ô»€ô»€ô»€ô»€ò¼€ò¼€ñ»ñ»ñ»ï¹}ï¹}ï¹}ï¹}ï¹}î¸|ï¹}ï¹}ì¶zì¶zì¶zì¶zëµyë¶wë¶wë¶wë¶wë¶wë¶wí´yì³xë²wë²wé³wé³wè²vè²vê±xê±xì±xé®ué°ué°ué±sê²tç±wç±wã¯uå±wå³xí»€ï½„ð¾…ç·~â²yÑ£p¹‹XžtJ…[1mD U,E 2 # - - - - - - - $2H) -\6vP+™lC­€WÊ—eÙ¦t×£mÓŸiÓŸgÔ hÔ fÓŸeÕŸeÓcÑ›_Òœ`Ñœ]Ñœ]Ñ›aÏ™_Ñ›cÏ™aΘ`Ë•]Ë•]Ë•]Ë—_Ë—_Ë—aË—aË—aË—aË—aÊ–`Ú¤jÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÛ¥iÛ¥iÛ¥iܦjܦjܦjܦjܦjÚ¦jÚ¦jÚ¦jÛ§kÛ§kÛ§kÛ§kÛ§kÛ§kÛ§kÛ§kܨlܨlܨlܨlܨlÞªnÞªnàªnàªnàªná«oá«oá«oá«oá«oá«oá«oá«oá«oà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qá­qá­qâ®râ®râ®rã¯sã¯sã¯så¯så¯så¯så¯så¯så¯sç±uç±uç²sç²sç²sè³tè³tè³tè³té´ué³wé³wé³wé³wê´xê´xê´xê´xê´xê´xé³wê´xê´xëµyëµyëµyë¶wë¶wë¶wë¶wë¶wë¶wë¶wë¶wëµyì¶zì¶zì¶zì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¸|î¸|ï¹}ï¹}ï¹}ï¹}ï¹}ðº~ðº~ðº~ðº~ðº~ðº~ó»}ó»}ó»}ô¼~ô¼~ó»}ô¼~ô¼~ó»}ô¼~ô¼~ô¼~ô¼~ô¼~õ½õ½õ½÷¿÷¿÷¿øÀ€øÀ€øÀ€ùÁùÁùÁú‚ú‚ú‚ú‚ü€ü€ÿÃÿÆ‚ÿÆ€ÿÆ€ÿÈÿÊÿË‚ÿÊÿÉÿÉÿÊ€ÿÊ€ÿÊ~ÿÌ€ÿÊ€ÿÊ€ÿÊ€ÿÈ~ÿÇ‚ÿÆÿƃÿƃÿÅ…ÿÅ…ÿÅ…ÿÅ…ÿŇÿĆÿÆÿÆÿÆÿÁ„ÿÁ‚ÿÁ‚ÿÀƒÿÀƒÿÀ‚ÿÀ‚ÿÆÿÆÿÆÿňÿňÿňÿÈÿÈÿÈÿĉÿÉÿÉÿćÿÆÿÇÿÇÿÇÿÇÿĉÿĉÿÈÿÈÿÆÿÆÿÆÿÆÿÈÿÈÿÇÿĈÿňÿňÿĈÿĈÿĉÿĉÿÊÿÄ‹ÿćÿćÿÆÿćÿĉÿĉÿˆÿÄŠÿÇÿÈŽÿÆ‹ÿÅŠÿÉŽÿÍ’ÿÑ•ÿДÿÑ•ÿÓ—ÿÒ–ÿÍ‘ÿÅ‹ÿµ{óŸjáXÕuCÃc1¿S$²F¸6 ²0À-Å2 Ø. -Ú0 ï0ö7ÿ6ÿ: ÿ;ÿ>ÿ8ÿ3ÿ0ÿ.ÿ.ÿ.ÿ/ÿ0ÿ2ÿ7ÿ9ÿ9ÿ4ÿ) ÿ#ÿ% ÿ+ -ÿ1ÿ1ÿ2ÿ3ÿ8ÿ=ÿ, -Ò¶~‚º Ó$ÿ,ÿ3ÿ+ ÿ/ÿ6ÿ9ÿJ ÿZ0ÿn4ÿt:ÿ0ÿ„5ÿ™5ÿ®JÿÉOÿÓYÿÕ`ÿÔ_ÿÓlÿÔmÿÖ{ÿÓxÿÑyÿÐxÿÎÿÏ‚ÿÒ‹ÿÔÿؘÿߟÿã¤ÿã¤ÿà¡ÿÝžÿÙšÿטÿÔ˜ÿÔ˜ÿÒ˜ÿЖÿÎ’ÿËÿÊÿÉŽÿÇŽÿÇŽÿÆ‹þÅŠüÈû‡üÈüÈüÈüÈüĆüĆüÈüÈþÈþÈþĆþĆÿƆÿƆÿƆÿƆüÄ„þƆþƆüÄ„üÈüÈüÈüÈüÊüÊû‰û‰û‰û‰û‰û‰úÁŠúÁŠúÁˆúÁˆùÀ…ø¿„ø¿„ø¿„ø¿†÷¾…ø¿†ø¿†ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒõ¼õ¼õ¼õ¼ô»€õ¼õ¼õ¼ô»€ô»€óºô»€ò¼€ò¼€ñ»ñ»ñ»ðº~ðº~ðº~ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}î¸|ì¶zì¶zì·xì·xë¶wë¶wë¶wë¶wè´xè´xéµyéµyê´xé³wé³yè²xç³wç³wè²vç±ué°ué°uè²xç±wç±uç±uå±uå±uå³vâ°så¯uè²xí´{ð·~ð·€ñ¸ïº‡ç²Ù¤sÆ‘`±|K”_.p?U$D>81&" -  ! 4P6#yW= ~d¾•qÊ¡}Ц|Ë¡wФvÊžpÌžkÌžkОgÏfÐœdÐœdÓšaÓšaјaјa̘`̘`Ë—_Ë—_Ë—aË—aË—a̘b̘b̘bÊ–`È”^Ú¤jÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÛ¥iÛ¥iܦjܦjܦjܦjܦjܦjÚ¦jÚ¦jÛ§kÛ§kÛ§kÛ§kÛ§kÛ§kܨlܨlܨlܨlܨlܨlܨlܨlÞªnÞªnàªnàªná«oá«oá«oá«oá«oá«oá«oá«oâ¬pâ¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qâ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯så¯så¯så¯så¯så¯så¯sç±uç±uç²sç²sç²sè³tè³tè³tè³té´ué³wé³wé³wê´xê´xê´xê´xê´xê´xê´xê´xê´xê´xëµyëµyëµyë¶wë¶wë¶wë¶wë¶wë¶wë¶wë¶wëµyì¶zì¶zì¶zì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¸|ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ðº~ðº~ðº~ðº~ðº~ðº~ó»}ó»}ó»}ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~õ½õ½õ½÷¿÷¿÷¿øÀ€øÀ€ùÁùÁú‚ú‚ûÃûÃûÃüÄ„þÄ‚þÄ‚ÿÆ‚ÿÈ„ÿɃÿɃÿÊÿÍ„ÿÍ„ÿÊÿÉÿÉÿÉÿÈ~ÿÈ|ÿÊ~ÿÊ€ÿÊ€ÿÉÿÇ}ÿÆÿÆÿƃÿƃÿÅ…ÿÅ…ÿÅ…ÿÅ…ÿĆÿĆÿćÿćÿÆÿÁ„ÿÁ‚ÿÁ‚ÿÁ„ÿÀƒÿÀ‚ÿÁƒÿÂ…ÿÂ…ÿÂ…ÿÆÿÆÿÆÿÈÿÈÿÈÿÈÿˆÿˆÿÆÿÆÿÇÿÇÿÇÿÇÿĉÿĉÿÈÿÈÿÆÿÆÿÂ…ÿÂ…ÿ‡ÿ‡ÿĈÿĈÿňÿňÿĈÿĈÿĉÿĉÿÊÿÄ‹ÿćÿćÿćÿćÿĉÿĉÿÉÿÄŠÿÄŠÿÄŠÿÅŠÿÅŠÿÈÿÈÿÉÿÉÿËÿÍ‘ÿДÿÒ–ÿÕ›ÿ×ÿÑœÿ͘ÿÑÿ¶„ÿ¥vÿ“dÿ}Pìj=ëX/ßL#é?á7î/í.þ*ÿ,ÿ-ÿ2ÿ5ÿ3ÿ2ÿ0ÿ/ÿ/ÿ1ÿ1ÿ2ÿ4ÿ5ÿ7ÿ5ÿ0ÿ1ÿ0ÿ1ÿ2ÿ1ÿ2ÿ3ÿ6ÿ9ÿ. Þ!È ÅÚ+ÿ0ÿ3ÿ/ÿ2ÿ8ÿ=ÿQ'ÿa7ÿu;ÿ{AÿŠ;ÿ’Cÿ©Eÿ¾ZÿÓYÿÔZÿÖaÿÖaÿÕnÿÕnÿÖ{ÿÕzÿÕ}ÿ×ÿÙŒÿÜÿà™ÿãœÿæ¦ÿç§ÿà¡ÿÞŸÿÚ›ÿØ™ÿטÿÕ–ÿÑ•ÿДÿÏ•ÿÍ“ÿÊŽÿÊŽÿÈÿÆ‹þÅŒÿÆÿÆ‹ÿÆ‹þÅŠû‡üÈüÈüÈüÈüĆüĆþÅŠþÅŠÿÅŠÿÅŠÿƈÿljÿLJÿȈÿȈÿȈÿLJüÄ„üÄ„üÄ„üÈüÈüÈüÈüÊüÊû‰û‰û‰úÁˆúÁˆúÁˆùÀ‰úÁŠúÁˆúÁˆùÀ…ùÀ…ùÀ…ùÀ…ùÀ‡ø¿†ø¿†ø¿†ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒõ¼õ¼õ¼õ¼õ¼õ¼õ¼õ¼õ¼ô»€ô»€ò¼€ò¼€ñ»ñ»ñ»ðº~ðº~ðº~ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}î¸|î¸|ì¶zì·xì·xë¶wë¶wë¶wë¶wè´xéµyéµyéµyê´xê´xé³yé³yç³wç³wè²vç±uê±vê±vè²xè²xè²vè²væ²væ²vå³vä²uç±wå¯uæ­tæ­tç®wê±z賀賀汀췆鴃ܧvÊ™n»Š_¦yT™lG‘hDX4e@ T/D"6*  - 0N,zQ-›rN¶Œb˜nËŸqÏ£uÏ¡nΠmÒ iÑŸhÑeÐœdÓšaÓšaÓšcÓšcÏ›cÏ›cÍ™aÍ™a̘b̘bË—a̘b̘bË—aË—a̘bÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÛ¥kÛ¥kܦlܦjܦjܦjܦjܦjܦjÚ¦jÚ¦jÛ§kÛ§kÛ§kÛ§kÛ§kܨlܨlܨlܨlܨlܨlܨlܨlÞªnÝ«nÝ«nÞªnÞªnß«oß«oß«oß«oß«oß«oß«oß«oà¬pà¬pà¬pà¬pá­qá­qá­qâ®râ®râ®râ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯så±uå±uç±uç±uç±uè²vè²vè²vè²vé³wé³wé³wê´xê´xê´xê´xê´xê´xê´xê´xê´xê´xê´xëµyëµyëµyëµyëµyëµyëµyëµyëµyëµyëµyëµyì¶zì·xì·xì·xì·xì·xî¹zî¹zî¹zî¸|ï¹}ï¹}ï¹}ï¹}ðº~ðº~ðº~ðº~ðº~ðº~ðº~ðº~ðº~òº|ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~õ½õ½õ½õ½ó¾õÀõÀõÀöÁ‚öÁ‚öÁ‚÷ƒùÁƒú„ú„ú„ýÃýÃÿÃÿÅÿÆ‚ÿÈ„ÿË‚ÿÍ„ÿÌ€ÿÌ€ÿÊ€ÿÊ€ÿÊ€ÿÉÿÉÿÈ~ÿÈ~ÿÉÿÉÿÉÿÉÿÈ€ÿÉ€ÿÉ€ÿÇÿÇÿÅ‚ÿÅ‚ÿÅ‚ÿÅ‚ÿÅ…ÿÅ…ÿÄ…ÿÄ…ÿÄÿÁ‚ÿ‚ÿÁÿÁƒÿÁƒÿÁ„ÿÂ…þÂ…þÂ…þÂ…ýÁ„þÂ…þÂ…ÿÂ…ÿňÿĈÿĈÿÈÿÈÿÆÿÆÿÇÿÇÿĈÿĈÿÈÿĉÿĉÿĉÿĉÿĉÿÈÿÈÿĈÿĈÿćÿňÿňÿƉÿƉÿƉÿʼnÿʼnÿÈÿÈÿÈÿÈÿĉÿĉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÄŠÿÄŠÿÉÿÉÿÉÿÅÿÅÿÇ‘ÿÇ‘ÿÈ’ÿË•ÿÑšÿÓœÿÔœÿÑ™ÿË‘ÿÄŠÿ¶~ü©qõ˜båˆRÛtAÍf3ÑU)ÈL ËDÆ?×6×6á3á3ò/ó0þ0ÿ4ÿ4ÿ4ÿ3ÿ4ÿ3ÿ3ÿ4ÿ0 ÿ0ÿ1ÿ0ÿ1ÿ4ÿ5ÿ7ÿ2ÿ,þÜÕíÿ(ÿ/ÿ1ÿ1ÿ5ÿ:ÿBÿ]*ÿm:ÿ?ÿŠHÿ¤Oÿ±\ÿÅbÿÒoÿÕmÿÖnÿÖxÿÖxÿׂÿ؃ÿÙŒÿÝÿá•ÿä˜ÿæŸÿé¢ÿë¥ÿê¤ÿç¤ÿã ÿÝ›ÿÛ™ÿÙ”ÿ×’ÿÕ“ÿÑÿÏÿÍ‹ÿÍÿËÿÉ‹ÿÈŠÿÈŠÿÈŠÿÆ‹ÿÆ‹ÿÆ‹þÅŠþÅŠüÈüÈüÈüĆüĆûÃ…üĆüĆüĆÿLJÿȈÿȈÿɉÿȃÿȃÿȆÿȆÿȈþƆüÄ„üÄ„üÈüÈüÊüÊû‰û‰û‰û‰û‰úÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆùÀ‡ùÀ…ùÀ…ùÀ‡ø¿†ø¿†ø¿†ø¿†ø¿†ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„õ¼õ¼õ¼õ¼õ¼õ¼õ¼õ¼õ¼ô»€ô»‚ô»‚óºóºñ»ðº~ðº~ðº~ðº~ðº~ð»|ð»|ïº{ð»|ð»|ïº{ï¹}ï¹}î¸|î¸|î¹zî¹zî¶xíµwî¶xî¶xì¶zì¶zéµ{ê¶|éµ{éµ{ê´zëµ{ê´zé³yé³yé³yê´zé³yæ²zç³{è²vç±uæ²væ²vå³vå³vè²xè²xê±xé°wé°wç®uç®uê±xé®sè­ræ­të²yí´}í´}굂ç²æ²xÞªpОeÂW³ƒLžn7„S$i8 M@8 -0(% - -'<&aE2uYF‰fO }f°‰i¼•uÈžxË¡{ÌŸvÌŸvПkПkÑŸhÏfÐœbÐœbÏ™]Ï™]͘eÌ—dÌ—dÌ—d̘bË—aË—a̘bÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÛ¥kÛ¥kÛ¥kܦlܦlܦjܦjܦjܦjܦjݧkÚ¦jÚ¦jÛ§kÛ§kÛ§kÛ§kÛ§kܨlܨlܨlܨlܨlܨlܨlܨlÞªnÝ«nÞ¬oß«oß«oß«oà¬pà¬pà¬pà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qâ®râ®râ®râ®râ®râ®râ®rã¯sã¯sã¯så±uå±uå±uå±uã¯så±uå±uå±uå±uå±uæ²vè²vè²vè²vè²vè²vé³wé³wé³wé³wê´xê´xê´xê´xê´xê´xê´xê´xê´xê´xëµyëµyëµyëµyëµyëµyëµyëµyëµyëµyëµyëµyëµyëµyì¶zì·xì·xì·xì·xì·xî¹zî¹zî¹zï¹}ï¹}ï¹}ï¹}ï¹}ðº~ðº~ðº~ðº~ðº~ðº~ðº~ðº~ñ»ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~õ½õ½õ½õ½õÀõÀõÀöÁ‚öÁ‚öÁ‚÷ƒ÷ƒú„ú„ú„ú„ýÃÿƆÿǃÿǃÿǃÿÈ„ÿÍ„ÿÎ…ÿÌ€ÿÌ€ÿÌ‚ÿÊ€ÿÊ€ÿÉÿÉÿÈ~ÿÈ~ÿÉÿÉÿÉÿÈ€ÿÈ€ÿÉ€ÿÉ€ÿÈ‚ÿÇÿÇ„ÿÅ‚ÿÅ‚ÿÅ‚ÿÅ…ÿÅ…ÿÄ…ÿÄ…ÿÄ…ÿÄÿ‚ÿÄ„ÿĆÿÁƒÿÁ„ÿÂ…þÂ…þÂ…þÂ…ýÁ„þÂ…þÂ…ÿÂ…ÿňÿĈÿĈÿÈÿÈÿÆÿÆÿÇÿÇÿĈÿĈÿÈÿĉÿĉÿĉÿĉÿĉÿÈÿÈÿĈÿĈÿćÿňÿňÿƉÿƉÿƉÿʼnÿʼnÿÈÿÈÿÈÿÈÿĉÿĉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÄŠÿÄŠÿÉÿÉÿÉÿÉÿÄŒÿÄŒÿÄŽÿÄŽÿÇ‘ÿÇ‘ÿÈ‘ÿÈ‘ÿÉ‘ÿÇÿÊÿЖÿÑ™ÿÒšÿΘÿÈ’ÿÀÿ¶ƒÿžrÿaÿ{Kõn>úY3ðO)í?æ8õ2ò/ú,ú,ÿ) ÿ+ÿ,ÿ0ÿ/ÿ2ÿ3ÿ0 ÿ0ÿ0ÿ1ÿ1ÿ5ÿ6ÿ6ÿ6ÿ7 ÿ.ò$ êú# ÿ-ÿ2ÿ1ÿ5ÿ8ÿ@ÿN)ÿj7ÿzGÿŒJÿ™Wÿ´_ÿÂmÿÓpÿÖsÿ×oÿØpÿÙ{ÿØzÿ؃ÿ܇ÿß’ÿä—ÿèœÿêžÿê£ÿê£ÿè¢ÿæ ÿã ÿÞ›ÿÛ™ÿØ–ÿÖ‘ÿÔÿÒÿÏÿÍ‹ÿˉÿËÿËÿÉ‹ÿÉ‹ÿljÿljÿÆ‹ÿÆ‹ÿÆ‹þÅŠþÅŠüÈþÅŠþÅŠþƈÿljÿljÿljÿljÿljÿLJÿɉÿɉÿÊŠÿÊ…ÿˆÿɇÿȆÿȈÿLJüÄ„üÄ„üÈüÈüÊüÊû‰û‰û‰û‰û‰úÁˆúÁˆúÁˆúÁˆúÁˆúÁˆùÀ‡ùÀ‡ø¿†ø¿„ùÀ…ø¿†ø¿†ùÀ‡ùÀ‡ø¿†ø¿†ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„ø¿„õ¼õ¼õ¼÷¾ƒ÷¾ƒõ¼õ¼õ¼õ¼õ¼ƒô»‚ô»€ô»€ñ»ñ»ñ»ðº~ðº~ðº~ð»|ð»|ð»|ð»|ð»|ïº{ï¹}ï¹}î¸|î¸|î¹zî¹zð¸zî¶xð¸zð¸zî¸|î¸|ì¸~ê¶|ê¶|ê¶|ì¶|ì¶|ëµ{ê´zëµ{ëµ{ëµ{ê´zè´|è´|é³wé³wç³wç³wå³vå³vè²xè²xê±xê±xê±xê±xê±xé°wé®sè­rç®uæ­tç®wé°yæ±~æ±~æ²xéµ{çµ|çµ|åµ~à°yÛª{Ç–g«|[a@zL5c5Q&H -<5( "  "( ;P-e>wP0Š`:sM¬V·ŠaÀ[Ç–bÎœeОgÒždÒždÒœ`Òœ`ÏšgÌ—dÌ—dÌ—dÍ™cÍ™c̘b̘bÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÛ¥kÛ¥kÛ¥kܦlܦlܦjܦjܦjܦjܦjݧkÛ§kÛ§kÛ§kÛ§kܨlܨlܨlܨlܨlܨlܨlÞªnÞªnÞªnÞªnÞªnÞ¬oÞ¬oß«oß«oà¬pà¬pà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qá­qâ®râ®râ®râ®râ®rã¯sã¯sã¯sã¯så±uå±uå±uå±uå±uå±uå±uå±uå±uå±uå±uæ²væ²vè²vè²vè²vé³wé³wé³wé³wé³wê´xê´xê´xê´xê´xê´xê´xê´xëµyëµyê´xëµyëµyëµyì¶zì¶zì¶zì¶zì¶zì¶zì¶zì¶zì¶zì¶zì¶|ì¶|ì¶zì¶zì·xì·xì¶zî¸|î¸|î¸|ï¹}ï¹}ï¹}ðº~ðº~ðº~ðº~ðº~ðº~ðº~ðº~ðº~ðº~ñ»ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~õ½õ½õ½õ½õ½÷¿õÀõÀöÁ‚öÁ‚÷ƒ÷ƒ÷ƒøÄú„ú„ú„ú„ü‚þÄ„ÿÆ…ÿdžÿÇÿɃÿÌ‚ÿÊ€ÿÊ~ÿÌ€ÿÌ‚ÿÉÿÉÿÉÿȃÿÇ‚ÿÇ‚ÿȃÿȃÿȃÿÇ‚ÿÇ‚ÿÉ€ÿÉ€ÿÈ‚ÿɃÿÇ‚ÿÇ‚ÿÇ„ÿÇ„ÿÅ‚ÿÅ‚ÿćÿćÿĉÿÅŠÿʼnÿĈÿĈÿ†ÿÂ…ÿÂ…þÂ…þÂ…þÂ…þÂ…ýÁ„ýÁ„ÿÆÿňÿĈÿĈÿÈÿÈÿĆÿĆÿŇÿŇÿćÿňÿÈÿÈÿÈÿÈÿÈÿÈÿÇÿÇÿÇÿÇÿĆÿƈÿLjÿLjÿňÿňÿňÿňÿĈÿĈÿĈÿĈÿĉÿĉÿĉÿĉÿÉÿÉÿÉÿÉÿĉÿĉÿĉÿĉÿÅŠÿÅŠÿÅŠÿĉÿÆŒÿÆŒÿÄŽÿÄŽÿÅŽÿÆÿÅÿÅÿÇ’ÿÇ’ÿÈÿÈÿÅÿÇÿÇÿÈÿË“ÿÑ™ÿÒ˜ÿÍ“ÿÊŒÿÃ…ÿ´|ù§oñ•bà„QÞtDÑg7ÎX+ÇQ$ËGÈDÔ;Ò9â4 â4 ó3÷7 ÿ4 ÿ5 ÿ2ÿ3ÿ3ÿ4ÿ6ÿ8ÿ7ÿ6ÿ1ÿ,ÿ+ÿ0ÿ3ÿ6ÿ<ÿD ÿW+ÿi=ÿ…Kÿ”Zÿ¦dÿ±oÿÁqÿÉyÿÒ}ÿÔÿ؇ÿÙˆÿÜÿà”ÿâ›ÿåžÿè ÿê¢ÿê¢ÿçŸÿèÿçœÿâ›ÿߘÿÜ•ÿÙ’ÿ×ÿ×ÿÔÿÑŒÿЋÿ͈ÿʈÿʈÿÊŒÿÊŒÿÉ‹ÿÉ‹ÿÉ‹ÿljÿljÿljÿljþƈþƈüĆÿljÿljÿLJÿȈÿÈŠÿÈŠÿɉÿɉÿʈÿˉÿʆÿʆÿ͈ÿ͈ÿʈÿȆÿLJÿLJþƆüÄ„üÈüÈüÊüÊû‰û‰û‰û‰úÁˆùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡úÁˆùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‰ùÀ‰ùÀ‡ø¿†ø¿†ø¿†ø¿†ø¿†ø¿†ø¿†ø¿„÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒõ¼õ¼÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒõ¼õ¼õ¼ƒô»‚ô»€ô»€ò¼€ñ»ñ»ðº~ñ»ñ»ñ¼}ñ¼}ñ¼}ð»|ð»|ð»|ðº~ï¹}ï¹}î¸|î¸|î¸|ð·~ð·~ð·~ð·~ì¶~î¸~ì¶|î¸~ì¶|ì¶|ì¶|ì¶|ëµ{ëµ{ëµ{ëµ{ëµ{éµ{éµ{ê´zé³yç³yç³yç³yç³yè²xè²xè²xè²xè²xè²xè²xè²xë°uë°uê­sê­së°wé®uå¬sæ­tç¯qå­oæ­ré°uê²të³uì³xîµzëµ{ܦlØ¢hÈ’X¶ƒO¨uA™h6Ž]+xJc5U+G9/%! - - - & 9L0^B+oQ4~`CŽmIž}Y«…\µf×kËŸsÊžrÉqÊœnÌžpÓ qΛlË–e͘gÚ¤jÚ¤jÚ¤jÚ¤jÚ¤jÛ¥kÛ¥kÛ¥kܦlܦlܦjܦjܦjܦjܦjݧkÛ§kÛ§kÛ§kܨlܨlܨlܨlܨlÞªnÞªnÞªnÞªnÞªnÞªnÞªnß«oÞ¬oÞ¬oß«oß«oà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qá­qá­qâ®râ®râ®râ®râ®râ®râ®rã¯sã¯sã¯sã¯så±uå±uå±uå±uå±uå±uå±uå±uæ²væ²væ²væ²væ²vè²vè²vè²vé³wé³wé³wé³wé³wê´xê´xê´xê´xëµyëµyê´xëµyëµyëµyëµyëµyëµyì¶zì¶zì¶zì¶zì¶zì¶zì¶zì¶zì¶zì¶zì¶zì¶|ì¶|î¸|î¸|î¹zî¹zî¸|î¸|î¸|ï¹}ï¹}ï¹}ï¹}ðº~ðº~ðº~ðº~ðº~ðº~ðº~ðº~ðº~ðº~ñ»ó»}ó»}ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~ô¼~õ½õ½õ½õ½õ½÷¿÷¿õÀöÁ‚öÁ‚öÁ‚÷ƒ÷ƒøÄøÄú„ûÃ…ûÃ…ûÃ…ýÃÿƆÿdžÿɈÿɃÿ̆ÿÌ‚ÿÊ€ÿÊ~ÿÌ€ÿÌ‚ÿÉÿÉÿÉÿÇ‚ÿÆÿÆÿÇ‚ÿÇ‚ÿÇ‚ÿÇ‚ÿÇ‚ÿÊÿÊÿÈ‚ÿɃÿÇ‚ÿÇ‚ÿÇ„ÿÇ„ÿƃÿƃÿňÿćÿÅŠÿÅŠÿʼnÿĈÿĈÿ†ÿÂ…ÿÂ…þÂ…þÂ…þÂ…þÂ…þÂ…ýÁ„ÿÁ„ÿÆÿ†ÿ†ÿÁ†ÿÈÿĆÿĆÿŇÿŇÿćÿňÿÈÿÈÿÈÿÈÿÈÿÈÿÇÿÇÿÇÿÇÿĆÿŇÿƇÿƇÿćÿňÿňÿćÿÇÿÇÿÇÿĈÿĉÿĉÿÅŠÿÅŠÿÄŠÿÄŠÿÄŠÿÄŠÿÅŠÿÅŠÿĉÿĉÿÈÿĉÿĉÿĉÿÅ‹ÿÆŒÿÄŽÿÄŽÿÅŽÿÅŽÿÄŽÿÅÿÆ‘ÿÆ‘ÿÇÿÇÿÅÿÇÿÇÿÉ‘ÿÉ‘ÿÊ’ÿΔÿÑ—ÿÔ–ÿÔ–ÿÏ—ÿÑ™ÿÏœÿÈ•ÿ»‹ÿ¯ÿžqÿcþzMònAö]5êQ)óEì>ù9ó3þ1 ü/ÿ+ ÿ, -ÿ, ÿ, ÿ0ÿ7ÿ2ÿ2ÿ2ÿ1ÿ2ÿ6ÿ8ÿ<ÿD ÿS/ÿk?ÿ~Rÿ—]ÿ¢hÿ±oÿ¹wÿÅuÿÉyÿÏzÿÓ~ÿ؇ÿÝŒÿß“ÿä˜ÿé¢ÿê£ÿì¤ÿë£ÿê¢ÿæžÿåšÿâ—ÿÝ–ÿÛ”ÿØ‘ÿØ‘ÿÕŽÿÒ‹ÿÒÿÑŒÿÏŠÿˆÿʈÿʈÿÊŒÿÊŒÿÉ‹ÿÉ‹ÿÉ‹ÿÈŠÿÈŠÿÈŠÿljþƈÿljÿÈŠÿÈŠÿÈŠÿɉÿɉÿÉ‹ÿÉ‹ÿÊŠÿÍÿÍ‹ÿÍ‹ÿ̈ÿ̈ÿ͈ÿÊ…ÿɇÿȆÿLJÿLJþƆüÄ„üÈüÈû‰û‰û‰úÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‰ùÀ‰ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡ø¿†ø¿†ø¿„÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒõ¼õ¼õ¼÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒõ¼õ¼õ¼ƒô»‚ô»€ô»€ò¼€ñ»ñ»ñ»ñ»ñ»ñ¼}ñ¼}ñ¼}ñ¼}ñ¼}ñ¼}ðº~ï¹}ï¹}î¸|î¸|ì¶zð·~ð·~ð·~ð·~ì¶|î¸~î¸~ì¶|ì¶|ì¶|ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{è´zè´zê´zê´zç³yç³yç³yç³yé³yé³yé³yè²xè²xè²xè²xè²xë°uë°uì¯uê­së°wë°wç®uæ­tç¯qæ®på¬qå¬qå­oä¬nå¬qç®sç±wè²xè²xé³yç´€æ³à¯}Ù¨vΠrÀ’d­ƒ]˜nH‚X9lB#O->1+##   !* 1=O. -d>uO&‹_3œpD©}Q·‹_À’dÅ—iÊ—hÊ—hÌ—fÏšiÛ¥mÛ¥mÛ¥mÛ¥mÛ¥mÛ¥mÛ¥mÛ¥mܦlܦlܦlݧmݧmݧmݧmݧmÛ§kÛ§kÛ§kܨlܨlܨlܨlܨlÞªnÞªnÞªnÞªnÞªnÞªnÞªnß«oÞ¬oß­pà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qá­qâ®râ®râ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯så±uå±uå±uå±uå±uå±uæ²væ²væ²væ²væ²væ²vç³wë²wë²wë²wë²wë²wì³xì³xì³xê´xê´xê´xëµyëµyëµyëµyëµyëµyëµyëµyëµyì¶zì¶zì¶zì¶zì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¸|î¸~î¸~î¸~î¸~î¸|î¸|î¸|ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ðº~ðº~ðº~ðº~ðº~ðº~ðº~ñ»ñ»ñ»ñ»ô»€ô»€ô»€ô»€ô»€ô»€ô»€õ¼õ¼õ¼õ½õ½÷¿÷¿÷¿÷¿öÁ‚öÁ‚öÁ‚÷ƒ÷Á…÷Á…ú„ú„ûÃ…ûÃ…ýÃýÃýÃÿÆ„ÿÇÿɃÿÊ„ÿ͇ÿ̓ÿ̓ÿÊ€ÿÊ€ÿÊ‚ÿÉÿȃÿȃÿÇ‚ÿÆÿǃÿÈ„ÿÈ…ÿÈ…ÿÈ„ÿÈ„ÿÉ…ÿÉ…ÿɃÿɃÿÉ„ÿÉ„ÿÉ„ÿÉ„ÿȃÿȃÿÈŠÿljÿÆŠÿÆŠÿƈÿŇÿÁˆÿÁˆÿˆÿÁ‡þÁ‡þÁ‡þÁ‡þÁ‡þÁ‡þÁ‡ÿˆÿˆÿÁˆÿÊÿŠÿŠÿÆÿÆÿćÿćÿ†ÿĈÿÈÿÈÿÈÿÈÿÈÿÈÿÇÿÇÿćÿćÿÆÿćÿŇÿŇÿŇÿŇÿćÿćÿĈÿĈÿĈÿʼnÿĉÿĉÿÅŒÿÅŒÿÅŒÿÅŒÿÄŒÿÄŒÿÅŒÿÄ‹ÿÅ‹ÿÄŠÿÆ‹ÿÆ‹ÿÇ‹ÿÇ‹ÿÆ‹ÿÇŒÿÅŽÿÅŽÿÄŽÿÄŽÿÅÿÅÿÄÿÄÿÇÿÇÿÇÿÇÿÇÿÇÿÈÿÈÿÉÿÉÿÈÿÊÿÍ“ÿЖÿÑ—ÿÒ˜ÿÔ™ÿÕšÿÖ›ÿÔ™ÿÍ’ÿÈÿ³xò¥jì–[߉NÙz>Ìm1Ùc,ÌVÎGÊCÞ=Þ=ê7ê7ö3 ö3 ÿ2ÿ3ÿ3ÿ:ÿAÿHÿa0ÿuDÿ’Wÿ¡fÿ­sÿ²xÿ¾ÿƒÿ̃ÿφÿÕŠÿÜ‘ÿá™ÿæžÿè¥ÿê§ÿé¤ÿé¤ÿçÿå›ÿä™ÿã˜ÿß’ÿÝÿÚŽÿÙÿÖÿÔÿÒÿÒÿÑ‘ÿÏÿÍÿÊŠÿÊŠÿÊŠÿÉ‹ÿÊŒÿÊŒÿÊŒÿÊŒÿÉ‹ÿÉ‹ÿÉ‹ÿÈŠÿljÿÈŠÿÈŠÿÉ‹ÿÉ‹ÿɉÿÊŠÿÊŠÿÊŠÿˉÿÎŒÿΉÿΉÿ͉ÿ͉ÿΉÿÊ…ÿɉÿɉÿljÿljüĆüĆüÊüÊüÊû‰û‰û‰û‰û‰úÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁŠúÁŠúÁŠùÀ‰ùÀ‡ùÀ‡ùÀ‰ùÀ‰ùÀ‰ùÀ‰ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡÷¾ƒ÷¾ƒ÷¾ƒ÷¾ƒù¾ƒù¾ƒõ¼õ¼÷¾…÷¾…÷¾…÷¾…õ¼ƒõ¼ƒõ¼ƒõ¼ƒò¼‚ò¼‚ò¼€ò¼€ò¼€ñ»ñ»ñ»ñ»ñ»ñ»ñ»ñ»ñ»ðº~ðº~ï¹}ï¹}ï¹}î¸|ð·~ð·~ð·~ð·~ï¹î¸~î¸~î¸~î¸~î¸~ì¶|ëµ{ì¶|ëµ{éµ{è´zê´zê´zè´|ç³{ç³{ç³{é³yé³yé³yè²xè²xç±wç±wè²xê±xê±xì±vì±vë°uë°uë°uë°ué®sé®sé®sè­rè­rè­rè­rè­rå¬sä«râ¬râ¬rá­uå±yçµ|ë¹€ë¹~è¶{å³vá¯rÚ¨mË™^´ƒO›j6€R"pBm9_+T"M>: 0("!  . >&S8bG+qS6…gJœrS¬‚cº†`¹…_Û¥mÛ¥mÛ¥mÛ¥mÛ¥mÛ¥mÛ¥mܦnܦlܦlݧmݧmݧmݧmݧmݧmÛ§kÛ§kܨlܨlܨlܨlܨlÞªnÞªnÞªnÞªnÞªnÞªnÞªnÞªnß«oß­pß­pà¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qá­qâ®râ®râ®râ®râ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯så±uå±uå±uå±uå±uæ²væ²væ²væ²væ²væ²vç³wç³wë²wë²wë²wë²wë²wì³xì³xì³xê´xê´xê´xëµyëµyëµyëµyëµyëµyì¶zëµyëµyì¶zì¶zì¶zì¶zì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¸|î¸~î¸~î¸~î¸~î¸|î¸|î¸|ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ðº~ðº~ðº~ðº~ðº~ðº~ðº~ñ»ñ»ñ»ñ»ô»€ô»€ô»€ô»€ô»€ô»€ô»€õ¼õ¼õ¼÷¿÷¿÷¿÷¿÷¿÷¿öÁ‚öÁ‚öÁ‚÷ƒ÷Á…÷Á…ûÃ…ûÃ…ûÃ…üĆþÄ„þÄ„ÿÆ„ÿȆÿɃÿÊ„ÿ̆ÿ͇ÿ̓ÿ̓ÿÊ€ÿÊ€ÿÊ‚ÿÊ‚ÿȃÿÇ‚ÿȃÿȃÿÈ„ÿÉ…ÿÈ…ÿÈ…ÿÉ…ÿÉ…ÿÉ…ÿÉ…ÿɃÿÊ„ÿˆÿˆÿˆÿˆÿ̇ÿˆÿÉ‹ÿÉ‹ÿÈŒÿÇ‹ÿƈÿƈÿÄ‹ÿÁˆÿˆÿˆÿˆþÁ‡ÿˆÿˆþÁ‡ÿˆÿˆÿÄŠÿÊÿÊÿŠÿŠÿćÿćÿćÿćÿĈÿĈÿÈÿÈÿÈÿÈÿÈÿÈÿĈÿÇÿćÿćÿÆÿćÿŇÿŇÿŇÿƈÿňÿćÿĈÿĈÿĈÿʼnÿĉÿĉÿÅŒÿÅŒÿÅŒÿÅŒÿÄŒÿÄŒÿÅŒÿÄ‹ÿÅ‹ÿÄŠÿÆ‹ÿÆ‹ÿÇ‹ÿÇ‹ÿÆ‹ÿÇŒÿÅŽÿÅŽÿÄŽÿÄŽÿÅÿÅÿÄÿÄÿÅÿÅÿÇÿÇÿÇÿÇÿÇÿÇÿÉÿÉÿÈÿÈÿÆŒÿÆŒÿÆŒÿÇÿÍ’ÿÓ˜ÿÒ—ÿÏ”ÿÕšÿÜ¡ÿÙžÿÔ™ÿÍ’ÿ‡ÿ´xÿ¬pÿ˜aÿ‰RþwDôm:ÿc=øW1ýJ!÷Dÿ?ý:ÿ9ÿ:ÿ8ÿ<ÿGÿT+ÿxGÿŽ]ÿ§lÿ¯tÿ¶|ÿ¼‚ÿņÿÉŠÿÔ‹ÿ׎ÿÞ“ÿâ—ÿçŸÿê¢ÿë¨ÿç¤ÿå ÿâÿá—ÿà–ÿÞ“ÿß”ÿÞ‘ÿÜÿÚŽÿØŒÿÕŽÿÑŠÿÐŽÿÐŽÿÏÿÍÿË‹ÿË‹ÿÊŠÿÊŠÿÊŒÿÉ‹ÿÉ‹ÿÉ‹ÿÉ‹ÿÉ‹ÿÉ‹ÿÉ‹ÿÉ‹ÿÈŠÿÈŠÿÈŠÿÊŒÿÊŒÿÊŠÿË‹ÿÍÿÍÿÎŒÿÏÿÏŠÿÏŠÿΊÿ͉ÿ͈ÿÊ…ÿɉÿȈÿljüĆüĆüĆüÊüÊüÊû‰û‰û‰û‰û‰úÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁŠúÁŠúÁŠúÁŠúÁˆùÀ‡ùÀ‰ùÀ‰ùÀ‰ùÀ‰ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡ùÀ‡÷¾ƒ÷¾ƒ÷¾ƒø¿„ú¿„ù¾ƒ÷¾ƒ÷¾ƒõ¼ƒ÷¾…÷¾…÷¾…õ¼ƒõ¼ƒõ¼ƒõ¼ƒó½ƒò¼‚ò¼€ò¼€ò¼€ò¼€ñ»ñ»ñ»ñ»ñ»ñ»ñ»ñ»ðº~ðº~ðº~ðº~ï¹}ï¹}ñ¸ñ¸ñ¸ñ¸ï¹î¸~î¸~î¸~î¸~î¸~î¸~ì¶|ì¶|ì¶|éµ{éµ{ê´zê´zè´|è´|è´|ç³{é³yé³yé³yé³yé³yè²xè²xè²xê±xê±xì±vì±vë°uì±vë°uë°ué®sé®sé®sé®sè­ré®sè­ré®sæ­tå¬sã­sã­sà¬tÞªrÝ«rÝ«rà®sà®sß­pà®qä²wä²wä³ݬxΠpÕeÅ‘i³W£qM–d@P/vG&c8P% C 5) #   &9Q']3wCˆT.Û¥mÛ¥mÛ¥mÛ¥mÛ¥mÛ¥mܦnܦnܦlܦlݧmݧmݧmݧmݧkÞ¨lÛ§kܨlܨlܨlÞªnÞªnÞªnÞªnÞªnÞªnÞªnÞªnÞªnÞªnß«oß«oß«oà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qâ®râ®râ®râ®râ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯så±uå±uå±uå±uå±uæ²væ²væ²væ²væ²væ²vç³wç³wé³wé³wé³wé³wé³wê´xê´xê´xê´xê´xê´xê´xê´xê´xëµyëµyëµyëµyëµyì¶zì¶zëµyì¶zì¶zê¶zê¶zê¶|ê¶|ì¸~ì¸~ì¸~ì¸~ì¸~ì¸~ì¸~ì¸~ì¸~ì¸~ì¸~ì¸~ï¹ï¹ï¹ðº€ðº€ðº€ðº€ðº€ðº€ðº€ðº€ñ»ñ»ñ»ñ»ñ»ò¼€ò¼€ò¼€ò¼€ò¼€ò¼€ò¼€ó½ó½ó½õ¿ƒõ¿ƒõ¿ƒõ¿ƒõ¿ƒöÀ„öÀ„öÀ„÷Á…÷Á…ùÀ…ùÀ…ûÃ…ûÃ…ýÃ…þĆþÄ„þÄ„ÿÅÿǃÿɃÿÊ„ÿÊ„ÿ̆ÿ̓ÿ̓ÿÊ€ÿÊ€ÿÊ‚ÿÉÿȃÿÇ‚ÿÇ„ÿÇ„ÿȇÿȇÿÈ„ÿÉ…ÿɃÿɃÿɃÿɃÿɃÿÊ„ÿ̆ÿ̆ÿÌ„ÿÌ„ÿÍ…ÿÍ…ÿ̉ÿɆÿȈÿLJÿljÿljÿʼnÿĈÿňÿňÿˆÿˆÿÆÿÆÿÆÿňÿĈÿĈÿĈÿʼnÿĉÿĉÿĉÿĉÿňÿňÿĈÿĈÿÈÿÈÿÈÿÈÿĈÿĈÿĈÿĈÿĉÿĉÿĉÿĉÿʼnÿʼnÿʼnÿʼnÿĉÿĉÿÄŠÿÅ‹ÿÅ‹ÿÅ‹ÿÅŒÿÅŒÿÅŒÿÅŒÿÅŒÿÅŒÿÄŒÿÄŒÿÄ‹ÿÄ‹ÿÄŠÿÄŠÿÆ‹ÿÆ‹ÿÆ‹ÿÅŠÿÅŠÿÆ‹ÿÆ‹ÿÇŒÿÆŒÿÆŒÿÆŒÿÆŒÿÇÿÇÿÆÿÆÿÆÿÈÿÈÿÈÿÆÿÆÿÆÿÆÿÆ‹ÿÆ‹ÿÇÿÇÿÇÿÉÿÈÿÈÿÉÿÊ‘ÿÉÿÊ‘ÿÍ“ÿЖÿÑ•ÿÒ–ÿÕ˜ÿÖ™ÿÔ—ÿÍÿĆÿ»}ÿ­sñŸeì‘Tá†Ià|AÙu:Þh/Üf-èi2êk4ép8øGþ“Xÿ§lÿ¶xÿº|ÿ‚ÿɉÿÏ”ÿÔ™ÿÝœÿâ¡ÿå ÿäŸÿä›ÿãšÿß•ÿÝ“ÿÞ•ÿÞ•ÿÞ‘ÿÞ‘ÿÛŒÿÛŒÿÛÿÛÿÙŒÿ׊ÿÔŽÿЊÿÏÿÎŒÿÍŽÿÊ‹ÿÊ‹ÿÊ‹ÿÉŠÿÉŠÿɇÿɇÿÉ‹ÿÉ‹ÿɉÿɉÿÈŠÿÈŠÿȈÿɉÿɉÿLJÿɉÿÊŠÿˉÿˉÿÍ‹ÿÎŒÿΉÿÏŠÿψÿψÿψÿψÿÍ‹ÿʈÿÉ‹ÿÈŠÿÆ‹üÈüÈüÈüÈüÈüÈû‡û‡û‡û‡û‡úÁˆû‰û‰û‰û‰û‰úÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆ÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†öÀ†õ¿…öÀ†õ¿…õ¿…õ¿…÷¾…÷¾…õ¿…õ¿…õ¿…ó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒò¼‚ò¼‚ò¼‚ò¼€ò¼€ñ»ñ»ñ»ñ»ñ»ñ»ðº~ðº~ðº€ðº€ðº€ï¹ï¹ï¹ï¹ï¹ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~ì¶|ì¶|ì¶|ì¶|ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{ê´zê´zê´zé³yê´zé³yé³yé³yé³yé³yé³yé³yé³yê±xê±xê±vé°uê²tê²té°ué°ué°wç®uç®uç®uç®uæ­tæ­tç®uâ®rá­qá­sâ®tá­uß«sá­uà¬tà¬rà¬rà¬tâ®vç±{é³}è´~å±{èµvå²sÝ«pÙ§lÏŸfÆ–]¼ŒU±J™k;„V&qEb6 -S)K!B==?=?=:7-&$ - - - - - - - -)Û¥mÛ¥mÛ¥mÛ¥mÛ¥mÛ¥mܦnܦnܦlݧmݧmݧmݧmݧmݧkÞ¨lÛ§kܨlܨlܨlÞªnÞªnÞªnÞªnÞªnÞªnÞªnÞªnÞªnÞªnß«oß«oß«oà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qâ®râ®râ®râ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯så±uå±uå±uå±uæ²væ²væ²vç³wç³wç³wç³wç³wè´xé³wé³wé³wê´xê´xê´xê´xê´xëµyëµyëµyëµyëµyê´xëµyëµyëµyëµyëµyì¶zì¶zëµyì¶zì¶zê¶zì¸|ì¸~ì¸~ì¸~ì¸~ì¸~ì¸~ì¸~í¹í¹í¹í¹í¹í¹í¹ðº€ðº€ðº€ðº€ðº€ðº€ðº€ñ»ñ»ñ»ñ»ñ»ñ»ñ»ò¼‚ò¼‚ò¼€ò¼€ò¼€ò¼€ò¼€ò¼€ò¼€ó½ó½ó½õ¿ƒõ¿ƒõ¿ƒõ¿ƒõ¿ƒöÀ„÷Á…÷Á…÷Á…÷Á…ùÀ…û‡ûÃ…ûÃ…ÿƈÿƈÿƆÿƆÿǃÿÉ…ÿÊ„ÿÊ„ÿÊ„ÿ̆ÿ̓ÿ̓ÿÊ€ÿÊ€ÿÊ‚ÿÉÿÇ‚ÿÇ‚ÿÇ„ÿƃÿȇÿȇÿÈ„ÿÉ…ÿɃÿɃÿɃÿɃÿɃÿÊ„ÿ̆ÿ̆ÿÌ„ÿÌ„ÿÍ…ÿΆÿÍŠÿ̉ÿɉÿLJÿljÿljÿʼnÿĈÿňÿňÿˆÿˆÿÆÿÆÿÆÿňÿĈÿĈÿĈÿĈÿĉÿĉÿĉÿĉÿňÿňÿĈÿĈÿÈÿÈÿÈÿÈÿĈÿĈÿĈÿĈÿĉÿĉÿĉÿĉÿʼnÿʼnÿʼnÿʼnÿĉÿĉÿÄŠÿÅ‹ÿÅ‹ÿÅ‹ÿÅŒÿÅŒÿÅŒÿÅŒÿÅŒÿÅŒÿÄŒÿÄŒÿÄ‹ÿÄ‹ÿÄŠÿÄŠÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÇŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆÿÆÿÆÿÆÿÆÿÆÿÆÿÆÿÆÿÆÿÆ‹ÿÅŠÿÆŒÿÇÿÇÿÇÿÆÿÈÿÈÿÆÿÈÿÈÿÇÿÆŒÿÇ‹ÿÉÿËŽÿÍÿÑ”ÿÔ—ÿÕ—ÿ×™ÿÓ™ÿÏ•ÿƉÿ¾ÿ±vÿ¥jÿšaÿ•\ÿYÿXÿ‘Yÿ›cÿ©nÿ´yÿ¾€ÿÁƒÿË‹ÿÒ’ÿÙžÿÝ¢ÿá ÿã¢ÿâÿÞ™ÿÞ•ÿÞ•ÿÝ“ÿÛ‘ÿÜ“ÿÜ“ÿÜÿÜÿÙŠÿÙŠÿØŠÿØŠÿÙŒÿ׊ÿÑ‹ÿωÿÎŒÿˉÿÊ‹ÿÊ‹ÿÉŠÿÉŠÿÉŠÿÉŠÿɇÿɇÿÉ‹ÿÉ‹ÿɉÿɉÿÈŠÿÈŠÿȈÿɉÿɉÿȈÿɉÿɉÿˉÿˉÿˉÿÎŒÿΉÿΉÿψÿψÿψÿψÿÍ‹ÿʈÿÉ‹ÿÈŠÿÆ‹üÈüÈüÈüÈüÈû‡û‡û‡û‡û‡û‡û‰û‰û‰û‰û‰û‰úÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆúÁˆøˆ÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡öÀ†öÀ†õ¿…öÀ†öÀ†öÀ†öÀ†ø¿†÷¾…õ¿…õ¿…õ¿…õ¿…ó½ƒó½ƒó½ƒó½ƒó½ƒò¼‚ò¼‚ò¼‚ò¼€ò¼€ò¼€ò¼€ñ»ñ»ñ»ñ»ñ»ðº~ðº€ðº€ðº€ðº€ðº€ðº€ðº€ï¹ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~ì¶|ì¶|ì¶|ì¶|ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{ê´zê´zëµ{ëµ{ëµ{ê´zê´zê´zê´zê´zê´zé³yé³yë²yë²yê±vé°uê²të³uê±vé°ué°wê±xé°wé°wç®uæ­tç®ué°wã¯sâ®râ®tã¯uã¯wâ®vã¯wâ®vá­sà¬rß«sà¬tâ¬vã­wâ®xá­wà­ná®oà®sâ°uã³zåµ|ã³|à°yÙ«{Πp–jµ‰]¥{Q•kA…[5W1|S/yP,xQ1xQ1}V9xQ4mH-\7N,@50( # - - -Û¥kÛ¥kÛ¥kÛ¥kÛ¥kܦlܦlܦlܦlݧmݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªnÞªnÞªnß«oß«oß«oß«oß«oß«oß«oà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qâ®râ®râ®râ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯så±uå±uå±uå±uæ²væ²væ²vç³wç³wç³wç³wè´xè´xç³wç³wè´xè´xè´xè´xè´xéµyëµyëµyëµyëµyëµyëµyëµyëµyëµyëµyì¶zì¶zì¶zì¶zì¶zì¶zê¶zì¸|ì¸~ì¸~ì¸~í¹í¹í¹í¹í¹í¹í¹í¹í¹í¹í¹ðº€ðº€ðº€ðº€ðº€ðº€ðº€ñ»ñ»ñ»ñ»ñ»ò¼‚ò¼‚ò¼‚ò¼‚ò¼€ò¼€ò¼€ò¼€ò¼€ò¼€ò¼€ó½ó½ó½õ¿ƒõ¿ƒõ¿ƒöÀ„öÀ„÷Á…÷Á…÷Á…÷Á…÷Á…ùÁƒûÃ…þĆþĆÿƆÿƆÿÅ„ÿdžÿɃÿ̆ÿÍ„ÿÎ…ÿÎ…ÿÍ„ÿÌ€ÿÌ€ÿÊ‚ÿÊ‚ÿȃÿȃÿÈ„ÿǃÿȇÿdžÿȇÿȇÿÈ„ÿÉ…ÿɃÿɃÿɃÿɃÿÊ„ÿ̆ÿÎ…ÿÎ…ÿÏ„ÿÏ„ÿ΄ÿÏ…ÿχÿχÿ͉ÿʆÿÉ…ÿÉ…ÿƇÿņÿňÿňÿÄŠÿˆÿÆÿňÿňÿňÿĈÿĈÿÈÿĉÿĉÿÅŠÿÅŠÿĉÿÄŠÿÄŠÿÄŠÿÄŠÿĈÿĈÿĈÿĈÿĈÿĈÿňÿňÿĉÿĉÿĉÿÈÿƉÿƉÿʼnÿÆŠÿÅŠÿÅŠÿÅŠÿÆ‹ÿÅ‹ÿÅ‹ÿÅŒÿÅŒÿÅŒÿÅŒÿÅŒÿÅŒÿÄŒÿÄŒÿÄ‹ÿÊÿÄŠÿÄŠÿÅŠÿÅŠÿÅŠÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÇŒÿÇŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆÿÆÿÆÿÆÿÆÿÆÿÅÿÅÿÆÿÅŽÿÅŽÿÅŽÿÅŽÿÆÿÆÿÆÿÆÿÆÿÅŽÿÆÿÆÿÆÿÆÿÆÿÆÿÅŒÿÆÿÈÿÊ‘ÿÍ”ÿÓ—ÿÓ—ÿÖ™ÿØ›ÿØ›ÿÔ—ÿÎ’ÿÉÿËÿÇ‹ÿÇ‹ÿÇ‹ÿÆŠÿÊŽÿΑÿÓ–ÿÖšÿÛŸÿÞ¡ÿÝ ÿÛ›ÿÚšÿÛ—ÿדÿÛ“ÿÛ“ÿÚ‘ÿÙÿÙŽÿÙŽÿÙŒÿ׊ÿ׊ÿ׊ÿÚŠÿ؈ÿØÿ׎ÿÓÿÏ‹ÿÍŒÿʉÿÉŠÿÉŠÿÉŠÿȉÿÉŠÿÉŠÿɉÿɉÿÉ‹ÿÉ‹ÿÉ‹ÿÉ‹ÿÇŒÿÇŒÿÉ‹ÿÉ‹ÿÉ‹ÿÊŒÿÊŒÿÊŒÿÊŒÿÍÿÍÿÍÿΉÿΉÿψÿψÿψÿψÿÎŒÿˉÿÉ‹ÿÈŠÿÆ‹þÈüÊüÊüÊüÊüÊüÊû‰üÊüÊüÊüÊû‰û‰û‰û‰û‰û‰û‰û‰û‰úÁˆúÁˆúÁˆúÁˆúÁˆúÁˆøˆøˆøˆ÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡öÀ†öÀ†õ¿…öÀ†öÀ†öÀ†öÀ†ø¿†÷¾…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…ó½ƒó½ƒó½ƒó½ƒó½ƒò¼€ò¼€ò¼€ò¼€ò¼€ñ»ñ»ñ»ñ»ñ»ñ»ðº€ðº€ðº€ðº€ðº€ðº€ðº€ðº€ðº€ðº€ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~î¸~ì¶|ì¶|ì¶|ì¶|ì¶|ëµ{ëµ{ì¶|ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{ê´zê´zê´zê´zê´zê´zé³yé³yé³yè²xé³yé³yé³yè²xè²xè²xé³yè²xç±wç±wç±wç±wç±wå¯uå¯uå¯uå¯wä®vå¯uå¯uä®tã­sä®tå¯uæ­tä«ræ­rä«pâ­ná¬mâ­nã®oã®oä¯pä®rå¯sæ²vè´xè´zæ²xá¯vÚ¨oÓ¡jÒ iÍœhÈ—cÈ—eɘfÍœmÇ–g¼‹^³‚U¤vHa3S'tH_5Q'B91 -%" -Û¥kÛ¥kÛ¥kÛ¥kÛ¥kܦlܦlܦlܦlݧmݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªnÞªnÞªnß«oß«oß«oß«oß«oß«oà¬pà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qâ®râ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯så±uå±uå±uå±uå±uå±uå±uæ²væ²væ²vç³wç³wç³wç³wç³wè´xè´xç³wç³wè´xè´xè´xè´xè´xéµyëµyëµyëµyëµyëµyëµyëµyëµyëµyëµyì¶zì¶zì¶zì¶zì¶zì¶zê¶zì¸|ì¸~ì¸~ì¸~í¹í¹í¹í¹í¹í¹í¹í¹í¹í¹í¹ðº€ðº€ðº€ðº€ðº€ðº€ðº€ñ»ñ»ñ»ñ»ñ»ò¼‚ò¼‚ò¼‚ò¼‚ò¼€ò¼€ò¼€ò¼€ò¼€ó½ó½ó½õ¿ƒõ¿ƒõ¿ƒöÀ„öÀ„öÀ„öÀ„÷Á…÷Á…÷Á…÷Á…÷Á…ùÁƒûÃ…ÿƈÿƈÿƆÿƆÿÆ…ÿȇÿÊ„ÿ͇ÿÎ…ÿÎ…ÿÎ…ÿÍ„ÿÌ€ÿÌ€ÿÉÿÉÿȃÿÇ‚ÿÈ„ÿǃÿȇÿȇÿȇÿȇÿÈ„ÿÉ…ÿɃÿɃÿÊ„ÿ̆ÿ̆ÿ͇ÿÎ…ÿφÿÏ„ÿÐ…ÿÏ…ÿÏ…ÿχÿΆÿΊÿ͉ÿ̈ÿÉ…ÿLjÿLjÿƉÿƉÿÅ‹ÿÄŠÿƉÿƉÿƉÿƉÿÆŠÿʼnÿĉÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÄŠÿÄŠÿÄŠÿÄŠÿĈÿĈÿĈÿĈÿĈÿĈÿňÿňÿĉÿĉÿĉÿĉÿƉÿƉÿʼnÿÆŠÿÅŠÿÅŠÿÅŠÿÅŠÿÄŠÿÅ‹ÿÄ‹ÿÄ‹ÿÅŒÿÅŒÿÅŒÿÅŒÿÄŒÿÄŒÿÄ‹ÿÊÿÄŠÿÄŠÿÅŠÿÅŠÿÅŠÿÅŠÿÆ‹ÿÅŠÿÅŠÿÆ‹ÿÆ‹ÿÆ‹ÿÅ‹ÿÅ‹ÿÅ‹ÿÅ‹ÿÆŒÿÆŒÿÅŒÿÆÿÆÿÆÿÆÿÆÿÅÿÄŽÿÄÿÄÿÄÿÅŽÿÅŽÿÅŽÿÅŽÿÅŽÿÅŽÿÅŽÿÅŽÿÅŽÿÆÿÆÿÅŒÿÅŒÿÅŒÿÅŒÿÆÿÆÿÈÿÉÿÊŽÿÉÿÍÿÍÿΑÿÏ’ÿДÿÎ’ÿÎ’ÿДÿÑ•ÿÑ•ÿÑ•ÿДÿÓ–ÿÖ™ÿÜ ÿÜ ÿÛžÿØ›ÿÕ•ÿÕ•ÿדÿÖ’ÿÖŽÿ×ÿÖÿ׎ÿ׌ÿ׌ÿ׊ÿÖ‰ÿÖ‰ÿ׊ÿ؈ÿ؈ÿÙÿ׎ÿÓÿΊÿÌ‹ÿɈÿÉŠÿÉŠÿÉŠÿȉÿȉÿȉÿɉÿɉÿÉ‹ÿÉ‹ÿÉ‹ÿÉ‹ÿÇŒÿÇŒÿÉ‹ÿÉ‹ÿÉ‹ÿÊŒÿÊŒÿÊŒÿÊŒÿËÿÎŽÿÎŽÿ͈ÿΉÿ·ÿ·ÿψÿψÿÎŒÿˉÿÉ‹ÿljÿÆ‹þÈüÊüÊüÊüÊüÊüÊüÊüÊüÊüÊüÊüÊû‰û‰û‰û‰û‰û‰û‰û‰û‰û‰úÁˆúÁˆúÁˆúÁˆøˆøˆøˆøˆøˆøˆøˆøˆøˆ÷Á‡÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†ø¿†ø¿†öÀ†õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…ó½ƒó½ƒó½ò¼€ò¼€ò¼€ò¼€ò¼€ñ»ñ»ñ»ñ»ñ»ñ»ñ»ñ»ñ»ñ»ðº€ðº€ðº€ðº€ðº€ðº€ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~ì¶|ì¶|ì¶|ì¶|ì¶|ì¶|ì¶|ì¶|ì¶|ì¶|ëµ{ëµ{ëµ{ëµ{ê´zê´zê´zê´zê´zê´zê´zé³yê´zê´zê´zé³yé³yé³yé³yè²xè²xé³yè²xé³yé³yè²xè²xè²xå¯uè²xè²zç±yè²xè²xç±wç±wå¯uä®tå¬så¬sæ­rç®så°qä¯pã®oã®oä¯pá¬màªnàªnß«oß«oß«qà¬rà®uá¯vä²{ä²{Þ­yÞ­yà¯}â±ீޭ~Ü«~ݬÛ­Ñ£uÅ™m¾’fª€Z–lF‚[;uN.fA(X3H(? 4-" - -Û¥kÛ¥kÛ¥kÛ¥kÛ¥kܦlܦlܦlܦlݧmݧkݧkݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªnß«oß«oß«oß«oß«oß«oà¬pà¬pà¬pà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qâ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯så±uå±uå±uå±uå±uæ²væ²væ²væ²vç³wç³wè´xè´xè´xè´xè´xè´xç³wè´xè´xè´xè´xéµyéµyéµyëµyëµyëµyëµyëµyëµyëµyëµyëµyëµyì¶zì¶zì¶zì¶zì¶zì¶zê¶zì¸|ì¸~ì¸~ì¸~í¹í¹í¹í¹í¹í¹í¹í¹ðº€ðº€ðº€ðº€ðº€ðº€ðº€ñ»ñ»ñ»ñ»ñ»ò¼‚ò¼‚ò¼‚ò¼‚ò¼€ò¼€ó½ó½ó½ó½ó½õ¿ƒõ¿ƒõ¿ƒöÀ„öÀ„öÀ„öÀ„öÀ„÷Á…÷Á…÷Á…úÁ†û‡ýÃ…þĆÿƆÿƆÿÆ„ÿÇ…ÿÈ„ÿÉ…ÿË‚ÿÎ…ÿ΃ÿ΃ÿ΃ÿÍ‚ÿÍ‚ÿÍ‚ÿɃÿɃÿÉ…ÿÈ„ÿȇÿdžÿȇÿȇÿȇÿȇÿÈ„ÿÉ…ÿɃÿÊ„ÿ̆ÿ̆ÿ̆ÿ͇ÿÎ…ÿφÿÏ„ÿÐ…ÿÐ…ÿÐ…ÿЇÿφÿ͉ÿ͉ÿ̈ÿ̈ÿÉŠÿȉÿȉÿȉÿÇŠÿÇŠÿÇŠÿÇŠÿÆŠÿÆŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÅŠÿÅŠÿÅŠÿÅŠÿÆŠÿÆŠÿÇŠÿÇŠÿÇŠÿƉÿÅŠÿÅŠÿÅŠÿÅŠÿÄŠÿÅ‹ÿÅ‹ÿÅ‹ÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅ‹ÿÄŠÿňÿňÿÆŠÿÆŠÿÆŠÿÆŠÿÆ‹ÿÅŠÿÅŠÿÆ‹ÿÆ‹ÿÆ‹ÿÅ‹ÿÅ‹ÿÅ‹ÿÅ‹ÿÅ‹ÿÅ‹ÿÄ‹ÿÅŒÿÆÿÆÿÆÿÆÿÆÿÅŽÿÅÿÅÿÅÿÅÿÅÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÅŽÿÅŽÿÅŽÿÅŽÿÆŒÿÆŒÿÆŒÿÅ‹ÿÆŒÿÇÿÆÿÅŒÿÅŽÿÅŽÿÅŽÿÅŽÿÇ‘ÿÈ’ÿÉ’ÿÈ‘ÿÈ‘ÿÈ‘ÿÉ“ÿÊ”ÿÍ•ÿИÿИÿИÿКÿΘÿÌ–ÿÌ–ÿÌ’ÿÍ“ÿÑ’ÿÓ”ÿÓŒÿÔÿÔ‰ÿÕŠÿÖ‹ÿÕŠÿÕŠÿÕŠÿÖÿÖÿ׉ÿØŠÿÙÿØÿÓÿ͉ÿÊ‹ÿȉÿÉŒÿÉŒÿÉŒÿÈ‹ÿÈ‹ÿÈ‹ÿÈŠÿÉ‹ÿÉ‹ÿÉ‹ÿÈŠÿÉ‹ÿÈÿÈÿÉ‹ÿÉ‹ÿÊŒÿÊŒÿÊŒÿËÿËÿËÿÍÿÍÿ͈ÿΉÿ·ÿψÿψÿψÿÏÿÍÿÉŽÿÇŒÿÆþÊüÊüÊüÊû‰üÊüÊüÊüÊüÊüÊüÊüÊüÊüÊüÊüÊüÊû‰û‰û‰û‰û‰û‰û‰û‰û‰ùÉøˆøˆøˆøˆøˆøˆøˆøˆ÷Á‡÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†ø¿†ø¿†öÀ†öÀ†õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…ó½ƒó½ƒó½ò¼€ò¼€ò¼€ò¼€ò¼€ñ»ñ»ñ»ò¼€ò¼‚ñ»ñ»ñ»ñ»ñ»ñ»ðº€ðº€ðº€ðº€ðº€ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~ì¶|ì¶|ì¶|ì¶|î¸~î¸~ì¶|ì¶|ì¶|ì¶|ì¶|ëµ{ëµ{ëµ{ê´zê´zëµ{ëµ{ëµ{ê´zê´zê´zê´zê´zê´zê´zé³yé³yé³yé³yé³yé³yé³yé³yè²xè²xè²xè²xè²xè²xè²zè²zè²xè²xè²xç±wå¯uä®tä®rä®rä®tå¯uæ­rå¬qæ­ræ­ræ­ræ­rå¬qãªoãªoãªoâ©nà§lÞ¨lá«oß«oà¬pà¬tá­uà¬tà¬tà¬tà¬tà¬tá­uá­sâ®tâ°uä²wÞ¬uÕ£lÌ™gÈ•c»Š[«zKoC_3xK"h;]3Q'@9. '% ܦlܦlܦlܦlܦlܦlÛ¥kܦlܦlݧmݧkݧkݧkÞ¨lÞ¨lÞ¨lÞ¨lÞ¨làªnàªnàªnàªnàªná«oß«oß«oß«oß«oß«oà¬pà¬pà¬pà¬pà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qâ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯så±uå±uå±uå±uå±uæ²væ²væ²væ²vç³wç³wè´xè´xè´xè´xè´xè´xè´xè´xè´xè´xè´xéµyéµyéµyëµyëµyëµyëµyëµyì¶zì¶zì¶zì¶zëµyì¶zî¸|ì¶zî¸|î¸|î¸|ì¸|ì¸|í¹í¹í¹í¹í¹í¹ðº€ñ»ñ»ñ»ñ»ñ»ñ»ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ó½ƒó½ƒó½ó½ó½ó½ó½ó½ó½õ¿ƒõ¿ƒõ¿ƒöÀ„öÀ„öÀ„öÀ„öÀ„÷Á…ø†ø†úÁ†û‡ýÃ…þĆÿƆÿƆÿÇ…ÿɇÿÉ…ÿÉ…ÿË‚ÿÎ…ÿ΃ÿ΃ÿ΃ÿÍ‚ÿÍ‚ÿÍ‚ÿɃÿɃÿÉ…ÿÈ„ÿȇÿdžÿȇÿȇÿȇÿȇÿÈ„ÿÉ…ÿɃÿÊ„ÿ̆ÿ͇ÿ͇ÿ͇ÿφÿφÿÐ…ÿÐ…ÿÐ…ÿцÿшÿЇÿΊÿΊÿΊÿ̈ÿÊ‹ÿÉŠÿȉÿȉÿÈ‹ÿÇŠÿÇŠÿÇŠÿʼnÿÆŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÄŠÿÅŠÿÅŠÿÅŠÿÅŠÿÆŠÿÆŠÿÇŠÿÇŠÿÇŠÿÇŠÿÅŠÿÅŠÿÅŠÿÅŠÿÄŠÿÅ‹ÿÅ‹ÿÅ‹ÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅ‹ÿÄŠÿňÿňÿÆŠÿÆŠÿÆŠÿÆŠÿÆ‹ÿÅŠÿÅŠÿÆ‹ÿÆ‹ÿÆ‹ÿÅ‹ÿÅ‹ÿÆŒÿÆŒÿÆŒÿÆŒÿÅŒÿÆÿÆÿÆÿÆÿÆÿÆÿÅŽÿÅÿÅÿÅÿÅÿÅÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÅŽÿÅŽÿÅŽÿÅŽÿÆŒÿÅ‹ÿÆŒÿÆŒÿÆŒÿÆŒÿÅŒÿÅŒÿÅŽÿÅŽÿÅŽÿÅŽÿÃÿÄŽÿÅŽÿÅŽÿÅŽÿÅŽÿÇ‘ÿÉ“ÿÍ•ÿÍ•ÿÍ•ÿÌ”ÿÌ–ÿÌ–ÿÊ”ÿÊ”ÿÊÿÌ’ÿÏÿÒ“ÿÔÿÔÿÔ‰ÿÕŠÿÕŠÿÕŠÿÕŠÿÕŠÿÕŒÿÖÿØŠÿ׉ÿ׎ÿ׎ÿÓÿΊÿÉŠÿȉÿÈ‹ÿÈ‹ÿÈ‹ÿÈ‹ÿÈ‹ÿÈ‹ÿÉ‹ÿÉ‹ÿÉ‹ÿÉ‹ÿÉ‹ÿÉ‹ÿÈÿÈÿÉ‹ÿÉ‹ÿÊŒÿÊŒÿÊŒÿËÿËÿËÿÍÿÍÿ͈ÿΉÿ·ÿ·ÿψÿψÿÏÿÍÿÉŽÿÈÿÈÿÆþÅŒþÅŒþÅŒüÊþÅŒþÅŒþÅŒþÅŒþÅŒþÅŒþÅŒüÊüÊüÊüÊüÊû‰üÊüÊüÊüÊû‰û‰û‰û‰û‰ùÉùÉùÉùÉùÉøˆøˆøˆøˆ÷Á‡÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†ø¿†ø¿†öÀ†öÀ†õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…ó½ƒó½ƒò¼€ó½ó½ó½ò¼€ñ»ò¼€ò¼€ñ»ò¼€ò¼‚ñ»ñ»ñ»ñ»ñ»ñ»ñ»ñ»ðº€ðº€ðº€ðº€ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~î¸~î¸~î¸~î¸~î¸~î¸~î¸~î¸~î¸~ì¶|ì¶|ì¶|ì¶|ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{ê´zê´zê´zëµ{ê´zê´zé³yê´zê´zê´zé³yê´zé³yè²xé³yé³yè²xç±wè²xè²zè²zè²xè²xè²xè²xè²xç±wå¯sç±uç±wç±wç®så¬qå¬qæ­rç®sæ­rå¬qå¬qä«pä«pä«pâ©nàªnàªnß«oà¬pà¬tá­uß«sß«sß«sß«sÞªrÞªrÞªpÞªpÚ¨mÝ«pß­vß­v߬z߬zݬ}Ü«|Ö¨|ÍŸs•l³†] vRfBU6lB#_:!T/E":, $ܦlܦlܦlܦlܦlܦlܦlܦlݧmݧmݧkݧkܨlܨlÞ¨làªnàªnàªnàªnàªnàªnàªná«oá«oß«oß«oß«oß«oß«oà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qâ®râ®râ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯så±uå±uå±uå±uå±uæ²væ²væ²væ²vç³wç³wè´xè´xè´xè´xè´xè´xè´xè´xè´xè´xè´xéµyéµyéµyëµyëµyì¶zì¶zì¶zì¶zì¶zì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¸|î¸|ì¸|í¹}í¹í¹í¹í¹í¹í¹ñ»ñ»ñ»ñ»ñ»ñ»ñ»ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ó½ƒó½ƒó½ƒó½ƒó½ó½ó½ò¼€ó½ó½ó½õ¿ƒõ¿ƒõ¿ƒöÀ„öÀ„öÀ„÷Á…÷Á…ø†ø†ùÇû‡üÈþĆþĆÿƆÿƆÿÇ…ÿɇÿ̈ÿ̈ÿÍ„ÿÎ…ÿ΃ÿÍ‚ÿÍ‚ÿÍ‚ÿÍ„ÿÍ„ÿɃÿɃÿÉ…ÿÈ„ÿȇÿdžÿȇÿȇÿȇÿȇÿÈ„ÿÉ…ÿÉ…ÿʆÿ̆ÿ͇ÿΈÿΈÿφÿЇÿÐ…ÿÐ…ÿцÿцÿЇÿЇÿωÿΈÿΈÿ͇ÿÌ‹ÿɈÿÉŠÿÉŠÿÉŠÿȉÿȉÿLjÿÆŠÿÆŠÿÅŠÿÅŠÿÅŠÿÅŠÿĉÿÅŠÿÅŠÿÅŠÿÄŠÿÄŠÿÄŠÿÄŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÆŠÿÆŠÿÇŠÿÇŠÿÇŠÿÈ‹ÿÇ‹ÿÇ‹ÿÆ‹ÿÆ‹ÿÅ‹ÿÅ‹ÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÅ‹ÿÄŠÿÅŠÿÅŠÿÆŠÿÆŠÿÆŠÿÆŠÿÆ‹ÿÅŠÿÅŠÿÆ‹ÿÆ‹ÿÆ‹ÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÇÿÇÿÇÿÆÿÆÿÇÿÇÿÆ‘ÿÅÿÇÿÇÿÆÿÆÿÆÿÇÿÆÿÆÿÆÿÆÿÆÿÆÿÅŽÿÅŽÿÅŽÿÄÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÅŽÿÅŽÿÅŽÿÅŽÿÅŽÿÄÿÆŽÿÆŽÿÆŽÿÆŽÿÉ‘ÿÊ’ÿÊ’ÿË“ÿË“ÿÊ’ÿÊ”ÿÊ”ÿÉ‘ÿÉ‘ÿÉÿÊÿÍŽÿÒ“ÿÔÿÔÿÕŠÿÕŠÿÕŠÿÕŠÿÕŠÿÔ‰ÿÔ‰ÿÔ‰ÿÖˆÿ׉ÿ׌ÿ׌ÿÔŽÿωÿÈŽÿÇÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÇŒÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÊŒÿÊŒÿÊŒÿÊŒÿËÿËÿË‹ÿÍÿÍ‹ÿÍ‹ÿÎŒÿÎŒÿÎŒÿÎŒÿÎŽÿÍÿÉŽÿÈÿÈÿÇŽþÅŒþÅŒþÅŒüÊþÅŒþÅŒþÅŒþÅŒþÅŒþÅŒþÅŒþÅŒüÊþÅŒþÅŒþÅŒþÅŒþÅŒüÊüÊüÊüÊû‰üÊüÊüÊúÄŠúÄŠúÄŠúÄŠùÉùÉùÉùÉøˆøˆøˆ÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡ùÀ‡ùÀ‡÷Á‡öÀ†öÀ†öÀ†õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…ò¼‚ó½ƒó½ó½ó½ƒó½ƒó½ƒó½ƒò¼‚ò¼‚ñ»ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ñ»ñ»ñ»ðº€ðº€ðº€ðº€ï¹ï¹ï¹ï¹î¸~î¸~ï¹ï¹ï¹î¸~î¸~î¸~î¸~î¸~î¸~ì¶|ì¶|ì¶|ì¶|ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{ê´zëµ{ëµ{ê´zê´zê´zê´zê´zê´zê´zê´zê´zê´zê´zé³yé³yè²xè²xç±wè²xç±wé³yè²xè²xè²xè²xç±wè²xç±wç±wå¯så¯så¯sç±ué°uç®sç®sæ­rä®râ¬pä«rä«rå¬qå¬qã­qâ¬pâ¬pâ¬pá«qá«qàªpàªpÞªpÞªpܨpܨpÛ§oÛ§oÛ§oÚ¦nÛ§oÚ¦nÚ¦nÚ¦nÚ¦pÞªtÞ«w߬xÛ¨tÐiÏœhÂ[°}K m;Ža8xK"b9W. -ܦlܦlܦlܦlܦlܦlܦlܦlݧmݧmݧkݧkܨlܨlÞ¨làªnàªnàªnàªnàªnàªnàªná«oá«oß«oß«oß«oß«oß«oà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qâ®râ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯så±uå±uå±uå±uå±uå±uæ²væ²væ²væ²væ²væ²væ²vç³wç³wç³wç³wè´xè´xè´xè´xè´xè´xè´xè´xè´xè´xè´xéµyéµyéµyëµyëµyëµyì¶zì¶zì¶zì¶zì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¸|î¸|ì¸|í¹}í¹í¹í¹í¹í¹í¹ñ»ñ»ñ»ñ»ñ»ñ»ñ»ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ó½ƒó½ƒó½ƒó½ƒó½ó½ó½ó½õ¿ƒõ¿ƒõ¿ƒöÀ„öÀ„öÀ„÷Á…÷Á…÷Á…÷Á…÷Á…÷Á…÷Á…ùÇüÈüÈþĆÿƈÿƆÿƆÿȆÿʈÿ̈ÿ̈ÿÍ„ÿÍ„ÿ΃ÿ΃ÿÍ‚ÿÍ‚ÿÍ„ÿÍ„ÿÊ„ÿÊ„ÿÈ„ÿÈ„ÿȇÿdžÿȇÿȇÿɈÿɈÿÉ…ÿÉ…ÿÉ…ÿʆÿ͇ÿ͇ÿ͇ÿΈÿφÿЇÿÐ…ÿцÿцÿцÿшÿшÿωÿΈÿΈÿ͇ÿÌ‹ÿɈÿÉŠÿÉŠÿÉŠÿÉŠÿȉÿLjÿÆŠÿÆŠÿĉÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÄŠÿÄŠÿÄŠÿÄŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÆ‹ÿÆ‹ÿÅŠÿÅŠÿÆŠÿÆŠÿÇŠÿÇŠÿÇŠÿÈ‹ÿÇ‹ÿÇ‹ÿÆ‹ÿÆ‹ÿÅ‹ÿÅ‹ÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÅ‹ÿÄŠÿÆ‹ÿÆ‹ÿÇ‹ÿÇ‹ÿÇ‹ÿÇ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÇŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÇÿÇÿÇÿÆÿÆÿÇÿÇÿÆ‘ÿÅÿÇÿÇÿÇÿÆÿÆÿÇÿÆÿÆÿÆÿÆÿÆÿÆÿÅŽÿÅŽÿÅŽÿÄÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÅŽÿÅŽÿÅŽÿÅŽÿÅŽÿÄÿÆŽÿÆŽÿÆŽÿÆŽÿÇÿÇÿÇÿÉ‘ÿÉ‘ÿÉ‘ÿÉ“ÿÉ“ÿÉ‘ÿÇÿÇÿÉÿÌÿÑ’ÿÓŒÿÕŽÿÕŠÿÕŠÿÕŠÿÕŠÿÕŠÿÔ‰ÿÔ‰ÿÕŠÿÖˆÿÖˆÿÖ‹ÿ׌ÿÔŽÿΈÿÈŽÿÇÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÊŒÿÊŒÿÊŒÿÊŒÿÊŒÿÊŒÿÊŠÿË‹ÿˉÿʈÿˉÿˉÿÍ‹ÿÍ‹ÿÍÿÍÿÉŽÿÈÿÈÿÇŽþÅŒþÅŒþÅŒþÅŒþÅŒþÅŒþÅŒþÅŒÿÆÿÆþÅŒþÅŒÿÆÿÆþÅŒÿÆÿÆþÅŒþÅŒþÅŒþÅŒüÊþÅŒþÅŒüÊüÊúÄŠúÄŠúÄŠúÄŠúÄŠùÉùÉùÉùÉøˆøˆøˆ÷Á‡÷Á‡÷Á‡÷Á‡ùÀ‡ùÀ‡÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†öÀ†öÀ†öÀ†õ¿…õ¿…õ¿…õ¿…õ¿…ó½ó½ó½ƒó½ƒó½ƒó½ƒó½ƒò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ñ»ñ»ñ»ñ»ðº€ðº€ðº€ðº€ï¹ï¹ï¹ï¹ï¹ï¹ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~ì¶|ì¶|ì¶|ì¶|ì¶|ì¶|ì¶|ì¶|ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{ê´zê´zê´zé³yê´zê´zê´zê´zê´zê´zé³yé³yé³yé³yé³yê´zê´zê´zé³yé³yé³yé³yè²xè²xè²xç±wç±uç±uç±uç±ué°ué°ué°uç®så¯sä®ræ­tå¬sæ­ræ­rä®rã­qâ¬pâ¬pá«qá«qàªpàªpÞªpÞªpܨpܨpÛ§oÚ¦nÛ§oÛ§oÚ¦nÚ¦nÚ¦nÚ¦nÚ¦pÙ¥oÙ¦rÙ¦rÚ§sÛ¨tÚ§sÙ¦rØ¥sÏœjÈ›r¸‹btP‡^:ܦlܦlܦlܦlܦlܦlܦlܦlݧmݧmݧmÞ¨nܨlܨlܨlÞªnàªnàªnàªnàªná«oá«oá«oá«oß«oß«oß«oà¬pà¬pà¬pà¬pá­qá­qá­qá­qâ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯sç±uç±uç±uç±uç±uç±uç±uç±uç±uè²vè²vè²vè²vè²vé³wé³wé³wé³wè´xè´xè´xè´xè´xè´xéµyéµyè´xéµyéµyéµyéµyéµyéµyéµyì¶zì¶zì¶zì¶zì¶zì¶zì¶zì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¸|î¸|ì¸|í¹}í¹í¹í¹í¹í¹í¹ñ»ñ»ñ»ñ»ñ»ñ»ñ»ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒõ¿…õ¿…õ¿…õ¿…õ¿…öÀ†öÀ†öÀ†÷Á…÷Á…÷Á…÷Á…÷Á…ø†ø†ùÇùÄ…ùÄ…þƈþƈþÄ„ÿLJÿɇÿɇÿˆÿ͈ÿÊ„ÿ̆ÿÍ„ÿÍ„ÿÍ„ÿÍ„ÿÍ„ÿÍ„ÿÊ„ÿÊ„ÿÉ…ÿÉ…ÿɈÿɈÿȇÿɈÿɈÿɈÿÉ…ÿʆÿʆÿ̈ÿ͇ÿΈÿΈÿΈÿωÿωÿшÿшÿЊÿЊÿЊÿЊÿЊÿωÿωÿΈÿ͉ÿ̈ÿɈÿɈÿɈÿɈÿÉŠÿȉÿȉÿLjÿÆŠÿÇ‹ÿÇ‹ÿÇ‹ÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÆ‹ÿÆ‹ÿÆŠÿÆŠÿÆŠÿÇ‹ÿÇ‹ÿÆŠÿÅŒÿÆÿÅÿÅÿÅÿÅÿÅ‹ÿÅ‹ÿÆŒÿÆŒÿÅ‹ÿÆŒÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÆ‹ÿÅÿÅÿÆÿÆÿÇÿÇÿÆÿÆÿÆÿÆÿÅÿÆŽÿÇŒÿÇŒÿÇÿÇÿÆŒÿÇÿÆÿÆÿÆÿÆÿÆÿÆÿÇÿÆŽÿÈŽÿÈŽÿÈŽÿÇÿÇÿÇÿÆÿÆÿÅÿÆŽÿÆŽÿÆŽÿÅŽÿÅŽÿÅŽÿÅŽÿÅŽÿÅŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇŽÿÆÿÈŽÿÈŽÿÇŽÿÇŽÿÇÿÇÿÆÿÇ‘ÿÇ‘ÿÇ‘ÿÉ‘ÿÉ‘ÿÆÿÆÿÆÿÆÿÈÿÊÿÌÿΑÿÑÿÓÿÕ‹ÿÕ‹ÿÔˆÿÕ‰ÿÕ‰ÿÖŠÿÖŠÿ׋ÿØ‹ÿ׊ÿÖ‹ÿÖ‹ÿÔÿψÿËÿÉ‹ÿÈÿÇŒÿÇŒÿÇŒÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÉŽÿÉŽÿÉŽÿÈÿÈÿÉŽÿÉŽÿÊŒÿÊŒÿÊŒÿÊŒÿË‹ÿË‹ÿË‹ÿË‹ÿÍÿÍÿÍÿÍÿÍÿÍÿÉŽÿÈÿÈÿÆþÅŒþÅŒþÅŒÿÆÿÆÿÆÿÆÿÆÿÆÿÆþÅŒÿÆÿÆÿÆÿÆþÅŒÿÆÿÆÿÆÿÆÿÆÿÆþÅŒþÅŒüÆŒüÆŒþÅŒüÊüÊüÊüÊû‰û‰û‰û‰úÁˆúÁˆúÁˆùÀ‡ùÀ‡ùÀ‡ùÀ‡÷Á‡÷Á‡÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†öÀ†öÀ†öÀ†öÀ†öÀ†öÀ†õ¿…õ¿…õ¿…õ¿…õ¿…ó½ƒó½ƒó½ƒó½ƒó½ƒò¼‚ó½ƒó½ƒó½ƒó½ƒó½ƒò¼‚ò¼‚ò¼‚ñ»ñ»ñ»ñ»ðº€ðº€ðº€ðº€ï¹ï¹ï¹ï¹ï¹ðº€ðº€ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~ì¶|ì¶|ëµ{ëµ{ì¶|ì¶|ì¶|ì¶|ì¶|ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{ëµ{ëµ}ê´|é³{ê´|ê´|ê´|ê´|ê´|ê´zê´zè²xê´zê´zê´zëµ{ê´zé³yê´zê´zé³yè²xé³yç³yæ²xæ²xã¯uå¯uå¯uç±wè²xç±yç±yç±yå¯wä®vã­uä®vä®vâ¬tã­uâ¬pá«oß«oÞªnܨlÞªnÞ¨nÞ¨nݧmݧmÛ§mÛ§mÚ¦nÛ§oÚ¦nÚ¦nÚ¦pÚ¦pÙ¥oÚ¦pؤnؤnÚ¦nÚ¦n×¥nئoØ¥qØ¥qئoОgÅ•^µ…NܦlܦlܦlܦlܦlܦlܦlܦlݧmݧmݧmÞ¨nܨlܨlܨlÞªnàªnàªnàªná«oá«oá«oá«oá«oß«oß«oà¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qâ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯sç±uç±uç±uç±uç±uç±uç±uç±uç±uè²vè²vè²vè²vè²vé³wé³wé³wé³wè´xè´xè´xéµyéµyéµyéµyéµyè´xéµyéµyéµyéµyê¶zê¶zê¶zì¶zì¶zì¶zì¶zì¶zî¸|î¸|î¸|î¸|ì¶zî¸|î¸|ï¹}ï¹}ï¹}ï¹}í¹}ì¸|í¹ï»ï»ï»ï»ñ»ñ»ñ»ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒõ¿…õ¿…ó½ƒó½ƒõ¿…õ¿…õ¿…õ¿…õ¿…öÀ†öÀ†öÀ†÷Á…÷Á…÷Á…÷Á…ø†ùÇùÇùÇùÄ…úņÿljÿljÿLJÿɉÿˉÿˉÿˆÿ͈ÿ̆ÿ̆ÿÍ„ÿÍ„ÿÍ„ÿÍ„ÿÎ…ÿÎ…ÿÊ„ÿÊ„ÿʆÿÉ…ÿɈÿɈÿȇÿɈÿɈÿɈÿÉ…ÿʆÿʆÿ̈ÿ͇ÿΈÿΈÿωÿωÿЊÿшÿшÿЊÿЊÿЊÿЊÿЊÿЊÿωÿωÿΊÿ̈ÿʉÿʉÿʉÿʉÿÉŠÿÉŠÿÉŠÿÉŠÿÈŒÿÇ‹ÿÇ‹ÿÇ‹ÿÆ‹ÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÇ‹ÿÇ‹ÿÇ‹ÿÇ‹ÿÇ‹ÿÇ‹ÿÆÿÇŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÆŒÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÆŽÿÆŽÿÆÿÆÿÇÿÇÿÆÿÆÿÆÿÆÿÅÿÅÿÆ‹ÿÈÿÇÿÇÿÆŒÿÇÿÆÿÆÿÆÿÆÿÆÿÆÿÇÿÆŽÿÈŽÿÈŽÿÈŽÿÇÿÇÿÇÿÆÿÆÿÅÿÆŽÿÆŽÿÆŽÿÅŽÿÅŽÿÅŽÿÅŽÿÅŽÿÅŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇŽÿÆÿÈŽÿÈŽÿÇŽÿÇŽÿÇÿÈ‘ÿÇ‘ÿÇ‘ÿÇ‘ÿÉ“ÿÉ‘ÿÉ‘ÿÈ‘ÿÈ‘ÿÈ‘ÿÈ‘ÿÊÿÊÿÌÿΑÿÑÿÓÿÖŒÿÖŒÿÖŠÿÖŠÿÖŠÿÖŠÿÖŠÿÖŠÿÙŒÿÙŒÿ׌ÿÖ‹ÿÔÿψÿËÿÉ‹ÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÈÿÈÿÈÿÈÿÉŽÿÉŽÿÉŽÿÈÿÉŽÿÉŽÿÉŽÿÊŒÿÊŒÿÊŒÿÊŒÿË‹ÿË‹ÿË‹ÿË‹ÿË‹ÿË‹ÿÍÿÍÿÍÿÍÿÉŽÿÇŒÿÇŽÿÇŽÿÆÿÆÿÆÿÆÿÆÿÆÿÆÿÆÿÇŽÿÇŽÿÆÿÇŽÿÇŽÿÇŽÿÇŽÿÇŽÿÇŽÿÇŽÿÆÿÆÿÆÿÆÿÆÿÆýÇýÇþÅŒþÅŒþÅŒüÊüÊüÊüÊüÊû‰û‰û‰úÁˆúÁˆúÁˆúÁˆúÁˆøˆ÷Á‡÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†öÀ†õ¿…õ¿…õ¿…õ¿…õ¿…ó½ƒó½ƒó½ƒò¼‚ó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒò¼‚ò¼‚ò¼‚ò¼‚ñ»ñ»ñ»ñ»ñ»ðº€ðº€ðº€ðº€ï¹ï¹ðº€ðº€ï¹ï¹ï¹ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~ì¶|ì¶|ì¶|ì¶|ì¶|ì¶|ì¶|ëµ{ì¶|ì¶|ëµ{ëµ{ëµ{ëµ}ëµ}ê´|ê´|ëµ}ê´|ê´|ê´|ê´zê´zê´zê´zê´zëµ{ëµ{ëµ{ëµ{ê´zê´zé³yé³yé³yç³yæ²xæ²xã¯uå¯uå¯uç±wç±wç±yç±yç±yç±yç±yå¯wå¯wä®vä®vä®vã­qã­qà¬pß«oÞªnÞªnÞ¨nÞ¨nÞ¨nÞ¨nÛ§mÛ§mÛ§oܨpÛ§oÛ§oÛ§qÛ§qÚ¦pÚ¦pÚ¦pÙ¥oÙ¥mÙ¥mÖ¤m×¥nÖ£oÖ£oÓ¡jÏfÇ—`Ó\ܦlܦlܦlܦlܦlܦlܦlܦlݧmݧmݧmÞ¨nܨlܨlܨlÞªnÞªnÞªnß«oß«oß«oß«oß«oß«oß«oß«oà¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qâ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯sç±uç±uç±uç±uç±uç±uç±uç±uç±uè²vè²vè²vè²vè²vé³wé³wé³wé³wè´xè´xè´xéµyéµyéµyéµyéµyè´xéµyéµyéµyéµyê¶zê¶zê¶zì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¸|î¸|î¸|î¸|ï¹}ï¹}ï¹}ï¹}ðº~îº~îº~í¹ï»ï»ï»ï»ï»ï»ï»ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒõ¿…õ¿…õ¿…õ¿…õ¿…õ¿…ó½ƒõ¿…õ¿…öÀ†öÀ†öÀ†öÀ†÷Á…÷Á…ø†ø†úÁ†û‡û‡û‡û‡þÅŠÿÈŠÿÈŠÿȉÿÊ‹ÿʉÿʉÿʆÿ̈ÿ̆ÿ̆ÿÍ„ÿÍ„ÿÍ„ÿÍ„ÿ̆ÿ̆ÿʆÿÉ…ÿÉ…ÿÉ…ÿɈÿɈÿɈÿɈÿɈÿɈÿÉ…ÿʆÿ̈ÿ͉ÿ͇ÿΈÿЇÿЇÿωÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿωÿωÿΈÿ͇ÿ͉ÿ͉ÿ͉ÿʆÿʉÿʉÿÈŠÿÈŠÿÈŒÿÇ‹ÿÇ‹ÿÇ‹ÿÇ‹ÿÆŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÅŠÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÇ‹ÿÇ‹ÿÈ‹ÿÈ‹ÿÆÿÆÿÆÿÇŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆÿÆÿÆÿÆÿÆŽÿÅÿÅÿÆŽÿÆŽÿÆŽÿÈÿÈÿÇÿÇÿÇÿÇÿÆÿÆÿÆÿÆÿÆÿÆÿÇÿÆŽÿÈŽÿÈŽÿÈŽÿÇÿÇÿÇÿÇÿÇÿÆÿÇŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇŽÿÇŽÿÈŽÿÇÿÈŽÿÈŽÿÈŽÿÈŽÿÈÿÉ‘ÿÈ‘ÿÈ‘ÿÇ‘ÿÉ“ÿÉ‘ÿÉ‘ÿÉ‘ÿÉ‘ÿÊÿÊÿÊÿÊÿÌÿΑÿÑÿÓ‘ÿÖŽÿÖŽÿÕ‹ÿÖŒÿÖŒÿÖŒÿÖŒÿÖŒÿØÿÙŽÿØÿÖÿÕ“ÿÏÿÉŽÿÈÿÇŒÿÈÿÈÿÈÿÈÿÈÿÈÿÉŽÿÉŽÿÉŽÿÉŽÿÉŽÿÉÿÉÿÉÿÉÿÉŽÿÉŽÿÉŽÿÉŽÿÉŽÿÉŽÿËÿËÿËÿËÿËÿËÿÍÿÍÿÍÿÍÿÉŽÿÈÿÈÿÈÿÆÿÆÿÆÿÆÿÆÿÆÿÆÿÇŽÿÇŽÿÇŽÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÇŽÿÈÿÈÿÈÿÈÿÉýÇÿÆÿÆÿÆþÅŒþÅŒþÅŒþÅŒþÅŒþÅŒüÊüÊüÊû‰û‰û‰û‰øˆøˆøˆ÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†öÀ†õ¿…õ¿…õ¿…õ¿…õ¿…ó½ƒó½ƒò¼‚ó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒò¼‚ò¼‚ò¼‚ò¼‚ñ»ñ»ñ»ñ»ñ»ñ»ðº€ðº€ðº€ðº€ðº€ðº€ðº€ðº€ðº€ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~î¸~î¸~î¸~ì¶|ì¶|ì¶|ì¶|ì¶|ì¶|ì¶|ëµ{ëµ{ëµ}ëµ}ëµ}ëµ}ëµ}ëµ}ëµ}ëµ}ê´zê´zê´zé³yê´zëµ{ëµ{ëµ{ê´zé³yé³yé³yé³yé³yç³yæ²xæ²zã¯wå±yå±yç±yç±yå±yã¯wã¯wã¯wã¯wã¯wã¯uã¯uå¯uä®tâ®râ®rá­sß«qß«qß«qÞ¨nÞ¨nÞ¨nÞ¨nÞ¨nݧmÞ¨nàªpàªpÞ¨nܨpܨpÛ§oÛ§oÛ§oÚ¦nÚ¦nÚ¦nÛ¥kÛ¥kÚ¤jÚ¤j×£iÓŸeÒždÐœbܦlܦlܦlܦlܦlܦlܦlݧmݧmݧmÞ¨nÞ¨nܨlÞªnÞªnÞªnÞªnÞªnß«oß«oß«oß«oß«oß«oß«oß«oà¬pà¬pà¬pà¬pá­qá­qâ®râ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯så±uå±uå±uç±uç±uç±uç±uç±uè²vè²vè²vè²vè²vé³wé³wé³wé³wé³wé³wê´xê´xè´xè´xè´xéµyéµyéµyéµyéµyè´xéµyéµyéµyéµyê¶zê¶zê¶zì¶zì¶zì¶zî¸|î¸|î¸|î¸|î¸|î¸|î¸|ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}í¹}îº~ï»îº€ï»ï»ï»ï»îº€ï»ï»ï»ï»ï»ï»ï»ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒõ¿…õ¿…õ¿…õ¿…õ¿…öÀ†öÀ†öÀ†öÀ†öÀ†÷Á‡÷Á‡÷Á‡ø†ø†ø†ùÇû‡û‡û‡þÅŠþÅŠþÅŠÿÈŠÿÉ‹ÿȉÿÊ‹ÿʉÿʉÿÉ…ÿʆÿÊ„ÿÊ„ÿÍ„ÿÍ„ÿÍ„ÿÍ„ÿ̆ÿ̆ÿ̈ÿÉ…ÿÉ…ÿÉ…ÿɈÿȇÿɈÿɈÿɈÿʉÿÉ…ÿ̈ÿ̈ÿΊÿΈÿωÿЇÿЇÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿÏ‹ÿÏ‹ÿÏ‹ÿ͉ÿÌ‹ÿÌ‹ÿÉ‹ÿÈŠÿÈŒÿÈŒÿÈŒÿÈŒÿÇ‹ÿÇ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÆ‹ÿÇŒÿÇŒÿÇ‹ÿÇ‹ÿÈ‹ÿÈ‹ÿÆÿÆÿÆÿÆÿÆŽÿÆŽÿÆŽÿÆŽÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇŽÿÆÿÆÿÇŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇÿÇÿÈÿÈÿÇÿÇÿÇÿÇÿÈ‘ÿÈ‘ÿÆÿÆÿÆÿÆÿÇÿÆŽÿÈŽÿÈŽÿÈŽÿÇÿÇÿÇÿÇÿÇÿÆÿÇŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇŽÿÇŽÿÈŽÿÇÿÈŽÿÈŽÿÈŽÿÈŽÿÈÿÉ‘ÿÈ‘ÿÈ‘ÿÇ‘ÿÉ“ÿÉ‘ÿÉ‘ÿÉ‘ÿÉ‘ÿÊÿÊÿÊÿËÿÍÿÏ’ÿÒÿÔ’ÿÖŽÿÖŽÿÖŒÿÖŒÿÖŒÿÖŒÿÖŒÿÖŒÿØÿÙŽÿ׎ÿÔ‹ÿÒÿÏÿÉŽÿÈÿÇŒÿÈÿÈÿÈÿÈÿÈÿÈÿÉŽÿÉŽÿÉŽÿÉŽÿÉŽÿÉÿÉÿÉÿÉÿÉŽÿÉŽÿÉŽÿÉŽÿÉŽÿÉŽÿÊŒÿÉ‹ÿÊŒÿÊŒÿËÿËÿÍÿÍÿÍÿËÿÈÿÈÿÈÿÈÿÇŽÿÇŽÿÆÿÇŽÿÇŽÿÇŽÿÇŽÿÈÿÈÿÈÿÇŽÿÇŽÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÇŽÿÈÿÈÿÈÿÈÿÉþÈŽÿÇŽÿÆÿÆÿÆÿÆÿÆÿÆþÅŒþÅŒþÅŒüÊüÊüÊüÊüÊüÊúÄŠúÄŠùÉùÉùÉùÉùÉùÉøˆøˆøˆ÷Á‡÷Á‡÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†öÀ†öÀ†öÀ†õ¿…õ¿…õ¿…õ¿…õ¿…ó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ñ»ñ»ñ»ñ»ðº€ðº€ñ»ñ»ðº€ðº€ðº€ðº€ï¹ï¹ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~î¸~î¸~î¸~ì¶|ì¶|ì¶|ì¶|ì¶|ì¶|ëµ{ì¶~ì¶~ëµ}ëµ}ëµ}ëµ}ëµ}ëµ}ëµ{ëµ{ê´zê´zê´zê´zëµ{ê´zê´zê´zê´zê´zê´zé³yç³yç³yæ²zæ²zå±yå±yç±yç±yã¯wã¯wã¯wã¯wã¯wã¯wã¯uã¯uå¯uä®tâ®râ®rá­sß«qß«qß«qàªpàªpàªpàªpàªpàªpàªpàªpÞ¨nÞ¨nܨpܨpܨpܨpܨpܨpÚ¦nÚ¦nܦlܦlÛ¥kÚ¤jؤj×£i×£iÖ¢hܦlܦlܦlܦlܦlݧmݧmݧmݧmÞ¨nÞ¨nÞ¨nܨlÞªnÞªnß«oß«oß«oß«oß«oß«oß«oß«oà¬pà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qâ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯så±uå±uå±uè²vè²vè²vè²vè²vè²vè²vè²vè²vé³wé³wé³wé³wé³wé³wé³wê´xê´xè´xéµyéµyéµyéµyéµyê¶zê¶zéµyéµyê¶zê¶zê¶zê¶zê¶zê¶zî¸|î¸|î¸|î¸|î¸|î¸|î¸|ï¹}ï¹}î¸|ï¹}ï¹}ï¹}ï¹}ï¹}ðº~îº~îº~ï»ï»ï»ï»ï»ï»ð¼‚ð¼‚ï»ï»ï»ï»ï»ï»ò¼‚ò¼‚ò¼‚ò¼‚ó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒõ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…öÀ†öÀ†öÀ†öÀ†öÀ†÷Á‡÷Á‡÷Á‡ø†ø†øÄùÄ…ûÃ…üĆüĆüĆÿljÿÈŠÿȈÿÊŠÿɈÿɈÿÉ…ÿʆÿÊ„ÿÊ„ÿÊ„ÿÊ„ÿÊ„ÿ̆ÿÍ„ÿË‚ÿÊ„ÿÊ„ÿʆÿʆÿʆÿʆÿÉ…ÿÉ…ÿÉ…ÿÉ…ÿʆÿʆÿ̈ÿ̈ÿ͉ÿΊÿωÿωÿЇÿЇÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿÏ‹ÿÏ‹ÿÌŒÿË‹ÿËÿÉ‹ÿÈŒÿÈŒÿÈŒÿÈŒÿÈŒÿÇ‹ÿÆÿÆÿÅÿÅÿÅÿÅÿÅÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇŽÿÇŽÿÆÿÆÿÆÿÇŽÿÆŽÿÅÿÅÿÅÿÆŽÿÆŽÿÆŽÿÆŽÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇŽÿÇŽÿÇŽÿÇŽÿÇÿÆŽÿÆŽÿÇÿÇÿÇÿÈÿÈÿÇÿÉÿÉÿÇÿÉ‘ÿÉ‘ÿÇÿÇÿÇÿÇÿÇÿÆŽÿÈŽÿÈŽÿÈŽÿÇÿÇÿÈŽÿÈŽÿÈŽÿÇŽÿÈÿÇÿÇÿÇÿÇÿÈÿÈÿÇŽÿÇŽÿÈŽÿÈŽÿÇŽÿÇŽÿÈŽÿÈŽÿÈŽÿÈŽÿÉŽÿÉŽÿÉŽÿÉŽÿÉÿÊ‘ÿÉ‘ÿÉ‘ÿÉ‘ÿË“ÿÉ‘ÿÉ‘ÿÉ‘ÿÉ‘ÿÉÿÉÿÉÿÊÿÍÿÏ’ÿÑ’ÿÓ”ÿÖ‘ÿÖ‘ÿÖŽÿÖŽÿÖŽÿÖŽÿÕ’ÿÕ’ÿבÿבÿÕ‘ÿÓÿÑÿÏÿÊÿÈÿÇŒÿÈÿÈÿÈÿÈÿÈÿÈÿÉŽÿÉŽÿÉŽÿÉŽÿÊÿÊ‘ÿÊ‘ÿÉÿÉÿÉŽÿÉŽÿÉŽÿÉŽÿÉŽÿÉŽÿÊŒÿÊŒÿÊŒÿËÿËÿËÿËÿËÿÊÿÊÿÉÿÈÿÈÿÈÿÇŽÿÇŽÿÇŽÿÇŽÿÇŽÿÈÿÈÿÈÿÈÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÈÿÈÿÇŽÿÈÿÈÿÉÿÉÿÊÿÉÿÈÿÇŽÿÇŽÿÆÿÆÿÆÿÆÿÆÿÆþÅŒþÅŒþÅŒüÊüÊüÊüÊúÄŠúÄŠúÄŠúÄŠúÄŠùÉùÉùÉùÉøˆøˆøˆøˆ÷Á‡÷Á‡÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†öÀ†õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…ó½ƒó½ƒó½ƒó½ƒó½ƒò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ñ»ñ»ñ»ñ»ðº€ðº€ñ»ñ»ñ»ðº€ðº€ðº€ï¹ï¹ï¹ï¹ï¹ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~î¸~ì¶|î¸~î¸~ì¶|îµ|îµ|îµ~îµ~îµ~í´}í´}í´}í´}í´}ëµ{ëµ{ê´zê´zê´zëµ{ëµ{ëµ{ëµ{ê´zê´zê´zê´zé³yç³yç³yæ²zæ²zæ²zå±yå±yå±yå±yå±yå±yå±yä²yâ°wã¯wã¯wã¯uã¯uã¯uã¯uâ®tà¬râ¬râ¬rá«qá«qá«qàªpàªpàªpàªpàªpàªpàªpÞªpÞªpܨnܨnܨnܨnÛ§mÛ§mÛ§mÚ¦lÙ¥mÙ¥mÛ¥iÚ¤h×£gÔ dܦlܦlܦlܦlݧmݧmݧmݧmÞ¨nÞ¨nÞ¨nÞ¨nÞªnÞªnÞªnß«oß«oß«oß«oß«oß«oß«oà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qâ®râ®râ®rã¯sã¯sã¯sã¯sã¯sã¯sã¯sã¯så±uå±uå±uè²vè²vè²vè²vè²vè²vè²vè²vè²vé³wé³wé³wé³wé³wé³wê´xê´xê´xéµyéµyéµyéµyéµyéµyê¶zê¶zéµyê¶zê¶zê¶zê¶zê¶zê¶zì¸|î¸|î¸|î¸|î¸|î¸|î¸|ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ðº~ðº~îº~ï»ï»ï»ï»ï»ï»ï»ð¼‚ð¼‚ï»ï»ð¼‚ð¼‚ð¼‚ð¼‚ò¼‚ò¼‚ò¼‚ó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒõ¿…õ¿…õ¿…öÀ†öÀ†öÀ†öÀ†öÀ†õ¿…öÀ†öÀ†÷Á‡÷Á‡÷Á‡÷Á‡ø†ø†øÄùÄ…ûÃ…üĆüĆüĆÿljÿÉ‹ÿË‹ÿË‹ÿʉÿʉÿʆÿʆÿÊ„ÿÊ„ÿÊ„ÿÊ„ÿÊ„ÿ̆ÿÍ„ÿË‚ÿÊ„ÿÊ„ÿʆÿʆÿʆÿʆÿÉ…ÿÉ…ÿÉ…ÿʆÿʆÿʆÿ̈ÿ͉ÿ͉ÿΊÿΈÿωÿЇÿЇÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿÏ‹ÿÏ‹ÿÎŽÿÌŒÿËÿÈŠÿÉÿÉÿÈŒÿÈŒÿÈŒÿÇ‹ÿÆÿÆÿÅÿÅÿÅÿÅÿÅÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇŽÿÇŽÿÇŽÿÇŽÿÇŽÿÇŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇŽÿÇŽÿÇŽÿÇŽÿÇÿÆŽÿÆŽÿÇÿÇÿÇÿÈÿÊÿÉÿÉÿÉÿÉÿÉ‘ÿÉ‘ÿÉ‘ÿÉ‘ÿÇÿÇÿÇÿÆŽÿÈŽÿÈŽÿÈŽÿÇÿÇÿÈŽÿÈŽÿÈŽÿÇŽÿÈÿÇÿÇÿÇÿÇÿÈÿÈÿÇŽÿÇŽÿÈŽÿÈŽÿÇŽÿÇŽÿÈŽÿÈŽÿÈŽÿÈŽÿÉŽÿÉŽÿÉŽÿÉŽÿÉÿÊ‘ÿÉ‘ÿÉ‘ÿÉ‘ÿË“ÿÉ‘ÿÉ‘ÿÉ‘ÿÉ‘ÿÉÿÊÿÊÿÊÿÍÿÏ’ÿÑ’ÿÓ”ÿÖ‘ÿÖ‘ÿÖŽÿÖŽÿÖŽÿÖŽÿÕ’ÿÔ‘ÿÕÿÕÿÔÿÑÿÏÿÎŒÿÌ‘ÿÈÿÇŒÿÈÿÈÿÈÿÈÿÈÿÈÿÉŽÿÉŽÿÉŽÿÉŽÿÊÿÊ‘ÿÊ‘ÿÉÿÊ‘ÿÊÿÊÿÊÿÊÿÊÿÊÿËÿËÿËÿËÿÍÿÍÿËÿËÿÊÿÊÿÉÿÈÿÈÿÈÿÇŽÿÇŽÿÇŽÿÈÿÈÿÇŽÿÈÿÈÿÈÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÈÿÊ‘ÿÌ“ÿÌ“ÿÍ“ÿÊÿÈÿÈÿÇŽÿÇŽÿÇŽÿÇŽÿÇŽÿÇŽÿÆÿÆÿÆþÅŒþÅŒþÅŒþÅŒþÅŒüÆŒüÆŒúÄŠúÄŠúÄŠùÉúÄŠúÄŠùÉùÉùÉùÉøˆøˆøˆ÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†öÀ†öÀ†õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…ó½ƒó½ƒó½ƒó½ƒò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ò¼‚ñ»ñ»ñ»ñ»ñ»ñ»ñ»ñ»ñ»ñ»ðº€ðº€ðº€ðº€ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~î¸~î¸~î¸~î¸~î¸~îµ|îµ|îµ~îµ~îµ~îµ~í´}í´}í´}í´}ëµ{ëµ{ê´zê´zê´zëµ{ê´zëµ{ëµ{ê´zê´zê´zê´zé³yç³yç³yæ²zæ²zæ²zæ²zå±yå±yå±yå±yå±yå±yä²yâ°wã¯wã¯wã¯uã¯uã¯uâ®tâ®tâ®tã­sã­sá«qá«qá«qàªpàªpàªpàªpàªpàªpàªpÞªpÞªpܨnܨnܨnܨnÛ§mÛ§mÛ§mÚ¦lÚ¦nÚ¦nÛ¥iÚ¤h×£gÖ¢fܦlܦlܦlܦlݧmݧmݧmݧmÞ¨nÞ¨nÞ¨nàªpÞªnÞªnÞªnß«oß«oß«oß«oß«oß«oß«oà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qá­qâ®râ®râ®rã¯sã¯sã¯sã¯så±uå±uå±uå±uå±uæ²væ²vè²vè²vè²vè²vè²vè²vé³wé³wé³wé³wé³wé³wé³wê´xê´xê´xê´xê´xéµyéµyéµyéµyéµyéµyê¶zê¶zéµyê¶zê¶zê¶zê¶zê¶zê¶zì¸|î¸|î¸|î¸|î¸|î¸|ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ðº~ðº~îº~ï»ï»ï»ð¼‚ð¼‚ð¼‚ð¼‚ð¼‚ð¼‚ï»ï»ð¼‚ð¼‚ð¼‚ð¼‚ò¼‚ò¼‚ò¼‚ó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒõ¿…õ¿…õ¿…öÀ†öÀ†öÀ†öÀ†öÀ†öÀ†÷Á‡÷Á‡÷Á‡øˆøˆøˆùÇùÇùÄ…ùÄ…üĆþƈþƈþƈÿÈŠÿËÿÍÿÍÿʉÿʉÿʆÿʆÿɃÿÊ„ÿÊ„ÿÊ„ÿ̆ÿ̆ÿÍ„ÿÍ„ÿ̆ÿ̆ÿ̈ÿ̈ÿʆÿʆÿʆÿÉ…ÿʆÿʆÿʆÿ̈ÿ̈ÿ͉ÿ͉ÿÏ‹ÿωÿωÿЇÿЇÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿÐŒÿÏ‹ÿÎŽÿÍÿËÿÉ‹ÿÉÿÉÿÉÿÉÿÈŒÿÈŒÿÇŽÿÆÿÆÿÆÿÆÿÆÿÆÿÇŽÿÇŽÿÇŽÿÇŽÿÇŽÿÈÿÈÿÈŽÿÈŽÿÇŽÿÇŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇÿÇÿÈÿÈÿÈÿÈÿÉ‘ÿÉ‘ÿÇÿÇÿÉ‘ÿÉ‘ÿÊÿÊÿÉÿÉÿÉÿÉÿÉ‘ÿÉ‘ÿÉ‘ÿÉ‘ÿÉ‘ÿÇÿÇÿÆŽÿÈŽÿÈŽÿÈŽÿÇÿÇÿÈŽÿÈŽÿÈŽÿÇŽÿÈÿÇÿÇÿÇÿÇÿÈÿÈÿÈŽÿÈŽÿÈŽÿÈŽÿÈŽÿÈŽÿÈŽÿÈŽÿÉŽÿÉŽÿÉŽÿÉŽÿÉŽÿÉŽÿÉÿÊ‘ÿÉ‘ÿÉ‘ÿÉ‘ÿË“ÿÊ‘ÿÊ‘ÿÉ‘ÿÊ’ÿËÿÌ‘ÿÌ‘ÿÌ‘ÿΑÿÏ’ÿÏÿÓ”ÿÖ‘ÿÖ‘ÿÖŽÿÖŽÿÖŽÿÖŽÿÕ•ÿÔ”ÿÕ‘ÿÕ‘ÿÔ“ÿÑÿÏÿÍÿÊÿÈÿÇŒÿÈÿÈÿÈÿÈÿÈÿÈÿÉŽÿÉŽÿÉŽÿÉŽÿÊÿÊ‘ÿÊ‘ÿÊ‘ÿÊ‘ÿÊÿÊÿÊÿÊÿÊÿÊÿÍÿÍÿÍÿÍÿÍÿÍÿÍÿÍÿÊÿÊÿÉÿÈÿÈÿÈÿÈÿÈÿÇŽÿÈÿÈÿÈÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÉÿÊ‘ÿÊ‘ÿÊ‘ÿÌ“ÿÌ“ÿÌ“ÿÍ“ÿÊÿÉÿÉÿÈÿÈÿÈÿÈÿÈÿÇŽÿÇŽÿÆÿÆþÅŒÿÆÿÆÿÆÿÆýÇüÆŒýÇýÇüÆŒúÄŠúÄŠúÄŠúÄŠúÄŠùÉùÉøˆøˆøˆ÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†öÀ†õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…ó½ƒó½ƒó½ƒó½ƒó½ƒò¼‚ò¼‚ò¼‚ò¼‚ñ»ñ»ò¼‚ò¼‚ñ»ñ»ñ»ñ»ñ»ðº€ðº€ðº€ðº€ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~î¸~î¸~î¸~î¸~î¸~ð·~îµ|îµ~îµ~îµ~îµ~îµ~í´}í´}í´}ëµ{ëµ{ê´zê´zê´zëµ{ê´zëµ{ëµ{ê´zê´zê´zè´zç³yç³yç³yå³zå³zå³zå³zä²yä²yä²yä²yä²{ä²{ä²{â°yã¯wã¯wã¯uã¯uã¯uâ®tâ®tâ®tã­sã­sá«qá«qá«qàªpàªpàªpàªpàªpàªpàªpÞªpÞªpÞªpÞªpÞªpÞªpÛ§mÛ§mÛ§mÚ¦lÙ¥mÙ¥mÞ¨lÛ¥iؤh×£gܦlܦlܦlܦlݧmݧmݧmݧmÞ¨nÞ¨nÞ¨nàªpÞªnÞªnÞªnß«oß«oß«oß«oß«oà¬pà¬pà¬pà¬pà¬pà¬pà¬pá­qá­qá­qá­qâ®râ®râ®râ®rã¯sã¯sã¯sã¯så±uå±uå±uå±uæ²væ²væ²vè²vè²vè²vè²vè²vé³wé³wé³wé³wé³wé³wé³wé³wê´xê´xê´xëµyëµyéµyéµyê¶zê¶zê¶zê¶zê¶zê¶zê¶zê¶zê¶zê¶zê¶zì¸|ì¸|ì¸|î¸|î¸|î¸|î¸|î¸|ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ï¹}ðº~ðº~îº~ï»ï»ï»ð¼‚ð¼‚ð¼‚ð¼‚ð¼‚ð¼‚ï»ï»ð¼‚ð¼‚ð¼‚ð¼‚ò¼‚ò¼‚ò¼‚ó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒó½ƒõ¿…õ¿…õ¿…öÀ†öÀ†öÀ†öÀ†÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡øˆøˆøˆùÇùÇùÄ…úņüĆþƈþƈþƈÿÈŠÿËÿÍÿÍÿʉÿʉÿʆÿʆÿɃÿÊ„ÿÊ„ÿÊ„ÿ̆ÿ̆ÿÍ„ÿÍ„ÿ̆ÿ̆ÿ̈ÿ̈ÿʆÿʆÿʆÿÉ…ÿʆÿʆÿʆÿ̈ÿ̈ÿ͉ÿΊÿÏ‹ÿωÿωÿЇÿЇÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿЊÿÐŒÿÐŒÿÏÿÏÿÍÿÌŽÿÉÿÉÿÉÿÉÿÈŒÿÈŒÿÇŽÿÆÿÆÿÆÿÆÿÆÿÆÿÇŽÿÇŽÿÇŽÿÇŽÿÇŽÿÈÿÈÿÈŽÿÈŽÿÇŽÿÇŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÇŒÿÆŽÿÆŽÿÆŽÿÆŽÿÆŽÿÇÿÇÿÇÿÈÿÈÿÈÿÈÿÉ‘ÿÇÿÇÿÉ‘ÿÉ‘ÿÉ‘ÿÊÿÊÿÉÿÉÿÉÿÉÿÉ‘ÿÉ‘ÿÉ‘ÿÉ‘ÿÉ‘ÿÇÿÇÿÆŽÿÈŽÿÈŽÿÈŽÿÇÿÇÿÈŽÿÈŽÿÈŽÿÇŽÿÈÿÇÿÇÿÇÿÇÿÈÿÈÿÈŽÿÈŽÿÈŽÿÈŽÿÈŽÿÈŽÿÈŽÿÈŽÿÉŽÿÉŽÿÉŽÿÉŽÿÉŽÿÉŽÿÉÿÊ‘ÿÉ‘ÿÉ‘ÿÉ‘ÿË“ÿÊ‘ÿÊ‘ÿÉ‘ÿÊ’ÿËÿÌ‘ÿÌ‘ÿÌ‘ÿΑÿÏ’ÿÏÿÓ”ÿÖ‘ÿÖ‘ÿÖŽÿÖŽÿÖŽÿÖŽÿÕ•ÿÕ•ÿÕ‘ÿÕ‘ÿÓ’ÿÐÿÏÿÍÿÊÿÈÿÇŒÿÈÿÈÿÈÿÈÿÈÿÈÿÉŽÿÉŽÿÉŽÿÉŽÿÊÿÊ‘ÿÊ‘ÿÊ‘ÿÊ‘ÿÊÿÊÿÊÿÊÿÊÿÊÿËÿËÿÍÿÍÿÍÿÍÿÍÿÍÿÊÿÊÿÉÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÈÿÉÿÉÿÉÿÉÿÉÿÉÿÈÿÉÿÉÿÉÿÉÿÉÿÉÿÊ‘ÿÊ‘ÿÊ‘ÿÌ“ÿÌ“ÿÌ“ÿÍ“ÿË‘ÿÉÿÉÿÈÿÈÿÈÿÈÿÈÿÇŽÿÇŽÿÇŽÿÆÿÆÿÆÿÆÿÆÿÆýÇýÇýÇýÇýÇüÆŒüÆŒüÆŒùÉùÉùÉùÉùÉùÉ÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡÷Á‡öÀ†öÀ†öÀ†öÀ†öÀ†õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…õ¿…ó½ƒó½ƒó½ƒó½ƒó½ƒò¼‚ò¼‚ò¼‚ò¼‚ñ»ñ»ò¼‚ò¼‚ñ»ñ»ñ»ñ»ñ»ðº€ðº€ðº€ðº€ï¹ï¹ï¹ï¹î¸~î¸~î¸~î¸~î¸~î¸~î¸~î¸~î¸~ð·~ð·~îµ~îµ~îµ~îµ~í´}í´}í´}í´}ëµ{ëµ{ê´zê´zê´zëµ{ê´zëµ{ëµ{ê´zê´zê´zè´zç³yç³yç³yå³zå³zå³zå³zä²yä²yä²yä²yä²{ä²{ä²{â°yã¯wã¯wã¯uã¯uã¯uâ®tâ®tâ®tã­sã­sá«qá«qá«qàªpàªpàªpàªpàªpàªpàªpÞªpÞªpÞªpÞªpÞªpÞªpÛ§mÛ§mÛ§mÚ¦lÙ¥mÙ¥mÞ¨lÛ¥iؤh×£g \ No newline at end of file diff --git a/tensorflow/contrib/ffmpeg/testdata/mono_10khz.ogg b/tensorflow/contrib/ffmpeg/testdata/mono_10khz.ogg deleted file mode 100644 index 8b1d9fea465..00000000000 Binary files a/tensorflow/contrib/ffmpeg/testdata/mono_10khz.ogg and /dev/null differ diff --git a/tensorflow/contrib/ffmpeg/testdata/mono_10khz.wav b/tensorflow/contrib/ffmpeg/testdata/mono_10khz.wav deleted file mode 100644 index 2a65e0e2559..00000000000 Binary files a/tensorflow/contrib/ffmpeg/testdata/mono_10khz.wav and /dev/null differ diff --git a/tensorflow/contrib/ffmpeg/testdata/mono_16khz.mp3 b/tensorflow/contrib/ffmpeg/testdata/mono_16khz.mp3 deleted file mode 100644 index 52a2b7d1f47..00000000000 Binary files a/tensorflow/contrib/ffmpeg/testdata/mono_16khz.mp3 and /dev/null differ diff --git a/tensorflow/contrib/ffmpeg/testdata/mono_16khz_mp3.mp4 b/tensorflow/contrib/ffmpeg/testdata/mono_16khz_mp3.mp4 deleted file mode 100644 index 424f4b6e1a7..00000000000 Binary files a/tensorflow/contrib/ffmpeg/testdata/mono_16khz_mp3.mp4 and /dev/null differ diff --git a/tensorflow/contrib/ffmpeg/testdata/mono_16khz_mp3_32khz_aac.mp4 b/tensorflow/contrib/ffmpeg/testdata/mono_16khz_mp3_32khz_aac.mp4 deleted file mode 100644 index 2485da86d60..00000000000 Binary files a/tensorflow/contrib/ffmpeg/testdata/mono_16khz_mp3_32khz_aac.mp4 and /dev/null differ diff --git a/tensorflow/contrib/ffmpeg/testdata/mono_32khz_aac.mp4 b/tensorflow/contrib/ffmpeg/testdata/mono_32khz_aac.mp4 deleted file mode 100644 index 6577e6f58af..00000000000 Binary files a/tensorflow/contrib/ffmpeg/testdata/mono_32khz_aac.mp4 and /dev/null differ diff --git a/tensorflow/contrib/ffmpeg/testdata/small.mp4 b/tensorflow/contrib/ffmpeg/testdata/small.mp4 deleted file mode 100644 index 1fc478842f5..00000000000 Binary files a/tensorflow/contrib/ffmpeg/testdata/small.mp4 and /dev/null differ diff --git a/tensorflow/contrib/ffmpeg/testdata/small_100.bmp b/tensorflow/contrib/ffmpeg/testdata/small_100.bmp deleted file mode 100644 index 61f53a2a21c..00000000000 Binary files a/tensorflow/contrib/ffmpeg/testdata/small_100.bmp and /dev/null differ diff --git a/tensorflow/contrib/ffmpeg/testdata/stereo_48khz.mp3 b/tensorflow/contrib/ffmpeg/testdata/stereo_48khz.mp3 deleted file mode 100644 index 0be1de48060..00000000000 Binary files a/tensorflow/contrib/ffmpeg/testdata/stereo_48khz.mp3 and /dev/null differ diff --git a/tensorflow/contrib/ffmpeg/testdata/stereo_48khz_aac.mp4 b/tensorflow/contrib/ffmpeg/testdata/stereo_48khz_aac.mp4 deleted file mode 100644 index bc71bf95e8e..00000000000 Binary files a/tensorflow/contrib/ffmpeg/testdata/stereo_48khz_aac.mp4 and /dev/null differ diff --git a/tensorflow/contrib/ffmpeg/testdata/stereo_48khz_mp3.mp4 b/tensorflow/contrib/ffmpeg/testdata/stereo_48khz_mp3.mp4 deleted file mode 100644 index 9f9b2072f80..00000000000 Binary files a/tensorflow/contrib/ffmpeg/testdata/stereo_48khz_mp3.mp4 and /dev/null differ diff --git a/tensorflow/contrib/ffmpeg/testdata/test_sound1.mp3 b/tensorflow/contrib/ffmpeg/testdata/test_sound1.mp3 deleted file mode 100644 index 51e8e2a3d2f..00000000000 Binary files a/tensorflow/contrib/ffmpeg/testdata/test_sound1.mp3 and /dev/null differ diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD deleted file mode 100644 index f04b970d156..00000000000 --- a/tensorflow/contrib/framework/BUILD +++ /dev/null @@ -1,306 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -load("//tensorflow:tensorflow.bzl", "py_test", "tf_custom_op_library", "tf_gen_op_libs", "tf_gen_op_wrapper_py", "tf_kernel_library") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = [ - "//learning/brain:__subpackages__", - "//tensorflow:__subpackages__", - "//tensorflow_model_optimization:__subpackages__", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_custom_op_py_library( - name = "framework_py", - srcs = [ - "__init__.py", - "python/framework/__init__.py", - "python/framework/checkpoint_utils.py", - "python/framework/experimental.py", - "python/framework/graph_util.py", - "python/framework/tensor_util.py", - "python/ops/__init__.py", - "python/ops/arg_scope.py", - "python/ops/audio_ops.py", - "python/ops/checkpoint_ops.py", - "python/ops/ops.py", - "python/ops/prettyprint_ops.py", - "python/ops/script_ops.py", - "python/ops/sort_ops.py", - "python/ops/variables.py", - ], - dso = [ - ":python/ops/_variable_ops.so", - ], - kernels = [ - ":variable_kernels", - ":variable_ops_op_lib", - ], - srcs_version = "PY2AND3", - visibility = [ - "//learning/brain:__subpackages__", - "//tensorflow:__subpackages__", - "//tensorflow_estimator:__subpackages__", - "//tensorflow_model_optimization:__subpackages__", - "//third_party/py/tf_slim:__subpackages__", - "//video/youtube/personalization:__subpackages__", - ], - deps = [ - ":gen_variable_ops", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:audio_ops_gen", - "//tensorflow/python:check_ops", - "//tensorflow/python:checkpoint_ops_gen", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:io_ops", - "//tensorflow/python:logging_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:pywrap_tensorflow", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:script_ops", - "//tensorflow/python:smart_cond", - "//tensorflow/python:sort_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:state_ops", - "//tensorflow/python:state_ops_gen", - "//tensorflow/python:tensor_array_ops", - "//tensorflow/python:tensor_util", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:function", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -tf_kernel_library( - name = "variable_kernels", - srcs = [ - "kernels/zero_initializer_op.cc", - "kernels/zero_initializer_op.h", - ], - gpu_srcs = [ - "kernels/zero_initializer_op_gpu.cu.cc", - "kernels/zero_initializer_op.h", - ], - deps = [ - "//tensorflow/core:framework", - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - ], - alwayslink = 1, -) - -tf_custom_op_library( - name = "python/ops/_variable_ops.so", - srcs = [ - "kernels/zero_initializer_op.cc", - "kernels/zero_initializer_op.h", - "ops/variable_ops.cc", - ], - gpu_srcs = [ - "kernels/zero_initializer_op_gpu.cu.cc", - "kernels/zero_initializer_op.h", - ], -) - -tf_gen_op_libs( - op_lib_names = [ - "variable_ops", - ], -) - -cc_library( - name = "all_ops", - deps = [ - ":variable_ops_op_lib", - ], -) - -tf_gen_op_wrapper_py( - name = "gen_variable_ops", - out = "python/ops/gen_variable_ops.py", - deps = [ - ":variable_ops_op_lib", - ], -) - -py_test( - name = "arg_scope_test", - size = "small", - srcs = ["python/ops/arg_scope_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":framework_py", - "//tensorflow/python:client_testlib", - ], -) - -py_test( - name = "checkpoint_utils_test", - size = "small", - srcs = ["python/framework/checkpoint_utils_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["manual"], # http://b/30468735 - deps = [ - ":framework_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:partitioned_variables", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "ops_test", - size = "small", - srcs = ["python/ops/ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":framework_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - ], -) - -py_test( - name = "prettyprint_ops_test", - size = "small", - srcs = ["python/ops/prettyprint_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":framework_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:tensor_array_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "experimental_test", - srcs = ["python/framework/experimental_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":framework_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:platform", - ], -) - -py_test( - name = "graph_util_test", - srcs = ["python/framework/graph_util_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":framework_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:platform", - ], -) - -py_test( - name = "tensor_util_test", - srcs = ["python/framework/tensor_util_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":framework_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "variables_test", - size = "small", - srcs = ["python/ops/variables_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["manual"], - deps = [ - ":framework_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:partitioned_variables", - "//tensorflow/python:platform", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -filegroup( - name = "checkpoint_ops_testdata", - srcs = [ - "testdata/bundle_checkpoint.data-00000-of-00001", - "testdata/bundle_checkpoint.index", - "testdata/bundle_checkpoint_vocab.txt", - "testdata/bundle_checkpoint_vocab_with_oov.txt", - "testdata/keyword.txt", - "testdata/keyword_new.txt", - ], -) - -py_test( - name = "checkpoint_ops_test", - size = "medium", - srcs = ["python/ops/checkpoint_ops_test.py"], - data = [":checkpoint_ops_testdata"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [ - ":framework_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:partitioned_variables", - "//tensorflow/python:platform", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/framework/README.md b/tensorflow/contrib/framework/README.md deleted file mode 100644 index 616606912ff..00000000000 --- a/tensorflow/contrib/framework/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# TensorFlow contrib framework. - -Common TensorFlow utilities, mostly used by tf.contrib.layers and -tf.contrib.losses. diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py deleted file mode 100644 index 063717f08aa..00000000000 --- a/tensorflow/contrib/framework/__init__.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Framework utilities. - -@@assert_same_float_dtype -@@assert_scalar -@@assert_scalar_int -@@convert_to_tensor_or_sparse_tensor -@@get_graph_from_inputs -@@is_numeric_tensor -@@is_non_decreasing -@@is_strictly_increasing -@@is_tensor -@@reduce_sum_n -@@remove_squeezable_dimensions -@@with_shape -@@with_same_shape - -@@deprecated -@@deprecated_args -@@deprecated_arg_values - -@@arg_scope -@@add_arg_scope -@@current_arg_scope -@@has_arg_scope -@@arg_scoped_arguments - -@@prepend_name_scope -@@strip_name_scope - -@@add_model_variable -@@assert_global_step -@@assert_or_get_global_step -@@assign_from_checkpoint -@@assign_from_checkpoint_fn -@@assign_from_values -@@assign_from_values_fn -@@create_global_step -@@filter_variables -@@fuse_op -@@get_global_step -@@get_or_create_global_step -@@get_local_variables -@@get_model_variables -@@get_name_scope -@@get_trainable_variables -@@get_unique_variable -@@get_variables_by_name -@@get_variables_by_suffix -@@get_variable_full_name -@@get_variables_to_restore -@@get_variables -@@global_variable -@@local_variable -@@model_variable -@@variable -@@VariableDeviceChooser -@@convolutional_delta_orthogonal -@@convolutional_orthogonal_1d -@@convolutional_orthogonal_2d -@@convolutional_orthogonal_3d -@@zero_initializer - -@@load_checkpoint -@@list_variables -@@load_variable -@@init_from_checkpoint -@@load_and_remap_matrix_initializer -@@load_embedding_initializer -@@load_linear_multiclass_bias_initializer -@@load_variable_slot_initializer - -@@argsort -@@py_func -@@sort - -@@get_placeholders - -@@smart_cond -@@smart_constant_value -@@smart_case - -@@BoundedTensorSpec -@@TensorSpec - -@@RecordInput -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.framework.python.framework import * -from tensorflow.contrib.framework.python.framework import nest -from tensorflow.contrib.framework.python.ops import * -# pylint: enable=unused-import,wildcard-import - -from tensorflow.python.framework.ops import prepend_name_scope -from tensorflow.python.framework.ops import strip_name_scope -from tensorflow.python.framework.smart_cond import smart_case -from tensorflow.python.framework.smart_cond import smart_cond -from tensorflow.python.framework.smart_cond import smart_constant_value -from tensorflow.python.framework.tensor_spec import BoundedTensorSpec -from tensorflow.python.framework.tensor_spec import TensorSpec -from tensorflow.python.ops.data_flow_ops import RecordInput -from tensorflow.python.ops.init_ops import convolutional_delta_orthogonal -from tensorflow.python.ops.init_ops import convolutional_orthogonal_1d -from tensorflow.python.ops.init_ops import convolutional_orthogonal_2d -from tensorflow.python.ops.init_ops import convolutional_orthogonal_3d -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = ['nest'] -_nest_allowed_symbols = [ - 'assert_same_structure', - 'is_nested', - 'is_sequence', - 'is_sequence_or_composite', - 'flatten', - 'flatten_dict_items', - 'pack_sequence_as', - 'map_structure', - 'map_structure_with_paths', - 'map_structure_with_tuple_paths', - 'assert_shallow_structure', - 'flatten_up_to', - 'flatten_with_tuple_paths_up_to', - 'map_structure_up_to', - 'map_structure_with_tuple_paths_up_to', - 'get_traverse_shallow_structure', - 'yield_flat_paths', - 'flatten_with_joined_string_paths', - 'flatten_with_tuple_paths', -] - -remove_undocumented(nest.__name__, allowed_exception_list=_nest_allowed_symbols) -remove_undocumented(__name__, allowed_exception_list=_allowed_symbols) diff --git a/tensorflow/contrib/framework/kernels/zero_initializer_op.cc b/tensorflow/contrib/framework/kernels/zero_initializer_op.cc deleted file mode 100644 index 3dd68d5ddf7..00000000000 --- a/tensorflow/contrib/framework/kernels/zero_initializer_op.cc +++ /dev/null @@ -1,158 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#define EIGEN_USE_THREADS - -#if GOOGLE_CUDA -#define EIGEN_USE_GPU -#endif // GOOGLE_CUDA - -#include "tensorflow/contrib/framework/kernels/zero_initializer_op.h" - -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/framework/resource_var.h" - -namespace tensorflow { - -using CPUDevice = Eigen::ThreadPoolDevice; -using GPUDevice = Eigen::GpuDevice; - -template -class ZeroInitializerOp : public OpKernel { - public: - explicit ZeroInitializerOp(OpKernelConstruction* ctx) : OpKernel(ctx) { - OP_REQUIRES(ctx, IsRefType(ctx->input_type(0)), - errors::InvalidArgument("input needs to be a ref type")); - } - - void Compute(OpKernelContext* ctx) override { - mutex_lock l(*ctx->input_ref_mutex(0)); - Tensor input = ctx->mutable_input(0, true); - OP_REQUIRES(ctx, !input.IsInitialized(), - errors::InvalidArgument("input is already initialized")); - AllocatorAttributes attr; - attr.set_gpu_compatible(true); - attr.set_nic_compatible(true); - PersistentTensor out_persistent; - Tensor* out_tensor = nullptr; - OP_REQUIRES_OK( - ctx, ctx->allocate_persistent(input.dtype(), input.shape(), - &out_persistent, &out_tensor, attr)); - functor::TensorSetZero()(ctx->eigen_device(), - out_tensor->flat()); - ctx->replace_ref_input(0, *out_tensor, true); - // we always return the input ref. - ctx->forward_ref_input_to_ref_output(0, 0); - } -}; - -#define REGISTER_KERNELS(D, T) \ - REGISTER_KERNEL_BUILDER( \ - Name("ZeroInitializer").Device(DEVICE_##D).TypeConstraint("T"), \ - ZeroInitializerOp); -#define REGISTER_CPU_KERNELS(T) REGISTER_KERNELS(CPU, T); -TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS); -#undef REGISTER_CPU_KERNELS - -#if GOOGLE_CUDA -namespace functor { -#define DECLARE_GPU_SPEC(T) \ - template <> \ - void TensorSetZero::operator()(const GPUDevice& d, \ - typename TTypes::Flat t); \ - extern template struct TensorSetZero; - -TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPEC); -#undef DECLARE_GPU_SPEC -} // namespace functor - -#define REGISTER_GPU_KERNELS(T) REGISTER_KERNELS(GPU, T); -TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); -#undef REGISTER_GPU_KERNELS -#endif // GOOGLE_CUDA - -#undef REGISTER_KERNELS - -template -class ZeroVarInitializer : public OpKernel { - public: - explicit ZeroVarInitializer(OpKernelConstruction* ctx) : OpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype_)); - OP_REQUIRES_OK(ctx, ctx->GetAttr("shape", &shape_)); - } - - void Compute(OpKernelContext* ctx) override { - core::RefCountPtr variable; - OP_REQUIRES_OK(ctx, LookupOrCreateResource( - ctx, HandleFromInput(ctx, 0), &variable, - [this, ctx](Var** var_ptr) { - *var_ptr = new Var(dtype_); - PersistentTensor unused; - Tensor* var_tensor = nullptr; - AllocatorAttributes attr; - attr.set_gpu_compatible(true); - attr.set_nic_compatible(true); - TF_RETURN_IF_ERROR(ctx->allocate_persistent( - dtype_, shape_, &unused, &var_tensor, attr)); - - functor::TensorSetZero()( - ctx->eigen_device(), - var_tensor->flat()); - - *(*var_ptr)->tensor() = *var_tensor; - - return Status::OK(); - })); - - mutex_lock ml(*variable->mu()); - - OP_REQUIRES(ctx, !variable->is_initialized, - errors::InvalidArgument("input is already initialized")); - - variable->is_initialized = true; - - Tensor* output = nullptr; - OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &output)); - output->scalar()() = HandleFromInput(ctx, 0); - } - - private: - DataType dtype_; - TensorShape shape_; -}; - -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER(Name("ZeroVarInitializer") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("dtype"), \ - ZeroVarInitializer); - -TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS); -#undef REGISTER_CPU_KERNELS - -#if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER(Name("ZeroVarInitializer") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("dtype") \ - .HostMemory("var"), \ - ZeroVarInitializer); - -TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); -#undef REGISTER_GPU_KERNELS -#endif // GOOGLE_CUDA - -} // namespace tensorflow diff --git a/tensorflow/contrib/framework/kernels/zero_initializer_op.h b/tensorflow/contrib/framework/kernels/zero_initializer_op.h deleted file mode 100644 index 99389a5ab6a..00000000000 --- a/tensorflow/contrib/framework/kernels/zero_initializer_op.h +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_FRAMEWORK_KERNELS_ZERO_INITIALIZER_OP_H_ -#define TENSORFLOW_CONTRIB_FRAMEWORK_KERNELS_ZERO_INITIALIZER_OP_H_ - -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/framework/tensor_types.h" - -namespace tensorflow { -namespace functor { -template -struct TensorSetZero { - void operator()(const Device& d, typename TTypes::Flat t) { - t.device(d) = t.constant(T(0)); - } -}; -} // namespace functor - -} // end namespace tensorflow -#endif // TENSORFLOW_CONTRIB_FRAMEWORK_KERNELS_ZERO_INITIALIZER_OP_H_ diff --git a/tensorflow/contrib/framework/kernels/zero_initializer_op_gpu.cu.cc b/tensorflow/contrib/framework/kernels/zero_initializer_op_gpu.cu.cc deleted file mode 100644 index ad2eabfd405..00000000000 --- a/tensorflow/contrib/framework/kernels/zero_initializer_op_gpu.cu.cc +++ /dev/null @@ -1,36 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#if GOOGLE_CUDA - -#define EIGEN_USE_GPU - -#include "tensorflow/contrib/framework/kernels/zero_initializer_op.h" - -#include "tensorflow/core/framework/register_types.h" - -namespace tensorflow { -namespace functor { - -using GPUDevice = Eigen::GpuDevice; - -#define DEFINE_GPU_SPECS(T) template struct TensorSetZero; -TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS); -#undef DEFINE_GPU_SPECS - -} // namespace functor -} // namespace tensorflow - -#endif // GOOGLE_CUDA diff --git a/tensorflow/contrib/framework/ops/variable_ops.cc b/tensorflow/contrib/framework/ops/variable_ops.cc deleted file mode 100644 index f6ee6cdb571..00000000000 --- a/tensorflow/contrib/framework/ops/variable_ops.cc +++ /dev/null @@ -1,71 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" - -namespace tensorflow { - -using shape_inference::InferenceContext; - -REGISTER_OP("ZeroInitializer") - .Input("ref: Ref(T)") - .Output("output_ref: Ref(T)") - .Attr("T: realnumbertype") - .SetAllowsUninitializedInput() - .SetShapeFn([](InferenceContext* c) { - c->set_output(0, c->input(0)); - return Status::OK(); - }) - .Doc(R"doc( -Initialize 'ref' with all zeros. This op requires that the tensor is not -initialized. The tensor will first be allocated memory, then be filled with all -zeros. This op is intended to save memory during initialization, -if you use this op, you should not run initializer of the 'ref' tensor. - -ref: Should be from a `Variable` node. -output_ref:= Same as "ref". -)doc"); - -REGISTER_OP("ZeroVarInitializer") - .Input("var: resource") - .Output("output_var: resource") - .Attr("dtype: type") - .Attr("shape: shape") - .SetAllowsUninitializedInput() - .SetShapeFn([](InferenceContext* c) { - c->set_output(0, c->Scalar()); - DataType t; - TF_RETURN_IF_ERROR(c->GetAttr("dtype", &t)); - PartialTensorShape p; - TF_RETURN_IF_ERROR(c->GetAttr("shape", &p)); - shape_inference::ShapeHandle s; - TF_RETURN_IF_ERROR(c->MakeShapeFromPartialTensorShape(p, &s)); - c->set_output_handle_shapes_and_types( - 0, std::vector{{s, t}}); - - return Status::OK(); - }) - .Doc(R"doc( -Initialize 'var' with all zeros. This op requires that the resource var is not -initialized. The var will first be allocated memory, then be filled with all -zeros. This op is intended to save memory during initialization, -if you use this op, you should not run initializer of the var. - -var: Should be a ResourceVariable. -output_var:= Same as "var". -)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/framework/python/framework/__init__.py b/tensorflow/contrib/framework/python/framework/__init__.py deleted file mode 100644 index 2d49771ab75..00000000000 --- a/tensorflow/contrib/framework/python/framework/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A module containing TensorFlow ops whose API may change in the future.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.framework.python.framework.checkpoint_utils import * -from tensorflow.contrib.framework.python.framework.experimental import experimental -from tensorflow.contrib.framework.python.framework.graph_util import * -from tensorflow.contrib.framework.python.framework.tensor_util import * -# pylint: enable=wildcard-import -from tensorflow.python.util import decorator_utils -from tensorflow.python.util import nest -from tensorflow.python.util.deprecation import deprecated -from tensorflow.python.util.deprecation import deprecated_arg_values -from tensorflow.python.util.deprecation import deprecated_args diff --git a/tensorflow/contrib/framework/python/framework/checkpoint_utils.py b/tensorflow/contrib/framework/python/framework/checkpoint_utils.py deleted file mode 100644 index 811df7a55ae..00000000000 --- a/tensorflow/contrib/framework/python/framework/checkpoint_utils.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tools to work with checkpoints.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import six - -from tensorflow.python.framework import ops -from tensorflow.python.ops import io_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.ops import variables -from tensorflow.python.platform import gfile -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import checkpoint_management -from tensorflow.python.training import training as train - -__all__ = [ - "load_checkpoint", - "load_variable", - "list_variables", - "init_from_checkpoint"] - - -def _get_checkpoint_filename(filepattern): - """Returns checkpoint filename given directory or specific filepattern.""" - if gfile.IsDirectory(filepattern): - return checkpoint_management.latest_checkpoint(filepattern) - return filepattern - - -def load_checkpoint(filepattern): - """Returns CheckpointReader for latest checkpoint. - - Args: - filepattern: Directory with checkpoints file or path to checkpoint. - - Returns: - `CheckpointReader` object. - - Raises: - ValueError: if checkpoint_dir doesn't have 'checkpoint' file or checkpoints. - """ - filename = _get_checkpoint_filename(filepattern) - if filename is None: - raise ValueError("Couldn't find 'checkpoint' file or checkpoints in " - "given directory %s" % filepattern) - return train.NewCheckpointReader(filename) - - -def load_variable(checkpoint_dir, name): - """Returns a Tensor with the contents of the given variable in the checkpoint. - - Args: - checkpoint_dir: Directory with checkpoints file or path to checkpoint. - name: Name of the tensor to return. - - Returns: - `Tensor` object. - """ - # TODO(b/29227106): Fix this in the right place and remove this. - if name.endswith(":0"): - name = name[:-2] - reader = load_checkpoint(checkpoint_dir) - return reader.get_tensor(name) - - -def list_variables(checkpoint_dir): - """Returns list of all variables in the latest checkpoint. - - Args: - checkpoint_dir: Directory with checkpoints file or path to checkpoint. - - Returns: - List of tuples `(name, shape)`. - """ - reader = load_checkpoint(checkpoint_dir) - variable_map = reader.get_variable_to_shape_map() - names = sorted(variable_map.keys()) - result = [] - for name in names: - result.append((name, variable_map[name])) - return result - - -# pylint: disable=protected-access -# Currently variable_scope doesn't provide very good APIs to access -# all variables under scope and retrieve and check existing scopes. -# TODO(ipolosukhin): Refactor variable_scope module to provide nicer APIs. - - -def _set_checkpoint_initializer(variable, file_pattern, tensor_name, slice_spec, - name="checkpoint_initializer"): - """Sets variable initializer to assign op form value in checkpoint's tensor. - - Args: - variable: `Variable` object. - file_pattern: string, where to load checkpoints from. - tensor_name: Name of the `Tensor` to load from checkpoint reader. - slice_spec: Slice specification for loading partitioned variables. - name: Name of the operation. - """ - base_type = variable.dtype.base_dtype - with ops.device(variable.device), ops.device("/cpu:0"): - restore_op = io_ops.restore_v2( - file_pattern, [tensor_name], [slice_spec], [base_type], name=name)[0] - variable._initializer_op = state_ops.assign(variable, restore_op) - - -def _set_variable_or_list_initializer(variable_or_list, file_pattern, - tensor_name): - if isinstance(variable_or_list, (list, tuple)): - # A set of slices. - slice_name = None - for v in variable_or_list: - if slice_name is None: - slice_name = v._save_slice_info.full_name - elif slice_name != v._save_slice_info.full_name: - raise ValueError("Slices must all be from the same tensor: %s != %s" % - (slice_name, v._save_slice_info.full_name)) - _set_checkpoint_initializer(v, file_pattern, tensor_name, - v._save_slice_info.spec) - else: - _set_checkpoint_initializer(variable_or_list, file_pattern, tensor_name, "") - - -def _collect_partitioned_variable(name, var_scope): - if name + "/part_0" in var_scope._vars: - var = [] - i = 0 - while name + "/part_%d" % i in var_scope._vars: - var.append(var_scope._vars[name + "/part_%d" % i]) - i += 1 - return var - return None - - -def init_from_checkpoint(checkpoint_dir, assignment_map): - """Using assignment map initializes current variables with loaded tensors. - - Note: This overrides default initialization ops of specified variables and - redefines dtype. - - Assignment map supports following syntax: - - * `'checkpoint_scope_name/': 'scope_name/'` - will load all variables in - current `scope_name` from `checkpoint_scope_name` with matching variable - names. - * `'checkpoint_scope_name/some_other_variable': 'scope_name/variable_name'` - - will initialize `scope_name/variable_name` variable - from `checkpoint_scope_name/some_other_variable`. - * `'scope_variable_name': variable` - will initialize given `tf.Variable` - object with variable from the checkpoint. - * `'scope_variable_name': list(variable)` - will initialize list of - partitioned variables with variable from the checkpoint. - * `'/': 'scope_name/'` - will load all variables in current `scope_name` from - checkpoint's root (e.g. no scope). - - Supports loading into partitioned variables, which are represented as - `'/part_'`. - - Example: - - ```python - # Create variables. - with tf.compat.v1.variable_scope('test'): - m = tf.compat.v1.get_variable('my_var') - with tf.compat.v1.variable_scope('test2'): - var2 = tf.compat.v1.get_variable('my_var') - var3 = tf.compat.v1.get_variable(name="my1", shape=[100, 100], - partitioner=lambda shape, dtype: [5, 1]) - ... - # Specify which variables to initialize from checkpoint. - init_from_checkpoint(checkpoint_dir, { - 'some_var': 'test/my_var', - 'some_scope/': 'test2/'}) - ... - # Or use `Variable` objects to identify what to initialize. - init_from_checkpoint(checkpoint_dir, { - 'some_scope/var2': var2, - }) - # Initialize partitioned variables - init_from_checkpoint(checkpoint_dir, { - 'some_var_from_ckpt': 'part_var', - }) - # Or specifying the list of `Variable` objects. - init_from_checkpoint(checkpoint_dir, { - 'some_var_from_ckpt': var3._get_variable_list(), - }) - ... - # Initialize variables as usual. - session.run(tf.get_all_variables()) - ``` - - Args: - checkpoint_dir: Directory with checkpoints file or path to checkpoint. - assignment_map: Dict, where keys are names of the variables in the - checkpoint and values are current variables or names of current variables - (in default graph). - - Raises: - tf.errors.OpError: If missing checkpoints or tensors in checkpoints. - ValueError: If missing variables in current graph. - """ - filepattern = _get_checkpoint_filename(checkpoint_dir) - reader = load_checkpoint(checkpoint_dir) - variable_map = reader.get_variable_to_shape_map() - for tensor_name_in_ckpt, current_var_or_name in six.iteritems(assignment_map): - var = None - # Check if this is Variable object or list of Variable objects (in case of - # partitioned variables). - is_var = lambda x: isinstance(x, variables.Variable) - if is_var(current_var_or_name) or ( - isinstance(current_var_or_name, list) - and all(is_var(v) for v in current_var_or_name)): - var = current_var_or_name - else: - var_scope = vs._get_default_variable_store() - # Check if this variable is in var_store. - var = var_scope._vars.get(current_var_or_name, None) - # Also check if variable is partitioned as list. - if var is None: - var = _collect_partitioned_variable(current_var_or_name, var_scope) - if var is not None: - # If 1 to 1 mapping was provided, find variable in the checkpoint. - if tensor_name_in_ckpt not in variable_map: - raise ValueError("Tensor %s is not found in %s checkpoint %s" % ( - tensor_name_in_ckpt, checkpoint_dir, variable_map - )) - if is_var(var): - # Additional at-call-time checks. - if not var.get_shape().is_compatible_with( - variable_map[tensor_name_in_ckpt]): - raise ValueError( - "Shape of variable %s (%s) doesn't match with shape of " - "tensor %s (%s) from checkpoint reader." % ( - var.name, str(var.get_shape()), - tensor_name_in_ckpt, str(variable_map[tensor_name_in_ckpt]) - )) - var_name = var.name - else: - var_name = ",".join([v.name for v in var]) - _set_variable_or_list_initializer(var, filepattern, tensor_name_in_ckpt) - logging.info("Initialize variable %s from checkpoint %s with %s" % ( - var_name, checkpoint_dir, tensor_name_in_ckpt - )) - else: - scopes = "" - # TODO(vihanjain): Support list of 'current_var_or_name' here. - if "/" in current_var_or_name: - scopes = current_var_or_name[:current_var_or_name.rindex("/")] - if not tensor_name_in_ckpt.endswith("/"): - raise ValueError( - "Assignment map with scope only name {} should map to scope only " - "{}. Should be 'scope/': 'other_scope/'.".format( - scopes, tensor_name_in_ckpt)) - # If scope to scope mapping was provided, find all variables in the scope - # and create variable to variable mapping. - scope_variables = set() - for var_name in var_scope._vars: - if not scopes or var_name.startswith(scopes + "/"): - # Consume /part_ if partitioned variable. - if "/part_" in var_name: - var_name = var_name[:var_name.index("/part_")] - scope_variables.add(var_name) - for var_name in scope_variables: - # Lookup name with specified prefix and suffix from current variable. - # If tensor_name given is '/' (root), don't use it for full name. - full_tensor_name = var_name[len(scopes):] - if current_var_or_name != "/": - full_tensor_name = full_tensor_name[1:] - if tensor_name_in_ckpt != "/": - full_tensor_name = tensor_name_in_ckpt + full_tensor_name - if full_tensor_name not in variable_map: - raise ValueError( - "Tensor %s (%s in %s) is not found in %s checkpoint" % ( - full_tensor_name, var_name[len(scopes) + 1:], - tensor_name_in_ckpt, checkpoint_dir - )) - var = var_scope._vars.get(var_name, None) - if var is None: - var = _collect_partitioned_variable(var_name, var_scope) - _set_variable_or_list_initializer(var, filepattern, full_tensor_name) - logging.info("Initialize variable %s from checkpoint %s with %s" % ( - var_name, checkpoint_dir, full_tensor_name - )) -# pylint: enable=protected-access diff --git a/tensorflow/contrib/framework/python/framework/checkpoint_utils_test.py b/tensorflow/contrib/framework/python/framework/checkpoint_utils_test.py deleted file mode 100644 index 77a424145a8..00000000000 --- a/tensorflow/contrib/framework/python/framework/checkpoint_utils_test.py +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for checkpoints tools.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from tensorflow.contrib.framework.python.framework import checkpoint_utils -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import saver as saver_lib - - -def _create_checkpoints(sess, checkpoint_dir): - checkpoint_prefix = os.path.join(checkpoint_dir, "model") - checkpoint_state_name = "checkpoint" - v1 = variable_scope.get_variable("var1", [1, 10]) - v2 = variable_scope.get_variable("var2", [10, 10]) - v3 = variable_scope.get_variable("var3", [100, 100]) - with variable_scope.variable_scope("useful_scope"): - v4 = variable_scope.get_variable("var4", [9, 9]) - sess.run(variables.global_variables_initializer()) - v1_value, v2_value, v3_value, v4_value = sess.run([v1, v2, v3, v4]) - saver = saver_lib.Saver() - saver.save( - sess, - checkpoint_prefix, - global_step=0, - latest_filename=checkpoint_state_name) - return v1_value, v2_value, v3_value, v4_value - - -def _create_partition_checkpoints(sess, checkpoint_dir): - checkpoint_prefix = os.path.join(checkpoint_dir, "model") - checkpoint_state_name = "checkpoint" - with variable_scope.variable_scope("scope"): - v1 = variable_scope.get_variable( - name="var1", - shape=[100, 100], - initializer=init_ops.truncated_normal_initializer(0.5), - partitioner=partitioned_variables.min_max_variable_partitioner( - max_partitions=5, axis=0, min_slice_size=8 << 10)) - sess.run(variables.global_variables_initializer()) - v1_value = sess.run(v1._get_variable_list()) - saver = saver_lib.Saver() - saver.save( - sess, - checkpoint_prefix, - global_step=0, - latest_filename=checkpoint_state_name) - return v1_value - - -class CheckpointsTest(test.TestCase): - - def testNoCheckpoints(self): - checkpoint_dir = self.get_temp_dir() + "/no_checkpoints" - with self.assertRaises(errors_impl.OpError): - self.assertAllEqual( - checkpoint_utils.load_variable(checkpoint_dir, "var1"), []) - - def testNoTensor(self): - checkpoint_dir = self.get_temp_dir() - with self.cached_session() as session: - _, _, _, _ = _create_checkpoints(session, checkpoint_dir) - with self.assertRaises(errors_impl.OpError): - self.assertAllEqual( - checkpoint_utils.load_variable(checkpoint_dir, "var5"), []) - - def testGetTensor(self): - checkpoint_dir = self.get_temp_dir() - with self.cached_session() as session: - v1, v2, v3, v4 = _create_checkpoints(session, checkpoint_dir) - self.assertAllEqual( - checkpoint_utils.load_variable(checkpoint_dir, "var1"), v1) - self.assertAllEqual( - checkpoint_utils.load_variable(checkpoint_dir, "var2"), v2) - self.assertAllEqual( - checkpoint_utils.load_variable(checkpoint_dir, "var3"), v3) - self.assertAllEqual( - checkpoint_utils.load_variable(checkpoint_dir, "useful_scope/var4"), v4) - - def testGetAllVariables(self): - checkpoint_dir = self.get_temp_dir() - with self.cached_session() as session: - _create_checkpoints(session, checkpoint_dir) - self.assertEqual( - checkpoint_utils.list_variables(checkpoint_dir), - [("useful_scope/var4", [9, 9]), ("var1", [1, 10]), ("var2", [10, 10]), - ("var3", [100, 100])]) - - def testInitFromCheckpoint(self): - checkpoint_dir = self.get_temp_dir() - with self.cached_session() as session: - v1, v2, v3, v4 = _create_checkpoints(session, checkpoint_dir) - - # New graph and session. - with ops.Graph().as_default() as g: - with self.session(graph=g) as session: - with variable_scope.variable_scope("some_scope"): - my1 = variable_scope.get_variable("my1", [1, 10]) - with variable_scope.variable_scope("some_other_scope"): - my2 = variable_scope.get_variable("my2", [10, 10]) - with variable_scope.variable_scope("other_useful_scope"): - my4 = variable_scope.get_variable("var4", [9, 9]) - my3 = variable_scope.get_variable("my3", [100, 100]) - - checkpoint_utils.init_from_checkpoint(checkpoint_dir, { - "var1": "some_scope/my1", - "useful_scope/": "some_scope/some_other_scope/other_useful_scope/", - }) - checkpoint_utils.init_from_checkpoint(checkpoint_dir, { - "var2": "some_scope/some_other_scope/my2", - "var3": my3, - }) - - session.run(variables.global_variables_initializer()) - self.assertAllEqual(my1.eval(session), v1) - self.assertAllEqual(my2.eval(session), v2) - self.assertAllEqual(my3.eval(session), v3) - self.assertAllEqual(my4.eval(session), v4) - - # Check that tensors are not explicitly in the graph. - self.assertLess(len(str(session.graph.as_graph_def())), 27000) - - def testInitWithScopeDoesNotCaptureSuffixes(self): - checkpoint_dir = self.get_temp_dir() - with self.cached_session() as session: - _, _, _, v4 = _create_checkpoints(session, checkpoint_dir) - - with ops.Graph().as_default() as g: - with variable_scope.variable_scope("useful_scope"): - my4 = variable_scope.get_variable("var4", [9, 9]) - with variable_scope.variable_scope("useful_scope_1"): - my5_init = [[1.0, 2.0], [3.0, 4.0]] - my5 = variable_scope.get_variable("var5", initializer=my5_init) - - checkpoint_utils.init_from_checkpoint(checkpoint_dir, - {"useful_scope/": "useful_scope/"}) - with self.session(graph=g) as session: - session.run(variables.global_variables_initializer()) - self.assertAllEqual(my4.eval(session), v4) - self.assertAllEqual(my5.eval(session), my5_init) - - def testInitFromRootCheckpoint(self): - checkpoint_dir = self.get_temp_dir() - with self.cached_session() as session: - v1, v2, v3, v4 = _create_checkpoints(session, checkpoint_dir) - - # New graph and session. - with ops.Graph().as_default() as g: - with self.session(graph=g) as session: - with variable_scope.variable_scope("some_scope"): - my1 = variable_scope.get_variable("var1", [1, 10]) - my2 = variable_scope.get_variable("var2", [10, 10]) - my3 = variable_scope.get_variable("var3", [100, 100]) - with variable_scope.variable_scope("useful_scope"): - my4 = variable_scope.get_variable("var4", [9, 9]) - - checkpoint_utils.init_from_checkpoint(checkpoint_dir, - {"/": "some_scope/",}) - - session.run(variables.global_variables_initializer()) - self.assertAllEqual(my1.eval(session), v1) - self.assertAllEqual(my2.eval(session), v2) - self.assertAllEqual(my3.eval(session), v3) - self.assertAllEqual(my4.eval(session), v4) - - def testInitToRootCheckpoint(self): - checkpoint_dir = self.get_temp_dir() - with self.cached_session() as session: - v1, v2, v3, v4 = _create_checkpoints(session, checkpoint_dir) - - # New graph and session. - with ops.Graph().as_default() as g: - with self.session(graph=g) as session: - my1 = variable_scope.get_variable("var1", [1, 10]) - my2 = variable_scope.get_variable("var2", [10, 10]) - my3 = variable_scope.get_variable("var3", [100, 100]) - with variable_scope.variable_scope("useful_scope"): - my4 = variable_scope.get_variable("var4", [9, 9]) - - checkpoint_utils.init_from_checkpoint(checkpoint_dir, - {"/": "/",}) - - session.run(variables.global_variables_initializer()) - self.assertAllEqual(my1.eval(session), v1) - self.assertAllEqual(my2.eval(session), v2) - self.assertAllEqual(my3.eval(session), v3) - self.assertAllEqual(my4.eval(session), v4) - - def testInitFromPartitionVar(self): - checkpoint_dir = self.get_temp_dir() - with self.cached_session() as session: - v1 = _create_partition_checkpoints(session, checkpoint_dir) - - # New graph and session. - with ops.Graph().as_default() as g: - with self.session(graph=g) as session: - with variable_scope.variable_scope("some_scope"): - my1 = variable_scope.get_variable( - name="my1", - shape=[100, 100], - initializer=init_ops.truncated_normal_initializer(0.5), - partitioner=partitioned_variables.min_max_variable_partitioner( - max_partitions=5, axis=0, min_slice_size=8 << 10)) - my1_var_list = my1._get_variable_list() - with variable_scope.variable_scope("some_other_scope"): - my2 = variable_scope.get_variable( - name="var1", - shape=[100, 100], - initializer=init_ops.truncated_normal_initializer(0.5), - partitioner=partitioned_variables.min_max_variable_partitioner( - max_partitions=5, axis=0, min_slice_size=8 << 10)) - my2_var_list = my2._get_variable_list() - - checkpoint_utils.init_from_checkpoint(checkpoint_dir, { - "scope/var1": "some_scope/my1", - "scope/": "some_other_scope/"}) - - session.run(variables.global_variables_initializer()) - my1_values = session.run(my1_var_list) - self.assertAllEqual(my1_values, v1) - my2_values = session.run(my2_var_list) - self.assertAllEqual(my2_values, v1) - - # New graph and session. - with ops.Graph().as_default() as g: - with self.session(graph=g) as session: - with variable_scope.variable_scope("some_scope"): - my1 = variable_scope.get_variable( - name="my1", - shape=[100, 100], - initializer=init_ops.truncated_normal_initializer(0.5), - partitioner=partitioned_variables.min_max_variable_partitioner( - max_partitions=5, axis=0, min_slice_size=8 << 10)) - my1_var_list = my1._get_variable_list() - - checkpoint_utils.init_from_checkpoint(checkpoint_dir, - {"scope/var1": my1_var_list,}) - - session.run(variables.global_variables_initializer()) - my1_values = session.run(my1_var_list) - self.assertAllEqual(my1_values, v1) - - def testInitFromCheckpointMissing(self): - checkpoint_dir = self.get_temp_dir() - with self.cached_session() as session: - _, _, _, _ = _create_checkpoints(session, checkpoint_dir) - - # New graph and session. - with ops.Graph().as_default() as g: - with self.session(graph=g) as session: - with variable_scope.variable_scope("some_scope"): - _ = variable_scope.get_variable("my1", [10, 10]) - _ = variable_scope.get_variable( - "my2", [1, 10], - dtype=dtypes.int64, - initializer=init_ops.zeros_initializer()) - - # No directory. - with self.assertRaises(errors_impl.OpError): - checkpoint_utils.init_from_checkpoint("no_dir", - {"var1": "some_scope/my1"}) - - # No variable in checkpoint. - with self.assertRaises(ValueError): - checkpoint_utils.init_from_checkpoint(checkpoint_dir, - {"no_var": "some_scope/my1"}) - - # No variable in the graph. - with self.assertRaises(ValueError): - checkpoint_utils.init_from_checkpoint(checkpoint_dir, - {"var3": "some_scope/no_var"}) - - # Shape mismatch. - with self.assertRaises(ValueError): - checkpoint_utils.init_from_checkpoint(checkpoint_dir, - {"var1": "some_scope/my1"}) - - # Variable 'my1' and 'my2' are missing in given checkpoint scope. - with self.assertRaises(ValueError): - checkpoint_utils.init_from_checkpoint( - checkpoint_dir, {"useful_scope/": "some_scope/"}) - - # Mapping is not to scope name. - with self.assertRaises(ValueError): - checkpoint_utils.init_from_checkpoint(checkpoint_dir, - {"useful_scope": "some_scope/"}) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/framework/python/framework/experimental.py b/tensorflow/contrib/framework/python/framework/experimental.py deleted file mode 100644 index 054b5a4a342..00000000000 --- a/tensorflow/contrib/framework/python/framework/experimental.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tensor utility functions.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util import decorator_utils - - -def _add_experimental_function_notice_to_docstring(doc): - """Adds an experimental notice to a docstring for experimental functions.""" - return decorator_utils.add_notice_to_docstring( - doc, '', - 'EXPERIMENTAL FUNCTION', - '(experimental)', ['THIS FUNCTION IS EXPERIMENTAL. It may change or ' - 'be removed at any time, and without warning.']) - - -def experimental(func): - """Decorator for marking functions or methods experimental. - - This decorator logs an experimental warning whenever the decorated function is - called. It has the following format: - - (from ) is experimental and may change or be removed at - any time, and without warning. - - will include the class name if it is a method. - - It also edits the docstring of the function: ' (experimental)' is appended - to the first line of the docstring and a notice is prepended to the rest of - the docstring. - - Args: - func: A function or method to mark experimental. - - Returns: - Decorated function or method. - """ - decorator_utils.validate_callable(func, 'experimental') - @functools.wraps(func) - def new_func(*args, **kwargs): - logging.warning( - '%s (from %s) is experimental and may change or be removed at ' - 'any time, and without warning.', - decorator_utils.get_qualified_name(func), func.__module__) - return func(*args, **kwargs) - new_func.__doc__ = _add_experimental_function_notice_to_docstring( - func.__doc__) - return new_func diff --git a/tensorflow/contrib/framework/python/framework/experimental_test.py b/tensorflow/contrib/framework/python/framework/experimental_test.py deleted file mode 100644 index 00e04b83ac4..00000000000 --- a/tensorflow/contrib/framework/python/framework/experimental_test.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""@experimental tests.""" - -# pylint: disable=unused-import -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework.python.framework import experimental -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging as logging - - -class ExperimentalTest(test.TestCase): - - @test.mock.patch.object(logging, "warning", autospec=True) - def test_warning(self, mock_warning): - - @experimental - def _fn(arg0, arg1): - """fn doc. - - Args: - arg0: Arg 0. - arg1: Arg 1. - - Returns: - Sum of args. - """ - return arg0 + arg1 - - # Assert function docs are properly updated. - self.assertEqual("_fn", _fn.__name__) - self.assertEqual( - "fn doc. (experimental)" - "\n" - "\nWarning: THIS FUNCTION IS EXPERIMENTAL. It may change " - "or be removed at any time, and without warning." - "\n" - "\nArgs:" - "\n arg0: Arg 0." - "\n arg1: Arg 1." - "\n" - "\nReturns:" - "\n Sum of args.", _fn.__doc__) - - # Assert calling new fn issues log warning. - self.assertEqual(3, _fn(1, 2)) - self.assertEqual(1, mock_warning.call_count) - (args, _) = mock_warning.call_args - self.assertRegexpMatches(args[0], r"is experimental and may change") - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/framework/python/framework/graph_util.py b/tensorflow/contrib/framework/python/framework/graph_util.py deleted file mode 100644 index 6dd82a47572..00000000000 --- a/tensorflow/contrib/framework/python/framework/graph_util.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Helpers to manipulate a tensor graph in python. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import copy -import six - -# pylint: disable=unused-import -from tensorflow.core.framework import graph_pb2 -from tensorflow.core.framework import node_def_pb2 -from tensorflow.python.framework import ops -from tensorflow.python.framework.graph_util_impl import _assert_nodes_are_present -from tensorflow.python.framework.graph_util_impl import _bfs_for_reachable_nodes -from tensorflow.python.framework.graph_util_impl import _extract_graph_summary -from tensorflow.python.framework.graph_util_impl import _node_name - - -__all__ = ["fuse_op", "get_placeholders"] - - -def fuse_op(graph_def, input_nodes, output_nodes, output_dtypes, - output_quantized, op_name, op_type): - """Fuse subgraph between input_nodes and output_nodes into a single custom op. - - Args: - graph_def: A graph_pb2.GraphDef proto. - input_nodes: input nodes to the subgraph to be fused. - output_nodes: output nodes to the subgraph to be fused. - output_dtypes: A list of output datatypes for the custom op - output_quantized: A boolean flag that indicates if output is quantized - op_name: fused op name. - op_type: fused op type. - Returns: - The GraphDef of the new graph. - - Raises: - TypeError: If 'graph_def' is not a graph_pb2.GraphDef proto. - """ - - if not isinstance(graph_def, graph_pb2.GraphDef): - raise TypeError("graph_def must be a graph_pb2.GraphDef proto.") - - if isinstance(input_nodes, six.string_types): - raise TypeError("input_nodes must be a list.") - - if isinstance(output_nodes, six.string_types): - raise TypeError("output_nodes must be a list.") - - name_to_input_name, name_to_node, name_to_seq_num = _extract_graph_summary( - graph_def) - _assert_nodes_are_present(name_to_node, input_nodes + output_nodes) - - # Nodes upto and including input_nodes - reachable_by_input = _bfs_for_reachable_nodes(input_nodes, name_to_input_name) - # Nodes upto and including output_nodes - reachable_by_output = _bfs_for_reachable_nodes(output_nodes, - name_to_input_name) - - # Set of nodes in the list input_nodes - input_nodes_set = set(input_nodes) - - # Set of nodes in the list output_nodes - output_nodes_set = set(output_nodes) - - nodes_post_output = [] - for node in graph_def.node: - n = _node_name(node.name) - if n in reachable_by_output: - if n not in reachable_by_input and n not in output_nodes_set: - # n is between input and output, i.e., part of the fused op - next_to_visit = [n] - visited = set() - while next_to_visit: - cur_node = next_to_visit[0] - visited.add(cur_node) - del next_to_visit[0] - if cur_node in reachable_by_input and cur_node not in input_nodes_set: - raise TypeError("Node %s uses input %s not in input_nodes." % - (n, cur_node)) - if cur_node not in input_nodes_set: - next_to_visit += [ - input_node for input_node in name_to_input_name[cur_node] - if input_node not in visited - ] - elif n not in reachable_by_input: - nodes_post_output.append(n) - - # Add all nodes upto the input nodes - out = graph_pb2.GraphDef() - reachable_by_input_sorted = sorted( - list(reachable_by_input), key=lambda n: name_to_seq_num[n]) - for node in reachable_by_input_sorted: - out.node.extend([copy.deepcopy(name_to_node[node])]) - - # Add the custom op - new_node = node_def_pb2.NodeDef() - for node in input_nodes: - new_node.input.append(node) - new_node.attr["_output_types"].list.type[:] = output_dtypes - new_node.attr["_output_quantized"].b = output_quantized - new_node.op = op_type - new_node.name = op_name - out.node.extend([new_node]) - - # Add the nodes in the output of the custom op - for index, n in enumerate(output_nodes): - assert len(name_to_node[n].input) == 1 - new_node = copy.deepcopy(name_to_node[n]) - del new_node.input[:] - new_node.input.append(op_name + (":" + str(index) if index != 0 else "")) - out.node.extend([new_node]) - - # Add the nodes post output_nodes - for n in nodes_post_output: - out.node.extend([copy.deepcopy(name_to_node[n])]) - - out.library.CopyFrom(graph_def.library) - out.versions.CopyFrom(graph_def.versions) - return out - - -def get_placeholders(graph): - """Get placeholders of a graph. - - For example: - - ```python - a = tf.compat.v1.placeholder(dtype=tf.float32, shape=[2, 2], name='a') - a = tf.compat.v1.placeholder(dtype=tf.int32, shape=[3, 2], name='b') - - tf.contrib.framework.get_placeholders(tf.compat.v1.get_default_graph()) - # Returns: - # [, - # ] - ``` - - Args: - graph: A tf.Graph. - Returns: - A list contains all placeholders of given graph. - - Raises: - TypeError: If `graph` is not a tensorflow graph. - """ - - if not isinstance(graph, ops.Graph): - raise TypeError("Input graph needs to be a Graph: %s" % graph) - - # For each placeholder() call, there is a corresponding - # operation of type 'Placeholder' registered to the graph. - # The return value (a Tensor) of placeholder() is the - # first output of this operation in fact. - operations = graph.get_operations() - result = [i.outputs[0] for i in operations if i.type == "Placeholder"] - return result diff --git a/tensorflow/contrib/framework/python/framework/graph_util_test.py b/tensorflow/contrib/framework/python/framework/graph_util_test.py deleted file mode 100644 index 812c5fbd8cb..00000000000 --- a/tensorflow/contrib/framework/python/framework/graph_util_test.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""@graph_util tests.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework.python.framework import graph_util -from tensorflow.core.framework import graph_pb2 -from tensorflow.core.framework import node_def_pb2 -from tensorflow.core.framework import types_pb2 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -def GetNewNode(name, op, input_nodes): - new_node = node_def_pb2.NodeDef() - new_node.op = op - new_node.name = name - for node in input_nodes: - new_node.input.append(node) - return new_node - - -class GraphUtilTest(test.TestCase): - - def testGraphUtil(self): - graph_def = graph_pb2.GraphDef() - node_a = GetNewNode('A', 'Placeholder', []) - node_b = GetNewNode('B', 'Op1', ['A']) - # A loop in the part that will be fused. - node_c = GetNewNode('C', 'Op1', ['B', 'C']) - node_d = GetNewNode('D', 'Op1', ['C']) - node_e = GetNewNode('E', 'Op1', ['D']) - graph_def.node.extend([node_a, node_b, node_c, node_d, node_e]) - fused_graph_def = graph_util.fuse_op( - graph_def, ['A'], ['D'], [types_pb2.DT_FLOAT], True, 'FusedOp', 'Op2') - self.assertEqual(len(fused_graph_def.node), 4) - self.assertEqual(fused_graph_def.node[0].name, 'A') - self.assertEqual(fused_graph_def.node[1].name, 'FusedOp') - self.assertEqual(fused_graph_def.node[1].input[0], 'A') - self.assertEqual(fused_graph_def.node[1].op, 'Op2') - self.assertEqual(fused_graph_def.node[1].attr['_output_quantized'].b, True) - self.assertEqual(fused_graph_def.node[1].attr['_output_types'].list.type, - [types_pb2.DT_FLOAT]) - self.assertEqual(fused_graph_def.node[2].name, 'D') - self.assertEqual(fused_graph_def.node[3].name, 'E') - - def testGraphUtilArtificialDependencyInjection(self): - graph_def = graph_pb2.GraphDef() - node_a = GetNewNode('A', 'Placeholder', []) - node_a1 = GetNewNode('A1', 'Placeholder', []) - node_b = GetNewNode('B', 'Op1', ['A']) - node_c = GetNewNode('C', 'Op1', ['B']) - node_d = GetNewNode('D', 'Op1', ['C']) - node_e = GetNewNode('E', 'Op1', ['D']) - graph_def.node.extend([node_a, node_a1, node_b, node_c, node_d, node_e]) - fused_graph_def = graph_util.fuse_op(graph_def, ['A', 'A1'], ['D'], - [types_pb2.DT_FLOAT], True, 'FusedOp', - 'Op2') - self.assertEqual(len(fused_graph_def.node), 5) - self.assertEqual(fused_graph_def.node[0].name, 'A') - self.assertEqual(fused_graph_def.node[1].name, 'A1') - self.assertEqual(fused_graph_def.node[2].name, 'FusedOp') - self.assertEqual(fused_graph_def.node[2].input[0], 'A') - self.assertEqual(fused_graph_def.node[2].op, 'Op2') - self.assertEqual(fused_graph_def.node[2].attr['_output_quantized'].b, True) - self.assertEqual(fused_graph_def.node[2].attr['_output_types'].list.type, - [types_pb2.DT_FLOAT]) - self.assertEqual(fused_graph_def.node[3].name, 'D') - self.assertEqual(fused_graph_def.node[4].name, 'E') - - -class GetPlaceholdersTest(test.TestCase): - - def test_get_placeholders(self): - with ops.Graph().as_default() as g: - placeholders = [array_ops.placeholder(dtypes.float32) for _ in range(5)] - results = graph_util.get_placeholders(g) - self.assertEqual( - sorted(placeholders, key=lambda x: x._id), # pylint: disable=protected-access - sorted(results, key=lambda x: x._id)) # pylint: disable=protected-access - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/framework/python/framework/tensor_util.py b/tensorflow/contrib/framework/python/framework/tensor_util.py deleted file mode 100644 index bdf8aeb2b8e..00000000000 --- a/tensorflow/contrib/framework/python/framework/tensor_util.py +++ /dev/null @@ -1,319 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tensor utility functions.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.util.deprecation import deprecated - - -__all__ = [ - 'assert_same_float_dtype', - 'assert_scalar', - 'assert_scalar_int', - 'convert_to_tensor_or_sparse_tensor', - 'is_tensor', - 'reduce_sum_n', - 'remove_squeezable_dimensions', - 'with_shape', - 'with_same_shape'] - - -# Temporary for backwards compatibility -is_tensor = tensor_util.is_tensor -assert_same_float_dtype = check_ops.assert_same_float_dtype -assert_scalar = check_ops.assert_scalar - -convert_to_tensor_or_sparse_tensor = ( - sparse_tensor.convert_to_tensor_or_sparse_tensor) - - -def reduce_sum_n(tensors, name=None): - """Reduce tensors to a scalar sum. - - This reduces each tensor in `tensors` to a scalar via `tf.reduce_sum`, then - adds them via `tf.add_n`. - - Args: - tensors: List of tensors, all of the same numeric type. - name: Tensor name, and scope for all other ops. - - Returns: - Total loss tensor, or None if no losses have been configured. - - Raises: - ValueError: if `losses` is missing or empty. - """ - if not tensors: - raise ValueError('No tensors provided.') - with ops.name_scope(name, 'reduce_sum_n', tensors) as name_scope: - tensors = [ - math_ops.reduce_sum(t, name='%s/sum' % t.op.name) for t in tensors] - if len(tensors) == 1: - return tensors[0] - return math_ops.add_n(tensors, name=name_scope) - -@deprecated( - None, "Please switch to remove_squeezable_dimensions from " - "tf.confusion_matrix. Note that the order of the inputs and outputs of " - "labels and predictions have also been switched.") -def remove_squeezable_dimensions(predictions, labels, name=None): - """Squeeze last dim if ranks of `predictions` and `labels` differ by 1. - - This will use static shape if available. Otherwise, it will add graph - operations, which could result in a performance hit. - - Args: - predictions: Predicted values, a `Tensor` of arbitrary dimensions. - labels: Label values, a `Tensor` whose dimensions match `predictions`. - name: Name of the op. - - Returns: - Tuple of `predictions` and `labels`, possibly with last dim squeezed. - """ - with ops.name_scope(name, 'remove_squeezable_dimensions', - [predictions, labels]): - predictions = ops.convert_to_tensor(predictions) - labels = ops.convert_to_tensor(labels) - predictions_shape = predictions.get_shape() - predictions_rank = predictions_shape.ndims - labels_shape = labels.get_shape() - labels_rank = labels_shape.ndims - if (labels_rank is not None) and (predictions_rank is not None): - # Use static rank. - rank_diff = predictions_rank - labels_rank - if rank_diff == -1: - labels = array_ops.squeeze(labels, [-1]) - elif rank_diff == 1: - predictions = array_ops.squeeze(predictions, [-1]) - return predictions, labels - - # Use dynamic rank. - rank_diff = array_ops.rank(predictions) - array_ops.rank(labels) - if (predictions_rank is None) or ( - predictions_shape.dims[-1].is_compatible_with(1)): - predictions = control_flow_ops.cond( - math_ops.equal(1, rank_diff), - lambda: array_ops.squeeze(predictions, [-1]), - lambda: predictions) - if (labels_rank is None) or ( - labels_shape.dims[-1].is_compatible_with(1)): - labels = control_flow_ops.cond( - math_ops.equal(-1, rank_diff), - lambda: array_ops.squeeze(labels, [-1]), - lambda: labels) - return predictions, labels - - -def _shape_tensor_compatible(expected_shape, actual_shape): - """Returns whether actual_shape is compatible with expected_shape. - - Note that -1 in `expected_shape` is recognized as unknown dimension. - - Args: - expected_shape: Integer list defining the expected shape, or tensor of same. - actual_shape: Shape of the tensor to test. - Returns: - New tensor. - """ - with ops.name_scope('shape_tensor_equal', - values=[expected_shape, actual_shape]) as scope: - return math_ops.reduce_all( - math_ops.logical_or( - math_ops.equal(expected_shape, -1), - math_ops.equal(expected_shape, actual_shape, 'equal'), - name='exclude_partial_shape'), - name=scope) - - -def _is_rank(expected_rank, actual_tensor): - """Returns whether actual_tensor's rank is expected_rank. - - Args: - expected_rank: Integer defining the expected rank, or tensor of same. - actual_tensor: Tensor to test. - Returns: - New tensor. - """ - with ops.name_scope('is_rank', values=[actual_tensor]) as scope: - expected = ops.convert_to_tensor(expected_rank, name='expected') - actual = array_ops.rank(actual_tensor, name='actual') - return math_ops.equal(expected, actual, name=scope) - - -def _is_shape(expected_shape, actual_tensor, actual_shape=None): - """Returns whether actual_tensor's shape is expected_shape. - - Note that -1 in `expected_shape` is recognized as unknown dimension. - - Args: - expected_shape: Integer list defining the expected shape, or tensor of same. - actual_tensor: Tensor to test. - actual_shape: Shape of actual_tensor, if we already have it. - Returns: - New tensor. - """ - with ops.name_scope('is_shape', values=[actual_tensor]) as scope: - is_rank = _is_rank(array_ops.size(expected_shape), actual_tensor) - if actual_shape is None: - actual_shape = array_ops.shape(actual_tensor, name='actual') - shape_equal = _shape_tensor_compatible(expected_shape, actual_shape) - return math_ops.logical_and(is_rank, shape_equal, name=scope) - - -def _assert_shape_op(expected_shape, actual_tensor): - """Asserts actual_tensor's shape is expected_shape. - - Note that unknown dimension in `expected_shape` will be ignored. - - Args: - expected_shape: List of integers defining the expected shape, or tensor of - same. - actual_tensor: Tensor to test. - Returns: - New assert tensor. - """ - with ops.name_scope('assert_shape', values=[actual_tensor]) as scope: - actual_shape = array_ops.shape(actual_tensor, name='actual') - if (isinstance(expected_shape, tensor_shape.TensorShape) - and not expected_shape.is_fully_defined()): - expected_shape = [d if d else -1 for d in expected_shape.as_list()] - is_shape = _is_shape(expected_shape, actual_tensor, actual_shape) - return control_flow_ops.Assert( - is_shape, [ - 'Wrong shape for %s [expected] [actual].' % actual_tensor.name, - expected_shape, - actual_shape - ], name=scope) - - -def with_same_shape(expected_tensor, tensor): - """Assert tensors are the same shape, from the same graph. - - Args: - expected_tensor: Tensor with expected shape. - tensor: Tensor of actual values. - Returns: - The original tensor argument, possibly with assert ops added. - """ - with ops.name_scope('%s/' % tensor.op.name, values=[expected_tensor, tensor]): - tensor_shape = expected_tensor.get_shape() - expected_shape = ( - tensor_shape.as_list() if tensor_shape.is_fully_defined() - else array_ops.shape(expected_tensor, name='expected_shape')) - return with_shape(expected_shape, tensor) - - -def with_shape(expected_shape, tensor): - """Asserts tensor has expected shape. - - If tensor shape and expected_shape, are fully defined, assert they match. - Otherwise, add assert op that will validate the shape when tensor is - evaluated, and set shape on tensor. - - Args: - expected_shape: Expected shape to assert, as a 1D array of ints, or tensor - of same. - tensor: Tensor whose shape we're validating. - Returns: - tensor, perhaps with a dependent assert operation. - Raises: - ValueError: if tensor has an invalid shape. - """ - if isinstance(tensor, sparse_tensor.SparseTensor): - raise ValueError('SparseTensor not supported.') - - # Shape type must be 1D int32. - if tensor_util.is_tensor(expected_shape): - if expected_shape.dtype.base_dtype != dtypes.int32: - raise ValueError( - 'Invalid dtype %s for shape %s expected of tensor %s.' % ( - expected_shape.dtype, expected_shape, tensor.name)) - if isinstance(expected_shape, (list, tuple)): - if not expected_shape: - expected_shape = np.asarray([], dtype=np.int32) - else: - np_expected_shape = np.asarray(expected_shape) - expected_shape = ( - np.asarray(expected_shape, dtype=np.int32) - if np_expected_shape.dtype == np.int64 else np_expected_shape) - if isinstance(expected_shape, np.ndarray): - if expected_shape.ndim > 1: - raise ValueError( - 'Invalid rank %s for shape %s expected of tensor %s.' % ( - expected_shape.ndim, expected_shape, tensor.name)) - if expected_shape.dtype != np.int32: - raise ValueError( - 'Invalid dtype %s for shape %s expected of tensor %s.' % ( - expected_shape.dtype, expected_shape, tensor.name)) - - actual_shape = tensor.get_shape() - - if (not actual_shape.is_fully_defined() - or tensor_util.is_tensor(expected_shape)): - with ops.name_scope('%s/' % tensor.op.name, values=[tensor]): - if (not tensor_util.is_tensor(expected_shape) - and (len(expected_shape) < 1)): - # TODO(irving): Remove scalar special case - return array_ops.reshape(tensor, []) - with ops.control_dependencies([_assert_shape_op(expected_shape, tensor)]): - result = array_ops.identity(tensor) - if not tensor_util.is_tensor(expected_shape): - result.set_shape(expected_shape) - return result - - if (not tensor_util.is_tensor(expected_shape) and - not actual_shape.is_compatible_with(expected_shape)): - if (len(expected_shape) < 1) and actual_shape.is_compatible_with([1]): - # TODO(irving): Remove scalar special case. - with ops.name_scope('%s/' % tensor.op.name, values=[tensor]): - return array_ops.reshape(tensor, []) - raise ValueError('Invalid shape for tensor %s, expected %s, got %s.' % ( - tensor.name, expected_shape, actual_shape)) - - return tensor - - -def assert_scalar_int(tensor, name=None): - """Assert `tensor` is 0-D, of type `tf.int32` or `tf.int64`. - - Args: - tensor: `Tensor` to test. - name: Name of the op and of the new `Tensor` if one is created. - Returns: - `tensor`, for chaining. - Raises: - ValueError: if `tensor` is not 0-D, of integer type. - """ - with ops.name_scope(name, 'assert_scalar_int', [tensor]) as name_scope: - tensor = ops.convert_to_tensor(tensor) - data_type = tensor.dtype - if not data_type.base_dtype.is_integer: - raise ValueError('Expected integer type for %s, received type: %s.' - % (tensor.name, data_type)) - return check_ops.assert_scalar(tensor, name=name_scope) diff --git a/tensorflow/contrib/framework/python/framework/tensor_util_test.py b/tensorflow/contrib/framework/python/framework/tensor_util_test.py deleted file mode 100644 index 05788d2e820..00000000000 --- a/tensorflow/contrib/framework/python/framework/tensor_util_test.py +++ /dev/null @@ -1,387 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""tensor_util tests.""" - -# pylint: disable=unused-import -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re - -import numpy as np - -from tensorflow.contrib.framework.python.framework import tensor_util -from tensorflow.contrib.framework.python.ops import variables as variables_lib2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.platform import test - - -class LocalVariabletest(test.TestCase): - - def test_local_variable(self): - with self.cached_session() as sess: - self.assertEquals([], variables_lib.local_variables()) - value0 = 42 - variables_lib2.local_variable(value0) - value1 = 43 - variables_lib2.local_variable(value1) - variables = variables_lib.local_variables() - self.assertEquals(2, len(variables)) - self.assertRaises(errors_impl.OpError, sess.run, variables) - variables_lib.variables_initializer(variables).run() - self.assertAllEqual(set([value0, value1]), set(sess.run(variables))) - - -class ReduceSumNTest(test.TestCase): - - def test_reduce_sum_n(self): - with self.cached_session(): - a = constant_op.constant(1) - b = constant_op.constant([2]) - c = constant_op.constant([[3, 4], [5, 6]]) - self.assertEqual(21, tensor_util.reduce_sum_n([a, b, c]).eval()) - - -class AssertScalarIntTest(test.TestCase): - - def test_assert_scalar_int(self): - tensor_util.assert_scalar_int(constant_op.constant(3, dtype=dtypes.int32)) - tensor_util.assert_scalar_int(constant_op.constant(3, dtype=dtypes.int64)) - tensor_util.assert_scalar_int(3) - with self.assertRaisesRegexp(ValueError, "Expected integer"): - tensor_util.assert_scalar_int( - constant_op.constant( - 3, dtype=dtypes.float32)) - with self.assertRaisesRegexp(ValueError, "Expected scalar"): - tensor_util.assert_scalar_int( - constant_op.constant( - [3, 4], dtype=dtypes.int32)) - - -class WithShapeTest(test.TestCase): - - def _assert_with_shape(self, tensor, expected_value, expected_shape, - unexpected_shapes): - for unexpected_shape in unexpected_shapes: - self.assertRaises(ValueError, tensor_util.with_shape, unexpected_shape, - tensor) - pattern = ( - r"\[Wrong shape for %s \[expected\] \[actual\].\] \[%s\] \[%s\]" % - (tensor.name, " ".join([str(dim) for dim in unexpected_shape]), - " ".join([str(dim) for dim in expected_shape]))) - self.assertRaisesRegexp(errors_impl.OpError, - re.compile(pattern), - tensor_util.with_shape( - constant_op.constant(unexpected_shape), - tensor).eval) - expected_placeholder = array_ops.placeholder(dtypes.float32) - self.assertRaisesRegexp(errors_impl.OpError, - re.compile(pattern), - tensor_util.with_same_shape(expected_placeholder, - tensor).eval, - {expected_placeholder: np.ones(unexpected_shape)}) - - self.assertIs(tensor, tensor_util.with_shape(expected_shape, tensor)) - self.assertIs( - tensor, - tensor_util.with_same_shape( - constant_op.constant( - 1, shape=expected_shape), tensor)) - tensor_with_shape = tensor_util.with_shape( - constant_op.constant(expected_shape), tensor) - np.testing.assert_array_equal(expected_value, tensor_with_shape.eval()) - tensor_with_same_shape = tensor_util.with_same_shape(expected_placeholder, - tensor) - np.testing.assert_array_equal(expected_value, - tensor_with_same_shape.eval({ - expected_placeholder: - np.ones(expected_shape) - })) - - def test_with_shape_invalid_expected_shape(self): - with self.cached_session(): - self.assertRaisesRegexp(ValueError, "Invalid rank", - tensor_util.with_shape, [[1], [2]], - constant_op.constant(1.0)) - - def test_with_shape_invalid_type(self): - with self.cached_session(): - self.assertRaisesRegexp(ValueError, "Invalid dtype", - tensor_util.with_shape, [1.1], - constant_op.constant([1.0])) - self.assertRaisesRegexp(ValueError, "Invalid dtype", - tensor_util.with_shape, - np.array([1.1]), constant_op.constant(1.0)) - self.assertRaisesRegexp(ValueError, "Invalid dtype", - tensor_util.with_shape, - constant_op.constant(np.array([1.1])), - constant_op.constant(1.0)) - - def test_with_shape_0(self): - with self.cached_session(): - value = 42 - shape = [0] - unexpected_shapes = [[1], [2], [1, 1]] - self._assert_with_shape( - constant_op.constant( - value, shape=shape), - value, - shape, - unexpected_shapes) - - def test_with_shape_1(self): - with self.cached_session(): - value = [42] - shape = [1] - unexpected_shapes = [[0], [2], [1, 1]] - self._assert_with_shape( - constant_op.constant( - value, shape=shape), - value, - shape, - unexpected_shapes) - - def test_with_shape_2(self): - with self.cached_session(): - value = [42, 43] - shape = [2] - unexpected_shapes = [[0], [1], [2, 1]] - self._assert_with_shape( - constant_op.constant( - value, shape=shape), - value, - shape, - unexpected_shapes) - - def test_with_shape_2x2(self): - with self.cached_session(): - value = [[42, 43], [44, 45]] - shape = [2, 2] - unexpected_shapes = [[0], [1], [2, 1]] - self._assert_with_shape( - constant_op.constant( - value, shape=shape), - value, - shape, - unexpected_shapes) - - def test_with_shape_2x2_with_partial_expected_shape(self): - with self.cached_session(): - value = [[42, 43], [44, 45]] - actual_shape = [2, 2] - tensor = constant_op.constant(value, shape=actual_shape) - partial_expected_shape = tensor_shape.TensorShape([None, 2]) - # Won't raise any exception here: - tensor_with_shape = tensor_util.with_shape(partial_expected_shape, tensor) - np.testing.assert_array_equal(value, tensor_with_shape.eval()) - - def test_with_shape_none(self): - with self.cached_session(): - tensor_no_shape = array_ops.placeholder(dtypes.float32) - - compatible_shape = [2, 2] - with_present_2x2 = tensor_util.with_shape(compatible_shape, - tensor_no_shape) - self.assertEquals(compatible_shape, with_present_2x2.get_shape().dims) - with_future_2x2 = tensor_util.with_shape( - constant_op.constant(compatible_shape), tensor_no_shape) - - array_2x2 = [[42.0, 43.0], [44.0, 45.0]] - for tensor_2x2 in [with_present_2x2, with_future_2x2]: - np.testing.assert_array_equal(array_2x2, - tensor_2x2.eval({ - tensor_no_shape: array_2x2 - })) - self.assertRaisesRegexp(errors_impl.OpError, "Wrong shape", - tensor_2x2.eval, - {tensor_no_shape: [42.0, 43.0]}) - self.assertRaisesRegexp(errors_impl.OpError, "Wrong shape", - tensor_2x2.eval, {tensor_no_shape: [42.0]}) - - def test_with_shape_partial(self): - with self.cached_session(): - tensor_partial_shape = array_ops.placeholder(dtypes.float32) - tensor_partial_shape.set_shape([None, 2]) - - for incompatible_shape in [[0], [1]]: - self.assertRaisesRegexp( - ValueError, "Shapes must be equal rank, but are 2 and 1", - tensor_util.with_shape, incompatible_shape, tensor_partial_shape) - for incompatible_shape in [[1, 2, 1]]: - self.assertRaisesRegexp(ValueError, "Dimensions must be equal", - tensor_util.with_shape, incompatible_shape, - tensor_partial_shape) - for incompatible_shape in [[2, 1]]: - self.assertRaisesRegexp( - ValueError, - r"Dimension 1 in both shapes must be equal, but are 2 and 1. " - r"Shapes are \[\?,2\] and \[2,1\].", - tensor_util.with_shape, incompatible_shape, tensor_partial_shape) - - compatible_shape = [2, 2] - with_present_2x2 = tensor_util.with_shape(compatible_shape, - tensor_partial_shape) - self.assertEquals(compatible_shape, with_present_2x2.get_shape().dims) - with_future_2x2 = tensor_util.with_shape( - constant_op.constant(compatible_shape), tensor_partial_shape) - - array_2x2 = [[42.0, 43.0], [44.0, 45.0]] - for tensor_2x2 in [with_present_2x2, with_future_2x2]: - np.testing.assert_array_equal(array_2x2, - tensor_2x2.eval({ - tensor_partial_shape: array_2x2 - })) - self.assertRaises(ValueError, tensor_2x2.eval, - {tensor_partial_shape: [42.0, 43.0]}) - self.assertRaises(ValueError, tensor_2x2.eval, - {tensor_partial_shape: [42.0]}) - - -class RemoveSqueezableDimensionsTest(test.TestCase): - - def testRemoveSqueezableDimensions(self): - self._testRemoveSqueezableDimensions( - predictions_have_static_shape=False, - predictions_have_extra_dim=False, - labels_have_static_shape=False, - labels_have_extra_dim=False) - - def testRemoveSqueezableDimensions_extraLabelDim(self): - self._testRemoveSqueezableDimensions( - predictions_have_static_shape=False, - predictions_have_extra_dim=False, - labels_have_static_shape=False, - labels_have_extra_dim=True) - - def testRemoveSqueezableDimensions_staticLabel(self): - self._testRemoveSqueezableDimensions( - predictions_have_static_shape=False, - predictions_have_extra_dim=False, - labels_have_static_shape=True, - labels_have_extra_dim=False) - - def testRemoveSqueezableDimensions_staticLabel_extraLabelDim(self): - self._testRemoveSqueezableDimensions( - predictions_have_static_shape=False, - predictions_have_extra_dim=False, - labels_have_static_shape=True, - labels_have_extra_dim=True) - - def testRemoveSqueezableDimensions_extraPredictionDim(self): - self._testRemoveSqueezableDimensions( - predictions_have_static_shape=False, - predictions_have_extra_dim=True, - labels_have_static_shape=False, - labels_have_extra_dim=False) - - def testRemoveSqueezableDimensions_extraPredictionDim_staticLabel(self): - self._testRemoveSqueezableDimensions( - predictions_have_static_shape=False, - predictions_have_extra_dim=True, - labels_have_static_shape=True, - labels_have_extra_dim=False) - - def testRemoveSqueezableDimensions_staticPrediction(self): - self._testRemoveSqueezableDimensions( - predictions_have_static_shape=True, - predictions_have_extra_dim=False, - labels_have_static_shape=False, - labels_have_extra_dim=False) - - def testRemoveSqueezableDimensions_staticPrediction_extraLabelDim(self): - self._testRemoveSqueezableDimensions( - predictions_have_static_shape=True, - predictions_have_extra_dim=False, - labels_have_static_shape=False, - labels_have_extra_dim=True) - - def testRemoveSqueezableDimensions_static(self): - self._testRemoveSqueezableDimensions( - predictions_have_static_shape=True, - predictions_have_extra_dim=False, - labels_have_static_shape=True, - labels_have_extra_dim=False) - - def testRemoveSqueezableDimensions_static_extraLabelDim(self): - self._testRemoveSqueezableDimensions( - predictions_have_static_shape=True, - predictions_have_extra_dim=False, - labels_have_static_shape=True, - labels_have_extra_dim=True) - - def testRemoveSqueezableDimensions_staticPrediction_extraPredictionDim(self): - self._testRemoveSqueezableDimensions( - predictions_have_static_shape=True, - predictions_have_extra_dim=True, - labels_have_static_shape=False, - labels_have_extra_dim=False) - - def testRemoveSqueezableDimensions_static_extraPredictionDim(self): - self._testRemoveSqueezableDimensions( - predictions_have_static_shape=True, - predictions_have_extra_dim=True, - labels_have_static_shape=True, - labels_have_extra_dim=False) - - # TODO(ptucker): Replace this with parameterized test. - def _testRemoveSqueezableDimensions(self, predictions_have_static_shape, - predictions_have_extra_dim, - labels_have_static_shape, - labels_have_extra_dim): - assert not (predictions_have_extra_dim and labels_have_extra_dim) - predictions_value = (0, 1, 1, 0, 0, 1, 0) - labels_value = (0, 0, 1, 1, 0, 0, 0) - - input_predictions_value = ([[p] for p in predictions_value] if - predictions_have_extra_dim else - predictions_value) - input_labels_value = ([[l] for l in labels_value] if labels_have_extra_dim - else labels_value) - - with ops.Graph().as_default() as g: - feed_dict = {} - if predictions_have_static_shape: - predictions = constant_op.constant( - input_predictions_value, dtype=dtypes.int32) - else: - predictions = array_ops.placeholder( - dtype=dtypes.int32, name="predictions") - feed_dict[predictions] = input_predictions_value - if labels_have_static_shape: - labels = constant_op.constant(input_labels_value, dtype=dtypes.int32) - else: - labels = array_ops.placeholder(dtype=dtypes.int32, name="labels") - feed_dict[labels] = input_labels_value - - squeezed_predictions, squeezed_labels = ( - tensor_util.remove_squeezable_dimensions(predictions, labels)) - with self.session(g): - variables_lib.local_variables_initializer().run() - self.assertAllClose( - predictions_value, squeezed_predictions.eval(feed_dict=feed_dict)) - self.assertAllClose( - labels_value, squeezed_labels.eval(feed_dict=feed_dict)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/framework/python/ops/__init__.py b/tensorflow/contrib/framework/python/ops/__init__.py deleted file mode 100644 index 8113bf7c095..00000000000 --- a/tensorflow/contrib/framework/python/ops/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A module containing TensorFlow ops whose API may change in the future.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# TODO(ptucker): Add these to tf.contrib.variables? -# pylint: disable=wildcard-import -from tensorflow.contrib.framework.python.ops.arg_scope import * -from tensorflow.contrib.framework.python.ops.checkpoint_ops import * -from tensorflow.contrib.framework.python.ops.ops import * -from tensorflow.contrib.framework.python.ops.prettyprint_ops import * -from tensorflow.contrib.framework.python.ops.script_ops import * -from tensorflow.contrib.framework.python.ops.sort_ops import * -from tensorflow.contrib.framework.python.ops.variables import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/framework/python/ops/arg_scope.py b/tensorflow/contrib/framework/python/ops/arg_scope.py deleted file mode 100644 index 0a02e76a265..00000000000 --- a/tensorflow/contrib/framework/python/ops/arg_scope.py +++ /dev/null @@ -1,211 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains the arg_scope used for scoping layers arguments. - - Allows one to define models much more compactly by eliminating boilerplate - code. This is accomplished through the use of argument scoping (arg_scope). - - Example of how to use tf.contrib.framework.arg_scope: - - ``` - from third_party.tensorflow.contrib.layers.python import layers - - arg_scope = tf.contrib.framework.arg_scope - - with arg_scope([layers.conv2d], padding='SAME', - initializer=layers.variance_scaling_initializer(), - regularizer=layers.l2_regularizer(0.05)): - net = layers.conv2d(inputs, 64, [11, 11], 4, padding='VALID', scope='conv1') - net = layers.conv2d(net, 256, [5, 5], scope='conv2') - ``` - The first call to conv2d will behave as follows: - layers.conv2d(inputs, 64, [11, 11], 4, padding='VALID', - initializer=layers.variance_scaling_initializer(), - regularizer=layers.l2_regularizer(0.05), scope='conv1') - - The second call to conv2d will also use the arg_scope's default for padding: - layers.conv2d(inputs, 256, [5, 5], padding='SAME', - initializer=layers.variance_scaling_initializer(), - regularizer=layers.l2_regularizer(0.05), scope='conv2') - - Example of how to reuse an arg_scope: - - ``` - with arg_scope([layers.conv2d], padding='SAME', - initializer=layers.variance_scaling_initializer(), - regularizer=layers.l2_regularizer(0.05)) as sc: - net = layers.conv2d(net, 256, [5, 5], scope='conv1') - .... - - with arg_scope(sc): - net = layers.conv2d(net, 256, [5, 5], scope='conv2') - ``` - - Example of how to use tf.contrib.framework.add_arg_scope to enable your - function to be called within an arg_scope later: - - @tf.contrib.framework.add_arg_scope - def conv2d(*args, **kwargs) -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.util import tf_contextlib -from tensorflow.python.util import tf_decorator - -__all__ = [ - 'arg_scope', 'add_arg_scope', 'current_arg_scope', 'has_arg_scope', - 'arg_scoped_arguments', 'arg_scope_func_key' -] - -_ARGSTACK = [{}] - -_DECORATED_OPS = {} - - -def _get_arg_stack(): - if _ARGSTACK: - return _ARGSTACK - else: - _ARGSTACK.append({}) - return _ARGSTACK - - -def current_arg_scope(): - stack = _get_arg_stack() - return stack[-1] - - -def arg_scope_func_key(op): - return getattr(op, '_key_op', str(op)) - - -def _name_op(op): - return (op.__module__, op.__name__) - - -def _kwarg_names(func): - kwargs_length = len(func.__defaults__) if func.__defaults__ else 0 - return func.__code__.co_varnames[-kwargs_length:func.__code__.co_argcount] - - -def _add_op(op): - key_op = arg_scope_func_key(op) - _DECORATED_OPS[key_op] = _kwarg_names(op) - - -@tf_contextlib.contextmanager -def arg_scope(list_ops_or_scope, **kwargs): - """Stores the default arguments for the given set of list_ops. - - For usage, please see examples at top of the file. - - Args: - list_ops_or_scope: List or tuple of operations to set argument scope for or - a dictionary containing the current scope. When list_ops_or_scope is a - dict, kwargs must be empty. When list_ops_or_scope is a list or tuple, - then every op in it need to be decorated with @add_arg_scope to work. - **kwargs: keyword=value that will define the defaults for each op in - list_ops. All the ops need to accept the given set of arguments. - - Yields: - the current_scope, which is a dictionary of {op: {arg: value}} - Raises: - TypeError: if list_ops is not a list or a tuple. - ValueError: if any op in list_ops has not be decorated with @add_arg_scope. - """ - if isinstance(list_ops_or_scope, dict): - # Assumes that list_ops_or_scope is a scope that is being reused. - if kwargs: - raise ValueError('When attempting to re-use a scope by suppling a' - 'dictionary, kwargs must be empty.') - current_scope = list_ops_or_scope.copy() - try: - _get_arg_stack().append(current_scope) - yield current_scope - finally: - _get_arg_stack().pop() - else: - # Assumes that list_ops_or_scope is a list/tuple of ops with kwargs. - if not isinstance(list_ops_or_scope, (list, tuple)): - raise TypeError('list_ops_or_scope must either be a list/tuple or reused ' - 'scope (i.e. dict)') - try: - current_scope = current_arg_scope().copy() - for op in list_ops_or_scope: - key = arg_scope_func_key(op) - if not has_arg_scope(op): - raise ValueError('%s is not decorated with @add_arg_scope', - _name_op(op)) - if key in current_scope: - current_kwargs = current_scope[key].copy() - current_kwargs.update(kwargs) - current_scope[key] = current_kwargs - else: - current_scope[key] = kwargs.copy() - _get_arg_stack().append(current_scope) - yield current_scope - finally: - _get_arg_stack().pop() - - -def add_arg_scope(func): - """Decorates a function with args so it can be used within an arg_scope. - - Args: - func: function to decorate. - - Returns: - A tuple with the decorated function func_with_args(). - """ - - def func_with_args(*args, **kwargs): - current_scope = current_arg_scope() - current_args = kwargs - key_func = arg_scope_func_key(func) - if key_func in current_scope: - current_args = current_scope[key_func].copy() - current_args.update(kwargs) - return func(*args, **current_args) - - _add_op(func) - setattr(func_with_args, '_key_op', arg_scope_func_key(func)) - return tf_decorator.make_decorator(func, func_with_args) - - -def has_arg_scope(func): - """Checks whether a func has been decorated with @add_arg_scope or not. - - Args: - func: function to check. - - Returns: - a boolean. - """ - return arg_scope_func_key(func) in _DECORATED_OPS - - -def arg_scoped_arguments(func): - """Returns the list kwargs that arg_scope can set for a func. - - Args: - func: function which has been decorated with @add_arg_scope. - - Returns: - a list of kwargs names. - """ - assert has_arg_scope(func) - return _DECORATED_OPS[arg_scope_func_key(func)] diff --git a/tensorflow/contrib/framework/python/ops/arg_scope_test.py b/tensorflow/contrib/framework/python/ops/arg_scope_test.py deleted file mode 100644 index 0e6c6f0e2fa..00000000000 --- a/tensorflow/contrib/framework/python/ops/arg_scope_test.py +++ /dev/null @@ -1,254 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""arg_scope tests.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework.python.ops import add_arg_scope -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.framework.python.ops import arg_scoped_arguments -from tensorflow.python.platform import test - - -@add_arg_scope -def func1(*args, **kwargs): - return (args, kwargs) - - -@add_arg_scope -def func2(*args, **kwargs): - return (args, kwargs) - - -@add_arg_scope -def func3(args, a=None, b=1, c=2): - """Some cool doc string.""" - return (args, a, b, c) - -@add_arg_scope -def func4(x='x', y='y'): - if x: - pass - if y: - pass - -def _key_op(op): - return getattr(op, '_key_op', str(op)) - - -class ArgScopeTest(test.TestCase): - - def testEmptyArgScope(self): - with self.cached_session(): - with arg_scope([]) as sc: - self.assertEqual(sc, {}) - - def testClearArgScope(self): - func1_kwargs = {'a': 1, 'b': None, 'c': [1]} - key_op = _key_op(func1) - func1_scope = {key_op: func1_kwargs.copy()} - with self.cached_session(): - with arg_scope([func1], a=1, b=None, c=[1]) as sc1: - self.assertEqual(sc1, func1_scope) - with arg_scope({}) as sc2: - self.assertEqual(sc2, {}) - with arg_scope([]) as current_arg_scope: - self.assertEqual(current_arg_scope, func1_scope) - - def testNonDecorated(self): - - def my_func(t, a=None): - return (t, a) - - with self.assertRaises(ValueError): - with arg_scope([my_func], a=1): - pass - - def testUnexpectedArg(self): - with self.assertRaises(TypeError): - with arg_scope([func3], d=1): - func3(1) - - def testCurrentArgScope(self): - func1_kwargs = {'a': 1, 'b': None, 'c': [1]} - key_op = _key_op(func1) - current_scope = {key_op: func1_kwargs.copy()} - with self.cached_session(): - with arg_scope([func1], a=1, b=None, c=[1]) as scope: - self.assertDictEqual(scope, current_scope) - - def testArgScopedArguments(self): - func3_kwargs = ('a', 'b', 'c') - self.assertEquals(arg_scoped_arguments(func3), func3_kwargs) - - def testCurrentArgScopeNested(self): - func1_kwargs = {'a': 1, 'b': None, 'c': [1]} - func2_kwargs = {'b': 2, 'd': [2]} - key = _key_op - current_scope = { - key(func1): func1_kwargs.copy(), - key(func2): func2_kwargs.copy() - } - with self.cached_session(): - with arg_scope([func1], a=1, b=None, c=[1]): - with arg_scope([func2], b=2, d=[2]) as scope: - self.assertDictEqual(scope, current_scope) - - def testReuseArgScope(self): - func1_kwargs = {'a': 1, 'b': None, 'c': [1]} - key_op = _key_op(func1) - current_scope = {key_op: func1_kwargs.copy()} - with self.cached_session(): - with arg_scope([func1], a=1, b=None, c=[1]) as scope1: - pass - with arg_scope(scope1) as scope: - self.assertDictEqual(scope, current_scope) - - def testReuseArgScopeNested(self): - func1_kwargs = {'a': 1, 'b': None, 'c': [1]} - func2_kwargs = {'b': 2, 'd': [2]} - key = _key_op - current_scope1 = {key(func1): func1_kwargs.copy()} - current_scope2 = { - key(func1): func1_kwargs.copy(), - key(func2): func2_kwargs.copy() - } - with self.cached_session(): - with arg_scope([func1], a=1, b=None, c=[1]) as scope1: - with arg_scope([func2], b=2, d=[2]) as scope2: - pass - with arg_scope(scope1): - with arg_scope([]) as current_arg_scope: - self.assertDictEqual(current_arg_scope, current_scope1) - with arg_scope(scope2): - with arg_scope([]) as current_arg_scope: - self.assertDictEqual(current_arg_scope, current_scope2) - - def testSimpleArgScope(self): - func1_args = (0,) - func1_kwargs = {'a': 1, 'b': None, 'c': [1]} - with self.cached_session(): - with arg_scope([func1], a=1, b=None, c=[1]): - args, kwargs = func1(0) - self.assertTupleEqual(args, func1_args) - self.assertDictEqual(kwargs, func1_kwargs) - - def testSimpleArgScopeWithTuple(self): - func1_args = (0,) - func1_kwargs = {'a': 1, 'b': None, 'c': [1]} - with self.cached_session(): - with arg_scope((func1,), a=1, b=None, c=[1]): - args, kwargs = func1(0) - self.assertTupleEqual(args, func1_args) - self.assertDictEqual(kwargs, func1_kwargs) - - def testOverwriteArgScope(self): - func1_args = (0,) - func1_kwargs = {'a': 1, 'b': 2, 'c': [1]} - with arg_scope([func1], a=1, b=None, c=[1]): - args, kwargs = func1(0, b=2) - self.assertTupleEqual(args, func1_args) - self.assertDictEqual(kwargs, func1_kwargs) - - def testNestedArgScope(self): - func1_args = (0,) - func1_kwargs = {'a': 1, 'b': None, 'c': [1]} - with arg_scope([func1], a=1, b=None, c=[1]): - args, kwargs = func1(0) - self.assertTupleEqual(args, func1_args) - self.assertDictEqual(kwargs, func1_kwargs) - func1_kwargs['b'] = 2 - with arg_scope([func1], b=2): - args, kwargs = func1(0) - self.assertTupleEqual(args, func1_args) - self.assertDictEqual(kwargs, func1_kwargs) - - def testNestedArgScopeObjectCreatedOutsideScopeOverridesArgScope(self): - - def get_scope_object(): - with arg_scope([func1], a=1, b=None, c=[1]) as sc: - return sc - - scope_object = get_scope_object() - with arg_scope([func1], b=2, d=10): - with arg_scope(scope_object): - args, kwargs = func1(0) - self.assertTupleEqual(args, (0,)) - self.assertDictEqual(kwargs, {'a': 1, 'b': None, 'c': [1]}) - - def testArgScopeObjectCreatedWithinScopeInheritsArgScope(self): - def get_scope_object(): - with arg_scope([func1], a=1, b=None, c=[1]) as sc: - return sc - - with arg_scope([func1], b=2, d=10): - with arg_scope(get_scope_object()): - args, kwargs = func1(0) - self.assertTupleEqual(args, (0,)) - self.assertDictEqual(kwargs, {'a': 1, 'b': None, 'c': [1], 'd': 10}) - - def testSharedArgScope(self): - func1_args = (0,) - func1_kwargs = {'a': 1, 'b': None, 'c': [1]} - with arg_scope([func1, func2], a=1, b=None, c=[1]): - args, kwargs = func1(0) - self.assertTupleEqual(args, func1_args) - self.assertDictEqual(kwargs, func1_kwargs) - args, kwargs = func2(0) - self.assertTupleEqual(args, func1_args) - self.assertDictEqual(kwargs, func1_kwargs) - - def testSharedArgScopeTuple(self): - func1_args = (0,) - func1_kwargs = {'a': 1, 'b': None, 'c': [1]} - with arg_scope((func1, func2), a=1, b=None, c=[1]): - args, kwargs = func1(0) - self.assertTupleEqual(args, func1_args) - self.assertDictEqual(kwargs, func1_kwargs) - args, kwargs = func2(0) - self.assertTupleEqual(args, func1_args) - self.assertDictEqual(kwargs, func1_kwargs) - - def testPartiallySharedArgScope(self): - func1_args = (0,) - func1_kwargs = {'a': 1, 'b': None, 'c': [1]} - func2_args = (1,) - func2_kwargs = {'a': 1, 'b': None, 'd': [2]} - with arg_scope([func1, func2], a=1, b=None): - with arg_scope([func1], c=[1]): - with arg_scope([func2], d=[2]): - args, kwargs = func1(0) - self.assertTupleEqual(args, func1_args) - self.assertDictEqual(kwargs, func1_kwargs) - args, kwargs = func2(1) - self.assertTupleEqual(args, func2_args) - self.assertDictEqual(kwargs, func2_kwargs) - - def testAddArgScopeRaceCondition(self): - func4_kwargs = ('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h') - for i in range(4): - # redefine the function with different args - @add_arg_scope - def func4(a=1, b=2, c=3, d=4, e=5, f=6, g=7, h=8): - pass - self.assertTupleEqual(arg_scoped_arguments(func4), func4_kwargs) - - def testDocString(self): - self.assertEqual(func3.__doc__, 'Some cool doc string.') - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/framework/python/ops/audio_ops.py b/tensorflow/contrib/framework/python/ops/audio_ops.py deleted file mode 100644 index 0aac269b90f..00000000000 --- a/tensorflow/contrib/framework/python/ops/audio_ops.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# pylint: disable=g-short-docstring-punctuation -"""Audio processing and decoding ops. - -@@decode_wav -@@encode_wav -@@audio_spectrogram -@@mfcc -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -# go/tf-wildcard-import -# pylint: disable=wildcard-import -from tensorflow.python.ops.gen_audio_ops import * -# pylint: enable=wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented - -remove_undocumented(__name__, []) diff --git a/tensorflow/contrib/framework/python/ops/checkpoint_ops.py b/tensorflow/contrib/framework/python/ops/checkpoint_ops.py deleted file mode 100644 index 26146790b65..00000000000 --- a/tensorflow/contrib/framework/python/ops/checkpoint_ops.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Operations for generating and loading vocab remappings.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import init_ops -from tensorflow.python.training import checkpoint_ops - - -# pylint: disable=protected-access,line-too-long -load_and_remap_matrix_initializer = checkpoint_ops._load_and_remap_matrix_initializer -# pylint: enable=line-too-long -load_embedding_initializer = checkpoint_ops._load_embedding_initializer -# pylint: enable=protected-access - - -def load_linear_multiclass_bias_initializer(ckpt_path, - bias_tensor_name, - new_class_vocab_size, - old_class_vocab_file, - new_class_vocab_file, - num_class_oov_buckets=0, - initializer=None, - max_rows_in_memory=-1): - """Loads pre-trained multi-class biases for linear models from checkpoint. - - Wrapper around `load_and_remap_matrix_initializer()` specialized for loading - multi-class bias and remapping according to the provided vocab files. See docs - for `load_and_remap_matrix_initializer()` for more details. In this case, the - provided row_vocab is the class vocabulary, and the expected shape is - `[new_class_vocab_size, 1]`. - - Args: - ckpt_path: Path to the TensorFlow checkpoint (version 2, `TensorBundle`) - from which the old matrix `Tensor` will be loaded. - bias_tensor_name: Tensor name to load from in the checkpoints. - new_class_vocab_size: Number of entries in the new class vocab. - old_class_vocab_file: A scalar `Tensor` of type `string` containing the - path to the old class vocabulary file. - new_class_vocab_file: A scalar `Tensor` of type `string` containing the - path to the new class vocabulary file. - num_class_oov_buckets: `int` specifying the number of out-of-vocabulary - buckets to use for the classes. Must be >= 0. - initializer: Initializer function that accepts a 1-D tensor as the arg to - specify the shape of the returned tensor. If `None`, defaults to using - `zeros_initializer()`. - max_rows_in_memory: `int` specifying the maximum number of rows to load from - the checkpoint at once. If less than or equal to 0, the entire matrix will - be loaded into memory. Setting this arg trades increased disk reads for - lower memory usage. - - Returns: - A variable initializer function. - """ - # Linear multi-class biases should be zero-initialized. - if initializer is None: - initializer = init_ops.zeros_initializer() - - return load_and_remap_matrix_initializer( - ckpt_path=ckpt_path, - old_tensor_name=bias_tensor_name, - new_row_vocab_size=new_class_vocab_size, - new_col_vocab_size=1, - old_row_vocab_file=old_class_vocab_file, - new_row_vocab_file=new_class_vocab_file, - old_col_vocab_file=None, - new_col_vocab_file=None, - num_row_oov_buckets=num_class_oov_buckets, - num_col_oov_buckets=0, - initializer=initializer, - max_rows_in_memory=max_rows_in_memory) - - -def load_variable_slot_initializer(ckpt_path, - old_tensor_name, - primary_partition_info, - new_row_vocab_size, - new_col_vocab_size, - old_row_vocab_file=None, - new_row_vocab_file=None, - old_col_vocab_file=None, - new_col_vocab_file=None, - num_row_oov_buckets=0, - num_col_oov_buckets=0, - initializer=None, - max_rows_in_memory=-1): - """Loads pre-trained multi-class slots for linear models from checkpoint. - - Wrapper around `load_and_remap_matrix_initializer()` specialized for loading - multi-class slots (such as optimizer accumulators) and remapping them - according to the provided vocab files. See docs for - `load_and_remap_matrix_initializer()` for more details. Takes in a - `variable_scope._PartitionInfo` representing the slot's primary `Variable`'s - partitioning. This is necessary since accumulator `Variable` creation ignores - primary scoping and partitioning information. - - Args: - ckpt_path: Path to the TensorFlow checkpoint (version 2, `TensorBundle`) - from which the old matrix `Tensor` will be loaded. - old_tensor_name: Name of the 2-D `Tensor` to load from checkpoint. - primary_partition_info: A `variable_scope._PartitionInfo` containing this - slot's primary `Variable`'s partitioning information. This is used to - calculate the offset and override the partition_info passed to the call to - _initialize. - new_row_vocab_size: `int` specifying the number of entries in - `new_row_vocab_file`. If no row remapping is needed (no row vocab - provided), this should be equal to the number of rows to load from the old - matrix (which can theoretically be smaller than the number of rows in the - old matrix). - new_col_vocab_size: `int` specifying the number of entries in - `new_col_vocab_file`. If no column remapping is needed (no column vocab - provided), this should be equal to the number of columns in the old - matrix. - old_row_vocab_file: A scalar `Tensor` of type `string` containing the - path to the old row vocabulary file. Can be None, which represents no - remapping on the row axis. - new_row_vocab_file: A scalar `Tensor` of type `string` containing the path - to the new row vocabulary file. Can be None, which represents no remapping - on the row axis. - old_col_vocab_file: A scalar `Tensor` of type `string` containing the - path to the old column vocabulary file. Can be None, which represents no - remapping on the column axis. - new_col_vocab_file: A scalar `Tensor` of type `string` containing the path - to the new column vocabulary file. Can be None, which represents no - remapping on the column axis. - num_row_oov_buckets: `int` specifying the number of out-of-vocabulary rows - to append. Must be >= 0. - num_col_oov_buckets: `int` specifying the number of out-of-vocabulary - columns to append. Must be >= 0. - initializer: Initializer function to initialize missing values. Accepts a - 1-D tensor as the arg to specify the shape of the returned tensor. If - `None`, defaults to using `zeros_initializer()`. - max_rows_in_memory: `int` specifying the maximum number of rows to load from - the checkpoint at once. If less than or equal to 0, the entire matrix will - be loaded into memory. Setting this arg trades increased disk reads for - lower memory usage. - - Returns: - A variable initializer function that should be used to initialize a - (potentially partitioned) `Variable` whose complete shape is - `[new_row_vocab_size + num_row_oov_buckets, new_col_vocab_size + - num_col_oov_buckets]`. - - Raises: - TypeError: If `initializer` is specified but not callable. - """ - initializer_fn = load_and_remap_matrix_initializer( - ckpt_path=ckpt_path, - old_tensor_name=old_tensor_name, - new_row_vocab_size=new_row_vocab_size, - new_col_vocab_size=new_col_vocab_size, - old_row_vocab_file=old_row_vocab_file, - new_row_vocab_file=new_row_vocab_file, - old_col_vocab_file=old_col_vocab_file, - new_col_vocab_file=new_col_vocab_file, - num_row_oov_buckets=num_row_oov_buckets, - num_col_oov_buckets=num_col_oov_buckets, - initializer=initializer, - max_rows_in_memory=max_rows_in_memory) - - def _initializer(shape, dtype=dtypes.float32, partition_info=None): - del partition_info # Unused by this override. - return initializer_fn(shape, dtype, partition_info=primary_partition_info) - - return _initializer diff --git a/tensorflow/contrib/framework/python/ops/checkpoint_ops_test.py b/tensorflow/contrib/framework/python/ops/checkpoint_ops_test.py deleted file mode 100644 index 4036c87b6d0..00000000000 --- a/tensorflow/contrib/framework/python/ops/checkpoint_ops_test.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functional tests for the op to generate vocab remapping.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import numpy as np - -from tensorflow.contrib import framework as contrib_framework -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import flags -from tensorflow.python.platform import test -from tensorflow.python.training import saver - -FLAGS = flags.FLAGS -_TESTDATA_PATH = 'contrib/framework/testdata' - - -class LoadMulticlassBiasTest(test.TestCase): - """Tests for the load_linear_multiclass_bias_initializer functionality.""" - - def setUp(self): - ops.reset_default_graph() - dim = 1 - num = 3 - with ops.name_scope('some_scope'): - # Basically from 0 to dim*num-1. - flat_data = math_ops.linspace(0.0, dim * num - 1, dim * num) - bias = variables.Variable( - array_ops.reshape(flat_data, (num, dim)), name='bias') - save = saver.Saver([bias]) - with self.cached_session() as sess: - variables.global_variables_initializer().run() - self.bundle_file = os.path.join(test.get_temp_dir(), 'bias_checkpoint') - save.save(sess, self.bundle_file) - - self.new_class_vocab_file = os.path.join( - test.test_src_dir_path(_TESTDATA_PATH), 'keyword_new.txt') - self.old_class_vocab_file = os.path.join( - test.test_src_dir_path(_TESTDATA_PATH), 'keyword.txt') - self.init_val = 42 - - def _init_val_initializer(shape, dtype=None, partition_info=None): - del dtype, partition_info # Unused by this unit-testing initializer. - return array_ops.tile( - constant_op.constant([[self.init_val]], dtype=dtypes.float32), shape) - - self.initializer = _init_val_initializer - - def test_load_linear_multiclass_bias_initializer(self): - """Tests for the bias initializer wrapper.""" - bias_loading_initializer = ( - contrib_framework.load_linear_multiclass_bias_initializer( - new_class_vocab_file=self.new_class_vocab_file, - old_class_vocab_file=self.old_class_vocab_file, - new_class_vocab_size=4, - bias_tensor_name='some_scope/bias', - ckpt_path=[self.bundle_file], - num_class_oov_buckets=1, - initializer=self.initializer)) - - expected_remapped_bias_vector = np.reshape( - [2, 0, self.init_val, 1, self.init_val], [5, 1]) - - # The new bias vector is of size [4 class vocab + 1 class OOV, 1]. - remapped_bias_vector = variable_scope.get_variable( - name='bias/obtained_bias_vector', - shape=[5, 1], - initializer=bias_loading_initializer, - partitioner=partitioned_variables.fixed_size_partitioner(3)) - - with self.cached_session(): - variables.global_variables_initializer().run() - self.assertAllClose(expected_remapped_bias_vector, - remapped_bias_vector.as_tensor().eval()) - - -class LoadVariableSlotTest(test.TestCase): - """Tests for the load_variable_slot_initializer functionality.""" - - def setUp(self): - ops.reset_default_graph() - dim = 1 - num = 3 - with ops.name_scope('some_scope'): - # Basically from 0 to dim*num-1. - flat_data = math_ops.linspace(0.0, dim * num - 1, dim * num) - accum = variables.Variable( - array_ops.reshape(flat_data, (num, dim)), name='accum') - save = saver.Saver([accum]) - with self.cached_session() as sess: - variables.global_variables_initializer().run() - self.bundle_file = os.path.join(test.get_temp_dir(), 'accum_checkpoint') - save.save(sess, self.bundle_file) - - self.new_class_vocab_file = os.path.join( - test.test_src_dir_path(_TESTDATA_PATH), 'keyword_new.txt') - self.old_class_vocab_file = os.path.join( - test.test_src_dir_path(_TESTDATA_PATH), 'keyword.txt') - self.init_val = 42 - - def _init_val_initializer(shape, dtype=None, partition_info=None): - del dtype, partition_info # Unused by this unit-testing initializer. - return array_ops.tile( - constant_op.constant([[self.init_val]], dtype=dtypes.float32), shape) - - self.initializer = _init_val_initializer - - def test_load_variable_slot_initializer(self): - """Tests for the slot initializer wrapper.""" - # We have an initializer for each of two partitioned variables, which will - # be [3, 1] and [2, 1]. The partitioning information is passed here in - # initializer construction, as opposed to through a variable scope during - # variable creation. - variable_slot_initializer_part_0 = ( - contrib_framework.load_variable_slot_initializer( - new_row_vocab_file=self.new_class_vocab_file, - old_row_vocab_file=self.old_class_vocab_file, - new_row_vocab_size=4, - new_col_vocab_size=1, - primary_partition_info=variable_scope._PartitionInfo( - full_shape=[5, 1], var_offset=[0, 0]), - old_tensor_name='some_scope/accum', - ckpt_path=[self.bundle_file], - num_row_oov_buckets=1, - initializer=self.initializer)) - variable_slot_initializer_part_1 = ( - contrib_framework.load_variable_slot_initializer( - new_row_vocab_file=self.new_class_vocab_file, - old_row_vocab_file=self.old_class_vocab_file, - new_row_vocab_size=4, - new_col_vocab_size=1, - primary_partition_info=variable_scope._PartitionInfo( - full_shape=[5, 1], var_offset=[3, 0]), - old_tensor_name='some_scope/accum', - ckpt_path=[self.bundle_file], - num_row_oov_buckets=1, - initializer=self.initializer)) - - expected_remapped_accum_vector_part_0 = np.reshape([2, 0, self.init_val], - [3, 1]) - - expected_remapped_accum_vector_part_1 = np.reshape([1, self.init_val], - [2, 1]) - - # Since there is no variable scope here, partition_info will be None, so - # if variable_slot_initializer_part_0 and variable_slot_initializer_part_1 - # were instead instances of load_and_remap_matrix_initializer, the part_0 - # obtained vector would still be [2, 0, self.init_val], but the part_1 - # obtained vector would be [2, 0], since the partition_info would default to - # assuming a single partition. - remapped_accum_vector_part_0 = variable_scope.get_variable( - name='accum/obtained_accum_vector_part_0', - shape=[3, 1], - initializer=variable_slot_initializer_part_0) - remapped_accum_vector_part_1 = variable_scope.get_variable( - name='accum/obtained_accum_vector_part_1', - shape=[2, 1], - initializer=variable_slot_initializer_part_1) - - with self.cached_session(): - variables.global_variables_initializer().run() - self.assertAllClose(expected_remapped_accum_vector_part_0, - remapped_accum_vector_part_0.eval()) - self.assertAllClose(expected_remapped_accum_vector_part_1, - remapped_accum_vector_part_1.eval()) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/framework/python/ops/ops.py b/tensorflow/contrib/framework/python/ops/ops.py deleted file mode 100644 index ac451a974d5..00000000000 --- a/tensorflow/contrib/framework/python/ops/ops.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Classes and functions used to construct graphs.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from tensorflow.python.framework import ops - - -__all__ = ['get_graph_from_inputs', - 'get_name_scope'] - - -def get_graph_from_inputs(op_input_list, graph=None): - """Returns the appropriate graph to use for the given inputs. - - 1. If `graph` is provided, we validate that all inputs in `op_input_list` are - from the same graph. - 2. Otherwise, we attempt to select a graph from the first Operation- or - Tensor-valued input in `op_input_list`, and validate that all other - such inputs are in the same graph. - 3. If the graph was not specified and it could not be inferred from - `op_input_list`, we attempt to use the default graph. - - Args: - op_input_list: A list of inputs to an operation, which may include `Tensor`, - `Operation`, and other objects that may be converted to a graph element. - graph: (Optional) The explicit graph to use. - - Raises: - TypeError: If `op_input_list` is not a list or tuple, or if graph is not a - Graph. - ValueError: If a graph is explicitly passed and not all inputs are from it, - or if the inputs are from multiple graphs, or we could not find a graph - and there was no default graph. - - Returns: - The appropriate graph to use for the given inputs. - """ - # pylint: disable=protected-access - return ops._get_graph_from_inputs(op_input_list, graph) - - -def get_name_scope(): - """Returns the current name scope of the default graph. - - For example: - - ```python - with tf.name_scope('scope1'): - with tf.name_scope('scope2'): - print(tf.contrib.framework.get_name_scope()) - ``` - would print the string `scope1/scope2`. - - Returns: - A string representing the current name scope. - """ - return ops.get_default_graph().get_name_scope() diff --git a/tensorflow/contrib/framework/python/ops/ops_test.py b/tensorflow/contrib/framework/python/ops/ops_test.py deleted file mode 100644 index 19bcb5d22e0..00000000000 --- a/tensorflow/contrib/framework/python/ops/ops_test.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""tensor_util tests.""" - -# pylint: disable=unused-import -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework.python.ops import ops as ops_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.platform import test - - -class OpsTest(test.TestCase): - - def testGetGraphFromEmptyInputs(self): - with ops.Graph().as_default() as g0: - self.assertIs(g0, ops_lib.get_graph_from_inputs([])) - - def testGetGraphFromValidInputs(self): - g0 = ops.Graph() - with g0.as_default(): - values = [constant_op.constant(0.0), constant_op.constant(1.0)] - self.assertIs(g0, ops_lib.get_graph_from_inputs(values)) - self.assertIs(g0, ops_lib.get_graph_from_inputs(values, g0)) - with ops.Graph().as_default(): - self.assertIs(g0, ops_lib.get_graph_from_inputs(values)) - self.assertIs(g0, ops_lib.get_graph_from_inputs(values, g0)) - - def testGetGraphFromInvalidInputs(self): - g0 = ops.Graph() - with g0.as_default(): - values = [constant_op.constant(0.0), constant_op.constant(1.0)] - g1 = ops.Graph() - with self.assertRaisesRegexp(ValueError, "not from the passed-in graph"): - ops_lib.get_graph_from_inputs(values, g1) - with g1.as_default(): - values.append(constant_op.constant(2.0)) - with self.assertRaisesRegexp(ValueError, "must be from the same graph"): - ops_lib.get_graph_from_inputs(values) - with self.assertRaisesRegexp(ValueError, "not from the passed-in graph"): - ops_lib.get_graph_from_inputs(values, g0) - with self.assertRaisesRegexp(ValueError, "not from the passed-in graph"): - ops_lib.get_graph_from_inputs(values, g1) - - def testGetNameScope(self): - with ops.name_scope("scope1"): - with ops.name_scope("scope2"): - with ops.name_scope("scope3"): - self.assertEqual("scope1/scope2/scope3", ops_lib.get_name_scope()) - self.assertEqual("scope1/scope2", ops_lib.get_name_scope()) - self.assertEqual("scope1", ops_lib.get_name_scope()) - self.assertEqual("", ops_lib.get_name_scope()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/framework/python/ops/prettyprint_ops.py b/tensorflow/contrib/framework/python/ops/prettyprint_ops.py deleted file mode 100644 index 2637aa15ea9..00000000000 --- a/tensorflow/contrib/framework/python/ops/prettyprint_ops.py +++ /dev/null @@ -1,170 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Functions to provide simpler and prettier logging.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import tensor_array_ops -from tensorflow.python.ops import variables - -__all__ = ["print_op"] - - -def _get_tensor_repr(t, - print_tensor_name=True, - print_tensor_type=True, - print_shape=True, - summarize_indicator_vector=True): - """Return a list of Tensors that summarize the given tensor t.""" - tensor_list = [] - if print_tensor_name and isinstance(t, ops.Tensor): - tensor_list.append(constant_op.constant("Name: " + t.name)) - - if print_tensor_type: - if isinstance(t, ops.Tensor): - t_type_str = "Type: Tensor ({})".format(t.dtype.name) - elif isinstance(t, sparse_tensor.SparseTensor): - t_type_str = "Type: SparseTensor ({})".format(t.dtype.name) - elif isinstance(t, tensor_array_ops.TensorArray): - t_type_str = "Type: TensorArray ({})".format(t.dtype.name) - elif isinstance(t, variables.Variable): - t_type_str = "Type: Variable ({})".format(t.dtype.name) - else: - raise ValueError("t must be a Tensor, SparseTensor, TensorArray or " - "Variable.") - - tensor_list.append(constant_op.constant(t_type_str)) - - if print_shape: - if isinstance(t, sparse_tensor.SparseTensor): - tensor_list.append(constant_op.constant("Shape:")) - tensor_list.append(t.dense_shape) - elif isinstance(t, ops.Tensor): - tensor_list.append(constant_op.constant("Shape: " + str(t.get_shape( - ).dims))) - elif isinstance(t, tensor_array_ops.TensorArray): - tensor_list.append(constant_op.constant("Size:")) - tensor_list.append(t.size()) - - if summarize_indicator_vector and t.dtype == dtypes.bool: - int_tensor = math_ops.cast(t, dtypes.uint8) - tensor_list.append(constant_op.constant("First True in Boolean tensor at:")) - tensor_list.append(math_ops.argmax(int_tensor, 0)) - - if isinstance(t, sparse_tensor.SparseTensor): - tensor_list.append(constant_op.constant("Sparse indices:")) - tensor_list.append(t.indices) - tensor_list.append(constant_op.constant("Sparse values:")) - tensor_list.append(t.values) - elif isinstance(t, ops.Tensor): - tensor_list.append(constant_op.constant("Value:")) - tensor_list.append(t) - elif isinstance(t, tensor_array_ops.TensorArray): - tensor_list.append(constant_op.constant("Value:")) - tensor_list.append(t.stack()) - - return tensor_list - - -def print_op(input_, - data=None, - message=None, - first_n=None, - summarize=20, - print_tensor_name=True, - print_tensor_type=True, - print_shape=True, - summarize_indicator_vector=True, - name=None): - """Creates a print op that will print when a tensor is accessed. - - Wraps the tensor passed in so that whenever that tensor is accessed, - the message `message` is printed, along with the current value of the - tensor `t` and an optional list of other tensors. - - Args: - input_: A Tensor/SparseTensor/TensorArray to print when it is evaluated. - data: A list of other tensors to print. - message: A string message to print as a prefix. - first_n: Only log `first_n` number of times. Negative numbers log always; - this is the default. - summarize: Print this number of elements in the tensor. - print_tensor_name: Print the tensor name. - print_tensor_type: Print the tensor type. - print_shape: Print the tensor's shape. - summarize_indicator_vector: Whether to print the index of the first true - value in an indicator vector (a Boolean tensor). - name: The name to give this op. - - Returns: - A Print op. The Print op returns `input_`. - - Raises: - ValueError: If the tensor `input_` is not a Tensor, SparseTensor or - TensorArray. - - """ - - message = message or "" - if input_ is None: - raise ValueError("input_ must be of type " - "Tensor, SparseTensor or TensorArray") - - tensor_list = _get_tensor_repr(input_, print_tensor_name, print_tensor_type, - print_shape, summarize_indicator_vector) - - if data is not None: - for t in data: - tensor_list.extend(_get_tensor_repr(t, print_tensor_name, - print_tensor_type, print_shape, - summarize_indicator_vector)) - - if isinstance(input_, ops.Tensor) or isinstance(input_, variables.Variable): - input_ = logging_ops.Print(input_, tensor_list, message, first_n, summarize, - name) - elif isinstance(input_, sparse_tensor.SparseTensor): - p = logging_ops.Print( - constant_op.constant([]), tensor_list, message, first_n, summarize, - name) - - with ops.control_dependencies([p]): - input_ = sparse_tensor.SparseTensor( - array_ops.identity(input_.indices), - array_ops.identity(input_.values), - array_ops.identity(input_.dense_shape)) - elif isinstance(input_, tensor_array_ops.TensorArray): - p = logging_ops.Print( - constant_op.constant([]), tensor_list, message, first_n, summarize, - name) - - with ops.control_dependencies([p]): - input_ = tensor_array_ops.TensorArray(dtype=input_.dtype, - handle=input_.handle, - flow=input_.flow) - else: - raise ValueError("input_ must be of type " - "Tensor, SparseTensor or TensorArray") - - return input_ diff --git a/tensorflow/contrib/framework/python/ops/prettyprint_ops_test.py b/tensorflow/contrib/framework/python/ops/prettyprint_ops_test.py deleted file mode 100644 index c104c51fef2..00000000000 --- a/tensorflow/contrib/framework/python/ops/prettyprint_ops_test.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# pylint: disable=unused-import -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib.framework.python.ops import prettyprint_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import tensor_array_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class PrettyPrintOpsTest(test.TestCase): - - def testPrintTensorPassthrough(self): - a = constant_op.constant([1]) - a = prettyprint_ops.print_op(a) - with self.cached_session(): - self.assertEqual(a.eval(), constant_op.constant([1]).eval()) - - def testPrintSparseTensorPassthrough(self): - a = sparse_tensor.SparseTensor( - indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4]) - b = sparse_tensor.SparseTensor( - indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4]) - a = prettyprint_ops.print_op(a) - with self.cached_session(): - self.assertAllEqual( - sparse_ops.sparse_tensor_to_dense(a).eval(), - sparse_ops.sparse_tensor_to_dense(b).eval()) - - def testPrintTensorArrayPassthrough(self): - a = tensor_array_ops.TensorArray( - size=2, dtype=dtypes.int32, clear_after_read=False) - a = a.write(1, 1) - a = a.write(0, 0) - a = prettyprint_ops.print_op(a) - with self.cached_session(): - self.assertAllEqual(a.stack().eval(), constant_op.constant([0, 1]).eval()) - - def testPrintVariable(self): - a = variables.Variable(1.0) - a = prettyprint_ops.print_op(a) - with self.cached_session(): - variables.global_variables_initializer().run() - a.eval() - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/framework/python/ops/script_ops.py b/tensorflow/contrib/framework/python/ops/script_ops.py deleted file mode 100644 index 21b00fcdaa8..00000000000 --- a/tensorflow/contrib/framework/python/ops/script_ops.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Script Language Operators. - -@@py_func -""" - -# pylint: disable=g-bad-name -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops.script_ops import py_func as _py_func -from tensorflow.python.util import nest - -__all__ = ['py_func'] - - -def py_func(func, - args=(), - kwargs=None, - output_types=None, - output_shapes=None, - stateful=True, - name=None): - """Wraps a python function and uses it as a TensorFlow op. - - This function is a wrapper around `tf.compat.v1.py_func` and improve it with - kwargs - and output_shapes. Further it changed some argument names. - - Given a python function `func`, which takes numpy arrays as its - inputs and returns numpy arrays as its outputs, wrap this function as an - operation in a TensorFlow graph. The following snippet constructs a simple - TensorFlow graph that invokes the `np.sinh()` NumPy function as a operation - in the graph: - - ```python - def my_func(x): - # x will be a numpy array with the contents of the placeholder below - return np.sinh(x) - inp = tf.compat.v1.placeholder(tf.float32) - y = tf.compat.v1.py_func(my_func, [inp], tf.float32) - ``` - - - **N.B.** The `tf.compat.v1.py_func()` operation has the following known - limitations: - - * The body of the function (i.e. `func`) will not be serialized in a - `GraphDef`. Therefore, you should not use this function if you need to - serialize your model and restore it in a different environment. - - * The operation must run in the same address space as the Python program - that calls `tf.compat.v1.py_func()`. If you are using distributed - TensorFlow, you - must run a `tf.distribute.Server` in the same process as the program that - calls - `tf.compat.v1.py_func()` and you must pin the created operation to a device - in that - server (e.g. using `with tf.device():`). - - Args: - func: A Python function, which accepts a list of NumPy `ndarray` objects - having element types that match the corresponding `tf.Tensor` objects in - `inp`, and returns a list of `ndarray` objects (or a single `ndarray`) - having element types that match the corresponding values in `Tout`. - args: A list of `Tensor` objects. - kwargs: A dict with `Tensor` objects as values. - output_types: A nested structure of tensorflow data types or a single - tensorflow data type if there is only one, indicating what `func` returns. - output_shapes: Same as output_types, except the types are replaces with - shapes (optional). - stateful: (Boolean.) If True, the function should be considered stateful. If - a function is stateless, when given the same input it will return the same - output and have no observable side effects. Optimizations such as common - subexpression elimination are only performed on stateless operations. - name: A name for the operation (optional). - - Returns: - Tensorflow op that wraps the input python function. - """ - - if kwargs is None: - kwargs = {} - - if not isinstance(args, (list, tuple)): - raise TypeError('args must be list and not {}. args: {}'.format( - type(args), args)) - - if not isinstance(kwargs, dict): - raise TypeError('kwargs must be dict and not {}. args: {}'.format( - type(kwargs), kwargs)) - - # For dynamic type inference use callable output_types and output_shapes - if callable(output_types): - # If callable assume same signature and call with tensors and get the types - output_types = output_types(*args, **kwargs) - if callable(output_shapes): - # If callable assume same signature and call with tensors and get the shapes - output_shapes = output_shapes(*args, **kwargs) - - flat_output_types = nest.flatten(output_types) - args = (args, kwargs) - flat_args = nest.flatten(args) - - def python_function_wrapper(*py_args): - py_args, py_kwargs = nest.pack_sequence_as(args, py_args) - - ret = func(*py_args, **py_kwargs) - # TODO(alextp): Catch Exceptions and improve msg, because tensorflow - # ist not able to preserve the traceback, i.e. the Exceptions does not - # contain any information where the Exception was raised. - nest.assert_shallow_structure(output_types, ret) - return nest.flatten(ret) - - flat_values = _py_func( - python_function_wrapper, - flat_args, - flat_output_types, - stateful=stateful, - name=name) - - if output_shapes is not None: - # I am not sure if this is nessesary - output_shapes = nest.map_structure_up_to(output_types, - tensor_shape.as_shape, - output_shapes) - - flattened_shapes = nest.flatten(output_shapes) - for ret_t, shape in zip(flat_values, flattened_shapes): - ret_t.set_shape(shape) - - return nest.pack_sequence_as(output_types, flat_values) diff --git a/tensorflow/contrib/framework/python/ops/sort_ops.py b/tensorflow/contrib/framework/python/ops/sort_ops.py deleted file mode 100644 index 42184a4e55e..00000000000 --- a/tensorflow/contrib/framework/python/ops/sort_ops.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Support for sorting tensors. - -@@argsort -@@sort -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import sort_ops - -sort = sort_ops.sort -argsort = sort_ops.argsort diff --git a/tensorflow/contrib/framework/python/ops/variables.py b/tensorflow/contrib/framework/python/ops/variables.py deleted file mode 100644 index 334262daf08..00000000000 --- a/tensorflow/contrib/framework/python/ops/variables.py +++ /dev/null @@ -1,878 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Variable functions.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import re - -from tensorflow.contrib.framework.python.ops import add_arg_scope as contrib_add_arg_scope -from tensorflow.contrib.framework.python.ops import gen_variable_ops -from tensorflow.contrib.util import loader -from tensorflow.core.protobuf import saver_pb2 -from tensorflow.python import pywrap_tensorflow -from tensorflow.python.framework import device as tf_device -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import resource_loader -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import saver as tf_saver -from tensorflow.python.training import training_util -from tensorflow.python.util.deprecation import deprecated - - -__all__ = ['add_model_variable', - 'assert_global_step', - 'assert_or_get_global_step', - 'assign_from_checkpoint', - 'assign_from_checkpoint_fn', - 'assign_from_values', - 'assign_from_values_fn', - 'create_global_step', - 'filter_variables', - 'get_global_step', - 'get_or_create_global_step', - 'get_local_variables', - 'get_model_variables', - 'get_trainable_variables', - 'get_unique_variable', - 'get_variables_by_name', - 'get_variables_by_suffix', - 'get_variable_full_name', - 'get_variables_to_restore', - 'get_variables', - 'global_variable', - 'local_variable', - 'model_variable', - 'variable', - 'VariableDeviceChooser', - 'zero_initializer'] - - -def zero_initializer(ref, use_locking=True, name="zero_initializer"): - """Initialize 'ref' with all zeros, ref tensor should be uninitialized. - - If already initialized, you will get ValueError. This op is intended to - save memory during initialization. - Args: - ref: ref of the tensor need to be zero initialized. - name: optional name for this operation. - - Returns: - ref that initialized. - Raises: - ValueError: If ref tensor is initialized. - """ - loader.load_op_library( - resource_loader.get_path_to_datafile('_variable_ops.so')) - if resource_variable_ops.is_resource_variable(ref): - return gen_variable_ops.zero_var_initializer( - ref.handle, shape=ref.shape, dtype=ref.dtype, name=name) - else: - return gen_variable_ops.zero_initializer(ref, name=name) - - -@deprecated(None, 'Please switch to tf.train.assert_global_step') -def assert_global_step(global_step_tensor): - training_util.assert_global_step(global_step_tensor) - - -def assert_or_get_global_step(graph=None, global_step_tensor=None): - """Verifies that a global step tensor is valid or gets one if None is given. - - If `global_step_tensor` is not None, check that it is a valid global step - tensor (using `assert_global_step`). Otherwise find a global step tensor using - `get_global_step` and return it. - - Args: - graph: The graph to find the global step tensor for. - global_step_tensor: The tensor to check for suitability as a global step. If - None is given (the default), find a global step tensor. - - Returns: - A tensor suitable as a global step, or `None` if none was provided and none - was found. - """ - if global_step_tensor is None: - # Get the global step tensor the same way the supervisor would. - global_step_tensor = get_global_step(graph) - else: - assert_global_step(global_step_tensor) - return global_step_tensor - - -@deprecated(None, 'Please switch to tf.train.get_global_step') -def get_global_step(graph=None): - return training_util.get_global_step(graph) - - -@deprecated(None, 'Please switch to tf.train.create_global_step') -def create_global_step(graph=None): - """Create global step tensor in graph. - - This API is deprecated. Use core framework training version instead. - - Args: - graph: The graph in which to create the global step tensor. If missing, use - default graph. - - Returns: - Global step tensor. - - Raises: - ValueError: if global step tensor is already defined. - """ - return training_util.create_global_step(graph) - - -@deprecated(None, 'Please switch to tf.train.get_or_create_global_step') -def get_or_create_global_step(graph=None): - """Returns and create (if necessary) the global step tensor. - - Args: - graph: The graph in which to create the global step tensor. If missing, use - default graph. - - Returns: - The global step tensor. - """ - return training_util.get_or_create_global_step(graph) - - -def local_variable(initial_value, - validate_shape=True, - name=None, - use_resource=None): - """Create a variable with a value and add it to `GraphKeys.LOCAL_VARIABLES`. - - Args: - initial_value: See variables.Variable.__init__. - validate_shape: See variables.Variable.__init__. - name: See variables.Variable.__init__. - use_resource: If `True` use a ResourceVariable instead of a Variable. - - Returns: - New variable. - """ - return variable_scope.variable( - initial_value, - trainable=False, - collections=[ops.GraphKeys.LOCAL_VARIABLES], - validate_shape=validate_shape, - use_resource=use_resource, - name=name) - - -def global_variable(initial_value, - validate_shape=True, - name=None, - use_resource=None): - """Create a variable with a value and add it to `GraphKeys.GLOBAL_VARIABLES`. - - Args: - initial_value: See variables.Variable.__init__. - validate_shape: See variables.Variable.__init__. - name: See variables.Variable.__init__. - use_resource: If `True` use a ResourceVariable instead of a Variable. - - Returns: - New variable. - """ - return variable_scope.variable( - initial_value, - trainable=False, - collections=[ops.GraphKeys.GLOBAL_VARIABLES], - validate_shape=validate_shape, - use_resource=use_resource, - name=name) - - -@contrib_add_arg_scope -def variable(name, - shape=None, - dtype=None, - initializer=None, - regularizer=None, - trainable=True, - collections=None, - caching_device=None, - device=None, - partitioner=None, - custom_getter=None, - use_resource=None, - synchronization=variables.VariableSynchronization.AUTO, - aggregation=variables.VariableAggregation.NONE): - """Gets an existing variable with these parameters or creates a new one. - - Args: - name: the name of the new or existing variable. - shape: shape of the new or existing variable. - dtype: type of the new or existing variable (defaults to `DT_FLOAT`). - initializer: initializer for the variable if one is created. - regularizer: a (Tensor -> Tensor or None) function; the result of applying - it on a newly created variable will be added to the collection - GraphKeys.REGULARIZATION_LOSSES and can be used for regularization. - trainable: If `True` also add the variable to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - collections: A list of collection names to which the Variable will be added. - If None it would default to `tf.GraphKeys.GLOBAL_VARIABLES`. - caching_device: Optional device string or function describing where the - Variable should be cached for reading. Defaults to the Variable's device. - device: Optional device to place the variable. It can be an string or a - function that is called to get the device for the variable. - partitioner: Optional callable that accepts a fully defined `TensorShape` - and dtype of the `Variable` to be created, and returns a list of - partitions for each axis (currently only one axis can be partitioned). - custom_getter: Callable that allows overwriting the internal get_variable - method and has to have the same signature. - use_resource: If `True` use a ResourceVariable instead of a Variable. - synchronization: Indicates when a distributed a variable will be aggregated. - Accepted values are constants defined in the class - `tf.VariableSynchronization`. By default the synchronization is set to - `AUTO` and the current `DistributionStrategy` chooses when to synchronize. - aggregation: Indicates how a distributed variable will be aggregated. - Accepted values are constants defined in the class - `tf.VariableAggregation`. - - Returns: - The created or existing variable. - """ - collections = list(collections if collections is not None else - [ops.GraphKeys.GLOBAL_VARIABLES]) - - # Remove duplicates - collections = list(set(collections)) - getter = variable_scope.get_variable - if custom_getter is not None: - getter = functools.partial( - custom_getter, reuse=variable_scope.get_variable_scope().reuse) - with ops.device(device or ''): - return getter( - name, - shape=shape, - dtype=dtype, - initializer=initializer, - regularizer=regularizer, - trainable=trainable, - collections=collections, - caching_device=caching_device, - partitioner=partitioner, - use_resource=use_resource, - synchronization=synchronization, - aggregation=aggregation) - - -@contrib_add_arg_scope -def model_variable(name, - shape=None, - dtype=dtypes.float32, - initializer=None, - regularizer=None, - trainable=True, - collections=None, - caching_device=None, - device=None, - partitioner=None, - custom_getter=None, - use_resource=None, - synchronization=variables.VariableSynchronization.AUTO, - aggregation=variables.VariableAggregation.NONE): - """Gets an existing model variable with these parameters or creates a new one. - - Args: - name: the name of the new or existing variable. - shape: shape of the new or existing variable. - dtype: type of the new or existing variable (defaults to `DT_FLOAT`). - initializer: initializer for the variable if one is created. - regularizer: a (Tensor -> Tensor or None) function; the result of applying - it on a newly created variable will be added to the collection - GraphKeys.REGULARIZATION_LOSSES and can be used for regularization. - trainable: If `True` also add the variable to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - collections: A list of collection names to which the Variable will be added. - Note that the variable is always also added to the - `GraphKeys.GLOBAL_VARIABLES` and `GraphKeys.MODEL_VARIABLES` collections. - caching_device: Optional device string or function describing where the - Variable should be cached for reading. Defaults to the Variable's device. - device: Optional device to place the variable. It can be an string or a - function that is called to get the device for the variable. - partitioner: Optional callable that accepts a fully defined `TensorShape` - and dtype of the `Variable` to be created, and returns a list of - partitions for each axis (currently only one axis can be partitioned). - custom_getter: Callable that allows overwriting the internal get_variable - method and has to have the same signature. - use_resource: If `True` use a ResourceVariable instead of a Variable. - synchronization: Indicates when a distributed a variable will be aggregated. - Accepted values are constants defined in the class - `tf.VariableSynchronization`. By default the synchronization is set to - `AUTO` and the current `DistributionStrategy` chooses when to synchronize. - aggregation: Indicates how a distributed variable will be aggregated. - Accepted values are constants defined in the class - `tf.VariableAggregation`. - - Returns: - The created or existing variable. - """ - collections = list(collections or []) - collections += [ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.MODEL_VARIABLES] - var = variable( - name, - shape=shape, - dtype=dtype, - initializer=initializer, - regularizer=regularizer, - trainable=trainable, - collections=collections, - caching_device=caching_device, - device=device, - partitioner=partitioner, - custom_getter=custom_getter, - use_resource=use_resource, - synchronization=synchronization, - aggregation=aggregation) - return var - - -def add_model_variable(var): - """Adds a variable to the `GraphKeys.MODEL_VARIABLES` collection. - - Args: - var: a variable. - """ - if var not in ops.get_collection(ops.GraphKeys.MODEL_VARIABLES): - ops.add_to_collection(ops.GraphKeys.MODEL_VARIABLES, var) - - -def get_variables(scope=None, - suffix=None, - collection=ops.GraphKeys.GLOBAL_VARIABLES): - """Gets the list of variables, filtered by scope and/or suffix. - - Args: - scope: an optional scope for filtering the variables to return. Can be a - variable scope or a string. - suffix: an optional suffix for filtering the variables to return. - collection: in which collection search for. Defaults to - `GraphKeys.GLOBAL_VARIABLES`. - - Returns: - a list of variables in collection with scope and suffix. - """ - if isinstance(scope, variable_scope.VariableScope): - scope = scope.name - if suffix is not None: - if ':' not in suffix: - suffix += ':' - scope = (scope or '') + '.*' + suffix - return ops.get_collection(collection, scope) - - -def get_model_variables(scope=None, suffix=None): - """Gets the list of model variables, filtered by scope and/or suffix. - - Args: - scope: an optional scope for filtering the variables to return. - suffix: an optional suffix for filtering the variables to return. - - Returns: - a list of variables in collection with scope and suffix. - """ - return get_variables(scope, suffix, ops.GraphKeys.MODEL_VARIABLES) - - -def get_local_variables(scope=None, suffix=None): - """Gets the list of local variables, filtered by scope and/or suffix. - - Args: - scope: an optional scope for filtering the variables to return. - suffix: an optional suffix for filtering the variables to return. - - Returns: - a list of variables in collection with scope and suffix. - """ - return get_variables(scope, suffix, ops.GraphKeys.LOCAL_VARIABLES) - - -def get_trainable_variables(scope=None, suffix=None): - """Gets the list of trainable variables, filtered by scope and/or suffix. - - Args: - scope: an optional scope for filtering the variables to return. - suffix: an optional suffix for filtering the variables to return. - - Returns: - a list of variables in the trainable collection with scope and suffix. - """ - return get_variables(scope, suffix, ops.GraphKeys.TRAINABLE_VARIABLES) - - -def get_variables_to_restore(include=None, exclude=None): - """Gets the list of the variables to restore. - - Args: - include: an optional list/tuple of scope strings for filtering which - variables from the VARIABLES collection to include. None would include all - the variables. - exclude: an optional list/tuple of scope strings for filtering which - variables from the VARIABLES collection to exclude. None it would not - exclude any. - - Returns: - a list of variables to restore. - - Raises: - TypeError: include or exclude is provided but is not a list or a tuple. - """ - if include is None: - # Include all variables. - vars_to_include = get_variables() - else: - if not isinstance(include, (list, tuple)): - raise TypeError('include is provided but is not a list or a tuple.') - vars_to_include = [] - for scope in include: - vars_to_include += get_variables(scope) - vars_to_exclude = set() - if exclude is not None: - if not isinstance(exclude, (list, tuple)): - raise TypeError('exclude is provided but is not a list or a tuple.') - for scope in exclude: - vars_to_exclude |= set(get_variables(scope)) - # Exclude the variables in vars_to_exclude - return [v for v in vars_to_include if v not in vars_to_exclude] - - -def get_variables_by_suffix(suffix, scope=None): - """Gets the list of variables that end with the given suffix. - - Args: - suffix: suffix for filtering the variables to return. - scope: an optional scope for filtering the variables to return. - - Returns: - a copied list of variables with the given name and prefix. - """ - return get_variables(scope=scope, suffix=suffix) - - -def get_variables_by_name(given_name, scope=None): - """Gets the list of variables that were given that name. - - Args: - given_name: name given to the variable without any scope. - scope: an optional scope for filtering the variables to return. - - Returns: - a copied list of variables with the given name and scope. - """ - suffix = '/' + given_name + ':|^' + given_name + ':' - return get_variables(scope=scope, suffix=suffix) - - -def get_unique_variable(var_op_name): - """Gets the variable uniquely identified by that var_op_name. - - Args: - var_op_name: the full name of the variable op, including the scope. - - Returns: - a tensorflow variable. - - Raises: - ValueError: if no variable uniquely identified by the name exists. - """ - candidates = get_variables(scope=var_op_name) - if not candidates: - raise ValueError('Couldn\'t find variable %s' % var_op_name) - - for candidate in candidates: - if candidate.op.name == var_op_name: - return candidate - raise ValueError('Variable %s does not uniquely identify a variable' % - var_op_name) - - -def assign_from_values(var_names_to_values): - """Creates an assignment operation from a given mapping. - - This function provides a mechanism for performing assignment of variables - to values in a way that does not fill the graph with large assignment values. - - Args: - var_names_to_values: A map from variable names to values. - - Returns: - assign_op: An `Operation` that assigns each of the given variables to the - requested values. - feed_dict: The feed dictionary to use when evaluating `assign_op`. - - Raises: - ValueError: if any of the given variable names were not found. - """ - feed_dict = {} - assign_ops = [] - - for var_name in var_names_to_values: - var_value = var_names_to_values[var_name] - var = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES, var_name) - if not var: - raise ValueError('Variable %s wasn\'t found' % var_name) - elif len(var) > 1: - # tf.compat.v1.get_collection is just a filter on the prefix: find the exact match: - found = False - for v in var: - if v.op.name == var_name: - var = v - found = True - break - - if not found: - raise ValueError('Variable %s doesn\'t uniquely identify a variable' % - var_name) - else: - var = var[0] - - # TODO(nsilberman): ensure placeholder and assign are on the same device. - # Assign a placeholder to the value that will be filled later. - placeholder_name = 'placeholder/' + var.op.name - placeholder_value = array_ops.placeholder( - dtype=var.dtype.base_dtype, - shape=var.get_shape(), - name=placeholder_name) - assign_ops.append(var.assign(placeholder_value)) - - feed_dict[placeholder_value] = var_value.reshape(var.get_shape()) - - assign_op = control_flow_ops.group(*assign_ops) - return assign_op, feed_dict - - -def assign_from_values_fn(var_names_to_values): - """Returns a function that assigns specific variables from the given values. - - This function provides a mechanism for performing assignment of variables - to values in a way that does not fill the graph with large assignment values. - - Args: - var_names_to_values: A map from variable names to values. - - Returns: - A function that takes a single argument, a `tf.compat.v1.Session`, that - applies the - assignment operation. - - Raises: - ValueError: if any of the given variable names were not found. - """ - assign_op, feed_dict = assign_from_values(var_names_to_values) - - def callback(session): - return session.run(assign_op, feed_dict) - - return callback - - -# pylint: disable=protected-access -# Currently variable_scope doesn't provide very good APIs to access -# all variables under scope and retrieve and check existing scopes. -def get_variable_full_name(var): - """Returns the full name of a variable. - - For normal Variables, this is the same as the var.op.name. For - sliced or PartitionedVariables, this name is the same for all the - slices/partitions. In both cases, this is normally the name used in - a checkpoint file. - - Args: - var: A `Variable` object. - - Returns: - A string that is the full name. - """ - if var._save_slice_info: - return var._save_slice_info.full_name - else: - return var.op.name - - -# TODO(nsilberman): add flag to load exponential moving averages instead -# -# TODO(sguada): Update docs in slim/g3doc/index.md to describe -# the new feature where the var_list dictionary can have values that -# are each a list of Variables. -def assign_from_checkpoint(model_path, var_list, ignore_missing_vars=False): - """Creates an operation to assign specific variables from a checkpoint. - - Args: - model_path: The full path to the model checkpoint. To get latest checkpoint - use `model_path = tf.train.latest_checkpoint(checkpoint_dir)` - var_list: A list of (possibly partitioned) `Variable` objects or a - dictionary mapping names in the checkpoint to the corresponding variables - or list of variables to initialize from that checkpoint value. For - partitioned Variables, the name in the checkpoint must be the full - variable, not the name of the partitioned variable, eg. "my_var" rather - than "my_var/part_4". If empty, returns no_op(), {}. - ignore_missing_vars: Boolean, if True ignore variables missing in the - checkpoint with a warning instead of failing. - - Returns: - the restore_op and the feed_dict that need to be run to restore var_list. - - Raises: - ValueError: If `ignore_missing_vars` is False and the checkpoint specified - at `model_path` is missing one of the variables in `var_list`. - """ - # Normalize var_list into a dictionary mapping names in the - # checkpoint to the list of variables to initialize from that - # checkpoint variable. Sliced (including partitioned) variables will - # end up under the same key. - grouped_vars = {} - if isinstance(var_list, (tuple, list)): - for var in var_list: - ckpt_name = get_variable_full_name(var) - if ckpt_name not in grouped_vars: - grouped_vars[ckpt_name] = [] - grouped_vars[ckpt_name].append(var) - - else: - for ckpt_name, value in var_list.items(): - if isinstance(value, (tuple, list)): - grouped_vars[ckpt_name] = value - else: - grouped_vars[ckpt_name] = [value] - - # Read each checkpoint entry. Create a placeholder variable and - # add the (possibly sliced) data from the checkpoint to the feed_dict. - reader = pywrap_tensorflow.NewCheckpointReader(model_path) - feed_dict = {} - assign_ops = [] - for ckpt_name in grouped_vars: - if not reader.has_tensor(ckpt_name): - log_str = 'Checkpoint is missing variable [%s]' % ckpt_name - if ignore_missing_vars: - logging.warning(log_str) - continue - else: - raise ValueError(log_str) - ckpt_value = reader.get_tensor(ckpt_name) - - for var in grouped_vars[ckpt_name]: - placeholder_tensor = array_ops.placeholder( - dtype=var.dtype.base_dtype, - shape=var.get_shape(), - name='placeholder/' + var.op.name) - assign_ops.append(var.assign(placeholder_tensor)) - - if not var._save_slice_info: - if var.get_shape() != ckpt_value.shape: - raise ValueError( - 'Total size of new array must be unchanged for %s ' - 'lh_shape: [%s], rh_shape: [%s]' % - (ckpt_name, str(ckpt_value.shape), str(var.get_shape()))) - - feed_dict[placeholder_tensor] = ckpt_value.reshape(ckpt_value.shape) - else: - slice_dims = zip(var._save_slice_info.var_offset, - var._save_slice_info.var_shape) - slice_dims = [(start, start + size) for (start, size) in slice_dims] - slice_dims = [slice(*x) for x in slice_dims] - slice_value = ckpt_value[slice_dims] - slice_value = slice_value.reshape(var._save_slice_info.var_shape) - feed_dict[placeholder_tensor] = slice_value - - assign_op = control_flow_ops.group(*assign_ops) - return assign_op, feed_dict - - -# pylint: enable=protected-access - - -def assign_from_checkpoint_fn(model_path, - var_list, - ignore_missing_vars=False, - reshape_variables=False): - """Returns a function that assigns specific variables from a checkpoint. - - If ignore_missing_vars is True and no variables are found in the checkpoint - it returns None. - - Args: - model_path: The full path to the model checkpoint. To get latest checkpoint - use `model_path = tf.train.latest_checkpoint(checkpoint_dir)` - var_list: A list of `Variable` objects or a dictionary mapping names in the - checkpoint to the corresponding variables to initialize. If empty or - `None`, it would return `no_op(), None`. - ignore_missing_vars: Boolean, if True it would ignore variables missing in - the checkpoint with a warning instead of failing. - reshape_variables: Boolean, if True it would automatically reshape variables - which are of different shape then the ones stored in the checkpoint but - which have the same number of elements. - - Returns: - A function that takes a single argument, a `tf.compat.v1.Session`, that - applies the - assignment operation. If no matching variables were found in the checkpoint - then `None` is returned. - - Raises: - ValueError: If var_list is empty. - """ - if not var_list: - raise ValueError('var_list cannot be empty') - if ignore_missing_vars: - reader = pywrap_tensorflow.NewCheckpointReader(model_path) - if isinstance(var_list, dict): - var_dict = var_list - else: - var_dict = {var.op.name: var for var in var_list} - available_vars = {} - for var in var_dict: - if reader.has_tensor(var): - available_vars[var] = var_dict[var] - else: - logging.warning('Variable %s missing in checkpoint %s', var, model_path) - var_list = available_vars - if var_list: - saver = tf_saver.Saver( - var_list, - reshape=reshape_variables, - write_version=saver_pb2.SaverDef.V1) - - def callback(session): - saver.restore(session, model_path) - - return callback - else: - logging.warning('No Variables to restore') - return None - - -class VariableDeviceChooser(object): - """Device chooser for variables. - - When using a parameter server it will assign them in a round-robin fashion. - When not using a parameter server it allows GPU or CPU placement. - """ - - def __init__(self, - num_tasks=0, - job_name='ps', - device_type='CPU', - device_index=0, - replica=None): - """Initialize VariableDeviceChooser. - - Usage: - To use with 2 parameter servers: - VariableDeviceChooser(2) - - To use without parameter servers: - VariableDeviceChooser() - VariableDeviceChooser(device_type='GPU') # For GPU placement - - Args: - num_tasks: number of tasks. - job_name: String, a name for the parameter server job. - device_type: Optional device type string (e.g. "CPU" or "GPU") - device_index: int. Optional device index. If left unspecified, device - represents 'any' device_index. - """ - self._job_name = job_name - self._device_type = device_type - self._device_index = device_index - self._replica = replica - self._num_tasks = num_tasks - self._next_task_id = 0 - - def __call__(self, op): - device_spec = tf_device.DeviceSpec( - replica=self._replica, - device_type=self._device_type, - device_index=self._device_index) - if self._num_tasks > 0: - task_id = self._next_task_id - self._next_task_id = (self._next_task_id + 1) % self._num_tasks - device_spec.job = self._job_name - device_spec.task = task_id - return device_spec.to_string() - - -def filter_variables(var_list, - include_patterns=None, - exclude_patterns=None, - reg_search=True): - """Filter a list of variables using regular expressions. - - First includes variables according to the list of include_patterns. - Afterwards, eliminates variables according to the list of exclude_patterns. - - For example, one can obtain a list of variables with the weights of all - convolutional layers (depending on the network definition) by: - - ```python - variables = tf.contrib.framework.get_model_variables() - conv_weight_variables = tf.contrib.framework.filter_variables( - variables, - include_patterns=['Conv'], - exclude_patterns=['biases', 'Logits']) - ``` - - Args: - var_list: list of variables. - include_patterns: list of regular expressions to include. Defaults to None, - which means all variables are selected according to the include rules. A - variable is included if it matches any of the include_patterns. - exclude_patterns: list of regular expressions to exclude. Defaults to None, - which means all variables are selected according to the exclude rules. A - variable is excluded if it matches any of the exclude_patterns. - reg_search: boolean. If True (default), performs re.search to find matches - (i.e. pattern can match any substring of the variable name). If False, - performs re.match (i.e. regexp should match from the beginning of the - variable name). - - Returns: - filtered list of variables. - """ - if reg_search: - reg_exp_func = re.search - else: - reg_exp_func = re.match - - # First include variables. - if include_patterns is None: - included_variables = list(var_list) - else: - included_variables = [] - for var in var_list: - if any(reg_exp_func(ptrn, var.name) for ptrn in include_patterns): - included_variables.append(var) - - # Afterwards, exclude variables. - if exclude_patterns is None: - filtered_variables = included_variables - else: - filtered_variables = [] - for var in included_variables: - if not any(reg_exp_func(ptrn, var.name) for ptrn in exclude_patterns): - filtered_variables.append(var) - - return filtered_variables diff --git a/tensorflow/contrib/framework/python/ops/variables_test.py b/tensorflow/contrib/framework/python/ops/variables_test.py deleted file mode 100644 index c223df5b6e9..00000000000 --- a/tensorflow/contrib/framework/python/ops/variables_test.py +++ /dev/null @@ -1,1426 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""variables tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import tempfile - -import numpy as np -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.framework.python.ops import variables as variables_lib2 -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test -from tensorflow.python.training import device_setter -from tensorflow.python.training import saver as saver_lib - - -class LocalVariableTest(test.TestCase): - - def test_local_variable(self): - with self.cached_session() as sess: - self.assertEquals([], variables_lib.local_variables()) - value0 = 42 - variables_lib2.local_variable(value0) - value1 = 43 - variables_lib2.local_variable(value1) - variables = variables_lib.local_variables() - self.assertEquals(2, len(variables)) - self.assertRaises(errors_impl.OpError, sess.run, variables) - variables_lib.variables_initializer(variables).run() - self.assertAllEqual(set([value0, value1]), set(sess.run(variables))) - - def testLocalVariableNameAndShape(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.local_variable([1, 1, 1, 1, 1], name='a') - self.assertEquals(a.op.name, 'A/a') - self.assertListEqual(a.get_shape().as_list(), [5]) - self.assertListEqual([a], variables_lib2.get_local_variables()) - - def testLocalVariableNotInAllVariables(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.local_variable(0) - self.assertFalse(a in variables_lib.global_variables()) - self.assertTrue(a in variables_lib.local_variables()) - - def testLocalVariableNotInVariablesToRestore(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.local_variable(0) - self.assertFalse(a in variables_lib2.get_variables_to_restore()) - self.assertTrue(a in variables_lib.local_variables()) - - def testGetVariablesDontReturnsTransients(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - variables_lib2.local_variable(0) - with variable_scope.variable_scope('B'): - variables_lib2.local_variable(0) - self.assertEquals([], variables_lib2.get_variables('A')) - self.assertEquals([], variables_lib2.get_variables('B')) - - def testGetLocalVariablesReturnsTransients(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.local_variable(0) - with variable_scope.variable_scope('B'): - b = variables_lib2.local_variable(0) - self.assertEquals([a], variables_lib2.get_local_variables('A')) - self.assertEquals([b], variables_lib2.get_local_variables('B')) - - def testInitializedVariableValue(self): - with self.cached_session() as sess: - a = variables_lib2.local_variable([0, 0, 0, 0, 0], name='a') - sess.run(variables_lib.local_variables_initializer()) - self.assertAllEqual(a.eval(), [0] * 5) - - def testResourceVariable(self): - a = variables_lib2.local_variable(0) - b = variables_lib2.local_variable(0, use_resource=True) - self.assertTrue(isinstance(a, variables_lib.Variable)) - self.assertFalse(isinstance(a, resource_variable_ops.ResourceVariable)) - self.assertTrue(isinstance(b, resource_variable_ops.ResourceVariable)) - - -class GlobalVariableTest(test.TestCase): - - def test_global_variable(self): - with self.cached_session() as sess: - self.assertEquals([], variables_lib.global_variables()) - value0 = 42 - variables_lib2.global_variable(value0) - value1 = 43 - variables_lib2.global_variable(value1) - variables = variables_lib.global_variables() - self.assertEquals(2, len(variables)) - with self.assertRaisesOpError( - 'Attempting to use uninitialized value Variable'): - sess.run(variables) - variables_lib.variables_initializer(variables).run() - self.assertAllEqual(set([value0, value1]), set(sess.run(variables))) - - def testVariableNameAndShape(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.global_variable([1, 1, 1, 1, 1], name='a') - self.assertEquals(a.op.name, 'A/a') - self.assertListEqual(a.get_shape().as_list(), [5]) - self.assertListEqual([a], variables_lib.global_variables()) - - def testGlobalVariableNotInLocalVariables(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.global_variable(0) - self.assertFalse(a in variables_lib.local_variables()) - self.assertTrue(a in variables_lib.global_variables()) - - def testGlobalVariableInVariablesToRestore(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.global_variable(0) - self.assertFalse(a in variables_lib.local_variables()) - self.assertTrue(a in variables_lib2.get_variables_to_restore()) - - def testGetVariablesReturnsThem(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.global_variable(0) - with variable_scope.variable_scope('B'): - b = variables_lib2.global_variable(0) - self.assertEquals([a], variables_lib2.get_variables('A')) - self.assertEquals([b], variables_lib2.get_variables('B')) - - def testGetLocalVariablesDontReturnsThem(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - variables_lib2.global_variable(0) - with variable_scope.variable_scope('B'): - variables_lib2.global_variable(0) - self.assertEquals([], variables_lib2.get_local_variables('A')) - self.assertEquals([], variables_lib2.get_local_variables('B')) - - def testInitializedVariableValue(self): - with self.cached_session() as sess: - a = variables_lib2.global_variable([0, 0, 0, 0, 0], name='a') - sess.run(variables_lib.global_variables_initializer()) - self.assertAllEqual(a.eval(), [0] * 5) - - def testResourceVariable(self): - a = variables_lib2.global_variable(0) - b = variables_lib2.global_variable(0, use_resource=True) - self.assertTrue(isinstance(a, variables_lib.Variable)) - self.assertFalse(isinstance(a, resource_variable_ops.ResourceVariable)) - self.assertTrue(isinstance(b, resource_variable_ops.ResourceVariable)) - - -class GlobalStepTest(test.TestCase): - - def _assert_global_step(self, global_step, expected_dtype=dtypes.int64): - self.assertEquals('%s:0' % ops.GraphKeys.GLOBAL_STEP, global_step.name) - self.assertEquals(expected_dtype, global_step.dtype.base_dtype) - self.assertEquals([], global_step.get_shape().as_list()) - - def test_invalid_dtype(self): - with ops.Graph().as_default() as g: - self.assertEquals(None, variables_lib2.get_global_step()) - variables_lib.VariableV1( - 0.0, - trainable=False, - dtype=dtypes.float32, - name=ops.GraphKeys.GLOBAL_STEP) - self.assertRaisesRegexp(TypeError, 'does not have integer type', - variables_lib2.get_global_step) - self.assertRaisesRegexp(TypeError, 'does not have integer type', - variables_lib2.get_global_step, g) - - def test_invalid_shape(self): - with ops.Graph().as_default() as g: - self.assertEquals(None, variables_lib2.get_global_step()) - variables_lib.VariableV1( - [0], - trainable=False, - dtype=dtypes.int32, - name=ops.GraphKeys.GLOBAL_STEP) - self.assertRaisesRegexp(TypeError, 'not scalar', - variables_lib2.get_global_step) - self.assertRaisesRegexp(TypeError, 'not scalar', - variables_lib2.get_global_step, g) - - def test_create_global_step(self): - self.assertEquals(None, variables_lib2.get_global_step()) - with ops.Graph().as_default() as g: - global_step = variables_lib2.create_global_step() - self._assert_global_step(global_step) - self.assertRaisesRegexp(ValueError, 'already exists', - variables_lib2.create_global_step) - self.assertRaisesRegexp(ValueError, 'already exists', - variables_lib2.create_global_step, g) - self._assert_global_step(variables_lib2.create_global_step(ops.Graph())) - - def test_get_global_step(self): - with ops.Graph().as_default() as g: - self.assertEquals(None, variables_lib2.get_global_step()) - variables_lib.VariableV1( - 0, - trainable=False, - dtype=dtypes.int32, - name=ops.GraphKeys.GLOBAL_STEP) - self._assert_global_step( - variables_lib2.get_global_step(), expected_dtype=dtypes.int32) - self._assert_global_step( - variables_lib2.get_global_step(g), expected_dtype=dtypes.int32) - - def test_get_or_create_global_step(self): - with ops.Graph().as_default() as g: - self.assertEquals(None, variables_lib2.get_global_step()) - self._assert_global_step(variables_lib2.get_or_create_global_step()) - self._assert_global_step(variables_lib2.get_or_create_global_step(g)) - - -class VariablesTest(test.TestCase): - - def testCreateVariable(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [5]) - self.assertEquals(a.op.name, 'A/a') - self.assertListEqual(a.get_shape().as_list(), [5]) - self.assertTrue(a in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)) - self.assertFalse(a in ops.get_collection(ops.GraphKeys.MODEL_VARIABLES)) - self.assertFalse(a in variables_lib.local_variables()) - - def testGetVariables(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [5]) - with variable_scope.variable_scope('B'): - b = variables_lib2.variable('a', [5]) - self.assertEquals([a, b], variables_lib2.get_variables()) - self.assertEquals([a], variables_lib2.get_variables('A')) - self.assertEquals([b], variables_lib2.get_variables('B')) - - def testGetVariablesWithScope(self): - with self.cached_session(): - with variable_scope.variable_scope('A') as var_scope: - a = variables_lib2.variable('a', [5]) - b = variables_lib2.variable('b', [5]) - self.assertSetEqual( - set([a, b]), set(variables_lib2.get_variables(var_scope))) - - def testGetVariablesSuffix(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [5]) - with variable_scope.variable_scope('A'): - b = variables_lib2.variable('b', [5]) - self.assertEquals([a], variables_lib2.get_variables(suffix='a')) - self.assertEquals([b], variables_lib2.get_variables(suffix='b')) - - def testGetVariableWithSingleVar(self): - with self.cached_session(): - with variable_scope.variable_scope('parent'): - a = variables_lib2.variable('child', [5]) - self.assertEquals(a, variables_lib2.get_unique_variable('parent/child')) - - def testGetVariableWithDistractors(self): - with self.cached_session(): - with variable_scope.variable_scope('parent'): - a = variables_lib2.variable('child', [5]) - with variable_scope.variable_scope('child'): - variables_lib2.variable('grandchild1', [7]) - variables_lib2.variable('grandchild2', [9]) - self.assertEquals(a, variables_lib2.get_unique_variable('parent/child')) - - def testGetVariableThrowsExceptionWithNoMatch(self): - var_name = 'cant_find_me' - with self.cached_session(): - with self.assertRaises(ValueError): - variables_lib2.get_unique_variable(var_name) - - def testGetThrowsExceptionWithChildrenButNoMatch(self): - var_name = 'parent/child' - with self.cached_session(): - with variable_scope.variable_scope(var_name): - variables_lib2.variable('grandchild1', [7]) - variables_lib2.variable('grandchild2', [9]) - with self.assertRaises(ValueError): - variables_lib2.get_unique_variable(var_name) - - def testGetVariablesToRestore(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [5]) - with variable_scope.variable_scope('B'): - b = variables_lib2.variable('a', [5]) - self.assertEquals([a, b], variables_lib2.get_variables_to_restore()) - - def testIncludeGetVariablesToRestore(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [5]) - with variable_scope.variable_scope('B'): - b = variables_lib2.variable('a', [5]) - self.assertEquals([a, b], variables_lib2.get_variables()) - self.assertEquals([a], variables_lib2.get_variables_to_restore(['A'])) - - def testExcludeGetVariablesToRestore(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [5]) - with variable_scope.variable_scope('B'): - b = variables_lib2.variable('a', [5]) - self.assertEquals([a, b], variables_lib2.get_variables()) - self.assertEquals( - [a], variables_lib2.get_variables_to_restore(exclude=['B'])) - - def testWrongIncludeGetVariablesToRestore(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [5]) - with variable_scope.variable_scope('B'): - b = variables_lib2.variable('a', [5]) - self.assertEquals([a, b], variables_lib2.get_variables()) - self.assertEquals([], variables_lib2.get_variables_to_restore(['a'])) - - def testGetMixedVariablesToRestore(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [5]) - b = variables_lib2.variable('b', [5]) - with variable_scope.variable_scope('B'): - c = variables_lib2.variable('c', [5]) - d = variables_lib2.variable('d', [5]) - self.assertEquals([a, b, c, d], variables_lib2.get_variables()) - self.assertEquals( - [a, c], - variables_lib2.get_variables_to_restore(include=['A/a', 'B/c'])) - - def testExcludeGetMixedVariablesToRestore(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [5]) - b = variables_lib2.variable('b', [5]) - with variable_scope.variable_scope('B'): - c = variables_lib2.variable('c', [5]) - d = variables_lib2.variable('d', [5]) - self.assertEquals([a, b, c, d], variables_lib2.get_variables()) - self.assertEquals( - [b, d], - variables_lib2.get_variables_to_restore(exclude=['A/a', 'B/c'])) - - def testReuseVariable(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', []) - with variable_scope.variable_scope('A', reuse=True): - b = variables_lib2.variable('a', []) - self.assertEquals(a, b) - self.assertListEqual([a], variables_lib2.get_variables()) - - def testVariableWithRegularizer(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [], regularizer=nn_ops.l2_loss) - loss = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)[0] - self.assertDeviceEqual(loss.device, a.device) - - def testVariableWithRegularizerColocate(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable( - 'a', [], device='gpu:0', regularizer=nn_ops.l2_loss) - loss = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)[0] - self.assertDeviceEqual(loss.device, a.device) - - def testVariableWithDevice(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [], device='cpu:0') - b = variables_lib2.variable('b', [], device='cpu:1') - self.assertDeviceEqual(a.device, 'cpu:0') - self.assertDeviceEqual(b.device, 'cpu:1') - - def testVariableWithDeviceFromScope(self): - with self.cached_session(): - with ops.device('/cpu:0'): - a = variables_lib2.variable('a', []) - b = variables_lib2.variable('b', [], device='cpu:1') - self.assertDeviceEqual(a.device, 'cpu:0') - self.assertDeviceEqual(b.device, 'cpu:1') - - def testVariableWithDeviceFunction(self): - - class DevFn(object): - - def __init__(self): - self.counter = -1 - - def __call__(self, op): - self.counter += 1 - return 'cpu:%d' % self.counter - - with self.cached_session(): - with arg_scope([variables_lib2.variable], device=DevFn()): - a = variables_lib2.variable('a', []) - b = variables_lib2.variable('b', []) - c = variables_lib2.variable('c', [], device='cpu:12') - d = variables_lib2.variable('d', []) - with ops.device('cpu:99'): - e_init = constant_op.constant(12) - e = variables_lib2.variable('e', initializer=e_init) - self.assertDeviceEqual(a.device, 'cpu:0') - self.assertEqual(a.initial_value.op.colocation_groups(), - a.op.colocation_groups()) - self.assertDeviceEqual(b.device, 'cpu:1') - self.assertEqual(b.initial_value.op.colocation_groups(), - b.op.colocation_groups()) - self.assertDeviceEqual(c.device, 'cpu:12') - self.assertEqual(c.initial_value.op.colocation_groups(), - c.op.colocation_groups()) - self.assertDeviceEqual(d.device, 'cpu:2') - self.assertEqual(d.initial_value.op.colocation_groups(), - d.op.colocation_groups()) - self.assertDeviceEqual(e.device, 'cpu:3') - self.assertDeviceEqual(e.initial_value.device, 'cpu:99') - - def testVariableWithReplicaDeviceSetter(self): - with self.cached_session(): - with ops.device(device_setter.replica_device_setter(ps_tasks=2)): - a = variables_lib2.variable('a', []) - b = variables_lib2.variable('b', []) - c = variables_lib2.variable('c', [], device='cpu:12') - d = variables_lib2.variable('d', []) - with ops.device('cpu:99'): - e_init = constant_op.constant(12) - e = variables_lib2.variable('e', initializer=e_init) - # The values below highlight how the replica_device_setter puts initial - # values on the worker job, and how it merges explicit devices. - self.assertDeviceEqual(a.device, '/job:ps/task:0/cpu:0') - self.assertEqual(a.initial_value.op.colocation_groups(), - a.op.colocation_groups()) - self.assertDeviceEqual(b.device, '/job:ps/task:1/cpu:0') - self.assertEqual(b.initial_value.op.colocation_groups(), - b.op.colocation_groups()) - self.assertDeviceEqual(c.device, '/job:ps/task:0/cpu:12') - self.assertEqual(c.initial_value.op.colocation_groups(), - c.op.colocation_groups()) - self.assertDeviceEqual(d.device, '/job:ps/task:1/cpu:0') - self.assertEqual(d.initial_value.op.colocation_groups(), - d.op.colocation_groups()) - self.assertDeviceEqual(e.device, '/job:ps/task:0/cpu:0') - self.assertDeviceEqual(e.initial_value.device, '/job:worker/cpu:99') - - def testVariableWithVariableDeviceChooser(self): - - with ops.Graph().as_default(): - device_fn = variables_lib2.VariableDeviceChooser(num_tasks=2) - with arg_scope([variables_lib2.variable], device=device_fn): - a = variables_lib2.variable('a', []) - b = variables_lib2.variable('b', []) - c = variables_lib2.variable('c', [], device='cpu:12') - d = variables_lib2.variable('d', []) - with ops.device('cpu:99'): - e_init = constant_op.constant(12) - e = variables_lib2.variable('e', initializer=e_init) - # The values below highlight how the VariableDeviceChooser puts initial - # values on the same device as the variable job. - self.assertDeviceEqual(a.device, '/job:ps/task:0/cpu:0') - self.assertEqual(a.initial_value.op.colocation_groups(), - a.op.colocation_groups()) - self.assertDeviceEqual(b.device, '/job:ps/task:1/cpu:0') - self.assertEqual(b.initial_value.op.colocation_groups(), - b.op.colocation_groups()) - self.assertDeviceEqual(c.device, '/cpu:12') - self.assertEqual(c.initial_value.op.colocation_groups(), - c.op.colocation_groups()) - self.assertDeviceEqual(d.device, '/job:ps/task:0/cpu:0') - self.assertEqual(d.initial_value.op.colocation_groups(), - d.op.colocation_groups()) - self.assertDeviceEqual(e.device, '/job:ps/task:1/cpu:0') - self.assertDeviceEqual(e.initial_value.device, '/cpu:99') - - def testVariableWithVariableDeviceChooserWithReplica(self): - - with ops.Graph().as_default(): - device_fn = variables_lib2.VariableDeviceChooser(replica=3, num_tasks=2) - with arg_scope([variables_lib2.variable], device=device_fn): - a = variables_lib2.variable('a', []) - b = variables_lib2.variable('b', []) - c = variables_lib2.variable('c', [], device='cpu:12') - d = variables_lib2.variable('d', []) - with ops.device('cpu:99'): - e_init = constant_op.constant(12) - e = variables_lib2.variable('e', initializer=e_init) - # The values below highlight how the VariableDeviceChooser puts initial - # values on the same device as the variable job. - self.assertDeviceEqual(a.device, '/job:ps/replica:3/task:0/cpu:0') - self.assertEqual(a.initial_value.op.colocation_groups(), - a.op.colocation_groups()) - self.assertDeviceEqual(b.device, '/job:ps/replica:3/task:1/cpu:0') - self.assertEqual(b.initial_value.op.colocation_groups(), - b.op.colocation_groups()) - self.assertDeviceEqual(c.device, '/cpu:12') - self.assertEqual(c.initial_value.op.colocation_groups(), - c.op.colocation_groups()) - self.assertDeviceEqual(d.device, '/job:ps/replica:3/task:0/cpu:0') - self.assertEqual(d.initial_value.op.colocation_groups(), - d.op.colocation_groups()) - self.assertDeviceEqual(e.device, '/job:ps/replica:3/task:1/cpu:0') - self.assertDeviceEqual(e.initial_value.device, '/cpu:99') - - def testVariableGPUPlacement(self): - - with ops.Graph().as_default(): - device_fn = variables_lib2.VariableDeviceChooser(device_type='GPU') - with arg_scope([variables_lib2.variable], device=device_fn): - a = variables_lib2.variable('a', []) - b = variables_lib2.variable('b', []) - c = variables_lib2.variable('c', [], device='cpu:12') - d = variables_lib2.variable('d', []) - with ops.device('cpu:99'): - e_init = constant_op.constant(12) - e = variables_lib2.variable('e', initializer=e_init) - # The values below highlight how the VariableDeviceChooser puts initial - # values on the same device as the variable job. - self.assertDeviceEqual(a.device, '/device:GPU:0') - self.assertEqual(a.initial_value.op.colocation_groups(), - a.op.colocation_groups()) - self.assertDeviceEqual(b.device, '/device:GPU:0') - self.assertEqual(b.initial_value.op.colocation_groups(), - b.op.colocation_groups()) - self.assertDeviceEqual(c.device, '/cpu:12') - self.assertEqual(c.initial_value.op.colocation_groups(), - c.op.colocation_groups()) - self.assertDeviceEqual(d.device, '/device:GPU:0') - self.assertEqual(d.initial_value.op.colocation_groups(), - d.op.colocation_groups()) - self.assertDeviceEqual(e.device, '/device:GPU:0') - self.assertDeviceEqual(e.initial_value.device, '/cpu:99') - - -class ModelVariablesTest(test.TestCase): - - def testNameAndShape(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.model_variable('a', [5]) - self.assertEquals(a.op.name, 'A/a') - self.assertListEqual(a.get_shape().as_list(), [5]) - self.assertListEqual([a], variables_lib2.get_model_variables('A')) - - def testNotInLocalVariables(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.model_variable('a', [5]) - self.assertTrue(a in variables_lib.global_variables()) - self.assertTrue(a in ops.get_collection(ops.GraphKeys.MODEL_VARIABLES)) - self.assertFalse(a in variables_lib.local_variables()) - - def testGetVariablesReturns(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.model_variable('a', [5]) - with variable_scope.variable_scope('B'): - b = variables_lib2.model_variable('a', [5]) - self.assertEquals([a], variables_lib2.get_variables('A')) - self.assertEquals([b], variables_lib2.get_variables('B')) - - def testGetModelVariables(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.model_variable('a', [5]) - with variable_scope.variable_scope('B'): - b = variables_lib2.model_variable('a', [5]) - self.assertEquals([a], variables_lib2.get_model_variables('A')) - self.assertEquals([b], variables_lib2.get_model_variables('B')) - - def testGetTrainableVariables(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - variables_lib2.local_variable([5]) - a = variables_lib.VariableV1([5]) - with variable_scope.variable_scope('B'): - variables_lib2.local_variable([5]) - b = variables_lib.VariableV1([5]) - self.assertEquals([a], variables_lib2.get_trainable_variables('A')) - self.assertEquals([b], variables_lib2.get_trainable_variables('B')) - - def testGetLocalVariables(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - _ = variables_lib2.model_variable('a', [5]) - with variable_scope.variable_scope('B'): - _ = variables_lib2.model_variable('a', [5]) - self.assertEquals([], variables_lib2.get_local_variables('A')) - self.assertEquals([], variables_lib2.get_local_variables('B')) - - def testInitializedVariableValue(self): - with self.cached_session() as sess: - a = variables_lib2.model_variable( - 'a', [5], initializer=init_ops.ones_initializer()) - sess.run(variables_lib.global_variables_initializer()) - self.assertAllEqual(a.eval(), [1] * 5) - - def testDeviceFn(self): - - class DevFn(object): - - def __init__(self): - self.counter = -1 - - def __call__(self, op): - self.counter += 1 - return '/cpu:%d' % self.counter - - with ops.Graph().as_default(): - with arg_scope([variables_lib2.model_variable], device=DevFn()): - a = variables_lib2.model_variable('a', [5]) - b = variables_lib2.model_variable('b', [20]) - self.assertDeviceEqual(a.device, '/cpu:0') - self.assertEqual(a.initial_value.op.colocation_groups(), - a.op.colocation_groups()) - self.assertDeviceEqual(b.device, '/cpu:1') - self.assertEqual(b.initial_value.op.colocation_groups(), - b.op.colocation_groups()) - - def testVariableWithVariableDeviceChooser(self): - - with ops.Graph().as_default(): - device_fn = variables_lib2.VariableDeviceChooser() - with arg_scope([variables_lib2.model_variable], device=device_fn): - a = variables_lib2.model_variable('a', [5]) - b = variables_lib2.model_variable('b', [20]) - self.assertDeviceEqual(a.device, 'cpu:0') - self.assertEqual(a.initial_value.op.colocation_groups(), - a.op.colocation_groups()) - self.assertDeviceEqual(b.device, 'cpu:0') - self.assertEqual(a.initial_value.op.colocation_groups(), - a.op.colocation_groups()) - - -class GetVariablesCollections(test.TestCase): - - def testVariableCollection(self): - with self.cached_session(): - a = variables_lib2.variable('a', [], collections='A') - b = variables_lib2.variable('b', [], collections='B') - self.assertEquals(a, ops.get_collection('A')[0]) - self.assertEquals(b, ops.get_collection('B')[0]) - - def testVariableCollections(self): - with self.cached_session(): - a = variables_lib2.variable('a', [], collections=['A', 'C']) - b = variables_lib2.variable('b', [], collections=['B', 'C']) - self.assertEquals(a, ops.get_collection('A')[0]) - self.assertEquals(b, ops.get_collection('B')[0]) - self.assertListEqual([a, b], ops.get_collection('C')) - - def testVariableCollectionsWithArgScope(self): - with self.cached_session(): - with arg_scope([variables_lib2.variable], collections='A'): - a = variables_lib2.variable('a', []) - b = variables_lib2.variable('b', []) - self.assertListEqual([a, b], ops.get_collection('A')) - - def testVariableCollectionsWithArgScopeNested(self): - with self.cached_session(): - with arg_scope([variables_lib2.variable], collections='A'): - a = variables_lib2.variable('a', []) - with arg_scope([variables_lib2.variable], collections='B'): - b = variables_lib2.variable('b', []) - self.assertEquals(a, ops.get_collection('A')[0]) - self.assertEquals(b, ops.get_collection('B')[0]) - - def testVariableCollectionsWithArgScopeNonNested(self): - with self.cached_session(): - with arg_scope([variables_lib2.variable], collections='A'): - a = variables_lib2.variable('a', []) - with arg_scope([variables_lib2.variable], collections='B'): - b = variables_lib2.variable('b', []) - variables_lib2.variable('c', []) - self.assertListEqual([a], ops.get_collection('A')) - self.assertListEqual([b], ops.get_collection('B')) - - def testVariableRestoreWithArgScopeNested(self): - with self.cached_session(): - a = variables_lib2.variable('a', []) - with arg_scope( - [variables_lib2.variable], trainable=False, collections=['A', 'B']): - b = variables_lib2.variable('b', []) - c = variables_lib2.variable('c', [], trainable=False) - self.assertEquals([a, c], variables_lib2.get_variables_to_restore()) - self.assertEquals([a], variables_lib.trainable_variables()) - self.assertEquals([b], ops.get_collection('A')) - self.assertEquals([b], ops.get_collection('B')) - - -class GetVariablesBySuffixTest(test.TestCase): - - def testGetVariableGivenNameScoped(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [5]) - b = variables_lib2.variable('b', [5]) - self.assertEquals([a], variables_lib2.get_variables_by_suffix('a')) - self.assertEquals([b], variables_lib2.get_variables_by_suffix('b')) - - def testGetVariableWithScope(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [5]) - fooa = variables_lib2.variable('fooa', [5]) - with variable_scope.variable_scope('B'): - a2 = variables_lib2.variable('a', [5]) - matched_variables = variables_lib2.get_variables_by_suffix('a') - self.assertEquals([a, fooa, a2], matched_variables) - matched_variables = variables_lib2.get_variables_by_suffix('/a') - self.assertEquals([a, a2], matched_variables) - matched_variables = variables_lib2.get_variables_by_suffix('a', scope='A') - self.assertEquals([a, fooa], matched_variables) - - def testGetVariableWithoutScope(self): - with self.cached_session(): - a = variables_lib2.variable('a', [5]) - fooa = variables_lib2.variable('fooa', [5]) - b_a = variables_lib2.variable('B/a', [5]) - matched_variables = variables_lib2.get_variables_by_suffix('a') - self.assertEquals([a, fooa, b_a], matched_variables) - matched_variables = variables_lib2.get_variables_by_suffix('fooa') - self.assertEquals([fooa], matched_variables) - - -class GetVariablesByNameTest(test.TestCase): - - def testGetVariableGivenNameScoped(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [5]) - b = variables_lib2.variable('b', [5]) - self.assertEquals([a], variables_lib2.get_variables_by_name('a')) - self.assertEquals([b], variables_lib2.get_variables_by_name('b')) - - def testGetVariableWithScope(self): - with self.cached_session(): - with variable_scope.variable_scope('A'): - a = variables_lib2.variable('a', [5]) - fooa = variables_lib2.variable('fooa', [5]) - with variable_scope.variable_scope('B'): - a2 = variables_lib2.variable('a', [5]) - matched_variables = variables_lib2.get_variables_by_name('a') - self.assertEquals([a, a2], matched_variables) - matched_variables = variables_lib2.get_variables_by_name('fooa') - self.assertEquals([fooa], matched_variables) - matched_variables = variables_lib2.get_variables_by_name('/a') - self.assertEquals([], matched_variables) - matched_variables = variables_lib2.get_variables_by_name('a', scope='A') - self.assertEquals([a], matched_variables) - - def testGetVariableWithoutScope(self): - with self.cached_session(): - a = variables_lib2.variable('a', [5]) - fooa = variables_lib2.variable('fooa', [5]) - b_a = variables_lib2.variable('B/a', [5]) - matched_variables = variables_lib2.get_variables_by_name('a') - self.assertEquals([a, b_a], matched_variables) - matched_variables = variables_lib2.get_variables_by_name('fooa') - self.assertEquals([fooa], matched_variables) - - -class GetVariableFullNameTest(test.TestCase): - - def testVariable(self): - my_var0 = variables_lib2.variable('my_var0', shape=[]) - full_name = variables_lib2.get_variable_full_name(my_var0) - self.assertEquals(full_name, my_var0.op.name) - - def testPartitionedVariable(self): - input_full_name = 'my_var0' - partitioner = partitioned_variables.variable_axis_size_partitioner(2) - my_var0 = variables_lib2.variable( - 'my_var0', shape=[2, 2], partitioner=partitioner) - for part_var in list(my_var0): - computed_full_name = variables_lib2.get_variable_full_name(part_var) - self.assertEquals(input_full_name, computed_full_name) - - -class AssignFromValuesTest(test.TestCase): - - def testNoScopes(self): - init_value0 = np.asarray([1.0, 3.0, 9.0]).reshape((1, 3, 1)) - init_value1 = np.asarray([2.0, 4.0, 6.0, 8.0]).reshape((2, 1, 2)) - - with self.cached_session() as sess: - initializer = init_ops.truncated_normal_initializer(stddev=.1) - var0 = variables_lib2.variable( - 'my_var0', shape=[1, 3, 1], initializer=initializer) - var1 = variables_lib2.variable( - 'my_var1', shape=[2, 1, 2], initializer=initializer) - - var_names_to_values = {'my_var0': init_value0, 'my_var1': init_value1} - assign_op, feed_dict = variables_lib2.assign_from_values( - var_names_to_values) - - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Perform the assignment. - sess.run(assign_op, feed_dict) - - # Request and test the variable values: - var0, var1 = sess.run([var0, var1]) - self.assertAllEqual(init_value0, var0) - self.assertAllEqual(init_value1, var1) - - def testWithScopes(self): - init_value0 = np.asarray([1.0, 3.0, 9.0]).reshape((1, 3, 1)) - init_value1 = np.asarray([2.0, 4.0, 6.0, 8.0]).reshape((2, 1, 2)) - - with self.cached_session() as sess: - initializer = init_ops.truncated_normal_initializer(stddev=.1) - - with variable_scope.variable_scope('my_model/my_layer0'): - var0 = variables_lib2.variable( - 'my_var0', shape=[1, 3, 1], initializer=initializer) - with variable_scope.variable_scope('my_model/my_layer1'): - var1 = variables_lib2.variable( - 'my_var1', shape=[2, 1, 2], initializer=initializer) - - var_names_to_values = { - 'my_model/my_layer0/my_var0': init_value0, - 'my_model/my_layer1/my_var1': init_value1 - } - assign_op, feed_dict = variables_lib2.assign_from_values( - var_names_to_values) - - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Perform the assignment. - sess.run(assign_op, feed_dict) - - # Request and test the variable values: - var0, var1 = sess.run([var0, var1]) - self.assertAllEqual(init_value0, var0) - self.assertAllEqual(init_value1, var1) - - -class AssignFromValuesFnTest(test.TestCase): - - def testNoScopes(self): - init_value0 = np.asarray([1.0, 3.0, 9.0]).reshape((1, 3, 1)) - init_value1 = np.asarray([2.0, 4.0, 6.0, 8.0]).reshape((2, 1, 2)) - - with self.cached_session() as sess: - initializer = init_ops.truncated_normal_initializer(stddev=.1) - var0 = variables_lib2.variable( - 'my_var0', shape=[1, 3, 1], initializer=initializer) - var1 = variables_lib2.variable( - 'my_var1', shape=[2, 1, 2], initializer=initializer) - - var_names_to_values = {'my_var0': init_value0, 'my_var1': init_value1} - init_fn = variables_lib2.assign_from_values_fn(var_names_to_values) - - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Perform the assignment. - init_fn(sess) - - # Request and test the variable values: - var0, var1 = sess.run([var0, var1]) - self.assertAllEqual(init_value0, var0) - self.assertAllEqual(init_value1, var1) - - def testWithScopes(self): - init_value0 = np.asarray([1.0, 3.0, 9.0]).reshape((1, 3, 1)) - init_value1 = np.asarray([2.0, 4.0, 6.0, 8.0]).reshape((2, 1, 2)) - - with self.cached_session() as sess: - initializer = init_ops.truncated_normal_initializer(stddev=.1) - - with variable_scope.variable_scope('my_model/my_layer0'): - var0 = variables_lib2.variable( - 'my_var0', shape=[1, 3, 1], initializer=initializer) - with variable_scope.variable_scope('my_model/my_layer1'): - var1 = variables_lib2.variable( - 'my_var1', shape=[2, 1, 2], initializer=initializer) - - var_names_to_values = { - 'my_model/my_layer0/my_var0': init_value0, - 'my_model/my_layer1/my_var1': init_value1 - } - init_fn = variables_lib2.assign_from_values_fn(var_names_to_values) - - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Perform the assignment. - init_fn(sess) - - # Request and test the variable values: - var0, var1 = sess.run([var0, var1]) - self.assertAllEqual(init_value0, var0) - self.assertAllEqual(init_value1, var1) - - -class AssignFromCheckpointTest(test.TestCase): - - def create_checkpoint_from_values(self, - var_names_to_values, - checkpoint_dir, - global_step=None): - """Creates a checkpoint from a mapping of name to values in model_dir. - - Args: - var_names_to_values: a map from variable names to values. - checkpoint_dir: the directory where the checkpoint will be saved. - global_step: the global step used to save the checkpoint. - - Returns: - the model_path to the checkpoint. - """ - var_list = [] - with session.Session('', graph=ops.Graph()) as sess: - # Create a set of variables to save in the checkpoint. - for var_name in var_names_to_values: - var_value = var_names_to_values[var_name] - var_list.append(variables_lib.VariableV1(var_value, name=var_name)) - saver = saver_lib.Saver(var_list) - init_op = variables_lib.variables_initializer(var_list) - sess.run(init_op) - # Save the initialized values in the file at 'checkpoint_dir' - return saver.save(sess, checkpoint_dir, global_step=global_step) - - def testLoadExistingVariables(self): - model_dir = tempfile.mkdtemp( - prefix=os.path.join(self.get_temp_dir(), 'load_existing_variables')) - - init_value0 = 10.0 - init_value1 = 20.0 - var_names_to_values = {'v0': init_value0, 'v1': init_value1} - - with self.cached_session() as sess: - model_path = self.create_checkpoint_from_values(var_names_to_values, - model_dir) - var0 = variables_lib2.variable('my_var0', shape=[]) - var1 = variables_lib2.variable('my_var1', shape=[]) - - vars_to_restore = {'v0': var0, 'v1': var1} - op, feed_dict = variables_lib2.assign_from_checkpoint( - model_path, vars_to_restore) - - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Perform the assignment. - sess.run(op, feed_dict) - - # Request and test the variable values: - self.assertEqual(init_value0, var0.eval()) - self.assertEqual(init_value1, var1.eval()) - - # Tests restoring PartitionedVariables and tests using a dictionary - # of lists as the assign_from_checkpoint() var_list param. - def testLoadPartitionedVariables(self): - model_dir = tempfile.mkdtemp( - prefix=os.path.join(self.get_temp_dir(), 'load_partitioned_variables')) - - init_value0 = np.array([[10.0, 11.0], [12.0, 13.0]]) - init_value1 = np.array([20.0]) # Partitioned into 1 part, edge case. - var_names_to_values = {'var0': init_value0, 'var1': init_value1} - - with self.cached_session() as sess: - model_path = self.create_checkpoint_from_values(var_names_to_values, - model_dir) - # var0 and var1 are PartitionedVariables. - partitioner = partitioned_variables.variable_axis_size_partitioner(2) - var0 = variables_lib2.variable( - 'var0', shape=init_value0.shape, partitioner=partitioner) - var0full = variables_lib2.variable('var0full', shape=init_value0.shape) - var1 = variables_lib2.variable( - 'var1', shape=init_value1.shape, partitioner=partitioner) - - # Convert var0 and var1 into a list of underlying variables. - vars_to_restore = {'var0': list(var0) + [var0full], 'var1': list(var1)} - op, feed_dict = variables_lib2.assign_from_checkpoint( - model_path, vars_to_restore) - - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Perform the assignment. - sess.run(op, feed_dict) - - # Request and test the variable values. PartitionedVariables can't - # be evaled so we wrap them in an identity. - self.assertTrue( - np.array_equal(init_value0, - array_ops.identity(var0).eval())) - self.assertTrue(np.array_equal(init_value0, var0full.eval())) - self.assertTrue( - np.array_equal(init_value1, - array_ops.identity(var1).eval())) - - def testRaisesValueErrorIfAVariableIsntFound(self): - model_dir = tempfile.mkdtemp( - prefix=os.path.join(self.get_temp_dir(), - 'raises_value_error_if_var_isnt_found')) - - init_value0 = 10.0 - init_value1 = 20.0 - var_names_to_values = {'v0': init_value0, 'v1': init_value1} - - with self.cached_session(): - model_path = self.create_checkpoint_from_values(var_names_to_values, - model_dir) - var0 = variables_lib2.variable('my_var0', shape=[]) - var1 = variables_lib2.variable('my_var1', shape=[]) - - vars_to_restore = {'v0_fake': var0, 'v1': var1} - - with self.assertRaises(ValueError): - variables_lib2.assign_from_checkpoint(model_path, vars_to_restore) - - def testInitFromCheckpointWithScopes(self): - model_dir = tempfile.mkdtemp( - prefix=os.path.join(self.get_temp_dir(), - 'init_from_checkpoint_with_scopes')) - - init_value0 = np.asarray( - [1.0, 3.0, 9.0], dtype=np.float32).reshape((1, 3, 1)) - init_value1 = np.asarray( - [2.0, 4.0, 6.0, 8.0], dtype=np.float32).reshape((2, 1, 2)) - - var_names_to_values = {'layer0/v0': init_value0, 'layer1/v1': init_value1} - - with self.cached_session() as sess: - model_path = self.create_checkpoint_from_values(var_names_to_values, - model_dir) - with variable_scope.variable_scope('my_model/my_layer0'): - var0 = variables_lib2.variable('my_var0', shape=init_value0.shape) - with variable_scope.variable_scope('my_model/my_layer1'): - var1 = variables_lib2.variable('my_var1', shape=init_value1.shape) - - vars_to_restore = {'layer0/v0': var0, 'layer1/v1': var1} - op, feed_dict = variables_lib2.assign_from_checkpoint( - model_path, vars_to_restore) - - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Perform the assignment. - sess.run(op, feed_dict) - - # Request and test the variable values: - self.assertAllEqual(init_value0, var0.eval()) - self.assertAllEqual(init_value1, var1.eval()) - - -class AssignFromCheckpointFnTest(test.TestCase): - - def create_checkpoint_from_values(self, - var_names_to_values, - checkpoint_dir, - global_step=None): - """Creates a checkpoint from a mapping of name to values in model_dir. - - Args: - var_names_to_values: a map from variable names to values. - checkpoint_dir: the directory where the checkpoint will be saved. - global_step: the global step used to save the checkpoint. - - Returns: - the model_path to the checkpoint. - """ - var_list = [] - with session.Session('', graph=ops.Graph()) as sess: - # Create a set of variables to save in the checkpoint. - for var_name in var_names_to_values: - var_value = var_names_to_values[var_name] - var_list.append(variables_lib.VariableV1(var_value, name=var_name)) - saver = saver_lib.Saver(var_list) - init_op = variables_lib.variables_initializer(var_list) - sess.run(init_op) - # Save the initialized values in the file at 'checkpoint_dir' - return saver.save(sess, checkpoint_dir, global_step=global_step) - - def testLoadExistingVariables(self): - model_dir = tempfile.mkdtemp( - prefix=os.path.join(self.get_temp_dir(), 'load_existing_variables')) - if gfile.Exists(model_dir): - gfile.DeleteRecursively(model_dir) - - init_value0 = 10.0 - init_value1 = 20.0 - var_names_to_values = {'v0': init_value0, 'v1': init_value1} - - with self.cached_session() as sess: - model_path = self.create_checkpoint_from_values(var_names_to_values, - model_dir) - var0 = variables_lib2.variable('my_var0', shape=[]) - var1 = variables_lib2.variable('my_var1', shape=[]) - - vars_to_restore = {'v0': var0, 'v1': var1} - init_fn = variables_lib2.assign_from_checkpoint_fn( - model_path, vars_to_restore) - - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Perform the assignment. - init_fn(sess) - - # Request and test the variable values: - self.assertEqual(init_value0, var0.eval()) - self.assertEqual(init_value1, var1.eval()) - - def testLoadExistingVariablesDifferentShapeDefaultDoesNotAllowReshape(self): - model_dir = tempfile.mkdtemp( - prefix=os.path.join(self.get_temp_dir(), - 'load_existing_vars_no_reshape')) - if gfile.Exists(model_dir): - gfile.DeleteRecursively(model_dir) - - init_value0 = [[10.0, 11.0]] - init_value1 = 20.0 - var_names_to_values = {'v0': init_value0, 'v1': init_value1} - - with self.cached_session() as sess: - model_path = self.create_checkpoint_from_values(var_names_to_values, - model_dir) - var0 = variables_lib2.variable('my_var0', shape=[2, 1]) - var1 = variables_lib2.variable('my_var1', shape=[]) - - vars_to_restore = {'v0': var0, 'v1': var1} - init_fn = variables_lib2.assign_from_checkpoint_fn( - model_path, vars_to_restore) - - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Perform the assignment. - with self.assertRaises(errors_impl.InvalidArgumentError): - init_fn(sess) - - def testLoadExistingVariablesDifferentShapeAllowReshape(self): - model_dir = tempfile.mkdtemp( - prefix=os.path.join( - self.get_temp_dir(), - 'load_existing_variables_different_shape_allow_reshape')) - if gfile.Exists(model_dir): - gfile.DeleteRecursively(model_dir) - - init_value0 = [[10.0, 11.0]] - init_value1 = 20.0 - var_names_to_values = {'v0': init_value0, 'v1': init_value1} - - with self.cached_session() as sess: - model_path = self.create_checkpoint_from_values(var_names_to_values, - model_dir) - var0 = variables_lib2.variable('my_var0', shape=[2, 1]) - var1 = variables_lib2.variable('my_var1', shape=[]) - - vars_to_restore = {'v0': var0, 'v1': var1} - init_fn = variables_lib2.assign_from_checkpoint_fn( - model_path, vars_to_restore, reshape_variables=True) - - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Perform the assignment. - init_fn(sess) - - # Request and test the variable values: - self.assertAllEqual(np.transpose(np.array(init_value0)), var0.eval()) - self.assertEqual(init_value1, var1.eval()) - - def testNotFoundError(self): - model_dir = tempfile.mkdtemp( - prefix=os.path.join(self.get_temp_dir(), 'not_found_error')) - if gfile.Exists(model_dir): - gfile.DeleteRecursively(model_dir) - - init_value0 = 10.0 - init_value1 = 20.0 - var_names_to_values = {'v0': init_value0, 'v1': init_value1} - - with self.cached_session() as sess: - model_path = self.create_checkpoint_from_values(var_names_to_values, - model_dir) - var0 = variables_lib2.variable('my_var0', shape=[]) - var1 = variables_lib2.variable('my_var1', shape=[]) - var2 = variables_lib2.variable('my_var2', shape=[]) - - vars_to_restore = {'v0': var0, 'v1': var1, 'v2': var2} - init_fn = variables_lib2.assign_from_checkpoint_fn( - model_path, vars_to_restore) - - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Perform the assignment. - with self.assertRaises(errors_impl.NotFoundError): - init_fn(sess) - - def testMissingVariablesList(self): - model_dir = tempfile.mkdtemp( - prefix=os.path.join(self.get_temp_dir(), 'missing_variables_list')) - if gfile.Exists(model_dir): - gfile.DeleteRecursively(model_dir) - - init_value0 = 10.0 - init_value1 = 20.0 - var_names_to_values = {'v0': init_value0, 'v1': init_value1} - - with self.cached_session() as sess: - model_path = self.create_checkpoint_from_values(var_names_to_values, - model_dir) - var0 = variables_lib2.variable('v0', shape=[]) - var1 = variables_lib2.variable('v1', shape=[]) - var2 = variables_lib2.variable('v2', shape=[]) - - vars_to_restore = [var0, var1, var2] - init_fn = variables_lib2.assign_from_checkpoint_fn( - model_path, vars_to_restore, ignore_missing_vars=True) - - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Perform the assignment. - init_fn(sess) - - # Request and test the variable values: - self.assertEqual(init_value0, var0.eval()) - self.assertEqual(init_value1, var1.eval()) - - def testMissingVariablesDict(self): - model_dir = tempfile.mkdtemp( - prefix=os.path.join(self.get_temp_dir(), 'missing_variables_dict')) - if gfile.Exists(model_dir): - gfile.DeleteRecursively(model_dir) - - init_value0 = 10.0 - init_value1 = 20.0 - var_names_to_values = {'v0': init_value0, 'v1': init_value1} - - with self.cached_session() as sess: - model_path = self.create_checkpoint_from_values(var_names_to_values, - model_dir) - var0 = variables_lib2.variable('my_var0', shape=[]) - var1 = variables_lib2.variable('my_var1', shape=[]) - var2 = variables_lib2.variable('my_var2', shape=[]) - - vars_to_restore = {'v0': var0, 'v1': var1, 'v2': var2} - init_fn = variables_lib2.assign_from_checkpoint_fn( - model_path, vars_to_restore, ignore_missing_vars=True) - - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Perform the assignment. - init_fn(sess) - - # Request and test the variable values: - self.assertEqual(init_value0, var0.eval()) - self.assertEqual(init_value1, var1.eval()) - - -class ZeroInitializerOpTest(test.TestCase): - - def _testZeroInitializer(self, shape, initializer, use_init): - var = variables_lib.VariableV1(initializer) - var_zero = variables_lib2.zero_initializer(var) - with self.cached_session() as sess: - with self.assertRaisesOpError('Attempting to use uninitialized value'): - var.eval() - if use_init: - sess.run(var.initializer) - with self.assertRaisesOpError('input is already initialized'): - var_zero.eval() - self.assertAllClose(np.ones(shape), var.eval()) - else: - var_zero.eval() - self.assertAllClose(np.zeros(shape), var.eval()) - - def testZeroInitializer(self): - for dtype in (dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64): - for use_init in (False, True): - self._testZeroInitializer([10, 20], array_ops.ones( - [10, 20], dtype=dtype), use_init) - - -class ZeroVarInitializerOpTest(test.TestCase): - - def _testZeroVarInitializer(self, shape, initializer, use_init): - var = resource_variable_ops.ResourceVariable(initializer) - var_zero = variables_lib2.zero_initializer(var) - - with self.cached_session() as sess: - with self.assertRaisesOpError('Error while reading resource variable'): - var.eval() - if use_init: - sess.run(var.initializer) - with self.assertRaisesOpError('input is already initialized'): - var_zero.eval() - self.assertAllClose(np.ones(shape), var.eval()) - else: - var_zero.eval() - self.assertAllClose(np.zeros(shape), var.eval()) - - def testZeroVarInitializer(self): - for dtype in (dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64): - for use_init in (False, True): - self._testZeroVarInitializer([10, 20], - array_ops.ones([10, 20], dtype=dtype), - use_init) - - -class FilterVariablesTest(test.TestCase): - - def setUp(self): - g = ops.Graph() - with g.as_default(): - var_list = [] - var_list.append(variables_lib.VariableV1(0, name='conv1/weights')) - var_list.append(variables_lib.VariableV1(0, name='conv1/biases')) - var_list.append(variables_lib.VariableV1(0, name='conv2/weights')) - var_list.append(variables_lib.VariableV1(0, name='conv2/biases')) - var_list.append(variables_lib.VariableV1(0, name='clfs/weights')) - var_list.append(variables_lib.VariableV1(0, name='clfs/biases')) - self._var_list = var_list - - def _test_filter_variables(self, - expected_var_names, - include_patterns=None, - exclude_patterns=None, - reg_search=True): - filtered_var_list = variables_lib2.filter_variables( - self._var_list, - include_patterns=include_patterns, - exclude_patterns=exclude_patterns, - reg_search=reg_search) - - filtered_var_names = [var.op.name for var in filtered_var_list] - - for name in filtered_var_names: - self.assertIn(name, expected_var_names) - for name in expected_var_names: - self.assertIn(name, filtered_var_names) - self.assertEqual(len(filtered_var_names), len(expected_var_names)) - - def testNoFiltering(self): - self._test_filter_variables(expected_var_names=[ - 'conv1/weights', 'conv1/biases', 'conv2/weights', 'conv2/biases', - 'clfs/weights', 'clfs/biases' - ]) - - def testIncludeBiases(self): - self._test_filter_variables( - expected_var_names=['conv1/biases', 'conv2/biases', 'clfs/biases'], - include_patterns=['biases']) - - def testExcludeWeights(self): - self._test_filter_variables( - expected_var_names=['conv1/biases', 'conv2/biases', 'clfs/biases'], - exclude_patterns=['weights']) - - def testExcludeWeightsAndConv1(self): - self._test_filter_variables( - expected_var_names=['conv2/biases', 'clfs/biases'], - exclude_patterns=['weights', 'conv1']) - - def testTwoIncludePatternsEnsureNoVariablesTwiceInFilteredList(self): - self._test_filter_variables( - expected_var_names=[ - 'conv1/weights', 'conv1/biases', 'conv2/weights', 'clfs/weights' - ], - include_patterns=['conv1', 'weights']) - - def testIncludeConv1ExcludeBiases(self): - self._test_filter_variables( - expected_var_names=['conv1/weights'], - include_patterns=['conv1'], - exclude_patterns=['biases']) - - def testRegMatchIncludeBiases(self): - self._test_filter_variables( - expected_var_names=['conv1/biases', 'conv2/biases', 'clfs/biases'], - include_patterns=['.*biases'], - reg_search=False) - - def testRegMatchIncludeBiasesWithIncompleteRegExpHasNoMatches(self): - self._test_filter_variables( - expected_var_names=[], include_patterns=['biases'], reg_search=False) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/framework/testdata/bundle_checkpoint.data-00000-of-00001 b/tensorflow/contrib/framework/testdata/bundle_checkpoint.data-00000-of-00001 deleted file mode 100644 index a0e27e341d0..00000000000 Binary files a/tensorflow/contrib/framework/testdata/bundle_checkpoint.data-00000-of-00001 and /dev/null differ diff --git a/tensorflow/contrib/framework/testdata/bundle_checkpoint.index b/tensorflow/contrib/framework/testdata/bundle_checkpoint.index deleted file mode 100644 index 264b524fe2b..00000000000 Binary files a/tensorflow/contrib/framework/testdata/bundle_checkpoint.index and /dev/null differ diff --git a/tensorflow/contrib/framework/testdata/bundle_checkpoint_vocab.txt b/tensorflow/contrib/framework/testdata/bundle_checkpoint_vocab.txt deleted file mode 100644 index fa220f83637..00000000000 --- a/tensorflow/contrib/framework/testdata/bundle_checkpoint_vocab.txt +++ /dev/null @@ -1,5 +0,0 @@ -zero -one -two -three -four diff --git a/tensorflow/contrib/framework/testdata/bundle_checkpoint_vocab_with_oov.txt b/tensorflow/contrib/framework/testdata/bundle_checkpoint_vocab_with_oov.txt deleted file mode 100644 index d7ed9e6a2d5..00000000000 --- a/tensorflow/contrib/framework/testdata/bundle_checkpoint_vocab_with_oov.txt +++ /dev/null @@ -1,4 +0,0 @@ -zero -one -two -three diff --git a/tensorflow/contrib/framework/testdata/keyword.txt b/tensorflow/contrib/framework/testdata/keyword.txt deleted file mode 100644 index 374f36fd738..00000000000 --- a/tensorflow/contrib/framework/testdata/keyword.txt +++ /dev/null @@ -1,3 +0,0 @@ -knitting -eminem -MISSING diff --git a/tensorflow/contrib/framework/testdata/keyword_new.txt b/tensorflow/contrib/framework/testdata/keyword_new.txt deleted file mode 100644 index 56a9c4d415d..00000000000 --- a/tensorflow/contrib/framework/testdata/keyword_new.txt +++ /dev/null @@ -1,4 +0,0 @@ -MISSING -knitting -flask -eminem diff --git a/tensorflow/contrib/fused_conv/BUILD b/tensorflow/contrib/fused_conv/BUILD deleted file mode 100644 index 35abaae8d63..00000000000 --- a/tensorflow/contrib/fused_conv/BUILD +++ /dev/null @@ -1,205 +0,0 @@ -# Description: -# A Fused Conv Bias Activation operator wrapper. -# APIs are meant to change over time. - -load( - "//tensorflow:tensorflow.bzl", - "tf_custom_op_library", - "tf_gen_op_libs", - "tf_gen_op_wrapper_py", - "tf_kernel_library", -) -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = ["//visibility:private"], - licenses = ["notice"], # Apache 2.0 -) - -package_group( - name = "friends", - packages = [ - "//tensorflow/...", - ], -) - -exports_files(["LICENSE"]) - -tf_custom_op_py_library( - name = "fused_conv_py", - srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - dso = [":python/ops/_fused_conv2d_bias_activation_op.so"], - kernels = [ - ":fused_conv2d_bias_activation_op_kernels", - ":fused_conv2d_bias_activation_op_op_lib", - ], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":fused_conv2d_bias_activation_op", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:session", - "//tensorflow/python:util", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -tf_kernel_library( - name = "fused_conv2d_bias_activation_op_kernels", - srcs = [ - "kernels/fused_conv2d_bias_activation_op.cc", - "kernels/fused_conv2d_bias_activation_op.h", - "kernels/fused_conv_ops_gpu.h", - ], - prefix = "fused_conv2d_bias_activation_op", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/core:autotuning_proto_cc", - "//tensorflow/core:conv_autotuning_proto_cc", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_proto_parsing", - "//tensorflow/core:logger", - "//tensorflow/core:stream_executor", - "//tensorflow/core/kernels:bounds_check", - "//tensorflow/core/kernels:conv_2d_hdrs", - "//tensorflow/core/kernels:conv_ops_gpu_hdrs", - "//tensorflow/core/kernels:cwise_lib_hdrs", - "//tensorflow/core/kernels:eigen_contraction_kernel", - "//tensorflow/core/kernels:gpu_util_hdrs", - "//tensorflow/core/kernels:ops_util_hdrs", - "//third_party/eigen3", - "@com_google_absl//absl/time", - "@local_config_cuda//cuda:cudnn_header", - ], - alwayslink = 1, -) - -tf_custom_op_library( - name = "python/ops/_fused_conv2d_bias_activation_op.so", - srcs = [ - "kernels/fused_conv2d_bias_activation_op.cc", - "kernels/fused_conv2d_bias_activation_op.h", - "kernels/fused_conv_ops_gpu.h", - "ops/fused_conv2d_bias_activation_op.cc", - ], - deps = [ - "//tensorflow/core:autotuning_proto_cc", - "//tensorflow/core:conv_autotuning_proto_cc", - "//tensorflow/core:lib_proto_parsing", - "//tensorflow/core/kernels:bounds_check_lib", - "//tensorflow/core/kernels:conv_2d_hdrs", - "//tensorflow/core/kernels:conv_ops_gpu_hdrs", - "//tensorflow/core/kernels:cwise_lib_hdrs", - "//tensorflow/core/kernels:eigen_contraction_kernel", - "//tensorflow/core/kernels:gpu_util_hdrs", - "//tensorflow/core/kernels:ops_util_hdrs", - "@local_config_cuda//cuda:cudnn_header", - ], -) - -tf_gen_op_libs( - op_lib_names = ["fused_conv2d_bias_activation_op"], - deps = ["//tensorflow/core:lib_proto_parsing"], -) - -tf_gen_op_wrapper_py( - name = "fused_conv2d_bias_activation_op", - deps = [":fused_conv2d_bias_activation_op_op_lib"], -) - -py_library( - name = "fused_conv2d_bias_activation_op_test_base", - testonly = 1, - srcs = ["python/ops/fused_conv2d_bias_activation_op_test_base.py"], - visibility = ["//tensorflow/compiler/tf2xla:internal"], - deps = [ - ":fused_conv_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:nn", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//third_party/py/numpy", - "@absl_py//absl/testing:parameterized", - ], -) - -cuda_py_test( - name = "fused_conv2d_bias_activation_op_test", - size = "large", - srcs = ["python/ops/fused_conv2d_bias_activation_op_test.py"], - additional_deps = [ - ":fused_conv2d_bias_activation_op_test_base", - "//tensorflow/python:client_testlib", - ], - tags = [ - "manual", # TODO(b/117128481): re-enable after fixing OSS build - "no_pip", - "requires-gpu-sm70", - ], -) - -cuda_py_test( - name = "fused_conv2d_bias_activation_benchmark", - srcs = ["python/ops/fused_conv2d_bias_activation_benchmark.py"], - additional_deps = [ - ":fused_conv_py", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_benchmark", - "//tensorflow/python:random_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - "//tensorflow/core:protos_all_py", - ], - main = "python/ops/fused_conv2d_bias_activation_benchmark.py", - tags = [ - "manual", # TODO(b/117128481): re-enable after fixing OSS build - "nogpu", - "requires-gpu-sm70", - ], -) - -cuda_py_test( - name = "tensorrt_fused_conv_test", - srcs = ["python/ops/tensorrt_fused_conv_test.py"], - additional_deps = [ - ":fused_conv_py", - "//tensorflow/python/compiler/tensorrt:tf_trt_integration_test_base", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - ], - tags = [ - "manual", # TODO(b/117128481): re-enable after fixing OSS build - "no_cuda_on_cpu_tap", - "no_rocm", - "no_windows", - "nomac", - ], - xla_enable_strict_auto_jit = True, -) diff --git a/tensorflow/contrib/fused_conv/__init__.py b/tensorflow/contrib/fused_conv/__init__.py deleted file mode 100644 index dd4d3fc707d..00000000000 --- a/tensorflow/contrib/fused_conv/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops and modules related to fused_conv2d_bias_activation.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.fused_conv.python.ops.fused_conv2d_bias_activation_op import * -from tensorflow.python.util.all_util import remove_undocumented - -remove_undocumented(__name__, ['fused_conv2d_bias_activation']) diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc deleted file mode 100644 index e3e2f51beb7..00000000000 --- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc +++ /dev/null @@ -1,1053 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#if GOOGLE_CUDA -#define EIGEN_USE_GPU -#endif // GOOGLE_CUDA - -#define EIGEN_USE_THREADS - -#include "tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.h" - -#include "tensorflow/core/framework/bounds_check.h" -#include "tensorflow/core/framework/numeric_op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_slice.h" -#include "tensorflow/core/kernels/conv_2d.h" -#include "tensorflow/core/kernels/cwise_ops.h" -#include "tensorflow/core/kernels/ops_util.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/util/padding.h" -#include "tensorflow/core/util/use_cudnn.h" - -#if defined(TENSORFLOW_USE_CUSTOM_CONTRACTION_KERNEL) -#include "tensorflow/core/kernels/eigen_contraction_kernel.h" -#endif // defined(TENSORFLOW_USE_CUSTOM_CONTRACTION_KERNEL) - -#if GOOGLE_CUDA -#include "google/protobuf/duration.pb.h" -#include "absl/time/time.h" -#include "third_party/gpus/cudnn/cudnn.h" -#include "tensorflow/core/kernels/conv_ops_gpu.h" -#include "tensorflow/core/platform/logger.h" -#include "tensorflow/core/platform/stream_executor.h" -#include "tensorflow/core/protobuf/autotuning.pb.h" -#include "tensorflow/core/protobuf/conv_autotuning.pb.h" -#include "tensorflow/core/util/activation_mode.h" -#include "tensorflow/stream_executor/dnn.h" -#endif // GOOGLE_CUDA - -namespace tensorflow { - -namespace { -typedef Eigen::ThreadPoolDevice CPUDevice; -typedef Eigen::GpuDevice GPUDevice; - -template -struct RawType { - using type = T; -}; - -template <> -struct RawType { - using type = int8; -}; - -// Template struct to convert int8x4 to int32. -// (for NCHW_VECT_C with element type int8, we can consider it to be -// an NCHW layout with element type int32 for operations like padding). -template -struct Int8x4ToInt32 { - // By default, do not change T. - using type = T; -}; - -template <> -struct Int8x4ToInt32 { - using type = int32; -}; -} // namespace - -// WARNING: Packing specializations defined in eigen_spatial_convolutions.h do -// not support packing expressions of QInt8 type. However, default Eigen -// gebp_kernel for QInt8 is too slow to be considered useful for anything. -#if defined(TENSORFLOW_USE_CUSTOM_CONTRACTION_KERNEL) - -template -class LaunchFusedConv2DBiasActivationOp { - using T = qint8; // conv_input and filter type - using ComputeT = float; // convert inputs to fp32 for tensor contraction - using TempT = float; // temporary accumulator type for tensor contraction - - public: - void launch(OpKernelContext* ctx, bool cudnn_use_autotune, - const Tensor& conv_input, const Tensor& conv_input_scale, - const Tensor& filter, int32 row_stride, int32 col_stride, - const Eigen::PaddingType& padding, const Tensor& side_input, - ScaleType side_input_scale, const Tensor& bias, - ActivationMode activation_mode, TensorFormat data_format, - FilterTensorFormat filter_format, Tensor* output) { - static_assert(std::is_same::value, - "Scale and Bias must be of the same type."); - - // Output tensor has type T (QInt8), but we can only evaluate Int8 Tensor - // contraction using 32-bit accumulation (QInt32). - Tensor temp_output(DataTypeToEnum::value, output->shape()); - - constexpr int32 row_dilation = 1; - constexpr int32 col_dilation = 1; - - auto& device = ctx->eigen_device(); - - // CPU convolution works with input in NHWC and filter in HWIO data formats. - // NOTE: This code is mostly shared with 'Conv2D' and 'FusedConv2D'. - - BiasActivationOutputKernel output_kernel(conv_input_scale, side_input, - side_input_scale, bias, - activation_mode, output); - - if (filter.dim_size(0) == 1 && filter.dim_size(1) == 1 && row_stride == 1 && - col_stride == 1) { - int conv_width = // Width for the convolution step. - output->dim_size(0) * output->dim_size(1) * output->dim_size(2); - - Eigen::array, 1> dim_pair; - dim_pair[0] = Eigen::IndexPair(1, 0); - - auto out = temp_output.shaped({conv_width, filter.dim_size(3)}); - auto in0 = conv_input.shaped({conv_width, filter.dim_size(2)}); - auto in1 = filter.shaped({filter.dim_size(2), filter.dim_size(3)}); - - out.device(device) = in0.cast().contract( - in1.cast(), dim_pair, output_kernel); - - } else if (filter.dim_size(0) == conv_input.dim_size(1) && - filter.dim_size(1) == conv_input.dim_size(2) && - row_dilation == 1 && col_dilation == 1 && - padding == Eigen::PaddingType::PADDING_VALID) { - // If the input data and filter have the same height/width, - // reduce the 2D convolution to matrix multiplication. - const auto k = // Length of reduction dimension. - filter.dim_size(0) * filter.dim_size(1) * filter.dim_size(2); - - Eigen::array, 1> dim_pair; - dim_pair[0] = Eigen::IndexPair(1, 0); - - auto out = temp_output.shaped( - {conv_input.dim_size(0), filter.dim_size(3)}); - auto in0 = conv_input.shaped({conv_input.dim_size(0), k}); - auto in1 = filter.shaped({k, filter.dim_size(3)}); - - out.device(device) = in0.cast().contract( - in1.cast(), dim_pair, output_kernel); - - } else { - auto out = temp_output.tensor(); - auto in0 = conv_input.tensor(); - auto in1 = filter.tensor(); - - // Need to swap row/col when calling Eigen. - out.device(device) = Eigen::SpatialConvolution( - in0.cast(), in1.cast(), col_stride, row_stride, - padding, col_dilation, row_dilation, output_kernel); - } - } - - private: - // Contraction output mapper for temporary QInt32 tensor. - using ContractionOutputMapper = - Eigen::internal::blas_data_mapper; - - // This output kernel computes an expressions corresponding to cuDNN - // implementation of INT8 cudnnConvolutionBiasActivationForward: - // https://docs.nvidia.com/deeplearning/sdk/cudnn-developer-guide/index.html#scaling-parameters__fig-conv-bias-activation-forward - struct BiasActivationOutputKernel { - static constexpr ScaleType kMaxRange = static_cast(127.f); - static constexpr ScaleType kMinRange = static_cast(-128.f); - - explicit BiasActivationOutputKernel(const Tensor& conv_input_scale, - const Tensor& side_input, - ScaleType side_input_scale, - const Tensor& bias, - ActivationMode activation_mode, - Tensor* output) - : activation_mode(activation_mode), - conv_input_scale_data(conv_input_scale.flat().data()), - bias_data(bias.flat().data()), - side_input_data(side_input.flat().data()), - side_input_scale(side_input_scale), - output_data(const_cast(output->flat().data())), - conv_input_scale_tensor_size(conv_input_scale.NumElements()) {} - - EIGEN_ALWAYS_INLINE void operator()( - const ContractionOutputMapper& conv_output_mapper, - const Eigen::TensorContractionParams& params, Eigen::Index i, - Eigen::Index j, Eigen::Index num_rows, Eigen::Index num_cols) const { - DCHECK(params.swapped_arguments); - - const auto stride = conv_output_mapper.stride(); - - const BiasType* bias_base = bias_data + i; - const T* side_input_base = side_input_data + i + j * stride; - T* output_base = output_data + i + j * stride; - - for (int col = 0; col < num_cols; ++col) { - // A column of an output tensor after QInt8xQInt8 -> QInt32 contraction. - // This is a temporary tensor, that we will scale, add bias with - // side_input, and quantize before writing to final output tensor. - typename TTypes::UnalignedTensor conv_output( - &conv_output_mapper(0, col), num_rows); - - // A column of output quantized tensor corresponding to conv output row. - typename TTypes::UnalignedTensor output(output_base + col * stride, - num_rows); - - // Pointers to the input data accounting for the column offset. - TempT* conv_output_ptr = conv_output.data(); - const T* side_input_ptr = side_input_base + col * stride; - const BiasType* bias_ptr = bias_base; - - static_assert( - std::is_same::value, - "Temporary contraction result type must match with scale type."); - - // (1) Scale and add bias. - // NOTE(ezhulenev): We do not use Eigen expressions for this loop, - // because it seems that packet FMA produces slightly different results, - // and we are targeting close equality with Nvidia implementation. - // We could use std::fmaf, but it can be ~50x slower, on machines - // without fma instruction. - double conv_input_scale = static_cast(*conv_input_scale_data); - for (int idx = 0; idx < num_rows; ++idx) { - if (conv_input_scale_tensor_size > 1) { - conv_input_scale = static_cast(conv_input_scale_data[idx]); - } - conv_output_ptr[idx] = - static_cast(conv_output_ptr[idx]) * conv_input_scale + - static_cast(bias_ptr[idx]); - if (side_input_scale != 0.0f) { - conv_output_ptr[idx] = static_cast(side_input_ptr[idx]) * - static_cast(side_input_scale) + - static_cast(conv_output_ptr[idx]); - } - } - - // (2) Round-up, clip and apply activation function. - ScaleType lower_bound = - (activation_mode == ActivationMode::NONE ? kMinRange : 0); - output = - conv_output - // scalar_round_op_google uses HALF_TO_EVEN. - .unaryExpr(Eigen::internal::scalar_round_op_google()) - .clip(lower_bound, kMaxRange) - .template cast(); - } - } - - private: - ActivationMode activation_mode; - const ScaleType* conv_input_scale_data; - const BiasType* bias_data; - const T* side_input_data; - ScaleType side_input_scale; - T* output_data; - const int conv_input_scale_tensor_size; - }; -}; -#endif // defined(TENSORFLOW_USE_CUSTOM_CONTRACTION_KERNEL) - -// T is the element type of the conv_input, filter and side_input tensors. -// BiasType is the element type of the bias tensor, which can be different. -// ScaleType is the type used for conv_input_scale, side_input_scale. -template -class FusedConv2DBiasActivationOp : public OpKernel { - public: - enum InputIndexes { - kConvInput = 0, - kFilter, - kBias, - kSideInput, - kConvInputScale, - kSideInputScale, - kNumInputs - }; - - explicit FusedConv2DBiasActivationOp(OpKernelConstruction* context) - : OpKernel(context) { - string data_format_str, filter_format_str; - CHECK_EQ(kNumInputs, context->num_inputs()); - OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format_str)); - OP_REQUIRES(context, FormatFromString(data_format_str, &data_format_), - errors::InvalidArgument("Invalid data format")); - OP_REQUIRES_OK(context, - context->GetAttr("filter_format", &filter_format_str)); - OP_REQUIRES(context, - FilterFormatFromString(filter_format_str, &filter_format_), - errors::InvalidArgument("Invalid filter format")); - - std::vector strides; - OP_REQUIRES_OK(context, context->GetAttr("strides", &strides)); - OP_REQUIRES(context, strides.size() == 4, - errors::InvalidArgument("Sliding window strides field must " - "specify 4 dimensions")); - - stride_rows_ = GetTensorDim(strides, data_format_, 'H'); - stride_cols_ = GetTensorDim(strides, data_format_, 'W'); - OP_REQUIRES( - context, - (GetTensorDim(strides, data_format_, 'N') == 1 && - GetTensorDim(strides, data_format_, 'C') == 1), - errors::Unimplemented("Convolutional strides are not supported in " - "the batch and depth dimensions.")); - - std::vector dilations; - OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations)); - OP_REQUIRES(context, dilations == std::vector({1, 1, 1, 1}), - errors::InvalidArgument("Dilations must be all equal to 1.")); - - constexpr bool is_cpu = std::is_same::value; - constexpr bool is_gpu = std::is_same::value; - OP_REQUIRES(context, is_cpu || is_gpu, - errors::InvalidArgument("Unknown Device type.")); - - constexpr bool is_qint8 = std::is_same::value; - - if (is_qint8 && is_gpu) { - // Assuming qint8 <--> NCHW_VECT_C, OIHW_VECT_I (int8x4) here. - - // Note: Only NCHW_VECT_C format is supported for int8 on GPU. - // This is because it is expected to be the fastest, and our previous - // tests found cudnn 6 does not fully support the other formats for int8 - // mode. - OP_REQUIRES( - context, data_format_ == FORMAT_NCHW_VECT_C, - errors::InvalidArgument( - "qint8 should be used with data_format NCHW_VECT_C on GPU.")); - OP_REQUIRES( - context, filter_format_ == FORMAT_OIHW_VECT_I, - errors::InvalidArgument( - "qint8 should be used with filter_format OIHW_VECT_I on GPU.")); - - } else if (is_qint8 && is_cpu) { - // On CPU we implement convolution with Eigen Tensor contraction, it - // requries NHWC and HWIO formats for input and kernel. - - OP_REQUIRES(context, data_format_ == FORMAT_NHWC, - errors::InvalidArgument( - "qint8 should be used with data_format NHWC on CPU.")); - OP_REQUIRES(context, filter_format_ == FORMAT_HWIO, - errors::InvalidArgument( - "qint8 should be used with filter_format HWIO on CPU.")); - } - - OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_type_)); - eigen_padding_type_ = BrainPadding2EigenPadding(padding_type_); - string activation_mode_str; - OP_REQUIRES_OK(context, - context->GetAttr("activation_mode", &activation_mode_str)); - OP_REQUIRES_OK(context, GetActivationModeFromString(activation_mode_str, - &activation_mode_)); - OP_REQUIRES(context, - activation_mode_ == ActivationMode::RELU || - activation_mode_ == ActivationMode::NONE, - errors::InvalidArgument( - "Current implementation only supports RELU or NONE " - "as the activation function.")); - cudnn_use_autotune_ = CudnnUseAutotune(); - } - - Status CheckShape(const Tensor& tensor, const string& tensor_name) { - const int num_dims = tensor.dims(); - for (int i = 0; i < num_dims; i++) { - if (!FastBoundsCheck(tensor.dim_size(i), - std::numeric_limits::max())) { - return errors::InvalidArgument(tensor_name, " dimension ", i, - " too large"); - } - } - // If there is a 5th dimension it is the VECT_C or VECT_I dimension. - if (num_dims == 5 && tensor.dim_size(4) != 4) { - return errors::InvalidArgument("The last dimension of ", tensor_name, - " must be of size 4 for qint8."); - } - return Status::OK(); - } - - void Compute(OpKernelContext* context) override { - // The conv_input tensor is one of the following formats: - // NHWC, NCHW, NCHW_VECT_C. - const Tensor& conv_input = context->input(kConvInput); - OP_REQUIRES_OK(context, CheckShape(conv_input, "conv_input")); - - // The filter tensor is one of the following formats: - // HWIO, OIHW, OIHW_VECT_I. - const Tensor& filter = context->input(kFilter); - OP_REQUIRES_OK(context, CheckShape(filter, "filter")); - - // Input bias is a 1-D tensor, with size matching output depth. - const Tensor& bias = context->input(kBias); - OP_REQUIRES_OK(context, CheckShape(bias, "bias")); - - const Tensor& conv_input_scale_tensor = context->input(kConvInputScale); - const Tensor& side_input_scale_tensor = context->input(kSideInputScale); - - auto side_input_scale = *reinterpret_cast( - side_input_scale_tensor.tensor_data().data()); - - // If side_input_scale != 0, then side_input is not ignored and - // has the same type and dimensions as the output. - const Tensor& side_input = context->input(kSideInput); - if (side_input_scale != 0) { - OP_REQUIRES_OK(context, CheckShape(side_input, "side_input")); - } - - // TODO(pauldonnelly): Switch to a more efficient mechanism to access - // dimension indexes and per-dimension attributes. - const int32 filter_rows = GetFilterDim(filter, filter_format_, 'H'); - const int32 filter_cols = GetFilterDim(filter, filter_format_, 'W'); - const int32 output_depth = GetFilterDim(filter, filter_format_, 'O'); - - const int32 batch_size = GetTensorDim(conv_input, data_format_, 'N'); - const int32 conv_input_rows = GetTensorDim(conv_input, data_format_, 'H'); - const int32 conv_input_cols = GetTensorDim(conv_input, data_format_, 'W'); - - int64 output_rows = 0, output_cols = 0, pad_rows = 0, pad_cols = 0; - OP_REQUIRES_OK(context, GetWindowedOutputSize(conv_input_rows, filter_rows, - stride_rows_, padding_type_, - &output_rows, &pad_rows)); - OP_REQUIRES_OK(context, GetWindowedOutputSize(conv_input_cols, filter_cols, - stride_cols_, padding_type_, - &output_cols, &pad_cols)); - // Initialize the output tensor shape according to data_format_ - TensorShape output_shape = ShapeFromFormat( - data_format_, batch_size, output_rows, output_cols, output_depth); - Tensor* output = nullptr; - OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, &output)); - - VLOG(2) << "FusedConv2DBiasActivation: conv_input_cols = " - << conv_input_cols << ", conv_input_rows = " << conv_input_rows - << ", filter_cols = " << filter_cols - << ", filter_rows = " << filter_rows - << ", stride_cols = " << stride_cols_ - << ", stride_rows = " << stride_rows_ - << ", output_depth = " << output_depth - << ", output_cols = " << output_cols - << ", output_rows = " << output_rows - << ", output_shape.num_elements = " << output_shape.num_elements(); - - // If there is nothing to compute, return. - if (output_shape.num_elements() == 0) { - return; - } - - launcher_.launch(context, cudnn_use_autotune_, conv_input, - conv_input_scale_tensor, filter, stride_rows_, - stride_cols_, eigen_padding_type_, side_input, - side_input_scale, bias, activation_mode_, data_format_, - filter_format_, output); - } - - private: - int32 stride_rows_, stride_cols_; - Padding padding_type_; - Eigen::PaddingType eigen_padding_type_; - ActivationMode activation_mode_; - TensorFormat data_format_; - FilterTensorFormat filter_format_; - LaunchFusedConv2DBiasActivationOp launcher_; - bool cudnn_use_autotune_; - - TF_DISALLOW_COPY_AND_ASSIGN(FusedConv2DBiasActivationOp); -}; - -#if defined(TENSORFLOW_USE_CUSTOM_CONTRACTION_KERNEL) -REGISTER_KERNEL_BUILDER( - Name("FusedConv2DBiasActivation") - .Device(DEVICE_CPU) - .TypeConstraint("T") - .TypeConstraint("Tbias"), - FusedConv2DBiasActivationOp); -#endif // defined(TENSORFLOW_USE_CUSTOM_CONTRACTION_KERNEL) - -#if GOOGLE_CUDA -namespace dnn = se::dnn; - -// Several functions are copyed over from tensorflow/core/kernels/gpu_utils, -// since this file may be compiled down to a tf_custom_op_library .so file, -// which can't depend on basic dependencies like tensorflow/core:lib. Instead, -// the code has to depend on whatever is the same in libtensorflow_framework.so. -// -// In theory, we can lift the dependencies of gpu_utils by turning it into a -// template library that provides duck typing, but I think duplication is the -// lesser of two evils. -namespace internal { -namespace { - -tensorflow::CudnnVersion GetCudnnVersion(se::StreamExecutor* stream_executor) { - tensorflow::CudnnVersion cudnn_version; - if (auto* dnn = stream_executor->AsDnn()) { - se::port::StatusOr version_or = dnn->GetVersion(); - if (version_or.ok()) { - const auto& version = version_or.ValueOrDie(); - cudnn_version.set_major(version.major_version()); - cudnn_version.set_minor(version.minor_version()); - cudnn_version.set_patch(version.patch()); - } - } - return cudnn_version; -} - -// Converts an absl::Duration to a google::protobuf::Duration. -inline google::protobuf::Duration ToDurationProto(absl::Duration duration) { - google::protobuf::Duration proto; - proto.set_seconds(absl::IDivDuration(duration, absl::Seconds(1), &duration)); - proto.set_nanos( - absl::IDivDuration(duration, absl::Nanoseconds(1), &duration)); - return proto; -} - -// Converts a google::protobuf::Duration to an absl::Duration. -inline absl::Duration FromDurationProto(google::protobuf::Duration proto) { - return absl::Seconds(proto.seconds()) + absl::Nanoseconds(proto.nanos()); -} - -tensorflow::ComputeCapability GetComputeCapability( - se::StreamExecutor* stream_executor) { - tensorflow::ComputeCapability cc; - int cc_major, cc_minor; - stream_executor->GetDeviceDescription().cuda_compute_capability(&cc_major, - &cc_minor); - cc.set_major(cc_major); - cc.set_minor(cc_minor); - return cc; -} - -void LogFusedConvForwardAutotuneResults( - se::dnn::DataType element_type, se::DeviceMemoryBase input_buffer, - se::DeviceMemoryBase filter_buffer, se::DeviceMemoryBase output_buffer, - se::DeviceMemoryBase bias_buffer, se::DeviceMemoryBase side_input_buffer, - const se::dnn::BatchDescriptor& input_desc, - const se::dnn::FilterDescriptor& filter_desc, - const se::dnn::BatchDescriptor& output_desc, - const se::dnn::ConvolutionDescriptor& conv_desc, double conv_scale, - double side_value_scale, se::dnn::ActivationMode activation_mode, - se::StreamExecutor* stream_exec, absl::Span results) { - AutotuningLog log; - { - ConvolutionProto instr; - instr.set_kind(se::dnn::ConvolutionKind::FORWARD_BIAS_ACTIVATION); - *instr.mutable_input() = input_desc.ToProto(element_type); - *instr.mutable_filter() = filter_desc.ToProto(element_type); - *instr.mutable_output() = output_desc.ToProto(element_type); - *instr.mutable_conv_desc() = conv_desc.ToProto(); - instr.set_conv_scale(conv_scale); - instr.set_side_value_scale(side_value_scale); - instr.set_activation(activation_mode); - instr.set_input_address(reinterpret_cast(input_buffer.opaque())); - instr.set_filter_address(reinterpret_cast(filter_buffer.opaque())); - instr.set_output_address(reinterpret_cast(output_buffer.opaque())); - instr.set_bias_address(reinterpret_cast(bias_buffer.opaque())); - instr.set_side_input_address( - reinterpret_cast(side_input_buffer.opaque())); - log.mutable_instr()->PackFrom(std::move(instr)); - } - *log.mutable_cudnn_version() = GetCudnnVersion(stream_exec); - *log.mutable_compute_capability() = GetComputeCapability(stream_exec); - log.set_device_pci_bus_id(stream_exec->GetDeviceDescription().pci_bus_id()); - { - string blas_version; - if (auto* blas = stream_exec->AsBlas()) { - if (blas->GetVersion(&blas_version).ok()) { - log.set_blas_version(blas_version); - } - } - } - for (const auto& result : results) { - *log.add_results() = result; - } - Logger::GetSingleton()->LogProto(log); -} - -Status BestCudnnConvAlgorithm(absl::Span results, - se::dnn::AlgorithmConfig* algo) { - const AutotuneResult* best_result = std::min_element( - results.begin(), results.end(), - [](const AutotuneResult& lhs, const AutotuneResult& rhs) { - return internal::FromDurationProto(lhs.run_time()) < - internal::FromDurationProto(rhs.run_time()); - }); - - const AutotuneResult* best_result_no_scratch = std::min_element( - results.begin(), results.end(), - [](const AutotuneResult& lhs, const AutotuneResult& rhs) { - return std::make_tuple(lhs.scratch_bytes(), - internal::FromDurationProto(lhs.run_time())) < - std::make_tuple(rhs.scratch_bytes(), - internal::FromDurationProto(rhs.run_time())); - }); - - if (best_result == results.end()) { - return errors::NotFound("No algorithm worked!"); - } - algo->set_algorithm({best_result->conv().algorithm(), - best_result->conv().tensor_ops_enabled()}); - if (best_result_no_scratch != results.end() && - best_result_no_scratch->scratch_bytes() == 0) { - algo->set_algorithm_no_scratch( - {best_result_no_scratch->conv().algorithm(), - best_result_no_scratch->conv().tensor_ops_enabled()}); - } - return Status::OK(); -} - -} // namespace -} // namespace internal - -// A dummy type to group forward convolution autotune results together. -struct ConvBiasActivationAutoTuneGroup { - static string name() { return "ConvBiasActivation"; } -}; -typedef AutoTuneSingleton - AutoTuneConvBiasActivation; - -// Allocates 'transformed_tensor' and transforms 'nhwc_tensor' into it -// using the specified 'batch_size', 'rows', 'cols', and 'depth' dimensions. -template -Status TransformNHWCToNCHW(OpKernelContext* ctx, const Tensor& nhwc_tensor, - int batch_size, int rows, int cols, int depth, - Tensor* transformed_tensor, const Tensor** result) { - TensorShape nchw_shape = - ShapeFromFormat(FORMAT_NCHW, batch_size, rows, cols, depth); - if (depth > 1) { - TF_RETURN_IF_ERROR(ctx->allocate_temp(DataTypeToEnum::value, nchw_shape, - transformed_tensor)); - functor::NHWCToNCHW()( - ctx->eigen_device(), nhwc_tensor.tensor(), - transformed_tensor->tensor()); - } else { - // If depth <= 1, then just reshape. - CHECK(transformed_tensor->CopyFrom(nhwc_tensor, nchw_shape)); - } - *result = transformed_tensor; - return Status::OK(); -} - -// Adjusts padding so cudnn supports it. Sets `adjusted_padding` to be the -// adjusted padding, and `extra_padding_before` and `extra_padding_after` to be -// the extra padding that FusedConv needs to apply before calling cudnn. -void AdjustPaddingForCudnn(int padding, bool is_int8x4, int filter_size, - int* adjusted_padding, int* extra_padding_before, - int* extra_padding_after) { -#if CUDNN_VERSION < 7000 - if (is_int8x4 && filter_size >= 6) { - // TODO(b/70795525): Remove after NVIDIA fixes this bug with int8 fused - // convolution. I don't know cuDNN7 still has the bug, so enable this - // workaround for cuDNN6 or older. - *adjusted_padding = 0; - *extra_padding_before = padding / 2; - *extra_padding_after = padding - *extra_padding_before; - return; - } -#endif - *adjusted_padding = padding / 2 * 2; - *extra_padding_before = 0; - *extra_padding_after = padding % 2; -} - -template -void LaunchFusedConv2DBiasActivationOp:: - launch(OpKernelContext* ctx, bool cudnn_use_autotune, - const Tensor& conv_input_param, const Tensor& conv_input_scale, - const Tensor& filter_param, int32 row_stride, int32 col_stride, - const Eigen::PaddingType& padding, const Tensor& side_input_param, - ScaleType side_input_scale, const Tensor& bias, - ActivationMode activation_mode, TensorFormat data_format, - FilterTensorFormat filter_format, Tensor* output_param) { - auto* stream = ctx->op_device_context()->stream(); - OP_REQUIRES(ctx, stream, errors::Internal("No GPU stream available.")); - - // TODO(yangzihao): refactor all the complicated/duplicated code in regular - // conv ops to a shared conv utility. - - // Assuming qint8 <--> NCHW_VECT_C, OIHW_VECT_I (int8x4) here. - constexpr bool is_int8x4 = std::is_same::value; - constexpr int rank = is_int8x4 ? 5 : 4; - constexpr int vect = is_int8x4 ? 4 : 1; - - if (is_int8x4) { - int cc_major, cc_minor; - stream->parent()->GetDeviceDescription().cuda_compute_capability(&cc_major, - &cc_minor); - OP_REQUIRES( - ctx, ((cc_major == 6 && cc_minor >= 1) || cc_major > 6), - errors::Unimplemented( - "FusedConv2DBiasActivation for int8 is only supported on GPUs with " - "compute capability 6.1 or later.")); - } - - const int batch_size = GetTensorDim(conv_input_param, data_format, 'N'); - int conv_input_rows = GetTensorDim(conv_input_param, data_format, 'H'); - int conv_input_cols = GetTensorDim(conv_input_param, data_format, 'W'); - - const int conv_input_depth = - GetTensorDim(conv_input_param, data_format, 'C') * vect; - const int output_rows = GetTensorDim(*output_param, data_format, 'H'); - const int output_cols = GetTensorDim(*output_param, data_format, 'W'); - const int output_depth = GetFilterDim(filter_param, filter_format, 'O'); - const int filter_rows = GetFilterDim(filter_param, filter_format, 'H'); - const int filter_cols = GetFilterDim(filter_param, filter_format, 'W'); - int padding_rows = 0; - int padding_cols = 0; - const Tensor* conv_input = &conv_input_param; - - Tensor maybe_padded_conv_input; - if (padding == Eigen::PADDING_SAME) { - // Total padding on rows and cols is - // Pr = (R' - 1) * S + Kr - R - // Pc = (C' - 1) * S + Kc - C - // where (R', C') are output dimensions, (R, C) are input dimensions, S - // is stride, (Kr, Kc) are filter dimensions. - // We pad Pr/2 on the left and Pr - Pr/2 on the right, Pc/2 on the top - // and Pc - Pc/2 on the bottom. When Pr or Pc is odd, this means - // we pad more on the right and bottom than on the top and left. - padding_rows = std::max( - 0, (output_rows - 1) * row_stride + filter_rows - conv_input_rows); - padding_cols = std::max( - 0, (output_cols - 1) * col_stride + filter_cols - conv_input_cols); - int extra_top_padding = 0; - int extra_bottom_padding = 0; - int extra_left_padding = 0; - int extra_right_padding = 0; - AdjustPaddingForCudnn(padding_rows, is_int8x4, filter_rows, &padding_rows, - &extra_top_padding, &extra_bottom_padding); - AdjustPaddingForCudnn(padding_cols, is_int8x4, filter_cols, &padding_cols, - &extra_left_padding, &extra_right_padding); - if (extra_top_padding != 0 || extra_bottom_padding != 0 || - extra_left_padding != 0 || extra_right_padding != 0) { - const int new_conv_input_rows = - conv_input_rows + extra_top_padding + extra_bottom_padding; - const int new_conv_input_cols = - conv_input_cols + extra_left_padding + extra_right_padding; - - using VectT = typename Int8x4ToInt32::type>::type; - auto pad_data_format = is_int8x4 ? FORMAT_NCHW : data_format; - - OP_REQUIRES_OK( - ctx, ctx->allocate_temp( - DataTypeToEnum::value, - ShapeFromFormat(data_format, batch_size, new_conv_input_rows, - new_conv_input_cols, conv_input_depth), - &maybe_padded_conv_input)); - - auto conv_input_eigen_tensor = - To32Bit(conv_input_param.reinterpret_last_dimension()); - auto padded_conv_input_eigen_tensor = To32Bit( - maybe_padded_conv_input.reinterpret_last_dimension()); - - functor::PadInput()( - ctx->eigen_device(), conv_input_eigen_tensor, - {{extra_top_padding, extra_left_padding}}, - {{extra_bottom_padding, extra_right_padding}}, - padded_conv_input_eigen_tensor, pad_data_format); - - conv_input = &maybe_padded_conv_input; - conv_input_rows = new_conv_input_rows; - conv_input_cols = new_conv_input_cols; - } - } - - Tensor maybe_transformed_conv_input, maybe_transformed_side_input; - Tensor maybe_transformed_output; - const Tensor* side_input = &side_input_param; - Tensor* output = output_param; - - // NOTE: Here and elsewhere, checking 'is_int8x4' may look unnecessary - // and inefficient, but it is actually both a time and code size optimization, - // since 'is_int8x4' is a constexpr determined by the template parameter. - if (!is_int8x4 && data_format == FORMAT_NHWC) { - OP_REQUIRES_OK(ctx, (TransformNHWCToNCHW( - ctx, *conv_input, batch_size, conv_input_rows, - conv_input_cols, conv_input_depth, - &maybe_transformed_conv_input, &conv_input))); - if (side_input_scale != 0) { - OP_REQUIRES_OK( - ctx, (TransformNHWCToNCHW( - ctx, side_input_param, batch_size, output_rows, output_cols, - output_depth, &maybe_transformed_side_input, &side_input))); - } - if (output_depth > 1) { - // Allocate a tensor for the NCHW output of the kernel and point output - // to it. Afterwards, we will transform it to NHWC while copying back to - // 'output_param'. - TensorShape nchw_shape = ShapeFromFormat( - FORMAT_NCHW, batch_size, output_rows, output_cols, output_depth); - OP_REQUIRES_OK(ctx, - ctx->allocate_temp(DataTypeToEnum::value, nchw_shape, - &maybe_transformed_output)); - output = &maybe_transformed_output; - } - } - - constexpr auto data_layout = is_int8x4 ? dnn::DataLayout::kBatchDepthYX4 - : dnn::DataLayout::kBatchDepthYX; - constexpr auto filter_layout = is_int8x4 ? dnn::FilterLayout::kOutputInputYX4 - : dnn::FilterLayout::kOutputInputYX; - constexpr auto compute_data_format = - is_int8x4 ? FORMAT_NCHW_VECT_C : FORMAT_NCHW; - - dnn::BatchDescriptor conv_input_desc; - conv_input_desc.set_count(batch_size) - .set_feature_map_count(conv_input_depth) - .set_height(conv_input_rows) - .set_width(conv_input_cols) - .set_layout(data_layout); - dnn::FilterDescriptor filter_desc; - filter_desc.set_input_filter_height(filter_rows) - .set_input_filter_width(filter_cols) - .set_input_feature_map_count(conv_input_depth) - .set_output_feature_map_count(output_depth) - .set_layout(filter_layout); - dnn::BatchDescriptor side_input_desc; - side_input_desc.set_count(batch_size) - .set_height(output_rows) - .set_width(output_cols) - .set_feature_map_count(output_depth) - .set_layout(data_layout); - dnn::BatchDescriptor bias_desc; - bias_desc.set_count(1) - .set_height(1) - .set_width(1) - .set_feature_map_count(output_depth) - .set_layout(dnn::DataLayout::kBatchDepthYX); - dnn::BatchDescriptor output_desc; - output_desc.set_count(batch_size) - .set_height(output_rows) - .set_width(output_cols) - .set_feature_map_count(output_depth) - .set_layout(data_layout); - dnn::ConvolutionDescriptor conv_desc; - CHECK_EQ(0, padding_rows % 2); - CHECK_EQ(0, padding_cols % 2); - conv_desc.set_vertical_filter_stride(row_stride) - .set_horizontal_filter_stride(col_stride) - .set_zero_padding_height(padding_rows / 2) - .set_zero_padding_width(padding_cols / 2); - - Tensor maybe_transformed_filter; - const Tensor* filter = &filter_param; - // For qint8, we have already checked filter is OIHW_VECT_I in the - // constructor, but we need to test for is_int8x4 so the if block doesn't - // generate code for qint8. - if (!is_int8x4 && filter_format == FORMAT_HWIO) { - // Shuffle filter tensor from HWIO to OIHW: - OP_REQUIRES_OK(ctx, ctx->allocate_temp( - DataTypeToEnum::value, - ShapeFromFilterFormat( - FORMAT_OIHW, filter_param.shape(), FORMAT_HWIO), - &maybe_transformed_filter)); - functor::TransformFilter()( - ctx->eigen_device(), FORMAT_OIHW, - To32Bit(filter_param.tensor()), - To32Bit(maybe_transformed_filter.tensor())); - filter = &maybe_transformed_filter; - } - - auto conv_input_ptr = - AsDeviceMemory(reinterpret_cast::type*>( - conv_input->template flat().data()), - conv_input->template flat().size()); - auto filter_ptr = - AsDeviceMemory(reinterpret_cast::type*>( - filter->template flat().data()), - filter->template flat().size()); - auto side_input_ptr = - AsDeviceMemory(reinterpret_cast::type*>( - side_input->template flat().data()), - side_input->template flat().size()); - auto output_ptr = - AsDeviceMemory(reinterpret_cast::type*>( - output->template flat().data()), - output->template flat().size()); - auto bias_ptr = AsDeviceMemory(bias.template flat().data(), - bias.template flat().size()); - - static int64 ConvolveScratchSize = GetDnnWorkspaceLimit( - // default value is in bytes despite the name of the environment variable - "TF_CUDNN_WORKSPACE_LIMIT_IN_MB", 1LL << 32 // 4GB - ); - - int device_id = stream->parent()->device_ordinal(); - FusedConvParameters fused_conv_parameters = { - batch_size, - conv_input_depth, - {{conv_input_rows, conv_input_cols}}, - compute_data_format, - output_depth, - {{filter_rows, filter_cols}}, - // TODO(yangzihao): Add support for arbitrary dilations for fused conv. - {{1, 1}}, // dilation_rows, dilation_cols - {{row_stride, col_stride}}, - {{padding_rows, padding_cols}}, - conv_input->dtype(), - device_id, - (side_input_scale != 0), - activation_mode, - }; - - dnn::ActivationMode dnn_activation_mode; - switch (activation_mode) { - case ActivationMode::NONE: - dnn_activation_mode = dnn::ActivationMode::kNone; - break; - case ActivationMode::RELU: - dnn_activation_mode = dnn::ActivationMode::kRelu; - break; - default: - LOG(FATAL) << "Activation mode " << activation_mode << " not supported"; - } - - dnn::AlgorithmConfig algorithm_config; - if (cudnn_use_autotune && !AutoTuneConvBiasActivation::GetInstance()->Find( - fused_conv_parameters, &algorithm_config)) { - std::vector algorithms; - CHECK(stream->parent()->GetConvolveAlgorithms( - fused_conv_parameters.ShouldIncludeWinogradNonfusedAlgo( - stream->parent()), - &algorithms)); - if (activation_mode == ActivationMode::NONE) { - // Only CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM is supported for - // identity activation, other algs seem to quietly do Relu. - // See - // https://docs.nvidia.com/deeplearning/sdk/cudnn-developer-guide/index.html#cudnnConvolutionBiasActivationForward - algorithms.erase( - std::remove_if( - algorithms.begin(), algorithms.end(), - [](dnn::AlgorithmDesc alg) { - return alg.algo_id() != - CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM; - }), - algorithms.end()); - } - std::vector results; - for (auto profile_algorithm : algorithms) { - // TODO(zhengxq): profile each algorithm multiple times to better - // accuracy. - DnnScratchAllocator scratch_allocator(ConvolveScratchSize, ctx); - dnn::ProfileResult profile_result; - bool cudnn_launch_status = - stream - ->ThenFusedConvolveWithAlgorithm( - conv_input_desc, conv_input_ptr, - *conv_input_scale.flat().data(), filter_desc, - filter_ptr, conv_desc, side_input_ptr, side_input_scale, - bias_desc, bias_ptr, dnn_activation_mode, output_desc, - &output_ptr, &scratch_allocator, - dnn::AlgorithmConfig(profile_algorithm), &profile_result) - .ok(); - if (cudnn_launch_status && profile_result.is_valid()) { - results.emplace_back(); - auto& result = results.back(); - result.mutable_conv()->set_algorithm(profile_algorithm.algo_id()); - result.mutable_conv()->set_tensor_ops_enabled( - profile_algorithm.tensor_ops_enabled()); - result.set_scratch_bytes(scratch_allocator.TotalByteSize()); - *result.mutable_run_time() = internal::ToDurationProto( - absl::Milliseconds(profile_result.elapsed_time_in_ms())); - } - } - internal::LogFusedConvForwardAutotuneResults( - se::dnn::ToDataType::type>::value, conv_input_ptr, - filter_ptr, output_ptr, bias_ptr, side_input_ptr, conv_input_desc, - filter_desc, output_desc, conv_desc, - *conv_input_scale.flat().data(), side_input_scale, - dnn_activation_mode, stream->parent(), results); - OP_REQUIRES_OK( - ctx, internal::BestCudnnConvAlgorithm(results, &algorithm_config)); - AutoTuneConvBiasActivation::GetInstance()->Insert(fused_conv_parameters, - algorithm_config); - } - - DnnScratchAllocator scratch_allocator(ConvolveScratchSize, ctx); - bool cudnn_launch_status = - stream - ->ThenFusedConvolveWithAlgorithm( - conv_input_desc, conv_input_ptr, - *conv_input_scale.flat().data(), filter_desc, - filter_ptr, conv_desc, side_input_ptr, side_input_scale, - bias_desc, bias_ptr, dnn_activation_mode, output_desc, - &output_ptr, &scratch_allocator, algorithm_config, - /*output_profile_result=*/nullptr) - .ok(); - - if (!cudnn_launch_status) { - ctx->SetStatus(errors::Internal("cuDNN launch failure : conv_input shape(", - conv_input->shape().DebugString(), - ") filter shape(", - filter->shape().DebugString(), ")")); - } - - // Convert the output tensor back from NCHW to NHWC if necessary. - if (!is_int8x4 && (data_format == FORMAT_NHWC) && (output_depth > 1)) { - functor::NCHWToNHWC()( - ctx->eigen_device(), - const_cast(output)->tensor(), - output_param->tensor()); - } -} - -// Forward declarations of the functor specializations for GPU used above. -namespace functor { -#define DECLARE_GPU_SPEC(T) \ - template <> \ - void PadInput::operator()( \ - const GPUDevice& d, typename TTypes::ConstTensor in, \ - const std::array& padding_left, \ - const std::array& padding_right, \ - typename TTypes::Tensor out, TensorFormat data_format); \ - extern template struct PadInput; - -DECLARE_GPU_SPEC(float); -DECLARE_GPU_SPEC(int32); -#undef DECLARE_GPU_SPEC -} // namespace functor - -// Registration of the GPU implementations. - -REGISTER_KERNEL_BUILDER( - Name("FusedConv2DBiasActivation") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tbias") - .HostMemory("conv_input_scale") - .HostMemory("side_input_scale"), - FusedConv2DBiasActivationOp); - -REGISTER_KERNEL_BUILDER( - Name("FusedConv2DBiasActivation") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tbias") - .HostMemory("conv_input_scale") - .HostMemory("side_input_scale"), - FusedConv2DBiasActivationOp); - -#endif // GOOGLE_CUDA - -} // namespace tensorflow diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.h b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.h deleted file mode 100644 index 2a47bc67bdd..00000000000 --- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.h +++ /dev/null @@ -1,70 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_FUSED_CONV_KERNELS_FUSED_CONV2D_BIAS_ACTIVATION_OP_H_ -#define TENSORFLOW_CONTRIB_FUSED_CONV_KERNELS_FUSED_CONV2D_BIAS_ACTIVATION_OP_H_ - -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/platform/mem.h" -#include "tensorflow/core/util/activation_mode.h" -#include "tensorflow/core/util/tensor_format.h" - -// FixedPoint header must be included after Tensor. -// clang-format off -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint" -// clang-format on - -#if GOOGLE_CUDA -#include "tensorflow/contrib/fused_conv/kernels/fused_conv_ops_gpu.h" -#include "tensorflow/core/platform/stream_executor.h" -#endif // GOOGLE_CUDA - -namespace tensorflow { - -// Forward declaration. -class OpKernelContext; - -template -class LaunchFusedConv2DBiasActivationOp { - public: - void launch(OpKernelContext* ctx, bool cudnn_use_autotune, - const Tensor& conv_input, const Tensor& conv_input_scale, - const Tensor& filter, int32 row_stride, int32 col_stride, - const Eigen::PaddingType& padding, const Tensor& side_input, - ScaleType side_input_scale, const Tensor& bias, - ActivationMode activation_mode, TensorFormat data_format, - FilterTensorFormat filter_format, Tensor* output); -}; - -#ifdef GOOGLE_CUDA -template -class LaunchFusedConv2DBiasActivationOp { - public: - void launch(OpKernelContext* ctx, bool cudnn_use_autotune, - const Tensor& conv_input, const Tensor& conv_input_scale, - const Tensor& filter, int32 row_stride, int32 col_stride, - const Eigen::PaddingType& padding, const Tensor& side_input, - ScaleType side_input_scale, const Tensor& bias, - ActivationMode activation_mode, TensorFormat data_format, - FilterTensorFormat filter_format, Tensor* output); -}; -#endif // GOOGLE_CUDA - -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_FUSED_CONV_KERNELS_FUSED_CONV2D_BIAS_ACTIVATION_OP_H_ diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv_ops_gpu.h b/tensorflow/contrib/fused_conv/kernels/fused_conv_ops_gpu.h deleted file mode 100644 index b9c131a2e91..00000000000 --- a/tensorflow/contrib/fused_conv/kernels/fused_conv_ops_gpu.h +++ /dev/null @@ -1,75 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_FUSED_CONV_KERNELS_FUSED_CONV_OPS_GPU_H_ -#define TENSORFLOW_CONTRIB_FUSED_CONV_KERNELS_FUSED_CONV_OPS_GPU_H_ - -#if GOOGLE_CUDA - -#include "tensorflow/core/kernels/conv_ops_gpu.h" -#include "tensorflow/core/util/activation_mode.h" - -// TODO(pauldonnelly): Merge this file into core/kernels/conv_ops_gpu.h. - -namespace tensorflow { - -// Add additional parameters specific to fused convolutions. -class FusedConvParameters : public ConvParameters { - public: - FusedConvParameters(int64 batch, int64 in_depths, const SpatialArray& in, - TensorFormat data_format, int64 out_depths, - const SpatialArray& filter, const SpatialArray& dilation, - const SpatialArray& stride, const SpatialArray& padding, - DataType dtype, int device_id, bool has_side_input, - ActivationMode activation_mode) - : ConvParameters(batch, in_depths, in, data_format, out_depths, filter, - dilation, stride, padding, dtype, device_id), - activation_mode_(activation_mode), - has_side_input_(has_side_input) { - hash_code_ = Hash64Combine(hash_code_, has_side_input); - hash_code_ = Hash64Combine(hash_code_, activation_mode); - } - - bool operator==(const FusedConvParameters& other) const { - return this->get_data_as_tuple() == other.get_data_as_tuple(); - } - - bool operator!=(const FusedConvParameters& other) const { - return !(*this == other); - } - - string ToString() const { - return strings::StrCat(ConvParameters::ToString(), ", ", has_side_input_, - ", ", activation_mode_, ", "); - } - - private: - using ParameterDataType = - std::tuple; - - ParameterDataType get_data_as_tuple() const { - return std::make_tuple(ConvParameters::get_data_as_tuple(), has_side_input_, - activation_mode_); - } - - ActivationMode activation_mode_; - bool has_side_input_; -}; - -} // namespace tensorflow - -#endif // GOOGLE_CUDA - -#endif // TENSORFLOW_CONTRIB_FUSED_CONV_KERNELS_FUSED_CONV_OPS_GPU_H_ diff --git a/tensorflow/contrib/fused_conv/ops/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/ops/fused_conv2d_bias_activation_op.cc deleted file mode 100644 index d4a2aa4b5f6..00000000000 --- a/tensorflow/contrib/fused_conv/ops/fused_conv2d_bias_activation_op.cc +++ /dev/null @@ -1,158 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/activation_mode.h" -#include "tensorflow/core/util/padding.h" -#include "tensorflow/core/util/tensor_format.h" - -namespace tensorflow { - -// -------------------------------------------------------------------------- - -// TODO(pauldonnelly): Add support for double inputs and scales to this Op, -// (currently Attr does not support double). - -REGISTER_OP("FusedConv2DBiasActivation") - .Input("conv_input: T") - .Input("filter: T") - .Input("bias: Tbias") - .Input("side_input: T") - .Input("conv_input_scale: float") - .Input("side_input_scale: float") - .Output("output: T") - .Attr("T: {float, half, qint8}") - .Attr("Tbias: {float, half}") - .Attr("strides: list(int)") - .Attr(GetPaddingAttrString()) - .Attr("data_format: {'NHWC', 'NCHW', 'NCHW_VECT_C'} = 'NHWC'") - .Attr("filter_format: {'HWIO', 'OIHW', 'OIHW_VECT_I'} = 'HWIO'") - .Attr("activation_mode: {'Relu', 'None'} = 'Relu'") - .Attr("dilations: list(int) = [1, 1, 1, 1]") - .SetShapeFn([](shape_inference::InferenceContext* c) { - using shape_inference::ShapeHandle; - using shape_inference::DimensionHandle; - TF_RETURN_IF_ERROR(shape_inference::Conv2DShape(c)); - - string data_format_str, filter_format_str; - TF_RETURN_IF_ERROR(c->GetAttr("data_format", &data_format_str)); - TF_RETURN_IF_ERROR(c->GetAttr("filter_format", &filter_format_str)); - - TensorFormat data_format; - FormatFromString(data_format_str, &data_format); - FilterTensorFormat filter_format; - FilterFormatFromString(filter_format_str, &filter_format); - - constexpr int num_spatial_dims = 2; - const int rank = - GetTensorDimsFromSpatialDims(num_spatial_dims, data_format); - ShapeHandle filter_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), rank, &filter_shape)); - - DimensionHandle output_depth_dim = - c->Dim(filter_shape, - GetFilterDimIndex(filter_format, 'O')); - int64 output_depth_dim_val = c->Value(output_depth_dim); - - ShapeHandle bias_shape; - // Bias should be a 1-D tensor. - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &bias_shape)); - DimensionHandle bias_dim = c->Dim(bias_shape, 0); - int64 bias_dim_val = c->Value(bias_dim); - - if (output_depth_dim_val != bias_dim_val) { - return errors::InvalidArgument( - "Output depth dimension (", output_depth_dim_val, - ") and bias dimension (", bias_dim_val, ") do not match."); - } - - // Check side input shape matches the output shape. - ShapeHandle side_input_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(3), 1, &side_input_shape)); - if (c->Rank(side_input_shape) > 1) { - ShapeHandle unused; - TF_RETURN_IF_ERROR(c->Merge(side_input_shape, c->output(0), &unused)); - } - - ShapeHandle unused; - // Check that conv_input_scale is a scalar or vector. - TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(4), 1, &unused)); - // Check that side_input_scale is a scalar tensor. - TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &unused)); - - return Status::OK(); - }) - .Doc(R"doc( - Computes a fused kernel which implements: 2-D convolution, adds side input, - with separate scaling on convolution and side inputs, then adds bias and - applies the RELU activation function to the result. Supports both float and - qint8 data formats. In the case of qint8, the output is clipped to [0..127]. - - conv_input: A tensor with format as specified by `data_format` (see below). - filter: A tensor with format depending on `data_format` as follows: - "NHWC", "NCHW": - `float [ filter_height, filter_width, in_channels, out_channels ]` - "NCHW_VECT_C": - `qint8 [ out_channels, in_channels, filter_height, filter_width ]` - bias: 1-D float tensor with size matching the `out_channels` dimension of - `filter`. - Note: this tensor is still float, even if other inputs are qint8. - side_input: A tensor with format as specified by `data_format` (see below). - This tensor will be ignored and can be [] if side_input_scale == 0. - Otherwise, the size of each dimension must match the `output` tensor. - conv_input_scale: scalar float value to be multiplied by `conv_input`. - (conceptually.. in reality it is applied after convolution). - For the CPU version, this can also be a 1-D Tensor of per output-channel - scales. - side_input_scale: scalar float value to be multiplied by `side_input`. - output: A tensor with format as specified by `data_format` (see below). - The dimension sizes are determined automatically based on other inputs - and attributes. - T: The element data type of `conv_input`, `side_input` and `output` tensors. - Note: must match with the `data_format`. - Tbias: The element data type of `bias`. - strides: 1-D tensor of length 4. The stride of the sliding window for each - dimension of `input`. The dimension order is determined by the value of - `data_format`, see below for details. - Note: the stride for batch and channel dimensions must be 1. - padding: The type of padding algorithm to use. - data_format: A string specifying the data format of `conv_input`, - `side_input` and `output` tensors with the following options: - "NHWC": `float [ batch, height, width, channels ]` - "NCHW": `float [ batch, channels, height, width ]` - "NCHW_VECT_C": - `qint8 [ batch, channels / 4, height, width, channels % 4 ]` - Note: for "NCHW_VECT_C", `channels` must be a multiple of 4. - filter_format: A string specifying the data format of `filter`, - "HWIO": `float [ kernel_height, kernel_width, input_channels, - output_channels ]` - "OIHW_VECT_I": - `qint8 [ output_channels, input_channels / 4, - kernel_height, kernel_width, input_channels % 4 ]` - activation_mode: The activation applied to the output. - Must be "Relu" or "None". - dilations: 1-D tensor of length 4. The dilation factor for each dimension - of `input`. If set to k > 1, there will be k-1 skipped cells between - each filter element on that dimension. The dimension order is determined - by the value of `data_format`, see above for details. Dilations in the - batch and depth dimensions must be 1. -)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/fused_conv/python/__init__.py b/tensorflow/contrib/fused_conv/python/__init__.py deleted file mode 100644 index 23d817cefbd..00000000000 --- a/tensorflow/contrib/fused_conv/python/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ops module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_benchmark.py b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_benchmark.py deleted file mode 100644 index 96cdd8b1ca4..00000000000 --- a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_benchmark.py +++ /dev/null @@ -1,243 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Benchmark for fused conv2d bias and activation op.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import time - -from tensorflow.contrib.fused_conv.python.ops import fused_conv2d_bias_activation_op -from tensorflow.python.client import session as session_lib -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def build_conv_bias_relu_graph(device, input_shape, filter_shape, strides, - padding, num_iters, data_format): - """builds a graph containing a sequence of conv2d operations. - - Args: - device: String, the device to run on. - input_shape: Shape of the input tensor. - filter_shape: Shape of the filter tensor. - strides: A list of ints. 1-D of length 4. The stride of sliding - window for each dimension of input. - padding: A string from: "SAME", "VALID". The type of padding - algorithm to use. - num_iters: number of iterations to run conv2d. - data_format: data format string of input, 'NHWC' and 'NCHW' are - supported. - - Returns: - An array of tensors to run() - """ - if data_format == "NCHW": - input_shape = [ - input_shape[0], input_shape[3], input_shape[1], input_shape[2] - ] - with ops.device("/%s:0" % device): - inp = variables.Variable(random_ops.truncated_normal(input_shape)) - filt = variables.Variable(random_ops.truncated_normal(filter_shape)) - bias_shape = [filter_shape[-1]] - bias = variables.Variable(random_ops.truncated_normal(bias_shape)) - - outputs = [] - conv2d_out = nn_ops.conv2d( - inp, filt, strides, padding, data_format=data_format) - bias_out = nn_ops.bias_add(conv2d_out, bias, data_format=data_format) - relu_out = nn_ops.relu(bias_out) - outputs.append(relu_out) - for _ in range(1, num_iters): - with ops.control_dependencies([relu_out]): - conv2d_out = nn_ops.conv2d( - inp, filt, strides, padding, data_format=data_format) - bias_out = nn_ops.bias_add(conv2d_out, bias, data_format=data_format) - relu_out = nn_ops.relu(bias_out) - outputs.append(relu_out) - return control_flow_ops.group(*outputs) - - -def build_fused_conv_bias_relu_graph(device, input_shape, filter_shape, strides, - padding, num_iters, data_format): - """builds a graph containing a sequence of conv2d operations. - - Args: - device: String, the device to run on. - input_shape: Shape of the input tensor. - filter_shape: Shape of the filter tensor. - strides: A list of ints. 1-D of length 4. The stride of sliding - window for each dimension of input. - padding: A string from: "SAME", "VALID". The type of padding - algorithm to use. - num_iters: number of iterations to run conv2d. - data_format: data format string of input, 'NHWC' and 'NCHW' are - supported. - - Returns: - An array of tensors to run() - """ - if data_format == "NCHW": - input_shape = [ - input_shape[0], input_shape[3], input_shape[1], input_shape[2] - ] - with ops.device("/%s:0" % device): - inp = variables.Variable(random_ops.truncated_normal(input_shape)) - filt = variables.Variable(random_ops.truncated_normal(filter_shape)) - bias_shape = [filter_shape[-1]] - bias = variables.Variable(random_ops.truncated_normal(bias_shape)) - - outputs = [] - fused_out = fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - inp, - filt, - bias, - strides, - padding, - data_format=data_format, - activation_mode="Relu") - outputs.append(fused_out) - for _ in range(1, num_iters): - with ops.control_dependencies([fused_out]): - # pylint: disable=g-line-too-long - fused_out = fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( # pylint: disable=line-too-long - inp, - filt, - bias, - strides, - padding, - data_format=data_format, - activation_mode="Relu") - outputs.append(fused_out) - return control_flow_ops.group(*outputs) - - -class FusedConv2DBiasActivationBenchmark(test.Benchmark): - """Benchmark conv2d!""" - - def _run_graph(self, device, input_shape, filter_shape, strides, padding, - num_iters, data_format): - """runs the graph and print its execution time. - - Args: - device: String, the device to run on. - input_shape: Shape of the input tensor. - filter_shape: Shape of the filter tensor. - strides: A list of ints. 1-D of length 4. The stride of sliding - window for each dimension of input. - padding: A string from: "SAME", "VALID". The type of padding - algorithm to use. num_iters: Number of iterations to run the - benchmark. - num_iters: number of iterations to run conv2d. - data_format: data format string of input, 'NHWC' and 'NCHW' are - supported. - - Returns: - The duration of the run in seconds. - """ - graph = ops.Graph() - with graph.as_default(): - outputs = build_fused_conv_bias_relu_graph(device, input_shape, - filter_shape, strides, padding, - num_iters, data_format) - with session_lib.Session(graph=graph) as session: - variables.global_variables_initializer().run() - # warmup runs - session.run(outputs) - - start_time = time.time() - session.run(outputs) - duration = (time.time() - start_time) / num_iters - - print("%s inputshape:%s filtershape:%s strides:%s padding:%s " - "%d iters: %.8f sec" % (device, str(input_shape).replace(" ", ""), - str(filter_shape).replace(" ", ""), - str(strides).replace(" ", ""), padding, - num_iters, duration)) - name_template = ( - "conv2d_{device}_input_shape_{inputshape}_filter_shape_{filtershape}_" - "strides_{strides}_padding_{padding}") - - self.report_benchmark( - name=name_template.format( - device=device, - inputshape=str(input_shape).replace(" ", ""), - filtershape=str(filter_shape).replace(" ", ""), - strides=str(strides).replace(" ", ""), - padding=padding).replace(" ", ""), - iters=num_iters, - wall_time=duration) - - return duration - - def benchmark_fused_conv2d_bias_activation(self): - - stride = [1, 1, 1, 1] - paddings = ["VALID", "SAME"] - data_formats = ["NHWC", "NCHW"] - - resnet50_input_shapes = [[64, 14, 14, 256], [64, 14, 14, 256], [ - 64, 14, 14, 1024 - ], [64, 55, 55, 64], [64, 28, 28, 128], [64, 28, 28, 128], [64, 55, 55, 64], - [64, 7, 7, 512], [64, 7, 7, 512], - [64, 28, 28, 512], [64, 55, 55, - 256], [64, 7, 7, 2048]] - - resnet50_filter_shapes = [[1, 1, 256, 1024], [3, 3, 256, 256], [ - 1, 1, 1024, 256 - ], [1, 1, 64, 256], [1, 1, 128, 512], [3, 3, 128, 128], [3, 3, 64, 64], [ - 3, 3, 512, 512 - ], [1, 1, 512, 2048], [1, 1, 512, 128], [1, 1, 256, 64], [1, 1, 2048, 512]] - - inception3_input_shapes = [[64, 17, 17, 768], [64, 35, 35, 96], [ - 64, 35, 35, 288 - ], [64, 8, 8, 384], [64, 8, 8, 384], [64, 17, 17, 192], [64, 35, 35, 64], [ - 64, 17, 17, 192 - ], [64, 17, 17, 160], [64, 17, 17, 160], [64, 17, 17, 768], [ - 64, 35, 35, 256 - ], [64, 35, 35, 48], [64, 35, 35, 192], [64, 17, 17, 128], [ - 64, 17, 17, 160 - ], [64, 8, 8, 448], [64, 17, 17, 128], [64, 17, 17, 768], [64, 17, 17, 160]] - inception3_filter_shapes = [[1, 1, 768, 192], [3, 3, 96, 96], [ - 1, 1, 288, 64 - ], [1, 3, 384, 384], [3, 1, 384, 384], [7, 1, 192, 192], [3, 3, 64, 96], [ - 1, 7, 192, 192 - ], [7, 1, 160, 160], [1, 7, 160, 160], [1, 1, 768, 160], [1, 1, 256, 64], [ - 5, 5, 48, 64 - ], [1, 1, 192, 64], [1, 7, 128, 128], [1, 7, 160, 192], [3, 3, 448, 384], - [7, 1, 128, 128], [1, 1, 768, - 128], [7, 1, 160, 192]] - - print("fused conv2d bias activation benchmark using resnet50's shapes:") - for ishape, fshape in zip(resnet50_input_shapes, resnet50_filter_shapes): - for padding in paddings: - for data_format in data_formats: - self._run_graph("gpu", ishape, fshape, stride, padding, 80, - data_format) - print("fused conv2d bias activation benchmark using inception3's shapes:") - for ishape, fshape in zip(inception3_input_shapes, - inception3_filter_shapes): - for padding in paddings: - for data_format in data_formats: - self._run_graph("gpu", ishape, fshape, stride, padding, 80, - data_format) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op.py b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op.py deleted file mode 100644 index ee29f5f6d06..00000000000 --- a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""Tensorflow op performing fused conv2d bias_add and relu.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.fused_conv.ops import gen_fused_conv2d_bias_activation_op -from tensorflow.contrib.util import loader -from tensorflow.python.platform import resource_loader - -_fused_conv2d_bias_activation_op_so = loader.load_op_library( - resource_loader.get_path_to_datafile("_fused_conv2d_bias_activation_op.so")) - - -# pylint: disable=redefined-builtin -def fused_conv2d_bias_activation(conv_input, - filter, - bias, - strides=None, - padding=None, - conv_input_scale=1.0, - side_input_scale=0.0, - side_input=None, - activation_mode="Relu", - data_format=None, - filter_format=None, - name=None): - """Fused 2D conv, bias and activation with optional side input. - - Computes a fused 2-D convolution scaled by conv_input_scale, - adds an optional side input scaled by side_input_scale, adds biases, - and applies ReLU. As an equation: - output = ReLU(conv_input_scale * Conv(conv_input, filter) + - side_input_scale * side_input + bias) - Note: In int8 mode, The ReLU will clip the output to the range [0..127]. - - Args: - conv_input: A `Tensor` of the format specified by `data_format`. - filter: A `Tensor` whose format depends on `data_format`: if `data_format` - is "NCHW_VECT_C", filter should be "OIHW_VECT_I" otherwise, it should be - "HWIO" format. - bias: A 1-D `Tensor` of type `float32`, and dimensions equal to the number - of output channels. - strides: A list of 4 `ints` specifying convolution strides. if `data_format` - is "NCHW" or "NCHW_VECT_C", the order should be NCHW. if `data_format` is - "NHWC", the order should be NHWC. - padding: A `string` from: `"SAME", "VALID"`. - conv_input_scale: Either a scalar `float32` or a Tensor of the same shape as - the `bias` Tensor, that will be multiplied by `conv_input`. This is - optional and defaults to 1. However it should be set to specify the - quantization scale when `data_format` is "NCHW_VECT_C". - side_input_scale: A scalar `float32` that will be multiplied by side_input. - This is optional and defaults to 0. - side_input: A `Tensor` of the format specified by `data_format`. This is - useful for implementing ResNet blocks. - activation_mode: (optional) currently supports the default "Relu", or "None" - activation function. - Note: in qint8 mode, "None" actually clips to the range [-128, 127], - while "Relu" clips to the range [0, 127]. - data_format: Specifies the data format. - Possible values are: "NHWC" float [batch, height, width, channels] - "NCHW" float [batch, channels, height, width] "NCHW_VECT_C" qint8 - [batch, channels / 4, height, width, channels % 4] Defaults to - `"NHWC"`. Performance is worst for `"NHWC"` and best for - `"NCHW_VECT_C"`. - filter_format: Specifies the filter format. - Possible values are: "HWIO" float [kernel_height, kernel_width, - input_channels, output_channels ] "OIHW" float [output_channels, - input_channels, kernel_height, kernel_width ] "OIHW_VECT_I" qint8 [ - output_channels, input_channels / 4, kernel_height, kernel_width, - input_channels % 4 ] Defaults to `"HWIO"`. - name: A name for the operation (optional). - - Returns: - A `Tensor` of the format specified by `data_format`. - """ - if strides is None: - strides = [1, 1, 1, 1] - if side_input is None: - side_input = [] - return gen_fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - conv_input, - filter, - bias, - side_input, - conv_input_scale, - side_input_scale, - padding=padding, - strides=strides, - activation_mode=activation_mode, - data_format=data_format, - filter_format=filter_format, - name=name) diff --git a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py deleted file mode 100644 index 6cc5d697efe..00000000000 --- a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for fused convolutions.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.fused_conv.python.ops import fused_conv2d_bias_activation_op_test_base as test_base -from tensorflow.python.platform import test - - -# Instantiate three test suites from test_base, mixing in test.TestCase as -# the test framework. -class FusedConv2DBiasActivationTest(test_base.FusedConv2DBiasActivationTest, - test.TestCase): - pass - - -class FusedConvInt8CPUTests(test_base.FusedConvInt8CPUTests, test.TestCase): - pass - - -class FusedConvInt8CorrespondenceTests( - test_base.FusedConvInt8CorrespondenceTests, test.TestCase): - pass - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test_base.py b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test_base.py deleted file mode 100644 index 205144fca56..00000000000 --- a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test_base.py +++ /dev/null @@ -1,1174 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Provides test suites that can be run to test fused convolutions. - -Each of the two test suites in this module, FusedConv2DBiasActivationTest and -FusedConvInt8Tests, should be "instantiated" by declaring a class which inherits -from the FusedConv test and a class that provides the standard test.TestCase -API. - -See e.g. fused_conv2d_bias_activation_op_test.py in this folder. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import contextlib -import numpy as np - -from tensorflow.contrib.fused_conv.python.ops import fused_conv2d_bias_activation_op -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging - - -def _GetShrunkInceptionShapes(shrink=10): - """Iterator for smaller versions of convolution shapes in 2015 Inception. - - Relative to inception, each depth value is `depth // shrink`. - - Args: - shrink: Factor to shrink each depth value by relative to Inception. - - Yields: - Tuple (input_size, filter_size, out_size, stride, padding), the convolution - parameters of Inception layers. - """ - input_sizes = [[4, 5, 5, 1248], [4, 8, 8, 384], [4, 8, 8, 384], - [4, 8, 8, 2048], [4, 8, 8, 448], [4, 8, 8, 2048], - [4, 8, 8, 2048], [4, 8, 8, 2048], [4, 8, 8, 1760], - [4, 8, 8, 1760], [4, 8, 8, 1760], [4, 8, 8, 1760], - [4, 17, 17, 192], [4, 17, 17, 192], [4, 17, 17, 1248], - [4, 17, 17, 128], [4, 17, 17, 1248], [4, 17, 17, 224], - [4, 17, 17, 192], [4, 17, 17, 192], [4, 17, 17, 1216], - [4, 17, 17, 1216], [4, 17, 17, 224], [4, 17, 17, 192], - [4, 17, 17, 192], [4, 17, 17, 1152], [4, 17, 17, 1152], - [4, 17, 17, 192], [4, 17, 17, 160], [4, 17, 17, 1152], - [4, 17, 17, 1024], [4, 17, 17, 128], [4, 17, 17, 1024], - [4, 17, 17, 128], [4, 17, 17, 1024], [4, 17, 17, 128], - [4, 17, 17, 768], [4, 17, 17, 128], [4, 17, 17, 128], - [4, 17, 17, 768], [4, 17, 17, 768], [4, 35, 35, 96], - [4, 35, 35, 288], [4, 35, 35, 64], [4, 35, 35, 288], - [4, 35, 35, 256], [4, 35, 35, 48], [4, 35, 35, 256], - [4, 35, 35, 96], [4, 35, 35, 192], [4, 35, 35, 192], - [4, 35, 35, 192], [4, 73, 73, 64], [4, 73, 73, 64], - [4, 147, 147, 24]] - filter_sizes = [[1, 1, 1248, 128], [1, 3, 384, 384], [3, 1, 384, 384], - [1, 1, 2048, 192], [3, 3, 448, 384], [1, 1, 2048, 320], - [1, 1, 2048, 448], [1, 1, 2048, 384], [1, 1, 1760, 384], - [1, 1, 1760, 192], [1, 1, 1760, 448], [1, 1, 1760, 320], - [3, 3, 192, 192], [3, 3, 192, 192], [1, 1, 1248, 192], - [3, 3, 128, 320], [1, 1, 1248, 128], [1, 3, 224, 224], - [3, 1, 192, 256], [1, 3, 192, 256], [1, 1, 1216, 192], - [1, 1, 1216, 96], [3, 1, 224, 224], [3, 3, 192, 224], - [1, 3, 192, 192], [1, 1, 1152, 192], [1, 1, 1152, 128], - [3, 1, 192, 192], [3, 3, 160, 192], [1, 1, 1152, 160], - [1, 1, 1024, 128], [1, 3, 128, 192], [1, 1, 1024, 160], - [3, 1, 128, 192], [1, 1, 1024, 256], [3, 1, 128, 128], - [1, 1, 768, 192], [1, 3, 128, 128], [3, 3, 128, 128], - [1, 1, 768, 128], [1, 1, 768, 320], [3, 3, 96, 96], - [3, 3, 288, 384], [3, 3, 64, 96], [1, 1, 288, 64], - [1, 1, 256, 64], [5, 5, 48, 64], [1, 1, 256, 48], - [3, 3, 96, 96], [1, 1, 192, 32], [1, 1, 192, 64], - [1, 1, 192, 48], [3, 3, 64, 192], [1, 1, 64, 64], - [1, 1, 24, 64]] - out_sizes = [[4, 5, 5, 128], [4, 8, 8, 384], [4, 8, 8, 384], [4, 8, 8, 192], - [4, 8, 8, 384], [4, 8, 8, 320], [4, 8, 8, 448], [4, 8, 8, 384], - [4, 8, 8, 384], [4, 8, 8, 192], [4, 8, 8, 448], [4, 8, 8, 320], - [4, 8, 8, 192], [4, 17, 17, 192], [4, 17, 17, 192], - [4, 8, 8, 320], [4, 17, 17, 128], [4, 17, 17, 224], - [4, 17, 17, 256], [4, 17, 17, 256], [4, 17, 17, 192], - [4, 17, 17, 96], [4, 17, 17, 224], [4, 17, 17, 224], - [4, 17, 17, 192], [4, 17, 17, 192], [4, 17, 17, 128], - [4, 17, 17, 192], [4, 17, 17, 192], [4, 17, 17, 160], - [4, 17, 17, 128], [4, 17, 17, 192], [4, 17, 17, 160], - [4, 17, 17, 192], [4, 17, 17, 256], [4, 17, 17, 128], - [4, 17, 17, 192], [4, 17, 17, 128], [4, 17, 17, 128], - [4, 17, 17, 128], [4, 17, 17, 320], [4, 17, 17, 96], - [4, 17, 17, 384], [4, 35, 35, 96], [4, 35, 35, 64], - [4, 35, 35, 64], [4, 35, 35, 64], [4, 35, 35, - 48], [4, 35, 35, 96], - [4, 35, 35, 32], [4, 35, 35, 64], [4, 35, 35, 48], - [4, 71, 71, 192], [4, 73, 73, 64], [4, 147, 147, 64]] - strides = [ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1 - ] - # Shrink sizes to make the test faster - for i in input_sizes: - i[3] //= shrink - for f in filter_sizes: - f[2] //= shrink - f[3] //= shrink - for o in out_sizes: - o[3] //= shrink - # pylint: disable=invalid-name - VALID = "VALID" - SAME = "SAME" - # pylint: enable=invalid-name - paddings = [ - SAME, SAME, SAME, SAME, SAME, SAME, SAME, SAME, SAME, SAME, SAME, SAME, - VALID, SAME, SAME, VALID, SAME, SAME, SAME, SAME, SAME, SAME, SAME, SAME, - SAME, SAME, SAME, SAME, SAME, SAME, SAME, SAME, SAME, SAME, SAME, SAME, - SAME, SAME, SAME, SAME, SAME, VALID, VALID, SAME, SAME, SAME, SAME, SAME, - SAME, SAME, SAME, SAME, VALID, VALID, VALID - ] - for i, f, o, s, p in zip(input_sizes, filter_sizes, out_sizes, strides, - paddings): - yield i, f, o, s, p - - -def _GetTestConfigs(): - """Get all the valid tests configs to run. - - Returns: - all the valid test configs as tuples of data_format and use_gpu. - """ - test_configs = [("NCHW", True), ("NHWC", True)] - return test_configs - - -def _IotaNdF32Constant(dim_sizes): - - def MakeList(dims): - if len(dims) == 1: - return [float(1 + f) for f in range(dims[0])] - return [MakeList(dims[1:]) for _ in range(dims[0])] - - return constant_op.constant(MakeList(dim_sizes), dtype=dtypes.float32) - - -def _GetInceptionFwdTest(input_size, - filter_size, - stride, - padding, - gpu_only=True): - - def Test(self): - if gpu_only and not test.is_gpu_available(): - tf_logging.info("Skipping InceptionFwd %s", - (input_size, filter_size, stride, padding)) - return - tf_logging.info("Testing InceptionFwd %s", - (input_size, filter_size, stride, padding)) - self.CompareFwdValues(input_size, filter_size, [stride, stride], padding) - - return Test - - -class FusedConv2DBiasActivationTest(object): - - @contextlib.contextmanager - def test_scope(self): # pylint: disable=invalid-name - """Can be overridden in base classes to provide a test scope.""" - yield - - def _DtypesToTest(self, use_gpu): - return [dtypes.float32] - - def _FilterFormatsToTest(self, use_gpu): - return ["HWIO", "OIHW"] - - def _SetupValuesForDevice(self, tensor_in_sizes, filter_in_sizes, bias, - strides, padding, activation_mode, data_format, - filter_format, dtype): - """Verifies the output values of the convolution function. - - Args: - tensor_in_sizes: Input tensor dimensions in [batch, input_rows, - input_cols, input_depth]. - filter_in_sizes: Filter tensor dimensions in [kernel_rows, kernel_cols, - input_depth, output_depth]. - bias: 1-D bias tensor of length output_depth. - strides: Stride: [col_stride, row_stride] - padding: Padding type. - activation_mode: Activation mode. - data_format: Format of the data tensors. - filter_format: Filter format to use for the fused convolution. - dtype: Data type for inputs and outputs. - - Returns: - Symbolic tensor value and reference value that can be used to - execute the computation and verify the results. - """ - input_size = np.prod(tensor_in_sizes) - filter_size = np.prod(filter_in_sizes) - bias_size = filter_in_sizes[-1] # equals to output depth - # Initializes the input tensor with array containing incrementing - # numbers from 1. - x1 = [f * 1.0 for f in range(1, input_size + 1)] - x2 = [f * 1.0 for f in range(1, filter_size + 1)] - # This is to guarantee that there are always negative values after - # bias add so that we can test whether relu works correctly. - x3 = bias - t1 = constant_op.constant(x1, shape=tensor_in_sizes, dtype=dtype) - t2 = constant_op.constant(x2, shape=filter_in_sizes, dtype=dtype) - fused_t2 = t2 - if filter_format == "OIHW": - fused_t2 = _HwioToOihw(t2) - t3 = constant_op.constant(x3, shape=[bias_size], dtype=dtype) - strides = [1] + strides + [1] - if data_format == "NCHW": - t1 = test_util.NHWCToNCHW(t1) - strides = test_util.NHWCToNCHW(strides) - output = fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - t1, - fused_t2, - t3, - strides=strides, - padding=padding, - data_format=data_format, - filter_format=filter_format, - activation_mode=activation_mode) - ref_conv_output = nn_ops.conv2d( - t1, t2, strides=strides, padding=padding, data_format=data_format) - ref_bias_output = nn_ops.bias_add( - ref_conv_output, t3, data_format=data_format) - ref_output = nn_ops.relu(ref_bias_output) - if data_format == "NCHW": - output = test_util.NCHWToNHWC(output) - ref_output = test_util.NCHWToNHWC(ref_output) - - return output, ref_output - - def CompareFwdValues(self, tensor_in_sizes, filter_in_sizes, conv_strides, - padding): - """Verifies that CPU and GPU produce the same values. - - Args: - tensor_in_sizes: Input tensor dimensions in [batch, input_rows, - input_cols, input_depth]. - filter_in_sizes: Filter tensor dimensions in [kernel_rows, kernel_cols, - input_depth, output_depth]. - conv_strides: [row_stride, col_stride] for the convolution; - padding: Padding type. - """ - x1 = np.random.rand(*tensor_in_sizes).astype(np.float32) - x2 = np.random.rand(*filter_in_sizes).astype(np.float32) - x3 = np.random.rand(*[filter_in_sizes[-1]]).astype(np.float32) - - def _SetupVal(data_format, use_gpu): - t1 = constant_op.constant(x1, shape=tensor_in_sizes) - t2 = constant_op.constant(x2, shape=filter_in_sizes) - t3 = constant_op.constant(x3, shape=[filter_in_sizes[-1]]) - strides = [1] + conv_strides + [1] - if data_format == "NCHW": - t1 = test_util.NHWCToNCHW(t1) - strides = test_util.NHWCToNCHW(strides) - output = fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - t1, - t2, - t3, - strides=strides, - padding=padding, - data_format=data_format, - activation_mode="Relu") - - if data_format == "NCHW": - output = test_util.NCHWToNHWC(output) - return output - - with self.session() as sess, self.test_scope(): - tensors = [] - for (data_format, use_gpu) in _GetTestConfigs(): - tensors.append(_SetupVal(data_format, use_gpu)) - values = sess.run(tensors) - for i in range(1, len(values)): - self.assertAllClose(values[0], values[i], rtol=1e-3, atol=1e-3) - - def _VerifyValues(self, tensor_in_sizes, filter_in_sizes, bias, strides, - padding): - with self.session() as sess, self.test_scope(): - tensors = [] - ref_tensors = [] - for (data_format, use_gpu) in _GetTestConfigs(): - with ops.device("/gpu:0" if use_gpu else "/cpu:0"): - for dtype in self._DtypesToTest(use_gpu): - for filter_format in self._FilterFormatsToTest(use_gpu): - result, expected = self._SetupValuesForDevice( - tensor_in_sizes, filter_in_sizes, bias, strides, padding, - "Relu", data_format, filter_format, dtype) - tensors.append(result) - ref_tensors.append(expected) - - values = sess.run(tensors) - ref_values = sess.run(ref_tensors) - for i in range(len(tensors)): - conv = tensors[i] - value = values[i] - ref_value = ref_values[i] - tf_logging.info("expected = %s", ref_value) - tf_logging.info("actual = %s", value) - tol = 1e-5 - if value.dtype == np.float16: - tol = 1e-3 - self.assertAllClose( - np.ravel(ref_value), np.ravel(value), atol=tol, rtol=tol) - self.assertShapeEqual(value, conv) - - def testConv2D1x1Filter(self, gpu_only=True): - if gpu_only and not test.is_gpu_available(): - tf_logging.info("Skipping Conv2D1x1Filter test.") - return - # expected_output = [ - # 0.0, 0.0, 0.0, 21.0, 0.0, 0.0, 57.0, 0.0, 0.0, 93.0, 41.0, 0.0, 129.0, - # 86.0, 43.0, 165.0, 131.0, 97.0 - # ] - medians = [-45.0, -130.0, -215.0] - self._VerifyValues( - tensor_in_sizes=[1, 2, 3, 3], - filter_in_sizes=[1, 1, 3, 3], - bias=medians, - strides=[1, 1], - padding="VALID") - - def testConv2DEmpty(self, gpu_only=True): - if gpu_only and not test.is_gpu_available(): - tf_logging.info("Skipping Conv2DEmpty test.") - return - # expected_output = [] - self._VerifyValues( - tensor_in_sizes=[0, 2, 3, 3], - filter_in_sizes=[1, 1, 3, 3], - bias=[0.0, 0.0, 0.0], - strides=[1, 1], - padding="VALID") - - def testConv2D2x2Filter(self, gpu_only=True): - if gpu_only and not test.is_gpu_available(): - tf_logging.info("Skipping Conv2D2x2Filter test.") - return - # expected_output = [0.0, 0.0, 0.0, 401.0, 533.0, 665.0] - self._VerifyValues( - tensor_in_sizes=[1, 2, 3, 3], - filter_in_sizes=[2, 2, 3, 3], - bias=[-2500.0, -2500.0, -2500.0], - strides=[1, 1], - padding="VALID") - - def testConv2D1x2Filter(self, gpu_only=True): - if gpu_only and not test.is_gpu_available(): - tf_logging.info("Skipping Conv2D1x2Filter test.") - return - # expected_output = [ - # 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 190.0, 265.0, 340.0, 343.0, 436.0, 529.0 - # ] - self._VerifyValues( - tensor_in_sizes=[1, 2, 3, 3], - filter_in_sizes=[1, 2, 3, 3], - bias=[-500.0, -500.0, -500.0], - strides=[1, 1], - padding="VALID") - - def testConv2D2x2FilterStride2(self, gpu_only=True): - if gpu_only and not test.is_gpu_available(): - tf_logging.info("Skipping Conv2D2x2FilterStride2 test.") - return - # expected_output = [0.0, 67.0, 163.0] - self._VerifyValues( - tensor_in_sizes=[1, 2, 3, 3], - filter_in_sizes=[2, 2, 3, 3], - bias=[-2300.0, -2300.0, -2300.0], - strides=[2, 2], - padding="VALID") - - def testConv2D2x2FilterStride2Same(self, gpu_only=True): - if gpu_only and not test.is_gpu_available(): - tf_logging.info("Skipping Conv2D2x2FilterStride2Same test.") - return - # expected_output = [0.0, 2367.0, 2463.0, 1230.0, 1305.0, 1380.0] - self._VerifyValues( - tensor_in_sizes=[1, 2, 3, 3], - filter_in_sizes=[2, 2, 3, 3], - bias=[-2300.0, -1000.0, -1000.0], - strides=[2, 2], - padding="SAME") - - def testConv2D2x2FilterStride1x2(self, gpu_only=True): - if gpu_only and not test.is_gpu_available(): - tf_logging.info("Skipping Conv2D2x2FilterStride1x2 test.") - return - # expected_output = [0.0, 0.0, 8.0, 28.0, 48.0, 68.0] - self._VerifyValues( - tensor_in_sizes=[1, 3, 6, 1], - filter_in_sizes=[2, 2, 1, 1], - bias=[-90.0], - strides=[1, 2], - padding="VALID") - - def testConv2DKernelSmallerThanStrideValid(self, gpu_only=True): - if gpu_only and not test.is_gpu_available(): - tf_logging.info("Skipping Conv2DKernelSmallerThanStrideValid test.") - return - # expected_output = [0, 0, 175, 205] - self._VerifyValues( - tensor_in_sizes=[1, 7, 7, 1], - filter_in_sizes=[2, 2, 1, 1], - bias=[-100.0], - strides=[3, 3], - padding="VALID") - - def testConv2DKernelSmallerThanStrideSame(self, gpu_only=True): - if gpu_only and not test.is_gpu_available(): - tf_logging.info("Skipping Conv2DKernelSmallerThanStrideSame test.") - return - # expected = [0, 0, 2, 4] - self._VerifyValues( - tensor_in_sizes=[1, 3, 3, 1], - filter_in_sizes=[1, 1, 1, 1], - bias=[-5.0], - strides=[2, 2], - padding="SAME") - - # expected = [0, 0, 4, 6] - self._VerifyValues( - tensor_in_sizes=[1, 4, 4, 1], - filter_in_sizes=[1, 1, 1, 1], - bias=[-5.0], - strides=[2, 2], - padding="SAME") - - # expected = [4, 0, 1, 0] - self._VerifyValues( - tensor_in_sizes=[1, 4, 4, 1], - filter_in_sizes=[2, 2, 1, 1], - bias=[-40.0], - strides=[3, 3], - padding="SAME") - - def testConv2DKernelSizeMatchesInputSize(self, gpu_only=True): - if gpu_only and not test.is_gpu_available(): - tf_logging.info("Skipping Conv2DKernelSizeMatchesInputSize test.") - return - # expected = [0, 5] - self._VerifyValues( - tensor_in_sizes=[1, 2, 2, 1], - filter_in_sizes=[2, 2, 1, 2], - bias=[-50.0, -55.0], - strides=[1, 1], - padding="VALID") - - # expected = [0, 2, 282, 322] - self._VerifyValues( - tensor_in_sizes=[1, 8, 8, 1], - filter_in_sizes=[2, 2, 1, 1], - bias=[-200.0], - strides=[4, 4], - padding="SAME") - - def testShapeFunctionEdgeCases(self): - # All shapes unknown. - c1 = fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - array_ops.placeholder(dtypes.float32), - array_ops.placeholder(dtypes.float32), - array_ops.placeholder(dtypes.float32), - strides=[1, 1, 1, 1], - padding="SAME", - activation_mode="Relu") - self.assertEqual([None, None, None, None], c1.get_shape().as_list()) - - # Incorrect input shape. - with self.assertRaises(ValueError): - fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - array_ops.placeholder(dtypes.float32, shape=[1, 3]), - array_ops.placeholder(dtypes.float32), - array_ops.placeholder(dtypes.float32), - strides=[1, 1, 1, 1], - padding="SAME", - activation_mode="Relu") - - # Incorrect filter shape. - with self.assertRaises(ValueError): - fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - array_ops.placeholder(dtypes.float32), - array_ops.placeholder(dtypes.float32, shape=[1, 3]), - array_ops.placeholder(dtypes.float32), - strides=[1, 1, 1, 1], - padding="SAME", - activation_mode="Relu") - - # Depth mismatch. - with self.assertRaises(ValueError): - fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - array_ops.placeholder(dtypes.float32, shape=[32, 20, 20, 3]), - array_ops.placeholder(dtypes.float32, shape=[4, 4, 2, 2]), - array_ops.placeholder(dtypes.float32), - strides=[1, 1, 1, 1], - padding="SAME", - activation_mode="Relu") - - def testOpEdgeCases(self, gpu_only=True): - if gpu_only and not test.is_gpu_available(): - tf_logging.info("Skipping OpEdgeCases tests.") - return - with self.session() as sess, self.test_scope(): - # Illegal strides. - with self.assertRaisesRegexp( - errors_impl.UnimplementedError, - ".*strides.*in the batch and depth dimensions"): - sess.run( - fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - _IotaNdF32Constant([1, 1, 1, 1]), - _IotaNdF32Constant([1, 1, 1, 1]), - _IotaNdF32Constant([1]), - strides=[2, 1, 1, 1], - padding="SAME", - activation_mode="Relu")) - with self.assertRaisesRegexp( - errors_impl.UnimplementedError, - ".*strides.*in the batch and depth dimensions"): - sess.run( - fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - _IotaNdF32Constant([1, 1, 1, 1]), - _IotaNdF32Constant([1, 1, 1, 1]), - _IotaNdF32Constant([1]), - strides=[1, 1, 1, 2], - padding="SAME", - activation_mode="Relu")) - - # Illegal activation mode. - with self.assertRaisesRegexp(ValueError, - "Op passed string 'Tanh' not in:"): - sess.run( - fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - _IotaNdF32Constant([1, 1, 1, 1]), - _IotaNdF32Constant([1, 1, 1, 1]), - _IotaNdF32Constant([1]), - strides=[1, 1, 1, 1], - padding="SAME", - activation_mode="Tanh")) - - # Filter larger than input. - with self.assertRaisesRegexp(ValueError, "Negative dimension size"): - sess.run( - fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - _IotaNdF32Constant([32, 20, 20, 3]), - _IotaNdF32Constant([20, 21, 3, 2]), - _IotaNdF32Constant([2]), - strides=[1, 1, 1, 1], - padding="VALID", - activation_mode="Relu")) - with self.assertRaisesRegexp(ValueError, "Negative dimension size"): - sess.run( - fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - _IotaNdF32Constant([32, 20, 20, 3]), - _IotaNdF32Constant([21, 20, 3, 2]), - _IotaNdF32Constant([2]), - strides=[1, 1, 1, 1], - padding="VALID", - activation_mode="Relu")) - - -# Add InceptionFwd tests to FusedConv2DBiasActivationTest. -for index, (input_size_, filter_size_, output_size_, stride_, - padding_) in enumerate(_GetShrunkInceptionShapes()): - setattr(FusedConv2DBiasActivationTest, "testInceptionFwd_" + str(index), - _GetInceptionFwdTest(input_size_, filter_size_, stride_, padding_)) - -# TODO(b/35359731) -# Fwd, BckInput, and BackFilter to test that for certain input parameter -# set, winograd nonfused algorithm will be excluded from conv autotune. If -# in such case, winograd nonfused algorithm is added as one option of the -# conv autotune, and cuDNN version is smaller than 7, the following tests -# will fail. -ishape = [1, 400, 400, 1] -fshape = [1, 1, 1, 256] -oshape = [1, 400, 400, 256] -setattr(FusedConv2DBiasActivationTest, "testInceptionFwd_No_Winograd_Nonfused", - _GetInceptionFwdTest(ishape, fshape, 1, "SAME", gpu_only=True)) - - -def _CalculateConvolvedOutputDim(input_dim, filter_dim, stride, padding_type): - """Calculates the size of an output dimension of a strided convolution. - - Given the sizes of the corresponding dimension of the input and filter shapes, - and the stride and padding_types, calculates the size of the output dimension. - This function can be called separately for each input dimension. - - Args: - input_dim: An `int` specifying the size of the input dimension. - filter_dim: An `int` specifying the size of the filter dimension. - stride: An `int` specifying the step size of the convolution along the input - dimension. - padding_type: either 'VALID' or 'SAME'. - - Returns: - The size of the output dimension. - """ - if padding_type == "VALID": - return (input_dim - filter_dim + stride) // stride - else: # padding_type == 'SAME' - return (input_dim + stride - 1) // stride - - -def _GetFusedConvInt8TestParams(): - """Returns test parameters shared by all Int8 FusedConv tests.""" - _test_params = [ - { - "batch_size": 4, - "input_channels": 256, - "output_channels": 256, - "input_height": 228, - "input_width": 228, - "filter_height": 6, - "filter_width": 6, - "vertical_stride": 1, - "horizontal_stride": 1, - "conv_input_scale": 0.00002, - "side_input_scale": 0.2, - "bias_scale": 1.0, - "padding_type": "SAME" - }, - { - "batch_size": 3, - "input_channels": 8, - "output_channels": 4, - "input_height": 5, - "input_width": 6, - "filter_height": 3, - "filter_width": 3, - "vertical_stride": 1, - "horizontal_stride": 1, - "conv_input_scale": np.linspace(0.006, 0.009, 4, dtype=np.float32), - "side_input_scale": 0.5, - "bias_scale": 1, - "padding_type": "SAME" - }, - { - "batch_size": 1, - "input_channels": 4, - "output_channels": 4, - "input_height": 8, - "input_width": 8, - "filter_height": 6, - "filter_width": 6, - "vertical_stride": 2, - "horizontal_stride": 2, - "conv_input_scale": 0.002, - "side_input_scale": 0.0, - "bias_scale": 1, - "padding_type": "SAME" - }, - { - "batch_size": 1, - "input_channels": 4, - "output_channels": 4, - "input_height": 6, - "input_width": 6, - "filter_height": 6, - "filter_width": 6, - "vertical_stride": 2, - "horizontal_stride": 2, - "conv_input_scale": 0.002, - "side_input_scale": 0.0, - "bias_scale": 1, - "padding_type": "SAME" - }, - { - "batch_size": 2, - "input_channels": 8, - "output_channels": 16, - "input_height": 8, - "input_width": 8, - "filter_height": 3, - "filter_width": 3, - "vertical_stride": 2, - "horizontal_stride": 2, - "conv_input_scale": 0.002, - "side_input_scale": 0.0, - "bias_scale": 1, - "padding_type": "VALID" - }, - { - "batch_size": 2, - "input_channels": 8, - "output_channels": 16, - "input_height": 8, - "input_width": 8, - "filter_height": 3, - "filter_width": 3, - "vertical_stride": 2, - "horizontal_stride": 2, - "conv_input_scale": 0.002, - "side_input_scale": 0.0, - "bias_scale": 1, - "padding_type": "SAME" - }, - { - "batch_size": 2, - "input_channels": 8, - "output_channels": 16, - "input_height": 8, - "input_width": 8, - "filter_height": 3, - "filter_width": 3, - "vertical_stride": 2, - "horizontal_stride": 2, - "conv_input_scale": 0.002, - "side_input_scale": 0.5, - "bias_scale": 1, - "padding_type": "VALID" - }, - { - "batch_size": 2, - "input_channels": 16, - "output_channels": 16, - "input_height": 9, - "input_width": 9, - "filter_height": 3, - "filter_width": 3, - "vertical_stride": 1, - "horizontal_stride": 1, - "conv_input_scale": 0.001, - "side_input_scale": 0.5, - "bias_scale": 1, - "padding_type": "SAME" - }, - { - "batch_size": 3, - "input_channels": 8, - "output_channels": 8, - "input_height": 9, - "input_width": 9, - "filter_height": 5, - "filter_width": 5, - "vertical_stride": 1, - "horizontal_stride": 1, - "conv_input_scale": 0.001, - "side_input_scale": 0.5, - "bias_scale": 1, - "padding_type": "SAME" - }, - { - "batch_size": 3, - "input_channels": 8, - "output_channels": 8, - "input_height": 9, - "input_width": 9, - "filter_height": 7, - "filter_width": 1, - "vertical_stride": 2, - "horizontal_stride": 1, - "conv_input_scale": 0.002, - "side_input_scale": 0.5, - "bias_scale": 1, - "padding_type": "SAME" - }, - { - "batch_size": 3, - "input_channels": 8, - "output_channels": 8, - "input_height": 9, - "input_width": 9, - "filter_height": 1, - "filter_width": 7, - "vertical_stride": 1, - "horizontal_stride": 1, - "conv_input_scale": 0.002, - "side_input_scale": 0.5, - "bias_scale": 1, - "padding_type": "SAME" - }, - ] - return _test_params - - -def _Int8Roundtrip(fn, tensor): - return array_ops.bitcast( - fn(array_ops.bitcast(tensor, dtypes.int8)), dtypes.qint8) - - -def _NchwVectCToNchw(in_tensor): - # [N, C / 4, H, W, 4] => [N, C / 4, 4, H, W] == [N, C, H, W] - t = array_ops.transpose(in_tensor, [0, 1, 4, 2, 3]) - n = in_tensor.shape.dims[0].value - c = in_tensor.shape.dims[1].value * in_tensor.shape.dims[4].value - h = in_tensor.shape.dims[2].value - w = in_tensor.shape.dims[3].value - return array_ops.reshape(t, [n, c, h, w]) - - -def _NchwVectCToNhwc(in_tensor): - # [N, C / 4, H, W, 4] => [N, H, W, C / 4, 4] == [N, H, W, C] - t = array_ops.transpose(in_tensor, [0, 2, 3, 1, 4]) - n = in_tensor.shape.dims[0].value - h = in_tensor.shape.dims[2].value - w = in_tensor.shape.dims[3].value - c = in_tensor.shape.dims[1].value * in_tensor.shape.dims[4].value - return array_ops.reshape(t, [n, h, w, c]) - - -def _OihwVectIToHwio(in_tensor): - # [O, I / 4, H, W, 4] => [O, I / 4, 4, H, W] == [O, I, H, W] - t = array_ops.transpose(in_tensor, [2, 3, 1, 4, 0]) - o = in_tensor.shape.dims[0].value - i = in_tensor.shape.dims[1].value * in_tensor.shape.dims[4].value - h = in_tensor.shape.dims[2].value - w = in_tensor.shape.dims[3].value - return array_ops.reshape(t, [h, w, i, o]) - - -def _NchwToNchwVectC(in_tensor): - n, c, h, w = in_tensor.shape.as_list() - assert c % 4 == 0 - t = array_ops.reshape(in_tensor, [n, c // 4, 4, h, w]) - return array_ops.transpose(t, [0, 1, 3, 4, 2]) - - -def _NhwcToNchwVectC(in_tensor): - # [H, H, W, C] => [N, H, W, C //4, 4] => [N, C / 4, H, W, 4] - n, h, w, c = in_tensor.shape.as_list() - assert c % 4 == 0 - t = array_ops.reshape(in_tensor, [n, h, w, c // 4, 4]) - return array_ops.transpose(t, [0, 3, 1, 2, 4]) - - -def _HwioToOihw(in_tensor): - return array_ops.transpose(in_tensor, [3, 2, 0, 1]) - - -def _SimulateFusedConv2dBiasActivationInt8OnCpu(conv_input_scale, conv_input, - kernel, padding, strides, - side_input_scale, side_input, - biases, apply_relu): - """Simulates the int8 fused 2-D convolution op using separate float ops. - - The arguments and return values have the same format, meanings and - restrictions as the actual op. - - Args: - conv_input_scale: A scalar 'float' Tensor or [1, 1, 1, C] Tensor for the - per-channel quantized mode. - conv_input: A `Tensor` of type `qint8` in NHWC layout. - kernel: A `Tensor` of type `qint8` in HWIO layout. - padding: A `string` from: `"SAME", "VALID"`. - strides: A list of `ints`. - side_input_scale: A scalar 'float'. - side_input: A `Tensor` of type `qint8` in NHWC layout. - biases: A `Tensor` of type `float32` in NHWC layout. - apply_relu: A boolean to specify whether to apply "Relu" activation function - that clips outputs to the range [0, 127], or "None" activation that clips - to the range [-128, 127]. - - Returns: - A `Tensor` of type `qint8` in NHWC layout. - """ - conv_result = nn_ops.conv2d( - math_ops.cast(conv_input, dtypes.float32), - math_ops.cast(kernel, dtypes.float32), - strides=strides, - padding=padding, - data_format="NHWC") * conv_input_scale - - conv_and_side_inputs = conv_result + side_input_scale * math_ops.cast( - side_input, dtypes.float32) - - output = nn_ops.bias_add(conv_and_side_inputs, biases, data_format="NHWC") - if apply_relu: - output = nn_ops.relu(output) - - # In this case quantization is identical to clipping and casting. - result, _, _ = gen_array_ops.quantize_v2(output, -128, 127, dtypes.qint8) - return result - - -# FusedConv2DBiasActivation on CPU supports only NHWC/HWIO data format. -class FusedConvInt8CPUTests(object): - """Verify quantization with CPU kernel.""" - _test_params = _GetFusedConvInt8TestParams() - - @contextlib.contextmanager - def test_scope(self): # pylint: disable=invalid-name - """Can be overridden in base classes to provide a test scope.""" - yield - - def runTest(self, test_param, apply_relu): - """Runs tests for dimensions configured in test_param.""" - - batch_size = test_param["batch_size"] - input_channels = test_param["input_channels"] - output_channels = test_param["output_channels"] - input_height = test_param["input_height"] - input_width = test_param["input_width"] - filter_height = test_param["filter_height"] - filter_width = test_param["filter_width"] - vertical_stride = test_param["vertical_stride"] - horizontal_stride = test_param["horizontal_stride"] - conv_input_scale = test_param["conv_input_scale"] - side_input_scale = test_param["side_input_scale"] - bias_scale = test_param["bias_scale"] - padding_type = test_param["padding_type"] - - with self.session() as sess, self.test_scope(): - conv_input, _, _ = gen_array_ops.quantize_v2( - random_ops.random_uniform( - [batch_size, input_height, input_width, input_channels], - minval=-0.0, - maxval=1.0, - dtype=dtypes.float32), - -1.0, - 1.0, - dtypes.qint8, - mode="SCALED") - self.assertTrue( - sess.run( - math_ops.reduce_all( - math_ops.greater_equal( - array_ops.bitcast(conv_input, dtypes.int8), 0)))) - - kernel, _, _ = gen_array_ops.quantize_v2( - random_ops.random_uniform( - [filter_height, filter_width, input_channels, output_channels], - minval=-1.0, - maxval=1.0, - dtype=dtypes.float32), - -1.0, - 1.0, - dtypes.qint8, - mode="SCALED") - - output_height = _CalculateConvolvedOutputDim(input_height, filter_height, - vertical_stride, - padding_type) - output_width = _CalculateConvolvedOutputDim(input_width, filter_width, - horizontal_stride, - padding_type) - tf_logging.info("output_height=%s, output_width=%s", output_height, - output_width) - - side_input, _, _ = gen_array_ops.quantize_v2( - random_ops.random_uniform( - [batch_size, output_height, output_width, output_channels], - minval=0.0, - maxval=1.0, - dtype=dtypes.float32), - -1.0, - 1.0, - dtypes.qint8, - mode="SCALED") - - biases = random_ops.random_uniform([output_channels], - minval=-10 * bias_scale, - maxval=20 * bias_scale, - dtype=dtypes.float32) - - strides = [1, vertical_stride, horizontal_stride, 1] - - actual = fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - conv_input, - kernel, - biases, - strides=strides, - padding=padding_type, - conv_input_scale=conv_input_scale, - side_input_scale=side_input_scale, - side_input=(None if side_input_scale == 0.0 else side_input), - activation_mode="Relu" if apply_relu else "None", - data_format="NHWC", - filter_format="HWIO") - - if isinstance(conv_input_scale, (list, np.ndarray)): - conv_input_scale = array_ops.reshape( - conv_input_scale, [1, 1, 1, len(conv_input_scale)]) - - expected = _SimulateFusedConv2dBiasActivationInt8OnCpu( - conv_input_scale, conv_input, kernel, padding_type, strides, - side_input_scale, side_input, biases, apply_relu) - - actual_y, expected_y = sess.run([actual, expected]) - self.assertAllClose(actual_y, expected_y, rtol=0, atol=1) - - def testFusedConvInt8(self): - for apply_relu in [True, False]: - for test_param in self._test_params: - self.runTest(test_param, apply_relu) - - def testRoundingMode(self): - """Verify the fused convolution op uses half-to-even rounding mode.""" - batches = 1 - input_size = 2 - input_channels = 1 - output_channels = 1 - conv_input = np.array([1, 2, 3, 4]).reshape( - (batches, input_size, input_size, input_channels)).astype(np.int8) - kernel = np.array([1]).reshape( - (1, 1, input_channels, output_channels)).astype(np.int8) - biases = np.zeros((output_channels)).astype(np.float32) - - with self.session() as sess, self.test_scope(): - actual = fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - math_ops.cast(conv_input, dtypes.qint8), - math_ops.cast(kernel, dtypes.qint8), - biases, - strides=[1, 1, 1, 1], - padding="SAME", - conv_input_scale=0.5, - side_input_scale=0.0, - activation_mode="None", - data_format="NHWC", - filter_format="HWIO") - actual_value = sess.run(actual) - # The convolution output scaled is [0.5, 1.0, 1.5, 2.0]. After rounding - # half to even, the final output is [0, 1, 2, 2]. - self.assertTrue( - np.array_equal(actual_value.flatten(), - np.array([0, 1, 2, 2]).astype(np.int8))) - - -# Test that GPU and CPU kernels produce identical results for QInt8 data type. -class FusedConvInt8CorrespondenceTests(object): - """Verify quantization with CPU kernel.""" - _test_params = _GetFusedConvInt8TestParams() - - @contextlib.contextmanager - def test_scope(self): # pylint: disable=invalid-name - """Can be overridden in base classes to provide a test scope.""" - yield - - def runTest(self, test_param, apply_relu): - """Runs tests for dimensions configured in test_param.""" - - batch_size = test_param["batch_size"] - input_channels = test_param["input_channels"] - output_channels = test_param["output_channels"] - input_height = test_param["input_height"] - input_width = test_param["input_width"] - filter_height = test_param["filter_height"] - filter_width = test_param["filter_width"] - vertical_stride = test_param["vertical_stride"] - horizontal_stride = test_param["horizontal_stride"] - conv_input_scale = test_param["conv_input_scale"] - side_input_scale = test_param["side_input_scale"] - bias_scale = test_param["bias_scale"] - padding_type = test_param["padding_type"] - - if isinstance(conv_input_scale, (list, np.ndarray)): - # GPU kernel (Cudnn) does not support per-channel scale. - return - - with self.session() as sess, self.test_scope(): - conv_input, _, _ = gen_array_ops.quantize_v2( - random_ops.random_uniform( - [batch_size, input_channels // 4, input_height, input_width, 4], - minval=0.0, - maxval=1.0, - dtype=dtypes.float32), - -1.0, - 1.0, - dtypes.qint8, - mode="SCALED") - self.assertTrue( - sess.run( - math_ops.reduce_all( - math_ops.greater_equal( - array_ops.bitcast(conv_input, dtypes.int8), 0)))) - - kernel, _, _ = gen_array_ops.quantize_v2( - random_ops.random_uniform([ - output_channels, input_channels // 4, filter_height, filter_width, - 4 - ], - minval=-128.0, - maxval=127.0, - dtype=dtypes.float32), - -128.0, - 127.0, - dtypes.qint8, - mode="SCALED") - - output_height = _CalculateConvolvedOutputDim(input_height, filter_height, - vertical_stride, - padding_type) - output_width = _CalculateConvolvedOutputDim(input_width, filter_width, - horizontal_stride, - padding_type) - tf_logging.info("output_height=%s, output_width=%s", output_height, - output_width) - - side_input, _, _ = gen_array_ops.quantize_v2( - random_ops.random_uniform([ - batch_size, output_channels // 4, output_height, output_width, 4 - ], - minval=0.0, - maxval=1.0, - dtype=dtypes.float32), - -1.0, - 1.0, - dtypes.qint8, - mode="SCALED") - - biases = random_ops.random_uniform([output_channels], - minval=-10 * bias_scale, - maxval=20 * bias_scale, - dtype=dtypes.float32) - - with ops.device("/cpu:0"): - t = fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - _Int8Roundtrip(_NchwVectCToNhwc, conv_input), - _Int8Roundtrip(_OihwVectIToHwio, kernel), - biases, - strides=[1, vertical_stride, horizontal_stride, 1], - padding=padding_type, - conv_input_scale=conv_input_scale, - side_input_scale=side_input_scale, - side_input=(None if side_input_scale == 0.0 else _Int8Roundtrip( - _NchwVectCToNhwc, side_input)), - activation_mode="Relu" if apply_relu else "None", - data_format="NHWC", - filter_format="HWIO") - cpu_result = _Int8Roundtrip(_NhwcToNchwVectC, t) - - with ops.device("/gpu:0"): - t = fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( - conv_input, - kernel, - biases, - strides=[1, 1, vertical_stride, horizontal_stride], - padding=padding_type, - conv_input_scale=conv_input_scale, - side_input_scale=side_input_scale, - side_input=(None if side_input_scale == 0.0 else side_input), - activation_mode="Relu" if apply_relu else "None", - data_format="NCHW_VECT_C", - filter_format="OIHW_VECT_I") - gpu_result = t - - cpu_y, gpu_y = sess.run([cpu_result, gpu_result]) - self.assertAllClose(cpu_y, gpu_y, rtol=0, atol=0) - - def testFusedConvInt8(self): - if not test.is_gpu_available( - cuda_only=True, min_cuda_compute_capability=(6, 1)): - tf_logging.info("int8 test skipped because not run with --config=cuda or " - "no GPUs with compute capability >= 6.1 are available.") - return - for apply_relu in [True, False]: - for test_param in self._test_params: - self.runTest(test_param, apply_relu) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/fused_conv/python/ops/tensorrt_fused_conv_test.py b/tensorflow/contrib/fused_conv/python/ops/tensorrt_fused_conv_test.py deleted file mode 100644 index d5c4a51caac..00000000000 --- a/tensorflow/contrib/fused_conv/python/ops/tensorrt_fused_conv_test.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Script to test TF-TensorRT conversion of FusedConv2DBiasActivation.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.fused_conv import fused_conv2d_bias_activation -from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -def conv2d_fused_layer(inputs, - filters, - kernel_size, - strides=(1, 1), - padding="valid", - data_format="NHWC", - filter_format="HWIO"): - dtype = inputs.dtype - c_axis = -1 if data_format == "NHWC" else 1 - nchan = inputs.shape[c_axis] - if filter_format == "HWIO": - weights_shape = (kernel_size[0], kernel_size[1], nchan, filters) - else: - assert filter_format == "OIHW" - weights_shape = (filters, nchan, kernel_size[0], kernel_size[1]) - weights = constant_op.constant(np.random.randn(*weights_shape), dtype=dtype) - biases_shape = (filters,) - biases = constant_op.constant(np.random.randn(*biases_shape), dtype=dtype) - padding = padding.upper() - if data_format == "NHWC": - strides = [1] + list(strides) + [1] - else: - assert data_format == "NCHW" - strides = [1, 1] + list(strides) - return fused_conv2d_bias_activation( - inputs, - weights, - biases, - strides=strides, - padding=padding, - data_format=data_format, - filter_format=filter_format, - activation_mode="Relu") - - -def build_fused_conv_graph(inp, - num_filters, - data_format, - filter_format, - kernel_sizes, - padding="same"): - results = [] - for kernel_size in kernel_sizes: - result = conv2d_fused_layer(inp, num_filters, kernel_size, (1, 1), - padding, data_format, filter_format) - results.append(result) - output = sum(results) - return array_ops.identity(output, name="output_0") - - -class Conv2DFusedHWIOTest(trt_test.TfTrtIntegrationTestBase): - """Testing conversion of Fused Conv2D+Bias+Activation (filter_format=HWIO).""" - - def GraphFn(self, inp): - np.random.seed(1234) - return build_fused_conv_graph( - inp=inp, - num_filters=5, - data_format="NHWC", - filter_format="HWIO", - kernel_sizes=[(3, 3), (3, 2)]) - - def GetParams(self): - # TODO(aaroey): test graph with different dtypes. - return self.BuildParams(self.GraphFn, dtypes.float32, [[13, 7, 11, 3]], - [[13, 7, 11, 5]]) - - def ExpectedEnginesToBuild(self, run_params): - """Return the expected engines to build.""" - return ["TRTEngineOp_0"] - - -class Conv2DFusedOIHWTest(trt_test.TfTrtIntegrationTestBase): - """Testing conversion of Fused Conv2D+Bias+Activation (filter_format=HWIO).""" - - def GraphFn(self, inp): - np.random.seed(1234) - return build_fused_conv_graph( - inp=inp, - num_filters=5, - data_format="NCHW", - filter_format="OIHW", - kernel_sizes=[(3, 3), (3, 2)]) - - def GetParams(self): - # TODO(aaroey): test graph with different dtypes. - return self.BuildParams(self.GraphFn, dtypes.float32, [[13, 3, 7, 11]], - [[13, 5, 7, 11]]) - - def ExpectedEnginesToBuild(self, run_params): - """Return the expected engines to build.""" - return ["TRTEngineOp_0"] - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/graph_editor/BUILD b/tensorflow/contrib/graph_editor/BUILD deleted file mode 100644 index 0683a90610b..00000000000 --- a/tensorflow/contrib/graph_editor/BUILD +++ /dev/null @@ -1,153 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "graph_editor_py", - srcs = [ - "__init__.py", - "edit.py", - "reroute.py", - "select.py", - "subgraph.py", - "transform.py", - "util.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:op_selector", - "//tensorflow/python:platform", - "//tensorflow/python:util", - "@six_archive//:six", - ], -) - -# Transitive dependencies of this target will be included in the pip package. -py_library( - name = "graph_editor_pip", - deps = [ - ":graph_editor_py", - ":match", - ], -) - -py_library( - name = "match", - srcs = ["tests/match.py"], - srcs_version = "PY2AND3", - deps = [ - ":graph_editor_py", - "//tensorflow/python:framework_ops", - "@six_archive//:six", - ], -) - -py_test( - name = "util_test", - srcs = ["tests/util_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":graph_editor_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - ], -) - -py_test( - name = "select_test", - srcs = ["tests/select_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":graph_editor_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - ], -) - -py_test( - name = "match_test", - srcs = ["tests/match_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":match", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - ], -) - -py_test( - name = "subgraph_test", - srcs = ["tests/subgraph_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":graph_editor_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - ], -) - -py_test( - name = "reroute_test", - srcs = ["tests/reroute_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":graph_editor_py", - ":match", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - ], -) - -py_test( - name = "edit_test", - srcs = ["tests/edit_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":graph_editor_py", - ":match", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - ], -) - -py_test( - name = "transform_test", - srcs = ["tests/transform_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":graph_editor_py", - ":match", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:gradients", - "//tensorflow/python:math_ops", - "//tensorflow/python:session", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/graph_editor/README.md b/tensorflow/contrib/graph_editor/README.md deleted file mode 100644 index f6f82668ed4..00000000000 --- a/tensorflow/contrib/graph_editor/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# TensorFlow Graph Editor - -The TensorFlow Graph Editor library allows for modification of an existing -tf.Graph instance in-place. - -The author's github username is [purpledog](https://github.com/purpledog). diff --git a/tensorflow/contrib/graph_editor/__init__.py b/tensorflow/contrib/graph_editor/__init__.py deleted file mode 100644 index 72f0ff420f6..00000000000 --- a/tensorflow/contrib/graph_editor/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""TensorFlow Graph Editor.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.graph_editor.edit import * -from tensorflow.contrib.graph_editor.reroute import * -from tensorflow.contrib.graph_editor.select import * -from tensorflow.contrib.graph_editor.subgraph import * -from tensorflow.contrib.graph_editor.transform import * -from tensorflow.contrib.graph_editor.util import * -# pylint: enable=wildcard-import - -# some useful aliases -# pylint: disable=g-bad-import-order -from tensorflow.contrib.graph_editor import subgraph as _subgraph -from tensorflow.contrib.graph_editor import util as _util -# pylint: enable=g-bad-import-order -ph = _util.make_placeholder_from_dtype_and_shape -sgv = _subgraph.make_view -sgv_scope = _subgraph.make_view_from_scope - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/graph_editor/edit.py b/tensorflow/contrib/graph_editor/edit.py deleted file mode 100644 index 3037eef4894..00000000000 --- a/tensorflow/contrib/graph_editor/edit.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Various function for graph editing.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.graph_editor import reroute -from tensorflow.contrib.graph_editor import select -from tensorflow.contrib.graph_editor import subgraph -from tensorflow.contrib.graph_editor import util -from tensorflow.python.ops import array_ops as tf_array_ops - -__all__ = [ - "detach_control_inputs", - "detach_control_outputs", - "detach_inputs", - "detach_outputs", - "detach", - "connect", - "bypass", -] - - -def detach_control_inputs(sgv): - """Detach all the external control inputs of the subgraph sgv. - - Args: - sgv: the subgraph view to be detached. This argument is converted to a - subgraph using the same rules as the function subgraph.make_view. - """ - sgv = subgraph.make_view(sgv) - for op in sgv.ops: - cops = [cop for cop in op.control_inputs if cop not in sgv.ops] - reroute.remove_control_inputs(op, cops) - - -def detach_control_outputs(sgv, control_outputs): - """Detach all the external control outputs of the subgraph sgv. - - Args: - sgv: the subgraph view to be detached. This argument is converted to a - subgraph using the same rules as the function subgraph.make_view. - control_outputs: a util.ControlOutputs instance. - """ - if not isinstance(control_outputs, util.ControlOutputs): - raise TypeError("Expected a util.ControlOutputs, got: {}", - type(control_outputs)) - control_outputs.update() - sgv = subgraph.make_view(sgv) - for op in sgv.ops: - for cop in control_outputs.get(op): - if cop not in sgv.ops: - reroute.remove_control_inputs(cop, op) - - -def detach_inputs(sgv, control_inputs=False): - """Detach the inputs of a subgraph view. - - Args: - sgv: the subgraph view to be detached. This argument is converted to a - subgraph using the same rules as the function subgraph.make_view. - Note that sgv is modified in place. - control_inputs: if True control_inputs are also detached. - Returns: - A tuple `(sgv, input_placeholders)` where - `sgv` is a new subgraph view of the detached subgraph; - `input_placeholders` is a list of the created input placeholders. - Raises: - StandardError: if sgv cannot be converted to a SubGraphView using - the same rules than the function subgraph.make_view. - """ - sgv = subgraph.make_view(sgv) - - with sgv.graph.as_default(): - input_placeholders = [ - tf_array_ops.placeholder( - dtype=input_t.dtype, name=util.placeholder_name(input_t)) - for input_t in sgv.inputs - ] - - reroute.swap_inputs(sgv, input_placeholders) - if control_inputs: - detach_control_inputs(sgv) - return sgv, input_placeholders - - -def detach_outputs(sgv, control_outputs=None): - """Detach the output of a subgraph view. - - Args: - sgv: the subgraph view to be detached. This argument is converted to a - subgraph using the same rules as the function subgraph.make_view. - Note that sgv is modified in place. - control_outputs: a util.ControlOutputs instance or None. If not None the - control outputs are also detached. - Returns: - A tuple `(sgv, output_placeholders)` where - `sgv` is a new subgraph view of the detached subgraph; - `output_placeholders` is a list of the created output placeholders. - Raises: - StandardError: if sgv cannot be converted to a SubGraphView using - the same rules than the function subgraph.make_view. - """ - sgv = subgraph.make_view(sgv) - # only select outputs with consumers - sgv_ = sgv.remap_outputs([output_id - for output_id, output_t in enumerate(sgv.outputs) - if output_t.consumers()]) - # create consumer subgraph and remap - consumers_sgv = subgraph.SubGraphView(sgv_.consumers()) - consumers_sgv = consumers_sgv.remap_inputs( - [input_id for input_id, input_t in enumerate(consumers_sgv.inputs) - if input_t in sgv_.outputs]) - - with sgv_.graph.as_default(): - output_placeholders = [ - util.make_placeholder_from_tensor(input_t) - for input_t in consumers_sgv.inputs - ] - - reroute.swap_outputs(sgv_, output_placeholders) - if control_outputs is not None: - detach_control_outputs(sgv_, control_outputs) - return sgv_, output_placeholders - - -def detach(sgv, control_inputs=False, control_outputs=None, control_ios=None): - """Detach both the inputs and the outputs of a subgraph view. - - Args: - sgv: the subgraph view to be detached. This argument is converted to a - subgraph using the same rules as the function subgraph.make_view. - Note that sgv is modified in place. - control_inputs: A boolean indicating whether control inputs are enabled. - control_outputs: An instance of util.ControlOutputs or None. If not None, - control outputs are enabled. - control_ios: An instance of util.ControlOutputs or None. If not None, both - control inputs and control outputs are enabled. This is equivalent to set - control_inputs to True and control_outputs to the util.ControlOutputs - instance. - Returns: - A tuple `(sgv, detached_inputs, detached_outputs)` where: - `sgv` is a new subgraph view of the detached subgraph; - `detach_inputs` is a list of the created input placeholders; - `detach_outputs` is a list of the created output placeholders. - Raises: - StandardError: if sgv cannot be converted to a SubGraphView using - the same rules than the function subgraph.make_view. - """ - control_inputs, control_outputs = select.check_cios(control_inputs, - control_outputs, - control_ios) - _, detached_inputs = detach_inputs(sgv, control_inputs) - _, detached_outputs = detach_outputs(sgv, control_outputs) - return sgv, detached_inputs, detached_outputs - - -def connect(sgv0, sgv1, disconnect_first=False): - """Connect the outputs of sgv0 to the inputs of sgv1. - - Args: - sgv0: the first subgraph to have its outputs swapped. This argument is - converted to a subgraph using the same rules as the function - subgraph.make_view. - Note that sgv0 is modified in place. - sgv1: the second subgraph to have its outputs swapped. This argument is - converted to a subgraph using the same rules as the function - subgraph.make_view. - Note that sgv1 is modified in place. - disconnect_first: if True the current outputs of sgv0 are disconnected. - Returns: - A tuple `(sgv0, sgv1)` of the now connected subgraphs. - Raises: - StandardError: if sgv0 or sgv1 cannot be converted to a SubGraphView using - the same rules than the function subgraph.make_view. - """ - sgv0 = subgraph.make_view(sgv0) - sgv1 = subgraph.make_view(sgv1) - util.check_graphs(sgv0, sgv1) - if disconnect_first: - detach_outputs(sgv0) - sgv0_outputs = subgraph.SubGraphView(passthrough_ts=sgv0.outputs) - reroute.reroute_inputs(sgv0_outputs, sgv1) - return sgv0, sgv1 - - -def bypass(sgv): - """Bypass the given subgraph by connecting its inputs to its outputs. - - Args: - sgv: the subgraph view to be bypassed. This argument is converted to a - subgraph using the same rules than the function subgraph.make_view. - Note that sgv is modified in place. - Returns: - A tuple `(sgv, detached_inputs)` where: - `sgv` is a new subgraph view of the bypassed subgraph; - `detached_inputs` is a list of the created input placeholders. - Raises: - StandardError: if sgv cannot be converted to a SubGraphView using - the same rules than the function subgraph.make_view. - """ - # TODO(fkp): allows to plug sgv.inputs to individual sgv.outputs consumers - sgv = subgraph.make_view(sgv) - sgv_inputs = list(sgv.inputs) - sgv, detached_inputs = detach_inputs(sgv) - reroute.reroute_ts(sgv_inputs, sgv.outputs) - return sgv, detached_inputs diff --git a/tensorflow/contrib/graph_editor/examples/edit_graph_example.py b/tensorflow/contrib/graph_editor/examples/edit_graph_example.py deleted file mode 100644 index af9cddc141e..00000000000 --- a/tensorflow/contrib/graph_editor/examples/edit_graph_example.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Simple GraphEditor example. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import tensorflow as tf -from tensorflow.contrib import graph_editor as ge - -FLAGS = tf.flags.FLAGS - - -def main(_): - # create a graph - g = tf.Graph() - with g.as_default(): - a = tf.constant(1.0, shape=[2, 3], name="a") - b = tf.constant(2.0, shape=[2, 3], name="b") - c = tf.add( - tf.placeholder(dtype=np.float32), - tf.placeholder(dtype=np.float32), - name="c") - - # modify the graph - ge.swap_inputs(c.op, [a, b]) - - # print the graph def - print(g.as_graph_def()) - - # and print the value of c - with tf.Session(graph=g) as sess: - res = sess.run(c) - print(res) - - -if __name__ == "__main__": - tf.app.run() diff --git a/tensorflow/contrib/graph_editor/reroute.py b/tensorflow/contrib/graph_editor/reroute.py deleted file mode 100644 index d42e0c01f45..00000000000 --- a/tensorflow/contrib/graph_editor/reroute.py +++ /dev/null @@ -1,502 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Various function for graph rerouting.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.graph_editor import subgraph as _subgraph -from tensorflow.contrib.graph_editor import util as _util -from tensorflow.python.framework import ops as _tf_ops - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - "swap_ts", - "reroute_ts", - "swap_inputs", - "reroute_inputs", - "swap_outputs", - "reroute_outputs", - "swap_ios", - "reroute_ios", - "remove_control_inputs", - "add_control_inputs", -] - - -def _check_ts_compatibility(ts0, ts1): - """Make sure the shape and dtype of the two tensor's lists are compatible. - - Args: - ts0: an object convertible to a list of `tf.Tensor`. - ts1: an object convertible to a list of `tf.Tensor`. - Raises: - ValueError: if any pair of tensors (same index in ts0 and ts1) have - a dtype or a shape which is not compatible. - """ - ts0 = _util.make_list_of_t(ts0) - ts1 = _util.make_list_of_t(ts1) - if len(ts0) != len(ts1): - raise ValueError("ts0 and ts1 have different sizes: {} != {}".format( - len(ts0), len(ts1))) - for t0, t1 in zip(ts0, ts1): - # check dtype - dtype0, dtype1 = t0.dtype, t1.dtype - if not dtype0.is_compatible_with(dtype1): - raise ValueError("Dtypes {} and {} are not compatible.".format(dtype0, - dtype1)) - # check shape - shape0, shape1 = t0.get_shape(), t1.get_shape() - if not shape0.is_compatible_with(shape1): - raise ValueError("Shapes {} and {} are not compatible.".format(shape0, - shape1)) - - -class _RerouteMode(object): - """Enums for reroute's mode. - - swap: the end of tensors a and b are swapped. - a2b: the end of the tensor a are also rerouted to the end of the tensor b - (the end of b is left dangling). - b2a: the end of the tensor b are also rerouted to the end of the tensor a - (the end of a is left dangling). - """ - swap, a2b, b2a = range(3) - - @classmethod - def check(cls, mode): - """Check swap mode. - - Args: - mode: an integer representing one of the modes. - Returns: - A tuple `(a2b, b2a)` boolean indicating what rerouting needs doing. - Raises: - ValueError: if mode is outside the enum range. - """ - if mode == cls.swap: - return True, True - elif mode == cls.b2a: - return False, True - elif mode == cls.a2b: - return True, False - else: - raise ValueError("Unknown _RerouteMode: {}".format(mode)) - - -def _reroute_t(t0, t1, consumers1, can_modify=None, cannot_modify=None): - """Reroute the end of the tensors (t0,t1). - - Warning: this function is directly manipulating the internals of the - `tf.Graph`. - - Args: - t0: a tf.Tensor. - t1: a tf.Tensor. - consumers1: The consumers of t1 which needs to be rerouted. - can_modify: iterable of operations which can be modified. Any operation - outside within_ops will be left untouched by this function. - cannot_modify: iterable of operations which cannot be modified. - Any operation within cannot_modify will be left untouched by this - function. - Returns: - The number of individual modifications made by the function. - """ - nb_update_inputs = 0 - if can_modify is not None: - consumers1 &= can_modify - if cannot_modify is not None: - consumers1 -= cannot_modify - consumers1_indices = {} - for consumer1 in consumers1: - consumers1_indices[consumer1] = [i for i, t in enumerate(consumer1.inputs) - if t is t1] - for consumer1 in consumers1: - for i in consumers1_indices[consumer1]: - consumer1._update_input(i, t0) # pylint: disable=protected-access - nb_update_inputs += 1 - return nb_update_inputs - - -def _reroute_ts(ts0, ts1, mode, can_modify=None, cannot_modify=None): - """Reroute the end of the tensors in each pair (t0,t1) in ts0 x ts1. - - This function is the back-bone of the Graph-Editor. It is essentially a thin - wrapper on top of the tf.Operation._update_input. - - Given a pair of tensor t0, t1 in ts0 x ts1, this function re-route the end - of t0 and t1 in three possible ways: - 1) The reroute mode is "a<->b" or "b<->a": the tensors' end are swapped. After - this operation, the previous consumers of t0 are now consumers of t1 and - vice-versa. - 2) The reroute mode is "a->b": the tensors' end of t0 are re-routed to the - tensors's end of t1 (which are left dangling). After this operation, the - previous consumers of t0 are still consuming t0 but the previous consumers of - t1 are not also consuming t0. The tensor t1 has no consumer. - 3) The reroute mode is "b->a": this mode is the symmetric of the "a->b" mode. - - Note that this function is re-routing the end of two tensors, not the start. - Re-routing the start of two tensors is not supported by this library. The - reason for that is the following: TensorFlow, by design, creates a strong bond - between an op and its output tensor. This Graph editor follows this design and - treats an operation A and its generating tensors {t_i} as an entity which - cannot be broken. In other words, an op cannot be detached from any of its - output tensors, ever. But it is possible to detach an op from its input - tensors, which is what this function concerns itself with. - - Warning: this function is directly manipulating the internals of the tf.Graph. - - Args: - ts0: an object convertible to a list of `tf.Tensor`. - ts1: an object convertible to a list of `tf.Tensor`. - mode: what to do with those tensors: "a->b" or "b<->a" for swaping and - "a->b" or "b->a" for one direction re-routing. - can_modify: iterable of operations which can be modified. Any operation - outside within_ops will be left untouched by this function. - cannot_modify: iterable of operations which cannot be modified. - Any operation within cannot_modify will be left untouched by this - function. - Returns: - The number of individual modifications made by the function. - Raises: - TypeError: if `ts0` or `ts1` cannot be converted to a list of `tf.Tensor`. - TypeError: if `can_modify` or `cannot_modify` is not `None` and cannot be - converted to a list of `tf.Operation`. - """ - a2b, b2a = _RerouteMode.check(mode) - ts0 = _util.make_list_of_t(ts0) - ts1 = _util.make_list_of_t(ts1) - _check_ts_compatibility(ts0, ts1) - if cannot_modify is not None: - cannot_modify = frozenset(_util.make_list_of_op(cannot_modify)) - if can_modify is not None: - can_modify = frozenset(_util.make_list_of_op(can_modify)) - nb_update_inputs = 0 - precomputed_consumers = [] - # precompute consumers to avoid issue with repeated tensors: - for t0, t1 in zip(ts0, ts1): - consumers0 = set(t0.consumers()) - consumers1 = set(t1.consumers()) - precomputed_consumers.append((consumers0, consumers1)) - for t0, t1, consumers in zip(ts0, ts1, precomputed_consumers): - if t0 is t1: - continue # Silently ignore identical tensors. - consumers0, consumers1 = consumers - if a2b: - nb_update_inputs += _reroute_t(t0, t1, consumers1, can_modify, - cannot_modify) - if b2a: - nb_update_inputs += _reroute_t(t1, t0, consumers0, can_modify, - cannot_modify) - return nb_update_inputs - - -def swap_ts(ts0, ts1, can_modify=None, cannot_modify=None): - """For each tensor's pair, swap the end of (t0,t1). - - B0 B1 B0 B1 - | | => X - A0 A1 A0 A1 - - Args: - ts0: an object convertible to a list of `tf.Tensor`. - ts1: an object convertible to a list of `tf.Tensor`. - can_modify: iterable of operations which can be modified. Any operation - outside within_ops will be left untouched by this function. - cannot_modify: iterable of operations which cannot be modified. - Any operation within cannot_modify will be left untouched by this - function. - Returns: - The number of individual modifications made by the function. - Raises: - TypeError: if ts0 or ts1 cannot be converted to a list of tf.Tensor. - TypeError: if can_modify or cannot_modify is not None and cannot be - converted to a list of tf.Operation. - """ - return _reroute_ts(ts0, ts1, _RerouteMode.swap, can_modify, cannot_modify) - - -def reroute_ts(ts0, ts1, can_modify=None, cannot_modify=None): - """For each tensor's pair, replace the end of t1 by the end of t0. - - B0 B1 B0 B1 - | | => |/ - A0 A1 A0 A1 - - The end of the tensors in ts1 are left dangling. - - Args: - ts0: an object convertible to a list of `tf.Tensor`. - ts1: an object convertible to a list of `tf.Tensor`. - can_modify: iterable of operations which can be modified. Any operation - outside within_ops will be left untouched by this function. - cannot_modify: iterable of operations which cannot be modified. Any - operation within cannot_modify will be left untouched by this function. - Returns: - The number of individual modifications made by the function. - Raises: - TypeError: if ts0 or ts1 cannot be converted to a list of tf.Tensor. - TypeError: if can_modify or cannot_modify is not None and cannot be - converted to a list of tf.Operation. - """ - return _reroute_ts(ts0, ts1, _RerouteMode.a2b, can_modify, cannot_modify) - - -def _reroute_sgv_remap(sgv0, sgv1, mode): - """Remap in place the inputs of two subgraph views to mimic the reroute. - - This function is meant to used by reroute_inputs only. - - Args: - sgv0: the first subgraph to have its inputs remapped. - sgv1: the second subgraph to have its inputs remapped. - mode: reroute mode, see _reroute_ts(...). - Raises: - TypeError: if svg0 or svg1 are not SubGraphView. - ValueError: if sgv0 and sgv1 do not belong to the same graph. - """ - a2b, b2a = _RerouteMode.check(mode) - if not isinstance(sgv0, _subgraph.SubGraphView): - raise TypeError("Expected a SubGraphView, got {}".format(type(sgv0))) - if not isinstance(sgv1, _subgraph.SubGraphView): - raise TypeError("Expected a SubGraphView, got {}".format(type(sgv1))) - _util.check_graphs(sgv0, sgv1) - sgv0_ = sgv0.copy() - sgv1_ = sgv1.copy() - # pylint: disable=protected-access - if a2b and b2a: - (sgv0_._input_ts, sgv1_._input_ts) = (sgv1_._input_ts, sgv0_._input_ts) - (sgv0_._passthrough_ts, sgv1_._passthrough_ts) = (sgv1_._passthrough_ts, - sgv0_._passthrough_ts) - elif a2b: - sgv1_._input_ts = sgv0_._input_ts[:] - sgv1_._passthrough_ts = sgv0_._passthrough_ts[:] - elif b2a: - sgv0_._input_ts = sgv1_._input_ts[:] - sgv0_._passthrough_ts = sgv1_._passthrough_ts[:] - # pylint: enable=protected-access - - # Update the passthrough outputs as well. - def update_passthrough_outputs(a, b): - # pylint: disable=protected-access - for i, t in enumerate(b._output_ts): - if t in a._passthrough_ts: - ii = a._input_ts.index(t) - b._output_ts[i] = b._input_ts[ii] - # pylint: enable=protected-access - - if a2b: - update_passthrough_outputs(sgv0_, sgv1_) - if b2a: - update_passthrough_outputs(sgv1_, sgv0_) - - # in-place - # pylint: disable=protected-access - sgv0._assign_from(sgv0_) - sgv1._assign_from(sgv1_) - # pylint: enable=protected-access - - -def _reroute_sgv_inputs(sgv0, sgv1, mode): - """Re-route all the inputs of two subgraphs. - - Args: - sgv0: the first subgraph to have its inputs swapped. This argument is - converted to a subgraph using the same rules than the function - subgraph.make_view. - sgv1: the second subgraph to have its inputs swapped. This argument is - converted to a subgraph using the same rules than the function - subgraph.make_view. - mode: reroute mode, see _reroute_ts(...). - Returns: - A tuple `(sgv0, sgv1)` of subgraph views with their inputs swapped. - Note that the function argument sgv0 and sgv1 are also modified in place. - Raises: - StandardError: if sgv0 or sgv1 cannot be converted to a SubGraphView using - the same rules than the function subgraph.make_view. - """ - sgv0 = _subgraph.make_view(sgv0) - sgv1 = _subgraph.make_view(sgv1) - _util.check_graphs(sgv0, sgv1) - can_modify = sgv0.ops + sgv1.ops - # also allow consumers of passthrough to be modified: - can_modify += _util.get_consuming_ops(sgv0.passthroughs) - can_modify += _util.get_consuming_ops(sgv1.passthroughs) - _reroute_ts(sgv0.inputs, sgv1.inputs, mode, can_modify=can_modify) - _reroute_sgv_remap(sgv0, sgv1, mode) - return sgv0, sgv1 - - -def _reroute_sgv_outputs(sgv0, sgv1, mode): - """Re-route all the outputs of two operations. - - Args: - sgv0: the first subgraph to have its outputs swapped. This argument is - converted to a subgraph using the same rules than the function - subgraph.make_view. - sgv1: the second subgraph to have its outputs swapped. This argument is - converted to a subgraph using the same rules than the function - subgraph.make_view. - mode: reroute mode, see _reroute_ts(...). - Returns: - A tuple `(sgv0, sgv1)` of subgraph views with their outputs swapped. - Note that the function argument sgv0 and sgv1 are also modified in place. - Raises: - StandardError: if sgv0 or sgv1 cannot be converted to a SubGraphView using - the same rules than the function subgraph.make_view. - """ - sgv0 = _subgraph.make_view(sgv0) - sgv1 = _subgraph.make_view(sgv1) - _util.check_graphs(sgv0, sgv1) - cannot_modify = sgv0.ops + sgv1.ops - _reroute_ts(sgv0.outputs, sgv1.outputs, mode, cannot_modify=cannot_modify) - return sgv0, sgv1 - - -def _reroute_sgv(sgv0, sgv1, mode): - """Re-route both the inputs and the outputs of the two subgraph views. - - This involves swapping all the inputs/outputs of the two subgraph views. - - Args: - sgv0: the first subgraph to be swapped. This argument is converted to a - subgraph using the same rules than the function subgraph.make_view. - sgv1: the second subgraph to be swapped. This argument is converted to a - subgraph using the same rules than the function subgraph.make_view. - mode: reroute mode, see _reroute_ts(...). - Returns: - A tuple `(sgv0, sgv1)` of subgraph views with their outputs and inputs - swapped. - Note that the function argument sgv0 and sgv1 are also modified in place. - Raises: - StandardError: if sgv0 or sgv1 cannot be converted to a SubGraphView using - the same rules than the function subgraph.make_view. - """ - _reroute_sgv_outputs(sgv0, sgv1, mode) - _reroute_sgv_inputs(sgv0, sgv1, mode) - return sgv0, sgv1 - - -def swap_inputs(sgv0, sgv1): - """Swap all the inputs of sgv0 and sgv1 (see reroute_inputs).""" - return _reroute_sgv_inputs(sgv0, sgv1, _RerouteMode.swap) - - -def reroute_inputs(sgv0, sgv1): - """Re-route all the inputs of two subgraphs. - - Args: - sgv0: the first subgraph to have its inputs swapped. This argument is - converted to a subgraph using the same rules than the function - subgraph.make_view. - sgv1: the second subgraph to have its inputs swapped. This argument is - converted to a subgraph using the same rules than the function - subgraph.make_view. - Returns: - A tuple `(sgv0, sgv1)` of subgraph views with their inputs swapped. - Note that the function argument sgv0 and sgv1 are also modified in place. - Raises: - StandardError: if sgv0 or sgv1 cannot be converted to a SubGraphView using - the same rules than the function subgraph.make_view. - """ - return _reroute_sgv_inputs(sgv0, sgv1, _RerouteMode.a2b) - - -def swap_outputs(sgv0, sgv1): - """Swap all the outputs of sgv0 and sgv1 (see reroute_outputs).""" - return _reroute_sgv_outputs(sgv0, sgv1, _RerouteMode.swap) - - -def reroute_outputs(sgv0, sgv1): - """Re-route all the outputs of two operations. - - Args: - sgv0: the first subgraph to have its outputs swapped. This argument is - converted to a subgraph using the same rules than the function - subgraph.make_view. - sgv1: the second subgraph to have its outputs swapped. This argument is - converted to a subgraph using the same rules than the function - subgraph.make_view. - Returns: - A tuple `(sgv0, sgv1)` of subgraph views with their outputs swapped. - Note that the function argument sgv0 and sgv1 are also modified in place. - Raises: - StandardError: if sgv0 or sgv1 cannot be converted to a SubGraphView using - the same rules than the function subgraph.make_view. - """ - return _reroute_sgv_outputs(sgv0, sgv1, _RerouteMode.a2b) - - -def swap_ios(sgv0, sgv1): - """Swap the inputs and outputs of sgv1 to sgv0 (see _reroute_sgv).""" - return _reroute_sgv(sgv0, sgv1, _RerouteMode.swap) - - -def reroute_ios(sgv0, sgv1): - """Re-route the inputs and outputs of sgv0 to sgv1 (see _reroute_sgv).""" - return _reroute_sgv(sgv0, sgv1, _RerouteMode.a2b) - - -def remove_control_inputs(op, cops): - """Remove the control inputs cops from co. - - Warning: this function is directly manipulating the internals of the - `tf.Graph`. - - Args: - op: a `tf.Operation` from which to remove the control inputs. - cops: an object convertible to a list of `tf.Operation`. - Raises: - TypeError: if op is not a `tf.Operation`. - ValueError: if any cop in cops is not a control input of op. - """ - if not isinstance(op, _tf_ops.Operation): - raise TypeError("Expected a tf.Operation, got: {}", type(op)) - cops = _util.make_list_of_op(cops, allow_graph=False) - for cop in cops: - if cop not in op.control_inputs: - raise ValueError("{} is not a control_input of {}".format(op.name, - cop.name)) - control_inputs = [cop for cop in op.control_inputs if cop not in cops] - # pylint: disable=protected-access - op._remove_all_control_inputs() - op._add_control_inputs(control_inputs) - # pylint: enable=protected-access - - -def add_control_inputs(op, cops): - """Add the control inputs cops to op. - - Warning: this function is directly manipulating the internals of the tf.Graph. - - Args: - op: a tf.Operation to which the control inputs are added. - cops: an object convertible to a list of `tf.Operation`. - Raises: - TypeError: if op is not a tf.Operation - ValueError: if any cop in cops is already a control input of op. - """ - if not isinstance(op, _tf_ops.Operation): - raise TypeError("Expected a tf.Operation, got: {}", type(op)) - cops = _util.make_list_of_op(cops, allow_graph=False) - for cop in cops: - if cop in op.control_inputs: - raise ValueError("{} is already a control_input of {}".format(cop.name, - op.name)) - op._add_control_inputs(cops) # pylint: disable=protected-access - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/graph_editor/select.py b/tensorflow/contrib/graph_editor/select.py deleted file mode 100644 index 3f7bc91ef05..00000000000 --- a/tensorflow/contrib/graph_editor/select.py +++ /dev/null @@ -1,773 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Various ways of selecting operations and tensors in a graph.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re - -from six import iteritems -from six import string_types - -from tensorflow.contrib.graph_editor import util -from tensorflow.python.ops import op_selector -from tensorflow.python.framework import ops as tf_ops -from tensorflow.python.util import deprecation - - - -__all__ = [ - "can_be_regex", - "make_regex", - "filter_ts", - "filter_ts_from_regex", - "filter_ops", - "filter_ops_from_regex", - "get_name_scope_ops", - "check_cios", - "get_ops_ios", - "compute_boundary_ts", - "get_within_boundary_ops", - "get_forward_walk_ops", - "get_backward_walk_ops", - "get_walks_intersection_ops", - "get_walks_union_ops", - "select_ops", - "select_ts", - "select_ops_and_ts", -] - -_RE_TYPE = type(re.compile("")) - - -def can_be_regex(obj): - """Return True if obj can be turned into a regular expression.""" - return isinstance(obj, string_types + (_RE_TYPE,)) - - -def make_regex(obj): - """Return a compiled regular expression. - - Args: - obj: a string or a regular expression. - Returns: - A compiled regular expression. - Raises: - ValueError: if obj could not be converted to a regular expression. - """ - if not can_be_regex(obj): - raise ValueError("Expected a string or a regex, got: {}".format(type(obj))) - - if isinstance(obj, string_types): - return re.compile(obj) - else: - return obj - - -def _get_input_ts(ops): - """Compute the list of unique input tensors of all the op in ops. - - Args: - ops: an object convertible to a list of `tf.Operation`. - Returns: - The list of unique input tensors of all the op in ops. - Raises: - TypeError: if ops cannot be converted to a list of `tf.Operation`. - """ - ops = util.make_list_of_op(ops) - ts = [] - ts_set = set() - for op in ops: - for t in op.inputs: - if t not in ts_set: - ts.append(t) - ts_set.add(t) - return ts - - -def _get_output_ts(ops): - """Compute the list of unique output tensors of all the op in ops. - - Args: - ops: an object convertible to a list of tf.Operation. - Returns: - The list of unique output tensors of all the op in ops. - Raises: - TypeError: if ops cannot be converted to a list of tf.Operation. - """ - ops = util.make_list_of_op(ops) - ts = [] - for op in ops: - ts += op.outputs - return ts - - -def filter_ts(ops, positive_filter): - """Get all the tensors which are input or output of an op in ops. - - Args: - ops: an object convertible to a list of `tf.Operation`. - positive_filter: a function deciding whether to keep a tensor or not. - If `True`, all the tensors are returned. - Returns: - A list of `tf.Tensor`. - Raises: - TypeError: if ops cannot be converted to a list of `tf.Operation`. - """ - ops = util.make_list_of_op(ops) - ts = _get_input_ts(ops) - util.concatenate_unique(ts, _get_output_ts(ops)) - if positive_filter is not True: - ts = [t for t in ts if positive_filter(t)] - return ts - - -def filter_ts_from_regex(ops, regex): - r"""Get all the tensors linked to ops that match the given regex. - - Args: - ops: an object convertible to a list of tf.Operation. - regex: a regular expression matching the tensors' name. - For example, "^foo(/.*)?:\d+$" will match all the tensors in the "foo" - scope. - Returns: - A list of tf.Tensor. - Raises: - TypeError: if ops cannot be converted to a list of tf.Operation. - """ - ops = util.make_list_of_op(ops) - regex_obj = make_regex(regex) - return filter_ts(ops, positive_filter=lambda op: regex_obj.search(op.name)) - - -def filter_ops(ops, positive_filter): - """Get the ops passing the given filter. - - Args: - ops: an object convertible to a list of tf.Operation. - positive_filter: a function deciding where to keep an operation or not. - If True, all the operations are returned. - Returns: - A list of selected tf.Operation. - Raises: - TypeError: if ops cannot be converted to a list of tf.Operation. - """ - ops = util.make_list_of_op(ops) - if positive_filter is not True: # pylint: disable=g-explicit-bool-comparison - ops = [op for op in ops if positive_filter(op)] - return ops - - -def filter_ops_from_regex(ops, regex): - """Get all the operations that match the given regex. - - Args: - ops: an object convertible to a list of `tf.Operation`. - regex: a regular expression matching the operation's name. - For example, `"^foo(/.*)?$"` will match all the operations in the "foo" - scope. - Returns: - A list of `tf.Operation`. - Raises: - TypeError: if ops cannot be converted to a list of `tf.Operation`. - """ - ops = util.make_list_of_op(ops) - regex_obj = make_regex(regex) - return filter_ops(ops, lambda op: regex_obj.search(op.name)) - - -def get_name_scope_ops(ops, scope): - """Get all the operations under the given scope path. - - Args: - ops: an object convertible to a list of tf.Operation. - scope: a scope path. - Returns: - A list of tf.Operation. - Raises: - TypeError: if ops cannot be converted to a list of tf.Operation. - """ - if scope and scope[-1] == "/": - scope = scope[:-1] - return filter_ops_from_regex(ops, "^{}(/.*)?$".format(scope)) - - -def check_cios(control_inputs=False, control_outputs=None, control_ios=None): - """Do various check on control_inputs and control_outputs. - - Args: - control_inputs: A boolean indicating whether control inputs are enabled. - control_outputs: An instance of util.ControlOutputs or None. If not None, - control outputs are enabled. - control_ios: An instance of util.ControlOutputs or None. If not None, both - control inputs and control outputs are enabled. This is equivalent to set - control_inputs to True and control_outputs to the util.ControlOutputs - instance. - Returns: - A tuple `(control_inputs, control_outputs)` where: - `control_inputs` is a boolean indicating whether to use control inputs. - `control_outputs` is an instance of util.ControlOutputs or None - Raises: - ValueError: if control_inputs is an instance of util.ControlOutputs but - control_outputs is not None - TypeError: if control_outputs is not None and is not a util.ControlOutputs. - """ - if control_ios is not None: - if not isinstance(control_ios, util.ControlOutputs): - raise TypeError("Expected a util.ControlOutputs, got: {}".format( - type(control_ios))) - if control_outputs is not None: - raise ValueError("control_outputs should be None when using control_ios.") - control_inputs = True - control_outputs = control_ios - elif control_outputs is not None: - if not isinstance(control_outputs, util.ControlOutputs): - raise TypeError("Expected a util.ControlOutputs, got: {}".format( - type(control_outputs))) - - if control_outputs is not None: - control_outputs.update() - return control_inputs, control_outputs - - -def get_ops_ios(ops, control_inputs=False, control_outputs=None, - control_ios=None): - """Return all the `tf.Operation` which are connected to an op in ops. - - Args: - ops: an object convertible to a list of `tf.Operation`. - control_inputs: A boolean indicating whether control inputs are enabled. - control_outputs: An instance of `util.ControlOutputs` or `None`. If not - `None`, control outputs are enabled. - control_ios: An instance of `util.ControlOutputs` or `None`. If not `None`, - both control inputs and control outputs are enabled. This is equivalent to - set `control_inputs` to `True` and `control_outputs` to the - `util.ControlOutputs` instance. - Returns: - All the `tf.Operation` surrounding the given ops. - Raises: - TypeError: if `ops` cannot be converted to a list of `tf.Operation`. - """ - control_inputs, control_outputs = check_cios(control_inputs, control_outputs, - control_ios) - ops = util.make_list_of_op(ops) - res = [] - for op in ops: - util.concatenate_unique(res, [t.op for t in op.inputs]) - for t in op.outputs: - util.concatenate_unique(res, t.consumers()) - if control_outputs is not None: - util.concatenate_unique(res, control_outputs.get(op)) - if control_inputs: - util.concatenate_unique(res, op.control_inputs) - return res - - -def compute_boundary_ts(ops): - """Compute the tensors at the boundary of a set of ops. - - This function looks at all the tensors connected to the given ops (in/out) - and classify them into three categories: - 1) input tensors: tensors whose generating operation is not in ops. - 2) output tensors: tensors whose consumer operations are not in ops - 3) inside tensors: tensors which are neither input nor output tensors. - - Note that a tensor can be both an inside tensor and an output tensor if it is - consumed by operations both outside and inside of `ops`. - - Args: - ops: an object convertible to a list of tf.Operation. - Returns: - A tuple `(outside_input_ts, outside_output_ts, inside_ts)` where: - `outside_input_ts` is a Python list of input tensors; - `outside_output_ts` is a python list of output tensors; - `inside_ts` is a python list of inside tensors. - Since a tensor can be both an inside tensor and an output tensor, - `outside_output_ts` and `inside_ts` might intersect. - Raises: - TypeError: if ops cannot be converted to a list of tf.Operation. - """ - ops = util.make_list_of_op(ops) - input_ts = _get_input_ts(ops) - output_ts = _get_output_ts(ops) - output_ts_set = frozenset(output_ts) - ops_set = frozenset(ops) - - # Compute inside tensors. - inside_ts = [] - only_inside_ts = [] - for t in input_ts: - # Skip if the input tensor is not also an output tensor. - if t not in output_ts_set: - continue - # Mark as "inside". - inside_ts.append(t) - # Mark as "only inside" if the tensor is not both inside and output. - consumers = frozenset(t.consumers()) - if consumers - ops_set: - continue - only_inside_ts.append(t) - - inside_ts_set = frozenset(inside_ts) - only_inside_ts_set = frozenset(only_inside_ts) - outside_output_ts = [t for t in output_ts if t not in only_inside_ts_set] - outside_input_ts = [t for t in input_ts if t not in inside_ts_set] - return outside_input_ts, outside_output_ts, inside_ts - - -def get_within_boundary_ops(ops, - seed_ops, - boundary_ops=(), - inclusive=True, - control_inputs=False, - control_outputs=None, - control_ios=None): - """Return all the `tf.Operation` within the given boundary. - - Args: - ops: an object convertible to a list of `tf.Operation`. those ops define the - set in which to perform the operation (if a `tf.Graph` is given, it - will be converted to the list of all its operations). - seed_ops: the operations from which to start expanding. - boundary_ops: the ops forming the boundary. - inclusive: if `True`, the result will also include the boundary ops. - control_inputs: A boolean indicating whether control inputs are enabled. - control_outputs: An instance of `util.ControlOutputs` or `None`. If not - `None`, control outputs are enabled. - control_ios: An instance of `util.ControlOutputs` or `None`. If not - `None`, both control inputs and control outputs are enabled. This is - equivalent to set control_inputs to True and control_outputs to - the `util.ControlOutputs` instance. - Returns: - All the `tf.Operation` surrounding the given ops. - Raises: - TypeError: if `ops` or `seed_ops` cannot be converted to a list of - `tf.Operation`. - ValueError: if the boundary is intersecting with the seeds. - """ - control_inputs, control_outputs = check_cios(control_inputs, control_outputs, - control_ios) - ops = util.make_list_of_op(ops) - seed_ops = util.make_list_of_op(seed_ops, allow_graph=False) - boundary_ops = set(util.make_list_of_op(boundary_ops)) - res = set(seed_ops) - if boundary_ops & res: - raise ValueError("Boundary is intersecting with the seeds.") - wave = set(seed_ops) - while wave: - new_wave = set() - ops_io = get_ops_ios(wave, control_inputs, control_outputs) - for op in ops_io: - if op in res: - continue - if op in boundary_ops: - if inclusive: - res.add(op) - else: - new_wave.add(op) - res.update(new_wave) - wave = new_wave - return [op for op in ops if op in res] - - -def get_forward_walk_ops(seed_ops, - inclusive=True, - within_ops=None, - within_ops_fn=None, - stop_at_ts=(), - control_outputs=None): - """Do a forward graph walk and return all the visited ops. - - Args: - seed_ops: an iterable of operations from which the forward graph - walk starts. If a list of tensors is given instead, the seed_ops are set - to be the consumers of those tensors. - inclusive: if True the given seed_ops are also part of the resulting set. - within_ops: an iterable of `tf.Operation` within which the search is - restricted. If `within_ops` is `None`, the search is performed within - the whole graph. - within_ops_fn: if provided, a function on ops that should return True iff - the op is within the graph traversal. This can be used along within_ops, - in which case an op is within if it is also in within_ops. - stop_at_ts: an iterable of tensors at which the graph walk stops. - control_outputs: a `util.ControlOutputs` instance or None. - If not `None`, it will be used while walking the graph forward. - Returns: - A Python set of all the `tf.Operation` ahead of `seed_ops`. - Raises: - TypeError: if `seed_ops` or `within_ops` cannot be converted to a list of - `tf.Operation`. - """ - _, control_outputs = check_cios(False, control_outputs) - if not util.is_iterable(seed_ops): - seed_ops = [seed_ops] - if not seed_ops: - return [] - if isinstance(seed_ops[0], tf_ops.Tensor): - ts = util.make_list_of_t(seed_ops, allow_graph=False) - seed_ops = util.get_consuming_ops(ts) - else: - seed_ops = util.make_list_of_op(seed_ops, allow_graph=False) - - seed_ops = frozenset(seed_ops) - stop_at_ts = frozenset(util.make_list_of_t(stop_at_ts)) - if within_ops: - within_ops = util.make_list_of_op(within_ops, allow_graph=False) - within_ops = frozenset(within_ops) - seed_ops &= within_ops - - def is_within(op): - return (within_ops is None or op in within_ops) and ( - within_ops_fn is None or within_ops_fn(op)) - - result = list(seed_ops) - wave = set(seed_ops) - while wave: - new_wave = set() - for op in wave: - for new_t in op.outputs: - if new_t in stop_at_ts: - continue - for new_op in new_t.consumers(): - if new_op not in result and is_within(new_op): - new_wave.add(new_op) - if control_outputs is not None: - for new_op in control_outputs.get(op): - if new_op not in result and is_within(new_op): - new_wave.add(new_op) - util.concatenate_unique(result, new_wave) - wave = new_wave - if not inclusive: - result = [op for op in result if op not in seed_ops] - return result - - -@deprecation.deprecated( - "2019-06-06", - "Please use tensorflow.python.ops.op_selector.get_backward_walk_ops.", - warn_once=True) -def get_backward_walk_ops(seed_ops, - inclusive=True, - within_ops=None, - within_ops_fn=None, - stop_at_ts=(), - control_inputs=False): - """Do a backward graph walk and return all the visited ops. - - Args: - seed_ops: an iterable of operations from which the backward graph - walk starts. If a list of tensors is given instead, the seed_ops are set - to be the generators of those tensors. - inclusive: if True the given seed_ops are also part of the resulting set. - within_ops: an iterable of `tf.Operation` within which the search is - restricted. If `within_ops` is `None`, the search is performed within - the whole graph. - within_ops_fn: if provided, a function on ops that should return True iff - the op is within the graph traversal. This can be used along within_ops, - in which case an op is within if it is also in within_ops. - stop_at_ts: an iterable of tensors at which the graph walk stops. - control_inputs: if True, control inputs will be used while moving backward. - Returns: - A Python set of all the `tf.Operation` behind `seed_ops`. - Raises: - TypeError: if `seed_ops` or `within_ops` cannot be converted to a list of - `tf.Operation`. - """ - return op_selector.get_backward_walk_ops( - seed_ops, - inclusive=inclusive, - within_ops=within_ops, - within_ops_fn=within_ops_fn, - stop_at_ts=stop_at_ts, - control_inputs=control_inputs) - - -def get_walks_intersection_ops(forward_seed_ops, - backward_seed_ops, - forward_inclusive=True, - backward_inclusive=True, - within_ops=None, - within_ops_fn=None, - control_inputs=False, - control_outputs=None, - control_ios=None): - """Return the intersection of a forward and a backward walk. - - Args: - forward_seed_ops: an iterable of operations from which the forward graph - walk starts. If a list of tensors is given instead, the seed_ops are set - to be the consumers of those tensors. - backward_seed_ops: an iterable of operations from which the backward graph - walk starts. If a list of tensors is given instead, the seed_ops are set - to be the generators of those tensors. - forward_inclusive: if True the given forward_seed_ops are also part of the - resulting set. - backward_inclusive: if True the given backward_seed_ops are also part of the - resulting set. - within_ops: an iterable of tf.Operation within which the search is - restricted. If within_ops is None, the search is performed within - the whole graph. - within_ops_fn: if provided, a function on ops that should return True iff - the op is within the graph traversal. This can be used along within_ops, - in which case an op is within if it is also in within_ops. - control_inputs: A boolean indicating whether control inputs are enabled. - control_outputs: An instance of util.ControlOutputs or None. If not None, - control outputs are enabled. - control_ios: An instance of util.ControlOutputs or None. If not None, both - control inputs and control outputs are enabled. This is equivalent to set - control_inputs to True and control_outputs to the util.ControlOutputs - instance. - Returns: - A Python set of all the tf.Operation in the intersection of a forward and a - backward walk. - Raises: - TypeError: if `forward_seed_ops` or `backward_seed_ops` or `within_ops` - cannot be converted to a list of `tf.Operation`. - """ - control_inputs, control_outputs = check_cios(control_inputs, control_outputs, - control_ios) - forward_ops = get_forward_walk_ops( - forward_seed_ops, - inclusive=forward_inclusive, - within_ops=within_ops, - within_ops_fn=within_ops_fn, - control_outputs=control_outputs) - backward_ops = get_backward_walk_ops( - backward_seed_ops, - inclusive=backward_inclusive, - within_ops=within_ops, - within_ops_fn=within_ops_fn, - control_inputs=control_inputs) - return [op for op in forward_ops if op in backward_ops] - - -def get_walks_union_ops(forward_seed_ops, - backward_seed_ops, - forward_inclusive=True, - backward_inclusive=True, - within_ops=None, - within_ops_fn=None, - control_inputs=False, - control_outputs=None, - control_ios=None): - """Return the union of a forward and a backward walk. - - Args: - forward_seed_ops: an iterable of operations from which the forward graph - walk starts. If a list of tensors is given instead, the seed_ops are set - to be the consumers of those tensors. - backward_seed_ops: an iterable of operations from which the backward graph - walk starts. If a list of tensors is given instead, the seed_ops are set - to be the generators of those tensors. - forward_inclusive: if True the given forward_seed_ops are also part of the - resulting set. - backward_inclusive: if True the given backward_seed_ops are also part of the - resulting set. - within_ops: restrict the search within those operations. If within_ops is - None, the search is done within the whole graph. - within_ops_fn: if provided, a function on ops that should return True iff - the op is within the graph traversal. This can be used along within_ops, - in which case an op is within if it is also in within_ops. - control_inputs: A boolean indicating whether control inputs are enabled. - control_outputs: An instance of util.ControlOutputs or None. If not None, - control outputs are enabled. - control_ios: An instance of util.ControlOutputs or None. If not None, both - control inputs and control outputs are enabled. This is equivalent to set - control_inputs to True and control_outputs to the util.ControlOutputs - instance. - Returns: - A Python set of all the tf.Operation in the union of a forward and a - backward walk. - Raises: - TypeError: if forward_seed_ops or backward_seed_ops or within_ops cannot be - converted to a list of tf.Operation. - """ - control_inputs, control_outputs = check_cios(control_inputs, control_outputs, - control_ios) - forward_ops = get_forward_walk_ops( - forward_seed_ops, - inclusive=forward_inclusive, - within_ops=within_ops, - within_ops_fn=within_ops_fn, - control_outputs=control_outputs) - backward_ops = get_backward_walk_ops( - backward_seed_ops, - inclusive=backward_inclusive, - within_ops=within_ops, - within_ops_fn=within_ops_fn, - control_inputs=control_inputs) - return util.concatenate_unique(forward_ops, backward_ops) - - -def select_ops(*args, **kwargs): - """Helper to select operations. - - Args: - *args: list of 1) regular expressions (compiled or not) or 2) (array of) - `tf.Operation`. `tf.Tensor` instances are silently ignored. - **kwargs: 'graph': `tf.Graph` in which to perform the regex query.This is - required when using regex. - 'positive_filter': an elem if selected only if `positive_filter(elem)` is - `True`. This is optional. - 'restrict_ops_regex': a regular expression is ignored if it doesn't start - with the substring "(?#ops)". - Returns: - A list of `tf.Operation`. - Raises: - TypeError: if the optional keyword argument graph is not a `tf.Graph` - or if an argument in args is not an (array of) `tf.Operation` - or an (array of) `tf.Tensor` (silently ignored) or a string - or a regular expression. - ValueError: if one of the keyword arguments is unexpected or if a regular - expression is used without passing a graph as a keyword argument. - """ - # get keywords arguments - graph = None - positive_filter = None - restrict_ops_regex = False - for k, v in iteritems(kwargs): - if k == "graph": - graph = v - if graph is not None and not isinstance(graph, tf_ops.Graph): - raise TypeError("Expected a tf.Graph, got: {}".format(type(graph))) - elif k == "positive_filter": - positive_filter = v - elif k == "restrict_ops_regex": - restrict_ops_regex = v - elif k == "restrict_ts_regex": - pass - else: - raise ValueError("Wrong keywords argument: {}.".format(k)) - - ops = [] - - for arg in args: - if can_be_regex(arg): - if graph is None: - raise ValueError("Use the keyword argument 'graph' to use regex.") - regex = make_regex(arg) - if regex.pattern.startswith("(?#ts)"): - continue - if restrict_ops_regex and not regex.pattern.startswith("(?#ops)"): - continue - ops_ = filter_ops_from_regex(graph, regex) - for op_ in ops_: - if op_ not in ops: - if positive_filter is None or positive_filter(op_): - ops.append(op_) - else: - ops_aux = util.make_list_of_op(arg, ignore_ts=True) - if positive_filter is not None: - ops_aux = [op for op in ops_aux if positive_filter(op)] - ops_aux = [op for op in ops_aux if op not in ops] - ops += ops_aux - - return ops - - -def select_ts(*args, **kwargs): - """Helper to select tensors. - - Args: - *args: list of 1) regular expressions (compiled or not) or 2) (array of) - `tf.Tensor`. `tf.Operation` instances are silently ignored. - **kwargs: 'graph': `tf.Graph` in which to perform the regex query.This is - required when using regex. - 'positive_filter': an elem if selected only if `positive_filter(elem)` is - `True`. This is optional. - 'restrict_ts_regex': a regular expression is ignored if it doesn't start - with the substring "(?#ts)". - Returns: - A list of `tf.Tensor`. - Raises: - TypeError: if the optional keyword argument graph is not a `tf.Graph` - or if an argument in args is not an (array of) `tf.Tensor` - or an (array of) `tf.Operation` (silently ignored) or a string - or a regular expression. - ValueError: if one of the keyword arguments is unexpected or if a regular - expression is used without passing a graph as a keyword argument. - """ - # get keywords arguments - graph = None - positive_filter = None - restrict_ts_regex = False - for k, v in iteritems(kwargs): - if k == "graph": - graph = v - if graph is not None and not isinstance(graph, tf_ops.Graph): - raise TypeError("Expected a tf.Graph, got {}".format(type(graph))) - elif k == "positive_filter": - positive_filter = v - elif k == "restrict_ts_regex": - restrict_ts_regex = v - elif k == "restrict_ops_regex": - pass - else: - raise ValueError("Wrong keywords argument: {}.".format(k)) - - ts = [] - - for arg in args: - if can_be_regex(arg): - if graph is None: - raise ValueError("Use the keyword argument 'graph' to use regex.") - regex = make_regex(arg) - if regex.pattern.startswith("(?#ops)"): - continue - if restrict_ts_regex and not regex.pattern.startswith("(?#ts)"): - continue - ts_ = filter_ts_from_regex(graph, regex) - for t_ in ts_: - if t_ not in ts: - if positive_filter is None or positive_filter(t_): - ts.append(t_) - else: - ts_aux = util.make_list_of_t(arg, ignore_ops=True) - if positive_filter is not None: - ts_aux = [t for t in ts_aux if positive_filter(t)] - ts_aux = [t for t in ts_aux if t not in ts] - ts += ts_aux - - return ts - - -def select_ops_and_ts(*args, **kwargs): - """Helper to select operations and tensors. - - Args: - *args: list of 1) regular expressions (compiled or not) or 2) (array of) - `tf.Operation` 3) (array of) tf.Tensor. Regular expressions matching - tensors must start with the comment `"(?#ts)"`, for instance: - `"(?#ts)^foo/.*"`. - **kwargs: 'graph': `tf.Graph` in which to perform the regex query.This is - required when using regex. - 'positive_filter': an elem if selected only if `positive_filter(elem)` is - `True`. This is optional. - Returns: - A tuple `(ops, ts)` where: - `ops` is a list of `tf.Operation`, and - `ts` is a list of `tf.Tensor` - Raises: - TypeError: if the optional keyword argument graph is not a `tf.Graph` - or if an argument in args is not an (array of) `tf.Tensor` - or an (array of) `tf.Operation` or a string or a regular expression. - ValueError: if one of the keyword arguments is unexpected or if a regular - expression is used without passing a graph as a keyword argument. - """ - ops = select_ops(*args, restrict_ops_regex=False, **kwargs) - ts = select_ts(*args, restrict_ts_regex=True, **kwargs) - return ops, ts diff --git a/tensorflow/contrib/graph_editor/subgraph.py b/tensorflow/contrib/graph_editor/subgraph.py deleted file mode 100644 index 6650e996d71..00000000000 --- a/tensorflow/contrib/graph_editor/subgraph.py +++ /dev/null @@ -1,668 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""SubGraphView: a subgraph view on an existing tf.Graph. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import copy - -import six -from six import iteritems -from six import StringIO - -from tensorflow.contrib.graph_editor import select -from tensorflow.contrib.graph_editor import util -from tensorflow.python.framework import ops as tf_ops - -__all__ = [ - "SubGraphView", - "make_view", - "make_view_from_scope", -] - - -def _finalize_index(index_or_t, ts): - """Returns index as is or return index of tensor in `ts`.""" - if isinstance(index_or_t, six.integer_types): - return index_or_t - else: - return ts.index(index_or_t) - - -def _finalize_indices(list_of_index_or_t, ts): - """Returns index in `indices` as is or replace with tensor's index.""" - return [_finalize_index(index_or_t, ts) for index_or_t in list_of_index_or_t] - - -def _check_within_range(mapping, n, repetition): - """Check is the mapping is valid. - - Args: - mapping: an iterable of integer. - n: define the input domain as [0, n-1]. Note that the mapping can be - under-complete, that is, it can only contain a subset of the integers on - [0, n-1]. - repetition: if True repetition are allowed (the function is surjective) - otherwise repetition are not allowed (the function is injective). - Raises: - ValueError: if the mapping is out of range ot if repetition is False and - the mapping has some repetition. - """ - for i in mapping: - if not 0 <= i < n: - raise ValueError("Out of [0, {}[ range: {}".format(n, i)) - if not repetition and len(set(mapping)) != len(mapping): - raise ValueError("Found repetition in mapping: {}".format(mapping)) - - -class SubGraphView(object): - """A subgraph view on an existing `tf.Graph`. - - An instance of this class is a subgraph view on an existing `tf.Graph`. - "subgraph" means that it can represent part of the whole `tf.Graph`. - "view" means that it only provides a passive observation and do not to act - on the `tf.Graph`. Note that in this documentation, the term "subgraph" is - often used as substitute to "subgraph view". - - A subgraph contains: - - * a list of input tensors, accessible via the `inputs` property. - * a list of output tensors, accessible via the `outputs` property. - * and the operations in between, accessible via the "ops" property. - - An subgraph can be seen as a function F(i0, i1, ...) -> o0, o1, ... It is a - function which takes as input some input tensors and returns as output some - output tensors. The computation that the function performs is encoded in the - operations of the subgraph. - - The tensors (input or output) can be of two kinds: - - - connected: a connected tensor connects to at least one operation contained - in the subgraph. One example is a subgraph representing a single operation - and its inputs and outputs: all the input and output tensors of the op - are "connected". - - passthrough: a passthrough tensor does not connect to any operation - contained in the subgraph. One example is a subgraph representing a - single tensor: this tensor is passthrough. By default a passthrough tensor is - present both in the input and output tensors of the subgraph. It can however - be remapped to only appear as an input (or output) only. - - The input and output tensors can be remapped. For instance, some input tensor - can be omitted. For instance, a subgraph representing an operation with two - inputs can be remapped to only take one input. Note that this does not change - at all the underlying `tf.Graph` (remember, it is a view). It means that - the other input is being ignored, or is being treated as "given". - The analogy with functions can be extended like this: F(x,y) is the original - function. Remapping the inputs from [x, y] to just [x] means that the subgraph - now represent the function F_y(x) (y is "given"). - - The output tensors can also be remapped. For instance, some output tensor can - be omitted. Other output tensor can be duplicated as well. As mentioned - before, this does not change at all the underlying `tf.Graph`. - The analogy with functions can be extended like this: F(...)->x,y is the - original function. Remapping the outputs from [x, y] to just [y,y] means that - the subgraph now represent the function M(F(...)) where M is the function - M(a,b)->b,b. - - It is useful to describe three other kind of tensors: - - * internal: an internal tensor is a tensor connecting operations contained - in the subgraph. One example in the subgraph representing the two - operations A and B connected sequentially: -> A -> B ->. The middle arrow - is an internal tensor. - * actual input: an input tensor of the subgraph, regardless of whether it is - listed in "inputs" or not (masked-out). - * actual output: an output tensor of the subgraph, regardless of whether it is - listed in "outputs" or not (masked-out). - * hidden input: an actual input which has been masked-out using an - input remapping. In other word, a hidden input is a non-internal tensor - not listed as a input tensor and one of whose consumers belongs to - the subgraph. - * hidden output: a actual output which has been masked-out using an output - remapping. In other word, a hidden output is a non-internal tensor - not listed as an output and one of whose generating operations belongs to - the subgraph. - - Here are some useful guarantees about an instance of a SubGraphView: - - * the input (or output) tensors are not internal. - * the input (or output) tensors are either "connected" or "passthrough". - * the passthrough tensors are not connected to any of the operation of - the subgraph. - - Note that there is no guarantee that an operation in a subgraph contributes - at all to its inputs or outputs. For instance, remapping both the inputs and - outputs to empty lists will produce a subgraph which still contains all the - original operations. However, the remove_unused_ops function can be used to - make a new subgraph view whose operations are connected to at least one of - the input or output tensors. - - An instance of this class is meant to be a lightweight object which is not - modified in-place by the user. Rather, the user can create new modified - instances of a given subgraph. In that sense, the class SubGraphView is meant - to be used like an immutable python object. - - A common problem when using views is that they can get out-of-sync with the - data they observe (in this case, a `tf.Graph`). This is up to the user to - ensure that this doesn't happen. To keep on the safe side, it is recommended - that the life time of subgraph views are kept very short. One way to achieve - this is to use subgraphs within a "with make_sgv(...) as sgv:" Python context. - - To alleviate the out-of-sync problem, some functions are granted the right to - modified subgraph in place. This is typically the case of graph manipulation - functions which, given some subgraphs as arguments, can modify the underlying - `tf.Graph`. Since this modification is likely to render the subgraph view - invalid, those functions can modify the argument in place to reflect the - change. For instance, calling the function swap_inputs(svg0, svg1) will modify - svg0 and svg1 in place to reflect the fact that their inputs have now being - swapped. - """ - - def __init__(self, inside_ops=(), passthrough_ts=()): - """Create a subgraph containing the given ops and the "passthrough" tensors. - - Args: - inside_ops: an object convertible to a list of `tf.Operation`. This list - defines all the operations in the subgraph. - passthrough_ts: an object convertible to a list of `tf.Tensor`. This list - define all the "passthrough" tensors. A passthrough tensor is a tensor - which goes directly from the input of the subgraph to it output, without - any intermediate operations. All the non passthrough tensors are - silently ignored. - Raises: - TypeError: if inside_ops cannot be converted to a list of `tf.Operation` - or if `passthrough_ts` cannot be converted to a list of `tf.Tensor`. - """ - - inside_ops = util.make_list_of_op(inside_ops) - passthrough_ts = util.make_list_of_t(passthrough_ts) - ops_and_ts = inside_ops + passthrough_ts - if ops_and_ts: - self._graph = util.get_unique_graph(ops_and_ts) - self._ops = inside_ops - - # Compute inside and outside tensor - inputs, outputs, insides = select.compute_boundary_ts(inside_ops) - - # Compute passthrough tensors, silently ignoring the non-passthrough ones. - all_tensors = frozenset(inputs + outputs + list(insides)) - self._passthrough_ts = [t for t in passthrough_ts if t not in all_tensors] - - # Set inputs and outputs. - self._input_ts = inputs + self._passthrough_ts - self._output_ts = outputs + self._passthrough_ts - else: - self._graph = None - self._passthrough_ts = [] - self._input_ts = [] - self._output_ts = [] - self._ops = [] - - def __copy__(self): - """Create a copy of this subgraph. - - Note that this class is a "view", copying it only create another view and - does not copy the underlying part of the `tf.Graph`. - - Returns: - A new identical instance of the original subgraph view. - """ - cls = self.__class__ - result = cls.__new__(cls) - for k, v in iteritems(self.__dict__): - if k == "_graph": - setattr(result, k, v) - else: - setattr(result, k, list(v)) # copy the list - return result - - def _assign_from(self, other): - """Assign other to itself. - - Args: - other: another subgraph-view. - Returns: - A new instance identical to the original one. - Raises: - TypeError: if other is not an SubGraphView. - """ - if not isinstance(other, SubGraphView): - raise TypeError("Expected SubGraphView, got: {}".format(type(other))) - # pylint: disable=protected-access - self._graph = other._graph - self._ops = list(other._ops) - self._passthrough_ts = list(other._passthrough_ts) - self._input_ts = list(other._input_ts) - self._output_ts = list(other._output_ts) - # pylint: enable=protected-access - - def copy(self): - """Return a copy of itself. - - Note that this class is a "view", copying it only create another view and - does not copy the underlying part of the tf.Graph. - - Returns: - A new instance identical to the original one. - """ - return copy.copy(self) - - def _remap_default(self, remove_input_map=True, remove_output_map=True): - """Remap in the place the inputs and/or outputs to the default mapping. - - Args: - remove_input_map: if True the input map is reset to the default one. - remove_output_map: if True the output map is reset to the default one. - """ - if not remove_input_map and not remove_output_map: - return - - # Compute inside and outside tensor - inputs, outputs, _ = select.compute_boundary_ts(self._ops) - if remove_input_map: - self._input_ts = list(inputs) + self._passthrough_ts - if remove_output_map: - self._output_ts = list(outputs) + self._passthrough_ts - - def remap_default(self, remove_input_map=True, remove_output_map=True): - """Remap the inputs and/or outputs to the default mapping. - - Args: - remove_input_map: if True the input map is reset to the default one. - remove_output_map: if True the output map is reset to the default one. - Returns: - A new modified instance of the original subgraph view with its - input and/or output mapping reset to the default one. - """ - res = self.copy() - res._remap_default(remove_input_map, remove_output_map) # pylint: disable=protected-access - return res - - def _remap_inputs(self, new_input_indices): - """Remap the inputs of the subgraph in-place.""" - new_input_indices = _finalize_indices(new_input_indices, self._input_ts) - _check_within_range( - new_input_indices, len(self._input_ts), repetition=False) - self._input_ts = [self._input_ts[i] for i in new_input_indices] - - def _remap_outputs(self, new_output_indices): - """Remap the outputs of the subgraph in-place.""" - new_output_indices = _finalize_indices(new_output_indices, self._output_ts) - _check_within_range( - new_output_indices, len(self._output_ts), repetition=True) - self._output_ts = [self._output_ts[i] for i in new_output_indices] - - def _remap_outputs_make_unique(self): - """Remap the outputs in place so that all the tensors appears only once.""" - output_ts = list(self._output_ts) - self._output_ts = [] - util.concatenate_unique(self._output_ts, output_ts) - - def _remap_outputs_to_consumers(self): - """Remap the outputs in place to match the number of consumers.""" - self._remap_outputs_make_unique() - output_ts = list(self._output_ts) - self._output_ts = [] - for t in output_ts: - self._output_ts += [t] * len(t.consumers()) - - def remap_outputs_make_unique(self): - """Remap the outputs so that all the tensors appears only once.""" - res = copy.copy(self) - res._remap_outputs_make_unique() # pylint: disable=protected-access - return res - - def remap_outputs_to_consumers(self): - """Remap the outputs to match the number of consumers.""" - res = copy.copy(self) - res._remap_outputs_to_consumers() # pylint: disable=protected-access - return res - - def _remove_unused_ops(self, control_inputs=True): - """Remove unused ops in place. - - Args: - control_inputs: if True, control inputs are used to detect used ops. - Returns: - A new subgraph view which only contains used operations. - """ - ops = select.get_walks_union_ops( - self.connected_inputs, - self.connected_outputs, - within_ops=self._ops, - control_inputs=control_inputs) - self._ops = [op for op in self._ops if op in ops] - - def remove_unused_ops(self, control_inputs=True): - """Remove unused ops. - - Args: - control_inputs: if True, control inputs are used to detect used ops. - Returns: - A new subgraph view which only contains used operations. - """ - res = copy.copy(self) - res._remove_unused_ops(control_inputs) # pylint: disable=protected-access - return res - - def remap_inputs(self, new_input_indices): - """Remap the inputs of the subgraph. - - If the inputs of the original subgraph are [t0, t1, t2], remapping to [2,0] - will create a new instance whose inputs is [t2, t0]. - - Note that this is only modifying the view: the underlying `tf.Graph` is not - affected. - - Args: - new_input_indices: an iterable of integers or tf.Tensors - representing a mapping between the old inputs and the new ones. - Integers must be positive and smaller than the number of old inputs. - tf.Tensors must belong to the old list of inputs. - This mapping can be under-complete and must be without repetitions. - Returns: - A new modified instance of the original subgraph view with remapped - inputs. - """ - res = self.copy() - res._remap_inputs(new_input_indices) # pylint: disable=protected-access - return res - - def remap_outputs(self, new_output_indices): - """Remap the output of the subgraph. - - If the output of the original subgraph are [t0, t1, t2], remapping to - [1,1,0] will create a new instance whose outputs is [t1, t1, t0]. - - Note that this is only modifying the view: the underlying tf.Graph is not - affected. - - Args: - new_output_indices: an iterable of integers or tf.Tensors - representing a mapping between the old outputs and the new ones. - Integers must be positive and smaller than the number of old outputs. - tf.Tensors must belong to the old list of outputs. - This mapping can be under-complete and can have repetitions. - Returns: - A new modified instance of the original subgraph view with remapped - outputs. - """ - res = copy.copy(self) - res._remap_outputs(new_output_indices) # pylint: disable=protected-access - return res - - def remap(self, new_input_indices=None, new_output_indices=None): - """Remap the inputs and outputs of the subgraph. - - Note that this is only modifying the view: the underlying tf.Graph is not - affected. - - Args: - new_input_indices: an iterable of integers or tf.Tensors - representing a mapping between the old inputs and the new ones. - Integers must be positive and smaller than the number of old inputs. - tf.Tensors must belong to the old list of inputs. - This mapping can be under-complete and must be without repetitions. - new_output_indices: an iterable of integers or tf.Tensors - representing a mapping between the old outputs and the new ones. - Integers must be positive and smaller than the number of old outputs. - tf.Tensors must belong to the old list of outputs. - This mapping can be under-complete and can have repetitions. - Returns: - A new modified instance of the original subgraph view with remapped - inputs and outputs. - """ - res = copy.copy(self) - if new_input_indices is not None: - res._remap_inputs(new_input_indices) # pylint: disable=protected-access - if new_output_indices is not None: - res._remap_outputs(new_output_indices) # pylint: disable=protected-access - return res - - def find_op_by_name(self, op_name): - """Return the op named op_name. - - Args: - op_name: the name to search for - Returns: - The op named op_name. - Raises: - ValueError: if the op_name could not be found. - AssertionError: if the name was found multiple time. - """ - res = [op for op in self._ops if op.name == op_name] - if not res: - raise ValueError("{} not in subgraph.".format(op_name)) - if len(res) > 1: - raise AssertionError("More than 1 op named: {}!".format(op_name)) - return res[0] - - def __str__(self): - if not self: - return "SubGraphView: empty" - - def op_name(op): - return op.name - - def tensor_name(t): - if t in self._passthrough_ts: - return "{} *".format(t.name) - else: - return t.name - - def print_list(name, iterable, get_name): - if iterable: - print("** {}[{}]:".format(name, len(iterable)), file=res) - print("\n".join([" {}".format(get_name(elem)) for elem in iterable]), - file=res) - else: - print("** {}: empty".format(name), file=res) - - res = StringIO() - print("SubGraphView (graphid={}):".format(id(self.graph)), file=res) - print_list("ops", self._ops, op_name) - print_list("inputs", self._input_ts, tensor_name) - print_list("outputs", self._output_ts, tensor_name) - return res.getvalue() - - @property - def graph(self): - """The underlying `tf.Graph`.""" - return self._graph - - @property - def ops(self): - """The operations in this subgraph view.""" - return self._ops - - @property - def inputs(self): - """The input tensors of this subgraph view.""" - return util.ListView(self._input_ts) - - @property - def connected_inputs(self): - """The connected input tensors of this subgraph view.""" - return [t for t in self._input_ts if t not in self._passthrough_ts] - - @property - def outputs(self): - """The output tensors of this subgraph view.""" - return util.ListView(self._output_ts) - - @property - def connected_outputs(self): - """The connected output tensors of this subgraph view.""" - return [t for t in self._output_ts if t not in self._passthrough_ts] - - @property - def passthroughs(self): - """The passthrough tensors, going straight from input to output.""" - return util.ListView(self._passthrough_ts) - - def __bool__(self): - """Allows for implicit boolean conversion.""" - return self._graph is not None - - # Python 3 wants __bool__, Python 2.7 wants __nonzero__ - __nonzero__ = __bool__ - - def op(self, op_id): - """Get an op by its index.""" - return self._ops[op_id] - - def is_passthrough(self, t): - """Check whether a tensor is passthrough.""" - return t in self._passthrough_ts - - def __enter__(self): - """Allow Python context to minimize the life time of a subgraph view. - - A subgraph view is meant to be a lightweight and transient object. A short - lifetime will alleviate the "out-of-sync" issue mentioned earlier. For that - reason, a SubGraphView instance can be used within a Python context. For - example: - - from tensorflow.contrib import graph_editor as ge - with ge.make_sgv(...) as sgv: - print(sgv) - - Returns: - Itself. - """ - return self - - def __exit__(self, exc_type, exc_value, traceback): - pass - - def input_index(self, t): - """Find the input index corresponding to the given input tensor t. - - Args: - t: the input tensor of this subgraph view. - Returns: - The index in the self.inputs list. - Raises: - Error: if t in not an input tensor. - """ - try: - subgraph_id = self._input_ts.index(t) - except: - raise ValueError("Can't find {} in inputs of subgraph {}.".format( - t.name, self.name)) - return subgraph_id - - def output_index(self, t): - """Find the output index corresponding to given output tensor t. - - Args: - t: the output tensor of this subgraph view. - Returns: - The index in the self.outputs list. - Raises: - Error: if t in not an output tensor. - """ - try: - subgraph_id = self._output_ts.index(t) - except: - raise ValueError("Can't find {} in outputs of subgraph {}.".format( - t.name, self.name)) - return subgraph_id - - def consumers(self): - """Return a Python set of all the consumers of this subgraph view. - - A consumer of a subgraph view is a tf.Operation which is a consumer - of one of the output tensors and is not in the subgraph. - - Returns: - A list of `tf.Operation` which are the consumers of this subgraph view. - """ - ops_set = frozenset(self._ops) - res = [] - for output in self._output_ts: - consumers = [op for op in output.consumers() if op not in ops_set] - util.concatenate_unique(res, consumers) - return res - - -def _check_graph(sgv, graph): - """Check if sgv belongs to the given graph. - - Args: - sgv: a SubGraphView. - graph: a graph or None. - Returns: - The SubGraphView sgv. - Raises: - TypeError: if sgv is not a SubGraphView or if graph is not None and not - a tf.Graph. - ValueError: if the graph of sgv and the given graph are not None and - different. - """ - if not isinstance(sgv, SubGraphView): - raise TypeError("Expected a SubGraphView, got: {}".format(type(graph))) - if graph is None or not sgv.graph: - return sgv - if not isinstance(graph, tf_ops.Graph): - raise TypeError("Expected a tf.Graph, got: {}".format(type(graph))) - if sgv.graph is not graph: - raise ValueError("Graph mismatch.") - return sgv - - -def make_view(*args, **kwargs): - """Create a SubGraphView from selected operations and passthrough tensors. - - Args: - *args: list of 1) regular expressions (compiled or not) or 2) (array of) - `tf.Operation` 3) (array of) `tf.Tensor`. Those objects will be converted - into a list of operations and a list of candidate for passthrough tensors. - **kwargs: keyword graph is used 1) to check that the ops and ts are from - the correct graph 2) for regular expression query - Returns: - A subgraph view. - Raises: - TypeError: if the optional keyword argument graph is not a `tf.Graph` - or if an argument in args is not an (array of) `tf.Tensor` - or an (array of) `tf.Operation` or a string or a regular expression. - ValueError: if one of the keyword arguments is unexpected. - """ - # get keywords arguments - graph = kwargs["graph"] if "graph" in kwargs else None - - # already a view? - if len(args) == 1 and isinstance(args[0], SubGraphView): - return _check_graph(args[0], graph) - - ops, ts = select.select_ops_and_ts(*args, **kwargs) - sgv = SubGraphView(ops, ts) - return _check_graph(sgv, graph) - - -def make_view_from_scope(scope, graph): - """Make a subgraph from a name scope. - - Args: - scope: the name of the scope. - graph: the `tf.Graph`. - Returns: - A subgraph view representing the given scope. - """ - ops = select.get_name_scope_ops(graph, scope) - return SubGraphView(ops) diff --git a/tensorflow/contrib/graph_editor/tests/edit_test.py b/tensorflow/contrib/graph_editor/tests/edit_test.py deleted file mode 100644 index 2f669c5d20e..00000000000 --- a/tensorflow/contrib/graph_editor/tests/edit_test.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.contrib.graph_editor.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import graph_editor as ge -from tensorflow.contrib.graph_editor.tests import match -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class EditTest(test.TestCase): - """edit module test. - - Generally the tests are in two steps: - - modify an existing graph. - - then make sure it has the expected topology using the graph matcher. - """ - - def setUp(self): - self.graph = ops.Graph() - with self.graph.as_default(): - self.a = constant_op.constant([1., 1.], shape=[2], name="a") - with ops.name_scope("foo"): - self.b = constant_op.constant([2., 2.], shape=[2], name="b") - self.c = math_ops.add(self.a, self.b, name="c") - self.d = constant_op.constant([3., 3.], shape=[2], name="d") - with ops.name_scope("bar"): - self.e = math_ops.add(self.c, self.d, name="e") - self.f = math_ops.add(self.c, self.d, name="f") - self.g = math_ops.add(self.c, self.a, name="g") - with ops.control_dependencies([self.c.op]): - self.h = math_ops.add(self.f, self.g, name="h") - - def test_detach(self): - """Test for ge.detach.""" - sgv = ge.sgv(self.c.op, self.a.op) - control_outputs = ge.ControlOutputs(self.graph) - ge.detach(sgv, control_ios=control_outputs) - # make sure the detached graph is as expected. - self.assertTrue( - match.OpMatcher("^foo/c$").input_ops("a", "geph__b_0")(self.c.op)) - - def test_connect(self): - """Test for ge.connect.""" - with self.graph.as_default(): - x = constant_op.constant([1., 1.], shape=[2], name="x") - y = constant_op.constant([2., 2.], shape=[2], name="y") - z = math_ops.add(x, y, name="z") - - sgv = ge.sgv(x.op, y.op, z.op) - ge.connect(sgv, ge.sgv(self.e.op).remap_inputs([0])) - self.assertTrue( - match.OpMatcher("^foo/bar/e$").input_ops("^z$", "foo/d$")(self.e.op)) - - def test_bypass(self): - """Test for ge.bypass.""" - ge.bypass(ge.sgv(self.f.op).remap_inputs([0])) - self.assertTrue( - match.OpMatcher("^foo/bar/h$").input_ops("^foo/c$", "foo/bar/g$")( - self.h.op)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/graph_editor/tests/match.py b/tensorflow/contrib/graph_editor/tests/match.py deleted file mode 100644 index 1bf482b6c21..00000000000 --- a/tensorflow/contrib/graph_editor/tests/match.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Simple graph matching functions.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from six import string_types - -from tensorflow.contrib.graph_editor import select -from tensorflow.python.framework import ops as tf_ops - -__all__ = [ - "op_type", - "OpMatcher", -] - - -def _make_graph_match(graph_match): - """Convert to a OpMatcher instance.""" - if graph_match is None: - return None - if not isinstance(graph_match, OpMatcher): - graph_match = OpMatcher(graph_match) - return graph_match - - -def op_type(op_types, op=None): - """Check if an op is of the given type. - - Args: - op_types: tuple of strings containing the types to check against. - For instance: ("Add", "Const") - op: the operation to check (or None). - Returns: - if op is not None, return True if the op is of the correct type. - if op is None, return a lambda function which does the type checking. - """ - if isinstance(op_types, string_types): - op_types = (op_types) - if op is None: - return lambda op: op.node_def.op in op_types - else: - return op.node_def.op in op_types - - -class OpMatcher(object): - """Graph match class.""" - - def __init__(self, positive_filter): - """Graph match constructor.""" - self.positive_filters = [] - self.input_op_matches = None - self.control_input_op_matches = None - self.output_op_matches = None - positive_filter = self._finalize_positive_filter(positive_filter) - self.positive_filters.append(positive_filter) - - def _finalize_positive_filter(self, elem): - """Convert to a filter function.""" - if select.can_be_regex(elem): - regex_ = select.make_regex(elem) - return lambda op, regex=regex_: regex.search(op.name) is not None - elif isinstance(elem, tf_ops.Operation): - return lambda op, match_op=elem: op is match_op - elif callable(elem): - return elem - elif elem is True: - return lambda op: True - else: - raise ValueError("Cannot finalize the positive filter: {}".format(elem)) - - def __call__(self, op): - """Evaluate if the op matches or not.""" - if not isinstance(op, tf_ops.Operation): - raise TypeError("Expect tf.Operation, got: {}".format(type(op))) - for positive_filter in self.positive_filters: - if not positive_filter(op): - return False - if self.input_op_matches is not None: - if len(op.inputs) != len(self.input_op_matches): - return False - for input_t, input_op_match in zip(op.inputs, self.input_op_matches): - if input_op_match is None: - continue - if not input_op_match(input_t.op): - return False - if self.control_input_op_matches is not None: - if len(op.control_inputs) != len(self.control_input_op_matches): - return False - for cinput_op, cinput_op_match in zip(op.control_inputs, - self.control_input_op_matches): - if cinput_op_match is None: - continue - if not cinput_op_match(cinput_op): - return False - if self.output_op_matches is not None: - if len(op.outputs) != len(self.output_op_matches): - return False - for output_t, output_op_matches in zip(op.outputs, - self.output_op_matches): - if output_op_matches is None: - continue - if len(output_t.consumers()) != len(output_op_matches): - return False - for consumer_op, consumer_op_match in zip(output_t.consumers(), - output_op_matches): - if consumer_op_match is None: - continue - if not consumer_op_match(consumer_op): - return False - return True - - def input_ops(self, *args): - """Add input matches.""" - if self.input_op_matches is not None: - raise ValueError("input_op_matches is already set.") - self.input_op_matches = [] - for input_match in args: - self.input_op_matches.append(_make_graph_match(input_match)) - return self - - def control_input_ops(self, *args): - """Add input matches.""" - if self.control_input_op_matches is not None: - raise ValueError("control_input_op_matches is already set.") - self.control_input_op_matches = [] - for input_match in args: - self.control_input_op_matches.append(_make_graph_match(input_match)) - return self - - def output_ops(self, *args): - """Add output matches.""" - if self.output_op_matches is not None: - raise ValueError("output_op_matches is already set.") - self.output_op_matches = [] - for consumer_op_matches in args: - if consumer_op_matches is None: - self.output_op_matches.append(None) - if not isinstance(consumer_op_matches, list): - consumer_op_matches = [consumer_op_matches] - consumer_op_matches = [_make_graph_match(consumer_op_match) - for consumer_op_match in consumer_op_matches] - self.output_op_matches.append(consumer_op_matches) - return self diff --git a/tensorflow/contrib/graph_editor/tests/match_test.py b/tensorflow/contrib/graph_editor/tests/match_test.py deleted file mode 100644 index d81dc34dba0..00000000000 --- a/tensorflow/contrib/graph_editor/tests/match_test.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.contrib.graph_editor.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.graph_editor.tests import match -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class MatchTest(test.TestCase): - - def setUp(self): - self.graph = ops.Graph() - with self.graph.as_default(): - self.a = constant_op.constant([1., 1.], shape=[2], name="a") - with ops.name_scope("foo"): - self.b = constant_op.constant([2., 2.], shape=[2], name="b") - self.c = math_ops.add(self.a, self.b, name="c") - self.d = constant_op.constant([3., 3.], shape=[2], name="d") - with ops.name_scope("bar"): - self.e = math_ops.add(self.c, self.d, name="e") - self.f = math_ops.add(self.c, self.d, name="f") - self.g = math_ops.add(self.c, self.a, name="g") - with ops.control_dependencies([self.c.op]): - self.h = math_ops.add(self.f, self.g, name="h") - - def test_simple_match(self): - self.assertTrue(match.OpMatcher("^.*/f$")(self.f.op)) - self.assertTrue( - match.OpMatcher("^.*/f$").input_ops("^.*/c$", "^.*/d$")(self.f.op)) - self.assertTrue( - match.OpMatcher("^.*/f$").input_ops(True, "^.*/d$")(self.f.op)) - self.assertTrue( - match.OpMatcher("^.*/f$").input_ops( - match.op_type("Add"), match.op_type("Const"))(self.f.op)) - self.assertTrue( - match.OpMatcher("^.*/f$").input_ops("^.*/c$", "^.*/d$") - .output_ops(match.OpMatcher("^.*/h$") - .control_input_ops("^.*/c$"))(self.f.op)) - self.assertTrue( - match.OpMatcher("^.*/f$").input_ops("^.*/c$", "^.*/d$").output_ops( - match.OpMatcher("^.*/h$").control_input_ops("^.*/c$") - .output_ops([]))(self.f.op)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/graph_editor/tests/reroute_test.py b/tensorflow/contrib/graph_editor/tests/reroute_test.py deleted file mode 100644 index 3c00304add4..00000000000 --- a/tensorflow/contrib/graph_editor/tests/reroute_test.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.contrib.graph_editor.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import graph_editor as ge -from tensorflow.contrib.graph_editor.tests import match -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class RerouteTest(test.TestCase): - - def setUp(self): - self.graph = ops.Graph() - with self.graph.as_default(): - self.a0 = constant_op.constant(1.0, shape=[2], name="a0") - self.b0 = constant_op.constant(2.0, shape=[2], name="b0") - self.c0 = math_ops.add(self.a0, self.b0, name="c0") - self.a1 = constant_op.constant(3.0, shape=[2], name="a1") - self.b1 = constant_op.constant(4.0, shape=[2], name="b1") - self.c1 = math_ops.add(self.a1, self.b1, name="c1") - self.a2 = constant_op.constant(3.0, shape=[3], name="a2") - self.b2 = constant_op.constant(4.0, shape=[3], name="b2") - self.c2 = math_ops.add(self.a2, self.b2, name="c2") - - def test_swap(self): - ge.swap_ts([self.a0, self.b0], [self.a1, self.b1]) - self.assertTrue(match.OpMatcher("c0").input_ops("a1", "b1")(self.c0.op)) - self.assertTrue(match.OpMatcher("c1").input_ops("a0", "b0")(self.c1.op)) - - def test_multiswap(self): - with self.graph.as_default(): - a3 = constant_op.constant(3.0, shape=[2], name="a3") - ge.swap_ios(ge.sgv(a3.op).remap_outputs([0, 0]), - ge.sgv(self.a0.op, self.a1.op)) - self.assertTrue(match.OpMatcher("c0").input_ops("a3", "b0")(self.c0.op)) - self.assertTrue(match.OpMatcher("c1").input_ops("a3", "b1")(self.c1.op)) - - def test_reroute(self): - ge.reroute_ts([self.a0, self.b0], [self.a1, self.b1]) - self.assertTrue(match.OpMatcher("c0").input_ops("a0", "b0")(self.c0.op)) - self.assertTrue(match.OpMatcher("c1").input_ops("a0", "b0")(self.c1.op)) - - ge.reroute_ts([self.a1, self.b1], [self.a0, self.b0]) - self.assertTrue(match.OpMatcher("c0").input_ops("a1", "b1")(self.c0.op)) - self.assertTrue(match.OpMatcher("c1").input_ops("a1", "b1")(self.c1.op)) - - def test_compatibility(self): - with self.assertRaises(ValueError): - ge.reroute_ts([self.a0, self.b0], [self.a2, self.b2]) - - def test_reroute_can_modify(self): - graph = ops.Graph() - # create a special graph where "a" is an ambiguous tensor. That is - # it is both an input and an output of the ops in sgv0. - with graph.as_default(): - a = constant_op.constant(1.0, shape=[2], name="a") - b = constant_op.constant(2.0, shape=[2], name="b") - c = math_ops.add(a, b, name="c") - d = math_ops.add(a, c, name="d") - - e = constant_op.constant(1.0, shape=[2], name="e") - f = constant_op.constant(2.0, shape=[2], name="f") - g = math_ops.add(e, f, name="g") - - sgv0 = ge.sgv(a.op, b.op, c.op) - sgv1 = ge.sgv(e.op, f.op) - - ge.swap_outputs(sgv0, sgv1) - self.assertTrue( - match.OpMatcher("g").input_ops( - "a", match.OpMatcher("c").input_ops("a", "b"))(g.op)) - self.assertTrue(match.OpMatcher("d").input_ops("e", "f")(d.op)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/graph_editor/tests/select_test.py b/tensorflow/contrib/graph_editor/tests/select_test.py deleted file mode 100644 index d12c6d3cbd1..00000000000 --- a/tensorflow/contrib/graph_editor/tests/select_test.py +++ /dev/null @@ -1,312 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.contrib.graph_editor.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re - -from tensorflow.contrib import graph_editor as ge -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops as ops_lib -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class SelectTest(test.TestCase): - - def setUp(self): - self.graph = ops_lib.Graph() - with self.graph.as_default(): - self.a = constant_op.constant([1., 1.], shape=[2], name="a") - with ops_lib.name_scope("foo"): - self.b = constant_op.constant([2., 2.], shape=[2], name="b") - self.c = math_ops.add(self.a, self.b, name="c") - self.d = constant_op.constant([3., 3.], shape=[2], name="d") - with ops_lib.name_scope("bar"): - self.e = math_ops.add(self.c, self.d, name="e") - self.f = math_ops.add(self.c, self.d, name="f") - self.g = math_ops.add(self.c, self.a, name="g") - with ops_lib.control_dependencies([self.c.op]): - self.h = math_ops.add(self.f, self.g, name="h") - - def test_regex(self): - """Test for ge.can_be_regex and ge.make_regex.""" - self.assertTrue(ge.can_be_regex("foo")) - self.assertTrue(ge.can_be_regex(re.compile("foo"))) - regex = re.compile("foo") - self.assertIs(ge.make_regex(regex), regex) - - def test_get_input_output_ts(self): - """Test for ge._get_input_ts abd ge._get_output_ts.""" - self.assertEqual(len(ge.select._get_input_ts(self.graph)), 6) - self.assertEqual(len(ge.select._get_output_ts(self.graph)), 8) - - def test_get_filter(self): - """Test for various filtering operations on ts ops.""" - # TODO(fkp): parameterise - self.assertEqual(len(ge.filter_ops(self.graph, True)), 8) - self.assertEqual( - len(ge.filter_ops(self.graph, lambda op: op.node_def.op == "Const")), 3) - self.assertEqual( - len(ge.filter_ops(self.graph, lambda op: op.node_def.op == "Add")), 5) - self.assertEqual( - len(ge.filter_ops_from_regex(self.graph, r"^.*\b[abc]$")), 3) - - self.assertEqual(len(ge.filter_ts(self.graph, True)), 8) - self.assertEqual( - len(ge.filter_ts_from_regex(self.graph, r"^.*/[fgh]:\d$")), 3) - - self.assertEqual(len(ge.get_name_scope_ops(self.graph, "foo/")), 7) - self.assertEqual(len(ge.get_name_scope_ops(self.graph, "foo/bar")), 4) - - def test_get_ops_ios(self): - """Test for ge.get_ops_ios.""" - control_outputs = ge.util.ControlOutputs(self.graph) - self.assertEqual( - len(ge.get_ops_ios(self.h.op, control_ios=control_outputs)), 3) - self.assertEqual(len(ge.get_ops_ios(self.h.op)), 2) - self.assertEqual( - len(ge.get_ops_ios(self.c.op, control_ios=control_outputs)), 6) - self.assertEqual(len(ge.get_ops_ios(self.c.op)), 5) - - def test_compute_boundary_ts_0(self): - """Test for ge.compute_boundary_ts.""" - input_ts, output_ts, inside_ts = ge.compute_boundary_ts(self.g.op) - self.assertEqual(list(input_ts), [self.c, self.a]) - self.assertEqual(list(output_ts), [self.g]) - self.assertEqual(list(inside_ts), []) - - def test_compute_boundary_ts_1(self): - """Test for ge.compute_boundary_ts.""" - input_ts, output_ts, inside_ts = ge.compute_boundary_ts( - [self.g.op, self.h.op]) - self.assertEqual(list(input_ts), [self.c, self.a, self.f]) - self.assertEqual(list(output_ts), [self.h]) - self.assertEqual(list(inside_ts), [self.g]) - - def test_compute_boundary_ts_2(self): - """Test for ge.compute_boundary_ts.""" - graph = ops_lib.Graph() - with graph.as_default(): - a = constant_op.constant(1, name="a") - b = constant_op.constant(1, name="b") - c = math_ops.add(a, b, name="c") - _ = a + c - input_ts, output_ts, inside_ts = ge.compute_boundary_ts([a.op, c.op]) - self.assertEqual(list(input_ts), [b]) - self.assertEqual(list(output_ts), [a, c]) - self.assertEqual(list(inside_ts), [a]) - - def test_get_within_boundary_ops_0(self): - """Test for test_get_within_boundary_ops.""" - control_outputs = ge.util.ControlOutputs(self.graph) - ops = ge.get_within_boundary_ops( - ops=self.graph, - seed_ops=self.f.op, - boundary_ops=[self.c.op, self.h.op], - inclusive=False, - control_ios=control_outputs) - self.assertEqual(len(ops), 3) - - def test_get_within_boundary_ops_1(self): - """Test for ge.test_get_within_boundary_ops.""" - ops = ge.get_within_boundary_ops( - ops=self.graph, seed_ops=self.h.op, boundary_ops=[self.f.op, self.g.op]) - self.assertEqual(len(ops), 3) - - def test_get_walks_intersection(self): - """Test for ge.get_walks_intersection_ops.""" - ops = ge.get_walks_intersection_ops([self.c.op], [self.g.op]) - self.assertEqual(len(ops), 2) - - ops = ge.get_walks_intersection_ops([self.a.op], [self.f.op]) - self.assertEqual(len(ops), 3) - self.assertTrue(self.a.op in ops) - self.assertTrue(self.c.op in ops) - self.assertTrue(self.f.op in ops) - - within_ops = [self.a.op, self.f.op] - ops = ge.get_walks_intersection_ops( - [self.a.op], [self.f.op], within_ops=within_ops) - self.assertEqual(len(ops), 0) - - within_ops_fn = lambda op: op in [self.a.op, self.f.op] - ops = ge.get_walks_intersection_ops( - [self.a.op], [self.f.op], within_ops_fn=within_ops_fn) - self.assertEqual(len(ops), 0) - - def test_get_walks_union(self): - """Test for ge.get_walks_union_ops.""" - ops = ge.get_walks_union_ops([self.f.op], [self.g.op]) - self.assertEqual(len(ops), 6) - - ops = ge.get_walks_union_ops([self.a.op], [self.f.op]) - self.assertEqual(len(ops), 8) - - within_ops = [self.a.op, self.c.op, self.d.op, self.f.op] - ops = ge.get_walks_union_ops([self.a.op], [self.f.op], - within_ops=within_ops) - self.assertEqual(len(ops), 4) - self.assertTrue(self.b.op not in ops) - - within_ops_fn = lambda op: op in [self.a.op, self.c.op, self.f.op] - ops = ge.get_walks_union_ops([self.a.op], [self.f.op], - within_ops_fn=within_ops_fn) - self.assertEqual(len(ops), 3) - self.assertTrue(self.b.op not in ops) - self.assertTrue(self.d.op not in ops) - - def test_select_ops(self): - parameters = ( - (("^foo/",), 7), - (("^foo/bar/",), 4), - (("^foo/bar/", "a"), 5), - ) - for param, length in parameters: - ops = ge.select_ops(*param, graph=self.graph) - self.assertEqual(len(ops), length) - - def test_select_ts(self): - parameters = ( - (".*:0", 8), - (r".*/bar/\w+:0", 4), - ) - for regex, length in parameters: - ts = ge.select_ts(regex, graph=self.graph) - self.assertEqual(len(ts), length) - - def test_select_ops_and_ts(self): - parameters = ( - (("^foo/.*",), 7, 0), - (("^foo/.*", "(?#ts)^foo/bar/.*"), 7, 4), - ) - for param, l0, l1 in parameters: - ops, ts = ge.select_ops_and_ts(*param, graph=self.graph) - self.assertEqual(len(ops), l0) - self.assertEqual(len(ts), l1) - - def test_forward_walk_ops(self): - seed_ops = [self.a.op, self.d.op] - # Include all ops except for self.g.op - within_ops = [ - x.op for x in [self.a, self.b, self.c, self.d, self.e, self.f, self.h] - ] - # For the fn, exclude self.e.op. - within_ops_fn = lambda op: op not in (self.e.op,) - stop_at_ts = (self.f,) - - with self.graph.as_default(): - # No b.op since it's an independent source node. - # No g.op from within_ops. - # No e.op from within_ops_fn. - # No h.op from stop_at_ts and within_ops. - ops = ge.select.get_forward_walk_ops( - seed_ops, - inclusive=True, - within_ops=within_ops, - within_ops_fn=within_ops_fn, - stop_at_ts=stop_at_ts) - self.assertEqual( - set(ops), set([self.a.op, self.c.op, self.d.op, self.f.op])) - - # Also no a.op and d.op when inclusive=False - ops = ge.select.get_forward_walk_ops( - seed_ops, - inclusive=False, - within_ops=within_ops, - within_ops_fn=within_ops_fn, - stop_at_ts=stop_at_ts) - self.assertEqual(set(ops), set([self.c.op, self.f.op])) - - # Not using within_ops_fn adds e.op. - ops = ge.select.get_forward_walk_ops( - seed_ops, - inclusive=False, - within_ops=within_ops, - stop_at_ts=stop_at_ts) - self.assertEqual(set(ops), set([self.c.op, self.e.op, self.f.op])) - - # Not using stop_at_ts adds back h.op. - ops = ge.select.get_forward_walk_ops( - seed_ops, inclusive=False, within_ops=within_ops) - self.assertEqual( - set(ops), set([self.c.op, self.e.op, self.f.op, self.h.op])) - - # Starting just form a (the tensor, not op) omits a, b, d. - ops = ge.select.get_forward_walk_ops([self.a], inclusive=True) - self.assertEqual( - set(ops), set([self.c.op, self.e.op, self.f.op, self.g.op, - self.h.op])) - - def test_backward_walk_ops(self): - seed_ops = [self.h.op] - # Include all ops except for self.g.op - within_ops = [ - x.op for x in [self.a, self.b, self.c, self.d, self.e, self.f, self.h] - ] - # For the fn, exclude self.c.op. - within_ops_fn = lambda op: op not in (self.c.op,) - stop_at_ts = (self.f,) - - with self.graph.as_default(): - # Backward walk only includes h since we stop at f and g is not within. - ops = ge.select.get_backward_walk_ops( - seed_ops, - inclusive=True, - within_ops=within_ops, - within_ops_fn=within_ops_fn, - stop_at_ts=stop_at_ts) - self.assertEqual(set(ops), set([self.h.op])) - - # If we do inclusive=False, the result is empty. - ops = ge.select.get_backward_walk_ops( - seed_ops, - inclusive=False, - within_ops=within_ops, - within_ops_fn=within_ops_fn, - stop_at_ts=stop_at_ts) - self.assertEqual(set(ops), set()) - - # Removing stop_at_fs adds f.op, d.op. - ops = ge.select.get_backward_walk_ops( - seed_ops, - inclusive=True, - within_ops=within_ops, - within_ops_fn=within_ops_fn) - self.assertEqual(set(ops), set([self.d.op, self.f.op, self.h.op])) - - # Not using within_ops_fn adds back ops for a, b, c. - ops = ge.select.get_backward_walk_ops( - seed_ops, inclusive=True, within_ops=within_ops) - self.assertEqual( - set(ops), - set([ - self.a.op, self.b.op, self.c.op, self.d.op, self.f.op, self.h.op - ])) - - # Vanially backward search via self.h.op includes everything excpet e.op. - ops = ge.select.get_backward_walk_ops(seed_ops, inclusive=True) - self.assertEqual( - set(ops), - set([ - self.a.op, self.b.op, self.c.op, self.d.op, self.f.op, self.g.op, - self.h.op - ])) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/graph_editor/tests/subgraph_test.py b/tensorflow/contrib/graph_editor/tests/subgraph_test.py deleted file mode 100644 index f91c2b5c25f..00000000000 --- a/tensorflow/contrib/graph_editor/tests/subgraph_test.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.contrib.graph_editor.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import graph_editor as ge -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class SubgraphTest(test.TestCase): - - def setUp(self): - self.graph = ops.Graph() - with self.graph.as_default(): - self.a = constant_op.constant([1., 1.], shape=[2], name="a") - with ops.name_scope("foo"): - self.b = constant_op.constant([2., 2.], shape=[2], name="b") - self.c = math_ops.add(self.a, self.b, name="c") - self.d = constant_op.constant([3., 3.], shape=[2], name="d") - with ops.name_scope("bar"): - self.e = math_ops.add(self.c, self.d, name="e") - self.f = math_ops.add(self.c, self.d, name="f") - self.g = math_ops.add(self.c, self.a, name="g") - with ops.control_dependencies([self.c.op]): - self.h = math_ops.add(self.f, self.g, name="h") - - def test_subgraph(self): - sgv = ge.sgv(self.graph) - self.assertEqual(list(sgv.outputs), [self.e, self.h]) - self.assertEqual(list(sgv.inputs), []) - self.assertEqual(len(sgv.ops), 8) - - sgv = ge.sgv(self.f.op, self.g.op) - self.assertEqual(list(sgv.outputs), [self.f, self.g]) - self.assertEqual(list(sgv.inputs), [self.c, self.d, self.a]) - - sgv = ge.sgv_scope("foo/bar", graph=self.graph) - self.assertEqual( - list(sgv.ops), [self.e.op, self.f.op, self.g.op, self.h.op]) - - def test_subgraph_remap(self): - sgv = ge.sgv(self.c.op) - self.assertEqual(list(sgv.outputs), [self.c]) - self.assertEqual(list(sgv.inputs), [self.a, self.b]) - - sgv = ge.sgv(self.c.op).remap([self.a], [0, self.c]) - self.assertEqual(list(sgv.outputs), [self.c, self.c]) - self.assertEqual(list(sgv.inputs), [self.a]) - - sgv = sgv.remap_outputs_to_consumers() - self.assertEqual(list(sgv.outputs), [self.c, self.c, self.c]) - sgv = sgv.remap_outputs_make_unique() - self.assertEqual(list(sgv.outputs), [self.c]) - - sgv = sgv.remap(new_input_indices=[], new_output_indices=[]) - self.assertEqual(len(sgv.inputs), 0) - self.assertEqual(len(sgv.outputs), 0) - sgv = sgv.remap_default() - self.assertEqual(list(sgv.outputs), [self.c]) - self.assertEqual(list(sgv.inputs), [self.a, self.b]) - - def test_remove_unused_ops(self): - sgv = ge.sgv(self.graph) - self.assertEqual(list(sgv.outputs), [self.e, self.h]) - self.assertEqual(len(sgv.ops), 8) - - sgv = sgv.remap_outputs(new_output_indices=[1]).remove_unused_ops() - self.assertEqual(list(sgv.outputs), [self.h]) - self.assertEqual(len(sgv.ops), 7) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/graph_editor/tests/transform_test.py b/tensorflow/contrib/graph_editor/tests/transform_test.py deleted file mode 100644 index 0ebcdc26889..00000000000 --- a/tensorflow/contrib/graph_editor/tests/transform_test.py +++ /dev/null @@ -1,286 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.contrib.graph_editor.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import functools -import numpy as np -from tensorflow.contrib import graph_editor as ge -from tensorflow.contrib.graph_editor.tests import match -from tensorflow.core.framework import attr_value_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - -# Precision tolerance for floating-point value tests. -ERROR_TOLERANCE = 1e-3 - - -class TransformTest(test.TestCase): - - def setUp(self): - self.graph = ops.Graph() - with self.graph.as_default(): - c0 = constant_op.constant(1.0, shape=[10], name="Const") - c0.op._set_attr("_foo", attr_value_pb2.AttrValue(s=b"foo")) - c1 = constant_op.constant(1.0, shape=[10], name="Const") - c2 = constant_op.constant(1.0, shape=[10], name="Const") - i = constant_op.constant(1.0, shape=[10], name="Input") - self.o = math_ops.add(c2, math_ops.add(c1, math_ops.add(c0, i))) - - def test_copy(self): - graph = ops.Graph() - _, info = ge.copy(self.graph, graph) - self.assertEqual( - set(op.name for op in self.graph.get_operations()), - set(op.name for op in graph.get_operations())) - src_ops = self.graph.get_operations() - dst_ops = graph.get_operations() - for op in src_ops: - op_ = info.transformed(op) - self.assertTrue(op_ in dst_ops) - self.assertEqual(op.name, op_.name) - self.assertEqual(info.original(op_), op) - src_ts = ge.util.get_tensors(self.graph) - dst_ts = ge.util.get_tensors(graph) - for t in src_ts: - t_ = info.transformed(t) - self.assertTrue(t_ in dst_ts) - self.assertEqual(t.name, t_.name) - self.assertEqual(info.original(t_), t) - - def test_copy_assert(self): - ops.reset_default_graph() - a = constant_op.constant(1) - b = constant_op.constant(1) - eq = math_ops.equal(a, b) - assert_op = control_flow_ops.Assert(eq, [a, b]) - with ops.control_dependencies([assert_op]): - _ = math_ops.add(a, b) - sgv = ge.make_view([assert_op, eq.op, a.op, b.op]) - copier = ge.Transformer() - _, info = copier(sgv, sgv.graph, "", "") - new_assert_op = info.transformed(assert_op) - self.assertIsNotNone(new_assert_op) - - def test_transform(self): - transformer = ge.Transformer() - - def my_transform_op_handler(info, op, new_inputs): - add_noise = op.name.startswith("Add") - op_, op_outputs_ = ge.transform.copy_op_handler(info, op, new_inputs) - if not add_noise: - return op_, op_outputs_ - # add some noise to op - with info.graph_.as_default(): - t_ = math_ops.add( - constant_op.constant(1.0, shape=[10], name="Noise"), - op_.outputs[0], - name="AddNoise") - # return the "noisy" op - return op_, [t_] - - transformer.transform_op_handler = my_transform_op_handler - - graph = ops.Graph() - transformer(self.graph, graph, "", "") - matcher0 = match.OpMatcher("AddNoise").input_ops( - "Noise", match.OpMatcher("Add").input_ops("Const", "Input")) - matcher1 = match.OpMatcher("AddNoise_1").input_ops( - "Noise_1", match.OpMatcher("Add_1").input_ops("Const_1", matcher0)) - matcher2 = match.OpMatcher("AddNoise_2").input_ops( - "Noise_2", match.OpMatcher("Add_2").input_ops("Const_2", matcher1)) - top = ge.select_ops("^AddNoise_2$", graph=graph)[0] - self.assertTrue(matcher2(top)) - - def test_transform_nodedef_fn(self): - transformer = ge.Transformer() - - def nodedef_fn(node_def): - if "_foo" in node_def.attr: - del node_def.attr["_foo"] - node_def.attr["_bar"].s = b"bar" - return node_def - - my_copy_op_handler = functools.partial( - ge.transform.copy_op_handler, nodedef_fn=nodedef_fn) - transformer.transform_op_handler = my_copy_op_handler - - graph = ops.Graph() - transformer(self.graph, graph, "", "") - - c0_before = self.graph.get_operation_by_name("Const") - c0_after = graph.get_operation_by_name("Const") - self.assertEquals(c0_before.get_attr("_foo"), b"foo") - with self.assertRaises(ValueError): - c0_after.get_attr("_foo") - - all_ops = graph.get_operations() - for op in all_ops: - self.assertEquals(op.get_attr("_bar"), b"bar") - - def test_copy_with_input_replacements(self): - with self.graph.as_default(): - ten = constant_op.constant(10.0, shape=[10], name="Input") - sgv, _ = ge.copy_with_input_replacements(self.o.op, - {self.o.op.inputs[1]: ten}) - with session.Session() as sess: - val = sess.run(sgv.outputs[0]) - self.assertNear( - np.linalg.norm(val - np.array([11])), 0.0, ERROR_TOLERANCE) - - def test_graph_replace(self): - ops.reset_default_graph() - a = constant_op.constant(1.0, name="a") - b = variables.Variable(1.0, name="b") - eps = constant_op.constant(0.001, name="eps") - c = array_ops.identity(a + b + eps, name="c") - a_new = constant_op.constant(2.0, name="a_new") - c_new = ge.graph_replace(c, {a: a_new}) - with session.Session() as sess: - sess.run(variables.global_variables_initializer()) - c_val, c_new_val = sess.run([c, c_new]) - self.assertNear(c_val, 2.001, ERROR_TOLERANCE) - self.assertNear(c_new_val, 3.001, ERROR_TOLERANCE) - - def test_graph_replace_dict(self): - ops.reset_default_graph() - a = constant_op.constant(1.0, name="a") - b = variables.Variable(1.0, name="b") - eps = constant_op.constant(0.001, name="eps") - c = array_ops.identity(a + b + eps, name="c") - a_new = constant_op.constant(2.0, name="a_new") - c_new = ge.graph_replace({"c": c}, {a: a_new}) - self.assertTrue(isinstance(c_new, dict)) - with session.Session() as sess: - sess.run(variables.global_variables_initializer()) - c_val, c_new_val = sess.run([c, c_new]) - self.assertTrue(isinstance(c_new_val, dict)) - self.assertNear(c_val, 2.001, ERROR_TOLERANCE) - self.assertNear(c_new_val["c"], 3.001, ERROR_TOLERANCE) - - def test_graph_replace_ordered_dict(self): - ops.reset_default_graph() - a = constant_op.constant(1.0, name="a") - b = variables.Variable(1.0, name="b") - eps = constant_op.constant(0.001, name="eps") - c = array_ops.identity(a + b + eps, name="c") - a_new = constant_op.constant(2.0, name="a_new") - c_new = ge.graph_replace(collections.OrderedDict({"c": c}), {a: a_new}) - self.assertTrue(isinstance(c_new, collections.OrderedDict)) - - def test_graph_replace_named_tuple(self): - ops.reset_default_graph() - a = constant_op.constant(1.0, name="a") - b = variables.Variable(1.0, name="b") - eps = constant_op.constant(0.001, name="eps") - c = array_ops.identity(a + b + eps, name="c") - a_new = constant_op.constant(2.0, name="a_new") - one_tensor = collections.namedtuple("OneTensor", ["t"]) - c_new = ge.graph_replace(one_tensor(c), {a: a_new}) - self.assertTrue(isinstance(c_new, one_tensor)) - - def test_graph_replace_missing(self): - ops.reset_default_graph() - a = constant_op.constant(1.0, name="a") - b = constant_op.constant(2.0, name="b") - c = a + 2 * b - d = constant_op.constant(2.0, name="d") - res = ge.graph_replace([b, c], {a: d}) - self.assertEqual(res[0].name, "b:0") - self.assertEqual(res[1].name, "add_1:0") - - def test_graph_replace_gradients(self): - ops.reset_default_graph() - w = variables.VariableV1(0.0, name="w") - y = math_ops.multiply(math_ops.multiply(w, w, name="mul1"), w, name="mul2") - g = gradients_impl.gradients(y, w, name="grad")[0] - - # Extract the operations. - replacement_ts = {w.value(): g} - original_mul1_grad = (ops.get_default_graph(). - get_operation_by_name("grad/mul1_grad/Mul_1")) - - # Should not raise exception. - res = ge.graph_replace(g, replacement_ts, dst_scope="res") - - # Extract the operations after graph_replace. - result_mul1_grad = (ops.get_default_graph(). - get_operation_by_name("res/grad/mul1_grad/Mul_1")) - - # Make sure _original_ops are as expected. - self.assertEqual(original_mul1_grad._original_op.name, u"mul1") - self.assertEqual(result_mul1_grad._original_op.name, u"res/mul1") - self.assertNotEqual(res.name, g.name) - with session.Session() as sess: - sess.run(variables.global_variables_initializer()) - g_val, res_val = sess.run([g, res]) - self.assertNear(g_val, 0.0, ERROR_TOLERANCE) - self.assertNear(res_val, 0.0, ERROR_TOLERANCE) - - def test_graph_while_loop(self): - graph = ops.Graph() - with graph.as_default(): - max_index = array_ops.placeholder(dtype=dtypes.int32, shape=tuple()) - index_start = constant_op.constant(1) - sum_start = constant_op.constant(0) - _, result = control_flow_ops.while_loop( - cond=lambda i, unused_s: i <= max_index, - body=lambda i, s: (i + 1, s + i), - loop_vars=[index_start, sum_start]) - copied_graph = ops.Graph() - _, copy_info = ge.copy( - graph, dst_graph=copied_graph, dst_scope="imported") - copied_result = copy_info.transformed(result) - copied_max_index = copy_info.transformed(max_index) - with copied_graph.as_default(): - with session.Session() as sess: - n = 10 - sum_val = sess.run(copied_result, feed_dict={copied_max_index: n}) - self.assertEqual(sum_val, 55) - - def test_graph_cond(self): - graph = ops.Graph() - with graph.as_default(): - choice = array_ops.placeholder(shape=(), dtype=dtypes.bool) - result = control_flow_ops.cond( - choice, - lambda: constant_op.constant(1), - lambda: constant_op.constant(2)) - copied_graph = ops.Graph() - _, copy_info = ge.copy( - graph, dst_graph=copied_graph, dst_scope="imported") - copied_result = copy_info.transformed(result) - copied_choice = copy_info.transformed(choice) - with copied_graph.as_default(): - with session.Session() as sess: - res = sess.run(copied_result, feed_dict={copied_choice: True}) - self.assertEqual(res, 1) - res = sess.run(copied_result, feed_dict={copied_choice: False}) - self.assertEqual(res, 2) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/graph_editor/tests/util_test.py b/tensorflow/contrib/graph_editor/tests/util_test.py deleted file mode 100644 index 8b1b2cfbdcf..00000000000 --- a/tensorflow/contrib/graph_editor/tests/util_test.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.contrib.graph_editor.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import graph_editor as ge -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class UtilTest(test.TestCase): - - def test_list_view(self): - """Test for ge.util.ListView.""" - l = [0, 1, 2] - lv = ge.util.ListView(l) - # Should not be the same id. - self.assertIsNot(l, lv) - # Should behave the same way than the original list. - self.assertTrue(len(lv) == 3 and lv[0] == 0 and lv[1] == 1 and lv[2] == 2) - # Should be read only. - with self.assertRaises(TypeError): - lv[0] = 0 - - def test_is_iterable(self): - """Test for ge.util.is_iterable.""" - self.assertTrue(ge.util.is_iterable([0, 1, 2])) - self.assertFalse(ge.util.is_iterable(3)) - - def test_unique_graph(self): - """Test for ge.util.check_graphs and ge.util.get_unique_graph.""" - g0 = ops.Graph() - with g0.as_default(): - a0 = constant_op.constant(1) - b0 = constant_op.constant(2) - g1 = ops.Graph() - with g1.as_default(): - a1 = constant_op.constant(1) - b1 = constant_op.constant(2) - # Same graph, should be fine. - self.assertIsNone(ge.util.check_graphs(a0, b0)) - # Two different graphs, should assert. - with self.assertRaises(ValueError): - ge.util.check_graphs(a0, b0, a1, b1) - # a0 and b0 belongs to the same graph, should be fine. - self.assertEqual(ge.util.get_unique_graph([a0, b0]), g0) - # Different graph, should raise an error. - with self.assertRaises(ValueError): - ge.util.get_unique_graph([a0, b0, a1, b1]) - - def test_make_list_of_op(self): - """Test for ge.util.make_list_of_op.""" - g0 = ops.Graph() - with g0.as_default(): - a0 = constant_op.constant(1) - b0 = constant_op.constant(2) - # Should extract the ops from the graph. - self.assertEqual(len(ge.util.make_list_of_op(g0)), 2) - # Should extract the ops from the tuple. - self.assertEqual(len(ge.util.make_list_of_op((a0.op, b0.op))), 2) - - def test_make_list_of_t(self): - """Test for ge.util.make_list_of_t.""" - g0 = ops.Graph() - with g0.as_default(): - a0 = constant_op.constant(1) - b0 = constant_op.constant(2) - c0 = math_ops.add(a0, b0) # pylint: disable=unused-variable - # Should extract the tensors from tre graph. - self.assertEqual(len(ge.util.make_list_of_t(g0)), 3) - # Should extract the tensors from the tuple - self.assertEqual(len(ge.util.make_list_of_t((a0, b0))), 2) - # Should extract the tensors and ignore the ops. - self.assertEqual( - len(ge.util.make_list_of_t( - (a0, a0.op, b0), ignore_ops=True)), 2) - - def test_get_generating_consuming(self): - """Test for ge.util.get_generating_ops and ge.util.get_generating_ops.""" - g0 = ops.Graph() - with g0.as_default(): - a0 = constant_op.constant(1) - b0 = constant_op.constant(2) - c0 = math_ops.add(a0, b0) - self.assertEqual(len(ge.util.get_generating_ops([a0, b0])), 2) - self.assertEqual(len(ge.util.get_consuming_ops([a0, b0])), 1) - self.assertEqual(len(ge.util.get_generating_ops([c0])), 1) - self.assertEqual(ge.util.get_consuming_ops([c0]), []) - - def test_control_outputs(self): - """Test for the ge.util.ControlOutputs class.""" - g0 = ops.Graph() - with g0.as_default(): - a0 = constant_op.constant(1) - b0 = constant_op.constant(2) - x0 = constant_op.constant(3) - with ops.control_dependencies([x0.op]): - c0 = math_ops.add(a0, b0) # pylint: disable=unused-variable - control_outputs = ge.util.ControlOutputs(g0).get_all() - self.assertEqual(len(control_outputs), 1) - self.assertEqual(len(control_outputs[x0.op]), 1) - self.assertIs(list(control_outputs[x0.op])[0], c0.op) - - def test_scope(self): - """Test simple path scope functionalities.""" - self.assertEqual(ge.util.scope_finalize("foo/bar"), "foo/bar/") - self.assertEqual(ge.util.scope_dirname("foo/bar/op"), "foo/bar/") - self.assertEqual(ge.util.scope_basename("foo/bar/op"), "op") - - def test_placeholder(self): - """Test placeholder functionalities.""" - g0 = ops.Graph() - with g0.as_default(): - a0 = constant_op.constant(1, name="foo") - # Test placeholder name. - self.assertEqual(ge.util.placeholder_name(a0), "geph__foo_0") - self.assertEqual(ge.util.placeholder_name(None), "geph") - self.assertEqual( - ge.util.placeholder_name( - a0, scope="foo/"), "foo/geph__foo_0") - self.assertEqual( - ge.util.placeholder_name( - a0, scope="foo"), "foo/geph__foo_0") - self.assertEqual(ge.util.placeholder_name(None, scope="foo/"), "foo/geph") - self.assertEqual(ge.util.placeholder_name(None, scope="foo"), "foo/geph") - # Test placeholder creation. - g0 = ops.Graph() - with g0.as_default(): - a0 = constant_op.constant(1, dtype=dtypes.float32, name="a0") - c0 = math_ops.add( - ge.util.make_placeholder_from_tensor(a0), - ge.util.make_placeholder_from_dtype_and_shape(dtype=dtypes.float32)) - self.assertEqual(c0.op.inputs[0].op.name, "geph__a0_0") - self.assertEqual(c0.op.inputs[1].op.name, "geph") - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/graph_editor/transform.py b/tensorflow/contrib/graph_editor/transform.py deleted file mode 100644 index 0a0c476dd1e..00000000000 --- a/tensorflow/contrib/graph_editor/transform.py +++ /dev/null @@ -1,752 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Class to transform an subgraph into another. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from copy import deepcopy -from functools import partial -from six import iteritems -from six import string_types -from six import StringIO -from tensorflow.contrib.graph_editor import reroute -from tensorflow.contrib.graph_editor import select -from tensorflow.contrib.graph_editor import subgraph -from tensorflow.contrib.graph_editor import util -from tensorflow.python.framework import ops as tf_ops -from tensorflow.python.platform import tf_logging as logging - - -__all__ = [ - "replace_t_with_placeholder_handler", - "keep_t_if_possible_handler", - "assign_renamed_collections_handler", - "transform_op_if_inside_handler", - "copy_op_handler", - "Transformer", - "TransformerInfo", - "copy", - "copy_with_input_replacements", - "graph_replace", -] - - -def replace_t_with_placeholder_handler(info, t): - """Transform a tensor into a placeholder tensor. - - This handler is typically used to transform a subgraph input tensor into a - placeholder. - - Args: - info: Transform._TmpInfo instance. - t: tensor whose input must be transformed into a place holder. - Returns: - The tensor generated by the newly created place holder. - """ - with info.graph_.as_default(): - t_ = util.make_placeholder_from_tensor(t, scope=info.scope_) - return t_ - - -def keep_t_if_possible_handler(info, t): - """Transform a tensor into itself (identity) if possible. - - This handler transform a tensor into itself if the source and destination - graph are the same. Otherwise it will create a placeholder. - This handler is typically used to transform a hidden input tensors. - - Args: - info: Transform._TmpInfo instance. - t: tensor whose input must be transformed into a place holder. - Returns: - The tensor generated by the newly created place holder. - """ - if info.graph is info.graph_: - return t - else: - return replace_t_with_placeholder_handler(info, t) - - -def assign_renamed_collections_handler(info, elem, elem_): - """Add the transformed elem to the (renamed) collections of elem. - - A collection is renamed only if is not a known key, as described in - `tf.compat.v1.GraphKeys`. - - Args: - info: Transform._TmpInfo instance. - elem: the original element (`tf.Tensor` or `tf.Operation`) - elem_: the transformed element - """ - known_collection_names = util.get_predefined_collection_names() - for name, collection in iteritems(info.collections): - if elem not in collection: - continue - - if name in known_collection_names: - transformed_name = name - else: - transformed_name = info.new_name(name) - info.graph_.add_to_collection(transformed_name, elem_) - - -def transform_op_if_inside_handler(info, op, keep_if_possible=True): - """Transform an optional op only if it is inside the subgraph. - - This handler is typically use to handle original op: it is fine to keep them - if they are inside the subgraph, otherwise they are just ignored. - - Args: - info: Transform._TmpInfo instance. - op: the optional op to transform (or ignore). - keep_if_possible: re-attach to the original op if possible, that is, - if the source graph and the destination graph are the same. - Returns: - The transformed op or None. - """ - if op in info.sgv.ops: - return info.transformed_ops[op] - else: - if keep_if_possible and info.graph is info.graph_: - return op - else: - return None - - -def copy_op_handler(info, op, new_inputs, copy_shape=False, nodedef_fn=None): - """Copy a `tf.Operation`. - - Args: - info: Transform._TmpInfo instance. - op: the `tf.Operation` to be copied. - new_inputs: The new inputs for this op. - copy_shape: also copy the shape of the tensor - nodedef_fn: If provided, a function that will be run on the NodeDef - and should return a mutated NodeDef before a new Operation is created. - This is useful as certain features cannot be set on the Operation and - must be modified in NodeDef. - - Returns: - A `(op, op_outputs)` tuple containing the transformed op and its outputs. - """ - # The `new_inputs` was added to this function. For compatibility reason, - # let's raise an error if `new_inputs` is a boolean. - if isinstance(new_inputs, bool): - raise TypeError("the `new_inputs` argument must be an iterable.") - - # pylint: disable=protected-access - - # Clone the node def: - node_def_ = deepcopy(op.node_def) - - # Transform name: - name_ = info.new_name(op.name) - name_ = info.graph_.unique_name(name_) - node_def_.name = name_ - - # Mutate NodeDef if requested: - if nodedef_fn is not None: - node_def_ = nodedef_fn(node_def_) - - # Copy the other inputs needed for initialization - output_types_ = op._output_types[:] - input_types_ = op._input_types[:] - - # Make a copy of the op_def too. - # Its unique to every _type_ of Operation. - op_def_ = deepcopy(op.op_def) - - # Initialize a new Operation instance - op_ = tf_ops.Operation(node_def_, info.graph_, new_inputs, output_types_, - [], input_types_, None, op_def_) - - # copy the shape over - if copy_shape: - for t, t_ in zip(op.outputs, op_.outputs): - t_.set_shape(t.get_shape()) - - # Original op cannot be finalised here yet. Because some ops require this - # attribute to exist, we will create a dummy original_op first and then - # later finalise it with the actual original_op when all the ops have - # been copied. - # TODO(fkp): Stop worrying about _original_op and remove this code? - if op._original_op: - op_._original_op = op._original_op - - return op_, op_.outputs - - -class TransformerInfo(object): - """"Contains information about the result of a transform operation.""" - - def __init__(self, info): - """Constructor. - - Args: - info: an instance of Transformer._TmpInfo containing various internal - information about the transform operation. - """ - self._graph = info.graph - self._scope = info.scope - self._graph_ = info.graph_ - self._scope_ = info.scope_ - self._transformed_ops = info.transformed_ops - self._transformed_ts = info.transformed_ts - - def _get_transformed_map(self, top): - """Return the correct container depending on the type of `top`.""" - if isinstance(top, tf_ops.Operation): - return self._transformed_ops - elif isinstance(top, tf_ops.Tensor): - return self._transformed_ts - else: - raise TypeError( - "Expected a tf.Tensor or a tf.Operation, got a {}".format( - type(top))) - - def _transformed_elem(self, original_top, missing_fn=None): - """Return the transformed op/tensor corresponding to the original one. - - Args: - original_top: the original tensor/operation. - missing_fn: function handling the case where the counterpart - cannot be found. By default, None is returned. - Returns: - the transformed tensor/operation (or None if no match is found). - """ - transformed_map = self._get_transformed_map(original_top) - if isinstance(original_top, string_types): - for original, transformed in iteritems(transformed_map): - if original.name == original_top: - return transformed - return None if missing_fn is None else missing_fn(original_top) - else: - if original_top not in transformed_map: - return None if missing_fn is None else missing_fn(original_top) - return transformed_map[original_top] - - def _original_elem(self, transformed_top, missing_fn=None): - """Return the original op/tensor corresponding to the transformed one. - - Args: - transformed_top: the transformed tensor/operation. - missing_fn: function handling the case where the counterpart - cannot be found. By default, None is returned. - Returns: - the original tensor/operation (or None if no match is found). - """ - transformed_map = self._get_transformed_map(transformed_top) - if isinstance(transformed_top, string_types): - finder = lambda transformed: transformed.name == transformed_top - else: - finder = lambda transformed: transformed == transformed_top - for original, transformed in iteritems(transformed_map): - if finder(transformed): - return original - return None if missing_fn is None else missing_fn(transformed_top) - - def transformed(self, original, missing_fn=None): - """Return the transformed op/tensor corresponding to the original one. - - Note that the output of this function mimics the hierarchy - of its input argument `original`. - Given an iterable, it returns a list. Given an operation or a tensor, - it will return an operation or a tensor. - - Args: - original: the original tensor/operation. - missing_fn: function handling the case where the counterpart - cannot be found. By default, None is returned. - Returns: - the transformed tensor/operation (or None if no match is found). - """ - transformed_elem = partial(self._transformed_elem, missing_fn=missing_fn) - return util.transform_tree(original, transformed_elem) - - def original(self, transformed, missing_fn=None): - """Return the original op/tensor corresponding to the transformed one. - - Note that the output of this function mimics the hierarchy - of its input argument `transformed`. - Given an iterable, it returns a list. Given an operation or a tensor, - it will return an operation or a tensor. - - Args: - transformed: the transformed tensor/operation. - missing_fn: function handling the case where the counterpart - cannot be found. By default, None is returned. - Returns: - the original tensor/operation (or None if no match is found). - """ - original_elem = partial(self._original_elem, missing_fn=missing_fn) - return util.transform_tree(transformed, original_elem) - - def __str__(self): - res = StringIO() - print("Transform result info:", file=res) - if self._graph == self._graph_: - in_place_str = "" if self._scope_ else " IN-PLACE" - print(" Within graph[{}]{}".format( - id(self._graph), in_place_str), file=res) - else: - print(" graph[{}] => graph[{}]".format( - id(self._graph), id(self._graph_)), file=res) - if self._scope: - print(" Relative to source scope: {}".format(self._scope), file=res) - if self._scope_: - print(" Scope destination: {}".format(self._scope_), file=res) - print("Operations mapping:", file=res) - for op, op_ in iteritems(self._transformed_ops): - print(" {} => {}".format(op.name, op_.name), file=res) - return res.getvalue() - - -class _TmpInfo(object): - """Transformer temporary data. - - An instance of this class holds all the information relevant to a call - to a transformer instance (that is, a call to __call__). An instance - is created for the life-time of the __call__ function and is passed as - argument to the handlers. - """ - - def __init__(self, sgv, dst_graph, dst_scope, src_scope): - self.sgv = sgv - self.sgv_inputs_set = frozenset(sgv.inputs) - self.ops = frozenset(sgv.ops) - self.control_outputs = util.ControlOutputs(sgv.graph) - self.graph = sgv.graph - self.scope = src_scope - self.graph_ = dst_graph - self.scope_ = dst_scope - self.transformed_ops = {} - self.transformed_ts = {} - self.collections = dict((key, self.graph.get_collection(key)) - for key in self.graph.get_all_collection_keys()) - self.cyclic_ops = [] - self.transform_original_op_handler = transform_op_if_inside_handler - # The graph is transformed op by op, in the same order the original ops - # were created. However, this is sometimes not possible due to cycles - # (i.e. while loops). So when the transformer creates a new op whose - # inputs do not exist yet, temporary placeholders are created and stored - # in this `tmp_cyclic_ts` container. During a second pass, - # those temporary tensors are replaced by the proper transformed tensors - # (see the function `_finalize_cycles`). - self.tmp_cyclic_ts = [] - - def new_name(self, name): - """Compute a destination name from a source name. - - Args: - name: the name to be "transformed". - Returns: - The transformed name. - Raises: - ValueError: if the source scope is used (that is, not an empty string) - and the source name does not belong to the source scope. - """ - scope = self.scope - if not name.startswith(scope): - raise ValueError("{} does not belong to source scope: {}.".format( - name, scope)) - rel_name = name[len(scope):] - name_ = self.scope_ + rel_name - return name_ - - -class Transformer(object): - """Transform a subgraph into another one. - - By default, the constructor create a transform which copy a subgraph and - replaces inputs with placeholders. This behavior can be modified by changing - the handlers. - """ - - def __init__(self): - """Transformer constructor. - - The following members can be modified: - transform_op_handler: handle the transformation of a `tf.Operation`. - This handler defaults to a simple copy. - assign_collections_handler: handle the assignment of collections. - This handler defaults to assigning new collections created under the - given name-scope. - transform_external_input_handler: handle the transform of the inputs to - the given subgraph. This handler defaults to creating placeholders - instead of the ops just before the input tensors of the subgraph. - transform_external_hidden_input_handler: handle the transform of the - hidden inputs of the subgraph, that is, the inputs which are not listed - in sgv.inputs. This handler defaults to a transform which keep the same - input if the source and destination graphs are the same, otherwise - use placeholders. - transform_original_op_handler: handle the transform of original_op. This - handler defaults to transforming original_op only if they are in the - subgraph, otherwise they are ignored. - """ - - # handlers - self.transform_op_handler = copy_op_handler - self.transform_control_input_handler = transform_op_if_inside_handler - self.assign_collections_handler = assign_renamed_collections_handler - self.transform_external_input_handler = replace_t_with_placeholder_handler - self.transform_external_hidden_input_handler = keep_t_if_possible_handler - self.transform_original_op_handler = transform_op_if_inside_handler - - def __call__(self, - sgv, - dst_graph, - dst_scope, - src_scope="", - reuse_dst_scope=False): - """Execute the transformation. - - Args: - sgv: the source subgraph-view. - dst_graph: the destination graph. - dst_scope: the destination scope. - src_scope: the source scope, which specify the path from which the - relative path of the transformed nodes are computed. For instance, if - src_scope is a/ and dst_scoped is b/, then the node a/x/y will have a - relative path of x/y and will be transformed into b/x/y. - reuse_dst_scope: if True the dst_scope is re-used if it already exists. - Otherwise, the scope is given a unique name based on the one given - by appending an underscore followed by a digit (default). - Returns: - A tuple `(sgv, info)` where: - `sgv` is the transformed subgraph view; - `info` is an instance of TransformerInfo containing - information about the transform, including mapping between - original and transformed tensors and operations. - Raises: - ValueError: if the arguments are invalid. - """ - sgv = subgraph.make_view(sgv) - if not isinstance(dst_graph, tf_ops.Graph): - raise TypeError("Expected a tf.Graph, got: {}".format(type(dst_graph))) - - src_scope = util.scope_finalize(src_scope) - dst_scope = util.scope_finalize(dst_scope) - - # Potentially create new scope if reuse_dst_scope is False - if dst_scope and not reuse_dst_scope: - dst_scope = util.scope_finalize(dst_graph.unique_name(dst_scope[:-1])) - - # Create temporary info used during this transform call - info = _TmpInfo(sgv, dst_graph, dst_scope, src_scope) - - self._copy_ops(info) - self._finalize_cycles(info) - self._connect_control_inputs(info) - - # Compute information about the transformation - res_info = TransformerInfo(info) - sgv_ = self._transform_sgv(info, sgv) - return sgv_, res_info - - def _copy_ops(self, info): - """Copy ops without connecting them.""" - sorted_ops = sorted(info.sgv.ops, key=lambda op: op._id) # pylint: disable=protected-access - for op in sorted_ops: - new_inputs = [self._transformed_t(info, t, op) for t in op.inputs] - op_, op_outputs_ = self.transform_op_handler(info, op, new_inputs) - if op is op_: - raise ValueError("In-place transformation not allowed.") - - # Process op. - info.transformed_ops[op] = op_ - self.assign_collections_handler(info, op, op_) - - # Process output tensors. - for op_output, op_output_ in zip(op.outputs, op_outputs_): - info.transformed_ts[op_output] = op_output_ - self.assign_collections_handler(info, op_output, op_output_) - - def _finalize_cycles(self, info): - """Reconnects the cyclic tensors.""" - for t, tmp_t_, consumer_op in info.tmp_cyclic_ts: - if t not in info.transformed_ts: - raise ValueError("The tensor {} should be transformed by now.".format( - t.name)) - if consumer_op not in info.transformed_ops: - raise ValueError("The op {} should be transformed by now.".format( - consumer_op.name)) - t_ = info.transformed_ts[t] - consumer_op_ = info.transformed_ops[consumer_op] - t_index_ = list(consumer_op_.inputs).index(tmp_t_) - consumer_op_._update_input(t_index_, t_) # pylint: disable=protected-access - - def _connect_control_inputs(self, info): - """Connect the previously copied ops.""" - for op in info.sgv.ops: - logging.debug("Connecting control inputs of op: %s", op.name) - op_ = info.transformed_ops[op] - - # Finalize original op. - # TODO(fkp): Stop worrying about _original_op and remove this code? - # pylint: disable=protected-access - if op._original_op: - original_op = self.transform_original_op_handler(info, op._original_op) - if original_op is None: - logging.debug("Could not find original op for: %s", op_.name) - else: - op_._original_op = original_op - # pylint: enable=protected-access - - # Finalize control inputs: - control_inputs_ = [self.transform_control_input_handler(info, ci) - for ci in op.control_inputs] - control_inputs_ = [ci for ci in control_inputs_ if ci is not None] - reroute.add_control_inputs(op_, control_inputs_) - - def _transform_sgv(self, info, sgv): - """Transform a subgraph view. - - For convenience, a transform operation returns a subgraph view of the - transformed graph. - - Args: - info: Temporary information for this transorfm call. - sgv: the subgraph to be transformed. - Returns: - The transformed subgraph. - """ - ops_ = [op_ for _, op_ in iteritems(info.transformed_ops)] - sgv_ = subgraph.SubGraphView(ops_) - sgv_inputs_ = sgv_.inputs - sgv_outputs_ = sgv_.outputs - - # re-order inputs - input_map_ = [] - for input_t in sgv.inputs: - if input_t not in info.transformed_ts: - continue - input_t_ = info.transformed_ts[input_t] - if input_t_ not in sgv_inputs_: - continue - input_t_index_ = sgv_.input_index(input_t_) - input_map_.append(input_t_index_) - - # re-order outputs - output_map_ = [] - for output_t in sgv.outputs: - if output_t not in info.transformed_ts: - continue - output_t_ = info.transformed_ts[output_t] - if output_t_ not in sgv_outputs_: - continue - output_t_index_ = sgv_.output_index(output_t_) - output_map_.append(output_t_index_) - - return sgv_.remap(input_map_, output_map_) - - def _transformed_t(self, info, t, consumer_op): - """Return tre transformed tensor of `t`.""" - if t in info.transformed_ts: - # If op is in the subgraph, just return its transformed counterpart. - return info.transformed_ts[t] - - if t in info.sgv_inputs_set: - # `t` is an input of the subgraph. - return self.transform_external_input_handler(info, t) - elif t.op in info.ops: - # `t` is an internal tensor but is not transformed yet because it - # belongs to a graph cycle. - logging.debug("Cyclic tensor: t.name = %s", t.name) - # Try to find an existing tensor we can use for now, - # otherwise create one. We'll rewire this later. - if consumer_op.type == "Merge": - first_input = consumer_op.inputs[0] - tmp_t_ = self._transformed_t(info, first_input, consumer_op) - elif t.op.type == "Enter": - enter_input = t.op.inputs[0] - tmp_t_ = self._transformed_t(info, enter_input, consumer_op) - else: - with info.graph_.as_default(): - tmp_t_ = util.make_placeholder_from_tensor(t, scope=info.scope_, - prefix="geph_tmp") - logging.debug("Created temporary placeholder: %s.", tmp_t_.name) - # Register as temporary and return. - info.tmp_cyclic_ts.append((t, tmp_t_, consumer_op)) - return tmp_t_ - else: - # `t` is a hidden input of the subgraph. - return self.transform_external_hidden_input_handler(info, t) - - -def copy(sgv, dst_graph=None, dst_scope="", src_scope="", - reuse_dst_scope=False): - """Copy a subgraph. - - Args: - sgv: the source subgraph-view. This argument is converted to a subgraph - using the same rules than the function subgraph.make_view. - dst_graph: the destination graph. - dst_scope: the destination scope. - src_scope: the source scope. - reuse_dst_scope: if True the dst_scope is re-used if it already exists. - Otherwise, the scope is given a unique name based on the one given - by appending an underscore followed by a digit (default). - Returns: - A tuple `(sgv, info)` where: - `sgv` is the transformed subgraph view; - `info` is an instance of TransformerInfo containing - information about the transform, including mapping between - original and transformed tensors and operations. - Raises: - TypeError: if `dst_graph` is not a `tf.Graph`. - StandardError: if sgv cannot be converted to a SubGraphView using - the same rules than the function subgraph.make_view. - """ - sgv = subgraph.make_view(sgv) - if dst_graph is None: - dst_graph = sgv.graph - if not isinstance(dst_graph, tf_ops.Graph): - raise TypeError("Expected a tf.Graph, got: {}".format(type(dst_graph))) - - copier = Transformer() - return copier( - sgv, dst_graph, dst_scope, src_scope, reuse_dst_scope=reuse_dst_scope) - - -def copy_with_input_replacements(sgv, replacement_ts, - dst_graph=None, dst_scope="", src_scope="", - reuse_dst_scope=False): - """Copy a subgraph, replacing some of its inputs. - - Note a replacement only happens if the tensor to be replaced - is an input of the given subgraph. The inputs of a subgraph can - be queried using sgv.inputs. - - Args: - sgv: the source subgraph-view. This argument is converted to a subgraph - using the same rules as the function subgraph.make_view. - replacement_ts: dictionary mapping from original tensors to the - replaced one. - dst_graph: the destination graph. - dst_scope: the destination scope. - src_scope: the source scope. - reuse_dst_scope: if True the dst_scope is re-used if it already exists. - Otherwise, the scope is given a unique name based on the one given - by appending an underscore followed by a digit (default). - Returns: - A tuple `(sgv, info)` where: - `sgv` is the transformed subgraph view; - `info` is an instance of TransformerInfo containing - information about the transform, including mapping between - original and transformed tensors and operations. - Raises: - TypeError: if dst_graph is not a tf.Graph. - StandardError: if sgv cannot be converted to a SubGraphView using - the same rules as the function subgraph.make_view. - """ - sgv = subgraph.make_view(sgv) - if dst_graph is None: - dst_graph = sgv.graph - if not isinstance(dst_graph, tf_ops.Graph): - raise TypeError("Expected a tf.Graph, got: {}".format(type(dst_graph))) - - copier = Transformer() - # Replace tensor if possible. - def replace_t_with_replacement_handler(info, t): - if t in replacement_ts: - return replacement_ts[t] - else: - return keep_t_if_possible_handler(info, t) - copier.transform_external_input_handler = replace_t_with_replacement_handler - return copier( - sgv, dst_graph, dst_scope, src_scope, reuse_dst_scope=reuse_dst_scope) - - -def _add_control_flow_ops(ops, control_ios): - """Complete `ops` so that the transformed graph is valid. - - Partially copying a graph can lead to a malformed graph. For instance, - copying half of a while construct is likely to result in an invalid graph. - This function attempts to add missing ops so that the transformation result - in a valid graph. - - Args: - ops: list of ops (modifed in-place). - control_ios: object created by a call to `util.ControlOutputs`. - """ - # Find while contexts. - control_flow_contexts = set() - for op in ops: - cfc = op._control_flow_context # pylint: disable=protected-access - if cfc: - control_flow_contexts.add(cfc) - # Find new ops. - new_ops = [] - for cfc in control_flow_contexts: - if cfc.IsWhileContext(): - new_ops += select.get_walks_intersection_ops( - [enter_t.op for enter_t in cfc.loop_enters], - [exit_t.op for exit_t in cfc.loop_exits], - control_ios=control_ios) - # Add new ops. - new_ops_set = set(new_ops) - ops_set = frozenset(ops) - for op in new_ops_set: - if op not in ops_set: - ops.append(op) - - -def graph_replace(target_ts, replacement_ts, dst_scope="", - src_scope="", reuse_dst_scope=False): - """Create a new graph which compute the targets from the replaced Tensors. - - Args: - target_ts: a single tf.Tensor or an iterable of tf.Tensor. - replacement_ts: dictionary mapping from original tensors to replaced tensors - dst_scope: the destination scope. - src_scope: the source scope. - reuse_dst_scope: if True the dst_scope is re-used if it already exists. - Otherwise, the scope is given a unique name based on the one given - by appending an underscore followed by a digit (default). - Returns: - A single tf.Tensor or a list of target tf.Tensor, depending on - the type of the input argument `target_ts`. - The returned tensors are recomputed using the tensors from replacement_ts. - Raises: - ValueError: if the targets are not connected to replacement_ts. - """ - # Identify operations in the graph that will change. - # Start forward walk at Tensors that will be replaced, and - # backward walk at the target output Tensors. - flatten_target_ts = util.flatten_tree(target_ts) - # Construct the forward control dependencies edges so that - # the get_walks_intersection_ops can also traverse the - # control dependencies. - graph = util.get_unique_graph(flatten_target_ts, check_types=(tf_ops.Tensor)) - control_ios = util.ControlOutputs(graph) - ops = select.get_walks_intersection_ops( - list(replacement_ts), flatten_target_ts, control_ios=control_ios) - if not ops: - raise ValueError("Targets and replacements are not connected!") - - # Complete ops to avoid malformed control flow. - # TODO(fkp): Consider moving this function deeper (in the transformer?). - _add_control_flow_ops(ops, control_ios) - - # Create a copy of the relevant subgraph - unused_sgv_, info = copy_with_input_replacements( - ops, replacement_ts, None, dst_scope, src_scope, reuse_dst_scope) - # Return the transformed targets but keep the original if the transformed - # counterpart cannot be found - missing_fn = lambda original_t: original_t - return info.transformed(target_ts, missing_fn) diff --git a/tensorflow/contrib/graph_editor/util.py b/tensorflow/contrib/graph_editor/util.py deleted file mode 100644 index 543c1da7e33..00000000000 --- a/tensorflow/contrib/graph_editor/util.py +++ /dev/null @@ -1,566 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utility functions for the graph_editor. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re -from six import iteritems -from tensorflow.python.framework import ops as tf_ops -from tensorflow.python.ops import array_ops as tf_array_ops -from tensorflow.python.util.compat import collections_abc - -__all__ = [ - "make_list_of_op", - "get_tensors", - "make_list_of_t", - "get_generating_ops", - "get_consuming_ops", - "ControlOutputs", - "placeholder_name", - "make_placeholder_from_tensor", - "make_placeholder_from_dtype_and_shape", -] - - -# The graph editor sometimes need to create placeholders, they are named -# "geph_*". "geph" stands for Graph-Editor PlaceHolder. -_DEFAULT_PLACEHOLDER_PREFIX = "geph" - - -def concatenate_unique(la, lb): - """Add all the elements of `lb` to `la` if they are not there already. - - The elements added to `la` maintain ordering with respect to `lb`. - - Args: - la: List of Python objects. - lb: List of Python objects. - Returns: - `la`: The list `la` with missing elements from `lb`. - """ - la_set = set(la) - for l in lb: - if l not in la_set: - la.append(l) - la_set.add(l) - return la - - -# TODO(fkp): very generic code, it should be moved in a more generic place. -class ListView(object): - """Immutable list wrapper. - - This class is strongly inspired by the one in tf.Operation. - """ - - def __init__(self, list_): - if not isinstance(list_, list): - raise TypeError("Expected a list, got: {}.".format(type(list_))) - self._list = list_ - - def __iter__(self): - return iter(self._list) - - def __len__(self): - return len(self._list) - - def __bool__(self): - return bool(self._list) - - # Python 3 wants __bool__, Python 2.7 wants __nonzero__ - __nonzero__ = __bool__ - - def __getitem__(self, i): - return self._list[i] - - def __add__(self, other): - if not isinstance(other, list): - other = list(other) - return list(self) + other - - -# TODO(fkp): very generic code, it should be moved in a more generic place. -def is_iterable(obj): - """Return true if the object is iterable.""" - if isinstance(obj, tf_ops.Tensor): - return False - try: - _ = iter(obj) - except Exception: # pylint: disable=broad-except - return False - return True - - -def flatten_tree(tree, leaves=None): - """Flatten a tree into a list. - - Args: - tree: iterable or not. If iterable, its elements (child) can also be - iterable or not. - leaves: list to which the tree leaves are appended (None by default). - Returns: - A list of all the leaves in the tree. - """ - if leaves is None: - leaves = [] - if isinstance(tree, dict): - for _, child in iteritems(tree): - flatten_tree(child, leaves) - elif is_iterable(tree): - for child in tree: - flatten_tree(child, leaves) - else: - leaves.append(tree) - return leaves - - -def transform_tree(tree, fn, iterable_type=tuple): - """Transform all the nodes of a tree. - - Args: - tree: iterable or not. If iterable, its elements (child) can also be - iterable or not. - fn: function to apply to each leaves. - iterable_type: type use to construct the resulting tree for unknown - iterable, typically `list` or `tuple`. - Returns: - A tree whose leaves has been transformed by `fn`. - The hierarchy of the output tree mimics the one of the input tree. - """ - if is_iterable(tree): - if isinstance(tree, dict): - res = tree.__new__(type(tree)) - res.__init__( - (k, transform_tree(child, fn)) for k, child in iteritems(tree)) - return res - elif isinstance(tree, tuple): - # NamedTuple? - if hasattr(tree, "_asdict"): - res = tree.__new__(type(tree), **transform_tree(tree._asdict(), fn)) - else: - res = tree.__new__(type(tree), - (transform_tree(child, fn) for child in tree)) - return res - elif isinstance(tree, collections_abc.Sequence): - res = tree.__new__(type(tree)) - res.__init__(transform_tree(child, fn) for child in tree) - return res - else: - return iterable_type(transform_tree(child, fn) for child in tree) - else: - return fn(tree) - - -def check_graphs(*args): - """Check that all the element in args belong to the same graph. - - Args: - *args: a list of object with a obj.graph property. - Raises: - ValueError: if all the elements do not belong to the same graph. - """ - graph = None - for i, sgv in enumerate(args): - if graph is None and sgv.graph is not None: - graph = sgv.graph - elif sgv.graph is not None and sgv.graph is not graph: - raise ValueError("Argument[{}]: Wrong graph!".format(i)) - - -def get_unique_graph(tops, check_types=None, none_if_empty=False): - """Return the unique graph used by the all the elements in tops. - - Args: - tops: list of elements to check (usually a list of tf.Operation and/or - tf.Tensor). Or a tf.Graph. - check_types: check that the element in tops are of given type(s). If None, - the types (tf.Operation, tf.Tensor) are used. - none_if_empty: don't raise an error if tops is an empty list, just return - None. - Returns: - The unique graph used by all the tops. - Raises: - TypeError: if tops is not a iterable of tf.Operation. - ValueError: if the graph is not unique. - """ - if isinstance(tops, tf_ops.Graph): - return tops - if not is_iterable(tops): - raise TypeError("{} is not iterable".format(type(tops))) - if check_types is None: - check_types = (tf_ops.Operation, tf_ops.Tensor) - elif not is_iterable(check_types): - check_types = (check_types,) - g = None - for op in tops: - if not isinstance(op, check_types): - raise TypeError("Expected a type in ({}), got: {}".format(", ".join([str( - t) for t in check_types]), type(op))) - if g is None: - g = op.graph - elif g is not op.graph: - raise ValueError("Operation {} does not belong to given graph".format(op)) - if g is None and not none_if_empty: - raise ValueError("Can't find the unique graph of an empty list") - return g - - -def make_list_of_op(ops, check_graph=True, allow_graph=True, ignore_ts=False): - """Convert ops to a list of `tf.Operation`. - - Args: - ops: can be an iterable of `tf.Operation`, a `tf.Graph` or a single - operation. - check_graph: if `True` check if all the operations belong to the same graph. - allow_graph: if `False` a `tf.Graph` cannot be converted. - ignore_ts: if True, silently ignore `tf.Tensor`. - Returns: - A newly created list of `tf.Operation`. - Raises: - TypeError: if ops cannot be converted to a list of `tf.Operation` or, - if `check_graph` is `True`, if all the ops do not belong to the - same graph. - """ - if isinstance(ops, tf_ops.Graph): - if allow_graph: - return ops.get_operations() - else: - raise TypeError("allow_graph is False: cannot convert a tf.Graph.") - else: - if not is_iterable(ops): - ops = [ops] - if not ops: - return [] - if check_graph: - check_types = None if ignore_ts else tf_ops.Operation - get_unique_graph(ops, check_types=check_types) - return [op for op in ops if isinstance(op, tf_ops.Operation)] - - -# TODO(fkp): move this function in tf.Graph? -def get_tensors(graph): - """get all the tensors which are input or output of an op in the graph. - - Args: - graph: a `tf.Graph`. - Returns: - A list of `tf.Tensor`. - Raises: - TypeError: if graph is not a `tf.Graph`. - """ - if not isinstance(graph, tf_ops.Graph): - raise TypeError("Expected a graph, got: {}".format(type(graph))) - ts = [] - for op in graph.get_operations(): - ts += op.outputs - return ts - - -def make_list_of_t(ts, check_graph=True, allow_graph=True, ignore_ops=False): - """Convert ts to a list of `tf.Tensor`. - - Args: - ts: can be an iterable of `tf.Tensor`, a `tf.Graph` or a single tensor. - check_graph: if `True` check if all the tensors belong to the same graph. - allow_graph: if `False` a `tf.Graph` cannot be converted. - ignore_ops: if `True`, silently ignore `tf.Operation`. - Returns: - A newly created list of `tf.Tensor`. - Raises: - TypeError: if `ts` cannot be converted to a list of `tf.Tensor` or, - if `check_graph` is `True`, if all the ops do not belong to the same graph. - """ - if isinstance(ts, tf_ops.Graph): - if allow_graph: - return get_tensors(ts) - else: - raise TypeError("allow_graph is False: cannot convert a tf.Graph.") - else: - if not is_iterable(ts): - ts = [ts] - if not ts: - return [] - if check_graph: - check_types = None if ignore_ops else tf_ops.Tensor - get_unique_graph(ts, check_types=check_types) - return [t for t in ts if isinstance(t, tf_ops.Tensor)] - - -def get_generating_ops(ts): - """Return all the generating ops of the tensors in `ts`. - - Args: - ts: a list of `tf.Tensor` - Returns: - A list of all the generating `tf.Operation` of the tensors in `ts`. - Raises: - TypeError: if `ts` cannot be converted to a list of `tf.Tensor`. - """ - ts = make_list_of_t(ts, allow_graph=False) - return [t.op for t in ts] - - -def get_consuming_ops(ts): - """Return all the consuming ops of the tensors in ts. - - Args: - ts: a list of `tf.Tensor` - Returns: - A list of all the consuming `tf.Operation` of the tensors in `ts`. - Raises: - TypeError: if ts cannot be converted to a list of `tf.Tensor`. - """ - ts = make_list_of_t(ts, allow_graph=False) - ops = [] - for t in ts: - for op in t.consumers(): - if op not in ops: - ops.append(op) - return ops - - -class ControlOutputs(object): - """The control outputs topology.""" - - def __init__(self, graph): - """Create a dictionary of control-output dependencies. - - Args: - graph: a `tf.Graph`. - Returns: - A dictionary where a key is a `tf.Operation` instance and the - corresponding value is a list of all the ops which have the key - as one of their control-input dependencies. - Raises: - TypeError: graph is not a `tf.Graph`. - """ - if not isinstance(graph, tf_ops.Graph): - raise TypeError("Expected a tf.Graph, got: {}".format(type(graph))) - self._control_outputs = {} - self._graph = graph - self._version = None - self._build() - - def update(self): - """Update the control outputs if the graph has changed.""" - if self._version != self._graph.version: - self._build() - return self - - def _build(self): - """Build the control outputs dictionary.""" - self._control_outputs.clear() - ops = self._graph.get_operations() - for op in ops: - for control_input in op.control_inputs: - if control_input not in self._control_outputs: - self._control_outputs[control_input] = [] - if op not in self._control_outputs[control_input]: - self._control_outputs[control_input].append(op) - self._version = self._graph.version - - def get_all(self): - return self._control_outputs - - def get(self, op): - """return the control outputs of op.""" - if op in self._control_outputs: - return self._control_outputs[op] - else: - return () - - @property - def graph(self): - return self._graph - - -def scope_finalize(scope): - if scope and scope[-1] != "/": - scope += "/" - return scope - - -def scope_dirname(scope): - slash = scope.rfind("/") - if slash == -1: - return "" - return scope[:slash + 1] - - -def scope_basename(scope): - slash = scope.rfind("/") - if slash == -1: - return scope - return scope[slash + 1:] - - -def placeholder_name(t=None, scope=None, prefix=_DEFAULT_PLACEHOLDER_PREFIX): - """Create placeholder name for the graph editor. - - Args: - t: optional tensor on which the placeholder operation's name will be based - on - scope: absolute scope with which to prefix the placeholder's name. None - means that the scope of t is preserved. "" means the root scope. - prefix: placeholder name prefix. - Returns: - A new placeholder name prefixed by "geph". Note that "geph" stands for - Graph Editor PlaceHolder. This convention allows to quickly identify the - placeholder generated by the Graph Editor. - Raises: - TypeError: if t is not None or a tf.Tensor. - """ - if scope is not None: - scope = scope_finalize(scope) - if t is not None: - if not isinstance(t, tf_ops.Tensor): - raise TypeError("Expected a tf.Tenfor, got: {}".format(type(t))) - op_dirname = scope_dirname(t.op.name) - op_basename = scope_basename(t.op.name) - if scope is None: - scope = op_dirname - - if op_basename.startswith("{}__".format(prefix)): - ph_name = op_basename - else: - ph_name = "{}__{}_{}".format(prefix, op_basename, t.value_index) - - return scope + ph_name - else: - if scope is None: - scope = "" - return "{}{}".format(scope, prefix) - - -def make_placeholder_from_tensor(t, scope=None, - prefix=_DEFAULT_PLACEHOLDER_PREFIX): - """Create a `tf.compat.v1.placeholder` for the Graph Editor. - - Note that the correct graph scope must be set by the calling function. - - Args: - t: a `tf.Tensor` whose name will be used to create the placeholder (see - function placeholder_name). - scope: absolute scope within which to create the placeholder. None means - that the scope of `t` is preserved. `""` means the root scope. - prefix: placeholder name prefix. - - Returns: - A newly created `tf.compat.v1.placeholder`. - Raises: - TypeError: if `t` is not `None` or a `tf.Tensor`. - """ - return tf_array_ops.placeholder( - dtype=t.dtype, shape=t.get_shape(), - name=placeholder_name(t, scope=scope, prefix=prefix)) - - -def make_placeholder_from_dtype_and_shape(dtype, shape=None, scope=None, - prefix=_DEFAULT_PLACEHOLDER_PREFIX): - """Create a tf.compat.v1.placeholder for the Graph Editor. - - Note that the correct graph scope must be set by the calling function. - The placeholder is named using the function placeholder_name (with no - tensor argument). - - Args: - dtype: the tensor type. - shape: the tensor shape (optional). - scope: absolute scope within which to create the placeholder. None means - that the scope of t is preserved. "" means the root scope. - prefix: placeholder name prefix. - - Returns: - A newly created tf.placeholder. - """ - return tf_array_ops.placeholder( - dtype=dtype, shape=shape, - name=placeholder_name(scope=scope, prefix=prefix)) - - -_INTERNAL_VARIABLE_RE = re.compile(r"^__\w+__$") - - -def get_predefined_collection_names(): - """Return all the predefined collection names.""" - return [getattr(tf_ops.GraphKeys, key) for key in dir(tf_ops.GraphKeys) - if not _INTERNAL_VARIABLE_RE.match(key)] - - -def find_corresponding_elem(target, dst_graph, dst_scope="", src_scope=""): - """Find corresponding op/tensor in a different graph. - - Args: - target: A `tf.Tensor` or a `tf.Operation` belonging to the original graph. - dst_graph: The graph in which the corresponding graph element must be found. - dst_scope: A scope which is prepended to the name to look for. - src_scope: A scope which is removed from the original of `target` name. - - Returns: - The corresponding tf.Tensor` or a `tf.Operation`. - - Raises: - ValueError: if `src_name` does not start with `src_scope`. - TypeError: if `target` is not a `tf.Tensor` or a `tf.Operation` - KeyError: If the corresponding graph element cannot be found. - """ - src_name = target.name - if src_scope: - src_scope = scope_finalize(src_scope) - if not src_name.startswidth(src_scope): - raise ValueError("{} does not start with {}".format(src_name, src_scope)) - src_name = src_name[len(src_scope):] - - dst_name = src_name - if dst_scope: - dst_scope = scope_finalize(dst_scope) - dst_name = dst_scope + dst_name - - if isinstance(target, tf_ops.Tensor): - return dst_graph.get_tensor_by_name(dst_name) - if isinstance(target, tf_ops.Operation): - return dst_graph.get_operation_by_name(dst_name) - raise TypeError("Expected tf.Tensor or tf.Operation, got: {}", type(target)) - - -def find_corresponding(targets, dst_graph, dst_scope="", src_scope=""): - """Find corresponding ops/tensors in a different graph. - - `targets` is a Python tree, that is, a nested structure of iterable - (list, tupple, dictionary) whose leaves are instances of - `tf.Tensor` or `tf.Operation` - - Args: - targets: A Python tree containing `tf.Tensor` or `tf.Operation` - belonging to the original graph. - dst_graph: The graph in which the corresponding graph element must be found. - dst_scope: A scope which is prepended to the name to look for. - src_scope: A scope which is removed from the original of `top` name. - - Returns: - A Python tree containin the corresponding tf.Tensor` or a `tf.Operation`. - - Raises: - ValueError: if `src_name` does not start with `src_scope`. - TypeError: if `top` is not a `tf.Tensor` or a `tf.Operation` - KeyError: If the corresponding graph element cannot be found. - """ - def func(top): - return find_corresponding_elem(top, dst_graph, dst_scope, src_scope) - return transform_tree(targets, func) diff --git a/tensorflow/contrib/grid_rnn/BUILD b/tensorflow/contrib/grid_rnn/BUILD deleted file mode 100644 index addebb6f827..00000000000 --- a/tensorflow/contrib/grid_rnn/BUILD +++ /dev/null @@ -1,44 +0,0 @@ -# Description: -# Contains classes to construct GridRNN cells -# APIs here are meant to evolve over time. - -load("//tensorflow:tensorflow.bzl", "cuda_py_tests") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "grid_rnn_py", - srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:platform", - "//tensorflow/python:variable_scope", - ], -) - -cuda_py_tests( - name = "grid_rnn_test", - srcs = ["python/kernel_tests/grid_rnn_test.py"], - additional_deps = [ - ":grid_rnn_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:rnn", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) diff --git a/tensorflow/contrib/grid_rnn/__init__.py b/tensorflow/contrib/grid_rnn/__init__.py deleted file mode 100644 index e005dd2083f..00000000000 --- a/tensorflow/contrib/grid_rnn/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""GridRNN cells - -## This package provides classes for GridRNN - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import, line-too-long -from tensorflow.contrib.grid_rnn.python.ops.grid_rnn_cell import * -# pylint: enable=unused-import,wildcard-import,line-too-long diff --git a/tensorflow/contrib/grid_rnn/python/__init__.py b/tensorflow/contrib/grid_rnn/python/__init__.py deleted file mode 100644 index d2b5550aa61..00000000000 --- a/tensorflow/contrib/grid_rnn/python/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/grid_rnn/python/kernel_tests/grid_rnn_test.py b/tensorflow/contrib/grid_rnn/python/kernel_tests/grid_rnn_test.py deleted file mode 100644 index 363a3c9b4a5..00000000000 --- a/tensorflow/contrib/grid_rnn/python/kernel_tests/grid_rnn_test.py +++ /dev/null @@ -1,745 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for GridRNN cells.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.grid_rnn.python.ops import grid_rnn_cell -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import rnn -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class GridRNNCellTest(test.TestCase): - - def testGrid2BasicLSTMCell(self): - with self.test_session(use_gpu=False) as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.2)) as root_scope: - x = array_ops.zeros([1, 3]) - m = ((array_ops.zeros([1, 2]), array_ops.zeros([1, 2])), - (array_ops.zeros([1, 2]), array_ops.zeros([1, 2]))) - cell = grid_rnn_cell.Grid2BasicLSTMCell(2) - self.assertEqual(cell.state_size, ((2, 2), (2, 2))) - - g, s = cell(x, m) - self.assertEqual(g[0].get_shape(), (1, 2)) - self.assertEqual(s[0].c.get_shape(), (1, 2)) - self.assertEqual(s[0].h.get_shape(), (1, 2)) - self.assertEqual(s[1].c.get_shape(), (1, 2)) - self.assertEqual(s[1].h.get_shape(), (1, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g, res_s = sess.run([g, s], { - x: - np.array([[1., 1., 1.]]), - m: ((np.array([[0.1, 0.2]]), np.array([[0.3, 0.4]])), - (np.array([[0.5, 0.6]]), np.array([[0.7, 0.8]]))) - }) - self.assertEqual(res_g[0].shape, (1, 2)) - self.assertEqual(res_s[0].c.shape, (1, 2)) - self.assertEqual(res_s[0].h.shape, (1, 2)) - self.assertEqual(res_s[1].c.shape, (1, 2)) - self.assertEqual(res_s[1].h.shape, (1, 2)) - - self.assertAllClose(res_g, ([[0.36617181, 0.36617181]],)) - self.assertAllClose( - res_s, (([[0.71053141, 0.71053141]], [[0.36617181, 0.36617181]]), - ([[0.72320831, 0.80555487]], [[0.39102408, 0.42150158]]))) - - # emulate a loop through the input sequence, - # where we call cell() multiple times - root_scope.reuse_variables() - g2, s2 = cell(x, m) - self.assertEqual(g2[0].get_shape(), (1, 2)) - self.assertEqual(s2[0].c.get_shape(), (1, 2)) - self.assertEqual(s2[0].h.get_shape(), (1, 2)) - self.assertEqual(s2[1].c.get_shape(), (1, 2)) - self.assertEqual(s2[1].h.get_shape(), (1, 2)) - - res_g2, res_s2 = sess.run([g2, s2], - {x: np.array([[2., 2., 2.]]), - m: res_s}) - self.assertEqual(res_g2[0].shape, (1, 2)) - self.assertEqual(res_s2[0].c.shape, (1, 2)) - self.assertEqual(res_s2[0].h.shape, (1, 2)) - self.assertEqual(res_s2[1].c.shape, (1, 2)) - self.assertEqual(res_s2[1].h.shape, (1, 2)) - self.assertAllClose(res_g2[0], [[0.58847463, 0.58847463]]) - self.assertAllClose( - res_s2, (([[1.40469193, 1.40469193]], [[0.58847463, 0.58847463]]), - ([[0.97726452, 1.04626071]], [[0.4927212, 0.51137757]]))) - - def testGrid2BasicLSTMCellTied(self): - with self.test_session(use_gpu=False) as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.2)): - x = array_ops.zeros([1, 3]) - m = ((array_ops.zeros([1, 2]), array_ops.zeros([1, 2])), - (array_ops.zeros([1, 2]), array_ops.zeros([1, 2]))) - cell = grid_rnn_cell.Grid2BasicLSTMCell(2, tied=True) - self.assertEqual(cell.state_size, ((2, 2), (2, 2))) - - g, s = cell(x, m) - self.assertEqual(g[0].get_shape(), (1, 2)) - self.assertEqual(s[0].c.get_shape(), (1, 2)) - self.assertEqual(s[0].h.get_shape(), (1, 2)) - self.assertEqual(s[1].c.get_shape(), (1, 2)) - self.assertEqual(s[1].h.get_shape(), (1, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g, res_s = sess.run([g, s], { - x: - np.array([[1., 1., 1.]]), - m: ((np.array([[0.1, 0.2]]), np.array([[0.3, 0.4]])), - (np.array([[0.5, 0.6]]), np.array([[0.7, 0.8]]))) - }) - self.assertEqual(res_g[0].shape, (1, 2)) - self.assertEqual(res_s[0].c.shape, (1, 2)) - self.assertEqual(res_s[0].h.shape, (1, 2)) - self.assertEqual(res_s[1].c.shape, (1, 2)) - self.assertEqual(res_s[1].h.shape, (1, 2)) - - self.assertAllClose(res_g[0], [[0.36617181, 0.36617181]]) - self.assertAllClose( - res_s, (([[0.71053141, 0.71053141]], [[0.36617181, 0.36617181]]), - ([[0.72320831, 0.80555487]], [[0.39102408, 0.42150158]]))) - - res_g, res_s = sess.run([g, s], {x: np.array([[1., 1., 1.]]), m: res_s}) - self.assertEqual(res_g[0].shape, (1, 2)) - - self.assertAllClose(res_g[0], [[0.36703536, 0.36703536]]) - self.assertAllClose( - res_s, (([[0.71200621, 0.71200621]], [[0.36703536, 0.36703536]]), - ([[0.80941606, 0.87550586]], [[0.40108523, 0.42199609]]))) - - def testGrid2BasicLSTMCellWithRelu(self): - with self.test_session(use_gpu=False) as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.2)): - x = array_ops.zeros([1, 3]) - m = ((array_ops.zeros([1, 2]), array_ops.zeros([1, 2])),) - cell = grid_rnn_cell.Grid2BasicLSTMCell( - 2, tied=False, non_recurrent_fn=nn_ops.relu) - self.assertEqual(cell.state_size, ((2, 2),)) - - g, s = cell(x, m) - self.assertEqual(g[0].get_shape(), (1, 2)) - self.assertEqual(s[0].c.get_shape(), (1, 2)) - self.assertEqual(s[0].h.get_shape(), (1, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g, res_s = sess.run([g, s], { - x: np.array([[1., 1., 1.]]), - m: ((np.array([[0.1, 0.2]]), np.array([[0.3, 0.4]])),) - }) - self.assertEqual(res_g[0].shape, (1, 2)) - self.assertAllClose(res_g[0], [[0.31667367, 0.31667367]]) - self.assertAllClose(res_s, (([[0.29530135, 0.37520045]], - [[0.17044567, 0.21292259]]),)) - - """LSTMCell - """ - - def testGrid2LSTMCell(self): - with self.test_session(use_gpu=False) as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 3]) - m = ((array_ops.zeros([1, 2]), array_ops.zeros([1, 2])), - (array_ops.zeros([1, 2]), array_ops.zeros([1, 2]))) - cell = grid_rnn_cell.Grid2LSTMCell(2, use_peepholes=True) - self.assertEqual(cell.state_size, ((2, 2), (2, 2))) - - g, s = cell(x, m) - self.assertEqual(g[0].get_shape(), (1, 2)) - self.assertEqual(s[0].c.get_shape(), (1, 2)) - self.assertEqual(s[0].h.get_shape(), (1, 2)) - self.assertEqual(s[1].c.get_shape(), (1, 2)) - self.assertEqual(s[1].h.get_shape(), (1, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g, res_s = sess.run([g, s], { - x: - np.array([[1., 1., 1.]]), - m: ((np.array([[0.1, 0.2]]), np.array([[0.3, 0.4]])), - (np.array([[0.5, 0.6]]), np.array([[0.7, 0.8]]))) - }) - self.assertEqual(res_g[0].shape, (1, 2)) - self.assertEqual(res_s[0].c.shape, (1, 2)) - self.assertEqual(res_s[0].h.shape, (1, 2)) - self.assertEqual(res_s[1].c.shape, (1, 2)) - self.assertEqual(res_s[1].h.shape, (1, 2)) - - self.assertAllClose(res_g[0], [[0.95686918, 0.95686918]]) - self.assertAllClose( - res_s, (([[2.41515064, 2.41515064]], [[0.95686918, 0.95686918]]), - ([[1.38917875, 1.49043763]], [[0.83884692, 0.86036491]]))) - - def testGrid2LSTMCellTied(self): - with self.test_session(use_gpu=False) as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 3]) - m = ((array_ops.zeros([1, 2]), array_ops.zeros([1, 2])), - (array_ops.zeros([1, 2]), array_ops.zeros([1, 2]))) - cell = grid_rnn_cell.Grid2LSTMCell(2, tied=True, use_peepholes=True) - self.assertEqual(cell.state_size, ((2, 2), (2, 2))) - - g, s = cell(x, m) - self.assertEqual(g[0].get_shape(), (1, 2)) - self.assertEqual(s[0].c.get_shape(), (1, 2)) - self.assertEqual(s[0].h.get_shape(), (1, 2)) - self.assertEqual(s[1].c.get_shape(), (1, 2)) - self.assertEqual(s[1].h.get_shape(), (1, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g, res_s = sess.run([g, s], { - x: - np.array([[1., 1., 1.]]), - m: ((np.array([[0.1, 0.2]]), np.array([[0.3, 0.4]])), - (np.array([[0.5, 0.6]]), np.array([[0.7, 0.8]]))) - }) - self.assertEqual(res_g[0].shape, (1, 2)) - self.assertEqual(res_s[0].c.shape, (1, 2)) - self.assertEqual(res_s[0].h.shape, (1, 2)) - self.assertEqual(res_s[1].c.shape, (1, 2)) - self.assertEqual(res_s[1].h.shape, (1, 2)) - - self.assertAllClose(res_g[0], [[0.95686918, 0.95686918]]) - self.assertAllClose( - res_s, (([[2.41515064, 2.41515064]], [[0.95686918, 0.95686918]]), - ([[1.38917875, 1.49043763]], [[0.83884692, 0.86036491]]))) - - def testGrid2LSTMCellWithRelu(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 3]) - m = ((array_ops.zeros([1, 2]), array_ops.zeros([1, 2])),) - cell = grid_rnn_cell.Grid2LSTMCell( - 2, use_peepholes=True, non_recurrent_fn=nn_ops.relu) - self.assertEqual(cell.state_size, ((2, 2),)) - - g, s = cell(x, m) - self.assertEqual(g[0].get_shape(), (1, 2)) - self.assertEqual(s[0].c.get_shape(), (1, 2)) - self.assertEqual(s[0].h.get_shape(), (1, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g, res_s = sess.run([g, s], { - x: np.array([[1., 1., 1.]]), - m: ((np.array([[0.1, 0.2]]), np.array([[0.3, 0.4]])),) - }) - self.assertEqual(res_g[0].shape, (1, 2)) - self.assertAllClose(res_g[0], [[2.1831727, 2.1831727]]) - self.assertAllClose(res_s, (([[0.92270052, 1.02325559]], - [[0.66159075, 0.70475441]]),)) - - """RNNCell - """ - - def testGrid2BasicRNNCell(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([2, 2]) - m = (array_ops.zeros([2, 2]), array_ops.zeros([2, 2])) - cell = grid_rnn_cell.Grid2BasicRNNCell(2) - self.assertEqual(cell.state_size, (2, 2)) - - g, s = cell(x, m) - self.assertEqual(g[0].get_shape(), (2, 2)) - self.assertEqual(s[0].get_shape(), (2, 2)) - self.assertEqual(s[1].get_shape(), (2, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g, res_s = sess.run([g, s], { - x: - np.array([[1., 1.], [2., 2.]]), - m: (np.array([[0.1, 0.1], [0.2, 0.2]]), np.array([[0.1, 0.1], - [0.2, 0.2]])) - }) - self.assertEqual(res_g[0].shape, (2, 2)) - self.assertEqual(res_s[0].shape, (2, 2)) - self.assertEqual(res_s[1].shape, (2, 2)) - - self.assertAllClose(res_g, ([[0.94685763, 0.94685763], - [0.99480951, 0.99480951]],)) - self.assertAllClose( - res_s, ([[0.94685763, 0.94685763], [0.99480951, 0.99480951]], - [[0.80049908, 0.80049908], [0.97574311, 0.97574311]])) - - def testGrid2BasicRNNCellTied(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([2, 2]) - m = (array_ops.zeros([2, 2]), array_ops.zeros([2, 2])) - cell = grid_rnn_cell.Grid2BasicRNNCell(2, tied=True) - self.assertEqual(cell.state_size, (2, 2)) - - g, s = cell(x, m) - self.assertEqual(g[0].get_shape(), (2, 2)) - self.assertEqual(s[0].get_shape(), (2, 2)) - self.assertEqual(s[1].get_shape(), (2, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g, res_s = sess.run([g, s], { - x: - np.array([[1., 1.], [2., 2.]]), - m: (np.array([[0.1, 0.1], [0.2, 0.2]]), np.array([[0.1, 0.1], - [0.2, 0.2]])) - }) - self.assertEqual(res_g[0].shape, (2, 2)) - self.assertEqual(res_s[0].shape, (2, 2)) - self.assertEqual(res_s[1].shape, (2, 2)) - - self.assertAllClose(res_g, ([[0.94685763, 0.94685763], - [0.99480951, 0.99480951]],)) - self.assertAllClose( - res_s, ([[0.94685763, 0.94685763], [0.99480951, 0.99480951]], - [[0.80049908, 0.80049908], [0.97574311, 0.97574311]])) - - def testGrid2BasicRNNCellWithRelu(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 2]) - m = (array_ops.zeros([1, 2]),) - cell = grid_rnn_cell.Grid2BasicRNNCell(2, non_recurrent_fn=nn_ops.relu) - self.assertEqual(cell.state_size, (2,)) - - g, s = cell(x, m) - self.assertEqual(g[0].get_shape(), (1, 2)) - self.assertEqual(s[0].get_shape(), (1, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g, res_s = sess.run( - [g, s], {x: np.array([[1., 1.]]), - m: np.array([[0.1, 0.1]])}) - self.assertEqual(res_g[0].shape, (1, 2)) - self.assertEqual(res_s[0].shape, (1, 2)) - self.assertAllClose(res_g, ([[1.80049896, 1.80049896]],)) - self.assertAllClose(res_s, ([[0.80049896, 0.80049896]],)) - - """1-LSTM - """ - - def testGrid1LSTMCell(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)) as root_scope: - x = array_ops.zeros([1, 3]) - m = ((array_ops.zeros([1, 2]), array_ops.zeros([1, 2])),) - cell = grid_rnn_cell.Grid1LSTMCell(2, use_peepholes=True) - self.assertEqual(cell.state_size, ((2, 2),)) - - g, s = cell(x, m) - self.assertEqual(g[0].get_shape(), (1, 2)) - self.assertEqual(s[0].c.get_shape(), (1, 2)) - self.assertEqual(s[0].h.get_shape(), (1, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g, res_s = sess.run([g, s], { - x: np.array([[1., 1., 1.]]), - m: ((np.array([[0.1, 0.2]]), np.array([[0.3, 0.4]])),) - }) - self.assertEqual(res_g[0].shape, (1, 2)) - self.assertEqual(res_s[0].c.shape, (1, 2)) - self.assertEqual(res_s[0].h.shape, (1, 2)) - - self.assertAllClose(res_g, ([[0.91287315, 0.91287315]],)) - self.assertAllClose(res_s, (([[2.26285243, 2.26285243]], - [[0.91287315, 0.91287315]]),)) - - root_scope.reuse_variables() - - x2 = array_ops.zeros([0, 0]) - g2, s2 = cell(x2, m) - self.assertEqual(g2[0].get_shape(), (1, 2)) - self.assertEqual(s2[0].c.get_shape(), (1, 2)) - self.assertEqual(s2[0].h.get_shape(), (1, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g2, res_s2 = sess.run([g2, s2], {m: res_s}) - self.assertEqual(res_g2[0].shape, (1, 2)) - self.assertEqual(res_s2[0].c.shape, (1, 2)) - self.assertEqual(res_s2[0].h.shape, (1, 2)) - - self.assertAllClose(res_g2, ([[0.9032144, 0.9032144]],)) - self.assertAllClose(res_s2, (([[2.79966092, 2.79966092]], - [[0.9032144, 0.9032144]]),)) - - g3, s3 = cell(x2, m) - self.assertEqual(g3[0].get_shape(), (1, 2)) - self.assertEqual(s3[0].c.get_shape(), (1, 2)) - self.assertEqual(s3[0].h.get_shape(), (1, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g3, res_s3 = sess.run([g3, s3], {m: res_s2}) - self.assertEqual(res_g3[0].shape, (1, 2)) - self.assertEqual(res_s3[0].c.shape, (1, 2)) - self.assertEqual(res_s3[0].h.shape, (1, 2)) - self.assertAllClose(res_g3, ([[0.92727238, 0.92727238]],)) - self.assertAllClose(res_s3, (([[3.3529923, 3.3529923]], - [[0.92727238, 0.92727238]]),)) - - """3-LSTM - """ - - def testGrid3LSTMCell(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 3]) - m = ((array_ops.zeros([1, 2]), array_ops.zeros([1, 2])), - (array_ops.zeros([1, 2]), array_ops.zeros([1, 2])), - (array_ops.zeros([1, 2]), array_ops.zeros([1, 2]))) - cell = grid_rnn_cell.Grid3LSTMCell(2, use_peepholes=True) - self.assertEqual(cell.state_size, ((2, 2), (2, 2), (2, 2))) - - g, s = cell(x, m) - self.assertEqual(g[0].get_shape(), (1, 2)) - self.assertEqual(s[0].c.get_shape(), (1, 2)) - self.assertEqual(s[0].h.get_shape(), (1, 2)) - self.assertEqual(s[1].c.get_shape(), (1, 2)) - self.assertEqual(s[1].h.get_shape(), (1, 2)) - self.assertEqual(s[2].c.get_shape(), (1, 2)) - self.assertEqual(s[2].h.get_shape(), (1, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g, res_s = sess.run([g, s], { - x: - np.array([[1., 1., 1.]]), - m: ((np.array([[0.1, 0.2]]), np.array([[0.3, 0.4]])), - (np.array([[0.5, 0.6]]), np.array([[0.7, 0.8]])), (np.array( - [[-0.1, -0.2]]), np.array([[-0.3, -0.4]]))) - }) - self.assertEqual(res_g[0].shape, (1, 2)) - self.assertEqual(res_s[0].c.shape, (1, 2)) - self.assertEqual(res_s[0].h.shape, (1, 2)) - self.assertEqual(res_s[1].c.shape, (1, 2)) - self.assertEqual(res_s[1].h.shape, (1, 2)) - self.assertEqual(res_s[2].c.shape, (1, 2)) - self.assertEqual(res_s[2].h.shape, (1, 2)) - - self.assertAllClose(res_g, ([[0.96892911, 0.96892911]],)) - self.assertAllClose( - res_s, (([[2.45227885, 2.45227885]], [[0.96892911, 0.96892911]]), - ([[1.33592629, 1.4373529]], [[0.80867189, 0.83247656]]), - ([[0.7317788, 0.63205892]], [[0.56548983, 0.50446129]]))) - - """Edge cases - """ - - def testGridRNNEdgeCasesLikeRelu(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([3, 2]) - m = () - - # this is equivalent to relu - cell = grid_rnn_cell.GridRNNCell( - num_units=2, - num_dims=1, - input_dims=0, - output_dims=0, - non_recurrent_dims=0, - non_recurrent_fn=nn_ops.relu) - g, s = cell(x, m) - self.assertEqual(g[0].get_shape(), (3, 2)) - self.assertEqual(s, ()) - - sess.run([variables.global_variables_initializer()]) - res_g, res_s = sess.run([g, s], - {x: np.array([[1., -1.], [-2, 1], [2, -1]])}) - self.assertEqual(res_g[0].shape, (3, 2)) - self.assertEqual(res_s, ()) - self.assertAllClose(res_g, ([[0, 0], [0, 0], [0.5, 0.5]],)) - - def testGridRNNEdgeCasesNoOutput(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 2]) - m = ((array_ops.zeros([1, 2]), array_ops.zeros([1, 2])),) - - # This cell produces no output - cell = grid_rnn_cell.GridRNNCell( - num_units=2, - num_dims=2, - input_dims=0, - output_dims=None, - non_recurrent_dims=0, - non_recurrent_fn=nn_ops.relu) - g, s = cell(x, m) - self.assertEqual(g, ()) - self.assertEqual(s[0].c.get_shape(), (1, 2)) - self.assertEqual(s[0].h.get_shape(), (1, 2)) - - sess.run([variables.global_variables_initializer()]) - res_g, res_s = sess.run([g, s], { - x: np.array([[1., 1.]]), - m: ((np.array([[0.1, 0.1]]), np.array([[0.1, 0.1]])),) - }) - self.assertEqual(res_g, ()) - self.assertEqual(res_s[0].c.shape, (1, 2)) - self.assertEqual(res_s[0].h.shape, (1, 2)) - - """Test with tf.nn.rnn - """ - - def testGrid2LSTMCellWithRNN(self): - batch_size = 3 - input_size = 5 - max_length = 6 # unrolled up to this length - num_units = 2 - - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - cell = grid_rnn_cell.Grid2LSTMCell(num_units=num_units) - - inputs = max_length * [ - array_ops.placeholder( - dtypes.float32, shape=(batch_size, input_size)) - ] - - outputs, state = rnn.static_rnn(cell, inputs, dtype=dtypes.float32) - - self.assertEqual(len(outputs), len(inputs)) - self.assertEqual(state[0].c.get_shape(), (batch_size, 2)) - self.assertEqual(state[0].h.get_shape(), (batch_size, 2)) - self.assertEqual(state[1].c.get_shape(), (batch_size, 2)) - self.assertEqual(state[1].h.get_shape(), (batch_size, 2)) - - for out, inp in zip(outputs, inputs): - self.assertEqual(len(out), 1) - self.assertEqual(out[0].get_shape()[0], inp.get_shape()[0]) - self.assertEqual(out[0].get_shape()[1], num_units) - self.assertEqual(out[0].dtype, inp.dtype) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - - input_value = np.ones((batch_size, input_size)) - values = sess.run(outputs + [state], feed_dict={inputs[0]: input_value}) - for tp in values[:-1]: - for v in tp: - self.assertTrue(np.all(np.isfinite(v))) - for tp in values[-1]: - for st in tp: - for v in st: - self.assertTrue(np.all(np.isfinite(v))) - - def testGrid2LSTMCellReLUWithRNN(self): - batch_size = 3 - input_size = 5 - max_length = 6 # unrolled up to this length - num_units = 2 - - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - cell = grid_rnn_cell.Grid2LSTMCell( - num_units=num_units, non_recurrent_fn=nn_ops.relu) - - inputs = max_length * [ - array_ops.placeholder(dtypes.float32, shape=(batch_size, input_size)) - ] - - outputs, state = rnn.static_rnn(cell, inputs, dtype=dtypes.float32) - - self.assertEqual(len(outputs), len(inputs)) - self.assertEqual(state[0].c.get_shape(), (batch_size, 2)) - self.assertEqual(state[0].h.get_shape(), (batch_size, 2)) - - for out, inp in zip(outputs, inputs): - self.assertEqual(len(out), 1) - self.assertEqual(out[0].get_shape()[0], inp.get_shape()[0]) - self.assertEqual(out[0].get_shape()[1], num_units) - self.assertEqual(out[0].dtype, inp.dtype) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - - input_value = np.ones((batch_size, input_size)) - values = sess.run(outputs + [state], feed_dict={inputs[0]: input_value}) - for tp in values[:-1]: - for v in tp: - self.assertTrue(np.all(np.isfinite(v))) - for tp in values[-1]: - for st in tp: - for v in st: - self.assertTrue(np.all(np.isfinite(v))) - - def testGrid3LSTMCellReLUWithRNN(self): - batch_size = 3 - input_size = 5 - max_length = 6 # unrolled up to this length - num_units = 2 - - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - cell = grid_rnn_cell.Grid3LSTMCell( - num_units=num_units, non_recurrent_fn=nn_ops.relu) - - inputs = max_length * [ - array_ops.placeholder(dtypes.float32, shape=(batch_size, input_size)) - ] - - outputs, state = rnn.static_rnn(cell, inputs, dtype=dtypes.float32) - - self.assertEqual(len(outputs), len(inputs)) - self.assertEqual(state[0].c.get_shape(), (batch_size, 2)) - self.assertEqual(state[0].h.get_shape(), (batch_size, 2)) - self.assertEqual(state[1].c.get_shape(), (batch_size, 2)) - self.assertEqual(state[1].h.get_shape(), (batch_size, 2)) - - for out, inp in zip(outputs, inputs): - self.assertEqual(len(out), 1) - self.assertEqual(out[0].get_shape()[0], inp.get_shape()[0]) - self.assertEqual(out[0].get_shape()[1], num_units) - self.assertEqual(out[0].dtype, inp.dtype) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - - input_value = np.ones((batch_size, input_size)) - values = sess.run(outputs + [state], feed_dict={inputs[0]: input_value}) - for tp in values[:-1]: - for v in tp: - self.assertTrue(np.all(np.isfinite(v))) - for tp in values[-1]: - for st in tp: - for v in st: - self.assertTrue(np.all(np.isfinite(v))) - - def testGrid1LSTMCellWithRNN(self): - batch_size = 3 - input_size = 5 - max_length = 6 # unrolled up to this length - num_units = 2 - - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - cell = grid_rnn_cell.Grid1LSTMCell(num_units=num_units) - - # for 1-LSTM, we only feed the first step - inputs = ([ - array_ops.placeholder( - dtypes.float32, shape=(batch_size, input_size)) - ] + (max_length - 1) * [array_ops.zeros([batch_size, input_size])]) - - outputs, state = rnn.static_rnn(cell, inputs, dtype=dtypes.float32) - - self.assertEqual(len(outputs), len(inputs)) - self.assertEqual(state[0].c.get_shape(), (batch_size, 2)) - self.assertEqual(state[0].h.get_shape(), (batch_size, 2)) - - for out, inp in zip(outputs, inputs): - self.assertEqual(len(out), 1) - self.assertEqual(out[0].get_shape(), (3, num_units)) - self.assertEqual(out[0].dtype, inp.dtype) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - - input_value = np.ones((batch_size, input_size)) - values = sess.run(outputs + [state], feed_dict={inputs[0]: input_value}) - for tp in values[:-1]: - for v in tp: - self.assertTrue(np.all(np.isfinite(v))) - for tp in values[-1]: - for st in tp: - for v in st: - self.assertTrue(np.all(np.isfinite(v))) - - def testGrid2LSTMCellWithRNNAndDynamicBatchSize(self): - """Test for #4296.""" - input_size = 5 - max_length = 6 # unrolled up to this length - num_units = 2 - - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - cell = grid_rnn_cell.Grid2LSTMCell(num_units=num_units) - - inputs = max_length * [ - array_ops.placeholder(dtypes.float32, shape=(None, input_size)) - ] - - outputs, state = rnn.static_rnn(cell, inputs, dtype=dtypes.float32) - - self.assertEqual(len(outputs), len(inputs)) - - for out, inp in zip(outputs, inputs): - self.assertEqual(len(out), 1) - self.assertTrue(out[0].get_shape().dims[0].value is None) - self.assertEqual(out[0].get_shape().dims[1], num_units) - self.assertEqual(out[0].dtype, inp.dtype) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - - input_value = np.ones((3, input_size)) - values = sess.run(outputs + [state], feed_dict={inputs[0]: input_value}) - for tp in values[:-1]: - for v in tp: - self.assertTrue(np.all(np.isfinite(v))) - for tp in values[-1]: - for st in tp: - for v in st: - self.assertTrue(np.all(np.isfinite(v))) - - def testGrid2LSTMCellLegacy(self): - """Test for legacy case (when state_is_tuple=False).""" - with self.cached_session() as sess: - with variable_scope.variable_scope( - 'root', initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 3]) - m = array_ops.zeros([1, 8]) - cell = grid_rnn_cell.Grid2LSTMCell( - 2, use_peepholes=True, state_is_tuple=False, output_is_tuple=False) - self.assertEqual(cell.state_size, 8) - - g, s = cell(x, m) - self.assertEqual(g.get_shape(), (1, 2)) - self.assertEqual(s.get_shape(), (1, 8)) - - sess.run([variables.global_variables_initializer()]) - res = sess.run([g, s], { - x: np.array([[1., 1., 1.]]), - m: np.array([[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]]) - }) - self.assertEqual(res[0].shape, (1, 2)) - self.assertEqual(res[1].shape, (1, 8)) - self.assertAllClose(res[0], [[0.95686918, 0.95686918]]) - self.assertAllClose(res[1], [[ - 2.41515064, 2.41515064, 0.95686918, 0.95686918, 1.38917875, - 1.49043763, 0.83884692, 0.86036491 - ]]) - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/grid_rnn/python/ops/__init__.py b/tensorflow/contrib/grid_rnn/python/ops/__init__.py deleted file mode 100644 index d2b5550aa61..00000000000 --- a/tensorflow/contrib/grid_rnn/python/ops/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/grid_rnn/python/ops/grid_rnn_cell.py b/tensorflow/contrib/grid_rnn/python/ops/grid_rnn_cell.py deleted file mode 100644 index 5f3af43a474..00000000000 --- a/tensorflow/contrib/grid_rnn/python/ops/grid_rnn_cell.py +++ /dev/null @@ -1,668 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Module for constructing GridRNN cells""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from collections import namedtuple -import functools - -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import variable_scope as vs - -from tensorflow.python.platform import tf_logging as logging -from tensorflow.contrib import layers -from tensorflow.contrib import rnn - - -class GridRNNCell(rnn.RNNCell): - """Grid recurrent cell. - - This implementation is based on: - - http://arxiv.org/pdf/1507.01526v3.pdf - - This is the generic implementation of GridRNN. Users can specify arbitrary - number of dimensions, - set some of them to be priority (section 3.2), non-recurrent (section 3.3) - and input/output dimensions (section 3.4). - Weight sharing can also be specified using the `tied` parameter. - Type of recurrent units can be specified via `cell_fn`. - """ - - def __init__(self, - num_units, - num_dims=1, - input_dims=None, - output_dims=None, - priority_dims=None, - non_recurrent_dims=None, - tied=False, - cell_fn=None, - non_recurrent_fn=None, - state_is_tuple=True, - output_is_tuple=True): - """Initialize the parameters of a Grid RNN cell - - Args: - num_units: int, The number of units in all dimensions of this GridRNN cell - num_dims: int, Number of dimensions of this grid. - input_dims: int or list, List of dimensions which will receive input data. - output_dims: int or list, List of dimensions from which the output will be - recorded. - priority_dims: int or list, List of dimensions to be considered as - priority dimensions. - If None, no dimension is prioritized. - non_recurrent_dims: int or list, List of dimensions that are not - recurrent. - The transfer function for non-recurrent dimensions is specified - via `non_recurrent_fn`, which is - default to be `tensorflow.nn.relu`. - tied: bool, Whether to share the weights among the dimensions of this - GridRNN cell. - If there are non-recurrent dimensions in the grid, weights are - shared between each group of recurrent and non-recurrent - dimensions. - cell_fn: function, a function which returns the recurrent cell object. - Has to be in the following signature: - ``` - def cell_func(num_units): - # ... - ``` - and returns an object of type `RNNCell`. If None, LSTMCell with - default parameters will be used. - Note that if you use a custom RNNCell (with `cell_fn`), it is your - responsibility to make sure the inner cell use `state_is_tuple=True`. - - non_recurrent_fn: a tensorflow Op that will be the transfer function of - the non-recurrent dimensions - state_is_tuple: If True, accepted and returned states are tuples of the - states of the recurrent dimensions. If False, they are concatenated - along the column axis. The latter behavior will soon be deprecated. - - Note that if you use a custom RNNCell (with `cell_fn`), it is your - responsibility to make sure the inner cell use `state_is_tuple=True`. - - output_is_tuple: If True, the output is a tuple of the outputs of the - recurrent dimensions. If False, they are concatenated along the - column axis. The later behavior will soon be deprecated. - - Raises: - TypeError: if cell_fn does not return an RNNCell instance. - """ - if not state_is_tuple: - logging.warning('%s: Using a concatenated state is slower and will ' - 'soon be deprecated. Use state_is_tuple=True.', self) - if not output_is_tuple: - logging.warning('%s: Using a concatenated output is slower and will ' - 'soon be deprecated. Use output_is_tuple=True.', self) - - if num_dims < 1: - raise ValueError('dims must be >= 1: {}'.format(num_dims)) - - self._config = _parse_rnn_config(num_dims, input_dims, output_dims, - priority_dims, non_recurrent_dims, - non_recurrent_fn or nn.relu, tied, - num_units) - - self._state_is_tuple = state_is_tuple - self._output_is_tuple = output_is_tuple - - if cell_fn is None: - my_cell_fn = functools.partial( - rnn.LSTMCell, num_units=num_units, state_is_tuple=state_is_tuple) - else: - my_cell_fn = lambda: cell_fn(num_units) - if tied: - self._cells = [my_cell_fn()] * num_dims - else: - self._cells = [my_cell_fn() for _ in range(num_dims)] - if not isinstance(self._cells[0], rnn.RNNCell): - raise TypeError('cell_fn must return an RNNCell instance, saw: %s' % - type(self._cells[0])) - - if self._output_is_tuple: - self._output_size = tuple(self._cells[0].output_size - for _ in self._config.outputs) - else: - self._output_size = self._cells[0].output_size * len(self._config.outputs) - - if self._state_is_tuple: - self._state_size = tuple(self._cells[0].state_size - for _ in self._config.recurrents) - else: - self._state_size = self._cell_state_size() * len(self._config.recurrents) - - @property - def output_size(self): - return self._output_size - - @property - def state_size(self): - return self._state_size - - def __call__(self, inputs, state, scope=None): - """Run one step of GridRNN. - - Args: - inputs: input Tensor, 2D, batch x input_size. Or None - state: state Tensor, 2D, batch x state_size. Note that state_size = - cell_state_size * recurrent_dims - scope: VariableScope for the created subgraph; defaults to "GridRNNCell". - - Returns: - A tuple containing: - - - A 2D, batch x output_size, Tensor representing the output of the cell - after reading "inputs" when previous state was "state". - - A 2D, batch x state_size, Tensor representing the new state of the cell - after reading "inputs" when previous state was "state". - """ - conf = self._config - dtype = inputs.dtype - - c_prev, m_prev, cell_output_size = self._extract_states(state) - - new_output = [None] * conf.num_dims - new_state = [None] * conf.num_dims - - with vs.variable_scope(scope or type(self).__name__): # GridRNNCell - # project input, populate c_prev and m_prev - self._project_input(inputs, c_prev, m_prev, cell_output_size > 0) - - # propagate along dimensions, first for non-priority dimensions - # then priority dimensions - _propagate(conf.non_priority, conf, self._cells, c_prev, m_prev, - new_output, new_state, True) - _propagate(conf.priority, conf, self._cells, - c_prev, m_prev, new_output, new_state, False) - - # collect outputs and states - output_tensors = [new_output[i] for i in self._config.outputs] - if self._output_is_tuple: - output = tuple(output_tensors) - else: - if output_tensors: - output = array_ops.concat(output_tensors, 1) - else: - output = array_ops.zeros([0, 0], dtype) - - if self._state_is_tuple: - states = tuple(new_state[i] for i in self._config.recurrents) - else: - # concat each state first, then flatten the whole thing - state_tensors = [ - x for i in self._config.recurrents for x in new_state[i] - ] - if state_tensors: - states = array_ops.concat(state_tensors, 1) - else: - states = array_ops.zeros([0, 0], dtype) - - return output, states - - def _extract_states(self, state): - """Extract the cell and previous output tensors from the given state. - - Args: - state: The RNN state. - - Returns: - Tuple of the cell value, previous output, and cell_output_size. - - Raises: - ValueError: If len(self._config.recurrents) != len(state). - """ - conf = self._config - - # c_prev is `m` (cell value), and - # m_prev is `h` (previous output) in the paper. - # Keeping c and m here for consistency with the codebase - c_prev = [None] * conf.num_dims - m_prev = [None] * conf.num_dims - - # for LSTM : state = memory cell + output, hence cell_output_size > 0 - # for GRU/RNN: state = output (whose size is equal to _num_units), - # hence cell_output_size = 0 - total_cell_state_size = self._cell_state_size() - cell_output_size = total_cell_state_size - conf.num_units - - if self._state_is_tuple: - if len(conf.recurrents) != len(state): - raise ValueError('Expected state as a tuple of {} ' - 'element'.format(len(conf.recurrents))) - - for recurrent_dim, recurrent_state in zip(conf.recurrents, state): - if cell_output_size > 0: - c_prev[recurrent_dim], m_prev[recurrent_dim] = recurrent_state - else: - m_prev[recurrent_dim] = recurrent_state - else: - for recurrent_dim, start_idx in zip(conf.recurrents, - range(0, self.state_size, - total_cell_state_size)): - if cell_output_size > 0: - c_prev[recurrent_dim] = array_ops.slice(state, [0, start_idx], - [-1, conf.num_units]) - m_prev[recurrent_dim] = array_ops.slice( - state, [0, start_idx + conf.num_units], [-1, cell_output_size]) - else: - m_prev[recurrent_dim] = array_ops.slice(state, [0, start_idx], - [-1, conf.num_units]) - return c_prev, m_prev, cell_output_size - - def _project_input(self, inputs, c_prev, m_prev, with_c): - """Fills in c_prev and m_prev with projected input, for input dimensions. - - Args: - inputs: inputs tensor - c_prev: cell value - m_prev: previous output - with_c: boolean; whether to include project_c. - - Raises: - ValueError: if len(self._config.input) != len(inputs) - """ - conf = self._config - - if (inputs is not None and - tensor_shape.dimension_value(inputs.shape.with_rank(2)[1]) > 0 and - conf.inputs): - if isinstance(inputs, tuple): - if len(conf.inputs) != len(inputs): - raise ValueError('Expect inputs as a tuple of {} ' - 'tensors'.format(len(conf.inputs))) - input_splits = inputs - else: - input_splits = array_ops.split( - value=inputs, num_or_size_splits=len(conf.inputs), axis=1) - input_sz = tensor_shape.dimension_value( - input_splits[0].shape.with_rank(2)[1]) - - for i, j in enumerate(conf.inputs): - input_project_m = vs.get_variable( - 'project_m_{}'.format(j), [input_sz, conf.num_units], - dtype=inputs.dtype) - m_prev[j] = math_ops.matmul(input_splits[i], input_project_m) - - if with_c: - input_project_c = vs.get_variable( - 'project_c_{}'.format(j), [input_sz, conf.num_units], - dtype=inputs.dtype) - c_prev[j] = math_ops.matmul(input_splits[i], input_project_c) - - def _cell_state_size(self): - """Total size of the state of the inner cell used in this grid. - - Returns: - Total size of the state of the inner cell. - """ - state_sizes = self._cells[0].state_size - if isinstance(state_sizes, tuple): - return sum(state_sizes) - return state_sizes - - -"""Specialized cells, for convenience -""" - - -class Grid1BasicRNNCell(GridRNNCell): - """1D BasicRNN cell""" - - def __init__(self, num_units, state_is_tuple=True, output_is_tuple=True): - super(Grid1BasicRNNCell, self).__init__( - num_units=num_units, - num_dims=1, - input_dims=0, - output_dims=0, - priority_dims=0, - tied=False, - cell_fn=lambda n: rnn.BasicRNNCell(num_units=n), - state_is_tuple=state_is_tuple, - output_is_tuple=output_is_tuple) - - -class Grid2BasicRNNCell(GridRNNCell): - """2D BasicRNN cell - - This creates a 2D cell which receives input and gives output in the first - dimension. - - The first dimension can optionally be non-recurrent if `non_recurrent_fn` is - specified. - """ - - def __init__(self, - num_units, - tied=False, - non_recurrent_fn=None, - state_is_tuple=True, - output_is_tuple=True): - super(Grid2BasicRNNCell, self).__init__( - num_units=num_units, - num_dims=2, - input_dims=0, - output_dims=0, - priority_dims=0, - tied=tied, - non_recurrent_dims=None if non_recurrent_fn is None else 0, - cell_fn=lambda n: rnn.BasicRNNCell(num_units=n), - non_recurrent_fn=non_recurrent_fn, - state_is_tuple=state_is_tuple, - output_is_tuple=output_is_tuple) - - -class Grid1BasicLSTMCell(GridRNNCell): - """1D BasicLSTM cell.""" - - def __init__(self, - num_units, - forget_bias=1, - state_is_tuple=True, - output_is_tuple=True): - def cell_fn(n): - return rnn.BasicLSTMCell(num_units=n, forget_bias=forget_bias) - super(Grid1BasicLSTMCell, self).__init__( - num_units=num_units, - num_dims=1, - input_dims=0, - output_dims=0, - priority_dims=0, - tied=False, - cell_fn=cell_fn, - state_is_tuple=state_is_tuple, - output_is_tuple=output_is_tuple) - - -class Grid2BasicLSTMCell(GridRNNCell): - """2D BasicLSTM cell. - - This creates a 2D cell which receives input and gives output in the first - dimension. - - The first dimension can optionally be non-recurrent if `non_recurrent_fn` is - specified. - """ - - def __init__(self, - num_units, - tied=False, - non_recurrent_fn=None, - forget_bias=1, - state_is_tuple=True, - output_is_tuple=True): - def cell_fn(n): - return rnn.BasicLSTMCell(num_units=n, forget_bias=forget_bias) - super(Grid2BasicLSTMCell, self).__init__( - num_units=num_units, - num_dims=2, - input_dims=0, - output_dims=0, - priority_dims=0, - tied=tied, - non_recurrent_dims=None if non_recurrent_fn is None else 0, - cell_fn=cell_fn, - non_recurrent_fn=non_recurrent_fn, - state_is_tuple=state_is_tuple, - output_is_tuple=output_is_tuple) - - -class Grid1LSTMCell(GridRNNCell): - """1D LSTM cell. - - This is different from Grid1BasicLSTMCell because it gives options to - specify the forget bias and enabling peepholes. - """ - - def __init__(self, - num_units, - use_peepholes=False, - forget_bias=1.0, - state_is_tuple=True, - output_is_tuple=True): - - def cell_fn(n): - return rnn.LSTMCell( - num_units=n, forget_bias=forget_bias, use_peepholes=use_peepholes) - - super(Grid1LSTMCell, self).__init__( - num_units=num_units, - num_dims=1, - input_dims=0, - output_dims=0, - priority_dims=0, - cell_fn=cell_fn, - state_is_tuple=state_is_tuple, - output_is_tuple=output_is_tuple) - - -class Grid2LSTMCell(GridRNNCell): - """2D LSTM cell. - - This creates a 2D cell which receives input and gives output in the first - dimension. - The first dimension can optionally be non-recurrent if `non_recurrent_fn` is - specified. - """ - - def __init__(self, - num_units, - tied=False, - non_recurrent_fn=None, - use_peepholes=False, - forget_bias=1.0, - state_is_tuple=True, - output_is_tuple=True): - - def cell_fn(n): - return rnn.LSTMCell( - num_units=n, forget_bias=forget_bias, use_peepholes=use_peepholes) - - super(Grid2LSTMCell, self).__init__( - num_units=num_units, - num_dims=2, - input_dims=0, - output_dims=0, - priority_dims=0, - tied=tied, - non_recurrent_dims=None if non_recurrent_fn is None else 0, - cell_fn=cell_fn, - non_recurrent_fn=non_recurrent_fn, - state_is_tuple=state_is_tuple, - output_is_tuple=output_is_tuple) - - -class Grid3LSTMCell(GridRNNCell): - """3D BasicLSTM cell. - - This creates a 2D cell which receives input and gives output in the first - dimension. - The first dimension can optionally be non-recurrent if `non_recurrent_fn` is - specified. - The second and third dimensions are LSTM. - """ - - def __init__(self, - num_units, - tied=False, - non_recurrent_fn=None, - use_peepholes=False, - forget_bias=1.0, - state_is_tuple=True, - output_is_tuple=True): - - def cell_fn(n): - return rnn.LSTMCell( - num_units=n, forget_bias=forget_bias, use_peepholes=use_peepholes) - - super(Grid3LSTMCell, self).__init__( - num_units=num_units, - num_dims=3, - input_dims=0, - output_dims=0, - priority_dims=0, - tied=tied, - non_recurrent_dims=None if non_recurrent_fn is None else 0, - cell_fn=cell_fn, - non_recurrent_fn=non_recurrent_fn, - state_is_tuple=state_is_tuple, - output_is_tuple=output_is_tuple) - - -class Grid2GRUCell(GridRNNCell): - """2D LSTM cell. - - This creates a 2D cell which receives input and gives output in the first - dimension. - The first dimension can optionally be non-recurrent if `non_recurrent_fn` is - specified. - """ - - def __init__(self, - num_units, - tied=False, - non_recurrent_fn=None, - state_is_tuple=True, - output_is_tuple=True): - super(Grid2GRUCell, self).__init__( - num_units=num_units, - num_dims=2, - input_dims=0, - output_dims=0, - priority_dims=0, - tied=tied, - non_recurrent_dims=None if non_recurrent_fn is None else 0, - cell_fn=lambda n: rnn.GRUCell(num_units=n), - non_recurrent_fn=non_recurrent_fn, - state_is_tuple=state_is_tuple, - output_is_tuple=output_is_tuple) - - -# Helpers - -_GridRNNDimension = namedtuple('_GridRNNDimension', [ - 'idx', 'is_input', 'is_output', 'is_priority', 'non_recurrent_fn' -]) - -_GridRNNConfig = namedtuple('_GridRNNConfig', - ['num_dims', 'dims', 'inputs', 'outputs', - 'recurrents', 'priority', 'non_priority', 'tied', - 'num_units']) - - -def _parse_rnn_config(num_dims, ls_input_dims, ls_output_dims, ls_priority_dims, - ls_non_recurrent_dims, non_recurrent_fn, tied, num_units): - def check_dim_list(ls): - if ls is None: - ls = [] - if not isinstance(ls, (list, tuple)): - ls = [ls] - ls = sorted(set(ls)) - if any(_ < 0 or _ >= num_dims for _ in ls): - raise ValueError('Invalid dims: {}. Must be in [0, {})'.format(ls, - num_dims)) - return ls - - input_dims = check_dim_list(ls_input_dims) - output_dims = check_dim_list(ls_output_dims) - priority_dims = check_dim_list(ls_priority_dims) - non_recurrent_dims = check_dim_list(ls_non_recurrent_dims) - - rnn_dims = [] - for i in range(num_dims): - rnn_dims.append( - _GridRNNDimension( - idx=i, - is_input=(i in input_dims), - is_output=(i in output_dims), - is_priority=(i in priority_dims), - non_recurrent_fn=non_recurrent_fn - if i in non_recurrent_dims else None)) - return _GridRNNConfig( - num_dims=num_dims, - dims=rnn_dims, - inputs=input_dims, - outputs=output_dims, - recurrents=[x for x in range(num_dims) if x not in non_recurrent_dims], - priority=priority_dims, - non_priority=[x for x in range(num_dims) if x not in priority_dims], - tied=tied, - num_units=num_units) - - -def _propagate(dim_indices, conf, cells, c_prev, m_prev, new_output, new_state, - first_call): - """Propagates through all the cells in dim_indices dimensions. - """ - if len(dim_indices) == 0: - return - - # Because of the way RNNCells are implemented, we take the last dimension - # (H_{N-1}) out and feed it as the state of the RNN cell - # (in `last_dim_output`). - # The input of the cell (H_0 to H_{N-2}) are concatenated into `cell_inputs` - if conf.num_dims > 1: - ls_cell_inputs = [None] * (conf.num_dims - 1) - for d in conf.dims[:-1]: - if new_output[d.idx] is None: - ls_cell_inputs[d.idx] = m_prev[d.idx] - else: - ls_cell_inputs[d.idx] = new_output[d.idx] - cell_inputs = array_ops.concat(ls_cell_inputs, 1) - else: - cell_inputs = array_ops.zeros([m_prev[0].get_shape().as_list()[0], 0], - m_prev[0].dtype) - - last_dim_output = (new_output[-1] - if new_output[-1] is not None else m_prev[-1]) - - for i in dim_indices: - d = conf.dims[i] - if d.non_recurrent_fn: - if conf.num_dims > 1: - linear_args = array_ops.concat([cell_inputs, last_dim_output], 1) - else: - linear_args = last_dim_output - with vs.variable_scope('non_recurrent' if conf.tied else - 'non_recurrent/cell_{}'.format(i)): - if conf.tied and not (first_call and i == dim_indices[0]): - vs.get_variable_scope().reuse_variables() - - new_output[d.idx] = layers.fully_connected( - linear_args, - num_outputs=conf.num_units, - activation_fn=d.non_recurrent_fn, - weights_initializer=(vs.get_variable_scope().initializer or - layers.initializers.xavier_initializer), - weights_regularizer=vs.get_variable_scope().regularizer) - else: - if c_prev[i] is not None: - cell_state = (c_prev[i], last_dim_output) - else: - # for GRU/RNN, the state is just the previous output - cell_state = last_dim_output - - with vs.variable_scope('recurrent' if conf.tied else - 'recurrent/cell_{}'.format(i)): - if conf.tied and not (first_call and i == dim_indices[0]): - vs.get_variable_scope().reuse_variables() - cell = cells[i] - new_output[d.idx], new_state[d.idx] = cell(cell_inputs, cell_state) diff --git a/tensorflow/contrib/hadoop/BUILD b/tensorflow/contrib/hadoop/BUILD deleted file mode 100644 index c948b5080e4..00000000000 --- a/tensorflow/contrib/hadoop/BUILD +++ /dev/null @@ -1,118 +0,0 @@ -load( - "//tensorflow:tensorflow.bzl", - "tf_custom_op_library", - "tf_gen_op_libs", - "tf_gen_op_wrapper_py", - "tf_kernel_library", - "tf_py_test", -) -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -filegroup( - name = "test_data", - srcs = glob(["python/kernel_tests/testdata/*"]), -) - -py_library( - name = "hadoop", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - deps = [ - ":dataset_ops", - ], -) - -tf_custom_op_library( - name = "_dataset_ops.so", - srcs = ["ops/dataset_ops.cc"], - deps = [ - ":dataset_kernels", - ], -) - -tf_gen_op_libs( - op_lib_names = ["dataset_ops"], -) - -cc_library( - name = "dataset_kernels", - srcs = ["kernels/hadoop_dataset_ops.cc"], - deps = [ - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - "@com_google_protobuf//:protobuf_headers", - ], - alwayslink = 1, -) - -py_library( - name = "dataset_ops", - srcs = [ - "python/ops/hadoop_dataset_ops.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":hadoop_op_loader", - "//tensorflow/python:dataset_ops_gen", - "//tensorflow/python:util", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/util:nest", - ], -) - -tf_gen_op_wrapper_py( - name = "gen_dataset_ops", - out = "python/ops/gen_dataset_ops.py", - deps = ["//tensorflow/contrib/hadoop:dataset_ops_op_lib"], -) - -tf_kernel_library( - name = "dataset_ops_kernels", - deps = [ - ":dataset_kernels", - "//tensorflow/core:framework", - ], - alwayslink = 1, -) - -tf_custom_op_py_library( - name = "hadoop_op_loader", - srcs = ["python/ops/hadoop_op_loader.py"], - dso = ["//tensorflow/contrib/hadoop:_dataset_ops.so"], - kernels = [ - ":dataset_ops_kernels", - "//tensorflow/contrib/hadoop:dataset_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":gen_dataset_ops", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:platform", - ], -) - -tf_py_test( - name = "hadoop_test", - srcs = ["python/kernel_tests/hadoop_test.py"], - additional_deps = [ - ":hadoop", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], - data = [ - ":test_data", - ], - tags = [ - "notap", - ], -) diff --git a/tensorflow/contrib/hadoop/__init__.py b/tensorflow/contrib/hadoop/__init__.py deleted file mode 100644 index abf8cd4845f..00000000000 --- a/tensorflow/contrib/hadoop/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Sequence File Dataset. - -@@SequenceFileDataset -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.hadoop.python.ops.hadoop_dataset_ops import SequenceFileDataset - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - "SequenceFileDataset", -] - -remove_undocumented(__name__) diff --git a/tensorflow/contrib/hadoop/kernels/hadoop_dataset_ops.cc b/tensorflow/contrib/hadoop/kernels/hadoop_dataset_ops.cc deleted file mode 100644 index 243c2a40298..00000000000 --- a/tensorflow/contrib/hadoop/kernels/hadoop_dataset_ops.cc +++ /dev/null @@ -1,345 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/dataset.h" -#include "tensorflow/core/lib/io/buffered_inputstream.h" -#include "tensorflow/core/platform/file_system.h" - -namespace tensorflow { -namespace data { -namespace { - -static const size_t kSyncMarkerSize = 16; -static const size_t kSequenceFileBufferSize = 1024 * 1024; - -class SequenceFileReader { - public: - explicit SequenceFileReader(RandomAccessFile* file) - : input_stream_( - new io::BufferedInputStream(file, kSequenceFileBufferSize)) {} - - Status ReadHeader() { - tstring version; - TF_RETURN_IF_ERROR(input_stream_->ReadNBytes(4, &version)); - StringPiece version_view(version); - if (version_view.substr(0, 3) != "SEQ" || version[3] != 6) { - return errors::InvalidArgument( - "sequence file header must starts with `SEQ6`, received \"", - version_view.substr(0, 3), static_cast(version[3]), "\""); - } - TF_RETURN_IF_ERROR(ReadString(&key_class_name_)); - TF_RETURN_IF_ERROR(ReadString(&value_class_name_)); - - // At the moment we only support `org.apache.hadoop.io.Text` for key/value. - // TODO (yongtang): Add more class name support. - if (key_class_name_ != "org.apache.hadoop.io.Text" || - value_class_name_ != "org.apache.hadoop.io.Text") { - return errors::Unimplemented("key/value of '", key_class_name_, "/", - value_class_name_, - "' is currently not supported"); - } - - tstring buffer; - TF_RETURN_IF_ERROR(input_stream_->ReadNBytes(2, &buffer)); - compression_ = buffer[0]; - block_compression_ = buffer[1]; - if (compression_ || block_compression_) { - TF_RETURN_IF_ERROR(ReadString(&compression_codec_class_name_)); - } - - // At the moment no compression is supported. - // TODO (yongtang): Add compression support. - if (compression_ || block_compression_) { - return errors::Unimplemented("compression is currently not supported"); - } - - // Not interested in metadata for now. - uint32 num_metadata_pairs = 0; - TF_RETURN_IF_ERROR(ReadUInt32(&num_metadata_pairs)); - if (num_metadata_pairs > 1024) { - return errors::InvalidArgument( - "sequence file metadata should have key value pairs < 1024, " - "received ", - num_metadata_pairs); - } - for (int i = 0; i < num_metadata_pairs; i++) { - TF_RETURN_IF_ERROR(ReadString(nullptr)); - TF_RETURN_IF_ERROR(ReadString(nullptr)); - } - - TF_RETURN_IF_ERROR( - input_stream_->ReadNBytes(kSyncMarkerSize, &sync_marker_)); - - return Status::OK(); - } - - Status ReadRecord(tstring* key, tstring* value) { - uint32 length = 0; - TF_RETURN_IF_ERROR(ReadUInt32(&length)); - if (length == static_cast(-1)) { - // Sync marker. - tstring sync_marker; - TF_RETURN_IF_ERROR( - input_stream_->ReadNBytes(kSyncMarkerSize, &sync_marker)); - if (sync_marker != sync_marker_) { - return errors::InvalidArgument( - "sequence file should have sync marker \"", sync_marker_, - "\" at pos ", input_stream_->Tell() - kSyncMarkerSize, - ", received \"", sync_marker, "\""); - } - return ReadRecord(key, value); - } - uint32 key_length = 0; - TF_RETURN_IF_ERROR(ReadUInt32(&key_length)); - if (key_length > length) { - return errors::InvalidArgument("key length (", key_length, - ") should be < record length (", length, - ")"); - } - // At the moment we only support `org.apache.hadoop.io.Text` for key/value. - // TODO (yongtang): Expand supported format. - TF_RETURN_IF_ERROR(ReadString(key)); - TF_RETURN_IF_ERROR(ReadString(value)); - return Status::OK(); - } - - Status ReadString(tstring* value) { - int64 length = 0; - TF_RETURN_IF_ERROR(ReadVInt(&length)); - if (value == nullptr) { - return input_stream_->SkipNBytes(length); - } - return input_stream_->ReadNBytes(length, value); - } - - Status ReadUInt32(uint32* value) { - tstring buffer; - TF_RETURN_IF_ERROR(input_stream_->ReadNBytes(4, &buffer)); - *value = ((static_cast(buffer[0]) << 24) | - static_cast(buffer[1]) << 16) | - (static_cast(buffer[2]) << 8) | - static_cast(buffer[3]); - return Status::OK(); - } - - Status ReadVInt(int64* value) { - tstring buffer; - TF_RETURN_IF_ERROR(input_stream_->ReadNBytes(1, &buffer)); - if (buffer[0] >= -112) { - *value = static_cast(buffer[0]); - return Status::OK(); - } - - int64 remaining = 0; - bool negative = false; - if (buffer[0] >= -120) { - remaining = static_cast(-112) - static_cast(buffer[0]); - } else { - remaining = static_cast(-120) - static_cast(buffer[0]); - negative = true; - } - buffer.clear(); - TF_RETURN_IF_ERROR(input_stream_->ReadNBytes(remaining, &buffer)); - - uint64 v = 0; - for (int i = 0; i < buffer.size(); i++) { - v = (v << 8) | static_cast(buffer[i]); - } - if (negative) { - v = ~v; - } - *value = static_cast(v); - return Status::OK(); - } - - virtual ~SequenceFileReader() = default; - - private: - std::unique_ptr input_stream_; - tstring key_class_name_; - tstring value_class_name_; - tstring sync_marker_; - bool compression_; - bool block_compression_; - tstring compression_codec_class_name_; - TF_DISALLOW_COPY_AND_ASSIGN(SequenceFileReader); -}; -class SequenceFileDatasetOp : public DatasetOpKernel { - public: - using DatasetOpKernel::DatasetOpKernel; - explicit SequenceFileDatasetOp(OpKernelConstruction* ctx) - : DatasetOpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_)); - for (const DataType& dt : output_types_) { - OP_REQUIRES(ctx, dt == DT_STRING, - errors::InvalidArgument( - "Each element of `output_types_` must be one of: " - "DT_STRING")); - } - } - void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override { - const Tensor* filenames_tensor; - OP_REQUIRES_OK(ctx, ctx->input("filenames", &filenames_tensor)); - OP_REQUIRES( - ctx, filenames_tensor->dims() <= 1, - errors::InvalidArgument("`filenames` must be a scalar or a vector.")); - - std::vector filenames; - filenames.reserve(filenames_tensor->NumElements()); - for (int i = 0; i < filenames_tensor->NumElements(); ++i) { - filenames.push_back(filenames_tensor->flat()(i)); - } - - *output = new Dataset(ctx, filenames, output_types_); - } - - private: - class Dataset : public DatasetBase { - public: - Dataset(OpKernelContext* ctx, const std::vector& filenames, - const DataTypeVector& output_types) - : DatasetBase(DatasetContext(ctx)), - filenames_(filenames), - output_types_(output_types) {} - - std::unique_ptr MakeIteratorInternal( - const string& prefix) const override { - return std::unique_ptr( - new Iterator({this, strings::StrCat(prefix, "::SequenceFile")})); - } - - const DataTypeVector& output_dtypes() const override { - return output_types_; - } - - const std::vector& output_shapes() const override { - static std::vector* shapes = - new std::vector({{}, {}}); - return *shapes; - } - - string DebugString() const override { - return "SequenceFileDatasetOp::Dataset"; - } - - Status CheckExternalState() const override { return Status::OK(); } - - protected: - Status AsGraphDefInternal(SerializationContext* ctx, - DatasetGraphDefBuilder* b, - Node** output) const override { - Node* filenames = nullptr; - TF_RETURN_IF_ERROR(b->AddVector(filenames_, &filenames)); - TF_RETURN_IF_ERROR(b->AddDataset(this, {filenames}, output)); - return Status::OK(); - } - - private: - class Iterator : public DatasetIterator { - public: - explicit Iterator(const Params& params) - : DatasetIterator(params) {} - - Status GetNextInternal(IteratorContext* ctx, - std::vector* out_tensors, - bool* end_of_sequence) override { - mutex_lock l(mu_); - do { - // We are currently processing a file, so try to read the next record. - if (reader_) { - tstring key, value; - Status status = reader_->ReadRecord(&key, &value); - if (!errors::IsOutOfRange(status)) { - TF_RETURN_IF_ERROR(status); - - Tensor key_tensor(ctx->allocator({}), DT_STRING, {}); - key_tensor.scalar()() = std::move(key); - out_tensors->emplace_back(std::move(key_tensor)); - - Tensor value_tensor(ctx->allocator({}), DT_STRING, {}); - value_tensor.scalar()() = std::move(value); - out_tensors->emplace_back(std::move(value_tensor)); - - *end_of_sequence = false; - return Status::OK(); - } - // We have reached the end of the current file, so maybe - // move on to next file. - ResetStreamsLocked(); - ++current_file_index_; - } - - // Iteration ends when there are no more files to process. - if (current_file_index_ == dataset()->filenames_.size()) { - *end_of_sequence = true; - return Status::OK(); - } - - TF_RETURN_IF_ERROR(SetupStreamsLocked(ctx->env())); - } while (true); - } - - protected: - Status SaveInternal(IteratorStateWriter* writer) override { - return errors::Unimplemented("SaveInternal is currently not supported"); - } - - Status RestoreInternal(IteratorContext* ctx, - IteratorStateReader* reader) override { - return errors::Unimplemented( - "RestoreInternal is currently not supported"); - } - - private: - // Sets up SequenceFile streams to read from the topic at - // `current_file_index_`. - Status SetupStreamsLocked(Env* env) EXCLUSIVE_LOCKS_REQUIRED(mu_) { - if (current_file_index_ >= dataset()->filenames_.size()) { - return errors::InvalidArgument( - "current_file_index_:", current_file_index_, - " >= filenames_.size():", dataset()->filenames_.size()); - } - - // Actually move on to next file. - const string& filename = dataset()->filenames_[current_file_index_]; - TF_RETURN_IF_ERROR(env->NewRandomAccessFile(filename, &file_)); - reader_.reset(new SequenceFileReader(file_.get())); - return reader_->ReadHeader(); - } - - // Resets all Hadoop SequenceFile streams. - void ResetStreamsLocked() EXCLUSIVE_LOCKS_REQUIRED(mu_) { - reader_.reset(); - file_.reset(); - } - - mutex mu_; - size_t current_file_index_ GUARDED_BY(mu_) = 0; - std::unique_ptr file_ GUARDED_BY(mu_); - std::unique_ptr reader_ GUARDED_BY(mu_); - }; - - const std::vector filenames_; - const DataTypeVector output_types_; - }; - DataTypeVector output_types_; -}; - -REGISTER_KERNEL_BUILDER(Name("SequenceFileDataset").Device(DEVICE_CPU), - SequenceFileDatasetOp); - -} // namespace -} // namespace data -} // namespace tensorflow diff --git a/tensorflow/contrib/hadoop/ops/dataset_ops.cc b/tensorflow/contrib/hadoop/ops/dataset_ops.cc deleted file mode 100644 index 66ad549b475..00000000000 --- a/tensorflow/contrib/hadoop/ops/dataset_ops.cc +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { - -REGISTER_OP("SequenceFileDataset") - .Input("filenames: string") - .Output("handle: variant") - .Attr("output_types: list(type) >= 1") - .SetIsStateful() - .SetShapeFn(shape_inference::ScalarShape); - -} // namespace tensorflow diff --git a/tensorflow/contrib/hadoop/python/kernel_tests/hadoop_test.py b/tensorflow/contrib/hadoop/python/kernel_tests/hadoop_test.py deleted file mode 100644 index bc941ae9f23..00000000000 --- a/tensorflow/contrib/hadoop/python/kernel_tests/hadoop_test.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# ============================================================================== -"""Tests for SequenceFileDataset.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from tensorflow.contrib.hadoop.python.ops import hadoop_dataset_ops -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.platform import resource_loader -from tensorflow.python.platform import test - - -class SequenceFileDatasetTest(test.TestCase): - - def test_sequence_file_dataset(self): - """Test case for SequenceFileDataset. - - The file is generated with `org.apache.hadoop.io.Text` for key/value. - There are 25 records in the file with the format of: - key = XXX - value = VALUEXXX - where XXX is replaced as the line number (starts with 001). - """ - filename = os.path.join(resource_loader.get_data_files_path(), - "testdata", "string.seq") - - filenames = constant_op.constant([filename], dtypes.string) - num_repeats = 2 - - dataset = hadoop_dataset_ops.SequenceFileDataset(filenames).repeat( - num_repeats) - iterator = dataset_ops.make_initializable_iterator(dataset) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - for _ in range(num_repeats): # Dataset is repeated. - for i in range(25): # 25 records. - v0 = b"%03d" % (i + 1) - v1 = b"VALUE%03d" % (i + 1) - self.assertEqual((v0, v1), sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/hadoop/python/kernel_tests/testdata/string.seq b/tensorflow/contrib/hadoop/python/kernel_tests/testdata/string.seq deleted file mode 100755 index b7175338af3..00000000000 Binary files a/tensorflow/contrib/hadoop/python/kernel_tests/testdata/string.seq and /dev/null differ diff --git a/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py b/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py deleted file mode 100644 index c8c873cd9c2..00000000000 --- a/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""SequenceFile Dataset.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.hadoop.python.ops import gen_dataset_ops -from tensorflow.contrib.hadoop.python.ops import hadoop_op_loader # pylint: disable=unused-import -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_spec -from tensorflow.python.util import deprecation - - -class SequenceFileDataset(dataset_ops.DatasetSource): - """A Sequence File Dataset that reads the sequence file.""" - - @deprecation.deprecated( - None, - "tf.contrib.hadoop will be removed in 2.0, the support for Apache Hadoop " - "will continue to be provided through the tensorflow/io GitHub project.") - def __init__(self, filenames): - """Create a `SequenceFileDataset`. - - `SequenceFileDataset` allows a user to read data from a hadoop sequence - file. A sequence file consists of (key value) pairs sequentially. At - the moment, `org.apache.hadoop.io.Text` is the only serialization type - being supported, and there is no compression support. - - For example: - - ```python - tf.compat.v1.enable_eager_execution() - - dataset = tf.contrib.hadoop.SequenceFileDataset("/foo/bar.seq") - # Prints the (key, value) pairs inside a hadoop sequence file. - for key, value in dataset: - print(key, value) - ``` - - Args: - filenames: A `tf.string` tensor containing one or more filenames. - """ - self._filenames = ops.convert_to_tensor( - filenames, dtype=dtypes.string, name="filenames") - variant_tensor = gen_dataset_ops.sequence_file_dataset( - self._filenames, self._flat_types) - super(SequenceFileDataset, self).__init__(variant_tensor) - - @property - def element_spec(self): - return (tensor_spec.TensorSpec([], dtypes.string), - tensor_spec.TensorSpec([], dtypes.string)) diff --git a/tensorflow/contrib/hadoop/python/ops/hadoop_op_loader.py b/tensorflow/contrib/hadoop/python/ops/hadoop_op_loader.py deleted file mode 100644 index 6dbf1253f3f..00000000000 --- a/tensorflow/contrib/hadoop/python/ops/hadoop_op_loader.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python helper for loading hadoop ops and kernels.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.util import loader -from tensorflow.python.platform import resource_loader - -_dataset_ops = loader.load_op_library( - resource_loader.get_path_to_datafile("../../_dataset_ops.so")) diff --git a/tensorflow/contrib/hooks/BUILD b/tensorflow/contrib/hooks/BUILD deleted file mode 100644 index 5dd4fd4b2c8..00000000000 --- a/tensorflow/contrib/hooks/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -# Description: -# Contains `SessionRunHook`s for use with `MonitoredSession` and the -# wrappers around it. - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "hooks", - srcs = [ - "__init__.py", - "python/training/__init__.py", - "python/training/profiler_hook.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:training", - "//tensorflow/python:util", - ], -) diff --git a/tensorflow/contrib/hooks/README.md b/tensorflow/contrib/hooks/README.md deleted file mode 100644 index 84dd6ac8792..00000000000 --- a/tensorflow/contrib/hooks/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# TensorFlow Experimental SessionRunHooks - -These hooks complement those in tensorflow/python/training. They are instances -of `SessionRunHook` and are to be used with helpers like `MonitoredSession` -and `learn.Estimator` that wrap `tensorflow.Session`. - -The hooks are called between invocations of `Session.run()` to perform custom -behavior. - -For example the `ProfilerHook` periodically collects `RunMetadata` after -`Session.run()` and saves profiling information that can be viewed in a -neat timeline through a Chromium-based web browser (via -[about:tracing](chrome://tracing)) or the standalone [Catapult](https://github.com/catapult-project/catapult/blob/master/tracing/README.md) tool. - -```python -from tensorflow.contrib.hooks import ProfilerHook - -hooks = [ProfilerHook(save_secs=30, output_dir="profiling")] -with SingularMonitoredSession(hooks=hooks) as sess: - while not sess.should_stop(): - sess.run(some_op) -``` - -Or similarly with contrib.learn: - -```python -hooks = [ProfilerHook(save_steps=10, output_dir="profiling")] -estimator = learn.Estimator(...) -estimator.fit(input_fn, monitors=hooks) -``` diff --git a/tensorflow/contrib/hooks/__init__.py b/tensorflow/contrib/hooks/__init__.py deleted file mode 100644 index 4b7319f843d..00000000000 --- a/tensorflow/contrib/hooks/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""hooks: A module containing `SessionRunHook`s for use with `MonitoredSession`. - -@@ProfilerHook -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.hooks.python.training import * -# pylint: enable=wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = ['ProfilerHook'] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/hooks/python/__init__.py b/tensorflow/contrib/hooks/python/__init__.py deleted file mode 100644 index 8f9c49896e9..00000000000 --- a/tensorflow/contrib/hooks/python/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Experimental `SessionRunHooks` for use with `MonitoredSession`.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.hooks.python.training import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/hooks/python/training/__init__.py b/tensorflow/contrib/hooks/python/training/__init__.py deleted file mode 100644 index cc4726497f8..00000000000 --- a/tensorflow/contrib/hooks/python/training/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""hooks: A module containing `SessionRunHook`s for use with `MonitoredSession`. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.hooks.python.training.profiler_hook import ProfilerHook diff --git a/tensorflow/contrib/hooks/python/training/profiler_hook.py b/tensorflow/contrib/hooks/python/training/profiler_hook.py deleted file mode 100644 index 6173aa07971..00000000000 --- a/tensorflow/contrib/hooks/python/training/profiler_hook.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Placeholder of ProfilerHook for backward compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.training import basic_session_run_hooks - -ProfilerHook = basic_session_run_hooks.ProfilerHook # pylint: disable=invalid-name diff --git a/tensorflow/contrib/hvx/README.md b/tensorflow/contrib/hvx/README.md deleted file mode 100644 index 68e34f3b093..00000000000 --- a/tensorflow/contrib/hvx/README.md +++ /dev/null @@ -1,161 +0,0 @@ -# TensorFlow Runtime with HVX Acceleration - -This README explain how to build and use the TensorFlow runtime with HVX Acceleration. HVX is an extension of Hexagon, a DSP provided by Qualcomm, which can compute vector calculations faster using less energy than ARM processors. - -## Dependencies - -* [Android SDK](https://developer.android.com/studio/index.html). -* [Android NDK](https://developer.android.com/ndk/index.html). Save the path in `${NDK_ROOT}`. -* A rooted Qualcomm-based Android device connected to the computer (preferably, a [Snapdragon Development Board](https://developer.qualcomm.com/hardware/additional-snapdragon), but it could be a rooted phone with a Qualcomm SoC, albeit this guide may not work with it). The device needs to be rooted for development and testing purposes, and shouldn't be needed in production. See [Behold, The Snapdragon MDP](https://developer.qualcomm.com/blog/behold-snapdragon-mdp) for more information. -* [Hexagon SDK v3.0](https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools). Save the path in `${QUALCOMM_SDK}`. -* The current directory should be TensorFlow source code (`git clone https://github.com/tensorflow/tensorflow.git && cd tensorflow`), and saved into `${TF_ROOT_DIR}`. - -You may also need to add a test signature in the device to run HVX-based binaries. Follow the instructions in `${QUALCOMM_SDK}/docs/Tools_Signing.html`, using Python 2. - -Note that if the device is not rooted, you may not be able to get the serial number, push the test signature and/or run binary files that call HVX libraries. - -## Quick Start Guide - -We provide several tools to build and run inference with this runtime quickly. - -### Run inception model with a prebuilt Hexagon library - -If you don’t need to build your own implementation of Hexagon HVX, we provide a shortcut to execute graphs by using pre-compiled binaries. - -```shell -./tensorflow/contrib/makefile/samples/build_and_run_inception_hexagon.sh -p -``` - -The `-p` option makes the script download dependencies (i.e., Hexagon HVX binaries and graphs models), copy them to the Android device and execute a test. - -### Run inception model by building all from the source code - -If you want to build your own implementation of Hexagon HVX, we provide a sample all-in-one script to execute graphs which downloads the source and builds everything that's necessary. - -```shell -./tensorflow/contrib/makefile/samples/build_and_run_inception_hexagon.sh -``` - -## Building libraries - -If you've finished walking through the quick start guide, you may want to try building each binary manually. - -### Build libhexagon\_nn\_skel.so - -Download Hexagon NN library from codeaurora.org and build it. For Hexagon SDK 3.0, we need use the compatible version([721b2d58f](https://source.codeaurora.org/quic/hexagon_nn/nnlib/commit/?id=721b2d58f0f4e2d5b182f41e6b7c4db5356bf0fb)) of nnlib. - -```shell -git clone https://source.codeaurora.org/quic/hexagon_nn/nnlib -cd nnlib -git reset 721b2d58f --hard -``` - -Just follow the instructions in `README.HOW_TO_BUILD`. You can find the file `libhexagon_nn_skel.so` in `hexagon_Release_dynamic_toolv72_v60/ship`. -Then copy the generated binary to `${GEN_LIBS_DIR}`. - -```shell -GEN_LIBS_DIR="/path/to/a/dir/to/store/hexagon/libraries" -cp -v "hexagon_Release_dynamic_toolv72_v60/ship/libhexagon_nn_skel.so" "${GEN_LIBS_DIR}" -``` - -### Build libhexagon\_controller.so - -Download tensorflow and build hexagon controller. - -```shell -GENERATED_NNLIB_DIRECTORY="/path/to/nnlib" -GENERATED_HEXAGON_CONTROLLER_DIRECTORY="${QUALCOMM_SDK}/examples/common/generated_hexagon_controller" -rm -rf "${GENERATED_HEXAGON_CONTROLLER_DIRECTORY}" -cp -af "${TF_ROOT_DIR}/tensorflow/contrib/hvx/hexagon_controller" \ - "${GENERATED_HEXAGON_CONTROLLER_DIRECTORY}" -cp -afv "${GENERATED_NNLIB_DIRECTORY}/interface" \ -"${GENERATED_HEXAGON_CONTROLLER_DIRECTORY}/" -cp -afv "${GENERATED_NNLIB_DIRECTORY}/glue" \ -"${GENERATED_HEXAGON_CONTROLLER_DIRECTORY}/" -make clean V=android_Release -rm -rf android_Release -make tree VERBOSE=1 V=android_Release -cp -v "${GENERATED_HEXAGON_CONTROLLER_DIRECTORY}/android_Release/ship/libhexagon_controller.so" "${GEN_LIBS_DIR}" -``` - -### Build TensorFlow linking Hexagon library - -Build TensorFlow with `build_all_android.sh` specifying the `-x` option. - -```shell -BUILD_ALL_ANDROID_PATH="${TF_ROOT_DIR}/tensorflow/contrib/makefile/build_all_android.sh" - -CC_PREFIX=${CC_PREFIX} NDK_ROOT=${NDK_ROOT} "${BUILD_ALL_ANDROID_PATH}" \ --x "${GEN_LIBS_DIR}" \ --s "${TF_ROOT_DIR}/tensorflow/contrib/makefile/sub_makefiles/hexagon_graph_execution/Makefile.in" \ --t hexagon_graph_execution -``` - -### Push binaries to your Android device - -Before running tests on your Android device, you need to push several binaries to it. - -```shell -adb push "${GEN_LIBS_DIR}/libhexagon_controller.so" "/data/local/tmp" -adb push "${GEN_LIBS_DIR}/libhexagon_nn_skel.so" "/vendor/lib/rfsa/adsp" -adb push -p \ -"${TF_ROOT_DIR}/tensorflow/contrib/makefile/gen/bin/hexagon_graph_execution" \ -"/data/local/tmp/" -adb wait-for-device -ANDROID_EXEC_FILE_MODE=755 -adb shell chmod "${ANDROID_EXEC_FILE_MODE}" \ -"/data/local/tmp/hexagon_graph_execution" -adb wait-for-device -``` - -### Run tests on the device - -Finally, you can run the inference tests on your device. - -```shell -adb shell 'LD_LIBRARY_PATH=/data/local/tmp:$LD_LIBRARY_PATH' \ -"/data/local/tmp/hexagon_graph_execution" -``` - -### Troubleshooting - -#### Testsig issue - -If you're using the Open-Q 820 Snapdragon Development Kit, you may run into an issue with running the executable due to a missing `testsig` library. From the Hexagon SDK documentation: *Dynamic shared objects are required to be digitally signed and then authenticated at runtime before they are allowed to be loaded and executed.* Generating a testsig library is necessary to run the unsigned sample library built from this project. - -If the lack of a `testsig` library is your problem, you will see errors of the type: -`vendor/qcom/proprietary/adsprpc/src/fastrpc_apps_user.c:169::error: -1: 0 == (nErr = remotectl_open(name, (int*)ph, dlerrstr, sizeof(dlerrstr), &dlerr))` -appearing in `adb logcat` or ["Expected: (version) >= (1), actual: 0 vs 1" while running a binary from adb](https://github.com/tensorflow/tensorflow/issues/11210). - -You need to add a test signature, as described at the beginning of this README. After rebooting your device, you should be able to run the sample application. - -#### Qualcomm SDK Linux installation fails with "Malformed \uxxxx encoding" - -The installation file is based on LaunchAnywhere, which fails in Linux if the `PS1` env variable contains non-common Unicode chars: - -``` -Preparing to install... -Extracting the JRE from the installer archive... -Unpacking the JRE... -Extracting the installation resources from the installer archive... -Configuring the installer for this system's environment... - -Launching installer... - -An internal LaunchAnywhere application error has occurred and this application cannot proceed. (LAX) - -Stack Trace: -java.lang.IllegalArgumentException: Malformed \uxxxx encoding. - at java.util.Properties.loadConvert(Properties.java:574) - at java.util.Properties.load0(Properties.java:391) - at java.util.Properties.load(Properties.java:317) - at com.zerog.common.java.util.PropertiesUtil.loadProperties(Unknown Source) - at com.zerog.lax.LAX.(Unknown Source) - at com.zerog.lax.LAX.main(Unknown Source) -``` - -It can be solved by temporarily assigning the `PS1` environment variable to something simple, such as '$'. - -## Maintainers - -* Satoshi Kataoka (satok@google.com, github.com/satok16) diff --git a/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD b/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD deleted file mode 100644 index dc0eea3f2e5..00000000000 --- a/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD +++ /dev/null @@ -1,49 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -load( - "//tensorflow:tensorflow.bzl", - "tf_cc_binary", - "tf_copts", -) - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_cc_binary( - name = "clock_cycle_profiling", - testonly = 1, - srcs = ["clock_cycle_profiling_main.cc"], - copts = tf_copts(), - linkopts = select({ - "//tensorflow:android": [ - "-pie", - "-s", - "-landroid", - "-ljnigraphics", - "-llog", - "-lm", - "-z defs", - "-s", - "-Wl,--exclude-libs,ALL", # Exclude syms in all libs from auto export - ], - "//conditions:default": [], - }), - linkstatic = 1, - visibility = ["//visibility:public"], - deps = select({ - "//tensorflow:android": [ - "//tensorflow/core:android_tensorflow_lib", - "//tensorflow/core:android_tensorflow_test_lib", - ], - "//conditions:default": [ - "//tensorflow/core:lib", - "//tensorflow/core:framework_internal", - "//tensorflow/core:tensorflow", - ], - }), -) diff --git a/tensorflow/contrib/hvx/clock_cycle_profiling/clock_cycle_profiling_main.cc b/tensorflow/contrib/hvx/clock_cycle_profiling/clock_cycle_profiling_main.cc deleted file mode 100644 index a87ef953b7e..00000000000 --- a/tensorflow/contrib/hvx/clock_cycle_profiling/clock_cycle_profiling_main.cc +++ /dev/null @@ -1,53 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/profile_utils/clock_cycle_profiler.h" -#include "tensorflow/core/platform/profile_utils/cpu_utils.h" - -int main(int argc, char** argv) { - static constexpr int LOOP_COUNT = 1000000; - -#if defined(__ANDROID_API__) -#if defined(__aarch64__) - LOG(INFO) << "android arm 64 bit"; -#endif -#if defined(__ARM_ARCH_7A__) - LOG(INFO) << "android arm 32 bit"; -#endif - LOG(INFO) << "Android API = " << __ANDROID_API__; - if (__ANDROID_API__ < 21) { - LOG(INFO) << "Cpu utils requires API level 21 or above."; - return 0; - } -#endif - - tensorflow::profile_utils::CpuUtils::EnableClockCycleProfiling(true); - - tensorflow::ClockCycleProfiler prof_global; - tensorflow::ClockCycleProfiler prof_internal; - - prof_global.Start(); - for (int i = 0; i < LOOP_COUNT; ++i) { - prof_internal.Start(); - prof_internal.Stop(); - } - prof_global.Stop(); - - prof_global.DumpStatistics("prof_global"); - prof_internal.DumpStatistics("prof_internal"); - - return 0; -} diff --git a/tensorflow/contrib/hvx/hexagon_controller/Makefile b/tensorflow/contrib/hvx/hexagon_controller/Makefile deleted file mode 100644 index 9fe2ed596af..00000000000 --- a/tensorflow/contrib/hvx/hexagon_controller/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -include glue/defines.min - -include target/make/android.min - -include $(RULES_MIN) diff --git a/tensorflow/contrib/hvx/hexagon_controller/src_dummy_data/inception_v1_graph_init.c b/tensorflow/contrib/hvx/hexagon_controller/src_dummy_data/inception_v1_graph_init.c deleted file mode 100644 index 3ca5532c38f..00000000000 --- a/tensorflow/contrib/hvx/hexagon_controller/src_dummy_data/inception_v1_graph_init.c +++ /dev/null @@ -1,16 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -void init_graph_v1(int nn_id) {} diff --git a/tensorflow/contrib/hvx/hexagon_controller/src_dummy_data/inception_v3_dummy_float_data.c b/tensorflow/contrib/hvx/hexagon_controller/src_dummy_data/inception_v3_dummy_float_data.c deleted file mode 100644 index dc61ae754a3..00000000000 --- a/tensorflow/contrib/hvx/hexagon_controller/src_dummy_data/inception_v3_dummy_float_data.c +++ /dev/null @@ -1,16 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -float inception_dummy_float_data_299x299[299*299*3] = {}; diff --git a/tensorflow/contrib/hvx/hexagon_controller/src_dummy_data/inception_v3_dummy_int_data.c b/tensorflow/contrib/hvx/hexagon_controller/src_dummy_data/inception_v3_dummy_int_data.c deleted file mode 100644 index 27e1ca40b9e..00000000000 --- a/tensorflow/contrib/hvx/hexagon_controller/src_dummy_data/inception_v3_dummy_int_data.c +++ /dev/null @@ -1,17 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include -uint8_t inception_dummy_int_data_299x299[299*299*3] = {}; diff --git a/tensorflow/contrib/hvx/hexagon_controller/src_dummy_data/inception_v3_graph_init.c b/tensorflow/contrib/hvx/hexagon_controller/src_dummy_data/inception_v3_graph_init.c deleted file mode 100644 index 9def6658279..00000000000 --- a/tensorflow/contrib/hvx/hexagon_controller/src_dummy_data/inception_v3_graph_init.c +++ /dev/null @@ -1,16 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -void init_graph(int nn_id) {} diff --git a/tensorflow/contrib/hvx/hexagon_controller/src_impl/graph_functions_wrapper.c b/tensorflow/contrib/hvx/hexagon_controller/src_impl/graph_functions_wrapper.c deleted file mode 100644 index f6a38fe8a94..00000000000 --- a/tensorflow/contrib/hvx/hexagon_controller/src_impl/graph_functions_wrapper.c +++ /dev/null @@ -1,358 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// to demonstrate the performance difference between ION and HLOS memory -// for sharing with ADSP. -#define USE_ION_MEMORY - -#include -#include - -#include "hexagon_controller.h" -#include "hexagon_nn.h" -#include "tfm_log.h" - -static const uint32_t MAX_NODES = 2048; -static const uint32_t MAX_EVENT_COUNT = 256; - -static const bool DUMP_OUTPUT = false; -static const bool DBG_EXECUTION = true; - -static const int OUT_RANKING_SIZE = 5; - -// static only for this file. -// TODO(satok): allocate dynamically -static float s_output_values[300 * 300 * 3 * 4]; - -extern void init_graph(uint32_t id); -extern void init_graph_v1(uint32_t id); -extern uint8_t inception_dummy_int_data_299x299[]; -extern uint8_t inception_dummy_int_data_224x224[]; -extern float inception_dummy_float_data_299x299[]; - -enum InceptionVersion { - INCEPTION_V1, - INCEPTION_V3, -}; - -static enum InceptionVersion s_inception_version = INCEPTION_V3; - -///////////////////////////////////////////////// -// file local functions - -static const char* ConvertGraphInfoIdToName(unsigned int id) { - // TODO(satok): implement - return "?"; -} - -static const char* ConvertGraphInfoIdToOpName(unsigned int id) { - // TODO(satok): implement - return "?"; -} - -///////////////////////////////////////////////// -// file local utilities -static uint32_t FindMaxIdxWithExcludeList(const float* data, uint32_t entries, - const int exclude_size, - const int* exclude_idx) { - int i; - float maxval = data[0]; - int maxidx = 0; - for (i = 0; i < entries; i++) { - bool exclude = false; - for (int j = 0; j < exclude_size; ++j) { - if (exclude_idx[j] == i) { - exclude = true; - break; - } - } - if (exclude) { - continue; - } - if (maxval < data[i]) { - maxval = data[i]; - maxidx = i; - } - } - return maxidx; -} - -static uint32_t FindMaxIdx(const float* data, uint32_t entries) { - return FindMaxIdxWithExcludeList(data, entries, 0, NULL); -} - -void hexagon_controller_PrintMaxNIdx(const float* data, const uint32_t entries, - const int n, int* out_ranking) { - if (DUMP_OUTPUT) { - for (int i = 0; i < entries; ++i) { - TFMLOGD("%d: val = %f", i, data[i]); - } - } - if (n >= entries) { - TFMLOGD("Too many N %d >= %d", n, entries); - } - for (int i = 0; i < n; ++i) { - out_ranking[i] = INT_MAX; - } - for (int i = 0; i < n; ++i) { - out_ranking[i] = FindMaxIdxWithExcludeList(data, entries, n, out_ranking); - } - TFMLOGD("=== RANKING ==="); - for (int i = 0; i < n; ++i) { - TFMLOGD("%d: id = %d, val = %f", i, out_ranking[i], data[out_ranking[i]]); - } -} - -static inline unsigned long long int GetCounter(hexagon_nn_perfinfo s) { - unsigned long long int ret; - ret = s.counter_hi; - ret <<= 32; - ret |= s.counter_lo; - return ret; -} - -static int CompareCycle(const void* va, const void* vb) { - const hexagon_nn_perfinfo* a = va; - const hexagon_nn_perfinfo* b = vb; - unsigned long long int acount = GetCounter(*a); - unsigned long long int bcount = GetCounter(*b); - if (acount < bcount) { - return -1; - } else if (acount > bcount) { - return 1; - } else { - return 0; - } -} - -///////////////////////////////////////////////// -// Graph functions - -uint32_t hexagon_controller_InstantiateGraph() { - const uint32_t nn_id = hexagon_nn_init(); - // TODO(satok): make this as argument - hexagon_nn_set_debug_level(nn_id, 0); - return nn_id; -} - -void hexagon_controller_InitGraph(int version, uint32_t nn_id) { - if (version == 1) { - s_inception_version = INCEPTION_V1; - } else if (version == 3) { - s_inception_version = INCEPTION_V3; - } else { - TFMLOGE("Unsupported inception version %d", version); - return; - } - if (s_inception_version == INCEPTION_V3) { - init_graph(nn_id); - } else if (s_inception_version == INCEPTION_V1) { - init_graph_v1(nn_id); - } - TFMLOGD("Init graph (inception version = %d) done.", version); -} - -bool hexagon_controller_ConstructGraph(uint32_t nn_id) { - int err; - if ((err = hexagon_nn_prepare(nn_id)) != 0) { - TFMLOGE("Prepare failed! returned 0x%x\n", err); - DumpNNId(nn_id); - return false; - } else { - TFMLOGD("Prepare success!\n"); - return true; - } -} - -uint32_t hexagon_controller_SetupGraph(int version) { - const uint32_t nn_id = hexagon_controller_InstantiateGraph(); - hexagon_controller_InitGraph(version, nn_id); - hexagon_controller_ConstructGraph(nn_id); - return nn_id; -} - -bool hexagon_controller_ExecuteGraphWithMultipleInOut( - const uint32_t nn_id, const int input_count, hexagon_nn_tensordef* inputs, - const int output_count, hexagon_nn_tensordef* outputs) { - if (DBG_EXECUTION) { - TFMLOGD("Preparing to execute... in = %d, out = %d", input_count, - output_count); - LogDHexagon("Execute graph!"); - } - - const int err = - hexagon_nn_execute_new(nn_id, inputs, input_count, outputs, output_count); - if (err != 0) { - if (DBG_EXECUTION) { - LogDHexagon("Execution failed!"); - TFMLOGE("execute got err: %d\n", err); - DumpNNId(nn_id); - } - return false; - } else { - if (DBG_EXECUTION) { - LogDHexagon("Execution succeeded!"); - } - return true; - } -} - -bool hexagon_controller_ExecuteGraph( - const uint32_t nn_id, const uint32_t batches, const uint32_t height, - const uint32_t width, const uint32_t depth, uint8_t* int_data, - const uint32_t int_data_size, uint32_t* out_batches, uint32_t* out_height, - uint32_t* out_width, uint32_t* out_depth, uint8_t* out_vals, - const uint32_t output_val_byte_size, uint32_t* out_data_byte_size) { - if (DBG_EXECUTION) { - TFMLOGD("Preparing to execute..."); - TFMLOGD("Input: %d, %d, %d, %d, %d, %d", batches, height, width, depth, - int_data[0], int_data_size); - TFMLOGD("Output: %d, %p", output_val_byte_size, out_vals); - LogDHexagon("Execute graph!"); - } - - hexagon_nn_tensordef input; - hexagon_nn_tensordef output; - - input.batches = batches; - input.height = height; - input.width = width; - input.depth = depth; - input.data = int_data; - input.dataLen = int_data_size; - - output.data = out_vals; - output.dataLen = output_val_byte_size; - - if (!hexagon_controller_ExecuteGraphWithMultipleInOut(nn_id, 1, &input, 1, - &output)) { - return false; - } else { - *out_batches = output.batches; - *out_height = output.height; - *out_width = output.width; - *out_depth = output.depth; - *out_data_byte_size = output.dataLen; - - if (DBG_EXECUTION) { - LogDHexagon("Execution succeeded!"); - TFMLOGD("%d x %d x %d x %d, byte size = %d\n", *out_batches, *out_height, - *out_width, *out_depth, *out_data_byte_size); - } - return true; - } -} - -bool hexagon_controller_ExecuteInceptionDummyData(uint32_t nn_id) { - uint32_t out_batches, out_height, out_width, out_depth; - uint32_t out_data_size; - // s_output_values = 300 * 300 * 3 * 4 * 4 - const bool success = hexagon_controller_ExecuteGraph( - nn_id, INCEPTION_PARAM_BATCHES, INCEPTION_PARAM_HEIGHT_V3, - INCEPTION_PARAM_WIDTH_V3, INCEPTION_PARAM_DEPTH, - (uint8_t*)inception_dummy_int_data_299x299, - INCEPTION_PARAM_HEIGHT_V3 * INCEPTION_PARAM_WIDTH_V3 * - INCEPTION_PARAM_DEPTH, - &out_batches, &out_height, &out_width, &out_depth, - (uint8_t*)s_output_values, sizeof(s_output_values), &out_data_size); - if (success) { - int out_ranking[OUT_RANKING_SIZE]; - hexagon_controller_PrintMaxNIdx( - s_output_values, out_batches * out_height * out_width * out_depth, - OUT_RANKING_SIZE, out_ranking); - TFMLOGD("%d x %d x %d x %d, size = %d\n", out_batches, out_height, - out_width, out_depth, out_data_size); - TFMLOGD("max idx: %d\n", - FindMaxIdx(s_output_values, - out_batches * out_height * out_width * out_depth)); - if (out_ranking[0] == 169 && out_ranking[1] == 7) { - return true; - } else { - TFMLOGD("Result is wrong! %d, %d", out_ranking[0], out_ranking[1]); - return false; - } - } else { - return false; - } -} - -void hexagon_controller_DumpPerf(uint32_t nn_id) { - hexagon_nn_perfinfo info[MAX_NODES]; - unsigned long long int total_cycles = 0; - unsigned long long int cum_cycles = 0; - unsigned long long int counter = 0; - unsigned int n_nodes; - int i; - TFMLOGD("Perf dump follows:"); - if (hexagon_nn_get_perfinfo(nn_id, info, MAX_NODES, &n_nodes) != 0) { - TFMLOGE("perf info failure"); - return; - } - TFMLOGD("Total %d nodes.", n_nodes); - qsort(info, n_nodes, sizeof(info[0]), CompareCycle); - for (i = 0; i < n_nodes; i++) { - total_cycles += GetCounter(info[i]); - } - TFMLOGD("Total %lld cycles.", total_cycles); - for (i = 0; i < n_nodes; i++) { - counter = GetCounter(info[i]); - cum_cycles += counter; - TFMLOGD( - "node,0x%x,%s,%s,executions,%d,cycles,%lld,%f %%," - "cum_cycles,%lld,%f %%\n", - info[i].node_id, ConvertGraphInfoIdToName(info[i].node_id), - ConvertGraphInfoIdToOpName(info[i].node_id), info[i].executions, - counter, 100 * ((double)counter) / total_cycles, cum_cycles, - 100 * ((double)cum_cycles) / total_cycles); - } -#ifdef ENABLE_HVX_FULL_DEBUG - DumpAllPerf(nn_id); -#endif -} - -void hexagon_controller_DumpNodeName(uint32_t nn_id) { - TFMLOGD("Show node name"); - const uint32_t id = nn_id; - hexagon_nn_perfinfo info[MAX_NODES]; - unsigned long long int total_cycles = 0; - unsigned long long int cum_cycles = 0; - unsigned long long int counter = 0; - unsigned int node_count; - int i; - TFMLOGD("Perf dump follows:"); - if (hexagon_nn_get_perfinfo(id, info, MAX_NODES, &node_count) != 0) { - TFMLOGD("perf info failure"); - return; - } - TFMLOGD("Total %d nodes.", node_count); - qsort(info, node_count, sizeof(info[0]), CompareCycle); - for (i = 0; i < node_count; i++) { - total_cycles += GetCounter(info[i]); - } - TFMLOGD("Total %lld cycles.", total_cycles); - for (i = 0; i < node_count; i++) { - counter = GetCounter(info[i]); - cum_cycles += counter; - TFMLOGD( - "node,0x%x,%s,%s,executions,%d,cycles,%lld,%f %%," - "cum_cycles,%lld,%f %%", - info[i].node_id, ConvertGraphInfoIdToName(info[i].node_id), - ConvertGraphInfoIdToOpName(info[i].node_id), info[i].executions, - counter, 100 * ((double)counter) / total_cycles, cum_cycles, - 100 * ((double)cum_cycles) / total_cycles); - } -} - -void hexagon_controller_Teardown(uint32_t nn_id) { hexagon_nn_teardown(nn_id); } diff --git a/tensorflow/contrib/hvx/hexagon_controller/src_impl/hexagon_controller.c b/tensorflow/contrib/hvx/hexagon_controller/src_impl/hexagon_controller.c deleted file mode 100644 index 2e5c84704f8..00000000000 --- a/tensorflow/contrib/hvx/hexagon_controller/src_impl/hexagon_controller.c +++ /dev/null @@ -1,511 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// to demonstrate the performance difference between ION and HLOS memory -// for sharing with ADSP. -#define USE_ION_MEMORY - -#include "hexagon_controller.h" - -#include -#include - -#include "adspmsgd.h" -#include "dspCV.h" -#include "node_data_float.h" -#include "rpcmem.h" // helper API's for shared buffer allocation -#include "soc_interface.h" -#include "tfm_log.h" - -// if false, use int data as input. This is only for acceleration purpose. -// Also you may need to change android.min. -static const bool USE_FLOAT_DATA = true; - -// if true, show id for each node -static const bool DBG_SHOW_ID = false; - -static const uint32_t OUTPUT_PARAM_MAX_LINE_SIZE = 1000; - -static const uint32_t PRINT_BUFSIZE = 2 * 1024 * 1024; - -// extern pre-generated inception dummy data -extern uint8_t inception_dummy_int_data_224x224[]; -extern uint8_t inception_dummy_int_data_299x299[]; -extern float inception_dummy_float_data_299x299[]; - -#define HEXAGON_CONTROLLER_VERSION 101 - -// allocate print bufsize in advance @MB -#define PRINT_BUFSIZE (2 * 1024 * 1024) - -static unsigned char s_print_buf[PRINT_BUFSIZE]; - -#define MAX_INPUTS 10 -#define MAX_OUTPUTS 10 - -static struct NodeDataFloat s_input_node_data_buffer[MAX_INPUTS]; -static uint8_t* s_output_node_data_buffer[MAX_OUTPUTS]; -static int s_output_node_data_buffer_max_byte_size[MAX_OUTPUTS]; -static int s_output_node_data_array_byte_size[MAX_OUTPUTS]; -static uint32_t s_target_graph_id; - -static bool s_dbg_use_inception_dummy_data = false; -static int s_dbg_inception_version = 3; - -static int GetInputNodeCount() { - for (int i = 0; i < MAX_INPUTS; ++i) { - if (s_input_node_data_buffer[i].max_buf_byte_size == 0) { - return i; - } - } - return 0; -} - -static int GetOutputNodeCount() { - for (int i = 0; i < MAX_OUTPUTS; ++i) { - if (s_output_node_data_buffer_max_byte_size[i] == 0) { - return i; - } - } - return 0; -} - -static bool SetInputTensorDef(int port, hexagon_nn_tensordef* tensordef) { - if (port >= GetInputNodeCount()) { - TFMLOGE("Error exceeds input count."); - return false; - } - struct NodeDataFloat* input_node_data_buffer = - &s_input_node_data_buffer[port]; - tensordef->batches = input_node_data_buffer->x; - tensordef->height = input_node_data_buffer->y; - tensordef->width = input_node_data_buffer->z; - tensordef->depth = input_node_data_buffer->d; - tensordef->data = input_node_data_buffer->byte_array_data; - tensordef->dataLen = input_node_data_buffer->array_byte_size; - - return true; -} - -bool hexagon_controller_SetAllInputTensorDef(int node_count, - hexagon_nn_tensordef* tensordef) { - bool success = true; - if (node_count != GetInputNodeCount()) { - TFMLOGE("Error invalid input node count."); - return false; - } - for (int i = 0; i < node_count; ++i) { - SetInputTensorDef(i, &tensordef[i]); - } - return success; -} - -static bool SetOutputTensorDef(int port, hexagon_nn_tensordef* tensordef) { - if (port >= GetOutputNodeCount()) { - TFMLOGE("Error exceeds output count."); - return false; - } - tensordef->data = s_output_node_data_buffer[port]; - tensordef->dataLen = s_output_node_data_buffer_max_byte_size[port]; - return true; -} - -bool hexagon_controller_SetAllOutputTensorDef(int node_count, - hexagon_nn_tensordef* tensordef) { - bool success = true; - if (node_count != GetOutputNodeCount()) { - TFMLOGE("Error invalid output node count. %d != %d", node_count, - GetOutputNodeCount()); - return false; - } - for (int i = 0; i < node_count; ++i) { - SetOutputTensorDef(i, &tensordef[i]); - } - return success; -} - -void hexagon_controller_InitInputNodeDataToInceptionDummyData(int version) { - if (version == 1) { - if (USE_FLOAT_DATA) { - TFMLOGE("ERROR!!!! Do not use float data for v1"); - return; - } - hexagon_controller_CopyByteNodeData( - 0, INCEPTION_PARAM_BATCHES, INCEPTION_PARAM_HEIGHT_V1, - INCEPTION_PARAM_WIDTH_V1, INCEPTION_PARAM_DEPTH, 1, - inception_dummy_int_data_224x224); - } else if (version == 3) { - if (USE_FLOAT_DATA) { - hexagon_controller_CopyByteNodeData( - 0, INCEPTION_PARAM_BATCHES, INCEPTION_PARAM_HEIGHT_V3, - INCEPTION_PARAM_WIDTH_V3, INCEPTION_PARAM_DEPTH, sizeof(float), - (uint8_t*)inception_dummy_float_data_299x299); - } else { - hexagon_controller_CopyByteNodeData( - 0, INCEPTION_PARAM_BATCHES, INCEPTION_PARAM_HEIGHT_V3, - INCEPTION_PARAM_WIDTH_V3, INCEPTION_PARAM_DEPTH, 1, - inception_dummy_int_data_299x299); - } - } -} - -bool hexagon_controller_ExecuteGraphWithBuffer(uint32_t nn_id, - bool show_ranking) { - const int input_node_count = GetInputNodeCount(); - hexagon_nn_tensordef inputs[input_node_count]; - const int output_node_count = GetOutputNodeCount(); - if (output_node_count <= 0) { - TFMLOGI("Error output node count is 0."); - return false; - } - hexagon_nn_tensordef outputs[output_node_count]; - hexagon_controller_SetAllInputTensorDef(input_node_count, inputs); - hexagon_controller_SetAllOutputTensorDef(output_node_count, outputs); - const bool success = hexagon_controller_ExecuteGraphWithMultipleInOut( - nn_id, input_node_count, inputs, output_node_count, outputs); - for (int i = 0; i < output_node_count; ++i) { - s_output_node_data_array_byte_size[i] = outputs[i].data_valid_len; - } - - const hexagon_nn_tensordef* output0 = &outputs[0]; - - const uint32_t out_batches = output0->batches; - const uint32_t out_height = output0->height; - const uint32_t out_width = output0->width; - const uint32_t out_depth = output0->depth; - const uint32_t out_data_size = output0->data_valid_len; - const uint32_t out_buf_byte_size = output0->dataLen; - - if (!success) { - TFMLOGE("Execution failed"); - DumpNNId(nn_id); - return false; - } else if (!show_ranking) { - return true; - } - - static const int OUT_RANKING_SIZE = 5; - int out_ranking[OUT_RANKING_SIZE]; - hexagon_controller_PrintMaxNIdx( - (float*)s_output_node_data_buffer[0], - out_batches * out_height * out_width * out_depth, OUT_RANKING_SIZE, - out_ranking); - TFMLOGD("%d x %d x %d x %d, byte size = %d, buf size = %d\n", out_batches, - out_height, out_width, out_depth, out_data_size, out_buf_byte_size); - if (s_dbg_use_inception_dummy_data) { - // Check the result of inception with a dummy data. This step shouldn't - // be passed when show_ranking != true to avoid adding unnecessary - // additional computation cost. - if (out_ranking[0] == 169 && out_ranking[1] == 7) { - TFMLOGD("Result is correct! %d, %d", out_ranking[0], out_ranking[1]); - return true; - } else { - TFMLOGD("Result is wrong! %d, %d", out_ranking[0], out_ranking[1]); - return false; - } - } - return true; -} - -uint32_t hexagon_controller_GetTargetGraphId() { return s_target_graph_id; } - -void hexagon_controller_SetTargetGraphId(uint32_t graph_id) { - s_target_graph_id = graph_id; -} - -void hexagon_controller_PrintGraph(uint32_t id) { - int retval = hexagon_nn_snpprint(id, s_print_buf, PRINT_BUFSIZE); - TFMLOGD("PrintGraph %s\n", s_print_buf); - if (retval) { - TFMLOGE("Error on print graph\n"); - } -} - -int hexagon_controller_GetWrapperVersion() { - return HEXAGON_CONTROLLER_VERSION; -} - -int hexagon_controller_GetHexagonBinaryVersion() { - int retval = 0; - hexagon_nn_version(&retval); - return retval; -} - -bool hexagon_controller_AllocateInputNodeDataBuffers(int port, - int input_buf_byte_size) { - TFMLOGD("Allocate memory for input node data. port = %d, size = %d", port, - input_buf_byte_size); - if (s_input_node_data_buffer[port].max_buf_byte_size != 0) { - TFMLOGE("ERROR! input buffer is already allocated!!"); - return false; - } else { - s_input_node_data_buffer[port].max_buf_byte_size = input_buf_byte_size; - posix_memalign((void**)&s_input_node_data_buffer[port].byte_array_data, 128, - input_buf_byte_size); - TFMLOGD("allocate input node data buffers done"); - } - return true; -} - -bool hexagon_controller_AllocateOutputNodeDataBuffers( - int port, int output_buf_byte_size) { - TFMLOGD("Allocate memory for output node data. port = %d, size = %d", port, - output_buf_byte_size); - if (s_output_node_data_buffer_max_byte_size[port] != 0) { - TFMLOGE("ERROR! input buffer is already allocated!!"); - return false; - } else { - // s_output_node_data_buffer = malloc(output_size * sizeof(float)); - posix_memalign((void**)&s_output_node_data_buffer[port], 128, - output_buf_byte_size); - s_output_node_data_buffer_max_byte_size[port] = output_buf_byte_size; - s_output_node_data_array_byte_size[port] = 0; - TFMLOGD("allocate output node data buffers"); - } - return true; -} - -bool hexagon_controller_AllocateMultipleNodeDataBuffers(int input_count, - int* input_sizes, - int output_count, - int* output_sizes) { - bool success = true; - for (int i = 0; i < input_count; ++i) { - success &= - hexagon_controller_AllocateInputNodeDataBuffers(i, input_sizes[i]); - } - for (int i = 0; i < output_count; ++i) { - success &= - hexagon_controller_AllocateOutputNodeDataBuffers(i, output_sizes[i]); - } - - if (s_dbg_use_inception_dummy_data) { - hexagon_controller_InitInputNodeDataToInceptionDummyData( - s_dbg_inception_version); - } - return success; -} - -bool hexagon_controller_AllocateNodeDataBuffers(int input_size, - int output_size) { - return hexagon_controller_AllocateMultipleNodeDataBuffers(1, &input_size, 1, - &output_size); -} - -bool hexagon_controller_ReleaseInputNodeDataBuffersWithPort(int port) { - struct NodeDataFloat* input_node_data_buffer = - &s_input_node_data_buffer[port]; - if (input_node_data_buffer->max_buf_byte_size == 0) { - TFMLOGE("ERROR! input buffer has not been allocated yet!!"); - return false; - } else { - input_node_data_buffer->max_buf_byte_size = 0; - input_node_data_buffer->array_byte_size = 0; - free(input_node_data_buffer->byte_array_data); - } - return true; -} - -bool hexagon_controller_ReleaseOutputNodeDataBuffersWithPort(int port) { - if (s_output_node_data_buffer_max_byte_size[port] == 0) { - TFMLOGE("ERROR! output buffer has not been allocated yet!!"); - return false; - } else { - s_output_node_data_buffer_max_byte_size[port] = 0; - s_output_node_data_array_byte_size[port] = 0; - free(s_output_node_data_buffer[port]); - } - return true; -} - -bool hexagon_controller_ReleaseNodeDataBuffers() { - bool success = true; - for (int i = 0; i < GetInputNodeCount(); ++i) { - success &= hexagon_controller_ReleaseInputNodeDataBuffersWithPort(i); - } - for (int i = 0; i < GetOutputNodeCount(); ++i) { - success &= hexagon_controller_ReleaseOutputNodeDataBuffersWithPort(i); - } - return success; -} - -bool hexagon_controller_CopyByteNodeData(int port, int x, int y, int z, int d, - int type_byte_size, - uint8_t* array_data) { - int array_byte_size = x * y * z * d * type_byte_size; - TFMLOGD("--- %d, %d, %d, %d, %d, %d", x, y, z, d, type_byte_size, - array_byte_size); - struct NodeDataFloat* input_node_data_buffer = &s_input_node_data_buffer[0]; - - if (input_node_data_buffer->max_buf_byte_size < array_byte_size) { - TFMLOGE("ERROR! input buffer size is too small! %d < %d", - input_node_data_buffer->max_buf_byte_size, array_byte_size); - return false; - } - memcpy(input_node_data_buffer->byte_array_data, array_data, array_byte_size); - input_node_data_buffer->array_byte_size = array_byte_size; - input_node_data_buffer->x = x; - input_node_data_buffer->y = y; - input_node_data_buffer->z = z; - input_node_data_buffer->d = d; - return true; -} - -int hexagon_controller_InitHexagonWithMaxAttributes(int enable_dcvs, - int bus_usage, - int version) { - TFMLOGI("Init hexagon with max attributes (Controller version = %d)", - HEXAGON_CONTROLLER_VERSION); - const int MCPS = 1000; - const int MBPS = 12000; - - adspmsgd_start(0, RPCMEM_HEAP_DEFAULT, 4096); - - dspCV_Attribute attrib[] = { - // The below values will result in the maximum aDSP performance, - // at Turbo voltage. - // Slightly more MCPS than are available on current targets - {DSP_TOTAL_MCPS, MCPS}, - // drive the clock to MAX on known targets - {DSP_MCPS_PER_THREAD, MCPS / 2}, - // 12 GB/sec is slightly higher than the max realistic - // max BW on existing targets. - {PEAK_BUS_BANDWIDTH_MBPS, MBPS}, - // This app is non-real time, and constantly reading/writing memory - {BUS_USAGE_PERCENT, bus_usage}, - }; - int retval = 0; - if (!enable_dcvs) { - retval = hexagon_nn_disable_dcvs(); - if (retval) { - TFMLOGE("Failed to disable DSP DCVS: %x\n", retval); - } - } - - retval = - dspCV_initQ6_with_attributes(attrib, sizeof(attrib) / sizeof(attrib[0])); - TFMLOGD("Return value from dspCV_initQ6() : %d\n", retval); - - s_target_graph_id = 0; - s_dbg_inception_version = version; - - return retval; -} - -int hexagon_controller_DeInitHexagon() { - adspmsgd_stop(); - TFMLOGI("Finalize hexagon"); - const int retval = dspCV_deinitQ6(); - TFMLOGD("return value from dspCV_deinitQ6(): %d \n", retval); - - hexagon_controller_ReleaseNodeDataBuffers(); - - return retval; -} - -void hexagon_controller_GrowMemorySize() { hexagon_nn_config(); } - -struct NodeDataFloat* hexagon_controller_GetInputNodeDataBuffer(int port) { - if (port >= GetInputNodeCount()) { - TFMLOGE("port should be less than 1"); - } - return &s_input_node_data_buffer[port]; -} - -uint8_t* hexagon_controller_GetOutputNodeDataBuffer(int port, - int* out_array_byte_size) { - if (port >= GetOutputNodeCount()) { - TFMLOGE("port should be less than 1"); - } - *out_array_byte_size = s_output_node_data_array_byte_size[port]; - return s_output_node_data_buffer[port]; -} - -// Append const node to the graph -int hexagon_controller_AppendConstNode(const char* const name, int graph_id, - int node_id, int batch, int height, - int width, int depth, - const uint8_t* const data, - int data_length) { - if (DBG_SHOW_ID) { - TFMLOGV("---(CONST) %s, %d, %d, %d, %d, %d, %d", name, node_id, batch, - height, width, depth, data_length); - } else { - TFMLOGV("---(CONST) %s, %d, %d, %d, %d, %d", name, batch, height, width, - depth, data_length); - } - const int retval = hexagon_nn_append_const_node( - graph_id, node_id, batch, height, width, depth, data, data_length); - if (retval != 0) { - TFMLOGE("Failed to append const node %d", node_id); - return retval; - } - return retval; -} - -// Append node to the graph -int hexagon_controller_AppendNode(const char* const name, int graph_id, - int node_id, int ops_id, int padding_id, - const hexagon_nn_input* const inputs, - int inputs_count, - const hexagon_nn_output* const outputs, - int outputs_count) { - char input_param_buf[OUTPUT_PARAM_MAX_LINE_SIZE]; - memset(input_param_buf, 0, OUTPUT_PARAM_MAX_LINE_SIZE); - int pos = 0; - pos += snprintf(&input_param_buf[pos], 500, "in: "); - for (int i = 0; i < inputs_count; ++i) { - if (DBG_SHOW_ID) { - pos += snprintf(&input_param_buf[pos], 500, "(%d, %d), ", - inputs[i].src_id, inputs[i].output_idx); - } else { - pos += - snprintf(&input_param_buf[pos], 500, "(%d), ", inputs[i].output_idx); - } - } - - char output_param_buf[OUTPUT_PARAM_MAX_LINE_SIZE]; - memset(output_param_buf, 0, OUTPUT_PARAM_MAX_LINE_SIZE); - pos = 0; - pos += snprintf(&output_param_buf[pos], 500, "out: "); - for (int i = 0; i < outputs_count; ++i) { - pos += snprintf(&output_param_buf[pos], 500, "(%d), ", outputs[i].max_size); - } - - if (DBG_SHOW_ID) { - TFMLOGV("---(OP) %s, %d, %d, %d, %d, %d, %s, %s", name, node_id, ops_id, - padding_id, inputs_count, outputs_count, input_param_buf, - output_param_buf); - } else { - TFMLOGV("---(OP) %s, %d, %d, %d, %d, %s, %s", name, ops_id, padding_id, - inputs_count, outputs_count, input_param_buf, output_param_buf); - } - const int retval = - hexagon_nn_append_node(graph_id, node_id, ops_id, padding_id, inputs, - inputs_count, outputs, outputs_count); - if (retval != 0) { - TFMLOGE("Failed to append const node %d", node_id); - return retval; - } - return retval; -} - -void hexagon_controller_EnableDbgUseInceptionDummyData(bool enable) { - s_dbg_use_inception_dummy_data = enable; -} - -bool hexagon_controller_IsDbgUseInceptionDummyDataEnabled() { - return s_dbg_use_inception_dummy_data; -} diff --git a/tensorflow/contrib/hvx/hexagon_controller/src_impl/include/hexagon_controller.h b/tensorflow/contrib/hvx/hexagon_controller/src_impl/include/hexagon_controller.h deleted file mode 100644 index fc921ff8b98..00000000000 --- a/tensorflow/contrib/hvx/hexagon_controller/src_impl/include/hexagon_controller.h +++ /dev/null @@ -1,149 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef GEMM_WRAPPER_H -#define GEMM_WRAPPER_H - -#include -#include - -#include "hexagon_nn.h" -#include "node_data_float.h" - -#ifdef __cplusplus -extern "C" { -#endif // __cplusplus - -#define INCEPTION_PARAM_BATCHES 1 -#define INCEPTION_PARAM_HEIGHT_V1 224 -#define INCEPTION_PARAM_WIDTH_V1 224 -#define INCEPTION_PARAM_HEIGHT_V3 299 -#define INCEPTION_PARAM_WIDTH_V3 299 -#define INCEPTION_PARAM_DEPTH 3 - -// General functions -void hexagon_controller_PrintGraph(uint32_t nn_id); - -int hexagon_controller_GetWrapperVersion(); - -int hexagon_controller_GetHexagonBinaryVersion(); - -// Buffer operations -bool hexagon_controller_SetAllInputTensorDef(int node_count, - hexagon_nn_tensordef* tensordef); - -bool hexagon_controller_SetAllInputTensorDef(int node_count, - hexagon_nn_tensordef* tensordef); - -// Hexagon perf functions -int hexagon_controller_InitHexagonWithMaxAttributes(int enable_dcvs, - int bus_usage, int version); - -bool hexagon_controller_AllocateInputNodeDataBuffersWithPort(int port, - int input_size); - -bool hexagon_controller_AllocateOutNodeDataBuffersWithPort(int port, - int output_size); - -bool hexagon_controller_AllocateNodeDataBuffers(int input_size, - int output_size); - -bool hexagon_controller_AllocateMultipleNodeDataBuffers(int input_count, - int* input_sizes, - int output_count, - int* output_sizes); - -bool hexagon_controller_ReleaseInputNodeDataBuffersWithPort(int port); -bool hexagon_controller_ReleaseOutputNodeDataBuffersWithPort(int port); - -bool hexagon_controller_ReleaseNodeDataBuffers(); - -bool hexagon_controller_CopyByteNodeData(int port, int x, int y, int z, int d, - int type_byte_size, - uint8_t* array_data); - -int hexagon_controller_DeInitHexagon(); - -uint32_t hexagon_controller_GetTargetGraphId(); - -void hexagon_controller_SetTargetGraphId(uint32_t graph_id); - -// Hexagon config functions -void hexagon_controller_GrowMemorySize(); - -// Graph data transfer functions -struct NodeDataFloat* hexagon_controller_GetInputNodeDataBuffer(int port); - -uint8_t* hexagon_controller_GetOutputNodeDataBuffer(int port, - int* out_array_byte_size); - -// Graph functions -uint32_t hexagon_controller_InstantiateGraph(); - -void hexagon_controller_InitGraph(int version, uint32_t nn_id); - -bool hexagon_controller_ConstructGraph(uint32_t nn_id); - -uint32_t hexagon_controller_SetupGraph(int version); - -bool hexagon_controller_ExecuteInceptionDummyData(uint32_t nn_id); - -bool hexagon_controller_ExecuteGraphWithMultipleInOut( - const uint32_t nn_id, const int input_count, hexagon_nn_tensordef* inputs, - const int output_count, hexagon_nn_tensordef* outputs); - -bool hexagon_controller_ExecuteGraph( - const uint32_t nn_id, const uint32_t batches, const uint32_t height, - const uint32_t width, const uint32_t depth, uint8_t* int_data, - const uint32_t int_data_size, uint32_t* out_batches, uint32_t* out_height, - uint32_t* out_width, uint32_t* out_depth, uint8_t* out_vals, - const uint32_t output_val_byte_size, uint32_t* out_data_byte_size); - -bool hexagon_controller_ExecuteGraphWithBuffer(uint32_t nn_id, - bool show_ranking); - -void hexagon_controller_DumpPerf(uint32_t nn_id); - -void hexagon_controller_DumpNodeName(uint32_t nn_id); - -void hexagon_controller_Teardown(uint32_t nn_id); - -void hexagon_controller_PrintMaxNIdx(const float* data, const uint32_t entries, - const int n, int* out_ranking); - -void hexagon_controller_InitInputNodeDataToInceptionDummyData(int version); - -int hexagon_controller_AppendNode(const char* const name, int graph_id, - int node_id, int op_id, int padding_id, - const hexagon_nn_input* const inputs, - int inputs_count, - const hexagon_nn_output* const outputs, - int outputs_count); - -int hexagon_controller_AppendConstNode(const char* const name, int graph_id, - int node_id, int batch, int height, - int width, int depth, - const uint8_t* const data, - int data_length); - -void hexagon_controller_EnableDbgUseInceptionDummyData(bool enable); - -bool hexagon_controller_IsDbgUseInceptionDummyDataEnabled(); - -#ifdef __cplusplus -} -#endif // __cplusplus - -#endif // GEMM_WRAPPER_H diff --git a/tensorflow/contrib/hvx/hexagon_controller/src_log/include/tfm_log.h b/tensorflow/contrib/hvx/hexagon_controller/src_log/include/tfm_log.h deleted file mode 100644 index 8d11ee4a340..00000000000 --- a/tensorflow/contrib/hvx/hexagon_controller/src_log/include/tfm_log.h +++ /dev/null @@ -1,82 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef GEMM_WRAPPER_LOG_H -#define GEMM_WRAPPER_LOG_H - -#include -#include -#include - -#define TFM_LOG_LEVEL_VERBOSE -2 -#define TFM_LOG_LEVEL_DEBUG -1 -#define TFM_LOG_LEVEL_INFO 0 -#define TFM_LOG_LEVEL_WARNING 1 -#define TFM_LOG_LEVEL_ERROR 2 -#define TFM_LOG_LEVEL_FATAL 3 - -static int s_log_level = TFM_LOG_LEVEL_INFO; - -static inline bool IsLogOn(int log_level) { return log_level >= s_log_level; } - -static inline void SetLogLevel(int log_level) { s_log_level = log_level; } - -// Do nothing -static inline void SetExperimentalDebug() {} - -#define TFMLOGV(fmt, ...) \ - do { \ - if (!IsLogOn(TFM_LOG_LEVEL_VERBOSE)) break; \ - printf(fmt "\n", ##__VA_ARGS__); \ - } while (0) - -#define TFMLOGD(fmt, ...) \ - do { \ - if (!IsLogOn(TFM_LOG_LEVEL_DEBUG)) break; \ - printf(fmt "\n", ##__VA_ARGS__); \ - } while (0) - -#define TFMLOGI(fmt, ...) \ - do { \ - if (!IsLogOn(TFM_LOG_LEVEL_INFO)) break; \ - printf(fmt "\n", ##__VA_ARGS__); \ - } while (0) - -#define TFMLOGE(fmt, ...) \ - do { \ - if (!IsLogOn(TFM_LOG_LEVEL_ERROR)) break; \ - printf(fmt "\n", ##__VA_ARGS__); \ - } while (0) - -static inline void PrintLogHexagon(const char* fmt, va_list ap) { - char buffer[200]; - const int count = snprintf(buffer, 200, fmt, ap); - buffer[count] = 0; - TFMLOGI("%s", buffer); -} - -static inline void LogDHexagon(const char* fmt, ...) { - va_list ap; - va_start(ap, fmt); - PrintLogHexagon(fmt, ap); - va_end(ap); -} - -static inline void DumpNNId(uint32_t nn_id) { - // TODO(satok): Dump more information - TFMLOGI("NN Id = %d", nn_id); -} - -#endif diff --git a/tensorflow/contrib/hvx/hexagon_controller/src_soc_interface/include/node_data_float.h b/tensorflow/contrib/hvx/hexagon_controller/src_soc_interface/include/node_data_float.h deleted file mode 100644 index c7034cc3a0d..00000000000 --- a/tensorflow/contrib/hvx/hexagon_controller/src_soc_interface/include/node_data_float.h +++ /dev/null @@ -1,40 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef NODE_DATA_FLOAT_H -#define NODE_DATA_FLOAT_H - -#ifdef __cplusplus -extern "C" { -#else -#include -#endif -#define NODE_DATA_FLOAT_NODE_NAME_BUF_SIZE 100 - -struct NodeDataFloat { - int x; - int y; - int z; - int d; - int max_buf_byte_size; - int array_byte_size; - uint8_t* byte_array_data; - char node_name[NODE_DATA_FLOAT_NODE_NAME_BUF_SIZE]; -}; -#ifdef __cplusplus -} -#endif - -#endif // NODE_DATA_FLOAT_H diff --git a/tensorflow/contrib/hvx/hexagon_controller/src_soc_interface/include/soc_interface.h b/tensorflow/contrib/hvx/hexagon_controller/src_soc_interface/include/soc_interface.h deleted file mode 100644 index 30fad13fb5f..00000000000 --- a/tensorflow/contrib/hvx/hexagon_controller/src_soc_interface/include/soc_interface.h +++ /dev/null @@ -1,115 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_PLATFORM_HEXAGON_SOC_INTERFACE_H_ -#define TENSORFLOW_PLATFORM_HEXAGON_SOC_INTERFACE_H_ - -#include - -// Declaration of APIs provided by hexagon shared library. This header is shared -// with both hexagon library built with qualcomm SDK and tensorflow. -// All functions defined here must have prefix "soc_interface" to avoid -// naming conflicts. -#ifdef __cplusplus -extern "C" { -#else -#include -#endif // __cplusplus -// Returns the version of loaded hexagon wrapper shared library. -// You should assert that the version matches the expected version before -// calling APIs defined in this header. -int soc_interface_GetWrapperVersion(); -// Returns the version of hexagon binary. -// You should assert that the version matches the expected version before -// calling APIs defined in this header. -int soc_interface_GetSocControllerVersion(); -// Initialize SOC -bool soc_interface_Init(); -// Finalize SOC -bool soc_interface_Finalize(); -// Execute graph on SOC -bool soc_interface_ExecuteGraph(); -// Teardown graph setup -bool soc_interface_TeardownGraph(); - -// Allocate buffers for input node and output node -bool soc_interface_AllocateInOutNodeBuffers(int input_count, int* input_sizes, - int output_count, - int* output_sizes); - -// Send input data to SOC with port -bool soc_interface_FillInputNodeWithPort(int port, int x, int y, int z, int d, - const uint8_t* const buf, - uint64_t buf_byte_size); - -// Send input data to SOC -bool soc_interface_FillInputNodeFloat(int x, int y, int z, int d, - const uint8_t* const buf, - uint64_t buf_byte_size); - -// Load output data from SOC with port -bool soc_interface_ReadOutputNodeWithPort(int port, uint8_t** buf, - uint64_t* buf_byte_size); - -// Load output data from SOC -bool soc_interface_ReadOutputNodeFloat(const char* const node_name, - uint8_t** buf, uint64_t* buf_byte_size); - -// Setup graph -// TODO(satok): Remove and use runtime version -bool soc_interface_setupDummyGraph(int version); - -// Allocate memory for params of node inputs and node outputs -bool soc_interface_AllocateNodeInputAndNodeOutputArray(int total_input_count, - int total_output_count); - -// Release memory for params of node inputs and node outputs -bool soc_interface_ReleaseNodeInputAndNodeOutputArray(); - -// Set one node's inputs and return pointer to that struct -void* soc_interface_SetOneNodeInputs(int input_count, const int* const node_id, - const int* const port); - -// Set one node's outputs and return pointer to that struct -void* soc_interface_SetOneNodeOutputs(int output_count, int* max_size); - -// Append const node to the graph -bool soc_interface_AppendConstNode(const char* const name, int node_id, - int batch, int height, int width, int depth, - const uint8_t* const data, int data_length); - -// Append node to the graph -bool soc_interface_AppendNode(const char* const name, int node_id, int op_id, - int padding_id, const void* const inputs, - int inputs_count, const void* const outputs, - int outputs_count); - -// Instantiate graph -bool soc_interface_InstantiateGraph(); - -// Construct graph -bool soc_interface_ConstructGraph(); - -// Set log level -void soc_interface_SetLogLevel(int log_level); - -// Set debug flag -void soc_interface_SetDebugFlag(uint64_t flag); - -#ifdef __cplusplus -} -#endif // __cplusplus - -#endif // TENSORFLOW_PLATFORM_HEXAGON_SOC_INTERFACE_H_ diff --git a/tensorflow/contrib/hvx/hexagon_controller/src_soc_interface/soc_interface.c b/tensorflow/contrib/hvx/hexagon_controller/src_soc_interface/soc_interface.c deleted file mode 100755 index a1387ee5736..00000000000 --- a/tensorflow/contrib/hvx/hexagon_controller/src_soc_interface/soc_interface.c +++ /dev/null @@ -1,266 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "soc_interface.h" - -#include - -#include "hexagon_controller.h" -#include "hexagon_nn.h" -#include "node_data_float.h" -#include "tfm_log.h" - -// to demonstrate the performance difference between ION and HLOS memory -// for sharing with ADSP. -#define USE_ION_MEMORY - -const int64_t FLAG_ENABLE_INCEPTION_DUMMY_BINARY_INPUT = 0x01; -const int64_t FLAG_ENABLE_EXPERIMENTAL_DEBUG = 0x02; - -static const int INCEPTION_VERSION = 3; - -static hexagon_nn_input* s_node_inputs_array; -static int s_node_inputs_array_index; -static int s_node_inputs_array_max_count; - -static hexagon_nn_output* s_node_outputs_array; -static int s_node_outputs_array_index; -static int s_node_outputs_array_max_count; - -int soc_interface_GetWrapperVersion() { - TFMLOGD("GetWrapperVersion"); - return hexagon_controller_GetWrapperVersion(); -} - -int soc_interface_GetSocControllerVersion() { - TFMLOGD("GetSocControllerVersion"); - return hexagon_controller_GetHexagonBinaryVersion(); -} - -bool soc_interface_Init() { - TFMLOGD("Init"); - hexagon_controller_InitHexagonWithMaxAttributes( - 0, 100, INCEPTION_VERSION /* version */); - hexagon_controller_GrowMemorySize(); - return true; -} - -bool soc_interface_Finalize() { - TFMLOGD("Finalize"); - hexagon_controller_DeInitHexagon(); - return true; -} - -bool soc_interface_ExecuteGraph() { - TFMLOGD("ExecuteGraph"); - if (hexagon_controller_IsDbgUseInceptionDummyDataEnabled()) { - hexagon_controller_InitInputNodeDataToInceptionDummyData( - INCEPTION_VERSION /* version */); - } - const uint32_t graph_id = hexagon_controller_GetTargetGraphId(); - if (graph_id == 0) { - TFMLOGE("Graph id has not been set yet."); - return false; - } - hexagon_controller_ExecuteGraphWithBuffer(graph_id, true); - return true; -} - -bool soc_interface_TeardownGraph() { - TFMLOGD("TeardownGraph"); - const uint32_t graph_id = hexagon_controller_GetTargetGraphId(); - if (graph_id == 0) { - TFMLOGE("Graph id has not been set yet."); - return false; - } - hexagon_controller_Teardown(graph_id); - return true; -} - -bool soc_interface_AllocateInOutNodeBuffers(int input_count, int* input_sizes, - int output_count, - int* output_sizes) { - TFMLOGD("AllocateInOutNodeBuffers"); - return hexagon_controller_AllocateMultipleNodeDataBuffers( - input_count, input_sizes, output_count, output_sizes); -} - -bool soc_interface_FillInputNodeWithPort(int port, int x, int y, int z, int d, - const uint8_t* const buf, - uint64_t buf_byte_size) { - TFMLOGD("FillInputNodeWithPort %d", port); - struct NodeDataFloat* node_data = - hexagon_controller_GetInputNodeDataBuffer(port); - if (buf_byte_size > node_data->max_buf_byte_size) { - TFMLOGE("buf size exceeds max buf size"); - return false; - } - memcpy(node_data->byte_array_data, buf, buf_byte_size); - node_data->x = x; - node_data->y = y; - node_data->z = z; - node_data->d = d; - node_data->array_byte_size = buf_byte_size; - return true; -} - -bool soc_interface_FillInputNodeFloat(int x, int y, int z, int d, - const uint8_t* const buf, - uint64_t buf_byte_size) { - return soc_interface_FillInputNodeWithPort( - /*port=*/0, x, y, z, d, buf, buf_byte_size); -} - -// TODO(satok): Remove and use runtime version -bool soc_interface_ReadOutputNodeWithPort(int port, uint8_t** buf, - uint64_t* buf_byte_size) { - TFMLOGD("ReadOutputNodeWithPort"); - int array_byte_size = -1; - uint8_t* output_node_data_buffer = - hexagon_controller_GetOutputNodeDataBuffer(port, &array_byte_size); - if (array_byte_size < 0) { - TFMLOGE("Failed to read data."); - return false; - } - *buf = output_node_data_buffer; - *buf_byte_size = array_byte_size; - return true; -} - -bool soc_interface_ReadOutputNodeFloat(const char* const node_name, - uint8_t** buf, uint64_t* buf_byte_size) { - return soc_interface_ReadOutputNodeWithPort(/*port=*/0, buf, buf_byte_size); -} - -bool soc_interface_setupDummyGraph(int version) { - TFMLOGD("SetupGraphDummy"); - const uint32_t graph_id = hexagon_controller_SetupGraph(version); - if (graph_id == 0) { - TFMLOGE("Failed to setup graph"); - return false; - } - hexagon_controller_SetTargetGraphId(graph_id); - return true; -} - -bool soc_interface_AllocateNodeInputAndNodeOutputArray(int total_input_count, - int total_output_count) { - TFMLOGD("Allocate node inputs and node outputs array %d, %d", - total_input_count, total_output_count); - posix_memalign((void**)&s_node_inputs_array, 128, - total_input_count * sizeof(hexagon_nn_input)); - posix_memalign((void**)&s_node_outputs_array, 128, - total_output_count * sizeof(hexagon_nn_output)); - s_node_inputs_array_index = 0; - s_node_outputs_array_index = 0; - s_node_inputs_array_max_count = total_input_count; - s_node_outputs_array_max_count = total_output_count; - return true; -} - -bool soc_interface_ReleaseNodeInputAndNodeOutputArray() { - TFMLOGD("Release node inputs and node outputs array"); - free(s_node_inputs_array); - free(s_node_outputs_array); - return true; -} - -void* soc_interface_SetOneNodeInputs( - int input_count, const int* const node_id, const int* const port) { - if (s_node_inputs_array_index + input_count > s_node_inputs_array_max_count) { - TFMLOGE("input count exceeds limit"); - return 0; - } - for (int i = 0; i < input_count; ++i) { - const int index = s_node_inputs_array_index + i; - s_node_inputs_array[index].src_id = node_id[i]; - s_node_inputs_array[index].output_idx = port[i]; - } - void* retval = (void*)(&s_node_inputs_array[s_node_inputs_array_index]); - s_node_inputs_array_index += input_count; - return retval; -} - -void* soc_interface_SetOneNodeOutputs(int output_count, int* max_size) { - if (s_node_outputs_array_index + output_count > - s_node_outputs_array_max_count) { - TFMLOGE("output count exceeds limit"); - return 0; - } - for (int i = 0; i < output_count; ++i) { - const int index = s_node_outputs_array_index + i; - s_node_outputs_array[index].max_size = max_size[i]; - } - void* retval = (void*)(&s_node_outputs_array[s_node_outputs_array_index]); - s_node_outputs_array_index += output_count; - return retval; -} - -// Append const node to the graph -bool soc_interface_AppendConstNode(const char* const name, int node_id, - int batch, int height, int width, int depth, - const uint8_t* const data, int data_length) { - const uint32_t graph_id = hexagon_controller_GetTargetGraphId(); - const int retval = hexagon_controller_AppendConstNode( - name, graph_id, node_id, batch, height, width, depth, data, data_length); - if (retval != 0) { - TFMLOGE("Failed to append const node %d", node_id); - return false; - } - return true; -} - -// Append node to the graph -bool soc_interface_AppendNode(const char* const name, int node_id, int ops_id, - int padding_id, const void* const inputs, - int inputs_count, const void* const outputs, - int outputs_count) { - const uint32_t graph_id = hexagon_controller_GetTargetGraphId(); - const int retval = hexagon_controller_AppendNode( - name, graph_id, node_id, ops_id, padding_id, (hexagon_nn_input*)inputs, - inputs_count, (hexagon_nn_output*)outputs, outputs_count); - if (retval != 0) { - TFMLOGE("Failed to append const node %d", node_id); - return false; - } - return true; -} - -// Instantiate graph -bool soc_interface_InstantiateGraph() { - const uint32_t nn_id = hexagon_controller_InstantiateGraph(); - hexagon_controller_SetTargetGraphId(nn_id); - return true; -} - -// Construct graph -bool soc_interface_ConstructGraph() { - const uint32_t graph_id = hexagon_controller_GetTargetGraphId(); - return hexagon_controller_ConstructGraph(graph_id); -} - -void soc_interface_SetLogLevel(int log_level) { - SetLogLevel(log_level); -} - -void soc_interface_SetDebugFlag(uint64_t flag) { - TFMLOGI("Set debug flag 0x%" PRIx64, flag); - if ((flag & FLAG_ENABLE_INCEPTION_DUMMY_BINARY_INPUT) != 0) { - TFMLOGI("Enable always use panda data"); - hexagon_controller_EnableDbgUseInceptionDummyData(true); - } else if ((flag & FLAG_ENABLE_EXPERIMENTAL_DEBUG) != 0) { - SetExperimentalDebug(); - } -} diff --git a/tensorflow/contrib/hvx/hexagon_controller/target/make/android.min b/tensorflow/contrib/hvx/hexagon_controller/target/make/android.min deleted file mode 100644 index 4770d31c56d..00000000000 --- a/tensorflow/contrib/hvx/hexagon_controller/target/make/android.min +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -$(info ------------------------------------------) -$(info --- V = $(V)) -$(info --- GLUE_DIR = $(GLUE_DIR)) -$(info --- HEXAGON_SDK_ROOT = $(HEXAGON_SDK_ROOT)) -$(info ------------------------------------------) - -INCDIRS += ../../../libs/common/adspmsgd/ship/android_Release - -INCDIRS += src_impl/include -INCDIRS += src_log/include -INCDIRS += src_soc_interface/include - -LIBDIRS += ../../../libs/common/adspmsgd/ship/android_Release - -BUILD_DLLS=libhexagon_controller - -hexagon_controller_lib_QAICIDLS += \ -interface/hexagon_nn \ -$(MAKE_D_DSPCV_INCDIR)/dspCV - -# hexagon controller library -hexagon_controller_lib_C_SRCS += \ -src_impl/hexagon_controller \ -src_impl/graph_functions_wrapper \ -src_soc_interface/soc_interface - -# dummy data -hexagon_controller_lib_C_SRCS += \ -src_dummy_data/inception_v1_graph_init \ -src_dummy_data/inception_v3_dummy_float_data \ -src_dummy_data/inception_v3_dummy_int_data \ -src_dummy_data/inception_v3_graph_init - -# hexagon interface -hexagon_controller_lib_C_SRCS += \ -$V/hexagon_nn_stub \ -$V/dspCV_stub - -hexagon_controller_lib_DLLS += libadsprpc -hexagon_controller_lib_LIBS += rpcmem adspmsgd -hexagon_controller_lib_LD_FLAGS += -llog -hexagon_controller_lib_DEFINES += VERIFY_PRINT_ERROR - -libhexagon_controller_QAICIDLS += $(hexagon_controller_lib_QAICIDLS) -libhexagon_controller_C_SRCS += $(hexagon_controller_lib_C_SRCS) -libhexagon_controller_DLLS += $(hexagon_controller_lib_DLLS) -libhexagon_controller_LIBS += $(hexagon_controller_lib_LIBS) -libhexagon_controller_LD_FLAGS += $(hexagon_controller_lib_LD_FLAGS) -libhexagon_controller_DEFINES += $(hexagon_controller_lib_DEFINES) - -BUILD_COPIES = \ - $(DLLS) \ - $(EXES) \ - $(LIBS) \ - $(SHIP_DIR)/ ; diff --git a/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD b/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD deleted file mode 100644 index 4f89f544ecb..00000000000 --- a/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD +++ /dev/null @@ -1,49 +0,0 @@ -# Description: -# Contains a tool to dump TensorFlow ops which are not supported -# in TensorFlow HVX runtime. - -load("//tensorflow:tensorflow.bzl", "tf_cc_binary") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_cc_binary( - name = "hvx_ops_support_checker", - testonly = 1, - srcs = ["hvx_ops_support_checker_main.cc"], - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/core:array_ops_op_lib", - "//tensorflow/core:candidate_sampling_ops_op_lib", - "//tensorflow/core:control_flow_ops_op_lib", - "//tensorflow/core:data_flow_ops_op_lib", - "//tensorflow/core:framework_internal", - "//tensorflow/core:functional_ops_op_lib", - "//tensorflow/core:io_ops_op_lib", - "//tensorflow/core:lib", - "//tensorflow/core:list_ops_op_lib", - "//tensorflow/core:logging_ops_op_lib", - "//tensorflow/core:lookup_ops_op_lib", - "//tensorflow/core:manip_ops_op_lib", - "//tensorflow/core:math_ops_op_lib", - "//tensorflow/core:nn_ops_op_lib", - "//tensorflow/core:no_op_op_lib", - "//tensorflow/core:parsing_ops_op_lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:random_ops_op_lib", - "//tensorflow/core:remote_fused_graph_ops_op_lib", - "//tensorflow/core:sendrecv_ops_op_lib", - "//tensorflow/core:sparse_ops_op_lib", - "//tensorflow/core:state_ops_op_lib", - "//tensorflow/core:string_ops_op_lib", - "//tensorflow/core:training_ops_op_lib", - "//tensorflow/core:user_ops_op_lib", - "//tensorflow/core/kernels:remote_fused_graph_execute_utils", - "//tensorflow/core/kernels/hexagon:graph_transferer", - "//tensorflow/tools/graph_transforms:file_utils", - ], -) diff --git a/tensorflow/contrib/hvx/hvx_ops_support_checker/hvx_ops_support_checker_main.cc b/tensorflow/contrib/hvx/hvx_ops_support_checker/hvx_ops_support_checker_main.cc deleted file mode 100644 index 66939fbb0f0..00000000000 --- a/tensorflow/contrib/hvx/hvx_ops_support_checker/hvx_ops_support_checker_main.cc +++ /dev/null @@ -1,177 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// bazel build tensorflow/contrib/hvx/hvx_ops_support_checker && -// bazel-bin/tensorflow/contrib/hvx/hvx_ops_support_checker/hvx_ops_support_checker -// \ -// --in_graph=graph_def.pb - -#include - -#include "tensorflow/core/framework/graph.pb.h" -#include "tensorflow/core/framework/node_def.pb.h" -#include "tensorflow/core/framework/remote_fused_graph_execute_info.pb.h" -#include "tensorflow/core/kernels/hexagon/hexagon_ops_definitions.h" -#include "tensorflow/core/kernels/i_remote_fused_graph_ops_definitions.h" -#include "tensorflow/core/kernels/remote_fused_graph_execute_utils.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/util/command_line_flags.h" -#include "tensorflow/tools/graph_transforms/file_utils.h" - -namespace tensorflow { - -namespace { -static int ParseFlags(int argc, char* argv[], string* in_graph, - bool* dump_all_nodes, bool* dump_shape_and_type) { - std::vector flag_list = { - Flag("in_graph", in_graph, "Input graph file name to check hvx support."), - Flag("dump_all_nodes", dump_all_nodes, "Dump all nodes in the model."), - Flag("dump_shape_and_type", dump_shape_and_type, - "Dump shape and type of nodes"), - }; - CHECK(Flags::Parse(&argc, argv, flag_list)); - // We need to call this to set up global state for TensorFlow. - port::InitMain(argv[0], &argc, &argv); - - string usage = Flags::Usage(argv[0], flag_list); - CHECK(!in_graph->empty()) << "in_graph graph can't be empty.\n" << usage; - - return 0; -} - -static void SummarizeNode(const NodeDef& node_def, - const bool dump_shape_and_type) { - LOG(INFO) << "Node(" << node_def.name() << ")"; - LOG(INFO) << " op: " << node_def.op(); - for (const string& input : node_def.input()) { - LOG(INFO) << " Input: " << input; - } - std::vector data_types; - std::vector shapes; - const Status status = RemoteFusedGraphExecuteUtils::GetOutputTensorShapeType( - node_def, &data_types, &shapes); - if (data_types.empty() || shapes.empty()) { - return; - } - CHECK_EQ(data_types.size(), shapes.size()); - for (int i = 0; i < data_types.size(); ++i) { - LOG(INFO) << " Output(" << i << "): " << DataType_Name(data_types.at(i)) - << ", " << shapes.at(i).DebugString(); - } -} - -static void DumpRemoteFusedGraph(const NodeDef& node_def) { - LOG(INFO) << "Remote fused graph found."; - RemoteFusedGraphExecuteInfo info; - string serialized_proto; - GetNodeAttr(node_def, - RemoteFusedGraphExecuteUtils:: - ATTR_SERIALIZED_REMOTE_FUSED_GRAPH_EXECUTE_INFO, - &serialized_proto) - .IgnoreError(); - info.ParseFromString(serialized_proto); - LOG(INFO) << "Node name: " << node_def.name(); - LOG(INFO) << "Executor name: " << info.executor_name(); - for (const string& input : info.graph_input_node_name()) { - LOG(INFO) << "Input: " << input; - } - for (const RemoteFusedGraphExecuteInfo::TensorShapeTypeProto& shape_type : - info.default_graph_input_tensor_shape()) { - LOG(INFO) << "Input shape type: " << shape_type.DebugString(); - } - for (const string& output : info.graph_output_node_name()) { - LOG(INFO) << "Output: " << output; - } - for (const RemoteFusedGraphExecuteInfo::TensorShapeTypeProto& shape_type : - info.default_graph_output_tensor_shape()) { - LOG(INFO) << "Output shape type: " << shape_type.DebugString(); - } - const int subgraph_node_size = info.remote_graph().node_size(); - LOG(INFO) << "Nodes in the graph: " << subgraph_node_size; - for (int i = 0; i < subgraph_node_size; ++i) { - LOG(INFO) << "node(" << i << "): " << info.remote_graph().node(i).name(); - } -} - -static void CheckOpsSupport(const GraphDef& graph_def, - const bool dump_all_nodes, - const bool dump_shape_and_type) { - const IRemoteFusedGraphOpsDefinitions& ops_definition = - HexagonOpsDefinitions::getInstance(); - LOG(INFO) << "Checking " << graph_def.node_size() << " nodes"; - LOG(INFO) << "dump_all_nodes = " << dump_all_nodes - << ", dump_shape_and_type = " << dump_shape_and_type; - - std::unordered_set unsupported_ops; - bool all_supported = true; - bool contains_remote_graph = false; - for (const NodeDef& node : graph_def.node()) { - if (node.op() == "RemoteFusedGraphExecute") { - contains_remote_graph = true; - DumpRemoteFusedGraph(node); - continue; - } - // TODO(satok): Set correct data type if it's given. - const int op_id = ops_definition.GetOpIdFor(node.op(), {}); - if (op_id == IRemoteFusedGraphOpsDefinitions::INVALID_OP_ID) { - all_supported = false; - LOG(ERROR) << "OP type: " << node.op() << " is not supported on hvx. " - << "Name = " << node.name(); - unsupported_ops.emplace(node.op()); - } - } - - LOG(INFO) << "\n"; - LOG(INFO) << "Unsupported ops:"; - int count = 0; - for (const string& op_type : unsupported_ops) { - LOG(INFO) << "(" << (++count) << ") " << op_type; - } - if (count == 0) { - LOG(INFO) << "All ops supported!"; - } else { - LOG(INFO) << count << " ops are not supported."; - } - - if (contains_remote_graph || dump_all_nodes) { - for (const NodeDef& node : graph_def.node()) { - SummarizeNode(node, dump_shape_and_type); - } - } -} - -} // namespace -} // namespace tensorflow - -int main(int argc, char** argv) { - tensorflow::string in_graph; - bool dump_all_nodes; - bool dump_shape_and_type; - const int ret = tensorflow::ParseFlags(argc, argv, &in_graph, &dump_all_nodes, - &dump_shape_and_type); - if (ret != 0) { - return ret; - } - - tensorflow::GraphDef graph_def; - TF_CHECK_OK(tensorflow::graph_transforms::LoadTextOrBinaryGraphFile( - in_graph, &graph_def)); - - tensorflow::CheckOpsSupport(graph_def, dump_all_nodes, dump_shape_and_type); - return 0; -} diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD deleted file mode 100755 index 4b14b9e08cf..00000000000 --- a/tensorflow/contrib/image/BUILD +++ /dev/null @@ -1,421 +0,0 @@ -# Description: -# Contains ops for image manipulation. - -load( - "//tensorflow:tensorflow.bzl", - "tf_cc_test", - "tf_custom_op_library", - "tf_gen_op_libs", - "tf_gen_op_wrapper_py", - "tf_kernel_library", - "tf_py_test", -) -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_custom_op_library( - name = "python/ops/_image_ops.so", - srcs = [ - "kernels/bipartite_match_op.cc", - "kernels/image_ops.cc", - "kernels/image_ops.h", - "kernels/segmentation_ops.cc", - "kernels/segmentation_ops.h", - "ops/image_ops.cc", - ], - gpu_srcs = [ - "kernels/image_ops_gpu.cu.cc", - "kernels/image_ops.h", - ], -) - -tf_kernel_library( - name = "image_ops_kernels", - srcs = [ - "kernels/bipartite_match_op.cc", - "kernels/image_ops.cc", - "kernels/image_ops.h", - "kernels/segmentation_ops.cc", - "kernels/segmentation_ops.h", - ], - gpu_srcs = [ - "kernels/image_ops_gpu.cu.cc", - "kernels/image_ops.h", - ], - deps = [ - ":image_ops_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//third_party/eigen3", - ], - alwayslink = 1, -) - -tf_gen_op_libs( - op_lib_names = ["image_ops"], -) - -tf_gen_op_wrapper_py( - name = "image_ops", - deps = [":image_ops_op_lib"], -) - -tf_custom_op_py_library( - name = "image_py", - srcs = [ - "__init__.py", - "python/ops/image_ops.py", - ], - dso = [":python/ops/_image_ops.so"], - kernels = [ - ":image_ops_kernels", - ], - srcs_version = "PY2AND3", - deps = [ - ":dense_image_warp_py", - ":image_ops", - ":interpolate_spline_py", - ":sparse_image_warp_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:common_shapes", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:util", - ], -) - -cuda_py_test( - name = "image_ops_test", - size = "medium", - srcs = ["python/kernel_tests/image_ops_test.py"], - additional_deps = [ - ":distort_image_py", - ":image_py", - ":single_image_random_dot_stereograms_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -tf_custom_op_library( - name = "python/ops/_distort_image_ops.so", - srcs = [ - "kernels/adjust_hsv_in_yiq_op.cc", - "kernels/adjust_hsv_in_yiq_op.h", - "ops/distort_image_ops.cc", - ], - gpu_srcs = [ - "kernels/adjust_hsv_in_yiq_op_gpu.cu.cc", - "kernels/adjust_hsv_in_yiq_op.h", - ], - deps = [ - "//tensorflow/core/kernels:gpu_util_hdrs", - ], -) - -tf_kernel_library( - name = "distort_image_ops_kernels", - srcs = [ - "kernels/adjust_hsv_in_yiq_op.cc", - "kernels/adjust_hsv_in_yiq_op.h", - ], - gpu_srcs = [ - "kernels/adjust_hsv_in_yiq_op_gpu.cu.cc", - "kernels/adjust_hsv_in_yiq_op.h", - ], - deps = [ - ":distort_image_ops_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core/kernels:gpu_util_hdrs", - "//third_party/eigen3", - ], - alwayslink = 1, -) - -tf_cc_test( - name = "adjust_hsv_in_yiq_op_test", - size = "small", - srcs = [ - "kernels/adjust_hsv_in_yiq_op.h", - "kernels/adjust_hsv_in_yiq_op_test.cc", - ], - deps = [ - "//tensorflow/core:framework", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - "//tensorflow/core/kernels:ops_testutil", - "//tensorflow/core/kernels:ops_util", - "//third_party/eigen3", - ], -) - -tf_gen_op_libs( - op_lib_names = ["distort_image_ops"], -) - -tf_gen_op_wrapper_py( - name = "distort_image_ops", - deps = [":distort_image_ops_op_lib"], -) - -tf_custom_op_py_library( - name = "distort_image_py", - srcs = [ - "__init__.py", - "python/ops/distort_image_ops.py", - ], - dso = [":python/ops/_distort_image_ops.so"], - kernels = [ - ":distort_image_ops_kernels", - ], - srcs_version = "PY2AND3", - deps = [ - ":distort_image_ops", - ":single_image_random_dot_stereograms_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:image_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:util", - ], -) - -cuda_py_test( - name = "distort_image_ops_test", - size = "medium", - srcs = ["python/kernel_tests/distort_image_ops_test.py"], - additional_deps = [ - ":distort_image_py", - ":image_py", - ":single_image_random_dot_stereograms_py", - "//third_party/py/numpy", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - "//tensorflow/python:variables", - "//tensorflow/core:protos_all_py", - ], -) - -py_library( - name = "dense_image_warp_py", - srcs = [ - "python/ops/dense_image_warp.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:platform", - "//tensorflow/python:util", - "//third_party/py/numpy", - ], -) - -py_library( - name = "interpolate_spline_py", - srcs = [ - "python/ops/interpolate_spline.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:platform", - "//tensorflow/python:util", - ], -) - -py_library( - name = "sparse_image_warp_py", - srcs = [ - "python/ops/sparse_image_warp.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":dense_image_warp_py", - ":interpolate_spline_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:platform", - "//tensorflow/python:util", - ], -) - -cuda_py_test( - name = "sparse_image_warp_test", - size = "medium", - srcs = ["python/kernel_tests/sparse_image_warp_test.py"], - additional_deps = [ - ":sparse_image_warp_py", - "//third_party/py/numpy", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:clip_ops", - "//tensorflow/python:io_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - "//tensorflow/python:image_ops", - "//tensorflow/python:variables", - "//tensorflow/core:protos_all_py", - ], - data = [":sparse_image_warp_test_data"], - tags = ["no_pip"], -) - -filegroup( - name = "sparse_image_warp_test_data", - srcs = glob(["python/kernel_tests/test_data/*.png"]), -) - -cuda_py_test( - name = "dense_image_warp_test", - size = "medium", - srcs = ["python/kernel_tests/dense_image_warp_test.py"], - additional_deps = [ - ":dense_image_warp_py", - "//third_party/py/numpy", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:clip_ops", - "//tensorflow/python:io_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - "//tensorflow/python:image_ops", - "//tensorflow/python:variables", - "//tensorflow/core:protos_all_py", - ], -) - -cuda_py_test( - name = "interpolate_spline_test", - size = "medium", - srcs = ["python/kernel_tests/interpolate_spline_test.py"], - additional_deps = [ - ":interpolate_spline_py", - "//third_party/py/numpy", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:clip_ops", - "//tensorflow/python:io_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:image_ops", - "//tensorflow/python:variables", - "//tensorflow/core:protos_all_py", - ], -) - -tf_py_test( - name = "segmentation_test", - size = "medium", - srcs = ["python/kernel_tests/segmentation_test.py"], - additional_deps = [ - ":distort_image_py", - ":image_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) - -tf_custom_op_library( - name = "python/ops/_single_image_random_dot_stereograms.so", - srcs = [ - "kernels/single_image_random_dot_stereograms_ops.cc", - "ops/single_image_random_dot_stereograms_ops.cc", - ], - deps = [ - "@com_google_protobuf//:protobuf", - ], -) - -tf_gen_op_libs( - op_lib_names = ["single_image_random_dot_stereograms_ops"], -) - -tf_kernel_library( - name = "single_image_random_dot_stereograms_kernels", - srcs = [ - "kernels/single_image_random_dot_stereograms_ops.cc", - ], - deps = [ - ":single_image_random_dot_stereograms_ops_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//third_party/eigen3", - ], -) - -tf_gen_op_wrapper_py( - name = "single_image_random_dot_stereograms_ops", - deps = [":single_image_random_dot_stereograms_ops_op_lib"], -) - -alias( - name = "image_ops_cc", - actual = ":image_ops_op_lib", -) - -tf_custom_op_py_library( - name = "single_image_random_dot_stereograms_py", - srcs = glob(["python/ops/single*.py"]) + ["__init__.py"], - dso = [":python/ops/_single_image_random_dot_stereograms.so"], - kernels = [ - ":single_image_random_dot_stereograms_kernels", - ], - srcs_version = "PY2AND3", - deps = [ - ":image_py", - ":single_image_random_dot_stereograms_ops", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:framework_ops", - "//tensorflow/python:platform", - "//tensorflow/python:util", - ], -) - -cuda_py_test( - name = "single_image_random_dot_stereograms_ops_test", - size = "medium", - srcs = ["python/kernel_tests/single_image_random_dot_stereograms_ops_test.py"], - additional_deps = [ - ":distort_image_py", - ":image_py", - ":single_image_random_dot_stereograms_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], -) diff --git a/tensorflow/contrib/image/__init__.py b/tensorflow/contrib/image/__init__.py deleted file mode 100755 index 91b8e8d0f93..00000000000 --- a/tensorflow/contrib/image/__init__.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for image manipulation. - -### API - -This module provides functions for image manipulation; currently, chrominance -transforms (including changing saturation and hue) in YIQ space and -projective transforms (including rotation) are supported. - -## Image Transformation `Ops` - -@@angles_to_projective_transforms -@@compose_transforms -@@adjust_yiq_hsv -@@flat_transforms_to_matrices -@@matrices_to_flat_transforms -@@random_yiq_hsv -@@rotate -@@transform -@@translate -@@translations_to_projective_transforms -@@dense_image_warp -@@interpolate_spline -@@sparse_image_warp - -## Image Segmentation `Ops` - -@@connected_components - -## Matching `Ops` - -@@bipartite_match - -## Random Dot Stereogram `Ops` - -@@single_image_random_dot_stereograms -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.image.python.ops.dense_image_warp import dense_image_warp - -from tensorflow.contrib.image.python.ops.distort_image_ops import adjust_hsv_in_yiq -from tensorflow.contrib.image.python.ops.distort_image_ops import random_hsv_in_yiq - -from tensorflow.contrib.image.python.ops.image_ops import angles_to_projective_transforms -from tensorflow.contrib.image.python.ops.image_ops import bipartite_match -from tensorflow.contrib.image.python.ops.image_ops import compose_transforms -from tensorflow.contrib.image.python.ops.image_ops import connected_components -from tensorflow.contrib.image.python.ops.image_ops import flat_transforms_to_matrices -from tensorflow.contrib.image.python.ops.image_ops import matrices_to_flat_transforms -from tensorflow.contrib.image.python.ops.image_ops import rotate -from tensorflow.contrib.image.python.ops.image_ops import transform -from tensorflow.contrib.image.python.ops.image_ops import translate -from tensorflow.contrib.image.python.ops.image_ops import translations_to_projective_transforms -from tensorflow.contrib.image.python.ops.interpolate_spline import interpolate_spline -from tensorflow.contrib.image.python.ops.single_image_random_dot_stereograms import single_image_random_dot_stereograms -from tensorflow.contrib.image.python.ops.sparse_image_warp import sparse_image_warp - -from tensorflow.python.util.all_util import remove_undocumented - -# pylint: enable=line-too-long - -remove_undocumented(__name__) diff --git a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.cc b/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.cc deleted file mode 100644 index 108da044946..00000000000 --- a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.cc +++ /dev/null @@ -1,168 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#if GOOGLE_CUDA -#define EIGEN_USE_GPU -#endif - -#include "tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.h" -#include -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -typedef Eigen::ThreadPoolDevice CPUDevice; -typedef Eigen::GpuDevice GPUDevice; - -class AdjustHsvInYiqOpBase : public OpKernel { - protected: - explicit AdjustHsvInYiqOpBase(OpKernelConstruction* context) - : OpKernel(context) {} - - struct ComputeOptions { - const Tensor* input = nullptr; - Tensor* output = nullptr; - const Tensor* delta_h = nullptr; - const Tensor* scale_s = nullptr; - const Tensor* scale_v = nullptr; - int64 channel_count = 0; - }; - - virtual void DoCompute(OpKernelContext* context, - const ComputeOptions& options) = 0; - - void Compute(OpKernelContext* context) override { - const Tensor& input = context->input(0); - const Tensor& delta_h = context->input(1); - const Tensor& scale_s = context->input(2); - const Tensor& scale_v = context->input(3); - OP_REQUIRES(context, input.dims() >= 3, - errors::InvalidArgument("input must be at least 3-D, got shape", - input.shape().DebugString())); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(delta_h.shape()), - errors::InvalidArgument("delta_h must be scalar: ", - delta_h.shape().DebugString())); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(scale_s.shape()), - errors::InvalidArgument("scale_s must be scalar: ", - scale_s.shape().DebugString())); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(scale_v.shape()), - errors::InvalidArgument("scale_v must be scalar: ", - scale_v.shape().DebugString())); - auto channels = input.dim_size(input.dims() - 1); - OP_REQUIRES( - context, channels == kChannelSize, - errors::InvalidArgument("input must have 3 channels but instead has ", - channels, " channels.")); - - Tensor* output = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(0, input.shape(), &output)); - - if (input.NumElements() > 0) { - const int64 channel_count = input.NumElements() / channels; - ComputeOptions options; - options.input = &input; - options.delta_h = &delta_h; - options.scale_s = &scale_s; - options.scale_v = &scale_v; - options.output = output; - options.channel_count = channel_count; - DoCompute(context, options); - } - } -}; - -template -class AdjustHsvInYiqOp; - -template <> -class AdjustHsvInYiqOp : public AdjustHsvInYiqOpBase { - public: - explicit AdjustHsvInYiqOp(OpKernelConstruction* context) - : AdjustHsvInYiqOpBase(context) {} - - void DoCompute(OpKernelContext* context, - const ComputeOptions& options) override { - const Tensor* input = options.input; - Tensor* output = options.output; - const int64 channel_count = options.channel_count; - auto input_data = input->shaped({channel_count, kChannelSize}); - const float delta_h = options.delta_h->scalar()(); - const float scale_s = options.scale_s->scalar()(); - const float scale_v = options.scale_v->scalar()(); - auto output_data = output->shaped({channel_count, kChannelSize}); - float tranformation_matrix[kChannelSize * kChannelSize] = {0}; - internal::compute_tranformation_matrix( - delta_h, scale_s, scale_v, tranformation_matrix); - const int kCostPerChannel = 10; - const DeviceBase::CpuWorkerThreads& worker_threads = - *context->device()->tensorflow_cpu_worker_threads(); - Shard(worker_threads.num_threads, worker_threads.workers, channel_count, - kCostPerChannel, - [&input_data, &output_data, &tranformation_matrix]( - int64 start_channel, int64 end_channel) { - // Applying projection matrix to input RGB vectors. - const float* p = input_data.data() + start_channel * kChannelSize; - float* q = output_data.data() + start_channel * kChannelSize; - for (int i = start_channel; i < end_channel; i++) { - for (int q_index = 0; q_index < kChannelSize; q_index++) { - q[q_index] = 0; - for (int p_index = 0; p_index < kChannelSize; p_index++) { - q[q_index] += - p[p_index] * - tranformation_matrix[q_index + kChannelSize * p_index]; - } - } - p += kChannelSize; - q += kChannelSize; - } - }); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("AdjustHsvInYiq").Device(DEVICE_CPU).TypeConstraint("T"), - AdjustHsvInYiqOp); - -#if GOOGLE_CUDA -template <> -class AdjustHsvInYiqOp : public AdjustHsvInYiqOpBase { - public: - explicit AdjustHsvInYiqOp(OpKernelConstruction* context) - : AdjustHsvInYiqOpBase(context) {} - - void DoCompute(OpKernelContext* ctx, const ComputeOptions& options) override { - const int64 number_of_elements = options.input->NumElements(); - if (number_of_elements <= 0) { - return; - } - const float* delta_h = options.delta_h->flat().data(); - const float* scale_s = options.scale_s->flat().data(); - const float* scale_v = options.scale_v->flat().data(); - functor::AdjustHsvInYiqGPU()(ctx, options.channel_count, options.input, - delta_h, scale_s, scale_v, options.output); - } -}; - -REGISTER_KERNEL_BUILDER( - Name("AdjustHsvInYiq").Device(DEVICE_GPU).TypeConstraint("T"), - AdjustHsvInYiqOp); -#endif - -} // namespace tensorflow diff --git a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.h b/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.h deleted file mode 100644 index 8968da6d824..00000000000 --- a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.h +++ /dev/null @@ -1,87 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#ifndef TENSORFLOW_CONTRIB_IMAGE_KERNELS_ADJUST_HSV_IN_YIQ_OP_H_ -#define TENSORFLOW_CONTRIB_IMAGE_KERNELS_ADJUST_HSV_IN_YIQ_OP_H_ - -#if GOOGLE_CUDA -#define EIGEN_USE_GPU -#endif // GOOGLE_CUDA - -#include -#include "third_party/eigen3/Eigen/Core" -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" - -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/framework/types.h" - -namespace tensorflow { - -static constexpr int kChannelSize = 3; - -namespace internal { - -template -EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void compute_tranformation_matrix( - const float delta_h, const float scale_s, const float scale_v, - float* matrix) { - static_assert(MATRIX_SIZE == kChannelSize * kChannelSize, - "Size of matrix should be 9."); - // Projection matrix from RGB to YIQ. Numbers from wikipedia - // https://en.wikipedia.org/wiki/YIQ - Eigen::Matrix3f yiq; - /* clang-format off */ - yiq << 0.299, 0.587, 0.114, - 0.596, -0.274, -0.322, - 0.211, -0.523, 0.312; - Eigen::Matrix3f yiq_inverse; - yiq_inverse << 1, 0.95617069, 0.62143257, - 1, -0.2726886, -0.64681324, - 1, -1.103744, 1.70062309; - /* clang-format on */ - // Construct hsv linear transformation matrix in YIQ space. - // https://beesbuzz.biz/code/hsv_color_transforms.php - float vsu = scale_v * scale_s * std::cos(delta_h); - float vsw = scale_v * scale_s * std::sin(delta_h); - Eigen::Matrix3f hsv_transform; - /* clang-format off */ - hsv_transform << scale_v, 0, 0, - 0, vsu, -vsw, - 0, vsw, vsu; - /* clang-format on */ - // Compute final transformation matrix = inverse_yiq * hsv_transform * yiq - Eigen::Map> eigen_matrix(matrix); - eigen_matrix = yiq_inverse * hsv_transform * yiq; -} -} // namespace internal - -#if GOOGLE_CUDA -typedef Eigen::GpuDevice GPUDevice; - -namespace functor { - -struct AdjustHsvInYiqGPU { - void operator()(OpKernelContext* ctx, int channel_count, - const Tensor* const input, const float* const delta_h, - const float* const scale_s, const float* const scale_v, - Tensor* const output); -}; - -} // namespace functor - -#endif // GOOGLE_CUDA - -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_IMAGE_KERNELS_ADJUST_HSV_IN_YIQ_OP_H_ diff --git a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op_gpu.cu.cc b/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op_gpu.cu.cc deleted file mode 100644 index cf11a670109..00000000000 --- a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op_gpu.cu.cc +++ /dev/null @@ -1,85 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#if GOOGLE_CUDA - -#define EIGEN_USE_GPU - -#include "tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.h" -#include "tensorflow/core/kernels/gpu_utils.h" -#include "tensorflow/core/platform/stream_executor.h" -#include "tensorflow/core/util/gpu_kernel_helper.h" - -namespace tensorflow { - -namespace internal { - -__global__ void compute_tranformation_matrix_cuda(const float* const delta_h, - const float* const scale_s, - const float* const scale_v, - float* const matrix, - const int matrix_size) { - if (matrix_size == kChannelSize * kChannelSize) { - compute_tranformation_matrix( - *delta_h, *scale_s, *scale_v, matrix); - } -} -} // namespace internal - -namespace functor { - -void AdjustHsvInYiqGPU::operator()(OpKernelContext* ctx, int channel_count, - const Tensor* const input, - const float* const delta_h, - const float* const scale_s, - const float* const scale_v, - Tensor* const output) { - const uint64 m = channel_count; - const uint64 k = kChannelSize; - const uint64 n = kChannelSize; - auto* cu_stream = ctx->eigen_device().stream(); - OP_REQUIRES(ctx, cu_stream, errors::Internal("No GPU stream available.")); - Tensor tranformation_matrix; - OP_REQUIRES_OK(ctx, ctx->allocate_temp( - DT_FLOAT, TensorShape({kChannelSize * kChannelSize}), - &tranformation_matrix)); - // TODO(huangyp): It takes about 3.5 us to compute tranformation_matrix - // with one thread. Improve its performance if necessary. - TF_CHECK_OK(GpuLaunchKernel(internal::compute_tranformation_matrix_cuda, 1, 1, - 0, cu_stream, delta_h, scale_s, scale_v, - tranformation_matrix.flat().data(), - tranformation_matrix.flat().size())); - // Call cuBlas C = A * B directly. - auto no_transpose = se::blas::Transpose::kNoTranspose; - auto a_ptr = - AsDeviceMemory(input->flat().data(), input->flat().size()); - auto b_ptr = AsDeviceMemory(tranformation_matrix.flat().data(), - tranformation_matrix.flat().size()); - auto c_ptr = AsDeviceMemory(output->flat().data(), - output->flat().size()); - auto* stream = ctx->op_device_context()->stream(); - OP_REQUIRES(ctx, stream, errors::Internal("No GPU stream available.")); - // TODO(huangyp): share/use autotune cublas algorithms in Matmul.op. - bool blas_launch_status = - stream - ->ThenBlasGemm(no_transpose, no_transpose, n, m, k, 1.0f, b_ptr, n, - a_ptr, k, 0.0f, &c_ptr, n) - .ok(); - if (!blas_launch_status) { - ctx->SetStatus(errors::Internal("Blas SGEMM launch failed : m=", m, - ", n=", n, ", k=", k)); - } -} -} // namespace functor -} // namespace tensorflow -#endif // GOOGLE_CUDA diff --git a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op_test.cc b/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op_test.cc deleted file mode 100644 index 4cbbd277840..00000000000 --- a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op_test.cc +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/kernels/ops_testutil.h" -#include "tensorflow/core/kernels/ops_util.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { - -class AdjustHsvInYiqOpTest : public OpsTestBase { - protected: -}; - -TEST_F(AdjustHsvInYiqOpTest, IdentiyTransformMatrix) { - Tensor matrix(allocator(), DT_FLOAT, TensorShape({9})); - internal::compute_tranformation_matrix<9>(0.0, 1.0, 1.0, - matrix.flat().data()); - Tensor expected(allocator(), DT_FLOAT, TensorShape({9})); - test::FillValues(&expected, {1, 0, 0, 0, 1, 0, 0, 0, 1}); - test::ExpectClose(matrix, expected); -} - -TEST_F(AdjustHsvInYiqOpTest, ScaleValueTransformMatrix) { - float scale_v = 2.3; - Tensor matrix(allocator(), DT_FLOAT, TensorShape({9})); - internal::compute_tranformation_matrix<9>(0.0, 1.0, scale_v, - matrix.flat().data()); - Tensor expected(allocator(), DT_FLOAT, TensorShape({9})); - test::FillValues(&expected, - {scale_v, 0, 0, 0, scale_v, 0, 0, 0, scale_v}); - test::ExpectClose(matrix, expected); -} - -} // end namespace tensorflow diff --git a/tensorflow/contrib/image/kernels/bipartite_match_op.cc b/tensorflow/contrib/image/kernels/bipartite_match_op.cc deleted file mode 100644 index 726adb07775..00000000000 --- a/tensorflow/contrib/image/kernels/bipartite_match_op.cc +++ /dev/null @@ -1,134 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include -#include -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.h" - -namespace { - -struct DistancePair { - DistancePair(int i1, int i2, double d) : index1(i1), index2(i2), dist(d) {} - - bool operator<(const DistancePair& b1) const { return b1.dist < dist; } - - int index1, index2; - float dist; -}; - -} // namespace - -namespace tensorflow { - -class BipartiteMatchOp : public OpKernel { - public: - explicit BipartiteMatchOp(OpKernelConstruction* context) : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr("top_k", &top_k_)); - } - - void Compute(OpKernelContext* context) override { - const Tensor& input_distance_mat = context->input(0); - OP_REQUIRES(context, input_distance_mat.dims() == 2, - errors::InvalidArgument( - "distance_mat should be 2-dimensional, but got ", - input_distance_mat.shape().DebugString())); - const int num_input_rows = input_distance_mat.dim_size(0); - const int num_input_columns = input_distance_mat.dim_size(1); - - const Tensor& input_num_valid_rows = context->input(1); - OP_REQUIRES( - context, input_num_valid_rows.NumElements() == 1, - errors::InvalidArgument( - "num_valid_rows argument should be a tensor with 1 element, " - "but got ", - input_num_valid_rows.NumElements())); - - const float num_valid_rows_f = input_num_valid_rows.flat()(0); - int num_valid_rows = num_input_rows; - // If num_valid_rows_f is non-negative, use it to set num_valid_rows. - if (num_valid_rows_f >= 0) { - num_valid_rows = static_cast(num_valid_rows_f + 0.1); - } - OP_REQUIRES( - context, num_input_rows >= num_valid_rows, - errors::InvalidArgument("There should be at least ", num_valid_rows, - " rows in distance_mat, but only got ", - num_input_rows, " rows.")); - - // If negative or zero then set it to the maximum possible matches. - auto valid_top_k = top_k_; - - if (valid_top_k <= 0) { - valid_top_k = num_valid_rows * num_input_columns; - } - - // Create output tensors. - Tensor* row_to_column_match_indices = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(0, TensorShape({num_input_rows}), - &row_to_column_match_indices)); - Tensor* column_to_row_match_indices = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(1, TensorShape({num_input_columns}), - &column_to_row_match_indices)); - - TTypes::ConstTensor distance_mat = - input_distance_mat.shaped( - {num_input_rows, num_input_columns}); - - // Greedy bi-partite matching. - std::priority_queue match_queue; - - for (int index1 = 0; index1 < num_valid_rows; index1++) { - for (int index2 = 0; index2 < num_input_columns; index2++) { - match_queue.push( - DistancePair(index1, index2, distance_mat(index1, index2))); - } - } - - std::vector row_to_col_match_vec(num_input_rows, -1); - std::vector col_to_row_match_vec(num_input_columns, -1); - int index = 0; - while (!match_queue.empty()) { - const auto& match = match_queue.top(); - if (row_to_col_match_vec[match.index1] == -1 && - col_to_row_match_vec[match.index2] == -1) { - row_to_col_match_vec[match.index1] = match.index2; - col_to_row_match_vec[match.index2] = match.index1; - - index++; - if (index >= valid_top_k) { - break; - } - } - match_queue.pop(); - } - - // Set the output tensors. - row_to_column_match_indices->vec() = - TTypes::Vec(row_to_col_match_vec.data(), num_input_rows); - column_to_row_match_indices->vec() = - TTypes::Vec(col_to_row_match_vec.data(), num_input_columns); - } - - private: - int top_k_; -}; - -REGISTER_KERNEL_BUILDER(Name("BipartiteMatch").Device(DEVICE_CPU), - BipartiteMatchOp); - -} // namespace tensorflow diff --git a/tensorflow/contrib/image/kernels/image_ops.cc b/tensorflow/contrib/image/kernels/image_ops.cc deleted file mode 100644 index 788bf04b28a..00000000000 --- a/tensorflow/contrib/image/kernels/image_ops.cc +++ /dev/null @@ -1,187 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#define EIGEN_USE_THREADS - -#if GOOGLE_CUDA -#define EIGEN_USE_GPU -#endif // GOOGLE_CUDA - -#include "tensorflow/contrib/image/kernels/image_ops.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { - -namespace functor { - -// Explicit instantiation of the CPU functor. -typedef Eigen::ThreadPoolDevice CPUDevice; - -template struct FillProjectiveTransform; -template struct FillProjectiveTransform; -template struct FillProjectiveTransform; -template struct FillProjectiveTransform; -template struct FillProjectiveTransform; -template struct FillProjectiveTransform; - -} // end namespace functor - -typedef Eigen::ThreadPoolDevice CPUDevice; - -using functor::FillProjectiveTransform; -using generator::Interpolation; -using generator::INTERPOLATION_BILINEAR; -using generator::INTERPOLATION_NEAREST; -using generator::ProjectiveGenerator; - -template -class ImageProjectiveTransform : public OpKernel { - private: - Interpolation interpolation_; - - public: - explicit ImageProjectiveTransform(OpKernelConstruction* ctx) : OpKernel(ctx) { - string interpolation_str; - OP_REQUIRES_OK(ctx, ctx->GetAttr("interpolation", &interpolation_str)); - if (interpolation_str == "NEAREST") { - interpolation_ = INTERPOLATION_NEAREST; - } else if (interpolation_str == "BILINEAR") { - interpolation_ = INTERPOLATION_BILINEAR; - } else { - LOG(FATAL) << "Invalid interpolation " << interpolation_str - << ". Supported types: NEAREST, BILINEAR"; - } - } - - void Compute(OpKernelContext* ctx) override { - const Tensor& images_t = ctx->input(0); - const Tensor& transform_t = ctx->input(1); - OP_REQUIRES(ctx, images_t.shape().dims() == 4, - errors::InvalidArgument("Input images must have rank 4")); - OP_REQUIRES(ctx, - (TensorShapeUtils::IsMatrix(transform_t.shape()) && - (transform_t.dim_size(0) == images_t.dim_size(0) || - transform_t.dim_size(0) == 1) && - transform_t.dim_size(1) == - ProjectiveGenerator::kNumParameters), - errors::InvalidArgument( - "Input transform should be num_images x 8 or 1 x 8")); - - int32 out_height, out_width; - // Kernel is shared by legacy "ImageProjectiveTransform" op with 2 args. - if (ctx->num_inputs() >= 3) { - const Tensor& shape_t = ctx->input(2); - OP_REQUIRES(ctx, shape_t.dims() == 1, - errors::InvalidArgument("output shape must be 1-dimensional", - shape_t.shape().DebugString())); - OP_REQUIRES(ctx, shape_t.NumElements() == 2, - errors::InvalidArgument("output shape must have two elements", - shape_t.shape().DebugString())); - auto shape_vec = shape_t.vec(); - out_height = shape_vec(0); - out_width = shape_vec(1); - OP_REQUIRES( - ctx, out_height > 0 && out_width > 0, - errors::InvalidArgument("output dimensions must be positive")); - } else { - // Shape is N (batch size), H (height), W (width), C (channels). - out_height = images_t.shape().dim_size(1); - out_width = images_t.shape().dim_size(2); - } - - Tensor* output_t; - OP_REQUIRES_OK(ctx, ctx->allocate_output( - 0, - TensorShape({images_t.dim_size(0), out_height, - out_width, images_t.dim_size(3)}), - &output_t)); - auto output = output_t->tensor(); - auto images = images_t.tensor(); - auto transform = transform_t.matrix(); - - (FillProjectiveTransform(interpolation_))( - ctx->eigen_device(), &output, images, transform); - } -}; - -#define REGISTER(TYPE) \ - REGISTER_KERNEL_BUILDER(Name("ImageProjectiveTransform") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("dtype"), \ - ImageProjectiveTransform); \ - REGISTER_KERNEL_BUILDER(Name("ImageProjectiveTransformV2") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("dtype"), \ - ImageProjectiveTransform) - -TF_CALL_uint8(REGISTER); -TF_CALL_int32(REGISTER); -TF_CALL_int64(REGISTER); -TF_CALL_half(REGISTER); -TF_CALL_float(REGISTER); -TF_CALL_double(REGISTER); - -#undef REGISTER - -#if GOOGLE_CUDA - -typedef Eigen::GpuDevice GPUDevice; - -namespace functor { - -// NOTE(ringwalt): We get an undefined symbol error if we don't explicitly -// instantiate the operator() in GCC'd code. -#define DECLARE_FUNCTOR(TYPE) \ - template <> \ - void FillProjectiveTransform::operator()( \ - const GPUDevice& device, OutputType* output, const InputType& images, \ - const TransformsType& transform) const; \ - extern template struct FillProjectiveTransform - -TF_CALL_uint8(DECLARE_FUNCTOR); -TF_CALL_int32(DECLARE_FUNCTOR); -TF_CALL_int64(DECLARE_FUNCTOR); -TF_CALL_half(DECLARE_FUNCTOR); -TF_CALL_float(DECLARE_FUNCTOR); -TF_CALL_double(DECLARE_FUNCTOR); - -} // end namespace functor - -#define REGISTER(TYPE) \ - REGISTER_KERNEL_BUILDER(Name("ImageProjectiveTransform") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("dtype"), \ - ImageProjectiveTransform); \ - REGISTER_KERNEL_BUILDER(Name("ImageProjectiveTransformV2") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("dtype") \ - .HostMemory("output_shape"), \ - ImageProjectiveTransform) - -TF_CALL_uint8(REGISTER); -TF_CALL_int32(REGISTER); -TF_CALL_int64(REGISTER); -TF_CALL_half(REGISTER); -TF_CALL_float(REGISTER); -TF_CALL_double(REGISTER); - -#undef REGISTER - -#endif // GOOGLE_CUDA - -} // end namespace tensorflow diff --git a/tensorflow/contrib/image/kernels/image_ops.h b/tensorflow/contrib/image/kernels/image_ops.h deleted file mode 100644 index 7fac774d07f..00000000000 --- a/tensorflow/contrib/image/kernels/image_ops.h +++ /dev/null @@ -1,172 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_IMAGE_KERNELS_IMAGE_OPS_H_ -#define TENSORFLOW_CONTRIB_IMAGE_KERNELS_IMAGE_OPS_H_ - -// See docs in ../ops/image_ops.cc. - -#define EIGEN_USE_THREADS - -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" - -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { - -namespace generator { - -enum Interpolation { INTERPOLATION_NEAREST, INTERPOLATION_BILINEAR }; - -using Eigen::array; -using Eigen::DenseIndex; - -template -class ProjectiveGenerator { - private: - typename TTypes::ConstTensor input_; - typename TTypes::ConstMatrix transforms_; - const Interpolation interpolation_; - - public: - static const int kNumParameters = 8; - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE - ProjectiveGenerator(typename TTypes::ConstTensor input, - typename TTypes::ConstMatrix transforms, - const Interpolation interpolation) - : input_(input), transforms_(transforms), interpolation_(interpolation) {} - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T - operator()(const array& coords) const { - const int64 output_y = coords[1]; - const int64 output_x = coords[2]; - const float* transform = - transforms_.dimension(0) == 1 - ? transforms_.data() - : &transforms_.data()[transforms_.dimension(1) * coords[0]]; - float projection = transform[6] * output_x + transform[7] * output_y + 1.f; - if (projection == 0) { - // Return the fill value (0) for infinite coordinates, - // which are outside the input image - return T(0); - } - const float input_x = - (transform[0] * output_x + transform[1] * output_y + transform[2]) / - projection; - const float input_y = - (transform[3] * output_x + transform[4] * output_y + transform[5]) / - projection; - - const T fill_value = T(0); - switch (interpolation_) { - case INTERPOLATION_NEAREST: - // Switch the order of x and y again for indexing into the image. - return nearest_interpolation(coords[0], input_y, input_x, coords[3], - fill_value); - case INTERPOLATION_BILINEAR: - return bilinear_interpolation(coords[0], input_y, input_x, coords[3], - fill_value); - } - // Unreachable; ImageProjectiveTransform only uses INTERPOLATION_NEAREST - // or INTERPOLATION_BILINEAR. - return T(0); - } - - private: - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T - nearest_interpolation(const DenseIndex batch, const float y, const float x, - const DenseIndex channel, const T fill_value) const { - return read_with_fill_value(batch, DenseIndex(std::round(y)), - DenseIndex(std::round(x)), channel, fill_value); - } - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T - bilinear_interpolation(const DenseIndex batch, const float y, const float x, - const DenseIndex channel, const T fill_value) const { - const float y_floor = std::floor(y); - const float x_floor = std::floor(x); - const float y_ceil = y_floor + 1; - const float x_ceil = x_floor + 1; - // f(x, y_floor) = (x_ceil - x) / (x_ceil - x_floor) * f(x_floor, y_floor) - // + (x - x_floor) / (x_ceil - x_floor) * f(x_ceil, y_floor) - const float value_yfloor = - (x_ceil - x) * static_cast(read_with_fill_value( - batch, DenseIndex(y_floor), DenseIndex(x_floor), - channel, fill_value)) + - (x - x_floor) * static_cast(read_with_fill_value( - batch, DenseIndex(y_floor), DenseIndex(x_ceil), - channel, fill_value)); - // f(x, y_ceil) = (x_ceil - x) / (x_ceil - x_floor) * f(x_floor, y_ceil) - // + (x - x_floor) / (x_ceil - x_floor) * f(x_ceil, y_ceil) - const float value_yceil = - (x_ceil - x) * static_cast(read_with_fill_value( - batch, DenseIndex(y_ceil), DenseIndex(x_floor), - channel, fill_value)) + - (x - x_floor) * static_cast(read_with_fill_value( - batch, DenseIndex(y_ceil), DenseIndex(x_ceil), - channel, fill_value)); - // f(x, y) = (y_ceil - y) / (y_ceil - y_floor) * f(x, y_floor) - // + (y - y_floor) / (y_ceil - y_floor) * f(x, y_ceil) - return T((y_ceil - y) * value_yfloor + (y - y_floor) * value_yceil); - } - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T read_with_fill_value( - const DenseIndex batch, const DenseIndex y, const DenseIndex x, - const DenseIndex channel, const T fill_value) const { - // batch and channel must be correct, because they are passed unchanged from - // the input. - return (0 <= y && y < input_.dimension(1) && 0 <= x && - x < input_.dimension(2)) - ? input_(array{batch, y, x, channel}) - : fill_value; - } -}; - -} // end namespace generator - -// NOTE(ringwalt): We MUST wrap the generate() call in a functor and explicitly -// instantiate the functor in image_ops_gpu.cu.cc. Otherwise, we will be missing -// some Eigen device code. -namespace functor { - -using generator::Interpolation; -using generator::ProjectiveGenerator; - -template -struct FillProjectiveTransform { - typedef typename TTypes::Tensor OutputType; - typedef typename TTypes::ConstTensor InputType; - typedef typename TTypes::ConstTensor TransformsType; - const Interpolation interpolation_; - - FillProjectiveTransform(Interpolation interpolation) - : interpolation_(interpolation) {} - - EIGEN_ALWAYS_INLINE - void operator()(const Device& device, OutputType* output, - const InputType& images, - const TransformsType& transform) const { - output->device(device) = output->generate( - ProjectiveGenerator(images, transform, interpolation_)); - } -}; - -} // end namespace functor - -} // end namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_IMAGE_KERNELS_IMAGE_OPS_H_ diff --git a/tensorflow/contrib/image/kernels/image_ops_gpu.cu.cc b/tensorflow/contrib/image/kernels/image_ops_gpu.cu.cc deleted file mode 100644 index 36b9a236a6e..00000000000 --- a/tensorflow/contrib/image/kernels/image_ops_gpu.cu.cc +++ /dev/null @@ -1,43 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#if GOOGLE_CUDA - -#define EIGEN_USE_GPU - -#include "tensorflow/contrib/image/kernels/image_ops.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { - -namespace functor { - -// Explicit instantiation of the GPU functor. -typedef Eigen::GpuDevice GPUDevice; - -template class FillProjectiveTransform; -template class FillProjectiveTransform; -template class FillProjectiveTransform; -template class FillProjectiveTransform; -template class FillProjectiveTransform; -template class FillProjectiveTransform; - -} // end namespace functor - -} // end namespace tensorflow - -#endif // GOOGLE_CUDA diff --git a/tensorflow/contrib/image/kernels/segmentation_ops.cc b/tensorflow/contrib/image/kernels/segmentation_ops.cc deleted file mode 100644 index b9d615613cc..00000000000 --- a/tensorflow/contrib/image/kernels/segmentation_ops.cc +++ /dev/null @@ -1,139 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// See docs for ImageConnectedComponents in ../ops/image_ops.cc, and description -// of the algorithm in segmentation_ops.h. - -#define EIGEN_USE_THREADS - -#include "tensorflow/contrib/image/kernels/segmentation_ops.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { - -using tensorflow::functor::BlockedImageUnionFindFunctor; -using tensorflow::functor::FindRootFunctor; -using tensorflow::functor::ImageConnectedComponentsFunctor; -using tensorflow::functor::TensorRangeFunctor; - -using OutputType = typename BlockedImageUnionFindFunctor::OutputType; - -// Computes connected components on batches of 2D images. -template -class ImageConnectedComponents : public OpKernel { - public: - explicit ImageConnectedComponents(OpKernelConstruction* ctx) - : OpKernel(ctx) {} - - void Compute(OpKernelContext* ctx) override { - const Tensor& images_t = ctx->input(0); - OP_REQUIRES(ctx, images_t.shape().dims() == 3, - errors::InvalidArgument("Input images must have rank 3")); - Tensor forest_t, rank_t; - OP_REQUIRES_OK(ctx, ctx->allocate_temp(tensorflow::DT_INT64, - images_t.shape(), &forest_t)); - OP_REQUIRES_OK(ctx, ctx->allocate_temp(tensorflow::DT_INT64, - images_t.shape(), &rank_t)); - Tensor* output_t; - OP_REQUIRES_OK(ctx, ctx->allocate_output(0, images_t.shape(), &output_t)); - - // Fill forest with values from 0 to n - 1, so that each node points to - // itself. - TensorRangeFunctor()(ctx->eigen_device(), - forest_t.flat()); - auto rank = rank_t.tensor(); - rank.device(ctx->eigen_device()) = rank.constant(OutputType(0)); - - const auto images = images_t.tensor(); - auto forest = forest_t.tensor(); - ImageConnectedComponentsFunctor()( - ctx, output_t->flat(), images, forest, rank); - } -}; - -using CPUDevice = Eigen::ThreadPoolDevice; - -namespace functor { - -// Connected components CPU implementation. See `segmentation_ops.h` for a -// description of the algorithm. -template -struct ImageConnectedComponentsFunctor { - void operator()(OpKernelContext* ctx, - typename TTypes::Flat output, - typename TTypes::ConstTensor images, - typename TTypes::Tensor forest, - typename TTypes::Tensor rank) { - const int64 num_images = images.dimension(0), - num_rows = images.dimension(1), num_cols = images.dimension(2), - num_elements = images.size(); - // Bail out early for an empty image--no work to do. - if (num_elements == 0) { - return; - } - auto worker_threads = ctx->device()->tensorflow_cpu_worker_threads(); - BlockedImageUnionFindFunctor union_find( - images.data(), num_rows, num_cols, forest.data(), rank.data()); - while (union_find.can_merge()) { - union_find.merge_blocks(); - int64 num_blocks_vertically = union_find.num_blocks_vertically(); - int64 num_blocks_horizontally = union_find.num_blocks_horizontally(); - // Merging each block calls union_down for each pixel in a row of the - // block, and union_right for each pixel in a column of the block. Assume - // 20 instructions for each call to union_down or union_right. find() may - // loop more while searching for the root, but this should not be very - // significant. - int cost = (union_find.block_height() + union_find.block_width()) * 20; - Shard(worker_threads->num_threads, worker_threads->workers, - num_images * num_blocks_vertically * num_blocks_horizontally, cost, - [&union_find, num_blocks_vertically, num_blocks_horizontally]( - int64 start_block, int64 limit_block) { - for (int64 i = start_block; i < limit_block; i++) { - int64 block_x = i % num_blocks_horizontally; - int64 block_y = - (i / num_blocks_horizontally) % num_blocks_vertically; - int64 image = - i / (num_blocks_horizontally * num_blocks_vertically); - union_find.merge_internal_block_edges(image, block_y, block_x); - } - }); - } - FindRootFunctor()(ctx->eigen_device(), output, - images.data(), union_find); - } -}; - -} // end namespace functor - -#define REGISTER_IMAGE_CONNECTED_COMPONENTS(TYPE) \ - REGISTER_KERNEL_BUILDER(Name("ImageConnectedComponents") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("dtype"), \ - ImageConnectedComponents) -// Connected components (arguably) make sense for number, bool, and string types -TF_CALL_NUMBER_TYPES(REGISTER_IMAGE_CONNECTED_COMPONENTS); -TF_CALL_bool(REGISTER_IMAGE_CONNECTED_COMPONENTS); -TF_CALL_tstring(REGISTER_IMAGE_CONNECTED_COMPONENTS); -#undef REGISTER_IMAGE_CONNECTED_COMPONENTS - -// TODO(ringwalt): Implement on GPU. We probably want to stick to the original -// algorithm by Stava and Benes there for efficiency (computing small blocks in -// shared memory in CUDA thread blocks, instead of starting with single-pixel -// blocks). - -} // end namespace tensorflow diff --git a/tensorflow/contrib/image/kernels/segmentation_ops.h b/tensorflow/contrib/image/kernels/segmentation_ops.h deleted file mode 100644 index 0957d5fd10f..00000000000 --- a/tensorflow/contrib/image/kernels/segmentation_ops.h +++ /dev/null @@ -1,303 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_IMAGE_KERNELS_SEGMENTATION_OPS_H_ -#define TENSORFLOW_CONTRIB_IMAGE_KERNELS_SEGMENTATION_OPS_H_ - -// Connected component analysis. The op is described in ../ops/image_ops.cc. A -// description of the algorithm appears below. - -#define EIGEN_USE_THREADS - -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -namespace functor { - -template -bool is_nonzero(T value) { - return value != T(0); -} - -template <> -bool is_nonzero(string value) { - return value.size() != 0; -} - -// Processes each pixel of an image for union-find, in parallel blocks. This is -// loosely based on the algorithm in "GPU Computing Gems" by Ondrej Stava and -// Bedrich Benes, available here: -// http://hpcg.purdue.edu/bbenes/papers/Stava2011CCL.pdf -// The bulk of the process uses blocks of each image, which have each been -// processed separately. As long as there are multiple blocks in the image, we -// double the height and width of the blocks, creating new blocks which each -// consist of 2x2 previous sub-blocks. On each new block, we process adjacent -// pixels from the previous sub-blocks serially. However, the new blocks are not -// connected, so we can process each block in parallel. -// The GPU algorithm first processes blocks of a fixed size in GPU shared -// memory, with one image block per CUDA thread block. On the CPU, we just start -// with a block size of a single pixel, and borrow the rest of the algorithm -// unchanged. -template -class BlockedImageUnionFindFunctor { - public: - using OutputType = int64; - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE BlockedImageUnionFindFunctor( - const T* images, const int64 num_rows, const int64 num_cols, - OutputType* forest, OutputType* rank) - : images_(images), - num_rows_(num_rows), - num_cols_(num_cols), - block_height_(1), - block_width_(1), - forest_(forest), - rank_(rank) {} - - // Returns the root of the tree that the pixel at the given index belongs to. - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE OutputType - find(OutputType index) const { - while (forest_[index] != index) { - index = forest_[index]; - } - return index; - } - - // Returns the number of blocks along the y axis. - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int64 num_blocks_vertically() const { - return (num_rows_ + block_height_ - 1) / block_height_; - } - - // Returns the number of blocks along the x axis. - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int64 num_blocks_horizontally() const { - return (num_cols_ + block_width_ - 1) / block_width_; - } - - // Returns the total number of blocks in each image. - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int64 num_blocks() const { - return num_blocks_vertically() * num_blocks_horizontally(); - } - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int64 block_height() const { - return block_height_; - } - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int64 block_width() const { - return block_width_; - } - - // Returns whether we may merge again (the image contains more than one - // block). - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool can_merge() const { - return block_height_ < num_rows_ || block_width_ < num_cols_; - } - - // Doubles the block size. After this method, you must call - // `merge_internal_block_edges` for each image and each *new* block's xy - // coordinates (typically in parallel). - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void merge_blocks() { - block_height_ *= 2; - block_width_ *= 2; - } - - // Processes pairs of pixels within the block which were adjacent in the four - // sub-blocks. This must be done at each stage so that the connected - // components in each block are joined correctly. - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void merge_internal_block_edges( - int64 image_index, int64 block_vertical_index, - int64 block_horizontal_index) const { - int64 block_start_y = block_vertical_index * block_height_; - int64 block_start_x = block_horizontal_index * block_width_; - // Merge the 4 sub-blocks horizontally (fixing the vertical seam). - int64 block_center_x = block_start_x + block_width_ / 2 - 1; - if (0 <= block_center_x && block_center_x + 1 < num_cols_) { - int64 merge_blocks_limit_y = - std::min(num_rows_, block_start_y + block_height_); - for (int64 y = block_start_y; y < merge_blocks_limit_y; y++) { - union_right(image_index, y, block_center_x); - } - } - // Merge the 4 sub-blocks vertically (fixing the horizontal seam). - int64 block_center_y = block_start_y + block_height_ / 2 - 1; - if (0 <= block_center_y && block_center_y + 1 < num_rows_) { - int64 merge_blocks_limit_x = - std::min(num_cols_, block_start_x + block_width_); - for (int64 x = block_start_x; x < merge_blocks_limit_x; x++) { - union_down(image_index, block_center_y, x); - } - } - } - - private: - // The input image(s). - const T* const images_; - const int64 num_rows_; - const int64 num_cols_; - // Current height of each sub-block of the image. - int64 block_height_; - // Current width of each sub-block of the image. - int64 block_width_; - // Union-find forest. This has the same size as `images_`, and each entry - // holds the index of its parent in `images_` (roots hold their own index). - // Cycles should not occur. - OutputType* const forest_; - // Union-find rank of each pixel. - OutputType* const rank_; - - // Unions the pixel with the pixel below it if applicable (both pixels are - // true, and the pixel is not in the last row). - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void union_down(OutputType batch, - OutputType row, - OutputType col) const { - T pixel = read_pixel(batch, row, col); - if (is_nonzero(pixel)) { - const int64 index_a = col + num_cols_ * (row + num_rows_ * batch); - if (row + 1 < num_rows_ && read_pixel(batch, row + 1, col) == pixel) { - const int64 index_b = col + num_cols_ * (row + 1 + num_rows_ * batch); - do_union(index_a, index_b); - } - } - } - - // Unions the pixel with the pixel to the right of it if applicable. - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void union_right(OutputType batch, - OutputType row, - OutputType col) const { - T pixel = read_pixel(batch, row, col); - if (is_nonzero(pixel)) { - const int64 index_a = col + num_cols_ * (row + num_rows_ * batch); - if (col + 1 < num_cols_ && read_pixel(batch, row, col + 1) == pixel) { - const int64 index_b = col + 1 + num_cols_ * (row + num_rows_ * batch); - do_union(index_a, index_b); - } - } - } - - // Reads a pixel value in the images. - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T - read_pixel(const OutputType batch, const OutputType row, - const OutputType col) const { - return images_[col + num_cols_ * (row + num_rows_ * batch)]; - } - - // Unions the trees that the two pixels belong to, using their index in the - // `images_` array. - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void do_union( - OutputType index_a, OutputType index_b) const { - // Find the roots of index_a and index_b in the forest, and make one the - // child of the other. - index_a = find(index_a); - index_b = find(index_b); - const OutputType rank_a = rank_[index_a]; - const OutputType rank_b = rank_[index_b]; - OutputType parent, child; - if (index_a == index_b) { - return; - } else if (rank_a < rank_b) { - parent = index_a; - child = index_b; - } else { - parent = index_b; - child = index_a; - rank_[parent]++; - } - forest_[child] = parent; - } -}; - -// Runs the ImageUnionFindFunctor on all pixels. Will require different CPU and -// GPU implementations. -template -class ImageConnectedComponentsFunctor { - public: - using OutputType = typename BlockedImageUnionFindFunctor::OutputType; - - void operator()(OpKernelContext* ctx, - typename TTypes::ConstTensor images, - typename TTypes::Tensor forest, - typename TTypes::Tensor rank); -}; - -// Fills a flat Tensor with indices from 0 to n - 1. -template -class TensorRangeFunctor { - public: - using OutputType = typename BlockedImageUnionFindFunctor::OutputType; - - void operator()(const Device& device, - typename TTypes::Flat tensor) { - tensor.device(device) = tensor.generate(TensorRangeGenerator()); - } - - private: - class TensorRangeGenerator { - public: - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE OutputType - operator()(const Eigen::array& coords) const { - return coords[0]; - } - }; -}; - -// Given the union-find forest, generates the root index for each node. This -// gives us arbitrary, usually non-consecutive ids for each connected component. -// The ids are massaged in Python to get deterministic, consecutive ids. -template -class FindRootFunctor { - public: - using OutputType = typename BlockedImageUnionFindFunctor::OutputType; - - void operator()(const Device& device, - typename TTypes::Flat component_ids, - const T* images, - const BlockedImageUnionFindFunctor& union_find) { - component_ids.device(device) = - component_ids.generate(FindRootGenerator(images, union_find)); - } - - private: - class FindRootGenerator { - const T* const images_; - const BlockedImageUnionFindFunctor union_find_; - - public: - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE FindRootGenerator( - const T* images, BlockedImageUnionFindFunctor union_find) - : images_(images), union_find_(union_find) {} - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE OutputType - operator()(const Eigen::array& coords) const { - if (is_nonzero(images_[coords[0]])) { - // True pixels have an arbitrary segment id > 0. The segment ids will be - // made contiguous later. - return union_find_.find(coords[0]) + 1; - } else { - // False pixels have a segment of 0. - return 0; - } - } - }; -}; - -} // end namespace functor - -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_IMAGE_KERNELS_SEGMENTATION_OPS_H_ diff --git a/tensorflow/contrib/image/kernels/single_image_random_dot_stereograms_ops.cc b/tensorflow/contrib/image/kernels/single_image_random_dot_stereograms_ops.cc deleted file mode 100755 index 8f9a5c28039..00000000000 --- a/tensorflow/contrib/image/kernels/single_image_random_dot_stereograms_ops.cc +++ /dev/null @@ -1,408 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { - -template -class SingleImageRandomDotStereogramsOp : public OpKernel { - private: - int E2Epixels; // Pixels from eye to eye = eye_to_eye_inches * DPI - - int input_Xvalue; // X value of input Z values (width) - int input_Yvalue; // Y value of input Z values (height) - - int output_Ximage; // X value of output image (width) - int output_Yimage; // Y value of output image (height) - int output_Cimage; // color value of output image (color, 1 or 3) (3 not - // implemented) - - int data_box_left; // X starting value for DATA window - int data_box_top; // Y starting value for DATA window - int data_box_width; // width of scan line - int data_box_height; // hight of image - - int converge_dot_box_end; // Row convergences dots end on - - uint8* outputImage; // Output Image flat as a buffer (Tensor Connection) - double* ZBuffer; // For internal use, allow for MASK, etc later, actual Z - // used for Stereogram, XxY (X is the row index, y is col - // index like a screen) - // 0 (far) -> 1.0(near) range - bool hidden_surface_removal; - int convergence_dots_size; - int dots_per_inch; - float eye_separation; - float mu; - bool normalize; - float normalize_max; - float normalize_min; - float border_level; - int number_colors; - ::tensorflow::PartialTensorShape output_image_shape; - ::tensorflow::PartialTensorShape output_data_window; - - uint8 Cblack = 0; - uint8 Cwhite = 255; - - int indexMode = 0; // 0 - truncate XY, 1 - round XY, 2 - Interpolate XY (not - // implemented yet, keep default of 0) - int interp_x, interp_y; // 1 - yes, 0 - no interpolation directions (not - // implemented yet) - - bool debugging = false; - - inline int separation(double z) { - return (std::round((1 - mu * z) * E2Epixels / (2 - mu * z))); - } - - inline int get_far_width() { return (separation(0.0)); } - inline int get_near_width() { return (separation(1.0)); } - - public: - explicit SingleImageRandomDotStereogramsOp(OpKernelConstruction* context) - : OpKernel(context) { // Constructor - OP_REQUIRES_OK(context, context->GetAttr("hidden_surface_removal", - &hidden_surface_removal)); - OP_REQUIRES_OK(context, context->GetAttr("convergence_dots_size", - &convergence_dots_size)); - OP_REQUIRES_OK(context, context->GetAttr("dots_per_inch", &dots_per_inch)); - OP_REQUIRES_OK(context, - context->GetAttr("eye_separation", &eye_separation)); - OP_REQUIRES_OK(context, context->GetAttr("mu", &mu)); - OP_REQUIRES_OK(context, context->GetAttr("normalize", &normalize)); - OP_REQUIRES_OK(context, context->GetAttr("normalize_max", &normalize_max)); - OP_REQUIRES_OK(context, context->GetAttr("normalize_min", &normalize_min)); - OP_REQUIRES_OK(context, context->GetAttr("border_level", &border_level)); - OP_REQUIRES_OK(context, context->GetAttr("number_colors", &number_colors)); - OP_REQUIRES_OK(context, - context->GetAttr("output_image_shape", &output_image_shape)); - OP_REQUIRES_OK(context, - context->GetAttr("output_data_window", &output_data_window)); - - E2Epixels = - eye_separation * dots_per_inch; // Initialize pixels from eye to eye - } - - ~SingleImageRandomDotStereogramsOp() override { // Destructor - } - - void Compute(OpKernelContext* context) override { - const Tensor& input_tensor = context->input(0); - input_Xvalue = input_tensor.shape().dim_size( - 1); // X value is the number of columns of the input matrix - input_Yvalue = - input_tensor.shape().dim_size(0); // Y value is the number of rows - - output_Ximage = output_image_shape.dim_size(0); - output_Yimage = output_image_shape.dim_size(1); - output_Cimage = output_image_shape.dim_size(2); - - if (number_colors > 256) // Go to full color image - output_Cimage = 3; - - int data_Xwindow = output_data_window.dim_size(0); - int data_Ywindow = output_data_window.dim_size(1); - - int deltaX_border_image = output_Ximage - data_Xwindow; - int deltaY_border_image = output_Yimage - data_Ywindow; - - if (convergence_dots_size > - 0) // 3 frame sections in Y direction due to DOTS - { - deltaY_border_image = - deltaY_border_image - - convergence_dots_size; // Take off space for Convergence Dots - deltaY_border_image = std::max(0, deltaY_border_image); - data_box_top = deltaY_border_image / 3; - - if (deltaY_border_image >= 0) { - converge_dot_box_end = output_Yimage - 1 - data_box_top; - } else { - converge_dot_box_end = output_Yimage - 1; - } - } else // Otherwise only 2, no convergence dot - { - data_box_top = deltaY_border_image / 2; // Center DATA in Y dimension - converge_dot_box_end = output_Yimage - 1; - } - - data_box_left = deltaX_border_image / 2; // Center DATA in X dimension - data_box_width = data_Xwindow; // width of scan line - data_box_height = data_Ywindow; // hight of image - - const T* inputZ = input_tensor.flat().data(); // Flatten input Z buffer - - BuildZBuffer(inputZ); - - // Output a scalar string. - Tensor* output_tensor = nullptr; - OP_REQUIRES_OK( - context, - context->allocate_output( - 0, TensorShape({output_Yimage, output_Ximage, output_Cimage}), - &output_tensor)); - - outputImage = output_tensor->flat().data(); - - generate_stereogram(); - - delete[] ZBuffer; - } - - //*************************************************************************** - //*************************************************************************** - // Move input into standard Z format to reduce complexity of algorithm - // - void BuildZBuffer(const T* Z, bool log = false) { - double MaxValue = 1.0; - double MinValue = 0.0; - ZBuffer = new double[input_Xvalue * input_Yvalue]; // Used to computer - // final Z values before - // rendering to output - - if (normalize) { - // Init Min/Max to first value - if (normalize_max < normalize_min) // Autoscale if MIN>MAX - { - MaxValue = *Z; - MinValue = *Z; - - for (int y = 0; y < input_Yvalue; ++y) - for (int x = 0; x < input_Xvalue; ++x) { - double value = getZfromInputImage(Z, x, y); - if (value > MaxValue) MaxValue = value; - if (value < MinValue) MinValue = value; - } - } else { - MaxValue = normalize_max; - MinValue = normalize_min; - } - } - - for (int y = 0; y < input_Yvalue; ++y) - for (int x = 0; x < input_Xvalue; ++x) { - double value = getZfromInputImage(Z, x, y); - - if (normalize) { - value = (value - MinValue) / (MaxValue - MinValue); - } - - if (value > 1.0) value = 1.0; - if (value < 0.0) value = 0.0; - - *(ZBuffer + (input_Xvalue * y + x)) = value; - } - } - - //*************************************************************************** - //*************************************************************************** - double getZfromInputImage(const T* Z, int x, int y) { - return *(Z + input_Xvalue * y + x); - } - - //*************************************************************************** - //*************************************************************************** - // All normalized, not checking required - // Possible Projection issue if DATA is bigger or smaller than Input - // Modes include: - // Truncate value (Default) - // Round-off value - // Interpolate between values - // - double getZfromZbuffer(double x, double y) { - int xi, yi; - - switch (indexMode) { - case 0: // Truncate - xi = int(x); - yi = int(y); - return (*(ZBuffer + (xi + input_Xvalue * yi))); - break; - case 1: // Round-off - xi = std::round(x); - yi = std::round(y); - return (*(ZBuffer + (xi + input_Xvalue * yi))); - break; - case 2: // Interpolate (Not implemented yet, will need 4 points - // [x,y],[x+1,y],[x,y+1],[x+1,y+1], then interpolate) - xi = int(x); - yi = int(y); - return (*(ZBuffer + (xi + input_Xvalue * yi))); - break; - default: // Round-off is the default - xi = int(x + 0.5); - yi = int(y + 0.5); - return (*(ZBuffer + (xi + input_Xvalue * yi))); - break; - } - } - - //*************************************************************************** - //*************************************************************************** - - int getOutputImageIndex(int x, int y, - int channel) { // No error checking for some - // optimization, calling routine - // required to make sure there is no - // violation - return ((output_Ximage * output_Cimage) * y + x * output_Cimage + channel); - } - - //*************************************************************************** - //*************************************************************************** - - double getZFromOutputPixel(int x, int y) { - // Convert pixel units to Z units, do this as "double" - double xofz = static_cast(input_Xvalue) * (x - data_box_left) / - (static_cast(data_box_width)); - double yofz = static_cast(input_Yvalue) * (y - data_box_top) / - (static_cast(data_box_height)); - - if ((xofz < 0) || (yofz < 0) || (yofz >= input_Yvalue) || - (xofz >= input_Xvalue)) { // Top of left side border hit or Right - // side or bottom border hit, - // send BORDER Z value - return border_level; - } else { - return getZfromZbuffer(xofz, yofz); - } - } - - //*************************************************************************** - //*************************************************************************** - - void generate_stereogram() { - int s, left, right, visible, t, l; - double zt, gz; - // Scan line - uint8* pix; // Scan row color for each pixel - int* same; // Used to determine if Pixel needs to be the same as another - // pixel in the row - - pix = new uint8[output_Ximage * output_Cimage]; - same = new int[output_Ximage]; - - for (int y = 0; y < output_Yimage; ++y) { - // Set no dependencies on any pixels, tie each one back to itself - for (int x = 0; x < output_Ximage; ++x) same[x] = x; - - for (int x = 0; x < output_Ximage; ++x) { - gz = getZFromOutputPixel(x, y); - s = separation(gz); - left = x - s / 2; - right = left + s; - - if ((left >= 0) && (right < output_Ximage)) { - t = 1; - visible = 1; - if (hidden_surface_removal) do { - zt = gz + 2 * (2 - mu * gz) * t / (mu * E2Epixels); - visible = (getZFromOutputPixel(x - t, y) < zt) && - (getZFromOutputPixel(x + t, y) < zt); - ++t; - } while ((visible) && (zt < 1)); - - if (visible) { - l = same[left]; - while ((l != left) && (l != right)) - if (l < right) { - left = l; - l = same[left]; - } else { - same[left] = right; - left = right; - l = same[left]; - right = l; - } - same[left] = right; - } - } - } - // Set colors for scan row, use channels and number_colors - for (int x = output_Ximage - 1; x >= 0; x--) { - for (int channel = 0; channel < output_Cimage; ++channel) { - if (same[x] == x) { // Pick a random color - if (number_colors == 2) { - if ((rand() % 2) == 0) { - pix[x * output_Cimage + channel] = Cblack; - } else { - pix[x * output_Cimage + channel] = Cwhite; - } - } else { - pix[x * output_Cimage + channel] = rand() % 256; - } - } else - pix[x * output_Cimage + channel] = - pix[same[x] * output_Cimage + channel]; - - setpixel(x, y, channel, pix[x * output_Cimage + channel]); - } - } - } - - draw_convergence_dots(); - - delete[] pix; - delete[] same; - } - - //*************************************************************************** - //*************************************************************************** - - void draw_convergence_dots() { - int x1, x2; // center position for convergence dots - - if (convergence_dots_size == 0) // No dot, return - return; - - x1 = output_Ximage / 2 - get_far_width() / 2; - x2 = output_Ximage / 2 + get_far_width() / 2; - - for (int lloop = 0; lloop < convergence_dots_size; ++lloop) - for (int wloop = 0; wloop < convergence_dots_size; ++wloop) - for (int channel = 0; channel < output_Cimage; ++channel) { - setpixel(x1 - (convergence_dots_size / 2) + wloop, - converge_dot_box_end - lloop, channel, Cblack); - setpixel(x2 - (convergence_dots_size / 2) + wloop, - converge_dot_box_end - lloop, channel, Cblack); - } - } - - //*************************************************************************** - //*************************************************************************** - - void setpixel(int x, int y, int channel, uint8 color) { - *(outputImage + getOutputImageIndex(x, y, channel)) = color; - } -}; - -#define REGISTER_KERNEL(T) \ - REGISTER_KERNEL_BUILDER(Name("SingleImageRandomDotStereograms") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T"), \ - SingleImageRandomDotStereogramsOp); - -REGISTER_KERNEL(int32); -REGISTER_KERNEL(int64); -REGISTER_KERNEL(float); -REGISTER_KERNEL(double); - -#undef REGISTER_KERNEL - -} // end namespace tensorflow diff --git a/tensorflow/contrib/image/ops/distort_image_ops.cc b/tensorflow/contrib/image/ops/distort_image_ops.cc deleted file mode 100644 index ca49635d5d0..00000000000 --- a/tensorflow/contrib/image/ops/distort_image_ops.cc +++ /dev/null @@ -1,60 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { - -using shape_inference::InferenceContext; - -// -------------------------------------------------------------------------- -REGISTER_OP("AdjustHsvInYiq") - .Input("images: T") - .Input("delta_h: float") - .Input("scale_s: float") - .Input("scale_v: float") - .Output("output: T") - .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}") - .SetShapeFn([](InferenceContext* c) { - return shape_inference::UnchangedShapeWithRankAtLeast(c, 3); - }) - .Doc(R"Doc( -Adjust the YIQ hue of one or more images. - -`images` is a tensor of at least 3 dimensions. The last dimension is -interpreted as channels, and must be three. - -We used linear transformation described in: - beesbuzz.biz/code/hsv_color_transforms.php -The input image is considered in the RGB colorspace. Conceptually, the RGB -colors are first mapped into YIQ space, rotated around the Y channel by -delta_h in radians, multiplying the chrominance channels (I, Q) by scale_s, -multiplying all channels (Y, I, Q) by scale_v, and then remapped back to RGB -colorspace. Each operation described above is a linear transformation. - -images: Images to adjust. At least 3-D. -delta_h: A float scale that represents the hue rotation amount, in radians. - Although delta_h can be any float value. -scale_s: A float scale that represents the factor to multiply the saturation by. - scale_s needs to be non-negative. -scale_v: A float scale that represents the factor to multiply the value by. - scale_v needs to be non-negative. -output: The hsv-adjusted image or images. No clipping will be done in this op. - The client can clip them using additional ops in their graph. -)Doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/image/ops/image_ops.cc b/tensorflow/contrib/image/ops/image_ops.cc deleted file mode 100644 index 6f7c9bb5204..00000000000 --- a/tensorflow/contrib/image/ops/image_ops.cc +++ /dev/null @@ -1,190 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { - -using shape_inference::DimensionHandle; -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -namespace { - -// Sets output[0] to shape [batch_dim,height,width,channel_dim], where -// height and width come from the size_tensor. -Status SetOutputToSizedImage(InferenceContext* c, DimensionHandle batch_dim, - int size_input_idx, DimensionHandle channel_dim) { - // Verify shape of size input. - ShapeHandle size; - TF_RETURN_IF_ERROR(c->WithRank(c->input(size_input_idx), 1, &size)); - DimensionHandle unused; - TF_RETURN_IF_ERROR(c->WithValue(c->Dim(size, 0), 2, &unused)); - - // Get size values from the size tensor. - const Tensor* size_tensor = c->input_tensor(size_input_idx); - DimensionHandle width; - DimensionHandle height; - if (size_tensor == nullptr) { - width = c->UnknownDim(); - height = c->UnknownDim(); - } else { - // TODO(petewarden) - Remove once we have constant evaluation in C++ only. - if (size_tensor->dtype() != DT_INT32) { - return errors::InvalidArgument( - "Bad size input type for SetOutputToSizedImage: Expected DT_INT32 " - "but got ", - DataTypeString(size_tensor->dtype()), " for input #", size_input_idx, - " in ", c->DebugString()); - } - auto vec = size_tensor->vec(); - height = c->MakeDim(vec(0)); - width = c->MakeDim(vec(1)); - } - c->set_output(0, c->MakeShape({batch_dim, height, width, channel_dim})); - return Status::OK(); -} - -// TODO(qyu): Move this to core/framework/common_shape_fns.h -Status ResizeShapeFn(InferenceContext* c) { - ShapeHandle input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 4, &input)); - return SetOutputToSizedImage(c, c->Dim(input, 0), 2 /* size_input_idx */, - c->Dim(input, 3)); -} - -static const char kImageProjectiveTransformDoc[] = R"doc( -Applies the given transform to each of the images. - -Input `image` is a `Tensor` in NHWC format (where the axes are image in batch, -rows, columns, and channels. Input `transforms` is a num_images x 8 or 1 x 8 -matrix, where each row corresponds to a 3 x 3 projective transformation matrix, -with the last entry assumed to be 1. If there is one row, the same -transformation will be applied to all images. - -If one row of `transforms` is `[a0, a1, a2, b0, b1, b2, c0, c1]`, then it maps -the *output* point `(x, y)` to a transformed *input* point -`(x', y') = ((a0 x + a1 y + a2) / k, (b0 x + b1 y + b2) / k)`, where -`k = c0 x + c1 y + 1`. If the transformed point lays outside of the input -image, the output pixel is set to 0. - -images: 4D `Tensor`, input image(s) in NHWC format. -transforms: 2D `Tensor`, projective transform(s) to apply to the image(s). - -transformed_images: 4D `Tensor`, image(s) in NHWC format, generated by applying -the `transforms` to the `images`. Satisfies the description above. -)doc"; - -} // namespace - -// TODO(ringwalt): Add a "fill_mode" attr with "constant", "mirror", etc. -// TODO(ringwalt): Add a "fill_constant" argument for constant mode (default 0). -REGISTER_OP("ImageProjectiveTransform") - .Input("images: dtype") - .Input("transforms: float32") - .Attr("dtype: {uint8, int32, int64, float16, float32, float64}") - .Attr("interpolation: string") - .Output("transformed_images: dtype") - // Output shape is identical to input images. - .SetShapeFn([](InferenceContext* c) { - c->set_output(0, c->input(0)); - return Status::OK(); - }) - .Doc(kImageProjectiveTransformDoc); - -// V2 op supports output_shape. -REGISTER_OP("ImageProjectiveTransformV2") - .Input("images: dtype") - .Input("transforms: float32") - .Input("output_shape: int32") - .Attr("dtype: {uint8, int32, int64, float16, float32, float64}") - .Attr("interpolation: string") - .Output("transformed_images: dtype") - .SetShapeFn(ResizeShapeFn) - .Doc(kImageProjectiveTransformDoc); - -REGISTER_OP("BipartiteMatch") - .Input("distance_mat: float") - .Input("num_valid_rows: float") - .Attr("top_k: int = -1") - .Output("row_to_col_match_indices: int32") - .Output("col_to_row_match_indices: int32") - .SetIsStateful() - .SetShapeFn([](InferenceContext* c) { - ShapeHandle input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 2, &input)); - c->set_output(0, c->MakeShape({c->Dim(input, 0)})); - c->set_output(1, c->MakeShape({c->Dim(input, 1)})); - return Status::OK(); - }) - .Doc(R"doc( -Find bipartite matching based on a given distance matrix. - -A greedy bi-partite matching algorithm is used to obtain the matching with the -(greedy) minimum distance. - -distance_mat: A 2-D float tensor of shape `[num_rows, num_columns]`. It is a - pair-wise distance matrix between the entities represented by each row and - each column. It is an asymmetric matrix. The smaller the distance is, the more - similar the pairs are. The bipartite matching is to minimize the distances. -num_valid_rows: A scalar or a 1-D tensor with one element describing the - number of valid rows of distance_mat to consider for the bipartite matching. - If set to be negative, then all rows from `distance_mat` are used. -top_k: A scalar that specifies the number of top-k matches to retrieve. - If set to be negative, then is set according to the maximum number of - matches from `distance_mat`. -row_to_col_match_indices: A vector of length num_rows, which is the number of - rows of the input `distance_matrix`. - If `row_to_col_match_indices[i]` is not -1, row i is matched to column - `row_to_col_match_indices[i]`. -col_to_row_match_indices: A vector of length num_columns, which is the number - of columns of the input distance matrix. - If `col_to_row_match_indices[j]` is not -1, column j is matched to row - `col_to_row_match_indices[j]`. -)doc"); - -REGISTER_OP("ImageConnectedComponents") - .Input("image: dtype") - .Output("components: int64") - .Attr( - "dtype: {int64, int32, uint16, int16, uint8, int8, half, float, " - "double, bool, string}") - .SetShapeFn([](InferenceContext* c) { - return shape_inference::UnchangedShape(c); - }) - .Doc(R"doc( -Find the connected components of image(s). - -For each image (along the 0th axis), all connected components of adjacent pixels -with the same non-zero value are detected and given unique ids. - -The returned `components` tensor has 0s for the zero pixels of `images`, and -arbitrary nonzero ids for the connected components of nonzero values. Ids are -unique across all of the images, and are in row-major order by the first pixel -in the component. - -Uses union-find with union by rank but not path compression, giving a runtime of -`O(n log n)`. See: - https://en.wikipedia.org/wiki/Disjoint-set_data_structure#Time_Complexity - -image: Image(s) with shape (N, H, W). -components: Component ids for each pixel in "image". Same shape as "image". Zero - pixels all have an output of 0, and all components of adjacent pixels with - the same value are given consecutive ids, starting from 1. -)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/image/ops/single_image_random_dot_stereograms_ops.cc b/tensorflow/contrib/image/ops/single_image_random_dot_stereograms_ops.cc deleted file mode 100755 index bd784c6bda0..00000000000 --- a/tensorflow/contrib/image/ops/single_image_random_dot_stereograms_ops.cc +++ /dev/null @@ -1,117 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { - -using shape_inference::DimensionHandle; -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -REGISTER_OP("SingleImageRandomDotStereograms") - .Attr("T: {double,float,int64,int32}") - .Input("depth_values: T") - .Output("image: uint8") - .Attr("hidden_surface_removal: bool = true") - .Attr("convergence_dots_size: int = 8") - .Attr("dots_per_inch: int = 72") - .Attr("eye_separation: float = 2.5") - .Attr("mu: float = .3333") - .Attr("normalize: bool = true") - .Attr("normalize_max: float = -100.0") - .Attr("normalize_min: float = 100.0") - .Attr("border_level: float = 0.0") - .Attr("number_colors: int = 256") - .Attr( - "output_image_shape: shape = { dim {size:1024} dim {size: 768} dim " - "{size: 1}}") - .Attr("output_data_window: shape = { dim {size:1022} dim {size: 757}}") - .SetShapeFn([](InferenceContext* c) { - // Validate that the output_image_shape attr is correct. - // NOTE: The output_image_shape is [X, Y, C] - // while the output data is [Y, X, C] (or [H, W, C]). - // As a result, by default the output_image_shape has the value - // of [1024, 768, 1] but the output data will be [768, 1024, 1]. - PartialTensorShape shape; - TF_RETURN_IF_ERROR(c->GetAttr("output_image_shape", &shape)); - ShapeHandle output_image_shape; - TF_RETURN_IF_ERROR( - c->MakeShapeFromPartialTensorShape(shape, &output_image_shape)); - DimensionHandle x_dim = c->Dim(output_image_shape, 0); - DimensionHandle y_dim = c->Dim(output_image_shape, 1); - - int colors; - TF_RETURN_IF_ERROR(c->GetAttr("number_colors", &colors)); - - c->set_output( - 0, c->MakeShape( - {y_dim, x_dim, colors > 256 ? c->MakeDim(3) : c->MakeDim(1)})); - return Status::OK(); - }) - .Doc(R"doc( -Outputs a single image random dot stereogram for export via encode_PNG/JPG OP. - -Given the 2-D tensor 'depth_values' with encoded Z values, this operation will -encode 3-D data into a 2-D image. The output of this Op is suitable for the -encode_PNG/JPG ops. Be careful with image compression as this may corrupt the -encode 3-D data within the image. - -This Op is based upon: -'http://www.learningace.com/doc/4331582/b6ab058d1e206d68ab60e4e1ead2fe6e/sirds-paper' - -Example use which outputs a SIRDS image as picture_out.png: -```python -img=[[1,2,3,3,2,1], - [1,2,3,4,5,2], - [1,2,3,4,5,3], - [1,2,3,4,5,4], - [6,5,4,4,5,5]] - -session = tf.InteractiveSession() - -sirds = single_image_random_dot_stereograms(img,convergence_dots_size=8,number_colors=256,normalize=True) - -out = sirds.eval() - -png = tf.image.encode_png(out).eval() - -with open('picture_out.png', 'wb') as f: - f.write(png) -``` - -depth_values: Z values of data to encode into 'output_data_window' window, - lower values are further away {0.0 floor(far), 1.0 ceiling(near) after normalization}, must be 2-D tensor -hidden_surface_removal: Activate hidden surface removal -convergence_dots_size: Black dot size in pixels to help view converge image, drawn on bottom of image -dots_per_inch: Output device in dots/inch -eye_separation: Separation between eyes in inches -mu: Depth of field, Fraction of viewing distance (eg. 1/3 = .3333) -normalize: Normalize input data to [0.0, 1.0] -normalize_max: Fix MAX value for Normalization - if < MIN, autoscale -normalize_min: Fix MIN value for Normalization - if > MAX, autoscale -border_level: Value of border depth 0.0 {far} to 1.0 {near} -number_colors: 2 (Black & White),256 (grayscale), and Numbers > 256 (Full Color) are all that are supported currently -output_image_shape: Output size of returned image in X,Y, Channels 1-grayscale, 3 color (1024, 768, 1), - channels will be updated to 3 if 'number_colors' > 256 -output_data_window: Size of "DATA" window, must be equal to or smaller than 'output_image_shape', will be centered - and use 'convergence_dots_size' for best fit to avoid overlap if possible - -image:= A tensor of size 'output_image_shape' with the encoded 'depth_values' -)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py b/tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py deleted file mode 100644 index ae9c7a61194..00000000000 --- a/tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py +++ /dev/null @@ -1,267 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for dense_image_warp.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import numpy as np - -from tensorflow.contrib.image.python.ops import dense_image_warp - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest - -from tensorflow.python.training import adam - - -class DenseImageWarpTest(test_util.TensorFlowTestCase): - - def setUp(self): - np.random.seed(0) - - def test_interpolate_small_grid_ij(self): - grid = constant_op.constant( - [[0., 1., 2.], [3., 4., 5.], [6., 7., 8.]], shape=[1, 3, 3, 1]) - query_points = constant_op.constant( - [[0., 0.], [1., 0.], [2., 0.5], [1.5, 1.5]], shape=[1, 4, 2]) - expected_results = np.reshape(np.array([0., 3., 6.5, 6.]), [1, 4, 1]) - - interp = dense_image_warp._interpolate_bilinear(grid, query_points) - - with self.cached_session() as sess: - predicted = sess.run(interp) - self.assertAllClose(expected_results, predicted) - - def test_interpolate_small_grid_xy(self): - grid = constant_op.constant( - [[0., 1., 2.], [3., 4., 5.], [6., 7., 8.]], shape=[1, 3, 3, 1]) - query_points = constant_op.constant( - [[0., 0.], [0., 1.], [0.5, 2.0], [1.5, 1.5]], shape=[1, 4, 2]) - expected_results = np.reshape(np.array([0., 3., 6.5, 6.]), [1, 4, 1]) - - interp = dense_image_warp._interpolate_bilinear( - grid, query_points, indexing='xy') - - with self.cached_session() as sess: - predicted = sess.run(interp) - self.assertAllClose(expected_results, predicted) - - def test_interpolate_small_grid_batched(self): - grid = constant_op.constant( - [[[0., 1.], [3., 4.]], [[5., 6.], [7., 8.]]], shape=[2, 2, 2, 1]) - query_points = constant_op.constant([[[0., 0.], [1., 0.], [0.5, 0.5]], - [[0.5, 0.], [1., 0.], [1., 1.]]]) - expected_results = np.reshape( - np.array([[0., 3., 2.], [6., 7., 8.]]), [2, 3, 1]) - - interp = dense_image_warp._interpolate_bilinear(grid, query_points) - - with self.cached_session() as sess: - predicted = sess.run(interp) - self.assertAllClose(expected_results, predicted) - - def get_image_and_flow_placeholders(self, shape, image_type, flow_type): - batch_size, height, width, numchannels = shape - image_shape = [batch_size, height, width, numchannels] - flow_shape = [batch_size, height, width, 2] - - tf_type = { - 'float16': dtypes.half, - 'float32': dtypes.float32, - 'float64': dtypes.float64 - } - - image = array_ops.placeholder(dtype=tf_type[image_type], shape=image_shape) - - flows = array_ops.placeholder(dtype=tf_type[flow_type], shape=flow_shape) - return image, flows - - def get_random_image_and_flows(self, shape, image_type, flow_type): - batch_size, height, width, numchannels = shape - image_shape = [batch_size, height, width, numchannels] - image = np.random.normal(size=image_shape) - flow_shape = [batch_size, height, width, 2] - flows = np.random.normal(size=flow_shape) * 3 - return image.astype(image_type), flows.astype(flow_type) - - def assert_correct_interpolation_value(self, - image, - flows, - pred_interpolation, - batch_index, - y_index, - x_index, - low_precision=False): - """Assert that the tf interpolation matches hand-computed value.""" - - height = image.shape[1] - width = image.shape[2] - displacement = flows[batch_index, y_index, x_index, :] - float_y = y_index - displacement[0] - float_x = x_index - displacement[1] - floor_y = max(min(height - 2, math.floor(float_y)), 0) - floor_x = max(min(width - 2, math.floor(float_x)), 0) - ceil_y = floor_y + 1 - ceil_x = floor_x + 1 - - alpha_y = min(max(0.0, float_y - floor_y), 1.0) - alpha_x = min(max(0.0, float_x - floor_x), 1.0) - - floor_y = int(floor_y) - floor_x = int(floor_x) - ceil_y = int(ceil_y) - ceil_x = int(ceil_x) - - top_left = image[batch_index, floor_y, floor_x, :] - top_right = image[batch_index, floor_y, ceil_x, :] - bottom_left = image[batch_index, ceil_y, floor_x, :] - bottom_right = image[batch_index, ceil_y, ceil_x, :] - - interp_top = alpha_x * (top_right - top_left) + top_left - interp_bottom = alpha_x * (bottom_right - bottom_left) + bottom_left - interp = alpha_y * (interp_bottom - interp_top) + interp_top - atol = 1e-6 - rtol = 1e-6 - if low_precision: - atol = 1e-2 - rtol = 1e-3 - self.assertAllClose( - interp, - pred_interpolation[batch_index, y_index, x_index, :], - atol=atol, - rtol=rtol) - - def check_zero_flow_correctness(self, shape, image_type, flow_type): - """Assert using zero flows doesn't change the input image.""" - - image, flows = self.get_image_and_flow_placeholders(shape, image_type, - flow_type) - interp = dense_image_warp.dense_image_warp(image, flows) - - with self.cached_session() as sess: - rand_image, rand_flows = self.get_random_image_and_flows( - shape, image_type, flow_type) - rand_flows *= 0 - - predicted_interpolation = sess.run( - interp, feed_dict={ - image: rand_image, - flows: rand_flows - }) - self.assertAllClose(rand_image, predicted_interpolation) - - def test_zero_flows(self): - """Apply check_zero_flow_correctness() for a few sizes and types.""" - - shapes_to_try = [[3, 4, 5, 6], [1, 2, 2, 1]] - for shape in shapes_to_try: - self.check_zero_flow_correctness( - shape, image_type='float32', flow_type='float32') - - def check_interpolation_correctness(self, - shape, - image_type, - flow_type, - num_probes=5): - """Interpolate, and then assert correctness for a few query locations.""" - - image, flows = self.get_image_and_flow_placeholders(shape, image_type, - flow_type) - interp = dense_image_warp.dense_image_warp(image, flows) - low_precision = image_type == 'float16' or flow_type == 'float16' - with self.cached_session() as sess: - rand_image, rand_flows = self.get_random_image_and_flows( - shape, image_type, flow_type) - - pred_interpolation = sess.run( - interp, feed_dict={ - image: rand_image, - flows: rand_flows - }) - - for _ in range(num_probes): - batch_index = np.random.randint(0, shape[0]) - y_index = np.random.randint(0, shape[1]) - x_index = np.random.randint(0, shape[2]) - - self.assert_correct_interpolation_value( - rand_image, - rand_flows, - pred_interpolation, - batch_index, - y_index, - x_index, - low_precision=low_precision) - - def test_interpolation(self): - """Apply check_interpolation_correctness() for a few sizes and types.""" - - shapes_to_try = [[3, 4, 5, 6], [1, 5, 5, 3], [1, 2, 2, 1]] - for im_type in ['float32', 'float64', 'float16']: - for flow_type in ['float32', 'float64', 'float16']: - for shape in shapes_to_try: - self.check_interpolation_correctness(shape, im_type, flow_type) - - def test_gradients_exist(self): - """Check that backprop can run. - - The correctness of the gradients is assumed, since the forward propagation - is tested to be correct and we only use built-in tf ops. - However, we perform a simple test to make sure that backprop can actually - run. We treat the flows as a tf.Variable and optimize them to minimize - the difference between the interpolated image and the input image. - """ - - batch_size, height, width, numchannels = [4, 5, 6, 7] - image_shape = [batch_size, height, width, numchannels] - image = random_ops.random_normal(image_shape) - flow_shape = [batch_size, height, width, 2] - init_flows = np.float32(np.random.normal(size=flow_shape) * 0.25) - flows = variables.Variable(init_flows) - - interp = dense_image_warp.dense_image_warp(image, flows) - loss = math_ops.reduce_mean(math_ops.square(interp - image)) - - optimizer = adam.AdamOptimizer(1.0) - grad = gradients.gradients(loss, [flows]) - opt_func = optimizer.apply_gradients(zip(grad, [flows])) - init_op = variables.global_variables_initializer() - - with self.cached_session() as sess: - sess.run(init_op) - for _ in range(10): - sess.run(opt_func) - - def test_size_exception(self): - """Make sure it throws an exception for images that are too small.""" - - shape = [1, 2, 1, 1] - msg = 'Should have raised an exception for invalid image size' - with self.assertRaises(errors.InvalidArgumentError, msg=msg): - self.check_interpolation_correctness(shape, 'float32', 'float32') - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/image/python/kernel_tests/distort_image_ops_test.py b/tensorflow/contrib/image/python/kernel_tests/distort_image_ops_test.py deleted file mode 100644 index ac8573445ca..00000000000 --- a/tensorflow/contrib/image/python/kernel_tests/distort_image_ops_test.py +++ /dev/null @@ -1,344 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for python distort_image_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time - -import numpy as np -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib.image.python.ops import distort_image_ops -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest -from tensorflow.python.platform import test - - -# TODO(huangyp): also measure the differences between AdjustHsvInYiq and -# AdjustHsv in core. -class AdjustHueInYiqTest(test_util.TensorFlowTestCase): - - def _adjust_hue_in_yiq_np(self, x_np, delta_h): - """Rotate hue in YIQ space. - - Mathematically we first convert rgb color to yiq space, rotate the hue - degrees, and then convert back to rgb. - - Args: - x_np: input x with last dimension = 3. - delta_h: degree of hue rotation, in radians. - - Returns: - Adjusted y with the same shape as x_np. - """ - self.assertEqual(x_np.shape[-1], 3) - x_v = x_np.reshape([-1, 3]) - y_v = np.ndarray(x_v.shape, dtype=x_v.dtype) - u = np.cos(delta_h) - w = np.sin(delta_h) - # Projection matrix from RGB to YIQ. Numbers from wikipedia - # https://en.wikipedia.org/wiki/YIQ - tyiq = np.array([[0.299, 0.587, 0.114], [0.596, -0.274, -0.322], - [0.211, -0.523, 0.312]]) - y_v = np.dot(x_v, tyiq.T) - # Hue rotation matrix in YIQ space. - hue_rotation = np.array([[1.0, 0.0, 0.0], [0.0, u, -w], [0.0, w, u]]) - y_v = np.dot(y_v, hue_rotation.T) - # Projecting back to RGB space. - y_v = np.dot(y_v, np.linalg.inv(tyiq).T) - return y_v.reshape(x_np.shape) - - def _adjust_hue_in_yiq_tf(self, x_np, delta_h): - with self.test_session(use_gpu=True): - x = constant_op.constant(x_np) - y = distort_image_ops.adjust_hsv_in_yiq(x, delta_h, 1, 1) - y_tf = y.eval() - return y_tf - - def test_adjust_random_hue_in_yiq(self): - x_shapes = [ - [2, 2, 3], - [4, 2, 3], - [2, 4, 3], - [2, 5, 3], - [1000, 1, 3], - ] - test_styles = [ - 'all_random', - 'rg_same', - 'rb_same', - 'gb_same', - 'rgb_same', - ] - for x_shape in x_shapes: - for test_style in test_styles: - x_np = np.random.rand(*x_shape) * 255. - delta_h = (np.random.rand() * 2.0 - 1.0) * np.pi - if test_style == 'all_random': - pass - elif test_style == 'rg_same': - x_np[..., 1] = x_np[..., 0] - elif test_style == 'rb_same': - x_np[..., 2] = x_np[..., 0] - elif test_style == 'gb_same': - x_np[..., 2] = x_np[..., 1] - elif test_style == 'rgb_same': - x_np[..., 1] = x_np[..., 0] - x_np[..., 2] = x_np[..., 0] - else: - raise AssertionError('Invalid test style: %s' % (test_style)) - y_np = self._adjust_hue_in_yiq_np(x_np, delta_h) - y_tf = self._adjust_hue_in_yiq_tf(x_np, delta_h) - self.assertAllClose(y_tf, y_np, rtol=2e-4, atol=1e-4) - - def test_invalid_shapes(self): - x_np = np.random.rand(2, 3) * 255. - delta_h = np.random.rand() * 2.0 - 1.0 - with self.assertRaisesRegexp(ValueError, 'Shape must be at least rank 3'): - self._adjust_hue_in_yiq_tf(x_np, delta_h) - x_np = np.random.rand(4, 2, 4) * 255. - delta_h = np.random.rand() * 2.0 - 1.0 - with self.assertRaisesOpError('input must have 3 channels but instead has ' - '4 channels'): - self._adjust_hue_in_yiq_tf(x_np, delta_h) - - -class AdjustValueInYiqTest(test_util.TensorFlowTestCase): - - def _adjust_value_in_yiq_np(self, x_np, scale): - return x_np * scale - - def _adjust_value_in_yiq_tf(self, x_np, scale): - with self.test_session(use_gpu=True): - x = constant_op.constant(x_np) - y = distort_image_ops.adjust_hsv_in_yiq(x, 0, 1, scale) - y_tf = y.eval() - return y_tf - - def test_adjust_random_value_in_yiq(self): - x_shapes = [ - [2, 2, 3], - [4, 2, 3], - [2, 4, 3], - [2, 5, 3], - [1000, 1, 3], - ] - test_styles = [ - 'all_random', - 'rg_same', - 'rb_same', - 'gb_same', - 'rgb_same', - ] - for x_shape in x_shapes: - for test_style in test_styles: - x_np = np.random.rand(*x_shape) * 255. - scale = np.random.rand() * 2.0 - 1.0 - if test_style == 'all_random': - pass - elif test_style == 'rg_same': - x_np[..., 1] = x_np[..., 0] - elif test_style == 'rb_same': - x_np[..., 2] = x_np[..., 0] - elif test_style == 'gb_same': - x_np[..., 2] = x_np[..., 1] - elif test_style == 'rgb_same': - x_np[..., 1] = x_np[..., 0] - x_np[..., 2] = x_np[..., 0] - else: - raise AssertionError('Invalid test style: %s' % (test_style)) - y_np = self._adjust_value_in_yiq_np(x_np, scale) - y_tf = self._adjust_value_in_yiq_tf(x_np, scale) - self.assertAllClose(y_tf, y_np, rtol=2e-4, atol=1e-4) - - def test_invalid_shapes(self): - x_np = np.random.rand(2, 3) * 255. - scale = np.random.rand() * 2.0 - 1.0 - with self.assertRaisesRegexp(ValueError, 'Shape must be at least rank 3'): - self._adjust_value_in_yiq_tf(x_np, scale) - x_np = np.random.rand(4, 2, 4) * 255. - scale = np.random.rand() * 2.0 - 1.0 - with self.assertRaisesOpError('input must have 3 channels but instead has ' - '4 channels'): - self._adjust_value_in_yiq_tf(x_np, scale) - - -class AdjustSaturationInYiqTest(test_util.TensorFlowTestCase): - - def _adjust_saturation_in_yiq_tf(self, x_np, scale): - with self.test_session(use_gpu=True): - x = constant_op.constant(x_np) - y = distort_image_ops.adjust_hsv_in_yiq(x, 0, scale, 1) - y_tf = y.eval() - return y_tf - - def _adjust_saturation_in_yiq_np(self, x_np, scale): - """Adjust saturation using linear interpolation.""" - rgb_weights = np.array([0.299, 0.587, 0.114]) - gray = np.sum(x_np * rgb_weights, axis=-1, keepdims=True) - y_v = x_np * scale + gray * (1 - scale) - return y_v - - def test_adjust_random_saturation_in_yiq(self): - x_shapes = [ - [2, 2, 3], - [4, 2, 3], - [2, 4, 3], - [2, 5, 3], - [1000, 1, 3], - ] - test_styles = [ - 'all_random', - 'rg_same', - 'rb_same', - 'gb_same', - 'rgb_same', - ] - with self.cached_session(): - for x_shape in x_shapes: - for test_style in test_styles: - x_np = np.random.rand(*x_shape) * 255. - scale = np.random.rand() * 2.0 - 1.0 - if test_style == 'all_random': - pass - elif test_style == 'rg_same': - x_np[..., 1] = x_np[..., 0] - elif test_style == 'rb_same': - x_np[..., 2] = x_np[..., 0] - elif test_style == 'gb_same': - x_np[..., 2] = x_np[..., 1] - elif test_style == 'rgb_same': - x_np[..., 1] = x_np[..., 0] - x_np[..., 2] = x_np[..., 0] - else: - raise AssertionError('Invalid test style: %s' % (test_style)) - y_baseline = self._adjust_saturation_in_yiq_np(x_np, scale) - y_tf = self._adjust_saturation_in_yiq_tf(x_np, scale) - self.assertAllClose(y_tf, y_baseline, rtol=2e-4, atol=1e-4) - - def test_invalid_shapes(self): - x_np = np.random.rand(2, 3) * 255. - scale = np.random.rand() * 2.0 - 1.0 - with self.assertRaisesRegexp(ValueError, 'Shape must be at least rank 3'): - self._adjust_saturation_in_yiq_tf(x_np, scale) - x_np = np.random.rand(4, 2, 4) * 255. - scale = np.random.rand() * 2.0 - 1.0 - with self.assertRaisesOpError('input must have 3 channels but instead has ' - '4 channels'): - self._adjust_saturation_in_yiq_tf(x_np, scale) - - -class AdjustHueInYiqBenchmark(test.Benchmark): - - def _benchmark_adjust_hue_in_yiq(self, device, cpu_count): - image_shape = [299, 299, 3] - warmup_rounds = 100 - benchmark_rounds = 1000 - config = config_pb2.ConfigProto() - if cpu_count is not None: - config.inter_op_parallelism_threads = 1 - config.intra_op_parallelism_threads = cpu_count - with session.Session('', graph=ops.Graph(), config=config) as sess: - with ops.device(device): - inputs = variables.Variable( - random_ops.random_uniform(image_shape, dtype=dtypes.float32) * 255, - trainable=False, - dtype=dtypes.float32) - delta = constant_op.constant(0.1, dtype=dtypes.float32) - outputs = distort_image_ops.adjust_hsv_in_yiq(inputs, delta, 1, 1) - run_op = control_flow_ops.group(outputs) - sess.run(variables.global_variables_initializer()) - for i in xrange(warmup_rounds + benchmark_rounds): - if i == warmup_rounds: - start = time.time() - sess.run(run_op) - end = time.time() - step_time = (end - start) / benchmark_rounds - tag = device + '_%s' % (cpu_count if cpu_count is not None else 'all') - print('benchmarkadjust_hue_in_yiq_299_299_3_%s step_time: %.2f us' % - (tag, step_time * 1e6)) - self.report_benchmark( - name='benchmarkadjust_hue_in_yiq_299_299_3_%s' % (tag), - iters=benchmark_rounds, - wall_time=step_time) - - def benchmark_adjust_hue_in_yiqCpu1(self): - self._benchmark_adjust_hue_in_yiq('/cpu:0', 1) - - def benchmark_adjust_hue_in_yiqCpuAll(self): - self._benchmark_adjust_hue_in_yiq('/cpu:0', None) - - def benchmark_adjust_hue_in_yiq_gpu_all(self): - self._benchmark_adjust_hue_in_yiq(test.gpu_device_name(), None) - - -class AdjustSaturationInYiqBenchmark(test.Benchmark): - - def _benchmark_adjust_saturation_in_yiq(self, device, cpu_count): - image_shape = [299, 299, 3] - warmup_rounds = 100 - benchmark_rounds = 1000 - config = config_pb2.ConfigProto() - if cpu_count is not None: - config.inter_op_parallelism_threads = 1 - config.intra_op_parallelism_threads = cpu_count - with session.Session('', graph=ops.Graph(), config=config) as sess: - with ops.device(device): - inputs = variables.Variable( - random_ops.random_uniform(image_shape, dtype=dtypes.float32) * 255, - trainable=False, - dtype=dtypes.float32) - scale = constant_op.constant(0.1, dtype=dtypes.float32) - outputs = distort_image_ops.adjust_hsv_in_yiq(inputs, 0, scale, 1) - run_op = control_flow_ops.group(outputs) - sess.run(variables.global_variables_initializer()) - for _ in xrange(warmup_rounds): - sess.run(run_op) - start = time.time() - for _ in xrange(benchmark_rounds): - sess.run(run_op) - end = time.time() - step_time = (end - start) / benchmark_rounds - tag = '%s' % (cpu_count) if cpu_count is not None else '_all' - print('benchmarkAdjustSaturationInYiq_299_299_3_cpu%s step_time: %.2f us' % - (tag, step_time * 1e6)) - self.report_benchmark( - name='benchmarkAdjustSaturationInYiq_299_299_3_cpu%s' % (tag), - iters=benchmark_rounds, - wall_time=step_time) - - def benchmark_adjust_saturation_in_yiq_cpu1(self): - self._benchmark_adjust_saturation_in_yiq('/cpu:0', 1) - - def benchmark_adjust_saturation_in_yiq_cpu_all(self): - self._benchmark_adjust_saturation_in_yiq('/cpu:0', None) - - def benchmark_adjust_saturation_in_yiq_gpu_all(self): - self._benchmark_adjust_saturation_in_yiq(test.gpu_device_name(), None) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py deleted file mode 100644 index a386e811d3f..00000000000 --- a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py +++ /dev/null @@ -1,361 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for image_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.image.ops import gen_image_ops -from tensorflow.contrib.image.python.ops import image_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradient_checker -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import googletest - -_DTYPES = set([ - dtypes.uint8, dtypes.int32, dtypes.int64, dtypes.float16, dtypes.float32, - dtypes.float64 -]) - - -class ImageOpsTest(test_util.TensorFlowTestCase): - - def test_zeros(self): - for dtype in _DTYPES: - with self.cached_session(): - for shape in [(5, 5), (24, 24), (2, 24, 24, 3)]: - for angle in [0, 1, np.pi / 2.0]: - image = array_ops.zeros(shape, dtype) - self.assertAllEqual( - image_ops.rotate(image, angle).eval(), - np.zeros(shape, dtype.as_numpy_dtype())) - - def test_rotate_even(self): - for dtype in _DTYPES: - with self.cached_session(): - image = array_ops.reshape( - math_ops.cast(math_ops.range(36), dtype), (6, 6)) - image_rep = array_ops.tile(image[None, :, :, None], [3, 1, 1, 1]) - angles = constant_op.constant([0.0, np.pi / 2.0, np.pi * 3. / 2.], - dtypes.float32) - image_rotated = image_ops.rotate(image_rep, angles) - # pyformat: disable - self.assertAllEqual(image_rotated[:, :, :, 0].eval(), - [[[0, 1, 2, 3, 4, 5], - [6, 7, 8, 9, 10, 11], - [12, 13, 14, 15, 16, 17], - [18, 19, 20, 21, 22, 23], - [24, 25, 26, 27, 28, 29], - [30, 31, 32, 33, 34, 35]], - [[5, 11, 17, 23, 29, 35], - [4, 10, 16, 22, 28, 34], - [3, 9, 15, 21, 27, 33], - [2, 8, 14, 20, 26, 32], - [1, 7, 13, 19, 25, 31], - [0, 6, 12, 18, 24, 30]], - [[30, 24, 18, 12, 6, 0], - [31, 25, 19, 13, 7, 1], - [32, 26, 20, 14, 8, 2], - [33, 27, 21, 15, 9, 3], - [34, 28, 22, 16, 10, 4], - [35, 29, 23, 17, 11, 5]]]) - - def test_rotate_odd(self): - for dtype in _DTYPES: - with self.cached_session(): - image = array_ops.reshape( - math_ops.cast(math_ops.range(25), dtype), (5, 5)) - image_rep = array_ops.tile(image[None, :, :, None], [3, 1, 1, 1]) - angles = constant_op.constant([np.pi / 4.0, 1.0, -np.pi / 2.0], - dtypes.float32) - image_rotated = image_ops.rotate(image_rep, angles) - self.assertAllEqual(image_rotated[:, :, :, 0].eval(), - [[[0, 3, 8, 9, 0], [1, 7, 8, 13, 19], - [6, 6, 12, 18, 18], [5, 11, 16, 17, 23], - [0, 15, 16, 21, 0]], - [[0, 3, 9, 14, 0], [2, 7, 8, 13, 19], - [1, 6, 12, 18, 23], [5, 11, 16, 17, 22], - [0, 10, 15, 21, 0]], - [[20, 15, 10, 5, 0], [21, 16, 11, 6, 1], - [22, 17, 12, 7, 2], [23, 18, 13, 8, 3], - [24, 19, 14, 9, 4]]]) - - def test_translate(self): - for dtype in _DTYPES: - with self.cached_session(): - image = constant_op.constant( - [[1, 0, 1, 0], - [0, 1, 0, 1], - [1, 0, 1, 0], - [0, 1, 0, 1]], dtype=dtype) - translation = constant_op.constant([-1, -1], dtypes.float32) - image_translated = image_ops.translate(image, translation) - self.assertAllEqual(image_translated.eval(), - [[1, 0, 1, 0], - [0, 1, 0, 0], - [1, 0, 1, 0], - [0, 0, 0, 0]]) - - def test_compose(self): - for dtype in _DTYPES: - with self.cached_session(): - image = constant_op.constant( - [[1, 1, 1, 0], - [1, 0, 0, 0], - [1, 1, 1, 0], - [0, 0, 0, 0]], dtype=dtype) - # Rotate counter-clockwise by pi / 2. - rotation = image_ops.angles_to_projective_transforms(np.pi / 2, 4, 4) - # Translate right by 1 (the transformation matrix is always inverted, - # hence the -1). - translation = constant_op.constant([1, 0, -1, - 0, 1, 0, - 0, 0], - dtype=dtypes.float32) - composed = image_ops.compose_transforms(rotation, translation) - image_transformed = image_ops.transform(image, composed) - self.assertAllEqual(image_transformed.eval(), - [[0, 0, 0, 0], - [0, 1, 0, 1], - [0, 1, 0, 1], - [0, 1, 1, 1]]) - - def test_extreme_projective_transform(self): - for dtype in _DTYPES: - with self.cached_session(): - image = constant_op.constant( - [[1, 0, 1, 0], - [0, 1, 0, 1], - [1, 0, 1, 0], - [0, 1, 0, 1]], dtype=dtype) - transformation = constant_op.constant([1, 0, 0, 0, 1, 0, -1, 0], - dtypes.float32) - image_transformed = image_ops.transform(image, transformation) - self.assertAllEqual(image_transformed.eval(), - [[1, 0, 0, 0], - [0, 0, 0, 0], - [1, 0, 0, 0], - [0, 0, 0, 0]]) - - def test_bilinear(self): - with self.cached_session(): - image = constant_op.constant( - [[0, 0, 0, 0, 0], - [0, 1, 1, 1, 0], - [0, 1, 0, 1, 0], - [0, 1, 1, 1, 0], - [0, 0, 0, 0, 0]], - dtypes.float32) - # The following result matches: - # >>> scipy.ndimage.rotate(image, 45, order=1, reshape=False) - # which uses spline interpolation of order 1, equivalent to bilinear - # interpolation. - self.assertAllClose( - image_ops.rotate(image, np.pi / 4.0, interpolation="BILINEAR").eval(), - [[0.000, 0.000, 0.343, 0.000, 0.000], - [0.000, 0.586, 0.914, 0.586, 0.000], - [0.343, 0.914, 0.000, 0.914, 0.343], - [0.000, 0.586, 0.914, 0.586, 0.000], - [0.000, 0.000, 0.343, 0.000, 0.000]], - atol=0.001) - self.assertAllClose( - image_ops.rotate(image, np.pi / 4.0, interpolation="NEAREST").eval(), - [[0, 0, 1, 0, 0], - [0, 1, 1, 1, 0], - [1, 1, 0, 1, 1], - [0, 1, 1, 1, 0], - [0, 0, 1, 0, 0]]) - - def test_bilinear_uint8(self): - with self.cached_session(): - image = constant_op.constant( - np.asarray( - [[0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 255, 255, 255, 0.0], - [0.0, 255, 0.0, 255, 0.0], - [0.0, 255, 255, 255, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0]], - np.uint8), - dtypes.uint8) - # == np.rint((expected image above) * 255) - self.assertAllEqual( - image_ops.rotate(image, np.pi / 4.0, interpolation="BILINEAR").eval(), - [[0.0, 0.0, 87., 0.0, 0.0], - [0.0, 149, 233, 149, 0.0], - [87., 233, 0.0, 233, 87.], - [0.0, 149, 233, 149, 0.0], - [0.0, 0.0, 87., 0.0, 0.0]]) - - def test_rotate_static_shape(self): - image = array_ops.diag([1., 2., 3.]) - result = image_ops.rotate( - image, random_ops.random_uniform((), -1, 1), interpolation="BILINEAR") - self.assertEqual(image.get_shape(), result.get_shape()) - - def test_transform_static_output_shape(self): - image = constant_op.constant([[1., 2.], [3., 4.]]) - result = image_ops.transform( - image, random_ops.random_uniform([8], -1, 1), - output_shape=constant_op.constant([3, 5])) - self.assertAllEqual([3, 5], result.get_shape()) - - def _test_grad(self, shape_to_test): - with self.cached_session(): - test_image_shape = shape_to_test - test_image = np.random.randn(*test_image_shape) - test_image_tensor = constant_op.constant( - test_image, shape=test_image_shape) - test_transform = image_ops.angles_to_projective_transforms( - np.pi / 2, 4, 4) - - output_shape = test_image_shape - output = image_ops.transform(test_image_tensor, test_transform) - left_err = gradient_checker.compute_gradient_error( - test_image_tensor, - test_image_shape, - output, - output_shape, - x_init_value=test_image) - self.assertLess(left_err, 1e-10) - - def _test_grad_different_shape(self, input_shape, output_shape): - with self.cached_session(): - test_image_shape = input_shape - test_image = np.random.randn(*test_image_shape) - test_image_tensor = constant_op.constant( - test_image, shape=test_image_shape) - test_transform = image_ops.angles_to_projective_transforms( - np.pi / 2, 4, 4) - - if len(output_shape) == 2: - resize_shape = output_shape - elif len(output_shape) == 3: - resize_shape = output_shape[0:2] - elif len(output_shape) == 4: - resize_shape = output_shape[1:3] - output = image_ops.transform( - images=test_image_tensor, - transforms=test_transform, - output_shape=resize_shape) - left_err = gradient_checker.compute_gradient_error( - test_image_tensor, - test_image_shape, - output, - output_shape, - x_init_value=test_image) - self.assertLess(left_err, 1e-10) - - def test_grad(self): - self._test_grad([16, 16]) - self._test_grad([4, 12, 12]) - self._test_grad([3, 4, 12, 12]) - self._test_grad_different_shape([16, 16], [8, 8]) - self._test_grad_different_shape([4, 12, 3], [8, 24, 3]) - self._test_grad_different_shape([3, 4, 12, 3], [3, 8, 24, 3]) - - def test_projective_transform_v1(self): - """The original ImageProjectiveTransform op should take 2 arguments.""" - image = constant_op.constant([[[[1], [0]], [[0], [1]]]]) - transform = constant_op.constant([[1., 0., 0., 0., 1., 0., 0., 0.]]) - result = gen_image_ops.image_projective_transform( - image, transform, interpolation="NEAREST") - with self.cached_session(): - self.assertAllEqual([[[[1], [0]], [[0], [1]]]], result.eval()) - - def test_transform_data_types(self): - for dtype in _DTYPES: - image = constant_op.constant([[1, 2], [3, 4]], dtype=dtype) - value = image_ops.transform(image, [1] * 8) - with self.test_session(use_gpu=True): - self.assertAllEqual( - value.eval(), - np.array([[4, 4], [4, 4]]).astype(dtype.as_numpy_dtype())) - - @test_util.run_in_graph_and_eager_modes - def test_transform_eager(self): - image = constant_op.constant([[1., 2.], [3., 4.]]) - value = image_ops.transform(image, [1] * 8) - with self.test_session(use_gpu=True): - self.assertAllEqual(self.evaluate(value), np.array([[4, 4], [4, 4]])) - - -class BipartiteMatchTest(test_util.TensorFlowTestCase): - - def _BipartiteMatchTest(self, distance_mat, distance_mat_shape, - num_valid_rows, - expected_row_to_col_match, - expected_col_to_row_match): - distance_mat_np = np.array(distance_mat, dtype=np.float32).reshape( - distance_mat_shape) - expected_row_to_col_match_np = np.array(expected_row_to_col_match, - dtype=np.int32) - expected_col_to_row_match_np = np.array(expected_col_to_row_match, - dtype=np.int32) - - with self.cached_session(): - distance_mat_tf = constant_op.constant(distance_mat_np, - shape=distance_mat_shape) - location_to_prior, prior_to_location = image_ops.bipartite_match( - distance_mat_tf, num_valid_rows) - location_to_prior_np = location_to_prior.eval() - prior_to_location_np = prior_to_location.eval() - self.assertAllEqual(location_to_prior_np, expected_row_to_col_match_np) - self.assertAllEqual(prior_to_location_np, expected_col_to_row_match_np) - - def testBipartiteMatch(self): - distance_mat = [0.5, 0.8, 0.1, - 0.3, 0.2, 0.15] - num_valid_rows = 2 - expected_row_to_col_match = [2, 1] - expected_col_to_row_match = [-1, 1, 0] - self._BipartiteMatchTest(distance_mat, [2, 3], num_valid_rows, - expected_row_to_col_match, - expected_col_to_row_match) - - # The case of num_valid_rows less than num-of-rows-in-distance-mat. - num_valid_rows = 1 - expected_row_to_col_match = [2, -1] - expected_col_to_row_match = [-1, -1, 0] - self._BipartiteMatchTest(distance_mat, [2, 3], num_valid_rows, - expected_row_to_col_match, - expected_col_to_row_match) - - # The case of num_valid_rows being 0. - num_valid_rows = 0 - expected_row_to_col_match = [-1, -1] - expected_col_to_row_match = [-1, -1, -1] - self._BipartiteMatchTest(distance_mat, [2, 3], num_valid_rows, - expected_row_to_col_match, - expected_col_to_row_match) - - # The case of num_valid_rows less being -1. - num_valid_rows = -1 - # The expected results are the same as num_valid_rows being 2. - expected_row_to_col_match = [2, 1] - expected_col_to_row_match = [-1, 1, 0] - self._BipartiteMatchTest(distance_mat, [2, 3], num_valid_rows, - expected_row_to_col_match, - expected_col_to_row_match) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py b/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py deleted file mode 100644 index d58a6542924..00000000000 --- a/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py +++ /dev/null @@ -1,340 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for interpolate_spline.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from scipy import interpolate as sc_interpolate - -from tensorflow.contrib.image.python.ops import interpolate_spline - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util - -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import gradients -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest - -from tensorflow.python.training import momentum - - -class _InterpolationProblem(object): - """Abstract class for interpolation problem descriptions.""" - - def get_problem(self, optimizable=False, extrapolate=True, dtype='float32'): - """Make data for an interpolation problem where all x vectors are n-d. - - Args: - optimizable: If True, then make train_points a tf.Variable. - extrapolate: If False, then clamp the query_points values to be within - the max and min of train_points. - dtype: The data type to use. - - Returns: - query_points, query_values, train_points, train_values: training and - test tensors for interpolation problem - """ - - # The values generated here depend on a seed of 0. - np.random.seed(0) - - batch_size = 1 - num_training_points = 10 - num_query_points = 4 - - init_points = np.random.uniform( - size=[batch_size, num_training_points, self.DATA_DIM]) - - init_points = init_points.astype(dtype) - train_points = ( - variables.Variable(init_points) - if optimizable else constant_op.constant(init_points)) - train_values = self.tf_function(train_points) - - query_points_np = np.random.uniform( - size=[batch_size, num_query_points, self.DATA_DIM]) - query_points_np = query_points_np.astype(dtype) - if not extrapolate: - query_points_np = np.clip(query_points_np, np.min(init_points), - np.max(init_points)) - - query_points = constant_op.constant(query_points_np) - query_values = self.np_function(query_points_np) - - return query_points, query_values, train_points, train_values - - -class _QuadraticPlusSinProblem1D(_InterpolationProblem): - """1D interpolation problem used for regression testing.""" - DATA_DIM = 1 - HARDCODED_QUERY_VALUES = { - (1.0, 0.0): [6.2647187603, -7.84362604077, -5.63690142322, 1.42928896387], - (1.0, - 0.01): [6.77688289946, -8.02163669853, -5.79491157027, 1.4063285693], - (2.0, - 0.0): [8.67110264937, -8.41281390883, -5.80190044693, 1.50155606059], - (2.0, - 0.01): [6.70797816797, -7.49709587663, -5.28965776238, 1.52284731741], - (3.0, - 0.0): [9.37691802935, -8.50390141515, -5.80786417426, 1.63467762122], - (3.0, - 0.01): [4.47106304758, -5.71266128361, -3.92529303296, 1.86755293857], - (4.0, - 0.0): [9.58172461111, -8.51432104771, -5.80967675388, 1.63361164256], - (4.0, 0.01): [ - -3.87902711352, -0.0253462273846, 1.79857618022, -0.769339675725 - ] - } - - def np_function(self, x): - """Takes np array, evaluates the test function, and returns np array.""" - return np.sum( - np.power((x - 0.5), 3) - 0.25 * x + 10 * np.sin(x * 10), - axis=2, - keepdims=True) - - def tf_function(self, x): - """Takes tf tensor, evaluates the test function, and returns tf tensor.""" - return math_ops.reduce_mean( - math_ops.pow((x - 0.5), 3) - 0.25 * x + 10 * math_ops.sin(x * 10), - 2, - keepdims=True) - - -class _QuadraticPlusSinProblemND(_InterpolationProblem): - """3D interpolation problem used for regression testing.""" - - DATA_DIM = 3 - HARDCODED_QUERY_VALUES = { - (1.0, 0.0): [1.06609663962, 1.28894849357, 1.10882405595, 1.63966936885], - (1.0, 0.01): [1.03123780748, 1.2952930985, 1.10366822954, 1.65265118569], - (2.0, 0.0): [0.627787735064, 1.43802857251, 1.00194632358, 1.91667538215], - (2.0, 0.01): [0.730159985046, 1.41702471595, 1.0065827217, 1.85758519312], - (3.0, 0.0): [0.350460417862, 1.67223539464, 1.00475331246, 2.31580322491], - (3.0, - 0.01): [0.624557250556, 1.63138876667, 0.976588193162, 2.12511237866], - (4.0, - 0.0): [0.898129669986, 1.24434133638, -0.938056116931, 1.59910338833], - (4.0, - 0.01): [0.0930360338179, -3.38791305538, -1.00969032567, 0.745535080382], - } - - def np_function(self, x): - """Takes np array, evaluates the test function, and returns np array.""" - return np.sum( - np.square(x - 0.5) + 0.25 * x + 1 * np.sin(x * 15), - axis=2, - keepdims=True) - - def tf_function(self, x): - """Takes tf tensor, evaluates the test function, and returns tf tensor.""" - return math_ops.reduce_sum( - math_ops.square(x - 0.5) + 0.25 * x + 1 * math_ops.sin(x * 15), - 2, - keepdims=True) - - -class InterpolateSplineTest(test_util.TensorFlowTestCase): - - def test_1d_linear_interpolation(self): - """For 1d linear interpolation, we can compare directly to scipy.""" - - tp = _QuadraticPlusSinProblem1D() - (query_points, _, train_points, train_values) = tp.get_problem( - extrapolate=False, dtype='float64') - interpolation_order = 1 - - with ops.name_scope('interpolator'): - interpolator = interpolate_spline.interpolate_spline( - train_points, train_values, query_points, interpolation_order) - with self.cached_session() as sess: - fetches = [query_points, train_points, train_values, interpolator] - query_points_, train_points_, train_values_, interp_ = sess.run(fetches) - - # Just look at the first element of the minibatch. - # Also, trim the final singleton dimension. - interp_ = interp_[0, :, 0] - query_points_ = query_points_[0, :, 0] - train_points_ = train_points_[0, :, 0] - train_values_ = train_values_[0, :, 0] - - # Compute scipy interpolation. - scipy_interp_function = sc_interpolate.interp1d( - train_points_, train_values_, kind='linear') - - scipy_interpolation = scipy_interp_function(query_points_) - scipy_interpolation_on_train = scipy_interp_function(train_points_) - - # Even with float64 precision, the interpolants disagree with scipy a - # bit due to the fact that we add the EPSILON to prevent sqrt(0), etc. - tol = 1e-3 - - self.assertAllClose( - train_values_, scipy_interpolation_on_train, atol=tol, rtol=tol) - self.assertAllClose(interp_, scipy_interpolation, atol=tol, rtol=tol) - - def test_1d_interpolation(self): - """Regression test for interpolation with 1-D points.""" - - tp = _QuadraticPlusSinProblem1D() - (query_points, _, train_points, - train_values) = tp.get_problem(dtype='float64') - - for order in (1, 2, 3): - for reg_weight in (0, 0.01): - interpolator = interpolate_spline.interpolate_spline( - train_points, train_values, query_points, order, reg_weight) - - target_interpolation = tp.HARDCODED_QUERY_VALUES[(order, reg_weight)] - target_interpolation = np.array(target_interpolation) - with self.cached_session() as sess: - interp_val = sess.run(interpolator) - self.assertAllClose(interp_val[0, :, 0], target_interpolation) - - def test_nd_linear_interpolation(self): - """Regression test for interpolation with N-D points.""" - - tp = _QuadraticPlusSinProblemND() - (query_points, _, train_points, - train_values) = tp.get_problem(dtype='float64') - - for order in (1, 2, 3): - for reg_weight in (0, 0.01): - interpolator = interpolate_spline.interpolate_spline( - train_points, train_values, query_points, order, reg_weight) - - target_interpolation = tp.HARDCODED_QUERY_VALUES[(order, reg_weight)] - target_interpolation = np.array(target_interpolation) - with self.cached_session() as sess: - interp_val = sess.run(interpolator) - self.assertAllClose(interp_val[0, :, 0], target_interpolation) - - def test_nd_linear_interpolation_unspecified_shape(self): - """Ensure that interpolation supports dynamic batch_size and num_points.""" - - tp = _QuadraticPlusSinProblemND() - (query_points, _, train_points, - train_values) = tp.get_problem(dtype='float64') - - # Construct placeholders such that the batch size, number of train points, - # and number of query points are not known at graph construction time. - feature_dim = query_points.shape[-1] - value_dim = train_values.shape[-1] - train_points_ph = array_ops.placeholder( - dtype=train_points.dtype, shape=[None, None, feature_dim]) - train_values_ph = array_ops.placeholder( - dtype=train_values.dtype, shape=[None, None, value_dim]) - query_points_ph = array_ops.placeholder( - dtype=query_points.dtype, shape=[None, None, feature_dim]) - - order = 1 - reg_weight = 0.01 - - interpolator = interpolate_spline.interpolate_spline( - train_points_ph, train_values_ph, query_points_ph, order, reg_weight) - - target_interpolation = tp.HARDCODED_QUERY_VALUES[(order, reg_weight)] - target_interpolation = np.array(target_interpolation) - with self.cached_session() as sess: - - (train_points_value, train_values_value, query_points_value) = sess.run( - [train_points, train_values, query_points]) - - interp_val = sess.run( - interpolator, - feed_dict={ - train_points_ph: train_points_value, - train_values_ph: train_values_value, - query_points_ph: query_points_value - }) - self.assertAllClose(interp_val[0, :, 0], target_interpolation) - - def test_fully_unspecified_shape(self): - """Ensure that erreor is thrown when input/output dim unspecified.""" - - tp = _QuadraticPlusSinProblemND() - (query_points, _, train_points, - train_values) = tp.get_problem(dtype='float64') - - # Construct placeholders such that the batch size, number of train points, - # and number of query points are not known at graph construction time. - feature_dim = query_points.shape[-1] - value_dim = train_values.shape[-1] - train_points_ph = array_ops.placeholder( - dtype=train_points.dtype, shape=[None, None, feature_dim]) - train_points_ph_invalid = array_ops.placeholder( - dtype=train_points.dtype, shape=[None, None, None]) - train_values_ph = array_ops.placeholder( - dtype=train_values.dtype, shape=[None, None, value_dim]) - train_values_ph_invalid = array_ops.placeholder( - dtype=train_values.dtype, shape=[None, None, None]) - query_points_ph = array_ops.placeholder( - dtype=query_points.dtype, shape=[None, None, feature_dim]) - - order = 1 - reg_weight = 0.01 - - with self.assertRaises(ValueError): - _ = interpolate_spline.interpolate_spline( - train_points_ph_invalid, train_values_ph, query_points_ph, order, - reg_weight) - - with self.assertRaises(ValueError): - _ = interpolate_spline.interpolate_spline( - train_points_ph, train_values_ph_invalid, query_points_ph, order, - reg_weight) - - def test_interpolation_gradient(self): - """Make sure that backprop can run. Correctness of gradients is assumed. - - Here, we create a use a small 'training' set and a more densely-sampled - set of query points, for which we know the true value in advance. The goal - is to choose x locations for the training data such that interpolating using - this training data yields the best reconstruction for the function - values at the query points. The training data locations are optimized - iteratively using gradient descent. - """ - tp = _QuadraticPlusSinProblemND() - (query_points, query_values, train_points, - train_values) = tp.get_problem(optimizable=True) - - regularization = 0.001 - for interpolation_order in (1, 2, 3, 4): - interpolator = interpolate_spline.interpolate_spline( - train_points, train_values, query_points, interpolation_order, - regularization) - - loss = math_ops.reduce_mean(math_ops.square(query_values - interpolator)) - - optimizer = momentum.MomentumOptimizer(0.001, 0.9) - grad = gradients.gradients(loss, [train_points]) - grad, _ = clip_ops.clip_by_global_norm(grad, 1.0) - opt_func = optimizer.apply_gradients(zip(grad, [train_points])) - init_op = variables.global_variables_initializer() - - with self.cached_session() as sess: - sess.run(init_op) - for _ in range(100): - sess.run([loss, opt_func]) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/image/python/kernel_tests/segmentation_test.py b/tensorflow/contrib/image/python/kernel_tests/segmentation_test.py deleted file mode 100644 index 3d39165ede2..00000000000 --- a/tensorflow/contrib/image/python/kernel_tests/segmentation_test.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for connected component analysis.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import logging - -import numpy as np - -from tensorflow.contrib.image.python.ops import image_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import googletest - -# Image for testing connected_components, with a single, winding component. -SNAKE = np.asarray( - [[0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 1, 1, 1, 1, 0, 0, 0, 0], - [0, 0, 0, 0, 1, 1, 1, 1, 0], - [0, 0, 0, 0, 0, 0, 0, 1, 0], - [0, 1, 1, 1, 1, 1, 1, 1, 0], - [0, 1, 0, 0, 0, 0, 0, 0, 0], - [0, 1, 0, 1, 1, 1, 1, 1, 0], - [0, 1, 0, 0, 0, 0, 0, 1, 0], - [0, 1, 1, 1, 1, 1, 1, 1, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0]]) # pyformat: disable - - -class SegmentationTest(test_util.TensorFlowTestCase): - - def testDisconnected(self): - arr = math_ops.cast( - [[1, 0, 0, 1, 0, 0, 0, 0, 1], - [0, 1, 0, 0, 0, 1, 0, 1, 0], - [1, 0, 1, 0, 0, 0, 1, 0, 0], - [0, 0, 0, 0, 1, 0, 0, 0, 0], - [0, 0, 1, 0, 0, 0, 0, 0, 0]], - dtypes.bool) # pyformat: disable - expected = ( - [[1, 0, 0, 2, 0, 0, 0, 0, 3], - [0, 4, 0, 0, 0, 5, 0, 6, 0], - [7, 0, 8, 0, 0, 0, 9, 0, 0], - [0, 0, 0, 0, 10, 0, 0, 0, 0], - [0, 0, 11, 0, 0, 0, 0, 0, 0]]) # pyformat: disable - with self.cached_session(): - self.assertAllEqual(image_ops.connected_components(arr).eval(), expected) - - def testSimple(self): - arr = [[0, 1, 0], [1, 1, 1], [0, 1, 0]] - with self.cached_session(): - # Single component with id 1. - self.assertAllEqual( - image_ops.connected_components(math_ops.cast( - arr, dtypes.bool)).eval(), arr) - - def testSnake(self): - with self.cached_session(): - # Single component with id 1. - self.assertAllEqual( - image_ops.connected_components(math_ops.cast( - SNAKE, dtypes.bool)).eval(), SNAKE) - - def testSnake_disconnected(self): - for i in range(SNAKE.shape[0]): - for j in range(SNAKE.shape[1]): - with self.cached_session(): - # If we disconnect any part of the snake except for the endpoints, - # there will be 2 components. - if SNAKE[i, j] and (i, j) not in [(1, 1), (6, 3)]: - disconnected_snake = SNAKE.copy() - disconnected_snake[i, j] = 0 - components = image_ops.connected_components( - math_ops.cast(disconnected_snake, dtypes.bool)).eval() - self.assertEqual(components.max(), 2, 'disconnect (%d, %d)' % (i, - j)) - bins = np.bincount(components.ravel()) - # Nonzero number of pixels labeled 0, 1, or 2. - self.assertGreater(bins[0], 0) - self.assertGreater(bins[1], 0) - self.assertGreater(bins[2], 0) - - def testMultipleImages(self): - images = [[[1, 1, 1, 1], - [1, 0, 0, 1], - [1, 0, 0, 1], - [1, 1, 1, 1]], - [[1, 0, 0, 1], - [0, 0, 0, 0], - [0, 0, 0, 0], - [1, 0, 0, 1]], - [[1, 1, 0, 1], - [0, 1, 1, 0], - [1, 0, 1, 0], - [0, 0, 1, 1]]] # pyformat: disable - expected = [[[1, 1, 1, 1], - [1, 0, 0, 1], - [1, 0, 0, 1], - [1, 1, 1, 1]], - [[2, 0, 0, 3], - [0, 0, 0, 0], - [0, 0, 0, 0], - [4, 0, 0, 5]], - [[6, 6, 0, 7], - [0, 6, 6, 0], - [8, 0, 6, 0], - [0, 0, 6, 6]]] # pyformat: disable - with self.cached_session(): - self.assertAllEqual( - image_ops.connected_components(math_ops.cast( - images, dtypes.bool)).eval(), expected) - - def testZeros(self): - with self.cached_session(): - self.assertAllEqual( - image_ops.connected_components( - array_ops.zeros((100, 20, 50), dtypes.bool)).eval(), - np.zeros((100, 20, 50))) - - def testOnes(self): - with self.cached_session(): - self.assertAllEqual( - image_ops.connected_components( - array_ops.ones((100, 20, 50), dtypes.bool)).eval(), - np.tile(np.arange(100)[:, None, None] + 1, [1, 20, 50])) - - def testOnes_small(self): - with self.cached_session(): - self.assertAllEqual( - image_ops.connected_components(array_ops.ones((3, 5), - dtypes.bool)).eval(), - np.ones((3, 5))) - - def testRandom_scipy(self): - np.random.seed(42) - images = np.random.randint(0, 2, size=(10, 100, 200)).astype(np.bool) - expected = connected_components_reference_implementation(images) - if expected is None: - return - with self.cached_session(): - self.assertAllEqual( - image_ops.connected_components(images).eval(), expected) - - -def connected_components_reference_implementation(images): - try: - # pylint: disable=g-import-not-at-top - from scipy.ndimage import measurements - except ImportError: - logging.exception('Skipping test method because scipy could not be loaded') - return - image_or_images = np.asarray(images) - if len(image_or_images.shape) == 2: - images = image_or_images[None, :, :] - elif len(image_or_images.shape) == 3: - images = image_or_images - components = np.asarray([measurements.label(image)[0] for image in images]) - # Get the count of nonzero ids for each image, and offset each image's nonzero - # ids using the cumulative sum. - num_ids_per_image = components.reshape( - [-1, components.shape[1] * components.shape[2]]).max(axis=-1) - positive_id_start_per_image = np.cumsum(num_ids_per_image) - for i in range(components.shape[0]): - new_id_start = positive_id_start_per_image[i - 1] if i > 0 else 0 - components[i, components[i] > 0] += new_id_start - if len(image_or_images.shape) == 2: - return components[0, :, :] - else: - return components - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/image/python/kernel_tests/single_image_random_dot_stereograms_ops_test.py b/tensorflow/contrib/image/python/kernel_tests/single_image_random_dot_stereograms_ops_test.py deleted file mode 100644 index e5980c53b22..00000000000 --- a/tensorflow/contrib/image/python/kernel_tests/single_image_random_dot_stereograms_ops_test.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for python single_image_random_dot_stereograms_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.image.python.ops.single_image_random_dot_stereograms \ - import single_image_random_dot_stereograms -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import test_util -from tensorflow.python.platform import googletest - -class SingleImageRandomDotStereogramsTest(test_util.TensorFlowTestCase): - - def test_shape_function_default(self): - """ - NOTE: The output_image_shape is [X, Y, C] - while the output data is [Y, X, C] (or [H, W, C]). - As a result, by default the output_image_shape has the value - of [1024, 768, 1], but the output data will be [768, 1024, 1]. - """ - x_np = [[1, 2, 3, 3, 2, 1], - [1, 2, 3, 4, 5, 2], - [1, 2, 3, 4, 5, 3], - [1, 2, 3, 4, 5, 4], - [6, 5, 4, 4, 5, 5]] - x_tf = constant_op.constant(x_np) - # By default [1024, 768, 1] => [768, 1024, 1]. - sirds_1 = single_image_random_dot_stereograms( - x_tf, - convergence_dots_size=8, - number_colors=256, - normalize=True) - shape_1 = sirds_1.get_shape().as_list() - self.assertEqual(shape_1, [768, 1024, 1]) - with self.cached_session(): - r_tf_1 = sirds_1.eval() - self.assertAllEqual(shape_1, r_tf_1.shape) - - # If color > 256 then [1024, 768, 3] => [768, 1024, 3]. - sirds_2 = single_image_random_dot_stereograms( - x_tf, - convergence_dots_size=8, - number_colors=512, - normalize=True) - shape_2 = sirds_2.get_shape().as_list() - self.assertEqual(shape_2, [768, 1024, 3]) - with self.cached_session(): - r_tf_2 = sirds_2.eval() - self.assertAllEqual(shape_2, r_tf_2.shape) - - # If explicitly set output_image_shape to [1200, 800, 1], - # then the output data should be [800, 1200, 1]. - sirds_3 = single_image_random_dot_stereograms( - x_tf, - convergence_dots_size=8, - number_colors=256, - normalize=True, - output_image_shape=[1200, 800, 1]) - shape_3 = sirds_3.get_shape().as_list() - self.assertEqual(shape_3, [800, 1200, 1]) - with self.cached_session(): - r_tf_3 = sirds_3.eval() - self.assertAllEqual(shape_3, r_tf_3.shape) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py b/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py deleted file mode 100644 index ce9e34df732..00000000000 --- a/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py +++ /dev/null @@ -1,254 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for sparse_image_warp.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.image.python.ops import sparse_image_warp - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import test_util -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import gradients -from tensorflow.python.ops import image_ops -from tensorflow.python.ops import io_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest -from tensorflow.python.platform import test - -from tensorflow.python.training import momentum - - -class SparseImageWarpTest(test_util.TensorFlowTestCase): - - def setUp(self): - np.random.seed(0) - - def testGetBoundaryLocations(self): - image_height = 11 - image_width = 11 - num_points_per_edge = 4 - locs = sparse_image_warp._get_boundary_locations(image_height, image_width, - num_points_per_edge) - num_points = locs.shape[0] - self.assertEqual(num_points, 4 + 4 * num_points_per_edge) - locs = [(locs[i, 0], locs[i, 1]) for i in range(num_points)] - for i in (0, image_height - 1): - for j in (0, image_width - 1): - self.assertIn((i, j), locs, '{},{} not in the locations'.format(i, j)) - - for i in (2, 4, 6, 8): - for j in (0, image_width - 1): - self.assertIn((i, j), locs, '{},{} not in the locations'.format(i, j)) - - for i in (0, image_height - 1): - for j in (2, 4, 6, 8): - self.assertIn((i, j), locs, '{},{} not in the locations'.format(i, j)) - - def testGetGridLocations(self): - image_height = 5 - image_width = 3 - grid = sparse_image_warp._get_grid_locations(image_height, image_width) - for i in range(image_height): - for j in range(image_width): - self.assertEqual(grid[i, j, 0], i) - self.assertEqual(grid[i, j, 1], j) - - def testZeroShift(self): - """Run assertZeroShift for various hyperparameters.""" - for order in (1, 2): - for regularization in (0, 0.01): - for num_boundary_points in (0, 1): - self.assertZeroShift(order, regularization, num_boundary_points) - - def assertZeroShift(self, order, regularization, num_boundary_points): - """Check that warping with zero displacements doesn't change the image.""" - batch_size = 1 - image_height = 4 - image_width = 4 - channels = 3 - - image = np.random.uniform( - size=[batch_size, image_height, image_width, channels]) - - input_image_op = constant_op.constant(np.float32(image)) - - control_point_locations = [[1., 1.], [2., 2.], [2., 1.]] - control_point_locations = constant_op.constant( - np.float32(np.expand_dims(control_point_locations, 0))) - - control_point_displacements = np.zeros( - control_point_locations.shape.as_list()) - control_point_displacements = constant_op.constant( - np.float32(control_point_displacements)) - - (warped_image_op, flow_field) = sparse_image_warp.sparse_image_warp( - input_image_op, - control_point_locations, - control_point_locations + control_point_displacements, - interpolation_order=order, - regularization_weight=regularization, - num_boundary_points=num_boundary_points) - - with self.cached_session() as sess: - warped_image, input_image, _ = sess.run( - [warped_image_op, input_image_op, flow_field]) - - self.assertAllClose(warped_image, input_image) - - def testMoveSinglePixel(self): - """Run assertMoveSinglePixel for various hyperparameters and data types.""" - for order in (1, 2): - for num_boundary_points in (1, 2): - for type_to_use in (dtypes.float32, dtypes.float64): - self.assertMoveSinglePixel(order, num_boundary_points, type_to_use) - - def assertMoveSinglePixel(self, order, num_boundary_points, type_to_use): - """Move a single block in a small grid using warping.""" - batch_size = 1 - image_height = 7 - image_width = 7 - channels = 3 - - image = np.zeros([batch_size, image_height, image_width, channels]) - image[:, 3, 3, :] = 1.0 - input_image_op = constant_op.constant(image, dtype=type_to_use) - - # Place a control point at the one white pixel. - control_point_locations = [[3., 3.]] - control_point_locations = constant_op.constant( - np.float32(np.expand_dims(control_point_locations, 0)), - dtype=type_to_use) - # Shift it one pixel to the right. - control_point_displacements = [[0., 1.0]] - control_point_displacements = constant_op.constant( - np.float32(np.expand_dims(control_point_displacements, 0)), - dtype=type_to_use) - - (warped_image_op, flow_field) = sparse_image_warp.sparse_image_warp( - input_image_op, - control_point_locations, - control_point_locations + control_point_displacements, - interpolation_order=order, - num_boundary_points=num_boundary_points) - - with self.cached_session() as sess: - warped_image, input_image, flow = sess.run( - [warped_image_op, input_image_op, flow_field]) - # Check that it moved the pixel correctly. - self.assertAllClose( - warped_image[0, 4, 5, :], - input_image[0, 4, 4, :], - atol=1e-5, - rtol=1e-5) - - # Test that there is no flow at the corners. - for i in (0, image_height - 1): - for j in (0, image_width - 1): - self.assertAllClose( - flow[0, i, j, :], np.zeros([2]), atol=1e-5, rtol=1e-5) - - def load_image(self, image_file, sess): - image_op = image_ops.decode_png( - io_ops.read_file(image_file), dtype=dtypes.uint8, channels=4)[:, :, 0:3] - return sess.run(image_op) - - def testSmileyFace(self): - """Check warping accuracy by comparing to hardcoded warped images.""" - - test_data_dir = test.test_src_dir_path('contrib/image/python/' - 'kernel_tests/test_data/') - input_file = test_data_dir + 'Yellow_Smiley_Face.png' - with self.cached_session() as sess: - input_image = self.load_image(input_file, sess) - control_points = np.asarray([[64, 59], [180 - 64, 59], [39, 111], - [180 - 39, 111], [90, 143], [58, 134], - [180 - 58, 134]]) # pyformat: disable - control_point_displacements = np.asarray( - [[-10.5, 10.5], [10.5, 10.5], [0, 0], [0, 0], [0, -10], [-20, 10.25], - [10, 10.75]]) - control_points_op = constant_op.constant( - np.expand_dims(np.float32(control_points[:, [1, 0]]), 0)) - control_point_displacements_op = constant_op.constant( - np.expand_dims(np.float32(control_point_displacements[:, [1, 0]]), 0)) - float_image = np.expand_dims(np.float32(input_image) / 255, 0) - input_image_op = constant_op.constant(float_image) - - for interpolation_order in (1, 2, 3): - for num_boundary_points in (0, 1, 4): - warp_op, _ = sparse_image_warp.sparse_image_warp( - input_image_op, - control_points_op, - control_points_op + control_point_displacements_op, - interpolation_order=interpolation_order, - num_boundary_points=num_boundary_points) - with self.cached_session() as sess: - warped_image = sess.run(warp_op) - out_image = np.uint8(warped_image[0, :, :, :] * 255) - target_file = ( - test_data_dir + - 'Yellow_Smiley_Face_Warp-interp' + '-{}-clamp-{}.png'.format( - interpolation_order, num_boundary_points)) - - target_image = self.load_image(target_file, sess) - - # Check that the target_image and out_image difference is no - # bigger than 2 (on a scale of 0-255). Due to differences in - # floating point computation on different devices, the float - # output in warped_image may get rounded to a different int - # than that in the saved png file loaded into target_image. - self.assertAllClose(target_image, out_image, atol=2, rtol=1e-3) - - def testThatBackpropRuns(self): - """Run optimization to ensure that gradients can be computed.""" - - batch_size = 1 - image_height = 9 - image_width = 12 - image = variables.Variable( - np.float32( - np.random.uniform(size=[batch_size, image_height, image_width, 3]))) - control_point_locations = [[3., 3.]] - control_point_locations = constant_op.constant( - np.float32(np.expand_dims(control_point_locations, 0))) - control_point_displacements = [[0.25, -0.5]] - control_point_displacements = constant_op.constant( - np.float32(np.expand_dims(control_point_displacements, 0))) - warped_image, _ = sparse_image_warp.sparse_image_warp( - image, - control_point_locations, - control_point_locations + control_point_displacements, - num_boundary_points=3) - - loss = math_ops.reduce_mean(math_ops.abs(warped_image - image)) - optimizer = momentum.MomentumOptimizer(0.001, 0.9) - grad = gradients.gradients(loss, [image]) - grad, _ = clip_ops.clip_by_global_norm(grad, 1.0) - opt_func = optimizer.apply_gradients(zip(grad, [image])) - init_op = variables.global_variables_initializer() - - with self.cached_session() as sess: - sess.run(init_op) - for _ in range(5): - sess.run([loss, opt_func]) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face.png deleted file mode 100644 index 7e303881e21..00000000000 Binary files a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face.png and /dev/null differ diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-0.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-0.png deleted file mode 100644 index 7fd9e4e6d69..00000000000 Binary files a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-0.png and /dev/null differ diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-1.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-1.png deleted file mode 100644 index 86d225e5d21..00000000000 Binary files a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-1.png and /dev/null differ diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-4.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-4.png deleted file mode 100644 index 37e8ffae114..00000000000 Binary files a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-4.png and /dev/null differ diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-0.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-0.png deleted file mode 100644 index e49b5816120..00000000000 Binary files a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-0.png and /dev/null differ diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-1.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-1.png deleted file mode 100644 index df3cf200431..00000000000 Binary files a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-1.png and /dev/null differ diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-4.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-4.png deleted file mode 100644 index e1799a87c85..00000000000 Binary files a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-4.png and /dev/null differ diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-0.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-0.png deleted file mode 100644 index 2c346e0ce54..00000000000 Binary files a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-0.png and /dev/null differ diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-1.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-1.png deleted file mode 100644 index 6f8b65451cc..00000000000 Binary files a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-1.png and /dev/null differ diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-4.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-4.png deleted file mode 100644 index 8e78146d955..00000000000 Binary files a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-4.png and /dev/null differ diff --git a/tensorflow/contrib/image/python/ops/dense_image_warp.py b/tensorflow/contrib/image/python/ops/dense_image_warp.py deleted file mode 100644 index f7ced440720..00000000000 --- a/tensorflow/contrib/image/python/ops/dense_image_warp.py +++ /dev/null @@ -1,218 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Image warping using per-pixel flow vectors.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops - -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops - - -def _interpolate_bilinear(grid, - query_points, - name='interpolate_bilinear', - indexing='ij'): - """Similar to Matlab's interp2 function. - - Finds values for query points on a grid using bilinear interpolation. - - Args: - grid: a 4-D float `Tensor` of shape `[batch, height, width, channels]`. - query_points: a 3-D float `Tensor` of N points with shape `[batch, N, 2]`. - name: a name for the operation (optional). - indexing: whether the query points are specified as row and column (ij), - or Cartesian coordinates (xy). - - Returns: - values: a 3-D `Tensor` with shape `[batch, N, channels]` - - Raises: - ValueError: if the indexing mode is invalid, or if the shape of the inputs - invalid. - """ - if indexing != 'ij' and indexing != 'xy': - raise ValueError('Indexing mode must be \'ij\' or \'xy\'') - - with ops.name_scope(name): - grid = ops.convert_to_tensor(grid) - query_points = ops.convert_to_tensor(query_points) - shape = grid.get_shape().as_list() - if len(shape) != 4: - msg = 'Grid must be 4 dimensional. Received size: ' - raise ValueError(msg + str(grid.get_shape())) - - batch_size, height, width, channels = (array_ops.shape(grid)[0], - array_ops.shape(grid)[1], - array_ops.shape(grid)[2], - array_ops.shape(grid)[3]) - - shape = [batch_size, height, width, channels] - query_type = query_points.dtype - grid_type = grid.dtype - - with ops.control_dependencies([ - check_ops.assert_equal( - len(query_points.get_shape()), - 3, - message='Query points must be 3 dimensional.'), - check_ops.assert_equal( - array_ops.shape(query_points)[2], - 2, - message='Query points must be size 2 in dim 2.') - ]): - num_queries = array_ops.shape(query_points)[1] - - with ops.control_dependencies([ - check_ops.assert_greater_equal( - height, 2, message='Grid height must be at least 2.'), - check_ops.assert_greater_equal( - width, 2, message='Grid width must be at least 2.') - ]): - alphas = [] - floors = [] - ceils = [] - index_order = [0, 1] if indexing == 'ij' else [1, 0] - unstacked_query_points = array_ops.unstack(query_points, axis=2) - - for dim in index_order: - with ops.name_scope('dim-' + str(dim)): - queries = unstacked_query_points[dim] - - size_in_indexing_dimension = shape[dim + 1] - - # max_floor is size_in_indexing_dimension - 2 so that max_floor + 1 - # is still a valid index into the grid. - max_floor = math_ops.cast(size_in_indexing_dimension - 2, query_type) - min_floor = constant_op.constant(0.0, dtype=query_type) - floor = math_ops.minimum( - math_ops.maximum(min_floor, math_ops.floor(queries)), max_floor) - int_floor = math_ops.cast(floor, dtypes.int32) - floors.append(int_floor) - ceil = int_floor + 1 - ceils.append(ceil) - - # alpha has the same type as the grid, as we will directly use alpha - # when taking linear combinations of pixel values from the image. - alpha = math_ops.cast(queries - floor, grid_type) - min_alpha = constant_op.constant(0.0, dtype=grid_type) - max_alpha = constant_op.constant(1.0, dtype=grid_type) - alpha = math_ops.minimum(math_ops.maximum(min_alpha, alpha), max_alpha) - - # Expand alpha to [b, n, 1] so we can use broadcasting - # (since the alpha values don't depend on the channel). - alpha = array_ops.expand_dims(alpha, 2) - alphas.append(alpha) - - with ops.control_dependencies([ - check_ops.assert_less_equal( - math_ops.cast(batch_size * height * width, dtype=dtypes.float32), - np.iinfo(np.int32).max / 8, - message="""The image size or batch size is sufficiently large - that the linearized addresses used by array_ops.gather - may exceed the int32 limit.""") - ]): - flattened_grid = array_ops.reshape( - grid, [batch_size * height * width, channels]) - batch_offsets = array_ops.reshape( - math_ops.range(batch_size) * height * width, [batch_size, 1]) - - # This wraps array_ops.gather. We reshape the image data such that the - # batch, y, and x coordinates are pulled into the first dimension. - # Then we gather. Finally, we reshape the output back. It's possible this - # code would be made simpler by using array_ops.gather_nd. - def gather(y_coords, x_coords, name): - with ops.name_scope('gather-' + name): - linear_coordinates = batch_offsets + y_coords * width + x_coords - gathered_values = array_ops.gather(flattened_grid, linear_coordinates) - return array_ops.reshape(gathered_values, - [batch_size, num_queries, channels]) - - # grab the pixel values in the 4 corners around each query point - top_left = gather(floors[0], floors[1], 'top_left') - top_right = gather(floors[0], ceils[1], 'top_right') - bottom_left = gather(ceils[0], floors[1], 'bottom_left') - bottom_right = gather(ceils[0], ceils[1], 'bottom_right') - - # now, do the actual interpolation - with ops.name_scope('interpolate'): - interp_top = alphas[1] * (top_right - top_left) + top_left - interp_bottom = alphas[1] * (bottom_right - bottom_left) + bottom_left - interp = alphas[0] * (interp_bottom - interp_top) + interp_top - - return interp - - -def dense_image_warp(image, flow, name='dense_image_warp'): - """Image warping using per-pixel flow vectors. - - Apply a non-linear warp to the image, where the warp is specified by a dense - flow field of offset vectors that define the correspondences of pixel values - in the output image back to locations in the source image. Specifically, the - pixel value at output[b, j, i, c] is - images[b, j - flow[b, j, i, 0], i - flow[b, j, i, 1], c]. - - The locations specified by this formula do not necessarily map to an int - index. Therefore, the pixel value is obtained by bilinear - interpolation of the 4 nearest pixels around - (b, j - flow[b, j, i, 0], i - flow[b, j, i, 1]). For locations outside - of the image, we use the nearest pixel values at the image boundary. - - - Args: - image: 4-D float `Tensor` with shape `[batch, height, width, channels]`. - flow: A 4-D float `Tensor` with shape `[batch, height, width, 2]`. - name: A name for the operation (optional). - - Note that image and flow can be of type tf.half, tf.float32, or tf.float64, - and do not necessarily have to be the same type. - - Returns: - A 4-D float `Tensor` with shape`[batch, height, width, channels]` - and same type as input image. - - Raises: - ValueError: if height < 2 or width < 2 or the inputs have the wrong number - of dimensions. - """ - with ops.name_scope(name): - batch_size, height, width, channels = (array_ops.shape(image)[0], - array_ops.shape(image)[1], - array_ops.shape(image)[2], - array_ops.shape(image)[3]) - - # The flow is defined on the image grid. Turn the flow into a list of query - # points in the grid space. - grid_x, grid_y = array_ops.meshgrid( - math_ops.range(width), math_ops.range(height)) - stacked_grid = math_ops.cast( - array_ops.stack([grid_y, grid_x], axis=2), flow.dtype) - batched_grid = array_ops.expand_dims(stacked_grid, axis=0) - query_points_on_grid = batched_grid - flow - query_points_flattened = array_ops.reshape(query_points_on_grid, - [batch_size, height * width, 2]) - # Compute values at the query points, then reshape the result back to the - # image grid. - interpolated = _interpolate_bilinear(image, query_points_flattened) - interpolated = array_ops.reshape(interpolated, - [batch_size, height, width, channels]) - return interpolated diff --git a/tensorflow/contrib/image/python/ops/distort_image_ops.py b/tensorflow/contrib/image/python/ops/distort_image_ops.py deleted file mode 100644 index 06e8e4ee720..00000000000 --- a/tensorflow/contrib/image/python/ops/distort_image_ops.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python layer for distort_image_ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.image.ops import gen_distort_image_ops -from tensorflow.contrib.util import loader -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import image_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import resource_loader - -_distort_image_ops = loader.load_op_library( - resource_loader.get_path_to_datafile('_distort_image_ops.so')) - - -# pylint: disable=invalid-name -def random_hsv_in_yiq(image, - max_delta_hue=0, - lower_saturation=1, - upper_saturation=1, - lower_value=1, - upper_value=1, - seed=None): - """Adjust hue, saturation, value of an RGB image randomly in YIQ color space. - - Equivalent to `adjust_yiq_hsv()` but uses a `delta_h` randomly - picked in the interval `[-max_delta_hue, max_delta_hue]`, a `scale_saturation` - randomly picked in the interval `[lower_saturation, upper_saturation]`, and - a `scale_value` randomly picked in the interval - `[lower_saturation, upper_saturation]`. - - Args: - image: RGB image or images. Size of the last dimension must be 3. - max_delta_hue: float. Maximum value for the random delta_hue. Passing 0 - disables adjusting hue. - lower_saturation: float. Lower bound for the random scale_saturation. - upper_saturation: float. Upper bound for the random scale_saturation. - lower_value: float. Lower bound for the random scale_value. - upper_value: float. Upper bound for the random scale_value. - seed: An operation-specific seed. It will be used in conjunction - with the graph-level seed to determine the real seeds that will be - used in this operation. Please see the documentation of - set_random_seed for its interaction with the graph-level random seed. - - Returns: - 3-D float tensor of shape `[height, width, channels]`. - - Raises: - ValueError: if `max_delta`, `lower_saturation`, `upper_saturation`, - `lower_value`, or `upper_Value` is invalid. - """ - if max_delta_hue < 0: - raise ValueError('max_delta must be non-negative.') - - if lower_saturation < 0: - raise ValueError('lower_saturation must be non-negative.') - - if lower_value < 0: - raise ValueError('lower_value must be non-negative.') - - if lower_saturation > upper_saturation: - raise ValueError('lower_saturation must be < upper_saturation.') - - if lower_value > upper_value: - raise ValueError('lower_value must be < upper_value.') - - if max_delta_hue == 0: - delta_hue = 0 - else: - delta_hue = random_ops.random_uniform( - [], -max_delta_hue, max_delta_hue, seed=seed) - if lower_saturation == upper_saturation: - scale_saturation = lower_saturation - else: - scale_saturation = random_ops.random_uniform( - [], lower_saturation, upper_saturation, seed=seed) - if lower_value == upper_value: - scale_value = lower_value - else: - scale_value = random_ops.random_uniform( - [], lower_value, upper_value, seed=seed) - return adjust_hsv_in_yiq(image, delta_hue, scale_saturation, scale_value) - - -def adjust_hsv_in_yiq(image, - delta_hue=0, - scale_saturation=1, - scale_value=1, - name=None): - """Adjust hue, saturation, value of an RGB image in YIQ color space. - - This is a convenience method that converts an RGB image to float - representation, converts it to YIQ, rotates the color around the Y channel by - delta_hue in radians, scales the chrominance channels (I, Q) by - scale_saturation, scales all channels (Y, I, Q) by scale_value, - converts back to RGB, and then back to the original data type. - - `image` is an RGB image. The image hue is adjusted by converting the - image to YIQ, rotating around the luminance channel (Y) by - `delta_hue` in radians, multiplying the chrominance channels (I, Q) by - `scale_saturation`, and multiplying all channels (Y, I, Q) by - `scale_value`. The image is then converted back to RGB. - - Args: - image: RGB image or images. Size of the last dimension must be 3. - delta_hue: float, the hue rotation amount, in radians. - scale_saturation: float, factor to multiply the saturation by. - scale_value: float, factor to multiply the value by. - name: A name for this operation (optional). - - Returns: - Adjusted image(s), same shape and DType as `image`. - """ - with ops.name_scope(name, 'adjust_hsv_in_yiq', [image]) as name: - image = ops.convert_to_tensor(image, name='image') - # Remember original dtype to so we can convert back if needed - orig_dtype = image.dtype - flt_image = image_ops.convert_image_dtype(image, dtypes.float32) - - rgb_altered = gen_distort_image_ops.adjust_hsv_in_yiq( - flt_image, delta_hue, scale_saturation, scale_value) - - return image_ops.convert_image_dtype(rgb_altered, orig_dtype) diff --git a/tensorflow/contrib/image/python/ops/image_ops.py b/tensorflow/contrib/image/python/ops/image_ops.py deleted file mode 100644 index 96f6af2ac51..00000000000 --- a/tensorflow/contrib/image/python/ops/image_ops.py +++ /dev/null @@ -1,537 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python layer for image_ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.eager import context -from tensorflow.contrib.image.ops import gen_image_ops -from tensorflow.contrib.util import loader -from tensorflow.python.framework import common_shapes -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import resource_loader - -_image_ops_so = loader.load_op_library( - resource_loader.get_path_to_datafile("_image_ops.so")) - -_IMAGE_DTYPES = set( - [dtypes.uint8, dtypes.int32, dtypes.int64, - dtypes.float16, dtypes.float32, dtypes.float64]) - -ops.RegisterShape("ImageConnectedComponents")(common_shapes.call_cpp_shape_fn) -ops.RegisterShape("ImageProjectiveTransform")(common_shapes.call_cpp_shape_fn) -ops.RegisterShape("ImageProjectiveTransformV2")(common_shapes.call_cpp_shape_fn) - - -# TODO(ringwalt): Support a "reshape" (name used by SciPy) or "expand" (name -# used by PIL, maybe more readable) mode, which determines the correct -# output_shape and translation for the transform. -def rotate(images, angles, interpolation="NEAREST", name=None): - """Rotate image(s) counterclockwise by the passed angle(s) in radians. - - Args: - images: A tensor of shape (num_images, num_rows, num_columns, num_channels) - (NHWC), (num_rows, num_columns, num_channels) (HWC), or - (num_rows, num_columns) (HW). The rank must be statically known (the - shape is not `TensorShape(None)`. - angles: A scalar angle to rotate all images by, or (if images has rank 4) - a vector of length num_images, with an angle for each image in the batch. - interpolation: Interpolation mode. Supported values: "NEAREST", "BILINEAR". - name: The name of the op. - - Returns: - Image(s) with the same type and shape as `images`, rotated by the given - angle(s). Empty space due to the rotation will be filled with zeros. - - Raises: - TypeError: If `image` is an invalid type. - """ - with ops.name_scope(name, "rotate"): - image_or_images = ops.convert_to_tensor(images) - if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES: - raise TypeError("Invalid dtype %s." % image_or_images.dtype) - elif image_or_images.get_shape().ndims is None: - raise TypeError("image_or_images rank must be statically known") - elif len(image_or_images.get_shape()) == 2: - images = image_or_images[None, :, :, None] - elif len(image_or_images.get_shape()) == 3: - images = image_or_images[None, :, :, :] - elif len(image_or_images.get_shape()) == 4: - images = image_or_images - else: - raise TypeError("Images should have rank between 2 and 4.") - - image_height = math_ops.cast(array_ops.shape(images)[1], - dtypes.float32)[None] - image_width = math_ops.cast(array_ops.shape(images)[2], - dtypes.float32)[None] - output = transform( - images, - angles_to_projective_transforms(angles, image_height, image_width), - interpolation=interpolation) - if image_or_images.get_shape().ndims is None: - raise TypeError("image_or_images rank must be statically known") - elif len(image_or_images.get_shape()) == 2: - return output[0, :, :, 0] - elif len(image_or_images.get_shape()) == 3: - return output[0, :, :, :] - else: - return output - - -def translate(images, translations, interpolation="NEAREST", name=None): - """Translate image(s) by the passed vectors(s). - - Args: - images: A tensor of shape (num_images, num_rows, num_columns, num_channels) - (NHWC), (num_rows, num_columns, num_channels) (HWC), or - (num_rows, num_columns) (HW). The rank must be statically known (the - shape is not `TensorShape(None)`. - translations: A vector representing [dx, dy] or (if images has rank 4) - a matrix of length num_images, with a [dx, dy] vector for each image in - the batch. - interpolation: Interpolation mode. Supported values: "NEAREST", "BILINEAR". - name: The name of the op. - - Returns: - Image(s) with the same type and shape as `images`, translated by the given - vector(s). Empty space due to the translation will be filled with zeros. - - Raises: - TypeError: If `image` is an invalid type. - """ - with ops.name_scope(name, "translate"): - return transform( - images, - translations_to_projective_transforms(translations), - interpolation=interpolation) - - -def angles_to_projective_transforms(angles, - image_height, - image_width, - name=None): - """Returns projective transform(s) for the given angle(s). - - Args: - angles: A scalar angle to rotate all images by, or (for batches of images) - a vector with an angle to rotate each image in the batch. The rank must - be statically known (the shape is not `TensorShape(None)`. - image_height: Height of the image(s) to be transformed. - image_width: Width of the image(s) to be transformed. - - Returns: - A tensor of shape (num_images, 8). Projective transforms which can be given - to `tf.contrib.image.transform`. - """ - with ops.name_scope(name, "angles_to_projective_transforms"): - angle_or_angles = ops.convert_to_tensor( - angles, name="angles", dtype=dtypes.float32) - if len(angle_or_angles.get_shape()) == 0: # pylint: disable=g-explicit-length-test - angles = angle_or_angles[None] - elif len(angle_or_angles.get_shape()) == 1: - angles = angle_or_angles - else: - raise TypeError("Angles should have rank 0 or 1.") - x_offset = ((image_width - 1) - (math_ops.cos(angles) * - (image_width - 1) - math_ops.sin(angles) * - (image_height - 1))) / 2.0 - y_offset = ((image_height - 1) - (math_ops.sin(angles) * - (image_width - 1) + math_ops.cos(angles) * - (image_height - 1))) / 2.0 - num_angles = array_ops.shape(angles)[0] - return array_ops.concat( - values=[ - math_ops.cos(angles)[:, None], - -math_ops.sin(angles)[:, None], - x_offset[:, None], - math_ops.sin(angles)[:, None], - math_ops.cos(angles)[:, None], - y_offset[:, None], - array_ops.zeros((num_angles, 2), dtypes.float32), - ], - axis=1) - - -def translations_to_projective_transforms(translations, name=None): - """Returns projective transform(s) for the given translation(s). - - Args: - translations: A 2-element list representing [dx, dy] or a matrix of - 2-element lists representing [dx, dy] to translate for each image - (for a batch of images). The rank must be statically known (the shape - is not `TensorShape(None)`. - name: The name of the op. - - Returns: - A tensor of shape (num_images, 8) projective transforms which can be given - to `tf.contrib.image.transform`. - """ - with ops.name_scope(name, "translations_to_projective_transforms"): - translation_or_translations = ops.convert_to_tensor( - translations, name="translations", dtype=dtypes.float32) - if translation_or_translations.get_shape().ndims is None: - raise TypeError( - "translation_or_translations rank must be statically known") - elif len(translation_or_translations.get_shape()) == 1: - translations = translation_or_translations[None] - elif len(translation_or_translations.get_shape()) == 2: - translations = translation_or_translations - else: - raise TypeError("Translations should have rank 1 or 2.") - num_translations = array_ops.shape(translations)[0] - # The translation matrix looks like: - # [[1 0 -dx] - # [0 1 -dy] - # [0 0 1]] - # where the last entry is implicit. - # Translation matrices are always float32. - return array_ops.concat( - values=[ - array_ops.ones((num_translations, 1), dtypes.float32), - array_ops.zeros((num_translations, 1), dtypes.float32), - -translations[:, 0, None], - array_ops.zeros((num_translations, 1), dtypes.float32), - array_ops.ones((num_translations, 1), dtypes.float32), - -translations[:, 1, None], - array_ops.zeros((num_translations, 2), dtypes.float32), - ], - axis=1) - - -def transform(images, - transforms, - interpolation="NEAREST", - output_shape=None, - name=None): - """Applies the given transform(s) to the image(s). - - Args: - images: A tensor of shape (num_images, num_rows, num_columns, num_channels) - (NHWC), (num_rows, num_columns, num_channels) (HWC), or - (num_rows, num_columns) (HW). The rank must be statically known (the - shape is not `TensorShape(None)`. - transforms: Projective transform matrix/matrices. A vector of length 8 or - tensor of size N x 8. If one row of transforms is - [a0, a1, a2, b0, b1, b2, c0, c1], then it maps the *output* point - `(x, y)` to a transformed *input* point - `(x', y') = ((a0 x + a1 y + a2) / k, (b0 x + b1 y + b2) / k)`, - where `k = c0 x + c1 y + 1`. The transforms are *inverted* compared to - the transform mapping input points to output points. Note that gradients - are not backpropagated into transformation parameters. - interpolation: Interpolation mode. Supported values: "NEAREST", "BILINEAR". - output_shape: Output dimesion after the transform, [height, width]. - If None, output is the same size as input image. - - name: The name of the op. - - Returns: - Image(s) with the same type and shape as `images`, with the given - transform(s) applied. Transformed coordinates outside of the input image - will be filled with zeros. - - Raises: - TypeError: If `image` is an invalid type. - ValueError: If output shape is not 1-D int32 Tensor. - """ - with ops.name_scope(name, "transform"): - image_or_images = ops.convert_to_tensor(images, name="images") - transform_or_transforms = ops.convert_to_tensor( - transforms, name="transforms", dtype=dtypes.float32) - if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES: - raise TypeError("Invalid dtype %s." % image_or_images.dtype) - elif image_or_images.get_shape().ndims is None: - raise TypeError("image_or_images rank must be statically known") - elif len(image_or_images.get_shape()) == 2: - images = image_or_images[None, :, :, None] - elif len(image_or_images.get_shape()) == 3: - images = image_or_images[None, :, :, :] - elif len(image_or_images.get_shape()) == 4: - images = image_or_images - else: - raise TypeError("Images should have rank between 2 and 4.") - - if output_shape is None: - output_shape = array_ops.shape(images)[1:3] - if not context.executing_eagerly(): - output_shape_value = tensor_util.constant_value(output_shape) - if output_shape_value is not None: - output_shape = output_shape_value - - output_shape = ops.convert_to_tensor( - output_shape, dtypes.int32, name="output_shape") - - if not output_shape.get_shape().is_compatible_with([2]): - raise ValueError("output_shape must be a 1-D Tensor of 2 elements: " - "new_height, new_width") - - if len(transform_or_transforms.get_shape()) == 1: - transforms = transform_or_transforms[None] - elif transform_or_transforms.get_shape().ndims is None: - raise TypeError( - "transform_or_transforms rank must be statically known") - elif len(transform_or_transforms.get_shape()) == 2: - transforms = transform_or_transforms - else: - raise TypeError("Transforms should have rank 1 or 2.") - - output = gen_image_ops.image_projective_transform_v2( - images, - output_shape=output_shape, - transforms=transforms, - interpolation=interpolation.upper()) - if len(image_or_images.get_shape()) == 2: - return output[0, :, :, 0] - elif len(image_or_images.get_shape()) == 3: - return output[0, :, :, :] - else: - return output - - -def compose_transforms(*transforms): - """Composes the transforms tensors. - - Args: - *transforms: List of image projective transforms to be composed. Each - transform is length 8 (single transform) or shape (N, 8) (batched - transforms). The shapes of all inputs must be equal, and at least one - input must be given. - - Returns: - A composed transform tensor. When passed to `tf.contrib.image.transform`, - equivalent to applying each of the given transforms to the image in - order. - """ - assert transforms, "transforms cannot be empty" - with ops.name_scope("compose_transforms"): - composed = flat_transforms_to_matrices(transforms[0]) - for tr in transforms[1:]: - # Multiply batches of matrices. - composed = math_ops.matmul(composed, flat_transforms_to_matrices(tr)) - return matrices_to_flat_transforms(composed) - - -def flat_transforms_to_matrices(transforms): - """Converts `tf.contrib.image` projective transforms to affine matrices. - - Note that the output matrices map output coordinates to input coordinates. For - the forward transformation matrix, call `tf.linalg.inv` on the result. - - Args: - transforms: Vector of length 8, or batches of transforms with shape - `(N, 8)`. - - Returns: - 3D tensor of matrices with shape `(N, 3, 3)`. The output matrices map the - *output coordinates* (in homogeneous coordinates) of each transform to the - corresponding *input coordinates*. - - Raises: - ValueError: If `transforms` have an invalid shape. - """ - with ops.name_scope("flat_transforms_to_matrices"): - transforms = ops.convert_to_tensor(transforms, name="transforms") - if transforms.shape.ndims not in (1, 2): - raise ValueError("Transforms should be 1D or 2D, got: %s" % transforms) - # Make the transform(s) 2D in case the input is a single transform. - transforms = array_ops.reshape(transforms, constant_op.constant([-1, 8])) - num_transforms = array_ops.shape(transforms)[0] - # Add a column of ones for the implicit last entry in the matrix. - return array_ops.reshape( - array_ops.concat( - [transforms, array_ops.ones([num_transforms, 1])], axis=1), - constant_op.constant([-1, 3, 3])) - - -def matrices_to_flat_transforms(transform_matrices): - """Converts affine matrices to `tf.contrib.image` projective transforms. - - Note that we expect matrices that map output coordinates to input coordinates. - To convert forward transformation matrices, call `tf.linalg.inv` on the - matrices and use the result here. - - Args: - transform_matrices: One or more affine transformation matrices, for the - reverse transformation in homogeneous coordinates. Shape `(3, 3)` or - `(N, 3, 3)`. - - Returns: - 2D tensor of flat transforms with shape `(N, 8)`, which may be passed into - `tf.contrib.image.transform`. - - Raises: - ValueError: If `transform_matrices` have an invalid shape. - """ - with ops.name_scope("matrices_to_flat_transforms"): - transform_matrices = ops.convert_to_tensor( - transform_matrices, name="transform_matrices") - if transform_matrices.shape.ndims not in (2, 3): - raise ValueError( - "Matrices should be 2D or 3D, got: %s" % transform_matrices) - # Flatten each matrix. - transforms = array_ops.reshape(transform_matrices, - constant_op.constant([-1, 9])) - # Divide each matrix by the last entry (normally 1). - transforms /= transforms[:, 8:9] - return transforms[:, :8] - - -@ops.RegisterGradient("ImageProjectiveTransformV2") -def _image_projective_transform_grad(op, grad): - """Computes the gradient for ImageProjectiveTransform.""" - images = op.inputs[0] - transforms = op.inputs[1] - interpolation = op.get_attr("interpolation") - - image_or_images = ops.convert_to_tensor(images, name="images") - transform_or_transforms = ops.convert_to_tensor( - transforms, name="transforms", dtype=dtypes.float32) - - if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES: - raise TypeError("Invalid dtype %s." % image_or_images.dtype) - if len(transform_or_transforms.get_shape()) == 1: - transforms = transform_or_transforms[None] - elif len(transform_or_transforms.get_shape()) == 2: - transforms = transform_or_transforms - else: - raise TypeError("Transforms should have rank 1 or 2.") - - # Invert transformations - transforms = flat_transforms_to_matrices(transforms=transforms) - inverse = linalg_ops.matrix_inverse(transforms) - transforms = matrices_to_flat_transforms(inverse) - output = gen_image_ops.image_projective_transform_v2( - images=grad, - transforms=transforms, - output_shape=array_ops.shape(image_or_images)[1:3], - interpolation=interpolation) - return [output, None, None] - - -def bipartite_match(distance_mat, - num_valid_rows, - top_k=-1, - name="bipartite_match"): - """Find bipartite matching based on a given distance matrix. - - A greedy bi-partite matching algorithm is used to obtain the matching with - the (greedy) minimum distance. - - Args: - distance_mat: A 2-D float tensor of shape `[num_rows, num_columns]`. It is a - pair-wise distance matrix between the entities represented by each row and - each column. It is an asymmetric matrix. The smaller the distance is, the - more similar the pairs are. The bipartite matching is to minimize the - distances. - num_valid_rows: A scalar or a 1-D tensor with one element describing the - number of valid rows of distance_mat to consider for the bipartite - matching. If set to be negative, then all rows from `distance_mat` are - used. - top_k: A scalar that specifies the number of top-k matches to retrieve. - If set to be negative, then is set according to the maximum number of - matches from `distance_mat`. - name: The name of the op. - - Returns: - row_to_col_match_indices: A vector of length num_rows, which is the number - of rows of the input `distance_matrix`. If `row_to_col_match_indices[i]` - is not -1, row i is matched to column `row_to_col_match_indices[i]`. - col_to_row_match_indices: A vector of length num_columns, which is the - number of columns of the input distance matrix. - If `col_to_row_match_indices[j]` is not -1, column j is matched to row - `col_to_row_match_indices[j]`. - """ - result = gen_image_ops.bipartite_match( - distance_mat, num_valid_rows, top_k, name=name) - return result - - -def connected_components(images): - """Labels the connected components in a batch of images. - - A component is a set of pixels in a single input image, which are all adjacent - and all have the same non-zero value. The components using a squared - connectivity of one (all True entries are joined with their neighbors above, - below, left, and right). Components across all images have consecutive ids 1 - through n. Components are labeled according to the first pixel of the - component appearing in row-major order (lexicographic order by - image_index_in_batch, row, col). Zero entries all have an output id of 0. - - This op is equivalent with `scipy.ndimage.measurements.label` on a 2D array - with the default structuring element (which is the connectivity used here). - - Args: - images: A 2D (H, W) or 3D (N, H, W) Tensor of boolean image(s). - - Returns: - Components with the same shape as `images`. False entries in `images` have - value 0, and all True entries map to a component id > 0. - - Raises: - TypeError: if `images` is not 2D or 3D. - """ - with ops.name_scope("connected_components"): - image_or_images = ops.convert_to_tensor(images, name="images") - if len(image_or_images.get_shape()) == 2: - images = image_or_images[None, :, :] - elif len(image_or_images.get_shape()) == 3: - images = image_or_images - else: - raise TypeError( - "images should have rank 2 (HW) or 3 (NHW). Static shape is %s" % - image_or_images.get_shape()) - components = gen_image_ops.image_connected_components(images) - - # TODO(ringwalt): Component id renaming should be done in the op, to avoid - # constructing multiple additional large tensors. - components_flat = array_ops.reshape(components, [-1]) - unique_ids, id_index = array_ops.unique(components_flat) - id_is_zero = array_ops.where_v2(math_ops.equal(unique_ids, 0))[:, 0] - # Map each nonzero id to consecutive values. - nonzero_consecutive_ids = math_ops.range( - array_ops.shape(unique_ids)[0] - array_ops.shape(id_is_zero)[0]) + 1 - - def no_zero(): - # No need to insert a zero into the ids. - return nonzero_consecutive_ids - - def has_zero(): - # Insert a zero in the consecutive ids where zero appears in unique_ids. - # id_is_zero has length 1. - zero_id_ind = math_ops.cast(id_is_zero[0], dtypes.int32) - ids_before = nonzero_consecutive_ids[:zero_id_ind] - ids_after = nonzero_consecutive_ids[zero_id_ind:] - return array_ops.concat([ids_before, [0], ids_after], axis=0) - - new_ids = control_flow_ops.cond( - math_ops.equal(array_ops.shape(id_is_zero)[0], 0), no_zero, has_zero) - components = array_ops.reshape( - array_ops.gather(new_ids, id_index), array_ops.shape(components)) - if len(image_or_images.get_shape()) == 2: - return components[0, :, :] - else: - return components - - -ops.NotDifferentiable("BipartiteMatch") -ops.NotDifferentiable("ImageConnectedComponents") diff --git a/tensorflow/contrib/image/python/ops/interpolate_spline.py b/tensorflow/contrib/image/python/ops/interpolate_spline.py deleted file mode 100644 index 3a444d26c2a..00000000000 --- a/tensorflow/contrib/image/python/ops/interpolate_spline.py +++ /dev/null @@ -1,299 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Polyharmonic spline interpolation.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops - -EPSILON = 0.0000000001 - - -def _cross_squared_distance_matrix(x, y): - """Pairwise squared distance between two (batch) matrices' rows (2nd dim). - - Computes the pairwise distances between rows of x and rows of y - Args: - x: [batch_size, n, d] float `Tensor` - y: [batch_size, m, d] float `Tensor` - - Returns: - squared_dists: [batch_size, n, m] float `Tensor`, where - squared_dists[b,i,j] = ||x[b,i,:] - y[b,j,:]||^2 - """ - x_norm_squared = math_ops.reduce_sum(math_ops.square(x), 2) - y_norm_squared = math_ops.reduce_sum(math_ops.square(y), 2) - - # Expand so that we can broadcast. - x_norm_squared_tile = array_ops.expand_dims(x_norm_squared, 2) - y_norm_squared_tile = array_ops.expand_dims(y_norm_squared, 1) - - x_y_transpose = math_ops.matmul(x, y, adjoint_b=True) - - # squared_dists[b,i,j] = ||x_bi - y_bj||^2 = x_bi'x_bi- 2x_bi'x_bj + x_bj'x_bj - squared_dists = x_norm_squared_tile - 2 * x_y_transpose + y_norm_squared_tile - - return squared_dists - - -def _pairwise_squared_distance_matrix(x): - """Pairwise squared distance among a (batch) matrix's rows (2nd dim). - - This saves a bit of computation vs. using _cross_squared_distance_matrix(x,x) - - Args: - x: `[batch_size, n, d]` float `Tensor` - - Returns: - squared_dists: `[batch_size, n, n]` float `Tensor`, where - squared_dists[b,i,j] = ||x[b,i,:] - x[b,j,:]||^2 - """ - - x_x_transpose = math_ops.matmul(x, x, adjoint_b=True) - x_norm_squared = array_ops.matrix_diag_part(x_x_transpose) - x_norm_squared_tile = array_ops.expand_dims(x_norm_squared, 2) - - # squared_dists[b,i,j] = ||x_bi - x_bj||^2 = x_bi'x_bi- 2x_bi'x_bj + x_bj'x_bj - squared_dists = x_norm_squared_tile - 2 * x_x_transpose + array_ops.transpose( - x_norm_squared_tile, [0, 2, 1]) - - return squared_dists - - -def _solve_interpolation(train_points, train_values, order, - regularization_weight): - """Solve for interpolation coefficients. - - Computes the coefficients of the polyharmonic interpolant for the 'training' - data defined by (train_points, train_values) using the kernel phi. - - Args: - train_points: `[b, n, d]` interpolation centers - train_values: `[b, n, k]` function values - order: order of the interpolation - regularization_weight: weight to place on smoothness regularization term - - Returns: - w: `[b, n, k]` weights on each interpolation center - v: `[b, d, k]` weights on each input dimension - Raises: - ValueError: if d or k is not fully specified. - """ - - # These dimensions are set dynamically at runtime. - b, n, _ = array_ops.unstack(array_ops.shape(train_points), num=3) - - d = train_points.shape[-1] - if tensor_shape.dimension_value(d) is None: - raise ValueError('The dimensionality of the input points (d) must be ' - 'statically-inferrable.') - - k = train_values.shape[-1] - if tensor_shape.dimension_value(k) is None: - raise ValueError('The dimensionality of the output values (k) must be ' - 'statically-inferrable.') - - # First, rename variables so that the notation (c, f, w, v, A, B, etc.) - # follows https://en.wikipedia.org/wiki/Polyharmonic_spline. - # To account for python style guidelines we use - # matrix_a for A and matrix_b for B. - - c = train_points - f = train_values - - # Next, construct the linear system. - with ops.name_scope('construct_linear_system'): - - matrix_a = _phi(_pairwise_squared_distance_matrix(c), order) # [b, n, n] - if regularization_weight > 0: - batch_identity_matrix = array_ops.expand_dims( - linalg_ops.eye(n, dtype=c.dtype), 0) - matrix_a += regularization_weight * batch_identity_matrix - - # Append ones to the feature values for the bias term in the linear model. - ones = array_ops.ones_like(c[..., :1], dtype=c.dtype) - matrix_b = array_ops.concat([c, ones], 2) # [b, n, d + 1] - - # [b, n + d + 1, n] - left_block = array_ops.concat( - [matrix_a, array_ops.transpose(matrix_b, [0, 2, 1])], 1) - - num_b_cols = matrix_b.get_shape()[2] # d + 1 - lhs_zeros = array_ops.zeros([b, num_b_cols, num_b_cols], train_points.dtype) - right_block = array_ops.concat([matrix_b, lhs_zeros], - 1) # [b, n + d + 1, d + 1] - lhs = array_ops.concat([left_block, right_block], - 2) # [b, n + d + 1, n + d + 1] - - rhs_zeros = array_ops.zeros([b, d + 1, k], train_points.dtype) - rhs = array_ops.concat([f, rhs_zeros], 1) # [b, n + d + 1, k] - - # Then, solve the linear system and unpack the results. - with ops.name_scope('solve_linear_system'): - w_v = linalg_ops.matrix_solve(lhs, rhs) - w = w_v[:, :n, :] - v = w_v[:, n:, :] - - return w, v - - -def _apply_interpolation(query_points, train_points, w, v, order): - """Apply polyharmonic interpolation model to data. - - Given coefficients w and v for the interpolation model, we evaluate - interpolated function values at query_points. - - Args: - query_points: `[b, m, d]` x values to evaluate the interpolation at - train_points: `[b, n, d]` x values that act as the interpolation centers - ( the c variables in the wikipedia article) - w: `[b, n, k]` weights on each interpolation center - v: `[b, d, k]` weights on each input dimension - order: order of the interpolation - - Returns: - Polyharmonic interpolation evaluated at points defined in query_points. - """ - - # First, compute the contribution from the rbf term. - pairwise_dists = _cross_squared_distance_matrix(query_points, train_points) - phi_pairwise_dists = _phi(pairwise_dists, order) - - rbf_term = math_ops.matmul(phi_pairwise_dists, w) - - # Then, compute the contribution from the linear term. - # Pad query_points with ones, for the bias term in the linear model. - query_points_pad = array_ops.concat([ - query_points, - array_ops.ones_like(query_points[..., :1], train_points.dtype) - ], 2) - linear_term = math_ops.matmul(query_points_pad, v) - - return rbf_term + linear_term - - -def _phi(r, order): - """Coordinate-wise nonlinearity used to define the order of the interpolation. - - See https://en.wikipedia.org/wiki/Polyharmonic_spline for the definition. - - Args: - r: input op - order: interpolation order - - Returns: - phi_k evaluated coordinate-wise on r, for k = r - """ - - # using EPSILON prevents log(0), sqrt0), etc. - # sqrt(0) is well-defined, but its gradient is not - with ops.name_scope('phi'): - if order == 1: - r = math_ops.maximum(r, EPSILON) - r = math_ops.sqrt(r) - return r - elif order == 2: - return 0.5 * r * math_ops.log(math_ops.maximum(r, EPSILON)) - elif order == 4: - return 0.5 * math_ops.square(r) * math_ops.log( - math_ops.maximum(r, EPSILON)) - elif order % 2 == 0: - r = math_ops.maximum(r, EPSILON) - return 0.5 * math_ops.pow(r, 0.5 * order) * math_ops.log(r) - else: - r = math_ops.maximum(r, EPSILON) - return math_ops.pow(r, 0.5 * order) - - -def interpolate_spline(train_points, - train_values, - query_points, - order, - regularization_weight=0.0, - name='interpolate_spline'): - r"""Interpolate signal using polyharmonic interpolation. - - The interpolant has the form - $$f(x) = \sum_{i = 1}^n w_i \phi(||x - c_i||) + v^T x + b.$$ - - This is a sum of two terms: (1) a weighted sum of radial basis function (RBF) - terms, with the centers \\(c_1, ... c_n\\), and (2) a linear term with a bias. - The \\(c_i\\) vectors are 'training' points. In the code, b is absorbed into v - by appending 1 as a final dimension to x. The coefficients w and v are - estimated such that the interpolant exactly fits the value of the function at - the \\(c_i\\) points, the vector w is orthogonal to each \\(c_i\\), and the - vector w sums to 0. With these constraints, the coefficients can be obtained - by solving a linear system. - - \\(\phi\\) is an RBF, parametrized by an interpolation - order. Using order=2 produces the well-known thin-plate spline. - - We also provide the option to perform regularized interpolation. Here, the - interpolant is selected to trade off between the squared loss on the training - data and a certain measure of its curvature - ([details](https://en.wikipedia.org/wiki/Polyharmonic_spline)). - Using a regularization weight greater than zero has the effect that the - interpolant will no longer exactly fit the training data. However, it may be - less vulnerable to overfitting, particularly for high-order interpolation. - - Note the interpolation procedure is differentiable with respect to all inputs - besides the order parameter. - - We support dynamically-shaped inputs, where batch_size, n, and m are None - at graph construction time. However, d and k must be known. - - Args: - train_points: `[batch_size, n, d]` float `Tensor` of n d-dimensional - locations. These do not need to be regularly-spaced. - train_values: `[batch_size, n, k]` float `Tensor` of n c-dimensional values - evaluated at train_points. - query_points: `[batch_size, m, d]` `Tensor` of m d-dimensional locations - where we will output the interpolant's values. - order: order of the interpolation. Common values are 1 for - \\(\phi(r) = r\\), 2 for \\(\phi(r) = r^2 * log(r)\\) (thin-plate spline), - or 3 for \\(\phi(r) = r^3\\). - regularization_weight: weight placed on the regularization term. - This will depend substantially on the problem, and it should always be - tuned. For many problems, it is reasonable to use no regularization. - If using a non-zero value, we recommend a small value like 0.001. - name: name prefix for ops created by this function - - Returns: - `[b, m, k]` float `Tensor` of query values. We use train_points and - train_values to perform polyharmonic interpolation. The query values are - the values of the interpolant evaluated at the locations specified in - query_points. - """ - with ops.name_scope(name): - train_points = ops.convert_to_tensor(train_points) - train_values = ops.convert_to_tensor(train_values) - query_points = ops.convert_to_tensor(query_points) - - # First, fit the spline to the observed data. - with ops.name_scope('solve'): - w, v = _solve_interpolation(train_points, train_values, order, - regularization_weight) - - # Then, evaluate the spline at the query locations. - with ops.name_scope('predict'): - query_values = _apply_interpolation(query_points, train_points, w, v, - order) - - return query_values diff --git a/tensorflow/contrib/image/python/ops/single_image_random_dot_stereograms.py b/tensorflow/contrib/image/python/ops/single_image_random_dot_stereograms.py deleted file mode 100755 index dfc6af3e558..00000000000 --- a/tensorflow/contrib/image/python/ops/single_image_random_dot_stereograms.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python layer for image_ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.image.ops import gen_single_image_random_dot_stereograms_ops -from tensorflow.contrib.util import loader -from tensorflow.python.framework import ops -from tensorflow.python.platform import resource_loader - -_sirds_ops = loader.load_op_library( - resource_loader.get_path_to_datafile( - "_single_image_random_dot_stereograms.so")) - - -def single_image_random_dot_stereograms(depth_values, - hidden_surface_removal=None, - convergence_dots_size=None, - dots_per_inch=None, - eye_separation=None, - mu=None, - normalize=None, - normalize_max=None, - normalize_min=None, - border_level=None, - number_colors=None, - output_image_shape=None, - output_data_window=None): - """Output a RandomDotStereogram Tensor for export via encode_PNG/JPG OP. - - Given the 2-D tensor 'depth_values' with encoded Z values, this operation - will encode 3-D data into a 2-D image. The output of this Op is suitable - for the encode_PNG/JPG ops. Be careful with image compression as this may - corrupt the encode 3-D data within the image. - - Based upon [this - paper](https://www.cs.waikato.ac.nz/~ihw/papers/94-HWT-SI-IHW-SIRDS-paper.pdf). - - This outputs a SIRDS image as picture_out.png: - - ```python - img=[[1,2,3,3,2,1], - [1,2,3,4,5,2], - [1,2,3,4,5,3], - [1,2,3,4,5,4], - [6,5,4,4,5,5]] - session = tf.compat.v1.InteractiveSession() - sirds = single_image_random_dot_stereograms( - img, - convergence_dots_size=8, - number_colors=256,normalize=True) - - out = sirds.eval() - png = tf.image.encode_png(out).eval() - with open('picture_out.png', 'wb') as f: - f.write(png) - ``` - - Args: - depth_values: A `Tensor`. Must be one of the following types: - `float64`, `float32`, `int64`, `int32`. Z values of data to encode - into 'output_data_window' window, lower further away {0.0 floor(far), - 1.0 ceiling(near) after norm}, must be 2-D tensor - hidden_surface_removal: An optional `bool`. Defaults to `True`. - Activate hidden surface removal - convergence_dots_size: An optional `int`. Defaults to `8`. - Black dot size in pixels to help view converge image, drawn on bottom - of the image - dots_per_inch: An optional `int`. Defaults to `72`. - Output device in dots/inch - eye_separation: An optional `float`. Defaults to `2.5`. - Separation between eyes in inches - mu: An optional `float`. Defaults to `0.3333`. - Depth of field, Fraction of viewing distance (eg. 1/3 = 0.3333) - normalize: An optional `bool`. Defaults to `True`. - Normalize input data to [0.0, 1.0] - normalize_max: An optional `float`. Defaults to `-100`. - Fix MAX value for Normalization (0.0) - if < MIN, autoscale - normalize_min: An optional `float`. Defaults to `100`. - Fix MIN value for Normalization (0.0) - if > MAX, autoscale - border_level: An optional `float`. Defaults to `0`. - Value of bord in depth 0.0 {far} to 1.0 {near} - number_colors: An optional `int`. Defaults to `256`. 2 (Black & - White), 256 (grayscale), and Numbers > 256 (Full Color) are - supported - output_image_shape: An optional `tf.TensorShape` or list of `ints`. - Defaults to shape `[1024, 768, 1]`. Defines output shape of returned - image in '[X,Y, Channels]' 1-grayscale, 3 color; channels will be - updated to 3 if number_colors > 256 - output_data_window: An optional `tf.TensorShape` or list of `ints`. - Defaults to `[1022, 757]`. Size of "DATA" window, must be equal to or - smaller than `output_image_shape`, will be centered and use - `convergence_dots_size` for best fit to avoid overlap if possible - - Returns: - A `Tensor` of type `uint8` of shape 'output_image_shape' with encoded - 'depth_values' - """ - - result = gen_single_image_random_dot_stereograms_ops.single_image_random_dot_stereograms( # pylint: disable=line-too-long - depth_values=depth_values, - hidden_surface_removal=hidden_surface_removal, - convergence_dots_size=convergence_dots_size, - dots_per_inch=dots_per_inch, - eye_separation=eye_separation, - mu=mu, - normalize=normalize, - normalize_max=normalize_max, - normalize_min=normalize_min, - border_level=border_level, - number_colors=number_colors, - output_image_shape=output_image_shape, - output_data_window=output_data_window) - return result - - -ops.NotDifferentiable("SingleImageRandomDotStereograms") diff --git a/tensorflow/contrib/image/python/ops/sparse_image_warp.py b/tensorflow/contrib/image/python/ops/sparse_image_warp.py deleted file mode 100644 index 51449ff5e93..00000000000 --- a/tensorflow/contrib/image/python/ops/sparse_image_warp.py +++ /dev/null @@ -1,202 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Image warping using sparse flow defined at control points.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.image.python.ops import dense_image_warp -from tensorflow.contrib.image.python.ops import interpolate_spline - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops - - -def _get_grid_locations(image_height, image_width): - """Wrapper for np.meshgrid.""" - - y_range = np.linspace(0, image_height - 1, image_height) - x_range = np.linspace(0, image_width - 1, image_width) - y_grid, x_grid = np.meshgrid(y_range, x_range, indexing='ij') - return np.stack((y_grid, x_grid), -1) - - -def _expand_to_minibatch(np_array, batch_size): - """Tile arbitrarily-sized np_array to include new batch dimension.""" - tiles = [batch_size] + [1] * np_array.ndim - return np.tile(np.expand_dims(np_array, 0), tiles) - - -def _get_boundary_locations(image_height, image_width, num_points_per_edge): - """Compute evenly-spaced indices along edge of image.""" - y_range = np.linspace(0, image_height - 1, num_points_per_edge + 2) - x_range = np.linspace(0, image_width - 1, num_points_per_edge + 2) - ys, xs = np.meshgrid(y_range, x_range, indexing='ij') - is_boundary = np.logical_or( - np.logical_or(xs == 0, xs == image_width - 1), - np.logical_or(ys == 0, ys == image_height - 1)) - return np.stack([ys[is_boundary], xs[is_boundary]], axis=-1) - - -def _add_zero_flow_controls_at_boundary(control_point_locations, - control_point_flows, image_height, - image_width, boundary_points_per_edge): - """Add control points for zero-flow boundary conditions. - - Augment the set of control points with extra points on the - boundary of the image that have zero flow. - - Args: - control_point_locations: input control points - control_point_flows: their flows - image_height: image height - image_width: image width - boundary_points_per_edge: number of points to add in the middle of each - edge (not including the corners). - The total number of points added is - 4 + 4*(boundary_points_per_edge). - - Returns: - merged_control_point_locations: augmented set of control point locations - merged_control_point_flows: augmented set of control point flows - """ - - batch_size = tensor_shape.dimension_value(control_point_locations.shape[0]) - - boundary_point_locations = _get_boundary_locations(image_height, image_width, - boundary_points_per_edge) - - boundary_point_flows = np.zeros([boundary_point_locations.shape[0], 2]) - - type_to_use = control_point_locations.dtype - boundary_point_locations = constant_op.constant( - _expand_to_minibatch(boundary_point_locations, batch_size), - dtype=type_to_use) - - boundary_point_flows = constant_op.constant( - _expand_to_minibatch(boundary_point_flows, batch_size), dtype=type_to_use) - - merged_control_point_locations = array_ops.concat( - [control_point_locations, boundary_point_locations], 1) - - merged_control_point_flows = array_ops.concat( - [control_point_flows, boundary_point_flows], 1) - - return merged_control_point_locations, merged_control_point_flows - - -def sparse_image_warp(image, - source_control_point_locations, - dest_control_point_locations, - interpolation_order=2, - regularization_weight=0.0, - num_boundary_points=0, - name='sparse_image_warp'): - """Image warping using correspondences between sparse control points. - - Apply a non-linear warp to the image, where the warp is specified by - the source and destination locations of a (potentially small) number of - control points. First, we use a polyharmonic spline - (`tf.contrib.image.interpolate_spline`) to interpolate the displacements - between the corresponding control points to a dense flow field. - Then, we warp the image using this dense flow field - (`tf.contrib.image.dense_image_warp`). - - Let t index our control points. For regularization_weight=0, we have: - warped_image[b, dest_control_point_locations[b, t, 0], - dest_control_point_locations[b, t, 1], :] = - image[b, source_control_point_locations[b, t, 0], - source_control_point_locations[b, t, 1], :]. - - For regularization_weight > 0, this condition is met approximately, since - regularized interpolation trades off smoothness of the interpolant vs. - reconstruction of the interpolant at the control points. - See `tf.contrib.image.interpolate_spline` for further documentation of the - interpolation_order and regularization_weight arguments. - - - Args: - image: `[batch, height, width, channels]` float `Tensor` - source_control_point_locations: `[batch, num_control_points, 2]` float - `Tensor` - dest_control_point_locations: `[batch, num_control_points, 2]` float - `Tensor` - interpolation_order: polynomial order used by the spline interpolation - regularization_weight: weight on smoothness regularizer in interpolation - num_boundary_points: How many zero-flow boundary points to include at - each image edge.Usage: - num_boundary_points=0: don't add zero-flow points - num_boundary_points=1: 4 corners of the image - num_boundary_points=2: 4 corners and one in the middle of each edge - (8 points total) - num_boundary_points=n: 4 corners and n-1 along each edge - name: A name for the operation (optional). - - Note that image and offsets can be of type tf.half, tf.float32, or - tf.float64, and do not necessarily have to be the same type. - - Returns: - warped_image: `[batch, height, width, channels]` float `Tensor` with same - type as input image. - flow_field: `[batch, height, width, 2]` float `Tensor` containing the dense - flow field produced by the interpolation. - """ - - image = ops.convert_to_tensor(image) - source_control_point_locations = ops.convert_to_tensor( - source_control_point_locations) - dest_control_point_locations = ops.convert_to_tensor( - dest_control_point_locations) - - control_point_flows = ( - dest_control_point_locations - source_control_point_locations) - - clamp_boundaries = num_boundary_points > 0 - boundary_points_per_edge = num_boundary_points - 1 - - with ops.name_scope(name): - - batch_size, image_height, image_width, _ = image.get_shape().as_list() - - # This generates the dense locations where the interpolant - # will be evaluated. - grid_locations = _get_grid_locations(image_height, image_width) - - flattened_grid_locations = np.reshape(grid_locations, - [image_height * image_width, 2]) - - flattened_grid_locations = constant_op.constant( - _expand_to_minibatch(flattened_grid_locations, batch_size), image.dtype) - - if clamp_boundaries: - (dest_control_point_locations, - control_point_flows) = _add_zero_flow_controls_at_boundary( - dest_control_point_locations, control_point_flows, image_height, - image_width, boundary_points_per_edge) - - flattened_flows = interpolate_spline.interpolate_spline( - dest_control_point_locations, control_point_flows, - flattened_grid_locations, interpolation_order, regularization_weight) - - dense_flows = array_ops.reshape(flattened_flows, - [batch_size, image_height, image_width, 2]) - - warped_image = dense_image_warp.dense_image_warp(image, dense_flows) - - return warped_image, dense_flows diff --git a/tensorflow/contrib/input_pipeline/BUILD b/tensorflow/contrib/input_pipeline/BUILD deleted file mode 100644 index 4fd9e2c5b95..00000000000 --- a/tensorflow/contrib/input_pipeline/BUILD +++ /dev/null @@ -1,118 +0,0 @@ -# Description: -# Contains ops to build an input pipeline for tensorflow. -# APIs here are meant to evolve over time. - -load("//tensorflow:tensorflow.bzl", "py_test") -load( - "//tensorflow:tensorflow.bzl", - "tf_cc_tests", - "tf_custom_op_library", - "tf_gen_op_libs", - "tf_gen_op_wrapper_py", - "tf_kernel_library", -) -load( - "//tensorflow/core/platform:default/build_config.bzl", - "tf_kernel_tests_linkstatic", -) -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_custom_op_library( - # TODO(sibyl-Mooth6ku,ptucker): Understand why 'python/ops/_' is needed and fix it. - name = "python/ops/_input_pipeline_ops.so", - srcs = [ - "ops/input_pipeline_ops.cc", - ], - deps = [ - "//tensorflow/contrib/input_pipeline/kernels:input_pipeline_kernels", - ], -) - -tf_gen_op_libs( - op_lib_names = ["input_pipeline_ops"], -) - -tf_gen_op_wrapper_py( - name = "input_pipeline_ops", - deps = [":input_pipeline_ops_op_lib"], -) - -tf_kernel_library( - name = "input_pipeline_ops_kernels", - deps = [ - "//tensorflow/contrib/input_pipeline/kernels:input_pipeline_kernels", - "//tensorflow/core:framework", - ], - alwayslink = 1, -) - -tf_custom_op_py_library( - name = "input_pipeline_py", - srcs = glob(["python/ops/*.py"]) + ["__init__.py"], - dso = [":python/ops/_input_pipeline_ops.so"], - kernels = [ - ":input_pipeline_ops_kernels", - ":input_pipeline_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":input_pipeline_ops", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:state_ops", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "input_pipeline_ops_test", - size = "small", - srcs = ["python/ops/input_pipeline_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":input_pipeline_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:state_ops", - "//tensorflow/python:variables", - ], -) - -tf_cc_tests( - name = "input_pipeline_ops_test", - size = "small", - srcs = [ - "ops/input_pipeline_ops_test.cc", - ], - linkstatic = tf_kernel_tests_linkstatic(), - deps = [ - ":input_pipeline_ops_op_lib", - "//tensorflow/cc:cc_ops", - "//tensorflow/core", - "//tensorflow/core:core_cpu", - "//tensorflow/core:core_cpu_internal", - "//tensorflow/core:framework", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//tensorflow/core:ops", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) diff --git a/tensorflow/contrib/input_pipeline/__init__.py b/tensorflow/contrib/input_pipeline/__init__.py deleted file mode 100644 index 02fd4135bfa..00000000000 --- a/tensorflow/contrib/input_pipeline/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops and modules related to input_pipeline. - -@@obtain_next -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.input_pipeline.python.ops.input_pipeline_ops import obtain_next - -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/input_pipeline/kernels/BUILD b/tensorflow/contrib/input_pipeline/kernels/BUILD deleted file mode 100644 index bad6d3c5cf3..00000000000 --- a/tensorflow/contrib/input_pipeline/kernels/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -# Description: -# Contains kernels for the input pipeline. - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -cc_library( - name = "input_pipeline_kernels", - srcs = ["input_pipeline_kernels.cc"], - deps = [ - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - "@com_google_protobuf//:protobuf_headers", - ], - alwayslink = 1, -) diff --git a/tensorflow/contrib/input_pipeline/kernels/input_pipeline_kernels.cc b/tensorflow/contrib/input_pipeline/kernels/input_pipeline_kernels.cc deleted file mode 100644 index d5da76a753f..00000000000 --- a/tensorflow/contrib/input_pipeline/kernels/input_pipeline_kernels.cc +++ /dev/null @@ -1,57 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/platform/mutex.h" - -namespace tensorflow { - -// This Op takes in a list of strings and a counter (ref). It increments the -// counter by 1 and returns the element at that position in the list (circling -// around if need to). -class ObtainNextOp : public OpKernel { - public: - explicit ObtainNextOp(OpKernelConstruction* context) : OpKernel(context) {} - - void Compute(OpKernelContext* ctx) override { - const Tensor* list; - OP_REQUIRES_OK(ctx, ctx->input("list", &list)); - int64 num_elements = list->NumElements(); - auto list_flat = list->flat(); - - // Allocate output. - Tensor* output_tensor = nullptr; - OP_REQUIRES_OK(ctx, ctx->allocate_output("out_element", TensorShape({}), - &output_tensor)); - - // Obtain mutex for the "counter" tensor. - mutex* mu; - OP_REQUIRES_OK(ctx, ctx->input_ref_mutex("counter", &mu)); - mutex_lock l(*mu); - // Increment "counter" tensor by 1. - Tensor counter_tensor; - OP_REQUIRES_OK(ctx, ctx->mutable_input("counter", &counter_tensor, true)); - int64* pos = &counter_tensor.scalar()(); - *pos = (*pos + 1) % num_elements; - - // Assign value to output. - output_tensor->scalar()() = list_flat(*pos); - } -}; - -REGISTER_KERNEL_BUILDER(Name("ObtainNext").Device(DEVICE_CPU), ObtainNextOp); - -} // namespace tensorflow diff --git a/tensorflow/contrib/input_pipeline/ops/input_pipeline_ops.cc b/tensorflow/contrib/input_pipeline/ops/input_pipeline_ops.cc deleted file mode 100644 index 01628c8e1b0..00000000000 --- a/tensorflow/contrib/input_pipeline/ops/input_pipeline_ops.cc +++ /dev/null @@ -1,45 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" - -namespace tensorflow { - -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -REGISTER_OP("ObtainNext") - .Input("list: string") - .Input("counter: Ref(int64)") - .Output("out_element: string") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle unused_input, input1; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &unused_input)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &input1)); - c->set_output(0, c->Scalar()); - return Status::OK(); - }) - .Doc(R"doc( -Takes a list and returns the next based on a counter in a round-robin fashion. - -Returns the element in the list at the new position of the counter, so if you -want to circle the list around start by setting the counter value = -1. - -list: A list of strings -counter: A reference to an int64 variable -)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/input_pipeline/ops/input_pipeline_ops_test.cc b/tensorflow/contrib/input_pipeline/ops/input_pipeline_ops_test.cc deleted file mode 100644 index fbfed207484..00000000000 --- a/tensorflow/contrib/input_pipeline/ops/input_pipeline_ops_test.cc +++ /dev/null @@ -1,38 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); - -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/node_def_builder.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference_testutil.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { - -TEST(InputPipelineOpsTest, ObtainNext_InvalidNumberOfInputs) { - ShapeInferenceTestOp op("ObtainNext"); - op.input_tensors.resize(3); - INFER_ERROR("Wrong number of inputs passed", op, "?;?;?"); -} - -TEST(InputPipelineOpsTest, ObtainNext) { - ShapeInferenceTestOp op("ObtainNext"); - INFER_OK(op, "[100];[]", "[]"); - - INFER_ERROR("Shape must be rank 1 but is rank 2", op, "[1,1];[]"); - INFER_ERROR("Shape must be rank 0 but is rank 1", op, "[1000];[1]"); -} - -} // end namespace tensorflow diff --git a/tensorflow/contrib/input_pipeline/python/ops/input_pipeline_ops.py b/tensorflow/contrib/input_pipeline/python/ops/input_pipeline_ops.py deleted file mode 100644 index 0092ae74b2a..00000000000 --- a/tensorflow/contrib/input_pipeline/python/ops/input_pipeline_ops.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python wrapper for input_pipeline_ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random - -from tensorflow.contrib.input_pipeline.ops import gen_input_pipeline_ops -from tensorflow.contrib.util import loader -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import resource_loader - - -_input_pipeline_ops = loader.load_op_library( - resource_loader.get_path_to_datafile("_input_pipeline_ops.so")) - - -def obtain_next(string_list_tensor, counter): - """Basic wrapper for the ObtainNextOp. - - Args: - string_list_tensor: A tensor that is a list of strings - counter: an int64 ref tensor to keep track of which element is returned. - - Returns: - An op that produces the element at counter + 1 in the list, round - robin style. - """ - return gen_input_pipeline_ops.obtain_next(string_list_tensor, counter) - - -def _maybe_randomize_list(string_list, shuffle): - if shuffle: - random.shuffle(string_list) - return string_list - - -def _create_list(string_list, shuffle, seed, num_epochs): - if shuffle and seed: - random.seed(seed) - expanded_list = _maybe_randomize_list(string_list, shuffle)[:] - if num_epochs: - for _ in range(num_epochs - 1): - expanded_list.extend(_maybe_randomize_list(string_list, shuffle)) - return expanded_list - - -def seek_next(string_list, shuffle=False, seed=None, num_epochs=None): - """Returns an op that seeks the next element in a list of strings. - - Seeking happens in a round robin fashion. This op creates a variable called - obtain_next_counter that is initialized to -1 and is used to keep track of - which element in the list was returned, and a variable - obtain_next_expanded_list to hold the list. If num_epochs is not None, then we - limit the number of times we go around the string_list before OutOfRangeError - is thrown. It creates a variable to keep track of this. - - Args: - string_list: A list of strings. - shuffle: If true, we shuffle the string_list differently for each epoch. - seed: Seed used for shuffling. - num_epochs: Returns OutOfRangeError once string_list has been repeated - num_epoch times. If unspecified then keeps on looping. - - Returns: - An op that produces the next element in the provided list. - """ - expanded_list = _create_list(string_list, shuffle, seed, num_epochs) - - with variable_scope.variable_scope("obtain_next"): - counter = variable_scope.get_variable( - name="obtain_next_counter", - initializer=constant_op.constant( - -1, dtype=dtypes.int64), - dtype=dtypes.int64, - trainable=False) - with ops.colocate_with(counter): - string_tensor = variable_scope.get_variable( - name="obtain_next_expanded_list", - initializer=constant_op.constant(expanded_list), - dtype=dtypes.string, - trainable=False) - if num_epochs: - filename_counter = variable_scope.get_variable( - name="obtain_next_filename_counter", - initializer=constant_op.constant( - 0, dtype=dtypes.int64), - dtype=dtypes.int64, - trainable=False) - c = filename_counter.count_up_to(len(expanded_list)) - with ops.control_dependencies([c]): - return obtain_next(string_tensor, counter) - else: - return obtain_next(string_tensor, counter) diff --git a/tensorflow/contrib/input_pipeline/python/ops/input_pipeline_ops_test.py b/tensorflow/contrib/input_pipeline/python/ops/input_pipeline_ops_test.py deleted file mode 100644 index f44edaa14c0..00000000000 --- a/tensorflow/contrib/input_pipeline/python/ops/input_pipeline_ops_test.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for input_pipeline_ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.input_pipeline.python.ops import input_pipeline_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class InputPipelineOpsTest(test.TestCase): - - def testObtainNext(self): - with self.cached_session(): - var = state_ops.variable_op([], dtypes.int64) - state_ops.assign(var, -1).op.run() - c = constant_op.constant(["a", "b"]) - sample1 = input_pipeline_ops.obtain_next(c, var) - self.assertEqual(b"a", sample1.eval()) - self.assertEqual(0, var.eval()) - sample2 = input_pipeline_ops.obtain_next(c, var) - self.assertEqual(b"b", sample2.eval()) - self.assertEqual(1, var.eval()) - sample3 = input_pipeline_ops.obtain_next(c, var) - self.assertEqual(b"a", sample3.eval()) - self.assertEqual(0, var.eval()) - - def testSeekNext(self): - string_list = ["a", "b", "c"] - with self.cached_session() as session: - elem = input_pipeline_ops.seek_next(string_list) - session.run([variables.global_variables_initializer()]) - self.assertEqual(b"a", session.run(elem)) - self.assertEqual(b"b", session.run(elem)) - self.assertEqual(b"c", session.run(elem)) - # Make sure we loop. - self.assertEqual(b"a", session.run(elem)) - - # Helper method that runs the op len(expected_list) number of times, asserts - # that the results are elements of the expected_list and then throws an - # OutOfRangeError. - def _assert_output(self, expected_list, session, op): - for element in expected_list: - self.assertEqual(element, session.run(op)) - with self.assertRaises(errors.OutOfRangeError): - session.run(op) - - def testSeekNextLimitEpochs(self): - string_list = ["a", "b", "c"] - with self.cached_session() as session: - elem = input_pipeline_ops.seek_next(string_list, num_epochs=1) - session.run([ - variables.local_variables_initializer(), - variables.global_variables_initializer() - ]) - self._assert_output([b"a", b"b", b"c"], session, elem) - - def testSeekNextLimitEpochsThree(self): - string_list = ["a", "b", "c"] - with self.cached_session() as session: - elem = input_pipeline_ops.seek_next(string_list, num_epochs=3) - session.run([ - variables.local_variables_initializer(), - variables.global_variables_initializer() - ]) - # Expect to see [a, b, c] three times. - self._assert_output([b"a", b"b", b"c"] * 3, session, elem) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/integrate/BUILD b/tensorflow/contrib/integrate/BUILD deleted file mode 100644 index 9a696fcb520..00000000000 --- a/tensorflow/contrib/integrate/BUILD +++ /dev/null @@ -1,46 +0,0 @@ -# Description: -# Integration and ODE solvers for TensorFlow. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "integrate_py", - srcs = [ - "__init__.py", - "python/ops/odes.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:functional_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:tensor_array_ops", - "//tensorflow/python:util", - "@six_archive//:six", - ], -) - -py_test( - name = "odes_test", - srcs = ["python/ops/odes_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":integrate_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/integrate/README.md b/tensorflow/contrib/integrate/README.md deleted file mode 100644 index beae6993b9d..00000000000 --- a/tensorflow/contrib/integrate/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Integration and ODE solvers for TensorFlow - -TensorFlow equivalents to the routines provided by `scipy.integrate`. Currently -contains a single function, `odeint`, for integrating ordinary differential -equations. - -Maintainers: -- Stephan Hoyer (shoyer@google.com, github.com/shoyer) -- Marc Coram (mcoram@google.com, github.com/mcoram) diff --git a/tensorflow/contrib/integrate/__init__.py b/tensorflow/contrib/integrate/__init__.py deleted file mode 100644 index 3c37f152e59..00000000000 --- a/tensorflow/contrib/integrate/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Integration and ODE solvers. - -See the -[Contrib Integrate](https://tensorflow.org/api_guides/python/contrib.integrate) -guide. - -@@odeint -@@odeint_fixed -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.integrate.python.ops.odes import * -from tensorflow.python.util.all_util import remove_undocumented - - -remove_undocumented(__name__) diff --git a/tensorflow/contrib/integrate/python/ops/odes.py b/tensorflow/contrib/integrate/python/ops/odes.py deleted file mode 100644 index b7d77130bd0..00000000000 --- a/tensorflow/contrib/integrate/python/ops/odes.py +++ /dev/null @@ -1,704 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ODE solvers for TensorFlow.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc -import collections - -import six - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import tensor_array_ops - -_ButcherTableau = collections.namedtuple('_ButcherTableau', - 'alpha beta c_sol c_mid c_error') - -# Parameters from Shampine (1986), section 4. -_DORMAND_PRINCE_TABLEAU = _ButcherTableau( - alpha=[1 / 5, 3 / 10, 4 / 5, 8 / 9, 1., 1.], - beta=[ - [1 / 5], - [3 / 40, 9 / 40], - [44 / 45, -56 / 15, 32 / 9], - [19372 / 6561, -25360 / 2187, 64448 / 6561, -212 / 729], - [9017 / 3168, -355 / 33, 46732 / 5247, 49 / 176, -5103 / 18656], - [35 / 384, 0, 500 / 1113, 125 / 192, -2187 / 6784, 11 / 84], - ], - c_sol=[35 / 384, 0, 500 / 1113, 125 / 192, -2187 / 6784, 11 / 84, 0], - c_mid=[ - 6025192743 / 30085553152 / 2, 0, 51252292925 / 65400821598 / 2, - -2691868925 / 45128329728 / 2, 187940372067 / 1594534317056 / 2, - -1776094331 / 19743644256 / 2, 11237099 / 235043384 / 2 - ], - c_error=[ - 1951 / 21600 - 35 / 384, - 0, - 22642 / 50085 - 500 / 1113, - 451 / 720 - 125 / 192, - -12231 / 42400 - -2187 / 6784, - 649 / 6300 - 11 / 84, - 1 / 60, - ],) - - -def _possibly_nonzero(x): - return isinstance(x, ops.Tensor) or x != 0 - - -def _scaled_dot_product(scale, xs, ys, name=None): - """Calculate a scaled, vector inner product between lists of Tensors.""" - with ops.name_scope(name, 'scaled_dot_product', [scale, xs, ys]) as scope: - # Some of the parameters in our Butcher tableau include zeros. Using - # _possibly_nonzero lets us avoid wasted computation. - return math_ops.add_n( - [(scale * x) * y for x, y in zip(xs, ys) - if _possibly_nonzero(x) and _possibly_nonzero(y)], - name=scope) - - -def _dot_product(xs, ys, name=None): - """Calculate the vector inner product between two lists of Tensors.""" - with ops.name_scope(name, 'dot_product', [xs, ys]) as scope: - return math_ops.add_n([x * y for x, y in zip(xs, ys)], name=scope) - - -def _runge_kutta_step(func, - y0, - f0, - t0, - dt, - tableau=_DORMAND_PRINCE_TABLEAU, - name=None): - """Take an arbitrary Runge-Kutta step and estimate error. - - Args: - func: Function to evaluate like `func(y, t)` to compute the time derivative - of `y`. - y0: Tensor initial value for the state. - f0: Tensor initial value for the derivative, computed from `func(y0, t0)`. - t0: float64 scalar Tensor giving the initial time. - dt: float64 scalar Tensor giving the size of the desired time step. - tableau: optional _ButcherTableau describing how to take the Runge-Kutta - step. - name: optional name for the operation. - - Returns: - Tuple `(y1, f1, y1_error, k)` giving the estimated function value after - the Runge-Kutta step at `t1 = t0 + dt`, the derivative of the state at `t1`, - estimated error at `t1`, and a list of Runge-Kutta coefficients `k` used for - calculating these terms. - """ - with ops.name_scope(name, 'runge_kutta_step', [y0, f0, t0, dt]) as scope: - y0 = ops.convert_to_tensor(y0, name='y0') - f0 = ops.convert_to_tensor(f0, name='f0') - t0 = ops.convert_to_tensor(t0, name='t0') - dt = ops.convert_to_tensor(dt, name='dt') - dt_cast = math_ops.cast(dt, y0.dtype) - - k = [f0] - for alpha_i, beta_i in zip(tableau.alpha, tableau.beta): - ti = t0 + alpha_i * dt - yi = y0 + _scaled_dot_product(dt_cast, beta_i, k) - k.append(func(yi, ti)) - - if not (tableau.c_sol[-1] == 0 and tableau.c_sol[:-1] == tableau.beta[-1]): - # This property (true for Dormand-Prince) lets us save a few FLOPs. - yi = y0 + _scaled_dot_product(dt_cast, tableau.c_sol, k) - - y1 = array_ops.identity(yi, name='%s/y1' % scope) - f1 = array_ops.identity(k[-1], name='%s/f1' % scope) - y1_error = _scaled_dot_product( - dt_cast, tableau.c_error, k, name='%s/y1_error' % scope) - return (y1, f1, y1_error, k) - - -def _interp_fit(y0, y1, y_mid, f0, f1, dt): - """Fit coefficients for 4th order polynomial interpolation. - - Args: - y0: function value at the start of the interval. - y1: function value at the end of the interval. - y_mid: function value at the mid-point of the interval. - f0: derivative value at the start of the interval. - f1: derivative value at the end of the interval. - dt: width of the interval. - - Returns: - List of coefficients `[a, b, c, d, e]` for interpolating with the polynomial - `p = a * x ** 4 + b * x ** 3 + c * x ** 2 + d * x + e` for values of `x` - between 0 (start of interval) and 1 (end of interval). - """ - # a, b, c, d, e = sympy.symbols('a b c d e') - # x, dt, y0, y1, y_mid, f0, f1 = sympy.symbols('x dt y0 y1 y_mid f0 f1') - # p = a * x ** 4 + b * x ** 3 + c * x ** 2 + d * x + e - # sympy.solve([p.subs(x, 0) - y0, - # p.subs(x, 1 / 2) - y_mid, - # p.subs(x, 1) - y1, - # (p.diff(x) / dt).subs(x, 0) - f0, - # (p.diff(x) / dt).subs(x, 1) - f1], - # [a, b, c, d, e]) - # {a: -2.0*dt*f0 + 2.0*dt*f1 - 8.0*y0 - 8.0*y1 + 16.0*y_mid, - # b: 5.0*dt*f0 - 3.0*dt*f1 + 18.0*y0 + 14.0*y1 - 32.0*y_mid, - # c: -4.0*dt*f0 + dt*f1 - 11.0*y0 - 5.0*y1 + 16.0*y_mid, - # d: dt*f0, - # e: y0} - a = _dot_product([-2 * dt, 2 * dt, -8, -8, 16], [f0, f1, y0, y1, y_mid]) - b = _dot_product([5 * dt, -3 * dt, 18, 14, -32], [f0, f1, y0, y1, y_mid]) - c = _dot_product([-4 * dt, dt, -11, -5, 16], [f0, f1, y0, y1, y_mid]) - d = dt * f0 - e = y0 - return [a, b, c, d, e] - - -def _interp_fit_rk(y0, y1, k, dt, tableau=_DORMAND_PRINCE_TABLEAU): - """Fit an interpolating polynomial to the results of a Runge-Kutta step.""" - with ops.name_scope('interp_fit_rk'): - dt = math_ops.cast(dt, y0.dtype) - y_mid = y0 + _scaled_dot_product(dt, tableau.c_mid, k) - f0 = k[0] - f1 = k[-1] - return _interp_fit(y0, y1, y_mid, f0, f1, dt) - - -def _interp_evaluate(coefficients, t0, t1, t): - """Evaluate polynomial interpolation at the given time point. - - Args: - coefficients: list of Tensor coefficients as created by `interp_fit`. - t0: scalar float64 Tensor giving the start of the interval. - t1: scalar float64 Tensor giving the end of the interval. - t: scalar float64 Tensor giving the desired interpolation point. - - Returns: - Polynomial interpolation of the coefficients at time `t`. - """ - with ops.name_scope('interp_evaluate'): - t0 = ops.convert_to_tensor(t0) - t1 = ops.convert_to_tensor(t1) - t = ops.convert_to_tensor(t) - - dtype = coefficients[0].dtype - - assert_op = control_flow_ops.Assert( - (t0 <= t) & (t <= t1), - ['invalid interpolation, fails `t0 <= t <= t1`:', t0, t, t1]) - with ops.control_dependencies([assert_op]): - x = math_ops.cast((t - t0) / (t1 - t0), dtype) - - xs = [constant_op.constant(1, dtype), x] - for _ in range(2, len(coefficients)): - xs.append(xs[-1] * x) - - return _dot_product(coefficients, reversed(xs)) - - -def _optimal_step_size(last_step, - error_ratio, - safety=0.9, - ifactor=10.0, - dfactor=0.2, - order=5, - name=None): - """Calculate the optimal size for the next Runge-Kutta step.""" - with ops.name_scope(name, 'optimal_step_size', [last_step, - error_ratio]) as scope: - error_ratio = math_ops.cast(error_ratio, last_step.dtype) - exponent = math_ops.cast(1 / order, last_step.dtype) - # this looks more complex than necessary, but importantly it keeps - # error_ratio in the numerator so we can't divide by zero: - factor = math_ops.maximum(1 / ifactor, - math_ops.minimum(error_ratio**exponent / safety, - 1 / dfactor)) - return math_ops.div(last_step, factor, name=scope) - - -def _abs_square(x): - if x.dtype.is_complex: - return math_ops.square(math_ops.real(x)) + math_ops.square(math_ops.imag(x)) - else: - return math_ops.square(x) - - -def _ta_append(tensor_array, value): - """Append a value to the end of a tf.TensorArray.""" - return tensor_array.write(tensor_array.size(), value) - - -class _RungeKuttaState( - collections.namedtuple('_RungeKuttaState', - 'y1, f1, t0, t1, dt, interp_coeff')): - """Saved state of the Runge Kutta solver. - - Attributes: - y1: Tensor giving the function value at the end of the last time step. - f1: Tensor giving derivative at the end of the last time step. - t0: scalar float64 Tensor giving start of the last time step. - t1: scalar float64 Tensor giving end of the last time step. - dt: scalar float64 Tensor giving the size for the next time step. - interp_coef: list of Tensors giving coefficients for polynomial - interpolation between `t0` and `t1`. - """ - - -class _History( - collections.namedtuple('_History', 'integrate_points, error_ratio')): - """Saved integration history for use in `info_dict`. - - Attributes: - integrate_points: tf.TensorArray storing integrating time points. - error_ratio: tf.TensorArray storing computed error ratios at each - integration step. - """ - - -def _assert_increasing(t): - assert_increasing = control_flow_ops.Assert( - math_ops.reduce_all(t[1:] > t[:-1]), ['`t` must be monotonic increasing']) - return ops.control_dependencies([assert_increasing]) - - -def _check_input_types(y0, t, dt=None): - if not (y0.dtype.is_floating or y0.dtype.is_complex): - raise TypeError('`y0` must have a floating point or complex floating ' - 'point dtype') - if not t.dtype.is_floating: - raise TypeError('`t` must have a floating point dtype') - - if dt is not None and not dt.dtype.is_floating: - raise TypeError('`dt` must have a floating point dtype') - - -def _check_input_sizes(t, dt): - if len(t.get_shape().as_list()) > 1: - raise ValueError('t must be a 1D tensor') - - if len(dt.get_shape().as_list()) > 1: - raise ValueError('t must be a 1D tensor') - - if t.get_shape()[0] != dt.get_shape()[0] + 1: - raise ValueError('t and dt have incompatible lengths, must be N and N-1') - - -def _dopri5(func, - y0, - t, - rtol, - atol, - full_output=False, - first_step=None, - safety=0.9, - ifactor=10.0, - dfactor=0.2, - max_num_steps=1000, - name=None): - """Solve an ODE for `odeint` using method='dopri5'.""" - - if first_step is None: - # at some point, we might want to switch to picking the step size - # automatically - first_step = 1.0 - - with ops.name_scope(name, 'dopri5', [ - y0, t, rtol, atol, safety, ifactor, dfactor, max_num_steps - ]) as scope: - - first_step = ops.convert_to_tensor( - first_step, dtype=t.dtype, name='first_step') - safety = ops.convert_to_tensor(safety, dtype=t.dtype, name='safety') - ifactor = ops.convert_to_tensor(ifactor, dtype=t.dtype, name='ifactor') - dfactor = ops.convert_to_tensor(dfactor, dtype=t.dtype, name='dfactor') - max_num_steps = ops.convert_to_tensor( - max_num_steps, dtype=dtypes.int32, name='max_num_steps') - - def adaptive_runge_kutta_step(rk_state, history, n_steps): - """Take an adaptive Runge-Kutta step to integrate the ODE.""" - y0, f0, _, t0, dt, interp_coeff = rk_state - with ops.name_scope('assertions'): - check_underflow = control_flow_ops.Assert(t0 + dt > t0, - ['underflow in dt', dt]) - check_max_num_steps = control_flow_ops.Assert( - n_steps < max_num_steps, ['max_num_steps exceeded']) - check_numerics = control_flow_ops.Assert( - math_ops.reduce_all(math_ops.is_finite(abs(y0))), - ['non-finite values in state `y`', y0]) - with ops.control_dependencies( - [check_underflow, check_max_num_steps, check_numerics]): - y1, f1, y1_error, k = _runge_kutta_step(func, y0, f0, t0, dt) - - with ops.name_scope('error_ratio'): - # We use the same approach as the dopri5 fortran code. - error_tol = atol + rtol * math_ops.maximum(abs(y0), abs(y1)) - tensor_error_ratio = _abs_square(y1_error) / _abs_square(error_tol) - # Could also use reduce_maximum here. - error_ratio = math_ops.sqrt(math_ops.reduce_mean(tensor_error_ratio)) - accept_step = error_ratio <= 1 - - with ops.name_scope('update/rk_state'): - # If we don't accept the step, the _RungeKuttaState will be useless - # (covering a time-interval of size 0), but that's OK, because in such - # cases we always immediately take another Runge-Kutta step. - y_next = control_flow_ops.cond(accept_step, lambda: y1, lambda: y0) - f_next = control_flow_ops.cond(accept_step, lambda: f1, lambda: f0) - t_next = control_flow_ops.cond(accept_step, lambda: t0 + dt, lambda: t0) - interp_coeff = control_flow_ops.cond( - accept_step, lambda: _interp_fit_rk(y0, y1, k, dt), - lambda: interp_coeff) - dt_next = _optimal_step_size(dt, error_ratio, safety, ifactor, dfactor) - rk_state = _RungeKuttaState(y_next, f_next, t0, t_next, dt_next, - interp_coeff) - - with ops.name_scope('update/history'): - history = _History( - _ta_append(history.integrate_points, t0 + dt), - _ta_append(history.error_ratio, error_ratio)) - return rk_state, history, n_steps + 1 - - def interpolate(solution, history, rk_state, i): - """Interpolate through the next time point, integrating as necessary.""" - with ops.name_scope('interpolate'): - rk_state, history, _ = control_flow_ops.while_loop( - lambda rk_state, *_: t[i] > rk_state.t1, - adaptive_runge_kutta_step, (rk_state, history, 0), - name='integrate_loop') - y = _interp_evaluate(rk_state.interp_coeff, rk_state.t0, rk_state.t1, - t[i]) - solution = solution.write(i, y) - return solution, history, rk_state, i + 1 - - with _assert_increasing(t): - num_times = array_ops.size(t) - - solution = tensor_array_ops.TensorArray( - y0.dtype, size=num_times).write(0, y0) - history = _History( - integrate_points=tensor_array_ops.TensorArray( - t.dtype, size=0, dynamic_size=True), - error_ratio=tensor_array_ops.TensorArray( - rtol.dtype, size=0, dynamic_size=True)) - rk_state = _RungeKuttaState( - y0, func(y0, t[0]), t[0], t[0], first_step, interp_coeff=[y0] * 5) - - solution, history, _, _ = control_flow_ops.while_loop( - lambda _, __, ___, i: i < num_times, - interpolate, (solution, history, rk_state, 1), - name='interpolate_loop') - - y = solution.stack(name=scope) - y.set_shape(t.get_shape().concatenate(y0.get_shape())) - if not full_output: - return y - else: - integrate_points = history.integrate_points.stack() - info_dict = { - 'num_func_evals': 6 * array_ops.size(integrate_points) + 1, - 'integrate_points': integrate_points, - 'error_ratio': history.error_ratio.stack() - } - return (y, info_dict) - - -def odeint(func, - y0, - t, - rtol=1e-6, - atol=1e-12, - method=None, - options=None, - full_output=False, - name=None): - """Integrate a system of ordinary differential equations. - - Solves the initial value problem for a non-stiff system of first order ODEs: - - ``` - dy/dt = func(y, t), y(t[0]) = y0 - ``` - - where y is a Tensor of any shape. - - For example: - - ``` - # solve `dy/dt = -y`, corresponding to exponential decay - tf.contrib.integrate.odeint(lambda y, _: -y, 1.0, [0, 1, 2]) - => [1, exp(-1), exp(-2)] - ``` - - Output dtypes and numerical precision are based on the dtypes of the inputs - `y0` and `t`. - - Currently, implements 5th order Runge-Kutta with adaptive step size control - and dense output, using the Dormand-Prince method. Similar to the 'dopri5' - method of `scipy.integrate.ode` and MATLAB's `ode45`. - - Based on: Shampine, Lawrence F. (1986), "Some Practical Runge-Kutta Formulas", - Mathematics of Computation, American Mathematical Society, 46 (173): 135-150, - doi:10.2307/2008219 - - Args: - func: Function that maps a Tensor holding the state `y` and a scalar Tensor - `t` into a Tensor of state derivatives with respect to time. - y0: N-D Tensor giving starting value of `y` at time point `t[0]`. May - have any floating point or complex dtype. - t: 1-D Tensor holding a sequence of time points for which to solve for - `y`. The initial time point should be the first element of this sequence, - and each time must be larger than the previous time. May have any floating - point dtype. If not provided as a Tensor, converted to a Tensor with - float64 dtype. - rtol: optional float64 Tensor specifying an upper bound on relative error, - per element of `y`. - atol: optional float64 Tensor specifying an upper bound on absolute error, - per element of `y`. - method: optional string indicating the integration method to use. Currently, - the only valid option is `'dopri5'`. - options: optional dict of configuring options for the indicated integration - method. Can only be provided if a `method` is explicitly set. For - `'dopri5'`, valid options include: - * first_step: an initial guess for the size of the first integration - (current default: 1.0, but may later be changed to use heuristics based - on the gradient). - * safety: safety factor for adaptive step control, generally a constant - in the range 0.8-1 (default: 0.9). - * ifactor: maximum factor by which the adaptive step may be increased - (default: 10.0). - * dfactor: maximum factor by which the adpative step may be decreased - (default: 0.2). - * max_num_steps: integer maximum number of integrate steps between time - points in `t` (default: 1000). - full_output: optional boolean. If True, `odeint` returns a tuple - `(y, info_dict)` describing the integration process. - name: Optional name for this operation. - - Returns: - y: (N+1)-D tensor, where the first dimension corresponds to different - time points. Contains the solved value of y for each desired time point in - `t`, with the initial value `y0` being the first element along the first - dimension. - info_dict: only if `full_output == True`. A dict with the following values: - * num_func_evals: integer Tensor counting the number of function - evaluations. - * integrate_points: 1D float64 Tensor with the upper bound of each - integration time step. - * error_ratio: 1D float Tensor with the estimated ratio of the integration - error to the error tolerance at each integration step. An ratio greater - than 1 corresponds to rejected steps. - - Raises: - ValueError: if an invalid `method` is provided. - TypeError: if `options` is supplied without `method`, or if `t` or `y0` has - an invalid dtype. - """ - if method is not None and method != 'dopri5': - raise ValueError('invalid method: %r' % method) - - if options is None: - options = {} - elif method is None: - raise ValueError('cannot supply `options` without specifying `method`') - - with ops.name_scope(name, 'odeint', [y0, t, rtol, atol]) as scope: - # TODO(shoyer): use nest.flatten (like tf.while_loop) to allow `y0` to be an - # arbitrarily nested tuple. This will help performance and usability by - # avoiding the need to pack/unpack in user functions. - y0 = ops.convert_to_tensor(y0, name='y0') - t = ops.convert_to_tensor(t, preferred_dtype=dtypes.float64, name='t') - _check_input_types(y0, t) - - error_dtype = abs(y0).dtype - rtol = ops.convert_to_tensor(rtol, dtype=error_dtype, name='rtol') - atol = ops.convert_to_tensor(atol, dtype=error_dtype, name='atol') - - return _dopri5( - func, - y0, - t, - rtol=rtol, - atol=atol, - full_output=full_output, - name=scope, - **options) - - -@six.add_metaclass(abc.ABCMeta) -class _FixedGridIntegrator(object): - """Base class for fixed-grid ODE integrators.""" - - def integrate(self, evol_func, y0, time_grid, dt_grid, steps_on_intervals): - """Returns integrated values of differential equation on the `time grid`. - - Numerically integrates differential equation defined via time derivative - evaluator `evol_func` using fixed time steps specified in dt_grid. - - Args: - evol_func: Callable, evaluates time derivative of y at a given time. - y0: N-D Tensor holds initial values of the solution. - time_grid: 1-D Tensor holding the time points at which the solution - will be recorded, must have a floating dtype. - dt_grid: 1-D Tensor holds fixed time steps to be used on time_grid - intervals. Must be a floating dtype and have one less element than that - of the time_grid. - steps_on_intervals: 1-D Tensor of integer dtype, must have the same size - as dt_grid. Specifies number of steps needed for every interval. Assumes - steps_on_intervals * dt_grid == time intervals. - - Returns: - (N+1)-D tensor, where the first dimension corresponds to different - time points. Contains the solved value of y for each desired time point in - `t`, with the initial value `y0` being the first element along the first - dimension. - """ - - iteration_func = self._make_iteration_func(evol_func, dt_grid) - integrate_interval = self._make_interval_integrator(iteration_func, - steps_on_intervals) - - num_times = array_ops.size(time_grid) - current_time = time_grid[0] - solution_array = tensor_array_ops.TensorArray(y0.dtype, num_times) - solution_array = solution_array.write(0, y0) - - solution_array, _, _, _ = control_flow_ops.while_loop( - lambda _, __, ___, i: i < num_times, - integrate_interval, - (solution_array, y0, current_time, 1) - ) - solution_array = solution_array.stack() - solution_array.set_shape(time_grid.get_shape().concatenate(y0.get_shape())) - return solution_array - - def _make_iteration_func(self, evol_func, dt_grid): - """Returns a function that builds operations of a single time step.""" - - def iteration_func(y, t, dt_step, interval_step): - """Performs a single time step advance.""" - dt = dt_grid[interval_step - 1] - dy = self._step_func(evol_func, t, dt, y) - dy = math_ops.cast(dy, dtype=y.dtype) - return y + dy, t + dt, dt_step + 1, interval_step - - return iteration_func - - def _make_interval_integrator(self, iteration_func, interval_sizes): - """Returns a function that builds operations for interval integration.""" - - def integrate_interval(solution_array, y, t, interval_num): - """Integrates y with fixed time step on interval `interval_num`.""" - y, t, _, _ = control_flow_ops.while_loop( - lambda _, __, j, interval_num: j < interval_sizes[interval_num - 1], - iteration_func, - (y, t, 0, interval_num) - ) - return solution_array.write(interval_num, y), y, t, interval_num + 1 - - return integrate_interval - - @abc.abstractmethod - def _step_func(self, evol_func, t, dt, y): - pass - - -class _MidpointFixedGridIntegrator(_FixedGridIntegrator): - """Fixed grid integrator implementing midpoint scheme.""" - - def _step_func(self, evol_func, t, dt, y): - dt_cast = math_ops.cast(dt, y.dtype) - # yn1 = yn + h * f(tn + h/2, yn + f(tn, yn) * h/2) - return dt_cast * evol_func(y + evol_func(y, t) * dt_cast / 2, t + dt / 2) - - -class _RK4FixedGridIntegrator(_FixedGridIntegrator): - """Fixed grid integrator implementing RK4 scheme.""" - - def _step_func(self, evol_func, t, dt, y): - k1 = evol_func(y, t) - half_step = t + dt / 2 - dt_cast = math_ops.cast(dt, y.dtype) - - k2 = evol_func(y + dt_cast * k1 / 2, half_step) - k3 = evol_func(y + dt_cast * k2 / 2, half_step) - k4 = evol_func(y + dt_cast * k3, t + dt) - return math_ops.add_n([k1, 2 * k2, 2 * k3, k4]) * (dt_cast / 6) - - -def odeint_fixed(func, y0, t, dt=None, method='rk4', name=None): - """ODE integration on a fixed grid (with no step size control). - - Useful in certain scenarios to avoid the overhead of adaptive step size - control, e.g. when differentiation of the integration result is desired and/or - the time grid is known a priori to be sufficient. - - Args: - func: Function that maps a Tensor holding the state `y` and a scalar Tensor - `t` into a Tensor of state derivatives with respect to time. - y0: N-D Tensor giving starting value of `y` at time point `t[0]`. - t: 1-D Tensor holding a sequence of time points for which to solve for - `y`. The initial time point should be the first element of this sequence, - and each time must be larger than the previous time. May have any floating - point dtype. - dt: 0-D or 1-D Tensor providing time step suggestion to be used on time - integration intervals in `t`. 1-D Tensor should provide values - for all intervals, must have 1 less element than that of `t`. - If given a 0-D Tensor, the value is interpreted as time step suggestion - same for all intervals. If passed None, then time step is set to be the - t[1:] - t[:-1]. Defaults to None. The actual step size is obtained by - insuring an integer number of steps per interval, potentially reducing the - time step. - method: One of 'midpoint' or 'rk4'. - name: Optional name for the resulting operation. - - Returns: - y: (N+1)-D tensor, where the first dimension corresponds to different - time points. Contains the solved value of y for each desired time point in - `t`, with the initial value `y0` being the first element along the first - dimension. - - Raises: - ValueError: Upon caller errors. - """ - with ops.name_scope(name, 'odeint_fixed', [y0, t, dt]): - t = ops.convert_to_tensor(t, preferred_dtype=dtypes.float64, name='t') - y0 = ops.convert_to_tensor(y0, name='y0') - - intervals = t[1:] - t[:-1] - if dt is None: - dt = intervals - dt = ops.convert_to_tensor(dt, preferred_dtype=dtypes.float64, name='dt') - - steps_on_intervals = math_ops.ceil(intervals / dt) - dt = intervals / steps_on_intervals - steps_on_intervals = math_ops.cast(steps_on_intervals, dtype=dtypes.int32) - - _check_input_types(y0, t, dt) - _check_input_sizes(t, dt) - - with _assert_increasing(t): - with ops.name_scope(method): - if method == 'midpoint': - return _MidpointFixedGridIntegrator().integrate(func, y0, t, dt, - steps_on_intervals) - elif method == 'rk4': - return _RK4FixedGridIntegrator().integrate(func, y0, t, dt, - steps_on_intervals) - else: - raise ValueError('method not supported: {!s}'.format(method)) diff --git a/tensorflow/contrib/integrate/python/ops/odes_test.py b/tensorflow/contrib/integrate/python/ops/odes_test.py deleted file mode 100644 index be915ef96fb..00000000000 --- a/tensorflow/contrib/integrate/python/ops/odes_test.py +++ /dev/null @@ -1,319 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for ODE solvers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.integrate.python.ops import odes -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class OdeIntTest(test.TestCase): - - def setUp(self): - super(OdeIntTest, self).setUp() - # simple defaults (solution is a sin-wave) - matrix = constant_op.constant([[0, 1], [-1, 0]], dtype=dtypes.float64) - self.func = lambda y, t: math_ops.matmul(matrix, y) - self.y0 = np.array([[1.0], [0.0]]) - - def test_odeint_exp(self): - # Test odeint by an exponential function: - # dy / dt = y, y(0) = 1.0. - # Its analytical solution is y = exp(t). - func = lambda y, t: y - y0 = constant_op.constant(1.0, dtype=dtypes.float64) - t = np.linspace(0.0, 1.0, 11) - y_solved = odes.odeint(func, y0, t) - self.assertIn('odeint', y_solved.name) - self.assertEqual(y_solved.get_shape(), tensor_shape.TensorShape([11])) - with self.cached_session() as sess: - y_solved = sess.run(y_solved) - y_true = np.exp(t) - self.assertAllClose(y_true, y_solved) - - def test_odeint_complex(self): - # Test a complex, linear ODE: - # dy / dt = k * y, y(0) = 1.0. - # Its analytical solution is y = exp(k * t). - k = 1j - 0.1 - func = lambda y, t: k * y - t = np.linspace(0.0, 1.0, 11) - y_solved = odes.odeint(func, 1.0 + 0.0j, t) - with self.cached_session() as sess: - y_solved = sess.run(y_solved) - y_true = np.exp(k * t) - self.assertAllClose(y_true, y_solved) - - def test_odeint_riccati(self): - # The Ricatti equation is: - # dy / dt = (y - t) ** 2 + 1.0, y(0) = 0.5. - # Its analytical solution is y = 1.0 / (2.0 - t) + t. - func = lambda t, y: (y - t)**2 + 1.0 - t = np.linspace(0.0, 1.0, 11) - y_solved = odes.odeint(func, np.float64(0.5), t) - with self.cached_session() as sess: - y_solved = sess.run(y_solved) - y_true = 1.0 / (2.0 - t) + t - self.assertAllClose(y_true, y_solved) - - def test_odeint_2d_linear(self): - # Solve the 2D linear differential equation: - # dy1 / dt = 3.0 * y1 + 4.0 * y2, - # dy2 / dt = -4.0 * y1 + 3.0 * y2, - # y1(0) = 0.0, - # y2(0) = 1.0. - # Its analytical solution is - # y1 = sin(4.0 * t) * exp(3.0 * t), - # y2 = cos(4.0 * t) * exp(3.0 * t). - matrix = constant_op.constant( - [[3.0, 4.0], [-4.0, 3.0]], dtype=dtypes.float64) - func = lambda y, t: math_ops.matmul(matrix, y) - - y0 = constant_op.constant([[0.0], [1.0]], dtype=dtypes.float64) - t = np.linspace(0.0, 1.0, 11) - - y_solved = odes.odeint(func, y0, t) - with self.cached_session() as sess: - y_solved = sess.run(y_solved) - - y_true = np.zeros((len(t), 2, 1)) - y_true[:, 0, 0] = np.sin(4.0 * t) * np.exp(3.0 * t) - y_true[:, 1, 0] = np.cos(4.0 * t) * np.exp(3.0 * t) - self.assertAllClose(y_true, y_solved, atol=1e-5) - - def test_odeint_higher_rank(self): - func = lambda y, t: y - y0 = constant_op.constant(1.0, dtype=dtypes.float64) - t = np.linspace(0.0, 1.0, 11) - for shape in [(), (1,), (1, 1)]: - expected_shape = (len(t),) + shape - y_solved = odes.odeint(func, array_ops.reshape(y0, shape), t) - self.assertEqual(y_solved.get_shape(), - tensor_shape.TensorShape(expected_shape)) - with self.cached_session() as sess: - y_solved = sess.run(y_solved) - self.assertEquals(y_solved.shape, expected_shape) - - def test_odeint_all_dtypes(self): - func = lambda y, t: y - t = np.linspace(0.0, 1.0, 11) - for y0_dtype in [ - dtypes.float32, dtypes.float64, dtypes.complex64, dtypes.complex128 - ]: - for t_dtype in [dtypes.float32, dtypes.float64]: - y0 = math_ops.cast(1.0, y0_dtype) - y_solved = odes.odeint(func, y0, math_ops.cast(t, t_dtype)) - with self.cached_session() as sess: - y_solved = sess.run(y_solved) - expected = np.asarray(np.exp(t)) - self.assertAllClose(y_solved, expected, rtol=1e-5) - self.assertEqual(dtypes.as_dtype(y_solved.dtype), y0_dtype) - - def test_odeint_required_dtypes(self): - with self.assertRaisesRegexp(TypeError, '`y0` must have a floating point'): - odes.odeint(self.func, math_ops.cast(self.y0, dtypes.int32), [0, 1]) - - with self.assertRaisesRegexp(TypeError, '`t` must have a floating point'): - odes.odeint(self.func, self.y0, math_ops.cast([0, 1], dtypes.int32)) - - def test_odeint_runtime_errors(self): - with self.assertRaisesRegexp(ValueError, 'cannot supply `options` without'): - odes.odeint(self.func, self.y0, [0, 1], options={'first_step': 1.0}) - - y = odes.odeint( - self.func, - self.y0, [0, 1], - method='dopri5', - options={'max_num_steps': 0}) - with self.cached_session() as sess: - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - 'max_num_steps'): - sess.run(y) - - y = odes.odeint(self.func, self.y0, [1, 0]) - with self.cached_session() as sess: - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - 'monotonic increasing'): - sess.run(y) - - def test_odeint_different_times(self): - # integrate steps should be independent of interpolation times - times0 = np.linspace(0, 10, num=11, dtype=float) - times1 = np.linspace(0, 10, num=101, dtype=float) - - with self.cached_session() as sess: - y_solved_0, info_0 = sess.run( - odes.odeint(self.func, self.y0, times0, full_output=True)) - y_solved_1, info_1 = sess.run( - odes.odeint(self.func, self.y0, times1, full_output=True)) - - self.assertAllClose(y_solved_0, y_solved_1[::10]) - self.assertEqual(info_0['num_func_evals'], info_1['num_func_evals']) - self.assertAllEqual(info_0['integrate_points'], info_1['integrate_points']) - self.assertAllEqual(info_0['error_ratio'], info_1['error_ratio']) - - def test_odeint_5th_order_accuracy(self): - t = [0, 20] - kwargs = dict( - full_output=True, method='dopri5', options=dict(max_num_steps=2000)) - with self.cached_session() as sess: - _, info_0 = sess.run( - odes.odeint(self.func, self.y0, t, rtol=0, atol=1e-6, **kwargs)) - _, info_1 = sess.run( - odes.odeint(self.func, self.y0, t, rtol=0, atol=1e-9, **kwargs)) - self.assertAllClose( - info_0['integrate_points'].size * 1000**0.2, - float(info_1['integrate_points'].size), - rtol=0.01) - - -class StepSizeTest(test.TestCase): - - def test_error_ratio_one(self): - new_step = odes._optimal_step_size( - last_step=constant_op.constant(1.0), - error_ratio=constant_op.constant(1.0)) - with self.cached_session() as sess: - new_step = sess.run(new_step) - self.assertAllClose(new_step, 0.9) - - def test_ifactor(self): - new_step = odes._optimal_step_size( - last_step=constant_op.constant(1.0), - error_ratio=constant_op.constant(0.0)) - with self.cached_session() as sess: - new_step = sess.run(new_step) - self.assertAllClose(new_step, 10.0) - - def test_dfactor(self): - new_step = odes._optimal_step_size( - last_step=constant_op.constant(1.0), - error_ratio=constant_op.constant(1e6)) - with self.cached_session() as sess: - new_step = sess.run(new_step) - self.assertAllClose(new_step, 0.2) - - -class InterpolationTest(test.TestCase): - - def test_5th_order_polynomial(self): - # this should be an exact fit - f = lambda x: x**4 + x**3 - 2 * x**2 + 4 * x + 5 - f_prime = lambda x: 4 * x**3 + 3 * x**2 - 4 * x + 4 - coeffs = odes._interp_fit( - f(0.0), f(10.0), f(5.0), f_prime(0.0), f_prime(10.0), 10.0) - times = np.linspace(0, 10, dtype=np.float32) - y_fit = array_ops.stack( - [odes._interp_evaluate(coeffs, 0.0, 10.0, t) for t in times]) - y_expected = f(times) - with self.cached_session() as sess: - y_actual = sess.run(y_fit) - self.assertAllClose(y_expected, y_actual) - - # attempt interpolation outside bounds - y_invalid = odes._interp_evaluate(coeffs, 0.0, 10.0, 100.0) - with self.cached_session() as sess: - with self.assertRaises(errors_impl.InvalidArgumentError): - sess.run(y_invalid) - - -class OdeIntFixedTest(test.TestCase): - - def _test_integrate_sine(self, method, t, dt=None): - - def evol_func(y, t): - del t - return array_ops.stack([y[1], -y[0]]) - - y0 = [0., 1.] - y_grid = odes.odeint_fixed(evol_func, y0, t, dt, method=method) - - with self.cached_session() as sess: - y_grid_array = sess.run(y_grid) - - np.testing.assert_allclose( - y_grid_array[:, 0], np.sin(t), rtol=1e-2, atol=1e-2) - - def _test_integrate_gaussian(self, method, t, dt=None): - - def evol_func(y, t): - return -math_ops.cast(t, dtype=y.dtype) * y[0] - - y0 = [1.] - y_grid = odes.odeint_fixed(evol_func, y0, t, dt, method=method) - - with self.cached_session() as sess: - y_grid_array = sess.run(y_grid) - - np.testing.assert_allclose( - y_grid_array[:, 0], np.exp(-t**2 / 2), rtol=1e-2, atol=1e-2) - - def _test_integrate_sine_all(self, method): - uniform_time_grid = np.linspace(0., 10., 200) - non_uniform_time_grid = np.asarray([0.0, 0.4, 4.7, 5.2, 7.0]) - uniform_dt = 0.02 - non_uniform_dt = np.asarray([0.01, 0.001, 0.05, 0.03]) - self._test_integrate_sine(method, uniform_time_grid) - self._test_integrate_sine(method, non_uniform_time_grid, uniform_dt) - self._test_integrate_sine(method, non_uniform_time_grid, non_uniform_dt) - - def _test_integrate_gaussian_all(self, method): - uniform_time_grid = np.linspace(0., 2., 100) - non_uniform_time_grid = np.asarray([0.0, 0.1, 0.7, 1.2, 2.0]) - uniform_dt = 0.01 - non_uniform_dt = np.asarray([0.01, 0.001, 0.1, 0.03]) - self._test_integrate_gaussian(method, uniform_time_grid) - self._test_integrate_gaussian(method, non_uniform_time_grid, uniform_dt) - self._test_integrate_gaussian(method, non_uniform_time_grid, non_uniform_dt) - - def _test_everything(self, method): - self._test_integrate_sine_all(method) - self._test_integrate_gaussian_all(method) - - def test_midpoint(self): - self._test_everything('midpoint') - - def test_rk4(self): - self._test_everything('rk4') - - def test_dt_size_exceptions(self): - times = np.linspace(0., 2., 100) - dt = np.ones(99) * 0.01 - dt_wrong_length = np.asarray([0.01, 0.001, 0.1, 0.03]) - dt_wrong_dim = np.expand_dims(np.linspace(0., 2., 99), axis=0) - times_wrong_dim = np.expand_dims(np.linspace(0., 2., 100), axis=0) - with self.assertRaises(ValueError): - self._test_integrate_gaussian('midpoint', times, dt_wrong_length) - - with self.assertRaises(ValueError): - self._test_integrate_gaussian('midpoint', times, dt_wrong_dim) - - with self.assertRaises(ValueError): - self._test_integrate_gaussian('midpoint', times_wrong_dim, dt) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/keras/BUILD b/tensorflow/contrib/keras/BUILD deleted file mode 100644 index a839693340e..00000000000 --- a/tensorflow/contrib/keras/BUILD +++ /dev/null @@ -1,55 +0,0 @@ -# Description: -# Contains the Keras API (internal TensorFlow version). -# Note that tf.contrib.keras has been deprecated in favor of tf.keras. - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "keras", - srcs = [ - "__init__.py", - "api/__init__.py", - "api/keras/__init__.py", - "api/keras/activations/__init__.py", - "api/keras/applications/__init__.py", - "api/keras/applications/inception_v3/__init__.py", - "api/keras/applications/mobilenet/__init__.py", - "api/keras/applications/resnet50/__init__.py", - "api/keras/applications/vgg16/__init__.py", - "api/keras/applications/vgg19/__init__.py", - "api/keras/applications/xception/__init__.py", - "api/keras/backend/__init__.py", - "api/keras/callbacks/__init__.py", - "api/keras/constraints/__init__.py", - "api/keras/datasets/__init__.py", - "api/keras/datasets/boston_housing/__init__.py", - "api/keras/datasets/cifar10/__init__.py", - "api/keras/datasets/cifar100/__init__.py", - "api/keras/datasets/imdb/__init__.py", - "api/keras/datasets/mnist/__init__.py", - "api/keras/datasets/reuters/__init__.py", - "api/keras/initializers/__init__.py", - "api/keras/layers/__init__.py", - "api/keras/losses/__init__.py", - "api/keras/metrics/__init__.py", - "api/keras/models/__init__.py", - "api/keras/optimizers/__init__.py", - "api/keras/preprocessing/__init__.py", - "api/keras/preprocessing/image/__init__.py", - "api/keras/preprocessing/sequence/__init__.py", - "api/keras/preprocessing/text/__init__.py", - "api/keras/regularizers/__init__.py", - "api/keras/utils/__init__.py", - "api/keras/wrappers/__init__.py", - "api/keras/wrappers/scikit_learn/__init__.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/keras", - ], -) diff --git a/tensorflow/contrib/keras/README.md b/tensorflow/contrib/keras/README.md deleted file mode 100644 index de4c81268d5..00000000000 --- a/tensorflow/contrib/keras/README.md +++ /dev/null @@ -1,9 +0,0 @@ -NOTE: THE `tensorflow.contrib.keras` MODULE HAS BEEN DEPRECATED. -USE INSTEAD `tensorflow.keras`, PART OF CORE TENSORFLOW. - -Keras is an object-oriented API for defining and training neural networks. - -This module contains a pure-TensorFlow implementation of the Keras API, -allowing for deep integration with TensorFlow functionality. - -See [keras.io](https://keras.io) for complete documentation and user guides. diff --git a/tensorflow/contrib/keras/__init__.py b/tensorflow/contrib/keras/__init__.py deleted file mode 100644 index cecf1ddcdb1..00000000000 --- a/tensorflow/contrib/keras/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implementation of the Keras API meant to be a high-level API for TensorFlow. - -This module an alias for `tf.keras`, for backwards compatibility. - -Detailed documentation and user guides are also available at -[keras.io](https://keras.io). -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.keras.api.keras import * - -try: - from tensorflow.contrib.keras import python # pylint: disable=g-import-not-at-top - del python -except ImportError: - pass - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/__init__.py b/tensorflow/contrib/keras/api/__init__.py deleted file mode 100644 index 52e83069cb0..00000000000 --- a/tensorflow/contrib/keras/api/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/keras/api/keras/__init__.py b/tensorflow/contrib/keras/api/keras/__init__.py deleted file mode 100644 index 53fb4a30c9b..00000000000 --- a/tensorflow/contrib/keras/api/keras/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implementation of the Keras API meant to be a high-level API for TensorFlow. - -Detailed documentation and user guides are available at -[keras.io](https://keras.io). -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.keras.api.keras import activations -from tensorflow.contrib.keras.api.keras import applications -from tensorflow.contrib.keras.api.keras import backend -from tensorflow.contrib.keras.api.keras import callbacks -from tensorflow.contrib.keras.api.keras import constraints -from tensorflow.contrib.keras.api.keras import datasets -from tensorflow.contrib.keras.api.keras import initializers -from tensorflow.contrib.keras.api.keras import layers -from tensorflow.contrib.keras.api.keras import losses -from tensorflow.contrib.keras.api.keras import metrics -from tensorflow.contrib.keras.api.keras import models -from tensorflow.contrib.keras.api.keras import optimizers -from tensorflow.contrib.keras.api.keras import preprocessing -from tensorflow.contrib.keras.api.keras import regularizers -from tensorflow.contrib.keras.api.keras import utils -from tensorflow.contrib.keras.api.keras import wrappers - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/activations/__init__.py b/tensorflow/contrib/keras/api/keras/activations/__init__.py deleted file mode 100644 index 3f0184276f6..00000000000 --- a/tensorflow/contrib/keras/api/keras/activations/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras built-in activation functions.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# Activation functions. -from tensorflow.python.keras.activations import elu -from tensorflow.python.keras.activations import hard_sigmoid -from tensorflow.python.keras.activations import linear -from tensorflow.python.keras.activations import relu -from tensorflow.python.keras.activations import selu -from tensorflow.python.keras.activations import sigmoid -from tensorflow.python.keras.activations import softmax -from tensorflow.python.keras.activations import softplus -from tensorflow.python.keras.activations import softsign -from tensorflow.python.keras.activations import tanh - -# Auxiliary utils. -# pylint: disable=g-bad-import-order -from tensorflow.python.keras.activations import deserialize -from tensorflow.python.keras.activations import serialize -from tensorflow.python.keras.activations import get - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/applications/__init__.py b/tensorflow/contrib/keras/api/keras/applications/__init__.py deleted file mode 100644 index f943e84606b..00000000000 --- a/tensorflow/contrib/keras/api/keras/applications/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras Applications are canned architectures with pre-trained weights.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.keras.api.keras.applications import inception_v3 -from tensorflow.contrib.keras.api.keras.applications import mobilenet -from tensorflow.contrib.keras.api.keras.applications import resnet50 -from tensorflow.contrib.keras.api.keras.applications import vgg16 -from tensorflow.contrib.keras.api.keras.applications import vgg19 -from tensorflow.contrib.keras.api.keras.applications import xception -from tensorflow.contrib.keras.api.keras.applications.inception_v3 import InceptionV3 -from tensorflow.contrib.keras.api.keras.applications.mobilenet import MobileNet -from tensorflow.contrib.keras.api.keras.applications.resnet50 import ResNet50 -from tensorflow.contrib.keras.api.keras.applications.vgg16 import VGG16 -from tensorflow.contrib.keras.api.keras.applications.vgg19 import VGG19 -from tensorflow.contrib.keras.api.keras.applications.xception import Xception - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/applications/inception_v3/__init__.py b/tensorflow/contrib/keras/api/keras/applications/inception_v3/__init__.py deleted file mode 100644 index 6dfb5cab17c..00000000000 --- a/tensorflow/contrib/keras/api/keras/applications/inception_v3/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Inception V3 Keras application.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.applications.inception_v3 import decode_predictions -from tensorflow.python.keras.applications.inception_v3 import InceptionV3 -from tensorflow.python.keras.applications.inception_v3 import preprocess_input - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/applications/mobilenet/__init__.py b/tensorflow/contrib/keras/api/keras/applications/mobilenet/__init__.py deleted file mode 100644 index 67306cc51e1..00000000000 --- a/tensorflow/contrib/keras/api/keras/applications/mobilenet/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""MobileNet Keras application.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.applications.mobilenet import decode_predictions -from tensorflow.python.keras.applications.mobilenet import MobileNet -from tensorflow.python.keras.applications.mobilenet import preprocess_input - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/applications/resnet50/__init__.py b/tensorflow/contrib/keras/api/keras/applications/resnet50/__init__.py deleted file mode 100644 index 8a7540ed40c..00000000000 --- a/tensorflow/contrib/keras/api/keras/applications/resnet50/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ResNet50 Keras application.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.applications.resnet import decode_predictions -from tensorflow.python.keras.applications.resnet import preprocess_input -from tensorflow.python.keras.applications.resnet import ResNet50 - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/applications/vgg16/__init__.py b/tensorflow/contrib/keras/api/keras/applications/vgg16/__init__.py deleted file mode 100644 index 4964b1b7deb..00000000000 --- a/tensorflow/contrib/keras/api/keras/applications/vgg16/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""VGG16 Keras application.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.applications.vgg16 import decode_predictions -from tensorflow.python.keras.applications.vgg16 import preprocess_input -from tensorflow.python.keras.applications.vgg16 import VGG16 - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/applications/vgg19/__init__.py b/tensorflow/contrib/keras/api/keras/applications/vgg19/__init__.py deleted file mode 100644 index afb3abebdd6..00000000000 --- a/tensorflow/contrib/keras/api/keras/applications/vgg19/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""VGG19 Keras application.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.applications.vgg19 import decode_predictions -from tensorflow.python.keras.applications.vgg19 import preprocess_input -from tensorflow.python.keras.applications.vgg19 import VGG19 - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/applications/xception/__init__.py b/tensorflow/contrib/keras/api/keras/applications/xception/__init__.py deleted file mode 100644 index 2e3335d02af..00000000000 --- a/tensorflow/contrib/keras/api/keras/applications/xception/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Xception Keras application.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.applications.xception import decode_predictions -from tensorflow.python.keras.applications.xception import preprocess_input -from tensorflow.python.keras.applications.xception import Xception - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/backend/__init__.py b/tensorflow/contrib/keras/api/keras/backend/__init__.py deleted file mode 100644 index a7553640142..00000000000 --- a/tensorflow/contrib/keras/api/keras/backend/__init__.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras backend API.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=redefined-builtin -from tensorflow.python.keras.backend import abs -from tensorflow.python.keras.backend import all -from tensorflow.python.keras.backend import any -from tensorflow.python.keras.backend import arange -from tensorflow.python.keras.backend import argmax -from tensorflow.python.keras.backend import argmin -from tensorflow.python.keras.backend import backend -from tensorflow.python.keras.backend import batch_dot -from tensorflow.python.keras.backend import batch_flatten -from tensorflow.python.keras.backend import batch_get_value -from tensorflow.python.keras.backend import batch_normalization -from tensorflow.python.keras.backend import batch_set_value -from tensorflow.python.keras.backend import bias_add -from tensorflow.python.keras.backend import binary_crossentropy -from tensorflow.python.keras.backend import cast -from tensorflow.python.keras.backend import cast_to_floatx -from tensorflow.python.keras.backend import categorical_crossentropy -from tensorflow.python.keras.backend import clear_session -from tensorflow.python.keras.backend import clip -from tensorflow.python.keras.backend import concatenate -from tensorflow.python.keras.backend import constant -from tensorflow.python.keras.backend import conv1d -from tensorflow.python.keras.backend import conv2d -from tensorflow.python.keras.backend import conv2d_transpose -from tensorflow.python.keras.backend import conv3d -from tensorflow.python.keras.backend import cos -from tensorflow.python.keras.backend import count_params -from tensorflow.python.keras.backend import ctc_batch_cost -from tensorflow.python.keras.backend import ctc_decode -from tensorflow.python.keras.backend import ctc_label_dense_to_sparse -from tensorflow.python.keras.backend import dot -from tensorflow.python.keras.backend import dropout -from tensorflow.python.keras.backend import dtype -from tensorflow.python.keras.backend import elu -from tensorflow.python.keras.backend import epsilon -from tensorflow.python.keras.backend import equal -from tensorflow.python.keras.backend import eval -from tensorflow.python.keras.backend import exp -from tensorflow.python.keras.backend import expand_dims -from tensorflow.python.keras.backend import eye -from tensorflow.python.keras.backend import flatten -from tensorflow.python.keras.backend import floatx -from tensorflow.python.keras.backend import foldl -from tensorflow.python.keras.backend import foldr -from tensorflow.python.keras.backend import function -from tensorflow.python.keras.backend import gather -from tensorflow.python.keras.backend import get_session -from tensorflow.python.keras.backend import get_uid -from tensorflow.python.keras.backend import get_value -from tensorflow.python.keras.backend import gradients -from tensorflow.python.keras.backend import greater -from tensorflow.python.keras.backend import greater_equal -from tensorflow.python.keras.backend import hard_sigmoid -from tensorflow.python.keras.backend import image_data_format -from tensorflow.python.keras.backend import in_test_phase -from tensorflow.python.keras.backend import in_top_k -from tensorflow.python.keras.backend import in_train_phase -from tensorflow.python.keras.backend import int_shape -from tensorflow.python.keras.backend import is_sparse -from tensorflow.python.keras.backend import l2_normalize -from tensorflow.python.keras.backend import learning_phase -from tensorflow.python.keras.backend import less -from tensorflow.python.keras.backend import less_equal -from tensorflow.python.keras.backend import log -from tensorflow.python.keras.backend import manual_variable_initialization -from tensorflow.python.keras.backend import map_fn -from tensorflow.python.keras.backend import max -from tensorflow.python.keras.backend import maximum -from tensorflow.python.keras.backend import mean -from tensorflow.python.keras.backend import min -from tensorflow.python.keras.backend import minimum -from tensorflow.python.keras.backend import moving_average_update -from tensorflow.python.keras.backend import name_scope -from tensorflow.python.keras.backend import ndim -from tensorflow.python.keras.backend import normalize_batch_in_training -from tensorflow.python.keras.backend import not_equal -from tensorflow.python.keras.backend import one_hot -from tensorflow.python.keras.backend import ones -from tensorflow.python.keras.backend import ones_like -from tensorflow.python.keras.backend import permute_dimensions -from tensorflow.python.keras.backend import placeholder -from tensorflow.python.keras.backend import pool2d -from tensorflow.python.keras.backend import pool3d -from tensorflow.python.keras.backend import pow -from tensorflow.python.keras.backend import print_tensor -from tensorflow.python.keras.backend import prod -from tensorflow.python.keras.backend import random_binomial -from tensorflow.python.keras.backend import random_normal -from tensorflow.python.keras.backend import random_normal_variable -from tensorflow.python.keras.backend import random_uniform -from tensorflow.python.keras.backend import random_uniform_variable -from tensorflow.python.keras.backend import relu -from tensorflow.python.keras.backend import repeat -from tensorflow.python.keras.backend import repeat_elements -from tensorflow.python.keras.backend import reset_uids -from tensorflow.python.keras.backend import reshape -from tensorflow.python.keras.backend import resize_images -from tensorflow.python.keras.backend import resize_volumes -from tensorflow.python.keras.backend import reverse -from tensorflow.python.keras.backend import rnn -from tensorflow.python.keras.backend import round -from tensorflow.python.keras.backend import separable_conv2d -from tensorflow.python.keras.backend import set_epsilon -from tensorflow.python.keras.backend import set_floatx -from tensorflow.python.keras.backend import set_image_data_format -from tensorflow.python.keras.backend import set_learning_phase -from tensorflow.python.keras.backend import set_session -from tensorflow.python.keras.backend import set_value -from tensorflow.python.keras.backend import shape -from tensorflow.python.keras.backend import sigmoid -from tensorflow.python.keras.backend import sign -from tensorflow.python.keras.backend import sin -from tensorflow.python.keras.backend import softmax -from tensorflow.python.keras.backend import softplus -from tensorflow.python.keras.backend import softsign -from tensorflow.python.keras.backend import sparse_categorical_crossentropy -from tensorflow.python.keras.backend import spatial_2d_padding -from tensorflow.python.keras.backend import spatial_3d_padding -from tensorflow.python.keras.backend import sqrt -from tensorflow.python.keras.backend import square -from tensorflow.python.keras.backend import squeeze -from tensorflow.python.keras.backend import stack -from tensorflow.python.keras.backend import std -from tensorflow.python.keras.backend import stop_gradient -from tensorflow.python.keras.backend import sum -from tensorflow.python.keras.backend import switch -from tensorflow.python.keras.backend import tanh -from tensorflow.python.keras.backend import temporal_padding -from tensorflow.python.keras.backend import to_dense -from tensorflow.python.keras.backend import transpose -from tensorflow.python.keras.backend import truncated_normal -from tensorflow.python.keras.backend import update -from tensorflow.python.keras.backend import update_add -from tensorflow.python.keras.backend import update_sub -from tensorflow.python.keras.backend import var -from tensorflow.python.keras.backend import variable -from tensorflow.python.keras.backend import zeros -from tensorflow.python.keras.backend import zeros_like - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/callbacks/__init__.py b/tensorflow/contrib/keras/api/keras/callbacks/__init__.py deleted file mode 100644 index 10e05f2969b..00000000000 --- a/tensorflow/contrib/keras/api/keras/callbacks/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras callback classes.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.callbacks import BaseLogger -from tensorflow.python.keras.callbacks import Callback -from tensorflow.python.keras.callbacks import CSVLogger -from tensorflow.python.keras.callbacks import EarlyStopping -from tensorflow.python.keras.callbacks import History -from tensorflow.python.keras.callbacks import LambdaCallback -from tensorflow.python.keras.callbacks import LearningRateScheduler -from tensorflow.python.keras.callbacks import ModelCheckpoint -from tensorflow.python.keras.callbacks import ProgbarLogger -from tensorflow.python.keras.callbacks import ReduceLROnPlateau -from tensorflow.python.keras.callbacks import RemoteMonitor -from tensorflow.python.keras.callbacks import TensorBoard -from tensorflow.python.keras.callbacks import TerminateOnNaN - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/constraints/__init__.py b/tensorflow/contrib/keras/api/keras/constraints/__init__.py deleted file mode 100644 index 08debf974ec..00000000000 --- a/tensorflow/contrib/keras/api/keras/constraints/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras built-in constraints functions.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# Constraints functions / callable classes. -from tensorflow.python.keras.constraints import Constraint -from tensorflow.python.keras.constraints import max_norm -from tensorflow.python.keras.constraints import MaxNorm -from tensorflow.python.keras.constraints import min_max_norm -from tensorflow.python.keras.constraints import MinMaxNorm -from tensorflow.python.keras.constraints import non_neg -from tensorflow.python.keras.constraints import NonNeg -from tensorflow.python.keras.constraints import unit_norm -from tensorflow.python.keras.constraints import UnitNorm - -# Auxiliary utils. -# pylint: disable=g-bad-import-order -from tensorflow.python.keras.constraints import deserialize -from tensorflow.python.keras.constraints import serialize -from tensorflow.python.keras.constraints import get - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/datasets/__init__.py b/tensorflow/contrib/keras/api/keras/datasets/__init__.py deleted file mode 100644 index 4513231bb47..00000000000 --- a/tensorflow/contrib/keras/api/keras/datasets/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras built-in datasets.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.keras.api.keras.datasets import boston_housing -from tensorflow.contrib.keras.api.keras.datasets import cifar10 -from tensorflow.contrib.keras.api.keras.datasets import cifar100 -from tensorflow.contrib.keras.api.keras.datasets import imdb -from tensorflow.contrib.keras.api.keras.datasets import mnist -from tensorflow.contrib.keras.api.keras.datasets import reuters - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/datasets/boston_housing/__init__.py b/tensorflow/contrib/keras/api/keras/datasets/boston_housing/__init__.py deleted file mode 100644 index a5a6fdab445..00000000000 --- a/tensorflow/contrib/keras/api/keras/datasets/boston_housing/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Boston housing price regression dataset.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.datasets.boston_housing import load_data - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/datasets/cifar10/__init__.py b/tensorflow/contrib/keras/api/keras/datasets/cifar10/__init__.py deleted file mode 100644 index e74e5f347df..00000000000 --- a/tensorflow/contrib/keras/api/keras/datasets/cifar10/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""CIFAR10 small image classification dataset.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.datasets.cifar10 import load_data - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/datasets/cifar100/__init__.py b/tensorflow/contrib/keras/api/keras/datasets/cifar100/__init__.py deleted file mode 100644 index 8f5753a6360..00000000000 --- a/tensorflow/contrib/keras/api/keras/datasets/cifar100/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""CIFAR100 small image classification dataset.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.datasets.cifar100 import load_data - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/datasets/imdb/__init__.py b/tensorflow/contrib/keras/api/keras/datasets/imdb/__init__.py deleted file mode 100644 index bd6ec4b8dfb..00000000000 --- a/tensorflow/contrib/keras/api/keras/datasets/imdb/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""IMDB movie review sentiment classification dataset.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.datasets.imdb import get_word_index -from tensorflow.python.keras.datasets.imdb import load_data - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/datasets/mnist/__init__.py b/tensorflow/contrib/keras/api/keras/datasets/mnist/__init__.py deleted file mode 100644 index f61145655bd..00000000000 --- a/tensorflow/contrib/keras/api/keras/datasets/mnist/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""MNIST handwritten digits classification dataset.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.datasets.mnist import load_data - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/datasets/reuters/__init__.py b/tensorflow/contrib/keras/api/keras/datasets/reuters/__init__.py deleted file mode 100644 index ade31f4ea9c..00000000000 --- a/tensorflow/contrib/keras/api/keras/datasets/reuters/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Reuters newswire topic classification dataset.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.datasets.reuters import get_word_index -from tensorflow.python.keras.datasets.reuters import load_data - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/initializers/__init__.py b/tensorflow/contrib/keras/api/keras/initializers/__init__.py deleted file mode 100644 index c6bdc4f0dac..00000000000 --- a/tensorflow/contrib/keras/api/keras/initializers/__init__.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras built-in initializers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# Initializer functions / callable classes. -from tensorflow.python.keras.initializers import Constant -from tensorflow.python.keras.initializers import Identity -from tensorflow.python.keras.initializers import Initializer -from tensorflow.python.keras.initializers import Ones -from tensorflow.python.keras.initializers import Orthogonal -from tensorflow.python.keras.initializers import RandomNormal -from tensorflow.python.keras.initializers import RandomUniform -from tensorflow.python.keras.initializers import TruncatedNormal -from tensorflow.python.keras.initializers import VarianceScaling -from tensorflow.python.keras.initializers import Zeros - -# Functional interface. -# pylint: disable=g-bad-import-order -from tensorflow.python.keras.initializers import glorot_normal -from tensorflow.python.keras.initializers import glorot_uniform -from tensorflow.python.keras.initializers import he_normal -from tensorflow.python.keras.initializers import he_uniform -from tensorflow.python.keras.initializers import lecun_normal -from tensorflow.python.keras.initializers import lecun_uniform - -# Auxiliary utils. -from tensorflow.python.keras.initializers import deserialize -from tensorflow.python.keras.initializers import serialize -from tensorflow.python.keras.initializers import get - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/layers/__init__.py b/tensorflow/contrib/keras/api/keras/layers/__init__.py deleted file mode 100644 index 9e19884df85..00000000000 --- a/tensorflow/contrib/keras/api/keras/layers/__init__.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras layers API.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# Generic layers. -# pylint: disable=g-bad-import-order -from tensorflow.python.keras.engine.input_spec import InputSpec -from tensorflow.python.keras.engine.base_layer import Layer -from tensorflow.python.keras.engine.input_layer import Input -from tensorflow.python.keras.engine.input_layer import InputLayer - -# Advanced activations. -from tensorflow.python.keras.layers.advanced_activations import LeakyReLU -from tensorflow.python.keras.layers.advanced_activations import PReLU -from tensorflow.python.keras.layers.advanced_activations import ELU -from tensorflow.python.keras.layers.advanced_activations import ThresholdedReLU - -# Convolution layers. -from tensorflow.python.keras.layers.convolutional import Conv1D -from tensorflow.python.keras.layers.convolutional import Conv2D -from tensorflow.python.keras.layers.convolutional import Conv3D -from tensorflow.python.keras.layers.convolutional import Conv2DTranspose -from tensorflow.python.keras.layers.convolutional import Conv3DTranspose -from tensorflow.python.keras.layers.convolutional import SeparableConv2D - -# Convolution layer aliases. -from tensorflow.python.keras.layers.convolutional import Convolution1D -from tensorflow.python.keras.layers.convolutional import Convolution2D -from tensorflow.python.keras.layers.convolutional import Convolution3D -from tensorflow.python.keras.layers.convolutional import Convolution2DTranspose -from tensorflow.python.keras.layers.convolutional import Convolution3DTranspose -from tensorflow.python.keras.layers.convolutional import SeparableConvolution2D - -# Image processing layers. -from tensorflow.python.keras.layers.convolutional import UpSampling1D -from tensorflow.python.keras.layers.convolutional import UpSampling2D -from tensorflow.python.keras.layers.convolutional import UpSampling3D -from tensorflow.python.keras.layers.convolutional import ZeroPadding1D -from tensorflow.python.keras.layers.convolutional import ZeroPadding2D -from tensorflow.python.keras.layers.convolutional import ZeroPadding3D -from tensorflow.python.keras.layers.convolutional import Cropping1D -from tensorflow.python.keras.layers.convolutional import Cropping2D -from tensorflow.python.keras.layers.convolutional import Cropping3D - -# Convolutional-recurrent layers. -from tensorflow.python.keras.layers.convolutional_recurrent import ConvLSTM2D - -# Core layers. -from tensorflow.python.keras.layers.core import Masking -from tensorflow.python.keras.layers.core import Dropout -from tensorflow.python.keras.layers.core import SpatialDropout1D -from tensorflow.python.keras.layers.core import SpatialDropout2D -from tensorflow.python.keras.layers.core import SpatialDropout3D -from tensorflow.python.keras.layers.core import Activation -from tensorflow.python.keras.layers.core import Reshape -from tensorflow.python.keras.layers.core import Permute -from tensorflow.python.keras.layers.core import Flatten -from tensorflow.python.keras.layers.core import RepeatVector -from tensorflow.python.keras.layers.core import Lambda -from tensorflow.python.keras.layers.core import Dense -from tensorflow.python.keras.layers.core import ActivityRegularization - -# Embedding layers. -from tensorflow.python.keras.layers.embeddings import Embedding - -# Locally-connected layers. -from tensorflow.python.keras.layers.local import LocallyConnected1D -from tensorflow.python.keras.layers.local import LocallyConnected2D - -# Merge layers. -from tensorflow.python.keras.layers.merge import Add -from tensorflow.python.keras.layers.merge import Multiply -from tensorflow.python.keras.layers.merge import Average -from tensorflow.python.keras.layers.merge import Maximum -from tensorflow.python.keras.layers.merge import Concatenate -from tensorflow.python.keras.layers.merge import Dot -from tensorflow.python.keras.layers.merge import add -from tensorflow.python.keras.layers.merge import multiply -from tensorflow.python.keras.layers.merge import average -from tensorflow.python.keras.layers.merge import maximum -from tensorflow.python.keras.layers.merge import concatenate -from tensorflow.python.keras.layers.merge import dot - -# Noise layers. -from tensorflow.python.keras.layers.noise import AlphaDropout -from tensorflow.python.keras.layers.noise import GaussianNoise -from tensorflow.python.keras.layers.noise import GaussianDropout - -# Normalization layers. -from tensorflow.python.keras.layers.normalization import BatchNormalization - -# Pooling layers. -from tensorflow.python.keras.layers.pooling import MaxPooling1D -from tensorflow.python.keras.layers.pooling import MaxPooling2D -from tensorflow.python.keras.layers.pooling import MaxPooling3D -from tensorflow.python.keras.layers.pooling import AveragePooling1D -from tensorflow.python.keras.layers.pooling import AveragePooling2D -from tensorflow.python.keras.layers.pooling import AveragePooling3D -from tensorflow.python.keras.layers.pooling import GlobalAveragePooling1D -from tensorflow.python.keras.layers.pooling import GlobalAveragePooling2D -from tensorflow.python.keras.layers.pooling import GlobalAveragePooling3D -from tensorflow.python.keras.layers.pooling import GlobalMaxPooling1D -from tensorflow.python.keras.layers.pooling import GlobalMaxPooling2D -from tensorflow.python.keras.layers.pooling import GlobalMaxPooling3D - -# Pooling layer aliases. -from tensorflow.python.keras.layers.pooling import MaxPool1D -from tensorflow.python.keras.layers.pooling import MaxPool2D -from tensorflow.python.keras.layers.pooling import MaxPool3D -from tensorflow.python.keras.layers.pooling import AvgPool1D -from tensorflow.python.keras.layers.pooling import AvgPool2D -from tensorflow.python.keras.layers.pooling import AvgPool3D -from tensorflow.python.keras.layers.pooling import GlobalAvgPool1D -from tensorflow.python.keras.layers.pooling import GlobalAvgPool2D -from tensorflow.python.keras.layers.pooling import GlobalAvgPool3D -from tensorflow.python.keras.layers.pooling import GlobalMaxPool1D -from tensorflow.python.keras.layers.pooling import GlobalMaxPool2D -from tensorflow.python.keras.layers.pooling import GlobalMaxPool3D - -# Recurrent layers. -from tensorflow.python.keras.layers.recurrent import SimpleRNN -from tensorflow.python.keras.layers.recurrent import GRU -from tensorflow.python.keras.layers.recurrent import LSTM - -# Wrapper functions -from tensorflow.python.keras.layers.wrappers import Wrapper -from tensorflow.python.keras.layers.wrappers import Bidirectional -from tensorflow.python.keras.layers.wrappers import TimeDistributed - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/losses/__init__.py b/tensorflow/contrib/keras/api/keras/losses/__init__.py deleted file mode 100644 index b12832d2e2a..00000000000 --- a/tensorflow/contrib/keras/api/keras/losses/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras built-in loss functions.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# Loss functions. -from tensorflow.python.keras.losses import binary_crossentropy -from tensorflow.python.keras.losses import categorical_crossentropy -from tensorflow.python.keras.losses import categorical_hinge -from tensorflow.python.keras.losses import cosine_similarity -from tensorflow.python.keras.losses import hinge -from tensorflow.python.keras.losses import kullback_leibler_divergence -from tensorflow.python.keras.losses import logcosh -from tensorflow.python.keras.losses import mean_absolute_error -from tensorflow.python.keras.losses import mean_absolute_percentage_error -from tensorflow.python.keras.losses import mean_squared_error -from tensorflow.python.keras.losses import mean_squared_logarithmic_error -from tensorflow.python.keras.losses import poisson -from tensorflow.python.keras.losses import sparse_categorical_crossentropy -from tensorflow.python.keras.losses import squared_hinge - -# Auxiliary utils. -# pylint: disable=g-bad-import-order -from tensorflow.python.keras.losses import deserialize -from tensorflow.python.keras.losses import serialize -from tensorflow.python.keras.losses import get - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/metrics/__init__.py b/tensorflow/contrib/keras/api/keras/metrics/__init__.py deleted file mode 100644 index 095b5d798df..00000000000 --- a/tensorflow/contrib/keras/api/keras/metrics/__init__.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras built-in metrics functions.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# Metrics functions. -from tensorflow.python.keras.metrics import binary_accuracy -from tensorflow.python.keras.metrics import binary_crossentropy -from tensorflow.python.keras.metrics import categorical_accuracy -from tensorflow.python.keras.metrics import categorical_crossentropy -from tensorflow.python.keras.metrics import cosine_similarity -from tensorflow.python.keras.metrics import hinge -from tensorflow.python.keras.metrics import kullback_leibler_divergence -from tensorflow.python.keras.metrics import mean_absolute_error -from tensorflow.python.keras.metrics import mean_absolute_percentage_error -from tensorflow.python.keras.metrics import mean_squared_error -from tensorflow.python.keras.metrics import mean_squared_logarithmic_error -from tensorflow.python.keras.metrics import poisson -from tensorflow.python.keras.metrics import sparse_categorical_crossentropy -from tensorflow.python.keras.metrics import sparse_top_k_categorical_accuracy -from tensorflow.python.keras.metrics import squared_hinge -from tensorflow.python.keras.metrics import top_k_categorical_accuracy - -# Auxiliary utils. -# pylint: disable=g-bad-import-order -from tensorflow.python.keras.metrics import deserialize -from tensorflow.python.keras.metrics import serialize -from tensorflow.python.keras.metrics import get - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/models/__init__.py b/tensorflow/contrib/keras/api/keras/models/__init__.py deleted file mode 100644 index 3a196984cd8..00000000000 --- a/tensorflow/contrib/keras/api/keras/models/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras models API.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.models import load_model -from tensorflow.python.keras.models import Model -from tensorflow.python.keras.models import model_from_config -from tensorflow.python.keras.models import model_from_json -from tensorflow.python.keras.models import model_from_yaml -from tensorflow.python.keras.models import save_model -from tensorflow.python.keras.models import Sequential - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/optimizers/__init__.py b/tensorflow/contrib/keras/api/keras/optimizers/__init__.py deleted file mode 100644 index 4849a067479..00000000000 --- a/tensorflow/contrib/keras/api/keras/optimizers/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras built-in optimizers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# Optimizer classes. -from tensorflow.python.keras.optimizers import Adadelta -from tensorflow.python.keras.optimizers import Adagrad -from tensorflow.python.keras.optimizers import Adam -from tensorflow.python.keras.optimizers import Adamax -from tensorflow.python.keras.optimizers import Nadam -from tensorflow.python.keras.optimizers import Optimizer -from tensorflow.python.keras.optimizers import RMSprop -from tensorflow.python.keras.optimizers import SGD - -# Auxiliary utils. -# pylint: disable=g-bad-import-order -from tensorflow.python.keras.optimizers import deserialize -from tensorflow.python.keras.optimizers import serialize -from tensorflow.python.keras.optimizers import get - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/preprocessing/__init__.py b/tensorflow/contrib/keras/api/keras/preprocessing/__init__.py deleted file mode 100644 index 4a200e3f584..00000000000 --- a/tensorflow/contrib/keras/api/keras/preprocessing/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras data preprocessing utils.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.keras.api.keras.preprocessing import image -from tensorflow.contrib.keras.api.keras.preprocessing import sequence -from tensorflow.contrib.keras.api.keras.preprocessing import text - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/preprocessing/image/__init__.py b/tensorflow/contrib/keras/api/keras/preprocessing/image/__init__.py deleted file mode 100644 index cb649a37510..00000000000 --- a/tensorflow/contrib/keras/api/keras/preprocessing/image/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras data preprocessing utils for image data.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.preprocessing.image import array_to_img -from tensorflow.python.keras.preprocessing.image import DirectoryIterator -from tensorflow.python.keras.preprocessing.image import ImageDataGenerator -from tensorflow.python.keras.preprocessing.image import img_to_array -from tensorflow.python.keras.preprocessing.image import Iterator -from tensorflow.python.keras.preprocessing.image import load_img -from tensorflow.python.keras.preprocessing.image import NumpyArrayIterator -from tensorflow.python.keras.preprocessing.image import random_channel_shift -from tensorflow.python.keras.preprocessing.image import random_rotation -from tensorflow.python.keras.preprocessing.image import random_shear -from tensorflow.python.keras.preprocessing.image import random_shift -from tensorflow.python.keras.preprocessing.image import random_zoom - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/preprocessing/sequence/__init__.py b/tensorflow/contrib/keras/api/keras/preprocessing/sequence/__init__.py deleted file mode 100644 index 9a93b6fb57f..00000000000 --- a/tensorflow/contrib/keras/api/keras/preprocessing/sequence/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras data preprocessing utils for sequence data.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.preprocessing.sequence import make_sampling_table -from tensorflow.python.keras.preprocessing.sequence import pad_sequences -from tensorflow.python.keras.preprocessing.sequence import skipgrams - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/preprocessing/text/__init__.py b/tensorflow/contrib/keras/api/keras/preprocessing/text/__init__.py deleted file mode 100644 index 86386a9b676..00000000000 --- a/tensorflow/contrib/keras/api/keras/preprocessing/text/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras data preprocessing utils for text data.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.preprocessing.text import one_hot -from tensorflow.python.keras.preprocessing.text import text_to_word_sequence -from tensorflow.python.keras.preprocessing.text import Tokenizer - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/regularizers/__init__.py b/tensorflow/contrib/keras/api/keras/regularizers/__init__.py deleted file mode 100644 index d668e39c09c..00000000000 --- a/tensorflow/contrib/keras/api/keras/regularizers/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras built-in regularizers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# Regularizer functions / callable classes. -from tensorflow.python.keras.regularizers import L1L2 -from tensorflow.python.keras.regularizers import Regularizer - -# Functional interface. -# pylint: disable=g-bad-import-order -from tensorflow.python.keras.regularizers import l1 -from tensorflow.python.keras.regularizers import l2 -from tensorflow.python.keras.regularizers import l1_l2 - -# Auxiliary utils. -from tensorflow.python.keras.regularizers import deserialize -from tensorflow.python.keras.regularizers import serialize -from tensorflow.python.keras.regularizers import get - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/utils/__init__.py b/tensorflow/contrib/keras/api/keras/utils/__init__.py deleted file mode 100644 index 47cd01b924f..00000000000 --- a/tensorflow/contrib/keras/api/keras/utils/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras utilities.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.utils.data_utils import GeneratorEnqueuer -from tensorflow.python.keras.utils.data_utils import get_file -from tensorflow.python.keras.utils.data_utils import Sequence -from tensorflow.python.keras.utils.data_utils import SequenceEnqueuer -from tensorflow.python.keras.utils.generic_utils import custom_object_scope -from tensorflow.python.keras.utils.generic_utils import CustomObjectScope -from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object -from tensorflow.python.keras.utils.generic_utils import get_custom_objects -from tensorflow.python.keras.utils.generic_utils import Progbar -from tensorflow.python.keras.utils.generic_utils import serialize_keras_object -from tensorflow.python.keras.utils.io_utils import HDF5Matrix -from tensorflow.python.keras.utils.layer_utils import convert_all_kernels_in_model -from tensorflow.python.keras.utils.np_utils import normalize -from tensorflow.python.keras.utils.np_utils import to_categorical -from tensorflow.python.keras.utils.vis_utils import plot_model - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/wrappers/__init__.py b/tensorflow/contrib/keras/api/keras/wrappers/__init__.py deleted file mode 100644 index d2c7c4bf144..00000000000 --- a/tensorflow/contrib/keras/api/keras/wrappers/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Wrappers for Keras models, providing compatibility with other frameworks.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.keras.api.keras.wrappers import scikit_learn - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/keras/api/keras/wrappers/scikit_learn/__init__.py b/tensorflow/contrib/keras/api/keras/wrappers/scikit_learn/__init__.py deleted file mode 100644 index c4b7aa765c2..00000000000 --- a/tensorflow/contrib/keras/api/keras/wrappers/scikit_learn/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keras scikit-learn API wrapper.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.wrappers.scikit_learn import KerasClassifier -from tensorflow.python.keras.wrappers.scikit_learn import KerasRegressor - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/kernel_methods/BUILD b/tensorflow/contrib/kernel_methods/BUILD deleted file mode 100644 index 5af95d192e8..00000000000 --- a/tensorflow/contrib/kernel_methods/BUILD +++ /dev/null @@ -1,96 +0,0 @@ -# Description: -# Contains kernel methods for TensorFlow. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "kernel_methods", - srcs = [ - "__init__.py", - "python/kernel_estimators.py", - "python/losses.py", - "python/mappers/random_fourier_features.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":dense_kernel_mapper_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform", - "//tensorflow/python:util", - "//tensorflow/python/ops/losses", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_library( - name = "dense_kernel_mapper_py", - srcs = ["python/mappers/dense_kernel_mapper.py"], - srcs_version = "PY2AND3", - deps = ["@six_archive//:six"], -) - -py_test( - name = "random_fourier_features_test", - srcs = ["python/mappers/random_fourier_features_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":dense_kernel_mapper_py", - ":kernel_methods", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - ], -) - -py_test( - name = "kernel_estimators_test", - srcs = ["python/kernel_estimators_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["notsan"], - deps = [ - ":kernel_methods", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:sparse_tensor", - "//third_party/py/numpy", - ], -) - -py_test( - name = "losses_test", - srcs = ["python/losses_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":kernel_methods", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/kernel_methods/README.md b/tensorflow/contrib/kernel_methods/README.md deleted file mode 100644 index 1bce3277ff4..00000000000 --- a/tensorflow/contrib/kernel_methods/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# TensorFlow contrib kernel_methods. - -This module contains operations and estimators that enable the use of primal -(explicit) kernel methods in TensorFlow. See also the [tutorial](https://www.tensorflow.org/code/tensorflow/contrib/kernel_methods/g3doc/tutorial.md) on how to use this module to improve the quality of -classification or regression tasks. - -## Kernel Mappers -Implement explicit kernel mapping Ops over tensors. Kernel mappers add -Tensor-In-Tensor-Out (TITO) Ops to the TensorFlow graph. They can be used in -conjunction with other layers or ML models. - -Sample usage: - -```python -kernel_mapper = tf.contrib.kernel_methods.SomeKernelMapper(...) -out_tensor = kernel_mapper.map(in_tensor) -... # code that consumes out_tensor. -``` - -Currently, there is a [RandomFourierFeatureMapper](https://www.tensorflow.org/code/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py) implemented that maps dense input to dense -output. More mappers are on the way. - -## Kernel-based Estimators - -These estimators inherit from the -[`tf.contrib.learn.Estimator`](https://www.tensorflow.org/code/tensorflow/contrib/learn/python/learn/estimators/estimator.py) -class and use kernel mappers internally to discover non-linearities in the -data. These canned estimators map their input features using kernel mapper -Ops and then apply linear models to the mapped features. Combining kernel -mappers with linear models and different loss functions leads to a variety of -models: linear and non-linear SVMs, linear regression (with and without -kernels) and (multinomial) logistic regression (with and without kernels). - -Currently there is a [KernelLinearClassifier](https://www.tensorflow.org/code/tensorflow/contrib/kernel_methods/python/kernel_estimators.py) implemented but more pre-packaged estimators -are on the way. - -Sample usage: - -```python -real_column_a = tf.contrib.layers.real_valued_column(name='real_column_a',...) -sparse_column_b = tf.contrib.layers.sparse_column_with_hash_bucket(...) -kernel_mappers = {real_column_a : [tf.contrib.kernel_methods.SomeKernelMapper(...)]} -optimizer = ... - -kernel_classifier = tf.contrib.kernel_methods.KernelLinearClassifier( - feature_columns=[real_column_a, sparse_column_b], - model_dir=..., - optimizer=optimizer, - kernel_mappers=kernel_mappers) - -# Construct input_fns -kernel_classifier.fit(...) -kernel_classifier.evaluate(...) -``` - diff --git a/tensorflow/contrib/kernel_methods/__init__.py b/tensorflow/contrib/kernel_methods/__init__.py deleted file mode 100644 index 0f3827d1870..00000000000 --- a/tensorflow/contrib/kernel_methods/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops and estimators that enable explicit kernel methods in TensorFlow. - -@@KernelLinearClassifier -@@RandomFourierFeatureMapper -@@sparse_multiclass_hinge_loss -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.kernel_methods.python.kernel_estimators import KernelLinearClassifier -from tensorflow.contrib.kernel_methods.python.losses import sparse_multiclass_hinge_loss -from tensorflow.contrib.kernel_methods.python.mappers.random_fourier_features import RandomFourierFeatureMapper - -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/kernel_methods/g3doc/acc-vs-trn_time.png b/tensorflow/contrib/kernel_methods/g3doc/acc-vs-trn_time.png deleted file mode 100644 index 1028bb39017..00000000000 Binary files a/tensorflow/contrib/kernel_methods/g3doc/acc-vs-trn_time.png and /dev/null differ diff --git a/tensorflow/contrib/kernel_methods/g3doc/acc_vs_outdim.png b/tensorflow/contrib/kernel_methods/g3doc/acc_vs_outdim.png deleted file mode 100644 index b3384e053b2..00000000000 Binary files a/tensorflow/contrib/kernel_methods/g3doc/acc_vs_outdim.png and /dev/null differ diff --git a/tensorflow/contrib/kernel_methods/g3doc/kernel_mapping.png b/tensorflow/contrib/kernel_methods/g3doc/kernel_mapping.png deleted file mode 100644 index e63303dab45..00000000000 Binary files a/tensorflow/contrib/kernel_methods/g3doc/kernel_mapping.png and /dev/null differ diff --git a/tensorflow/contrib/kernel_methods/g3doc/tutorial.md b/tensorflow/contrib/kernel_methods/g3doc/tutorial.md deleted file mode 100644 index f39a8d80d22..00000000000 --- a/tensorflow/contrib/kernel_methods/g3doc/tutorial.md +++ /dev/null @@ -1,279 +0,0 @@ -# Improving Linear Models Using Explicit Kernel Methods - -In this tutorial, we demonstrate how combining (explicit) kernel methods with -linear models can drastically increase the latters' quality of predictions -without significantly increasing training and inference times. Unlike dual -kernel methods, explicit (primal) kernel methods scale well with the size of the -training dataset both in terms of training/inference times and in terms of -memory requirements. - -Currently, explicit kernel mappings are supported for dense features. Support -for sparse features is in the works. - -We will use [tf.contrib.learn](https://www.tensorflow.org/code/tensorflow/contrib/learn/python/learn) (TensorFlow's high-level Machine Learning API) Estimators for our ML models. The -tf.contrib.learn API reduces the boilerplate code one needs to write for -configuring, training and evaluating models and will let us focus on the core -ideas. If you are not familiar with this API, [tf.estimator Quickstart](https://www.tensorflow.org/get_started/estimator) is a good place to start. We -will use MNIST, a widely-used dataset containing images of handwritten digits -(between 0 and 9). The tutorial consists of the following steps: - -* Load and prepare MNIST data for classification. -* Construct a simple linear model, train it and evaluate it on the eval data. -* Replace the linear model with a kernelized linear model, re-train and -re-evaluate. - -## Load and prepare MNIST data for classification -The first step is to prepare the data to be fed to the ML models. The following -utility command from tf.contrib.learn loads the MNIST dataset: - -```python -data = tf.contrib.learn.datasets.mnist.load_mnist() -``` -This loads the entire MNIST dataset (containing 70K samples) and splits it into -train, validation and test data with 55K, 5K and 10K samples respectively. Each -split contains one numpy array for images (with shape [sample_size, 784]) and -one for labels (with shape [sample_size, 1]). In this tutorial, we only use the -train and validation splits (to train and evaluate our models respectively). - -In order to feed data to a tf.contrib.learn Estimator, it is helpful to convert -it to Tensors. For this, we will use an `input function` which adds Ops to the -TensorFlow graph that, when executed, create mini-batches of Tensors to be used -downstream. For more background on input functions, check -[Building Input Functions with tf.contrib.learn](https://www.tensorflow.org/get_started/input_fn). -In this example, we will use the `tf.train.shuffle_batch` Op which, besides -converting numpy arrays to Tensors, allows us to specify the batch_size and -whether to randomize the input every time the input_fn Ops are executed -(randomization typically expedites convergence during training). The full code -for loading and preparing the data is shown in the snippet below. In this -example, we use mini-batches of size 256 for training and the entire sample (5K -entries) for evaluation. Feel free to experiment with different batch sizes. - -```python -import numpy as np -import tensorflow as tf - -def get_input_fn(dataset_split, batch_size, capacity=10000, min_after_dequeue=3000): - - def _input_fn(): - images_batch, labels_batch = tf.train.shuffle_batch( - tensors=[dataset_split.images, dataset_split.labels.astype(np.int32)], - batch_size=batch_size, - capacity=capacity, - min_after_dequeue=min_after_dequeue, - enqueue_many=True, - num_threads=4) - features_map = {'images': images_batch} - return features_map, labels_batch - - return _input_fn - -data = tf.contrib.learn.datasets.mnist.load_mnist() - -train_input_fn = get_input_fn(data.train, batch_size=256) -eval_input_fn = get_input_fn(data.validation, batch_size=5000) - -``` - -## Training a simple linear model -We can now train a linear model over the MNIST dataset. We will use the -[tf.contrib.learn.LinearClassifier](https://www.tensorflow.org/code/tensorflow/contrib/learn/python/learn/estimators/linear.py) estimator with 10 classes (representing the 10 digits). -The input features form a 784-dimensional (dense) vector which can be specified -as follows: - -```python -image_column = tf.contrib.layers.real_valued_column('images', dimension=784) -``` - -The full code for constructing, training and evaluating a LinearClassifier -estimator is shown below. - -```python -import time - -# Specify the feature(s) to be used by the estimator. -image_column = tf.contrib.layers.real_valued_column('images', dimension=784) -estimator = tf.contrib.learn.LinearClassifier(feature_columns=[image_column], n_classes=10) - -# Train. -start = time.time() -estimator.fit(input_fn=train_input_fn, steps=2000) -end = time.time() -print('Elapsed time: {} seconds'.format(end - start)) - -# Evaluate and report metrics. -eval_metrics = estimator.evaluate(input_fn=eval_input_fn, steps=1) -print(eval_metrics) -``` -On eval data, the loss (i.e., the value of the objective function being -minimized during training) lies between **0.25** and **0.30** (depending on the -parameters used) while the accuracy of the classifier is approximately **92.5%** -(training is randomized so the exact loss and accuracy will vary). Also, the -training time is around 25 seconds (this will also vary based on the machine you -run the code on). - -In addition to experimenting with the (training) batch size and the number of -training steps, there are a couple other parameters that can be tuned as well. -For instance, you can change the optimization method used to minimize the loss -by explicitly selecting another optimizer from the collection of -[available optimizers](https://www.tensorflow.org/code/tensorflow/python/training). -As an example, the following code constructs a LinearClassifier estimator that -uses the Follow-The-Regularized-Leader (FTRL) optimization strategy with a -specific learning rate and L2-regularization. - - -```python -optimizer = tf.train.FtrlOptimizer(learning_rate=5.0, l2_regularization_strength=1.0) -estimator = tf.contrib.learn.LinearClassifier( - feature_columns=[image_column], n_classes=10, optimizer=optimizer) -``` - -Regardless of the values of the parameters, the max accuracy a linear model can -achieve on this dataset caps at around **93%**. - -## Using explicit kernel mappings with the linear model. -The relatively high error (~7%) of the linear model over MNIST indicates that -the input data is not linearly separable. We will use explicit kernel mappings -to reduce the classification error. - -**Intuition:** The high-level idea is to use a non-linear map to transform the -input space to another feature space (of possibly higher dimension) where the -(transformed) features are (almost) linearly separable and then apply a linear -model on the mapped features. This is shown in the following figure: - -![image](./kernel_mapping.png) - -**Technical details overview:** In this example we will use **Random Fourier -Features** (introduced in the -["Random Features for Large-Scale Kernel Machines"](https://people.eecs.berkeley.edu/~brecht/papers/07.rah.rec.nips.pdf) paper by -Rahimi and Recht) to map the input data. Random Fourier Features map a vector -\\(\mathbf{x} \in \mathbb{R}^d\\) to \\(\mathbf{x'} \in \mathbb{R}^D\\) via the -following mapping: - -$$ -RFFM(\cdot): \mathbb{R}^d \to \mathbb{R}^D, \quad -RFFM(\mathbf{x}) = \cos(\mathbf{\Omega} \cdot \mathbf{x}+ \mathbf{b}) -$$ - -where \\(\mathbf{\Omega} \in \mathbb{R}^{D \times d}\\), -\\(\mathbf{x} \in \mathbb{R}^d,\\) \\(\mathbf{b} \in \mathbb{R}^D\\) and the -cosine is applied element-wise. - -In this example, the entries of \\(\mathbf{\Omega}\\) and \\(\mathbf{b}\\) are -sampled from distributions such that the mapping satisfies the following -property: - -$$ -RFFM(\mathbf{x})^T \cdot RFFM(\mathbf{y}) \approx -e^{-\frac{\|\mathbf{x} - \mathbf{y}\|^2}{2 \sigma^2}} -$$ - -The right-hand-side quantity of the expression above is known as the RBF (or -Gaussian) kernel function. This function is one of the most-widely used kernel -functions in Machine Learning and measures (implicitly) similarity in a -different (much higher dimensional) space than the original one. See -[Radial basis function kernel](https://en.wikipedia.org/wiki/Radial_basis_function_kernel) -for more details. - -**Kernel Classifier:** `tf.contrib.kernel_methods.KernelLinearClassifier` is a -pre-packaged `tf.contrib.learn` estimator that combines the power of explicit -kernel mappings with linear models. Its API is very similar to that of the -LinearClassifier with the additional ability to specify a list of explicit -kernel mappings to be applied to each feature used by the classifier. The -following code snippet demonstrates how to replace LinearClassifier with -KernelLinearClassifier. - - -```python -# Specify the feature(s) to be used by the estimator. This is identical to the -# code used for the LinearClassifier. -image_column = tf.contrib.layers.real_valued_column('images', dimension=784) -optimizer = tf.train.FtrlOptimizer( - learning_rate=50.0, l2_regularization_strength=0.001) - - -kernel_mapper = tf.contrib.kernel_methods.RandomFourierFeatureMapper( - input_dim=784, output_dim=2000, stddev=5.0, name='rffm') -kernel_mappers = {image_column: [kernel_mapper]} -estimator = tf.contrib.kernel_methods.KernelLinearClassifier( - n_classes=10, optimizer=optimizer, kernel_mappers=kernel_mappers) - -# Train. -start = time.time() -estimator.fit(input_fn=train_input_fn, steps=2000) -end = time.time() -print('Elapsed time: {} seconds'.format(end - start)) - -# Evaluate and report metrics. -eval_metrics = estimator.evaluate(input_fn=eval_input_fn, steps=1) -print(eval_metrics) -``` -The only additional parameter passed to `KernelLinearClassifier` is a dictionary -from feature_columns to a list of kernel mappings to be applied to the -corresponding feature column. In this example, the lines - -```python -kernel_mapper = tf.contrib.kernel_methods.RandomFourierFeatureMapper( - input_dim=784, output_dim=2000, stddev=5.0, name='rffm') -kernel_mappers = {image_column: [kernel_mapper]} -estimator = tf.contrib.kernel_methods.KernelLinearClassifier( - n_classes=10, optimizer=optimizer, kernel_mappers=kernel_mappers) -``` -instruct the classifier to first map the initial 784-dimensional images to -2000-dimensional vectors using random Fourier features and then learn a linear -model on the transformed vectors. Note that, besides the output dimension, there -is one more parameter (stddev) involved. This parameter is the standard -deviation (\\(\sigma\\)) of the approximated RBF kernel and controls the -similarity measure used in classification. This parameter is typically -determined via hyperparameter tuning. - -Running the code above yields a loss of approximately **0.10** while the -accuracy is increased to approximately **97%** on eval data (an increase of 4% -over the plain linear model). The training time hovers around 35 seconds. We can -increase the accuracy even more, by increasing the output dimension of the -mapping and tuning the standard deviation even more. - -**On the role of stddev:** The classification quality is very sensitive to the -value of the stddev parameter used to define the similarity measure between the -pairs of input features. The following table shows the accuracy of the -classifier on the eval data for different values of stddev (for all experiments -the output dimension was fixed to 3000). The optimal value is stddev=5.0. Notice -how too small or too high stddev values can dramatically decrease the accuracy -of the classification. - -stddev | eval accuracy -:----- | :------------ -1.0 | 0.1362 -2.0 | 0.4764 -4.0 | 0.9654 -5.0 | 0.9766 -8.0 | 0.9714 -16.0 | 0.8878 - -**On the role of the output dimension:** Intuitively, the larger the output -dimension of the mapping, the closer the inner product of two mapped vectors -approximates the kernel which typically translates to better classification -accuracy. Another way to think about this is that the output dimension equals -the number of weights of the linear model (the larger this dimension, the larger -the "degrees of freedom" of the model). However, after a certain threshold, -higher output dimensions increase the accuracy by very little (while still -increasing the training time). This is shown in the following 2 Figures which -depict the eval accuracy as a function of the output dimension and the training -time respectively. - -![image](./acc_vs_outdim.png) ![image](./acc-vs-trn_time.png) - - -## Explicit kernel mappings: summary and practical tips -* Explicit kernel mappings combine the predictive power of non-linear models -with the scalability of linear models. -* Unlike traditional dual kernel methods, they can scale to millions or hundreds -of millions of samples. -* Random Fourier Features can be particularly effective for datasets with dense -features. -* The parameters of the kernel mapping are often data-dependent. Model quality -can be very sensitive to these parameters. Use hyperparameter tuning to find the -optimal values. -* If you have multiple numerical features, concatenate them into a single -multi-dimensional feature and apply the kernel mapping to the concatenated -vector. - diff --git a/tensorflow/contrib/kernel_methods/python/kernel_estimators.py b/tensorflow/contrib/kernel_methods/python/kernel_estimators.py deleted file mode 100644 index 0f863c5a906..00000000000 --- a/tensorflow/contrib/kernel_methods/python/kernel_estimators.py +++ /dev/null @@ -1,339 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Estimators that combine explicit kernel mappings with linear models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import six - -from tensorflow.contrib import layers -from tensorflow.contrib.kernel_methods.python.mappers import dense_kernel_mapper as dkm -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import linear -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import tf_logging as logging - -_FEATURE_COLUMNS = "feature_columns" -_KERNEL_MAPPERS = "kernel_mappers" -_OPTIMIZER = "optimizer" - - -def _check_valid_kernel_mappers(kernel_mappers): - """Checks that the input kernel_mappers are valid.""" - if kernel_mappers is None: - return True - for kernel_mappers_list in six.itervalues(kernel_mappers): - for kernel_mapper in kernel_mappers_list: - if not isinstance(kernel_mapper, dkm.DenseKernelMapper): - return False - return True - - -def _check_valid_head(head): - """Returns true if the provided head is supported.""" - if head is None: - return False - # pylint: disable=protected-access - return isinstance(head, head_lib._BinaryLogisticHead) or isinstance( - head, head_lib._MultiClassHead) - # pylint: enable=protected-access - - -def _update_features_and_columns(features, feature_columns, - kernel_mappers_dict): - """Updates features and feature_columns based on provided kernel mappers. - - Currently supports the update of `RealValuedColumn`s only. - - Args: - features: Initial features dict. The key is a `string` (feature column name) - and the value is a tensor. - feature_columns: Initial iterable containing all the feature columns to be - consumed (possibly after being updated) by the model. All items should be - instances of classes derived from `FeatureColumn`. - kernel_mappers_dict: A dict from feature column (type: _FeatureColumn) to - objects inheriting from KernelMapper class. - - Returns: - updated features and feature_columns based on provided kernel_mappers_dict. - """ - if kernel_mappers_dict is None: - return features, feature_columns - - # First construct new columns and features affected by kernel_mappers_dict. - mapped_features = {} - mapped_columns = set() - for feature_column in kernel_mappers_dict: - column_name = feature_column.name - # Currently only mappings over RealValuedColumns are supported. - if not isinstance(feature_column, layers.feature_column._RealValuedColumn): # pylint: disable=protected-access - logging.warning( - "Updates are currently supported on RealValuedColumns only. Metadata " - "for FeatureColumn {} will not be updated.".format(column_name)) - continue - mapped_column_name = column_name + "_MAPPED" - # Construct new feature columns based on provided kernel_mappers. - column_kernel_mappers = kernel_mappers_dict[feature_column] - new_dim = sum(mapper.output_dim for mapper in column_kernel_mappers) - mapped_columns.add( - layers.feature_column.real_valued_column(mapped_column_name, new_dim)) - - # Get mapped features by concatenating mapped tensors (one mapped tensor - # per kernel mappers from the list of kernel mappers corresponding to each - # feature column). - output_tensors = [] - for kernel_mapper in column_kernel_mappers: - output_tensors.append(kernel_mapper.map(features[column_name])) - tensor = array_ops.concat(output_tensors, 1) - mapped_features[mapped_column_name] = tensor - - # Finally update features dict and feature_columns. - features = features.copy() - features.update(mapped_features) - feature_columns = set(feature_columns) - feature_columns.update(mapped_columns) - - return features, feature_columns - - -def _kernel_model_fn(features, labels, mode, params, config=None): - """model_fn for the Estimator using kernel methods. - - Args: - features: `Tensor` or dict of `Tensor` (depends on data passed to `fit`). - labels: `Tensor` of shape [batch_size, 1] or [batch_size] labels of - dtype `int32` or `int64` in the range `[0, n_classes)`. - mode: Defines whether this is training, evaluation or prediction. See - `ModeKeys`. - params: A dict of hyperparameters. - The following hyperparameters are expected: - * head: A `Head` instance. - * feature_columns: An iterable containing all the feature columns used by - the model. - * optimizer: string, `Optimizer` object, or callable that defines the - optimizer to use for training. If `None`, will use a FTRL optimizer. - * kernel_mappers: Dictionary of kernel mappers to be applied to the input - features before training. - config: `RunConfig` object to configure the runtime settings. - - Returns: - A `ModelFnOps` instance. - - Raises: - ValueError: If mode is not any of the `ModeKeys`. - """ - feature_columns = params[_FEATURE_COLUMNS] - kernel_mappers = params[_KERNEL_MAPPERS] - - updated_features, updated_columns = _update_features_and_columns( - features, feature_columns, kernel_mappers) - params[_FEATURE_COLUMNS] = updated_columns - - return linear._linear_model_fn( # pylint: disable=protected-access - updated_features, labels, mode, params, config) - - -class _KernelEstimator(estimator.Estimator): - """Generic kernel-based linear estimator.""" - - def __init__(self, - feature_columns=None, - model_dir=None, - weight_column_name=None, - head=None, - optimizer=None, - kernel_mappers=None, - config=None): - """Constructs a `_KernelEstimator` object.""" - if not feature_columns and not kernel_mappers: - raise ValueError( - "You should set at least one of feature_columns, kernel_mappers.") - if not _check_valid_kernel_mappers(kernel_mappers): - raise ValueError("Invalid kernel mappers.") - - if not _check_valid_head(head): - raise ValueError( - "head type: {} is not supported. Supported head types: " - "_BinaryLogisticHead, _MultiClassHead.".format(type(head))) - - params = { - "head": head, - _FEATURE_COLUMNS: feature_columns or [], - _OPTIMIZER: optimizer, - _KERNEL_MAPPERS: kernel_mappers, - } - super(_KernelEstimator, self).__init__( - model_fn=_kernel_model_fn, - model_dir=model_dir, - config=config, - params=params) - - -class KernelLinearClassifier(_KernelEstimator): - """Linear classifier using kernel methods as feature preprocessing. - - It trains a linear model after possibly mapping initial input features into - a mapped space using explicit kernel mappings. Due to the kernel mappings, - training a linear classifier in the mapped (output) space can detect - non-linearities in the input space. - - The user can provide a list of kernel mappers to be applied to all or a subset - of existing feature_columns. This way, the user can effectively provide 2 - types of feature columns: - - * those passed as elements of feature_columns in the classifier's constructor - * those appearing as a key of the kernel_mappers dict. - - If a column appears in feature_columns only, no mapping is applied to it. If - it appears as a key in kernel_mappers, the corresponding kernel mappers are - applied to it. Note that it is possible that a column appears in both places. - Currently kernel_mappers are supported for _RealValuedColumns only. - - Example usage: - ``` - real_column_a = real_valued_column(name='real_column_a',...) - sparse_column_b = sparse_column_with_hash_bucket(...) - kernel_mappers = {real_column_a : [RandomFourierFeatureMapper(...)]} - optimizer = ... - - # real_column_a is used as a feature in both its initial and its transformed - # (mapped) form. sparse_column_b is not affected by kernel mappers. - kernel_classifier = KernelLinearClassifier( - feature_columns=[real_column_a, sparse_column_b], - model_dir=..., - optimizer=optimizer, - kernel_mappers=kernel_mappers) - - # real_column_a is used as a feature in its transformed (mapped) form only. - # sparse_column_b is not affected by kernel mappers. - kernel_classifier = KernelLinearClassifier( - feature_columns=[sparse_column_b], - model_dir=..., - optimizer=optimizer, - kernel_mappers=kernel_mappers) - - # Input builders - def train_input_fn: # returns x, y - ... - def eval_input_fn: # returns x, y - ... - - kernel_classifier.fit(input_fn=train_input_fn) - kernel_classifier.evaluate(input_fn=eval_input_fn) - kernel_classifier.predict(...) - ``` - - Input of `fit` and `evaluate` should have following features, otherwise there - will be a `KeyError`: - - * if `weight_column_name` is not `None`, a feature with - `key=weight_column_name` whose value is a `Tensor`. - * for each `column` in `feature_columns`: - - if `column` is a `SparseColumn`, a feature with `key=column.name` - whose `value` is a `SparseTensor`. - - if `column` is a `WeightedSparseColumn`, two features: the first with - `key` the id column name, the second with `key` the weight column name. - Both features' `value` must be a `SparseTensor`. - - if `column` is a `RealValuedColumn`, a feature with `key=column.name` - whose `value` is a `Tensor`. - """ - - def __init__(self, - feature_columns=None, - model_dir=None, - n_classes=2, - weight_column_name=None, - optimizer=None, - kernel_mappers=None, - config=None): - """Construct a `KernelLinearClassifier` estimator object. - - Args: - feature_columns: An iterable containing all the feature columns used by - the model. All items in the set should be instances of classes derived - from `FeatureColumn`. - model_dir: Directory to save model parameters, graph etc. This can also be - used to load checkpoints from the directory into an estimator to - continue training a previously saved model. - n_classes: number of label classes. Default is binary classification. - Note that class labels are integers representing the class index (i.e. - values from 0 to n_classes-1). For arbitrary label values (e.g. string - labels), convert to class indices first. - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - optimizer: The optimizer used to train the model. If specified, it should - be an instance of `tf.Optimizer`. If `None`, the Ftrl optimizer is used - by default. - kernel_mappers: Dictionary of kernel mappers to be applied to the input - features before training a (linear) model. Keys are feature columns and - values are lists of mappers to be applied to the corresponding feature - column. Currently only _RealValuedColumns are supported and therefore - all mappers should conform to the `DenseKernelMapper` interface (see - ./mappers/dense_kernel_mapper.py). - config: `RunConfig` object to configure the runtime settings. - - Returns: - A `KernelLinearClassifier` estimator. - - Raises: - ValueError: if n_classes < 2. - ValueError: if neither feature_columns nor kernel_mappers are provided. - ValueError: if mappers provided as kernel_mappers values are invalid. - """ - super(KernelLinearClassifier, self).__init__( - feature_columns=feature_columns, - model_dir=model_dir, - weight_column_name=weight_column_name, - head=head_lib.multi_class_head( - n_classes=n_classes, weight_column_name=weight_column_name), - optimizer=optimizer, - kernel_mappers=kernel_mappers, - config=config) - - def predict_classes(self, input_fn=None): - """Runs inference to determine the predicted class per instance. - - Args: - input_fn: The input function providing features. - - Returns: - A generator of predicted classes for the features provided by input_fn. - Each predicted class is represented by its class index (i.e. integer from - 0 to n_classes-1) - """ - key = prediction_key.PredictionKey.CLASSES - predictions = super(KernelLinearClassifier, self).predict( - input_fn=input_fn, outputs=[key]) - return (pred[key] for pred in predictions) - - def predict_proba(self, input_fn=None): - """Runs inference to determine the class probability predictions. - - Args: - input_fn: The input function providing features. - - Returns: - A generator of predicted class probabilities for the features provided by - input_fn. - """ - key = prediction_key.PredictionKey.PROBABILITIES - predictions = super(KernelLinearClassifier, self).predict( - input_fn=input_fn, outputs=[key]) - return (pred[key] for pred in predictions) diff --git a/tensorflow/contrib/kernel_methods/python/kernel_estimators_test.py b/tensorflow/contrib/kernel_methods/python/kernel_estimators_test.py deleted file mode 100644 index a461ba81345..00000000000 --- a/tensorflow/contrib/kernel_methods/python/kernel_estimators_test.py +++ /dev/null @@ -1,269 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for kernel_estimators.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib import layers -from tensorflow.contrib.kernel_methods.python import kernel_estimators -from tensorflow.contrib.kernel_methods.python.mappers.random_fourier_features import RandomFourierFeatureMapper -from tensorflow.contrib.learn.python.learn.estimators import test_data -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework.test_util import TensorFlowTestCase -from tensorflow.python.platform import googletest - - -def _linearly_separable_binary_input_fn(): - """Returns linearly-separable data points (binary classification).""" - return { - 'feature1': constant_op.constant([[0.0], [1.0], [3.0]]), - 'feature2': constant_op.constant([[1.0], [-1.2], [1.0]]), - }, constant_op.constant([[1], [0], [1]]) - - -def _linearly_inseparable_binary_input_fn(): - """Returns non-linearly-separable data points (binary classification).""" - return { - 'multi_dim_feature': - constant_op.constant([[1.0, 1.0], [1.0, -1.0], [-1.0, -1.0], - [-1.0, 1.0]]), - }, constant_op.constant([[1], [0], [1], [0]]) - - -class KernelLinearClassifierTest(TensorFlowTestCase): - - def testNoFeatureColumnsOrKernelMappers(self): - """Tests that at least one of feature columns or kernels is provided.""" - with self.assertRaises(ValueError): - _ = kernel_estimators.KernelLinearClassifier() - - def testInvalidKernelMapper(self): - """ValueError raised when the kernel mappers provided have invalid type.""" - - class DummyKernelMapper(object): - - def __init__(self): - pass - - feature = layers.real_valued_column('feature') - kernel_mappers = {feature: [DummyKernelMapper()]} - with self.assertRaises(ValueError): - _ = kernel_estimators.KernelLinearClassifier( - feature_columns=[feature], kernel_mappers=kernel_mappers) - - def testInvalidNumberOfClasses(self): - """ValueError raised when the kernel mappers provided have invalid type.""" - - feature = layers.real_valued_column('feature') - with self.assertRaises(ValueError): - _ = kernel_estimators.KernelLinearClassifier( - feature_columns=[feature], n_classes=1) - - def testLinearlySeparableBinaryDataNoKernels(self): - """Tests classifier w/o kernels (log. regression) for lin-separable data.""" - - feature1 = layers.real_valued_column('feature1') - feature2 = layers.real_valued_column('feature2') - - logreg_classifier = kernel_estimators.KernelLinearClassifier( - feature_columns=[feature1, feature2]) - logreg_classifier.fit( - input_fn=_linearly_separable_binary_input_fn, steps=100) - - metrics = logreg_classifier.evaluate( - input_fn=_linearly_separable_binary_input_fn, steps=1) - # Since the data is linearly separable, the classifier should have small - # loss and perfect accuracy. - self.assertLess(metrics['loss'], 0.1) - self.assertEqual(metrics['accuracy'], 1.0) - - # As a result, it should assign higher probability to class 1 for the 1st - # and 3rd example and higher probability to class 0 for the second example. - logreg_prob_predictions = list( - logreg_classifier.predict_proba(input_fn= - _linearly_separable_binary_input_fn)) - self.assertGreater(logreg_prob_predictions[0][1], 0.5) - self.assertGreater(logreg_prob_predictions[1][0], 0.5) - self.assertGreater(logreg_prob_predictions[2][1], 0.5) - - def testLinearlyInseparableBinaryDataWithAndWithoutKernels(self): - """Tests classifier w/ and w/o kernels on non-linearly-separable data.""" - multi_dim_feature = layers.real_valued_column( - 'multi_dim_feature', dimension=2) - - # Data points are non-linearly separable so there will be at least one - # mis-classified sample (accuracy < 0.8). In fact, the loss is minimized for - # w1=w2=0.0, in which case each example incurs a loss of ln(2). The overall - # (average) loss should then be ln(2) and the logits should be approximately - # 0.0 for each sample. - logreg_classifier = kernel_estimators.KernelLinearClassifier( - feature_columns=[multi_dim_feature]) - logreg_classifier.fit( - input_fn=_linearly_inseparable_binary_input_fn, steps=50) - logreg_metrics = logreg_classifier.evaluate( - input_fn=_linearly_inseparable_binary_input_fn, steps=1) - logreg_loss = logreg_metrics['loss'] - logreg_accuracy = logreg_metrics['accuracy'] - logreg_predictions = logreg_classifier.predict( - input_fn=_linearly_inseparable_binary_input_fn, as_iterable=False) - self.assertAlmostEqual(logreg_loss, np.log(2), places=3) - self.assertLess(logreg_accuracy, 0.8) - self.assertAllClose(logreg_predictions['logits'], [[0.0], [0.0], [0.0], - [0.0]]) - - # Using kernel mappers allows to discover non-linearities in data. Mapping - # the data to a higher dimensional feature space using approx RBF kernels, - # substantially reduces the loss and leads to perfect classification - # accuracy. - kernel_mappers = { - multi_dim_feature: [RandomFourierFeatureMapper(2, 30, 0.6, 1, 'rffm')] - } - kernelized_logreg_classifier = kernel_estimators.KernelLinearClassifier( - feature_columns=[], kernel_mappers=kernel_mappers) - kernelized_logreg_classifier.fit( - input_fn=_linearly_inseparable_binary_input_fn, steps=50) - kernelized_logreg_metrics = kernelized_logreg_classifier.evaluate( - input_fn=_linearly_inseparable_binary_input_fn, steps=1) - kernelized_logreg_loss = kernelized_logreg_metrics['loss'] - kernelized_logreg_accuracy = kernelized_logreg_metrics['accuracy'] - self.assertLess(kernelized_logreg_loss, 0.2) - self.assertEqual(kernelized_logreg_accuracy, 1.0) - - def testVariablesWithAndWithoutKernels(self): - """Tests variables w/ and w/o kernel.""" - multi_dim_feature = layers.real_valued_column( - 'multi_dim_feature', dimension=2) - - linear_classifier = kernel_estimators.KernelLinearClassifier( - feature_columns=[multi_dim_feature]) - linear_classifier.fit( - input_fn=_linearly_inseparable_binary_input_fn, steps=50) - linear_variables = linear_classifier.get_variable_names() - self.assertIn('linear/multi_dim_feature/weight', linear_variables) - self.assertIn('linear/bias_weight', linear_variables) - linear_weights = linear_classifier.get_variable_value( - 'linear/multi_dim_feature/weight') - linear_bias = linear_classifier.get_variable_value('linear/bias_weight') - - kernel_mappers = { - multi_dim_feature: [RandomFourierFeatureMapper(2, 30, 0.6, 1, 'rffm')] - } - kernel_linear_classifier = kernel_estimators.KernelLinearClassifier( - feature_columns=[], kernel_mappers=kernel_mappers) - kernel_linear_classifier.fit( - input_fn=_linearly_inseparable_binary_input_fn, steps=50) - kernel_linear_variables = kernel_linear_classifier.get_variable_names() - self.assertIn('linear/multi_dim_feature_MAPPED/weight', - kernel_linear_variables) - self.assertIn('linear/bias_weight', kernel_linear_variables) - kernel_linear_weights = kernel_linear_classifier.get_variable_value( - 'linear/multi_dim_feature_MAPPED/weight') - kernel_linear_bias = kernel_linear_classifier.get_variable_value( - 'linear/bias_weight') - - # The feature column used for linear classification (no kernels) has - # dimension 2 so the model will learn a 2-dimension weights vector (and a - # scalar for the bias). In the kernelized model, the features are mapped to - # a 30-dimensional feature space and so the weights variable will also have - # dimension 30. - self.assertEqual(2, len(linear_weights)) - self.assertEqual(1, len(linear_bias)) - self.assertEqual(30, len(kernel_linear_weights)) - self.assertEqual(1, len(kernel_linear_bias)) - - def testClassifierWithAndWithoutKernelsNoRealValuedColumns(self): - """Tests kernels have no effect for non-real valued columns .""" - - def input_fn(): - return { - 'price': - constant_op.constant([[0.4], [0.6], [0.3]]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - }, constant_op.constant([[1], [0], [1]]) - - price = layers.real_valued_column('price') - country = layers.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - - linear_classifier = kernel_estimators.KernelLinearClassifier( - feature_columns=[price, country]) - linear_classifier.fit(input_fn=input_fn, steps=100) - linear_metrics = linear_classifier.evaluate(input_fn=input_fn, steps=1) - linear_loss = linear_metrics['loss'] - linear_accuracy = linear_metrics['accuracy'] - - kernel_mappers = { - country: [RandomFourierFeatureMapper(2, 30, 0.6, 1, 'rffm')] - } - - kernel_linear_classifier = kernel_estimators.KernelLinearClassifier( - feature_columns=[price, country], kernel_mappers=kernel_mappers) - kernel_linear_classifier.fit(input_fn=input_fn, steps=100) - kernel_linear_metrics = kernel_linear_classifier.evaluate( - input_fn=input_fn, steps=1) - kernel_linear_loss = kernel_linear_metrics['loss'] - kernel_linear_accuracy = kernel_linear_metrics['accuracy'] - - # The kernel mapping is applied to a non-real-valued feature column and so - # it should have no effect on the model. The loss and accuracy of the - # "kernelized" model should match the loss and accuracy of the initial model - # (without kernels). - self.assertAlmostEqual(linear_loss, kernel_linear_loss, delta=0.01) - self.assertAlmostEqual(linear_accuracy, kernel_linear_accuracy, delta=0.01) - - def testMulticlassDataWithAndWithoutKernels(self): - """Tests classifier w/ and w/o kernels on multiclass data.""" - feature_column = layers.real_valued_column('feature', dimension=4) - - # Metrics for linear classifier (no kernels). - linear_classifier = kernel_estimators.KernelLinearClassifier( - feature_columns=[feature_column], n_classes=3) - linear_classifier.fit(input_fn=test_data.iris_input_multiclass_fn, steps=50) - linear_metrics = linear_classifier.evaluate( - input_fn=test_data.iris_input_multiclass_fn, steps=1) - linear_loss = linear_metrics['loss'] - linear_accuracy = linear_metrics['accuracy'] - - # Using kernel mappers allows to discover non-linearities in data (via RBF - # kernel approximation), reduces loss and increases accuracy. - kernel_mappers = { - feature_column: [ - RandomFourierFeatureMapper( - input_dim=4, output_dim=50, stddev=1.0, name='rffm') - ] - } - kernel_linear_classifier = kernel_estimators.KernelLinearClassifier( - feature_columns=[], n_classes=3, kernel_mappers=kernel_mappers) - kernel_linear_classifier.fit( - input_fn=test_data.iris_input_multiclass_fn, steps=50) - kernel_linear_metrics = kernel_linear_classifier.evaluate( - input_fn=test_data.iris_input_multiclass_fn, steps=1) - kernel_linear_loss = kernel_linear_metrics['loss'] - kernel_linear_accuracy = kernel_linear_metrics['accuracy'] - self.assertLess(kernel_linear_loss, linear_loss) - self.assertGreater(kernel_linear_accuracy, linear_accuracy) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/kernel_methods/python/losses.py b/tensorflow/contrib/kernel_methods/python/losses.py deleted file mode 100644 index 0d43bc2101b..00000000000 --- a/tensorflow/contrib/kernel_methods/python/losses.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implementation of kernel-methods-related loss operations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops.losses import losses - - -def sparse_multiclass_hinge_loss( - labels, - logits, - weights=1.0, - scope=None, - loss_collection=ops.GraphKeys.LOSSES, - reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS): - r"""Adds Ops for computing the multiclass hinge loss. - - The implementation is based on the following paper: - On the Algorithmic Implementation of Multiclass Kernel-based Vector Machines - by Crammer and Singer. - link: http://jmlr.csail.mit.edu/papers/volume2/crammer01a/crammer01a.pdf - - This is a generalization of standard (binary) hinge loss. For a given instance - with correct label c*, the loss is given by: - $$loss = max_{c != c*} logits_c - logits_{c*} + 1.$$ - or equivalently - $$loss = max_c { logits_c - logits_{c*} + I_{c != c*} }$$ - where \\(I_{c != c*} = 1\ \text{if}\ c != c*\\) and 0 otherwise. - - Args: - labels: `Tensor` of shape [batch_size] or [batch_size, 1]. Corresponds to - the ground truth. Each entry must be an index in `[0, num_classes)`. - logits: `Tensor` of shape [batch_size, num_classes] corresponding to the - unscaled logits. Its dtype should be either `float32` or `float64`. - weights: Optional (python) scalar or `Tensor`. If a non-scalar `Tensor`, its - rank should be either 1 ([batch_size]) or 2 ([batch_size, 1]). - scope: The scope for the operations performed in computing the loss. - loss_collection: collection to which the loss will be added. - reduction: Type of reduction to apply to loss. - - Returns: - Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same - shape as `labels`; otherwise, it is a scalar. - - Raises: - ValueError: If `logits`, `labels` or `weights` have invalid or inconsistent - shapes. - ValueError: If `labels` tensor has invalid dtype. - """ - - with ops.name_scope(scope, 'sparse_multiclass_hinge_loss', (logits, - labels)) as scope: - - # Check logits Tensor has valid rank. - logits_rank = logits.get_shape().ndims - if logits_rank != 2: - raise ValueError( - 'logits should have rank 2 ([batch_size, num_classes]). Given rank is' - ' {}'.format(logits_rank)) - logits_shape = array_ops.shape(logits) - batch_size, num_classes = logits_shape[0], logits_shape[1] - logits = math_ops.cast(logits, dtypes.float32) - - # Check labels have valid type. - if labels.dtype != dtypes.int32 and labels.dtype != dtypes.int64: - raise ValueError( - 'Invalid dtype for labels: {}. Acceptable dtypes: int32 and int64'. - format(labels.dtype)) - - # Check labels and weights have valid ranks and are consistent. - labels_rank = labels.get_shape().ndims - if labels_rank not in [1, 2]: - raise ValueError( - 'labels should have rank 1 ([batch_size]) or 2 ([batch_size, 1]). ' - 'Given rank is {}'.format(labels_rank)) - with ops.control_dependencies([ - check_ops.assert_less(labels, math_ops.cast(num_classes, labels.dtype)) - ]): - labels = array_ops.reshape(labels, shape=[-1]) - - weights = ops.convert_to_tensor(weights) - weights_rank = weights.get_shape().ndims - if weights_rank not in [0, 1, 2]: - raise ValueError( - 'non-scalar weights should have rank 1 ([batch_size]) or 2 ' - '([batch_size, 1]). Given rank is {}'.format(labels_rank)) - - if weights_rank > 0: - weights = array_ops.reshape(weights, shape=[-1]) - # Check weights and labels have the same number of elements. - weights.get_shape().assert_is_compatible_with(labels.get_shape()) - - # Compute the logits tensor corresponding to the correct class per instance. - example_indices = array_ops.reshape( - math_ops.range(batch_size), shape=[batch_size, 1]) - indices = array_ops.concat( - [ - example_indices, - array_ops.reshape( - math_ops.cast(labels, example_indices.dtype), - shape=[batch_size, 1]) - ], - axis=1) - label_logits = array_ops.reshape( - array_ops.gather_nd(params=logits, indices=indices), - shape=[batch_size, 1]) - - one_cold_labels = array_ops.one_hot( - indices=labels, depth=num_classes, on_value=0.0, off_value=1.0) - margin = logits - label_logits + one_cold_labels - margin = nn_ops.relu(margin) - loss = math_ops.reduce_max(margin, axis=1) - return losses.compute_weighted_loss( - loss, weights, scope, loss_collection, reduction=reduction) diff --git a/tensorflow/contrib/kernel_methods/python/losses_test.py b/tensorflow/contrib/kernel_methods/python/losses_test.py deleted file mode 100644 index 4d5cc24ce09..00000000000 --- a/tensorflow/contrib/kernel_methods/python/losses_test.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for third_party.tensorflow.contrib.kernel_methods.python.losses.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.kernel_methods.python import losses -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class SparseMulticlassHingeLossTest(test.TestCase): - - def testInvalidLogitsShape(self): - """An error is raised when logits have invalid shape.""" - with self.cached_session(): - logits = constant_op.constant([-1.0, 2.1], shape=(2,)) - labels = constant_op.constant([0, 1]) - with self.assertRaises(ValueError): - _ = losses.sparse_multiclass_hinge_loss(labels, logits) - - def testInvalidLabelsShape(self): - """An error is raised when labels have invalid shape.""" - with self.cached_session(): - logits = constant_op.constant([-1.0, 2.1], shape=(2, 1)) - labels = constant_op.constant([1, 0], shape=(1, 1, 2)) - with self.assertRaises(ValueError): - _ = losses.sparse_multiclass_hinge_loss(labels, logits) - - def testInvalidWeightsShape(self): - """An error is raised when weights have invalid shape.""" - with self.cached_session(): - logits = constant_op.constant([-1.0, 2.1], shape=(2, 1)) - labels = constant_op.constant([1, 0], shape=(2,)) - weights = constant_op.constant([1.5, 0.2], shape=(2, 1, 1)) - with self.assertRaises(ValueError): - _ = losses.sparse_multiclass_hinge_loss(labels, logits, weights) - - def testInvalidLabelsDtype(self): - """An error is raised when labels have invalid shape.""" - with self.cached_session(): - logits = constant_op.constant([-1.0, 2.1], shape=(2, 1)) - labels = constant_op.constant([1, 0], dtype=dtypes.float32) - with self.assertRaises(ValueError): - _ = losses.sparse_multiclass_hinge_loss(labels, logits) - - def testNoneWeightRaisesValueError(self): - """An error is raised when weights are None.""" - with self.cached_session(): - logits = constant_op.constant([-1.0, 2.1], shape=(2, 1)) - labels = constant_op.constant([1, 0]) - with self.assertRaises(ValueError): - _ = losses.sparse_multiclass_hinge_loss(labels, logits, weights=None) - - def testInconsistentLabelsAndWeightsShapesSameRank(self): - """Error raised when weights and labels have same ranks, different sizes.""" - with self.cached_session(): - logits = constant_op.constant([-1.0, 2.1, 4.1], shape=(3, 1)) - labels = constant_op.constant([1, 0, 2], shape=(3, 1)) - weights = constant_op.constant([1.1, 2.0], shape=(2, 1)) - with self.assertRaises(ValueError): - _ = losses.sparse_multiclass_hinge_loss(labels, logits, weights) - - def testInconsistentLabelsAndWeightsShapesDifferentRank(self): - """Error raised when weights and labels have different ranks and sizes.""" - with self.cached_session(): - logits = constant_op.constant([-1.0, 2.1], shape=(2, 1)) - labels = constant_op.constant([1, 0], shape=(2, 1)) - weights = constant_op.constant([1.1, 2.0, 2.8], shape=(3,)) - with self.assertRaises(ValueError): - _ = losses.sparse_multiclass_hinge_loss(labels, logits, weights) - - def testOutOfRangeLabels(self): - """An error is raised when labels are not in [0, num_classes).""" - with self.cached_session(): - logits = constant_op.constant([[1.2, -1.4, -1.0], [1.4, 1.8, 4.0], - [0.5, 1.8, -1.0]]) - labels = constant_op.constant([1, 0, 4]) - loss = losses.sparse_multiclass_hinge_loss(labels, logits) - with self.assertRaises(errors.InvalidArgumentError): - loss.eval() - - def testZeroLossInt32Labels(self): - """Loss is 0 if true class logits sufficiently higher than other classes.""" - with self.cached_session(): - logits = constant_op.constant([[1.2, -1.4, -1.0], [1.4, 1.8, 4.0], - [0.5, 1.8, -1.0]]) - labels = constant_op.constant([0, 2, 1], dtype=dtypes.int32) - loss = losses.sparse_multiclass_hinge_loss(labels, logits) - self.assertAlmostEqual(loss.eval(), 0.0, 3) - - def testZeroLossInt64Labels(self): - """Loss is 0 if true class logits sufficiently higher than other classes.""" - with self.cached_session(): - logits = constant_op.constant([[2.1, -0.4, -1.0], [1.4, 2.8, 4.0], - [-0.5, 0.8, -1.0]]) - labels = constant_op.constant([0, 2, 1], dtype=dtypes.int64) - loss = losses.sparse_multiclass_hinge_loss(labels, logits) - self.assertAlmostEqual(loss.eval(), 0.0, 3) - - def testUnknownShape(self): - """Result keeps same with `testZeroLossInt32Labels`""" - logits_np = np.array([[1.2, -1.4, -1.0], [1.4, 1.8, 4.0], [0.5, 1.8, -1.0]]) - labels_np = np.array([0, 2, 1], dtype=np.int32) - - logits_shapes = [ - [3, 3], # batch_size, num_classes - [None, 3], - [3, None], - [None, None] - ] - - for batch_size, num_classes in logits_shapes: - with self.cached_session(): - logits = array_ops.placeholder( - dtypes.float32, shape=(batch_size, num_classes)) - labels = array_ops.placeholder(dtypes.int32, shape=(batch_size,)) - loss = losses.sparse_multiclass_hinge_loss(labels, logits) - result = loss.eval(feed_dict={logits: logits_np, labels: labels_np}) - self.assertAlmostEqual(result, 0.0, 3) - - def testCorrectPredictionsSomeClassesInsideMargin(self): - """Loss is > 0 even if true class logits are higher than other classes.""" - with self.cached_session(): - logits = constant_op.constant([[1.2, -1.4, 0.8], [1.4, 1.8, 4.0], - [1.5, 1.8, -1.0]]) - labels = constant_op.constant([0, 2, 1]) - loss = losses.sparse_multiclass_hinge_loss(labels, logits) - # The first and third samples incur some loss (0.6 and 0.7 respectively). - self.assertAlmostEqual(loss.eval(), 0.4333, 3) - - def testIncorrectPredictions(self): - """Loss is >0 when an incorrect class has higher logits than true class.""" - with self.cached_session(): - logits = constant_op.constant([[2.6, 0.4, 0.8], [1.4, 0.8, -1.0], - [0.5, -1.8, 2.0]]) - labels = constant_op.constant([1, 0, 2]) - loss = losses.sparse_multiclass_hinge_loss(labels, logits) - # The first examples incurs a high loss (3.2) since the logits of an - # incorrect class (0) are higher than the logits of the ground truth. The - # second example also incures a (smaller) loss (0.4). - self.assertAlmostEqual(loss.eval(), 1.2, 3) - - def testIncorrectPredictionsColumnLabels(self): - """Same as above but labels is a rank-2 tensor.""" - with self.cached_session(): - logits = constant_op.constant([[1.6, -0.4, 0.8], [1.5, 0.8, -1.0], - [0.2, -1.8, 4.0]]) - labels = constant_op.constant([1, 0, 2], shape=(3, 1)) - loss = losses.sparse_multiclass_hinge_loss(labels, logits) - # The first examples incurs a high loss (3.0) since the logits of an - # incorrect class (0) are higher than the logits of the ground truth. The - # second example also incures a (smaller) loss (0.3). - self.assertAlmostEqual(loss.eval(), 1.1, 3) - - def testIncorrectPredictionsZeroWeights(self): - """Loss is 0 when all weights are missing even if predictions are wrong.""" - with self.cached_session(): - logits = constant_op.constant([[1.6, -0.4, 0.8], [1.5, 0.8, -1.0], - [0.2, -1.8, 4.0]]) - labels = constant_op.constant([1, 0, 2], shape=(3, 1)) - weights = constant_op.constant([0.0, 0.0, 0.0], shape=(3, 1)) - loss = losses.sparse_multiclass_hinge_loss(labels, logits, weights) - # No overall loss since all weights are 0. - self.assertAlmostEqual(loss.eval(), 0.0, 3) - - def testNonZeroLossWithPythonScalarWeights(self): - """Weighted loss is correctly computed when weights is a python scalar.""" - with self.cached_session(): - logits = constant_op.constant([[1.6, -0.4, 0.8], [1.5, 0.8, -1.0], - [0.2, -1.8, 4.0]]) - labels = constant_op.constant([1, 0, 2], shape=(3, 1)) - weights = 10.0 - loss = losses.sparse_multiclass_hinge_loss(labels, logits, weights) - self.assertAlmostEqual(loss.eval(), 11.0, 3) - - def testNonZeroLossWithScalarTensorWeights(self): - """Weighted loss is correctly computed when weights is a rank-0 tensor.""" - with self.cached_session(): - logits = constant_op.constant([[1.6, -0.4, 0.8], [1.5, 0.8, -1.0], - [0.2, -1.8, 4.0]]) - labels = constant_op.constant([1, 0, 2], shape=(3, 1)) - weights = constant_op.constant(5.0) - loss = losses.sparse_multiclass_hinge_loss(labels, logits, weights) - self.assertAlmostEqual(loss.eval(), 5.5, 3) - - def testNonZeroLossWith1DTensorWeightsColumnLabels(self): - """Weighted loss is correctly computed when weights is a rank-0 tensor.""" - with self.cached_session(): - logits = constant_op.constant([[1.6, -0.4, 0.8], [1.5, 0.8, -1.0], - [0.2, -1.8, 4.0]]) - labels = constant_op.constant([1, 0, 2], shape=(3, 1)) - weights = constant_op.constant([1.0, 0.5, 2.0], shape=(3,)) - loss = losses.sparse_multiclass_hinge_loss(labels, logits, weights) - # The overall loss is 1/3 *(3.0*1.0 + 0.5*0.3+ 2.0*0.0) = 1.05 - self.assertAlmostEqual(loss.eval(), 1.05, 3) - - def testNonZeroLossWith2DTensorWeights1DLabelsSomeWeightsMissing(self): - """Weighted loss is correctly computed when weights is a rank-0 tensor.""" - with self.cached_session(): - logits = constant_op.constant([[1.6, -0.4, 0.8], [1.5, 0.8, -1.0], - [0.2, -1.8, 4.0], [1.6, 1.8, -4.0]]) - labels = constant_op.constant([1, 0, 2, 1]) - weights = constant_op.constant([[1.0], [0.0], [2.0], [4.0]]) - loss = losses.sparse_multiclass_hinge_loss(labels, logits, weights) - # The overall loss is 1/3 *(3.0*1.0 + 0.0*0.3+ 2.0*0.0 + 4.0*0.8) = 6.2/3. - self.assertAlmostEqual(loss.eval(), 2.06666, 3) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/kernel_methods/python/mappers/dense_kernel_mapper.py b/tensorflow/contrib/kernel_methods/python/mappers/dense_kernel_mapper.py deleted file mode 100644 index 04ecdbfdb66..00000000000 --- a/tensorflow/contrib/kernel_methods/python/mappers/dense_kernel_mapper.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""API class for dense (approximate) kernel mappers. - -See ./random_fourier_features.py for a concrete instantiation of this class. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - - -class InvalidShapeError(Exception): - """Exception thrown when a tensor's shape deviates from an expected shape.""" - - -@six.add_metaclass(abc.ABCMeta) -class DenseKernelMapper(object): - """Abstract class for a kernel mapper that maps dense inputs to dense outputs. - - This class is abstract. Users should not create instances of this class. - """ - - @abc.abstractmethod - def map(self, input_tensor): - """Main Dense-Tensor-In-Dense-Tensor-Out (DTIDTO) map method. - - Should be implemented by subclasses. - Args: - input_tensor: The dense input tensor to be mapped using the (approximate) - kernel mapper. - """ - raise NotImplementedError('map is not implemented for {}.'.format(self)) - - @abc.abstractproperty - def name(self): - """Returns the name of the kernel mapper.""" - pass - - @abc.abstractproperty - def output_dim(self): - """Returns the output dimension of the mapping.""" - pass diff --git a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py b/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py deleted file mode 100644 index 9a721a9d440..00000000000 --- a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Approximate kernel mapper for RBF kernel based on Random Fourier Features.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -import numpy as np - -from tensorflow.contrib.kernel_methods.python.mappers import dense_kernel_mapper as dkm -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import math_ops - - -# TODO(sibyl-vie3Poto,felixyu): add an option to control whether the parameters in the -# kernel map are trainable. -class RandomFourierFeatureMapper(dkm.DenseKernelMapper): - r"""Class that implements Random Fourier Feature Mapping (RFFM) in TensorFlow. - - The RFFM mapping is used to approximate the Gaussian (RBF) kernel: - $$(exp(-||x-y||_2^2 / (2 * \sigma^2))$$ - - The implementation of RFFM is based on the following paper: - "Random Features for Large-Scale Kernel Machines" by Ali Rahimi and Ben Recht. - (link: https://people.eecs.berkeley.edu/~brecht/papers/07.rah.rec.nips.pdf) - - The mapping uses a matrix \\(\Omega \in R^{d x D}\\) and a bias vector - \\(b \in R^D\\) where \\(d\\) is the input dimension (number of dense input - features) and \\(D\\) is the output dimension (i.e., dimension of the feature - space the input is mapped to). Each entry of \\(\Omega\\) is sampled i.i.d. - from a (scaled) Gaussian distribution and each entry of \\(b\\) is sampled - independently and uniformly from [0, \\(2 * \pi\\)]. - - For a single input feature vector \\(x \in R^d\\), its RFFM is defined as: - $$\sqrt(2/D) * cos(x * \Omega + b)$$ - - where \\(cos\\) is the element-wise cosine function and \\(x, b\\) are - represented as row vectors. The aforementioned paper shows that the linear - kernel of RFFM-mapped vectors approximates the Gaussian kernel of the initial - vectors. - - """ - - def __init__(self, input_dim, output_dim, stddev=1.0, seed=1, name=None): - r"""Constructs a RandomFourierFeatureMapper instance. - - Args: - input_dim: The dimension (number of features) of the tensors to be mapped. - output_dim: The output dimension of the mapping. - stddev: The standard deviation of the Gaussian kernel to be approximated. - The error of the classifier trained using this approximation is very - sensitive to this parameter. - seed: An integer used to initialize the parameters (\\(\Omega\\) and - \\(b\\)) of the mapper. For repeatable sequences across different - invocations of the mapper object (for instance, to ensure consistent - mapping both at training and eval/inference if these happen in - different invocations), set this to the same integer. - name: name for the mapper object. - """ - # TODO(sibyl-vie3Poto): Maybe infer input_dim and/or output_dim (if not explicitly - # provided). input_dim can be inferred lazily, the first time map is called. - # output_dim can be inferred from input_dim using heuristics on the error of - # the approximation (and, by extension, the error of the classification - # based on the approximation). - self._input_dim = input_dim - self._output_dim = output_dim - self._stddev = stddev - self._seed = seed - self._name = name - - @property - def name(self): - """Returns a name for the `RandomFourierFeatureMapper` instance. - - If the name provided in the constructor is `None`, then the object's unique - id is returned. - - Returns: - A name for the `RandomFourierFeatureMapper` instance. - """ - return self._name or str(id(self)) - - @property - def input_dim(self): - return self._input_dim - - @property - def output_dim(self): - return self._output_dim - - def map(self, input_tensor): - """Maps each row of input_tensor using random Fourier features. - - Args: - input_tensor: a `Tensor` containing input features. It's shape is - [batch_size, self._input_dim]. - - Returns: - A `Tensor` of shape [batch_size, self._output_dim] containing RFFM-mapped - features. - - Raises: - InvalidShapeError: if the shape of the `input_tensor` is inconsistent with - expected input dimension. - """ - input_tensor_shape = input_tensor.get_shape() - if len(input_tensor_shape) != 2: - raise dkm.InvalidShapeError( - 'The shape of the tensor should be 2. Got %d instead.' % - len(input_tensor_shape)) - - features_dim = input_tensor_shape[1] - if features_dim != self._input_dim: - raise dkm.InvalidShapeError( - 'Invalid dimension: expected %d input features, got %d instead.' % - (self._input_dim, features_dim)) - - # Add ops that compute (deterministically) omega_matrix and bias based on - # the provided seed. - # TODO(sibyl-vie3Poto): Storing the mapper's parameters (omega_matrix and bias) as - # constants incurs no RPC calls to the parameter server during distributed - # training. However, if the parameters grow too large (for instance if they - # don't fit into memory or if they blow up the size of the GraphDef proto), - # stroring them as constants is no longer an option. In this case, we should - # have a heuristic to choose out of one of the following alternatives: - # a) store them as variables (in the parameter server) - # b) store them as worker local variables - # c) generating on the fly the omega matrix at each step - np.random.seed(self._seed) - omega_matrix_shape = [self._input_dim, self._output_dim] - bias_shape = [self._output_dim] - - omega_matrix = constant_op.constant( - np.random.normal( - scale=1.0 / self._stddev, size=omega_matrix_shape), - dtype=dtypes.float32) - bias = constant_op.constant( - np.random.uniform( - low=0.0, high=2 * np.pi, size=bias_shape), - dtype=dtypes.float32) - - x_omega_plus_bias = math_ops.add( - math_ops.matmul(input_tensor, omega_matrix), bias) - return math.sqrt(2.0 / self._output_dim) * math_ops.cos(x_omega_plus_bias) diff --git a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features_test.py b/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features_test.py deleted file mode 100644 index bad0a596a78..00000000000 --- a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features_test.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for RandomFourierFeatureMapper.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib.kernel_methods.python.mappers import dense_kernel_mapper -from tensorflow.contrib.kernel_methods.python.mappers.random_fourier_features import RandomFourierFeatureMapper -from tensorflow.python.framework import constant_op -from tensorflow.python.framework.test_util import TensorFlowTestCase -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import googletest - - -def _inner_product(x, y): - r"""Inner product between tensors x and y. - - The input tensors are assumed to be in ROW representation, that is, the method - returns \\(x * y^T\\). - - Args: - x: input tensor in row format - y: input tensor in row format - - Returns: - the inner product of x, y - """ - return math_ops.matmul(x, y, transpose_b=True) - - -def _compute_exact_rbf_kernel(x, y, stddev): - """Computes exact RBF kernel given input tensors x and y and stddev.""" - diff = math_ops.subtract(x, y) - diff_squared_norm = _inner_product(diff, diff) - return math_ops.exp(-diff_squared_norm / (2 * stddev * stddev)) - - -class RandomFourierFeatureMapperTest(TensorFlowTestCase): - - def testInvalidInputShape(self): - x = constant_op.constant([[2.0, 1.0]]) - - with self.cached_session(): - rffm = RandomFourierFeatureMapper(3, 10) - with self.assertRaisesWithPredicateMatch( - dense_kernel_mapper.InvalidShapeError, - r'Invalid dimension: expected 3 input features, got 2 instead.'): - rffm.map(x) - - def testMappedShape(self): - x1 = constant_op.constant([[2.0, 1.0, 0.0]]) - x2 = constant_op.constant([[1.0, -1.0, 2.0], [-1.0, 10.0, 1.0], - [4.0, -2.0, -1.0]]) - - with self.cached_session(): - rffm = RandomFourierFeatureMapper(3, 10, 1.0) - mapped_x1 = rffm.map(x1) - mapped_x2 = rffm.map(x2) - self.assertEqual([1, 10], mapped_x1.get_shape()) - self.assertEqual([3, 10], mapped_x2.get_shape()) - - def testSameOmegaReused(self): - x = constant_op.constant([[2.0, 1.0, 0.0]]) - - with self.cached_session(): - rffm = RandomFourierFeatureMapper(3, 100) - mapped_x = rffm.map(x) - mapped_x_copy = rffm.map(x) - # Two different evaluations of tensors output by map on the same input - # are identical because the same parameters are used for the mappings. - self.assertAllClose(mapped_x.eval(), mapped_x_copy.eval(), atol=0.001) - - def testTwoMapperObjects(self): - x = constant_op.constant([[2.0, 1.0, 0.0]]) - y = constant_op.constant([[1.0, -1.0, 2.0]]) - stddev = 3.0 - - with self.cached_session(): - # The mapped dimension is fairly small, so the kernel approximation is - # very rough. - rffm1 = RandomFourierFeatureMapper(3, 100, stddev) - rffm2 = RandomFourierFeatureMapper(3, 100, stddev) - mapped_x1 = rffm1.map(x) - mapped_y1 = rffm1.map(y) - mapped_x2 = rffm2.map(x) - mapped_y2 = rffm2.map(y) - - approx_kernel_value1 = _inner_product(mapped_x1, mapped_y1) - approx_kernel_value2 = _inner_product(mapped_x2, mapped_y2) - self.assertAllClose( - approx_kernel_value1.eval(), approx_kernel_value2.eval(), atol=0.01) - - def testBadKernelApproximation(self): - x = constant_op.constant([[2.0, 1.0, 0.0]]) - y = constant_op.constant([[1.0, -1.0, 2.0]]) - stddev = 3.0 - - with self.cached_session(): - # The mapped dimension is fairly small, so the kernel approximation is - # very rough. - rffm = RandomFourierFeatureMapper(3, 100, stddev, seed=0) - mapped_x = rffm.map(x) - mapped_y = rffm.map(y) - exact_kernel_value = _compute_exact_rbf_kernel(x, y, stddev) - approx_kernel_value = _inner_product(mapped_x, mapped_y) - self.assertAllClose( - exact_kernel_value.eval(), approx_kernel_value.eval(), atol=0.2) - - def testGoodKernelApproximationAmortized(self): - # Parameters. - num_points = 20 - input_dim = 5 - mapped_dim = 5000 - stddev = 5.0 - - points_shape = [1, input_dim] - points = [ - random_ops.random_uniform(shape=points_shape, maxval=1.0) - for _ in xrange(num_points) - ] - - normalized_points = [nn.l2_normalize(point, dim=1) for point in points] - total_absolute_error = 0.0 - with self.cached_session(): - rffm = RandomFourierFeatureMapper(input_dim, mapped_dim, stddev, seed=0) - # Cache mappings so that they are not computed multiple times. - cached_mappings = dict((point, rffm.map(point)) - for point in normalized_points) - for x in normalized_points: - mapped_x = cached_mappings[x] - for y in normalized_points: - mapped_y = cached_mappings[y] - exact_kernel_value = _compute_exact_rbf_kernel(x, y, stddev) - approx_kernel_value = _inner_product(mapped_x, mapped_y) - abs_error = math_ops.abs(exact_kernel_value - approx_kernel_value) - total_absolute_error += abs_error - self.assertAllClose( - [[0.0]], - total_absolute_error.eval() / (num_points * num_points), - atol=0.02) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/kfac/README.md b/tensorflow/contrib/kfac/README.md deleted file mode 100644 index 19daffea6c7..00000000000 --- a/tensorflow/contrib/kfac/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# K-FAC: Kronecker-Factored Approximate Curvature - -## KFAC moved to https://github.com/tensorflow/kfac. diff --git a/tensorflow/contrib/labeled_tensor/BUILD b/tensorflow/contrib/labeled_tensor/BUILD deleted file mode 100644 index 7ec7fa8a8cb..00000000000 --- a/tensorflow/contrib/labeled_tensor/BUILD +++ /dev/null @@ -1,225 +0,0 @@ -# Description: -# Labels for TensorFlow. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "labeled_tensor", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - deps = [ - ":core", - ":io_ops", - ":nn", - ":ops", - ":sugar", - ], -) - -# Transitive dependencies of this target will be included in the pip package. -py_library( - name = "labeled_tensor_pip", - deps = [ - ":labeled_tensor", - ":test_util", - ], -) - -py_library( - name = "_typecheck", - srcs = ["python/ops/_typecheck.py"], - srcs_version = "PY2AND3", - visibility = [":__subpackages__"], - deps = ["//tensorflow/python:util"], -) - -py_library( - name = "core", - srcs = ["python/ops/core.py"], - srcs_version = "PY2AND3", - deps = [ - ":_typecheck", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_library( - name = "test_util", - srcs = ["python/ops/test_util.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:client_testlib", - "//tensorflow/python:training", - ], -) - -py_test( - name = "core_test", - size = "medium", - srcs = [ - "python/ops/core_test.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_windows", # TODO: needs investigation on Windows - "noasan", # TODO(b/119323169) - ], - deps = [ - ":_typecheck", - ":core", - ":test_util", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//third_party/py/numpy", - ], -) - -py_library( - name = "io_ops", - srcs = ["python/ops/io_ops.py"], - srcs_version = "PY2AND3", - deps = [ - ":_typecheck", - ":core", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:parsing_ops", - "@six_archive//:six", - ], -) - -py_test( - name = "io_ops_test", - size = "small", - srcs = [ - "python/ops/io_ops_test.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":core", - ":io_ops", - ":test_util", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:session", - ], -) - -py_library( - name = "nn", - srcs = ["python/ops/nn.py"], - srcs_version = "PY2AND3", - deps = [ - ":core", - "//tensorflow/contrib/nn:nn_py", - "//tensorflow/python:nn", - ], -) - -py_test( - name = "nn_test", - size = "small", - srcs = [ - "python/ops/nn_test.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":core", - ":nn", - ":test_util", - "//tensorflow/python:nn", - "//tensorflow/python:nn_ops", - ], -) - -py_library( - name = "ops", - srcs = ["python/ops/ops.py"], - srcs_version = "PY2AND3", - deps = [ - ":_typecheck", - ":core", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:map_fn", - "//tensorflow/python:math_ops", - "//tensorflow/python:numerics", - "//tensorflow/python:random_ops", - "//tensorflow/python:training", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_test( - name = "ops_test", - size = "medium", - srcs = [ - "python/ops/ops_test.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":core", - ":ops", - ":test_util", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:string_ops", - "//third_party/py/numpy", - ], -) - -py_library( - name = "sugar", - srcs = ["python/ops/sugar.py"], - srcs_version = "PY2AND3", - deps = [ - ":_typecheck", - ":core", - ":ops", - "//tensorflow/python:framework_for_generated_wrappers", - "@six_archive//:six", - ], -) - -py_test( - name = "sugar_test", - size = "small", - srcs = [ - "python/ops/sugar_test.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":core", - ":ops", - ":sugar", - ":test_util", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - ], -) diff --git a/tensorflow/contrib/labeled_tensor/README.md b/tensorflow/contrib/labeled_tensor/README.md deleted file mode 100644 index adce979e2ac..00000000000 --- a/tensorflow/contrib/labeled_tensor/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Labels for TensorFlow - -LabeledTensor is a library for adding semantically meaningful dimension and -coordinate labels to tensors in Tensorflow. - -LabeledTensor was inspired by [xarray](http://xarray.pydata.org) and -[pandas](http://pandas.pydata.org), projects that adds labels to NumPy array. - -## Data model - -`LabeledTensor` is an immutable object consisting of two components: - -- `tensor`: the `tf.Tensor` object containing the labeled tensor's data. -- `axes`: an OrderedDict-like object with keys given by axis names (e.g., - ``"channel"``) and values given by `Axis` objects. - -`Axis` objects keep track of the size of a dimension and, optionally, coordinate -labels along that axis (e.g., `("red", "green", "blue")`) in the form of a -tuple stored in `Axis.labels`. - -Operations on `LabeledTensors` use, preserve and transform axis names and -labels. - -## Quick start - -Try out the following snippet in a script or Jupyter notebook: - - import tensorflow as tf - - lt = tf.contrib.labeled_tensor - - # Create two LabeledTensors: - raw_image = tf.ones((299, 299, 3)) - axes = ['row', 'column', ('channel', ['red', 'green', 'blue'])] - image = lt.LabeledTensor(raw_image, axes) - assert image.tensor is raw_image - weights = lt.LabeledTensor(tf.constant([0.1, 0.3, 0.6]), - [image.axes['channel']]) - - # Examples of valid operations: - lt.transpose(image, ['column', 'row', 'channel']) - lt.reshape(image, ['row', 'column'], ['pixel']) - lt.concat([image, image], 'row') - lt.reduce_sum(image, ['channel']) - lt.select(image, {'channel': 'red'}) - lt.cast(image / 256.0, tf.uint8) - image * weights - lt.matmul(image[0, :, :], weights) - tf.cos(image) # automatically converts to tf.Tensor - -## Adding a custom op - -LabeledTensor has wrappers for [quite a -few](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/labeled_tensor/__init__.py) -TensorFlow ops. - -To easily add your own, you can use the `define_unary_op`, `define_binary_op` -and `define_reduce_op` functions, e.g., - - log = lt.define_unary_op('log', tf.log) - -## Questions - -Please reach out to the authors: - -- Stephan Hoyer (shoyer@google.com, github.com/shoyer) -- Eric Christiansen (ericmc@google.com, github.com/emchristiansen) diff --git a/tensorflow/contrib/labeled_tensor/__init__.py b/tensorflow/contrib/labeled_tensor/__init__.py deleted file mode 100644 index d19811d49ca..00000000000 --- a/tensorflow/contrib/labeled_tensor/__init__.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Labels for TensorFlow.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.labeled_tensor.python.ops import core as _core -from tensorflow.contrib.labeled_tensor.python.ops import io_ops as _io_ops -from tensorflow.contrib.labeled_tensor.python.ops import nn -from tensorflow.contrib.labeled_tensor.python.ops import ops as _ops -from tensorflow.contrib.labeled_tensor.python.ops import sugar as _sugar - -# pylint: disable=invalid-name - -# Core types. -Axis = _core.Axis -Axes = _core.Axes -LabeledTensor = _core.LabeledTensor - -as_axis = _core.as_axis -convert_to_labeled_tensor = _core.convert_to_labeled_tensor - -identity = _core.identity -slice = _core.slice_function # pylint: disable=redefined-builtin -transpose = _core.transpose -expand_dims = _core.expand_dims -align = _core.align - -axis_order_scope = _core.axis_order_scope -check_axis_order = _core.check_axis_order -impose_axis_order = _core.impose_axis_order -AxisOrderError = _core.AxisOrderError - -define_unary_op = _core.define_unary_op -define_binary_op = _core.define_binary_op -define_reduce_op = _ops.define_reduce_op - -abs = _core.abs_function # pylint: disable=redefined-builtin -neg = _core.neg -sign = _core.sign -reciprocal = _core.reciprocal -square = _core.square -round = _core.round_function # pylint: disable=redefined-builtin -sqrt = _core.sqrt -rsqrt = _core.rsqrt -exp = _core.exp -log = _core.log -ceil = _core.ceil -floor = _core.floor -cos = _core.cos -sin = _core.sin -tan = _core.tan -acos = _core.acos -asin = _core.asin -atan = _core.atan -lgamma = _core.lgamma -digamma = _core.digamma -erf = _core.erf -erfc = _core.erfc -logical_not = _core.logical_not -tanh = _core.tanh -sigmoid = _core.sigmoid - -add = _core.add -sub = _core.sub -mul = _core.mul -div = _core.div -mod = _core.mod -pow = _core.pow_function # pylint: disable=redefined-builtin - -equal = _core.equal -greater = _core.greater -greater_equal = _core.greater_equal -not_equal = _core.not_equal -less = _core.less -less_equal = _core.less_equal -logical_and = _core.logical_and -logical_or = _core.logical_or -logical_xor = _core.logical_xor - -maximum = _core.maximum -minimum = _core.minimum -squared_difference = _core.squared_difference -igamma = _core.igamma -igammac = _core.igammac -zeta = _core.zeta -polygamma = _core.polygamma - -select = _ops.select -concat = _ops.concat -pack = _ops.pack -unpack = _ops.unpack -reshape = _ops.reshape -rename_axis = _ops.rename_axis -random_crop = _ops.random_crop -map_fn = _ops.map_fn -foldl = _ops.foldl -squeeze = _ops.squeeze -matmul = _ops.matmul -tile = _ops.tile -pad = _ops.pad -constant = _ops.constant -zeros_like = _ops.zeros_like -ones_like = _ops.ones_like -cast = _ops.cast -verify_tensor_all_finite = _ops.verify_tensor_all_finite -boolean_mask = _ops.boolean_mask -where = _ops.where - -reduce_all = _ops.reduce_all -reduce_any = _ops.reduce_any -reduce_logsumexp = _ops.reduce_logsumexp -reduce_max = _ops.reduce_max -reduce_mean = _ops.reduce_mean -reduce_min = _ops.reduce_min -reduce_prod = _ops.reduce_prod -reduce_sum = _ops.reduce_sum - -batch = _ops.batch -shuffle_batch = _ops.shuffle_batch - -FixedLenFeature = _io_ops.FixedLenFeature -parse_example = _io_ops.parse_example -parse_single_example = _io_ops.parse_single_example -placeholder = _io_ops.placeholder - -ReshapeCoder = _sugar.ReshapeCoder diff --git a/tensorflow/contrib/labeled_tensor/python/ops/_typecheck.py b/tensorflow/contrib/labeled_tensor/python/ops/_typecheck.py deleted file mode 100644 index 3a257d81887..00000000000 --- a/tensorflow/contrib/labeled_tensor/python/ops/_typecheck.py +++ /dev/null @@ -1,319 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Minimal runtime type checking library. - -This module should not be considered public API. -""" -# TODO(ericmc,shoyer): Delete this in favor of using pytype or mypy -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import re - -from tensorflow.python.util import tf_inspect -from tensorflow.python.util.compat import collections_abc - -# used for register_type_abbreviation and _type_repr below. -_TYPE_ABBREVIATIONS = {} - - -class Type(object): - """Base class for type checker types. - - The custom types defined in this module are based on types in the standard - library's typing module (in Python 3.5): - https://docs.python.org/3/library/typing.html - - The only difference should be that we use actual instances of Type classes to - represent custom types rather than the metaclass magic typing uses to create - new class objects. In practice, all this should mean is that we use - `List(int)` rather than `List[int]`. - - Custom types should implement __instancecheck__ and inherit from Type. Every - argument in the constructor must be a type or Type instance, and these - arguments must be stored as a tuple on the `_types` attribute. - """ - - def __init__(self, *types): - self._types = types - - def __repr__(self): - args_repr = ", ".join(repr(t) for t in self._types) - return "typecheck.%s(%s)" % (type(self).__name__, args_repr) - - -class _SingleArgumentType(Type): - """Use this subclass for parametric types that accept only one argument.""" - - def __init__(self, tpe): - super(_SingleArgumentType, self).__init__(tpe) - - @property - def _type(self): - tpe, = self._types # pylint: disable=unbalanced-tuple-unpacking - return tpe - - -class _TwoArgumentType(Type): - """Use this subclass for parametric types that accept two arguments.""" - - def __init__(self, first_type, second_type): - super(_TwoArgumentType, self).__init__(first_type, second_type) - - -class Union(Type): - """A sum type. - - A correct type is any of the types provided. - """ - - def __instancecheck__(self, instance): - return isinstance(instance, self._types) - - -class Optional(_SingleArgumentType): - """An optional type. - - A correct type is either the provided type or NoneType. - """ - - def __instancecheck__(self, instance): - # types.NoneType does not exist in Python 3 - return isinstance(instance, (self._type, type(None))) - - -class List(_SingleArgumentType): - """A typed list. - - A correct type is a list where each element has the single provided type. - """ - - def __instancecheck__(self, instance): - return (isinstance(instance, list) and - all(isinstance(x, self._type) for x in instance)) - - -class Sequence(_SingleArgumentType): - """A typed sequence. - - A correct type is a sequence where each element has the single provided type. - """ - - def __instancecheck__(self, instance): - return (isinstance(instance, collections_abc.Sequence) and - all(isinstance(x, self._type) for x in instance)) - - -class Collection(_SingleArgumentType): - """A sized, iterable container. - - A correct type is an iterable and container with known size where each element - has the single provided type. - - We use this in preference to Iterable because we check each instance of the - iterable at runtime, and hence need to avoid iterables that could be - exhausted. - """ - - def __instancecheck__(self, instance): - return (isinstance(instance, collections_abc.Iterable) and - isinstance(instance, collections_abc.Sized) and - isinstance(instance, collections_abc.Container) and - all(isinstance(x, self._type) for x in instance)) - - -class Tuple(Type): - """A typed tuple. - - A correct type is a tuple with the correct length where each element has - the correct type. - """ - - def __instancecheck__(self, instance): - return (isinstance(instance, tuple) and - len(instance) == len(self._types) and - all(isinstance(x, t) for x, t in zip(instance, self._types))) - - -class Mapping(_TwoArgumentType): - """A typed mapping. - - A correct type has the correct parametric types for keys and values. - """ - - def __instancecheck__(self, instance): - key_type, value_type = self._types # pylint: disable=unbalanced-tuple-unpacking - return (isinstance(instance, collections_abc.Mapping) and - all(isinstance(k, key_type) for k in instance.keys()) and - all(isinstance(k, value_type) for k in instance.values())) - - -class Dict(Mapping): - """A typed dict. - - A correct type has the correct parametric types for keys and values. - """ - - def __instancecheck__(self, instance): - return (isinstance(instance, dict) and - super(Dict, self).__instancecheck__(instance)) - - -def _replace_forward_references(t, context): - """Replace forward references in the given type.""" - if isinstance(t, str): - return context[t] - elif isinstance(t, Type): - return type(t)(*[_replace_forward_references(t, context) for t in t._types]) # pylint: disable=protected-access - else: - return t - - -def register_type_abbreviation(name, alias): - """Register an abbreviation for a type in typecheck tracebacks. - - This makes otherwise very long typecheck errors much more readable. - - Example: - typecheck.register_type_abbreviation(tf.compat.v1.Dimension, - 'tf.compat.v1.Dimension') - - Args: - name: type or class to abbreviate. - alias: string alias to substitute. - """ - _TYPE_ABBREVIATIONS[name] = alias - - -def _type_repr(t): - """A more succinct repr for typecheck tracebacks.""" - string = repr(t) - for type_, alias in _TYPE_ABBREVIATIONS.items(): - string = string.replace(repr(type_), alias) - string = re.sub(r"<(class|type) '([\w.]+)'>", r"\2", string) - string = re.sub(r"typecheck\.(\w+)", r"\1", string) - return string - - -class Error(TypeError): - """Exception for typecheck failures.""" - - -def accepts(*types): - """A decorator which checks the input types of a function. - - Based on: - http://stackoverflow.com/questions/15299878/how-to-use-python-decorators-to-check-function-arguments - The above draws from: - https://www.python.org/dev/peps/pep-0318/ - - Args: - *types: A list of Python types. - - Returns: - A function to use as a decorator. - """ - - def check_accepts(f): - """Check the types.""" - spec = tf_inspect.getargspec(f) - - num_function_arguments = len(spec.args) - if len(types) != num_function_arguments: - raise Error( - "Function %r has %d arguments but only %d types were provided in the " - "annotation." % (f, num_function_arguments, len(types))) - - if spec.defaults: - num_defaults = len(spec.defaults) - for (name, a, t) in zip(spec.args[-num_defaults:], spec.defaults, - types[-num_defaults:]): - allowed_type = _replace_forward_references(t, f.__globals__) - if not isinstance(a, allowed_type): - raise Error("default argument value %r of type %r is not an instance " - "of the allowed type %s for the %s argument to %r" % - (a, type(a), _type_repr(allowed_type), name, f)) - - @functools.wraps(f) - def new_f(*args, **kwds): - """A helper function.""" - for (a, t) in zip(args, types): - allowed_type = _replace_forward_references(t, f.__globals__) - if not isinstance(a, allowed_type): - raise Error("%r of type %r is not an instance of the allowed type %s " - "for %r" % (a, type(a), _type_repr(allowed_type), f)) - return f(*args, **kwds) - - return new_f - - return check_accepts - - -def returns(*types): - """A decorator which checks the return types of a function. - - Based on: - http://stackoverflow.com/questions/15299878/how-to-use-python-decorators-to-check-function-arguments - The above draws from: - https://www.python.org/dev/peps/pep-0318/ - - Args: - *types: A list of Python types. A list of one element corresponds to a - single return value. A list of several elements corresponds to several - return values. Note that a function with no explicit return value has an - implicit NoneType return and should be annotated correspondingly. - - Returns: - A function to use as a decorator. - """ - - def check_returns(f): - """Check the types.""" - if not types: - raise TypeError("A return type annotation must contain at least one type") - - @functools.wraps(f) - def new_f(*args, **kwds): - """A helper function.""" - return_value = f(*args, **kwds) - - if len(types) == 1: - # The function has a single return value. - allowed_type = _replace_forward_references(types[0], f.__globals__) - if not isinstance(return_value, allowed_type): - raise Error( - "%r of type %r is not an instance of the allowed type %s " - "for %r" % - (return_value, type(return_value), _type_repr(allowed_type), f)) - - else: - if len(return_value) != len(types): - raise Error("Function %r has %d return values but only %d types were " - "provided in the annotation." % - (f, len(return_value), len(types))) - - for (r, t) in zip(return_value, types): - allowed_type = _replace_forward_references(t, f.__globals__) - if not isinstance(r, allowed_type): - raise Error("%r of type %r is not an instance of allowed type %s " - "for %r" % (r, type(r), _type_repr(allowed_type), f)) - - return return_value - - return new_f - - return check_returns diff --git a/tensorflow/contrib/labeled_tensor/python/ops/core.py b/tensorflow/contrib/labeled_tensor/python/ops/core.py deleted file mode 100644 index 394254cbd90..00000000000 --- a/tensorflow/contrib/labeled_tensor/python/ops/core.py +++ /dev/null @@ -1,1199 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Core classes and core ops for LabeledTensor. - -Core ops are ops which will eventually be called by LabeledTensor methods, -and ops which a core op depends upon. -For example, `add` is a core op because we'll eventually support the `+` -operator. -Non-core ops should go in `ops.py`. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import contextlib -import numbers -import types - -import numpy as np -from six import binary_type -from six import string_types -from six import text_type -from six.moves import range # pylint: disable=redefined-builtin - -from tensorflow.contrib.labeled_tensor.python.ops import _typecheck as tc -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.util.compat import collections_abc - -# pylint: disable=invalid-name - -# Types coercible to Axis.labels -# We use this instead of collections_abc.Sequence to exclude strings. -LabelsLike = tc.Union(np.ndarray, range, list, tuple) - -# Types coercible to a tf.compat.v1.Dimension -DimensionLike = tc.Optional(tc.Union(tensor_shape.Dimension, int)) - -# Types usable for axis values -AxisValue = tc.Union(LabelsLike, DimensionLike) - -# Valid scalar values for TensorFlow -Scalar = tc.Union(numbers.Number, bool, binary_type, text_type) - -# pylint: enable=invalid-name - - -class Axis(object): - """Size and label information for an axis. - - Axis contains either a tf.compat.v1.Dimension indicating the size of an axis, - or a tuple of tick labels for the axis. - - If tick labels are provided, they must be unique. - """ - - @tc.accepts(object, string_types, AxisValue) - def __init__(self, name, value): - """Construct an Axis. - - Args: - name: Name of the axis. - value: Either None, an int or tf.compat.v1.Dimension giving the size of - the axis, or a sequence that is not a string additionally providing - coordinate (tick) labels. - - Raises: - ValueError: If the user provides labels with duplicate values. - """ - if isinstance(value, tensor_shape.Dimension): - dimension = value - labels = None - elif isinstance(value, int) or value is None: - dimension = tensor_shape.Dimension(value) - labels = None - else: - dimension = tensor_shape.Dimension(len(value)) - labels = tuple(value) - - if dimension.value == 0: - # Treat a zero-length axis as if it has labels. - labels = () - - if labels is not None: - index = dict(zip(labels, range(len(labels)))) - if len(index) != len(labels): - raise ValueError( - 'Tick labels must be unique, but got {}'.format(labels)) - else: - index = None - - self._name = name # type: string_types - self._dimension = dimension # type: tensor_shape.Dimension - self._labels = labels # type: Optional[tuple] - self._index = index # type: Optional[Dict[Any, int]] - - @property - @tc.returns(string_types) - def name(self): - return self._name - - @tc.returns(string_types) - def __repr__(self): - # Axis('x', Dimension(2)) - # TODO(shoyer): make very long reprs more succint? - return "%s('%s', %r)" % (type(self).__name__, self.name, self.value) - - @tc.returns(bool) - def __eq__(self, other): - return (isinstance(other, Axis) and self.name == other.name and - self.size == other.size and self.labels == other.labels) - - def __hash__(self): - return hash((self.name, self.size, self.labels)) - - @tc.returns(bool) - def __ne__(self, other): - return not self == other - - @tc.returns(int) - def __len__(self): - size = self.size - if size is None: - raise ValueError('axis %r has unknown length' % self.name) - return size - - @property - @tc.returns(tc.Optional(tensor_shape.Dimension)) - def dimension(self): - return self._dimension - - @property - @tc.returns(tc.Optional(int)) - def size(self): - return self._dimension.value - - @property - @tc.returns(tc.Union(tuple, tensor_shape.Dimension)) - def value(self): - """Returns the tf.compat.v1.Dimension or tuple specifying axis ticks.""" - if self.labels is None: - return self.dimension - else: - return self.labels - - @property - @tc.returns(tc.Optional(tuple)) - def labels(self): - """Returns the tuple containing coordinate labels, else None.""" - return self._labels - - def index(self, value): - """Returns the integer position of the given tick label.""" - if self._index is None: - raise ValueError('Axis does not have tick labels') - return self._index[value] - - -# tc class for anything that can be coerced into an Axis -# pylint: disable=invalid-name -AxisLike = tc.Union(Axis, tc.Tuple(string_types, AxisValue)) -# pylint: enable=invalid-name - - -@tc.returns(Axis) -@tc.accepts(AxisLike) -def as_axis(axis_data): - """Convert an AxisLike object into an Axis. - - Args: - axis_data: Axis object or tuple (axis_name, axis_value) describing an axis. - - Returns: - Axis object. This may be the original object if axis_data is an Axis. - """ - if isinstance(axis_data, Axis): - axis = axis_data - else: - axis = Axis(*axis_data) - return axis - - -class Axes(collections_abc.Mapping): - """Axis names and indices for a tensor. - - It is an ordered mapping, with keys given by axis name and values given - by Axis objects. Duplicate axis names are not allowed. - """ - - @tc.accepts(object, tc.List(AxisLike)) - def __init__(self, axes): - """Construct an Axes. - - Args: - axes: A list of Axis objects or (axis_name, axis_value) tuples. - - Raises: - ValueError: If the user provides empty or duplicate axis names. - """ - self._axes = collections.OrderedDict() - - for axis_data in axes: - axis = as_axis(axis_data) - - name = axis.name - if name in self._axes: - raise ValueError('Duplicate axis name: %s' % name) - - self._axes[name] = axis - - def __iter__(self): - return iter(self._axes) - - @tc.returns(string_types) - def __repr__(self): - # Axes([('x', Dimension(2)), - # ('y', ['a', 'b', 'c']), - # ('z', Dimension(4))]) - cls_name = type(self).__name__ - values = ["('%s', %r)" % (v.name, v.value) for v in self._axes.values()] - values_repr = (',\n' + ' ' * len(cls_name + '([')).join(values) - return '%s([%s])' % (cls_name, values_repr) - - @tc.returns(Axis) - @tc.accepts(object, string_types) - def __getitem__(self, name): - return self._axes[name] - - @tc.returns(bool) - def __contains__(self, name): - return name in self._axes - - @tc.returns(int) - def __len__(self): - return len(self._axes) - - def __hash__(self): - return hash(tuple(self.items())) - - @tc.accepts(object, string_types) - def remove(self, axis_name): - """Creates a new Axes object without the given axis.""" - if axis_name not in self: - raise KeyError(axis_name) - remaining_axes = [axis for axis in self.values() if axis.name != axis_name] - return Axes(remaining_axes) - - -class LabeledTensor(object): - """A tensor with annotated axes. - - It has the following invariants: - 1) The dimensionality of the tensor is equal to the number of elements - in axes. - 2) The number of coordinate values in the ith dimension is equal to the - size of the tensor in the ith dimension. - - Attributes: - tensor: tf.Tensor containing the data. - axes: lt.Axes containing axis names and coordinate labels. - """ - - @tc.accepts(object, ops.Tensor, - tc.Union(Axes, tc.Collection(tc.Union(string_types, AxisLike)))) - def __init__(self, tensor, axes): - """Construct a LabeledTensor. - - Args: - tensor: The underlying tensor containing the data. - axes: An Axes object, or a collection of strings, Axis objects or tuples - of (name, value) pairs indicating the axes. - - Raises: - ValueError: If the provided axes do not satisfy the class invariants. - """ - self._tensor = tensor - shape = tensor.get_shape() - - if isinstance(axes, Axes): - unvalidated_axes = axes - else: - mutable_axes = [] - - for position, axis_like in enumerate(axes): - if isinstance(axis_like, string_types): - # The coordinates for this axes are unlabeled. - # Infer the size of the axis. - value = shape[position] - axis_like = (axis_like, value) - - mutable_axes.append(axis_like) - - # Construct the Axis object, which will additionally validate the contents - # of the object. - unvalidated_axes = Axes(mutable_axes) - - # Check our invariants. - - # First, the rank of the tensor must be equal to the number of axes. - if len(shape) != len(unvalidated_axes): - raise ValueError( - 'Tensor rank was not equal to the number of axes: %r, %r' % - (shape, unvalidated_axes)) - - # Second, the size of each tensor dimension must match the size of the - # corresponding indices. - for (d, axis) in zip(shape, unvalidated_axes.values()): - if d != axis.size: - raise ValueError( - 'Provided axis size %d does not match tensor dimension size %d' - 'in tensor %r' % (axis.size, d, tensor)) - - self._axes = unvalidated_axes - - def __repr__(self): - # - axes = ["('%s', %r)" % (v.name, v.value) for v in self.axes.values()] - axes_repr = (',\n' + ' ' * len(' axes=[')).join(axes) - return ("<%s '%s' shape=%s dtype=%s\n axes=[%s]>" % - (type(self).__name__, self.tensor.name, self.tensor.get_shape(), - self.tensor.dtype.name, axes_repr)) - - @property - def tensor(self): - return self._tensor - - def _as_graph_element(self): - """Support tf.Graph.as_graph_element on LabeledTensor objects. - - This allows operations such as tf.name_scope to take labeled tensors. - - Returns: - self.tensor - """ - return self.tensor - - @property - def axes(self): - return self._axes - - # properties/methods directly borrowed from tf.Tensor: - - @property - def dtype(self): - return self._tensor.dtype - - @property - def shape(self): - return self._tensor.shape - - @property - def name(self): - return self._tensor.name - - def get_shape(self): - """Returns the TensorShape that represents the shape of this tensor. - - See tf.Tensor.get_shape(). - - Returns: - A TensorShape representing the shape of this tensor. - """ - return self._tensor.get_shape() - - # TODO(shoyer): consider how/if to implement .eval(). Maybe it should return - # an xarray.DataArray? - - def __getitem__(self, key): - # This should work exactly like tf.Tensor.__getitem__, except it preserves - # labels. - if not isinstance(key, tuple): - key = (key,) - if len(key) != len(self.axes): - raise ValueError('indexer %r must have the same length as the Tensor ' - 'rank (%r)' % (key, len(self.axes))) - selection = {a: k for a, k in zip(self.axes.keys(), key)} - return slice_function(self, selection) - - # special methods for overloading arithmetic operations: - - def __abs__(self): - return abs_function(self) - - def __neg__(self): - return neg(self) - - def __pos__(self): - return self - - def __add__(self, other): - return add(self, other) - - def __radd__(self, other): - return add(other, self) - - def __sub__(self, other): - return sub(self, other) - - def __rsub__(self, other): - return sub(other, self) - - def __mul__(self, other): - return mul(self, other) - - def __rmul__(self, other): - return mul(other, self) - - def __truediv__(self, other): - return div(self, other) - - __div__ = __truediv__ - - def __rtruediv__(self, other): - return div(other, self) - - __rdiv__ = __rtruediv__ - - def __mod__(self, other): - return mod(self, other) - - def __rmod__(self, other): - return mod(other, self) - - def __pow__(self, other): - return pow_function(self, other) - - def __rpow__(self, other): - return pow_function(other, self) - - # logical operations: - - def __invert__(self): - return logical_not(self) - - def __and__(self, other): - return logical_and(self, other) - - def __or__(self, other): - return logical_or(self, other) - - def __xor__(self, other): - return logical_xor(self, other) - - # boolean operations: - - def __lt__(self, other): - return less(self, other) - - def __le__(self, other): - return less_equal(self, other) - - def __gt__(self, other): - return greater(self, other) - - def __ge__(self, other): - return greater_equal(self, other) - - def __eq__(self, other): - # for consistency with tf.Tensor - if not isinstance(other, LabeledTensor): - return False - - return self.tensor == other.tensor and self.axes == other.axes - - def __ne__(self, other): - return not self == other - - def __hash__(self): - return hash((self.tensor, self.axes)) - - -# typecheck type abbreviations: -# abbreviations for third-party types with very long reprs -tc.register_type_abbreviation(tensor_shape.Dimension, 'tensorflow.Dimension') -tc.register_type_abbreviation(ops.Tensor, 'tensorflow.Tensor') -tc.register_type_abbreviation(dtypes.DType, 'tensorflow.DType') -# core LabeledTensor types -tc.register_type_abbreviation(Axis, 'labeled_tensor.Axis') -tc.register_type_abbreviation(Axes, 'labeled_tensor.Axes') -tc.register_type_abbreviation(LabeledTensor, 'labeled_tensor.LabeledTensor') - - -@tc.returns(ops.Tensor) -@tc.accepts(LabeledTensor) -def _convert_labeled_tensor_to_tensor(value, *args, **kwargs): - # call ops.convert_to_tensor to handle optional arguments appropriately - return ops.internal_convert_to_tensor(value.tensor, *args, **kwargs) - - -ops.register_tensor_conversion_function(LabeledTensor, - _convert_labeled_tensor_to_tensor) - -# tc class for anything that can be coerced into a LabeledTensor -# pylint: disable=invalid-name -LabeledTensorLike = tc.Union(LabeledTensor, ops.Tensor, np.ndarray, Scalar) -# pylint: enable=invalid-name - - -@tc.returns(LabeledTensor) -@tc.accepts(LabeledTensorLike, object, tc.Optional(string_types)) -def convert_to_labeled_tensor(value, dtype=None, name=None): - """Converts the given `value` to a `LabeledTensor`. - - This function accepts `LabeledTensor` objects, 0-dimensional `Tensor` objects - and numpy arrays, and Python scalars. Higher dimensional unlabeled tensors - must use the `LabeledTensor` constructor explicitly. - - Args: - value: Object to convert. - dtype: Optional element type for the returned tensor. If missing, the type - is inferred from the type of value. - name: Optional name to use if a new Tensor is created. - - Returns: - `value` converted into a `LabeledTensor` object. - - Raises: - ValueError: If the output would have rank>0 but the input was not already a - `LabeledTensor`. - """ - # TODO(shoyer): consider extending to accept xarray.DataArray as input. - if isinstance(value, LabeledTensor): - axes = value.axes.values() - value = value.tensor - else: - axes = [] - - # We call convert_to_tensor even for LabeledTensor input because it also - # checks to make sure the dtype argument is compatible. - tensor = ops.convert_to_tensor(value, dtype=dtype, name=name) - if len(tensor.get_shape()) != len(axes): - raise ValueError('cannot automatically convert unlabeled arrays or tensors ' - 'with rank>0 into LabeledTensors: %r' % value) - return LabeledTensor(tensor, axes) - - -@tc.returns(Axis) -@tc.accepts(tc.Collection(Axis)) -def concat_axes(axes): - """Concatenate a list of Axes. - - Args: - axes: A collection of Axis objects. - - Returns: - The concatenation of the axes. - If all axes have labels, the result has the concatenation of the labels. - Else, the result has no labels, and its size is the sum of the sizes - of the axes. - - Raises: - ValueError: If `others` is not a collection of Axes or if it is empty. - """ - if not axes: - raise ValueError('axes must not be empty') - for a in axes: - if not isinstance(a, Axis): - raise ValueError('Expected an Axis, but got %r of type %r' % (a, type(a))) - - names = set(a.name for a in axes) - if len(names) > 1: - raise ValueError('axes do not all have the same name: %r' % names) - name, = names - - all_have_labels = all(a.labels is not None for a in axes) - any_has_unknown_size = any(a.size is None for a in axes) - - if all_have_labels: - value = tuple(label for a in axes for label in a.labels) - elif any_has_unknown_size: - value = None - else: - value = sum(len(a) for a in axes) - return Axis(name, value) - - -@tc.returns(LabeledTensor) -@tc.accepts(LabeledTensorLike, tc.Optional(string_types)) -def identity(labeled_tensor, name=None): - """The identity op. - - See tf.identity. - - Args: - labeled_tensor: The input tensor. - name: Optional op name. - - Returns: - The tensor. - """ - with ops.name_scope(name, 'lt_identity', [labeled_tensor]) as scope: - labeled_tensor = convert_to_labeled_tensor(labeled_tensor) - return LabeledTensor( - array_ops.identity(labeled_tensor.tensor, name=scope), - labeled_tensor.axes) - - -# We don't call this slice because that shadows a built-in. Instead, we alias -# this to lt.slice in __init__.py. -@tc.returns(LabeledTensor) -@tc.accepts(LabeledTensorLike, tc.Mapping(string_types, tc.Union(int, slice)), - tc.Optional(string_types)) -def slice_function(labeled_tensor, selection, name=None): - """Slice out a subset of the tensor. - - This is an analog of tf.slice. - For example: - >>> tensor = tf.reshape(tf.range(0, 6), [3, 2]) - >>> labeled_tensor = lt.LabeledTensor(tensor, ['a', ('b', ['foo', 'bar'])]) - >>> lt.slice(labeled_tensor, {'a': slice(0, 2), 'b': 1}) - - - Args: - labeled_tensor: The input tensor. - selection: A dictionary of type str -> Union(int, slice of int) mapping axis - names to sub-selections. - name: Optional op name. - - Returns: - The slice as a `LabeledTensor`. - """ - with ops.name_scope(name, 'lt_slice', [labeled_tensor]) as scope: - labeled_tensor = convert_to_labeled_tensor(labeled_tensor) - - slices = [] - - for axis_name in labeled_tensor.axes: - if axis_name not in selection: - # We're not sub-selecting this axis, so use the full slice. - slices.append(slice(None)) - else: - slices.append(selection[axis_name]) - - sliced_tensor = labeled_tensor.tensor[tuple(slices)] - - sliced_axes = [] - for axis, s in zip(labeled_tensor.axes.values(), slices): - # We sub-select this axis's index with the slice s. - - # `s` is either an int or a proper slice. - if isinstance(s, slice): - if axis.labels is None: - # We're not tracking coordinate names for this axis. - sliced_axes.append(axis.name) - else: - sliced_axes.append((axis.name, axis.labels[s])) - else: - # If the slice is an int this dimension now has size 1, so we remove it. - assert isinstance(s, int) - - return LabeledTensor( - array_ops.identity(sliced_tensor, name=scope), sliced_axes) - - -@tc.returns(LabeledTensor) -@tc.accepts(LabeledTensorLike, tc.Optional(tc.Collection(string_types)), - tc.Optional(string_types)) -def transpose(labeled_tensor, axis_order=None, name=None): - """Permute a tensor's axes. - - See tf.transpose. - - Args: - labeled_tensor: The input tensor. - axis_order: Optional desired axis order, as a list of names. By default, the - order of axes is reversed. - name: Optional op name. - - Returns: - The permuted tensor. - - Raises: - ValueError: If axis_order isn't a permutation of the existing axes. - """ - with ops.name_scope(name, 'lt_transpose', [labeled_tensor]) as scope: - labeled_tensor = convert_to_labeled_tensor(labeled_tensor) - - original_order = list(labeled_tensor.axes.keys()) - if axis_order is None: - axis_order = list(reversed(original_order)) - elif sorted(axis_order) != sorted(original_order): - raise ValueError( - 'The new axis order must have the same names as the original axes, ' - 'but the new order is %r while the original order is %r' % - (axis_order, original_order)) - - axis_names = list(labeled_tensor.axes.keys()) - permutation = [axis_names.index(n) for n in axis_order] - - # Note: TensorFlow doesn't copy data for the identity transpose. - transpose_tensor = array_ops.transpose( - labeled_tensor.tensor, permutation, name=scope) - - permuted_axes = [labeled_tensor.axes[n] for n in axis_order] - - return LabeledTensor(transpose_tensor, permuted_axes) - - -@tc.returns(LabeledTensor) -@tc.accepts(LabeledTensorLike, - tc.Collection( - tc.Union(string_types, - tc.Tuple(string_types, collections_abc.Hashable))), - tc.Optional(string_types)) -def expand_dims(labeled_tensor, axes, name=None): - """Insert dimensions of size 1. - - See tf.expand_dims. - - Args: - labeled_tensor: The input tensor. - axes: The desired axis names as strings or tuples of (name, label), where - `label` is the coordinate name for the new dimension `name`. These must - include the existing axis names, and the existing names must appear in the - same order in this list as they do in the input tensor. - name: Optional op name. - - Returns: - A tensor with an axis for each axis in axes. - New axes are created with size 1 and do not have labeled coordinates. - - Raises: - AxisOrderError: If axis names don't appear in the same order in axes - and the labeled tensor. - """ - with ops.name_scope(name, 'lt_expand_dims', [labeled_tensor]) as scope: - labeled_tensor = convert_to_labeled_tensor(labeled_tensor) - - axis_names = [a if isinstance(a, string_types) else a[0] for a in axes] - check_axis_order(labeled_tensor, axis_names) - - reshaped_axes = [] - shape = [] - for axis_spec in axes: - if axis_spec in labeled_tensor.axes: - axis = labeled_tensor.axes[axis_spec] - reshaped_axes.append(axis) - shape.append(-1 if axis.size is None else axis.size) - else: - if isinstance(axis_spec, string_types): - reshaped_axes.append((axis_spec, 1)) - else: - (name, label) = axis_spec - reshaped_axes.append((name, (label,))) - - shape.append(1) - - reshaped_tensor = array_ops.reshape( - labeled_tensor.tensor, shape, name=scope) - - return LabeledTensor(reshaped_tensor, reshaped_axes) - - -# This should only be added to a graph collection once. -_AXIS_ORDER_KEY = ('__axis_order',) - - -@tc.returns(tc.Optional(tc.List(string_types))) -def get_axis_order(): - """Get the axis_order set by any containing axis_order_scope. - - Returns: - List of strings giving an order to use for axis names, or None, if no axis - order is set. - """ - # By storing axis_order in the graph, we can ensure that axis_order_scope is - # thread-safe. - axis_order_list = ops.get_collection(_AXIS_ORDER_KEY) - if axis_order_list: - axis_order, = axis_order_list - else: - axis_order = None - return axis_order - - -@tc.accepts(tc.Optional(tc.List(string_types))) -def _set_axis_order(axis_order): - axis_order_list = ops.get_collection_ref(_AXIS_ORDER_KEY) - if axis_order_list: - axis_order_list[0] = axis_order - else: - axis_order_list.append(axis_order) - - -@contextlib.contextmanager -@tc.accepts(tc.Optional(tc.List(string_types))) -def axis_order_scope(axis_order=None): - """Set axis order for the result of broadcasting operations within a scope. - - This allows you to ensure that tensors resulting from arithmetic have a - predictable axis order. - - Example usage: - - with lt.axis_order_scope(['x', 'y', 'z']): - # result is guaranteed to have the correct axis order - result = w + b - - You can nest scopes, in which case only the inner-most scope applies, e.g., - - with lt.axis_order(['x', 'y', 'z']): - with lt.axis_order(): - result = w + b # uses the default (left-most) axis ordering - - Args: - axis_order: optional list of strings providing axis names. By default, - creates a scope without axis order. - - Yields: - The provided axis_order or `None`. - """ - original_axis_order = get_axis_order() - _set_axis_order(axis_order) - try: - yield axis_order - finally: - _set_axis_order(original_axis_order) - - -@tc.returns(tc.List(string_types)) -def _get_valid_axis_order(): - axis_order = get_axis_order() - if axis_order is None: - raise AxisOrderError('an explicit axis order must be provided with the ' - 'axis_order argument or by using an axis_order_scope') - return axis_order - - -class AxisOrderError(ValueError): - """Error class for cases where there is no valid axis order.""" - - -# TODO(shoyer): should this function accept a list of labeled tensors instead? -@tc.returns(type(None)) -@tc.accepts(LabeledTensorLike, tc.Optional(tc.Collection(string_types))) -def check_axis_order(labeled_tensor, axis_order=None): - """Verify that the given tensor has a consistent axis order. - - Args: - labeled_tensor: The input tensor. All axes on this tensor must appear in - axis_order. - axis_order: Optional desired axis order, as a list of names. If not - provided, defaults to the current axis_order_scope (if set). - - Raises: - AxisOrderError: If the axis_order is unavailable, inconsistent or does not - include all existing axes. - """ - labeled_tensor = convert_to_labeled_tensor(labeled_tensor) - - if axis_order is None: - axis_order = _get_valid_axis_order() - - relevant_axis_order = [a for a in axis_order if a in labeled_tensor.axes] - - if len(relevant_axis_order) < len(labeled_tensor.axes): - raise AxisOrderError( - 'not all axis names appear in the required axis order %r: %r' % - (axis_order, labeled_tensor)) - - if relevant_axis_order != list(labeled_tensor.axes): - raise AxisOrderError( - 'axes on a labeled tensor do not appear in the same order as the ' - 'required axis order %r: %r' % (axis_order, labeled_tensor)) - - -@tc.returns(LabeledTensor) -@tc.accepts(LabeledTensorLike, tc.Optional(tc.Collection(string_types)), - tc.Optional(string_types)) -def impose_axis_order(labeled_tensor, axis_order=None, name=None): - """Impose desired axis order on a labeled tensor. - - Args: - labeled_tensor: The input tensor. - axis_order: Optional desired axis order, as a list of names. If not - provided, defaults to the current axis_order_scope (if set). - name: Optional op name. - - Returns: - Labeled tensor with possibly transposed axes. - - Raises: - AxisOrderError: If no axis_order is provided or axis_order does not contain - all axes on the input tensor. - """ - with ops.name_scope(name, 'lt_impose_axis_order', [labeled_tensor]) as scope: - labeled_tensor = convert_to_labeled_tensor(labeled_tensor) - - if axis_order is None: - axis_order = _get_valid_axis_order() - - relevant_axis_order = [a for a in axis_order if a in labeled_tensor.axes] - - return transpose(labeled_tensor, relevant_axis_order, name=scope) - - -@tc.returns(tc.Optional(list)) -@tc.accepts(list, list) -def _find_consistent_ordering(a, b): - """Find the left-most consistent ordering between two lists of unique items. - - A consistent ordering combines all elements in both a and b while keeping all - elements in their original order in both inputs. The left-most consistent - ordering orders elements from `a` not found in `b` before elements in `b` not - found in `a`. - - For example, given ['x', 'z'] and ['y', 'z'], both ['x', 'y', 'z'] and ['y', - 'x', 'z'] are consistent orderings because each of the inputs appears in - each consistent ordering in the same order, and ['x', 'y', 'z'] is the - left-most, because 'x' appears only in `a` and 'y' appears only in `b`. In - contrast, there is no consistent ordering between ['x', 'y'] and ['y', 'x']. - - Args: - a: list with unique elements. - b: list with unique elements. - - Returns: - List containing all elements in either a or b, or None, if no consistent - ordering exists. - """ - a_set = set(a) - b_set = set(b) - i = 0 - j = 0 - ordering = [] - while i < len(a) and j < len(b): - if a[i] not in b_set: - ordering.append(a[i]) - i += 1 - elif b[j] not in a_set: - ordering.append(b[j]) - j += 1 - elif a[i] == b[j]: - ordering.append(a[i]) - i += 1 - j += 1 - else: - return None - - ordering.extend(a[i:]) - ordering.extend(b[j:]) - - return ordering - - -@tc.returns(LabeledTensor, LabeledTensor, Axes) -@tc.accepts(LabeledTensorLike, LabeledTensorLike, tc.Optional(string_types)) -def align(labeled_tensor_0, labeled_tensor_1, name=None): - """Align the axes of two tensors so they may be broadcast to each other. - - Axes are ordered by the current axis order scope, if present, or by the left- - most consistent ordering. An exception is raised if it is impossible to align - the tensors without a transpose (align never copies the input data). - - Example usage: - - >>> a = lt.LabeledTensor(tf.ones((2, 4)), ['x', 'z']) - >>> b = lt.LabeledTensor(tf.ones((3, 4)), ['y', 'z']) - >>> a2, b2, axes = lt.align(a, b) - >>> a2 - - >>> b2 - - >>> axes - Axes([('x', Dimension(2)), - ('y', Dimension(3)), - ('z', Dimension(4))]) - - Args: - labeled_tensor_0: An input tensor. - labeled_tensor_1: An input tensor. - name: Optional op name. - - Returns: - The aligned tensors and the axes the resulting tensor would have if the two - aligned tensors were broadcast to each other. The aligned tensors have the - same rank but not necessarily the same shape, with axes in the same order. - - Raises: - ValueError: If axes with the same name on the inputs are not equal. - AxisOrderError: If there is no way to reshape the input tensors into the - output without a transpose. - """ - with ops.name_scope(name, 'lt_align', - [labeled_tensor_0, labeled_tensor_1]) as scope: - - labeled_tensor_0 = convert_to_labeled_tensor(labeled_tensor_0) - labeled_tensor_1 = convert_to_labeled_tensor(labeled_tensor_1) - - axes_0 = labeled_tensor_0.axes - axes_1 = labeled_tensor_1.axes - for axis_name in axes_0: - if axis_name in axes_1: - if axes_0[axis_name] != axes_1[axis_name]: - raise ValueError('Mismatched %r axis on input tensors: %r and %r' % - (axis_name, axes_0[axis_name], axes_1[axis_name])) - - axis_scope_order = get_axis_order() - if axis_scope_order is not None: - # we are in an axis_order_scope - axis_names_set = set(axes_0) | set(axes_1) - new_axis_names = [a for a in axis_scope_order if a in axis_names_set] - - check_axis_order(labeled_tensor_0, axis_scope_order) - check_axis_order(labeled_tensor_1, axis_scope_order) - - else: - # attempt to find a consistent ordering - new_axis_names = _find_consistent_ordering(list(axes_0), list(axes_1)) - if new_axis_names is None: - raise AxisOrderError( - 'No consistent axis order allows for aligning tensors with axis ' - 'orders %r and %r without copying data. Use transpose or ' - 'impose_axis_order to reorder axes on one of more of the inputs.' % - (axes_0.keys(), axes_1.keys())) - - labeled_tensor_0 = expand_dims( - labeled_tensor_0, new_axis_names, name=scope + '0') - labeled_tensor_1 = expand_dims( - labeled_tensor_1, new_axis_names, name=scope + '1') - - broadcast_axes = [] - for axis_name in new_axis_names: - if axis_name in axes_0: - broadcast_axes.append(axes_0[axis_name]) - else: - broadcast_axes.append(axes_1[axis_name]) - - return labeled_tensor_0, labeled_tensor_1, Axes(broadcast_axes) - - -@tc.returns(types.FunctionType) -@tc.accepts(string_types, collections_abc.Callable) -def define_unary_op(op_name, elementwise_function): - """Define a unary operation for labeled tensors. - - Args: - op_name: string name of the TensorFlow op. - elementwise_function: function to call to evaluate the op on a single - tf.Tensor object. This function must accept two arguments: a tf.Tensor - object, and an optional `name`. - - Returns: - Function defining the given op that acts on LabeledTensors. - """ - - default_name = 'lt_%s' % op_name - - @tc.returns(LabeledTensor) - @tc.accepts(LabeledTensorLike, tc.Optional(string_types)) - def op(labeled_tensor, name=None): - """LabeledTensor version of `tf.{op_name}`. - - See `tf.{op_name}` for full details. - - Args: - labeled_tensor: Input tensor. - name: Optional op name. - - Returns: - A LabeledTensor with result of applying `tf.{op_name}` elementwise. - """ - with ops.name_scope(name, default_name, [labeled_tensor]) as scope: - labeled_tensor = convert_to_labeled_tensor(labeled_tensor) - result_tensor = elementwise_function(labeled_tensor.tensor, name=scope) - return LabeledTensor(result_tensor, labeled_tensor.axes) - - op.__doc__ = op.__doc__.format(op_name=op_name) - op.__name__ = op_name - - return op - - -abs_function = define_unary_op('abs', math_ops.abs) -neg = define_unary_op('neg', math_ops.negative) -sign = define_unary_op('sign', math_ops.sign) -reciprocal = define_unary_op('reciprocal', math_ops.reciprocal) -square = define_unary_op('square', math_ops.square) -round_function = define_unary_op('round', math_ops.round) -sqrt = define_unary_op('sqrt', math_ops.sqrt) -rsqrt = define_unary_op('rsqrt', math_ops.rsqrt) -exp = define_unary_op('exp', math_ops.exp) -log = define_unary_op('log', math_ops.log) -ceil = define_unary_op('ceil', math_ops.ceil) -floor = define_unary_op('floor', math_ops.floor) -cos = define_unary_op('cos', math_ops.cos) -sin = define_unary_op('sin', math_ops.sin) -tan = define_unary_op('tan', math_ops.tan) -acos = define_unary_op('acos', math_ops.acos) -asin = define_unary_op('asin', math_ops.asin) -atan = define_unary_op('atan', math_ops.atan) -lgamma = define_unary_op('lgamma', math_ops.lgamma) -digamma = define_unary_op('digamma', math_ops.digamma) -erf = define_unary_op('erf', math_ops.erf) -erfc = define_unary_op('erfc', math_ops.erfc) -logical_not = define_unary_op('logical_not', math_ops.logical_not) -tanh = define_unary_op('tanh', math_ops.tanh) -sigmoid = define_unary_op('sigmoid', math_ops.sigmoid) - - -@tc.returns(types.FunctionType) -@tc.accepts(string_types, collections_abc.Callable) -def define_binary_op(op_name, elementwise_function): - """Define a binary operation that broadcasts labeled tensors. - - Args: - op_name: string name of the TensorFlow op. - elementwise_function: function to call to evaluate the op on tf.Tensor - objects. This function must accept three arguments: two tf.Tensor objects, - and an optional `name`. - - Returns: - Function defining the given op that acts on LabeledTensors. - """ - - default_name = 'lt_%s' % op_name - - @tc.returns(LabeledTensor) - @tc.accepts(LabeledTensorLike, LabeledTensorLike, tc.Optional(string_types)) - def op(labeled_tensor_0, labeled_tensor_1, name=None): - """LabeledTensor version of `tf.{op_name}` with label based alignment. - - See `tf.{op_name}` for full details. - - Args: - labeled_tensor_0: Input tensor. - labeled_tensor_1: Input tensor. - name: Optional op name. - - Returns: - A LabeledTensor with result of applying `tf.{op_name}` elementwise. - """ - with ops.name_scope(name, default_name, - [labeled_tensor_0, labeled_tensor_1]) as scope: - - align_0, align_1, broadcast_axes = align(labeled_tensor_0, - labeled_tensor_1) - - tensor = elementwise_function(align_0.tensor, align_1.tensor, name=scope) - - return LabeledTensor(tensor, broadcast_axes) - - op.__doc__ = op.__doc__.format(op_name=op_name) - op.__name__ = op_name - - return op - - -add = define_binary_op('add', math_ops.add) -sub = define_binary_op('sub', math_ops.subtract) -mul = define_binary_op('mul', math_ops.multiply) -div = define_binary_op('div', math_ops.div) -mod = define_binary_op('mod', math_ops.mod) -pow_function = define_binary_op('pow', math_ops.pow) - -equal = define_binary_op('equal', math_ops.equal) -greater = define_binary_op('greater', math_ops.greater) -greater_equal = define_binary_op('greater_equal', math_ops.greater_equal) -not_equal = define_binary_op('not_equal', math_ops.not_equal) -less = define_binary_op('less', math_ops.less) -less_equal = define_binary_op('less_equal', math_ops.less_equal) -logical_and = define_binary_op('logical_and', math_ops.logical_and) -logical_or = define_binary_op('logical_or', math_ops.logical_or) -logical_xor = define_binary_op('logical_xor', math_ops.logical_xor) - -maximum = define_binary_op('maximum', math_ops.maximum) -minimum = define_binary_op('minimum', math_ops.minimum) -squared_difference = define_binary_op('squared_difference', - math_ops.squared_difference) -igamma = define_binary_op('igamma', math_ops.igamma) -igammac = define_binary_op('igammac', math_ops.igammac) -zeta = define_binary_op('zeta', math_ops.zeta) -polygamma = define_binary_op('polygamma', math_ops.polygamma) diff --git a/tensorflow/contrib/labeled_tensor/python/ops/core_test.py b/tensorflow/contrib/labeled_tensor/python/ops/core_test.py deleted file mode 100644 index e378db56afb..00000000000 --- a/tensorflow/contrib/labeled_tensor/python/ops/core_test.py +++ /dev/null @@ -1,864 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import operator -import re -import textwrap - -import numpy as np -from six.moves import range # pylint: disable=redefined-builtin - -from tensorflow.contrib.labeled_tensor.python.ops import _typecheck as tc -from tensorflow.contrib.labeled_tensor.python.ops import core -from tensorflow.contrib.labeled_tensor.python.ops import test_util -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test as test_lib - - -class AxisTest(test_lib.TestCase): - - def setUp(self): - d_7 = tensor_shape.Dimension(7) - p_rgb = ['red', 'green', 'blue'] - - self.i_7 = core.Axis('7', d_7) - self.i_7p = core.Axis('7prime', d_7) - self.i_rgb = core.Axis('rgb', p_rgb) - self.i_range = core.Axis('range', range(7)) - self.i_unknown = core.Axis('unknown', None) - - def test_equality(self): - - axes = [self.i_7, self.i_7p, self.i_rgb, self.i_range, self.i_unknown] - for i, axis_0 in enumerate(axes): - for j, axis_1 in enumerate(axes): - if i == j: - self.assertEqual(axis_0, axis_1) - else: - self.assertNotEqual(axis_0, axis_1) - - def test_axis_value(self): - self.assertEqual(self.i_7.value, tensor_shape.Dimension(7)) - self.assertTrue(self.i_range.value == tuple(range(7))) - - def test_axis_input(self): - axes = [self.i_7, self.i_7p, self.i_rgb, self.i_range, self.i_unknown] - for axis in axes: - self.assertEqual(axis, core.Axis(axis.name, axis.value)) - - def test_axis_value_input(self): - axis = self.i_range - for value in [range(7), list(range(7)), np.arange(7)]: - self.assertEqual(axis, core.Axis(axis.name, value)) - - def test_size(self): - self.assertEqual(len(self.i_7), 7) - self.assertEqual(len(self.i_rgb), 3) - self.assertEqual(len(self.i_range), 7) - self.assertEqual(self.i_unknown.size, None) - - def test_concat_single(self): - red = core.Axis('rgb', ['red']) - - self.assertEqual(core.concat_axes([red]), red) - - def test_concat_many(self): - red = core.Axis('rgb', ['red']) - green = core.Axis('rgb', ['green']) - blue = core.Axis('rgb', ['blue']) - red_green_blue = core.Axis('rgb', ['red', 'green', 'blue']) - - self.assertEqual(core.concat_axes([red, green, blue]), red_green_blue) - - def test_concat_different_names(self): - red = core.Axis('red', ['red']) - green = core.Axis('green', ['red']) - with self.assertRaises(ValueError): - core.concat_axes([red, green]) - - def test_concat_unknown(self): - red = core.Axis('rgb', None) - green = core.Axis('rgb', None) - self.assertEqual(core.concat_axes([red, green]), red) - - def test_repr(self): - self.assertEqual("Axis('7', Dimension(7))", repr(self.i_7)) - - def test_invalid_input(self): - with self.assertRaises(TypeError): - core.Axis('foo', [{}]) - with self.assertRaises(ValueError): - core.Axis('foo', [1, 2, 3, 1]) - red = core.Axis('foo', ['red']) - with self.assertRaises(tc.Error): - core.concat_axes([red, 1]) - - def test_as_axis(self): - self.assertEqual(self.i_7, core.as_axis(('7', 7))) - self.assertEqual(self.i_7, core.as_axis(self.i_7)) - - -class AxesTest(test_lib.TestCase): - - def setUp(self): - d_7 = tensor_shape.Dimension(7) - d_8 = tensor_shape.Dimension(8) - p_rgb = ['red', 'green', 'blue'] - p_range = range(7) - - self.i_8 = core.Axis('8', d_8) - - self.a0 = core.Axes([('d7', d_7)]) - self.a1 = core.Axes([('d7', d_7)]) - self.a2 = core.Axes([('d7', d_7), ('rgb', p_rgb)]) - self.a3 = core.Axes([('8', d_8), ('range', p_range)]) - - def test_equality(self): - self.assertEqual(self.a0, self.a0) - self.assertEqual(self.a0, self.a1) - self.assertNotEqual(self.a0, self.a2) - - def test_repr(self): - self.assertEqual("Axes([('d7', Dimension(7))])", repr(self.a0)) - - def test_remove(self): - a = self.a3.remove('range') - self.assertEqual(a, core.Axes([self.i_8])) - with self.assertRaises(KeyError): - self.a3.remove('foobar') - - def test_typecheck_error_message(self): - pattern = ('List(Union(labeled_tensor.Axis, Tuple(..., ' - 'Union(Union(numpy.ndarray, %s, list, tuple), ' - 'Optional(Union(tensorflow.Dimension, int))))))' % - range.__name__) - regexp = re.escape(pattern).replace(re.escape('...'), '.*') - with self.assertRaisesRegexp(tc.Error, 'allowed type ' + regexp): - core.Axes(None) - - -class LabeledTensorTest(test_util.Base): - - def setUp(self): - tensor = array_ops.ones([7, 3, 8, 1]) - a0 = ('x', range(7)) - a1 = ('channel', ['red', 'green', 'blue']) - a2 = ('y', 8) - a3 = ('z', tensor_shape.Dimension(1)) - - self.lt = core.LabeledTensor(tensor, [a0, a1, a2, a3]) - - def test_repr(self): - pattern = textwrap.dedent("""\ - """) - regexp = re.escape(pattern).replace(re.escape('...'), '.*') - self.assertRegexpMatches(repr(self.lt), regexp) - - def test_reuse_existing_axes(self): - alt_lt = core.LabeledTensor(self.lt.tensor, self.lt.axes) - self.assertLabeledTensorsEqual(alt_lt, self.lt) - - def test_reuse_existing_axis_objects(self): - alt_lt = core.LabeledTensor(self.lt.tensor, self.lt.axes.values()) - self.assertLabeledTensorsEqual(alt_lt, self.lt) - - def test_indexing_scalars(self): - actual = self.lt[:, :, :, 0] - expected = core.LabeledTensor(self.lt.tensor[:, :, :, 0], - list(self.lt.axes.values())[:-1]) - self.assertLabeledTensorsEqual(actual, expected) - - actual = self.lt[1, :, :, 0] - expected = core.LabeledTensor(self.lt.tensor[1, :, :, 0], - list(self.lt.axes.values())[1:-1]) - self.assertLabeledTensorsEqual(actual, expected) - - actual = self.lt[1, 2, :, 0] - expected = core.LabeledTensor(self.lt.tensor[1, 2, :, 0], - list(self.lt.axes.values())[2:-1]) - self.assertLabeledTensorsEqual(actual, expected) - - def test_indexing_1d(self): - lt_1d = self.lt[1, 2, :, 0] - actual = lt_1d[3] - expected = core.LabeledTensor(lt_1d.tensor[3], []) - self.assertLabeledTensorsEqual(actual, expected) - - def test_indexing_slices(self): - actual = self.lt[:3, :, :, :] - axes = [('x', range(3))] + list(self.lt.axes.values())[1:] - expected = core.LabeledTensor(self.lt.tensor[:3, :, :, :], axes) - self.assertLabeledTensorsEqual(actual, expected) - - def test_invalid_indexing(self): - with self.assertRaises(ValueError): - self.lt[0] # pylint: disable=pointless-statement - with self.assertRaises(ValueError): - self.lt[:, :, :, :, 0] # pylint: disable=pointless-statement - - def test_unknown_size(self): - tensor = array_ops.placeholder(dtypes.string, [None]) - actual = core.LabeledTensor(tensor, ['x']) - self.assertIsNone(actual.axes['x'].size) - self.assertIsNone(actual.axes['x'].value.value) - - def test_eq(self): - self.assertEqual(self.lt, self.lt) - self.assertNotEqual(self.lt, self.lt.tensor) - self.assertNotEqual(self.lt.tensor, self.lt) - - def test_hash(self): - lt1 = self.lt - lt2 = core.LabeledTensor(self.lt.tensor, self.lt.axes) - self.assertEqual(lt1, lt2) - self.assertEqual(hash(lt1), hash(lt2)) - - def test_name(self): - self.assertEqual(self.lt.name, self.lt.tensor.name) - - def test_dtype(self): - self.assertEqual(self.lt.dtype, self.lt.tensor.dtype) - - def test_shape(self): - self.assertEqual(self.lt.shape, self.lt.tensor.shape) - - def test_get_shape(self): - self.assertEqual(self.lt.get_shape(), self.lt.tensor.get_shape()) - - def test_convert_to_tensor(self): - expected = self.lt.tensor - actual = ops.convert_to_tensor(self.lt) - self.assertIs(expected, actual) - - -class Base(test_util.Base): - - def setUp(self): - self.x_size = 7 - self.channel_size = 3 - self.z_size = 4 - self.probs_size = 11 - - tensor = math_ops.range(0, self.x_size * self.channel_size * self.z_size * - self.probs_size) - tensor = array_ops.reshape( - tensor, [self.x_size, self.channel_size, self.z_size, self.probs_size]) - a0 = ('x', range(self.x_size)) - a1 = ('channel', ['red', 'green', 'blue']) - a2 = 'z' - a3 = ('probs', np.linspace(0.0, 1.0, self.probs_size)) - - self.tensor = tensor - self.a0 = a0 - self.a1 = a1 - self.a2 = a2 - self.a3 = a3 - self.original_lt = core.LabeledTensor(tensor, [a0, a1, a2, a3]) - - self.x_probs_lt = core.slice_function(self.original_lt, - {'z': 0, - 'channel': 0}) - self.channel_probs_lt = core.slice_function(self.original_lt, - {'x': 3, - 'z': 0}) - - -class IdentityTest(Base): - - def test_name(self): - identity_lt = core.identity(self.original_lt) - self.assertIn('lt_identity', identity_lt.name) - - -class SliceFunctionTest(Base): - - def test_name(self): - select_lt = core.slice_function(self.original_lt, {'channel': 1}) - self.assertIn('lt_slice', select_lt.name) - - def test_scalar(self): - select_lt = core.slice_function(self.original_lt, {'channel': 1}) - golden_lt = core.LabeledTensor(self.tensor[:, 1, :, :], - [self.a0, self.a2, self.a3]) - - self.assertLabeledTensorsEqual(select_lt, golden_lt) - - def test_slice(self): - select_lt = core.slice_function(self.original_lt, {'channel': slice(0, 2)}) - - a1_sliced = ('channel', ['red', 'green']) - golden_lt = core.LabeledTensor(self.tensor[:, :2, :, :], - [self.a0, a1_sliced, self.a2, self.a3]) - - self.assertLabeledTensorsEqual(select_lt, golden_lt) - - def test_slices(self): - select_lt = core.slice_function( - self.original_lt, {'x': slice(1, 5), - 'channel': slice(1, None)}) - - a0_sliced = ('x', range(1, 5)) - a1_sliced = ('channel', ['green', 'blue']) - golden_lt = core.LabeledTensor(self.tensor[1:5, 1:, :, :], - [a0_sliced, a1_sliced, self.a2, self.a3]) - - self.assertLabeledTensorsEqual(select_lt, golden_lt) - - def test_slice_unlabeled(self): - select_lt = core.slice_function(self.original_lt, {'z': slice(1, 3)}) - - a2_sliced = 'z' - golden_lt = core.LabeledTensor(self.tensor[:, :, 1:3, :], - [self.a0, self.a1, a2_sliced, self.a3]) - - self.assertLabeledTensorsEqual(select_lt, golden_lt) - - def test_slice_unknown_shape(self): - lt = core.LabeledTensor( - array_ops.placeholder(dtypes.float32, [None, 1]), ['x', 'y']) - sliced_lt = core.slice_function(lt, {'y': 0}) - self.assertEqual(list(sliced_lt.axes.values()), [lt.axes['x']]) - - -class TransposeTest(Base): - - def test_name(self): - transpose_lt = core.transpose(self.original_lt, - self.original_lt.axes.keys()) - self.assertIn('lt_transpose', transpose_lt.name) - - def test_identity(self): - transpose_lt = core.transpose(self.original_lt, - self.original_lt.axes.keys()) - golden_lt = self.original_lt - - self.assertLabeledTensorsEqual(transpose_lt, golden_lt) - - def test(self): - transpose_lt = core.transpose(self.original_lt, - ['z', 'channel', 'x', 'probs']) - golden_lt = core.LabeledTensor( - array_ops.transpose(self.tensor, [2, 1, 0, 3]), - [self.a2, self.a1, self.a0, self.a3]) - - self.assertLabeledTensorsEqual(transpose_lt, golden_lt) - - def test_default_axis_order(self): - transpose_lt = core.transpose(self.original_lt) - golden_lt = core.LabeledTensor( - array_ops.transpose(self.tensor, [3, 2, 1, 0]), - list(reversed(list(self.original_lt.axes.values())))) - - self.assertLabeledTensorsEqual(transpose_lt, golden_lt) - - def test_invalid_input(self): - with self.assertRaises(ValueError): - core.transpose(self.original_lt, ['channel', 'x', 'probs']) - with self.assertRaises(ValueError): - core.transpose(self.original_lt, ['z', 'foo', 'x', 'probs']) - - -class ExpandDimsTest(Base): - - def test_name(self): - expand_lt = core.expand_dims(self.original_lt, self.original_lt.axes.keys()) - self.assertIn('lt_expand', expand_lt.name) - - def test_identity(self): - expand_lt = core.expand_dims(self.original_lt, self.original_lt.axes.keys()) - golden_lt = self.original_lt - - self.assertLabeledTensorsEqual(expand_lt, golden_lt) - - def test(self): - expand_lt = core.expand_dims( - self.original_lt, ['foo', 'x', 'bar', 'channel', 'z', 'probs', 'grok']) - golden_lt = core.LabeledTensor( - array_ops.reshape(self.tensor, [ - 1, self.x_size, 1, self.channel_size, self.z_size, self.probs_size, - 1 - ]), ['foo', self.a0, 'bar', self.a1, self.a2, self.a3, 'grok']) - - self.assertLabeledTensorsEqual(expand_lt, golden_lt) - - def test_label(self): - expand_lt = core.expand_dims(self.original_lt, [ - 'x', - 'channel', - ('foo', 'bar'), - 'z', - 'probs', - ]) - golden_lt = core.LabeledTensor( - array_ops.reshape( - self.tensor, - [self.x_size, self.channel_size, 1, self.z_size, self.probs_size]), - [self.a0, self.a1, ('foo', ['bar']), self.a2, self.a3]) - - self.assertLabeledTensorsEqual(expand_lt, golden_lt) - - def test_unknown_dimension(self): - orig_lt = core.LabeledTensor( - array_ops.placeholder(dtypes.float32, [None]), ['x']) - expand_lt = core.expand_dims(orig_lt, ['x', 'y']) - self.assertEqual(expand_lt.axes, core.Axes([('x', None), ('y', 1)])) - - def test_invalid_input(self): - with self.assertRaises(core.AxisOrderError): - core.expand_dims(self.original_lt, - ['foo', 'not_x', 'bar', 'channel', 'z', 'probs', 'grok']) - with self.assertRaises(core.AxisOrderError): - core.expand_dims(self.original_lt, - ['foo', 'z', 'bar', 'channel', 'x', 'probs', 'grok']) - - -class AxisOrderScopeTest(Base): - - def test(self): - xyz = ['x', 'y', 'z'] - abc = ['a', 'b', 'c'] - - self.assertIsNone(core.get_axis_order()) - - with core.axis_order_scope(xyz): - self.assertEqual(core.get_axis_order(), xyz) - - with core.axis_order_scope(): - self.assertIsNone(core.get_axis_order()) - - with core.axis_order_scope(abc): - self.assertEqual(core.get_axis_order(), abc) - - self.assertIsNone(core.get_axis_order()) - - self.assertEqual(core.get_axis_order(), xyz) - - self.assertIsNone(core.get_axis_order()) - - -class CheckAxisOrderTest(Base): - - def test_passes(self): - axis_order = ['w', 'x', 'y', 'z'] - - lt = core.LabeledTensor(array_ops.ones((1, 1, 1, 1)), axis_order) - core.check_axis_order(lt, axis_order) - - lt = core.LabeledTensor(array_ops.ones((1, 1, 1)), axis_order[1:]) - core.check_axis_order(lt, axis_order) - - lt = core.LabeledTensor(array_ops.ones((1, 1, 1)), axis_order[:-1]) - core.check_axis_order(lt, axis_order) - - def test_invalid(self): - axis_order = ['w', 'x', 'y', 'z'] - lt = core.LabeledTensor(array_ops.ones((1, 1, 1, 1)), axis_order) - with self.assertRaises(core.AxisOrderError): - core.check_axis_order(lt) - with self.assertRaises(core.AxisOrderError): - core.check_axis_order(lt, axis_order[:-1]) - with self.assertRaises(core.AxisOrderError): - core.check_axis_order(lt, axis_order[::-1]) - - def test_scope(self): - axis_order = ['w', 'x', 'y', 'z'] - lt = core.LabeledTensor(array_ops.ones((1, 1, 1, 1)), axis_order) - with core.axis_order_scope(axis_order): - core.check_axis_order(lt) - - -class ImposeAxisOrderTest(Base): - - def test_identity(self): - axis_order = ['w', 'x', 'y', 'z'] - lt = core.LabeledTensor( - array_ops.reshape(math_ops.range(24), (1, 2, 3, 4)), axis_order) - actual = core.impose_axis_order(lt, axis_order) - self.assertLabeledTensorsEqual(lt, actual) - - lt = core.LabeledTensor( - array_ops.reshape(math_ops.range(6), (1, 2, 3)), axis_order[:3]) - actual = core.impose_axis_order(lt, axis_order) - self.assertLabeledTensorsEqual(lt, actual) - - def test_reverse(self): - axis_order = ['w', 'x', 'y', 'z'] - - lt = core.LabeledTensor( - array_ops.reshape(math_ops.range(24), (1, 2, 3, 4)), axis_order) - actual = core.impose_axis_order(lt, axis_order[::-1]) - expected = core.transpose(lt, axis_order[::-1]) - self.assertLabeledTensorsEqual(expected, actual) - - lt = core.LabeledTensor( - array_ops.reshape(math_ops.range(6), (1, 2, 3)), axis_order[:3]) - actual = core.impose_axis_order(lt, axis_order[::-1]) - expected = core.transpose(lt, ['y', 'x', 'w']) - self.assertLabeledTensorsEqual(expected, actual) - - def test_scope(self): - axis_order = ['w', 'x', 'y', 'z'] - - lt = core.LabeledTensor( - array_ops.reshape(math_ops.range(24), (1, 2, 3, 4)), axis_order) - expected = core.transpose(lt, axis_order[::-1]) - with core.axis_order_scope(axis_order[::-1]): - actual = core.impose_axis_order(lt) - self.assertLabeledTensorsEqual(expected, actual) - - def test_invalid(self): - lt = core.LabeledTensor( - array_ops.reshape(math_ops.range(2), (1, 2)), ['x', 'y']) - with self.assertRaises(ValueError): - core.impose_axis_order(lt) - with self.assertRaises(ValueError): - core.impose_axis_order(lt, ['x']) - - -class FindConsistentOrderingTest(Base): - - def test(self): - cases = [ - ([], [], []), - (['x'], [], ['x']), - ([], ['x'], ['x']), - (['x'], ['x'], ['x']), - (['x'], ['y'], ['x', 'y']), - (['y'], ['x'], ['y', 'x']), - (['x', 'y'], ['x', 'y'], ['x', 'y']), - (['x', 'y'], ['y', 'x'], None), - (['x', 'y'], ['y', 'z'], ['x', 'y', 'z']), - (['x', 'z'], ['y', 'z'], ['x', 'y', 'z']), - (['x', 'y'], ['x', 'z'], ['x', 'y', 'z']), - (['w', 'x'], ['y', 'z'], ['w', 'x', 'y', 'z']), - (['x', 'y', 'z'], ['z', 'x'], None), - (['x', 'y', 'z'], ['x'], ['x', 'y', 'z']), - ([], ['x', 'y', 'z'], ['x', 'y', 'z']), - ] - for a, b, expected in cases: - actual = core._find_consistent_ordering(a, b) - msg = ('unexpected ordering between %r and %r:\nexpected: %r\nactual: %r' - % (a, b, expected, actual)) - self.assertEqual(expected, actual, msg=msg) - - -class AlignTest(Base): - - def test_name(self): - align_lt_0, align_lt_1, _ = core.align(self.original_lt, self.original_lt) - self.assertIn('lt_align', align_lt_0.name) - self.assertIn('/0', align_lt_0.name) - self.assertIn('lt_align', align_lt_1.name) - self.assertIn('/1', align_lt_1.name) - - def test_identical_shaped_inputs(self): - offset_tensor = self.original_lt.tensor + 1 - offset_lt = core.LabeledTensor(offset_tensor, self.original_lt.axes) - - align_lt, align_offset_lt, broadcast_axes = core.align(self.original_lt, - offset_lt) - - self.assertLabeledTensorsEqual(align_lt, self.original_lt) - self.assertLabeledTensorsEqual(align_offset_lt, offset_lt) - self.assertEqual(broadcast_axes, self.original_lt.axes) - - def test_different_inputs(self): - # The correct axis ordering is ['x', 'channel', 'probs']. - align_x_probs_lt, align_channel_probs_lt, broadcast_axes = core.align( - self.x_probs_lt, self.channel_probs_lt) - - x_probs_golden_lt = core.LabeledTensor( - array_ops.reshape(self.x_probs_lt.tensor, - [self.x_size, 1, self.probs_size]), - [self.a0, 'channel', self.a3]) - - self.assertLabeledTensorsEqual(align_x_probs_lt, x_probs_golden_lt) - - channel_probs_golden_lt = core.LabeledTensor( - array_ops.reshape(self.channel_probs_lt.tensor, - [1, self.channel_size, self.probs_size]), - ['x', self.a1, self.a3]) - - self.assertLabeledTensorsEqual(align_channel_probs_lt, - channel_probs_golden_lt) - - self.assertEqual(broadcast_axes, core.Axes([self.a0, self.a1, self.a3])) - - def test_axis_order_scope(self): - xz_lt = core.LabeledTensor(array_ops.ones((2, 3)), ['x', 'z']) - yz_lt = core.LabeledTensor(array_ops.ones((4, 3)), ['y', 'z']) - - _, _, broadcast_axes = core.align(xz_lt, yz_lt) - self.assertEqual(list(broadcast_axes.keys()), ['x', 'y', 'z']) - - _, _, broadcast_axes = core.align(yz_lt, xz_lt) - self.assertEqual(list(broadcast_axes.keys()), ['y', 'x', 'z']) - - with core.axis_order_scope(['x', 'y', 'z']): - _, _, broadcast_axes = core.align(yz_lt, xz_lt) - self.assertEqual(list(broadcast_axes.keys()), ['x', 'y', 'z']) - - with core.axis_order_scope(['x', 'y']): - with self.assertRaises(core.AxisOrderError): - core.align(xz_lt, yz_lt) - with self.assertRaises(core.AxisOrderError): - core.align(yz_lt, xz_lt) - - def test_invalid_input(self): - lt_0 = core.LabeledTensor(array_ops.zeros([5]), [('a', range(5))]) - lt_1 = core.LabeledTensor(array_ops.zeros([5]), [('a', range(1, 6))]) - with self.assertRaises(ValueError): - core.align(lt_0, lt_1) - - -class ConvertToLabeledTensorTest(Base): - - # TODO(shoyer): Simplify these tests once we can reuse labeled tensors in - # assertLabeledTensorsEqual. - - def test_labeled_tensor(self): - actual = core.convert_to_labeled_tensor(self.original_lt) - self.assertLabeledTensorsEqual(actual, self.original_lt) - - def test_python_scalar(self): - actual = core.convert_to_labeled_tensor(42) - golden_lt = core.LabeledTensor(ops.convert_to_tensor(42), []) - self.assertLabeledTensorsEqual(actual, golden_lt) - - def test_numpy_array(self): - actual = core.convert_to_labeled_tensor(np.array(42)) - golden_lt = core.LabeledTensor(ops.convert_to_tensor(42), []) - self.assertLabeledTensorsEqual(actual, golden_lt) - - def test_tensor(self): - actual = core.convert_to_labeled_tensor(constant_op.constant(42)) - golden_lt = core.LabeledTensor(ops.convert_to_tensor(42), []) - self.assertLabeledTensorsEqual(actual, golden_lt) - - def test_invalid_input(self): - with self.assertRaises(ValueError): - core.convert_to_labeled_tensor(math_ops.range(5)) - with self.assertRaises(ValueError): - core.convert_to_labeled_tensor(np.array([1, 2])) - - -class DocStringCheckMixin(object): - # requires self.ops to be defined - - def test_function_docstring_and_name(self): - for op_name, _, _, lt_op in self.ops: - if lt_op is not None: - self.assertIn('tf.%s' % op_name, lt_op.__doc__) - self.assertEqual(op_name, lt_op.__name__) - - -class UnaryOpsTestsMixin(object): - # requires self.ops and self.test_lt to be defined - - def test_core_op(self): - for op_name, _, tf_op, lt_op in self.ops: - if tf_op is not None: - golden_lt = core.LabeledTensor( - tf_op(self.test_lt.tensor), self.test_lt.axes) - actual_lt = lt_op(self.test_lt) - self.assertIn(op_name, actual_lt.name) - self.assertLabeledTensorsEqual(golden_lt, actual_lt) - - def test_infix(self): - for op_name, infix_op, _, _ in self.ops: - if infix_op is not None: - expected_lt = core.LabeledTensor( - infix_op(self.test_lt.tensor), self.test_lt.axes) - actual_lt = infix_op(self.test_lt) - self.assertIn(op_name, actual_lt.name) - self.assertLabeledTensorsEqual(expected_lt, actual_lt) - - -class CoreUnaryOpsTest(Base, DocStringCheckMixin, UnaryOpsTestsMixin): - - def setUp(self): - super(CoreUnaryOpsTest, self).setUp() - - self.ops = [ - ('abs', operator.abs, math_ops.abs, core.abs_function), - ('neg', operator.neg, math_ops.negative, core.neg), - # TODO(shoyer): add unary + to core TensorFlow - ('pos', None, None, None), - ('sign', None, math_ops.sign, core.sign), - ('reciprocal', None, math_ops.reciprocal, core.reciprocal), - ('square', None, math_ops.square, core.square), - ('round', None, math_ops.round, core.round_function), - ('sqrt', None, math_ops.sqrt, core.sqrt), - ('rsqrt', None, math_ops.rsqrt, core.rsqrt), - ('log', None, math_ops.log, core.log), - ('exp', None, math_ops.exp, core.exp), - ('log', None, math_ops.log, core.log), - ('ceil', None, math_ops.ceil, core.ceil), - ('floor', None, math_ops.floor, core.floor), - ('cos', None, math_ops.cos, core.cos), - ('sin', None, math_ops.sin, core.sin), - ('tan', None, math_ops.tan, core.tan), - ('acos', None, math_ops.acos, core.acos), - ('asin', None, math_ops.asin, core.asin), - ('atan', None, math_ops.atan, core.atan), - ('lgamma', None, math_ops.lgamma, core.lgamma), - ('digamma', None, math_ops.digamma, core.digamma), - ('erf', None, math_ops.erf, core.erf), - ('erfc', None, math_ops.erfc, core.erfc), - ('lgamma', None, math_ops.lgamma, core.lgamma), - ] - total_size = np.prod([v.size for v in self.original_lt.axes.values()]) - self.test_lt = core.LabeledTensor( - math_ops.cast(self.original_lt, dtypes.float32) / total_size, - self.original_lt.axes) - - -class LogicalNotTest(Base, DocStringCheckMixin, UnaryOpsTestsMixin): - - def setUp(self): - super(LogicalNotTest, self).setUp() - self.ops = [('logical_not', operator.invert, math_ops.logical_not, - core.logical_not),] - self.test_lt = self.original_lt < 10 - - -class BinaryOpsTestsMixin(object): - # requires self.ops, self.test_lt_1, self.test_lt_2, self.test_lt_1_broadcast - # and self.test_lt_2_broadcast to be defined - - def test_core_op(self): - for op_name, _, tf_op, lt_op in self.ops: - golden_tensor = tf_op(self.test_lt_1_broadcast, self.test_lt_2_broadcast) - golden_lt = core.LabeledTensor(golden_tensor, self.broadcast_axes) - actual_lt = lt_op(self.test_lt_1, self.test_lt_2) - self.assertIn(op_name, actual_lt.name) - self.assertLabeledTensorsEqual(golden_lt, actual_lt) - - def test_infix(self): - for op_name, infix_op, _, lt_op in self.ops: - if infix_op is not None: - expected_lt = lt_op(self.test_lt_1, self.test_lt_2) - actual_lt = infix_op(self.test_lt_1, self.test_lt_2) - self.assertIn(op_name, actual_lt.name) - self.assertLabeledTensorsEqual(expected_lt, actual_lt) - - -class CoreBinaryOpsTest(Base, DocStringCheckMixin, BinaryOpsTestsMixin): - - def setUp(self): - super(CoreBinaryOpsTest, self).setUp() - - self.x_probs_broadcast_tensor = array_ops.reshape( - self.x_probs_lt.tensor, [self.x_size, 1, self.probs_size]) - - self.channel_probs_broadcast_tensor = array_ops.reshape( - self.channel_probs_lt.tensor, [1, self.channel_size, self.probs_size]) - - # == and != are not element-wise for tf.Tensor, so they shouldn't be - # elementwise for LabeledTensor, either. - self.ops = [ - ('add', operator.add, math_ops.add, core.add), - ('sub', operator.sub, math_ops.subtract, core.sub), - ('mul', operator.mul, math_ops.multiply, core.mul), - ('div', operator.truediv, math_ops.div, core.div), - ('mod', operator.mod, math_ops.mod, core.mod), - ('pow', operator.pow, math_ops.pow, core.pow_function), - ('equal', None, math_ops.equal, core.equal), - ('less', operator.lt, math_ops.less, core.less), - ('less_equal', operator.le, math_ops.less_equal, core.less_equal), - ('not_equal', None, math_ops.not_equal, core.not_equal), - ('greater', operator.gt, math_ops.greater, core.greater), - ('greater_equal', operator.ge, math_ops.greater_equal, - core.greater_equal), - ] - self.test_lt_1 = self.x_probs_lt - self.test_lt_2 = self.channel_probs_lt - self.test_lt_1_broadcast = self.x_probs_broadcast_tensor - self.test_lt_2_broadcast = self.channel_probs_broadcast_tensor - self.broadcast_axes = [self.a0, self.a1, self.a3] - - def test_reflexive(self): - labeled_tensor = self.x_probs_lt + 1 # all elements must be >0 for division - for op_name, infix_op, _, lt_op in self.ops: - if infix_op is not None: - expected_lt = lt_op(2, labeled_tensor) - actual_lt = infix_op(2, labeled_tensor) - # Python uses greater for the reflexive version of less (and vise-versa) - if 'less' in op_name: - op_name = op_name.replace('less', 'greater') - elif 'greater' in op_name: - op_name = op_name.replace('greater', 'less') - self.assertIn(op_name, actual_lt.name) - self.assertLabeledTensorsEqual(expected_lt, actual_lt) - - -class LogicalBinaryOpsTest(Base, DocStringCheckMixin, BinaryOpsTestsMixin): - - def setUp(self): - super(LogicalBinaryOpsTest, self).setUp() - - self.ops = [ - ('logical_and', operator.and_, math_ops.logical_and, core.logical_and), - ('logical_or', operator.or_, math_ops.logical_or, core.logical_or), - ('logical_xor', operator.xor, math_ops.logical_xor, core.logical_xor), - ] - self.test_lt_1 = self.original_lt < 10 - self.test_lt_2 = self.original_lt < 5 - self.test_lt_1_broadcast = self.test_lt_1.tensor - self.test_lt_2_broadcast = self.test_lt_2.tensor - self.broadcast_axes = self.test_lt_1.axes - - -class FloatBinaryOpsTest(Base, DocStringCheckMixin, BinaryOpsTestsMixin): - - def setUp(self): - super(FloatBinaryOpsTest, self).setUp() - - self.ops = [ - ('igamma', None, math_ops.igamma, core.igamma), - ('igammac', None, math_ops.igammac, core.igammac), - ('zeta', None, math_ops.zeta, core.zeta), - ('polygamma', None, math_ops.polygamma, core.polygamma), - ('maximum', None, math_ops.maximum, core.maximum), - ('minimum', None, math_ops.minimum, core.minimum), - ('squared_difference', None, math_ops.squared_difference, - core.squared_difference), - ] - total_size = np.prod([v.size for v in self.original_lt.axes.values()]) - test_lt = core.LabeledTensor( - math_ops.cast(self.original_lt, dtypes.float32) / total_size, - self.original_lt.axes) - self.test_lt_1 = test_lt - self.test_lt_2 = 1.0 - test_lt - self.test_lt_1_broadcast = self.test_lt_1.tensor - self.test_lt_2_broadcast = self.test_lt_2.tensor - self.broadcast_axes = self.test_lt_1.axes - - -if __name__ == '__main__': - test_lib.main() diff --git a/tensorflow/contrib/labeled_tensor/python/ops/io_ops.py b/tensorflow/contrib/labeled_tensor/python/ops/io_ops.py deleted file mode 100644 index 3cda9c82788..00000000000 --- a/tensorflow/contrib/labeled_tensor/python/ops/io_ops.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Input parsing code for LabeledTensors.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from six import string_types - -from tensorflow.contrib.labeled_tensor.python.ops import _typecheck as tc -from tensorflow.contrib.labeled_tensor.python.ops import core -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import parsing_ops - - -class FixedLenFeature(object): - """Configuration for parsing a fixed-length input feature. - - Fields: - axes: A list of Axis objects or tuples (axis_name, axis_value), - where `axis_name` is a string and `axis_value` is None (unknown size), an - integer or a list of tick labels. - dtype: Data type of input. - default_value: Value to be used if an example is missing this feature. It - must be compatible with `dtype`. - """ - - def __init__(self, axes, dtype, default_value=None): - self._axes = [core.as_axis(a) for a in axes] - self._dtype = dtype - self._default_value = default_value - - @property - def axes(self): - return self._axes - - @property - def dtype(self): - return self._dtype - - @property - def default_value(self): - return self._default_value - - -@tc.returns(tc.Dict(string_types, parsing_ops.FixedLenFeature)) -@tc.accepts(tc.Mapping(string_types, FixedLenFeature)) -def _labeled_to_unlabeled_features(features): - """Convert a dict of lt.FixedLenFeature into a dict of tf.FixedLenFeature.""" - unlabeled_features = {} - for name, labeled_feature in features.items(): - shape = [ax.size for ax in labeled_feature.axes] - if any(size is None for size in shape): - # This should be caught on the TensorFlow side, but it isn't yet: - # https://github.com/tensorflow/tensorflow/issues/2874 - raise ValueError('axes with unknown size are not supported') - dtype = labeled_feature.dtype - default_value = labeled_feature.default_value - unlabeled_features[name] = parsing_ops.FixedLenFeature( - shape, dtype, default_value) - return unlabeled_features - - -@tc.returns(tc.Dict(string_types, core.LabeledTensor)) -@tc.accepts(core.LabeledTensorLike, tc.Mapping(string_types, FixedLenFeature), - tc.Optional(string_types), object) -def parse_example(serialized, features, name=None, example_names=None): - """Parse `Example` protos into a `dict` of labeled tensors. - - See tf.parse_example. - - Args: - serialized: A 1-D LabeledTensor of strings, a batch of binary serialized - `Example` protos. - features: A `dict` mapping feature keys to `labeled_tensor.FixedLenFeature` - values. - name: A name for this operation (optional). - example_names: A vector (1-D Tensor) of strings (optional), the names of - the serialized protos in the batch. - - Returns: - A `dict` mapping feature keys to `LabeledTensor` values. The single axis - from `serialized` will be prepended to the axes provided by each feature. - - Raises: - ValueError: if any feature is invalid. - """ - serialized = core.convert_to_labeled_tensor(serialized) - unlabeled_features = _labeled_to_unlabeled_features(features) - - unlabeled_parsed = parsing_ops.parse_example( - serialized.tensor, unlabeled_features, name, example_names) - - parsed = {} - for name, parsed_feature in unlabeled_parsed.items(): - axes = list(serialized.axes.values()) + features[name].axes - parsed[name] = core.LabeledTensor(parsed_feature, axes) - - return parsed - - -@tc.returns(tc.Dict(string_types, core.LabeledTensor)) -@tc.accepts(core.LabeledTensorLike, tc.Mapping(string_types, FixedLenFeature), - tc.Optional(string_types), object) -def parse_single_example(serialized, features, name=None, example_names=None): - """Parses a single `Example` proto. - - See tf.parse_single_example. - - Args: - serialized: A scalar string Tensor or LabeledTensor, a single serialized - Example. - features: A `dict` mapping feature keys to `labeled_tensor.FixedLenFeature` - values. - name: A name for this operation (optional). - example_names: (Optional) A scalar string Tensor, the associated name. - - Returns: - A `dict` mapping feature keys to `LabeledTensor` values. - - Raises: - ValueError: if any feature is invalid. - """ - serialized = core.convert_to_labeled_tensor(serialized) - unlabeled_features = _labeled_to_unlabeled_features(features) - - unlabeled_parsed = parsing_ops.parse_single_example( - serialized.tensor, unlabeled_features, name, example_names) - - parsed = {} - for name, parsed_feature in unlabeled_parsed.items(): - parsed[name] = core.LabeledTensor(parsed_feature, features[name].axes) - - return parsed - - -@tc.returns(core.LabeledTensor) -@tc.accepts(dtypes.DType, tc.Collection(tc.Union(string_types, core.AxisLike)), - tc.Optional(string_types)) -def placeholder(dtype, axes, name=None): - """Create a placeholder for a labeled tensor. - - For example: - - lt.placeholder(tf.float32, ['batch', ('channel', ['r', 'g', 'b'])]) - - See tf.compat.v1.placeholder for more details. - - Args: - dtype: The type of elements in the tensor to be fed. - axes: sequence of strings (denoting axes of unknown size) and/or objects - convertable to lt.Axis to label the result. - name: Optional op name. - - Returns: - Placeholder labeled tensor. - """ - with ops.name_scope(name, 'lt_placeholder', []) as scope: - axes = core.Axes([(axis, None) if isinstance(axis, string_types) else axis - for axis in axes]) - shape = [axis.size for axis in axes.values()] - tensor = array_ops.placeholder(dtype, shape, name=scope) - return core.LabeledTensor(tensor, axes) diff --git a/tensorflow/contrib/labeled_tensor/python/ops/io_ops_test.py b/tensorflow/contrib/labeled_tensor/python/ops/io_ops_test.py deleted file mode 100644 index 72dc3a8e445..00000000000 --- a/tensorflow/contrib/labeled_tensor/python/ops/io_ops_test.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.labeled_tensor.python.ops import core -from tensorflow.contrib.labeled_tensor.python.ops import io_ops -from tensorflow.contrib.labeled_tensor.python.ops import test_util -from tensorflow.core.example import example_pb2 -from tensorflow.core.example import feature_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test as test_lib - - -class ParseBase(test_util.Base): - - def setUp(self): - super(ParseBase, self).setUp() - examples = [ - example_pb2.Example(features=feature_pb2.Features(feature={ - 'a': - feature_pb2.Feature( - int64_list=feature_pb2.Int64List(value=[1])), - 'b': - feature_pb2.Feature( - int64_list=feature_pb2.Int64List(value=[2, 3, 4])), - })), - example_pb2.Example(features=feature_pb2.Features(feature={ - 'a': - feature_pb2.Feature( - int64_list=feature_pb2.Int64List(value=[5])), - 'b': - feature_pb2.Feature( - int64_list=feature_pb2.Int64List(value=[6, 7, 8])), - })), - ] - self.serialized = core.LabeledTensor( - constant_op.constant([ex.SerializeToString() for ex in examples]), - ['batch']) - self.features = { - 'a': io_ops.FixedLenFeature([], dtypes.int64), - 'b': io_ops.FixedLenFeature([('x', 3)], dtypes.int64) - } - - -class TestParseExample(ParseBase): - - def test(self): - expected_a = core.LabeledTensor(constant_op.constant([1, 5]), ['batch']) - expected_b = core.LabeledTensor( - constant_op.constant([[2, 3, 4], [6, 7, 8]]), ['batch', 'x']) - parsed = io_ops.parse_example(self.serialized, self.features) - self.assertLabeledTensorsEqual(expected_a, parsed['a']) - self.assertLabeledTensorsEqual(expected_b, parsed['b']) - - def test_placeholder(self): - serialized = core.LabeledTensor( - array_ops.placeholder(dtypes.string, [None]), ['batch']) - # should not raise - io_ops.parse_example(serialized, self.features) - - -class TestParseSingleExample(ParseBase): - - def test(self): - expected_a = core.LabeledTensor(constant_op.constant(1), []) - expected_b = core.LabeledTensor(constant_op.constant([2, 3, 4]), ['x']) - parsed = io_ops.parse_single_example(self.serialized[0], self.features) - self.assertLabeledTensorsEqual(expected_a, parsed['a']) - self.assertLabeledTensorsEqual(expected_b, parsed['b']) - - def test_unknown_size(self): - features = {'a': io_ops.FixedLenFeature([('x', None)], dtypes.int64)} - serialized = array_ops.placeholder(dtypes.string, []) - with self.assertRaisesRegexp(ValueError, 'unknown size'): - io_ops.parse_single_example(serialized, features) - - -class PlaceholderTest(test_util.Base): - - def test_name(self): - placeholder_lt = io_ops.placeholder(dtypes.float32, []) - self.assertIn('lt_placeholder', placeholder_lt.name) - - def test(self): - placeholder_lt = io_ops.placeholder(dtypes.float32, - ['batch', ('x', ['a', 'b'])]) - self.assertEqual(placeholder_lt.dtype, dtypes.float32) - self.assertEqual(placeholder_lt.axes, - core.Axes([('batch', None), ('x', ['a', 'b'])])) - - def test_feed(self): - sess = session.Session() - placeholder_lt = io_ops.placeholder(dtypes.float32, []) - two_times = 2.0 * placeholder_lt - result = sess.run(two_times, {placeholder_lt.tensor: 1}) - self.assertEqual(result, 2.0) - - -if __name__ == '__main__': - test_lib.main() diff --git a/tensorflow/contrib/labeled_tensor/python/ops/nn.py b/tensorflow/contrib/labeled_tensor/python/ops/nn.py deleted file mode 100644 index 90c2ee0b90f..00000000000 --- a/tensorflow/contrib/labeled_tensor/python/ops/nn.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Neural network ops for LabeledTensors.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import nn as contrib_nn -from tensorflow.contrib.labeled_tensor.python.ops import core -from tensorflow.python.ops import nn - -relu = core.define_unary_op('relu', nn.relu) -relu6 = core.define_unary_op('relu6', nn.relu6) -crelu = core.define_unary_op('crelu', nn.crelu) -elu = core.define_unary_op('elu', nn.elu) -softplus = core.define_unary_op('softplus', nn.softplus) - -l2_loss = core.define_unary_op('l2_loss', nn.l2_loss) -sigmoid_cross_entropy_with_logits = core.define_binary_op( - 'sigmoid_cross_entropy_with_logits', - contrib_nn.deprecated_flipped_sigmoid_cross_entropy_with_logits) -softmax = core.define_unary_op('softmax', nn.softmax) -log_softmax = core.define_unary_op('log_softmax', nn.log_softmax) -softmax_cross_entropy_with_logits = core.define_binary_op( - 'softmax_cross_entropy_with_logits', - contrib_nn.deprecated_flipped_softmax_cross_entropy_with_logits) -sparse_softmax_cross_entropy_with_logits = core.define_binary_op( - 'sparse_softmax_cross_entropy_with_logits', - contrib_nn.deprecated_flipped_sparse_softmax_cross_entropy_with_logits) diff --git a/tensorflow/contrib/labeled_tensor/python/ops/nn_test.py b/tensorflow/contrib/labeled_tensor/python/ops/nn_test.py deleted file mode 100644 index a5baf07dffd..00000000000 --- a/tensorflow/contrib/labeled_tensor/python/ops/nn_test.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.labeled_tensor.python.ops import core -from tensorflow.contrib.labeled_tensor.python.ops import nn -from tensorflow.contrib.labeled_tensor.python.ops import test_util -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import nn_ops - - -class NNTests(test_util.Base): - - def setUp(self): - super(NNTests, self).setUp() - self.axes = ['x'] - self.original_lt = core.LabeledTensor([0.0, 0.5, 1.0], self.axes) - self.other_lt = 1 - self.original_lt - - def test_unary_ops(self): - ops = [ - ('relu', nn_ops.relu, nn.relu), - ('relu6', nn_ops.relu6, nn.relu6), - ('crelu', nn_ops.crelu, nn.crelu), - ('elu', nn_ops.elu, nn.elu), - ('softplus', nn_ops.softplus, nn.softplus), - ('l2_loss', nn_ops.l2_loss, nn.l2_loss), - ('softmax', nn_ops.softmax, nn.softmax), - ('log_softmax', nn_ops.log_softmax, nn.log_softmax), - ] - for op_name, tf_op, lt_op in ops: - golden_tensor = tf_op(self.original_lt.tensor) - golden_lt = core.LabeledTensor(golden_tensor, self.axes) - actual_lt = lt_op(self.original_lt) - self.assertIn(op_name, actual_lt.name) - self.assertLabeledTensorsEqual(golden_lt, actual_lt) - - def test_binary_ops(self): - ops = [ - ('sigmoid_cross_entropy_with_logits', - nn_impl.sigmoid_cross_entropy_with_logits, - nn.sigmoid_cross_entropy_with_logits), - ('softmax_cross_entropy_with_logits', - nn_ops.softmax_cross_entropy_with_logits, - nn.softmax_cross_entropy_with_logits), - ('sparse_softmax_cross_entropy_with_logits', - nn_ops.sparse_softmax_cross_entropy_with_logits, - nn.sparse_softmax_cross_entropy_with_logits), - ] - for op_name, tf_op, lt_op in ops: - golden_tensor = tf_op(self.original_lt.tensor, self.other_lt.tensor) - golden_lt = core.LabeledTensor(golden_tensor, self.axes) - actual_lt = lt_op(self.original_lt, self.other_lt) - self.assertIn(op_name, actual_lt.name) - self.assertLabeledTensorsEqual(golden_lt, actual_lt) diff --git a/tensorflow/contrib/labeled_tensor/python/ops/ops.py b/tensorflow/contrib/labeled_tensor/python/ops/ops.py deleted file mode 100644 index 35ab141a18f..00000000000 --- a/tensorflow/contrib/labeled_tensor/python/ops/ops.py +++ /dev/null @@ -1,1283 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Non-core ops for LabeledTensor.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import types - -import numpy as np -from six import string_types - -from tensorflow.contrib.labeled_tensor.python.ops import _typecheck as tc -from tensorflow.contrib.labeled_tensor.python.ops import core -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import functional_ops -from tensorflow.python.ops import map_fn as map_fn_lib -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import numerics -from tensorflow.python.ops import random_ops -from tensorflow.python.training import input # pylint: disable=redefined-builtin -from tensorflow.python.util.compat import collections_abc - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensor, ops.Tensor, core.Axis, - tc.Optional(string_types)) -def _gather_1d_on_axis(labeled_tensor, indexer, axis, name=None): - with ops.name_scope(name, 'lt_take', [labeled_tensor]) as scope: - temp_axes = core.Axes([axis] + list( - labeled_tensor.axes.remove(axis.name).values())) - transposed = core.transpose(labeled_tensor, temp_axes.keys()) - indexed = core.LabeledTensor( - array_ops.gather(transposed.tensor, indexer), temp_axes) - return core.transpose(indexed, labeled_tensor.axes.keys(), name=scope) - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, - tc.Mapping(string_types, - tc.Union(slice, collections_abc.Hashable, list)), - tc.Optional(string_types)) -def select(labeled_tensor, selection, name=None): - """Slice out a subset of the tensor. - - Args: - labeled_tensor: The input tensor. - selection: A dictionary mapping an axis name to a scalar, slice or list of - values to select. Currently supports two types of selections: - (a) Any number of scalar and/or slice selections. - (b) Exactly one list selection, without any scalars or slices. - name: Optional op name. - - Returns: - The selection as a `LabeledTensor`. - - Raises: - ValueError: If the tensor doesn't have an axis in the selection or if - that axis lacks labels. - KeyError: If any labels in a selection are not found in the original axis. - NotImplementedError: If you attempt to combine a list selection with - scalar selection or another list selection. - """ - with ops.name_scope(name, 'lt_select', [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - - slices = {} - indexers = {} - for axis_name, value in selection.items(): - if axis_name not in labeled_tensor.axes: - raise ValueError( - 'The tensor does not have an axis named %s. Its axes are: %r' % - (axis_name, labeled_tensor.axes.keys())) - axis = labeled_tensor.axes[axis_name] - if axis.labels is None: - raise ValueError( - 'The axis named %s does not have labels. The axis is: %r' % - (axis_name, axis)) - - if isinstance(value, slice): - # TODO(shoyer): consider deprecating using slices in favor of lists - if value.start is None: - start = None - else: - start = axis.index(value.start) - - if value.stop is None: - stop = None - else: - # For now, follow the pandas convention of making labeled slices - # inclusive of both bounds. - stop = axis.index(value.stop) + 1 - - if value.step is not None: - raise NotImplementedError('slicing with a step is not yet supported') - - slices[axis_name] = slice(start, stop) - - # Needs to be after checking for slices, since slice objects claim to be - # instances of collections_abc.Hashable but hash() on them fails. - elif isinstance(value, collections_abc.Hashable): - slices[axis_name] = axis.index(value) - - elif isinstance(value, list): - if indexers: - raise NotImplementedError( - 'select does not yet support more than one list selection at ' - 'the same time') - indexer = [axis.index(v) for v in value] - indexers[axis_name] = ops.convert_to_tensor(indexer, dtype=dtypes.int64) - - else: - # If type checking is working properly, this shouldn't be possible. - raise TypeError('cannot handle arbitrary types') - - if indexers and slices: - raise NotImplementedError( - 'select does not yet support combined scalar and list selection') - - # For now, handle array selection separately, because tf.gather_nd does - # not support gradients yet. Later, using gather_nd will let us combine - # these paths. - if indexers: - (axis_name, indexer), = indexers.items() - axis = core.Axis(axis_name, selection[axis_name]) - return _gather_1d_on_axis(labeled_tensor, indexer, axis, name=scope) - else: - return core.slice_function(labeled_tensor, slices, name=scope) - - -@tc.returns(core.LabeledTensor) -@tc.accepts( - tc.Collection(core.LabeledTensorLike), string_types, - tc.Optional(string_types)) -def concat(labeled_tensors, axis_name, name=None): - """Concatenate tensors along a dimension. - - See tf.concat. - - Args: - labeled_tensors: A list of input LabeledTensors. - axis_name: The name of the axis along which to concatenate. - name: Optional op name. - - Returns: - The concatenated tensor. - The coordinate labels for the concatenation dimension are also concatenated, - if they are available for every tensor. - - Raises: - ValueError: If fewer than one tensor inputs is provided, if the tensors - have incompatible axes, or if `axis_name` isn't the name of an axis. - """ - with ops.name_scope(name, 'lt_concat', labeled_tensors) as scope: - labeled_tensors = [ - core.convert_to_labeled_tensor(lt) for lt in labeled_tensors - ] - - if len(labeled_tensors) < 1: - raise ValueError('concat expects at least 1 tensor, but received %s' % - labeled_tensors) - - # All tensors must have these axes. - axes_0 = labeled_tensors[0].axes - axis_names = list(axes_0.keys()) - - if axis_name not in axis_names: - raise ValueError('%s not in %s' % (axis_name, axis_names)) - - shared_axes = axes_0.remove(axis_name) - - tensors = [labeled_tensors[0].tensor] - concat_axis_list = [axes_0[axis_name]] - for labeled_tensor in labeled_tensors[1:]: - current_shared_axes = labeled_tensor.axes.remove(axis_name) - if current_shared_axes != shared_axes: - # TODO(shoyer): add more specific checks about what went wrong, - # including raising AxisOrderError when appropriate - raise ValueError('Mismatched shared axes: the first tensor ' - 'had axes %r but this tensor has axes %r.' % - (shared_axes, current_shared_axes)) - - # Accumulate the axis labels, if they're available. - concat_axis_list.append(labeled_tensor.axes[axis_name]) - tensors.append(labeled_tensor.tensor) - - concat_axis = core.concat_axes(concat_axis_list) - concat_dimension = axis_names.index(axis_name) - concat_tensor = array_ops.concat(tensors, concat_dimension, name=scope) - values = list(axes_0.values()) - concat_axes = (values[:concat_dimension] + [concat_axis] + - values[concat_dimension + 1:]) - - return core.LabeledTensor(concat_tensor, concat_axes) - - -# TODO(shoyer): rename pack/unpack to stack/unstack - - -@tc.returns(core.LabeledTensor) -@tc.accepts( - tc.Collection(core.LabeledTensorLike), - tc.Union(string_types, core.AxisLike), int, tc.Optional(string_types)) -def pack(labeled_tensors, new_axis, axis_position=0, name=None): - """Pack tensors along a new axis. - - See tf.pack. - - Args: - labeled_tensors: The input tensors, which must have identical axes. - new_axis: The name of the new axis, or a tuple containing the name - and coordinate labels. - axis_position: Optional integer position at which to insert the new axis. - name: Optional op name. - - Returns: - The packed tensors as a single LabeledTensor, with `new_axis` in the given - `axis_position`. - - Raises: - ValueError: If fewer than one input tensors is provided, or if the tensors - don't have identical axes. - """ - with ops.name_scope(name, 'lt_pack', labeled_tensors) as scope: - labeled_tensors = [ - core.convert_to_labeled_tensor(lt) for lt in labeled_tensors - ] - - if len(labeled_tensors) < 1: - raise ValueError('pack expects at least 1 tensors, but received %s' % - labeled_tensors) - - axes_0 = labeled_tensors[0].axes - for t in labeled_tensors: - if t.axes != axes_0: - raise ValueError('Non-identical axes. Expected %s but got %s' % - (axes_0, t.axes)) - - pack_op = array_ops.stack( - [t.tensor for t in labeled_tensors], axis=axis_position, name=scope) - axes = list(axes_0.values()) - axes.insert(axis_position, new_axis) - return core.LabeledTensor(pack_op, axes) - - -@tc.returns(tc.List(core.LabeledTensor)) -@tc.accepts(core.LabeledTensorLike, - tc.Optional(string_types), tc.Optional(string_types)) -def unpack(labeled_tensor, axis_name=None, name=None): - """Unpack the tensor. - - See tf.unpack. - - Args: - labeled_tensor: The input tensor. - axis_name: Optional name of axis to unpack. By default, the first axis is - used. - name: Optional op name. - - Returns: - The list of unpacked LabeledTensors. - - Raises: - ValueError: If `axis_name` is not an axis on the input. - """ - with ops.name_scope(name, 'lt_unpack', [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - - axis_names = list(labeled_tensor.axes.keys()) - if axis_name is None: - axis_name = axis_names[0] - - if axis_name not in axis_names: - raise ValueError('%s not in %s' % (axis_name, axis_names)) - axis = axis_names.index(axis_name) - - unpack_ops = array_ops.unstack(labeled_tensor.tensor, axis=axis, name=scope) - axes = [a for i, a in enumerate(labeled_tensor.axes.values()) if i != axis] - return [core.LabeledTensor(t, axes) for t in unpack_ops] - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, - tc.Collection(string_types), - tc.Collection(tc.Union(string_types, core.AxisLike)), - tc.Optional(string_types)) -def reshape(labeled_tensor, existing_axes, new_axes, name=None): - """Reshape specific axes of a LabeledTensor. - - Non-indicated axes remain in their original locations. - - Args: - labeled_tensor: The input tensor. - existing_axes: List of axis names found on the input tensor. These must - appear sequentially in the list of axis names on the input. In other - words, they must be a valid slice of `list(labeled_tensor.axes.keys())`. - new_axes: List of strings, tuples of (axis_name, axis_value) or Axis objects - providing new axes with which to replace `existing_axes` in the reshaped - result. At most one element of `new_axes` may be a string, indicating an - axis with unknown size. - name: Optional op name. - - Returns: - The reshaped LabeledTensor. - - Raises: - ValueError: If `existing_axes` are not all axes on the input, or if more - than one of `new_axes` has unknown size. - AxisOrderError: If `existing_axes` are not a slice of axis names on the - input. - """ - with ops.name_scope(name, 'lt_reshape', [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - - original_axis_names = list(labeled_tensor.axes.keys()) - existing_axes = list(existing_axes) - if not set(existing_axes) <= set(original_axis_names): - raise ValueError('existing_axes %r are not contained in the set of axis ' - 'names %r on the input labeled tensor' % - (existing_axes, original_axis_names)) - - start = original_axis_names.index(existing_axes[0]) - stop = original_axis_names.index(existing_axes[-1]) + 1 - - if existing_axes != original_axis_names[start:stop]: - # We could support existing_axes that aren't a slice by using transpose, - # but that could lead to unpredictable performance consequences because - # transposes are not free in TensorFlow. If we did transpose - # automatically, the user might never realize that their data is being - # produced with the wrong order. (The later will occur with some frequency - # because of how broadcasting automatically choose axis order.) - # So for now we've taken the strict approach. - raise core.AxisOrderError( - 'existing_axes %r are not a slice of axis names %r on the input ' - 'labeled tensor. Use `transpose` or `impose_axis_order` to reorder ' - 'axes on the input explicitly.' % - (existing_axes, original_axis_names)) - - if sum(isinstance(axis, string_types) for axis in new_axes) > 1: - raise ValueError( - 'at most one axis in new_axes can have unknown size. All other ' - 'axes must have an indicated integer size or labels: %r' % new_axes) - - original_values = list(labeled_tensor.axes.values()) - axis_size = lambda axis: -1 if axis.size is None else axis.size - shape = [axis_size(axis) for axis in original_values[:start]] - for axis_ref in new_axes: - if isinstance(axis_ref, string_types): - shape.append(-1) - else: - axis = core.as_axis(axis_ref) - shape.append(axis_size(axis)) - shape.extend(axis_size(axis) for axis in original_values[stop:]) - - reshaped_tensor = array_ops.reshape( - labeled_tensor.tensor, shape, name=scope) - axes = original_values[:start] + list(new_axes) + original_values[stop:] - return core.LabeledTensor(reshaped_tensor, axes) - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, string_types, string_types, - tc.Optional(string_types)) -def rename_axis(labeled_tensor, existing_name, new_name, name=None): - """Rename an axis of LabeledTensor. - - Args: - labeled_tensor: The input tensor. - existing_name: Name for an existing axis on the input. - new_name: Desired replacement name. - name: Optional op name. - - Returns: - LabeledTensor with renamed axis. - - Raises: - ValueError: If `existing_name` is not an axis on the input. - """ - with ops.name_scope(name, 'lt_rename_axis', [labeled_tensor]) as scope: - if existing_name not in labeled_tensor.axes: - raise ValueError('existing_name %r are not contained in the set of axis ' - 'names %r on the input labeled tensor' % - (existing_name, labeled_tensor.axes.keys())) - new_axis = core.Axis(new_name, labeled_tensor.axes[existing_name].value) - return reshape(labeled_tensor, [existing_name], [new_axis], name=scope) - - -@tc.returns(tc.List(core.LabeledTensor)) -@tc.accepts(string_types, collections_abc.Callable, int, bool, - tc.Collection(core.LabeledTensorLike), bool, - tc.Optional(string_types)) -def _batch_helper(default_name, - batch_fn, - batch_size, - enqueue_many, - labeled_tensors, - allow_smaller_final_batch, - name=None): - with ops.name_scope(name, default_name, labeled_tensors) as scope: - labeled_tensors = [ - core.convert_to_labeled_tensor(lt) for lt in labeled_tensors - ] - - batch_ops = batch_fn([t.tensor for t in labeled_tensors], scope) - # TODO(shoyer): Remove this when they sanitize the TF API. - if not isinstance(batch_ops, list): - assert isinstance(batch_ops, ops.Tensor) - batch_ops = [batch_ops] - - if allow_smaller_final_batch: - batch_size = None - - @tc.returns(core.Axes) - @tc.accepts(core.Axes) - def output_axes(axes): - if enqueue_many: - if 'batch' not in axes or list(axes.keys()).index('batch') != 0: - raise ValueError( - 'When enqueue_many is True, input tensors must have an axis ' - 'called "batch" as their first dimension, ' - 'but axes were %s' % axes) - culled_axes = axes.remove('batch') - return core.Axes([('batch', batch_size)] + list(culled_axes.values())) - else: - return core.Axes([('batch', batch_size)] + list(axes.values())) - - output_labeled_tensors = [] - for i, tensor in enumerate(batch_ops): - axes = output_axes(labeled_tensors[i].axes) - output_labeled_tensors.append(core.LabeledTensor(tensor, axes)) - - return output_labeled_tensors - - -@tc.returns(tc.List(core.LabeledTensor)) -@tc.accepts( - tc.Collection(core.LabeledTensorLike), int, int, int, bool, bool, - tc.Optional(string_types)) -def batch(labeled_tensors, - batch_size, - num_threads=1, - capacity=32, - enqueue_many=False, - allow_smaller_final_batch=False, - name=None): - """Rebatch a tensor. - - See tf.batch. - - Args: - labeled_tensors: The input tensors. - batch_size: The output batch size. - num_threads: See tf.batch. - capacity: See tf.batch. - enqueue_many: If true, the input tensors must contain a 'batch' axis as - their first axis. - If false, the input tensors must not contain a 'batch' axis. - See tf.batch. - allow_smaller_final_batch: See tf.batch. - name: Optional op name. - - Returns: - The rebatched tensors. - If enqueue_many is false, the output tensors will have a new 'batch' axis - as their first axis. - - Raises: - ValueError: If enqueue_many is True and the first axis of the tensors - isn't "batch". - """ - - def fn(tensors, scope): - return input.batch( - tensors, - batch_size=batch_size, - num_threads=num_threads, - capacity=capacity, - enqueue_many=enqueue_many, - allow_smaller_final_batch=allow_smaller_final_batch, - name=scope) - - return _batch_helper('lt_batch', fn, batch_size, enqueue_many, - labeled_tensors, allow_smaller_final_batch, name) - - -@tc.returns(tc.List(core.LabeledTensor)) -@tc.accepts( - tc.Collection(core.LabeledTensorLike), int, int, int, bool, int, - tc.Optional(int), bool, tc.Optional(string_types)) -def shuffle_batch(labeled_tensors, - batch_size, - num_threads=1, - capacity=32, - enqueue_many=False, - min_after_dequeue=0, - seed=None, - allow_smaller_final_batch=False, - name=None): - """Rebatch a tensor, with shuffling. - - See tf.batch. - - Args: - labeled_tensors: The input tensors. - batch_size: The output batch size. - num_threads: See tf.batch. - capacity: See tf.batch. - enqueue_many: If true, the input tensors must contain a 'batch' axis as - their first axis. - If false, the input tensors must not contain a 'batch' axis. - See tf.batch. - min_after_dequeue: Minimum number of elements in the queue after a dequeue, - used to ensure mixing. - seed: Optional random seed. - allow_smaller_final_batch: See tf.batch. - name: Optional op name. - - Returns: - The rebatched tensors. - If enqueue_many is false, the output tensors will have a new 'batch' axis - as their first axis. - - Raises: - ValueError: If enqueue_many is True and the first axis of the tensors - isn't "batch". - """ - - def fn(tensors, scope): - return input.shuffle_batch( - tensors, - batch_size=batch_size, - num_threads=num_threads, - capacity=capacity, - enqueue_many=enqueue_many, - min_after_dequeue=min_after_dequeue, - seed=seed, - allow_smaller_final_batch=allow_smaller_final_batch, - name=scope) - - return _batch_helper('lt_shuffle_batch', fn, batch_size, enqueue_many, - labeled_tensors, allow_smaller_final_batch, name) - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, - tc.Mapping(string_types, int), - tc.Optional(int), tc.Optional(string_types)) -def random_crop(labeled_tensor, shape_map, seed=None, name=None): - """Randomly crops a tensor to a given size. - - See tf.random_crop. - - Args: - labeled_tensor: The input tensor. - shape_map: A dictionary mapping axis names to the size of the random crop - for that dimension. - seed: An optional random seed. - name: An optional op name. - - Returns: - A tensor of the same rank as `labeled_tensor`, cropped randomly in the - selected dimensions. - - Raises: - ValueError: If the shape map contains an axis name not in the input tensor. - """ - with ops.name_scope(name, 'lt_random_crop', [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - - for axis_name in shape_map: - if axis_name not in labeled_tensor.axes: - raise ValueError('Selection axis %s not in axes %s' % - (axis_name, labeled_tensor.axes)) - - shape = [] - axes = [] - for axis in labeled_tensor.axes.values(): - if axis.name in shape_map: - size = shape_map[axis.name] - shape.append(size) - # We lose labels for the axes we crop, leaving just the size. - axes.append((axis.name, size)) - else: - shape.append(len(axis)) - axes.append(axis) - - crop_op = random_ops.random_crop( - labeled_tensor.tensor, shape, seed=seed, name=scope) - - return core.LabeledTensor(crop_op, axes) - - -# TODO(shoyer): Allow the user to select the axis over which to map. -@tc.returns(core.LabeledTensor) -@tc.accepts(collections_abc.Callable, core.LabeledTensorLike, - tc.Optional(string_types)) -def map_fn(fn, labeled_tensor, name=None): - """Map on the list of tensors unpacked from labeled_tensor. - - See tf.map_fn. - - Args: - fn: The function to apply to each unpacked LabeledTensor. - It should have type LabeledTensor -> LabeledTensor. - labeled_tensor: The input tensor. - name: Optional op name. - - Returns: - A tensor that packs the results of applying fn to the list of tensors - unpacked from labeled_tensor. - """ - with ops.name_scope(name, 'lt_map_fn', [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - - unpack_lts = unpack(labeled_tensor) - - # TODO(ericmc): Fix this upstream. - if labeled_tensor.dtype == dtypes.string: - # We must construct the full graph here, because map_fn_lib.map_fn - # doesn't work for string-valued tensors. - # Constructing the full graph may be slow. - map_lts = [fn(t) for t in unpack_lts] - return pack(map_lts, list(labeled_tensor.axes.values())[0], name=scope) - else: - # Figure out what the axis labels should be, but use tf.map_fn to - # construct the graph because it's efficient. - # It may be slow to construct the full graph, so we infer the labels from - # the first element. - # TODO(ericmc): This builds a subgraph which then gets thrown away. - # Find a more elegant solution. - first_map_lt = fn(unpack_lts[0]) - final_axes = list(labeled_tensor.axes.values())[:1] + list( - first_map_lt.axes.values()) - - @tc.returns(ops.Tensor) - @tc.accepts(ops.Tensor) - def tf_fn(tensor): - original_axes = list(labeled_tensor.axes.values())[1:] - tensor_lt = core.LabeledTensor(tensor, original_axes) - return fn(tensor_lt).tensor - - map_op = map_fn_lib.map_fn( - tf_fn, labeled_tensor.tensor, dtype=first_map_lt.dtype) - map_lt = core.LabeledTensor(map_op, final_axes) - - return core.identity(map_lt, name=scope) - - -@tc.returns(core.LabeledTensor) -@tc.accepts(collections_abc.Callable, core.LabeledTensorLike, - core.LabeledTensorLike, tc.Optional(string_types)) -def foldl(fn, labeled_tensor, initial_value, name=None): - """Left fold on the list of tensors unpacked from labeled_tensor. - - See tf.foldl. - - Args: - fn: The function to apply to each unpacked LabeledTensor. - It should have type (LabeledTensor, LabeledTensor) -> LabeledTensor. - Its arguments are (accumulated_value, next_value). - labeled_tensor: The input tensor. - initial_value: The initial value of the accumulator. - name: Optional op name. - - Returns: - The accumulated value. - """ - with ops.name_scope(name, 'lt_foldl', - [labeled_tensor, initial_value]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - initial_value = core.convert_to_labeled_tensor(initial_value) - - @tc.returns(ops.Tensor) - @tc.accepts(ops.Tensor, ops.Tensor) - def tf_fn(accumulator, next_element): - accumulator_lt = core.LabeledTensor(accumulator, initial_value.axes) - next_element_lt = core.LabeledTensor( - next_element, list(labeled_tensor.axes.values())[1:]) - return fn(accumulator_lt, next_element_lt).tensor - - foldl_op = functional_ops.foldl( - tf_fn, labeled_tensor.tensor, initializer=initial_value.tensor) - foldl_lt = core.LabeledTensor(foldl_op, initial_value.axes) - - return core.identity(foldl_lt, name=scope) - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, - tc.Optional(tc.Collection(string_types)), tc.Optional(string_types)) -def squeeze(labeled_tensor, axis_names=None, name=None): - """Remove size-1 dimensions. - - See tf.squeeze. - - Args: - labeled_tensor: The input tensor. - axis_names: The names of the dimensions to remove, or None to remove - all size-1 dimensions. - name: Optional op name. - - Returns: - A tensor with the specified dimensions removed. - - Raises: - ValueError: If the named axes are not in the tensor, or if they are - not size-1. - """ - with ops.name_scope(name, 'lt_squeeze', [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - - if axis_names is None: - axis_names = [a.name for a in labeled_tensor.axes.values() if len(a) == 1] - - for axis_name in axis_names: - if axis_name not in labeled_tensor.axes: - raise ValueError('axis %s is not in tensor axes %s' % - (axis_name, labeled_tensor.axes)) - elif len(labeled_tensor.axes[axis_name]) != 1: - raise ValueError( - 'cannot squeeze axis with size greater than 1: (%s, %s)' % - (axis_name, labeled_tensor.axes[axis_name])) - - squeeze_dimensions = [] - axes = [] - for i, axis in enumerate(labeled_tensor.axes.values()): - if axis.name in axis_names: - squeeze_dimensions.append(i) - else: - axes.append(axis) - - if squeeze_dimensions: - squeeze_op = array_ops.squeeze( - labeled_tensor.tensor, squeeze_dimensions, name=scope) - else: - squeeze_op = array_ops.identity(labeled_tensor.tensor, name=scope) - - return core.LabeledTensor(squeeze_op, axes) - - -# pylint: disable=invalid-name -ReduceAxis = tc.Union(string_types, - tc.Tuple(string_types, collections_abc.Hashable)) -ReduceAxes = tc.Optional(tc.Union(ReduceAxis, tc.Collection(ReduceAxis))) -# pylint: enable=invalid-name - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, core.LabeledTensorLike, - tc.Optional(string_types)) -def matmul(a, b, name=None): - """Matrix multiply two tensors with rank 1 or 2. - - If both tensors have rank 2, a matrix-matrix product is performed. - If one tensor has rank 1 and the other has rank 2, then a matrix-vector - product is performed. - If both tensors have rank 1, then a vector dot-product is performed. - (This behavior matches that of `numpy.dot`.) - - Both tensors must share exactly one dimension in common, which is the - dimension the operation is summed along. The inputs will be automatically - transposed if necessary as part of the matmul op. - - We intend to eventually support `matmul` on higher rank input, and also - eventually support summing over any number shared dimensions (via an `axis` - argument), but neither of these features has been implemented yet. - - Args: - a: First LabeledTensor. - b: Second LabeledTensor. - name: Optional op name. - - Returns: - LabeledTensor with the result of matrix multiplication. Axes are ordered by - the current axis_order_scope, if set, or in or order of appearance on the - inputs. - - Raises: - NotImplementedError: If inputs have rank >2 or share multiple axes. - ValueError: If the inputs have rank 0 or do not share any axes. - """ - with ops.name_scope(name, 'lt_matmul', [a, b]) as scope: - - a = core.convert_to_labeled_tensor(a) - b = core.convert_to_labeled_tensor(b) - - if len(a.axes) > 2 or len(b.axes) > 2: - # We could pass batched inputs to tf.matmul to make this work, but we - # would also need to use tf.tile and/or tf.transpose. These are more - # expensive than doing reshapes, so it's not clear if it's a good idea to - # do this automatically. - raise NotImplementedError( - 'matmul currently requires inputs with rank 2 or less, but ' - 'inputs have ranks %r and %r' % (len(a.axes), len(b.axes))) - - if not a.axes or not b.axes: - raise ValueError( - 'matmul currently requires inputs with at least rank 1, but ' - 'inputs have ranks %r and %r' % (len(a.axes), len(b.axes))) - - shared_axes = set(a.axes) & set(b.axes) - if len(shared_axes) > 1: - raise NotImplementedError( - 'matmul does not yet support summing over multiple shared axes: %r. ' - 'Use transpose and reshape to create a single shared axis to sum ' - 'over.' % shared_axes) - if not shared_axes: - raise ValueError('there must have exactly one axis in common between ' - 'input to matmul: %r, %r' % - (a.axes.keys(), b.axes.keys())) - shared_axis, = shared_axes - - if a.axes[shared_axis] != b.axes[shared_axis]: - raise ValueError('axis %r does not match on input arguments: %r vs %r' % - (shared_axis, a.axes[shared_axis].value, - b.axes[shared_axis].value)) - - result_axes = [] - for axes in [a.axes, b.axes]: - for axis in axes.values(): - if axis.name != shared_axis: - result_axes.append(axis) - - axis_scope_order = core.get_axis_order() - if axis_scope_order is not None: - result_axis_names = [axis.name for axis in result_axes] - new_axis_names = [ - name for name in axis_scope_order if name in result_axis_names - ] - if new_axis_names != result_axis_names: - # switch a and b - b, a = a, b - # result_axes is a list of length 1 or 2 - result_axes = result_axes[::-1] - - squeeze_dims = [] - - if len(a.axes) == 1: - a_tensor = array_ops.reshape(a.tensor, (1, -1)) - squeeze_dims.append(0) - transpose_a = False - else: - a_tensor = a.tensor - transpose_a = list(a.axes.keys()).index(shared_axis) == 0 - - if len(b.axes) == 1: - b_tensor = array_ops.reshape(b.tensor, (-1, 1)) - squeeze_dims.append(1) - transpose_b = False - else: - b_tensor = b.tensor - transpose_b = list(b.axes.keys()).index(shared_axis) == 1 - - result_op = math_ops.matmul( - a_tensor, b_tensor, transpose_a=transpose_a, transpose_b=transpose_b) - - if squeeze_dims: - result_op = array_ops.squeeze(result_op, squeeze_dims) - result_op = array_ops.identity(result_op, name=scope) - - return core.LabeledTensor(result_op, result_axes) - - -@tc.returns(types.FunctionType) -@tc.accepts(string_types, collections_abc.Callable) -def define_reduce_op(op_name, reduce_fn): - """Define a reduction op for labeled tensors. - - Args: - op_name: string name of the TensorFlow op. - reduce_fn: function to call to evaluate the op on a tf.Tensor. - - Returns: - Function defining the given reduction op that acts on a LabeledTensor. - """ - - default_name = 'lt_%s' % op_name - - @tc.returns(core.LabeledTensor) - @tc.accepts(core.LabeledTensorLike, ReduceAxes, tc.Optional(string_types)) - def op(labeled_tensor, axes=None, name=None): - """Computes the given reduction across the given axes of a LabeledTensor. - - See `tf.{op_name}` for full details. - - Args: - labeled_tensor: The input tensor. - axes: A set of axes or None. - If None, all axes will be reduced. - Axes must all be strings, in which case those dimensions will be - removed, or pairs of (name, None) or (name, label), in which case those - dimensions will be kept. - name: Optional op name. - - Returns: - The reduced LabeledTensor. - - Raises: - ValueError: if any of the axes to reduce over are not found on - `labeled_tensor`. - """ - with ops.name_scope(name, default_name, [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - - if axes is None: - axes = labeled_tensor.axes.keys() - - if isinstance(axes, (string_types, tuple)): - axes = [axes] - - reduction_axes = {} - axes_to_squeeze = [] - for a in axes: - if isinstance(a, string_types): - # We squeeze out this axis. - reduction_axes[a] = a - axes_to_squeeze.append(a) - else: - # We keep this axis, with the user-provided labels. - (axis_name, label) = a - if label is not None: - # The input was a single label, so make it a list so it can be - # turned into an Axis. - label = [label] - reduction_axes[axis_name] = (axis_name, label) - - for axis_name in reduction_axes: - if axis_name not in labeled_tensor.axes: - raise ValueError('Axis %s not in axes %s' % - (axis_name, labeled_tensor.axes)) - - intermediate_axes = [] - reduction_dimensions = [] - for i, axis in enumerate(labeled_tensor.axes.values()): - if axis.name in reduction_axes: - intermediate_axes.append(reduction_axes[axis.name]) - reduction_dimensions.append(i) - else: - intermediate_axes.append(axis) - - reduce_op = reduce_fn( - labeled_tensor.tensor, reduction_dimensions, keepdims=True) - reduce_lt = core.LabeledTensor(reduce_op, intermediate_axes) - - return squeeze(reduce_lt, axes_to_squeeze, name=scope) - - op.__doc__ = op.__doc__.format(op_name=op_name) - op.__name__ = op_name - - return op - - -reduce_all = define_reduce_op('reduce_all', math_ops.reduce_all) -reduce_any = define_reduce_op('reduce_any', math_ops.reduce_any) -reduce_logsumexp = define_reduce_op('reduce_logsumexp', - math_ops.reduce_logsumexp) -reduce_max = define_reduce_op('reduce_max', math_ops.reduce_max) -reduce_mean = define_reduce_op('reduce_mean', math_ops.reduce_mean) -reduce_min = define_reduce_op('reduce_min', math_ops.reduce_min) -reduce_prod = define_reduce_op('reduce_prod', math_ops.reduce_prod) -reduce_sum = define_reduce_op('reduce_sum', math_ops.reduce_sum) - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, - tc.Mapping(str, tc.Union(int, ops.Tensor)), - tc.Optional(string_types)) -def tile(labeled_tensor, multiples, name=None): - """Constructs a tensor by tiling a given tensor. - - Only axes without tick-labels can be tiled. (Otherwise, axis labels on tiled - tensors would no longer be unique.) - - See lt.tile. - - Args: - labeled_tensor: The input tensor. - multiples: A mapping where the keys are axis names and the values are the - integer number of times to tile along that axis. Only axes with a multiple - different than 1 need be included. - name: Optional op name. - - Returns: - A tensor with the indicated axes tiled. - - Raises: - ValueError: If the tiled axes are not axes in the input tensor, or if any - axes in multiples have tick labels. - """ - with ops.name_scope(name, 'lt_tile', [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - - if not set(multiples.keys()) <= set(labeled_tensor.axes.keys()): - raise ValueError('tile axes %r are not contained in the set of axis ' - 'names %r on the input labeled tensor' % - (multiples.keys(), labeled_tensor.axes)) - - labeled_axes = [ - name for name in multiples - if labeled_tensor.axes[name].labels is not None - ] - if labeled_axes: - raise ValueError('cannot tile axes with tick labels: %r' % labeled_axes) - - multiples_list = [multiples.get(name, 1) for name in labeled_tensor.axes] - tile_op = array_ops.tile(labeled_tensor.tensor, multiples_list, name=scope) - - new_axes = [ - axis.name if axis.labels is None else axis - for axis in labeled_tensor.axes.values() - ] - return core.LabeledTensor(tile_op, new_axes) - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, - tc.Mapping(str, tc.Tuple(core.AxisValue, core.AxisValue)), - string_types, tc.Optional(string_types)) -def pad(labeled_tensor, paddings, mode='CONSTANT', name=None): - """Pads a tensor. - - See tf.pad. - - Args: - labeled_tensor: The input tensor. - paddings: A mapping where the keys are axis names and the values are - tuples where the first element is the padding to insert at the beginning - of the axis and the second is the padding to insert at the end of the - axis. - mode: One of "CONSTANT", "REFLECT", or "SYMMETRIC". - name: Optional op name. - - Returns: - A tensor with the indicated axes padded, optionally with those axes extended - with the provided labels. - - Raises: - ValueError: If the padded axes are not axes in the input tensor. - """ - with ops.name_scope(name, 'lt_pad', [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - - if not set(paddings.keys()) <= set(labeled_tensor.axes.keys()): - raise ValueError('pad axes %r are not contained in the set of axis ' - 'names %r on the input labeled tensor' % - (paddings.keys(), labeled_tensor.axes)) - - new_axes = [] - padding_pairs = [] - for name, axis in labeled_tensor.axes.items(): - if name in paddings: - padding_before, padding_after = paddings[name] - axis_before = core.Axis(name, padding_before) - axis_after = core.Axis(name, padding_after) - new_axes.append(core.concat_axes([axis_before, axis, axis_after])) - padding_pairs.append((len(axis_before), len(axis_after))) - else: - new_axes.append(axis) - padding_pairs.append((0, 0)) - - pad_op = array_ops.pad(labeled_tensor.tensor, - padding_pairs, - mode, - name=scope) - - return core.LabeledTensor(pad_op, new_axes) - - -@tc.returns(core.LabeledTensor) -@tc.accepts( - tc.Union(np.ndarray, list, tuple, core.Scalar), - tc.Optional(dtypes.DType), - tc.Optional( - tc.Union(core.Axes, tc.Collection( - tc.Union(string_types, core.AxisLike)))), tc.Optional(string_types)) -def constant(value, dtype=None, axes=None, name=None): - """Creates a constant tensor. - - If `axes` includes any strings, shape is inferred from `value`. Otherwise, - the sizes of the given `axes` are used to set `shape` for `tf.constant`. - - See tf.constant for more details. - - Args: - value: The input tensor. - dtype: The type of the returned tensor. - axes: Optional Axes, list of strings or list of objects coercible to Axis - objects. By default, axes are assumed to be an empty list (i.e., `value` - is treated as a scalar). - name: Optional op name. - - Returns: - The tensor with elements set to zero. - """ - with ops.name_scope(name, 'lt_constant', [value]) as scope: - - if axes is None: - axes = [] - - if isinstance(axes, core.Axes): - axes = axes.values() - - if any(isinstance(ax, string_types) for ax in axes): - # need to infer shape - shape = None - else: - # axes already indicate shape - axes = [core.as_axis(a) for a in axes] - shape = [a.size for a in axes] - - op = array_ops.constant(value, dtype=dtype, shape=shape, name=scope) - return core.LabeledTensor(op, axes) - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, - tc.Optional(dtypes.DType), tc.Optional(string_types)) -def zeros_like(labeled_tensor, dtype=None, name=None): - """Creates an identical tensor with all elements set to zero. - - Args: - labeled_tensor: The input tensor. - dtype: The type of the returned tensor. - name: Optional op name. - - Returns: - The tensor with elements set to zero. - """ - with ops.name_scope(name, 'lt_zeros_like', [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - op = array_ops.zeros_like(labeled_tensor.tensor, dtype=dtype, name=scope) - return core.LabeledTensor(op, labeled_tensor.axes) - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, - tc.Optional(dtypes.DType), tc.Optional(string_types)) -def ones_like(labeled_tensor, dtype=None, name=None): - """Creates an identical tensor with all elements set to one. - - Args: - labeled_tensor: The input tensor. - dtype: The type of the returned tensor. - name: Optional op name. - - Returns: - The tensor with elements set to one. - """ - with ops.name_scope(name, 'lt_ones_like', [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - op = array_ops.ones_like(labeled_tensor.tensor, dtype=dtype, name=scope) - return core.LabeledTensor(op, labeled_tensor.axes) - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, - tc.Optional(dtypes.DType), tc.Optional(string_types)) -def cast(labeled_tensor, dtype=None, name=None): - """Casts a labeled tensor to a new type. - - Args: - labeled_tensor: The input tensor. - dtype: The type of the returned tensor. - name: Optional op name. - - Returns: - A labeled tensor with the new dtype. - """ - with ops.name_scope(name, 'lt_cast', [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - op = math_ops.cast(labeled_tensor.tensor, dtype=dtype, name=scope) - return core.LabeledTensor(op, labeled_tensor.axes) - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, string_types, tc.Optional(string_types)) -def verify_tensor_all_finite(labeled_tensor, message, name=None): - """Asserts a tensor doesn't contain NaNs or Infs. - - See tf.verify_tensor_all_finite. - - Args: - labeled_tensor: The input tensor. - message: Message to log on failure. - name: Optional op name. - - Returns: - The input tensor. - """ - with ops.name_scope(name, 'lt_verify_tensor_all_finite', - [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - op = numerics.verify_tensor_all_finite( - labeled_tensor.tensor, msg=message, name=scope) - return core.LabeledTensor(op, labeled_tensor.axes) - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, core.LabeledTensorLike, - tc.Optional(string_types)) -def boolean_mask(labeled_tensor, mask, name=None): - """Apply a boolean mask to a labeled tensor. - - Unlike `tf.boolean_mask`, this currently only works on 1-dimensional masks. - The mask is applied to the first axis of `labeled_tensor`. Labels on the first - axis are removed, because True indices in `mask` may not be known dynamically. - - Args: - labeled_tensor: The input tensor. - mask: The type of the returned tensor. - name: Optional op name. - - Returns: - The masked labeled tensor. - - Raises: - ValueError: if the first axis of the mask - """ - with ops.name_scope(name, 'lt_boolean_mask', [labeled_tensor, mask]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - mask = core.convert_to_labeled_tensor(mask) - - if len(mask.axes) > 1: - raise NotImplementedError( - "LabeledTensor's boolean_mask currently only supports 1D masks") - mask_axis = list(mask.axes.values())[0] - lt_axis = list(labeled_tensor.axes.values())[0] - if mask_axis != lt_axis: - raise ValueError('the first axis of the labeled tensor and the mask ' - 'are not equal:\n%r\n%r' % (lt_axis, mask_axis)) - op = array_ops.boolean_mask(labeled_tensor.tensor, mask.tensor, name=scope) - # TODO(shoyer): attempt to infer labels for the masked values, by calling - # tf.get_static_value on the mask? - axes = [lt_axis.name] + list(labeled_tensor.axes.values())[1:] - return core.LabeledTensor(op, axes) - - -@tc.returns(core.LabeledTensor) -@tc.accepts(core.LabeledTensorLike, core.LabeledTensorLike, - core.LabeledTensorLike, tc.Optional(string_types)) -def where(condition, x, y, name=None): - """Return elements from x or y depending on condition. - - See `tf.where` for more details. This function currently only implements the - three argument version of where. - - Args: - condition: LabeledTensor of type `bool`. - x: LabeledTensor for values where condition is true. - y: LabeledTensor for values where condition is false. - name: Optional op name. - - Returns: - The labeled tensor with values according to condition. - - Raises: - ValueError: if `x` and `y` have different axes, or if the axes of `x` do not - start with the axes of `condition`. - """ - with ops.name_scope(name, 'lt_where', [condition, x, y]) as scope: - condition = core.convert_to_labeled_tensor(condition) - x = core.convert_to_labeled_tensor(x) - y = core.convert_to_labeled_tensor(y) - - if not condition.axes == x.axes == y.axes: - raise ValueError('all inputs to `where` must have equal axes') - - op = array_ops.where(condition.tensor, x.tensor, y.tensor, name=scope) - return core.LabeledTensor(op, x.axes) diff --git a/tensorflow/contrib/labeled_tensor/python/ops/ops_test.py b/tensorflow/contrib/labeled_tensor/python/ops/ops_test.py deleted file mode 100644 index 9a402d888cf..00000000000 --- a/tensorflow/contrib/labeled_tensor/python/ops/ops_test.py +++ /dev/null @@ -1,978 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from six.moves import range # pylint: disable=redefined-builtin - -from tensorflow.contrib.labeled_tensor.python.ops import core -from tensorflow.contrib.labeled_tensor.python.ops import ops -from tensorflow.contrib.labeled_tensor.python.ops import test_util -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import string_ops -from tensorflow.python.platform import test as test_lib - - -class Base(test_util.Base): - - def setUp(self): - super(Base, self).setUp() - - self.x_size = 7 - self.channel_size = 3 - self.z_size = 4 - self.probs_size = 11 - - tensor = math_ops.range(0, self.x_size * self.channel_size * self.z_size * - self.probs_size) - tensor = array_ops.reshape( - tensor, [self.x_size, self.channel_size, self.z_size, self.probs_size]) - a0 = ('x', range(self.x_size)) - a1 = ('channel', ['red', 'green', 'blue']) - a2 = 'z' - a3 = ('probs', np.linspace(0.0, 1.0, self.probs_size)) - - self.tensor = tensor - self.a0 = a0 - self.a1 = a1 - self.a2 = a2 - self.a2_resolved = ('z', self.z_size) - self.a3 = a3 - self.original_lt = core.LabeledTensor(tensor, [a0, a1, a2, a3]) - - self.x_probs_lt = core.slice_function(self.original_lt, {'z': 0}) - self.x_probs_lt = ops.select(self.x_probs_lt, {'channel': 'red'}) - self.channel_probs_lt = core.slice_function(self.original_lt, - {'x': 3, - 'z': 0}) - - -class SelectTest(Base): - - def test_name(self): - select_lt = ops.select(self.original_lt, {'channel': 'green'}) - self.assertIn('lt_select', select_lt.name) - - def test_scalar(self): - select_lt = ops.select(self.original_lt, {'channel': 'green'}) - golden_lt = core.LabeledTensor(self.tensor[:, 1, :, :], - [self.a0, self.a2, self.a3]) - self.assertLabeledTensorsEqual(select_lt, golden_lt) - - def test_slice(self): - select_lt = ops.select(self.original_lt, {'channel': slice('red', 'green')}) - a1_sliced = ('channel', ['red', 'green']) - golden_lt = core.LabeledTensor(self.tensor[:, :2, :, :], - [self.a0, a1_sliced, self.a2, self.a3]) - self.assertLabeledTensorsEqual(select_lt, golden_lt) - - def test_slices(self): - select_lt = ops.select(self.original_lt, - {'x': slice(1, 4), - 'channel': slice('green', None)}) - - a0_sliced = ('x', range(1, 5)) - a1_sliced = ('channel', ['green', 'blue']) - golden_lt = core.LabeledTensor(self.tensor[1:5, 1:, :, :], - [a0_sliced, a1_sliced, self.a2, self.a3]) - self.assertLabeledTensorsEqual(select_lt, golden_lt) - - def test_list(self): - select_lt = ops.select(self.original_lt, {'channel': ['red', 'green']}) - a1_sliced = ('channel', ['red', 'green']) - golden_lt = core.LabeledTensor(self.tensor[:, :2, :, :], - [self.a0, a1_sliced, self.a2, self.a3]) - self.assertLabeledTensorsEqual(select_lt, golden_lt) - - def test_list_one_item(self): - select_lt = ops.select(self.original_lt, {'channel': ['red']}) - a1_sliced = ('channel', ['red']) - golden_lt = core.LabeledTensor(self.tensor[:, :1, :, :], - [self.a0, a1_sliced, self.a2, self.a3]) - self.assertLabeledTensorsEqual(select_lt, golden_lt) - - def test_list_zero_items(self): - select_lt = ops.select(self.original_lt, {'channel': []}) - golden_lt = core.LabeledTensor(self.tensor[:, :0, :, :], - [self.a0, 'channel', self.a2, self.a3]) - self.assertLabeledTensorsEqual(select_lt, golden_lt) - - def test_scalars(self): - select_lt = ops.select(self.original_lt, {'x': 1, 'channel': 'green'}) - golden_lt = core.LabeledTensor(self.tensor[1, 1, :, :], [self.a2, self.a3]) - self.assertLabeledTensorsEqual(select_lt, golden_lt) - - def test_tuple(self): - original_lt = core.LabeledTensor(constant_op.constant([5, 6]), - [('x', [(1, 2), (3, 4)])]) - select_lt = ops.select(original_lt, {'x': (1, 2)}) - golden_lt = core.LabeledTensor(constant_op.constant(5), []) - self.assertLabeledTensorsEqual(select_lt, golden_lt) - - def test_invalid_input(self): - with self.assertRaises(ValueError): - ops.select(self.original_lt, {'foo': 1}) - with self.assertRaises(ValueError): - ops.select(self.original_lt, {'z': 1}) - with self.assertRaises(KeyError): - ops.select(self.original_lt, {'channel': 'purple'}) - with self.assertRaises(KeyError): - ops.select(self.original_lt, {'channel': ['red', 'purple']}) - with self.assertRaises(NotImplementedError): - ops.select(self.original_lt, {'channel': ['red'], 'x': [1]}) - with self.assertRaises(NotImplementedError): - ops.select(self.original_lt, {'channel': ['red'], 'x': 1}) - with self.assertRaises(NotImplementedError): - ops.select(self.original_lt, {'channel': slice('red', 'green', 2)}) - - -class ConcatTest(Base): - - def setUp(self): - super(ConcatTest, self).setUp() - - self.red_lt = ops.select(self.original_lt, {'channel': ['red']}) - self.green_lt = ops.select(self.original_lt, {'channel': ['green']}) - self.blue_lt = ops.select(self.original_lt, {'channel': ['blue']}) - - def test_name(self): - concat_lt = ops.concat([self.red_lt, self.blue_lt], 'channel') - self.assertIn('lt_concat', concat_lt.name) - - def test(self): - concat_lt = ops.concat([self.red_lt, self.green_lt], 'channel') - golden_lt = ops.select(self.original_lt, {'channel': ['red', 'green']}) - - self.assertLabeledTensorsEqual(concat_lt, golden_lt) - - def test_transposed(self): - green_transposed = core.transpose(self.green_lt, - ['probs', 'channel', 'z', 'x']) - with self.assertRaises(ValueError): - ops.concat([self.red_lt, green_transposed], 'channel') - - def test_invalid_input(self): - with self.assertRaises(ValueError): - ops.concat([], 'channel') - with self.assertRaises(ValueError): - ops.concat([self.red_lt, self.red_lt], 'channel') - with self.assertRaises(ValueError): - ops.concat([self.red_lt, self.red_lt], 'foo') - - -class PackTest(Base): - - def test_name(self): - pack_lt = ops.pack([self.original_lt, self.original_lt], 'batch') - self.assertIn('lt_pack', pack_lt.name) - - def test(self): - pack_lt = ops.pack([self.original_lt, self.original_lt], 'batch') - golden_lt = core.LabeledTensor( - array_ops.stack([self.original_lt.tensor, self.original_lt.tensor]), - ['batch', self.a0, self.a1, self.a2, self.a3]) - - self.assertLabeledTensorsEqual(pack_lt, golden_lt) - - def test_axis(self): - pack_lt = ops.pack( - [self.original_lt, self.original_lt], new_axis='batch', axis_position=4) - golden_lt = core.LabeledTensor( - array_ops.stack( - [self.original_lt.tensor, self.original_lt.tensor], axis=4), - [self.a0, self.a1, self.a2, self.a3, 'batch']) - - self.assertLabeledTensorsEqual(pack_lt, golden_lt) - - def test_invalid_input(self): - with self.assertRaises(ValueError): - ops.pack([self.original_lt, self.original_lt], 'channel') - - -class UnpackTest(Base): - - def test_name(self): - unpack_lts = ops.unpack(self.original_lt) - for t in unpack_lts: - self.assertIn('lt_unpack', t.name) - - def test(self): - unpack_lt = ops.unpack(self.original_lt)[0] - golden_lt = core.LabeledTensor( - array_ops.unstack(self.original_lt.tensor)[0], - [self.a1, self.a2, self.a3]) - - self.assertLabeledTensorsEqual(unpack_lt, golden_lt) - - def test_axis(self): - unpack_lt = ops.unpack(self.original_lt, axis_name='z')[0] - golden_lt = core.LabeledTensor( - array_ops.unstack( - self.original_lt.tensor, axis=2)[0], [self.a0, self.a1, self.a3]) - - self.assertLabeledTensorsEqual(unpack_lt, golden_lt) - - def test_invalid_input(self): - with self.assertRaises(ValueError): - ops.unpack(self.original_lt, axis_name='not_found') - - -class ReshapeTest(Base): - - def test_name(self): - reshape_lt = ops.reshape(self.original_lt, ['channel'], ['foo']) - self.assertIn('lt_reshape', reshape_lt.name) - - def test_identity(self): - reshape_lt = ops.reshape(self.original_lt, - self.original_lt.axes.keys(), - self.original_lt.axes.values()) - self.assertLabeledTensorsEqual(reshape_lt, self.original_lt) - - def test_known_size(self): - new_dim_size = self.channel_size * self.z_size * self.probs_size - reshape_lt = ops.reshape(self.original_lt, ['channel', 'z', 'probs'], - [('new_dim', new_dim_size)]) - golden_lt = core.LabeledTensor( - array_ops.reshape(self.original_lt.tensor, [self.x_size, -1]), - [self.original_lt.axes['x'], 'new_dim']) - self.assertLabeledTensorsEqual(reshape_lt, golden_lt) - - def test_unknown_size(self): - reshape_lt = ops.reshape(self.original_lt, ['channel', 'z', 'probs'], - ['new_dim']) - golden_lt = core.LabeledTensor( - array_ops.reshape(self.original_lt.tensor, [self.x_size, -1]), - [self.original_lt.axes['x'], 'new_dim']) - self.assertLabeledTensorsEqual(reshape_lt, golden_lt) - - def test_unknown_dimension(self): - orig_lt = core.LabeledTensor( - array_ops.placeholder(dtypes.float32, [None]), ['x']) - reshape_lt = ops.reshape(orig_lt, ['x'], ['y', ('z', 1)]) - self.assertEqual(reshape_lt.axes, core.Axes([('y', None), ('z', 1)])) - with self.cached_session() as sess: - result = sess.run(reshape_lt, feed_dict={orig_lt.tensor: [1, 2]}) - np.testing.assert_array_equal(result, [[1], [2]]) - - def test_with_labels(self): - new_dim_size = self.channel_size * self.z_size * self.probs_size - reshape_lt = ops.reshape(self.original_lt, ['channel', 'z', 'probs'], - [('new_dim', range(new_dim_size))]) - golden_lt = core.LabeledTensor( - array_ops.reshape(self.original_lt.tensor, [self.x_size, -1]), - [self.original_lt.axes['x'], ('new_dim', range(new_dim_size))]) - self.assertLabeledTensorsEqual(reshape_lt, golden_lt) - - def test_invalid_input(self): - with self.assertRaisesRegexp(ValueError, 'not contained in the set'): - ops.reshape(self.original_lt, ['foo'], ['bar']) - with self.assertRaisesRegexp(core.AxisOrderError, - 'not a slice of axis names'): - ops.reshape(self.original_lt, ['probs', 'z'], ['bar']) - with self.assertRaisesRegexp(ValueError, 'at most one axis in new_axes'): - ops.reshape(self.original_lt, ['probs'], ['foo', 'bar']) - - -class RenameAxisTest(Base): - - def test_name(self): - rename_axis_lt = ops.rename_axis(self.original_lt, 'channel', 'foo') - self.assertIn('lt_rename_axis', rename_axis_lt.name) - - def test_identity(self): - rename_axis_lt = ops.rename_axis(self.original_lt, 'channel', 'channel') - self.assertLabeledTensorsEqual(rename_axis_lt, self.original_lt) - - def test_new_name(self): - rename_axis_lt = ops.rename_axis(self.original_lt, 'channel', 'foo') - expected_axes = [(name if name != 'channel' else 'foo', axis.value) - for name, axis in self.original_lt.axes.items()] - expected_lt = core.LabeledTensor(self.original_lt.tensor, expected_axes) - self.assertLabeledTensorsEqual(rename_axis_lt, expected_lt) - - def test_invalid_input(self): - with self.assertRaisesRegexp(ValueError, 'not contained in the set'): - ops.rename_axis(self.original_lt, 'foo', 'bar') - - -class BatchTest(Base): - - def setUp(self): - super(BatchTest, self).setUp() - - tensors = [] - for i in range(10): - offset_lt = core.LabeledTensor(constant_op.constant(i), []) - tensors.append(core.add(self.original_lt, offset_lt)) - self.pack_lt = ops.pack(tensors, 'batch') - - def test_name(self): - batch_ops = ops.batch( - [self.pack_lt, self.pack_lt], batch_size=2, enqueue_many=True) - for bo in batch_ops: - self.assertIn('lt_batch', bo.name) - - def test_enqueue_many(self): - [batch_2_op] = ops.batch([self.pack_lt], batch_size=2, enqueue_many=True) - self.assertEqual(len(batch_2_op.axes['batch']), 2) - - [batch_10_op] = ops.batch([batch_2_op], batch_size=10, enqueue_many=True) - - self.assertLabeledTensorsEqual(self.pack_lt, batch_10_op) - - def test_no_enqueue_many(self): - [batch_2_op] = ops.batch([self.original_lt], batch_size=2) - self.assertEqual(len(batch_2_op.axes['batch']), 2) - - [batch_10_op] = ops.batch([batch_2_op], batch_size=10, enqueue_many=True) - - self.assertLabeledTensorsEqual( - ops.pack(10 * [self.original_lt], 'batch'), batch_10_op) - - def test_invalid_input(self): - with self.assertRaises(ValueError): - ops.batch([self.original_lt], 3, enqueue_many=True) - - def test_allow_smaller_final_batch(self): - [batch_2_op] = ops.batch( - [self.original_lt], batch_size=2, allow_smaller_final_batch=True) - self.assertEqual(batch_2_op.axes['batch'].size, None) - - -class ShuffleBatchTest(Base): - - def setUp(self): - super(ShuffleBatchTest, self).setUp() - - tensors = [] - for i in range(10): - offset_lt = core.LabeledTensor(constant_op.constant(i), []) - tensors.append(core.add(self.original_lt, offset_lt)) - self.pack_lt = ops.pack(tensors, 'batch') - - def test_name(self): - batch_lts = ops.shuffle_batch( - [self.pack_lt, self.pack_lt], batch_size=2, enqueue_many=True) - for blt in batch_lts: - self.assertIn('lt_shuffle_batch', blt.name) - - def test_enqueue_many(self): - [batch_2_lt] = ops.shuffle_batch( - [self.pack_lt], - batch_size=2, - enqueue_many=True, - min_after_dequeue=8, - seed=0) - self.assertEqual(len(batch_2_lt.axes['batch']), 2) - - [batch_10_lt] = ops.batch([batch_2_lt], batch_size=10, enqueue_many=True) - - self.assertEqual(batch_10_lt.axes, self.pack_lt.axes) - [batch_10, pack] = self.eval([batch_10_lt.tensor, self.pack_lt.tensor]) - self.assertFalse((batch_10 == pack).all()) - - def test_allow_smaller_final_batch(self): - [batch_2_op] = ops.shuffle_batch( - [self.original_lt], batch_size=2, allow_smaller_final_batch=True) - self.assertEqual(batch_2_op.axes['batch'].size, None) - - -class RandomCropTest(Base): - - def test_name(self): - crop_lt = ops.random_crop(self.original_lt, {'probs': 3}) - self.assertIn('lt_random_crop', crop_lt.name) - - def test_single(self): - crop_lt = ops.random_crop(self.original_lt, {'probs': 3}) - - self.assertEqual( - core.Axes([self.a0, self.a1, self.a2_resolved, ('probs', 3)]), - crop_lt.axes) - - def test_double(self): - crop_lt = ops.random_crop(self.original_lt, {'probs': 3, 'channel': 2}) - - self.assertEqual( - core.Axes([self.a0, ('channel', 2), self.a2_resolved, ('probs', 3)]), - crop_lt.axes) - - def test_size1(self): - crop_lt = ops.random_crop(self.original_lt, {'probs': 1}) - - self.assertEqual( - core.Axes([self.a0, self.a1, self.a2_resolved, ('probs', 1)]), - crop_lt.axes) - - def test_different_seeds(self): - crop_0_lt = ops.random_crop( - self.original_lt, {'probs': 3, - 'channel': 2}, seed=0) - crop_1_lt = ops.random_crop( - self.original_lt, {'probs': 3, - 'channel': 2}, seed=1) - - self.assertEqual(crop_0_lt.axes, crop_1_lt.axes) - [crop_0, crop_1] = self.eval([crop_0_lt.tensor, crop_1_lt.tensor]) - self.assertFalse((crop_0 == crop_1).all()) - - def test_identical_seeds(self): - crop_0_lt = ops.random_crop( - self.original_lt, {'probs': 3, - 'channel': 2}, seed=0) - crop_1_lt = ops.random_crop( - self.original_lt, {'probs': 3, - 'channel': 2}, seed=0) - - self.assertLabeledTensorsEqual(crop_0_lt, crop_1_lt) - - def test_crop_idempotent(self): - crop_0_lt = ops.random_crop( - self.original_lt, {'probs': 3, - 'channel': 2}, seed=0) - crop_1_lt = ops.random_crop(crop_0_lt, {'probs': 3, 'channel': 2}, seed=1) - - self.assertLabeledTensorsEqual(crop_0_lt, crop_1_lt) - - def test_invalid_input(self): - with self.assertRaises(ValueError): - ops.random_crop(self.original_lt, {'foobar': 2}) - - -class MapFnTest(Base): - - def test_name(self): - map_lt = ops.map_fn(core.identity, self.original_lt) - self.assertIn('lt_map_fn', map_lt.name) - - def test_identity(self): - map_lt = ops.map_fn(core.identity, self.original_lt) - self.assertLabeledTensorsEqual(map_lt, self.original_lt) - - def test_callable_object(self): - - class Identity(object): - - def __call__(self, other): - return other - - map_lt = ops.map_fn(Identity(), self.original_lt) - self.assertLabeledTensorsEqual(map_lt, self.original_lt) - - def test_slice(self): - map_lt = ops.map_fn(lambda t: core.slice_function(t, {'channel': 1}), - self.original_lt) - slice_lt = core.slice_function(self.original_lt, {'channel': 1}) - self.assertLabeledTensorsEqual(map_lt, slice_lt) - - def test_string(self): - - def fn(entry_lt): - op = string_ops.string_join([entry_lt, 'world']) - return core.LabeledTensor(op, []) - - tensor_lt = ops.constant(['hi', 'bye'], axes=['batch']) - map_lt = ops.map_fn(fn, tensor_lt) - golden_lt = ops.constant(['hiworld', 'byeworld'], axes=['batch']) - - self.assertLabeledTensorsEqual(map_lt, golden_lt) - - -class FoldlTest(Base): - - def test_name(self): - foldl_lt = ops.foldl(core.add, self.original_lt, - core.slice_function(self.original_lt, {'x': 0})) - self.assertIn('lt_foldl', foldl_lt.name) - - def test_sum(self): - initializer_lt = ops.constant([0, 10], axes=['y']) - tensor_lt = ops.constant([[1, 2], [3, 4], [5, 6]], axes=['x', 'y']) - foldl_lt = ops.foldl(core.add, tensor_lt, initializer_lt) - golden_lt = ops.constant([9, 22], axes=['y']) - self.assertLabeledTensorsEqual(foldl_lt, golden_lt) - - -class SqueezeTest(Base): - - def setUp(self): - super(SqueezeTest, self).setUp() - - self.squeezable_lt = core.slice_function( - self.original_lt, {'channel': slice(0, 1), - 'probs': slice(0, 1)}) - - def test_name(self): - squeeze_lt = ops.squeeze(self.squeezable_lt) - self.assertIn('lt_squeeze', squeeze_lt.name) - - def test_none(self): - none_lt = ops.squeeze(self.squeezable_lt, None) - axes_lt = ops.squeeze(self.squeezable_lt, ['channel', 'probs']) - self.assertLabeledTensorsEqual(none_lt, axes_lt) - - def test(self): - squeeze_lt = ops.squeeze(self.squeezable_lt, ['probs']) - golden_lt = core.slice_function(self.squeezable_lt, {'probs': 0}) - self.assertLabeledTensorsEqual(squeeze_lt, golden_lt) - - def test_invalid_input(self): - with self.assertRaises(ValueError): - ops.squeeze(self.original_lt, ['channel']) - with self.assertRaises(ValueError): - ops.squeeze(self.squeezable_lt, ['foo']) - - -class MatMulTest(Base): - - def test_name(self): - x_lt = core.LabeledTensor(array_ops.ones((3,)), ['x']) - matmul_lt = ops.matmul(x_lt, x_lt) - self.assertIn('lt_matmul', matmul_lt.name) - - def test_vector_vector(self): - x_lt = core.LabeledTensor(math_ops.range(3), ['x']) - matmul_lt = ops.matmul(x_lt, x_lt) - golden_lt = core.convert_to_labeled_tensor(5) - self.assertLabeledTensorsEqual(matmul_lt, golden_lt) - - def test_matrix_vector(self): - xy_lt = core.LabeledTensor( - array_ops.reshape(math_ops.range(6), (2, 3)), ['x', 'y']) - y_lt = core.LabeledTensor(math_ops.range(3), ['y']) - - matmul_lt = ops.matmul(xy_lt, y_lt) - golden_lt = core.LabeledTensor( - math_ops.matmul(xy_lt.tensor, array_ops.reshape(y_lt.tensor, - (-1, 1)))[:, 0], ['x']) - self.assertLabeledTensorsEqual(matmul_lt, golden_lt) - - matmul_lt = ops.matmul(y_lt, xy_lt) - self.assertLabeledTensorsEqual(matmul_lt, golden_lt) - - def test_matrix_matrix(self): - xy_lt = core.LabeledTensor( - array_ops.reshape(math_ops.range(6), (2, 3)), ['x', 'y']) - yz_lt = core.LabeledTensor( - array_ops.reshape(math_ops.range(12), (3, 4)), ['y', 'z']) - - matmul_lt = ops.matmul(xy_lt, yz_lt) - golden_lt = core.LabeledTensor( - math_ops.matmul(xy_lt.tensor, yz_lt.tensor), ['x', 'z']) - self.assertLabeledTensorsEqual(matmul_lt, golden_lt) - - transpose = lambda x: core.transpose(x, list(x.axes.keys())[::-1]) - - matmul_lt = ops.matmul(xy_lt, transpose(yz_lt)) - self.assertLabeledTensorsEqual(matmul_lt, golden_lt) - - matmul_lt = ops.matmul(transpose(xy_lt), yz_lt) - self.assertLabeledTensorsEqual(matmul_lt, golden_lt) - - matmul_lt = ops.matmul(transpose(xy_lt), transpose(yz_lt)) - self.assertLabeledTensorsEqual(matmul_lt, golden_lt) - - matmul_lt = ops.matmul(yz_lt, xy_lt) - self.assertLabeledTensorsEqual(matmul_lt, transpose(golden_lt)) - - def test_matrix_matrix_axis_order(self): - xy_lt = core.LabeledTensor( - array_ops.reshape(math_ops.range(6), (2, 3)), ['x', 'y']) - yz_lt = core.LabeledTensor( - array_ops.reshape(math_ops.range(12), (3, 4)), ['y', 'z']) - - golden_lt = core.LabeledTensor( - math_ops.matmul(xy_lt.tensor, yz_lt.tensor), ['x', 'z']) - - with core.axis_order_scope(['x', 'y', 'z']): - - matmul_lt = ops.matmul(xy_lt, yz_lt) - self.assertLabeledTensorsEqual(matmul_lt, golden_lt) - - matmul_lt = ops.matmul(yz_lt, xy_lt) - self.assertLabeledTensorsEqual(matmul_lt, golden_lt) - - def test_invalid(self): - scalar_lt = core.LabeledTensor(array_ops.ones(()), []) - x_lt = core.LabeledTensor(array_ops.ones((2,)), ['x']) - x2_lt = core.LabeledTensor(array_ops.ones((3,)), ['x']) - y_lt = core.LabeledTensor(array_ops.ones((3,)), ['y']) - xy_lt = core.LabeledTensor(array_ops.ones((2, 3)), ['x', 'y']) - xyz_lt = core.LabeledTensor(array_ops.ones((2, 3, 1)), ['x', 'y', 'z']) - - with self.assertRaisesRegexp(ValueError, 'inputs with at least rank'): - ops.matmul(x_lt, scalar_lt) - - with self.assertRaises(NotImplementedError): - ops.matmul(x_lt, xyz_lt) - - with self.assertRaisesRegexp(ValueError, 'exactly one axis in common'): - ops.matmul(x_lt, y_lt) - - with self.assertRaises(NotImplementedError): - ops.matmul(xy_lt, xy_lt) - - with self.assertRaisesRegexp(ValueError, 'does not match'): - ops.matmul(x_lt, x2_lt) - - -class ReduceSumTest(Base): - - def test_name(self): - sum_lt = ops.reduce_sum(self.original_lt, {'channel'}) - self.assertIn('lt_reduce_sum', sum_lt.name) - - def test_drop_axis(self): - sum_lt = ops.reduce_sum(self.original_lt, {'channel'}) - golden_lt = core.LabeledTensor( - math_ops.reduce_sum(self.original_lt.tensor, 1), - [self.a0, self.a2, self.a3]) - self.assertLabeledTensorsEqual(sum_lt, golden_lt) - - def test_drop_scalar_axis(self): - sum_lt = ops.reduce_sum(self.original_lt, 'channel') - golden_lt = core.LabeledTensor( - math_ops.reduce_sum(self.original_lt.tensor, 1), - [self.a0, self.a2, self.a3]) - self.assertLabeledTensorsEqual(sum_lt, golden_lt) - - def test_keep_axis(self): - sum_lt = ops.reduce_sum(self.original_lt, {('channel', 'hihowareyou')}) - golden_lt = core.LabeledTensor( - math_ops.reduce_sum( - self.original_lt.tensor, 1, keepdims=True), - [self.a0, ('channel', ['hihowareyou']), self.a2, self.a3]) - self.assertLabeledTensorsEqual(sum_lt, golden_lt) - - def test_keep_scalar_axis(self): - sum_lt = ops.reduce_sum(self.original_lt, ('channel', 'hihowareyou')) - golden_lt = core.LabeledTensor( - math_ops.reduce_sum( - self.original_lt.tensor, 1, keepdims=True), - [self.a0, ('channel', ['hihowareyou']), self.a2, self.a3]) - self.assertLabeledTensorsEqual(sum_lt, golden_lt) - - def test_scalar(self): - scalar_lt = core.LabeledTensor(constant_op.constant(42), []) - reduce_lt = ops.reduce_sum(scalar_lt, []) - self.assertLabeledTensorsEqual(reduce_lt, scalar_lt) - - def test_empty_list(self): - reduce_lt = ops.reduce_sum(self.original_lt, []) - self.assertLabeledTensorsEqual(reduce_lt, self.original_lt) - - def test_none(self): - sum_lt = ops.reduce_sum(self.original_lt) - golden_lt = core.LabeledTensor( - math_ops.reduce_sum(self.original_lt.tensor), []) - self.assertLabeledTensorsEqual(sum_lt, golden_lt) - - def test_function_docstring_and_name(self): - self.assertIn('tf.reduce_sum', ops.reduce_sum.__doc__) - self.assertEqual('reduce_sum', ops.reduce_sum.__name__) - - -class ReduceMeanTest(Base): - - def test_name(self): - actual_lt = ops.reduce_mean(self.original_lt, {'channel'}) - self.assertIn('lt_reduce_mean', actual_lt.name) - - def test(self): - actual_lt = ops.reduce_mean(self.original_lt, {'channel'}) - golden_lt = core.LabeledTensor( - math_ops.reduce_mean(self.original_lt.tensor, 1), - [self.a0, self.a2, self.a3]) - self.assertLabeledTensorsEqual(actual_lt, golden_lt) - - -class ReduceProdTest(Base): - - def test_name(self): - result_lt = ops.reduce_prod(self.original_lt, {'channel'}) - self.assertIn('lt_reduce_prod', result_lt.name) - - def test(self): - result_lt = ops.reduce_prod(self.original_lt, {'channel'}) - golden_lt = core.LabeledTensor( - math_ops.reduce_prod(self.original_lt.tensor, 1), - [self.a0, self.a2, self.a3]) - self.assertLabeledTensorsEqual(result_lt, golden_lt) - - -class ReduceMinTest(Base): - - def test_name(self): - result_lt = ops.reduce_min(self.original_lt, {'channel'}) - self.assertIn('lt_reduce_min', result_lt.name) - - def test(self): - result_lt = ops.reduce_min(self.original_lt, {'channel'}) - golden_lt = core.LabeledTensor( - math_ops.reduce_min(self.original_lt.tensor, 1), - [self.a0, self.a2, self.a3]) - self.assertLabeledTensorsEqual(result_lt, golden_lt) - - -class ReduceMaxTest(Base): - - def test_name(self): - result_lt = ops.reduce_max(self.original_lt, {'channel'}) - self.assertIn('lt_reduce_max', result_lt.name) - - def test(self): - result_lt = ops.reduce_max(self.original_lt, {'channel'}) - golden_lt = core.LabeledTensor( - math_ops.reduce_max(self.original_lt.tensor, 1), - [self.a0, self.a2, self.a3]) - self.assertLabeledTensorsEqual(result_lt, golden_lt) - - -class BaseReduceBoolean(Base): - - def setUp(self): - super(BaseReduceBoolean, self).setUp() - self.bool_tensor = math_ops.cast(self.original_lt.tensor > 5, dtypes.bool) - self.bool_lt = core.LabeledTensor(self.bool_tensor, self.original_lt.axes) - - -class ReduceAllTest(BaseReduceBoolean): - - def test_name(self): - result_lt = ops.reduce_all(self.bool_lt, {'channel'}) - self.assertIn('lt_reduce_all', result_lt.name) - - def test(self): - result_lt = ops.reduce_all(self.bool_lt, {'channel'}) - golden_lt = core.LabeledTensor( - math_ops.reduce_all(self.bool_tensor, 1), [self.a0, self.a2, self.a3]) - self.assertLabeledTensorsEqual(result_lt, golden_lt) - - -class ReduceAnyTest(BaseReduceBoolean): - - def test_name(self): - result_lt = ops.reduce_any(self.bool_lt, {'channel'}) - self.assertIn('lt_reduce_any', result_lt.name) - - def test(self): - result_lt = ops.reduce_any(self.bool_lt, {'channel'}) - golden_lt = core.LabeledTensor( - math_ops.reduce_any(self.bool_tensor, 1), [self.a0, self.a2, self.a3]) - self.assertLabeledTensorsEqual(result_lt, golden_lt) - - -class TileTest(Base): - - def test_name(self): - tile_lt = ops.tile(self.original_lt, {'z': 2}) - self.assertIn('lt_tile', tile_lt.name) - - def test(self): - for multiple in [2, constant_op.constant(2)]: - tile_lt = ops.tile(self.original_lt, {'z': multiple}) - golden_op = array_ops.tile(self.original_lt.tensor, [1, 1, multiple, 1]) - golden_axes = [ - 'z' if axis.name == 'z' else axis - for axis in self.original_lt.axes.values() - ] - golden_lt = core.LabeledTensor(golden_op, golden_axes) - self.assertLabeledTensorsEqual(tile_lt, golden_lt) - - def test_invalid_input(self): - with self.assertRaisesRegexp(ValueError, 'are not contained in the set'): - ops.tile(self.original_lt, {'foo': 5}) - with self.assertRaisesRegexp(ValueError, 'axes with tick labels'): - ops.tile(self.original_lt, {'x': 5}) - - -class PadTest(Base): - - def test_name(self): - pad_lt = ops.pad(self.original_lt, - {'x': (1, 1), - 'channel': ([], ['alpha'])}) - self.assertIn('lt_pad', pad_lt.name) - - def test(self): - pad_lt = ops.pad(self.original_lt, - {'x': (1, 1), - 'channel': ([], ['alpha'])}) - - golden_op = array_ops.pad(self.original_lt.tensor, [[1, 1], [0, 1], [0, 0], - [0, 0]]) - golden_axes = [('x', self.x_size + 2), - ('channel', ['red', 'green', 'blue', 'alpha']), self.a2, - self.a3] - golden_lt = core.LabeledTensor(golden_op, golden_axes) - self.assertLabeledTensorsEqual(pad_lt, golden_lt) - - def test_invalid_input(self): - with self.assertRaisesRegexp(ValueError, 'are not contained in the set'): - ops.pad(self.original_lt, {'foo': (1, 1), 'channel': ([], ['alpha'])}) - - -class ConstantTest(Base): - - def test_name(self): - constant_lt = ops.constant(1) - self.assertIn('lt_constant', constant_lt.name) - - def test_scalar(self): - constant_lt = ops.constant(1) - golden_lt = core.LabeledTensor(constant_op.constant(1), []) - self.assertLabeledTensorsEqual(constant_lt, golden_lt) - - def test_infer_shape(self): - constant_lt = ops.constant([1, 2], axes=['x']) - golden_lt = core.LabeledTensor(constant_op.constant([1, 2]), ['x']) - self.assertLabeledTensorsEqual(constant_lt, golden_lt) - - def test_specify_shape(self): - constant_lt = ops.constant(1, axes=[('x', 3)]) - golden_lt = core.LabeledTensor(constant_op.constant(1, shape=(3,)), ['x']) - self.assertLabeledTensorsEqual(constant_lt, golden_lt) - - def test_existing_axes(self): - golden_lt = core.LabeledTensor(constant_op.constant([1, 2]), ['x']) - constant_lt = ops.constant([1, 2], axes=golden_lt.axes) - self.assertLabeledTensorsEqual(constant_lt, golden_lt) - - -class ZerosLikeTest(Base): - - def test_name(self): - like_lt = ops.zeros_like(self.original_lt) - self.assertIn('lt_zeros_like', like_lt.name) - - def test(self): - like_lt = ops.zeros_like(self.original_lt) - golden_lt = core.LabeledTensor( - array_ops.zeros_like(self.original_lt.tensor), self.original_lt.axes) - self.assertLabeledTensorsEqual(like_lt, golden_lt) - - -class OnesLikeTest(Base): - - def test_name(self): - like_lt = ops.ones_like(self.original_lt) - self.assertIn('lt_ones_like', like_lt.name) - - def test(self): - like_lt = ops.ones_like(self.original_lt) - golden_lt = core.LabeledTensor( - array_ops.ones_like(self.original_lt.tensor), self.original_lt.axes) - self.assertLabeledTensorsEqual(like_lt, golden_lt) - - -class CastTest(Base): - - def test_name(self): - cast_lt = ops.cast(self.original_lt, dtypes.float16) - self.assertIn('lt_cast', cast_lt.name) - - def test(self): - cast_lt = ops.cast(self.original_lt, dtypes.float16) - golden_lt = core.LabeledTensor( - math_ops.cast(self.original_lt.tensor, dtypes.float16), - self.original_lt.axes) - self.assertLabeledTensorsEqual(cast_lt, golden_lt) - - -class VerifyTensorAllFiniteTest(Base): - - def setUp(self): - super(VerifyTensorAllFiniteTest, self).setUp() - - self.finite_lt = core.LabeledTensor(constant_op.constant(42.0), []) - self.nan_lt = core.LabeledTensor(constant_op.constant(np.nan), []) - - self.checked_finite_lt = ops.verify_tensor_all_finite(self.finite_lt, '') - self.checked_nan_lt = ops.verify_tensor_all_finite(self.nan_lt, '') - - def test_name(self): - self.assertIn('lt_verify_tensor_all_finite', self.checked_finite_lt.name) - self.assertIn('lt_verify_tensor_all_finite', self.checked_nan_lt.name) - - def test_finite(self): - self.assertLabeledTensorsEqual(self.finite_lt, self.checked_finite_lt) - - def test_nan(self): - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - 'Tensor had NaN values'): - self.eval([self.checked_nan_lt]) - - -class BooleanMaskTest(Base): - - def test_name(self): - mask = core.LabeledTensor(math_ops.range(7) > 3, [self.a0]) - masked_lt = ops.boolean_mask(self.original_lt, mask) - self.assertIn('lt_boolean_mask', masked_lt.name) - - def test(self): - mask = core.LabeledTensor(math_ops.range(7) > 3, [self.a0]) - masked_lt = ops.boolean_mask(self.original_lt, mask) - golden_lt = core.LabeledTensor( - array_ops.boolean_mask(self.original_lt.tensor, mask.tensor), - ['x', self.a1, self.a2, self.a3]) - self.assertLabeledTensorsEqual(masked_lt, golden_lt) - - def test_invalid_rank(self): - mask = core.LabeledTensor(array_ops.ones((7, 3)) > 3, [self.a0, self.a1]) - with self.assertRaises(NotImplementedError): - ops.boolean_mask(self.original_lt, mask) - - def test_mismatched_axis(self): - mask = core.LabeledTensor(math_ops.range(7) > 3, ['foo']) - with self.assertRaisesRegexp(ValueError, 'not equal'): - ops.boolean_mask(self.original_lt, mask) - - -class WhereTest(Base): - - def test_name(self): - condition = core.LabeledTensor(math_ops.range(5) < 3, ['x']) - where_lt = ops.where(condition, condition, condition) - self.assertIn('lt_where', where_lt.name) - - def test(self): - condition = core.LabeledTensor(math_ops.range(5) < 3, ['x']) - x = core.LabeledTensor(array_ops.ones(5), ['x']) - y = core.LabeledTensor(array_ops.zeros(5), ['x']) - where_lt = ops.where(condition, x, y) - - golden_lt = core.LabeledTensor( - array_ops.concat([array_ops.ones(3), array_ops.zeros(2)], 0), ['x']) - self.assertLabeledTensorsEqual(where_lt, golden_lt) - - def test_mismatched_axes(self): - condition = core.LabeledTensor(math_ops.range(5) < 3, ['x']) - with self.assertRaisesRegexp(ValueError, 'equal axes'): - ops.where(condition, condition[:3], condition) - with self.assertRaisesRegexp(ValueError, 'equal axes'): - ops.where(condition, condition, condition[:3]) - - -if __name__ == '__main__': - test_lib.main() diff --git a/tensorflow/contrib/labeled_tensor/python/ops/sugar.py b/tensorflow/contrib/labeled_tensor/python/ops/sugar.py deleted file mode 100644 index 914493f4736..00000000000 --- a/tensorflow/contrib/labeled_tensor/python/ops/sugar.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tools to make it a bit easier to use LabeledTensor.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from six import string_types - -from tensorflow.contrib.labeled_tensor.python.ops import _typecheck as tc -from tensorflow.contrib.labeled_tensor.python.ops import core -from tensorflow.contrib.labeled_tensor.python.ops import ops -from tensorflow.python.framework import ops as tf_ops - - -class ReshapeCoder(object): - """Utility class for mapping to and from another shape. - - For example, say you have a function `crop_center` which expects a - LabeledTensor with axes named ['batch', 'row', 'column', 'depth'], and - you have a LabeledTensor `masked_image_lt` with axes ['batch', 'row', - 'column', 'channel', 'mask']. - - To call `crop_center` with `masked_image_lt` you'd normally have to write: - - >>> reshape_lt = lt.reshape(masked_image_lt, ['channel', 'mask'], ['depth']) - >>> crop_lt = crop_center(reshape_lt) - >>> result_lt = lt.reshape(crop_lt, ['depth'], - ... [masked_image_lt.axes['channel'], masked_image_lt.axes['mask']]) - - ReshapeCoder takes care of this renaming logic for you, allowing you to - instead write: - - >>> rc = ReshapeCoder(['channel', 'mask'], ['depth']) - >>> result_lt = rc.decode(crop_center(rc.encode(masked_image_lt))) - - Here, `decode` restores the original axes 'channel' and 'mask', so - `crop_center` must not have modified the size of the 'depth' axis. - """ - - @tc.accepts(object, tc.Collection(str), - tc.Collection(tc.Union(str, core.AxisLike)), tc.Optional(str)) - def __init__(self, existing_axis_names, new_axes, name=None): - self._name = name - self._existing_axis_names = existing_axis_names - self._new_axes = new_axes - - self._existing_axes = None - - @tc.returns(core.LabeledTensor) - @tc.accepts(object, core.LabeledTensorLike) - def encode(self, labeled_tensor): - """Reshape the input to the target shape. - - If called several times, the axes named in existing_axis_names must be - identical. - - Args: - labeled_tensor: The input tensor. - - Returns: - The input reshaped to the target shape. - - Raises: - ValueError: If the axes in existing_axis_names don't match the axes of - a tensor in a previous invocation of this method. - """ - with tf_ops.name_scope(self._name, 'lt_reshape_encode', - [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - - reshape_lt = ops.reshape(labeled_tensor, - self._existing_axis_names, - self._new_axes, - name=scope) - - axes = [labeled_tensor.axes[n] for n in self._existing_axis_names] - if self._existing_axes is not None and self._existing_axes != axes: - raise ValueError( - 'input axes %r do not match axes from previous method call %r' % - (axes, self._existing_axes)) - else: - self._existing_axes = axes - - return reshape_lt - - @tc.returns(core.LabeledTensor) - @tc.accepts(object, core.LabeledTensorLike) - def decode(self, labeled_tensor): - """Reshape the input to the original shape. - - This is the inverse of encode. - Encode must have been called at least once prior to this method being - called. - - Args: - labeled_tensor: The input tensor. - - Returns: - The input reshaped to the original shape. - - Raises: - ValueError: If this method was called before encode was called. - """ - if self._existing_axes is None: - raise ValueError('decode called before encode') - - with tf_ops.name_scope(self._name, 'lt_reshape_decode', - [labeled_tensor]) as scope: - labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) - - new_axis_names = [axis if isinstance(axis, string_types) else - core.as_axis(axis).name for axis in self._new_axes] - - return ops.reshape(labeled_tensor, - new_axis_names, - self._existing_axes, - name=scope) diff --git a/tensorflow/contrib/labeled_tensor/python/ops/sugar_test.py b/tensorflow/contrib/labeled_tensor/python/ops/sugar_test.py deleted file mode 100644 index 2797e7d5251..00000000000 --- a/tensorflow/contrib/labeled_tensor/python/ops/sugar_test.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from six.moves import range # pylint: disable=redefined-builtin - -from tensorflow.contrib.labeled_tensor.python.ops import core -from tensorflow.contrib.labeled_tensor.python.ops import ops -from tensorflow.contrib.labeled_tensor.python.ops import sugar -from tensorflow.contrib.labeled_tensor.python.ops import test_util -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class Base(test_util.Base): - - def setUp(self): - super(Base, self).setUp() - - self.small_lt = core.LabeledTensor(constant_op.constant([1]), [('x', 1)]) - - -class ReshapeCoderTest(Base): - - def setUp(self): - super(ReshapeCoderTest, self).setUp() - - self.batch_size = 8 - self.num_rows = 50 - self.num_columns = 100 - self.channels = ['red', 'green', 'blue'] - self.masks = [False, True] - - tensor = math_ops.range(0, - self.batch_size * self.num_rows * self.num_columns * - len(self.channels) * len(self.masks)) - tensor = array_ops.reshape(tensor, [ - self.batch_size, self.num_rows, self.num_columns, len(self.channels), - len(self.masks) - ]) - - self.batch_axis = ('batch', range(self.batch_size)) - self.row_axis = ('row', range(self.num_rows)) - self.column_axis = ('column', range(self.num_columns)) - self.channel_axis = ('channel', self.channels) - self.mask_axis = ('mask', self.masks) - - axes = [ - self.batch_axis, self.row_axis, self.column_axis, self.channel_axis, - self.mask_axis - ] - self.masked_image_lt = core.LabeledTensor(tensor, axes) - - def test_name(self): - rc = sugar.ReshapeCoder(['channel', 'mask'], ['depth']) - encode_lt = rc.encode(self.masked_image_lt) - decode_lt = rc.decode(encode_lt) - self.assertIn('lt_reshape_encode', encode_lt.name) - self.assertIn('lt_reshape_decode', decode_lt.name) - - def test_bijection_flat(self): - rc = sugar.ReshapeCoder(['channel', 'mask'], ['depth']) - - encode_lt = rc.encode(self.masked_image_lt) - golden_axes = core.Axes([ - self.batch_axis, self.row_axis, self.column_axis, - ('depth', len(self.channels) * len(self.masks)) - ]) - self.assertEqual(encode_lt.axes, golden_axes) - - decode_lt = rc.decode(encode_lt) - self.assertLabeledTensorsEqual(decode_lt, self.masked_image_lt) - - def test_bijection_with_labels(self): - depth_axis = core.Axis('depth', range(len(self.channels) * len(self.masks))) - rc = sugar.ReshapeCoder(['channel', 'mask'], - [depth_axis, ('other', ['label'])]) - - encode_lt = rc.encode(self.masked_image_lt) - golden_axes = core.Axes([ - self.batch_axis, self.row_axis, self.column_axis, depth_axis, - ('other', ['label']) - ]) - self.assertEqual(encode_lt.axes, golden_axes) - - decode_lt = rc.decode(encode_lt) - self.assertLabeledTensorsEqual(decode_lt, self.masked_image_lt) - - def test_invalid_input(self): - with self.assertRaises(ValueError): - rc = sugar.ReshapeCoder(['channel', 'mask'], ['depth']) - rc.decode(self.masked_image_lt) - with self.assertRaises(ValueError): - rc = sugar.ReshapeCoder(['channel', 'mask'], ['depth']) - rc.encode(self.masked_image_lt) - rc.encode(ops.select(self.masked_image_lt, {'channel': 'red'})) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/labeled_tensor/python/ops/test_util.py b/tensorflow/contrib/labeled_tensor/python/ops/test_util.py deleted file mode 100644 index 900c9217c39..00000000000 --- a/tensorflow/contrib/labeled_tensor/python/ops/test_util.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utils for writing tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.platform import test -from tensorflow.python.training import coordinator -from tensorflow.python.training import queue_runner_impl - - -class Base(test.TestCase): - """A class with some useful methods for testing.""" - - def eval(self, tensors): - with self.cached_session() as sess: - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(sess=sess, coord=coord) - - try: - results = sess.run(tensors) - finally: - coord.request_stop() - coord.join(threads) - - return results - - def assertTensorsEqual(self, tensor_0, tensor_1): - [tensor_0_eval, tensor_1_eval] = self.eval([tensor_0, tensor_1]) - self.assertAllEqual(tensor_0_eval, tensor_1_eval) - - def assertLabeledTensorsEqual(self, tensor_0, tensor_1): - self.assertEqual(tensor_0.axes, tensor_1.axes) - self.assertTensorsEqual(tensor_0.tensor, tensor_1.tensor) diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD deleted file mode 100644 index 6010b072418..00000000000 --- a/tensorflow/contrib/layers/BUILD +++ /dev/null @@ -1,416 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") -load("//tensorflow:tensorflow.bzl", "py_test", "tf_custom_op_library", "tf_gen_op_libs", "tf_gen_op_wrapper_py", "tf_kernel_library") - -package( - default_visibility = [ - "//learning/brain:__subpackages__", - "//tensorflow:__subpackages__", - "//tensorflow_model_optimization:__subpackages__", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_custom_op_library( - # TODO(sibyl-Mooth6ku,ptucker): Understand why 'python/ops/_' is needed and fix it. - name = "python/ops/_sparse_feature_cross_op.so", - srcs = [ - "ops/sparse_feature_cross_op.cc", - ], - deps = [ - "//tensorflow/contrib/layers/kernels:sparse_feature_cross_kernel", - ], -) - -tf_gen_op_libs( - op_lib_names = ["sparse_feature_cross_op"], -) - -tf_gen_op_wrapper_py( - name = "sparse_feature_cross_op", - deps = [":sparse_feature_cross_op_op_lib"], -) - -tf_kernel_library( - name = "sparse_feature_cross_op_kernel", - deps = [ - "//tensorflow/contrib/layers/kernels:sparse_feature_cross_kernel", - "//tensorflow/core:framework", - ], - alwayslink = 1, -) - -tf_custom_op_py_library( - name = "layers_py", - srcs = [ - "__init__.py", - "python/layers/__init__.py", - "python/layers/embedding_ops.py", - "python/layers/encoders.py", - "python/layers/feature_column.py", - "python/layers/feature_column_ops.py", - "python/layers/initializers.py", - "python/layers/layers.py", - "python/layers/normalization.py", - "python/layers/optimizers.py", - "python/layers/regularizers.py", - "python/layers/rev_block_lib.py", - "python/layers/summaries.py", - "python/layers/target_column.py", - "python/layers/utils.py", - "python/ops/bucketization_op.py", - "python/ops/sparse_feature_cross_op.py", - "python/ops/sparse_ops.py", - ], - dso = [ - ":python/ops/_sparse_feature_cross_op.so", - ], - kernels = [ - ":sparse_feature_cross_op_kernel", - ":sparse_feature_cross_op_op_lib", - ], - srcs_version = "PY2AND3", - visibility = [ - "//learning/brain:__subpackages__", - "//learning/lib/ami/simple_ml/link_other_ml_tools/tensorflow:__subpackages__", - "//storage/d/analysis/prefetch:__pkg__", - "//tensorflow:__subpackages__", - "//tensorflow_model_optimization:__subpackages__", - "//third_party/py/tf_slim:__subpackages__", - "//video/youtube/personalization:__subpackages__", - ], - deps = [ - ":sparse_feature_cross_op", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/lookup:lookup_py", - "//tensorflow/contrib/losses:losses_py", - "//tensorflow/contrib/metrics:metrics_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:clip_ops", - "//tensorflow/python:common_shapes", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:layers", - "//tensorflow/python:layers_base", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:nn_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:standard_ops", - "//tensorflow/python:string_ops", - "//tensorflow/python:summary", - "//tensorflow/python:tensor_util", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/eager:context", - "//tensorflow/python/feature_column", - "@six_archive//:six", - ], -) - -cuda_py_test( - name = "layers_test", - size = "medium", - srcs = ["python/layers/layers_test.py"], - additional_deps = [ - ":layers_py", - "//third_party/py/numpy", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:partitioned_variables", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:template", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/ops/losses:losses", - ], - xla_enable_strict_auto_jit = False, -) - -py_test( - name = "regularizers_test", - size = "small", - srcs = ["python/layers/regularizers_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:session", - "//third_party/py/numpy", - ], -) - -py_test( - name = "initializers_test", - size = "small", - srcs = ["python/layers/initializers_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":layers_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:session", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "normalization_test", - size = "medium", - srcs = ["python/layers/normalization_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_windows"], # TODO: needs investigation on Windows - deps = [ - ":layers_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:random_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "optimizers_test", - srcs = ["python/layers/optimizers_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "summaries_test", - size = "small", - srcs = ["python/layers/summaries_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:nn_ops", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "feature_column_test", - size = "small", - srcs = ["python/layers/feature_column_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":layers_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:lookup_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/feature_column:feature_column_py", - "//third_party/py/numpy", - ], -) - -py_test( - name = "feature_column_ops_test", - size = "medium", - srcs = ["python/layers/feature_column_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":layers_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:gradients", - "//tensorflow/python:init_ops", - "//tensorflow/python:lookup_ops", - "//tensorflow/python:partitioned_variables", - "//tensorflow/python:random_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/feature_column:feature_column_py", - "//third_party/py/numpy", - ], -) - -py_test( - name = "target_column_test", - size = "small", - srcs = ["python/layers/target_column_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":layers_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:session", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "sparse_feature_cross_op_test", - size = "medium", - srcs = ["python/kernel_tests/sparse_feature_cross_op_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":layers_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:session", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:sparse_tensor", - "//third_party/py/numpy", - ], -) - -py_test( - name = "embedding_ops_test", - size = "small", - timeout = "moderate", - srcs = ["python/layers/embedding_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:gradient_checker", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:partitioned_variables", - "//tensorflow/python:random_seed", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:util", - "//third_party/py/numpy", - ], -) - -py_test( - name = "utils_test", - size = "small", - srcs = ["python/layers/utils_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "sparse_ops_test", - size = "small", - srcs = ["python/ops/sparse_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_windows"], # TODO: needs investigation on Windows - deps = [ - ":layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:sparse_tensor", - "//third_party/py/numpy", - ], -) - -py_test( - name = "encoders_test", - size = "small", - srcs = ["python/layers/encoders_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":layers_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:init_ops", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "rev_block_lib_test", - size = "medium", - srcs = ["python/layers/rev_block_lib_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":layers_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:init_ops", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) diff --git a/tensorflow/contrib/layers/README.md b/tensorflow/contrib/layers/README.md deleted file mode 100644 index 9310b194dff..00000000000 --- a/tensorflow/contrib/layers/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# TensorFlow contrib layers. - -## initializers.py - -Functions that produce variable initializer functions with signature: - -`foo(shape, dtype) : Tensor` - -These are typically consumed by functions in [layers.py](#layers.py). - -## layers.py {#.py} - -Functions that produce layer operations and associated weight & bias variables. Signatures will vary for different functions, but they will often take many of -these arguments. - -`foo(x, - num_outputs, - …, - weight_init=, - bias_init=, - weight_regularizer=None, - bias_regularizer=None, - name=None) : Tensor` - -`x` is the input tensor. - -Weights and biases are added to `tf.GraphKeys.GLOBAL_VARIABLES` and -`tf.GraphKeys.TRAINABLE_VARIABLES`. - -## optimizers.py - -Functions that add optimization ops given `loss` and `global_step` tensors. - -## regularizers.py - -Functions that produce weight regularization functions with signature - -`foo(weight_vars, name=None) : Operation` - -These are typically consumed by functions in [layers.py](#layers.py). - -## summaries.py - -Functions that add summary ops to the standard `tf.GraphKeys.SUMMARIES` -collection. They also avoid name conflicts in the summary key. diff --git a/tensorflow/contrib/layers/__init__.py b/tensorflow/contrib/layers/__init__.py deleted file mode 100644 index 32f3006b749..00000000000 --- a/tensorflow/contrib/layers/__init__.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for building neural network layers, regularizers, summaries, etc. - -@@avg_pool2d -@@avg_pool3d -@@batch_norm -@@convolution -@@convolution1d -@@convolution2d -@@convolution3d -@@conv2d_in_plane -@@convolution2d_in_plane -@@conv2d_transpose -@@convolution2d_transpose -@@conv3d_transpose -@@convolution3d_transpose -@@dense_to_sparse -@@dropout -@@elu -@@embedding_lookup_unique -@@flatten -@@fully_connected -@@GDN -@@gdn -@@images_to_sequence -@@layer_norm -@@linear -@@max_pool2d -@@max_pool3d -@@one_hot_encoding -@@relu -@@relu6 -@@repeat -@@recompute_grad -@@RevBlock -@@rev_block -@@safe_embedding_lookup_sparse -@@scale_gradient -@@separable_conv2d -@@separable_convolution2d -@@sequence_to_images -@@softmax -@@spatial_softmax -@@stack -@@unit_norm -@@bow_encoder -@@embed_sequence -@@maxout - -@@apply_regularization -@@l1_l2_regularizer -@@l1_regularizer -@@l2_regularizer -@@sum_regularizer - -@@xavier_initializer -@@xavier_initializer_conv2d -@@variance_scaling_initializer - -@@optimize_loss - -@@summarize_activation -@@summarize_tensor -@@summarize_tensors -@@summarize_collection - -@@summarize_activations - -@@bucketized_column -@@check_feature_columns -@@create_feature_spec_for_parsing -@@crossed_column -@@embedding_column -@@scattered_embedding_column -@@input_from_feature_columns -@@transform_features -@@joint_weighted_sum_from_feature_columns -@@make_place_holder_tensors_for_base_features -@@multi_class_target -@@one_hot_column -@@parse_feature_columns_from_examples -@@parse_feature_columns_from_sequence_examples -@@real_valued_column -@@shared_embedding_columns -@@sparse_column_with_hash_bucket -@@sparse_column_with_integerized_feature -@@sparse_column_with_keys -@@sparse_column_with_vocabulary_file -@@weighted_sparse_column -@@weighted_sum_from_feature_columns -@@infer_real_valued_columns -@@sequence_input_from_feature_columns - -@@group_norm -@@instance_norm -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.layers.python.layers import * -# pylint: enable=unused-import,wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = ['bias_add', - 'conv1d', - 'conv2d', - 'conv3d', - 'elu', - 'feature_column', - 'group_norm', - 'instance_norm', - 'legacy_fully_connected', - 'legacy_linear', - 'legacy_relu', - 'OPTIMIZER_CLS_NAMES', - 'OPTIMIZER_SUMMARIES', - 'regression_target', - 'SPARSE_FEATURE_CROSS_DEFAULT_HASH_KEY', - 'summaries'] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/layers/kernels/BUILD b/tensorflow/contrib/layers/kernels/BUILD deleted file mode 100644 index 464550d7413..00000000000 --- a/tensorflow/contrib/layers/kernels/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -# Description: -# Contains kernels for layers. - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -cc_library( - name = "sparse_feature_cross_kernel", - srcs = ["sparse_feature_cross_kernel.cc"], - deps = [ - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - "@com_google_protobuf//:protobuf_headers", - "@farmhash_archive//:farmhash", - ], - alwayslink = 1, -) diff --git a/tensorflow/contrib/layers/kernels/sparse_feature_cross_kernel.cc b/tensorflow/contrib/layers/kernels/sparse_feature_cross_kernel.cc deleted file mode 100644 index 3fe4bd48748..00000000000 --- a/tensorflow/contrib/layers/kernels/sparse_feature_cross_kernel.cc +++ /dev/null @@ -1,649 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Contains OP to generate sparse crosses. -#include -#include -#include -#include - -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/framework/kernel_def_builder.h" -#include "tensorflow/core/framework/op_def_builder.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/platform/fingerprint.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -namespace { -// An interface that represents a column with batches. -template -class ColumnInterface { - public: - // Returns the number of features in the specified batch. - virtual int64 FeatureCount(int64 batch) const = 0; - - // Returns the fingerprint of nth feature from the specified batch. - virtual InternalType Feature(int64 batch, int64 n) const = 0; - - virtual ~ColumnInterface() {} -}; - -// A column that is backed by a sparse tensor. -template -class SparseTensorColumn : public ColumnInterface { - public: - SparseTensorColumn(const Tensor& values, std::vector feature_counts, - std::vector feature_start_indices) - : values_(values), - feature_counts_(std::move(feature_counts)), - feature_start_indices_(std::move(feature_start_indices)) { - CHECK_EQ(feature_counts_.size(), feature_start_indices_.size()); - } - - int64 FeatureCount(int64 batch) const override { - return feature_counts_[batch]; - } - - InternalType Feature(int64 batch, int64 n) const override; - - ~SparseTensorColumn() override {} - - private: - const Tensor& values_; - std::vector feature_counts_; - std::vector feature_start_indices_; -}; - -// InternalType is int64 only when using HashCrosser. -template <> -int64 SparseTensorColumn::Feature(int64 batch, int64 n) const { - const int64 start = feature_start_indices_[batch]; - if (DT_STRING == values_.dtype()) - return Fingerprint64(values_.vec().data()[start + n]); - return values_.vec().data()[start + n]; -} - -// InternalType is string or StringPiece when using StringCrosser. -template <> -tstring SparseTensorColumn::Feature(int64 batch, int64 n) const { - const int64 start = feature_start_indices_[batch]; - if (DT_STRING == values_.dtype()) - return values_.vec().data()[start + n]; - return std::to_string(values_.vec().data()[start + n]); -} - -template <> -StringPiece SparseTensorColumn::Feature(int64 batch, - int64 n) const { - const int64 start = feature_start_indices_[batch]; - return values_.vec().data()[start + n]; -} - -// A column that is backed by a dense tensor. -template -class DenseTensorColumn : public ColumnInterface { - public: - explicit DenseTensorColumn(const Tensor& tensor) : tensor_(tensor) {} - - int64 FeatureCount(int64 batch) const override { return tensor_.dim_size(1); } - - InternalType Feature(int64 batch, int64 n) const override; - - ~DenseTensorColumn() override {} - - private: - const Tensor& tensor_; -}; - -// InternalType is int64 only when using HashCrosser. -template <> -int64 DenseTensorColumn::Feature(int64 batch, int64 n) const { - if (DT_STRING == tensor_.dtype()) - return Fingerprint64(tensor_.matrix()(batch, n)); - return tensor_.matrix()(batch, n); -} - -// Internal type is string or StringPiece when using StringCrosser. -template <> -tstring DenseTensorColumn::Feature(int64 batch, int64 n) const { - if (DT_STRING == tensor_.dtype()) return tensor_.matrix()(batch, n); - return std::to_string(tensor_.matrix()(batch, n)); -} - -template <> -StringPiece DenseTensorColumn::Feature(int64 batch, - int64 n) const { - return tensor_.matrix()(batch, n); -} - -// Updates Output tensors with sparse crosses. -template -class OutputUpdater { - public: - OutputUpdater(const std::vector& output_start_indices, - Tensor* indices_out, Tensor* values_out) - : output_start_indices_(output_start_indices), - indices_out_(indices_out), - values_out_(values_out) {} - - void Update(const int64 batch_index, const int64 cross_count, - const OutType& cross) const { - const int64 output_index = output_start_indices_[batch_index] + cross_count; - - auto indices_matrix = indices_out_->matrix(); - indices_matrix(output_index, 0) = batch_index; - indices_matrix(output_index, 1) = cross_count; - - auto value_vec = values_out_->vec(); - value_vec(output_index) = cross; - } - - private: - const std::vector& output_start_indices_; - Tensor* indices_out_; - Tensor* values_out_; -}; - -// Generates the sparse crosses as concatenation of strings. -template -class StringCrosser { - public: - StringCrosser(const std::vector< - std::unique_ptr>>& columns, - const int64 num_buckets_unused, const uint64 hash_key_unused) - : columns_(columns) {} - - string Generate(const int64 batch_index, - const std::vector& permutation) const { - static const auto k_feature_separator = "_X_"; - - gtl::InlinedVector cross_vec(columns_.size()); - for (size_t i = 0; i < permutation.size(); i++) { - cross_vec[i] = columns_[i]->Feature(batch_index, permutation[i]); - } - // TODO(zakaria): this will copy the string twice, might effect - // performance. - return absl::StrJoin(cross_vec, k_feature_separator); - } - - private: - const std::vector>>& columns_; -}; - -// Generates the sparse crosses as nested hash to avoid string manipulations. -class HashCrosser { - public: - HashCrosser( - const std::vector>>& columns, - const int64 num_buckets, const uint64 hash_key_unused) - : columns_(columns), num_buckets_(num_buckets) {} - - int64 Generate(const int64 batch_index, - const std::vector& permutation) const { - // Seed is chosen based on third_party/tensorflow/core/lib/hash/hash.h - static const int64 kInitialHashSeed = 0xDECAFCAFFE; - - uint64 hashed_output = kInitialHashSeed; - for (size_t i = 0; i < permutation.size(); ++i) { - int64 hash_i = columns_[i]->Feature(batch_index, permutation[i]); - hashed_output = HashCombine(hashed_output, hash_i); - } - if (num_buckets_ > 0) { - return hashed_output % num_buckets_; - } else { - // To prevent negative output we take modulo to max int64. - return hashed_output % std::numeric_limits::max(); - } - } - - private: - static int64 HashCombine(int64 a, int64 b) { - return a ^ (b + 0x9e3779b97f4a7800 + (a << 10) + (a >> 4)); - } - - const std::vector>>& columns_; - const int64 num_buckets_; -}; - -// Generates the sparse crosses as nested hash to avoid string manipulations. -class HashCrosserV2 { - public: - HashCrosserV2( - const std::vector>>& columns, - const int64 num_buckets, const uint64 hash_key) - : columns_(columns), num_buckets_(num_buckets), hash_key_(hash_key) {} - - int64 Generate(const int64 batch_index, - const std::vector& permutation) const { - // Do the fingerprint concatenation on uint64. - uint64 hashed_output = hash_key_; - for (size_t i = 0; i < permutation.size(); ++i) { - uint64 hash_i = columns_[i]->Feature(batch_index, permutation[i]); - hashed_output = FingerprintCat64(hashed_output, hash_i); - } - // The return value is int64 based on the number of buckets. - if (num_buckets_ > 0) { - return hashed_output % num_buckets_; - } else { - // To prevent negative output we take modulo to max int64. - return hashed_output % std::numeric_limits::max(); - } - } - - private: - const std::vector>>& columns_; - const int64 num_buckets_; - const uint64 hash_key_; -}; - -// ProductIterator generates cartesian products based on indices. -template -class ProductIterator { - public: - explicit ProductIterator( - const std::vector>>& - columns, - int64 batch_index) - : columns_(columns), batch_index_(batch_index) { - next_permutation_.resize(columns_.size(), 0); - // Sets has_next_ to false if any feature column has 0 features. - has_next_ = true; - for (size_t i = 0; i < columns_.size(); i++) { - if (columns_[i]->FeatureCount(batch_index_) == 0) { - has_next_ = false; - break; - } - } - } - - std::vector Next() { - std::vector permutation(next_permutation_); - - // Generates next permutation, if available. - bool carry = true; - for (int i = next_permutation_.size() - 1; i >= 0; i--) { - if (carry) { - next_permutation_[i] = next_permutation_[i] + 1; - } - if (next_permutation_[i] == columns_[i]->FeatureCount(batch_index_)) { - next_permutation_[i] = 0; - } else { - carry = false; - break; - } - } - has_next_ = !carry; - return permutation; - } - - bool HasNext() { return has_next_; } - - private: - bool has_next_; - const std::vector>>& columns_; - const int64 batch_index_; - std::vector next_permutation_; -}; - -template -struct CrossTraits; - -template -struct CrossTraits { - typedef StringCrosser Crosser; - typedef OutputUpdater Updater; -}; - -template <> -struct CrossTraits { - typedef HashCrosser Crosser; - typedef OutputUpdater Updater; -}; - -template <> -struct CrossTraits { - typedef HashCrosserV2 Crosser; - typedef OutputUpdater Updater; -}; -} // namespace - -template -class SparseFeatureCrossOp : public OpKernel { - public: - explicit SparseFeatureCrossOp(OpKernelConstruction* context) - : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr("num_buckets", &num_buckets_)); - if (VERSION_2) { - // Read signed_hash_key_ as int64 since uint64 attributes are not - // supported by REGISTER_OP. - int64 signed_hash_key_; - OP_REQUIRES_OK(context, context->GetAttr("hash_key", &signed_hash_key_)); - hash_key_ = static_cast(signed_hash_key_); - } - } - - void Compute(OpKernelContext* context) override { - OpInputList indices_list_in; - OP_REQUIRES_OK(context, context->input_list("indices", &indices_list_in)); - OpInputList values_list_in; - OP_REQUIRES_OK(context, context->input_list("values", &values_list_in)); - OpInputList shapes_list_in; - OP_REQUIRES_OK(context, context->input_list("shapes", &shapes_list_in)); - OpInputList dense_list_in; - OP_REQUIRES_OK(context, context->input_list("dense", &dense_list_in)); - - ValidateInput(context, indices_list_in, values_list_in, shapes_list_in, - dense_list_in); - - std::vector>> columns = - GenerateColumnsFromInput(indices_list_in, values_list_in, - shapes_list_in, dense_list_in); - - typename CrossTraits::Crosser - crosser(columns, num_buckets_, hash_key_); - Tensor* indices_out; - Tensor* values_out; - Tensor* shape_out; - const int64 batch_size = CalculateBatchSize(shapes_list_in, dense_list_in); - std::vector output_start_indices(batch_size); - CreateOutputTensors(columns, batch_size, context, &indices_out, &values_out, - &shape_out, &output_start_indices); - - typename CrossTraits::Updater - updater(output_start_indices, indices_out, values_out); - auto do_work = [this, &columns, crosser, updater](int64 begin, int64 end) { - for (int b = begin; b < end; b++) { - ProductIterator product_iterator(columns, b); - int64 cross_count = 0; - while (product_iterator.HasNext()) { - const auto permutation = product_iterator.Next(); - updater.Update(b, cross_count, crosser.Generate(b, permutation)); - cross_count++; - } - } - }; - - auto* worker_threads = context->device()->tensorflow_cpu_worker_threads(); - // TODO(zakaria): optimize kCostPerUnit - const int kCostPerUnit = 5000 * indices_list_in.size(); - Shard(worker_threads->num_threads, worker_threads->workers, batch_size, - kCostPerUnit, do_work); - } - - private: - // Validates input tensors. - void ValidateInput(OpKernelContext* context, - const OpInputList& indices_list_in, - const OpInputList& values_list_in, - const OpInputList& shapes_list_in, - const OpInputList& dense_list_in) { - const auto size = indices_list_in.size(); - // Validates indices_list_in OpInputList. - for (int i = 0; i < size; i++) { - OP_REQUIRES( - context, TensorShapeUtils::IsMatrix(indices_list_in[i].shape()), - errors::InvalidArgument( - "Input indices should be a matrix but received shape ", - indices_list_in[i].shape().DebugString(), " at position ", i)); - OP_REQUIRES( - context, indices_list_in[i].shape().dim_size(1) == 2, - errors::InvalidArgument("Expected D2 of index to be 2 got ", - indices_list_in[i].shape().dim_size(1), - " at position ", i)); - } - - // Validates values_list_in OpInputList. - OP_REQUIRES( - context, values_list_in.size() == size, - errors::InvalidArgument("Expected ", size, " input values, got ", - values_list_in.size())); - for (int i = 0; i < size; i++) { - OP_REQUIRES( - context, TensorShapeUtils::IsVector(values_list_in[i].shape()), - errors::InvalidArgument( - "Input values should be a std::vector but received shape ", - values_list_in[i].shape().DebugString(), " at position ", i)); - OP_REQUIRES( - context, - indices_list_in[i].shape().dim_size(0) == - values_list_in[i].shape().dim_size(0), - errors::InvalidArgument( - "Expected size of values to be ", - indices_list_in[i].shape().dim_size(0), " got ", - values_list_in[i].shape().dim_size(0), " at position ", i)); - } - - // Validates shapes_list_in OpInputList - OP_REQUIRES( - context, shapes_list_in.size() == size, - errors::InvalidArgument("Expected ", size, " input shapes, got ", - shapes_list_in.size())); - const auto batch_size = CalculateBatchSize(shapes_list_in, dense_list_in); - for (int i = 0; i < size; i++) { - OP_REQUIRES( - context, TensorShapeUtils::IsVector(shapes_list_in[i].shape()), - errors::InvalidArgument( - "Input shapes should be a std::vector but received shape ", - shapes_list_in[i].shape().DebugString(), " at position ", i)); - - OP_REQUIRES( - context, shapes_list_in[i].vec().size() == 2, - errors::InvalidArgument("shape should imply a 2D tensor, but got ", - shapes_list_in[i].shape().DebugString(), - " at position ", i)); - OP_REQUIRES(context, shapes_list_in[i].vec()(0) == batch_size, - errors::InvalidArgument( - "Expected batch size ", batch_size, " got ", - shapes_list_in[i].vec()(0), " at position ", i)); - } - - // Validates dense_list_in OpInputList - for (int i = 0; i < dense_list_in.size(); ++i) { - OP_REQUIRES( - context, TensorShapeUtils::IsMatrix(dense_list_in[i].shape()), - errors::InvalidArgument( - "Dense inputs should be a matrix but received shape ", - indices_list_in[i].shape().DebugString(), " at position ", i)); - OP_REQUIRES(context, dense_list_in[i].dim_size(0) == batch_size, - errors::InvalidArgument("Expected batch size ", batch_size, - " got ", dense_list_in[i].dim_size(0), - " at dense tensor ", i)); - } - } - - // Calculate the batch size from either the shapes input or the dense input. - int64 CalculateBatchSize(const OpInputList& shapes_list_in, - const OpInputList& dense_list_in) { - if (shapes_list_in.size() > 0) { - return shapes_list_in[0].vec()(0); - } - - if (dense_list_in.size() > 0) { - return dense_list_in[0].dim_size(0); - } - - return 0; - } - - // Generate the columns given the sparse and dense inputs. - std::vector>> - GenerateColumnsFromInput(const OpInputList& indices_list_in, - const OpInputList& values_list_in, - const OpInputList& shapes_list_in, - const OpInputList& dense_list_in) { - std::vector>> columns; - const int64 batch_size = CalculateBatchSize(shapes_list_in, dense_list_in); - const int64 number_of_columns = shapes_list_in.size(); - - std::vector> feature_counts(number_of_columns, - std::vector()); - std::vector> feature_start_indices(number_of_columns, - std::vector()); - - ExtractFeatureData(indices_list_in, batch_size, &feature_counts, - &feature_start_indices); - - columns.reserve(values_list_in.size()); - for (int i = 0; i < values_list_in.size(); ++i) { - columns.emplace_back(new SparseTensorColumn( - values_list_in[i], std::move(feature_counts[i]), - std::move(feature_start_indices[i]))); - } - for (int i = 0; i < dense_list_in.size(); ++i) { - columns.emplace_back( - new DenseTensorColumn(dense_list_in[i])); - } - - return columns; - } - - // Extracts data about the features and populates feature data. - void ExtractFeatureData( - const OpInputList& indices_list_in, int64 batch_size, - std::vector>* feature_counts, - std::vector>* feature_start_indices) { - gtl::InlinedVector current_row(indices_list_in.size(), 0); - for (int b = 0; b < batch_size; b++) { - for (int i = 0; i < indices_list_in.size(); i++) { - const auto indices = indices_list_in[i].matrix(); - int64 feature_count = 0; - int64 start_index = current_row[i]; - // Loops until we reach next batch index for current feature column. - while (current_row[i] < indices_list_in[i].dim_size(0) && - indices(current_row[i], 0) == b) { - feature_count++; - current_row[i]++; - } - (*feature_counts)[i].push_back(feature_count); - (*feature_start_indices)[i].push_back(start_index); - } - } - } - - // Allocates output tensors with proper size and sets the shape tensor of - // the output SparseTensor. - // It also output_start_indices which contains the start indices for each - // input in the output SparseTensor. - void CreateOutputTensors( - const std::vector>>& - columns, - int64 batch_size, OpKernelContext* context, Tensor** indices_out, - Tensor** values_out, Tensor** shape_out, - std::vector* output_start_indices) { - // Calculates dimensions for output tensors. - int64 cross_count_total = 0; - int64 max_cross_count = 0; - for (int64 b = 0; b < batch_size; b++) { - // For each input, sets starting indices in output SparseTensor - (*output_start_indices)[b] = cross_count_total; - const auto cross_count = CrossCountByBatchIndex(columns, b); - max_cross_count = std::max(max_cross_count, cross_count); - cross_count_total += cross_count; - } - - // Allocates tensors. - OP_REQUIRES_OK(context, - context->allocate_output( - 0, TensorShape({cross_count_total, 2}), indices_out)); - OP_REQUIRES_OK(context, - context->allocate_output(1, TensorShape({cross_count_total}), - values_out)); - OP_REQUIRES_OK(context, - context->allocate_output(2, TensorShape({2}), shape_out)); - - // Sets shape. - auto shape_vec = (*shape_out)->vec(); - shape_vec(0) = batch_size; - shape_vec(1) = max_cross_count; - } - - // Returns number of crosses for a given batch_index - int64 CrossCountByBatchIndex( - const std::vector>>& - columns, - int batch_index) { - int64 cross_count = 1; - for (size_t i = 0; i < columns.size(); i++) { - const auto feature_count = columns[i]->FeatureCount(batch_index); - // If one column is missing any feature, there won't be any cross. - if (feature_count == 0) { - return 0; - } - cross_count *= feature_count; - } - return cross_count; - } - int64 num_buckets_; - uint64 hash_key_; -}; - -REGISTER_KERNEL_BUILDER(Name("SparseFeatureCross") - .Device(DEVICE_CPU) - .TypeConstraint("out_type") - .TypeConstraint("internal_type"), - SparseFeatureCrossOp); - -REGISTER_KERNEL_BUILDER(Name("SparseFeatureCross") - .Device(DEVICE_CPU) - .TypeConstraint("out_type") - .TypeConstraint("internal_type"), - SparseFeatureCrossOp); - -REGISTER_KERNEL_BUILDER(Name("SparseFeatureCross") - .Device(DEVICE_CPU) - .TypeConstraint("out_type") - .TypeConstraint("internal_type"), - SparseFeatureCrossOp); - -REGISTER_KERNEL_BUILDER(Name("SparseFeatureCross") - .Device(DEVICE_CPU) - .TypeConstraint("out_type") - .TypeConstraint("internal_type"), - SparseFeatureCrossOp); - -// The following builders enable FingerprintCat64 concatenation for the -// crosses features. -REGISTER_KERNEL_BUILDER(Name("SparseFeatureCrossV2") - .Device(DEVICE_CPU) - .TypeConstraint("out_type") - .TypeConstraint("internal_type"), - SparseFeatureCrossOp); - -REGISTER_KERNEL_BUILDER(Name("SparseFeatureCrossV2") - .Device(DEVICE_CPU) - .TypeConstraint("out_type") - .TypeConstraint("internal_type"), - SparseFeatureCrossOp); - -REGISTER_KERNEL_BUILDER(Name("SparseFeatureCrossV2") - .Device(DEVICE_CPU) - .TypeConstraint("out_type") - .TypeConstraint("internal_type"), - SparseFeatureCrossOp); - -REGISTER_KERNEL_BUILDER(Name("SparseFeatureCrossV2") - .Device(DEVICE_CPU) - .TypeConstraint("out_type") - .TypeConstraint("internal_type"), - SparseFeatureCrossOp); - -} // namespace tensorflow diff --git a/tensorflow/contrib/layers/ops/sparse_feature_cross_op.cc b/tensorflow/contrib/layers/ops/sparse_feature_cross_op.cc deleted file mode 100644 index f73ea5e2c9e..00000000000 --- a/tensorflow/contrib/layers/ops/sparse_feature_cross_op.cc +++ /dev/null @@ -1,162 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { -REGISTER_OP("SparseFeatureCross") - .Input("indices: N * int64") - .Input("values: sparse_types") - .Input("shapes: N * int64") - .Input("dense: dense_types") - .Output("output_indices: int64") - .Output("output_values: out_type") - .Output("output_shape: int64") - .Attr("N: int >= 0") - .Attr("hashed_output: bool") - .Attr("num_buckets: int >= 0") - .Attr("sparse_types: list({int64, string}) >= 0") - .Attr("dense_types: list({int64, string}) >= 0") - .Attr("out_type: {int64, string}") - .Attr("internal_type: {int64, string}") - .SetShapeFn([](shape_inference::InferenceContext* c) { - c->set_output(0, c->Matrix(c->UnknownDim(), 2)); - c->set_output(1, c->Vector(c->UnknownDim())); - c->set_output(2, c->Vector(2)); - return Status::OK(); - }) - .Doc(R"doc( -Generates sparse cross form a list of sparse tensors. - -The op takes two lists, one of 2D `SparseTensor` and one of 2D `Tensor`, each -representing features of one feature column. It outputs a 2D `SparseTensor` with -the batchwise crosses of these features. - -For example, if the inputs are - - inputs[0]: SparseTensor with shape = [2, 2] - [0, 0]: "a" - [1, 0]: "b" - [1, 1]: "c" - - inputs[1]: SparseTensor with shape = [2, 1] - [0, 0]: "d" - [1, 0]: "e" - - inputs[2]: Tensor [["f"], ["g"]] - -then the output will be - - shape = [2, 2] - [0, 0]: "a_X_d_X_f" - [1, 0]: "b_X_e_X_g" - [1, 1]: "c_X_e_X_g" - -if hashed_output=true then the output will be - - shape = [2, 2] - [0, 0]: HashCombine( - Fingerprint64("f"), HashCombine( - Fingerprint64("d"), Fingerprint64("a"))) - [1, 0]: HashCombine( - Fingerprint64("g"), HashCombine( - Fingerprint64("e"), Fingerprint64("b"))) - [1, 1]: HashCombine( - Fingerprint64("g"), HashCombine( - Fingerprint64("e"), Fingerprint64("c"))) - -indices: 2-D. Indices of each input `SparseTensor`. -values: 1-D. values of each `SparseTensor`. -shapes: 1-D. Shapes of each `SparseTensor`. -dense: 2-D. Columns represented by dense `Tensor`. -output_indices: 2-D. Indices of the concatenated `SparseTensor`. -output_values: 1-D. Non-empty values of the concatenated or hashed - `SparseTensor`. -output_shape: 1-D. Shape of the concatenated `SparseTensor`. -)doc"); - -REGISTER_OP("SparseFeatureCrossV2") - .Input("indices: N * int64") - .Input("values: sparse_types") - .Input("shapes: N * int64") - .Input("dense: dense_types") - .Output("output_indices: int64") - .Output("output_values: out_type") - .Output("output_shape: int64") - .Attr("N: int >= 0") - .Attr("hashed_output: bool") - .Attr("num_buckets: int >= 0") - .Attr("hash_key: int") - .Attr("sparse_types: list({int64, string}) >= 0") - .Attr("dense_types: list({int64, string}) >= 0") - .Attr("out_type: {int64, string}") - .Attr("internal_type: {int64, string}") - .SetShapeFn([](shape_inference::InferenceContext* c) { - c->set_output(0, c->Matrix(c->UnknownDim(), 2)); - c->set_output(1, c->Vector(c->UnknownDim())); - c->set_output(2, c->Vector(2)); - return Status::OK(); - }) - .Doc(R"doc( -Generates sparse cross form a list of sparse tensors. - -The op takes two lists, one of 2D `SparseTensor` and one of 2D `Tensor`, each -representing features of one feature column. It outputs a 2D `SparseTensor` with -the batchwise crosses of these features. - -For example, if the inputs are - - inputs[0]: SparseTensor with shape = [2, 2] - [0, 0]: "a" - [1, 0]: "b" - [1, 1]: "c" - - inputs[1]: SparseTensor with shape = [2, 1] - [0, 0]: "d" - [1, 0]: "e" - - inputs[2]: Tensor [["f"], ["g"]] - -then the output will be - - shape = [2, 2] - [0, 0]: "a_X_d_X_f" - [1, 0]: "b_X_e_X_g" - [1, 1]: "c_X_e_X_g" - -if hashed_output=true then the output will be - - shape = [2, 2] - [0, 0]: FingerprintCat64( - Fingerprint64("f"), FingerprintCat64( - Fingerprint64("d"), Fingerprint64("a"))) - [1, 0]: FingerprintCat64( - Fingerprint64("g"), FingerprintCat64( - Fingerprint64("e"), Fingerprint64("b"))) - [1, 1]: FingerprintCat64( - Fingerprint64("g"), FingerprintCat64( - Fingerprint64("e"), Fingerprint64("c"))) - -indices: 2-D. Indices of each input `SparseTensor`. -values: 1-D. values of each `SparseTensor`. -shapes: 1-D. Shapes of each `SparseTensor`. -dense: 2-D. Columns represented by dense `Tensor`. -output_indices: 2-D. Indices of the concatenated `SparseTensor`. -output_values: 1-D. Non-empty values of the concatenated or hashed - `SparseTensor`. -output_shape: 1-D. Shape of the concatenated `SparseTensor`. -)doc"); -} // namespace tensorflow diff --git a/tensorflow/contrib/layers/python/kernel_tests/sparse_feature_cross_op_test.py b/tensorflow/contrib/layers/python/kernel_tests/sparse_feature_cross_op_test.py deleted file mode 100644 index 155d06a08e6..00000000000 --- a/tensorflow/contrib/layers/python/kernel_tests/sparse_feature_cross_op_test.py +++ /dev/null @@ -1,437 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.contrib.layers.sparse_feature_cross.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy - -from tensorflow.contrib import layers -from tensorflow.contrib.layers.python.ops import sparse_feature_cross_op -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import sparse_ops -from tensorflow.python.platform import test - - -class SparseCrossOpTest(test.TestCase): - - def test_simple(self): - """Tests a simple scenario. - """ - op = sparse_feature_cross_op.sparse_feature_cross([ - self._sparse_tensor([['batch1-FC1-F1'], - ['batch2-FC1-F1', 'batch2-FC1-F2']]), - self._sparse_tensor([['batch1-FC2-F1'], - ['batch2-FC2-F1', 'batch2-FC2-F2']]) - ]) - expected_out = self._sparse_tensor([['batch1-FC1-F1_X_batch1-FC2-F1'], [ - 'batch2-FC1-F1_X_batch2-FC2-F1', 'batch2-FC1-F1_X_batch2-FC2-F2', - 'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2' - ]]) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - def test_dense(self): - """Tests only dense inputs. - """ - op = sparse_feature_cross_op.sparse_feature_cross([ - constant_op.constant([['batch1-FC1-F1', 'batch1-FC1-F2'], - ['batch2-FC1-F1', 'batch2-FC1-F2']], - dtypes.string), - constant_op.constant([['batch1-FC2-F1', 'batch1-FC2-F2'], - ['batch2-FC2-F1', 'batch2-FC2-F2']], - dtypes.string), - ]) - expected_out = self._sparse_tensor([[ - 'batch1-FC1-F1_X_batch1-FC2-F1', 'batch1-FC1-F1_X_batch1-FC2-F2', - 'batch1-FC1-F2_X_batch1-FC2-F1', 'batch1-FC1-F2_X_batch1-FC2-F2' - ], [ - 'batch2-FC1-F1_X_batch2-FC2-F1', 'batch2-FC1-F1_X_batch2-FC2-F2', - 'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2' - ]]) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - def test_integer_mixed_string_sparse(self): - """Tests mixed type.""" - op = sparse_feature_cross_op.sparse_feature_cross([ - self._sparse_tensor([[11], [333, 55555]]), - self._sparse_tensor([['batch1-FC2-F1'], - ['batch2-FC2-F1', 'batch2-FC2-F2']]) - ]) - expected_out = self._sparse_tensor([['11_X_batch1-FC2-F1'], [ - '333_X_batch2-FC2-F1', '333_X_batch2-FC2-F2', '55555_X_batch2-FC2-F1', - '55555_X_batch2-FC2-F2' - ]]) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - def test_integer_mixed_string_dense(self): - """Tests mixed dense inputs. - """ - op = sparse_feature_cross_op.sparse_feature_cross([ - constant_op.constant([[11, 333], [55555, 999999]], dtypes.int64), - constant_op.constant([['batch1-FC2-F1', 'batch1-FC2-F2'], - ['batch2-FC2-F1', 'batch2-FC2-F2']], - dtypes.string), - ]) - expected_out = self._sparse_tensor([[ - '11_X_batch1-FC2-F1', '11_X_batch1-FC2-F2', '333_X_batch1-FC2-F1', - '333_X_batch1-FC2-F2' - ], [ - '55555_X_batch2-FC2-F1', '55555_X_batch2-FC2-F2', - '999999_X_batch2-FC2-F1', '999999_X_batch2-FC2-F2' - ]]) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - def test_sparse_cross_dense(self): - """Tests sparse and dense inputs. - """ - op = sparse_feature_cross_op.sparse_feature_cross([ - self._sparse_tensor([['batch1-FC1-F1'], - ['batch2-FC1-F1', 'batch2-FC1-F2']]), - constant_op.constant([['batch1-FC2-F1', 'batch1-FC2-F2'], - ['batch2-FC2-F1', 'batch2-FC2-F2']], - dtypes.string), - ]) - expected_out = self._sparse_tensor( - [['batch1-FC1-F1_X_batch1-FC2-F1', 'batch1-FC1-F1_X_batch1-FC2-F2'], [ - 'batch2-FC1-F1_X_batch2-FC2-F1', 'batch2-FC1-F1_X_batch2-FC2-F2', - 'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2' - ]]) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - def test_integer_sparse_input(self): - """Tests mixed type sparse and dense inputs.""" - op = sparse_feature_cross_op.sparse_feature_cross([ - self._sparse_tensor([[11], [333, 5555]]), - constant_op.constant([['batch1-FC2-F1', 'batch1-FC2-F2'], - ['batch2-FC2-F1', 'batch2-FC2-F2']], - dtypes.string), - ]) - expected_out = self._sparse_tensor( - [['11_X_batch1-FC2-F1', '11_X_batch1-FC2-F2'], [ - '333_X_batch2-FC2-F1', '333_X_batch2-FC2-F2', - '5555_X_batch2-FC2-F1', '5555_X_batch2-FC2-F2' - ]]) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - def test_permutation_3x3x3(self): - """Tests 3x3x3 permutation. - """ - op = sparse_feature_cross_op.sparse_feature_cross([ - self._sparse_tensor( - [['batch1-FC1-F1', 'batch1-FC1-F2', 'batch1-FC1-F3']]), - self._sparse_tensor( - [['batch1-FC2-F1', 'batch1-FC2-F2', 'batch1-FC2-F3']]), - self._sparse_tensor( - [['batch1-FC3-F1', 'batch1-FC3-F2', 'batch1-FC3-F3']]) - ]) - expected_out = self._sparse_tensor([[ - 'batch1-FC1-F1_X_batch1-FC2-F1_X_batch1-FC3-F1', - 'batch1-FC1-F1_X_batch1-FC2-F1_X_batch1-FC3-F2', - 'batch1-FC1-F1_X_batch1-FC2-F1_X_batch1-FC3-F3', - 'batch1-FC1-F1_X_batch1-FC2-F2_X_batch1-FC3-F1', - 'batch1-FC1-F1_X_batch1-FC2-F2_X_batch1-FC3-F2', - 'batch1-FC1-F1_X_batch1-FC2-F2_X_batch1-FC3-F3', - 'batch1-FC1-F1_X_batch1-FC2-F3_X_batch1-FC3-F1', - 'batch1-FC1-F1_X_batch1-FC2-F3_X_batch1-FC3-F2', - 'batch1-FC1-F1_X_batch1-FC2-F3_X_batch1-FC3-F3', - 'batch1-FC1-F2_X_batch1-FC2-F1_X_batch1-FC3-F1', - 'batch1-FC1-F2_X_batch1-FC2-F1_X_batch1-FC3-F2', - 'batch1-FC1-F2_X_batch1-FC2-F1_X_batch1-FC3-F3', - 'batch1-FC1-F2_X_batch1-FC2-F2_X_batch1-FC3-F1', - 'batch1-FC1-F2_X_batch1-FC2-F2_X_batch1-FC3-F2', - 'batch1-FC1-F2_X_batch1-FC2-F2_X_batch1-FC3-F3', - 'batch1-FC1-F2_X_batch1-FC2-F3_X_batch1-FC3-F1', - 'batch1-FC1-F2_X_batch1-FC2-F3_X_batch1-FC3-F2', - 'batch1-FC1-F2_X_batch1-FC2-F3_X_batch1-FC3-F3', - 'batch1-FC1-F3_X_batch1-FC2-F1_X_batch1-FC3-F1', - 'batch1-FC1-F3_X_batch1-FC2-F1_X_batch1-FC3-F2', - 'batch1-FC1-F3_X_batch1-FC2-F1_X_batch1-FC3-F3', - 'batch1-FC1-F3_X_batch1-FC2-F2_X_batch1-FC3-F1', - 'batch1-FC1-F3_X_batch1-FC2-F2_X_batch1-FC3-F2', - 'batch1-FC1-F3_X_batch1-FC2-F2_X_batch1-FC3-F3', - 'batch1-FC1-F3_X_batch1-FC2-F3_X_batch1-FC3-F1', - 'batch1-FC1-F3_X_batch1-FC2-F3_X_batch1-FC3-F2', - 'batch1-FC1-F3_X_batch1-FC2-F3_X_batch1-FC3-F3' - ]]) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - def test_permutation_3x1x2(self): - """Tests 3x1x2 permutation. - """ - op = sparse_feature_cross_op.sparse_feature_cross([ - self._sparse_tensor( - [['batch1-FC1-F1', 'batch1-FC1-F2', 'batch1-FC1-F3']]), - self._sparse_tensor([['batch1-FC2-F1']]), - self._sparse_tensor([['batch1-FC3-F1', 'batch1-FC3-F2']]) - ]) - expected_out = self._sparse_tensor([[ - 'batch1-FC1-F1_X_batch1-FC2-F1_X_batch1-FC3-F1', - 'batch1-FC1-F1_X_batch1-FC2-F1_X_batch1-FC3-F2', - 'batch1-FC1-F2_X_batch1-FC2-F1_X_batch1-FC3-F1', - 'batch1-FC1-F2_X_batch1-FC2-F1_X_batch1-FC3-F2', - 'batch1-FC1-F3_X_batch1-FC2-F1_X_batch1-FC3-F1', - 'batch1-FC1-F3_X_batch1-FC2-F1_X_batch1-FC3-F2' - ]]) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - def test_large_batch(self): - """Tests with large batch size to force multithreading. - """ - batch_size = 5000 - col1 = [] - col2 = [] - col3 = [] - for b in range(batch_size): - col1.append( - ['batch%d-FC1-F1' % b, 'batch%d-FC1-F2' % b, 'batch%d-FC1-F3' % b]) - col2.append(['batch%d-FC2-F1' % b]) - col3.append(['batch%d-FC3-F1' % b, 'batch%d-FC3-F2' % b]) - - op = sparse_feature_cross_op.sparse_feature_cross([ - self._sparse_tensor(col1), self._sparse_tensor(col2), - self._sparse_tensor(col3) - ]) - - col_out = [] - for b in range(batch_size): - col_out.append([ - 'batch%d-FC1-F1_X_batch%d-FC2-F1_X_batch%d-FC3-F1' % (b, b, b), - 'batch%d-FC1-F1_X_batch%d-FC2-F1_X_batch%d-FC3-F2' % (b, b, b), - 'batch%d-FC1-F2_X_batch%d-FC2-F1_X_batch%d-FC3-F1' % (b, b, b), - 'batch%d-FC1-F2_X_batch%d-FC2-F1_X_batch%d-FC3-F2' % (b, b, b), - 'batch%d-FC1-F3_X_batch%d-FC2-F1_X_batch%d-FC3-F1' % (b, b, b), - 'batch%d-FC1-F3_X_batch%d-FC2-F1_X_batch%d-FC3-F2' % (b, b, b) - ]) - - expected_out = self._sparse_tensor(col_out) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - def test_one_column_empty(self): - """Tests when one column is empty. - - The crossed tensor should be empty. - """ - op = sparse_feature_cross_op.sparse_feature_cross([ - self._sparse_tensor([['batch1-FC1-F1', 'batch1-FC1-F2']]), - self._sparse_tensor([], 1), - self._sparse_tensor([['batch1-FC3-F1', 'batch1-FC3-F2']]) - ]) - with self.cached_session() as sess: - self._assert_sparse_tensor_empty(sess.run(op)) - - def test_some_columns_empty(self): - """Tests when more than one columns are empty. - - Cross for the corresponding batch should be empty. - """ - op = sparse_feature_cross_op.sparse_feature_cross([ - self._sparse_tensor([['batch1-FC1-F1', 'batch1-FC1-F2']], 2), - self._sparse_tensor([['batch1-FC2-F1'], ['batch2-FC2-F1']], 2), - self._sparse_tensor([['batch1-FC3-F1', 'batch1-FC3-F2']], 2) - ]) - expected_out = self._sparse_tensor([[ - 'batch1-FC1-F1_X_batch1-FC2-F1_X_batch1-FC3-F1', - 'batch1-FC1-F1_X_batch1-FC2-F1_X_batch1-FC3-F2', - 'batch1-FC1-F2_X_batch1-FC2-F1_X_batch1-FC3-F1', - 'batch1-FC1-F2_X_batch1-FC2-F1_X_batch1-FC3-F2' - ]], 2) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - def test_all_columns_empty(self): - """Tests when all columns are empty. - - The crossed tensor should be empty. - """ - op = sparse_feature_cross_op.sparse_feature_cross([ - self._sparse_tensor([]), self._sparse_tensor([]), - self._sparse_tensor([]) - ]) - with self.cached_session() as sess: - self._assert_sparse_tensor_empty(sess.run(op)) - - def test_hashed_output_zero_bucket(self): - """Tests a simple scenario. - """ - op = sparse_feature_cross_op.sparse_feature_cross( - [ - self._sparse_tensor([['batch1-FC1-F1']]), - self._sparse_tensor([['batch1-FC2-F1']]), - self._sparse_tensor([['batch1-FC3-F1']]) - ], - hashed_output=True) - # Check actual hashed output to prevent unintentional hashing changes. - expected_out = self._sparse_tensor([[3735511728867393167]]) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - def test_hashed_output_zero_bucket_v2(self): - """Tests a simple scenario. - """ - op = sparse_feature_cross_op.sparse_feature_cross( - [ - self._sparse_tensor([['batch1-FC1-F1']]), - self._sparse_tensor([['batch1-FC2-F1']]), - self._sparse_tensor([['batch1-FC3-F1']]) - ], - hashed_output=True, - hash_key=layers.SPARSE_FEATURE_CROSS_DEFAULT_HASH_KEY) - # Check actual hashed output to prevent unintentional hashing changes. - expected_out = self._sparse_tensor([[1971693436396284976]]) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - # TODO(sibyl-Aix6ihai): Add benchmark to compare Hashed vs Non-hashed. - def test_hashed_output(self): - """Tests a simple scenario. - """ - op = sparse_feature_cross_op.sparse_feature_cross( - [ - self._sparse_tensor([['batch1-FC1-F1']]), - self._sparse_tensor([['batch1-FC2-F1']]), - self._sparse_tensor([['batch1-FC3-F1']]) - ], - hashed_output=True, - num_buckets=100) - # Check actual hashed output to prevent unintentional hashing changes. - expected_out = self._sparse_tensor([[74]]) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - def test_hashed_output_v2(self): - """Tests a simple scenario. - """ - op = sparse_feature_cross_op.sparse_feature_cross( - [ - self._sparse_tensor([['batch1-FC1-F1']]), - self._sparse_tensor([['batch1-FC2-F1']]), - self._sparse_tensor([['batch1-FC3-F1']]) - ], - hashed_output=True, - num_buckets=100, - hash_key=layers.SPARSE_FEATURE_CROSS_DEFAULT_HASH_KEY) - # Check actual hashed output to prevent unintentional hashing changes. - expected_out = self._sparse_tensor([[83]]) - with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) - - def test_hashed_output_v1_has_collision(self): - """Tests the old version of the fingerprint concatenation has collisions. - """ - # The last 10 bits of 359 and 1024+359 are identical. - # As a result, all the crosses collide. - t1 = constant_op.constant([[359], [359 + 1024]]) - t2 = constant_op.constant([list(range(10)), list(range(10))]) - cross = sparse_feature_cross_op.sparse_feature_cross( - [t2, t1], hashed_output=True, num_buckets=1024) - cross_dense = sparse_ops.sparse_tensor_to_dense(cross) - with session.Session(): - values = cross_dense.eval() - self.assertTrue(numpy.equal(values[0], values[1]).all()) - - def test_hashed_output_v2_has_no_collision(self): - """Tests the new version of the fingerprint concatenation has no collisions. - """ - # Although the last 10 bits of 359 and 1024+359 are identical. - # As a result, all the crosses shouldn't collide. - t1 = constant_op.constant([[359], [359 + 1024]]) - t2 = constant_op.constant([list(range(10)), list(range(10))]) - cross = sparse_feature_cross_op.sparse_feature_cross( - [t2, t1], - hashed_output=True, - num_buckets=1024, - hash_key=layers.SPARSE_FEATURE_CROSS_DEFAULT_HASH_KEY) - cross_dense = sparse_ops.sparse_tensor_to_dense(cross) - with session.Session(): - values = cross_dense.eval() - self.assertTrue(numpy.not_equal(values[0], values[1]).all()) - - def test_hashed_3x1x2(self): - """Tests 3x1x2 permutation with hashed output. - """ - op = sparse_feature_cross_op.sparse_feature_cross( - [ - self._sparse_tensor( - [['batch1-FC1-F1', 'batch1-FC1-F2', 'batch1-FC1-F3']]), - self._sparse_tensor([['batch1-FC2-F1']]), - self._sparse_tensor([['batch1-FC3-F1', 'batch1-FC3-F2']]) - ], - hashed_output=True, - num_buckets=1000) - with self.cached_session() as sess: - out = sess.run(op) - self.assertEqual(6, len(out.values)) - self.assertAllEqual([[0, i] for i in range(6)], out.indices) - self.assertTrue(all(x < 1000 and x >= 0 for x in out.values)) - all_values_are_different = len(out.values) == len(set(out.values)) - self.assertTrue(all_values_are_different) - - def _assert_sparse_tensor_empty(self, sp): - self.assertEquals(0, sp.indices.size) - self.assertEquals(0, sp.values.size) - # TODO(zakaria): check if we can ignore the first dim of the shape. - self.assertEquals(0, sp.dense_shape[1]) - - def _assert_sparse_tensor_equals(self, sp1, sp2): - self.assertAllEqual(sp1.indices.eval(), sp2.indices) - self.assertAllEqual(sp1.values.eval(), sp2.values) - self.assertAllEqual(sp1.dense_shape.eval(), sp2.dense_shape) - - def _sparse_tensor(self, data, batch_size=-1): - """Generates a SparseTensor. - - Args: - data: Should be a list of list of strings or int64. Each item of the outer - list represents a batch. Each item of the batch is a feature of a - specific feature column. - batch_size: optional batch size, especially for cases when data has no - entry for some batches. - - Returns: - A SparseTensor. - """ - indices = [] - values = [] - max_col_count = 0 - for batch, batch_ix in zip(data, range(len(data))): - for column, column_ix in zip(batch, range(len(batch))): - indices.append([batch_ix, column_ix]) - values.append(column) - max_col_count = max(max_col_count, column_ix + 1) - shape = [batch_size if batch_size != -1 else len(data), max_col_count] - value_type = (dtypes.string if not values or isinstance(values[0], str) else - dtypes.int64) - return sparse_tensor.SparseTensor( - constant_op.constant(indices, dtypes.int64, [len(indices), 2]), - constant_op.constant(values, value_type, [len(indices)]), - constant_op.constant(shape, dtypes.int64)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/layers/python/layers/__init__.py b/tensorflow/contrib/layers/python/layers/__init__.py deleted file mode 100644 index f1ae2de68be..00000000000 --- a/tensorflow/contrib/layers/python/layers/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""layers module with higher level NN primitives.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.layers.python.layers.embedding_ops import * -from tensorflow.contrib.layers.python.layers.encoders import * -from tensorflow.contrib.layers.python.layers.feature_column import * -from tensorflow.contrib.layers.python.layers.feature_column_ops import * -from tensorflow.contrib.layers.python.layers.initializers import * -from tensorflow.contrib.layers.python.layers.layers import * -from tensorflow.contrib.layers.python.layers.normalization import * -from tensorflow.contrib.layers.python.layers.optimizers import * -from tensorflow.contrib.layers.python.layers.regularizers import * -from tensorflow.contrib.layers.python.layers.rev_block_lib import * -from tensorflow.contrib.layers.python.layers.summaries import * -from tensorflow.contrib.layers.python.layers.target_column import * -from tensorflow.contrib.layers.python.ops.bucketization_op import * -from tensorflow.contrib.layers.python.ops.sparse_feature_cross_op import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops.py b/tensorflow/contrib/layers/python/layers/embedding_ops.py deleted file mode 100644 index 14bbe5f9b30..00000000000 --- a/tensorflow/contrib/layers/python/layers/embedding_ops.py +++ /dev/null @@ -1,943 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Embedding functions.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib.framework.python.framework import tensor_util as contrib_tensor_util -from tensorflow.contrib.layers.python.ops import sparse_feature_cross_op - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import tf_logging as logging - -__all__ = [ - "safe_embedding_lookup_sparse", "scattered_embedding_lookup", - "scattered_embedding_lookup_sparse", "embedding_lookup_unique", - "embedding_lookup_sparse_with_distributed_aggregation" -] - - -def safe_embedding_lookup_sparse(embedding_weights, - sparse_ids, - sparse_weights=None, - combiner=None, - default_id=None, - name=None, - partition_strategy="div", - max_norm=None): - """Lookup embedding results, accounting for invalid IDs and empty features. - - The partitioned embedding in `embedding_weights` must all be the same shape - except for the first dimension. The first dimension is allowed to vary as the - vocabulary size is not necessarily a multiple of `P`. `embedding_weights` - may be a `PartitionedVariable` as returned by using - `tf.compat.v1.get_variable()` with a - partitioner. - - Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs - with non-positive weight. For an entry with no features, the embedding vector - for `default_id` is returned, or the 0-vector if `default_id` is not supplied. - - The ids and weights may be multi-dimensional. Embeddings are always aggregated - along the last dimension. - - Args: - embedding_weights: A list of `P` float tensors or values representing - partitioned embedding tensors. Alternatively, a `PartitionedVariable`, - created by partitioning along dimension 0. The total unpartitioned shape - should be `[e_0, e_1, ..., e_m]`, where `e_0` represents the vocab size - and `e_1, ..., e_m` are the embedding dimensions. - sparse_ids: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the - ids. `d_0` is typically batch size. - sparse_weights: `SparseTensor` of same shape as `sparse_ids`, containing - float weights corresponding to `sparse_ids`, or `None` if all weights are - be assumed to be 1.0. - combiner: A string specifying how to combine embedding results for each - entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean" the - default. - default_id: The id to use for an entry with no features. - name: A name for this operation (optional). - partition_strategy: A string specifying the partitioning strategy. Currently - `"div"` and `"mod"` are supported. Default is `"div"`. - max_norm: If not None, all embeddings are l2-normalized to max_norm before - combining. - - Returns: - Dense tensor of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`. - - Raises: - ValueError: if `embedding_weights` is empty. - """ - if combiner is None: - logging.warn("The default value of combiner will change from \"mean\" " - "to \"sqrtn\" after 2016/11/01.") - combiner = "mean" - if embedding_weights is None: - raise ValueError("Missing embedding_weights %s." % embedding_weights) - if isinstance(embedding_weights, variables.PartitionedVariable): - embedding_weights = list(embedding_weights) # get underlying Variables. - if not isinstance(embedding_weights, list): - embedding_weights = [embedding_weights] - if len(embedding_weights) < 1: - raise ValueError("Missing embedding_weights %s." % embedding_weights) - - dtype = sparse_weights.dtype if sparse_weights is not None else None - if isinstance(embedding_weights, variables.PartitionedVariable): - embedding_weights = list(embedding_weights) - embedding_weights = [ - ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights - ] - - contrib_tensor_util.assert_same_float_dtype(embedding_weights + - [sparse_weights]) - - with ops.name_scope(name, "embedding_lookup", embedding_weights + - [sparse_ids, sparse_weights]) as scope: - # Reshape higher-rank sparse ids and weights to linear segment ids. - original_shape = sparse_ids.dense_shape - original_rank_dim = tensor_shape.Dimension( - tensor_shape.dimension_value(sparse_ids.dense_shape.get_shape()[0])) - original_rank = ( - array_ops.size(original_shape) - if original_rank_dim.value is None else original_rank_dim.value) - sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ - math_ops.reduce_prod( - array_ops.slice(original_shape, [0], [original_rank - 1])), - array_ops.gather(original_shape, original_rank - 1) - ]) - if sparse_weights is not None: - sparse_weights = sparse_tensor.SparseTensor(sparse_ids.indices, - sparse_weights.values, - sparse_ids.dense_shape) - - # Prune invalid ids and weights. - sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) - if combiner != "sum": - sparse_ids, sparse_weights = _prune_invalid_weights( - sparse_ids, sparse_weights) - - # Fill in dummy values for empty features, if necessary. - sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows( - sparse_ids, default_id or 0) - if sparse_weights is not None: - sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) - - result = embedding_ops.embedding_lookup_sparse( - embedding_weights, - sparse_ids, - sparse_weights, - combiner=combiner, - partition_strategy=partition_strategy, - name=None if default_id is None else scope, - max_norm=max_norm) - - if default_id is None: - # Broadcast is_row_empty to the same shape as embedding_lookup_result, - # for use in Select. - is_row_empty = array_ops.tile( - array_ops.reshape(is_row_empty, [-1, 1]), - array_ops.stack([1, array_ops.shape(result)[1]])) - - result = array_ops.where( - is_row_empty, array_ops.zeros_like(result), result, name=scope) - - # Reshape back from linear ids back into higher-dimensional dense result. - final_result = array_ops.reshape( - result, - array_ops.concat([ - array_ops.slice( - math_ops.cast(original_shape, dtypes.int32), [0], - [original_rank - 1]), - array_ops.slice(array_ops.shape(result), [1], [-1]) - ], 0)) - final_result.set_shape( - tensor_shape.unknown_shape( - (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) - return final_result - - -def _prune_invalid_ids(sparse_ids, sparse_weights): - """Prune invalid IDs (< 0) from the input ids and weights.""" - is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) - if sparse_weights is not None: - is_id_valid = math_ops.logical_and( - is_id_valid, - array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) - sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) - if sparse_weights is not None: - sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) - return sparse_ids, sparse_weights - - -def _prune_invalid_weights(sparse_ids, sparse_weights): - """Prune invalid weights (< 0) from the input ids and weights.""" - if sparse_weights is not None: - is_weights_valid = math_ops.greater(sparse_weights.values, 0) - sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_weights_valid) - sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_weights_valid) - return sparse_ids, sparse_weights - - -def scattered_embedding_lookup(params, - values, - dimension, - name=None, - hash_key=None): - """Looks up embeddings using parameter hashing for each value in `values`. - - The i-th embedding component of a value v in `values` is found by retrieving - the weight whose index is a fingerprint of the pair (v,i). - The concept is explored as "feature hashing" for model compression in this - paper: http://arxiv.org/pdf/1504.04788.pdf - - Feature hashing has the pleasant effect of allowing us to compute an embedding - without needing a pre-determined vocabulary, relieving some amount of process - complexity. It also allows for us to maintain embeddings for possibly - trillions of features with a fixed amount of memory. - - Note that this is superior to out-of-vocabulary shared "hash buckets" in that - the embedding is extremely likely to be unique for each token as opposed to - being shared across probably-colliding tokens. The price is that we must - compute a hash once for each scalar in the token's embedding as opposed to - once per token. - - If `params` is a list, it represents a partition of the embedding parameters. - Each tensor in the list should have the same length, except for the first ones - which may have an additional element. For instance 10 parameters can be - partitioned in 4 tensors with length `[3, 3, 2, 2]`. - - Args: - params: A `Tensor`, `list` of `Tensors`, or `PartitionedVariable`. Each - tensor must be of rank 1 with fully-defined shape. - values: `Tensor` of values to be embedded with shape `[d0, ..., dn]`. - dimension: Embedding dimension. - name: An optional name for this op. - hash_key: Specify the hash_key that will be used by the `FingerprintCat64` - function to combine the crosses fingerprints on SparseFeatureCrossOp - (optional). - - Returns: - A `Tensor` with shape `[d0, ..., dn, dimension]`. - - Raises: - ValueError: if dimension is not positive or the partition size is invalid. - """ - if dimension is None: - raise ValueError("You must specify dimension.") - return _sampled_scattered_embedding_lookup( - params, - values, - dimension=dimension, - sampled_candidates=None, - hash_key=hash_key, - name=name) - - -def _sampled_scattered_embedding_lookup(params, - values, - dimension=None, - sampled_candidates=None, - hash_key=None, - name=None): - """Looks up embeddings using parameter hashing for each value in `values`. - - This method looks up selected embedding dimensions if `sampled_candidates` is - given, otherwise looks up all dimensions. - - The i-th embedding component of a value v in `values` is found by retrieving - the weight whose index is a fingerprint of the pair (v,i). - The concept is explored as "feature hashing" for model compression in this - paper: http://arxiv.org/pdf/1504.04788.pdf - - Feature hashing has the pleasant effect of allowing us to compute an embedding - without needing a pre-determined vocabulary, relieving some amount of process - complexity. It also allows for us to maintain embeddings for possibly - trillions of features with a fixed amount of memory. - - Note that this is superior to out-of-vocabulary shared "hash buckets" in that - the embedding is extremely likely to be unique for each token as opposed to - being shared across probably-colliding tokens. The price is that we must - compute a hash once for each scalar in the token's embedding as opposed to - once per token. - - If `params` is a list, it represents a partition of the embedding parameters. - Each tensor in the list should have the same length, except for the first ones - which may have an additional element. For instance 10 parameters can be - partitioned in 4 tensors with length `[3, 3, 2, 2]`. - - Args: - params: A `Tensor`, `list` of `Tensors`, or `PartitionedVariable`. Each - tensor must be of rank 1 with fully-defined shape. - values: `Tensor` of values to be embedded with shape `[d0, ..., dn]`. - dimension: Embedding dimension. The user must specify either `dimension` or - `sampled_candidates`. - sampled_candidates: An optional `Tensor` of slice indices to keep along the - final dimension with shape `[d0, ..., dn, N]`. If given, `dimension` is - ignored. If `None`, looks up all candidates. - hash_key: Specify the hash_key that will be used by the `FingerprintCat64` - function to combine the crosses fingerprints on SparseFeatureCrossOp - (optional). - name: An optional name for this op. - - Returns: - A `Tensor` with shape `[d0, ..., dn, dimension]`. - If `sampled_candidates` is given, the output shape is `[d0, ..., dn, N]` - - Raises: - ValueError: if dimension is not positive or the partition size is invalid. - """ - if isinstance(params, variables.PartitionedVariable): - params = list(params) - if not isinstance(params, list): - params = [params] - - with ops.name_scope(name, "scattered_embedding_lookup", - params + [dimension, values]): - # Flatten the values - values_shape = array_ops.shape(values) - values = array_ops.reshape(values, [-1, 1]) - - if sampled_candidates is None: - if dimension is None: - raise ValueError( - "You must specify either dimension or sampled_candidates.") - if dimension <= 0: - raise ValueError("Dimension must be >0. Given is %d" % dimension) - sampled_candidates = array_ops.tile( - array_ops.expand_dims(math_ops.range(0, dimension), 0), - array_ops.shape(values)) - else: - dimension = array_ops.shape(sampled_candidates)[math_ops.subtract( - array_ops.rank(sampled_candidates), 1)] - sampled_candidates_shape = array_ops.shape(sampled_candidates) - dimension_tensor = array_ops.reshape( - dimension, shape=[ - 1, - ]) - expected_shape = array_ops.concat([values_shape, dimension_tensor], 0) - with ops.control_dependencies([ - control_flow_ops.Assert( - math_ops.reduce_all( - math_ops.equal(sampled_candidates_shape, expected_shape)), - [ - "The shape of sampled_candidates: ", sampled_candidates_shape, - " does not match the shape of values: ", values_shape - ]) - ]): - # Flatten sampled_candidates, same way as values are flattened. - sampled_candidates = array_ops.reshape(sampled_candidates, - [-1, dimension]) - - num_partitions = len(params) - partition_sizes = [] - for p in range(num_partitions): - shape = params[p].get_shape() - shape.assert_has_rank(1) - shape.assert_is_fully_defined() - partition_sizes.append(tensor_shape.dimension_value(shape[0])) - num_params = sum(partition_sizes) # Total number of parameters. - - # Assert the size of each partition. - for p in range(num_partitions): - expected_size = (num_params - p - 1) // num_partitions + 1 - if partition_sizes[p] != expected_size: - raise ValueError("Tensor %d in params has size %d, expected %d." % - (p, partition_sizes[p], expected_size)) - - # With two values v1 and v2 and 3 dimensions, we will cross - # [[0, 1, 2], [0, 1, 2]] with [[v1], [v2]]. - tensors_to_cross = [sampled_candidates, values] - ids = sparse_feature_cross_op.sparse_feature_cross( - tensors_to_cross, - hashed_output=True, - num_buckets=num_params, - hash_key=hash_key) - ids = sparse_ops.sparse_tensor_to_dense(ids) - - # No need to validate the indices since we have checked the params - # dimensions and we know the largest id. - result = embedding_ops.embedding_lookup( - params, ids, partition_strategy="div") - - return array_ops.reshape(result, - array_ops.concat([values_shape, [dimension]], 0)) - - -def scattered_embedding_lookup_sparse(params, - sparse_values, - dimension, - combiner=None, - default_value=None, - name=None, - hash_key=None): - """Looks up embeddings of a sparse feature using parameter hashing. - - See `tf.contrib.layers.scattered_embedding_lookup` for embedding with hashing. - - Args: - params: A `Tensor`, `list` of `Tensors`, or `PartitionedVariable`. Each - tensor must be of rank 1 with fully-defined shape. - sparse_values: A 2-D `SparseTensor` containing the values to be embedded. - Some rows may be empty. - dimension: Embedding dimension - combiner: A string specifying how to combine embedding results for each - entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean" the - default. - default_value: The value to use for an entry with no features. - name: An optional name for this op. - hash_key: Specify the hash_key that will be used by the `FingerprintCat64` - function to combine the crosses fingerprints on SparseFeatureCrossOp - (optional). - - Returns: - Dense tensor with shape [N, dimension] with N the number of rows in - sparse_values. - - Raises: - TypeError: If sparse_values is not a SparseTensor. - ValueError: If combiner is not one of {"mean", "sqrtn", "sum"}. - """ - if combiner is None: - logging.warn("The default value of combiner will change from \"mean\" " - "to \"sqrtn\" after 2016/11/01.") - combiner = "mean" - if isinstance(params, variables.PartitionedVariable): - params = list(params) - if not isinstance(params, list): - params = [params] - if not isinstance(sparse_values, sparse_tensor.SparseTensor): - raise TypeError("sparse_values must be SparseTensor") - - with ops.name_scope(name, "scattered_embedding_lookup_sparse", - params + [sparse_values]) as scope: - # Fill in the empty rows. - if default_value is None: - # Random default values to reduce the risk of collision. - if sparse_values.dtype == dtypes.string: - default_value = "6ZxWzWOHxZ" - else: - default_value = 1288896567 - sparse_values, _ = sparse_ops.sparse_fill_empty_rows( - sparse_values, default_value) - - segment_ids = sparse_values.indices[:, 0] - if segment_ids.dtype != dtypes.int32: - segment_ids = math_ops.cast(segment_ids, dtypes.int32) - - values = sparse_values.values - values, idx = array_ops.unique(values) - - embeddings = scattered_embedding_lookup( - params, values, dimension, hash_key=hash_key) - - if combiner == "sum": - embeddings = math_ops.sparse_segment_sum( - embeddings, idx, segment_ids, name=scope) - elif combiner == "mean": - embeddings = math_ops.sparse_segment_mean( - embeddings, idx, segment_ids, name=scope) - elif combiner == "sqrtn": - embeddings = math_ops.sparse_segment_sqrt_n( - embeddings, idx, segment_ids, name=scope) - else: - raise ValueError("Combiner must be one of 'mean', 'sqrtn' or 'sum'.") - - return embeddings - - -def embedding_lookup_unique(params, ids, partition_strategy="mod", name=None): - """Version of embedding_lookup that avoids duplicate lookups. - - This can save communication in the case of repeated ids. - Same interface as embedding_lookup. Except it supports multi-dimensional `ids` - which allows to not reshape input/output to fit gather. - - Args: - params: A list of tensors with the same shape and type, or a - `PartitionedVariable`. Shape `[index, d1, d2, ...]`. - ids: A one-dimensional `Tensor` with type `int32` or `int64` containing the - ids to be looked up in `params`. Shape `[ids1, ids2, ...]`. - partition_strategy: A string specifying the partitioning strategy, relevant - if `len(params) > 1`. Currently `"div"` and `"mod"` are supported. Default - is `"mod"`. - name: A name for this operation (optional). - - Returns: - A `Tensor` with the same type as the tensors in `params` and dimension of - `[ids1, ids2, d1, d2, ...]`. - - Raises: - ValueError: If `params` is empty. - """ - with ops.name_scope(name, "EmbeddingLookupUnique", [params, ids]): - ids = ops.convert_to_tensor(ids) - shape = array_ops.shape(ids) - ids_flat = array_ops.reshape(ids, - math_ops.reduce_prod(shape, keepdims=True)) - unique_ids, idx = array_ops.unique(ids_flat) - unique_embeddings = embedding_ops.embedding_lookup(params, unique_ids, - partition_strategy) - embeds_flat = array_ops.gather(unique_embeddings, idx) - embed_shape = array_ops.concat( - [shape, array_ops.shape(unique_embeddings)[1:]], 0) - embeds = array_ops.reshape(embeds_flat, embed_shape) - embeds.set_shape(ids.get_shape().concatenate( - unique_embeddings.get_shape()[1:])) - return embeds - - -def _sampled_scattered_embedding_lookup_sparse(params, - sp_values, - dimension=None, - sampled_candidates=None, - hash_key=None, - with_sign_hash=False, - name=None): - """Looks up embeddings using parameter hashing for sparse values. - - This method looks up selected embedding dimensions if `sampled_candidates` is - given, otherwise looks up all dimensions. - - The i-th embedding component of a value v in `values` is found by retrieving - the weight whose index is a fingerprint of the pair (v,i). - The concept is explored as "feature hashing" for model compression in this - paper: http://arxiv.org/pdf/1504.04788.pdf - - This is logically equivalent to: - * Transforming `sp_values` (which has shape `[d0, d1]`) into a one-hot - `Tensor` of shape `[d0, N]`. - * Multiplying with a `Tensor` `h` of shape `[N, dimension]`, where - `h(i, j) = params[hash(i, j)]`. - - Args: - params: A float `Tensor` with rank 1 and fully-defined shape. - sp_values: A 2D `SparseTensor` to be embedded with shape `[d0, d1]`. - dimension: An int `Tensor` of the final dimension. The user needs to provide - either `dimension` or `sampled_candidates`. - sampled_candidates: An optional `Tensor` of column indices to keep along the - final dimension with shape `[d0, N]`. If given, `dimension` is ignored. If - `None`, looks up all candidates. - hash_key: Specify the hash_key that will be used by the `FingerprintCat64` - function to combine the crosses fingerprints on SparseFeatureCrossOp - (optional). - with_sign_hash: A `bool` indicating whether `h(i, j)` should be multiplied - by `+1` or `-1`, where the value selected is determined by hashing `(i, - j)`. This is often necessary to remove bias resulting from hash - collisions. - name: An optional name for this op. - - Returns: - A `Tensor` of shape `[d0, dimension]`. - If `sampled_candidates` is given, the output shape is `[d0, N]`. - - Raises: - TypeError: If sp_values is not `SparseTensor`. - ValueError: If both `dimension` and `sampled_candidates` are `None`. - """ - if not isinstance(sp_values, sparse_tensor.SparseTensor): - raise TypeError("sp_values must be SparseTensor") - - with ops.name_scope( - name=name, - default_name="sampled_scattered_embedding_lookup_sparse", - values=[sp_values, params, dimension, sampled_candidates]) as name_scope: - segment_ids = sp_values.indices[:, 0] - if sampled_candidates is not None: - # Tile sampled_candidates so there is one line corresponding to each - # element in sp_values.values - sampled_candidates = array_ops.gather(sampled_candidates, segment_ids) - - embeddings = _sampled_scattered_embedding_lookup( - params, - sp_values.values, - dimension=dimension, - sampled_candidates=sampled_candidates, - hash_key=hash_key, - name="values_lookup") - if with_sign_hash: - signs = _sampled_scattered_embedding_lookup( - array_ops.constant([-1., 1.]), - sp_values.values, - dimension=dimension, - sampled_candidates=sampled_candidates, - hash_key=hash_key, - name="signs_lookup") - embeddings = math_ops.multiply(signs, embeddings, name="signs_hash") - - if segment_ids.dtype != dtypes.int32: - segment_ids = math_ops.cast(segment_ids, dtypes.int32) - num_segments = array_ops.shape(sp_values)[0] - - return math_ops.unsorted_segment_sum( - embeddings, segment_ids, num_segments=num_segments, name=name_scope) - - -def embedding_lookup_sparse_with_distributed_aggregation( - params, - sp_ids, - sp_weights, - partition_strategy="mod", - name=None, - combiner=None, - max_norm=None): - """Computes embeddings for the given ids and weights. - - Embeddings belonging to same param are aggregated on that device first. This - op is intended to decrease data transmission and improve parallelism. See - `tf.nn.embedding_lookup_sparse` for the functionality and example of this op. - - Args: - params: A single tensor representing the complete embedding tensor, or a - list of P tensors all of same shape except for the first dimension, - representing sharded embedding tensors. Alternatively, a - `PartitionedVariable`, created by partitioning along dimension 0. Each - element must be appropriately sized for the given `partition_strategy`. - sp_ids: N x M SparseTensor of int64 ids (typically from FeatureValueToId), - where N is typically batch size and M is arbitrary. - sp_weights: either a SparseTensor of float / double weights, or None to - indicate all weights should be taken to be 1. If specified, sp_weights - must have exactly the same shape and indices as sp_ids. - partition_strategy: A string specifying the partitioning strategy, relevant - if `len(params) > 1`. Currently `"div"` and `"mod"` are supported. Default - is `"mod"`. See `tf.nn.embedding_lookup` for more details. - name: Optional name for the op. - combiner: A string specifying the reduction op. Currently "mean", "sqrtn" - and "sum" are supported. "sum" computes the weighted sum of the embedding - results for each row. "mean" is the weighted sum divided by the total - weight. "sqrtn" is the weighted sum divided by the square root of the sum - of the squares of the weights. - max_norm: If not None, each embedding is normalized to have l2 norm equal to - max_norm before combining. - - Returns: - A dense tensor representing the combined embeddings for the - sparse ids. For each row in the dense tensor represented by sp_ids, the op - looks up the embeddings for all ids in that row, multiplies them by the - corresponding weight, and combines these embeddings as specified. - - Raises: - TypeError: If sp_ids is not a SparseTensor, or if sp_weights is neither - None nor SparseTensor. - ValueError: If combiner is not one of {"mean", "sqrtn", "sum"}. - """ - if combiner is None: - logging.warn("The default value of combiner will change from \"mean\" " - "to \"sqrtn\" after 2016/11/01.") - combiner = "mean" - if combiner not in ("mean", "sqrtn", "sum"): - raise ValueError("combiner must be one of 'mean', 'sqrtn' or 'sum'") - if isinstance(params, variables.PartitionedVariable): - params = list(params) # Iterate to get the underlying Variables. - if not isinstance(params, list): - params = [params] - if not isinstance(sp_ids, sparse_tensor.SparseTensor): - raise TypeError("sp_ids must be SparseTensor") - ignore_weights = sp_weights is None - if not ignore_weights: - if not isinstance(sp_weights, sparse_tensor.SparseTensor): - raise TypeError("sp_weights must be either None or SparseTensor") - sp_ids.values.get_shape().assert_is_compatible_with( - sp_weights.values.get_shape()) - sp_ids.indices.get_shape().assert_is_compatible_with( - sp_weights.indices.get_shape()) - sp_ids.dense_shape.get_shape().assert_is_compatible_with( - sp_weights.dense_shape.get_shape()) - # TODO(yleon): Add enhanced node assertions to verify that sp_ids and - # sp_weights have equal indices and shapes. - - with ops.name_scope(name, "embedding_lookup_sparse", - params + [sp_ids]) as name: - segment_ids = sp_ids.indices[:, 0] - if segment_ids.dtype != dtypes.int32: - segment_ids = math_ops.cast(segment_ids, dtypes.int32) - - ids = sp_ids.values - if ignore_weights: - ids, idx = array_ops.unique(ids) - else: - idx = None - - weights = None if ignore_weights else sp_weights.values - embeddings = _embedding_lookup_with_distributed_aggregation( - params, - ids, - partition_strategy=partition_strategy, - max_norm=max_norm, - weights=weights, - idx=idx, - segment_ids=segment_ids) - # Set weights to all one if ignore weights. - if ignore_weights: - weights = array_ops.fill([array_ops.shape(segment_ids)[0]], 1) - if weights.dtype != embeddings.dtype: - weights = math_ops.cast(weights, embeddings.dtype) - # Reshape weights. - ones = array_ops.fill( - array_ops.expand_dims(array_ops.rank(embeddings) - 1, 0), 1) - bcast_weights_shape = array_ops.concat([array_ops.shape(weights), ones], 0) - orig_weights_shape = weights.get_shape() - weights = array_ops.reshape(weights, bcast_weights_shape) - if embeddings.get_shape().ndims is not None: - weights.set_shape( - orig_weights_shape.concatenate( - [1 for _ in range(embeddings.get_shape().ndims - 1)])) - - if combiner == "mean": - weight_sum = math_ops.segment_sum(weights, segment_ids) - embeddings = math_ops.div(embeddings, weight_sum) - elif combiner == "sqrtn": - weights_squared = math_ops.pow(weights, 2) - weight_sum = math_ops.segment_sum(weights_squared, segment_ids) - weight_sum_sqrt = math_ops.sqrt(weight_sum) - embeddings = math_ops.div(embeddings, weight_sum_sqrt) - elif combiner != "sum": - assert False, "Unrecognized combiner" - return embeddings - - -def _do_gather(params, ids, name=None): - """Deals with doing gather differently for resource variables.""" - if isinstance(params, resource_variable_ops.ResourceVariable): - return params.sparse_read(ids, name=name) - return array_ops.gather(params, ids, name=name) - - -def _embedding_lookup_with_distributed_aggregation(params, - ids, - partition_strategy="mod", - name=None, - max_norm=None, - weights=None, - idx=None, - segment_ids=None): - """Lookup helper for embedding_lookup_sparse_with_distributed_aggregation.""" - if params is None or params == []: # pylint: disable=g-explicit-bool-comparison - raise ValueError("Need at least one param") - if isinstance(params, variables.PartitionedVariable): - params = list(params) # Iterate to get the underlying Variables. - if not isinstance(params, list): - params = [params] - - def maybe_normalize(x): - if max_norm is not None: - if x.get_shape().ndims is not None: - ndims = x.get_shape().ndims - else: - ndims = array_ops.size(array_ops.shape(x)) - return clip_ops.clip_by_norm(x, max_norm, axes=list(range(1, ndims))) - return x - - with ops.name_scope(name, "embedding_lookup_with_distributed_aggregation", - params + [ids]) as name: - np = len(params) # Number of partitions - # Preserve the resource variable status to avoid accidental dense reads. - if not any( - isinstance(p, resource_variable_ops.ResourceVariable) for p in params): - params = ops.convert_n_to_tensor_or_indexed_slices(params, name="params") - if np == 1: - with ops.colocate_with(params[0]): - ret = maybe_normalize(_do_gather(params[0], ids)) - ignore_weights = weights is None - if not ignore_weights: - if weights.dtype != ret.dtype: - weights = math_ops.cast(weights, ret.dtype) - # Reshape to allow broadcast - ones = array_ops.fill( - array_ops.expand_dims(array_ops.rank(ret) - 1, 0), 1) - bcast_weights_shape = array_ops.concat( - [array_ops.shape(weights), ones], 0) - orig_weights_shape = weights.get_shape() - weights = array_ops.reshape(weights, bcast_weights_shape) - # Set weights shape after reshape - if ret.get_shape().ndims is not None: - weights.set_shape( - orig_weights_shape.concatenate( - [1 for _ in range(ret.get_shape().ndims - 1)])) - ret *= weights - return math_ops.segment_sum(ret, segment_ids, name=name) - else: - return math_ops.sparse_segment_sum(ret, idx, segment_ids, name=name) - else: - ids = ops.convert_to_tensor(ids, name="ids") - flat_ids = array_ops.reshape(ids, [-1]) - original_indices = math_ops.range(array_ops.size(flat_ids)) - - # Create p_assignments and set new_ids depending on the strategy. - if partition_strategy == "mod": - p_assignments = flat_ids % np - new_ids = flat_ids // np - elif partition_strategy == "div": - # Compute num_total_ids as the sum of dim-0 of params, then assign to - # partitions based on a constant number of ids per partition. Optimize - # if we already know the full shape statically. - dim_0_size = params[0].get_shape().dims[0] - for p in xrange(1, np): - dim_0_size += params[p].get_shape().dims[0] - if dim_0_size.value: - num_total_ids = constant_op.constant(dim_0_size, flat_ids.dtype) - else: - dim_0_sizes = [] - for p in xrange(np): - if params[p].get_shape().dims[0].value is not None: - dim_0_sizes.append(params[p].get_shape().dims[0].value) - else: - with ops.colocate_with(params[p]): - dim_0_sizes.append(array_ops.shape(params[p])[0]) - num_total_ids = math_ops.reduce_sum( - math_ops.cast(array_ops.stack(dim_0_sizes), flat_ids.dtype)) - ids_per_partition = num_total_ids // np - extras = num_total_ids % np - - p_assignments = math_ops.maximum(flat_ids // (ids_per_partition + 1), - (flat_ids - extras) // - ids_per_partition) - - # Emulate a conditional using a boolean indicator tensor - is_in_first_extras_partitions = math_ops.cast(p_assignments < extras, - flat_ids.dtype) - new_ids = ( - is_in_first_extras_partitions * (flat_ids % - (ids_per_partition + 1)) + - (1 - is_in_first_extras_partitions) * - ((flat_ids - extras) % ids_per_partition)) - else: - raise ValueError("Unrecognized partition strategy: " + - partition_strategy) - - # Cast partition assignments to int32 for use in dynamic_partition. - # There really should not be more than 2^32 partitions. - p_assignments = math_ops.cast(p_assignments, dtypes.int32) - # Partition list of ids based on assignments into np separate lists - gather_ids = data_flow_ops.dynamic_partition(new_ids, p_assignments, np) - # Similarly, partition the original indices. - pindices = data_flow_ops.dynamic_partition(original_indices, - p_assignments, np) - # Do np separate lookups, finding embeddings for plist[p] in params[p] - partitioned_result = [] - for p in xrange(np): - with ops.colocate_with(params[p]): - partitioned_result.append(_do_gather(params[p], gather_ids[p])) - - ignore_weights = weights is None - if not ignore_weights: - # Partition weights according to pindices. - partitioned_weight = [] - for p in xrange(np): - partitioned_weight.append(array_ops.gather(weights, pindices[p])) - # Reshape each partition result. - element_shape = params[0].get_shape()[1:] - for p in params[1:]: - element_shape = element_shape.merge_with(p.get_shape()[1:]) - if element_shape.is_fully_defined(): - for p in xrange(np): - with ops.colocate_with(params[p]): - partitioned_result[p] = array_ops.reshape( - partitioned_result[p], - array_ops.concat([array_ops.shape(pindices[p]), element_shape], - 0)) - else: - with ops.colocate_with(params[0]): - params_shape = array_ops.shape(params[0]) - for p in xrange(np): - with ops.colocate_with(params[p]): - partitioned_result[p] = array_ops.reshape( - partitioned_result[p], - array_ops.concat([ - array_ops.shape(pindices[p]), - array_ops.slice(params_shape, [1], [-1]) - ], 0)) - # Normalize each partition result. - for p in xrange(np): - with ops.colocate_with(params[p]): - partitioned_result[p] = maybe_normalize(partitioned_result[p]) - if not ignore_weights: - # Multiply each partition result with partition weights. - for p in xrange(np): - with ops.colocate_with(params[p]): - if partitioned_weight[p].dtype != partitioned_result[p].dtype: - partitioned_weight[p] = math_ops.cast(partitioned_weight[p], - partitioned_result[p].dtype) - # Reshape partition weights. - ones = array_ops.fill( - array_ops.expand_dims( - array_ops.rank(partitioned_result[p]) - 1, 0), 1) - bcast_weights_shape = array_ops.concat( - [array_ops.shape(partitioned_weight[p]), ones], 0) - orig_weights_shape = partitioned_weight[p].get_shape() - partitioned_weight[p] = array_ops.reshape(partitioned_weight[p], - bcast_weights_shape) - if partitioned_result[p].get_shape().ndims is not None: - partitioned_weight[p].set_shape( - orig_weights_shape.concatenate([ - 1 for _ in range(partitioned_result[p].get_shape().ndims - - 1) - ])) - partitioned_result[p] *= partitioned_weight[p] - partitioned_segment_ids = [] - for p in xrange(np): - if not ignore_weights: - # Partition segment_ids according to pindices. - p_segment_ids = array_ops.gather(segment_ids, pindices[p]) - # Number the p_segment_ids to meet segment_sum's requirements. Note - # that unique_p_segment_ids contains unique segment ids of this - # partition and these ids' order is unchanged. - unique_p_segment_ids, unique_p_segment_idx = array_ops.unique( - p_segment_ids) - partitioned_segment_ids.append(unique_p_segment_ids) - # segment_sum this partition's result. - with ops.colocate_with(params[p]): - partitioned_result[p] = math_ops.segment_sum( - partitioned_result[p], unique_p_segment_idx) - else: - # When ignore weights, we need to get indexs of elements in idx and - # segment_ids. - _, exclude_idx = array_ops.setdiff1d(idx, pindices[p]) - all_idx = math_ops.range(array_ops.shape(idx)[0]) - _, include_idx = array_ops.setdiff1d(all_idx, exclude_idx) - # Gather segment_ids and idx according to indexs. - p_segment_ids = array_ops.gather(segment_ids, include_idx) - p_idx = array_ops.gather(idx, include_idx) - # Number the p_segment_ids, same as ignore_weights case above. - unique_p_segment_ids, unique_p_segment_idx = array_ops.unique( - p_segment_ids) - _, unique_p_idx_idx = array_ops.unique(p_idx) - partitioned_segment_ids.append(unique_p_segment_ids) - with ops.colocate_with(params[p]): - partitioned_result[p] = math_ops.sparse_segment_sum( - partitioned_result[p], unique_p_idx_idx, unique_p_segment_idx) - # Concat each partition's segment_ids and result for final segment_sum. - concat_segment_ids = array_ops.concat(partitioned_segment_ids, 0) - concat_partitioned_result = array_ops.concat(partitioned_result, 0) - return math_ops.unsorted_segment_sum( - concat_partitioned_result, - concat_segment_ids, - math_ops.reduce_max(concat_segment_ids) + 1, - name=name) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py deleted file mode 100644 index 295c721fced..00000000000 --- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py +++ /dev/null @@ -1,799 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""embedding_ops tests.""" - -# pylint: disable=unused-import -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools -import math -import sys - -import numpy as np - -from tensorflow.contrib.layers.python.layers import embedding_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradient_checker -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import test -from tensorflow.python.util import compat - - -class SafeEmbeddingLookupSparseTest(test.TestCase): - - def _random_weights(self, vocab_size=4, embed_dim=4, num_shards=1): - assert vocab_size > 0 - assert embed_dim > 0 - assert num_shards > 0 - assert num_shards <= vocab_size - - initializer = init_ops.truncated_normal_initializer( - mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32) - embedding_weights = list(variable_scope.get_variable( - "embedding_weights", - shape=[vocab_size, embed_dim], - partitioner=partitioned_variables.fixed_size_partitioner(num_shards), - initializer=initializer)) - for w in embedding_weights: - w.initializer.run() - embedding_weights = [w.eval() for w in embedding_weights] - return embedding_weights - - def _ids_and_weights_2d(self): - # Each row demonstrates a test case: - # Row 0: multiple valid ids, 1 invalid id, weighted mean - # Row 1: all ids are invalid (leaving no valid ids after pruning) - # Row 2: no ids to begin with - # Row 3: single id - # Row 4: all ids have <=0 weight - indices = [[0, 0], [0, 1], [0, 2], [1, 0], [3, 0], [4, 0], [4, 1]] - ids = [0, 1, -1, -1, 2, 0, 1] - weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] - shape = [5, 4] - - sparse_ids = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(ids, dtypes.int64), - constant_op.constant(shape, dtypes.int64)) - - sparse_weights = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(weights, dtypes.float32), - constant_op.constant(shape, dtypes.int64)) - - return sparse_ids, sparse_weights - - def _ids_and_weights_3d(self): - # Each (2-D) index demonstrates a test case: - # Index 0, 0: multiple valid ids, 1 invalid id, weighted mean - # Index 0, 1: all ids are invalid (leaving no valid ids after pruning) - # Index 0, 2: no ids to begin with - # Index 1, 0: single id - # Index 1, 1: all ids have <=0 weight - # Index 1, 2: no ids to begin with - indices = [[0, 0, 0], [0, 0, 1], [0, 0, 2], [0, 1, 0], [1, 0, 0], [1, 1, 0], - [1, 1, 1]] - ids = [0, 1, -1, -1, 2, 0, 1] - weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] - shape = [2, 3, 4] - - sparse_ids = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(ids, dtypes.int64), - constant_op.constant(shape, dtypes.int64)) - - sparse_weights = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(weights, dtypes.float32), - constant_op.constant(shape, dtypes.int64)) - - return sparse_ids, sparse_weights - - def test_safe_embedding_lookup_sparse_return_zero_vector(self): - with self.cached_session(): - embedding_weights = self._random_weights() - sparse_ids, sparse_weights = self._ids_and_weights_2d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, sparse_weights).eval()) - - self.assertAllClose( - embedding_lookup_result, - [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / - 3.0, [0] * 4, [0] * 4, embedding_weights[0][2], [0] * 4]) - - def test_safe_embedding_lookup_sparse_return_special_vector(self): - with self.cached_session(): - embedding_weights = self._random_weights() - sparse_ids, sparse_weights = self._ids_and_weights_2d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) - - self.assertAllClose( - embedding_lookup_result, - [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / - 3.0, embedding_weights[0][3], embedding_weights[0][3], - embedding_weights[0][2], embedding_weights[0][3]]) - - def test_safe_embedding_lookup_sparse_no_weights(self): - with self.cached_session(): - embedding_weights = self._random_weights() - sparse_ids, _ = self._ids_and_weights_2d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, None).eval()) - - self.assertAllClose( - embedding_lookup_result, - [(embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, - [0] * 4, embedding_weights[0][2], ( - embedding_weights[0][0] + embedding_weights[0][1]) / 2.0]) - - def test_safe_embedding_lookup_sparse_partitioned(self): - with self.cached_session(): - embedding_weights = self._random_weights(num_shards=3) - sparse_ids, _ = self._ids_and_weights_2d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, None).eval()) - - embedding_weights = list(itertools.chain(*embedding_weights)) - self.assertAllClose(embedding_lookup_result, - [(embedding_weights[0] + embedding_weights[1]) / 2.0, - [0] * 4, [0] * 4, embedding_weights[2], - (embedding_weights[0] + embedding_weights[1]) / 2.0]) - - def test_safe_embedding_lookup_sparse_partitioned_inconsistent_weights(self): - with self.cached_session(): - embedding_weights = self._random_weights(num_shards=3) - sparse_ids, sparse_weights = self._ids_and_weights_2d() - - embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, - embedding_weights, sparse_ids) - embedding_weights = [ - constant_op.constant(w, dtype=dtypes.float64) - for w in embedding_weights - ] - self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, - embedding_weights, sparse_ids, sparse_weights) - - def test_safe_embedding_lookup_sparse_3d_return_zero_vector(self): - with self.cached_session(): - embedding_weights = self._random_weights() - sparse_ids, sparse_weights = self._ids_and_weights_3d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, sparse_weights).eval()) - - self.assertAllClose(embedding_lookup_result, [[ - (1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / 3.0, - [0] * 4, [0] * 4 - ], [embedding_weights[0][2], [0] * 4, [0] * 4]]) - - def test_safe_embedding_lookup_sparse_3d_return_special_vector(self): - with self.cached_session(): - embedding_weights = self._random_weights() - sparse_ids, sparse_weights = self._ids_and_weights_3d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) - - self.assertAllClose( - embedding_lookup_result, - [[(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / - 3.0, embedding_weights[0][3], embedding_weights[0][3]], [ - embedding_weights[0][2], embedding_weights[0][3], - embedding_weights[0][3] - ]]) - - def test_safe_embedding_lookup_sparse_3d_no_weights(self): - with self.cached_session(): - embedding_weights = self._random_weights() - sparse_ids, _ = self._ids_and_weights_3d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, None).eval()) - - self.assertAllClose(embedding_lookup_result, [[( - embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, [ - 0 - ] * 4], [ - embedding_weights[0][2], - (embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4 - ]]) - - def test_safe_embedding_lookup_sparse_3d_partitioned(self): - with self.cached_session(): - embedding_weights = self._random_weights(num_shards=3) - sparse_ids, _ = self._ids_and_weights_3d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, None).eval()) - - embedding_weights = list(itertools.chain(*embedding_weights)) - self.assertAllClose(embedding_lookup_result, [[ - (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4, [0] * 4 - ], [ - embedding_weights[2], - (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4 - ]]) - - def test_safe_embedding_lookup_sparse_3d_partitioned_inconsistent_weights( - self): - with self.cached_session(): - embedding_weights = self._random_weights(num_shards=3) - sparse_ids, sparse_weights = self._ids_and_weights_3d() - - embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, - embedding_weights, sparse_ids) - embedding_weights = [ - constant_op.constant(w, dtype=dtypes.float64) - for w in embedding_weights - ] - self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, - embedding_weights, sparse_ids, sparse_weights) - - -# pylint: disable=invalid-name -def local_variable_scope(): - """Create a variable scope named like the caller function.""" - return variable_scope.variable_scope(sys._getframe(1).f_code.co_name) -# pylint: enable=invalid-name - - -class ScatteredEmbeddingLookupTest(test.TestCase): - - def setUp(self): - random_seed.set_random_seed(1) - - def _random_weights(self, size=50, num_shards=1): - assert size > 0 - assert num_shards > 0 - assert num_shards <= size - - embedding_weights = list(variable_scope.get_variable( - "embedding_weights", - shape=[size], - partitioner=partitioned_variables.fixed_size_partitioner(num_shards), - initializer=init_ops.truncated_normal_initializer( - mean=0.0, stddev=1.0, dtype=dtypes.float32))) - for w in embedding_weights: - w.initializer.run() - return embedding_weights - - def test_scattered_embedding_consistency(self): - with self.cached_session(), local_variable_scope(): - embedding_weights = self._random_weights() - values = constant_op.constant(["foo", "foo"]) - - embedding_lookup_result = embedding_ops.scattered_embedding_lookup( - embedding_weights, values, dimension=10).eval() - - self.assertAllEqual(embedding_lookup_result.shape, [2, 10]) - self.assertAllEqual(embedding_lookup_result[0], - embedding_lookup_result[1]) - - def test_scattered_embedding_multiple_partition(self): - with self.cached_session(), local_variable_scope(): - embedding_weights = self._random_weights(num_shards=7) - values = constant_op.constant([4, 4, 5]) - - embedding_lookup_result = embedding_ops.scattered_embedding_lookup( - embedding_weights, values, dimension=5).eval() - - self.assertAllEqual(embedding_lookup_result.shape, [3, 5]) - self.assertAllEqual(embedding_lookup_result[0], - embedding_lookup_result[1]) - # Different embedding expected for different value. - embedding_diff = np.min( - (embedding_lookup_result[2] - embedding_lookup_result[0])**2) - self.assertGreater(embedding_diff, 0) - - def test_scattered_embedding_coverage(self): - with self.cached_session(), local_variable_scope(): - size = 8 - embedding_weights = self._random_weights(size=size, num_shards=3) - values = constant_op.constant(["foo"]) - - # Large embedding dimension to cover the full range of weights. - embedding_lookup_result = embedding_ops.scattered_embedding_lookup( - embedding_weights, values, dimension=100).eval() - - self.assertEqual(len(np.unique(embedding_lookup_result[0])), size) - - def test_scattered_embedding_multi_dimension(self): - with self.cached_session(), local_variable_scope(): - embedding_weights = self._random_weights() - values = constant_op.constant([["foo", "bar", "bar"], - ["bar", "bar", "foo"]]) - - embedding_lookup_result = embedding_ops.scattered_embedding_lookup( - embedding_weights, values, dimension=10).eval() - - self.assertAllEqual(embedding_lookup_result.shape, [2, 3, 10]) - self.assertAllEqual(embedding_lookup_result[0][0], - embedding_lookup_result[1][2]) - - def test_scattered_embedding_lookup_sparse(self): - with self.cached_session(), local_variable_scope(): - embedding_weights = self._random_weights(num_shards=3) - sparse_tensor = sparse_tensor_lib.SparseTensor( - values=["foo", "bar", "foo", "bar"], - indices=[[0, 0], [1, 0], [1, 1], [3, 0]], - dense_shape=[5, 2]) - - embedding_lookup_result = ( - embedding_ops.scattered_embedding_lookup_sparse( - embedding_weights, sparse_tensor, dimension=5, - combiner="mean").eval()) - - self.assertAllEqual(embedding_lookup_result.shape, [5, 5]) - # Same non-zero embedding for the empty rows filled with a default value. - self.assertAllEqual(embedding_lookup_result[2], - embedding_lookup_result[4]) - embedding_norm = np.sum(embedding_lookup_result[2]**2) - self.assertGreater(embedding_norm, 0) - - self.assertAllEqual(embedding_lookup_result[1], 0.5 * ( - embedding_lookup_result[0] + embedding_lookup_result[3])) - - def test_embedding_lookup_unique(self): - d_embed = 5 - n_embed = 10 - idx_shape = (2, 3, 4) - embeds = np.random.randn(n_embed, d_embed) - idx = np.random.randint(0, n_embed, idx_shape) - - with self.cached_session(), local_variable_scope(): - embedded_np = embeds[idx] - embedded_tf = embedding_ops.embedding_lookup_unique(embeds, idx).eval() - - self.assertEqual(embedded_np.shape, embedded_tf.shape) - np.testing.assert_almost_equal(embedded_np, embedded_tf) - - def test_embedding_lookup_unique_param3d(self): - embeds = np.random.randn(5, 3, 3) - idx = np.random.randint(0, 5, 10) - idx2d = np.random.randint(0, 5, (10, 2)) - - with self.cached_session(), local_variable_scope(): - embedded_np = embeds[idx] - embedded_np2d = embeds[idx2d] - embedded_tf = embedding_ops.embedding_lookup_unique(embeds, idx).eval() - embedded_tf_lst = embedding_ops.embedding_lookup_unique([embeds], - idx).eval() - embedded_tf2d = embedding_ops.embedding_lookup_unique(embeds, - idx2d).eval() - - self.assertEqual(embedded_np.shape, embedded_tf.shape) - np.testing.assert_almost_equal(embedded_np, embedded_tf) - self.assertEqual(embedded_np.shape, embedded_tf_lst.shape) - np.testing.assert_almost_equal(embedded_np, embedded_tf_lst) - self.assertEqual(embedded_np2d.shape, embedded_tf2d.shape) - np.testing.assert_almost_equal(embedded_np2d, embedded_tf2d) - - -class SampledScatteredEmbeddingLookupTest(test.TestCase): - - def setUp(self): - random_seed.set_random_seed(1) - self._hash_key = 1 - - def _random_weights(self, size=50, num_shards=1): - assert size > 0 - assert num_shards > 0 - assert num_shards <= size - - embedding_weights = list(variable_scope.get_variable( - "embedding_weights", - shape=[size], - partitioner=partitioned_variables.fixed_size_partitioner(num_shards), - initializer=init_ops.truncated_normal_initializer( - mean=0.0, stddev=1.0, dtype=dtypes.float32))) - for w in embedding_weights: - w.initializer.run() - return embedding_weights - - def test_hashed_embedding_consistency(self): - with self.cached_session(), local_variable_scope(): - embedding_weights = self._random_weights() - values = constant_op.constant(["foo", "foo"]) - # The first three sampled_candidates are equal, so the first three - # embedding weights will be equal. - sampled_candidates = constant_op.constant([[1, 3, 4, 6], [1, 3, 4, 7]]) - - embedding_lookup_result = ( # pylint: disable=protected-access - embedding_ops._sampled_scattered_embedding_lookup( - embedding_weights, - values, - sampled_candidates=sampled_candidates, - hash_key=self._hash_key).eval()) - - self.assertAllEqual(embedding_lookup_result.shape, [2, 4]) - self.assertAllEqual(embedding_lookup_result[0][:3], - embedding_lookup_result[1][:3]) - self.assertNotEqual(embedding_lookup_result[0][3], - embedding_lookup_result[1][3]) - - def test_hashed_embedding_multi_dimension(self): - with self.cached_session(), local_variable_scope(): - embedding_weights = self._random_weights() - values = constant_op.constant([["foo", "bar", "bar"], - ["bar", "bar", "foo"]]) - sampled_candidates = constant_op.constant( - [[[1, 3, 4, 6], [1, 7, 8, 9], [1, 7, 8, 9]], - [[1, 7, 8, 9], [1, 7, 8, 9], [1, 3, 4, 6]]]) - - embedding_lookup_result = ( # pylint: disable=protected-access - embedding_ops._sampled_scattered_embedding_lookup( - embedding_weights, - values, - sampled_candidates=sampled_candidates, - hash_key=self._hash_key).eval()) - - self.assertAllEqual(embedding_lookup_result.shape, [2, 3, 4]) - self.assertAllEqual(embedding_lookup_result[0][0], - embedding_lookup_result[1][2]) - - invalid_indices = constant_op.constant([[[1, 3, 4, 6], [1, 7, 8, 9]], - [[1, 7, 8, 9], [1, 7, 8, 9]]]) - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, ( - r"\[The shape of sampled_candidates: \] \[2 2 4\] " - r"\[ does not match the shape of values: \] \[2 3\]")): - # pylint: disable=protected-access - embedding_ops._sampled_scattered_embedding_lookup( - embedding_weights, values, - sampled_candidates=invalid_indices).eval() - - -class SampledScatteredEmbeddingLookupSparseTest(test.TestCase): - - def setUp(self): - random_seed.set_random_seed(1) - self._hash_key = 1 - - def test_output_shape(self): - """Verifies the shape of the output tensor.""" - with self.cached_session(): - sp_values = sparse_tensor_lib.SparseTensor( - values=["a", "a", "b", "c", "d", "e", "f"], - indices=[[1, 0], [2, 0], [2, 1], [2, 2], [2, 3], [2, 4], [2, 5]], - dense_shape=[3, 6]) - params = constant_op.constant([.1, .2, .3]) - - result = embedding_ops._sampled_scattered_embedding_lookup_sparse( - params, sp_values, dimension=4, hash_key=self._hash_key) - - self.assertEqual(result.eval().shape, (3, 4)) - - def test_output_values(self): - """Verifies the values in a trivial case.""" - with self.cached_session(): - sp_values = sparse_tensor_lib.SparseTensor( - values=["a"], indices=[[1, 0]], dense_shape=[3, 1]) - params = constant_op.constant([.1, .2, .3]) - - result = embedding_ops._sampled_scattered_embedding_lookup_sparse( - params, sp_values, dimension=5, hash_key=self._hash_key) - - self.assertAllClose(result.eval(), [[0., 0., 0., 0., - 0.], [.3, .2, .2, .3, .1], - [0., 0., 0., 0., 0.]]) - - def test_output_values_with_sampled_candidates(self): - """Verifies the values for given sampled_candidates.""" - with self.cached_session(): - sp_values = sparse_tensor_lib.SparseTensor( - values=["a", "a", "b", "c", "d", "e", "f"], - indices=[[1, 0], [2, 0], [2, 1], [2, 2], [2, 3], [2, 4], [2, 5]], - dense_shape=[3, 6]) - params = constant_op.constant([.1, .2, .3]) - - sampled_candidates = [[1, 0], [2, 1], [3, 2]] - sampled_result = embedding_ops._sampled_scattered_embedding_lookup_sparse( - params, - sp_values, - sampled_candidates=constant_op.constant(sampled_candidates), - hash_key=self._hash_key) - full_result = embedding_ops._sampled_scattered_embedding_lookup_sparse( - params, sp_values, dimension=4, hash_key=self._hash_key) - - sampled_result_val = sampled_result.eval() - full_result_val = full_result.eval() - self.assertEqual(sampled_result_val.shape, (3, 2)) - for i in range(len(sampled_candidates)): - self.assertAllClose(sampled_result_val[i], - full_result_val[i, sampled_candidates[i]]) - - def test_output_values_with_sign_hash(self): - """Verifies the values in a trivial case with hash_signs=True.""" - with self.cached_session(): - sp_values = sparse_tensor_lib.SparseTensor( - values=["a"], indices=[[1, 0]], dense_shape=[3, 1]) - params = constant_op.constant([.1, .1, .1]) - - result = embedding_ops._sampled_scattered_embedding_lookup_sparse( - params, - sp_values, - dimension=4, - with_sign_hash=True, - hash_key=self._hash_key) - - self.assertAllClose(result.eval(), [[0., 0., 0., 0.], [-.1, -.1, -.1, .1], - [0., 0., 0., 0.]]) - - def test_distributive_property(self): - """Verifies the distributive property of matrix multiplication.""" - with self.cached_session(): - params = constant_op.constant([.1, .2, .3]) - sp_values_a = sparse_tensor_lib.SparseTensor( - values=["a"], indices=[[0, 0]], dense_shape=[3, 1]) - sp_values_b = sparse_tensor_lib.SparseTensor( - values=["b"], indices=[[2, 0]], dense_shape=[3, 1]) - sp_values_c = sparse_tensor_lib.SparseTensor( - values=["c"], indices=[[2, 0]], dense_shape=[3, 1]) - sp_values = sparse_tensor_lib.SparseTensor( - values=["a", "b", "c"], - indices=[[0, 0], [2, 0], [2, 1]], - dense_shape=[3, 2]) - - result_a = embedding_ops._sampled_scattered_embedding_lookup_sparse( - params, sp_values_a, dimension=4, hash_key=self._hash_key) - result_b = embedding_ops._sampled_scattered_embedding_lookup_sparse( - params, sp_values_b, dimension=4, hash_key=self._hash_key) - result_c = embedding_ops._sampled_scattered_embedding_lookup_sparse( - params, sp_values_c, dimension=4, hash_key=self._hash_key) - result = embedding_ops._sampled_scattered_embedding_lookup_sparse( - params, sp_values, dimension=4, hash_key=self._hash_key) - - result_abc = math_ops.add_n([result_a, result_b, result_c]) - self.assertAllClose(result.eval(), result_abc.eval()) - - -def _PName(param_id): - return "p" + str(param_id) - - -def _EmbeddingParams(num_shards, - vocab_size, - dtype=dtypes.float32, - shape=None, - use_shapeless_placeholder=False): - p = [] - params = {} - feed_dict = {} - if not shape: - shape = [10] - for i in range(num_shards): - shard_shape = [vocab_size // num_shards] + shape - if i < vocab_size % num_shards: # Excess goes evenly on the first shards - shard_shape[0] += 1 - - param_name = _PName(i) - - if use_shapeless_placeholder: - param = array_ops.placeholder(dtype, shape=None, name=param_name) - else: - param = constant_op.constant( - 1.0, shape=shard_shape, dtype=dtype, name=param_name) - p.append(param) - np_type = "f" if dtype == dtypes.float32 else "d" - val = (np.random.rand(*shard_shape).astype(np_type)) + 1 - params[param_name + ":0"] = val - feed_dict[param.name] = val - return p, params, feed_dict - - -def _EmbeddingResult(params, - id_vals, - num_shards, - vocab_size, - partition_strategy="mod", - weight_vals=None): - if weight_vals is None: - weight_vals = np.copy(id_vals) - weight_vals.fill(1) - values = [] - weights = [] - weights_squared = [] - for ids, wts in zip(id_vals, weight_vals): - value_aggregation = None - weight_aggregation = None - squared_weight_aggregation = None - if isinstance(ids, compat.integral_types): - ids = [ids] - wts = [wts] - for i, weight_value in zip(ids, wts): - if partition_strategy == "mod": - val = np.copy(params[_PName(i % num_shards) + ":0"][ - i // num_shards, :]) * weight_value - elif partition_strategy == "div": - ids_per_partition, extras = divmod(vocab_size, num_shards) - threshold = extras * (ids_per_partition + 1) - if i < threshold: - partition = i // (ids_per_partition + 1) - offset = i % (ids_per_partition + 1) - else: - partition = extras + (i - threshold) // ids_per_partition - offset = (i - threshold) % ids_per_partition - val = np.copy( - params[_PName(partition) + ":0"][offset, :]) * weight_value - else: - assert False - if value_aggregation is None: - assert weight_aggregation is None - assert squared_weight_aggregation is None - value_aggregation = val - weight_aggregation = weight_value - squared_weight_aggregation = weight_value * weight_value - else: - assert weight_aggregation is not None - assert squared_weight_aggregation is not None - value_aggregation += val - weight_aggregation += weight_value - squared_weight_aggregation += weight_value * weight_value - values.append(value_aggregation) - weights.append(weight_aggregation) - weights_squared.append(squared_weight_aggregation) - values = np.array(values).astype(np.float32) - weights = np.array(weights).astype(np.float32) - weights_squared = np.array(weights_squared).astype(np.float32) - return values, weights, weights_squared - - -class EmbeddingLookupSparseWithDistributedAggregationTest(test.TestCase): - - def _RandomIdsAndWeights(self, batch_size, vocab_size): - max_val_per_entry = 6 - vals_per_batch_entry = np.random.randint( - 1, max_val_per_entry, size=batch_size) - num_vals = np.sum(vals_per_batch_entry) - - ids = np.random.randint(vocab_size, size=num_vals) - weights = 1 + np.random.rand(num_vals) - - indices = [] - for batch_entry, num_val in enumerate(vals_per_batch_entry): - for val_index in range(num_val): - indices.append([batch_entry, val_index]) - - shape = [batch_size, max_val_per_entry] - - sp_ids = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(ids, dtypes.int32), - constant_op.constant(shape, dtypes.int64)) - sp_weights = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(weights, dtypes.float32), - constant_op.constant(shape, dtypes.int64)) - - return sp_ids, sp_weights, ids, weights, vals_per_batch_entry - - def _GroupByBatchEntry(self, vals, vals_per_batch_entry): - grouped_vals = [] - index = 0 - for num_val in vals_per_batch_entry: - grouped_vals.append(list(vals[index:(index + num_val)])) - index += num_val - return grouped_vals - - def testEmbeddingLookupSparse(self): - vocab_size = 13 - batch_size = 10 - param_shape = [2, 5] - expected_lookup_result_shape = param_shape - - sp_ids, sp_weights, ids, weights, vals_per_batch_entry = ( - self._RandomIdsAndWeights(batch_size, vocab_size)) - - grouped_ids = self._GroupByBatchEntry(ids, vals_per_batch_entry) - grouped_weights = self._GroupByBatchEntry(weights, vals_per_batch_entry) - grouped_ignored_weights = self._GroupByBatchEntry( - np.ones(np.sum(vals_per_batch_entry)), vals_per_batch_entry) - - for num_shards, combiner, dtype, ignore_weights in itertools.product( - [1, 5], ["sum", "mean", "sqrtn"], [dtypes.float32, - dtypes.float64], [True, False]): - - with self.cached_session(): - p, params, feed_dict = _EmbeddingParams( - num_shards, vocab_size, shape=param_shape, dtype=dtype) - embedding_sum = \ - embedding_ops.embedding_lookup_sparse_with_distributed_aggregation( - p, - sp_ids, - None if ignore_weights else sp_weights, - combiner=combiner) - - self.assertEqual(embedding_sum.get_shape().as_list()[1:], - expected_lookup_result_shape) - - tf_embedding_sum = embedding_sum.eval(feed_dict=feed_dict) - - np_embedding_sum, np_weight_sum, np_weight_sq_sum = _EmbeddingResult( - params, - grouped_ids, - num_shards, - vocab_size, - weight_vals=grouped_ignored_weights - if ignore_weights else grouped_weights) - if combiner == "mean": - np_embedding_sum /= np.reshape(np_weight_sum, (batch_size, 1, 1)) - if combiner == "sqrtn": - np_embedding_sum /= np.reshape( - np.sqrt(np_weight_sq_sum), (batch_size, 1, 1)) - self.assertAllClose(np_embedding_sum, tf_embedding_sum) - - def testGradientsEmbeddingLookupSparse(self): - vocab_size = 12 - batch_size = 4 - param_shape = [2, 3] - sp_ids, sp_weights, _, _, _ = (self._RandomIdsAndWeights( - batch_size, vocab_size)) - - for num_shards, combiner, dtype, ignore_weights in itertools.product( - [1, 3], ["sum", "mean", "sqrtn"], [dtypes.float32, - dtypes.float64], [True, False]): - with self.cached_session(): - x, params, _ = _EmbeddingParams( - num_shards, vocab_size, shape=param_shape, dtype=dtype) - - y = embedding_ops.embedding_lookup_sparse_with_distributed_aggregation( - x, - sp_ids, - None if ignore_weights else sp_weights, - combiner=combiner) - x_name = [_PName(i) for i in range(num_shards)] - x_init_value = [params[x_n + ":0"] for x_n in x_name] - x_shape = [i.shape for i in x_init_value] - y_shape = [batch_size] + list(params[_PName(0) + ":0"].shape[1:]) - err = gradient_checker.compute_gradient_error( - x, x_shape, y, y_shape, x_init_value=x_init_value) - self.assertLess(err, 1e-5 if dtype == dtypes.float64 else 2e-3) - - def testIncompatibleShapes(self): - with self.cached_session(): - x, _, _ = _EmbeddingParams(1, 10, dtype=dtypes.float32) - sp_ids = sparse_tensor_lib.SparseTensor( - constant_op.constant([[0, 0], [0, 1], [1, 0]], dtypes.int64), - constant_op.constant([0, 1, 2], dtypes.int32), - constant_op.constant([2, 2], dtypes.int64)) - sp_weights = sparse_tensor_lib.SparseTensor( - constant_op.constant([[0, 0], [0, 1]], dtypes.int64), - constant_op.constant([12.0, 5.0], dtypes.float32), - constant_op.constant([1, 2], dtypes.int64)) - - with self.assertRaises(ValueError): - embedding_ops.embedding_lookup_sparse_with_distributed_aggregation( - x, sp_ids, sp_weights, combiner="mean") - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/layers/python/layers/encoders.py b/tensorflow/contrib/layers/python/layers/encoders.py deleted file mode 100644 index 3671633c8d7..00000000000 --- a/tensorflow/contrib/layers/python/layers/encoders.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Encoders to transform sequence of symbols into vector representation.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework.python.ops import variables -from tensorflow.contrib.layers.python.layers import embedding_ops as contrib_embedding_ops -from tensorflow.contrib.layers.python.ops import sparse_ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variable_scope - -__all__ = ['bow_encoder', 'embed_sequence'] - - -def bow_encoder(ids, - vocab_size, - embed_dim, - sparse_lookup=True, - initializer=None, - regularizer=None, - trainable=True, - scope=None, - reuse=None): - """Maps a sequence of symbols to a vector per example by averaging embeddings. - - Args: - ids: `[batch_size, doc_length]` `Tensor` or `SparseTensor` of type - `int32` or `int64` with symbol ids. - vocab_size: Integer number of symbols in vocabulary. - embed_dim: Integer number of dimensions for embedding matrix. - sparse_lookup: `bool`, if `True`, converts ids to a `SparseTensor` - and performs a sparse embedding lookup. This is usually faster, - but not desirable if padding tokens should have an embedding. Empty rows - are assigned a special embedding. - initializer: An initializer for the embeddings, if `None` default for - current scope is used. - regularizer: Optional regularizer for the embeddings. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). - scope: Optional string specifying the variable scope for the op, required - if `reuse=True`. - reuse: If `True`, variables inside the op will be reused. - - Returns: - Encoding `Tensor` `[batch_size, embed_dim]` produced by - averaging embeddings. - - Raises: - ValueError: If `embed_dim` or `vocab_size` are not specified. - """ - if not vocab_size or not embed_dim: - raise ValueError('Must specify vocab size and embedding dimension') - with variable_scope.variable_scope( - scope, 'bow_encoder', [ids], reuse=reuse): - embeddings = variables.model_variable( - 'embeddings', shape=[vocab_size, embed_dim], - initializer=initializer, regularizer=regularizer, - trainable=trainable) - if sparse_lookup: - if isinstance(ids, sparse_tensor.SparseTensor): - sparse_ids = ids - else: - sparse_ids = sparse_ops.dense_to_sparse_tensor(ids) - return contrib_embedding_ops.safe_embedding_lookup_sparse( - [embeddings], sparse_ids, combiner='mean', default_id=0) - else: - if isinstance(ids, sparse_tensor.SparseTensor): - raise TypeError('ids are expected to be dense Tensor, got: %s', ids) - return math_ops.reduce_mean( - embedding_ops.embedding_lookup(embeddings, ids), axis=1) - - -def embed_sequence(ids, - vocab_size=None, - embed_dim=None, - unique=False, - initializer=None, - regularizer=None, - trainable=True, - scope=None, - reuse=None): - """Maps a sequence of symbols to a sequence of embeddings. - - Typical use case would be reusing embeddings between an encoder and decoder. - - Args: - ids: `[batch_size, doc_length]` `Tensor` of type `int32` or `int64` - with symbol ids. - vocab_size: Integer number of symbols in vocabulary. - embed_dim: Integer number of dimensions for embedding matrix. - unique: If `True`, will first compute the unique set of indices, and then - lookup each embedding once, repeating them in the output as needed. - initializer: An initializer for the embeddings, if `None` default for - current scope is used. - regularizer: Optional regularizer for the embeddings. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - scope: Optional string specifying the variable scope for the op, required - if `reuse=True`. - reuse: If `True`, variables inside the op will be reused. - - Returns: - `Tensor` of `[batch_size, doc_length, embed_dim]` with embedded sequences. - - Raises: - ValueError: if `embed_dim` or `vocab_size` are not specified when - `reuse` is `None` or `False`. - """ - if not (reuse or (vocab_size and embed_dim)): - raise ValueError('Must specify vocab size and embedding dimension when not ' - 'reusing. Got vocab_size=%s and embed_dim=%s' % ( - vocab_size, embed_dim)) - with variable_scope.variable_scope( - scope, 'EmbedSequence', [ids], reuse=reuse): - shape = [vocab_size, embed_dim] - if reuse and vocab_size is None or embed_dim is None: - shape = None - embeddings = variables.model_variable( - 'embeddings', shape=shape, - initializer=initializer, regularizer=regularizer, - trainable=trainable) - if unique: - return contrib_embedding_ops.embedding_lookup_unique(embeddings, ids) - return embedding_ops.embedding_lookup(embeddings, ids) diff --git a/tensorflow/contrib/layers/python/layers/encoders_test.py b/tensorflow/contrib/layers/python/layers/encoders_test.py deleted file mode 100644 index 1a2aa710d5d..00000000000 --- a/tensorflow/contrib/layers/python/layers/encoders_test.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.contrib.layers.python.layers.encoders.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.layers.python.layers import encoders -from tensorflow.contrib.layers.python.ops import sparse_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def _get_const_var(name, shape, value): - return variable_scope.get_variable( - name, shape, initializer=init_ops.constant_initializer(value)) - - -class EncodersTest(test.TestCase): - - def testBowEncoderSparse(self): - with self.cached_session() as sess: - docs = [[0, 1], [2, 3]] - enc = encoders.bow_encoder(docs, 4, 3) - sess.run(variables.global_variables_initializer()) - self.assertAllEqual([2, 3], enc.eval().shape) - - def testBowEncoderSparseTensor(self): - with self.cached_session() as sess: - docs = [[0, 1], [2, 3]] - sparse_docs = sparse_ops.dense_to_sparse_tensor(docs) - enc = encoders.bow_encoder(sparse_docs, 4, 3) - sess.run(variables.global_variables_initializer()) - self.assertAllEqual([2, 3], enc.eval().shape) - - def testBowEncoderSparseEmptyRow(self): - with self.cached_session() as sess: - docs = [[0, 1], [2, 3], [0, 0]] - enc = encoders.bow_encoder(docs, 4, 5) - sess.run(variables.global_variables_initializer()) - self.assertAllEqual([3, 5], enc.eval().shape) - - def testBowEncoderDense(self): - with self.cached_session() as sess: - docs = [[0, 1], [2, 3], [0, 0], [0, 0]] - enc = encoders.bow_encoder(docs, 4, 3, sparse_lookup=False) - sess.run(variables.global_variables_initializer()) - self.assertAllEqual([4, 3], enc.eval().shape) - - def testBowEncoderSparseTensorDenseLookup(self): - with self.cached_session(): - docs = [[0, 1]] - sparse_docs = sparse_ops.dense_to_sparse_tensor(docs) - with self.assertRaises(TypeError): - encoders.bow_encoder(sparse_docs, 4, 3, sparse_lookup=False) - - def testBowEncodersSharingEmbeddings(self): - with self.cached_session() as sess: - docs = [[0, 1], [2, 3]] - enc_1 = encoders.bow_encoder(docs, 4, 3, scope='test') - enc_2 = encoders.bow_encoder(docs, 4, 3, scope='test', reuse=True) - sess.run(variables.global_variables_initializer()) - avg_1, avg_2 = sess.run([enc_1, enc_2]) - self.assertAllEqual(avg_1, avg_2) - - def testBowEncodersSharingEmbeddingsInheritedScopes(self): - with self.cached_session() as sess: - docs = [[0, 1], [2, 3]] - with variable_scope.variable_scope('test'): - enc_1 = encoders.bow_encoder(docs, 4, 3) - with variable_scope.variable_scope('test', reuse=True): - enc_2 = encoders.bow_encoder(docs, 4, 3) - sess.run(variables.global_variables_initializer()) - avg_1, avg_2 = sess.run([enc_1, enc_2]) - self.assertAllEqual(avg_1, avg_2) - - def testBowEncodersSharingEmbeddingsSharedScope(self): - with self.cached_session() as sess: - docs = [[0, 1], [2, 3]] - enc_1 = encoders.bow_encoder(docs, 4, 3, scope='bow') - variable_scope.get_variable_scope().reuse_variables() - enc_2 = encoders.bow_encoder(docs, 4, 3, scope='bow') - sess.run(variables.global_variables_initializer()) - avg_1, avg_2 = sess.run([enc_1, enc_2]) - self.assertAllEqual(avg_1, avg_2) - - def testBowEncoderReuseEmbeddingsVariable(self): - with self.cached_session() as sess: - docs = [[1, 1], [2, 3]] - with variable_scope.variable_scope('test'): - v = _get_const_var('embeddings', (4, 3), - [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]]) - self.assertEqual(v.name, 'test/embeddings:0') - enc = encoders.bow_encoder(docs, 4, 3, scope='test', reuse=True) - sess.run(variables.global_variables_initializer()) - self.assertAllClose([[3., 4., 5.], [7.5, 8.5, 9.5]], enc.eval()) - - def testEmbedSequence(self): - with self.cached_session() as sess: - docs = [[1, 1], [2, 3]] - with variable_scope.variable_scope('test'): - v = _get_const_var('embeddings', (4, 3), - [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]]) - self.assertEqual(v.name, 'test/embeddings:0') - emb = encoders.embed_sequence(docs, 4, 3, scope='test', reuse=True) - sess.run(variables.global_variables_initializer()) - self.assertAllClose( - [[[3., 4., 5.], [3., 4., 5.]], [[6., 7., 8.], [9., 10., 11.]]], - emb.eval()) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/layers/python/layers/feature_column.py b/tensorflow/contrib/layers/python/layers/feature_column.py deleted file mode 100644 index 385dcc0d80a..00000000000 --- a/tensorflow/contrib/layers/python/layers/feature_column.py +++ /dev/null @@ -1,2724 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""This API defines FeatureColumn abstraction. - -FeatureColumns provide a high level abstraction for ingesting and representing -features in `Estimator` models. - -FeatureColumns are the primary way of encoding features for pre-canned -`Estimator` models. - -When using FeatureColumns with `Estimator` models, the type of feature column -you should choose depends on (1) the feature type and (2) the model type. - -(1) Feature type: - - * Continuous features can be represented by `real_valued_column`. - * Categorical features can be represented by any `sparse_column_with_*` - column (`sparse_column_with_keys`, `sparse_column_with_vocabulary_file`, - `sparse_column_with_hash_bucket`, `sparse_column_with_integerized_feature`). - -(2) Model type: - - * Deep neural network models (`DNNClassifier`, `DNNRegressor`). - - Continuous features can be directly fed into deep neural network models. - - age_column = real_valued_column("age") - - To feed sparse features into DNN models, wrap the column with - `embedding_column` or `one_hot_column`. `one_hot_column` will create a dense - boolean tensor with an entry for each possible value, and thus the - computation cost is linear in the number of possible values versus the number - of values that occur in the sparse tensor. Thus using a "one_hot_column" is - only recommended for features with only a few possible values. For features - with many possible values or for very sparse features, `embedding_column` is - recommended. - - embedded_dept_column = embedding_column( - sparse_column_with_keys("department", ["math", "philosophy", ...]), - dimension=10) - -* Wide (aka linear) models (`LinearClassifier`, `LinearRegressor`). - - Sparse features can be fed directly into linear models. When doing so - an embedding_lookups are used to efficiently perform the sparse matrix - multiplication. - - dept_column = sparse_column_with_keys("department", - ["math", "philosophy", "english"]) - - It is recommended that continuous features be bucketized before being - fed into linear models. - - bucketized_age_column = bucketized_column( - source_column=age_column, - boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65]) - - Sparse features can be crossed (also known as conjuncted or combined) in - order to form non-linearities, and then fed into linear models. - - cross_dept_age_column = crossed_column( - columns=[department_column, bucketized_age_column], - hash_bucket_size=1000) - -Example of building an `Estimator` model using FeatureColumns: - - # Define features and transformations - deep_feature_columns = [age_column, embedded_dept_column] - wide_feature_columns = [dept_column, bucketized_age_column, - cross_dept_age_column] - - # Build deep model - estimator = DNNClassifier( - feature_columns=deep_feature_columns, - hidden_units=[500, 250, 50]) - estimator.train(...) - - # Or build a wide model - estimator = LinearClassifier( - feature_columns=wide_feature_columns) - estimator.train(...) - - # Or build a wide and deep model! - estimator = DNNLinearCombinedClassifier( - linear_feature_columns=wide_feature_columns, - dnn_feature_columns=deep_feature_columns, - dnn_hidden_units=[500, 250, 50]) - estimator.train(...) - - -FeatureColumns can also be transformed into a generic input layer for -custom models using `input_from_feature_columns` within -`feature_column_ops.py`. - -Example of building a non-`Estimator` model using FeatureColumns: - - # Building model via layers - - deep_feature_columns = [age_column, embedded_dept_column] - columns_to_tensor = parse_feature_columns_from_examples( - serialized=my_data, - feature_columns=deep_feature_columns) - first_layer = input_from_feature_columns( - columns_to_tensors=columns_to_tensor, - feature_columns=deep_feature_columns) - second_layer = fully_connected(first_layer, ...) - -See feature_column_ops_test for more examples. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc -import collections -import math - -import six - -from tensorflow.contrib import lookup -from tensorflow.contrib.framework.python.framework import checkpoint_utils -from tensorflow.contrib.framework.python.framework import experimental -from tensorflow.contrib.framework.python.ops import variables as contrib_variables -from tensorflow.contrib.layers.python.layers import embedding_ops -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.layers.python.ops import bucketization_op -from tensorflow.contrib.layers.python.ops import sparse_feature_cross_op -from tensorflow.contrib.layers.python.ops import sparse_ops as contrib_sparse_ops -from tensorflow.python.feature_column import feature_column as fc_core -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor as sparse_tensor_py -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import string_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util import deprecation -from tensorflow.python.util import nest -from tensorflow.python.util.compat import collections_abc - -# Imports the core `InputLayer` symbol in contrib during development. -InputLayer = fc_core.InputLayer # pylint: disable=invalid-name - - -class _LinearEmbeddingLookupArguments( - collections.namedtuple("_LinearEmbeddingLookupArguments", [ - "input_tensor", "weight_tensor", "vocab_size", "initializer", "combiner" - ])): - """Represents the information needed from a column for embedding lookup. - - Used to compute DNN inputs and weighted sum. - """ - pass - - -class _DeepEmbeddingLookupArguments( - collections.namedtuple("_DeepEmbeddingLookupArguments", [ - "input_tensor", "weight_tensor", "vocab_size", "initializer", - "combiner", "dimension", "shared_embedding_name", "hash_key", - "max_norm", "trainable" - ])): - """Represents the information needed from a column for embedding lookup. - - Used to compute DNN inputs and weighted sum. - """ - pass - - -@six.add_metaclass(abc.ABCMeta) -class _FeatureColumn(object): - """Represents a feature column abstraction. - - To distinguish the concept of a feature family and a specific binary feature - within a family, we refer to a feature family like "country" as a feature - column. For example "country:US" is a feature which is in "country" feature - column and has a feature value ("US"). - This class is an abstract class. User should not create one instance of this. - Following classes (_SparseColumn, _RealValuedColumn, ...) are concrete - instances. - """ - - @abc.abstractproperty - @deprecation.deprecated("2016-09-25", "Should be private.") - def name(self): - """Returns the name of column or transformed column.""" - pass - - @abc.abstractproperty - @deprecation.deprecated("2016-09-25", "Should be private.") - def config(self): - """Returns configuration of the base feature for `tf.io.parse_example`.""" - pass - - @abc.abstractproperty - @deprecation.deprecated("2016-09-25", "Should be private.") - def key(self): - """Returns a string which will be used as a key when we do sorting.""" - pass - - @abc.abstractmethod - @deprecation.deprecated("2016-09-25", "Should be private.") - def insert_transformed_feature(self, columns_to_tensors): - """Apply transformation and inserts it into columns_to_tensors. - - Args: - columns_to_tensors: A mapping from feature columns to tensors. 'string' - key means a base feature (not-transformed). It can have _FeatureColumn - as a key too. That means that _FeatureColumn is already transformed. - """ - raise NotImplementedError( - "Transform is not implemented for {}.".format(self)) - - # pylint: disable=unused-argument - def _to_dnn_input_layer(self, - input_tensor, - weight_collection=None, - trainable=True, - output_rank=2): - """Returns a Tensor as an input to the first layer of neural network.""" - raise ValueError("Calling an abstract method.") - - def _deep_embedding_lookup_arguments(self, input_tensor): - """Returns arguments to embedding lookup to build an input layer.""" - raise NotImplementedError( - "No deep embedding lookup arguments for column {}.".format(self)) - - # It is expected that classes implement either wide_embedding_lookup_arguments - # or to_dense_tensor to be used in linear models. - # pylint: disable=unused-argument - def _wide_embedding_lookup_arguments(self, input_tensor): - """Returns arguments to look up embeddings for this column.""" - raise NotImplementedError( - "No wide embedding lookup arguments for column {}.".format(self)) - - # pylint: disable=unused-argument - def _to_dense_tensor(self, input_tensor): - """Returns a dense tensor representing this column's values.""" - raise NotImplementedError( - "No dense tensor representation for column {}.".format(self)) - - def _checkpoint_path(self): - """Returns None, or a (path,tensor_name) to load a checkpoint from.""" - return None - - def _key_without_properties(self, properties): - """Helper method for self.key() that omits particular properties.""" - fields_values = [] - # pylint: disable=protected-access - for i, k in enumerate(self._fields): - if k in properties: - # Excludes a property from the key. - # For instance, exclude `initializer` from the key of EmbeddingColumn - # since we don't support users specifying different initializers for - # the same embedding column. Ditto for `normalizer` and - # RealValuedColumn. - # Special treatment is needed since the default str form of a - # function contains its address, which could introduce non-determinism - # in sorting. - continue - fields_values.append("{}={}".format(k, self[i])) - # pylint: enable=protected-access - - # This is effectively the same format as str(self), except with our special - # treatment. - return "{}({})".format(type(self).__name__, ", ".join(fields_values)) - - -# TODO(b/30410315): Support warm starting in all feature columns. -class _SparseColumn( - _FeatureColumn, - fc_core._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple("_SparseColumn", [ - "column_name", "is_integerized", "bucket_size", "lookup_config", - "combiner", "dtype" - ])): - """Represents a sparse feature column also known as categorical features. - - Instances of this class are immutable. A sparse column means features are - sparse and dictionary returned by InputBuilder contains a - ("column_name", SparseTensor) pair. - One and only one of bucket_size or lookup_config should be set. If - is_integerized is True then bucket_size should be set. - - Attributes: - column_name: A string defining sparse column name. - is_integerized: A bool if True means type of feature is an integer. - Integerized means we can use the feature itself as id. - bucket_size: An int that is > 0. The number of buckets. - lookup_config: A _SparseIdLookupConfig defining feature-to-id lookup - configuration - combiner: A string specifying how to reduce if the sparse column is - multivalent. Currently "mean", "sqrtn" and "sum" are supported, with "sum" - the default. "sqrtn" often achieves good accuracy, in particular with - bag-of-words columns. - * "sum": do not normalize features in the column - * "mean": do l1 normalization on features in the column - * "sqrtn": do l2 normalization on features in the column - For more information: `tf.embedding_lookup_sparse`. - dtype: Type of features, either `tf.string` or `tf.int64`. - - Raises: - TypeError: if lookup_config is not a _SparseIdLookupConfig. - ValueError: if above expectations about input fails. - """ - - def __new__(cls, - column_name, - is_integerized=False, - bucket_size=None, - lookup_config=None, - combiner="sum", - dtype=dtypes.string): - if is_integerized and bucket_size is None: - raise ValueError("bucket_size must be set if is_integerized is True. " - "column_name: {}".format(column_name)) - - if is_integerized and not dtype.is_integer: - raise ValueError("dtype must be an integer if is_integerized is True. " - "dtype: {}, column_name: {}.".format(dtype, column_name)) - if dtype != dtypes.string and not dtype.is_integer: - raise ValueError("dtype must be string or integer. " - "dtype: {}, column_name: {}".format(dtype, column_name)) - - if bucket_size is None and lookup_config is None: - raise ValueError("one of bucket_size or lookup_config must be set. " - "column_name: {}".format(column_name)) - - if bucket_size is not None and lookup_config: - raise ValueError("one and only one of bucket_size or lookup_config " - "must be set. column_name: {}".format(column_name)) - - if bucket_size is not None and bucket_size < 1: - raise ValueError("bucket_size must be at least 1. " - "bucket_size: {}, column_name: {}".format( - bucket_size, column_name)) - - if ((lookup_config) and - (not isinstance(lookup_config, _SparseIdLookupConfig))): - raise TypeError( - "lookup_config must be an instance of _SparseIdLookupConfig. " - "Given one is in type {} for column_name {}".format( - type(lookup_config), column_name)) - - if (lookup_config and lookup_config.vocabulary_file and - lookup_config.vocab_size is None): - raise ValueError("vocab_size must be defined. " - "column_name: {}".format(column_name)) - - return super(_SparseColumn, cls).__new__( - cls, - column_name, - is_integerized=is_integerized, - bucket_size=bucket_size, - lookup_config=lookup_config, - combiner=combiner, - dtype=dtype) - - @property - def name(self): - return self.column_name - - @property - def length(self): - """Returns vocabulary or hash_bucket size.""" - if self.bucket_size is not None: - return self.bucket_size - return self.lookup_config.vocab_size + self.lookup_config.num_oov_buckets - - @property - def config(self): - return {self.column_name: parsing_ops.VarLenFeature(self.dtype)} - - @property - def key(self): - """Returns a string which will be used as a key when we do sorting.""" - return "{}".format(self) - - def id_tensor(self, input_tensor): - """Returns the id tensor from the given transformed input_tensor.""" - return input_tensor - - # pylint: disable=unused-argument - def weight_tensor(self, input_tensor): - """Returns the weight tensor from the given transformed input_tensor.""" - return None - - # pylint: disable=unused-argument - def _to_dnn_input_layer(self, - input_tensor, - weight_collections=None, - trainable=True, - output_rank=2): - raise ValueError( - "SparseColumn is not supported in DNN. " - "Please use embedding_column or one_hot_column. column: {}".format( - self)) - - def _wide_embedding_lookup_arguments(self, input_tensor): - return _LinearEmbeddingLookupArguments( - input_tensor=self.id_tensor(input_tensor), - weight_tensor=self.weight_tensor(input_tensor), - vocab_size=self.length, - initializer=init_ops.zeros_initializer(), - combiner=self.combiner) - - def _get_input_sparse_tensor(self, input_tensor): - """sparsify input_tensor if dense.""" - if not isinstance(input_tensor, sparse_tensor_py.SparseTensor): - # To avoid making any assumptions about which values are to be ignored, - # we set ignore_value to -1 for numeric tensors to avoid excluding valid - # indices. - if input_tensor.dtype == dtypes.string: - ignore_value = "" - else: - ignore_value = -1 - input_tensor = _reshape_real_valued_tensor(input_tensor, 2, self.name) - input_tensor = contrib_sparse_ops.dense_to_sparse_tensor( - input_tensor, ignore_value=ignore_value) - - return input_tensor - - def is_compatible(self, other_column): - """Check compatibility of two sparse columns.""" - if self.lookup_config and other_column.lookup_config: - return self.lookup_config == other_column.lookup_config - compatible = ( - self.length == other_column.length and - (self.dtype == other_column.dtype or - (self.dtype.is_integer and other_column.dtype.is_integer))) - if compatible: - logging.warn("Column {} and {} may not have the same vocabulary.".format( - self.name, other_column.name)) - return compatible - - @abc.abstractmethod - def _do_transform(self, input_tensor): - pass - - def insert_transformed_feature(self, columns_to_tensors): - """Handles sparse column to id conversion.""" - input_tensor = self._get_input_sparse_tensor(columns_to_tensors[self.name]) - columns_to_tensors[self] = self._do_transform(input_tensor) - - def _transform_feature(self, inputs): - input_tensor = self._get_input_sparse_tensor(inputs.get(self.name)) - return self._do_transform(input_tensor) - - @property - def _parse_example_spec(self): - return self.config - - @property - def _num_buckets(self): - return self.length - - def _get_sparse_tensors(self, inputs, weight_collections=None, - trainable=None): - del weight_collections - del trainable - input_tensor = inputs.get(self) - return fc_core._CategoricalColumn.IdWeightPair( # pylint: disable=protected-access - self.id_tensor(input_tensor), self.weight_tensor(input_tensor)) - - -class _SparseColumnIntegerized(_SparseColumn): - """See `sparse_column_with_integerized_feature`.""" - - def _do_transform(self, input_tensor): - sparse_id_values = math_ops.mod( - input_tensor.values, self.bucket_size, name="mod") - return sparse_tensor_py.SparseTensor(input_tensor.indices, sparse_id_values, - input_tensor.dense_shape) - - -def sparse_column_with_integerized_feature(column_name, - bucket_size, - combiner="sum", - dtype=dtypes.int64): - """Creates an integerized _SparseColumn. - - Use this when your features are already pre-integerized into int64 IDs, that - is, when the set of values to output is already coming in as what's desired in - the output. Integerized means we can use the feature value itself as id. - - Typically this is used for reading contiguous ranges of integers indexes, but - it doesn't have to be. The output value is simply copied from the - input_feature, whatever it is. Just be aware, however, that if you have large - gaps of unused integers it might affect what you feed those in (for instance, - if you make up a one-hot tensor from these, the unused integers will appear as - values in the tensor which are always zero.) - - Args: - column_name: A string defining sparse column name. - bucket_size: An int that is >= 1. The number of buckets. It should be bigger - than maximum feature. In other words features in this column should be an - int64 in range [0, bucket_size) - combiner: A string specifying how to reduce if the sparse column is - multivalent. Currently "mean", "sqrtn" and "sum" are supported, with "sum" - the default. "sqrtn" often achieves good accuracy, in particular with - bag-of-words columns. - * "sum": do not normalize features in the column - * "mean": do l1 normalization on features in the column - * "sqrtn": do l2 normalization on features in the column - For more information: `tf.embedding_lookup_sparse`. - dtype: Type of features. It should be an integer type. Default value is - dtypes.int64. - - Returns: - An integerized _SparseColumn definition. - - Raises: - ValueError: bucket_size is less than 1. - ValueError: dtype is not integer. - """ - return _SparseColumnIntegerized( - column_name, - is_integerized=True, - bucket_size=bucket_size, - combiner=combiner, - dtype=dtype) - - -class _SparseColumnHashed(_SparseColumn): - """See `sparse_column_with_hash_bucket`.""" - - def __new__(cls, - column_name, - is_integerized=False, - bucket_size=None, - lookup_config=None, - combiner="sum", - dtype=dtypes.string, - hash_keys=None): - if hash_keys is not None: - if not isinstance(hash_keys, list) or not hash_keys: - raise ValueError("hash_keys must be a non-empty list.") - if (any([not isinstance(key_pair, list) for key_pair in hash_keys]) or - any([len(key_pair) != 2 for key_pair in hash_keys]) or - any([not isinstance(key, int) for key in nest.flatten(hash_keys)])): - raise ValueError( - "Each element of hash_keys must be a pair of integers.") - obj = super(_SparseColumnHashed, cls).__new__( - cls, - column_name, - is_integerized=is_integerized, - bucket_size=bucket_size, - lookup_config=lookup_config, - combiner=combiner, - dtype=dtype) - obj.hash_keys = hash_keys - return obj - - def _do_transform(self, input_tensor): - if self.dtype.is_integer: - sparse_values = string_ops.as_string(input_tensor.values) - else: - sparse_values = input_tensor.values - - if self.hash_keys: - result = [] - for key in self.hash_keys: - sparse_id_values = string_ops.string_to_hash_bucket_strong( - sparse_values, self.bucket_size, key) - result.append( - sparse_tensor_py.SparseTensor(input_tensor.indices, - sparse_id_values, - input_tensor.dense_shape)) - return sparse_ops.sparse_concat(axis=1, sp_inputs=result, name="lookup") - else: - sparse_id_values = string_ops.string_to_hash_bucket_fast( - sparse_values, self.bucket_size, name="lookup") - return sparse_tensor_py.SparseTensor(input_tensor.indices, - sparse_id_values, - input_tensor.dense_shape) - - -def sparse_column_with_hash_bucket(column_name, - hash_bucket_size, - combiner="sum", - dtype=dtypes.string, - hash_keys=None): - """Creates a _SparseColumn with hashed bucket configuration. - - Use this when your sparse features are in string or integer format, but you - don't have a vocab file that maps each value to an integer ID. - output_id = Hash(input_feature_string) % bucket_size - - When hash_keys is set, multiple integer IDs would be created with each key - pair in the `hash_keys`. This is useful to reduce the collision of hashed ids. - - Args: - column_name: A string defining sparse column name. - hash_bucket_size: An int that is > 1. The number of buckets. - combiner: A string specifying how to reduce if the sparse column is - multivalent. Currently "mean", "sqrtn" and "sum" are supported, with "sum" - the default. "sqrtn" often achieves good accuracy, in particular with - bag-of-words columns. - * "sum": do not normalize features in the column - * "mean": do l1 normalization on features in the column - * "sqrtn": do l2 normalization on features in the column - For more information: `tf.embedding_lookup_sparse`. - dtype: The type of features. Only string and integer types are supported. - hash_keys: The hash keys to use. It is a list of lists of two uint64s. If - None, simple and fast hashing algorithm is used. Otherwise, multiple - strong hash ids would be produced with each two uint64s in this argument. - - Returns: - A _SparseColumn with hashed bucket configuration - - Raises: - ValueError: hash_bucket_size is not greater than 2. - ValueError: dtype is neither string nor integer. - """ - return _SparseColumnHashed( - column_name, - bucket_size=hash_bucket_size, - combiner=combiner, - dtype=dtype, - hash_keys=hash_keys) - - -class _SparseColumnKeys(_SparseColumn): - """See `sparse_column_with_keys`.""" - - def _do_transform(self, input_tensor): - table = lookup.index_table_from_tensor( - mapping=tuple(self.lookup_config.keys), - default_value=self.lookup_config.default_value, - dtype=self.dtype, - name="lookup") - return table.lookup(input_tensor) - - -def sparse_column_with_keys(column_name, - keys, - default_value=-1, - combiner="sum", - dtype=dtypes.string): - """Creates a _SparseColumn with keys. - - Look up logic is as follows: - lookup_id = index_of_feature_in_keys if feature in keys else default_value - - Args: - column_name: A string defining sparse column name. - keys: A list or tuple defining vocabulary. Must be castable to `dtype`. - default_value: The value to use for out-of-vocabulary feature values. - Default is -1. - combiner: A string specifying how to reduce if the sparse column is - multivalent. Currently "mean", "sqrtn" and "sum" are supported, with "sum" - the default. "sqrtn" often achieves good accuracy, in particular with - bag-of-words columns. - * "sum": do not normalize features in the column - * "mean": do l1 normalization on features in the column - * "sqrtn": do l2 normalization on features in the column - For more information: `tf.embedding_lookup_sparse`. - dtype: Type of features. Only integer and string are supported. - - Returns: - A _SparseColumnKeys with keys configuration. - """ - keys = tuple(keys) - return _SparseColumnKeys( - column_name, - lookup_config=_SparseIdLookupConfig( - keys=keys, vocab_size=len(keys), default_value=default_value), - combiner=combiner, - dtype=dtype) - - -class _SparseColumnVocabulary(_SparseColumn): - """See `sparse_column_with_vocabulary_file`.""" - - def _do_transform(self, st): - if self.dtype.is_integer: - sparse_string_values = string_ops.as_string(st.values) - sparse_string_tensor = sparse_tensor_py.SparseTensor( - st.indices, sparse_string_values, st.dense_shape) - else: - sparse_string_tensor = st - - table = lookup.index_table_from_file( - vocabulary_file=self.lookup_config.vocabulary_file, - num_oov_buckets=self.lookup_config.num_oov_buckets, - vocab_size=self.lookup_config.vocab_size, - default_value=self.lookup_config.default_value, - name=self.name + "_lookup") - return table.lookup(sparse_string_tensor) - - -def sparse_column_with_vocabulary_file(column_name, - vocabulary_file, - num_oov_buckets=0, - vocab_size=None, - default_value=-1, - combiner="sum", - dtype=dtypes.string): - """Creates a _SparseColumn with vocabulary file configuration. - - Use this when your sparse features are in string or integer format, and you - have a vocab file that maps each value to an integer ID. - output_id = LookupIdFromVocab(input_feature_string) - - Args: - column_name: A string defining sparse column name. - vocabulary_file: The vocabulary filename. - num_oov_buckets: The number of out-of-vocabulary buckets. If zero all out of - vocabulary features will be ignored. - vocab_size: Number of the elements in the vocabulary. - default_value: The value to use for out-of-vocabulary feature values. - Defaults to -1. - combiner: A string specifying how to reduce if the sparse column is - multivalent. Currently "mean", "sqrtn" and "sum" are supported, with "sum" - the default. "sqrtn" often achieves good accuracy, in particular with - bag-of-words columns. - * "sum": do not normalize features in the column - * "mean": do l1 normalization on features in the column - * "sqrtn": do l2 normalization on features in the column - For more information: `tf.embedding_lookup_sparse`. - dtype: The type of features. Only string and integer types are supported. - - Returns: - A _SparseColumn with vocabulary file configuration. - - Raises: - ValueError: vocab_size is not defined. - ValueError: dtype is neither string nor integer. - """ - if vocab_size is None: - raise ValueError("vocab_size should be defined. " - "column_name: {}".format(column_name)) - - return _SparseColumnVocabulary( - column_name, - lookup_config=_SparseIdLookupConfig( - vocabulary_file=vocabulary_file, - num_oov_buckets=num_oov_buckets, - vocab_size=vocab_size, - default_value=default_value), - combiner=combiner, - dtype=dtype) - - -class _WeightedSparseColumn( - _FeatureColumn, - fc_core._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple("_WeightedSparseColumn", - ["sparse_id_column", "weight_column_name", "dtype"]) -): - """See `weighted_sparse_column`.""" - - def __new__(cls, sparse_id_column, weight_column_name, dtype): - return super(_WeightedSparseColumn, cls).__new__(cls, sparse_id_column, - weight_column_name, dtype) - - @property - def name(self): - return "{}_weighted_by_{}".format(self.sparse_id_column.name, - self.weight_column_name) - - @property - def length(self): - """Returns id size.""" - return self.sparse_id_column.length - - @property - def config(self): - config = _get_feature_config(self.sparse_id_column) - config.update( - {self.weight_column_name: parsing_ops.VarLenFeature(self.dtype)}) - return config - - @property - def lookup_config(self): - return self.sparse_id_column.lookup_config - - @property - def key(self): - """Returns a string which will be used as a key when we do sorting.""" - return "{}".format(self) - - def id_tensor(self, input_tensor): - """Returns the id tensor from the given transformed input_tensor.""" - return input_tensor[0] - - def weight_tensor(self, input_tensor): - """Returns the weight tensor from the given transformed input_tensor.""" - return input_tensor[1] - - # pylint: disable=unused-argument - def _to_dnn_input_layer(self, - input_tensor, - weight_collections=None, - trainable=True, - output_rank=2): - raise ValueError( - "WeightedSparseColumn is not supported in DNN. " - "Please use embedding_column or one_hot_column. column: {}".format( - self)) - - def _wide_embedding_lookup_arguments(self, input_tensor): - return _LinearEmbeddingLookupArguments( - input_tensor=self.id_tensor(input_tensor), - weight_tensor=self.weight_tensor(input_tensor), - vocab_size=self.length, - initializer=init_ops.zeros_initializer(), - combiner=self.sparse_id_column.combiner) - - def _do_transform(self, id_tensor, weight_tensor): - if not isinstance(weight_tensor, sparse_tensor_py.SparseTensor): - # The weight tensor can be a regular Tensor. In such case, sparsify it. - weight_tensor = contrib_sparse_ops.dense_to_sparse_tensor(weight_tensor) - if not self.dtype.is_floating: - weight_tensor = math_ops.cast(weight_tensor, dtypes.float32) - return tuple([id_tensor, weight_tensor]) - - def insert_transformed_feature(self, columns_to_tensors): - """Inserts a tuple with the id and weight tensors.""" - if self.sparse_id_column not in columns_to_tensors: - self.sparse_id_column.insert_transformed_feature(columns_to_tensors) - - weight_tensor = columns_to_tensors[self.weight_column_name] - columns_to_tensors[self] = self._do_transform( - columns_to_tensors[self.sparse_id_column], weight_tensor) - - def _transform_feature(self, inputs): - return self._do_transform( - inputs.get(self.sparse_id_column), inputs.get(self.weight_column_name)) - - @property - def _parse_example_spec(self): - return self.config - - @property - def _num_buckets(self): - return self.length - - def _get_sparse_tensors(self, inputs, weight_collections=None, - trainable=None): - del weight_collections - del trainable - input_tensor = inputs.get(self) - return fc_core._CategoricalColumn.IdWeightPair( # pylint: disable=protected-access - self.id_tensor(input_tensor), self.weight_tensor(input_tensor)) - - def is_compatible(self, other_column): - """Check compatibility with other sparse column.""" - if isinstance(other_column, _WeightedSparseColumn): - return self.sparse_id_column.is_compatible(other_column.sparse_id_column) - return self.sparse_id_column.is_compatible(other_column) - - -def weighted_sparse_column(sparse_id_column, - weight_column_name, - dtype=dtypes.float32): - """Creates a _SparseColumn by combining sparse_id_column with a weight column. - - Example: - - ```python - sparse_feature = sparse_column_with_hash_bucket(column_name="sparse_col", - hash_bucket_size=1000) - weighted_feature = weighted_sparse_column(sparse_id_column=sparse_feature, - weight_column_name="weights_col") - ``` - - This configuration assumes that input dictionary of model contains the - following two items: - * (key="sparse_col", value=sparse_tensor) where sparse_tensor is - a SparseTensor. - * (key="weights_col", value=weights_tensor) where weights_tensor - is a SparseTensor. - Following are assumed to be true: - * sparse_tensor.indices = weights_tensor.indices - * sparse_tensor.dense_shape = weights_tensor.dense_shape - - Args: - sparse_id_column: A `_SparseColumn` which is created by - `sparse_column_with_*` functions. - weight_column_name: A string defining a sparse column name which represents - weight or value of the corresponding sparse id feature. - dtype: Type of weights, such as `tf.float32`. Only floating and integer - weights are supported. - - Returns: - A _WeightedSparseColumn composed of two sparse features: one represents id, - the other represents weight (value) of the id feature in that example. - - Raises: - ValueError: if dtype is not convertible to float. - """ - if not (dtype.is_integer or dtype.is_floating): - raise ValueError( - "dtype is not convertible to float. Given {}".format(dtype)) - - return _WeightedSparseColumn(sparse_id_column, weight_column_name, dtype) - - -class _OneHotColumn( - _FeatureColumn, - fc_core._DenseColumn, # pylint: disable=protected-access - collections.namedtuple("_OneHotColumn", ["sparse_id_column"])): - """Represents a one-hot column for use in deep networks. - - Args: - sparse_id_column: A _SparseColumn which is created by `sparse_column_with_*` - function. - """ - - @property - def name(self): - return "{}_one_hot".format(self.sparse_id_column.name) - - @property - def length(self): - """Returns vocabulary or hash_bucket size.""" - return self.sparse_id_column.length - - @property - def config(self): - """Returns the parsing config of the origin column.""" - return _get_feature_config(self.sparse_id_column) - - @property - def key(self): - """Returns a string which will be used as a key when we do sorting.""" - return "{}".format(self) - - def insert_transformed_feature(self, columns_to_tensors): - """Used by the Transformer to prevent double transformations.""" - if self.sparse_id_column not in columns_to_tensors: - self.sparse_id_column.insert_transformed_feature(columns_to_tensors) - columns_to_tensors[self] = columns_to_tensors[self.sparse_id_column] - - def _to_dnn_input_layer(self, - transformed_input_tensor, - unused_weight_collections=None, - unused_trainable=False, - output_rank=2): - """Returns a Tensor as an input to the first layer of neural network. - - Args: - transformed_input_tensor: A tensor that has undergone the transformations - in `insert_transformed_feature`. Rank should be >= `output_rank`. - unused_weight_collections: Unused. One hot encodings are not variable. - unused_trainable: Unused. One hot encodings are not trainable. - output_rank: the desired rank of the output `Tensor`. - - Returns: - A multi-hot Tensor to be fed into the first layer of neural network. - - Raises: - ValueError: When using one_hot_column with weighted_sparse_column. - This is not yet supported. - """ - - # Reshape ID column to `output_rank`. - sparse_id_column = self.sparse_id_column.id_tensor(transformed_input_tensor) - # pylint: disable=protected-access - sparse_id_column = layers._inner_flatten(sparse_id_column, output_rank) - - weight_tensor = self.sparse_id_column.weight_tensor( - transformed_input_tensor) - if weight_tensor is not None: - weighted_column = sparse_ops.sparse_merge( - sp_ids=sparse_id_column, - sp_values=weight_tensor, - vocab_size=self.length) - # Remove (?, -1) index - weighted_column = sparse_ops.sparse_slice( - weighted_column, array_ops.zeros_like(weighted_column.dense_shape), - weighted_column.dense_shape) - dense_tensor = sparse_ops.sparse_tensor_to_dense(weighted_column) - batch_shape = array_ops.shape(dense_tensor)[:-1] - dense_tensor_shape = array_ops.concat([batch_shape, [self.length]], - axis=0) - dense_tensor = array_ops.reshape(dense_tensor, dense_tensor_shape) - return dense_tensor - - dense_id_tensor = sparse_ops.sparse_tensor_to_dense( - sparse_id_column, default_value=-1) - - # One hot must be float for tf.concat reasons since all other inputs to - # input_layer are float32. - one_hot_id_tensor = array_ops.one_hot( - dense_id_tensor, depth=self.length, on_value=1.0, off_value=0.0) - - # Reduce to get a multi-hot per example. - return math_ops.reduce_sum(one_hot_id_tensor, axis=[output_rank - 1]) - - @property - def _variable_shape(self): - return tensor_shape.TensorShape([self.length]) - - def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): - del weight_collections - del trainable - return inputs.get(self) - - def _transform_feature(self, inputs): - return self._to_dnn_input_layer(inputs.get(self.sparse_id_column)) - - @property - def _parse_example_spec(self): - return self.config - - -class _EmbeddingColumn( - _FeatureColumn, - fc_core._DenseColumn, # pylint: disable=protected-access - collections.namedtuple("_EmbeddingColumn", [ - "sparse_id_column", "dimension", "combiner", "initializer", - "ckpt_to_load_from", "tensor_name_in_ckpt", "shared_embedding_name", - "shared_vocab_size", "max_norm", "trainable" - ])): - """Represents an embedding column. - - Args: - sparse_id_column: A `_SparseColumn` which is created by - `sparse_column_with_*` or `weighted_sparse_column` functions. - dimension: An integer specifying dimension of the embedding. - combiner: A string specifying how to reduce if there are multiple entries in - a single row. Currently "mean", "sqrtn" and "sum" are supported, with - "mean" the default. "sqrtn" often achieves good accuracy, in particular - with bag-of-words columns. Each of this can be thought as example level - normalizations on the column: - * "sum": do not normalize features in the column - * "mean": do l1 normalization on features in the column - * "sqrtn": do l2 normalization on features in the column - For more information: `tf.embedding_lookup_sparse`. - initializer: A variable initializer function to be used in embedding - variable initialization. If not specified, defaults to - `tf.compat.v1.truncated_normal_initializer` with mean 0.0 and standard - deviation 1/sqrt(sparse_id_column.length). - ckpt_to_load_from: (Optional). String representing checkpoint name/pattern - to restore the column weights. Required if `tensor_name_in_ckpt` is not - None. - tensor_name_in_ckpt: (Optional). Name of the `Tensor` in the provided - checkpoint from which to restore the column weights. Required if - `ckpt_to_load_from` is not None. - shared_embedding_name: (Optional). The common name for shared embedding. - shared_vocab_size: (Optional). The common vocab_size used for shared - embedding space. - max_norm: (Optional). If not None, embedding values are l2-normalized to the - value of max_norm. - trainable: (Optional). Should the embedding be trainable. Default is True. - - Raises: - ValueError: if `initializer` is specified and is not callable. Also, - if only one of `ckpt_to_load_from` and `tensor_name_in_ckpt` is specified. - """ - - def __new__(cls, - sparse_id_column, - dimension, - combiner="mean", - initializer=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - shared_embedding_name=None, - shared_vocab_size=None, - max_norm=None, - trainable=True): - if initializer is not None and not callable(initializer): - raise ValueError("initializer must be callable if specified. " - "Embedding of column_name: {}".format( - sparse_id_column.name)) - - if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError("Must specify both `ckpt_to_load_from` and " - "`tensor_name_in_ckpt` or none of them.") - if initializer is None: - logging.warn("The default stddev value of initializer was changed from " - "\"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core " - "implementation (tf.feature_column.embedding_column).") - stddev = 1 / math.sqrt(sparse_id_column.length) - initializer = init_ops.truncated_normal_initializer( - mean=0.0, stddev=stddev) - return super(_EmbeddingColumn, - cls).__new__(cls, sparse_id_column, dimension, combiner, - initializer, ckpt_to_load_from, - tensor_name_in_ckpt, shared_embedding_name, - shared_vocab_size, max_norm, trainable) - - @property - def name(self): - if self.shared_embedding_name is None: - return "{}_embedding".format(self.sparse_id_column.name) - else: - return "{}_shared_embedding".format(self.sparse_id_column.name) - - @property - def length(self): - """Returns id size.""" - if self.shared_vocab_size is None: - return self.sparse_id_column.length - else: - return self.shared_vocab_size - - @property - def config(self): - return _get_feature_config(self.sparse_id_column) - - @property - def key(self): - """Returns a string which will be used as a key when we do sorting.""" - return self._key_without_properties(["initializer"]) - - def insert_transformed_feature(self, columns_to_tensors): - if self.sparse_id_column not in columns_to_tensors: - self.sparse_id_column.insert_transformed_feature(columns_to_tensors) - columns_to_tensors[self] = columns_to_tensors[self.sparse_id_column] - - def _deep_embedding_lookup_arguments(self, input_tensor): - return _DeepEmbeddingLookupArguments( - input_tensor=self.sparse_id_column.id_tensor(input_tensor), - weight_tensor=self.sparse_id_column.weight_tensor(input_tensor), - vocab_size=self.length, - dimension=self.dimension, - initializer=self.initializer, - combiner=self.combiner, - shared_embedding_name=self.shared_embedding_name, - hash_key=None, - max_norm=self.max_norm, - trainable=self.trainable) - - def _checkpoint_path(self): - if self.ckpt_to_load_from is not None: - return self.ckpt_to_load_from, self.tensor_name_in_ckpt - return None - - # pylint: disable=unused-argument - def _wide_embedding_lookup_arguments(self, input_tensor): - raise ValueError("Column {} is not supported in linear models. " - "Please use sparse_column.".format(self)) - - @property - def _variable_shape(self): - return tensor_shape.TensorShape([self.dimension]) - - def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): - return _embeddings_from_arguments( - self, self._deep_embedding_lookup_arguments(inputs.get(self)), - weight_collections, trainable) - - def _transform_feature(self, inputs): - return inputs.get(self.sparse_id_column) - - @property - def _parse_example_spec(self): - return self.config - - -def _is_variable(v): - """Returns true if `v` is a variable.""" - return isinstance( - v, (variables.Variable, resource_variable_ops.ResourceVariable)) - - -def _embeddings_from_arguments(column, - args, - weight_collections, - trainable, - output_rank=2): - """Returns embeddings for a column based on the computed arguments. - - Args: - column: the column name. - args: the _DeepEmbeddingLookupArguments for this column. - weight_collections: collections to store weights in. - trainable: whether these embeddings should be trainable. - output_rank: the desired rank of the returned `Tensor`. Inner dimensions will - be combined to produce the desired rank. - - Returns: - the embeddings. - - Raises: - ValueError: if not possible to create. - """ - # pylint: disable=protected-access - input_tensor = layers._inner_flatten(args.input_tensor, output_rank) - weight_tensor = None - if args.weight_tensor is not None: - weight_tensor = layers._inner_flatten(args.weight_tensor, output_rank) - # pylint: enable=protected-access - - # This option is only enabled for scattered_embedding_column. - if args.hash_key: - embeddings = contrib_variables.model_variable( - name="weights", - shape=[args.vocab_size], - dtype=dtypes.float32, - initializer=args.initializer, - trainable=(trainable and args.trainable), - collections=weight_collections) - - return embedding_ops.scattered_embedding_lookup_sparse( - embeddings, - input_tensor, - args.dimension, - hash_key=args.hash_key, - combiner=args.combiner, - name="lookup") - - if args.shared_embedding_name is not None: - shared_embedding_collection_name = ("SHARED_EMBEDDING_COLLECTION_" + - args.shared_embedding_name.upper()) - graph = ops.get_default_graph() - shared_embedding_collection = ( - graph.get_collection_ref(shared_embedding_collection_name)) - shape = [args.vocab_size, args.dimension] - if shared_embedding_collection: - if len(shared_embedding_collection) > 1: - raise ValueError("Collection %s can only contain one " - "(partitioned) variable." % - shared_embedding_collection_name) - else: - embeddings = shared_embedding_collection[0] - if embeddings.get_shape() != shape: - raise ValueError("The embedding variable with name {} already " - "exists, but its shape does not match required " - "embedding shape here. Please make sure to use " - "different shared_embedding_name for different " - "shared embeddings.".format( - args.shared_embedding_name)) - else: - embeddings = contrib_variables.model_variable( - name=args.shared_embedding_name, - shape=shape, - dtype=dtypes.float32, - initializer=args.initializer, - trainable=(trainable and args.trainable), - collections=weight_collections) - graph.add_to_collection(shared_embedding_collection_name, embeddings) - else: - embeddings = contrib_variables.model_variable( - name="weights", - shape=[args.vocab_size, args.dimension], - dtype=dtypes.float32, - initializer=args.initializer, - trainable=(trainable and args.trainable), - collections=weight_collections) - - if _is_variable(embeddings): - embeddings = [embeddings] - else: - embeddings = embeddings._get_variable_list() # pylint: disable=protected-access - # pylint: disable=protected-access - _maybe_restore_from_checkpoint(column._checkpoint_path(), embeddings) - return embedding_ops.safe_embedding_lookup_sparse( - embeddings, - input_tensor, - sparse_weights=weight_tensor, - combiner=args.combiner, - name=column.name + "weights", - max_norm=args.max_norm) - - -def _maybe_restore_from_checkpoint(checkpoint_path, variable): - if checkpoint_path is not None: - path, tensor_name = checkpoint_path - weights_to_restore = variable - if len(variable) == 1: - weights_to_restore = variable[0] - checkpoint_utils.init_from_checkpoint(path, - {tensor_name: weights_to_restore}) - - -def one_hot_column(sparse_id_column): - """Creates an `_OneHotColumn` for a one-hot or multi-hot repr in a DNN. - - Args: - sparse_id_column: A _SparseColumn which is created by - `sparse_column_with_*` or crossed_column functions. Note that `combiner` - defined in `sparse_id_column` is ignored. - - Returns: - An _OneHotColumn. - """ - return _OneHotColumn(sparse_id_column) - - -def embedding_column(sparse_id_column, - dimension, - combiner="mean", - initializer=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True): - """Creates an `_EmbeddingColumn` for feeding sparse data into a DNN. - - Args: - sparse_id_column: A `_SparseColumn` which is created by for example - `sparse_column_with_*` or crossed_column functions. Note that `combiner` - defined in `sparse_id_column` is ignored. - dimension: An integer specifying dimension of the embedding. - combiner: A string specifying how to reduce if there are multiple entries in - a single row. Currently "mean", "sqrtn" and "sum" are supported, with - "mean" the default. "sqrtn" often achieves good accuracy, in particular - with bag-of-words columns. Each of this can be thought as example level - normalizations on the column: - * "sum": do not normalize - * "mean": do l1 normalization - * "sqrtn": do l2 normalization - For more information: `tf.embedding_lookup_sparse`. - initializer: A variable initializer function to be used in embedding - variable initialization. If not specified, defaults to - `tf.compat.v1.truncated_normal_initializer` with mean 0.0 and standard - deviation 1/sqrt(sparse_id_column.length). - ckpt_to_load_from: (Optional). String representing checkpoint name/pattern - to restore the column weights. Required if `tensor_name_in_ckpt` is not - None. - tensor_name_in_ckpt: (Optional). Name of the `Tensor` in the provided - checkpoint from which to restore the column weights. Required if - `ckpt_to_load_from` is not None. - max_norm: (Optional). If not None, embedding values are l2-normalized to the - value of max_norm. - trainable: (Optional). Should the embedding be trainable. Default is True - - Returns: - An `_EmbeddingColumn`. - """ - return _EmbeddingColumn( - sparse_id_column, - dimension, - combiner, - initializer, - ckpt_to_load_from, - tensor_name_in_ckpt, - max_norm=max_norm, - trainable=trainable) - - -def shared_embedding_columns(sparse_id_columns, - dimension, - combiner="mean", - shared_embedding_name=None, - initializer=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True): - """Creates a list of `_EmbeddingColumn` sharing the same embedding. - - Args: - sparse_id_columns: An iterable of `_SparseColumn`, such as those created by - `sparse_column_with_*` or crossed_column functions. Note that `combiner` - defined in each sparse_id_column is ignored. - dimension: An integer specifying dimension of the embedding. - combiner: A string specifying how to reduce if there are multiple entries in - a single row. Currently "mean", "sqrtn" and "sum" are supported, with - "mean" the default. "sqrtn" often achieves good accuracy, in particular - with bag-of-words columns. Each of this can be thought as example level - normalizations on the column: - * "sum": do not normalize - * "mean": do l1 normalization - * "sqrtn": do l2 normalization - For more information: `tf.embedding_lookup_sparse`. - shared_embedding_name: (Optional). A string specifying the name of shared - embedding weights. This will be needed if you want to reference the shared - embedding separately from the generated `_EmbeddingColumn`. - initializer: A variable initializer function to be used in embedding - variable initialization. If not specified, defaults to - `tf.compat.v1.truncated_normal_initializer` with mean 0.0 and standard - deviation 1/sqrt(sparse_id_columns[0].length). - ckpt_to_load_from: (Optional). String representing checkpoint name/pattern - to restore the column weights. Required if `tensor_name_in_ckpt` is not - None. - tensor_name_in_ckpt: (Optional). Name of the `Tensor` in the provided - checkpoint from which to restore the column weights. Required if - `ckpt_to_load_from` is not None. - max_norm: (Optional). If not None, embedding values are l2-normalized to the - value of max_norm. - trainable: (Optional). Should the embedding be trainable. Default is True - - Returns: - A tuple of `_EmbeddingColumn` with shared embedding space. - - Raises: - ValueError: if sparse_id_columns is empty, or its elements are not - compatible with each other. - TypeError: if `sparse_id_columns` is not a sequence or is a string. If at - least one element of `sparse_id_columns` is not a `SparseColumn` or a - `WeightedSparseColumn`. - """ - if (not isinstance(sparse_id_columns, collections_abc.Sequence) or - isinstance(sparse_id_columns, six.string_types)): - raise TypeError( - "sparse_id_columns must be a non-string sequence (ex: list or tuple) " - "instead of type {}.".format(type(sparse_id_columns))) - if len(sparse_id_columns) < 1: - raise ValueError("The input sparse_id_columns should have at least one " - "element.") - for sparse_id_column in sparse_id_columns: - if not (isinstance(sparse_id_column, _SparseColumn) or - isinstance(sparse_id_column, _WeightedSparseColumn)): - raise TypeError( - "Elements of sparse_id_columns must be _SparseColumn or " - "_WeightedSparseColumn, but {} is not.".format(sparse_id_column)) - - if len(sparse_id_columns) == 1: - return [ - _EmbeddingColumn( - sparse_id_columns[0], - dimension, - combiner, - initializer, - ckpt_to_load_from, - tensor_name_in_ckpt, - shared_embedding_name, - max_norm=max_norm, - trainable=trainable) - ] - else: - # Check compatibility of sparse_id_columns - compatible = True - for column in sparse_id_columns[1:]: - if isinstance(sparse_id_columns[0], _WeightedSparseColumn): - compatible = compatible and sparse_id_columns[0].is_compatible(column) - else: - compatible = compatible and column.is_compatible(sparse_id_columns[0]) - if not compatible: - raise ValueError("The input sparse id columns are not compatible.") - # Construct the shared name and size for shared embedding space. - if not shared_embedding_name: - # Sort the columns so that shared_embedding_name will be deterministic - # even if users pass in unsorted columns from a dict or something. - # Since they are different classes, ordering is SparseColumns first, - # then WeightedSparseColumns. - sparse_columns = [] - weighted_sparse_columns = [] - for column in sparse_id_columns: - if isinstance(column, _SparseColumn): - sparse_columns.append(column) - else: - weighted_sparse_columns.append(column) - sorted_columns = sorted(sparse_columns) + sorted( - weighted_sparse_columns, key=lambda x: x.name) - if len(sorted_columns) <= 3: - shared_embedding_name = "_".join( - [column.name for column in sorted_columns]) - else: - shared_embedding_name = "_".join( - [column.name for column in sorted_columns[0:3]]) - shared_embedding_name += ( - "_plus_{}_others".format(len(sorted_columns) - 3)) - shared_embedding_name += "_shared_embedding" - shared_vocab_size = sparse_id_columns[0].length - - embedded_columns = [] - for column in sparse_id_columns: - embedded_columns.append( - _EmbeddingColumn( - column, - dimension, - combiner, - initializer, - ckpt_to_load_from, - tensor_name_in_ckpt, - shared_embedding_name, - shared_vocab_size, - max_norm=max_norm, - trainable=trainable)) - return tuple(embedded_columns) - - -class _ScatteredEmbeddingColumn( - _FeatureColumn, - fc_core._DenseColumn, # pylint: disable=protected-access - collections.namedtuple("_ScatteredEmbeddingColumn", [ - "column_name", "size", "dimension", "hash_key", "combiner", - "initializer" - ])): - """See `scattered_embedding_column`.""" - - def __new__(cls, - column_name, - size, - dimension, - hash_key, - combiner="sqrtn", - initializer=None): - if initializer is not None and not callable(initializer): - raise ValueError("initializer must be callable if specified. " - "column_name: {}".format(column_name)) - if initializer is None: - stddev = 0.1 - initializer = init_ops.truncated_normal_initializer( - mean=0.0, stddev=stddev) - return super(_ScatteredEmbeddingColumn, - cls).__new__(cls, column_name, size, dimension, hash_key, - combiner, initializer) - - @property - def name(self): - return "{}_scattered_embedding".format(self.column_name) - - @property - def config(self): - return {self.column_name: parsing_ops.VarLenFeature(dtypes.string)} - - @property - def key(self): - """Returns a string which will be used as a key when we do sorting.""" - return self._key_without_properties(["initializer"]) - - def insert_transformed_feature(self, columns_to_tensors): - columns_to_tensors[self] = columns_to_tensors[self.column_name] - - def _deep_embedding_lookup_arguments(self, input_tensor): - return _DeepEmbeddingLookupArguments( - input_tensor=input_tensor, - weight_tensor=None, - vocab_size=self.size, - initializer=self.initializer, - combiner=self.combiner, - dimension=self.dimension, - shared_embedding_name=None, - hash_key=self.hash_key, - max_norm=None, - trainable=True) - - @property - def _variable_shape(self): - return tensor_shape.TensorShape([self.dimension]) - - def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): - return _embeddings_from_arguments( - self, self._deep_embedding_lookup_arguments(inputs.get(self)), - weight_collections, trainable) - - def _transform_feature(self, inputs): - return inputs.get(self.column_name) - - @property - def _parse_example_spec(self): - return self.config - - -def scattered_embedding_column(column_name, - size, - dimension, - hash_key, - combiner="mean", - initializer=None): - """Creates an embedding column of a sparse feature using parameter hashing. - - This is a useful shorthand when you have a sparse feature you want to use an - embedding for, but also want to hash the embedding's values in each dimension - to a variable based on a different hash. - - Specifically, the i-th embedding component of a value v is found by retrieving - an embedding weight whose index is a fingerprint of the pair (v,i). - - An embedding column with sparse_column_with_hash_bucket such as - - embedding_column( - sparse_column_with_hash_bucket(column_name, bucket_size), - dimension) - - could be replaced by - - scattered_embedding_column( - column_name, - size=bucket_size * dimension, - dimension=dimension, - hash_key=tf.contrib.layers.SPARSE_FEATURE_CROSS_DEFAULT_HASH_KEY) - - for the same number of embedding parameters. This should hopefully reduce the - impact of collisions, but adds the cost of slowing down training. - - Args: - column_name: A string defining sparse column name. - size: An integer specifying the number of parameters in the embedding layer. - dimension: An integer specifying dimension of the embedding. - hash_key: Specify the hash_key that will be used by the `FingerprintCat64` - function to combine the crosses fingerprints on SparseFeatureCrossOp. - combiner: A string specifying how to reduce if there are multiple entries in - a single row. Currently "mean", "sqrtn" and "sum" are supported, with - "mean" the default. "sqrtn" often achieves good accuracy, in particular - with bag-of-words columns. Each of this can be thought as example level - normalizations on the column: - * "sum": do not normalize features in the column - * "mean": do l1 normalization on features in the column - * "sqrtn": do l2 normalization on features in the column - For more information: `tf.embedding_lookup_sparse`. - initializer: A variable initializer function to be used in embedding - variable initialization. If not specified, defaults to - `tf.compat.v1.truncated_normal_initializer` with mean 0 and standard - deviation 0.1. - - Returns: - A _ScatteredEmbeddingColumn. - - Raises: - ValueError: if dimension or size is not a positive integer; or if combiner - is not supported. - - """ - if (dimension < 1) or (size < 1): - raise ValueError("Dimension and size must be greater than 0. " - "dimension: {}, size: {}, column_name: {}".format( - dimension, size, column_name)) - - if combiner not in ("mean", "sqrtn", "sum"): - raise ValueError("Combiner must be one of 'mean', 'sqrtn' or 'sum'. " - "combiner: {}, column_name: {}".format( - combiner, column_name)) - - return _ScatteredEmbeddingColumn(column_name, size, dimension, hash_key, - combiner, initializer) - - -def _reshape_real_valued_tensor(input_tensor, output_rank, column_name=None): - """Reshaping logic for dense, numeric `Tensors`. - - Follows the following rules: - 1. If `output_rank > input_rank + 1` raise a `ValueError`. - 2. If `output_rank == input_rank + 1`, expand `input_tensor` by one - dimension and return - 3. If `output_rank == input_rank`, return `input_tensor`. - 4. If `output_rank < input_rank`, flatten the inner dimensions of - `input_tensor` and return a `Tensor` with `output_rank` - - Args: - input_tensor: a dense `Tensor` to be reshaped. - output_rank: the desired rank of the reshaped `Tensor`. - column_name: (optional) the name of the associated column. Used for error - messages. - - Returns: - A `Tensor` with the same entries as `input_tensor` and rank `output_rank`. - Raises: - ValueError: if `output_rank > input_rank + 1`. - """ - input_rank = input_tensor.get_shape().ndims - if input_rank is not None: - if output_rank > input_rank + 1: - error_string = ("Rank of input Tensor ({}) should be the same as " - "output_rank ({}). For example, sequence data should " - "typically be 3 dimensional (rank 3) while non-sequence " - "data is typically 2 dimensional (rank 2).".format( - input_rank, output_rank)) - if column_name is not None: - error_string = ( - "Error while processing column {}.".format(column_name) + - error_string) - raise ValueError(error_string) - if output_rank == input_rank + 1: - logging.warning( - "Rank of input Tensor ({}) should be the same as output_rank ({}) " - "for column. Will attempt to expand dims. It is highly recommended " - "that you resize your input, as this behavior may change.".format( - input_rank, output_rank)) - return array_ops.expand_dims(input_tensor, -1, name="expand_dims") - if output_rank == input_rank: - return input_tensor - # Here, either `input_rank` is unknown or it is greater than `output_rank`. - return layers._inner_flatten(input_tensor, output_rank) # pylint: disable=protected-access - - -class _RealValuedVarLenColumn( - _FeatureColumn, - collections.namedtuple( - "_RealValuedVarLenColumn", - ["column_name", "default_value", "dtype", "normalizer", "is_sparse"])): - """Represents a real valued feature column for variable length Features. - - Instances of this class are immutable. - If is_sparse=False, the dictionary returned by InputBuilder contains a - ("column_name", Tensor) pair with a Tensor shape of (batch_size, dimension). - If is_sparse=True, the dictionary contains a ("column_name", SparseTensor) - pair instead with shape inferred after parsing. - """ - - @property - def name(self): - return self.column_name - - @property - def config(self): - if self.is_sparse: - return {self.column_name: parsing_ops.VarLenFeature(self.dtype)} - else: - return { - self.column_name: - parsing_ops.FixedLenSequenceFeature( - [], - self.dtype, - allow_missing=True, - default_value=self.default_value) - } - - @property - def key(self): - """Returns a string which will be used as a key when we do sorting.""" - return self._key_without_properties(["normalizer"]) - - @property - def normalizer_fn(self): - """Returns the function used to normalize the column.""" - return self.normalizer - - def _normalized_input_tensor(self, input_tensor): - """Returns the input tensor after custom normalization is applied.""" - if self.normalizer is None: - return input_tensor - if self.is_sparse: - return sparse_tensor_py.SparseTensor(input_tensor.indices, - self.normalizer(input_tensor.values), - input_tensor.dense_shape) - else: - return self.normalizer(input_tensor) - - def insert_transformed_feature(self, columns_to_tensors): - """Apply transformation and inserts it into columns_to_tensors. - - Args: - columns_to_tensors: A mapping from feature columns to tensors. 'string' - key means a base feature (not-transformed). It can have _FeatureColumn - as a key too. That means that _FeatureColumn is already transformed. - """ - # Transform the input tensor according to the normalizer function. - input_tensor = self._normalized_input_tensor(columns_to_tensors[self.name]) - columns_to_tensors[self] = math_ops.cast(input_tensor, dtypes.float32) - - # pylint: disable=unused-argument - def _to_dnn_input_layer(self, - input_tensor, - weight_collections=None, - trainable=True, - output_rank=2): - return _reshape_real_valued_tensor( - self._to_dense_tensor(input_tensor), output_rank, self.name) - - def _to_dense_tensor(self, input_tensor): - if not self.is_sparse: - return input_tensor - raise ValueError("Set is_sparse to False if you want a dense Tensor for " - "column_name: {}".format(self.name)) - - -@experimental -def _real_valued_var_len_column(column_name, - default_value=None, - dtype=dtypes.float32, - normalizer=None, - is_sparse=False): - """Creates a `_RealValuedVarLenColumn` for variable-length numeric data. - - Note, this is not integrated with any of the DNNEstimators, except the RNN - ones DynamicRNNEstimator and the StateSavingRNNEstimator. - - It can either create a parsing config for a SparseTensor (with is_sparse=True) - or a padded Tensor. - The (dense_)shape of the result will be [batch_size, None], which can be used - with is_sparse=False as input into an RNN (see DynamicRNNEstimator or - StateSavingRNNEstimator) or with is_sparse=True as input into a tree (see - gtflow). - - Use real_valued_column if the Feature has a fixed length. Use some - SparseColumn for columns to be embedded / one-hot-encoded. - - Args: - column_name: A string defining real valued column name. - default_value: A scalar value compatible with dtype. Needs to be specified - if is_sparse=False. - dtype: Defines the type of values. Default value is tf.float32. Needs to be - convertible to tf.float32. - normalizer: If not None, a function that can be used to normalize the value - of the real valued column after default_value is applied for parsing. - Normalizer function takes the input tensor as its argument, and returns - the output tensor. (e.g. lambda x: (x - 3.0) / 4.2). Note that for - is_sparse=False, the normalizer will be run on the values of the - `SparseTensor`. - is_sparse: A boolean defining whether to create a SparseTensor or a Tensor. - - Returns: - A _RealValuedSparseColumn. - Raises: - TypeError: if default_value is not a scalar value compatible with dtype. - TypeError: if dtype is not convertible to tf.float32. - ValueError: if default_value is None and is_sparse is False. - """ - if not (dtype.is_integer or dtype.is_floating): - raise TypeError("dtype must be convertible to float. " - "dtype: {}, column_name: {}".format(dtype, column_name)) - - if default_value is None and not is_sparse: - raise ValueError("default_value must be provided when is_sparse=False to " - "parse a padded Tensor. " - "column_name: {}".format(column_name)) - if isinstance(default_value, list): - raise ValueError( - "Only scalar default value. default_value: {}, column_name: {}".format( - default_value, column_name)) - if default_value is not None: - if dtype.is_integer: - default_value = int(default_value) - elif dtype.is_floating: - default_value = float(default_value) - - return _RealValuedVarLenColumn(column_name, default_value, dtype, normalizer, - is_sparse) - - -class _RealValuedColumn( - _FeatureColumn, - fc_core._DenseColumn, # pylint: disable=protected-access - collections.namedtuple( - "_RealValuedColumn", - ["column_name", "dimension", "default_value", "dtype", "normalizer"])): - """Represents a real valued feature column also known as continuous features. - - Instances of this class are immutable. The dictionary returned by InputBuilder - contains a ("column_name", Tensor) pair with a Tensor shape of - (batch_size, dimension). - """ - - def __new__(cls, column_name, dimension, default_value, dtype, normalizer): - if default_value is not None: - default_value = tuple(default_value) - return super(_RealValuedColumn, - cls).__new__(cls, column_name, dimension, default_value, dtype, - normalizer) - - @property - def name(self): - return self.column_name - - @property - def config(self): - default_value = self.default_value - if default_value is not None: - default_value = list(default_value) - return { - self.column_name: - parsing_ops.FixedLenFeature([self.dimension], self.dtype, - default_value) - } - - @property - def key(self): - """Returns a string which will be used as a key when we do sorting.""" - return self._key_without_properties(["normalizer"]) - - @property - def normalizer_fn(self): - """Returns the function used to normalize the column.""" - return self.normalizer - - def _normalized_input_tensor(self, input_tensor): - """Returns the input tensor after custom normalization is applied.""" - return (self.normalizer(input_tensor) - if self.normalizer is not None else input_tensor) - - def insert_transformed_feature(self, columns_to_tensors): - """Apply transformation and inserts it into columns_to_tensors. - - Args: - columns_to_tensors: A mapping from feature columns to tensors. 'string' - key means a base feature (not-transformed). It can have _FeatureColumn - as a key too. That means that _FeatureColumn is already transformed. - """ - # Transform the input tensor according to the normalizer function. - input_tensor = self._normalized_input_tensor(columns_to_tensors[self.name]) - columns_to_tensors[self] = math_ops.cast(input_tensor, dtypes.float32) - - # pylint: disable=unused-argument - def _to_dnn_input_layer(self, - input_tensor, - weight_collections=None, - trainable=True, - output_rank=2): - input_tensor = self._to_dense_tensor(input_tensor) - if input_tensor.dtype != dtypes.float32: - input_tensor = math_ops.cast(input_tensor, dtypes.float32) - return _reshape_real_valued_tensor(input_tensor, output_rank, self.name) - - def _to_dense_tensor(self, input_tensor): - return input_tensor - - @property - def _variable_shape(self): - return tensor_shape.TensorShape([self.dimension]) - - def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): - del weight_collections - del trainable - return inputs.get(self) - - def _transform_feature(self, inputs): - return math_ops.cast( - self._normalized_input_tensor(inputs.get(self.name)), dtypes.float32) - - @property - def _parse_example_spec(self): - return self.config - - -def real_valued_column(column_name, - dimension=1, - default_value=None, - dtype=dtypes.float32, - normalizer=None): - """Creates a `_RealValuedColumn` for dense numeric data. - - Args: - column_name: A string defining real valued column name. - dimension: An integer specifying dimension of the real valued column. The - default is 1. - default_value: A single value compatible with dtype or a list of values - compatible with dtype which the column takes on during tf.Example parsing - if data is missing. When dimension is not None, a default value of None - will cause tf.io.parse_example to fail if an example does not contain this - column. If a single value is provided, the same value will be applied as - the default value for every dimension. If a list of values is provided, - the length of the list should be equal to the value of `dimension`. Only - scalar default value is supported in case dimension is not specified. - dtype: defines the type of values. Default value is tf.float32. Must be a - non-quantized, real integer or floating point type. - normalizer: If not None, a function that can be used to normalize the value - of the real valued column after default_value is applied for parsing. - Normalizer function takes the input tensor as its argument, and returns - the output tensor. (e.g. lambda x: (x - 3.0) / 4.2). Note that for - variable length columns, the normalizer should expect an input_tensor of - type `SparseTensor`. - - Returns: - A _RealValuedColumn. - Raises: - TypeError: if dimension is not an int - ValueError: if dimension is not a positive integer - TypeError: if default_value is a list but its length is not equal to the - value of `dimension`. - TypeError: if default_value is not compatible with dtype. - ValueError: if dtype is not convertible to tf.float32. - """ - - if dimension is None: - raise TypeError("dimension must be an integer. Use the " - "_real_valued_var_len_column for variable length features." - "dimension: {}, column_name: {}".format( - dimension, column_name)) - if not isinstance(dimension, int): - raise TypeError("dimension must be an integer. " - "dimension: {}, column_name: {}".format( - dimension, column_name)) - if dimension < 1: - raise ValueError("dimension must be greater than 0. " - "dimension: {}, column_name: {}".format( - dimension, column_name)) - - if not (dtype.is_integer or dtype.is_floating): - raise ValueError("dtype must be convertible to float. " - "dtype: {}, column_name: {}".format(dtype, column_name)) - - if default_value is None: - return _RealValuedColumn(column_name, dimension, default_value, dtype, - normalizer) - - if isinstance(default_value, int): - if dtype.is_integer: - default_value = ([default_value for _ in range(dimension)] - if dimension else [default_value]) - return _RealValuedColumn(column_name, dimension, default_value, dtype, - normalizer) - if dtype.is_floating: - default_value = float(default_value) - default_value = ([default_value for _ in range(dimension)] - if dimension else [default_value]) - return _RealValuedColumn(column_name, dimension, default_value, dtype, - normalizer) - - if isinstance(default_value, float): - if dtype.is_floating and (not dtype.is_integer): - default_value = ([default_value for _ in range(dimension)] - if dimension else [default_value]) - return _RealValuedColumn(column_name, dimension, default_value, dtype, - normalizer) - - if isinstance(default_value, list): - if len(default_value) != dimension: - raise ValueError( - "The length of default_value must be equal to dimension. " - "default_value: {}, dimension: {}, column_name: {}".format( - default_value, dimension, column_name)) - # Check if the values in the list are all integers or are convertible to - # floats. - is_list_all_int = True - is_list_all_float = True - for v in default_value: - if not isinstance(v, int): - is_list_all_int = False - if not (isinstance(v, float) or isinstance(v, int)): - is_list_all_float = False - if is_list_all_int: - if dtype.is_integer: - return _RealValuedColumn(column_name, dimension, default_value, dtype, - normalizer) - elif dtype.is_floating: - default_value = [float(v) for v in default_value] - return _RealValuedColumn(column_name, dimension, default_value, dtype, - normalizer) - if is_list_all_float: - if dtype.is_floating and (not dtype.is_integer): - default_value = [float(v) for v in default_value] - return _RealValuedColumn(column_name, dimension, default_value, dtype, - normalizer) - - raise TypeError("default_value must be compatible with dtype. " - "default_value: {}, dtype: {}, column_name: {}".format( - default_value, dtype, column_name)) - - -class _BucketizedColumn( - _FeatureColumn, - fc_core._CategoricalColumn, # pylint: disable=protected-access - fc_core._DenseColumn, # pylint: disable=protected-access - collections.namedtuple("_BucketizedColumn", - ["source_column", "boundaries"])): - """Represents a bucketization transformation also known as binning. - - Instances of this class are immutable. Values in `source_column` will be - bucketized based on `boundaries`. - For example, if the inputs are: - boundaries = [0, 10, 100] - source_column = [[-5], [150], [10], [0], [4], [19]] - - then the bucketized feature will be: - output = [[0], [3], [2], [1], [1], [2]] - - Attributes: - source_column: A _RealValuedColumn defining dense column. - boundaries: A list or tuple of floats specifying the boundaries. It has to - be sorted. [a, b, c] defines following buckets: (-inf., a), [a, b), [b, - c), [c, inf.) - - Raises: - ValueError: if 'boundaries' is empty or not sorted. - """ - - def __new__(cls, source_column, boundaries): - if not isinstance(source_column, _RealValuedColumn): - raise TypeError("source_column must be an instance of _RealValuedColumn. " - "source_column: {}".format(source_column)) - - if source_column.dimension is None: - raise ValueError("source_column must have a defined dimension. " - "source_column: {}".format(source_column)) - - if (not isinstance(boundaries, list) and - not isinstance(boundaries, tuple)) or not boundaries: - raise ValueError("boundaries must be a non-empty list or tuple. " - "boundaries: {}".format(boundaries)) - - # We allow bucket boundaries to be monotonically increasing - # (ie a[i+1] >= a[i]). When two bucket boundaries are the same, we - # de-duplicate. - sanitized_boundaries = [] - for i in range(len(boundaries) - 1): - if boundaries[i] == boundaries[i + 1]: - continue - elif boundaries[i] < boundaries[i + 1]: - sanitized_boundaries.append(boundaries[i]) - else: - raise ValueError("boundaries must be a sorted list. " - "boundaries: {}".format(boundaries)) - sanitized_boundaries.append(boundaries[len(boundaries) - 1]) - - return super(_BucketizedColumn, cls).__new__(cls, source_column, - tuple(sanitized_boundaries)) - - @property - def name(self): - return "{}_bucketized".format(self.source_column.name) - - @property - def length(self): - """Returns total number of buckets.""" - return len(self.boundaries) + 1 - - @property - def config(self): - return self.source_column.config - - @property - def key(self): - """Returns a string which will be used as a key when we do sorting.""" - return "{}".format(self) - - # pylint: disable=unused-argument - def _to_dnn_input_layer(self, - input_tensor, - weight_collections=None, - trainable=True, - output_rank=2): - if output_rank != 2: - raise ValueError("BucketizedColumn currently only supports output_rank=2") - return array_ops.reshape( - array_ops.one_hot( - math_ops.cast(input_tensor, dtypes.int64), - self.length, - 1., - 0., - name="one_hot"), [-1, self.length * self.source_column.dimension], - name="reshape") - - def to_sparse_tensor(self, input_tensor): - """Creates a SparseTensor from the bucketized Tensor.""" - dimension = self.source_column.dimension - batch_size = array_ops.shape(input_tensor, name="shape")[0] - - if dimension > 1: - i1 = array_ops.reshape( - array_ops.tile( - array_ops.expand_dims( - math_ops.range(0, batch_size), 1, name="expand_dims"), - [1, dimension], - name="tile"), [-1], - name="reshape") - i2 = array_ops.tile( - math_ops.range(0, dimension), [batch_size], name="tile") - # Flatten the bucket indices and unique them across dimensions - # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets - bucket_indices = array_ops.reshape( - input_tensor, [-1], name="reshape") + self.length * i2 - else: - # Simpler indices when dimension=1 - i1 = math_ops.range(0, batch_size) - i2 = array_ops.zeros([batch_size], dtype=dtypes.int32, name="zeros") - bucket_indices = array_ops.reshape(input_tensor, [-1], name="reshape") - - indices = math_ops.cast( - array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64) - shape = math_ops.cast( - array_ops.stack([batch_size, dimension]), dtypes.int64) - sparse_id_values = sparse_tensor_py.SparseTensor(indices, bucket_indices, - shape) - - return sparse_id_values - - def _wide_embedding_lookup_arguments(self, input_tensor): - return _LinearEmbeddingLookupArguments( - input_tensor=self.to_sparse_tensor(input_tensor), - weight_tensor=None, - vocab_size=self.length * self.source_column.dimension, - initializer=init_ops.zeros_initializer(), - combiner="sum") - - def _transform_feature(self, inputs): - """Handles cross transformation.""" - # Bucketize the source column. - return bucketization_op.bucketize( - inputs.get(self.source_column), - boundaries=list(self.boundaries), - name="bucketize") - - def insert_transformed_feature(self, columns_to_tensors): - """Handles sparse column to id conversion.""" - columns_to_tensors[self] = self._transform_feature( - _LazyBuilderByColumnsToTensor(columns_to_tensors)) - - @property - def _parse_example_spec(self): - return self.config - - @property - def _num_buckets(self): - return self.length * self.source_column.dimension - - def _get_sparse_tensors(self, inputs, weight_collections=None, - trainable=None): - del weight_collections - del trainable - return fc_core._CategoricalColumn.IdWeightPair( # pylint: disable=protected-access - self.to_sparse_tensor(inputs.get(self)), None) - - @property - def _variable_shape(self): - return tensor_shape.TensorShape( - [self.length * self.source_column.dimension]) - - def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): - return self._to_dnn_input_layer( - inputs.get(self), weight_collections, trainable) - - -def bucketized_column(source_column, boundaries): - """Creates a _BucketizedColumn for discretizing dense input. - - Args: - source_column: A _RealValuedColumn defining dense column. - boundaries: A list or tuple of floats specifying the boundaries. It has to - be sorted. - - Returns: - A _BucketizedColumn. - - Raises: - ValueError: if 'boundaries' is empty or not sorted. - """ - return _BucketizedColumn(source_column, boundaries) - - -class _CrossedColumn( - _FeatureColumn, - fc_core._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple("_CrossedColumn", [ - "columns", "hash_bucket_size", "hash_key", "combiner", - "ckpt_to_load_from", "tensor_name_in_ckpt" - ])): - """Represents a cross transformation also known as conjunction or combination. - - Instances of this class are immutable. It crosses given `columns`. Crossed - column output will be hashed to hash_bucket_size. - Conceptually, transformation can be thought as: - Hash(cartesian product of features in columns) % `hash_bucket_size` - - For example, if the columns are - - SparseTensor referred by first column: shape = [2, 2] - [0, 0]: "a" - [1, 0]: "b" - [1, 1]: "c" - - SparseTensor referred by second column: : shape = [2, 1] - [0, 0]: "d" - [1, 0]: "e" - - then crossed feature will look like: - - shape = [2, 2] - [0, 0]: Hash64("d", Hash64("a")) % hash_bucket_size - [1, 0]: Hash64("e", Hash64("b")) % hash_bucket_size - [1, 1]: Hash64("e", Hash64("c")) % hash_bucket_size - - Attributes: - columns: An iterable of _FeatureColumn. Items can be an instance of - _SparseColumn, _CrossedColumn, or _BucketizedColumn. - hash_bucket_size: An int that is > 1. The number of buckets. - combiner: A string specifying how to reduce if there are multiple entries in - a single row. Currently "mean", "sqrtn" and "sum" are supported, with - "sum" the default. "sqrtn" often achieves good accuracy, in particular - with bag-of-words columns. Each of this can be thought as example level - normalizations on the column:: - * "sum": do not normalize - * "mean": do l1 normalization - * "sqrtn": do l2 normalization - For more information: `tf.embedding_lookup_sparse`. - ckpt_to_load_from: (Optional). String representing checkpoint name/pattern - to restore the column weights. Required if `tensor_name_in_ckpt` is not - None. - tensor_name_in_ckpt: (Optional). Name of the `Tensor` in the provided - checkpoint from which to restore the column weights. Required if - `ckpt_to_load_from` is not None. - - Raises: - TypeError: if all items in columns are not an instance of _SparseColumn, - _CrossedColumn, or _BucketizedColumn. - ValueError: if hash_bucket_size is not > 1 or len(columns) is not > 1. Also, - if only one of `ckpt_to_load_from` and `tensor_name_in_ckpt` is specified. - """ - - @staticmethod - def _assert_is_crossable(column): - if isinstance(column, (_SparseColumn, _CrossedColumn, _BucketizedColumn)): - return - raise TypeError("columns must be a set of _SparseColumn, " - "_CrossedColumn, or _BucketizedColumn instances. " - "(column {} is a {})".format(column, - column.__class__.__name__)) - - def __new__(cls, - columns, - hash_bucket_size, - hash_key, - combiner="sum", - ckpt_to_load_from=None, - tensor_name_in_ckpt=None): - for column in columns: - _CrossedColumn._assert_is_crossable(column) - - if len(columns) < 2: - raise ValueError("columns must contain at least 2 elements. " - "columns: {}".format(columns)) - - if hash_bucket_size < 2: - raise ValueError("hash_bucket_size must be at least 2. " - "hash_bucket_size: {}".format(hash_bucket_size)) - - if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError("Must specify both `ckpt_to_load_from` and " - "`tensor_name_in_ckpt` or none of them.") - - sorted_columns = sorted([column for column in columns], - key=lambda column: column.name) - return super(_CrossedColumn, - cls).__new__(cls, tuple(sorted_columns), hash_bucket_size, - hash_key, combiner, ckpt_to_load_from, - tensor_name_in_ckpt) - - @property - def name(self): - sorted_names = sorted([column.name for column in self.columns]) - return "_X_".join(sorted_names) - - @property - def config(self): - config = {} - for column in self.columns: - config.update(_get_feature_config(column)) - return config - - @property - def length(self): - """Returns total number of buckets.""" - return self.hash_bucket_size - - @property - def key(self): - """Returns a string which will be used as a key when we do sorting.""" - return "{}".format(self) - - def id_tensor(self, input_tensor): - """Returns the id tensor from the given transformed input_tensor.""" - return input_tensor - - def weight_tensor(self, input_tensor): - """Returns the weight tensor from the given transformed input_tensor.""" - del input_tensor - return None - - def _to_dnn_input_layer(self, - input_tensor, - weight_collections=None, - trainable=True, - output_rank=2): - del input_tensor - del weight_collections - del trainable - del output_rank - raise ValueError("CrossedColumn is not supported in DNN. " - "Please use embedding_column. column: {}".format(self)) - - def _checkpoint_path(self): - if self.ckpt_to_load_from is not None: - return self.ckpt_to_load_from, self.tensor_name_in_ckpt - return None - - def _wide_embedding_lookup_arguments(self, input_tensor): - return _LinearEmbeddingLookupArguments( - input_tensor=input_tensor, - weight_tensor=None, - vocab_size=self.length, - initializer=init_ops.zeros_initializer(), - combiner=self.combiner) - - def _transform_feature(self, inputs): - """Handles cross transformation.""" - - def _collect_leaf_level_columns(cross): - """Collects base columns contained in the cross.""" - leaf_level_columns = [] - for c in cross.columns: - if isinstance(c, _CrossedColumn): - leaf_level_columns.extend(_collect_leaf_level_columns(c)) - else: - leaf_level_columns.append(c) - return leaf_level_columns - - feature_tensors = [] - for c in _collect_leaf_level_columns(self): - if isinstance(c, _SparseColumn): - feature_tensors.append(inputs.get(c.name)) - else: - if isinstance(c, _BucketizedColumn): - feature_tensors.append(c.to_sparse_tensor(inputs.get(c))) - else: - feature_tensors.append(inputs.get(c)) - return sparse_feature_cross_op.sparse_feature_cross( - feature_tensors, - hashed_output=True, - num_buckets=self.hash_bucket_size, - hash_key=self.hash_key, - name="cross") - - def insert_transformed_feature(self, columns_to_tensors): - """Handles sparse column to id conversion.""" - columns_to_tensors[self] = self._transform_feature( - _LazyBuilderByColumnsToTensor(columns_to_tensors)) - - @property - def _parse_example_spec(self): - return self.config - - @property - def _num_buckets(self): - return self.length - - def _get_sparse_tensors(self, inputs, weight_collections=None, - trainable=None): - del weight_collections - del trainable - return fc_core._CategoricalColumn.IdWeightPair(inputs.get(self), None) # pylint: disable=protected-access - - -class _LazyBuilderByColumnsToTensor(object): - - def __init__(self, columns_to_tensors): - self._columns_to_tensors = columns_to_tensors - - def get(self, key): - """Gets the transformed feature column.""" - if key in self._columns_to_tensors: - return self._columns_to_tensors[key] - if isinstance(key, str): - raise ValueError( - "features dictionary doesn't contain key ({})".format(key)) - if not isinstance(key, _FeatureColumn): - raise TypeError('"key" must be either a "str" or "_FeatureColumn". ' - "Provided: {}".format(key)) - - key.insert_transformed_feature(self._columns_to_tensors) - return self._columns_to_tensors[key] - - -def crossed_column(columns, - hash_bucket_size, - combiner="sum", - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - hash_key=None): - """Creates a _CrossedColumn for performing feature crosses. - - Args: - columns: An iterable of _FeatureColumn. Items can be an instance of - _SparseColumn, _CrossedColumn, or _BucketizedColumn. - hash_bucket_size: An int that is > 1. The number of buckets. - combiner: A string specifying how to reduce if there are multiple entries in - a single row. Currently "mean", "sqrtn" and "sum" are supported, with - "sum" the default. "sqrtn" often achieves good accuracy, in particular - with bag-of-words columns. Each of this can be thought as example level - normalizations on the column:: - * "sum": do not normalize - * "mean": do l1 normalization - * "sqrtn": do l2 normalization - For more information: `tf.embedding_lookup_sparse`. - ckpt_to_load_from: (Optional). String representing checkpoint name/pattern - to restore the column weights. Required if `tensor_name_in_ckpt` is not - None. - tensor_name_in_ckpt: (Optional). Name of the `Tensor` in the provided - checkpoint from which to restore the column weights. Required if - `ckpt_to_load_from` is not None. - hash_key: Specify the hash_key that will be used by the `FingerprintCat64` - function to combine the crosses fingerprints on SparseFeatureCrossOp - (optional). - - Returns: - A _CrossedColumn. - - Raises: - TypeError: if any item in columns is not an instance of _SparseColumn, - _CrossedColumn, or _BucketizedColumn, or - hash_bucket_size is not an int. - ValueError: if hash_bucket_size is not > 1 or - len(columns) is not > 1. - """ - return _CrossedColumn( - columns, - hash_bucket_size, - hash_key, - combiner=combiner, - ckpt_to_load_from=ckpt_to_load_from, - tensor_name_in_ckpt=tensor_name_in_ckpt) - - -class DataFrameColumn(_FeatureColumn, - collections.namedtuple("DataFrameColumn", - ["column_name", "series"])): - """Represents a feature column produced from a `DataFrame`. - - Instances of this class are immutable. A `DataFrame` column may be dense or - sparse, and may have any shape, with the constraint that dimension 0 is - batch_size. - - Args: - column_name: a name for this column - series: a `Series` to be wrapped, which has already had its base features - substituted with `PredefinedSeries`. - """ - - def __new__(cls, column_name, series): - return super(DataFrameColumn, cls).__new__(cls, column_name, series) - - @property - def name(self): - return self.column_name - - @property - def config(self): - return self.series.required_base_features() - - @property - def key(self): - """Returns a string which will be used as a key when we do sorting.""" - return self.name - - def insert_transformed_feature(self, columns_to_tensors): - # The cache must already contain mappings from the expected base feature - # names to Tensors. - - # Passing columns_to_tensors as the cache here means that multiple outputs - # of the transform will be cached, keyed by the repr of their associated - # TransformedSeries. - # The specific requested output ends up in columns_to_tensors twice: once - # keyed by the TransformedSeries repr, and once keyed by this - # DataFrameColumn instance. - columns_to_tensors[self] = self.series.build(columns_to_tensors) - - # pylint: disable=unused-argument - def _to_dnn_input_layer(self, - input_tensor, - weight_collections=None, - trainable=True, - output_rank=2): - if input_tensor.dtype != dtypes.float32: - input_tensor = math_ops.cast(input_tensor, dtypes.float32) - return _reshape_real_valued_tensor(input_tensor, output_rank, self.name) - - def _to_dense_tensor(self, input_tensor): - return self._to_dnn_input_layer(input_tensor) - - def __eq__(self, other): - if isinstance(other, self.__class__): - return self.__dict__ == other.__dict__ - else: - return False - - def __ne__(self, other): - return not self.__eq__(other) - - -def _get_feature_config(feature_column): - """Returns configuration for the base feature defined in feature_column.""" - if not isinstance(feature_column, _FeatureColumn): - raise TypeError( - "feature_columns should only contain instances of _FeatureColumn. " - "Given column is {}".format(feature_column)) - if isinstance(feature_column, - (_SparseColumn, _WeightedSparseColumn, _EmbeddingColumn, - _RealValuedColumn, _RealValuedVarLenColumn, _BucketizedColumn, - _CrossedColumn, _OneHotColumn, _ScatteredEmbeddingColumn)): - return feature_column.config - - raise TypeError("Not supported _FeatureColumn type. " - "Given column is {}".format(feature_column)) - - -def create_feature_spec_for_parsing(feature_columns): - """Helper that prepares features config from input feature_columns. - - The returned feature config can be used as arg 'features' in tf.parse_example. - - Typical usage example: - - ```python - # Define features and transformations - feature_a = sparse_column_with_vocabulary_file(...) - feature_b = real_valued_column(...) - feature_c_bucketized = bucketized_column(real_valued_column("feature_c"), ...) - feature_a_x_feature_c = crossed_column( - columns=[feature_a, feature_c_bucketized], ...) - - feature_columns = set( - [feature_b, feature_c_bucketized, feature_a_x_feature_c]) - batch_examples = tf.io.parse_example( - serialized=serialized_examples, - features=create_feature_spec_for_parsing(feature_columns)) - ``` - - For the above example, create_feature_spec_for_parsing would return the dict: - { - "feature_a": parsing_ops.VarLenFeature(tf.string), - "feature_b": parsing_ops.FixedLenFeature([1], dtype=tf.float32), - "feature_c": parsing_ops.FixedLenFeature([1], dtype=tf.float32) - } - - Args: - feature_columns: An iterable containing all the feature columns. All items - should be instances of classes derived from _FeatureColumn, unless - feature_columns is a dict -- in which case, this should be true of all - values in the dict. - - Returns: - A dict mapping feature keys to FixedLenFeature or VarLenFeature values. - """ - if isinstance(feature_columns, dict): - feature_columns = feature_columns.values() - - features_config = {} - for column in feature_columns: - features_config.update(_get_feature_config(column)) - return features_config - - -def _create_sequence_feature_spec_for_parsing(sequence_feature_columns, - allow_missing_by_default=False): - """Prepares a feature spec for parsing `tf.SequenceExample`s. - - Args: - sequence_feature_columns: an iterable containing all the feature columns. - All items should be instances of classes derived from `_FeatureColumn`. - allow_missing_by_default: whether to set `allow_missing=True` by default for - `FixedLenSequenceFeature`s. - - Returns: - A dict mapping feature keys to `FixedLenSequenceFeature` or `VarLenFeature`. - """ - feature_spec = create_feature_spec_for_parsing(sequence_feature_columns) - sequence_feature_spec = {} - for key, feature in feature_spec.items(): - if isinstance(feature, parsing_ops.VarLenFeature): - sequence_feature = feature - elif (isinstance(feature, parsing_ops.FixedLenFeature) or - isinstance(feature, parsing_ops.FixedLenSequenceFeature)): - default_is_set = feature.default_value is not None - if default_is_set: - logging.warning( - 'Found default value {} for feature "{}". Ignoring this value and ' - "setting `allow_missing=True` instead.".format( - feature.default_value, key)) - sequence_feature = parsing_ops.FixedLenSequenceFeature( - shape=feature.shape, - dtype=feature.dtype, - allow_missing=(allow_missing_by_default or default_is_set)) - else: - raise TypeError("Unsupported feature type: {}".format( - type(feature).__name__)) - sequence_feature_spec[key] = sequence_feature - return sequence_feature_spec - - -def make_place_holder_tensors_for_base_features(feature_columns): - """Returns placeholder tensors for inference. - - Args: - feature_columns: An iterable containing all the feature columns. All items - should be instances of classes derived from _FeatureColumn. - - Returns: - A dict mapping feature keys to SparseTensors (sparse columns) or - placeholder Tensors (dense columns). - """ - # Get dict mapping features to FixedLenFeature or VarLenFeature values. - dict_for_parse_example = create_feature_spec_for_parsing(feature_columns) - placeholders = {} - for column_name, column_type in dict_for_parse_example.items(): - if isinstance(column_type, parsing_ops.VarLenFeature): - # Sparse placeholder for sparse tensors. - placeholders[column_name] = array_ops.sparse_placeholder( - column_type.dtype, name="Placeholder_{}".format(column_name)) - else: - # Simple placeholder for dense tensors. - placeholders[column_name] = array_ops.placeholder( - column_type.dtype, - shape=(None, column_type.shape[0]), - name="Placeholder_{}".format(column_name)) - return placeholders - - -class _SparseIdLookupConfig( - collections.namedtuple("_SparseIdLookupConfig", [ - "vocabulary_file", "keys", "num_oov_buckets", "vocab_size", - "default_value" - ])): - """Defines lookup configuration for a sparse feature. - - An immutable object defines lookup table configuration used by - tf.feature_to_id_v2. - - Attributes: - vocabulary_file: The vocabulary filename. vocabulary_file cannot be combined - with keys. - keys: A 1-D string iterable that specifies the mapping of strings to - indices. It means a feature in keys will map to it's index in keys. - num_oov_buckets: The number of out-of-vocabulary buckets. If zero all out of - vocabulary features will be ignored. - vocab_size: Number of the elements in the vocabulary. - default_value: The value to use for out-of-vocabulary feature values. - Defaults to -1. - """ - - def __new__(cls, - vocabulary_file=None, - keys=None, - num_oov_buckets=0, - vocab_size=None, - default_value=-1): - - return super(_SparseIdLookupConfig, - cls).__new__(cls, vocabulary_file, keys, num_oov_buckets, - vocab_size, default_value) diff --git a/tensorflow/contrib/layers/python/layers/feature_column_ops.py b/tensorflow/contrib/layers/python/layers/feature_column_ops.py deleted file mode 100644 index 37594fb81dd..00000000000 --- a/tensorflow/contrib/layers/python/layers/feature_column_ops.py +++ /dev/null @@ -1,919 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities related to FeatureColumn.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -from tensorflow.contrib.framework.python.framework import experimental -from tensorflow.contrib.framework.python.ops import variables as contrib_variables -from tensorflow.contrib.layers.python.layers import embedding_ops -from tensorflow.contrib.layers.python.layers import feature_column as fc -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor as sparse_tensor_py -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util import nest - - -def _maybe_reshape_input_tensor(tensor, column_name, output_rank): - """Reshape the input tensor by the following rule. - - 1. If `output_rank > input_rank + 1`, raise a `ValueError`. - 2. If `output_rank == input_rank + 1`, expand the tensor by one dimension. - 3. If `output_rank == input_rank`, do nothing. - 4. If `output_rank < input_rank`, flatten the inner dimensions of the tensor. - - Args: - tensor: A Tensor or SparseTensor to be reshaped. - column_name: A string name of the feature column for the tensor. - output_rank: the desired rank of the tensor. - Returns: - A reshaped Tensor or SparseTensor. - Raises: - ValueError: if `output_rank > input_rank + 1` for the input tensor. - """ - input_rank = tensor.get_shape().ndims - - if input_rank is None and isinstance(tensor, sparse_tensor_py.SparseTensor): - # Try to get the rank of a sparse tensor by its dense_shape's shape. - input_rank = tensor.dense_shape.get_shape().as_list()[0] - - if input_rank is None: - raise ValueError('Error while processing column {}. Rank of input Tensor ' - 'can not be None.'.format(column_name)) - - if output_rank > input_rank + 1: - raise ValueError('Error while processing column {}. Rank of input Tensor ' - '({}) should be the same as output_rank ({}). For ' - 'example, sequence data should typically be 3 ' - 'dimensional (rank 3) while non-sequence data is ' - 'typically 2 dimensional (rank 2).'.format( - column_name, input_rank, output_rank)) - elif output_rank == input_rank + 1: - # Expand the tensor's shape by 1 dimension. - if isinstance(tensor, sparse_tensor_py.SparseTensor): - output_shape = array_ops.concat([tensor.dense_shape, [1]], 0) - return sparse_ops.sparse_reshape(tensor, output_shape) - else: - reshaped = array_ops.expand_dims(tensor, -1) - # Try to calculate the new shape. - static_shape = tensor.get_shape() - if static_shape is not None and static_shape.dims is not None: - reshaped.set_shape(static_shape.as_list() + [1]) - return reshaped - elif output_rank < input_rank: - return layers._inner_flatten(tensor, output_rank) # pylint: disable=protected-access - else: - return tensor - - -def _input_from_feature_columns(columns_to_tensors, - feature_columns, - weight_collections, - trainable, - scope, - output_rank, - default_name, - cols_to_outs=None): - """Implementation of `input_from(_sequence)_feature_columns`.""" - columns_to_tensors = columns_to_tensors.copy() - check_feature_columns(feature_columns) - if cols_to_outs is not None and not isinstance(cols_to_outs, dict): - raise ValueError('cols_to_outs must be a dict unless None') - with variable_scope.variable_scope(scope, - default_name=default_name, - values=columns_to_tensors.values()): - output_tensors = [] - transformer = _Transformer(columns_to_tensors) - if weight_collections: - weight_collections = list(set(list(weight_collections) + - [ops.GraphKeys.GLOBAL_VARIABLES])) - - for column in sorted(set(feature_columns), key=lambda x: x.key): - with variable_scope.variable_scope(None, - default_name=column.name, - values=columns_to_tensors.values()): - transformed_tensor = transformer.transform(column) - if output_rank == 3: - transformed_tensor = nest.map_structure( - functools.partial( - _maybe_reshape_input_tensor, - column_name=column.name, - output_rank=output_rank), transformed_tensor) - try: - # pylint: disable=protected-access - arguments = column._deep_embedding_lookup_arguments( - transformed_tensor) - output_tensors.append( - fc._embeddings_from_arguments( # pylint: disable=protected-access - column, - arguments, - weight_collections, - trainable, - output_rank=output_rank)) - - except NotImplementedError as ee: - try: - # pylint: disable=protected-access - output_tensors.append(column._to_dnn_input_layer( - transformed_tensor, - weight_collections, - trainable, - output_rank=output_rank)) - except ValueError as e: - raise ValueError('Error creating input layer for column: {}.\n' - '{}, {}'.format(column.name, e, ee)) - if cols_to_outs is not None: - cols_to_outs[column] = output_tensors[-1] - return array_ops.concat(output_tensors, output_rank - 1) - - -def input_from_feature_columns(columns_to_tensors, - feature_columns, - weight_collections=None, - trainable=True, - scope=None, - cols_to_outs=None): - """A tf.contrib.layers style input layer builder based on FeatureColumns. - - Generally a single example in training data is described with feature columns. - At the first layer of the model, this column oriented data should be converted - to a single tensor. Each feature column needs a different kind of operation - during this conversion. For example sparse features need a totally different - handling than continuous features. - - Example: - - ```python - # Building model for training - columns_to_tensor = tf.io.parse_example(...) - first_layer = input_from_feature_columns( - columns_to_tensors=columns_to_tensor, - feature_columns=feature_columns) - second_layer = fully_connected(inputs=first_layer, ...) - ... - ``` - - where feature_columns can be defined as follows: - - ```python - sparse_feature = sparse_column_with_hash_bucket( - column_name="sparse_col", ...) - sparse_feature_emb = embedding_column(sparse_id_column=sparse_feature, ...) - real_valued_feature = real_valued_column(...) - real_valued_buckets = bucketized_column( - source_column=real_valued_feature, ...) - - feature_columns=[sparse_feature_emb, real_valued_buckets] - ``` - - Args: - columns_to_tensors: A mapping from feature column to tensors. 'string' key - means a base feature (not-transformed). It can have FeatureColumn as a - key too. That means that FeatureColumn is already transformed by input - pipeline. - feature_columns: A set containing all the feature columns. All items in the - set should be instances of classes derived by FeatureColumn. - weight_collections: List of graph collections to which weights are added. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). - scope: Optional scope for variable_scope. - cols_to_outs: Optional dict from feature column to output tensor, - which is concatenated into the returned tensor. - - Returns: - A Tensor which can be consumed by hidden layers in the neural network. - - Raises: - ValueError: if FeatureColumn cannot be consumed by a neural network. - """ - return _input_from_feature_columns(columns_to_tensors, - feature_columns, - weight_collections, - trainable, - scope, - output_rank=2, - default_name='input_from_feature_columns', - cols_to_outs=cols_to_outs) - - -@experimental -def sequence_input_from_feature_columns(columns_to_tensors, - feature_columns, - weight_collections=None, - trainable=True, - scope=None): - """Builds inputs for sequence models from `FeatureColumn`s. - - See documentation for `input_from_feature_columns`. The following types of - `FeatureColumn` are permitted in `feature_columns`: `_OneHotColumn`, - `_EmbeddingColumn`, `_ScatteredEmbeddingColumn`, `_RealValuedColumn`, - `_DataFrameColumn`. In addition, columns in `feature_columns` may not be - constructed using any of the following: `ScatteredEmbeddingColumn`, - `BucketizedColumn`, `CrossedColumn`. - - Args: - columns_to_tensors: A mapping from feature column to tensors. 'string' key - means a base feature (not-transformed). It can have FeatureColumn as a - key too. That means that FeatureColumn is already transformed by input - pipeline. - feature_columns: A set containing all the feature columns. All items in the - set should be instances of classes derived by FeatureColumn. - weight_collections: List of graph collections to which weights are added. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). - scope: Optional scope for variable_scope. - - Returns: - A Tensor which can be consumed by hidden layers in the neural network. - - Raises: - ValueError: if FeatureColumn cannot be consumed by a neural network. - """ - _check_supported_sequence_columns(feature_columns) - _check_forbidden_sequence_columns(feature_columns) - - return _input_from_feature_columns( - columns_to_tensors, - feature_columns, - weight_collections, - trainable, - scope, - output_rank=3, - default_name='sequence_input_from_feature_columns') - - -def _create_embedding_lookup(column, - columns_to_tensors, - embedding_lookup_arguments, - num_outputs, - trainable, - weight_collections): - """Creates variables and returns predictions for linear weights in a model. - - Args: - column: the column we're working on. - columns_to_tensors: a map from column name to tensors. - embedding_lookup_arguments: arguments for embedding lookup. - num_outputs: how many outputs. - trainable: whether the variable we create is trainable. - weight_collections: weights will be placed here. - - Returns: - variables: the created embeddings. - predictions: the computed predictions. - """ - with variable_scope.variable_scope( - None, default_name=column.name, values=columns_to_tensors.values()): - variable = contrib_variables.model_variable( - name='weights', - shape=[embedding_lookup_arguments.vocab_size, num_outputs], - dtype=dtypes.float32, - initializer=embedding_lookup_arguments.initializer, - trainable=trainable, - collections=weight_collections) - if fc._is_variable(variable): # pylint: disable=protected-access - variable = [variable] - else: - variable = variable._get_variable_list() # pylint: disable=protected-access - predictions = embedding_ops.safe_embedding_lookup_sparse( - variable, - embedding_lookup_arguments.input_tensor, - sparse_weights=embedding_lookup_arguments.weight_tensor, - combiner=embedding_lookup_arguments.combiner, - name=column.name + '_weights') - return variable, predictions - - -def _create_joint_embedding_lookup(columns_to_tensors, - embedding_lookup_arguments, - num_outputs, - trainable, - weight_collections): - """Creates an embedding lookup for all columns sharing a single weight.""" - for arg in embedding_lookup_arguments: - assert arg.weight_tensor is None, ( - 'Joint sums for weighted sparse columns are not supported. ' - 'Please use weighted_sum_from_feature_columns instead.') - assert arg.combiner == 'sum', ( - 'Combiners other than sum are not supported for joint sums. ' - 'Please use weighted_sum_from_feature_columns instead.') - assert len(embedding_lookup_arguments) >= 1, ( - 'At least one column must be in the model.') - prev_size = 0 - sparse_tensors = [] - for a in embedding_lookup_arguments: - t = a.input_tensor - values = t.values + prev_size - prev_size += a.vocab_size - sparse_tensors.append( - sparse_tensor_py.SparseTensor(t.indices, - values, - t.dense_shape)) - sparse_tensor = sparse_ops.sparse_concat(1, sparse_tensors) - with variable_scope.variable_scope( - None, default_name='linear_weights', values=columns_to_tensors.values()): - variable = contrib_variables.model_variable( - name='weights', - shape=[prev_size, num_outputs], - dtype=dtypes.float32, - initializer=init_ops.zeros_initializer(), - trainable=trainable, - collections=weight_collections) - if fc._is_variable(variable): # pylint: disable=protected-access - variable = [variable] - else: - variable = variable._get_variable_list() # pylint: disable=protected-access - predictions = embedding_ops.safe_embedding_lookup_sparse( - variable, - sparse_tensor, - sparse_weights=None, - combiner='sum', - name='_weights') - return variable, predictions - - -def joint_weighted_sum_from_feature_columns(columns_to_tensors, - feature_columns, - num_outputs, - weight_collections=None, - trainable=True, - scope=None): - """A restricted linear prediction builder based on FeatureColumns. - - As long as all feature columns are unweighted sparse columns this computes the - prediction of a linear model which stores all weights in a single variable. - - Args: - columns_to_tensors: A mapping from feature column to tensors. 'string' key - means a base feature (not-transformed). It can have FeatureColumn as a - key too. That means that FeatureColumn is already transformed by input - pipeline. For example, `inflow` may have handled transformations. - feature_columns: A set containing all the feature columns. All items in the - set should be instances of classes derived from FeatureColumn. - num_outputs: An integer specifying number of outputs. Default value is 1. - weight_collections: List of graph collections to which weights are added. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). - scope: Optional scope for variable_scope. - - Returns: - A tuple containing: - - * A Tensor which represents predictions of a linear model. - * A list of Variables storing the weights. - * A Variable which is used for bias. - - Raises: - ValueError: if FeatureColumn cannot be used for linear predictions. - - """ - columns_to_tensors = columns_to_tensors.copy() - check_feature_columns(feature_columns) - with variable_scope.variable_scope( - scope, - default_name='joint_weighted_sum_from_feature_columns', - values=columns_to_tensors.values()): - transformer = _Transformer(columns_to_tensors) - embedding_lookup_arguments = [] - for column in sorted(set(feature_columns), key=lambda x: x.key): - transformed_tensor = transformer.transform(column) - try: - embedding_lookup_arguments.append( - column._wide_embedding_lookup_arguments(transformed_tensor)) # pylint: disable=protected-access - except NotImplementedError: - raise NotImplementedError('Real-valued columns are not supported. ' - 'Use weighted_sum_from_feature_columns ' - 'instead, or bucketize these columns.') - - variable, predictions_no_bias = _create_joint_embedding_lookup( - columns_to_tensors, - embedding_lookup_arguments, - num_outputs, - trainable, - weight_collections) - bias = contrib_variables.model_variable( - 'bias_weight', - shape=[num_outputs], - initializer=init_ops.zeros_initializer(), - trainable=trainable, - collections=_add_variable_collection(weight_collections)) - _log_variable(bias) - predictions = nn_ops.bias_add(predictions_no_bias, bias) - - return predictions, variable, bias - - -def weighted_sum_from_feature_columns(columns_to_tensors, - feature_columns, - num_outputs, - weight_collections=None, - trainable=True, - scope=None): - """A tf.contrib.layers style linear prediction builder based on FeatureColumn. - - Generally a single example in training data is described with feature columns. - This function generates weighted sum for each num_outputs. Weighted sum refers - to logits in classification problems. It refers to prediction itself for - linear regression problems. - - Example: - - ``` - # Building model for training - feature_columns = ( - real_valued_column("my_feature1"), - ... - ) - columns_to_tensor = tf.io.parse_example(...) - logits = weighted_sum_from_feature_columns( - columns_to_tensors=columns_to_tensor, - feature_columns=feature_columns, - num_outputs=1) - loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, - logits=logits) - ``` - - Args: - columns_to_tensors: A mapping from feature column to tensors. 'string' key - means a base feature (not-transformed). It can have FeatureColumn as a - key too. That means that FeatureColumn is already transformed by input - pipeline. For example, `inflow` may have handled transformations. - feature_columns: A set containing all the feature columns. All items in the - set should be instances of classes derived from FeatureColumn. - num_outputs: An integer specifying number of outputs. Default value is 1. - weight_collections: List of graph collections to which weights are added. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). - scope: Optional scope for variable_scope. - - Returns: - A tuple containing: - - * A Tensor which represents predictions of a linear model. - * A dictionary which maps feature_column to corresponding Variable. - * A Variable which is used for bias. - - Raises: - ValueError: if FeatureColumn cannot be used for linear predictions. - """ - columns_to_tensors = columns_to_tensors.copy() - check_feature_columns(feature_columns) - with variable_scope.variable_scope( - scope, - default_name='weighted_sum_from_feature_columns', - values=columns_to_tensors.values()): - output_tensors = [] - column_to_variable = {} - transformer = _Transformer(columns_to_tensors) - # pylint: disable=protected-access - for column in sorted(set(feature_columns), key=lambda x: x.key): - transformed_tensor = transformer.transform(column) - try: - embedding_lookup_arguments = column._wide_embedding_lookup_arguments( - transformed_tensor) - variable, predictions = _create_embedding_lookup( - column, - columns_to_tensors, - embedding_lookup_arguments, - num_outputs, - trainable, - weight_collections) - except NotImplementedError: - with variable_scope.variable_scope( - None, - default_name=column.name, - values=columns_to_tensors.values()): - tensor = column._to_dense_tensor(transformed_tensor) - tensor = _maybe_reshape_input_tensor( - tensor, column.name, output_rank=2) - variable = [ - contrib_variables.model_variable( - name='weight', - shape=[tensor.get_shape()[1], num_outputs], - initializer=init_ops.zeros_initializer(), - trainable=trainable, - collections=weight_collections) - ] - predictions = math_ops.matmul(tensor, variable[0], name='matmul') - except ValueError as ee: - raise ValueError('Error creating weighted sum for column: {}.\n' - '{}'.format(column.name, ee)) - output_tensors.append(array_ops.reshape( - predictions, shape=(-1, num_outputs))) - column_to_variable[column] = variable - _log_variable(variable) - fc._maybe_restore_from_checkpoint(column._checkpoint_path(), variable) # pylint: disable=protected-access - # pylint: enable=protected-access - predictions_no_bias = math_ops.add_n(output_tensors) - bias = contrib_variables.model_variable( - 'bias_weight', - shape=[num_outputs], - initializer=init_ops.zeros_initializer(), - trainable=trainable, - collections=_add_variable_collection(weight_collections)) - _log_variable(bias) - predictions = nn_ops.bias_add(predictions_no_bias, bias) - - return predictions, column_to_variable, bias - - -def parse_feature_columns_from_examples(serialized, - feature_columns, - name=None, - example_names=None): - """Parses tf.Examples to extract tensors for given feature_columns. - - This is a wrapper of 'tf.io.parse_example'. - - Example: - - ```python - columns_to_tensor = parse_feature_columns_from_examples( - serialized=my_data, - feature_columns=my_features) - - # Where my_features are: - # Define features and transformations - sparse_feature_a = sparse_column_with_keys( - column_name="sparse_feature_a", keys=["AB", "CD", ...]) - - embedding_feature_a = embedding_column( - sparse_id_column=sparse_feature_a, dimension=3, combiner="sum") - - sparse_feature_b = sparse_column_with_hash_bucket( - column_name="sparse_feature_b", hash_bucket_size=1000) - - embedding_feature_b = embedding_column( - sparse_id_column=sparse_feature_b, dimension=16, combiner="sum") - - crossed_feature_a_x_b = crossed_column( - columns=[sparse_feature_a, sparse_feature_b], hash_bucket_size=10000) - - real_feature = real_valued_column("real_feature") - real_feature_buckets = bucketized_column( - source_column=real_feature, boundaries=[...]) - - my_features = [embedding_feature_b, real_feature_buckets, embedding_feature_a] - ``` - - Args: - serialized: A vector (1-D Tensor) of strings, a batch of binary - serialized `Example` protos. - feature_columns: An iterable containing all the feature columns. All items - should be instances of classes derived from _FeatureColumn. - name: A name for this operation (optional). - example_names: A vector (1-D Tensor) of strings (optional), the names of - the serialized protos in the batch. - - Returns: - A `dict` mapping FeatureColumn to `Tensor` and `SparseTensor` values. - """ - check_feature_columns(feature_columns) - columns_to_tensors = parsing_ops.parse_example( - serialized=serialized, - features=fc.create_feature_spec_for_parsing(feature_columns), - name=name, - example_names=example_names) - - transformer = _Transformer(columns_to_tensors) - for column in sorted(set(feature_columns), key=lambda x: x.key): - transformer.transform(column) - return columns_to_tensors - - -def transform_features(features, feature_columns): - """Returns transformed features based on features columns passed in. - - Example: - - ```python - columns_to_tensor = transform_features(features=features, - feature_columns=feature_columns) - - # Where my_features are: - # Define features and transformations - sparse_feature_a = sparse_column_with_keys( - column_name="sparse_feature_a", keys=["AB", "CD", ...]) - - embedding_feature_a = embedding_column( - sparse_id_column=sparse_feature_a, dimension=3, combiner="sum") - - sparse_feature_b = sparse_column_with_hash_bucket( - column_name="sparse_feature_b", hash_bucket_size=1000) - - embedding_feature_b = embedding_column( - sparse_id_column=sparse_feature_b, dimension=16, combiner="sum") - - crossed_feature_a_x_b = crossed_column( - columns=[sparse_feature_a, sparse_feature_b], hash_bucket_size=10000) - - real_feature = real_valued_column("real_feature") - real_feature_buckets = bucketized_column( - source_column=real_feature, boundaries=[...]) - - feature_columns = [embedding_feature_b, - real_feature_buckets, - embedding_feature_a] - ``` - - Args: - features: A dictionary of features. - feature_columns: An iterable containing all the feature columns. All items - should be instances of classes derived from _FeatureColumn. - - Returns: - A `dict` mapping FeatureColumn to `Tensor` and `SparseTensor` values. - """ - columns_to_tensor = features.copy() - check_feature_columns(feature_columns) - transformer = _Transformer(columns_to_tensor) - for column in sorted(set(feature_columns), key=lambda x: x.key): - transformer.transform(column) - keys = list(columns_to_tensor.keys()) - for k in keys: - if k not in feature_columns: - columns_to_tensor.pop(k) - return columns_to_tensor - - -def parse_feature_columns_from_sequence_examples( - serialized, - context_feature_columns, - sequence_feature_columns, - name=None, - example_name=None): - """Parses tf.SequenceExamples to extract tensors for given `FeatureColumn`s. - - Args: - serialized: A scalar (0-D Tensor) of type string, a single serialized - `SequenceExample` proto. - context_feature_columns: An iterable containing the feature columns for - context features. All items should be instances of classes derived from - `_FeatureColumn`. Can be `None`. - sequence_feature_columns: An iterable containing the feature columns for - sequence features. All items should be instances of classes derived from - `_FeatureColumn`. Can be `None`. - name: A name for this operation (optional). - example_name: A scalar (0-D Tensor) of type string (optional), the names of - the serialized proto. - - Returns: - A tuple consisting of (context_features, sequence_features) - - * context_features: a dict mapping `FeatureColumns` from - `context_feature_columns` to their parsed `Tensors`/`SparseTensor`s. - * sequence_features: a dict mapping `FeatureColumns` from - `sequence_feature_columns` to their parsed `Tensors`/`SparseTensor`s. - """ - # Sequence example parsing requires a single (scalar) example. - try: - serialized = array_ops.reshape(serialized, []) - except ValueError as e: - raise ValueError( - 'serialized must contain as single sequence example. Batching must be ' - 'done after parsing for sequence examples. Error: {}'.format(e)) - - if context_feature_columns is None: - context_feature_columns = [] - if sequence_feature_columns is None: - sequence_feature_columns = [] - - check_feature_columns(context_feature_columns) - context_feature_spec = fc.create_feature_spec_for_parsing( - context_feature_columns) - - check_feature_columns(sequence_feature_columns) - sequence_feature_spec = fc._create_sequence_feature_spec_for_parsing( # pylint: disable=protected-access - sequence_feature_columns, allow_missing_by_default=False) - - return parsing_ops.parse_single_sequence_example(serialized, - context_feature_spec, - sequence_feature_spec, - example_name, - name) - - -def _log_variable(variable): - if isinstance(variable, list): - for var in variable: - if fc._is_variable(variable): # pylint: disable=protected-access - logging.info('Created variable %s, with device=%s', var.name, - var.device) - elif fc._is_variable(variable): # pylint: disable=protected-access - logging.info('Created variable %s, with device=%s', variable.name, - variable.device) - - -def _infer_real_valued_column_for_tensor(name, tensor): - """Creates a real_valued_column for given tensor and name.""" - if isinstance(tensor, sparse_tensor_py.SparseTensor): - raise ValueError( - 'SparseTensor is not supported for auto detection. Please define ' - 'corresponding FeatureColumn for tensor {} {}.', name, tensor) - - if not (tensor.dtype.is_integer or tensor.dtype.is_floating): - raise ValueError( - 'Non integer or non floating types are not supported for auto detection' - '. Please define corresponding FeatureColumn for tensor {} {}.', name, - tensor) - - shape = tensor.get_shape().as_list() - dimension = 1 - for i in range(1, len(shape)): - dimension *= shape[i] - return fc.real_valued_column(name, dimension=dimension, dtype=tensor.dtype) - - -def infer_real_valued_columns(features): - if not isinstance(features, dict): - return [_infer_real_valued_column_for_tensor('', features)] - - feature_columns = [] - for key, value in features.items(): - feature_columns.append(_infer_real_valued_column_for_tensor(key, value)) - - return feature_columns - - -def check_feature_columns(feature_columns): - """Checks the validity of the set of FeatureColumns. - - Args: - feature_columns: An iterable of instances or subclasses of FeatureColumn. - - Raises: - ValueError: If `feature_columns` is a dict. - ValueError: If there are duplicate feature column keys. - """ - if isinstance(feature_columns, dict): - raise ValueError('Expected feature_columns to be iterable, found dict.') - seen_keys = set() - for f in feature_columns: - key = f.key - if key in seen_keys: - raise ValueError('Duplicate feature column key found for column: {}. ' - 'This usually means that the column is almost identical ' - 'to another column, and one must be discarded.'.format( - f.name)) - seen_keys.add(key) - - -class _Transformer(object): - """Handles all the transformations defined by FeatureColumn if needed. - - FeatureColumn specifies how to digest an input column to the network. Some - feature columns require data transformations. This class handles those - transformations if they are not handled already. - - Some features may be used in more than one place. For example, one can use a - bucketized feature by itself and a cross with it. In that case Transformer - should create only one bucketization op instead of multiple ops for each - feature column. To handle re-use of transformed columns, Transformer keeps all - previously transformed columns. - - Example: - - ```python - sparse_feature = sparse_column_with_hash_bucket(...) - real_valued_feature = real_valued_column(...) - real_valued_buckets = bucketized_column(source_column=real_valued_feature, - ...) - sparse_x_real = crossed_column( - columns=[sparse_feature, real_valued_buckets], hash_bucket_size=10000) - - columns_to_tensor = tf.io.parse_example(...) - transformer = Transformer(columns_to_tensor) - - sparse_x_real_tensor = transformer.transform(sparse_x_real) - sparse_tensor = transformer.transform(sparse_feature) - real_buckets_tensor = transformer.transform(real_valued_buckets) - ``` - """ - - def __init__(self, columns_to_tensors): - """Initializes transformer. - - Args: - columns_to_tensors: A mapping from feature columns to tensors. 'string' - key means a base feature (not-transformed). It can have FeatureColumn as - a key too. That means that FeatureColumn is already transformed by input - pipeline. For example, `inflow` may have handled transformations. - Transformed features are inserted in columns_to_tensors. - """ - self._columns_to_tensors = columns_to_tensors - - def transform(self, feature_column): - """Returns a Tensor which represents given feature_column. - - Args: - feature_column: An instance of FeatureColumn. - - Returns: - A Tensor which represents given feature_column. It may create a new Tensor - or re-use an existing one. - - Raises: - ValueError: if FeatureColumn cannot be handled by this Transformer. - """ - logging.debug('Transforming feature_column %s', feature_column) - if feature_column in self._columns_to_tensors: - # Feature_column is already transformed. - return self._columns_to_tensors[feature_column] - - feature_column.insert_transformed_feature(self._columns_to_tensors) - - if feature_column not in self._columns_to_tensors: - raise ValueError('Column {} is not supported.'.format( - feature_column.name)) - - return self._columns_to_tensors[feature_column] - - -def _add_variable_collection(weight_collections): - if weight_collections: - weight_collections = list( - set(list(weight_collections) + [ops.GraphKeys.GLOBAL_VARIABLES])) - return weight_collections - - -# TODO(jamieas): remove the following logic once all FeatureColumn types are -# supported for sequences. -# pylint: disable=protected-access -_SUPPORTED_SEQUENCE_COLUMNS = (fc._OneHotColumn, - fc._EmbeddingColumn, - fc._RealValuedColumn, - fc._RealValuedVarLenColumn) - -_FORBIDDEN_SEQUENCE_COLUMNS = (fc._ScatteredEmbeddingColumn, - fc._BucketizedColumn, - fc._CrossedColumn) - - -def _check_supported_sequence_columns(feature_columns): - """Asserts `feature_columns` are in `_SUPPORTED_SEQUENCE_COLUMNS`.""" - for col in feature_columns: - if not isinstance(col, _SUPPORTED_SEQUENCE_COLUMNS): - raise ValueError( - 'FeatureColumn type {} is not currently supported for sequence data.'. - format(type(col).__name__)) - - -def _get_parent_columns(feature_column): - """Returns the tuple of `FeatureColumn`s that `feature_column` depends on.""" - if isinstance(feature_column, (fc._WeightedSparseColumn, - fc._OneHotColumn, - fc._EmbeddingColumn,)): - return (feature_column.sparse_id_column,) - if isinstance(feature_column, (fc._BucketizedColumn,)): - return (feature_column.source_column,) - if isinstance(feature_column, (fc._CrossedColumn)): - return tuple(feature_column.columns) - return tuple() - - -def _gather_feature_columns(feature_columns): - """Returns a list of all ancestor `FeatureColumns` of `feature_columns`.""" - gathered = list(feature_columns) - i = 0 - while i < len(gathered): - for column in _get_parent_columns(gathered[i]): - if column not in gathered: - gathered.append(column) - i += 1 - return gathered - - -def _check_forbidden_sequence_columns(feature_columns): - """Recursively checks `feature_columns` for `_FORBIDDEN_SEQUENCE_COLUMNS`.""" - all_feature_columns = _gather_feature_columns(feature_columns) - for feature_column in all_feature_columns: - if isinstance(feature_column, _FORBIDDEN_SEQUENCE_COLUMNS): - raise ValueError( - 'Column {} is of type {}, which is not currently supported for ' - 'sequences.'.format(feature_column.name, - type(feature_column).__name__)) diff --git a/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py b/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py deleted file mode 100644 index 00e41026d00..00000000000 --- a/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py +++ /dev/null @@ -1,2803 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for layers.feature_column_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -import numpy as np - -from tensorflow.contrib import layers -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.layers.python.layers import feature_column_ops -from tensorflow.core.example import example_pb2 -from tensorflow.core.example import feature_pb2 -from tensorflow.python.feature_column import feature_column_lib as fc_core -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.platform import test - - -class TransformerTest(test.TestCase): - - def testRealValuedColumnIsIdentityTransformation(self): - real_valued = feature_column.real_valued_column("price") - features = {"price": constant_op.constant([[20.], [110], [-3]])} - output = feature_column_ops._Transformer(features).transform(real_valued) - with self.cached_session(): - self.assertAllEqual(output.eval(), [[20.], [110], [-3]]) - - def testSparseRealValuedColumnIdentityTransformation(self): - sparse_real_valued = feature_column._real_valued_var_len_column( - "rating", is_sparse=True) - rating_tensor = sparse_tensor.SparseTensor( - values=[2.0, 5.0], indices=[[0, 0], [2, 0]], dense_shape=[3, 1]) - features = {"rating": rating_tensor} - output = feature_column_ops._Transformer(features).transform( - sparse_real_valued) - with self.cached_session(): - self.assertAllEqual(output.values.eval(), rating_tensor.values.eval()) - self.assertAllEqual(output.indices.eval(), rating_tensor.indices.eval()) - self.assertAllEqual(output.dense_shape.eval(), - rating_tensor.dense_shape.eval()) - - def testSparseRealValuedColumnWithTransformation(self): - - def square_fn(x): - return x**2 - - sparse_real_valued = feature_column._real_valued_var_len_column( - "rating", normalizer=square_fn, is_sparse=True) - rating_tensor = sparse_tensor.SparseTensor( - values=[2.0, 5.0], indices=[[0, 0], [2, 0]], dense_shape=[3, 1]) - features = {"rating": rating_tensor} - output_dict = feature_column_ops.transform_features(features, - [sparse_real_valued]) - self.assertTrue(sparse_real_valued in output_dict) - output = output_dict[sparse_real_valued] - with self.cached_session(): - self.assertArrayNear(output.values.eval(), [4.0, 25.0], 1e-5) - self.assertAllEqual(output.indices.eval(), rating_tensor.indices.eval()) - self.assertAllEqual(output.dense_shape.eval(), - rating_tensor.dense_shape.eval()) - - def testBucketizedColumn(self): - bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price"), boundaries=[0., 10., 100.]) - # buckets 2, 3, 0 - features = {"price": constant_op.constant([[20.], [110], [-3]])} - - # Test transform features. - output = feature_column_ops.transform_features( - features=features, feature_columns=[bucket]) - self.assertEqual(len(output), 1) - self.assertIn(bucket, output) - with self.cached_session(): - self.assertAllEqual(output[bucket].eval(), [[2], [3], [0]]) - - def testBucketizedColumnWithMultiDimensions(self): - bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price", 2), - boundaries=[0., 10., 100.]) - # buckets 2, 3, 0 - features = { - "price": constant_op.constant([[20., 110], [110., 20], [-3, -3]]) - } - output = feature_column_ops._Transformer(features).transform(bucket) - with self.cached_session(): - self.assertAllEqual(output.eval(), [[2, 3], [3, 2], [0, 0]]) - - def testCachedTransformation(self): - bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price"), boundaries=[0., 10., 100.]) - # buckets 2, 3, 0 - features = {"price": constant_op.constant([[20.], [110], [-3]])} - transformer = feature_column_ops._Transformer(features) - with self.cached_session() as sess: - transformer.transform(bucket) - num_of_ops = len(sess.graph.get_operations()) - # Verify that the second call to transform the same feature - # doesn't increase the number of ops. - transformer.transform(bucket) - self.assertEqual(num_of_ops, len(sess.graph.get_operations())) - - def testSparseColumnWithHashBucket(self): - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - # Test transform features. - output = feature_column_ops.transform_features( - features=features, feature_columns=[hashed_sparse]) - self.assertEqual(len(output), 1) - self.assertIn(hashed_sparse, output) - with self.cached_session(): - self.assertEqual(output[hashed_sparse].values.dtype, dtypes.int64) - self.assertTrue( - all(x < 10 and x >= 0 for x in output[hashed_sparse].values.eval())) - self.assertAllEqual(output[hashed_sparse].indices.eval(), - wire_tensor.indices.eval()) - self.assertAllEqual(output[hashed_sparse].dense_shape.eval(), - wire_tensor.dense_shape.eval()) - - def testSparseIntColumnWithHashBucket(self): - """Tests a sparse column with int values.""" - hashed_sparse = feature_column.sparse_column_with_hash_bucket( - "wire", 10, dtype=dtypes.int64) - wire_tensor = sparse_tensor.SparseTensor( - values=[101, 201, 301], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - # Test transform features. - output = feature_column_ops.transform_features( - features=features, feature_columns=[hashed_sparse]) - self.assertEqual(len(output), 1) - self.assertIn(hashed_sparse, output) - with self.cached_session(): - self.assertEqual(output[hashed_sparse].values.dtype, dtypes.int64) - self.assertTrue( - all(x < 10 and x >= 0 for x in output[hashed_sparse].values.eval())) - self.assertAllEqual(output[hashed_sparse].indices.eval(), - wire_tensor.indices.eval()) - self.assertAllEqual(output[hashed_sparse].dense_shape.eval(), - wire_tensor.dense_shape.eval()) - - def testSparseColumnWithHashBucketWithDenseInputTensor(self): - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_tensor = constant_op.constant( - [["omar", "stringer"], ["marlo", "rick"]]) - features = {"wire": wire_tensor} - output = feature_column_ops._Transformer(features).transform(hashed_sparse) - - with self.cached_session(): - # While the input is a dense Tensor, the output should be a SparseTensor. - self.assertIsInstance(output, sparse_tensor.SparseTensor) - self.assertEqual(output.values.dtype, dtypes.int64) - self.assertTrue(all(x < 10 and x >= 0 for x in output.values.eval())) - self.assertAllEqual(output.indices.eval(), - [[0, 0], [0, 1], [1, 0], [1, 1]]) - self.assertAllEqual(output.dense_shape.eval(), [2, 2]) - - def testEmbeddingColumn(self): - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_embedding = feature_column.embedding_column(hashed_sparse, 10) - - # Test transform features. - output = feature_column_ops.transform_features( - features=features, feature_columns=[hashed_sparse, wire_embedding]) - # Check that features dict haven't changed - self.assertEqual({"wire": wire_tensor}, features) - self.assertEqual(len(output), 2) - self.assertIn(hashed_sparse, output) - self.assertIn(wire_embedding, output) - with self.cached_session(): - self.assertAllEqual(output[wire_embedding].indices.eval(), - wire_tensor.indices.eval()) - self.assertAllEqual(output[wire_embedding].dense_shape.eval(), [2, 2]) - self.assertAllEqual(output[wire_embedding].values.eval(), - output[hashed_sparse].values.eval()) - - def testSparseColumnWithKeys(self): - keys_sparse = feature_column.sparse_column_with_keys( - "wire", ["marlo", "omar", "stringer"]) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - # Test transform features. - output = feature_column_ops.transform_features( - features=features, feature_columns=[keys_sparse]) - self.assertEqual(len(output), 1) - self.assertIn(keys_sparse, output) - with self.cached_session(): - lookup_ops.tables_initializer().run() - self.assertEqual(output[keys_sparse].values.dtype, dtypes.int64) - self.assertAllEqual(output[keys_sparse].values.eval(), [1, 2, 0]) - self.assertAllEqual(output[keys_sparse].indices.eval(), - wire_tensor.indices.eval()) - self.assertAllEqual(output[keys_sparse].dense_shape.eval(), - wire_tensor.dense_shape.eval()) - - def testSparseColumnWithKeysWithDenseInputTensor(self): - keys_sparse = feature_column.sparse_column_with_keys( - "wire", ["marlo", "omar", "stringer", "rick"]) - wire_tensor = constant_op.constant( - [["omar", "stringer"], ["marlo", "rick"]]) - - features = {"wire": wire_tensor} - output = feature_column_ops._Transformer(features).transform(keys_sparse) - - with self.cached_session(): - lookup_ops.tables_initializer().run() - # While the input is a dense Tensor, the output should be a SparseTensor. - self.assertIsInstance(output, sparse_tensor.SparseTensor) - self.assertEqual(output.dtype, dtypes.int64) - self.assertAllEqual(output.values.eval(), [1, 2, 0, 3]) - self.assertAllEqual(output.indices.eval(), - [[0, 0], [0, 1], [1, 0], [1, 1]]) - self.assertAllEqual(output.dense_shape.eval(), [2, 2]) - - def testSparseColumnWithHashBucket_IsIntegerized(self): - hashed_sparse = feature_column.sparse_column_with_integerized_feature( - "wire", 10) - wire_tensor = sparse_tensor.SparseTensor( - values=[100, 1, 25], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - # Test transform features. - output = feature_column_ops.transform_features( - features=features, feature_columns=[hashed_sparse]) - self.assertEqual(len(output), 1) - self.assertIn(hashed_sparse, output) - with self.cached_session(): - self.assertEqual(output[hashed_sparse].values.dtype, dtypes.int32) - self.assertTrue( - all(x < 10 and x >= 0 for x in output[hashed_sparse].values.eval())) - self.assertAllEqual(output[hashed_sparse].indices.eval(), - wire_tensor.indices.eval()) - self.assertAllEqual(output[hashed_sparse].dense_shape.eval(), - wire_tensor.dense_shape.eval()) - - def testSparseColumnWithHashBucketWithDenseInputTensor_IsIntegerized(self): - hashed_sparse = feature_column.sparse_column_with_integerized_feature( - "wire", 10) - # wire_tensor = tf.SparseTensor(values=[100, 1, 25], - # indices=[[0, 0], [1, 0], [1, 1]], - # dense_shape=[2, 2]) - wire_tensor = constant_op.constant([[100, 0], [1, 25]]) - features = {"wire": wire_tensor} - output = feature_column_ops._Transformer(features).transform(hashed_sparse) - with self.cached_session(): - # While the input is a dense Tensor, the output should be a SparseTensor. - self.assertIsInstance(output, sparse_tensor.SparseTensor) - self.assertEqual(output.values.dtype, dtypes.int32) - self.assertTrue(all(x < 10 and x >= 0 for x in output.values.eval())) - self.assertAllEqual(output.indices.eval(), - [[0, 0], [0, 1], [1, 0], [1, 1]]) - self.assertAllEqual(output.dense_shape.eval(), [2, 2]) - - def testWeightedSparseColumn(self): - ids = feature_column.sparse_column_with_keys("ids", - ["marlo", "omar", "stringer"]) - ids_tensor = sparse_tensor.SparseTensor( - values=["stringer", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - weighted_ids = feature_column.weighted_sparse_column(ids, "weights") - weights_tensor = sparse_tensor.SparseTensor( - values=[10.0, 20.0, 30.0], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"ids": ids_tensor, "weights": weights_tensor} - # Test transform features. - output = feature_column_ops.transform_features( - features=features, feature_columns=[weighted_ids]) - self.assertEqual(len(output), 1) - self.assertIn(weighted_ids, output) - - with self.cached_session(): - lookup_ops.tables_initializer().run() - self.assertAllEqual(output[weighted_ids][0].dense_shape.eval(), - ids_tensor.dense_shape.eval()) - self.assertAllEqual(output[weighted_ids][0].indices.eval(), - ids_tensor.indices.eval()) - self.assertAllEqual(output[weighted_ids][0].values.eval(), [2, 2, 0]) - self.assertAllEqual(output[weighted_ids][1].dense_shape.eval(), - weights_tensor.dense_shape.eval()) - self.assertAllEqual(output[weighted_ids][1].indices.eval(), - weights_tensor.indices.eval()) - self.assertEqual(output[weighted_ids][1].values.dtype, dtypes.float32) - self.assertAllEqual(output[weighted_ids][1].values.eval(), - weights_tensor.values.eval()) - - def testSparseColumnWithVocabulary(self): - vocabulary_file = os.path.join(self.get_temp_dir(), "movies.txt") - with open(vocabulary_file, "w") as f: - f.write("\n".join(["marlo", "omar", "stringer"]) + "\n") - vocab_sparse = feature_column.sparse_column_with_vocabulary_file( - "wire", vocabulary_file, vocab_size=3) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - output = feature_column_ops.transform_features( - features=features, feature_columns=[vocab_sparse]) - self.assertEqual(len(output), 1) - self.assertIn(vocab_sparse, output) - with self.cached_session(): - lookup_ops.tables_initializer().run() - self.assertEqual(output[vocab_sparse].values.dtype, dtypes.int64) - self.assertAllEqual(output[vocab_sparse].values.eval(), [1, 2, 0]) - self.assertAllEqual(output[vocab_sparse].indices.eval(), - wire_tensor.indices.eval()) - self.assertAllEqual(output[vocab_sparse].dense_shape.eval(), - wire_tensor.dense_shape.eval()) - - def testSparseColumnWithVocabularyWithDenseInputTensor(self): - vocabulary_file = os.path.join(self.get_temp_dir(), "movies.txt") - with open(vocabulary_file, "w") as f: - f.write("\n".join(["marlo", "omar", "stringer"]) + "\n") - vocab_sparse = feature_column.sparse_column_with_vocabulary_file( - "wire", vocabulary_file, vocab_size=3) - wire_tensor = constant_op.constant( - [["omar", "stringer"], ["marlo", "omar"]]) - features = {"wire": wire_tensor} - output = feature_column_ops.transform_features( - features=features, feature_columns=[vocab_sparse]) - self.assertEqual(len(output), 1) - self.assertIn(vocab_sparse, output) - with self.cached_session(): - lookup_ops.tables_initializer().run() - self.assertEqual(output[vocab_sparse].values.dtype, dtypes.int64) - self.assertAllEqual(output[vocab_sparse].values.eval(), [1, 2, 0, 1]) - self.assertAllEqual(output[vocab_sparse].indices.eval(), - [[0, 0], [0, 1], [1, 0], [1, 1]]) - self.assertAllEqual(output[vocab_sparse].dense_shape.eval(), [2, 2]) - - def testSparseIntColumnWithVocabulary(self): - """Tests a sparse integer column with vocabulary.""" - vocabulary_file = os.path.join(self.get_temp_dir(), "courses.txt") - with open(vocabulary_file, "w") as f: - f.write("\n".join(["101", "201", "301"]) + "\n") - vocab_sparse = feature_column.sparse_column_with_vocabulary_file( - "wire", vocabulary_file, vocab_size=3, dtype=dtypes.int64) - wire_tensor = sparse_tensor.SparseTensor( - values=[201, 301, 101], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - output = feature_column_ops.transform_features( - features=features, feature_columns=[vocab_sparse]) - self.assertEqual(len(output), 1) - self.assertIn(vocab_sparse, output) - with self.cached_session(): - lookup_ops.tables_initializer().run() - self.assertEqual(output[vocab_sparse].values.dtype, dtypes.int64) - self.assertAllEqual(output[vocab_sparse].values.eval(), [1, 2, 0]) - self.assertAllEqual(output[vocab_sparse].indices.eval(), - wire_tensor.indices.eval()) - self.assertAllEqual(output[vocab_sparse].dense_shape.eval(), - wire_tensor.dense_shape.eval()) - - def testSparseIntColumnWithVocabularyWithDenseInputTensor(self): - """Tests a sparse integer column with vocabulary.""" - vocabulary_file = os.path.join(self.get_temp_dir(), "courses.txt") - with open(vocabulary_file, "w") as f: - f.write("\n".join(["101", "201", "301"]) + "\n") - vocab_sparse = feature_column.sparse_column_with_vocabulary_file( - "wire", vocabulary_file, vocab_size=3, dtype=dtypes.int64) - wire_tensor = constant_op.constant([[201, 301], [101, 201]]) - features = {"wire": wire_tensor} - output = feature_column_ops.transform_features( - features=features, feature_columns=[vocab_sparse]) - self.assertEqual(len(output), 1) - self.assertIn(vocab_sparse, output) - with self.cached_session(): - lookup_ops.tables_initializer().run() - self.assertEqual(output[vocab_sparse].values.dtype, dtypes.int64) - self.assertAllEqual(output[vocab_sparse].values.eval(), [1, 2, 0, 1]) - self.assertAllEqual(output[vocab_sparse].indices.eval(), - [[0, 0], [0, 1], [1, 0], [1, 1]]) - self.assertAllEqual(output[vocab_sparse].dense_shape.eval(), [2, 2]) - - def testCrossColumn(self): - language = feature_column.sparse_column_with_hash_bucket( - "language", hash_bucket_size=3) - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=5) - country_language = feature_column.crossed_column( - [language, country], hash_bucket_size=15) - features = { - "language": - sparse_tensor.SparseTensor( - values=["english", "spanish"], - indices=[[0, 0], [1, 0]], - dense_shape=[2, 1]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV"], - indices=[[0, 0], [1, 0]], - dense_shape=[2, 1]) - } - # Test transform features. - output = feature_column_ops.transform_features( - features=features, feature_columns=[country_language]) - self.assertEqual(len(output), 1) - self.assertIn(country_language, output) - with self.cached_session(): - self.assertEqual(output[country_language].values.dtype, dtypes.int64) - self.assertTrue( - all(x < 15 and x >= 0 for x in output[country_language].values.eval( - ))) - - def testCrossWithBucketizedColumn(self): - price_bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price"), boundaries=[0., 10., 100.]) - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=5) - country_price = feature_column.crossed_column( - [country, price_bucket], hash_bucket_size=15) - features = { - "price": - constant_op.constant([[20.]]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV"], - indices=[[0, 0], [0, 1]], - dense_shape=[1, 2]) - } - # Test transform features. - output = feature_column_ops.transform_features( - features=features, feature_columns=[country_price]) - self.assertEqual(len(output), 1) - self.assertIn(country_price, output) - with self.cached_session(): - self.assertEqual(output[country_price].values.dtype, dtypes.int64) - self.assertTrue( - all(x < 15 and x >= 0 for x in output[country_price].values.eval())) - - def testCrossWithMultiDimensionBucketizedColumn(self): - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=5) - price_bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price", 2), - boundaries=[0., 10., 100.]) - country_price = feature_column.crossed_column( - [country, price_bucket], hash_bucket_size=1000) - - with ops.Graph().as_default(): - features = { - "price": - constant_op.constant([[20., 210.], [110., 50.], [-3., -30.]]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV", "US"], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 2]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [country_price], num_outputs=1)) - - weights = column_to_variable[country_price][0] - grad = array_ops.squeeze( - gradients_impl.gradients(output, weights)[0].values) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - self.assertEqual(len(grad.eval()), 6) - - # Test transform features. - output = feature_column_ops.transform_features( - features=features, feature_columns=[country_price]) - self.assertEqual(len(output), 1) - self.assertIn(country_price, output) - - def testCrossWithCrossedColumn(self): - price_bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price"), boundaries=[0., 10., 100.]) - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=5) - country_price = feature_column.crossed_column( - [country, price_bucket], hash_bucket_size=15) - wire = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_country_price = feature_column.crossed_column( - [wire, country_price], hash_bucket_size=15) - features = { - "price": - constant_op.constant([[20.]]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV"], - indices=[[0, 0], [0, 1]], - dense_shape=[1, 2]), - "wire": - sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [0, 1], [0, 2]], - dense_shape=[1, 3]) - } - # Test transform features. - output = feature_column_ops.transform_features( - features=features, feature_columns=[wire_country_price]) - self.assertEqual(len(output), 1) - self.assertIn(wire_country_price, output) - with self.cached_session(): - self.assertEqual(output[wire_country_price].values.dtype, dtypes.int64) - self.assertTrue( - all(x < 15 and x >= 0 for x in output[wire_country_price].values.eval( - ))) - - def testIfFeatureTableContainsTransformationReturnIt(self): - any_column = feature_column.sparse_column_with_hash_bucket("sparse", 10) - features = {any_column: "any-thing-even-not-a-tensor"} - output = feature_column_ops._Transformer(features).transform(any_column) - self.assertEqual(output, "any-thing-even-not-a-tensor") - - -class CreateInputLayersForDNNsTest(test.TestCase): - - def testFeatureColumnDictFails(self): - real_valued = feature_column.real_valued_column("price") - features = {"price": constant_op.constant([[20.], [110], [-3]])} - with self.assertRaisesRegexp( - ValueError, - "Expected feature_columns to be iterable, found dict"): - feature_column_ops.input_from_feature_columns( - features, {"feature": real_valued}) - - def testSparseTensorRealValuedColumn(self): - var_len_sparse_real_valued_column = ( - feature_column._real_valued_var_len_column("rating", is_sparse=True)) - features = { - "ids": - sparse_tensor.SparseTensor( - values=["c", "b", "a"], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - "income": - constant_op.constant([[20.3, 10], [110.3, 0.4], [-3.0, 30.4]]), - "rating": - sparse_tensor.SparseTensor( - values=[3.5, 5.0], indices=[[0, 0], [2, 0]], dense_shape=[3, 1]) - } - with self.assertRaisesRegexp( - ValueError, - "dd"): - feature_column_ops.input_from_feature_columns( - features, [var_len_sparse_real_valued_column]) - - def testAllDNNColumns(self): - sparse_column = feature_column.sparse_column_with_keys( - "ids", ["a", "b", "c", "unseen"]) - real_valued_column = feature_column.real_valued_column("income", 2) - one_hot_column = feature_column.one_hot_column(sparse_column) - embedding_column = feature_column.embedding_column(sparse_column, 10) - features = { - "ids": - sparse_tensor.SparseTensor( - values=["c", "b", "a"], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - "income": - constant_op.constant([[20.3, 10], [110.3, 0.4], [-3.0, 30.4]]), - } - columns = [one_hot_column, embedding_column, real_valued_column] - output = feature_column_ops.input_from_feature_columns(features, columns) - output_core = fc_core.input_layer(features, columns) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - self.assertAllEqual(output.eval().shape, [3, 2 + 4 + 10]) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(output.eval().shape, output_core.eval().shape) - - def testAllDNNColumnsWithColumnwiseOutputs(self): - sparse_column = feature_column.sparse_column_with_keys( - "ids", ["a", "b", "c", "unseen"]) - real_valued_column = feature_column.real_valued_column("income", 2) - one_hot_column = feature_column.one_hot_column(sparse_column) - embedding_column = feature_column.embedding_column(sparse_column, 10) - features = { - "ids": - sparse_tensor.SparseTensor( - values=["c", "b", "a"], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - "income": - constant_op.constant([[20.3, 10], [110.3, 0.4], [-3.0, 30.4]]), - } - columns = [one_hot_column, embedding_column, real_valued_column] - cols_to_outs = {} - feature_column_ops.input_from_feature_columns( - features, columns, cols_to_outs=cols_to_outs) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - for column in columns: - self.assertTrue(column in cols_to_outs) - - def testRealValuedColumn(self): - real_valued = feature_column.real_valued_column("price") - features = {"price": constant_op.constant([[20.], [110], [-3]])} - output = feature_column_ops.input_from_feature_columns(features, - [real_valued]) - with self.cached_session(): - self.assertAllClose(output.eval(), features["price"].eval()) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllClose(output.eval(), - fc_core.input_layer(features, [real_valued]).eval()) - - def testRealValuedColumnWithMultiDimensions(self): - real_valued = feature_column.real_valued_column("price", 2) - features = { - "price": constant_op.constant([[20., 10.], [110, 0.], [-3, 30]]) - } - output = feature_column_ops.input_from_feature_columns(features, - [real_valued]) - with self.cached_session(): - self.assertAllClose(output.eval(), features["price"].eval()) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllClose(output.eval(), - fc_core.input_layer(features, [real_valued]).eval()) - - def testRealValuedColumnDense(self): - var_len_real_valued = feature_column._real_valued_var_len_column( - "rating", default_value=-1) - rating = np.array([[0., 1., 2., -1.], - [3., 4., 5., 6.]]) - features = {"rating": constant_op.constant(rating)} - with self.cached_session() as sess: - output = sess.run(feature_column_ops.input_from_feature_columns( - features, [var_len_real_valued])) - self.assertAllClose(rating, output) - - def testRealValuedColumnTypeConversion(self): - var_len_real_valued = feature_column._real_valued_var_len_column( - "rating", default_value=-1) - rating = np.array([[0, 1, 2, -1], - [3, 4, 5, 6]]) - features = {"rating": constant_op.constant(rating, dtype=dtypes.int64)} - with self.cached_session() as sess: - output = sess.run(feature_column_ops.input_from_feature_columns( - features, [var_len_real_valued])) - self.assertAllClose(rating.astype(np.float32), output) - - def testRealValuedColumnWithNormalizer(self): - real_valued = feature_column.real_valued_column( - "price", normalizer=lambda x: x - 2) - features = {"price": constant_op.constant([[20.], [110], [-3]])} - output = feature_column_ops.input_from_feature_columns(features, - [real_valued]) - with self.cached_session(): - self.assertAllClose(output.eval(), features["price"].eval() - 2) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllClose(output.eval(), - fc_core.input_layer(features, [real_valued]).eval()) - - def testRealValuedColumnWithMultiDimensionsAndNormalizer(self): - real_valued = feature_column.real_valued_column( - "price", 2, normalizer=lambda x: x - 2) - features = { - "price": constant_op.constant([[20., 10.], [110, 0.], [-3, 30]]) - } - output = feature_column_ops.input_from_feature_columns(features, - [real_valued]) - with self.cached_session(): - self.assertAllClose(output.eval(), features["price"].eval() - 2) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllClose(output.eval(), - fc_core.input_layer(features, [real_valued]).eval()) - - def testBucketizedColumnWithNormalizerSucceedsForDNN(self): - bucket = feature_column.bucketized_column( - feature_column.real_valued_column( - "price", normalizer=lambda x: x - 15), - boundaries=[0., 10., 100.]) - # buckets 2, 3, 0 - features = {"price": constant_op.constant([[20.], [110], [-3]])} - output = feature_column_ops.input_from_feature_columns(features, [bucket]) - expected = [[0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0]] - with self.cached_session(): - self.assertAllClose(output.eval(), expected) - self.assertAllClose(output.eval(), - fc_core.input_layer(features, [bucket]).eval()) - - def testBucketizedColumnWithMultiDimensionsSucceedsForDNN(self): - bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price", 2), - boundaries=[0., 10., 100.]) - # buckets [2, 3], [3, 2], [0, 0]. dimension = 2 - features = { - "price": constant_op.constant([[20., 200], [110, 50], [-3, -3]]) - } - output = feature_column_ops.input_from_feature_columns(features, [bucket]) - expected = [[0, 0, 1, 0, 0, 0, 0, 1], [0, 0, 0, 1, 0, 0, 1, 0], - [1, 0, 0, 0, 1, 0, 0, 0]] - with self.cached_session(): - self.assertAllClose(output.eval(), expected) - self.assertAllClose(output.eval(), - fc_core.input_layer(features, [bucket]).eval()) - - def testOneHotColumnFromWeightedSparseColumnSucceedsForDNN(self): - ids_column = feature_column.sparse_column_with_keys( - "ids", ["a", "b", "c", "unseen"]) - ids_tensor = sparse_tensor.SparseTensor( - values=["c", "b", "a", "c"], - indices=[[0, 0], [1, 0], [2, 0], [2, 1]], - dense_shape=[3, 2]) - weighted_ids_column = feature_column.weighted_sparse_column(ids_column, - "weights") - weights_tensor = sparse_tensor.SparseTensor( - values=[10.0, 20.0, 30.0, 40.0], - indices=[[0, 0], [1, 0], [2, 0], [2, 1]], - dense_shape=[3, 2]) - features = {"ids": ids_tensor, "weights": weights_tensor} - one_hot_column = feature_column.one_hot_column(weighted_ids_column) - output = feature_column_ops.input_from_feature_columns(features, - [one_hot_column]) - output_core = fc_core.input_layer(features, [one_hot_column]) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - self.assertAllEqual([[0, 0, 10., 0], [0, 20., 0, 0], [30., 0, 40., 0]], - output.eval()) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(output.eval(), output_core.eval()) - - def testOneHotColumnFromSparseColumnWithKeysSucceedsForDNN(self): - ids_column = feature_column.sparse_column_with_keys( - "ids", ["a", "b", "c", "unseen"]) - ids_tensor = sparse_tensor.SparseTensor( - values=["c", "b", "a"], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]) - one_hot_sparse = feature_column.one_hot_column(ids_column) - features = {"ids": ids_tensor} - output = feature_column_ops.input_from_feature_columns(features, - [one_hot_sparse]) - output_core = fc_core.input_layer(features, [one_hot_sparse]) - - with self.cached_session(): - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - self.assertAllEqual([[0, 0, 1, 0], [0, 1, 0, 0], [1, 0, 0, 0]], - output.eval()) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(output.eval(), output_core.eval()) - - def testOneHotColumnFromMultivalentSparseColumnWithKeysSucceedsForDNN(self): - ids_column = feature_column.sparse_column_with_keys( - "ids", ["a", "b", "c", "unseen"]) - ids_tensor = sparse_tensor.SparseTensor( - values=["c", "b", "a", "c"], - indices=[[0, 0], [1, 0], [2, 0], [2, 1]], - dense_shape=[3, 2]) - one_hot_sparse = feature_column.one_hot_column(ids_column) - features = {"ids": ids_tensor} - output = feature_column_ops.input_from_feature_columns(features, - [one_hot_sparse]) - output_core = fc_core.input_layer(features, [one_hot_sparse]) - - with self.cached_session(): - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - self.assertAllEqual([[0, 0, 1, 0], [0, 1, 0, 0], [1, 0, 1, 0]], - output.eval()) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(output.eval(), output_core.eval()) - - def testOneHotColumnFromSparseColumnWithIntegerizedFeaturePassesForDNN(self): - ids_column = feature_column.sparse_column_with_integerized_feature( - "ids", bucket_size=4) - one_hot_sparse = feature_column.one_hot_column(ids_column) - features = { - "ids": - sparse_tensor.SparseTensor( - values=[2, 1, 0, 2], - indices=[[0, 0], [1, 0], [2, 0], [2, 1]], - dense_shape=[3, 2]) - } - output = feature_column_ops.input_from_feature_columns(features, - [one_hot_sparse]) - output_core = fc_core.input_layer(features, [one_hot_sparse]) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - self.assertAllEqual([[0, 0, 1, 0], [0, 1, 0, 0], [1, 0, 1, 0]], - output.eval()) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(output.eval(), output_core.eval()) - - def testOneHotColumnFromSparseColumnWithHashBucketSucceedsForDNN(self): - hashed_sparse = feature_column.sparse_column_with_hash_bucket("feat", 10) - wire_tensor = sparse_tensor.SparseTensor( - values=["a", "b", "c1", "c2"], - indices=[[0, 0], [1, 0], [2, 0], [2, 1]], - dense_shape=[3, 2]) - features = {"feat": wire_tensor} - one_hot_sparse = feature_column.one_hot_column(hashed_sparse) - output = feature_column_ops.input_from_feature_columns(features, - [one_hot_sparse]) - output_core = fc_core.input_layer(features, [one_hot_sparse]) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - self.assertAllEqual([3, 10], output.eval().shape) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(output.eval(), output_core.eval()) - - def testEmbeddingColumnSucceedsForDNN(self): - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo", "xx", "yy"], - indices=[[0, 0], [1, 0], [1, 1], [2, 0], [3, 0]], - dense_shape=[4, 2]) - features = {"wire": wire_tensor} - embeded_sparse = feature_column.embedding_column(hashed_sparse, 10) - output = feature_column_ops.input_from_feature_columns(features, - [embeded_sparse]) - output_core = fc_core.input_layer(features, [embeded_sparse]) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - self.assertAllEqual(output.eval().shape, [4, 10]) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(output.eval().shape, output_core.eval().shape) - - def testScatteredEmbeddingColumnSucceedsForDNN(self): - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo", "omar"], - indices=[[0, 0], [1, 0], [1, 1], [2, 0]], - dense_shape=[3, 2]) - - features = {"wire": wire_tensor} - # Big enough hash space so that hopefully there is no collision - embedded_sparse = feature_column.scattered_embedding_column( - "wire", 1000, 3, layers.SPARSE_FEATURE_CROSS_DEFAULT_HASH_KEY) - output = feature_column_ops.input_from_feature_columns( - features, [embedded_sparse], weight_collections=["my_collection"]) - weights = ops.get_collection("my_collection") - grad = gradients_impl.gradients(output, weights) - # Calcuates the tensors calculated by FC core libs. Later, the values will - # be compared with the contrib version. - output_core = fc_core.input_layer( - features, [embedded_sparse], weight_collections=["my_collection_core"]) - weights_core = ops.get_collection("my_collection_core") - grad_core = gradients_impl.gradients(output_core, weights_core) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - gradient_values = [] - gradient_values_core = [] - # Collect the gradient from the different partitions (one in this test) - for p in range(len(grad)): - gradient_values.extend(grad[p].values.eval()) - gradient_values_core.extend(grad_core[p].values.eval()) - gradient_values.sort() - gradient_values_core.sort() - self.assertAllEqual(gradient_values, [0.5] * 6 + [2] * 3) - self.assertAllEqual(gradient_values, gradient_values_core) - - def testEmbeddingColumnWithInitializerSucceedsForDNN(self): - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - init_value = 133.7 - embeded_sparse = feature_column.embedding_column( - hashed_sparse, - 10, - initializer=init_ops.constant_initializer(init_value)) - output = feature_column_ops.input_from_feature_columns(features, - [embeded_sparse]) - output_core = fc_core.input_layer(features, [embeded_sparse]) - - with self.cached_session(): - variables_lib.global_variables_initializer().run() - output_eval = output.eval() - self.assertAllEqual(output_eval.shape, [2, 10]) - self.assertAllClose(output_eval, np.tile(init_value, [2, 10])) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(output.eval(), output_core.eval()) - - def testEmbeddingColumnWithMultipleInitializersFails(self): - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - embedded_sparse = feature_column.embedding_column( - hashed_sparse, - 10, - initializer=init_ops.truncated_normal_initializer( - mean=42, stddev=1337)) - embedded_sparse_alternate = feature_column.embedding_column( - hashed_sparse, - 10, - initializer=init_ops.truncated_normal_initializer( - mean=1337, stddev=42)) - - # Makes sure that trying to use different initializers with the same - # embedding column explicitly fails. - with self.cached_session(): - with self.assertRaisesRegexp( - ValueError, - "Duplicate feature column key found for column: wire_embedding"): - feature_column_ops.input_from_feature_columns( - features, [embedded_sparse, embedded_sparse_alternate]) - - def testEmbeddingColumnWithWeightedSparseColumnSucceedsForDNN(self): - """Tests DNN input with embedded weighted sparse column.""" - ids = feature_column.sparse_column_with_keys("ids", - ["marlo", "omar", "stringer"]) - ids_tensor = sparse_tensor.SparseTensor( - values=["stringer", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - weighted_ids = feature_column.weighted_sparse_column(ids, "weights") - weights_tensor = sparse_tensor.SparseTensor( - values=[10.0, 20.0, 30.0], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"ids": ids_tensor, "weights": weights_tensor} - embeded_sparse = feature_column.embedding_column(weighted_ids, 10) - output = feature_column_ops.input_from_feature_columns(features, - [embeded_sparse]) - output_core = fc_core.input_layer(features, [embeded_sparse]) - - with self.cached_session(): - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - self.assertAllEqual(output.eval().shape, [2, 10]) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(output.eval().shape, output_core.eval().shape) - - def testEmbeddingColumnWithIntegerWeightedSparseColumnSucceedsForDNN(self): - """Same as the previous test, but with integer weights.""" - ids = feature_column.sparse_column_with_keys("ids", - ["marlo", "omar", "stringer"]) - ids_tensor = sparse_tensor.SparseTensor( - values=["stringer", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - weighted_ids = feature_column.weighted_sparse_column( - ids, "weights", dtype=dtypes.int32) - weights_tensor = sparse_tensor.SparseTensor( - values=constant_op.constant([10, 20, 30], dtype=dtypes.int32), - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"ids": ids_tensor, "weights": weights_tensor} - embeded_sparse = feature_column.embedding_column(weighted_ids, 10) - output = feature_column_ops.input_from_feature_columns(features, - [embeded_sparse]) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - self.assertAllEqual(output.eval().shape, [2, 10]) - - def testEmbeddingColumnWithCrossedColumnSucceedsForDNN(self): - a = feature_column.sparse_column_with_hash_bucket( - "aaa", hash_bucket_size=100) - b = feature_column.sparse_column_with_hash_bucket( - "bbb", hash_bucket_size=100) - crossed = feature_column.crossed_column(set([a, b]), hash_bucket_size=10000) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"aaa": wire_tensor, "bbb": wire_tensor} - embeded_sparse = feature_column.embedding_column(crossed, 10) - output = feature_column_ops.input_from_feature_columns(features, - [embeded_sparse]) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - self.assertAllEqual(output.eval().shape, [2, 10]) - - def testSparseColumnFailsForDNN(self): - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - with self.cached_session(): - with self.assertRaisesRegexp( - ValueError, "Error creating input layer for column: wire"): - variables_lib.global_variables_initializer().run() - feature_column_ops.input_from_feature_columns(features, [hashed_sparse]) - - def testWeightedSparseColumnFailsForDNN(self): - ids = feature_column.sparse_column_with_keys("ids", - ["marlo", "omar", "stringer"]) - ids_tensor = sparse_tensor.SparseTensor( - values=["stringer", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - weighted_ids = feature_column.weighted_sparse_column(ids, "weights") - weights_tensor = sparse_tensor.SparseTensor( - values=[10.0, 20.0, 30.0], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"ids": ids_tensor, "weights": weights_tensor} - with self.cached_session(): - with self.assertRaisesRegexp( - ValueError, - "Error creating input layer for column: ids_weighted_by_weights"): - lookup_ops.tables_initializer().run() - feature_column_ops.input_from_feature_columns(features, [weighted_ids]) - - def testCrossedColumnFailsForDNN(self): - a = feature_column.sparse_column_with_hash_bucket( - "aaa", hash_bucket_size=100) - b = feature_column.sparse_column_with_hash_bucket( - "bbb", hash_bucket_size=100) - crossed = feature_column.crossed_column(set([a, b]), hash_bucket_size=10000) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"aaa": wire_tensor, "bbb": wire_tensor} - with self.cached_session(): - with self.assertRaisesRegexp( - ValueError, "Error creating input layer for column: aaa_X_bbb"): - variables_lib.global_variables_initializer().run() - feature_column_ops.input_from_feature_columns(features, [crossed]) - - def testDeepColumnsSucceedForDNN(self): - real_valued = feature_column.real_valued_column("income", 3) - bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price", 2), - boundaries=[0., 10., 100.]) - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - features = { - "income": - constant_op.constant([[20., 10, -5], [110, 0, -7], [-3, 30, 50]]), - "price": - constant_op.constant([[20., 200], [110, 2], [-20, -30]]), - "wire": - sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]) - } - embeded_sparse = feature_column.embedding_column( - hashed_sparse, 10, initializer=init_ops.constant_initializer(133.7)) - output = feature_column_ops.input_from_feature_columns( - features, [real_valued, bucket, embeded_sparse]) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - # size of output = 3 (real_valued) + 2 * 4 (bucket) + 10 (embedding) = 21 - self.assertAllEqual(output.eval().shape, [3, 21]) - - def testEmbeddingColumnForDNN(self): - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[3, 2]) - features = {"wire": wire_tensor} - embeded_sparse = feature_column.embedding_column( - hashed_sparse, - 1, - combiner="sum", - initializer=init_ops.ones_initializer()) - output = feature_column_ops.input_from_feature_columns(features, - [embeded_sparse]) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - # score: (number of values) - self.assertAllEqual(output.eval(), [[1.], [2.], [0.]]) - - def testEmbeddingColumnWithMaxNormForDNN(self): - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[3, 2]) - features = {"wire": wire_tensor} - embedded_sparse = feature_column.embedding_column( - hashed_sparse, - 1, - combiner="sum", - initializer=init_ops.ones_initializer(), - max_norm=0.5) - output = feature_column_ops.input_from_feature_columns(features, - [embedded_sparse]) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - # score: (number of values * 0.5) - self.assertAllClose(output.eval(), [[0.5], [1.], [0.]]) - - def testEmbeddingColumnWithWeightedSparseColumnForDNN(self): - ids = feature_column.sparse_column_with_keys("ids", - ["marlo", "omar", "stringer"]) - ids_tensor = sparse_tensor.SparseTensor( - values=["stringer", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[3, 2]) - weighted_ids = feature_column.weighted_sparse_column(ids, "weights") - weights_tensor = sparse_tensor.SparseTensor( - values=[10.0, 20.0, 30.0], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[3, 2]) - features = {"ids": ids_tensor, "weights": weights_tensor} - embeded_sparse = feature_column.embedding_column( - weighted_ids, - 1, - combiner="sum", - initializer=init_ops.ones_initializer()) - output = feature_column_ops.input_from_feature_columns(features, - [embeded_sparse]) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - # score: (sum of weights) - self.assertAllEqual(output.eval(), [[10.], [50.], [0.]]) - - def testInputLayerWithCollectionsForDNN(self): - real_valued = feature_column.real_valued_column("price") - bucket = feature_column.bucketized_column( - real_valued, boundaries=[0., 10., 100.]) - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - features = { - "price": - constant_op.constant([[20.], [110], [-3]]), - "wire": - sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]) - } - embeded_sparse = feature_column.embedding_column(hashed_sparse, 10) - feature_column_ops.input_from_feature_columns( - features, [real_valued, bucket, embeded_sparse], - weight_collections=["my_collection"]) - weights = ops.get_collection("my_collection") - # one variable for embeded sparse - self.assertEqual(1, len(weights)) - - def testInputLayerWithTrainableArgForDNN(self): - real_valued = feature_column.real_valued_column("price") - bucket = feature_column.bucketized_column( - real_valued, boundaries=[0., 10., 100.]) - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - features = { - "price": - constant_op.constant([[20.], [110], [-3]]), - "wire": - sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]) - } - embeded_sparse = feature_column.embedding_column(hashed_sparse, 10) - feature_column_ops.input_from_feature_columns( - features, [real_valued, bucket, embeded_sparse], - weight_collections=["my_collection"], - trainable=False) - # There should not be any trainable variables - self.assertEqual(0, len(variables_lib.trainable_variables())) - - feature_column_ops.input_from_feature_columns( - features, [real_valued, bucket, embeded_sparse], - weight_collections=["my_collection"], - trainable=True) - # There should one trainable variable for embeded sparse - self.assertEqual(1, len(variables_lib.trainable_variables())) - - def testInputLayerWithNonTrainableEmbeddingForDNN(self): - sparse_1 = feature_column.sparse_column_with_hash_bucket("wire_1", 10) - sparse_2 = feature_column.sparse_column_with_hash_bucket("wire_2", 10) - features = { - "wire_1": - sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - "wire_2": - sparse_tensor.SparseTensor( - values=["jack", "jill"], - indices=[[0, 0], [1, 0]], - dense_shape=[4, 1]) - } - dims_1 = 10 - init_1 = 3.14 - embeded_1 = feature_column.embedding_column( - sparse_1, dims_1, initializer=init_ops.constant_initializer(init_1), - trainable=False) - output_1 = feature_column_ops.input_from_feature_columns( - features, [embeded_1]) - # There should be no trainable variables for sparse_1 - self.assertEqual(0, len(variables_lib.trainable_variables())) - - dims_2 = 7 - init_2 = 6.14 - embeded_2 = feature_column.embedding_column( - sparse_2, dims_2, initializer=init_ops.constant_initializer(init_2), - trainable=True) - output_2 = feature_column_ops.input_from_feature_columns( - features, [embeded_2]) - # There should be one trainable variables for sparse_2 - self.assertEqual(1, len(variables_lib.trainable_variables())) - - with self.cached_session(): - variables_lib.global_variables_initializer().run() - output_1_eval = output_1.eval() - output_2_eval = output_2.eval() - self.assertAllEqual(output_1_eval.shape, [3, dims_1]) - self.assertAllClose(output_1_eval, np.tile(init_1, [3, dims_1])) - self.assertAllEqual(output_2_eval.shape, [4, dims_2]) - self.assertAllClose(output_2_eval, np.concatenate( - (np.tile(init_2, [2, dims_2]), np.tile(0, [2, dims_2])))) - - -class SequenceInputFromFeatureColumnTest(test.TestCase): - - def testSupportedColumns(self): - measurement = feature_column.real_valued_column("measurements") - country = feature_column.sparse_column_with_hash_bucket("country", 100) - pets = feature_column.sparse_column_with_hash_bucket("pets", 100) - ids = feature_column.sparse_column_with_integerized_feature("id", 100) - - country_x_pets = feature_column.crossed_column([country, pets], 100) - country_x_pets_onehot = feature_column.one_hot_column(country_x_pets) - bucketized_measurement = feature_column.bucketized_column(measurement, - [.25, .5, .75]) - embedded_id = feature_column.embedding_column(ids, 100) - - # `_BucketizedColumn` is not supported. - self.assertRaisesRegexp( - ValueError, - "FeatureColumn type _BucketizedColumn is not currently supported", - feature_column_ops.sequence_input_from_feature_columns, {}, - [measurement, bucketized_measurement]) - - # `_CrossedColumn` is not supported. - self.assertRaisesRegexp( - ValueError, - "FeatureColumn type _CrossedColumn is not currently supported", - feature_column_ops.sequence_input_from_feature_columns, {}, - [embedded_id, country_x_pets]) - - # `country_x_pets_onehot` depends on a `_CrossedColumn` which is forbidden. - self.assertRaisesRegexp( - ValueError, "Column country_X_pets .* _CrossedColumn", - feature_column_ops.sequence_input_from_feature_columns, {}, - [embedded_id, country_x_pets_onehot]) - - def testRealValuedColumn(self): - batch_size = 4 - sequence_length = 8 - dimension = 3 - - np.random.seed(1111) - measurement_input = np.random.rand(batch_size, sequence_length, dimension) - measurement_column = feature_column.real_valued_column("measurements") - columns_to_tensors = { - "measurements": constant_op.constant(measurement_input) - } - model_input_tensor = feature_column_ops.sequence_input_from_feature_columns( - columns_to_tensors, [measurement_column]) - - with self.cached_session() as sess: - model_inputs = sess.run(model_input_tensor) - self.assertAllClose(measurement_input, model_inputs) - - def testRealValuedVarLenColumn(self): - var_len_real_valued = feature_column._real_valued_var_len_column( - "rating", default_value=-1) - rating = np.array([[0., 1., 2., -1.], - [3., 4., 5., 6.]]) - features = {"rating": constant_op.constant(rating)} - with self.cached_session() as sess: - output = sess.run( - feature_column_ops.sequence_input_from_feature_columns( - features, [var_len_real_valued])) - reshaped_rating = np.reshape(rating, [2, 4, 1]) - self.assertAllClose(reshaped_rating, output) - - def testRealValuedColumnWithExtraDimensions(self): - batch_size = 4 - sequence_length = 8 - dimensions = [3, 4, 5] - - np.random.seed(2222) - measurement_input = np.random.rand(batch_size, sequence_length, *dimensions) - measurement_column = feature_column.real_valued_column("measurements") - columns_to_tensors = { - "measurements": constant_op.constant(measurement_input) - } - model_input_tensor = feature_column_ops.sequence_input_from_feature_columns( - columns_to_tensors, [measurement_column]) - - expected_shape = [batch_size, sequence_length, np.prod(dimensions)] - reshaped_measurements = np.reshape(measurement_input, expected_shape) - - with self.cached_session() as sess: - model_inputs = sess.run(model_input_tensor) - - self.assertAllClose(reshaped_measurements, model_inputs) - - def testRealValuedColumnWithNormalizer(self): - batch_size = 4 - sequence_length = 8 - dimension = 3 - normalizer = lambda x: x - 2 - - np.random.seed(3333) - measurement_input = np.random.rand(batch_size, sequence_length, dimension) - measurement_column = feature_column.real_valued_column( - "measurements", normalizer=normalizer) - columns_to_tensors = { - "measurements": constant_op.constant(measurement_input) - } - model_input_tensor = feature_column_ops.sequence_input_from_feature_columns( - columns_to_tensors, [measurement_column]) - - with self.cached_session() as sess: - model_inputs = sess.run(model_input_tensor) - self.assertAllClose(normalizer(measurement_input), model_inputs) - - def testRealValuedColumnWithMultiDimensionsAndNormalizer(self): - batch_size = 4 - sequence_length = 8 - dimensions = [3, 4, 5] - normalizer = lambda x: x / 2.0 - - np.random.seed(1234) - measurement_input = np.random.rand(batch_size, sequence_length, *dimensions) - measurement_column = feature_column.real_valued_column( - "measurements", normalizer=normalizer) - columns_to_tensors = { - "measurements": constant_op.constant(measurement_input) - } - model_input_tensor = feature_column_ops.sequence_input_from_feature_columns( - columns_to_tensors, [measurement_column]) - - expected_shape = [batch_size, sequence_length, np.prod(dimensions)] - reshaped_measurements = np.reshape(measurement_input, expected_shape) - - with self.cached_session() as sess: - model_inputs = sess.run(model_input_tensor) - - self.assertAllClose(normalizer(reshaped_measurements), model_inputs) - - def testOneHotColumnFromSparseColumnWithKeys(self): - ids_tensor = sparse_tensor.SparseTensor( - values=["c", "b", - "a", "c", "b", - "b"], - indices=[[0, 0, 0], [0, 1, 0], - [1, 0, 0], [1, 0, 1], [1, 1, 0], - [3, 2, 0]], - dense_shape=[4, 3, 2]) - - ids_column = feature_column.sparse_column_with_keys( - "ids", ["a", "b", "c", "unseen"]) - one_hot_column = feature_column.one_hot_column(ids_column) - columns_to_tensors = {"ids": ids_tensor} - model_input_tensor = feature_column_ops.sequence_input_from_feature_columns( - columns_to_tensors, [one_hot_column]) - - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - model_input = sess.run(model_input_tensor) - - expected_input_shape = np.array([4, 3, 4]) - expected_model_input = np.array( - [[[0, 0, 1, 0], [0, 1, 0, 0], [0, 0, 0, 0]], - [[1, 0, 1, 0], [0, 1, 0, 0], [0, 0, 0, 0]], - [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], - [[0, 0, 0, 0], [0, 0, 0, 0], [0, 1, 0, 0]]], - dtype=np.float32) - - self.assertAllEqual(expected_input_shape, model_input.shape) - self.assertAllClose(expected_model_input, model_input) - - def testOneHotColumnFromSparseColumnWithHashBucket(self): - hash_buckets = 10 - ids_tensor = sparse_tensor.SparseTensor( - values=["c", "b", - "a", "c", "b", - "b"], - indices=[[0, 0, 0], [0, 1, 0], - [1, 0, 0], [1, 0, 1], [1, 1, 0], - [3, 2, 0]], - dense_shape=[4, 3, 2]) - - hashed_ids_column = feature_column.sparse_column_with_hash_bucket( - "ids", hash_buckets) - one_hot_column = feature_column.one_hot_column(hashed_ids_column) - columns_to_tensors = {"ids": ids_tensor} - model_input_tensor = feature_column_ops.sequence_input_from_feature_columns( - columns_to_tensors, [one_hot_column]) - - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - model_input = sess.run(model_input_tensor) - - expected_input_shape = np.array([4, 3, hash_buckets]) - self.assertAllEqual(expected_input_shape, model_input.shape) - - def testEmbeddingColumn(self): - hash_buckets = 10 - embedding_dimension = 5 - ids_tensor = sparse_tensor.SparseTensor( - values=["c", "b", - "a", "c", "b", - "b"], - indices=[[0, 0, 0], [0, 1, 0], - [1, 0, 0], [1, 0, 1], [1, 1, 0], - [3, 2, 0]], - dense_shape=[4, 3, 2]) - - expected_input_shape = np.array([4, 3, embedding_dimension]) - - hashed_ids_column = feature_column.sparse_column_with_hash_bucket( - "ids", hash_buckets) - embedded_column = feature_column.embedding_column(hashed_ids_column, - embedding_dimension) - columns_to_tensors = {"ids": ids_tensor} - model_input_tensor = feature_column_ops.sequence_input_from_feature_columns( - columns_to_tensors, [embedded_column]) - - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - model_input = sess.run(model_input_tensor) - - self.assertAllEqual(expected_input_shape, model_input.shape) - - def testEmbeddingColumnWithAutoReshape(self): - hash_buckets = 10 - embedding_dimension = 5 - ids_tensor = sparse_tensor.SparseTensor( - values=["c", "b", - "a", "c", "b", - "b"], - indices=[[0, 0], [0, 1], - [1, 0], [1, 1], [1, 2], - [3, 2]], - dense_shape=[4, 3]) - - expected_input_shape = np.array([4, 3, embedding_dimension]) - - hashed_ids_column = feature_column.sparse_column_with_hash_bucket( - "ids", hash_buckets) - embedded_column = feature_column.embedding_column(hashed_ids_column, - embedding_dimension) - columns_to_tensors = {"ids": ids_tensor} - model_input_tensor = feature_column_ops.sequence_input_from_feature_columns( - columns_to_tensors, [embedded_column]) - - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - model_input = sess.run(model_input_tensor) - - self.assertAllEqual(expected_input_shape, model_input.shape) - - def testEmbeddingColumnGradient(self): - hash_buckets = 1000 - embedding_dimension = 3 - ids_tensor = sparse_tensor.SparseTensor( - values=["c", "b", - "a", "c", "b", - "b"], - indices=[[0, 0, 0], [0, 1, 0], - [1, 0, 0], [1, 0, 1], [1, 1, 0], - [3, 2, 0]], - dense_shape=[4, 3, 2]) - - hashed_ids_column = feature_column.sparse_column_with_hash_bucket( - "ids", hash_buckets) - embedded_column = feature_column.embedding_column( - hashed_ids_column, embedding_dimension, combiner="sum") - columns_to_tensors = {"ids": ids_tensor} - model_input_tensor = feature_column_ops.sequence_input_from_feature_columns( - columns_to_tensors, [embedded_column], - weight_collections=["my_collection"]) - embedding_weights = ops.get_collection("my_collection") - gradient_tensor = gradients_impl.gradients(model_input_tensor, - embedding_weights) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - model_input, gradients = sess.run([model_input_tensor, gradient_tensor]) - - expected_input_shape = [4, 3, embedding_dimension] - self.assertAllEqual(expected_input_shape, model_input.shape) - - # `ids_tensor` consists of 7 instances of , 3 occurrences of "b", - # 2 occurrences of "c" and 1 instance of "a". - expected_gradient_values = sorted([0., 3., 2., 1.] * embedding_dimension) - actual_gradient_values = np.sort(gradients[0].values, axis=None) - self.assertAllClose(expected_gradient_values, actual_gradient_values) - - def testMultipleColumns(self): - batch_size = 4 - sequence_length = 3 - measurement_dimension = 5 - country_hash_size = 10 - max_id = 200 - id_embedding_dimension = 11 - normalizer = lambda x: x / 10.0 - - measurement_tensor = random_ops.random_uniform( - [batch_size, sequence_length, measurement_dimension]) - country_tensor = sparse_tensor.SparseTensor( - values=["us", "ca", - "ru", "fr", "ca", - "mx"], - indices=[[0, 0, 0], [0, 1, 0], - [1, 0, 0], [1, 0, 1], [1, 1, 0], - [3, 2, 0]], - dense_shape=[4, 3, 2]) - id_tensor = sparse_tensor.SparseTensor( - values=[2, 5, - 26, 123, 1, - 0], - indices=[[0, 0, 0], [0, 0, 1], - [0, 1, 1], [1, 0, 0], [1, 1, 0], - [3, 2, 0]], - dense_shape=[4, 3, 2]) - - columns_to_tensors = { - "measurements": measurement_tensor, - "country": country_tensor, - "id": id_tensor - } - - measurement_column = feature_column.real_valued_column( - "measurements", normalizer=normalizer) - country_column = feature_column.sparse_column_with_hash_bucket( - "country", country_hash_size) - id_column = feature_column.sparse_column_with_integerized_feature("id", - max_id) - - onehot_country_column = feature_column.one_hot_column(country_column) - embedded_id_column = feature_column.embedding_column(id_column, - id_embedding_dimension) - - model_input_columns = [ - measurement_column, onehot_country_column, embedded_id_column - ] - - model_input_tensor = feature_column_ops.sequence_input_from_feature_columns( - columns_to_tensors, model_input_columns) - self.assertEqual(dtypes.float32, model_input_tensor.dtype) - - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - model_input = sess.run(model_input_tensor) - - expected_input_shape = [ - batch_size, sequence_length, - measurement_dimension + country_hash_size + id_embedding_dimension - ] - self.assertAllEqual(expected_input_shape, model_input.shape) - - -class WeightedSumTest(test.TestCase): - - def testFeatureColumnDictFails(self): - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - with self.assertRaisesRegexp( - ValueError, - "Expected feature_columns to be iterable, found dict"): - feature_column_ops.weighted_sum_from_feature_columns( - features, {"feature": hashed_sparse}, num_outputs=5) - - def testSparseColumn(self): - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - logits, _, _ = feature_column_ops.weighted_sum_from_feature_columns( - features, [hashed_sparse], num_outputs=5) - logits_core = fc_core.linear_model(features, [hashed_sparse], units=5) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - self.assertAllEqual(logits.eval().shape, [2, 5]) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(logits.eval(), logits_core.eval()) - - def testSparseIntColumn(self): - """Tests a sparse column with int values.""" - hashed_sparse = feature_column.sparse_column_with_hash_bucket( - "wire", 10, dtype=dtypes.int64) - wire_tensor = sparse_tensor.SparseTensor( - values=[101, 201, 301], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - logits, _, _ = feature_column_ops.weighted_sum_from_feature_columns( - features, [hashed_sparse], num_outputs=5) - logits_core = fc_core.linear_model(features, [hashed_sparse], units=5) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - self.assertAllEqual(logits.eval().shape, [2, 5]) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(logits.eval(), logits_core.eval()) - - def testSparseColumnWithDenseInputTensor(self): - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_tensor = constant_op.constant( - [["omar", "stringer"], ["marlo", "rick"]]) - features = {"wire": wire_tensor} - logits, _, _ = feature_column_ops.weighted_sum_from_feature_columns( - features, [hashed_sparse], num_outputs=5) - logits_core = fc_core.linear_model(features, [hashed_sparse], units=5) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - self.assertAllEqual(logits.eval().shape, [2, 5]) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(logits.eval(), logits_core.eval()) - - def testWeightedSparseColumn(self): - ids = feature_column.sparse_column_with_keys("ids", - ["marlo", "omar", "stringer"]) - ids_tensor = sparse_tensor.SparseTensor( - values=["stringer", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - weighted_ids = feature_column.weighted_sparse_column(ids, "weights") - weights_tensor = sparse_tensor.SparseTensor( - values=[10.0, 20.0, 30.0], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"ids": ids_tensor, "weights": weights_tensor} - logits, _, _ = feature_column_ops.weighted_sum_from_feature_columns( - features, [weighted_ids], num_outputs=5) - logits_core = fc_core.linear_model(features, [weighted_ids], units=5) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - self.assertAllEqual(logits.eval().shape, [2, 5]) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(logits.eval(), logits_core.eval()) - - def testWeightedSparseColumnWithDenseInputTensor(self): - ids = feature_column.sparse_column_with_keys( - "ids", ["marlo", "omar", "stringer", "rick"]) - ids_tensor = constant_op.constant([["omar", "stringer"], ["marlo", "rick"]]) - weighted_ids = feature_column.weighted_sparse_column(ids, "weights") - weights_tensor = constant_op.constant([[10.0, 20.0], [30.0, 40.0]]) - - features = {"ids": ids_tensor, "weights": weights_tensor} - logits, _, _ = feature_column_ops.weighted_sum_from_feature_columns( - features, [weighted_ids], num_outputs=5) - logits_core = fc_core.linear_model(features, [weighted_ids], units=5) - - with self.cached_session(): - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - self.assertAllEqual(logits.eval().shape, [2, 5]) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(logits.eval(), logits_core.eval()) - - def testCrossedColumn(self): - a = feature_column.sparse_column_with_hash_bucket( - "aaa", hash_bucket_size=100) - b = feature_column.sparse_column_with_hash_bucket( - "bbb", hash_bucket_size=100) - crossed = feature_column.crossed_column(set([a, b]), hash_bucket_size=10000) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"aaa": wire_tensor, "bbb": wire_tensor} - logits, _, _ = feature_column_ops.weighted_sum_from_feature_columns( - features, [crossed], num_outputs=5) - logits_core = fc_core.linear_model(features, [crossed], units=5) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - self.assertAllEqual(logits.eval().shape, [2, 5]) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(logits.eval(), logits_core.eval()) - - def testEmbeddingColumn(self): - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - wire_tensor = sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]) - features = {"wire": wire_tensor} - embeded_sparse = feature_column.embedding_column(hashed_sparse, 10) - with self.cached_session(): - with self.assertRaisesRegexp( - ValueError, "Error creating weighted sum for column: wire_embedding"): - variables_lib.global_variables_initializer().run() - feature_column_ops.weighted_sum_from_feature_columns( - features, [embeded_sparse], num_outputs=5) - - def testSparseFeatureColumnWithVocabularyFile(self): - vocabulary_file = os.path.join(self.get_temp_dir(), "movies.txt") - with open(vocabulary_file, "w") as f: - f.write("\n".join(["head-on", "matrix", "winter sleep"]) + "\n") - movies = feature_column.sparse_column_with_vocabulary_file( - column_name="movies", vocabulary_file=vocabulary_file, vocab_size=3) - with ops.Graph().as_default(): - features = { - "movies": - sparse_tensor.SparseTensor( - values=["matrix", "head-on", "winter sleep"], - indices=[[0, 0], [0, 1], [1, 0]], - dense_shape=[2, 2]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [movies], num_outputs=1)) - logits_core = fc_core.linear_model(features, [movies]) - - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - weights = column_to_variable[movies][0] - self.assertEqual(weights.get_shape(), (3, 1)) - sess.run(weights.assign([[0.1], [0.3], [0.5]])) - # score for first example = 0.3 (matrix) + 0.1 (head-on) = 0.4 - # score for second example = 0.5 (winter sleep) - self.assertAllClose(output.eval(), [[0.4], [0.5]]) - # Cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(output.eval().shape, logits_core.eval().shape) - - def testRealValuedColumnWithMultiDimensions(self): - real_valued = feature_column.real_valued_column("price", 2) - features = { - "price": constant_op.constant([[20., 10.], [110, 0.], [-3, 30]]) - } - logits, _, _ = feature_column_ops.weighted_sum_from_feature_columns( - features, [real_valued], num_outputs=5) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - self.assertAllEqual(logits.eval().shape, [3, 5]) - - def testBucketizedColumnWithMultiDimensions(self): - bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price", 2), - boundaries=[0., 10., 100.]) - features = { - "price": constant_op.constant([[20., 10.], [110, 0.], [-3, 30]]) - } - logits, _, _ = feature_column_ops.weighted_sum_from_feature_columns( - features, [bucket], num_outputs=5) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - self.assertAllEqual(logits.eval().shape, [3, 5]) - - def testAllWideColumns(self): - real_valued = feature_column.real_valued_column("income", 2) - bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price"), boundaries=[0., 10., 100.]) - hashed_sparse = feature_column.sparse_column_with_hash_bucket("wire", 10) - crossed = feature_column.crossed_column([bucket, hashed_sparse], 100) - features = { - "income": - constant_op.constant([[20., 10], [110, 0], [-3, 30]]), - "price": - constant_op.constant([[20.], [110], [-3]]), - "wire": - sparse_tensor.SparseTensor( - values=["omar", "stringer", "marlo"], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]) - } - output, _, _ = feature_column_ops.weighted_sum_from_feature_columns( - features, [real_valued, bucket, hashed_sparse, crossed], num_outputs=5) - output_core = fc_core.linear_model( - features, [real_valued, bucket, hashed_sparse, crossed], units=5) - with self.cached_session(): - variables_lib.global_variables_initializer().run() - self.assertAllEqual(output.eval().shape, [3, 5]) - # Verify cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(output.eval(), output_core.eval()) - - def testPredictions(self): - language = feature_column.sparse_column_with_keys( - column_name="language", keys=["english", "finnish", "hindi"]) - age = feature_column.real_valued_column("age") - with ops.Graph().as_default(): - features = { - "age": - constant_op.constant([[1], [2]]), - "language": - sparse_tensor.SparseTensor( - values=["hindi", "english"], - indices=[[0, 0], [1, 0]], - dense_shape=[2, 1]), - } - output, column_to_variable, bias = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [age, language], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - self.assertAllClose(output.eval(), [[0.], [0.]]) - - sess.run(bias.assign([0.1])) - self.assertAllClose(output.eval(), [[0.1], [0.1]]) - - # score: 0.1 + age*0.1 - sess.run(column_to_variable[age][0].assign([[0.2]])) - self.assertAllClose(output.eval(), [[0.3], [0.5]]) - - # score: 0.1 + age*0.1 + language_weight[language_index] - sess.run(column_to_variable[language][0].assign([[0.1], [0.3], [0.2]])) - self.assertAllClose(output.eval(), [[0.5], [0.6]]) - - def testJointPredictions(self): - country = feature_column.sparse_column_with_keys( - column_name="country", keys=["us", "finland"]) - language = feature_column.sparse_column_with_keys( - column_name="language", keys=["english", "finnish", "hindi"]) - with ops.Graph().as_default(): - features = { - "country": - sparse_tensor.SparseTensor( - values=["finland", "us"], - indices=[[0, 0], [1, 0]], - dense_shape=[2, 1]), - "language": - sparse_tensor.SparseTensor( - values=["hindi", "english"], - indices=[[0, 0], [1, 0]], - dense_shape=[2, 1]), - } - output, variables, bias = ( - feature_column_ops.joint_weighted_sum_from_feature_columns( - features, [country, language], num_outputs=1)) - # Assert that only a single weight is created. - self.assertEqual(len(variables), 1) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - self.assertAllClose(output.eval(), [[0.], [0.]]) - - sess.run(bias.assign([0.1])) - self.assertAllClose(output.eval(), [[0.1], [0.1]]) - - # shape is [5,1] because 1 class and 2 + 3 features. - self.assertEquals(variables[0].get_shape().as_list(), [5, 1]) - - # score: bias + country_weight + language_weight - sess.run(variables[0].assign([[0.1], [0.2], [0.3], [0.4], [0.5]])) - self.assertAllClose(output.eval(), [[0.8], [0.5]]) - - def testJointPredictionsWeightedFails(self): - language = feature_column.weighted_sparse_column( - feature_column.sparse_column_with_keys( - column_name="language", keys=["english", "finnish", "hindi"]), - "weight") - with ops.Graph().as_default(): - features = { - "weight": - constant_op.constant([[1], [2]]), - "language": - sparse_tensor.SparseTensor( - values=["hindi", "english"], - indices=[[0, 0], [1, 0]], - dense_shape=[2, 1]), - } - with self.assertRaises(AssertionError): - feature_column_ops.joint_weighted_sum_from_feature_columns( - features, [language], num_outputs=1) - - def testJointPredictionsRealFails(self): - age = feature_column.real_valued_column("age") - with ops.Graph().as_default(): - features = {"age": constant_op.constant([[1], [2]]),} - with self.assertRaises(NotImplementedError): - feature_column_ops.joint_weighted_sum_from_feature_columns( - features, [age], num_outputs=1) - - def testPredictionsWithWeightedSparseColumn(self): - language = feature_column.sparse_column_with_keys( - column_name="language", keys=["english", "finnish", "hindi"]) - weighted_language = feature_column.weighted_sparse_column( - sparse_id_column=language, weight_column_name="age") - with ops.Graph().as_default(): - features = { - "language": - sparse_tensor.SparseTensor( - values=["hindi", "english"], - indices=[[0, 0], [1, 0]], - dense_shape=[2, 1]), - "age": - sparse_tensor.SparseTensor( - values=[10.0, 20.0], - indices=[[0, 0], [1, 0]], - dense_shape=[2, 1]) - } - output, column_to_variable, bias = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [weighted_language], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - self.assertAllClose(output.eval(), [[0.], [0.]]) - - sess.run(bias.assign([0.1])) - self.assertAllClose(output.eval(), [[0.1], [0.1]]) - - # score: bias + age*language_weight[index] - sess.run(column_to_variable[weighted_language][0].assign([[0.1], [0.2], - [0.3]])) - self.assertAllClose(output.eval(), [[3.1], [2.1]]) - - def testPredictionsWithMultivalentColumnButNoCross(self): - language = feature_column.sparse_column_with_keys( - column_name="language", keys=["english", "turkish", "hindi"]) - with ops.Graph().as_default(): - features = { - "language": - sparse_tensor.SparseTensor( - values=["hindi", "english"], - indices=[[0, 0], [0, 1]], - dense_shape=[1, 2]) - } - output, column_to_variable, bias = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [language], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - # score: 0.1 + language_weight['hindi'] + language_weight['english'] - sess.run(bias.assign([0.1])) - sess.run(column_to_variable[language][0].assign([[0.1], [0.3], [0.2]])) - self.assertAllClose(output.eval(), [[0.4]]) - - def testSparseFeatureColumnWithHashedBucketSize(self): - movies = feature_column.sparse_column_with_hash_bucket( - column_name="movies", hash_bucket_size=15) - with ops.Graph().as_default(): - features = { - "movies": - sparse_tensor.SparseTensor( - values=["matrix", "head-on", "winter sleep"], - indices=[[0, 0], [0, 1], [1, 0]], - dense_shape=[2, 2]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [movies], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - weights = column_to_variable[movies][0] - self.assertEqual(weights.get_shape(), (15, 1)) - sess.run(weights.assign(weights + 0.4)) - # score for first example = 0.4 (matrix) + 0.4 (head-on) = 0.8 - # score for second example = 0.4 (winter sleep) - self.assertAllClose(output.eval(), [[0.8], [0.4]]) - - def testCrossUsageInPredictions(self): - language = feature_column.sparse_column_with_hash_bucket( - "language", hash_bucket_size=3) - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=5) - country_language = feature_column.crossed_column( - [language, country], hash_bucket_size=10) - with ops.Graph().as_default(): - features = { - "language": - sparse_tensor.SparseTensor( - values=["english", "spanish"], - indices=[[0, 0], [1, 0]], - dense_shape=[2, 1]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV"], - indices=[[0, 0], [1, 0]], - dense_shape=[2, 1]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [country_language], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - weights = column_to_variable[country_language][0] - sess.run(weights.assign(weights + 0.4)) - self.assertAllClose(output.eval(), [[0.4], [0.4]]) - - def testCrossColumnByItself(self): - language = feature_column.sparse_column_with_hash_bucket( - "language", hash_bucket_size=3) - language_language = feature_column.crossed_column( - [language, language], hash_bucket_size=10) - with ops.Graph().as_default(): - features = { - "language": - sparse_tensor.SparseTensor( - values=["english", "spanish"], - indices=[[0, 0], [0, 1]], - dense_shape=[1, 2]), - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [language_language], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - weights = column_to_variable[language_language][0] - sess.run(weights.assign(weights + 0.4)) - # There are two features inside language. If we cross it by itself we'll - # have four crossed features. - self.assertAllClose(output.eval(), [[1.6]]) - - def testMultivalentCrossUsageInPredictions(self): - language = feature_column.sparse_column_with_hash_bucket( - "language", hash_bucket_size=3) - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=5) - country_language = feature_column.crossed_column( - [language, country], hash_bucket_size=10) - with ops.Graph().as_default(): - features = { - "language": - sparse_tensor.SparseTensor( - values=["english", "spanish"], - indices=[[0, 0], [0, 1]], - dense_shape=[1, 2]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV"], - indices=[[0, 0], [0, 1]], - dense_shape=[1, 2]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [country_language], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - weights = column_to_variable[country_language][0] - sess.run(weights.assign(weights + 0.4)) - # There are four crosses each with 0.4 weight. - # score = 0.4 + 0.4 + 0.4 + 0.4 - self.assertAllClose(output.eval(), [[1.6]]) - - def testMultivalentCrossUsageInPredictionsWithPartition(self): - # bucket size has to be big enough to allow sharding. - language = feature_column.sparse_column_with_hash_bucket( - "language", hash_bucket_size=64 << 19) - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=64 << 18) - country_language = feature_column.crossed_column( - [language, country], hash_bucket_size=64 << 18) - with ops.Graph().as_default(): - features = { - "language": - sparse_tensor.SparseTensor( - values=["english", "spanish"], - indices=[[0, 0], [0, 1]], - dense_shape=[1, 2]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV"], - indices=[[0, 0], [0, 1]], - dense_shape=[1, 2]) - } - with variable_scope.variable_scope( - "weighted_sum_from_feature_columns", - features.values(), - partitioner=partitioned_variables.min_max_variable_partitioner( - max_partitions=10, min_slice_size=((64 << 20) - 1))) as scope: - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [country, language, country_language], - num_outputs=1, - scope=scope)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - self.assertEqual(2, len(column_to_variable[country])) - self.assertEqual(3, len(column_to_variable[language])) - self.assertEqual(2, len(column_to_variable[country_language])) - - weights = column_to_variable[country_language] - for partition_variable in weights: - sess.run(partition_variable.assign(partition_variable + 0.4)) - # There are four crosses each with 0.4 weight. - # score = 0.4 + 0.4 + 0.4 + 0.4 - self.assertAllClose(output.eval(), [[1.6]]) - - def testRealValuedColumnHavingMultiDimensions(self): - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=5) - age = feature_column.real_valued_column("age") - # The following RealValuedColumn has 3 dimensions. - incomes = feature_column.real_valued_column("incomes", 3) - - with ops.Graph().as_default(): - features = { - "age": - constant_op.constant([[1], [1]]), - "incomes": - constant_op.constant([[100., 200., 300.], [10., 20., 30.]]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV"], - indices=[[0, 0], [1, 0]], - dense_shape=[2, 2]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [country, age, incomes], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - incomes_weights = column_to_variable[incomes][0] - sess.run(incomes_weights.assign([[0.1], [0.2], [0.3]])) - self.assertAllClose(output.eval(), [[140.], [14.]]) - - def testMulticlassWithRealValuedColumnHavingMultiDimensionsAndSparse(self): - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=5) - age = feature_column.real_valued_column("age") - # The following RealValuedColumn has no predefined dimension so it - # can be missing. - height = feature_column._real_valued_var_len_column("height", - default_value=0, - is_sparse=False) - # The following RealValuedColumn has 3 dimensions. - incomes = feature_column.real_valued_column("incomes", 3) - with ops.Graph().as_default(): - features = { - "age": - constant_op.constant([[1], [1]]), - "incomes": - constant_op.constant([[100., 200., 300.], [10., 20., 30.]]), - "height": - constant_op.constant([[5., 4.], [0., 6.]]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV"], - indices=[[0, 0], [1, 0]], - dense_shape=[2, 2]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [country, age, height, incomes], num_outputs=5)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - height_weights = column_to_variable[height][0] - sess.run( - height_weights.assign( - [[1., 2., 3., 5., 10.], [1., 2., 3., 5., 10.]])) - self.assertAllClose(output.eval(), [[9., 18., 27., 45., 90.], - [6., 12., 18., 30., 60.]]) - - incomes_weights = column_to_variable[incomes][0] - sess.run( - incomes_weights.assign([[0.01, 0.1, 1., 10., 100.], - [0.02, 0.2, 2., 20., 200.], - [0.03, 0.3, 3., 30., 300.]])) - self.assertAllClose( - output.eval(), - [[14. + 9., 140. + 18., 1400. + 27., 14000. + 45., 140000. + 90.], - [1.4 + 6., 14. + 12., 140. + 18., 1400. + 30., 14000. + 60.]]) - - def testBucketizedColumn(self): - bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price"), boundaries=[0., 10., 100.]) - with ops.Graph().as_default(): - # buckets 2, 3, 0 - features = {"price": constant_op.constant([[20.], [110], [-3]])} - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [bucket], num_outputs=1)) - output_core = fc_core.linear_model(features, [bucket]) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - # Cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(output.eval(), output_core.eval()) - - sess.run(column_to_variable[bucket][0].assign([[0.1], [0.2], [0.3], - [0.4]])) - self.assertAllClose(output.eval(), [[0.3], [0.4], [0.1]]) - - def testBucketizedColumnHavingMultiDimensions(self): - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=5) - bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price", 2), - boundaries=[0., 10., 100.]) - with ops.Graph().as_default(): - # buckets 2, 3, 0 - features = { - "price": - constant_op.constant([[20., 210], [110, 50], [-3, -30]]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV"], - indices=[[0, 0], [1, 0]], - dense_shape=[3, 2]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [bucket, country], num_outputs=1)) - output_core = fc_core.linear_model(features, [bucket, country]) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - # Cross compatibility: Core builder output should equal to contrib. - self.assertAllEqual(output.eval(), output_core.eval()) - - # dimension = 2, bucket_size = 4, num_classes = 1 - sess.run(column_to_variable[bucket][0].assign( - [[0.1], [0.2], [0.3], [0.4], [1], [2], [3], [4]])) - self.assertAllClose(output.eval(), [[0.3 + 4], [0.4 + 3], [0.1 + 1]]) - - def testMulticlassWithBucketizedColumnHavingMultiDimensions(self): - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=5) - bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price", 2), - boundaries=[0., 10., 100.]) - with ops.Graph().as_default(): - # buckets 2, 3, 0 - features = { - "price": - constant_op.constant([[20., 210], [110, 50], [-3, -30]]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV"], - indices=[[0, 0], [1, 0]], - dense_shape=[3, 2]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [bucket, country], num_outputs=5)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - # dimension = 2, bucket_size = 4, num_classes = 5 - sess.run(column_to_variable[bucket][0].assign( - [[0.1, 1, 10, 100, 1000], [0.2, 2, 20, 200, 2000], - [0.3, 3, 30, 300, 3000], [0.4, 4, 40, 400, 4000], - [5, 50, 500, 5000, 50000], [6, 60, 600, 6000, 60000], - [7, 70, 700, 7000, 70000], [8, 80, 800, 8000, 80000]])) - self.assertAllClose( - output.eval(), - [[0.3 + 8, 3 + 80, 30 + 800, 300 + 8000, 3000 + 80000], - [0.4 + 7, 4 + 70, 40 + 700, 400 + 7000, 4000 + 70000], - [0.1 + 5, 1 + 50, 10 + 500, 100 + 5000, 1000 + 50000]]) - - def testCrossWithBucketizedColumn(self): - price_bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price"), boundaries=[0., 10., 100.]) - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=5) - country_price = feature_column.crossed_column( - [country, price_bucket], hash_bucket_size=10) - with ops.Graph().as_default(): - features = { - "price": - constant_op.constant([[20.]]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV"], - indices=[[0, 0], [0, 1]], - dense_shape=[1, 2]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [country_price], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - weights = column_to_variable[country_price][0] - sess.run(weights.assign(weights + 0.4)) - # There are two crosses each with 0.4 weight. - # score = 0.4 + 0.4 - self.assertAllClose(output.eval(), [[0.8]]) - - def testCrossWithCrossedColumn(self): - price_bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price"), boundaries=[0., 10., 100.]) - language = feature_column.sparse_column_with_hash_bucket( - "language", hash_bucket_size=3) - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=5) - country_language = feature_column.crossed_column( - [language, country], hash_bucket_size=10) - country_language_price = feature_column.crossed_column( - set([country_language, price_bucket]), hash_bucket_size=15) - with ops.Graph().as_default(): - features = { - "price": - constant_op.constant([[20.]]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV"], - indices=[[0, 0], [0, 1]], - dense_shape=[1, 2]), - "language": - sparse_tensor.SparseTensor( - values=["english", "spanish"], - indices=[[0, 0], [0, 1]], - dense_shape=[1, 2]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [country_language_price], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - weights = column_to_variable[country_language_price][0] - sess.run(weights.assign(weights + 0.4)) - # There are two crosses each with 0.4 weight. - # score = 0.4 + 0.4 + 0.4 + 0.4 - self.assertAllClose(output.eval(), [[1.6]]) - - def testIntegerizedColumn(self): - product = feature_column.sparse_column_with_integerized_feature( - "product", bucket_size=5) - with ops.Graph().as_default(): - features = { - "product": - sparse_tensor.SparseTensor( - values=[0, 4, 2], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [product], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - product_weights = column_to_variable[product][0] - sess.run(product_weights.assign([[0.1], [0.2], [0.3], [0.4], [0.5]])) - self.assertAllClose(output.eval(), [[0.1], [0.5], [0.3]]) - - def testIntegerizedColumnWithDenseInputTensor(self): - product = feature_column.sparse_column_with_integerized_feature( - "product", bucket_size=5) - with ops.Graph().as_default(): - features = {"product": constant_op.constant([[0], [4], [2]])} - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [product], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - product_weights = column_to_variable[product][0] - sess.run(product_weights.assign([[0.1], [0.2], [0.3], [0.4], [0.5]])) - self.assertAllClose(output.eval(), [[0.1], [0.5], [0.3]]) - - def testIntegerizedColumnWithDenseInputTensor2(self): - product = feature_column.sparse_column_with_integerized_feature( - "product", bucket_size=5) - with ops.Graph().as_default(): - features = {"product": constant_op.constant([[0, 4], [2, 3]])} - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [product], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - product_weights = column_to_variable[product][0] - sess.run(product_weights.assign([[0.1], [0.2], [0.3], [0.4], [0.5]])) - self.assertAllClose(output.eval(), [[0.6], [0.7]]) - - def testIntegerizedColumnWithInvalidId(self): - product = feature_column.sparse_column_with_integerized_feature( - "product", bucket_size=5) - with ops.Graph().as_default(): - features = { - "product": - sparse_tensor.SparseTensor( - values=[5, 4, 7], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [product], num_outputs=1)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - product_weights = column_to_variable[product][0] - sess.run(product_weights.assign([[0.1], [0.2], [0.3], [0.4], [0.5]])) - self.assertAllClose(output.eval(), [[0.1], [0.5], [0.3]]) - - def testMulticlassWithOnlyBias(self): - with ops.Graph().as_default(): - features = {"age": constant_op.constant([[10.], [20.], [30.], [40.]])} - output, _, bias = feature_column_ops.weighted_sum_from_feature_columns( - features, [feature_column.real_valued_column("age")], num_outputs=3) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - sess.run(bias.assign([0.1, 0.2, 0.3])) - self.assertAllClose(output.eval(), [[0.1, 0.2, 0.3], [0.1, 0.2, 0.3], - [0.1, 0.2, 0.3], [0.1, 0.2, 0.3]]) - - def testMulticlassWithRealValuedColumn(self): - with ops.Graph().as_default(): - column = feature_column.real_valued_column("age") - features = {"age": constant_op.constant([[10.], [20.], [30.], [40.]])} - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [column], num_outputs=3)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - weights = column_to_variable[column][0] - self.assertEqual(weights.get_shape(), (1, 3)) - sess.run(weights.assign([[0.01, 0.03, 0.05]])) - self.assertAllClose(output.eval(), [[0.1, 0.3, 0.5], [0.2, 0.6, 1.0], - [0.3, 0.9, 1.5], [0.4, 1.2, 2.0]]) - - def testMulticlassWithSparseColumn(self): - with ops.Graph().as_default(): - column = feature_column.sparse_column_with_keys( - column_name="language", - keys=["english", "arabic", "hindi", "russian", "swahili"]) - features = { - "language": - sparse_tensor.SparseTensor( - values=["hindi", "english", "arabic", "russian"], - indices=[[0, 0], [1, 0], [2, 0], [3, 0]], - dense_shape=[4, 1]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [column], num_outputs=3)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - weights = column_to_variable[column][0] - self.assertEqual(weights.get_shape(), (5, 3)) - sess.run( - weights.assign([[0.1, 0.4, 0.7], - [0.2, 0.5, 0.8], - [0.3, 0.6, 0.9], - [0.4, 0.7, 1.0], - [0.5, 0.8, 1.1]])) - self.assertAllClose(output.eval(), [[0.3, 0.6, 0.9], - [0.1, 0.4, 0.7], - [0.2, 0.5, 0.8], - [0.4, 0.7, 1.0]]) - - def testMulticlassWithBucketizedColumn(self): - column = feature_column.bucketized_column( - feature_column.real_valued_column("price"), - boundaries=[0., 100., 500., 1000.]) - with ops.Graph().as_default(): - # buckets 0, 2, 1, 2 - features = {"price": constant_op.constant([[-3], [110], [20.], [210]])} - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [column], num_outputs=3)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - weights = column_to_variable[column][0] - self.assertEqual(weights.get_shape(), (5, 3)) - sess.run( - weights.assign([[0.1, 0.4, 0.7], - [0.2, 0.5, 0.8], - [0.3, 0.6, 0.9], - [0.4, 0.7, 1.0], - [0.5, 0.8, 1.1]])) - self.assertAllClose(output.eval(), [[0.1, 0.4, 0.7], - [0.3, 0.6, 0.9], - [0.2, 0.5, 0.8], - [0.3, 0.6, 0.9]]) - - def testMulticlassWithCrossedColumn(self): - language = feature_column.sparse_column_with_hash_bucket( - "language", hash_bucket_size=3) - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=2) - column = feature_column.crossed_column( - {language, country}, hash_bucket_size=5) - with ops.Graph().as_default(): - features = { - "language": - sparse_tensor.SparseTensor( - values=["english", "spanish", "russian", "swahili"], - indices=[[0, 0], [1, 0], [2, 0], [3, 0]], - dense_shape=[4, 1]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV", "RU", "KE"], - indices=[[0, 0], [1, 0], [2, 0], [3, 0]], - dense_shape=[4, 1]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [column], num_outputs=3)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - weights = column_to_variable[column][0] - self.assertEqual(weights.get_shape(), (5, 3)) - sess.run( - weights.assign([[0.1, 0.4, 0.7], - [0.2, 0.5, 0.8], - [0.3, 0.6, 0.9], - [0.4, 0.7, 1.0], - [0.5, 0.8, 1.1]])) - self.assertAllClose(array_ops.shape(output).eval(), [4, 3]) - - def testMulticlassWithMultivalentColumn(self): - column = feature_column.sparse_column_with_keys( - column_name="language", - keys=["english", "turkish", "hindi", "russian", "swahili"]) - with ops.Graph().as_default(): - features = { - "language": - sparse_tensor.SparseTensor( - values=["hindi", "english", "turkish", "turkish", "english"], - indices=[[0, 0], [0, 1], [1, 0], [2, 0], [3, 0]], - dense_shape=[4, 2]) - } - output, column_to_variable, _ = ( - feature_column_ops.weighted_sum_from_feature_columns( - features, [column], num_outputs=3)) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - lookup_ops.tables_initializer().run() - - weights = column_to_variable[column][0] - self.assertEqual(weights.get_shape(), (5, 3)) - sess.run( - weights.assign([[0.1, 0.4, 0.7], - [0.2, 0.5, 0.8], - [0.3, 0.6, 0.9], - [0.4, 0.7, 1.0], - [0.5, 0.8, 1.1]])) - self.assertAllClose(output.eval(), [[0.4, 1.0, 1.6], - [0.2, 0.5, 0.8], - [0.2, 0.5, 0.8], - [0.1, 0.4, 0.7]]) - - def testVariablesAddedToCollection(self): - price_bucket = feature_column.bucketized_column( - feature_column.real_valued_column("price"), boundaries=[0., 10., 100.]) - country = feature_column.sparse_column_with_hash_bucket( - "country", hash_bucket_size=5) - country_price = feature_column.crossed_column( - [country, price_bucket], hash_bucket_size=10) - with ops.Graph().as_default(): - features = { - "price": - constant_op.constant([[20.]]), - "country": - sparse_tensor.SparseTensor( - values=["US", "SV"], - indices=[[0, 0], [0, 1]], - dense_shape=[1, 2]) - } - feature_column_ops.weighted_sum_from_feature_columns( - features, [country_price, price_bucket], - num_outputs=1, - weight_collections=["my_collection"]) - weights = ops.get_collection("my_collection") - # 3 = bias + price_bucket + country_price - self.assertEqual(3, len(weights)) - - -class ParseExampleTest(test.TestCase): - - def testParseExample(self): - bucket = feature_column.bucketized_column( - feature_column.real_valued_column( - "price", dimension=3), - boundaries=[0., 10., 100.]) - wire_cast = feature_column.sparse_column_with_keys( - "wire_cast", ["marlo", "omar", "stringer"]) - # buckets 2, 3, 0 - data = example_pb2.Example(features=feature_pb2.Features(feature={ - "price": - feature_pb2.Feature(float_list=feature_pb2.FloatList( - value=[20., 110, -3])), - "wire_cast": - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b"stringer", b"marlo"])), - })) - output = feature_column_ops.parse_feature_columns_from_examples( - serialized=[data.SerializeToString()], - feature_columns=[bucket, wire_cast]) - self.assertIn(bucket, output) - self.assertIn(wire_cast, output) - with self.cached_session(): - lookup_ops.tables_initializer().run() - self.assertAllEqual(output[bucket].eval(), [[2, 3, 0]]) - self.assertAllEqual(output[wire_cast].indices.eval(), [[0, 0], [0, 1]]) - self.assertAllEqual(output[wire_cast].values.eval(), [2, 0]) - - def testParseSequenceExample(self): - location_keys = ["east_side", "west_side", "nyc"] - embedding_dimension = 10 - - location = feature_column.sparse_column_with_keys( - "location", keys=location_keys) - location_onehot = feature_column.one_hot_column(location) - wire_cast = feature_column.sparse_column_with_keys( - "wire_cast", ["marlo", "omar", "stringer"]) - wire_cast_embedded = feature_column.embedding_column( - wire_cast, dimension=embedding_dimension) - measurements = feature_column.real_valued_column( - "measurements", dimension=2) - - context_feature_columns = [location_onehot] - sequence_feature_columns = [wire_cast_embedded, measurements] - - sequence_example = example_pb2.SequenceExample( - context=feature_pb2.Features(feature={ - "location": - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b"west_side"])), - }), - feature_lists=feature_pb2.FeatureLists(feature_list={ - "wire_cast": - feature_pb2.FeatureList(feature=[ - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b"marlo", b"stringer"])), - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b"omar", b"stringer", b"marlo"])), - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b"marlo"])), - ]), - "measurements": - feature_pb2.FeatureList(feature=[ - feature_pb2.Feature(float_list=feature_pb2.FloatList( - value=[0.2, 0.3])), - feature_pb2.Feature(float_list=feature_pb2.FloatList( - value=[0.1, 0.8])), - feature_pb2.Feature(float_list=feature_pb2.FloatList( - value=[0.5, 0.0])), - ]) - })) - - ctx, seq = feature_column_ops.parse_feature_columns_from_sequence_examples( - serialized=sequence_example.SerializeToString(), - context_feature_columns=context_feature_columns, - sequence_feature_columns=sequence_feature_columns) - - self.assertIn("location", ctx) - self.assertIsInstance(ctx["location"], sparse_tensor.SparseTensor) - self.assertIn("wire_cast", seq) - self.assertIsInstance(seq["wire_cast"], sparse_tensor.SparseTensor) - self.assertIn("measurements", seq) - self.assertIsInstance(seq["measurements"], ops.Tensor) - - with self.cached_session() as sess: - location_val, wire_cast_val, measurement_val = sess.run( - [ctx["location"], seq["wire_cast"], seq["measurements"]]) - - self.assertAllEqual(location_val.indices, np.array([[0]])) - self.assertAllEqual(location_val.values, np.array([b"west_side"])) - self.assertAllEqual(location_val.dense_shape, np.array([1])) - - self.assertAllEqual(wire_cast_val.indices, - np.array( - [[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [2, 0]])) - self.assertAllEqual( - wire_cast_val.values, - np.array( - [b"marlo", b"stringer", b"omar", b"stringer", b"marlo", b"marlo"])) - self.assertAllEqual(wire_cast_val.dense_shape, np.array([3, 3])) - - self.assertAllClose(measurement_val, - np.array([[0.2, 0.3], [0.1, 0.8], [0.5, 0.0]])) - - -class InferRealValuedColumnTest(test.TestCase): - - def testTensorInt32(self): - self.assertEqual( - feature_column_ops.infer_real_valued_columns( - array_ops.zeros( - shape=[33, 4], dtype=dtypes.int32)), [ - feature_column.real_valued_column( - "", dimension=4, dtype=dtypes.int32) - ]) - - def testTensorInt64(self): - self.assertEqual( - feature_column_ops.infer_real_valued_columns( - array_ops.zeros( - shape=[33, 4], dtype=dtypes.int64)), [ - feature_column.real_valued_column( - "", dimension=4, dtype=dtypes.int64) - ]) - - def testTensorFloat32(self): - self.assertEqual( - feature_column_ops.infer_real_valued_columns( - array_ops.zeros( - shape=[33, 4], dtype=dtypes.float32)), [ - feature_column.real_valued_column( - "", dimension=4, dtype=dtypes.float32) - ]) - - def testTensorFloat64(self): - self.assertEqual( - feature_column_ops.infer_real_valued_columns( - array_ops.zeros( - shape=[33, 4], dtype=dtypes.float64)), [ - feature_column.real_valued_column( - "", dimension=4, dtype=dtypes.float64) - ]) - - def testDictionary(self): - self.assertItemsEqual( - feature_column_ops.infer_real_valued_columns({ - "a": array_ops.zeros( - shape=[33, 4], dtype=dtypes.int32), - "b": array_ops.zeros( - shape=[3, 2], dtype=dtypes.float32) - }), [ - feature_column.real_valued_column( - "a", dimension=4, dtype=dtypes.int32), - feature_column.real_valued_column( - "b", dimension=2, dtype=dtypes.float32) - ]) - - def testNotGoodDtype(self): - with self.assertRaises(ValueError): - feature_column_ops.infer_real_valued_columns( - constant_op.constant( - [["a"]], dtype=dtypes.string)) - - def testSparseTensor(self): - with self.assertRaises(ValueError): - feature_column_ops.infer_real_valued_columns( - sparse_tensor.SparseTensor( - indices=[[0, 0]], values=["a"], dense_shape=[1, 1])) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/layers/python/layers/feature_column_test.py b/tensorflow/contrib/layers/python/layers/feature_column_test.py deleted file mode 100644 index cab8da808b6..00000000000 --- a/tensorflow/contrib/layers/python/layers/feature_column_test.py +++ /dev/null @@ -1,1212 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for layers.feature_column.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import copy -import itertools -import os -import tempfile - -import numpy as np - -from tensorflow.contrib.layers.python.layers import feature_column as fc -from tensorflow.contrib.layers.python.layers import feature_column_ops -from tensorflow.python.feature_column import feature_column_lib as fc_core -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import saver - - -def _sparse_id_tensor(shape, vocab_size, seed=112123): - # Returns a arbitrary `SparseTensor` with given shape and vocab size. - np.random.seed(seed) - indices = np.array(list(itertools.product(*[range(s) for s in shape]))) - - # In order to create some sparsity, we include a value outside the vocab. - values = np.random.randint(0, vocab_size + 1, size=np.prod(shape)) - - # Remove entries outside the vocabulary. - keep = values < vocab_size - indices = indices[keep] - values = values[keep] - - return sparse_tensor_lib.SparseTensor( - indices=indices, values=values, dense_shape=shape) - - -def _sparse_id_tensor_with_weights(shape, vocab_size, seed=112123): - # Returns a arbitrary `SparseTensor` with given shape and vocab size. - assert vocab_size >= shape[-1] - np.random.seed(seed) - indices = np.array(list(itertools.product(*[range(s) for s in shape]))) - - # Values must be distinct from the vocab - values = np.ndarray.flatten(np.array([ - np.random.choice(vocab_size, size=shape[-1], replace=False) - for _ in range(np.prod(shape[:-1]))])) - weights = np.sort(np.random.rand(*shape), axis=len(shape)-1) - - # Remove entries if weight < 0.5 for sparsity. - keep = np.ndarray.flatten(weights < 0.5) # Remove half of them - indices = indices[keep] - values = values[keep] - weights = np.ndarray.flatten(weights)[keep] - return (sparse_tensor_lib.SparseTensor( - indices=indices, values=values, dense_shape=shape), - sparse_tensor_lib.SparseTensor( - indices=indices, values=weights, dense_shape=shape)) - - -class FeatureColumnTest(test.TestCase): - - def testImmutability(self): - a = fc.sparse_column_with_hash_bucket("aaa", hash_bucket_size=100) - with self.assertRaises(AttributeError): - a.column_name = "bbb" - - def testSparseColumnWithHashBucket(self): - a = fc.sparse_column_with_hash_bucket("aaa", hash_bucket_size=100) - self.assertEqual(a.name, "aaa") - self.assertEqual(a.dtype, dtypes.string) - - a = fc.sparse_column_with_hash_bucket( - "aaa", hash_bucket_size=100, dtype=dtypes.int64) - self.assertEqual(a.name, "aaa") - self.assertEqual(a.dtype, dtypes.int64) - - with self.assertRaisesRegexp(ValueError, "dtype must be string or integer"): - a = fc.sparse_column_with_hash_bucket( - "aaa", hash_bucket_size=100, dtype=dtypes.float32) - - def testSparseColumnWithVocabularyFile(self): - b = fc.sparse_column_with_vocabulary_file( - "bbb", vocabulary_file="a_file", vocab_size=454) - self.assertEqual(b.dtype, dtypes.string) - self.assertEqual(b.lookup_config.vocab_size, 454) - self.assertEqual(b.lookup_config.vocabulary_file, "a_file") - - with self.assertRaises(ValueError): - # Vocabulary size should be defined if vocabulary_file is used. - fc.sparse_column_with_vocabulary_file("bbb", vocabulary_file="somefile") - - b = fc.sparse_column_with_vocabulary_file( - "bbb", vocabulary_file="a_file", vocab_size=454, dtype=dtypes.int64) - self.assertEqual(b.dtype, dtypes.int64) - - with self.assertRaisesRegexp(ValueError, "dtype must be string or integer"): - b = fc.sparse_column_with_vocabulary_file( - "bbb", vocabulary_file="a_file", vocab_size=454, dtype=dtypes.float32) - - def testWeightedSparseColumn(self): - ids = fc.sparse_column_with_keys("ids", ["marlo", "omar", "stringer"]) - weighted_ids = fc.weighted_sparse_column(ids, "weights") - self.assertEqual(weighted_ids.name, "ids_weighted_by_weights") - - def testWeightedSparseColumnWithVocabularyFile(self): - ids = fc.sparse_column_with_vocabulary_file( - "ids", "a_file", num_oov_buckets=7, vocab_size=3) - weighted_ids = fc.weighted_sparse_column(ids, "weights") - self.assertEqual(weighted_ids.name, "ids_weighted_by_weights") - self.assertEqual(weighted_ids.lookup_config, ids.lookup_config) - self.assertEqual(weighted_ids.lookup_config.vocab_size, 3) - self.assertEqual(weighted_ids.lookup_config.num_oov_buckets, 7) - self.assertEqual(weighted_ids.lookup_config.vocabulary_file, "a_file") - - def testWeightedSparseColumnDeepCopy(self): - ids = fc.sparse_column_with_keys("ids", ["marlo", "omar", "stringer"]) - weighted = fc.weighted_sparse_column(ids, "weights") - weighted_copy = copy.deepcopy(weighted) - self.assertEqual(weighted_copy.sparse_id_column.name, "ids") - self.assertEqual(weighted_copy.weight_column_name, "weights") - self.assertEqual(weighted_copy.name, "ids_weighted_by_weights") - - def testEmbeddingColumn(self): - a = fc.sparse_column_with_hash_bucket( - "aaa", hash_bucket_size=100, combiner="sum") - b = fc.embedding_column(a, dimension=4, combiner="mean") - self.assertEqual(b.sparse_id_column.name, "aaa") - self.assertEqual(b.dimension, 4) - self.assertEqual(b.combiner, "mean") - - def testEmbeddingColumnDeepCopy(self): - a = fc.sparse_column_with_hash_bucket( - "aaa", hash_bucket_size=100, combiner="sum") - column = fc.embedding_column(a, dimension=4, combiner="mean") - column_copy = copy.deepcopy(column) - self.assertEqual(column_copy.name, "aaa_embedding") - self.assertEqual(column_copy.sparse_id_column.name, "aaa") - self.assertEqual(column_copy.dimension, 4) - self.assertEqual(column_copy.combiner, "mean") - - def testScatteredEmbeddingColumn(self): - column = fc.scattered_embedding_column( - "aaa", size=100, dimension=10, hash_key=1) - self.assertEqual(column.column_name, "aaa") - self.assertEqual(column.size, 100) - self.assertEqual(column.dimension, 10) - self.assertEqual(column.hash_key, 1) - self.assertEqual(column.name, "aaa_scattered_embedding") - - def testScatteredEmbeddingColumnDeepCopy(self): - column = fc.scattered_embedding_column( - "aaa", size=100, dimension=10, hash_key=1) - column_copy = copy.deepcopy(column) - self.assertEqual(column_copy.column_name, "aaa") - self.assertEqual(column_copy.size, 100) - self.assertEqual(column_copy.dimension, 10) - self.assertEqual(column_copy.hash_key, 1) - self.assertEqual(column_copy.name, "aaa_scattered_embedding") - - def testSharedEmbeddingColumn(self): - a1 = fc.sparse_column_with_keys("a1", ["marlo", "omar", "stringer"]) - a2 = fc.sparse_column_with_keys("a2", ["marlo", "omar", "stringer"]) - b = fc.shared_embedding_columns([a1, a2], dimension=4, combiner="mean") - self.assertEqual(len(b), 2) - self.assertEqual(b[0].shared_embedding_name, "a1_a2_shared_embedding") - self.assertEqual(b[1].shared_embedding_name, "a1_a2_shared_embedding") - - # Create a sparse id tensor for a1. - input_tensor_c1 = sparse_tensor_lib.SparseTensor( - indices=[[0, 0], [1, 1], [2, 2]], values=[0, 1, 2], dense_shape=[3, 3]) - # Create a sparse id tensor for a2. - input_tensor_c2 = sparse_tensor_lib.SparseTensor( - indices=[[0, 0], [1, 1], [2, 2]], values=[0, 1, 2], dense_shape=[3, 3]) - with variable_scope.variable_scope("run_1"): - b1 = feature_column_ops.input_from_feature_columns({ - b[0]: input_tensor_c1 - }, [b[0]]) - b2 = feature_column_ops.input_from_feature_columns({ - b[1]: input_tensor_c2 - }, [b[1]]) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - b1_value = b1.eval() - b2_value = b2.eval() - for i in range(len(b1_value)): - self.assertAllClose(b1_value[i], b2_value[i]) - - # Test the case when a shared_embedding_name is explicitly specified. - d = fc.shared_embedding_columns( - [a1, a2], - dimension=4, - combiner="mean", - shared_embedding_name="my_shared_embedding") - # a3 is a completely different sparse column with a1 and a2, but since the - # same shared_embedding_name is passed in, a3 will have the same embedding - # as a1 and a2 - a3 = fc.sparse_column_with_keys("a3", [42, 1, -1000], dtype=dtypes.int32) - e = fc.shared_embedding_columns( - [a3], - dimension=4, - combiner="mean", - shared_embedding_name="my_shared_embedding") - with variable_scope.variable_scope("run_2"): - d1 = feature_column_ops.input_from_feature_columns({ - d[0]: input_tensor_c1 - }, [d[0]]) - e1 = feature_column_ops.input_from_feature_columns({ - e[0]: input_tensor_c1 - }, [e[0]]) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - d1_value = d1.eval() - e1_value = e1.eval() - for i in range(len(d1_value)): - self.assertAllClose(d1_value[i], e1_value[i]) - - def testSharedEmbeddingColumnWithWeightedSparseColumn(self): - # Tests creation of shared embeddings containing weighted sparse columns. - sparse_col = fc.sparse_column_with_keys("a1", ["marlo", "omar", "stringer"]) - ids = fc.sparse_column_with_keys("ids", ["marlo", "omar", "stringer"]) - weighted_sparse_col = fc.weighted_sparse_column(ids, "weights") - self.assertEqual(weighted_sparse_col.name, "ids_weighted_by_weights") - - b = fc.shared_embedding_columns( - [sparse_col, weighted_sparse_col], dimension=4, combiner="mean") - self.assertEqual(len(b), 2) - self.assertEqual(b[0].shared_embedding_name, - "a1_ids_weighted_by_weights_shared_embedding") - self.assertEqual(b[1].shared_embedding_name, - "a1_ids_weighted_by_weights_shared_embedding") - - # Tries reversing order to check compatibility condition. - b = fc.shared_embedding_columns( - [weighted_sparse_col, sparse_col], dimension=4, combiner="mean") - self.assertEqual(len(b), 2) - self.assertEqual(b[0].shared_embedding_name, - "a1_ids_weighted_by_weights_shared_embedding") - self.assertEqual(b[1].shared_embedding_name, - "a1_ids_weighted_by_weights_shared_embedding") - - # Tries adding two weighted columns to check compatibility between them. - weighted_sparse_col_2 = fc.weighted_sparse_column(ids, "weights_2") - b = fc.shared_embedding_columns( - [weighted_sparse_col, weighted_sparse_col_2], - dimension=4, - combiner="mean") - self.assertEqual(len(b), 2) - self.assertEqual( - b[0].shared_embedding_name, - "ids_weighted_by_weights_ids_weighted_by_weights_2_shared_embedding") - self.assertEqual( - b[1].shared_embedding_name, - "ids_weighted_by_weights_ids_weighted_by_weights_2_shared_embedding") - - def testSharedEmbeddingColumnDeterminism(self): - # Tests determinism in auto-generated shared_embedding_name. - sparse_id_columns = tuple([ - fc.sparse_column_with_keys(k, ["foo", "bar"]) - for k in ["07", "02", "00", "03", "05", "01", "09", "06", "04", "08"] - ]) - output = fc.shared_embedding_columns( - sparse_id_columns, dimension=2, combiner="mean") - self.assertEqual(len(output), 10) - for x in output: - self.assertEqual(x.shared_embedding_name, - "00_01_02_plus_7_others_shared_embedding") - - def testSharedEmbeddingColumnErrors(self): - # Tries passing in a string. - with self.assertRaises(TypeError): - invalid_string = "Invalid string." - fc.shared_embedding_columns(invalid_string, dimension=2, combiner="mean") - - # Tries passing in a set of sparse columns. - with self.assertRaises(TypeError): - invalid_set = set([ - fc.sparse_column_with_keys("a", ["foo", "bar"]), - fc.sparse_column_with_keys("b", ["foo", "bar"]), - ]) - fc.shared_embedding_columns(invalid_set, dimension=2, combiner="mean") - - def testSharedEmbeddingColumnDeepCopy(self): - a1 = fc.sparse_column_with_keys("a1", ["marlo", "omar", "stringer"]) - a2 = fc.sparse_column_with_keys("a2", ["marlo", "omar", "stringer"]) - columns = fc.shared_embedding_columns( - [a1, a2], dimension=4, combiner="mean") - columns_copy = copy.deepcopy(columns) - self.assertEqual(columns_copy[0].shared_embedding_name, - "a1_a2_shared_embedding") - self.assertEqual(columns_copy[1].shared_embedding_name, - "a1_a2_shared_embedding") - - def testOneHotColumn(self): - a = fc.sparse_column_with_keys("a", ["a", "b", "c", "d"]) - onehot_a = fc.one_hot_column(a) - self.assertEqual(onehot_a.sparse_id_column.name, "a") - self.assertEqual(onehot_a.length, 4) - - b = fc.sparse_column_with_hash_bucket( - "b", hash_bucket_size=100, combiner="sum") - onehot_b = fc.one_hot_column(b) - self.assertEqual(onehot_b.sparse_id_column.name, "b") - self.assertEqual(onehot_b.length, 100) - - def testOneHotReshaping(self): - """Tests reshaping behavior of `OneHotColumn`.""" - id_tensor_shape = [3, 2, 4, 5] - - sparse_column = fc.sparse_column_with_keys( - "animals", ["squirrel", "moose", "dragon", "octopus"]) - one_hot = fc.one_hot_column(sparse_column) - - vocab_size = len(sparse_column.lookup_config.keys) - id_tensor = _sparse_id_tensor(id_tensor_shape, vocab_size) - - for output_rank in range(1, len(id_tensor_shape) + 1): - with variable_scope.variable_scope("output_rank_{}".format(output_rank)): - one_hot_output = one_hot._to_dnn_input_layer( - id_tensor, output_rank=output_rank) - with self.cached_session() as sess: - one_hot_value = sess.run(one_hot_output) - expected_shape = (id_tensor_shape[:output_rank - 1] + [vocab_size]) - self.assertEquals(expected_shape, list(one_hot_value.shape)) - - def testOneHotColumnForWeightedSparseColumn(self): - ids = fc.sparse_column_with_keys("ids", ["marlo", "omar", "stringer"]) - weighted_ids = fc.weighted_sparse_column(ids, "weights") - one_hot = fc.one_hot_column(weighted_ids) - self.assertEqual(one_hot.sparse_id_column.name, "ids_weighted_by_weights") - self.assertEqual(one_hot.length, 3) - - def testIntegerizedOneHotColumnForWeightedSparseColumn(self): - vocab_size = 5 - ids = fc.sparse_column_with_integerized_feature("ids", vocab_size) - weighted_ids = fc.weighted_sparse_column(ids, "weights") - one_hot = fc.one_hot_column(weighted_ids) - self.assertEqual(one_hot.sparse_id_column.name, "ids_weighted_by_weights") - self.assertEqual(one_hot.length, vocab_size) - - def testIntegerizedOneHotWeightedSparseColumnShape(self): - vocab_size = 5 - for id_tensor_shape in [[4, 3], [2, 4], [3, 3, 3]]: - output_rank = len(id_tensor_shape) - a = fc.sparse_column_with_integerized_feature("a", vocab_size) - weighted = fc.weighted_sparse_column(a, "weights") - one_hot = fc.one_hot_column(weighted) - id_tensor, weight_tensor = _sparse_id_tensor_with_weights( - id_tensor_shape, vocab_size) - - one_hot_output = one_hot._to_dnn_input_layer( - (id_tensor, weight_tensor), - output_rank=output_rank) - one_hot_output_shape = one_hot_output.get_shape().as_list() - expected_shape = id_tensor_shape[:-1] + [vocab_size] - self.assertEquals(expected_shape, one_hot_output_shape) - with self.cached_session() as sess: - one_hot_value = sess.run(one_hot_output) - self.assertEquals(expected_shape, list(one_hot_value.shape)) - - def testOneHotColumnWithSparseColumnWithHashKeys(self): - input_values = ["marlo", "unknown", "omar"] - inputs = constant_op.constant(input_values) - hash_keys = [[10, 20], [20, 30]] - hash_column = fc.sparse_column_with_hash_bucket( - column_name="ids", hash_bucket_size=10, hash_keys=hash_keys) - columns_to_tensors = {} - columns_to_tensors["ids"] = inputs - hash_column.insert_transformed_feature(columns_to_tensors) - self.assertEqual(len(columns_to_tensors), 2) - self.assertTrue(hash_column in columns_to_tensors) - - one_hot_column = fc.one_hot_column(hash_column) - one_hot_output = one_hot_column._to_dnn_input_layer( - columns_to_tensors[hash_column]) - - expected = np.array([[0., 1., 0., 0., 0., 0., 0., 1., 0., - 0.], [0., 1., 0., 0., 0., 0., 0., 0., 0., 1.], - [1., 0., 0., 0., 0., 0., 0., 0., 0., 1.]]) - with self.cached_session() as sess: - one_hot_value = sess.run(one_hot_output) - self.assertTrue(np.array_equal(one_hot_value, expected)) - - def testSparseColumnWithHashKeysWithUnexpectedHashKeys(self): - with self.assertRaisesRegexp(ValueError, - "hash_keys must be a non-empty list."): - fc.sparse_column_with_hash_bucket( - column_name="ids", hash_bucket_size=100, hash_keys=[]) - - with self.assertRaisesRegexp(ValueError, - "hash_keys must be a non-empty list."): - fc.sparse_column_with_hash_bucket( - column_name="ids", hash_bucket_size=100, hash_keys=1) - - with self.assertRaisesRegexp( - ValueError, "Each element of hash_keys must be a pair of integers."): - fc.sparse_column_with_hash_bucket( - column_name="ids", hash_bucket_size=100, hash_keys=[1, 2]) - - with self.assertRaisesRegexp( - ValueError, "Each element of hash_keys must be a pair of integers."): - fc.sparse_column_with_hash_bucket( - column_name="ids", hash_bucket_size=100, hash_keys=["key"]) - - with self.assertRaisesRegexp( - ValueError, "Each element of hash_keys must be a pair of integers."): - fc.sparse_column_with_hash_bucket( - column_name="ids", hash_bucket_size=100, hash_keys=[[1, 2.0]]) - - def testMissingValueInOneHotColumnForWeightedSparseColumn(self): - # Github issue 12583 - ids = fc.sparse_column_with_keys("ids", ["marlo", "omar", "stringer"]) - weighted_ids = fc.weighted_sparse_column(ids, "weights") - one_hot = fc.one_hot_column(weighted_ids) - features = { - "ids": constant_op.constant([["marlo", "unknown", "omar"]]), - "weights": constant_op.constant([[2., 4., 6.]]) - } - one_hot_tensor = feature_column_ops.input_from_feature_columns( - features, [one_hot]) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - sess.run(lookup_ops.tables_initializer()) - self.assertAllEqual([[2., 6., 0.]], one_hot_tensor.eval()) - - def testMissingValueInOneHotColumnForSparseColumnWithKeys(self): - ids = fc.sparse_column_with_keys("ids", ["marlo", "omar", "stringer"]) - one_hot = fc.one_hot_column(ids) - features = {"ids": constant_op.constant([["marlo", "unknown", "omar"]])} - one_hot_tensor = feature_column_ops.input_from_feature_columns( - features, [one_hot]) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - sess.run(lookup_ops.tables_initializer()) - self.assertAllEqual([[1., 1., 0.]], one_hot_tensor.eval()) - - def testOneHotColumnDeepCopy(self): - a = fc.sparse_column_with_keys("a", ["a", "b", "c", "d"]) - column = fc.one_hot_column(a) - column_copy = copy.deepcopy(column) - self.assertEqual(column_copy.sparse_id_column.name, "a") - self.assertEqual(column.name, "a_one_hot") - self.assertEqual(column.length, 4) - - def testRealValuedVarLenColumn(self): - c = fc._real_valued_var_len_column("ccc", is_sparse=True) - self.assertTrue(c.is_sparse) - self.assertTrue(c.default_value is None) - # default_value is an integer. - c5 = fc._real_valued_var_len_column("c5", default_value=2) - self.assertEqual(c5.default_value, 2) - # default_value is a float. - d4 = fc._real_valued_var_len_column("d4", is_sparse=True) - self.assertEqual(d4.default_value, None) - self.assertEqual(d4.is_sparse, True) - # Default value is a list but dimension is None. - with self.assertRaisesRegexp(ValueError, "Only scalar default value.*"): - fc._real_valued_var_len_column("g5", default_value=[2., 3.]) - - def testRealValuedVarLenColumnDtypes(self): - rvc = fc._real_valued_var_len_column("rvc", is_sparse=True) - self.assertDictEqual( - { - "rvc": parsing_ops.VarLenFeature(dtype=dtypes.float32) - }, rvc.config) - - rvc = fc._real_valued_var_len_column( - "rvc", default_value=0, is_sparse=False) - self.assertDictEqual({ - "rvc": - parsing_ops.FixedLenSequenceFeature( - shape=[], - dtype=dtypes.float32, - allow_missing=True, - default_value=0.0) - }, rvc.config) - - rvc = fc._real_valued_var_len_column( - "rvc", dtype=dtypes.int32, default_value=0, is_sparse=True) - self.assertDictEqual( - { - "rvc": parsing_ops.VarLenFeature(dtype=dtypes.int32) - }, rvc.config) - - with self.assertRaisesRegexp(TypeError, - "dtype must be convertible to float"): - fc._real_valued_var_len_column( - "rvc", dtype=dtypes.string, default_value="", is_sparse=True) - - def testRealValuedColumn(self): - a = fc.real_valued_column("aaa") - self.assertEqual(a.name, "aaa") - self.assertEqual(a.dimension, 1) - b = fc.real_valued_column("bbb", 10) - self.assertEqual(b.dimension, 10) - self.assertTrue(b.default_value is None) - - with self.assertRaisesRegexp(TypeError, "dimension must be an integer"): - fc.real_valued_column("d3", dimension=1.0) - - with self.assertRaisesRegexp(ValueError, - "dimension must be greater than 0"): - fc.real_valued_column("d3", dimension=0) - - with self.assertRaisesRegexp(ValueError, - "dtype must be convertible to float"): - fc.real_valued_column("d3", dtype=dtypes.string) - - # default_value is an integer. - c1 = fc.real_valued_column("c1", default_value=2) - self.assertListEqual(list(c1.default_value), [2.]) - c2 = fc.real_valued_column("c2", default_value=2, dtype=dtypes.int32) - self.assertListEqual(list(c2.default_value), [2]) - c3 = fc.real_valued_column("c3", dimension=4, default_value=2) - self.assertListEqual(list(c3.default_value), [2, 2, 2, 2]) - c4 = fc.real_valued_column( - "c4", dimension=4, default_value=2, dtype=dtypes.int32) - self.assertListEqual(list(c4.default_value), [2, 2, 2, 2]) - - # default_value is a float. - d1 = fc.real_valued_column("d1", default_value=2.) - self.assertListEqual(list(d1.default_value), [2.]) - d2 = fc.real_valued_column("d2", dimension=4, default_value=2.) - self.assertListEqual(list(d2.default_value), [2., 2., 2., 2.]) - with self.assertRaisesRegexp(TypeError, - "default_value must be compatible with dtype"): - fc.real_valued_column("d3", default_value=2., dtype=dtypes.int32) - - # default_value is neither integer nor float. - with self.assertRaisesRegexp(TypeError, - "default_value must be compatible with dtype"): - fc.real_valued_column("e1", default_value="string") - with self.assertRaisesRegexp(TypeError, - "default_value must be compatible with dtype"): - fc.real_valued_column("e1", dimension=3, default_value=[1, 3., "string"]) - - # default_value is a list of integers. - f1 = fc.real_valued_column("f1", default_value=[2]) - self.assertListEqual(list(f1.default_value), [2]) - f2 = fc.real_valued_column("f2", dimension=3, default_value=[2, 2, 2]) - self.assertListEqual(list(f2.default_value), [2., 2., 2.]) - f3 = fc.real_valued_column( - "f3", dimension=3, default_value=[2, 2, 2], dtype=dtypes.int32) - self.assertListEqual(list(f3.default_value), [2, 2, 2]) - - # default_value is a list of floats. - g1 = fc.real_valued_column("g1", default_value=[2.]) - self.assertListEqual(list(g1.default_value), [2.]) - g2 = fc.real_valued_column("g2", dimension=3, default_value=[2., 2, 2]) - self.assertListEqual(list(g2.default_value), [2., 2., 2.]) - with self.assertRaisesRegexp(TypeError, - "default_value must be compatible with dtype"): - fc.real_valued_column("g3", default_value=[2.], dtype=dtypes.int32) - with self.assertRaisesRegexp( - ValueError, "The length of default_value must be equal to dimension"): - fc.real_valued_column("g4", dimension=3, default_value=[2.]) - - # Test that the normalizer_fn gets stored for a real_valued_column - normalizer = lambda x: x - 1 - h1 = fc.real_valued_column("h1", normalizer=normalizer) - self.assertEqual(normalizer(10), h1.normalizer_fn(10)) - - # Test that normalizer is not stored within key - self.assertFalse("normalizer" in g1.key) - self.assertFalse("normalizer" in g2.key) - self.assertFalse("normalizer" in h1.key) - - def testRealValuedColumnReshaping(self): - """Tests reshaping behavior of `RealValuedColumn`.""" - batch_size = 4 - sequence_length = 8 - dimensions = [3, 4, 5] - - np.random.seed(2222) - input_shape = [batch_size, sequence_length] + dimensions - real_valued_input = np.random.rand(*input_shape) - real_valued_column = fc.real_valued_column("values") - - for output_rank in range(1, 3 + len(dimensions)): - with variable_scope.variable_scope("output_rank_{}".format(output_rank)): - real_valued_output = real_valued_column._to_dnn_input_layer( - constant_op.constant(real_valued_input, dtype=dtypes.float32), - output_rank=output_rank) - with self.cached_session() as sess: - real_valued_eval = sess.run(real_valued_output) - expected_shape = ( - input_shape[:output_rank - 1] + - [np.prod(input_shape[output_rank - 1:])]) - self.assertEquals(expected_shape, list(real_valued_eval.shape)) - - def testRealValuedColumnDensification(self): - """Tests densification behavior of `RealValuedColumn`.""" - # No default value, dimension 1 float. - real_valued_column = fc._real_valued_var_len_column( - "sparse_real_valued1", is_sparse=True) - sparse_tensor = sparse_tensor_lib.SparseTensor( - values=[2.0, 5.0], indices=[[0, 0], [2, 0]], dense_shape=[3, 1]) - with self.assertRaisesRegexp(ValueError, "Set is_sparse to False"): - real_valued_column._to_dnn_input_layer(sparse_tensor) - - def testRealValuedColumnDeepCopy(self): - column = fc.real_valued_column( - "aaa", dimension=3, default_value=[1, 2, 3], dtype=dtypes.int32) - column_copy = copy.deepcopy(column) - self.assertEqual(column_copy.name, "aaa") - self.assertEqual(column_copy.dimension, 3) - self.assertEqual(column_copy.default_value, (1, 2, 3)) - - def testBucketizedColumnNameEndsWithUnderscoreBucketized(self): - a = fc.bucketized_column(fc.real_valued_column("aaa"), [0, 4]) - self.assertEqual(a.name, "aaa_bucketized") - - def testBucketizedColumnRequiresRealValuedColumn(self): - with self.assertRaisesRegexp( - TypeError, "source_column must be an instance of _RealValuedColumn"): - fc.bucketized_column("bbb", [0]) - with self.assertRaisesRegexp( - TypeError, "source_column must be an instance of _RealValuedColumn"): - fc.bucketized_column( - fc.sparse_column_with_integerized_feature( - column_name="bbb", bucket_size=10), [0]) - - def testBucketizedColumnRequiresRealValuedColumnDimension(self): - with self.assertRaisesRegexp( - TypeError, "source_column must be an instance of _RealValuedColumn.*"): - fc.bucketized_column( - fc._real_valued_var_len_column("bbb", is_sparse=True), [0]) - - def testBucketizedColumnRequiresSortedBuckets(self): - with self.assertRaisesRegexp(ValueError, - "boundaries must be a sorted list"): - fc.bucketized_column(fc.real_valued_column("ccc"), [5, 0, 4]) - - def testBucketizedColumnWithSameBucketBoundaries(self): - a_bucketized = fc.bucketized_column( - fc.real_valued_column("a"), [1., 2., 2., 3., 3.]) - self.assertEqual(a_bucketized.name, "a_bucketized") - self.assertTupleEqual(a_bucketized.boundaries, (1., 2., 3.)) - - def testBucketizedColumnDeepCopy(self): - """Tests that we can do a deepcopy of a bucketized column. - - This test requires that the bucketized column also accept boundaries - as tuples. - """ - bucketized = fc.bucketized_column( - fc.real_valued_column("a"), [1., 2., 2., 3., 3.]) - self.assertEqual(bucketized.name, "a_bucketized") - self.assertTupleEqual(bucketized.boundaries, (1., 2., 3.)) - bucketized_copy = copy.deepcopy(bucketized) - self.assertEqual(bucketized_copy.name, "a_bucketized") - self.assertTupleEqual(bucketized_copy.boundaries, (1., 2., 3.)) - - def testCrossedColumnNameCreatesSortedNames(self): - a = fc.sparse_column_with_hash_bucket("aaa", hash_bucket_size=100) - b = fc.sparse_column_with_hash_bucket("bbb", hash_bucket_size=100) - bucket = fc.bucketized_column(fc.real_valued_column("cost"), [0, 4]) - crossed = fc.crossed_column(set([b, bucket, a]), hash_bucket_size=10000) - - self.assertEqual("aaa_X_bbb_X_cost_bucketized", crossed.name, - "name should be generated by sorted column names") - self.assertEqual("aaa", crossed.columns[0].name) - self.assertEqual("bbb", crossed.columns[1].name) - self.assertEqual("cost_bucketized", crossed.columns[2].name) - - def testCrossedColumnNotSupportRealValuedColumn(self): - b = fc.sparse_column_with_hash_bucket("bbb", hash_bucket_size=100) - with self.assertRaisesRegexp( - TypeError, "columns must be a set of _SparseColumn, _CrossedColumn, " - "or _BucketizedColumn instances"): - fc.crossed_column( - set([b, fc.real_valued_column("real")]), hash_bucket_size=10000) - - def testCrossedColumnDeepCopy(self): - a = fc.sparse_column_with_hash_bucket("aaa", hash_bucket_size=100) - b = fc.sparse_column_with_hash_bucket("bbb", hash_bucket_size=100) - bucket = fc.bucketized_column(fc.real_valued_column("cost"), [0, 4]) - crossed = fc.crossed_column(set([b, bucket, a]), hash_bucket_size=10000) - crossed_copy = copy.deepcopy(crossed) - self.assertEqual("aaa_X_bbb_X_cost_bucketized", crossed_copy.name, - "name should be generated by sorted column names") - self.assertEqual("aaa", crossed_copy.columns[0].name) - self.assertEqual("bbb", crossed_copy.columns[1].name) - self.assertEqual("cost_bucketized", crossed_copy.columns[2].name) - - def testFloat32WeightedSparseInt32ColumnDtypes(self): - ids = fc.sparse_column_with_keys("ids", [42, 1, -1000], dtype=dtypes.int32) - weighted_ids = fc.weighted_sparse_column(ids, "weights") - self.assertDictEqual({ - "ids": parsing_ops.VarLenFeature(dtypes.int32), - "weights": parsing_ops.VarLenFeature(dtypes.float32) - }, weighted_ids.config) - - def testFloat32WeightedSparseStringColumnDtypes(self): - ids = fc.sparse_column_with_keys("ids", ["marlo", "omar", "stringer"]) - weighted_ids = fc.weighted_sparse_column(ids, "weights") - self.assertDictEqual({ - "ids": parsing_ops.VarLenFeature(dtypes.string), - "weights": parsing_ops.VarLenFeature(dtypes.float32) - }, weighted_ids.config) - - def testInt32WeightedSparseStringColumnDtypes(self): - ids = fc.sparse_column_with_keys("ids", ["marlo", "omar", "stringer"]) - weighted_ids = fc.weighted_sparse_column(ids, "weights", dtype=dtypes.int32) - self.assertDictEqual({ - "ids": parsing_ops.VarLenFeature(dtypes.string), - "weights": parsing_ops.VarLenFeature(dtypes.int32) - }, weighted_ids.config) - - with self.assertRaisesRegexp(ValueError, - "dtype is not convertible to float"): - weighted_ids = fc.weighted_sparse_column( - ids, "weights", dtype=dtypes.string) - - def testInt32WeightedSparseInt64ColumnDtypes(self): - ids = fc.sparse_column_with_keys("ids", [42, 1, -1000], dtype=dtypes.int64) - weighted_ids = fc.weighted_sparse_column(ids, "weights", dtype=dtypes.int32) - self.assertDictEqual({ - "ids": parsing_ops.VarLenFeature(dtypes.int64), - "weights": parsing_ops.VarLenFeature(dtypes.int32) - }, weighted_ids.config) - - with self.assertRaisesRegexp(ValueError, - "dtype is not convertible to float"): - weighted_ids = fc.weighted_sparse_column( - ids, "weights", dtype=dtypes.string) - - def testRealValuedColumnDtypes(self): - rvc = fc.real_valued_column("rvc") - self.assertDictEqual({ - "rvc": parsing_ops.FixedLenFeature([1], dtype=dtypes.float32) - }, rvc.config) - - rvc = fc.real_valued_column("rvc", dtype=dtypes.int32) - self.assertDictEqual({ - "rvc": parsing_ops.FixedLenFeature([1], dtype=dtypes.int32) - }, rvc.config) - - with self.assertRaisesRegexp(ValueError, - "dtype must be convertible to float"): - fc.real_valued_column("rvc", dtype=dtypes.string) - - def testSparseColumnDtypes(self): - sc = fc.sparse_column_with_integerized_feature("sc", 10) - self.assertDictEqual( - { - "sc": parsing_ops.VarLenFeature(dtype=dtypes.int64) - }, sc.config) - - sc = fc.sparse_column_with_integerized_feature("sc", 10, dtype=dtypes.int32) - self.assertDictEqual( - { - "sc": parsing_ops.VarLenFeature(dtype=dtypes.int32) - }, sc.config) - - with self.assertRaisesRegexp(ValueError, "dtype must be an integer"): - fc.sparse_column_with_integerized_feature("sc", 10, dtype=dtypes.float32) - - def testSparseColumnSingleBucket(self): - sc = fc.sparse_column_with_integerized_feature("sc", 1) - self.assertDictEqual( - { - "sc": parsing_ops.VarLenFeature(dtype=dtypes.int64) - }, sc.config) - self.assertEqual(1, sc._wide_embedding_lookup_arguments(None).vocab_size) - - def testSparseColumnAcceptsDenseScalar(self): - """Tests that `SparseColumn`s accept dense scalar inputs.""" - batch_size = 4 - dense_scalar_input = [1, 2, 3, 4] - sparse_column = fc.sparse_column_with_integerized_feature("values", 10) - features = { - "values": constant_op.constant(dense_scalar_input, dtype=dtypes.int64) - } - sparse_column.insert_transformed_feature(features) - sparse_output = features[sparse_column] - expected_shape = [batch_size, 1] - with self.cached_session() as sess: - sparse_result = sess.run(sparse_output) - self.assertEquals(expected_shape, list(sparse_result.dense_shape)) - - def testSparseColumnIntegerizedDeepCopy(self): - """Tests deepcopy of sparse_column_with_integerized_feature.""" - column = fc.sparse_column_with_integerized_feature("a", 10) - self.assertEqual("a", column.name) - column_copy = copy.deepcopy(column) - self.assertEqual("a", column_copy.name) - self.assertEqual(10, column_copy.bucket_size) - self.assertTrue(column_copy.is_integerized) - - def testSparseColumnHashBucketDeepCopy(self): - """Tests deepcopy of sparse_column_with_hash_bucket.""" - column = fc.sparse_column_with_hash_bucket("a", 10) - self.assertEqual("a", column.name) - column_copy = copy.deepcopy(column) - self.assertEqual("a", column_copy.name) - self.assertEqual(10, column_copy.bucket_size) - self.assertFalse(column_copy.is_integerized) - - def testSparseColumnKeysDeepCopy(self): - """Tests deepcopy of sparse_column_with_keys.""" - column = fc.sparse_column_with_keys("a", keys=["key0", "key1", "key2"]) - self.assertEqual("a", column.name) - column_copy = copy.deepcopy(column) - self.assertEqual("a", column_copy.name) - self.assertEqual( - fc._SparseIdLookupConfig( # pylint: disable=protected-access - keys=("key0", "key1", "key2"), - vocab_size=3, - default_value=-1), - column_copy.lookup_config) - self.assertFalse(column_copy.is_integerized) - - def testSparseColumnVocabularyDeepCopy(self): - """Tests deepcopy of sparse_column_with_vocabulary_file.""" - column = fc.sparse_column_with_vocabulary_file( - "a", vocabulary_file="path_to_file", vocab_size=3) - self.assertEqual("a", column.name) - column_copy = copy.deepcopy(column) - self.assertEqual("a", column_copy.name) - self.assertEqual( - fc._SparseIdLookupConfig( # pylint: disable=protected-access - vocabulary_file="path_to_file", - num_oov_buckets=0, - vocab_size=3, - default_value=-1), - column_copy.lookup_config) - self.assertFalse(column_copy.is_integerized) - - def testCreateFeatureSpec(self): - sparse_col = fc.sparse_column_with_hash_bucket( - "sparse_column", hash_bucket_size=100) - embedding_col = fc.embedding_column( - fc.sparse_column_with_hash_bucket( - "sparse_column_for_embedding", hash_bucket_size=10), - dimension=4) - str_sparse_id_col = fc.sparse_column_with_keys( - "str_id_column", ["marlo", "omar", "stringer"]) - int32_sparse_id_col = fc.sparse_column_with_keys( - "int32_id_column", [42, 1, -1000], dtype=dtypes.int32) - int64_sparse_id_col = fc.sparse_column_with_keys( - "int64_id_column", [42, 1, -1000], dtype=dtypes.int64) - weighted_id_col = fc.weighted_sparse_column(str_sparse_id_col, - "str_id_weights_column") - real_valued_col1 = fc.real_valued_column("real_valued_column1") - real_valued_col2 = fc.real_valued_column("real_valued_column2", 5) - bucketized_col1 = fc.bucketized_column( - fc.real_valued_column("real_valued_column_for_bucketization1"), [0, 4]) - bucketized_col2 = fc.bucketized_column( - fc.real_valued_column("real_valued_column_for_bucketization2", 4), - [0, 4]) - a = fc.sparse_column_with_hash_bucket("cross_aaa", hash_bucket_size=100) - b = fc.sparse_column_with_hash_bucket("cross_bbb", hash_bucket_size=100) - cross_col = fc.crossed_column(set([a, b]), hash_bucket_size=10000) - one_hot_col = fc.one_hot_column( - fc.sparse_column_with_hash_bucket( - "sparse_column_for_one_hot", hash_bucket_size=100)) - scattered_embedding_col = fc.scattered_embedding_column( - "scattered_embedding_column", size=100, dimension=10, hash_key=1) - feature_columns = set([ - sparse_col, embedding_col, weighted_id_col, int32_sparse_id_col, - int64_sparse_id_col, real_valued_col1, real_valued_col2, - bucketized_col1, bucketized_col2, cross_col, one_hot_col, - scattered_embedding_col - ]) - expected_config = { - "sparse_column": - parsing_ops.VarLenFeature(dtypes.string), - "sparse_column_for_embedding": - parsing_ops.VarLenFeature(dtypes.string), - "str_id_column": - parsing_ops.VarLenFeature(dtypes.string), - "int32_id_column": - parsing_ops.VarLenFeature(dtypes.int32), - "int64_id_column": - parsing_ops.VarLenFeature(dtypes.int64), - "str_id_weights_column": - parsing_ops.VarLenFeature(dtypes.float32), - "real_valued_column1": - parsing_ops.FixedLenFeature([1], dtype=dtypes.float32), - "real_valued_column2": - parsing_ops.FixedLenFeature([5], dtype=dtypes.float32), - "real_valued_column_for_bucketization1": - parsing_ops.FixedLenFeature([1], dtype=dtypes.float32), - "real_valued_column_for_bucketization2": - parsing_ops.FixedLenFeature([4], dtype=dtypes.float32), - "cross_aaa": - parsing_ops.VarLenFeature(dtypes.string), - "cross_bbb": - parsing_ops.VarLenFeature(dtypes.string), - "sparse_column_for_one_hot": - parsing_ops.VarLenFeature(dtypes.string), - "scattered_embedding_column": - parsing_ops.VarLenFeature(dtypes.string), - } - - config = fc.create_feature_spec_for_parsing(feature_columns) - self.assertDictEqual(expected_config, config) - - # Tests that contrib feature columns work with core library: - config_core = fc_core.make_parse_example_spec(feature_columns) - self.assertDictEqual(expected_config, config_core) - - # Test that the same config is parsed out if we pass a dictionary. - feature_columns_dict = { - str(i): val - for i, val in enumerate(feature_columns) - } - config = fc.create_feature_spec_for_parsing(feature_columns_dict) - self.assertDictEqual(expected_config, config) - - def testCreateFeatureSpec_ExperimentalColumns(self): - real_valued_col0 = fc._real_valued_var_len_column( - "real_valued_column0", is_sparse=True) - real_valued_col1 = fc._real_valued_var_len_column( - "real_valued_column1", - dtype=dtypes.int64, - default_value=0, - is_sparse=False) - feature_columns = set([real_valued_col0, real_valued_col1]) - expected_config = { - "real_valued_column0": - parsing_ops.VarLenFeature(dtype=dtypes.float32), - "real_valued_column1": - parsing_ops.FixedLenSequenceFeature( - [], dtype=dtypes.int64, allow_missing=True, default_value=0), - } - - config = fc.create_feature_spec_for_parsing(feature_columns) - self.assertDictEqual(expected_config, config) - - def testCreateFeatureSpec_RealValuedColumnWithDefaultValue(self): - real_valued_col1 = fc.real_valued_column( - "real_valued_column1", default_value=2) - real_valued_col2 = fc.real_valued_column( - "real_valued_column2", 5, default_value=4) - real_valued_col3 = fc.real_valued_column( - "real_valued_column3", default_value=[8]) - real_valued_col4 = fc.real_valued_column( - "real_valued_column4", 3, default_value=[1, 0, 6]) - real_valued_col5 = fc._real_valued_var_len_column( - "real_valued_column5", default_value=2, is_sparse=True) - real_valued_col6 = fc._real_valued_var_len_column( - "real_valued_column6", - dtype=dtypes.int64, - default_value=1, - is_sparse=False) - feature_columns = [ - real_valued_col1, real_valued_col2, real_valued_col3, real_valued_col4, - real_valued_col5, real_valued_col6 - ] - config = fc.create_feature_spec_for_parsing(feature_columns) - self.assertEqual(6, len(config)) - self.assertDictEqual( - { - "real_valued_column1": - parsing_ops.FixedLenFeature( - [1], dtype=dtypes.float32, default_value=[2.]), - "real_valued_column2": - parsing_ops.FixedLenFeature( - [5], - dtype=dtypes.float32, - default_value=[4., 4., 4., 4., 4.]), - "real_valued_column3": - parsing_ops.FixedLenFeature( - [1], dtype=dtypes.float32, default_value=[8.]), - "real_valued_column4": - parsing_ops.FixedLenFeature( - [3], dtype=dtypes.float32, default_value=[1., 0., 6.]), - "real_valued_column5": - parsing_ops.VarLenFeature(dtype=dtypes.float32), - "real_valued_column6": - parsing_ops.FixedLenSequenceFeature( - [], dtype=dtypes.int64, allow_missing=True, default_value=1) - }, - config) - - def testCreateSequenceFeatureSpec(self): - sparse_col = fc.sparse_column_with_hash_bucket( - "sparse_column", hash_bucket_size=100) - embedding_col = fc.embedding_column( - fc.sparse_column_with_hash_bucket( - "sparse_column_for_embedding", hash_bucket_size=10), - dimension=4) - sparse_id_col = fc.sparse_column_with_keys("id_column", - ["marlo", "omar", "stringer"]) - weighted_id_col = fc.weighted_sparse_column(sparse_id_col, - "id_weights_column") - real_valued_col1 = fc.real_valued_column("real_valued_column", dimension=2) - real_valued_col2 = fc.real_valued_column( - "real_valued_default_column", dimension=5, default_value=3.0) - real_valued_col3 = fc._real_valued_var_len_column( - "real_valued_var_len_column", default_value=3.0, is_sparse=True) - real_valued_col4 = fc._real_valued_var_len_column( - "real_valued_var_len_dense_column", default_value=4.0, is_sparse=False) - - feature_columns = set([ - sparse_col, embedding_col, weighted_id_col, real_valued_col1, - real_valued_col2, real_valued_col3, real_valued_col4 - ]) - - feature_spec = fc._create_sequence_feature_spec_for_parsing(feature_columns) - - expected_feature_spec = { - "sparse_column": - parsing_ops.VarLenFeature(dtypes.string), - "sparse_column_for_embedding": - parsing_ops.VarLenFeature(dtypes.string), - "id_column": - parsing_ops.VarLenFeature(dtypes.string), - "id_weights_column": - parsing_ops.VarLenFeature(dtypes.float32), - "real_valued_column": - parsing_ops.FixedLenSequenceFeature( - shape=[2], dtype=dtypes.float32, allow_missing=False), - "real_valued_default_column": - parsing_ops.FixedLenSequenceFeature( - shape=[5], dtype=dtypes.float32, allow_missing=True), - "real_valued_var_len_column": - parsing_ops.VarLenFeature(dtype=dtypes.float32), - "real_valued_var_len_dense_column": - parsing_ops.FixedLenSequenceFeature( - shape=[], dtype=dtypes.float32, allow_missing=True), - } - - self.assertDictEqual(expected_feature_spec, feature_spec) - - def testMakePlaceHolderTensorsForBaseFeatures(self): - sparse_col = fc.sparse_column_with_hash_bucket( - "sparse_column", hash_bucket_size=100) - real_valued_col = fc.real_valued_column("real_valued_column", 5) - vlen_real_valued_col = fc._real_valued_var_len_column( - "vlen_real_valued_column", is_sparse=True) - - bucketized_col = fc.bucketized_column( - fc.real_valued_column("real_valued_column_for_bucketization"), [0, 4]) - feature_columns = set( - [sparse_col, real_valued_col, vlen_real_valued_col, bucketized_col]) - placeholders = ( - fc.make_place_holder_tensors_for_base_features(feature_columns)) - - self.assertEqual(4, len(placeholders)) - self.assertTrue( - isinstance(placeholders["sparse_column"], - sparse_tensor_lib.SparseTensor)) - self.assertTrue( - isinstance(placeholders["vlen_real_valued_column"], - sparse_tensor_lib.SparseTensor)) - placeholder = placeholders["real_valued_column"] - self.assertGreaterEqual( - placeholder.name.find(u"Placeholder_real_valued_column"), 0) - self.assertEqual(dtypes.float32, placeholder.dtype) - self.assertEqual([None, 5], placeholder.get_shape().as_list()) - placeholder = placeholders["real_valued_column_for_bucketization"] - self.assertGreaterEqual( - placeholder.name.find( - u"Placeholder_real_valued_column_for_bucketization"), 0) - self.assertEqual(dtypes.float32, placeholder.dtype) - self.assertEqual([None, 1], placeholder.get_shape().as_list()) - - def testInitEmbeddingColumnWeightsFromCkpt(self): - sparse_col = fc.sparse_column_with_hash_bucket( - column_name="object_in_image", hash_bucket_size=4) - # Create _EmbeddingColumn which randomly initializes embedding of size - # [4, 16]. - embedding_col = fc.embedding_column(sparse_col, dimension=16) - - # Creating a SparseTensor which has all the ids possible for the given - # vocab. - input_tensor = sparse_tensor_lib.SparseTensor( - indices=[[0, 0], [1, 1], [2, 2], [3, 3]], - values=[0, 1, 2, 3], - dense_shape=[4, 4]) - - # Invoking 'layers.input_from_feature_columns' will create the embedding - # variable. Creating under scope 'run_1' so as to prevent name conflicts - # when creating embedding variable for 'embedding_column_pretrained'. - with variable_scope.variable_scope("run_1"): - with variable_scope.variable_scope(embedding_col.name): - # This will return a [4, 16] tensor which is same as embedding variable. - embeddings = feature_column_ops.input_from_feature_columns({ - embedding_col: input_tensor - }, [embedding_col]) - - save = saver.Saver() - ckpt_dir_prefix = os.path.join(self.get_temp_dir(), - "init_embedding_col_w_from_ckpt") - ckpt_dir = tempfile.mkdtemp(prefix=ckpt_dir_prefix) - checkpoint_path = os.path.join(ckpt_dir, "model.ckpt") - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - saved_embedding = embeddings.eval() - save.save(sess, checkpoint_path) - - embedding_col_initialized = fc.embedding_column( - sparse_id_column=sparse_col, - dimension=16, - ckpt_to_load_from=checkpoint_path, - tensor_name_in_ckpt=("run_1/object_in_image_embedding/" - "input_from_feature_columns/object" - "_in_image_embedding/weights")) - - with variable_scope.variable_scope("run_2"): - # This will initialize the embedding from provided checkpoint and return a - # [4, 16] tensor which is same as embedding variable. Since we didn't - # modify embeddings, this should be same as 'saved_embedding'. - pretrained_embeddings = feature_column_ops.input_from_feature_columns({ - embedding_col_initialized: input_tensor - }, [embedding_col_initialized]) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - loaded_embedding = pretrained_embeddings.eval() - - self.assertAllClose(saved_embedding, loaded_embedding) - - def testInitCrossedColumnWeightsFromCkpt(self): - sparse_col_1 = fc.sparse_column_with_hash_bucket( - column_name="col_1", hash_bucket_size=4) - sparse_col_2 = fc.sparse_column_with_keys( - column_name="col_2", keys=("foo", "bar", "baz")) - sparse_col_3 = fc.sparse_column_with_keys( - column_name="col_3", keys=(42, 1, -1000), dtype=dtypes.int64) - - crossed_col = fc.crossed_column( - columns=[sparse_col_1, sparse_col_2, sparse_col_3], hash_bucket_size=4) - - input_tensor = sparse_tensor_lib.SparseTensor( - indices=[[0, 0], [1, 1], [2, 2], [3, 3]], - values=[0, 1, 2, 3], - dense_shape=[4, 4]) - - # Invoking 'weighted_sum_from_feature_columns' will create the crossed - # column weights variable. - with variable_scope.variable_scope("run_1"): - with variable_scope.variable_scope(crossed_col.name): - # Returns looked up column weights which is same as crossed column - # weights as well as actual references to weights variables. - _, col_weights, _ = ( - feature_column_ops.weighted_sum_from_feature_columns({ - sparse_col_1.name: input_tensor, - sparse_col_2.name: input_tensor, - sparse_col_3.name: input_tensor - }, [crossed_col], 1)) - # Update the weights since default initializer initializes all weights - # to 0.0. - for weight in col_weights.values(): - assign_op = state_ops.assign(weight[0], weight[0] + 0.5) - - save = saver.Saver() - ckpt_dir_prefix = os.path.join(self.get_temp_dir(), - "init_crossed_col_w_from_ckpt") - ckpt_dir = tempfile.mkdtemp(prefix=ckpt_dir_prefix) - checkpoint_path = os.path.join(ckpt_dir, "model.ckpt") - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - sess.run(assign_op) - saved_col_weights = col_weights[crossed_col][0].eval() - save.save(sess, checkpoint_path) - - crossed_col_initialized = fc.crossed_column( - columns=[sparse_col_1, sparse_col_2], - hash_bucket_size=4, - ckpt_to_load_from=checkpoint_path, - tensor_name_in_ckpt=("run_1/col_1_X_col_2_X_col_3/" - "weighted_sum_from_feature_columns/" - "col_1_X_col_2_X_col_3/weights")) - - with variable_scope.variable_scope("run_2"): - # This will initialize the crossed column weights from provided checkpoint - # and return a [4, 1] tensor which is same as weights variable. Since we - # won't modify weights, this should be same as 'saved_col_weights'. - _, col_weights, _ = ( - feature_column_ops.weighted_sum_from_feature_columns({ - sparse_col_1.name: input_tensor, - sparse_col_2.name: input_tensor - }, [crossed_col_initialized], 1)) - col_weights_from_ckpt = col_weights[crossed_col_initialized][0] - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - loaded_col_weights = col_weights_from_ckpt.eval() - - self.assertAllClose(saved_col_weights, loaded_col_weights) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/layers/python/layers/initializers.py b/tensorflow/contrib/layers/python/layers/initializers.py deleted file mode 100644 index 51e5f4d68b9..00000000000 --- a/tensorflow/contrib/layers/python/layers/initializers.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Weight initializers for use with layers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import random_ops - - -__all__ = ['xavier_initializer', 'xavier_initializer_conv2d', - 'variance_scaling_initializer'] - - -def xavier_initializer(uniform=True, seed=None, dtype=dtypes.float32): - """Returns an initializer performing "Xavier" initialization for weights. - - This function implements the weight initialization from: - - Xavier Glorot and Yoshua Bengio (2010): - [Understanding the difficulty of training deep feedforward neural - networks. International conference on artificial intelligence and - statistics.]( - http://www.jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf) - - This initializer is designed to keep the scale of the gradients roughly the - same in all layers. In uniform distribution this ends up being the range: - `x = sqrt(6. / (in + out)); [-x, x]` and for normal distribution a standard - deviation of `sqrt(2. / (in + out))` is used. - - Args: - uniform: Whether to use uniform or normal distributed random initialization. - seed: A Python integer. Used to create random seeds. See - `tf.compat.v1.set_random_seed` for behavior. - dtype: The data type. Only floating point types are supported. - - Returns: - An initializer for a weight matrix. - """ - return variance_scaling_initializer(factor=1.0, mode='FAN_AVG', - uniform=uniform, seed=seed, dtype=dtype) - -xavier_initializer_conv2d = xavier_initializer - - -def variance_scaling_initializer(factor=2.0, mode='FAN_IN', uniform=False, - seed=None, dtype=dtypes.float32): - """Returns an initializer that generates tensors without scaling variance. - - When initializing a deep network, it is in principle advantageous to keep - the scale of the input variance constant, so it does not explode or diminish - by reaching the final layer. This initializer use the following formula: - - ```python - if mode='FAN_IN': # Count only number of input connections. - n = fan_in - elif mode='FAN_OUT': # Count only number of output connections. - n = fan_out - elif mode='FAN_AVG': # Average number of inputs and output connections. - n = (fan_in + fan_out)/2.0 - - truncated_normal(shape, 0.0, stddev=sqrt(factor / n)) - ``` - - * To get [Delving Deep into Rectifiers]( - http://arxiv.org/pdf/1502.01852v1.pdf) (also know as the "MSRA - initialization"), use (Default):
- `factor=2.0 mode='FAN_IN' uniform=False` - * To get [Convolutional Architecture for Fast Feature Embedding]( - http://arxiv.org/abs/1408.5093), use:
- `factor=1.0 mode='FAN_IN' uniform=True` - * To get [Understanding the difficulty of training deep feedforward neural - networks](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf), - use:
- `factor=1.0 mode='FAN_AVG' uniform=True.` - * To get `xavier_initializer` use either:
- `factor=1.0 mode='FAN_AVG' uniform=True`, or
- `factor=1.0 mode='FAN_AVG' uniform=False`. - - Args: - factor: Float. A multiplicative factor. - mode: String. 'FAN_IN', 'FAN_OUT', 'FAN_AVG'. - uniform: Whether to use uniform or normal distributed random initialization. - seed: A Python integer. Used to create random seeds. See - `tf.compat.v1.set_random_seed` for behavior. - dtype: The data type. Only floating point types are supported. - - Returns: - An initializer that generates tensors with unit variance. - - Raises: - ValueError: if `dtype` is not a floating point type. - TypeError: if `mode` is not in ['FAN_IN', 'FAN_OUT', 'FAN_AVG']. - """ - if not dtype.is_floating: - raise TypeError('Cannot create initializer for non-floating point type.') - if mode not in ['FAN_IN', 'FAN_OUT', 'FAN_AVG']: - raise TypeError('Unknown mode %s [FAN_IN, FAN_OUT, FAN_AVG]', mode) - - # pylint: disable=unused-argument - def _initializer(shape, dtype=dtype, partition_info=None): - """Initializer function.""" - if not dtype.is_floating: - raise TypeError('Cannot create initializer for non-floating point type.') - # Estimating fan_in and fan_out is not possible to do perfectly, but we try. - # This is the right thing for matrix multiply and convolutions. - if shape: - fan_in = float(shape[-2]) if len(shape) > 1 else float(shape[-1]) - fan_out = float(shape[-1]) - else: - fan_in = 1.0 - fan_out = 1.0 - for dim in shape[:-2]: - fan_in *= float(dim) - fan_out *= float(dim) - if mode == 'FAN_IN': - # Count only number of input connections. - n = fan_in - elif mode == 'FAN_OUT': - # Count only number of output connections. - n = fan_out - elif mode == 'FAN_AVG': - # Average number of inputs and output connections. - n = (fan_in + fan_out) / 2.0 - if uniform: - # To get stddev = math.sqrt(factor / n) need to adjust for uniform. - limit = math.sqrt(3.0 * factor / n) - return random_ops.random_uniform(shape, -limit, limit, - dtype, seed=seed) - else: - # To get stddev = math.sqrt(factor / n) need to adjust for truncated. - trunc_stddev = math.sqrt(1.3 * factor / n) - return random_ops.truncated_normal(shape, 0.0, trunc_stddev, dtype, - seed=seed) - # pylint: enable=unused-argument - - return _initializer diff --git a/tensorflow/contrib/layers/python/layers/initializers_test.py b/tensorflow/contrib/layers/python/layers/initializers_test.py deleted file mode 100644 index bd3692b2585..00000000000 --- a/tensorflow/contrib/layers/python/layers/initializers_test.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for initializers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib import layers -from tensorflow.contrib.layers.python.layers import initializers -from tensorflow.contrib.layers.python.layers import regularizers -from tensorflow.python.client import session -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class InitializerTest(test.TestCase): - - def test_xavier_wrong_dtype(self): - with self.assertRaisesRegexp( - TypeError, 'Cannot create initializer for non-floating point type.'): - initializers.xavier_initializer(dtype=dtypes.int32) - - self.assertIsNone(regularizers.l1_regularizer(0.)(None)) - - def _test_xavier(self, initializer, shape, variance, uniform): - with session.Session() as sess: - var = variable_scope.get_variable( - name='test', - shape=shape, - dtype=dtypes.float32, - initializer=initializer( - uniform=uniform, seed=1)) - sess.run(variables.global_variables_initializer()) - values = var.eval() - self.assertAllClose(np.var(values), variance, 1e-3, 1e-3) - - def test_xavier_uniform(self): - self._test_xavier(initializers.xavier_initializer, [100, 40], - 2. / (100. + 40.), True) - - def test_xavier_normal(self): - self._test_xavier(initializers.xavier_initializer, [100, 40], - 2. / (100. + 40.), False) - - def test_xavier_scalar(self): - self._test_xavier(initializers.xavier_initializer, [], 0.0, True) - - def test_xavier_conv2d_uniform(self): - self._test_xavier(layers.xavier_initializer_conv2d, [100, 40, 5, 7], - 2. / (100. * 40 * (5 + 7)), True) - - def test_xavier_conv2d_normal(self): - self._test_xavier(layers.xavier_initializer_conv2d, [100, 40, 5, 7], - 2. / (100. * 40 * (5 + 7)), False) - - -class VarianceScalingInitializerTest(test.TestCase): - - def test_wrong_dtype(self): - with self.assertRaisesRegexp( - TypeError, 'Cannot create initializer for non-floating point type.'): - initializers.variance_scaling_initializer(dtype=dtypes.int32) - initializer = initializers.variance_scaling_initializer() - with self.assertRaisesRegexp( - TypeError, 'Cannot create initializer for non-floating point type.'): - initializer([], dtype=dtypes.int32) - - def _test_variance(self, initializer, shape, variance, factor, mode, uniform): - with ops.Graph().as_default() as g: - with self.session(graph=g) as sess: - var = variable_scope.get_variable( - name='test', - shape=shape, - dtype=dtypes.float32, - initializer=initializer( - factor=factor, mode=mode, uniform=uniform, seed=1)) - sess.run(variables.global_variables_initializer()) - values = var.eval() - self.assertAllClose(np.var(values), variance, 1e-3, 1e-3) - - def test_fan_in(self): - for uniform in [False, True]: - self._test_variance( - initializers.variance_scaling_initializer, - shape=[100, 40], - variance=2. / 100., - factor=2.0, - mode='FAN_IN', - uniform=uniform) - - def test_fan_out(self): - for uniform in [False, True]: - self._test_variance( - initializers.variance_scaling_initializer, - shape=[100, 40], - variance=2. / 40., - factor=2.0, - mode='FAN_OUT', - uniform=uniform) - - def test_fan_avg(self): - for uniform in [False, True]: - self._test_variance( - initializers.variance_scaling_initializer, - shape=[100, 40], - variance=4. / (100. + 40.), - factor=2.0, - mode='FAN_AVG', - uniform=uniform) - - def test_conv2d_fan_in(self): - for uniform in [False, True]: - self._test_variance( - initializers.variance_scaling_initializer, - shape=[100, 40, 5, 7], - variance=2. / (100. * 40. * 5.), - factor=2.0, - mode='FAN_IN', - uniform=uniform) - - def test_conv2d_fan_out(self): - for uniform in [False, True]: - self._test_variance( - initializers.variance_scaling_initializer, - shape=[100, 40, 5, 7], - variance=2. / (100. * 40. * 7.), - factor=2.0, - mode='FAN_OUT', - uniform=uniform) - - def test_conv2d_fan_avg(self): - for uniform in [False, True]: - self._test_variance( - initializers.variance_scaling_initializer, - shape=[100, 40, 5, 7], - variance=2. / (100. * 40. * (5. + 7.)), - factor=2.0, - mode='FAN_AVG', - uniform=uniform) - - def test_xavier_uniform(self): - self._test_variance( - initializers.variance_scaling_initializer, - shape=[100, 40], - variance=2. / (100. + 40.), - factor=1.0, - mode='FAN_AVG', - uniform=True) - - def test_xavier_normal(self): - self._test_variance( - initializers.variance_scaling_initializer, - shape=[100, 40], - variance=2. / (100. + 40.), - factor=1.0, - mode='FAN_AVG', - uniform=False) - - def test_xavier_scalar(self): - self._test_variance( - initializers.variance_scaling_initializer, - shape=[], - variance=0.0, - factor=1.0, - mode='FAN_AVG', - uniform=False) - - def test_xavier_conv2d_uniform(self): - self._test_variance( - initializers.variance_scaling_initializer, - shape=[100, 40, 5, 7], - variance=2. / (100. * 40. * (5. + 7.)), - factor=1.0, - mode='FAN_AVG', - uniform=True) - - def test_xavier_conv2d_normal(self): - self._test_variance( - initializers.variance_scaling_initializer, - shape=[100, 40, 5, 7], - variance=2. / (100. * 40. * (5. + 7.)), - factor=1.0, - mode='FAN_AVG', - uniform=True) - - def test_1d_shape_fan_in(self): - for uniform in [False, True]: - self._test_variance( - initializers.variance_scaling_initializer, - shape=[100], - variance=2. / 100., - factor=2.0, - mode='FAN_IN', - uniform=uniform) - - def test_1d_shape_fan_out(self): - for uniform in [False, True]: - self._test_variance( - initializers.variance_scaling_initializer, - shape=[100], - variance=2. / 100., - factor=2.0, - mode='FAN_OUT', - uniform=uniform) - - def test_1d_shape_fan_avg(self): - for uniform in [False, True]: - self._test_variance( - initializers.variance_scaling_initializer, - shape=[100], - variance=4. / (100. + 100.), - factor=2.0, - mode='FAN_AVG', - uniform=uniform) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py deleted file mode 100644 index bb3d73f7a17..00000000000 --- a/tensorflow/contrib/layers/python/layers/layers.py +++ /dev/null @@ -1,3336 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# pylint: disable=g-short-docstring-punctuation -"""Higher level ops for building layers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import six - -from tensorflow.contrib.framework.python.ops import add_arg_scope -from tensorflow.contrib.framework.python.ops import variables -from tensorflow.contrib.layers.python.layers import initializers -from tensorflow.contrib.layers.python.layers import utils -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import function -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.keras.engine import input_spec -from tensorflow.python.layers import base -from tensorflow.python.layers import convolutional as convolutional_layers -from tensorflow.python.layers import core as core_layers -from tensorflow.python.layers import normalization as normalization_layers -from tensorflow.python.layers import pooling as pooling_layers -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import standard_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables as tf_variables -from tensorflow.python.training import moving_averages - -# TODO(b/28426988): Replace legacy_* fns migrated from slim. -# TODO(b/28426988): Remove legacy_* when all uses have migrated to new API. -__all__ = [ - 'avg_pool2d', 'avg_pool3d', 'batch_norm', 'bias_add', 'conv1d', 'conv2d', - 'conv3d', 'conv2d_in_plane', 'conv2d_transpose', 'conv3d_transpose', - 'convolution', 'convolution1d', 'convolution2d', 'convolution2d_in_plane', - 'convolution2d_transpose', 'convolution3d', 'convolution3d_transpose', - 'dense_to_sparse', 'dropout', 'elu', 'flatten', 'fully_connected', 'GDN', - 'gdn', 'images_to_sequence', 'layer_norm', 'linear', 'pool', 'max_pool2d', - 'max_pool3d', 'one_hot_encoding', 'relu', 'relu6', 'repeat', - 'scale_gradient', 'separable_conv2d', 'separable_convolution2d', - 'sequence_to_images', 'softmax', 'spatial_softmax', 'stack', 'unit_norm', - 'legacy_fully_connected', 'legacy_linear', 'legacy_relu', 'maxout' -] - -DATA_FORMAT_NCHW = 'NCHW' -DATA_FORMAT_NHWC = 'NHWC' -DATA_FORMAT_NCDHW = 'NCDHW' -DATA_FORMAT_NDHWC = 'NDHWC' - - -@add_arg_scope -def avg_pool2d(inputs, - kernel_size, - stride=2, - padding='VALID', - data_format=DATA_FORMAT_NHWC, - outputs_collections=None, - scope=None): - """Adds a 2D average pooling op. - - It is assumed that the pooling is done per image but not in batch or channels. - - Args: - inputs: A 4-D tensor of shape `[batch_size, height, width, channels]` if - `data_format` is `NHWC`, and `[batch_size, channels, height, width]` if - `data_format` is `NCHW`. - kernel_size: A list of length 2: [kernel_height, kernel_width] of the - pooling kernel over which the op is computed. Can be an int if both values - are the same. - stride: A list of length 2: [stride_height, stride_width]. Can be an int if - both strides are the same. Note that presently both strides must have the - same value. - padding: The padding method, either 'VALID' or 'SAME'. - data_format: A string. `NHWC` (default) and `NCHW` are supported. - outputs_collections: The collections to which the outputs are added. - scope: Optional scope for name_scope. - - Returns: - A `Tensor` representing the results of the pooling operation. - - Raises: - ValueError: If `data_format` is neither `NHWC` nor `NCHW`. - """ - if data_format not in (DATA_FORMAT_NCHW, DATA_FORMAT_NHWC): - raise ValueError('data_format has to be either NCHW or NHWC.') - with ops.name_scope(scope, 'AvgPool2D', [inputs]) as sc: - inputs = ops.convert_to_tensor(inputs) - df = ('channels_first' - if data_format and data_format.startswith('NC') else 'channels_last') - layer = pooling_layers.AveragePooling2D( - pool_size=kernel_size, - strides=stride, - padding=padding, - data_format=df, - _scope=sc) - outputs = layer.apply(inputs) - return utils.collect_named_outputs(outputs_collections, sc, outputs) - - -@add_arg_scope -def avg_pool3d(inputs, - kernel_size, - stride=2, - padding='VALID', - data_format=DATA_FORMAT_NDHWC, - outputs_collections=None, - scope=None): - """Adds a 3D average pooling op. - - It is assumed that the pooling is done per image but not in batch or channels. - - Args: - inputs: A 5-D tensor of shape `[batch_size, depth, height, width, channels]` - if `data_format` is `NDHWC`, and `[batch_size, channels, depth, height, - width]` if `data_format` is `NCDHW`. - kernel_size: A list of length 3: [kernel_depth, kernel_height, kernel_width] - of the pooling kernel over which the op is computed. Can be an int if both - values are the same. - stride: A list of length 3: [stride_depth, stride_height, stride_width]. Can - be an int if both strides are the same. Note that presently both strides - must have the same value. - padding: The padding method, either 'VALID' or 'SAME'. - data_format: A string. `NDHWC` (default) and `NCDHW` are supported. - outputs_collections: The collections to which the outputs are added. - scope: Optional scope for name_scope. - - Returns: - A `Tensor` representing the results of the pooling operation. - - Raises: - ValueError: If `data_format` is neither `NDHWC` nor `NCDHW`. - """ - if data_format not in (DATA_FORMAT_NCDHW, DATA_FORMAT_NDHWC): - raise ValueError('data_format has to be either NCDHW or NDHWC.') - with ops.name_scope(scope, 'AvgPool3D', [inputs]) as sc: - inputs = ops.convert_to_tensor(inputs) - df = ('channels_first' - if data_format and data_format.startswith('NC') else 'channels_last') - layer = pooling_layers.AveragePooling3D( - pool_size=kernel_size, - strides=stride, - padding=padding, - data_format=df, - _scope=sc) - outputs = layer.apply(inputs) - return utils.collect_named_outputs(outputs_collections, sc, outputs) - - -def _fused_batch_norm(inputs, - decay=0.999, - center=True, - scale=False, - epsilon=0.001, - activation_fn=None, - param_initializers=None, - param_regularizers=None, - updates_collections=ops.GraphKeys.UPDATE_OPS, - is_training=True, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - data_format=DATA_FORMAT_NHWC, - zero_debias_moving_mean=False, - scope=None): - """Adds a Batch Normalization layer from http://arxiv.org/abs/1502.03167. - - "Batch Normalization: Accelerating Deep Network Training by Reducing - Internal Covariate Shift" - - Sergey Ioffe, Christian Szegedy - - Can be used as a normalizer function for conv2d and fully_connected. - - Note: when training, the moving_mean and moving_variance need to be updated. - By default the update ops are placed in `tf.GraphKeys.UPDATE_OPS`, so they - need to be added as a dependency to the `train_op`. For example: - - ```python - update_ops = tf.compat.v1.get_collection(tf.GraphKeys.UPDATE_OPS) - with tf.control_dependencies(update_ops): - train_op = optimizer.minimize(loss) - ``` - - One can set updates_collections=None to force the updates in place, but that - can have a speed penalty, especially in distributed settings. - - Args: - inputs: A tensor with 2 or more dimensions, where the first dimension has - `batch_size`. The normalization is over all but the last dimension if - `data_format` is `NHWC` and the second dimension if `data_format` is - `NCHW`. - decay: Decay for the moving average. Reasonable values for `decay` are close - to 1.0, typically in the multiple-nines range: 0.999, 0.99, 0.9, etc. - Lower `decay` value (recommend trying `decay`=0.9) if model experiences - reasonably good training performance but poor validation and/or test - performance. - center: If True, add offset of `beta` to normalized tensor. If False, - `beta` is ignored. - scale: If True, multiply by `gamma`. If False, `gamma` is not used. When the - next layer is linear (also e.g. `nn.relu`), this can be disabled since the - scaling can be done by the next layer. - epsilon: Small float added to variance to avoid dividing by zero. - activation_fn: Activation function, default set to None to skip it and - maintain a linear activation. - param_initializers: Optional initializers for beta, gamma, moving mean and - moving variance. - param_regularizers: Optional regularizer for beta and gamma. - updates_collections: Collections to collect the update ops for computation. - The updates_ops need to be executed with the train_op. If None, a control - dependency would be added to make sure the updates are computed in place. - is_training: Whether or not the layer is in training mode. In training mode - it would accumulate the statistics of the moments into `moving_mean` and - `moving_variance` using an exponential moving average with the given - `decay`. When it is not in training mode then it would use the values of - the `moving_mean` and the `moving_variance`. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional collections for the variables. - outputs_collections: Collections to add the outputs. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - data_format: A string. `NHWC` (default) and `NCHW` are supported. - zero_debias_moving_mean: Use zero_debias for moving_mean. - scope: Optional scope for `variable_scope`. - - Returns: - A `Tensor` representing the output of the operation. - - Raises: - ValueError: If `data_format` is neither `NHWC` nor `NCHW`. - ValueError: If the rank of `inputs` is undefined. - ValueError: If the rank of `inputs` is neither 2 or 4. - ValueError: If rank or `C` dimension of `inputs` is undefined. - """ - if data_format not in (DATA_FORMAT_NCHW, DATA_FORMAT_NHWC): - raise ValueError('data_format has to be either NCHW or NHWC.') - with variable_scope.variable_scope( - scope, 'BatchNorm', [inputs], reuse=reuse) as sc: - inputs = ops.convert_to_tensor(inputs) - original_shape = inputs.get_shape() - original_inputs = inputs - original_rank = original_shape.ndims - if original_rank is None: - raise ValueError('Inputs %s has undefined rank' % inputs.name) - elif original_rank not in [2, 4]: - raise ValueError('Inputs %s has unsupported rank.' - ' Expected 2 or 4 but got %d' % - (inputs.name, original_rank)) - if original_rank == 2: - channels = inputs.get_shape().dims[-1].value - if channels is None: - raise ValueError('`C` dimension must be known but is None') - new_shape = [-1, 1, 1, channels] - if data_format == DATA_FORMAT_NCHW: - new_shape = [-1, channels, 1, 1] - inputs = array_ops.reshape(inputs, new_shape) - inputs_shape = inputs.get_shape() - if data_format == DATA_FORMAT_NHWC: - params_shape = inputs_shape[-1:] - else: - params_shape = inputs_shape[1:2] - if not params_shape.is_fully_defined(): - raise ValueError('Inputs %s has undefined `C` dimension %s.' % - (inputs.name, params_shape)) - - # Allocate parameters for the beta and gamma of the normalization. - beta_collections = utils.get_variable_collections(variables_collections, - 'beta') - # Float32 required to avoid precision-loss when using fp16 input/output - variable_dtype = dtypes.float32 - if not param_initializers: - param_initializers = {} - if not param_regularizers: - param_regularizers = {} - beta_regularizer = param_regularizers.get('beta') - gamma_regularizer = param_regularizers.get('gamma') - - if center: - beta_initializer = param_initializers.get('beta', - init_ops.zeros_initializer()) - beta = variables.model_variable( - 'beta', - shape=params_shape, - dtype=variable_dtype, - initializer=beta_initializer, - regularizer=beta_regularizer, - collections=beta_collections, - trainable=trainable) - else: - beta = array_ops.constant(0.0, dtype=variable_dtype, shape=params_shape) - - if scale: - gamma_collections = utils.get_variable_collections( - variables_collections, 'gamma') - gamma_initializer = param_initializers.get('gamma', - init_ops.ones_initializer()) - gamma = variables.model_variable( - 'gamma', - shape=params_shape, - dtype=variable_dtype, - initializer=gamma_initializer, - regularizer=gamma_regularizer, - collections=gamma_collections, - trainable=trainable) - else: - gamma = array_ops.constant(1.0, dtype=variable_dtype, shape=params_shape) - - # Create moving_mean and moving_variance variables and add them to the - # appropriate collections. We disable variable partitioning while creating - # them, because assign_moving_average is not yet supported for partitioned - # variables (this needs to be handled carefully, as it may break - # the checkpoint backward compatibility). - with variable_scope.variable_scope( - variable_scope.get_variable_scope()) as local_scope: - local_scope.set_partitioner(None) - moving_mean_collections = utils.get_variable_collections( - variables_collections, 'moving_mean') - moving_mean_initializer = param_initializers.get( - 'moving_mean', init_ops.zeros_initializer()) - moving_mean = variables.model_variable( - 'moving_mean', - shape=params_shape, - dtype=variable_dtype, - initializer=moving_mean_initializer, - trainable=False, - collections=moving_mean_collections) - moving_variance_collections = utils.get_variable_collections( - variables_collections, 'moving_variance') - moving_variance_initializer = param_initializers.get( - 'moving_variance', init_ops.ones_initializer()) - moving_variance = variables.model_variable( - 'moving_variance', - shape=params_shape, - dtype=variable_dtype, - initializer=moving_variance_initializer, - trainable=False, - collections=moving_variance_collections) - - def _fused_batch_norm_training(): - return nn.fused_batch_norm( - inputs, gamma, beta, epsilon=epsilon, data_format=data_format) - - def _fused_batch_norm_inference(): - return nn.fused_batch_norm( - inputs, - gamma, - beta, - mean=moving_mean, - variance=moving_variance, - epsilon=epsilon, - is_training=False, - data_format=data_format) - - outputs, mean, variance = utils.smart_cond(is_training, - _fused_batch_norm_training, - _fused_batch_norm_inference) - - # If `is_training` doesn't have a constant value, because it is a `Tensor`, - # a `Variable` or `Placeholder` then is_training_value will be None and - # `need_updates` will be true. - is_training_value = utils.constant_value(is_training) - need_updates = is_training_value is None or is_training_value - if need_updates: - if updates_collections is None: - no_updates = lambda: outputs - - def _force_updates(): - """Internal function forces updates moving_vars if is_training.""" - update_moving_mean = moving_averages.assign_moving_average( - moving_mean, mean, decay, zero_debias=zero_debias_moving_mean) - update_moving_variance = moving_averages.assign_moving_average( - moving_variance, variance, decay, zero_debias=False) - with ops.control_dependencies( - [update_moving_mean, update_moving_variance]): - return array_ops.identity(outputs) - - outputs = utils.smart_cond(is_training, _force_updates, no_updates) - else: - moving_vars_fn = lambda: (moving_mean, moving_variance) - - def _delay_updates(): - """Internal function that delay updates moving_vars if is_training.""" - update_moving_mean = moving_averages.assign_moving_average( - moving_mean, mean, decay, zero_debias=zero_debias_moving_mean) - update_moving_variance = moving_averages.assign_moving_average( - moving_variance, variance, decay, zero_debias=False) - return update_moving_mean, update_moving_variance - - update_mean, update_variance = utils.smart_cond(is_training, - _delay_updates, - moving_vars_fn) - ops.add_to_collections(updates_collections, update_mean) - ops.add_to_collections(updates_collections, update_variance) - - outputs.set_shape(inputs_shape) - if original_shape.ndims == 2: - outputs = array_ops.reshape(outputs, array_ops.shape(original_inputs)) - if activation_fn is not None: - outputs = activation_fn(outputs) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - -@add_arg_scope -def batch_norm(inputs, - decay=0.999, - center=True, - scale=False, - epsilon=0.001, - activation_fn=None, - param_initializers=None, - param_regularizers=None, - updates_collections=ops.GraphKeys.UPDATE_OPS, - is_training=True, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - batch_weights=None, - fused=None, - data_format=DATA_FORMAT_NHWC, - zero_debias_moving_mean=False, - scope=None, - renorm=False, - renorm_clipping=None, - renorm_decay=0.99, - adjustment=None): - """Adds a Batch Normalization layer from http://arxiv.org/abs/1502.03167. - - "Batch Normalization: Accelerating Deep Network Training by Reducing - Internal Covariate Shift" - - Sergey Ioffe, Christian Szegedy - - Can be used as a normalizer function for conv2d and fully_connected. The - normalization is over all but the last dimension if `data_format` is `NHWC` - and all but the second dimension if `data_format` is `NCHW`. In case of a 2D - tensor this corresponds to the batch dimension, while in case of a 4D tensor - this - corresponds to the batch and space dimensions. - - Note: when training, the moving_mean and moving_variance need to be updated. - By default the update ops are placed in `tf.GraphKeys.UPDATE_OPS`, so they - need to be added as a dependency to the `train_op`. For example: - - ```python - update_ops = tf.compat.v1.get_collection(tf.GraphKeys.UPDATE_OPS) - with tf.control_dependencies(update_ops): - train_op = optimizer.minimize(loss) - ``` - - One can set updates_collections=None to force the updates in place, but that - can have a speed penalty, especially in distributed settings. - - Args: - inputs: A tensor with 2 or more dimensions, where the first dimension has - `batch_size`. The normalization is over all but the last dimension if - `data_format` is `NHWC` and the second dimension if `data_format` is - `NCHW`. - decay: Decay for the moving average. Reasonable values for `decay` are close - to 1.0, typically in the multiple-nines range: 0.999, 0.99, 0.9, etc. - Lower `decay` value (recommend trying `decay`=0.9) if model experiences - reasonably good training performance but poor validation and/or test - performance. Try zero_debias_moving_mean=True for improved stability. - center: If True, add offset of `beta` to normalized tensor. If False, `beta` - is ignored. - scale: If True, multiply by `gamma`. If False, `gamma` is not used. When the - next layer is linear (also e.g. `nn.relu`), this can be disabled since the - scaling can be done by the next layer. - epsilon: Small float added to variance to avoid dividing by zero. - activation_fn: Activation function, default set to None to skip it and - maintain a linear activation. - param_initializers: Optional initializers for beta, gamma, moving mean and - moving variance. - param_regularizers: Optional regularizer for beta and gamma. - updates_collections: Collections to collect the update ops for computation. - The updates_ops need to be executed with the train_op. If None, a control - dependency would be added to make sure the updates are computed in place. - is_training: Whether or not the layer is in training mode. In training mode - it would accumulate the statistics of the moments into `moving_mean` and - `moving_variance` using an exponential moving average with the given - `decay`. When it is not in training mode then it would use the values of - the `moving_mean` and the `moving_variance`. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional collections for the variables. - outputs_collections: Collections to add the outputs. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - batch_weights: An optional tensor of shape `[batch_size]`, containing a - frequency weight for each batch item. If present, then the batch - normalization uses weighted mean and variance. (This can be used to - correct for bias in training example selection.) - fused: if `None` or `True`, use a faster, fused implementation if possible. - If `False`, use the system recommended implementation. - data_format: A string. `NHWC` (default) and `NCHW` are supported. - zero_debias_moving_mean: Use zero_debias for moving_mean. It creates a new - pair of variables 'moving_mean/biased' and 'moving_mean/local_step'. - scope: Optional scope for `variable_scope`. - renorm: Whether to use Batch Renormalization - (https://arxiv.org/abs/1702.03275). This adds extra variables during - training. The inference is the same for either value of this parameter. - renorm_clipping: A dictionary that may map keys 'rmax', 'rmin', 'dmax' to - scalar `Tensors` used to clip the renorm correction. The correction `(r, - d)` is used as `corrected_value = normalized_value * r + d`, with `r` - clipped to [rmin, rmax], and `d` to [-dmax, dmax]. Missing rmax, rmin, - dmax are set to inf, 0, inf, respectively. - renorm_decay: Momentum used to update the moving means and standard - deviations with renorm. Unlike `momentum`, this affects training and - should be neither too small (which would add noise) nor too large (which - would give stale estimates). Note that `decay` is still applied to get the - means and variances for inference. - adjustment: A function taking the `Tensor` containing the (dynamic) shape of - the input tensor and returning a pair (scale, bias) to apply to the - normalized values (before gamma and beta), only during training. For - example, - `adjustment = lambda shape: ( - tf.random.uniform(shape[-1:], 0.93, 1.07), - tf.random.uniform(shape[-1:], -0.1, 0.1))` will scale the normalized - value by up to 7% up or down, then shift the result by up to 0.1 - (with independent scaling and bias for each feature but shared - across all examples), and finally apply gamma and/or beta. If - `None`, no adjustment is applied. - - Returns: - A `Tensor` representing the output of the operation. - - Raises: - ValueError: If `data_format` is neither `NHWC` nor `NCHW`. - ValueError: If the rank of `inputs` is undefined. - ValueError: If rank or channels dimension of `inputs` is undefined. - """ - if fused is None: - fused = True - - # Only use _fused_batch_norm if all of the following three - # conditions are true: - # (1) fused is set True; - # (2) it is possible to use (currently it doesn't support batch weights, - # renorm, and the case when rank is neither 2 nor 4); - # (3) it is used with zero_debias_moving_mean, or an input shape of rank 2, - # or non-default updates_collections (not implemented in - # normalization_layers.BatchNormalization yet); otherwise use the fused - # implementation in normalization_layers.BatchNormalization. - inputs = ops.convert_to_tensor(inputs) - rank = inputs.get_shape().ndims - possible_to_fuse = ( - batch_weights is None and not renorm and rank in [2, 4] and - adjustment is None) - if fused and possible_to_fuse and ( - zero_debias_moving_mean or rank == 2 or - updates_collections is not ops.GraphKeys.UPDATE_OPS): - return _fused_batch_norm( - inputs, - decay=decay, - center=center, - scale=scale, - epsilon=epsilon, - activation_fn=activation_fn, - param_initializers=param_initializers, - param_regularizers=param_regularizers, - updates_collections=updates_collections, - is_training=is_training, - reuse=reuse, - variables_collections=variables_collections, - outputs_collections=outputs_collections, - trainable=trainable, - data_format=data_format, - zero_debias_moving_mean=zero_debias_moving_mean, - scope=scope) - - if data_format not in (DATA_FORMAT_NCHW, DATA_FORMAT_NHWC): - raise ValueError('data_format has to be either NCHW or NHWC.') - - layer_variable_getter = _build_variable_getter() - with variable_scope.variable_scope( - scope, - 'BatchNorm', [inputs], - reuse=reuse, - custom_getter=layer_variable_getter) as sc: - inputs = ops.convert_to_tensor(inputs) - - # Determine whether we can use the core layer class. - if (batch_weights is None and - updates_collections is ops.GraphKeys.UPDATE_OPS and - not zero_debias_moving_mean): - # Use the core layer class. - axis = 1 if data_format == DATA_FORMAT_NCHW else -1 - if not param_initializers: - param_initializers = {} - beta_initializer = param_initializers.get('beta', - init_ops.zeros_initializer()) - gamma_initializer = param_initializers.get('gamma', - init_ops.ones_initializer()) - moving_mean_initializer = param_initializers.get( - 'moving_mean', init_ops.zeros_initializer()) - moving_variance_initializer = param_initializers.get( - 'moving_variance', init_ops.ones_initializer()) - if not param_regularizers: - param_regularizers = {} - beta_regularizer = param_regularizers.get('beta') - gamma_regularizer = param_regularizers.get('gamma') - layer = normalization_layers.BatchNormalization( - axis=axis, - momentum=decay, - epsilon=epsilon, - center=center, - scale=scale, - beta_initializer=beta_initializer, - gamma_initializer=gamma_initializer, - moving_mean_initializer=moving_mean_initializer, - moving_variance_initializer=moving_variance_initializer, - beta_regularizer=beta_regularizer, - gamma_regularizer=gamma_regularizer, - trainable=trainable, - renorm=renorm, - renorm_clipping=renorm_clipping, - renorm_momentum=renorm_decay, - adjustment=adjustment, - name=sc.name, - _scope=sc, - _reuse=reuse, - fused=fused) - outputs = layer.apply(inputs, training=is_training) - - # Add variables to collections. - _add_variable_to_collections(layer.moving_mean, variables_collections, - 'moving_mean') - _add_variable_to_collections(layer.moving_variance, variables_collections, - 'moving_variance') - if layer.beta is not None: - _add_variable_to_collections(layer.beta, variables_collections, 'beta') - if layer.gamma is not None: - _add_variable_to_collections(layer.gamma, variables_collections, - 'gamma') - - if activation_fn is not None: - outputs = activation_fn(outputs) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - # Not supported by layer class: batch_weights argument, - # and custom updates_collections. In that case, use the legacy BN - # implementation. - # Custom updates collections are not supported because the update logic - # is different in this case, in particular w.r.t. "forced updates" and - # update op reuse. - if renorm: - raise ValueError('renorm is not supported with batch_weights, ' - 'updates_collections or zero_debias_moving_mean') - inputs_shape = inputs.get_shape() - inputs_rank = inputs_shape.ndims - if inputs_rank is None: - raise ValueError('Inputs %s has undefined rank.' % inputs.name) - dtype = inputs.dtype.base_dtype - if batch_weights is not None: - batch_weights = ops.convert_to_tensor(batch_weights) - inputs_shape[0:1].assert_is_compatible_with(batch_weights.get_shape()) - # Reshape batch weight values so they broadcast across inputs. - nshape = [-1] + [1 for _ in range(inputs_rank - 1)] - batch_weights = array_ops.reshape(batch_weights, nshape) - - if data_format == DATA_FORMAT_NCHW: - moments_axes = [0] + list(range(2, inputs_rank)) - params_shape = inputs_shape[1:2] - # For NCHW format, rather than relying on implicit broadcasting, we - # explicitly reshape the params to params_shape_broadcast when computing - # the moments and the batch normalization. - params_shape_broadcast = list([1, inputs_shape.dims[1].value] + - [1 for _ in range(2, inputs_rank)]) - else: - moments_axes = list(range(inputs_rank - 1)) - params_shape = inputs_shape[-1:] - params_shape_broadcast = None - if not params_shape.is_fully_defined(): - raise ValueError('Inputs %s has undefined channels dimension %s.' % - (inputs.name, params_shape)) - - # Allocate parameters for the beta and gamma of the normalization. - beta, gamma = None, None - if not param_initializers: - param_initializers = {} - if center: - beta_collections = utils.get_variable_collections(variables_collections, - 'beta') - beta_initializer = param_initializers.get('beta', - init_ops.zeros_initializer()) - beta = variables.model_variable( - 'beta', - shape=params_shape, - dtype=dtype, - initializer=beta_initializer, - collections=beta_collections, - trainable=trainable) - if scale: - gamma_collections = utils.get_variable_collections( - variables_collections, 'gamma') - gamma_initializer = param_initializers.get('gamma', - init_ops.ones_initializer()) - gamma = variables.model_variable( - 'gamma', - shape=params_shape, - dtype=dtype, - initializer=gamma_initializer, - collections=gamma_collections, - trainable=trainable) - - # Create moving_mean and moving_variance variables and add them to the - # appropriate collections. We disable variable partitioning while creating - # them, because assign_moving_average is not yet supported for partitioned - # variables (this needs to be handled carefully, as it may break - # the checkpoint backward compatibility). - with variable_scope.variable_scope( - variable_scope.get_variable_scope()) as local_scope: - local_scope.set_partitioner(None) - moving_mean_collections = utils.get_variable_collections( - variables_collections, 'moving_mean') - moving_mean_initializer = param_initializers.get( - 'moving_mean', init_ops.zeros_initializer()) - moving_mean = variables.model_variable( - 'moving_mean', - shape=params_shape, - dtype=dtype, - initializer=moving_mean_initializer, - trainable=False, - collections=moving_mean_collections) - moving_variance_collections = utils.get_variable_collections( - variables_collections, 'moving_variance') - moving_variance_initializer = param_initializers.get( - 'moving_variance', init_ops.ones_initializer()) - moving_variance = variables.model_variable( - 'moving_variance', - shape=params_shape, - dtype=dtype, - initializer=moving_variance_initializer, - trainable=False, - collections=moving_variance_collections) - - # If `is_training` doesn't have a constant value, because it is a `Tensor`, - # a `Variable` or `Placeholder` then is_training_value will be None and - # `needs_moments` will be true. - is_training_value = utils.constant_value(is_training) - need_moments = is_training_value is None or is_training_value - if need_moments: - # Calculate the moments based on the individual batch. - if batch_weights is None: - if data_format == DATA_FORMAT_NCHW: - mean, variance = nn.moments(inputs, moments_axes, keep_dims=True) - mean = array_ops.reshape(mean, [-1]) - variance = array_ops.reshape(variance, [-1]) - else: - mean, variance = nn.moments(inputs, moments_axes) - else: - if data_format == DATA_FORMAT_NCHW: - mean, variance = nn.weighted_moments( - inputs, moments_axes, batch_weights, keepdims=True) - mean = array_ops.reshape(mean, [-1]) - variance = array_ops.reshape(variance, [-1]) - else: - mean, variance = nn.weighted_moments(inputs, moments_axes, - batch_weights) - - moving_vars_fn = lambda: (moving_mean, moving_variance) - if updates_collections is None: - - def _force_updates(): - """Internal function forces updates moving_vars if is_training.""" - update_moving_mean = moving_averages.assign_moving_average( - moving_mean, mean, decay, zero_debias=zero_debias_moving_mean) - update_moving_variance = moving_averages.assign_moving_average( - moving_variance, variance, decay, zero_debias=False) - with ops.control_dependencies( - [update_moving_mean, update_moving_variance]): - return array_ops.identity(mean), array_ops.identity(variance) - - mean, variance = utils.smart_cond(is_training, _force_updates, - moving_vars_fn) - else: - - def _delay_updates(): - """Internal function that delay updates moving_vars if is_training.""" - update_moving_mean = moving_averages.assign_moving_average( - moving_mean, mean, decay, zero_debias=zero_debias_moving_mean) - update_moving_variance = moving_averages.assign_moving_average( - moving_variance, variance, decay, zero_debias=False) - return update_moving_mean, update_moving_variance - - update_mean, update_variance = utils.smart_cond(is_training, - _delay_updates, - moving_vars_fn) - ops.add_to_collections(updates_collections, update_mean) - ops.add_to_collections(updates_collections, update_variance) - # Use computed moments during training and moving_vars otherwise. - vars_fn = lambda: (mean, variance) - mean, variance = utils.smart_cond(is_training, vars_fn, moving_vars_fn) - else: - mean, variance = moving_mean, moving_variance - if data_format == DATA_FORMAT_NCHW: - mean = array_ops.reshape(mean, params_shape_broadcast) - variance = array_ops.reshape(variance, params_shape_broadcast) - if beta is not None: - beta = array_ops.reshape(beta, params_shape_broadcast) - if gamma is not None: - gamma = array_ops.reshape(gamma, params_shape_broadcast) - - # Compute batch_normalization. - outputs = nn.batch_normalization(inputs, mean, variance, beta, gamma, - epsilon) - outputs.set_shape(inputs_shape) - if activation_fn is not None: - outputs = activation_fn(outputs) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - -@add_arg_scope -def bias_add(inputs, - activation_fn=None, - initializer=init_ops.zeros_initializer(), - regularizer=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - data_format=DATA_FORMAT_NHWC, - scope=None): - """Adds a bias to the inputs. - - Can be used as a normalizer function for conv2d and fully_connected. - - Args: - inputs: A tensor of with at least rank 2 and value for the last dimension, - e.g. `[batch_size, depth]`, `[None, None, None, depth]`. - activation_fn: Activation function, default set to None to skip it and - maintain a linear activation. - initializer: An initializer for the bias, defaults to 0. - regularizer: A regularizer like the result of `l1_regularizer` or - `l2_regularizer`. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional collections for the variables. - outputs_collections: Collections to add the outputs. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). - data_format: A string. 'NHWC' and 'NCHW' are supported. - scope: Optional scope for variable_scope. - - Returns: - A tensor representing the result of adding biases to the inputs. - - Raises: - ValueError: If `data_format` is neither `NHWC` nor `NCHW`. - ValueError: If `data_format` is `NCHW` and rank of `inputs` is not 4. - ValueError: If the rank of `inputs` is undefined. - ValueError: If rank or `C` dimension of `inputs` is undefined. - """ - if data_format not in (DATA_FORMAT_NCHW, DATA_FORMAT_NHWC): - raise ValueError('data_format has to be either NCHW or NHWC.') - with variable_scope.variable_scope( - scope, 'BiasAdd', [inputs], reuse=reuse) as sc: - inputs = ops.convert_to_tensor(inputs) - dtype = inputs.dtype.base_dtype - inputs_shape = inputs.get_shape() - inputs_rank = inputs_shape.ndims - if inputs_rank is None: - raise ValueError('Dims of shape must be known but is None') - elif inputs_rank != 4 and data_format == DATA_FORMAT_NCHW: - raise ValueError('Data format NCHW only supports 4D Tensor') - axis = 1 if data_format == DATA_FORMAT_NCHW else -1 - num_features = inputs_shape.dims[axis].value - if num_features is None: - raise ValueError('`C` dimension must be known but is None') - biases_collections = utils.get_variable_collections(variables_collections, - 'biases') - biases = variables.model_variable( - 'biases', - shape=[ - num_features, - ], - dtype=dtype, - initializer=initializer, - regularizer=regularizer, - collections=biases_collections, - trainable=trainable) - outputs = nn.bias_add(inputs, biases, data_format=data_format) - if activation_fn is not None: - outputs = activation_fn(outputs) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - -# TODO(jbms): change `rate` parameter to `dilation_rate` for consistency with -# underlying op. -@add_arg_scope -def convolution(inputs, - num_outputs, - kernel_size, - stride=1, - padding='SAME', - data_format=None, - rate=1, - activation_fn=nn.relu, - normalizer_fn=None, - normalizer_params=None, - weights_initializer=initializers.xavier_initializer(), - weights_regularizer=None, - biases_initializer=init_ops.zeros_initializer(), - biases_regularizer=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - scope=None, - conv_dims=None): - """Adds an N-D convolution followed by an optional batch_norm layer. - - It is required that 1 <= N <= 3. - - `convolution` creates a variable called `weights`, representing the - convolutional kernel, that is convolved (actually cross-correlated) with the - `inputs` to produce a `Tensor` of activations. If a `normalizer_fn` is - provided (such as `batch_norm`), it is then applied. Otherwise, if - `normalizer_fn` is None and a `biases_initializer` is provided then a `biases` - variable would be created and added the activations. Finally, if - `activation_fn` is not `None`, it is applied to the activations as well. - - Performs atrous convolution with input stride/dilation rate equal to `rate` - if a value > 1 for any dimension of `rate` is specified. In this case - `stride` values != 1 are not supported. - - Args: - inputs: A Tensor of rank N+2 of shape `[batch_size] + input_spatial_shape + - [in_channels]` if data_format does not start with "NC" (default), or - `[batch_size, in_channels] + input_spatial_shape` if data_format starts - with "NC". - num_outputs: Integer, the number of output filters. - kernel_size: A sequence of N positive integers specifying the spatial - dimensions of the filters. Can be a single integer to specify the same - value for all spatial dimensions. - stride: A sequence of N positive integers specifying the stride at which to - compute output. Can be a single integer to specify the same value for all - spatial dimensions. Specifying any `stride` value != 1 is incompatible - with specifying any `rate` value != 1. - padding: One of `"VALID"` or `"SAME"`. - data_format: A string or None. Specifies whether the channel dimension of - the `input` and output is the last dimension (default, or if `data_format` - does not start with "NC"), or the second dimension (if `data_format` - starts with "NC"). For N=1, the valid values are "NWC" (default) and - "NCW". For N=2, the valid values are "NHWC" (default) and "NCHW". For - N=3, the valid values are "NDHWC" (default) and "NCDHW". - rate: A sequence of N positive integers specifying the dilation rate to use - for atrous convolution. Can be a single integer to specify the same value - for all spatial dimensions. Specifying any `rate` value != 1 is - incompatible with specifying any `stride` value != 1. - activation_fn: Activation function. The default value is a ReLU function. - Explicitly set it to None to skip it and maintain a linear activation. - normalizer_fn: Normalization function to use instead of `biases`. If - `normalizer_fn` is provided then `biases_initializer` and - `biases_regularizer` are ignored and `biases` are not created nor added. - default set to None for no normalizer function - normalizer_params: Normalization function parameters. - weights_initializer: An initializer for the weights. - weights_regularizer: Optional regularizer for the weights. - biases_initializer: An initializer for the biases. If None skip biases. - biases_regularizer: Optional regularizer for the biases. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional list of collections for all the variables or - a dictionary containing a different list of collection per variable. - outputs_collections: Collection to add the outputs. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). - scope: Optional scope for `variable_scope`. - conv_dims: Optional convolution dimensionality, when set it would use the - corresponding convolution (e.g. 2 for Conv 2D, 3 for Conv 3D, ..). When - leaved to None it would select the convolution dimensionality based on the - input rank (i.e. Conv ND, with N = input_rank - 2). - - Returns: - A tensor representing the output of the operation. - - Raises: - ValueError: If `data_format` is invalid. - ValueError: Both 'rate' and `stride` are not uniformly 1. - """ - if data_format not in [None, 'NWC', 'NCW', 'NHWC', 'NCHW', 'NDHWC', 'NCDHW']: - raise ValueError('Invalid data_format: %r' % (data_format,)) - - layer_variable_getter = _build_variable_getter({ - 'bias': 'biases', - 'kernel': 'weights' - }) - - with variable_scope.variable_scope( - scope, 'Conv', [inputs], reuse=reuse, - custom_getter=layer_variable_getter) as sc: - inputs = ops.convert_to_tensor(inputs) - input_rank = inputs.get_shape().ndims - - if conv_dims is not None and conv_dims + 2 != input_rank: - raise ValueError('Convolution expects input with rank %d, got %d' % - (conv_dims + 2, input_rank)) - if input_rank == 3: - layer_class = convolutional_layers.Convolution1D - elif input_rank == 4: - layer_class = convolutional_layers.Convolution2D - elif input_rank == 5: - layer_class = convolutional_layers.Convolution3D - else: - raise ValueError('Convolution not supported for input with rank', - input_rank) - - df = ('channels_first' - if data_format and data_format.startswith('NC') else 'channels_last') - layer = layer_class( - filters=num_outputs, - kernel_size=kernel_size, - strides=stride, - padding=padding, - data_format=df, - dilation_rate=rate, - activation=None, - use_bias=not normalizer_fn and biases_initializer, - kernel_initializer=weights_initializer, - bias_initializer=biases_initializer, - kernel_regularizer=weights_regularizer, - bias_regularizer=biases_regularizer, - activity_regularizer=None, - trainable=trainable, - name=sc.name, - dtype=inputs.dtype.base_dtype, - _scope=sc, - _reuse=reuse) - outputs = layer.apply(inputs) - - # Add variables to collections. - _add_variable_to_collections(layer.kernel, variables_collections, 'weights') - if layer.use_bias: - _add_variable_to_collections(layer.bias, variables_collections, 'biases') - - if normalizer_fn is not None: - normalizer_params = normalizer_params or {} - outputs = normalizer_fn(outputs, **normalizer_params) - - if activation_fn is not None: - outputs = activation_fn(outputs) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - -@add_arg_scope -def convolution1d(inputs, - num_outputs, - kernel_size, - stride=1, - padding='SAME', - data_format=None, - rate=1, - activation_fn=nn.relu, - normalizer_fn=None, - normalizer_params=None, - weights_initializer=initializers.xavier_initializer(), - weights_regularizer=None, - biases_initializer=init_ops.zeros_initializer(), - biases_regularizer=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - scope=None): - return convolution( - inputs, - num_outputs, - kernel_size, - stride, - padding, - data_format, - rate, - activation_fn, - normalizer_fn, - normalizer_params, - weights_initializer, - weights_regularizer, - biases_initializer, - biases_regularizer, - reuse, - variables_collections, - outputs_collections, - trainable, - scope, - conv_dims=1) - - -convolution1d.__doc__ = convolution.__doc__ - - -@add_arg_scope -def convolution2d(inputs, - num_outputs, - kernel_size, - stride=1, - padding='SAME', - data_format=None, - rate=1, - activation_fn=nn.relu, - normalizer_fn=None, - normalizer_params=None, - weights_initializer=initializers.xavier_initializer(), - weights_regularizer=None, - biases_initializer=init_ops.zeros_initializer(), - biases_regularizer=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - scope=None): - return convolution( - inputs, - num_outputs, - kernel_size, - stride, - padding, - data_format, - rate, - activation_fn, - normalizer_fn, - normalizer_params, - weights_initializer, - weights_regularizer, - biases_initializer, - biases_regularizer, - reuse, - variables_collections, - outputs_collections, - trainable, - scope, - conv_dims=2) - - -convolution2d.__doc__ = convolution.__doc__ - - -@add_arg_scope -def convolution3d(inputs, - num_outputs, - kernel_size, - stride=1, - padding='SAME', - data_format=None, - rate=1, - activation_fn=nn.relu, - normalizer_fn=None, - normalizer_params=None, - weights_initializer=initializers.xavier_initializer(), - weights_regularizer=None, - biases_initializer=init_ops.zeros_initializer(), - biases_regularizer=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - scope=None): - return convolution( - inputs, - num_outputs, - kernel_size, - stride, - padding, - data_format, - rate, - activation_fn, - normalizer_fn, - normalizer_params, - weights_initializer, - weights_regularizer, - biases_initializer, - biases_regularizer, - reuse, - variables_collections, - outputs_collections, - trainable, - scope, - conv_dims=3) - - -convolution3d.__doc__ = convolution.__doc__ - - -@add_arg_scope -def convolution2d_in_plane( - inputs, - kernel_size, - stride=1, - padding='SAME', - activation_fn=nn.relu, - normalizer_fn=None, - normalizer_params=None, - weights_initializer=initializers.xavier_initializer(), - weights_regularizer=None, - biases_initializer=init_ops.zeros_initializer(), - biases_regularizer=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - scope=None): - """Performs the same in-plane convolution to each channel independently. - - This is useful for performing various simple channel-independent convolution - operations such as image gradients: - - image = tf.constant(..., shape=(16, 240, 320, 3)) - vert_gradients = layers.conv2d_in_plane(image, - kernel=[1, -1], - kernel_size=[2, 1]) - horz_gradients = layers.conv2d_in_plane(image, - kernel=[1, -1], - kernel_size=[1, 2]) - - Args: - inputs: A 4-D tensor with dimensions [batch_size, height, width, channels]. - kernel_size: A list of length 2 holding the [kernel_height, kernel_width] of - of the pooling. Can be an int if both values are the same. - stride: A list of length 2 `[stride_height, stride_width]`. Can be an int if - both strides are the same. Note that presently both strides must have the - same value. - padding: The padding type to use, either 'SAME' or 'VALID'. - activation_fn: Activation function. The default value is a ReLU function. - Explicitly set it to None to skip it and maintain a linear activation. - normalizer_fn: Normalization function to use instead of `biases`. If - `normalizer_fn` is provided then `biases_initializer` and - `biases_regularizer` are ignored and `biases` are not created nor added. - default set to None for no normalizer function - normalizer_params: Normalization function parameters. - weights_initializer: An initializer for the weights. - weights_regularizer: Optional regularizer for the weights. - biases_initializer: An initializer for the biases. If None skip biases. - biases_regularizer: Optional regularizer for the biases. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional list of collections for all the variables or - a dictionary containing a different list of collection per variable. - outputs_collections: Collection to add the outputs. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). - scope: Optional scope for `variable_scope`. - - Returns: - A `Tensor` representing the output of the operation. - """ - with variable_scope.variable_scope( - scope, 'ConvInPlane', [inputs], reuse=reuse) as sc: - dtype = inputs.dtype.base_dtype - kernel_h, kernel_w = utils.two_element_tuple(kernel_size) - stride_h, stride_w = utils.two_element_tuple(stride) - num_filters_in = utils.last_dimension(inputs.get_shape(), min_rank=4) - weights_shape = [kernel_h, kernel_w, 1, 1] - weights_collections = utils.get_variable_collections( - variables_collections, 'weights') - weights = variables.model_variable( - 'weights', - shape=weights_shape, - dtype=dtype, - initializer=weights_initializer, - regularizer=weights_regularizer, - collections=weights_collections, - trainable=trainable) - depthwise_weights = array_ops.tile(weights, [1, 1, num_filters_in, 1]) - outputs = nn.depthwise_conv2d(inputs, depthwise_weights, - [1, stride_h, stride_w, 1], padding) - if normalizer_fn is not None: - normalizer_params = normalizer_params or {} - outputs = normalizer_fn(outputs, **normalizer_params) - else: - if biases_initializer is not None: - biases_collections = utils.get_variable_collections( - variables_collections, 'biases') - biases = variables.model_variable( - 'biases', - shape=[ - num_filters_in, - ], - dtype=dtype, - initializer=biases_initializer, - regularizer=biases_regularizer, - collections=biases_collections, - trainable=trainable) - outputs = nn.bias_add(outputs, biases) - - if activation_fn is not None: - outputs = activation_fn(outputs) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - -@add_arg_scope -def convolution2d_transpose( - inputs, - num_outputs, - kernel_size, - stride=1, - padding='SAME', - data_format=DATA_FORMAT_NHWC, - activation_fn=nn.relu, - normalizer_fn=None, - normalizer_params=None, - weights_initializer=initializers.xavier_initializer(), - weights_regularizer=None, - biases_initializer=init_ops.zeros_initializer(), - biases_regularizer=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - scope=None): - """Adds a convolution2d_transpose with an optional batch normalization layer. - - The function creates a variable called `weights`, representing the - kernel, that is convolved with the input. If `normalizer_fn` is `None`, a - second variable called 'biases' is added to the result of the operation. - - Args: - inputs: A 4-D `Tensor` of type `float` and shape `[batch, height, width, - in_channels]` for `NHWC` data format or `[batch, in_channels, height, - width]` for `NCHW` data format. - num_outputs: Integer, the number of output filters. - kernel_size: A list of length 2 holding the [kernel_height, kernel_width] of - of the filters. Can be an int if both values are the same. - stride: A list of length 2: [stride_height, stride_width]. Can be an int if - both strides are the same. Note that presently both strides must have the - same value. - padding: One of 'VALID' or 'SAME'. - data_format: A string. `NHWC` (default) and `NCHW` are supported. - activation_fn: Activation function. The default value is a ReLU function. - Explicitly set it to None to skip it and maintain a linear activation. - normalizer_fn: Normalization function to use instead of `biases`. If - `normalizer_fn` is provided then `biases_initializer` and - `biases_regularizer` are ignored and `biases` are not created nor added. - default set to None for no normalizer function - normalizer_params: Normalization function parameters. - weights_initializer: An initializer for the weights. - weights_regularizer: Optional regularizer for the weights. - biases_initializer: An initializer for the biases. If None skip biases. - biases_regularizer: Optional regularizer for the biases. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional list of collections for all the variables or - a dictionary containing a different list of collection per variable. - outputs_collections: Collection to add the outputs. - trainable: Whether or not the variables should be trainable or not. - scope: Optional scope for variable_scope. - - Returns: - A tensor representing the output of the operation. - - Raises: - ValueError: If 'kernel_size' is not a list of length 2. - ValueError: If `data_format` is neither `NHWC` nor `NCHW`. - ValueError: If `C` dimension of `inputs` is None. - """ - layer_variable_getter = _build_variable_getter({ - 'bias': 'biases', - 'kernel': 'weights' - }) - - with variable_scope.variable_scope( - scope, - 'Conv2d_transpose', [inputs], - reuse=reuse, - custom_getter=layer_variable_getter) as sc: - if data_format not in (DATA_FORMAT_NCHW, DATA_FORMAT_NHWC): - raise ValueError('data_format has to be either NCHW or NHWC.') - - inputs = ops.convert_to_tensor(inputs) - - df = ('channels_first' - if data_format and data_format.startswith('NC') else 'channels_last') - layer = convolutional_layers.Convolution2DTranspose( - filters=num_outputs, - kernel_size=kernel_size, - strides=stride, - padding=padding, - data_format=df, - activation=None, - use_bias=not normalizer_fn and biases_initializer, - kernel_initializer=weights_initializer, - bias_initializer=biases_initializer, - kernel_regularizer=weights_regularizer, - bias_regularizer=biases_regularizer, - activity_regularizer=None, - trainable=trainable, - name=sc.name, - dtype=inputs.dtype.base_dtype, - _scope=sc, - _reuse=reuse) - outputs = layer.apply(inputs) - - # Add variables to collections. - _add_variable_to_collections(layer.kernel, variables_collections, 'weights') - if layer.bias is not None: - _add_variable_to_collections(layer.bias, variables_collections, 'biases') - - if normalizer_fn is not None: - normalizer_params = normalizer_params or {} - outputs = normalizer_fn(outputs, **normalizer_params) - - if activation_fn is not None: - outputs = activation_fn(outputs) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - -@add_arg_scope -def convolution3d_transpose( - inputs, - num_outputs, - kernel_size, - stride=1, - padding='SAME', - data_format=DATA_FORMAT_NDHWC, - activation_fn=nn.relu, - normalizer_fn=None, - normalizer_params=None, - weights_initializer=initializers.xavier_initializer(), - weights_regularizer=None, - biases_initializer=init_ops.zeros_initializer(), - biases_regularizer=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - scope=None): - """Adds a convolution3d_transpose with an optional batch normalization layer. - - The function creates a variable called `weights`, representing the - kernel, that is convolved with the input. If `batch_norm_params` is `None`, a - second variable called 'biases' is added to the result of the operation. - Args: - inputs: A 5-D `Tensor` of type `float` and shape `[batch, depth, height, - width, in_channels]` for `NDHWC` data format or `[batch, in_channels, - depth, height, width]` for `NCDHW` data format. - num_outputs: Integer, the number of output filters. - kernel_size: A list of length 3 holding the [kernel_depth, kernel_height, - kernel_width] of the filters. Can be an int if both values are the same. - stride: A list of length 3: [stride_depth, stride_height, stride_width]. Can - be an int if both strides are the same. Note that presently both strides - must have the same value. - padding: One of 'VALID' or 'SAME'. - data_format: A string. `NDHWC` (default) and `NCDHW` are supported. - activation_fn: Activation function. The default value is a ReLU function. - Explicitly set it to None to skip it and maintain a linear activation. - normalizer_fn: Normalization function to use instead of `biases`. If - `normalizer_fn` is provided then `biases_initializer` and - `biases_regularizer` are ignored and `biases` are not created nor added. - default set to None for no normalizer function - normalizer_params: Normalization function parameters. - weights_initializer: An initializer for the weights. - weights_regularizer: Optional regularizer for the weights. - biases_initializer: An initializer for the biases. If None skip biases. - biases_regularizer: Optional regularizer for the biases. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional list of collections for all the variables or - a dictionary containing a different list of collection per variable. - outputs_collections: Collection to add the outputs. - trainable: Whether or not the variables should be trainable or not. - scope: Optional scope for variable_scope. - - Returns: - A tensor representing the output of the operation. - Raises: - ValueError: If 'kernel_size' is not a list of length 3. - ValueError: If `data_format` is neither `NDHWC` nor `NCDHW`. - ValueError: If `C` dimension of `inputs` is None. - """ - layer_variable_getter = _build_variable_getter({ - 'bias': 'biases', - 'kernel': 'weights' - }) - - with variable_scope.variable_scope( - scope, - 'Conv3d_transpose', [inputs], - reuse=reuse, - custom_getter=layer_variable_getter) as sc: - if data_format not in (DATA_FORMAT_NCDHW, DATA_FORMAT_NDHWC): - raise ValueError('data_format has to be either NCDHW or NDHWC.') - - inputs = ops.convert_to_tensor(inputs) - - df = ('channels_first' - if data_format and data_format.startswith('NC') else 'channels_last') - layer = convolutional_layers.Convolution3DTranspose( - filters=num_outputs, - kernel_size=kernel_size, - strides=stride, - padding=padding, - data_format=df, - activation=None, - use_bias=not normalizer_fn and biases_initializer, - kernel_initializer=weights_initializer, - bias_initializer=biases_initializer, - kernel_regularizer=weights_regularizer, - bias_regularizer=biases_regularizer, - activity_regularizer=None, - trainable=trainable, - name=sc.name, - dtype=inputs.dtype.base_dtype, - _scope=sc, - _reuse=reuse) - outputs = layer.apply(inputs) - - # Add variables to collections. - _add_variable_to_collections(layer.kernel, variables_collections, 'weights') - if layer.bias is not None: - _add_variable_to_collections(layer.bias, variables_collections, 'biases') - - if normalizer_fn is not None: - normalizer_params = normalizer_params or {} - outputs = normalizer_fn(outputs, **normalizer_params) - - if activation_fn is not None: - outputs = activation_fn(outputs) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - -@add_arg_scope -def dense_to_sparse(tensor, eos_token=0, outputs_collections=None, scope=None): - """Converts a dense tensor into a sparse tensor. - - An example use would be to convert dense labels to sparse ones - so that they can be fed to the ctc_loss. - - Args: - tensor: An `int` `Tensor` to be converted to a `Sparse`. - eos_token: An integer. It is part of the target label that signifies the - end of a sentence. - outputs_collections: Collection to add the outputs. - scope: Optional scope for name_scope. - """ - with variable_scope.variable_scope(scope, 'dense_to_sparse', [tensor]) as sc: - tensor = ops.convert_to_tensor(tensor) - indices = array_ops.where( - math_ops.not_equal(tensor, constant_op.constant(eos_token, - tensor.dtype))) - values = array_ops.gather_nd(tensor, indices) - shape = array_ops.shape(tensor, out_type=dtypes.int64) - outputs = sparse_tensor.SparseTensor(indices, values, shape) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - -@add_arg_scope -def dropout(inputs, - keep_prob=0.5, - noise_shape=None, - is_training=True, - outputs_collections=None, - scope=None, - seed=None): - """Returns a dropout op applied to the input. - - With probability `keep_prob`, outputs the input element scaled up by - `1 / keep_prob`, otherwise outputs `0`. The scaling is so that the expected - sum is unchanged. - - Args: - inputs: The tensor to pass to the nn.dropout op. - keep_prob: A scalar `Tensor` with the same type as x. The probability that - each element is kept. - noise_shape: A 1-D `Tensor` of type `int32`, representing the shape for - randomly generated keep/drop flags. - is_training: A bool `Tensor` indicating whether or not the model is in - training mode. If so, dropout is applied and values scaled. Otherwise, - inputs is returned. - outputs_collections: Collection to add the outputs. - scope: Optional scope for name_scope. - seed: A Python integer. Used to create random seeds. See - `tf.compat.v1.set_random_seed` for behavior. - - Returns: - A tensor representing the output of the operation. - """ - with variable_scope.variable_scope( - scope, 'Dropout', [inputs], custom_getter=_model_variable_getter) as sc: - inputs = ops.convert_to_tensor(inputs) - layer = core_layers.Dropout( - rate=1 - keep_prob, - noise_shape=noise_shape, - seed=seed, - name=sc.name, - _scope=sc) - outputs = layer.apply(inputs, training=is_training) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - -@add_arg_scope -def flatten(inputs, outputs_collections=None, scope=None): - """Flattens the input while maintaining the batch_size. - - Assumes that the first dimension represents the batch. - - Args: - inputs: A tensor of size [batch_size, ...]. - outputs_collections: Collection to add the outputs. - scope: Optional scope for name_scope. - - Returns: - A flattened tensor with shape [batch_size, k]. - Raises: - ValueError: If inputs rank is unknown or less than 2. - """ - with ops.name_scope(scope, 'Flatten', [inputs]) as sc: - inputs = ops.convert_to_tensor(inputs) - outputs = core_layers.flatten(inputs) - return utils.collect_named_outputs(outputs_collections, sc, outputs) - - -def _sparse_inner_flatten(inputs, new_rank): - """Helper function for `inner_flatten`.""" - inputs_rank = inputs.dense_shape.get_shape().as_list()[0] - if inputs_rank < new_rank: - raise ValueError( - 'Inputs has rank less than new_rank. {} must have rank at least' - ' {}. Received rank {}, shape {}'.format(inputs, new_rank, inputs_rank, - inputs.get_shape())) - - outer_dimensions = inputs.dense_shape[:new_rank - 1] - inner_dimensions = inputs.dense_shape[new_rank - 1:] - new_shape = array_ops.concat( - (outer_dimensions, [math_ops.reduce_prod(inner_dimensions)]), 0) - flattened = sparse_ops.sparse_reshape(inputs, new_shape) - return flattened - - -def _dense_inner_flatten(inputs, new_rank): - """Helper function for `inner_flatten`.""" - rank_assertion = check_ops.assert_rank_at_least( - inputs, new_rank, message='inputs has rank less than new_rank') - with ops.control_dependencies([rank_assertion]): - outer_dimensions = array_ops.strided_slice( - array_ops.shape(inputs), [0], [new_rank - 1]) - new_shape = array_ops.concat((outer_dimensions, [-1]), 0) - reshaped = array_ops.reshape(inputs, new_shape) - - # if `new_rank` is an integer, try to calculate new shape. - if isinstance(new_rank, six.integer_types): - static_shape = inputs.get_shape() - if static_shape is not None and static_shape.dims is not None: - static_shape = static_shape.as_list() - static_outer_dims = static_shape[:new_rank - 1] - static_inner_dims = static_shape[new_rank - 1:] - flattened_dimension = 1 - for inner_dim in static_inner_dims: - if inner_dim is None: - flattened_dimension = None - break - flattened_dimension *= inner_dim - reshaped.set_shape(static_outer_dims + [flattened_dimension]) - return reshaped - - -@add_arg_scope -def _inner_flatten(inputs, new_rank, output_collections=None, scope=None): - """Flattens inner dimensions of `inputs`, returns a Tensor with `new_rank`. - - For example: - ''' - x = tf.random.uniform(shape=[1, 2, 3, 4, 5, 6]) - y = _inner_flatten(x, 4) - assert y.get_shape().as_list() == [1, 2, 3, (4 * 5 * 6)] - ''' - This layer will fail at run time if `new_rank` is greater than the current - rank of `inputs`. - - Args: - inputs: A `Tensor` or `SparseTensor`. - new_rank: The desired rank of the returned `Tensor` or `SparseTensor`. - output_collections: Collection to which the outputs will be added. - scope: Optional scope for `name_scope`. - - Returns: - A `Tensor` or `SparseTensor` containing the same values as `inputs`, but - with innermost dimensions flattened to obtain rank `new_rank`. - - Raises: - TypeError: `inputs` is not a `Tensor` or `SparseTensor`. - """ - with ops.name_scope(scope, 'InnerFlatten', [inputs, new_rank]) as sc: - if isinstance(inputs, sparse_tensor.SparseTensor): - flattened = _sparse_inner_flatten(inputs, new_rank) - else: - inputs = ops.convert_to_tensor(inputs) - flattened = _dense_inner_flatten(inputs, new_rank) - return utils.collect_named_outputs(output_collections, sc, flattened) - - -def _model_variable_getter( - getter, - name, - shape=None, - dtype=None, - initializer=None, - regularizer=None, - trainable=True, - collections=None, - caching_device=None, - partitioner=None, - rename=None, - use_resource=None, - synchronization=tf_variables.VariableSynchronization.AUTO, - aggregation=tf_variables.VariableAggregation.NONE, - **_): - """Getter that uses model_variable for compatibility with core layers.""" - short_name = name.split('/')[-1] - if rename and short_name in rename: - name_components = name.split('/') - name_components[-1] = rename[short_name] - name = '/'.join(name_components) - return variables.model_variable( - name, - shape=shape, - dtype=dtype, - initializer=initializer, - regularizer=regularizer, - collections=collections, - trainable=trainable, - caching_device=caching_device, - partitioner=partitioner, - custom_getter=getter, - use_resource=use_resource, - synchronization=synchronization, - aggregation=aggregation) - - -def _build_variable_getter(rename=None): - """Build a model variable getter that respects scope getter and renames.""" - - # VariableScope will nest the getters - def layer_variable_getter(getter, *args, **kwargs): - kwargs['rename'] = rename - return _model_variable_getter(getter, *args, **kwargs) - - return layer_variable_getter - - -def _add_variable_to_collections(variable, collections_set, collections_name): - """Adds variable (or all its parts) to all collections with that name.""" - collections = utils.get_variable_collections(collections_set, - collections_name) or [] - variables_list = [variable] - if isinstance(variable, tf_variables.PartitionedVariable): - variables_list = [v for v in variable] - for collection in collections: - for var in variables_list: - if var not in ops.get_collection(collection): - ops.add_to_collection(collection, var) - - -@add_arg_scope -def fully_connected(inputs, - num_outputs, - activation_fn=nn.relu, - normalizer_fn=None, - normalizer_params=None, - weights_initializer=initializers.xavier_initializer(), - weights_regularizer=None, - biases_initializer=init_ops.zeros_initializer(), - biases_regularizer=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - scope=None): - """Adds a fully connected layer. - - `fully_connected` creates a variable called `weights`, representing a fully - connected weight matrix, which is multiplied by the `inputs` to produce a - `Tensor` of hidden units. If a `normalizer_fn` is provided (such as - `batch_norm`), it is then applied. Otherwise, if `normalizer_fn` is - None and a `biases_initializer` is provided then a `biases` variable would be - created and added the hidden units. Finally, if `activation_fn` is not `None`, - it is applied to the hidden units as well. - - Note: that if `inputs` have a rank greater than 2, then `inputs` is flattened - prior to the initial matrix multiply by `weights`. - - Args: - inputs: A tensor of at least rank 2 and static value for the last dimension; - i.e. `[batch_size, depth]`, `[None, None, None, channels]`. - num_outputs: Integer or long, the number of output units in the layer. - activation_fn: Activation function. The default value is a ReLU function. - Explicitly set it to None to skip it and maintain a linear activation. - normalizer_fn: Normalization function to use instead of `biases`. If - `normalizer_fn` is provided then `biases_initializer` and - `biases_regularizer` are ignored and `biases` are not created nor added. - default set to None for no normalizer function - normalizer_params: Normalization function parameters. - weights_initializer: An initializer for the weights. - weights_regularizer: Optional regularizer for the weights. - biases_initializer: An initializer for the biases. If None skip biases. - biases_regularizer: Optional regularizer for the biases. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional list of collections for all the variables or - a dictionary containing a different list of collections per variable. - outputs_collections: Collection to add the outputs. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). - scope: Optional scope for variable_scope. - - Returns: - The tensor variable representing the result of the series of operations. - - Raises: - ValueError: If x has rank less than 2 or if its last dimension is not set. - """ - if not isinstance(num_outputs, six.integer_types): - raise ValueError('num_outputs type should be one of %s, got %s.' % - (list(six.integer_types), type(num_outputs))) - - layer_variable_getter = _build_variable_getter({ - 'bias': 'biases', - 'kernel': 'weights' - }) - - with variable_scope.variable_scope( - scope, - 'fully_connected', [inputs], - reuse=reuse, - custom_getter=layer_variable_getter) as sc: - inputs = ops.convert_to_tensor(inputs) - layer = core_layers.Dense( - units=num_outputs, - activation=None, - use_bias=not normalizer_fn and biases_initializer, - kernel_initializer=weights_initializer, - bias_initializer=biases_initializer, - kernel_regularizer=weights_regularizer, - bias_regularizer=biases_regularizer, - activity_regularizer=None, - trainable=trainable, - name=sc.name, - dtype=inputs.dtype.base_dtype, - _scope=sc, - _reuse=reuse) - outputs = layer.apply(inputs) - - # Add variables to collections. - _add_variable_to_collections(layer.kernel, variables_collections, 'weights') - if layer.bias is not None: - _add_variable_to_collections(layer.bias, variables_collections, 'biases') - - # Apply normalizer function / layer. - if normalizer_fn is not None: - if not normalizer_params: - normalizer_params = {} - outputs = normalizer_fn(outputs, **normalizer_params) - - if activation_fn is not None: - outputs = activation_fn(outputs) - - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - -class GDN(base.Layer): - """Generalized divisive normalization layer. - - Based on the papers: - - "Density Modeling of Images using a Generalized Normalization - Transformation" - - Johannes Ballé, Valero Laparra, Eero P. Simoncelli - - https://arxiv.org/abs/1511.06281 - - "End-to-end Optimized Image Compression" - - Johannes Ballé, Valero Laparra, Eero P. Simoncelli - - https://arxiv.org/abs/1611.01704 - - Implements an activation function that is essentially a multivariate - generalization of a particular sigmoid-type function: - - ``` - y[i] = x[i] / sqrt(beta[i] + sum_j(gamma[j, i] * x[j])) - ``` - - where `i` and `j` run over channels. This implementation never sums across - spatial dimensions. It is similar to local response normalization, but much - more flexible, as `beta` and `gamma` are trainable parameters. - - Arguments: - inverse: If `False` (default), compute GDN response. If `True`, compute IGDN - response (one step of fixed point iteration to invert GDN; the division is - replaced by multiplication). - beta_min: Lower bound for beta, to prevent numerical error from causing - square root of zero or negative values. - gamma_init: The gamma matrix will be initialized as the identity matrix - multiplied with this value. If set to zero, the layer is effectively - initialized to the identity operation, since beta is initialized as one. A - good default setting is somewhere between 0 and 0.5. - reparam_offset: Offset added to the reparameterization of beta and gamma. - The reparameterization of beta and gamma as their square roots lets the - training slow down when their values are close to zero, which is desirable - as small values in the denominator can lead to a situation where gradient - noise on beta/gamma leads to extreme amounts of noise in the GDN - activations. However, without the offset, we would get zero gradients if - any elements of beta or gamma were exactly zero, and thus the training - could get stuck. To prevent this, we add this small constant. The default - value was empirically determined as a good starting point. Making it - bigger potentially leads to more gradient noise on the activations, making - it too small may lead to numerical precision issues. - data_format: Format of input tensor. Currently supports `'channels_first'` - and `'channels_last'`. - activity_regularizer: Regularizer function for the output. - trainable: Boolean, if `True`, also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - name: String, the name of the layer. Layers with the same name will share - weights, but to avoid mistakes we require `reuse=True` in such cases. - Properties: - inverse: Boolean, whether GDN is computed (`True`) or IGDN (`False`). - data_format: Format of input tensor. Currently supports `'channels_first'` - and `'channels_last'`. - beta: The beta parameter as defined above (1D `Tensor`). - gamma: The gamma parameter as defined above (2D `Tensor`). - """ - - def __init__(self, - inverse=False, - beta_min=1e-6, - gamma_init=.1, - reparam_offset=2**-18, - data_format='channels_last', - activity_regularizer=None, - trainable=True, - name=None, - **kwargs): - super(GDN, self).__init__( - trainable=trainable, - name=name, - activity_regularizer=activity_regularizer, - **kwargs) - self.inverse = inverse - self._beta_min = beta_min - self._gamma_init = gamma_init - self._reparam_offset = reparam_offset - self.data_format = data_format - self._channel_axis() # trigger ValueError early - self.input_spec = input_spec.InputSpec(min_ndim=3, max_ndim=5) - - def _channel_axis(self): - try: - return {'channels_first': 1, 'channels_last': -1}[self.data_format] - except KeyError: - raise ValueError('Unsupported `data_format` for GDN layer: {}.'.format( - self.data_format)) - - @staticmethod - def _lower_bound(inputs, bound, name=None): - """Same as tf.maximum, but with helpful gradient for inputs < bound. - - The gradient is overwritten so that it is passed through if the input is not - hitting the bound. If it is, only gradients that push `inputs` higher than - the bound are passed through. No gradients are passed through to the bound. - - Args: - inputs: input tensor - bound: lower bound for the input tensor - name: name for this op - - Returns: - tf.maximum(inputs, bound) - """ - with ops.name_scope(name, 'GDNLowerBound', [inputs, bound]) as scope: - inputs = ops.convert_to_tensor(inputs, name='inputs') - bound = ops.convert_to_tensor(bound, name='bound') - with ops.get_default_graph().gradient_override_map( - {'Maximum': 'GDNLowerBound'}): - return math_ops.maximum(inputs, bound, name=scope) - - @staticmethod - def _lower_bound_grad(op, grad): - """Gradient for `_lower_bound`. - - Args: - op: the tensorflow op for which to calculate a gradient - grad: gradient with respect to the output of the op - - Returns: - gradients with respect to the inputs of the op - """ - inputs = op.inputs[0] - bound = op.inputs[1] - pass_through_if = math_ops.logical_or(inputs >= bound, grad < 0) - return [math_ops.cast(pass_through_if, grad.dtype) * grad, None] - - def build(self, input_shape): - channel_axis = self._channel_axis() - input_shape = tensor_shape.TensorShape(input_shape) - num_channels = input_shape.dims[channel_axis].value - if num_channels is None: - raise ValueError('The channel dimension of the inputs to `GDN` ' - 'must be defined.') - self._input_rank = input_shape.ndims - self.input_spec = input_spec.InputSpec( - ndim=input_shape.ndims, axes={channel_axis: num_channels}) - - pedestal = array_ops.constant(self._reparam_offset**2, dtype=self.dtype) - beta_bound = array_ops.constant( - (self._beta_min + self._reparam_offset**2)**.5, dtype=self.dtype) - gamma_bound = array_ops.constant(self._reparam_offset, dtype=self.dtype) - - def beta_initializer(shape, dtype=None, partition_info=None): - del partition_info # unused - pedestal = array_ops.constant(self._reparam_offset**2, dtype=self.dtype) - return math_ops.sqrt(array_ops.ones(shape, dtype=dtype) + pedestal) - - def gamma_initializer(shape, dtype=None, partition_info=None): - del partition_info # unused - assert len(shape) == 2 - assert shape[0] == shape[1] - eye = linalg_ops.eye(shape[0], dtype=dtype) - pedestal = array_ops.constant(self._reparam_offset**2, dtype=self.dtype) - return math_ops.sqrt(self._gamma_init * eye + pedestal) - - beta = self.add_variable( - 'reparam_beta', - shape=[num_channels], - initializer=beta_initializer, - dtype=self.dtype, - trainable=True) - beta = self._lower_bound(beta, beta_bound) - self.beta = math_ops.square(beta) - pedestal - - gamma = self.add_variable( - 'reparam_gamma', - shape=[num_channels, num_channels], - initializer=gamma_initializer, - dtype=self.dtype, - trainable=True) - gamma = self._lower_bound(gamma, gamma_bound) - self.gamma = math_ops.square(gamma) - pedestal - - self.built = True - - def call(self, inputs): - inputs = ops.convert_to_tensor(inputs, dtype=self.dtype) - ndim = self._input_rank - - shape = self.gamma.get_shape().as_list() - gamma = array_ops.reshape(self.gamma, (ndim - 2) * [1] + shape) - - # Compute normalization pool. - if self.data_format == 'channels_first': - norm_pool = nn.convolution( - math_ops.square(inputs), - gamma, - 'VALID', - data_format='NC' + 'DHW' [-(ndim - 2):]) - if ndim == 3: - norm_pool = array_ops.expand_dims(norm_pool, 2) - norm_pool = nn.bias_add(norm_pool, self.beta, data_format='NCHW') - norm_pool = array_ops.squeeze(norm_pool, [2]) - elif ndim == 5: - shape = array_ops.shape(norm_pool) - norm_pool = array_ops.reshape(norm_pool, shape[:3] + [-1]) - norm_pool = nn.bias_add(norm_pool, self.beta, data_format='NCHW') - norm_pool = array_ops.reshape(norm_pool, shape) - else: # ndim == 4 - norm_pool = nn.bias_add(norm_pool, self.beta, data_format='NCHW') - else: # channels_last - norm_pool = nn.convolution(math_ops.square(inputs), gamma, 'VALID') - norm_pool = nn.bias_add(norm_pool, self.beta, data_format='NHWC') - norm_pool = math_ops.sqrt(norm_pool) - - if self.inverse: - outputs = inputs * norm_pool - else: - outputs = inputs / norm_pool - outputs.set_shape(inputs.get_shape()) - return outputs - - def compute_output_shape(self, input_shape): - channel_axis = self._channel_axis() - input_shape = tensor_shape.TensorShape(input_shape) - if not 3 <= input_shape.ndim <= 5: - raise ValueError('`input_shape` must be of rank 3 to 5, inclusive.') - if input_shape.dims[channel_axis].value is None: - raise ValueError( - 'The channel dimension of `input_shape` must be defined.') - return input_shape - - -ops.RegisterGradient('GDNLowerBound')(GDN._lower_bound_grad) # pylint:disable=protected-access - - -def gdn(inputs, - inverse=False, - beta_min=1e-6, - gamma_init=.1, - reparam_offset=2**-18, - data_format='channels_last', - activity_regularizer=None, - trainable=True, - name=None, - reuse=None): - """Functional interface for GDN layer. - - Based on the papers: - - "Density Modeling of Images using a Generalized Normalization - Transformation" - Johannes Ballé, Valero Laparra, Eero P. Simoncelli - https://arxiv.org/abs/1511.06281 - - "End-to-end Optimized Image Compression" - Johannes Ballé, Valero Laparra, Eero P. Simoncelli - https://arxiv.org/abs/1611.01704 - - Implements an activation function that is essentially a multivariate - generalization of a particular sigmoid-type function: - - ``` - y[i] = x[i] / sqrt(beta[i] + sum_j(gamma[j, i] * x[j])) - ``` - - where `i` and `j` run over channels. This implementation never sums across - spatial dimensions. It is similar to local response normalization, but much - more flexible, as `beta` and `gamma` are trainable parameters. - - Args: - inputs: Tensor input. - inverse: If `False` (default), compute GDN response. If `True`, compute IGDN - response (one step of fixed point iteration to invert GDN; the division is - replaced by multiplication). - beta_min: Lower bound for beta, to prevent numerical error from causing - square root of zero or negative values. - gamma_init: The gamma matrix will be initialized as the identity matrix - multiplied with this value. If set to zero, the layer is effectively - initialized to the identity operation, since beta is initialized as one. A - good default setting is somewhere between 0 and 0.5. - reparam_offset: Offset added to the reparameterization of beta and gamma. - The reparameterization of beta and gamma as their square roots lets the - training slow down when their values are close to zero, which is desirable - as small values in the denominator can lead to a situation where gradient - noise on beta/gamma leads to extreme amounts of noise in the GDN - activations. However, without the offset, we would get zero gradients if - any elements of beta or gamma were exactly zero, and thus the training - could get stuck. To prevent this, we add this small constant. The default - value was empirically determined as a good starting point. Making it - bigger potentially leads to more gradient noise on the activations, making - it too small may lead to numerical precision issues. - data_format: Format of input tensor. Currently supports `'channels_first'` - and `'channels_last'`. - activity_regularizer: Regularizer function for the output. - trainable: Boolean, if `True`, also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - name: String, the name of the layer. Layers with the same name will share - weights, but to avoid mistakes we require `reuse=True` in such cases. - reuse: Boolean, whether to reuse the weights of a previous layer by the same - name. - - Returns: - Output tensor. - """ - layer = GDN( - inverse=inverse, - beta_min=beta_min, - gamma_init=gamma_init, - reparam_offset=reparam_offset, - data_format=data_format, - activity_regularizer=activity_regularizer, - trainable=trainable, - name=name, - dtype=inputs.dtype.base_dtype, - _scope=name, - _reuse=reuse) - return layer.apply(inputs) - - -@add_arg_scope -def layer_norm(inputs, - center=True, - scale=True, - activation_fn=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - begin_norm_axis=1, - begin_params_axis=-1, - scope=None): - """Adds a Layer Normalization layer. - - Based on the paper: - - "Layer Normalization" - - Jimmy Lei Ba, Jamie Ryan Kiros, Geoffrey E. Hinton - - https://arxiv.org/abs/1607.06450. - - Can be used as a normalizer function for conv2d and fully_connected. - - Given a tensor `inputs` of rank `R`, moments are calculated and normalization - is performed over axes `begin_norm_axis ... R - 1`. Scaling and centering, - if requested, is performed over axes `begin_params_axis .. R - 1`. - - By default, `begin_norm_axis = 1` and `begin_params_axis = -1`, - meaning that normalization is performed over all but the first axis - (the `HWC` if `inputs` is `NHWC`), while the `beta` and `gamma` trainable - parameters are calculated for the rightmost axis (the `C` if `inputs` is - `NHWC`). Scaling and recentering is performed via broadcast of the - `beta` and `gamma` parameters with the normalized tensor. - - The shapes of `beta` and `gamma` are `inputs.shape[begin_params_axis:]`, - and this part of the inputs' shape must be fully defined. - - Args: - inputs: A tensor having rank `R`. The normalization is performed over axes - `begin_norm_axis ... R - 1` and centering and scaling parameters are - calculated over `begin_params_axis ... R - 1`. - center: If True, add offset of `beta` to normalized tensor. If False, `beta` - is ignored. - scale: If True, multiply by `gamma`. If False, `gamma` is not used. When the - next layer is linear (also e.g. `nn.relu`), this can be disabled since the - scaling can be done by the next layer. - activation_fn: Activation function, default set to None to skip it and - maintain a linear activation. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional collections for the variables. - outputs_collections: Collections to add the outputs. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). - begin_norm_axis: The first normalization dimension: normalization will be - performed along dimensions `begin_norm_axis : rank(inputs)` - begin_params_axis: The first parameter (beta, gamma) dimension: scale and - centering parameters will have dimensions - `begin_params_axis : rank(inputs)` and will be broadcast with the - normalized inputs accordingly. - scope: Optional scope for `variable_scope`. - - Returns: - A `Tensor` representing the output of the operation, having the same - shape and dtype as `inputs`. - - Raises: - ValueError: If the rank of `inputs` is not known at graph build time, - or if `inputs.shape[begin_params_axis:]` is not fully defined at - graph build time. - """ - with variable_scope.variable_scope( - scope, 'LayerNorm', [inputs], reuse=reuse) as sc: - inputs = ops.convert_to_tensor(inputs) - inputs_shape = inputs.shape - inputs_rank = inputs_shape.ndims - if inputs_rank is None: - raise ValueError('Inputs %s has undefined rank.' % inputs.name) - dtype = inputs.dtype.base_dtype - if begin_norm_axis < 0: - begin_norm_axis = inputs_rank + begin_norm_axis - if begin_params_axis >= inputs_rank or begin_norm_axis >= inputs_rank: - raise ValueError('begin_params_axis (%d) and begin_norm_axis (%d) ' - 'must be < rank(inputs) (%d)' % - (begin_params_axis, begin_norm_axis, inputs_rank)) - params_shape = inputs_shape[begin_params_axis:] - if not params_shape.is_fully_defined(): - raise ValueError( - 'Inputs %s: shape(inputs)[%s:] is not fully defined: %s' % - (inputs.name, begin_params_axis, inputs_shape)) - # Allocate parameters for the beta and gamma of the normalization. - beta, gamma = None, None - if center: - beta_collections = utils.get_variable_collections(variables_collections, - 'beta') - beta = variables.model_variable( - 'beta', - shape=params_shape, - dtype=dtype, - initializer=init_ops.zeros_initializer(), - collections=beta_collections, - trainable=trainable) - if scale: - gamma_collections = utils.get_variable_collections( - variables_collections, 'gamma') - gamma = variables.model_variable( - 'gamma', - shape=params_shape, - dtype=dtype, - initializer=init_ops.ones_initializer(), - collections=gamma_collections, - trainable=trainable) - # By default, compute the moments across all the dimensions except the one with index 0. - norm_axes = list(range(begin_norm_axis, inputs_rank)) - mean, variance = nn.moments(inputs, norm_axes, keep_dims=True) - # Compute layer normalization using the batch_normalization function. - # Note that epsilon must be increased for float16 due to the limited - # representable range. - variance_epsilon = 1e-12 if dtype != dtypes.float16 else 1e-3 - outputs = nn.batch_normalization( - inputs, - mean, - variance, - offset=beta, - scale=gamma, - variance_epsilon=variance_epsilon) - outputs.set_shape(inputs_shape) - if activation_fn is not None: - outputs = activation_fn(outputs) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - -@add_arg_scope -def images_to_sequence(inputs, - data_format=DATA_FORMAT_NHWC, - outputs_collections=None, - scope=None): - """Convert a batch of images into a batch of sequences. - - Args: - inputs: a (num_images, height, width, depth) tensor - data_format: A string. `NHWC` (default) and `NCHW` are supported. - outputs_collections: The collections to which the outputs are added. - scope: Optional scope for name_scope. - - Raises: - ValueError: If `data_format` is not either NCHW or NHWC. - - Returns: - (width, num_images*height, depth) sequence tensor - """ - if data_format not in (DATA_FORMAT_NCHW, DATA_FORMAT_NHWC): - raise ValueError('data_format has to be either NCHW or NHWC.') - with ops.name_scope(scope, 'ImagesToSequence', [inputs]) as sc: - inputs = ops.convert_to_tensor(inputs) - df = ('channels_first' - if data_format and data_format.startswith('NC') else 'channels_last') - if df == 'channels_first': - inputs = array_ops.transpose(inputs, [0, 2, 3, 1]) - _, _, width, depth = inputs.get_shape().as_list() - s = array_ops.shape(inputs) - batch_size, height = s[0], s[1] - transposed = array_ops.transpose(inputs, [2, 0, 1, 3]) - outputs = array_ops.reshape(transposed, [width, batch_size * height, depth]) - return utils.collect_named_outputs(outputs_collections, sc, outputs) - - -@add_arg_scope -def max_pool2d(inputs, - kernel_size, - stride=2, - padding='VALID', - data_format=DATA_FORMAT_NHWC, - outputs_collections=None, - scope=None): - """Adds a 2D Max Pooling op. - - It is assumed that the pooling is done per image but not in batch or channels. - - Args: - inputs: A 4-D tensor of shape `[batch_size, height, width, channels]` if - `data_format` is `NHWC`, and `[batch_size, channels, height, width]` if - `data_format` is `NCHW`. - kernel_size: A list of length 2: [kernel_height, kernel_width] of the - pooling kernel over which the op is computed. Can be an int if both values - are the same. - stride: A list of length 2: [stride_height, stride_width]. Can be an int if - both strides are the same. Note that presently both strides must have the - same value. - padding: The padding method, either 'VALID' or 'SAME'. - data_format: A string. `NHWC` (default) and `NCHW` are supported. - outputs_collections: The collections to which the outputs are added. - scope: Optional scope for name_scope. - - Returns: - A `Tensor` representing the results of the pooling operation. - - Raises: - ValueError: If `data_format` is neither `NHWC` nor `NCHW`. - ValueError: If 'kernel_size' is not a 2-D list - """ - if data_format not in (DATA_FORMAT_NCHW, DATA_FORMAT_NHWC): - raise ValueError('data_format has to be either NCHW or NHWC.') - with ops.name_scope(scope, 'MaxPool2D', [inputs]) as sc: - inputs = ops.convert_to_tensor(inputs) - df = ('channels_first' - if data_format and data_format.startswith('NC') else 'channels_last') - layer = pooling_layers.MaxPooling2D( - pool_size=kernel_size, - strides=stride, - padding=padding, - data_format=df, - _scope=sc) - outputs = layer.apply(inputs) - return utils.collect_named_outputs(outputs_collections, sc, outputs) - - -@add_arg_scope -def max_pool3d(inputs, - kernel_size, - stride=2, - padding='VALID', - data_format=DATA_FORMAT_NDHWC, - outputs_collections=None, - scope=None): - """Adds a 3D Max Pooling op. - - It is assumed that the pooling is done per image but not in batch or channels. - - Args: - inputs: A 5-D tensor of shape `[batch_size, depth, height, width, channels]` - if `data_format` is `NDHWC`, and `[batch_size, channels, depth, height, - width]` if `data_format` is `NCDHW`. - kernel_size: A list of length 3: [kernel_depth, kernel_height, kernel_width] - of the pooling kernel over which the op is computed. Can be an int if both - values are the same. - stride: A list of length 3: [stride_depth, stride_height, stride_width]. Can - be an int if both strides are the same. Note that presently both strides - must have the same value. - padding: The padding method, either 'VALID' or 'SAME'. - data_format: A string. `NDHWC` (default) and `NCDHW` are supported. - outputs_collections: The collections to which the outputs are added. - scope: Optional scope for name_scope. - - Returns: - A `Tensor` representing the results of the pooling operation. - - Raises: - ValueError: If `data_format` is neither `NDHWC` nor `NCDHW`. - ValueError: If 'kernel_size' is not a 3-D list - """ - if data_format not in (DATA_FORMAT_NCDHW, DATA_FORMAT_NDHWC): - raise ValueError('data_format has to be either NCDHW or NDHWC.') - with ops.name_scope(scope, 'MaxPool3D', [inputs]) as sc: - inputs = ops.convert_to_tensor(inputs) - df = ('channels_first' - if data_format and data_format.startswith('NC') else 'channels_last') - layer = pooling_layers.MaxPooling3D( - pool_size=kernel_size, - strides=stride, - padding=padding, - data_format=df, - _scope=sc) - outputs = layer.apply(inputs) - return utils.collect_named_outputs(outputs_collections, sc, outputs) - - -@add_arg_scope -def pool(inputs, - kernel_size, - pooling_type, - padding='VALID', - data_format=None, - dilation_rate=1, - stride=1, - outputs_collections=None, - scope=None): - # pylint: disable=line-too-long - """Adds a pooling op. - - - Args: - inputs: Tensor of rank N+2, of shape `[batch_size] + input_spatial_shape + - [num_channels]` if data_format does not start with "NC" (default), or - `[batch_size, num_channels] + input_spatial_shape` if data_format starts - with "NC". Pooling happens over the spatial dimensions only. - kernel_size: Sequence of N ints >= 1. Can also be a single integer to - specify the same value for all spatial dimensions. - pooling_type: Specifies pooling operation, must be "AVG" or "MAX". - padding: The padding algorithm, must be "SAME" or "VALID". - data_format: A string or None. Specifies whether the channel dimension of - the `input` and output is the last dimension (default, or if `data_format` - does not start with "NC"), or the second dimension (if `data_format` - starts with "NC"). For N=1, the valid values are "NWC" (default) and - "NCW". For N=2, the valid values are "NHWC" (default) and "NCHW". For - N=3, the valid values are "NDHWC" (default) and "NCDHW". - dilation_rate: Optional. Dilation rate. Sequence of N ints >= 1. Defaults - to [1]*N. Can also be a single integer to specify the same value for all - spatial dimensions. If any value of dilation_rate is > 1, then all values - of stride must be 1. - stride: Optional. Sequence of N ints >= 1. Defaults to [1]*N. Can also be - a single integer to specify the same value for all spatial dimensions. If - any value of stride is > 1, then all values of dilation_rate must be 1. - outputs_collections: The collections to which the outputs are added. - scope: Optional scope for name_scope. - - Returns: - A `Tensor` representing the results of the pooling operation. - - Raises: - ValueError: If arguments are invalid. - - """ - # pylint: enable=line-too-long - with ops.name_scope(scope, '%s_pool' % (pooling_type.lower()), - [inputs]) as sc: - inputs = ops.convert_to_tensor(inputs) - input_rank = inputs.get_shape().ndims - if input_rank is None: - raise ValueError('Rank of inputs must be known') - if input_rank < 3: - raise ValueError('Rank of inputs must be >= 3') - num_spatial_dims = input_rank - 2 - output = nn.pool( - input=inputs, - window_shape=utils.n_positive_integers(num_spatial_dims, kernel_size), - pooling_type=pooling_type, - padding=padding, - data_format=data_format, - dilation_rate=utils.n_positive_integers(num_spatial_dims, - dilation_rate), - strides=utils.n_positive_integers(num_spatial_dims, stride), - name=sc) - return utils.collect_named_outputs(outputs_collections, sc, output) - - -@add_arg_scope -def one_hot_encoding(labels, - num_classes, - on_value=1.0, - off_value=0.0, - outputs_collections=None, - scope=None): - """Transform numeric labels into onehot_labels using `tf.one_hot`. - - Args: - labels: [batch_size] target labels. - num_classes: Total number of classes. - on_value: A scalar defining the on-value. - off_value: A scalar defining the off-value. - outputs_collections: Collection to add the outputs. - scope: Optional scope for name_scope. - - Returns: - One-hot encoding of the labels. - """ - with ops.name_scope(scope, 'OneHotEncoding', [labels, num_classes]) as sc: - labels = ops.convert_to_tensor(labels) - if labels.dtype == dtypes.int32: - labels = standard_ops.to_int64(labels) - outputs = standard_ops.one_hot( - labels, num_classes, on_value=on_value, off_value=off_value) - return utils.collect_named_outputs(outputs_collections, sc, outputs) - - -def _apply_activation(y, activation_fn, output_collections): - if activation_fn is not None: - y = activation_fn(y) - ops.add_to_collections( - list(output_collections or []) + [ops.GraphKeys.ACTIVATIONS], y) - return y - - -def repeat(inputs, repetitions, layer, *args, **kwargs): - """Applies the same layer with the same arguments repeatedly. - - ```python - y = repeat(x, 3, conv2d, 64, [3, 3], scope='conv1') - # It is equivalent to: - - x = conv2d(x, 64, [3, 3], scope='conv1/conv1_1') - x = conv2d(x, 64, [3, 3], scope='conv1/conv1_2') - y = conv2d(x, 64, [3, 3], scope='conv1/conv1_3') - ``` - - If the `scope` argument is not given in `kwargs`, it is set to - `layer.__name__`, or `layer.func.__name__` (for `functools.partial` - objects). If neither `__name__` nor `func.__name__` is available, the - layers are called with `scope='stack'`. - - Args: - inputs: A `Tensor` suitable for layer. - repetitions: Int, number of repetitions. - layer: A layer with arguments `(inputs, *args, **kwargs)` - *args: Extra args for the layer. - **kwargs: Extra kwargs for the layer. - - Returns: - A tensor result of applying the layer, repetitions times. - Raises: - ValueError: If the op is unknown or wrong. - """ - scope = kwargs.pop('scope', None) - with variable_scope.variable_scope(scope, 'Repeat', [inputs]): - inputs = ops.convert_to_tensor(inputs) - if scope is None: - if hasattr(layer, '__name__'): - scope = layer.__name__ - elif hasattr(layer, 'func') and hasattr(layer.func, '__name__'): - scope = layer.func.__name__ # In case layer is a functools.partial. - else: - scope = 'repeat' - outputs = inputs - for i in range(repetitions): - kwargs['scope'] = scope + '_' + str(i + 1) - outputs = layer(outputs, *args, **kwargs) - return outputs - - -def _scale_gradient_shape(op): - """Shape helper function for scale_gradient function below.""" - return [op.inputs[0].shape] - - -def _scale_gradient_grad(op, grad): - """Python gradient helper function for scale_gradient function below.""" - return [grad * op.inputs[1], None] - - -@function.Defun( - python_grad_func=_scale_gradient_grad, shape_func=_scale_gradient_shape) -def scale_gradient(inputs, gradient_multiplier): - """Identity operation, but with the gradient multiplied by a tensor. - - The TensorFlow gradient system will compute the gradient with respect to - `inputs` as the product of the gradient with respect to the `output` - multiplied by a specified `gradient_multiplier` tensor. If - `gradient_multiplier` is equal to 1, then this results in the true gradient. - Otherwise, it results in a scaled gradient. - - This can be useful for adjusting the relative learning rate of different - parameter tensors when performing gradient descent, and because this rescaling - can be inserted at arbitrary locations within a graph, is often more - convenient to apply than simply rescaling the final computed gradients. - - Args: - inputs: Tensor to be output. - gradient_multiplier: Tensor by which to multiply the gradient with respect - to `output` to compute the gradient with respect to `inputs`. Its shape - must be broadcastable to the shape of `inputs`. - - Returns: - output Tensor, equal to `inputs`. - """ - # gradient_multiplier is implicitly saved by decorator, and only used for - # gradient computation. - del gradient_multiplier - - return inputs - - -@add_arg_scope -def separable_convolution2d( - inputs, - num_outputs, - kernel_size, - depth_multiplier=1, - stride=1, - padding='SAME', - data_format=DATA_FORMAT_NHWC, - rate=1, - activation_fn=nn.relu, - normalizer_fn=None, - normalizer_params=None, - weights_initializer=initializers.xavier_initializer(), - pointwise_initializer=None, - weights_regularizer=None, - biases_initializer=init_ops.zeros_initializer(), - biases_regularizer=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - scope=None): - """Adds a depth-separable 2D convolution with optional batch_norm layer. - - This op first performs a depthwise convolution that acts separately on - channels, creating a variable called `depthwise_weights`. If `num_outputs` - is not None, it adds a pointwise convolution that mixes channels, creating a - variable called `pointwise_weights`. Then, if `normalizer_fn` is None, - it adds bias to the result, creating a variable called 'biases', otherwise, - the `normalizer_fn` is applied. It finally applies an activation function - to produce the end result. - - Args: - inputs: A tensor of size [batch_size, height, width, channels]. - num_outputs: The number of pointwise convolution output filters. If is None, - then we skip the pointwise convolution stage. - kernel_size: A list of length 2: [kernel_height, kernel_width] of of the - filters. Can be an int if both values are the same. - depth_multiplier: The number of depthwise convolution output channels for - each input channel. The total number of depthwise convolution output - channels will be equal to `num_filters_in * depth_multiplier`. - stride: A list of length 2: [stride_height, stride_width], specifying the - depthwise convolution stride. Can be an int if both strides are the same. - padding: One of 'VALID' or 'SAME'. - data_format: A string. `NHWC` (default) and `NCHW` are supported. - rate: A list of length 2: [rate_height, rate_width], specifying the dilation - rates for atrous convolution. Can be an int if both rates are the same. If - any value is larger than one, then both stride values need to be one. - activation_fn: Activation function. The default value is a ReLU function. - Explicitly set it to None to skip it and maintain a linear activation. - normalizer_fn: Normalization function to use instead of `biases`. If - `normalizer_fn` is provided then `biases_initializer` and - `biases_regularizer` are ignored and `biases` are not created nor added. - default set to None for no normalizer function - normalizer_params: Normalization function parameters. - weights_initializer: An initializer for the depthwise weights. - pointwise_initializer: An initializer for the pointwise weights. default set - to None, means use weights_initializer. - weights_regularizer: Optional regularizer for the weights. - biases_initializer: An initializer for the biases. If None skip biases. - biases_regularizer: Optional regularizer for the biases. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional list of collections for all the variables or - a dictionary containing a different list of collection per variable. - outputs_collections: Collection to add the outputs. - trainable: Whether or not the variables should be trainable or not. - scope: Optional scope for variable_scope. - - Returns: - A `Tensor` representing the output of the operation. - Raises: - ValueError: If `data_format` is invalid. - """ - if data_format not in (DATA_FORMAT_NCHW, DATA_FORMAT_NHWC): - raise ValueError('data_format has to be either NCHW or NHWC.') - layer_variable_getter = _build_variable_getter({ - 'bias': 'biases', - 'depthwise_kernel': 'depthwise_weights', - 'pointwise_kernel': 'pointwise_weights' - }) - - with variable_scope.variable_scope( - scope, - 'SeparableConv2d', [inputs], - reuse=reuse, - custom_getter=layer_variable_getter) as sc: - inputs = ops.convert_to_tensor(inputs) - - if pointwise_initializer is None: - pointwise_initializer = weights_initializer - - df = ('channels_first' - if data_format and data_format.startswith('NC') else 'channels_last') - if num_outputs is not None: - # Apply separable conv using the SeparableConvolution2D layer. - layer = convolutional_layers.SeparableConvolution2D( - filters=num_outputs, - kernel_size=kernel_size, - strides=stride, - padding=padding, - data_format=df, - dilation_rate=utils.two_element_tuple(rate), - activation=None, - depth_multiplier=depth_multiplier, - use_bias=not normalizer_fn and biases_initializer, - depthwise_initializer=weights_initializer, - pointwise_initializer=pointwise_initializer, - bias_initializer=biases_initializer, - depthwise_regularizer=weights_regularizer, - pointwise_regularizer=weights_regularizer, - bias_regularizer=biases_regularizer, - activity_regularizer=None, - trainable=trainable, - name=sc.name, - dtype=inputs.dtype.base_dtype, - _scope=sc, - _reuse=reuse) - outputs = layer.apply(inputs) - - # Add variables to collections. - _add_variable_to_collections(layer.depthwise_kernel, - variables_collections, 'weights') - _add_variable_to_collections(layer.pointwise_kernel, - variables_collections, 'weights') - if layer.bias is not None: - _add_variable_to_collections(layer.bias, variables_collections, - 'biases') - - if normalizer_fn is not None: - normalizer_params = normalizer_params or {} - outputs = normalizer_fn(outputs, **normalizer_params) - else: - # Actually apply depthwise conv instead of separable conv. - dtype = inputs.dtype.base_dtype - kernel_h, kernel_w = utils.two_element_tuple(kernel_size) - stride_h, stride_w = utils.two_element_tuple(stride) - num_filters_in = utils.channel_dimension( - inputs.get_shape(), df, min_rank=4) - weights_collections = utils.get_variable_collections( - variables_collections, 'weights') - - depthwise_shape = [kernel_h, kernel_w, num_filters_in, depth_multiplier] - depthwise_weights = variables.model_variable( - 'depthwise_weights', - shape=depthwise_shape, - dtype=dtype, - initializer=weights_initializer, - regularizer=weights_regularizer, - trainable=trainable, - collections=weights_collections) - strides = [ - 1, 1, stride_h, stride_w - ] if data_format.startswith('NC') else [1, stride_h, stride_w, 1] - - outputs = nn.depthwise_conv2d( - inputs, - depthwise_weights, - strides, - padding, - rate=utils.two_element_tuple(rate), - data_format=data_format) - num_outputs = depth_multiplier * num_filters_in - - if normalizer_fn is not None: - normalizer_params = normalizer_params or {} - outputs = normalizer_fn(outputs, **normalizer_params) - else: - if biases_initializer is not None: - biases_collections = utils.get_variable_collections( - variables_collections, 'biases') - biases = variables.model_variable( - 'biases', - shape=[ - num_outputs, - ], - dtype=dtype, - initializer=biases_initializer, - regularizer=biases_regularizer, - trainable=trainable, - collections=biases_collections) - outputs = nn.bias_add(outputs, biases, data_format=data_format) - - if activation_fn is not None: - outputs = activation_fn(outputs) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - -@add_arg_scope -def sequence_to_images(inputs, - height, - output_data_format='channels_last', - outputs_collections=None, - scope=None): - """Convert a batch of sequences into a batch of images. - - Args: - inputs: (num_steps, num_batches, depth) sequence tensor - height: the height of the images - output_data_format: Format of output tensor. Currently supports - `'channels_first'` and `'channels_last'`. - outputs_collections: The collections to which the outputs are added. - scope: Optional scope for name_scope. - - Returns: - A tensor representing the output of the operation. - """ - with ops.name_scope(scope, 'SequenceToImages', [inputs]) as sc: - inputs = ops.convert_to_tensor(inputs) - width, num_batches, depth = inputs.get_shape().as_list() - if num_batches is None: - num_batches = -1 - else: - num_batches //= height - reshaped = array_ops.reshape(inputs, [width, num_batches, height, depth]) - if output_data_format == 'channels_first': - outputs = array_ops.transpose(reshaped, [1, 3, 2, 0]) - else: - outputs = array_ops.transpose(reshaped, [1, 2, 0, 3]) - return utils.collect_named_outputs(outputs_collections, sc, outputs) - - -@add_arg_scope -def softmax(logits, scope=None): - """Performs softmax on Nth dimension of N-dimensional logit tensor. - - For two-dimensional logits this reduces to tf.nn.softmax. The N-th dimension - needs to have a specified number of elements (number of classes). - - Args: - logits: N-dimensional `Tensor` with logits, where N > 1. - scope: Optional scope for variable_scope. - - Returns: - A `Tensor` with same shape and type as logits. - """ - # TODO(jrru): Add axis argument which defaults to last dimension. - with variable_scope.variable_scope(scope, 'softmax', [logits]): - num_logits = utils.last_dimension(logits.get_shape(), min_rank=2) - logits_2d = array_ops.reshape(logits, [-1, num_logits]) - predictions = nn.softmax(logits_2d) - predictions = array_ops.reshape(predictions, array_ops.shape(logits)) - if not context.executing_eagerly(): - predictions.set_shape(logits.get_shape()) - return predictions - - -@add_arg_scope -def spatial_softmax(features, - temperature=None, - name=None, - variables_collections=None, - trainable=True, - data_format='NHWC'): - """Computes the spatial softmax of a convolutional feature map. - - First computes the softmax over the spatial extent of each channel of a - convolutional feature map. Then computes the expected 2D position of the - points of maximal activation for each channel, resulting in a set of - feature keypoints [i1, j1, ... iN, jN] for all N channels. - - Read more here: - "Learning visual feature spaces for robotic manipulation with - deep spatial autoencoders." Finn et al., http://arxiv.org/abs/1509.06113. - - Args: - features: A `Tensor` of size [batch_size, W, H, num_channels]; the - convolutional feature map. - temperature: Softmax temperature (optional). If None, a learnable - temperature is created. - name: A name for this operation (optional). - variables_collections: Collections for the temperature variable. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - data_format: A string. `NHWC` (default) and `NCHW` are supported. - - Returns: - feature_keypoints: A `Tensor` with size [batch_size, num_channels * 2]; - the expected 2D locations of each channel's feature keypoint (normalized - to the range (-1,1)). The inner dimension is arranged as - [i1, j1, ... iN, jN]. - Raises: - ValueError: If unexpected data_format specified. - ValueError: If num_channels dimension is unspecified. - """ - with variable_scope.variable_scope(name, 'spatial_softmax'): - shape = array_ops.shape(features) - static_shape = features.shape - if data_format == DATA_FORMAT_NHWC: - height, width, num_channels = shape[1], shape[2], static_shape[3] - elif data_format == DATA_FORMAT_NCHW: - num_channels, height, width = static_shape[1], shape[2], shape[3] - else: - raise ValueError('data_format has to be either NCHW or NHWC.') - if tensor_shape.dimension_value(num_channels) is None: - raise ValueError('The num_channels dimension of the inputs to ' - '`spatial_softmax` should be defined. Found `None`.') - - with ops.name_scope('spatial_softmax_op', 'spatial_softmax_op', [features]): - # Create tensors for x and y coordinate values, scaled to range [-1, 1]. - pos_x, pos_y = array_ops.meshgrid( - math_ops.lin_space(-1., 1., num=height), - math_ops.lin_space(-1., 1., num=width), - indexing='ij') - pos_x = array_ops.reshape(pos_x, [height * width]) - pos_y = array_ops.reshape(pos_y, [height * width]) - - if temperature is None: - temp_initializer = init_ops.ones_initializer() - else: - temp_initializer = init_ops.constant_initializer(temperature) - - if not trainable: - temp_collections = None - else: - temp_collections = utils.get_variable_collections( - variables_collections, 'temperature') - - temperature = variables.model_variable( - 'temperature', - shape=(), - dtype=dtypes.float32, - initializer=temp_initializer, - collections=temp_collections, - trainable=trainable) - if data_format == 'NCHW': - features = array_ops.reshape(features, [-1, height * width]) - else: - features = array_ops.reshape( - array_ops.transpose(features, [0, 3, 1, 2]), [-1, height * width]) - - softmax_attention = nn.softmax(features / temperature) - expected_x = math_ops.reduce_sum( - pos_x * softmax_attention, [1], keepdims=True) - expected_y = math_ops.reduce_sum( - pos_y * softmax_attention, [1], keepdims=True) - expected_xy = array_ops.concat([expected_x, expected_y], 1) - feature_keypoints = array_ops.reshape( - expected_xy, [-1, tensor_shape.dimension_value(num_channels) * 2]) - feature_keypoints.set_shape( - [None, tensor_shape.dimension_value(num_channels) * 2]) - return feature_keypoints - - -def stack(inputs, layer, stack_args, **kwargs): - """Builds a stack of layers by applying layer repeatedly using stack_args. - - `stack` allows you to repeatedly apply the same operation with different - arguments `stack_args[i]`. For each application of the layer, `stack` creates - a new scope appended with an increasing number. For example: - - ```python - y = stack(x, fully_connected, [32, 64, 128], scope='fc') - # It is equivalent to: - - x = fully_connected(x, 32, scope='fc/fc_1') - x = fully_connected(x, 64, scope='fc/fc_2') - y = fully_connected(x, 128, scope='fc/fc_3') - ``` - - If the `scope` argument is not given in `kwargs`, it is set to - `layer.__name__`, or `layer.func.__name__` (for `functools.partial` - objects). If neither `__name__` nor `func.__name__` is available, the - layers are called with `scope='stack'`. - - Args: - inputs: A `Tensor` suitable for layer. - layer: A layer with arguments `(inputs, *args, **kwargs)` - stack_args: A list/tuple of parameters for each call of layer. - **kwargs: Extra kwargs for the layer. - - Returns: - A `Tensor` result of applying the stacked layers. - - Raises: - ValueError: If the op is unknown or wrong. - """ - scope = kwargs.pop('scope', None) - if not isinstance(stack_args, (list, tuple)): - raise ValueError('stack_args need to be a list or tuple') - with variable_scope.variable_scope(scope, 'Stack', [inputs]): - inputs = ops.convert_to_tensor(inputs) - if scope is None: - if hasattr(layer, '__name__'): - scope = layer.__name__ - elif hasattr(layer, 'func') and hasattr(layer.func, '__name__'): - scope = layer.func.__name__ # In case layer is a functools.partial. - else: - scope = 'stack' - outputs = inputs - for i in range(len(stack_args)): - kwargs['scope'] = scope + '_' + str(i + 1) - layer_args = stack_args[i] - if not isinstance(layer_args, (list, tuple)): - layer_args = [layer_args] - outputs = layer(outputs, *layer_args, **kwargs) - return outputs - - -@add_arg_scope -def unit_norm(inputs, dim, epsilon=1e-7, scope=None): - """Normalizes the given input across the specified dimension to unit length. - - Note that the rank of `input` must be known. - - Args: - inputs: A `Tensor` of arbitrary size. - dim: The dimension along which the input is normalized. - epsilon: A small value to add to the inputs to avoid dividing by zero. - scope: Optional scope for variable_scope. - - Returns: - The normalized `Tensor`. - - Raises: - ValueError: If dim is smaller than the number of dimensions in 'inputs'. - """ - with variable_scope.variable_scope(scope, 'UnitNorm', [inputs]): - if not inputs.get_shape(): - raise ValueError('The input rank must be known.') - input_rank = len(inputs.get_shape().as_list()) - if dim < 0 or dim >= input_rank: - raise ValueError('dim must be positive but smaller than the input rank.') - - lengths = math_ops.sqrt( - epsilon + math_ops.reduce_sum(math_ops.square(inputs), dim, True)) - multiples = [] - if dim > 0: - multiples.append(array_ops.ones([dim], dtypes.int32)) - multiples.append( - array_ops.strided_slice(array_ops.shape(inputs), [dim], [dim + 1])) - if dim < (input_rank - 1): - multiples.append(array_ops.ones([input_rank - 1 - dim], dtypes.int32)) - multiples = array_ops.concat(multiples, 0) - return math_ops.div(inputs, array_ops.tile(lengths, multiples)) - - -@add_arg_scope -def maxout(inputs, num_units, axis=-1, scope=None): - """Adds a maxout op from https://arxiv.org/abs/1302.4389 - - "Maxout Networks" Ian J. Goodfellow, David Warde-Farley, Mehdi Mirza, Aaron - Courville, - Yoshua Bengio - - Usually the operation is performed in the filter/channel dimension. This can - also be - used after fully-connected layers to reduce number of features. - - Arguments: - inputs: Tensor input - num_units: Specifies how many features will remain after maxout in the - `axis` dimension (usually channel). This must be a factor of number of - features. - axis: The dimension where max pooling will be performed. Default is the last - dimension. - scope: Optional scope for variable_scope. - - Returns: - A `Tensor` representing the results of the pooling operation. - - Raises: - ValueError: if num_units is not multiple of number of features. - """ - with variable_scope.variable_scope(scope, 'MaxOut', [inputs]): - inputs = ops.convert_to_tensor(inputs) - shape = inputs.get_shape().as_list() - num_channels = shape[axis] - if num_channels % num_units: - raise ValueError('number of features({}) is not ' - 'a multiple of num_units({})'.format( - num_channels, num_units)) - shape[axis] = num_units - shape += [num_channels // num_units] - - # Dealing with batches with arbitrary sizes - for i in range(len(shape)): - if shape[i] is None: - shape[i] = array_ops.shape(inputs)[i] - outputs = math_ops.reduce_max( - array_ops.reshape(inputs, shape), -1, keepdims=False) - return outputs - - -def poincare_normalize(x, axis=1, epsilon=1e-5, name=None): - """Project into the Poincare ball with norm <= 1.0 - epsilon. - - https://en.wikipedia.org/wiki/Poincare_ball_model - - Used in - Poincare Embeddings for Learning Hierarchical Representations - Maximilian Nickel, Douwe Kiela - https://arxiv.org/pdf/1705.08039.pdf - - For a 1-D tensor with `axis = 0`, computes - - (x * (1 - epsilon)) / ||x|| if ||x|| > 1 - epsilon - output = - x otherwise - - For `x` with more dimensions, independently normalizes each 1-D slice along - dimension `axis`. - - Args: - x: A `Tensor`. - axis: Axis along which to normalize. A scalar or a vector of integers. - epsilon: A small deviation from the edge of the unit sphere for numerical - stability. - name: A name for this operation (optional). - - Returns: - A `Tensor` with the same shape as `x`. - """ - with ops.name_scope(name, 'poincare_normalize', [x]) as name: - x = ops.convert_to_tensor(x, name='x') - square_sum = math_ops.reduce_sum(math_ops.square(x), axis, keepdims=True) - x_inv_norm = math_ops.rsqrt(square_sum) - x_inv_norm = math_ops.minimum((1. - epsilon) * x_inv_norm, 1.) - return math_ops.multiply(x, x_inv_norm, name=name) - - -def legacy_fully_connected(x, - num_output_units, - activation_fn=None, - weight_init=initializers.xavier_initializer(), - bias_init=init_ops.zeros_initializer(), - name=None, - weight_collections=(ops.GraphKeys.WEIGHTS,), - bias_collections=(ops.GraphKeys.BIASES,), - output_collections=(ops.GraphKeys.ACTIVATIONS,), - trainable=True, - weight_regularizer=None, - bias_regularizer=None): - # pylint: disable=anomalous-backslash-in-string - r"""Adds the parameters for a fully connected layer and returns the output. - - A fully connected layer is generally defined as a matrix multiply: - `y = f(w * x + b)` where `f` is given by `activation_fn`. If - `activation_fn` is `None`, the result of `y = w * x + b` is - returned. - - If `x` has shape [\\(\text{dim}_0, \text{dim}_1, ..., \text{dim}_n\\)] - with more than 2 dimensions (\\(n > 1\\)), then we repeat the matrix - multiply along the first dimensions. The result r is a tensor of shape - [\\(\text{dim}_0, ..., \text{dim}_{n-1},\\) `num_output_units`], - where \\( r_{i_0, ..., i_{n-1}, k} = - \sum_{0 \leq j < \text{dim}_n} x_{i_0, ... i_{n-1}, j} \cdot w_{j, k}\\). - This is accomplished by reshaping `x` to 2-D - [\\(\text{dim}_0 \cdot ... \cdot \text{dim}_{n-1}, \text{dim}_n\\)] - before the matrix multiply and afterwards reshaping it to - [\\(\text{dim}_0, ..., \text{dim}_{n-1},\\) `num_output_units`]. - - This op creates `w` and optionally `b`. Bias (`b`) can be disabled by setting - `bias_init` to `None`. - - The variable creation is compatible with `tf.compat.v1.variable_scope` and so - can be - reused with `tf.compat.v1.variable_scope` or `tf.compat.v1.make_template`. - - Most of the details of variable creation can be controlled by specifying the - initializers (`weight_init` and `bias_init`) and in which collections to place - the created variables (`weight_collections` and `bias_collections`; note that - the variables are always added to the `VARIABLES` collection). The output of - the layer can be placed in custom collections using `output_collections`. - The collections arguments default to `WEIGHTS`, `BIASES` and `ACTIVATIONS`, - respectively. - - A per layer regularization can be specified by setting `weight_regularizer` - and `bias_regularizer`, which are applied to the weights and biases - respectively, and whose output is added to the `REGULARIZATION_LOSSES` - collection. - - Args: - x: The input `Tensor`. - num_output_units: The size of the output. - activation_fn: Activation function, default set to None to skip it and - maintain a linear activation. - weight_init: An optional weight initialization, defaults to - `xavier_initializer`. - bias_init: An initializer for the bias, defaults to 0. Set to `None` in - order to disable bias. - name: The name for this operation is used to name operations and to find - variables. If specified it must be unique for this scope, otherwise a - unique name starting with "fully_connected" will be created. See - `tf.compat.v1.variable_scope` for details. - weight_collections: List of graph collections to which weights are added. - bias_collections: List of graph collections to which biases are added. - output_collections: List of graph collections to which outputs are added. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). - weight_regularizer: A regularizer like the result of `l1_regularizer` or - `l2_regularizer`. Used for weights. - bias_regularizer: A regularizer like the result of `l1_regularizer` or - `l2_regularizer`. Used for biases. - - Returns: - The output of the fully connected layer. - - Raises: - ValueError: If x has rank less than 2 or if its last dimension is not set. - """ - with variable_scope.variable_scope(name, 'fully_connected', [x]): - x = ops.convert_to_tensor(x) - dims = x.get_shape().dims - if dims is None: - raise ValueError('dims of x must be known but is None') - if len(dims) < 2: - raise ValueError('rank of x must be at least 2 not: %d' % len(dims)) - num_input_units = dims[-1].value - if num_input_units is None: - raise ValueError('last dimension of x must be known but is None') - dtype = x.dtype.base_dtype - - weight_collections = set( - list(weight_collections or []) + [ops.GraphKeys.GLOBAL_VARIABLES]) - w = variable_scope.get_variable( - 'weights', - shape=[num_input_units, num_output_units], - dtype=dtype, - initializer=weight_init, - collections=weight_collections, - regularizer=weight_regularizer, - trainable=trainable) - x_2_dim = x if len(dims) <= 2 else array_ops.reshape( - x, [-1, num_input_units]) - y = standard_ops.matmul(x_2_dim, w) - - if bias_init is not None: - bias_collections = set( - list(bias_collections or []) + [ops.GraphKeys.GLOBAL_VARIABLES]) - b = variable_scope.get_variable( - 'bias', - shape=[num_output_units], - dtype=dtype, - initializer=bias_init, - collections=bias_collections, - regularizer=bias_regularizer, - trainable=trainable) - - y = nn.bias_add(y, b) - - if len(dims) > 2: - out_shape = array_ops.unstack(array_ops.shape(x)) - out_shape[-1] = num_output_units - - y = array_ops.reshape(y, array_ops.stack(out_shape)) - - static_shape = x.get_shape().as_list() - static_shape[-1] = num_output_units - y.set_shape(static_shape) - - return _apply_activation(y, activation_fn, output_collections) - - -# TODO(eiderm): Verify and fix autocomplete in colab (also relu6). -# Simple aliases which remove the activation_fn parameter. -elu = functools.partial(fully_connected, activation_fn=nn.elu) -legacy_relu = functools.partial(legacy_fully_connected, activation_fn=nn.relu) -legacy_linear = functools.partial(legacy_fully_connected, activation_fn=None) -relu = functools.partial(fully_connected, activation_fn=nn.relu) -relu6 = functools.partial(fully_connected, activation_fn=nn.relu6) -linear = functools.partial(fully_connected, activation_fn=None) - -# Simple alias. -conv1d = convolution1d -conv2d = convolution2d -conv3d = convolution3d -conv2d_transpose = convolution2d_transpose -conv3d_transpose = convolution3d_transpose -conv2d_in_plane = convolution2d_in_plane -separable_conv2d = separable_convolution2d diff --git a/tensorflow/contrib/layers/python/layers/layers_test.py b/tensorflow/contrib/layers/python/layers/layers_test.py deleted file mode 100644 index 1b1e4cb04e7..00000000000 --- a/tensorflow/contrib/layers/python/layers/layers_test.py +++ /dev/null @@ -1,4219 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.contrib.layers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -import numpy as np - -from tensorflow.contrib import layers as layers_lib -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.framework.python.ops import variables -from tensorflow.contrib.layers.python.layers import layers as _layers -from tensorflow.contrib.layers.python.layers import regularizers -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gradient_checker -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import template -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import test - - -class AvgPool2DTest(test.TestCase): - - def testInvalidDataFormat(self): - height, width = 3, 6 - images = np.random.uniform(size=(5, height, width, 3)) - with self.assertRaisesRegexp(ValueError, - 'data_format has to be either NCHW or NHWC.'): - _layers.avg_pool2d(images, [3, 3], data_format='CHWN') - - def testCreateAvgPool(self): - height, width = 3, 6 - images = np.random.uniform(size=(5, height, width, 3)) - output = _layers.avg_pool2d(images, [3, 3]) - self.assertEqual(output.op.name, 'AvgPool2D/AvgPool') - self.assertListEqual(output.get_shape().as_list(), [5, 1, 2, 3]) - - def testCreateAvgPoolNCHW(self): - height, width = 3, 6 - images = np.random.uniform(size=(5, 2, height, width)) - output = _layers.avg_pool2d(images, [3, 3], data_format='NCHW') - self.assertListEqual(output.get_shape().as_list(), [5, 2, 1, 2]) - - def testCollectOutputs(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.avg_pool2d(images, [3, 3], outputs_collections='outputs') - output_collected = ops.get_collection('outputs')[0] - self.assertEqual(output_collected.aliases, ['AvgPool2D']) - self.assertEqual(output_collected, output) - - def testCreateSquareAvgPool(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.avg_pool2d(images, 3) - self.assertEqual(output.op.name, 'AvgPool2D/AvgPool') - self.assertListEqual(output.get_shape().as_list(), [5, 1, 2, 3]) - - def testCreateAvgPoolWithScope(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.avg_pool2d(images, [3, 3], scope='pool1') - self.assertEqual(output.op.name, 'pool1/AvgPool') - - def testCreateAvgPoolWithSamePadding(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.avg_pool2d(images, [3, 3], padding='SAME') - self.assertListEqual(output.get_shape().as_list(), [5, 2, 3, 3]) - - def testCreateAvgPoolWithSamePaddingNCHW(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, 3, height, width), seed=1) - output = _layers.avg_pool2d( - images, [3, 3], padding='SAME', data_format='NCHW') - self.assertListEqual(output.get_shape().as_list(), [5, 3, 2, 3]) - - def testCreateAvgPoolStrideWithSamePadding(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.avg_pool2d(images, [3, 3], stride=1, padding='SAME') - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 3]) - - def testGlobalAvgPool(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.avg_pool2d(images, images.get_shape()[1:3], stride=1) - self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3]) - - -class AvgPool3DTest(test.TestCase): - - def testInvalidDataFormat(self): - depth, height, width = 3, 6, 9 - images = np.random.uniform(size=(5, depth, height, width, 3)) - with self.assertRaisesRegexp( - ValueError, 'data_format has to be either NCDHW or NDHWC.'): - _layers.avg_pool3d(images, [3, 3, 3], data_format='CDHWN') - - def testCreateAvgPool(self): - depth, height, width = 3, 6, 9 - images = np.random.uniform(size=(5, depth, height, width, 3)) - output = _layers.avg_pool3d(images, [3, 3, 3]) - self.assertEqual(output.op.name, 'AvgPool3D/AvgPool3D') - self.assertListEqual(output.get_shape().as_list(), [5, 1, 2, 4, 3]) - - def testCreateAvgPoolNCDHW(self): - depth, height, width = 3, 6, 9 - images = np.random.uniform(size=(5, 2, depth, height, width)) - output = _layers.avg_pool3d(images, [3, 3, 3], data_format='NCDHW') - self.assertEquals(output.op.name, 'AvgPool3D/transpose_1') - self.assertListEqual(output.get_shape().as_list(), [5, 2, 1, 2, 4]) - - def testCollectOutputs(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, depth, height, width, 3), seed=1) - output = _layers.avg_pool3d( - images, [3, 3, 3], outputs_collections='outputs') - output_collected = ops.get_collection('outputs')[0] - self.assertEqual(output_collected.aliases, ['AvgPool3D']) - self.assertEqual(output_collected, output) - - def testCreateSquareAvgPool(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, depth, height, width, 3), seed=1) - output = _layers.avg_pool3d(images, 3) - self.assertEqual(output.op.name, 'AvgPool3D/AvgPool3D') - self.assertListEqual(output.get_shape().as_list(), [5, 1, 2, 4, 3]) - - def testCreateAvgPoolWithScope(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, depth, height, width, 3), seed=1) - output = _layers.avg_pool3d(images, [3, 3, 3], scope='pool1') - self.assertEqual(output.op.name, 'pool1/AvgPool3D') - - def testCreateAvgPoolWithSamePadding(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, depth, height, width, 3), seed=1) - output = _layers.avg_pool3d(images, [3, 3, 3], padding='SAME') - self.assertListEqual(output.get_shape().as_list(), [5, 2, 3, 5, 3]) - - def testCreateAvgPoolWithSamePaddingNCDHW(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, 3, depth, height, width), seed=1) - output = _layers.avg_pool3d( - images, [3, 3, 3], padding='SAME', data_format='NCDHW') - self.assertListEqual(output.get_shape().as_list(), [5, 3, 2, 3, 5]) - - def testCreateAvgPoolStrideWithSamePadding(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, depth, height, width, 3), seed=1) - output = _layers.avg_pool3d(images, [3, 3, 3], stride=1, padding='SAME') - self.assertListEqual(output.get_shape().as_list(), - [5, depth, height, width, 3]) - - def testGlobalAvgPool(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, depth, height, width, 3), seed=1) - output = _layers.avg_pool3d(images, images.get_shape()[1:4], stride=1) - self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 1, 3]) - - -class PoolTest(test.TestCase): - - def testCreatePool(self): - height, width = 3, 3 - images = np.random.uniform(size=(5, height, width, 3)) - output = _layers.pool(images, [3, 3], pooling_type='AVG') - self.assertEqual(output.op.name, 'avg_pool') - self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3]) - - def testCreatePoolNCHW(self): - height, width = 3, 3 - images = np.random.uniform(size=(5, 3, height, width)) - output = _layers.pool( - images, [3, 3], pooling_type='AVG', data_format='NCHW') - self.assertEqual(output.op.name, 'avg_pool') - self.assertListEqual(output.get_shape().as_list(), [5, 3, 1, 1]) - - def testCollectOutputs(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.pool( - images, [3, 3], pooling_type='AVG', outputs_collections='outputs') - output_collected = ops.get_collection('outputs')[0] - self.assertEqual(output_collected.aliases, ['avg_pool']) - self.assertEqual(output_collected, output) - - def testCreateSquareAvgPool(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.pool(images, 3, pooling_type='AVG') - self.assertEqual(output.op.name, 'avg_pool') - self.assertEqual(output.get_shape().as_list(), [5, 1, 1, 3]) - - def testCreateMaxPoolWithScope(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.pool(images, [3, 3], pooling_type='MAX', scope='pool1') - self.assertEqual(output.op.name, 'pool1') - - def testCreateMaxPoolWithSamePadding(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.pool(images, [3, 3], pooling_type='MAX', padding='SAME') - self.assertEqual(output.get_shape().as_list(), [5, 3, 3, 3]) - - def testCreateAvgPoolStrideWithSamePadding(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.pool( - images, [3, 3], stride=1, padding='SAME', pooling_type='AVG') - self.assertEqual(output.get_shape().as_list(), [5, height, width, 3]) - - def testGlobalAvgPool(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.pool( - images, images.get_shape()[1:3], stride=1, pooling_type='AVG') - self.assertEqual(output.get_shape().as_list(), [5, 1, 1, 3]) - - def testAvgPoolWithStride(self): - height, width = 5, 8 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.pool(images, [2, 3], stride=[1, 2], pooling_type='AVG') - self.assertEqual(output.get_shape().as_list(), [5, 4, 3, 3]) - - def testAvgPoolWithDilation(self): - height, width = 5, 8 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.pool( - images, [2, 3], dilation_rate=[1, 2], pooling_type='AVG') - self.assertEqual(output.get_shape().as_list(), [5, 4, 4, 3]) - - def testAvgPoolWithDilationNCHW(self): - height, width = 5, 8 - images = random_ops.random_uniform((5, 3, height, width), seed=1) - output = _layers.pool( - images, [2, 3], - dilation_rate=[1, 2], - pooling_type='AVG', - data_format='NCHW') - self.assertEqual(output.get_shape().as_list(), [5, 3, 4, 4]) - - -class BiasAddTest(test.TestCase): - - def testCreate(self): - height, width = 3, 3 - with self.cached_session(): - images = np.random.uniform(size=(5, height, width, 3)) - output = _layers.bias_add(images) - self.assertEqual(output.op.name, 'BiasAdd/BiasAdd') - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 3]) - - def testCreateWithActivation(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.bias_add(images, activation_fn=nn_ops.relu) - self.assertEqual(output.op.name, 'BiasAdd/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 3]) - - def testCreateDimensions(self): - dims = (2, 3, 4) - shape = [5, 2, 3, 4] - with self.cached_session(): - for d in dims: - input_shape = shape[:d] - inputs = random_ops.random_uniform(input_shape, seed=1) - output = _layers.bias_add(inputs) - self.assertListEqual(output.get_shape().as_list(), input_shape) - biases = variables.get_variables_by_name('biases')[-1] - self.assertListEqual(biases.get_shape().as_list(), [input_shape[-1]]) - - -class ConvolutionTest(test.TestCase): - - def testInvalidShape(self): - with self.cached_session(): - images_2d = random_ops.random_uniform((5, 7, 9, 3), seed=1) - with self.assertRaisesRegexp( - ValueError, 'Convolution expects input with rank 5, got 4'): - layers_lib.convolution3d(images_2d, 32, 3) - images_3d = random_ops.random_uniform((5, 6, 7, 9, 3), seed=1) - with self.assertRaisesRegexp( - ValueError, 'Convolution expects input with rank 4, got 5'): - layers_lib.convolution2d(images_3d, 32, 3) - - def testInvalidDataFormat(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - with self.assertRaisesRegexp(ValueError, 'data_format'): - layers_lib.convolution2d(images, 32, 3, data_format='CHWN') - - def testCreateConv(self): - height, width = 7, 9 - with self.cached_session(): - images = np.random.uniform(size=(5, height, width, 4)).astype(np.float32) - output = layers_lib.convolution2d(images, 32, [3, 3]) - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 32]) - weights = variables.get_variables_by_name('weights')[0] - self.assertListEqual(weights.get_shape().as_list(), [3, 3, 4, 32]) - biases = variables.get_variables_by_name('biases')[0] - self.assertListEqual(biases.get_shape().as_list(), [32]) - - def testCreateConvNCHW(self): - height, width = 7, 9 - with self.cached_session(): - images = np.random.uniform(size=(5, 4, height, width)).astype(np.float32) - output = layers_lib.convolution2d(images, 32, [3, 3], data_format='NCHW') - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, 32, height, width]) - weights = variables.get_variables_by_name('weights')[0] - self.assertListEqual(weights.get_shape().as_list(), [3, 3, 4, 32]) - biases = variables.get_variables_by_name('biases')[0] - self.assertListEqual(biases.get_shape().as_list(), [32]) - - def testCreateSquareConv(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = layers_lib.convolution2d(images, 32, 3) - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 32]) - - def testCreateConvWithTensorShape(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = layers_lib.convolution2d(images, 32, images.get_shape()[1:3]) - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 32]) - - def testCreateFullyConv(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 32), seed=1) - output = layers_lib.convolution2d( - images, 64, images.get_shape()[1:3], padding='VALID') - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 64]) - biases = variables.get_variables_by_name('biases')[0] - self.assertListEqual(biases.get_shape().as_list(), [64]) - - def testFullyConvWithCustomGetter(self): - height, width = 7, 9 - with self.cached_session(): - called = [0] - - def custom_getter(getter, *args, **kwargs): - called[0] += 1 - return getter(*args, **kwargs) - - with variable_scope.variable_scope('test', custom_getter=custom_getter): - images = random_ops.random_uniform((5, height, width, 32), seed=1) - layers_lib.convolution2d(images, 64, images.get_shape()[1:3]) - self.assertEqual(called[0], 2) # Custom getter called twice. - - def testCreateVerticalConv(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 4), seed=1) - output = layers_lib.convolution2d(images, 32, [3, 1]) - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 32]) - weights = variables.get_variables_by_name('weights')[0] - self.assertListEqual(weights.get_shape().as_list(), [3, 1, 4, 32]) - biases = variables.get_variables_by_name('biases')[0] - self.assertListEqual(biases.get_shape().as_list(), [32]) - - def testCreateHorizontalConv(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 4), seed=1) - output = layers_lib.convolution2d(images, 32, [1, 3]) - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 32]) - weights = variables.get_variables_by_name('weights')[0] - self.assertListEqual(weights.get_shape().as_list(), [1, 3, 4, 32]) - - def testCreateConvWithStride(self): - height, width = 6, 8 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = layers_lib.convolution2d(images, 32, [3, 3], stride=2) - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(output.get_shape().as_list(), - [5, height / 2, width / 2, 32]) - - def testCreateConvCreatesWeightsAndBiasesVars(self): - height, width = 7, 9 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - with self.cached_session(): - self.assertFalse(variables.get_variables('conv1/weights')) - self.assertFalse(variables.get_variables('conv1/biases')) - layers_lib.convolution2d(images, 32, [3, 3], scope='conv1') - self.assertTrue(variables.get_variables('conv1/weights')) - self.assertTrue(variables.get_variables('conv1/biases')) - - def testCreateConvWithScope(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = layers_lib.convolution2d(images, 32, [3, 3], scope='conv1') - self.assertEqual(output.op.name, 'conv1/Relu') - - def testCreateConvWithCollection(self): - height, width = 7, 9 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - with ops.name_scope('fe'): - conv = layers_lib.convolution2d( - images, 32, [3, 3], outputs_collections='outputs', scope='Conv') - output_collected = ops.get_collection('outputs')[0] - self.assertEqual(output_collected.aliases, ['Conv']) - self.assertEqual(output_collected, conv) - - def testCreateConvWithoutActivation(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = layers_lib.convolution2d(images, 32, [3, 3], activation_fn=None) - self.assertEqual(output.op.name, 'Conv/BiasAdd') - - def testCreateConvValid(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = layers_lib.convolution2d(images, 32, [3, 3], padding='VALID') - self.assertListEqual(output.get_shape().as_list(), [5, 5, 7, 32]) - - def testCreateConvWithWD(self): - height, width = 7, 9 - weight_decay = 0.01 - with self.cached_session() as sess: - images = random_ops.random_uniform((5, height, width, 3), seed=1) - regularizer = regularizers.l2_regularizer(weight_decay) - layers_lib.convolution2d( - images, 32, [3, 3], weights_regularizer=regularizer) - l2_loss = nn_ops.l2_loss(variables.get_variables_by_name('weights')[0]) - wd = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)[0] - self.assertEqual(wd.op.name, 'Conv/kernel/Regularizer/l2_regularizer') - sess.run(variables_lib.global_variables_initializer()) - self.assertAlmostEqual(sess.run(wd), weight_decay * l2_loss.eval()) - - def testCreateConvNoRegularizers(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - layers_lib.convolution2d(images, 32, [3, 3]) - self.assertEqual( - ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES), []) - - def testReuseVars(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - layers_lib.convolution2d(images, 32, [3, 3], scope='conv1') - self.assertEqual(len(variables.get_variables()), 2) - layers_lib.convolution2d(images, 32, [3, 3], scope='conv1', reuse=True) - self.assertEqual(len(variables.get_variables()), 2) - - def testNonReuseVars(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - layers_lib.convolution2d(images, 32, [3, 3]) - self.assertEqual(len(variables.get_variables()), 2) - layers_lib.convolution2d(images, 32, [3, 3]) - self.assertEqual(len(variables.get_variables()), 4) - - def testReuseConvWithWD(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - weight_decay = regularizers.l2_regularizer(0.01) - with arg_scope( - [layers_lib.convolution2d], weights_regularizer=weight_decay): - layers_lib.convolution2d(images, 32, [3, 3], scope='conv1') - self.assertEqual(len(variables.get_variables()), 2) - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 1) - layers_lib.convolution2d(images, 32, [3, 3], scope='conv1', reuse=True) - self.assertEqual(len(variables.get_variables()), 2) - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 1) - - def testConvWithBatchNorm(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 32), seed=1) - with arg_scope( - [layers_lib.convolution2d], - normalizer_fn=_layers.batch_norm, - normalizer_params={ - 'decay': 0.9 - }): - net = layers_lib.convolution2d(images, 32, [3, 3]) - net = layers_lib.convolution2d(net, 32, [3, 3]) - self.assertEqual(len(variables.get_variables()), 8) - self.assertEqual(len(variables.get_variables('Conv/BatchNorm')), 3) - self.assertEqual(len(variables.get_variables('Conv_1/BatchNorm')), 3) - - def testReuseConvWithBatchNorm(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 32), seed=1) - with arg_scope( - [layers_lib.convolution2d], - normalizer_fn=_layers.batch_norm, - normalizer_params={ - 'decay': 0.9 - }): - net = layers_lib.convolution2d(images, 32, [3, 3], scope='Conv') - net = layers_lib.convolution2d( - net, 32, [3, 3], scope='Conv', reuse=True) - self.assertEqual(len(variables.get_variables()), 4) - self.assertEqual(len(variables.get_variables('Conv/BatchNorm')), 3) - self.assertEqual(len(variables.get_variables('Conv_1/BatchNorm')), 0) - - def testCreateConvCreatesWeightsAndBiasesVarsWithRateTwo(self): - height, width = 7, 9 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - with self.cached_session(): - self.assertFalse(variables.get_variables('conv1/weights')) - self.assertFalse(variables.get_variables('conv1/biases')) - layers_lib.convolution2d(images, 32, [3, 3], rate=2, scope='conv1') - self.assertTrue(variables.get_variables('conv1/weights')) - self.assertTrue(variables.get_variables('conv1/biases')) - - def testOutputSizeWithRateTwoSamePadding(self): - num_filters = 32 - input_size = [5, 10, 12, 3] - expected_size = [5, 10, 12, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.convolution2d( - images, num_filters, [3, 3], rate=2, padding='SAME') - self.assertListEqual(list(output.get_shape().as_list()), expected_size) - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWithRateTwoValidPadding(self): - num_filters = 32 - input_size = [5, 10, 12, 3] - expected_size = [5, 6, 8, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.convolution2d( - images, num_filters, [3, 3], rate=2, padding='VALID') - self.assertListEqual(list(output.get_shape().as_list()), expected_size) - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWithRateTwoThreeValidPadding(self): - num_filters = 32 - input_size = [5, 10, 12, 3] - expected_size = [5, 6, 6, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.convolution2d( - images, num_filters, [3, 3], rate=[2, 3], padding='VALID') - self.assertListEqual(list(output.get_shape().as_list()), expected_size) - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertEquals(output.op.name, 'Conv/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testDynamicOutputSizeWithRateOneValidPadding(self): - num_filters = 32 - input_size = [5, 9, 11, 3] - expected_size = [None, None, None, num_filters] - expected_size_dynamic = [5, 7, 9, num_filters] - - with self.cached_session(): - images = array_ops.placeholder(np.float32, - [None, None, None, input_size[3]]) - output = layers_lib.convolution2d( - images, num_filters, [3, 3], rate=1, padding='VALID') - variables_lib.global_variables_initializer().run() - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(output.get_shape().as_list(), expected_size) - eval_output = output.eval({images: np.zeros(input_size, np.float32)}) - self.assertListEqual(list(eval_output.shape), expected_size_dynamic) - - def testDynamicOutputSizeWithRateOneValidPaddingNCHW(self): - if test.is_gpu_available(cuda_only=True): - num_filters = 32 - input_size = [5, 3, 9, 11] - expected_size = [None, num_filters, None, None] - expected_size_dynamic = [5, num_filters, 7, 9] - - with self.session(use_gpu=True): - images = array_ops.placeholder(np.float32, - [None, input_size[1], None, None]) - output = layers_lib.convolution2d( - images, - num_filters, [3, 3], - rate=1, - padding='VALID', - data_format='NCHW') - variables_lib.global_variables_initializer().run() - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(output.get_shape().as_list(), expected_size) - eval_output = output.eval({images: np.zeros(input_size, np.float32)}) - self.assertListEqual(list(eval_output.shape), expected_size_dynamic) - - def testDynamicOutputSizeWithRateTwoValidPadding(self): - num_filters = 32 - input_size = [5, 9, 11, 3] - expected_size = [None, None, None, num_filters] - expected_size_dynamic = [5, 5, 7, num_filters] - - with self.cached_session(): - images = array_ops.placeholder(np.float32, - [None, None, None, input_size[3]]) - output = layers_lib.convolution2d( - images, num_filters, [3, 3], rate=2, padding='VALID') - variables_lib.global_variables_initializer().run() - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(output.get_shape().as_list(), expected_size) - eval_output = output.eval({images: np.zeros(input_size, np.float32)}) - self.assertListEqual(list(eval_output.shape), expected_size_dynamic) - - def testWithScope(self): - num_filters = 32 - input_size = [5, 9, 11, 3] - expected_size = [5, 5, 7, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.convolution2d( - images, num_filters, [3, 3], rate=2, padding='VALID', scope='conv7') - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'conv7/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testWithScopeWithoutActivation(self): - num_filters = 32 - input_size = [5, 9, 11, 3] - expected_size = [5, 5, 7, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.convolution2d( - images, - num_filters, [3, 3], - rate=2, - padding='VALID', - activation_fn=None, - scope='conv7') - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'conv7/BiasAdd') - self.assertListEqual(list(output.eval().shape), expected_size) - - -class Convolution2dTransposeTests(test.TestCase): - - def testTrainableFlagIsPassedOn(self): - for trainable in [True, False]: - with ops.Graph().as_default(): - num_filters = 32 - input_size = [5, 10, 12, 3] - - images = random_ops.random_uniform(input_size, seed=1) - layers_lib.conv2d_transpose( - images, num_filters, [3, 3], stride=1, trainable=trainable) - model_variables = variables.get_model_variables() - trainable_variables = variables_lib.trainable_variables() - for model_variable in model_variables: - self.assertEqual(trainable, model_variable in trainable_variables) - - def testInvalidDataFormat(self): - height, width = 7, 9 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - with self.assertRaisesRegexp( - ValueError, 'data_format has to be either NCHW or NHWC.'): - _layers.convolution2d_transpose(images, 32, 3, data_format='CHWN') - - def testOutputSizeWithStrideOneSamePaddingNCHW(self): - # `NCHW` data format is only supported for `GPU` device. - if test.is_gpu_available(cuda_only=True): - with self.session(use_gpu=True) as sess: - num_filters = 32 - input_size = [5, 3, 10, 12] - expected_size = [5, num_filters, 10, 12] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, - num_filters, [3, 3], - stride=1, - padding='SAME', - data_format='NCHW') - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - - sess.run(variables_lib.global_variables_initializer()) - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWithStrideOneValidPaddingNCHW(self): - if test.is_gpu_available(cuda_only=True): - with self.session(use_gpu=True) as sess: - num_filters = 32 - input_size = [5, 3, 10, 12] - expected_size = [5, num_filters, 12, 14] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, - num_filters, [3, 3], - stride=1, - padding='VALID', - data_format='NCHW') - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - - sess.run(variables_lib.global_variables_initializer()) - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWithStrideTwoValidPaddingNCHW(self): - if test.is_gpu_available(cuda_only=True): - with self.session(use_gpu=True) as sess: - num_filters = 32 - input_size = [5, 3, 9, 11] - expected_size = [5, num_filters, 19, 23] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, - num_filters, [3, 3], - stride=[2, 2], - padding='VALID', - data_format='NCHW') - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.get_shape().as_list()), expected_size) - - sess.run(variables_lib.global_variables_initializer()) - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWith1x1StrideTwoSamePaddingNCHW(self): - if test.is_gpu_available(cuda_only=True): - with self.session(use_gpu=True) as sess: - num_filters = 1 - input_size = [1, 1, 1, 1] - expected_size = [1, num_filters, 2, 2] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, - num_filters, [2, 2], - stride=[2, 2], - padding='SAME', - data_format='NCHW') - self.assertListEqual(list(output.get_shape().as_list()), expected_size) - - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWith1x1StrideTwoValidPaddingNCHW(self): - if test.is_gpu_available(cuda_only=True): - with self.session(use_gpu=True) as sess: - num_filters = 1 - input_size = [1, 1, 1, 1] - expected_size = [1, num_filters, 2, 2] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, - num_filters, [2, 2], - stride=[2, 2], - padding='VALID', - data_format='NCHW') - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWith2x2StrideTwoSamePaddingNCHW(self): - if test.is_gpu_available(cuda_only=True): - with self.session(use_gpu=True) as sess: - num_filters = 1 - input_size = [1, 1, 2, 2] - expected_size = [1, num_filters, 4, 4] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, - num_filters, [2, 2], - stride=[2, 2], - padding='SAME', - data_format='NCHW') - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWith2x2StrideTwoValidPaddingNCHW(self): - if test.is_gpu_available(cuda_only=True): - with self.session(use_gpu=True) as sess: - num_filters = 1 - input_size = [1, 1, 2, 2] - expected_size = [1, num_filters, 4, 4] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, - num_filters, [2, 2], - stride=[2, 2], - padding='VALID', - data_format='NCHW') - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWithStride2x1NCHW(self): - if test.is_gpu_available(cuda_only=True): - with self.session(use_gpu=True) as sess: - num_filters = 1 - input_size = [1, 1, 3, 2] - expected_size = [1, num_filters, 6, 5] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, - num_filters, [2, 4], - stride=[2, 1], - padding='VALID', - data_format='NCHW') - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWithStride2x4NCHW(self): - if test.is_gpu_available(cuda_only=True): - with self.session(use_gpu=True) as sess: - num_filters = 1 - input_size = [1, 1, 3, 2] - expected_size = [1, num_filters, 6, 8] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, - num_filters, [2, 4], - stride=[2, 4], - padding='VALID', - data_format='NCHW') - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWithStride2x5NCHW(self): - if test.is_gpu_available(cuda_only=True): - with self.session(use_gpu=True) as sess: - num_filters = 1 - input_size = [1, 1, 3, 2] - expected_size = [1, num_filters, 6, 10] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, - num_filters, [2, 4], - stride=[2, 5], - padding='VALID', - data_format='NCHW') - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWithStrideOneSamePadding(self): - num_filters = 32 - input_size = [5, 10, 12, 3] - expected_size = [5, 10, 12, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, num_filters, [3, 3], stride=1, padding='SAME') - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWithStrideOneValidPadding(self): - num_filters = 32 - input_size = [5, 10, 12, 3] - expected_size = [5, 12, 14, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, num_filters, [3, 3], stride=1, padding='VALID') - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWithStrideTwoValidPadding(self): - num_filters = 32 - input_size = [5, 9, 11, 3] - expected_size = [5, 19, 23, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, num_filters, [3, 3], stride=[2, 2], padding='VALID') - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.get_shape().as_list()), expected_size) - - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWith1x1StrideTwoSamePadding(self): - num_filters = 1 - input_size = [1, 1, 1, 1] - expected_size = [1, 2, 2, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, num_filters, [2, 2], stride=[2, 2], padding='SAME') - self.assertListEqual(list(output.get_shape().as_list()), expected_size) - - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWith1x1StrideTwoValidPadding(self): - num_filters = 1 - input_size = [1, 1, 1, 1] - expected_size = [1, 2, 2, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, num_filters, [2, 2], stride=[2, 2], padding='VALID') - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWith2x2StrideTwoSamePadding(self): - num_filters = 1 - input_size = [1, 2, 2, 1] - expected_size = [1, 4, 4, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, num_filters, [2, 2], stride=[2, 2], padding='SAME') - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWith2x2StrideTwoValidPadding(self): - num_filters = 1 - input_size = [1, 2, 2, 1] - expected_size = [1, 4, 4, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, num_filters, [2, 2], stride=[2, 2], padding='VALID') - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWithStride2x1(self): - num_filters = 1 - input_size = [1, 3, 2, 1] - expected_size = [1, 6, 5, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, num_filters, [2, 4], stride=[2, 1], padding='VALID') - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWithStride2x4(self): - num_filters = 1 - input_size = [1, 3, 2, 1] - expected_size = [1, 6, 8, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, num_filters, [2, 4], stride=[2, 4], padding='VALID') - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeWithStride2x5(self): - num_filters = 1 - input_size = [1, 3, 2, 1] - expected_size = [1, 6, 10, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, num_filters, [2, 4], stride=[2, 5], padding='VALID') - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(list(output.eval().shape), expected_size) - - def testOutputSizeRandomSizesAndStridesValidPadding(self): - np.random.seed(0) - max_image_size = 10 - - for _ in range(10): - num_filters = 1 - input_size = [ - 1, - np.random.randint(1, max_image_size), - np.random.randint(1, max_image_size), 1 - ] - filter_size = [ - np.random.randint(1, input_size[1] + 1), - np.random.randint(1, input_size[2] + 1) - ] - stride = [np.random.randint(1, 3), np.random.randint(1, 3)] - - ops.reset_default_graph() - graph = ops.Graph() - with graph.as_default(): - images = random_ops.random_uniform(input_size, seed=1) - transpose = layers_lib.conv2d_transpose( - images, num_filters, filter_size, stride=stride, padding='VALID') - conv = layers_lib.conv2d( - transpose, num_filters, filter_size, stride=stride, padding='VALID') - - with self.session(graph=graph) as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertListEqual(list(conv.eval().shape), input_size) - - def testDynamicOutputSizeWithStrideTwoValidPadding(self): - num_filters = 32 - input_size = [5, 9, 11, 3] - expected_size = [None, None, None, num_filters] - expected_size_dynamic = [5, 19, 23, num_filters] - - images = array_ops.placeholder(np.float32, - [None, None, None, input_size[3]]) - output = layers_lib.conv2d_transpose( - images, num_filters, [3, 3], stride=[2, 2], padding='VALID') - self.assertListEqual(output.get_shape().as_list(), expected_size) - - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - eval_output = output.eval({images: np.zeros(input_size, np.float32)}) - self.assertListEqual(list(eval_output.shape), expected_size_dynamic) - - def testDynamicOutputSizeWithStrideTwoSamePadding(self): - num_filters = 32 - input_size = [5, 9, 11, 3] - expected_size = [None, None, None, num_filters] - expected_size_dynamic = [5, 18, 22, num_filters] - - with self.cached_session(): - images = array_ops.placeholder(np.float32, - [None, None, None, input_size[3]]) - output = layers_lib.conv2d_transpose( - images, num_filters, [3, 3], stride=[2, 2], padding='SAME') - variables_lib.global_variables_initializer().run() - self.assertEqual(output.op.name, 'Conv2d_transpose/Relu') - self.assertListEqual(output.get_shape().as_list(), expected_size) - eval_output = output.eval({images: np.zeros(input_size, np.float32)}) - self.assertListEqual(list(eval_output.shape), expected_size_dynamic) - - def testWithScope(self): - num_filters = 32 - input_size = [5, 9, 11, 3] - expected_size = [5, 19, 23, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, num_filters, [3, 3], stride=2, padding='VALID', scope='conv7') - self.assertEqual(output.op.name, 'conv7/Relu') - - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertListEqual(list(output.eval().shape), expected_size) - - def testWithScopeWithoutActivation(self): - num_filters = 32 - input_size = [5, 9, 11, 3] - expected_size = [5, 19, 23, num_filters] - - images = random_ops.random_uniform(input_size, seed=1) - output = layers_lib.conv2d_transpose( - images, - num_filters, [3, 3], - stride=2, - padding='VALID', - activation_fn=None, - scope='conv7') - self.assertEqual(output.op.name, 'conv7/BiasAdd') - - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertListEqual(list(output.eval().shape), expected_size) - - def testDeconvWithoutBiasesProducesConv2dTranspose(self): - num_filters = 32 - input_size = [5, 9, 11, 3] - expected_size = [5, 19, 23, num_filters] - stride = 2 - padding = 'VALID' - - with self.cached_session() as sess: - images = random_ops.random_uniform(input_size, seed=1) - output_deconv = layers_lib.conv2d_transpose( - images, - num_filters, [3, 3], - stride=stride, - padding=padding, - activation_fn=None, - scope='conv7') - - weights = variables.get_variables_by_name('conv7/weights')[0] - output_conv2d_transpose = nn_ops.conv2d_transpose( - images, - weights, - expected_size, [1, stride, stride, 1], - padding=padding) - - sess.run(variables_lib.global_variables_initializer()) - - output_deconv, output_conv2d_transpose = sess.run( - [output_deconv, output_conv2d_transpose]) - - self.assertTrue( - np.isclose(output_deconv, output_conv2d_transpose, 1e-5, 1e-5).all()) - - -class ConvolutionInPlaneTest(test.TestCase): - - def testHorzConvWithBlankImage(self): - image = array_ops.ones((1, 10, 10, 1)) - horz_gradients = layers_lib.conv2d_in_plane( - image, - weights_initializer=init_ops.constant_initializer([1, -1]), - kernel_size=[1, 2], - padding='VALID', - activation_fn=None) - init_op = variables_lib.global_variables_initializer() - - with self.cached_session() as sess: - sess.run(init_op) - result = sess.run(horz_gradients) - expected = np.zeros((1, 10, 9, 1)) - - self.assertAllClose(result, expected, rtol=1e-5, atol=1e-5) - - def testHorzConvWithBlankImageAndPlaceholder(self): - image = array_ops.placeholder(dtypes.float32, shape=(None, None, None, 1)) - horz_gradients = layers_lib.conv2d_in_plane( - image, - weights_initializer=init_ops.constant_initializer([1, -1]), - kernel_size=[1, 2], - padding='VALID', - activation_fn=None) - init_op = variables_lib.global_variables_initializer() - - with self.cached_session() as sess: - sess.run(init_op) - result = sess.run( - horz_gradients, feed_dict={ - image: np.ones((1, 10, 10, 1)) - }) - expected = np.zeros((1, 10, 9, 1)) - - self.assertAllClose(result, expected, rtol=1e-5, atol=1e-5) - - def testHorzConvWithRandomImageMultiBatch(self): - np.random.seed(1) - image = np.random.rand(5, 10, 10, 1) - expected = image[:, :, 0:-1, :] - image[:, :, 1:, :] - - tf_image = constant_op.constant(image, dtype=dtypes.float32) - horz_gradients = layers_lib.conv2d_in_plane( - tf_image, - weights_initializer=init_ops.constant_initializer([1, -1]), - kernel_size=[1, 2], - padding='VALID', - activation_fn=None) - init_op = variables_lib.global_variables_initializer() - - with self.cached_session() as sess: - sess.run(init_op) - result = sess.run(horz_gradients) - - self.assertAllClose(result, expected, rtol=1e-5, atol=1e-5) - - def testHorzConvWithRandomImageMultiBatchMultiChannel(self): - np.random.seed(1) - image = np.random.rand(5, 10, 10, 7) - expected = image[:, :, 0:-1, :] - image[:, :, 1:, :] - - tf_image = constant_op.constant(image, dtype=dtypes.float32) - horz_gradients = layers_lib.conv2d_in_plane( - tf_image, - weights_initializer=init_ops.constant_initializer([1, -1]), - kernel_size=[1, 2], - padding='VALID', - activation_fn=None) - init_op = variables_lib.global_variables_initializer() - - with self.cached_session() as sess: - sess.run(init_op) - result = sess.run(horz_gradients) - - self.assertAllClose(result, expected, rtol=1e-5, atol=1e-5) - - def testHorzConvWithVaryingImage(self): - image = np.asmatrix(('1.0 2.0 3.0;' '1.1 2.0 4.0;' '-4.3 0.0 8.9')) - - expected = np.asmatrix(('-1.0 -1.0;' '-0.9 -2.0;' '-4.3 -8.9')) - expected = np.reshape(np.asarray(expected), (1, 3, 2, 1)) - - tf_image = constant_op.constant( - image, shape=(1, 3, 3, 1), dtype=dtypes.float32) - horz_gradients = layers_lib.conv2d_in_plane( - tf_image, - weights_initializer=init_ops.constant_initializer([1, -1]), - kernel_size=[1, 2], - padding='VALID', - activation_fn=None) - init_op = variables_lib.global_variables_initializer() - - with self.cached_session() as sess: - sess.run(init_op) - result = sess.run(horz_gradients) - - self.assertAllClose(result, expected, rtol=1e-5, atol=1e-5) - - def testVertConvWithBlankImage(self): - image = array_ops.ones((1, 10, 10, 1)) - vert_gradients = layers_lib.conv2d_in_plane( - image, - weights_initializer=init_ops.constant_initializer([1, -1]), - kernel_size=[2, 1], - padding='VALID', - activation_fn=None) - init_op = variables_lib.global_variables_initializer() - - with self.cached_session() as sess: - sess.run(init_op) - result = sess.run(vert_gradients) - expected = np.zeros((1, 9, 10, 1)) - - self.assertAllClose(result, expected, rtol=1e-5, atol=1e-5) - - def testVertConvWithVaryingImage(self): - image = np.asmatrix(('1.0 2.0 3.0;' '1.1 2.0 4.0;' '-4.3 0.0 8.9')) - - expected = np.asmatrix(('-0.1 0.0 -1.0;' ' 5.4 2.0 -4.9')) - expected = np.reshape(np.asarray(expected), (1, 2, 3, 1)) - - tf_image = constant_op.constant( - image, shape=(1, 3, 3, 1), dtype=dtypes.float32) - vert_gradients = layers_lib.conv2d_in_plane( - tf_image, - weights_initializer=init_ops.constant_initializer([1, -1]), - kernel_size=[2, 1], - padding='VALID', - activation_fn=None) - init_op = variables_lib.global_variables_initializer() - - with self.cached_session() as sess: - sess.run(init_op) - result = sess.run(vert_gradients) - - self.assertAllClose(result, expected, rtol=1e-5, atol=1e-5) - - def testConv1dShape(self): - width = 7 - with self.cached_session(): - images = random_ops.random_uniform((5, width, 3), seed=1) - output = layers_lib.convolution1d(images, 32, 3) - self.assertEqual(output.op.name, 'Conv/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, width, 32]) - - def testConvInferSpatialDims(self): - depth, height, width = 7, 9, 11 - with self.cached_session(): - images = np.random.uniform(size=(5, width, 4)).astype(np.float32) - output = layers_lib.convolution(images, 32, [3]) - self.assertListEqual(output.get_shape().as_list(), [5, width, 32]) - images = np.random.uniform(size=(5, height, width, 4)).astype(np.float32) - output = layers_lib.convolution(images, 32, [3, 3]) - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 32]) - images = np.random.uniform(size=(5, depth, height, width, - 4)).astype(np.float32) - output = layers_lib.convolution(images, 32, [3, 3, 3]) - self.assertListEqual(output.get_shape().as_list(), - [5, depth, height, width, 32]) - - -class DenseToSparseTest(test.TestCase): - - def testDenseFromConstantToSparse(self): - expected_constant = np.reshape(np.arange(24, dtype=np.int64), (3, 4, 2)) - tensor = constant_op.constant(expected_constant) - sparse = _layers.dense_to_sparse(tensor) - dense = sparse_ops.sparse_to_dense(sparse.indices, sparse.dense_shape, - sparse.values) - with self.cached_session() as sess: - constant = sess.run(dense) - self.assertAllEqual(expected_constant, constant) - - -class DropoutTest(test.TestCase): - - def testCreateDropout(self): - height, width = 3, 3 - with self.cached_session(): - images = np.random.uniform(size=(5, height, width, 3)) - output = _layers.dropout(images) - self.assertEqual(output.op.name, 'Dropout/dropout_1/mul_1') - output.get_shape().assert_is_compatible_with( - ops.convert_to_tensor(images).get_shape()) - - def testCreateDropoutWithConstantTrue(self): - height, width = 3, 3 - with self.cached_session(): - is_training = constant_op.constant(True) - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.dropout(images, is_training=is_training) - output.get_shape().assert_is_compatible_with(images.get_shape()) - - def testCreateDropoutWithConstantFalse(self): - height, width = 3, 3 - with self.cached_session(): - is_training = constant_op.constant(False) - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.dropout(images, is_training=is_training) - output.get_shape().assert_is_compatible_with(images.get_shape()) - - def testCreateDropoutWithPlaceholder(self): - height, width = 3, 3 - with self.cached_session(): - is_training = array_ops.placeholder(dtype=dtypes.bool, shape=[]) - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.dropout(images, is_training=is_training) - self.assertEqual(output.op.name, 'Dropout/cond/Merge') - output.get_shape().assert_is_compatible_with(images.get_shape()) - - def testCollectOutputs(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.dropout(images, outputs_collections='outputs') - c_output = ops.get_collection('outputs')[0] - self.assertEqual(c_output.aliases, ['Dropout']) - self.assertEqual(c_output, output) - - def testDropout(self): - height, width = 10, 10 - with self.cached_session() as sess: - images = random_ops.random_uniform( - (5, height, width, 3), seed=1, name='images') - num_elem_initial = math_ops.reduce_mean( - math_ops.cast(images > 0, dtypes.float32)) - output = _layers.dropout(images) - num_elem = math_ops.reduce_mean(math_ops.cast(output > 0, dtypes.float32)) - num_elem, num_elem_initial = sess.run([num_elem, num_elem_initial]) - self.assertLess(num_elem, num_elem_initial / 2 + 0.1) - self.assertGreater(num_elem, num_elem_initial / 2 - 0.1) - - def testDropoutSeed(self): - """Test that providing the same seed produces the same result.""" - height, width = 10, 10 - with self.cached_session() as sess: - images = random_ops.random_uniform( - (5, height, width, 3), seed=1, name='images') - output1 = _layers.dropout(images, seed=1) - output2 = _layers.dropout(images, seed=1) - self.assertAllEqual(*sess.run([output1, output2])) - - def testCreateDropoutNoTraining(self): - height, width = 3, 3 - with self.cached_session() as sess: - images = random_ops.random_uniform( - (5, height, width, 3), seed=1, name='images') - num_elem_initial = math_ops.reduce_mean( - math_ops.cast(images > 0, dtypes.float32)) - output = _layers.dropout(images, is_training=False) - num_elem = math_ops.reduce_mean(math_ops.cast(output > 0, dtypes.float32)) - num_elem, num_elem_initial = sess.run([num_elem, num_elem_initial]) - self.assertEqual(num_elem, num_elem_initial) - outputs, inputs = sess.run([output, images]) - self.assertAllClose(outputs, inputs) - - def testCreateFCFollowByDropout(self): - height, width = 3, 3 - with self.cached_session() as sess: - images = random_ops.random_uniform( - (5, height, width, 3), seed=1, name='images') - output = _layers.fully_connected(images, 50) - num_elem_initial = math_ops.reduce_mean( - math_ops.cast(output > 0, dtypes.float32)) - output = _layers.dropout(output) - num_elem = math_ops.reduce_mean(math_ops.cast(output > 0, dtypes.float32)) - sess.run(variables_lib.global_variables_initializer()) - num_elem, num_elem_initial = sess.run([num_elem, num_elem_initial]) - self.assertLess(num_elem, num_elem_initial / 2 + 0.1) - self.assertGreater(num_elem, num_elem_initial / 2 - 0.1) - - def testCreateFCWithDropout(self): - height, width = 3, 3 - with self.cached_session() as sess: - images = random_ops.random_uniform( - (5, height, width, 3), seed=1, name='images') - output = _layers.fully_connected( - images, 50, normalizer_fn=_layers.dropout) - num_elem = math_ops.reduce_mean(math_ops.cast(output > 0, dtypes.float32)) - sess.run(variables_lib.global_variables_initializer()) - num_elem = sess.run(num_elem) - self.assertLess(num_elem, 0.5) - self.assertGreater(num_elem, 0.1) - - -class FlattenTest(test.TestCase): - - def testUnknownLastDim(self): - with ops.Graph().as_default() as g, self.session(g): - inputs = array_ops.placeholder(dtype=dtypes.float32) - inputs.set_shape(tensor_shape.TensorShape((5, None))) - output = _layers.flatten(inputs) - self.assertEqual(output.get_shape().as_list(), [5, None]) - - def testCollectOutputs(self): - height, width = 3, 3 - with self.cached_session(): - images = np.random.uniform(size=(5, height, width, 3)) - output = _layers.flatten(images, outputs_collections='outputs') - c_output = ops.get_collection('outputs')[0] - self.assertEqual(c_output.aliases, ['Flatten']) - self.assertEqual(c_output, output) - - def testFlatten4D(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform( - (5, height, width, 3), seed=1, name='images') - output = _layers.flatten(images) - self.assertEqual(output.get_shape().num_elements(), - images.get_shape().num_elements()) - self.assertEqual(output.get_shape()[0], images.get_shape()[0]) - - def testFlatten3D(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform( - (5, height, width), seed=1, name='images') - output = _layers.flatten(images) - self.assertEqual(output.get_shape().num_elements(), - images.get_shape().num_elements()) - self.assertEqual(output.get_shape()[0], images.get_shape()[0]) - - def testFlatten0D(self): - with self.cached_session(): - scalars = random_ops.random_uniform((5,), seed=1, name='scalars') - output = _layers.flatten(scalars) - self.assertEqual(output.shape, (5, 1)) - - def testFlattenBatchSize(self): - height, width = 3, 3 - with self.cached_session() as sess: - images = random_ops.random_uniform( - (5, height, width, 3), seed=1, name='images') - inputs = array_ops.placeholder(dtypes.int32, (None, height, width, 3)) - output = _layers.flatten(inputs) - self.assertEqual(output.get_shape().as_list(), [None, height * width * 3]) - output = sess.run(output, {inputs: images.eval()}) - self.assertEqual(output.size, images.get_shape().num_elements()) - self.assertEqual(output.shape[0], images.get_shape()[0]) - - def testUnknownDims(self): - height = width = depth = 3 - with self.cached_session() as sess: - images = random_ops.random_uniform( - (5, height, width, depth), seed=1, name='images') - inputs = array_ops.placeholder(dtypes.int32, (None, None, None, None)) - output = _layers.flatten(inputs) - output = sess.run(output, {inputs: images.eval()}) - self.assertEqual(output.size, images.get_shape().num_elements()) - self.assertEqual(output.shape[0], images.get_shape()[0]) - - -def _sparsify(array, threshold=0.5): - array[array < threshold] = 0 - non_zero = np.where(array) - indices = np.vstack(non_zero).T - values = array[non_zero] - shape = array.shape - return indices, values, shape - - -class PartialFlattenTest(test.TestCase): - - def testDensePartialFlatten(self): - """Test `_inner_flatten` on `Tensor`s.""" - shape = [2, 3, 4, 5, 6] - np.random.seed(5446) - inputs = np.random.randint(0, 100, size=shape) - - for new_rank in [1, 2, 3, 4, 5]: - expected_new_shape = ( - shape[:new_rank - 1] + [np.prod(shape[new_rank - 1:])]) - expected_flattened = np.reshape(inputs, expected_new_shape) - - flattened_t = _layers._inner_flatten(inputs, new_rank) - static_shape = flattened_t.get_shape().as_list() - self.assertEqual(static_shape, expected_new_shape) - with self.cached_session() as sess: - flattened = sess.run(flattened_t) - np.testing.assert_array_equal(expected_flattened, flattened) - - def testSparsePartialFlatten(self): - """Test `_inner_flatten` on `SparseTensor`s.""" - shape = [4, 3, 11, 6] - np.random.seed(10301) - random_ = np.random.rand(*shape) - indices, values, _ = _sparsify(random_) - - for new_rank in [1, 2, 3]: - expected_shape = (shape[:new_rank - 1] + [np.prod(shape[new_rank - 1:])]) - reshaped_random_ = np.reshape(random_, expected_shape) - expected_indices, expected_values, _ = _sparsify(reshaped_random_) - - inputs_t = sparse_tensor.SparseTensor(indices, values, shape) - - flattened_t = _layers._inner_flatten(inputs_t, new_rank) - - with self.cached_session() as sess: - flattened = sess.run(flattened_t) - - np.testing.assert_array_equal(expected_indices, flattened.indices) - np.testing.assert_array_equal(expected_values, flattened.values) - np.testing.assert_array_equal(expected_shape, flattened.dense_shape) - - def testIncompleteShape(self): - """Test `_inner_flatten` shape inference for incomplete shapes.""" - shape = [2, None, 4, None, 5, 6] - inputs = array_ops.placeholder(dtypes.int32) - inputs.set_shape(shape) - - flattened1 = _layers._inner_flatten(inputs, 1) - self.assertEqual([None], flattened1.get_shape().as_list()) - - flattened2 = _layers._inner_flatten(inputs, 2) - self.assertEqual([2, None], flattened2.get_shape().as_list()) - - flattened3 = _layers._inner_flatten(inputs, 3) - self.assertEqual([2, None, None], flattened3.get_shape().as_list()) - - flattened4 = _layers._inner_flatten(inputs, 4) - self.assertEqual([2, None, 4, None], flattened4.get_shape().as_list()) - - flattened5 = _layers._inner_flatten(inputs, 5) - self.assertEqual([2, None, 4, None, 30], flattened5.get_shape().as_list()) - - def testDenseFlattenRankAssertion(self): - """Test `_inner_flatten` rank assertion for dense tensors.""" - shape = [2, 3] - new_rank = 3 - inputs = array_ops.placeholder(dtypes.int32) - inputs.set_shape(shape) - - with self.assertRaisesRegexp(ValueError, - 'inputs has rank less than new_rank'): - _layers._inner_flatten(inputs, new_rank) - - def testSparseFlattenRankAssertion(self): - """Test `_inner_flatten` rank assertion for sparse tensors.""" - shape = [2, 3] - new_rank = 3 - np.random.seed(10301) - random_ = np.random.rand(*shape) - indices, values, _ = _sparsify(random_) - inputs = sparse_tensor.SparseTensor(indices, values, shape) - - with self.assertRaisesRegexp(ValueError, - 'Inputs has rank less than new_rank'): - _layers._inner_flatten(inputs, new_rank) - - -class FCTest(test.TestCase): - - def testCreateFC(self): - height, width = 3, 3 - for layer_fn in (_layers.fully_connected, layers_lib.relu): - with ops.Graph().as_default() as g, self.session(g): - inputs = np.random.uniform(size=(5, height * width * 3)) - output = layer_fn(inputs, 32) - self.assertEqual(output.op.name, 'fully_connected/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, 32]) - weights = variables.get_variables_by_name('weights')[0] - self.assertListEqual(weights.get_shape().as_list(), [3 * 3 * 3, 32]) - biases = variables.get_variables_by_name('biases')[0] - self.assertListEqual(biases.get_shape().as_list(), [32]) - - def testCreateFCWithScope(self): - height, width = 3, 3 - with self.cached_session(): - inputs = random_ops.random_uniform((5, height * width * 3), seed=1) - output = _layers.fully_connected(inputs, 32, scope='fc1') - self.assertEqual(output.op.name, 'fc1/Relu') - - def testCreateFCWithCollection(self): - height, width = 3, 3 - inputs = random_ops.random_uniform((5, height * width * 3), seed=1) - with ops.name_scope('fe'): - fc = _layers.fully_connected( - inputs, 7, outputs_collections='outputs', scope='fc') - output_collected = ops.get_collection('outputs')[0] - self.assertEqual(output_collected.aliases, ['fc']) - self.assertEqual(output_collected, fc) - - def testCreateFcCreatesWeightsAndBiasesVars(self): - height, width = 3, 3 - inputs = random_ops.random_uniform((5, height * width * 3), seed=1) - with self.cached_session(): - self.assertFalse(variables.get_variables('fc1/weights')) - self.assertFalse(variables.get_variables('fc1/biases')) - _layers.fully_connected(inputs, 32, scope='fc1') - self.assertTrue(variables.get_variables('fc1/weights')) - self.assertTrue(variables.get_variables('fc1/biases')) - - def testReuseVars(self): - height, width = 3, 3 - inputs = random_ops.random_uniform((5, height * width * 3), seed=1) - with self.cached_session(): - _layers.fully_connected(inputs, 32, scope='fc1') - self.assertEqual(len(variables.get_variables('fc1')), 2) - _layers.fully_connected(inputs, 32, scope='fc1', reuse=True) - self.assertEqual(len(variables.get_variables('fc1')), 2) - - def testNonReuseVars(self): - height, width = 3, 3 - inputs = random_ops.random_uniform((5, height * width * 3), seed=1) - with self.cached_session(): - _layers.fully_connected(inputs, 32) - self.assertEqual(len(variables.get_variables('fully_connected')), 2) - _layers.fully_connected(inputs, 32) - self.assertEqual(len(variables.get_variables('fully_connected')), 4) - - def testReuseWithRegularizer(self): - height, width = 3, 3 - regularizer = lambda x: math_ops.reduce_sum(x) * 1e-3 - inputs = random_ops.random_uniform((5, height * width * 3), seed=1) - - _layers.fully_connected( - inputs, 32, scope='fc1', weights_regularizer=regularizer) - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 1) - self.assertEqual(len(losses.get_regularization_losses()), 1) - _layers.fully_connected( - inputs, 32, scope='fc1', weights_regularizer=regularizer, reuse=True) - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 1) - self.assertEqual(len(losses.get_regularization_losses()), 1) - - with variable_scope.variable_scope('outer', reuse=False): - _layers.fully_connected(inputs, 32, weights_regularizer=regularizer) - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 2) - self.assertEqual(len(losses.get_regularization_losses()), 2) - with variable_scope.variable_scope('outer', reuse=True): - _layers.fully_connected(inputs, 32, weights_regularizer=regularizer) - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 2) - self.assertEqual(len(losses.get_regularization_losses()), 2) - - def testCreateFCWithoutActivation(self): - height, width = 3, 3 - with self.cached_session(): - inputs = random_ops.random_uniform((5, height * width * 3), seed=1) - output = _layers.fully_connected(inputs, 32, activation_fn=None) - self.assertEqual(output.op.name, 'fully_connected/BiasAdd') - - def testCreateFCWithWD(self): - height, width = 3, 3 - with self.cached_session() as sess: - inputs = random_ops.random_uniform((5, height * width * 3), seed=1) - weight_decay = regularizers.l2_regularizer(0.01) - _layers.fully_connected(inputs, 32, weights_regularizer=weight_decay) - wd = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)[0] - self.assertEqual(wd.op.name, - 'fully_connected/kernel/Regularizer/l2_regularizer') - sess.run(variables_lib.global_variables_initializer()) - self.assertLess(sess.run(wd), 0.4) - - def testCreateFCWithBD(self): - height, width = 3, 3 - with self.cached_session() as sess: - inputs = random_ops.random_uniform((5, height * width * 3), seed=1) - bias_decay = regularizers.l2_regularizer(0.01) - _layers.fully_connected(inputs, 32, biases_regularizer=bias_decay) - wd = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)[0] - self.assertEqual(wd.op.name, - 'fully_connected/bias/Regularizer/l2_regularizer') - sess.run(variables_lib.global_variables_initializer()) - self.assertLess(sess.run(wd), 0.4) - - def testCreateNoRegularizers(self): - height, width = 3, 3 - with self.cached_session(): - inputs = random_ops.random_uniform((5, height * width * 3), seed=1) - _layers.fully_connected(inputs, 32) - self.assertEqual( - ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES), []) - - def testReuseFCWithWD(self): - height, width = 3, 3 - with self.cached_session(): - inputs = random_ops.random_uniform((5, height * width * 3), seed=1) - weight_decay = regularizers.l2_regularizer(0.01) - _layers.fully_connected( - inputs, 32, weights_regularizer=weight_decay, scope='FC') - self.assertEqual(len(variables.get_variables()), 2) - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 1) - _layers.fully_connected( - inputs, 32, weights_regularizer=weight_decay, scope='FC', reuse=True) - self.assertEqual(len(variables.get_variables()), 2) - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 1) - - def testFCWithBatchNorm(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height * width * 3), seed=1) - with arg_scope( - [_layers.fully_connected], - normalizer_fn=_layers.batch_norm, - normalizer_params={ - 'decay': 0.9 - }): - net = _layers.fully_connected(images, 27) - net = _layers.fully_connected(net, 27) - self.assertEqual(len(variables.get_variables()), 8) - self.assertEqual( - len(variables.get_variables('fully_connected/BatchNorm')), 3) - self.assertEqual( - len(variables.get_variables('fully_connected_1/BatchNorm')), 3) - - def testReuseFCWithBatchNorm(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height * width * 3), seed=1) - with arg_scope( - [_layers.fully_connected], - normalizer_fn=_layers.batch_norm, - normalizer_params={ - 'decay': 0.9 - }): - net = _layers.fully_connected(images, 27, scope='fc1') - net = _layers.fully_connected(net, 27, scope='fc1', reuse=True) - self.assertEqual(len(variables.get_variables()), 4) - self.assertEqual(len(variables.get_variables('fc1/BatchNorm')), 3) - - -class BatchNormTest(test.TestCase): - - def _addBesselsCorrection(self, sample_size, expected_var): - correction_factor = sample_size / (sample_size - 1) - expected_var *= correction_factor - return expected_var, correction_factor - - def testBatchNormCenterFalse(self): - a = array_ops.placeholder(dtype=dtypes.float32, shape=(10, 10, 10, 10)) - # Test that center=False builds a valid graph. - _layers.batch_norm( - a, center=False, data_format='NCHW', zero_debias_moving_mean=True) - - def testUnknownShape(self): - with ops.Graph().as_default() as g, self.session(g): - inputs = array_ops.placeholder(dtype=dtypes.float32) - with self.assertRaisesRegexp(ValueError, 'undefined rank'): - _layers.batch_norm(inputs) - - def testInvalidDataFormat(self): - with ops.Graph().as_default() as g, self.session(g): - inputs = array_ops.placeholder(dtype=dtypes.float32) - with self.assertRaisesRegexp( - ValueError, 'data_format has to be either NCHW or NHWC.'): - _layers.batch_norm(inputs, data_format='CHWN') - - def testUnknownChannelsDimNHWC(self): - with ops.Graph().as_default() as g, self.session(g): - inputs = array_ops.placeholder(dtype=dtypes.float32) - inputs.set_shape(tensor_shape.TensorShape((5, 3, 3, None))) - with self.assertRaisesRegexp(ValueError, 'undefined'): - _layers.batch_norm(inputs, data_format='NHWC') - - def testUnknownChannelsDimNCHW(self): - with ops.Graph().as_default() as g, self.session(g): - inputs = array_ops.placeholder(dtype=dtypes.float32) - inputs.set_shape(tensor_shape.TensorShape((5, None, 3, 3))) - with self.assertRaisesRegexp(ValueError, 'undefined'): - _layers.batch_norm(inputs, data_format='NCHW') - - def _testCreateOp(self, fused, dtype=None): - if dtype is None: - dtype = dtypes.float32 - height, width = 3, 3 - with self.cached_session(): - images = np.random.uniform(size=(5, height, width, 3)).astype( - dtype.as_numpy_dtype) - output = _layers.batch_norm(images, fused=fused) - expected_name = ('BatchNorm/FusedBatchNorm' - if fused else 'BatchNorm/batchnorm') - self.assertTrue(output.op.name.startswith(expected_name)) - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 3]) - self.assertEqual( - ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES), []) - - def testCreateOpDefault(self): - self._testCreateOp(False) - - def testCreateOpFused(self): - self._testCreateOp(True) - - def testCreateOpFusedFloat16(self): - self._testCreateOp(True, dtypes.float16) - - def _testCreateOpBetaRegularizer(self, fused=True): - height, width = 3, 3 - with self.cached_session(): - reg = lambda x: 0.1 * math_ops.reduce_sum(x) - images = np.random.uniform(size=(5, height, width, 3)).astype('f') - _layers.batch_norm(images, param_regularizers={'beta': reg}, fused=fused) - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 1) - beta_decay = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)[0] - self.assertEqual(beta_decay.op.name, 'BatchNorm/beta/Regularizer/mul') - - def testCreateOpBetaRegularizerFused(self): - self._testCreateOpBetaRegularizer(fused=True) - - def testCreateOpBetaRegularizerNonFused(self): - self._testCreateOpBetaRegularizer(fused=False) - - def _testCreateOpGammaRegularizer(self, fused=True): - height, width = 3, 3 - with self.cached_session(): - reg = lambda x: 0.1 * math_ops.reduce_sum(x) - images = np.random.uniform(size=(5, height, width, 3)).astype('f') - _layers.batch_norm( - images, param_regularizers={'gamma': reg}, scale=True, fused=fused) - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 1) - gamma_decay = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)[0] - self.assertEqual(gamma_decay.op.name, 'BatchNorm/gamma/Regularizer/mul') - - def testCreateOpGammaRegularizerFused(self): - self._testCreateOpGammaRegularizer(fused=True) - - def testCreateOpGammaRegularizerNonFused(self): - self._testCreateOpGammaRegularizer(fused=False) - - def testCreateVariables(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - _layers.batch_norm(images, scale=True) - beta = variables.get_variables_by_name('beta')[0] - gamma = variables.get_variables_by_name('gamma')[0] - self.assertEqual(beta.op.name, 'BatchNorm/beta') - self.assertEqual(gamma.op.name, 'BatchNorm/gamma') - moving_mean = variables.get_variables_by_name('moving_mean')[0] - moving_variance = variables.get_variables_by_name('moving_variance')[0] - self.assertEqual(moving_mean.op.name, 'BatchNorm/moving_mean') - self.assertEqual(moving_variance.op.name, 'BatchNorm/moving_variance') - - def testMovingAverageVariables(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - _layers.batch_norm(images, scale=True) - self.assertEqual(len(variables.get_model_variables()), 4) - moving_mean = variables.get_variables_by_name('moving_mean')[0] - moving_variance = variables.get_variables_by_name('moving_variance')[0] - self.assertEqual(moving_mean.op.name, 'BatchNorm/moving_mean') - self.assertEqual(moving_variance.op.name, 'BatchNorm/moving_variance') - - def testMovingAverageVariablesZeroDebias(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - _layers.batch_norm( - images, scale=True, zero_debias_moving_mean=True, fused=False) - self.assertEqual(len(variables.get_model_variables()), 6) - moving_mean = variables.get_variables_by_name('moving_mean')[0] - moving_variance = variables.get_variables_by_name('moving_variance')[0] - biased = variables.get_variables_by_name('biased')[0] - local_step = variables.get_variables_by_name('local_step')[0] - self.assertEqual(moving_mean.op.name, 'BatchNorm/moving_mean') - self.assertEqual(moving_variance.op.name, 'BatchNorm/moving_variance') - self.assertEqual(biased.op.name, 'BatchNorm/BatchNorm/moving_mean/biased') - self.assertEqual(local_step.op.name, - 'BatchNorm/BatchNorm/moving_mean/local_step') - - def testUpdatesCollection(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - _layers.batch_norm(images, updates_collections='my_update_ops') - update_layers = ops.get_collection('my_update_ops') - update_moving_mean = update_layers[0] - update_moving_variance = update_layers[1] - self.assertEqual(update_moving_mean.op.name, 'BatchNorm/AssignMovingAvg') - self.assertEqual(update_moving_variance.op.name, - 'BatchNorm/AssignMovingAvg_1') - - def testVariablesCollections(self): - variables_collections = { - 'beta': ['beta'], - 'gamma': ['gamma'], - 'moving_mean': ['moving_mean'], - 'moving_variance': ['moving_variance'], - } - images = random_ops.random_uniform((5, 5, 5, 3), seed=1) - _layers.batch_norm( - images, scale=True, variables_collections=variables_collections) - for var_name, collection_names in variables_collections.items(): - collection = ops.get_collection(collection_names[0]) - self.assertEqual(len(collection), 1) - var_name_in_collection = collection[0].op.name - self.assertEqual(var_name_in_collection, 'BatchNorm/' + var_name) - - def testReuseVariables(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - _layers.batch_norm(images, scale=True, scope='bn') - _layers.batch_norm(images, scale=True, scope='bn', reuse=True) - beta = variables.get_variables_by_name('beta') - gamma = variables.get_variables_by_name('gamma') - self.assertEqual(len(beta), 1) - self.assertEqual(len(gamma), 1) - moving_mean = variables.get_variables_by_name('moving_mean') - moving_variance = variables.get_variables_by_name('moving_variance') - moving_vars = moving_mean + moving_variance - self.assertEqual(len(moving_vars), 2) - - def testReuseUpdateOps(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - with arg_scope([_layers.batch_norm], updates_collections='update_ops'): - _layers.batch_norm(images, scope='bn') - self.assertEqual(len(ops.get_collection('update_ops')), 2) - _layers.batch_norm(images, scope='bn', reuse=True) - self.assertEqual(len(ops.get_collection('update_ops')), 4) - - def testCreateMovingVars(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - _ = _layers.batch_norm(images) - moving_mean = variables.get_variables('BatchNorm/moving_mean') - self.assertEqual(len(moving_mean), 1) - self.assertEqual(moving_mean[0].op.name, 'BatchNorm/moving_mean') - moving_variance = variables.get_variables('BatchNorm/moving_variance') - self.assertEqual(len(moving_variance), 1) - self.assertEqual(moving_variance[0].op.name, 'BatchNorm/moving_variance') - - def testZeroDebiasMovingMean(self): - height, width = 3, 3 - batch_size = 10 - channels = 3 - np.random.seed(1) - image_shape = (batch_size, height, width, channels) - axis = (0, 1, 2) - image_values = np.random.rand(*image_shape) - expected_mean = np.mean(image_values, axis=axis) - expected_var = np.var(image_values, axis=axis) - - images = constant_op.constant( - image_values, shape=image_shape, dtype=dtypes.float32) - output = _layers.batch_norm( - images, - decay=0.1, - updates_collections=None, - zero_debias_moving_mean=True, - fused=False) - moving_mean = variables.get_variables_by_name('BatchNorm/moving_mean')[0] - moving_variance = variables.get_variables_by_name('moving_variance')[0] - biased = variables.get_variables_by_name('biased')[0] - local_step = variables.get_variables_by_name('local_step')[0] - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - self.assertAllClose(local_step.eval(), 0) - self.assertAllClose(moving_mean.eval(), [0] * channels) - self.assertAllClose(biased.eval(), [0] * channels) - self.assertAllClose(moving_variance.eval(), [1] * channels) - for i in range(10): - self.assertAllClose(local_step.eval(), i) - sess.run([output]) - # In this case moving_mean == expected_mean after each update - self.assertAllClose(moving_mean.eval(), expected_mean) - - # After 10 updates with decay 0.1 moving_mean == expected_mean, - # biased == expected_mean and moving_variance == expected_var. - self.assertAllClose(moving_mean.eval(), expected_mean) - self.assertAllClose(moving_variance.eval(), expected_var) - self.assertAllClose(biased.eval(), expected_mean) - - def _testNoneUpdatesCollections(self, - fused, - data_format='NHWC', - zero_debias_moving_mean=False): - height, width = 2, 2 - batch_size = 10 - channels = 3 - np.random.seed(1) - use_gpu = fused - with self.session(use_gpu=use_gpu) as sess: - if data_format == 'NHWC': - image_shape = (batch_size, height, width, channels) - axis = (0, 1, 2) - else: - image_shape = (batch_size, channels, height, width) - axis = (0, 2, 3) - image_values = np.random.rand(*image_shape) - expected_mean = np.mean(image_values, axis=axis) - expected_var = np.var(image_values, axis=axis) - if fused: - # Add Bessel's correction - expected_var, _ = self._addBesselsCorrection( - batch_size * height * width, expected_var) - images = constant_op.constant( - image_values, shape=image_shape, dtype=dtypes.float32) - output = _layers.batch_norm( - images, - decay=0.1, - updates_collections=None, - fused=fused, - data_format=data_format, - zero_debias_moving_mean=zero_debias_moving_mean) - # updates_ops are not added to UPDATE_OPS collection. - self.assertEqual(ops.get_collection(ops.GraphKeys.UPDATE_OPS), []) - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] - moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] - mean, variance = sess.run([moving_mean, moving_variance]) - # After initialization moving_mean == 0 and moving_variance == 1. - self.assertAllClose(mean, [0] * channels) - self.assertAllClose(variance, [1] * channels) - for _ in range(10): - sess.run([output]) - if zero_debias_moving_mean: - # In this case moving_mean == expected_mean after update - self.assertAllClose(moving_mean.eval(), expected_mean) - mean = moving_mean.eval() - variance = moving_variance.eval() - # After 10 updates with decay 0.1 moving_mean == expected_mean and - # moving_variance == expected_var. - self.assertAllClose(mean, expected_mean) - self.assertAllClose(variance, expected_var) - - def testNoneUpdatesCollectionsNHWC(self): - self._testNoneUpdatesCollections(False, data_format='NHWC') - - def testNoneUpdatesCollectionsNCHW(self): - self._testNoneUpdatesCollections(False, data_format='NCHW') - - def testNoneUpdatesCollectionsNHWCZeroDebias(self): - self._testNoneUpdatesCollections( - False, data_format='NHWC', zero_debias_moving_mean=True) - - def testNoneUpdatesCollectionsNCHWZeroDebias(self): - self._testNoneUpdatesCollections( - False, data_format='NCHW', zero_debias_moving_mean=True) - - def testNoneUpdatesCollectionsFusedNCHW(self): - if test.is_gpu_available(cuda_only=True): - self._testNoneUpdatesCollections(True, data_format='NCHW') - - def testNoneUpdatesCollectionsFusedNHWC(self): - self._testNoneUpdatesCollections(True, data_format='NHWC') - - def testNoneUpdatesCollectionsFusedNCHWZeroDebias(self): - if test.is_gpu_available(cuda_only=True): - self._testNoneUpdatesCollections( - True, data_format='NCHW', zero_debias_moving_mean=True) - - def testNoneUpdatesCollectionsFusedNHWCZeroDebias(self): - self._testNoneUpdatesCollections( - True, data_format='NHWC', zero_debias_moving_mean=True) - - def _testDelayedUpdateMovingVars(self, - fused, - data_format='NHWC', - zero_debias_moving_mean=False): - height, width = 2, 2 - batch_size = 10 - channels = 3 - np.random.seed(1) - use_gpu = fused - with self.session(use_gpu=use_gpu) as sess: - if data_format == 'NHWC': - image_shape = (batch_size, height, width, channels) - axis = (0, 1, 2) - else: - image_shape = (batch_size, channels, height, width) - axis = (0, 2, 3) - image_values = np.random.rand(*image_shape) - expected_mean = np.mean(image_values, axis=axis) - expected_var = np.var(image_values, axis=axis) - if fused: - # Add Bessel's correction - expected_var, correction_factor = self._addBesselsCorrection( - batch_size * height * width, expected_var) - images = constant_op.constant( - image_values, shape=image_shape, dtype=dtypes.float32) - output = _layers.batch_norm( - images, - decay=0.1, - fused=fused, - data_format=data_format, - zero_debias_moving_mean=zero_debias_moving_mean) - update_ops = ops.get_collection(ops.GraphKeys.UPDATE_OPS) - # updates_ops are added to UPDATE_OPS collection. - self.assertEqual(len(update_ops), 2) - with ops.control_dependencies(update_ops): - barrier = control_flow_ops.no_op(name='barrier') - output = control_flow_ops.with_dependencies([barrier], output) - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] - moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] - mean, variance = sess.run([moving_mean, moving_variance]) - # After initialization moving_mean == 0 and moving_variance == 1. - self.assertAllClose(mean, [0] * channels) - self.assertAllClose(variance, [1] * channels) - for _ in range(10): - sess.run([output]) - if zero_debias_moving_mean: - # In this case moving_mean == expected_mean after update - self.assertAllClose(moving_mean.eval(), expected_mean) - - mean = moving_mean.eval() - variance = moving_variance.eval() - # After 10 updates with decay 0.1 moving_mean == expected_mean and - # moving_variance == expected_var. - self.assertAllClose(mean, expected_mean) - if fused: - # Add Bessel's correction - moving_variance_corrected = moving_variance / correction_factor - correct_moving_variance = state_ops.assign(moving_variance, - moving_variance_corrected) - sess.run(correct_moving_variance) - self.assertAllClose(variance, expected_var) - - def testDelayedUpdateMovingVarsNHWC(self): - self._testDelayedUpdateMovingVars(False, data_format='NHWC') - - def testDelayedUpdateMovingVarsNCHW(self): - self._testDelayedUpdateMovingVars(False, data_format='NCHW') - - def testDelayedUpdateMovingVarsFusedNCHW(self): - if test.is_gpu_available(cuda_only=True): - self._testDelayedUpdateMovingVars(True, data_format='NCHW') - - def testDelayedUpdateMovingVarsFusedNHWC(self): - self._testDelayedUpdateMovingVars(True, data_format='NHWC') - - def testDelayedUpdateMovingVars(self): - self._testDelayedUpdateMovingVars(False) - - def _testEvalMovingVars(self, zero_debias_moving_mean=False): - height, width = 3, 3 - with self.cached_session() as sess: - image_shape = (10, height, width, 3) - image_values = np.random.rand(*image_shape) - expected_mean = np.mean(image_values, axis=(0, 1, 2)) - expected_var = np.var(image_values, axis=(0, 1, 2)) - images = constant_op.constant( - image_values, shape=image_shape, dtype=dtypes.float32) - output = _layers.batch_norm(images, decay=0.1, is_training=False) - self.assertEqual(ops.get_collection(ops.GraphKeys.UPDATE_OPS), []) - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] - moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] - mean, variance = sess.run([moving_mean, moving_variance]) - # After initialization moving_mean == 0 and moving_variance == 1. - self.assertAllClose(mean, [0] * 3) - self.assertAllClose(variance, [1] * 3) - # Simulate assignment from saver restore. - init_assigns = [ - state_ops.assign(moving_mean, expected_mean), - state_ops.assign(moving_variance, expected_var) - ] - sess.run(init_assigns) - for _ in range(10): - sess.run([output], {images: np.random.rand(*image_shape)}) - mean = moving_mean.eval() - variance = moving_variance.eval() - # Although we feed different images, the moving_mean and moving_variance - # shouldn't change. - self.assertAllClose(mean, expected_mean) - self.assertAllClose(variance, expected_var) - - def testEvalMovingVars(self): - self._testEvalMovingVars() - - def testEvalMovingVarsZeroDebias(self): - self._testEvalMovingVars(True) - - def testEvalMovingVarsWithPartitioner(self): - # This test makes sure that the moving-mean and moving-variance logic works - # when `batch_norm` is called within a variable-scope that has a variable - # partitioner. - partitioner = partitioned_variables.fixed_size_partitioner(2, axis=0) - with variable_scope.variable_scope( - variable_scope.get_variable_scope(), partitioner=partitioner): - self.testEvalMovingVars() - - def _testReuseVars(self, fused, zero_debias_moving_mean=False): - height, width = 3, 3 - batch_size = 10 - channels = 3 - with self.cached_session() as sess: - image_shape = (batch_size, height, width, channels) - image_values = np.random.rand(*image_shape) - expected_mean = np.mean(image_values, axis=(0, 1, 2)) - expected_var = np.var(image_values, axis=(0, 1, 2)) - if fused: - # Add Bessel's correction - expected_var, correction_factor = self._addBesselsCorrection( - batch_size * height * width, expected_var) - images = constant_op.constant( - image_values, shape=image_shape, dtype=dtypes.float32) - output_train = _layers.batch_norm( - images, - decay=0.1, - is_training=True, - scope='BN', - fused=fused, - zero_debias_moving_mean=zero_debias_moving_mean) - output_eval = _layers.batch_norm( - images, - decay=0.1, - is_training=False, - scope='BN', - reuse=True, - fused=fused, - zero_debias_moving_mean=zero_debias_moving_mean) - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - moving_mean = variables.get_variables('BN/moving_mean')[0] - moving_variance = variables.get_variables('BN/moving_variance')[0] - mean, variance = sess.run([moving_mean, moving_variance]) - # After initialization moving_mean == 0 and moving_variance == 1. - self.assertAllClose(mean, [0] * channels) - self.assertAllClose(variance, [1] * channels) - update_ops = ops.get_collection(ops.GraphKeys.UPDATE_OPS) - with ops.control_dependencies(update_ops): - barrier = control_flow_ops.no_op(name='barrier') - train_op = control_flow_ops.with_dependencies([barrier], output_train) - # Before updates the outputs are different for train and eval. - self.assertFalse( - np.allclose(sess.run([output_train]), sess.run([output_eval]))) - for _ in range(10): - sess.run([train_op]) - mean = moving_mean.eval() - variance = moving_variance.eval() - # After 10 updates with decay 0.1 moving_mean == expected_mean and - # moving_variance == expected_var. - self.assertAllClose(mean, expected_mean) - if fused: - # Add Bessel's correction - moving_variance_corrected = moving_variance / correction_factor - correct_moving_variance = state_ops.assign(moving_variance, - moving_variance_corrected) - sess.run(correct_moving_variance) - self.assertAllClose(variance, expected_var) - # After convergence output_train and output_eval should be the same. - self.assertAllClose(sess.run([output_train]), sess.run([output_eval])) - - def testReuseVarsDefault(self): - self._testReuseVars(False) - - def testReuseVarsFused(self): - self._testReuseVars(True) - - def testReuseVarsDefaultZeroDebias(self): - self._testReuseVars(False, True) - - def testReuseVarsFusedZeroDebias(self): - self._testReuseVars(True, True) - - def _testIsTrainingVariable(self, - fused, - data_format='NHWC', - zero_debias_moving_mean=False): - height, width = 2, 2 - batch_size = 10 - channels = 3 - np.random.seed(1) - use_gpu = fused - np.random.seed(1) - with self.session(use_gpu=use_gpu) as sess: - if data_format == 'NHWC': - image_shape = (batch_size, height, width, channels) - axis = (0, 1, 2) - else: - image_shape = (batch_size, channels, height, width) - axis = (0, 2, 3) - image_values = np.random.rand(*image_shape) - expected_mean = np.mean(image_values, axis=axis) - expected_var = np.var(image_values, axis=axis) - if fused: - # Add Bessel's correction - expected_var, correction_factor = self._addBesselsCorrection( - batch_size * height * width, expected_var) - images = constant_op.constant( - image_values, shape=image_shape, dtype=dtypes.float32) - is_training = variables_lib.VariableV1(True) - output = _layers.batch_norm( - images, - decay=0.1, - is_training=is_training, - fused=fused, - data_format=data_format, - zero_debias_moving_mean=zero_debias_moving_mean) - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] - moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] - mean, variance = sess.run([moving_mean, moving_variance]) - # After initialization moving_mean == 0 and moving_variance == 1. - self.assertAllClose(mean, [0] * channels) - self.assertAllClose(variance, [1] * channels) - # Before updates the outputs are different depending of is_training. - output_true = sess.run([output], {is_training: True}) - output_false = sess.run([output], {is_training: False}) - self.assertFalse(np.allclose(output_true, output_false)) - update_ops = ops.get_collection(ops.GraphKeys.UPDATE_OPS) - with ops.control_dependencies(update_ops): - barrier = control_flow_ops.no_op(name='barrier') - train_op = control_flow_ops.with_dependencies([barrier], output) - for _ in range(10): - sess.run([train_op]) - mean = moving_mean.eval() - variance = moving_variance.eval() - # After 10 updates with decay 0.1 moving_mean == expected_mean and - # moving_variance == expected_var. - self.assertAllClose(mean, expected_mean) - self.assertAllClose(variance, expected_var) - # After updates to convergence the outputs don't depend on is_training. - output_true = sess.run([output], {is_training: True}) - if fused: - # Add Bessel's correction - moving_variance_corrected = moving_variance / correction_factor - correct_moving_variance = state_ops.assign(moving_variance, - moving_variance_corrected) - sess.run(correct_moving_variance) - output_false = sess.run([output], {is_training: False}) - self.assertAllClose(output_true, output_false) - - def testIsTrainingVariableNHWC(self): - self._testIsTrainingVariable(False, data_format='NHWC') - - def testIsTrainingVariableNCHW(self): - self._testIsTrainingVariable(False, data_format='NCHW') - - def testIsTrainingVariableNHWCZeroDebias(self): - self._testIsTrainingVariable( - False, data_format='NHWC', zero_debias_moving_mean=True) - - def testIsTrainingVariableNCHWZeroDebias(self): - self._testIsTrainingVariable( - False, data_format='NCHW', zero_debias_moving_mean=True) - - def testIsTrainingVariableFusedNCHW(self): - if test.is_gpu_available(cuda_only=True): - self._testIsTrainingVariable(True, data_format='NCHW') - - def testIsTrainingVariableFusedNHWC(self): - self._testIsTrainingVariable(True, data_format='NHWC') - - def testIsTrainingVariableFusedNCHWZeroDebias(self): - if test.is_gpu_available(cuda_only=True): - self._testIsTrainingVariable( - True, data_format='NCHW', zero_debias_moving_mean=True) - - def testIsTrainingVariableFusedNHWCZeroDebias(self): - self._testIsTrainingVariable( - True, data_format='NHWC', zero_debias_moving_mean=True) - - def testNoUpdatesWhenIsTrainingFalse(self): - height, width = 3, 3 - with self.cached_session() as sess: - image_shape = (10, height, width, 3) - image_values = np.random.rand(*image_shape) - images = constant_op.constant( - image_values, shape=image_shape, dtype=dtypes.float32) - output = _layers.batch_norm(images, decay=0.1, is_training=False) - update_ops = ops.get_collection(ops.GraphKeys.UPDATE_OPS) - # updates_ops are not added to UPDATE_OPS collection. - self.assertEqual(len(update_ops), 0) - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] - moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] - mean, variance = sess.run([moving_mean, moving_variance]) - # After initialization moving_mean == 0 and moving_variance == 1. - self.assertAllClose(mean, [0] * 3) - self.assertAllClose(variance, [1] * 3) - # When is_training is False batch_norm doesn't update moving_vars. - for _ in range(10): - sess.run([output]) - self.assertAllClose(moving_mean.eval(), [0] * 3) - self.assertAllClose(moving_variance.eval(), [1] * 3) - - def testNoneUpdatesCollectionNoTraining(self): - height, width = 3, 3 - with self.cached_session() as sess: - image_shape = (10, height, width, 3) - image_values = np.random.rand(*image_shape) - images = constant_op.constant( - image_values, shape=image_shape, dtype=dtypes.float32) - output = _layers.batch_norm( - images, decay=0.1, updates_collections=None, is_training=False) - # updates_ops are not added to UPDATE_OPS collection. - self.assertEqual(ops.get_collection(ops.GraphKeys.UPDATE_OPS), []) - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] - moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] - mean, variance = sess.run([moving_mean, moving_variance]) - # After initialization moving_mean == 0 and moving_variance == 1. - self.assertAllClose(mean, [0] * 3) - self.assertAllClose(variance, [1] * 3) - # When is_training is False batch_norm doesn't update moving_vars. - for _ in range(10): - sess.run([output]) - self.assertAllClose(moving_mean.eval(), [0] * 3) - self.assertAllClose(moving_variance.eval(), [1] * 3) - - def _testNoneUpdatesCollectionIsTrainingVariable(self, - fused, - data_format='NHWC'): - height, width = 2, 2 - batch_size = 10 - channels = 3 - np.random.seed(1) - use_gpu = fused - with self.session(use_gpu=use_gpu) as sess: - if data_format == 'NHWC': - image_shape = (batch_size, height, width, channels) - axis = (0, 1, 2) - else: - image_shape = (batch_size, channels, height, width) - axis = (0, 2, 3) - image_values = np.random.rand(*image_shape) - expected_mean = np.mean(image_values, axis=axis) - expected_var = np.var(image_values, axis=axis) - if fused: - # Add Bessel's correction - expected_var, correction_factor = self._addBesselsCorrection( - batch_size * height * width, expected_var) - images = constant_op.constant( - image_values, shape=image_shape, dtype=dtypes.float32) - is_training = variables_lib.VariableV1(True) - output = _layers.batch_norm( - images, - decay=0.1, - updates_collections=None, - is_training=is_training, - fused=fused, - data_format=data_format) - # updates_ops are not added to UPDATE_OPS collection. - self.assertEqual(ops.get_collection(ops.GraphKeys.UPDATE_OPS), []) - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] - moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] - mean, variance = sess.run([moving_mean, moving_variance]) - # After initialization moving_mean == 0 and moving_variance == 1. - self.assertAllClose(mean, [0] * channels) - self.assertAllClose(variance, [1] * channels) - # When is_training is False batch_norm doesn't update moving_vars. - for _ in range(10): - sess.run([output], {is_training: False}) - self.assertAllClose(moving_mean.eval(), [0] * channels) - self.assertAllClose(moving_variance.eval(), [1] * channels) - # Before updates the outputs are different depending of is_training. - output_true = sess.run([output], {is_training: True}) - output_false = sess.run([output], {is_training: False}) - self.assertFalse(np.allclose(output_true, output_false)) - # When is_training is True update moving_vars. - for _ in range(10): - sess.run([output], {is_training: True}) - # After 10 updates with decay 0.1 moving_mean == expected_mean and - # moving_variance == expected_var. - self.assertAllClose(moving_mean.eval(), expected_mean) - self.assertAllClose(moving_variance.eval(), expected_var) - # After updates to convergence the outputs don't depend on is_training. - output_true = sess.run([output], {is_training: True}) - if fused: - # Add Bessel's correction - moving_variance_corrected = moving_variance / correction_factor - correct_moving_variance = state_ops.assign(moving_variance, - moving_variance_corrected) - sess.run(correct_moving_variance) - output_false = sess.run([output], {is_training: False}) - self.assertTrue(np.allclose(output_true, output_false)) - - def testNoneUpdatesCollectionIsTrainingVariableNHWC(self): - self._testNoneUpdatesCollectionIsTrainingVariable(False, data_format='NHWC') - - def testNoneUpdatesCollectionIsTrainingVariableNCHW(self): - self._testNoneUpdatesCollectionIsTrainingVariable(False, data_format='NCHW') - - def testNoneUpdatesCollectionIsTrainingVariableFusedNCHW(self): - if test.is_gpu_available(cuda_only=True): - self._testNoneUpdatesCollectionIsTrainingVariable( - True, data_format='NCHW') - - def testNoneUpdatesCollectionIsTrainingVariableFusedNHWC(self): - self._testNoneUpdatesCollectionIsTrainingVariable(True, data_format='NHWC') - - def _testTrainMovingVars(self, fused, data_format='NHWC'): - # Test that the gradients are stable while the moving_mean is updated. - # Since the moving_mean is used as shift to compute the tf.momments, the - # gradients could diverge, this test checks that gradients remains stable - # while the moving_mean is updated. - height, width = 7, 7 - batch_size = 10 - channels = 32 - np.random.seed(1) - use_gpu = fused - with self.session(use_gpu=use_gpu) as sess: - if data_format == 'NHWC': - image_shape = (batch_size, height, width, channels) - axis = (0, 1, 2) - else: - image_shape = (batch_size, channels, height, width) - axis = (0, 2, 3) - image_values = np.random.rand(*image_shape) + 256 - expected_mean = np.mean(image_values, axis=axis) - expected_var = np.var(image_values, axis=axis) - if fused: - # Add Bessel's correction - expected_var, _ = self._addBesselsCorrection( - batch_size * height * width, expected_var) - images = constant_op.constant( - image_values, shape=image_shape, dtype=dtypes.float32) - output = _layers.batch_norm( - images, - decay=0.2, - updates_collections=None, - is_training=True, - fused=fused, - data_format=data_format) - self.assertEqual(ops.get_collection(ops.GraphKeys.UPDATE_OPS), []) - - objective = math_ops.reduce_sum(output) - - [images_gradients] = gradients_impl.gradients(objective, images) - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] - moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] - mean, variance = sess.run([moving_mean, moving_variance]) - # After initialization moving_mean == 0 and moving_variance == 1. - self.assertAllClose(mean, [0] * channels) - self.assertAllClose(variance, [1] * channels) - - # Initial input gradients. - images_gradients_value = sess.run(images_gradients) - for _ in range(10): - np_output, new_images_gradients = sess.run([output, images_gradients]) - # The outputs should be close to 0.0 mean and 1.0 variance - self.assertAllClose( - np.mean(np_output, axis=axis), [0] * channels, - rtol=0.001, - atol=0.001) - self.assertAllClose( - np.var(np_output, axis=axis), [1] * channels, rtol=0.01, atol=0.01) - # The gradients should change slowly while updating moving_mean. - max_diff = np.max(np.abs(images_gradients_value - new_images_gradients)) - self.assertGreaterEqual(max_diff, 0.0) - self.assertLess(max_diff, 5e-5) - self.assertAllClose(moving_mean.eval(), expected_mean) - self.assertAllClose(moving_variance.eval(), expected_var) - - def testTrainMovingVarsNHWC(self): - self._testTrainMovingVars(False, data_format='NHWC') - - def testTrainMovingVarsNCHW(self): - self._testTrainMovingVars(False, data_format='NCHW') - - def testTrainMovingVarsFusedNCHW(self): - if test.is_gpu_available(cuda_only=True): - self._testTrainMovingVars(True, data_format='NCHW') - - def testTrainMovingVarsFusedNHWC(self): - self._testTrainMovingVars(True, data_format='NHWC') - - def testCustomInitializer(self): - height, width = 3, 3 - channels = 3 - with self.cached_session() as sess: - images = (np.ones((5, height, width, channels)) * 9.0).astype('f') - beta = init_ops.constant_initializer( - (np.ones(channels) * 5.0).astype('f')) - gamma = init_ops.constant_initializer( - (np.ones(channels) * 2.0).astype('f')) - mean = init_ops.constant_initializer( - (np.ones(channels) * 5.0).astype('f')) - variance = init_ops.constant_initializer( - (np.ones(channels) * 4.0).astype('f')) - output = _layers.batch_norm( - images, - is_training=False, - scale=True, - epsilon=0.0, - param_initializers={ - 'beta': beta, - 'gamma': gamma, - 'moving_mean': mean, - 'moving_variance': variance, - }) - sess.run(variables_lib.global_variables_initializer()) - outs = sess.run(output) - self.assertAllClose(outs, images) - - def _runBatchNormalizationWithFormat(self, shape, data_format, is_training): - channels = shape[-1] - with self.session(use_gpu=True) as sess: - images = np.arange(np.product(shape), dtype=np.float32).reshape(shape) - beta = init_ops.constant_initializer( - np.arange(2, channels + 2, dtype=np.float32)) - gamma = init_ops.constant_initializer( - np.arange(10, channels + 10, dtype=np.float32) * 2.0) - mean = init_ops.constant_initializer( - np.arange(3, channels + 3, dtype=np.float32) * 5.0) - variance = init_ops.constant_initializer( - np.arange(1, channels + 1, dtype=np.float32) * 4.0) - if data_format == 'NCHW': - # Reshape inputs from NHWC to NCHW format. - images = array_ops.transpose( - images, [0, len(shape) - 1] + list(range(1, - len(shape) - 1))) - output = _layers.batch_norm( - images, - is_training=is_training, - scale=True, - epsilon=0.5, - param_initializers={ - 'beta': beta, - 'gamma': gamma, - 'moving_mean': mean, - 'moving_variance': variance, - }, - data_format=data_format) - if data_format == 'NCHW': - # Reshape outputs from NCHW back to NHWC format. - output = array_ops.transpose(output, - [0] + list(range(2, len(shape))) + [1]) - sess.run(variables_lib.global_variables_initializer()) - return sess.run(output) - - def testNHWCAndNCHWInferenceProduceSameOutput(self): - if test.is_gpu_available(cuda_only=True): - for shape in [[7, 3, 5], [5, 2, 3, 4], [11, 3, 2, 4, 5]]: - nhwc = self._runBatchNormalizationWithFormat( - data_format='NHWC', shape=shape, is_training=False) - nchw = self._runBatchNormalizationWithFormat( - data_format='NCHW', shape=shape, is_training=False) - self.assertAllClose(nhwc, nchw, atol=1e-4, rtol=1e-4) - - def testNHWCAndNCHWTrainingProduceSameOutput(self): - if test.is_gpu_available(cuda_only=True): - for shape in [[7, 3, 5], [5, 2, 3, 4], [11, 3, 2, 4, 5]]: - nhwc = self._runBatchNormalizationWithFormat( - data_format='NHWC', shape=shape, is_training=True) - nchw = self._runBatchNormalizationWithFormat( - data_format='NCHW', shape=shape, is_training=True) - self.assertAllClose(nhwc, nchw, atol=1e-4, rtol=1e-4) - - def testBatchNormBeta(self): - # Test case for 11673 - with self.cached_session() as sess: - a_32 = array_ops.placeholder(dtypes.float32, shape=(10, 10, 10, 10)) - _layers.batch_norm( - a_32, center=False, data_format='NCHW', zero_debias_moving_mean=True) - a_16 = array_ops.placeholder(dtypes.float16, shape=(10, 10, 10, 10)) - _layers.batch_norm( - a_16, center=False, data_format='NCHW', zero_debias_moving_mean=True) - sess.run(variables_lib.global_variables_initializer()) - - def testVariablesAreFloat32(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform( - (5, height, width, 3), seed=1, dtype=dtypes.float16) - _layers.batch_norm(images, scale=True) - beta = variables.get_variables_by_name('beta')[0] - gamma = variables.get_variables_by_name('gamma')[0] - self.assertEqual(beta.dtype, dtypes.float32_ref) - self.assertEqual(gamma.dtype, dtypes.float32_ref) - moving_mean = variables.get_variables_by_name('moving_mean')[0] - moving_variance = variables.get_variables_by_name('moving_variance')[0] - self.assertEqual(moving_mean.dtype, dtypes.float32_ref) - self.assertEqual(moving_variance.dtype, dtypes.float32_ref) - - def _runFusedBatchNorm(self, shape, dtype): - channels = shape[1] - images = np.arange(np.product(shape), dtype=dtype).reshape(shape) - beta = init_ops.constant_initializer( - np.arange(2, channels + 2, dtype=np.float32)) - gamma = init_ops.constant_initializer( - np.arange(10, channels + 10, dtype=np.float32) * 2.0) - mean = init_ops.constant_initializer( - np.arange(3, channels + 3, dtype=np.float32) * 5.0) - variance = init_ops.constant_initializer( - np.arange(1, channels + 1, dtype=np.float32) * 4.0) - output = _layers.batch_norm( - images, - fused=True, - is_training=True, - scale=True, - epsilon=0.5, - param_initializers={ - 'beta': beta, - 'gamma': gamma, - 'moving_mean': mean, - 'moving_variance': variance, - }, - data_format='NCHW') - with self.session(use_gpu=True) as sess: - sess.run(variables_lib.global_variables_initializer()) - return sess.run(output) - - def testFusedBatchNormFloat16MatchesFloat32(self): - if test.is_gpu_available(cuda_only=True): - shape = [5, 4, 2, 3] - res_32 = self._runFusedBatchNorm(shape, np.float32) - res_16 = self._runFusedBatchNorm(shape, np.float16) - self.assertAllClose(res_32, res_16, rtol=1e-3) - - def testAdjustmentCreated(self): - # Tests that the adjustment is appropriately passed to and used by the core - # BN layer. - all_adjustments = [] - - def _create_adjustment(shape): - adjustments = [array_ops.ones(shape[-1:]), array_ops.zeros(shape[-1:])] - all_adjustments.extend(adjustments) - return adjustments - - depth = 8 - images = array_ops.zeros([10, 5, 5, depth]) - output = _layers.batch_norm( - images, is_training=True, adjustment=_create_adjustment) - self.assertListEqual(output.shape.as_list(), images.shape.as_list()) - self.assertEqual(len(all_adjustments), 2) - self.assertListEqual(all_adjustments[0].shape.as_list(), [depth]) - self.assertListEqual(all_adjustments[1].shape.as_list(), [depth]) - - -class LayerNormTest(test.TestCase): - - def testUnknownShape(self): - with ops.Graph().as_default() as g, self.session(g): - inputs = array_ops.placeholder(dtype=dtypes.float32) - with self.assertRaisesRegexp(ValueError, 'undefined rank'): - _layers.layer_norm(inputs) - - def testParamsDimsNotFullyDefined(self): - with ops.Graph().as_default() as g, self.session(g): - inputs = array_ops.placeholder(dtype=dtypes.float32) - inputs.set_shape(tensor_shape.TensorShape((5, 3, 3, None))) - with self.assertRaisesRegexp(ValueError, 'is not fully defined'): - _layers.layer_norm(inputs) - - def testCreateOp(self): - height, width = 3, 3 - with self.cached_session(): - images = np.random.uniform(size=(5, height, width, 3)) - output = _layers.layer_norm(images) - self.assertTrue(output.op.name.startswith('LayerNorm/batchnorm')) - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 3]) - - def testCreateVariables(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - _layers.layer_norm(images) - beta = variables.get_variables_by_name('beta')[0] - gamma = variables.get_variables_by_name('gamma')[0] - self.assertEqual(beta.op.name, 'LayerNorm/beta') - self.assertEqual(gamma.op.name, 'LayerNorm/gamma') - - def testReuseVariables(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - _layers.layer_norm(images, scope='ln') - _layers.layer_norm(images, scope='ln', reuse=True) - beta = variables.get_variables_by_name('beta') - gamma = variables.get_variables_by_name('gamma') - self.assertEqual(len(beta), 1) - self.assertEqual(len(gamma), 1) - - def testReuseVars(self): - height, width = 3, 3 - with self.cached_session() as sess: - image_shape = (10, height, width, 3) - image_values = np.random.rand(*image_shape) - images = constant_op.constant( - image_values, shape=image_shape, dtype=dtypes.float32) - output_train = _layers.layer_norm(images, scope='LN') - output_eval = _layers.layer_norm(images, scope='LN', reuse=True) - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - # output_train and output_eval should be the same. - self.assertAllClose(sess.run([output_train]), sess.run([output_eval])) - - def doOutputTest(self, - input_shape, - tol=1e-5, - begin_norm_axis=1, - dtype=dtypes.float64): - eps = 1e-12 if dtype != dtypes.float16 else 1e-3 - expected_mean = np.zeros(input_shape[:begin_norm_axis]) - expected_var_uncorrected = np.ones(input_shape[:begin_norm_axis]) - sigma_list = [1.0, 0.1] - if dtype == dtypes.float16: - # This causes the variance to underflow in float16, and requires that - # variance_epsilon be set appropriately to avoid NaNs in the output. - sigma_list.append(1e-4) - # Note that the mean:variance ratio must be limited to the representable - # range for float16. - for mu in [0.0, 1e2 if dtype != dtypes.float16 else 1e1]: - for sigma in sigma_list: - expected_var = expected_var_uncorrected / (1.0 + eps / sigma**2) - input_values = np.random.randn(*input_shape) * sigma + mu - with ops.Graph().as_default() as g: - with self.session(graph=g) as sess: - inputs = constant_op.constant( - input_values, shape=input_shape, dtype=dtype) - output_t = _layers.layer_norm( - inputs, begin_norm_axis=begin_norm_axis, scope='LN') - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - # The mean and variance of the output should be close to 0 and 1 - # respectively. - if begin_norm_axis < 0: - begin_norm_axis = len(input_shape) + begin_norm_axis - moments_axis = tuple(range(begin_norm_axis, len(input_shape))) - with variable_scope.variable_scope('LN', reuse=True): - beta_var = variable_scope.get_variable('beta', dtype=dtype) - gamma_var = variable_scope.get_variable('gamma', dtype=dtype) - outputs, beta, gamma = sess.run((output_t, beta_var, gamma_var)) - # Make sure that there are no NaNs - self.assertFalse(np.isnan(outputs).any()) - if outputs.dtype != np.float64: - # Cast to float64 before computing mean/variance to avoid - # overflow and precision issues. - outputs = outputs.astype(np.float64) - mean = np.mean(outputs, axis=moments_axis) - var = np.var(outputs, axis=moments_axis) - # Layer-norm implemented in numpy - expected_out = ( - (gamma * (input_values - np.mean( - input_values, axis=moments_axis, keepdims=True)) / - np.sqrt(eps + np.var( - input_values, axis=moments_axis, keepdims=True))) + beta) - self.assertAllClose(expected_mean, mean, atol=tol, rtol=tol) - self.assertAllClose(expected_var, var, atol=tol) - # The full computation gets a bigger tolerance - self.assertAllClose(expected_out, outputs, atol=5 * tol) - - def testOutput2DInput(self): - self.doOutputTest((10, 300)) - - def testOutput2DInputDegenerateNormAxis(self): - with self.assertRaisesRegexp(ValueError, r'must be < rank\(inputs\)'): - self.doOutputTest((10, 300), begin_norm_axis=2) - - def testOutput4DInput(self): - self.doOutputTest((100, 10, 10, 3)) - - def testOutput4DInputNormOnInnermostAxis(self): - # Equivalent tests - self.doOutputTest( - (100, 10, 10, 3), begin_norm_axis=3, tol=1e-4, dtype=dtypes.float64) - self.doOutputTest( - (100, 10, 10, 3), begin_norm_axis=-1, tol=1e-4, dtype=dtypes.float64) - - def testOutputSmallInput(self): - self.doOutputTest((10, 10, 10, 30)) - - def testOutputSmallInputNormOnInnermostAxis(self): - self.doOutputTest((10, 10, 10, 30), begin_norm_axis=3) - - def testOutputBigInput(self): - self.doOutputTest((1, 100, 100, 1)) - - def testOutputBigInputFloat32(self): - self.doOutputTest((1, 100, 1000, 1), tol=1e-4, dtype=dtypes.float32) - - def testOutputBigInputFloat16(self): - self.doOutputTest((1, 100, 1000, 1), tol=5e-2, dtype=dtypes.float16) - - -class GDNTest(test.TestCase): - - def _runGDN(self, x, shape, inverse, data_format): - inputs = array_ops.placeholder(dtypes.float32, shape) - outputs = _layers.gdn(inputs, inverse=inverse, data_format=data_format) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - y, = sess.run([outputs], {inputs: x}) - return y - - def testInvalidDataFormat(self): - x = np.random.uniform(size=(1, 2, 3, 4)) - with self.assertRaises(ValueError): - self._runGDN(x, x.shape, False, 'NHWC') - - def testUnknownDim(self): - x = np.random.uniform(size=(1, 2, 3, 4)) - with self.assertRaises(ValueError): - self._runGDN(x, 4 * [None], False, 'channels_last') - - def testChannelsLast(self): - for ndim in [3, 4, 5]: - x = np.random.uniform(size=(1, 2, 3, 4)[:ndim]) - y = self._runGDN(x, x.shape, False, 'channels_last') - self.assertEqual(x.shape, y.shape) - self.assertAllClose(y, x / np.sqrt(1 + .1 * (x**2)), rtol=0, atol=1e-6) - - def testChannelsFirst(self): - # `bias_add` doesn't support NCHW on CPU. - if test.is_gpu_available(cuda_only=True): - for ndim in [3, 4, 5]: - x = np.random.uniform(size=(4, 3, 2, 1)[:ndim]) - y = self._runGDN(x, x.shape, False, 'channels_first') - self.assertEqual(x.shape, y.shape) - self.assertAllClose(y, x / np.sqrt(1 + .1 * (x**2)), rtol=0, atol=1e-6) - - def testWrongDims(self): - for ndim in [1, 2, 6]: - x = np.random.uniform(size=(1, 2, 3, 4, 3, 2)[:ndim]) - with self.assertRaises(ValueError): - self._runGDN(x, x.shape, False, 'channels_last') - - def testIGDN(self): - x = np.random.uniform(size=(1, 2, 3, 4)) - y = self._runGDN(x, x.shape, True, 'channels_last') - self.assertEqual(x.shape, y.shape) - self.assertAllClose(y, x * np.sqrt(1 + .1 * (x**2)), rtol=0, atol=1e-6) - - -class ImagesToSequenceTest(test.TestCase): - - def testInvalidDataFormat(self): - height, width = 7, 11 - images = np.random.uniform(size=(5, height, width, 2)) - with self.assertRaisesRegexp(ValueError, - 'data_format has to be either NCHW or NHWC.'): - _layers.images_to_sequence(images, data_format='CHWN') - - def testImagesToSequenceDims(self): - height, width = 7, 11 - images = np.random.uniform(size=(2, height, width, 5)).astype(np.float32) - output = _layers.images_to_sequence(images) - self.assertListEqual(output.get_shape().as_list(), [11, 14, 5]) - - def testImagesToSequenceNCHW(self): - height, width = 7, 11 - images = np.random.uniform(size=(2, 5, height, width)).astype(np.float32) - output = _layers.images_to_sequence(images, data_format='NCHW') - self.assertListEqual(output.get_shape().as_list(), [11, 14, 5]) - - -class MaxPool2DTest(test.TestCase): - - def testInvalidDataFormat(self): - height, width = 3, 6 - images = np.random.uniform(size=(5, height, width, 3)) - with self.assertRaisesRegexp(ValueError, - 'data_format has to be either NCHW or NHWC.'): - _layers.max_pool2d(images, [3, 3], data_format='CHWN') - - def testCreateMaxPool(self): - height, width = 3, 6 - images = np.random.uniform(size=(5, height, width, 3)).astype(np.float32) - output = _layers.max_pool2d(images, [3, 3]) - self.assertEqual(output.op.name, 'MaxPool2D/MaxPool') - self.assertListEqual(output.get_shape().as_list(), [5, 1, 2, 3]) - - def testCreateMaxPoolNCHW(self): - height, width = 3, 6 - images = np.random.uniform(size=(5, 3, height, width)).astype(np.float32) - output = _layers.max_pool2d(images, [3, 3], data_format='NCHW') - self.assertListEqual(output.get_shape().as_list(), [5, 3, 1, 2]) - - def testCollectOutputs(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.max_pool2d(images, [3, 3], outputs_collections='outputs') - output_collected = ops.get_collection('outputs')[0] - self.assertEqual(output_collected.aliases, ['MaxPool2D']) - self.assertEqual(output_collected, output) - - def testCreateSquareMaxPool(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.max_pool2d(images, 3) - self.assertEqual(output.op.name, 'MaxPool2D/MaxPool') - self.assertListEqual(output.get_shape().as_list(), [5, 1, 2, 3]) - - def testCreateMaxPoolWithScope(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.max_pool2d(images, [3, 3], scope='pool1') - self.assertEqual(output.op.name, 'pool1/MaxPool') - - def testCreateMaxPoolWithSamePadding(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.max_pool2d(images, [3, 3], padding='SAME') - self.assertListEqual(output.get_shape().as_list(), [5, 2, 3, 3]) - - def testCreateMaxPoolWithSamePaddingNCHW(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, 3, height, width), seed=1) - output = _layers.max_pool2d( - images, [3, 3], padding='SAME', data_format='NCHW') - self.assertListEqual(output.get_shape().as_list(), [5, 3, 2, 3]) - - def testCreateMaxPoolStrideWithSamePadding(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.max_pool2d(images, [3, 3], stride=1, padding='SAME') - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 3]) - - def testGlobalMaxPool(self): - height, width = 3, 6 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = _layers.max_pool2d(images, images.get_shape()[1:3], stride=1) - self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3]) - - -class MaxPool3DTest(test.TestCase): - - def testInvalidDataFormat(self): - depth, height, width = 3, 6, 9 - images = np.random.uniform(size=(5, depth, height, width, 3)) - with self.assertRaisesRegexp( - ValueError, 'data_format has to be either NCDHW or NDHWC.'): - _layers.max_pool3d(images, [3, 3, 3], data_format='CDHWN') - - def testCreateMaxPool(self): - depth, height, width = 3, 6, 9 - images = np.random.uniform(size=(5, depth, height, width, 3)).astype( - np.float32) - output = _layers.max_pool3d(images, [3, 3, 3]) - self.assertEqual(output.op.name, 'MaxPool3D/MaxPool3D') - self.assertListEqual(output.get_shape().as_list(), [5, 1, 2, 4, 3]) - - def testCreateMaxPoolNCDHW(self): - depth, height, width = 3, 6, 9 - images = np.random.uniform(size=(5, 3, depth, height, width)).astype( - np.float32) - output = _layers.max_pool3d(images, [3, 3, 3], data_format='NCDHW') - self.assertEquals(output.op.name, 'MaxPool3D/transpose_1') - self.assertListEqual(output.get_shape().as_list(), [5, 3, 1, 2, 4]) - - def testCollectOutputs(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, depth, height, width, 3), seed=1) - output = _layers.max_pool3d( - images, [3, 3, 3], outputs_collections='outputs') - output_collected = ops.get_collection('outputs')[0] - self.assertEqual(output_collected.aliases, ['MaxPool3D']) - self.assertEqual(output_collected, output) - - def testCreateSquareMaxPool(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, depth, height, width, 3), seed=1) - output = _layers.max_pool3d(images, 3) - self.assertEqual(output.op.name, 'MaxPool3D/MaxPool3D') - self.assertListEqual(output.get_shape().as_list(), [5, 1, 2, 4, 3]) - - def testCreateMaxPoolWithScope(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, depth, height, width, 3), seed=1) - output = _layers.max_pool3d(images, [3, 3, 3], scope='pool1') - self.assertEqual(output.op.name, 'pool1/MaxPool3D') - - def testCreateMaxPoolWithSamePadding(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, depth, height, width, 3), seed=1) - output = _layers.max_pool3d(images, [3, 3, 3], padding='SAME') - self.assertListEqual(output.get_shape().as_list(), [5, 2, 3, 5, 3]) - - def testCreateMaxPoolWithSamePaddingNCDHW(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, 3, depth, height, width), seed=1) - output = _layers.max_pool3d( - images, [3, 3, 3], padding='SAME', data_format='NCDHW') - self.assertListEqual(output.get_shape().as_list(), [5, 3, 2, 3, 5]) - - def testCreateMaxPoolStrideWithSamePadding(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, depth, height, width, 3), seed=1) - output = _layers.max_pool3d(images, [3, 3, 3], stride=1, padding='SAME') - self.assertListEqual(output.get_shape().as_list(), - [5, depth, height, width, 3]) - - def testGlobalMaxPool(self): - depth, height, width = 3, 6, 9 - images = random_ops.random_uniform((5, depth, height, width, 3), seed=1) - output = _layers.max_pool3d(images, images.get_shape()[1:4], stride=1) - self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 1, 3]) - - -class OneHotEncodingTest(test.TestCase): - - def testOneHotEncodingCreate(self): - with self.cached_session(): - labels = np.array([0, 1, 2]) - output = _layers.one_hot_encoding(labels, num_classes=3) - self.assertEqual(output.op.name, 'OneHotEncoding/one_hot') - self.assertListEqual(output.get_shape().as_list(), [3, 3]) - - def testCollectOutputs(self): - with self.cached_session(): - labels = constant_op.constant([0, 1, 2]) - output = _layers.one_hot_encoding( - labels, num_classes=3, outputs_collections='outputs') - c_output = ops.get_collection('outputs')[0] - self.assertEqual(c_output.aliases, ['OneHotEncoding']) - self.assertEqual(c_output, output) - - def testOneHotEncoding(self): - with self.cached_session(): - labels = constant_op.constant([0, 1, 2]) - one_hot_labels = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) - output = _layers.one_hot_encoding(labels, num_classes=3) - self.assertAllClose(output.eval(), one_hot_labels.eval()) - - def testOneHotEncodingInt32(self): - with self.cached_session(): - labels = constant_op.constant([0, 1, 2], dtype=dtypes.int32) - one_hot_labels = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) - output = _layers.one_hot_encoding(labels, num_classes=3) - self.assertAllClose(output.eval(), one_hot_labels.eval()) - - -class RepeatTests(test.TestCase): - - def testRepeat(self): - height, width = 3, 3 - with self.cached_session(): - images = np.random.uniform(size=(5, height, width, 3)).astype(np.float32) - output = _layers.repeat(images, 3, layers_lib.conv2d, 32, [3, 3]) - self.assertEqual(output.op.name, 'Repeat/convolution2d_3/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, 3, 3, 32]) - - def testRepeatWithScope(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform( - (5, height, width, 3), seed=1, name='images') - output = _layers.repeat( - images, 3, layers_lib.conv2d, 32, [3, 3], scope='conv1') - self.assertEqual(output.op.name, 'conv1/conv1_3/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, 3, 3, 32]) - - -class SeparableConv2dTest(test.TestCase): - - def testCreateConvInt32(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform( - (5, height, width, 3), seed=1, dtype=dtypes.int32, maxval=12345) - with self.assertRaisesRegexp(TypeError, 'non-floating point type'): - layers_lib.separable_conv2d(images, 32, [3, 3], 2) - - def testCreateConvFloat32(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform( - (5, height, width, 3), seed=1, dtype=dtypes.float32) - output = layers_lib.separable_conv2d(images, 32, [3, 3], 2) - self.assertEqual(output.op.name, 'SeparableConv2d/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 32]) - - def testCreateDepthwiseConv(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = layers_lib.separable_conv2d(images, None, [3, 3], 2) - self.assertEqual(output.op.name, 'SeparableConv2d/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, height, width, 6]) - - def testCreateConvCreatesWeightsAndBiasesVars(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - with self.cached_session(): - self.assertFalse(variables.get_variables('conv1/depthwise_weights')) - self.assertFalse(variables.get_variables('conv1/pointwise_weights')) - self.assertFalse(variables.get_variables('conv1/biases')) - layers_lib.separable_conv2d(images, 32, [3, 3], 4, scope='conv1') - self.assertTrue(variables.get_variables('conv1/depthwise_weights')) - self.assertTrue(variables.get_variables('conv1/pointwise_weights')) - self.assertTrue(variables.get_variables('conv1/biases')) - - def testCreateAtrousConvCreatesWeightsAndBiasesVars(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - with self.cached_session(): - self.assertFalse(variables.get_variables('conv1/depthwise_weights')) - self.assertFalse(variables.get_variables('conv1/pointwise_weights')) - self.assertFalse(variables.get_variables('conv1/biases')) - layers_lib.separable_conv2d(images, 32, [3, 3], 4, rate=2, scope='conv1') - self.assertTrue(variables.get_variables('conv1/depthwise_weights')) - self.assertTrue(variables.get_variables('conv1/pointwise_weights')) - self.assertTrue(variables.get_variables('conv1/biases')) - - def testCreateDepthwiseConvCreatesWeightsAndBiasesVars(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - with self.cached_session(): - self.assertFalse(variables.get_variables('conv1/depthwise_weights')) - self.assertFalse(variables.get_variables('conv1/pointwise_weights')) - self.assertFalse(variables.get_variables('conv1/biases')) - layers_lib.separable_conv2d(images, None, [3, 3], 4, scope='conv1') - self.assertTrue(variables.get_variables('conv1/depthwise_weights')) - self.assertFalse(variables.get_variables('conv1/pointwise_weights')) - self.assertTrue(variables.get_variables('conv1/biases')) - - def testCreateConvWithScope(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = layers_lib.separable_conv2d(images, 32, [3, 3], 6, scope='conv1') - self.assertEqual(output.op.name, 'conv1/Relu') - - def testCreateConvWithoutActivation(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = layers_lib.separable_conv2d( - images, 32, [3, 3], 8, activation_fn=None) - self.assertEqual(output.op.name, 'SeparableConv2d/BiasAdd') - - def testCreateConvValid(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = layers_lib.separable_conv2d( - images, 32, [3, 3], 2, padding='VALID') - self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 32]) - - def testCreateAtrousConvValid(self): - height, width = 5, 5 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = layers_lib.separable_conv2d( - images, 32, [3, 3], 2, padding='VALID', rate=2) - self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 32]) - - def testCreateDepthwiseConvValid(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = layers_lib.separable_conv2d( - images, None, [3, 3], 2, padding='VALID') - self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 6]) - - def testCreateAtrousDepthwiseConvValid(self): - height, width = 5, 5 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = layers_lib.separable_conv2d( - images, None, [3, 3], 2, padding='VALID', rate=2) - self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 6]) - - def testCreateConvWithWeightDecay(self): - random_seed.set_random_seed(0) - height, width = 3, 3 - with self.cached_session() as sess: - images = random_ops.random_uniform((5, height, width, 3), seed=1) - regularizer = regularizers.l2_regularizer(0.01) - layers_lib.separable_conv2d( - images, - 32, [3, 3], - 2, - weights_regularizer=regularizer, - weights_initializer=init_ops.ones_initializer()) - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 2) - weight_decay = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)[0] - self.assertEqual( - weight_decay.op.name, - 'SeparableConv2d/depthwise_kernel/Regularizer/l2_regularizer') - sess.run(variables_lib.global_variables_initializer()) - depth_weight_one = sess.run(weight_decay) - weight_decay = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)[1] - self.assertEqual( - weight_decay.op.name, - 'SeparableConv2d/pointwise_kernel/Regularizer/l2_regularizer') - pointwise_weight_one = sess.run(weight_decay) - - regularizer = regularizers.l2_regularizer(1.0) - layers_lib.separable_conv2d( - images, - 32, [3, 3], - 2, - weights_regularizer=regularizer, - weights_initializer=init_ops.ones_initializer()) - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 4) - weight_decay = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)[2] - sess.run(variables_lib.global_variables_initializer()) - depth_weight_two = sess.run(weight_decay) - weight_decay = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)[3] - pointwise_weight_two = sess.run(weight_decay) - - self.assertAllClose( - [100.0 * depth_weight_one, 100.0 * pointwise_weight_one], - [depth_weight_two, pointwise_weight_two]) - - def testReuseConvWithWeightDecay(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform((5, height, width, 3), seed=1) - regularizer = regularizers.l2_regularizer(0.01) - layers_lib.separable_conv2d( - images, 32, [3, 3], 2, weights_regularizer=regularizer, scope='conv1') - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 2) - layers_lib.separable_conv2d( - images, - 32, [3, 3], - 2, - weights_regularizer=regularizer, - scope='conv1', - reuse=True) - self.assertEqual( - len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 2) - - def testConvWithBatchNorm(self): - height, width = 3, 3 - batch_norm_collection = 'moving_vars' - normalizer_params = { - 'variables_collections': { - 'beta': [batch_norm_collection], - 'gamma': [batch_norm_collection], - 'moving_mean': [batch_norm_collection], - 'moving_variance': [batch_norm_collection], - } - } - images = random_ops.random_uniform((5, height, width, 3), seed=1) - net = layers_lib.separable_conv2d( - images, - 8, [3, 3], - 2, - normalizer_fn=_layers.batch_norm, - normalizer_params=normalizer_params, - scope='conv1') - net = layers_lib.separable_conv2d( - net, - 32, [3, 3], - 2, - normalizer_fn=_layers.batch_norm, - normalizer_params=normalizer_params, - scope='conv2') - self.assertEqual(len(ops.get_collection(batch_norm_collection)), 6) - self.assertEqual(len(variables.get_variables('conv1/BatchNorm')), 3) - self.assertEqual(len(variables.get_variables('conv2/BatchNorm')), 3) - - def testConvWithInputsViaPlaceHolder(self): - height, width = 3, 3 - images_placeholder = array_ops.placeholder( - dtypes.float32, shape=(None, None, None, 3)) - net = layers_lib.separable_conv2d( - images_placeholder, - 8, [3, 3], - 2, - normalizer_fn=_layers.batch_norm, - normalizer_params={}, - scope='conv1') - init_op = variables_lib.global_variables_initializer() - with self.cached_session() as sess: - images = np.random.rand(5, height, width, 3) - sess.run(init_op) - sess.run(net, feed_dict={images_placeholder: images}) - - def testTrainableFlagIsPassedOn(self): - for trainable in [True, False]: - for num_filters in [None, 8]: - with ops.Graph().as_default(): - input_size = [5, 10, 12, 3] - - images = random_ops.random_uniform(input_size, seed=1) - layers_lib.separable_conv2d( - images, num_filters, [3, 3], 1, trainable=trainable) - model_variables = variables.get_model_variables() - trainable_variables = variables_lib.trainable_variables() - for model_variable in model_variables: - self.assertEqual(trainable, model_variable in trainable_variables) - - def testSepConvNCHW(self): - for num_filters, correct_output_filters in zip((None, 5), (6, 5)): - with self.cached_session(): - batch, height, width = 4, 10, 12 - kernel_dim, stride = 3, 2 - images = random_ops.random_uniform((batch, 3, height, width), seed=1) - output = layers_lib.separable_conv2d( - images, - num_outputs=num_filters, - kernel_size=[kernel_dim, kernel_dim], - depth_multiplier=2, - stride=stride, - padding='VALID', - data_format='NCHW') - self.assertListEqual(output.get_shape().as_list(), [ - batch, correct_output_filters, (height - kernel_dim + 1) // stride, - (width - kernel_dim + 1) // stride - ]) - - -class ScaleGradientTests(test.TestCase): - """Simple tests of the scale_gradient function.""" - - def testBasic(self): - with self.cached_session(): - x = np.array([42], np.float32) - gradient_scale = np.array([2], np.float32) - - x = ops.convert_to_tensor(x) - y = layers_lib.scale_gradient(x, gradient_scale) - - np.testing.assert_array_equal(x.eval(), y.eval()) - g_x, = gradients_impl.gradients(y, [x], [np.array([3], np.float32)]) - np.testing.assert_array_equal([3 * 2], g_x.eval()) - - -class SequenceToImagesTest(test.TestCase): - - def testImagesToSequenceDims(self): - num_batches = 14 - num_time_steps = 11 - num_channels = 5 - desired_height = 7 - sequence = np.random.uniform(size=(num_time_steps, - num_batches, - num_channels)).astype(np.float32) - output = _layers.sequence_to_images(sequence, desired_height) - self.assertListEqual(output.get_shape().as_list(), [2, 7, 11, 5]) - - def testImagesToSequenceNCHW(self): - num_batches = 14 - num_time_steps = 11 - num_channels = 5 - desired_height = 7 - sequence = np.random.uniform(size=(num_time_steps, - num_batches, - num_channels)).astype(np.float32) - output = _layers.sequence_to_images(sequence, - desired_height, - output_data_format='channels_first') - self.assertListEqual(output.get_shape().as_list(), [2, 5, 7, 11]) - - -class SoftmaxTests(test.TestCase): - - def setUp(self): - self.low = 1 / (1 + math.e) - self.high = math.e / (1 + math.e) - - def testSoftmax2D(self): - logits = constant_op.constant([[0.0, 1], [1, 1], [1, 0]]) - prediction = _layers.softmax(logits) - exp_prediction = np.array([[self.low, self.high], [0.5, 0.5], - [self.high, self.low]]) - - with self.cached_session() as sess: - prediction = sess.run(prediction) - self.assertAllClose(exp_prediction, prediction) - - def testSoftmax3D(self): - logits = np.ones((2, 3, 2)) - logits[0, 0, 0] = 0 - logits[1, 1, 1] = 0 - logits = constant_op.constant(logits) - exp_prediction = 0.5 * np.ones((2, 3, 2)) - exp_prediction[0, 0, 0] = self.low - exp_prediction[0, 0, 1] = self.high - exp_prediction[1, 1, 0] = self.high - exp_prediction[1, 1, 1] = self.low - - prediction = _layers.softmax(logits) - with self.cached_session() as sess: - prediction = sess.run(prediction) - self.assertAllClose(exp_prediction, prediction) - - def testSoftmax3DUnknownSize(self): - logits = np.ones((2, 3, 2)) - logits[0, 0, 0] = 0 - logits[1, 1, 1] = 0 - logit_placeholder = array_ops.placeholder( - dtypes.float32, shape=(None, None, 2)) - feed_dict = {logit_placeholder: logits} - exp_prediction = 0.5 * np.ones((2, 3, 2)) - exp_prediction[0, 0, 0] = self.low - exp_prediction[0, 0, 1] = self.high - exp_prediction[1, 1, 0] = self.high - exp_prediction[1, 1, 1] = self.low - - prediction = _layers.softmax(logit_placeholder) - with self.cached_session() as sess: - prediction = sess.run(prediction, feed_dict=feed_dict) - self.assertAllClose(exp_prediction, prediction) - - def testSoftmaxUndefinedNthDimension(self): - logits = array_ops.placeholder(dtypes.float32) - with self.assertRaises(ValueError): - _layers.softmax(logits) - - -class SpatialSoftmaxTests(test.TestCase): - - def _SpatialSoftmax(self, x_loc, y_loc, height, width, batch_size, nchannels): - # Convert specified activation locations to range [-1, 1]. - height_lin = np.linspace(-1, 1, height) - width_lin = np.linspace(-1, 1, width) - x_lin = np.expand_dims(np.array([height_lin[i] for i in x_loc]), 1) - y_lin = np.expand_dims(np.array([width_lin[i] for i in y_loc]), 1) - np_keypoints = np.array( - [np.concatenate([x_lin, y_lin], axis=1) for i in range(batch_size)]) - np_keypoints = np.reshape(np_keypoints, [-1, nchannels * 2]) - return np_keypoints - - def testSpatialSoftmaxShape(self): - batch_shape = (2, 35, 30, 2) - features = array_ops.placeholder(dtypes.float32, shape=batch_shape) - np_features = np.zeros(batch_shape, dtype=np.float32) - spatial_softmax = _layers.spatial_softmax(features) - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - feed_dict = {features: np_features} - keypoints = sess.run(spatial_softmax, feed_dict) - self.assertAllEqual(keypoints.shape, (batch_shape[0], batch_shape[3] * 2)) - - def testSpatialSoftmaxShapeNCHW(self): - batch_shape = (2, 2, 35, 35) - features = array_ops.placeholder(dtypes.float32, shape=batch_shape) - np_features = np.zeros(batch_shape, dtype=np.float32) - spatial_softmax = _layers.spatial_softmax(features, data_format='NCHW') - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - feed_dict = {features: np_features} - keypoints = sess.run(spatial_softmax, feed_dict) - self.assertAllEqual(keypoints.shape, (batch_shape[0], batch_shape[1] * 2)) - - def testTwoMaxActivationsSameChannel(self): - batch_size, height, width, nchannels = (2, 35, 35, 1) - batch_shape = (batch_size, height, width, nchannels) - - # Put high equal activations on different locations in the same channel. - features = array_ops.placeholder(dtypes.float32, shape=batch_shape) - spatial_softmax = _layers.spatial_softmax(features) - np_features = np.zeros(batch_shape, dtype=np.float32) - x0, y0 = (10, 10) - x1, y1 = (20, 20) - avg_x = (x0 + x1) // 2 - avg_y = (y0 + y1) // 2 - np_features[:, x0, y0, :] = 100. - np_features[:, x1, y1, :] = 100. - x_loc = [avg_x] - y_loc = [avg_y] - - np_keypoints = self._SpatialSoftmax(x_loc, y_loc, height, width, batch_size, - nchannels) - - # Make sure expected location keypoints matches actual location keypoints. - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - feed_dict = {features: np_features} - keypoints = sess.run(spatial_softmax, feed_dict) - self.assertAllClose(keypoints, np_keypoints) - - def testMaxActivationsAtEdges(self): - batch_size, height, width, nchannels = (2, 35, 35, 4) - batch_shape = (batch_size, height, width, nchannels) - - # Put high activations on edges of spatial extent. - features = array_ops.placeholder(dtypes.float32, shape=batch_shape) - spatial_softmax = _layers.spatial_softmax(features) - np_features = np.zeros(batch_shape, dtype=np.float32) - - edges = [(0, 0), (0, width - 1), (height - 1, 0), (height - 1, width - 1)] - x_loc, y_loc = zip(*edges) - for c in range(nchannels): - np_features[:, x_loc[c], y_loc[c], c] = 100. - - np_keypoints = self._SpatialSoftmax(x_loc, y_loc, height, width, batch_size, - nchannels) - - # Make sure expected location keypoints matches actual location keypoints. - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - feed_dict = {features: np_features} - keypoints = sess.run(spatial_softmax, feed_dict) - self.assertAllClose(keypoints, np_keypoints) - - def testSpatialSoftmaxVariableSized(self): - batch_size = 2 - nchannels = 2 - height1, width1 = (35, 30) - height2, width2 = (20, 20) - batch_shape1 = (batch_size, height1, width1, nchannels) - batch_shape2 = (batch_size, height2, width2, nchannels) - variable_sized_shape = (None, None, None, 2) - - # Put high activations on single spatial locations. - features = array_ops.placeholder(dtypes.float32, shape=variable_sized_shape) - spatial_softmax = _layers.spatial_softmax(features) - np_features1 = np.zeros(batch_shape1, dtype=np.float32) - np_features2 = np.zeros(batch_shape2, dtype=np.float32) - x_loc = [15, 2] - y_loc = [10, 9] - for c in range(nchannels): - np_features1[:, x_loc[c], y_loc[c], c] = 100. - np_features2[:, x_loc[c], y_loc[c], c] = 100. - - np_keypoints1 = self._SpatialSoftmax(x_loc, y_loc, height1, width1, - batch_size, nchannels) - np_keypoints2 = self._SpatialSoftmax(x_loc, y_loc, height2, width2, - batch_size, nchannels) - - # Make sure expected location keypoints matches actual location keypoints. - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - feed_dict = {features: np_features1} - tf_keypoints1 = sess.run(spatial_softmax, feed_dict) - self.assertAllClose(tf_keypoints1, np_keypoints1) - - feed_dict = {features: np_features2} - tf_keypoints2 = sess.run(spatial_softmax, feed_dict) - self.assertAllClose(tf_keypoints2, np_keypoints2) - - def testSpatialSoftmax(self): - batch_size, height, width, nchannels = (2, 35, 35, 2) - batch_shape = (batch_size, height, width, nchannels) - - # Put high activations on single spatial locations. - features = array_ops.placeholder(dtypes.float32, shape=batch_shape) - spatial_softmax = _layers.spatial_softmax(features) - np_features = np.zeros(batch_shape, dtype=np.float32) - x_loc = [15, 2] - y_loc = [10, 28] - for c in range(nchannels): - np_features[:, x_loc[c], y_loc[c], c] = 100. - - np_keypoints = self._SpatialSoftmax(x_loc, y_loc, height, width, batch_size, - nchannels) - - # Make sure expected location keypoints matches actual location keypoints. - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - feed_dict = {features: np_features} - keypoints = sess.run(spatial_softmax, feed_dict) - self.assertAllClose(keypoints, np_keypoints) - - def testSpatialSoftmaxNCHW(self): - batch_size, nchannels, height, width = (2, 2, 35, 35) - batch_shape = (batch_size, nchannels, height, width) - - # Put high activations on single spatial locations. - features = array_ops.placeholder(dtypes.float32, shape=batch_shape) - spatial_softmax = _layers.spatial_softmax(features, data_format='NCHW') - np_features = np.zeros(batch_shape, dtype=np.float32) - x_loc = [15, 2] - y_loc = [10, 28] - for c in range(nchannels): - np_features[:, c, x_loc[c], y_loc[c]] = 100. - - np_keypoints = self._SpatialSoftmax(x_loc, y_loc, height, width, batch_size, - nchannels) - - # Make sure expected location keypoints matches actual location keypoints. - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - feed_dict = {features: np_features} - keypoints = sess.run(spatial_softmax, feed_dict) - self.assertAllClose(keypoints, np_keypoints) - - def testSpatialSoftmaxToFullyConnected(self): - batch_shape = (2, 35, 35, 2) - features = array_ops.placeholder(dtypes.float32, shape=batch_shape) - spatial_softmax = _layers.spatial_softmax(features) - net = _layers.fully_connected(spatial_softmax, 10) - np_features = np.zeros(batch_shape, dtype=np.float32) - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - feed_dict = {features: np_features} - sess.run(net, feed_dict) - - -class StackTests(test.TestCase): - - def testStackFullyConnected(self): - height, width = 3, 3 - with self.cached_session(): - images = np.random.uniform(size=(5, height * width * 3)) - output = _layers.stack(images, _layers.fully_connected, [10, 20, 30]) - self.assertEqual(output.op.name, 'Stack/fully_connected_3/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, 30]) - - def testStackFullyConnectedFailOnReuse(self): - height, width = 3, 3 - with self.cached_session(): - with variable_scope.variable_scope('test', reuse=True): - images = np.random.uniform(size=(5, height * width * 3)) - with self.assertRaises(ValueError): - _layers.stack(images, _layers.fully_connected, [10, 20, 30]) - - def testStackRelu(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform( - (5, height * width * 3), seed=1, name='images') - output = _layers.stack(images, layers_lib.relu, [10, 20, 30]) - self.assertEqual(output.op.name, 'Stack/fully_connected_3/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, 30]) - - def testStackElu(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform( - (5, height * width * 3), seed=1, name='images') - output = _layers.stack(images, layers_lib.elu, [10, 20, 30]) - self.assertEqual(output.op.name, 'Stack/fully_connected_3/Elu') - self.assertListEqual(output.get_shape().as_list(), [5, 30]) - - def testStackConvolution2d(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform( - (5, height, width, 3), seed=1, name='images') - output = _layers.stack( - images, - layers_lib.convolution2d, [10, 20, 30], - kernel_size=[3, 3], - padding='SAME') - self.assertEqual(output.op.name, 'Stack/convolution2d_3/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, 3, 3, 30]) - - def testStackWithScope(self): - height, width = 3, 3 - with self.cached_session(): - images = random_ops.random_uniform( - (5, height, width, 3), seed=1, name='images') - output = _layers.stack( - images, - layers_lib.convolution2d, [10, 20, 30], - kernel_size=[3, 3], - padding='SAME', - scope='conv1') - self.assertEqual(output.op.name, 'conv1/conv1_3/Relu') - self.assertListEqual(output.get_shape().as_list(), [5, 3, 3, 30]) - - -class UnitNormTests(test.TestCase): - - def testUnitNormWithRandomMatrix(self): - height, width = 2, 3 - - for dim in range(3): - random_seed.set_random_seed(0) - image = random_ops.random_uniform((height, width, 3)) - output = _layers.unit_norm(image, dim=dim, epsilon=1e-6) - norms = math_ops.sqrt( - math_ops.reduce_sum(math_ops.square(output), axis=dim)) - - shape = [height, width, 3] - del shape[dim] - expected = np.ones(shape) - - with self.cached_session(): - actual = norms.eval() - self.assertAllClose(expected, actual, 1e-4, 1e-4) - - def testDimEqualToRankRaisesError(self): - height, width = 2, 3 - - random_seed.set_random_seed(0) - image = random_ops.random_uniform((height, width, 3)) - - with self.assertRaises(ValueError): - _layers.unit_norm(image, dim=3, epsilon=1e-6) - - def testUnknownRankRaisesError(self): - image = array_ops.placeholder(dtypes.float32) - with self.assertRaises(ValueError): - _layers.unit_norm(image, dim=2) - - def testKnownRankUnknownDimsSucceeds(self): - height, width = 2, 3 - - for dim in range(3): - placeholder_value = np.ones((height, width, 3)) - shape = [height, width, 3] - del shape[dim] - expected = np.ones(shape) - - image = array_ops.placeholder(dtypes.float32, (None, None, 3)) - output = _layers.unit_norm(image, dim=dim, epsilon=1e-6) - norms = math_ops.sqrt( - math_ops.reduce_sum(math_ops.square(output), axis=dim)) - - with self.cached_session(): - actual = norms.eval({image: placeholder_value}) - self.assertAllClose(expected, actual, 1e-4, 1e-4) - - -class PoincareNormalizeTest(test.TestCase): - - def _PoincareNormalize(self, x, dim, epsilon=1e-5): - if isinstance(dim, list): - norm = np.linalg.norm(x, axis=tuple(dim)) - for d in dim: - norm = np.expand_dims(norm, d) - norm_x = ((1. - epsilon) * x) / norm - else: - norm = np.expand_dims(np.apply_along_axis(np.linalg.norm, dim, x), dim) - norm_x = ((1. - epsilon) * x) / norm - return np.where(norm > 1.0 - epsilon, norm_x, x) - - def testPoincareNormalize(self): - x_shape = [20, 7, 3] - epsilon = 1e-5 - tol = 1e-6 - np.random.seed(1) - x_np = np.random.random_sample(x_shape).astype(np.float32) - for dim in range(len(x_shape)): - y_np = self._PoincareNormalize(x_np, dim, epsilon) - with self.cached_session(): - x_tf = constant_op.constant(x_np, name='x') - y_tf = _layers.poincare_normalize(x_tf, dim, epsilon) - y_tf_eval = y_tf.eval() - norm = np.linalg.norm(y_np, axis=dim) - self.assertLessEqual(norm.max(), 1. - epsilon + tol) - norm = np.linalg.norm(y_tf_eval, axis=dim) - self.assertLessEqual(norm.max(), 1. - epsilon + tol) - self.assertAllClose(y_np, y_tf_eval) - - def testPoincareNormalizeDimArray(self): - x_shape = [20, 7, 3] - epsilon = 1e-5 - tol = 1e-6 - np.random.seed(1) - x_np = np.random.random_sample(x_shape).astype(np.float32) - dim = [1, 2] - y_np = self._PoincareNormalize(x_np, dim, epsilon) - with self.cached_session(): - x_tf = constant_op.constant(x_np, name='x') - y_tf = _layers.poincare_normalize(x_tf, dim, epsilon) - y_tf_eval = y_tf.eval() - norm = np.linalg.norm(y_np, axis=tuple(dim)) - self.assertLess(norm.max(), 1. - epsilon + tol) - norm = np.linalg.norm(y_tf_eval, axis=tuple(dim)) - self.assertLess(norm.max(), 1. - epsilon + tol) - self.assertAllClose(y_np, y_tf_eval, rtol=1e-6, atol=1e-6) - - def testPoincareNormalizeGradient(self): - x_shape = [20, 7, 3] - np.random.seed(1) - x_np = np.random.random_sample(x_shape).astype(np.float64) - for dim in range(len(x_shape)): - with self.cached_session(): - x_tf = constant_op.constant(x_np, name='x') - y_tf = _layers.poincare_normalize(x_tf, dim) - err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf, - x_shape) - print('PoinCareNormalize gradient err = %g ' % err) - self.assertLess(err, 1e-4) - - -# TODO(b/28426988): Add separate tests for non-legacy versions. -class LegacyFullyConnectedTest(test.TestCase): - - def setUp(self): - test.TestCase.setUp(self) - random_seed.set_random_seed(1234) - self.input = constant_op.constant([[1., 2., 3.], [-4., 15., -6.]]) - self.input_3_dim_arr = [[[1., 1.1, 1.2], [2., 2.1, 2.2], [3., 3.1, 3.2], - [4., 4.1, 4.2]], [[5., 5.1, 5.2], [6., 6.1, 6.2], - [7., 7.1, 7.2], [8., 8.1, 8.2]]] - self.input_3_dim = constant_op.constant(self.input_3_dim_arr) - - assert not ops.get_collection(ops.GraphKeys.SUMMARIES) - - def _fully_connected_basic_use(self, x, num_output_units, expected_shape): - output = _layers.legacy_fully_connected( - x, num_output_units, activation_fn=nn_ops.relu) - - with session.Session() as sess: - with self.assertRaises(errors_impl.FailedPreconditionError): - sess.run(output) - - variables_lib.global_variables_initializer().run() - out_value, shape_value = sess.run([output, array_ops.shape(output)]) - - self.assertAllClose(shape_value, expected_shape) - self.assertEqual(output.get_shape().as_list(), expected_shape) - self.assertTrue(np.all(out_value >= 0), 'Relu should have all values >= 0.') - - self.assertEqual(2, - len(ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES))) - self.assertEqual( - 0, len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES))) - - def test_fully_connected_basic_use(self): - self._fully_connected_basic_use(self.input, 8, [2, 8]) - - def test_fully_connected_basic_use_multi_dim(self): - for last_dim in [1, 3]: - self.setUp() - self._fully_connected_basic_use(self.input_3_dim, last_dim, - [2, 4, last_dim]) - - def test_relu_layer_basic_use(self): - output = layers_lib.legacy_relu(self.input, 8) - - with session.Session() as sess: - with self.assertRaises(errors_impl.FailedPreconditionError): - sess.run(output) - - variables_lib.global_variables_initializer().run() - out_value = sess.run(output) - - self.assertEqual(output.get_shape().as_list(), [2, 8]) - self.assertTrue(np.all(out_value >= 0), 'Relu should have all values >= 0.') - - self.assertEqual(2, - len(ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES))) - self.assertEqual( - 0, len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES))) - - def test_variable_reuse_with_scope(self): - with variable_scope.variable_scope('test') as vs: - output1 = layers_lib.legacy_relu(self.input, 8) - output2 = layers_lib.legacy_relu(self.input, 8) - - with variable_scope.variable_scope(vs, reuse=True): - output3 = layers_lib.legacy_relu(self.input, 8) - - with session.Session() as sess: - variables_lib.global_variables_initializer().run() - out_value1, out_value2, out_value3 = sess.run([output1, output2, output3]) - - self.assertFalse(np.allclose(out_value1, out_value2)) - self.assertAllClose(out_value1, out_value3) - - def test_variable_reuse_with_template(self): - tmpl1 = template.make_template( - 'test', _layers.legacy_fully_connected, num_output_units=8) - output1 = tmpl1(self.input) - output2 = tmpl1(self.input) - - with session.Session() as sess: - variables_lib.global_variables_initializer().run() - out_value1, out_value2 = sess.run([output1, output2]) - self.assertAllClose(out_value1, out_value2) - - def _custom_initializers(self, x, num_output_units, expected_outputs): - output = layers_lib.legacy_relu( - x, - num_output_units, - weight_init=init_ops.constant_initializer(2.0), - bias_init=init_ops.constant_initializer(1.0)) - - with session.Session() as sess: - variables_lib.global_variables_initializer().run() - out_value = sess.run(output) - - self.assertAllClose(np.array(expected_outputs), out_value) - - def test_custom_initializers(self): - self._custom_initializers(self.input, 2, [[13.0, 13.0], [11.0, 11.0]]) - - def test_custom_initializers_multi_dim(self): - self._custom_initializers( - self.input_3_dim, 2, - [[[7.6, 7.6], [13.6, 13.6], [19.6, 19.6], [25.6, 25.6]], - [[31.6, 31.6], [37.6, 37.6], [43.6, 43.6], [49.6, 49.6]]]) - - def test_custom_collections(self): - layers_lib.legacy_relu( - self.input, - 2, - weight_collections=['unbiased'], - bias_collections=['biased'], - output_collections=['output']) - - self.assertEqual(1, len(ops.get_collection('unbiased'))) - self.assertEqual(1, len(ops.get_collection('biased'))) - self.assertEqual(1, len(ops.get_collection('output'))) - self.assertEqual(2, len(ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))) - - def test_all_custom_collections(self): - layers_lib.legacy_relu( - self.input, - 2, - weight_collections=['unbiased', 'all'], - bias_collections=['biased', 'all']) - - self.assertEqual(1, len(ops.get_collection('unbiased'))) - self.assertEqual(1, len(ops.get_collection('biased'))) - self.assertEqual( - ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES), - ops.get_collection('all')) - - def test_no_bias(self): - layers_lib.legacy_relu(self.input, 2, bias_init=None) - self.assertEqual(1, len(ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))) - - def test_no_activation(self): - y = _layers.legacy_fully_connected(self.input, 2) - self.assertEqual(2, len(ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))) - self.assertEqual('BiasAdd', y.op.type) - - def test_no_activation_no_bias(self): - y = _layers.legacy_fully_connected(self.input, 2, bias_init=None) - self.assertEqual(1, len(ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))) - self.assertEqual('MatMul', y.op.type) - - def test_regularizer(self): - tensor = constant_op.constant(5.0) - - def test_fn(_): - return tensor - - _layers.legacy_fully_connected(self.input, 2, weight_regularizer=test_fn) - - regs = [ - ops.convert_to_tensor(r) - for r in ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES) - ] - self.assertEqual([tensor], regs) - - def test_regularizer_with_multiple_variables(self): - tensor = constant_op.constant(5.0) - - def test_fn(_): - return tensor - - _layers.legacy_fully_connected(self.input, 2, weight_regularizer=test_fn) - _layers.legacy_fully_connected(self.input, 2, weight_regularizer=test_fn) - - regs = [ - ops.convert_to_tensor(r) - for r in ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES) - ] - self.assertEqual([tensor, tensor], regs) - - def test_regularizer_with_variable_reuse(self): - tensor = constant_op.constant(5.0) - - def test_fn(_): - return tensor - - with variable_scope.variable_scope('test') as vs: - _layers.legacy_fully_connected(self.input, 2, weight_regularizer=test_fn) - - with variable_scope.variable_scope(vs, reuse=True): - _layers.legacy_fully_connected(self.input, 2, weight_regularizer=test_fn) - - regs = [ - ops.convert_to_tensor(r) - for r in ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES) - ] - self.assertEqual([tensor], regs) - - def test_empty_x_results_in_empty_output(self): - # Empty x is common if someone masks their input with tf.boolean_mask in - # order to drop missing entries, and in a particular batch all entries are - # missing. - with self.cached_session(): - x = np.array([]).reshape(0, 3) - self.assertEqual(0, array_ops.size(x).eval()) - y = _layers.legacy_fully_connected(x, 2, activation_fn=nn_ops.softmax) - variables_lib.global_variables_initializer().run() - expected_y = np.array([]).reshape(0, 2) - np.testing.assert_array_equal(expected_y, y.eval()) - - def test_shapes_variable_first_dim(self): - # first dimension is not known statically. - x = array_ops.placeholder(dtypes.float32, shape=[None, 4, 3]) - y = _layers.legacy_fully_connected(x, 1) - # in the output we still only know the 2nd and 3rd dimensions statically. - self.assertEqual(y.get_shape().as_list(), [None, 4, 1]) - with self.cached_session() as sess: - variables_lib.global_variables_initializer().run() - # we can feed in input with first dimension 2 - shape_value = sess.run( - array_ops.shape(y), feed_dict={ - x: self.input_3_dim_arr - }) - self.assertAllClose(shape_value, [2, 4, 1]) - # we can feed in input with first dimension 1 - shape_value = sess.run( - array_ops.shape(y), feed_dict={ - x: [self.input_3_dim_arr[0]] - }) - self.assertAllClose(shape_value, [1, 4, 1]) - # we cannot feed in input with inconsistent dimensions - with self.assertRaises(ValueError): - sess.run(array_ops.shape(y), feed_dict={x: [[[]]]}) - - def _unknown_dim_invalid_input(self, last_dim): - x = array_ops.placeholder(dtypes.float32, shape=[3, last_dim]) - _layers.legacy_fully_connected(x, 2, activation_fn=None) - - def test_known_dim_valid_input(self): - self._unknown_dim_invalid_input(last_dim=3) - - def test_unknown_dim_invalid_input(self): - with self.assertRaisesRegexp( - ValueError, 'last dimension of x must be known but is None'): - self._unknown_dim_invalid_input(last_dim=None) - - def test_1d_invalid_input(self): - with self.cached_session(): - with self.assertRaisesRegexp(ValueError, - 'rank of x must be at least 2 not: 1'): - x = constant_op.constant([[]], shape=[0]) - _layers.legacy_fully_connected(x, 2, activation_fn=nn_ops.softmax) - - -class MaxOutTest(test.TestCase): - - def test_simple(self): - inputs = random_ops.random_uniform((64, 10, 36), seed=1) - graph = _layers.maxout(inputs, num_units=3) - self.assertEqual(graph.get_shape().as_list(), [64, 10, 3]) - - def test_fully_connected(self): - inputs = random_ops.random_uniform((64, 50), seed=1) - graph = _layers.fully_connected(inputs, 50) - graph = _layers.maxout(graph, num_units=10) - self.assertEqual(graph.get_shape().as_list(), [64, 10]) - - def test_nchw(self): - inputs = random_ops.random_uniform((10, 100, 100, 3), seed=1) - graph = _layers.conv2d(inputs, 10, 3, padding='SAME') - graph = _layers.maxout(graph, num_units=1) - self.assertEqual(graph.get_shape().as_list(), [10, 100, 100, 1]) - - def test_invalid_shape(self): - inputs = random_ops.random_uniform((10, 100, 100, 3), seed=1) - graph = _layers.conv2d(inputs, 3, 10) - with self.assertRaisesRegexp(ValueError, 'number of features'): - graph = _layers.maxout(graph, num_units=2) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/layers/python/layers/normalization.py b/tensorflow/contrib/layers/python/layers/normalization.py deleted file mode 100644 index 76b03ff5148..00000000000 --- a/tensorflow/contrib/layers/python/layers/normalization.py +++ /dev/null @@ -1,384 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Contains the normalization layer classes and their functional aliases.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -from tensorflow.contrib.framework.python.ops import add_arg_scope -from tensorflow.contrib.framework.python.ops import variables -from tensorflow.contrib.layers.python.layers import utils -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import variable_scope - - -__all__ = [ - 'group_norm', - 'instance_norm', -] - -DATA_FORMAT_NCHW = 'NCHW' -DATA_FORMAT_NHWC = 'NHWC' - - -@add_arg_scope -def instance_norm(inputs, - center=True, - scale=True, - epsilon=1e-6, - activation_fn=None, - param_initializers=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - data_format=DATA_FORMAT_NHWC, - scope=None): - """Functional interface for the instance normalization layer. - - Reference: https://arxiv.org/abs/1607.08022. - - "Instance Normalization: The Missing Ingredient for Fast Stylization" - Dmitry Ulyanov, Andrea Vedaldi, Victor Lempitsky - - Args: - inputs: A tensor with 2 or more dimensions, where the first dimension has - `batch_size`. The normalization is over all but the last dimension if - `data_format` is `NHWC` and the second dimension if `data_format` is - `NCHW`. - center: If True, add offset of `beta` to normalized tensor. If False, `beta` - is ignored. - scale: If True, multiply by `gamma`. If False, `gamma` is - not used. When the next layer is linear (also e.g. `nn.relu`), this can be - disabled since the scaling can be done by the next layer. - epsilon: Small float added to variance to avoid dividing by zero. - activation_fn: Activation function, default set to None to skip it and - maintain a linear activation. - param_initializers: Optional initializers for beta, gamma, moving mean and - moving variance. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional collections for the variables. - outputs_collections: Collections to add the outputs. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - data_format: A string. `NHWC` (default) and `NCHW` are supported. - scope: Optional scope for `variable_scope`. - - Returns: - A `Tensor` representing the output of the operation. - - Raises: - ValueError: If `data_format` is neither `NHWC` nor `NCHW`. - ValueError: If the rank of `inputs` is undefined. - ValueError: If rank or channels dimension of `inputs` is undefined. - """ - inputs = ops.convert_to_tensor(inputs) - inputs_shape = inputs.shape - inputs_rank = inputs.shape.ndims - - if inputs_rank is None: - raise ValueError('Inputs %s has undefined rank.' % inputs.name) - if data_format not in (DATA_FORMAT_NCHW, DATA_FORMAT_NHWC): - raise ValueError('data_format has to be either NCHW or NHWC.') - - with variable_scope.variable_scope( - scope, 'InstanceNorm', [inputs], reuse=reuse) as sc: - if data_format == DATA_FORMAT_NCHW: - reduction_axis = 1 - # For NCHW format, rather than relying on implicit broadcasting, we - # explicitly reshape the params to params_shape_broadcast when computing - # the moments and the batch normalization. - params_shape_broadcast = list( - [1, inputs_shape[1].value] + [1 for _ in range(2, inputs_rank)]) - else: - reduction_axis = inputs_rank - 1 - params_shape_broadcast = None - moments_axes = list(range(inputs_rank)) - del moments_axes[reduction_axis] - del moments_axes[0] - params_shape = inputs_shape[reduction_axis:reduction_axis + 1] - if not params_shape.is_fully_defined(): - raise ValueError('Inputs %s has undefined channels dimension %s.' % ( - inputs.name, params_shape)) - - # Allocate parameters for the beta and gamma of the normalization. - beta, gamma = None, None - dtype = inputs.dtype.base_dtype - if param_initializers is None: - param_initializers = {} - if center: - beta_collections = utils.get_variable_collections( - variables_collections, 'beta') - beta_initializer = param_initializers.get( - 'beta', init_ops.zeros_initializer()) - beta = variables.model_variable('beta', - shape=params_shape, - dtype=dtype, - initializer=beta_initializer, - collections=beta_collections, - trainable=trainable) - if params_shape_broadcast: - beta = array_ops.reshape(beta, params_shape_broadcast) - if scale: - gamma_collections = utils.get_variable_collections( - variables_collections, 'gamma') - gamma_initializer = param_initializers.get( - 'gamma', init_ops.ones_initializer()) - gamma = variables.model_variable('gamma', - shape=params_shape, - dtype=dtype, - initializer=gamma_initializer, - collections=gamma_collections, - trainable=trainable) - if params_shape_broadcast: - gamma = array_ops.reshape(gamma, params_shape_broadcast) - - # Calculate the moments (instance activations). - mean, variance = nn.moments(inputs, moments_axes, keep_dims=True) - - # Compute instance normalization. - outputs = nn.batch_normalization( - inputs, mean, variance, beta, gamma, epsilon, name='instancenorm') - if activation_fn is not None: - outputs = activation_fn(outputs) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) - - -@add_arg_scope -def group_norm(inputs, - groups=32, - channels_axis=-1, - reduction_axes=(-3, -2), - center=True, - scale=True, - epsilon=1e-6, - activation_fn=None, - param_initializers=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - scope=None, - mean_close_to_zero=False): - """Functional interface for the group normalization layer. - - Reference: https://arxiv.org/abs/1803.08494. - - "Group Normalization", Yuxin Wu, Kaiming He - - Args: - inputs: A Tensor with at least 2 dimensions one which is channels. All - shape dimensions except for batch must be fully defined. - groups: Integer. Divide the channels into this number of groups over which - normalization statistics are computed. This number must be commensurate - with the number of channels in `inputs`. - channels_axis: An integer. Specifies index of channels axis which will be - broken into `groups`, each of which whose statistics will be computed - across. Must be mutually exclusive with `reduction_axes`. Preferred usage - is to specify negative integers to be agnostic as to whether a batch - dimension is included. - reduction_axes: Tuple of integers. Specifies dimensions over which - statistics will be accumulated. Must be mutually exclusive with - `channels_axis`. Statistics will not be accumulated across axes not - specified in `reduction_axes` nor `channel_axis`. Preferred usage is to - specify negative integers to be agnostic to whether a batch dimension is - included. - - Some sample usage cases: - NHWC format: channels_axis=-1, reduction_axes=[-3, -2] - NCHW format: channels_axis=-3, reduction_axes=[-2, -1] - - center: If True, add offset of `beta` to normalized tensor. If False, `beta` - is ignored. - scale: If True, multiply by `gamma`. If False, `gamma` is - not used. When the next layer is linear (also e.g. `nn.relu`), this can be - disabled since the scaling can be done by the next layer. - epsilon: Small float added to variance to avoid dividing by zero. - activation_fn: Activation function, default set to None to skip it and - maintain a linear activation. - param_initializers: Optional initializers for beta, gamma, moving mean and - moving variance. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional collections for the variables. - outputs_collections: Collections to add the outputs. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - scope: Optional scope for `variable_scope`. - mean_close_to_zero: The mean of `input` before ReLU will be close to zero - when batch size >= 4k for Resnet-50 on TPU. If `True`, use - `nn.sufficient_statistics` and `nn.normalize_moments` to calculate the - variance. This is the same behavior as `fused` equals `True` in batch - normalization. If `False`, use `nn.moments` to calculate the variance. - When `mean` is close to zero, like 1e-4, use `mean` to calculate the - variance may have poor result due to repeated roundoff error and - denormalization in `mean`. When `mean` is large, like 1e2, - sum(`input`^2) is so large that only the high-order digits of the elements - are being accumulated. Thus, use sum(`input` - `mean`)^2/n to calculate - the variance has better accuracy compared to (sum(`input`^2)/n - `mean`^2) - when `mean` is large. - - - Returns: - A `Tensor` representing the output of the operation. - - Raises: - ValueError: If the rank of `inputs` is undefined. - ValueError: If rank or channels dimension of `inputs` is undefined. - ValueError: If number of groups is not commensurate with number of channels. - ValueError: If reduction_axes or channels_axis are out of bounds. - ValueError: If reduction_axes are not mutually exclusive with channels_axis. - """ - # TODO(shlens): Support partially defined shapes for the inputs. - inputs = ops.convert_to_tensor(inputs) - - if inputs.shape.ndims is None: - raise ValueError('Inputs %s has undefined rank.' % inputs.name) - if channels_axis > (inputs.shape.ndims - 1): - raise ValueError('Axis is out of bounds.') - - # Use dynamic shape for not fully defined dimensions in the inputs. - dyanmic_shape = array_ops.shape(inputs) - input_shape_list = [] - for i, dim in enumerate(inputs.shape): - if dim.value is None: - input_shape_list.append(dyanmic_shape[i]) - else: - input_shape_list.append(dim) - - # Standardize the channels_axis to be positive and identify # of channels. - if channels_axis < 0: - channels_axis = inputs.shape.ndims + channels_axis - channels = inputs.shape[channels_axis].value - - if channels is None: - raise ValueError('Inputs %s has undefined channel dimension: %d.' % ( - inputs.name, channels_axis)) - - # Standardize the reduction_axes to be positive. - reduction_axes = list(reduction_axes) - for i in range(len(reduction_axes)): - if reduction_axes[i] < 0: - reduction_axes[i] += inputs.shape.ndims - - for a in reduction_axes: - if a > inputs.shape.ndims: - raise ValueError('Axis is out of bounds.') - if inputs.shape[a].value is None: - raise ValueError('Inputs %s has undefined dimensions %d.' % ( - inputs.name, a)) - if channels_axis == a: - raise ValueError('reduction_axis must be mutually exclusive ' - 'with channels_axis') - if groups > channels: - raise ValueError('Invalid groups %d for %d channels.' % (groups, channels)) - if channels % groups != 0: - raise ValueError('%d channels is not commensurate with %d groups.' % - (channels, groups)) - - # Determine axes before channels. Some examples of common image formats: - # 'NCHW': before = [N], after = [HW] - # 'NHWC': before = [NHW], after = [] - axes_before_channels = input_shape_list[:channels_axis] - axes_after_channels = input_shape_list[channels_axis+1:] - - # Manually broadcast the parameters to conform to the number of groups. - params_shape_broadcast = ([1] * len(axes_before_channels) + - [groups, channels // groups] + - [1] * len(axes_after_channels)) - - # Reshape the input by the group within the channel dimension. - inputs_shape = (axes_before_channels + [groups, channels // groups] + - axes_after_channels) - inputs = array_ops.reshape(inputs, inputs_shape) - - # Determine the dimensions across which moments are calculated. - moments_axes = [channels_axis + 1] - for a in reduction_axes: - if a > channels_axis: - moments_axes.append(a + 1) - else: - moments_axes.append(a) - - with variable_scope.variable_scope( - scope, 'GroupNorm', [inputs], reuse=reuse) as sc: - # Note that the params_shape is the number of channels always. - params_shape = [channels] - - # Allocate parameters for the beta and gamma of the normalization. - beta, gamma = None, None - dtype = inputs.dtype.base_dtype - if param_initializers is None: - param_initializers = {} - if center: - beta_collections = utils.get_variable_collections( - variables_collections, 'beta') - beta_initializer = param_initializers.get( - 'beta', init_ops.zeros_initializer()) - beta = variables.model_variable('beta', - shape=params_shape, - dtype=dtype, - initializer=beta_initializer, - collections=beta_collections, - trainable=trainable) - beta = array_ops.reshape(beta, params_shape_broadcast) - - if scale: - gamma_collections = utils.get_variable_collections( - variables_collections, 'gamma') - gamma_initializer = param_initializers.get( - 'gamma', init_ops.ones_initializer()) - gamma = variables.model_variable('gamma', - shape=params_shape, - dtype=dtype, - initializer=gamma_initializer, - collections=gamma_collections, - trainable=trainable) - gamma = array_ops.reshape(gamma, params_shape_broadcast) - - # Calculate the moments. - if mean_close_to_zero: - # One pass algorithm returns better result when mean is close to zero. - counts, means_ss, variance_ss, _ = nn.sufficient_statistics( - inputs, moments_axes, keep_dims=True) - mean, variance = nn.normalize_moments( - counts, means_ss, variance_ss, shift=None) - else: - mean, variance = nn.moments(inputs, moments_axes, keep_dims=True) - - # Compute normalization. - # TODO(shlens): Fix nn.batch_normalization to handle the 5-D Tensor - # appropriately so that this operation may be faster. - gain = math_ops.rsqrt(variance + epsilon) - offset = -mean * gain - if gamma is not None: - gain *= gamma - offset *= gamma - if beta is not None: - offset += beta - outputs = inputs * gain + offset - - # Collapse the groups into the channel dimension. - outputs = array_ops.reshape(outputs, input_shape_list) - - if activation_fn is not None: - outputs = activation_fn(outputs) - return utils.collect_named_outputs(outputs_collections, sc.name, outputs) diff --git a/tensorflow/contrib/layers/python/layers/normalization_test.py b/tensorflow/contrib/layers/python/layers/normalization_test.py deleted file mode 100644 index 9a85084b239..00000000000 --- a/tensorflow/contrib/layers/python/layers/normalization_test.py +++ /dev/null @@ -1,473 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Tests for contrib.layers.python.layers.normalization.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.framework.python.ops import variables as contrib_variables -from tensorflow.contrib.layers.python.layers import normalization -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class InstanceNormTest(test.TestCase): - - def testUnknownShape(self): - inputs = array_ops.placeholder(dtypes.float32) - with self.assertRaisesRegexp(ValueError, 'undefined rank'): - normalization.instance_norm(inputs) - - def testBadDataFormat(self): - inputs = array_ops.placeholder(dtypes.float32, shape=(2, 5, 5)) - with self.assertRaisesRegexp(ValueError, - 'data_format has to be either NCHW or NHWC.'): - normalization.instance_norm(inputs, data_format='NHCW') - - def testParamsShapeNotFullyDefinedNCHW(self): - inputs = array_ops.placeholder(dtypes.float32, shape=(3, None, 4)) - with self.assertRaisesRegexp(ValueError, 'undefined channels dimension'): - normalization.instance_norm(inputs, data_format='NCHW') - - def testParamsShapeNotFullyDefinedNHWC(self): - inputs = array_ops.placeholder(dtypes.float32, shape=(3, 4, None)) - with self.assertRaisesRegexp(ValueError, 'undefined channels dimension'): - normalization.instance_norm(inputs, data_format='NHWC') - - def testCreateOp(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - output = normalization.instance_norm(images) - print('name: ', output.op.name) - self.assertStartsWith( - output.op.name, 'InstanceNorm/instancenorm') - self.assertListEqual([5, height, width, 3], output.shape.as_list()) - - def testCreateOpFloat64(self): - height, width = 3, 3 - images = random_ops.random_uniform( - (5, height, width, 3), dtype=dtypes.float64, seed=1) - output = normalization.instance_norm(images) - self.assertStartsWith( - output.op.name, 'InstanceNorm/instancenorm') - self.assertListEqual([5, height, width, 3], output.shape.as_list()) - - def testCreateOpNoScaleCenter(self): - height, width = 3, 3 - images = random_ops.random_uniform( - (5, height, width, 3), dtype=dtypes.float64, seed=1) - output = normalization.instance_norm(images, center=False, scale=False) - self.assertStartsWith( - output.op.name, 'InstanceNorm/instancenorm') - self.assertListEqual([5, height, width, 3], output.shape.as_list()) - self.assertEqual(0, len(contrib_variables.get_variables_by_name('beta'))) - self.assertEqual(0, len(contrib_variables.get_variables_by_name('gamma'))) - - def testCreateVariables(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - normalization.instance_norm(images, center=True, scale=True) - beta = contrib_variables.get_variables_by_name('beta')[0] - gamma = contrib_variables.get_variables_by_name('gamma')[0] - self.assertEqual('InstanceNorm/beta', beta.op.name) - self.assertEqual('InstanceNorm/gamma', gamma.op.name) - - def testReuseVariables(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 3), seed=1) - normalization.instance_norm(images, scale=True, scope='IN') - normalization.instance_norm(images, scale=True, scope='IN', reuse=True) - beta = contrib_variables.get_variables_by_name('beta') - gamma = contrib_variables.get_variables_by_name('gamma') - self.assertEqual(1, len(beta)) - self.assertEqual(1, len(gamma)) - - def testValueCorrectWithReuseVars(self): - height, width = 3, 3 - image_shape = (10, height, width, 3) - images = random_ops.random_uniform(image_shape, seed=1) - output_train = normalization.instance_norm(images, scope='IN') - output_eval = normalization.instance_norm(images, scope='IN', reuse=True) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - # output_train and output_eval should be the same. - train_np, eval_np = sess.run([output_train, output_eval]) - self.assertAllClose(train_np, eval_np) - - def doOutputTest(self, input_shape, data_format, tol=1e-3): - axis = -1 if data_format == 'NHWC' else 1 - for mu in (0.0, 1e2): - for sigma in (1.0, 0.1): - # Determine shape of Tensor after normalization. - reduced_shape = (input_shape[0], input_shape[axis]) - expected_mean = np.zeros(reduced_shape) - expected_var = np.ones(reduced_shape) - - # Determine axes that will be normalized. - reduced_axes = list(range(len(input_shape))) - del reduced_axes[axis] - del reduced_axes[0] - reduced_axes = tuple(reduced_axes) - - inputs = random_ops.random_uniform(input_shape, seed=0) * sigma + mu - output_op = normalization.instance_norm( - inputs, center=False, scale=False, data_format=data_format) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - outputs = sess.run(output_op) - # Make sure that there are no NaNs - self.assertFalse(np.isnan(outputs).any()) - mean = np.mean(outputs, axis=reduced_axes) - var = np.var(outputs, axis=reduced_axes) - # The mean and variance of each example should be close to 0 and 1 - # respectively. - self.assertAllClose(expected_mean, mean, rtol=tol, atol=tol) - self.assertAllClose(expected_var, var, rtol=tol, atol=tol) - - def testOutputSmallInput4DNHWC(self): - self.doOutputTest((10, 10, 10, 30), 'NHWC', tol=1e-2) - - def testOutputSmallInput4DNCHW(self): - self.doOutputTest((10, 10, 10, 30), 'NCHW', tol=1e-2) - - def testOutputBigInput4DNHWC(self): - self.doOutputTest((1, 100, 100, 1), 'NHWC', tol=1e-3) - - def testOutputBigInput4DNCHW(self): - self.doOutputTest((1, 100, 100, 1), 'NCHW', tol=1e-3) - - def testOutputSmallInput5DNHWC(self): - self.doOutputTest((10, 10, 10, 10, 30), 'NHWC', tol=1e-2) - - def testOutputSmallInput5DNCHW(self): - self.doOutputTest((10, 10, 10, 10, 30), 'NCHW', tol=1e-2) - - def testOutputBigInput5DNHWC(self): - self.doOutputTest((1, 100, 100, 1, 1), 'NHWC', tol=1e-3) - - def testOutputBigInput5DNCHW(self): - self.doOutputTest((1, 100, 100, 1, 1), 'NCHW', tol=1e-3) - - -class GroupNormTest(test.TestCase): - - def testInvalidGroupSize(self): - inputs = array_ops.placeholder(dtypes.float32, shape=(5, 2, 10, 10)) - with self.assertRaisesRegexp(ValueError, - 'Invalid groups 10 for 2 channels.'): - normalization.group_norm(inputs, groups=10, - reduction_axes=[-2, -1], channels_axis=-3) - - def testBadCommensurateGroup(self): - inputs = array_ops.placeholder(dtypes.float32, shape=(5, 4, 10, 10)) - with self.assertRaisesRegexp(ValueError, - '4 channels is not commensurate with ' - '3 groups.'): - normalization.group_norm(inputs, groups=3, - reduction_axes=[-2, -1], channels_axis=-3) - - def testAxisIsBad(self): - inputs = array_ops.placeholder(dtypes.float32, shape=(1, 2, 4, 5)) - with self.assertRaisesRegexp(ValueError, - 'Axis is out of bounds.'): - normalization.group_norm(inputs, channels_axis=5) - with self.assertRaisesRegexp(ValueError, - 'Axis is out of bounds.'): - normalization.group_norm(inputs, reduction_axes=[1, 5]) - - def testNotMutuallyExclusiveAxis(self): - inputs = array_ops.placeholder(dtypes.float32, shape=(10, 32, 32, 32)) - # Specify axis with negative values. - with self.assertRaisesRegexp(ValueError, 'mutually exclusive'): - normalization.group_norm(inputs, channels_axis=-2, reduction_axes=[-2]) - # Specify axis with positive values. - with self.assertRaisesRegexp(ValueError, 'mutually exclusive'): - normalization.group_norm(inputs, channels_axis=1, reduction_axes=[1, 3]) - # Specify axis with mixed positive and negative values. - with self.assertRaisesRegexp(ValueError, 'mutually exclusive'): - normalization.group_norm(inputs, channels_axis=-2, reduction_axes=[2]) - - def testUnknownShape(self): - inputs = array_ops.placeholder(dtypes.float32) - with self.assertRaisesRegexp(ValueError, 'undefined rank'): - normalization.group_norm(inputs) - - def testParamsShapeNotFullyDefinedReductionAxes(self): - inputs = array_ops.placeholder(dtypes.float32, shape=(1, 32, None, 4)) - with self.assertRaisesRegexp(ValueError, 'undefined dimensions'): - normalization.group_norm(inputs) - - def testParamsShapeNotFullyDefinedChannelsAxis(self): - inputs = array_ops.placeholder(dtypes.float32, shape=(1, 3, 4, None)) - with self.assertRaisesRegexp(ValueError, 'undefined channel dimension'): - normalization.group_norm(inputs, channels_axis=-1, - reduction_axes=[-3, -2]) - - def testParamsShapeNotFullyDefinedBatchAxis(self): - height, width, groups = 3, 3, 4 - inputs = array_ops.placeholder(dtypes.float32, - shape=(None, height, width, 2*groups)) - output = normalization.group_norm(inputs, channels_axis=-1, - reduction_axes=[-3, -2], groups=groups) - self.assertListEqual([None, height, width, 2 * groups], - output.shape.as_list()) - - def testCreateOp(self): - height, width, groups = 3, 3, 4 - images = random_ops.random_uniform((5, height, width, 2*groups), seed=1) - output = normalization.group_norm(images, groups=groups, channels_axis=-1, - reduction_axes=[-3, -2]) - print('name: ', output.op.name) - self.assertListEqual([5, height, width, 2*groups], output.shape.as_list()) - - def testCreateOpFloat64(self): - height, width, groups = 3, 3, 5 - images = random_ops.random_uniform( - (5, height, width, 4*groups), dtype=dtypes.float64, seed=1) - output = normalization.group_norm(images, groups=groups) - self.assertEqual(dtypes.float64, output.dtype) - self.assertListEqual([5, height, width, 4*groups], output.shape.as_list()) - - def testCreateOpNoScaleCenter(self): - height, width, groups = 3, 3, 7 - images = random_ops.random_uniform( - (5, height, width, 3*groups), dtype=dtypes.float32, seed=1) - output = normalization.group_norm(images, groups=groups, center=False, - scale=False) - self.assertListEqual([5, height, width, 3*groups], output.shape.as_list()) - self.assertEqual(0, len(contrib_variables.get_variables_by_name('beta'))) - self.assertEqual(0, len(contrib_variables.get_variables_by_name('gamma'))) - - def testCreateVariables_NHWC(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 8), seed=1) - normalization.group_norm(images, groups=4, - channels_axis=-1, reduction_axes=(-3, -2), - center=True, scale=True) - beta = contrib_variables.get_variables_by_name('beta')[0] - gamma = contrib_variables.get_variables_by_name('gamma')[0] - self.assertEqual('GroupNorm/beta', beta.op.name) - self.assertEqual('GroupNorm/gamma', gamma.op.name) - - def testCreateVariables_NCHW(self): - height, width, groups = 3, 3, 4 - images = random_ops.random_uniform((5, 2*groups, height, width), seed=1) - normalization.group_norm(images, groups=4, - channels_axis=-3, reduction_axes=(-2, -1), - center=True, scale=True) - beta = contrib_variables.get_variables_by_name('beta')[0] - gamma = contrib_variables.get_variables_by_name('gamma')[0] - self.assertEqual('GroupNorm/beta', beta.op.name) - self.assertEqual('GroupNorm/gamma', gamma.op.name) - - def testReuseVariables(self): - height, width = 3, 3 - images = random_ops.random_uniform((5, height, width, 4), seed=1) - normalization.group_norm(images, groups=2, scale=True, scope='IN') - normalization.group_norm(images, groups=2, scale=True, scope='IN', - reuse=True) - beta = contrib_variables.get_variables_by_name('beta') - gamma = contrib_variables.get_variables_by_name('gamma') - self.assertEqual(1, len(beta)) - self.assertEqual(1, len(gamma)) - - def testValueCorrectWithReuseVars(self): - height, width = 3, 3 - image_shape = (10, height, width, 4) - images = random_ops.random_uniform(image_shape, seed=1) - output_train = normalization.group_norm(images, groups=2, scope='IN') - output_eval = normalization.group_norm(images, groups=2, scope='IN', - reuse=True) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - # output_train and output_eval should be the same. - train_np, eval_np = sess.run([output_train, output_eval]) - self.assertAllClose(train_np, eval_np) - - def doOutputTest(self, - input_shape, - channels_axis=None, - reduction_axes=None, - mean_close_to_zero=False, - groups=2, - tol=1e-2): - # Select the axis for the channel and the dimensions along which statistics - # are accumulated. - if channels_axis < 0: - channels_axis += len(input_shape) - reduced_axes = [channels_axis + 1] - for a in reduction_axes: - if a < 0: - a += len(input_shape) - if a < channels_axis: - reduced_axes.append(a) - else: - reduced_axes.append(a+1) - reduced_axes = tuple(reduced_axes) - - # Calculate the final shape for the output Tensor. - axes_before_channels = input_shape[:channels_axis] - axes_after_channels = input_shape[channels_axis+1:] - channels = input_shape[channels_axis] - outputs_shape = (axes_before_channels + [groups, channels // groups] + - axes_after_channels) - - # Calculate the final shape for the output statistics. - reduced_shape = [] - for i, a in enumerate(outputs_shape): - if i not in reduced_axes: - reduced_shape.append(a) - - if mean_close_to_zero: - mu_tuple = (1e-4, 1e-2, 1.0) - sigma_tuple = (1e-2, 0.1, 1.0) - else: - mu_tuple = (1.0, 1e2) - sigma_tuple = (1.0, 0.1) - - for mu in mu_tuple: - for sigma in sigma_tuple: - # Determine shape of Tensor after normalization. - expected_mean = np.zeros(reduced_shape) - expected_var = np.ones(reduced_shape) - - inputs = random_ops.random_normal(input_shape, seed=0) * sigma + mu - output_op = normalization.group_norm( - inputs, - groups=groups, - center=False, - scale=False, - channels_axis=channels_axis, - reduction_axes=reduction_axes, - mean_close_to_zero=mean_close_to_zero) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - outputs = sess.run(output_op) - # Make sure that there are no NaNs - self.assertFalse(np.isnan(outputs).any()) - - outputs = np.reshape(outputs, outputs_shape) - mean = np.mean(outputs, axis=reduced_axes) - var = np.var(outputs, axis=reduced_axes) - # The mean and variance of each example should be close to 0 and 1 - # respectively. - self.assertAllClose(expected_mean, mean, rtol=tol, atol=tol) - self.assertAllClose(expected_var, var, rtol=tol, atol=tol) - - def doOutputTestForMeanCloseToZero(self, - input_shape, - channels_axis=None, - reduction_axes=None, - groups=2, - tol=5e-2): - self.doOutputTest( - input_shape, - channels_axis=channels_axis, - reduction_axes=reduction_axes, - groups=groups, - tol=tol, - mean_close_to_zero=True) - - def testOutputSmallInput4D_NHWC(self): - input_shape = [10, 10, 10, 30] - # Specify axes with positive values. - self.doOutputTest(input_shape, channels_axis=3, reduction_axes=[1, 2]) - # Specify axes with negative values. - self.doOutputTest(input_shape, channels_axis=-1, reduction_axes=[-3, -2]) - # Specify axes with positive values. - self.doOutputTestForMeanCloseToZero( - input_shape, channels_axis=3, reduction_axes=[1, 2]) - # Specify axes with negative values. - self.doOutputTestForMeanCloseToZero( - input_shape, channels_axis=-1, reduction_axes=[-3, -2]) - - def testOutputSmallInput3D_NHWC(self): - input_shape = [10, 10, 30] - # Specify axes with positive values. - self.doOutputTest(input_shape, channels_axis=2, reduction_axes=[0, 1]) - # Specify axes with negative values. - self.doOutputTest(input_shape, channels_axis=-1, reduction_axes=[-3, -2]) - # Specify axes with positive values. - self.doOutputTestForMeanCloseToZero( - input_shape, channels_axis=2, reduction_axes=[0, 1]) - # Specify axes with negative values. - self.doOutputTestForMeanCloseToZero( - input_shape, channels_axis=-1, reduction_axes=[-3, -2]) - - def testOutputSmallInput4D_NCHW(self): - input_shape = [10, 10, 10, 30] - # Specify axes with positive values. - self.doOutputTest(input_shape, channels_axis=1, reduction_axes=[2, 3]) - # Specify axes with negative values. - self.doOutputTest(input_shape, channels_axis=-3, reduction_axes=[-2, -1]) - # Specify axes with positive values. - self.doOutputTestForMeanCloseToZero( - input_shape, channels_axis=1, reduction_axes=[2, 3]) - # Specify axes with negative values. - self.doOutputTestForMeanCloseToZero( - input_shape, channels_axis=-3, reduction_axes=[-2, -1]) - - def testOutputSmallInput3D_NCHW(self): - input_shape = [10, 10, 30] - # Specify axes with positive values. - self.doOutputTest(input_shape, channels_axis=0, reduction_axes=[1, 2]) - # Specify axes with negative values. - self.doOutputTest(input_shape, channels_axis=-3, reduction_axes=[-2, -1]) - # Specify axes with positive values. - self.doOutputTestForMeanCloseToZero( - input_shape, channels_axis=0, reduction_axes=[1, 2]) - # Specify axes with negative values. - self.doOutputTestForMeanCloseToZero( - input_shape, channels_axis=-3, reduction_axes=[-2, -1]) - - def testOutputBigInput4D_NHWC(self): - self.doOutputTest( - [5, 100, 100, 1], channels_axis=3, reduction_axes=[1, 2], groups=1) - self.doOutputTestForMeanCloseToZero( - [5, 100, 100, 1], channels_axis=3, reduction_axes=[1, 2], groups=1) - - def testOutputBigInput4D_NCHW(self): - self.doOutputTest( - [1, 100, 100, 4], channels_axis=1, reduction_axes=[2, 3], groups=4) - self.doOutputTestForMeanCloseToZero( - [1, 100, 100, 4], channels_axis=1, reduction_axes=[2, 3], groups=4) - - def testOutputSmallInput2D_NC(self): - self.doOutputTest( - [10, 7 * 100], channels_axis=1, reduction_axes=[], groups=7) - self.doOutputTestForMeanCloseToZero( - [10, 7 * 100], channels_axis=1, reduction_axes=[], groups=7) - - def testOutputSmallInput5D_NCXXX(self): - self.doOutputTest( - [10, 10, 20, 40, 5], - channels_axis=1, - reduction_axes=[2, 3, 4], - groups=5) - self.doOutputTestForMeanCloseToZero( - [10, 10, 20, 40, 5], - channels_axis=1, - reduction_axes=[2, 3, 4], - groups=5) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/layers/python/layers/optimizers.py b/tensorflow/contrib/layers/python/layers/optimizers.py deleted file mode 100644 index 2c18bfa7b91..00000000000 --- a/tensorflow/contrib/layers/python/layers/optimizers.py +++ /dev/null @@ -1,440 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Optimizer ops for use in layers and tf.learn.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import six - -from tensorflow.contrib import framework as contrib_framework -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.ops import variables as vars_ -from tensorflow.python.summary import summary -from tensorflow.python.training import moving_averages -from tensorflow.python.training import optimizer as optimizer_ -from tensorflow.python.training import training as train - -OPTIMIZER_CLS_NAMES = { - "Adagrad": train.AdagradOptimizer, - "Adam": train.AdamOptimizer, - "Ftrl": train.FtrlOptimizer, - "Momentum": lambda learning_rate: train.MomentumOptimizer(learning_rate, momentum=0.9), # pylint: disable=line-too-long - "RMSProp": train.RMSPropOptimizer, - "SGD": train.GradientDescentOptimizer, -} - -OPTIMIZER_SUMMARIES = [ - "learning_rate", - "loss", - "gradients", - "gradient_norm", - "global_gradient_norm", -] - - -def optimize_loss(loss, - global_step, - learning_rate, - optimizer, - gradient_noise_scale=None, - gradient_multipliers=None, - clip_gradients=None, - learning_rate_decay_fn=None, - update_ops=None, - variables=None, - name=None, - summaries=None, - colocate_gradients_with_ops=False, - increment_global_step=True): - """Given loss and parameters for optimizer, returns a training op. - - Various ways of passing optimizers include: - - - by string specifying the name of the optimizer. See OPTIMIZER_CLS_NAMES - for full list. E.g. `optimize_loss(..., optimizer='Adam')`. - - by function taking learning rate `Tensor` as argument and returning an - `Optimizer` instance. E.g. `optimize_loss(..., - optimizer=lambda lr: tf.compat.v1.train.MomentumOptimizer(lr, - momentum=0.5))`. - Alternatively, if `learning_rate` is `None`, the function takes no - arguments. E.g. `optimize_loss(..., learning_rate=None, - optimizer=lambda: tf.compat.v1.train.MomentumOptimizer(0.5, - momentum=0.5))`. - - by a subclass of `Optimizer` having a single-argument constructor - (the argument is the learning rate), such as AdamOptimizer or - AdagradOptimizer. E.g. `optimize_loss(..., - optimizer=tf.compat.v1.train.AdagradOptimizer)`. - - by an instance of a subclass of `Optimizer`. - E.g., `optimize_loss(..., - optimizer=tf.compat.v1.train.AdagradOptimizer(0.5))`. - - Args: - loss: Scalar `Tensor`. - global_step: Scalar int `Tensor`, step counter to update on each step unless - `increment_global_step` is `False`. If not supplied, it will be fetched - from the default graph (see `tf.compat.v1.train.get_global_step` for - details). If it has not been created, no step will be incremented with - each weight update. `learning_rate_decay_fn` requires `global_step`. - learning_rate: float or `Tensor`, magnitude of update per each training - step. Can be `None`. - optimizer: string, class or optimizer instance, used as trainer. string - should be name of optimizer, like 'SGD', 'Adam', 'Adagrad'. Full list in - OPTIMIZER_CLS_NAMES constant. class should be sub-class of `tf.Optimizer` - that implements `compute_gradients` and `apply_gradients` functions. - optimizer instance should be instantiation of `tf.Optimizer` sub-class and - have `compute_gradients` and `apply_gradients` functions. - gradient_noise_scale: float or None, adds 0-mean normal noise scaled by this - value. - gradient_multipliers: dict of variables or variable names to floats. If - present, gradients for specified variables will be multiplied by given - constant. - clip_gradients: float, callable or `None`. If a float is provided, a global - clipping is applied to prevent the norm of the gradient from exceeding - this value. Alternatively, a callable can be provided, e.g., - `adaptive_clipping_fn()`. This callable takes a list of `(gradients, - variables)` tuples and returns the same thing with the gradients modified. - learning_rate_decay_fn: function, takes `learning_rate` and `global_step` - `Tensor`s, returns `Tensor`. Can be used to implement any learning rate - decay functions. - For example: `tf.compat.v1.train.exponential_decay`. - Ignored if `learning_rate` is not supplied. - update_ops: list of update `Operation`s to execute at each step. If `None`, - uses elements of UPDATE_OPS collection. The order of execution between - `update_ops` and `loss` is non-deterministic. - variables: list of variables to optimize or `None` to use all trainable - variables. - name: The name for this operation is used to scope operations and summaries. - summaries: List of internal quantities to visualize on tensorboard. If not - set, the loss, the learning rate, and the global norm of the gradients - will be reported. The complete list of possible values is in - OPTIMIZER_SUMMARIES. - colocate_gradients_with_ops: If True, try colocating gradients with the - corresponding op. - increment_global_step: Whether to increment `global_step`. If your model - calls `optimize_loss` multiple times per training step (e.g. to optimize - different parts of the model), use this arg to avoid incrementing - `global_step` more times than necessary. - - Returns: - Training op. - - Raises: - ValueError: if: - * `loss` is an invalid type or shape. - * `global_step` is an invalid type or shape. - * `learning_rate` is an invalid type or value. - * `optimizer` has the wrong type. - * `clip_gradients` is neither float nor callable. - * `learning_rate` and `learning_rate_decay_fn` are supplied, but no - `global_step` is available. - * `gradients` is empty. - """ - loss = ops.convert_to_tensor(loss) - contrib_framework.assert_scalar(loss) - if global_step is None: - global_step = train.get_global_step() - else: - train.assert_global_step(global_step) - with vs.variable_scope(name, "OptimizeLoss", [loss, global_step]): - # Update ops take UPDATE_OPS collection if not provided. - if update_ops is None: - update_ops = set(ops.get_collection(ops.GraphKeys.UPDATE_OPS)) - # Make sure update ops are ran before computing loss. - if update_ops: - loss = control_flow_ops.with_dependencies(list(update_ops), loss) - - # Learning rate variable, with possible decay. - lr = None - if learning_rate is not None: - if (isinstance(learning_rate, ops.Tensor) and - learning_rate.get_shape().ndims == 0): - lr = learning_rate - elif isinstance(learning_rate, float): - if learning_rate < 0.0: - raise ValueError("Invalid learning_rate %s.", learning_rate) - lr = vs.get_variable( - "learning_rate", [], - trainable=False, - initializer=init_ops.constant_initializer(learning_rate)) - else: - raise ValueError("Learning rate should be 0d Tensor or float. " - "Got %s of type %s" % - (str(learning_rate), str(type(learning_rate)))) - if summaries is None: - summaries = ["loss", "learning_rate", "global_gradient_norm"] - else: - for summ in summaries: - if summ not in OPTIMIZER_SUMMARIES: - raise ValueError("Summaries should be one of [%s], you provided %s." % - (", ".join(OPTIMIZER_SUMMARIES), summ)) - if learning_rate is not None and learning_rate_decay_fn is not None: - if global_step is None: - raise ValueError("global_step is required for learning_rate_decay_fn.") - lr = learning_rate_decay_fn(lr, global_step) - if "learning_rate" in summaries: - summary.scalar("learning_rate", lr) - - # Create optimizer, given specified parameters. - if isinstance(optimizer, six.string_types): - if lr is None: - raise ValueError("Learning rate is None, but should be specified if " - "optimizer is string (%s)." % optimizer) - if optimizer not in OPTIMIZER_CLS_NAMES: - raise ValueError( - "Optimizer name should be one of [%s], you provided %s." % - (", ".join(OPTIMIZER_CLS_NAMES), optimizer)) - opt = OPTIMIZER_CLS_NAMES[optimizer](learning_rate=lr) - elif (isinstance(optimizer, type) and - issubclass(optimizer, optimizer_.Optimizer)): - if lr is None: - raise ValueError("Learning rate is None, but should be specified if " - "optimizer is class (%s)." % optimizer) - opt = optimizer(learning_rate=lr) - elif isinstance(optimizer, optimizer_.Optimizer): - opt = optimizer - elif callable(optimizer): - if learning_rate is not None: - opt = optimizer(lr) - else: - opt = optimizer() - if not isinstance(opt, optimizer_.Optimizer): - raise ValueError("Unrecognized optimizer: function should return " - "subclass of Optimizer. Got %s." % str(opt)) - else: - raise ValueError("Unrecognized optimizer: should be string, " - "subclass of Optimizer, instance of " - "subclass of Optimizer or function with one argument. " - "Got %s." % str(optimizer)) - - # All trainable variables, if specific variables are not specified. - if variables is None: - variables = vars_.trainable_variables() - - # Compute gradients. - gradients = opt.compute_gradients( - loss, - variables, - colocate_gradients_with_ops=colocate_gradients_with_ops) - - # Optionally add gradient noise. - if gradient_noise_scale is not None: - gradients = _add_scaled_noise_to_gradients(gradients, - gradient_noise_scale) - - # Multiply some gradients. - if gradient_multipliers is not None: - gradients = _multiply_gradients(gradients, gradient_multipliers) - if not gradients: - raise ValueError( - "Empty list of (gradient, var) pairs encountered. This is most " - "likely to be caused by an improper value of gradient_multipliers.") - - if "global_gradient_norm" in summaries or "gradient_norm" in summaries: - summary.scalar("global_norm/gradient_norm", - clip_ops.global_norm(list(zip(*gradients))[0])) - - # Optionally clip gradients by global norm. - if isinstance(clip_gradients, float): - gradients = _clip_gradients_by_norm(gradients, clip_gradients) - elif callable(clip_gradients): - gradients = clip_gradients(gradients) - elif clip_gradients is not None: - raise ValueError("Unknown type %s for clip_gradients" % - type(clip_gradients)) - - # Add scalar summary for loss. - if "loss" in summaries: - summary.scalar("loss", loss) - - # Add histograms for variables, gradients and gradient norms. - for gradient, variable in gradients: - if isinstance(gradient, ops.IndexedSlices): - grad_values = gradient.values - else: - grad_values = gradient - - if grad_values is not None: - var_name = variable.name.replace(":", "_") - if "gradients" in summaries: - summary.histogram("gradients/%s" % var_name, grad_values) - if "gradient_norm" in summaries: - summary.scalar("gradient_norm/%s" % var_name, - clip_ops.global_norm([grad_values])) - - if clip_gradients is not None and ("global_gradient_norm" in summaries or - "gradient_norm" in summaries): - summary.scalar("global_norm/clipped_gradient_norm", - clip_ops.global_norm(list(zip(*gradients))[0])) - - # Create gradient updates. - grad_updates = opt.apply_gradients( - gradients, - global_step=global_step if increment_global_step else None, - name="train") - - # Ensure the train_tensor computes grad_updates. - train_tensor = control_flow_ops.with_dependencies([grad_updates], loss) - - return train_tensor - - -def _clip_gradients_by_norm(grads_and_vars, clip_gradients): - """Clips gradients by global norm.""" - gradients, variables = zip(*grads_and_vars) - clipped_gradients, _ = clip_ops.clip_by_global_norm(gradients, clip_gradients) - return list(zip(clipped_gradients, variables)) - - -def _adaptive_max_norm(norm, std_factor, decay, global_step, epsilon, name): - """Find max_norm given norm and previous average.""" - with vs.variable_scope(name, "AdaptiveMaxNorm", [norm]): - log_norm = math_ops.log(norm + epsilon) - - def moving_average(name, value, decay): - moving_average_variable = vs.get_variable( - name, - shape=value.get_shape(), - dtype=value.dtype, - initializer=init_ops.zeros_initializer(), - trainable=False) - return moving_averages.assign_moving_average( - moving_average_variable, value, decay, zero_debias=False) - - # quicker adaptation at the beginning - if global_step is not None: - n = math_ops.cast(global_step, dtypes.float32) - decay = math_ops.minimum(decay, n / (n + 1.)) - - # update averages - mean = moving_average("mean", log_norm, decay) - sq_mean = moving_average("sq_mean", math_ops.square(log_norm), decay) - - variance = sq_mean - math_ops.square(mean) - std = math_ops.sqrt(math_ops.maximum(epsilon, variance)) - max_norms = math_ops.exp(mean + std_factor * std) - return max_norms, mean - - -def adaptive_clipping_fn(std_factor=2., - decay=0.95, - static_max_norm=None, - global_step=None, - report_summary=False, - epsilon=1e-8, - name=None): - """Adapt the clipping value using statistics on the norms. - - Implement adaptive gradient as presented in section 3.2.1 of - https://arxiv.org/abs/1412.1602. - - Keeps a moving average of the mean and std of the log(norm) of the gradient. - If the norm exceeds `exp(mean + std_factor*std)` then all gradients will be - rescaled such that the global norm becomes `exp(mean)`. - - Args: - std_factor: Python scaler (or tensor). `max_norm = exp(mean + - std_factor*std)` - decay: The smoothing factor of the moving averages. - static_max_norm: If provided, will threshold the norm to this value as an - extra safety. - global_step: Optional global_step. If provided, `decay = decay*n/(n+1)`. - This provides a quicker adaptation of the mean for the first steps. - report_summary: If `True`, will add histogram summaries of the `max_norm`. - epsilon: Small value chosen to avoid zero variance. - name: The name for this operation is used to scope operations and summaries. - - Returns: - A function for applying gradient clipping. - """ - - def gradient_clipping(grads_and_vars): - """Internal function for adaptive clipping.""" - grads, variables = zip(*grads_and_vars) - - norm = clip_ops.global_norm(grads) - - max_norm, log_mean = _adaptive_max_norm(norm, std_factor, decay, - global_step, epsilon, name) - - # reports the max gradient norm for debugging - if report_summary: - summary.scalar("global_norm/adaptive_max_gradient_norm", max_norm) - - # factor will be 1. if norm is smaller than max_norm - factor = array_ops.where(norm < max_norm, array_ops.ones_like(norm), - math_ops.exp(log_mean) / norm) - - if static_max_norm is not None: - factor = math_ops.minimum(static_max_norm / norm, factor) - - # apply factor - clipped_grads = [] - for grad in grads: - if grad is None: - clipped_grads.append(None) - elif isinstance(grad, ops.IndexedSlices): - clipped_grads.append( - ops.IndexedSlices(grad.values * factor, grad.indices, - grad.dense_shape)) - else: - clipped_grads.append(grad * factor) - - return list(zip(clipped_grads, variables)) - - return gradient_clipping - - -def _add_scaled_noise_to_gradients(grads_and_vars, gradient_noise_scale): - """Adds scaled noise from a 0-mean normal distribution to gradients.""" - gradients, variables = zip(*grads_and_vars) - noisy_gradients = [] - for gradient in gradients: - if gradient is None: - noisy_gradients.append(None) - continue - if isinstance(gradient, ops.IndexedSlices): - gradient_shape = gradient.dense_shape - else: - gradient_shape = gradient.get_shape() - noise = random_ops.truncated_normal(gradient_shape) * gradient_noise_scale - noisy_gradients.append(gradient + noise) - return list(zip(noisy_gradients, variables)) - - -def _multiply_gradients(grads_and_vars, gradient_multipliers): - """Multiply specified gradients.""" - multiplied_grads_and_vars = [] - for grad, var in grads_and_vars: - if (grad is not None and - (var in gradient_multipliers or var.name in gradient_multipliers)): - key = var if var in gradient_multipliers else var.name - multiplier = gradient_multipliers[key] - if isinstance(grad, ops.IndexedSlices): - grad_values = grad.values * multiplier - grad = ops.IndexedSlices(grad_values, grad.indices, grad.dense_shape) - else: - grad *= math_ops.cast(multiplier, grad.dtype) - multiplied_grads_and_vars.append((grad, var)) - return multiplied_grads_and_vars diff --git a/tensorflow/contrib/layers/python/layers/optimizers_test.py b/tensorflow/contrib/layers/python/layers/optimizers_test.py deleted file mode 100644 index b4d1239e768..00000000000 --- a/tensorflow/contrib/layers/python/layers/optimizers_test.py +++ /dev/null @@ -1,537 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for optimizers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.layers.python.layers import optimizers as optimizers_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import gradient_descent - - -def _setup_model(): - x = array_ops.placeholder(dtypes.float32, []) - var = variable_scope.get_variable( - "test", [], initializer=init_ops.constant_initializer(10)) - loss = math_ops.abs(var * x) - global_step = variable_scope.get_variable( - "global_step", [], - trainable=False, - dtype=dtypes.int64, - initializer=init_ops.constant_initializer( - 0, dtype=dtypes.int64)) - return x, var, loss, global_step - - -def _no_op_learning_rate_decay_fn(lr, global_step): - assert lr is not None - assert global_step is not None - return lr - - -class OptimizersTest(test.TestCase): - - def testSGDOptimizer(self): - optimizers = [ - "SGD", gradient_descent.GradientDescentOptimizer, - gradient_descent.GradientDescentOptimizer(learning_rate=0.1), - lambda lr: gradient_descent.GradientDescentOptimizer(learning_rate=lr), - "Momentum" - ] - for optimizer in optimizers: - with ops.Graph().as_default() as g: - with self.session(graph=g) as session: - x, var, loss, global_step = _setup_model() - train = optimizers_lib.optimize_loss( - loss, global_step, learning_rate=0.1, optimizer=optimizer) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5}) - var_value, global_step_value = session.run([var, global_step]) - self.assertEqual(var_value, 9.5) - self.assertEqual(global_step_value, 1) - - def testNoLrCallable(self): - - def optimizer_fn(): - return gradient_descent.GradientDescentOptimizer(learning_rate=0.1) - - with ops.Graph().as_default() as g: - with self.session(graph=g) as session: - x, var, loss, global_step = _setup_model() - train = optimizers_lib.optimize_loss( - loss, global_step, learning_rate=None, optimizer=optimizer_fn) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5}) - var_value, global_step_value = session.run([var, global_step]) - self.assertEqual(var_value, 9.5) - self.assertEqual(global_step_value, 1) - - def testWrongOptimizer(self): - optimizers = ["blah", variables.Variable, object(), lambda x: None] - for optimizer in optimizers: - with ops.Graph().as_default() as g: - with self.session(graph=g): - _, _, loss, global_step = _setup_model() - with self.assertRaises(ValueError): - optimizers_lib.optimize_loss( - loss, global_step, learning_rate=0.1, optimizer=optimizer) - - def testBadSummaries(self): - with ops.Graph().as_default() as g, self.session(graph=g): - _, _, loss, global_step = _setup_model() - with self.assertRaises(ValueError): - optimizers_lib.optimize_loss( - loss, global_step, learning_rate=0.1, optimizer="SGD", - summaries=["loss", "bad_summary"]) - - def testInvalidLoss(self): - with ops.Graph().as_default() as g, self.session(graph=g): - _, _, _, global_step = _setup_model() - with self.assertRaises(ValueError): - optimizers_lib.optimize_loss( - None, global_step, learning_rate=0.1, optimizer="SGD") - with self.assertRaises(ValueError): - optimizers_lib.optimize_loss( - [[1.0]], global_step, learning_rate=0.1, optimizer="SGD") - - def testInvalidGlobalStep(self): - with ops.Graph().as_default() as g, self.session(graph=g): - x = array_ops.placeholder(dtypes.float32, []) - var = variable_scope.get_variable( - "test", [], initializer=init_ops.constant_initializer(10)) - loss = math_ops.abs(var * x) - with self.assertRaises(AttributeError): - optimizers_lib.optimize_loss( - loss, - global_step=constant_op.constant( - 43, dtype=dtypes.int64), - learning_rate=0.1, - optimizer="SGD") - with self.assertRaises(TypeError): - optimizers_lib.optimize_loss( - loss, - global_step=variable_scope.get_variable( - "global_step", [], - trainable=False, - dtype=dtypes.float64, - initializer=init_ops.constant_initializer( - 0.0, dtype=dtypes.float64)), - learning_rate=0.1, - optimizer="SGD") - with self.assertRaises(ValueError): - optimizers_lib.optimize_loss( - loss, - global_step=variable_scope.get_variable( - "global_step", [1], - trainable=False, - dtype=dtypes.int64, - initializer=init_ops.constant_initializer( - [0], dtype=dtypes.int64)), - learning_rate=0.1, - optimizer="SGD") - - def testInvalidLearningRate(self): - with ops.Graph().as_default() as g, self.session(graph=g): - _, _, loss, global_step = _setup_model() - with self.assertRaises(ValueError): - optimizers_lib.optimize_loss( - loss, global_step, learning_rate=-0.1, optimizer="SGD") - - def testGradientNoise(self): - random_seed.set_random_seed(42) - with self.cached_session() as session: - x, var, loss, global_step = _setup_model() - train = optimizers_lib.optimize_loss( - loss, - global_step, - learning_rate=0.1, - optimizer="SGD", - gradient_noise_scale=10.0) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5}) - var_value, global_step_value = session.run([var, global_step]) - # Due to randomness the following number may change if graph is different. - self.assertAlmostEqual(var_value, 9.86912, 4) - self.assertEqual(global_step_value, 1) - - def testGradientNoiseWithClipping(self): - random_seed.set_random_seed(42) - with self.cached_session() as session: - x, var, loss, global_step = _setup_model() - train = optimizers_lib.optimize_loss( - loss, - global_step, - learning_rate=0.1, - optimizer="SGD", - gradient_noise_scale=10.0, - clip_gradients=10.0) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5}) - var_value, global_step_value = session.run([var, global_step]) - self.assertAlmostEqual(var_value, 9.86912, 4) - self.assertEqual(global_step_value, 1) - - def testGradientClip(self): - with self.cached_session() as session: - x, var, loss, global_step = _setup_model() - train = optimizers_lib.optimize_loss( - loss, - global_step, - learning_rate=0.1, - optimizer="SGD", - clip_gradients=0.1) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5}) - var_value, global_step_value = session.run([var, global_step]) - self.assertAlmostEqual(var_value, 9.98999, 4) - self.assertEqual(global_step_value, 1) - - def testAdaptiveGradientClip(self): - with self.cached_session() as session: - x, var, loss, global_step = _setup_model() - clip_gradients = optimizers_lib.adaptive_clipping_fn() - train = optimizers_lib.optimize_loss( - loss, - global_step, - learning_rate=0.1, - optimizer="SGD", - clip_gradients=clip_gradients) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5}) - var_value, global_step_value = session.run([var, global_step]) - self.assertAlmostEqual(var_value, 9.8916, 4) - self.assertEqual(global_step_value, 1) - var_count = 0 - for var in variables.global_variables(): - if var.name.startswith("OptimizeLoss/AdaptiveMaxNorm"): - var_count += 1 - self.assertEqual(2, var_count) - - def testGradientMultiply(self): - with self.cached_session() as session: - x, var, loss, global_step = _setup_model() - train = optimizers_lib.optimize_loss( - loss, - global_step, - learning_rate=0.1, - optimizer="SGD", - gradient_multipliers={var: 7.}) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5}) - var_value, global_step_value = session.run([var, global_step]) - # var(0) = 10, x = 5, var(0)/dx = 5, - # var(1) = var(0) - learning_rate * gradient_multiplier * var(0)/dx - self.assertAlmostEqual(var_value, 6.5, 4) - self.assertEqual(global_step_value, 1) - - def testGradientMultiplyInt32Tensor(self): - with self.cached_session() as session: - x, var, loss, global_step = _setup_model() - v = array_ops.placeholder(dtypes.float32, []) - train = optimizers_lib.optimize_loss( - loss, - global_step, - learning_rate=0.1, - optimizer="SGD", - gradient_multipliers={var: v}) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5, v: 7.}) - var_value, global_step_value = session.run([var, global_step]) - # var(0) = 10, x = 5, var(0)/dx = 5, - # var(1) = var(0) - learning_rate * gradient_multiplier * var(0)/dx - self.assertAlmostEqual(var_value, 6.5, 4) - self.assertEqual(global_step_value, 1) - - def testGradientMultiplyInt64Tensor(self): - with self.cached_session() as session: - x, var, loss, global_step = _setup_model() - v = array_ops.placeholder(dtypes.float64, []) - train = optimizers_lib.optimize_loss( - loss, - global_step, - learning_rate=0.1, - optimizer="SGD", - gradient_multipliers={var: v}) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5, v: 7.}) - var_value, global_step_value = session.run([var, global_step]) - # var(0) = 10, x = 5, var(0)/dx = 5, - # var(1) = var(0) - learning_rate * gradient_multiplier * var(0)/dx - self.assertAlmostEqual(var_value, 6.5, 4) - self.assertEqual(global_step_value, 1) - - def testIgnoreVariablesWithNoGradients(self): - _, _, loss, global_step = _setup_model() - - unused_variable = variable_scope.get_variable("ignore_me", []) - - optimizers_lib.optimize_loss( - loss, - global_step, - learning_rate=0.1, - optimizer="SGD", - gradient_noise_scale=10.0, - gradient_multipliers={unused_variable: 1.}, - clip_gradients=10.0) - - def testNoGlobalStep(self): - optimizers = [ - "SGD", gradient_descent.GradientDescentOptimizer, - gradient_descent.GradientDescentOptimizer(learning_rate=0.1) - ] - for optimizer in optimizers: - with ops.Graph().as_default() as g, self.session(graph=g) as session: - x = array_ops.placeholder(dtypes.float32, []) - var = variable_scope.get_variable( - "test", [], initializer=init_ops.constant_initializer(10)) - loss = math_ops.abs(var * x) - update_var = variable_scope.get_variable( - "update", [], initializer=init_ops.constant_initializer(10)) - update_op = state_ops.assign(update_var, 20) - train = optimizers_lib.optimize_loss( - loss, - global_step=None, - learning_rate=0.1, - optimizer=optimizer, - update_ops=[update_op]) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5}) - self.assertEqual(9.5, var.eval()) - self.assertEqual(20, update_var.eval()) - - def testNoGlobalStepWithDecay(self): - optimizers = [ - "SGD", gradient_descent.GradientDescentOptimizer, - gradient_descent.GradientDescentOptimizer(learning_rate=0.1) - ] - for optimizer in optimizers: - with ops.Graph().as_default() as g, self.session(graph=g): - x = array_ops.placeholder(dtypes.float32, []) - var = variable_scope.get_variable( - "test", [], initializer=init_ops.constant_initializer(10)) - loss = math_ops.abs(var * x) - update_var = variable_scope.get_variable( - "update", [], initializer=init_ops.constant_initializer(10)) - update_op = state_ops.assign(update_var, 20) - with self.assertRaisesRegexp( - ValueError, "global_step is required for learning_rate_decay_fn"): - optimizers_lib.optimize_loss( - loss, - global_step=None, - learning_rate=0.1, - learning_rate_decay_fn=_no_op_learning_rate_decay_fn, - optimizer=optimizer, - update_ops=[update_op]) - - def testNoGlobalStepArg(self): - optimizers = [ - "SGD", gradient_descent.GradientDescentOptimizer, - gradient_descent.GradientDescentOptimizer(learning_rate=0.1) - ] - for optimizer in optimizers: - with ops.Graph().as_default() as g, self.session(graph=g) as session: - x, var, loss, global_step = _setup_model() - update_var = variable_scope.get_variable( - "update", [], initializer=init_ops.constant_initializer(10)) - update_op = state_ops.assign(update_var, 20) - train = optimizers_lib.optimize_loss( - loss, - global_step=None, - learning_rate=0.1, - optimizer=optimizer, - update_ops=[update_op]) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5}) - self.assertEqual(9.5, var.eval()) - self.assertEqual(20, update_var.eval()) - self.assertEqual(1, global_step.eval()) - - def testUpdateOp(self): - optimizers = [ - "SGD", gradient_descent.GradientDescentOptimizer, - gradient_descent.GradientDescentOptimizer(learning_rate=0.1) - ] - for optimizer in optimizers: - with ops.Graph().as_default() as g, self.session(graph=g) as session: - x, var, loss, global_step = _setup_model() - update_var = variable_scope.get_variable( - "update", [], initializer=init_ops.constant_initializer(10)) - update_op = state_ops.assign(update_var, 20) - train = optimizers_lib.optimize_loss( - loss, - global_step, - learning_rate=0.1, - optimizer=optimizer, - update_ops=[update_op]) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5}) - self.assertEqual(9.5, var.eval()) - self.assertEqual(20, update_var.eval()) - self.assertEqual(1, global_step.eval()) - - def testUpdateOpNoIncrementGlobalStep(self): - optimizers = [ - "SGD", gradient_descent.GradientDescentOptimizer, - gradient_descent.GradientDescentOptimizer(learning_rate=0.1) - ] - for optimizer in optimizers: - with ops.Graph().as_default() as g, self.session(graph=g) as session: - x, var, loss, global_step = _setup_model() - update_var = variable_scope.get_variable( - "update", [], initializer=init_ops.constant_initializer(10)) - update_op = state_ops.assign(update_var, 20) - train = optimizers_lib.optimize_loss( - loss, - global_step, - learning_rate=0.1, - optimizer=optimizer, - update_ops=[update_op], - increment_global_step=False) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5}) - self.assertEqual(9.5, var.eval()) - self.assertEqual(20, update_var.eval()) - self.assertEqual(0, global_step.eval()) - - def testUpdateOpWithNoOpDecay(self): - optimizers = [ - "SGD", gradient_descent.GradientDescentOptimizer, - gradient_descent.GradientDescentOptimizer(learning_rate=0.1) - ] - for optimizer in optimizers: - with ops.Graph().as_default() as g, self.session(graph=g) as session: - x, var, loss, global_step = _setup_model() - update_var = variable_scope.get_variable( - "update", [], initializer=init_ops.constant_initializer(10)) - update_op = state_ops.assign(update_var, 20) - train = optimizers_lib.optimize_loss( - loss, - global_step, - learning_rate=0.1, - learning_rate_decay_fn=_no_op_learning_rate_decay_fn, - optimizer=optimizer, - update_ops=[update_op]) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5}) - self.assertEqual(9.5, var.eval()) - self.assertEqual(20, update_var.eval()) - self.assertEqual(1, global_step.eval()) - - def testUpdateOpFromCollection(self): - optimizers = [ - "SGD", gradient_descent.GradientDescentOptimizer, - gradient_descent.GradientDescentOptimizer(learning_rate=0.1) - ] - for optimizer in optimizers: - with ops.Graph().as_default() as g, self.session(graph=g) as session: - x, var, loss, global_step = _setup_model() - update_var = variable_scope.get_variable( - "update", [], initializer=init_ops.constant_initializer(10)) - update_op = state_ops.assign(update_var, 20) - ops.add_to_collection(ops.GraphKeys.UPDATE_OPS, update_op) - train = optimizers_lib.optimize_loss( - loss, global_step, learning_rate=0.1, optimizer=optimizer) - variables.global_variables_initializer().run() - session.run(train, feed_dict={x: 5}) - var_value, update_var_value, global_step_value = session.run( - [var, update_var, global_step]) - self.assertEqual(var_value, 9.5) - self.assertEqual(update_var_value, 20) - self.assertEqual(global_step_value, 1) - - -class AdaptiveClipping(test.TestCase): - - def testAverages(self): - with self.cached_session() as session: - scale = 2. - grad = array_ops.ones([3, 4]) * scale - log_norm = np.log(np.sqrt(scale**2 * grad.get_shape().num_elements())) - grads_and_vars = [(grad, grad)] - grads_and_vars = optimizers_lib.adaptive_clipping_fn( - decay=0.5)(grads_and_vars) - - var_dict = {} - for var in variables.global_variables(): - if var.name.startswith("AdaptiveMaxNorm"): - var_dict[var.name.split(":")[0]] = var - self.assertEqual(2, len(var_dict)) - moving_mean = var_dict["AdaptiveMaxNorm/mean"] - moving_sq_mean = var_dict["AdaptiveMaxNorm/sq_mean"] - variables.global_variables_initializer().run() - mean, sq_mean = session.run([moving_mean, moving_sq_mean]) - self.assertEqual([0], mean) - self.assertEqual([0], sq_mean) - for i in range(20): - mean, sq_mean, _ = session.run( - [moving_mean, moving_sq_mean, grads_and_vars[0][0]]) - if i == 0: - self.assertLess(mean, 0.9 * log_norm) - self.assertLess(sq_mean, 0.9 * log_norm**2) - - self.assertAlmostEqual(float(mean), log_norm, places=4) - self.assertAlmostEqual(float(sq_mean), log_norm**2, places=4) - - def testClip(self): - with self.cached_session() as session: - spike = 1000. - multiplier = array_ops.placeholder(dtypes.float32, [], "multiplier") - step = array_ops.placeholder(dtypes.int32, [], "step") - - grad = array_ops.ones([3, 4]) * multiplier - grads_and_vars = [(grad, grad)] - grads_and_vars = optimizers_lib.adaptive_clipping_fn( - decay=0.9, global_step=step)(grads_and_vars) - - variables.global_variables_initializer().run() - - def run(scale, i): - return session.run(grads_and_vars[0][0], - feed_dict={multiplier: scale, - step: i}) - - for i in range(20): - scale = [1., -2.][i % 2] - clipped_grad = run(scale, i) - if i > 3: - self.assertAllClose(np.ones(clipped_grad.shape) * scale, clipped_grad) - - # assert that the spike will have low influence. - clipped_grad = run(spike, 20) - self.assertTrue((clipped_grad < 25.).all()) - - # assert that a repeated spike will converge to this new value. - for i in range(10): - clipped_grad = run(spike, i + 21) - - self.assertAllClose(np.ones(clipped_grad.shape) * spike, clipped_grad) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/layers/python/layers/regularizers.py b/tensorflow/contrib/layers/python/layers/regularizers.py deleted file mode 100644 index e23dea7d245..00000000000 --- a/tensorflow/contrib/layers/python/layers/regularizers.py +++ /dev/null @@ -1,209 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Regularizers for use with layers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numbers - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import standard_ops -from tensorflow.python.platform import tf_logging as logging - -__all__ = ['l1_regularizer', - 'l2_regularizer', - 'l1_l2_regularizer', - 'sum_regularizer', - 'apply_regularization'] - - -def l1_regularizer(scale, scope=None): - """Returns a function that can be used to apply L1 regularization to weights. - - L1 regularization encourages sparsity. - - Args: - scale: A scalar multiplier `Tensor`. 0.0 disables the regularizer. - scope: An optional scope name. - - Returns: - A function with signature `l1(weights)` that apply L1 regularization. - - Raises: - ValueError: If scale is negative or if scale is not a float. - """ - if isinstance(scale, numbers.Integral): - raise ValueError('scale cannot be an integer: %s' % scale) - if isinstance(scale, numbers.Real): - if scale < 0.: - raise ValueError('Setting a scale less than 0 on a regularizer: %g' % - scale) - if scale == 0.: - logging.info('Scale of 0 disables regularizer.') - return lambda _: None - - def l1(weights, name=None): - """Applies L1 regularization to weights.""" - with ops.name_scope(scope, 'l1_regularizer', [weights]) as name: - my_scale = ops.convert_to_tensor(scale, - dtype=weights.dtype.base_dtype, - name='scale') - return standard_ops.multiply( - my_scale, - standard_ops.reduce_sum(standard_ops.abs(weights)), - name=name) - - return l1 - - -def l2_regularizer(scale, scope=None): - """Returns a function that can be used to apply L2 regularization to weights. - - Small values of L2 can help prevent overfitting the training data. - - Args: - scale: A scalar multiplier `Tensor`. 0.0 disables the regularizer. - scope: An optional scope name. - - Returns: - A function with signature `l2(weights)` that applies L2 regularization. - - Raises: - ValueError: If scale is negative or if scale is not a float. - """ - if isinstance(scale, numbers.Integral): - raise ValueError('scale cannot be an integer: %s' % (scale,)) - if isinstance(scale, numbers.Real): - if scale < 0.: - raise ValueError('Setting a scale less than 0 on a regularizer: %g.' % - scale) - if scale == 0.: - logging.info('Scale of 0 disables regularizer.') - return lambda _: None - - def l2(weights): - """Applies l2 regularization to weights.""" - with ops.name_scope(scope, 'l2_regularizer', [weights]) as name: - my_scale = ops.convert_to_tensor(scale, - dtype=weights.dtype.base_dtype, - name='scale') - return standard_ops.multiply(my_scale, nn.l2_loss(weights), name=name) - - return l2 - - -def l1_l2_regularizer(scale_l1=1.0, scale_l2=1.0, scope=None): - """Returns a function that can be used to apply L1 L2 regularizations. - - Args: - scale_l1: A scalar multiplier `Tensor` for L1 regularization. - scale_l2: A scalar multiplier `Tensor` for L2 regularization. - scope: An optional scope name. - - Returns: - A function with signature `l1_l2(weights)` that applies a weighted sum of - L1 L2 regularization. - - Raises: - ValueError: If scale is negative or if scale is not a float. - """ - if isinstance(scale_l1, numbers.Integral): - raise ValueError('scale_l1 cannot be an integer: %s' % (scale_l1,)) - if isinstance(scale_l2, numbers.Integral): - raise ValueError('scale_l2 cannot be an integer: %s' % (scale_l2,)) - scope = scope or 'l1_l2_regularizer' - if scale_l1 == 0.: - return l2_regularizer(scale_l2, scope) - if scale_l2 == 0.: - return l1_regularizer(scale_l1, scope) - return sum_regularizer([l1_regularizer(scale_l1), - l2_regularizer(scale_l2)], - scope=scope) - - -def sum_regularizer(regularizer_list, scope=None): - """Returns a function that applies the sum of multiple regularizers. - - Args: - regularizer_list: A list of regularizers to apply. - scope: An optional scope name - - Returns: - A function with signature `sum_reg(weights)` that applies the - sum of all the input regularizers. - """ - regularizer_list = [reg for reg in regularizer_list if reg is not None] - if not regularizer_list: - return None - - def sum_reg(weights): - """Applies the sum of all the input regularizers.""" - with ops.name_scope(scope, 'sum_regularizer', [weights]) as name: - regularizer_tensors = [] - for reg in regularizer_list: - tensor = reg(weights) - if tensor is not None: - regularizer_tensors.append(tensor) - return math_ops.add_n( - regularizer_tensors, name=name) if regularizer_tensors else None - - return sum_reg - - -def apply_regularization(regularizer, weights_list=None): - """Returns the summed penalty by applying `regularizer` to the `weights_list`. - - Adding a regularization penalty over the layer weights and embedding weights - can help prevent overfitting the training data. Regularization over layer - biases is less common/useful, but assuming proper data preprocessing/mean - subtraction, it usually shouldn't hurt much either. - - Args: - regularizer: A function that takes a single `Tensor` argument and returns - a scalar `Tensor` output. - weights_list: List of weights `Tensors` or `Variables` to apply - `regularizer` over. Defaults to the `GraphKeys.WEIGHTS` collection if - `None`. - - Returns: - A scalar representing the overall regularization penalty. - - Raises: - ValueError: If `regularizer` does not return a scalar output, or if we find - no weights. - """ - if not weights_list: - weights_list = ops.get_collection(ops.GraphKeys.WEIGHTS) - if not weights_list: - raise ValueError('No weights to regularize.') - with ops.name_scope('get_regularization_penalty', - values=weights_list) as scope: - penalties = [regularizer(w) for w in weights_list] - penalties = [ - p if p is not None else constant_op.constant(0.0) for p in penalties - ] - for p in penalties: - if p.get_shape().ndims != 0: - raise ValueError('regularizer must return a scalar Tensor instead of a ' - 'Tensor with rank %d.' % p.get_shape().ndims) - - summed_penalty = math_ops.add_n(penalties, name=scope) - ops.add_to_collection(ops.GraphKeys.REGULARIZATION_LOSSES, summed_penalty) - return summed_penalty diff --git a/tensorflow/contrib/layers/python/layers/regularizers_test.py b/tensorflow/contrib/layers/python/layers/regularizers_test.py deleted file mode 100644 index 0ed10783584..00000000000 --- a/tensorflow/contrib/layers/python/layers/regularizers_test.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for regularizers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.layers.python.layers import regularizers -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class RegularizerTest(test.TestCase): - - def test_l1(self): - with self.assertRaises(ValueError): - regularizers.l1_regularizer(-1.) - with self.assertRaises(ValueError): - regularizers.l1_regularizer(0) - - self.assertIsNone(regularizers.l1_regularizer(0.)(None)) - - values = np.array([1., -1., 4., 2.]) - weights = constant_op.constant(values) - with session.Session() as sess: - result = sess.run(regularizers.l1_regularizer(.5)(weights)) - - self.assertAllClose(np.abs(values).sum() * .5, result) - - def test_l2(self): - with self.assertRaises(ValueError): - regularizers.l2_regularizer(-1.) - with self.assertRaises(ValueError): - regularizers.l2_regularizer(0) - - self.assertIsNone(regularizers.l2_regularizer(0.)(None)) - - values = np.array([1., -1., 4., 2.]) - weights = constant_op.constant(values) - with session.Session() as sess: - result = sess.run(regularizers.l2_regularizer(.42)(weights)) - - self.assertAllClose(np.power(values, 2).sum() / 2.0 * .42, result) - - def test_l1_l2(self): - with self.assertRaises(ValueError): - regularizers.l1_l2_regularizer(-1., 0.5) - with self.assertRaises(ValueError): - regularizers.l1_l2_regularizer(0.5, -1.) - with self.assertRaises(ValueError): - regularizers.l1_l2_regularizer(0, 0.5) - with self.assertRaises(ValueError): - regularizers.l1_l2_regularizer(0.5, 0) - - with self.cached_session(): - shape = [5, 5, 5] - num_elem = 5 * 5 * 5 - tensor = constant_op.constant(1.0, shape=shape) - loss = regularizers.l1_l2_regularizer(1.0, 1.0)(tensor) - self.assertEquals(loss.op.name, 'l1_l2_regularizer') - self.assertAlmostEqual(loss.eval(), num_elem + num_elem / 2, 5) - - def test_l1_l2_scale_l1Zero(self): - shape = [5, 5, 5] - num_elem = 5 * 5 * 5 - tensor = constant_op.constant(1.0, shape=shape) - loss = regularizers.l1_l2_regularizer(0.0, 1.0)(tensor) - with self.cached_session(): - self.assertEquals(loss.op.name, 'l1_l2_regularizer') - self.assertAlmostEqual(loss.eval(), num_elem / 2, 5) - - def test_l1_l2_scale_l2Zero(self): - shape = [5, 5, 5] - num_elem = 5 * 5 * 5 - tensor = constant_op.constant(1.0, shape=shape) - loss = regularizers.l1_l2_regularizer(1.0, 0.0)(tensor) - with self.cached_session(): - self.assertEquals(loss.op.name, 'l1_l2_regularizer') - self.assertAlmostEqual(loss.eval(), num_elem, 5) - - def test_l1_l2_scales_Zero(self): - shape = [5, 5, 5] - tensor = constant_op.constant(1.0, shape=shape) - loss = regularizers.l1_l2_regularizer(0.0, 0.0)(tensor) - self.assertEquals(loss, None) - - def testL1L2RegularizerWithScope(self): - with self.cached_session(): - shape = [5, 5, 5] - num_elem = 5 * 5 * 5 - tensor = constant_op.constant(1.0, shape=shape) - with ops.name_scope('foo'): - loss = regularizers.l1_l2_regularizer(1.0, 1.0, scope='l1_l2')(tensor) - self.assertEquals(loss.op.name, 'foo/l1_l2') - self.assertAlmostEqual(loss.eval(), num_elem + num_elem / 2, 5) - - def test_sum_regularizer(self): - l1_function = regularizers.l1_regularizer(.1) - l2_function = regularizers.l2_regularizer(.2) - self.assertIsNone(regularizers.sum_regularizer([])) - self.assertIsNone(regularizers.sum_regularizer([None])) - self.assertIsNone( - regularizers.sum_regularizer([regularizers.l1_regularizer(.0)])(None)) - - values = np.array([-3.]) - weights = constant_op.constant(values) - with session.Session() as sess: - l1_reg1 = regularizers.sum_regularizer([l1_function]) - l1_result1 = sess.run(l1_reg1(weights)) - - l1_reg2 = regularizers.sum_regularizer([l1_function, None]) - l1_result2 = sess.run(l1_reg2(weights)) - - l1_reg3 = regularizers.sum_regularizer( - [l1_function, regularizers.l2_regularizer(.0)]) - l1_result3 = sess.run(l1_reg3(weights)) - - l1_l2_reg = regularizers.sum_regularizer([l1_function, l2_function]) - l1_l2_result = sess.run(l1_l2_reg(weights)) - - self.assertAllClose(.1 * np.abs(values).sum(), l1_result1) - self.assertAllClose(.1 * np.abs(values).sum(), l1_result2) - self.assertAllClose(.1 * np.abs(values).sum(), l1_result3) - self.assertAllClose( - .1 * np.abs(values).sum() + .2 * np.power(values, 2).sum() / 2.0, - l1_l2_result) - - def test_apply_regularization(self): - dummy_regularizer = lambda x: math_ops.reduce_sum(2 * x) - array_weights_list = [[1.5], [2, 3, 4.2], [10, 42, 666.6]] - tensor_weights_list = [constant_op.constant(x) for x in array_weights_list] - expected = sum(2 * x for l in array_weights_list for x in l) - with self.cached_session(): - result = regularizers.apply_regularization(dummy_regularizer, - tensor_weights_list) - self.assertAllClose(expected, result.eval()) - - def test_apply_zero_regularization(self): - regularizer = regularizers.l2_regularizer(0.0) - array_weights_list = [[1.5], [2, 3, 4.2], [10, 42, 666.6]] - tensor_weights_list = [constant_op.constant(x) for x in array_weights_list] - with self.cached_session(): - result = regularizers.apply_regularization(regularizer, - tensor_weights_list) - self.assertAllClose(0.0, result.eval()) - - def test_apply_regularization_invalid_regularizer(self): - non_scalar_regularizer = lambda x: array_ops.tile(x, [2]) - tensor_weights_list = [ - constant_op.constant(x) for x in [[1.5], [2, 3, 4.2], [10, 42, 666.6]] - ] - with self.cached_session(): - with self.assertRaises(ValueError): - regularizers.apply_regularization(non_scalar_regularizer, - tensor_weights_list) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/layers/python/layers/rev_block_lib.py b/tensorflow/contrib/layers/python/layers/rev_block_lib.py deleted file mode 100644 index 06da32072f3..00000000000 --- a/tensorflow/contrib/layers/python/layers/rev_block_lib.py +++ /dev/null @@ -1,703 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Reversible Residual Block. - -From -[The Reversible Residual Network: Backpropagation Without Storing -Activations](https://arxiv.org/abs/1707.04585). - -Also contains the @recompute_grad decorator, which recomputes the forward -function on the backwards pass. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import re - -import numpy as np -import six -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib.framework.python import ops as contrib_framework_ops -from tensorflow.python.eager import backprop -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops as framework_ops -from tensorflow.python.layers import base -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import control_flow_util -from tensorflow.python.ops import custom_gradient -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util import nest -from tensorflow.python.util import tf_inspect - -__all__ = ["rev_block", "RevBlock", "recompute_grad"] - -LAYER_RE = re.compile(".*revlayer_([0-9]*)/([fg])/.*") -_USE_DEFAULT = "__rev_block_lib_default" -_WRONG_VARS_ERR = """\ -The variables used on recompute were different than the variables originally -used. The function wrapped with @recompute_grad likley creates its own variable -scope with a default name and has been called twice in the same enclosing scope. -To fix, ensure each call to the function happens in its own unique variable -scope. -""" - - -def _acc_grads(*lists_of_grads): - """Accumulates lists of gradients.""" - acc_grads = [] - for grads in zip(*lists_of_grads): - grads = [g for g in grads if g is not None] - if grads: - acc_grads.append(math_ops.add_n(grads)) - else: - acc_grads.append(None) - return acc_grads - - -def _rev_layer_forward(xs, f, g, f_side_input, g_side_input, - gate_outputs=False): - """Forward for 1 reversible layer.""" - x1, x2 = xs - y1 = x1 + (f(x2, f_side_input) if f_side_input else f(x2)) - y2 = x2 + (g(y1, g_side_input) if g_side_input else g(y1)) - if gate_outputs: - return control_flow_ops.tuple([y1, y2]) - else: - return (y1, y2) - - -def _rev_layer_backward(ys, grad_ys, f, g, f_vars, f_side_input, g_vars, - g_side_input): - """Backprop for 1 layer.""" - y1, y2 = ys - grad_y1, grad_y2 = grad_ys - - # Reconstruct intermediates and inputs (x1, x2) - # stop_gradients required on fn inputs to prevent infinite recursion into this - # grad function on the calls to gradients. - y1_stop = array_ops.stop_gradient(y1) - g_side_input = [array_ops.stop_gradient(t) for t in g_side_input] - gy1 = g(y1_stop, g_side_input) if g_side_input else g(y1_stop) - - x2 = y2 - gy1 - x2_stop = array_ops.stop_gradient(x2) - f_side_input = [array_ops.stop_gradient(t) for t in f_side_input] - fx2 = f(x2_stop, f_side_input) if f_side_input else f(x2_stop) - - x1 = y1 - fx2 - - # Compute gradients wrt to inputs - # dL/dy2 * dG(y1)/y1 - grad_gy1_y2 = gradients_impl.gradients(gy1, y1_stop, grad_y2)[0] - grad_x1 = grad_y1 + grad_gy1_y2 - grad_x2 = ( - gradients_impl.gradients(fx2, x2_stop, grad_y1)[0] + grad_y2 + - gradients_impl.gradients(fx2, x2_stop, grad_gy1_y2)[0]) - - # Compute gradients wrt to vars and side inputs in f and g - grads1 = gradients_impl.gradients(gy1, g_vars + g_side_input, grad_y2) - grad_g_vars, grad_g_side = grads1[:len(g_vars)], grads1[len(g_vars):] - grads2 = gradients_impl.gradients(fx2, f_vars + f_side_input, grad_y1) - grad_f_y1, grad_f_side1 = grads2[:len(f_vars)], grads2[len(f_vars):] - grads3 = gradients_impl.gradients(fx2, f_vars + f_side_input, grad_gy1_y2) - grad_f_y2, grad_f_side2 = grads3[:len(f_vars)], grads3[len(f_vars):] - grad_f_vars = _acc_grads(grad_f_y1, grad_f_y2) - - grad_f_side = _acc_grads(grad_f_side1, grad_f_side2) - - # Put returns in a tuple to ensure a constant memory budget (i.e. don't want - # the subsequent layer to start computing and consuming memory based on a - # subset of these values). - outputs = ((x1, x2), (grad_x1, grad_x2), (grad_f_vars, grad_f_side), - (grad_g_vars, grad_g_side)) - tupled = control_flow_ops.tuple(nest.flatten(outputs)) - return nest.pack_sequence_as(outputs, tupled) - - -def _rev_block_forward(x1, - x2, - f, - g, - num_layers=1, - f_side_input=None, - g_side_input=None, - gate_outputs=False): - """Forward for a series of reversible layers.""" - out = (x1, x2) - for i in xrange(num_layers): - out = _rev_layer_forward( - out, f[i], g[i], f_side_input, g_side_input, gate_outputs=gate_outputs) - - y1, y2 = out - return y1, y2 - - -def _safe_wraps(fn): - if isinstance(fn, functools.partial): - # functools.partial objects cannot be wrapped as they are missing the - # necessary properties (__name__, __module__, __doc__). - def passthrough(f): - return f - return passthrough - return functools.wraps(fn) - - -def _scope_wrap(fn, scope): - - @_safe_wraps(fn) - def wrap(*args, **kwargs): - with variable_scope.variable_scope(scope, use_resource=True): - return fn(*args, **kwargs) - - return wrap - - -class RevBlock(base.Layer): - """Block of reversible layers. See rev_block.""" - - def __init__(self, - f, - g, - num_layers=1, - f_side_input=None, - g_side_input=None, - use_efficient_backprop=True, - name="revblock", - **kwargs): - super(RevBlock, self).__init__(name=name, **kwargs) - - if isinstance(f, list): - assert len(f) == num_layers - else: - f = [f] * num_layers - - if isinstance(g, list): - assert len(g) == num_layers - else: - g = [g] * num_layers - - f = [_scope_wrap(fn, "revlayer_%d/f" % i) for i, fn in enumerate(f)] - g = [_scope_wrap(fn, "revlayer_%d/g" % i) for i, fn in enumerate(g)] - - self.f = f - self.g = g - - self.num_layers = num_layers - self.f_side_input = f_side_input or [] - self.g_side_input = g_side_input or [] - - self._use_efficient_backprop = use_efficient_backprop - - def call(self, inputs, forward=True): - vs = variable_scope.get_variable_scope() - vars_before = vs.global_variables() - - if forward: - x1, x2 = inputs - out = self._forward(x1, x2) - else: - y1, y2 = inputs - out = self._backward(y1, y2) - - # Add any created variables to the Layer's variable stores - new_vars = vs.global_variables()[len(vars_before):] - train_vars = vs.trainable_variables() - for new_var in new_vars: - if new_var in train_vars: - self._trainable_weights.append(new_var) - else: - self._non_trainable_weights.append(new_var) - - return out - - def forward(self, x1, x2): - return self.apply([x1, x2]) - - def backward(self, y1, y2): - return self.apply([y1, y2], forward=False) - - def build(self, _): - logging.warn("RevBlock constructs its variables on first call, not on " - "build.") - self.built = True - - def _make_efficient_grad_fn(self, inputs_, ys_): - def _efficient_grad_fn(*grad_ys, **kwargs): - """Custom gradient fn for a block of reversible residual layers.""" - inputs = inputs_ - ys = ys_ - variables = kwargs["variables"] - side_inputs = inputs[2:] - - f_side_idxs = [None] * len(self.f_side_input) - g_side_idxs = [None] * len(self.g_side_input) - assert len(side_inputs) == len(self.f_side_input) + len(self.g_side_input) - - for i, t in enumerate(side_inputs): - if t in self.f_side_input: - f_side_idxs[self.f_side_input.index(t)] = i - elif t in self.g_side_input: - g_side_idxs[self.g_side_input.index(t)] = i - else: - assert False - - f_vars = [[] for _ in range(self.num_layers)] - g_vars = [[] for _ in range(self.num_layers)] - f_vars_idxs = [[] for _ in range(self.num_layers)] - g_vars_idxs = [[] for _ in range(self.num_layers)] - - for i, ref in enumerate(variables): - # Use the name to identify the layer number and function (f or g) - regex = LAYER_RE.match(ref.name) - layer_no = int(regex.group(1)) - fn_name = regex.group(2) - if fn_name == "f": - f_vars[layer_no].append(ref) - f_vars_idxs[layer_no].append(i) - else: - assert fn_name == "g" - g_vars[layer_no].append(ref) - g_vars_idxs[layer_no].append(i) - - f_var_grads = [] - g_var_grads = [] - f_side_grads = [] - g_side_grads = [] - - # Reverse variable containers to go backward - f_vars.reverse() - g_vars.reverse() - f = list(self.f) - g = list(self.g) - f.reverse() - g.reverse() - - with variable_scope.variable_scope(self.scope_name, reuse=True): - for i in xrange(self.num_layers): - ys, grad_ys, f_ret, g_ret = _rev_layer_backward( - ys, grad_ys, f[i], g[i], f_vars[i], self.f_side_input, g_vars[i], - self.g_side_input) - - grad_f_vars, grad_f_side = f_ret - grad_g_vars, grad_g_side = g_ret - f_var_grads.append(grad_f_vars) - g_var_grads.append(grad_g_vars) - f_side_grads.append(grad_f_side) - g_side_grads.append(grad_g_side) - - # Accumulate layer gradients for f_side_input and g_side_input - acc_f_side_grads = _acc_grads(*f_side_grads) - acc_g_side_grads = _acc_grads(*g_side_grads) - - # Use the stored idxs to put gradients in the passed-in order. - side_input_grads = [None] * len(side_inputs) - variable_grads = [None] * len(variables) - - # Variable gradients were collected in reverse layer order. Reverse to - # match idxs. - f_var_grads.reverse() - g_var_grads.reverse() - for idxs, grads in list(zip(f_vars_idxs, f_var_grads)) + list( - zip(g_vars_idxs, g_var_grads)): - for i, grad in zip(idxs, grads): - variable_grads[i] = grad - - for i, grad in zip(f_side_idxs, acc_f_side_grads): - side_input_grads[i] = grad - for i, grad in zip(g_side_idxs, acc_g_side_grads): - side_input_grads[i] = grad - - grad_x1, grad_x2 = grad_ys - return [grad_x1, grad_x2] + side_input_grads, variable_grads - return _efficient_grad_fn - - def _forward(self, x1, x2): - """Run forward through the reversible layers.""" - - side_inputs = [self.f_side_input, self.g_side_input] - flat_side_inputs = nest.flatten(side_inputs) - - def _forward_wrap(x1_, x2_, *flat_side_inputs): - f_side, g_side = nest.pack_sequence_as(side_inputs, flat_side_inputs) - return _rev_block_forward( - x1_, - x2_, - self.f, - self.g, - num_layers=self.num_layers, - f_side_input=f_side, - g_side_input=g_side, - gate_outputs=self._use_efficient_backprop) - - @custom_gradient.custom_gradient - def _forward_with_custom_grad(*args): - out = _forward_wrap(*args) # pylint: disable=no-value-for-parameter - grad_fn = self._make_efficient_grad_fn(args, out) - return out, grad_fn - - if self._use_efficient_backprop: - return _forward_with_custom_grad(x1, x2, *flat_side_inputs) - else: - return _forward_wrap(x1, x2, *flat_side_inputs) - - def _backward(self, y1, y2): - """Run backward through the reversible layers.""" - - f = list(self.f) - g = list(self.g) - f.reverse() - g.reverse() - - for i in xrange(self.num_layers): - gy1 = g[i](y1, self.g_side_input) if self.g_side_input else g[i](y1) - x2 = y2 - gy1 - fx2 = f[i](x2, self.f_side_input) if self.f_side_input else f[i](x2) - x1 = y1 - fx2 - - y1, y2 = x1, x2 - - return x1, x2 - - -def rev_block(x1, - x2, - f, - g, - num_layers=1, - f_side_input=None, - g_side_input=None, - is_training=True): - """A block of reversible residual layers. - - A reversible residual layer is defined as: - - ``` - y1 = x1 + f(x2, f_side_input) - y2 = x2 + g(y1, g_side_input) - ``` - - A reversible residual block, defined here, is a series of reversible residual - layers. - - Limitations: - * f and g must not close over any Tensors; all side inputs to f and g should - be passed in with f_side_input and g_side_input which will be forwarded to - f and g. - * f and g must not change the dimensionality of their inputs in order for the - addition in the equations above to work. - - Args: - x1: a float Tensor. - x2: a float Tensor. - f: a function, (Tensor) -> (Tensor) (or list of such of length num_layers). - Should not change the shape of the Tensor. Can make calls to get_variable. - See f_side_input if there are side inputs. - g: a function, (Tensor) -> (Tensor) (or list of such of length num_layers). - Should not change the shape of the Tensor. Can make calls to get_variable. - See g_side_input if there are side inputs. - num_layers: int, number of reversible residual layers. Each layer will - apply f and g according to the equations above, with new variables in each - layer. - f_side_input: list of Tensors, side input to f. If not None, signature of f - should be (Tensor, list) -> (Tensor). - g_side_input: list of Tensors, side input to g. If not None, signature of g - should be (Tensor, list) -> (Tensor). - is_training: bool, whether to actually use the efficient backprop codepath. - - Returns: - y1, y2: tuple of float Tensors. - """ - block = RevBlock( - f=f, - g=g, - num_layers=num_layers, - f_side_input=f_side_input, - g_side_input=g_side_input, - use_efficient_backprop=is_training, - _reuse=variable_scope.get_variable_scope().reuse) - return block.forward(x1, x2) - - -def enable_with_args(dec): - """A decorator for decorators to enable their usage with or without args.""" - - @_safe_wraps(dec) - def new_dec(*args, **kwargs): - if len(args) == 1 and not kwargs and callable(args[0]): - # Used as decorator without args - fn = args[0] - return dec(fn) - else: - return lambda fn: dec(fn, *args, **kwargs) - - return new_dec - - -@enable_with_args -def recompute_grad(fn, use_data_dep=_USE_DEFAULT, tupleize_grads=False): - """Decorator that recomputes the function on the backwards pass. - - To use this function, you must use `ResourceVariable`s (i.e. - `variable_scope(name, use_resource=True), which are the default in Eager mode - and when running on TPU. - - Warning: Because the function will be called again on the backwards pass, the - user should be careful to not use ops in their function that mutate state or - have randomness (for example, batch normalization or dropout). If the function - does have such operations, it is recommended that the function take the - `is_recomputing` keyword argument which will be `False` on the forward pass - and `True` on the backwards pass so that it can disable state changes when - `is_recomputing=True` (for example, not updating the moving averages in batch - normalization). - - Args: - fn: a function that takes Tensors (all as positional arguments) and returns - a tuple of Tensors. Note that `fn` should not close over any other - Tensors or Variables. - use_data_dep: `bool`, if `True` will use a dummy data dependency to force - the recompute to happen. If `False` will use a control dependency. By - default will be `True` if in an XLA context and `False` otherwise. XLA - ignores control dependencies and so this data dependency is necessary. - tupleize_grads: `bool`, if `True` will use control dependencies to ensure - that all gradients are produced before any are consumed by downstream ops. - If `use_data_dep` is also `True`, will use a data dependency instead of - a control dependency. - - Returns: - A wrapped fn that is identical to fn when called, but its activations will - be discarded and recomputed on the backwards pass (i.e. on a call to - tf.gradients). - - Raises: - ValueError: if `fn` closes over any Tensors or Variables. - """ - # Check for closed-over Tensors/Variables - if fn.__code__.co_freevars: - closed_over_vars = dict(zip(fn.__code__.co_freevars, - [c.cell_contents for c in fn.__closure__])) - for var_name, value in six.iteritems(closed_over_vars): - if isinstance(value, (framework_ops.Tensor, variables_lib.Variable)): - raise ValueError( - "fn decorated with @recompute_grad closes over Tensor %s " - "(local variable name: %s). The decorated fn must not close over " - "Tensors or Variables because gradients will NOT be computed for " - "them through fn. To ensure correct gradients, make the " - "Tensor an input to fn." % (value.name, var_name)) - - @_safe_wraps(fn) - def wrapped(*args): - return _recompute_grad( - fn, args, use_data_dep=use_data_dep, tupleize_grads=tupleize_grads) - - return wrapped - - -def _is_on_tpu(): - ctxt = framework_ops.get_default_graph()._get_control_flow_context() # pylint: disable=protected-access - return control_flow_util.GetContainingXLAContext(ctxt) is not None - - -def _recomputing_grad_fn(compute_fn, - original_args, - original_vars, - output_grads, - grad_fn_variables, - use_data_dep, - tupleize_grads, - arg_scope, - var_scope, - has_is_recompute_kwarg): - """Grad fn for recompute_grad.""" - variables = grad_fn_variables or [] - - # Identity ops around the inputs ensures correct gradient graph-walking. - inputs = [array_ops.identity(x) for x in list(original_args)] - - # Recompute outputs - # Use a control dependency to ensure that the recompute is not eliminated by - # CSE and that it happens on the backwards pass. - ctrl_dep_grads = [g for g in output_grads if g is not None] - with framework_ops.control_dependencies(ctrl_dep_grads): - if use_data_dep: - inputs = _force_data_dependency(output_grads, inputs) - # Re-enter scopes - with contrib_framework_ops.arg_scope(arg_scope): - with variable_scope.variable_scope(var_scope, reuse=True): - # Re-call the function and ensure that the touched variables are the - # same as in the first call. - with backprop.GradientTape() as tape: - fn_kwargs = {} - if has_is_recompute_kwarg: - fn_kwargs["is_recomputing"] = True - outputs = compute_fn(*inputs, **fn_kwargs) - recompute_vars = set(tape.watched_variables()) - if original_vars != recompute_vars: - raise ValueError(_WRONG_VARS_ERR) - - if not isinstance(outputs, (list, tuple)): - outputs = [outputs] - outputs = list(outputs) - - # Compute gradients - grads = gradients_impl.gradients(outputs, inputs + variables, - output_grads) - - if tupleize_grads: - if use_data_dep: - grads = _tuple_with_data_dep(grads) - else: - grads = control_flow_ops.tuple(grads) - - grad_inputs = grads[:len(inputs)] - grad_vars = grads[len(inputs):] - return grad_inputs, grad_vars - - -def _recompute_grad(fn, args, use_data_dep=_USE_DEFAULT, tupleize_grads=False): - """See recompute_grad.""" - has_is_recompute_kwarg = "is_recomputing" in tf_inspect.getargspec(fn).args - for arg in args: - if not isinstance(arg, framework_ops.Tensor): - raise ValueError("All inputs to function must be Tensors") - use_data_dep_ = use_data_dep - if use_data_dep_ == _USE_DEFAULT: - use_data_dep_ = _is_on_tpu() - - # Use custom_gradient and return a grad_fn that recomputes on the backwards - # pass. - @custom_gradient.custom_gradient - def fn_with_recompute(*args): - """Wrapper for fn.""" - # Capture the variable and arg scopes so we can re-enter them when - # recomputing. - vs = variable_scope.get_variable_scope() - arg_scope = contrib_framework_ops.current_arg_scope() - # Track all variables touched in the function. - with backprop.GradientTape() as tape: - fn_kwargs = {} - if has_is_recompute_kwarg: - fn_kwargs["is_recomputing"] = False - outputs = fn(*args, **fn_kwargs) - original_vars = set(tape.watched_variables()) - - def _grad_fn(output_grads, variables=None): - # Validate that custom_gradient passes the right variables into grad_fn. - if original_vars: - assert variables, ("Fn created variables but the variables were not " - "passed to the gradient fn.") - if set(variables) != original_vars: - raise ValueError(_WRONG_VARS_ERR) - - return _recomputing_grad_fn( - compute_fn=fn, - original_args=args, - original_vars=original_vars, - output_grads=output_grads, - grad_fn_variables=variables, - use_data_dep=use_data_dep_, - tupleize_grads=tupleize_grads, - arg_scope=arg_scope, - var_scope=vs, - has_is_recompute_kwarg=has_is_recompute_kwarg) - - # custom_gradient inspects the signature of the function to determine - # whether the user expects variables passed in the grad_fn. If the function - # created variables, the grad_fn should accept the "variables" kwarg. - if original_vars: - def grad_fn(*output_grads, **kwargs): - return _grad_fn(output_grads, kwargs["variables"]) - else: - def grad_fn(*output_grads): - return _grad_fn(output_grads) - - return outputs, grad_fn - - return fn_with_recompute(*args) - - -def _underlying_variable_ref(t): - """Find the underlying variable ref. - - Traverses through Identity, ReadVariableOp, and Enter ops. - Stops when op type has Variable or VarHandle in name. - - Args: - t: a Tensor - - Returns: - a Tensor that is a variable ref, or None on error. - """ - while t.op.type in ["Identity", "ReadVariableOp", "Enter"]: - t = t.op.inputs[0] - - op_type = t.op.type - if "Variable" in op_type or "VarHandle" in op_type: - return t - else: - return None - - -def _force_data_dependency(first_compute, then_compute): - """Force all of `then_compute` to depend on all of `first_compute`. - - Uses a dummy data dependency, which is useful when running on TPUs because - XLA ignores control dependencies. Only supports float arguments. - - Args: - first_compute: `list`. These will be made to run before the - `Tensor`s `then_compute`. - then_compute: `list`. These will run after all the `Tensor`s in - `first_compute`. - - Returns: - `list`, same length as `then_compute`. - - Raises: - ValueError: if ranks are unknown or types are not floating. - """ - - def _first_element(x): - if x.get_shape().ndims is None: - raise ValueError("Rank of Tensor %s must be known" % x) - ndims = x.get_shape().ndims - begin = framework_ops.convert_to_tensor([0] * ndims, dtype=dtypes.int32) - size = framework_ops.convert_to_tensor([1] * ndims, dtype=dtypes.int32) - return array_ops.reshape(array_ops.slice(x, begin, size), []) - - first_compute_sum = math_ops.add_n( - [_first_element(x) for x in first_compute if x is not None]) - dtype = first_compute_sum.dtype - if not dtype.is_floating: - raise ValueError("_force_data_dependency only supports floating dtypes.") - epsilon = np.finfo(dtype.as_numpy_dtype).tiny - zero = array_ops.stop_gradient(epsilon * first_compute_sum) - - return [ - array_ops.identity(x) + zero if x is not None else None - for x in then_compute - ] - - -def _tuple_with_data_dep(tensors): - return _force_data_dependency(tensors, tensors) diff --git a/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py b/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py deleted file mode 100644 index 2c7463acc0f..00000000000 --- a/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py +++ /dev/null @@ -1,407 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for RevBlock.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.layers.python.layers import rev_block_lib -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.layers import convolutional -from tensorflow.python.layers import core as core_layers -from tensorflow.python.layers import normalization as normalization_layers -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class RevBlockTest(test.TestCase): - CHANNELS = 8 - NUM_LAYERS = 4 - BATCH_SIZE = 16 - - def testForwardBackward(self): - - def f(x): - return core_layers.dense(x, self.CHANNELS // 2, use_bias=True) - - def g(x): - return core_layers.dense(x, self.CHANNELS // 2, use_bias=True) - - x = random_ops.random_uniform( - [self.BATCH_SIZE, self.CHANNELS], dtype=dtypes.float32) - x1, x2 = array_ops.split(x, 2, axis=-1) - - block = rev_block_lib.RevBlock(f, g, num_layers=3) - y1, y2 = block.forward(x1, x2) - x1_inv, x2_inv = block.backward(y1, y2) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - x1, x2, x1_inv, x2_inv = sess.run([x1, x2, x1_inv, x2_inv]) - - self.assertAllClose(x1, x1_inv, atol=1e-5) - self.assertAllClose(x2, x2_inv, atol=1e-5) - - def testBackwardForward(self): - - def f(x): - return core_layers.dense(x, self.CHANNELS // 2, use_bias=True) - - def g(x): - return core_layers.dense(x, self.CHANNELS // 2, use_bias=True) - - y = random_ops.random_uniform( - [self.BATCH_SIZE, self.CHANNELS], dtype=dtypes.float32) - y1, y2 = array_ops.split(y, 2, axis=-1) - - block = rev_block_lib.RevBlock(f, g, num_layers=3) - x1, x2 = block.backward(y1, y2) - y1_inv, y2_inv = block.forward(x1, x2) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - y1, y2, y1_inv, y2_inv = sess.run([y1, y2, y1_inv, y2_inv]) - - self.assertAllClose(y1, y1_inv, rtol=1e-5) - self.assertAllClose(y2, y2_inv, rtol=1e-5) - - def _testRevBlock(self, - x=None, - f=None, - g=None, - f_side_input=None, - g_side_input=None): - random_seed.set_random_seed(1234) - - if f is None: - - def f(x): # pylint: disable=function-redefined - return core_layers.dense(x, self.CHANNELS // 2, use_bias=True) - - if g is None: - - def g(x): # pylint: disable=function-redefined - return core_layers.dense(x, self.CHANNELS // 2, use_bias=True) - - if f_side_input is None: - f_side_input = [] - - if g_side_input is None: - g_side_input = [] - - if x is None: - x = random_ops.random_uniform( - [self.BATCH_SIZE, self.CHANNELS], dtype=dtypes.float32) - x1, x2 = array_ops.split(x, 2, axis=-1) - - with variable_scope.variable_scope("rev_test") as vs: - y1_rev, y2_rev = rev_block_lib.rev_block( - x1, - x2, - f, - g, - f_side_input=f_side_input, - g_side_input=g_side_input, - num_layers=self.NUM_LAYERS) - y_rev = array_ops.concat([y1_rev, y2_rev], axis=1) - fg_vars = vs.trainable_variables() - - num_vars = len(variables.global_variables()) - with variable_scope.variable_scope(vs, reuse=True): - y1, y2 = rev_block_lib.rev_block( - x1, - x2, - f, - g, - f_side_input=f_side_input, - g_side_input=g_side_input, - num_layers=self.NUM_LAYERS, - is_training=False) - y = array_ops.concat([y1, y2], axis=1) - # Ensure no new vars were created - full reuse - assert len(variables.global_variables()) == num_vars - - loss_rev = math_ops.reduce_mean(y_rev + 10.) - loss = math_ops.reduce_mean(y + 10.) - - wrt = [x] + f_side_input + g_side_input + fg_vars - grads_rev = gradients_impl.gradients(loss_rev, wrt) - grads = gradients_impl.gradients(loss, wrt) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - y_val, yd_val, gd_val, g_val = sess.run([y, y_rev, grads_rev, grads]) - self.assertAllClose(y_val, yd_val) - for g1, g2 in zip(gd_val, g_val): - self.assertAllClose(g1, g2, rtol=1e-5) - - def testRevBlock(self): - self._testRevBlock() - - def testSideInput(self): - f_side_input = random_ops.random_uniform( - [self.BATCH_SIZE, self.CHANNELS // 2]) - - def f(x, side_input): - return core_layers.dense( - x, self.CHANNELS // 2, use_bias=True) + side_input[0] - - self._testRevBlock(f=f, f_side_input=[f_side_input]) - - def testMultipleFns(self): - - def f1(x): - return core_layers.dense(x, self.CHANNELS // 2) - - def f2(x): - return core_layers.dense(x, self.CHANNELS // 2, activation=nn_ops.relu) - - self._testRevBlock(f=[f1, f2, f1, f2]) - - def testConvAndBatchNorm(self): - - x = random_ops.random_uniform( - [self.BATCH_SIZE, 10, self.CHANNELS], dtype=dtypes.float32) - - def f(x): - x = convolutional.conv1d(x, self.CHANNELS // 2, 3, padding="same") - x = layers.batch_norm(x, is_training=False) - x = convolutional.conv1d(x, self.CHANNELS // 2, 3, padding="same") - x = layers.batch_norm(x, is_training=False) - return x - - self._testRevBlock(x=x, f=f) - - def testReuse(self): - - def f(x): - return core_layers.dense(x, self.CHANNELS // 2) - - def g(x): - return core_layers.dense(x, self.CHANNELS // 2) - - x = random_ops.random_uniform( - [self.BATCH_SIZE, self.CHANNELS], dtype=dtypes.float32) - x1, x2 = array_ops.split(x, 2, axis=-1) - - with variable_scope.variable_scope("test"): - y1, y2 = rev_block_lib.rev_block(x1, x2, f, g, num_layers=self.NUM_LAYERS) - - num_vars_before = len(variables.global_variables()) - - with variable_scope.variable_scope("test", reuse=True): - y1, y2 = rev_block_lib.rev_block(x1, x2, f, g, num_layers=self.NUM_LAYERS) - - num_vars_after = len(variables.global_variables()) - self.assertEqual(num_vars_before, num_vars_after) - - loss = math_ops.reduce_mean(y1 + y2) - _ = gradients_impl.gradients(loss, - [x] + variables.trainable_variables()) - - with variable_scope.variable_scope("test", reuse=True): - y1, y2 = rev_block_lib.rev_block(x1, x2, f, g, num_layers=self.NUM_LAYERS) - - num_vars_after = len(variables.global_variables()) - self.assertEqual(num_vars_before, num_vars_after) - - -class RecomputeTest(test.TestCase): - - def testRecompute(self): - - def layer(x, name=None): - with variable_scope.variable_scope(name, default_name="layer"): - x = layers.layer_norm(x) - x = convolutional.conv1d( - x, - 10, - 1, - use_bias=False, - kernel_initializer=init_ops.constant_initializer(42.42)) - x = nn_ops.relu(x) - return x - - def fn(x): - out = x - for _ in range(3): - out = layer(out) - return out - - @rev_block_lib.recompute_grad - def fn_recompute(x): - return fn(x) - - @rev_block_lib.recompute_grad(use_data_dep=True) - def fn_use_data_dep(x): - return fn(x) - - @rev_block_lib.recompute_grad(tupleize_grads=True) - def fn_tupleize(x): - return fn(x) - - @rev_block_lib.recompute_grad(use_data_dep=True, tupleize_grads=True) - def fn_both(x): - return fn(x) - - x = random_ops.random_uniform((3, 1, 3)) - - names_and_fns = [ - ("recompute", fn_recompute), - ("regular", fn), - ("use_data_dep", fn_use_data_dep), - ("tupleize", fn_tupleize), - ("tuple_and_data_dep", fn_both), - ] - outputs_and_vars = [] - for name, wrapped_fn in names_and_fns: - with variable_scope.variable_scope(name, use_resource=True) as vs: - out = math_ops.reduce_sum(wrapped_fn(x)) - outputs_and_vars.append((out, vs.trainable_variables())) - - all_grads = [] - for out, scope_vars in outputs_and_vars: - all_grads.append(gradients_impl.gradients(out, scope_vars)) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - outputs = list(zip(*outputs_and_vars))[0] - outs, all_grads_val = sess.run([outputs, all_grads]) - - # All outputs are the same - current = outs[0] - for out in outs[1:]: - self.assertAllClose(current, out) - current = out - - # All gradients are the same - for grads in zip(all_grads_val): - current = grads[0] - for g in grads[1:]: - self.assertAllClose(current, g) - current = g - - def testDoubleCallInSameScopeFails(self): - - @rev_block_lib.recompute_grad - def layer_with_recompute(inputs): - return core_layers.dense(inputs, 2) - - with variable_scope.variable_scope("layer", use_resource=True): - inputs = array_ops.ones((2, 4), dtypes.float32) - out1 = layer_with_recompute(inputs) - out2 = layer_with_recompute(inputs) + out1 - out = math_ops.reduce_sum(out2) - - tvars = variables.trainable_variables() - assert len(tvars) == 4 - with self.assertRaisesWithPredicateMatch( - ValueError, "called twice in the same enclosing scope"): - gradients_impl.gradients(out, [inputs] + tvars) - - def testDoubleCallInUniqueScope(self): - - @rev_block_lib.recompute_grad - def layer_with_recompute(inputs): - with variable_scope.variable_scope("inner", use_resource=True): - return core_layers.dense(inputs, 2) - - with variable_scope.variable_scope("layer", use_resource=True): - inputs = array_ops.ones((2, 4), dtypes.float32) - - with variable_scope.variable_scope("layer1", use_resource=True): - out1 = layer_with_recompute(inputs) - with variable_scope.variable_scope("layer2", use_resource=True): - out2 = layer_with_recompute(inputs) + out1 - out = math_ops.reduce_sum(out2) - - tvars = variables.trainable_variables() - assert len(tvars) == 4 - grads = gradients_impl.gradients(out, [inputs] + tvars) - for grad in grads: - self.assertTrue(grad is not None) - - def testWithIsRecomputeKwarg(self): - - kwarg_values = [] - - @rev_block_lib.recompute_grad - def layer_with_recompute(inputs, is_recomputing=False): - kwarg_values.append(is_recomputing) - out = core_layers.dense(inputs, 2) - out = normalization_layers.batch_normalization(out, training=True) - if is_recomputing: - # Ensure that the updates are not duplicated by popping off the latest - # 2 additions. - update_ops = ops.get_collection_ref(ops.GraphKeys.UPDATE_OPS) - update_ops.pop() - update_ops.pop() - return out - - x = array_ops.ones((2, 4), dtypes.float32) - with variable_scope.variable_scope("layer1", use_resource=True): - y = layer_with_recompute(x) - loss = math_ops.reduce_sum(y) - tvars = variables.trainable_variables() - gradients_impl.gradients(loss, [x] + tvars) - - update_ops = ops.get_collection(ops.GraphKeys.UPDATE_OPS) - self.assertEqual(2, len(update_ops)) - self.assertEqual([False, True], kwarg_values) - - def testWithoutVariables(self): - - def concat_n(layer_list, num_inputs): - return math_ops.reduce_sum( - array_ops.concat([x for x in layer_list[-num_inputs:]], axis=-1), - axis=1, keepdims=True) - - @rev_block_lib.recompute_grad - def concat_n_wrap(*args): - return concat_n(args, 3) - - # DenseNet-style layers - layer_list = [random_ops.random_uniform((4, 8))] - for _ in range(5): - layer_list.append(math_ops.sqrt(concat_n_wrap(*layer_list))) - - grads = gradients_impl.gradients(layer_list[-1], layer_list[0]) - with self.cached_session() as sess: - sess.run(grads) - - def testErrorOnClosedOverTensor(self): - x = random_ops.random_uniform((4, 8)) - y = random_ops.random_uniform((4, 8)) - z = x * y - - with self.assertRaisesWithPredicateMatch(ValueError, "closes over"): - @rev_block_lib.recompute_grad - def fn_with_capture(a): # pylint: disable=unused-variable - return a * z - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/layers/python/layers/summaries.py b/tensorflow/contrib/layers/python/layers/summaries.py deleted file mode 100644 index 6e61eb0dfa9..00000000000 --- a/tensorflow/contrib/layers/python/layers/summaries.py +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utility functions for summary creation.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import re - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import standard_ops -from tensorflow.python.summary import summary - -__all__ = [ - 'summarize_tensor', - 'summarize_activation', - 'summarize_tensors', - 'summarize_collection', - 'summarize_variables', - 'summarize_weights', - 'summarize_biases', - 'summarize_activations', -] - -# TODO(wicke): add more unit tests for summarization functions. - - -def _add_scalar_summary(tensor, tag=None): - """Add a scalar summary operation for the tensor. - - Args: - tensor: The tensor to summarize. - tag: The tag to use, if None then use tensor's op's name. - - Returns: - The created histogram summary. - - Raises: - ValueError: If the tag is already in use or the rank is not 0. - """ - tensor.get_shape().assert_has_rank(0) - tag = tag or '%s_summary' % tensor.op.name - return summary.scalar(tag, tensor) - - -def _add_histogram_summary(tensor, tag=None): - """Add a summary operation for the histogram of a tensor. - - Args: - tensor: The tensor to summarize. - tag: The tag to use, if None then use tensor's op's name. - - Returns: - The created histogram summary. - - Raises: - ValueError: If the tag is already in use. - """ - tag = tag or '%s_summary' % tensor.op.name - return summary.histogram(tag, tensor) - - -def summarize_activation(op): - """Summarize an activation. - - This applies the given activation and adds useful summaries specific to the - activation. - - Args: - op: The tensor to summarize (assumed to be a layer activation). - Returns: - The summary op created to summarize `op`. - """ - if op.op.type in ('Relu', 'Softplus', 'Relu6'): - # Using inputs to avoid floating point equality and/or epsilons. - _add_scalar_summary( - standard_ops.reduce_mean( - standard_ops.to_float( - standard_ops.less(op.op.inputs[ - 0], standard_ops.cast(0.0, op.op.inputs[0].dtype)))), - '%s/zeros' % op.op.name) - if op.op.type == 'Relu6': - _add_scalar_summary( - standard_ops.reduce_mean( - standard_ops.to_float( - standard_ops.greater(op.op.inputs[ - 0], standard_ops.cast(6.0, op.op.inputs[0].dtype)))), - '%s/sixes' % op.op.name) - return _add_histogram_summary(op, '%s/activation' % op.op.name) - - -def summarize_tensor(tensor, tag=None): - """Summarize a tensor using a suitable summary type. - - This function adds a summary op for `tensor`. The type of summary depends on - the shape of `tensor`. For scalars, a `scalar_summary` is created, for all - other tensors, `histogram_summary` is used. - - Args: - tensor: The tensor to summarize - tag: The tag to use, if None then use tensor's op's name. - - Returns: - The summary op created or None for string tensors. - """ - # Skips string tensors and boolean tensors (not handled by the summaries). - if (tensor.dtype.is_compatible_with(dtypes.string) or - tensor.dtype.base_dtype == dtypes.bool): - return None - - if tensor.get_shape().ndims == 0: - # For scalars, use a scalar summary. - return _add_scalar_summary(tensor, tag) - else: - # We may land in here if the rank is still unknown. The histogram won't - # hurt if this ends up being a scalar. - return _add_histogram_summary(tensor, tag) - - -def summarize_tensors(tensors, summarizer=summarize_tensor): - """Summarize a set of tensors.""" - return [summarizer(tensor) for tensor in tensors] - - -def summarize_collection(collection, - name_filter=None, - summarizer=summarize_tensor): - """Summarize a graph collection of tensors, possibly filtered by name.""" - tensors = [] - for op in ops.get_collection(collection): - if name_filter is None or re.match(name_filter, op.op.name): - tensors.append(op) - return summarize_tensors(tensors, summarizer) - - -# Utility functions for commonly used collections -summarize_variables = functools.partial(summarize_collection, - ops.GraphKeys.GLOBAL_VARIABLES) - -summarize_weights = functools.partial(summarize_collection, - ops.GraphKeys.WEIGHTS) - -summarize_biases = functools.partial(summarize_collection, ops.GraphKeys.BIASES) - - -def summarize_activations(name_filter=None, summarizer=summarize_activation): - """Summarize activations, using `summarize_activation` to summarize.""" - return summarize_collection(ops.GraphKeys.ACTIVATIONS, name_filter, - summarizer) diff --git a/tensorflow/contrib/layers/python/layers/summaries_test.py b/tensorflow/contrib/layers/python/layers/summaries_test.py deleted file mode 100644 index 2ec2af9d442..00000000000 --- a/tensorflow/contrib/layers/python/layers/summaries_test.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for regularizers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.layers.python.layers import summaries as summaries_lib -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class SummariesTest(test.TestCase): - - def test_summarize_scalar_tensor(self): - with self.cached_session(): - scalar_var = variables.Variable(1) - summary_op = summaries_lib.summarize_tensor(scalar_var) - self.assertEquals(summary_op.op.type, 'ScalarSummary') - - def test_summarize_multidim_tensor(self): - with self.cached_session(): - tensor_var = variables.Variable([1, 2, 3]) - summary_op = summaries_lib.summarize_tensor(tensor_var) - self.assertEquals(summary_op.op.type, 'HistogramSummary') - - def test_summarize_activation(self): - with self.cached_session(): - var = variables.Variable(1) - op = array_ops.identity(var, name='SummaryTest') - summary_op = summaries_lib.summarize_activation(op) - - self.assertEquals(summary_op.op.type, 'HistogramSummary') - names = [op.op.name for op in ops.get_collection(ops.GraphKeys.SUMMARIES)] - self.assertEquals(len(names), 1) - self.assertIn(u'SummaryTest/activation', names) - - def test_summarize_activation_relu(self): - with self.cached_session(): - var = variables.Variable(1) - op = nn_ops.relu(var, name='SummaryTest') - summary_op = summaries_lib.summarize_activation(op) - - self.assertEquals(summary_op.op.type, 'HistogramSummary') - names = [op.op.name for op in ops.get_collection(ops.GraphKeys.SUMMARIES)] - self.assertEquals(len(names), 2) - self.assertIn(u'SummaryTest/zeros', names) - self.assertIn(u'SummaryTest/activation', names) - - def test_summarize_activation_relu6(self): - with self.cached_session(): - var = variables.Variable(1) - op = nn_ops.relu6(var, name='SummaryTest') - summary_op = summaries_lib.summarize_activation(op) - - self.assertEquals(summary_op.op.type, 'HistogramSummary') - names = [op.op.name for op in ops.get_collection(ops.GraphKeys.SUMMARIES)] - self.assertEquals(len(names), 3) - self.assertIn(u'SummaryTest/zeros', names) - self.assertIn(u'SummaryTest/sixes', names) - self.assertIn(u'SummaryTest/activation', names) - - def test_summarize_collection_regex(self): - with self.cached_session(): - var = variables.Variable(1) - array_ops.identity(var, name='Test1') - ops.add_to_collection('foo', array_ops.identity(var, name='Test2')) - ops.add_to_collection('foo', array_ops.identity(var, name='Foobar')) - ops.add_to_collection('foo', array_ops.identity(var, name='Test3')) - summaries = summaries_lib.summarize_collection('foo', r'Test[123]') - names = [op.op.name for op in summaries] - self.assertEquals(len(names), 2) - self.assertIn(u'Test2_summary', names) - self.assertIn(u'Test3_summary', names) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/layers/python/layers/target_column.py b/tensorflow/contrib/layers/python/layers/target_column.py deleted file mode 100644 index 131b1e0dba2..00000000000 --- a/tensorflow/contrib/layers/python/layers/target_column.py +++ /dev/null @@ -1,529 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""TargetColumn abstract a single head in the model. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import six - -from tensorflow.contrib.framework import deprecated -from tensorflow.contrib.losses.python.losses import loss_ops -from tensorflow.contrib.metrics.python.ops import metric_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn - - -@deprecated( - "2016-11-12", "This file will be removed after the deprecation date." - "Please switch to " - "third_party/tensorflow/contrib/learn/python/learn/estimators/head.py") -def regression_target(label_name=None, - weight_column_name=None, - label_dimension=1): - """Creates a _TargetColumn for linear regression. - - Args: - label_name: String, name of the key in label dict. Can be null if label - is a tensor (single headed models). - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - label_dimension: dimension of the target for multilabels. - - Returns: - An instance of _TargetColumn - """ - return _RegressionTargetColumn( - loss_fn=_mean_squared_loss, - label_name=label_name, - weight_column_name=weight_column_name, - label_dimension=label_dimension) - - -# TODO(zakaria): Add logistic_regression_target - - -@deprecated( - "2016-11-12", "This file will be removed after the deprecation date." - "Please switch to " - "third_party/tensorflow/contrib/learn/python/learn/estimators/head.py") -def multi_class_target(n_classes, label_name=None, weight_column_name=None): - """Creates a _TargetColumn for multi class single label classification. - - The target column uses softmax cross entropy loss. - - Args: - n_classes: Integer, number of classes, must be >= 2 - label_name: String, name of the key in label dict. Can be null if label - is a tensor (single headed models). - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - - Returns: - An instance of _MultiClassTargetColumn. - - Raises: - ValueError: if n_classes is < 2 - """ - if n_classes < 2: - raise ValueError("n_classes must be > 1 for classification.") - if n_classes == 2: - loss_fn = _log_loss_with_two_classes - else: - loss_fn = _softmax_cross_entropy_loss - return _MultiClassTargetColumn( - loss_fn=loss_fn, - n_classes=n_classes, - label_name=label_name, - weight_column_name=weight_column_name) - - -@deprecated( - "2016-11-12", "This file will be removed after the deprecation date." - "Please switch to " - "third_party/tensorflow/contrib/learn/python/learn/estimators/head.py") -def binary_svm_target(label_name=None, weight_column_name=None): - """Creates a _TargetColumn for binary classification with SVMs. - - The target column uses binary hinge loss. - - Args: - label_name: String, name of the key in label dict. Can be null if label - is a tensor (single headed models). - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - - Returns: - An instance of _TargetColumn. - - """ - return _BinarySvmTargetColumn( - label_name=label_name, weight_column_name=weight_column_name) - - -@deprecated( - "2016-11-12", "This file will be removed after the deprecation date." - "Please switch to " - "third_party/tensorflow/contrib/learn/python/learn/estimators/head.py") -class ProblemType(object): - UNSPECIFIED = 0 - CLASSIFICATION = 1 - LINEAR_REGRESSION = 2 - LOGISTIC_REGRESSION = 3 - - -class _TargetColumn(object): - """_TargetColumn is the abstraction for a single head in a model. - - Args: - loss_fn: a function that returns the loss tensor. - num_label_columns: Integer, number of label columns. - label_name: String, name of the key in label dict. Can be null if label - is a tensor (single headed models). - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - - Raises: - ValueError: if loss_fn or n_classes are missing. - """ - - def __init__(self, loss_fn, num_label_columns, label_name, weight_column_name, - problem_type): - if not loss_fn: - raise ValueError("loss_fn must be provided") - if num_label_columns is None: # n_classes can be 0 - raise ValueError("num_label_columns must be provided") - - self._loss_fn = loss_fn - self._num_label_columns = num_label_columns - self._label_name = label_name - self._weight_column_name = weight_column_name - self._problem_type = problem_type - - def logits_to_predictions(self, logits, proba=False): - # Abstrat, Subclasses must implement. - raise NotImplementedError() - - def get_eval_ops(self, features, logits, labels, metrics=None): - """Returns eval op.""" - raise NotImplementedError - - @property - def label_name(self): - return self._label_name - - @property - def weight_column_name(self): - return self._weight_column_name - - @property - def num_label_columns(self): - return self._num_label_columns - - def get_weight_tensor(self, features): - if not self._weight_column_name: - return None - else: - return array_ops.reshape( - math_ops.cast(features[self._weight_column_name], dtypes.float32), - shape=(-1,)) - - @property - def problem_type(self): - return self._problem_type - - def _weighted_loss(self, loss, weight_tensor): - """Returns cumulative weighted loss.""" - unweighted_loss = array_ops.reshape(loss, shape=(-1,)) - weighted_loss = math_ops.multiply(unweighted_loss, - array_ops.reshape( - weight_tensor, shape=(-1,))) - return weighted_loss - - def training_loss(self, logits, target, features, name="training_loss"): - """Returns training loss tensor for this head. - - Training loss is different from the loss reported on the tensorboard as we - should respect the example weights when computing the gradient. - - L = sum_{i} w_{i} * l_{i} / B - - where B is the number of examples in the batch, l_{i}, w_{i} are individual - losses, and example weight. - - Args: - logits: logits, a float tensor. - target: either a tensor for labels or in multihead case, a dict of string - to target tensor. - features: features dict. - name: Op name. - - Returns: - Loss tensor. - """ - target = target[self.name] if isinstance(target, dict) else target - loss_unweighted = self._loss_fn(logits, target) - - weight_tensor = self.get_weight_tensor(features) - if weight_tensor is None: - return math_ops.reduce_mean(loss_unweighted, name=name) - loss_weighted = self._weighted_loss(loss_unweighted, weight_tensor) - return math_ops.reduce_mean(loss_weighted, name=name) - - def loss(self, logits, target, features): - """Returns loss tensor for this head. - - The loss returned is the weighted average. - - L = sum_{i} w_{i} * l_{i} / sum_{i} w_{i} - - Args: - logits: logits, a float tensor. - target: either a tensor for labels or in multihead case, a dict of string - to target tensor. - features: features dict. - - Returns: - Loss tensor. - """ - target = target[self.name] if isinstance(target, dict) else target - loss_unweighted = self._loss_fn(logits, target) - - weight_tensor = self.get_weight_tensor(features) - if weight_tensor is None: - return math_ops.reduce_mean(loss_unweighted, name="loss") - loss_weighted = self._weighted_loss(loss_unweighted, weight_tensor) - return math_ops.div( - math_ops.reduce_sum(loss_weighted), - math_ops.cast(math_ops.reduce_sum(weight_tensor), dtypes.float32), - name="loss") - - -class _RegressionTargetColumn(_TargetColumn): - """_TargetColumn for regression.""" - - def __init__(self, loss_fn, label_name, weight_column_name, label_dimension): - super(_RegressionTargetColumn, self).__init__( - loss_fn=loss_fn, - num_label_columns=label_dimension, - label_name=label_name, - weight_column_name=weight_column_name, - problem_type=ProblemType.LINEAR_REGRESSION) - - def logits_to_predictions(self, logits, proba=False): - if self.num_label_columns == 1: - return array_ops.squeeze(logits, axis=[1]) - return logits - - def get_eval_ops(self, features, logits, labels, metrics=None): - loss = self.loss(logits, labels, features) - result = {"loss": metric_ops.streaming_mean(loss)} - if metrics: - predictions = self.logits_to_predictions(logits, proba=False) - result.update( - _run_metrics(predictions, labels, metrics, - self.get_weight_tensor(features))) - return result - - -class _MultiClassTargetColumn(_TargetColumn): - """_TargetColumn for classification.""" - - # TODO(zakaria): support multilabel. - def __init__(self, loss_fn, n_classes, label_name, weight_column_name): - if n_classes < 2: - raise ValueError("n_classes must be >= 2") - super(_MultiClassTargetColumn, self).__init__( - loss_fn=loss_fn, - num_label_columns=1 if n_classes == 2 else n_classes, - label_name=label_name, - weight_column_name=weight_column_name, - problem_type=ProblemType.CLASSIFICATION) - - def logits_to_predictions(self, logits, proba=False): - if self.num_label_columns == 1: - logits = array_ops.concat([array_ops.zeros_like(logits), logits], 1) - - if proba: - return nn.softmax(logits) - else: - return math_ops.argmax(logits, 1) - - def _default_eval_metrics(self): - if self._num_label_columns == 1: - return get_default_binary_metrics_for_eval(thresholds=[.5]) - return {} - - def get_eval_ops(self, features, logits, labels, metrics=None): - loss = self.loss(logits, labels, features) - result = {"loss": metric_ops.streaming_mean(loss)} - - # Adds default metrics. - if metrics is None: - # TODO(b/29366811): This currently results in both an "accuracy" and an - # "accuracy/threshold_0.500000_mean" metric for binary classification. - metrics = {("accuracy", "classes"): metric_ops.streaming_accuracy} - - predictions = math_ops.sigmoid(logits) - labels_float = math_ops.cast(labels, dtypes.float32) - - default_metrics = self._default_eval_metrics() - for metric_name, metric_op in default_metrics.items(): - result[metric_name] = metric_op(predictions, labels_float) - - class_metrics = {} - proba_metrics = {} - for name, metric_op in six.iteritems(metrics): - if isinstance(name, tuple): - if len(name) != 2: - raise ValueError("Ignoring metric {}. It returned a tuple with " - "len {}, expected 2.".format(name, len(name))) - else: - if name[1] not in ["classes", "probabilities"]: - raise ValueError("Ignoring metric {}. The 2nd element of its " - "name should be either 'classes' or " - "'probabilities'.".format(name)) - elif name[1] == "classes": - class_metrics[name[0]] = metric_op - else: - proba_metrics[name[0]] = metric_op - elif isinstance(name, str): - class_metrics[name] = metric_op - else: - raise ValueError("Ignoring metric {}. Its name is not in the correct " - "form.".format(name)) - if class_metrics: - class_predictions = self.logits_to_predictions(logits, proba=False) - result.update( - _run_metrics(class_predictions, labels, class_metrics, - self.get_weight_tensor(features))) - if proba_metrics: - predictions = self.logits_to_predictions(logits, proba=True) - result.update( - _run_metrics(predictions, labels, proba_metrics, - self.get_weight_tensor(features))) - return result - - -class _BinarySvmTargetColumn(_MultiClassTargetColumn): - """_TargetColumn for binary classification using SVMs.""" - - def __init__(self, label_name, weight_column_name): - - def loss_fn(logits, target): - check_shape_op = control_flow_ops.Assert( - math_ops.less_equal(array_ops.rank(target), 2), - ["target's shape should be either [batch_size, 1] or [batch_size]"]) - with ops.control_dependencies([check_shape_op]): - target = array_ops.reshape( - target, shape=[array_ops.shape(target)[0], 1]) - return loss_ops.hinge_loss(logits, target) - - super(_BinarySvmTargetColumn, self).__init__( - loss_fn=loss_fn, - n_classes=2, - label_name=label_name, - weight_column_name=weight_column_name) - - def logits_to_predictions(self, logits, proba=False): - if proba: - raise ValueError( - "logits to probabilities is not supported for _BinarySvmTargetColumn") - - logits = array_ops.concat([array_ops.zeros_like(logits), logits], 1) - return math_ops.argmax(logits, 1) - - -# TODO(zakaria): use contrib losses. -def _mean_squared_loss(logits, target): - # To prevent broadcasting inside "-". - if len(target.get_shape()) == 1: - target = array_ops.expand_dims(target, axis=1) - - logits.get_shape().assert_is_compatible_with(target.get_shape()) - return math_ops.squared_difference(logits, - math_ops.cast(target, dtypes.float32)) - - -def _log_loss_with_two_classes(logits, target): - # sigmoid_cross_entropy_with_logits requires [batch_size, 1] target. - if len(target.get_shape()) == 1: - target = array_ops.expand_dims(target, axis=1) - loss_vec = nn.sigmoid_cross_entropy_with_logits( - labels=math_ops.cast(target, dtypes.float32), logits=logits) - return loss_vec - - -def _softmax_cross_entropy_loss(logits, target): - # Check that we got integer for classification. - if not target.dtype.is_integer: - raise ValueError("Target's dtype should be integer " - "Instead got %s." % target.dtype) - # sparse_softmax_cross_entropy_with_logits requires [batch_size] target. - if len(target.get_shape()) == 2: - target = array_ops.squeeze(target, axis=[1]) - loss_vec = nn.sparse_softmax_cross_entropy_with_logits( - labels=target, logits=logits) - return loss_vec - - -def _run_metrics(predictions, labels, metrics, weights): - result = {} - labels = math_ops.cast(labels, predictions.dtype) - for name, metric in six.iteritems(metrics or {}): - if weights is not None: - result[name] = metric(predictions, labels, weights=weights) - else: - result[name] = metric(predictions, labels) - - return result - - -@deprecated( - "2016-11-12", "This file will be removed after the deprecation date." - "Please switch to " - "third_party/tensorflow/contrib/learn/python/learn/estimators/head.py") -def get_default_binary_metrics_for_eval(thresholds): - """Returns a dictionary of basic metrics for logistic regression. - - Args: - thresholds: List of floating point thresholds to use for accuracy, - precision, and recall metrics. If None, defaults to [0.5]. - - Returns: - Dictionary mapping metrics string names to metrics functions. - """ - metrics = {} - metrics[_MetricKeys.PREDICTION_MEAN] = _predictions_streaming_mean - metrics[_MetricKeys.TARGET_MEAN] = _labels_streaming_mean - # Also include the streaming mean of the label as an accuracy baseline, as - # a reminder to users. - metrics[_MetricKeys.ACCURACY_BASELINE] = _labels_streaming_mean - - metrics[_MetricKeys.AUC] = _streaming_auc - - for threshold in thresholds: - metrics[_MetricKeys.ACCURACY_MEAN % - threshold] = _accuracy_at_threshold(threshold) - # Precision for positive examples. - metrics[_MetricKeys.PRECISION_MEAN % threshold] = _streaming_at_threshold( - metric_ops.streaming_precision_at_thresholds, threshold) - # Recall for positive examples. - metrics[_MetricKeys.RECALL_MEAN % threshold] = _streaming_at_threshold( - metric_ops.streaming_recall_at_thresholds, threshold) - - return metrics - - -def _float_weights_or_none(weights): - if weights is None: - return None - return math_ops.cast(weights, dtypes.float32) - - -def _labels_streaming_mean(unused_predictions, labels, weights=None): - return metric_ops.streaming_mean(labels, weights=weights) - - -def _predictions_streaming_mean(predictions, unused_labels, weights=None): - return metric_ops.streaming_mean(predictions, weights=weights) - - -def _streaming_auc(predictions, labels, weights=None): - return metric_ops.streaming_auc( - predictions, labels, weights=_float_weights_or_none(weights)) - - -def _accuracy_at_threshold(threshold): - - def _accuracy_metric(predictions, labels, weights=None): - threshold_predictions = math_ops.cast( - math_ops.greater_equal(predictions, threshold), dtypes.float32) - return metric_ops.streaming_accuracy( - predictions=threshold_predictions, labels=labels, weights=weights) - - return _accuracy_metric - - -def _streaming_at_threshold(streaming_metrics_fn, threshold): - - def _streaming_metrics(predictions, labels, weights=None): - precision_tensor, update_op = streaming_metrics_fn( - predictions, - labels=labels, - thresholds=[threshold], - weights=_float_weights_or_none(weights)) - return array_ops.squeeze(precision_tensor), update_op - - return _streaming_metrics - - -class _MetricKeys(object): - AUC = "auc" - PREDICTION_MEAN = "labels/prediction_mean" - TARGET_MEAN = "labels/actual_target_mean" - ACCURACY_BASELINE = "accuracy/baseline_target_mean" - ACCURACY_MEAN = "accuracy/threshold_%f_mean" - PRECISION_MEAN = "precision/positive_threshold_%f_mean" - RECALL_MEAN = "recall/positive_threshold_%f_mean" diff --git a/tensorflow/contrib/layers/python/layers/target_column_test.py b/tensorflow/contrib/layers/python/layers/target_column_test.py deleted file mode 100644 index d5d03fb1ebc..00000000000 --- a/tensorflow/contrib/layers/python/layers/target_column_test.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for TargetColumn.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.layers.python.layers import target_column as target_column_lib -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class RegressionTargetColumnTest(test.TestCase): - - # TODO(zakaria): test multilabel regression. - def testRegression(self): - target_column = target_column_lib.regression_target() - with ops.Graph().as_default(), session.Session() as sess: - prediction = constant_op.constant([[1.], [1.], [3.]]) - labels = constant_op.constant([[0.], [1.], [1.]]) - self.assertAlmostEqual( - 5. / 3, sess.run(target_column.loss(prediction, labels, {}))) - - def testRegressionWithWeights(self): - target_column = target_column_lib.regression_target( - weight_column_name="label_weight") - with ops.Graph().as_default(), session.Session() as sess: - features = {"label_weight": constant_op.constant([[2.], [5.], [0.]])} - prediction = constant_op.constant([[1.], [1.], [3.]]) - labels = constant_op.constant([[0.], [1.], [1.]]) - self.assertAlmostEqual( - 2. / 7, - sess.run(target_column.loss(prediction, labels, features)), - places=3) - self.assertAlmostEqual( - 2. / 3, - sess.run(target_column.training_loss(prediction, labels, features)), - places=3) - - -class MultiClassTargetColumnTest(test.TestCase): - - def testBinaryClassification(self): - target_column = target_column_lib.multi_class_target(n_classes=2) - with ops.Graph().as_default(), session.Session() as sess: - logits = constant_op.constant([[1.], [1.]]) - labels = constant_op.constant([[1.], [0.]]) - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - self.assertAlmostEqual( - 0.81326175, - sess.run(target_column.loss(logits, labels, {})), - delta=1e-6) - - def testBinaryClassificationWithWeights(self): - target_column = target_column_lib.multi_class_target( - n_classes=2, weight_column_name="label_weight") - with ops.Graph().as_default(), session.Session() as sess: - features = {"label_weight": constant_op.constant([[1.], [0.]])} - logits = constant_op.constant([[1.], [1.]]) - labels = constant_op.constant([[1.], [0.]]) - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - self.assertAlmostEqual( - .31326166, - sess.run(target_column.loss(logits, labels, features)), - delta=1e-6) - - def testBinaryEvalMetrics(self): - target_column = target_column_lib.multi_class_target(n_classes=2) - with ops.Graph().as_default(), session.Session() as sess: - logits = constant_op.constant([[1.], [1.], [-1.]]) - labels = constant_op.constant([[1.], [0.], [1.]]) - eval_dict = target_column.get_eval_ops({}, logits, labels) - # TODO(zakaria): test all metrics - accuracy_op, update_op = eval_dict["accuracy/threshold_0.500000_mean"] - sess.run(variables.global_variables_initializer()) - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertAlmostEqual(1.0 / 3, sess.run(accuracy_op)) - - def testMultiClass(self): - target_column = target_column_lib.multi_class_target(n_classes=3) - with ops.Graph().as_default(), session.Session() as sess: - logits = constant_op.constant([[1., 0., 0.]]) - labels = constant_op.constant([2]) - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - self.assertAlmostEqual(1.5514446, - sess.run(target_column.loss(logits, labels, {}))) - - def testMultiClassWithWeight(self): - target_column = target_column_lib.multi_class_target( - n_classes=3, weight_column_name="label_weight") - with ops.Graph().as_default(), session.Session() as sess: - features = {"label_weight": constant_op.constant([0.1])} - logits = constant_op.constant([[1., 0., 0.]]) - labels = constant_op.constant([2]) - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - self.assertAlmostEqual( - 1.5514446, sess.run(target_column.loss(logits, labels, features))) - - def testMultiClassWithInvalidNClass(self): - try: - target_column_lib.multi_class_target(n_classes=1) - self.fail("Softmax with no n_classes did not raise error.") - except ValueError: - # Expected - pass - - def testMultiClassEvalMetrics(self): - target_column = target_column_lib.multi_class_target(n_classes=3) - with ops.Graph().as_default(), session.Session() as sess: - logits = constant_op.constant([[1., 0., 0.]]) - labels = constant_op.constant([2]) - eval_dict = target_column.get_eval_ops({}, logits, labels) - loss_op, update_op = eval_dict["loss"] - sess.run(variables.global_variables_initializer()) - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - self.assertAlmostEqual(1.5514446, sess.run(loss_op)) - - def testBinarySVMDefaultWeights(self): - target_column = target_column_lib.binary_svm_target() - predictions = constant_op.constant([[-0.5], [1.2]]) - labels = constant_op.constant([0, 1]) - loss = target_column.loss(predictions, labels, {}) - # Prediction for first example is in the right side of the hyperplane (i.e., - # < 0) but it is within the [-1,1] margin. There is a 0.5 loss incurred by - # this example. The 2nd prediction is outside the margin so it incurs no - # loss at all. The overall (normalized) loss is therefore 0.5/(1+1) = 0.25. - with session.Session() as sess: - self.assertAlmostEqual(0.25, sess.run(loss)) - - def testBinarySVMWithWeights(self): - target_column = target_column_lib.binary_svm_target( - weight_column_name="weights") - predictions = constant_op.constant([[-0.7], [0.2]]) - labels = constant_op.constant([0, 1]) - features = {"weights": constant_op.constant([2.0, 10.0])} - loss = target_column.loss(predictions, labels, features) - training_loss = target_column.training_loss(predictions, labels, features) - # Prediction for both examples are in the right side of the hyperplane but - # within the margin. The (weighted) loss incurred is 2*0.3=0.6 and 10*0.8=8 - # respectively. The overall (normalized) loss is therefore 8.6/12. - with session.Session() as sess: - self.assertAlmostEqual(8.6 / 12, sess.run(loss), places=3) - self.assertAlmostEqual(8.6 / 2, sess.run(training_loss), places=3) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/layers/python/layers/utils.py b/tensorflow/contrib/layers/python/layers/utils.py deleted file mode 100644 index 3080d36d652..00000000000 --- a/tensorflow/contrib/layers/python/layers/utils.py +++ /dev/null @@ -1,369 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Common util functions used by layers. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from collections import namedtuple -from collections import OrderedDict -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import variables - -__all__ = ['collect_named_outputs', - 'constant_value', - 'static_cond', - 'smart_cond', - 'get_variable_collections', - 'two_element_tuple', - 'n_positive_integers', - 'channel_dimension', - 'last_dimension'] - -NamedOutputs = namedtuple('NamedOutputs', ['name', 'outputs']) - - -def collect_named_outputs(collections, alias, outputs): - """Add `Tensor` outputs tagged with alias to collections. - - It is useful to collect end-points or tags for summaries. Example of usage: - - logits = collect_named_outputs('end_points', 'inception_v3/logits', logits) - assert 'inception_v3/logits' in logits.aliases - - Args: - collections: A collection or list of collections. If None skip collection. - alias: String to append to the list of aliases of outputs, for example, - 'inception_v3/conv1'. - outputs: Tensor, an output tensor to collect - - Returns: - The outputs Tensor to allow inline call. - """ - if collections: - append_tensor_alias(outputs, alias) - ops.add_to_collections(collections, outputs) - return outputs - - -def append_tensor_alias(tensor, alias): - """Append an alias to the list of aliases of the tensor. - - Args: - tensor: A `Tensor`. - alias: String, to add to the list of aliases of the tensor. - - Returns: - The tensor with a new alias appended to its list of aliases. - """ - # Remove ending '/' if present. - if alias[-1] == '/': - alias = alias[:-1] - if hasattr(tensor, 'aliases'): - tensor.aliases.append(alias) - else: - tensor.aliases = [alias] - return tensor - - -def gather_tensors_aliases(tensors): - """Given a list of tensors, gather their aliases. - - Args: - tensors: A list of `Tensors`. - - Returns: - A list of strings with the aliases of all tensors. - """ - aliases = [] - for tensor in tensors: - aliases += get_tensor_aliases(tensor) - return aliases - - -def get_tensor_aliases(tensor): - """Get a list with the aliases of the input tensor. - - If the tensor does not have any alias, it would default to its its op.name or - its name. - - Args: - tensor: A `Tensor`. - - Returns: - A list of strings with the aliases of the tensor. - """ - if hasattr(tensor, 'aliases'): - aliases = tensor.aliases - else: - if tensor.name[-2:] == ':0': - # Use op.name for tensor ending in :0 - aliases = [tensor.op.name] - else: - aliases = [tensor.name] - return aliases - - -def convert_collection_to_dict(collection, clear_collection=False): - """Returns an OrderedDict of Tensors with their aliases as keys. - - Args: - collection: A collection. - clear_collection: When True, it clears the collection after converting to - OrderedDict. - - Returns: - An OrderedDict of {alias: tensor} - """ - output = OrderedDict((alias, tensor) - for tensor in ops.get_collection(collection) - for alias in get_tensor_aliases(tensor)) - if clear_collection: - ops.get_default_graph().clear_collection(collection) - return output - - -def constant_value(value_or_tensor_or_var, dtype=None): - """Returns value if value_or_tensor_or_var has a constant value. - - Args: - value_or_tensor_or_var: A value, a `Tensor` or a `Variable`. - dtype: Optional `tf.dtype`, if set it would check it has the right - dtype. - - Returns: - The constant value or None if it not constant. - - Raises: - ValueError: if value_or_tensor_or_var is None or the tensor_variable has the - wrong dtype. - """ - if value_or_tensor_or_var is None: - raise ValueError('value_or_tensor_or_var cannot be None') - value = value_or_tensor_or_var - if isinstance(value_or_tensor_or_var, (ops.Tensor, variables.Variable)): - if dtype and value_or_tensor_or_var.dtype != dtype: - raise ValueError('It has the wrong type %s instead of %s' % ( - value_or_tensor_or_var.dtype, dtype)) - if isinstance(value_or_tensor_or_var, variables.Variable): - value = None - else: - value = tensor_util.constant_value(value_or_tensor_or_var) - return value - - -def static_cond(pred, fn1, fn2): - """Return either fn1() or fn2() based on the boolean value of `pred`. - - Same signature as `control_flow_ops.cond()` but requires pred to be a bool. - - Args: - pred: A value determining whether to return the result of `fn1` or `fn2`. - fn1: The callable to be performed if pred is true. - fn2: The callable to be performed if pred is false. - - Returns: - Tensors returned by the call to either `fn1` or `fn2`. - - Raises: - TypeError: if `fn1` or `fn2` is not callable. - """ - if not callable(fn1): - raise TypeError('fn1 must be callable.') - if not callable(fn2): - raise TypeError('fn2 must be callable.') - if pred: - return fn1() - else: - return fn2() - - -def smart_cond(pred, fn1, fn2, name=None): - """Return either fn1() or fn2() based on the boolean predicate/value `pred`. - - If `pred` is bool or has a constant value it would use `static_cond`, - otherwise it would use `tf.cond`. - - Args: - pred: A scalar determining whether to return the result of `fn1` or `fn2`. - fn1: The callable to be performed if pred is true. - fn2: The callable to be performed if pred is false. - name: Optional name prefix when using tf.cond - Returns: - Tensors returned by the call to either `fn1` or `fn2`. - """ - pred_value = constant_value(pred) - if pred_value is not None: - # Use static_cond if pred has a constant value. - return static_cond(pred_value, fn1, fn2) - else: - # Use dynamic cond otherwise. - return control_flow_ops.cond(pred, fn1, fn2, name) - - -def get_variable_collections(variables_collections, name): - if isinstance(variables_collections, dict): - variable_collections = variables_collections.get(name, None) - else: - variable_collections = variables_collections - return variable_collections - - -def _get_dimension(shape, dim, min_rank=1): - """Returns the `dim` dimension of `shape`, while checking it has `min_rank`. - - Args: - shape: A `TensorShape`. - dim: Integer, which dimension to return. - min_rank: Integer, minimum rank of shape. - - Returns: - The value of the `dim` dimension. - - Raises: - ValueError: if inputs don't have at least min_rank dimensions, or if the - first dimension value is not defined. - """ - dims = shape.dims - if dims is None: - raise ValueError('dims of shape must be known but is None') - if len(dims) < min_rank: - raise ValueError('rank of shape must be at least %d not: %d' % (min_rank, - len(dims))) - value = dims[dim].value - if value is None: - raise ValueError( - 'dimension %d of shape must be known but is None: %s' % (dim, shape)) - return value - - -def channel_dimension(shape, data_format, min_rank=1): - """Returns the channel dimension of shape, while checking it has min_rank. - - Args: - shape: A `TensorShape`. - data_format: `channels_first` or `channels_last`. - min_rank: Integer, minimum rank of shape. - - Returns: - The value of the first dimension. - - Raises: - ValueError: if inputs don't have at least min_rank dimensions, or if the - first dimension value is not defined. - """ - return _get_dimension(shape, 1 if data_format == 'channels_first' else -1, - min_rank=min_rank) - - -def last_dimension(shape, min_rank=1): - """Returns the last dimension of shape while checking it has min_rank. - - Args: - shape: A `TensorShape`. - min_rank: Integer, minimum rank of shape. - - Returns: - The value of the last dimension. - - Raises: - ValueError: if inputs don't have at least min_rank dimensions, or if the - last dimension value is not defined. - """ - return _get_dimension(shape, -1, min_rank=min_rank) - - -def two_element_tuple(int_or_tuple): - """Converts `int_or_tuple` to height, width. - - Several of the functions that follow accept arguments as either - a tuple of 2 integers or a single integer. A single integer - indicates that the 2 values of the tuple are the same. - - This functions normalizes the input value by always returning a tuple. - - Args: - int_or_tuple: A list of 2 ints, a single int or a `TensorShape`. - - Returns: - A tuple with 2 values. - - Raises: - ValueError: If `int_or_tuple` it not well formed. - """ - if isinstance(int_or_tuple, (list, tuple)): - if len(int_or_tuple) != 2: - raise ValueError('Must be a list with 2 elements: %s' % int_or_tuple) - return int(int_or_tuple[0]), int(int_or_tuple[1]) - if isinstance(int_or_tuple, int): - return int(int_or_tuple), int(int_or_tuple) - if isinstance(int_or_tuple, tensor_shape.TensorShape): - if len(int_or_tuple) == 2: - return int_or_tuple[0], int_or_tuple[1] - raise ValueError('Must be an int, a list with 2 elements or a TensorShape of ' - 'length 2') - - -def n_positive_integers(n, value): - """Converts `value` to a sequence of `n` positive integers. - - `value` may be either be a sequence of values convertible to `int`, or a - single value convertible to `int`, in which case the resulting integer is - duplicated `n` times. It may also be a TensorShape of rank `n`. - - Args: - n: Length of sequence to return. - value: Either a single value convertible to a positive `int` or an - `n`-element sequence of values convertible to a positive `int`. - - Returns: - A tuple of `n` positive integers. - - Raises: - TypeError: If `n` is not convertible to an integer. - ValueError: If `n` or `value` are invalid. - """ - - n_orig = n - n = int(n) - if n < 1 or n != n_orig: - raise ValueError('n must be a positive integer') - - try: - value = int(value) - except (TypeError, ValueError): - sequence_len = len(value) - if sequence_len != n: - raise ValueError( - 'Expected sequence of %d positive integers, but received %r' % - (n, value)) - try: - values = tuple(int(x) for x in value) - except: - raise ValueError( - 'Expected sequence of %d positive integers, but received %r' % - (n, value)) - for x in values: - if x < 1: - raise ValueError('expected positive integer, but received %d' % x) - return values - - if value < 1: - raise ValueError('expected positive integer, but received %d' % value) - return (value,) * n diff --git a/tensorflow/contrib/layers/python/layers/utils_test.py b/tensorflow/contrib/layers/python/layers/utils_test.py deleted file mode 100644 index 34f63f5d862..00000000000 --- a/tensorflow/contrib/layers/python/layers/utils_test.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for regularizers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.layers.python.layers import utils -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class ConstantValueTest(test.TestCase): - - def test_value(self): - for v in [True, False, 1, 0, 1.0]: - value = utils.constant_value(v) - self.assertEqual(value, v) - - def test_constant(self): - for v in [True, False, 1, 0, 1.0]: - c = constant_op.constant(v) - value = utils.constant_value(c) - self.assertEqual(value, v) - with self.cached_session(): - self.assertEqual(c.eval(), v) - - def test_variable(self): - for v in [True, False, 1, 0, 1.0]: - with ops.Graph().as_default() as g, self.session(g) as sess: - x = variables.Variable(v) - value = utils.constant_value(x) - self.assertEqual(value, None) - sess.run(variables.global_variables_initializer()) - self.assertEqual(x.eval(), v) - - def test_placeholder(self): - for v in [True, False, 1, 0, 1.0]: - p = array_ops.placeholder(np.dtype(type(v)), []) - x = array_ops.identity(p) - value = utils.constant_value(p) - self.assertEqual(value, None) - with self.cached_session(): - self.assertEqual(x.eval(feed_dict={p: v}), v) - - -class StaticCondTest(test.TestCase): - - def test_value(self): - fn1 = lambda: 'fn1' - fn2 = lambda: 'fn2' - expected = lambda v: 'fn1' if v else 'fn2' - for v in [True, False, 1, 0]: - o = utils.static_cond(v, fn1, fn2) - self.assertEqual(o, expected(v)) - - def test_constant(self): - fn1 = lambda: constant_op.constant('fn1') - fn2 = lambda: constant_op.constant('fn2') - expected = lambda v: b'fn1' if v else b'fn2' - for v in [True, False, 1, 0]: - o = utils.static_cond(v, fn1, fn2) - with self.cached_session(): - self.assertEqual(o.eval(), expected(v)) - - def test_variable(self): - fn1 = lambda: variables.Variable('fn1') - fn2 = lambda: variables.Variable('fn2') - expected = lambda v: b'fn1' if v else b'fn2' - for v in [True, False, 1, 0]: - o = utils.static_cond(v, fn1, fn2) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertEqual(o.eval(), expected(v)) - - def test_tensors(self): - fn1 = lambda: constant_op.constant(0) - constant_op.constant(1) - fn2 = lambda: constant_op.constant(0) - constant_op.constant(2) - expected = lambda v: -1 if v else -2 - for v in [True, False, 1, 0]: - o = utils.static_cond(v, fn1, fn2) - with self.cached_session(): - self.assertEqual(o.eval(), expected(v)) - - -class SmartCondStaticTest(test.TestCase): - - def test_value(self): - fn1 = lambda: 'fn1' - fn2 = lambda: 'fn2' - expected = lambda v: 'fn1' if v else 'fn2' - for v in [True, False, 1, 0]: - o = utils.smart_cond(constant_op.constant(v), fn1, fn2) - self.assertEqual(o, expected(v)) - - def test_constant(self): - fn1 = lambda: constant_op.constant('fn1') - fn2 = lambda: constant_op.constant('fn2') - expected = lambda v: b'fn1' if v else b'fn2' - for v in [True, False, 1, 0]: - o = utils.smart_cond(constant_op.constant(v), fn1, fn2) - with self.cached_session(): - self.assertEqual(o.eval(), expected(v)) - - def test_variable(self): - fn1 = lambda: variables.Variable('fn1') - fn2 = lambda: variables.Variable('fn2') - expected = lambda v: b'fn1' if v else b'fn2' - for v in [True, False, 1, 0]: - o = utils.smart_cond(constant_op.constant(v), fn1, fn2) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertEqual(o.eval(), expected(v)) - - def test_tensors(self): - fn1 = lambda: constant_op.constant(0) - constant_op.constant(1) - fn2 = lambda: constant_op.constant(0) - constant_op.constant(2) - expected = lambda v: -1 if v else -2 - for v in [True, False, 1, 0]: - o = utils.smart_cond(constant_op.constant(v), fn1, fn2) - with self.cached_session(): - self.assertEqual(o.eval(), expected(v)) - - -class SmartCondDynamicTest(test.TestCase): - - def test_value(self): - fn1 = lambda: ops.convert_to_tensor('fn1') - fn2 = lambda: ops.convert_to_tensor('fn2') - expected = lambda v: b'fn1' if v else b'fn2' - p = array_ops.placeholder(dtypes.bool, []) - for v in [True, False, 1, 0]: - o = utils.smart_cond(p, fn1, fn2) - with self.cached_session(): - self.assertEqual(o.eval(feed_dict={p: v}), expected(v)) - - def test_constant(self): - fn1 = lambda: constant_op.constant('fn1') - fn2 = lambda: constant_op.constant('fn2') - expected = lambda v: b'fn1' if v else b'fn2' - p = array_ops.placeholder(dtypes.bool, []) - for v in [True, False, 1, 0]: - o = utils.smart_cond(p, fn1, fn2) - with self.cached_session(): - self.assertEqual(o.eval(feed_dict={p: v}), expected(v)) - - def test_variable(self): - fn1 = lambda: variables.Variable('fn1') - fn2 = lambda: variables.Variable('fn2') - expected = lambda v: b'fn1' if v else b'fn2' - p = array_ops.placeholder(dtypes.bool, []) - for v in [True, False, 1, 0]: - o = utils.smart_cond(p, fn1, fn2) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertEqual(o.eval(feed_dict={p: v}), expected(v)) - - def test_tensors(self): - fn1 = lambda: constant_op.constant(0) - constant_op.constant(1) - fn2 = lambda: constant_op.constant(0) - constant_op.constant(2) - expected = lambda v: -1 if v else -2 - p = array_ops.placeholder(dtypes.bool, []) - for v in [True, False, 1, 0]: - o = utils.smart_cond(p, fn1, fn2) - with self.cached_session(): - self.assertEqual(o.eval(feed_dict={p: v}), expected(v)) - - -class CollectNamedOutputsTest(test.TestCase): - - def test_collect(self): - t1 = constant_op.constant(1.0, name='t1') - t2 = constant_op.constant(2.0, name='t2') - utils.collect_named_outputs('end_points', 'a1', t1) - utils.collect_named_outputs('end_points', 'a2', t2) - self.assertEqual(ops.get_collection('end_points'), [t1, t2]) - - def test_aliases(self): - t1 = constant_op.constant(1.0, name='t1') - t2 = constant_op.constant(2.0, name='t2') - utils.collect_named_outputs('end_points', 'a1', t1) - utils.collect_named_outputs('end_points', 'a2', t2) - self.assertEqual(t1.aliases, ['a1']) - self.assertEqual(t2.aliases, ['a2']) - - def test_multiple_aliases(self): - t1 = constant_op.constant(1.0, name='t1') - t2 = constant_op.constant(2.0, name='t2') - utils.collect_named_outputs('end_points', 'a11', t1) - utils.collect_named_outputs('end_points', 'a12', t1) - utils.collect_named_outputs('end_points', 'a21', t2) - utils.collect_named_outputs('end_points', 'a22', t2) - self.assertEqual(t1.aliases, ['a11', 'a12']) - self.assertEqual(t2.aliases, ['a21', 'a22']) - - def test_gather_aliases(self): - t1 = constant_op.constant(1.0, name='t1') - t2 = constant_op.constant(2.0, name='t2') - t3 = constant_op.constant(2.0, name='t3') - utils.collect_named_outputs('end_points', 'a1', t1) - utils.collect_named_outputs('end_points', 'a2', t2) - ops.add_to_collection('end_points', t3) - aliases = utils.gather_tensors_aliases(ops.get_collection('end_points')) - self.assertEqual(aliases, ['a1', 'a2', 't3']) - - def test_convert_collection_to_dict(self): - t1 = constant_op.constant(1.0, name='t1') - t2 = constant_op.constant(2.0, name='t2') - utils.collect_named_outputs('end_points', 'a1', t1) - utils.collect_named_outputs('end_points', 'a21', t2) - utils.collect_named_outputs('end_points', 'a22', t2) - end_points = utils.convert_collection_to_dict('end_points') - self.assertEqual(end_points['a1'], t1) - self.assertEqual(end_points['a21'], t2) - self.assertEqual(end_points['a22'], t2) - - def test_convert_collection_to_dict_clear_collection(self): - t1 = constant_op.constant(1.0, name='t1') - t2 = constant_op.constant(2.0, name='t2') - utils.collect_named_outputs('end_points', 'a1', t1) - utils.collect_named_outputs('end_points', 'a21', t2) - utils.collect_named_outputs('end_points', 'a22', t2) - utils.convert_collection_to_dict('end_points', clear_collection=True) - self.assertEqual(ops.get_collection('end_points'), []) - - -class NPositiveIntegersTest(test.TestCase): - - def test_invalid_input(self): - with self.assertRaises(ValueError): - utils.n_positive_integers('3', [1]) - - with self.assertRaises(ValueError): - utils.n_positive_integers(3.3, [1]) - - with self.assertRaises(ValueError): - utils.n_positive_integers(-1, [1]) - - with self.assertRaises(ValueError): - utils.n_positive_integers(0, [1]) - - with self.assertRaises(ValueError): - utils.n_positive_integers(1, [1, 2]) - - with self.assertRaises(ValueError): - utils.n_positive_integers(1, [-1]) - - with self.assertRaises(ValueError): - utils.n_positive_integers(1, [0]) - - with self.assertRaises(ValueError): - utils.n_positive_integers(1, [0]) - - with self.assertRaises(ValueError): - utils.n_positive_integers(2, [1]) - - with self.assertRaises(ValueError): - utils.n_positive_integers(2, [1, 2, 3]) - - with self.assertRaises(ValueError): - utils.n_positive_integers(2, ['hello', 2]) - - with self.assertRaises(ValueError): - utils.n_positive_integers(2, tensor_shape.TensorShape([2, 3, 1])) - - with self.assertRaises(ValueError): - utils.n_positive_integers(3, tensor_shape.TensorShape([2, None, 1])) - - with self.assertRaises(ValueError): - utils.n_positive_integers(3, tensor_shape.TensorShape(None)) - - def test_valid_input(self): - self.assertEqual(utils.n_positive_integers(1, 2), (2,)) - self.assertEqual(utils.n_positive_integers(2, 2), (2, 2)) - self.assertEqual(utils.n_positive_integers(2, (2, 3)), (2, 3)) - self.assertEqual(utils.n_positive_integers(3, (2, 3, 1)), (2, 3, 1)) - self.assertEqual( - utils.n_positive_integers(3, tensor_shape.TensorShape([2, 3, 1])), - (2, 3, 1)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/layers/python/ops/bucketization_op.py b/tensorflow/contrib/layers/python/ops/bucketization_op.py deleted file mode 100644 index f498352855f..00000000000 --- a/tensorflow/contrib/layers/python/ops/bucketization_op.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Wrappers for bucketization operations.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import math_ops - - -def bucketize(input_tensor, boundaries, name=None): - """Bucketizes input_tensor by given boundaries. - - See bucketize_op.cc for more details. - - Args: - input_tensor: A `Tensor` which will be bucketize. - boundaries: A list of floats gives the boundaries. It has to be sorted. - name: A name prefix for the returned tensors (optional). - - Returns: - A `Tensor` with type int32 which indicates the corresponding bucket for - each value in `input_tensor`. - - Raises: - TypeError: If boundaries is not a list. - """ - return math_ops._bucketize( # pylint: disable=protected-access - input_tensor, boundaries=boundaries, name=name) diff --git a/tensorflow/contrib/layers/python/ops/sparse_feature_cross_op.py b/tensorflow/contrib/layers/python/ops/sparse_feature_cross_op.py deleted file mode 100644 index 934a7f06069..00000000000 --- a/tensorflow/contrib/layers/python/ops/sparse_feature_cross_op.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Wrappers for sparse cross operations.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework import deprecated_arg_values -from tensorflow.contrib.layers.ops import gen_sparse_feature_cross_op -from tensorflow.contrib.util import loader -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import resource_loader - -_sparse_feature_cross_op = loader.load_op_library( - resource_loader.get_path_to_datafile("_sparse_feature_cross_op.so")) - -# Default hash key for the FingerprintCat64. -SPARSE_FEATURE_CROSS_DEFAULT_HASH_KEY = 0xDECAFCAFFE - - -@deprecated_arg_values( - "2016-11-20", - "The default behavior of sparse_feature_cross is changing, the default\n" - "value for hash_key will change to SPARSE_FEATURE_CROSS_DEFAULT_HASH_KEY.\n" - "From that point on sparse_feature_cross will always use FingerprintCat64\n" - "to concatenate the feature fingerprints. And the underlying\n" - "_sparse_feature_cross_op.sparse_feature_cross operation will be marked\n" - "as deprecated.", - hash_key=None) -def sparse_feature_cross(inputs, hashed_output=False, num_buckets=0, - name=None, hash_key=None): - """Crosses a list of Tensor or SparseTensor objects. - - See sparse_feature_cross_kernel.cc for more details. - - Args: - inputs: List of `SparseTensor` or `Tensor` to be crossed. - hashed_output: If true, returns the hash of the cross instead of the string. - This will allow us avoiding string manipulations. - num_buckets: It is used if hashed_output is true. - output = hashed_value%num_buckets if num_buckets > 0 else hashed_value. - name: A name prefix for the returned tensors (optional). - hash_key: Specify the hash_key that will be used by the `FingerprintCat64` - function to combine the crosses fingerprints on SparseFeatureCrossOp. - The default value is None, but will become - SPARSE_FEATURE_CROSS_DEFAULT_HASH_KEY after 2016-11-20 (optional). - - Returns: - A `SparseTensor` with the crossed features. - Return type is string if hashed_output=False, int64 otherwise. - - Raises: - TypeError: If the inputs aren't either SparseTensor or Tensor. - """ - if not isinstance(inputs, list): - raise TypeError("Inputs must be a list") - if not all(isinstance(i, sparse_tensor.SparseTensor) or - isinstance(i, ops.Tensor) for i in inputs): - raise TypeError("All inputs must be SparseTensors") - - sparse_inputs = [i for i in inputs - if isinstance(i, sparse_tensor.SparseTensor)] - dense_inputs = [i for i in inputs - if not isinstance(i, sparse_tensor.SparseTensor)] - - indices = [sp_input.indices for sp_input in sparse_inputs] - values = [sp_input.values for sp_input in sparse_inputs] - shapes = [sp_input.dense_shape for sp_input in sparse_inputs] - out_type = dtypes.int64 if hashed_output else dtypes.string - - internal_type = dtypes.string - for i in range(len(values)): - if values[i].dtype != dtypes.string: - values[i] = math_ops.cast(values[i], dtypes.int64) - internal_type = dtypes.int64 - for i in range(len(dense_inputs)): - if dense_inputs[i].dtype != dtypes.string: - dense_inputs[i] = math_ops.cast(dense_inputs[i], dtypes.int64) - internal_type = dtypes.int64 - - if hash_key: - indices_out, values_out, shape_out = ( - gen_sparse_feature_cross_op.sparse_feature_cross_v2( - indices, - values, - shapes, - dense_inputs, - hashed_output, - num_buckets, - hash_key=hash_key, - out_type=out_type, - internal_type=internal_type, - name=name)) - else: - indices_out, values_out, shape_out = ( - gen_sparse_feature_cross_op.sparse_feature_cross( - indices, - values, - shapes, - dense_inputs, - hashed_output, - num_buckets, - out_type=out_type, - internal_type=internal_type, - name=name)) - - return sparse_tensor.SparseTensor(indices_out, values_out, shape_out) - - -ops.NotDifferentiable("SparseFeatureCross") - - -ops.NotDifferentiable("SparseFeatureCrossV2") diff --git a/tensorflow/contrib/layers/python/ops/sparse_ops.py b/tensorflow/contrib/layers/python/ops/sparse_ops.py deleted file mode 100644 index 7e79630c5e0..00000000000 --- a/tensorflow/contrib/layers/python/ops/sparse_ops.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops to work with `SparseTensor`.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.util import compat - - -def _multiplier_helper(shape): - """Returns moving offset for each dimension given shape.""" - multipliers = [] - for dim in reversed(shape): - if multipliers: - multipliers.append(dim * multipliers[-1]) - else: - multipliers.append(dim) - multipliers.reverse() - return multipliers - - -def _ignore_value_tensor(dtype, ignore_value=None): - """Create `Tensor` from provided `ignore_value` and `dtype`.""" - if ignore_value is None: - if dtype == dtypes.string: - # Exception due to TF strings are converted to numpy objects by default. - ignore_value = "" - else: - # NOTE: `as_numpy_dtype` is a property, so with the parentheses this is - # constructing a new numpy object of the given type, which yields the - # default value for that type. - ignore_value = dtype.as_numpy_dtype() - return math_ops.cast(ignore_value, dtype, name="ignore_value") - - -def dense_to_sparse_tensor(dense_tensor, ignore_value=None): - """Converts dense `Tensor` to `SparseTensor`, dropping `ignore_value` cells. - - Args: - dense_tensor: A `Tensor`. - ignore_value: Entries in `dense_tensor` equal to this value will be - absent from the return `SparseTensor`. If `None`, default value of - `dense_tensor` dtype will be used (e.g. '' for `str`, 0 for `int`). - - Returns: - A `SparseTensor` with the same shape as `dense_tensor`. - - Raises: - ValueError: when `dense_tensor`'s rank is `None`. - """ - with ops.name_scope("DenseToSparseTensor"): - dense_tensor = ops.convert_to_tensor(dense_tensor) - ignore_value = _ignore_value_tensor(dense_tensor.dtype, ignore_value) - indices = array_ops.where( - math_ops.not_equal(dense_tensor, ignore_value), name="indices") - return sparse_tensor.SparseTensor( - indices=indices, - values=array_ops.gather_nd(dense_tensor, indices, name="values"), - dense_shape=array_ops.shape( - dense_tensor, out_type=dtypes.int64, name="dense_shape")) - - -def indicators_to_sparse_ids(indicators, ignore_value=None, dtype=dtypes.int64): - """Convert a dense indicator tensor to sparse IDs. - - This is commonly used for converting a dense classification label to sparse. - In the following example, we have an input of shape (2, 2, num_classes), - where num_classes=4. - - ```python - indicators = [ - [ - [0, 0, 1, 0], - [0, 0, 0, 0] - ], [ - [1, 0, 1, 1], - [0, 0, 1, 0] - ] - ] - sparse_ids = indicator_to_sparse_ids(indicators) - ``` - - `sparse_ids` in "jagged" format: - [ - [ - [2], - [] - ], [ - [0, 2, 3], - [2] - ] - ] - - `sparse_ids` in `SparseTensor` format: - ```python - { - indices: [[0, 0, 1], [1, 0, 0], [1, 0, 1], [1, 0, 2], [1, 1, 0]], - values: [2, 0, 2, 3, 2], - dense_shape: [2, 2, 3] - } - ``` - - Args: - indicators: Dense `Tensor` of shape `(d0, ..., dn, num_classes)`. - `ignore_value` values are ignored. For other values (typically, ones), the - index along the last dimension is returned. - ignore_value: Entries in `indicators` equal to this value will be - absent from the returned `SparseTensor`. If `None`, default value of - `indicators` dtype will be used (e.g. '' for `str`, 0 for `int`). - dtype: Type of result, must be integer type. - - Returns: - `SparseTensor` of type `dtype` and shape `(d0, ..., dn, max_num_labels)`, - where `max_num_labels` is the maximum number of non-zero values in any - row (in the example above, row (1, 1) has 3 non-zero values, so the result - shape is (2, 2, 3)). The values of this `SparseTensor` are in the range - `[0, num_classes)` and correspond to the index of non-ignore values along - the last dimension of `indicators`. - - Raises: - ValueError: if `dtype` is not integer. - """ - if not dtype.is_integer: - raise ValueError("Invalid dtype {} not integer.".format(dtype)) - with ops.name_scope( - None, "indicators_to_sparse_ids", (indicators, ignore_value)): - # Convert indicators to binary ones and zeros. We use int64 since - # SparseTensor requires int64 indices. - indicators = ops.convert_to_tensor(indicators, name="indicators") - missing_indicators = math_ops.equal( - indicators, _ignore_value_tensor(indicators.dtype, ignore_value), - name="missing") - zeros_like_indicators = array_ops.zeros_like( - indicators, dtype=dtypes.int64, name="zeros") - binary_indicators = array_ops.where( - missing_indicators, zeros_like_indicators, - array_ops.ones_like(indicators, dtype=dtypes.int64, name="ones"), - name="binary_indicators") - - # Use cumsum along the last dimension to generate per-row indexes. - # Note that these are 1-based (since 0 indicates missing values), so they're - # off-by-1 from the actual indices. We'll subtract 1 below. Since they're - # off-by-one, the max value is the size of the last dimension (i.e., - # last_index + 1). - row_index_indicators = array_ops.where( - missing_indicators, zeros_like_indicators, - math_ops.cumsum(binary_indicators, axis=-1), "row_index_indicators") - result_last_dim = array_ops.reshape( - math_ops.reduce_max(row_index_indicators), shape=(1,), - name="result_last_dim") - - # Convert to a SparseTensor. The values of this SparseTensor are the last - # indices of our result, and the last indices of this SparseTensor (i.e., - # the class IDs indicated by `indicators`) are the values of our result, so - # we use tensor slicing and concat to swap them. - sparse_row_index_indicators = dense_to_sparse_tensor( - row_index_indicators, ignore_value=0) - return sparse_tensor.SparseTensor( - indices=array_ops.concat(( - sparse_row_index_indicators.indices[:, :-1], - array_ops.reshape(sparse_row_index_indicators.values - 1, (-1, 1)) - ), axis=1, name="indices"), - values=math_ops.cast( - sparse_row_index_indicators.indices[:, -1], dtype=dtype, - name="values"), - dense_shape=array_ops.concat( - (sparse_row_index_indicators.dense_shape[0:-1], result_last_dim), - axis=0, name="dense_shape")) - - -def sparse_row_envelope(sparse_input, row_axis=0, col_axis=1, name=None): - """Returns the length of each 'row' in a `SparseTensor`. - - For example, if `sparse_input` has indices `[[0,0], [2, 0], [2, 1], [2, 2]]` - and shape `[3, 3]`, this function will return `[1, 0, 3]`. - - Args: - sparse_input: a `SparseTensor` of rank at least 2. - row_axis: An integer. The axis for the row of the envelope matrix. Default - is 0. - col_axis: An integer. The axis for the col of the envelope matrix. Default - is 1. - name: A name for the operation (optional). - - Returns: - A one-dimensional `Tensor` whose entries correspond to the length of each - row of `SparseTensor`. - - Raises: - ValueError: If row_axis and col_axis are the same axis or they are not - integers. - """ - if not (isinstance(row_axis, compat.integral_types) and - isinstance(col_axis, compat.integral_types)): - raise ValueError("`row_axis` and `col_axis` must be integers.") - - if row_axis == col_axis: - raise ValueError("Row and column can not be the same axis.") - - with ops.name_scope(name, "sparse_row_envelope", [sparse_input]): - indices = sparse_input.indices - row_indices = indices[:, row_axis] - col_indices = indices[:, col_axis] - num_rows = math_ops.cast(sparse_input.dense_shape[row_axis], dtypes.int32) - row_envelope = math_ops.unsorted_segment_max( - col_indices + 1, row_indices, num_rows, name=name) - zeros = array_ops.zeros_like(row_envelope) - return array_ops.where(row_envelope > zeros, row_envelope, zeros) diff --git a/tensorflow/contrib/layers/python/ops/sparse_ops_test.py b/tensorflow/contrib/layers/python/ops/sparse_ops_test.py deleted file mode 100644 index b6c2cab64ac..00000000000 --- a/tensorflow/contrib/layers/python/ops/sparse_ops_test.py +++ /dev/null @@ -1,337 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.contrib.layers.python.ops.sparse_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.layers.python.ops import sparse_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -def _assert_sparse_tensor_value(test_case, expected, actual): - test_case.assertEqual(np.int64, np.array(actual.indices).dtype) - test_case.assertAllEqual(expected.indices, actual.indices) - - test_case.assertEqual( - np.array(expected.values).dtype, np.array(actual.values).dtype) - test_case.assertAllEqual(expected.values, actual.values) - - test_case.assertEqual(np.int64, np.array(actual.dense_shape).dtype) - test_case.assertAllEqual(expected.dense_shape, actual.dense_shape) - - -class DenseToSparseTensorTest(test.TestCase): - - def test_dense_to_sparse_tensor_1d(self): - with self.cached_session() as sess: - st = sparse_ops.dense_to_sparse_tensor([1, 0, 2, 0]) - result = sess.run(st) - self.assertEqual(result.indices.dtype, np.int64) - self.assertEqual(result.values.dtype, np.int32) - self.assertEqual(result.dense_shape.dtype, np.int64) - self.assertAllEqual([[0], [2]], result.indices) - self.assertAllEqual([1, 2], result.values) - self.assertAllEqual([4], result.dense_shape) - - def test_dense_to_sparse_tensor_1d_float(self): - with self.cached_session() as sess: - st = sparse_ops.dense_to_sparse_tensor([1.5, 0.0, 2.3, 0.0]) - result = sess.run(st) - self.assertEqual(result.indices.dtype, np.int64) - self.assertEqual(result.values.dtype, np.float32) - self.assertEqual(result.dense_shape.dtype, np.int64) - self.assertAllEqual([[0], [2]], result.indices) - self.assertAllClose([1.5, 2.3], result.values) - self.assertAllEqual([4], result.dense_shape) - - def test_dense_to_sparse_tensor_1d_bool(self): - with self.cached_session() as sess: - st = sparse_ops.dense_to_sparse_tensor([True, False, True, False]) - result = sess.run(st) - self.assertEqual(result.indices.dtype, np.int64) - self.assertEqual(result.values.dtype, np.bool) - self.assertEqual(result.dense_shape.dtype, np.int64) - self.assertAllEqual([[0], [2]], result.indices) - self.assertAllEqual([True, True], result.values) - self.assertAllEqual([4], result.dense_shape) - - def test_dense_to_sparse_tensor_1d_str(self): - with self.cached_session() as sess: - st = sparse_ops.dense_to_sparse_tensor([b'qwe', b'', b'ewq', b'']) - result = sess.run(st) - self.assertEqual(result.indices.dtype, np.int64) - self.assertEqual(result.values.dtype, np.object) - self.assertEqual(result.dense_shape.dtype, np.int64) - self.assertAllEqual([[0], [2]], result.indices) - self.assertAllEqual([b'qwe', b'ewq'], result.values) - self.assertAllEqual([4], result.dense_shape) - - def test_dense_to_sparse_tensor_1d_str_special_ignore(self): - with self.cached_session() as sess: - st = sparse_ops.dense_to_sparse_tensor( - [b'qwe', b'', b'ewq', b''], ignore_value=b'qwe') - result = sess.run(st) - self.assertEqual(result.indices.dtype, np.int64) - self.assertEqual(result.values.dtype, np.object) - self.assertEqual(result.dense_shape.dtype, np.int64) - self.assertAllEqual([[1], [2], [3]], result.indices) - self.assertAllEqual([b'', b'ewq', b''], result.values) - self.assertAllEqual([4], result.dense_shape) - - def test_dense_to_sparse_tensor_2d(self): - with self.cached_session() as sess: - st = sparse_ops.dense_to_sparse_tensor([[1, 2, 0, 0], [3, 4, 5, 0]]) - result = sess.run(st) - self.assertAllEqual([[0, 0], [0, 1], [1, 0], [1, 1], [1, 2]], - result.indices) - self.assertAllEqual([1, 2, 3, 4, 5], result.values) - self.assertAllEqual([2, 4], result.dense_shape) - - def test_dense_to_sparse_tensor_3d(self): - with self.cached_session() as sess: - st = sparse_ops.dense_to_sparse_tensor([[[1, 2, 0, 0], [3, 4, 5, 0]], - [[7, 8, 0, 0], [9, 0, 0, 0]]]) - result = sess.run(st) - self.assertAllEqual([[0, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 1], [0, 1, 2], - [1, 0, 0], [1, 0, 1], [1, 1, 0]], result.indices) - self.assertAllEqual([1, 2, 3, 4, 5, 7, 8, 9], result.values) - self.assertAllEqual([2, 2, 4], result.dense_shape) - - def test_dense_to_sparse_tensor_unknown_1d_shape(self): - with self.cached_session() as sess: - tensor = array_ops.placeholder(shape=[None], dtype=dtypes.int32) - st = sparse_ops.dense_to_sparse_tensor(tensor) - result = sess.run(st, feed_dict={tensor: [0, 100, 0, 3]}) - self.assertAllEqual([[1], [3]], result.indices) - self.assertAllEqual([100, 3], result.values) - self.assertAllEqual([4], result.dense_shape) - - def test_dense_to_sparse_tensor_unknown_3d_shape(self): - with self.cached_session() as sess: - tensor = array_ops.placeholder( - shape=[None, None, None], dtype=dtypes.int32) - st = sparse_ops.dense_to_sparse_tensor(tensor) - result = sess.run(st, - feed_dict={ - tensor: [[[1, 2, 0, 0], [3, 4, 5, 0]], - [[7, 8, 0, 0], [9, 0, 0, 0]]] - }) - self.assertAllEqual([[0, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 1], [0, 1, 2], - [1, 0, 0], [1, 0, 1], [1, 1, 0]], result.indices) - self.assertAllEqual([1, 2, 3, 4, 5, 7, 8, 9], result.values) - self.assertAllEqual([2, 2, 4], result.dense_shape) - - def test_dense_to_sparse_unknown_rank(self): - ph = array_ops.placeholder(dtype=dtypes.int32) - with self.cached_session() as sess: - st = sparse_ops.dense_to_sparse_tensor(ph) - result = sess.run(st, feed_dict={ph: [[1, 2, 0, 0], [3, 4, 5, 0]]}) - self.assertAllEqual([[0, 0], [0, 1], [1, 0], [1, 1], [1, 2]], - result.indices) - self.assertAllEqual([1, 2, 3, 4, 5], result.values) - self.assertAllEqual([2, 4], result.dense_shape) - - -class SparseRowEnvelopeTest(test.TestCase): - - def test_sparse_row_envelope(self): - expected_sparse_row_envelope = [1, 0, 3] - with self.cached_session() as sess: - sparse_input = sparse_tensor.SparseTensor( - indices=[[0, 0], [2, 0], [2, 1], [2, 2]], - values=[0, 1, 2, 3], - dense_shape=[3, 3]) - sparse_row_envelope = sess.run( - sparse_ops.sparse_row_envelope(sparse_input)) - self.assertAllEqual(expected_sparse_row_envelope, - sparse_row_envelope) - - def test_sparse_row_envelope_unsorted_indices(self): - expected_sparse_row_envelope = [1, 0, 3] - with self.cached_session() as sess: - sparse_input = sparse_tensor.SparseTensor( - indices=[[2, 0], [2, 2], [2, 1], [0, 0]], - values=[0, 1, 2, 3], - dense_shape=[3, 3]) - sparse_row_envelope = sess.run( - sparse_ops.sparse_row_envelope(sparse_input)) - self.assertAllEqual(expected_sparse_row_envelope, - sparse_row_envelope) - - def test_sparse_row_envelope_empty_in_the_end(self): - expected_sparse_row_envelope = [1, 0, 3, 0, 0] - with self.cached_session() as sess: - sparse_input = sparse_tensor.SparseTensor( - indices=[[0, 0], [2, 0], [2, 1], [2, 2]], - values=[0, 1, 2, 3], - dense_shape=[5, 3]) - sparse_row_envelope = sess.run( - sparse_ops.sparse_row_envelope(sparse_input)) - self.assertAllEqual(expected_sparse_row_envelope, - sparse_row_envelope) - - def test_sparse_row_envelope_empty_3d(self): - expected_sparse_row_envelope = [1, 0, 3, 0, 0] - with self.cached_session() as sess: - sparse_input = sparse_tensor.SparseTensor( - indices=[[0, 0, 0], [0, 2, 0], [0, 2, 1], [0, 2, 2]], - values=[0, 1, 2, 3], - dense_shape=[1, 5, 3]) - sparse_row_envelope = sess.run( - sparse_ops.sparse_row_envelope(sparse_input, 1, 2)) - self.assertAllEqual(expected_sparse_row_envelope, - sparse_row_envelope) - - -class IndicatorToSparseIdsTest(test.TestCase): - - def test_indicators_to_sparse_ids_1d(self): - indicators = (0, 0, 1, 0) - sparse_ids = sparse_ops.indicators_to_sparse_ids(indicators) - with self.cached_session(): - _assert_sparse_tensor_value(self, sparse_tensor.SparseTensorValue( - indices=((0,),), - values=(2,), - dense_shape=(1,), - ), sparse_ids.eval()) - - def test_indicators_to_sparse_ids_2d(self): - indicators = ( - (0, 0, 1, 0), - (1, 0, 0, 1), - ) - sparse_ids = sparse_ops.indicators_to_sparse_ids(indicators) - with self.cached_session(): - _assert_sparse_tensor_value(self, sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=(2, 0, 3), - dense_shape=(2, 2), - ), sparse_ids.eval()) - - def test_indicators_to_sparse_ids_3d(self): - indicators = ( - ((0, 0, 1, 0, 0), (0, 0, 0, 0, 0)), - ((1, 0, 0, 1, 0), (0, 0, 1, 0, 0)), - ((0, 0, 0, 0, 0), (0, 0, 0, 0, 0)), - ((1, 0, 0, 1, 1), (0, 0, 1, 0, 0)), - ) - sparse_ids = sparse_ops.indicators_to_sparse_ids(indicators) - with self.cached_session(): - _assert_sparse_tensor_value(self, sparse_tensor.SparseTensorValue( - indices=( - (0, 0, 0), - (1, 0, 0), (1, 0, 1), (1, 1, 0), - (3, 0, 0), (3, 0, 1), (3, 0, 2), (3, 1, 0) - ), values=( - 2, - 0, 3, 2, - 0, 3, 4, 2 - ), dense_shape=(4, 2, 3), - ), sparse_ids.eval()) - - def test_int16_to_sparse_ids_2d(self): - indicators = ( - (0, 0, 1, 0), - (1, 0, 0, 1), - ) - sparse_ids = sparse_ops.indicators_to_sparse_ids( - indicators, dtype=dtypes.int16) - with self.cached_session(): - _assert_sparse_tensor_value(self, sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=np.array((2, 0, 3), dtype=np.int16), - dense_shape=(2, 2), - ), sparse_ids.eval()) - - def test_indicators_to_sparse_ids_ignore_value(self): - indicators = ( - ((-1, -1, 10, -1), (-1, -1, -1, -1)), - ((11, -1, -1, 12), (-1, -1, 13, -1)), - ) - sparse_ids = sparse_ops.indicators_to_sparse_ids( - indicators, ignore_value=-1) - with self.cached_session(): - _assert_sparse_tensor_value(self, sparse_tensor.SparseTensorValue( - indices=((0, 0, 0), (1, 0, 0), (1, 0, 1), (1, 1, 0)), - values=(2, 0, 3, 2), - dense_shape=(2, 2, 2), - ), sparse_ids.eval()) - - def test_string_indicators_to_sparse_ids(self): - indicators = ( - (('', '', 'A', ''), ('', '', '', '')), - (('B', '', '', 'C'), ('', '', 'D', '')), - ) - sparse_ids = sparse_ops.indicators_to_sparse_ids(indicators) - with self.cached_session(): - _assert_sparse_tensor_value(self, sparse_tensor.SparseTensorValue( - indices=((0, 0, 0), (1, 0, 0), (1, 0, 1), (1, 1, 0)), - values=(2, 0, 3, 2), - dense_shape=(2, 2, 2), - ), sparse_ids.eval()) - - def test_string_indicators_to_sparse_ids_ignore_value(self): - indicators = ( - (('x', 'x', 'A', 'x'), ('x', 'x', 'x', 'x')), - (('B', 'x', 'x', 'C'), ('x', 'x', 'D', 'x')), - ) - sparse_ids = sparse_ops.indicators_to_sparse_ids( - indicators, ignore_value='x') - with self.cached_session(): - _assert_sparse_tensor_value(self, sparse_tensor.SparseTensorValue( - indices=((0, 0, 0), (1, 0, 0), (1, 0, 1), (1, 1, 0)), - values=(2, 0, 3, 2), - dense_shape=(2, 2, 2), - ), sparse_ids.eval()) - - def test_indicators_to_sparse_ids_unknown_3d_shape(self): - indicators_values = ( - ((0, 0, 1, 0), (0, 0, 0, 0)), - ((1, 0, 0, 1), (0, 0, 1, 0)), - ) - indicators = array_ops.placeholder( - dtype=dtypes.int32, shape=(None, None, None)) - sparse_ids = sparse_ops.indicators_to_sparse_ids(indicators) - with self.cached_session(): - _assert_sparse_tensor_value(self, sparse_tensor.SparseTensorValue( - indices=((0, 0, 0), (1, 0, 0), (1, 0, 1), (1, 1, 0)), - values=(2, 0, 3, 2), - dense_shape=(2, 2, 2), - ), sparse_ids.eval(feed_dict={indicators: indicators_values})) - - def test_indicators_to_sparse_ids_unknown_rank(self): - indicators_values = ( - ((0, 0, 1, 0), (0, 0, 0, 0)), - ((1, 0, 0, 1), (0, 0, 1, 0)), - ) - indicators = array_ops.placeholder(dtype=dtypes.int32) - sparse_ids = sparse_ops.indicators_to_sparse_ids(indicators) - with self.cached_session(): - _assert_sparse_tensor_value(self, sparse_tensor.SparseTensorValue( - indices=((0, 0, 0), (1, 0, 0), (1, 0, 1), (1, 1, 0)), - values=(2, 0, 3, 2), - dense_shape=(2, 2, 2), - ), sparse_ids.eval(feed_dict={indicators: indicators_values})) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD deleted file mode 100644 index e4ed2c7841a..00000000000 --- a/tensorflow/contrib/learn/BUILD +++ /dev/null @@ -1,939 +0,0 @@ -# Description: -# Contains TF Learn (aka Scikit Flow) sub-project with high level tensorflow API. - -load("//tensorflow:tensorflow.bzl", "py_test", "tf_py_test") - -package( - default_visibility = [ - "//engedu/ml/tf_from_scratch:__pkg__", - "//tensorflow:internal", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "learn", - srcs = [ - "__init__.py", - "python/__init__.py", - ] + glob( - ["python/learn/**/*.py"], - exclude = ["python/learn/**/*_test.py"], - ), - srcs_version = "PY2AND3", - visibility = [ - "//learning/brain:__subpackages__", - "//storage/d/analysis/prefetch:__pkg__", - "//tensorflow:__subpackages__", - "//video/youtube/personalization:__subpackages__", - ], - # This library should not depend on sklearn, even though some of the code - # refers to it. (The code handles the presence of sklearn conditionally.) - deps = [ - "//tensorflow/contrib/factorization:factorization_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/input_pipeline:input_pipeline_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/contrib/linear_optimizer:sdca_ops_py", - "//tensorflow/contrib/lookup:lookup_py", - "//tensorflow/contrib/losses:losses_py", - "//tensorflow/contrib/meta_graph_transform", - "//tensorflow/contrib/metrics:metrics_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/contrib/session_bundle:exporter", - "//tensorflow/contrib/session_bundle:gc", - "//tensorflow/contrib/tpu:tpu_estimator", - "//tensorflow/contrib/training:training_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:client", - "//tensorflow/python:clip_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:functional_ops", - "//tensorflow/python:gradients", - "//tensorflow/python:init_ops", - "//tensorflow/python:io_ops", - "//tensorflow/python:logging_ops", - "//tensorflow/python:lookup_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:metrics", - "//tensorflow/python:nn", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:partitioned_variables", - "//tensorflow/python:platform", - "//tensorflow/python:random_seed", - "//tensorflow/python:resources", - "//tensorflow/python:rnn", - "//tensorflow/python:session", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:state_ops", - "//tensorflow/python:string_ops", - "//tensorflow/python:summary", - "//tensorflow/python:tensor_util", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python:weights_broadcast_ops", - "//tensorflow/python/estimator:estimator_py", - "//tensorflow/python/feature_column", - "//tensorflow/python/feature_column:feature_column_py", - "//tensorflow/python/ops/losses", - "//tensorflow/python/saved_model:builder", - "//tensorflow/python/saved_model:loader", - "//tensorflow/python/saved_model:signature_constants", - "//tensorflow/python/saved_model:signature_def_utils", - "//tensorflow/python/saved_model:tag_constants", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -# Exposes constants without having to build the entire :learn target. -py_library( - name = "estimator_constants_py", - srcs = [ - "python/learn/estimators/constants.py", - "python/learn/estimators/prediction_key.py", - ], - srcs_version = "PY2AND3", -) - -py_test( - name = "data_feeder_test", - size = "small", - srcs = ["python/learn/learn_io/data_feeder_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_test( - name = "estimators_test", - size = "medium", - srcs = ["python/learn/estimators/estimators_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:string_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "metric_spec_test", - size = "small", - srcs = ["python/learn/metric_spec_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:client_testlib", - ], -) - -tf_py_test( - name = "experiment_test", - size = "medium", - srcs = ["python/learn/experiment_test.py"], - additional_deps = [ - ":learn", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:platform", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variables", - "//tensorflow/python/estimator:estimator_py", - ], -) - -py_test( - name = "export_strategy_test", - size = "small", - srcs = ["python/learn/export_strategy_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:client_testlib", - ], -) - -py_test( - name = "graph_actions_test", - size = "small", - srcs = ["python/learn/graph_actions_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_windows"], # TODO: needs investigation on Windows - deps = [ - ":learn", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/testing:testing_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:extra_py_tests_deps", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:resources", - "//tensorflow/python:state_ops", - "//tensorflow/python:summary", - "//tensorflow/python:test_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "learn_runner_test", - size = "small", - srcs = ["python/learn/learn_runner_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/contrib/training:training_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:platform", - "//tensorflow/python/estimator:estimator_py", - ], -) - -py_test( - name = "monitors_test", - size = "small", - srcs = ["python/learn/monitors_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_pip_gpu"], # b/74437598 - deps = [ - ":learn", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/testing:testing_py", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:session", - "//tensorflow/python:state_ops", - "//tensorflow/python:summary", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//tensorflow/python/estimator:estimator_py", - ], -) - -py_test( - name = "run_config_test", - size = "small", - srcs = ["python/learn/estimators/run_config_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:training", - "//tensorflow/python/estimator:estimator_py", - ], -) - -py_test( - name = "tensor_signature_test", - srcs = ["python/learn/estimators/tensor_signature_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "manual", # b/130760310 - ], - deps = [ - ":learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:sparse_tensor", - ], -) - -py_test( - name = "estimator_test", - size = "medium", - srcs = ["python/learn/estimators/estimator_test.py"], - python_version = "PY2", - shard_count = 2, - srcs_version = "PY2AND3", - tags = [ - "manual", - "noasan", # times out - "optonly", # test is flaky without optimization. - ], - deps = [ - ":learn", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/contrib/lookup:lookup_py", - "//tensorflow/contrib/metrics:metrics_py", - "//tensorflow/contrib/testing:testing_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:platform", - "//tensorflow/python:protos_all_py", - "//tensorflow/python:session", - "//tensorflow/python:summary", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variables", - "//tensorflow/python/saved_model:loader", - "//tensorflow/python/saved_model:tag_constants", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_test( - name = "estimator_input_test", - size = "medium", - srcs = ["python/learn/estimators/estimator_input_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/contrib/metrics:metrics_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) - -py_test( - name = "logistic_regressor_test", - size = "small", - srcs = ["python/learn/estimators/logistic_regressor_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python/ops/losses", - "//third_party/py/numpy", - ], -) - -py_test( - name = "dnn_linear_combined_test", - size = "medium", - srcs = ["python/learn/estimators/dnn_linear_combined_test.py"], - python_version = "PY2", - shard_count = 8, - srcs_version = "PY2AND3", - tags = ["no_oss"], # flaky b/70524820 - deps = [ - ":learn", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/contrib/metrics:metrics_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:training", - "//tensorflow/python/feature_column", - "//tensorflow/python/ops/losses", - "//third_party/py/numpy", - ], -) - -py_test( - name = "head_test", - size = "medium", - srcs = ["python/learn/estimators/head_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - tags = ["noasan"], # times out b/63678675 - deps = [":head_test_lib"], -) - -py_library( - name = "head_test_lib", - srcs = ["python/learn/estimators/head_test.py"], - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:lookup_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:session", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:variables", - "//tensorflow/python/ops/losses", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_test( - name = "dnn_test", - size = "medium", - srcs = ["python/learn/estimators/dnn_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - tags = ["notap"], - deps = [ - ":learn", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/contrib/metrics:metrics_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:training", - "//tensorflow/python/feature_column", - "//third_party/py/numpy", - ], -) - -py_test( - name = "kmeans_test", - size = "medium", - srcs = ["python/learn/estimators/kmeans_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - tags = [ - "noasan", # b/73741358 - "nomac", - ], - deps = [ - ":learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_benchmark", - "//tensorflow/python:random_ops", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) - -py_test( - name = "dynamic_rnn_estimator_test", - size = "medium", - srcs = ["python/learn/estimators/dynamic_rnn_estimator_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:functional_ops", - "//tensorflow/python:lookup_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:rnn_cell", - "//tensorflow/python:session", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "state_saving_rnn_estimator_test", - size = "medium", - srcs = ["python/learn/estimators/state_saving_rnn_estimator_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - tags = ["noasan"], - deps = [ - ":learn", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/lookup:lookup_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:lookup_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "linear_test", - size = "medium", - srcs = ["python/learn/estimators/linear_test.py"], - python_version = "PY2", - shard_count = 20, - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [ - ":learn", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/contrib/linear_optimizer:sdca_ops_py", - "//tensorflow/contrib/metrics:metrics_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:training", - "//tensorflow/python/feature_column", - "//third_party/py/numpy", - ], -) - -py_test( - name = "debug_test", - size = "medium", - srcs = ["python/learn/estimators/debug_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/contrib/metrics:metrics_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:math_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) - -py_test( - name = "composable_model_test", - size = "medium", - srcs = ["python/learn/estimators/composable_model_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:state_ops", - ], -) - -py_test( - name = "svm_test", - size = "medium", - srcs = ["python/learn/estimators/svm_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_oss", # b/134605801 - ], - deps = [ - ":learn", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:sparse_tensor", - ], -) - -py_test( - name = "grid_search_test", - size = "small", - srcs = ["python/learn/grid_search_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:client_testlib", - ], -) - -py_test( - name = "io_test", - size = "small", - srcs = ["python/learn/learn_io/io_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - ], -) - -py_test( - name = "model_fn_test", - size = "small", - srcs = ["python/learn/estimators/model_fn_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python/estimator:estimator_py", - "//tensorflow/python/saved_model:signature_constants", - "@six_archive//:six", - ], -) - -py_test( - name = "multioutput_test", - size = "small", - srcs = ["python/learn/estimators/multioutput_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_oss", - ], - deps = [ - ":learn", - "//tensorflow/python:client_testlib", - "//third_party/py/numpy", - ], -) - -py_test( - name = "nonlinear_test", - size = "medium", - srcs = ["python/learn/estimators/nonlinear_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/python:client_testlib", - "//tensorflow/python:random_seed", - ], -) - -py_test( - name = "regression_test", - size = "small", - srcs = ["python/learn/estimators/regression_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:client_testlib", - "//third_party/py/numpy", - ], -) - -py_test( - name = "rnn_common_test", - size = "medium", - srcs = ["python/learn/estimators/rnn_common_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:session", - "//third_party/py/numpy", - ], -) - -py_test( - name = "ops_test", - size = "small", - srcs = ["python/learn/ops/ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:random_seed", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "seq2seq_ops_test", - size = "small", - srcs = ["python/learn/ops/seq2seq_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:rnn_cell", - "//third_party/py/numpy", - ], -) - -py_test( - name = "categorical_test", - size = "small", - srcs = ["python/learn/preprocessing/tests/categorical_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:client_testlib", - "//third_party/py/numpy", - ], -) - -py_test( - name = "categorical_vocabulary_test", - size = "small", - srcs = ["python/learn/preprocessing/tests/categorical_vocabulary_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:client_testlib", - ], -) - -py_test( - name = "text_test", - size = "small", - srcs = ["python/learn/preprocessing/tests/text_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:client_testlib", - ], -) - -tf_py_test( - name = "graph_io_test", - size = "medium", - srcs = ["python/learn/learn_io/graph_io_test.py"], - additional_deps = [ - ":learn", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:io_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:platform", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], - grpc_enabled = True, -) - -py_test( - name = "pandas_io_test", - size = "small", - srcs = ["python/learn/learn_io/pandas_io_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) - -py_test( - name = "export_test", - size = "small", - timeout = "moderate", - srcs = ["python/learn/utils/export_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "manual", # http://b/31032996 - "notap", # TODO(b/37950026): Test is flaky - ], - deps = [ - ":learn", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/session_bundle:exporter", - "//tensorflow/contrib/session_bundle:manifest_proto_py_pb2", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_test( - name = "gc_test", - size = "small", - srcs = ["python/learn/utils/gc_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform", - "//tensorflow/python:util", - ], -) - -py_test( - name = "saved_model_export_utils_test", - size = "small", - srcs = ["python/learn/utils/saved_model_export_utils_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_windows"], # TODO: needs investigation on Windows - deps = [ - ":learn", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:util", - "//tensorflow/python/saved_model:signature_constants", - "//tensorflow/python/saved_model:signature_def_utils", - ], -) - -py_test( - name = "input_fn_utils_test", - size = "small", - srcs = ["python/learn/utils/input_fn_utils_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - ], -) - -py_test( - name = "stability_test", - size = "small", - srcs = ["python/learn/estimators/stability_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":learn", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:random_ops", - "//tensorflow/python:training", - ], -) - -py_binary( - name = "inspect_checkpoint", - srcs = ["python/learn/utils/inspect_checkpoint.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:platform", - ], -) diff --git a/tensorflow/contrib/learn/README.md b/tensorflow/contrib/learn/README.md deleted file mode 100644 index b2d3a6273ab..00000000000 --- a/tensorflow/contrib/learn/README.md +++ /dev/null @@ -1,141 +0,0 @@ -EVERYTHING IN THIS DIRECTORY IS DEPRECATED. - -Using functions or classes will result in warnings. - -Instructions for converting to current alternatives are included in the -warnings. A high-level overview is below. - -## Canned Estimators - -Many canned estimators (subclasses of `Estimator`) have equivalents in core -exposed under `tf.estimator`: -`DNNClassifier`, `DNNRegressor`, `DNNEstimator`, `LinearClassifier`, -`LinearRegressor`, `LinearEstimator`, `DNNLinearCombinedClassifier`, -`DNNLinearCombinedRegressor` and `DNNLinearCombinedEstimator`. - -To migrate to the new api, users need to take the following steps: - -* Replace `tf.contrib.learn` with `tf.estimator`. -* If you subclass any of the estimators, stop doing that. You should be able to - write a factory method that returns a canned estimator instead. If this is not - possible (if you override methods from the canned estimator), consider writing - a custom estimator instead. See `tf.estimator.Estimator`. -* Set `loss_reduction=tf.losses.Reduction.SUM_OVER_BATCH_SIZE` to preserve loss - reduction as the average over batch. -* Some optimizer-related arguments are no longer passed in the estimator - constructor. Instead, we provide methods that perform the same job by wrapping - an optimizer. Specifically: - * `gradient_clip_norm`: Use `tf.contrib.estimator.clip_gradients_by_norm` - * `embedding_lr_multipliers`: Not supported. - Other arguments: - * `input_layer_min_slice_size`: Replaced by `input_layer_partitioner` - * `enable_centered_bias`: Not supported. Dropping this argument is unlikely to - harm your model. - * `feature_engineering_fn`: Not supported. You can call your - `feature_engineering_fn` inside your input_fn: - ```python - def new_input_fn(): - features, labels = old_input_fn() - return feature_engineering_fn(features, labels) - ``` -* Use `tf.reshape` to reshape labels in your `input_fn`. `tf.estimator` - classifiers and regressors expect labels as a 2D Tensor of shape - `[batch_size, 1]`, or `[batch_size, n_labels]`. In contrast, - `tf.contrib.learn` classifiers and regressors supported labels with shape - `[batch_size]`. -* If you pass custom metrics from the `evaluate()` method call, use - `tf.estimator.add_metrics`. -* Replace your `serving_input_fn` with a `serving_input_receiver_fn`. - Note this should be entirely distinct from your training `input_fn`, so if you - previously had one `input_fn` with different "modes", you should now factor - that apart. Where the former returned either a simple `(features, labels)` - tuple or `InputFnOps`, you should now return a `ServingInputReceiver`. - If you were generating your `serving_input_fn` using the - `build_parsing_serving_input_fn` helper, you can simply drop in the - replacement `build_parsing_serving_input_receiver_fn`. - -Some remaining estimators/classes: - -* `DynamicRnnEstimator`: Consider a custom `model_fn`. -* `KMeansClustering`: Use `tf.contrib.factorization.KMeansClustering`. -* `LogisticRegressor`: Not supported. Instead, use `binary_classification_head` - with a custom `model_fn`, or with `DNNEstimator`. -* `StateSavingRnnEstimator`: Consider a custom `model_fn`. -* SVM: Consider a custom `model_fn`. -* `LinearComposableModel` and `DNNComposableModel`: Not supported. - Consider `tf.contrib.estimator.DNNEstimator`, or write a custom model_fn. -* `MetricSpec`: Deprecated. For adding custom metrics to canned Estimators, use - `tf.estimator.add_metrics`. - -## Estimator -`tf.contrib.learn.Estimator` is migrated to `tf.estimator.Estimator`. - -To migrate, users need to take the following steps: - -* Replace `tf.contrib.learn.Estimator` with `tf.estimator.Estimator`. -* If you pass a `config` argument to `Estimator`, this must be - `tf.estimator.RunConfig`. You may need to edit your code accordingly. -* Edit your `model_fn` to return `tf.estimator.EstimatorSpec`. Refer to - `EstimatorSpec` for documentation of specific fields. -* If your `model_fn` uses the `mode` argument, use `tf.estimator.ModeKeys`. - -Some related classes: -* `Evaluable`, `Trainable`: Not supported, merged into `tf.estimator.Estimator`. -* ExportStrategy: Replaced by `tf.estimator.Exporter`. - -## Head/MultiHead -These classes are now supported under `tf.contrib.estimator`, e.g. -`tf.contrib.estimator.multi_class_head` and `tf.contrib.estimator.multi_head`. - -Some differences: - -* `multi_class_head`: If you use `tf.contrib.learn.multi_class_head` with - `n_classes=2`, switch to `tf.contrib.estimator.binary_classification_head`. -* `loss_only_head`: Not supported. -* `poisson_regression_head`: Not supported (yet). -* `binary_svm_head`: Not supported (yet). -* `no_op_train_fn`: Replace it with `tf.no_op`. - -Some arguments are renamed, please refer to documentation. In addition: - -* `loss_fn`: Supported for `multi_label_head`. If you need it for other heads, - please open an issue. -* `metric_class_ids`: Not supported (yet). -* `enable_centered_bias`: Not supported. Dropping this argument is unlikely to - harm your model. -* `label_name`: Not needed in `tf.estimator`. If you don’t use `multi_head`, - drop this argument. If you use `multi_head`, refer to - `tf.contrib.estimator.multi_head` documentation. - -## Experiment Class - Distributed Training Tooling - -Switch to `tf.estimator.train_and_evaluate`. Some differences: - -* Most of the constructor arguments, like `train_input_fn`, `eval_input_fn`, - should be wrapped into `tf.estimator.TrainSpec` and `tf.estimator.EvalSpec`. -* Remove the `experiment_fn`. Instead, create the `Estimator`, `train_spec` - and `eval_spec`, then call `tf.estimator.train_and_evaluate` directly. -* Inside `tf.estimator.EvalSpec`, the `exporter` field is the replacement for - `export_strategy`. To be precise, `tf.estimator.LatestExporter` is the - replacement for `tf.contrib.learn.make_export_strategy`. If you want to - export only at the end of training use `tf.estimator.FinalExporter`. -* If the `TF_CONFIG` environment variable is constructed manually, please read - the `train_and_evaluate` documentation for the new requirements (in - particular, the chief node and evaluator node). - -## Others Classes and Functions - -* `tf.contrib.learn.datasets` is deprecated. We are adding ready to use datasets - to tensorflow/models. Many smaller datasets are available from other sources, - such as scikits.learn. Some Python processing may have to be written, but this - is straightforward to implement using the standard modules. -* `tf.contrib.learn.preprocessing`: Deprecated. The python-only preprocessing - functions are not a good fit for TensorFlow. Please use `tf.data`, and - consider tensorflow/transform for more complex use cases. -* `tf.contrib.learn.models`: Not supported, use canned estimators instead. -* `tf.contrib.learn.monitors`: Implement `SessionRunHook` instead. Hook - implementations are in `tf.train`. -* `tf.contrib.learn.learn_io`: Use the methods in `tf.estimator.inputs`, such as - `tf.estimator.inputs.numpy_input_fn`. Some utility functions have no - equivalent, we encourage the use of `tf.data`. - diff --git a/tensorflow/contrib/learn/__init__.py b/tensorflow/contrib/learn/__init__.py deleted file mode 100644 index 7bf2ac62d76..00000000000 --- a/tensorflow/contrib/learn/__init__.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""High level API for learning (DEPRECATED). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. - -@@BaseEstimator -@@Estimator -@@Trainable -@@Evaluable -@@KMeansClustering -@@ModeKeys -@@ModelFnOps -@@MetricSpec -@@PredictionKey -@@DNNClassifier -@@DNNEstimator -@@DNNRegressor -@@DNNLinearCombinedRegressor -@@DNNLinearCombinedEstimator -@@DNNLinearCombinedClassifier -@@DynamicRnnEstimator -@@LinearClassifier -@@LinearEstimator -@@LinearRegressor -@@LogisticRegressor -@@StateSavingRnnEstimator -@@SVM -@@SKCompat - -@@Head -@@multi_class_head -@@multi_label_head -@@binary_svm_head -@@regression_head -@@poisson_regression_head -@@multi_head -@@no_op_train_fn - -@@Experiment -@@ExportStrategy -@@TaskType - -@@NanLossDuringTrainingError -@@RunConfig -@@evaluate -@@infer -@@run_feeds -@@run_n -@@train - -@@extract_dask_data -@@extract_dask_labels -@@extract_pandas_data -@@extract_pandas_labels -@@extract_pandas_matrix -@@infer_real_valued_columns_from_input -@@infer_real_valued_columns_from_input_fn -@@read_batch_examples -@@read_batch_features -@@read_batch_record_features -@@read_keyed_batch_examples -@@read_keyed_batch_examples_shared_queue -@@read_keyed_batch_features -@@read_keyed_batch_features_shared_queue - -@@InputFnOps -@@ProblemType -@@build_parsing_serving_input_fn -@@make_export_strategy -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.learn.python.learn import * -# pylint: enable=wildcard-import - -from tensorflow.contrib.learn.python.learn import learn_runner_lib as learn_runner - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = ['datasets', 'head', 'io', 'learn_runner', 'models', - 'monitors', 'NotFittedError', 'ops', 'preprocessing', - 'utils', 'graph_actions'] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/learn/python/__init__.py b/tensorflow/contrib/learn/python/__init__.py deleted file mode 100644 index df23aeb2c43..00000000000 --- a/tensorflow/contrib/learn/python/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""High level API for learning with TensorFlow (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.learn.python.learn import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/learn/python/learn/__init__.py b/tensorflow/contrib/learn/python/learn/__init__.py deleted file mode 100644 index 76e0e8ac8f1..00000000000 --- a/tensorflow/contrib/learn/python/learn/__init__.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""High level API for learning with TensorFlow (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.learn.python.learn import basic_session_run_hooks -from tensorflow.contrib.learn.python.learn import datasets -from tensorflow.contrib.learn.python.learn import estimators -from tensorflow.contrib.learn.python.learn import graph_actions -from tensorflow.contrib.learn.python.learn import learn_io as io -from tensorflow.contrib.learn.python.learn import models -from tensorflow.contrib.learn.python.learn import monitors -from tensorflow.contrib.learn.python.learn import ops -from tensorflow.contrib.learn.python.learn import preprocessing -from tensorflow.contrib.learn.python.learn import utils -from tensorflow.contrib.learn.python.learn.estimators import * -from tensorflow.contrib.learn.python.learn.evaluable import Evaluable -from tensorflow.contrib.learn.python.learn.experiment import Experiment -from tensorflow.contrib.learn.python.learn.export_strategy import ExportStrategy -from tensorflow.contrib.learn.python.learn.graph_actions import evaluate -from tensorflow.contrib.learn.python.learn.graph_actions import infer -from tensorflow.contrib.learn.python.learn.graph_actions import run_feeds -from tensorflow.contrib.learn.python.learn.graph_actions import run_n -from tensorflow.contrib.learn.python.learn.graph_actions import train -from tensorflow.contrib.learn.python.learn.learn_io import * -from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec -from tensorflow.contrib.learn.python.learn.monitors import NanLossDuringTrainingError -from tensorflow.contrib.learn.python.learn.trainable import Trainable -from tensorflow.contrib.learn.python.learn.utils import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/learn/python/learn/basic_session_run_hooks.py b/tensorflow/contrib/learn/python/learn/basic_session_run_hooks.py deleted file mode 100644 index fed1c44d197..00000000000 --- a/tensorflow/contrib/learn/python/learn/basic_session_run_hooks.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Some common SessionRunHook classes (deprected). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.util.deprecation import deprecated_alias - -# pylint: disable=invalid-name -LoggingTensorHook = deprecated_alias( - 'tf.contrib.learn.basic_session_run_hooks.LoggingTensorHook', - 'tf.train.LoggingTensorHook', - basic_session_run_hooks.LoggingTensorHook) -StopAtStepHook = deprecated_alias( - 'tf.contrib.learn.basic_session_run_hooks.StopAtStepHook', - 'tf.train.StopAtStepHook', - basic_session_run_hooks.StopAtStepHook) -CheckpointSaverHook = deprecated_alias( - 'tf.contrib.learn.basic_session_run_hooks.CheckpointSaverHook', - 'tf.train.CheckpointSaverHook', - basic_session_run_hooks.CheckpointSaverHook) -StepCounterHook = deprecated_alias( - 'tf.contrib.learn.basic_session_run_hooks.StepCounterHook', - 'tf.train.StepCounterHook', - basic_session_run_hooks.StepCounterHook) -NanLossDuringTrainingError = deprecated_alias( - 'tf.contrib.learn.basic_session_run_hooks.NanLossDuringTrainingError', - 'tf.train.NanLossDuringTrainingError', - basic_session_run_hooks.NanLossDuringTrainingError) -NanTensorHook = deprecated_alias( - 'tf.contrib.learn.basic_session_run_hooks.NanTensorHook', - 'tf.train.NanTensorHook', - basic_session_run_hooks.NanTensorHook) -SummarySaverHook = deprecated_alias( - 'tf.contrib.learn.basic_session_run_hooks.SummarySaverHook', - 'tf.train.SummarySaverHook', - basic_session_run_hooks.SummarySaverHook) -# pylint: enable=invalid-name diff --git a/tensorflow/contrib/learn/python/learn/datasets/BUILD b/tensorflow/contrib/learn/python/learn/datasets/BUILD deleted file mode 100644 index fdd85e7213b..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/BUILD +++ /dev/null @@ -1,85 +0,0 @@ -# Prepare training and testing data. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -filegroup( - name = "data_csv", - srcs = glob(["data/*.csv"]), -) - -py_library( - name = "datasets", - srcs = [ - "__init__.py", - "base.py", - "mnist.py", - "produce_small_datasets.py", - "synthetic.py", - "text_datasets.py", - ], - data = [":data_csv"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:random_seed", - "//third_party/py/numpy", - ], -) - -py_binary( - name = "produce_small_datasets", - srcs = ["produce_small_datasets.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":datasets", - "//tensorflow/python:platform", - ], -) - -py_test( - name = "base_test", - size = "small", - srcs = ["base_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":datasets", - "//tensorflow/python:client_testlib", - ], -) - -py_test( - name = "load_csv_test", - size = "small", - srcs = ["load_csv_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":datasets", - "//tensorflow/python:client_testlib", - ], -) - -py_test( - name = "synthetic_test", - size = "small", - srcs = ["synthetic_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":datasets", - "//tensorflow/python:client_testlib", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) diff --git a/tensorflow/contrib/learn/python/learn/datasets/__init__.py b/tensorflow/contrib/learn/python/learn/datasets/__init__.py deleted file mode 100644 index 3c34712ac85..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/__init__.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Dataset utilities and synthetic/reference datasets (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import csv -from os import path - -import numpy as np - -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.contrib.learn.python.learn.datasets import mnist -from tensorflow.contrib.learn.python.learn.datasets import synthetic -from tensorflow.contrib.learn.python.learn.datasets import text_datasets -from tensorflow.python.util.deprecation import deprecated - -# Export load_iris and load_boston. -load_iris = base.load_iris -load_boston = base.load_boston - -# List of all available datasets. -# Note, currently they may return different types. -DATASETS = { - # Returns base.Dataset. - 'iris': base.load_iris, - 'boston': base.load_boston, - # Returns base.Datasets (train/validation/test sets). - 'mnist': mnist.load_mnist, - 'dbpedia': text_datasets.load_dbpedia, -} - -# List of all synthetic datasets -SYNTHETIC = { - # All of these will return ['data', 'target'] -> base.Dataset - 'circles': synthetic.circles, - 'spirals': synthetic.spirals -} - - -@deprecated(None, 'Please use tf.data.') -def load_dataset(name, size='small', test_with_fake_data=False): - """Loads dataset by name. - - Args: - name: Name of the dataset to load. - size: Size of the dataset to load. - test_with_fake_data: If true, load with fake dataset. - - Returns: - Features and labels for given dataset. Can be numpy or iterator. - - Raises: - ValueError: if `name` is not found. - """ - if name not in DATASETS: - raise ValueError('Name of dataset is not found: %s' % name) - if name == 'dbpedia': - return DATASETS[name](size, test_with_fake_data) - else: - return DATASETS[name]() - - -@deprecated(None, 'Please use tf.data.') -def make_dataset(name, n_samples=100, noise=None, seed=42, *args, **kwargs): - """Creates binary synthetic datasets. - - Args: - name: str, name of the dataset to generate - n_samples: int, number of datapoints to generate - noise: float or None, standard deviation of the Gaussian noise added - seed: int or None, seed for noise - - Returns: - Shuffled features and labels for given synthetic dataset of type - `base.Dataset` - - Raises: - ValueError: Raised if `name` not found - - Note: - - This is a generic synthetic data generator - individual generators might - have more parameters! - See documentation for individual parameters - - Note that the `noise` parameter uses `numpy.random.normal` and depends on - `numpy`'s seed - - TODO: - - Support multiclass datasets - - Need shuffling routine. Currently synthetic datasets are reshuffled to - avoid train/test correlation, - but that hurts reprodusability - """ - # seed = kwargs.pop('seed', None) - if name not in SYNTHETIC: - raise ValueError('Synthetic dataset not found or not implemeted: %s' % name) - else: - return SYNTHETIC[name]( - n_samples=n_samples, noise=noise, seed=seed, *args, **kwargs) diff --git a/tensorflow/contrib/learn/python/learn/datasets/base.py b/tensorflow/contrib/learn/python/learn/datasets/base.py deleted file mode 100644 index 4676eedb206..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/base.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Base utilities for loading datasets (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import csv -import os -from os import path -import random -import time - -import numpy as np -from six.moves import urllib - -from tensorflow.python.platform import gfile -from tensorflow.python.util.deprecation import deprecated - - -Dataset = collections.namedtuple('Dataset', ['data', 'target']) -Datasets = collections.namedtuple('Datasets', ['train', 'validation', 'test']) - - -@deprecated(None, 'Use tf.data instead.') -def load_csv_with_header(filename, - target_dtype, - features_dtype, - target_column=-1): - """Load dataset from CSV file with a header row.""" - with gfile.Open(filename) as csv_file: - data_file = csv.reader(csv_file) - header = next(data_file) - n_samples = int(header[0]) - n_features = int(header[1]) - data = np.zeros((n_samples, n_features), dtype=features_dtype) - target = np.zeros((n_samples,), dtype=target_dtype) - for i, row in enumerate(data_file): - target[i] = np.asarray(row.pop(target_column), dtype=target_dtype) - data[i] = np.asarray(row, dtype=features_dtype) - - return Dataset(data=data, target=target) - - -@deprecated(None, 'Use tf.data instead.') -def load_csv_without_header(filename, - target_dtype, - features_dtype, - target_column=-1): - """Load dataset from CSV file without a header row.""" - with gfile.Open(filename) as csv_file: - data_file = csv.reader(csv_file) - data, target = [], [] - for row in data_file: - target.append(row.pop(target_column)) - data.append(np.asarray(row, dtype=features_dtype)) - - target = np.array(target, dtype=target_dtype) - data = np.array(data) - return Dataset(data=data, target=target) - - -@deprecated(None, 'Use tf.data instead.') -def shrink_csv(filename, ratio): - """Create a smaller dataset of only 1/ratio of original data.""" - filename_small = filename.replace('.', '_small.') - with gfile.Open(filename_small, 'w') as csv_file_small: - writer = csv.writer(csv_file_small) - with gfile.Open(filename) as csv_file: - reader = csv.reader(csv_file) - i = 0 - for row in reader: - if i % ratio == 0: - writer.writerow(row) - i += 1 - - -@deprecated(None, 'Use scikits.learn.datasets.') -def load_iris(data_path=None): - """Load Iris dataset. - - Args: - data_path: string, path to iris dataset (optional) - - Returns: - Dataset object containing data in-memory. - """ - if data_path is None: - module_path = path.dirname(__file__) - data_path = path.join(module_path, 'data', 'iris.csv') - return load_csv_with_header( - data_path, target_dtype=np.int, features_dtype=np.float) - - -@deprecated(None, 'Use scikits.learn.datasets.') -def load_boston(data_path=None): - """Load Boston housing dataset. - - Args: - data_path: string, path to boston dataset (optional) - - Returns: - Dataset object containing data in-memory. - """ - if data_path is None: - module_path = path.dirname(__file__) - data_path = path.join(module_path, 'data', 'boston_house_prices.csv') - return load_csv_with_header( - data_path, target_dtype=np.float, features_dtype=np.float) - - -@deprecated(None, 'Use the retry module or similar alternatives.') -def retry(initial_delay, - max_delay, - factor=2.0, - jitter=0.25, - is_retriable=None): - """Simple decorator for wrapping retriable functions. - - Args: - initial_delay: the initial delay. - max_delay: the maximum delay allowed (actual max is - max_delay * (1 + jitter). - factor: each subsequent retry, the delay is multiplied by this value. - (must be >= 1). - jitter: to avoid lockstep, the returned delay is multiplied by a random - number between (1-jitter) and (1+jitter). To add a 20% jitter, set - jitter = 0.2. Must be < 1. - is_retriable: (optional) a function that takes an Exception as an argument - and returns true if retry should be applied. - - Returns: - A function that wraps another function to automatically retry it. - """ - return _internal_retry( - initial_delay=initial_delay, - max_delay=max_delay, - factor=factor, - jitter=jitter, - is_retriable=is_retriable) - - -def _internal_retry(initial_delay, - max_delay, - factor=2.0, - jitter=0.25, - is_retriable=None): - """Simple decorator for wrapping retriable functions, for internal use only. - - Args: - initial_delay: the initial delay. - max_delay: the maximum delay allowed (actual max is - max_delay * (1 + jitter). - factor: each subsequent retry, the delay is multiplied by this value. - (must be >= 1). - jitter: to avoid lockstep, the returned delay is multiplied by a random - number between (1-jitter) and (1+jitter). To add a 20% jitter, set - jitter = 0.2. Must be < 1. - is_retriable: (optional) a function that takes an Exception as an argument - and returns true if retry should be applied. - - Returns: - A function that wraps another function to automatically retry it. - """ - if factor < 1: - raise ValueError('factor must be >= 1; was %f' % (factor,)) - - if jitter >= 1: - raise ValueError('jitter must be < 1; was %f' % (jitter,)) - - # Generator to compute the individual delays - def delays(): - delay = initial_delay - while delay <= max_delay: - yield delay * random.uniform(1 - jitter, 1 + jitter) - delay *= factor - - def wrap(fn): - """Wrapper function factory invoked by decorator magic.""" - - def wrapped_fn(*args, **kwargs): - """The actual wrapper function that applies the retry logic.""" - for delay in delays(): - try: - return fn(*args, **kwargs) - except Exception as e: # pylint: disable=broad-except - if is_retriable is None: - continue - - if is_retriable(e): - time.sleep(delay) - else: - raise - return fn(*args, **kwargs) - - return wrapped_fn - - return wrap - - -_RETRIABLE_ERRNOS = { - 110, # Connection timed out [socket.py] -} - - -def _is_retriable(e): - return isinstance(e, IOError) and e.errno in _RETRIABLE_ERRNOS - - -@deprecated(None, 'Please use urllib or similar directly.') -@_internal_retry(initial_delay=1.0, max_delay=16.0, is_retriable=_is_retriable) -def urlretrieve_with_retry(url, filename=None): - return urllib.request.urlretrieve(url, filename) - - -@deprecated(None, 'Please write your own downloading logic.') -def maybe_download(filename, work_directory, source_url): - """Download the data from source url, unless it's already here. - - Args: - filename: string, name of the file in the directory. - work_directory: string, path to working directory. - source_url: url to download from if file doesn't exist. - - Returns: - Path to resulting file. - """ - if not gfile.Exists(work_directory): - gfile.MakeDirs(work_directory) - filepath = os.path.join(work_directory, filename) - if not gfile.Exists(filepath): - temp_file_name, _ = urlretrieve_with_retry(source_url) - gfile.Copy(temp_file_name, filepath) - with gfile.GFile(filepath) as f: - size = f.size() - print('Successfully downloaded', filename, size, 'bytes.') - return filepath diff --git a/tensorflow/contrib/learn/python/learn/datasets/base_test.py b/tensorflow/contrib/learn/python/learn/datasets/base_test.py deleted file mode 100644 index bc60d3797dc..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/base_test.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.python.platform import test - -mock = test.mock - -_TIMEOUT = IOError(110, "timeout") - - -class BaseTest(test.TestCase): - """Test load csv functions.""" - - def testUrlretrieveRetriesOnIOError(self): - with mock.patch.object(base, "time") as mock_time: - with mock.patch.object(base, "urllib") as mock_urllib: - mock_urllib.request.urlretrieve.side_effect = [ - _TIMEOUT, _TIMEOUT, _TIMEOUT, _TIMEOUT, _TIMEOUT, None - ] - base.urlretrieve_with_retry("http://dummy.com", "/tmp/dummy") - - # Assert full backoff was tried - actual_list = [arg[0][0] for arg in mock_time.sleep.call_args_list] - expected_list = [1, 2, 4, 8, 16] - for actual, expected in zip(actual_list, expected_list): - self.assertLessEqual(abs(actual - expected), 0.25 * expected) - self.assertEquals(len(actual_list), len(expected_list)) - - def testUrlretrieveRaisesAfterRetriesAreExhausted(self): - with mock.patch.object(base, "time") as mock_time: - with mock.patch.object(base, "urllib") as mock_urllib: - mock_urllib.request.urlretrieve.side_effect = [ - _TIMEOUT, - _TIMEOUT, - _TIMEOUT, - _TIMEOUT, - _TIMEOUT, - _TIMEOUT, - ] - with self.assertRaises(IOError): - base.urlretrieve_with_retry("http://dummy.com", "/tmp/dummy") - - # Assert full backoff was tried - actual_list = [arg[0][0] for arg in mock_time.sleep.call_args_list] - expected_list = [1, 2, 4, 8, 16] - for actual, expected in zip(actual_list, expected_list): - self.assertLessEqual(abs(actual - expected), 0.25 * expected) - self.assertEquals(len(actual_list), len(expected_list)) - - def testUrlretrieveRaisesOnNonRetriableErrorWithoutRetry(self): - with mock.patch.object(base, "time") as mock_time: - with mock.patch.object(base, "urllib") as mock_urllib: - mock_urllib.request.urlretrieve.side_effect = [ - IOError(2, "No such file or directory"), - ] - with self.assertRaises(IOError): - base.urlretrieve_with_retry("http://dummy.com", "/tmp/dummy") - - # Assert no retries - self.assertFalse(mock_time.called) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/datasets/data/boston_house_prices.csv b/tensorflow/contrib/learn/python/learn/datasets/data/boston_house_prices.csv deleted file mode 100644 index 5f667b6d1ab..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/data/boston_house_prices.csv +++ /dev/null @@ -1,507 +0,0 @@ -506,13,"CRIM","ZN","INDUS","CHAS","NOX","RM","AGE","DIS","RAD","TAX","PTRATIO","B","LSTAT","MEDV" -0.00632,18,2.31,0,0.538,6.575,65.2,4.09,1,296,15.3,396.9,4.98,24 -0.02731,0,7.07,0,0.469,6.421,78.9,4.9671,2,242,17.8,396.9,9.14,21.6 -0.02729,0,7.07,0,0.469,7.185,61.1,4.9671,2,242,17.8,392.83,4.03,34.7 -0.03237,0,2.18,0,0.458,6.998,45.8,6.0622,3,222,18.7,394.63,2.94,33.4 -0.06905,0,2.18,0,0.458,7.147,54.2,6.0622,3,222,18.7,396.9,5.33,36.2 -0.02985,0,2.18,0,0.458,6.43,58.7,6.0622,3,222,18.7,394.12,5.21,28.7 -0.08829,12.5,7.87,0,0.524,6.012,66.6,5.5605,5,311,15.2,395.6,12.43,22.9 -0.14455,12.5,7.87,0,0.524,6.172,96.1,5.9505,5,311,15.2,396.9,19.15,27.1 -0.21124,12.5,7.87,0,0.524,5.631,100,6.0821,5,311,15.2,386.63,29.93,16.5 -0.17004,12.5,7.87,0,0.524,6.004,85.9,6.5921,5,311,15.2,386.71,17.1,18.9 -0.22489,12.5,7.87,0,0.524,6.377,94.3,6.3467,5,311,15.2,392.52,20.45,15 -0.11747,12.5,7.87,0,0.524,6.009,82.9,6.2267,5,311,15.2,396.9,13.27,18.9 -0.09378,12.5,7.87,0,0.524,5.889,39,5.4509,5,311,15.2,390.5,15.71,21.7 -0.62976,0,8.14,0,0.538,5.949,61.8,4.7075,4,307,21,396.9,8.26,20.4 -0.63796,0,8.14,0,0.538,6.096,84.5,4.4619,4,307,21,380.02,10.26,18.2 -0.62739,0,8.14,0,0.538,5.834,56.5,4.4986,4,307,21,395.62,8.47,19.9 -1.05393,0,8.14,0,0.538,5.935,29.3,4.4986,4,307,21,386.85,6.58,23.1 -0.7842,0,8.14,0,0.538,5.99,81.7,4.2579,4,307,21,386.75,14.67,17.5 -0.80271,0,8.14,0,0.538,5.456,36.6,3.7965,4,307,21,288.99,11.69,20.2 -0.7258,0,8.14,0,0.538,5.727,69.5,3.7965,4,307,21,390.95,11.28,18.2 -1.25179,0,8.14,0,0.538,5.57,98.1,3.7979,4,307,21,376.57,21.02,13.6 -0.85204,0,8.14,0,0.538,5.965,89.2,4.0123,4,307,21,392.53,13.83,19.6 -1.23247,0,8.14,0,0.538,6.142,91.7,3.9769,4,307,21,396.9,18.72,15.2 -0.98843,0,8.14,0,0.538,5.813,100,4.0952,4,307,21,394.54,19.88,14.5 -0.75026,0,8.14,0,0.538,5.924,94.1,4.3996,4,307,21,394.33,16.3,15.6 -0.84054,0,8.14,0,0.538,5.599,85.7,4.4546,4,307,21,303.42,16.51,13.9 -0.67191,0,8.14,0,0.538,5.813,90.3,4.682,4,307,21,376.88,14.81,16.6 -0.95577,0,8.14,0,0.538,6.047,88.8,4.4534,4,307,21,306.38,17.28,14.8 -0.77299,0,8.14,0,0.538,6.495,94.4,4.4547,4,307,21,387.94,12.8,18.4 -1.00245,0,8.14,0,0.538,6.674,87.3,4.239,4,307,21,380.23,11.98,21 -1.13081,0,8.14,0,0.538,5.713,94.1,4.233,4,307,21,360.17,22.6,12.7 -1.35472,0,8.14,0,0.538,6.072,100,4.175,4,307,21,376.73,13.04,14.5 -1.38799,0,8.14,0,0.538,5.95,82,3.99,4,307,21,232.6,27.71,13.2 -1.15172,0,8.14,0,0.538,5.701,95,3.7872,4,307,21,358.77,18.35,13.1 -1.61282,0,8.14,0,0.538,6.096,96.9,3.7598,4,307,21,248.31,20.34,13.5 -0.06417,0,5.96,0,0.499,5.933,68.2,3.3603,5,279,19.2,396.9,9.68,18.9 -0.09744,0,5.96,0,0.499,5.841,61.4,3.3779,5,279,19.2,377.56,11.41,20 -0.08014,0,5.96,0,0.499,5.85,41.5,3.9342,5,279,19.2,396.9,8.77,21 -0.17505,0,5.96,0,0.499,5.966,30.2,3.8473,5,279,19.2,393.43,10.13,24.7 -0.02763,75,2.95,0,0.428,6.595,21.8,5.4011,3,252,18.3,395.63,4.32,30.8 -0.03359,75,2.95,0,0.428,7.024,15.8,5.4011,3,252,18.3,395.62,1.98,34.9 -0.12744,0,6.91,0,0.448,6.77,2.9,5.7209,3,233,17.9,385.41,4.84,26.6 -0.1415,0,6.91,0,0.448,6.169,6.6,5.7209,3,233,17.9,383.37,5.81,25.3 -0.15936,0,6.91,0,0.448,6.211,6.5,5.7209,3,233,17.9,394.46,7.44,24.7 -0.12269,0,6.91,0,0.448,6.069,40,5.7209,3,233,17.9,389.39,9.55,21.2 -0.17142,0,6.91,0,0.448,5.682,33.8,5.1004,3,233,17.9,396.9,10.21,19.3 -0.18836,0,6.91,0,0.448,5.786,33.3,5.1004,3,233,17.9,396.9,14.15,20 -0.22927,0,6.91,0,0.448,6.03,85.5,5.6894,3,233,17.9,392.74,18.8,16.6 -0.25387,0,6.91,0,0.448,5.399,95.3,5.87,3,233,17.9,396.9,30.81,14.4 -0.21977,0,6.91,0,0.448,5.602,62,6.0877,3,233,17.9,396.9,16.2,19.4 -0.08873,21,5.64,0,0.439,5.963,45.7,6.8147,4,243,16.8,395.56,13.45,19.7 -0.04337,21,5.64,0,0.439,6.115,63,6.8147,4,243,16.8,393.97,9.43,20.5 -0.0536,21,5.64,0,0.439,6.511,21.1,6.8147,4,243,16.8,396.9,5.28,25 -0.04981,21,5.64,0,0.439,5.998,21.4,6.8147,4,243,16.8,396.9,8.43,23.4 -0.0136,75,4,0,0.41,5.888,47.6,7.3197,3,469,21.1,396.9,14.8,18.9 -0.01311,90,1.22,0,0.403,7.249,21.9,8.6966,5,226,17.9,395.93,4.81,35.4 -0.02055,85,0.74,0,0.41,6.383,35.7,9.1876,2,313,17.3,396.9,5.77,24.7 -0.01432,100,1.32,0,0.411,6.816,40.5,8.3248,5,256,15.1,392.9,3.95,31.6 -0.15445,25,5.13,0,0.453,6.145,29.2,7.8148,8,284,19.7,390.68,6.86,23.3 -0.10328,25,5.13,0,0.453,5.927,47.2,6.932,8,284,19.7,396.9,9.22,19.6 -0.14932,25,5.13,0,0.453,5.741,66.2,7.2254,8,284,19.7,395.11,13.15,18.7 -0.17171,25,5.13,0,0.453,5.966,93.4,6.8185,8,284,19.7,378.08,14.44,16 -0.11027,25,5.13,0,0.453,6.456,67.8,7.2255,8,284,19.7,396.9,6.73,22.2 -0.1265,25,5.13,0,0.453,6.762,43.4,7.9809,8,284,19.7,395.58,9.5,25 -0.01951,17.5,1.38,0,0.4161,7.104,59.5,9.2229,3,216,18.6,393.24,8.05,33 -0.03584,80,3.37,0,0.398,6.29,17.8,6.6115,4,337,16.1,396.9,4.67,23.5 -0.04379,80,3.37,0,0.398,5.787,31.1,6.6115,4,337,16.1,396.9,10.24,19.4 -0.05789,12.5,6.07,0,0.409,5.878,21.4,6.498,4,345,18.9,396.21,8.1,22 -0.13554,12.5,6.07,0,0.409,5.594,36.8,6.498,4,345,18.9,396.9,13.09,17.4 -0.12816,12.5,6.07,0,0.409,5.885,33,6.498,4,345,18.9,396.9,8.79,20.9 -0.08826,0,10.81,0,0.413,6.417,6.6,5.2873,4,305,19.2,383.73,6.72,24.2 -0.15876,0,10.81,0,0.413,5.961,17.5,5.2873,4,305,19.2,376.94,9.88,21.7 -0.09164,0,10.81,0,0.413,6.065,7.8,5.2873,4,305,19.2,390.91,5.52,22.8 -0.19539,0,10.81,0,0.413,6.245,6.2,5.2873,4,305,19.2,377.17,7.54,23.4 -0.07896,0,12.83,0,0.437,6.273,6,4.2515,5,398,18.7,394.92,6.78,24.1 -0.09512,0,12.83,0,0.437,6.286,45,4.5026,5,398,18.7,383.23,8.94,21.4 -0.10153,0,12.83,0,0.437,6.279,74.5,4.0522,5,398,18.7,373.66,11.97,20 -0.08707,0,12.83,0,0.437,6.14,45.8,4.0905,5,398,18.7,386.96,10.27,20.8 -0.05646,0,12.83,0,0.437,6.232,53.7,5.0141,5,398,18.7,386.4,12.34,21.2 -0.08387,0,12.83,0,0.437,5.874,36.6,4.5026,5,398,18.7,396.06,9.1,20.3 -0.04113,25,4.86,0,0.426,6.727,33.5,5.4007,4,281,19,396.9,5.29,28 -0.04462,25,4.86,0,0.426,6.619,70.4,5.4007,4,281,19,395.63,7.22,23.9 -0.03659,25,4.86,0,0.426,6.302,32.2,5.4007,4,281,19,396.9,6.72,24.8 -0.03551,25,4.86,0,0.426,6.167,46.7,5.4007,4,281,19,390.64,7.51,22.9 -0.05059,0,4.49,0,0.449,6.389,48,4.7794,3,247,18.5,396.9,9.62,23.9 -0.05735,0,4.49,0,0.449,6.63,56.1,4.4377,3,247,18.5,392.3,6.53,26.6 -0.05188,0,4.49,0,0.449,6.015,45.1,4.4272,3,247,18.5,395.99,12.86,22.5 -0.07151,0,4.49,0,0.449,6.121,56.8,3.7476,3,247,18.5,395.15,8.44,22.2 -0.0566,0,3.41,0,0.489,7.007,86.3,3.4217,2,270,17.8,396.9,5.5,23.6 -0.05302,0,3.41,0,0.489,7.079,63.1,3.4145,2,270,17.8,396.06,5.7,28.7 -0.04684,0,3.41,0,0.489,6.417,66.1,3.0923,2,270,17.8,392.18,8.81,22.6 -0.03932,0,3.41,0,0.489,6.405,73.9,3.0921,2,270,17.8,393.55,8.2,22 -0.04203,28,15.04,0,0.464,6.442,53.6,3.6659,4,270,18.2,395.01,8.16,22.9 -0.02875,28,15.04,0,0.464,6.211,28.9,3.6659,4,270,18.2,396.33,6.21,25 -0.04294,28,15.04,0,0.464,6.249,77.3,3.615,4,270,18.2,396.9,10.59,20.6 -0.12204,0,2.89,0,0.445,6.625,57.8,3.4952,2,276,18,357.98,6.65,28.4 -0.11504,0,2.89,0,0.445,6.163,69.6,3.4952,2,276,18,391.83,11.34,21.4 -0.12083,0,2.89,0,0.445,8.069,76,3.4952,2,276,18,396.9,4.21,38.7 -0.08187,0,2.89,0,0.445,7.82,36.9,3.4952,2,276,18,393.53,3.57,43.8 -0.0686,0,2.89,0,0.445,7.416,62.5,3.4952,2,276,18,396.9,6.19,33.2 -0.14866,0,8.56,0,0.52,6.727,79.9,2.7778,5,384,20.9,394.76,9.42,27.5 -0.11432,0,8.56,0,0.52,6.781,71.3,2.8561,5,384,20.9,395.58,7.67,26.5 -0.22876,0,8.56,0,0.52,6.405,85.4,2.7147,5,384,20.9,70.8,10.63,18.6 -0.21161,0,8.56,0,0.52,6.137,87.4,2.7147,5,384,20.9,394.47,13.44,19.3 -0.1396,0,8.56,0,0.52,6.167,90,2.421,5,384,20.9,392.69,12.33,20.1 -0.13262,0,8.56,0,0.52,5.851,96.7,2.1069,5,384,20.9,394.05,16.47,19.5 -0.1712,0,8.56,0,0.52,5.836,91.9,2.211,5,384,20.9,395.67,18.66,19.5 -0.13117,0,8.56,0,0.52,6.127,85.2,2.1224,5,384,20.9,387.69,14.09,20.4 -0.12802,0,8.56,0,0.52,6.474,97.1,2.4329,5,384,20.9,395.24,12.27,19.8 -0.26363,0,8.56,0,0.52,6.229,91.2,2.5451,5,384,20.9,391.23,15.55,19.4 -0.10793,0,8.56,0,0.52,6.195,54.4,2.7778,5,384,20.9,393.49,13,21.7 -0.10084,0,10.01,0,0.547,6.715,81.6,2.6775,6,432,17.8,395.59,10.16,22.8 -0.12329,0,10.01,0,0.547,5.913,92.9,2.3534,6,432,17.8,394.95,16.21,18.8 -0.22212,0,10.01,0,0.547,6.092,95.4,2.548,6,432,17.8,396.9,17.09,18.7 -0.14231,0,10.01,0,0.547,6.254,84.2,2.2565,6,432,17.8,388.74,10.45,18.5 -0.17134,0,10.01,0,0.547,5.928,88.2,2.4631,6,432,17.8,344.91,15.76,18.3 -0.13158,0,10.01,0,0.547,6.176,72.5,2.7301,6,432,17.8,393.3,12.04,21.2 -0.15098,0,10.01,0,0.547,6.021,82.6,2.7474,6,432,17.8,394.51,10.3,19.2 -0.13058,0,10.01,0,0.547,5.872,73.1,2.4775,6,432,17.8,338.63,15.37,20.4 -0.14476,0,10.01,0,0.547,5.731,65.2,2.7592,6,432,17.8,391.5,13.61,19.3 -0.06899,0,25.65,0,0.581,5.87,69.7,2.2577,2,188,19.1,389.15,14.37,22 -0.07165,0,25.65,0,0.581,6.004,84.1,2.1974,2,188,19.1,377.67,14.27,20.3 -0.09299,0,25.65,0,0.581,5.961,92.9,2.0869,2,188,19.1,378.09,17.93,20.5 -0.15038,0,25.65,0,0.581,5.856,97,1.9444,2,188,19.1,370.31,25.41,17.3 -0.09849,0,25.65,0,0.581,5.879,95.8,2.0063,2,188,19.1,379.38,17.58,18.8 -0.16902,0,25.65,0,0.581,5.986,88.4,1.9929,2,188,19.1,385.02,14.81,21.4 -0.38735,0,25.65,0,0.581,5.613,95.6,1.7572,2,188,19.1,359.29,27.26,15.7 -0.25915,0,21.89,0,0.624,5.693,96,1.7883,4,437,21.2,392.11,17.19,16.2 -0.32543,0,21.89,0,0.624,6.431,98.8,1.8125,4,437,21.2,396.9,15.39,18 -0.88125,0,21.89,0,0.624,5.637,94.7,1.9799,4,437,21.2,396.9,18.34,14.3 -0.34006,0,21.89,0,0.624,6.458,98.9,2.1185,4,437,21.2,395.04,12.6,19.2 -1.19294,0,21.89,0,0.624,6.326,97.7,2.271,4,437,21.2,396.9,12.26,19.6 -0.59005,0,21.89,0,0.624,6.372,97.9,2.3274,4,437,21.2,385.76,11.12,23 -0.32982,0,21.89,0,0.624,5.822,95.4,2.4699,4,437,21.2,388.69,15.03,18.4 -0.97617,0,21.89,0,0.624,5.757,98.4,2.346,4,437,21.2,262.76,17.31,15.6 -0.55778,0,21.89,0,0.624,6.335,98.2,2.1107,4,437,21.2,394.67,16.96,18.1 -0.32264,0,21.89,0,0.624,5.942,93.5,1.9669,4,437,21.2,378.25,16.9,17.4 -0.35233,0,21.89,0,0.624,6.454,98.4,1.8498,4,437,21.2,394.08,14.59,17.1 -0.2498,0,21.89,0,0.624,5.857,98.2,1.6686,4,437,21.2,392.04,21.32,13.3 -0.54452,0,21.89,0,0.624,6.151,97.9,1.6687,4,437,21.2,396.9,18.46,17.8 -0.2909,0,21.89,0,0.624,6.174,93.6,1.6119,4,437,21.2,388.08,24.16,14 -1.62864,0,21.89,0,0.624,5.019,100,1.4394,4,437,21.2,396.9,34.41,14.4 -3.32105,0,19.58,1,0.871,5.403,100,1.3216,5,403,14.7,396.9,26.82,13.4 -4.0974,0,19.58,0,0.871,5.468,100,1.4118,5,403,14.7,396.9,26.42,15.6 -2.77974,0,19.58,0,0.871,4.903,97.8,1.3459,5,403,14.7,396.9,29.29,11.8 -2.37934,0,19.58,0,0.871,6.13,100,1.4191,5,403,14.7,172.91,27.8,13.8 -2.15505,0,19.58,0,0.871,5.628,100,1.5166,5,403,14.7,169.27,16.65,15.6 -2.36862,0,19.58,0,0.871,4.926,95.7,1.4608,5,403,14.7,391.71,29.53,14.6 -2.33099,0,19.58,0,0.871,5.186,93.8,1.5296,5,403,14.7,356.99,28.32,17.8 -2.73397,0,19.58,0,0.871,5.597,94.9,1.5257,5,403,14.7,351.85,21.45,15.4 -1.6566,0,19.58,0,0.871,6.122,97.3,1.618,5,403,14.7,372.8,14.1,21.5 -1.49632,0,19.58,0,0.871,5.404,100,1.5916,5,403,14.7,341.6,13.28,19.6 -1.12658,0,19.58,1,0.871,5.012,88,1.6102,5,403,14.7,343.28,12.12,15.3 -2.14918,0,19.58,0,0.871,5.709,98.5,1.6232,5,403,14.7,261.95,15.79,19.4 -1.41385,0,19.58,1,0.871,6.129,96,1.7494,5,403,14.7,321.02,15.12,17 -3.53501,0,19.58,1,0.871,6.152,82.6,1.7455,5,403,14.7,88.01,15.02,15.6 -2.44668,0,19.58,0,0.871,5.272,94,1.7364,5,403,14.7,88.63,16.14,13.1 -1.22358,0,19.58,0,0.605,6.943,97.4,1.8773,5,403,14.7,363.43,4.59,41.3 -1.34284,0,19.58,0,0.605,6.066,100,1.7573,5,403,14.7,353.89,6.43,24.3 -1.42502,0,19.58,0,0.871,6.51,100,1.7659,5,403,14.7,364.31,7.39,23.3 -1.27346,0,19.58,1,0.605,6.25,92.6,1.7984,5,403,14.7,338.92,5.5,27 -1.46336,0,19.58,0,0.605,7.489,90.8,1.9709,5,403,14.7,374.43,1.73,50 -1.83377,0,19.58,1,0.605,7.802,98.2,2.0407,5,403,14.7,389.61,1.92,50 -1.51902,0,19.58,1,0.605,8.375,93.9,2.162,5,403,14.7,388.45,3.32,50 -2.24236,0,19.58,0,0.605,5.854,91.8,2.422,5,403,14.7,395.11,11.64,22.7 -2.924,0,19.58,0,0.605,6.101,93,2.2834,5,403,14.7,240.16,9.81,25 -2.01019,0,19.58,0,0.605,7.929,96.2,2.0459,5,403,14.7,369.3,3.7,50 -1.80028,0,19.58,0,0.605,5.877,79.2,2.4259,5,403,14.7,227.61,12.14,23.8 -2.3004,0,19.58,0,0.605,6.319,96.1,2.1,5,403,14.7,297.09,11.1,23.8 -2.44953,0,19.58,0,0.605,6.402,95.2,2.2625,5,403,14.7,330.04,11.32,22.3 -1.20742,0,19.58,0,0.605,5.875,94.6,2.4259,5,403,14.7,292.29,14.43,17.4 -2.3139,0,19.58,0,0.605,5.88,97.3,2.3887,5,403,14.7,348.13,12.03,19.1 -0.13914,0,4.05,0,0.51,5.572,88.5,2.5961,5,296,16.6,396.9,14.69,23.1 -0.09178,0,4.05,0,0.51,6.416,84.1,2.6463,5,296,16.6,395.5,9.04,23.6 -0.08447,0,4.05,0,0.51,5.859,68.7,2.7019,5,296,16.6,393.23,9.64,22.6 -0.06664,0,4.05,0,0.51,6.546,33.1,3.1323,5,296,16.6,390.96,5.33,29.4 -0.07022,0,4.05,0,0.51,6.02,47.2,3.5549,5,296,16.6,393.23,10.11,23.2 -0.05425,0,4.05,0,0.51,6.315,73.4,3.3175,5,296,16.6,395.6,6.29,24.6 -0.06642,0,4.05,0,0.51,6.86,74.4,2.9153,5,296,16.6,391.27,6.92,29.9 -0.0578,0,2.46,0,0.488,6.98,58.4,2.829,3,193,17.8,396.9,5.04,37.2 -0.06588,0,2.46,0,0.488,7.765,83.3,2.741,3,193,17.8,395.56,7.56,39.8 -0.06888,0,2.46,0,0.488,6.144,62.2,2.5979,3,193,17.8,396.9,9.45,36.2 -0.09103,0,2.46,0,0.488,7.155,92.2,2.7006,3,193,17.8,394.12,4.82,37.9 -0.10008,0,2.46,0,0.488,6.563,95.6,2.847,3,193,17.8,396.9,5.68,32.5 -0.08308,0,2.46,0,0.488,5.604,89.8,2.9879,3,193,17.8,391,13.98,26.4 -0.06047,0,2.46,0,0.488,6.153,68.8,3.2797,3,193,17.8,387.11,13.15,29.6 -0.05602,0,2.46,0,0.488,7.831,53.6,3.1992,3,193,17.8,392.63,4.45,50 -0.07875,45,3.44,0,0.437,6.782,41.1,3.7886,5,398,15.2,393.87,6.68,32 -0.12579,45,3.44,0,0.437,6.556,29.1,4.5667,5,398,15.2,382.84,4.56,29.8 -0.0837,45,3.44,0,0.437,7.185,38.9,4.5667,5,398,15.2,396.9,5.39,34.9 -0.09068,45,3.44,0,0.437,6.951,21.5,6.4798,5,398,15.2,377.68,5.1,37 -0.06911,45,3.44,0,0.437,6.739,30.8,6.4798,5,398,15.2,389.71,4.69,30.5 -0.08664,45,3.44,0,0.437,7.178,26.3,6.4798,5,398,15.2,390.49,2.87,36.4 -0.02187,60,2.93,0,0.401,6.8,9.9,6.2196,1,265,15.6,393.37,5.03,31.1 -0.01439,60,2.93,0,0.401,6.604,18.8,6.2196,1,265,15.6,376.7,4.38,29.1 -0.01381,80,0.46,0,0.422,7.875,32,5.6484,4,255,14.4,394.23,2.97,50 -0.04011,80,1.52,0,0.404,7.287,34.1,7.309,2,329,12.6,396.9,4.08,33.3 -0.04666,80,1.52,0,0.404,7.107,36.6,7.309,2,329,12.6,354.31,8.61,30.3 -0.03768,80,1.52,0,0.404,7.274,38.3,7.309,2,329,12.6,392.2,6.62,34.6 -0.0315,95,1.47,0,0.403,6.975,15.3,7.6534,3,402,17,396.9,4.56,34.9 -0.01778,95,1.47,0,0.403,7.135,13.9,7.6534,3,402,17,384.3,4.45,32.9 -0.03445,82.5,2.03,0,0.415,6.162,38.4,6.27,2,348,14.7,393.77,7.43,24.1 -0.02177,82.5,2.03,0,0.415,7.61,15.7,6.27,2,348,14.7,395.38,3.11,42.3 -0.0351,95,2.68,0,0.4161,7.853,33.2,5.118,4,224,14.7,392.78,3.81,48.5 -0.02009,95,2.68,0,0.4161,8.034,31.9,5.118,4,224,14.7,390.55,2.88,50 -0.13642,0,10.59,0,0.489,5.891,22.3,3.9454,4,277,18.6,396.9,10.87,22.6 -0.22969,0,10.59,0,0.489,6.326,52.5,4.3549,4,277,18.6,394.87,10.97,24.4 -0.25199,0,10.59,0,0.489,5.783,72.7,4.3549,4,277,18.6,389.43,18.06,22.5 -0.13587,0,10.59,1,0.489,6.064,59.1,4.2392,4,277,18.6,381.32,14.66,24.4 -0.43571,0,10.59,1,0.489,5.344,100,3.875,4,277,18.6,396.9,23.09,20 -0.17446,0,10.59,1,0.489,5.96,92.1,3.8771,4,277,18.6,393.25,17.27,21.7 -0.37578,0,10.59,1,0.489,5.404,88.6,3.665,4,277,18.6,395.24,23.98,19.3 -0.21719,0,10.59,1,0.489,5.807,53.8,3.6526,4,277,18.6,390.94,16.03,22.4 -0.14052,0,10.59,0,0.489,6.375,32.3,3.9454,4,277,18.6,385.81,9.38,28.1 -0.28955,0,10.59,0,0.489,5.412,9.8,3.5875,4,277,18.6,348.93,29.55,23.7 -0.19802,0,10.59,0,0.489,6.182,42.4,3.9454,4,277,18.6,393.63,9.47,25 -0.0456,0,13.89,1,0.55,5.888,56,3.1121,5,276,16.4,392.8,13.51,23.3 -0.07013,0,13.89,0,0.55,6.642,85.1,3.4211,5,276,16.4,392.78,9.69,28.7 -0.11069,0,13.89,1,0.55,5.951,93.8,2.8893,5,276,16.4,396.9,17.92,21.5 -0.11425,0,13.89,1,0.55,6.373,92.4,3.3633,5,276,16.4,393.74,10.5,23 -0.35809,0,6.2,1,0.507,6.951,88.5,2.8617,8,307,17.4,391.7,9.71,26.7 -0.40771,0,6.2,1,0.507,6.164,91.3,3.048,8,307,17.4,395.24,21.46,21.7 -0.62356,0,6.2,1,0.507,6.879,77.7,3.2721,8,307,17.4,390.39,9.93,27.5 -0.6147,0,6.2,0,0.507,6.618,80.8,3.2721,8,307,17.4,396.9,7.6,30.1 -0.31533,0,6.2,0,0.504,8.266,78.3,2.8944,8,307,17.4,385.05,4.14,44.8 -0.52693,0,6.2,0,0.504,8.725,83,2.8944,8,307,17.4,382,4.63,50 -0.38214,0,6.2,0,0.504,8.04,86.5,3.2157,8,307,17.4,387.38,3.13,37.6 -0.41238,0,6.2,0,0.504,7.163,79.9,3.2157,8,307,17.4,372.08,6.36,31.6 -0.29819,0,6.2,0,0.504,7.686,17,3.3751,8,307,17.4,377.51,3.92,46.7 -0.44178,0,6.2,0,0.504,6.552,21.4,3.3751,8,307,17.4,380.34,3.76,31.5 -0.537,0,6.2,0,0.504,5.981,68.1,3.6715,8,307,17.4,378.35,11.65,24.3 -0.46296,0,6.2,0,0.504,7.412,76.9,3.6715,8,307,17.4,376.14,5.25,31.7 -0.57529,0,6.2,0,0.507,8.337,73.3,3.8384,8,307,17.4,385.91,2.47,41.7 -0.33147,0,6.2,0,0.507,8.247,70.4,3.6519,8,307,17.4,378.95,3.95,48.3 -0.44791,0,6.2,1,0.507,6.726,66.5,3.6519,8,307,17.4,360.2,8.05,29 -0.33045,0,6.2,0,0.507,6.086,61.5,3.6519,8,307,17.4,376.75,10.88,24 -0.52058,0,6.2,1,0.507,6.631,76.5,4.148,8,307,17.4,388.45,9.54,25.1 -0.51183,0,6.2,0,0.507,7.358,71.6,4.148,8,307,17.4,390.07,4.73,31.5 -0.08244,30,4.93,0,0.428,6.481,18.5,6.1899,6,300,16.6,379.41,6.36,23.7 -0.09252,30,4.93,0,0.428,6.606,42.2,6.1899,6,300,16.6,383.78,7.37,23.3 -0.11329,30,4.93,0,0.428,6.897,54.3,6.3361,6,300,16.6,391.25,11.38,22 -0.10612,30,4.93,0,0.428,6.095,65.1,6.3361,6,300,16.6,394.62,12.4,20.1 -0.1029,30,4.93,0,0.428,6.358,52.9,7.0355,6,300,16.6,372.75,11.22,22.2 -0.12757,30,4.93,0,0.428,6.393,7.8,7.0355,6,300,16.6,374.71,5.19,23.7 -0.20608,22,5.86,0,0.431,5.593,76.5,7.9549,7,330,19.1,372.49,12.5,17.6 -0.19133,22,5.86,0,0.431,5.605,70.2,7.9549,7,330,19.1,389.13,18.46,18.5 -0.33983,22,5.86,0,0.431,6.108,34.9,8.0555,7,330,19.1,390.18,9.16,24.3 -0.19657,22,5.86,0,0.431,6.226,79.2,8.0555,7,330,19.1,376.14,10.15,20.5 -0.16439,22,5.86,0,0.431,6.433,49.1,7.8265,7,330,19.1,374.71,9.52,24.5 -0.19073,22,5.86,0,0.431,6.718,17.5,7.8265,7,330,19.1,393.74,6.56,26.2 -0.1403,22,5.86,0,0.431,6.487,13,7.3967,7,330,19.1,396.28,5.9,24.4 -0.21409,22,5.86,0,0.431,6.438,8.9,7.3967,7,330,19.1,377.07,3.59,24.8 -0.08221,22,5.86,0,0.431,6.957,6.8,8.9067,7,330,19.1,386.09,3.53,29.6 -0.36894,22,5.86,0,0.431,8.259,8.4,8.9067,7,330,19.1,396.9,3.54,42.8 -0.04819,80,3.64,0,0.392,6.108,32,9.2203,1,315,16.4,392.89,6.57,21.9 -0.03548,80,3.64,0,0.392,5.876,19.1,9.2203,1,315,16.4,395.18,9.25,20.9 -0.01538,90,3.75,0,0.394,7.454,34.2,6.3361,3,244,15.9,386.34,3.11,44 -0.61154,20,3.97,0,0.647,8.704,86.9,1.801,5,264,13,389.7,5.12,50 -0.66351,20,3.97,0,0.647,7.333,100,1.8946,5,264,13,383.29,7.79,36 -0.65665,20,3.97,0,0.647,6.842,100,2.0107,5,264,13,391.93,6.9,30.1 -0.54011,20,3.97,0,0.647,7.203,81.8,2.1121,5,264,13,392.8,9.59,33.8 -0.53412,20,3.97,0,0.647,7.52,89.4,2.1398,5,264,13,388.37,7.26,43.1 -0.52014,20,3.97,0,0.647,8.398,91.5,2.2885,5,264,13,386.86,5.91,48.8 -0.82526,20,3.97,0,0.647,7.327,94.5,2.0788,5,264,13,393.42,11.25,31 -0.55007,20,3.97,0,0.647,7.206,91.6,1.9301,5,264,13,387.89,8.1,36.5 -0.76162,20,3.97,0,0.647,5.56,62.8,1.9865,5,264,13,392.4,10.45,22.8 -0.7857,20,3.97,0,0.647,7.014,84.6,2.1329,5,264,13,384.07,14.79,30.7 -0.57834,20,3.97,0,0.575,8.297,67,2.4216,5,264,13,384.54,7.44,50 -0.5405,20,3.97,0,0.575,7.47,52.6,2.872,5,264,13,390.3,3.16,43.5 -0.09065,20,6.96,1,0.464,5.92,61.5,3.9175,3,223,18.6,391.34,13.65,20.7 -0.29916,20,6.96,0,0.464,5.856,42.1,4.429,3,223,18.6,388.65,13,21.1 -0.16211,20,6.96,0,0.464,6.24,16.3,4.429,3,223,18.6,396.9,6.59,25.2 -0.1146,20,6.96,0,0.464,6.538,58.7,3.9175,3,223,18.6,394.96,7.73,24.4 -0.22188,20,6.96,1,0.464,7.691,51.8,4.3665,3,223,18.6,390.77,6.58,35.2 -0.05644,40,6.41,1,0.447,6.758,32.9,4.0776,4,254,17.6,396.9,3.53,32.4 -0.09604,40,6.41,0,0.447,6.854,42.8,4.2673,4,254,17.6,396.9,2.98,32 -0.10469,40,6.41,1,0.447,7.267,49,4.7872,4,254,17.6,389.25,6.05,33.2 -0.06127,40,6.41,1,0.447,6.826,27.6,4.8628,4,254,17.6,393.45,4.16,33.1 -0.07978,40,6.41,0,0.447,6.482,32.1,4.1403,4,254,17.6,396.9,7.19,29.1 -0.21038,20,3.33,0,0.4429,6.812,32.2,4.1007,5,216,14.9,396.9,4.85,35.1 -0.03578,20,3.33,0,0.4429,7.82,64.5,4.6947,5,216,14.9,387.31,3.76,45.4 -0.03705,20,3.33,0,0.4429,6.968,37.2,5.2447,5,216,14.9,392.23,4.59,35.4 -0.06129,20,3.33,1,0.4429,7.645,49.7,5.2119,5,216,14.9,377.07,3.01,46 -0.01501,90,1.21,1,0.401,7.923,24.8,5.885,1,198,13.6,395.52,3.16,50 -0.00906,90,2.97,0,0.4,7.088,20.8,7.3073,1,285,15.3,394.72,7.85,32.2 -0.01096,55,2.25,0,0.389,6.453,31.9,7.3073,1,300,15.3,394.72,8.23,22 -0.01965,80,1.76,0,0.385,6.23,31.5,9.0892,1,241,18.2,341.6,12.93,20.1 -0.03871,52.5,5.32,0,0.405,6.209,31.3,7.3172,6,293,16.6,396.9,7.14,23.2 -0.0459,52.5,5.32,0,0.405,6.315,45.6,7.3172,6,293,16.6,396.9,7.6,22.3 -0.04297,52.5,5.32,0,0.405,6.565,22.9,7.3172,6,293,16.6,371.72,9.51,24.8 -0.03502,80,4.95,0,0.411,6.861,27.9,5.1167,4,245,19.2,396.9,3.33,28.5 -0.07886,80,4.95,0,0.411,7.148,27.7,5.1167,4,245,19.2,396.9,3.56,37.3 -0.03615,80,4.95,0,0.411,6.63,23.4,5.1167,4,245,19.2,396.9,4.7,27.9 -0.08265,0,13.92,0,0.437,6.127,18.4,5.5027,4,289,16,396.9,8.58,23.9 -0.08199,0,13.92,0,0.437,6.009,42.3,5.5027,4,289,16,396.9,10.4,21.7 -0.12932,0,13.92,0,0.437,6.678,31.1,5.9604,4,289,16,396.9,6.27,28.6 -0.05372,0,13.92,0,0.437,6.549,51,5.9604,4,289,16,392.85,7.39,27.1 -0.14103,0,13.92,0,0.437,5.79,58,6.32,4,289,16,396.9,15.84,20.3 -0.06466,70,2.24,0,0.4,6.345,20.1,7.8278,5,358,14.8,368.24,4.97,22.5 -0.05561,70,2.24,0,0.4,7.041,10,7.8278,5,358,14.8,371.58,4.74,29 -0.04417,70,2.24,0,0.4,6.871,47.4,7.8278,5,358,14.8,390.86,6.07,24.8 -0.03537,34,6.09,0,0.433,6.59,40.4,5.4917,7,329,16.1,395.75,9.5,22 -0.09266,34,6.09,0,0.433,6.495,18.4,5.4917,7,329,16.1,383.61,8.67,26.4 -0.1,34,6.09,0,0.433,6.982,17.7,5.4917,7,329,16.1,390.43,4.86,33.1 -0.05515,33,2.18,0,0.472,7.236,41.1,4.022,7,222,18.4,393.68,6.93,36.1 -0.05479,33,2.18,0,0.472,6.616,58.1,3.37,7,222,18.4,393.36,8.93,28.4 -0.07503,33,2.18,0,0.472,7.42,71.9,3.0992,7,222,18.4,396.9,6.47,33.4 -0.04932,33,2.18,0,0.472,6.849,70.3,3.1827,7,222,18.4,396.9,7.53,28.2 -0.49298,0,9.9,0,0.544,6.635,82.5,3.3175,4,304,18.4,396.9,4.54,22.8 -0.3494,0,9.9,0,0.544,5.972,76.7,3.1025,4,304,18.4,396.24,9.97,20.3 -2.63548,0,9.9,0,0.544,4.973,37.8,2.5194,4,304,18.4,350.45,12.64,16.1 -0.79041,0,9.9,0,0.544,6.122,52.8,2.6403,4,304,18.4,396.9,5.98,22.1 -0.26169,0,9.9,0,0.544,6.023,90.4,2.834,4,304,18.4,396.3,11.72,19.4 -0.26938,0,9.9,0,0.544,6.266,82.8,3.2628,4,304,18.4,393.39,7.9,21.6 -0.3692,0,9.9,0,0.544,6.567,87.3,3.6023,4,304,18.4,395.69,9.28,23.8 -0.25356,0,9.9,0,0.544,5.705,77.7,3.945,4,304,18.4,396.42,11.5,16.2 -0.31827,0,9.9,0,0.544,5.914,83.2,3.9986,4,304,18.4,390.7,18.33,17.8 -0.24522,0,9.9,0,0.544,5.782,71.7,4.0317,4,304,18.4,396.9,15.94,19.8 -0.40202,0,9.9,0,0.544,6.382,67.2,3.5325,4,304,18.4,395.21,10.36,23.1 -0.47547,0,9.9,0,0.544,6.113,58.8,4.0019,4,304,18.4,396.23,12.73,21 -0.1676,0,7.38,0,0.493,6.426,52.3,4.5404,5,287,19.6,396.9,7.2,23.8 -0.18159,0,7.38,0,0.493,6.376,54.3,4.5404,5,287,19.6,396.9,6.87,23.1 -0.35114,0,7.38,0,0.493,6.041,49.9,4.7211,5,287,19.6,396.9,7.7,20.4 -0.28392,0,7.38,0,0.493,5.708,74.3,4.7211,5,287,19.6,391.13,11.74,18.5 -0.34109,0,7.38,0,0.493,6.415,40.1,4.7211,5,287,19.6,396.9,6.12,25 -0.19186,0,7.38,0,0.493,6.431,14.7,5.4159,5,287,19.6,393.68,5.08,24.6 -0.30347,0,7.38,0,0.493,6.312,28.9,5.4159,5,287,19.6,396.9,6.15,23 -0.24103,0,7.38,0,0.493,6.083,43.7,5.4159,5,287,19.6,396.9,12.79,22.2 -0.06617,0,3.24,0,0.46,5.868,25.8,5.2146,4,430,16.9,382.44,9.97,19.3 -0.06724,0,3.24,0,0.46,6.333,17.2,5.2146,4,430,16.9,375.21,7.34,22.6 -0.04544,0,3.24,0,0.46,6.144,32.2,5.8736,4,430,16.9,368.57,9.09,19.8 -0.05023,35,6.06,0,0.4379,5.706,28.4,6.6407,1,304,16.9,394.02,12.43,17.1 -0.03466,35,6.06,0,0.4379,6.031,23.3,6.6407,1,304,16.9,362.25,7.83,19.4 -0.05083,0,5.19,0,0.515,6.316,38.1,6.4584,5,224,20.2,389.71,5.68,22.2 -0.03738,0,5.19,0,0.515,6.31,38.5,6.4584,5,224,20.2,389.4,6.75,20.7 -0.03961,0,5.19,0,0.515,6.037,34.5,5.9853,5,224,20.2,396.9,8.01,21.1 -0.03427,0,5.19,0,0.515,5.869,46.3,5.2311,5,224,20.2,396.9,9.8,19.5 -0.03041,0,5.19,0,0.515,5.895,59.6,5.615,5,224,20.2,394.81,10.56,18.5 -0.03306,0,5.19,0,0.515,6.059,37.3,4.8122,5,224,20.2,396.14,8.51,20.6 -0.05497,0,5.19,0,0.515,5.985,45.4,4.8122,5,224,20.2,396.9,9.74,19 -0.06151,0,5.19,0,0.515,5.968,58.5,4.8122,5,224,20.2,396.9,9.29,18.7 -0.01301,35,1.52,0,0.442,7.241,49.3,7.0379,1,284,15.5,394.74,5.49,32.7 -0.02498,0,1.89,0,0.518,6.54,59.7,6.2669,1,422,15.9,389.96,8.65,16.5 -0.02543,55,3.78,0,0.484,6.696,56.4,5.7321,5,370,17.6,396.9,7.18,23.9 -0.03049,55,3.78,0,0.484,6.874,28.1,6.4654,5,370,17.6,387.97,4.61,31.2 -0.03113,0,4.39,0,0.442,6.014,48.5,8.0136,3,352,18.8,385.64,10.53,17.5 -0.06162,0,4.39,0,0.442,5.898,52.3,8.0136,3,352,18.8,364.61,12.67,17.2 -0.0187,85,4.15,0,0.429,6.516,27.7,8.5353,4,351,17.9,392.43,6.36,23.1 -0.01501,80,2.01,0,0.435,6.635,29.7,8.344,4,280,17,390.94,5.99,24.5 -0.02899,40,1.25,0,0.429,6.939,34.5,8.7921,1,335,19.7,389.85,5.89,26.6 -0.06211,40,1.25,0,0.429,6.49,44.4,8.7921,1,335,19.7,396.9,5.98,22.9 -0.0795,60,1.69,0,0.411,6.579,35.9,10.7103,4,411,18.3,370.78,5.49,24.1 -0.07244,60,1.69,0,0.411,5.884,18.5,10.7103,4,411,18.3,392.33,7.79,18.6 -0.01709,90,2.02,0,0.41,6.728,36.1,12.1265,5,187,17,384.46,4.5,30.1 -0.04301,80,1.91,0,0.413,5.663,21.9,10.5857,4,334,22,382.8,8.05,18.2 -0.10659,80,1.91,0,0.413,5.936,19.5,10.5857,4,334,22,376.04,5.57,20.6 -8.98296,0,18.1,1,0.77,6.212,97.4,2.1222,24,666,20.2,377.73,17.6,17.8 -3.8497,0,18.1,1,0.77,6.395,91,2.5052,24,666,20.2,391.34,13.27,21.7 -5.20177,0,18.1,1,0.77,6.127,83.4,2.7227,24,666,20.2,395.43,11.48,22.7 -4.26131,0,18.1,0,0.77,6.112,81.3,2.5091,24,666,20.2,390.74,12.67,22.6 -4.54192,0,18.1,0,0.77,6.398,88,2.5182,24,666,20.2,374.56,7.79,25 -3.83684,0,18.1,0,0.77,6.251,91.1,2.2955,24,666,20.2,350.65,14.19,19.9 -3.67822,0,18.1,0,0.77,5.362,96.2,2.1036,24,666,20.2,380.79,10.19,20.8 -4.22239,0,18.1,1,0.77,5.803,89,1.9047,24,666,20.2,353.04,14.64,16.8 -3.47428,0,18.1,1,0.718,8.78,82.9,1.9047,24,666,20.2,354.55,5.29,21.9 -4.55587,0,18.1,0,0.718,3.561,87.9,1.6132,24,666,20.2,354.7,7.12,27.5 -3.69695,0,18.1,0,0.718,4.963,91.4,1.7523,24,666,20.2,316.03,14,21.9 -13.5222,0,18.1,0,0.631,3.863,100,1.5106,24,666,20.2,131.42,13.33,23.1 -4.89822,0,18.1,0,0.631,4.97,100,1.3325,24,666,20.2,375.52,3.26,50 -5.66998,0,18.1,1,0.631,6.683,96.8,1.3567,24,666,20.2,375.33,3.73,50 -6.53876,0,18.1,1,0.631,7.016,97.5,1.2024,24,666,20.2,392.05,2.96,50 -9.2323,0,18.1,0,0.631,6.216,100,1.1691,24,666,20.2,366.15,9.53,50 -8.26725,0,18.1,1,0.668,5.875,89.6,1.1296,24,666,20.2,347.88,8.88,50 -11.1081,0,18.1,0,0.668,4.906,100,1.1742,24,666,20.2,396.9,34.77,13.8 -18.4982,0,18.1,0,0.668,4.138,100,1.137,24,666,20.2,396.9,37.97,13.8 -19.6091,0,18.1,0,0.671,7.313,97.9,1.3163,24,666,20.2,396.9,13.44,15 -15.288,0,18.1,0,0.671,6.649,93.3,1.3449,24,666,20.2,363.02,23.24,13.9 -9.82349,0,18.1,0,0.671,6.794,98.8,1.358,24,666,20.2,396.9,21.24,13.3 -23.6482,0,18.1,0,0.671,6.38,96.2,1.3861,24,666,20.2,396.9,23.69,13.1 -17.8667,0,18.1,0,0.671,6.223,100,1.3861,24,666,20.2,393.74,21.78,10.2 -88.9762,0,18.1,0,0.671,6.968,91.9,1.4165,24,666,20.2,396.9,17.21,10.4 -15.8744,0,18.1,0,0.671,6.545,99.1,1.5192,24,666,20.2,396.9,21.08,10.9 -9.18702,0,18.1,0,0.7,5.536,100,1.5804,24,666,20.2,396.9,23.6,11.3 -7.99248,0,18.1,0,0.7,5.52,100,1.5331,24,666,20.2,396.9,24.56,12.3 -20.0849,0,18.1,0,0.7,4.368,91.2,1.4395,24,666,20.2,285.83,30.63,8.8 -16.8118,0,18.1,0,0.7,5.277,98.1,1.4261,24,666,20.2,396.9,30.81,7.2 -24.3938,0,18.1,0,0.7,4.652,100,1.4672,24,666,20.2,396.9,28.28,10.5 -22.5971,0,18.1,0,0.7,5,89.5,1.5184,24,666,20.2,396.9,31.99,7.4 -14.3337,0,18.1,0,0.7,4.88,100,1.5895,24,666,20.2,372.92,30.62,10.2 -8.15174,0,18.1,0,0.7,5.39,98.9,1.7281,24,666,20.2,396.9,20.85,11.5 -6.96215,0,18.1,0,0.7,5.713,97,1.9265,24,666,20.2,394.43,17.11,15.1 -5.29305,0,18.1,0,0.7,6.051,82.5,2.1678,24,666,20.2,378.38,18.76,23.2 -11.5779,0,18.1,0,0.7,5.036,97,1.77,24,666,20.2,396.9,25.68,9.7 -8.64476,0,18.1,0,0.693,6.193,92.6,1.7912,24,666,20.2,396.9,15.17,13.8 -13.3598,0,18.1,0,0.693,5.887,94.7,1.7821,24,666,20.2,396.9,16.35,12.7 -8.71675,0,18.1,0,0.693,6.471,98.8,1.7257,24,666,20.2,391.98,17.12,13.1 -5.87205,0,18.1,0,0.693,6.405,96,1.6768,24,666,20.2,396.9,19.37,12.5 -7.67202,0,18.1,0,0.693,5.747,98.9,1.6334,24,666,20.2,393.1,19.92,8.5 -38.3518,0,18.1,0,0.693,5.453,100,1.4896,24,666,20.2,396.9,30.59,5 -9.91655,0,18.1,0,0.693,5.852,77.8,1.5004,24,666,20.2,338.16,29.97,6.3 -25.0461,0,18.1,0,0.693,5.987,100,1.5888,24,666,20.2,396.9,26.77,5.6 -14.2362,0,18.1,0,0.693,6.343,100,1.5741,24,666,20.2,396.9,20.32,7.2 -9.59571,0,18.1,0,0.693,6.404,100,1.639,24,666,20.2,376.11,20.31,12.1 -24.8017,0,18.1,0,0.693,5.349,96,1.7028,24,666,20.2,396.9,19.77,8.3 -41.5292,0,18.1,0,0.693,5.531,85.4,1.6074,24,666,20.2,329.46,27.38,8.5 -67.9208,0,18.1,0,0.693,5.683,100,1.4254,24,666,20.2,384.97,22.98,5 -20.7162,0,18.1,0,0.659,4.138,100,1.1781,24,666,20.2,370.22,23.34,11.9 -11.9511,0,18.1,0,0.659,5.608,100,1.2852,24,666,20.2,332.09,12.13,27.9 -7.40389,0,18.1,0,0.597,5.617,97.9,1.4547,24,666,20.2,314.64,26.4,17.2 -14.4383,0,18.1,0,0.597,6.852,100,1.4655,24,666,20.2,179.36,19.78,27.5 -51.1358,0,18.1,0,0.597,5.757,100,1.413,24,666,20.2,2.6,10.11,15 -14.0507,0,18.1,0,0.597,6.657,100,1.5275,24,666,20.2,35.05,21.22,17.2 -18.811,0,18.1,0,0.597,4.628,100,1.5539,24,666,20.2,28.79,34.37,17.9 -28.6558,0,18.1,0,0.597,5.155,100,1.5894,24,666,20.2,210.97,20.08,16.3 -45.7461,0,18.1,0,0.693,4.519,100,1.6582,24,666,20.2,88.27,36.98,7 -18.0846,0,18.1,0,0.679,6.434,100,1.8347,24,666,20.2,27.25,29.05,7.2 -10.8342,0,18.1,0,0.679,6.782,90.8,1.8195,24,666,20.2,21.57,25.79,7.5 -25.9406,0,18.1,0,0.679,5.304,89.1,1.6475,24,666,20.2,127.36,26.64,10.4 -73.5341,0,18.1,0,0.679,5.957,100,1.8026,24,666,20.2,16.45,20.62,8.8 -11.8123,0,18.1,0,0.718,6.824,76.5,1.794,24,666,20.2,48.45,22.74,8.4 -11.0874,0,18.1,0,0.718,6.411,100,1.8589,24,666,20.2,318.75,15.02,16.7 -7.02259,0,18.1,0,0.718,6.006,95.3,1.8746,24,666,20.2,319.98,15.7,14.2 -12.0482,0,18.1,0,0.614,5.648,87.6,1.9512,24,666,20.2,291.55,14.1,20.8 -7.05042,0,18.1,0,0.614,6.103,85.1,2.0218,24,666,20.2,2.52,23.29,13.4 -8.79212,0,18.1,0,0.584,5.565,70.6,2.0635,24,666,20.2,3.65,17.16,11.7 -15.8603,0,18.1,0,0.679,5.896,95.4,1.9096,24,666,20.2,7.68,24.39,8.3 -12.2472,0,18.1,0,0.584,5.837,59.7,1.9976,24,666,20.2,24.65,15.69,10.2 -37.6619,0,18.1,0,0.679,6.202,78.7,1.8629,24,666,20.2,18.82,14.52,10.9 -7.36711,0,18.1,0,0.679,6.193,78.1,1.9356,24,666,20.2,96.73,21.52,11 -9.33889,0,18.1,0,0.679,6.38,95.6,1.9682,24,666,20.2,60.72,24.08,9.5 -8.49213,0,18.1,0,0.584,6.348,86.1,2.0527,24,666,20.2,83.45,17.64,14.5 -10.0623,0,18.1,0,0.584,6.833,94.3,2.0882,24,666,20.2,81.33,19.69,14.1 -6.44405,0,18.1,0,0.584,6.425,74.8,2.2004,24,666,20.2,97.95,12.03,16.1 -5.58107,0,18.1,0,0.713,6.436,87.9,2.3158,24,666,20.2,100.19,16.22,14.3 -13.9134,0,18.1,0,0.713,6.208,95,2.2222,24,666,20.2,100.63,15.17,11.7 -11.1604,0,18.1,0,0.74,6.629,94.6,2.1247,24,666,20.2,109.85,23.27,13.4 -14.4208,0,18.1,0,0.74,6.461,93.3,2.0026,24,666,20.2,27.49,18.05,9.6 -15.1772,0,18.1,0,0.74,6.152,100,1.9142,24,666,20.2,9.32,26.45,8.7 -13.6781,0,18.1,0,0.74,5.935,87.9,1.8206,24,666,20.2,68.95,34.02,8.4 -9.39063,0,18.1,0,0.74,5.627,93.9,1.8172,24,666,20.2,396.9,22.88,12.8 -22.0511,0,18.1,0,0.74,5.818,92.4,1.8662,24,666,20.2,391.45,22.11,10.5 -9.72418,0,18.1,0,0.74,6.406,97.2,2.0651,24,666,20.2,385.96,19.52,17.1 -5.66637,0,18.1,0,0.74,6.219,100,2.0048,24,666,20.2,395.69,16.59,18.4 -9.96654,0,18.1,0,0.74,6.485,100,1.9784,24,666,20.2,386.73,18.85,15.4 -12.8023,0,18.1,0,0.74,5.854,96.6,1.8956,24,666,20.2,240.52,23.79,10.8 -0.6718,0,18.1,0,0.74,6.459,94.8,1.9879,24,666,20.2,43.06,23.98,11.8 -6.28807,0,18.1,0,0.74,6.341,96.4,2.072,24,666,20.2,318.01,17.79,14.9 -9.92485,0,18.1,0,0.74,6.251,96.6,2.198,24,666,20.2,388.52,16.44,12.6 -9.32909,0,18.1,0,0.713,6.185,98.7,2.2616,24,666,20.2,396.9,18.13,14.1 -7.52601,0,18.1,0,0.713,6.417,98.3,2.185,24,666,20.2,304.21,19.31,13 -6.71772,0,18.1,0,0.713,6.749,92.6,2.3236,24,666,20.2,0.32,17.44,13.4 -5.44114,0,18.1,0,0.713,6.655,98.2,2.3552,24,666,20.2,355.29,17.73,15.2 -5.09017,0,18.1,0,0.713,6.297,91.8,2.3682,24,666,20.2,385.09,17.27,16.1 -8.24809,0,18.1,0,0.713,7.393,99.3,2.4527,24,666,20.2,375.87,16.74,17.8 -9.51363,0,18.1,0,0.713,6.728,94.1,2.4961,24,666,20.2,6.68,18.71,14.9 -4.75237,0,18.1,0,0.713,6.525,86.5,2.4358,24,666,20.2,50.92,18.13,14.1 -4.66883,0,18.1,0,0.713,5.976,87.9,2.5806,24,666,20.2,10.48,19.01,12.7 -8.20058,0,18.1,0,0.713,5.936,80.3,2.7792,24,666,20.2,3.5,16.94,13.5 -7.75223,0,18.1,0,0.713,6.301,83.7,2.7831,24,666,20.2,272.21,16.23,14.9 -6.80117,0,18.1,0,0.713,6.081,84.4,2.7175,24,666,20.2,396.9,14.7,20 -4.81213,0,18.1,0,0.713,6.701,90,2.5975,24,666,20.2,255.23,16.42,16.4 -3.69311,0,18.1,0,0.713,6.376,88.4,2.5671,24,666,20.2,391.43,14.65,17.7 -6.65492,0,18.1,0,0.713,6.317,83,2.7344,24,666,20.2,396.9,13.99,19.5 -5.82115,0,18.1,0,0.713,6.513,89.9,2.8016,24,666,20.2,393.82,10.29,20.2 -7.83932,0,18.1,0,0.655,6.209,65.4,2.9634,24,666,20.2,396.9,13.22,21.4 -3.1636,0,18.1,0,0.655,5.759,48.2,3.0665,24,666,20.2,334.4,14.13,19.9 -3.77498,0,18.1,0,0.655,5.952,84.7,2.8715,24,666,20.2,22.01,17.15,19 -4.42228,0,18.1,0,0.584,6.003,94.5,2.5403,24,666,20.2,331.29,21.32,19.1 -15.5757,0,18.1,0,0.58,5.926,71,2.9084,24,666,20.2,368.74,18.13,19.1 -13.0751,0,18.1,0,0.58,5.713,56.7,2.8237,24,666,20.2,396.9,14.76,20.1 -4.34879,0,18.1,0,0.58,6.167,84,3.0334,24,666,20.2,396.9,16.29,19.9 -4.03841,0,18.1,0,0.532,6.229,90.7,3.0993,24,666,20.2,395.33,12.87,19.6 -3.56868,0,18.1,0,0.58,6.437,75,2.8965,24,666,20.2,393.37,14.36,23.2 -4.64689,0,18.1,0,0.614,6.98,67.6,2.5329,24,666,20.2,374.68,11.66,29.8 -8.05579,0,18.1,0,0.584,5.427,95.4,2.4298,24,666,20.2,352.58,18.14,13.8 -6.39312,0,18.1,0,0.584,6.162,97.4,2.206,24,666,20.2,302.76,24.1,13.3 -4.87141,0,18.1,0,0.614,6.484,93.6,2.3053,24,666,20.2,396.21,18.68,16.7 -15.0234,0,18.1,0,0.614,5.304,97.3,2.1007,24,666,20.2,349.48,24.91,12 -10.233,0,18.1,0,0.614,6.185,96.7,2.1705,24,666,20.2,379.7,18.03,14.6 -14.3337,0,18.1,0,0.614,6.229,88,1.9512,24,666,20.2,383.32,13.11,21.4 -5.82401,0,18.1,0,0.532,6.242,64.7,3.4242,24,666,20.2,396.9,10.74,23 -5.70818,0,18.1,0,0.532,6.75,74.9,3.3317,24,666,20.2,393.07,7.74,23.7 -5.73116,0,18.1,0,0.532,7.061,77,3.4106,24,666,20.2,395.28,7.01,25 -2.81838,0,18.1,0,0.532,5.762,40.3,4.0983,24,666,20.2,392.92,10.42,21.8 -2.37857,0,18.1,0,0.583,5.871,41.9,3.724,24,666,20.2,370.73,13.34,20.6 -3.67367,0,18.1,0,0.583,6.312,51.9,3.9917,24,666,20.2,388.62,10.58,21.2 -5.69175,0,18.1,0,0.583,6.114,79.8,3.5459,24,666,20.2,392.68,14.98,19.1 -4.83567,0,18.1,0,0.583,5.905,53.2,3.1523,24,666,20.2,388.22,11.45,20.6 -0.15086,0,27.74,0,0.609,5.454,92.7,1.8209,4,711,20.1,395.09,18.06,15.2 -0.18337,0,27.74,0,0.609,5.414,98.3,1.7554,4,711,20.1,344.05,23.97,7 -0.20746,0,27.74,0,0.609,5.093,98,1.8226,4,711,20.1,318.43,29.68,8.1 -0.10574,0,27.74,0,0.609,5.983,98.8,1.8681,4,711,20.1,390.11,18.07,13.6 -0.11132,0,27.74,0,0.609,5.983,83.5,2.1099,4,711,20.1,396.9,13.35,20.1 -0.17331,0,9.69,0,0.585,5.707,54,2.3817,6,391,19.2,396.9,12.01,21.8 -0.27957,0,9.69,0,0.585,5.926,42.6,2.3817,6,391,19.2,396.9,13.59,24.5 -0.17899,0,9.69,0,0.585,5.67,28.8,2.7986,6,391,19.2,393.29,17.6,23.1 -0.2896,0,9.69,0,0.585,5.39,72.9,2.7986,6,391,19.2,396.9,21.14,19.7 -0.26838,0,9.69,0,0.585,5.794,70.6,2.8927,6,391,19.2,396.9,14.1,18.3 -0.23912,0,9.69,0,0.585,6.019,65.3,2.4091,6,391,19.2,396.9,12.92,21.2 -0.17783,0,9.69,0,0.585,5.569,73.5,2.3999,6,391,19.2,395.77,15.1,17.5 -0.22438,0,9.69,0,0.585,6.027,79.7,2.4982,6,391,19.2,396.9,14.33,16.8 -0.06263,0,11.93,0,0.573,6.593,69.1,2.4786,1,273,21,391.99,9.67,22.4 -0.04527,0,11.93,0,0.573,6.12,76.7,2.2875,1,273,21,396.9,9.08,20.6 -0.06076,0,11.93,0,0.573,6.976,91,2.1675,1,273,21,396.9,5.64,23.9 -0.10959,0,11.93,0,0.573,6.794,89.3,2.3889,1,273,21,393.45,6.48,22 -0.04741,0,11.93,0,0.573,6.03,80.8,2.505,1,273,21,396.9,7.88,11.9 diff --git a/tensorflow/contrib/learn/python/learn/datasets/data/iris.csv b/tensorflow/contrib/learn/python/learn/datasets/data/iris.csv deleted file mode 100644 index 93fca4db848..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/data/iris.csv +++ /dev/null @@ -1,151 +0,0 @@ -150,4,setosa,versicolor,virginica -5.1,3.5,1.4,0.2,0 -4.9,3.0,1.4,0.2,0 -4.7,3.2,1.3,0.2,0 -4.6,3.1,1.5,0.2,0 -5.0,3.6,1.4,0.2,0 -5.4,3.9,1.7,0.4,0 -4.6,3.4,1.4,0.3,0 -5.0,3.4,1.5,0.2,0 -4.4,2.9,1.4,0.2,0 -4.9,3.1,1.5,0.1,0 -5.4,3.7,1.5,0.2,0 -4.8,3.4,1.6,0.2,0 -4.8,3.0,1.4,0.1,0 -4.3,3.0,1.1,0.1,0 -5.8,4.0,1.2,0.2,0 -5.7,4.4,1.5,0.4,0 -5.4,3.9,1.3,0.4,0 -5.1,3.5,1.4,0.3,0 -5.7,3.8,1.7,0.3,0 -5.1,3.8,1.5,0.3,0 -5.4,3.4,1.7,0.2,0 -5.1,3.7,1.5,0.4,0 -4.6,3.6,1.0,0.2,0 -5.1,3.3,1.7,0.5,0 -4.8,3.4,1.9,0.2,0 -5.0,3.0,1.6,0.2,0 -5.0,3.4,1.6,0.4,0 -5.2,3.5,1.5,0.2,0 -5.2,3.4,1.4,0.2,0 -4.7,3.2,1.6,0.2,0 -4.8,3.1,1.6,0.2,0 -5.4,3.4,1.5,0.4,0 -5.2,4.1,1.5,0.1,0 -5.5,4.2,1.4,0.2,0 -4.9,3.1,1.5,0.1,0 -5.0,3.2,1.2,0.2,0 -5.5,3.5,1.3,0.2,0 -4.9,3.1,1.5,0.1,0 -4.4,3.0,1.3,0.2,0 -5.1,3.4,1.5,0.2,0 -5.0,3.5,1.3,0.3,0 -4.5,2.3,1.3,0.3,0 -4.4,3.2,1.3,0.2,0 -5.0,3.5,1.6,0.6,0 -5.1,3.8,1.9,0.4,0 -4.8,3.0,1.4,0.3,0 -5.1,3.8,1.6,0.2,0 -4.6,3.2,1.4,0.2,0 -5.3,3.7,1.5,0.2,0 -5.0,3.3,1.4,0.2,0 -7.0,3.2,4.7,1.4,1 -6.4,3.2,4.5,1.5,1 -6.9,3.1,4.9,1.5,1 -5.5,2.3,4.0,1.3,1 -6.5,2.8,4.6,1.5,1 -5.7,2.8,4.5,1.3,1 -6.3,3.3,4.7,1.6,1 -4.9,2.4,3.3,1.0,1 -6.6,2.9,4.6,1.3,1 -5.2,2.7,3.9,1.4,1 -5.0,2.0,3.5,1.0,1 -5.9,3.0,4.2,1.5,1 -6.0,2.2,4.0,1.0,1 -6.1,2.9,4.7,1.4,1 -5.6,2.9,3.6,1.3,1 -6.7,3.1,4.4,1.4,1 -5.6,3.0,4.5,1.5,1 -5.8,2.7,4.1,1.0,1 -6.2,2.2,4.5,1.5,1 -5.6,2.5,3.9,1.1,1 -5.9,3.2,4.8,1.8,1 -6.1,2.8,4.0,1.3,1 -6.3,2.5,4.9,1.5,1 -6.1,2.8,4.7,1.2,1 -6.4,2.9,4.3,1.3,1 -6.6,3.0,4.4,1.4,1 -6.8,2.8,4.8,1.4,1 -6.7,3.0,5.0,1.7,1 -6.0,2.9,4.5,1.5,1 -5.7,2.6,3.5,1.0,1 -5.5,2.4,3.8,1.1,1 -5.5,2.4,3.7,1.0,1 -5.8,2.7,3.9,1.2,1 -6.0,2.7,5.1,1.6,1 -5.4,3.0,4.5,1.5,1 -6.0,3.4,4.5,1.6,1 -6.7,3.1,4.7,1.5,1 -6.3,2.3,4.4,1.3,1 -5.6,3.0,4.1,1.3,1 -5.5,2.5,4.0,1.3,1 -5.5,2.6,4.4,1.2,1 -6.1,3.0,4.6,1.4,1 -5.8,2.6,4.0,1.2,1 -5.0,2.3,3.3,1.0,1 -5.6,2.7,4.2,1.3,1 -5.7,3.0,4.2,1.2,1 -5.7,2.9,4.2,1.3,1 -6.2,2.9,4.3,1.3,1 -5.1,2.5,3.0,1.1,1 -5.7,2.8,4.1,1.3,1 -6.3,3.3,6.0,2.5,2 -5.8,2.7,5.1,1.9,2 -7.1,3.0,5.9,2.1,2 -6.3,2.9,5.6,1.8,2 -6.5,3.0,5.8,2.2,2 -7.6,3.0,6.6,2.1,2 -4.9,2.5,4.5,1.7,2 -7.3,2.9,6.3,1.8,2 -6.7,2.5,5.8,1.8,2 -7.2,3.6,6.1,2.5,2 -6.5,3.2,5.1,2.0,2 -6.4,2.7,5.3,1.9,2 -6.8,3.0,5.5,2.1,2 -5.7,2.5,5.0,2.0,2 -5.8,2.8,5.1,2.4,2 -6.4,3.2,5.3,2.3,2 -6.5,3.0,5.5,1.8,2 -7.7,3.8,6.7,2.2,2 -7.7,2.6,6.9,2.3,2 -6.0,2.2,5.0,1.5,2 -6.9,3.2,5.7,2.3,2 -5.6,2.8,4.9,2.0,2 -7.7,2.8,6.7,2.0,2 -6.3,2.7,4.9,1.8,2 -6.7,3.3,5.7,2.1,2 -7.2,3.2,6.0,1.8,2 -6.2,2.8,4.8,1.8,2 -6.1,3.0,4.9,1.8,2 -6.4,2.8,5.6,2.1,2 -7.2,3.0,5.8,1.6,2 -7.4,2.8,6.1,1.9,2 -7.9,3.8,6.4,2.0,2 -6.4,2.8,5.6,2.2,2 -6.3,2.8,5.1,1.5,2 -6.1,2.6,5.6,1.4,2 -7.7,3.0,6.1,2.3,2 -6.3,3.4,5.6,2.4,2 -6.4,3.1,5.5,1.8,2 -6.0,3.0,4.8,1.8,2 -6.9,3.1,5.4,2.1,2 -6.7,3.1,5.6,2.4,2 -6.9,3.1,5.1,2.3,2 -5.8,2.7,5.1,1.9,2 -6.8,3.2,5.9,2.3,2 -6.7,3.3,5.7,2.5,2 -6.7,3.0,5.2,2.3,2 -6.3,2.5,5.0,1.9,2 -6.5,3.0,5.2,2.0,2 -6.2,3.4,5.4,2.3,2 -5.9,3.0,5.1,1.8,2 diff --git a/tensorflow/contrib/learn/python/learn/datasets/data/text_test.csv b/tensorflow/contrib/learn/python/learn/datasets/data/text_test.csv deleted file mode 100644 index 9590637e0f1..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/data/text_test.csv +++ /dev/null @@ -1 +0,0 @@ -2,"Sergey Brin","Sergey Brin is the co-founder of Google." diff --git a/tensorflow/contrib/learn/python/learn/datasets/data/text_train.csv b/tensorflow/contrib/learn/python/learn/datasets/data/text_train.csv deleted file mode 100644 index 44e27afe7b6..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/data/text_train.csv +++ /dev/null @@ -1,3 +0,0 @@ -1,"Google","Google is a company." -2,"Larry Page","Larry Page is the co-founder of Google." -3,"Time","Time is a news magazine." diff --git a/tensorflow/contrib/learn/python/learn/datasets/load_csv_test.py b/tensorflow/contrib/learn/python/learn/datasets/load_csv_test.py deleted file mode 100644 index 9a62feac575..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/load_csv_test.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn import datasets -from tensorflow.python.platform import test - - -class LoadCsvTest(test.TestCase): - """Test load csv functions.""" - - def testIris(self): - iris = datasets.load_iris() - self.assertTupleEqual(iris.data.shape, (150, 4)) - self.assertTupleEqual(iris.target.shape, (150,)) - - def testBoston(self): - boston = datasets.load_boston() - self.assertTupleEqual(boston.data.shape, (506, 13)) - self.assertTupleEqual(boston.target.shape, (506,)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/datasets/mnist.py b/tensorflow/contrib/learn/python/learn/datasets/mnist.py deleted file mode 100644 index abbb44c2f5b..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/mnist.py +++ /dev/null @@ -1,300 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functions for downloading and reading MNIST data (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import gzip - -import numpy -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import random_seed -from tensorflow.python.platform import gfile -from tensorflow.python.util.deprecation import deprecated - -# CVDF mirror of http://yann.lecun.com/exdb/mnist/ -DEFAULT_SOURCE_URL = 'https://storage.googleapis.com/cvdf-datasets/mnist/' - - -def _read32(bytestream): - dt = numpy.dtype(numpy.uint32).newbyteorder('>') - return numpy.frombuffer(bytestream.read(4), dtype=dt)[0] - - -@deprecated(None, 'Please use tf.data to implement this functionality.') -def extract_images(f): - """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. - - Args: - f: A file object that can be passed into a gzip reader. - - Returns: - data: A 4D uint8 numpy array [index, y, x, depth]. - - Raises: - ValueError: If the bytestream does not start with 2051. - - """ - print('Extracting', f.name) - with gzip.GzipFile(fileobj=f) as bytestream: - magic = _read32(bytestream) - if magic != 2051: - raise ValueError('Invalid magic number %d in MNIST image file: %s' % - (magic, f.name)) - num_images = _read32(bytestream) - rows = _read32(bytestream) - cols = _read32(bytestream) - buf = bytestream.read(rows * cols * num_images) - data = numpy.frombuffer(buf, dtype=numpy.uint8) - data = data.reshape(num_images, rows, cols, 1) - return data - - -@deprecated(None, 'Please use tf.one_hot on tensors.') -def dense_to_one_hot(labels_dense, num_classes): - """Convert class labels from scalars to one-hot vectors.""" - num_labels = labels_dense.shape[0] - index_offset = numpy.arange(num_labels) * num_classes - labels_one_hot = numpy.zeros((num_labels, num_classes)) - labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1 - return labels_one_hot - - -@deprecated(None, 'Please use tf.data to implement this functionality.') -def extract_labels(f, one_hot=False, num_classes=10): - """Extract the labels into a 1D uint8 numpy array [index]. - - Args: - f: A file object that can be passed into a gzip reader. - one_hot: Does one hot encoding for the result. - num_classes: Number of classes for the one hot encoding. - - Returns: - labels: a 1D uint8 numpy array. - - Raises: - ValueError: If the bystream doesn't start with 2049. - """ - print('Extracting', f.name) - with gzip.GzipFile(fileobj=f) as bytestream: - magic = _read32(bytestream) - if magic != 2049: - raise ValueError('Invalid magic number %d in MNIST label file: %s' % - (magic, f.name)) - num_items = _read32(bytestream) - buf = bytestream.read(num_items) - labels = numpy.frombuffer(buf, dtype=numpy.uint8) - if one_hot: - return dense_to_one_hot(labels, num_classes) - return labels - - -class DataSet(object): - """Container class for a dataset (deprecated). - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - @deprecated(None, 'Please use alternatives such as official/mnist/dataset.py' - ' from tensorflow/models.') - def __init__(self, - images, - labels, - fake_data=False, - one_hot=False, - dtype=dtypes.float32, - reshape=True, - seed=None): - """Construct a DataSet. - one_hot arg is used only if fake_data is true. `dtype` can be either - `uint8` to leave the input as `[0, 255]`, or `float32` to rescale into - `[0, 1]`. Seed arg provides for convenient deterministic testing. - """ - seed1, seed2 = random_seed.get_seed(seed) - # If op level seed is not set, use whatever graph level seed is returned - numpy.random.seed(seed1 if seed is None else seed2) - dtype = dtypes.as_dtype(dtype).base_dtype - if dtype not in (dtypes.uint8, dtypes.float32): - raise TypeError( - 'Invalid image dtype %r, expected uint8 or float32' % dtype) - if fake_data: - self._num_examples = 10000 - self.one_hot = one_hot - else: - assert images.shape[0] == labels.shape[0], ( - 'images.shape: %s labels.shape: %s' % (images.shape, labels.shape)) - self._num_examples = images.shape[0] - - # Convert shape from [num examples, rows, columns, depth] - # to [num examples, rows*columns] (assuming depth == 1) - if reshape: - assert images.shape[3] == 1 - images = images.reshape(images.shape[0], - images.shape[1] * images.shape[2]) - if dtype == dtypes.float32: - # Convert from [0, 255] -> [0.0, 1.0]. - images = images.astype(numpy.float32) - images = numpy.multiply(images, 1.0 / 255.0) - self._images = images - self._labels = labels - self._epochs_completed = 0 - self._index_in_epoch = 0 - - @property - def images(self): - return self._images - - @property - def labels(self): - return self._labels - - @property - def num_examples(self): - return self._num_examples - - @property - def epochs_completed(self): - return self._epochs_completed - - def next_batch(self, batch_size, fake_data=False, shuffle=True): - """Return the next `batch_size` examples from this data set.""" - if fake_data: - fake_image = [1] * 784 - if self.one_hot: - fake_label = [1] + [0] * 9 - else: - fake_label = 0 - return [fake_image for _ in xrange(batch_size)], [ - fake_label for _ in xrange(batch_size) - ] - start = self._index_in_epoch - # Shuffle for the first epoch - if self._epochs_completed == 0 and start == 0 and shuffle: - perm0 = numpy.arange(self._num_examples) - numpy.random.shuffle(perm0) - self._images = self.images[perm0] - self._labels = self.labels[perm0] - # Go to the next epoch - if start + batch_size > self._num_examples: - # Finished epoch - self._epochs_completed += 1 - # Get the rest examples in this epoch - rest_num_examples = self._num_examples - start - images_rest_part = self._images[start:self._num_examples] - labels_rest_part = self._labels[start:self._num_examples] - # Shuffle the data - if shuffle: - perm = numpy.arange(self._num_examples) - numpy.random.shuffle(perm) - self._images = self.images[perm] - self._labels = self.labels[perm] - # Start next epoch - start = 0 - self._index_in_epoch = batch_size - rest_num_examples - end = self._index_in_epoch - images_new_part = self._images[start:end] - labels_new_part = self._labels[start:end] - return numpy.concatenate( - (images_rest_part, images_new_part), axis=0), numpy.concatenate( - (labels_rest_part, labels_new_part), axis=0) - else: - self._index_in_epoch += batch_size - end = self._index_in_epoch - return self._images[start:end], self._labels[start:end] - - -@deprecated(None, 'Please use alternatives such as official/mnist/dataset.py' - ' from tensorflow/models.') -def read_data_sets(train_dir, - fake_data=False, - one_hot=False, - dtype=dtypes.float32, - reshape=True, - validation_size=5000, - seed=None, - source_url=DEFAULT_SOURCE_URL): - if fake_data: - - def fake(): - return DataSet( - [], [], fake_data=True, one_hot=one_hot, dtype=dtype, seed=seed) - - train = fake() - validation = fake() - test = fake() - return base.Datasets(train=train, validation=validation, test=test) - - if not source_url: # empty string check - source_url = DEFAULT_SOURCE_URL - - TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' - TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' - TEST_IMAGES = 't10k-images-idx3-ubyte.gz' - TEST_LABELS = 't10k-labels-idx1-ubyte.gz' - - local_file = base.maybe_download(TRAIN_IMAGES, train_dir, - source_url + TRAIN_IMAGES) - with gfile.Open(local_file, 'rb') as f: - train_images = extract_images(f) - - local_file = base.maybe_download(TRAIN_LABELS, train_dir, - source_url + TRAIN_LABELS) - with gfile.Open(local_file, 'rb') as f: - train_labels = extract_labels(f, one_hot=one_hot) - - local_file = base.maybe_download(TEST_IMAGES, train_dir, - source_url + TEST_IMAGES) - with gfile.Open(local_file, 'rb') as f: - test_images = extract_images(f) - - local_file = base.maybe_download(TEST_LABELS, train_dir, - source_url + TEST_LABELS) - with gfile.Open(local_file, 'rb') as f: - test_labels = extract_labels(f, one_hot=one_hot) - - if not 0 <= validation_size <= len(train_images): - raise ValueError('Validation size should be between 0 and {}. Received: {}.' - .format(len(train_images), validation_size)) - - validation_images = train_images[:validation_size] - validation_labels = train_labels[:validation_size] - train_images = train_images[validation_size:] - train_labels = train_labels[validation_size:] - - options = dict(dtype=dtype, reshape=reshape, seed=seed) - - train = DataSet(train_images, train_labels, **options) - validation = DataSet(validation_images, validation_labels, **options) - test = DataSet(test_images, test_labels, **options) - - return base.Datasets(train=train, validation=validation, test=test) - - -@deprecated(None, 'Please use alternatives such as official/mnist/dataset.py' - ' from tensorflow/models.') -def load_mnist(train_dir='MNIST-data'): - return read_data_sets(train_dir) diff --git a/tensorflow/contrib/learn/python/learn/datasets/produce_small_datasets.py b/tensorflow/contrib/learn/python/learn/datasets/produce_small_datasets.py deleted file mode 100644 index a4848fa64a7..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/produce_small_datasets.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Produce DBpedia datasets of a smaller size (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.contrib.learn.python.learn.datasets import text_datasets -from tensorflow.python.platform import app - - -def main(unused_argv): - text_datasets.maybe_download_dbpedia('dbpedia_data') - # Reduce the size of original data by a factor of 1000. - base.shrink_csv('dbpedia_data/dbpedia_csv/train.csv', 1000) - base.shrink_csv('dbpedia_data/dbpedia_csv/test.csv', 1000) - - -if __name__ == '__main__': - app.run() diff --git a/tensorflow/contrib/learn/python/learn/datasets/synthetic.py b/tensorflow/contrib/learn/python/learn/datasets/synthetic.py deleted file mode 100644 index 6a0e3350b3d..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/synthetic.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Synthetic dataset generators (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.learn.python.learn.datasets.base import Dataset -from tensorflow.python.util.deprecation import deprecated - - -@deprecated(None, 'Consider using synthetic datasets from scikits.learn.') -def circles(n_samples=100, - noise=None, - seed=None, - factor=0.8, - n_classes=2, - *args, - **kwargs): - """Create circles separated by some value - - Args: - n_samples: int, number of datapoints to generate - noise: float or None, standard deviation of the Gaussian noise added - seed: int or None, seed for the noise - factor: float, size factor of the inner circles with respect to the outer - ones - n_classes: int, number of classes to generate - - Returns: - Shuffled features and labels for 'circles' synthetic dataset of type - `base.Dataset` - - Note: - The multi-class support might not work as expected if `noise` is enabled - - TODO: - - Generation of unbalanced data - - Credit goes to (under BSD 3 clause): - B. Thirion, - G. Varoquaux, - A. Gramfort, - V. Michel, - O. Grisel, - G. Louppe, - J. Nothman - """ - if seed is not None: - np.random.seed(seed) - # Algo: 1) Generate initial circle, 2) For ever class generate a smaller radius circle - linspace = np.linspace(0, 2 * np.pi, n_samples // n_classes) - circ_x = np.empty(0, dtype=np.int32) - circ_y = np.empty(0, dtype=np.int32) - base_cos = np.cos(linspace) - base_sin = np.sin(linspace) - - y = np.empty(0, dtype=np.int32) - for label in range(n_classes): - circ_x = np.append(circ_x, base_cos) - circ_y = np.append(circ_y, base_sin) - base_cos *= factor - base_sin *= factor - y = np.append(y, label * np.ones(n_samples // n_classes, dtype=np.int32)) - - # Add more points if n_samples is not divisible by n_classes (unbalanced!) - extras = n_samples % n_classes - circ_x = np.append(circ_x, np.cos(np.random.rand(extras) * 2 * np.pi)) - circ_y = np.append(circ_y, np.sin(np.random.rand(extras) * 2 * np.pi)) - y = np.append(y, np.zeros(extras, dtype=np.int32)) - - # Reshape the features/labels - X = np.vstack((circ_x, circ_y)).T - y = np.hstack(y) - - # Shuffle the data - indices = np.random.permutation(range(n_samples)) - if noise is not None: - X += np.random.normal(scale=noise, size=X.shape) - return Dataset(data=X[indices], target=y[indices]) - - -@deprecated(None, 'Consider using synthetic datasets from scikits.learn.') -def spirals(n_samples=100, - noise=None, - seed=None, - mode='archimedes', - n_loops=2, - *args, - **kwargs): - """Create spirals - - Currently only binary classification is supported for spiral generation - - Args: - n_samples: int, number of datapoints to generate - noise: float or None, standard deviation of the Gaussian noise added - seed: int or None, seed for the noise - n_loops: int, number of spiral loops, doesn't play well with 'bernoulli' - mode: str, how the spiral should be generated. Current implementations: - 'archimedes': a spiral with equal distances between branches - 'bernoulli': logarithmic spiral with branch distances increasing - 'fermat': a spiral with branch distances decreasing (sqrt) - - Returns: - Shuffled features and labels for 'spirals' synthetic dataset of type - `base.Dataset` - - Raises: - ValueError: If the generation `mode` is not valid - - TODO: - - Generation of unbalanced data - """ - n_classes = 2 # I am not sure how to make it multiclass - - _modes = { - 'archimedes': _archimedes_spiral, - 'bernoulli': _bernoulli_spiral, - 'fermat': _fermat_spiral - } - - if mode is None or mode not in _modes: - raise ValueError('Cannot generate spiral with mode %s' % mode) - - if seed is not None: - np.random.seed(seed) - linspace = np.linspace(0, 2 * n_loops * np.pi, n_samples // n_classes) - spir_x = np.empty(0, dtype=np.int32) - spir_y = np.empty(0, dtype=np.int32) - - y = np.empty(0, dtype=np.int32) - for label in range(n_classes): - base_cos, base_sin = _modes[mode](linspace, label * np.pi, *args, **kwargs) - spir_x = np.append(spir_x, base_cos) - spir_y = np.append(spir_y, base_sin) - y = np.append(y, label * np.ones(n_samples // n_classes, dtype=np.int32)) - - # Add more points if n_samples is not divisible by n_classes (unbalanced!) - extras = n_samples % n_classes - if extras > 0: - x_extra, y_extra = _modes[mode](np.random.rand(extras) * 2 * np.pi, *args, - **kwargs) - spir_x = np.append(spir_x, x_extra) - spir_y = np.append(spir_y, y_extra) - y = np.append(y, np.zeros(extras, dtype=np.int32)) - - # Reshape the features/labels - X = np.vstack((spir_x, spir_y)).T - y = np.hstack(y) - - # Shuffle the data - indices = np.random.permutation(range(n_samples)) - if noise is not None: - X += np.random.normal(scale=noise, size=X.shape) - return Dataset(data=X[indices], target=y[indices]) - - -def _archimedes_spiral(theta, theta_offset=0., *args, **kwargs): - """Return Archimedes spiral - - Args: - theta: array-like, angles from polar coordinates to be converted - theta_offset: float, angle offset in radians (2*pi = 0) - """ - x, y = theta * np.cos(theta + theta_offset), theta * np.sin( - theta + theta_offset) - x_norm = np.max(np.abs(x)) - y_norm = np.max(np.abs(y)) - x, y = x / x_norm, y / y_norm - return x, y - - -def _bernoulli_spiral(theta, theta_offset=0., *args, **kwargs): - """Return Equiangular (Bernoulli's) spiral - - Args: - theta: array-like, angles from polar coordinates to be converted - theta_offset: float, angle offset in radians (2*pi = 0) - - Kwargs: - exp_scale: growth rate of the exponential - """ - exp_scale = kwargs.pop('exp_scale', 0.1) - - x, y = np.exp(exp_scale * theta) * np.cos(theta + theta_offset), np.exp( - exp_scale * theta) * np.sin(theta + theta_offset) - x_norm = np.max(np.abs(x)) - y_norm = np.max(np.abs(y)) - x, y = x / x_norm, y / y_norm - return x, y - - -def _fermat_spiral(theta, theta_offset=0., *args, **kwargs): - """Return Parabolic (Fermat's) spiral - - Args: - theta: array-like, angles from polar coordinates to be converted - theta_offset: float, angle offset in radians (2*pi = 0) - """ - x, y = np.sqrt(theta) * np.cos(theta + theta_offset), np.sqrt(theta) * np.sin( - theta + theta_offset) - x_norm = np.max(np.abs(x)) - y_norm = np.max(np.abs(y)) - x, y = x / x_norm, y / y_norm - return x, y diff --git a/tensorflow/contrib/learn/python/learn/datasets/synthetic_test.py b/tensorflow/contrib/learn/python/learn/datasets/synthetic_test.py deleted file mode 100644 index 5809995c8c7..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/synthetic_test.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import six - -import numpy as np -from tensorflow.python.platform import test -from tensorflow.contrib.learn.python.learn import datasets -from tensorflow.contrib.learn.python.learn.datasets import synthetic - - -class SyntheticTest(test.TestCase): - """Test synthetic dataset generation""" - - def test_make_dataset(self): - """Test if the synthetic routine wrapper complains about the name""" - self.assertRaises( - ValueError, datasets.make_dataset, name='_non_existing_name') - - def test_all_datasets_callable(self): - """Test if all methods inside the `SYNTHETIC` are callable""" - self.assertIsInstance(datasets.SYNTHETIC, dict) - if len(datasets.SYNTHETIC) > 0: - for name, method in six.iteritems(datasets.SYNTHETIC): - self.assertTrue(callable(method)) - - def test_circles(self): - """Test if the circles are generated correctly - - Tests: - - return type is `Dataset` - - returned `data` shape is (n_samples, n_features) - - returned `target` shape is (n_samples,) - - set of unique classes range is [0, n_classes) - - TODO: - - all points have the same radius, if no `noise` specified - """ - n_samples = 100 - n_classes = 2 - circ = synthetic.circles( - n_samples=n_samples, noise=None, n_classes=n_classes) - self.assertIsInstance(circ, datasets.base.Dataset) - self.assertTupleEqual(circ.data.shape, (n_samples, 2)) - self.assertTupleEqual(circ.target.shape, (n_samples,)) - self.assertSetEqual(set(circ.target), set(range(n_classes))) - - def test_circles_replicable(self): - """Test if the data generation is replicable with a specified `seed` - - Tests: - - return the same value if raised with the same seed - - return different values if noise or seed is different - """ - seed = 42 - noise = 0.1 - circ0 = synthetic.circles( - n_samples=100, noise=noise, n_classes=2, seed=seed) - circ1 = synthetic.circles( - n_samples=100, noise=noise, n_classes=2, seed=seed) - np.testing.assert_array_equal(circ0.data, circ1.data) - np.testing.assert_array_equal(circ0.target, circ1.target) - - circ1 = synthetic.circles( - n_samples=100, noise=noise, n_classes=2, seed=seed + 1) - self.assertRaises(AssertionError, np.testing.assert_array_equal, circ0.data, - circ1.data) - self.assertRaises(AssertionError, np.testing.assert_array_equal, - circ0.target, circ1.target) - - circ1 = synthetic.circles( - n_samples=100, noise=noise / 2., n_classes=2, seed=seed) - self.assertRaises(AssertionError, np.testing.assert_array_equal, circ0.data, - circ1.data) - - def test_spirals(self): - """Test if the circles are generated correctly - - Tests: - - if mode is unknown, ValueError is raised - - return type is `Dataset` - - returned `data` shape is (n_samples, n_features) - - returned `target` shape is (n_samples,) - - set of unique classes range is [0, n_classes) - """ - self.assertRaises( - ValueError, synthetic.spirals, mode='_unknown_mode_spiral_') - n_samples = 100 - modes = ('archimedes', 'bernoulli', 'fermat') - for mode in modes: - spir = synthetic.spirals(n_samples=n_samples, noise=None, mode=mode) - self.assertIsInstance(spir, datasets.base.Dataset) - self.assertTupleEqual(spir.data.shape, (n_samples, 2)) - self.assertTupleEqual(spir.target.shape, (n_samples,)) - self.assertSetEqual(set(spir.target), set(range(2))) - - def test_spirals_replicable(self): - """Test if the data generation is replicable with a specified `seed` - - Tests: - - return the same value if raised with the same seed - - return different values if noise or seed is different - """ - seed = 42 - noise = 0.1 - modes = ('archimedes', 'bernoulli', 'fermat') - for mode in modes: - spir0 = synthetic.spirals(n_samples=1000, noise=noise, seed=seed) - spir1 = synthetic.spirals(n_samples=1000, noise=noise, seed=seed) - np.testing.assert_array_equal(spir0.data, spir1.data) - np.testing.assert_array_equal(spir0.target, spir1.target) - - spir1 = synthetic.spirals(n_samples=1000, noise=noise, seed=seed + 1) - self.assertRaises(AssertionError, np.testing.assert_array_equal, - spir0.data, spir1.data) - self.assertRaises(AssertionError, np.testing.assert_array_equal, - spir0.target, spir1.target) - - spir1 = synthetic.spirals(n_samples=1000, noise=noise / 2., seed=seed) - self.assertRaises(AssertionError, np.testing.assert_array_equal, - spir0.data, spir1.data) - - def test_spirals_synthetic(self): - synthetic.spirals(3) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/datasets/text_datasets.py b/tensorflow/contrib/learn/python/learn/datasets/text_datasets.py deleted file mode 100644 index ce946630172..00000000000 --- a/tensorflow/contrib/learn/python/learn/datasets/text_datasets.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Text datasets (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import tarfile - -import numpy as np - -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.python.platform import gfile -from tensorflow.python.util.deprecation import deprecated - -DBPEDIA_URL = 'https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz' - - -@deprecated(None, 'See contrib/learn/README.md') -def maybe_download_dbpedia(data_dir): - """Download if DBpedia data is not present.""" - train_path = os.path.join(data_dir, 'dbpedia_csv/train.csv') - test_path = os.path.join(data_dir, 'dbpedia_csv/test.csv') - if not (gfile.Exists(train_path) and gfile.Exists(test_path)): - archive_path = base.maybe_download( - 'dbpedia_csv.tar.gz', data_dir, DBPEDIA_URL) - tfile = tarfile.open(archive_path, 'r:*') - tfile.extractall(data_dir) - - -@deprecated(None, 'See contrib/learn/README.md') -def load_dbpedia(size='small', test_with_fake_data=False): - """Get DBpedia datasets from CSV files.""" - if not test_with_fake_data: - data_dir = os.path.join(os.getenv('TF_EXP_BASE_DIR', ''), 'dbpedia_data') - maybe_download_dbpedia(data_dir) - - train_path = os.path.join(data_dir, 'dbpedia_csv', 'train.csv') - test_path = os.path.join(data_dir, 'dbpedia_csv', 'test.csv') - - if size == 'small': - # Reduce the size of original data by a factor of 1000. - base.shrink_csv(train_path, 1000) - base.shrink_csv(test_path, 1000) - train_path = train_path.replace('train.csv', 'train_small.csv') - test_path = test_path.replace('test.csv', 'test_small.csv') - else: - module_path = os.path.dirname(__file__) - train_path = os.path.join(module_path, 'data', 'text_train.csv') - test_path = os.path.join(module_path, 'data', 'text_test.csv') - - train = base.load_csv_without_header( - train_path, target_dtype=np.int32, features_dtype=np.str, target_column=0) - test = base.load_csv_without_header( - test_path, target_dtype=np.int32, features_dtype=np.str, target_column=0) - - return base.Datasets(train=train, validation=None, test=test) diff --git a/tensorflow/contrib/learn/python/learn/estimators/__init__.py b/tensorflow/contrib/learn/python/learn/estimators/__init__.py deleted file mode 100644 index ce644dde04f..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/__init__.py +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""An estimator is a rule for calculating an estimate of a given quantity (deprecated). - -These classes are deprecated and replaced with `tf.estimator`. - -See [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. - -# Estimators - -* **Estimators** are used to train and evaluate TensorFlow models. -They support regression and classification problems. -* **Classifiers** are functions that have discrete outcomes. -* **Regressors** are functions that predict continuous values. - -## Choosing the correct estimator - -* For **Regression** problems use one of the following: - * `LinearRegressor`: Uses linear model. - * `DNNRegressor`: Uses DNN. - * `DNNLinearCombinedRegressor`: Uses Wide & Deep. - * `TensorForestEstimator`: Uses RandomForest. - See tf.contrib.tensor_forest.client.random_forest.TensorForestEstimator. - * `Estimator`: Use when you need a custom model. - -* For **Classification** problems use one of the following: - * `LinearClassifier`: Multiclass classifier using Linear model. - * `DNNClassifier`: Multiclass classifier using DNN. - * `DNNLinearCombinedClassifier`: Multiclass classifier using Wide & Deep. - * `TensorForestEstimator`: Uses RandomForest. - See tf.contrib.tensor_forest.client.random_forest.TensorForestEstimator. - * `SVM`: Binary classifier using linear SVMs. - * `LogisticRegressor`: Use when you need custom model for binary - classification. - * `Estimator`: Use when you need custom model for N class classification. - -## Pre-canned Estimators - -Pre-canned estimators are machine learning estimators premade for general -purpose problems. If you need more customization, you can always write your -own custom estimator as described in the section below. - -Pre-canned estimators are tested and optimized for speed and quality. - -### Define the feature columns - -Here are some possible types of feature columns used as inputs to a pre-canned -estimator. - -Feature columns may vary based on the estimator used. So you can see which -feature columns are fed to each estimator in the below section. - -```python -sparse_feature_a = sparse_column_with_keys( - column_name="sparse_feature_a", keys=["AB", "CD", ...]) - -embedding_feature_a = embedding_column( - sparse_id_column=sparse_feature_a, dimension=3, combiner="sum") - -sparse_feature_b = sparse_column_with_hash_bucket( - column_name="sparse_feature_b", hash_bucket_size=1000) - -embedding_feature_b = embedding_column( - sparse_id_column=sparse_feature_b, dimension=16, combiner="sum") - -crossed_feature_a_x_b = crossed_column( - columns=[sparse_feature_a, sparse_feature_b], hash_bucket_size=10000) - -real_feature = real_valued_column("real_feature") -real_feature_buckets = bucketized_column( - source_column=real_feature, - boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65]) -``` - -### Create the pre-canned estimator - -DNNClassifier, DNNRegressor, and DNNLinearCombinedClassifier are all pretty -similar to each other in how you use them. You can easily plug in an -optimizer and/or regularization to those estimators. - -#### DNNClassifier - -A classifier for TensorFlow DNN models. - -```python -my_features = [embedding_feature_a, embedding_feature_b] -estimator = DNNClassifier( - feature_columns=my_features, - hidden_units=[1024, 512, 256], - optimizer=tf.compat.v1.train.ProximalAdagradOptimizer( - learning_rate=0.1, - l1_regularization_strength=0.001 - )) -``` - -#### DNNRegressor - -A regressor for TensorFlow DNN models. - -```python -my_features = [embedding_feature_a, embedding_feature_b] - -estimator = DNNRegressor( -feature_columns=my_features, -hidden_units=[1024, 512, 256]) - -# Or estimator using the ProximalAdagradOptimizer optimizer with -# regularization. -estimator = DNNRegressor( - feature_columns=my_features, - hidden_units=[1024, 512, 256], - optimizer=tf.compat.v1.train.ProximalAdagradOptimizer( - learning_rate=0.1, - l1_regularization_strength=0.001 - )) -``` - -#### DNNLinearCombinedClassifier - -A classifier for TensorFlow Linear and DNN joined training models. - -* Wide and deep model -* Multi class (2 by default) - -```python -my_linear_features = [crossed_feature_a_x_b] -my_deep_features = [embedding_feature_a, embedding_feature_b] -estimator = DNNLinearCombinedClassifier( - # Common settings - n_classes=n_classes, - weight_column_name=weight_column_name, - # Wide settings - linear_feature_columns=my_linear_features, - linear_optimizer=tf.compat.v1.train.FtrlOptimizer(...), - # Deep settings - dnn_feature_columns=my_deep_features, - dnn_hidden_units=[1000, 500, 100], - dnn_optimizer=tf.compat.v1.train.AdagradOptimizer(...)) -``` - -#### LinearClassifier - -Train a linear model to classify instances into one of multiple possible -classes. When number of possible classes is 2, this is binary classification. - -```python -my_features = [sparse_feature_b, crossed_feature_a_x_b] -estimator = LinearClassifier( - feature_columns=my_features, - optimizer=tf.compat.v1.train.FtrlOptimizer( - learning_rate=0.1, - l1_regularization_strength=0.001 - )) -``` - -#### LinearRegressor - -Train a linear regression model to predict a label value given observation of -feature values. - -```python -my_features = [sparse_feature_b, crossed_feature_a_x_b] -estimator = LinearRegressor( - feature_columns=my_features) -``` - -### LogisticRegressor - -Logistic regression estimator for binary classification. - -```python -# See tf.contrib.learn.Estimator(...) for details on model_fn structure -def my_model_fn(...): - pass - -estimator = LogisticRegressor(model_fn=my_model_fn) - -# Input builders -def input_fn_train: - pass - -estimator.fit(input_fn=input_fn_train) -estimator.predict(x=x) -``` - -#### SVM - Support Vector Machine - -Support Vector Machine (SVM) model for binary classification. - -Currently only linear SVMs are supported. - -```python -my_features = [real_feature, sparse_feature_a] -estimator = SVM( - example_id_column='example_id', - feature_columns=my_features, - l2_regularization=10.0) -``` - -#### DynamicRnnEstimator - -An `Estimator` that uses a recurrent neural network with dynamic unrolling. - -```python -problem_type = ProblemType.CLASSIFICATION # or REGRESSION -prediction_type = PredictionType.SINGLE_VALUE # or MULTIPLE_VALUE - -estimator = DynamicRnnEstimator(problem_type, - prediction_type, - my_feature_columns) -``` - -### Use the estimator - -There are two main functions for using estimators, one of which is for -training, and one of which is for evaluation. -You can specify different data sources for each one in order to use different -datasets for train and eval. - -```python -# Input builders -def input_fn_train: # returns x, Y - ... -estimator.fit(input_fn=input_fn_train) - -def input_fn_eval: # returns x, Y - ... -estimator.evaluate(input_fn=input_fn_eval) -estimator.predict(x=x) -``` - -## Creating Custom Estimator - -To create a custom `Estimator`, provide a function to `Estimator`'s -constructor that builds your model (`model_fn`, below): - - -```python -estimator = tf.contrib.learn.Estimator( - model_fn=model_fn, - model_dir=model_dir) # Where the model's data (e.g., checkpoints) - # are saved. -``` - -Here is a skeleton of this function, with descriptions of its arguments and -return values in the accompanying tables: - -```python -def model_fn(features, targets, mode, params): - # Logic to do the following: - # 1. Configure the model via TensorFlow operations - # 2. Define the loss function for training/evaluation - # 3. Define the training operation/optimizer - # 4. Generate predictions - return predictions, loss, train_op -``` - -You may use `mode` and check against -`tf.contrib.learn.ModeKeys.{TRAIN, EVAL, INFER}` to parameterize `model_fn`. - -In the Further Reading section below, there is an end-to-end TensorFlow -tutorial for building a custom estimator. - -## Additional Estimators - -There is an additional estimators under -`tensorflow.contrib.factorization.python.ops`: - -* Gaussian mixture model (GMM) clustering - -## Further reading - -For further reading, there are several tutorials with relevant topics, -including: - -* [Overview of linear models](../../../tutorials/linear/overview.md) -* [Linear model tutorial](../../../tutorials/wide/index.md) -* [Wide and deep learning tutorial](../../../tutorials/wide_and_deep/index.md) -* [Custom estimator tutorial](../../../tutorials/estimators/index.md) -* [Building input functions](../../../tutorials/input_fn/index.md) -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.estimators._sklearn import NotFittedError -from tensorflow.contrib.learn.python.learn.estimators.constants import ProblemType -from tensorflow.contrib.learn.python.learn.estimators.dnn import DNNClassifier -from tensorflow.contrib.learn.python.learn.estimators.dnn import DNNEstimator -from tensorflow.contrib.learn.python.learn.estimators.dnn import DNNRegressor -from tensorflow.contrib.learn.python.learn.estimators.dnn_linear_combined import DNNLinearCombinedClassifier -from tensorflow.contrib.learn.python.learn.estimators.dnn_linear_combined import DNNLinearCombinedEstimator -from tensorflow.contrib.learn.python.learn.estimators.dnn_linear_combined import DNNLinearCombinedRegressor -from tensorflow.contrib.learn.python.learn.estimators.dynamic_rnn_estimator import DynamicRnnEstimator -from tensorflow.contrib.learn.python.learn.estimators.estimator import BaseEstimator -from tensorflow.contrib.learn.python.learn.estimators.estimator import Estimator -from tensorflow.contrib.learn.python.learn.estimators.estimator import GraphRewriteSpec -from tensorflow.contrib.learn.python.learn.estimators.estimator import infer_real_valued_columns_from_input -from tensorflow.contrib.learn.python.learn.estimators.estimator import infer_real_valued_columns_from_input_fn -from tensorflow.contrib.learn.python.learn.estimators.estimator import SKCompat -from tensorflow.contrib.learn.python.learn.estimators.head import binary_svm_head -from tensorflow.contrib.learn.python.learn.estimators.head import Head -from tensorflow.contrib.learn.python.learn.estimators.head import loss_only_head -from tensorflow.contrib.learn.python.learn.estimators.head import multi_class_head -from tensorflow.contrib.learn.python.learn.estimators.head import multi_head -from tensorflow.contrib.learn.python.learn.estimators.head import multi_label_head -from tensorflow.contrib.learn.python.learn.estimators.head import no_op_train_fn -from tensorflow.contrib.learn.python.learn.estimators.head import poisson_regression_head -from tensorflow.contrib.learn.python.learn.estimators.head import regression_head -from tensorflow.contrib.learn.python.learn.estimators.kmeans import KMeansClustering -from tensorflow.contrib.learn.python.learn.estimators.linear import LinearClassifier -from tensorflow.contrib.learn.python.learn.estimators.linear import LinearEstimator -from tensorflow.contrib.learn.python.learn.estimators.linear import LinearRegressor -from tensorflow.contrib.learn.python.learn.estimators.logistic_regressor import LogisticRegressor -from tensorflow.contrib.learn.python.learn.estimators.metric_key import MetricKey -from tensorflow.contrib.learn.python.learn.estimators.model_fn import ModeKeys -from tensorflow.contrib.learn.python.learn.estimators.model_fn import ModelFnOps -from tensorflow.contrib.learn.python.learn.estimators.prediction_key import PredictionKey -from tensorflow.contrib.learn.python.learn.estimators.rnn_common import PredictionType -from tensorflow.contrib.learn.python.learn.estimators.run_config import ClusterConfig -from tensorflow.contrib.learn.python.learn.estimators.run_config import Environment -from tensorflow.contrib.learn.python.learn.estimators.run_config import RunConfig -from tensorflow.contrib.learn.python.learn.estimators.run_config import TaskType -from tensorflow.contrib.learn.python.learn.estimators.svm import SVM diff --git a/tensorflow/contrib/learn/python/learn/estimators/_sklearn.py b/tensorflow/contrib/learn/python/learn/estimators/_sklearn.py deleted file mode 100644 index a15bbce515b..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/_sklearn.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""sklearn cross-support (deprecated).""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -import numpy as np -import six - -from tensorflow.python.util.compat import collections_abc - - -def _pprint(d): - return ', '.join(['%s=%s' % (key, str(value)) for key, value in d.items()]) - - -class _BaseEstimator(object): - """This is a cross-import when sklearn is not available. - - Adopted from sklearn.BaseEstimator implementation. - https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py - """ - - def get_params(self, deep=True): - """Get parameters for this estimator. - - Args: - deep: boolean, optional - - If `True`, will return the parameters for this estimator and - contained subobjects that are estimators. - - Returns: - params : mapping of string to any - Parameter names mapped to their values. - """ - out = {} - param_names = [name for name in self.__dict__ if not name.startswith('_')] - for key in param_names: - value = getattr(self, key, None) - - if isinstance(value, collections_abc.Callable): - continue - - # XXX: should we rather test if instance of estimator? - if deep and hasattr(value, 'get_params'): - deep_items = value.get_params().items() - out.update((key + '__' + k, val) for k, val in deep_items) - out[key] = value - return out - - def set_params(self, **params): - """Set the parameters of this estimator. - - The method works on simple estimators as well as on nested objects - (such as pipelines). The former have parameters of the form - ``__`` so that it's possible to update each - component of a nested object. - - Args: - **params: Parameters. - - Returns: - self - - Raises: - ValueError: If params contain invalid names. - """ - if not params: - # Simple optimisation to gain speed (inspect is slow) - return self - valid_params = self.get_params(deep=True) - for key, value in six.iteritems(params): - split = key.split('__', 1) - if len(split) > 1: - # nested objects case - name, sub_name = split - if name not in valid_params: - raise ValueError('Invalid parameter %s for estimator %s. ' - 'Check the list of available parameters ' - 'with `estimator.get_params().keys()`.' % - (name, self)) - sub_object = valid_params[name] - sub_object.set_params(**{sub_name: value}) - else: - # simple objects case - if key not in valid_params: - raise ValueError('Invalid parameter %s for estimator %s. ' - 'Check the list of available parameters ' - 'with `estimator.get_params().keys()`.' % - (key, self.__class__.__name__)) - setattr(self, key, value) - return self - - def __repr__(self): - class_name = self.__class__.__name__ - return '%s(%s)' % (class_name, - _pprint(self.get_params(deep=False)),) - - -# pylint: disable=old-style-class -class _ClassifierMixin(): - """Mixin class for all classifiers.""" - pass - - -class _RegressorMixin(): - """Mixin class for all regression estimators.""" - pass - - -class _TransformerMixin(): - """Mixin class for all transformer estimators.""" - - -class NotFittedError(ValueError, AttributeError): - """Exception class to raise if estimator is used before fitting. - - USE OF THIS EXCEPTION IS DEPRECATED. - - This class inherits from both ValueError and AttributeError to help with - exception handling and backward compatibility. - - Examples: - >>> from sklearn.svm import LinearSVC - >>> from sklearn.exceptions import NotFittedError - >>> try: - ... LinearSVC().predict([[1, 2], [2, 3], [3, 4]]) - ... except NotFittedError as e: - ... print(repr(e)) - ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS - NotFittedError('This LinearSVC instance is not fitted yet',) - - Copied from - https://github.com/scikit-learn/scikit-learn/master/sklearn/exceptions.py - """ - -# pylint: enable=old-style-class - - -def _accuracy_score(y_true, y_pred): - score = y_true == y_pred - return np.average(score) - - -def _mean_squared_error(y_true, y_pred): - if len(y_true.shape) > 1: - y_true = np.squeeze(y_true) - if len(y_pred.shape) > 1: - y_pred = np.squeeze(y_pred) - return np.average((y_true - y_pred)**2) - - -def _train_test_split(*args, **options): - # pylint: disable=missing-docstring - test_size = options.pop('test_size', None) - train_size = options.pop('train_size', None) - random_state = options.pop('random_state', None) - - if test_size is None and train_size is None: - train_size = 0.75 - elif train_size is None: - train_size = 1 - test_size - train_size = int(train_size * args[0].shape[0]) - - np.random.seed(random_state) - indices = np.random.permutation(args[0].shape[0]) - train_idx, test_idx = indices[:train_size], indices[train_size:] - result = [] - for x in args: - result += [x.take(train_idx, axis=0), x.take(test_idx, axis=0)] - return tuple(result) - - -# If "TENSORFLOW_SKLEARN" flag is defined then try to import from sklearn. -TRY_IMPORT_SKLEARN = os.environ.get('TENSORFLOW_SKLEARN', False) -if TRY_IMPORT_SKLEARN: - # pylint: disable=g-import-not-at-top,g-multiple-import,unused-import - from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin, TransformerMixin - from sklearn.metrics import accuracy_score, log_loss, mean_squared_error - from sklearn.model_selection import train_test_split - try: - from sklearn.exceptions import NotFittedError - except ImportError: - try: - from sklearn.utils.validation import NotFittedError - except ImportError: - pass -else: - # Naive implementations of sklearn classes and functions. - BaseEstimator = _BaseEstimator - ClassifierMixin = _ClassifierMixin - RegressorMixin = _RegressorMixin - TransformerMixin = _TransformerMixin - accuracy_score = _accuracy_score - log_loss = None - mean_squared_error = _mean_squared_error - train_test_split = _train_test_split diff --git a/tensorflow/contrib/learn/python/learn/estimators/composable_model.py b/tensorflow/contrib/learn/python/learn/estimators/composable_model.py deleted file mode 100644 index 1fa58271e2b..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/composable_model.py +++ /dev/null @@ -1,409 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""TensorFlow composable models used as building blocks for estimators (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import re - -import six - -from tensorflow.contrib import layers -from tensorflow.contrib.framework import list_variables -from tensorflow.contrib.framework import load_variable -from tensorflow.contrib.layers.python.layers import feature_column_ops -from tensorflow.python.framework import ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import gradients -from tensorflow.python.ops import nn -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import variable_scope -from tensorflow.python.summary import summary -from tensorflow.python.util.deprecation import deprecated - - -class _ComposableModel(object): - """ABC for building blocks that can be used to create estimators. - - Subclasses need to implement the following methods: - - build_model - - _get_optimizer - See below for the required signatures. - _ComposableModel and its subclasses are not part of the public tf.learn API. - """ - - @deprecated(None, "Please use model_fns in tf.estimator.") - def __init__(self, - num_label_columns, - optimizer, - gradient_clip_norm, - num_ps_replicas, - scope, - trainable=True): - """Common initialization for all _ComposableModel objects. - - Args: - num_label_columns: The number of label columns. - optimizer: An instance of `tf.Optimizer` used to apply gradients to - the model. If `None`, will use a FTRL optimizer. - gradient_clip_norm: A float > 0. If provided, gradients are clipped - to their global norm with this clipping ratio. See - tf.clip_by_global_norm for more details. - num_ps_replicas: The number of parameter server replicas. - scope: Scope for variables created in this model. - trainable: True if this model contains variables that can be trained. - False otherwise (in cases where the variables are used strictly for - transforming input labels for training). - """ - self._num_label_columns = num_label_columns - self._optimizer = optimizer - self._gradient_clip_norm = gradient_clip_norm - self._num_ps_replicas = num_ps_replicas - self._scope = scope - self._trainable = trainable - self._feature_columns = None - - def get_scope_name(self): - """Returns the scope name used by this model for variables.""" - return self._scope - - def build_model(self, features, feature_columns, is_training): - """Builds the model that can calculate the logits. - - Args: - features: A mapping from feature columns to tensors. - feature_columns: An iterable containing all the feature columns used - by the model. All items in the set should be instances of - classes derived from `FeatureColumn`. - is_training: Set to True when training, False otherwise. - - Returns: - The logits for this model. - """ - raise NotImplementedError - - def get_train_step(self, loss): - """Returns the ops to run to perform a training step on this estimator. - - Args: - loss: The loss to use when calculating gradients. - - Returns: - The ops to run to perform a training step. - """ - my_vars = self._get_vars() - if not (self._get_feature_columns() or my_vars): - return [] - - grads = gradients.gradients(loss, my_vars) - if self._gradient_clip_norm: - grads, _ = clip_ops.clip_by_global_norm(grads, self._gradient_clip_norm) - return [self._get_optimizer().apply_gradients(zip(grads, my_vars))] - - def _get_feature_columns(self): - if not self._feature_columns: - return None - feature_column_ops.check_feature_columns(self._feature_columns) - return sorted(set(self._feature_columns), key=lambda x: x.key) - - def _get_vars(self): - if self._get_feature_columns(): - return ops.get_collection(self._scope) - return [] - - def _get_optimizer(self): - if (self._optimizer is None or isinstance(self._optimizer, - six.string_types)): - optimizer = self._get_default_optimizer(self._optimizer) - elif callable(self._optimizer): - optimizer = self._optimizer() - else: - optimizer = self._optimizer - return optimizer - - def _get_default_optimizer(self, optimizer_name=None): - raise NotImplementedError - - -class LinearComposableModel(_ComposableModel): - """A _ComposableModel that implements linear regression. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Instances of this class can be used to build estimators through the use - of composition. - """ - - def __init__(self, - num_label_columns, - optimizer=None, - _joint_weights=False, - gradient_clip_norm=None, - num_ps_replicas=0, - scope=None, - trainable=True): - """Initializes LinearComposableModel objects. - - Args: - num_label_columns: The number of label columns. - optimizer: An instance of `tf.Optimizer` used to apply gradients to - the model. If `None`, will use a FTRL optimizer. - _joint_weights: If True use a single (possibly partitioned) variable - to store all weights in this model. Faster, but requires that all - feature columns are sparse and have the 'sum' combiner. - gradient_clip_norm: A float > 0. If provided, gradients are clipped - to their global norm with this clipping ratio. See - tf.clip_by_global_norm for more details. - num_ps_replicas: The number of parameter server replicas. - scope: Optional scope for variables created in this model. If scope - is not supplied, it will default to 'linear'. - trainable: True if this model contains variables that can be trained. - False otherwise (in cases where the variables are used strictly for - transforming input labels for training). - """ - scope = "linear" if not scope else scope - super(LinearComposableModel, self).__init__( - num_label_columns=num_label_columns, - optimizer=optimizer, - gradient_clip_norm=gradient_clip_norm, - num_ps_replicas=num_ps_replicas, - scope=scope, - trainable=trainable) - self._joint_weights = _joint_weights - - def get_weights(self, model_dir): - """Returns weights per feature of the linear part. - - Args: - model_dir: Directory where model parameters, graph and etc. are saved. - - Returns: - The weights created by this model (without the optimizer weights). - """ - all_variables = [name for name, _ in list_variables(model_dir)] - values = {} - optimizer_regex = r".*/" + self._get_optimizer().get_name() + r"(_\d)?$" - for name in all_variables: - if (name.startswith(self._scope + "/") and - name != self._scope + "/bias_weight" and - not re.match(optimizer_regex, name)): - values[name] = load_variable(model_dir, name) - if len(values) == 1: - return values[list(values.keys())[0]] - return values - - def get_bias(self, model_dir): - """Returns bias of the model. - - Args: - model_dir: Directory where model parameters, graph and etc. are saved. - - Returns: - The bias weights created by this model. - """ - return load_variable(model_dir, name=(self._scope + "/bias_weight")) - - def build_model(self, features, feature_columns, is_training): - """See base class.""" - self._feature_columns = feature_columns - partitioner = partitioned_variables.min_max_variable_partitioner( - max_partitions=self._num_ps_replicas, min_slice_size=64 << 20) - with variable_scope.variable_scope( - self._scope, values=features.values(), - partitioner=partitioner) as scope: - if self._joint_weights: - logits, _, _ = layers.joint_weighted_sum_from_feature_columns( - columns_to_tensors=features, - feature_columns=self._get_feature_columns(), - num_outputs=self._num_label_columns, - weight_collections=[self._scope], - trainable=self._trainable, - scope=scope) - else: - logits, _, _ = layers.weighted_sum_from_feature_columns( - columns_to_tensors=features, - feature_columns=self._get_feature_columns(), - num_outputs=self._num_label_columns, - weight_collections=[self._scope], - trainable=self._trainable, - scope=scope) - return logits - - def _get_default_optimizer(self, optimizer_name=None): - if optimizer_name is None: - optimizer_name = "Ftrl" - default_learning_rate = 1. / math.sqrt(len(self._get_feature_columns())) - default_learning_rate = min(0.2, default_learning_rate) - return layers.OPTIMIZER_CLS_NAMES[optimizer_name]( - learning_rate=default_learning_rate) - - -class DNNComposableModel(_ComposableModel): - """A _ComposableModel that implements a DNN. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Instances of this class can be used to build estimators through the use - of composition. - """ - - def __init__(self, - num_label_columns, - hidden_units, - optimizer=None, - activation_fn=nn.relu, - dropout=None, - gradient_clip_norm=None, - num_ps_replicas=0, - scope=None, - trainable=True): - """Initializes DNNComposableModel objects. - - Args: - num_label_columns: The number of label columns. - hidden_units: List of hidden units per layer. All layers are fully - connected. - optimizer: An instance of `tf.Optimizer` used to apply gradients to - the model. If `None`, will use a FTRL optimizer. - activation_fn: Activation function applied to each layer. If `None`, - will use `tf.nn.relu`. - dropout: When not None, the probability we will drop out - a given coordinate. - gradient_clip_norm: A float > 0. If provided, gradients are clipped - to their global norm with this clipping ratio. See - tf.clip_by_global_norm for more details. - num_ps_replicas: The number of parameter server replicas. - scope: Optional scope for variables created in this model. If not scope - is supplied, one is generated. - trainable: True if this model contains variables that can be trained. - False otherwise (in cases where the variables are used strictly for - transforming input labels for training). - """ - scope = "dnn" if not scope else scope - super(DNNComposableModel, self).__init__( - num_label_columns=num_label_columns, - optimizer=optimizer, - gradient_clip_norm=gradient_clip_norm, - num_ps_replicas=num_ps_replicas, - scope=scope, - trainable=trainable) - self._hidden_units = hidden_units - self._activation_fn = activation_fn - self._dropout = dropout - - def get_weights(self, model_dir): - """Returns the weights of the model. - - Args: - model_dir: Directory where model parameters, graph and etc. are saved. - - Returns: - The weights created by this model. - """ - return [ - load_variable( - model_dir, name=(self._scope + "/hiddenlayer_%d/weights" % i)) - for i, _ in enumerate(self._hidden_units) - ] + [load_variable( - model_dir, name=(self._scope + "/logits/weights"))] - - def get_bias(self, model_dir): - """Returns the bias of the model. - - Args: - model_dir: Directory where model parameters, graph and etc. are saved. - - Returns: - The bias weights created by this model. - """ - return [ - load_variable( - model_dir, name=(self._scope + "/hiddenlayer_%d/biases" % i)) - for i, _ in enumerate(self._hidden_units) - ] + [load_variable( - model_dir, name=(self._scope + "/logits/biases"))] - - def _add_hidden_layer_summary(self, value, tag): - # TODO(zakaria): Move this code to tf.learn and add test. - summary.scalar("%s/fraction_of_zero_values" % tag, nn.zero_fraction(value)) - summary.histogram("%s/activation" % tag, value) - - def build_model(self, features, feature_columns, is_training): - """See base class.""" - self._feature_columns = feature_columns - - input_layer_partitioner = ( - partitioned_variables.min_max_variable_partitioner( - max_partitions=self._num_ps_replicas, min_slice_size=64 << 20)) - with variable_scope.variable_scope( - self._scope + "/input_from_feature_columns", - values=features.values(), - partitioner=input_layer_partitioner) as scope: - net = layers.input_from_feature_columns( - features, - self._get_feature_columns(), - weight_collections=[self._scope], - trainable=self._trainable, - scope=scope) - - hidden_layer_partitioner = ( - partitioned_variables.min_max_variable_partitioner( - max_partitions=self._num_ps_replicas)) - for layer_id, num_hidden_units in enumerate(self._hidden_units): - with variable_scope.variable_scope( - self._scope + "/hiddenlayer_%d" % layer_id, - values=[net], - partitioner=hidden_layer_partitioner) as scope: - net = layers.fully_connected( - net, - num_hidden_units, - activation_fn=self._activation_fn, - variables_collections=[self._scope], - trainable=self._trainable, - scope=scope) - if self._dropout is not None and is_training: - net = layers.dropout(net, keep_prob=(1.0 - self._dropout)) - self._add_hidden_layer_summary(net, scope.name) - - with variable_scope.variable_scope( - self._scope + "/logits", - values=[net], - partitioner=hidden_layer_partitioner) as scope: - logits = layers.fully_connected( - net, - self._num_label_columns, - activation_fn=None, - variables_collections=[self._scope], - trainable=self._trainable, - scope=scope) - self._add_hidden_layer_summary(logits, "logits") - return logits - - def _get_default_optimizer(self, optimizer_name=None): - if optimizer_name is None: - optimizer_name = "Adagrad" - return layers.OPTIMIZER_CLS_NAMES[optimizer_name](learning_rate=0.05) diff --git a/tensorflow/contrib/learn/python/learn/estimators/composable_model_test.py b/tensorflow/contrib/learn/python/learn/estimators/composable_model_test.py deleted file mode 100644 index ef5e620e8f0..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/composable_model_test.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for ComposableModel classes.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.training import training_util -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.contrib.learn.python.learn.estimators import composable_model -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import state_ops -from tensorflow.python.platform import test - - -def _iris_input_fn(): - iris = base.load_iris() - return { - 'feature': constant_op.constant( - iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=[150, 1], dtype=dtypes.int32) - - -def _base_model_fn(features, labels, mode, params): - model = params['model'] - feature_columns = params['feature_columns'] - head = params['head'] - - if mode == model_fn_lib.ModeKeys.TRAIN: - logits = model.build_model(features, feature_columns, is_training=True) - elif mode == model_fn_lib.ModeKeys.EVAL: - logits = model.build_model(features, feature_columns, is_training=False) - else: - raise NotImplementedError - - def _train_op_fn(loss): - global_step = training_util.get_global_step() - assert global_step - train_step = model.get_train_step(loss) - - with ops.control_dependencies(train_step): - with ops.get_default_graph().colocate_with(global_step): - return state_ops.assign_add(global_step, 1).op - - return head.create_model_fn_ops( - features=features, - mode=mode, - labels=labels, - train_op_fn=_train_op_fn, - logits=logits) - - -def _linear_estimator(head, feature_columns): - return estimator.Estimator( - model_fn=_base_model_fn, - params={ - 'model': - composable_model.LinearComposableModel( - num_label_columns=head.logits_dimension), - 'feature_columns': - feature_columns, - 'head': - head - }) - - -def _joint_linear_estimator(head, feature_columns): - return estimator.Estimator( - model_fn=_base_model_fn, - params={ - 'model': - composable_model.LinearComposableModel( - num_label_columns=head.logits_dimension, _joint_weights=True), - 'feature_columns': - feature_columns, - 'head': - head - }) - - -def _dnn_estimator(head, feature_columns, hidden_units): - return estimator.Estimator( - model_fn=_base_model_fn, - params={ - 'model': - composable_model.DNNComposableModel( - num_label_columns=head.logits_dimension, - hidden_units=hidden_units), - 'feature_columns': - feature_columns, - 'head': - head - }) - - -class ComposableModelTest(test.TestCase): - - def testLinearModel(self): - """Tests that loss goes down with training.""" - - def input_fn(): - return { - 'age': - constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column.sparse_column_with_hash_bucket('language', 100) - age = feature_column.real_valued_column('age') - - head = head_lib.multi_class_head(n_classes=2) - classifier = _linear_estimator(head, feature_columns=[age, language]) - - classifier.fit(input_fn=input_fn, steps=1000) - loss1 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - classifier.fit(input_fn=input_fn, steps=2000) - loss2 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss2, loss1) - self.assertLess(loss2, 0.01) - - def testJointLinearModel(self): - """Tests that loss goes down with training.""" - - def input_fn(): - return { - 'age': - sparse_tensor.SparseTensor( - values=['1'], indices=[[0, 0]], dense_shape=[1, 1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column.sparse_column_with_hash_bucket('language', 100) - age = feature_column.sparse_column_with_hash_bucket('age', 2) - - head = head_lib.multi_class_head(n_classes=2) - classifier = _joint_linear_estimator(head, feature_columns=[age, language]) - - classifier.fit(input_fn=input_fn, steps=1000) - loss1 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - classifier.fit(input_fn=input_fn, steps=2000) - loss2 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss2, loss1) - self.assertLess(loss2, 0.01) - - def testDNNModel(self): - """Tests multi-class classification using matrix data as input.""" - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - head = head_lib.multi_class_head(n_classes=3) - classifier = _dnn_estimator( - head, feature_columns=cont_features, hidden_units=[3, 3]) - - classifier.fit(input_fn=_iris_input_fn, steps=1000) - classifier.evaluate(input_fn=_iris_input_fn, steps=100) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/constants.py b/tensorflow/contrib/learn/python/learn/estimators/constants.py deleted file mode 100644 index d2548946bc7..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/constants.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Constants regarding Estimators (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -class ProblemType(object): - """Enum-like values for the type of problem that the model solves. - - THIS CLASS IS DEPRECATED. - - These values are used when exporting the model to produce the appropriate - signature function for serving. - - The following values are supported: - UNSPECIFIED: Produces a predict signature_fn. - CLASSIFICATION: Produces a classify signature_fn. - LINEAR_REGRESSION: Produces a regression signature_fn. - LOGISTIC_REGRESSION: Produces a classify signature_fn. - """ - UNSPECIFIED = 0 - CLASSIFICATION = 1 - LINEAR_REGRESSION = 2 - LOGISTIC_REGRESSION = 3 - - -# CollectionDef key for the input feature keys. -# TODO(b/34388557): This is a stopgap; please follow the bug to learn of changes -COLLECTION_DEF_KEY_FOR_INPUT_FEATURE_KEYS = "input_feature_keys" diff --git a/tensorflow/contrib/learn/python/learn/estimators/debug.py b/tensorflow/contrib/learn/python/learn/estimators/debug.py deleted file mode 100644 index 24b067b7e38..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/debug.py +++ /dev/null @@ -1,337 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Debug estimators (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. - -Debug estimators are bias-only estimators that can be used for debugging -and as simple baselines. - -Example: - -``` -# Build DebugClassifier -classifier = DebugClassifier() - -# Input builders -def input_fn_train: # returns x, y (where y represents label's class index). - pass - -def input_fn_eval: # returns x, y (where y represents label's class index). - pass - -# Fit model. -classifier.fit(input_fn=input_fn_train) - -# Evaluate cross entropy between the test and train labels. -loss = classifier.evaluate(input_fn=input_fn_eval)["loss"] - -# predict_classes outputs the most commonly seen class in training. -predicted_label = classifier.predict_classes(new_samples) - -# predict_proba outputs the class distribution from training. -label_distribution = classifier.predict_proba(new_samples) -``` -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.layers.python.layers import optimizers -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops - - -def _get_feature_dict(features): - if isinstance(features, dict): - return features - return {"": features} - - -def debug_model_fn(features, labels, mode, params, config=None): - """Model_fn for debug models. - - Args: - features: `Tensor` or dict of `Tensor` (depends on data passed to `fit`). - labels: Labels that are compatible with the `_Head` instance in `params`. - mode: Defines whether this is training, evaluation or prediction. - See `ModeKeys`. - params: A dict of hyperparameters containing: - * head: A `_Head` instance. - config: `RunConfig` object to configure the runtime settings. - - Raises: - KeyError: If weight column is specified but not present. - ValueError: If features is an empty dictionary. - - Returns: - A `ModelFnOps` instance. - """ - del config # Unused. - - features = _get_feature_dict(features) - if not features: - raise ValueError("Features cannot be empty.") - - head = params["head"] - size_checks = [] - batch_size = None - - # The first dimension is assumed to be a batch size and must be consistent - # among all of the features. - for feature in features.values(): - first_dim = array_ops.shape(feature)[0] - if batch_size is None: - batch_size = first_dim - else: - size_checks.append(check_ops.assert_equal(batch_size, first_dim)) - - with ops.control_dependencies(size_checks): - logits = array_ops.zeros([batch_size, head.logits_dimension]) - - def train_op_fn(loss): - return optimizers.optimize_loss( - loss, global_step=None, learning_rate=0.3, optimizer="Adagrad") - - return head.create_model_fn_ops( - features=features, - labels=labels, - mode=mode, - train_op_fn=train_op_fn, - logits=logits) - - -class DebugClassifier(estimator.Estimator): - """A classifier for TensorFlow Debug models. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Example: - - ```python - - # Build DebugClassifier - classifier = DebugClassifier() - - # Input builders - def input_fn_train: # returns x, y (where y represents label's class index). - pass - - def input_fn_eval: # returns x, y (where y represents label's class index). - pass - - # Fit model. - classifier.fit(input_fn=input_fn_train) - - # Evaluate cross entropy between the test and train labels. - loss = classifier.evaluate(input_fn=input_fn_eval)["loss"] - - # predict_class outputs the most commonly seen class in training. - predicted_label = classifier.predict_class(new_samples) - - # predict_proba outputs the class distribution from training. - label_distribution = classifier.predict_proba(new_samples) - ``` - - Input of `fit` and `evaluate` should have following features, - otherwise there will be a `KeyError`: - - * if `weight_column_name` is not `None`, a feature with - `key=weight_column_name` whose value is a `Tensor`. - """ - - def __init__(self, - model_dir=None, - n_classes=2, - weight_column_name=None, - config=None, - feature_engineering_fn=None, - label_keys=None): - """Initializes a DebugClassifier instance. - - Args: - model_dir: Directory to save model parameters, graph and etc. This can - also be used to load checkpoints from the directory into a estimator to - continue training a previously saved model. - n_classes: number of label classes. Default is binary classification. - It must be greater than 1. Note: Class labels are integers representing - the class index (i.e. values from 0 to n_classes-1). For arbitrary - label values (e.g. string labels), convert to class indices first. - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns - features and labels which will be fed into the model. - label_keys: Optional list of strings with size `[n_classes]` defining the - label vocabulary. Only supported for `n_classes` > 2. - Returns: - A `DebugClassifier` estimator. - - Raises: - ValueError: If `n_classes` < 2. - """ - params = {"head": - head_lib.multi_class_head( - n_classes=n_classes, - weight_column_name=weight_column_name, - enable_centered_bias=True, - label_keys=label_keys)} - - super(DebugClassifier, self).__init__( - model_fn=debug_model_fn, - model_dir=model_dir, - config=config, - params=params, - feature_engineering_fn=feature_engineering_fn) - - def predict_classes(self, input_fn=None, batch_size=None): - """Returns predicted classes for given features. - - Args: - input_fn: Input function. - batch_size: Override default batch size. - - Returns: - An iterable of predicted classes. Each predicted class is represented by - its class index (i.e. integer from 0 to n_classes-1). - """ - key = prediction_key.PredictionKey.CLASSES - preds = self.predict( - input_fn=input_fn, batch_size=batch_size, outputs=[key]) - return (pred[key] for pred in preds) - - def predict_proba(self, - input_fn=None, - batch_size=None): - """Returns prediction probabilities for given features. - - Args: - input_fn: Input function. - batch_size: Override default batch size. - - Returns: - An iterable of predicted probabilities with shape [batch_size, n_classes]. - """ - key = prediction_key.PredictionKey.PROBABILITIES - preds = self.predict( - input_fn=input_fn, - batch_size=batch_size, - outputs=[key]) - return (pred[key] for pred in preds) - - -class DebugRegressor(estimator.Estimator): - """A regressor for TensorFlow Debug models. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Example: - - ```python - - # Build DebugRegressor - regressor = DebugRegressor() - - # Input builders - def input_fn_train: # returns x, y (where y represents label's class index). - pass - - def input_fn_eval: # returns x, y (where y represents label's class index). - pass - - # Fit model. - regressor.fit(input_fn=input_fn_train) - - # Evaluate squared-loss between the test and train targets. - loss = regressor.evaluate(input_fn=input_fn_eval)["loss"] - - # predict_scores outputs mean value seen during training. - predicted_targets = regressor.predict_scores(new_samples) - ``` - - Input of `fit` and `evaluate` should have following features, - otherwise there will be a `KeyError`: - - * if `weight_column_name` is not `None`, a feature with - `key=weight_column_name` whose value is a `Tensor`. - """ - - def __init__(self, - model_dir=None, - label_dimension=1, - weight_column_name=None, - config=None, - feature_engineering_fn=None): - """Initializes a DebugRegressor instance. - - Args: - model_dir: Directory to save model parameters, graph and etc. This can - also be used to load checkpoints from the directory into a estimator to - continue training a previously saved model. - label_dimension: Number of regression targets per example. This is the - size of the last dimension of the labels and logits `Tensor` objects - (typically, these have shape `[batch_size, label_dimension]`). - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns - features and labels which will be fed into the model. - Returns: - A `DebugRegressor` estimator. - """ - - params = { - "head": - head_lib.regression_head( - weight_column_name=weight_column_name, - label_dimension=label_dimension, - enable_centered_bias=True) - } - - super(DebugRegressor, self).__init__( - model_fn=debug_model_fn, - model_dir=model_dir, - config=config, - params=params, - feature_engineering_fn=feature_engineering_fn) - - def predict_scores(self, input_fn=None, batch_size=None): - """Returns predicted scores for given features. - - Args: - input_fn: Input function. - batch_size: Override default batch size. - - Returns: - An iterable of predicted scores. - """ - key = prediction_key.PredictionKey.SCORES - preds = self.predict( - input_fn=input_fn, batch_size=batch_size, outputs=[key]) - return (pred[key] for pred in preds) diff --git a/tensorflow/contrib/learn/python/learn/estimators/debug_test.py b/tensorflow/contrib/learn/python/learn/estimators/debug_test.py deleted file mode 100644 index ab0ce6d581a..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/debug_test.py +++ /dev/null @@ -1,855 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Debug estimators.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import functools -import operator -import tempfile - -import numpy as np - -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.layers.python.layers import feature_column_ops -from tensorflow.contrib.learn.python.learn import experiment -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.contrib.learn.python.learn.estimators import _sklearn -from tensorflow.contrib.learn.python.learn.estimators import debug -from tensorflow.contrib.learn.python.learn.estimators import estimator_test_utils -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.contrib.learn.python.learn.estimators import test_data -from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec -from tensorflow.contrib.metrics.python.ops import metric_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test -from tensorflow.python.training import input as input_lib - -NUM_EXAMPLES = 100 -N_CLASSES = 5 # Cardinality of multiclass labels. -LABEL_DIMENSION = 3 # Dimensionality of regression labels. - - -def _train_test_split(features_and_labels): - features, labels = features_and_labels - train_set = (features[:int(len(features) / 2)], - labels[:int(len(features) / 2)]) - test_set = (features[int(len(features) / 2):], - labels[int(len(features) / 2):]) - return train_set, test_set - - -def _input_fn_builder(features, labels): - - def input_fn(): - feature_dict = {'features': constant_op.constant(features)} - my_labels = labels - if my_labels is not None: - my_labels = constant_op.constant(my_labels) - return feature_dict, my_labels - - return input_fn - - -class DebugClassifierTest(test.TestCase): - - def setUp(self): - np.random.seed(100) - self.features = np.random.rand(NUM_EXAMPLES, 5) - self.labels = np.random.choice( - range(N_CLASSES), p=[0.1, 0.3, 0.4, 0.1, 0.1], size=NUM_EXAMPLES) - self.binary_labels = np.random.choice( - range(2), p=[0.2, 0.8], size=NUM_EXAMPLES) - self.binary_float_labels = np.random.choice( - range(2), p=[0.2, 0.8], size=NUM_EXAMPLES) - - def testPredict(self): - """Tests that DebugClassifier outputs the majority class.""" - (train_features, train_labels), (test_features, - test_labels) = _train_test_split( - [self.features, self.labels]) - majority_class, _ = max( - collections.Counter(train_labels).items(), key=operator.itemgetter(1)) - expected_prediction = np.vstack( - [[majority_class] for _ in range(test_labels.shape[0])]) - - classifier = debug.DebugClassifier(n_classes=N_CLASSES) - classifier.fit( - input_fn=_input_fn_builder(train_features, train_labels), steps=50) - - pred = classifier.predict_classes( - input_fn=_input_fn_builder(test_features, None)) - self.assertAllEqual(expected_prediction, np.vstack(pred)) - - def testPredictBinary(self): - """Same as above for binary predictions.""" - (train_features, train_labels), (test_features, - test_labels) = _train_test_split( - [self.features, self.binary_labels]) - - majority_class, _ = max( - collections.Counter(train_labels).items(), key=operator.itemgetter(1)) - expected_prediction = np.vstack( - [[majority_class] for _ in range(test_labels.shape[0])]) - - classifier = debug.DebugClassifier(n_classes=2) - classifier.fit( - input_fn=_input_fn_builder(train_features, train_labels), steps=50) - - pred = classifier.predict_classes( - input_fn=_input_fn_builder(test_features, None)) - self.assertAllEqual(expected_prediction, np.vstack(pred)) - - (train_features, - train_labels), (test_features, test_labels) = _train_test_split( - [self.features, self.binary_float_labels]) - - majority_class, _ = max( - collections.Counter(train_labels).items(), key=operator.itemgetter(1)) - expected_prediction = np.vstack( - [[majority_class] for _ in range(test_labels.shape[0])]) - - classifier = debug.DebugClassifier(n_classes=2) - classifier.fit( - input_fn=_input_fn_builder(train_features, train_labels), steps=50) - - pred = classifier.predict_classes( - input_fn=_input_fn_builder(test_features, None)) - self.assertAllEqual(expected_prediction, np.vstack(pred)) - - def testPredictProba(self): - """Tests that DebugClassifier outputs observed class distribution.""" - (train_features, train_labels), (test_features, - test_labels) = _train_test_split( - [self.features, self.labels]) - - class_distribution = np.zeros((1, N_CLASSES)) - for label in train_labels: - class_distribution[0, label] += 1 - class_distribution /= len(train_labels) - - expected_prediction = np.vstack( - [class_distribution for _ in range(test_labels.shape[0])]) - - classifier = debug.DebugClassifier(n_classes=N_CLASSES) - classifier.fit( - input_fn=_input_fn_builder(train_features, train_labels), steps=50) - - pred = classifier.predict_proba( - input_fn=_input_fn_builder(test_features, None)) - - self.assertAllClose(expected_prediction, np.vstack(pred), atol=0.1) - - def testPredictProbaBinary(self): - """Same as above but for binary classification.""" - (train_features, train_labels), (test_features, - test_labels) = _train_test_split( - [self.features, self.binary_labels]) - - class_distribution = np.zeros((1, 2)) - for label in train_labels: - class_distribution[0, label] += 1 - class_distribution /= len(train_labels) - - expected_prediction = np.vstack( - [class_distribution for _ in range(test_labels.shape[0])]) - - classifier = debug.DebugClassifier(n_classes=2) - classifier.fit( - input_fn=_input_fn_builder(train_features, train_labels), steps=50) - - pred = classifier.predict_proba( - input_fn=_input_fn_builder(test_features, None)) - - self.assertAllClose(expected_prediction, np.vstack(pred), atol=0.1) - - (train_features, - train_labels), (test_features, test_labels) = _train_test_split( - [self.features, self.binary_float_labels]) - - class_distribution = np.zeros((1, 2)) - for label in train_labels: - class_distribution[0, int(label)] += 1 - class_distribution /= len(train_labels) - - expected_prediction = np.vstack( - [class_distribution for _ in range(test_labels.shape[0])]) - - classifier = debug.DebugClassifier(n_classes=2) - classifier.fit( - input_fn=_input_fn_builder(train_features, train_labels), steps=50) - - pred = classifier.predict_proba( - input_fn=_input_fn_builder(test_features, None)) - - self.assertAllClose(expected_prediction, np.vstack(pred), atol=0.1) - - def testExperimentIntegration(self): - exp = experiment.Experiment( - estimator=debug.DebugClassifier(n_classes=3), - train_input_fn=test_data.iris_input_multiclass_fn, - eval_input_fn=test_data.iris_input_multiclass_fn) - exp.test() - - def _assertInRange(self, expected_min, expected_max, actual): - self.assertLessEqual(expected_min, actual) - self.assertGreaterEqual(expected_max, actual) - - def testEstimatorContract(self): - estimator_test_utils.assert_estimator_contract(self, debug.DebugClassifier) - - def testLogisticRegression_MatrixData(self): - """Tests binary classification using matrix data as input.""" - classifier = debug.DebugClassifier( - config=run_config.RunConfig(tf_random_seed=1)) - input_fn = test_data.iris_input_logistic_fn - classifier.fit(input_fn=input_fn, steps=5) - scores = classifier.evaluate(input_fn=input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - self.assertIn('loss', scores) - - def testLogisticRegression_MatrixData_Labels1D(self): - """Same as the last test, but label shape is [100] instead of [100, 1].""" - - def _input_fn(): - iris = test_data.prepare_iris_data_for_logistic_regression() - return { - 'feature': constant_op.constant(iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=[100], dtype=dtypes.int32) - - classifier = debug.DebugClassifier( - config=run_config.RunConfig(tf_random_seed=1)) - classifier.fit(input_fn=_input_fn, steps=5) - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores) - - def testLogisticRegression_NpMatrixData(self): - """Tests binary classification using numpy matrix data as input.""" - iris = test_data.prepare_iris_data_for_logistic_regression() - train_x = iris.data - train_y = iris.target - classifier = debug.DebugClassifier( - config=run_config.RunConfig(tf_random_seed=1)) - classifier.fit(x=train_x, y=train_y, steps=5) - scores = classifier.evaluate(x=train_x, y=train_y, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - - def _assertBinaryPredictions(self, expected_len, predictions): - self.assertEqual(expected_len, len(predictions)) - for prediction in predictions: - self.assertIn(prediction, (0, 1)) - - def _assertProbabilities(self, expected_batch_size, expected_n_classes, - probabilities): - self.assertEqual(expected_batch_size, len(probabilities)) - for b in range(expected_batch_size): - self.assertEqual(expected_n_classes, len(probabilities[b])) - for i in range(expected_n_classes): - self._assertInRange(0.0, 1.0, probabilities[b][i]) - - def testLogisticRegression_TensorData(self): - """Tests binary classification using tensor data as input.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[.8], [0.2], [.1]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([[1], [0], [0]], dtype=dtypes.int32) - - classifier = debug.DebugClassifier(n_classes=2) - - classifier.fit(input_fn=_input_fn, steps=50) - - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - self.assertIn('loss', scores) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = list(classifier.predict_classes(input_fn=predict_input_fn)) - self._assertBinaryPredictions(3, predictions) - - def testLogisticRegression_FloatLabel(self): - """Tests binary classification with float labels.""" - - def _input_fn_float_label(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[50], [20], [10]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - labels = constant_op.constant([[0.8], [0.], [0.2]], dtype=dtypes.float32) - return features, labels - - classifier = debug.DebugClassifier(n_classes=2) - - classifier.fit(input_fn=_input_fn_float_label, steps=50) - - predict_input_fn = functools.partial(_input_fn_float_label, num_epochs=1) - predictions = list(classifier.predict_classes(input_fn=predict_input_fn)) - self._assertBinaryPredictions(3, predictions) - predictions_proba = list( - classifier.predict_proba(input_fn=predict_input_fn)) - self._assertProbabilities(3, 2, predictions_proba) - - def testMultiClass_MatrixData(self): - """Tests multi-class classification using matrix data as input.""" - classifier = debug.DebugClassifier(n_classes=3) - - input_fn = test_data.iris_input_multiclass_fn - classifier.fit(input_fn=input_fn, steps=200) - scores = classifier.evaluate(input_fn=input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - self.assertIn('loss', scores) - - def testMultiClass_MatrixData_Labels1D(self): - """Same as the last test, but label shape is [150] instead of [150, 1].""" - - def _input_fn(): - iris = base.load_iris() - return { - 'feature': constant_op.constant(iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=[150], dtype=dtypes.int32) - - classifier = debug.DebugClassifier(n_classes=3) - - classifier.fit(input_fn=_input_fn, steps=200) - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - - def testMultiClass_NpMatrixData(self): - """Tests multi-class classification using numpy matrix data as input.""" - iris = base.load_iris() - train_x = iris.data - train_y = iris.target - classifier = debug.DebugClassifier(n_classes=3) - classifier.fit(x=train_x, y=train_y, steps=200) - scores = classifier.evaluate(x=train_x, y=train_y, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - - def testMultiClass_StringLabel(self): - """Tests multi-class classification with string labels.""" - - def _input_fn_train(): - labels = constant_op.constant([['foo'], ['bar'], ['baz'], ['bar']]) - features = { - 'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - } - return features, labels - - classifier = debug.DebugClassifier( - n_classes=3, label_keys=['foo', 'bar', 'baz']) - - classifier.fit(input_fn=_input_fn_train, steps=5) - scores = classifier.evaluate(input_fn=_input_fn_train, steps=1) - self.assertIn('loss', scores) - - def testLoss(self): - """Tests loss calculation.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # The logistic prediction should be (y = 0.25). - labels = constant_op.constant([[1], [0], [0], [0]]) - features = { - 'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - } - return features, labels - - classifier = debug.DebugClassifier(n_classes=2) - - classifier.fit(input_fn=_input_fn_train, steps=5) - scores = classifier.evaluate(input_fn=_input_fn_train, steps=1) - self.assertIn('loss', scores) - - def testLossWithWeights(self): - """Tests loss calculation with weights.""" - - def _input_fn_train(): - # 4 rows with equal weight, one of them (y = x), three of them (y=Not(x)) - # The logistic prediction should be (y = 0.25). - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - def _input_fn_eval(): - # 4 rows, with different weights. - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[7.], [1.], [1.], [1.]]) - } - return features, labels - - classifier = debug.DebugClassifier( - weight_column_name='w', - n_classes=2, - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn_train, steps=5) - scores = classifier.evaluate(input_fn=_input_fn_eval, steps=1) - self.assertIn('loss', scores) - - def testTrainWithWeights(self): - """Tests training with given weight column.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # First row has more weight than others. Model should fit (y=x) better - # than (y=Not(x)) due to the relative higher weight of the first row. - labels = constant_op.constant([[1], [0], [0], [0]]) - features = { - 'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[100.], [3.], [2.], [2.]]) - } - return features, labels - - def _input_fn_eval(): - # Create 4 rows (y = x) - labels = constant_op.constant([[1], [1], [1], [1]]) - features = { - 'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - classifier = debug.DebugClassifier(weight_column_name='w') - - classifier.fit(input_fn=_input_fn_train, steps=5) - scores = classifier.evaluate(input_fn=_input_fn_eval, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - - def testCustomMetrics(self): - """Tests custom evaluation metrics.""" - - def _input_fn(num_epochs=None): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - labels = constant_op.constant([[1], [0], [0], [0]]) - features = { - 'x': - input_lib.limit_epochs( - array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - num_epochs=num_epochs), - } - return features, labels - - def _my_metric_op(predictions, labels): - # For the case of binary classification, the 2nd column of "predictions" - # denotes the model predictions. - labels = math_ops.cast(labels, dtypes.float32) - predictions = array_ops.strided_slice( - predictions, [0, 1], [-1, 2], end_mask=1) - labels = math_ops.cast(labels, predictions.dtype) - return math_ops.reduce_sum(math_ops.multiply(predictions, labels)) - - classifier = debug.DebugClassifier( - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn, steps=5) - scores = classifier.evaluate( - input_fn=_input_fn, - steps=5, - metrics={ - 'my_accuracy': - MetricSpec( - metric_fn=metric_ops.streaming_accuracy, - prediction_key='classes'), - 'my_precision': - MetricSpec( - metric_fn=metric_ops.streaming_precision, - prediction_key='classes'), - 'my_metric': - MetricSpec( - metric_fn=_my_metric_op, prediction_key='probabilities') - }) - self.assertTrue( - set(['loss', 'my_accuracy', 'my_precision', 'my_metric']).issubset( - set(scores.keys()))) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = np.array( - list(classifier.predict_classes(input_fn=predict_input_fn))) - self.assertEqual( - _sklearn.accuracy_score([1, 0, 0, 0], predictions), - scores['my_accuracy']) - - # Test the case where the 2nd element of the key is neither "classes" nor - # "probabilities". - with self.assertRaisesRegexp(KeyError, 'bad_type'): - classifier.evaluate( - input_fn=_input_fn, - steps=5, - metrics={ - 'bad_name': - MetricSpec( - metric_fn=metric_ops.streaming_auc, - prediction_key='bad_type') - }) - - def testTrainSaveLoad(self): - """Tests that insures you can save and reload a trained model.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[.8], [.2], [.1]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([[1], [0], [0]], dtype=dtypes.int32) - - model_dir = tempfile.mkdtemp() - classifier = debug.DebugClassifier( - model_dir=model_dir, - n_classes=3, - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn, steps=5) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions1 = classifier.predict_classes(input_fn=predict_input_fn) - del classifier - - classifier2 = debug.DebugClassifier( - model_dir=model_dir, - n_classes=3, - config=run_config.RunConfig(tf_random_seed=1)) - predictions2 = classifier2.predict_classes(input_fn=predict_input_fn) - self.assertEqual(list(predictions1), list(predictions2)) - - def testExport(self): - """Tests export model for servo.""" - - def input_fn(): - return { - 'age': - constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column.sparse_column_with_hash_bucket('language', 100) - feature_columns = [ - feature_column.real_valued_column('age'), - feature_column.embedding_column(language, dimension=1) - ] - - classifier = debug.DebugClassifier( - config=run_config.RunConfig(tf_random_seed=1)) - classifier.fit(input_fn=input_fn, steps=5) - - def default_input_fn(unused_estimator, examples): - return feature_column_ops.parse_feature_columns_from_examples( - examples, feature_columns) - - export_dir = tempfile.mkdtemp() - classifier.export(export_dir, input_fn=default_input_fn) - - -class DebugRegressorTest(test.TestCase): - - def setUp(self): - np.random.seed(100) - self.features = np.random.rand(NUM_EXAMPLES, 5) - self.targets = np.random.rand(NUM_EXAMPLES, LABEL_DIMENSION) - - def testPredictScores(self): - """Tests that DebugRegressor outputs the mean target.""" - (train_features, train_labels), (test_features, - test_labels) = _train_test_split( - [self.features, self.targets]) - mean_target = np.mean(train_labels, 0) - expected_prediction = np.vstack( - [mean_target for _ in range(test_labels.shape[0])]) - - classifier = debug.DebugRegressor(label_dimension=LABEL_DIMENSION) - classifier.fit( - input_fn=_input_fn_builder(train_features, train_labels), steps=50) - - pred = classifier.predict_scores( - input_fn=_input_fn_builder(test_features, None)) - self.assertAllClose(expected_prediction, np.vstack(pred), atol=0.1) - - def testExperimentIntegration(self): - exp = experiment.Experiment( - estimator=debug.DebugRegressor(), - train_input_fn=test_data.iris_input_logistic_fn, - eval_input_fn=test_data.iris_input_logistic_fn) - exp.test() - - def testEstimatorContract(self): - estimator_test_utils.assert_estimator_contract(self, debug.DebugRegressor) - - def testRegression_MatrixData(self): - """Tests regression using matrix data as input.""" - regressor = debug.DebugRegressor( - config=run_config.RunConfig(tf_random_seed=1)) - input_fn = test_data.iris_input_logistic_fn - regressor.fit(input_fn=input_fn, steps=200) - scores = regressor.evaluate(input_fn=input_fn, steps=1) - self.assertIn('loss', scores) - - def testRegression_MatrixData_Labels1D(self): - """Same as the last test, but label shape is [100] instead of [100, 1].""" - - def _input_fn(): - iris = test_data.prepare_iris_data_for_logistic_regression() - return { - 'feature': constant_op.constant(iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=[100], dtype=dtypes.int32) - - regressor = debug.DebugRegressor( - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=200) - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores) - - def testRegression_NpMatrixData(self): - """Tests binary classification using numpy matrix data as input.""" - iris = test_data.prepare_iris_data_for_logistic_regression() - train_x = iris.data - train_y = iris.target - regressor = debug.DebugRegressor( - config=run_config.RunConfig(tf_random_seed=1)) - regressor.fit(x=train_x, y=train_y, steps=200) - scores = regressor.evaluate(x=train_x, y=train_y, steps=1) - self.assertIn('loss', scores) - - def testRegression_TensorData(self): - """Tests regression using tensor data as input.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[.8], [.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([1., 0., 0.2], dtype=dtypes.float32) - - regressor = debug.DebugRegressor( - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=200) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores) - - def testLoss(self): - """Tests loss calculation.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # The algorithm should learn (y = 0.25). - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - } - return features, labels - - regressor = debug.DebugRegressor( - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn_train, steps=5) - scores = regressor.evaluate(input_fn=_input_fn_train, steps=1) - self.assertIn('loss', scores) - - def testLossWithWeights(self): - """Tests loss calculation with weights.""" - - def _input_fn_train(): - # 4 rows with equal weight, one of them (y = x), three of them (y=Not(x)) - # The algorithm should learn (y = 0.25). - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - def _input_fn_eval(): - # 4 rows, with different weights. - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[7.], [1.], [1.], [1.]]) - } - return features, labels - - regressor = debug.DebugRegressor( - weight_column_name='w', config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn_train, steps=5) - scores = regressor.evaluate(input_fn=_input_fn_eval, steps=1) - self.assertIn('loss', scores) - - def testTrainWithWeights(self): - """Tests training with given weight column.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # First row has more weight than others. Model should fit (y=x) better - # than (y=Not(x)) due to the relative higher weight of the first row. - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[100.], [3.], [2.], [2.]]) - } - return features, labels - - def _input_fn_eval(): - # Create 4 rows (y = x) - labels = constant_op.constant([[1.], [1.], [1.], [1.]]) - features = { - 'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - regressor = debug.DebugRegressor( - weight_column_name='w', config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn_train, steps=5) - scores = regressor.evaluate(input_fn=_input_fn_eval, steps=1) - self.assertIn('loss', scores) - - def testCustomMetrics(self): - """Tests custom evaluation metrics.""" - - def _input_fn(num_epochs=None): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': - input_lib.limit_epochs( - array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - num_epochs=num_epochs), - } - return features, labels - - def _my_metric_op(predictions, labels): - return math_ops.reduce_sum(math_ops.multiply(predictions, labels)) - - regressor = debug.DebugRegressor( - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=5) - scores = regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - 'my_error': - MetricSpec( - metric_fn=metric_ops.streaming_mean_squared_error, - prediction_key='scores'), - 'my_metric': - MetricSpec(metric_fn=_my_metric_op, prediction_key='scores') - }) - self.assertIn('loss', set(scores.keys())) - self.assertIn('my_error', set(scores.keys())) - self.assertIn('my_metric', set(scores.keys())) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = np.array( - list(regressor.predict_scores(input_fn=predict_input_fn))) - self.assertAlmostEqual( - _sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions), - scores['my_error']) - - # Tests the case where the prediction_key is not "scores". - with self.assertRaisesRegexp(KeyError, 'bad_type'): - regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - 'bad_name': - MetricSpec( - metric_fn=metric_ops.streaming_auc, - prediction_key='bad_type') - }) - - def testTrainSaveLoad(self): - """Tests that insures you can save and reload a trained model.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([1., 0., 0.2], dtype=dtypes.float32) - - model_dir = tempfile.mkdtemp() - regressor = debug.DebugRegressor( - model_dir=model_dir, config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=5) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = list(regressor.predict_scores(input_fn=predict_input_fn)) - del regressor - - regressor2 = debug.DebugRegressor( - model_dir=model_dir, config=run_config.RunConfig(tf_random_seed=1)) - predictions2 = list(regressor2.predict_scores(input_fn=predict_input_fn)) - self.assertAllClose(predictions, predictions2) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn.py b/tensorflow/contrib/learn/python/learn/estimators/dnn.py deleted file mode 100644 index 7e4e3a8d287..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/dnn.py +++ /dev/null @@ -1,935 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Deep Neural Network estimators (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import six - -from tensorflow.contrib import layers -from tensorflow.contrib.framework import deprecated -from tensorflow.contrib.framework import deprecated_arg_values -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.layers.python.layers import optimizers -from tensorflow.contrib.learn.python.learn import metric_spec -from tensorflow.contrib.learn.python.learn.estimators import dnn_linear_combined -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.contrib.learn.python.learn.utils import export -from tensorflow.python.feature_column import feature_column_lib as fc_core -from tensorflow.python.ops import nn -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import variable_scope -from tensorflow.python.summary import summary -from tensorflow.python.training import training_util - -# The default learning rate of 0.05 is a historical artifact of the initial -# implementation, but seems a reasonable choice. -_LEARNING_RATE = 0.05 - - -def _get_feature_dict(features): - if isinstance(features, dict): - return features - return {"": features} - - -def _get_optimizer(optimizer): - if callable(optimizer): - return optimizer() - else: - return optimizer - - -_ACTIVATION_FUNCTIONS = { - "relu": nn.relu, - "tanh": nn.tanh, - "sigmoid": nn.sigmoid -} - - -def _get_activation_fn(activation_fn): - if not isinstance(activation_fn, six.string_types): - return activation_fn - if activation_fn not in _ACTIVATION_FUNCTIONS.keys(): - raise ValueError("Activation name should be one of [%s], you provided %s." % - (", ".join(_ACTIVATION_FUNCTIONS.keys()), activation_fn)) - return _ACTIVATION_FUNCTIONS[activation_fn] - - -def _add_hidden_layer_summary(value, tag): - summary.scalar("%s_fraction_of_zero_values" % tag, nn.zero_fraction(value)) - summary.histogram("%s_activation" % tag, value) - - -def _dnn_model_fn(features, labels, mode, params, config=None): - """Deep Neural Net model_fn. - - Args: - features: `Tensor` or dict of `Tensor` (depends on data passed to `fit`). - labels: `Tensor` of shape [batch_size, 1] or [batch_size] labels of - dtype `int32` or `int64` in the range `[0, n_classes)`. - mode: Defines whether this is training, evaluation or prediction. - See `ModeKeys`. - params: A dict of hyperparameters. - The following hyperparameters are expected: - * head: A `_Head` instance. - * hidden_units: List of hidden units per layer. - * feature_columns: An iterable containing all the feature columns used by - the model. - * optimizer: string, `Optimizer` object, or callable that defines the - optimizer to use for training. If `None`, will use the Adagrad - optimizer with a default learning rate of 0.05. - * activation_fn: Activation function applied to each layer. If `None`, - will use `tf.nn.relu`. Note that a string containing the unqualified - name of the op may also be provided, e.g., "relu", "tanh", or - "sigmoid". - * dropout: When not `None`, the probability we will drop out a given - coordinate. - * gradient_clip_norm: A float > 0. If provided, gradients are - clipped to their global norm with this clipping ratio. - * embedding_lr_multipliers: Optional. A dictionary from - `EmbeddingColumn` to a `float` multiplier. Multiplier will be used to - multiply with learning rate for the embedding variables. - * input_layer_min_slice_size: Optional. The min slice size of input layer - partitions. If not provided, will use the default of 64M. - config: `RunConfig` object to configure the runtime settings. - - Returns: - predictions: A dict of `Tensor` objects. - loss: A scalar containing the loss of the step. - train_op: The op for training. - """ - head = params["head"] - hidden_units = params["hidden_units"] - feature_columns = params["feature_columns"] - optimizer = params.get("optimizer") or "Adagrad" - activation_fn = _get_activation_fn(params.get("activation_fn")) - dropout = params.get("dropout") - gradient_clip_norm = params.get("gradient_clip_norm") - input_layer_min_slice_size = ( - params.get("input_layer_min_slice_size") or 64 << 20) - num_ps_replicas = config.num_ps_replicas if config else 0 - embedding_lr_multipliers = params.get("embedding_lr_multipliers", {}) - - features = _get_feature_dict(features) - parent_scope = "dnn" - - partitioner = partitioned_variables.min_max_variable_partitioner( - max_partitions=num_ps_replicas) - with variable_scope.variable_scope( - parent_scope, - values=tuple(six.itervalues(features)), - partitioner=partitioner): - input_layer_partitioner = ( - partitioned_variables.min_max_variable_partitioner( - max_partitions=num_ps_replicas, - min_slice_size=input_layer_min_slice_size)) - with variable_scope.variable_scope( - "input_from_feature_columns", - values=tuple(six.itervalues(features)), - partitioner=input_layer_partitioner) as input_layer_scope: - if all( - isinstance(fc, feature_column._FeatureColumn) # pylint: disable=protected-access - for fc in feature_columns - ): - net = layers.input_from_feature_columns( - columns_to_tensors=features, - feature_columns=feature_columns, - weight_collections=[parent_scope], - scope=input_layer_scope) - else: - net = fc_core.input_layer( - features=features, - feature_columns=feature_columns, - weight_collections=[parent_scope]) - - for layer_id, num_hidden_units in enumerate(hidden_units): - with variable_scope.variable_scope( - "hiddenlayer_%d" % layer_id, - values=(net,)) as hidden_layer_scope: - net = layers.fully_connected( - net, - num_hidden_units, - activation_fn=activation_fn, - variables_collections=[parent_scope], - scope=hidden_layer_scope) - if dropout is not None and mode == model_fn.ModeKeys.TRAIN: - net = layers.dropout(net, keep_prob=(1.0 - dropout)) - _add_hidden_layer_summary(net, hidden_layer_scope.name) - - with variable_scope.variable_scope( - "logits", - values=(net,)) as logits_scope: - logits = layers.fully_connected( - net, - head.logits_dimension, - activation_fn=None, - variables_collections=[parent_scope], - scope=logits_scope) - _add_hidden_layer_summary(logits, logits_scope.name) - - def _train_op_fn(loss): - """Returns the op to optimize the loss.""" - return optimizers.optimize_loss( - loss=loss, - global_step=training_util.get_global_step(), - learning_rate=_LEARNING_RATE, - optimizer=_get_optimizer(optimizer), - gradient_multipliers=( - dnn_linear_combined._extract_embedding_lr_multipliers( # pylint: disable=protected-access - embedding_lr_multipliers, parent_scope, - input_layer_scope.name)), - clip_gradients=gradient_clip_norm, - name=parent_scope, - # Empty summaries to prevent optimizers from logging training_loss. - summaries=[]) - - return head.create_model_fn_ops( - features=features, - mode=mode, - labels=labels, - train_op_fn=_train_op_fn, - logits=logits) - - -class DNNClassifier(estimator.Estimator): - """A classifier for TensorFlow DNN models. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Example: - - ```python - sparse_feature_a = sparse_column_with_hash_bucket(...) - sparse_feature_b = sparse_column_with_hash_bucket(...) - - sparse_feature_a_emb = embedding_column(sparse_id_column=sparse_feature_a, - ...) - sparse_feature_b_emb = embedding_column(sparse_id_column=sparse_feature_b, - ...) - - estimator = DNNClassifier( - feature_columns=[sparse_feature_a_emb, sparse_feature_b_emb], - hidden_units=[1024, 512, 256]) - - # Or estimator using the ProximalAdagradOptimizer optimizer with - # regularization. - estimator = DNNClassifier( - feature_columns=[sparse_feature_a_emb, sparse_feature_b_emb], - hidden_units=[1024, 512, 256], - optimizer=tf.compat.v1.train.ProximalAdagradOptimizer( - learning_rate=0.1, - l1_regularization_strength=0.001 - )) - - # Input builders - def input_fn_train: # returns x, y (where y represents label's class index). - pass - estimator.fit(input_fn=input_fn_train) - - def input_fn_eval: # returns x, y (where y represents label's class index). - pass - estimator.evaluate(input_fn=input_fn_eval) - - def input_fn_predict: # returns x, None - pass - # predict_classes returns class indices. - estimator.predict_classes(input_fn=input_fn_predict) - ``` - - If the user specifies `label_keys` in constructor, labels must be strings from - the `label_keys` vocabulary. Example: - - ```python - label_keys = ['label0', 'label1', 'label2'] - estimator = DNNClassifier( - feature_columns=[sparse_feature_a_emb, sparse_feature_b_emb], - hidden_units=[1024, 512, 256], - label_keys=label_keys) - - def input_fn_train: # returns x, y (where y is one of label_keys). - pass - estimator.fit(input_fn=input_fn_train) - - def input_fn_eval: # returns x, y (where y is one of label_keys). - pass - estimator.evaluate(input_fn=input_fn_eval) - def input_fn_predict: # returns x, None - # predict_classes returns one of label_keys. - estimator.predict_classes(input_fn=input_fn_predict) - ``` - - Input of `fit` and `evaluate` should have following features, - otherwise there will be a `KeyError`: - - * if `weight_column_name` is not `None`, a feature with - `key=weight_column_name` whose value is a `Tensor`. - * for each `column` in `feature_columns`: - - if `column` is a `SparseColumn`, a feature with `key=column.name` - whose `value` is a `SparseTensor`. - - if `column` is a `WeightedSparseColumn`, two features: the first with - `key` the id column name, the second with `key` the weight column name. - Both features' `value` must be a `SparseTensor`. - - if `column` is a `RealValuedColumn`, a feature with `key=column.name` - whose `value` is a `Tensor`. - """ - - def __init__(self, - hidden_units, - feature_columns, - model_dir=None, - n_classes=2, - weight_column_name=None, - optimizer=None, - activation_fn=nn.relu, - dropout=None, - gradient_clip_norm=None, - enable_centered_bias=False, - config=None, - feature_engineering_fn=None, - embedding_lr_multipliers=None, - input_layer_min_slice_size=None, - label_keys=None): - """Initializes a DNNClassifier instance. - - Args: - hidden_units: List of hidden units per layer. All layers are fully - connected. Ex. `[64, 32]` means first layer has 64 nodes and second one - has 32. - feature_columns: An iterable containing all the feature columns used by - the model. All items in the set should be instances of classes derived - from `FeatureColumn`. - model_dir: Directory to save model parameters, graph and etc. This can - also be used to load checkpoints from the directory into a estimator to - continue training a previously saved model. - n_classes: number of label classes. Default is binary classification. - It must be greater than 1. Note: Class labels are integers representing - the class index (i.e. values from 0 to n_classes-1). For arbitrary - label values (e.g. string labels), convert to class indices first. - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - optimizer: An instance of `tf.Optimizer` used to train the model. If - `None`, will use an Adagrad optimizer. - activation_fn: Activation function applied to each layer. If `None`, will - use tf.nn.relu. Note that a string containing the unqualified - name of the op may also be provided, e.g., "relu", "tanh", or "sigmoid". - dropout: When not `None`, the probability we will drop out a given - coordinate. - gradient_clip_norm: A float > 0. If provided, gradients are - clipped to their global norm with this clipping ratio. See - `tf.clip_by_global_norm` for more details. - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - embedding_lr_multipliers: Optional. A dictionary from `EmbeddingColumn` to - a `float` multiplier. Multiplier will be used to multiply with learning - rate for the embedding variables. - input_layer_min_slice_size: Optional. The min slice size of input layer - partitions. If not provided, will use the default of 64M. - label_keys: Optional list of strings with size `[n_classes]` defining the - label vocabulary. Only supported for `n_classes` > 2. - - Returns: - A `DNNClassifier` estimator. - - Raises: - ValueError: If `n_classes` < 2. - """ - self._feature_columns = tuple(feature_columns or []) - super(DNNClassifier, self).__init__( - model_fn=_dnn_model_fn, - model_dir=model_dir, - config=config, - params={ - "head": - head_lib.multi_class_head( - n_classes, - weight_column_name=weight_column_name, - enable_centered_bias=enable_centered_bias, - label_keys=label_keys), - "hidden_units": hidden_units, - "feature_columns": self._feature_columns, - "optimizer": optimizer, - "activation_fn": activation_fn, - "dropout": dropout, - "gradient_clip_norm": gradient_clip_norm, - "embedding_lr_multipliers": embedding_lr_multipliers, - "input_layer_min_slice_size": input_layer_min_slice_size, - }, - feature_engineering_fn=feature_engineering_fn) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, - estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - @deprecated_arg_values( - "2017-03-01", - "Please switch to predict_classes, or set `outputs` argument.", - outputs=None) - def predict(self, x=None, input_fn=None, batch_size=None, outputs=None, - as_iterable=True): - """Returns predictions for given features. - - By default, returns predicted classes. But this default will be dropped - soon. Users should either pass `outputs`, or call `predict_classes` method. - - Args: - x: features. - input_fn: Input function. If set, x must be None. - batch_size: Override default batch size. - outputs: list of `str`, name of the output to predict. - If `None`, returns classes. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted classes with shape [batch_size] (or an iterable - of predicted classes if as_iterable is True). Each predicted class is - represented by its class index (i.e. integer from 0 to n_classes-1). - If `outputs` is set, returns a dict of predictions. - """ - if not outputs: - return self.predict_classes( - x=x, - input_fn=input_fn, - batch_size=batch_size, - as_iterable=as_iterable) - return super(DNNClassifier, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=outputs, - as_iterable=as_iterable) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, - estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - def predict_classes(self, x=None, input_fn=None, batch_size=None, - as_iterable=True): - """Returns predicted classes for given features. - - Args: - x: features. - input_fn: Input function. If set, x must be None. - batch_size: Override default batch size. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted classes with shape [batch_size] (or an iterable - of predicted classes if as_iterable is True). Each predicted class is - represented by its class index (i.e. integer from 0 to n_classes-1). - """ - key = prediction_key.PredictionKey.CLASSES - preds = super(DNNClassifier, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=[key], - as_iterable=as_iterable) - if as_iterable: - return (pred[key] for pred in preds) - return preds[key].reshape(-1) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, - estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - def predict_proba(self, - x=None, - input_fn=None, - batch_size=None, - as_iterable=True): - """Returns predicted probabilities for given features. - - Args: - x: features. - input_fn: Input function. If set, x and y must be None. - batch_size: Override default batch size. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted probabilities with shape [batch_size, n_classes] - (or an iterable of predicted probabilities if as_iterable is True). - """ - key = prediction_key.PredictionKey.PROBABILITIES - preds = super(DNNClassifier, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=[key], - as_iterable=as_iterable) - if as_iterable: - return (pred[key] for pred in preds) - return preds[key] - - @deprecated("2017-03-25", "Please use Estimator.export_savedmodel() instead.") - def export(self, - export_dir, - input_fn=None, - input_feature_key=None, - use_deprecated_input_fn=True, - signature_fn=None, - default_batch_size=1, - exports_to_keep=None): - """See BaseEstimator.export.""" - - def default_input_fn(unused_estimator, examples): - return layers.parse_feature_columns_from_examples(examples, - self._feature_columns) - - return super(DNNClassifier, self).export( - export_dir=export_dir, - input_fn=input_fn or default_input_fn, - input_feature_key=input_feature_key, - use_deprecated_input_fn=use_deprecated_input_fn, - signature_fn=(signature_fn or - export.classification_signature_fn_with_prob), - prediction_key=prediction_key.PredictionKey.PROBABILITIES, - default_batch_size=default_batch_size, - exports_to_keep=exports_to_keep) - - -class DNNRegressor(estimator.Estimator): - """A regressor for TensorFlow DNN models. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Example: - - ```python - sparse_feature_a = sparse_column_with_hash_bucket(...) - sparse_feature_b = sparse_column_with_hash_bucket(...) - - sparse_feature_a_emb = embedding_column(sparse_id_column=sparse_feature_a, - ...) - sparse_feature_b_emb = embedding_column(sparse_id_column=sparse_feature_b, - ...) - - estimator = DNNRegressor( - feature_columns=[sparse_feature_a, sparse_feature_b], - hidden_units=[1024, 512, 256]) - - # Or estimator using the ProximalAdagradOptimizer optimizer with - # regularization. - estimator = DNNRegressor( - feature_columns=[sparse_feature_a, sparse_feature_b], - hidden_units=[1024, 512, 256], - optimizer=tf.compat.v1.train.ProximalAdagradOptimizer( - learning_rate=0.1, - l1_regularization_strength=0.001 - )) - - # Input builders - def input_fn_train: # returns x, y - pass - estimator.fit(input_fn=input_fn_train) - - def input_fn_eval: # returns x, y - pass - estimator.evaluate(input_fn=input_fn_eval) - def input_fn_predict: # returns x, None - pass - estimator.predict_scores(input_fn=input_fn_predict) - ``` - - Input of `fit` and `evaluate` should have following features, - otherwise there will be a `KeyError`: - - * if `weight_column_name` is not `None`, a feature with - `key=weight_column_name` whose value is a `Tensor`. - * for each `column` in `feature_columns`: - - if `column` is a `SparseColumn`, a feature with `key=column.name` - whose `value` is a `SparseTensor`. - - if `column` is a `WeightedSparseColumn`, two features: the first with - `key` the id column name, the second with `key` the weight column name. - Both features' `value` must be a `SparseTensor`. - - if `column` is a `RealValuedColumn`, a feature with `key=column.name` - whose `value` is a `Tensor`. - """ - - def __init__(self, - hidden_units, - feature_columns, - model_dir=None, - weight_column_name=None, - optimizer=None, - activation_fn=nn.relu, - dropout=None, - gradient_clip_norm=None, - enable_centered_bias=False, - config=None, - feature_engineering_fn=None, - label_dimension=1, - embedding_lr_multipliers=None, - input_layer_min_slice_size=None): - """Initializes a `DNNRegressor` instance. - - Args: - hidden_units: List of hidden units per layer. All layers are fully - connected. Ex. `[64, 32]` means first layer has 64 nodes and second one - has 32. - feature_columns: An iterable containing all the feature columns used by - the model. All items in the set should be instances of classes derived - from `FeatureColumn`. - model_dir: Directory to save model parameters, graph and etc. This can - also be used to load checkpoints from the directory into a estimator to - continue training a previously saved model. - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - optimizer: An instance of `tf.Optimizer` used to train the model. If - `None`, will use an Adagrad optimizer. - activation_fn: Activation function applied to each layer. If `None`, will - use `tf.nn.relu`. Note that a string containing the unqualified name of - the op may also be provided, e.g., "relu", "tanh", or "sigmoid". - dropout: When not `None`, the probability we will drop out a given - coordinate. - gradient_clip_norm: A `float` > 0. If provided, gradients are clipped - to their global norm with this clipping ratio. See - `tf.clip_by_global_norm` for more details. - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and - returns features and labels which will be fed - into the model. - label_dimension: Number of regression targets per example. This is the - size of the last dimension of the labels and logits `Tensor` objects - (typically, these have shape `[batch_size, label_dimension]`). - embedding_lr_multipliers: Optional. A dictionary from `EbeddingColumn` to - a `float` multiplier. Multiplier will be used to multiply with - learning rate for the embedding variables. - input_layer_min_slice_size: Optional. The min slice size of input layer - partitions. If not provided, will use the default of 64M. - - Returns: - A `DNNRegressor` estimator. - """ - self._feature_columns = tuple(feature_columns or []) - super(DNNRegressor, self).__init__( - model_fn=_dnn_model_fn, - model_dir=model_dir, - config=config, - params={ - "head": - head_lib.regression_head( - label_dimension=label_dimension, - weight_column_name=weight_column_name, - enable_centered_bias=enable_centered_bias), - "hidden_units": hidden_units, - "feature_columns": self._feature_columns, - "optimizer": optimizer, - "activation_fn": activation_fn, - "dropout": dropout, - "gradient_clip_norm": gradient_clip_norm, - "embedding_lr_multipliers": embedding_lr_multipliers, - "input_layer_min_slice_size": input_layer_min_slice_size, - }, - feature_engineering_fn=feature_engineering_fn) - - def evaluate(self, - x=None, - y=None, - input_fn=None, - feed_fn=None, - batch_size=None, - steps=None, - metrics=None, - name=None, - checkpoint_path=None, - hooks=None): - """See evaluable.Evaluable.""" - # TODO(zakaria): remove once deprecation is finished (b/31229024) - custom_metrics = {} - if metrics: - for key, metric in six.iteritems(metrics): - if (not isinstance(metric, metric_spec.MetricSpec) and - not isinstance(key, tuple)): - custom_metrics[(key, prediction_key.PredictionKey.SCORES)] = metric - else: - custom_metrics[key] = metric - - return super(DNNRegressor, self).evaluate( - x=x, - y=y, - input_fn=input_fn, - feed_fn=feed_fn, - batch_size=batch_size, - steps=steps, - metrics=custom_metrics, - name=name, - checkpoint_path=checkpoint_path, - hooks=hooks) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, - estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - @deprecated_arg_values( - "2017-03-01", - "Please switch to predict_scores, or set `outputs` argument.", - outputs=None) - def predict(self, x=None, input_fn=None, batch_size=None, outputs=None, - as_iterable=True): - """Returns predictions for given features. - - By default, returns predicted scores. But this default will be dropped - soon. Users should either pass `outputs`, or call `predict_scores` method. - - Args: - x: features. - input_fn: Input function. If set, x must be None. - batch_size: Override default batch size. - outputs: list of `str`, name of the output to predict. - If `None`, returns scores. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted scores (or an iterable of predicted scores if - as_iterable is True). If `label_dimension == 1`, the shape of the output - is `[batch_size]`, otherwise the shape is `[batch_size, label_dimension]`. - If `outputs` is set, returns a dict of predictions. - """ - if not outputs: - return self.predict_scores( - x=x, - input_fn=input_fn, - batch_size=batch_size, - as_iterable=as_iterable) - return super(DNNRegressor, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=outputs, - as_iterable=as_iterable) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, - estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - def predict_scores(self, x=None, input_fn=None, batch_size=None, - as_iterable=True): - """Returns predicted scores for given features. - - Args: - x: features. - input_fn: Input function. If set, x must be None. - batch_size: Override default batch size. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted scores (or an iterable of predicted scores if - as_iterable is True). If `label_dimension == 1`, the shape of the output - is `[batch_size]`, otherwise the shape is `[batch_size, label_dimension]`. - """ - key = prediction_key.PredictionKey.SCORES - preds = super(DNNRegressor, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=[key], - as_iterable=as_iterable) - if as_iterable: - return (pred[key] for pred in preds) - return preds[key] - - @deprecated("2017-03-25", "Please use Estimator.export_savedmodel() instead.") - def export(self, - export_dir, - input_fn=None, - input_feature_key=None, - use_deprecated_input_fn=True, - signature_fn=None, - default_batch_size=1, - exports_to_keep=None): - """See BaseEstimator.export.""" - def default_input_fn(unused_estimator, examples): - return layers.parse_feature_columns_from_examples(examples, - self._feature_columns) - - return super(DNNRegressor, self).export( - export_dir=export_dir, - input_fn=input_fn or default_input_fn, - input_feature_key=input_feature_key, - use_deprecated_input_fn=use_deprecated_input_fn, - signature_fn=signature_fn or export.regression_signature_fn, - prediction_key=prediction_key.PredictionKey.SCORES, - default_batch_size=default_batch_size, - exports_to_keep=exports_to_keep) - - -class DNNEstimator(estimator.Estimator): - """A Estimator for TensorFlow DNN models with user specified _Head. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Example: - - ```python - sparse_feature_a = sparse_column_with_hash_bucket(...) - sparse_feature_b = sparse_column_with_hash_bucket(...) - - sparse_feature_a_emb = embedding_column(sparse_id_column=sparse_feature_a, - ...) - sparse_feature_b_emb = embedding_column(sparse_id_column=sparse_feature_b, - ...) - To create a DNNEstimator for binary classification, where - estimator = DNNEstimator( - feature_columns=[sparse_feature_a_emb, sparse_feature_b_emb], - head=tf.contrib.learn.multi_class_head(n_classes=2), - hidden_units=[1024, 512, 256]) - - If your label is keyed with "y" in your labels dict, and weights are keyed - with "w" in features dict, and you want to enable centered bias, - head = tf.contrib.learn.multi_class_head( - n_classes=2, - label_name="x", - weight_column_name="w", - enable_centered_bias=True) - estimator = DNNEstimator( - feature_columns=[sparse_feature_a_emb, sparse_feature_b_emb], - head=head, - hidden_units=[1024, 512, 256]) - - # Input builders - def input_fn_train: # returns x, y (where y represents label's class index). - pass - estimator.fit(input_fn=input_fn_train) - - def input_fn_eval: # returns x, y (where y represents label's class index). - pass - estimator.evaluate(input_fn=input_fn_eval) - estimator.predict(x=x) # returns predicted labels (i.e. label's class index). - ``` - - Input of `fit` and `evaluate` should have following features, - otherwise there will be a `KeyError`: - - * if `weight_column_name` is not `None`, a feature with - `key=weight_column_name` whose value is a `Tensor`. - * for each `column` in `feature_columns`: - - if `column` is a `SparseColumn`, a feature with `key=column.name` - whose `value` is a `SparseTensor`. - - if `column` is a `WeightedSparseColumn`, two features: the first with - `key` the id column name, the second with `key` the weight column name. - Both features' `value` must be a `SparseTensor`. - - if `column` is a `RealValuedColumn`, a feature with `key=column.name` - whose `value` is a `Tensor`. - """ - - def __init__(self, - head, - hidden_units, - feature_columns, - model_dir=None, - optimizer=None, - activation_fn=nn.relu, - dropout=None, - gradient_clip_norm=None, - config=None, - feature_engineering_fn=None, - embedding_lr_multipliers=None, - input_layer_min_slice_size=None): - """Initializes a `DNNEstimator` instance. - - Args: - head: `Head` instance. - hidden_units: List of hidden units per layer. All layers are fully - connected. Ex. `[64, 32]` means first layer has 64 nodes and second one - has 32. - feature_columns: An iterable containing all the feature columns used by - the model. All items in the set should be instances of classes derived - from `FeatureColumn`. - model_dir: Directory to save model parameters, graph and etc. This can - also be used to load checkpoints from the directory into a estimator to - continue training a previously saved model. - optimizer: An instance of `tf.Optimizer` used to train the model. If - `None`, will use an Adagrad optimizer. - activation_fn: Activation function applied to each layer. If `None`, will - use `tf.nn.relu`. Note that a string containing the unqualified name of - the op may also be provided, e.g., "relu", "tanh", or "sigmoid". - dropout: When not `None`, the probability we will drop out a given - coordinate. - gradient_clip_norm: A float > 0. If provided, gradients are - clipped to their global norm with this clipping ratio. See - `tf.clip_by_global_norm` for more details. - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and - returns features and labels which will be fed - into the model. - embedding_lr_multipliers: Optional. A dictionary from `EmbeddingColumn` to - a `float` multiplier. Multiplier will be used to multiply with - learning rate for the embedding variables. - input_layer_min_slice_size: Optional. The min slice size of input layer - partitions. If not provided, will use the default of 64M. - - Returns: - A `DNNEstimator` estimator. - """ - super(DNNEstimator, self).__init__( - model_fn=_dnn_model_fn, - model_dir=model_dir, - config=config, - params={ - "head": head, - "hidden_units": hidden_units, - "feature_columns": feature_columns, - "optimizer": optimizer, - "activation_fn": activation_fn, - "dropout": dropout, - "gradient_clip_norm": gradient_clip_norm, - "embedding_lr_multipliers": embedding_lr_multipliers, - "input_layer_min_slice_size": input_layer_min_slice_size, - }, - feature_engineering_fn=feature_engineering_fn) diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py deleted file mode 100644 index 9dcc297e628..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py +++ /dev/null @@ -1,1168 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""TensorFlow estimators for Linear and DNN joined training models (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import six - -from tensorflow.contrib import layers -from tensorflow.contrib.framework import deprecated -from tensorflow.contrib.framework import deprecated_arg_values -from tensorflow.contrib.layers.python.layers import feature_column as feature_column_lib -from tensorflow.contrib.layers.python.layers import optimizers -from tensorflow.contrib.learn.python.learn import metric_spec -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.contrib.learn.python.learn.utils import export -from tensorflow.python.feature_column import feature_column_lib as fc_core -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.summary import summary -from tensorflow.python.training import sync_replicas_optimizer -from tensorflow.python.training import training_util - - -# The default learning rates are a historical artifact of the initial -# implementation, but seem a reasonable choice. -_DNN_LEARNING_RATE = 0.05 -_LINEAR_LEARNING_RATE = 0.2 - - -_FIX_GLOBAL_STEP_INCREMENT_DATE = "2017-04-15" -_FIX_GLOBAL_STEP_INCREMENT_INSTRUCTIONS = ( - "Please set fix_global_step_increment_bug=True and update training steps " - "in your pipeline. See pydoc for details.") - - -def _as_iterable(preds, output): - for pred in preds: - yield pred[output] - - -def _get_feature_dict(features): - if isinstance(features, dict): - return features - return {"": features} - - -def _get_optimizer(optimizer): - if callable(optimizer): - return optimizer() - else: - return optimizer - - -def _check_no_sync_replicas_optimizer(optimizer): - if isinstance(optimizer, sync_replicas_optimizer.SyncReplicasOptimizer): - raise ValueError( - "SyncReplicasOptimizer is not supported in DNNLinearCombined model. " - "If you want to use this optimizer, please use either DNN or Linear " - "model.") - - -def _linear_learning_rate(num_linear_feature_columns): - """Returns the default learning rate of the linear model. - - The calculation is a historical artifact of this initial implementation, but - has proven a reasonable choice. - - Args: - num_linear_feature_columns: The number of feature columns of the linear - model. - - Returns: - A float. - """ - default_learning_rate = 1. / math.sqrt(num_linear_feature_columns) - return min(_LINEAR_LEARNING_RATE, default_learning_rate) - - -def _add_hidden_layer_summary(value, tag): - summary.scalar("%s/fraction_of_zero_values" % tag, nn.zero_fraction(value)) - summary.histogram("%s/activation" % tag, value) - - -def _add_layer_summary(value, tag): - summary.scalar("%s/fraction_of_zero_values" % tag, nn.zero_fraction(value)) - summary.histogram("%s/activation" % tag, value) - - -def _get_embedding_variable(column, collection_key, input_layer_scope): - return ops.get_collection(collection_key, - input_layer_scope + "/" + column.name) - - -def _extract_embedding_lr_multipliers(embedding_lr_multipliers, collection_key, - input_layer_scope): - """Converts embedding lr multipliers to variable based gradient multiplier.""" - if not embedding_lr_multipliers: - return None - gradient_multipliers = {} - for column, lr_mult in embedding_lr_multipliers.items(): - if not isinstance(column, feature_column_lib._EmbeddingColumn): # pylint: disable=protected-access - raise ValueError( - "learning rate multipler can only be defined for embedding columns. " - "It is defined for {}".format(column)) - embedding = _get_embedding_variable( - column, collection_key, input_layer_scope) - if not embedding: - raise ValueError("Couldn't find a variable for column {}".format(column)) - for v in embedding: - gradient_multipliers[v] = lr_mult - return gradient_multipliers - - -def _dnn_linear_combined_model_fn(features, labels, mode, params, config=None): - """Deep Neural Net and Linear combined model_fn. - - Args: - features: `Tensor` or dict of `Tensor` (depends on data passed to `fit`). - labels: `Tensor` of shape [batch_size, 1] or [batch_size] labels of dtype - `int32` or `int64` in the range `[0, n_classes)`. - mode: Defines whether this is training, evaluation or prediction. - See `ModeKeys`. - params: A dict of hyperparameters. - The following hyperparameters are expected: - * head: A `Head` instance. - * linear_feature_columns: An iterable containing all the feature columns - used by the Linear model. - * linear_optimizer: string, `Optimizer` object, or callable that defines - the optimizer to use for training the Linear model. Defaults to the - Ftrl optimizer. - * joint_linear_weights: If True a single (possibly partitioned) variable - will be used to store the linear model weights. It's faster, but - requires all columns are sparse and have the 'sum' combiner. - * dnn_feature_columns: An iterable containing all the feature columns used - by the DNN model. - * dnn_optimizer: string, `Optimizer` object, or callable that defines the - optimizer to use for training the DNN model. Defaults to the Adagrad - optimizer. - * dnn_hidden_units: List of hidden units per DNN layer. - * dnn_activation_fn: Activation function applied to each DNN layer. If - `None`, will use `tf.nn.relu`. - * dnn_dropout: When not `None`, the probability we will drop out a given - DNN coordinate. - * gradient_clip_norm: A float > 0. If provided, gradients are - clipped to their global norm with this clipping ratio. - * embedding_lr_multipliers: Optional. A dictionary from - `EmbeddingColumn` to a `float` multiplier. Multiplier will be used to - multiply with learning rate for the embedding variables. - * input_layer_partitioner: Optional. Partitioner for input layer. - config: `RunConfig` object to configure the runtime settings. - - Returns: - `ModelFnOps` - - Raises: - ValueError: If both `linear_feature_columns` and `dnn_features_columns` - are empty at the same time, or `input_layer_partitioner` is missing. - """ - head = params["head"] - linear_feature_columns = params.get("linear_feature_columns") - linear_optimizer = params.get("linear_optimizer") or "Ftrl" - joint_linear_weights = params.get("joint_linear_weights") - dnn_feature_columns = params.get("dnn_feature_columns") - dnn_optimizer = params.get("dnn_optimizer") or "Adagrad" - dnn_hidden_units = params.get("dnn_hidden_units") - dnn_activation_fn = params.get("dnn_activation_fn") or nn.relu - dnn_dropout = params.get("dnn_dropout") - gradient_clip_norm = params.get("gradient_clip_norm") - num_ps_replicas = config.num_ps_replicas if config else 0 - input_layer_partitioner = params.get("input_layer_partitioner") or ( - partitioned_variables.min_max_variable_partitioner( - max_partitions=num_ps_replicas, - min_slice_size=64 << 20)) - embedding_lr_multipliers = params.get("embedding_lr_multipliers", {}) - fix_global_step_increment_bug = params.get( - "fix_global_step_increment_bug", True) - - if not linear_feature_columns and not dnn_feature_columns: - raise ValueError( - "Either linear_feature_columns or dnn_feature_columns must be defined.") - - features = _get_feature_dict(features) - - linear_optimizer = _get_optimizer(linear_optimizer) - _check_no_sync_replicas_optimizer(linear_optimizer) - dnn_optimizer = _get_optimizer(dnn_optimizer) - _check_no_sync_replicas_optimizer(dnn_optimizer) - - # Build DNN Logits. - dnn_parent_scope = "dnn" - - if not dnn_feature_columns: - dnn_logits = None - else: - if not dnn_hidden_units: - raise ValueError( - "dnn_hidden_units must be defined when dnn_feature_columns is " - "specified.") - dnn_partitioner = ( - partitioned_variables.min_max_variable_partitioner( - max_partitions=num_ps_replicas)) - with variable_scope.variable_scope( - dnn_parent_scope, - values=tuple(six.itervalues(features)), - partitioner=dnn_partitioner): - with variable_scope.variable_scope( - "input_from_feature_columns", - values=tuple(six.itervalues(features)), - partitioner=input_layer_partitioner) as dnn_input_scope: - if all( - isinstance(fc, feature_column_lib._FeatureColumn) # pylint: disable=protected-access - for fc in dnn_feature_columns - ): - net = layers.input_from_feature_columns( - columns_to_tensors=features, - feature_columns=dnn_feature_columns, - weight_collections=[dnn_parent_scope], - scope=dnn_input_scope) - else: - net = fc_core.input_layer( - features=features, - feature_columns=dnn_feature_columns, - weight_collections=[dnn_parent_scope]) - - for layer_id, num_hidden_units in enumerate(dnn_hidden_units): - with variable_scope.variable_scope( - "hiddenlayer_%d" % layer_id, - values=(net,)) as dnn_hidden_layer_scope: - net = layers.fully_connected( - net, - num_hidden_units, - activation_fn=dnn_activation_fn, - variables_collections=[dnn_parent_scope], - scope=dnn_hidden_layer_scope) - if dnn_dropout is not None and mode == model_fn.ModeKeys.TRAIN: - net = layers.dropout( - net, - keep_prob=(1.0 - dnn_dropout)) - # TODO(b/31209633): Consider adding summary before dropout. - _add_layer_summary(net, dnn_hidden_layer_scope.name) - - with variable_scope.variable_scope( - "logits", - values=(net,)) as dnn_logits_scope: - dnn_logits = layers.fully_connected( - net, - head.logits_dimension, - activation_fn=None, - variables_collections=[dnn_parent_scope], - scope=dnn_logits_scope) - _add_layer_summary(dnn_logits, dnn_logits_scope.name) - - # Build Linear logits. - linear_parent_scope = "linear" - - if not linear_feature_columns: - linear_logits = None - else: - linear_partitioner = partitioned_variables.min_max_variable_partitioner( - max_partitions=num_ps_replicas, - min_slice_size=64 << 20) - with variable_scope.variable_scope( - linear_parent_scope, - values=tuple(six.itervalues(features)), - partitioner=linear_partitioner) as scope: - if all(isinstance(fc, feature_column_lib._FeatureColumn) # pylint: disable=protected-access - for fc in linear_feature_columns): - if joint_linear_weights: - linear_logits, _, _ = layers.joint_weighted_sum_from_feature_columns( - columns_to_tensors=features, - feature_columns=linear_feature_columns, - num_outputs=head.logits_dimension, - weight_collections=[linear_parent_scope], - scope=scope) - else: - linear_logits, _, _ = layers.weighted_sum_from_feature_columns( - columns_to_tensors=features, - feature_columns=linear_feature_columns, - num_outputs=head.logits_dimension, - weight_collections=[linear_parent_scope], - scope=scope) - else: - linear_logits = fc_core.linear_model( - features=features, - feature_columns=linear_feature_columns, - units=head.logits_dimension, - weight_collections=[linear_parent_scope]) - - _add_layer_summary(linear_logits, scope.name) - - # Combine logits and build full model. - if dnn_logits is not None and linear_logits is not None: - logits = dnn_logits + linear_logits - elif dnn_logits is not None: - logits = dnn_logits - else: - logits = linear_logits - - def _make_training_op(training_loss): - """Training op for the DNN linear combined model.""" - train_ops = [] - global_step = training_util.get_global_step() - if dnn_logits is not None: - train_ops.append( - optimizers.optimize_loss( - loss=training_loss, - global_step=global_step, - learning_rate=_DNN_LEARNING_RATE, - optimizer=dnn_optimizer, - gradient_multipliers=_extract_embedding_lr_multipliers( # pylint: disable=protected-access - embedding_lr_multipliers, dnn_parent_scope, - dnn_input_scope.name), - clip_gradients=gradient_clip_norm, - variables=ops.get_collection(dnn_parent_scope), - name=dnn_parent_scope, - # Empty summaries, because head already logs "loss" summary. - summaries=[], - increment_global_step=not fix_global_step_increment_bug)) - if linear_logits is not None: - train_ops.append( - optimizers.optimize_loss( - loss=training_loss, - global_step=global_step, - learning_rate=_linear_learning_rate(len(linear_feature_columns)), - optimizer=linear_optimizer, - clip_gradients=gradient_clip_norm, - variables=ops.get_collection(linear_parent_scope), - name=linear_parent_scope, - # Empty summaries, because head already logs "loss" summary. - summaries=[], - increment_global_step=not fix_global_step_increment_bug)) - - train_op = control_flow_ops.group(*train_ops) - if fix_global_step_increment_bug: - with ops.control_dependencies([train_op]): - with ops.colocate_with(global_step): - return state_ops.assign_add(global_step, 1).op - return train_op - - return head.create_model_fn_ops( - features=features, - mode=mode, - labels=labels, - train_op_fn=_make_training_op, - logits=logits) - - -class DNNLinearCombinedEstimator(estimator.Estimator): - """An estimator for TensorFlow Linear and DNN joined training models. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Note: New users must set `fix_global_step_increment_bug=True` when creating an - estimator. - - Input of `fit`, `train`, and `evaluate` should have following features, - otherwise there will be a `KeyError`: - if `weight_column_name` is not `None`, a feature with - `key=weight_column_name` whose value is a `Tensor`. - for each `column` in `dnn_feature_columns` + `linear_feature_columns`: - - if `column` is a `SparseColumn`, a feature with `key=column.name` - whose `value` is a `SparseTensor`. - - if `column` is a `WeightedSparseColumn`, two features: the first with - `key` the id column name, the second with `key` the weight column - name. Both features' `value` must be a `SparseTensor`. - - if `column` is a `RealValuedColumn, a feature with `key=column.name` - whose `value` is a `Tensor`. - """ - - @deprecated_arg_values( - _FIX_GLOBAL_STEP_INCREMENT_DATE, - _FIX_GLOBAL_STEP_INCREMENT_INSTRUCTIONS, - fix_global_step_increment_bug=False) - def __init__(self, # _joint_linear_weights pylint: disable=invalid-name - head, - model_dir=None, - linear_feature_columns=None, - linear_optimizer=None, - _joint_linear_weights=False, - dnn_feature_columns=None, - dnn_optimizer=None, - dnn_hidden_units=None, - dnn_activation_fn=None, - dnn_dropout=None, - gradient_clip_norm=None, - config=None, - feature_engineering_fn=None, - embedding_lr_multipliers=None, - fix_global_step_increment_bug=False, - input_layer_partitioner=None): - """Initializes a DNNLinearCombinedEstimator instance. - - Note: New users must set `fix_global_step_increment_bug=True` when creating - an estimator. - - Args: - head: A _Head object. - model_dir: Directory to save model parameters, graph and etc. This can - also be used to load checkpoints from the directory into a estimator - to continue training a previously saved model. - linear_feature_columns: An iterable containing all the feature columns - used by linear part of the model. All items in the set should be - instances of classes derived from `FeatureColumn`. - linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to - the linear part of the model. If `None`, will use a FTRL optimizer. - _joint_linear_weights: If True will use a single (possibly partitioned) - variable to store all weights for the linear model. More efficient if - there are many columns, however requires all columns are sparse and - have the 'sum' combiner. - dnn_feature_columns: An iterable containing all the feature columns used - by deep part of the model. All items in the set should be instances of - classes derived from `FeatureColumn`. - dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to - the deep part of the model. If `None`, will use an Adagrad optimizer. - dnn_hidden_units: List of hidden units per layer. All layers are fully - connected. - dnn_activation_fn: Activation function applied to each layer. If `None`, - will use `tf.nn.relu`. - dnn_dropout: When not None, the probability we will drop out - a given coordinate. - gradient_clip_norm: A float > 0. If provided, gradients are clipped - to their global norm with this clipping ratio. See - tf.clip_by_global_norm for more details. - config: RunConfig object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - embedding_lr_multipliers: Optional. A dictionary from `EmbeddingColumn` to - a `float` multiplier. Multiplier will be used to multiply with - learning rate for the embedding variables. - fix_global_step_increment_bug: If `False`, the estimator needs two fit - steps to optimize both linear and dnn parts. If `True`, this bug is - fixed. New users must set this to `True`, but the default value is - `False` for backwards compatibility. - input_layer_partitioner: Optional. Partitioner for input layer. - - Raises: - ValueError: If both linear_feature_columns and dnn_features_columns are - empty at the same time. - """ - linear_feature_columns = tuple(linear_feature_columns or []) - dnn_feature_columns = tuple(dnn_feature_columns or []) - if not linear_feature_columns + dnn_feature_columns: - raise ValueError("Either linear_feature_columns or dnn_feature_columns " - "must be defined.") - super(DNNLinearCombinedEstimator, self).__init__( - model_fn=_dnn_linear_combined_model_fn, - model_dir=model_dir, - config=config, - params={ - "head": head, - "linear_feature_columns": linear_feature_columns, - "linear_optimizer": linear_optimizer, - "joint_linear_weights": _joint_linear_weights, - "dnn_feature_columns": dnn_feature_columns, - "dnn_optimizer": dnn_optimizer, - "dnn_hidden_units": dnn_hidden_units, - "dnn_activation_fn": dnn_activation_fn, - "dnn_dropout": dnn_dropout, - "gradient_clip_norm": gradient_clip_norm, - "embedding_lr_multipliers": embedding_lr_multipliers, - "fix_global_step_increment_bug": fix_global_step_increment_bug, - "input_layer_partitioner": input_layer_partitioner - }, - feature_engineering_fn=feature_engineering_fn) - - -class DNNLinearCombinedClassifier(estimator.Estimator): - """A classifier for TensorFlow Linear and DNN joined training models. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Note: New users must set `fix_global_step_increment_bug=True` when creating an - estimator. - - Example: - - ```python - sparse_feature_a = sparse_column_with_hash_bucket(...) - sparse_feature_b = sparse_column_with_hash_bucket(...) - - sparse_feature_a_x_sparse_feature_b = crossed_column(...) - - sparse_feature_a_emb = embedding_column(sparse_id_column=sparse_feature_a, - ...) - sparse_feature_b_emb = embedding_column(sparse_id_column=sparse_feature_b, - ...) - - estimator = DNNLinearCombinedClassifier( - # common settings - n_classes=n_classes, - weight_column_name=weight_column_name, - # wide settings - linear_feature_columns=[sparse_feature_a_x_sparse_feature_b], - linear_optimizer=tf.compat.v1.train.FtrlOptimizer(...), - # deep settings - dnn_feature_columns=[sparse_feature_a_emb, sparse_feature_b_emb], - dnn_hidden_units=[1000, 500, 100], - dnn_optimizer=tf.compat.v1.train.AdagradOptimizer(...)) - - # Input builders - def input_fn_train: # returns x, y (where y represents label's class index). - ... - def input_fn_eval: # returns x, y (where y represents label's class index). - ... - def input_fn_predict: # returns x, None. - ... - estimator.fit(input_fn=input_fn_train) - estimator.evaluate(input_fn=input_fn_eval) - # predict_classes returns class indices. - estimator.predict_classes(input_fn=input_fn_predict) - ``` - - If the user specifies `label_keys` in constructor, labels must be strings from - the `label_keys` vocabulary. Example: - - ```python - label_keys = ['label0', 'label1', 'label2'] - estimator = DNNLinearCombinedClassifier( - n_classes=n_classes, - linear_feature_columns=[sparse_feature_a_x_sparse_feature_b], - dnn_feature_columns=[sparse_feature_a_emb, sparse_feature_b_emb], - dnn_hidden_units=[1000, 500, 100], - label_keys=label_keys) - - def input_fn_train: # returns x, y (where y is one of label_keys). - pass - estimator.fit(input_fn=input_fn_train) - - def input_fn_eval: # returns x, y (where y is one of label_keys). - pass - estimator.evaluate(input_fn=input_fn_eval) - def input_fn_predict: # returns x, None - # predict_classes returns one of label_keys. - estimator.predict_classes(input_fn=input_fn_predict) - ``` - - Input of `fit` and `evaluate` should have following features, - otherwise there will be a `KeyError`: - - * if `weight_column_name` is not `None`, a feature with - `key=weight_column_name` whose value is a `Tensor`. - * for each `column` in `dnn_feature_columns` + `linear_feature_columns`: - - if `column` is a `SparseColumn`, a feature with `key=column.name` - whose `value` is a `SparseTensor`. - - if `column` is a `WeightedSparseColumn`, two features: the first with - `key` the id column name, the second with `key` the weight column name. - Both features' `value` must be a `SparseTensor`. - - if `column` is a `RealValuedColumn, a feature with `key=column.name` - whose `value` is a `Tensor`. - """ - - @deprecated_arg_values( - _FIX_GLOBAL_STEP_INCREMENT_DATE, - _FIX_GLOBAL_STEP_INCREMENT_INSTRUCTIONS, - fix_global_step_increment_bug=False) - def __init__(self, # _joint_linear_weights pylint: disable=invalid-name - model_dir=None, - n_classes=2, - weight_column_name=None, - linear_feature_columns=None, - linear_optimizer=None, - _joint_linear_weights=False, - dnn_feature_columns=None, - dnn_optimizer=None, - dnn_hidden_units=None, - dnn_activation_fn=nn.relu, - dnn_dropout=None, - gradient_clip_norm=None, - enable_centered_bias=False, - config=None, - feature_engineering_fn=None, - embedding_lr_multipliers=None, - input_layer_min_slice_size=None, - label_keys=None, - fix_global_step_increment_bug=False): - """Constructs a DNNLinearCombinedClassifier instance. - - Note: New users must set `fix_global_step_increment_bug=True` when creating - an estimator. - - Args: - model_dir: Directory to save model parameters, graph and etc. This can - also be used to load checkpoints from the directory into a estimator - to continue training a previously saved model. - n_classes: number of label classes. Default is binary classification. - Note that class labels are integers representing the class index (i.e. - values from 0 to n_classes-1). For arbitrary label values (e.g. string - labels), convert to class indices first. - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. - It will be multiplied by the loss of the example. - linear_feature_columns: An iterable containing all the feature columns - used by linear part of the model. All items in the set must be - instances of classes derived from `FeatureColumn`. - linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to - the linear part of the model. If `None`, will use a FTRL optimizer. - _joint_linear_weights: If True a single (possibly partitioned) variable - will be used to store the linear model weights. It's faster, but - requires all columns are sparse and have the 'sum' combiner. - dnn_feature_columns: An iterable containing all the feature columns used - by deep part of the model. All items in the set must be instances of - classes derived from `FeatureColumn`. - dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to - the deep part of the model. If `None`, will use an Adagrad optimizer. - dnn_hidden_units: List of hidden units per layer. All layers are fully - connected. - dnn_activation_fn: Activation function applied to each layer. If `None`, - will use `tf.nn.relu`. - dnn_dropout: When not None, the probability we will drop out - a given coordinate. - gradient_clip_norm: A float > 0. If provided, gradients are clipped - to their global norm with this clipping ratio. See - tf.clip_by_global_norm for more details. - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - config: RunConfig object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - embedding_lr_multipliers: Optional. A dictionary from `EmbeddingColumn` to - a `float` multiplier. Multiplier will be used to multiply with - learning rate for the embedding variables. - input_layer_min_slice_size: Optional. The min slice size of input layer - partitions. If not provided, will use the default of 64M. - label_keys: Optional list of strings with size `[n_classes]` defining the - label vocabulary. Only supported for `n_classes` > 2. - fix_global_step_increment_bug: If `False`, the estimator needs two fit - steps to optimize both linear and dnn parts. If `True`, this bug is - fixed. New users must set this to `True`, but it the default value is - `False` for backwards compatibility. - - Raises: - ValueError: If `n_classes` < 2. - ValueError: If both `linear_feature_columns` and `dnn_features_columns` - are empty at the same time. - """ - head = head_lib.multi_class_head( - n_classes=n_classes, - weight_column_name=weight_column_name, - enable_centered_bias=enable_centered_bias, - label_keys=label_keys) - linear_feature_columns = tuple(linear_feature_columns or []) - dnn_feature_columns = tuple(dnn_feature_columns or []) - self._feature_columns = linear_feature_columns + dnn_feature_columns - if not self._feature_columns: - raise ValueError("Either linear_feature_columns or dnn_feature_columns " - "must be defined.") - - # TODO(b/35922130): Replace with `input_layer_partitioner` arg. - input_layer_partitioner = None - if input_layer_min_slice_size is not None: - input_layer_partitioner = ( - partitioned_variables.min_max_variable_partitioner( - max_partitions=config.num_ps_replicas if config else 0, - min_slice_size=input_layer_min_slice_size)) - - super(DNNLinearCombinedClassifier, self).__init__( - model_fn=_dnn_linear_combined_model_fn, - model_dir=model_dir, - config=config, - params={ - "head": head, - "linear_feature_columns": linear_feature_columns, - "linear_optimizer": linear_optimizer, - "joint_linear_weights": _joint_linear_weights, - "dnn_feature_columns": dnn_feature_columns, - "dnn_optimizer": dnn_optimizer, - "dnn_hidden_units": dnn_hidden_units, - "dnn_activation_fn": dnn_activation_fn, - "dnn_dropout": dnn_dropout, - "gradient_clip_norm": gradient_clip_norm, - "embedding_lr_multipliers": embedding_lr_multipliers, - "input_layer_partitioner": input_layer_partitioner, - "fix_global_step_increment_bug": fix_global_step_increment_bug, - }, - feature_engineering_fn=feature_engineering_fn) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - @deprecated_arg_values( - "2017-03-01", - "Please switch to predict_classes, or set `outputs` argument.", - outputs=None) - def predict(self, x=None, input_fn=None, batch_size=None, outputs=None, - as_iterable=True): - """Returns predictions for given features. - - By default, returns predicted classes. But this default will be dropped - soon. Users should either pass `outputs`, or call `predict_classes` method. - - Args: - x: features. - input_fn: Input function. If set, x must be None. - batch_size: Override default batch size. - outputs: list of `str`, name of the output to predict. - If `None`, returns classes. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted classes with shape [batch_size] (or an iterable - of predicted classes if as_iterable is True). Each predicted class is - represented by its class index (i.e. integer from 0 to n_classes-1). - If `outputs` is set, returns a dict of predictions. - """ - if not outputs: - return self.predict_classes( - x=x, - input_fn=input_fn, - batch_size=batch_size, - as_iterable=as_iterable) - return super(DNNLinearCombinedClassifier, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=outputs, - as_iterable=as_iterable) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - def predict_classes(self, x=None, input_fn=None, batch_size=None, - as_iterable=True): - """Returns predicted classes for given features. - - Args: - x: features. - input_fn: Input function. If set, x must be None. - batch_size: Override default batch size. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted classes with shape [batch_size] (or an iterable - of predicted classes if as_iterable is True). Each predicted class is - represented by its class index (i.e. integer from 0 to n_classes-1). - """ - key = prediction_key.PredictionKey.CLASSES - preds = super(DNNLinearCombinedClassifier, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=[key], - as_iterable=as_iterable) - if as_iterable: - return _as_iterable(preds, output=key) - return preds[key].reshape(-1) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - def predict_proba( - self, x=None, input_fn=None, batch_size=None, as_iterable=True): - """Returns prediction probabilities for given features. - - Args: - x: features. - input_fn: Input function. If set, x and y must be None. - batch_size: Override default batch size. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted probabilities with shape [batch_size, n_classes] - (or an iterable of predicted probabilities if as_iterable is True). - """ - key = prediction_key.PredictionKey.PROBABILITIES - preds = super(DNNLinearCombinedClassifier, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=[key], - as_iterable=as_iterable) - if as_iterable: - return _as_iterable(preds, output=key) - return preds[key] - - @deprecated("2017-03-25", "Please use Estimator.export_savedmodel() instead.") - def export(self, - export_dir, - input_fn=None, - input_feature_key=None, - use_deprecated_input_fn=True, - signature_fn=None, - default_batch_size=1, - exports_to_keep=None): - """See BasEstimator.export.""" - def default_input_fn(unused_estimator, examples): - return layers.parse_feature_columns_from_examples( - examples, self._feature_columns) - return super(DNNLinearCombinedClassifier, self).export( - export_dir=export_dir, - input_fn=input_fn or default_input_fn, - input_feature_key=input_feature_key, - use_deprecated_input_fn=use_deprecated_input_fn, - signature_fn=(signature_fn or - export.classification_signature_fn_with_prob), - prediction_key=prediction_key.PredictionKey.PROBABILITIES, - default_batch_size=default_batch_size, - exports_to_keep=exports_to_keep) - - -class DNNLinearCombinedRegressor(estimator.Estimator): - """A regressor for TensorFlow Linear and DNN joined training models. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Note: New users must set `fix_global_step_increment_bug=True` when creating an - estimator. - - Example: - - ```python - sparse_feature_a = sparse_column_with_hash_bucket(...) - sparse_feature_b = sparse_column_with_hash_bucket(...) - - sparse_feature_a_x_sparse_feature_b = crossed_column(...) - - sparse_feature_a_emb = embedding_column(sparse_id_column=sparse_feature_a, - ...) - sparse_feature_b_emb = embedding_column(sparse_id_column=sparse_feature_b, - ...) - - estimator = DNNLinearCombinedRegressor( - # common settings - weight_column_name=weight_column_name, - # wide settings - linear_feature_columns=[sparse_feature_a_x_sparse_feature_b], - linear_optimizer=tf.compat.v1.train.FtrlOptimizer(...), - # deep settings - dnn_feature_columns=[sparse_feature_a_emb, sparse_feature_b_emb], - dnn_hidden_units=[1000, 500, 100], - dnn_optimizer=tf.compat.v1.train.ProximalAdagradOptimizer(...)) - - # To apply L1 and L2 regularization, you can set optimizers as follows: - tf.compat.v1.train.ProximalAdagradOptimizer( - learning_rate=0.1, - l1_regularization_strength=0.001, - l2_regularization_strength=0.001) - # It is same for FtrlOptimizer. - - # Input builders - def input_fn_train: # returns x, y - ... - def input_fn_eval: # returns x, y - ... - def input_fn_predict: # returns x, None - ... - estimator.train(input_fn_train) - estimator.evaluate(input_fn_eval) - estimator.predict(input_fn_predict) - ``` - - Input of `fit`, `train`, and `evaluate` should have following features, - otherwise there will be a `KeyError`: - if `weight_column_name` is not `None`, a feature with - `key=weight_column_name` whose value is a `Tensor`. - for each `column` in `dnn_feature_columns` + `linear_feature_columns`: - - if `column` is a `SparseColumn`, a feature with `key=column.name` - whose `value` is a `SparseTensor`. - - if `column` is a `WeightedSparseColumn`, two features: the first with - `key` the id column name, the second with `key` the weight column name. - Both features' `value` must be a `SparseTensor`. - - if `column` is a `RealValuedColumn, a feature with `key=column.name` - whose `value` is a `Tensor`. - """ - - @deprecated_arg_values( - _FIX_GLOBAL_STEP_INCREMENT_DATE, - _FIX_GLOBAL_STEP_INCREMENT_INSTRUCTIONS, - fix_global_step_increment_bug=False) - def __init__(self, # _joint_linear_weights pylint: disable=invalid-name - model_dir=None, - weight_column_name=None, - linear_feature_columns=None, - linear_optimizer=None, - _joint_linear_weights=False, - dnn_feature_columns=None, - dnn_optimizer=None, - dnn_hidden_units=None, - dnn_activation_fn=nn.relu, - dnn_dropout=None, - gradient_clip_norm=None, - enable_centered_bias=False, - label_dimension=1, - config=None, - feature_engineering_fn=None, - embedding_lr_multipliers=None, - input_layer_min_slice_size=None, - fix_global_step_increment_bug=False): - """Initializes a DNNLinearCombinedRegressor instance. - - Note: New users must set `fix_global_step_increment_bug=True` when creating - an estimator. - - Args: - model_dir: Directory to save model parameters, graph and etc. This can - also be used to load checkpoints from the directory into a estimator - to continue training a previously saved model. - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - linear_feature_columns: An iterable containing all the feature columns - used by linear part of the model. All items in the set must be - instances of classes derived from `FeatureColumn`. - linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to - the linear part of the model. If `None`, will use a FTRL optimizer. - _joint_linear_weights: If True a single (possibly partitioned) variable - will be used to store the linear model weights. It's faster, but - requires that all columns are sparse and have the 'sum' combiner. - dnn_feature_columns: An iterable containing all the feature columns used - by deep part of the model. All items in the set must be instances of - classes derived from `FeatureColumn`. - dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to - the deep part of the model. If `None`, will use an Adagrad optimizer. - dnn_hidden_units: List of hidden units per layer. All layers are fully - connected. - dnn_activation_fn: Activation function applied to each layer. If None, - will use `tf.nn.relu`. - dnn_dropout: When not None, the probability we will drop out - a given coordinate. - gradient_clip_norm: A float > 0. If provided, gradients are clipped - to their global norm with this clipping ratio. See - tf.clip_by_global_norm for more details. - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - label_dimension: Number of regression targets per example. This is the - size of the last dimension of the labels and logits `Tensor` objects - (typically, these have shape `[batch_size, label_dimension]`). - config: RunConfig object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - embedding_lr_multipliers: Optional. A dictionary from `EmbeddingColumn` to - a `float` multiplier. Multiplier will be used to multiply with - learning rate for the embedding variables. - input_layer_min_slice_size: Optional. The min slice size of input layer - partitions. If not provided, will use the default of 64M. - fix_global_step_increment_bug: If `False`, the estimator needs two fit - steps to optimize both linear and dnn parts. If `True`, this bug is - fixed. New users must set this to `True`, but it the default value is - `False` for backwards compatibility. - - Raises: - ValueError: If both linear_feature_columns and dnn_features_columns are - empty at the same time. - """ - linear_feature_columns = linear_feature_columns or [] - dnn_feature_columns = dnn_feature_columns or [] - self._feature_columns = linear_feature_columns + dnn_feature_columns - if not self._feature_columns: - raise ValueError("Either linear_feature_columns or dnn_feature_columns " - "must be defined.") - - # TODO(b/35922130): Replace with `input_layer_partitioner` arg. - input_layer_partitioner = None - if input_layer_min_slice_size is not None: - input_layer_partitioner = ( - partitioned_variables.min_max_variable_partitioner( - max_partitions=config.num_ps_replicas if config else 0, - min_slice_size=input_layer_min_slice_size)) - - head = head_lib.regression_head( - weight_column_name=weight_column_name, - label_dimension=label_dimension, - enable_centered_bias=enable_centered_bias) - super(DNNLinearCombinedRegressor, self).__init__( - model_fn=_dnn_linear_combined_model_fn, - model_dir=model_dir, - config=config, - params={ - "head": head, - "linear_feature_columns": linear_feature_columns, - "linear_optimizer": linear_optimizer, - "joint_linear_weights": _joint_linear_weights, - "dnn_feature_columns": dnn_feature_columns, - "dnn_optimizer": dnn_optimizer, - "dnn_hidden_units": dnn_hidden_units, - "dnn_activation_fn": dnn_activation_fn, - "dnn_dropout": dnn_dropout, - "gradient_clip_norm": gradient_clip_norm, - "embedding_lr_multipliers": embedding_lr_multipliers, - "input_layer_partitioner": input_layer_partitioner, - "fix_global_step_increment_bug": fix_global_step_increment_bug, - }, - feature_engineering_fn=feature_engineering_fn) - - def evaluate(self, - x=None, - y=None, - input_fn=None, - feed_fn=None, - batch_size=None, - steps=None, - metrics=None, - name=None, - checkpoint_path=None, - hooks=None): - """See evaluable.Evaluable.""" - # TODO(zakaria): remove once deprecation is finished (b/31229024) - custom_metrics = {} - if metrics: - for key, metric in six.iteritems(metrics): - if (not isinstance(metric, metric_spec.MetricSpec) and - not isinstance(key, tuple)): - custom_metrics[(key, prediction_key.PredictionKey.SCORES)] = metric - else: - custom_metrics[key] = metric - - return super(DNNLinearCombinedRegressor, self).evaluate( - x=x, - y=y, - input_fn=input_fn, - feed_fn=feed_fn, - batch_size=batch_size, - steps=steps, - metrics=custom_metrics, - name=name, - checkpoint_path=checkpoint_path, - hooks=hooks) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - @deprecated_arg_values( - "2017-03-01", - "Please switch to predict_scores, or set `outputs` argument.", - outputs=None) - def predict(self, x=None, input_fn=None, batch_size=None, outputs=None, - as_iterable=True): - """Returns predictions for given features. - - By default, returns predicted scores. But this default will be dropped - soon. Users should either pass `outputs`, or call `predict_scores` method. - - Args: - x: features. - input_fn: Input function. If set, x must be None. - batch_size: Override default batch size. - outputs: list of `str`, name of the output to predict. - If `None`, returns scores. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted scores (or an iterable of predicted scores if - as_iterable is True). If `label_dimension == 1`, the shape of the output - is `[batch_size]`, otherwise the shape is `[batch_size, label_dimension]`. - If `outputs` is set, returns a dict of predictions. - """ - if not outputs: - return self.predict_scores( - x=x, - input_fn=input_fn, - batch_size=batch_size, - as_iterable=as_iterable) - return super(DNNLinearCombinedRegressor, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=outputs, - as_iterable=as_iterable) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - def predict_scores(self, x=None, input_fn=None, batch_size=None, - as_iterable=True): - """Returns predicted scores for given features. - - Args: - x: features. - input_fn: Input function. If set, x must be None. - batch_size: Override default batch size. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted scores (or an iterable of predicted scores if - as_iterable is True). If `label_dimension == 1`, the shape of the output - is `[batch_size]`, otherwise the shape is `[batch_size, label_dimension]`. - """ - key = prediction_key.PredictionKey.SCORES - preds = super(DNNLinearCombinedRegressor, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=[key], - as_iterable=as_iterable) - if as_iterable: - return (pred[key] for pred in preds) - return preds[key] - - @deprecated("2017-03-25", "Please use Estimator.export_savedmodel() instead.") - def export(self, - export_dir, - input_fn=None, - input_feature_key=None, - use_deprecated_input_fn=True, - signature_fn=None, - default_batch_size=1, - exports_to_keep=None): - """See BaseEstimator.export.""" - def default_input_fn(unused_estimator, examples): - return layers.parse_feature_columns_from_examples( - examples, self._feature_columns) - return super(DNNLinearCombinedRegressor, self).export( - export_dir=export_dir, - input_fn=input_fn or default_input_fn, - input_feature_key=input_feature_key, - use_deprecated_input_fn=use_deprecated_input_fn, - signature_fn=signature_fn or export.regression_signature_fn, - prediction_key=prediction_key.PredictionKey.SCORES, - default_batch_size=default_batch_size, - exports_to_keep=exports_to_keep) - -# Aliases -# TODO(zakaria): Remove these aliases, See b/34751732 -_DNNLinearCombinedEstimator = DNNLinearCombinedEstimator diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py deleted file mode 100644 index 4f636ce69dd..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py +++ /dev/null @@ -1,1818 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for DNNLinearCombinedEstimators.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import json -import tempfile - -import numpy as np - -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.learn.python.learn import experiment -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.contrib.learn.python.learn.estimators import _sklearn -from tensorflow.contrib.learn.python.learn.estimators import dnn_linear_combined -from tensorflow.contrib.learn.python.learn.estimators import estimator_test_utils -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.contrib.learn.python.learn.estimators import test_data -from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec -from tensorflow.contrib.metrics.python.ops import metric_ops -from tensorflow.python.feature_column import feature_column_lib as fc_core -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import test -from tensorflow.python.training import adagrad -from tensorflow.python.training import ftrl -from tensorflow.python.training import input as input_lib -from tensorflow.python.training import learning_rate_decay -from tensorflow.python.training import monitored_session -from tensorflow.python.training import server_lib -from tensorflow.python.training import session_run_hook -from tensorflow.python.training import sync_replicas_optimizer -from tensorflow.python.training import training_util - - -def _assert_metrics_in_range(keys, metrics): - epsilon = 0.00001 # Added for floating point edge cases. - for key in keys: - estimator_test_utils.assert_in_range(0.0 - epsilon, 1.0 + epsilon, key, - metrics) - - -class _CheckCallsHead(head_lib.Head): - """Head that checks whether head_ops is called.""" - - def __init__(self): - self._head_ops_called_times = 0 - - @property - def logits_dimension(self): - return 1 - - def create_model_fn_ops( - self, mode, features, labels=None, train_op_fn=None, logits=None, - logits_input=None, scope=None): - """See `_Head`.""" - self._head_ops_called_times += 1 - loss = losses.mean_squared_error(labels, logits) - return model_fn.ModelFnOps( - mode, - predictions={'loss': loss}, - loss=loss, - train_op=train_op_fn(loss), - eval_metric_ops={'loss': loss}) - - @property - def head_ops_called_times(self): - return self._head_ops_called_times - - -class _StepCounterHook(session_run_hook.SessionRunHook): - """Counts the number of training steps.""" - - def __init__(self): - self._steps = 0 - - def after_run(self, run_context, run_values): - del run_context, run_values - self._steps += 1 - - @property - def steps(self): - return self._steps - - -class EmbeddingMultiplierTest(test.TestCase): - """dnn_model_fn tests.""" - - def testRaisesNonEmbeddingColumn(self): - one_hot_language = feature_column.one_hot_column( - feature_column.sparse_column_with_hash_bucket('language', 10)) - - params = { - 'dnn_feature_columns': [one_hot_language], - 'head': head_lib.multi_class_head(2), - 'dnn_hidden_units': [1], - # Set lr mult to 0. to keep embeddings constant. - 'embedding_lr_multipliers': { - one_hot_language: 0.0 - }, - 'dnn_optimizer': 'Adagrad', - } - features = { - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - } - labels = constant_op.constant([[0], [0], [0]], dtype=dtypes.int32) - with self.assertRaisesRegexp(ValueError, - 'can only be defined for embedding columns'): - dnn_linear_combined._dnn_linear_combined_model_fn(features, labels, - model_fn.ModeKeys.TRAIN, - params) - - def testMultipliesGradient(self): - embedding_language = feature_column.embedding_column( - feature_column.sparse_column_with_hash_bucket('language', 10), - dimension=1, - initializer=init_ops.constant_initializer(0.1)) - embedding_wire = feature_column.embedding_column( - feature_column.sparse_column_with_hash_bucket('wire', 10), - dimension=1, - initializer=init_ops.constant_initializer(0.1)) - - params = { - 'dnn_feature_columns': [embedding_language, embedding_wire], - 'head': head_lib.multi_class_head(2), - 'dnn_hidden_units': [1], - # Set lr mult to 0. to keep language embeddings constant, whereas wire - # embeddings will be trained. - 'embedding_lr_multipliers': { - embedding_language: 0.0 - }, - 'dnn_optimizer': 'Adagrad', - } - with ops.Graph().as_default(): - features = { - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - 'wire': - sparse_tensor.SparseTensor( - values=['omar', 'stringer', 'marlo'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - } - labels = constant_op.constant([[1], [0], [0]], dtype=dtypes.int32) - training_util.create_global_step() - model_ops = dnn_linear_combined._dnn_linear_combined_model_fn( - features, labels, model_fn.ModeKeys.TRAIN, params) - with monitored_session.MonitoredSession() as sess: - language_var = dnn_linear_combined._get_embedding_variable( - embedding_language, 'dnn', 'dnn/input_from_feature_columns') - language_initial_value = sess.run(language_var) - for _ in range(2): - _, language_value = sess.run([model_ops.train_op, language_var]) - - self.assertAllClose(language_value, language_initial_value) - # We could also test that wire_value changed, but that test would be flaky. - - -class DNNLinearCombinedEstimatorTest(test.TestCase): - - def testEstimatorContract(self): - estimator_test_utils.assert_estimator_contract( - self, dnn_linear_combined.DNNLinearCombinedEstimator) - - def testNoFeatureColumns(self): - with self.assertRaisesRegexp( - ValueError, - 'Either linear_feature_columns or dnn_feature_columns must be defined'): - dnn_linear_combined.DNNLinearCombinedEstimator( - head=_CheckCallsHead(), - linear_feature_columns=None, - dnn_feature_columns=None, - dnn_hidden_units=[3, 3]) - - def testCheckCallsHead(self): - """Tests binary classification using matrix data as input.""" - head = _CheckCallsHead() - iris = test_data.prepare_iris_data_for_logistic_regression() - cont_features = [ - feature_column.real_valued_column('feature', dimension=4)] - bucketized_feature = [feature_column.bucketized_column( - cont_features[0], test_data.get_quantile_based_buckets(iris.data, 10))] - - estimator = dnn_linear_combined.DNNLinearCombinedEstimator( - head, - linear_feature_columns=bucketized_feature, - dnn_feature_columns=cont_features, - dnn_hidden_units=[3, 3]) - - estimator.fit(input_fn=test_data.iris_input_multiclass_fn, steps=10) - self.assertEqual(1, head.head_ops_called_times) - - estimator.evaluate(input_fn=test_data.iris_input_multiclass_fn, steps=10) - self.assertEqual(2, head.head_ops_called_times) - - estimator.predict(input_fn=test_data.iris_input_multiclass_fn) - self.assertEqual(3, head.head_ops_called_times) - - -class DNNLinearCombinedClassifierTest(test.TestCase): - - def testEstimatorContract(self): - estimator_test_utils.assert_estimator_contract( - self, dnn_linear_combined.DNNLinearCombinedClassifier) - - def testExperimentIntegration(self): - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - exp = experiment.Experiment( - estimator=dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=cont_features, - dnn_feature_columns=cont_features, - dnn_hidden_units=[3, 3]), - train_input_fn=test_data.iris_input_logistic_fn, - eval_input_fn=test_data.iris_input_logistic_fn) - exp.test() - - def testNoFeatureColumns(self): - with self.assertRaisesRegexp( - ValueError, - 'Either linear_feature_columns or dnn_feature_columns must be defined'): - dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=None, - dnn_feature_columns=None, - dnn_hidden_units=[3, 3]) - - def testNoDnnHiddenUnits(self): - def _input_fn(): - return { - 'age': - constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column.sparse_column_with_hash_bucket('language', 100) - age = feature_column.real_valued_column('age') - - with self.assertRaisesRegexp( - ValueError, - 'dnn_hidden_units must be defined when dnn_feature_columns is ' - 'specified'): - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - dnn_feature_columns=[age, language]) - classifier.fit(input_fn=_input_fn, steps=2) - - def testSyncReplicasOptimizerUnsupported(self): - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - sync_optimizer = sync_replicas_optimizer.SyncReplicasOptimizer( - opt=adagrad.AdagradOptimizer(learning_rate=0.1), - replicas_to_aggregate=1, - total_num_replicas=1) - sync_hook = sync_optimizer.make_session_run_hook(is_chief=True) - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - n_classes=3, - dnn_feature_columns=cont_features, - dnn_hidden_units=[3, 3], - dnn_optimizer=sync_optimizer) - - with self.assertRaisesRegexp( - ValueError, - 'SyncReplicasOptimizer is not supported in DNNLinearCombined model'): - classifier.fit( - input_fn=test_data.iris_input_multiclass_fn, steps=100, - monitors=[sync_hook]) - - def testEmbeddingMultiplier(self): - embedding_language = feature_column.embedding_column( - feature_column.sparse_column_with_hash_bucket('language', 10), - dimension=1, - initializer=init_ops.constant_initializer(0.1)) - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - dnn_feature_columns=[embedding_language], - dnn_hidden_units=[3, 3], - embedding_lr_multipliers={embedding_language: 0.8}) - self.assertEqual({ - embedding_language: 0.8 - }, classifier.params['embedding_lr_multipliers']) - - def testInputPartitionSize(self): - def _input_fn_float_label(num_epochs=None): - features = { - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - labels = constant_op.constant([[0.8], [0.], [0.2]], dtype=dtypes.float32) - return features, labels - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column(language_column, dimension=1), - ] - - # Set num_ps_replica to be 10 and the min slice size to be extremely small, - # so as to ensure that there'll be 10 partititions produced. - config = run_config.RunConfig(tf_random_seed=1) - config._num_ps_replicas = 10 - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - n_classes=2, - dnn_feature_columns=feature_columns, - dnn_hidden_units=[3, 3], - dnn_optimizer='Adagrad', - config=config, - input_layer_min_slice_size=1) - - # Ensure the param is passed in. - self.assertTrue(callable(classifier.params['input_layer_partitioner'])) - - # Ensure the partition count is 10. - classifier.fit(input_fn=_input_fn_float_label, steps=50) - partition_count = 0 - for name in classifier.get_variable_names(): - if 'language_embedding' in name and 'Adagrad' in name: - partition_count += 1 - self.assertEqual(10, partition_count) - - def testLogisticRegression_MatrixData(self): - """Tests binary classification using matrix data as input.""" - iris = test_data.prepare_iris_data_for_logistic_regression() - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - bucketized_feature = [ - feature_column.bucketized_column( - cont_features[0], - test_data.get_quantile_based_buckets(iris.data, 10)) - ] - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=bucketized_feature, - dnn_feature_columns=cont_features, - dnn_hidden_units=[3, 3]) - - classifier.fit(input_fn=test_data.iris_input_logistic_fn, steps=100) - scores = classifier.evaluate( - input_fn=test_data.iris_input_logistic_fn, steps=100) - _assert_metrics_in_range(('accuracy', 'auc'), scores) - - def testLogisticRegression_TensorData(self): - """Tests binary classification using Tensor data as input.""" - - def _input_fn(): - iris = test_data.prepare_iris_data_for_logistic_regression() - features = {} - for i in range(4): - # The following shows how to provide the Tensor data for - # RealValuedColumns. - features.update({ - str(i): - array_ops.reshape( - constant_op.constant( - iris.data[:, i], dtype=dtypes.float32), [-1, 1]) - }) - # The following shows how to provide the SparseTensor data for - # a SparseColumn. - features['dummy_sparse_column'] = sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [60, 0]], - dense_shape=[len(iris.target), 2]) - labels = array_ops.reshape( - constant_op.constant( - iris.target, dtype=dtypes.int32), [-1, 1]) - return features, labels - - iris = test_data.prepare_iris_data_for_logistic_regression() - cont_features = [ - feature_column.real_valued_column(str(i)) for i in range(4) - ] - linear_features = [ - feature_column.bucketized_column(cont_features[i], - test_data.get_quantile_based_buckets( - iris.data[:, i], 10)) - for i in range(4) - ] - linear_features.append( - feature_column.sparse_column_with_hash_bucket( - 'dummy_sparse_column', hash_bucket_size=100)) - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=linear_features, - dnn_feature_columns=cont_features, - dnn_hidden_units=[3, 3]) - - classifier.fit(input_fn=_input_fn, steps=100) - scores = classifier.evaluate(input_fn=_input_fn, steps=100) - _assert_metrics_in_range(('accuracy', 'auc'), scores) - - def testEstimatorWithCoreFeatureColumns(self): - """Tests binary classification using Tensor data as input.""" - - def _input_fn(): - iris = test_data.prepare_iris_data_for_logistic_regression() - features = {} - for i in range(4): - # The following shows how to provide the Tensor data for - # RealValuedColumns. - features.update({ - str(i): - array_ops.reshape( - constant_op.constant(iris.data[:, i], dtype=dtypes.float32), - [-1, 1]) - }) - # The following shows how to provide the SparseTensor data for - # a SparseColumn. - features['dummy_sparse_column'] = sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [60, 0]], - dense_shape=[len(iris.target), 2]) - labels = array_ops.reshape( - constant_op.constant(iris.target, dtype=dtypes.int32), [-1, 1]) - return features, labels - - iris = test_data.prepare_iris_data_for_logistic_regression() - cont_features = [fc_core.numeric_column(str(i)) for i in range(4)] - linear_features = [ - fc_core.bucketized_column( - cont_features[i], - sorted(set(test_data.get_quantile_based_buckets( - iris.data[:, i], 10)))) for i in range(4) - ] - linear_features.append( - fc_core.categorical_column_with_hash_bucket( - 'dummy_sparse_column', hash_bucket_size=100)) - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=linear_features, - dnn_feature_columns=cont_features, - dnn_hidden_units=[3, 3]) - - classifier.fit(input_fn=_input_fn, steps=100) - scores = classifier.evaluate(input_fn=_input_fn, steps=100) - _assert_metrics_in_range(('accuracy', 'auc'), scores) - - def testTrainWithPartitionedVariables(self): - """Tests training with partitioned variables.""" - - def _input_fn(): - features = { - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - labels = constant_op.constant([[1], [0], [0]]) - return features, labels - - sparse_features = [ - # The given hash_bucket_size results in variables larger than the - # default min_slice_size attribute, so the variables are partitioned. - feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=2e7) - ] - embedding_features = [ - feature_column.embedding_column( - sparse_features[0], dimension=1) - ] - - tf_config = { - 'cluster': { - run_config.TaskType.PS: ['fake_ps_0', 'fake_ps_1'] - } - } - with test.mock.patch.dict('os.environ', - {'TF_CONFIG': json.dumps(tf_config)}): - config = run_config.RunConfig() - # Because we did not start a distributed cluster, we need to pass an - # empty ClusterSpec, otherwise the device_setter will look for - # distributed jobs, such as "/job:ps" which are not present. - config._cluster_spec = server_lib.ClusterSpec({}) - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=sparse_features, - dnn_feature_columns=embedding_features, - dnn_hidden_units=[3, 3], - config=config) - - classifier.fit(input_fn=_input_fn, steps=100) - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - _assert_metrics_in_range(('accuracy', 'auc'), scores) - - def testMultiClass(self): - """Tests multi-class classification using matrix data as input. - - Please see testLogisticRegression_TensorData() for how to use Tensor - data as input instead. - """ - iris = base.load_iris() - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - bucketized_features = [ - feature_column.bucketized_column( - cont_features[0], - test_data.get_quantile_based_buckets(iris.data, 10)) - ] - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - n_classes=3, - linear_feature_columns=bucketized_features, - dnn_feature_columns=cont_features, - dnn_hidden_units=[3, 3]) - - classifier.fit(input_fn=test_data.iris_input_multiclass_fn, steps=100) - scores = classifier.evaluate( - input_fn=test_data.iris_input_multiclass_fn, steps=100) - _assert_metrics_in_range(('accuracy',), scores) - - def testMultiClassLabelKeys(self): - """Tests n_classes > 2 with label_keys vocabulary for labels.""" - # Byte literals needed for python3 test to pass. - label_keys = [b'label0', b'label1', b'label2'] - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[.8], [0.2], [.1]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - labels = constant_op.constant( - [[label_keys[1]], [label_keys[0]], [label_keys[0]]], - dtype=dtypes.string) - return features, labels - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - n_classes=3, - linear_feature_columns=[language_column], - dnn_feature_columns=[ - feature_column.embedding_column( - language_column, dimension=1), - feature_column.real_valued_column('age') - ], - dnn_hidden_units=[3, 3], - label_keys=label_keys) - - classifier.fit(input_fn=_input_fn, steps=50) - - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - _assert_metrics_in_range(('accuracy',), scores) - self.assertIn('loss', scores) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predicted_classes = list( - classifier.predict_classes( - input_fn=predict_input_fn, as_iterable=True)) - self.assertEqual(3, len(predicted_classes)) - for pred in predicted_classes: - self.assertIn(pred, label_keys) - predictions = list( - classifier.predict(input_fn=predict_input_fn, as_iterable=True)) - self.assertAllEqual(predicted_classes, predictions) - - def testLoss(self): - """Tests loss calculation.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # The logistic prediction should be (y = 0.25). - features = {'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32),} - labels = constant_op.constant([[1], [0], [0], [0]]) - return features, labels - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - n_classes=2, - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn_train, steps=100) - scores = classifier.evaluate(input_fn=_input_fn_train, steps=1) - # Cross entropy = -0.25*log(0.25)-0.75*log(0.75) = 0.562 - self.assertAlmostEqual(0.562, scores['loss'], delta=0.1) - - def testLossWithWeights(self): - """Tests loss calculation with weights.""" - - def _input_fn_train(): - # 4 rows with equal weight, one of them (y = x), three of them (y=Not(x)) - # The logistic prediction should be (y = 0.25). - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - return features, labels - - def _input_fn_eval(): - # 4 rows, with different weights. - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[7.], [1.], [1.], [1.]]) - } - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - return features, labels - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - weight_column_name='w', - n_classes=2, - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - classifier.fit(input_fn=_input_fn_train, steps=100) - scores = classifier.evaluate(input_fn=_input_fn_eval, steps=1) - # Weighted cross entropy = (-7*log(0.25)-3*log(0.75))/10 = 1.06 - self.assertAlmostEqual(1.06, scores['loss'], delta=0.1) - - def testTrainWithWeights(self): - """Tests training with given weight column.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # First row has more weight than others. Model should fit (y=x) better - # than (y=Not(x)) due to the relative higher weight of the first row. - labels = constant_op.constant([[1], [0], [0], [0]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[100.], [3.], [2.], [2.]]) - } - return features, labels - - def _input_fn_eval(): - # Create 4 rows (y = x). - labels = constant_op.constant([[1], [1], [1], [1]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - weight_column_name='w', - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - classifier.fit(input_fn=_input_fn_train, steps=100) - scores = classifier.evaluate(input_fn=_input_fn_eval, steps=1) - _assert_metrics_in_range(('accuracy',), scores) - - def testCustomOptimizerByObject(self): - """Tests binary classification using matrix data as input.""" - iris = test_data.prepare_iris_data_for_logistic_regression() - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - bucketized_features = [ - feature_column.bucketized_column( - cont_features[0], - test_data.get_quantile_based_buckets(iris.data, 10)) - ] - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=bucketized_features, - linear_optimizer=ftrl.FtrlOptimizer(learning_rate=0.1), - dnn_feature_columns=cont_features, - dnn_hidden_units=[3, 3], - dnn_optimizer=adagrad.AdagradOptimizer(learning_rate=0.1)) - - classifier.fit(input_fn=test_data.iris_input_logistic_fn, steps=100) - scores = classifier.evaluate( - input_fn=test_data.iris_input_logistic_fn, steps=100) - _assert_metrics_in_range(('accuracy',), scores) - - def testCustomOptimizerByString(self): - """Tests binary classification using matrix data as input.""" - iris = test_data.prepare_iris_data_for_logistic_regression() - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - bucketized_features = [ - feature_column.bucketized_column( - cont_features[0], - test_data.get_quantile_based_buckets(iris.data, 10)) - ] - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=bucketized_features, - linear_optimizer='Ftrl', - dnn_feature_columns=cont_features, - dnn_hidden_units=[3, 3], - dnn_optimizer='Adagrad') - - classifier.fit(input_fn=test_data.iris_input_logistic_fn, steps=100) - scores = classifier.evaluate( - input_fn=test_data.iris_input_logistic_fn, steps=100) - _assert_metrics_in_range(('accuracy',), scores) - - def testCustomOptimizerByFunction(self): - """Tests binary classification using matrix data as input.""" - iris = test_data.prepare_iris_data_for_logistic_regression() - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - bucketized_features = [ - feature_column.bucketized_column( - cont_features[0], - test_data.get_quantile_based_buckets(iris.data, 10)) - ] - - def _optimizer_exp_decay(): - global_step = training_util.get_global_step() - learning_rate = learning_rate_decay.exponential_decay( - learning_rate=0.1, - global_step=global_step, - decay_steps=100, - decay_rate=0.001) - return adagrad.AdagradOptimizer(learning_rate=learning_rate) - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=bucketized_features, - linear_optimizer=_optimizer_exp_decay, - dnn_feature_columns=cont_features, - dnn_hidden_units=[3, 3], - dnn_optimizer=_optimizer_exp_decay) - - classifier.fit(input_fn=test_data.iris_input_logistic_fn, steps=100) - scores = classifier.evaluate( - input_fn=test_data.iris_input_logistic_fn, steps=100) - _assert_metrics_in_range(('accuracy',), scores) - - def testPredict(self): - """Tests weight column in evaluation.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - labels = constant_op.constant([[1], [0], [0], [0]]) - features = {'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32)} - return features, labels - - def _input_fn_predict(): - y = input_lib.limit_epochs( - array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), num_epochs=1) - features = {'x': y} - return features - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3]) - - classifier.fit(input_fn=_input_fn_train, steps=100) - - probs = list(classifier.predict_proba(input_fn=_input_fn_predict)) - self.assertAllClose([[0.75, 0.25]] * 4, probs, 0.05) - classes = list(classifier.predict_classes(input_fn=_input_fn_predict)) - self.assertListEqual([0] * 4, classes) - - def testCustomMetrics(self): - """Tests custom evaluation metrics.""" - - def _input_fn(num_epochs=None): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - labels = constant_op.constant([[1], [0], [0], [0]]) - features = { - 'x': - input_lib.limit_epochs( - array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - num_epochs=num_epochs) - } - return features, labels - - def _my_metric_op(predictions, labels): - # For the case of binary classification, the 2nd column of "predictions" - # denotes the model predictions. - labels = math_ops.cast(labels, dtypes.float32) - predictions = array_ops.strided_slice( - predictions, [0, 1], [-1, 2], end_mask=1) - return math_ops.reduce_sum(math_ops.multiply(predictions, labels)) - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3]) - - classifier.fit(input_fn=_input_fn, steps=100) - scores = classifier.evaluate( - input_fn=_input_fn, - steps=100, - metrics={ - 'my_accuracy': - MetricSpec( - metric_fn=metric_ops.streaming_accuracy, - prediction_key='classes'), - 'my_precision': - MetricSpec( - metric_fn=metric_ops.streaming_precision, - prediction_key='classes'), - 'my_metric': - MetricSpec( - metric_fn=_my_metric_op, prediction_key='probabilities') - }) - self.assertTrue( - set(['loss', 'my_accuracy', 'my_precision', 'my_metric']).issubset( - set(scores.keys()))) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = np.array(list(classifier.predict_classes( - input_fn=predict_input_fn))) - self.assertEqual( - _sklearn.accuracy_score([1, 0, 0, 0], predictions), - scores['my_accuracy']) - - # Test the case where the 2nd element of the key is neither "classes" nor - # "probabilities". - with self.assertRaisesRegexp(KeyError, 'bad_type'): - classifier.evaluate( - input_fn=_input_fn, - steps=100, - metrics={('bad_name', 'bad_type'): metric_ops.streaming_auc}) - - # Test the case where the tuple of the key doesn't have 2 elements. - with self.assertRaises(ValueError): - classifier.evaluate( - input_fn=_input_fn, - steps=100, - metrics={ - ('bad_length_name', 'classes', 'bad_length'): - metric_ops.streaming_accuracy - }) - - # Test the case where the prediction_key is neither "classes" nor - # "probabilities". - with self.assertRaisesRegexp(KeyError, 'bad_type'): - classifier.evaluate( - input_fn=_input_fn, - steps=100, - metrics={ - 'bad_name': - MetricSpec( - metric_fn=metric_ops.streaming_auc, - prediction_key='bad_type') - }) - - def testVariableQuery(self): - """Tests get_variable_names and get_variable_value.""" - - def _input_fn_train(): - # Create 4 rows, three (y = x), one (y=Not(x)) - labels = constant_op.constant([[1], [1], [1], [0]]) - features = {'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32),} - return features, labels - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3]) - - classifier.fit(input_fn=_input_fn_train, steps=500) - var_names = classifier.get_variable_names() - self.assertGreater(len(var_names), 3) - for name in var_names: - classifier.get_variable_value(name) - - def testExport(self): - """Tests export model for servo.""" - - def input_fn(): - return { - 'age': - constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column.sparse_column_with_hash_bucket('language', 100) - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=[ - feature_column.real_valued_column('age'), - language, - ], - dnn_feature_columns=[ - feature_column.embedding_column( - language, dimension=1), - ], - dnn_hidden_units=[3, 3]) - classifier.fit(input_fn=input_fn, steps=100) - - export_dir = tempfile.mkdtemp() - input_feature_key = 'examples' - - def serving_input_fn(): - features, targets = input_fn() - features[input_feature_key] = array_ops.placeholder(dtypes.string) - return features, targets - - classifier.export( - export_dir, - serving_input_fn, - input_feature_key, - use_deprecated_input_fn=False) - - def testCenteredBias(self): - """Tests bias is centered or not.""" - - def _input_fn_train(): - # Create 4 rows, three (y = x), one (y=Not(x)) - labels = constant_op.constant([[1], [1], [1], [0]]) - features = {'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32),} - return features, labels - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - enable_centered_bias=True) - - classifier.fit(input_fn=_input_fn_train, steps=1000) - self.assertIn('binary_logistic_head/centered_bias_weight', - classifier.get_variable_names()) - # logodds(0.75) = 1.09861228867 - self.assertAlmostEqual( - 1.0986, - float(classifier.get_variable_value( - 'binary_logistic_head/centered_bias_weight')[0]), - places=2) - - def testDisableCenteredBias(self): - """Tests bias is centered or not.""" - - def _input_fn_train(): - # Create 4 rows, three (y = x), one (y=Not(x)) - labels = constant_op.constant([[1], [1], [1], [0]]) - features = {'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32),} - return features, labels - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - enable_centered_bias=False) - - classifier.fit(input_fn=_input_fn_train, steps=500) - self.assertNotIn('centered_bias_weight', classifier.get_variable_names()) - - def testGlobalStepLinearOnly(self): - """Tests global step update for linear-only model.""" - - def input_fn(): - return { - 'age': constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column.sparse_column_with_hash_bucket('language', 10) - age = feature_column.real_valued_column('age') - - step_counter = _StepCounterHook() - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=[age, language]) - classifier.fit(input_fn=input_fn, steps=100, monitors=[step_counter]) - - self.assertEqual(100, step_counter.steps) - - def testGlobalStepDNNOnly(self): - """Tests global step update for dnn-only model.""" - - def input_fn(): - return { - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column.sparse_column_with_hash_bucket('language', 10) - - step_counter = _StepCounterHook() - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - dnn_feature_columns=[ - feature_column.embedding_column(language, dimension=1)], - dnn_hidden_units=[3, 3]) - classifier.fit(input_fn=input_fn, steps=100, monitors=[step_counter]) - - self.assertEqual(100, step_counter.steps) - - def testGlobalStepDNNLinearCombinedBug(self): - """Tests global step update for dnn-linear combined model.""" - - def input_fn(): - return { - 'age': constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column.sparse_column_with_hash_bucket('language', 10) - age = feature_column.real_valued_column('age') - - step_counter = _StepCounterHook() - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=[age, language], - dnn_feature_columns=[ - feature_column.embedding_column(language, dimension=1)], - dnn_hidden_units=[3, 3], - fix_global_step_increment_bug=False) - classifier.fit(input_fn=input_fn, steps=100, monitors=[step_counter]) - global_step = classifier.get_variable_value('global_step') - - if global_step == 100: - # Expected is 100, but because of the global step increment bug, is 50. - # Occasionally, step increments one more time due to a race condition, - # reaching 51 steps. - self.assertIn(step_counter.steps, [50, 51]) - else: - # Occasionally, training stops when global_step == 102, due to a race - # condition. In addition, occasionally step increments one more time due - # to a race condition reaching 52 steps. - self.assertIn(step_counter.steps, [51, 52]) - - def testGlobalStepDNNLinearCombinedBugFixed(self): - """Tests global step update for dnn-linear combined model.""" - - def input_fn(): - return { - 'age': constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column.sparse_column_with_hash_bucket('language', 10) - age = feature_column.real_valued_column('age') - - step_counter = _StepCounterHook() - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=[age, language], - dnn_feature_columns=[ - feature_column.embedding_column(language, dimension=1)], - dnn_hidden_units=[3, 3], - fix_global_step_increment_bug=True) - classifier.fit(input_fn=input_fn, steps=100, monitors=[step_counter]) - - self.assertEqual(100, step_counter.steps) - - def testLinearOnly(self): - """Tests that linear-only instantiation works.""" - - def input_fn(): - return { - 'age': - constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column.sparse_column_with_hash_bucket('language', 100) - age = feature_column.real_valued_column('age') - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=[age, language]) - classifier.fit(input_fn=input_fn, steps=100) - loss1 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - classifier.fit(input_fn=input_fn, steps=200) - loss2 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss2, loss1) - - variable_names = classifier.get_variable_names() - self.assertNotIn('dnn/logits/biases', variable_names) - self.assertNotIn('dnn/logits/weights', variable_names) - self.assertIn('linear/bias_weight', variable_names) - self.assertIn('linear/age/weight', variable_names) - self.assertIn('linear/language/weights', variable_names) - self.assertEquals( - 1, len(classifier.get_variable_value('linear/age/weight'))) - self.assertEquals( - 100, len(classifier.get_variable_value('linear/language/weights'))) - - def testLinearOnlyOneFeature(self): - """Tests that linear-only instantiation works for one feature only.""" - - def input_fn(): - return { - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column.sparse_column_with_hash_bucket('language', 99) - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=[language]) - classifier.fit(input_fn=input_fn, steps=100) - loss1 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - classifier.fit(input_fn=input_fn, steps=200) - loss2 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss2, loss1) - - variable_names = classifier.get_variable_names() - self.assertNotIn('dnn/logits/biases', variable_names) - self.assertNotIn('dnn/logits/weights', variable_names) - self.assertIn('linear/bias_weight', variable_names) - self.assertIn('linear/language/weights', variable_names) - self.assertEquals( - 1, len(classifier.get_variable_value('linear/bias_weight'))) - self.assertEquals( - 99, len(classifier.get_variable_value('linear/language/weights'))) - - def testDNNOnly(self): - """Tests that DNN-only instantiation works.""" - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - n_classes=3, dnn_feature_columns=cont_features, dnn_hidden_units=[3, 3]) - - classifier.fit(input_fn=test_data.iris_input_multiclass_fn, steps=1000) - classifier.evaluate(input_fn=test_data.iris_input_multiclass_fn, steps=100) - - variable_names = classifier.get_variable_names() - self.assertIn('dnn/hiddenlayer_0/weights', variable_names) - self.assertIn('dnn/hiddenlayer_0/biases', variable_names) - self.assertIn('dnn/hiddenlayer_1/weights', variable_names) - self.assertIn('dnn/hiddenlayer_1/biases', variable_names) - self.assertIn('dnn/logits/weights', variable_names) - self.assertIn('dnn/logits/biases', variable_names) - self.assertNotIn('linear/bias_weight', variable_names) - self.assertNotIn('linear/feature_BUCKETIZED/weight', variable_names) - - def testDNNWeightsBiasesNames(self): - """Tests the names of DNN weights and biases in the checkpoints.""" - - def _input_fn_train(): - # Create 4 rows, three (y = x), one (y=Not(x)) - labels = constant_op.constant([[1], [1], [1], [0]]) - features = {'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32),} - return features, labels - - classifier = dnn_linear_combined.DNNLinearCombinedClassifier( - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3]) - - classifier.fit(input_fn=_input_fn_train, steps=5) - variable_names = classifier.get_variable_names() - self.assertIn('dnn/hiddenlayer_0/weights', variable_names) - self.assertIn('dnn/hiddenlayer_0/biases', variable_names) - self.assertIn('dnn/hiddenlayer_1/weights', variable_names) - self.assertIn('dnn/hiddenlayer_1/biases', variable_names) - self.assertIn('dnn/logits/weights', variable_names) - self.assertIn('dnn/logits/biases', variable_names) - - -class DNNLinearCombinedRegressorTest(test.TestCase): - - def testExperimentIntegration(self): - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - exp = experiment.Experiment( - estimator=dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=cont_features, - dnn_feature_columns=cont_features, - dnn_hidden_units=[3, 3]), - train_input_fn=test_data.iris_input_logistic_fn, - eval_input_fn=test_data.iris_input_logistic_fn) - exp.test() - - def testEstimatorContract(self): - estimator_test_utils.assert_estimator_contract( - self, dnn_linear_combined.DNNLinearCombinedRegressor) - - def testRegression_MatrixData(self): - """Tests regression using matrix data as input.""" - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - regressor = dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=cont_features, - dnn_feature_columns=cont_features, - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=test_data.iris_input_logistic_fn, steps=10) - scores = regressor.evaluate( - input_fn=test_data.iris_input_logistic_fn, steps=1) - self.assertIn('loss', scores.keys()) - - def testRegression_TensorData(self): - """Tests regression using tensor data as input.""" - - def _input_fn(): - # Create 4 rows of (y = x) - labels = constant_op.constant([[100.], [3.], [2.], [2.]]) - features = {'x': constant_op.constant([[100.], [3.], [2.], [2.]])} - return features, labels - - classifier = dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn, steps=10) - classifier.evaluate(input_fn=_input_fn, steps=1) - - def testLoss(self): - """Tests loss calculation.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # The algorithm should learn (y = 0.25). - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = {'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32),} - return features, labels - - regressor = dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn_train, steps=100) - scores = regressor.evaluate(input_fn=_input_fn_train, steps=1) - # Average square loss = (0.75^2 + 3*0.25^2) / 4 = 0.1875 - self.assertAlmostEqual(0.1875, scores['loss'], delta=0.1) - - def testLossWithWeights(self): - """Tests loss calculation with weights.""" - - def _input_fn_train(): - # 4 rows with equal weight, one of them (y = x), three of them (y=Not(x)) - # The algorithm should learn (y = 0.25). - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - def _input_fn_eval(): - # 4 rows, with different weights. - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[7.], [1.], [1.], [1.]]) - } - return features, labels - - regressor = dnn_linear_combined.DNNLinearCombinedRegressor( - weight_column_name='w', - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn_train, steps=100) - scores = regressor.evaluate(input_fn=_input_fn_eval, steps=1) - # Weighted average square loss = (7*0.75^2 + 3*0.25^2) / 10 = 0.4125 - self.assertAlmostEqual(0.4125, scores['loss'], delta=0.1) - - def testTrainWithWeights(self): - """Tests training with given weight column.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # First row has more weight than others. Model should fit (y=x) better - # than (y=Not(x)) due to the relative higher weight of the first row. - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[100.], [3.], [2.], [2.]]) - } - return features, labels - - def _input_fn_eval(): - # Create 4 rows (y = x) - labels = constant_op.constant([[1.], [1.], [1.], [1.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - regressor = dnn_linear_combined.DNNLinearCombinedRegressor( - weight_column_name='w', - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn_train, steps=100) - scores = regressor.evaluate(input_fn=_input_fn_eval, steps=1) - # The model should learn (y = x) because of the weights, so the loss should - # be close to zero. - self.assertLess(scores['loss'], 0.2) - - def testPredict_AsIterableFalse(self): - """Tests predict method with as_iterable=False.""" - labels = [1., 0., 0.2] - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant(labels, dtype=dtypes.float32) - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - - regressor = dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=[ - language_column, feature_column.real_valued_column('age') - ], - dnn_feature_columns=[ - feature_column.embedding_column( - language_column, dimension=1), - feature_column.real_valued_column('age') - ], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=10) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores.keys()) - regressor.predict_scores(input_fn=_input_fn, as_iterable=False) - - def testPredict_AsIterable(self): - """Tests predict method with as_iterable=True.""" - labels = [1., 0., 0.2] - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant(labels, dtype=dtypes.float32) - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - - regressor = dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=[ - language_column, feature_column.real_valued_column('age') - ], - dnn_feature_columns=[ - feature_column.embedding_column( - language_column, dimension=1), - feature_column.real_valued_column('age') - ], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=10) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores.keys()) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - regressor.predict_scores(input_fn=predict_input_fn, as_iterable=True) - - def testCustomMetrics(self): - """Tests custom evaluation metrics.""" - - def _input_fn(num_epochs=None): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': - input_lib.limit_epochs( - array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - num_epochs=num_epochs) - } - return features, labels - - def _my_metric_op(predictions, labels): - return math_ops.reduce_sum(math_ops.multiply(predictions, labels)) - - regressor = dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=10) - scores = regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - 'my_error': metric_ops.streaming_mean_squared_error, - ('my_metric', 'scores'): _my_metric_op - }) - self.assertIn('loss', set(scores.keys())) - self.assertIn('my_error', set(scores.keys())) - self.assertIn('my_metric', set(scores.keys())) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = np.array(list(regressor.predict_scores( - input_fn=predict_input_fn))) - self.assertAlmostEqual( - _sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions), - scores['my_error']) - - # Tests the case that the 2nd element of the key is not "scores". - with self.assertRaises(KeyError): - regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - ('my_error', 'predictions'): - metric_ops.streaming_mean_squared_error - }) - - # Tests the case where the tuple of the key doesn't have 2 elements. - with self.assertRaises(ValueError): - regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - ('bad_length_name', 'scores', 'bad_length'): - metric_ops.streaming_mean_squared_error - }) - - def testCustomMetricsWithMetricSpec(self): - """Tests custom evaluation metrics.""" - - def _input_fn(num_epochs=None): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': - input_lib.limit_epochs( - array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - num_epochs=num_epochs) - } - return features, labels - - def _my_metric_op(predictions, labels): - return math_ops.reduce_sum(math_ops.multiply(predictions, labels)) - - regressor = dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=5) - scores = regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - 'my_error': - MetricSpec( - metric_fn=metric_ops.streaming_mean_squared_error, - prediction_key='scores'), - 'my_metric': - MetricSpec( - metric_fn=_my_metric_op, prediction_key='scores') - }) - self.assertIn('loss', set(scores.keys())) - self.assertIn('my_error', set(scores.keys())) - self.assertIn('my_metric', set(scores.keys())) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = np.array(list(regressor.predict_scores( - input_fn=predict_input_fn))) - self.assertAlmostEqual( - _sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions), - scores['my_error']) - - # Tests the case where the prediction_key is not "scores". - with self.assertRaisesRegexp(KeyError, 'bad_type'): - regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - 'bad_name': - MetricSpec( - metric_fn=metric_ops.streaming_auc, - prediction_key='bad_type') - }) - - def testExport(self): - """Tests export model for servo.""" - labels = [1., 0., 0.2] - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant(labels, dtype=dtypes.float32) - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - - regressor = dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=[ - language_column, feature_column.real_valued_column('age') - ], - dnn_feature_columns=[ - feature_column.embedding_column( - language_column, dimension=1), - ], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=10) - - export_dir = tempfile.mkdtemp() - input_feature_key = 'examples' - - def serving_input_fn(): - features, targets = _input_fn() - features[input_feature_key] = array_ops.placeholder(dtypes.string) - return features, targets - - regressor.export( - export_dir, - serving_input_fn, - input_feature_key, - use_deprecated_input_fn=False) - - def testTrainSaveLoad(self): - """Tests regression with restarting training / evaluate.""" - - def _input_fn(num_epochs=None): - # Create 4 rows of (y = x) - labels = constant_op.constant([[100.], [3.], [2.], [2.]]) - features = { - 'x': - input_lib.limit_epochs( - constant_op.constant([[100.], [3.], [2.], [2.]]), - num_epochs=num_epochs) - } - return features, labels - - model_dir = tempfile.mkdtemp() - # pylint: disable=g-long-lambda - new_regressor = lambda: dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - model_dir=model_dir, - config=run_config.RunConfig(tf_random_seed=1)) - - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - regressor = new_regressor() - regressor.fit(input_fn=_input_fn, steps=10) - predictions = list(regressor.predict_scores(input_fn=predict_input_fn)) - del regressor - - regressor = new_regressor() - predictions2 = list(regressor.predict_scores(input_fn=predict_input_fn)) - self.assertAllClose(predictions, predictions2) - - def testTrainWithPartitionedVariables(self): - """Tests training with partitioned variables.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([1., 0., 0.2], dtype=dtypes.float32) - - # The given hash_bucket_size results in variables larger than the - # default min_slice_size attribute, so the variables are partitioned. - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=2e7) - - tf_config = { - 'cluster': { - run_config.TaskType.PS: ['fake_ps_0', 'fake_ps_1'] - } - } - with test.mock.patch.dict('os.environ', - {'TF_CONFIG': json.dumps(tf_config)}): - config = run_config.RunConfig(tf_random_seed=1) - # Because we did not start a distributed cluster, we need to pass an - # empty ClusterSpec, otherwise the device_setter will look for - # distributed jobs, such as "/job:ps" which are not present. - config._cluster_spec = server_lib.ClusterSpec({}) - - regressor = dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=[ - language_column, feature_column.real_valued_column('age') - ], - dnn_feature_columns=[ - feature_column.embedding_column( - language_column, dimension=1), - feature_column.real_valued_column('age') - ], - dnn_hidden_units=[3, 3], - config=config) - - regressor.fit(input_fn=_input_fn, steps=100) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores.keys()) - - def testDisableCenteredBias(self): - """Tests that we can disable centered bias.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([1., 0., 0.2], dtype=dtypes.float32) - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - - regressor = dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=[ - language_column, feature_column.real_valued_column('age') - ], - dnn_feature_columns=[ - feature_column.embedding_column( - language_column, dimension=1), - feature_column.real_valued_column('age') - ], - dnn_hidden_units=[3, 3], - enable_centered_bias=False, - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=100) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores.keys()) - - def testLinearOnly(self): - """Tests linear-only instantiation and training.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([1., 0., 0.2], dtype=dtypes.float32) - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - - regressor = dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=[ - language_column, feature_column.real_valued_column('age') - ], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=100) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores.keys()) - - def testDNNOnly(self): - """Tests DNN-only instantiation and training.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([1., 0., 0.2], dtype=dtypes.float32) - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - - regressor = dnn_linear_combined.DNNLinearCombinedRegressor( - dnn_feature_columns=[ - feature_column.embedding_column( - language_column, dimension=1), - feature_column.real_valued_column('age') - ], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=100) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores.keys()) - - -class FeatureEngineeringFunctionTest(test.TestCase): - """Tests feature_engineering_fn.""" - - def testNoneFeatureEngineeringFn(self): - - def input_fn(): - # Create 4 rows of (y = x) - labels = constant_op.constant([[100.], [3.], [2.], [2.]]) - features = {'x': constant_op.constant([[100.], [3.], [2.], [2.]])} - return features, labels - - def feature_engineering_fn(features, labels): - _, _ = features, labels - labels = constant_op.constant([[1000.], [30.], [20.], [20.]]) - features = {'x': constant_op.constant([[1000.], [30.], [20.], [20.]])} - return features, labels - - estimator_with_fe_fn = dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1), - feature_engineering_fn=feature_engineering_fn) - estimator_with_fe_fn.fit(input_fn=input_fn, steps=110) - - estimator_without_fe_fn = dnn_linear_combined.DNNLinearCombinedRegressor( - linear_feature_columns=[feature_column.real_valued_column('x')], - dnn_feature_columns=[feature_column.real_valued_column('x')], - dnn_hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - estimator_without_fe_fn.fit(input_fn=input_fn, steps=110) - - # predictions = y - prediction_with_fe_fn = next( - estimator_with_fe_fn.predict_scores( - input_fn=input_fn, as_iterable=True)) - self.assertAlmostEqual(1000., prediction_with_fe_fn, delta=10.0) - prediction_without_fe_fn = next( - estimator_without_fe_fn.predict_scores( - input_fn=input_fn, as_iterable=True)) - self.assertAlmostEqual(100., prediction_without_fe_fn, delta=1.0) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py deleted file mode 100644 index d779495720b..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py +++ /dev/null @@ -1,1630 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for DNNEstimators.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import json -import tempfile - -import numpy as np - -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.learn.python.learn import experiment -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.contrib.learn.python.learn.estimators import _sklearn -from tensorflow.contrib.learn.python.learn.estimators import dnn -from tensorflow.contrib.learn.python.learn.estimators import dnn_linear_combined -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import estimator_test_utils -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.contrib.learn.python.learn.estimators import test_data -from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec -from tensorflow.contrib.metrics.python.ops import metric_ops -from tensorflow.python.feature_column import feature_column_lib as fc_core -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test -from tensorflow.python.training import input as input_lib -from tensorflow.python.training import monitored_session -from tensorflow.python.training import server_lib - - -class EmbeddingMultiplierTest(test.TestCase): - """dnn_model_fn tests.""" - - def testRaisesNonEmbeddingColumn(self): - one_hot_language = feature_column.one_hot_column( - feature_column.sparse_column_with_hash_bucket('language', 10)) - - params = { - 'feature_columns': [one_hot_language], - 'head': head_lib.multi_class_head(2), - 'hidden_units': [1], - # Set lr mult to 0. to keep embeddings constant. - 'embedding_lr_multipliers': { - one_hot_language: 0.0 - }, - } - features = { - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - } - labels = constant_op.constant([[0], [0], [0]], dtype=dtypes.int32) - with self.assertRaisesRegexp(ValueError, - 'can only be defined for embedding columns'): - dnn._dnn_model_fn(features, labels, model_fn.ModeKeys.TRAIN, params) - - def testMultipliesGradient(self): - embedding_language = feature_column.embedding_column( - feature_column.sparse_column_with_hash_bucket('language', 10), - dimension=1, - initializer=init_ops.constant_initializer(0.1)) - embedding_wire = feature_column.embedding_column( - feature_column.sparse_column_with_hash_bucket('wire', 10), - dimension=1, - initializer=init_ops.constant_initializer(0.1)) - - params = { - 'feature_columns': [embedding_language, embedding_wire], - 'head': head_lib.multi_class_head(2), - 'hidden_units': [1], - # Set lr mult to 0. to keep embeddings constant. - 'embedding_lr_multipliers': { - embedding_language: 0.0 - }, - } - features = { - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - 'wire': - sparse_tensor.SparseTensor( - values=['omar', 'stringer', 'marlo'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - } - labels = constant_op.constant([[0], [0], [0]], dtype=dtypes.int32) - model_ops = dnn._dnn_model_fn(features, labels, model_fn.ModeKeys.TRAIN, - params) - with monitored_session.MonitoredSession() as sess: - language_var = dnn_linear_combined._get_embedding_variable( - embedding_language, 'dnn', 'dnn/input_from_feature_columns') - wire_var = dnn_linear_combined._get_embedding_variable( - embedding_wire, 'dnn', 'dnn/input_from_feature_columns') - for _ in range(2): - _, language_value, wire_value = sess.run( - [model_ops.train_op, language_var, wire_var]) - initial_value = np.full_like(language_value, 0.1) - self.assertTrue(np.all(np.isclose(language_value, initial_value))) - self.assertFalse(np.all(np.isclose(wire_value, initial_value))) - - -class ActivationFunctionTest(test.TestCase): - - def _getModelForActivation(self, activation_fn): - embedding_language = feature_column.embedding_column( - feature_column.sparse_column_with_hash_bucket('language', 10), - dimension=1, - initializer=init_ops.constant_initializer(0.1)) - params = { - 'feature_columns': [embedding_language], - 'head': head_lib.multi_class_head(2), - 'hidden_units': [1], - 'activation_fn': activation_fn, - } - features = { - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - } - labels = constant_op.constant([[0], [0], [0]], dtype=dtypes.int32) - return dnn._dnn_model_fn(features, labels, model_fn.ModeKeys.TRAIN, params) - - def testValidActivation(self): - _ = self._getModelForActivation('relu') - - def testRaisesOnBadActivationName(self): - with self.assertRaisesRegexp(ValueError, - 'Activation name should be one of'): - self._getModelForActivation('max_pool') - - -class DNNEstimatorTest(test.TestCase): - - def _assertInRange(self, expected_min, expected_max, actual): - self.assertLessEqual(expected_min, actual) - self.assertGreaterEqual(expected_max, actual) - - def testExperimentIntegration(self): - exp = experiment.Experiment( - estimator=dnn.DNNClassifier( - n_classes=3, - feature_columns=[ - feature_column.real_valued_column( - 'feature', dimension=4) - ], - hidden_units=[3, 3]), - train_input_fn=test_data.iris_input_multiclass_fn, - eval_input_fn=test_data.iris_input_multiclass_fn) - exp.test() - - def testEstimatorContract(self): - estimator_test_utils.assert_estimator_contract(self, dnn.DNNEstimator) - - def testTrainWithWeights(self): - """Tests training with given weight column.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # First row has more weight than others. Model should fit (y=x) better - # than (y=Not(x)) due to the relative higher weight of the first row. - labels = constant_op.constant([[1], [0], [0], [0]]) - features = { - 'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[100.], [3.], [2.], [2.]]) - } - return features, labels - - def _input_fn_eval(): - # Create 4 rows (y = x) - labels = constant_op.constant([[1], [1], [1], [1]]) - features = { - 'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - dnn_estimator = dnn.DNNEstimator( - head=head_lib.multi_class_head(2, weight_column_name='w'), - feature_columns=[feature_column.real_valued_column('x')], - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - dnn_estimator.fit(input_fn=_input_fn_train, steps=5) - scores = dnn_estimator.evaluate(input_fn=_input_fn_eval, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - - -class DNNClassifierTest(test.TestCase): - - def testExperimentIntegration(self): - exp = experiment.Experiment( - estimator=dnn.DNNClassifier( - n_classes=3, - feature_columns=[ - feature_column.real_valued_column( - 'feature', dimension=4) - ], - hidden_units=[3, 3]), - train_input_fn=test_data.iris_input_multiclass_fn, - eval_input_fn=test_data.iris_input_multiclass_fn) - exp.test() - - def _assertInRange(self, expected_min, expected_max, actual): - self.assertLessEqual(expected_min, actual) - self.assertGreaterEqual(expected_max, actual) - - def testEstimatorContract(self): - estimator_test_utils.assert_estimator_contract(self, dnn.DNNClassifier) - - def testEmbeddingMultiplier(self): - embedding_language = feature_column.embedding_column( - feature_column.sparse_column_with_hash_bucket('language', 10), - dimension=1, - initializer=init_ops.constant_initializer(0.1)) - classifier = dnn.DNNClassifier( - feature_columns=[embedding_language], - hidden_units=[3, 3], - embedding_lr_multipliers={embedding_language: 0.8}) - self.assertEqual({ - embedding_language: 0.8 - }, classifier.params['embedding_lr_multipliers']) - - def testInputPartitionSize(self): - def _input_fn_float_label(num_epochs=None): - features = { - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - labels = constant_op.constant([[0.8], [0.], [0.2]], dtype=dtypes.float32) - return features, labels - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column(language_column, dimension=1), - ] - - # Set num_ps_replica to be 10 and the min slice size to be extremely small, - # so as to ensure that there'll be 10 partititions produced. - config = run_config.RunConfig(tf_random_seed=1) - config._num_ps_replicas = 10 - classifier = dnn.DNNClassifier( - n_classes=2, - feature_columns=feature_columns, - hidden_units=[3, 3], - optimizer='Adagrad', - config=config, - input_layer_min_slice_size=1) - - # Ensure the param is passed in. - self.assertEqual(1, classifier.params['input_layer_min_slice_size']) - - # Ensure the partition count is 10. - classifier.fit(input_fn=_input_fn_float_label, steps=50) - partition_count = 0 - for name in classifier.get_variable_names(): - if 'language_embedding' in name and 'Adagrad' in name: - partition_count += 1 - self.assertEqual(10, partition_count) - - def testLogisticRegression_MatrixData(self): - """Tests binary classification using matrix data as input.""" - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - classifier = dnn.DNNClassifier( - feature_columns=cont_features, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - input_fn = test_data.iris_input_logistic_fn - classifier.fit(input_fn=input_fn, steps=5) - scores = classifier.evaluate(input_fn=input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - self.assertIn('loss', scores) - - def testLogisticRegression_MatrixData_Labels1D(self): - """Same as the last test, but label shape is [100] instead of [100, 1].""" - - def _input_fn(): - iris = test_data.prepare_iris_data_for_logistic_regression() - return { - 'feature': constant_op.constant( - iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=[100], dtype=dtypes.int32) - - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - classifier = dnn.DNNClassifier( - feature_columns=cont_features, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn, steps=5) - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores) - - def testLogisticRegression_NpMatrixData(self): - """Tests binary classification using numpy matrix data as input.""" - iris = test_data.prepare_iris_data_for_logistic_regression() - train_x = iris.data - train_y = iris.target - feature_columns = [feature_column.real_valued_column('', dimension=4)] - classifier = dnn.DNNClassifier( - feature_columns=feature_columns, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(x=train_x, y=train_y, steps=5) - scores = classifier.evaluate(x=train_x, y=train_y, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - - def _assertBinaryPredictions(self, expected_len, predictions): - self.assertEqual(expected_len, len(predictions)) - for prediction in predictions: - self.assertIn(prediction, (0, 1)) - - def _assertClassificationPredictions( - self, expected_len, n_classes, predictions): - self.assertEqual(expected_len, len(predictions)) - for prediction in predictions: - self.assertIn(prediction, range(n_classes)) - - def _assertProbabilities(self, expected_batch_size, expected_n_classes, - probabilities): - self.assertEqual(expected_batch_size, len(probabilities)) - for b in range(expected_batch_size): - self.assertEqual(expected_n_classes, len(probabilities[b])) - for i in range(expected_n_classes): - self._assertInRange(0.0, 1.0, probabilities[b][i]) - - def testEstimatorWithCoreFeatureColumns(self): - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[.8], [0.2], [.1]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([[1], [0], [0]], dtype=dtypes.int32) - - language_column = fc_core.categorical_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - fc_core.embedding_column(language_column, dimension=1), - fc_core.numeric_column('age') - ] - - classifier = dnn.DNNClassifier( - n_classes=2, - feature_columns=feature_columns, - hidden_units=[10, 10], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn, steps=50) - - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - self.assertIn('loss', scores) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predicted_classes = list( - classifier.predict_classes(input_fn=predict_input_fn, as_iterable=True)) - self._assertBinaryPredictions(3, predicted_classes) - predictions = list( - classifier.predict(input_fn=predict_input_fn, as_iterable=True)) - self.assertAllEqual(predicted_classes, predictions) - - def testLogisticRegression_TensorData(self): - """Tests binary classification using tensor data as input.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[.8], [0.2], [.1]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([[1], [0], [0]], dtype=dtypes.int32) - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column( - language_column, dimension=1), - feature_column.real_valued_column('age') - ] - - classifier = dnn.DNNClassifier( - n_classes=2, - feature_columns=feature_columns, - hidden_units=[10, 10], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn, steps=50) - - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - self.assertIn('loss', scores) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predicted_classes = list( - classifier.predict_classes( - input_fn=predict_input_fn, as_iterable=True)) - self._assertBinaryPredictions(3, predicted_classes) - predictions = list( - classifier.predict(input_fn=predict_input_fn, as_iterable=True)) - self.assertAllEqual(predicted_classes, predictions) - - def testLogisticRegression_FloatLabel(self): - """Tests binary classification with float labels.""" - - def _input_fn_float_label(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[50], [20], [10]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - labels = constant_op.constant([[0.8], [0.], [0.2]], dtype=dtypes.float32) - return features, labels - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column( - language_column, dimension=1), - feature_column.real_valued_column('age') - ] - - classifier = dnn.DNNClassifier( - n_classes=2, - feature_columns=feature_columns, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn_float_label, steps=50) - - predict_input_fn = functools.partial(_input_fn_float_label, num_epochs=1) - predicted_classes = list( - classifier.predict_classes( - input_fn=predict_input_fn, as_iterable=True)) - self._assertBinaryPredictions(3, predicted_classes) - predictions = list( - classifier.predict( - input_fn=predict_input_fn, as_iterable=True)) - self.assertAllEqual(predicted_classes, predictions) - predictions_proba = list( - classifier.predict_proba( - input_fn=predict_input_fn, as_iterable=True)) - self._assertProbabilities(3, 2, predictions_proba) - - def testMultiClass_MatrixData(self): - """Tests multi-class classification using matrix data as input.""" - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - classifier = dnn.DNNClassifier( - n_classes=3, - feature_columns=cont_features, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - input_fn = test_data.iris_input_multiclass_fn - classifier.fit(input_fn=input_fn, steps=200) - scores = classifier.evaluate(input_fn=input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - self.assertIn('loss', scores) - - def testMultiClass_MatrixData_Labels1D(self): - """Same as the last test, but label shape is [150] instead of [150, 1].""" - - def _input_fn(): - iris = base.load_iris() - return { - 'feature': constant_op.constant( - iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=[150], dtype=dtypes.int32) - - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - classifier = dnn.DNNClassifier( - n_classes=3, - feature_columns=cont_features, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn, steps=200) - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - - def testMultiClass_NpMatrixData(self): - """Tests multi-class classification using numpy matrix data as input.""" - iris = base.load_iris() - train_x = iris.data - train_y = iris.target - feature_columns = [feature_column.real_valued_column('', dimension=4)] - classifier = dnn.DNNClassifier( - n_classes=3, - feature_columns=feature_columns, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(x=train_x, y=train_y, steps=200) - scores = classifier.evaluate(x=train_x, y=train_y, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - - def testMultiClassLabelKeys(self): - """Tests n_classes > 2 with label_keys vocabulary for labels.""" - # Byte literals needed for python3 test to pass. - label_keys = [b'label0', b'label1', b'label2'] - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[.8], [0.2], [.1]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - labels = constant_op.constant( - [[label_keys[1]], [label_keys[0]], [label_keys[0]]], - dtype=dtypes.string) - return features, labels - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column( - language_column, dimension=1), - feature_column.real_valued_column('age') - ] - - classifier = dnn.DNNClassifier( - n_classes=3, - feature_columns=feature_columns, - hidden_units=[10, 10], - label_keys=label_keys, - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn, steps=50) - - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - self.assertIn('loss', scores) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predicted_classes = list( - classifier.predict_classes( - input_fn=predict_input_fn, as_iterable=True)) - self.assertEqual(3, len(predicted_classes)) - for pred in predicted_classes: - self.assertIn(pred, label_keys) - predictions = list( - classifier.predict(input_fn=predict_input_fn, as_iterable=True)) - self.assertAllEqual(predicted_classes, predictions) - - def testLoss(self): - """Tests loss calculation.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # The logistic prediction should be (y = 0.25). - labels = constant_op.constant([[1], [0], [0], [0]]) - features = {'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32),} - return features, labels - - classifier = dnn.DNNClassifier( - n_classes=2, - feature_columns=[feature_column.real_valued_column('x')], - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn_train, steps=5) - scores = classifier.evaluate(input_fn=_input_fn_train, steps=1) - self.assertIn('loss', scores) - - def testLossWithWeights(self): - """Tests loss calculation with weights.""" - - def _input_fn_train(): - # 4 rows with equal weight, one of them (y = x), three of them (y=Not(x)) - # The logistic prediction should be (y = 0.25). - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - def _input_fn_eval(): - # 4 rows, with different weights. - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[7.], [1.], [1.], [1.]]) - } - return features, labels - - classifier = dnn.DNNClassifier( - weight_column_name='w', - n_classes=2, - feature_columns=[feature_column.real_valued_column('x')], - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn_train, steps=5) - scores = classifier.evaluate(input_fn=_input_fn_eval, steps=1) - self.assertIn('loss', scores) - - def testTrainWithWeights(self): - """Tests training with given weight column.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # First row has more weight than others. Model should fit (y=x) better - # than (y=Not(x)) due to the relative higher weight of the first row. - labels = constant_op.constant([[1], [0], [0], [0]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[100.], [3.], [2.], [2.]]) - } - return features, labels - - def _input_fn_eval(): - # Create 4 rows (y = x) - labels = constant_op.constant([[1], [1], [1], [1]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - classifier = dnn.DNNClassifier( - weight_column_name='w', - feature_columns=[feature_column.real_valued_column('x')], - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn_train, steps=5) - scores = classifier.evaluate(input_fn=_input_fn_eval, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - - def testPredict_AsIterableFalse(self): - """Tests predict and predict_prob methods with as_iterable=False.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[.8], [.2], [.1]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([[1], [0], [0]], dtype=dtypes.int32) - - sparse_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column( - sparse_column, dimension=1) - ] - - n_classes = 3 - classifier = dnn.DNNClassifier( - n_classes=n_classes, - feature_columns=feature_columns, - hidden_units=[10, 10], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn, steps=100) - - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - self.assertIn('loss', scores) - predicted_classes = classifier.predict_classes( - input_fn=_input_fn, as_iterable=False) - self._assertClassificationPredictions(3, n_classes, predicted_classes) - predictions = classifier.predict(input_fn=_input_fn, as_iterable=False) - self.assertAllEqual(predicted_classes, predictions) - probabilities = classifier.predict_proba( - input_fn=_input_fn, as_iterable=False) - self._assertProbabilities(3, n_classes, probabilities) - - def testPredict_AsIterable(self): - """Tests predict and predict_prob methods with as_iterable=True.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[.8], [.2], [.1]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([[1], [0], [0]], dtype=dtypes.int32) - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column( - language_column, dimension=1), - feature_column.real_valued_column('age') - ] - - n_classes = 3 - classifier = dnn.DNNClassifier( - n_classes=n_classes, - feature_columns=feature_columns, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn, steps=300) - - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - self.assertIn('loss', scores) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predicted_classes = list( - classifier.predict_classes( - input_fn=predict_input_fn, as_iterable=True)) - self._assertClassificationPredictions(3, n_classes, predicted_classes) - predictions = list( - classifier.predict( - input_fn=predict_input_fn, as_iterable=True)) - self.assertAllEqual(predicted_classes, predictions) - predicted_proba = list( - classifier.predict_proba( - input_fn=predict_input_fn, as_iterable=True)) - self._assertProbabilities(3, n_classes, predicted_proba) - - def testCustomMetrics(self): - """Tests custom evaluation metrics.""" - - def _input_fn(num_epochs=None): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - labels = constant_op.constant([[1], [0], [0], [0]]) - features = { - 'x': - input_lib.limit_epochs( - array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - num_epochs=num_epochs), - } - return features, labels - - def _my_metric_op(predictions, labels): - # For the case of binary classification, the 2nd column of "predictions" - # denotes the model predictions. - labels = math_ops.cast(labels, dtypes.float32) - predictions = array_ops.strided_slice( - predictions, [0, 1], [-1, 2], end_mask=1) - labels = math_ops.cast(labels, predictions.dtype) - return math_ops.reduce_sum(math_ops.multiply(predictions, labels)) - - classifier = dnn.DNNClassifier( - feature_columns=[feature_column.real_valued_column('x')], - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn, steps=5) - scores = classifier.evaluate( - input_fn=_input_fn, - steps=5, - metrics={ - 'my_accuracy': - MetricSpec( - metric_fn=metric_ops.streaming_accuracy, - prediction_key='classes'), - 'my_precision': - MetricSpec( - metric_fn=metric_ops.streaming_precision, - prediction_key='classes'), - 'my_metric': - MetricSpec( - metric_fn=_my_metric_op, prediction_key='probabilities') - }) - self.assertTrue( - set(['loss', 'my_accuracy', 'my_precision', 'my_metric']).issubset( - set(scores.keys()))) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = np.array(list(classifier.predict_classes( - input_fn=predict_input_fn))) - self.assertEqual( - _sklearn.accuracy_score([1, 0, 0, 0], predictions), - scores['my_accuracy']) - - # Test the case where the 2nd element of the key is neither "classes" nor - # "probabilities". - with self.assertRaisesRegexp(KeyError, 'bad_type'): - classifier.evaluate( - input_fn=_input_fn, - steps=5, - metrics={ - 'bad_name': - MetricSpec( - metric_fn=metric_ops.streaming_auc, - prediction_key='bad_type') - }) - - def testTrainSaveLoad(self): - """Tests that insures you can save and reload a trained model.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[.8], [.2], [.1]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([[1], [0], [0]], dtype=dtypes.int32) - - sparse_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column( - sparse_column, dimension=1) - ] - - model_dir = tempfile.mkdtemp() - classifier = dnn.DNNClassifier( - model_dir=model_dir, - n_classes=3, - feature_columns=feature_columns, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - classifier.fit(input_fn=_input_fn, steps=5) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions1 = classifier.predict_classes(input_fn=predict_input_fn) - del classifier - - classifier2 = dnn.DNNClassifier( - model_dir=model_dir, - n_classes=3, - feature_columns=feature_columns, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - predictions2 = classifier2.predict_classes(input_fn=predict_input_fn) - self.assertEqual(list(predictions1), list(predictions2)) - - def testTrainWithPartitionedVariables(self): - """Tests training with partitioned variables.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[.8], [.2], [.1]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([[1], [0], [0]], dtype=dtypes.int32) - - # The given hash_bucket_size results in variables larger than the - # default min_slice_size attribute, so the variables are partitioned. - sparse_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=2e7) - feature_columns = [ - feature_column.embedding_column( - sparse_column, dimension=1) - ] - - tf_config = { - 'cluster': { - run_config.TaskType.PS: ['fake_ps_0', 'fake_ps_1'] - } - } - with test.mock.patch.dict('os.environ', - {'TF_CONFIG': json.dumps(tf_config)}): - config = run_config.RunConfig(tf_random_seed=1) - # Because we did not start a distributed cluster, we need to pass an - # empty ClusterSpec, otherwise the device_setter will look for - # distributed jobs, such as "/job:ps" which are not present. - config._cluster_spec = server_lib.ClusterSpec({}) - - classifier = dnn.DNNClassifier( - n_classes=3, - feature_columns=feature_columns, - hidden_units=[3, 3], - config=config) - - classifier.fit(input_fn=_input_fn, steps=5) - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - self.assertIn('loss', scores) - - def testExport(self): - """Tests export model for servo.""" - - def input_fn(): - return { - 'age': - constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column.sparse_column_with_hash_bucket('language', 100) - feature_columns = [ - feature_column.real_valued_column('age'), - feature_column.embedding_column( - language, dimension=1) - ] - - classifier = dnn.DNNClassifier( - feature_columns=feature_columns, hidden_units=[3, 3]) - classifier.fit(input_fn=input_fn, steps=5) - - export_dir = tempfile.mkdtemp() - classifier.export(export_dir) - - def testEnableCenteredBias(self): - """Tests that we can enable centered bias.""" - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - classifier = dnn.DNNClassifier( - n_classes=3, - feature_columns=cont_features, - hidden_units=[3, 3], - enable_centered_bias=True, - config=run_config.RunConfig(tf_random_seed=1)) - - input_fn = test_data.iris_input_multiclass_fn - classifier.fit(input_fn=input_fn, steps=5) - self.assertIn('dnn/multi_class_head/centered_bias_weight', - classifier.get_variable_names()) - scores = classifier.evaluate(input_fn=input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - self.assertIn('loss', scores) - - def testDisableCenteredBias(self): - """Tests that we can disable centered bias.""" - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - classifier = dnn.DNNClassifier( - n_classes=3, - feature_columns=cont_features, - hidden_units=[3, 3], - enable_centered_bias=False, - config=run_config.RunConfig(tf_random_seed=1)) - - input_fn = test_data.iris_input_multiclass_fn - classifier.fit(input_fn=input_fn, steps=5) - self.assertNotIn('centered_bias_weight', classifier.get_variable_names()) - scores = classifier.evaluate(input_fn=input_fn, steps=1) - self._assertInRange(0.0, 1.0, scores['accuracy']) - self.assertIn('loss', scores) - - -class DNNRegressorTest(test.TestCase): - - def testExperimentIntegration(self): - exp = experiment.Experiment( - estimator=dnn.DNNRegressor( - feature_columns=[ - feature_column.real_valued_column( - 'feature', dimension=4) - ], - hidden_units=[3, 3]), - train_input_fn=test_data.iris_input_logistic_fn, - eval_input_fn=test_data.iris_input_logistic_fn) - exp.test() - - def testEstimatorContract(self): - estimator_test_utils.assert_estimator_contract(self, dnn.DNNRegressor) - - def testRegression_MatrixData(self): - """Tests regression using matrix data as input.""" - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - regressor = dnn.DNNRegressor( - feature_columns=cont_features, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - input_fn = test_data.iris_input_logistic_fn - regressor.fit(input_fn=input_fn, steps=200) - scores = regressor.evaluate(input_fn=input_fn, steps=1) - self.assertIn('loss', scores) - - def testRegression_MatrixData_Labels1D(self): - """Same as the last test, but label shape is [100] instead of [100, 1].""" - - def _input_fn(): - iris = test_data.prepare_iris_data_for_logistic_regression() - return { - 'feature': constant_op.constant( - iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=[100], dtype=dtypes.int32) - - cont_features = [feature_column.real_valued_column('feature', dimension=4)] - - regressor = dnn.DNNRegressor( - feature_columns=cont_features, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=200) - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores) - - def testRegression_NpMatrixData(self): - """Tests binary classification using numpy matrix data as input.""" - iris = test_data.prepare_iris_data_for_logistic_regression() - train_x = iris.data - train_y = iris.target - feature_columns = [feature_column.real_valued_column('', dimension=4)] - regressor = dnn.DNNRegressor( - feature_columns=feature_columns, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(x=train_x, y=train_y, steps=200) - scores = regressor.evaluate(x=train_x, y=train_y, steps=1) - self.assertIn('loss', scores) - - def testRegression_TensorData(self): - """Tests regression using tensor data as input.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[.8], [.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([1., 0., 0.2], dtype=dtypes.float32) - - language_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column( - language_column, dimension=1), - feature_column.real_valued_column('age') - ] - - regressor = dnn.DNNRegressor( - feature_columns=feature_columns, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=200) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores) - - def testLoss(self): - """Tests loss calculation.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # The algorithm should learn (y = 0.25). - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = {'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32),} - return features, labels - - regressor = dnn.DNNRegressor( - feature_columns=[feature_column.real_valued_column('x')], - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn_train, steps=5) - scores = regressor.evaluate(input_fn=_input_fn_train, steps=1) - self.assertIn('loss', scores) - - def testLossWithWeights(self): - """Tests loss calculation with weights.""" - - def _input_fn_train(): - # 4 rows with equal weight, one of them (y = x), three of them (y=Not(x)) - # The algorithm should learn (y = 0.25). - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - def _input_fn_eval(): - # 4 rows, with different weights. - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[7.], [1.], [1.], [1.]]) - } - return features, labels - - regressor = dnn.DNNRegressor( - weight_column_name='w', - feature_columns=[feature_column.real_valued_column('x')], - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn_train, steps=5) - scores = regressor.evaluate(input_fn=_input_fn_eval, steps=1) - self.assertIn('loss', scores) - - def testTrainWithWeights(self): - """Tests training with given weight column.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # First row has more weight than others. Model should fit (y=x) better - # than (y=Not(x)) due to the relative higher weight of the first row. - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[100.], [3.], [2.], [2.]]) - } - return features, labels - - def _input_fn_eval(): - # Create 4 rows (y = x) - labels = constant_op.constant([[1.], [1.], [1.], [1.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - regressor = dnn.DNNRegressor( - weight_column_name='w', - feature_columns=[feature_column.real_valued_column('x')], - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn_train, steps=5) - scores = regressor.evaluate(input_fn=_input_fn_eval, steps=1) - self.assertIn('loss', scores) - - def _assertRegressionOutputs( - self, predictions, expected_shape): - predictions_nparray = np.array(predictions) - self.assertAllEqual(expected_shape, predictions_nparray.shape) - self.assertTrue(np.issubdtype(predictions_nparray.dtype, np.floating)) - - def testPredict_AsIterableFalse(self): - """Tests predict method with as_iterable=False.""" - labels = [1., 0., 0.2] - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant(labels, dtype=dtypes.float32) - - sparse_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column( - sparse_column, dimension=1), - feature_column.real_valued_column('age') - ] - - regressor = dnn.DNNRegressor( - feature_columns=feature_columns, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=200) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores) - predicted_scores = regressor.predict_scores( - input_fn=_input_fn, as_iterable=False) - self._assertRegressionOutputs(predicted_scores, [3]) - predictions = regressor.predict(input_fn=_input_fn, as_iterable=False) - self.assertAllClose(predicted_scores, predictions) - - def testPredict_AsIterable(self): - """Tests predict method with as_iterable=True.""" - labels = [1., 0., 0.2] - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant(labels, dtype=dtypes.float32) - - sparse_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column( - sparse_column, dimension=1), - feature_column.real_valued_column('age') - ] - - regressor = dnn.DNNRegressor( - feature_columns=feature_columns, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=200) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predicted_scores = list( - regressor.predict_scores( - input_fn=predict_input_fn, as_iterable=True)) - self._assertRegressionOutputs(predicted_scores, [3]) - predictions = list( - regressor.predict(input_fn=predict_input_fn, as_iterable=True)) - self.assertAllClose(predicted_scores, predictions) - - def testCustomMetrics(self): - """Tests custom evaluation metrics.""" - - def _input_fn(num_epochs=None): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': - input_lib.limit_epochs( - array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - num_epochs=num_epochs), - } - return features, labels - - def _my_metric_op(predictions, labels): - return math_ops.reduce_sum(math_ops.multiply(predictions, labels)) - - regressor = dnn.DNNRegressor( - feature_columns=[feature_column.real_valued_column('x')], - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=5) - scores = regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - 'my_error': metric_ops.streaming_mean_squared_error, - ('my_metric', 'scores'): _my_metric_op - }) - self.assertIn('loss', set(scores.keys())) - self.assertIn('my_error', set(scores.keys())) - self.assertIn('my_metric', set(scores.keys())) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = np.array(list(regressor.predict_scores( - input_fn=predict_input_fn))) - self.assertAlmostEqual( - _sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions), - scores['my_error']) - - # Tests the case that the 2nd element of the key is not "scores". - with self.assertRaises(KeyError): - regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - ('my_error', 'predictions'): - metric_ops.streaming_mean_squared_error - }) - - # Tests the case where the tuple of the key doesn't have 2 elements. - with self.assertRaises(ValueError): - regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - ('bad_length_name', 'scores', 'bad_length'): - metric_ops.streaming_mean_squared_error - }) - - def testCustomMetricsWithMetricSpec(self): - """Tests custom evaluation metrics that use MetricSpec.""" - - def _input_fn(num_epochs=None): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': - input_lib.limit_epochs( - array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - num_epochs=num_epochs), - } - return features, labels - - def _my_metric_op(predictions, labels): - return math_ops.reduce_sum(math_ops.multiply(predictions, labels)) - - regressor = dnn.DNNRegressor( - feature_columns=[feature_column.real_valued_column('x')], - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=5) - scores = regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - 'my_error': - MetricSpec( - metric_fn=metric_ops.streaming_mean_squared_error, - prediction_key='scores'), - 'my_metric': - MetricSpec( - metric_fn=_my_metric_op, prediction_key='scores') - }) - self.assertIn('loss', set(scores.keys())) - self.assertIn('my_error', set(scores.keys())) - self.assertIn('my_metric', set(scores.keys())) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = np.array(list(regressor.predict_scores( - input_fn=predict_input_fn))) - self.assertAlmostEqual( - _sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions), - scores['my_error']) - - # Tests the case where the prediction_key is not "scores". - with self.assertRaisesRegexp(KeyError, 'bad_type'): - regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - 'bad_name': - MetricSpec( - metric_fn=metric_ops.streaming_auc, - prediction_key='bad_type') - }) - - def testTrainSaveLoad(self): - """Tests that insures you can save and reload a trained model.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([1., 0., 0.2], dtype=dtypes.float32) - - sparse_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column( - sparse_column, dimension=1), - feature_column.real_valued_column('age') - ] - - model_dir = tempfile.mkdtemp() - regressor = dnn.DNNRegressor( - model_dir=model_dir, - feature_columns=feature_columns, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=5) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = list(regressor.predict_scores(input_fn=predict_input_fn)) - del regressor - - regressor2 = dnn.DNNRegressor( - model_dir=model_dir, - feature_columns=feature_columns, - hidden_units=[3, 3], - config=run_config.RunConfig(tf_random_seed=1)) - predictions2 = list(regressor2.predict_scores(input_fn=predict_input_fn)) - self.assertAllClose(predictions, predictions2) - - def testTrainWithPartitionedVariables(self): - """Tests training with partitioned variables.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([1., 0., 0.2], dtype=dtypes.float32) - - # The given hash_bucket_size results in variables larger than the - # default min_slice_size attribute, so the variables are partitioned. - sparse_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=2e7) - feature_columns = [ - feature_column.embedding_column( - sparse_column, dimension=1), - feature_column.real_valued_column('age') - ] - - tf_config = { - 'cluster': { - run_config.TaskType.PS: ['fake_ps_0', 'fake_ps_1'] - } - } - with test.mock.patch.dict('os.environ', - {'TF_CONFIG': json.dumps(tf_config)}): - config = run_config.RunConfig(tf_random_seed=1) - # Because we did not start a distributed cluster, we need to pass an - # empty ClusterSpec, otherwise the device_setter will look for - # distributed jobs, such as "/job:ps" which are not present. - config._cluster_spec = server_lib.ClusterSpec({}) - - regressor = dnn.DNNRegressor( - feature_columns=feature_columns, hidden_units=[3, 3], config=config) - - regressor.fit(input_fn=_input_fn, steps=5) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores) - - def testEnableCenteredBias(self): - """Tests that we can enable centered bias.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([1., 0., 0.2], dtype=dtypes.float32) - - sparse_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column( - sparse_column, dimension=1), - feature_column.real_valued_column('age') - ] - - regressor = dnn.DNNRegressor( - feature_columns=feature_columns, - hidden_units=[3, 3], - enable_centered_bias=True, - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=5) - self.assertIn('dnn/regression_head/centered_bias_weight', - regressor.get_variable_names()) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores) - - def testDisableCenteredBias(self): - """Tests that we can disable centered bias.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([1., 0., 0.2], dtype=dtypes.float32) - - sparse_column = feature_column.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [ - feature_column.embedding_column( - sparse_column, dimension=1), - feature_column.real_valued_column('age') - ] - - regressor = dnn.DNNRegressor( - feature_columns=feature_columns, - hidden_units=[3, 3], - enable_centered_bias=False, - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=5) - self.assertNotIn('centered_bias_weight', regressor.get_variable_names()) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertIn('loss', scores) - - -def boston_input_fn(): - boston = base.load_boston() - features = math_ops.cast( - array_ops.reshape(constant_op.constant(boston.data), [-1, 13]), - dtypes.float32) - labels = math_ops.cast( - array_ops.reshape(constant_op.constant(boston.target), [-1, 1]), - dtypes.float32) - return features, labels - - -class FeatureColumnTest(test.TestCase): - - def testTrain(self): - feature_columns = estimator.infer_real_valued_columns_from_input_fn( - boston_input_fn) - est = dnn.DNNRegressor(feature_columns=feature_columns, hidden_units=[3, 3]) - est.fit(input_fn=boston_input_fn, steps=1) - _ = est.evaluate(input_fn=boston_input_fn, steps=1) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py deleted file mode 100644 index 35352b32ec8..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py +++ /dev/null @@ -1,702 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Estimator for Dynamic RNNs (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.layers.python.layers import optimizers -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.contrib.learn.python.learn.estimators import rnn_common -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import rnn -from tensorflow.python.training import momentum as momentum_opt -from tensorflow.python.util import nest - - -# TODO(jtbates): Remove PredictionType when all non-experimental targets which -# depend on it point to rnn_common.PredictionType. -class PredictionType(object): - SINGLE_VALUE = 1 - MULTIPLE_VALUE = 2 - - -def _get_state_name(i): - """Constructs the name string for state component `i`.""" - return '{}_{}'.format(rnn_common.RNNKeys.STATE_PREFIX, i) - - -def state_tuple_to_dict(state): - """Returns a dict containing flattened `state`. - - Args: - state: A `Tensor` or a nested tuple of `Tensors`. All of the `Tensor`s must - have the same rank and agree on all dimensions except the last. - - Returns: - A dict containing the `Tensor`s that make up `state`. The keys of the dict - are of the form "STATE_PREFIX_i" where `i` is the place of this `Tensor` - in a depth-first traversal of `state`. - """ - with ops.name_scope('state_tuple_to_dict'): - flat_state = nest.flatten(state) - state_dict = {} - for i, state_component in enumerate(flat_state): - state_name = _get_state_name(i) - state_value = (None if state_component is None - else array_ops.identity(state_component, name=state_name)) - state_dict[state_name] = state_value - return state_dict - - -def dict_to_state_tuple(input_dict, cell): - """Reconstructs nested `state` from a dict containing state `Tensor`s. - - Args: - input_dict: A dict of `Tensor`s. - cell: An instance of `RNNCell`. - Returns: - If `input_dict` does not contain keys 'STATE_PREFIX_i' for `0 <= i < n` - where `n` is the number of nested entries in `cell.state_size`, this - function returns `None`. Otherwise, returns a `Tensor` if `cell.state_size` - is an `int` or a nested tuple of `Tensor`s if `cell.state_size` is a nested - tuple. - Raises: - ValueError: State is partially specified. The `input_dict` must contain - values for all state components or none at all. - """ - flat_state_sizes = nest.flatten(cell.state_size) - state_tensors = [] - with ops.name_scope('dict_to_state_tuple'): - for i, state_size in enumerate(flat_state_sizes): - state_name = _get_state_name(i) - state_tensor = input_dict.get(state_name) - if state_tensor is not None: - rank_check = check_ops.assert_rank( - state_tensor, 2, name='check_state_{}_rank'.format(i)) - shape_check = check_ops.assert_equal( - array_ops.shape(state_tensor)[1], - state_size, - name='check_state_{}_shape'.format(i)) - with ops.control_dependencies([rank_check, shape_check]): - state_tensor = array_ops.identity(state_tensor, name=state_name) - state_tensors.append(state_tensor) - if not state_tensors: - return None - elif len(state_tensors) == len(flat_state_sizes): - dummy_state = cell.zero_state(batch_size=1, dtype=dtypes.bool) - return nest.pack_sequence_as(dummy_state, state_tensors) - else: - raise ValueError( - 'RNN state was partially specified.' - 'Expected zero or {} state Tensors; got {}'. - format(len(flat_state_sizes), len(state_tensors))) - - -def _concatenate_context_input(sequence_input, context_input): - """Replicates `context_input` across all timesteps of `sequence_input`. - - Expands dimension 1 of `context_input` then tiles it `sequence_length` times. - This value is appended to `sequence_input` on dimension 2 and the result is - returned. - - Args: - sequence_input: A `Tensor` of dtype `float32` and shape `[batch_size, - padded_length, d0]`. - context_input: A `Tensor` of dtype `float32` and shape `[batch_size, d1]`. - - Returns: - A `Tensor` of dtype `float32` and shape `[batch_size, padded_length, - d0 + d1]`. - - Raises: - ValueError: If `sequence_input` does not have rank 3 or `context_input` does - not have rank 2. - """ - seq_rank_check = check_ops.assert_rank( - sequence_input, - 3, - message='sequence_input must have rank 3', - data=[array_ops.shape(sequence_input)]) - seq_type_check = check_ops.assert_type( - sequence_input, - dtypes.float32, - message='sequence_input must have dtype float32; got {}.'.format( - sequence_input.dtype)) - ctx_rank_check = check_ops.assert_rank( - context_input, - 2, - message='context_input must have rank 2', - data=[array_ops.shape(context_input)]) - ctx_type_check = check_ops.assert_type( - context_input, - dtypes.float32, - message='context_input must have dtype float32; got {}.'.format( - context_input.dtype)) - with ops.control_dependencies( - [seq_rank_check, seq_type_check, ctx_rank_check, ctx_type_check]): - padded_length = array_ops.shape(sequence_input)[1] - tiled_context_input = array_ops.tile( - array_ops.expand_dims(context_input, 1), - array_ops.concat([[1], [padded_length], [1]], 0)) - return array_ops.concat([sequence_input, tiled_context_input], 2) - - -def build_sequence_input(features, - sequence_feature_columns, - context_feature_columns, - weight_collections=None, - scope=None): - """Combine sequence and context features into input for an RNN. - - Args: - features: A `dict` containing the input and (optionally) sequence length - information and initial state. - sequence_feature_columns: An iterable containing all the feature columns - describing sequence features. All items in the set should be instances - of classes derived from `FeatureColumn`. - context_feature_columns: An iterable containing all the feature columns - describing context features i.e. features that apply across all time - steps. All items in the set should be instances of classes derived from - `FeatureColumn`. - weight_collections: List of graph collections to which weights are added. - scope: Optional scope, passed through to parsing ops. - Returns: - A `Tensor` of dtype `float32` and shape `[batch_size, padded_length, ?]`. - This will be used as input to an RNN. - """ - features = features.copy() - features.update(layers.transform_features( - features, - list(sequence_feature_columns) + list(context_feature_columns or []))) - sequence_input = layers.sequence_input_from_feature_columns( - columns_to_tensors=features, - feature_columns=sequence_feature_columns, - weight_collections=weight_collections, - scope=scope) - if context_feature_columns is not None: - context_input = layers.input_from_feature_columns( - columns_to_tensors=features, - feature_columns=context_feature_columns, - weight_collections=weight_collections, - scope=scope) - sequence_input = _concatenate_context_input(sequence_input, context_input) - return sequence_input - - -def construct_rnn(initial_state, - sequence_input, - cell, - num_label_columns, - dtype=dtypes.float32, - parallel_iterations=32, - swap_memory=True): - """Build an RNN and apply a fully connected layer to get the desired output. - - Args: - initial_state: The initial state to pass the RNN. If `None`, the - default starting state for `self._cell` is used. - sequence_input: A `Tensor` with shape `[batch_size, padded_length, d]` - that will be passed as input to the RNN. - cell: An initialized `RNNCell`. - num_label_columns: The desired output dimension. - dtype: dtype of `cell`. - parallel_iterations: Number of iterations to run in parallel. Values >> 1 - use more memory but take less time, while smaller values use less memory - but computations take longer. - swap_memory: Transparently swap the tensors produced in forward inference - but needed for back prop from GPU to CPU. This allows training RNNs - which would typically not fit on a single GPU, with very minimal (or no) - performance penalty. - Returns: - activations: The output of the RNN, projected to `num_label_columns` - dimensions. - final_state: A `Tensor` or nested tuple of `Tensor`s representing the final - state output by the RNN. - """ - with ops.name_scope('RNN'): - rnn_outputs, final_state = rnn.dynamic_rnn( - cell=cell, - inputs=sequence_input, - initial_state=initial_state, - dtype=dtype, - parallel_iterations=parallel_iterations, - swap_memory=swap_memory, - time_major=False) - activations = layers.fully_connected( - inputs=rnn_outputs, - num_outputs=num_label_columns, - activation_fn=None, - trainable=True) - return activations, final_state - - -def _single_value_predictions(activations, - sequence_length, - target_column, - problem_type, - predict_probabilities): - """Maps `activations` from the RNN to predictions for single value models. - - If `predict_probabilities` is `False`, this function returns a `dict` - containing single entry with key `PREDICTIONS_KEY`. If `predict_probabilities` - is `True`, it will contain a second entry with key `PROBABILITIES_KEY`. The - value of this entry is a `Tensor` of probabilities with shape - `[batch_size, num_classes]`. - - Args: - activations: Output from an RNN. Should have dtype `float32` and shape - `[batch_size, padded_length, ?]`. - sequence_length: A `Tensor` with shape `[batch_size]` and dtype `int32` - containing the length of each sequence in the batch. If `None`, sequences - are assumed to be unpadded. - target_column: An initialized `TargetColumn`, calculate predictions. - problem_type: Either `ProblemType.CLASSIFICATION` or - `ProblemType.LINEAR_REGRESSION`. - predict_probabilities: A Python boolean, indicating whether probabilities - should be returned. Should only be set to `True` for - classification/logistic regression problems. - Returns: - A `dict` mapping strings to `Tensors`. - """ - with ops.name_scope('SingleValuePrediction'): - last_activations = rnn_common.select_last_activations( - activations, sequence_length) - predictions_name = (prediction_key.PredictionKey.CLASSES - if problem_type == constants.ProblemType.CLASSIFICATION - else prediction_key.PredictionKey.SCORES) - if predict_probabilities: - probabilities = target_column.logits_to_predictions( - last_activations, proba=True) - prediction_dict = { - prediction_key.PredictionKey.PROBABILITIES: probabilities, - predictions_name: math_ops.argmax(probabilities, 1)} - else: - predictions = target_column.logits_to_predictions( - last_activations, proba=False) - prediction_dict = {predictions_name: predictions} - return prediction_dict - - -def _multi_value_loss( - activations, labels, sequence_length, target_column, features): - """Maps `activations` from the RNN to loss for multi value models. - - Args: - activations: Output from an RNN. Should have dtype `float32` and shape - `[batch_size, padded_length, ?]`. - labels: A `Tensor` with length `[batch_size, padded_length]`. - sequence_length: A `Tensor` with shape `[batch_size]` and dtype `int32` - containing the length of each sequence in the batch. If `None`, sequences - are assumed to be unpadded. - target_column: An initialized `TargetColumn`, calculate predictions. - features: A `dict` containing the input and (optionally) sequence length - information and initial state. - Returns: - A scalar `Tensor` containing the loss. - """ - with ops.name_scope('MultiValueLoss'): - activations_masked, labels_masked = rnn_common.mask_activations_and_labels( - activations, labels, sequence_length) - return target_column.loss(activations_masked, labels_masked, features) - - -def _single_value_loss( - activations, labels, sequence_length, target_column, features): - """Maps `activations` from the RNN to loss for multi value models. - - Args: - activations: Output from an RNN. Should have dtype `float32` and shape - `[batch_size, padded_length, ?]`. - labels: A `Tensor` with length `[batch_size]`. - sequence_length: A `Tensor` with shape `[batch_size]` and dtype `int32` - containing the length of each sequence in the batch. If `None`, sequences - are assumed to be unpadded. - target_column: An initialized `TargetColumn`, calculate predictions. - features: A `dict` containing the input and (optionally) sequence length - information and initial state. - Returns: - A scalar `Tensor` containing the loss. - """ - - with ops.name_scope('SingleValueLoss'): - last_activations = rnn_common.select_last_activations( - activations, sequence_length) - return target_column.loss(last_activations, labels, features) - - -def _get_output_alternatives(prediction_type, - problem_type, - prediction_dict): - """Constructs output alternatives dict for `ModelFnOps`. - - Args: - prediction_type: either `MULTIPLE_VALUE` or `SINGLE_VALUE`. - problem_type: either `CLASSIFICATION` or `LINEAR_REGRESSION`. - prediction_dict: a dictionary mapping strings to `Tensor`s containing - predictions. - - Returns: - `None` or a dictionary mapping a string to an output alternative. - - Raises: - ValueError: `prediction_type` is not one of `SINGLE_VALUE` or - `MULTIPLE_VALUE`. - """ - if prediction_type == rnn_common.PredictionType.MULTIPLE_VALUE: - return None - if prediction_type == rnn_common.PredictionType.SINGLE_VALUE: - prediction_dict_no_state = { - k: v - for k, v in prediction_dict.items() - if rnn_common.RNNKeys.STATE_PREFIX not in k - } - return {'dynamic_rnn_output': (problem_type, prediction_dict_no_state)} - raise ValueError('Unrecognized prediction_type: {}'.format(prediction_type)) - - -def _get_dynamic_rnn_model_fn( - cell_type, - num_units, - target_column, - problem_type, - prediction_type, - optimizer, - sequence_feature_columns, - context_feature_columns=None, - predict_probabilities=False, - learning_rate=None, - gradient_clipping_norm=None, - dropout_keep_probabilities=None, - sequence_length_key=rnn_common.RNNKeys.SEQUENCE_LENGTH_KEY, - dtype=dtypes.float32, - parallel_iterations=None, - swap_memory=True, - name='DynamicRNNModel'): - """Creates an RNN model function for an `Estimator`. - - The model function returns an instance of `ModelFnOps`. When - `problem_type == ProblemType.CLASSIFICATION` and - `predict_probabilities == True`, the returned `ModelFnOps` includes an output - alternative containing the classes and their associated probabilities. When - `predict_probabilities == False`, only the classes are included. When - `problem_type == ProblemType.LINEAR_REGRESSION`, the output alternative - contains only the predicted values. - - Args: - cell_type: A string, a subclass of `RNNCell` or an instance of an `RNNCell`. - num_units: A single `int` or a list of `int`s. The size of the `RNNCell`s. - target_column: An initialized `TargetColumn`, used to calculate prediction - and loss. - problem_type: `ProblemType.CLASSIFICATION` or - `ProblemType.LINEAR_REGRESSION`. - prediction_type: `PredictionType.SINGLE_VALUE` or - `PredictionType.MULTIPLE_VALUE`. - optimizer: A subclass of `Optimizer`, an instance of an `Optimizer` or a - string. - sequence_feature_columns: An iterable containing all the feature columns - describing sequence features. All items in the set should be instances - of classes derived from `FeatureColumn`. - context_feature_columns: An iterable containing all the feature columns - describing context features, i.e., features that apply across all time - steps. All items in the set should be instances of classes derived from - `FeatureColumn`. - predict_probabilities: A boolean indicating whether to predict probabilities - for all classes. Must only be used with - `ProblemType.CLASSIFICATION`. - learning_rate: Learning rate used for optimization. This argument has no - effect if `optimizer` is an instance of an `Optimizer`. - gradient_clipping_norm: A float. Gradients will be clipped to this value. - dropout_keep_probabilities: a list of dropout keep probabilities or `None`. - If a list is given, it must have length `len(num_units) + 1`. - sequence_length_key: The key that will be used to look up sequence length in - the `features` dict. - dtype: The dtype of the state and output of the given `cell`. - parallel_iterations: Number of iterations to run in parallel. Values >> 1 - use more memory but take less time, while smaller values use less memory - but computations take longer. - swap_memory: Transparently swap the tensors produced in forward inference - but needed for back prop from GPU to CPU. This allows training RNNs - which would typically not fit on a single GPU, with very minimal (or no) - performance penalty. - name: A string that will be used to create a scope for the RNN. - - Returns: - A model function to be passed to an `Estimator`. - - Raises: - ValueError: `problem_type` is not one of - `ProblemType.LINEAR_REGRESSION` or `ProblemType.CLASSIFICATION`. - ValueError: `prediction_type` is not one of `PredictionType.SINGLE_VALUE` - or `PredictionType.MULTIPLE_VALUE`. - ValueError: `predict_probabilities` is `True` for `problem_type` other - than `ProblemType.CLASSIFICATION`. - ValueError: `len(dropout_keep_probabilities)` is not `len(num_units) + 1`. - """ - if problem_type not in (constants.ProblemType.CLASSIFICATION, - constants.ProblemType.LINEAR_REGRESSION): - raise ValueError( - 'problem_type must be ProblemType.LINEAR_REGRESSION or ' - 'ProblemType.CLASSIFICATION; got {}'. - format(problem_type)) - if prediction_type not in (rnn_common.PredictionType.SINGLE_VALUE, - rnn_common.PredictionType.MULTIPLE_VALUE): - raise ValueError( - 'prediction_type must be PredictionType.MULTIPLE_VALUEs or ' - 'PredictionType.SINGLE_VALUE; got {}'. - format(prediction_type)) - if (problem_type != constants.ProblemType.CLASSIFICATION - and predict_probabilities): - raise ValueError( - 'predict_probabilities can only be set to True for problem_type' - ' ProblemType.CLASSIFICATION; got {}.'.format(problem_type)) - def _dynamic_rnn_model_fn(features, labels, mode): - """The model to be passed to an `Estimator`.""" - with ops.name_scope(name): - sequence_length = features.get(sequence_length_key) - sequence_input = build_sequence_input(features, - sequence_feature_columns, - context_feature_columns) - dropout = (dropout_keep_probabilities - if mode == model_fn.ModeKeys.TRAIN - else None) - # This class promises to use the cell type selected by that function. - cell = rnn_common.construct_rnn_cell(num_units, cell_type, dropout) - initial_state = dict_to_state_tuple(features, cell) - rnn_activations, final_state = construct_rnn( - initial_state, - sequence_input, - cell, - target_column.num_label_columns, - dtype=dtype, - parallel_iterations=parallel_iterations, - swap_memory=swap_memory) - - loss = None # Created below for modes TRAIN and EVAL. - if prediction_type == rnn_common.PredictionType.MULTIPLE_VALUE: - prediction_dict = rnn_common.multi_value_predictions( - rnn_activations, target_column, problem_type, predict_probabilities) - if mode != model_fn.ModeKeys.INFER: - loss = _multi_value_loss( - rnn_activations, labels, sequence_length, target_column, features) - elif prediction_type == rnn_common.PredictionType.SINGLE_VALUE: - prediction_dict = _single_value_predictions( - rnn_activations, sequence_length, target_column, - problem_type, predict_probabilities) - if mode != model_fn.ModeKeys.INFER: - loss = _single_value_loss( - rnn_activations, labels, sequence_length, target_column, features) - state_dict = state_tuple_to_dict(final_state) - prediction_dict.update(state_dict) - - eval_metric_ops = None - if mode != model_fn.ModeKeys.INFER: - eval_metric_ops = rnn_common.get_eval_metric_ops( - problem_type, prediction_type, sequence_length, prediction_dict, - labels) - - train_op = None - if mode == model_fn.ModeKeys.TRAIN: - train_op = optimizers.optimize_loss( - loss=loss, - global_step=None, # Get it internally. - learning_rate=learning_rate, - optimizer=optimizer, - clip_gradients=gradient_clipping_norm, - summaries=optimizers.OPTIMIZER_SUMMARIES) - - output_alternatives = _get_output_alternatives(prediction_type, - problem_type, - prediction_dict) - - return model_fn.ModelFnOps(mode=mode, - predictions=prediction_dict, - loss=loss, - train_op=train_op, - eval_metric_ops=eval_metric_ops, - output_alternatives=output_alternatives) - return _dynamic_rnn_model_fn - - -class DynamicRnnEstimator(estimator.Estimator): - """Dynamically unrolled RNN (deprecated). - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - def __init__(self, - problem_type, - prediction_type, - sequence_feature_columns, - context_feature_columns=None, - num_classes=None, - num_units=None, - cell_type='basic_rnn', - optimizer='SGD', - learning_rate=0.1, - predict_probabilities=False, - momentum=None, - gradient_clipping_norm=5.0, - dropout_keep_probabilities=None, - model_dir=None, - feature_engineering_fn=None, - config=None): - """Initializes a `DynamicRnnEstimator`. - - The input function passed to this `Estimator` optionally contains keys - `RNNKeys.SEQUENCE_LENGTH_KEY`. The value corresponding to - `RNNKeys.SEQUENCE_LENGTH_KEY` must be vector of size `batch_size` where - entry `n` corresponds to the length of the `n`th sequence in the batch. The - sequence length feature is required for batches of varying sizes. It will be - used to calculate loss and evaluation metrics. If - `RNNKeys.SEQUENCE_LENGTH_KEY` is not included, all sequences are assumed to - have length equal to the size of dimension 1 of the input to the RNN. - - In order to specify an initial state, the input function must include keys - `STATE_PREFIX_i` for all `0 <= i < n` where `n` is the number of nested - elements in `cell.state_size`. The input function must contain values for - all state components or none of them. If none are included, then the default - (zero) state is used as an initial state. See the documentation for - `dict_to_state_tuple` and `state_tuple_to_dict` for further details. - The input function can call rnn_common.construct_rnn_cell() to obtain the - same cell type that this class will select from arguments to __init__. - - The `predict()` method of the `Estimator` returns a dictionary with keys - `STATE_PREFIX_i` for `0 <= i < n` where `n` is the number of nested elements - in `cell.state_size`, along with `PredictionKey.CLASSES` for problem type - `CLASSIFICATION` or `PredictionKey.SCORES` for problem type - `LINEAR_REGRESSION`. The value keyed by - `PredictionKey.CLASSES` or `PredictionKey.SCORES` has shape - `[batch_size, padded_length]` in the multi-value case and shape - `[batch_size]` in the single-value case. Here, `padded_length` is the - largest value in the `RNNKeys.SEQUENCE_LENGTH` `Tensor` passed as input. - Entry `[i, j]` is the prediction associated with sequence `i` and time step - `j`. If the problem type is `CLASSIFICATION` and `predict_probabilities` is - `True`, it will also include key`PredictionKey.PROBABILITIES`. - - Args: - problem_type: whether the `Estimator` is intended for a regression or - classification problem. Value must be one of - `ProblemType.CLASSIFICATION` or `ProblemType.LINEAR_REGRESSION`. - prediction_type: whether the `Estimator` should return a value for each - step in the sequence, or just a single value for the final time step. - Must be one of `PredictionType.SINGLE_VALUE` or - `PredictionType.MULTIPLE_VALUE`. - sequence_feature_columns: An iterable containing all the feature columns - describing sequence features. All items in the iterable should be - instances of classes derived from `FeatureColumn`. - context_feature_columns: An iterable containing all the feature columns - describing context features, i.e., features that apply across all time - steps. All items in the set should be instances of classes derived from - `FeatureColumn`. - num_classes: the number of classes for a classification problem. Only - used when `problem_type=ProblemType.CLASSIFICATION`. - num_units: A list of integers indicating the number of units in the - `RNNCell`s in each layer. - cell_type: A subclass of `RNNCell` or one of 'basic_rnn,' 'lstm' or 'gru'. - optimizer: The type of optimizer to use. Either a subclass of - `Optimizer`, an instance of an `Optimizer`, a callback that returns an - optimizer, or a string. Strings must be one of 'Adagrad', 'Adam', - 'Ftrl', 'Momentum', 'RMSProp' or 'SGD'. See `layers.optimize_loss` for - more details. - learning_rate: Learning rate. This argument has no effect if `optimizer` - is an instance of an `Optimizer`. - predict_probabilities: A boolean indicating whether to predict - probabilities for all classes. Used only if `problem_type` is - `ProblemType.CLASSIFICATION` - momentum: Momentum value. Only used if `optimizer` is 'Momentum'. - gradient_clipping_norm: Parameter used for gradient clipping. If `None`, - then no clipping is performed. - dropout_keep_probabilities: a list of dropout probabilities or `None`. - If a list is given, it must have length `len(num_units) + 1`. If - `None`, then no dropout is applied. - model_dir: The directory in which to save and restore the model graph, - parameters, etc. - feature_engineering_fn: Takes features and labels which are the output of - `input_fn` and returns features and labels which will be fed into - `model_fn`. Please check `model_fn` for a definition of features and - labels. - config: A `RunConfig` instance. - - Raises: - ValueError: `problem_type` is not one of - `ProblemType.LINEAR_REGRESSION` or `ProblemType.CLASSIFICATION`. - ValueError: `problem_type` is `ProblemType.CLASSIFICATION` but - `num_classes` is not specified. - ValueError: `prediction_type` is not one of - `PredictionType.MULTIPLE_VALUE` or `PredictionType.SINGLE_VALUE`. - """ - if prediction_type == rnn_common.PredictionType.MULTIPLE_VALUE: - name = 'MultiValueDynamicRNN' - elif prediction_type == rnn_common.PredictionType.SINGLE_VALUE: - name = 'SingleValueDynamicRNN' - else: - raise ValueError( - 'prediction_type must be one of PredictionType.MULTIPLE_VALUE or ' - 'PredictionType.SINGLE_VALUE; got {}'.format(prediction_type)) - - if problem_type == constants.ProblemType.LINEAR_REGRESSION: - name += 'Regressor' - target_column = layers.regression_target() - elif problem_type == constants.ProblemType.CLASSIFICATION: - if not num_classes: - raise ValueError('For CLASSIFICATION problem_type, num_classes must be ' - 'specified.') - target_column = layers.multi_class_target(n_classes=num_classes) - name += 'Classifier' - else: - raise ValueError( - 'problem_type must be either ProblemType.LINEAR_REGRESSION ' - 'or ProblemType.CLASSIFICATION; got {}'.format( - problem_type)) - - if optimizer == 'Momentum': - optimizer = momentum_opt.MomentumOptimizer(learning_rate, momentum) - dynamic_rnn_model_fn = _get_dynamic_rnn_model_fn( - cell_type=cell_type, - num_units=num_units, - target_column=target_column, - problem_type=problem_type, - prediction_type=prediction_type, - optimizer=optimizer, - sequence_feature_columns=sequence_feature_columns, - context_feature_columns=context_feature_columns, - predict_probabilities=predict_probabilities, - learning_rate=learning_rate, - gradient_clipping_norm=gradient_clipping_norm, - dropout_keep_probabilities=dropout_keep_probabilities, - name=name) - - super(DynamicRnnEstimator, self).__init__( - model_fn=dynamic_rnn_model_fn, - model_dir=model_dir, - config=config, - feature_engineering_fn=feature_engineering_fn) diff --git a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py deleted file mode 100644 index 7a96f6d3ea4..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py +++ /dev/null @@ -1,785 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for learn.estimators.dynamic_rnn_estimator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tempfile - -import numpy as np - -from tensorflow.contrib import rnn -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.layers.python.layers import target_column as target_column_lib -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import dynamic_rnn_estimator -from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.contrib.learn.python.learn.estimators import rnn_common -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import map_fn -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class IdentityRNNCell(rnn.RNNCell): - - def __init__(self, state_size, output_size): - self._state_size = state_size - self._output_size = output_size - - @property - def state_size(self): - return self._state_size - - @property - def output_size(self): - return self._output_size - - def __call__(self, inputs, state): - return array_ops.identity(inputs), array_ops.ones( - [array_ops.shape(inputs)[0], self.state_size]) - - -class MockTargetColumn(object): - - def __init__(self, num_label_columns=None): - self._num_label_columns = num_label_columns - - def get_eval_ops(self, features, activations, labels, metrics): - raise NotImplementedError( - 'MockTargetColumn.get_eval_ops called unexpectedly.') - - def logits_to_predictions(self, flattened_activations, proba=False): - raise NotImplementedError( - 'MockTargetColumn.logits_to_predictions called unexpectedly.') - - def loss(self, activations, labels, features): - raise NotImplementedError('MockTargetColumn.loss called unexpectedly.') - - @property - def num_label_columns(self): - if self._num_label_columns is None: - raise ValueError('MockTargetColumn.num_label_columns has not been set.') - return self._num_label_columns - - def set_num_label_columns(self, n): - self._num_label_columns = n - - -def sequence_length_mask(values, lengths): - masked = values - for i, length in enumerate(lengths): - masked[i, length:, :] = np.zeros_like(masked[i, length:, :]) - return masked - - -class DynamicRnnEstimatorTest(test.TestCase): - - NUM_RNN_CELL_UNITS = 8 - NUM_LABEL_COLUMNS = 6 - INPUTS_COLUMN = feature_column.real_valued_column( - 'inputs', dimension=NUM_LABEL_COLUMNS) - - def setUp(self): - super(DynamicRnnEstimatorTest, self).setUp() - self.rnn_cell = rnn_cell.BasicRNNCell(self.NUM_RNN_CELL_UNITS) - self.mock_target_column = MockTargetColumn( - num_label_columns=self.NUM_LABEL_COLUMNS) - - location = feature_column.sparse_column_with_keys( - 'location', keys=['west_side', 'east_side', 'nyc']) - location_onehot = feature_column.one_hot_column(location) - self.context_feature_columns = [location_onehot] - - wire_cast = feature_column.sparse_column_with_keys( - 'wire_cast', ['marlo', 'omar', 'stringer']) - wire_cast_embedded = feature_column.embedding_column(wire_cast, dimension=8) - measurements = feature_column.real_valued_column( - 'measurements', dimension=2) - self.sequence_feature_columns = [measurements, wire_cast_embedded] - - def GetColumnsToTensors(self): - """Get columns_to_tensors matching setUp(), in the current default graph.""" - return { - 'location': - sparse_tensor.SparseTensor( - indices=[[0, 0], [1, 0], [2, 0]], - values=['west_side', 'west_side', 'nyc'], - dense_shape=[3, 1]), - 'wire_cast': - sparse_tensor.SparseTensor( - indices=[[0, 0, 0], [0, 1, 0], - [1, 0, 0], [1, 1, 0], [1, 1, 1], - [2, 0, 0]], - values=[b'marlo', b'stringer', - b'omar', b'stringer', b'marlo', - b'marlo'], - dense_shape=[3, 2, 2]), - 'measurements': - random_ops.random_uniform( - [3, 2, 2], seed=4711) - } - - def GetClassificationTargetsOrNone(self, mode): - """Get targets matching setUp() and mode, in the current default graph.""" - return (random_ops.random_uniform( - [3, 2, 1], 0, 2, dtype=dtypes.int64, seed=1412) if - mode != model_fn_lib.ModeKeys.INFER else None) - - def testBuildSequenceInputInput(self): - sequence_input = dynamic_rnn_estimator.build_sequence_input( - self.GetColumnsToTensors(), self.sequence_feature_columns, - self.context_feature_columns) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - sess.run(lookup_ops.tables_initializer()) - sequence_input_val = sess.run(sequence_input) - expected_shape = np.array([ - 3, # expected batch size - 2, # padded sequence length - 3 + 8 + 2 # location keys + embedding dim + measurement dimension - ]) - self.assertAllEqual(expected_shape, sequence_input_val.shape) - - def testConstructRNN(self): - initial_state = None - sequence_input = dynamic_rnn_estimator.build_sequence_input( - self.GetColumnsToTensors(), self.sequence_feature_columns, - self.context_feature_columns) - activations_t, final_state_t = dynamic_rnn_estimator.construct_rnn( - initial_state, sequence_input, self.rnn_cell, - self.mock_target_column.num_label_columns) - - # Obtain values of activations and final state. - with session.Session() as sess: - sess.run(variables.global_variables_initializer()) - sess.run(lookup_ops.tables_initializer()) - activations, final_state = sess.run([activations_t, final_state_t]) - - expected_activations_shape = np.array([3, 2, self.NUM_LABEL_COLUMNS]) - self.assertAllEqual(expected_activations_shape, activations.shape) - expected_state_shape = np.array([3, self.NUM_RNN_CELL_UNITS]) - self.assertAllEqual(expected_state_shape, final_state.shape) - - def testGetOutputAlternatives(self): - test_cases = ( - (rnn_common.PredictionType.SINGLE_VALUE, - constants.ProblemType.CLASSIFICATION, - {prediction_key.PredictionKey.CLASSES: True, - prediction_key.PredictionKey.PROBABILITIES: True, - dynamic_rnn_estimator._get_state_name(0): True}, - {'dynamic_rnn_output': - (constants.ProblemType.CLASSIFICATION, - {prediction_key.PredictionKey.CLASSES: True, - prediction_key.PredictionKey.PROBABILITIES: True})}), - - (rnn_common.PredictionType.SINGLE_VALUE, - constants.ProblemType.LINEAR_REGRESSION, - {prediction_key.PredictionKey.SCORES: True, - dynamic_rnn_estimator._get_state_name(0): True, - dynamic_rnn_estimator._get_state_name(1): True}, - {'dynamic_rnn_output': - (constants.ProblemType.LINEAR_REGRESSION, - {prediction_key.PredictionKey.SCORES: True})}), - - (rnn_common.PredictionType.MULTIPLE_VALUE, - constants.ProblemType.CLASSIFICATION, - {prediction_key.PredictionKey.CLASSES: True, - prediction_key.PredictionKey.PROBABILITIES: True, - dynamic_rnn_estimator._get_state_name(0): True}, - None)) - - for pred_type, prob_type, pred_dict, expected_alternatives in test_cases: - actual_alternatives = dynamic_rnn_estimator._get_output_alternatives( - pred_type, prob_type, pred_dict) - self.assertEqual(expected_alternatives, actual_alternatives) - - # testGetDynamicRnnModelFn{Train,Eval,Infer}() test which fields - # of ModelFnOps are set depending on mode. - def testGetDynamicRnnModelFnTrain(self): - model_fn_ops = self._GetModelFnOpsForMode(model_fn_lib.ModeKeys.TRAIN) - self.assertIsNotNone(model_fn_ops.predictions) - self.assertIsNotNone(model_fn_ops.loss) - self.assertIsNotNone(model_fn_ops.train_op) - # None may get normalized to {}; we accept neither. - self.assertNotEqual(len(model_fn_ops.eval_metric_ops), 0) - - def testGetDynamicRnnModelFnEval(self): - model_fn_ops = self._GetModelFnOpsForMode(model_fn_lib.ModeKeys.EVAL) - self.assertIsNotNone(model_fn_ops.predictions) - self.assertIsNotNone(model_fn_ops.loss) - self.assertIsNone(model_fn_ops.train_op) - # None may get normalized to {}; we accept neither. - self.assertNotEqual(len(model_fn_ops.eval_metric_ops), 0) - - def testGetDynamicRnnModelFnInfer(self): - model_fn_ops = self._GetModelFnOpsForMode(model_fn_lib.ModeKeys.INFER) - self.assertIsNotNone(model_fn_ops.predictions) - self.assertIsNone(model_fn_ops.loss) - self.assertIsNone(model_fn_ops.train_op) - # None may get normalized to {}; we accept both. - self.assertFalse(model_fn_ops.eval_metric_ops) - - def _GetModelFnOpsForMode(self, mode): - """Helper for testGetDynamicRnnModelFn{Train,Eval,Infer}().""" - model_fn = dynamic_rnn_estimator._get_dynamic_rnn_model_fn( - cell_type='basic_rnn', - num_units=[10], - target_column=target_column_lib.multi_class_target(n_classes=2), - # Only CLASSIFICATION yields eval metrics to test for. - problem_type=constants.ProblemType.CLASSIFICATION, - prediction_type=rnn_common.PredictionType.MULTIPLE_VALUE, - optimizer='SGD', - sequence_feature_columns=self.sequence_feature_columns, - context_feature_columns=self.context_feature_columns, - learning_rate=0.1) - labels = self.GetClassificationTargetsOrNone(mode) - model_fn_ops = model_fn( - features=self.GetColumnsToTensors(), labels=labels, mode=mode) - return model_fn_ops - - def testExport(self): - input_feature_key = 'magic_input_feature_key' - - def get_input_fn(mode): - - def input_fn(): - features = self.GetColumnsToTensors() - if mode == model_fn_lib.ModeKeys.INFER: - input_examples = array_ops.placeholder(dtypes.string) - features[input_feature_key] = input_examples - # Real code would now parse features out of input_examples, - # but this test can just stick to the constants above. - return features, self.GetClassificationTargetsOrNone(mode) - - return input_fn - - model_dir = tempfile.mkdtemp() - - def estimator_fn(): - return dynamic_rnn_estimator.DynamicRnnEstimator( - problem_type=constants.ProblemType.CLASSIFICATION, - prediction_type=rnn_common.PredictionType.MULTIPLE_VALUE, - num_classes=2, - num_units=self.NUM_RNN_CELL_UNITS, - sequence_feature_columns=self.sequence_feature_columns, - context_feature_columns=self.context_feature_columns, - predict_probabilities=True, - model_dir=model_dir) - - # Train a bit to create an exportable checkpoint. - estimator_fn().fit(input_fn=get_input_fn(model_fn_lib.ModeKeys.TRAIN), - steps=100) - # Now export, but from a fresh estimator instance, like you would - # in an export binary. That means .export() has to work without - # .fit() being called on the same object. - export_dir = tempfile.mkdtemp() - print('Exporting to', export_dir) - estimator_fn().export( - export_dir, - input_fn=get_input_fn(model_fn_lib.ModeKeys.INFER), - use_deprecated_input_fn=False, - input_feature_key=input_feature_key) - - def testStateTupleDictConversion(self): - """Test `state_tuple_to_dict` and `dict_to_state_tuple`.""" - cell_sizes = [5, 3, 7] - # A MultiRNNCell of LSTMCells is both a common choice and an interesting - # test case, because it has two levels of nesting, with an inner class that - # is not a plain tuple. - cell = rnn_cell.MultiRNNCell( - [rnn_cell.LSTMCell(i) for i in cell_sizes]) - state_dict = { - dynamic_rnn_estimator._get_state_name(i): - array_ops.expand_dims(math_ops.range(cell_size), 0) - for i, cell_size in enumerate([5, 5, 3, 3, 7, 7]) - } - expected_state = (rnn_cell.LSTMStateTuple( - np.reshape(np.arange(5), [1, -1]), np.reshape(np.arange(5), [1, -1])), - rnn_cell.LSTMStateTuple( - np.reshape(np.arange(3), [1, -1]), - np.reshape(np.arange(3), [1, -1])), - rnn_cell.LSTMStateTuple( - np.reshape(np.arange(7), [1, -1]), - np.reshape(np.arange(7), [1, -1]))) - actual_state = dynamic_rnn_estimator.dict_to_state_tuple(state_dict, cell) - flattened_state = dynamic_rnn_estimator.state_tuple_to_dict(actual_state) - - with self.cached_session() as sess: - (state_dict_val, actual_state_val, flattened_state_val) = sess.run( - [state_dict, actual_state, flattened_state]) - - def _recursive_assert_equal(x, y): - self.assertEqual(type(x), type(y)) - if isinstance(x, (list, tuple)): - self.assertEqual(len(x), len(y)) - for i, _ in enumerate(x): - _recursive_assert_equal(x[i], y[i]) - elif isinstance(x, np.ndarray): - np.testing.assert_array_equal(x, y) - else: - self.fail('Unexpected type: {}'.format(type(x))) - - for k in state_dict_val.keys(): - np.testing.assert_array_almost_equal( - state_dict_val[k], - flattened_state_val[k], - err_msg='Wrong value for state component {}.'.format(k)) - _recursive_assert_equal(expected_state, actual_state_val) - - def testMultiRNNState(self): - """Test that state flattening/reconstruction works for `MultiRNNCell`.""" - batch_size = 11 - sequence_length = 16 - train_steps = 5 - cell_sizes = [4, 8, 7] - learning_rate = 0.1 - - def get_shift_input_fn(batch_size, sequence_length, seed=None): - - def input_fn(): - random_sequence = random_ops.random_uniform( - [batch_size, sequence_length + 1], - 0, - 2, - dtype=dtypes.int32, - seed=seed) - labels = array_ops.slice(random_sequence, [0, 0], - [batch_size, sequence_length]) - inputs = array_ops.expand_dims( - math_ops.cast( - array_ops.slice(random_sequence, [0, 1], - [batch_size, sequence_length]), - dtypes.float32), 2) - input_dict = { - dynamic_rnn_estimator._get_state_name(i): random_ops.random_uniform( - [batch_size, cell_size], seed=((i + 1) * seed)) - for i, cell_size in enumerate([4, 4, 8, 8, 7, 7]) - } - input_dict['inputs'] = inputs - return input_dict, labels - - return input_fn - - seq_columns = [feature_column.real_valued_column('inputs', dimension=1)] - config = run_config.RunConfig(tf_random_seed=21212) - cell_type = 'lstm' - sequence_estimator = dynamic_rnn_estimator.DynamicRnnEstimator( - problem_type=constants.ProblemType.CLASSIFICATION, - prediction_type=rnn_common.PredictionType.MULTIPLE_VALUE, - num_classes=2, - num_units=cell_sizes, - sequence_feature_columns=seq_columns, - cell_type=cell_type, - learning_rate=learning_rate, - config=config, - predict_probabilities=True) - - train_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=12321) - eval_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=32123) - - sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) - - prediction_dict = sequence_estimator.predict( - input_fn=eval_input_fn, as_iterable=False) - for i, state_size in enumerate([4, 4, 8, 8, 7, 7]): - state_piece = prediction_dict[dynamic_rnn_estimator._get_state_name(i)] - self.assertListEqual(list(state_piece.shape), [batch_size, state_size]) - - def testMultipleRuns(self): - """Tests resuming training by feeding state.""" - cell_sizes = [4, 7] - batch_size = 11 - learning_rate = 0.1 - train_sequence_length = 21 - train_steps = 121 - dropout_keep_probabilities = [0.5, 0.5, 0.5] - prediction_steps = [3, 2, 5, 11, 6] - - def get_input_fn(batch_size, sequence_length, state_dict, starting_step=0): - - def input_fn(): - sequence = constant_op.constant( - [[(starting_step + i + j) % 2 for j in range(sequence_length + 1)] - for i in range(batch_size)], - dtype=dtypes.int32) - labels = array_ops.slice(sequence, [0, 0], - [batch_size, sequence_length]) - inputs = array_ops.expand_dims( - math_ops.cast( - array_ops.slice(sequence, [0, 1], [batch_size, sequence_length - ]), - dtypes.float32), 2) - input_dict = state_dict - input_dict['inputs'] = inputs - return input_dict, labels - - return input_fn - - seq_columns = [feature_column.real_valued_column('inputs', dimension=1)] - config = run_config.RunConfig(tf_random_seed=21212) - - model_dir = tempfile.mkdtemp() - sequence_estimator = dynamic_rnn_estimator.DynamicRnnEstimator( - problem_type=constants.ProblemType.CLASSIFICATION, - prediction_type=rnn_common.PredictionType.MULTIPLE_VALUE, - num_classes=2, - sequence_feature_columns=seq_columns, - num_units=cell_sizes, - cell_type='lstm', - dropout_keep_probabilities=dropout_keep_probabilities, - learning_rate=learning_rate, - config=config, - model_dir=model_dir) - - train_input_fn = get_input_fn( - batch_size, train_sequence_length, state_dict={}) - - sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) - - def incremental_predict(estimator, increments): - """Run `estimator.predict` for `i` steps for `i` in `increments`.""" - step = 0 - incremental_state_dict = {} - for increment in increments: - input_fn = get_input_fn( - batch_size, - increment, - state_dict=incremental_state_dict, - starting_step=step) - prediction_dict = estimator.predict( - input_fn=input_fn, as_iterable=False) - step += increment - incremental_state_dict = { - k: v - for (k, v) in prediction_dict.items() - if k.startswith(rnn_common.RNNKeys.STATE_PREFIX) - } - return prediction_dict - - pred_all_at_once = incremental_predict(sequence_estimator, - [sum(prediction_steps)]) - pred_step_by_step = incremental_predict(sequence_estimator, - prediction_steps) - - # Check that the last `prediction_steps[-1]` steps give the same - # predictions. - np.testing.assert_array_equal( - pred_all_at_once[prediction_key.PredictionKey.CLASSES] - [:, -1 * prediction_steps[-1]:], - pred_step_by_step[prediction_key.PredictionKey.CLASSES], - err_msg='Mismatch on last {} predictions.'.format(prediction_steps[-1])) - # Check that final states are identical. - for k, v in pred_all_at_once.items(): - if k.startswith(rnn_common.RNNKeys.STATE_PREFIX): - np.testing.assert_array_equal( - v, pred_step_by_step[k], err_msg='Mismatch on state {}.'.format(k)) - - -# TODO(jamieas): move all tests below to a benchmark test. -class DynamicRNNEstimatorLearningTest(test.TestCase): - """Learning tests for dynamic RNN Estimators.""" - - def testLearnSineFunction(self): - """Tests learning a sine function.""" - batch_size = 8 - sequence_length = 64 - train_steps = 200 - eval_steps = 20 - cell_size = [4] - learning_rate = 0.1 - loss_threshold = 0.02 - - def get_sin_input_fn(batch_size, sequence_length, increment, seed=None): - - def _sin_fn(x): - ranger = math_ops.linspace( - array_ops.reshape(x[0], []), (sequence_length - 1) * increment, - sequence_length + 1) - return math_ops.sin(ranger) - - def input_fn(): - starts = random_ops.random_uniform( - [batch_size], maxval=(2 * np.pi), seed=seed) - sin_curves = map_fn.map_fn( - _sin_fn, (starts,), dtype=dtypes.float32) - inputs = array_ops.expand_dims( - array_ops.slice(sin_curves, [0, 0], [batch_size, sequence_length]), - 2) - labels = array_ops.slice(sin_curves, [0, 1], - [batch_size, sequence_length]) - return {'inputs': inputs}, labels - - return input_fn - - seq_columns = [ - feature_column.real_valued_column( - 'inputs', dimension=cell_size[0]) - ] - config = run_config.RunConfig(tf_random_seed=1234) - sequence_estimator = dynamic_rnn_estimator.DynamicRnnEstimator( - problem_type=constants.ProblemType.LINEAR_REGRESSION, - prediction_type=rnn_common.PredictionType.MULTIPLE_VALUE, - num_units=cell_size, - sequence_feature_columns=seq_columns, - learning_rate=learning_rate, - dropout_keep_probabilities=[0.9, 0.9], - config=config) - - train_input_fn = get_sin_input_fn( - batch_size, sequence_length, np.pi / 32, seed=1234) - eval_input_fn = get_sin_input_fn( - batch_size, sequence_length, np.pi / 32, seed=4321) - - sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) - loss = sequence_estimator.evaluate( - input_fn=eval_input_fn, steps=eval_steps)['loss'] - self.assertLess(loss, loss_threshold, - 'Loss should be less than {}; got {}'.format(loss_threshold, - loss)) - - def testLearnShiftByOne(self): - """Tests that learning a 'shift-by-one' example. - - Each label sequence consists of the input sequence 'shifted' by one place. - The RNN must learn to 'remember' the previous input. - """ - batch_size = 16 - sequence_length = 32 - train_steps = 200 - eval_steps = 20 - cell_size = 4 - learning_rate = 0.3 - accuracy_threshold = 0.9 - - def get_shift_input_fn(batch_size, sequence_length, seed=None): - - def input_fn(): - random_sequence = random_ops.random_uniform( - [batch_size, sequence_length + 1], - 0, - 2, - dtype=dtypes.int32, - seed=seed) - labels = array_ops.slice(random_sequence, [0, 0], - [batch_size, sequence_length]) - inputs = array_ops.expand_dims( - math_ops.cast( - array_ops.slice(random_sequence, [0, 1], - [batch_size, sequence_length]), - dtypes.float32), - 2) - return {'inputs': inputs}, labels - - return input_fn - - seq_columns = [ - feature_column.real_valued_column( - 'inputs', dimension=cell_size) - ] - config = run_config.RunConfig(tf_random_seed=21212) - sequence_estimator = dynamic_rnn_estimator.DynamicRnnEstimator( - problem_type=constants.ProblemType.CLASSIFICATION, - prediction_type=rnn_common.PredictionType.MULTIPLE_VALUE, - num_classes=2, - num_units=cell_size, - sequence_feature_columns=seq_columns, - learning_rate=learning_rate, - config=config, - predict_probabilities=True) - - train_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=12321) - eval_input_fn = get_shift_input_fn(batch_size, sequence_length, seed=32123) - - sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) - - evaluation = sequence_estimator.evaluate( - input_fn=eval_input_fn, steps=eval_steps) - accuracy = evaluation['accuracy'] - self.assertGreater(accuracy, accuracy_threshold, - 'Accuracy should be higher than {}; got {}'.format( - accuracy_threshold, accuracy)) - - # Testing `predict` when `predict_probabilities=True`. - prediction_dict = sequence_estimator.predict( - input_fn=eval_input_fn, as_iterable=False) - self.assertListEqual( - sorted(list(prediction_dict.keys())), - sorted([ - prediction_key.PredictionKey.CLASSES, - prediction_key.PredictionKey.PROBABILITIES, - dynamic_rnn_estimator._get_state_name(0) - ])) - predictions = prediction_dict[prediction_key.PredictionKey.CLASSES] - probabilities = prediction_dict[ - prediction_key.PredictionKey.PROBABILITIES] - self.assertListEqual(list(predictions.shape), [batch_size, sequence_length]) - self.assertListEqual( - list(probabilities.shape), [batch_size, sequence_length, 2]) - - def testLearnMean(self): - """Test learning to calculate a mean.""" - batch_size = 16 - sequence_length = 3 - train_steps = 200 - eval_steps = 20 - cell_type = 'basic_rnn' - cell_size = 8 - optimizer_type = 'Momentum' - learning_rate = 0.1 - momentum = 0.9 - loss_threshold = 0.1 - - def get_mean_input_fn(batch_size, sequence_length, seed=None): - - def input_fn(): - # Create examples by choosing 'centers' and adding uniform noise. - centers = math_ops.matmul( - random_ops.random_uniform( - [batch_size, 1], -0.75, 0.75, dtype=dtypes.float32, seed=seed), - array_ops.ones([1, sequence_length])) - noise = random_ops.random_uniform( - [batch_size, sequence_length], - -0.25, - 0.25, - dtype=dtypes.float32, - seed=seed) - sequences = centers + noise - - inputs = array_ops.expand_dims(sequences, 2) - labels = math_ops.reduce_mean(sequences, axis=[1]) - return {'inputs': inputs}, labels - - return input_fn - - seq_columns = [ - feature_column.real_valued_column( - 'inputs', dimension=cell_size) - ] - config = run_config.RunConfig(tf_random_seed=6) - sequence_estimator = dynamic_rnn_estimator.DynamicRnnEstimator( - problem_type=constants.ProblemType.LINEAR_REGRESSION, - prediction_type=rnn_common.PredictionType.SINGLE_VALUE, - num_units=cell_size, - sequence_feature_columns=seq_columns, - cell_type=cell_type, - optimizer=optimizer_type, - learning_rate=learning_rate, - momentum=momentum, - config=config) - - train_input_fn = get_mean_input_fn(batch_size, sequence_length, 121) - eval_input_fn = get_mean_input_fn(batch_size, sequence_length, 212) - - sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) - evaluation = sequence_estimator.evaluate( - input_fn=eval_input_fn, steps=eval_steps) - loss = evaluation['loss'] - self.assertLess(loss, loss_threshold, - 'Loss should be less than {}; got {}'.format(loss_threshold, - loss)) - - def DISABLED_testLearnMajority(self): - """Test learning the 'majority' function.""" - batch_size = 16 - sequence_length = 7 - train_steps = 500 - eval_steps = 20 - cell_type = 'lstm' - cell_size = 4 - optimizer_type = 'Momentum' - learning_rate = 2.0 - momentum = 0.9 - accuracy_threshold = 0.6 - - def get_majority_input_fn(batch_size, sequence_length, seed=None): - random_seed.set_random_seed(seed) - - def input_fn(): - random_sequence = random_ops.random_uniform( - [batch_size, sequence_length], 0, 2, dtype=dtypes.int32, seed=seed) - inputs = array_ops.expand_dims( - math_ops.cast(random_sequence, dtypes.float32), 2) - labels = math_ops.cast( - array_ops.squeeze( - math_ops.reduce_sum(inputs, axis=[1]) > ( - sequence_length / 2.0)), - dtypes.int32) - return {'inputs': inputs}, labels - - return input_fn - - seq_columns = [ - feature_column.real_valued_column( - 'inputs', dimension=cell_size) - ] - config = run_config.RunConfig(tf_random_seed=77) - sequence_estimator = dynamic_rnn_estimator.DynamicRnnEstimator( - problem_type=constants.ProblemType.CLASSIFICATION, - prediction_type=rnn_common.PredictionType.SINGLE_VALUE, - num_classes=2, - num_units=cell_size, - sequence_feature_columns=seq_columns, - cell_type=cell_type, - optimizer=optimizer_type, - learning_rate=learning_rate, - momentum=momentum, - config=config, - predict_probabilities=True) - - train_input_fn = get_majority_input_fn(batch_size, sequence_length, 1111) - eval_input_fn = get_majority_input_fn(batch_size, sequence_length, 2222) - - sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) - evaluation = sequence_estimator.evaluate( - input_fn=eval_input_fn, steps=eval_steps) - accuracy = evaluation['accuracy'] - self.assertGreater(accuracy, accuracy_threshold, - 'Accuracy should be higher than {}; got {}'.format( - accuracy_threshold, accuracy)) - - # Testing `predict` when `predict_probabilities=True`. - prediction_dict = sequence_estimator.predict( - input_fn=eval_input_fn, as_iterable=False) - self.assertListEqual( - sorted(list(prediction_dict.keys())), - sorted([ - prediction_key.PredictionKey.CLASSES, - prediction_key.PredictionKey.PROBABILITIES, - dynamic_rnn_estimator._get_state_name(0), - dynamic_rnn_estimator._get_state_name(1) - ])) - predictions = prediction_dict[prediction_key.PredictionKey.CLASSES] - probabilities = prediction_dict[ - prediction_key.PredictionKey.PROBABILITIES] - self.assertListEqual(list(predictions.shape), [batch_size]) - self.assertListEqual(list(probabilities.shape), [batch_size, 2]) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator.py b/tensorflow/contrib/learn/python/learn/estimators/estimator.py deleted file mode 100644 index c762227b20b..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/estimator.py +++ /dev/null @@ -1,1582 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Base Estimator class (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc -import collections -import copy -import os -import tempfile - -import numpy as np -import six - -from google.protobuf import message -from tensorflow.contrib import layers -from tensorflow.contrib.framework import deprecated -from tensorflow.contrib.framework import deprecated_args -from tensorflow.contrib.framework import list_variables -from tensorflow.contrib.framework import load_variable -from tensorflow.contrib.learn.python.learn import evaluable -from tensorflow.contrib.learn.python.learn import metric_spec -from tensorflow.contrib.learn.python.learn import monitors as monitor_lib -from tensorflow.contrib.learn.python.learn import trainable -from tensorflow.contrib.learn.python.learn.estimators import _sklearn as sklearn -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import metric_key -from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.contrib.learn.python.learn.estimators import tensor_signature -from tensorflow.contrib.learn.python.learn.estimators._sklearn import NotFittedError -from tensorflow.contrib.learn.python.learn.learn_io import data_feeder -from tensorflow.contrib.learn.python.learn.utils import export -from tensorflow.contrib.learn.python.learn.utils import saved_model_export_utils -from tensorflow.contrib.meta_graph_transform import meta_graph_transform -from tensorflow.contrib.training.python.training import evaluation -from tensorflow.core.framework import summary_pb2 -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session as tf_session -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import metrics as metrics_lib -from tensorflow.python.ops import resources -from tensorflow.python.ops import variables -from tensorflow.python.platform import gfile -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.saved_model import builder as saved_model_builder -from tensorflow.python.saved_model import tag_constants -from tensorflow.python.summary import summary as core_summary -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.training import checkpoint_management -from tensorflow.python.training import device_setter -from tensorflow.python.training import monitored_session -from tensorflow.python.training import saver -from tensorflow.python.training import training_util -from tensorflow.python.util import compat -from tensorflow.python.util import tf_decorator -from tensorflow.python.util import tf_inspect - -AS_ITERABLE_DATE = '2016-09-15' -AS_ITERABLE_INSTRUCTIONS = ( - 'The default behavior of predict() is changing. The default value for\n' - 'as_iterable will change to True, and then the flag will be removed\n' - 'altogether. The behavior of this flag is described below.') -SCIKIT_DECOUPLE_DATE = '2016-12-01' -SCIKIT_DECOUPLE_INSTRUCTIONS = ( - 'Estimator is decoupled from Scikit Learn interface by moving into\n' - 'separate class SKCompat. Arguments x, y and batch_size are only\n' - 'available in the SKCompat class, Estimator will only accept input_fn.\n' - 'Example conversion:\n' - ' est = Estimator(...) -> est = SKCompat(Estimator(...))') - - -def _verify_input_args(x, y, input_fn, feed_fn, batch_size): - """Verifies validity of co-existence of input arguments.""" - if input_fn is None: - if x is None: - raise ValueError('Either x or input_fn must be provided.') - - if tensor_util.is_tensor(x) or y is not None and tensor_util.is_tensor(y): - raise ValueError('Inputs cannot be tensors. Please provide input_fn.') - - if feed_fn is not None: - raise ValueError('Can not provide both feed_fn and x or y.') - else: - if (x is not None) or (y is not None): - raise ValueError('Can not provide both input_fn and x or y.') - if batch_size is not None: - raise ValueError('Can not provide both input_fn and batch_size.') - - -def _get_input_fn(x, y, input_fn, feed_fn, batch_size, shuffle=False, epochs=1): - """Make inputs into input and feed functions. - - Args: - x: Numpy, Pandas or Dask matrix or iterable. - y: Numpy, Pandas or Dask matrix or iterable. - input_fn: Pre-defined input function for training data. - feed_fn: Pre-defined data feeder function. - batch_size: Size to split data into parts. Must be >= 1. - shuffle: Whether to shuffle the inputs. - epochs: Number of epochs to run. - - Returns: - Data input and feeder function based on training data. - - Raises: - ValueError: Only one of `(x & y)` or `input_fn` must be provided. - """ - _verify_input_args(x, y, input_fn, feed_fn, batch_size) - if input_fn is not None: - return input_fn, feed_fn - df = data_feeder.setup_train_data_feeder( - x, - y, - n_classes=None, - batch_size=batch_size, - shuffle=shuffle, - epochs=epochs) - return df.input_builder, df.get_feed_dict_fn() - - -@deprecated(None, 'Please specify feature columns explicitly.') -def infer_real_valued_columns_from_input_fn(input_fn): - """Creates `FeatureColumn` objects for inputs defined by `input_fn`. - - This interprets all inputs as dense, fixed-length float values. This creates - a local graph in which it calls `input_fn` to build the tensors, then discards - it. - - Args: - input_fn: Input function returning a tuple of: - features - Dictionary of string feature name to `Tensor` or `Tensor`. - labels - `Tensor` of label values. - - Returns: - List of `FeatureColumn` objects. - """ - with ops.Graph().as_default(): - features, _ = input_fn() - return layers.infer_real_valued_columns(features) - - -@deprecated(None, 'Please specify feature columns explicitly.') -def infer_real_valued_columns_from_input(x): - """Creates `FeatureColumn` objects for inputs defined by input `x`. - - This interprets all inputs as dense, fixed-length float values. - - Args: - x: Real-valued matrix of shape [n_samples, n_features...]. Can be - iterator that returns arrays of features. - - Returns: - List of `FeatureColumn` objects. - """ - input_fn, _ = _get_input_fn( - x=x, y=None, input_fn=None, feed_fn=None, batch_size=None) - return infer_real_valued_columns_from_input_fn(input_fn) - - -def _model_fn_args(fn): - """Get argument names for function-like object. - - Args: - fn: Function, or function-like object (e.g., result of `functools.partial`). - - Returns: - `tuple` of string argument names. - - Raises: - ValueError: if partial function has positionally bound arguments - """ - _, fn = tf_decorator.unwrap(fn) - if hasattr(fn, 'func') and hasattr(fn, 'keywords') and hasattr(fn, 'args'): - # Handle functools.partial and similar objects. - return tuple([ - arg for arg in tf_inspect.getargspec(fn.func).args[len(fn.args):] - if arg not in set(fn.keywords.keys()) - ]) - # Handle function. - return tuple(tf_inspect.getargspec(fn).args) - - -def _get_replica_device_setter(config): - """Creates a replica device setter if required. - - Args: - config: A RunConfig instance. - - Returns: - A replica device setter, or None. - """ - ps_ops = [ - 'Variable', 'VariableV2', 'AutoReloadVariable', 'MutableHashTable', - 'MutableHashTableV2', 'MutableHashTableOfTensors', - 'MutableHashTableOfTensorsV2', 'MutableDenseHashTable', - 'MutableDenseHashTableV2', 'VarHandleOp' - ] - - if config.task_type: - worker_device = '/job:%s/task:%d' % (config.task_type, config.task_id) - else: - worker_device = '/job:worker' - - if config.num_ps_replicas > 0: - return device_setter.replica_device_setter( - ps_tasks=config.num_ps_replicas, - worker_device=worker_device, - merge_devices=True, - ps_ops=ps_ops, - cluster=config.cluster_spec) - else: - return None - - -def _make_metrics_ops(metrics, features, labels, predictions): - """Add metrics based on `features`, `labels`, and `predictions`. - - `metrics` contains a specification for how to run metrics. It is a dict - mapping friendly names to either `MetricSpec` objects, or directly to a metric - function (assuming that `predictions` and `labels` are single tensors), or to - `(pred_name, metric)` `tuple`, which passes `predictions[pred_name]` and - `labels` to `metric` (assuming `labels` is a single tensor). - - Users are encouraged to use `MetricSpec` objects, which are more flexible and - cleaner. They also lead to clearer errors. - - Args: - metrics: A dict mapping names to metrics specification, for example - `MetricSpec` objects. - features: A dict of tensors returned from an input_fn as features/inputs. - labels: A single tensor or a dict of tensors returned from an input_fn as - labels. - predictions: A single tensor or a dict of tensors output from a model as - predictions. - - Returns: - A dict mapping the friendly given in `metrics` to the result of calling the - given metric function. - - Raises: - ValueError: If metrics specifications do not work with the type of - `features`, `labels`, or `predictions` provided. Mostly, a dict is given - but no pred_name specified. - """ - metrics = metrics or {} - - # If labels is a dict with a single key, unpack into a single tensor. - labels_tensor_or_dict = labels - if isinstance(labels, dict) and len(labels) == 1: - labels_tensor_or_dict = labels[list(labels.keys())[0]] - - result = {} - # Iterate in lexicographic order, so the graph is identical among runs. - for name, metric in sorted(six.iteritems(metrics)): - if isinstance(metric, metric_spec.MetricSpec): - result[name] = metric.create_metric_ops(features, labels, predictions) - continue - - # TODO(b/31229024): Remove the rest of this loop - logging.warning('Please specify metrics using MetricSpec. Using bare ' - 'functions or (key, fn) tuples is deprecated and support ' - 'for it will be removed on Oct 1, 2016.') - - if isinstance(name, tuple): - # Multi-head metrics. - if len(name) != 2: - raise ValueError('Invalid metric for {}. It returned a tuple with ' - 'len {}, expected 2.'.format(name, len(name))) - if not isinstance(predictions, dict): - raise ValueError('Metrics passed provide (name, prediction), ' - 'but predictions are not dict. ' - 'Metrics: %s, Predictions: %s.' % (metrics, - predictions)) - # Here are two options: labels are single Tensor or a dict. - if isinstance(labels, dict) and name[1] in labels: - # If labels are dict and the prediction name is in it, apply metric. - result[name[0]] = metric(predictions[name[1]], labels[name[1]]) - else: - # Otherwise pass the labels to the metric. - result[name[0]] = metric(predictions[name[1]], labels_tensor_or_dict) - else: - # Single head metrics. - if isinstance(predictions, dict): - raise ValueError('Metrics passed provide only name, no prediction, ' - 'but predictions are dict. ' - 'Metrics: %s, Labels: %s.' % (metrics, - labels_tensor_or_dict)) - result[name] = metric(predictions, labels_tensor_or_dict) - return result - - -def _dict_to_str(dictionary): - """Get a `str` representation of a `dict`. - - Args: - dictionary: The `dict` to be represented as `str`. - - Returns: - A `str` representing the `dictionary`. - """ - results = [] - for k, v in sorted(dictionary.items()): - if isinstance(v, float) or isinstance(v, np.float32) or isinstance( - v, int) or isinstance(v, np.int64) or isinstance(v, np.int32): - results.append('%s = %s' % (k, v)) - else: - results.append('Type of %s = %s' % (k, type(v))) - - return ', '.join(results) - - -def _write_dict_to_summary(output_dir, dictionary, current_global_step): - """Writes a `dict` into summary file in given output directory. - - Args: - output_dir: `str`, directory to write the summary file in. - dictionary: the `dict` to be written to summary file. - current_global_step: `int`, the current global step. - """ - logging.info('Saving dict for global step %d: %s', current_global_step, - _dict_to_str(dictionary)) - summary_writer = core_summary.FileWriterCache.get(output_dir) - summary_proto = summary_pb2.Summary() - for key in dictionary: - if dictionary[key] is None: - continue - if key == 'global_step': - continue - if (isinstance(dictionary[key], np.float32) or - isinstance(dictionary[key], float)): - summary_proto.value.add(tag=key, simple_value=float(dictionary[key])) - elif (isinstance(dictionary[key], np.int64) or - isinstance(dictionary[key], np.int32) or - isinstance(dictionary[key], int)): - summary_proto.value.add(tag=key, simple_value=int(dictionary[key])) - elif isinstance(dictionary[key], six.string_types): - try: - summ = summary_pb2.Summary.FromString(dictionary[key]) - for i, _ in enumerate(summ.value): - summ.value[i].tag = key - summary_proto.value.extend(summ.value) - except message.DecodeError: - logging.warn('Skipping summary for %s, cannot parse string to Summary.', - key) - continue - elif isinstance(dictionary[key], np.ndarray): - value = summary_proto.value.add() - value.tag = key - value.node_name = key - tensor_proto = tensor_util.make_tensor_proto(dictionary[key]) - value.tensor.CopyFrom(tensor_proto) - logging.info( - 'Summary for np.ndarray is not visible in Tensorboard by default. ' - 'Consider using a Tensorboard plugin for visualization (see ' - 'https://github.com/tensorflow/tensorboard-plugin-example/blob/master/README.md' - ' for more information).') - else: - logging.warn( - 'Skipping summary for %s, must be a float, np.float32, np.int64, ' - 'np.int32 or int or np.ndarray or a serialized string of Summary.', - key) - summary_writer.add_summary(summary_proto, current_global_step) - summary_writer.flush() - - -GraphRewriteSpec = collections.namedtuple('GraphRewriteSpec', - ['tags', 'transforms']) - - -class BaseEstimator(sklearn.BaseEstimator, evaluable.Evaluable, - trainable.Trainable): - """Abstract BaseEstimator class to train and evaluate TensorFlow models. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Users should not instantiate or subclass this class. Instead, use an - `Estimator`. - """ - - # Note that for Google users, this is overridden with - # learn_runner.EstimatorConfig. - # TODO(wicke): Remove this once launcher takes over config functionality - _Config = run_config.RunConfig # pylint: disable=invalid-name - - @deprecated(None, 'Please replace uses of any Estimator from tf.contrib.learn' - ' with an Estimator from tf.estimator.*') - def __init__(self, model_dir=None, config=None): - """Initializes a BaseEstimator instance. - - Args: - model_dir: Directory to save model parameters, graph and etc. This can - also be used to load checkpoints from the directory into a estimator to - continue training a previously saved model. If `None`, the model_dir in - `config` will be used if set. If both are set, they must be same. - config: A RunConfig instance. - """ - # Create a run configuration. - if config is None: - self._config = BaseEstimator._Config() - logging.info('Using default config.') - else: - self._config = config - - if self._config.session_config is None: - self._session_config = config_pb2.ConfigProto(allow_soft_placement=True) - else: - self._session_config = self._config.session_config - - # Model directory. - if (model_dir is not None) and (self._config.model_dir is not None): - if model_dir != self._config.model_dir: - # TODO(b/9965722): remove this suppression after it is no longer - # necessary. - # pylint: disable=g-doc-exception - raise ValueError( - 'model_dir are set both in constructor and RunConfig, but with ' - "different values. In constructor: '{}', in RunConfig: " - "'{}' ".format(model_dir, self._config.model_dir)) - # pylint: enable=g-doc-exception - - self._model_dir = model_dir or self._config.model_dir - if self._model_dir is None: - self._model_dir = tempfile.mkdtemp() - logging.warning('Using temporary folder as model directory: %s', - self._model_dir) - if self._config.model_dir is None: - self._config = self._config.replace(model_dir=self._model_dir) - logging.info('Using config: %s', str(vars(self._config))) - - # Set device function depending if there are replicas or not. - self._device_fn = _get_replica_device_setter(self._config) - - # Features and labels TensorSignature objects. - # TODO(wicke): Rename these to something more descriptive - self._features_info = None - self._labels_info = None - - self._graph = None - - @property - def config(self): - # TODO(wicke): make RunConfig immutable, and then return it without a copy. - return copy.deepcopy(self._config) - - @property - def model_fn(self): - """Returns the model_fn which is bound to self.params. - - Returns: - The model_fn with the following signature: - `def model_fn(features, labels, mode, metrics)` - """ - - def public_model_fn(features, labels, mode, config): - return self._call_model_fn(features, labels, mode, config=config) - - return public_model_fn - - @deprecated_args(SCIKIT_DECOUPLE_DATE, SCIKIT_DECOUPLE_INSTRUCTIONS, - ('x', None), ('y', None), ('batch_size', None)) - def fit(self, - x=None, - y=None, - input_fn=None, - steps=None, - batch_size=None, - monitors=None, - max_steps=None): - # pylint: disable=g-doc-args,g-doc-return-or-yield - """See `Trainable`. - - Raises: - ValueError: If `x` or `y` are not `None` while `input_fn` is not `None`. - ValueError: If both `steps` and `max_steps` are not `None`. - """ - if (steps is not None) and (max_steps is not None): - raise ValueError('Can not provide both steps and max_steps.') - _verify_input_args(x, y, input_fn, None, batch_size) - if x is not None: - SKCompat(self).fit(x, y, batch_size, steps, max_steps, monitors) - return self - - if max_steps is not None: - try: - start_step = load_variable(self._model_dir, ops.GraphKeys.GLOBAL_STEP) - if max_steps <= start_step: - logging.info('Skipping training since max_steps has already saved.') - return self - except: # pylint: disable=bare-except - pass - - hooks = monitor_lib.replace_monitors_with_hooks(monitors, self) - if steps is not None or max_steps is not None: - hooks.append(basic_session_run_hooks.StopAtStepHook(steps, max_steps)) - - loss = self._train_model(input_fn=input_fn, hooks=hooks) - logging.info('Loss for final step: %s.', loss) - return self - - @deprecated_args(SCIKIT_DECOUPLE_DATE, SCIKIT_DECOUPLE_INSTRUCTIONS, - ('x', None), ('y', None), ('batch_size', None)) - def partial_fit(self, - x=None, - y=None, - input_fn=None, - steps=1, - batch_size=None, - monitors=None): - """Incremental fit on a batch of samples. - - This method is expected to be called several times consecutively - on different or the same chunks of the dataset. This either can - implement iterative training or out-of-core/online training. - - This is especially useful when the whole dataset is too big to - fit in memory at the same time. Or when model is taking long time - to converge, and you want to split up training into subparts. - - Args: - x: Matrix of shape [n_samples, n_features...]. Can be iterator that - returns arrays of features. The training input samples for fitting the - model. If set, `input_fn` must be `None`. - y: Vector or matrix [n_samples] or [n_samples, n_outputs]. Can be - iterator that returns array of labels. The training label values - (class labels in classification, real numbers in regression). If set, - `input_fn` must be `None`. - input_fn: Input function. If set, `x`, `y`, and `batch_size` must be - `None`. - steps: Number of steps for which to train model. If `None`, train forever. - batch_size: minibatch size to use on the input, defaults to first - dimension of `x`. Must be `None` if `input_fn` is provided. - monitors: List of `BaseMonitor` subclass instances. Used for callbacks - inside the training loop. - - Returns: - `self`, for chaining. - - Raises: - ValueError: If at least one of `x` and `y` is provided, and `input_fn` is - provided. - """ - logging.warning('The current implementation of partial_fit is not optimized' - ' for use in a loop. Consider using fit() instead.') - return self.fit( - x=x, - y=y, - input_fn=input_fn, - steps=steps, - batch_size=batch_size, - monitors=monitors) - - @deprecated_args(SCIKIT_DECOUPLE_DATE, SCIKIT_DECOUPLE_INSTRUCTIONS, - ('x', None), ('y', None), ('batch_size', None)) - def evaluate(self, - x=None, - y=None, - input_fn=None, - feed_fn=None, - batch_size=None, - steps=None, - metrics=None, - name=None, - checkpoint_path=None, - hooks=None, - log_progress=True): - # pylint: disable=g-doc-args,g-doc-return-or-yield - """See `Evaluable`. - - Raises: - ValueError: If at least one of `x` or `y` is provided, and at least one of - `input_fn` or `feed_fn` is provided. - Or if `metrics` is not `None` or `dict`. - """ - _verify_input_args(x, y, input_fn, feed_fn, batch_size) - if x is not None: - return SKCompat(self).score(x, y, batch_size, steps, metrics, name) - - if metrics is not None and not isinstance(metrics, dict): - raise ValueError('Metrics argument should be None or dict. ' - 'Got %s.' % metrics) - eval_results, global_step = self._evaluate_model( - input_fn=input_fn, - feed_fn=feed_fn, - steps=steps, - metrics=metrics, - name=name, - checkpoint_path=checkpoint_path, - hooks=hooks, - log_progress=log_progress) - - if eval_results is not None: - eval_results.update({'global_step': global_step}) - return eval_results - - @deprecated_args(SCIKIT_DECOUPLE_DATE, SCIKIT_DECOUPLE_INSTRUCTIONS, - ('x', None), ('batch_size', None), ('as_iterable', True)) - def predict(self, - x=None, - input_fn=None, - batch_size=None, - outputs=None, - as_iterable=True, - iterate_batches=False): - """Returns predictions for given features. - - Args: - x: Matrix of shape [n_samples, n_features...]. Can be iterator that - returns arrays of features. The training input samples for fitting the - model. If set, `input_fn` must be `None`. - input_fn: Input function. If set, `x` and 'batch_size' must be `None`. - batch_size: Override default batch size. If set, 'input_fn' must be - 'None'. - outputs: list of `str`, name of the output to predict. - If `None`, returns all. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - iterate_batches: If True, yield the whole batch at once instead of - decomposing the batch into individual samples. Only relevant when - as_iterable is True. - - Returns: - A numpy array of predicted classes or regression values if the - constructor's `model_fn` returns a `Tensor` for `predictions` or a `dict` - of numpy arrays if `model_fn` returns a `dict`. Returns an iterable of - predictions if as_iterable is True. - - Raises: - ValueError: If x and input_fn are both provided or both `None`. - """ - _verify_input_args(x, None, input_fn, None, batch_size) - if x is not None and not as_iterable: - return SKCompat(self).predict(x, batch_size) - - input_fn, feed_fn = _get_input_fn(x, None, input_fn, None, batch_size) - return self._infer_model( - input_fn=input_fn, - feed_fn=feed_fn, - outputs=outputs, - as_iterable=as_iterable, - iterate_batches=iterate_batches) - - def get_variable_value(self, name): - """Returns value of the variable given by name. - - Args: - name: string, name of the tensor. - - Returns: - Numpy array - value of the tensor. - """ - return load_variable(self.model_dir, name) - - def get_variable_names(self): - """Returns list of all variable names in this model. - - Returns: - List of names. - """ - return [name for name, _ in list_variables(self.model_dir)] - - @property - def model_dir(self): - return self._model_dir - - @deprecated('2017-03-25', 'Please use Estimator.export_savedmodel() instead.') - def export( - self, - export_dir, - input_fn=export._default_input_fn, # pylint: disable=protected-access - input_feature_key=None, - use_deprecated_input_fn=True, - signature_fn=None, - prediction_key=None, - default_batch_size=1, - exports_to_keep=None, - checkpoint_path=None): - """Exports inference graph into given dir. - - Args: - export_dir: A string containing a directory to write the exported graph - and checkpoints. - input_fn: If `use_deprecated_input_fn` is true, then a function that given - `Tensor` of `Example` strings, parses it into features that are then - passed to the model. Otherwise, a function that takes no argument and - returns a tuple of (features, labels), where features is a dict of - string key to `Tensor` and labels is a `Tensor` that's currently not - used (and so can be `None`). - input_feature_key: Only used if `use_deprecated_input_fn` is false. String - key into the features dict returned by `input_fn` that corresponds to a - the raw `Example` strings `Tensor` that the exported model will take as - input. Can only be `None` if you're using a custom `signature_fn` that - does not use the first arg (examples). - use_deprecated_input_fn: Determines the signature format of `input_fn`. - signature_fn: Function that returns a default signature and a named - signature map, given `Tensor` of `Example` strings, `dict` of `Tensor`s - for features and `Tensor` or `dict` of `Tensor`s for predictions. - prediction_key: The key for a tensor in the `predictions` dict (output - from the `model_fn`) to use as the `predictions` input to the - `signature_fn`. Optional. If `None`, predictions will pass to - `signature_fn` without filtering. - default_batch_size: Default batch size of the `Example` placeholder. - exports_to_keep: Number of exports to keep. - checkpoint_path: the checkpoint path of the model to be exported. If it is - `None` (which is default), will use the latest checkpoint in - export_dir. - - Returns: - The string path to the exported directory. NB: this functionality was - added ca. 2016/09/25; clients that depend on the return value may need - to handle the case where this function returns None because subclasses - are not returning a value. - """ - # pylint: disable=protected-access - return export._export_estimator( - estimator=self, - export_dir=export_dir, - signature_fn=signature_fn, - prediction_key=prediction_key, - input_fn=input_fn, - input_feature_key=input_feature_key, - use_deprecated_input_fn=use_deprecated_input_fn, - default_batch_size=default_batch_size, - exports_to_keep=exports_to_keep, - checkpoint_path=checkpoint_path) - - @abc.abstractproperty - def _get_train_ops(self, features, labels): - """Method that builds model graph and returns trainer ops. - - Expected to be overridden by sub-classes that require custom support. - - Args: - features: `Tensor` or `dict` of `Tensor` objects. - labels: `Tensor` or `dict` of `Tensor` objects. - - Returns: - A `ModelFnOps` object. - """ - pass - - @abc.abstractproperty - def _get_predict_ops(self, features): - """Method that builds model graph and returns prediction ops. - - Args: - features: `Tensor` or `dict` of `Tensor` objects. - - Returns: - A `ModelFnOps` object. - """ - pass - - def _get_eval_ops(self, features, labels, metrics): - """Method that builds model graph and returns evaluation ops. - - Expected to be overridden by sub-classes that require custom support. - - Args: - features: `Tensor` or `dict` of `Tensor` objects. - labels: `Tensor` or `dict` of `Tensor` objects. - metrics: Dict of metrics to run. If None, the default metric functions - are used; if {}, no metrics are used. Otherwise, `metrics` should map - friendly names for the metric to a `MetricSpec` object defining which - model outputs to evaluate against which labels with which metric - function. Metric ops should support streaming, e.g., returning - update_op and value tensors. See more details in - `../../../../metrics/python/metrics/ops/streaming_metrics.py` and - `../metric_spec.py`. - - Returns: - A `ModelFnOps` object. - """ - raise NotImplementedError('_get_eval_ops not implemented in BaseEstimator') - - @deprecated( - '2016-09-23', - 'The signature of the input_fn accepted by export is changing to be ' - 'consistent with what\'s used by tf.Learn Estimator\'s train/evaluate, ' - 'which makes this function useless. This will be removed after the ' - 'deprecation date.') - def _get_feature_ops_from_example(self, examples_batch): - """Returns feature parser for given example batch using features info. - - This function requires `fit()` has been called. - - Args: - examples_batch: batch of tf.Example - - Returns: - features: `Tensor` or `dict` of `Tensor` objects. - - Raises: - ValueError: If `_features_info` attribute is not available (usually - because `fit()` has not been called). - """ - if self._features_info is None: - raise ValueError('Features information missing, was fit() ever called?') - return tensor_signature.create_example_parser_from_signatures( - self._features_info, examples_batch) - - def _check_inputs(self, features, labels): - if self._features_info is not None: - logging.debug('Given features: %s, required signatures: %s.', - str(features), str(self._features_info)) - if not tensor_signature.tensors_compatible(features, self._features_info): - raise ValueError('Features are incompatible with given information. ' - 'Given features: %s, required signatures: %s.' % - (str(features), str(self._features_info))) - else: - self._features_info = tensor_signature.create_signatures(features) - logging.debug('Setting feature info to %s.', str(self._features_info)) - if labels is not None: - if self._labels_info is not None: - logging.debug('Given labels: %s, required signatures: %s.', str(labels), - str(self._labels_info)) - if not tensor_signature.tensors_compatible(labels, self._labels_info): - raise ValueError('Labels are incompatible with given information. ' - 'Given labels: %s, required signatures: %s.' % - (str(labels), str(self._labels_info))) - else: - self._labels_info = tensor_signature.create_signatures(labels) - logging.debug('Setting labels info to %s', str(self._labels_info)) - - def _extract_metric_update_ops(self, eval_dict): - """Separate update operations from metric value operations.""" - update_ops = [] - value_ops = {} - for name, metric_ops in six.iteritems(eval_dict): - if isinstance(metric_ops, (list, tuple)): - if len(metric_ops) == 2: - value_ops[name] = metric_ops[0] - update_ops.append(metric_ops[1]) - else: - logging.warning( - 'Ignoring metric {}. It returned a list|tuple with len {}, ' - 'expected 2'.format(name, len(metric_ops))) - value_ops[name] = metric_ops - else: - value_ops[name] = metric_ops - - if update_ops: - update_ops = control_flow_ops.group(*update_ops) - else: - update_ops = None - - return update_ops, value_ops - - def _evaluate_model(self, - input_fn, - steps, - feed_fn=None, - metrics=None, - name='', - checkpoint_path=None, - hooks=None, - log_progress=True): - # TODO(wicke): Remove this once Model and associated code are gone. - if (hasattr(self._config, 'execution_mode') and - self._config.execution_mode not in ('all', 'evaluate', 'eval_evalset')): - return None, None - - # Check that model has been trained (if nothing has been set explicitly). - if not checkpoint_path: - latest_path = checkpoint_management.latest_checkpoint(self._model_dir) - if not latest_path: - raise NotFittedError( - "Couldn't find trained model at %s." % self._model_dir) - checkpoint_path = latest_path - - # Setup output directory. - eval_dir = os.path.join(self._model_dir, 'eval' - if not name else 'eval_' + name) - - with ops.Graph().as_default() as g: - random_seed.set_random_seed(self._config.tf_random_seed) - global_step = training_util.create_global_step(g) - features, labels = input_fn() - self._check_inputs(features, labels) - - model_fn_results = self._get_eval_ops(features, labels, metrics) - eval_dict = model_fn_results.eval_metric_ops - - update_op, eval_dict = self._extract_metric_update_ops(eval_dict) - - # We need to copy the hook array as we modify it, thus [:]. - hooks = hooks[:] if hooks else [] - if feed_fn: - hooks.append(basic_session_run_hooks.FeedFnHook(feed_fn)) - if steps == 0: - logging.warning('evaluation steps are 0. If `input_fn` does not raise ' - '`OutOfRangeError`, the evaluation will never stop. ' - 'Use steps=None if intended.') - if steps: - hooks.append( - evaluation.StopAfterNEvalsHook(steps, log_progress=log_progress)) - - global_step_key = 'global_step' - while global_step_key in eval_dict: - global_step_key = '_' + global_step_key - eval_dict[global_step_key] = global_step - - eval_results = evaluation.evaluate_once( - checkpoint_path=checkpoint_path, - master=self._config.evaluation_master, - scaffold=model_fn_results.scaffold, - eval_ops=update_op, - final_ops=eval_dict, - hooks=hooks, - config=self._session_config) - current_global_step = eval_results[global_step_key] - - _write_dict_to_summary(eval_dir, eval_results, current_global_step) - - return eval_results, current_global_step - - def _get_features_from_input_fn(self, input_fn): - result = input_fn() - if isinstance(result, (list, tuple)): - return result[0] - return result - - def _infer_model(self, - input_fn, - feed_fn=None, - outputs=None, - as_iterable=True, - iterate_batches=False): - # Check that model has been trained. - checkpoint_path = checkpoint_management.latest_checkpoint(self._model_dir) - if not checkpoint_path: - raise NotFittedError( - "Couldn't find trained model at %s." % self._model_dir) - - with ops.Graph().as_default() as g: - random_seed.set_random_seed(self._config.tf_random_seed) - training_util.create_global_step(g) - features = self._get_features_from_input_fn(input_fn) - infer_ops = self._get_predict_ops(features) - predictions = self._filter_predictions(infer_ops.predictions, outputs) - mon_sess = monitored_session.MonitoredSession( - session_creator=monitored_session.ChiefSessionCreator( - checkpoint_filename_with_path=checkpoint_path, - scaffold=infer_ops.scaffold, - config=self._session_config)) - if not as_iterable: - with mon_sess: - if not mon_sess.should_stop(): - return mon_sess.run(predictions, feed_fn() if feed_fn else None) - else: - return self._predict_generator(mon_sess, predictions, feed_fn, - iterate_batches) - - def _predict_generator(self, mon_sess, predictions, feed_fn, iterate_batches): - with mon_sess: - while not mon_sess.should_stop(): - preds = mon_sess.run(predictions, feed_fn() if feed_fn else None) - if iterate_batches: - yield preds - elif not isinstance(predictions, dict): - for pred in preds: - yield pred - else: - first_tensor = list(preds.values())[0] - if isinstance(first_tensor, sparse_tensor.SparseTensorValue): - batch_length = first_tensor.dense_shape[0] - else: - batch_length = first_tensor.shape[0] - for i in range(batch_length): - yield {key: value[i] for key, value in six.iteritems(preds)} - if self._is_input_constant(feed_fn, mon_sess.graph): - return - - def _is_input_constant(self, feed_fn, graph): - # If there are no queue_runners, the input `predictions` is a - # constant, and we should stop after the first epoch. If, - # instead, there are queue_runners, eventually they should throw - # an `OutOfRangeError`. - if graph.get_collection(ops.GraphKeys.QUEUE_RUNNERS): - return False - # data_feeder uses feed_fn to generate `OutOfRangeError`. - if feed_fn is not None: - return False - return True - - def _filter_predictions(self, predictions, outputs): - if not outputs: - return predictions - if not isinstance(predictions, dict): - raise ValueError( - 'outputs argument is not valid in case of non-dict predictions.') - existing_keys = predictions.keys() - predictions = { - key: value - for key, value in six.iteritems(predictions) - if key in outputs - } - if not predictions: - raise ValueError('Expected to run at least one output from %s, ' - 'provided %s.' % (existing_keys, outputs)) - return predictions - - def _train_model(self, input_fn, hooks): - all_hooks = [] - self._graph = ops.Graph() - with self._graph.as_default() as g, g.device(self._device_fn): - random_seed.set_random_seed(self._config.tf_random_seed) - global_step = training_util.create_global_step(g) - features, labels = input_fn() - self._check_inputs(features, labels) - training_util._get_or_create_global_step_read() # pylint: disable=protected-access - model_fn_ops = self._get_train_ops(features, labels) - ops.add_to_collection(ops.GraphKeys.LOSSES, model_fn_ops.loss) - all_hooks.extend(hooks) - all_hooks.extend([ - basic_session_run_hooks.NanTensorHook(model_fn_ops.loss), - basic_session_run_hooks.LoggingTensorHook( - { - 'loss': model_fn_ops.loss, - 'step': global_step - }, - every_n_iter=100) - ]) - - scaffold = model_fn_ops.scaffold or monitored_session.Scaffold() - if not (scaffold.saver or ops.get_collection(ops.GraphKeys.SAVERS)): - ops.add_to_collection( - ops.GraphKeys.SAVERS, - saver.Saver( - sharded=True, - max_to_keep=self._config.keep_checkpoint_max, - keep_checkpoint_every_n_hours=( - self._config.keep_checkpoint_every_n_hours), - defer_build=True, - save_relative_paths=True)) - - chief_hooks = [] - if (self._config.save_checkpoints_secs or - self._config.save_checkpoints_steps): - saver_hook_exists = any( - isinstance(h, basic_session_run_hooks.CheckpointSaverHook) - for h in (all_hooks + model_fn_ops.training_hooks + chief_hooks + - model_fn_ops.training_chief_hooks) - ) - if not saver_hook_exists: - chief_hooks = [ - basic_session_run_hooks.CheckpointSaverHook( - self._model_dir, - save_secs=self._config.save_checkpoints_secs, - save_steps=self._config.save_checkpoints_steps, - scaffold=scaffold) - ] - with monitored_session.MonitoredTrainingSession( - master=self._config.master, - is_chief=self._config.is_chief, - checkpoint_dir=self._model_dir, - scaffold=scaffold, - hooks=all_hooks + model_fn_ops.training_hooks, - chief_only_hooks=chief_hooks + model_fn_ops.training_chief_hooks, - save_checkpoint_secs=0, # Saving is handled by a hook. - save_summaries_steps=self._config.save_summary_steps, - max_wait_secs=self._config.session_creation_timeout_secs, - config=self._session_config) as mon_sess: - loss = None - while not mon_sess.should_stop(): - _, loss = mon_sess.run([model_fn_ops.train_op, model_fn_ops.loss]) - return loss - - def latest_checkpoint(self): - """Finds the filename of the latest saved checkpoint file in `model_dir`. - - Returns: - The full path to the latest checkpoint or `None` if no checkpoint was - found. - """ - return checkpoint_management.latest_checkpoint(self.model_dir) - - -def _identity_feature_engineering_fn(features, labels): - return features, labels - - -class Estimator(BaseEstimator): - """Estimator class is the basic TensorFlow model trainer/evaluator. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - def __init__(self, - model_fn=None, - model_dir=None, - config=None, - params=None, - feature_engineering_fn=None): - """Constructs an `Estimator` instance. - - Args: - model_fn: Model function. Follows the signature: - * Args: - * `features`: single `Tensor` or `dict` of `Tensor`s - (depending on data passed to `fit`), - * `labels`: `Tensor` or `dict` of `Tensor`s (for multi-head - models). If mode is `ModeKeys.INFER`, `labels=None` will be - passed. If the `model_fn`'s signature does not accept - `mode`, the `model_fn` must still be able to handle - `labels=None`. - * `mode`: Optional. Specifies if this training, evaluation or - prediction. See `ModeKeys`. - * `params`: Optional `dict` of hyperparameters. Will receive what - is passed to Estimator in `params` parameter. This allows - to configure Estimators from hyper parameter tuning. - * `config`: Optional configuration object. Will receive what is passed - to Estimator in `config` parameter, or the default `config`. - Allows updating things in your model_fn based on configuration - such as `num_ps_replicas`. - * `model_dir`: Optional directory where model parameters, graph etc - are saved. Will receive what is passed to Estimator in - `model_dir` parameter, or the default `model_dir`. Allows - updating things in your model_fn that expect model_dir, such as - training hooks. - - * Returns: - `ModelFnOps` - - Also supports a legacy signature which returns tuple of: - - * predictions: `Tensor`, `SparseTensor` or dictionary of same. - Can also be any type that is convertible to a `Tensor` or - `SparseTensor`, or dictionary of same. - * loss: Scalar loss `Tensor`. - * train_op: Training update `Tensor` or `Operation`. - - Supports next three signatures for the function: - - * `(features, labels) -> (predictions, loss, train_op)` - * `(features, labels, mode) -> (predictions, loss, train_op)` - * `(features, labels, mode, params) -> (predictions, loss, train_op)` - * `(features, labels, mode, params, config) -> - (predictions, loss, train_op)` - * `(features, labels, mode, params, config, model_dir) -> - (predictions, loss, train_op)` - - model_dir: Directory to save model parameters, graph and etc. This can - also be used to load checkpoints from the directory into a estimator to - continue training a previously saved model. - config: Configuration object. - params: `dict` of hyper parameters that will be passed into `model_fn`. - Keys are names of parameters, values are basic python types. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and - returns features and labels which will be fed - into `model_fn`. Please check `model_fn` for - a definition of features and labels. - - Raises: - ValueError: parameters of `model_fn` don't match `params`. - """ - super(Estimator, self).__init__(model_dir=model_dir, config=config) - if model_fn is not None: - # Check number of arguments of the given function matches requirements. - model_fn_args = _model_fn_args(model_fn) - if params is not None and 'params' not in model_fn_args: - raise ValueError('Estimator\'s model_fn (%s) does not have a params ' - 'argument, but params (%s) were passed to the ' - 'Estimator\'s constructor.' % (model_fn, params)) - if params is None and 'params' in model_fn_args: - logging.warning('Estimator\'s model_fn (%s) includes params ' - 'argument, but params are not passed to Estimator.', - model_fn) - self._model_fn = model_fn - self.params = params - self._feature_engineering_fn = ( - feature_engineering_fn or _identity_feature_engineering_fn) - - def _call_model_fn(self, features, labels, mode, metrics=None, config=None): - """Calls model function with support of 2, 3 or 4 arguments. - - Args: - features: features dict. - labels: labels dict. - mode: ModeKeys - metrics: Dict of metrics. - config: RunConfig. - - Returns: - A `ModelFnOps` object. If model_fn returns a tuple, wraps them up in a - `ModelFnOps` object. - - Raises: - ValueError: if model_fn returns invalid objects. - """ - features, labels = self._feature_engineering_fn(features, labels) - model_fn_args = _model_fn_args(self._model_fn) - kwargs = {} - if 'mode' in model_fn_args: - kwargs['mode'] = mode - if 'params' in model_fn_args: - kwargs['params'] = self.params - if 'config' in model_fn_args: - if config: - kwargs['config'] = config - else: - kwargs['config'] = self.config - if 'model_dir' in model_fn_args: - kwargs['model_dir'] = self.model_dir - model_fn_results = self._model_fn(features, labels, **kwargs) - - if isinstance(model_fn_results, model_fn_lib.ModelFnOps): - model_fn_ops = model_fn_results - else: - # Here model_fn_results should be a tuple with 3 elements. - if len(model_fn_results) != 3: - raise ValueError('Unrecognized value returned by model_fn, ' - 'please return ModelFnOps.') - model_fn_ops = model_fn_lib.ModelFnOps( - mode=mode, - predictions=model_fn_results[0], - loss=model_fn_results[1], - train_op=model_fn_results[2]) - - # Custom metrics should overwrite defaults. - if metrics: - model_fn_ops.eval_metric_ops.update( - _make_metrics_ops(metrics, features, labels, - model_fn_ops.predictions)) - - return model_fn_ops - - def _get_train_ops(self, features, labels): - """Method that builds model graph and returns trainer ops. - - Expected to be overridden by sub-classes that require custom support. - This implementation uses `model_fn` passed as parameter to constructor to - build model. - - Args: - features: `Tensor` or `dict` of `Tensor` objects. - labels: `Tensor` or `dict` of `Tensor` objects. - - Returns: - `ModelFnOps` object. - """ - return self._call_model_fn(features, labels, model_fn_lib.ModeKeys.TRAIN) - - def _get_eval_ops(self, features, labels, metrics): - """Method that builds model graph and returns evaluation ops. - - Expected to be overridden by sub-classes that require custom support. - This implementation uses `model_fn` passed as parameter to constructor to - build model. - - Args: - features: `Tensor` or `dict` of `Tensor` objects. - labels: `Tensor` or `dict` of `Tensor` objects. - metrics: Dict of metrics to run. If None, the default metric functions - are used; if {}, no metrics are used. Otherwise, `metrics` should map - friendly names for the metric to a `MetricSpec` object defining which - model outputs to evaluate against which labels with which metric - function. Metric ops should support streaming, e.g., returning - update_op and value tensors. See more details in - `../../../../metrics/python/metrics/ops/streaming_metrics.py` and - `../metric_spec.py`. - - Returns: - `ModelFnOps` object. - - Raises: - ValueError: if `metrics` don't match `labels`. - """ - model_fn_ops = self._call_model_fn(features, labels, - model_fn_lib.ModeKeys.EVAL, metrics) - - if metric_key.MetricKey.LOSS not in model_fn_ops.eval_metric_ops: - model_fn_ops.eval_metric_ops[metric_key.MetricKey.LOSS] = ( - metrics_lib.mean(model_fn_ops.loss)) - return model_fn_ops - - def _get_predict_ops(self, features): - """Method that builds model graph and returns prediction ops. - - Expected to be overridden by sub-classes that require custom support. - This implementation uses `model_fn` passed as parameter to constructor to - build model. - - Args: - features: `Tensor` or `dict` of `Tensor` objects. - - Returns: - `ModelFnOps` object. - """ - labels = tensor_signature.create_placeholders_from_signatures( - self._labels_info) - return self._call_model_fn(features, labels, model_fn_lib.ModeKeys.INFER) - - def export_savedmodel(self, - export_dir_base, - serving_input_fn, - default_output_alternative_key=None, - assets_extra=None, - as_text=False, - checkpoint_path=None, - graph_rewrite_specs=(GraphRewriteSpec( - (tag_constants.SERVING,), ()),), - strip_default_attrs=False): - # pylint: disable=line-too-long - """Exports inference graph as a SavedModel into given dir. - - Args: - export_dir_base: A string containing a directory to write the exported - graph and checkpoints. - serving_input_fn: A function that takes no argument and - returns an `InputFnOps`. - default_output_alternative_key: the name of the head to serve when none is - specified. Not needed for single-headed models. - assets_extra: A dict specifying how to populate the assets.extra directory - within the exported SavedModel. Each key should give the destination - path (including the filename) relative to the assets.extra directory. - The corresponding value gives the full path of the source file to be - copied. For example, the simple case of copying a single file without - renaming it is specified as - `{'my_asset_file.txt': '/path/to/my_asset_file.txt'}`. - as_text: whether to write the SavedModel proto in text format. - checkpoint_path: The checkpoint path to export. If None (the default), - the most recent checkpoint found within the model directory is chosen. - graph_rewrite_specs: an iterable of `GraphRewriteSpec`. Each element will - produce a separate MetaGraphDef within the exported SavedModel, tagged - and rewritten as specified. Defaults to a single entry using the - default serving tag ("serve") and no rewriting. - strip_default_attrs: Boolean. If `True`, default-valued attributes will be - removed from the NodeDefs. For a detailed guide, see - [Stripping Default-Valued - Attributes](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md#stripping-default-valued-attributes). - - Returns: - The string path to the exported directory. - - Raises: - ValueError: if an unrecognized export_type is requested. - """ - # pylint: enable=line-too-long - if serving_input_fn is None: - raise ValueError('serving_input_fn must be defined.') - - if not checkpoint_path: - # Locate the latest checkpoint - checkpoint_path = checkpoint_management.latest_checkpoint(self._model_dir) - if not checkpoint_path: - raise NotFittedError( - "Couldn't find trained model at %s." % self._model_dir) - - export_dir = saved_model_export_utils.get_timestamped_export_dir( - export_dir_base) - # We'll write the SavedModel to a temporary directory and then atomically - # rename it at the end. This helps to avoid corrupt / incomplete outputs, - # which could otherwise occur if the job is preempted or otherwise fails - # in the middle of SavedModel creation. - temp_export_dir = saved_model_export_utils.get_temp_export_dir(export_dir) - builder = saved_model_builder.SavedModelBuilder(temp_export_dir) - - # Build the base graph - with ops.Graph().as_default() as g: - training_util.create_global_step(g) - - # Call the serving_input_fn and collect the input alternatives. - input_ops = serving_input_fn() - input_alternatives, features = ( - saved_model_export_utils.get_input_alternatives(input_ops)) - - # TODO(b/34388557) This is a stopgap, pending recording model provenance. - # Record which features are expected at serving time. It is assumed that - # these are the features that were used in training. - for feature_key in input_ops.features.keys(): - ops.add_to_collection( - constants.COLLECTION_DEF_KEY_FOR_INPUT_FEATURE_KEYS, feature_key) - - # Call the model_fn and collect the output alternatives. - model_fn_ops = self._call_model_fn(features, None, - model_fn_lib.ModeKeys.INFER) - output_alternatives, actual_default_output_alternative_key = ( - saved_model_export_utils.get_output_alternatives( - model_fn_ops, default_output_alternative_key)) - - init_op = control_flow_ops.group(variables.local_variables_initializer(), - resources.initialize_resources( - resources.shared_resources()), - lookup_ops.tables_initializer()) - - # Build the SignatureDefs from all pairs of input and output alternatives - signature_def_map = saved_model_export_utils.build_all_signature_defs( - input_alternatives, output_alternatives, - actual_default_output_alternative_key) - - # Export the first MetaGraphDef with variables, assets etc. - with tf_session.Session('') as session: - - # pylint: disable=protected-access - saveables = variables._all_saveable_objects() - # pylint: enable=protected-access - - if (model_fn_ops.scaffold is not None and - model_fn_ops.scaffold.saver is not None): - saver_for_restore = model_fn_ops.scaffold.saver - elif saveables: - saver_for_restore = saver.Saver(saveables, sharded=True) - - saver_for_restore.restore(session, checkpoint_path) - - # Perform the export - if not graph_rewrite_specs or graph_rewrite_specs[0].transforms: - raise ValueError('The first element of graph_rewrite_specs ' - 'must specify no transforms.') - untransformed_tags = graph_rewrite_specs[0].tags - - builder.add_meta_graph_and_variables( - session, - untransformed_tags, - signature_def_map=signature_def_map, - assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), - main_op=init_op, - strip_default_attrs=strip_default_attrs) - - # pylint: disable=protected-access - base_meta_graph_def = builder._saved_model.meta_graphs[0] - # pylint: enable=protected-access - - if graph_rewrite_specs[1:]: - # Prepare the input_names and output_names needed for the - # meta_graph_transform call below. - input_names = [ - tensor.name - for input_dict in input_alternatives.values() - for tensor in input_dict.values() - ] - output_names = [ - tensor.name - for output_alternative in output_alternatives.values() - for tensor in output_alternative[1].values() - ] - - # Write the additional MetaGraphDefs - for graph_rewrite_spec in graph_rewrite_specs[1:]: - - # TODO(soergel) consider moving most of this to saved_model.builder_impl - # as e.g. builder.add_rewritten_meta_graph(rewritten_graph_def, tags) - - transformed_meta_graph_def = meta_graph_transform.meta_graph_transform( - base_meta_graph_def, input_names, output_names, - graph_rewrite_spec.transforms, graph_rewrite_spec.tags) - - # pylint: disable=protected-access - meta_graph_def = builder._saved_model.meta_graphs.add() - # pylint: enable=protected-access - meta_graph_def.CopyFrom(transformed_meta_graph_def) - - # Add the extra assets - if assets_extra: - assets_extra_path = os.path.join( - compat.as_bytes(temp_export_dir), compat.as_bytes('assets.extra')) - for dest_relative, source in assets_extra.items(): - dest_absolute = os.path.join( - compat.as_bytes(assets_extra_path), compat.as_bytes(dest_relative)) - dest_path = os.path.dirname(dest_absolute) - gfile.MakeDirs(dest_path) - gfile.Copy(source, dest_absolute) - - builder.save(as_text) - gfile.Rename(temp_export_dir, export_dir) - return export_dir - - -# For time of deprecation x,y from Estimator allow direct access. -# pylint: disable=protected-access -class SKCompat(sklearn.BaseEstimator): - """Scikit learn wrapper for TensorFlow Learn Estimator. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - @deprecated(None, 'Please switch to the Estimator interface.') - def __init__(self, estimator): - self._estimator = estimator - - def fit(self, x, y, batch_size=128, steps=None, max_steps=None, - monitors=None): - input_fn, feed_fn = _get_input_fn( - x, - y, - input_fn=None, - feed_fn=None, - batch_size=batch_size, - shuffle=True, - epochs=None) - all_monitors = [] - if feed_fn: - all_monitors = [basic_session_run_hooks.FeedFnHook(feed_fn)] - if monitors: - all_monitors.extend(monitors) - - self._estimator.fit( - input_fn=input_fn, - steps=steps, - max_steps=max_steps, - monitors=all_monitors) - return self - - def score(self, x, y, batch_size=128, steps=None, metrics=None, name=None): - input_fn, feed_fn = _get_input_fn( - x, - y, - input_fn=None, - feed_fn=None, - batch_size=batch_size, - shuffle=False, - epochs=1) - if metrics is not None and not isinstance(metrics, dict): - raise ValueError('Metrics argument should be None or dict. ' - 'Got %s.' % metrics) - eval_results, global_step = self._estimator._evaluate_model( - input_fn=input_fn, - feed_fn=feed_fn, - steps=steps, - metrics=metrics, - name=name) - if eval_results is not None: - eval_results.update({'global_step': global_step}) - return eval_results - - def predict(self, x, batch_size=128, outputs=None): - input_fn, feed_fn = _get_input_fn( - x, - None, - input_fn=None, - feed_fn=None, - batch_size=batch_size, - shuffle=False, - epochs=1) - results = list( - self._estimator._infer_model( - input_fn=input_fn, - feed_fn=feed_fn, - outputs=outputs, - as_iterable=True, - iterate_batches=True)) - if not isinstance(results[0], dict): - return np.concatenate([output for output in results], axis=0) - return { - key: np.concatenate([output[key] for output in results], axis=0) - for key in results[0] - } diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator_input_test.py b/tensorflow/contrib/learn/python/learn/estimators/estimator_input_test.py deleted file mode 100644 index d4a46b41d0c..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/estimator_input_test.py +++ /dev/null @@ -1,349 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Estimator input.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import tempfile - -import numpy as np - -from tensorflow.python.training import training_util -from tensorflow.contrib.layers.python.layers import optimizers -from tensorflow.contrib.learn.python.learn import metric_spec -from tensorflow.contrib.learn.python.learn import models -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.contrib.learn.python.learn.estimators import _sklearn -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.metrics.python.ops import metric_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test -from tensorflow.python.training import input as input_lib -from tensorflow.python.training import queue_runner_impl - -_BOSTON_INPUT_DIM = 13 -_IRIS_INPUT_DIM = 4 - - -def boston_input_fn(num_epochs=None): - boston = base.load_boston() - features = input_lib.limit_epochs( - array_ops.reshape( - constant_op.constant(boston.data), [-1, _BOSTON_INPUT_DIM]), - num_epochs=num_epochs) - labels = array_ops.reshape(constant_op.constant(boston.target), [-1, 1]) - return features, labels - - -def boston_input_fn_with_queue(num_epochs=None): - features, labels = boston_input_fn(num_epochs=num_epochs) - - # Create a minimal queue runner. - fake_queue = data_flow_ops.FIFOQueue(30, dtypes.int32) - queue_runner = queue_runner_impl.QueueRunner(fake_queue, - [constant_op.constant(0)]) - queue_runner_impl.add_queue_runner(queue_runner) - - return features, labels - - -def iris_input_fn(): - iris = base.load_iris() - features = array_ops.reshape( - constant_op.constant(iris.data), [-1, _IRIS_INPUT_DIM]) - labels = array_ops.reshape(constant_op.constant(iris.target), [-1]) - return features, labels - - -def iris_input_fn_labels_dict(): - iris = base.load_iris() - features = array_ops.reshape( - constant_op.constant(iris.data), [-1, _IRIS_INPUT_DIM]) - labels = { - 'labels': array_ops.reshape(constant_op.constant(iris.target), [-1]) - } - return features, labels - - -def boston_eval_fn(): - boston = base.load_boston() - n_examples = len(boston.target) - features = array_ops.reshape( - constant_op.constant(boston.data), [n_examples, _BOSTON_INPUT_DIM]) - labels = array_ops.reshape( - constant_op.constant(boston.target), [n_examples, 1]) - return array_ops.concat([features, features], - 0), array_ops.concat([labels, labels], 0) - - -def extract(data, key): - if isinstance(data, dict): - assert key in data - return data[key] - else: - return data - - -def linear_model_params_fn(features, labels, mode, params): - features = extract(features, 'input') - labels = extract(labels, 'labels') - - assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL, - model_fn.ModeKeys.INFER) - prediction, loss = (models.linear_regression_zero_init(features, labels)) - train_op = optimizers.optimize_loss( - loss, - training_util.get_global_step(), - optimizer='Adagrad', - learning_rate=params['learning_rate']) - return prediction, loss, train_op - - -def linear_model_fn(features, labels, mode): - features = extract(features, 'input') - labels = extract(labels, 'labels') - assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL, - model_fn.ModeKeys.INFER) - if isinstance(features, dict): - (_, features), = features.items() - prediction, loss = (models.linear_regression_zero_init(features, labels)) - train_op = optimizers.optimize_loss( - loss, - training_util.get_global_step(), - optimizer='Adagrad', - learning_rate=0.1) - return prediction, loss, train_op - - -def linear_model_fn_with_model_fn_ops(features, labels, mode): - """Same as linear_model_fn, but returns `ModelFnOps`.""" - assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL, - model_fn.ModeKeys.INFER) - prediction, loss = (models.linear_regression_zero_init(features, labels)) - train_op = optimizers.optimize_loss( - loss, - training_util.get_global_step(), - optimizer='Adagrad', - learning_rate=0.1) - return model_fn.ModelFnOps( - mode=mode, predictions=prediction, loss=loss, train_op=train_op) - - -def logistic_model_no_mode_fn(features, labels): - features = extract(features, 'input') - labels = extract(labels, 'labels') - labels = array_ops.one_hot(labels, 3, 1, 0) - prediction, loss = (models.logistic_regression_zero_init(features, labels)) - train_op = optimizers.optimize_loss( - loss, - training_util.get_global_step(), - optimizer='Adagrad', - learning_rate=0.1) - return { - 'class': math_ops.argmax(prediction, 1), - 'prob': prediction - }, loss, train_op - - -VOCAB_FILE_CONTENT = 'emerson\nlake\npalmer\n' -EXTRA_FILE_CONTENT = 'kermit\npiggy\nralph\n' - - -class EstimatorInputTest(test.TestCase): - - def testContinueTrainingDictionaryInput(self): - boston = base.load_boston() - output_dir = tempfile.mkdtemp() - est = estimator.Estimator(model_fn=linear_model_fn, model_dir=output_dir) - boston_input = {'input': boston.data} - float64_target = {'labels': boston.target.astype(np.float64)} - est.fit(x=boston_input, y=float64_target, steps=50) - scores = est.evaluate( - x=boston_input, - y=float64_target, - metrics={ - 'MSE': metric_ops.streaming_mean_squared_error - }) - del est - # Create another estimator object with the same output dir. - est2 = estimator.Estimator(model_fn=linear_model_fn, model_dir=output_dir) - - # Check we can evaluate and predict. - scores2 = est2.evaluate( - x=boston_input, - y=float64_target, - metrics={ - 'MSE': metric_ops.streaming_mean_squared_error - }) - self.assertAllClose(scores2['MSE'], scores['MSE']) - predictions = np.array(list(est2.predict(x=boston_input))) - other_score = _sklearn.mean_squared_error(predictions, - float64_target['labels']) - self.assertAllClose(other_score, scores['MSE']) - - def testBostonAll(self): - boston = base.load_boston() - est = estimator.SKCompat(estimator.Estimator(model_fn=linear_model_fn)) - float64_labels = boston.target.astype(np.float64) - est.fit(x=boston.data, y=float64_labels, steps=100) - scores = est.score( - x=boston.data, - y=float64_labels, - metrics={ - 'MSE': metric_ops.streaming_mean_squared_error - }) - predictions = np.array(list(est.predict(x=boston.data))) - other_score = _sklearn.mean_squared_error(predictions, boston.target) - self.assertAllClose(scores['MSE'], other_score) - self.assertTrue('global_step' in scores) - self.assertEqual(100, scores['global_step']) - - def testBostonAllDictionaryInput(self): - boston = base.load_boston() - est = estimator.Estimator(model_fn=linear_model_fn) - boston_input = {'input': boston.data} - float64_target = {'labels': boston.target.astype(np.float64)} - est.fit(x=boston_input, y=float64_target, steps=100) - scores = est.evaluate( - x=boston_input, - y=float64_target, - metrics={ - 'MSE': metric_ops.streaming_mean_squared_error - }) - predictions = np.array(list(est.predict(x=boston_input))) - other_score = _sklearn.mean_squared_error(predictions, boston.target) - self.assertAllClose(other_score, scores['MSE']) - self.assertTrue('global_step' in scores) - self.assertEqual(scores['global_step'], 100) - - def testIrisAll(self): - iris = base.load_iris() - est = estimator.SKCompat( - estimator.Estimator(model_fn=logistic_model_no_mode_fn)) - est.fit(iris.data, iris.target, steps=100) - scores = est.score( - x=iris.data, - y=iris.target, - metrics={ - ('accuracy', 'class'): metric_ops.streaming_accuracy - }) - predictions = est.predict(x=iris.data) - predictions_class = est.predict(x=iris.data, outputs=['class'])['class'] - self.assertEqual(predictions['prob'].shape[0], iris.target.shape[0]) - self.assertAllClose(predictions['class'], predictions_class) - self.assertAllClose(predictions['class'], - np.argmax(predictions['prob'], axis=1)) - other_score = _sklearn.accuracy_score(iris.target, predictions['class']) - self.assertAllClose(scores['accuracy'], other_score) - self.assertTrue('global_step' in scores) - self.assertEqual(100, scores['global_step']) - - def testIrisAllDictionaryInput(self): - iris = base.load_iris() - est = estimator.Estimator(model_fn=logistic_model_no_mode_fn) - iris_data = {'input': iris.data} - iris_target = {'labels': iris.target} - est.fit(iris_data, iris_target, steps=100) - scores = est.evaluate( - x=iris_data, - y=iris_target, - metrics={ - ('accuracy', 'class'): metric_ops.streaming_accuracy - }) - predictions = list(est.predict(x=iris_data)) - predictions_class = list(est.predict(x=iris_data, outputs=['class'])) - self.assertEqual(len(predictions), iris.target.shape[0]) - classes_batch = np.array([p['class'] for p in predictions]) - self.assertAllClose(classes_batch, - np.array([p['class'] for p in predictions_class])) - self.assertAllClose(classes_batch, - np.argmax( - np.array([p['prob'] for p in predictions]), axis=1)) - other_score = _sklearn.accuracy_score(iris.target, classes_batch) - self.assertAllClose(other_score, scores['accuracy']) - self.assertTrue('global_step' in scores) - self.assertEqual(scores['global_step'], 100) - - def testIrisInputFn(self): - iris = base.load_iris() - est = estimator.Estimator(model_fn=logistic_model_no_mode_fn) - est.fit(input_fn=iris_input_fn, steps=100) - _ = est.evaluate(input_fn=iris_input_fn, steps=1) - predictions = list(est.predict(x=iris.data)) - self.assertEqual(len(predictions), iris.target.shape[0]) - - def testIrisInputFnLabelsDict(self): - iris = base.load_iris() - est = estimator.Estimator(model_fn=logistic_model_no_mode_fn) - est.fit(input_fn=iris_input_fn_labels_dict, steps=100) - _ = est.evaluate( - input_fn=iris_input_fn_labels_dict, - steps=1, - metrics={ - 'accuracy': - metric_spec.MetricSpec( - metric_fn=metric_ops.streaming_accuracy, - prediction_key='class', - label_key='labels') - }) - predictions = list(est.predict(x=iris.data)) - self.assertEqual(len(predictions), iris.target.shape[0]) - - def testTrainInputFn(self): - est = estimator.Estimator(model_fn=linear_model_fn) - est.fit(input_fn=boston_input_fn, steps=1) - _ = est.evaluate(input_fn=boston_eval_fn, steps=1) - - def testPredictInputFn(self): - est = estimator.Estimator(model_fn=linear_model_fn) - boston = base.load_boston() - est.fit(input_fn=boston_input_fn, steps=1) - input_fn = functools.partial(boston_input_fn, num_epochs=1) - output = list(est.predict(input_fn=input_fn)) - self.assertEqual(len(output), boston.target.shape[0]) - - def testPredictInputFnWithQueue(self): - est = estimator.Estimator(model_fn=linear_model_fn) - boston = base.load_boston() - est.fit(input_fn=boston_input_fn, steps=1) - input_fn = functools.partial(boston_input_fn_with_queue, num_epochs=2) - output = list(est.predict(input_fn=input_fn)) - self.assertEqual(len(output), boston.target.shape[0] * 2) - - def testPredictConstInputFn(self): - est = estimator.Estimator(model_fn=linear_model_fn) - boston = base.load_boston() - est.fit(input_fn=boston_input_fn, steps=1) - - def input_fn(): - features = array_ops.reshape( - constant_op.constant(boston.data), [-1, _BOSTON_INPUT_DIM]) - labels = array_ops.reshape(constant_op.constant(boston.target), [-1, 1]) - return features, labels - - output = list(est.predict(input_fn=input_fn)) - self.assertEqual(len(output), boston.target.shape[0]) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator_test.py b/tensorflow/contrib/learn/python/learn/estimators/estimator_test.py deleted file mode 100644 index 153d4867961..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/estimator_test.py +++ /dev/null @@ -1,1411 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Estimator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import itertools -import json -import os -import tempfile - -import numpy as np -import six -from six.moves import xrange # pylint: disable=redefined-builtin - -from google.protobuf import text_format - -from tensorflow.contrib import learn -from tensorflow.contrib import lookup -from tensorflow.python.training import training_util -from tensorflow.contrib.layers.python.layers import feature_column as feature_column_lib -from tensorflow.contrib.layers.python.layers import optimizers -from tensorflow.contrib.learn.python.learn import experiment -from tensorflow.contrib.learn.python.learn import models -from tensorflow.contrib.learn.python.learn import monitors as monitors_lib -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.contrib.learn.python.learn.estimators import _sklearn -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import linear -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.contrib.learn.python.learn.utils import input_fn_utils -from tensorflow.contrib.metrics.python.ops import metric_ops -from tensorflow.contrib.testing.python.framework import util_test -from tensorflow.python.client import session as session_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.lib.io import file_io -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test -from tensorflow.python.saved_model import loader -from tensorflow.python.saved_model import tag_constants -from tensorflow.python.summary import summary -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.training import checkpoint_state_pb2 -from tensorflow.python.training import input as input_lib -from tensorflow.python.training import monitored_session -from tensorflow.python.training import saver as saver_lib -from tensorflow.python.training import session_run_hook -from tensorflow.python.util import compat - -_BOSTON_INPUT_DIM = 13 -_IRIS_INPUT_DIM = 4 - - -def boston_input_fn(num_epochs=None): - boston = base.load_boston() - features = input_lib.limit_epochs( - array_ops.reshape( - constant_op.constant(boston.data), [-1, _BOSTON_INPUT_DIM]), - num_epochs=num_epochs) - labels = array_ops.reshape(constant_op.constant(boston.target), [-1, 1]) - return features, labels - - -def iris_input_fn(): - iris = base.load_iris() - features = array_ops.reshape( - constant_op.constant(iris.data), [-1, _IRIS_INPUT_DIM]) - labels = array_ops.reshape(constant_op.constant(iris.target), [-1]) - return features, labels - - -def iris_input_fn_labels_dict(): - iris = base.load_iris() - features = array_ops.reshape( - constant_op.constant(iris.data), [-1, _IRIS_INPUT_DIM]) - labels = { - 'labels': array_ops.reshape(constant_op.constant(iris.target), [-1]) - } - return features, labels - - -def boston_eval_fn(): - boston = base.load_boston() - n_examples = len(boston.target) - features = array_ops.reshape( - constant_op.constant(boston.data), [n_examples, _BOSTON_INPUT_DIM]) - labels = array_ops.reshape( - constant_op.constant(boston.target), [n_examples, 1]) - return array_ops.concat([features, features], - 0), array_ops.concat([labels, labels], 0) - - -def extract(data, key): - if isinstance(data, dict): - assert key in data - return data[key] - else: - return data - - -def linear_model_params_fn(features, labels, mode, params): - features = extract(features, 'input') - labels = extract(labels, 'labels') - - assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL, - model_fn.ModeKeys.INFER) - prediction, loss = (models.linear_regression_zero_init(features, labels)) - train_op = optimizers.optimize_loss( - loss, - training_util.get_global_step(), - optimizer='Adagrad', - learning_rate=params['learning_rate']) - return prediction, loss, train_op - - -def linear_model_fn(features, labels, mode): - features = extract(features, 'input') - labels = extract(labels, 'labels') - assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL, - model_fn.ModeKeys.INFER) - if isinstance(features, dict): - (_, features), = features.items() - prediction, loss = (models.linear_regression_zero_init(features, labels)) - train_op = optimizers.optimize_loss( - loss, - training_util.get_global_step(), - optimizer='Adagrad', - learning_rate=0.1) - return prediction, loss, train_op - - -def linear_model_fn_with_model_fn_ops(features, labels, mode): - """Same as linear_model_fn, but returns `ModelFnOps`.""" - assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL, - model_fn.ModeKeys.INFER) - prediction, loss = (models.linear_regression_zero_init(features, labels)) - train_op = optimizers.optimize_loss( - loss, - training_util.get_global_step(), - optimizer='Adagrad', - learning_rate=0.1) - return model_fn.ModelFnOps( - mode=mode, predictions=prediction, loss=loss, train_op=train_op) - - -def logistic_model_no_mode_fn(features, labels): - features = extract(features, 'input') - labels = extract(labels, 'labels') - labels = array_ops.one_hot(labels, 3, 1, 0) - prediction, loss = (models.logistic_regression_zero_init(features, labels)) - train_op = optimizers.optimize_loss( - loss, - training_util.get_global_step(), - optimizer='Adagrad', - learning_rate=0.1) - return { - 'class': math_ops.argmax(prediction, 1), - 'prob': prediction - }, loss, train_op - - -VOCAB_FILE_CONTENT = 'emerson\nlake\npalmer\n' -EXTRA_FILE_CONTENT = 'kermit\npiggy\nralph\n' - - -def _build_estimator_for_export_tests(tmpdir): - - def _input_fn(): - iris = base.load_iris() - return { - 'feature': constant_op.constant(iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=[150], dtype=dtypes.int32) - - feature_columns = [ - feature_column_lib.real_valued_column('feature', dimension=4) - ] - - est = linear.LinearRegressor(feature_columns) - est.fit(input_fn=_input_fn, steps=20) - - feature_spec = feature_column_lib.create_feature_spec_for_parsing( - feature_columns) - serving_input_fn = input_fn_utils.build_parsing_serving_input_fn(feature_spec) - - # hack in an op that uses an asset, in order to test asset export. - # this is not actually valid, of course. - def serving_input_fn_with_asset(): - features, labels, inputs = serving_input_fn() - - vocab_file_name = os.path.join(tmpdir, 'my_vocab_file') - vocab_file = gfile.GFile(vocab_file_name, mode='w') - vocab_file.write(VOCAB_FILE_CONTENT) - vocab_file.close() - hashtable = lookup.HashTable( - lookup.TextFileStringTableInitializer(vocab_file_name), 'x') - features['bogus_lookup'] = hashtable.lookup( - math_ops.cast(features['feature'], dtypes.int64)) - - return input_fn_utils.InputFnOps(features, labels, inputs) - - return est, serving_input_fn_with_asset - - -def _build_estimator_for_resource_export_test(): - - def _input_fn(): - iris = base.load_iris() - return { - 'feature': constant_op.constant(iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=[150], dtype=dtypes.int32) - - feature_columns = [ - feature_column_lib.real_valued_column('feature', dimension=4) - ] - - def resource_constant_model_fn(unused_features, unused_labels, mode): - """A model_fn that loads a constant from a resource and serves it.""" - assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL, - model_fn.ModeKeys.INFER) - - const = constant_op.constant(-1, dtype=dtypes.int64) - table = lookup.MutableHashTable( - dtypes.string, dtypes.int64, const, name='LookupTableModel') - update_global_step = training_util.get_global_step().assign_add(1) - if mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL): - key = constant_op.constant(['key']) - value = constant_op.constant([42], dtype=dtypes.int64) - train_op_1 = table.insert(key, value) - training_state = lookup.MutableHashTable( - dtypes.string, dtypes.int64, const, name='LookupTableTrainingState') - training_op_2 = training_state.insert(key, value) - return (const, const, - control_flow_ops.group(train_op_1, training_op_2, - update_global_step)) - if mode == model_fn.ModeKeys.INFER: - key = constant_op.constant(['key']) - prediction = table.lookup(key) - return prediction, const, update_global_step - - est = estimator.Estimator(model_fn=resource_constant_model_fn) - est.fit(input_fn=_input_fn, steps=1) - - feature_spec = feature_column_lib.create_feature_spec_for_parsing( - feature_columns) - serving_input_fn = input_fn_utils.build_parsing_serving_input_fn(feature_spec) - return est, serving_input_fn - - -class CheckCallsMonitor(monitors_lib.BaseMonitor): - - def __init__(self, expect_calls): - super(CheckCallsMonitor, self).__init__() - self.begin_calls = None - self.end_calls = None - self.expect_calls = expect_calls - - def begin(self, max_steps): - self.begin_calls = 0 - self.end_calls = 0 - - def step_begin(self, step): - self.begin_calls += 1 - return {} - - def step_end(self, step, outputs): - self.end_calls += 1 - return False - - def end(self): - assert (self.end_calls == self.expect_calls and - self.begin_calls == self.expect_calls) - - -def _model_fn_ops(expected_features, expected_labels, actual_features, - actual_labels, mode): - assert_ops = tuple([ - check_ops.assert_equal( - expected_features[k], actual_features[k], name='assert_%s' % k) - for k in expected_features - ] + [ - check_ops.assert_equal( - expected_labels, actual_labels, name='assert_labels') - ]) - with ops.control_dependencies(assert_ops): - return model_fn.ModelFnOps( - mode=mode, - predictions=constant_op.constant(0.), - loss=constant_op.constant(0.), - train_op=training_util.get_global_step().assign_add(1)) - - -def _make_input_fn(features, labels): - - def _input_fn(): - return {k: constant_op.constant(v) - for k, v in six.iteritems(features)}, constant_op.constant(labels) - - return _input_fn - - -class EstimatorModelFnTest(test.TestCase): - - def testModelFnArgs(self): - features = {'x': 42., 'y': 43.} - labels = 44. - expected_params = {'some_param': 'some_value'} - expected_config = run_config.RunConfig() - expected_config.i_am_test = True - - # TODO(ptucker): We have to roll our own mock since Estimator._get_arguments - # doesn't work with mock fns. - model_fn_call_count = [0] - - # `features` and `labels` are passed by position, `arg0` and `arg1` here. - def _model_fn(arg0, arg1, mode, params, config): - model_fn_call_count[0] += 1 - self.assertItemsEqual(features.keys(), arg0.keys()) - self.assertEqual(model_fn.ModeKeys.TRAIN, mode) - self.assertEqual(expected_params, params) - self.assertTrue(config.i_am_test) - return _model_fn_ops(features, labels, arg0, arg1, mode) - - est = estimator.Estimator( - model_fn=_model_fn, params=expected_params, config=expected_config) - self.assertEqual(0, model_fn_call_count[0]) - est.fit(input_fn=_make_input_fn(features, labels), steps=1) - self.assertEqual(1, model_fn_call_count[0]) - - def testPartialModelFnArgs(self): - features = {'x': 42., 'y': 43.} - labels = 44. - expected_params = {'some_param': 'some_value'} - expected_config = run_config.RunConfig() - expected_config.i_am_test = True - expected_foo = 45. - expected_bar = 46. - - # TODO(ptucker): We have to roll our own mock since Estimator._get_arguments - # doesn't work with mock fns. - model_fn_call_count = [0] - - # `features` and `labels` are passed by position, `arg0` and `arg1` here. - def _model_fn(arg0, arg1, foo, mode, params, config, bar): - model_fn_call_count[0] += 1 - self.assertEqual(expected_foo, foo) - self.assertEqual(expected_bar, bar) - self.assertItemsEqual(features.keys(), arg0.keys()) - self.assertEqual(model_fn.ModeKeys.TRAIN, mode) - self.assertEqual(expected_params, params) - self.assertTrue(config.i_am_test) - return _model_fn_ops(features, labels, arg0, arg1, mode) - - partial_model_fn = functools.partial( - _model_fn, foo=expected_foo, bar=expected_bar) - - est = estimator.Estimator( - model_fn=partial_model_fn, - params=expected_params, - config=expected_config) - self.assertEqual(0, model_fn_call_count[0]) - est.fit(input_fn=_make_input_fn(features, labels), steps=1) - self.assertEqual(1, model_fn_call_count[0]) - - def testModelFnWithModelDir(self): - expected_param = {'some_param': 'some_value'} - expected_model_dir = tempfile.mkdtemp() - - def _argument_checker(features, - labels, - mode, - params, - config=None, - model_dir=None): - _, _, _ = features, labels, config - self.assertEqual(model_fn.ModeKeys.TRAIN, mode) - self.assertEqual(expected_param, params) - self.assertEqual(model_dir, expected_model_dir) - return (constant_op.constant(0.), constant_op.constant(0.), - training_util.get_global_step().assign_add(1)) - - est = estimator.Estimator( - model_fn=_argument_checker, - params=expected_param, - model_dir=expected_model_dir) - est.fit(input_fn=boston_input_fn, steps=1) - - def testInvalidModelFn_no_train_op(self): - - def _invalid_model_fn(features, labels): - # pylint: disable=unused-argument - w = variables_lib.Variable(42.0, 'weight') - update_global_step = training_util.get_global_step().assign_add(1) - with ops.control_dependencies([update_global_step]): - loss = 100.0 - w - return None, loss, None - - est = estimator.Estimator(model_fn=_invalid_model_fn) - with self.assertRaisesRegexp(ValueError, 'Missing train_op'): - est.fit(input_fn=boston_input_fn, steps=1) - - def testInvalidModelFn_no_loss(self): - - def _invalid_model_fn(features, labels, mode): - # pylint: disable=unused-argument - w = variables_lib.Variable(42.0, 'weight') - loss = 100.0 - w - update_global_step = training_util.get_global_step().assign_add(1) - with ops.control_dependencies([update_global_step]): - train_op = w.assign_add(loss / 100.0) - predictions = loss - if mode == model_fn.ModeKeys.EVAL: - loss = None - return predictions, loss, train_op - - est = estimator.Estimator(model_fn=_invalid_model_fn) - est.fit(input_fn=boston_input_fn, steps=1) - with self.assertRaisesRegexp(ValueError, 'Missing loss'): - est.evaluate(input_fn=boston_eval_fn, steps=1) - - def testInvalidModelFn_no_prediction(self): - - def _invalid_model_fn(features, labels): - # pylint: disable=unused-argument - w = variables_lib.Variable(42.0, 'weight') - loss = 100.0 - w - update_global_step = training_util.get_global_step().assign_add(1) - with ops.control_dependencies([update_global_step]): - train_op = w.assign_add(loss / 100.0) - return None, loss, train_op - - est = estimator.Estimator(model_fn=_invalid_model_fn) - est.fit(input_fn=boston_input_fn, steps=1) - with self.assertRaisesRegexp(ValueError, 'Missing prediction'): - est.evaluate(input_fn=boston_eval_fn, steps=1) - with self.assertRaisesRegexp(ValueError, 'Missing prediction'): - est.predict(input_fn=boston_input_fn) - with self.assertRaisesRegexp(ValueError, 'Missing prediction'): - est.predict( - input_fn=functools.partial(boston_input_fn, num_epochs=1), - as_iterable=True) - - def testModelFnScaffoldInTraining(self): - self.is_init_fn_called = False - - def _init_fn(scaffold, session): - _, _ = scaffold, session - self.is_init_fn_called = True - - def _model_fn_scaffold(features, labels, mode): - _, _ = features, labels - return model_fn.ModelFnOps( - mode=mode, - predictions=constant_op.constant(0.), - loss=constant_op.constant(0.), - train_op=training_util.get_global_step().assign_add(1), - scaffold=monitored_session.Scaffold(init_fn=_init_fn)) - - est = estimator.Estimator(model_fn=_model_fn_scaffold) - est.fit(input_fn=boston_input_fn, steps=1) - self.assertTrue(self.is_init_fn_called) - - def testModelFnScaffoldSaverUsage(self): - - def _model_fn_scaffold(features, labels, mode): - _, _ = features, labels - variables_lib.Variable(1., 'weight') - real_saver = saver_lib.Saver() - self.mock_saver = test.mock.Mock( - wraps=real_saver, saver_def=real_saver.saver_def) - return model_fn.ModelFnOps( - mode=mode, - predictions=constant_op.constant([[1.]]), - loss=constant_op.constant(0.), - train_op=training_util.get_global_step().assign_add(1), - scaffold=monitored_session.Scaffold(saver=self.mock_saver)) - - def input_fn(): - return { - 'x': constant_op.constant([[1.]]), - }, constant_op.constant([[1.]]) - - est = estimator.Estimator(model_fn=_model_fn_scaffold) - est.fit(input_fn=input_fn, steps=1) - self.assertTrue(self.mock_saver.save.called) - est.evaluate(input_fn=input_fn, steps=1) - self.assertTrue(self.mock_saver.restore.called) - est.predict(input_fn=input_fn) - self.assertTrue(self.mock_saver.restore.called) - - def serving_input_fn(): - serialized_tf_example = array_ops.placeholder( - dtype=dtypes.string, shape=[None], name='input_example_tensor') - features, labels = input_fn() - return input_fn_utils.InputFnOps(features, labels, { - 'examples': serialized_tf_example - }) - - est.export_savedmodel( - os.path.join(est.model_dir, 'export'), serving_input_fn) - self.assertTrue(self.mock_saver.restore.called) - - -class EstimatorTest(test.TestCase): - - def testExperimentIntegration(self): - exp = experiment.Experiment( - estimator=estimator.Estimator(model_fn=linear_model_fn), - train_input_fn=boston_input_fn, - eval_input_fn=boston_input_fn) - exp.test() - - def testCheckpointSaverHookSuppressesTheDefaultOne(self): - saver_hook = test.mock.Mock( - spec=basic_session_run_hooks.CheckpointSaverHook) - saver_hook.before_run.return_value = None - est = estimator.Estimator(model_fn=linear_model_fn) - est.fit(input_fn=boston_input_fn, steps=1, monitors=[saver_hook]) - # test nothing is saved, due to suppressing default saver - with self.assertRaises(learn.NotFittedError): - est.evaluate(input_fn=boston_input_fn, steps=1) - - def testCustomConfig(self): - test_random_seed = 5783452 - - class TestInput(object): - - def __init__(self): - self.random_seed = 0 - - def config_test_input_fn(self): - self.random_seed = ops.get_default_graph().seed - return constant_op.constant([[1.]]), constant_op.constant([1.]) - - config = run_config.RunConfig(tf_random_seed=test_random_seed) - test_input = TestInput() - est = estimator.Estimator(model_fn=linear_model_fn, config=config) - est.fit(input_fn=test_input.config_test_input_fn, steps=1) - # If input_fn ran, it will have given us the random seed set on the graph. - self.assertEquals(test_random_seed, test_input.random_seed) - - def testRunConfigModelDir(self): - config = run_config.RunConfig(model_dir='test_dir') - est = estimator.Estimator(model_fn=linear_model_fn, config=config) - self.assertEqual('test_dir', est.config.model_dir) - self.assertEqual('test_dir', est.model_dir) - - def testModelDirAndRunConfigModelDir(self): - config = run_config.RunConfig(model_dir='test_dir') - est = estimator.Estimator( - model_fn=linear_model_fn, config=config, model_dir='test_dir') - self.assertEqual('test_dir', est.config.model_dir) - - with self.assertRaisesRegexp( - ValueError, 'model_dir are set both in constructor and RunConfig, ' - 'but with different'): - estimator.Estimator( - model_fn=linear_model_fn, config=config, model_dir='different_dir') - - def testModelDirIsCopiedToRunConfig(self): - config = run_config.RunConfig() - self.assertIsNone(config.model_dir) - - est = estimator.Estimator( - model_fn=linear_model_fn, model_dir='test_dir', config=config) - self.assertEqual('test_dir', est.config.model_dir) - self.assertEqual('test_dir', est.model_dir) - - def testModelDirAsTempDir(self): - with test.mock.patch.object(tempfile, 'mkdtemp', return_value='temp_dir'): - est = estimator.Estimator(model_fn=linear_model_fn) - self.assertEqual('temp_dir', est.config.model_dir) - self.assertEqual('temp_dir', est.model_dir) - - def testCheckInputs(self): - est = estimator.SKCompat(estimator.Estimator(model_fn=linear_model_fn)) - # Lambdas so we have to different objects to compare - right_features = lambda: np.ones(shape=[7, 8], dtype=np.float32) - right_labels = lambda: np.ones(shape=[7, 10], dtype=np.int32) - est.fit(right_features(), right_labels(), steps=1) - # TODO(wicke): This does not fail for np.int32 because of data_feeder magic. - wrong_type_features = np.ones(shape=[7, 8], dtype=np.int64) - wrong_size_features = np.ones(shape=[7, 10]) - wrong_type_labels = np.ones(shape=[7, 10], dtype=np.float32) - wrong_size_labels = np.ones(shape=[7, 11]) - est.fit(x=right_features(), y=right_labels(), steps=1) - with self.assertRaises(ValueError): - est.fit(x=wrong_type_features, y=right_labels(), steps=1) - with self.assertRaises(ValueError): - est.fit(x=wrong_size_features, y=right_labels(), steps=1) - with self.assertRaises(ValueError): - est.fit(x=right_features(), y=wrong_type_labels, steps=1) - with self.assertRaises(ValueError): - est.fit(x=right_features(), y=wrong_size_labels, steps=1) - - def testBadInput(self): - est = estimator.Estimator(model_fn=linear_model_fn) - self.assertRaisesRegexp( - ValueError, - 'Either x or input_fn must be provided.', - est.fit, - x=None, - input_fn=None, - steps=1) - self.assertRaisesRegexp( - ValueError, - 'Can not provide both input_fn and x or y', - est.fit, - x='X', - input_fn=iris_input_fn, - steps=1) - self.assertRaisesRegexp( - ValueError, - 'Can not provide both input_fn and x or y', - est.fit, - y='Y', - input_fn=iris_input_fn, - steps=1) - self.assertRaisesRegexp( - ValueError, - 'Can not provide both input_fn and batch_size', - est.fit, - input_fn=iris_input_fn, - batch_size=100, - steps=1) - self.assertRaisesRegexp( - ValueError, - 'Inputs cannot be tensors. Please provide input_fn.', - est.fit, - x=constant_op.constant(1.), - steps=1) - - def testUntrained(self): - boston = base.load_boston() - est = estimator.SKCompat(estimator.Estimator(model_fn=linear_model_fn)) - with self.assertRaises(learn.NotFittedError): - _ = est.score(x=boston.data, y=boston.target.astype(np.float64)) - with self.assertRaises(learn.NotFittedError): - est.predict(x=boston.data) - - def testContinueTraining(self): - boston = base.load_boston() - output_dir = tempfile.mkdtemp() - est = estimator.SKCompat( - estimator.Estimator(model_fn=linear_model_fn, model_dir=output_dir)) - float64_labels = boston.target.astype(np.float64) - est.fit(x=boston.data, y=float64_labels, steps=50) - scores = est.score( - x=boston.data, - y=float64_labels, - metrics={ - 'MSE': metric_ops.streaming_mean_squared_error - }) - del est - # Create another estimator object with the same output dir. - est2 = estimator.SKCompat( - estimator.Estimator(model_fn=linear_model_fn, model_dir=output_dir)) - - # Check we can evaluate and predict. - scores2 = est2.score( - x=boston.data, - y=float64_labels, - metrics={ - 'MSE': metric_ops.streaming_mean_squared_error - }) - self.assertAllClose(scores['MSE'], scores2['MSE']) - predictions = np.array(list(est2.predict(x=boston.data))) - other_score = _sklearn.mean_squared_error(predictions, float64_labels) - self.assertAllClose(scores['MSE'], other_score) - - # Check we can keep training. - est2.fit(x=boston.data, y=float64_labels, steps=100) - scores3 = est2.score( - x=boston.data, - y=float64_labels, - metrics={ - 'MSE': metric_ops.streaming_mean_squared_error - }) - self.assertLess(scores3['MSE'], scores['MSE']) - - def test_checkpoint_contains_relative_paths(self): - tmpdir = tempfile.mkdtemp() - est = estimator.Estimator( - model_dir=tmpdir, model_fn=linear_model_fn_with_model_fn_ops) - est.fit(input_fn=boston_input_fn, steps=5) - - checkpoint_file_content = file_io.read_file_to_string( - os.path.join(tmpdir, 'checkpoint')) - ckpt = checkpoint_state_pb2.CheckpointState() - text_format.Merge(checkpoint_file_content, ckpt) - self.assertEqual(ckpt.model_checkpoint_path, 'model.ckpt-5') - # TODO(b/78461127): Please modify tests to not directly rely on names of - # checkpoints. - self.assertAllEqual(['model.ckpt-0', 'model.ckpt-5'], - ckpt.all_model_checkpoint_paths) - - def test_train_save_copy_reload(self): - tmpdir = tempfile.mkdtemp() - model_dir1 = os.path.join(tmpdir, 'model_dir1') - est1 = estimator.Estimator( - model_dir=model_dir1, model_fn=linear_model_fn_with_model_fn_ops) - est1.fit(input_fn=boston_input_fn, steps=5) - - model_dir2 = os.path.join(tmpdir, 'model_dir2') - os.renames(model_dir1, model_dir2) - est2 = estimator.Estimator( - model_dir=model_dir2, model_fn=linear_model_fn_with_model_fn_ops) - self.assertEqual(5, est2.get_variable_value('global_step')) - est2.fit(input_fn=boston_input_fn, steps=5) - self.assertEqual(10, est2.get_variable_value('global_step')) - - def testEstimatorParams(self): - boston = base.load_boston() - est = estimator.SKCompat( - estimator.Estimator( - model_fn=linear_model_params_fn, params={ - 'learning_rate': 0.01 - })) - est.fit(x=boston.data, y=boston.target, steps=100) - - def testHooksNotChanged(self): - est = estimator.Estimator(model_fn=logistic_model_no_mode_fn) - # We pass empty array and expect it to remain empty after calling - # fit and evaluate. Requires inside to copy this array if any hooks were - # added. - my_array = [] - est.fit(input_fn=iris_input_fn, steps=100, monitors=my_array) - _ = est.evaluate(input_fn=iris_input_fn, steps=1, hooks=my_array) - self.assertEqual(my_array, []) - - def testIrisIterator(self): - iris = base.load_iris() - est = estimator.Estimator(model_fn=logistic_model_no_mode_fn) - x_iter = itertools.islice(iris.data, 100) - y_iter = itertools.islice(iris.target, 100) - estimator.SKCompat(est).fit(x_iter, y_iter, steps=20) - eval_result = est.evaluate(input_fn=iris_input_fn, steps=1) - x_iter_eval = itertools.islice(iris.data, 100) - y_iter_eval = itertools.islice(iris.target, 100) - score_result = estimator.SKCompat(est).score(x_iter_eval, y_iter_eval) - print(score_result) - self.assertItemsEqual(eval_result.keys(), score_result.keys()) - self.assertItemsEqual(['global_step', 'loss'], score_result.keys()) - predictions = estimator.SKCompat(est).predict(x=iris.data)['class'] - self.assertEqual(len(predictions), iris.target.shape[0]) - - def testIrisIteratorArray(self): - iris = base.load_iris() - est = estimator.Estimator(model_fn=logistic_model_no_mode_fn) - x_iter = itertools.islice(iris.data, 100) - y_iter = (np.array(x) for x in iris.target) - est.fit(x_iter, y_iter, steps=100) - _ = est.evaluate(input_fn=iris_input_fn, steps=1) - _ = six.next(est.predict(x=iris.data))['class'] - - def testIrisIteratorPlainInt(self): - iris = base.load_iris() - est = estimator.Estimator(model_fn=logistic_model_no_mode_fn) - x_iter = itertools.islice(iris.data, 100) - y_iter = (v for v in iris.target) - est.fit(x_iter, y_iter, steps=100) - _ = est.evaluate(input_fn=iris_input_fn, steps=1) - _ = six.next(est.predict(x=iris.data))['class'] - - def testIrisTruncatedIterator(self): - iris = base.load_iris() - est = estimator.Estimator(model_fn=logistic_model_no_mode_fn) - x_iter = itertools.islice(iris.data, 50) - y_iter = ([np.int32(v)] for v in iris.target) - est.fit(x_iter, y_iter, steps=100) - - def testTrainStepsIsIncremental(self): - est = estimator.Estimator(model_fn=linear_model_fn) - est.fit(input_fn=boston_input_fn, steps=10) - self.assertEqual(10, est.get_variable_value('global_step')) - est.fit(input_fn=boston_input_fn, steps=15) - self.assertEqual(25, est.get_variable_value('global_step')) - - def testTrainMaxStepsIsNotIncremental(self): - est = estimator.Estimator(model_fn=linear_model_fn) - est.fit(input_fn=boston_input_fn, max_steps=10) - self.assertEqual(10, est.get_variable_value('global_step')) - est.fit(input_fn=boston_input_fn, max_steps=15) - self.assertEqual(15, est.get_variable_value('global_step')) - - def testPredict(self): - est = estimator.Estimator(model_fn=linear_model_fn) - boston = base.load_boston() - est.fit(input_fn=boston_input_fn, steps=1) - output = list(est.predict(x=boston.data, batch_size=10)) - self.assertEqual(len(output), boston.target.shape[0]) - - def testWithModelFnOps(self): - """Test for model_fn that returns `ModelFnOps`.""" - est = estimator.Estimator(model_fn=linear_model_fn_with_model_fn_ops) - boston = base.load_boston() - est.fit(input_fn=boston_input_fn, steps=1) - input_fn = functools.partial(boston_input_fn, num_epochs=1) - scores = est.evaluate(input_fn=input_fn, steps=1) - self.assertIn('loss', scores.keys()) - output = list(est.predict(input_fn=input_fn)) - self.assertEqual(len(output), boston.target.shape[0]) - - def testWrongInput(self): - - def other_input_fn(): - return { - 'other': constant_op.constant([0, 0, 0]) - }, constant_op.constant([0, 0, 0]) - - est = estimator.Estimator(model_fn=linear_model_fn) - est.fit(input_fn=boston_input_fn, steps=1) - with self.assertRaises(ValueError): - est.fit(input_fn=other_input_fn, steps=1) - - def testMonitorsForFit(self): - est = estimator.Estimator(model_fn=linear_model_fn) - est.fit( - input_fn=boston_input_fn, - steps=21, - monitors=[CheckCallsMonitor(expect_calls=21)]) - - def testHooksForEvaluate(self): - - class CheckCallHook(session_run_hook.SessionRunHook): - - def __init__(self): - self.run_count = 0 - - def after_run(self, run_context, run_values): - self.run_count += 1 - - est = learn.Estimator(model_fn=linear_model_fn) - est.fit(input_fn=boston_input_fn, steps=1) - hook = CheckCallHook() - est.evaluate(input_fn=boston_eval_fn, steps=3, hooks=[hook]) - - self.assertEqual(3, hook.run_count) - - def testSummaryWriting(self): - est = estimator.Estimator(model_fn=linear_model_fn) - est.fit(input_fn=boston_input_fn, steps=200) - est.evaluate(input_fn=boston_input_fn, steps=200) - loss_summary = util_test.simple_values_from_events( - util_test.latest_events(est.model_dir), ['OptimizeLoss/loss']) - self.assertEqual(1, len(loss_summary)) - - def testSummaryWritingWithSummaryProto(self): - - def _streaming_mean_squared_error_histogram(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - metrics, update_ops = metric_ops.streaming_mean_squared_error( - predictions, - labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - return summary.histogram('histogram', metrics), update_ops - - est = estimator.Estimator(model_fn=linear_model_fn) - est.fit(input_fn=boston_input_fn, steps=200) - est.evaluate( - input_fn=boston_input_fn, - steps=200, - metrics={ - 'MSE': _streaming_mean_squared_error_histogram - }) - events = util_test.latest_events(est.model_dir + '/eval') - output_values = {} - for e in events: - if e.HasField('summary'): - for v in e.summary.value: - output_values[v.tag] = v - self.assertTrue('MSE' in output_values) - self.assertTrue(output_values['MSE'].HasField('histo')) - - def testSummaryWritingWithTensor(self): - - def _streaming_precition_mean_tensor(predictions, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - return metric_ops.streaming_mean_tensor( - predictions, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - est = estimator.Estimator(model_fn=linear_model_fn) - est.fit(input_fn=boston_input_fn, steps=200) - est.evaluate( - input_fn=boston_input_fn, - steps=200, - metrics={ - 'PMT': _streaming_precition_mean_tensor - }) - events = util_test.latest_events(est.model_dir + '/eval') - output_values = {} - for e in events: - if e.HasField('summary'): - for v in e.summary.value: - output_values[v.tag] = v - self.assertTrue('PMT' in output_values) - self.assertTrue(output_values['PMT'].HasField('tensor')) - - def testLossInGraphCollection(self): - - class _LossCheckerHook(session_run_hook.SessionRunHook): - - def begin(self): - self.loss_collection = ops.get_collection(ops.GraphKeys.LOSSES) - - hook = _LossCheckerHook() - est = estimator.Estimator(model_fn=linear_model_fn) - est.fit(input_fn=boston_input_fn, steps=200, monitors=[hook]) - self.assertTrue(hook.loss_collection) - - def test_export_returns_exported_dirname(self): - expected = '/path/to/some_dir' - with test.mock.patch.object(estimator, 'export') as mock_export_module: - mock_export_module._export_estimator.return_value = expected - - est = estimator.Estimator(model_fn=linear_model_fn) - actual = est.export('/path/to') - - self.assertEquals(expected, actual) - - def test_export_savedmodel(self): - tmpdir = tempfile.mkdtemp() - est, serving_input_fn = _build_estimator_for_export_tests(tmpdir) - - extra_file_name = os.path.join( - compat.as_bytes(tmpdir), compat.as_bytes('my_extra_file')) - extra_file = gfile.GFile(extra_file_name, mode='w') - extra_file.write(EXTRA_FILE_CONTENT) - extra_file.close() - assets_extra = {'some/sub/directory/my_extra_file': extra_file_name} - - export_dir_base = os.path.join( - compat.as_bytes(tmpdir), compat.as_bytes('export')) - export_dir = est.export_savedmodel( - export_dir_base, serving_input_fn, assets_extra=assets_extra) - - self.assertTrue(gfile.Exists(export_dir_base)) - self.assertTrue(gfile.Exists(export_dir)) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('saved_model.pb')))) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), compat.as_bytes('variables')))) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('variables/variables.index')))) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('variables/variables.data-00000-of-00001')))) - - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), compat.as_bytes('assets')))) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('assets/my_vocab_file')))) - self.assertEqual( - compat.as_bytes(VOCAB_FILE_CONTENT), - compat.as_bytes( - gfile.GFile( - os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('assets/my_vocab_file'))).read())) - - expected_extra_path = os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('assets.extra/some/sub/directory/my_extra_file')) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), compat.as_bytes('assets.extra')))) - self.assertTrue(gfile.Exists(expected_extra_path)) - self.assertEqual( - compat.as_bytes(EXTRA_FILE_CONTENT), - compat.as_bytes(gfile.GFile(expected_extra_path).read())) - - expected_vocab_file = os.path.join( - compat.as_bytes(tmpdir), compat.as_bytes('my_vocab_file')) - # Restore, to validate that the export was well-formed. - with ops.Graph().as_default() as graph: - with session_lib.Session(graph=graph) as sess: - loader.load(sess, [tag_constants.SERVING], export_dir) - assets = [ - x.eval() - for x in graph.get_collection(ops.GraphKeys.ASSET_FILEPATHS) - ] - self.assertItemsEqual([expected_vocab_file], assets) - graph_ops = [x.name for x in graph.get_operations()] - self.assertTrue('input_example_tensor' in graph_ops) - self.assertTrue('ParseExample/ParseExample' in graph_ops) - self.assertTrue('linear/linear/feature/matmul' in graph_ops) - self.assertItemsEqual(['bogus_lookup', 'feature'], [ - compat.as_str_any(x) - for x in graph.get_collection( - constants.COLLECTION_DEF_KEY_FOR_INPUT_FEATURE_KEYS) - ]) - - # cleanup - gfile.DeleteRecursively(tmpdir) - - def test_export_savedmodel_with_resource(self): - tmpdir = tempfile.mkdtemp() - est, serving_input_fn = _build_estimator_for_resource_export_test() - - export_dir_base = os.path.join( - compat.as_bytes(tmpdir), compat.as_bytes('export')) - export_dir = est.export_savedmodel(export_dir_base, serving_input_fn) - - self.assertTrue(gfile.Exists(export_dir_base)) - self.assertTrue(gfile.Exists(export_dir)) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('saved_model.pb')))) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), compat.as_bytes('variables')))) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('variables/variables.index')))) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('variables/variables.data-00000-of-00001')))) - - # Restore, to validate that the export was well-formed. - with ops.Graph().as_default() as graph: - with session_lib.Session(graph=graph) as sess: - loader.load(sess, [tag_constants.SERVING], export_dir) - graph_ops = [x.name for x in graph.get_operations()] - self.assertTrue('input_example_tensor' in graph_ops) - self.assertTrue('ParseExample/ParseExample' in graph_ops) - self.assertTrue('LookupTableModel' in graph_ops) - self.assertFalse('LookupTableTrainingState' in graph_ops) - - # cleanup - gfile.DeleteRecursively(tmpdir) - - def test_export_savedmodel_with_graph_transforms(self): - tmpdir = tempfile.mkdtemp() - est, serving_input_fn = _build_estimator_for_export_tests(tmpdir) - - extra_file_name = os.path.join( - compat.as_bytes(tmpdir), compat.as_bytes('my_extra_file')) - extra_file = gfile.GFile(extra_file_name, mode='w') - extra_file.write(EXTRA_FILE_CONTENT) - extra_file.close() - assets_extra = {'some/sub/directory/my_extra_file': extra_file_name} - - export_dir_base = os.path.join( - compat.as_bytes(tmpdir), compat.as_bytes('export')) - export_dir = est.export_savedmodel( - export_dir_base, - serving_input_fn, - assets_extra=assets_extra, - graph_rewrite_specs=[ - estimator.GraphRewriteSpec(['tag_1'], []), - estimator.GraphRewriteSpec(['tag_2', 'tag_3'], - ['strip_unused_nodes']) - ]) - - self.assertTrue(gfile.Exists(export_dir_base)) - self.assertTrue(gfile.Exists(export_dir)) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('saved_model.pb')))) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), compat.as_bytes('variables')))) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('variables/variables.index')))) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('variables/variables.data-00000-of-00001')))) - - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), compat.as_bytes('assets')))) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('assets/my_vocab_file')))) - self.assertEqual( - compat.as_bytes(VOCAB_FILE_CONTENT), - compat.as_bytes( - gfile.GFile( - os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('assets/my_vocab_file'))).read())) - - expected_extra_path = os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes('assets.extra/some/sub/directory/my_extra_file')) - self.assertTrue( - gfile.Exists( - os.path.join( - compat.as_bytes(export_dir), compat.as_bytes('assets.extra')))) - self.assertTrue(gfile.Exists(expected_extra_path)) - self.assertEqual( - compat.as_bytes(EXTRA_FILE_CONTENT), - compat.as_bytes(gfile.GFile(expected_extra_path).read())) - - expected_vocab_file = os.path.join( - compat.as_bytes(tmpdir), compat.as_bytes('my_vocab_file')) - - # Restore, to validate that the export was well-formed. - # tag_1 is untransformed. - tags = ['tag_1'] - with ops.Graph().as_default() as graph: - with session_lib.Session(graph=graph) as sess: - loader.load(sess, tags, export_dir) - assets = [ - x.eval() - for x in graph.get_collection(ops.GraphKeys.ASSET_FILEPATHS) - ] - self.assertItemsEqual([expected_vocab_file], assets) - graph_ops = [x.name for x in graph.get_operations()] - self.assertIn('input_example_tensor', graph_ops) - self.assertIn('ParseExample/ParseExample', graph_ops) - self.assertIn('linear/linear/feature/matmul', graph_ops) - # Since there were no transforms, both save ops are still present. - self.assertIn('save/SaveV2/tensor_names', graph_ops) - self.assertIn('save_1/SaveV2/tensor_names', graph_ops) - # Since there were no transforms, the hash table lookup is still there. - self.assertIn('hash_table_Lookup/LookupTableFindV2', graph_ops) - - # Restore, to validate that the export was well-formed. - # tag_2, tag_3 was subjected to strip_unused_nodes. - tags = ['tag_2', 'tag_3'] - with ops.Graph().as_default() as graph: - with session_lib.Session(graph=graph) as sess: - loader.load(sess, tags, export_dir) - assets = [ - x.eval() - for x in graph.get_collection(ops.GraphKeys.ASSET_FILEPATHS) - ] - self.assertItemsEqual([expected_vocab_file], assets) - graph_ops = [x.name for x in graph.get_operations()] - self.assertTrue('input_example_tensor' in graph_ops) - self.assertTrue('ParseExample/ParseExample' in graph_ops) - self.assertTrue('linear/linear/feature/matmul' in graph_ops) - # The Saver used to restore the checkpoint into the export Session - # was not added to the SAVERS collection, so strip_unused_nodes removes - # it. The one explicitly created in export_savedmodel is tracked in - # the MetaGraphDef saver_def field, so that one is retained. - # TODO(soergel): Make Savers sane again. I understand this is all a bit - # nuts but for now the test demonstrates what actually happens. - self.assertFalse('save/SaveV2/tensor_names' in graph_ops) - self.assertTrue('save_1/SaveV2/tensor_names' in graph_ops) - # The fake hash table lookup wasn't connected to anything; stripped. - self.assertFalse('hash_table_Lookup' in graph_ops) - - # cleanup - gfile.DeleteRecursively(tmpdir) - - -class InferRealValuedColumnsTest(test.TestCase): - - def testInvalidArgs(self): - with self.assertRaisesRegexp(ValueError, 'x or input_fn must be provided'): - estimator.infer_real_valued_columns_from_input(None) - - with self.assertRaisesRegexp(ValueError, 'cannot be tensors'): - estimator.infer_real_valued_columns_from_input(constant_op.constant(1.0)) - - def _assert_single_feature_column(self, expected_shape, expected_dtype, - feature_columns): - self.assertEqual(1, len(feature_columns)) - feature_column = feature_columns[0] - self.assertEqual('', feature_column.name) - self.assertEqual({ - '': - parsing_ops.FixedLenFeature( - shape=expected_shape, dtype=expected_dtype) - }, feature_column.config) - - def testInt32Input(self): - feature_columns = estimator.infer_real_valued_columns_from_input( - np.ones(shape=[7, 8], dtype=np.int32)) - self._assert_single_feature_column([8], dtypes.int32, feature_columns) - - def testInt32InputFn(self): - feature_columns = estimator.infer_real_valued_columns_from_input_fn( - lambda: (array_ops.ones(shape=[7, 8], dtype=dtypes.int32), None)) - self._assert_single_feature_column([8], dtypes.int32, feature_columns) - - def testInt64Input(self): - feature_columns = estimator.infer_real_valued_columns_from_input( - np.ones(shape=[7, 8], dtype=np.int64)) - self._assert_single_feature_column([8], dtypes.int64, feature_columns) - - def testInt64InputFn(self): - feature_columns = estimator.infer_real_valued_columns_from_input_fn( - lambda: (array_ops.ones(shape=[7, 8], dtype=dtypes.int64), None)) - self._assert_single_feature_column([8], dtypes.int64, feature_columns) - - def testFloat32Input(self): - feature_columns = estimator.infer_real_valued_columns_from_input( - np.ones(shape=[7, 8], dtype=np.float32)) - self._assert_single_feature_column([8], dtypes.float32, feature_columns) - - def testFloat32InputFn(self): - feature_columns = estimator.infer_real_valued_columns_from_input_fn( - lambda: (array_ops.ones(shape=[7, 8], dtype=dtypes.float32), None)) - self._assert_single_feature_column([8], dtypes.float32, feature_columns) - - def testFloat64Input(self): - feature_columns = estimator.infer_real_valued_columns_from_input( - np.ones(shape=[7, 8], dtype=np.float64)) - self._assert_single_feature_column([8], dtypes.float64, feature_columns) - - def testFloat64InputFn(self): - feature_columns = estimator.infer_real_valued_columns_from_input_fn( - lambda: (array_ops.ones(shape=[7, 8], dtype=dtypes.float64), None)) - self._assert_single_feature_column([8], dtypes.float64, feature_columns) - - def testBoolInput(self): - with self.assertRaisesRegexp( - ValueError, 'on integer or non floating types are not supported'): - estimator.infer_real_valued_columns_from_input( - np.array([[False for _ in xrange(8)] for _ in xrange(7)])) - - def testBoolInputFn(self): - with self.assertRaisesRegexp( - ValueError, 'on integer or non floating types are not supported'): - # pylint: disable=g-long-lambda - estimator.infer_real_valued_columns_from_input_fn( - lambda: (constant_op.constant(False, shape=[7, 8], dtype=dtypes.bool), None) - ) - - def testStringInput(self): - with self.assertRaisesRegexp( - ValueError, 'on integer or non floating types are not supported'): - # pylint: disable=g-long-lambda - estimator.infer_real_valued_columns_from_input( - np.array([['%d.0' % i for i in xrange(8)] for _ in xrange(7)])) - - def testStringInputFn(self): - with self.assertRaisesRegexp( - ValueError, 'on integer or non floating types are not supported'): - # pylint: disable=g-long-lambda - estimator.infer_real_valued_columns_from_input_fn( - lambda: ( - constant_op.constant([['%d.0' % i - for i in xrange(8)] - for _ in xrange(7)]), - None)) - - def testBostonInputFn(self): - feature_columns = estimator.infer_real_valued_columns_from_input_fn( - boston_input_fn) - self._assert_single_feature_column([_BOSTON_INPUT_DIM], dtypes.float64, - feature_columns) - - def testIrisInputFn(self): - feature_columns = estimator.infer_real_valued_columns_from_input_fn( - iris_input_fn) - self._assert_single_feature_column([_IRIS_INPUT_DIM], dtypes.float64, - feature_columns) - - -class ReplicaDeviceSetterTest(test.TestCase): - - def testVariablesAreOnPs(self): - tf_config = {'cluster': {run_config.TaskType.PS: ['fake_ps_0']}} - with test.mock.patch.dict('os.environ', { - 'TF_CONFIG': json.dumps(tf_config) - }): - config = run_config.RunConfig() - - with ops.device(estimator._get_replica_device_setter(config)): - v = variables_lib.Variable([1, 2]) - w = variables_lib.Variable([2, 1]) - a = v + w - self.assertDeviceEqual('/job:ps/task:0', v.device) - self.assertDeviceEqual('/job:ps/task:0', v.initializer.device) - self.assertDeviceEqual('/job:ps/task:0', w.device) - self.assertDeviceEqual('/job:ps/task:0', w.initializer.device) - self.assertDeviceEqual('/job:worker', a.device) - - def testVariablesAreLocal(self): - with ops.device( - estimator._get_replica_device_setter(run_config.RunConfig())): - v = variables_lib.Variable([1, 2]) - w = variables_lib.Variable([2, 1]) - a = v + w - self.assertDeviceEqual('', v.device) - self.assertDeviceEqual('', v.initializer.device) - self.assertDeviceEqual('', w.device) - self.assertDeviceEqual('', w.initializer.device) - self.assertDeviceEqual('', a.device) - - def testMutableHashTableIsOnPs(self): - tf_config = {'cluster': {run_config.TaskType.PS: ['fake_ps_0']}} - with test.mock.patch.dict('os.environ', { - 'TF_CONFIG': json.dumps(tf_config) - }): - config = run_config.RunConfig() - - with ops.device(estimator._get_replica_device_setter(config)): - default_val = constant_op.constant([-1, -1], dtypes.int64) - table = lookup.MutableHashTable(dtypes.string, dtypes.int64, default_val) - input_string = constant_op.constant(['brain', 'salad', 'tank']) - output = table.lookup(input_string) - self.assertDeviceEqual('/job:ps/task:0', table.resource_handle.device) - self.assertDeviceEqual('/job:ps/task:0', output.device) - - def testMutableHashTableIsLocal(self): - with ops.device( - estimator._get_replica_device_setter(run_config.RunConfig())): - default_val = constant_op.constant([-1, -1], dtypes.int64) - table = lookup.MutableHashTable(dtypes.string, dtypes.int64, default_val) - input_string = constant_op.constant(['brain', 'salad', 'tank']) - output = table.lookup(input_string) - self.assertDeviceEqual('', table.resource_handle.device) - self.assertDeviceEqual('', output.device) - - def testTaskIsSetOnWorkerWhenJobNameIsSet(self): - tf_config = { - 'cluster': { - run_config.TaskType.PS: ['fake_ps_0'] - }, - 'task': { - 'type': run_config.TaskType.WORKER, - 'index': 3 - } - } - with test.mock.patch.dict('os.environ', { - 'TF_CONFIG': json.dumps(tf_config) - }): - config = run_config.RunConfig() - - with ops.device(estimator._get_replica_device_setter(config)): - v = variables_lib.Variable([1, 2]) - w = variables_lib.Variable([2, 1]) - a = v + w - self.assertDeviceEqual('/job:ps/task:0', v.device) - self.assertDeviceEqual('/job:ps/task:0', v.initializer.device) - self.assertDeviceEqual('/job:ps/task:0', w.device) - self.assertDeviceEqual('/job:ps/task:0', w.initializer.device) - self.assertDeviceEqual('/job:worker/task:3', a.device) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator_test_utils.py b/tensorflow/contrib/learn/python/learn/estimators/estimator_test_utils.py deleted file mode 100644 index e4c31396baf..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/estimator_test_utils.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utils for Estimator (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.util import tf_inspect - - -def assert_estimator_contract(tester, estimator_class): - """Asserts whether given estimator satisfies the expected contract. - - This doesn't check every details of contract. This test is used for that a - function is not forgotten to implement in a precanned Estimator. - - Args: - tester: A tf.test.TestCase. - estimator_class: 'type' object of pre-canned estimator. - """ - attributes = tf_inspect.getmembers(estimator_class) - attribute_names = [a[0] for a in attributes] - - tester.assertTrue('config' in attribute_names) - tester.assertTrue('evaluate' in attribute_names) - tester.assertTrue('export' in attribute_names) - tester.assertTrue('fit' in attribute_names) - tester.assertTrue('get_variable_names' in attribute_names) - tester.assertTrue('get_variable_value' in attribute_names) - tester.assertTrue('model_dir' in attribute_names) - tester.assertTrue('predict' in attribute_names) - - -def assert_in_range(min_value, max_value, key, metrics): - actual_value = metrics[key] - if actual_value < min_value: - raise ValueError('%s: %s < %s.' % (key, actual_value, min_value)) - if actual_value > max_value: - raise ValueError('%s: %s > %s.' % (key, actual_value, max_value)) diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimators_test.py b/tensorflow/contrib/learn/python/learn/estimators/estimators_test.py deleted file mode 100644 index 2113fae3940..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/estimators_test.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Custom optimizer tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random - -import numpy as np - -from tensorflow.python.training import training_util -from tensorflow.contrib.learn.python import learn -from tensorflow.contrib.learn.python.learn import datasets -from tensorflow.contrib.learn.python.learn import metric_spec -from tensorflow.contrib.learn.python.learn.estimators import estimator as estimator_lib -from tensorflow.contrib.learn.python.learn.estimators._sklearn import accuracy_score -from tensorflow.contrib.learn.python.learn.estimators._sklearn import train_test_split -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import string_ops -from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.platform import test -from tensorflow.python.training import momentum as momentum_lib - - -class FeatureEngineeringFunctionTest(test.TestCase): - """Tests feature_engineering_fn.""" - - def testFeatureEngineeringFn(self): - - def input_fn(): - return { - "x": constant_op.constant([1.]) - }, { - "y": constant_op.constant([11.]) - } - - def feature_engineering_fn(features, labels): - _, _ = features, labels - return { - "transformed_x": constant_op.constant([9.]) - }, { - "transformed_y": constant_op.constant([99.]) - } - - def model_fn(features, labels): - # dummy variable: - _ = variables_lib.Variable([0.]) - _ = labels - predictions = features["transformed_x"] - loss = constant_op.constant([2.]) - update_global_step = training_util.get_global_step().assign_add(1) - return predictions, loss, update_global_step - - estimator = estimator_lib.Estimator( - model_fn=model_fn, feature_engineering_fn=feature_engineering_fn) - estimator.fit(input_fn=input_fn, steps=1) - prediction = next(estimator.predict(input_fn=input_fn, as_iterable=True)) - # predictions = transformed_x (9) - self.assertEqual(9., prediction) - metrics = estimator.evaluate( - input_fn=input_fn, - steps=1, - metrics={ - "label": metric_spec.MetricSpec(lambda predictions, labels: labels) - }) - # labels = transformed_y (99) - self.assertEqual(99., metrics["label"]) - - def testFeatureEngineeringFnWithSameName(self): - - def input_fn(): - return { - "x": constant_op.constant(["9."]) - }, { - "y": constant_op.constant(["99."]) - } - - def feature_engineering_fn(features, labels): - # Github #12205: raise a TypeError if called twice. - _ = string_ops.string_split(features["x"]) - features["x"] = constant_op.constant([9.]) - labels["y"] = constant_op.constant([99.]) - return features, labels - - def model_fn(features, labels): - # dummy variable: - _ = variables_lib.Variable([0.]) - _ = labels - predictions = features["x"] - loss = constant_op.constant([2.]) - update_global_step = training_util.get_global_step().assign_add(1) - return predictions, loss, update_global_step - - estimator = estimator_lib.Estimator( - model_fn=model_fn, feature_engineering_fn=feature_engineering_fn) - estimator.fit(input_fn=input_fn, steps=1) - prediction = next(estimator.predict(input_fn=input_fn, as_iterable=True)) - # predictions = transformed_x (9) - self.assertEqual(9., prediction) - metrics = estimator.evaluate( - input_fn=input_fn, - steps=1, - metrics={ - "label": metric_spec.MetricSpec(lambda predictions, labels: labels) - }) - # labels = transformed_y (99) - self.assertEqual(99., metrics["label"]) - - def testNoneFeatureEngineeringFn(self): - - def input_fn(): - return { - "x": constant_op.constant([1.]) - }, { - "y": constant_op.constant([11.]) - } - - def feature_engineering_fn(features, labels): - _, _ = features, labels - return { - "x": constant_op.constant([9.]) - }, { - "y": constant_op.constant([99.]) - } - - def model_fn(features, labels): - # dummy variable: - _ = variables_lib.Variable([0.]) - _ = labels - predictions = features["x"] - loss = constant_op.constant([2.]) - update_global_step = training_util.get_global_step().assign_add(1) - return predictions, loss, update_global_step - - estimator_with_fe_fn = estimator_lib.Estimator( - model_fn=model_fn, feature_engineering_fn=feature_engineering_fn) - estimator_with_fe_fn.fit(input_fn=input_fn, steps=1) - estimator_without_fe_fn = estimator_lib.Estimator(model_fn=model_fn) - estimator_without_fe_fn.fit(input_fn=input_fn, steps=1) - - # predictions = x - prediction_with_fe_fn = next( - estimator_with_fe_fn.predict(input_fn=input_fn, as_iterable=True)) - self.assertEqual(9., prediction_with_fe_fn) - prediction_without_fe_fn = next( - estimator_without_fe_fn.predict(input_fn=input_fn, as_iterable=True)) - self.assertEqual(1., prediction_without_fe_fn) - - -class CustomOptimizer(test.TestCase): - """Custom optimizer tests.""" - - def testIrisMomentum(self): - random.seed(42) - - iris = datasets.load_iris() - x_train, x_test, y_train, y_test = train_test_split( - iris.data, iris.target, test_size=0.2, random_state=42) - - def custom_optimizer(): - return momentum_lib.MomentumOptimizer(learning_rate=0.01, momentum=0.9) - - classifier = learn.DNNClassifier( - hidden_units=[10, 20, 10], - feature_columns=learn.infer_real_valued_columns_from_input(x_train), - n_classes=3, - optimizer=custom_optimizer, - config=learn.RunConfig(tf_random_seed=1)) - classifier.fit(x_train, y_train, steps=400) - predictions = np.array(list(classifier.predict_classes(x_test))) - score = accuracy_score(y_test, predictions) - - self.assertGreater(score, 0.65, "Failed with score = {0}".format(score)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/g3doc/svm.md b/tensorflow/contrib/learn/python/learn/estimators/g3doc/svm.md deleted file mode 100644 index 238ac075c50..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/g3doc/svm.md +++ /dev/null @@ -1,30 +0,0 @@ -# Support Vector Machines (SVMs) - -Support Vector Machines (SVMs) is a class of supervised machine learning -algorithms that scale well in high dimensional spaces. SVMs search for the -boundary (hyperplane for linear SVM classifier) that maximizes the distance to -the nearest training data point of any class (i.e., the **margin**). - -![svm](svm.png) - -The [**tf.contrib.learn.SVM**] -(https://www.tensorflow.org/code/tensorflow/contrib/learn/python/learn/estimators/svm.py) -Estimator currently implements linear SVMs for binary classification and uses -the hinge loss. Note that tf.contrib.learn.SVM is a **soft margin** classifier, -that is, it returns a linear boundary even when the training data is not -linearly separable. More specifically, tf.contrib.learn.SVM finds the optimal -hyperplane by solving the following optimization problem: - -$$ -\min_{{\bf w}, {\bf \xi}} \quad \frac{1}{N}\sum_{i=1}^N \xi_i -+ \frac{\lambda}{2}\|{\bf w}\|^2 \\ -\textrm{subject to: } -\forall i, \, \xi_i \ge 0, \, \xi_i \ge 1 - y_i {\bf w}^T {\bf x}_i -$$ - -For the solution of the optimization problem, the Stochastic Dual Coordinate -Ascent ([SDCA](https://www.tensorflow.org/code/tensorflow/contrib/linear_optimizer/kernels/g3doc/readme.md)) -algorithm is currently being used as the default optimizer. tf.contrib.learn.SVM -supports (global) $$L1$$ and $$L2$$ regularization and also works with weighted -examples. The implementation works in both single-machine and distributed -settings. diff --git a/tensorflow/contrib/learn/python/learn/estimators/g3doc/svm.png b/tensorflow/contrib/learn/python/learn/estimators/g3doc/svm.png deleted file mode 100644 index ebabafa2aab..00000000000 Binary files a/tensorflow/contrib/learn/python/learn/estimators/g3doc/svm.png and /dev/null differ diff --git a/tensorflow/contrib/learn/python/learn/estimators/head.py b/tensorflow/contrib/learn/python/learn/estimators/head.py deleted file mode 100644 index 0dd835f8fb5..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/head.py +++ /dev/null @@ -1,2142 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Abstractions for the head(s) of a model (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - -from tensorflow.contrib import framework as framework_lib -from tensorflow.contrib import layers as layers_lib -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.contrib.learn.python.learn.estimators.metric_key import MetricKey as mkey -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import metrics as metrics_lib -from tensorflow.python.ops import nn -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import string_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import weights_broadcast_ops -from tensorflow.python.ops.losses import losses as losses_lib -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.summary import summary -from tensorflow.python.training import training -from tensorflow.python.util import tf_decorator -from tensorflow.python.util import tf_inspect -from tensorflow.python.util.deprecation import deprecated - - -@six.add_metaclass(abc.ABCMeta) -class Head(object): - """Interface for the head/top of a model. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Given logits (or output of a hidden layer), a Head knows how to compute - predictions, loss, default metric and export signature. It is meant to, - - 1) Simplify writing model_fn and to make model_fn more configurable - 2) Support wide range of machine learning models. Since most heads can work - with logits, they can support DNN, RNN, Wide, Wide&Deep, - Global objectives, Gradient boosted trees and many other types - of machine learning models. - 2) To allow users to seamlessly switch between 1 to n heads for multi - objective learning (See _MultiHead implementation for more details) - - Common usage: - Here is simplified model_fn to build a multiclass DNN model. - ```python - def _my_dnn_model_fn(features, labels, mode, params, config=None): - # Optionally your callers can pass head to model_fn as a param. - head = tf.contrib.learn.multi_class_head(...) - input = tf.contrib.layers.input_from_feature_columns(features, ...) - last_hidden_layer_out = tf.contrib.layers.stack( - input, tf.contrib.layers.fully_connected, [1000, 500]) - logits = tf.contrib.layers.fully_connected( - last_hidden_layer_out, head.logits_dimension, activation_fn=None) - - def _train_op_fn(loss): - return optimizer.minimize(loss) - - return head.create_model_fn_ops( - features=features, - labels=labels, - mode=mode, - train_op_fn=_train_op_fn, - logits=logits, - scope=...) - ``` - - Most heads also support logits_input which is typically the output of the last - hidden layer. Some heads (like heads responsible for candidate sampling or - hierarchical softmax) intrinsically will not support logits and you have - to pass logits_input. Here is a common usage, - ```python - return head.create_model_fn_ops( - features=features, - labels=labels, - mode=mode, - train_op_fn=_train_op_fn, - logits_input=last_hidden_layer_out, - scope=...) - ```python - - There are cases where computing and applying gradients can not be meaningfully - captured with train_op_fn we support (for example, with sync optimizer). In - such case, you can take the responsibility on your own. Here is a common - use case, - ```python - model_fn_ops = head.create_model_fn_ops( - features=features, - labels=labels, - mode=mode, - train_op_fn=tf.contrib.learn.no_op_train_fn, - logits=logits, - scope=...) - if mode == tf.contrib.learn.ModeKeys.TRAIN: - optimizer = ... - sync = tf.compat.v1.train.SyncReplicasOptimizer(opt=optimizer, ...) - update_op = tf.contrib.layers.optimize_loss(optimizer=sync, - loss=model_fn_ops.loss, ...) - hooks = [sync.make_session_run_hook(is_chief)] - ... update train_op and hooks in ModelFnOps and return - ``` - """ - - @abc.abstractproperty - def logits_dimension(self): - """Size of the last dimension of the logits `Tensor`. - - Typically, logits is of shape `[batch_size, logits_dimension]`. - - Returns: - The expected size of the `logits` tensor. - """ - raise NotImplementedError("Calling an abstract method.") - - @abc.abstractmethod - def create_model_fn_ops(self, - features, - mode, - labels=None, - train_op_fn=None, - logits=None, - logits_input=None, - scope=None): - """Returns `ModelFnOps` that a model_fn can return. - - Please note that, - + Exactly one of `logits` and `logits_input` must be provided. - + All args must be passed via name. - - Args: - features: Input `dict` of `Tensor` objects. - mode: Estimator's `ModeKeys`. - labels: Labels `Tensor`, or `dict` of same. - train_op_fn: Function that takes a scalar loss `Tensor` and returns an op - to optimize the model with the loss. This is used in TRAIN mode and - must not be None. None is allowed in other modes. If you want to - optimize loss yourself you can pass `no_op_train_fn` and then use - ModeFnOps.loss to compute and apply gradients. - logits: logits `Tensor` to be used by the head. - logits_input: `Tensor` from which to build logits, often needed when you - don't want to compute the logits. Typically this is the activation of - the last hidden layer in a DNN. Some heads (like the ones responsible - for candidate sampling) intrinsically avoid computing full logits and - only accepts logits_input. - scope: Optional scope for `variable_scope`. - - Returns: - An instance of `ModelFnOps`. - - Raises: - ValueError: If `mode` is not recognized. - ValueError: If neither or both of `logits` and `logits_input` is provided. - """ - raise NotImplementedError("Calling an abstract method.") - - -@deprecated(None, "Please switch to tf.contrib.estimator.*_head.") -def regression_head(label_name=None, - weight_column_name=None, - label_dimension=1, - enable_centered_bias=False, - head_name=None, - link_fn=None): - """Creates a `Head` for linear regression. - - Args: - label_name: String, name of the key in label dict. Can be null if label - is a tensor (single headed models). - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - label_dimension: Number of regression labels per example. This is the size - of the last dimension of the labels `Tensor` (typically, this has shape - `[batch_size, label_dimension]`). - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - head_name: name of the head. If provided, predictions, summary and metrics - keys will be suffixed by `"/" + head_name` and the default variable scope - will be `head_name`. - link_fn: link function to convert logits to predictions. If provided, - this link function will be used instead of identity. - - Returns: - An instance of `Head` for linear regression. - """ - return _RegressionHead( - label_name=label_name, - weight_column_name=weight_column_name, - label_dimension=label_dimension, - enable_centered_bias=enable_centered_bias, - head_name=head_name, - loss_fn=_mean_squared_loss, - link_fn=(link_fn if link_fn is not None else array_ops.identity)) - - -@deprecated(None, "Please switch to tf.contrib.estimator.*_head.") -def poisson_regression_head(label_name=None, - weight_column_name=None, - label_dimension=1, - enable_centered_bias=False, - head_name=None): - """Creates a `Head` for poisson regression. - - Args: - label_name: String, name of the key in label dict. Can be null if label - is a tensor (single headed models). - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - label_dimension: Number of regression labels per example. This is the size - of the last dimension of the labels `Tensor` (typically, this has shape - `[batch_size, label_dimension]`). - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - head_name: name of the head. If provided, predictions, summary and metrics - keys will be suffixed by `"/" + head_name` and the default variable scope - will be `head_name`. - - Returns: - An instance of `Head` for poisson regression. - """ - return _RegressionHead( - label_name=label_name, - weight_column_name=weight_column_name, - label_dimension=label_dimension, - enable_centered_bias=enable_centered_bias, - head_name=head_name, - loss_fn=_poisson_loss, - link_fn=math_ops.exp) - -# TODO(zakaria): Consider adding a _RegressionHead for logistic_regression - - -@deprecated(None, "Please switch to tf.contrib.estimator.*_head.") -def multi_class_head(n_classes, - label_name=None, - weight_column_name=None, - enable_centered_bias=False, - head_name=None, - thresholds=None, - metric_class_ids=None, - loss_fn=None, - label_keys=None): - """Creates a `Head` for multi class single label classification. - - The Head uses softmax cross entropy loss. - - This head expects to be fed integer labels specifying the class index. But - if `label_keys` is specified, then labels must be strings from this - vocabulary, and the predicted classes will be strings from the same - vocabulary. - - Args: - n_classes: Integer, number of classes, must be >= 2 - label_name: String, name of the key in label dict. Can be null if label - is a tensor (single headed models). - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - head_name: name of the head. If provided, predictions, summary and metrics - keys will be suffixed by `"/" + head_name` and the default variable scope - will be `head_name`. - thresholds: thresholds for eval metrics, defaults to [.5] - metric_class_ids: List of class IDs for which we should report per-class - metrics. Must all be in the range `[0, n_classes)`. Invalid if - `n_classes` is 2. - loss_fn: Optional function that takes (`labels`, `logits`, `weights`) as - parameter and returns a weighted scalar loss. `weights` should be - optional. See `tf.losses` - label_keys: Optional list of strings with size `[n_classes]` defining the - label vocabulary. Only supported for `n_classes` > 2. - - Returns: - An instance of `Head` for multi class classification. - - Raises: - ValueError: if `n_classes` is < 2. - ValueError: If `metric_class_ids` is provided when `n_classes` is 2. - ValueError: If `len(label_keys) != n_classes`. - """ - if (n_classes is None) or (n_classes < 2): - raise ValueError("n_classes must be > 1 for classification: %s." % - n_classes) - if loss_fn: - _verify_loss_fn_args(loss_fn) - - loss_fn = _wrap_custom_loss_fn(loss_fn) if loss_fn else None - if n_classes == 2: - if metric_class_ids: - raise ValueError("metric_class_ids invalid for n_classes==2.") - if label_keys: - raise ValueError("label_keys is not supported for n_classes=2.") - return _BinaryLogisticHead( - label_name=label_name, - weight_column_name=weight_column_name, - enable_centered_bias=enable_centered_bias, - head_name=head_name, - thresholds=thresholds, - loss_fn=loss_fn) - - return _MultiClassHead( - n_classes=n_classes, - label_name=label_name, - weight_column_name=weight_column_name, - enable_centered_bias=enable_centered_bias, - head_name=head_name, - thresholds=thresholds, - metric_class_ids=metric_class_ids, - loss_fn=loss_fn, - label_keys=label_keys) - - -@deprecated(None, "Please switch to tf.contrib.estimator.*_head.") -def binary_svm_head( - label_name=None, - weight_column_name=None, - enable_centered_bias=False, - head_name=None, - thresholds=None,): - """Creates a `Head` for binary classification with SVMs. - - The head uses binary hinge loss. - - Args: - label_name: String, name of the key in label dict. Can be null if label - is a tensor (single headed models). - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - head_name: name of the head. If provided, predictions, summary and metrics - keys will be suffixed by `"/" + head_name` and the default variable scope - will be `head_name`. - thresholds: thresholds for eval metrics, defaults to [.5] - - Returns: - An instance of `Head` for binary classification with SVM. - """ - return _BinarySvmHead( - label_name=label_name, - weight_column_name=weight_column_name, - enable_centered_bias=enable_centered_bias, - head_name=head_name, - thresholds=thresholds) - - -@deprecated(None, "Please switch to tf.contrib.estimator.*_head.") -def multi_label_head(n_classes, - label_name=None, - weight_column_name=None, - enable_centered_bias=False, - head_name=None, - thresholds=None, - metric_class_ids=None, - loss_fn=None): - """Creates a Head for multi label classification. - - Multi-label classification handles the case where each example may have zero - or more associated labels, from a discrete set. This is distinct from - `multi_class_head` which has exactly one label from a discrete set. - - This head by default uses sigmoid cross entropy loss, which expects as input - a multi-hot tensor of shape `(batch_size, num_classes)`. - - Args: - n_classes: Integer, number of classes, must be >= 2 - label_name: String, name of the key in label dict. Can be null if label - is a tensor (single headed models). - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - head_name: name of the head. If provided, predictions, summary and metrics - keys will be suffixed by `"/" + head_name` and the default variable scope - will be `head_name`. - thresholds: thresholds for eval metrics, defaults to [.5] - metric_class_ids: List of class IDs for which we should report per-class - metrics. Must all be in the range `[0, n_classes)`. - loss_fn: Optional function that takes (`labels`, `logits`, `weights`) as - parameter and returns a weighted scalar loss. `weights` should be - optional. See `tf.losses` - - Returns: - An instance of `Head` for multi label classification. - - Raises: - ValueError: If n_classes is < 2 - ValueError: If loss_fn does not have expected signature. - """ - if n_classes < 2: - raise ValueError("n_classes must be > 1 for classification.") - if loss_fn: - _verify_loss_fn_args(loss_fn) - - return _MultiLabelHead( - n_classes=n_classes, - label_name=label_name, - weight_column_name=weight_column_name, - enable_centered_bias=enable_centered_bias, - head_name=head_name, - thresholds=thresholds, - metric_class_ids=metric_class_ids, - loss_fn=_wrap_custom_loss_fn(loss_fn) if loss_fn else None) - - -@deprecated(None, "Please switch to tf.contrib.estimator.*_head.") -def loss_only_head(loss_fn, head_name=None): - """Creates a Head that contains only loss terms. - - Loss only head holds additional loss terms to be added to other heads and - usually represents additional regularization terms in the objective function. - - Args: - loss_fn: a function that takes no argument and returns a list of - scalar tensors. - head_name: a name for the head. - - Returns: - An instance of `Head` to hold the additional losses. - """ - return _LossOnlyHead(loss_fn, head_name=head_name) - - -@deprecated(None, "Please switch to tf.contrib.estimator.*_head.") -def multi_head(heads, loss_weights=None): - """Creates a MultiHead stemming from same logits/hidden layer. - - Args: - heads: list of Head objects. - loss_weights: optional list of weights to be used to merge losses from - each head. All losses are weighted equally if not provided. - - Returns: - A instance of `Head` that merges multiple heads. - - Raises: - ValueError: if heads and loss_weights have different size. - """ - if loss_weights: - if len(loss_weights) != len(heads): - raise ValueError("heads and loss_weights must have same size") - - def _weighted_loss_merger(losses): - if loss_weights: - if len(losses) != len(loss_weights): - raise ValueError("losses and loss_weights must have same size") - weighted_losses = [] - for loss, weight in zip(losses, loss_weights): - weighted_losses.append(math_ops.multiply(loss, weight)) - return math_ops.add_n(weighted_losses) - else: - return math_ops.add_n(losses) - - return _MultiHead(heads, loss_merger=_weighted_loss_merger) - - -@deprecated(None, "Use 'lambda _: tf.no_op()'.") -def no_op_train_fn(loss): - del loss - return control_flow_ops.no_op() - - -class _SingleHead(Head): - """Interface for a single head/top of a model.""" - - def __init__( - self, problem_type, logits_dimension, label_name=None, - weight_column_name=None, head_name=None): - if problem_type is None: - raise ValueError("Invalid problem_type %s." % problem_type) - if logits_dimension is None or logits_dimension < 1: - raise ValueError("Invalid logits_dimension %s." % logits_dimension) - self._problem_type = problem_type - self._logits_dimension = logits_dimension - self._label_name = label_name - self._weight_column_name = weight_column_name - self._head_name = head_name - - @property - def logits_dimension(self): - return self._logits_dimension - - @property - def label_name(self): - return self._label_name - - @property - def weight_column_name(self): - return self._weight_column_name - - @property - def head_name(self): - return self._head_name - - def _create_output_alternatives(self, predictions): - """Creates output alternative for the Head. - - Args: - predictions: a dict of {tensor_name: Tensor}, where 'tensor_name' is a - symbolic name for an output Tensor possibly but not necessarily taken - from `PredictionKey`, and 'Tensor' is the corresponding output Tensor - itself. - - Returns: - `dict` of {submodel_name: (problem_type, {tensor_name: Tensor})}, where - 'submodel_name' is a submodel identifier that should be consistent across - the pipeline (here likely taken from the head_name), - 'problem_type' is a `ProblemType`, - 'tensor_name' is a symbolic name for an output Tensor possibly but not - necessarily taken from `PredictionKey`, and - 'Tensor' is the corresponding output Tensor itself. - """ - return {self._head_name: (self._problem_type, predictions)} - - -# TODO(zakaria): use contrib losses. -def _mean_squared_loss(labels, logits, weights=None): - with ops.name_scope(None, "mean_squared_loss", (logits, labels)) as name: - logits = ops.convert_to_tensor(logits) - labels = ops.convert_to_tensor(labels) - # To prevent broadcasting inside "-". - if len(labels.get_shape()) == 1: - labels = array_ops.expand_dims(labels, axis=1) - # TODO(zakaria): make sure it does not recreate the broadcast bug. - if len(logits.get_shape()) == 1: - logits = array_ops.expand_dims(logits, axis=1) - logits.get_shape().assert_is_compatible_with(labels.get_shape()) - loss = math_ops.squared_difference( - logits, math_ops.cast(labels, dtypes.float32), name=name) - return _compute_weighted_loss(loss, weights) - - -def _poisson_loss(labels, logits, weights=None): - """Computes poisson loss from logits.""" - with ops.name_scope(None, "_poisson_loss", (logits, labels)) as name: - logits = ops.convert_to_tensor(logits) - labels = ops.convert_to_tensor(labels) - # To prevent broadcasting inside "-". - if len(labels.get_shape()) == 1: - labels = array_ops.expand_dims(labels, axis=1) - # TODO(zakaria): make sure it does not recreate the broadcast bug. - if len(logits.get_shape()) == 1: - logits = array_ops.expand_dims(logits, axis=1) - logits.get_shape().assert_is_compatible_with(labels.get_shape()) - loss = nn.log_poisson_loss(labels, logits, compute_full_loss=True, - name=name) - return _compute_weighted_loss(loss, weights) - - -def _logits(logits_input, logits, logits_dimension): - """Validate logits args, and create `logits` if necessary. - - Exactly one of `logits_input` and `logits` must be provided. - - Args: - logits_input: `Tensor` input to `logits`. - logits: `Tensor` output. - logits_dimension: Integer, last dimension of `logits`. This is used to - create `logits` from `logits_input` if `logits` is `None`; otherwise, it's - used to validate `logits`. - - Returns: - `logits` `Tensor`. - - Raises: - ValueError: if neither or both of `logits` and `logits_input` are supplied. - """ - if (logits_dimension is None) or (logits_dimension < 1): - raise ValueError("Invalid logits_dimension %s." % logits_dimension) - - # If not provided, create logits. - if logits is None: - if logits_input is None: - raise ValueError("Neither logits nor logits_input supplied.") - return layers_lib.linear(logits_input, logits_dimension, scope="logits") - - if logits_input is not None: - raise ValueError("Both logits and logits_input supplied.") - - logits = ops.convert_to_tensor(logits, name="logits") - logits_dims = logits.get_shape().dims - if logits_dims is not None: - logits_dims[-1].assert_is_compatible_with(logits_dimension) - - return logits - - -def _create_model_fn_ops(features, - mode, - loss_fn, - logits_to_predictions_fn, - metrics_fn, - create_output_alternatives_fn, - labels=None, - train_op_fn=None, - logits=None, - logits_dimension=None, - head_name=None, - weight_column_name=None, - enable_centered_bias=False): - """Returns a `ModelFnOps` object.""" - _check_mode_valid(mode) - - centered_bias = None - if enable_centered_bias: - centered_bias = _centered_bias(logits_dimension, head_name) - logits = nn.bias_add(logits, centered_bias) - - predictions = logits_to_predictions_fn(logits) - loss = None - train_op = None - eval_metric_ops = None - if (mode != model_fn.ModeKeys.INFER) and (labels is not None): - weight_tensor = _weight_tensor(features, weight_column_name) - loss, weighted_average_loss = loss_fn(labels, logits, weight_tensor) - # The name_scope escapism is needed to maintain the same summary tag - # after switching away from the now unsupported API. - with ops.name_scope(""): - summary_loss = array_ops.identity(weighted_average_loss) - summary.scalar(_summary_key(head_name, mkey.LOSS), summary_loss) - - if mode == model_fn.ModeKeys.TRAIN: - if train_op_fn is None: - raise ValueError("train_op_fn can not be None in TRAIN mode") - batch_size = array_ops.shape(logits)[0] - train_op = _train_op(loss, labels, train_op_fn, centered_bias, - batch_size, loss_fn, weight_tensor) - eval_metric_ops = metrics_fn( - weighted_average_loss, predictions, labels, weight_tensor) - return model_fn.ModelFnOps( - mode=mode, - predictions=predictions, - loss=loss, - train_op=train_op, - eval_metric_ops=eval_metric_ops, - output_alternatives=create_output_alternatives_fn(predictions)) - - -class _RegressionHead(_SingleHead): - """`Head` for regression with a generalized linear model.""" - - def __init__(self, - label_dimension, - loss_fn, - link_fn, - logits_dimension=None, - label_name=None, - weight_column_name=None, - enable_centered_bias=False, - head_name=None): - """`Head` for regression. - - Args: - label_dimension: Number of regression labels per example. This is the - size of the last dimension of the labels `Tensor` (typically, this has - shape `[batch_size, label_dimension]`). - loss_fn: Loss function, takes logits and labels and returns loss. - link_fn: Link function, takes a logits tensor and returns the output. - logits_dimension: Number of logits per example. This is the - size of the last dimension of the logits `Tensor` (typically, this has - shape `[batch_size, label_dimension]`). - Default value: `label_dimension`. - label_name: String, name of the key in label dict. Can be null if label - is a tensor (single headed models). - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - head_name: name of the head. Predictions, summary and metrics keys are - suffixed by `"/" + head_name` and the default variable scope is - `head_name`. - """ - super(_RegressionHead, self).__init__( - problem_type=constants.ProblemType.LINEAR_REGRESSION, - logits_dimension=(logits_dimension if logits_dimension is not None - else label_dimension), - label_name=label_name, - weight_column_name=weight_column_name, - head_name=head_name) - - self._loss_fn = loss_fn - self._link_fn = link_fn - self._enable_centered_bias = enable_centered_bias - - def create_model_fn_ops(self, - features, - mode, - labels=None, - train_op_fn=None, - logits=None, - logits_input=None, - scope=None): - """See `Head`.""" - with variable_scope.variable_scope( - scope, - default_name=self.head_name or "regression_head", - values=(tuple(six.itervalues(features)) + - (labels, logits, logits_input))): - labels = self._transform_labels(mode=mode, labels=labels) - logits = _logits(logits_input, logits, self.logits_dimension) - return _create_model_fn_ops( - features=features, - mode=mode, - loss_fn=self._loss_fn, - logits_to_predictions_fn=self._logits_to_predictions, - metrics_fn=self._metrics, - create_output_alternatives_fn=self._create_output_alternatives, - labels=labels, - train_op_fn=train_op_fn, - logits=logits, - logits_dimension=self.logits_dimension, - head_name=self.head_name, - weight_column_name=self.weight_column_name, - enable_centered_bias=self._enable_centered_bias) - - def _transform_labels(self, mode, labels): - """Applies transformations to labels tensor.""" - if (mode == model_fn.ModeKeys.INFER) or (labels is None): - return None - labels_tensor = _to_labels_tensor(labels, self._label_name) - _check_no_sparse_tensor(labels_tensor) - return labels_tensor - - def _logits_to_predictions(self, logits): - """Returns a dict of predictions. - - Args: - logits: logits `Tensor` after applying possible centered bias. - - Returns: - Dict of prediction `Tensor` keyed by `PredictionKey`. - """ - key = prediction_key.PredictionKey.SCORES - with ops.name_scope(None, "predictions", (logits,)): - if self.logits_dimension == 1: - logits = array_ops.squeeze(logits, axis=(1,), name=key) - return {key: self._link_fn(logits)} - - def _metrics(self, eval_loss, predictions, labels, weights): - """Returns a dict of metrics keyed by name.""" - del predictions, labels, weights # Unused by this head. - with ops.name_scope("metrics", values=[eval_loss]): - return { - _summary_key(self.head_name, mkey.LOSS): - metrics_lib.mean(eval_loss)} - - -def _log_loss_with_two_classes(labels, logits, weights=None): - with ops.name_scope(None, "log_loss_with_two_classes", - (logits, labels)) as name: - logits = ops.convert_to_tensor(logits) - labels = math_ops.cast(labels, dtypes.float32) - # TODO(ptucker): This will break for dynamic shapes. - # sigmoid_cross_entropy_with_logits requires [batch_size, 1] labels. - if len(labels.get_shape()) == 1: - labels = array_ops.expand_dims(labels, axis=1) - loss = nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits, - name=name) - return _compute_weighted_loss(loss, weights) - - -def _one_class_to_two_class_logits(logits): - return array_ops.concat((array_ops.zeros_like(logits), logits), 1) - - -class _BinaryLogisticHead(_SingleHead): - """`Head` for binary classification with logistic regression.""" - - def __init__(self, - label_name=None, - weight_column_name=None, - enable_centered_bias=False, - head_name=None, - loss_fn=None, - thresholds=None): - """`Head` for binary classification with logistic regression. - - Args: - label_name: String, name of the key in label dict. Can be `None` if label - is a tensor (single headed models). - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - head_name: name of the head. Predictions, summary, metrics keys are - suffixed by `"/" + head_name` and the default variable scope is - `head_name`. - loss_fn: Loss function. - thresholds: thresholds for eval. - - Raises: - ValueError: if n_classes is invalid. - """ - super(_BinaryLogisticHead, self).__init__( - problem_type=constants.ProblemType.LOGISTIC_REGRESSION, - logits_dimension=1, - label_name=label_name, - weight_column_name=weight_column_name, - head_name=head_name) - self._thresholds = thresholds if thresholds else (.5,) - self._loss_fn = loss_fn if loss_fn else _log_loss_with_two_classes - self._enable_centered_bias = enable_centered_bias - - def create_model_fn_ops(self, - features, - mode, - labels=None, - train_op_fn=None, - logits=None, - logits_input=None, - scope=None): - """See `Head`.""" - with variable_scope.variable_scope( - scope, - default_name=self.head_name or "binary_logistic_head", - values=(tuple(six.itervalues(features)) + - (labels, logits, logits_input))): - labels = self._transform_labels(mode=mode, labels=labels) - logits = _logits(logits_input, logits, self.logits_dimension) - return _create_model_fn_ops( - features=features, - mode=mode, - loss_fn=self._loss_fn, - logits_to_predictions_fn=self._logits_to_predictions, - metrics_fn=self._metrics, - create_output_alternatives_fn=_classification_output_alternatives( - self.head_name, self._problem_type), - labels=labels, - train_op_fn=train_op_fn, - logits=logits, - logits_dimension=self.logits_dimension, - head_name=self.head_name, - weight_column_name=self.weight_column_name, - enable_centered_bias=self._enable_centered_bias) - - def _transform_labels(self, mode, labels): - """Applies transformations to labels tensor.""" - if (mode == model_fn.ModeKeys.INFER) or (labels is None): - return None - labels_tensor = _to_labels_tensor(labels, self._label_name) - _check_no_sparse_tensor(labels_tensor) - return labels_tensor - - def _logits_to_predictions(self, logits): - """Returns a dict of predictions. - - Args: - logits: logits `Output` after applying possible centered bias. - - Returns: - Dict of prediction `Output` keyed by `PredictionKey`. - """ - with ops.name_scope(None, "predictions", (logits,)): - two_class_logits = _one_class_to_two_class_logits(logits) - return { - prediction_key.PredictionKey.LOGITS: - logits, - prediction_key.PredictionKey.LOGISTIC: - math_ops.sigmoid( - logits, name=prediction_key.PredictionKey.LOGISTIC), - prediction_key.PredictionKey.PROBABILITIES: - nn.softmax( - two_class_logits, - name=prediction_key.PredictionKey.PROBABILITIES), - prediction_key.PredictionKey.CLASSES: - math_ops.argmax( - two_class_logits, - 1, - name=prediction_key.PredictionKey.CLASSES) - } - - def _metrics(self, eval_loss, predictions, labels, weights): - """Returns a dict of metrics keyed by name.""" - with ops.name_scope("metrics", values=( - [eval_loss, labels, weights] + list(six.itervalues(predictions)))): - classes = predictions[prediction_key.PredictionKey.CLASSES] - logistic = predictions[prediction_key.PredictionKey.LOGISTIC] - - metrics = {_summary_key(self.head_name, mkey.LOSS): - metrics_lib.mean(eval_loss)} - # TODO(b/29366811): This currently results in both an "accuracy" and an - # "accuracy/threshold_0.500000_mean" metric for binary classification. - metrics[_summary_key(self.head_name, mkey.ACCURACY)] = ( - metrics_lib.accuracy(labels, classes, weights)) - metrics[_summary_key(self.head_name, mkey.PREDICTION_MEAN)] = ( - _predictions_streaming_mean(logistic, weights)) - metrics[_summary_key(self.head_name, mkey.LABEL_MEAN)] = ( - _indicator_labels_streaming_mean(labels, weights)) - - # Also include the streaming mean of the label as an accuracy baseline, as - # a reminder to users. - metrics[_summary_key(self.head_name, mkey.ACCURACY_BASELINE)] = ( - _indicator_labels_streaming_mean(labels, weights)) - metrics[_summary_key(self.head_name, mkey.AUC)] = ( - _streaming_auc(logistic, labels, weights)) - metrics[_summary_key(self.head_name, mkey.AUC_PR)] = ( - _streaming_auc(logistic, labels, weights, curve="PR")) - - for threshold in self._thresholds: - metrics[_summary_key( - self.head_name, mkey.ACCURACY_MEAN % threshold)] = ( - _streaming_accuracy_at_threshold(logistic, labels, weights, - threshold)) - # Precision for positive examples. - metrics[_summary_key( - self.head_name, mkey.PRECISION_MEAN % threshold)] = ( - _streaming_precision_at_threshold(logistic, labels, weights, - threshold)) - # Recall for positive examples. - metrics[_summary_key( - self.head_name, mkey.RECALL_MEAN % threshold)] = ( - _streaming_recall_at_threshold(logistic, labels, weights, - threshold)) - - return metrics - - -def _softmax_cross_entropy_loss(labels, logits, weights=None): - with ops.name_scope( - None, "softmax_cross_entropy_loss", (logits, labels,)) as name: - labels = ops.convert_to_tensor(labels) - # Check that we got integer for classification. - if not labels.dtype.is_integer: - raise ValueError("Labels dtype should be integer " - "Instead got %s." % labels.dtype) - - # sparse_softmax_cross_entropy_with_logits requires [batch_size] labels. - is_squeezed_labels = False - # TODO(ptucker): This will break for dynamic shapes. - if len(labels.get_shape()) == 2: - labels = array_ops.squeeze(labels, axis=(1,)) - is_squeezed_labels = True - - loss = nn.sparse_softmax_cross_entropy_with_logits( - labels=labels, logits=logits, name=name) - - # Restore squeezed dimension, if necessary, so loss matches weights shape. - if is_squeezed_labels: - loss = array_ops.expand_dims(loss, axis=(1,)) - - return _compute_weighted_loss(loss, weights) - - -class _MultiClassHead(_SingleHead): - """'Head' for multi class classification.""" - - def __init__(self, - n_classes, - label_name=None, - weight_column_name=None, - enable_centered_bias=False, - head_name=None, - loss_fn=None, - thresholds=None, - metric_class_ids=None, - label_keys=None): - """'Head' for multi class classification. - - This head expects to be fed integer labels specifying the class index. But - if `label_keys` is specified, then labels must be strings from this - vocabulary, and the predicted classes will be strings from the same - vocabulary. - - Args: - n_classes: Number of classes, must be greater than 2 (for 2 classes, use - `_BinaryLogisticHead`). - label_name: String, name of the key in label dict. Can be null if label - is a tensor (single headed models). - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - head_name: name of the head. If provided, predictions, summary, metrics - keys will be suffixed by `"/" + head_name` and the default variable - scope will be `head_name`. - loss_fn: Loss function. Defaults to softmax cross entropy loss. - thresholds: thresholds for eval. - metric_class_ids: List of class IDs for which we should report per-class - metrics. Must all be in the range `[0, n_classes)`. - label_keys: Optional list of strings with size `[n_classes]` defining the - label vocabulary. - - Raises: - ValueError: if `n_classes`, `metric_class_ids` or `label_keys` is invalid. - """ - super(_MultiClassHead, self).__init__( - problem_type=constants.ProblemType.CLASSIFICATION, - logits_dimension=n_classes, - label_name=label_name, - weight_column_name=weight_column_name, - head_name=head_name) - - if (n_classes is None) or (n_classes <= 2): - raise ValueError("n_classes must be > 2: %s." % n_classes) - self._thresholds = thresholds if thresholds else (.5,) - self._loss_fn = loss_fn if loss_fn else _softmax_cross_entropy_loss - self._enable_centered_bias = enable_centered_bias - self._metric_class_ids = tuple([] if metric_class_ids is None else - metric_class_ids) - for class_id in self._metric_class_ids: - if (class_id < 0) or (class_id >= n_classes): - raise ValueError("Class ID %s not in [0, %s)." % (class_id, n_classes)) - if label_keys and len(label_keys) != n_classes: - raise ValueError("Length of label_keys must equal n_classes.") - self._label_keys = label_keys - - def create_model_fn_ops(self, - features, - mode, - labels=None, - train_op_fn=None, - logits=None, - logits_input=None, - scope=None): - """See `Head`.""" - with variable_scope.variable_scope( - scope, - default_name=self.head_name or "multi_class_head", - values=(tuple(six.itervalues(features)) + - (labels, logits, logits_input))): - labels = self._transform_labels(mode=mode, labels=labels) - logits = _logits(logits_input, logits, self.logits_dimension) - return _create_model_fn_ops( - features=features, - mode=mode, - loss_fn=self._wrapped_loss_fn, - logits_to_predictions_fn=self._logits_to_predictions, - metrics_fn=self._metrics, - create_output_alternatives_fn=_classification_output_alternatives( - self.head_name, self._problem_type, self._label_keys), - labels=labels, - train_op_fn=train_op_fn, - logits=logits, - logits_dimension=self.logits_dimension, - head_name=self.head_name, - weight_column_name=self.weight_column_name, - enable_centered_bias=self._enable_centered_bias) - - def _transform_labels(self, mode, labels): - """Returns a dict that contains both the original labels and label IDs.""" - if (mode == model_fn.ModeKeys.INFER) or (labels is None): - return None - labels_tensor = _to_labels_tensor(labels, self._label_name) - _check_no_sparse_tensor(labels_tensor) - if self._label_keys: - table = lookup_ops.index_table_from_tensor( - self._label_keys, name="label_id_lookup") - return { - "labels": labels_tensor, - "label_ids": table.lookup(labels_tensor), - } - return { - "labels": labels_tensor, - "label_ids": labels_tensor, - } - - def _labels(self, labels_dict): - """Returns labels `Tensor` of the same type as classes.""" - return labels_dict["labels"] - - def _label_ids(self, labels_dict): - """Returns integer label ID `Tensor`.""" - return labels_dict["label_ids"] - - def _wrapped_loss_fn(self, labels, logits, weights=None): - return self._loss_fn(self._label_ids(labels), logits, weights=weights) - - def _logits_to_predictions(self, logits): - """Returns a dict of predictions. - - Args: - logits: logits `Tensor` after applying possible centered bias. - - Returns: - Dict of prediction `Tensor` keyed by `PredictionKey`. - """ - with ops.name_scope(None, "predictions", (logits,)): - class_ids = math_ops.argmax( - logits, 1, name=prediction_key.PredictionKey.CLASSES) - if self._label_keys: - table = lookup_ops.index_to_string_table_from_tensor( - self._label_keys, name="class_string_lookup") - classes = table.lookup(class_ids) - else: - classes = class_ids - return { - prediction_key.PredictionKey.LOGITS: logits, - prediction_key.PredictionKey.PROBABILITIES: - nn.softmax( - logits, name=prediction_key.PredictionKey.PROBABILITIES), - prediction_key.PredictionKey.CLASSES: classes - } - - def _metrics(self, eval_loss, predictions, labels, weights): - """Returns a dict of metrics keyed by name.""" - with ops.name_scope( - "metrics", - values=((eval_loss, self._labels(labels), self._label_ids(labels), - weights) + tuple(six.itervalues(predictions)))): - logits = predictions[prediction_key.PredictionKey.LOGITS] - probabilities = predictions[prediction_key.PredictionKey.PROBABILITIES] - classes = predictions[prediction_key.PredictionKey.CLASSES] - - metrics = {_summary_key(self.head_name, mkey.LOSS): - metrics_lib.mean(eval_loss)} - # TODO(b/29366811): This currently results in both an "accuracy" and an - # "accuracy/threshold_0.500000_mean" metric for binary classification. - metrics[_summary_key(self.head_name, mkey.ACCURACY)] = ( - metrics_lib.accuracy(self._labels(labels), classes, weights)) - - if not self._label_keys: - # Classes are IDs. Add some metrics. - for class_id in self._metric_class_ids: - metrics[_summary_key( - self.head_name, mkey.CLASS_PREDICTION_MEAN % class_id)] = ( - _class_predictions_streaming_mean(classes, weights, class_id)) - # TODO(ptucker): Add per-class accuracy, precision, recall. - metrics[_summary_key( - self.head_name, mkey.CLASS_LABEL_MEAN % class_id)] = ( - _class_labels_streaming_mean( - self._label_ids(labels), weights, class_id)) - metrics[_summary_key( - self.head_name, mkey.CLASS_PROBABILITY_MEAN % class_id)] = ( - _predictions_streaming_mean(probabilities, weights, class_id)) - metrics[_summary_key( - self.head_name, mkey.CLASS_LOGITS_MEAN % class_id)] = ( - _predictions_streaming_mean(logits, weights, class_id)) - - return metrics - - -def _to_labels_tensor(labels, label_name): - """Returns label as a tensor. - - Args: - labels: Label `Tensor` or `SparseTensor` or a dict containing labels. - label_name: Label name if labels is a dict. - - Returns: - Label `Tensor` or `SparseTensor`. - """ - labels = labels[label_name] if isinstance(labels, dict) else labels - return framework_lib.convert_to_tensor_or_sparse_tensor(labels) - - -def _check_no_sparse_tensor(x): - """Raises ValueError if the given tensor is `SparseTensor`.""" - if isinstance(x, sparse_tensor.SparseTensor): - raise ValueError("SparseTensor is not supported.") - - -def _sparse_labels_to_indicator(labels, num_classes): - """If labels is `SparseTensor`, converts it to indicator `Tensor`. - - Args: - labels: Label `Tensor` or `SparseTensor`. - num_classes: Number of classes. - - Returns: - Dense label `Tensor`. - - Raises: - ValueError: If labels is `SparseTensor` and `num_classes` < 2. - """ - if isinstance(labels, sparse_tensor.SparseTensor): - if num_classes < 2: - raise ValueError("Must set num_classes >= 2 when passing labels as a " - "SparseTensor.") - return math_ops.cast( - sparse_ops.sparse_to_indicator(labels, num_classes), dtypes.int64) - return labels - - -def _assert_labels_rank(labels): - return control_flow_ops.Assert( - math_ops.less_equal(array_ops.rank(labels), 2), - ("labels shape should be either [batch_size, 1] or [batch_size]",)) - - -class _BinarySvmHead(_SingleHead): - """`Head` for binary classification using SVM.""" - - def __init__(self, label_name, weight_column_name, enable_centered_bias, - head_name, thresholds): - - def _loss_fn(labels, logits, weights=None): - with ops.name_scope(None, "hinge_loss", (logits, labels)) as name: - with ops.control_dependencies((_assert_labels_rank(labels),)): - labels = array_ops.reshape(labels, shape=(-1, 1)) - loss = losses_lib.hinge_loss(labels=labels, logits=logits, scope=name, - reduction=losses_lib.Reduction.NONE) - return _compute_weighted_loss(loss, weights) - - super(_BinarySvmHead, self).__init__( - problem_type=constants.ProblemType.LOGISTIC_REGRESSION, - logits_dimension=1, - label_name=label_name, - weight_column_name=weight_column_name, - head_name=head_name) - self._thresholds = thresholds if thresholds else (.5,) - self._loss_fn = _loss_fn - self._enable_centered_bias = enable_centered_bias - - def create_model_fn_ops(self, - features, - mode, - labels=None, - train_op_fn=None, - logits=None, - logits_input=None, - scope=None): - """See `Head`.""" - with variable_scope.variable_scope( - scope, - default_name=self.head_name or "binary_svm_head", - values=(tuple(six.itervalues(features)) + - (labels, logits, logits_input))): - labels = self._transform_labels(mode=mode, labels=labels) - logits = _logits(logits_input, logits, self.logits_dimension) - return _create_model_fn_ops( - features=features, - mode=mode, - loss_fn=self._loss_fn, - logits_to_predictions_fn=self._logits_to_predictions, - metrics_fn=self._metrics, - # TODO(zakaria): Handle labels for export. - create_output_alternatives_fn=self._create_output_alternatives, - labels=labels, - train_op_fn=train_op_fn, - logits=logits, - logits_dimension=self.logits_dimension, - head_name=self.head_name, - weight_column_name=self.weight_column_name, - enable_centered_bias=self._enable_centered_bias) - - def _transform_labels(self, mode, labels): - """Applies transformations to labels tensor.""" - if (mode == model_fn.ModeKeys.INFER) or (labels is None): - return None - labels_tensor = _to_labels_tensor(labels, self._label_name) - _check_no_sparse_tensor(labels_tensor) - return labels_tensor - - def _logits_to_predictions(self, logits): - """See `_MultiClassHead`.""" - with ops.name_scope(None, "predictions", (logits,)): - return { - prediction_key.PredictionKey.LOGITS: - logits, - prediction_key.PredictionKey.CLASSES: - math_ops.argmax( - _one_class_to_two_class_logits(logits), - 1, - name=prediction_key.PredictionKey.CLASSES) - } - - def _metrics(self, eval_loss, predictions, labels, weights): - """See `_MultiClassHead`.""" - with ops.name_scope("metrics", values=( - [eval_loss, labels, weights] + list(six.itervalues(predictions)))): - metrics = {_summary_key(self.head_name, mkey.LOSS): - metrics_lib.mean(eval_loss)} - - # TODO(b/29366811): This currently results in both an "accuracy" and an - # "accuracy/threshold_0.500000_mean" metric for binary classification. - classes = predictions[prediction_key.PredictionKey.CLASSES] - metrics[_summary_key(self.head_name, mkey.ACCURACY)] = ( - metrics_lib.accuracy(labels, classes, weights)) - # TODO(sibyl-vie3Poto): add more metrics relevant for svms. - - return metrics - - -class _MultiLabelHead(_SingleHead): - """`Head` for multi-label classification.""" - - # TODO(zakaria): add signature and metric for multilabel. - def __init__(self, - n_classes, - label_name, - weight_column_name, - enable_centered_bias, - head_name, - thresholds, - metric_class_ids=None, - loss_fn=None): - - super(_MultiLabelHead, self).__init__( - problem_type=constants.ProblemType.CLASSIFICATION, - logits_dimension=n_classes, - label_name=label_name, - weight_column_name=weight_column_name, - head_name=head_name) - - self._thresholds = thresholds if thresholds else (.5,) - self._loss_fn = loss_fn if loss_fn else _sigmoid_cross_entropy_loss - self._enable_centered_bias = enable_centered_bias - self._metric_class_ids = tuple([] if metric_class_ids is None else - metric_class_ids) - for class_id in self._metric_class_ids: - if (class_id < 0) or (class_id >= n_classes): - raise ValueError("Class ID %s not in [0, %s)." % (class_id, n_classes)) - - def create_model_fn_ops(self, - features, - mode, - labels=None, - train_op_fn=None, - logits=None, - logits_input=None, - scope=None): - """See `Head`.""" - with variable_scope.variable_scope( - scope, - default_name=self.head_name or "multi_label_head", - values=(tuple(six.itervalues(features)) + - (labels, logits, logits_input))): - labels = self._transform_labels(mode=mode, labels=labels) - logits = _logits(logits_input, logits, self.logits_dimension) - return _create_model_fn_ops( - features=features, - mode=mode, - loss_fn=self._loss_fn, - logits_to_predictions_fn=self._logits_to_predictions, - metrics_fn=self._metrics, - create_output_alternatives_fn=_classification_output_alternatives( - self.head_name, self._problem_type), - labels=labels, - train_op_fn=train_op_fn, - logits=logits, - logits_dimension=self.logits_dimension, - head_name=self.head_name, - weight_column_name=self.weight_column_name, - enable_centered_bias=self._enable_centered_bias) - - def _transform_labels(self, mode, labels): - """Applies transformations to labels tensor.""" - if (mode == model_fn.ModeKeys.INFER) or (labels is None): - return None - labels_tensor = _to_labels_tensor(labels, self._label_name) - labels_tensor = _sparse_labels_to_indicator(labels_tensor, - self._logits_dimension) - return labels_tensor - - def _logits_to_predictions(self, logits): - """See `_MultiClassHead`.""" - with ops.name_scope(None, "predictions", (logits,)): - return { - prediction_key.PredictionKey.LOGITS: - logits, - prediction_key.PredictionKey.PROBABILITIES: - math_ops.sigmoid( - logits, name=prediction_key.PredictionKey.PROBABILITIES), - prediction_key.PredictionKey.CLASSES: - math_ops.cast( - math_ops.greater(logits, 0), - dtypes.int64, - name=prediction_key.PredictionKey.CLASSES) - } - - def _metrics(self, eval_loss, predictions, labels, weights): - """Returns a dict of metrics keyed by name.""" - with ops.name_scope("metrics", values=( - [eval_loss, labels, weights] + list(six.itervalues(predictions)))): - classes = predictions[prediction_key.PredictionKey.CLASSES] - probabilities = predictions[prediction_key.PredictionKey.PROBABILITIES] - logits = predictions[prediction_key.PredictionKey.LOGITS] - - metrics = {_summary_key(self.head_name, mkey.LOSS): - metrics_lib.mean(eval_loss)} - # TODO(b/29366811): This currently results in both an "accuracy" and an - # "accuracy/threshold_0.500000_mean" metric for binary classification. - metrics[_summary_key(self.head_name, mkey.ACCURACY)] = ( - metrics_lib.accuracy(labels, classes, weights)) - metrics[_summary_key(self.head_name, mkey.AUC)] = _streaming_auc( - probabilities, labels, weights) - metrics[_summary_key(self.head_name, mkey.AUC_PR)] = _streaming_auc( - probabilities, labels, weights, curve="PR") - - for class_id in self._metric_class_ids: - # TODO(ptucker): Add per-class accuracy, precision, recall. - metrics[_summary_key( - self.head_name, mkey.CLASS_PREDICTION_MEAN % class_id)] = ( - _predictions_streaming_mean(classes, weights, class_id)) - metrics[_summary_key( - self.head_name, mkey.CLASS_LABEL_MEAN % class_id)] = ( - _indicator_labels_streaming_mean(labels, weights, class_id)) - metrics[_summary_key( - self.head_name, mkey.CLASS_PROBABILITY_MEAN % class_id)] = ( - _predictions_streaming_mean(probabilities, weights, class_id)) - metrics[_summary_key( - self.head_name, mkey.CLASS_LOGITS_MEAN % class_id)] = ( - _predictions_streaming_mean(logits, weights, class_id)) - metrics[_summary_key(self.head_name, mkey.CLASS_AUC % class_id)] = ( - _streaming_auc(probabilities, labels, weights, class_id)) - metrics[_summary_key(self.head_name, mkey.CLASS_AUC_PR % class_id)] = ( - _streaming_auc(probabilities, labels, weights, class_id, - curve="PR")) - - return metrics - - -class _LossOnlyHead(Head): - """`Head` implementation for additional loss terms. - - This class only holds loss terms unrelated to any other heads (labels), - e.g. regularization. - - Common usage: - This is oftem combine with other heads in a multi head setup. - ```python - head = multi_head([ - head1, head2, loss_only_head('regularizer', regularizer)]) - ``` - """ - - def __init__(self, loss_fn, head_name=None): - self._loss_fn = loss_fn - self.head_name = head_name or "loss_only_head" - - @property - def logits_dimension(self): - return 0 - - def create_model_fn_ops(self, - features, - mode, - labels=None, - train_op_fn=None, - logits=None, - logits_input=None, - scope=None): - """See `_Head.create_model_fn_ops`. - - Args: - features: Not been used. - mode: Estimator's `ModeKeys`. - labels: Labels `Tensor`, or `dict` of same. - train_op_fn: Function that takes a scalar loss and returns an op to - optimize with the loss. - logits: Not been used. - logits_input: Not been used. - scope: Optional scope for variable_scope. If provided, will be passed to - all heads. Most users will want to set this to `None`, so each head - constructs a separate variable_scope according to its `head_name`. - - Returns: - A `ModelFnOps` object. - - Raises: - ValueError: if `mode` is not recognition. - """ - _check_mode_valid(mode) - loss = None - train_op = None - if mode != model_fn.ModeKeys.INFER: - with variable_scope.variable_scope(scope, default_name=self.head_name): - loss = self._loss_fn() - if isinstance(loss, list): - loss = math_ops.add_n(loss) - # The name_scope escapism is needed to maintain the same summary tag - # after switching away from the now unsupported API. - with ops.name_scope(""): - summary_loss = array_ops.identity(loss) - summary.scalar(_summary_key(self.head_name, mkey.LOSS), - summary_loss) - if mode == model_fn.ModeKeys.TRAIN: - if train_op_fn is None: - raise ValueError("train_op_fn can not be None in TRAIN mode") - with ops.name_scope(None, "train_op", (loss,)): - train_op = train_op_fn(loss) - - return model_fn.ModelFnOps( - mode=mode, - loss=loss, - train_op=train_op, - predictions={}, - eval_metric_ops={}) - - -class _MultiHead(Head): - """`Head` implementation for multi objective learning. - - This class is responsible for using and merging the output of multiple - `Head` objects. - - All heads stem from the same logits/logit_input tensor. - - Common usage: - For simple use cases you can pass the activation of hidden layer like - this from your model_fn, - ```python - last_hidden_layer_activation = ... Build your model. - multi_head = ... - return multi_head.create_model_fn_ops( - ..., logits_input=last_hidden_layer_activation, ...) - ``` - - Or you can create a logits tensor of - [batch_size, multi_head.logits_dimension] shape. _MultiHead will split the - logits for you. - return multi_head.create_model_fn_ops(..., logits=logits, ...) - - For more complex use cases like a multi-task/multi-tower model or when logits - for each head has to be created separately, you can pass a dict of logits - where the keys match the name of the single heads. - ```python - logits = {"head1": logits1, "head2": logits2} - return multi_head.create_model_fn_ops(..., logits=logits, ...) - ``` - - Here is what this class does, - + For training, merges losses of each heads according a function provided by - user, calls user provided train_op_fn with this final loss. - + For eval, merges metrics by adding head_name suffix to the keys in eval - metrics. - + For inference, updates keys in prediction dict to a 2-tuple, - (head_name, prediction_key) - """ - - def __init__(self, heads, loss_merger): - """_Head to merges multiple _Head objects. - - Args: - heads: list of _Head objects. - loss_merger: function that takes a list of loss tensors for the heads - and returns the final loss tensor for the multi head. - - Raises: - ValueError: if any head does not have a name. - """ - self._logits_dimension = 0 - for head in heads: - if not head.head_name: - raise ValueError("Members of MultiHead must have names.") - self._logits_dimension += head.logits_dimension - - self._heads = heads - self._loss_merger = loss_merger - - @property - def logits_dimension(self): - return self._logits_dimension - - def create_model_fn_ops(self, - features, - mode, - labels=None, - train_op_fn=None, - logits=None, - logits_input=None, - scope=None): - """See `_Head.create_model_fn_ops`. - - Args: - features: Input `dict` of `Tensor` objects. - mode: Estimator's `ModeKeys`. - labels: Labels `Tensor`, or `dict` of same. - train_op_fn: Function that takes a scalar loss and returns an op to - optimize with the loss. - logits: Concatenated logits for all heads or a dict of head name to logits - tensor. If concatenated logits, it should have (batchsize, x) shape - where x is the sum of `logits_dimension` of all the heads, - i.e., same as `logits_dimension` of this class. create_model_fn_ops - will split the logits tensor and pass logits of proper size to each - head. This is useful if we want to be agnostic about whether you - creating a single versus multihead. logits can also be a dict for - convenience where you are creating the head specific logits explicitly - and don't want to concatenate them yourself. - logits_input: tensor to build logits from. - scope: Optional scope for variable_scope. If provided, will be passed to - all heads. Most users will want to set this to `None`, so each head - constructs a separate variable_scope according to its `head_name`. - - Returns: - `ModelFnOps`. - - Raises: - ValueError: if `mode` is not recognized, or neither or both of `logits` - and `logits_input` is provided. - """ - _check_mode_valid(mode) - all_model_fn_ops = [] - if logits is None: - # Use logits_input. - for head in self._heads: - all_model_fn_ops.append( - head.create_model_fn_ops( - features=features, - mode=mode, - labels=labels, - train_op_fn=no_op_train_fn, - logits_input=logits_input, - scope=scope)) - else: - head_logits_pairs = [] - if isinstance(logits, dict): - head_logits_pairs = [] - for head in self._heads: - if isinstance(head, _LossOnlyHead): - head_logits_pairs.append((head, None)) - else: - head_logits_pairs.append((head, logits[head.head_name])) - else: - # Split logits for each head. - head_logits_pairs = zip(self._heads, self._split_logits(logits)) - - for head, head_logits in head_logits_pairs: - all_model_fn_ops.append( - head.create_model_fn_ops( - features=features, - mode=mode, - labels=labels, - train_op_fn=no_op_train_fn, - logits=head_logits, - scope=scope)) - - if mode == model_fn.ModeKeys.TRAIN: - if train_op_fn is None: - raise ValueError("train_op_fn can not be None in TRAIN mode.") - return self._merge_train(all_model_fn_ops, train_op_fn) - if mode == model_fn.ModeKeys.INFER: - return self._merge_infer(all_model_fn_ops) - if mode == model_fn.ModeKeys.EVAL: - return self._merge_eval(all_model_fn_ops) - raise ValueError("mode=%s unrecognized" % str(mode)) - - def _split_logits(self, logits): - """Splits logits for heads. - - Args: - logits: the logits tensor. - - Returns: - A list of logits for the individual heads. - """ - all_logits = [] - begin = 0 - for head in self._heads: - current_logits_size = head.logits_dimension - current_logits = array_ops.slice(logits, [0, begin], - [-1, current_logits_size]) - all_logits.append(current_logits) - begin += current_logits_size - return all_logits - - def _merge_train(self, all_model_fn_ops, train_op_fn): - """Merges list of ModelFnOps for training. - - Args: - all_model_fn_ops: list of ModelFnOps for the individual heads. - train_op_fn: Function to create train op. See `create_model_fn_ops` - documentation for more details. - - Returns: - ModelFnOps that merges all heads for TRAIN. - """ - losses = [] - metrics = {} - additional_train_ops = [] - for m in all_model_fn_ops: - losses.append(m.loss) - if m.eval_metric_ops is not None: - for k, v in six.iteritems(m.eval_metric_ops): - # metrics["%s/%s" % (k, head_name)] = v - metrics[k] = v - additional_train_ops.append(m.train_op) - loss = self._loss_merger(losses) - - train_op = train_op_fn(loss) - train_op = control_flow_ops.group(train_op, *additional_train_ops) - return model_fn.ModelFnOps( - mode=model_fn.ModeKeys.TRAIN, - loss=loss, - train_op=train_op, - eval_metric_ops=metrics) - - def _merge_infer(self, all_model_fn_ops): - """Merges list of ModelFnOps for inference. - - Args: - all_model_fn_ops: list of ModelFnOps for the individual heads. - - Returns: - ModelFnOps that Merges all the heads for INFER. - """ - predictions = {} - output_alternatives = {} - for head, m in zip(self._heads, all_model_fn_ops): - if isinstance(head, _LossOnlyHead): - continue - head_name = head.head_name - output_alternatives[head_name] = m.output_alternatives[head_name] - for k, v in m.predictions.items(): - predictions[(head_name, k)] = v - - return model_fn.ModelFnOps( - mode=model_fn.ModeKeys.INFER, - predictions=predictions, - output_alternatives=output_alternatives) - - def _merge_eval(self, all_model_fn_ops): - """Merges list of ModelFnOps for eval. - - Args: - all_model_fn_ops: list of ModelFnOps for the individual heads. - - Returns: - ModelFnOps that merges all the heads for EVAL. - """ - predictions = {} - metrics = {} - losses = [] - for head, m in zip(self._heads, all_model_fn_ops): - losses.append(m.loss) - head_name = head.head_name - for k, v in m.predictions.items(): - predictions[(head_name, k)] = v - for k, v in m.eval_metric_ops.items(): - # metrics["%s/%s" % (k, head_name)] = v - metrics[k] = v - loss = self._loss_merger(losses) - - return model_fn.ModelFnOps( - mode=model_fn.ModeKeys.EVAL, - predictions=predictions, - loss=loss, - eval_metric_ops=metrics) - - -def _weight_tensor(features, weight_column_name): - """Returns weights as `Tensor` of rank 0, or at least 2.""" - if not weight_column_name: - return None - if weight_column_name not in features: - raise ValueError("Weights {} missing from features.".format( - weight_column_name)) - with ops.name_scope(None, "weight_tensor", tuple(six.itervalues(features))): - weight_tensor = math_ops.cast(features[weight_column_name], dtypes.float32) - shape = weight_tensor.get_shape() - rank = shape.ndims - # We don't bother with expanding dims of non-staticly shaped tensors or - # scalars, and >1d is already in a good format. - if rank == 1: - logging.warning("Weights {} has shape {}, expanding to make it 2d.". - format(weight_column_name, shape)) - return ( - sparse_ops.sparse_reshape(weight_tensor, (-1, 1)) - if isinstance(weight_tensor, sparse_tensor.SparseTensor) else - array_ops.reshape(weight_tensor, (-1, 1))) - return weight_tensor - - -# TODO(zakaria): This function is needed for backward compatibility and should -# be removed when we migrate to core. -def _compute_weighted_loss(loss_unweighted, weight, name="loss"): - """Returns a tuple of (loss_train, loss_report). - - loss is used for gradient descent while weighted_average_loss is used for - summaries to be backward compatible. - - loss is different from the loss reported on the tensorboard as we - should respect the example weights when computing the gradient. - - L = sum_{i} w_{i} * l_{i} / B - - where B is the number of examples in the batch, l_{i}, w_{i} are individual - losses, and example weight. - - Args: - loss_unweighted: Unweighted loss - weight: Weight tensor - name: Optional name - - Returns: - A tuple of losses. First one for training and the second one for reporting. - """ - with ops.name_scope(name, values=(loss_unweighted, weight)) as name_scope: - if weight is None: - loss = math_ops.reduce_mean(loss_unweighted, name=name_scope) - return loss, loss - weight = weights_broadcast_ops.broadcast_weights(weight, loss_unweighted) - with ops.name_scope(None, "weighted_loss", - (loss_unweighted, weight)) as name: - weighted_loss = math_ops.multiply(loss_unweighted, weight, name=name) - weighted_loss_mean = math_ops.reduce_mean(weighted_loss, name=name_scope) - weighted_loss_normalized = math_ops.div( - math_ops.reduce_sum(weighted_loss), - math_ops.cast(math_ops.reduce_sum(weight), dtypes.float32), - name="weighted_average_loss") - - return weighted_loss_mean, weighted_loss_normalized - - -def _wrap_custom_loss_fn(loss_fn): - def _wrapper(labels, logits, weights=None): - if weights is None: - loss = loss_fn(labels, logits) - else: - loss = loss_fn(labels, logits, weights) - return loss, loss - return _wrapper - - -def _check_mode_valid(mode): - """Raises ValueError if the given mode is invalid.""" - if (mode != model_fn.ModeKeys.TRAIN and mode != model_fn.ModeKeys.INFER and - mode != model_fn.ModeKeys.EVAL): - raise ValueError("mode=%s unrecognized." % str(mode)) - - -def _get_arguments(func): - """Returns a spec of given func.""" - _, func = tf_decorator.unwrap(func) - if hasattr(func, "__code__"): - # Regular function. - return tf_inspect.getargspec(func) - elif hasattr(func, "func"): - # Partial function. - return _get_arguments(func.func) - elif hasattr(func, "__call__"): - # Callable object. - return _get_arguments(func.__call__) - - -def _verify_loss_fn_args(loss_fn): - args = _get_arguments(loss_fn).args - for arg_name in ["labels", "logits", "weights"]: - if arg_name not in args: - raise ValueError("Argument %s not found in loss_fn." % arg_name) - - -def _centered_bias(logits_dimension, head_name=None): - """Returns centered_bias `Variable`. - - Args: - logits_dimension: Last dimension of `logits`. Must be >= 1. - head_name: Optional name of the head. - - Returns: - `Variable` with shape `[logits_dimension]`. - - Raises: - ValueError: if `logits_dimension` is invalid. - """ - if (logits_dimension is None) or (logits_dimension < 1): - raise ValueError("Invalid logits_dimension %s." % logits_dimension) - # Do not create a variable with variable_scope.get_variable, because that may - # create a PartitionedVariable, which does not support indexing, so - # summary.scalar will not work. - centered_bias = variable_scope.variable( - name="centered_bias_weight", - initial_value=array_ops.zeros(shape=(logits_dimension,)), - trainable=True) - for dim in range(logits_dimension): - if head_name: - summary.scalar("centered_bias/bias_%d/%s" % (dim, head_name), - centered_bias[dim]) - else: - summary.scalar("centered_bias/bias_%d" % dim, centered_bias[dim]) - return centered_bias - - -def _centered_bias_step(centered_bias, batch_size, labels, loss_fn, weights): - """Creates and returns training op for centered bias.""" - with ops.name_scope(None, "centered_bias_step", (labels,)) as name: - logits_dimension = array_ops.shape(centered_bias)[0] - logits = array_ops.reshape( - array_ops.tile(centered_bias, (batch_size,)), - (batch_size, logits_dimension)) - with ops.name_scope(None, "centered_bias", (labels, logits)): - centered_bias_loss = math_ops.reduce_mean( - loss_fn(labels, logits, weights), name="training_loss") - # Learn central bias by an optimizer. 0.1 is a convervative lr for a - # single variable. - return training.AdagradOptimizer(0.1).minimize( - centered_bias_loss, var_list=(centered_bias,), name=name) - - -def _summary_key(head_name, val): - return "%s/%s" % (val, head_name) if head_name else val - - -def _train_op(loss, labels, train_op_fn, centered_bias, batch_size, loss_fn, - weights): - """Returns op for the training step.""" - if centered_bias is not None: - centered_bias_step = _centered_bias_step( - centered_bias=centered_bias, - batch_size=batch_size, - labels=labels, - loss_fn=loss_fn, - weights=weights) - else: - centered_bias_step = None - with ops.name_scope(None, "train_op", (loss, labels)): - train_op = train_op_fn(loss) - if centered_bias_step is not None: - train_op = control_flow_ops.group(train_op, centered_bias_step) - return train_op - - -def _sigmoid_cross_entropy_loss(labels, logits, weights=None): - with ops.name_scope(None, "sigmoid_cross_entropy_loss", - (logits, labels)) as name: - # sigmoid_cross_entropy_with_logits requires [batch_size, n_classes] labels. - loss = nn.sigmoid_cross_entropy_with_logits( - labels=math_ops.cast(labels, dtypes.float32), logits=logits, name=name) - return _compute_weighted_loss(loss, weights) - - -def _float_weights_or_none(weights): - if weights is None: - return None - with ops.name_scope(None, "float_weights", (weights,)) as name: - return math_ops.cast(weights, dtypes.float32, name=name) - - -def _indicator_labels_streaming_mean(labels, weights=None, class_id=None): - labels = math_ops.cast(labels, dtypes.float32) - weights = _float_weights_or_none(weights) - if weights is not None: - weights = weights_broadcast_ops.broadcast_weights(weights, labels) - if class_id is not None: - if weights is not None: - weights = weights[:, class_id] - labels = labels[:, class_id] - return metrics_lib.mean(labels, weights) - - -def _predictions_streaming_mean(predictions, - weights=None, - class_id=None): - predictions = math_ops.cast(predictions, dtypes.float32) - weights = _float_weights_or_none(weights) - if weights is not None: - weights = weights_broadcast_ops.broadcast_weights(weights, predictions) - if class_id is not None: - if weights is not None: - weights = weights[:, class_id] - predictions = predictions[:, class_id] - return metrics_lib.mean(predictions, weights) - - -# TODO(ptucker): Add support for SparseTensor labels. -def _class_id_labels_to_indicator(labels, num_classes): - if (num_classes is None) or (num_classes < 2): - raise ValueError("Invalid num_classes %s." % num_classes) - with ops.control_dependencies((_assert_labels_rank(labels),)): - labels = array_ops.reshape(labels, (-1,)) - return array_ops.one_hot(labels, depth=num_classes, axis=-1) - - -def _class_predictions_streaming_mean(predictions, weights, class_id): - return metrics_lib.mean( - array_ops.where( - math_ops.equal( - math_ops.cast(class_id, dtypes.int32), - math_ops.cast(predictions, dtypes.int32)), - array_ops.ones_like(predictions), array_ops.zeros_like(predictions)), - weights=weights) - - -def _class_labels_streaming_mean(labels, weights, class_id): - return metrics_lib.mean( - array_ops.where( - math_ops.equal( - math_ops.cast(class_id, dtypes.int32), - math_ops.cast(labels, dtypes.int32)), array_ops.ones_like(labels), - array_ops.zeros_like(labels)), - weights=weights) - - -def _streaming_auc(predictions, labels, weights=None, class_id=None, - curve="ROC"): - # pylint: disable=missing-docstring - predictions = math_ops.cast(predictions, dtypes.float32) - if labels.dtype.base_dtype != dtypes.bool: - logging.warning("Casting %s labels to bool.", labels.dtype) - labels = math_ops.cast(labels, dtypes.bool) - weights = _float_weights_or_none(weights) - if weights is not None: - weights = weights_broadcast_ops.broadcast_weights(weights, predictions) - if class_id is not None: - if weights is not None: - weights = weights[:, class_id] - predictions = predictions[:, class_id] - labels = labels[:, class_id] - return metrics_lib.auc(labels, predictions, weights, curve=curve) - - -def _assert_class_id(class_id, num_classes=None): - """Average label value for class `class_id`.""" - if (class_id is None) or (class_id < 0): - raise ValueError("Invalid class_id %s." % class_id) - if num_classes is not None: - if num_classes < 2: - raise ValueError("Invalid num_classes %s." % num_classes) - if class_id >= num_classes: - raise ValueError("Invalid class_id %s." % class_id) - - -def _streaming_accuracy_at_threshold(predictions, labels, weights, threshold): - threshold_predictions = math_ops.cast( - math_ops.greater_equal(predictions, threshold), dtypes.float32) - return metrics_lib.accuracy(labels, threshold_predictions, weights) - - -def _streaming_precision_at_threshold(predictions, labels, weights, threshold): - precision_tensor, update_op = metrics_lib.precision_at_thresholds( - labels, predictions, (threshold,), _float_weights_or_none(weights)) - return array_ops.squeeze(precision_tensor), array_ops.squeeze(update_op) - - -def _streaming_recall_at_threshold(predictions, labels, weights, threshold): - precision_tensor, update_op = metrics_lib.recall_at_thresholds( - labels, predictions, (threshold,), _float_weights_or_none(weights)) - return array_ops.squeeze(precision_tensor), array_ops.squeeze(update_op) - - -def _classification_output_alternatives(head_name, problem_type, - label_keys=None): - """Creates a func to generate output alternatives for classification. - - Servo expects classes to be a string tensor, and have the same dimensions - as the probabilities tensor. It should contain the labels of the corresponding - entries in probabilities. This function creates a new classes tensor that - satisfies these conditions and can be exported. - - Args: - head_name: Name of the head. - problem_type: `ProblemType` - label_keys: Optional label keys - - Returns: - A function to generate output alternatives. - """ - def _create_output_alternatives(predictions): - """Creates output alternative for the Head. - - Args: - predictions: a dict of {tensor_name: Tensor}, where 'tensor_name' is a - symbolic name for an output Tensor possibly but not necessarily taken - from `PredictionKey`, and 'Tensor' is the corresponding output Tensor - itself. - - Returns: - `dict` of {submodel_name: (problem_type, {tensor_name: Tensor})}, where - 'submodel_name' is a submodel identifier that should be consistent across - the pipeline (here likely taken from the head_name), - 'problem_type' is a `ProblemType`, - 'tensor_name' is a symbolic name for an output Tensor possibly but not - necessarily taken from `PredictionKey`, and - 'Tensor' is the corresponding output Tensor itself. - - Raises: - ValueError: if predictions does not have PredictionKey.PROBABILITIES key. - """ - probabilities = predictions.get(prediction_key.PredictionKey.PROBABILITIES) - if probabilities is None: - raise ValueError("%s missing in predictions" % - prediction_key.PredictionKey.PROBABILITIES) - - with ops.name_scope(None, "_classification_output_alternatives", - (probabilities,)): - batch_size = array_ops.shape(probabilities)[0] - if label_keys: - classes = array_ops.tile( - input=array_ops.expand_dims(input=label_keys, axis=0), - multiples=[batch_size, 1], - name="classes_tensor") - else: - n = array_ops.shape(probabilities)[1] - classes = array_ops.tile( - input=array_ops.expand_dims(input=math_ops.range(n), axis=0), - multiples=[batch_size, 1]) - classes = string_ops.as_string(classes, name="classes_tensor") - - exported_predictions = { - prediction_key.PredictionKey.PROBABILITIES: probabilities, - prediction_key.PredictionKey.CLASSES: classes} - return {head_name: (problem_type, exported_predictions)} - - return _create_output_alternatives - -# Aliases -# TODO(zakaria): Remove these aliases, See b/34751732 -_regression_head = regression_head -_poisson_regression_head = poisson_regression_head -_multi_class_head = multi_class_head -_binary_svm_head = binary_svm_head -_multi_label_head = multi_label_head -_multi_head = multi_head -_Head = Head diff --git a/tensorflow/contrib/learn/python/learn/estimators/head_test.py b/tensorflow/contrib/learn/python/learn/estimators/head_test.py deleted file mode 100644 index a52d25acf40..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/head_test.py +++ /dev/null @@ -1,1867 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for head.py.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -# pylint: disable=g-bad-todo,g-import-not-at-top -import numpy as np -import six - -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.core.framework import summary_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables -from tensorflow.python.ops.losses import losses as losses_lib -from tensorflow.python.platform import test - - -def _assert_variables(test_case, - expected_global=None, - expected_model=None, - expected_trainable=None): - test_case.assertItemsEqual( - tuple([] if expected_global is None else expected_global), - tuple([k.name for k in variables.global_variables()])) - test_case.assertItemsEqual( - tuple([] if expected_model is None else expected_model), - tuple([k.name for k in variables.model_variables()])) - test_case.assertItemsEqual( - tuple([] if expected_trainable is None else expected_trainable), - tuple([k.name for k in variables.trainable_variables()])) - - -def _assert_no_variables(test_case): - _assert_variables(test_case) - - -# This must be called from within a tf.Session. -def _assert_metrics(test_case, expected_loss, expected_eval_metrics, - model_fn_ops): - test_case.assertAlmostEqual(expected_loss, model_fn_ops.loss.eval(), places=4) - for k in expected_eval_metrics: - test_case.assertIn(k, model_fn_ops.eval_metric_ops) - variables.initialize_local_variables().run() - for key, expected_value in six.iteritems(expected_eval_metrics): - value_tensor, update_tensor = model_fn_ops.eval_metric_ops[key] - update = update_tensor.eval() - test_case.assertAlmostEqual( - expected_value, - update, - places=4, - msg="%s: update, expected %s, got %s." % (key, expected_value, update)) - value = value_tensor.eval() - test_case.assertAlmostEqual( - expected_value, - value, - places=4, - msg="%s: value, expected %s, got %s." % (key, expected_value, value)) - - -# This must be called from within a tf.Session. -def _assert_summary_tags(test_case, expected_tags=None): - actual_tags = [] - for summary_op in ops.get_collection(ops.GraphKeys.SUMMARIES): - summ = summary_pb2.Summary() - summ.ParseFromString(summary_op.eval()) - actual_tags.append(summ.value[0].tag) - test_case.assertItemsEqual(expected_tags or [], actual_tags) - - -def _sigmoid(x): - return 1. / (1. + math.exp(-1 * x)) - - -class PoissonHeadTest(test.TestCase): - - def _assert_output_alternatives(self, model_fn_ops): - self.assertEquals({ - None: constants.ProblemType.LINEAR_REGRESSION - }, { - k: v[0] for k, v in six.iteritems(model_fn_ops.output_alternatives) - }) - - def _log_poisson_loss(self, logits, labels): - x = np.array([f[0] for f in logits]) - z = np.array([f[0] for f in labels]) - lpl = np.exp(x) - z * x - stirling_approx = z * np.log(z) - z + 0.5 * np.log(2. * np.pi * z) - lpl += np.ma.masked_array(stirling_approx, mask=(z <= 1)).filled(0.) - return sum(lpl)/len(lpl) - - def testPoissonWithLogits(self): - head = head_lib.poisson_regression_head() - labels = ((0.,), (1.,), (1.,)) - logits = ((0.,), (-1.,), (3.,)) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, - labels=labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=logits) - self._assert_output_alternatives(model_fn_ops) - _assert_summary_tags(self, ["loss"]) - _assert_no_variables(self) - loss = self._log_poisson_loss(logits, labels) - _assert_metrics(self, loss, {"loss": loss}, model_fn_ops) - - -class RegressionHeadTest(test.TestCase): - - def _assert_output_alternatives(self, model_fn_ops): - self.assertEquals({ - None: constants.ProblemType.LINEAR_REGRESSION - }, { - k: v[0] for k, v in six.iteritems(model_fn_ops.output_alternatives) - }) - - # TODO(zakaria): test multilabel regression. - def testRegressionWithLogits(self): - head = head_lib.regression_head() - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, - labels=((0.,), (1.,), (1.,)), - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=((1.,), (1.,), (3.,))) - self._assert_output_alternatives(model_fn_ops) - _assert_summary_tags(self, ["loss"]) - _assert_no_variables(self) - _assert_metrics(self, 5. / 3, {"loss": 5. / 3}, model_fn_ops) - - def testRegressionWithLogitFn(self): - head = head_lib.regression_head(link_fn=math_ops.square) - def _assert_preditions(test_case, expected_predictions, model_fn_ops): - variables.initialize_local_variables().run() - test_case.assertAllClose(expected_predictions, - model_fn_ops.predictions["scores"].eval()) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, - labels=((0.,), (1.,), (1.,)), - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=((1.,), (1.,), (3.,))) - self._assert_output_alternatives(model_fn_ops) - _assert_summary_tags(self, ["loss"]) - _assert_no_variables(self) - _assert_metrics(self, 5. / 3, {"loss": 5. / 3}, model_fn_ops) - _assert_preditions(self, ([1.0, 1.0, 9.0]), model_fn_ops) - - def testRegressionWithInvalidLogits(self): - head = head_lib.regression_head() - with ops.Graph().as_default(), session.Session(): - with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"): - head.create_model_fn_ops( - {}, - labels=((0.,), (1.,), (1.,)), - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=((1., 1.), (1., 1.), (3., 1.))) - - def testRegressionWithLogitsInput(self): - head = head_lib.regression_head() - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, - labels=((0.,), (1.,), (1.,)), - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits_input=((0., 0.), (0., 0.), (0., 0.))) - self._assert_output_alternatives(model_fn_ops) - w = ("regression_head/logits/weights:0", - "regression_head/logits/biases:0") - _assert_variables( - self, expected_global=w, expected_model=w, expected_trainable=w) - variables.global_variables_initializer().run() - _assert_summary_tags(self, ["loss"]) - _assert_metrics(self, 2. / 3, {"loss": 2. / 3}, model_fn_ops) - - def testRegressionWithLogitsAndLogitsInput(self): - head = head_lib.regression_head() - with ops.Graph().as_default(), session.Session(): - with self.assertRaisesRegexp( - ValueError, "Both logits and logits_input supplied"): - head.create_model_fn_ops( - {}, - labels=((0.,), (1.,), (1.,)), - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits_input=((0., 0.), (0., 0.), (0., 0.)), - logits=((1.,), (1.,), (3.,))) - - def testRegressionEvalMode(self): - head = head_lib.regression_head() - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, - labels=((1.,), (1.,), (3.,)), - mode=model_fn.ModeKeys.EVAL, - train_op_fn=head_lib.no_op_train_fn, - logits=((0.,), (1.,), (1.,))) - self._assert_output_alternatives(model_fn_ops) - self.assertIsNone(model_fn_ops.train_op) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - _assert_metrics(self, 5. / 3, {"loss": 5. / 3}, model_fn_ops) - - def testRegressionWithLabelName(self): - label_name = "my_label" - head = head_lib.regression_head(label_name=label_name) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, - labels={label_name: ((0.,), (1.,), (1.,))}, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=((1.,), (1.,), (3.,))) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - _assert_metrics(self, 5. / 3, {"loss": 5. / 3}, model_fn_ops) - - def testRegressionWithScalarWeights(self): - head = head_lib.regression_head(weight_column_name="label_weight") - with ops.Graph().as_default(), session.Session(): - weights = 2. - labels = ((0.,), (1.,), (1.,)) - model_fn_ops = head.create_model_fn_ops( - features={"label_weight": weights}, - labels=labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=((1.,), (1.,), (3.,))) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - _assert_metrics(self, (weights * 5.) / len(labels), { - "loss": (weights * 5.) / (weights * len(labels)) - }, model_fn_ops) - - def testRegressionWith1DWeights(self): - head = head_lib.regression_head(weight_column_name="label_weight") - with ops.Graph().as_default(), session.Session(): - weights = (2., 5., 0.) - labels = ((0.,), (1.,), (1.,)) - model_fn_ops = head.create_model_fn_ops( - features={"label_weight": weights}, - labels=labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=((1.,), (1.,), (3.,))) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - _assert_metrics(self, 2. / len(labels), {"loss": 2. / np.sum(weights)}, - model_fn_ops) - - def testRegressionWith2DWeights(self): - head = head_lib.regression_head(weight_column_name="label_weight") - with ops.Graph().as_default(), session.Session(): - weights = ((2.,), (5.,), (0.,)) - labels = ((0.,), (1.,), (1.,)) - model_fn_ops = head.create_model_fn_ops( - features={"label_weight": weights}, - labels=labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=((1.,), (1.,), (3.,))) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - _assert_metrics(self, 2. / len(labels), {"loss": 2. / np.sum(weights)}, - model_fn_ops) - - def testRegressionWithCenteredBias(self): - head = head_lib.regression_head(enable_centered_bias=True) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, - labels=((0.,), (1.,), (1.,)), - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=((1.,), (1.,), (3.,))) - self._assert_output_alternatives(model_fn_ops) - _assert_variables( - self, - expected_global=( - "regression_head/centered_bias_weight:0", - "regression_head/regression_head/centered_bias_weight/Adagrad:0", - ), - expected_trainable=("regression_head/centered_bias_weight:0",)) - variables.global_variables_initializer().run() - _assert_summary_tags(self, [ - "loss", - "regression_head/centered_bias/bias_0" - ]) - _assert_metrics(self, 5. / 3, {"loss": 5. / 3}, model_fn_ops) - - def testRegressionErrorInSparseTensorLabels(self): - head = head_lib.regression_head() - with ops.Graph().as_default(): - labels = sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (2, 0)), - values=(0., 1., 1.), - dense_shape=(3, 1)) - with self.assertRaisesRegexp(ValueError, - "SparseTensor is not supported"): - head.create_model_fn_ops( - {}, - labels=labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=((1.,), (1.,), (3.,))) - - -class MultiLabelHeadTest(test.TestCase): - - def _assert_output_alternatives(self, model_fn_ops): - self.assertEquals({ - None: constants.ProblemType.CLASSIFICATION - }, { - k: v[0] for k, v in six.iteritems(model_fn_ops.output_alternatives) - }) - - def setUp(self): - self._logits = ((1., 0., 0.),) - self._labels = ((0, 0, 1),) - - def _expected_eval_metrics(self, expected_loss): - return { - "accuracy": 1. / 3, - "loss": expected_loss, - "auc": 1. / 4, - "auc/class0": 1., - "auc/class1": 1., - "auc/class2": 0., - "auc_precision_recall": 0.166667, - "auc_precision_recall/class0": 0, - "auc_precision_recall/class1": 0., - "auc_precision_recall/class2": 1., - "labels/actual_label_mean/class0": self._labels[0][0], - "labels/actual_label_mean/class1": self._labels[0][1], - "labels/actual_label_mean/class2": self._labels[0][2], - "labels/logits_mean/class0": self._logits[0][0], - "labels/logits_mean/class1": self._logits[0][1], - "labels/logits_mean/class2": self._logits[0][2], - "labels/prediction_mean/class0": self._logits[0][0], - "labels/prediction_mean/class1": self._logits[0][1], - "labels/prediction_mean/class2": self._logits[0][2], - "labels/probability_mean/class0": _sigmoid(self._logits[0][0]), - "labels/probability_mean/class1": _sigmoid(self._logits[0][1]), - "labels/probability_mean/class2": _sigmoid(self._logits[0][2]), - } - - def testMultiLabelWithLogits(self): - n_classes = 3 - head = head_lib.multi_label_head( - n_classes=n_classes, metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = .89985204 - _assert_metrics(self, expected_loss, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testMultiLabelTwoClasses(self): - n_classes = 2 - labels = ((0, 1),) - logits = ((1., 0.),) - head = head_lib.multi_label_head( - n_classes=n_classes, metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, labels=labels, - train_op_fn=head_lib.no_op_train_fn, logits=logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = 1.00320443 - _assert_metrics(self, expected_loss, { - "accuracy": 0., - "auc": 0., - "loss": expected_loss, - "auc/class0": 1., - "auc/class1": 0., - "labels/actual_label_mean/class0": labels[0][0], - "labels/actual_label_mean/class1": labels[0][1], - "labels/logits_mean/class0": logits[0][0], - "labels/logits_mean/class1": logits[0][1], - "labels/prediction_mean/class0": logits[0][0], - "labels/prediction_mean/class1": logits[0][1], - "labels/probability_mean/class0": _sigmoid(logits[0][0]), - "labels/probability_mean/class1": _sigmoid(logits[0][1]), - }, model_fn_ops) - - def testMultiLabelWithInvalidLogits(self): - head = head_lib.multi_label_head(n_classes=len(self._labels[0]) + 1) - with ops.Graph().as_default(), session.Session(): - with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"): - head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - - def testMultiLabelWithLogitsInput(self): - n_classes = 3 - head = head_lib.multi_label_head( - n_classes=n_classes, metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits_input=((0., 0.),)) - self._assert_output_alternatives(model_fn_ops) - w = ("multi_label_head/logits/weights:0", - "multi_label_head/logits/biases:0") - _assert_variables( - self, expected_global=w, expected_model=w, expected_trainable=w) - variables.global_variables_initializer().run() - _assert_summary_tags(self, ["loss"]) - expected_loss = .69314718 - _assert_metrics(self, expected_loss, { - "accuracy": 2. / 3, - "auc": 2. / 4, - "loss": expected_loss, - "auc/class0": 1., - "auc/class1": 1., - "auc/class2": 0., - "labels/actual_label_mean/class0": self._labels[0][0], - "labels/actual_label_mean/class1": self._labels[0][1], - "labels/actual_label_mean/class2": self._labels[0][2], - "labels/logits_mean/class0": 0., - "labels/logits_mean/class1": 0., - "labels/logits_mean/class2": 0., - "labels/prediction_mean/class0": 0., - "labels/prediction_mean/class1": 0., - "labels/prediction_mean/class2": 0., - "labels/probability_mean/class0": .5, - "labels/probability_mean/class1": .5, - "labels/probability_mean/class2": .5, - }, model_fn_ops) - - def testMultiLabelWithLogitsAndLogitsInput(self): - n_classes = 3 - head = head_lib.multi_label_head( - n_classes=n_classes, metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - with self.assertRaisesRegexp( - ValueError, "Both logits and logits_input supplied"): - head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits_input=((0., 0.),), logits=self._logits) - - def testMultiLabelEval(self): - n_classes = 3 - head = head_lib.multi_label_head( - n_classes=n_classes, metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.EVAL, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - self.assertIsNone(model_fn_ops.train_op) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = .89985204 - _assert_metrics(self, expected_loss, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testMultiClassEvalWithLargeLogits(self): - n_classes = 3 - head = head_lib.multi_label_head( - n_classes=n_classes, metric_class_ids=range(n_classes)) - logits = ((2., 0., -1),) - with ops.Graph().as_default(), session.Session(): - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.EVAL, self._labels, head_lib.no_op_train_fn, - logits=logits) - self._assert_output_alternatives(model_fn_ops) - self.assertIsNone(model_fn_ops.train_op) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = 1.377779 - expected_eval_metrics = { - "accuracy": 1. / 3, - "auc": 9.99999e-07, - "loss": expected_loss, - "auc/class0": 1., - "auc/class1": 1., - "auc/class2": 0., - "labels/actual_label_mean/class0": 0. / 1, - "labels/actual_label_mean/class1": 0. / 1, - "labels/actual_label_mean/class2": 1. / 1, - "labels/logits_mean/class0": logits[0][0], - "labels/logits_mean/class1": logits[0][1], - "labels/logits_mean/class2": logits[0][2], - "labels/prediction_mean/class0": 1, - "labels/prediction_mean/class1": 0, - "labels/prediction_mean/class2": 0, - "labels/probability_mean/class0": _sigmoid(logits[0][0]), - "labels/probability_mean/class1": _sigmoid(logits[0][1]), - "labels/probability_mean/class2": _sigmoid(logits[0][2]), - } - _assert_metrics(self, expected_loss, - expected_eval_metrics, model_fn_ops) - - def testMultiLabelInfer(self): - n_classes = 3 - head = head_lib.multi_label_head(n_classes=n_classes, head_name="head_name") - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.INFER, self._labels, head_lib.no_op_train_fn, - logits=((1., 0., 0.), (0., 0., 1))) - self.assertIsNone(model_fn_ops.train_op) - _assert_no_variables(self) - with session.Session(): - self.assertListEqual( - [1, 0, 0], model_fn_ops.predictions["classes"].eval().tolist()[0]) - self.assertItemsEqual(["head_name"], - list(model_fn_ops.output_alternatives)) - self.assertEqual( - constants.ProblemType.CLASSIFICATION, - model_fn_ops.output_alternatives["head_name"][0]) - - predictions_for_serving = ( - model_fn_ops.output_alternatives["head_name"][1]) - self.assertIn("classes", predictions_for_serving) - self.assertAllEqual( - [[b"0", b"1", b"2"], [b"0", b"1", b"2"]], - predictions_for_serving["classes"].eval()) - self.assertIn("probabilities", predictions_for_serving) - self.assertAllClose( - [[0.731059, 0.5, 0.5], - [0.5, 0.5, 0.731059,]], - predictions_for_serving["probabilities"].eval()) - - def testMultiLabelWithLabelName(self): - n_classes = 3 - label_name = "my_label" - head = head_lib.multi_label_head( - n_classes=n_classes, - label_name=label_name, - metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, {label_name: self._labels}, - head_lib.no_op_train_fn, logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = .89985204 - _assert_metrics(self, expected_loss, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testMultiLabelWithScalarWeight(self): - n_classes = 3 - head = head_lib.multi_label_head( - n_classes=n_classes, - weight_column_name="label_weight", - metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - features={"label_weight": .1}, - labels=self._labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - _assert_metrics(self, .089985214, - self._expected_eval_metrics(.89985214), model_fn_ops) - - def testMultiLabelWith1DWeight(self): - n_classes = 3 - head = head_lib.multi_label_head( - n_classes=n_classes, - weight_column_name="label_weight", - metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - with self.assertRaisesRegexp( - ValueError, "weights can not be broadcast to values"): - head.create_model_fn_ops( - features={"label_weight": (.1, .1, .1)}, - labels=self._labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=self._logits) - - def testMultiLabelWith2DWeight(self): - n_classes = 3 - head = head_lib.multi_label_head( - n_classes=n_classes, - weight_column_name="label_weight", - metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - features={"label_weight": ((.1, .1, .1),)}, - labels=self._labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - _assert_metrics(self, .089985214, - self._expected_eval_metrics(.89985214), model_fn_ops) - - def testMultiLabelWithCustomLoss(self): - n_classes = 3 - head = head_lib.multi_label_head( - n_classes=n_classes, - weight_column_name="label_weight", - metric_class_ids=range(n_classes), - loss_fn=_sigmoid_cross_entropy) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - features={"label_weight": .1}, - labels=self._labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = .089985214 - _assert_metrics(self, expected_loss, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testMultiLabelWithCenteredBias(self): - n_classes = 3 - head = head_lib.multi_label_head( - n_classes=n_classes, - enable_centered_bias=True, - metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_variables( - self, - expected_global=( - "multi_label_head/centered_bias_weight:0", - ("multi_label_head/multi_label_head/centered_bias_weight/" - "Adagrad:0"),), - expected_trainable=("multi_label_head/centered_bias_weight:0",)) - variables.global_variables_initializer().run() - _assert_summary_tags(self, ( - "loss", - "multi_label_head/centered_bias/bias_0", - "multi_label_head/centered_bias/bias_1", - "multi_label_head/centered_bias/bias_2" - )) - expected_loss = .89985204 - _assert_metrics(self, expected_loss, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testMultiLabelSparseTensorLabels(self): - n_classes = 3 - head = head_lib.multi_label_head( - n_classes=n_classes, metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - labels = sparse_tensor.SparseTensorValue( - indices=((0, 0),), - values=(2,), - dense_shape=(1, 1)) - model_fn_ops = head.create_model_fn_ops( - features={}, - mode=model_fn.ModeKeys.TRAIN, - labels=labels, - train_op_fn=head_lib.no_op_train_fn, - logits=self._logits) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = .89985204 - _assert_metrics(self, expected_loss, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testMultiLabelSparseTensorLabelsTooFewClasses(self): - n_classes = 3 - head = head_lib.multi_label_head( - n_classes=n_classes, metric_class_ids=range(n_classes)) - # Set _logits_dimension (n_classes) to a lower value; if it's set to 1 - # upfront, the class throws an error during initialization. - head._logits_dimension = 1 - with ops.Graph().as_default(), session.Session(): - labels = sparse_tensor.SparseTensorValue( - indices=((0, 0),), - values=(2,), - dense_shape=(1, 1)) - with self.assertRaisesRegexp(ValueError, - "Must set num_classes >= 2 when passing"): - head.create_model_fn_ops( - features={}, - labels=labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=[0.]) - - -class BinaryClassificationHeadTest(test.TestCase): - - def _assert_output_alternatives(self, model_fn_ops): - self.assertEquals({ - None: constants.ProblemType.LOGISTIC_REGRESSION - }, { - k: v[0] for k, v in six.iteritems(model_fn_ops.output_alternatives) - }) - - def setUp(self): - self._logits = ((1.,), (1.,)) - self._labels = ((1.,), (0.,)) - - def _expected_eval_metrics(self, expected_loss): - label_mean = np.mean(self._labels) - return { - "accuracy": 1. / 2, - "accuracy/baseline_label_mean": label_mean, - "accuracy/threshold_0.500000_mean": 1. / 2, - "auc": 1. / 2, - "auc_precision_recall": 0.749999, - "labels/actual_label_mean": label_mean, - "labels/prediction_mean": .731059, # softmax - "loss": expected_loss, - "precision/positive_threshold_0.500000_mean": 1. / 2, - "recall/positive_threshold_0.500000_mean": 1. / 1, - } - - def testBinaryClassificationWithLogits(self): - n_classes = 2 - head = head_lib.multi_class_head(n_classes=n_classes) - with ops.Graph().as_default(), session.Session(): - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = .81326175 - _assert_metrics(self, expected_loss, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testBinaryClassificationWithInvalidLogits(self): - head = head_lib.multi_class_head(n_classes=len(self._labels) + 1) - with ops.Graph().as_default(), session.Session(): - with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"): - head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - - def testBinaryClassificationWithLogitsInput(self): - n_classes = 2 - head = head_lib.multi_class_head(n_classes=n_classes) - with ops.Graph().as_default(), session.Session(): - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits_input=((0., 0.), (0., 0.))) - self._assert_output_alternatives(model_fn_ops) - w = ("binary_logistic_head/logits/weights:0", - "binary_logistic_head/logits/biases:0") - _assert_variables( - self, expected_global=w, expected_model=w, expected_trainable=w) - variables.global_variables_initializer().run() - _assert_summary_tags(self, ["loss"]) - expected_loss = .69314718 - label_mean = np.mean(self._labels) - _assert_metrics(self, expected_loss, { - "accuracy": 1. / 2, - "accuracy/baseline_label_mean": label_mean, - "accuracy/threshold_0.500000_mean": 1. / 2, - "auc": 1. / 2, - "labels/actual_label_mean": label_mean, - "labels/prediction_mean": .5, # softmax - "loss": expected_loss, - "precision/positive_threshold_0.500000_mean": 0. / 2, - "recall/positive_threshold_0.500000_mean": 0. / 1, - }, model_fn_ops) - - def testBinaryClassificationWithLogitsAndLogitsInput(self): - head = head_lib.multi_class_head(n_classes=2) - with ops.Graph().as_default(), session.Session(): - with self.assertRaisesRegexp( - ValueError, "Both logits and logits_input supplied"): - head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits_input=((0., 0.), (0., 0.)), logits=self._logits) - - def testBinaryClassificationEval(self): - n_classes = 2 - head = head_lib.multi_class_head(n_classes=n_classes) - with ops.Graph().as_default(), session.Session(): - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.EVAL, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - self.assertIsNone(model_fn_ops.train_op) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = .81326175 - _assert_metrics(self, expected_loss, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testBinaryClassificationInfer(self): - n_classes = 2 - head = head_lib.multi_class_head(n_classes=n_classes, head_name="head_name") - with ops.Graph().as_default(), session.Session(): - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.INFER, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - self.assertIsNone(model_fn_ops.train_op) - _assert_no_variables(self) - with session.Session(): - self.assertListEqual( - [1, 1], list(model_fn_ops.predictions["classes"].eval())) - self.assertItemsEqual(["head_name"], - list(model_fn_ops.output_alternatives)) - self.assertEqual( - constants.ProblemType.LOGISTIC_REGRESSION, - model_fn_ops.output_alternatives["head_name"][0]) - predictions_for_serving = ( - model_fn_ops.output_alternatives["head_name"][1]) - self.assertIn("classes", predictions_for_serving) - predicted_classes = predictions_for_serving["classes"].eval().tolist() - self.assertListEqual( - [b"0", b"1"], predicted_classes[0]) - self.assertIn("probabilities", predictions_for_serving) - - def testBinaryClassificationInferMode_withWeightColumn(self): - n_classes = 2 - head = head_lib.multi_class_head(n_classes=n_classes, - weight_column_name="label_weight") - with ops.Graph().as_default(), session.Session(): - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - # This is what is being tested, features should not have weight for - # inference. - {}, model_fn.ModeKeys.INFER, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - self.assertIsNone(model_fn_ops.train_op) - _assert_no_variables(self) - - def testErrorInSparseTensorLabels(self): - n_classes = 2 - head = head_lib.multi_class_head(n_classes=n_classes) - with ops.Graph().as_default(): - labels = sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (2, 0)), - values=(0, 1, 1), - dense_shape=(3, 1)) - with self.assertRaisesRegexp(ValueError, - "SparseTensor is not supported"): - head.create_model_fn_ops( - {}, - model_fn.ModeKeys.TRAIN, - labels, - head_lib.no_op_train_fn, - logits=((1.,), (1.,), (3.,))) - - def testBinaryClassificationWithLabelName(self): - label_name = "my_label" - head = head_lib.multi_class_head(n_classes=2, label_name=label_name) - with ops.Graph().as_default(), session.Session(): - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - {}, - labels={label_name: self._labels}, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = .81326175 - _assert_metrics(self, expected_loss, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testBinaryClassificationWith1DWeights(self): - n_classes = 2 - head = head_lib.multi_class_head( - n_classes=n_classes, weight_column_name="label_weight") - with ops.Graph().as_default(), session.Session(): - weights = (1., 0.) - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - features={"label_weight": weights}, - labels=self._labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_total_loss = .31326166 - _assert_metrics( - self, - expected_total_loss / len(weights), - { - "accuracy": 1. / 1, - "accuracy/baseline_label_mean": 1. / 1, - "accuracy/threshold_0.500000_mean": 1. / 1, - "auc": 0. / 1, - "labels/actual_label_mean": 1. / 1, - "labels/prediction_mean": .731059, # softmax - # eval loss is weighted loss divided by sum of weights. - "loss": expected_total_loss, - "precision/positive_threshold_0.500000_mean": 1. / 1, - "recall/positive_threshold_0.500000_mean": 1. / 1, - }, - model_fn_ops) - - def testBinaryClassificationWith2DWeights(self): - n_classes = 2 - head = head_lib.multi_class_head( - n_classes=n_classes, weight_column_name="label_weight") - with ops.Graph().as_default(), session.Session(): - weights = ((1.,), (0.,)) - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - features={"label_weight": weights}, - labels=self._labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_total_loss = .31326166 - _assert_metrics( - self, - expected_total_loss / len(weights), - { - "accuracy": 1. / 1, - "accuracy/baseline_label_mean": 1. / 1, - "accuracy/threshold_0.500000_mean": 1. / 1, - "auc": 0. / 1, - "labels/actual_label_mean": 1. / 1, - "labels/prediction_mean": .731059, # softmax - # eval loss is weighted loss divided by sum of weights. - "loss": expected_total_loss, - "precision/positive_threshold_0.500000_mean": 1. / 1, - "recall/positive_threshold_0.500000_mean": 1. / 1, - }, - model_fn_ops) - - def testBinaryClassificationWithCustomLoss(self): - head = head_lib.multi_class_head( - n_classes=2, weight_column_name="label_weight", - loss_fn=_sigmoid_cross_entropy) - with ops.Graph().as_default(), session.Session(): - weights = ((.2,), (0.,)) - model_fn_ops = head.create_model_fn_ops( - features={"label_weight": weights}, - labels=self._labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - # expected_loss is (total_weighted_loss)/1 since there is 1 nonzero - # weight. - expected_loss = 0.062652342 - _assert_metrics( - self, - expected_loss, - { - "accuracy": 1. / 1, - "accuracy/baseline_label_mean": 1. / 1, - "accuracy/threshold_0.500000_mean": 1. / 1, - "auc": 0. / 1, - "labels/actual_label_mean": 1. / 1, - "labels/prediction_mean": .731059, # softmax - "loss": expected_loss, - "precision/positive_threshold_0.500000_mean": 1. / 1, - "recall/positive_threshold_0.500000_mean": 1. / 1, - }, - model_fn_ops) - - def testBinaryClassificationWithCenteredBias(self): - head = head_lib.multi_class_head(n_classes=2, enable_centered_bias=True) - with ops.Graph().as_default(), session.Session(): - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_variables( - self, - expected_global=( - "binary_logistic_head/centered_bias_weight:0", - ("binary_logistic_head/binary_logistic_head/centered_bias_weight/" - "Adagrad:0"),), - expected_trainable=("binary_logistic_head/centered_bias_weight:0",)) - variables.global_variables_initializer().run() - _assert_summary_tags(self, [ - "loss", - "binary_logistic_head/centered_bias/bias_0" - ]) - expected_loss = .81326175 - _assert_metrics(self, expected_loss, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - -class MultiClassHeadTest(test.TestCase): - - def _assert_output_alternatives(self, model_fn_ops): - self.assertEquals({ - None: constants.ProblemType.CLASSIFICATION - }, { - k: v[0] for k, v in six.iteritems(model_fn_ops.output_alternatives) - }) - - def setUp(self): - self._logits = ((1., 0., 0.),) - self._labels = ((2,),) - - def _expected_eval_metrics(self, expected_loss): - return { - "accuracy": 0., - "loss": expected_loss, - "labels/actual_label_mean/class0": 0. / 1, - "labels/actual_label_mean/class1": 0. / 1, - "labels/actual_label_mean/class2": 1. / 1, - "labels/logits_mean/class0": self._logits[0][0], - "labels/logits_mean/class1": self._logits[0][1], - "labels/logits_mean/class2": self._logits[0][2], - "labels/prediction_mean/class0": self._logits[0][0], - "labels/prediction_mean/class1": self._logits[0][1], - "labels/prediction_mean/class2": self._logits[0][2], - "labels/probability_mean/class0": 0.576117, # softmax - "labels/probability_mean/class1": 0.211942, # softmax - "labels/probability_mean/class2": 0.211942, # softmax - } - - def testMultiClassWithLogits(self): - n_classes = 3 - head = head_lib.multi_class_head( - n_classes=n_classes, metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = 1.5514447 - _assert_metrics(self, expected_loss, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testMultiClassWithInvalidLogits(self): - head = head_lib.multi_class_head(n_classes=len(self._logits[0]) + 1) - with ops.Graph().as_default(), session.Session(): - with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"): - head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - - def testMultiClassWithNoneTrainOpFnInTrain(self): - head = head_lib.multi_class_head(n_classes=3) - with ops.Graph().as_default(), session.Session(): - with self.assertRaisesRegexp( - ValueError, "train_op_fn can not be None in TRAIN mode"): - head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, - train_op_fn=None, - logits=self._logits) - - def testMultiClassWithLogitsInput(self): - n_classes = 3 - head = head_lib.multi_class_head( - n_classes=n_classes, metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits_input=((0., 0.),)) - self._assert_output_alternatives(model_fn_ops) - w = ("multi_class_head/logits/weights:0", - "multi_class_head/logits/biases:0") - _assert_variables( - self, expected_global=w, expected_model=w, expected_trainable=w) - variables.global_variables_initializer().run() - _assert_summary_tags(self, ["loss"]) - expected_loss = 1.0986123 - _assert_metrics(self, expected_loss, { - "accuracy": 0., - "loss": expected_loss, - "labels/actual_label_mean/class0": 0. / 1, - "labels/actual_label_mean/class1": 0. / 1, - "labels/actual_label_mean/class2": 1. / 1, - "labels/logits_mean/class0": 0., - "labels/logits_mean/class1": 0., - "labels/logits_mean/class2": 0., - "labels/prediction_mean/class0": 1., - "labels/prediction_mean/class1": 0., - "labels/prediction_mean/class2": 0., - "labels/probability_mean/class0": 0.333333, # softmax - "labels/probability_mean/class1": 0.333333, # softmax - "labels/probability_mean/class2": 0.333333, # softmax - }, model_fn_ops) - - def testMultiClassWithLogitsAndLogitsInput(self): - n_classes = 3 - head = head_lib.multi_class_head( - n_classes=n_classes, metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - with self.assertRaisesRegexp( - ValueError, "Both logits and logits_input supplied"): - head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits_input=((0., 0.),), logits=self._logits) - - def testMultiClassEnableCenteredBias(self): - n_classes = 3 - head = head_lib.multi_class_head( - n_classes=n_classes, enable_centered_bias=True) - with ops.Graph().as_default(), session.Session(): - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_variables( - self, - expected_global=( - "multi_class_head/centered_bias_weight:0", - ("multi_class_head/multi_class_head/centered_bias_weight/" - "Adagrad:0"), - ), - expected_trainable=("multi_class_head/centered_bias_weight:0",)) - variables.global_variables_initializer().run() - _assert_summary_tags(self, - ["loss", - "multi_class_head/centered_bias/bias_0", - "multi_class_head/centered_bias/bias_1", - "multi_class_head/centered_bias/bias_2"]) - - def testMultiClassEval(self): - n_classes = 3 - head = head_lib.multi_class_head( - n_classes=n_classes, metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.EVAL, self._labels, head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - self.assertIsNone(model_fn_ops.train_op) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = 1.5514447 - _assert_metrics(self, expected_loss, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testMultiClassEvalModeWithLargeLogits(self): - n_classes = 3 - head = head_lib.multi_class_head( - n_classes=n_classes, metric_class_ids=range(n_classes)) - logits = ((2., 0., -1),) - with ops.Graph().as_default(), session.Session(): - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - {}, model_fn.ModeKeys.EVAL, self._labels, head_lib.no_op_train_fn, - logits=logits) - self._assert_output_alternatives(model_fn_ops) - self.assertIsNone(model_fn_ops.train_op) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = 3.1698461 - expected_eval_metrics = { - "accuracy": 0., - "loss": expected_loss, - "labels/actual_label_mean/class0": 0. / 1, - "labels/actual_label_mean/class1": 0. / 1, - "labels/actual_label_mean/class2": 1. / 1, - "labels/logits_mean/class0": logits[0][0], - "labels/logits_mean/class1": logits[0][1], - "labels/logits_mean/class2": logits[0][2], - "labels/prediction_mean/class0": 1, - "labels/prediction_mean/class1": 0, - "labels/prediction_mean/class2": 0, - "labels/probability_mean/class0": 0.843795, # softmax - "labels/probability_mean/class1": 0.114195, # softmax - "labels/probability_mean/class2": 0.0420101, # softmax - } - _assert_metrics(self, expected_loss, - expected_eval_metrics, model_fn_ops) - - def testMultiClassWithScalarWeight(self): - n_classes = 3 - head = head_lib.multi_class_head( - n_classes=n_classes, - weight_column_name="label_weight", - metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - weight = .1 - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - features={"label_weight": weight}, - labels=self._labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = 1.5514447 - _assert_metrics(self, expected_loss * weight, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testMultiClassWith1DWeight(self): - n_classes = 3 - head = head_lib.multi_class_head( - n_classes=n_classes, - weight_column_name="label_weight", - metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - weight = .1 - weights = (weight,) - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - features={"label_weight": weights}, - labels=self._labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = 1.5514447 - _assert_metrics(self, expected_loss * weight, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testMultiClassWith2DWeight(self): - n_classes = 3 - head = head_lib.multi_class_head( - n_classes=n_classes, - weight_column_name="label_weight", - metric_class_ids=range(n_classes)) - with ops.Graph().as_default(), session.Session(): - weight = .1 - weights = ((weight,),) - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - features={"label_weight": weights}, - labels=self._labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = 1.5514447 - _assert_metrics(self, expected_loss * weight, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testMultiClassWithCustomLoss(self): - n_classes = 3 - head = head_lib.multi_class_head( - n_classes=n_classes, - weight_column_name="label_weight", - metric_class_ids=range(n_classes), - loss_fn=losses_lib.sparse_softmax_cross_entropy) - with ops.Graph().as_default(), session.Session(): - weight = .1 - # logloss: z:label, x:logit - # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - model_fn_ops = head.create_model_fn_ops( - features={"label_weight": weight}, - labels=self._labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=self._logits) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = 1.5514447 * weight - _assert_metrics(self, expected_loss, - self._expected_eval_metrics(expected_loss), model_fn_ops) - - def testMultiClassInfer(self): - n_classes = 3 - head = head_lib._multi_class_head( - n_classes=n_classes, - head_name="head_name") - with ops.Graph().as_default(): - model_fn_ops = head.create_model_fn_ops( - features={}, - mode=model_fn.ModeKeys.INFER, - train_op_fn=head_lib.no_op_train_fn, - logits=((1., 0., 0.), (0., 0., 1.),)) - with session.Session(): - lookup_ops.tables_initializer().run() - self.assertAllEqual( - [0, 2], - model_fn_ops.predictions["classes"].eval()) - self.assertItemsEqual(["head_name"], - list(model_fn_ops.output_alternatives)) - self.assertEqual( - constants.ProblemType.CLASSIFICATION, - model_fn_ops.output_alternatives["head_name"][0]) - predictions_for_serving = ( - model_fn_ops.output_alternatives["head_name"][1]) - self.assertIn("classes", predictions_for_serving) - self.assertAllEqual( - [[b"0", b"1", b"2"], [b"0", b"1", b"2"]], - predictions_for_serving["classes"].eval()) - self.assertIn("probabilities", predictions_for_serving) - self.assertAllClose( - [[0.576117, 0.2119416, 0.2119416], - [0.2119416, 0.2119416, 0.576117]], - predictions_for_serving["probabilities"].eval()) - - def testInvalidNClasses(self): - for n_classes in (None, -1, 0, 1): - with self.assertRaisesRegexp(ValueError, "n_classes must be > 1"): - head_lib.multi_class_head(n_classes=n_classes) - - def testMultiClassWithLabelKeysInvalidShape(self): - with self.assertRaisesRegexp( - ValueError, "Length of label_keys must equal n_classes"): - head_lib._multi_class_head( - n_classes=3, label_keys=("key0", "key1")) - - def testMultiClassWithLabelKeysTwoClasses(self): - with self.assertRaisesRegexp( - ValueError, "label_keys is not supported for n_classes=2"): - head_lib._multi_class_head( - n_classes=2, label_keys=("key0", "key1")) - - def testMultiClassWithLabelKeysInfer(self): - n_classes = 3 - label_keys = ("key0", "key1", "key2") - head = head_lib._multi_class_head( - n_classes=n_classes, label_keys=label_keys, - metric_class_ids=range(n_classes), - head_name="head_name") - with ops.Graph().as_default(): - model_fn_ops = head.create_model_fn_ops( - features={}, - mode=model_fn.ModeKeys.INFER, - train_op_fn=head_lib.no_op_train_fn, - logits=((1., 0., 0.), (0., 0., 1.),)) - with session.Session(): - lookup_ops.tables_initializer().run() - self.assertAllEqual( - [b"key0", b"key2"], - model_fn_ops.predictions["classes"].eval()) - self.assertItemsEqual(["head_name"], - list(model_fn_ops.output_alternatives)) - self.assertEqual( - constants.ProblemType.CLASSIFICATION, - model_fn_ops.output_alternatives["head_name"][0]) - predictions_for_serving = ( - model_fn_ops.output_alternatives["head_name"][1]) - self.assertIn("classes", predictions_for_serving) - self.assertAllEqual( - [[b"key0", b"key1", b"key2"], [b"key0", b"key1", b"key2"]], - predictions_for_serving["classes"].eval()) - self.assertIn("probabilities", predictions_for_serving) - self.assertAllClose( - [[0.576117, 0.2119416, 0.2119416], - [0.2119416, 0.2119416, 0.576117]], - predictions_for_serving["probabilities"].eval()) - - def testMultiClassWithLabelKeysEvalAccuracy0(self): - n_classes = 3 - label_keys = ("key0", "key1", "key2") - head = head_lib._multi_class_head( - n_classes=n_classes, - label_keys=label_keys) - with ops.Graph().as_default(): - model_fn_ops = head.create_model_fn_ops( - features={}, - mode=model_fn.ModeKeys.EVAL, - labels=("key2",), - train_op_fn=head_lib.no_op_train_fn, - logits=((1., 0., 0.),)) - with session.Session(): - lookup_ops.tables_initializer().run() - self.assertIsNone(model_fn_ops.train_op) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = 1.5514447 - expected_eval_metrics = { - "accuracy": 0., - "loss": expected_loss, - } - _assert_metrics(self, expected_loss, - expected_eval_metrics, model_fn_ops) - - def testMultiClassWithLabelKeysEvalAccuracy1(self): - n_classes = 3 - label_keys = ("key0", "key1", "key2") - head = head_lib._multi_class_head( - n_classes=n_classes, - label_keys=label_keys) - with ops.Graph().as_default(): - model_fn_ops = head.create_model_fn_ops( - features={}, - mode=model_fn.ModeKeys.EVAL, - labels=("key2",), - train_op_fn=head_lib.no_op_train_fn, - logits=((0., 0., 1.),)) - with session.Session(): - lookup_ops.tables_initializer().run() - self.assertIsNone(model_fn_ops.train_op) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = 0.5514447 - expected_eval_metrics = { - "accuracy": 1., - "loss": expected_loss, - } - _assert_metrics(self, expected_loss, - expected_eval_metrics, model_fn_ops) - - -class BinarySvmHeadTest(test.TestCase): - - def _assert_output_alternatives(self, model_fn_ops): - self.assertEquals({ - None: constants.ProblemType.LOGISTIC_REGRESSION - }, { - k: v[0] for k, v in six.iteritems(model_fn_ops.output_alternatives) - }) - - def setUp(self): - # Prediction for first example is in the right side of the hyperplane - # (i.e., < 0) but it is within the [-1,1] margin. There is a 0.5 loss - # incurred by this example. The 2nd prediction is outside the margin so it - # incurs no loss at all. - self._predictions = ((-.5,), (1.2,)) - self._labels = (0, 1) - self._expected_losses = (.5, 0.) - - def testBinarySVMWithLogits(self): - head = head_lib.binary_svm_head() - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, - model_fn.ModeKeys.TRAIN, - self._labels, - head_lib.no_op_train_fn, - logits=self._predictions) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = np.average(self._expected_losses) - _assert_metrics(self, expected_loss, { - "accuracy": 1., - "loss": expected_loss, - }, model_fn_ops) - - def testBinarySVMWithInvalidLogits(self): - head = head_lib.binary_svm_head() - with ops.Graph().as_default(), session.Session(): - with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"): - head.create_model_fn_ops( - {}, model_fn.ModeKeys.TRAIN, self._labels, head_lib.no_op_train_fn, - logits=np.ones((2, 2))) - - def testBinarySVMWithLogitsInput(self): - head = head_lib.binary_svm_head() - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, - model_fn.ModeKeys.TRAIN, - self._labels, - head_lib.no_op_train_fn, - logits_input=((0., 0.), (0., 0.))) - self._assert_output_alternatives(model_fn_ops) - w = ("binary_svm_head/logits/weights:0", - "binary_svm_head/logits/biases:0") - _assert_variables( - self, expected_global=w, expected_model=w, expected_trainable=w) - variables.global_variables_initializer().run() - _assert_summary_tags(self, ["loss"]) - expected_loss = 1. - _assert_metrics(self, expected_loss, { - "accuracy": .5, - "loss": expected_loss, - }, model_fn_ops) - - def testBinarySVMWithLogitsAndLogitsInput(self): - head = head_lib.binary_svm_head() - with ops.Graph().as_default(), session.Session(): - with self.assertRaisesRegexp( - ValueError, "Both logits and logits_input supplied"): - head.create_model_fn_ops( - {}, - model_fn.ModeKeys.TRAIN, - self._labels, - head_lib.no_op_train_fn, - logits_input=((0., 0.), (0., 0.)), - logits=self._predictions) - - def testBinarySVMEvalMode(self): - head = head_lib.binary_svm_head() - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, - model_fn.ModeKeys.EVAL, - self._labels, - head_lib.no_op_train_fn, - logits=self._predictions) - self._assert_output_alternatives(model_fn_ops) - self.assertIsNone(model_fn_ops.train_op) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = np.average(self._expected_losses) - _assert_metrics(self, expected_loss, { - "accuracy": 1., - "loss": expected_loss, - }, model_fn_ops) - - def testBinarySVMWithLabelName(self): - label_name = "my_label" - head = head_lib.binary_svm_head(label_name=label_name) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, - model_fn.ModeKeys.TRAIN, - {label_name: self._labels}, - head_lib.no_op_train_fn, - logits=self._predictions) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_loss = np.average(self._expected_losses) - _assert_metrics(self, expected_loss, { - "accuracy": 1., - "loss": expected_loss, - }, model_fn_ops) - - def testBinarySVMWith1DWeights(self): - head = head_lib.binary_svm_head(weight_column_name="weights") - with ops.Graph().as_default(), session.Session(): - weights = (7., 11.) - model_fn_ops = head.create_model_fn_ops( - # We have to add an extra dim here for weights broadcasting to work. - features={"weights": weights}, - mode=model_fn.ModeKeys.TRAIN, - labels=self._labels, - train_op_fn=head_lib.no_op_train_fn, - logits=self._predictions) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_weighted_losses = np.multiply(weights, self._expected_losses) - _assert_metrics(self, np.mean(expected_weighted_losses), { - "accuracy": 1., - "loss": np.sum(expected_weighted_losses) / np.sum(weights), - }, model_fn_ops) - - def testBinarySVMWith2DWeights(self): - head = head_lib.binary_svm_head(weight_column_name="weights") - with ops.Graph().as_default(), session.Session(): - weights = (7., 11.) - model_fn_ops = head.create_model_fn_ops( - # We have to add an extra dim here for weights broadcasting to work. - features={"weights": tuple([(w,) for w in weights])}, - mode=model_fn.ModeKeys.TRAIN, - labels=self._labels, - train_op_fn=head_lib.no_op_train_fn, - logits=self._predictions) - self._assert_output_alternatives(model_fn_ops) - _assert_no_variables(self) - _assert_summary_tags(self, ["loss"]) - expected_weighted_losses = np.multiply(weights, self._expected_losses) - _assert_metrics(self, np.mean(expected_weighted_losses), { - "accuracy": 1., - "loss": np.sum(expected_weighted_losses) / np.sum(weights), - }, model_fn_ops) - - def testBinarySVMWithCenteredBias(self): - head = head_lib.binary_svm_head(enable_centered_bias=True) - with ops.Graph().as_default(), session.Session(): - model_fn_ops = head.create_model_fn_ops( - {}, - model_fn.ModeKeys.TRAIN, - self._labels, - head_lib.no_op_train_fn, - logits=self._predictions) - self._assert_output_alternatives(model_fn_ops) - _assert_variables( - self, - expected_global=( - "binary_svm_head/centered_bias_weight:0", - ("binary_svm_head/binary_svm_head/centered_bias_weight/" - "Adagrad:0"), - ), - expected_trainable=("binary_svm_head/centered_bias_weight:0",)) - variables.global_variables_initializer().run() - _assert_summary_tags(self, [ - "loss", - "binary_svm_head/centered_bias/bias_0" - ]) - expected_loss = np.average(self._expected_losses) - _assert_metrics(self, expected_loss, { - "accuracy": 1., - "loss": expected_loss, - }, model_fn_ops) - - -class LossOnlyHead(test.TestCase): - - def testNoPredictionsAndNoMetrics(self): - head = head_lib.loss_only_head(lambda: 1, head_name="const") - model_fn_ops = head.create_model_fn_ops( - features={}, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn) - self.assertDictEqual(model_fn_ops.predictions, {}) - self.assertDictEqual(model_fn_ops.eval_metric_ops, {}) - self.assertIsNotNone(model_fn_ops.loss) - with session.Session() as sess: - self.assertEqual(1, sess.run(model_fn_ops.loss)) - - -class MultiHeadTest(test.TestCase): - - def testInvalidHeads(self): - named_head = head_lib.multi_class_head( - n_classes=3, label_name="label", head_name="head1") - unnamed_head = head_lib.multi_class_head( - n_classes=4, label_name="label") - with self.assertRaisesRegexp(ValueError, "must have names"): - head_lib.multi_head((named_head, unnamed_head)) - - def testTrainWithNoneTrainOpFn(self): - head1 = head_lib.multi_class_head( - n_classes=3, label_name="label1", head_name="head1") - head2 = head_lib.multi_class_head( - n_classes=4, label_name="label2", head_name="head2") - head = head_lib.multi_head((head1, head2)) - labels = { - "label1": (1,), - "label2": (1,) - } - with self.assertRaisesRegexp( - ValueError, "train_op_fn can not be None in TRAIN mode"): - head.create_model_fn_ops( - features={"weights": (2.0, 10.0)}, - labels=labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=None, - logits=((-0.7, 0.2, .1, .1, .1, .1, .1),)) - - def testTrain_withNoHeadWeights(self): - head1 = head_lib.multi_class_head( - n_classes=3, label_name="label1", head_name="head1") - head2 = head_lib.multi_class_head( - n_classes=4, label_name="label2", head_name="head2") - head3 = head_lib.loss_only_head(lambda: 1.0, head_name="const") - head = head_lib.multi_head((head1, head2, head3)) - labels = { - "label1": (1,), - "label2": (1,) - } - model_fn_ops = head.create_model_fn_ops( - features={"weights": (2.0, 10.0)}, - labels=labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=((-0.7, 0.2, .1, .1, .1, .1, .1),)) - - self.assertIsNone(model_fn_ops.predictions) - self.assertIsNotNone(model_fn_ops.loss) - self.assertIsNotNone(model_fn_ops.train_op) - self.assertTrue(model_fn_ops.eval_metric_ops) - self.assertIsNone(model_fn_ops.output_alternatives) - - with session.Session() as sess: - self.assertAlmostEqual(3.224, sess.run(model_fn_ops.loss), places=3) - - def testTrain_withHeadWeights(self): - head1 = head_lib.multi_class_head( - n_classes=3, label_name="label1", head_name="head1") - head2 = head_lib.multi_class_head( - n_classes=4, label_name="label2", head_name="head2") - head = head_lib.multi_head((head1, head2), (1, .5)) - labels = { - "label1": (1,), - "label2": (1,) - } - model_fn_ops = head.create_model_fn_ops( - features={"weights": (2.0, 10.0)}, - labels=labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits=((-0.7, 0.2, .1, .1, .1, .1, .1),)) - self.assertIsNone(model_fn_ops.predictions) - self.assertIsNotNone(model_fn_ops.loss) - self.assertIsNotNone(model_fn_ops.train_op) - self.assertTrue(model_fn_ops.eval_metric_ops) - self.assertIsNone(model_fn_ops.output_alternatives) - - with session.Session() as sess: - self.assertAlmostEqual(1.531, sess.run(model_fn_ops.loss), places=3) - - def testTrain_withDictLogits(self): - head1 = head_lib.multi_class_head( - n_classes=3, label_name="label1", head_name="head1") - head2 = head_lib.multi_class_head( - n_classes=4, label_name="label2", head_name="head2") - head = head_lib.multi_head((head1, head2)) - labels = { - "label1": (1,), - "label2": (1,) - } - model_fn_ops = head.create_model_fn_ops( - features={"weights": (2.0, 10.0)}, - labels=labels, - mode=model_fn.ModeKeys.TRAIN, - train_op_fn=head_lib.no_op_train_fn, - logits={head1.head_name: ((-0.7, 0.2, .1),), - head2.head_name: ((.1, .1, .1, .1),)}) - - self.assertIsNone(model_fn_ops.predictions) - self.assertIsNotNone(model_fn_ops.loss) - self.assertIsNotNone(model_fn_ops.train_op) - self.assertTrue(model_fn_ops.eval_metric_ops) - self.assertIsNone(model_fn_ops.output_alternatives) - - with session.Session() as sess: - self.assertAlmostEqual(2.224, sess.run(model_fn_ops.loss), places=3) - - def testInfer(self): - head1 = head_lib.multi_class_head( - n_classes=3, label_name="label1", head_name="head1") - head2 = head_lib.multi_class_head( - n_classes=4, label_name="label2", head_name="head2") - head = head_lib.multi_head((head1, head2), (1, .5)) - labels = { - "label1": (1,), - "label2": (1,) - } - model_fn_ops = head.create_model_fn_ops( - features={"weights": (2.0, 10.0)}, - labels=labels, - mode=model_fn.ModeKeys.INFER, - train_op_fn=head_lib.no_op_train_fn, - logits=((-0.7, 0.2, .1, .1, .1, .1, .1),)) - - self.assertIsNotNone(model_fn_ops.predictions) - self.assertIsNone(model_fn_ops.loss) - self.assertIsNone(model_fn_ops.train_op) - self.assertFalse(model_fn_ops.eval_metric_ops) - - # Tests predictions keys. - self.assertItemsEqual(( - ("head1", prediction_key.PredictionKey.LOGITS), - ("head1", prediction_key.PredictionKey.PROBABILITIES), - ("head1", prediction_key.PredictionKey.CLASSES), - ("head2", prediction_key.PredictionKey.LOGITS), - ("head2", prediction_key.PredictionKey.PROBABILITIES), - ("head2", prediction_key.PredictionKey.CLASSES), - ), model_fn_ops.predictions.keys()) - - # Tests output alternative. - self.assertEquals({ - "head1": constants.ProblemType.CLASSIFICATION, - "head2": constants.ProblemType.CLASSIFICATION, - }, { - k: v[0] for k, v in six.iteritems(model_fn_ops.output_alternatives) - }) - self.assertItemsEqual(( - prediction_key.PredictionKey.PROBABILITIES, - prediction_key.PredictionKey.CLASSES, - ), model_fn_ops.output_alternatives["head1"][1].keys()) - self.assertItemsEqual(( - prediction_key.PredictionKey.PROBABILITIES, - prediction_key.PredictionKey.CLASSES, - ), model_fn_ops.output_alternatives["head2"][1].keys()) - - def testEval(self): - head1 = head_lib.multi_class_head( - n_classes=3, label_name="label1", head_name="head1") - head2 = head_lib.multi_class_head( - n_classes=4, label_name="label2", head_name="head2") - head = head_lib.multi_head((head1, head2), (1, .5)) - labels = { - "label1": (1,), - "label2": (1,) - } - model_fn_ops = head.create_model_fn_ops( - features={"weights": (2.0, 10.0)}, - labels=labels, - mode=model_fn.ModeKeys.EVAL, - train_op_fn=head_lib.no_op_train_fn, - logits=((-0.7, 0.2, .1, .1, .1, .1, .1),)) - - self.assertIsNotNone(model_fn_ops.predictions) - self.assertIsNotNone(model_fn_ops.loss) - self.assertIsNone(model_fn_ops.train_op) - self.assertIsNotNone(model_fn_ops.eval_metric_ops) - self.assertIsNone(model_fn_ops.output_alternatives) - - metric_ops = model_fn_ops.eval_metric_ops - - # Tests eval keys. - self.assertIn("accuracy/head1", metric_ops.keys()) - self.assertIn("accuracy/head2", metric_ops.keys()) - - -def _sigmoid_cross_entropy(labels, logits, weights): - return losses_lib.sigmoid_cross_entropy(labels, logits, weights) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/kmeans.py b/tensorflow/contrib/learn/python/learn/estimators/kmeans.py deleted file mode 100644 index 21f7dcc5e42..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/kmeans.py +++ /dev/null @@ -1,281 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implementation of k-means clustering on top of `Estimator` API (deprecated). - -This module is deprecated. Please use -`tf.contrib.factorization.KMeansClustering` instead of -`tf.contrib.learn.KMeansClustering`. It has a similar interface, but uses the -`tf.estimator.Estimator` API instead of `tf.contrib.learn.Estimator`. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time -import numpy as np - -from tensorflow.contrib.factorization.python.ops import clustering_ops -from tensorflow.python.training import training_util -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators.model_fn import ModelFnOps -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops.control_flow_ops import with_dependencies -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.summary import summary -from tensorflow.python.training import session_run_hook -from tensorflow.python.training.session_run_hook import SessionRunArgs -from tensorflow.python.util.deprecation import deprecated - -_USE_TF_CONTRIB_FACTORIZATION = ( - 'Please use tf.contrib.factorization.KMeansClustering instead of' - ' tf.contrib.learn.KMeansClustering. It has a similar interface, but uses' - ' the tf.estimator.Estimator API instead of tf.contrib.learn.Estimator.') - - -class _LossRelativeChangeHook(session_run_hook.SessionRunHook): - """Stops when the change in loss goes below a tolerance.""" - - def __init__(self, tolerance): - """Initializes _LossRelativeChangeHook. - - Args: - tolerance: A relative tolerance of change between iterations. - """ - self._tolerance = tolerance - self._prev_loss = None - - def begin(self): - self._loss_tensor = ops.get_default_graph().get_tensor_by_name( - KMeansClustering.LOSS_OP_NAME + ':0') - assert self._loss_tensor is not None - - def before_run(self, run_context): - del run_context - return SessionRunArgs( - fetches={KMeansClustering.LOSS_OP_NAME: self._loss_tensor}) - - def after_run(self, run_context, run_values): - loss = run_values.results[KMeansClustering.LOSS_OP_NAME] - assert loss is not None - if self._prev_loss is not None: - relative_change = (abs(loss - self._prev_loss) / - (1 + abs(self._prev_loss))) - if relative_change < self._tolerance: - run_context.request_stop() - self._prev_loss = loss - - -class _InitializeClustersHook(session_run_hook.SessionRunHook): - """Initializes clusters or waits for cluster initialization.""" - - def __init__(self, init_op, is_initialized_op, is_chief): - self._init_op = init_op - self._is_chief = is_chief - self._is_initialized_op = is_initialized_op - - def after_create_session(self, session, _): - assert self._init_op.graph == ops.get_default_graph() - assert self._is_initialized_op.graph == self._init_op.graph - while True: - try: - if session.run(self._is_initialized_op): - break - elif self._is_chief: - session.run(self._init_op) - else: - time.sleep(1) - except RuntimeError as e: - logging.info(e) - - -def _parse_tensor_or_dict(features): - """Helper function to parse features.""" - if isinstance(features, dict): - keys = sorted(features.keys()) - with ops.colocate_with(features[keys[0]]): - features = array_ops.concat([features[k] for k in keys], 1) - return features - - -def _kmeans_clustering_model_fn(features, labels, mode, params, config): - """Model function for KMeansClustering estimator.""" - assert labels is None, labels - (all_scores, model_predictions, losses, - is_initialized, init_op, training_op) = clustering_ops.KMeans( - _parse_tensor_or_dict(features), - params.get('num_clusters'), - initial_clusters=params.get('training_initial_clusters'), - distance_metric=params.get('distance_metric'), - use_mini_batch=params.get('use_mini_batch'), - mini_batch_steps_per_iteration=params.get( - 'mini_batch_steps_per_iteration'), - random_seed=params.get('random_seed'), - kmeans_plus_plus_num_retries=params.get( - 'kmeans_plus_plus_num_retries')).training_graph() - incr_step = state_ops.assign_add(training_util.get_global_step(), 1) - loss = math_ops.reduce_sum(losses, name=KMeansClustering.LOSS_OP_NAME) - summary.scalar('loss/raw', loss) - training_op = with_dependencies([training_op, incr_step], loss) - predictions = { - KMeansClustering.ALL_SCORES: all_scores[0], - KMeansClustering.CLUSTER_IDX: model_predictions[0], - } - eval_metric_ops = {KMeansClustering.SCORES: loss} - training_hooks = [_InitializeClustersHook( - init_op, is_initialized, config.is_chief)] - relative_tolerance = params.get('relative_tolerance') - if relative_tolerance is not None: - training_hooks.append(_LossRelativeChangeHook(relative_tolerance)) - return ModelFnOps( - mode=mode, - predictions=predictions, - eval_metric_ops=eval_metric_ops, - loss=loss, - train_op=training_op, - training_hooks=training_hooks) - - -# TODO(agarwal,ands): support sharded input. -class KMeansClustering(estimator.Estimator): - """An Estimator for K-Means clustering. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - SQUARED_EUCLIDEAN_DISTANCE = clustering_ops.SQUARED_EUCLIDEAN_DISTANCE - COSINE_DISTANCE = clustering_ops.COSINE_DISTANCE - RANDOM_INIT = clustering_ops.RANDOM_INIT - KMEANS_PLUS_PLUS_INIT = clustering_ops.KMEANS_PLUS_PLUS_INIT - SCORES = 'scores' - CLUSTER_IDX = 'cluster_idx' - CLUSTERS = 'clusters' - ALL_SCORES = 'all_scores' - LOSS_OP_NAME = 'kmeans_loss' - - @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION) - def __init__(self, - num_clusters, - model_dir=None, - initial_clusters=RANDOM_INIT, - distance_metric=SQUARED_EUCLIDEAN_DISTANCE, - random_seed=0, - use_mini_batch=True, - mini_batch_steps_per_iteration=1, - kmeans_plus_plus_num_retries=2, - relative_tolerance=None, - config=None): - """Creates a model for running KMeans training and inference. - - Args: - num_clusters: number of clusters to train. - model_dir: the directory to save the model results and log files. - initial_clusters: specifies how to initialize the clusters for training. - See clustering_ops.kmeans for the possible values. - distance_metric: the distance metric used for clustering. - See clustering_ops.kmeans for the possible values. - random_seed: Python integer. Seed for PRNG used to initialize centers. - use_mini_batch: If true, use the mini-batch k-means algorithm. Else assume - full batch. - mini_batch_steps_per_iteration: number of steps after which the updated - cluster centers are synced back to a master copy. See clustering_ops.py - for more details. - kmeans_plus_plus_num_retries: For each point that is sampled during - kmeans++ initialization, this parameter specifies the number of - additional points to draw from the current distribution before selecting - the best. If a negative value is specified, a heuristic is used to - sample O(log(num_to_sample)) additional points. - relative_tolerance: A relative tolerance of change in the loss between - iterations. Stops learning if the loss changes less than this amount. - Note that this may not work correctly if use_mini_batch=True. - config: See Estimator - """ - params = {} - params['num_clusters'] = num_clusters - params['training_initial_clusters'] = initial_clusters - params['distance_metric'] = distance_metric - params['random_seed'] = random_seed - params['use_mini_batch'] = use_mini_batch - params['mini_batch_steps_per_iteration'] = mini_batch_steps_per_iteration - params['kmeans_plus_plus_num_retries'] = kmeans_plus_plus_num_retries - params['relative_tolerance'] = relative_tolerance - super(KMeansClustering, self).__init__( - model_fn=_kmeans_clustering_model_fn, - params=params, - model_dir=model_dir, - config=config) - - @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION) - def predict_cluster_idx(self, input_fn=None): - """Yields predicted cluster indices.""" - key = KMeansClustering.CLUSTER_IDX - results = super(KMeansClustering, self).predict( - input_fn=input_fn, outputs=[key]) - for result in results: - yield result[key] - - @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION) - def score(self, input_fn=None, steps=None): - """Predict total sum of distances to nearest clusters. - - Note that this function is different from the corresponding one in sklearn - which returns the negative of the sum of distances. - - Args: - input_fn: see predict. - steps: see predict. - - Returns: - Total sum of distances to nearest clusters. - """ - return np.sum( - self.evaluate( - input_fn=input_fn, steps=steps)[KMeansClustering.SCORES]) - - @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION) - def transform(self, input_fn=None, as_iterable=False): - """Transforms each element to distances to cluster centers. - - Note that this function is different from the corresponding one in sklearn. - For SQUARED_EUCLIDEAN distance metric, sklearn transform returns the - EUCLIDEAN distance, while this function returns the SQUARED_EUCLIDEAN - distance. - - Args: - input_fn: see predict. - as_iterable: see predict - - Returns: - Array with same number of rows as x, and num_clusters columns, containing - distances to the cluster centers. - """ - key = KMeansClustering.ALL_SCORES - results = super(KMeansClustering, self).predict( - input_fn=input_fn, - outputs=[key], - as_iterable=as_iterable) - if not as_iterable: - return results[key] - else: - return results - - @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION) - def clusters(self): - """Returns cluster centers.""" - return super(KMeansClustering, self).get_variable_value(self.CLUSTERS) diff --git a/tensorflow/contrib/learn/python/learn/estimators/kmeans_test.py b/tensorflow/contrib/learn/python/learn/estimators/kmeans_test.py deleted file mode 100644 index 584556992a0..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/kmeans_test.py +++ /dev/null @@ -1,585 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for KMeans.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import time - -import numpy as np -from sklearn.cluster import KMeans as SklearnKMeans - -# pylint: disable=g-import-not-at-top -from tensorflow.contrib.learn.python import learn -from tensorflow.contrib.learn.python.learn.estimators import kmeans as kmeans_lib -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import benchmark -from tensorflow.python.platform import flags -from tensorflow.python.platform import test -from tensorflow.python.training import input as input_lib -from tensorflow.python.training import queue_runner - -FLAGS = flags.FLAGS - - -def normalize(x): - return x / np.sqrt(np.sum(x * x, axis=-1, keepdims=True)) - - -def cosine_similarity(x, y): - return np.dot(normalize(x), np.transpose(normalize(y))) - - -def make_random_centers(num_centers, num_dims, center_norm=500): - return np.round( - np.random.rand(num_centers, num_dims).astype(np.float32) * center_norm) - - -def make_random_points(centers, num_points, max_offset=20): - num_centers, num_dims = centers.shape - assignments = np.random.choice(num_centers, num_points) - offsets = np.round( - np.random.randn(num_points, num_dims).astype(np.float32) * max_offset) - return (centers[assignments] + offsets, assignments, np.add.reduce( - offsets * offsets, 1)) - - -class KMeansTestBase(test.TestCase): - - def input_fn(self, - batch_size=None, - points=None, - randomize=None, - num_epochs=None): - """Returns an input_fn that randomly selects batches from given points.""" - batch_size = batch_size or self.batch_size - points = points if points is not None else self.points - num_points = points.shape[0] - if randomize is None: - randomize = (self.use_mini_batch and - self.mini_batch_steps_per_iteration <= 1) - - def _fn(): - x = constant_op.constant(points) - if batch_size == num_points: - return input_lib.limit_epochs(x, num_epochs=num_epochs), None - if randomize: - indices = random_ops.random_uniform( - constant_op.constant([batch_size]), - minval=0, - maxval=num_points - 1, - dtype=dtypes.int32, - seed=10) - else: - # We need to cycle through the indices sequentially. We create a queue - # to maintain the list of indices. - q = data_flow_ops.FIFOQueue(num_points, dtypes.int32, ()) - - # Conditionally initialize the Queue. - def _init_q(): - with ops.control_dependencies( - [q.enqueue_many(math_ops.range(num_points))]): - return control_flow_ops.no_op() - - init_q = control_flow_ops.cond(q.size() <= 0, _init_q, - control_flow_ops.no_op) - with ops.control_dependencies([init_q]): - offsets = q.dequeue_many(batch_size) - with ops.control_dependencies([q.enqueue_many(offsets)]): - indices = array_ops.identity(offsets) - batch = array_ops.gather(x, indices) - return (input_lib.limit_epochs(batch, num_epochs=num_epochs), None) - - return _fn - - @staticmethod - def config(tf_random_seed): - return run_config.RunConfig(tf_random_seed=tf_random_seed) - - @property - def initial_clusters(self): - return kmeans_lib.KMeansClustering.KMEANS_PLUS_PLUS_INIT - - @property - def batch_size(self): - return self.num_points - - @property - def use_mini_batch(self): - return False - - @property - def mini_batch_steps_per_iteration(self): - return 1 - - -class KMeansTest(KMeansTestBase): - - def setUp(self): - np.random.seed(3) - self.num_centers = 5 - self.num_dims = 2 - self.num_points = 1000 - self.true_centers = make_random_centers(self.num_centers, self.num_dims) - self.points, _, self.scores = make_random_points(self.true_centers, - self.num_points) - self.true_score = np.add.reduce(self.scores) - - def _kmeans(self, relative_tolerance=None): - return kmeans_lib.KMeansClustering( - self.num_centers, - initial_clusters=self.initial_clusters, - distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE, - use_mini_batch=self.use_mini_batch, - mini_batch_steps_per_iteration=self.mini_batch_steps_per_iteration, - random_seed=24, - relative_tolerance=relative_tolerance) - - def test_clusters(self): - kmeans = self._kmeans() - kmeans.fit(input_fn=self.input_fn(), steps=1) - clusters = kmeans.clusters() - self.assertAllEqual(list(clusters.shape), [self.num_centers, self.num_dims]) - - def test_fit(self): - kmeans = self._kmeans() - kmeans.fit(input_fn=self.input_fn(), steps=1) - score1 = kmeans.score( - input_fn=self.input_fn(batch_size=self.num_points), steps=1) - steps = 10 * self.num_points // self.batch_size - kmeans.fit(input_fn=self.input_fn(), steps=steps) - score2 = kmeans.score( - input_fn=self.input_fn(batch_size=self.num_points), steps=1) - self.assertTrue(score1 > score2) - self.assertNear(self.true_score, score2, self.true_score * 0.05) - - def test_monitor(self): - if self.use_mini_batch: - # We don't test for use_mini_batch case since the loss value can be noisy. - return - kmeans = kmeans_lib.KMeansClustering( - self.num_centers, - initial_clusters=self.initial_clusters, - distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE, - use_mini_batch=self.use_mini_batch, - mini_batch_steps_per_iteration=self.mini_batch_steps_per_iteration, - config=learn.RunConfig(tf_random_seed=14), - random_seed=12, - relative_tolerance=1e-4) - - kmeans.fit( - input_fn=self.input_fn(), - # Force it to train until the relative tolerance monitor stops it. - steps=None) - score = kmeans.score( - input_fn=self.input_fn(batch_size=self.num_points), steps=1) - self.assertNear(self.true_score, score, self.true_score * 0.01) - - def _infer_helper(self, kmeans, clusters, num_points): - points, true_assignments, true_offsets = make_random_points( - clusters, num_points) - # Test predict - assignments = list( - kmeans.predict_cluster_idx(input_fn=self.input_fn( - batch_size=num_points, points=points, num_epochs=1))) - self.assertAllEqual(assignments, true_assignments) - - # Test score - score = kmeans.score( - input_fn=lambda: (constant_op.constant(points), None), steps=1) - self.assertNear(score, np.sum(true_offsets), 0.01 * score) - - # Test transform - transform = kmeans.transform( - input_fn=lambda: (constant_op.constant(points), None)) - true_transform = np.maximum( - 0, - np.sum(np.square(points), axis=1, - keepdims=True) - 2 * np.dot(points, np.transpose(clusters)) + - np.transpose(np.sum(np.square(clusters), axis=1, keepdims=True))) - self.assertAllClose(transform, true_transform, rtol=0.05, atol=10) - - def test_infer(self): - kmeans = self._kmeans() - # Make a call to fit to initialize the cluster centers. - max_steps = 1 - kmeans.fit(input_fn=self.input_fn(), max_steps=max_steps) - clusters = kmeans.clusters() - - # Run inference on small datasets. - self._infer_helper(kmeans, clusters, num_points=10) - self._infer_helper(kmeans, clusters, num_points=1) - - -class KMeansTestMultiStageInit(KMeansTestBase): - - def test_random(self): - points = np.array( - [[1, 2], [3, 4], [5, 6], [7, 8], [9, 0]], dtype=np.float32) - kmeans = kmeans_lib.KMeansClustering( - num_clusters=points.shape[0], - initial_clusters=kmeans_lib.KMeansClustering.RANDOM_INIT, - distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE, - use_mini_batch=True, - mini_batch_steps_per_iteration=100, - random_seed=24, - relative_tolerance=None) - kmeans.fit( - input_fn=self.input_fn(batch_size=1, points=points, randomize=False), - steps=1) - clusters = kmeans.clusters() - self.assertAllEqual(points, clusters) - - def test_kmeans_plus_plus_batch_just_right(self): - points = np.array([[1, 2]], dtype=np.float32) - kmeans = kmeans_lib.KMeansClustering( - num_clusters=points.shape[0], - initial_clusters=kmeans_lib.KMeansClustering.KMEANS_PLUS_PLUS_INIT, - distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE, - use_mini_batch=True, - mini_batch_steps_per_iteration=100, - random_seed=24, - relative_tolerance=None) - kmeans.fit( - input_fn=self.input_fn(batch_size=1, points=points, randomize=False), - steps=1) - clusters = kmeans.clusters() - self.assertAllEqual(points, clusters) - - def test_kmeans_plus_plus_batch_too_small(self): - points = np.array( - [[1, 2], [3, 4], [5, 6], [7, 8], [9, 0]], dtype=np.float32) - kmeans = kmeans_lib.KMeansClustering( - num_clusters=points.shape[0], - initial_clusters=kmeans_lib.KMeansClustering.KMEANS_PLUS_PLUS_INIT, - distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE, - use_mini_batch=True, - mini_batch_steps_per_iteration=100, - random_seed=24, - relative_tolerance=None) - with self.assertRaisesOpError(AssertionError): - kmeans.fit( - input_fn=self.input_fn(batch_size=4, points=points, randomize=False), - steps=1) - - -class MiniBatchKMeansTest(KMeansTest): - - @property - def batch_size(self): - return 50 - - @property - def use_mini_batch(self): - return True - - -class FullBatchAsyncKMeansTest(KMeansTest): - - @property - def batch_size(self): - return 50 - - @property - def use_mini_batch(self): - return True - - @property - def mini_batch_steps_per_iteration(self): - return self.num_points // self.batch_size - - -class KMeansCosineDistanceTest(KMeansTestBase): - - def setUp(self): - self.points = np.array( - [[2.5, 0.1], [2, 0.2], [3, 0.1], [4, 0.2], [0.1, 2.5], [0.2, 2], - [0.1, 3], [0.2, 4]], - dtype=np.float32) - self.num_points = self.points.shape[0] - self.true_centers = np.array( - [ - normalize( - np.mean(normalize(self.points)[0:4, :], axis=0, keepdims=True))[ - 0], - normalize( - np.mean(normalize(self.points)[4:, :], axis=0, keepdims=True))[ - 0] - ], - dtype=np.float32) - self.true_assignments = np.array([0] * 4 + [1] * 4) - self.true_score = len(self.points) - np.tensordot( - normalize(self.points), self.true_centers[self.true_assignments]) - - self.num_centers = 2 - self.kmeans = kmeans_lib.KMeansClustering( - self.num_centers, - initial_clusters=kmeans_lib.KMeansClustering.RANDOM_INIT, - distance_metric=kmeans_lib.KMeansClustering.COSINE_DISTANCE, - use_mini_batch=self.use_mini_batch, - mini_batch_steps_per_iteration=self.mini_batch_steps_per_iteration, - config=self.config(3)) - - def test_fit(self): - max_steps = 10 * self.num_points // self.batch_size - self.kmeans.fit(input_fn=self.input_fn(), max_steps=max_steps) - centers = normalize(self.kmeans.clusters()) - centers = centers[centers[:, 0].argsort()] - true_centers = self.true_centers[self.true_centers[:, 0].argsort()] - self.assertAllClose(centers, true_centers, atol=0.04) - - def test_transform(self): - self.kmeans.fit(input_fn=self.input_fn(), steps=10) - centers = normalize(self.kmeans.clusters()) - true_transform = 1 - cosine_similarity(self.points, centers) - transform = self.kmeans.transform(input_fn=self.input_fn( - batch_size=self.num_points)) - self.assertAllClose(transform, true_transform, atol=1e-3) - - def test_predict(self): - max_steps = 10 * self.num_points // self.batch_size - self.kmeans.fit(input_fn=self.input_fn(), max_steps=max_steps) - centers = normalize(self.kmeans.clusters()) - - assignments = list( - self.kmeans.predict_cluster_idx(input_fn=self.input_fn( - num_epochs=1, batch_size=self.num_points))) - self.assertAllClose( - centers[assignments], - self.true_centers[self.true_assignments], - atol=1e-2) - - centers = centers[centers[:, 0].argsort()] - true_centers = self.true_centers[self.true_centers[:, 0].argsort()] - self.assertAllClose(centers, true_centers, atol=0.04) - score = self.kmeans.score( - input_fn=self.input_fn(batch_size=self.num_points), steps=1) - self.assertAllClose(score, self.true_score, atol=1e-2) - - def test_predict_kmeans_plus_plus(self): - # Most points are concetrated near one center. KMeans++ is likely to find - # the less populated centers. - points = np.array( - [[2.5, 3.5], [2.5, 3.5], [-2, 3], [-2, 3], [-3, -3], [-3.1, -3.2], - [-2.8, -3.], [-2.9, -3.1], [-3., -3.1], [-3., -3.1], [-3.2, -3.], - [-3., -3.]], - dtype=np.float32) - true_centers = np.array( - [ - normalize( - np.mean(normalize(points)[0:2, :], axis=0, keepdims=True))[0], - normalize( - np.mean(normalize(points)[2:4, :], axis=0, keepdims=True))[0], - normalize( - np.mean(normalize(points)[4:, :], axis=0, keepdims=True))[0] - ], - dtype=np.float32) - true_assignments = [0] * 2 + [1] * 2 + [2] * 8 - true_score = len(points) - np.tensordot( - normalize(points), true_centers[true_assignments]) - - kmeans = kmeans_lib.KMeansClustering( - 3, - initial_clusters=self.initial_clusters, - distance_metric=kmeans_lib.KMeansClustering.COSINE_DISTANCE, - use_mini_batch=self.use_mini_batch, - mini_batch_steps_per_iteration=self.mini_batch_steps_per_iteration, - config=self.config(3)) - kmeans.fit(input_fn=lambda: (constant_op.constant(points), None), steps=30) - - centers = normalize(kmeans.clusters()) - self.assertAllClose( - sorted(centers.tolist()), sorted(true_centers.tolist()), atol=1e-2) - - def _input_fn(): - return (input_lib.limit_epochs( - constant_op.constant(points), num_epochs=1), None) - - assignments = list(kmeans.predict_cluster_idx(input_fn=_input_fn)) - self.assertAllClose( - centers[assignments], true_centers[true_assignments], atol=1e-2) - - score = kmeans.score( - input_fn=lambda: (constant_op.constant(points), None), steps=1) - self.assertAllClose(score, true_score, atol=1e-2) - - -class MiniBatchKMeansCosineTest(KMeansCosineDistanceTest): - - @property - def batch_size(self): - return 2 - - @property - def use_mini_batch(self): - return True - - -class FullBatchAsyncKMeansCosineTest(KMeansCosineDistanceTest): - - @property - def batch_size(self): - return 2 - - @property - def use_mini_batch(self): - return True - - @property - def mini_batch_steps_per_iteration(self): - return self.num_points // self.batch_size - - -class KMeansBenchmark(benchmark.Benchmark): - """Base class for benchmarks.""" - - def SetUp(self, - dimension=50, - num_clusters=50, - points_per_cluster=10000, - center_norm=500, - cluster_width=20): - np.random.seed(123456) - self.num_clusters = num_clusters - self.num_points = num_clusters * points_per_cluster - self.centers = make_random_centers( - self.num_clusters, dimension, center_norm=center_norm) - self.points, _, scores = make_random_points( - self.centers, self.num_points, max_offset=cluster_width) - self.score = float(np.sum(scores)) - - def _report(self, num_iters, start, end, scores): - print(scores) - self.report_benchmark( - iters=num_iters, - wall_time=(end - start) / num_iters, - extras={'true_sum_squared_distances': self.score, - 'fit_scores': scores}) - - def _fit(self, num_iters=10): - pass - - def benchmark_01_2dim_5center_500point(self): - self.SetUp(dimension=2, num_clusters=5, points_per_cluster=100) - self._fit() - - def benchmark_02_20dim_20center_10kpoint(self): - self.SetUp(dimension=20, num_clusters=20, points_per_cluster=500) - self._fit() - - def benchmark_03_100dim_50center_50kpoint(self): - self.SetUp(dimension=100, num_clusters=50, points_per_cluster=1000) - self._fit() - - def benchmark_03_100dim_50center_50kpoint_unseparated(self): - self.SetUp( - dimension=100, - num_clusters=50, - points_per_cluster=1000, - cluster_width=250) - self._fit() - - def benchmark_04_100dim_500center_500kpoint(self): - self.SetUp(dimension=100, num_clusters=500, points_per_cluster=1000) - self._fit(num_iters=4) - - def benchmark_05_100dim_500center_500kpoint_unseparated(self): - self.SetUp( - dimension=100, - num_clusters=500, - points_per_cluster=1000, - cluster_width=250) - self._fit(num_iters=4) - - -class TensorflowKMeansBenchmark(KMeansBenchmark): - - def _fit(self, num_iters=10): - scores = [] - start = time.time() - for i in range(num_iters): - print('Starting tensorflow KMeans: %d' % i) - tf_kmeans = kmeans_lib.KMeansClustering( - self.num_clusters, - initial_clusters=kmeans_lib.KMeansClustering.KMEANS_PLUS_PLUS_INIT, - kmeans_plus_plus_num_retries=int(math.log(self.num_clusters) + 2), - random_seed=i * 42, - relative_tolerance=1e-6, - config=run_config.RunConfig(tf_random_seed=3)) - tf_kmeans.fit( - input_fn=lambda: (constant_op.constant(self.points), None), steps=50) - _ = tf_kmeans.clusters() - scores.append( - tf_kmeans.score( - input_fn=lambda: (constant_op.constant(self.points), None), - steps=1)) - self._report(num_iters, start, time.time(), scores) - - -class SklearnKMeansBenchmark(KMeansBenchmark): - - def _fit(self, num_iters=10): - scores = [] - start = time.time() - for i in range(num_iters): - print('Starting sklearn KMeans: %d' % i) - sklearn_kmeans = SklearnKMeans( - n_clusters=self.num_clusters, - init='k-means++', - max_iter=50, - n_init=1, - tol=1e-4, - random_state=i * 42) - sklearn_kmeans.fit(self.points) - scores.append(sklearn_kmeans.inertia_) - self._report(num_iters, start, time.time(), scores) - - -class KMeansTestQueues(test.TestCase): - - def input_fn(self): - - def _fn(): - queue = data_flow_ops.FIFOQueue( - capacity=10, dtypes=dtypes.float32, shapes=[10, 3]) - enqueue_op = queue.enqueue(array_ops.zeros([10, 3], dtype=dtypes.float32)) - queue_runner.add_queue_runner( - queue_runner.QueueRunner(queue, [enqueue_op])) - return queue.dequeue(), None - - return _fn - - # This test makes sure that there are no deadlocks when using a QueueRunner. - # Note that since cluster initialization is dependendent on inputs, if input - # is generated using a QueueRunner, one has to make sure that these runners - # are started before the initialization. - def test_queues(self): - kmeans = kmeans_lib.KMeansClustering(5) - kmeans.fit(input_fn=self.input_fn(), steps=1) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear.py b/tensorflow/contrib/learn/python/learn/estimators/linear.py deleted file mode 100644 index d2ee0ebfa48..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/linear.py +++ /dev/null @@ -1,982 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Linear Estimators (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -import six - -from tensorflow.contrib import layers -from tensorflow.contrib.framework import deprecated -from tensorflow.contrib.framework import deprecated_arg_values -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.contrib.learn.python.learn.utils import export -from tensorflow.contrib.linear_optimizer.python import sdca_optimizer -from tensorflow.python.feature_column import feature_column_lib as fc_core -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import gradients -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import session_run_hook -from tensorflow.python.training import training as train -from tensorflow.python.training import training_util - - -# The default learning rate of 0.2 is a historical artifact of the initial -# implementation, but seems a reasonable choice. -_LEARNING_RATE = 0.2 - - -def _get_optimizer(spec): - if isinstance(spec, six.string_types): - return layers.OPTIMIZER_CLS_NAMES[spec]( - learning_rate=_LEARNING_RATE) - elif callable(spec): - return spec() - return spec - - -# TODO(ispir): Remove this function by fixing '_infer_model' with single outputs -# and as_iteable case. -def _as_iterable(preds, output): - for pred in preds: - yield pred[output] - - -def _add_bias_column(feature_columns, columns_to_tensors, bias_variable, - columns_to_variables): - """Adds a fake bias feature column filled with all 1s.""" - # TODO(b/31008490): Move definition to a common constants place. - bias_column_name = "tf_virtual_bias_column" - if any(col.name is bias_column_name for col in feature_columns): - raise ValueError("%s is a reserved column name." % bias_column_name) - if not feature_columns: - raise ValueError("feature_columns can't be empty.") - - # Loop through input tensors until we can figure out batch_size. - batch_size = None - for column in columns_to_tensors.values(): - if isinstance(column, tuple): - column = column[0] - if isinstance(column, sparse_tensor.SparseTensor): - shape = tensor_util.constant_value(column.dense_shape) - if shape is not None: - batch_size = shape[0] - break - else: - batch_size = array_ops.shape(column)[0] - break - if batch_size is None: - raise ValueError("Could not infer batch size from input features.") - - bias_column = layers.real_valued_column(bias_column_name) - columns_to_tensors[bias_column] = array_ops.ones([batch_size, 1], - dtype=dtypes.float32) - columns_to_variables[bias_column] = [bias_variable] - - -def _linear_model_fn(features, labels, mode, params, config=None): - """A model_fn for linear models that use a gradient-based optimizer. - - Args: - features: `Tensor` or dict of `Tensor` (depends on data passed to `fit`). - labels: `Tensor` of shape [batch_size, 1] or [batch_size] labels of - dtype `int32` or `int64` in the range `[0, n_classes)`. - mode: Defines whether this is training, evaluation or prediction. - See `ModeKeys`. - params: A dict of hyperparameters. - The following hyperparameters are expected: - * head: A `Head` instance. - * feature_columns: An iterable containing all the feature columns used by - the model. - * optimizer: string, `Optimizer` object, or callable that defines the - optimizer to use for training. If `None`, will use a FTRL optimizer. - * gradient_clip_norm: A float > 0. If provided, gradients are - clipped to their global norm with this clipping ratio. - * joint_weights: If True, the weights for all columns will be stored in a - single (possibly partitioned) variable. It's more efficient, but it's - incompatible with SDCAOptimizer, and requires all feature columns are - sparse and use the 'sum' combiner. - config: `RunConfig` object to configure the runtime settings. - - Returns: - A `ModelFnOps` instance. - - Raises: - ValueError: If mode is not any of the `ModeKeys`. - """ - head = params["head"] - feature_columns = params["feature_columns"] - optimizer = params.get("optimizer") or _get_default_optimizer(feature_columns) - gradient_clip_norm = params.get("gradient_clip_norm", None) - num_ps_replicas = config.num_ps_replicas if config else 0 - joint_weights = params.get("joint_weights", False) - - if not isinstance(features, dict): - features = {"": features} - - parent_scope = "linear" - partitioner = partitioned_variables.min_max_variable_partitioner( - max_partitions=num_ps_replicas, - min_slice_size=64 << 20) - - with variable_scope.variable_scope( - parent_scope, - values=tuple(six.itervalues(features)), - partitioner=partitioner) as scope: - if all(isinstance(fc, feature_column._FeatureColumn) # pylint: disable=protected-access - for fc in feature_columns): - if joint_weights: - layer_fn = layers.joint_weighted_sum_from_feature_columns - else: - layer_fn = layers.weighted_sum_from_feature_columns - logits, _, _ = layer_fn( - columns_to_tensors=features, - feature_columns=feature_columns, - num_outputs=head.logits_dimension, - weight_collections=[parent_scope], - scope=scope) - else: - logits = fc_core.linear_model( - features=features, - feature_columns=feature_columns, - units=head.logits_dimension, - weight_collections=[parent_scope]) - - def _train_op_fn(loss): - global_step = training_util.get_global_step() - my_vars = ops.get_collection(parent_scope) - grads = gradients.gradients(loss, my_vars) - if gradient_clip_norm: - grads, _ = clip_ops.clip_by_global_norm(grads, gradient_clip_norm) - return (_get_optimizer(optimizer).apply_gradients( - zip(grads, my_vars), global_step=global_step)) - - return head.create_model_fn_ops( - features=features, - mode=mode, - labels=labels, - train_op_fn=_train_op_fn, - logits=logits) - - -def sdca_model_fn(features, labels, mode, params): - """A model_fn for linear models that use the SDCA optimizer. - - Args: - features: A dict of `Tensor` keyed by column name. - labels: `Tensor` of shape [batch_size, 1] or [batch_size] labels of - dtype `int32` or `int64` in the range `[0, n_classes)`. - mode: Defines whether this is training, evaluation or prediction. - See `ModeKeys`. - params: A dict of hyperparameters. - The following hyperparameters are expected: - * head: A `Head` instance. Type must be one of `_BinarySvmHead`, - `_RegressionHead` or `_BinaryLogisticHead`. - * feature_columns: An iterable containing all the feature columns used by - the model. - * optimizer: An `SDCAOptimizer` instance. - * weight_column_name: A string defining the weight feature column, or - None if there are no weights. - * update_weights_hook: A `SessionRunHook` object or None. Used to update - model weights. - - Returns: - A `ModelFnOps` instance. - - Raises: - ValueError: If `optimizer` is not an `SDCAOptimizer` instance. - ValueError: If the type of head is neither `_BinarySvmHead`, nor - `_RegressionHead` nor `_MultiClassHead`. - ValueError: If mode is not any of the `ModeKeys`. - """ - head = params["head"] - feature_columns = params["feature_columns"] - optimizer = params["optimizer"] - weight_column_name = params["weight_column_name"] - update_weights_hook = params.get("update_weights_hook", None) - - if not isinstance(optimizer, sdca_optimizer.SDCAOptimizer): - raise ValueError("Optimizer must be of type SDCAOptimizer") - - if isinstance(head, head_lib._BinarySvmHead): # pylint: disable=protected-access - loss_type = "hinge_loss" - elif isinstance(head, head_lib._BinaryLogisticHead): # pylint: disable=protected-access - loss_type = "logistic_loss" - elif isinstance(head, head_lib._RegressionHead): # pylint: disable=protected-access - assert head.logits_dimension == 1, ("SDCA only applies for " - "logits_dimension=1.") - loss_type = "squared_loss" - else: - raise ValueError("Unsupported head type: {}".format(head)) - - parent_scope = "linear" - - with variable_scope.variable_scope( - values=features.values(), - name_or_scope=parent_scope, - partitioner=optimizer.partitioner) as scope: - features = features.copy() - features.update(layers.transform_features(features, feature_columns)) - logits, columns_to_variables, bias = ( - layers.weighted_sum_from_feature_columns( - columns_to_tensors=features, - feature_columns=feature_columns, - num_outputs=1, - scope=scope)) - - _add_bias_column(feature_columns, features, bias, columns_to_variables) - - def _train_op_fn(unused_loss): - global_step = training_util.get_global_step() - sdca_model, train_op = optimizer.get_train_step(columns_to_variables, - weight_column_name, - loss_type, features, - labels, global_step) - if update_weights_hook is not None: - update_weights_hook.set_parameters(sdca_model, train_op) - return train_op - - model_fn_ops = head.create_model_fn_ops( - features=features, - labels=labels, - mode=mode, - train_op_fn=_train_op_fn, - logits=logits) - if update_weights_hook is not None: - return model_fn_ops._replace( - training_chief_hooks=(model_fn_ops.training_chief_hooks + - [update_weights_hook])) - return model_fn_ops - - -# Ensures consistency with LinearComposableModel. -def _get_default_optimizer(feature_columns): - learning_rate = min(_LEARNING_RATE, 1.0 / math.sqrt(len(feature_columns))) - return train.FtrlOptimizer(learning_rate=learning_rate) - - -class _SdcaUpdateWeightsHook(session_run_hook.SessionRunHook): - """SessionRunHook to update and shrink SDCA model weights.""" - - def __init__(self): - pass - - def set_parameters(self, sdca_model, train_op): - self._sdca_model = sdca_model - self._train_op = train_op - - def begin(self): - """Construct the update_weights op. - - The op is implicitly added to the default graph. - """ - self._update_op = self._sdca_model.update_weights(self._train_op) - - def before_run(self, run_context): - """Return the update_weights op so that it is executed during this run.""" - return session_run_hook.SessionRunArgs(self._update_op) - - -class LinearClassifier(estimator.Estimator): - """Linear classifier model. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Train a linear model to classify instances into one of multiple possible - classes. When number of possible classes is 2, this is binary classification. - - Example: - - ```python - sparse_column_a = sparse_column_with_hash_bucket(...) - sparse_column_b = sparse_column_with_hash_bucket(...) - - sparse_feature_a_x_sparse_feature_b = crossed_column(...) - - # Estimator using the default optimizer. - estimator = LinearClassifier( - feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b]) - - # Or estimator using the FTRL optimizer with regularization. - estimator = LinearClassifier( - feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b], - optimizer=tf.compat.v1.train.FtrlOptimizer( - learning_rate=0.1, - l1_regularization_strength=0.001 - )) - - # Or estimator using the SDCAOptimizer. - estimator = LinearClassifier( - feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b], - optimizer=tf.contrib.linear_optimizer.SDCAOptimizer( - example_id_column='example_id', - num_loss_partitions=..., - symmetric_l2_regularization=2.0 - )) - - # Input builders - def input_fn_train: # returns x, y (where y represents label's class index). - ... - def input_fn_eval: # returns x, y (where y represents label's class index). - ... - def input_fn_predict: # returns x, None. - ... - estimator.fit(input_fn=input_fn_train) - estimator.evaluate(input_fn=input_fn_eval) - # predict_classes returns class indices. - estimator.predict_classes(input_fn=input_fn_predict) - ``` - - If the user specifies `label_keys` in constructor, labels must be strings from - the `label_keys` vocabulary. Example: - - ```python - label_keys = ['label0', 'label1', 'label2'] - estimator = LinearClassifier( - n_classes=n_classes, - feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b], - label_keys=label_keys) - - def input_fn_train: # returns x, y (where y is one of label_keys). - pass - estimator.fit(input_fn=input_fn_train) - - def input_fn_eval: # returns x, y (where y is one of label_keys). - pass - estimator.evaluate(input_fn=input_fn_eval) - def input_fn_predict: # returns x, None - # predict_classes returns one of label_keys. - estimator.predict_classes(input_fn=input_fn_predict) - ``` - - Input of `fit` and `evaluate` should have following features, - otherwise there will be a `KeyError`: - - * if `weight_column_name` is not `None`, a feature with - `key=weight_column_name` whose value is a `Tensor`. - * for each `column` in `feature_columns`: - - if `column` is a `SparseColumn`, a feature with `key=column.name` - whose `value` is a `SparseTensor`. - - if `column` is a `WeightedSparseColumn`, two features: the first with - `key` the id column name, the second with `key` the weight column name. - Both features' `value` must be a `SparseTensor`. - - if `column` is a `RealValuedColumn`, a feature with `key=column.name` - whose `value` is a `Tensor`. - """ - - def __init__(self, # _joint_weight pylint: disable=invalid-name - feature_columns, - model_dir=None, - n_classes=2, - weight_column_name=None, - optimizer=None, - gradient_clip_norm=None, - enable_centered_bias=False, - _joint_weight=False, - config=None, - feature_engineering_fn=None, - label_keys=None): - """Construct a `LinearClassifier` estimator object. - - Args: - feature_columns: An iterable containing all the feature columns used by - the model. All items in the set should be instances of classes derived - from `FeatureColumn`. - model_dir: Directory to save model parameters, graph and etc. This can - also be used to load checkpoints from the directory into a estimator - to continue training a previously saved model. - n_classes: number of label classes. Default is binary classification. - Note that class labels are integers representing the class index (i.e. - values from 0 to n_classes-1). For arbitrary label values (e.g. string - labels), convert to class indices first. - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - optimizer: The optimizer used to train the model. If specified, it should - be either an instance of `tf.Optimizer` or the SDCAOptimizer. If `None`, - the Ftrl optimizer will be used. - gradient_clip_norm: A `float` > 0. If provided, gradients are clipped - to their global norm with this clipping ratio. See - `tf.clip_by_global_norm` for more details. - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - _joint_weight: If True, the weights for all columns will be stored in a - single (possibly partitioned) variable. It's more efficient, but it's - incompatible with SDCAOptimizer, and requires all feature columns are - sparse and use the 'sum' combiner. - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and - returns features and labels which will be fed - into the model. - label_keys: Optional list of strings with size `[n_classes]` defining the - label vocabulary. Only supported for `n_classes` > 2. - - Returns: - A `LinearClassifier` estimator. - - Raises: - ValueError: if n_classes < 2. - ValueError: if enable_centered_bias=True and optimizer is SDCAOptimizer. - """ - if (isinstance(optimizer, sdca_optimizer.SDCAOptimizer) and - enable_centered_bias): - raise ValueError("enable_centered_bias is not supported with SDCA") - - self._feature_columns = tuple(feature_columns or []) - assert self._feature_columns - - chief_hook = None - head = head_lib.multi_class_head( - n_classes, - weight_column_name=weight_column_name, - enable_centered_bias=enable_centered_bias, - label_keys=label_keys) - params = { - "head": head, - "feature_columns": feature_columns, - "optimizer": optimizer, - } - - if isinstance(optimizer, sdca_optimizer.SDCAOptimizer): - assert not _joint_weight, ("_joint_weight is incompatible with the" - " SDCAOptimizer") - assert n_classes == 2, "SDCA only applies to binary classification." - - model_fn = sdca_model_fn - # The model_fn passes the model parameters to the chief_hook. We then use - # the hook to update weights and shrink step only on the chief. - chief_hook = _SdcaUpdateWeightsHook() - params.update({ - "weight_column_name": weight_column_name, - "update_weights_hook": chief_hook, - }) - else: - model_fn = _linear_model_fn - params.update({ - "gradient_clip_norm": gradient_clip_norm, - "joint_weights": _joint_weight, - }) - - super(LinearClassifier, self).__init__( - model_fn=model_fn, - model_dir=model_dir, - config=config, - params=params, - feature_engineering_fn=feature_engineering_fn) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - @deprecated_arg_values( - "2017-03-01", - "Please switch to predict_classes, or set `outputs` argument.", - outputs=None) - def predict(self, x=None, input_fn=None, batch_size=None, outputs=None, - as_iterable=True): - """Returns predictions for given features. - - By default, returns predicted classes. But this default will be dropped - soon. Users should either pass `outputs`, or call `predict_classes` method. - - Args: - x: features. - input_fn: Input function. If set, x must be None. - batch_size: Override default batch size. - outputs: list of `str`, name of the output to predict. - If `None`, returns classes. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted classes with shape [batch_size] (or an iterable - of predicted classes if as_iterable is True). Each predicted class is - represented by its class index (i.e. integer from 0 to n_classes-1). - If `outputs` is set, returns a dict of predictions. - """ - if not outputs: - return self.predict_classes( - x=x, - input_fn=input_fn, - batch_size=batch_size, - as_iterable=as_iterable) - return super(LinearClassifier, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=outputs, - as_iterable=as_iterable) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - def predict_classes(self, x=None, input_fn=None, batch_size=None, - as_iterable=True): - """Returns predicted classes for given features. - - Args: - x: features. - input_fn: Input function. If set, x must be None. - batch_size: Override default batch size. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted classes with shape [batch_size] (or an iterable - of predicted classes if as_iterable is True). Each predicted class is - represented by its class index (i.e. integer from 0 to n_classes-1). - """ - key = prediction_key.PredictionKey.CLASSES - preds = super(LinearClassifier, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=[key], - as_iterable=as_iterable) - if as_iterable: - return _as_iterable(preds, output=key) - return preds[key] - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - def predict_proba(self, x=None, input_fn=None, batch_size=None, - as_iterable=True): - """Returns predicted probabilities for given features. - - Args: - x: features. - input_fn: Input function. If set, x and y must be None. - batch_size: Override default batch size. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted probabilities with shape [batch_size, n_classes] - (or an iterable of predicted probabilities if as_iterable is True). - """ - key = prediction_key.PredictionKey.PROBABILITIES - preds = super(LinearClassifier, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=[key], - as_iterable=as_iterable) - if as_iterable: - return _as_iterable(preds, output=key) - return preds[key] - - @deprecated("2017-03-25", "Please use Estimator.export_savedmodel() instead.") - def export(self, - export_dir, - input_fn=None, - input_feature_key=None, - use_deprecated_input_fn=True, - signature_fn=None, - default_batch_size=1, - exports_to_keep=None): - """See BaseEstimator.export.""" - def default_input_fn(unused_estimator, examples): - return layers.parse_feature_columns_from_examples( - examples, self._feature_columns) - - return super(LinearClassifier, self).export( - export_dir=export_dir, - input_fn=input_fn or default_input_fn, - input_feature_key=input_feature_key, - use_deprecated_input_fn=use_deprecated_input_fn, - signature_fn=(signature_fn or - export.classification_signature_fn_with_prob), - prediction_key=prediction_key.PredictionKey.PROBABILITIES, - default_batch_size=default_batch_size, - exports_to_keep=exports_to_keep) - - -class LinearRegressor(estimator.Estimator): - """Linear regressor model. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Train a linear regression model to predict label value given observation of - feature values. - - Example: - - ```python - sparse_column_a = sparse_column_with_hash_bucket(...) - sparse_column_b = sparse_column_with_hash_bucket(...) - - sparse_feature_a_x_sparse_feature_b = crossed_column(...) - - estimator = LinearRegressor( - feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b]) - - # Input builders - def input_fn_train: # returns x, y - ... - def input_fn_eval: # returns x, y - ... - estimator.fit(input_fn=input_fn_train) - estimator.evaluate(input_fn=input_fn_eval) - estimator.predict(x=x) - ``` - - Input of `fit` and `evaluate` should have following features, - otherwise there will be a KeyError: - - * if `weight_column_name` is not `None`: - key=weight_column_name, value=a `Tensor` - * for column in `feature_columns`: - - if isinstance(column, `SparseColumn`): - key=column.name, value=a `SparseTensor` - - if isinstance(column, `WeightedSparseColumn`): - {key=id column name, value=a `SparseTensor`, - key=weight column name, value=a `SparseTensor`} - - if isinstance(column, `RealValuedColumn`): - key=column.name, value=a `Tensor` - """ - - def __init__(self, # _joint_weights: pylint: disable=invalid-name - feature_columns, - model_dir=None, - weight_column_name=None, - optimizer=None, - gradient_clip_norm=None, - enable_centered_bias=False, - label_dimension=1, - _joint_weights=False, - config=None, - feature_engineering_fn=None): - """Construct a `LinearRegressor` estimator object. - - Args: - feature_columns: An iterable containing all the feature columns used by - the model. All items in the set should be instances of classes derived - from `FeatureColumn`. - model_dir: Directory to save model parameters, graph, etc. This can - also be used to load checkpoints from the directory into a estimator - to continue training a previously saved model. - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - optimizer: An instance of `tf.Optimizer` used to train the model. If - `None`, will use an Ftrl optimizer. - gradient_clip_norm: A `float` > 0. If provided, gradients are clipped - to their global norm with this clipping ratio. See - `tf.clip_by_global_norm` for more details. - enable_centered_bias: A bool. If True, estimator will learn a centered - bias variable for each class. Rest of the model structure learns the - residual after centered bias. - label_dimension: Number of regression targets per example. This is the - size of the last dimension of the labels and logits `Tensor` objects - (typically, these have shape `[batch_size, label_dimension]`). - _joint_weights: If True use a single (possibly partitioned) variable to - store the weights. It's faster, but requires all feature columns are - sparse and have the 'sum' combiner. Incompatible with SDCAOptimizer. - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and - returns features and labels which will be fed - into the model. - - Returns: - A `LinearRegressor` estimator. - """ - self._feature_columns = tuple(feature_columns or []) - assert self._feature_columns - - chief_hook = None - if (isinstance(optimizer, sdca_optimizer.SDCAOptimizer) and - enable_centered_bias): - enable_centered_bias = False - logging.warning("centered_bias is not supported with SDCA, " - "please disable it explicitly.") - head = head_lib.regression_head( - weight_column_name=weight_column_name, - label_dimension=label_dimension, - enable_centered_bias=enable_centered_bias) - params = { - "head": head, - "feature_columns": feature_columns, - "optimizer": optimizer, - } - - if isinstance(optimizer, sdca_optimizer.SDCAOptimizer): - assert label_dimension == 1, "SDCA only applies for label_dimension=1." - assert not _joint_weights, ("_joint_weights is incompatible with" - " SDCAOptimizer.") - - model_fn = sdca_model_fn - # The model_fn passes the model parameters to the chief_hook. We then use - # the hook to update weights and shrink step only on the chief. - chief_hook = _SdcaUpdateWeightsHook() - params.update({ - "weight_column_name": weight_column_name, - "update_weights_hook": chief_hook, - }) - else: - model_fn = _linear_model_fn - params.update({ - "gradient_clip_norm": gradient_clip_norm, - "joint_weights": _joint_weights, - }) - - super(LinearRegressor, self).__init__( - model_fn=model_fn, - model_dir=model_dir, - config=config, - params=params, - feature_engineering_fn=feature_engineering_fn) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - @deprecated_arg_values( - "2017-03-01", - "Please switch to predict_scores, or set `outputs` argument.", - outputs=None) - def predict(self, x=None, input_fn=None, batch_size=None, outputs=None, - as_iterable=True): - """Returns predictions for given features. - - By default, returns predicted scores. But this default will be dropped - soon. Users should either pass `outputs`, or call `predict_scores` method. - - Args: - x: features. - input_fn: Input function. If set, x must be None. - batch_size: Override default batch size. - outputs: list of `str`, name of the output to predict. - If `None`, returns scores. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted scores (or an iterable of predicted scores if - as_iterable is True). If `label_dimension == 1`, the shape of the output - is `[batch_size]`, otherwise the shape is `[batch_size, label_dimension]`. - If `outputs` is set, returns a dict of predictions. - """ - if not outputs: - return self.predict_scores( - x=x, - input_fn=input_fn, - batch_size=batch_size, - as_iterable=as_iterable) - return super(LinearRegressor, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=outputs, - as_iterable=as_iterable) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - def predict_scores(self, x=None, input_fn=None, batch_size=None, - as_iterable=True): - """Returns predicted scores for given features. - - Args: - x: features. - input_fn: Input function. If set, x must be None. - batch_size: Override default batch size. - as_iterable: If True, return an iterable which keeps yielding predictions - for each example until inputs are exhausted. Note: The inputs must - terminate if you want the iterable to terminate (e.g. be sure to pass - num_epochs=1 if you are using something like read_batch_features). - - Returns: - Numpy array of predicted scores (or an iterable of predicted scores if - as_iterable is True). If `label_dimension == 1`, the shape of the output - is `[batch_size]`, otherwise the shape is `[batch_size, label_dimension]`. - """ - key = prediction_key.PredictionKey.SCORES - preds = super(LinearRegressor, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=[key], - as_iterable=as_iterable) - if as_iterable: - return _as_iterable(preds, output=key) - return preds[key] - - @deprecated("2017-03-25", "Please use Estimator.export_savedmodel() instead.") - def export(self, - export_dir, - input_fn=None, - input_feature_key=None, - use_deprecated_input_fn=True, - signature_fn=None, - default_batch_size=1, - exports_to_keep=None): - """See BaseEstimator.export.""" - def default_input_fn(unused_estimator, examples): - return layers.parse_feature_columns_from_examples( - examples, self._feature_columns) - - return super(LinearRegressor, self).export( - export_dir=export_dir, - input_fn=input_fn or default_input_fn, - input_feature_key=input_feature_key, - use_deprecated_input_fn=use_deprecated_input_fn, - signature_fn=(signature_fn or export.regression_signature_fn), - prediction_key=prediction_key.PredictionKey.SCORES, - default_batch_size=default_batch_size, - exports_to_keep=exports_to_keep) - - -class LinearEstimator(estimator.Estimator): - """Linear model with user specified head. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Train a generalized linear model to predict label value given observation of - feature values. - - Example: - To do poisson regression, - - ```python - sparse_column_a = sparse_column_with_hash_bucket(...) - sparse_column_b = sparse_column_with_hash_bucket(...) - - sparse_feature_a_x_sparse_feature_b = crossed_column(...) - - estimator = LinearEstimator( - feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b], - head=head_lib.poisson_regression_head()) - - # Input builders - def input_fn_train: # returns x, y - ... - def input_fn_eval: # returns x, y - ... - estimator.fit(input_fn=input_fn_train) - estimator.evaluate(input_fn=input_fn_eval) - estimator.predict(x=x) - ``` - - Input of `fit` and `evaluate` should have following features, - otherwise there will be a KeyError: - - * if `weight_column_name` is not `None`: - key=weight_column_name, value=a `Tensor` - * for column in `feature_columns`: - - if isinstance(column, `SparseColumn`): - key=column.name, value=a `SparseTensor` - - if isinstance(column, `WeightedSparseColumn`): - {key=id column name, value=a `SparseTensor`, - key=weight column name, value=a `SparseTensor`} - - if isinstance(column, `RealValuedColumn`): - key=column.name, value=a `Tensor` - """ - - def __init__(self, # _joint_weights: pylint: disable=invalid-name - feature_columns, - head, - model_dir=None, - weight_column_name=None, - optimizer=None, - gradient_clip_norm=None, - _joint_weights=False, - config=None, - feature_engineering_fn=None): - """Construct a `LinearEstimator` object. - - Args: - feature_columns: An iterable containing all the feature columns used by - the model. All items in the set should be instances of classes derived - from `FeatureColumn`. - head: An instance of _Head class. - model_dir: Directory to save model parameters, graph, etc. This can - also be used to load checkpoints from the directory into a estimator - to continue training a previously saved model. - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - optimizer: An instance of `tf.Optimizer` used to train the model. If - `None`, will use an Ftrl optimizer. - gradient_clip_norm: A `float` > 0. If provided, gradients are clipped - to their global norm with this clipping ratio. See - `tf.clip_by_global_norm` for more details. - _joint_weights: If True use a single (possibly partitioned) variable to - store the weights. It's faster, but requires all feature columns are - sparse and have the 'sum' combiner. Incompatible with SDCAOptimizer. - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and - returns features and labels which will be fed - into the model. - - Returns: - A `LinearEstimator` estimator. - - Raises: - ValueError: if optimizer is not supported, e.g., SDCAOptimizer - """ - assert feature_columns - if isinstance(optimizer, sdca_optimizer.SDCAOptimizer): - raise ValueError("LinearEstimator does not support SDCA optimizer.") - - params = { - "head": head, - "feature_columns": feature_columns, - "optimizer": optimizer, - "gradient_clip_norm": gradient_clip_norm, - "joint_weights": _joint_weights, - } - super(LinearEstimator, self).__init__( - model_fn=_linear_model_fn, - model_dir=model_dir, - config=config, - params=params, - feature_engineering_fn=feature_engineering_fn) diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear_test.py b/tensorflow/contrib/learn/python/learn/estimators/linear_test.py deleted file mode 100644 index dfc76bfde6c..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/linear_test.py +++ /dev/null @@ -1,1982 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for estimators.linear.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import json -import tempfile - -import numpy as np - -from tensorflow.contrib.layers.python.layers import feature_column as feature_column_lib -from tensorflow.contrib.learn.python.learn import experiment -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.contrib.learn.python.learn.estimators import _sklearn -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import estimator_test_utils -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import linear -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.contrib.learn.python.learn.estimators import test_data -from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec -from tensorflow.contrib.linear_optimizer.python import sdca_optimizer as sdca_optimizer_lib -from tensorflow.contrib.metrics.python.ops import metric_ops -from tensorflow.python.feature_column import feature_column_lib as fc_core -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.platform import test -from tensorflow.python.training import ftrl -from tensorflow.python.training import input as input_lib -from tensorflow.python.training import server_lib - - -def _prepare_iris_data_for_logistic_regression(): - # Converts iris data to a logistic regression problem. - iris = base.load_iris() - ids = np.where((iris.target == 0) | (iris.target == 1)) - iris = base.Dataset(data=iris.data[ids], target=iris.target[ids]) - return iris - - -class LinearClassifierTest(test.TestCase): - - def testExperimentIntegration(self): - cont_features = [ - feature_column_lib.real_valued_column( - 'feature', dimension=4) - ] - - exp = experiment.Experiment( - estimator=linear.LinearClassifier( - n_classes=3, feature_columns=cont_features), - train_input_fn=test_data.iris_input_multiclass_fn, - eval_input_fn=test_data.iris_input_multiclass_fn) - exp.test() - - def testEstimatorContract(self): - estimator_test_utils.assert_estimator_contract(self, - linear.LinearClassifier) - - def testTrain(self): - """Tests that loss goes down with training.""" - - def input_fn(): - return { - 'age': - constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column_lib.sparse_column_with_hash_bucket('language', - 100) - age = feature_column_lib.real_valued_column('age') - - classifier = linear.LinearClassifier(feature_columns=[age, language]) - classifier.fit(input_fn=input_fn, steps=100) - loss1 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - classifier.fit(input_fn=input_fn, steps=200) - loss2 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss2, loss1) - self.assertLess(loss2, 0.01) - - def testJointTrain(self): - """Tests that loss goes down with training with joint weights.""" - - def input_fn(): - return { - 'age': - sparse_tensor.SparseTensor( - values=['1'], indices=[[0, 0]], dense_shape=[1, 1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column_lib.sparse_column_with_hash_bucket('language', - 100) - age = feature_column_lib.sparse_column_with_hash_bucket('age', 2) - - classifier = linear.LinearClassifier( - _joint_weight=True, feature_columns=[age, language]) - classifier.fit(input_fn=input_fn, steps=100) - loss1 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - classifier.fit(input_fn=input_fn, steps=200) - loss2 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss2, loss1) - self.assertLess(loss2, 0.01) - - def testMultiClass_MatrixData(self): - """Tests multi-class classification using matrix data as input.""" - feature_column = feature_column_lib.real_valued_column( - 'feature', dimension=4) - - classifier = linear.LinearClassifier( - n_classes=3, feature_columns=[feature_column]) - - classifier.fit(input_fn=test_data.iris_input_multiclass_fn, steps=100) - scores = classifier.evaluate( - input_fn=test_data.iris_input_multiclass_fn, steps=100) - self.assertGreater(scores['accuracy'], 0.9) - - def testMultiClass_MatrixData_Labels1D(self): - """Same as the last test, but labels shape is [150] instead of [150, 1].""" - - def _input_fn(): - iris = base.load_iris() - return { - 'feature': constant_op.constant( - iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=[150], dtype=dtypes.int32) - - feature_column = feature_column_lib.real_valued_column( - 'feature', dimension=4) - - classifier = linear.LinearClassifier( - n_classes=3, feature_columns=[feature_column]) - - classifier.fit(input_fn=_input_fn, steps=100) - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self.assertGreater(scores['accuracy'], 0.9) - - def testMultiClass_NpMatrixData(self): - """Tests multi-class classification using numpy matrix data as input.""" - iris = base.load_iris() - train_x = iris.data - train_y = iris.target - feature_column = feature_column_lib.real_valued_column('', dimension=4) - classifier = linear.LinearClassifier( - n_classes=3, feature_columns=[feature_column]) - - classifier.fit(x=train_x, y=train_y, steps=100) - scores = classifier.evaluate(x=train_x, y=train_y, steps=1) - self.assertGreater(scores['accuracy'], 0.9) - - def testMultiClassLabelKeys(self): - """Tests n_classes > 2 with label_keys vocabulary for labels.""" - # Byte literals needed for python3 test to pass. - label_keys = [b'label0', b'label1', b'label2'] - - def _input_fn(num_epochs=None): - features = { - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - labels = constant_op.constant( - [[label_keys[1]], [label_keys[0]], [label_keys[0]]], - dtype=dtypes.string) - return features, labels - - language_column = feature_column_lib.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20) - - classifier = linear.LinearClassifier( - n_classes=3, - feature_columns=[language_column], - label_keys=label_keys) - - classifier.fit(input_fn=_input_fn, steps=50) - - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self.assertGreater(scores['accuracy'], 0.9) - self.assertIn('loss', scores) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predicted_classes = list( - classifier.predict_classes( - input_fn=predict_input_fn, as_iterable=True)) - self.assertEqual(3, len(predicted_classes)) - for pred in predicted_classes: - self.assertIn(pred, label_keys) - predictions = list( - classifier.predict(input_fn=predict_input_fn, as_iterable=True)) - self.assertAllEqual(predicted_classes, predictions) - - def testLogisticRegression_MatrixData(self): - """Tests binary classification using matrix data as input.""" - - def _input_fn(): - iris = _prepare_iris_data_for_logistic_regression() - return { - 'feature': constant_op.constant( - iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=[100, 1], dtype=dtypes.int32) - - feature_column = feature_column_lib.real_valued_column( - 'feature', dimension=4) - - classifier = linear.LinearClassifier(feature_columns=[feature_column]) - - classifier.fit(input_fn=_input_fn, steps=100) - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self.assertGreater(scores['accuracy'], 0.9) - - def testEstimatorWithCoreFeatureColumns(self): - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[.8], [0.2], [.1]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=input_lib.limit_epochs( - ['en', 'fr', 'zh'], num_epochs=num_epochs), - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant([[1], [0], [0]], dtype=dtypes.int32) - - language_column = fc_core.categorical_column_with_hash_bucket( - 'language', hash_bucket_size=20) - feature_columns = [language_column, fc_core.numeric_column('age')] - - classifier = linear.LinearClassifier(feature_columns=feature_columns) - classifier.fit(input_fn=_input_fn, steps=100) - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self.assertGreater(scores['accuracy'], 0.9) - - def testLogisticRegression_MatrixData_Labels1D(self): - """Same as the last test, but labels shape is [100] instead of [100, 1].""" - - def _input_fn(): - iris = _prepare_iris_data_for_logistic_regression() - return { - 'feature': constant_op.constant( - iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=[100], dtype=dtypes.int32) - - feature_column = feature_column_lib.real_valued_column( - 'feature', dimension=4) - - classifier = linear.LinearClassifier(feature_columns=[feature_column]) - - classifier.fit(input_fn=_input_fn, steps=100) - scores = classifier.evaluate(input_fn=_input_fn, steps=1) - self.assertGreater(scores['accuracy'], 0.9) - - def testLogisticRegression_NpMatrixData(self): - """Tests binary classification using numpy matrix data as input.""" - iris = _prepare_iris_data_for_logistic_regression() - train_x = iris.data - train_y = iris.target - feature_columns = [feature_column_lib.real_valued_column('', dimension=4)] - classifier = linear.LinearClassifier(feature_columns=feature_columns) - - classifier.fit(x=train_x, y=train_y, steps=100) - scores = classifier.evaluate(x=train_x, y=train_y, steps=1) - self.assertGreater(scores['accuracy'], 0.9) - - def testWeightAndBiasNames(self): - """Tests that weight and bias names haven't changed.""" - feature_column = feature_column_lib.real_valued_column( - 'feature', dimension=4) - - classifier = linear.LinearClassifier( - n_classes=3, feature_columns=[feature_column]) - - classifier.fit(input_fn=test_data.iris_input_multiclass_fn, steps=100) - - variable_names = classifier.get_variable_names() - self.assertIn('linear/feature/weight', variable_names) - self.assertIn('linear/bias_weight', variable_names) - self.assertEqual( - 4, len(classifier.get_variable_value('linear/feature/weight'))) - self.assertEqual( - 3, len(classifier.get_variable_value('linear/bias_weight'))) - - def testCustomOptimizerByObject(self): - """Tests multi-class classification using matrix data as input.""" - feature_column = feature_column_lib.real_valued_column( - 'feature', dimension=4) - - classifier = linear.LinearClassifier( - n_classes=3, - optimizer=ftrl.FtrlOptimizer(learning_rate=0.1), - feature_columns=[feature_column]) - - classifier.fit(input_fn=test_data.iris_input_multiclass_fn, steps=100) - scores = classifier.evaluate( - input_fn=test_data.iris_input_multiclass_fn, steps=100) - self.assertGreater(scores['accuracy'], 0.9) - - def testCustomOptimizerByString(self): - """Tests multi-class classification using matrix data as input.""" - feature_column = feature_column_lib.real_valued_column( - 'feature', dimension=4) - - def _optimizer(): - return ftrl.FtrlOptimizer(learning_rate=0.1) - - classifier = linear.LinearClassifier( - n_classes=3, optimizer=_optimizer, feature_columns=[feature_column]) - - classifier.fit(input_fn=test_data.iris_input_multiclass_fn, steps=100) - scores = classifier.evaluate( - input_fn=test_data.iris_input_multiclass_fn, steps=100) - self.assertGreater(scores['accuracy'], 0.9) - - def testCustomOptimizerByFunction(self): - """Tests multi-class classification using matrix data as input.""" - feature_column = feature_column_lib.real_valued_column( - 'feature', dimension=4) - - classifier = linear.LinearClassifier( - n_classes=3, optimizer='Ftrl', feature_columns=[feature_column]) - - classifier.fit(input_fn=test_data.iris_input_multiclass_fn, steps=100) - scores = classifier.evaluate( - input_fn=test_data.iris_input_multiclass_fn, steps=100) - self.assertGreater(scores['accuracy'], 0.9) - - def testCustomMetrics(self): - """Tests custom evaluation metrics.""" - - def _input_fn(num_epochs=None): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - labels = constant_op.constant([[1], [0], [0], [0]], dtype=dtypes.float32) - features = { - 'x': - input_lib.limit_epochs( - array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - num_epochs=num_epochs) - } - return features, labels - - def _my_metric_op(predictions, labels): - # For the case of binary classification, the 2nd column of "predictions" - # denotes the model predictions. - predictions = array_ops.strided_slice( - predictions, [0, 1], [-1, 2], end_mask=1) - return math_ops.reduce_sum(math_ops.multiply(predictions, labels)) - - classifier = linear.LinearClassifier( - feature_columns=[feature_column_lib.real_valued_column('x')]) - - classifier.fit(input_fn=_input_fn, steps=100) - scores = classifier.evaluate( - input_fn=_input_fn, - steps=100, - metrics={ - 'my_accuracy': - MetricSpec( - metric_fn=metric_ops.streaming_accuracy, - prediction_key='classes'), - 'my_precision': - MetricSpec( - metric_fn=metric_ops.streaming_precision, - prediction_key='classes'), - 'my_metric': - MetricSpec( - metric_fn=_my_metric_op, prediction_key='probabilities') - }) - self.assertTrue( - set(['loss', 'my_accuracy', 'my_precision', 'my_metric']).issubset( - set(scores.keys()))) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = np.array(list(classifier.predict_classes( - input_fn=predict_input_fn))) - self.assertEqual( - _sklearn.accuracy_score([1, 0, 0, 0], predictions), - scores['my_accuracy']) - - # Tests the case where the prediction_key is neither "classes" nor - # "probabilities". - with self.assertRaisesRegexp(KeyError, 'bad_type'): - classifier.evaluate( - input_fn=_input_fn, - steps=100, - metrics={ - 'bad_name': - MetricSpec( - metric_fn=metric_ops.streaming_auc, - prediction_key='bad_type') - }) - - # Tests the case where the 2nd element of the key is neither "classes" nor - # "probabilities". - with self.assertRaises(KeyError): - classifier.evaluate( - input_fn=_input_fn, - steps=100, - metrics={('bad_name', 'bad_type'): metric_ops.streaming_auc}) - - # Tests the case where the tuple of the key doesn't have 2 elements. - with self.assertRaises(ValueError): - classifier.evaluate( - input_fn=_input_fn, - steps=100, - metrics={ - ('bad_length_name', 'classes', 'bad_length'): - metric_ops.streaming_accuracy - }) - - def testLogisticFractionalLabels(self): - """Tests logistic training with fractional labels.""" - - def input_fn(num_epochs=None): - return { - 'age': - input_lib.limit_epochs( - constant_op.constant([[1], [2]]), num_epochs=num_epochs), - }, constant_op.constant( - [[.7], [0]], dtype=dtypes.float32) - - age = feature_column_lib.real_valued_column('age') - - classifier = linear.LinearClassifier( - feature_columns=[age], config=run_config.RunConfig(tf_random_seed=1)) - classifier.fit(input_fn=input_fn, steps=500) - - predict_input_fn = functools.partial(input_fn, num_epochs=1) - predictions_proba = list( - classifier.predict_proba(input_fn=predict_input_fn)) - # Prediction probabilities mirror the labels column, which proves that the - # classifier learns from float input. - self.assertAllClose([[.3, .7], [1., 0.]], predictions_proba, atol=.1) - - def testTrainWithPartitionedVariables(self): - """Tests training with partitioned variables.""" - - def _input_fn(): - features = { - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - labels = constant_op.constant([[1], [0], [0]]) - return features, labels - - sparse_features = [ - # The given hash_bucket_size results in variables larger than the - # default min_slice_size attribute, so the variables are partitioned. - feature_column_lib.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=2e7) - ] - - tf_config = { - 'cluster': { - run_config.TaskType.PS: ['fake_ps_0', 'fake_ps_1'] - } - } - with test.mock.patch.dict('os.environ', - {'TF_CONFIG': json.dumps(tf_config)}): - config = run_config.RunConfig() - # Because we did not start a distributed cluster, we need to pass an - # empty ClusterSpec, otherwise the device_setter will look for - # distributed jobs, such as "/job:ps" which are not present. - config._cluster_spec = server_lib.ClusterSpec({}) - - classifier = linear.LinearClassifier( - feature_columns=sparse_features, config=config) - classifier.fit(input_fn=_input_fn, steps=200) - loss = classifier.evaluate(input_fn=_input_fn, steps=1)['loss'] - self.assertLess(loss, 0.07) - - def testTrainSaveLoad(self): - """Tests that insures you can save and reload a trained model.""" - - def input_fn(num_epochs=None): - return { - 'age': - input_lib.limit_epochs( - constant_op.constant([1]), num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]), - }, constant_op.constant([[1]]) - - language = feature_column_lib.sparse_column_with_hash_bucket('language', - 100) - age = feature_column_lib.real_valued_column('age') - - model_dir = tempfile.mkdtemp() - classifier = linear.LinearClassifier( - model_dir=model_dir, feature_columns=[age, language]) - classifier.fit(input_fn=input_fn, steps=30) - predict_input_fn = functools.partial(input_fn, num_epochs=1) - out1_class = list( - classifier.predict_classes( - input_fn=predict_input_fn, as_iterable=True)) - out1_proba = list( - classifier.predict_proba( - input_fn=predict_input_fn, as_iterable=True)) - del classifier - - classifier2 = linear.LinearClassifier( - model_dir=model_dir, feature_columns=[age, language]) - out2_class = list( - classifier2.predict_classes( - input_fn=predict_input_fn, as_iterable=True)) - out2_proba = list( - classifier2.predict_proba( - input_fn=predict_input_fn, as_iterable=True)) - self.assertTrue(np.array_equal(out1_class, out2_class)) - self.assertTrue(np.array_equal(out1_proba, out2_proba)) - - def testWeightColumn(self): - """Tests training with given weight column.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # First row has more weight than others. Model should fit (y=x) better - # than (y=Not(x)) due to the relative higher weight of the first row. - labels = constant_op.constant([[1], [0], [0], [0]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[100.], [3.], [2.], [2.]]) - } - return features, labels - - def _input_fn_eval(): - # Create 4 rows (y = x) - labels = constant_op.constant([[1], [1], [1], [1]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - classifier = linear.LinearClassifier( - weight_column_name='w', - feature_columns=[feature_column_lib.real_valued_column('x')], - config=run_config.RunConfig(tf_random_seed=3)) - - classifier.fit(input_fn=_input_fn_train, steps=100) - scores = classifier.evaluate(input_fn=_input_fn_eval, steps=1) - # All examples in eval data set are y=x. - self.assertGreater(scores['labels/actual_label_mean'], 0.9) - # If there were no weight column, model would learn y=Not(x). Because of - # weights, it learns y=x. - self.assertGreater(scores['labels/prediction_mean'], 0.9) - # All examples in eval data set are y=x. So if weight column were ignored, - # then accuracy would be zero. Because of weights, accuracy should be close - # to 1.0. - self.assertGreater(scores['accuracy'], 0.9) - - scores_train_set = classifier.evaluate(input_fn=_input_fn_train, steps=1) - # Considering weights, the mean label should be close to 1.0. - # If weights were ignored, it would be 0.25. - self.assertGreater(scores_train_set['labels/actual_label_mean'], 0.9) - # The classifier has learned y=x. If weight column were ignored in - # evaluation, then accuracy for the train set would be 0.25. - # Because weight is not ignored, accuracy is greater than 0.6. - self.assertGreater(scores_train_set['accuracy'], 0.6) - - def testWeightColumnLoss(self): - """Test ensures that you can specify per-example weights for loss.""" - - def _input_fn(): - features = { - 'age': constant_op.constant([[20], [20], [20]]), - 'weights': constant_op.constant([[100], [1], [1]]), - } - labels = constant_op.constant([[1], [0], [0]]) - return features, labels - - age = feature_column_lib.real_valued_column('age') - - classifier = linear.LinearClassifier(feature_columns=[age]) - classifier.fit(input_fn=_input_fn, steps=100) - loss_unweighted = classifier.evaluate(input_fn=_input_fn, steps=1)['loss'] - - classifier = linear.LinearClassifier( - feature_columns=[age], weight_column_name='weights') - classifier.fit(input_fn=_input_fn, steps=100) - loss_weighted = classifier.evaluate(input_fn=_input_fn, steps=1)['loss'] - - self.assertLess(loss_weighted, loss_unweighted) - - def testExport(self): - """Tests that export model for servo works.""" - - def input_fn(): - return { - 'age': - constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column_lib.sparse_column_with_hash_bucket('language', - 100) - age = feature_column_lib.real_valued_column('age') - - classifier = linear.LinearClassifier(feature_columns=[age, language]) - classifier.fit(input_fn=input_fn, steps=100) - - export_dir = tempfile.mkdtemp() - classifier.export(export_dir) - - def testDisableCenteredBias(self): - """Tests that we can disable centered bias.""" - - def input_fn(): - return { - 'age': - constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column_lib.sparse_column_with_hash_bucket('language', - 100) - age = feature_column_lib.real_valued_column('age') - - classifier = linear.LinearClassifier( - feature_columns=[age, language], enable_centered_bias=False) - classifier.fit(input_fn=input_fn, steps=100) - self.assertNotIn('centered_bias_weight', classifier.get_variable_names()) - - def testEnableCenteredBias(self): - """Tests that we can enable centered bias.""" - - def input_fn(): - return { - 'age': - constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column_lib.sparse_column_with_hash_bucket('language', - 100) - age = feature_column_lib.real_valued_column('age') - - classifier = linear.LinearClassifier( - feature_columns=[age, language], enable_centered_bias=True) - classifier.fit(input_fn=input_fn, steps=100) - self.assertIn('linear/binary_logistic_head/centered_bias_weight', - classifier.get_variable_names()) - - def testTrainOptimizerWithL1Reg(self): - """Tests l1 regularized model has higher loss.""" - - def input_fn(): - return { - 'language': - sparse_tensor.SparseTensor( - values=['hindi'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[1]]) - - language = feature_column_lib.sparse_column_with_hash_bucket('language', - 100) - classifier_no_reg = linear.LinearClassifier(feature_columns=[language]) - classifier_with_reg = linear.LinearClassifier( - feature_columns=[language], - optimizer=ftrl.FtrlOptimizer( - learning_rate=1.0, l1_regularization_strength=100.)) - loss_no_reg = classifier_no_reg.fit(input_fn=input_fn, steps=100).evaluate( - input_fn=input_fn, steps=1)['loss'] - loss_with_reg = classifier_with_reg.fit(input_fn=input_fn, - steps=100).evaluate( - input_fn=input_fn, - steps=1)['loss'] - self.assertLess(loss_no_reg, loss_with_reg) - - def testTrainWithMissingFeature(self): - """Tests that training works with missing features.""" - - def input_fn(): - return { - 'language': - sparse_tensor.SparseTensor( - values=['Swahili', 'turkish'], - indices=[[0, 0], [2, 0]], - dense_shape=[3, 1]) - }, constant_op.constant([[1], [1], [1]]) - - language = feature_column_lib.sparse_column_with_hash_bucket('language', - 100) - classifier = linear.LinearClassifier(feature_columns=[language]) - classifier.fit(input_fn=input_fn, steps=100) - loss = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss, 0.07) - - def testSdcaOptimizerRealValuedFeatures(self): - """Tests LinearClassifier with SDCAOptimizer and real valued features.""" - - def input_fn(): - return { - 'example_id': constant_op.constant(['1', '2']), - 'maintenance_cost': constant_op.constant([[500.0], [200.0]]), - 'sq_footage': constant_op.constant([[800.0], [600.0]]), - 'weights': constant_op.constant([[1.0], [1.0]]) - }, constant_op.constant([[0], [1]]) - - maintenance_cost = feature_column_lib.real_valued_column('maintenance_cost') - sq_footage = feature_column_lib.real_valued_column('sq_footage') - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id') - classifier = linear.LinearClassifier( - feature_columns=[maintenance_cost, sq_footage], - weight_column_name='weights', - optimizer=sdca_optimizer) - classifier.fit(input_fn=input_fn, steps=100) - loss = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss, 0.05) - - def testSdcaOptimizerRealValuedFeatureWithHigherDimension(self): - """Tests SDCAOptimizer with real valued features of higher dimension.""" - - # input_fn is identical to the one in testSdcaOptimizerRealValuedFeatures - # where 2 1-dimensional dense features have been replaced by 1 2-dimensional - # feature. - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2']), - 'dense_feature': - constant_op.constant([[500.0, 800.0], [200.0, 600.0]]) - }, constant_op.constant([[0], [1]]) - - dense_feature = feature_column_lib.real_valued_column( - 'dense_feature', dimension=2) - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id') - classifier = linear.LinearClassifier( - feature_columns=[dense_feature], optimizer=sdca_optimizer) - classifier.fit(input_fn=input_fn, steps=100) - loss = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss, 0.05) - - def testSdcaOptimizerBucketizedFeatures(self): - """Tests LinearClassifier with SDCAOptimizer and bucketized features.""" - - def input_fn(): - return { - 'example_id': constant_op.constant(['1', '2', '3']), - 'price': constant_op.constant([[600.0], [1000.0], [400.0]]), - 'sq_footage': constant_op.constant([[1000.0], [600.0], [700.0]]), - 'weights': constant_op.constant([[1.0], [1.0], [1.0]]) - }, constant_op.constant([[1], [0], [1]]) - - price_bucket = feature_column_lib.bucketized_column( - feature_column_lib.real_valued_column('price'), - boundaries=[500.0, 700.0]) - sq_footage_bucket = feature_column_lib.bucketized_column( - feature_column_lib.real_valued_column('sq_footage'), boundaries=[650.0]) - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id', symmetric_l2_regularization=1.0) - classifier = linear.LinearClassifier( - feature_columns=[price_bucket, sq_footage_bucket], - weight_column_name='weights', - optimizer=sdca_optimizer) - classifier.fit(input_fn=input_fn, steps=50) - scores = classifier.evaluate(input_fn=input_fn, steps=1) - self.assertGreater(scores['accuracy'], 0.9) - - def testSdcaOptimizerSparseFeatures(self): - """Tests LinearClassifier with SDCAOptimizer and sparse features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([0.4, 0.6, 0.3]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - 'weights': - constant_op.constant([[1.0], [1.0], [1.0]]) - }, constant_op.constant([[1], [0], [1]]) - - price = feature_column_lib.real_valued_column('price') - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id') - classifier = linear.LinearClassifier( - feature_columns=[price, country], - weight_column_name='weights', - optimizer=sdca_optimizer) - classifier.fit(input_fn=input_fn, steps=50) - scores = classifier.evaluate(input_fn=input_fn, steps=1) - self.assertGreater(scores['accuracy'], 0.9) - - def testSdcaOptimizerWeightedSparseFeatures(self): - """LinearClassifier with SDCAOptimizer and weighted sparse features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - sparse_tensor.SparseTensor( - values=[2., 3., 1.], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 5]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 5]) - }, constant_op.constant([[1], [0], [1]]) - - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - country_weighted_by_price = feature_column_lib.weighted_sparse_column( - country, 'price') - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id') - classifier = linear.LinearClassifier( - feature_columns=[country_weighted_by_price], optimizer=sdca_optimizer) - classifier.fit(input_fn=input_fn, steps=50) - scores = classifier.evaluate(input_fn=input_fn, steps=1) - self.assertGreater(scores['accuracy'], 0.9) - - def testSdcaOptimizerWeightedSparseFeaturesOOVWithNoOOVBuckets(self): - """LinearClassifier with SDCAOptimizer with OOV features (-1 IDs).""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - sparse_tensor.SparseTensor( - values=[2., 3., 1.], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 5]), - 'country': - sparse_tensor.SparseTensor( - # 'GB' is out of the vocabulary. - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 5]) - }, constant_op.constant([[1], [0], [1]]) - - country = feature_column_lib.sparse_column_with_keys( - 'country', keys=['US', 'CA', 'MK', 'IT', 'CN']) - country_weighted_by_price = feature_column_lib.weighted_sparse_column( - country, 'price') - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id') - classifier = linear.LinearClassifier( - feature_columns=[country_weighted_by_price], optimizer=sdca_optimizer) - classifier.fit(input_fn=input_fn, steps=50) - scores = classifier.evaluate(input_fn=input_fn, steps=1) - self.assertGreater(scores['accuracy'], 0.9) - - def testSdcaOptimizerCrossedFeatures(self): - """Tests LinearClassifier with SDCAOptimizer and crossed features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'language': - sparse_tensor.SparseTensor( - values=['english', 'italian', 'spanish'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - 'country': - sparse_tensor.SparseTensor( - values=['US', 'IT', 'MX'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]) - }, constant_op.constant([[0], [0], [1]]) - - language = feature_column_lib.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=5) - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - country_language = feature_column_lib.crossed_column( - [language, country], hash_bucket_size=10) - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id') - classifier = linear.LinearClassifier( - feature_columns=[country_language], optimizer=sdca_optimizer) - classifier.fit(input_fn=input_fn, steps=10) - scores = classifier.evaluate(input_fn=input_fn, steps=1) - self.assertGreater(scores['accuracy'], 0.9) - - def testSdcaOptimizerMixedFeatures(self): - """Tests LinearClassifier with SDCAOptimizer and a mix of features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([[0.6], [0.8], [0.3]]), - 'sq_footage': - constant_op.constant([[900.0], [700.0], [600.0]]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - 'weights': - constant_op.constant([[3.0], [1.0], [1.0]]) - }, constant_op.constant([[1], [0], [1]]) - - price = feature_column_lib.real_valued_column('price') - sq_footage_bucket = feature_column_lib.bucketized_column( - feature_column_lib.real_valued_column('sq_footage'), - boundaries=[650.0, 800.0]) - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - sq_footage_country = feature_column_lib.crossed_column( - [sq_footage_bucket, country], hash_bucket_size=10) - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id') - classifier = linear.LinearClassifier( - feature_columns=[price, sq_footage_bucket, country, sq_footage_country], - weight_column_name='weights', - optimizer=sdca_optimizer) - classifier.fit(input_fn=input_fn, steps=50) - scores = classifier.evaluate(input_fn=input_fn, steps=1) - self.assertGreater(scores['accuracy'], 0.9) - - def testSdcaOptimizerPartitionedVariables(self): - """Tests LinearClassifier with SDCAOptimizer with partitioned variables.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([[0.6], [0.8], [0.3]]), - 'sq_footage': - constant_op.constant([[900.0], [700.0], [600.0]]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - 'weights': - constant_op.constant([[3.0], [1.0], [1.0]]) - }, constant_op.constant([[1], [0], [1]]) - - price = feature_column_lib.real_valued_column('price') - sq_footage_bucket = feature_column_lib.bucketized_column( - feature_column_lib.real_valued_column('sq_footage'), - boundaries=[650.0, 800.0]) - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - sq_footage_country = feature_column_lib.crossed_column( - [sq_footage_bucket, country], hash_bucket_size=10) - - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id', - partitioner=partitioned_variables.fixed_size_partitioner( - num_shards=2, axis=0)) - - tf_config = { - 'cluster': { - run_config.TaskType.PS: ['fake_ps_0', 'fake_ps_1'] - } - } - with test.mock.patch.dict('os.environ', - {'TF_CONFIG': json.dumps(tf_config)}): - config = run_config.RunConfig() - # Because we did not start a distributed cluster, we need to pass an - # empty ClusterSpec, otherwise the device_setter will look for - # distributed jobs, such as "/job:ps" which are not present. - config._cluster_spec = server_lib.ClusterSpec({}) - - classifier = linear.LinearClassifier( - feature_columns=[price, sq_footage_bucket, country, sq_footage_country], - weight_column_name='weights', - optimizer=sdca_optimizer, - config=config) - classifier.fit(input_fn=input_fn, steps=50) - scores = classifier.evaluate(input_fn=input_fn, steps=1) - print('all scores = {}'.format(scores)) - self.assertGreater(scores['accuracy'], 0.9) - - def testEval(self): - """Tests that eval produces correct metrics. - """ - - def input_fn(): - return { - 'age': - constant_op.constant([[1], [2]]), - 'language': - sparse_tensor.SparseTensor( - values=['greek', 'chinese'], - indices=[[0, 0], [1, 0]], - dense_shape=[2, 1]), - }, constant_op.constant([[1], [0]]) - - language = feature_column_lib.sparse_column_with_hash_bucket('language', - 100) - age = feature_column_lib.real_valued_column('age') - classifier = linear.LinearClassifier(feature_columns=[age, language]) - - # Evaluate on trained model - classifier.fit(input_fn=input_fn, steps=100) - classifier.evaluate(input_fn=input_fn, steps=1) - - # TODO(ispir): Enable accuracy check after resolving the randomness issue. - # self.assertLess(evaluated_values['loss/mean'], 0.3) - # self.assertGreater(evaluated_values['accuracy/mean'], .95) - - -class LinearRegressorTest(test.TestCase): - - def testExperimentIntegration(self): - cont_features = [ - feature_column_lib.real_valued_column( - 'feature', dimension=4) - ] - - exp = experiment.Experiment( - estimator=linear.LinearRegressor(feature_columns=cont_features), - train_input_fn=test_data.iris_input_logistic_fn, - eval_input_fn=test_data.iris_input_logistic_fn) - exp.test() - - def testEstimatorContract(self): - estimator_test_utils.assert_estimator_contract(self, linear.LinearRegressor) - - def testRegression(self): - """Tests that loss goes down with training.""" - - def input_fn(): - return { - 'age': - constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[10.]]) - - language = feature_column_lib.sparse_column_with_hash_bucket('language', - 100) - age = feature_column_lib.real_valued_column('age') - - classifier = linear.LinearRegressor(feature_columns=[age, language]) - classifier.fit(input_fn=input_fn, steps=100) - loss1 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - classifier.fit(input_fn=input_fn, steps=200) - loss2 = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - - self.assertLess(loss2, loss1) - self.assertLess(loss2, 0.5) - - def testRegression_MatrixData(self): - """Tests regression using matrix data as input.""" - cont_features = [ - feature_column_lib.real_valued_column( - 'feature', dimension=4) - ] - - regressor = linear.LinearRegressor( - feature_columns=cont_features, - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=test_data.iris_input_multiclass_fn, steps=100) - scores = regressor.evaluate( - input_fn=test_data.iris_input_multiclass_fn, steps=1) - self.assertLess(scores['loss'], 0.2) - - def testRegression_TensorData(self): - """Tests regression using tensor data as input.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant( - [1.0, 0., 0.2], dtype=dtypes.float32) - - feature_columns = [ - feature_column_lib.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20), - feature_column_lib.real_valued_column('age') - ] - - regressor = linear.LinearRegressor( - feature_columns=feature_columns, - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=100) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertLess(scores['loss'], 0.2) - - def testLoss(self): - """Tests loss calculation.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # The algorithm should learn (y = 0.25). - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = {'x': array_ops.ones(shape=[4, 1], dtype=dtypes.float32),} - return features, labels - - regressor = linear.LinearRegressor( - feature_columns=[feature_column_lib.real_valued_column('x')], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn_train, steps=100) - scores = regressor.evaluate(input_fn=_input_fn_train, steps=1) - # Average square loss = (0.75^2 + 3*0.25^2) / 4 = 0.1875 - self.assertAlmostEqual(0.1875, scores['loss'], delta=0.1) - - def testLossWithWeights(self): - """Tests loss calculation with weights.""" - - def _input_fn_train(): - # 4 rows with equal weight, one of them (y = x), three of them (y=Not(x)) - # The algorithm should learn (y = 0.25). - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - def _input_fn_eval(): - # 4 rows, with different weights. - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[7.], [1.], [1.], [1.]]) - } - return features, labels - - regressor = linear.LinearRegressor( - weight_column_name='w', - feature_columns=[feature_column_lib.real_valued_column('x')], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn_train, steps=100) - scores = regressor.evaluate(input_fn=_input_fn_eval, steps=1) - # Weighted average square loss = (7*0.75^2 + 3*0.25^2) / 10 = 0.4125 - self.assertAlmostEqual(0.4125, scores['loss'], delta=0.1) - - def testTrainWithWeights(self): - """Tests training with given weight column.""" - - def _input_fn_train(): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - # First row has more weight than others. Model should fit (y=x) better - # than (y=Not(x)) due to the relative higher weight of the first row. - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[100.], [3.], [2.], [2.]]) - } - return features, labels - - def _input_fn_eval(): - # Create 4 rows (y = x) - labels = constant_op.constant([[1.], [1.], [1.], [1.]]) - features = { - 'x': array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - 'w': constant_op.constant([[1.], [1.], [1.], [1.]]) - } - return features, labels - - regressor = linear.LinearRegressor( - weight_column_name='w', - feature_columns=[feature_column_lib.real_valued_column('x')], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn_train, steps=100) - scores = regressor.evaluate(input_fn=_input_fn_eval, steps=1) - # The model should learn (y = x) because of the weights, so the loss should - # be close to zero. - self.assertLess(scores['loss'], 0.1) - - def testPredict_AsIterableFalse(self): - """Tests predict method with as_iterable=False.""" - labels = [1.0, 0., 0.2] - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant(labels, dtype=dtypes.float32) - - feature_columns = [ - feature_column_lib.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20), - feature_column_lib.real_valued_column('age') - ] - - regressor = linear.LinearRegressor( - feature_columns=feature_columns, - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=100) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertLess(scores['loss'], 0.1) - predicted_scores = regressor.predict_scores( - input_fn=_input_fn, as_iterable=False) - self.assertAllClose(labels, predicted_scores, atol=0.1) - predictions = regressor.predict(input_fn=_input_fn, as_iterable=False) - self.assertAllClose(predicted_scores, predictions) - - def testPredict_AsIterable(self): - """Tests predict method with as_iterable=True.""" - labels = [1.0, 0., 0.2] - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant(labels, dtype=dtypes.float32) - - feature_columns = [ - feature_column_lib.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20), - feature_column_lib.real_valued_column('age') - ] - - regressor = linear.LinearRegressor( - feature_columns=feature_columns, - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=100) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertLess(scores['loss'], 0.1) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predicted_scores = list( - regressor.predict_scores( - input_fn=predict_input_fn, as_iterable=True)) - self.assertAllClose(labels, predicted_scores, atol=0.1) - predictions = list( - regressor.predict( - input_fn=predict_input_fn, as_iterable=True)) - self.assertAllClose(predicted_scores, predictions) - - def testCustomMetrics(self): - """Tests custom evaluation metrics.""" - - def _input_fn(num_epochs=None): - # Create 4 rows, one of them (y = x), three of them (y=Not(x)) - labels = constant_op.constant([[1.], [0.], [0.], [0.]]) - features = { - 'x': - input_lib.limit_epochs( - array_ops.ones( - shape=[4, 1], dtype=dtypes.float32), - num_epochs=num_epochs) - } - return features, labels - - def _my_metric_op(predictions, labels): - return math_ops.reduce_sum(math_ops.multiply(predictions, labels)) - - regressor = linear.LinearRegressor( - feature_columns=[feature_column_lib.real_valued_column('x')], - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=100) - scores = regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - 'my_error': - MetricSpec( - metric_fn=metric_ops.streaming_mean_squared_error, - prediction_key='scores'), - 'my_metric': - MetricSpec( - metric_fn=_my_metric_op, prediction_key='scores') - }) - self.assertIn('loss', set(scores.keys())) - self.assertIn('my_error', set(scores.keys())) - self.assertIn('my_metric', set(scores.keys())) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = np.array(list( - regressor.predict_scores(input_fn=predict_input_fn))) - self.assertAlmostEqual( - _sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions), - scores['my_error']) - - # Tests the case where the prediction_key is not "scores". - with self.assertRaisesRegexp(KeyError, 'bad_type'): - regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - 'bad_name': - MetricSpec( - metric_fn=metric_ops.streaming_auc, - prediction_key='bad_type') - }) - - # Tests the case where the 2nd element of the key is not "scores". - with self.assertRaises(KeyError): - regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - ('my_error', 'predictions'): - metric_ops.streaming_mean_squared_error - }) - - # Tests the case where the tuple of the key doesn't have 2 elements. - with self.assertRaises(ValueError): - regressor.evaluate( - input_fn=_input_fn, - steps=1, - metrics={ - ('bad_length_name', 'scores', 'bad_length'): - metric_ops.streaming_mean_squared_error - }) - - def testTrainSaveLoad(self): - """Tests that insures you can save and reload a trained model.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant( - [1.0, 0., 0.2], dtype=dtypes.float32) - - feature_columns = [ - feature_column_lib.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20), - feature_column_lib.real_valued_column('age') - ] - - model_dir = tempfile.mkdtemp() - regressor = linear.LinearRegressor( - model_dir=model_dir, - feature_columns=feature_columns, - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=100) - predict_input_fn = functools.partial(_input_fn, num_epochs=1) - predictions = list(regressor.predict_scores(input_fn=predict_input_fn)) - del regressor - - regressor2 = linear.LinearRegressor( - model_dir=model_dir, feature_columns=feature_columns) - predictions2 = list(regressor2.predict_scores(input_fn=predict_input_fn)) - self.assertAllClose(predictions, predictions2) - - def testTrainWithPartitionedVariables(self): - """Tests training with partitioned variables.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant( - [1.0, 0., 0.2], dtype=dtypes.float32) - - feature_columns = [ - # The given hash_bucket_size results in variables larger than the - # default min_slice_size attribute, so the variables are partitioned. - feature_column_lib.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=2e7), - feature_column_lib.real_valued_column('age') - ] - - tf_config = { - 'cluster': { - run_config.TaskType.PS: ['fake_ps_0', 'fake_ps_1'] - } - } - with test.mock.patch.dict('os.environ', - {'TF_CONFIG': json.dumps(tf_config)}): - config = run_config.RunConfig(tf_random_seed=1) - # Because we did not start a distributed cluster, we need to pass an - # empty ClusterSpec, otherwise the device_setter will look for - # distributed jobs, such as "/job:ps" which are not present. - config._cluster_spec = server_lib.ClusterSpec({}) - - regressor = linear.LinearRegressor( - feature_columns=feature_columns, config=config) - - regressor.fit(input_fn=_input_fn, steps=100) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertLess(scores['loss'], 0.1) - - def testDisableCenteredBias(self): - """Tests that we can disable centered bias.""" - - def _input_fn(num_epochs=None): - features = { - 'age': - input_lib.limit_epochs( - constant_op.constant([[0.8], [0.15], [0.]]), - num_epochs=num_epochs), - 'language': - sparse_tensor.SparseTensor( - values=['en', 'fr', 'zh'], - indices=[[0, 0], [0, 1], [2, 0]], - dense_shape=[3, 2]) - } - return features, constant_op.constant( - [1.0, 0., 0.2], dtype=dtypes.float32) - - feature_columns = [ - feature_column_lib.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=20), - feature_column_lib.real_valued_column('age') - ] - - regressor = linear.LinearRegressor( - feature_columns=feature_columns, - enable_centered_bias=False, - config=run_config.RunConfig(tf_random_seed=1)) - - regressor.fit(input_fn=_input_fn, steps=100) - - scores = regressor.evaluate(input_fn=_input_fn, steps=1) - self.assertLess(scores['loss'], 0.1) - - def testRecoverWeights(self): - rng = np.random.RandomState(67) - n = 1000 - n_weights = 10 - bias = 2 - x = rng.uniform(-1, 1, (n, n_weights)) - weights = 10 * rng.randn(n_weights) - y = np.dot(x, weights) - y += rng.randn(len(x)) * 0.05 + rng.normal(bias, 0.01) - feature_columns = estimator.infer_real_valued_columns_from_input(x) - regressor = linear.LinearRegressor( - feature_columns=feature_columns, - optimizer=ftrl.FtrlOptimizer(learning_rate=0.8)) - regressor.fit(x, y, batch_size=64, steps=2000) - self.assertIn('linear//weight', regressor.get_variable_names()) - regressor_weights = regressor.get_variable_value('linear//weight') - # Have to flatten weights since they come in (x, 1) shape. - self.assertAllClose(weights, regressor_weights.flatten(), rtol=1) - # TODO(ispir): Disable centered_bias. - # assert abs(bias - regressor.bias_) < 0.1 - - def testSdcaOptimizerRealValuedLinearFeatures(self): - """Tests LinearRegressor with SDCAOptimizer and real valued features.""" - x = [[1.2, 2.0, -1.5], [-2.0, 3.0, -0.5], [1.0, -0.5, 4.0]] - weights = [[3.0], [-1.2], [0.5]] - y = np.dot(x, weights) - - def input_fn(): - return { - 'example_id': constant_op.constant(['1', '2', '3']), - 'x': constant_op.constant(x), - 'weights': constant_op.constant([[10.0], [10.0], [10.0]]) - }, constant_op.constant(y) - - x_column = feature_column_lib.real_valued_column('x', dimension=3) - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id') - regressor = linear.LinearRegressor( - feature_columns=[x_column], - weight_column_name='weights', - optimizer=sdca_optimizer) - regressor.fit(input_fn=input_fn, steps=20) - loss = regressor.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss, 0.01) - self.assertIn('linear/x/weight', regressor.get_variable_names()) - regressor_weights = regressor.get_variable_value('linear/x/weight') - self.assertAllClose( - [w[0] for w in weights], regressor_weights.flatten(), rtol=0.1) - - def testSdcaOptimizerMixedFeaturesArbitraryWeights(self): - """Tests LinearRegressor with SDCAOptimizer and a mix of features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([0.6, 0.8, 0.3]), - 'sq_footage': - constant_op.constant([[900.0], [700.0], [600.0]]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - 'weights': - constant_op.constant([[3.0], [5.0], [7.0]]) - }, constant_op.constant([[1.55], [-1.25], [-3.0]]) - - price = feature_column_lib.real_valued_column('price') - sq_footage_bucket = feature_column_lib.bucketized_column( - feature_column_lib.real_valued_column('sq_footage'), - boundaries=[650.0, 800.0]) - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - sq_footage_country = feature_column_lib.crossed_column( - [sq_footage_bucket, country], hash_bucket_size=10) - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id', symmetric_l2_regularization=1.0) - regressor = linear.LinearRegressor( - feature_columns=[price, sq_footage_bucket, country, sq_footage_country], - weight_column_name='weights', - optimizer=sdca_optimizer) - regressor.fit(input_fn=input_fn, steps=20) - loss = regressor.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss, 0.05) - - def testSdcaOptimizerPartitionedVariables(self): - """Tests LinearRegressor with SDCAOptimizer with partitioned variables.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([0.6, 0.8, 0.3]), - 'sq_footage': - constant_op.constant([[900.0], [700.0], [600.0]]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - 'weights': - constant_op.constant([[3.0], [5.0], [7.0]]) - }, constant_op.constant([[1.55], [-1.25], [-3.0]]) - - price = feature_column_lib.real_valued_column('price') - sq_footage_bucket = feature_column_lib.bucketized_column( - feature_column_lib.real_valued_column('sq_footage'), - boundaries=[650.0, 800.0]) - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - sq_footage_country = feature_column_lib.crossed_column( - [sq_footage_bucket, country], hash_bucket_size=10) - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id', symmetric_l2_regularization=1.0, - partitioner=partitioned_variables.fixed_size_partitioner( - num_shards=2, axis=0)) - tf_config = { - 'cluster': { - run_config.TaskType.PS: ['fake_ps_0', 'fake_ps_1'] - } - } - with test.mock.patch.dict('os.environ', - {'TF_CONFIG': json.dumps(tf_config)}): - config = run_config.RunConfig() - # Because we did not start a distributed cluster, we need to pass an - # empty ClusterSpec, otherwise the device_setter will look for - # distributed jobs, such as "/job:ps" which are not present. - config._cluster_spec = server_lib.ClusterSpec({}) - - regressor = linear.LinearRegressor( - feature_columns=[price, sq_footage_bucket, country, sq_footage_country], - weight_column_name='weights', - optimizer=sdca_optimizer, - config=config) - regressor.fit(input_fn=input_fn, steps=20) - loss = regressor.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss, 0.05) - - def testSdcaOptimizerSparseFeaturesWithL1Reg(self): - """Tests LinearClassifier with SDCAOptimizer and sparse features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([[0.4], [0.6], [0.3]]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - 'weights': - constant_op.constant([[10.0], [10.0], [10.0]]) - }, constant_op.constant([[1.4], [-0.8], [2.6]]) - - price = feature_column_lib.real_valued_column('price') - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - # Regressor with no L1 regularization. - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id') - regressor = linear.LinearRegressor( - feature_columns=[price, country], - weight_column_name='weights', - optimizer=sdca_optimizer) - regressor.fit(input_fn=input_fn, steps=20) - no_l1_reg_loss = regressor.evaluate(input_fn=input_fn, steps=1)['loss'] - variable_names = regressor.get_variable_names() - self.assertIn('linear/price/weight', variable_names) - self.assertIn('linear/country/weights', variable_names) - no_l1_reg_weights = { - 'linear/price/weight': regressor.get_variable_value( - 'linear/price/weight'), - 'linear/country/weights': regressor.get_variable_value( - 'linear/country/weights'), - } - - # Regressor with L1 regularization. - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id', symmetric_l1_regularization=1.0) - regressor = linear.LinearRegressor( - feature_columns=[price, country], - weight_column_name='weights', - optimizer=sdca_optimizer) - regressor.fit(input_fn=input_fn, steps=20) - l1_reg_loss = regressor.evaluate(input_fn=input_fn, steps=1)['loss'] - l1_reg_weights = { - 'linear/price/weight': regressor.get_variable_value( - 'linear/price/weight'), - 'linear/country/weights': regressor.get_variable_value( - 'linear/country/weights'), - } - - # Unregularized loss is lower when there is no L1 regularization. - self.assertLess(no_l1_reg_loss, l1_reg_loss) - self.assertLess(no_l1_reg_loss, 0.05) - - # But weights returned by the regressor with L1 regularization have smaller - # L1 norm. - l1_reg_weights_norm, no_l1_reg_weights_norm = 0.0, 0.0 - for var_name in sorted(l1_reg_weights): - l1_reg_weights_norm += sum( - np.absolute(l1_reg_weights[var_name].flatten())) - no_l1_reg_weights_norm += sum( - np.absolute(no_l1_reg_weights[var_name].flatten())) - print('Var name: %s, value: %s' % - (var_name, no_l1_reg_weights[var_name].flatten())) - self.assertLess(l1_reg_weights_norm, no_l1_reg_weights_norm) - - def testSdcaOptimizerBiasOnly(self): - """Tests LinearClassifier with SDCAOptimizer and validates bias weight.""" - - def input_fn(): - """Testing the bias weight when it's the only feature present. - - All of the instances in this input only have the bias feature, and a - 1/4 of the labels are positive. This means that the expected weight for - the bias should be close to the average prediction, i.e 0.25. - Returns: - Training data for the test. - """ - num_examples = 40 - return { - 'example_id': - constant_op.constant([str(x + 1) for x in range(num_examples)]), - # place_holder is an empty column which is always 0 (absent), because - # LinearClassifier requires at least one column. - 'place_holder': - constant_op.constant([[0.0]] * num_examples), - }, constant_op.constant( - [[1 if i % 4 == 0 else 0] for i in range(num_examples)]) - - place_holder = feature_column_lib.real_valued_column('place_holder') - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id') - regressor = linear.LinearRegressor( - feature_columns=[place_holder], optimizer=sdca_optimizer) - regressor.fit(input_fn=input_fn, steps=100) - - self.assertNear( - regressor.get_variable_value('linear/bias_weight')[0], 0.25, err=0.1) - - def testSdcaOptimizerBiasAndOtherColumns(self): - """Tests LinearClassifier with SDCAOptimizer and validates bias weight.""" - - def input_fn(): - """Testing the bias weight when there are other features present. - - 1/2 of the instances in this input have feature 'a', the rest have - feature 'b', and we expect the bias to be added to each instance as well. - 0.4 of all instances that have feature 'a' are positive, and 0.2 of all - instances that have feature 'b' are positive. The labels in the dataset - are ordered to appear shuffled since SDCA expects shuffled data, and - converges faster with this pseudo-random ordering. - If the bias was centered we would expect the weights to be: - bias: 0.3 - a: 0.1 - b: -0.1 - Until b/29339026 is resolved, the bias gets regularized with the same - global value for the other columns, and so the expected weights get - shifted and are: - bias: 0.2 - a: 0.2 - b: 0.0 - Returns: - The test dataset. - """ - num_examples = 200 - half = int(num_examples / 2) - return { - 'example_id': - constant_op.constant([str(x + 1) for x in range(num_examples)]), - 'a': - constant_op.constant([[1]] * int(half) + [[0]] * int(half)), - 'b': - constant_op.constant([[0]] * int(half) + [[1]] * int(half)), - }, constant_op.constant( - [[x] - for x in [1, 0, 0, 1, 1, 0, 0, 0, 1, 0] * int(half / 10) + - [0, 1, 0, 0, 0, 0, 0, 0, 1, 0] * int(half / 10)]) - - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id') - regressor = linear.LinearRegressor( - feature_columns=[ - feature_column_lib.real_valued_column('a'), - feature_column_lib.real_valued_column('b') - ], - optimizer=sdca_optimizer) - - regressor.fit(input_fn=input_fn, steps=200) - - variable_names = regressor.get_variable_names() - self.assertIn('linear/bias_weight', variable_names) - self.assertIn('linear/a/weight', variable_names) - self.assertIn('linear/b/weight', variable_names) - # TODO(b/29339026): Change the expected results to expect a centered bias. - self.assertNear( - regressor.get_variable_value('linear/bias_weight')[0], 0.2, err=0.05) - self.assertNear( - regressor.get_variable_value('linear/a/weight')[0], 0.2, err=0.05) - self.assertNear( - regressor.get_variable_value('linear/b/weight')[0], 0.0, err=0.05) - - def testSdcaOptimizerBiasAndOtherColumnsFabricatedCentered(self): - """Tests LinearClassifier with SDCAOptimizer and validates bias weight.""" - - def input_fn(): - """Testing the bias weight when there are other features present. - - 1/2 of the instances in this input have feature 'a', the rest have - feature 'b', and we expect the bias to be added to each instance as well. - 0.1 of all instances that have feature 'a' have a label of 1, and 0.1 of - all instances that have feature 'b' have a label of -1. - We can expect the weights to be: - bias: 0.0 - a: 0.1 - b: -0.1 - Returns: - The test dataset. - """ - num_examples = 200 - half = int(num_examples / 2) - return { - 'example_id': - constant_op.constant([str(x + 1) for x in range(num_examples)]), - 'a': - constant_op.constant([[1]] * int(half) + [[0]] * int(half)), - 'b': - constant_op.constant([[0]] * int(half) + [[1]] * int(half)), - }, constant_op.constant([[1 if x % 10 == 0 else 0] for x in range(half)] + - [[-1 if x % 10 == 0 else 0] for x in range(half)]) - - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id') - regressor = linear.LinearRegressor( - feature_columns=[ - feature_column_lib.real_valued_column('a'), - feature_column_lib.real_valued_column('b') - ], - optimizer=sdca_optimizer) - - regressor.fit(input_fn=input_fn, steps=100) - - variable_names = regressor.get_variable_names() - self.assertIn('linear/bias_weight', variable_names) - self.assertIn('linear/a/weight', variable_names) - self.assertIn('linear/b/weight', variable_names) - self.assertNear( - regressor.get_variable_value('linear/bias_weight')[0], 0.0, err=0.05) - self.assertNear( - regressor.get_variable_value('linear/a/weight')[0], 0.1, err=0.05) - self.assertNear( - regressor.get_variable_value('linear/b/weight')[0], -0.1, err=0.05) - - -class LinearEstimatorTest(test.TestCase): - - def testExperimentIntegration(self): - cont_features = [ - feature_column_lib.real_valued_column( - 'feature', dimension=4) - ] - exp = experiment.Experiment( - estimator=linear.LinearEstimator(feature_columns=cont_features, - head=head_lib.regression_head()), - train_input_fn=test_data.iris_input_logistic_fn, - eval_input_fn=test_data.iris_input_logistic_fn) - exp.test() - - def testEstimatorContract(self): - estimator_test_utils.assert_estimator_contract(self, - linear.LinearEstimator) - - def testLinearRegression(self): - """Tests that loss goes down with training.""" - - def input_fn(): - return { - 'age': - constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[10.]]) - - language = feature_column_lib.sparse_column_with_hash_bucket('language', - 100) - age = feature_column_lib.real_valued_column('age') - - linear_estimator = linear.LinearEstimator(feature_columns=[age, language], - head=head_lib.regression_head()) - linear_estimator.fit(input_fn=input_fn, steps=100) - loss1 = linear_estimator.evaluate(input_fn=input_fn, steps=1)['loss'] - linear_estimator.fit(input_fn=input_fn, steps=400) - loss2 = linear_estimator.evaluate(input_fn=input_fn, steps=1)['loss'] - - self.assertLess(loss2, loss1) - self.assertLess(loss2, 0.5) - - def testPoissonRegression(self): - """Tests that loss goes down with training.""" - - def input_fn(): - return { - 'age': - constant_op.constant([1]), - 'language': - sparse_tensor.SparseTensor( - values=['english'], indices=[[0, 0]], dense_shape=[1, 1]) - }, constant_op.constant([[10.]]) - - language = feature_column_lib.sparse_column_with_hash_bucket('language', - 100) - age = feature_column_lib.real_valued_column('age') - - linear_estimator = linear.LinearEstimator( - feature_columns=[age, language], - head=head_lib.poisson_regression_head()) - linear_estimator.fit(input_fn=input_fn, steps=10) - loss1 = linear_estimator.evaluate(input_fn=input_fn, steps=1)['loss'] - linear_estimator.fit(input_fn=input_fn, steps=100) - loss2 = linear_estimator.evaluate(input_fn=input_fn, steps=1)['loss'] - - self.assertLess(loss2, loss1) - # Here loss of 2.1 implies a prediction of ~9.9998 - self.assertLess(loss2, 2.1) - - def testSDCANotSupported(self): - """Tests that we detect error for SDCA.""" - maintenance_cost = feature_column_lib.real_valued_column('maintenance_cost') - sq_footage = feature_column_lib.real_valued_column('sq_footage') - sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( - example_id_column='example_id') - with self.assertRaises(ValueError): - linear.LinearEstimator( - head=head_lib.regression_head(label_dimension=1), - feature_columns=[maintenance_cost, sq_footage], - optimizer=sdca_optimizer, - _joint_weights=True) - - -def boston_input_fn(): - boston = base.load_boston() - features = math_ops.cast( - array_ops.reshape(constant_op.constant(boston.data), [-1, 13]), - dtypes.float32) - labels = math_ops.cast( - array_ops.reshape(constant_op.constant(boston.target), [-1, 1]), - dtypes.float32) - return features, labels - - -class FeatureColumnTest(test.TestCase): - - def testTrain(self): - feature_columns = estimator.infer_real_valued_columns_from_input_fn( - boston_input_fn) - est = linear.LinearRegressor(feature_columns=feature_columns) - est.fit(input_fn=boston_input_fn, steps=1) - _ = est.evaluate(input_fn=boston_input_fn, steps=1) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/logistic_regressor.py b/tensorflow/contrib/learn/python/learn/estimators/logistic_regressor.py deleted file mode 100644 index 8981432f7f2..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/logistic_regressor.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Logistic regression (aka binary classifier) class (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. - -This defines some useful basic metrics for using logistic regression to classify -a binary event (0 vs 1). -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import metrics as metrics_lib -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import metric_key -from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import math_ops - - -def _get_model_fn_with_logistic_metrics(model_fn): - """Returns a model_fn with additional logistic metrics. - - Args: - model_fn: Model function with the signature: - `(features, labels, mode) -> (predictions, loss, train_op)`. - Expects the returned predictions to be probabilities in [0.0, 1.0]. - - Returns: - model_fn that can be used with Estimator. - """ - - def _model_fn(features, labels, mode, params): - """Model function that appends logistic evaluation metrics.""" - thresholds = params.get('thresholds') or [.5] - - predictions, loss, train_op = model_fn(features, labels, mode) - if mode == model_fn_lib.ModeKeys.EVAL: - eval_metric_ops = _make_logistic_eval_metric_ops( - labels=labels, - predictions=predictions, - thresholds=thresholds) - else: - eval_metric_ops = None - return model_fn_lib.ModelFnOps( - mode=mode, - predictions=predictions, - loss=loss, - train_op=train_op, - eval_metric_ops=eval_metric_ops, - output_alternatives={ - 'head': (constants.ProblemType.LOGISTIC_REGRESSION, { - 'predictions': predictions - }) - }) - - return _model_fn - - -# TODO(roumposg): Deprecate and delete after converting users to use head. -def LogisticRegressor( # pylint: disable=invalid-name - model_fn, thresholds=None, model_dir=None, config=None, - feature_engineering_fn=None): - """Builds a logistic regression Estimator for binary classification. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - This method provides a basic Estimator with some additional metrics for custom - binary classification models, including AUC, precision/recall and accuracy. - - Example: - - ```python - # See tf.contrib.learn.Estimator(...) for details on model_fn structure - def my_model_fn(...): - pass - - estimator = LogisticRegressor(model_fn=my_model_fn) - - # Input builders - def input_fn_train: - pass - - estimator.fit(input_fn=input_fn_train) - estimator.predict(x=x) - ``` - - Args: - model_fn: Model function with the signature: - `(features, labels, mode) -> (predictions, loss, train_op)`. - Expects the returned predictions to be probabilities in [0.0, 1.0]. - thresholds: List of floating point thresholds to use for accuracy, - precision, and recall metrics. If `None`, defaults to `[0.5]`. - model_dir: Directory to save model parameters, graphs, etc. This can also - be used to load checkpoints from the directory into a estimator to - continue training a previously saved model. - config: A RunConfig configuration object. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and - returns features and labels which will be fed - into the model. - - Returns: - An `Estimator` instance. - """ - return estimator.Estimator( - model_fn=_get_model_fn_with_logistic_metrics(model_fn), - model_dir=model_dir, - config=config, - params={'thresholds': thresholds}, - feature_engineering_fn=feature_engineering_fn) - - -def _make_logistic_eval_metric_ops(labels, predictions, thresholds): - """Returns a dictionary of evaluation metric ops for logistic regression. - - Args: - labels: The labels `Tensor`, or a dict with only one `Tensor` keyed by name. - predictions: The predictions `Tensor`. - thresholds: List of floating point thresholds to use for accuracy, - precision, and recall metrics. - - Returns: - A dict of metric results keyed by name. - """ - # If labels is a dict with a single key, unpack into a single tensor. - labels_tensor = labels - if isinstance(labels, dict) and len(labels) == 1: - labels_tensor = labels.values()[0] - - metrics = {} - metrics[metric_key.MetricKey.PREDICTION_MEAN] = metrics_lib.streaming_mean( - predictions) - metrics[metric_key.MetricKey.LABEL_MEAN] = metrics_lib.streaming_mean( - labels_tensor) - # Also include the streaming mean of the label as an accuracy baseline, as - # a reminder to users. - metrics[metric_key.MetricKey.ACCURACY_BASELINE] = metrics_lib.streaming_mean( - labels_tensor) - - metrics[metric_key.MetricKey.AUC] = metrics_lib.streaming_auc( - labels=labels_tensor, predictions=predictions) - - for threshold in thresholds: - predictions_at_threshold = math_ops.cast( - math_ops.greater_equal(predictions, threshold), - dtypes.float32, - name='predictions_at_threshold_%f' % threshold) - metrics[metric_key.MetricKey.ACCURACY_MEAN % threshold] = ( - metrics_lib.streaming_accuracy(labels=labels_tensor, - predictions=predictions_at_threshold)) - # Precision for positive examples. - metrics[metric_key.MetricKey.PRECISION_MEAN % threshold] = ( - metrics_lib.streaming_precision(labels=labels_tensor, - predictions=predictions_at_threshold)) - # Recall for positive examples. - metrics[metric_key.MetricKey.RECALL_MEAN % threshold] = ( - metrics_lib.streaming_recall(labels=labels_tensor, - predictions=predictions_at_threshold)) - - return metrics diff --git a/tensorflow/contrib/learn/python/learn/estimators/logistic_regressor_test.py b/tensorflow/contrib/learn/python/learn/estimators/logistic_regressor_test.py deleted file mode 100644 index ac2d10011e2..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/logistic_regressor_test.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for LogisticRegressor.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib import layers -from tensorflow.python.training import training_util -from tensorflow.contrib.layers.python.layers import optimizers -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.contrib.learn.python.learn.estimators import logistic_regressor -from tensorflow.contrib.learn.python.learn.estimators import metric_key -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import test - - -def _iris_data_input_fn(): - # Converts iris data to a logistic regression problem. - iris = base.load_iris() - ids = np.where((iris.target == 0) | (iris.target == 1)) - features = constant_op.constant(iris.data[ids], dtype=dtypes.float32) - labels = constant_op.constant(iris.target[ids], dtype=dtypes.float32) - labels = array_ops.reshape(labels, labels.get_shape().concatenate(1)) - return features, labels - - -def _logistic_regression_model_fn(features, labels, mode): - _ = mode - logits = layers.linear( - features, - 1, - weights_initializer=init_ops.zeros_initializer(), - # Intentionally uses really awful initial values so that - # AUC/precision/recall/etc will change meaningfully even on a toy dataset. - biases_initializer=init_ops.constant_initializer(-10.0)) - predictions = math_ops.sigmoid(logits) - loss = losses.sigmoid_cross_entropy(labels, logits) - train_op = optimizers.optimize_loss( - loss, - training_util.get_global_step(), - optimizer='Adagrad', - learning_rate=0.1) - return predictions, loss, train_op - - -class LogisticRegressorTest(test.TestCase): - - def test_fit_and_evaluate_metrics(self): - """Tests basic fit and evaluate, and checks the evaluation metrics.""" - regressor = logistic_regressor.LogisticRegressor( - model_fn=_logistic_regression_model_fn) - - # Get some (intentionally horrible) baseline metrics. - regressor.fit(input_fn=_iris_data_input_fn, steps=1) - eval_metrics = regressor.evaluate(input_fn=_iris_data_input_fn, steps=1) - self.assertNear( - 0.0, eval_metrics[metric_key.MetricKey.PREDICTION_MEAN], err=1e-3) - self.assertNear( - 0.5, eval_metrics[metric_key.MetricKey.LABEL_MEAN], err=1e-6) - self.assertNear( - 0.5, eval_metrics[metric_key.MetricKey.ACCURACY_BASELINE], err=1e-6) - self.assertNear(0.5, eval_metrics[metric_key.MetricKey.AUC], err=1e-6) - self.assertNear( - 0.5, eval_metrics[metric_key.MetricKey.ACCURACY_MEAN % 0.5], err=1e-6) - self.assertNear( - 0.0, eval_metrics[metric_key.MetricKey.PRECISION_MEAN % 0.5], err=1e-6) - self.assertNear( - 0.0, eval_metrics[metric_key.MetricKey.RECALL_MEAN % 0.5], err=1e-6) - - # Train for more steps and check the metrics again. - regressor.fit(input_fn=_iris_data_input_fn, steps=100) - eval_metrics = regressor.evaluate(input_fn=_iris_data_input_fn, steps=1) - # Mean prediction moves from ~0.0 to ~0.5 as we stop predicting all 0's. - self.assertNear( - 0.5, eval_metrics[metric_key.MetricKey.PREDICTION_MEAN], err=1e-2) - # Label mean and baseline both remain the same at 0.5. - self.assertNear( - 0.5, eval_metrics[metric_key.MetricKey.LABEL_MEAN], err=1e-6) - self.assertNear( - 0.5, eval_metrics[metric_key.MetricKey.ACCURACY_BASELINE], err=1e-6) - # AUC improves from 0.5 to 1.0. - self.assertNear(1.0, eval_metrics[metric_key.MetricKey.AUC], err=1e-6) - # Accuracy improves from 0.5 to >0.9. - self.assertTrue( - eval_metrics[metric_key.MetricKey.ACCURACY_MEAN % 0.5] > 0.9) - # Precision improves from 0.0 to 1.0. - self.assertNear( - 1.0, eval_metrics[metric_key.MetricKey.PRECISION_MEAN % 0.5], err=1e-6) - # Recall improves from 0.0 to >0.9. - self.assertTrue(eval_metrics[metric_key.MetricKey.RECALL_MEAN % 0.5] > 0.9) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/metric_key.py b/tensorflow/contrib/learn/python/learn/estimators/metric_key.py deleted file mode 100644 index f264248e44d..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/metric_key.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Enum for metric keys (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -class MetricKey(object): - """Metric key strings (deprecated).""" - - LOSS = "loss" - AUC = "auc" - AUC_PR = "auc_precision_recall" - CLASS_AUC = "auc/class%d" - CLASS_AUC_PR = "auc_precision_recall/class%d" - PREDICTION_MEAN = "labels/prediction_mean" - CLASS_PREDICTION_MEAN = "labels/prediction_mean/class%d" - CLASS_LOGITS_MEAN = "labels/logits_mean/class%d" - CLASS_PROBABILITY_MEAN = "labels/probability_mean/class%d" - LABEL_MEAN = "labels/actual_label_mean" - CLASS_LABEL_MEAN = "labels/actual_label_mean/class%d" - ACCURACY = "accuracy" - ACCURACY_BASELINE = "accuracy/baseline_label_mean" - ACCURACY_MEAN = "accuracy/threshold_%f_mean" - PRECISION_MEAN = "precision/positive_threshold_%f_mean" - RECALL_MEAN = "recall/positive_threshold_%f_mean" diff --git a/tensorflow/contrib/learn/python/learn/estimators/model_fn.py b/tensorflow/contrib/learn/python/learn/estimators/model_fn.py deleted file mode 100644 index fcabbf69425..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/model_fn.py +++ /dev/null @@ -1,307 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Classes and methods related to model_fn (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -import six - -from tensorflow.contrib.framework import get_graph_from_inputs -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import metric_key -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.python.estimator import model_fn as core_model_fn_lib -from tensorflow.python.estimator.export import export_output as core_export_lib -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.saved_model import signature_constants -from tensorflow.python.training import session_run_hook -from tensorflow.python.util.deprecation import deprecated - - -class ModeKeys(object): - """Standard names for model modes (deprecated). - - THIS CLASS IS DEPRECATED. - - The following standard keys are defined: - - * `TRAIN`: training mode. - * `EVAL`: evaluation mode. - * `INFER`: inference mode. - """ - - TRAIN = 'train' - EVAL = 'eval' - INFER = 'infer' - - @classmethod - def validate(cls, key): - if key not in (cls.TRAIN, cls.EVAL, cls.INFER): - raise ValueError('Invalid mode %s.' % key) - - -class ModelFnOps( - collections.namedtuple('ModelFnOps', [ - 'predictions', 'loss', 'train_op', 'eval_metric_ops', - 'output_alternatives', 'training_chief_hooks', 'training_hooks', - 'scaffold', 'mode' - ])): - """Ops returned from a model_fn. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - @deprecated(None, 'When switching to tf.estimator.Estimator, use ' - 'tf.estimator.EstimatorSpec. You can use the `estimator_spec`' - ' method to create an equivalent one.') - def __new__(cls, - mode, - predictions=None, - loss=None, - train_op=None, - eval_metric_ops=None, - output_alternatives=None, - training_chief_hooks=None, - training_hooks=None, - scaffold=None): - """Creates a validated `ModelFnOps` instance. - - For a multi-headed model, the predictions dict here will contain the outputs - of all of the heads. However: at serving time, requests will be made - specifically for one or more heads, and the RPCs used for these requests may - differ by problem type (i.e., regression, classification, other). The - purpose of the output_alternatives dict is to aid in exporting a SavedModel - from which such head-specific queries can be served. These - output_alternatives will be combined with input_alternatives (see - `saved_model_export_utils`) to produce a set of `SignatureDef`s specifying - the valid requests that can be served from this model. - - For a single-headed model, it is still adviseable to provide - output_alternatives with a single entry, because this is how the problem - type is communicated for export and serving. If output_alternatives is not - given, the resulting SavedModel will support only one head of unspecified - type. - - Args: - mode: One of `ModeKeys`. Specifies if this training, evaluation or - prediction. - predictions: Predictions `Tensor` or dict of `Tensor`. - loss: Training loss `Tensor`. - train_op: Op for the training step. - eval_metric_ops: Dict of metric results keyed by name. The values of the - dict are the results of calling a metric function, such as `Tensor`. - output_alternatives: a dict of - `{submodel_name: (problem_type, {tensor_name: Tensor})}`, where - `submodel_name` is a submodel identifier that should be consistent - across the pipeline (here likely taken from the name of each `Head`, - for models that use them), `problem_type` is a `ProblemType`, - `tensor_name` is a symbolic name for an output Tensor possibly but not - necessarily taken from `PredictionKey`, and `Tensor` is the - corresponding output Tensor itself. - training_chief_hooks: A list of `SessionRunHook` objects that will be - run on the chief worker during training. - training_hooks: A list of `SessionRunHook` objects that will be run on - all workers during training. - scaffold: A `tf.compat.v1.train.Scaffold` object that can be used to set - initialization, saver, and more to be used in training. - - Returns: - A validated `ModelFnOps` object. - - Raises: - ValueError: If validation fails. - """ - ModeKeys.validate(mode) - - # Assert all ops are from the same graph. - get_graph_from_inputs((predictions, loss, train_op)) - - # Validate train_op. - if train_op is None: - if mode == ModeKeys.TRAIN: - raise ValueError('Missing train_op.') - elif not isinstance(train_op, ops.Operation): - # TODO(ptucker): Should this be allowed? Consider raising error. - train_op = ops.convert_to_tensor(train_op).op - - # Validate loss. - if loss is None: - if mode in (ModeKeys.TRAIN, ModeKeys.EVAL): - raise ValueError('Missing loss.') - else: - loss = ops.convert_to_tensor(loss) - loss_shape = loss.get_shape() - if loss_shape.num_elements() not in (None, 1): - raise ValueError('Loss must be scalar: %s.' % loss) - if not loss_shape.is_compatible_with(tensor_shape.TensorShape([])): - loss = array_ops.reshape(loss, []) - - # Validate predictions. - if predictions is None: - if mode == ModeKeys.INFER or mode == ModeKeys.EVAL: - raise ValueError('Missing predictions.') - else: - if isinstance(predictions, dict): - predictions = { - k: sparse_tensor.convert_to_tensor_or_sparse_tensor(v) - for k, v in six.iteritems(predictions) - } - else: - predictions = sparse_tensor.convert_to_tensor_or_sparse_tensor( - predictions) - - # Validate eval_metric_ops - if eval_metric_ops is None: - eval_metric_ops = {} - else: - if not isinstance(eval_metric_ops, dict): - raise ValueError('eval_metric_ops must be a dict.') - - # Validate hooks - if training_chief_hooks is None: - training_chief_hooks = [] - if training_hooks is None: - training_hooks = [] - for hook in training_hooks + training_chief_hooks: - if not isinstance(hook, session_run_hook.SessionRunHook): - raise TypeError('All hooks returned from model_fn must be ' - 'SessionRunHook instances, got instance of %s: %s' % - (type(hook), hook)) - - return super(ModelFnOps, cls).__new__( - cls, - predictions=predictions, - loss=loss, - train_op=train_op, - eval_metric_ops=eval_metric_ops, - output_alternatives=output_alternatives, - training_chief_hooks=training_chief_hooks, - training_hooks=training_hooks, - scaffold=scaffold, - mode=mode) - - def estimator_spec(self, default_serving_output_alternative_key=None): - """Creates an equivalent `EstimatorSpec`. - - Args: - default_serving_output_alternative_key: Required for multiple heads. If - you have multiple entries in `output_alternatives` dict (comparable to - multiple heads), `EstimatorSpec` requires a default head that will be - used if a Servo request does not explicitly mention which head to infer - on. Pass the key of the output alternative here that you want to - designate as default. A separate ExportOutpout for this default head - will be added to the export_outputs dict with the special key - saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY, unless there is - already an enry in output_alternatives with this special key. - - Returns: - Instance of `EstimatorSpec` that is equivalent to this `ModelFnOps` - - Raises: - ValueError: If problem type is unknown. - """ - def _scores(output_tensors): - scores = output_tensors.get(prediction_key.PredictionKey.SCORES) - if scores is None: - scores = output_tensors.get(prediction_key.PredictionKey.PROBABILITIES) - return scores - - def _classes(output_tensors): # pylint: disable=missing-docstring - classes = output_tensors.get(prediction_key.PredictionKey.CLASSES) - if classes is None: - logging.warning( - 'classes is None, Servo inference will not have class ids.') - return None - elif classes.dtype != dtypes.string: - # Servo classification can only serve string classes - logging.warning( - 'classes is not string, Servo inference will not have class ids.') - return None - - return classes - - def _export_output(problem_type, predictions): # pylint: disable=missing-docstring - if problem_type == constants.ProblemType.LINEAR_REGRESSION: - return core_export_lib.RegressionOutput(_scores(predictions)) - - if (problem_type == constants.ProblemType.CLASSIFICATION or - problem_type == constants.ProblemType.LOGISTIC_REGRESSION): - return core_export_lib.ClassificationOutput( - scores=_scores(predictions), classes=_classes(predictions)) - - if problem_type == constants.ProblemType.UNSPECIFIED: - return core_export_lib.PredictOutput(predictions) - - raise ValueError('Unknown problem_type=%s' % problem_type) - - # Converts output_alternatives - export_outputs_dict = None - if self.output_alternatives: - output_alternatives = self.output_alternatives - # Adds default output_alternative if needed. - if (len(output_alternatives) > 1 and - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY not in - output_alternatives): - output_alternatives = output_alternatives.copy() - output_alternatives[ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] = ( - output_alternatives[default_serving_output_alternative_key]) - export_outputs_dict = {key: _export_output(*val) for key, val in - output_alternatives.items()} - - def _get_eval_metric_ops(): - """Returns self.eval_metric_ops without loss metric.""" - result = {} - for key, value in six.iteritems(self.eval_metric_ops): - if key != metric_key.MetricKey.LOSS: - result[key] = value - return result - - # Convert the contrib mode enum to the core mode enum. - # Note: mode already validated in __new__(). - if self.mode == ModeKeys.TRAIN: - core_mode = core_model_fn_lib.ModeKeys.TRAIN - elif self.mode == ModeKeys.EVAL: - core_mode = core_model_fn_lib.ModeKeys.EVAL - elif self.mode == ModeKeys.INFER: - core_mode = core_model_fn_lib.ModeKeys.PREDICT - - return core_model_fn_lib.EstimatorSpec( - mode=core_mode, - predictions=self.predictions, - loss=self.loss, - train_op=self.train_op, - eval_metric_ops=_get_eval_metric_ops(), - export_outputs=export_outputs_dict, - training_chief_hooks=self.training_chief_hooks, - training_hooks=self.training_hooks, - scaffold=self.scaffold) diff --git a/tensorflow/contrib/learn/python/learn/estimators/model_fn_test.py b/tensorflow/contrib/learn/python/learn/estimators/model_fn_test.py deleted file mode 100644 index 284e2cfd7ac..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/model_fn_test.py +++ /dev/null @@ -1,297 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ModelFnOps tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import six - -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.python.client import session -from tensorflow.python.estimator.export import export_output as core_export_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.platform import test -from tensorflow.python.saved_model import signature_constants -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.training import monitored_session - - -class ModelFnopsTest(test.TestCase): - """Multi-output tests.""" - - def create_predictions(self): - probabilities = constant_op.constant([1., 1., 1.]) - scores = constant_op.constant([1., 2., 3.]) - classes = constant_op.constant([b"0", b"1", b"2"]) - return { - "probabilities": probabilities, - "scores": scores, - "classes": classes} - - def create_model_fn_ops(self, predictions, output_alternatives, - mode=model_fn.ModeKeys.INFER): - - return model_fn.ModelFnOps( - model_fn.ModeKeys.INFER, - predictions=predictions, - loss=constant_op.constant([1]), - train_op=control_flow_ops.no_op(), - eval_metric_ops={ - "metric_key": (constant_op.constant(1.), control_flow_ops.no_op()), - "loss": (constant_op.constant(1.), control_flow_ops.no_op()), - }, - training_chief_hooks=[basic_session_run_hooks.StepCounterHook()], - training_hooks=[basic_session_run_hooks.StepCounterHook()], - output_alternatives=output_alternatives, - scaffold=monitored_session.Scaffold()) - - def assertEquals_except_export_and_eval_loss( - self, model_fn_ops, estimator_spec): - expected_eval_metric_ops = {} - for key, value in six.iteritems(model_fn_ops.eval_metric_ops): - if key != "loss": - expected_eval_metric_ops[key] = value - self.assertEqual(model_fn_ops.predictions, estimator_spec.predictions) - self.assertEqual(model_fn_ops.loss, estimator_spec.loss) - self.assertEqual(model_fn_ops.train_op, estimator_spec.train_op) - self.assertEqual(expected_eval_metric_ops, - estimator_spec.eval_metric_ops) - self.assertAllEqual(model_fn_ops.training_chief_hooks, - estimator_spec.training_chief_hooks) - self.assertAllEqual(model_fn_ops.training_hooks, - estimator_spec.training_hooks) - self.assertEqual(model_fn_ops.scaffold, estimator_spec.scaffold) - - def testEstimatorSpec_except_export(self): - predictions = self.create_predictions() - model_fn_ops = self.create_model_fn_ops( - predictions, None, mode=model_fn.ModeKeys.INFER) - - estimator_spec = model_fn_ops.estimator_spec() - self.assertEquals_except_export_and_eval_loss(model_fn_ops, estimator_spec) - - def testEstimatorSpec_export_regression_with_scores(self): - predictions = self.create_predictions() - output_alternatives = {"regression_head": ( - constants.ProblemType.LINEAR_REGRESSION, predictions)} - model_fn_ops = self.create_model_fn_ops( - predictions, output_alternatives, mode=model_fn.ModeKeys.INFER) - - estimator_spec = model_fn_ops.estimator_spec() - self.assertEquals_except_export_and_eval_loss(model_fn_ops, estimator_spec) - - with session.Session(): - regression_output = estimator_spec.export_outputs["regression_head"] - self.assertTrue(isinstance( - regression_output, core_export_lib.RegressionOutput)) - self.assertAllEqual(predictions["scores"].eval(), - regression_output.value.eval()) - - def testEstimatorSpec_export_regression_with_probabilities(self): - predictions = self.create_predictions() - output_alternatives_predictions = predictions.copy() - del output_alternatives_predictions["scores"] - output_alternatives = {"regression_head": ( - constants.ProblemType.LINEAR_REGRESSION, - output_alternatives_predictions)} - model_fn_ops = self.create_model_fn_ops( - predictions, output_alternatives, mode=model_fn.ModeKeys.INFER) - - estimator_spec = model_fn_ops.estimator_spec() - self.assertEquals_except_export_and_eval_loss(model_fn_ops, estimator_spec) - - with session.Session(): - regression_output = estimator_spec.export_outputs["regression_head"] - self.assertTrue(isinstance( - regression_output, core_export_lib.RegressionOutput)) - self.assertAllEqual(predictions["probabilities"].eval(), - regression_output.value.eval()) - - def testEstimatorSpec_export_classification(self): - predictions = self.create_predictions() - output_alternatives = {"classification_head": ( - constants.ProblemType.CLASSIFICATION, predictions)} - model_fn_ops = self.create_model_fn_ops( - predictions, output_alternatives, mode=model_fn.ModeKeys.INFER) - - estimator_spec = model_fn_ops.estimator_spec() - self.assertEquals_except_export_and_eval_loss(model_fn_ops, estimator_spec) - - with session.Session(): - classification_output = estimator_spec.export_outputs[ - "classification_head"] - self.assertTrue(isinstance(classification_output, - core_export_lib.ClassificationOutput)) - self.assertAllEqual(predictions["scores"].eval(), - classification_output.scores.eval()) - self.assertAllEqual(predictions["classes"].eval(), - classification_output.classes.eval()) - - def testEstimatorSpec_export_classification_with_missing_scores(self): - predictions = self.create_predictions() - output_alternatives_predictions = predictions.copy() - del output_alternatives_predictions["scores"] - output_alternatives = {"classification_head": ( - constants.ProblemType.CLASSIFICATION, output_alternatives_predictions)} - model_fn_ops = self.create_model_fn_ops( - predictions, output_alternatives, mode=model_fn.ModeKeys.INFER) - - estimator_spec = model_fn_ops.estimator_spec() - self.assertEquals_except_export_and_eval_loss(model_fn_ops, estimator_spec) - - with session.Session(): - classification_output = estimator_spec.export_outputs[ - "classification_head"] - self.assertTrue(isinstance(classification_output, - core_export_lib.ClassificationOutput)) - self.assertAllEqual(predictions["probabilities"].eval(), - classification_output.scores.eval()) - self.assertAllEqual(predictions["classes"].eval(), - classification_output.classes.eval()) - - def testEstimatorSpec_export_classification_with_missing_scores_proba(self): - predictions = self.create_predictions() - output_alternatives_predictions = predictions.copy() - del output_alternatives_predictions["scores"] - del output_alternatives_predictions["probabilities"] - output_alternatives = {"classification_head": ( - constants.ProblemType.CLASSIFICATION, output_alternatives_predictions)} - model_fn_ops = self.create_model_fn_ops( - predictions, output_alternatives, mode=model_fn.ModeKeys.INFER) - - estimator_spec = model_fn_ops.estimator_spec() - self.assertEquals_except_export_and_eval_loss(model_fn_ops, estimator_spec) - - with session.Session(): - classification_output = estimator_spec.export_outputs[ - "classification_head"] - self.assertTrue(isinstance(classification_output, - core_export_lib.ClassificationOutput)) - self.assertIsNone(classification_output.scores) - self.assertAllEqual(predictions["classes"].eval(), - classification_output.classes.eval()) - - def testEstimatorSpec_export_classification_with_missing_classes(self): - predictions = self.create_predictions() - output_alternatives_predictions = predictions.copy() - del output_alternatives_predictions["classes"] - output_alternatives = {"classification_head": ( - constants.ProblemType.CLASSIFICATION, output_alternatives_predictions)} - model_fn_ops = self.create_model_fn_ops( - predictions, output_alternatives, mode=model_fn.ModeKeys.INFER) - - estimator_spec = model_fn_ops.estimator_spec() - self.assertEquals_except_export_and_eval_loss(model_fn_ops, estimator_spec) - - with session.Session(): - classification_output = estimator_spec.export_outputs[ - "classification_head"] - self.assertTrue(isinstance(classification_output, - core_export_lib.ClassificationOutput)) - self.assertAllEqual(predictions["scores"].eval(), - classification_output.scores.eval()) - self.assertIsNone(classification_output.classes) - - def testEstimatorSpec_export_classification_with_nonstring_classes(self): - predictions = self.create_predictions() - output_alternatives_predictions = predictions.copy() - output_alternatives_predictions["classes"] = constant_op.constant( - [1, 2, 3]) - output_alternatives = {"classification_head": ( - constants.ProblemType.CLASSIFICATION, output_alternatives_predictions)} - model_fn_ops = self.create_model_fn_ops( - predictions, output_alternatives, mode=model_fn.ModeKeys.INFER) - - estimator_spec = model_fn_ops.estimator_spec() - self.assertEquals_except_export_and_eval_loss(model_fn_ops, estimator_spec) - - with session.Session(): - classification_output = estimator_spec.export_outputs[ - "classification_head"] - self.assertTrue(isinstance(classification_output, - core_export_lib.ClassificationOutput)) - self.assertAllEqual(predictions["scores"].eval(), - classification_output.scores.eval()) - self.assertIsNone(classification_output.classes) - - def testEstimatorSpec_export_logistic(self): - predictions = self.create_predictions() - output_alternatives = {"logistic_head": ( - constants.ProblemType.LOGISTIC_REGRESSION, predictions)} - model_fn_ops = self.create_model_fn_ops( - predictions, output_alternatives, mode=model_fn.ModeKeys.INFER) - - estimator_spec = model_fn_ops.estimator_spec() - self.assertEquals_except_export_and_eval_loss(model_fn_ops, estimator_spec) - - with session.Session(): - logistic_output = estimator_spec.export_outputs["logistic_head"] - self.assertTrue(isinstance(logistic_output, - core_export_lib.ClassificationOutput)) - self.assertAllEqual(predictions["scores"].eval(), - logistic_output.scores.eval()) - self.assertAllEqual(predictions["classes"].eval(), - logistic_output.classes.eval()) - - def testEstimatorSpec_export_unspecified(self): - predictions = self.create_predictions() - output_alternatives = {"unspecified_head": ( - constants.ProblemType.UNSPECIFIED, predictions)} - - model_fn_ops = self.create_model_fn_ops( - predictions, output_alternatives, mode=model_fn.ModeKeys.INFER) - - estimator_spec = model_fn_ops.estimator_spec() - self.assertEquals_except_export_and_eval_loss(model_fn_ops, estimator_spec) - - with session.Session(): - unspecified_output = estimator_spec.export_outputs["unspecified_head"] - self.assertTrue(isinstance(unspecified_output, - core_export_lib.PredictOutput)) - self.assertEqual(predictions, unspecified_output.outputs) - - def testEstimatorSpec_export_multihead(self): - predictions = self.create_predictions() - output_alternatives = { - "regression_head": ( - constants.ProblemType.LINEAR_REGRESSION, predictions), - "classification_head": ( - constants.ProblemType.CLASSIFICATION, predictions)} - model_fn_ops = self.create_model_fn_ops( - predictions, output_alternatives, mode=model_fn.ModeKeys.INFER) - - estimator_spec = model_fn_ops.estimator_spec("regression_head") - self.assertEquals_except_export_and_eval_loss(model_fn_ops, estimator_spec) - - with session.Session(): - regression_output = estimator_spec.export_outputs["regression_head"] - self.assertTrue(isinstance( - regression_output, core_export_lib.RegressionOutput)) - self.assertAllEqual(predictions["scores"].eval(), - regression_output.value.eval()) - - default_output = estimator_spec.export_outputs[ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] - self.assertTrue(isinstance(default_output, - core_export_lib.RegressionOutput)) - self.assertAllEqual(predictions["scores"].eval(), - default_output.value.eval()) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/multioutput_test.py b/tensorflow/contrib/learn/python/learn/estimators/multioutput_test.py deleted file mode 100644 index 325c543a080..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/multioutput_test.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Multi-output tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random - -import numpy as np - -from tensorflow.contrib.learn.python import learn -from tensorflow.contrib.learn.python.learn.estimators._sklearn import mean_squared_error -from tensorflow.python.platform import test - - -class MultiOutputTest(test.TestCase): - """Multi-output tests.""" - - def testMultiRegression(self): - random.seed(42) - rng = np.random.RandomState(1) - x = np.sort(200 * rng.rand(100, 1) - 100, axis=0) - y = np.array([np.pi * np.sin(x).ravel(), np.pi * np.cos(x).ravel()]).T - regressor = learn.LinearRegressor( - feature_columns=learn.infer_real_valued_columns_from_input(x), - label_dimension=2) - regressor.fit(x, y, steps=100) - score = mean_squared_error(np.array(list(regressor.predict_scores(x))), y) - self.assertLess(score, 10, "Failed with score = {0}".format(score)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/nonlinear_test.py b/tensorflow/contrib/learn/python/learn/estimators/nonlinear_test.py deleted file mode 100644 index 8cf62707926..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/nonlinear_test.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Non-linear estimator tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random - -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.contrib.learn.python.learn.estimators import dnn -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.python.framework import random_seed -from tensorflow.python.platform import test - - -class NonLinearTest(test.TestCase): - """Non-linear estimator tests.""" - - def setUp(self): - random.seed(42) - random_seed.set_random_seed(42) - - def testIrisDNN(self): - iris = base.load_iris() - feature_columns = [feature_column.real_valued_column("", dimension=4)] - classifier = dnn.DNNClassifier( - feature_columns=feature_columns, - hidden_units=[10, 20, 10], - n_classes=3, - config=run_config.RunConfig(tf_random_seed=1)) - classifier.fit(iris.data, iris.target, max_steps=200) - variable_names = classifier.get_variable_names() - self.assertEqual( - classifier.get_variable_value("dnn/hiddenlayer_0/weights").shape, - (4, 10)) - self.assertEqual( - classifier.get_variable_value("dnn/hiddenlayer_1/weights").shape, - (10, 20)) - self.assertEqual( - classifier.get_variable_value("dnn/hiddenlayer_2/weights").shape, - (20, 10)) - self.assertEqual( - classifier.get_variable_value("dnn/logits/weights").shape, (10, 3)) - self.assertIn("dnn/hiddenlayer_0/biases", variable_names) - self.assertIn("dnn/hiddenlayer_1/biases", variable_names) - self.assertIn("dnn/hiddenlayer_2/biases", variable_names) - self.assertIn("dnn/logits/biases", variable_names) - - def testBostonDNN(self): - boston = base.load_boston() - feature_columns = [feature_column.real_valued_column("", dimension=13)] - regressor = dnn.DNNRegressor( - feature_columns=feature_columns, - hidden_units=[10, 20, 10], - config=run_config.RunConfig(tf_random_seed=1)) - regressor.fit(boston.data, - boston.target, - steps=300, - batch_size=boston.data.shape[0]) - weights = ([regressor.get_variable_value("dnn/hiddenlayer_0/weights")] + - [regressor.get_variable_value("dnn/hiddenlayer_1/weights")] + - [regressor.get_variable_value("dnn/hiddenlayer_2/weights")] + - [regressor.get_variable_value("dnn/logits/weights")]) - self.assertEqual(weights[0].shape, (13, 10)) - self.assertEqual(weights[1].shape, (10, 20)) - self.assertEqual(weights[2].shape, (20, 10)) - self.assertEqual(weights[3].shape, (10, 1)) - - biases = ([regressor.get_variable_value("dnn/hiddenlayer_0/biases")] + - [regressor.get_variable_value("dnn/hiddenlayer_1/biases")] + - [regressor.get_variable_value("dnn/hiddenlayer_2/biases")] + - [regressor.get_variable_value("dnn/logits/biases")]) - self.assertEqual(biases[0].shape, (10,)) - self.assertEqual(biases[1].shape, (20,)) - self.assertEqual(biases[2].shape, (10,)) - self.assertEqual(biases[3].shape, (1,)) - - def testDNNDropout0(self): - # Dropout prob == 0. - iris = base.load_iris() - feature_columns = [feature_column.real_valued_column("", dimension=4)] - classifier = dnn.DNNClassifier( - feature_columns=feature_columns, - hidden_units=[10, 20, 10], - n_classes=3, - dropout=0.0, - config=run_config.RunConfig(tf_random_seed=1)) - classifier.fit(iris.data, iris.target, max_steps=200) - - def testDNNDropout0_1(self): - # Dropping only a little. - iris = base.load_iris() - feature_columns = [feature_column.real_valued_column("", dimension=4)] - classifier = dnn.DNNClassifier( - feature_columns=feature_columns, - hidden_units=[10, 20, 10], - n_classes=3, - dropout=0.1, - config=run_config.RunConfig(tf_random_seed=1)) - classifier.fit(iris.data, iris.target, max_steps=200) - - def testDNNDropout0_9(self): - # Dropping out most of it. - iris = base.load_iris() - feature_columns = [feature_column.real_valued_column("", dimension=4)] - classifier = dnn.DNNClassifier( - feature_columns=feature_columns, - hidden_units=[10, 20, 10], - n_classes=3, - dropout=0.9, - config=run_config.RunConfig(tf_random_seed=1)) - classifier.fit(iris.data, iris.target, max_steps=200) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/prediction_key.py b/tensorflow/contrib/learn/python/learn/estimators/prediction_key.py deleted file mode 100644 index 6fd2fc9d592..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/prediction_key.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Enum for model prediction keys (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. - -This file is obsoleted in the move of Estimator to core. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -class PredictionKey(object): - """THIS CLASS IS DEPRECATED.""" - - CLASSES = "classes" - PROBABILITIES = "probabilities" - LOGITS = "logits" - LOGISTIC = "logistic" - SCORES = "scores" - TOP_K = "top_k" - GENERIC = "output" diff --git a/tensorflow/contrib/learn/python/learn/estimators/regression_test.py b/tensorflow/contrib/learn/python/learn/estimators/regression_test.py deleted file mode 100644 index fef0a084d17..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/regression_test.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Linear regression tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.learn.python import learn -from tensorflow.python.platform import test - - -class RegressionTest(test.TestCase): - """Linear regression tests.""" - - def testLinearRegression(self): - rng = np.random.RandomState(67) - n = 1000 - n_weights = 10 - bias = 2 - x = rng.uniform(-1, 1, (n, n_weights)) - weights = 10 * rng.randn(n_weights) - y = np.dot(x, weights) - y += rng.randn(len(x)) * 0.05 + rng.normal(bias, 0.01) - regressor = learn.LinearRegressor( - feature_columns=learn.infer_real_valued_columns_from_input(x), - optimizer="SGD") - regressor.fit(x, y, steps=200) - self.assertIn("linear//weight", regressor.get_variable_names()) - regressor_weights = regressor.get_variable_value("linear//weight") - # Have to flatten weights since they come in (x, 1) shape. - self.assertAllClose(weights, regressor_weights.flatten(), rtol=0.01) - # TODO(ispir): Disable centered_bias. - # assert abs(bias - regressor.bias_) < 0.1 - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/rnn_common.py b/tensorflow/contrib/learn/python/learn/estimators/rnn_common.py deleted file mode 100644 index 215022e5d9e..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/rnn_common.py +++ /dev/null @@ -1,308 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Common operations for RNN Estimators (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import metrics -from tensorflow.contrib import rnn as contrib_rnn -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops - - -# NOTE(jtbates): As of February 10, 2017, some of the `RNNKeys` have been -# removed and replaced with values from `prediction_key.PredictionKey`. The key -# `RNNKeys.PREDICTIONS_KEY` has been replaced by -# `prediction_key.PredictionKey.SCORES` for regression and -# `prediction_key.PredictionKey.CLASSES` for classification. The key -# `RNNKeys.PROBABILITIES_KEY` has been replaced by -# `prediction_key.PredictionKey.PROBABILITIES`. -class RNNKeys(object): - FINAL_STATE_KEY = 'final_state' - LABELS_KEY = '__labels__' - SEQUENCE_LENGTH_KEY = 'sequence_length' - STATE_PREFIX = 'rnn_cell_state' - - -class PredictionType(object): - """Enum-like values for the type of prediction that the model makes. - """ - SINGLE_VALUE = 1 - MULTIPLE_VALUE = 2 - - -_CELL_TYPES = {'basic_rnn': contrib_rnn.BasicRNNCell, - 'lstm': contrib_rnn.LSTMCell, - 'gru': contrib_rnn.GRUCell,} - - -def _get_single_cell(cell_type, num_units): - """Constructs and return a single `RNNCell`. - - Args: - cell_type: Either a string identifying the `RNNCell` type or a subclass of - `RNNCell`. - num_units: The number of units in the `RNNCell`. - Returns: - An initialized `RNNCell`. - Raises: - ValueError: `cell_type` is an invalid `RNNCell` name. - TypeError: `cell_type` is not a string or a subclass of `RNNCell`. - """ - cell_type = _CELL_TYPES.get(cell_type, cell_type) - if not cell_type or not issubclass(cell_type, contrib_rnn.RNNCell): - raise ValueError('The supported cell types are {}; got {}'.format( - list(_CELL_TYPES.keys()), cell_type)) - return cell_type(num_units=num_units) - - -def construct_rnn_cell(num_units, cell_type='basic_rnn', - dropout_keep_probabilities=None): - """Constructs cells, applies dropout and assembles a `MultiRNNCell`. - - The cell type chosen by DynamicRNNEstimator.__init__() is the same as - returned by this function when called with the same arguments. - - Args: - num_units: A single `int` or a list/tuple of `int`s. The size of the - `RNNCell`s. - cell_type: A string identifying the `RNNCell` type or a subclass of - `RNNCell`. - dropout_keep_probabilities: a list of dropout probabilities or `None`. If a - list is given, it must have length `len(cell_type) + 1`. - - Returns: - An initialized `RNNCell`. - """ - if not isinstance(num_units, (list, tuple)): - num_units = (num_units,) - - cells = [_get_single_cell(cell_type, n) for n in num_units] - if dropout_keep_probabilities: - cells = apply_dropout(cells, dropout_keep_probabilities) - if len(cells) == 1: - return cells[0] - return contrib_rnn.MultiRNNCell(cells) - - -def apply_dropout(cells, dropout_keep_probabilities, random_seed=None): - """Applies dropout to the outputs and inputs of `cell`. - - Args: - cells: A list of `RNNCell`s. - dropout_keep_probabilities: a list whose elements are either floats in - `[0.0, 1.0]` or `None`. It must have length one greater than `cells`. - random_seed: Seed for random dropout. - - Returns: - A list of `RNNCell`s, the result of applying the supplied dropouts. - - Raises: - ValueError: If `len(dropout_keep_probabilities) != len(cells) + 1`. - """ - if len(dropout_keep_probabilities) != len(cells) + 1: - raise ValueError( - 'The number of dropout probabilities must be one greater than the ' - 'number of cells. Got {} cells and {} dropout probabilities.'.format( - len(cells), len(dropout_keep_probabilities))) - wrapped_cells = [ - contrib_rnn.DropoutWrapper(cell, prob, 1.0, seed=random_seed) - for cell, prob in zip(cells[:-1], dropout_keep_probabilities[:-2]) - ] - wrapped_cells.append( - contrib_rnn.DropoutWrapper(cells[-1], dropout_keep_probabilities[-2], - dropout_keep_probabilities[-1])) - return wrapped_cells - - -def get_eval_metric_ops(problem_type, prediction_type, sequence_length, - prediction_dict, labels): - """Returns eval metric ops for given `problem_type` and `prediction_type`. - - Args: - problem_type: `ProblemType.CLASSIFICATION` or - `ProblemType.LINEAR_REGRESSION`. - prediction_type: `PredictionType.SINGLE_VALUE` or - `PredictionType.MULTIPLE_VALUE`. - sequence_length: A `Tensor` with shape `[batch_size]` and dtype `int32` - containing the length of each sequence in the batch. If `None`, sequences - are assumed to be unpadded. - prediction_dict: A dict of prediction tensors. - labels: The label `Tensor`. - - Returns: - A `dict` mapping strings to the result of calling the metric_fn. - """ - eval_metric_ops = {} - if problem_type == constants.ProblemType.CLASSIFICATION: - # Multi value classification - if prediction_type == PredictionType.MULTIPLE_VALUE: - mask_predictions, mask_labels = mask_activations_and_labels( - prediction_dict[prediction_key.PredictionKey.CLASSES], labels, - sequence_length) - eval_metric_ops['accuracy'] = metrics.streaming_accuracy( - predictions=mask_predictions, labels=mask_labels) - # Single value classification - elif prediction_type == PredictionType.SINGLE_VALUE: - eval_metric_ops['accuracy'] = metrics.streaming_accuracy( - predictions=prediction_dict[prediction_key.PredictionKey.CLASSES], - labels=labels) - elif problem_type == constants.ProblemType.LINEAR_REGRESSION: - # Multi value regression - if prediction_type == PredictionType.MULTIPLE_VALUE: - pass - # Single value regression - elif prediction_type == PredictionType.SINGLE_VALUE: - pass - return eval_metric_ops - - -def select_last_activations(activations, sequence_lengths): - """Selects the nth set of activations for each n in `sequence_length`. - - Returns a `Tensor` of shape `[batch_size, k]`. If `sequence_length` is not - `None`, then `output[i, :] = activations[i, sequence_length[i] - 1, :]`. If - `sequence_length` is `None`, then `output[i, :] = activations[i, -1, :]`. - - Args: - activations: A `Tensor` with shape `[batch_size, padded_length, k]`. - sequence_lengths: A `Tensor` with shape `[batch_size]` or `None`. - Returns: - A `Tensor` of shape `[batch_size, k]`. - """ - with ops.name_scope( - 'select_last_activations', values=[activations, sequence_lengths]): - activations_shape = array_ops.shape(activations) - batch_size = activations_shape[0] - padded_length = activations_shape[1] - num_label_columns = activations_shape[2] - if sequence_lengths is None: - sequence_lengths = padded_length - reshaped_activations = array_ops.reshape(activations, - [-1, num_label_columns]) - indices = math_ops.range(batch_size) * padded_length + sequence_lengths - 1 - last_activations = array_ops.gather(reshaped_activations, indices) - last_activations.set_shape( - [activations.get_shape()[0], activations.get_shape()[2]]) - return last_activations - - -def mask_activations_and_labels(activations, labels, sequence_lengths): - """Remove entries outside `sequence_lengths` and returned flattened results. - - Args: - activations: Output of the RNN, shape `[batch_size, padded_length, k]`. - labels: Label values, shape `[batch_size, padded_length]`. - sequence_lengths: A `Tensor` of shape `[batch_size]` with the unpadded - length of each sequence. If `None`, then each sequence is unpadded. - - Returns: - activations_masked: `logit` values with those beyond `sequence_lengths` - removed for each batch. Batches are then concatenated. Shape - `[tf.sum(sequence_lengths), k]` if `sequence_lengths` is not `None` and - shape `[batch_size * padded_length, k]` otherwise. - labels_masked: Label values after removing unneeded entries. Shape - `[tf.sum(sequence_lengths)]` if `sequence_lengths` is not `None` and shape - `[batch_size * padded_length]` otherwise. - """ - with ops.name_scope( - 'mask_activations_and_labels', - values=[activations, labels, sequence_lengths]): - labels_shape = array_ops.shape(labels) - batch_size = labels_shape[0] - padded_length = labels_shape[1] - if sequence_lengths is None: - flattened_dimension = padded_length * batch_size - activations_masked = array_ops.reshape(activations, - [flattened_dimension, -1]) - labels_masked = array_ops.reshape(labels, [flattened_dimension]) - else: - mask = array_ops.sequence_mask(sequence_lengths, padded_length) - activations_masked = array_ops.boolean_mask(activations, mask) - labels_masked = array_ops.boolean_mask(labels, mask) - return activations_masked, labels_masked - - -def multi_value_predictions(activations, target_column, problem_type, - predict_probabilities): - """Maps `activations` from the RNN to predictions for multi value models. - - If `predict_probabilities` is `False`, this function returns a `dict` - containing single entry with key `prediction_key.PredictionKey.CLASSES` for - `problem_type` `ProblemType.CLASSIFICATION` or - `prediction_key.PredictionKey.SCORE` for `problem_type` - `ProblemType.LINEAR_REGRESSION`. - - If `predict_probabilities` is `True`, it will contain a second entry with key - `prediction_key.PredictionKey.PROBABILITIES`. The - value of this entry is a `Tensor` of probabilities with shape - `[batch_size, padded_length, num_classes]`. - - Note that variable length inputs will yield some predictions that don't have - meaning. For example, if `sequence_length = [3, 2]`, then prediction `[1, 2]` - has no meaningful interpretation. - - Args: - activations: Output from an RNN. Should have dtype `float32` and shape - `[batch_size, padded_length, ?]`. - target_column: An initialized `TargetColumn`, calculate predictions. - problem_type: Either `ProblemType.CLASSIFICATION` or - `ProblemType.LINEAR_REGRESSION`. - predict_probabilities: A Python boolean, indicating whether probabilities - should be returned. Should only be set to `True` for - classification/logistic regression problems. - Returns: - A `dict` mapping strings to `Tensors`. - """ - with ops.name_scope('MultiValuePrediction'): - activations_shape = array_ops.shape(activations) - flattened_activations = array_ops.reshape(activations, - [-1, activations_shape[2]]) - prediction_dict = {} - if predict_probabilities: - flat_probabilities = target_column.logits_to_predictions( - flattened_activations, proba=True) - flat_predictions = math_ops.argmax(flat_probabilities, 1) - if target_column.num_label_columns == 1: - probability_shape = array_ops.concat([activations_shape[:2], [2]], 0) - else: - probability_shape = activations_shape - probabilities = array_ops.reshape( - flat_probabilities, - probability_shape, - name=prediction_key.PredictionKey.PROBABILITIES) - prediction_dict[ - prediction_key.PredictionKey.PROBABILITIES] = probabilities - else: - flat_predictions = target_column.logits_to_predictions( - flattened_activations, proba=False) - predictions_name = (prediction_key.PredictionKey.CLASSES - if problem_type == constants.ProblemType.CLASSIFICATION - else prediction_key.PredictionKey.SCORES) - predictions = array_ops.reshape( - flat_predictions, [activations_shape[0], activations_shape[1]], - name=predictions_name) - prediction_dict[predictions_name] = predictions - return prediction_dict diff --git a/tensorflow/contrib/learn/python/learn/estimators/rnn_common_test.py b/tensorflow/contrib/learn/python/learn/estimators/rnn_common_test.py deleted file mode 100644 index ebf5f5617d7..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/rnn_common_test.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for layers.rnn_common.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.learn.python.learn.estimators import rnn_common -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.platform import test - - -class RnnCommonTest(test.TestCase): - - def testMaskActivationsAndLabels(self): - """Test `mask_activations_and_labels`.""" - batch_size = 4 - padded_length = 6 - num_classes = 4 - np.random.seed(1234) - sequence_length = np.random.randint(0, padded_length + 1, batch_size) - activations = np.random.rand(batch_size, padded_length, num_classes) - labels = np.random.randint(0, num_classes, [batch_size, padded_length]) - (activations_masked_t, - labels_masked_t) = rnn_common.mask_activations_and_labels( - constant_op.constant(activations, dtype=dtypes.float32), - constant_op.constant(labels, dtype=dtypes.int32), - constant_op.constant(sequence_length, dtype=dtypes.int32)) - - with self.cached_session() as sess: - activations_masked, labels_masked = sess.run( - [activations_masked_t, labels_masked_t]) - - expected_activations_shape = [sum(sequence_length), num_classes] - np.testing.assert_equal( - expected_activations_shape, activations_masked.shape, - 'Wrong activations shape. Expected {}; got {}.'.format( - expected_activations_shape, activations_masked.shape)) - - expected_labels_shape = [sum(sequence_length)] - np.testing.assert_equal(expected_labels_shape, labels_masked.shape, - 'Wrong labels shape. Expected {}; got {}.'.format( - expected_labels_shape, labels_masked.shape)) - masked_index = 0 - for i in range(batch_size): - for j in range(sequence_length[i]): - actual_activations = activations_masked[masked_index] - expected_activations = activations[i, j, :] - np.testing.assert_almost_equal( - expected_activations, - actual_activations, - err_msg='Unexpected logit value at index [{}, {}, :].' - ' Expected {}; got {}.'.format(i, j, expected_activations, - actual_activations)) - - actual_labels = labels_masked[masked_index] - expected_labels = labels[i, j] - np.testing.assert_almost_equal( - expected_labels, - actual_labels, - err_msg='Unexpected logit value at index [{}, {}].' - ' Expected {}; got {}.'.format(i, j, expected_labels, - actual_labels)) - masked_index += 1 - - def testSelectLastActivations(self): - """Test `select_last_activations`.""" - batch_size = 4 - padded_length = 6 - num_classes = 4 - np.random.seed(4444) - sequence_length = np.random.randint(0, padded_length + 1, batch_size) - activations = np.random.rand(batch_size, padded_length, num_classes) - last_activations_t = rnn_common.select_last_activations( - constant_op.constant(activations, dtype=dtypes.float32), - constant_op.constant(sequence_length, dtype=dtypes.int32)) - - with session.Session() as sess: - last_activations = sess.run(last_activations_t) - - expected_activations_shape = [batch_size, num_classes] - np.testing.assert_equal( - expected_activations_shape, last_activations.shape, - 'Wrong activations shape. Expected {}; got {}.'.format( - expected_activations_shape, last_activations.shape)) - - for i in range(batch_size): - actual_activations = last_activations[i, :] - expected_activations = activations[i, sequence_length[i] - 1, :] - np.testing.assert_almost_equal( - expected_activations, - actual_activations, - err_msg='Unexpected logit value at index [{}, :].' - ' Expected {}; got {}.'.format(i, expected_activations, - actual_activations)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/run_config.py b/tensorflow/contrib/learn/python/learn/estimators/run_config.py deleted file mode 100644 index e435fd65702..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/run_config.py +++ /dev/null @@ -1,495 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Run Config (deprecated, use tf.estimator.RunConfig instead). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import json -import os - -import six - -from tensorflow.contrib.framework.python.framework import experimental -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.estimator import run_config as core_run_config -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import server_lib -from tensorflow.python.util.deprecation import deprecated - - -# A list of the property names in RunConfig user allows to change. They will -# not affect the execution framework, so when execution framework checks the -# `uid` of the RunConfig, it should be ignored. -_DEFAULT_UID_WHITE_LIST = [ - 'tf_random_seed', - 'save_summary_steps', - 'save_checkpoints_steps', - 'save_checkpoints_secs', - 'session_config', - 'keep_checkpoint_max', - 'keep_checkpoint_every_n_hours', - 'log_step_count_steps', -] - - -class Environment(object): - """DEPRECATED CLASS.""" - # For running general distributed training. - CLOUD = 'cloud' - # For running Google-internal distributed training. - GOOGLE = 'google' - # For running on local desktop. - LOCAL = 'local' - - -class TaskType(object): - """DEPRECATED CLASS.""" - MASTER = 'master' - PS = 'ps' - WORKER = 'worker' - - -class ClusterConfig(object): - """This class specifies the configurations for a distributed run. - - THIS CLASS IS DEPRECATED. Use tf.estimator.RunConfig instead. - - If you're using an `Estimator`, you should probably use the subclass - RunConfig instead. - """ - - def __init__(self, master=None, evaluation_master=None): - """Constructor. - - Sets the properties `cluster_spec`, `is_chief`, `master` (if `None` in the - args), `num_ps_replicas`, `task_id`, and `task_type` based on the - `TF_CONFIG` environment variable, if the pertinent information is - present. The `TF_CONFIG` environment variable is a JSON object with - attributes: `cluster`, `environment`, and `task`. - - `cluster` is a JSON serialized version of `ClusterSpec`'s Python dict from - `server_lib.py`, mapping task types (usually one of the TaskType enums) to a - list of task addresses. - - `environment` specifies the runtime environment for the job (usually one of - the `Environment` enums). Defaults to `LOCAL`. - - `task` has two attributes: `type` and `index`, where `type` can be any of - the task types in `cluster`. When `TF_CONFIG` contains said information, the - following properties are set on this class: - - * `task_type` is set to `TF_CONFIG['task']['type']`. Defaults to `None`. - * `task_id` is set to `TF_CONFIG['task']['index']`. Defaults to 0. - * `cluster_spec` is parsed from `TF_CONFIG['cluster']`. Defaults to {}. - * `master` is determined by looking up `task_type` and `task_id` in the - `cluster_spec`. Defaults to ''. - * `num_ps_replicas` is set by counting the number of nodes listed - in the `ps` attribute of `cluster_spec`. Defaults to 0. - * `num_worker_replicas` is set by counting the number of nodes listed - in the `worker` attribute of `cluster_spec`. Defaults to 0. - * `is_chief` is deteremined based on `task_type`, `type_id`, and - `environment`. - - Example: - ``` - cluster = {'ps': ['host1:2222', 'host2:2222'], - 'worker': ['host3:2222', 'host4:2222', 'host5:2222']} - os.environ['TF_CONFIG'] = json.dumps( - {'cluster': cluster, - 'task': {'type': 'worker', 'index': 1}}) - config = ClusterConfig() - assert config.master == 'host4:2222' - assert config.task_id == 1 - assert config.num_ps_replicas == 2 - assert config.num_worker_replicas == 3 - assert config.cluster_spec == server_lib.ClusterSpec(cluster) - assert config.task_type == 'worker' - assert not config.is_chief - ``` - - Args: - master: TensorFlow master. Defaults to empty string for local. - evaluation_master: The master on which to perform evaluation. - """ - # If not explicitly specified in the constructor and the TF_CONFIG - # environment variable is present, load cluster_spec from TF_CONFIG. - config = json.loads(os.environ.get('TF_CONFIG') or '{}') - - # Set task_type and task_id if the TF_CONFIG environment variable is - # present. Otherwise, use the respective default (None / 0). - task_env = config.get('task', {}) - self._task_type = task_env.get('type', None) - self._task_id = self.get_task_id() - - self._cluster_spec = server_lib.ClusterSpec(config.get('cluster', {})) - self._master = (master if master is not None else - _get_master(self._cluster_spec, self._task_type, - self._task_id) or '') - self._num_ps_replicas = _count_ps(self._cluster_spec) or 0 - self._num_worker_replicas = _count_worker(self._cluster_spec) or 0 - - # Set is_chief. - self._environment = config.get('environment', Environment.LOCAL) - self._is_chief = None - if self._task_type is None: - self._is_chief = (self._task_id == 0) - elif self._environment == Environment.CLOUD: - # When the TF_CONFIG environment variable is set, we can set the - # default of is_chief to 0 when task_type is "master" and task_id is 0. - self._is_chief = (self._task_type == TaskType.MASTER and - self._task_id == 0) - else: - # Legacy behavior is that is_chief is None if task_id == 0. - self._is_chief = (self._task_type == TaskType.WORKER and - self._task_id == 0) - - self._evaluation_master = evaluation_master or '' - - @property - def cluster_spec(self): - return self._cluster_spec - - @property - def environment(self): - return self._environment - - @property - def evaluation_master(self): - return self._evaluation_master - - @property - def is_chief(self): - return self._is_chief - - @property - def master(self): - return self._master - - @property - def num_ps_replicas(self): - return self._num_ps_replicas - - @property - def num_worker_replicas(self): - return self._num_worker_replicas - - @property - def task_id(self): - return self._task_id - - @property - def task_type(self): - return self._task_type - - @staticmethod - def get_task_id(): - """Returns task index from `TF_CONFIG` environmental variable. - - If you have a ClusterConfig instance, you can just access its task_id - property instead of calling this function and re-parsing the environmental - variable. - - Returns: - `TF_CONFIG['task']['index']`. Defaults to 0. - """ - config = json.loads(os.environ.get('TF_CONFIG') or '{}') - task_env = config.get('task', {}) - task_index = task_env.get('index') - return int(task_index) if task_index else 0 - - -class RunConfig(ClusterConfig, core_run_config.RunConfig): - """This class specifies the configurations for an `Estimator` run. - - This class is a deprecated implementation of `tf.estimator.RunConfig` - interface. - """ - _USE_DEFAULT = 0 - - @deprecated(None, 'When switching to tf.estimator.Estimator, use' - ' tf.estimator.RunConfig instead.') - def __init__(self, - master=None, - num_cores=0, - log_device_placement=False, - gpu_memory_fraction=1, - tf_random_seed=None, - save_summary_steps=100, - save_checkpoints_secs=_USE_DEFAULT, - save_checkpoints_steps=None, - keep_checkpoint_max=5, - keep_checkpoint_every_n_hours=10000, - log_step_count_steps=100, - protocol=None, - evaluation_master='', - model_dir=None, - session_config=None, - session_creation_timeout_secs=7200): - """Constructor. - - The superclass `ClusterConfig` may set properties like `cluster_spec`, - `is_chief`, `master` (if `None` in the args), `num_ps_replicas`, `task_id`, - and `task_type` based on the `TF_CONFIG` environment variable. See - `ClusterConfig` for more details. - - N.B.: If `save_checkpoints_steps` or `save_checkpoints_secs` is set, - `keep_checkpoint_max` might need to be adjusted accordingly, especially in - distributed training. For example, setting `save_checkpoints_secs` as 60 - without adjusting `keep_checkpoint_max` (defaults to 5) leads to situation - that checkpoint would be garbage collected after 5 minutes. In distributed - training, the evaluation job starts asynchronously and might fail to load or - find the checkpoint due to race condition. - - Args: - master: TensorFlow master. Defaults to empty string for local. - num_cores: Number of cores to be used. If 0, the system picks an - appropriate number (default: 0). - log_device_placement: Log the op placement to devices (default: False). - gpu_memory_fraction: Fraction of GPU memory used by the process on - each GPU uniformly on the same machine. - tf_random_seed: Random seed for TensorFlow initializers. - Setting this value allows consistency between reruns. - save_summary_steps: Save summaries every this many steps. - save_checkpoints_secs: Save checkpoints every this many seconds. Can not - be specified with `save_checkpoints_steps`. - save_checkpoints_steps: Save checkpoints every this many steps. Can not be - specified with `save_checkpoints_secs`. - keep_checkpoint_max: The maximum number of recent checkpoint files to - keep. As new files are created, older files are deleted. If None or 0, - all checkpoint files are kept. Defaults to 5 (that is, the 5 most recent - checkpoint files are kept.) - keep_checkpoint_every_n_hours: Number of hours between each checkpoint - to be saved. The default value of 10,000 hours effectively disables - the feature. - log_step_count_steps: The frequency, in number of global steps, that the - global step/sec will be logged during training. - protocol: An optional argument which specifies the protocol used when - starting server. None means default to grpc. - evaluation_master: the master on which to perform evaluation. - model_dir: directory where model parameters, graph etc are saved. If - `None`, will use `model_dir` property in `TF_CONFIG` environment - variable. If both are set, must have same value. If both are `None`, see - `Estimator` about where the model will be saved. - session_config: a ConfigProto used to set session parameters, or None. - Note - using this argument, it is easy to provide settings which break - otherwise perfectly good models. Use with care. - session_creation_timeout_secs: Max time workers should wait for a session - to become available (on initialization or when recovering a session) - with MonitoredTrainingSession. Defaults to 7200 seconds, but users may - want to set a lower value to detect problems with variable / session - (re)-initialization more quickly. - """ - # Neither parent class calls super().__init__(), so here we have to - # manually call their __init__() methods. - ClusterConfig.__init__( - self, master=master, evaluation_master=evaluation_master) - # For too long this code didn't call: - # core_run_config.RunConfig.__init__(self) - # so instead of breaking compatibility with that assumption, we - # just manually initialize this field: - self._train_distribute = None - self._eval_distribute = None - self._experimental_max_worker_delay_secs = None - self._device_fn = None - - gpu_options = config_pb2.GPUOptions( - per_process_gpu_memory_fraction=gpu_memory_fraction) - self._tf_config = config_pb2.ConfigProto( - log_device_placement=log_device_placement, - inter_op_parallelism_threads=num_cores, - intra_op_parallelism_threads=num_cores, - gpu_options=gpu_options) - - self._tf_random_seed = tf_random_seed - self._save_summary_steps = save_summary_steps - self._save_checkpoints_secs = save_checkpoints_secs - self._log_step_count_steps = log_step_count_steps - self._protocol = protocol - self._session_config = session_config - if save_checkpoints_secs == RunConfig._USE_DEFAULT: - if save_checkpoints_steps is None: - self._save_checkpoints_secs = 600 - else: - self._save_checkpoints_secs = None - self._save_checkpoints_steps = save_checkpoints_steps - - # TODO(weiho): Remove these after ModelFn refactoring, when users can - # create Scaffold and Saver in their model_fn to set these. - self._keep_checkpoint_max = keep_checkpoint_max - self._keep_checkpoint_every_n_hours = keep_checkpoint_every_n_hours - self._model_dir = _get_model_dir(model_dir) - self._session_creation_timeout_secs = session_creation_timeout_secs - - @experimental - def uid(self, whitelist=None): - """Generates a 'Unique Identifier' based on all internal fields. - - Caller should use the uid string to check `RunConfig` instance integrity - in one session use, but should not rely on the implementation details, which - is subject to change. - - Args: - whitelist: A list of the string names of the properties uid should not - include. If `None`, defaults to `_DEFAULT_UID_WHITE_LIST`, which - includes most properties user allowes to change. - - Returns: - A uid string. - """ - if whitelist is None: - whitelist = _DEFAULT_UID_WHITE_LIST - - state = {k: v for k, v in self.__dict__.items() if not k.startswith('__')} - # Pop out the keys in whitelist. - for k in whitelist: - state.pop('_' + k, None) - - ordered_state = collections.OrderedDict( - sorted(state.items(), key=lambda t: t[0])) - # For class instance without __repr__, some special cares are required. - # Otherwise, the object address will be used. - if '_cluster_spec' in ordered_state: - ordered_state['_cluster_spec'] = collections.OrderedDict( - sorted(ordered_state['_cluster_spec'].as_dict().items(), - key=lambda t: t[0])) - return ', '.join( - '%s=%r' % (k, v) for (k, v) in six.iteritems(ordered_state)) - - @property - def model_dir(self): - return self._model_dir - - @property - def tf_config(self): - return self._tf_config - - @property - def tf_random_seed(self): - return self._tf_random_seed - - @property - def save_summary_steps(self): - return self._save_summary_steps - - @property - def save_checkpoints_secs(self): - return self._save_checkpoints_secs - - @property - def save_checkpoints_steps(self): - return self._save_checkpoints_steps - - @property - def session_config(self): - return self._session_config - - @property - def keep_checkpoint_max(self): - return self._keep_checkpoint_max - - @property - def keep_checkpoint_every_n_hours(self): - return self._keep_checkpoint_every_n_hours - - @property - def log_step_count_steps(self): - return self._log_step_count_steps - - @property - def session_creation_timeout_secs(self): - return self._session_creation_timeout_secs - - -def _count_ps(cluster_spec): - """Counts the number of parameter servers in cluster_spec.""" - return len(cluster_spec.as_dict().get('ps', [])) if cluster_spec else 0 - - -def _count_worker(cluster_spec): - """Counts the number of workers in cluster_spec. - - Workers with TaskType.WORKER and TaskType.MASTER are included in the return - value. - - Args: - cluster_spec: a ClusterSpec instance that describes current deployment. - - Returns: - The total number of eligible workers. - - If 'cluster_spec' was None, then 0 is returned. - """ - return (len(cluster_spec.as_dict().get('worker', [])) + - len(cluster_spec.as_dict().get('master', []))) if cluster_spec else 0 - - -def _get_master(cluster_spec, task_type, task_id): - """Returns the appropriate string for the TensorFlow master.""" - if not cluster_spec: - return '' - - # If there is only one node in the cluster, do things locally. - jobs = cluster_spec.jobs - if len(jobs) == 1 and len(cluster_spec.job_tasks(jobs[0])) == 1: - return '' - - # Lookup the master in cluster_spec using task_type and task_id, - # if possible. - if task_type: - if task_type not in jobs: - raise ValueError( - '%s is not a valid task_type in the cluster_spec:\n' - '%s\n\n' - 'Note that these values may be coming from the TF_CONFIG environment ' - 'variable.' % (task_type, cluster_spec)) - addresses = cluster_spec.job_tasks(task_type) - if task_id >= len(addresses) or task_id < 0: - raise ValueError( - '%d is not a valid task_id for task_type %s in the ' - 'cluster_spec:\n' - '%s\n\n' - 'Note that these value may be coming from the TF_CONFIG environment ' - 'variable.' % (task_id, task_type, cluster_spec)) - return 'grpc://' + addresses[task_id] - - # For backwards compatibility, we return empty string if task_type was - # not set (task_type did not previously exist). - return '' - - -def _get_model_dir(model_dir): - """Returns `model_dir` based user provided `model_dir` or `TF_CONFIG`.""" - - model_dir_in_tf_config = json.loads( - os.environ.get('TF_CONFIG') or '{}').get('model_dir', None) - if model_dir_in_tf_config is not None: - if model_dir is not None and model_dir_in_tf_config != model_dir: - raise ValueError( - '`model_dir` provided in RunConfig construct, if set, ' - 'must have the same value as the model_dir in TF_CONFIG. ' - 'model_dir: {}\nTF_CONFIG["model_dir"]: {}.\n'.format( - model_dir, model_dir_in_tf_config)) - - logging.info('Using model_dir in TF_CONFIG: %s', model_dir_in_tf_config) - - return model_dir or model_dir_in_tf_config diff --git a/tensorflow/contrib/learn/python/learn/estimators/run_config_test.py b/tensorflow/contrib/learn/python/learn/estimators/run_config_test.py deleted file mode 100644 index e2cea9652ae..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/run_config_test.py +++ /dev/null @@ -1,398 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""run_config.py tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import copy -import json - -from tensorflow.contrib.learn.python.learn.estimators import run_config as run_config_lib -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.estimator import run_config as core_run_config -from tensorflow.python.platform import test -from tensorflow.python.training import server_lib - -TEST_DIR = "test_dir" -ANOTHER_TEST_DIR = "another_test_dir" -MASTER = "master_" -RANDOM_SEED = 123 - -patch = test.mock.patch - - -def _create_run_config_with_cluster_spec(tf_config_str): - with patch.dict("os.environ", {"TF_CONFIG": tf_config_str}): - return run_config_lib.RunConfig( - tf_random_seed=RANDOM_SEED, model_dir=TEST_DIR) - - -class RunConfigTest(test.TestCase): - - def test_instance_of_core_run_config(self): - config = run_config_lib.RunConfig() - self.assertTrue(isinstance(config, core_run_config.RunConfig)) - - def test_defaults_with_no_tf_config(self): - config = run_config_lib.RunConfig() - self.assertEqual(config.master, "") - self.assertEqual(config.task_id, 0) - self.assertEqual(config.num_ps_replicas, 0) - self.assertEqual(config.cluster_spec, {}) - self.assertIsNone(config.task_type) - self.assertTrue(config.is_chief) - self.assertEqual(config.evaluation_master, "") - - def test_values_from_tf_config(self): - tf_config = { - "cluster": { - run_config_lib.TaskType.PS: ["host1:1", "host2:2"], - run_config_lib.TaskType.WORKER: ["host3:3", "host4:4", "host5:5"] - }, - "task": { - "type": run_config_lib.TaskType.WORKER, - "index": 1 - } - } - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - config = run_config_lib.RunConfig() - - self.assertEqual(config.master, "grpc://host4:4") - self.assertEqual(config.task_id, 1) - self.assertEqual(config.num_ps_replicas, 2) - self.assertEqual(config.num_worker_replicas, 3) - self.assertEqual(config.cluster_spec.as_dict(), tf_config["cluster"]) - self.assertEqual(config.task_type, run_config_lib.TaskType.WORKER) - self.assertFalse(config.is_chief) - self.assertEqual(config.evaluation_master, "") - - def test_explicitly_specified_values(self): - cluster_spec = { - run_config_lib.TaskType.PS: ["localhost:9990"], - "my_job_name": ["localhost:9991", "localhost:9992", "localhost:0"] - } - tf_config = { - "cluster": cluster_spec, - "task": { - "type": run_config_lib.TaskType.WORKER, - "index": 2 - } - } - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - config = run_config_lib.RunConfig( - master="localhost:0", evaluation_master="localhost:9991") - - self.assertEqual(config.master, "localhost:0") - self.assertEqual(config.task_id, 2) - self.assertEqual(config.num_ps_replicas, 1) - self.assertEqual(config.num_worker_replicas, 0) - self.assertEqual(config.cluster_spec, server_lib.ClusterSpec(cluster_spec)) - self.assertEqual(config.task_type, run_config_lib.TaskType.WORKER) - self.assertFalse(config.is_chief) - self.assertEqual(config.evaluation_master, "localhost:9991") - - def test_single_node_in_cluster_spec_produces_empty_master(self): - tf_config = {"cluster": {run_config_lib.TaskType.WORKER: ["host1:1"]}} - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - config = run_config_lib.RunConfig() - self.assertEqual(config.master, "") - - def test_no_task_type_produces_empty_master(self): - tf_config = { - "cluster": { - run_config_lib.TaskType.PS: ["host1:1", "host2:2"], - run_config_lib.TaskType.WORKER: ["host3:3", "host4:4", "host5:5"] - }, - # Omits "task": {"type": "worker} - } - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - config = run_config_lib.RunConfig() - self.assertEqual(config.master, "") - - def test_invalid_job_name_raises(self): - tf_config = { - "cluster": { - run_config_lib.TaskType.PS: ["host1:1", "host2:2"], - run_config_lib.TaskType.WORKER: ["host3:3", "host4:4", "host5:5"] - }, - "task": { - "type": "not_in_cluster_spec" - } - } - expected_msg_regexp = "not_in_cluster_spec is not a valid task" - with patch.dict( - "os.environ", - {"TF_CONFIG": json.dumps(tf_config)}), self.assertRaisesRegexp( - ValueError, expected_msg_regexp): - run_config_lib.RunConfig() - - def test_illegal_task_index_raises(self): - tf_config = { - "cluster": { - run_config_lib.TaskType.PS: ["host1:1", "host2:2"], - run_config_lib.TaskType.WORKER: ["host3:3", "host4:4", "host5:5"] - }, - "task": { - "type": run_config_lib.TaskType.WORKER, - "index": 3 - } - } - expected_msg_regexp = "3 is not a valid task_id" - with patch.dict( - "os.environ", - {"TF_CONFIG": json.dumps(tf_config)}), self.assertRaisesRegexp( - ValueError, expected_msg_regexp): - run_config_lib.RunConfig() - - def test_is_chief_from_cloud_tf_config(self): - # is_chief should be true when ["task"]["type"] == "master" and - # index == 0 and ["task"]["environment"] == "cloud". Note that - # test_values_from_tf_config covers the non-master case. - tf_config = { - "cluster": { - run_config_lib.TaskType.PS: ["host1:1", "host2:2"], - run_config_lib.TaskType.MASTER: ["host3:3"], - run_config_lib.TaskType.WORKER: ["host4:4", "host5:5", "host6:6"] - }, - "task": { - "type": run_config_lib.TaskType.MASTER, - "index": 0 - }, - "environment": "cloud" - } - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - config = run_config_lib.RunConfig() - - self.assertTrue(config.is_chief) - - def test_is_chief_from_noncloud_tf_config(self): - # is_chief should be true when ["task"]["type"] == "worker" and - # index == 0 if ["task"]["environment"] != "cloud". - tf_config = { - "cluster": { - run_config_lib.TaskType.PS: ["host1:1", "host2:2"], - run_config_lib.TaskType.MASTER: ["host3:3"], - run_config_lib.TaskType.WORKER: ["host4:4", "host5:5", "host6:6"] - }, - "task": { - "type": run_config_lib.TaskType.WORKER, - "index": 0 - }, - "environment": "random" - } - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - config = run_config_lib.RunConfig() - - self.assertTrue(config.is_chief) - - # But task 0 for a job named "master" should not be. - tf_config = { - "cluster": { - run_config_lib.TaskType.PS: ["host1:1", "host2:2"], - run_config_lib.TaskType.MASTER: ["host3:3"], - run_config_lib.TaskType.WORKER: ["host4:4", "host5:5", "host6:6"] - }, - "task": { - "type": run_config_lib.TaskType.MASTER, - "index": 0 - }, - "environment": "random" - } - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - config = run_config_lib.RunConfig() - - self.assertFalse(config.is_chief) - - def test_default_is_chief_from_tf_config_without_job_name(self): - tf_config = {"cluster": {}, "task": {}} - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - config = run_config_lib.RunConfig() - - self.assertTrue(config.is_chief) - - def test_model_dir(self): - empty_config = run_config_lib.RunConfig() - self.assertIsNone(empty_config.model_dir) - - config = run_config_lib.RunConfig(model_dir=TEST_DIR) - self.assertEqual(TEST_DIR, config.model_dir) - - def test_model_dir_in_tf_config(self): - tf_config = {"model_dir": TEST_DIR} - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - run_config = run_config_lib.RunConfig() - self.assertEqual(TEST_DIR, run_config.model_dir) - - def test_model_dir_both_in_tf_config_and_constructor(self): - tf_config = {"model_dir": TEST_DIR} - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - run_config = run_config_lib.RunConfig(model_dir=TEST_DIR) - self.assertEqual(TEST_DIR, run_config.model_dir) - - def test_model_dir_fail_if_constructor_value_mismatch_tf_config(self): - tf_config = {"model_dir": TEST_DIR} - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - with self.assertRaisesRegexp( - ValueError, - "`model_dir` provided in RunConfig .* must have " - "the same value .* in TF_CONFIG"): - run_config_lib.RunConfig(model_dir=TEST_DIR + "/sub_dir") - - def test_replace(self): - config = run_config_lib.RunConfig( - tf_random_seed=RANDOM_SEED, model_dir=TEST_DIR) - self.assertEqual(TEST_DIR, config.model_dir) - self.assertEqual(RANDOM_SEED, config.tf_random_seed) - - new_config = config.replace(model_dir=ANOTHER_TEST_DIR) - self.assertEqual(ANOTHER_TEST_DIR, new_config.model_dir) - self.assertEqual(RANDOM_SEED, new_config.tf_random_seed) - self.assertEqual(RANDOM_SEED, config.tf_random_seed) - - def test_uid_for_different_configs(self): - config = run_config_lib.RunConfig( - tf_random_seed=RANDOM_SEED, model_dir=TEST_DIR) - - expected_uid = config.uid() - # Check for 10 times, which should prove something. - for _ in range(10): - self.assertEqual(expected_uid, config.uid()) - - new_config = config.replace(model_dir=ANOTHER_TEST_DIR) - self.assertEqual(TEST_DIR, config.model_dir) - self.assertNotEqual(expected_uid, new_config.uid()) - self.assertEqual(ANOTHER_TEST_DIR, new_config.model_dir) - - def test_uid_for_whitelist(self): - whitelist = ["model_dir"] - config = run_config_lib.RunConfig( - tf_random_seed=RANDOM_SEED, model_dir=TEST_DIR) - - expected_uid = config.uid(whitelist) - self.assertEqual(expected_uid, config.uid(whitelist)) - - new_config = config.replace(model_dir=ANOTHER_TEST_DIR) - self.assertEqual(TEST_DIR, config.model_dir) - self.assertEqual(expected_uid, new_config.uid(whitelist)) - self.assertEqual(ANOTHER_TEST_DIR, new_config.model_dir) - - def test_uid_for_default_whitelist(self): - config = run_config_lib.RunConfig( - tf_random_seed=11, - save_summary_steps=12, - save_checkpoints_steps=13, - save_checkpoints_secs=14, - session_config=config_pb2.ConfigProto(allow_soft_placement=True), - keep_checkpoint_max=16, - keep_checkpoint_every_n_hours=17) - self.assertEqual(11, config.tf_random_seed) - self.assertEqual(12, config.save_summary_steps) - self.assertEqual(13, config.save_checkpoints_steps) - self.assertEqual(14, config.save_checkpoints_secs) - self.assertEqual(config_pb2.ConfigProto(allow_soft_placement=True), - config.session_config) - self.assertEqual(16, config.keep_checkpoint_max) - self.assertEqual(17, config.keep_checkpoint_every_n_hours) - - new_config = run_config_lib.RunConfig( - tf_random_seed=21, - save_summary_steps=22, - save_checkpoints_steps=23, - save_checkpoints_secs=24, - session_config=config_pb2.ConfigProto(allow_soft_placement=False), - keep_checkpoint_max=26, - keep_checkpoint_every_n_hours=27) - self.assertEqual(config.uid(), new_config.uid()) - # model_dir is not on the default whitelist. - self.assertNotEqual(config.uid(whitelist=[]), - new_config.uid(whitelist=[])) - new_config = new_config.replace(model_dir=ANOTHER_TEST_DIR) - self.assertNotEqual(config.uid(), new_config.uid()) - - def test_uid_for_deepcopy(self): - tf_config = { - "cluster": { - run_config_lib.TaskType.PS: ["host1:1", "host2:2"], - run_config_lib.TaskType.WORKER: ["host3:3", "host4:4", "host5:5"] - }, - "task": { - "type": run_config_lib.TaskType.WORKER, - "index": 1 - } - } - - config = _create_run_config_with_cluster_spec(json.dumps(tf_config)) - expected_uid = config.uid() - self.assertEqual(tf_config["cluster"], config.cluster_spec.as_dict()) - - new_config = copy.deepcopy(config) - self.assertEqual(tf_config["cluster"], new_config.cluster_spec.as_dict()) - self.assertEqual(expected_uid, new_config.uid()) - - def test_uid_for_different_cluster_spec_order(self): - tf_config_1_str = ( - "{\"cluster\": {\"ps\": [\"host1:1\", \"host2:2\"], " - "\"worker\": [\"host3:3\", \"host4:4\", \"host5:5\"]}}") - - tf_config_2_str = ( - "{\"cluster\": {\"worker\": [\"host3:3\", \"host4:4\", \"host5:5\"]," - "\"ps\": [\"host1:1\", \"host2:2\"]}}") - - # Wraps in a loop to check flakiness. - for _ in range(100): - uid_1 = _create_run_config_with_cluster_spec(tf_config_1_str).uid() - uid_2 = _create_run_config_with_cluster_spec(tf_config_2_str).uid() - self.assertEqual(uid_1, uid_2) - - def test_uid_for_different_cluster_specs(self): - tf_config_1 = { - "cluster": { - run_config_lib.TaskType.PS: ["host1:1", "host2:2"], - run_config_lib.TaskType.WORKER: ["host3:3", "host4:4", "host5:5"] - }, - } - - tf_config_2 = { - "cluster": { - run_config_lib.TaskType.PS: ["host1:1"], - run_config_lib.TaskType.WORKER: ["host3:3", "host4:4", "host5:5"] - }, - } - - uid_1 = _create_run_config_with_cluster_spec(json.dumps(tf_config_1)).uid() - uid_2 = _create_run_config_with_cluster_spec(json.dumps(tf_config_2)).uid() - self.assertNotEqual(uid_1, uid_2) - - def test_num_worker_replicas_counts_in_master_too(self): - tf_config = { - "cluster": { - run_config_lib.TaskType.PS: ["host1:1", "host2:2"], - run_config_lib.TaskType.MASTER: ["host6:6"], - run_config_lib.TaskType.WORKER: ["host3:3", "host4:4", "host5:5"], - }, - "task": { - "type": run_config_lib.TaskType.WORKER, - "index": 1 - } - } - - config = _create_run_config_with_cluster_spec(json.dumps(tf_config)) - self.assertEqual(config.num_worker_replicas, 4) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/stability_test.py b/tensorflow/contrib/learn/python/learn/estimators/stability_test.py deleted file mode 100644 index 81376c0e2af..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/stability_test.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Estimator regression tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random - -from tensorflow.contrib.framework.python.ops import variables -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.contrib.learn.python.learn.estimators import dnn -from tensorflow.contrib.learn.python.learn.estimators import linear -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.contrib.learn.python.learn.learn_io import data_feeder -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import test -from tensorflow.python.training import optimizer as optimizer_lib - - -def _get_input_fn(x, y, batch_size=None): - df = data_feeder.setup_train_data_feeder( - x, y, n_classes=None, batch_size=batch_size) - return df.input_builder, df.get_feed_dict_fn() - - -# We use a null optimizer since we can't get deterministic results out of -# supervisor's multiple threads. -class _NullOptimizer(optimizer_lib.Optimizer): - - def __init__(self): - super(_NullOptimizer, self).__init__(use_locking=False, name='Null') - - def _apply_dense(self, grad, var): - return control_flow_ops.no_op() - - def _apply_sparse(self, grad, var): - return control_flow_ops.no_op() - - def _prepare(self): - pass - - -_NULL_OPTIMIZER = _NullOptimizer() - - -class StabilityTest(test.TestCase): - """Tests that estiamtors are reproducible.""" - - def testRandomStability(self): - my_seed = 42 - minval = -0.3333 - maxval = 0.3333 - with ops.Graph().as_default() as g: - with self.session(graph=g) as session: - g.seed = my_seed - x = random_ops.random_uniform([10, 10], minval=minval, maxval=maxval) - val1 = session.run(x) - with ops.Graph().as_default() as g: - with self.session(graph=g) as session: - g.seed = my_seed - x = random_ops.random_uniform([10, 10], minval=minval, maxval=maxval) - val2 = session.run(x) - self.assertAllClose(val1, val2) - - def testLinearRegression(self): - my_seed = 42 - config = run_config.RunConfig(tf_random_seed=my_seed) - boston = base.load_boston() - columns = [feature_column.real_valued_column('', dimension=13)] - - # We train with - - with ops.Graph().as_default() as g1: - random.seed(my_seed) - g1.seed = my_seed - variables.create_global_step() - regressor1 = linear.LinearRegressor( - optimizer=_NULL_OPTIMIZER, feature_columns=columns, config=config) - regressor1.fit(x=boston.data, y=boston.target, steps=1) - - with ops.Graph().as_default() as g2: - random.seed(my_seed) - g2.seed = my_seed - variables.create_global_step() - regressor2 = linear.LinearRegressor( - optimizer=_NULL_OPTIMIZER, feature_columns=columns, config=config) - regressor2.fit(x=boston.data, y=boston.target, steps=1) - - variable_names = regressor1.get_variable_names() - self.assertIn('linear//weight', variable_names) - self.assertIn('linear/bias_weight', variable_names) - regressor1_weights = regressor1.get_variable_value('linear//weight') - regressor2_weights = regressor2.get_variable_value('linear//weight') - regressor1_bias = regressor1.get_variable_value('linear/bias_weight') - regressor2_bias = regressor2.get_variable_value('linear/bias_weight') - self.assertAllClose(regressor1_weights, regressor2_weights) - self.assertAllClose(regressor1_bias, regressor2_bias) - self.assertAllClose( - list(regressor1.predict_scores( - boston.data, as_iterable=True)), - list(regressor2.predict_scores( - boston.data, as_iterable=True)), - atol=1e-05) - - def testDNNRegression(self): - my_seed = 42 - config = run_config.RunConfig(tf_random_seed=my_seed) - boston = base.load_boston() - columns = [feature_column.real_valued_column('', dimension=13)] - - with ops.Graph().as_default() as g1: - random.seed(my_seed) - g1.seed = my_seed - variables.create_global_step() - regressor1 = dnn.DNNRegressor( - hidden_units=[10], - feature_columns=columns, - optimizer=_NULL_OPTIMIZER, - config=config) - regressor1.fit(x=boston.data, y=boston.target, steps=1) - - with ops.Graph().as_default() as g2: - random.seed(my_seed) - g2.seed = my_seed - variables.create_global_step() - regressor2 = dnn.DNNRegressor( - hidden_units=[10], - feature_columns=columns, - optimizer=_NULL_OPTIMIZER, - config=config) - regressor2.fit(x=boston.data, y=boston.target, steps=1) - - weights1 = ([regressor1.get_variable_value('dnn/hiddenlayer_0/weights')] + - [regressor1.get_variable_value('dnn/logits/weights')]) - weights2 = ([regressor2.get_variable_value('dnn/hiddenlayer_0/weights')] + - [regressor2.get_variable_value('dnn/logits/weights')]) - for w1, w2 in zip(weights1, weights2): - self.assertAllClose(w1, w2) - - biases1 = ([regressor1.get_variable_value('dnn/hiddenlayer_0/biases')] + - [regressor1.get_variable_value('dnn/logits/biases')]) - biases2 = ([regressor2.get_variable_value('dnn/hiddenlayer_0/biases')] + - [regressor2.get_variable_value('dnn/logits/biases')]) - for b1, b2 in zip(biases1, biases2): - self.assertAllClose(b1, b2) - self.assertAllClose( - list(regressor1.predict_scores( - boston.data, as_iterable=True)), - list(regressor2.predict_scores( - boston.data, as_iterable=True)), - atol=1e-05) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/state_saving_rnn_estimator.py b/tensorflow/contrib/learn/python/learn/estimators/state_saving_rnn_estimator.py deleted file mode 100644 index de78c72c3ae..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/state_saving_rnn_estimator.py +++ /dev/null @@ -1,664 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Estimator for State Saving RNNs (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib import rnn as rnn_cell -from tensorflow.contrib.layers.python.layers import feature_column_ops -from tensorflow.contrib.layers.python.layers import optimizers -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.estimators import rnn_common -from tensorflow.contrib.training.python.training import sequence_queueing_state_saver as sqss -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import rnn -from tensorflow.python.training import momentum as momentum_opt -from tensorflow.python.util import nest - - -def construct_state_saving_rnn(cell, - inputs, - num_label_columns, - state_saver, - state_name, - scope='rnn'): - """Build a state saving RNN and apply a fully connected layer. - - Args: - cell: An instance of `RNNCell`. - inputs: A length `T` list of inputs, each a `Tensor` of shape - `[batch_size, input_size, ...]`. - num_label_columns: The desired output dimension. - state_saver: A state saver object with methods `state` and `save_state`. - state_name: Python string or tuple of strings. The name to use with the - state_saver. If the cell returns tuples of states (i.e., - `cell.state_size` is a tuple) then `state_name` should be a tuple of - strings having the same length as `cell.state_size`. Otherwise it should - be a single string. - scope: `VariableScope` for the created subgraph; defaults to "rnn". - - Returns: - activations: The output of the RNN, projected to `num_label_columns` - dimensions, a `Tensor` of shape `[batch_size, T, num_label_columns]`. - final_state: The final state output by the RNN - """ - with ops.name_scope(scope): - rnn_outputs, final_state = rnn.static_state_saving_rnn( - cell=cell, - inputs=inputs, - state_saver=state_saver, - state_name=state_name, - scope=scope) - # Convert rnn_outputs from a list of time-major order Tensors to a single - # Tensor of batch-major order. - rnn_outputs = array_ops.stack(rnn_outputs, axis=1) - activations = layers.fully_connected( - inputs=rnn_outputs, - num_outputs=num_label_columns, - activation_fn=None, - trainable=True) - # Use `identity` to rename `final_state`. - final_state = array_ops.identity( - final_state, name=rnn_common.RNNKeys.FINAL_STATE_KEY) - return activations, final_state - - -def _multi_value_loss( - activations, labels, sequence_length, target_column, features): - """Maps `activations` from the RNN to loss for multi value models. - - Args: - activations: Output from an RNN. Should have dtype `float32` and shape - `[batch_size, padded_length, ?]`. - labels: A `Tensor` with length `[batch_size, padded_length]`. - sequence_length: A `Tensor` with shape `[batch_size]` and dtype `int32` - containing the length of each sequence in the batch. If `None`, sequences - are assumed to be unpadded. - target_column: An initialized `TargetColumn`, calculate predictions. - features: A `dict` containing the input and (optionally) sequence length - information and initial state. - Returns: - A scalar `Tensor` containing the loss. - """ - with ops.name_scope('MultiValueLoss'): - activations_masked, labels_masked = rnn_common.mask_activations_and_labels( - activations, labels, sequence_length) - return target_column.loss(activations_masked, labels_masked, features) - - -def _get_name_or_parent_names(column): - """Gets the name of a column or its parent columns' names. - - Args: - column: A sequence feature column derived from `FeatureColumn`. - - Returns: - A list of the name of `column` or the names of its parent columns, - if any exist. - """ - # pylint: disable=protected-access - parent_columns = feature_column_ops._get_parent_columns(column) - if parent_columns: - return [x.name for x in parent_columns] - return [column.name] - - -def _prepare_features_for_sqss(features, labels, mode, - sequence_feature_columns, - context_feature_columns): - """Prepares features for batching by the SQSS. - - In preparation for batching by the SQSS, this function: - - Extracts the input key from the features dict. - - Separates sequence and context features dicts from the features dict. - - Adds the labels tensor to the sequence features dict. - - Args: - features: A dict of Python string to an iterable of `Tensor` or - `SparseTensor` of rank 2, the `features` argument of a TF.Learn model_fn. - labels: An iterable of `Tensor`. - mode: Defines whether this is training, evaluation or prediction. - See `ModeKeys`. - sequence_feature_columns: An iterable containing all the feature columns - describing sequence features. All items in the set should be instances - of classes derived from `FeatureColumn`. - context_feature_columns: An iterable containing all the feature columns - describing context features, i.e., features that apply across all time - steps. All items in the set should be instances of classes derived from - `FeatureColumn`. - - Returns: - sequence_features: A dict mapping feature names to sequence features. - context_features: A dict mapping feature names to context features. - - Raises: - ValueError: If `features` does not contain a value for every key in - `sequence_feature_columns` or `context_feature_columns`. - """ - - # Extract sequence features. - feature_column_ops._check_supported_sequence_columns(sequence_feature_columns) # pylint: disable=protected-access - sequence_features = {} - for column in sequence_feature_columns: - for name in _get_name_or_parent_names(column): - feature = features.get(name, None) - if feature is None: - raise ValueError('No key in features for sequence feature: ' + name) - sequence_features[name] = feature - - # Extract context features. - context_features = {} - if context_feature_columns is not None: - for column in context_feature_columns: - name = column.name - feature = features.get(name, None) - if feature is None: - raise ValueError('No key in features for context feature: ' + name) - context_features[name] = feature - - # Add labels to the resulting sequence features dict. - if mode != model_fn.ModeKeys.INFER: - sequence_features[rnn_common.RNNKeys.LABELS_KEY] = labels - - return sequence_features, context_features - - -def _get_state_names(cell): - """Gets the state names for an `RNNCell`. - - Args: - cell: A `RNNCell` to be used in the RNN. - - Returns: - State names in the form of a string, a list of strings, or a list of - string pairs, depending on the type of `cell.state_size`. - - Raises: - TypeError: If cell.state_size is of type TensorShape. - """ - state_size = cell.state_size - if isinstance(state_size, tensor_shape.TensorShape): - raise TypeError('cell.state_size of type TensorShape is not supported.') - if isinstance(state_size, int): - return '{}_{}'.format(rnn_common.RNNKeys.STATE_PREFIX, 0) - if isinstance(state_size, rnn_cell.LSTMStateTuple): - return [ - '{}_{}_c'.format(rnn_common.RNNKeys.STATE_PREFIX, 0), - '{}_{}_h'.format(rnn_common.RNNKeys.STATE_PREFIX, 0), - ] - if isinstance(state_size[0], rnn_cell.LSTMStateTuple): - return [[ - '{}_{}_c'.format(rnn_common.RNNKeys.STATE_PREFIX, i), - '{}_{}_h'.format(rnn_common.RNNKeys.STATE_PREFIX, i), - ] for i in range(len(state_size))] - return [ - '{}_{}'.format(rnn_common.RNNKeys.STATE_PREFIX, i) - for i in range(len(state_size))] - - -def _get_initial_states(cell): - """Gets the initial state of the `RNNCell` used in the RNN. - - Args: - cell: A `RNNCell` to be used in the RNN. - - Returns: - A Python dict mapping state names to the `RNNCell`'s initial state for - consumption by the SQSS. - """ - names = nest.flatten(_get_state_names(cell)) - values = nest.flatten(cell.zero_state(1, dtype=dtypes.float32)) - return {n: array_ops.squeeze(v, axis=0) for [n, v] in zip(names, values)} - - -def _read_batch(cell, - features, - labels, - mode, - num_unroll, - batch_size, - sequence_feature_columns, - context_feature_columns=None, - num_threads=3, - queue_capacity=1000, - seed=None): - """Reads a batch from a state saving sequence queue. - - Args: - cell: An initialized `RNNCell` to be used in the RNN. - features: A dict of Python string to an iterable of `Tensor`, the - `features` argument of a TF.Learn model_fn. - labels: An iterable of `Tensor`, the `labels` argument of a - TF.Learn model_fn. - mode: Defines whether this is training, evaluation or prediction. - See `ModeKeys`. - num_unroll: Python integer, how many time steps to unroll at a time. - The input sequences of length `k` are then split into `k / num_unroll` - many segments. - batch_size: Python integer, the size of the minibatch produced by the SQSS. - sequence_feature_columns: An iterable containing all the feature columns - describing sequence features. All items in the set should be instances - of classes derived from `FeatureColumn`. - context_feature_columns: An iterable containing all the feature columns - describing context features, i.e., features that apply across all time - steps. All items in the set should be instances of classes derived from - `FeatureColumn`. - num_threads: The Python integer number of threads enqueuing input examples - into a queue. Defaults to 3. - queue_capacity: The max capacity of the queue in number of examples. - Needs to be at least `batch_size`. Defaults to 1000. When iterating - over the same input example multiple times reusing their keys the - `queue_capacity` must be smaller than the number of examples. - seed: Fixes the random seed used for generating input keys by the SQSS. - - Returns: - batch: A `NextQueuedSequenceBatch` containing batch_size `SequenceExample` - values and their saved internal states. - """ - states = _get_initial_states(cell) - - sequences, context = _prepare_features_for_sqss( - features, labels, mode, sequence_feature_columns, - context_feature_columns) - - return sqss.batch_sequences_with_states( - input_key='key', - input_sequences=sequences, - input_context=context, - input_length=None, # infer sequence lengths - initial_states=states, - num_unroll=num_unroll, - batch_size=batch_size, - pad=True, # pad to a multiple of num_unroll - make_keys_unique=True, - make_keys_unique_seed=seed, - num_threads=num_threads, - capacity=queue_capacity) - - -def _get_state_name(i): - """Constructs the name string for state component `i`.""" - return '{}_{}'.format(rnn_common.RNNKeys.STATE_PREFIX, i) - - -def state_tuple_to_dict(state): - """Returns a dict containing flattened `state`. - - Args: - state: A `Tensor` or a nested tuple of `Tensors`. All of the `Tensor`s must - have the same rank and agree on all dimensions except the last. - - Returns: - A dict containing the `Tensor`s that make up `state`. The keys of the dict - are of the form "STATE_PREFIX_i" where `i` is the place of this `Tensor` - in a depth-first traversal of `state`. - """ - with ops.name_scope('state_tuple_to_dict'): - flat_state = nest.flatten(state) - state_dict = {} - for i, state_component in enumerate(flat_state): - state_name = _get_state_name(i) - state_value = (None if state_component is None else array_ops.identity( - state_component, name=state_name)) - state_dict[state_name] = state_value - return state_dict - - -def _prepare_inputs_for_rnn(sequence_features, context_features, - sequence_feature_columns, num_unroll): - """Prepares features batched by the SQSS for input to a state-saving RNN. - - Args: - sequence_features: A dict of sequence feature name to `Tensor` or - `SparseTensor`, with `Tensor`s of shape `[batch_size, num_unroll, ...]` - or `SparseTensors` of dense shape `[batch_size, num_unroll, d]`. - context_features: A dict of context feature name to `Tensor`, with - tensors of shape `[batch_size, 1, ...]` and type float32. - sequence_feature_columns: An iterable containing all the feature columns - describing sequence features. All items in the set should be instances - of classes derived from `FeatureColumn`. - num_unroll: Python integer, how many time steps to unroll at a time. - The input sequences of length `k` are then split into `k / num_unroll` - many segments. - - Returns: - features_by_time: A list of length `num_unroll` with `Tensor` entries of - shape `[batch_size, sum(sequence_features dimensions) + - sum(context_features dimensions)]` of type float32. - Context features are copied into each time step. - """ - - def _tile(feature): - return array_ops.squeeze( - array_ops.tile(array_ops.expand_dims(feature, 1), [1, num_unroll, 1]), - axis=2) - for feature in sequence_features.values(): - if isinstance(feature, sparse_tensor.SparseTensor): - # Explicitly set dense_shape's shape to 3 ([batch_size, num_unroll, d]) - # since it can't be statically inferred. - feature.dense_shape.set_shape([3]) - sequence_features = layers.sequence_input_from_feature_columns( - columns_to_tensors=sequence_features, - feature_columns=sequence_feature_columns, - weight_collections=None, - scope=None) - # Explicitly set shape along dimension 1 to num_unroll for the unstack op. - sequence_features.set_shape([None, num_unroll, None]) - - if not context_features: - return array_ops.unstack(sequence_features, axis=1) - # TODO(jtbates): Call layers.input_from_feature_columns for context features. - context_features = [ - _tile(context_features[k]) for k in sorted(context_features) - ] - return array_ops.unstack( - array_ops.concat( - [sequence_features, array_ops.stack(context_features, 2)], axis=2), - axis=1) - - -def _get_rnn_model_fn(cell_type, - target_column, - problem_type, - optimizer, - num_unroll, - num_units, - num_threads, - queue_capacity, - batch_size, - sequence_feature_columns, - context_feature_columns=None, - predict_probabilities=False, - learning_rate=None, - gradient_clipping_norm=None, - dropout_keep_probabilities=None, - name='StateSavingRNNModel', - seed=None): - """Creates a state saving RNN model function for an `Estimator`. - - Args: - cell_type: A subclass of `RNNCell` or one of 'basic_rnn,' 'lstm' or 'gru'. - target_column: An initialized `TargetColumn`, used to calculate prediction - and loss. - problem_type: `ProblemType.CLASSIFICATION` or - `ProblemType.LINEAR_REGRESSION`. - optimizer: A subclass of `Optimizer`, an instance of an `Optimizer` or a - string. - num_unroll: Python integer, how many time steps to unroll at a time. - The input sequences of length `k` are then split into `k / num_unroll` - many segments. - num_units: The number of units in the `RNNCell`. - num_threads: The Python integer number of threads enqueuing input examples - into a queue. - queue_capacity: The max capacity of the queue in number of examples. - Needs to be at least `batch_size`. When iterating over the same input - example multiple times reusing their keys the `queue_capacity` must be - smaller than the number of examples. - batch_size: Python integer, the size of the minibatch produced by the SQSS. - sequence_feature_columns: An iterable containing all the feature columns - describing sequence features. All items in the set should be instances - of classes derived from `FeatureColumn`. - context_feature_columns: An iterable containing all the feature columns - describing context features, i.e., features that apply across all time - steps. All items in the set should be instances of classes derived from - `FeatureColumn`. - predict_probabilities: A boolean indicating whether to predict probabilities - for all classes. - Must only be used with `ProblemType.CLASSIFICATION`. - learning_rate: Learning rate used for optimization. This argument has no - effect if `optimizer` is an instance of an `Optimizer`. - gradient_clipping_norm: A float. Gradients will be clipped to this value. - dropout_keep_probabilities: a list of dropout keep probabilities or `None`. - If given a list, it must have length `len(num_units) + 1`. - name: A string that will be used to create a scope for the RNN. - seed: Fixes the random seed used for generating input keys by the SQSS. - - Returns: - A model function to be passed to an `Estimator`. - - Raises: - ValueError: `problem_type` is not one of - `ProblemType.LINEAR_REGRESSION` - or `ProblemType.CLASSIFICATION`. - ValueError: `predict_probabilities` is `True` for `problem_type` other - than `ProblemType.CLASSIFICATION`. - ValueError: `num_unroll` is not positive. - """ - if problem_type not in (constants.ProblemType.CLASSIFICATION, - constants.ProblemType.LINEAR_REGRESSION): - raise ValueError( - 'problem_type must be ProblemType.LINEAR_REGRESSION or ' - 'ProblemType.CLASSIFICATION; got {}'. - format(problem_type)) - if (problem_type != constants.ProblemType.CLASSIFICATION and - predict_probabilities): - raise ValueError( - 'predict_probabilities can only be set to True for problem_type' - ' ProblemType.CLASSIFICATION; got {}.'.format(problem_type)) - if num_unroll <= 0: - raise ValueError('num_unroll must be positive; got {}.'.format(num_unroll)) - - def _rnn_model_fn(features, labels, mode): - """The model to be passed to an `Estimator`.""" - with ops.name_scope(name): - dropout = (dropout_keep_probabilities - if mode == model_fn.ModeKeys.TRAIN - else None) - cell = rnn_common.construct_rnn_cell(num_units, cell_type, dropout) - - batch = _read_batch( - cell=cell, - features=features, - labels=labels, - mode=mode, - num_unroll=num_unroll, - batch_size=batch_size, - sequence_feature_columns=sequence_feature_columns, - context_feature_columns=context_feature_columns, - num_threads=num_threads, - queue_capacity=queue_capacity, - seed=seed) - sequence_features = batch.sequences - context_features = batch.context - if mode != model_fn.ModeKeys.INFER: - labels = sequence_features.pop(rnn_common.RNNKeys.LABELS_KEY) - inputs = _prepare_inputs_for_rnn(sequence_features, context_features, - sequence_feature_columns, num_unroll) - state_name = _get_state_names(cell) - rnn_activations, final_state = construct_state_saving_rnn( - cell=cell, - inputs=inputs, - num_label_columns=target_column.num_label_columns, - state_saver=batch, - state_name=state_name) - - loss = None # Created below for modes TRAIN and EVAL. - prediction_dict = rnn_common.multi_value_predictions( - rnn_activations, target_column, problem_type, predict_probabilities) - if mode != model_fn.ModeKeys.INFER: - loss = _multi_value_loss(rnn_activations, labels, batch.length, - target_column, features) - - eval_metric_ops = None - if mode != model_fn.ModeKeys.INFER: - eval_metric_ops = rnn_common.get_eval_metric_ops( - problem_type, rnn_common.PredictionType.MULTIPLE_VALUE, - batch.length, prediction_dict, labels) - - state_dict = state_tuple_to_dict(final_state) - prediction_dict.update(state_dict) - - train_op = None - if mode == model_fn.ModeKeys.TRAIN: - train_op = optimizers.optimize_loss( - loss=loss, - global_step=None, # Get it internally. - learning_rate=learning_rate, - optimizer=optimizer, - clip_gradients=gradient_clipping_norm, - summaries=optimizers.OPTIMIZER_SUMMARIES) - - return model_fn.ModelFnOps(mode=mode, - predictions=prediction_dict, - loss=loss, - train_op=train_op, - eval_metric_ops=eval_metric_ops) - return _rnn_model_fn - - -class StateSavingRnnEstimator(estimator.Estimator): - """RNN with static unrolling and state saving (deprecated). - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - def __init__(self, - problem_type, - num_unroll, - batch_size, - sequence_feature_columns, - context_feature_columns=None, - num_classes=None, - num_units=None, - cell_type='basic_rnn', - optimizer_type='SGD', - learning_rate=0.1, - predict_probabilities=False, - momentum=None, - gradient_clipping_norm=5.0, - dropout_keep_probabilities=None, - model_dir=None, - config=None, - feature_engineering_fn=None, - num_threads=3, - queue_capacity=1000, - seed=None): - """Initializes a StateSavingRnnEstimator. - - Args: - problem_type: `ProblemType.CLASSIFICATION` or - `ProblemType.LINEAR_REGRESSION`. - num_unroll: Python integer, how many time steps to unroll at a time. - The input sequences of length `k` are then split into `k / num_unroll` - many segments. - batch_size: Python integer, the size of the minibatch. - sequence_feature_columns: An iterable containing all the feature columns - describing sequence features. All items in the set should be instances - of classes derived from `FeatureColumn`. - context_feature_columns: An iterable containing all the feature columns - describing context features, i.e., features that apply across all time - steps. All items in the set should be instances of classes derived from - `FeatureColumn`. - num_classes: The number of classes for categorization. Used only and - required if `problem_type` is `ProblemType.CLASSIFICATION`. - num_units: A list of integers indicating the number of units in the - `RNNCell`s in each layer. Either `num_units` is specified or `cell_type` - is an instance of `RNNCell`. - cell_type: A subclass of `RNNCell` or one of 'basic_rnn,' 'lstm' or 'gru'. - optimizer_type: The type of optimizer to use. Either a subclass of - `Optimizer`, an instance of an `Optimizer` or a string. Strings must be - one of 'Adagrad', 'Adam', 'Ftrl', Momentum', 'RMSProp', or 'SGD'. - learning_rate: Learning rate. This argument has no effect if `optimizer` - is an instance of an `Optimizer`. - predict_probabilities: A boolean indicating whether to predict - probabilities for all classes. Used only if `problem_type` is - `ProblemType.CLASSIFICATION`. - momentum: Momentum value. Only used if `optimizer_type` is 'Momentum'. - gradient_clipping_norm: Parameter used for gradient clipping. If `None`, - then no clipping is performed. - dropout_keep_probabilities: a list of dropout keep probabilities or - `None`. If given a list, it must have length `len(num_units) + 1`. - model_dir: The directory in which to save and restore the model graph, - parameters, etc. - config: A `RunConfig` instance. - feature_engineering_fn: Takes features and labels which are the output of - `input_fn` and returns features and labels which will be fed into - `model_fn`. Please check `model_fn` for a definition of features and - labels. - num_threads: The Python integer number of threads enqueuing input examples - into a queue. Defaults to 3. - queue_capacity: The max capacity of the queue in number of examples. - Needs to be at least `batch_size`. Defaults to 1000. When iterating - over the same input example multiple times reusing their keys the - `queue_capacity` must be smaller than the number of examples. - seed: Fixes the random seed used for generating input keys by the SQSS. - - Raises: - ValueError: Both or neither of the following are true: (a) `num_units` is - specified and (b) `cell_type` is an instance of `RNNCell`. - ValueError: `problem_type` is not one of - `ProblemType.LINEAR_REGRESSION` or `ProblemType.CLASSIFICATION`. - ValueError: `problem_type` is `ProblemType.CLASSIFICATION` but - `num_classes` is not specified. - """ - name = 'MultiValueStateSavingRNN' - if problem_type == constants.ProblemType.LINEAR_REGRESSION: - name += 'Regressor' - target_column = layers.regression_target() - elif problem_type == constants.ProblemType.CLASSIFICATION: - if not num_classes: - raise ValueError('For CLASSIFICATION problem_type, num_classes must be ' - 'specified.') - target_column = layers.multi_class_target(n_classes=num_classes) - name += 'Classifier' - else: - raise ValueError( - 'problem_type must be either ProblemType.LINEAR_REGRESSION ' - 'or ProblemType.CLASSIFICATION; got {}'.format( - problem_type)) - - if optimizer_type == 'Momentum': - optimizer_type = momentum_opt.MomentumOptimizer(learning_rate, momentum) - - rnn_model_fn = _get_rnn_model_fn( - cell_type=cell_type, - target_column=target_column, - problem_type=problem_type, - optimizer=optimizer_type, - num_unroll=num_unroll, - num_units=num_units, - num_threads=num_threads, - queue_capacity=queue_capacity, - batch_size=batch_size, - sequence_feature_columns=sequence_feature_columns, - context_feature_columns=context_feature_columns, - predict_probabilities=predict_probabilities, - learning_rate=learning_rate, - gradient_clipping_norm=gradient_clipping_norm, - dropout_keep_probabilities=dropout_keep_probabilities, - name=name, - seed=seed) - - super(StateSavingRnnEstimator, self).__init__( - model_fn=rnn_model_fn, - model_dir=model_dir, - config=config, - feature_engineering_fn=feature_engineering_fn) diff --git a/tensorflow/contrib/learn/python/learn/estimators/state_saving_rnn_estimator_test.py b/tensorflow/contrib/learn/python/learn/estimators/state_saving_rnn_estimator_test.py deleted file mode 100644 index 0689be88c5e..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/state_saving_rnn_estimator_test.py +++ /dev/null @@ -1,665 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for learn.estimators.state_saving_rnn_estimator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tempfile - -import numpy as np - -from tensorflow.contrib import lookup -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.layers.python.layers import target_column as target_column_lib -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.contrib.learn.python.learn.estimators import rnn_common -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.contrib.learn.python.learn.estimators import state_saving_rnn_estimator as ssre -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class PrepareInputsForRnnTest(test.TestCase): - - def _test_prepare_inputs_for_rnn(self, sequence_features, context_features, - sequence_feature_columns, num_unroll, - expected): - features_by_time = ssre._prepare_inputs_for_rnn(sequence_features, - context_features, - sequence_feature_columns, - num_unroll) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - sess.run(lookup_ops.tables_initializer()) - features_val = sess.run(features_by_time) - self.assertAllEqual(expected, features_val) - - def testPrepareInputsForRnnBatchSize1(self): - num_unroll = 3 - - expected = [ - np.array([[11., 31., 5., 7.]]), np.array([[12., 32., 5., 7.]]), - np.array([[13., 33., 5., 7.]]) - ] - - sequence_features = { - 'seq_feature0': constant_op.constant([[11., 12., 13.]]), - 'seq_feature1': constant_op.constant([[31., 32., 33.]]) - } - - sequence_feature_columns = [ - feature_column.real_valued_column( - 'seq_feature0', dimension=1), - feature_column.real_valued_column( - 'seq_feature1', dimension=1), - ] - - context_features = { - 'ctx_feature0': constant_op.constant([[5.]]), - 'ctx_feature1': constant_op.constant([[7.]]) - } - self._test_prepare_inputs_for_rnn(sequence_features, context_features, - sequence_feature_columns, num_unroll, - expected) - - def testPrepareInputsForRnnBatchSize2(self): - - num_unroll = 3 - - expected = [ - np.array([[11., 31., 5., 7.], [21., 41., 6., 8.]]), - np.array([[12., 32., 5., 7.], [22., 42., 6., 8.]]), - np.array([[13., 33., 5., 7.], [23., 43., 6., 8.]]) - ] - - sequence_features = { - 'seq_feature0': - constant_op.constant([[11., 12., 13.], [21., 22., 23.]]), - 'seq_feature1': - constant_op.constant([[31., 32., 33.], [41., 42., 43.]]) - } - - sequence_feature_columns = [ - feature_column.real_valued_column( - 'seq_feature0', dimension=1), - feature_column.real_valued_column( - 'seq_feature1', dimension=1), - ] - - context_features = { - 'ctx_feature0': constant_op.constant([[5.], [6.]]), - 'ctx_feature1': constant_op.constant([[7.], [8.]]) - } - - self._test_prepare_inputs_for_rnn(sequence_features, context_features, - sequence_feature_columns, num_unroll, - expected) - - def testPrepareInputsForRnnNoContext(self): - num_unroll = 3 - - expected = [ - np.array([[11., 31.], [21., 41.]]), np.array([[12., 32.], [22., 42.]]), - np.array([[13., 33.], [23., 43.]]) - ] - - sequence_features = { - 'seq_feature0': - constant_op.constant([[11., 12., 13.], [21., 22., 23.]]), - 'seq_feature1': - constant_op.constant([[31., 32., 33.], [41., 42., 43.]]) - } - - sequence_feature_columns = [ - feature_column.real_valued_column( - 'seq_feature0', dimension=1), - feature_column.real_valued_column( - 'seq_feature1', dimension=1), - ] - - context_features = None - - self._test_prepare_inputs_for_rnn(sequence_features, context_features, - sequence_feature_columns, num_unroll, - expected) - - def testPrepareInputsForRnnSparse(self): - num_unroll = 2 - embedding_dimension = 8 - - expected = [ - np.array([[1., 1., 1., 1., 1., 1., 1., 1.], - [1., 1., 1., 1., 1., 1., 1., 1.], - [1., 1., 1., 1., 1., 1., 1., 1.]]), - np.array([[1., 1., 1., 1., 1., 1., 1., 1.], - [2., 2., 2., 2., 2., 2., 2., 2.], - [1., 1., 1., 1., 1., 1., 1., 1.]]) - ] - - sequence_features = { - 'wire_cast': - sparse_tensor.SparseTensor( - indices=[[0, 0, 0], [0, 1, 0], [1, 0, 0], [1, 1, 0], [1, 1, 1], - [2, 0, 0], [2, 1, 1]], - values=[ - b'marlo', b'stringer', b'omar', b'stringer', b'marlo', - b'marlo', b'omar' - ], - dense_shape=[3, 2, 2]) - } - - wire_cast = feature_column.sparse_column_with_keys( - 'wire_cast', ['marlo', 'omar', 'stringer']) - sequence_feature_columns = [ - feature_column.embedding_column( - wire_cast, - dimension=embedding_dimension, - combiner='sum', - initializer=init_ops.ones_initializer()) - ] - - context_features = None - - self._test_prepare_inputs_for_rnn(sequence_features, context_features, - sequence_feature_columns, num_unroll, - expected) - - def testPrepareInputsForRnnSparseAndDense(self): - num_unroll = 2 - embedding_dimension = 8 - dense_dimension = 2 - - expected = [ - np.array([[1., 1., 1., 1., 1., 1., 1., 1., 111., 112.], - [1., 1., 1., 1., 1., 1., 1., 1., 211., 212.], - [1., 1., 1., 1., 1., 1., 1., 1., 311., 312.]]), - np.array([[1., 1., 1., 1., 1., 1., 1., 1., 121., 122.], - [2., 2., 2., 2., 2., 2., 2., 2., 221., 222.], - [1., 1., 1., 1., 1., 1., 1., 1., 321., 322.]]) - ] - - sequence_features = { - 'wire_cast': - sparse_tensor.SparseTensor( - indices=[[0, 0, 0], [0, 1, 0], [1, 0, 0], [1, 1, 0], [1, 1, 1], - [2, 0, 0], [2, 1, 1]], - values=[ - b'marlo', b'stringer', b'omar', b'stringer', b'marlo', - b'marlo', b'omar' - ], - dense_shape=[3, 2, 2]), - 'seq_feature0': - constant_op.constant([[[111., 112.], [121., 122.]], - [[211., 212.], [221., 222.]], - [[311., 312.], [321., 322.]]]) - } - - wire_cast = feature_column.sparse_column_with_keys( - 'wire_cast', ['marlo', 'omar', 'stringer']) - wire_cast_embedded = feature_column.embedding_column( - wire_cast, - dimension=embedding_dimension, - combiner='sum', - initializer=init_ops.ones_initializer()) - seq_feature0_column = feature_column.real_valued_column( - 'seq_feature0', dimension=dense_dimension) - - sequence_feature_columns = [seq_feature0_column, wire_cast_embedded] - - context_features = None - - self._test_prepare_inputs_for_rnn(sequence_features, context_features, - sequence_feature_columns, num_unroll, - expected) - - -class StateSavingRnnEstimatorTest(test.TestCase): - - def testPrepareFeaturesForSQSS(self): - mode = model_fn_lib.ModeKeys.TRAIN - seq_feature_name = 'seq_feature' - sparse_seq_feature_name = 'wire_cast' - ctx_feature_name = 'ctx_feature' - sequence_length = 4 - embedding_dimension = 8 - - features = { - sparse_seq_feature_name: - sparse_tensor.SparseTensor( - indices=[[0, 0, 0], [0, 1, 0], [1, 0, 0], [1, 1, 0], [1, 1, 1], - [2, 0, 0], [2, 1, 1]], - values=[ - b'marlo', b'stringer', b'omar', b'stringer', b'marlo', - b'marlo', b'omar' - ], - dense_shape=[3, 2, 2]), - seq_feature_name: - constant_op.constant( - 1.0, shape=[sequence_length]), - ctx_feature_name: - constant_op.constant(2.0) - } - - labels = constant_op.constant(5.0, shape=[sequence_length]) - - wire_cast = feature_column.sparse_column_with_keys( - 'wire_cast', ['marlo', 'omar', 'stringer']) - sequence_feature_columns = [ - feature_column.real_valued_column( - seq_feature_name, dimension=1), feature_column.embedding_column( - wire_cast, - dimension=embedding_dimension, - initializer=init_ops.ones_initializer()) - ] - - context_feature_columns = [ - feature_column.real_valued_column( - ctx_feature_name, dimension=1) - ] - - expected_sequence = { - rnn_common.RNNKeys.LABELS_KEY: - np.array([5., 5., 5., 5.]), - seq_feature_name: - np.array([1., 1., 1., 1.]), - sparse_seq_feature_name: - sparse_tensor.SparseTensor( - indices=[[0, 0, 0], [0, 1, 0], [1, 0, 0], [1, 1, 0], [1, 1, 1], - [2, 0, 0], [2, 1, 1]], - values=[ - b'marlo', b'stringer', b'omar', b'stringer', b'marlo', - b'marlo', b'omar' - ], - dense_shape=[3, 2, 2]), - } - - expected_context = {ctx_feature_name: 2.} - - sequence, context = ssre._prepare_features_for_sqss( - features, labels, mode, sequence_feature_columns, - context_feature_columns) - - def assert_equal(expected, got): - self.assertEqual(sorted(expected), sorted(got)) - for k, v in expected.items(): - if isinstance(v, sparse_tensor.SparseTensor): - self.assertAllEqual(v.values.eval(), got[k].values) - self.assertAllEqual(v.indices.eval(), got[k].indices) - self.assertAllEqual(v.dense_shape.eval(), got[k].dense_shape) - else: - self.assertAllEqual(v, got[k]) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - sess.run(lookup_ops.tables_initializer()) - actual_sequence, actual_context = sess.run( - [sequence, context]) - assert_equal(expected_sequence, actual_sequence) - assert_equal(expected_context, actual_context) - - def _getModelFnOpsForMode(self, mode): - """Helper for testGetRnnModelFn{Train,Eval,Infer}().""" - num_units = [4] - seq_columns = [ - feature_column.real_valued_column( - 'inputs', dimension=1) - ] - features = { - 'inputs': constant_op.constant([1., 2., 3.]), - } - labels = constant_op.constant([1., 0., 1.]) - model_fn = ssre._get_rnn_model_fn( - cell_type='basic_rnn', - target_column=target_column_lib.multi_class_target(n_classes=2), - optimizer='SGD', - num_unroll=2, - num_units=num_units, - num_threads=1, - queue_capacity=10, - batch_size=1, - # Only CLASSIFICATION yields eval metrics to test for. - problem_type=constants.ProblemType.CLASSIFICATION, - sequence_feature_columns=seq_columns, - context_feature_columns=None, - learning_rate=0.1) - model_fn_ops = model_fn(features=features, labels=labels, mode=mode) - return model_fn_ops - - # testGetRnnModelFn{Train,Eval,Infer}() test which fields - # of ModelFnOps are set depending on mode. - def testGetRnnModelFnTrain(self): - model_fn_ops = self._getModelFnOpsForMode(model_fn_lib.ModeKeys.TRAIN) - self.assertIsNotNone(model_fn_ops.predictions) - self.assertIsNotNone(model_fn_ops.loss) - self.assertIsNotNone(model_fn_ops.train_op) - # None may get normalized to {}; we accept neither. - self.assertNotEqual(len(model_fn_ops.eval_metric_ops), 0) - - def testGetRnnModelFnEval(self): - model_fn_ops = self._getModelFnOpsForMode(model_fn_lib.ModeKeys.EVAL) - self.assertIsNotNone(model_fn_ops.predictions) - self.assertIsNotNone(model_fn_ops.loss) - self.assertIsNone(model_fn_ops.train_op) - # None may get normalized to {}; we accept neither. - self.assertNotEqual(len(model_fn_ops.eval_metric_ops), 0) - - def testGetRnnModelFnInfer(self): - model_fn_ops = self._getModelFnOpsForMode(model_fn_lib.ModeKeys.INFER) - self.assertIsNotNone(model_fn_ops.predictions) - self.assertIsNone(model_fn_ops.loss) - self.assertIsNone(model_fn_ops.train_op) - # None may get normalized to {}; we accept both. - self.assertFalse(model_fn_ops.eval_metric_ops) - - def testExport(self): - input_feature_key = 'magic_input_feature_key' - batch_size = 8 - num_units = [4] - sequence_length = 10 - num_unroll = 2 - num_classes = 2 - - seq_columns = [ - feature_column.real_valued_column( - 'inputs', dimension=4) - ] - - def get_input_fn(mode, seed): - - def input_fn(): - features = {} - random_sequence = random_ops.random_uniform( - [sequence_length + 1], 0, 2, dtype=dtypes.int32, seed=seed) - labels = array_ops.slice(random_sequence, [0], [sequence_length]) - inputs = math_ops.cast( - array_ops.slice(random_sequence, [1], [sequence_length]), - dtypes.float32) - features = {'inputs': inputs} - - if mode == model_fn_lib.ModeKeys.INFER: - input_examples = array_ops.placeholder(dtypes.string) - features[input_feature_key] = input_examples - labels = None - return features, labels - - return input_fn - - model_dir = tempfile.mkdtemp() - - def estimator_fn(): - return ssre.StateSavingRnnEstimator( - constants.ProblemType.CLASSIFICATION, - num_units=num_units, - num_unroll=num_unroll, - batch_size=batch_size, - sequence_feature_columns=seq_columns, - num_classes=num_classes, - predict_probabilities=True, - model_dir=model_dir, - queue_capacity=2 + batch_size, - seed=1234) - - # Train a bit to create an exportable checkpoint. - estimator_fn().fit(input_fn=get_input_fn( - model_fn_lib.ModeKeys.TRAIN, seed=1234), - steps=100) - # Now export, but from a fresh estimator instance, like you would - # in an export binary. That means .export() has to work without - # .fit() being called on the same object. - export_dir = tempfile.mkdtemp() - print('Exporting to', export_dir) - estimator_fn().export( - export_dir, - input_fn=get_input_fn( - model_fn_lib.ModeKeys.INFER, seed=4321), - use_deprecated_input_fn=False, - input_feature_key=input_feature_key) - - -# Smoke tests to ensure deprecated constructor functions still work. -class LegacyConstructorTest(test.TestCase): - - def _get_input_fn(self, - sequence_length, - seed=None): - def input_fn(): - random_sequence = random_ops.random_uniform( - [sequence_length + 1], 0, 2, dtype=dtypes.int32, seed=seed) - labels = array_ops.slice(random_sequence, [0], [sequence_length]) - inputs = math_ops.cast( - array_ops.slice(random_sequence, [1], [sequence_length]), - dtypes.float32) - return {'inputs': inputs}, labels - return input_fn - - -# TODO(jtbates): move all tests below to a benchmark test. -class StateSavingRNNEstimatorLearningTest(test.TestCase): - """Learning tests for state saving RNN Estimators.""" - - def testLearnSineFunction(self): - """Tests learning a sine function.""" - batch_size = 8 - num_unroll = 5 - sequence_length = 64 - train_steps = 250 - eval_steps = 20 - num_rnn_layers = 1 - num_units = [4] * num_rnn_layers - learning_rate = 0.3 - loss_threshold = 0.035 - - def get_sin_input_fn(sequence_length, increment, seed=None): - - def input_fn(): - start = random_ops.random_uniform( - (), minval=0, maxval=(np.pi * 2.0), dtype=dtypes.float32, seed=seed) - sin_curves = math_ops.sin( - math_ops.linspace(start, (sequence_length - 1) * increment, - sequence_length + 1)) - inputs = array_ops.slice(sin_curves, [0], [sequence_length]) - labels = array_ops.slice(sin_curves, [1], [sequence_length]) - return {'inputs': inputs}, labels - - return input_fn - - seq_columns = [ - feature_column.real_valued_column( - 'inputs', dimension=1) - ] - config = run_config.RunConfig(tf_random_seed=1234) - dropout_keep_probabilities = [0.9] * (num_rnn_layers + 1) - sequence_estimator = ssre.StateSavingRnnEstimator( - constants.ProblemType.LINEAR_REGRESSION, - num_units=num_units, - cell_type='lstm', - num_unroll=num_unroll, - batch_size=batch_size, - sequence_feature_columns=seq_columns, - learning_rate=learning_rate, - dropout_keep_probabilities=dropout_keep_probabilities, - config=config, - queue_capacity=2 * batch_size, - seed=1234) - - train_input_fn = get_sin_input_fn(sequence_length, np.pi / 32, seed=1234) - eval_input_fn = get_sin_input_fn(sequence_length, np.pi / 32, seed=4321) - - sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) - loss = sequence_estimator.evaluate( - input_fn=eval_input_fn, steps=eval_steps)['loss'] - self.assertLess(loss, loss_threshold, - 'Loss should be less than {}; got {}'.format(loss_threshold, - loss)) - - def testLearnShiftByOne(self): - """Tests that learning a 'shift-by-one' example. - - Each label sequence consists of the input sequence 'shifted' by one place. - The RNN must learn to 'remember' the previous input. - """ - batch_size = 16 - num_classes = 2 - num_unroll = 32 - sequence_length = 32 - train_steps = 300 - eval_steps = 20 - num_units = [4] - learning_rate = 0.5 - accuracy_threshold = 0.9 - - def get_shift_input_fn(sequence_length, seed=None): - - def input_fn(): - random_sequence = random_ops.random_uniform( - [sequence_length + 1], 0, 2, dtype=dtypes.int32, seed=seed) - labels = array_ops.slice(random_sequence, [0], [sequence_length]) - inputs = math_ops.cast( - array_ops.slice(random_sequence, [1], [sequence_length]), - dtypes.float32) - return {'inputs': inputs}, labels - - return input_fn - - seq_columns = [ - feature_column.real_valued_column( - 'inputs', dimension=1) - ] - config = run_config.RunConfig(tf_random_seed=21212) - sequence_estimator = ssre.StateSavingRnnEstimator( - constants.ProblemType.CLASSIFICATION, - num_units=num_units, - cell_type='lstm', - num_unroll=num_unroll, - batch_size=batch_size, - sequence_feature_columns=seq_columns, - num_classes=num_classes, - learning_rate=learning_rate, - config=config, - predict_probabilities=True, - queue_capacity=2 + batch_size, - seed=1234) - - train_input_fn = get_shift_input_fn(sequence_length, seed=12321) - eval_input_fn = get_shift_input_fn(sequence_length, seed=32123) - - sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) - - evaluation = sequence_estimator.evaluate( - input_fn=eval_input_fn, steps=eval_steps) - accuracy = evaluation['accuracy'] - self.assertGreater(accuracy, accuracy_threshold, - 'Accuracy should be higher than {}; got {}'.format( - accuracy_threshold, accuracy)) - - # Testing `predict` when `predict_probabilities=True`. - prediction_dict = sequence_estimator.predict( - input_fn=eval_input_fn, as_iterable=False) - self.assertListEqual( - sorted(list(prediction_dict.keys())), - sorted([ - prediction_key.PredictionKey.CLASSES, - prediction_key.PredictionKey.PROBABILITIES, ssre._get_state_name(0) - ])) - predictions = prediction_dict[prediction_key.PredictionKey.CLASSES] - probabilities = prediction_dict[prediction_key.PredictionKey.PROBABILITIES] - self.assertListEqual(list(predictions.shape), [batch_size, sequence_length]) - self.assertListEqual( - list(probabilities.shape), [batch_size, sequence_length, 2]) - - def testLearnLyrics(self): - lyrics = 'if I go there will be trouble and if I stay it will be double' - lyrics_list = lyrics.split() - sequence_length = len(lyrics_list) - vocab = set(lyrics_list) - batch_size = 16 - num_classes = len(vocab) - num_unroll = 7 # not a divisor of sequence_length - train_steps = 350 - eval_steps = 30 - num_units = [4] - learning_rate = 0.4 - accuracy_threshold = 0.65 - - def get_lyrics_input_fn(seed): - - def input_fn(): - start = random_ops.random_uniform( - (), minval=0, maxval=sequence_length, dtype=dtypes.int32, seed=seed) - # Concatenate lyrics_list so inputs and labels wrap when start > 0. - lyrics_list_concat = lyrics_list + lyrics_list - inputs_dense = array_ops.slice(lyrics_list_concat, [start], - [sequence_length]) - indices = array_ops.constant( - [[i, 0] for i in range(sequence_length)], dtype=dtypes.int64) - dense_shape = [sequence_length, 1] - inputs = sparse_tensor.SparseTensor( - indices=indices, values=inputs_dense, dense_shape=dense_shape) - table = lookup.string_to_index_table_from_tensor( - mapping=list(vocab), default_value=-1, name='lookup') - labels = table.lookup( - array_ops.slice(lyrics_list_concat, [start + 1], [sequence_length])) - return {'lyrics': inputs}, labels - - return input_fn - - sequence_feature_columns = [ - feature_column.embedding_column( - feature_column.sparse_column_with_keys('lyrics', vocab), - dimension=8) - ] - config = run_config.RunConfig(tf_random_seed=21212) - sequence_estimator = ssre.StateSavingRnnEstimator( - constants.ProblemType.CLASSIFICATION, - num_units=num_units, - cell_type='basic_rnn', - num_unroll=num_unroll, - batch_size=batch_size, - sequence_feature_columns=sequence_feature_columns, - num_classes=num_classes, - learning_rate=learning_rate, - config=config, - predict_probabilities=True, - queue_capacity=2 + batch_size, - seed=1234) - - train_input_fn = get_lyrics_input_fn(seed=12321) - eval_input_fn = get_lyrics_input_fn(seed=32123) - - sequence_estimator.fit(input_fn=train_input_fn, steps=train_steps) - - evaluation = sequence_estimator.evaluate( - input_fn=eval_input_fn, steps=eval_steps) - accuracy = evaluation['accuracy'] - self.assertGreater(accuracy, accuracy_threshold, - 'Accuracy should be higher than {}; got {}'.format( - accuracy_threshold, accuracy)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/svm.py b/tensorflow/contrib/learn/python/learn/estimators/svm.py deleted file mode 100644 index 3459997baba..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/svm.py +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Support Vector Machine (SVM) Estimator (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.framework import deprecated -from tensorflow.contrib.framework import deprecated_arg_values -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import linear -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.contrib.linear_optimizer.python import sdca_optimizer - - -def _as_iterable(preds, output): - for pred in preds: - yield pred[output] - - -class SVM(estimator.Estimator): - """Support Vector Machine (SVM) model for binary classification. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Currently, only linear SVMs are supported. For the underlying optimization - problem, the `SDCAOptimizer` is used. For performance and convergence tuning, - the num_loss_partitions parameter passed to `SDCAOptimizer` (see `__init__()` - method), should be set to (#concurrent train ops per worker) x (#workers). If - num_loss_partitions is larger or equal to this value, convergence is - guaranteed but becomes slower as num_loss_partitions increases. If it is set - to a smaller value, the optimizer is more aggressive in reducing the global - loss but convergence is not guaranteed. The recommended value in an - `Estimator` (where there is one process per worker) is the number of workers - running the train steps. It defaults to 1 (single machine). - - Example: - - ```python - real_feature_column = real_valued_column(...) - sparse_feature_column = sparse_column_with_hash_bucket(...) - - estimator = SVM( - example_id_column='example_id', - feature_columns=[real_feature_column, sparse_feature_column], - l2_regularization=10.0) - - # Input builders - def input_fn_train: # returns x, y - ... - def input_fn_eval: # returns x, y - ... - - estimator.fit(input_fn=input_fn_train) - estimator.evaluate(input_fn=input_fn_eval) - estimator.predict(x=x) - ``` - - Input of `fit` and `evaluate` should have following features, otherwise there - will be a `KeyError`: - a feature with `key=example_id_column` whose value is a `Tensor` of dtype - string. - if `weight_column_name` is not `None`, a feature with - `key=weight_column_name` whose value is a `Tensor`. - for each `column` in `feature_columns`: - - if `column` is a `SparseColumn`, a feature with `key=column.name` - whose `value` is a `SparseTensor`. - - if `column` is a `RealValuedColumn, a feature with `key=column.name` - whose `value` is a `Tensor`. - """ - - def __init__(self, - example_id_column, - feature_columns, - weight_column_name=None, - model_dir=None, - l1_regularization=0.0, - l2_regularization=0.0, - num_loss_partitions=1, - kernels=None, - config=None, - feature_engineering_fn=None): - """Constructs an `SVM` estimator object. - - Args: - example_id_column: A string defining the feature column name representing - example ids. Used to initialize the underlying optimizer. - feature_columns: An iterable containing all the feature columns used by - the model. All items in the set should be instances of classes derived - from `FeatureColumn`. - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - model_dir: Directory to save model parameters, graph and etc. This can - also be used to load checkpoints from the directory into a estimator to - continue training a previously saved model. - l1_regularization: L1-regularization parameter. Refers to global L1 - regularization (across all examples). - l2_regularization: L2-regularization parameter. Refers to global L2 - regularization (across all examples). - num_loss_partitions: number of partitions of the (global) loss function - optimized by the underlying optimizer (SDCAOptimizer). - kernels: A list of kernels for the SVM. Currently, no kernels are - supported. Reserved for future use for non-linear SVMs. - config: RunConfig object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and - returns features and labels which will be fed - into the model. - - Raises: - ValueError: if kernels passed is not None. - """ - if kernels is not None: - raise ValueError("Kernel SVMs are not currently supported.") - optimizer = sdca_optimizer.SDCAOptimizer( - example_id_column=example_id_column, - num_loss_partitions=num_loss_partitions, - symmetric_l1_regularization=l1_regularization, - symmetric_l2_regularization=l2_regularization) - - self._feature_columns = feature_columns - chief_hook = linear._SdcaUpdateWeightsHook() # pylint: disable=protected-access - super(SVM, self).__init__( - model_fn=linear.sdca_model_fn, - model_dir=model_dir, - config=config, - params={ - "head": head_lib.binary_svm_head( - weight_column_name=weight_column_name, - enable_centered_bias=False), - "feature_columns": feature_columns, - "optimizer": optimizer, - "weight_column_name": weight_column_name, - "update_weights_hook": chief_hook, - }, - feature_engineering_fn=feature_engineering_fn) - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - def predict_classes(self, x=None, input_fn=None, batch_size=None, - as_iterable=True): - """Runs inference to determine the predicted class.""" - key = prediction_key.PredictionKey.CLASSES - preds = super(SVM, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=[key], - as_iterable=as_iterable) - if as_iterable: - return _as_iterable(preds, output=key) - return preds[key] - - @deprecated_arg_values( - estimator.AS_ITERABLE_DATE, estimator.AS_ITERABLE_INSTRUCTIONS, - as_iterable=False) - def predict_proba(self, x=None, input_fn=None, batch_size=None, outputs=None, - as_iterable=True): - """Runs inference to determine the class probability predictions.""" - key = prediction_key.PredictionKey.PROBABILITIES - preds = super(SVM, self).predict( - x=x, - input_fn=input_fn, - batch_size=batch_size, - outputs=[key], - as_iterable=as_iterable) - if as_iterable: - return _as_iterable(preds, output=key) - return preds[key] - # pylint: enable=protected-access - - @deprecated("2017-03-25", "Please use Estimator.export_savedmodel() instead.") - def export(self, export_dir, signature_fn=None, - input_fn=None, default_batch_size=1, - exports_to_keep=None): - """See BaseEstimator.export.""" - return self.export_with_defaults( - export_dir=export_dir, - signature_fn=signature_fn, - input_fn=input_fn, - default_batch_size=default_batch_size, - exports_to_keep=exports_to_keep) - - @deprecated("2017-03-25", "Please use Estimator.export_savedmodel() instead.") - def export_with_defaults( - self, - export_dir, - signature_fn=None, - input_fn=None, - default_batch_size=1, - exports_to_keep=None): - """Same as BaseEstimator.export, but uses some defaults.""" - def default_input_fn(unused_estimator, examples): - return layers.parse_feature_columns_from_examples( - examples, self._feature_columns) - return super(SVM, self).export(export_dir=export_dir, - signature_fn=signature_fn, - input_fn=input_fn or default_input_fn, - default_batch_size=default_batch_size, - exports_to_keep=exports_to_keep) diff --git a/tensorflow/contrib/learn/python/learn/estimators/svm_test.py b/tensorflow/contrib/learn/python/learn/estimators/svm_test.py deleted file mode 100644 index f67f181d1ad..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/svm_test.py +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for estimators.SVM.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.learn.python.learn.estimators import svm -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.platform import test - - -class SVMTest(test.TestCase): - - def testRealValuedFeaturesPerfectlySeparable(self): - """Tests SVM classifier with real valued features.""" - - def input_fn(): - return { - 'example_id': constant_op.constant(['1', '2', '3']), - 'feature1': constant_op.constant([[0.0], [1.0], [3.0]]), - 'feature2': constant_op.constant([[1.0], [-1.2], [1.0]]), - }, constant_op.constant([[1], [0], [1]]) - - feature1 = feature_column.real_valued_column('feature1') - feature2 = feature_column.real_valued_column('feature2') - svm_classifier = svm.SVM(feature_columns=[feature1, feature2], - example_id_column='example_id', - l1_regularization=0.0, - l2_regularization=0.0) - svm_classifier.fit(input_fn=input_fn, steps=30) - metrics = svm_classifier.evaluate(input_fn=input_fn, steps=1) - loss = metrics['loss'] - accuracy = metrics['accuracy'] - # The points are not only separable but there exist weights (for instance - # w1=0.0, w2=1.0) that satisfy the margin inequalities (y_i* w^T*x_i >=1). - # The unregularized loss should therefore be 0.0. - self.assertAlmostEqual(loss, 0.0, places=3) - self.assertAlmostEqual(accuracy, 1.0, places=3) - - def testRealValuedFeaturesWithL2Regularization(self): - """Tests SVM classifier with real valued features and L2 regularization.""" - - def input_fn(): - return { - 'example_id': constant_op.constant(['1', '2', '3']), - 'feature1': constant_op.constant([0.5, 1.0, 1.0]), - 'feature2': constant_op.constant([1.0, -1.0, 0.5]), - }, constant_op.constant([1, 0, 1]) - - feature1 = feature_column.real_valued_column('feature1') - feature2 = feature_column.real_valued_column('feature2') - svm_classifier = svm.SVM(feature_columns=[feature1, feature2], - example_id_column='example_id', - l1_regularization=0.0, - l2_regularization=1.0) - svm_classifier.fit(input_fn=input_fn, steps=30) - metrics = svm_classifier.evaluate(input_fn=input_fn, steps=1) - loss = metrics['loss'] - accuracy = metrics['accuracy'] - # The points are in general separable. Also, if there was no regularization, - # the margin inequalities would be satisfied too (for instance by w1=1.0, - # w2=5.0). Due to regularization, smaller weights are chosen. This results - # to a small but non-zero uneregularized loss. Still, all the predictions - # will be correct resulting to perfect accuracy. - self.assertLess(loss, 0.1) - self.assertAlmostEqual(accuracy, 1.0, places=3) - - def testMultiDimensionalRealValuedFeaturesWithL2Regularization(self): - """Tests SVM with multi-dimensional real features and L2 regularization.""" - - # This is identical to the one in testRealValuedFeaturesWithL2Regularization - # where 2 tensors (dense features) of shape [3, 1] have been replaced by a - # single tensor (dense feature) of shape [3, 2]. - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'multi_dim_feature': - constant_op.constant([[0.5, 1.0], [1.0, -1.0], [1.0, 0.5]]), - }, constant_op.constant([[1], [0], [1]]) - - multi_dim_feature = feature_column.real_valued_column( - 'multi_dim_feature', dimension=2) - svm_classifier = svm.SVM(feature_columns=[multi_dim_feature], - example_id_column='example_id', - l1_regularization=0.0, - l2_regularization=1.0) - svm_classifier.fit(input_fn=input_fn, steps=30) - metrics = svm_classifier.evaluate(input_fn=input_fn, steps=1) - loss = metrics['loss'] - accuracy = metrics['accuracy'] - self.assertLess(loss, 0.1) - self.assertAlmostEqual(accuracy, 1.0, places=3) - - def testRealValuedFeaturesWithMildL1Regularization(self): - """Tests SVM classifier with real valued features and L2 regularization.""" - - def input_fn(): - return { - 'example_id': constant_op.constant(['1', '2', '3']), - 'feature1': constant_op.constant([[0.5], [1.0], [1.0]]), - 'feature2': constant_op.constant([[1.0], [-1.0], [0.5]]), - }, constant_op.constant([[1], [0], [1]]) - - feature1 = feature_column.real_valued_column('feature1') - feature2 = feature_column.real_valued_column('feature2') - svm_classifier = svm.SVM(feature_columns=[feature1, feature2], - example_id_column='example_id', - l1_regularization=0.5, - l2_regularization=1.0) - svm_classifier.fit(input_fn=input_fn, steps=30) - metrics = svm_classifier.evaluate(input_fn=input_fn, steps=1) - loss = metrics['loss'] - accuracy = metrics['accuracy'] - - # Adding small L1 regularization favors even smaller weights. This results - # to somewhat moderate unregularized loss (bigger than the one when there is - # no L1 regularization. Still, since L1 is small, all the predictions will - # be correct resulting to perfect accuracy. - self.assertGreater(loss, 0.1) - self.assertAlmostEqual(accuracy, 1.0, places=3) - - def testRealValuedFeaturesWithBigL1Regularization(self): - """Tests SVM classifier with real valued features and L2 regularization.""" - - def input_fn(): - return { - 'example_id': constant_op.constant(['1', '2', '3']), - 'feature1': constant_op.constant([0.5, 1.0, 1.0]), - 'feature2': constant_op.constant([[1.0], [-1.0], [0.5]]), - }, constant_op.constant([[1], [0], [1]]) - - feature1 = feature_column.real_valued_column('feature1') - feature2 = feature_column.real_valued_column('feature2') - svm_classifier = svm.SVM(feature_columns=[feature1, feature2], - example_id_column='example_id', - l1_regularization=3.0, - l2_regularization=1.0) - svm_classifier.fit(input_fn=input_fn, steps=30) - metrics = svm_classifier.evaluate(input_fn=input_fn, steps=1) - loss = metrics['loss'] - accuracy = metrics['accuracy'] - - # When L1 regularization parameter is large, the loss due to regularization - # outweights the unregularized loss. In this case, the classifier will favor - # very small weights (in current case 0) resulting both big unregularized - # loss and bad accuracy. - self.assertAlmostEqual(loss, 1.0, places=3) - self.assertAlmostEqual(accuracy, 1 / 3, places=3) - - def testSparseFeatures(self): - """Tests SVM classifier with (hashed) sparse features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([[0.8], [0.6], [0.3]]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - }, constant_op.constant([[0], [1], [1]]) - - price = feature_column.real_valued_column('price') - country = feature_column.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - svm_classifier = svm.SVM(feature_columns=[price, country], - example_id_column='example_id', - l1_regularization=0.0, - l2_regularization=1.0) - svm_classifier.fit(input_fn=input_fn, steps=30) - accuracy = svm_classifier.evaluate(input_fn=input_fn, steps=1)['accuracy'] - self.assertAlmostEqual(accuracy, 1.0, places=3) - - def testBucketizedFeatures(self): - """Tests SVM classifier with bucketized features.""" - - def input_fn(): - return { - 'example_id': constant_op.constant(['1', '2', '3']), - 'price': constant_op.constant([[600.0], [800.0], [400.0]]), - 'sq_footage': constant_op.constant([[1000.0], [800.0], [500.0]]), - 'weights': constant_op.constant([[1.0], [1.0], [1.0]]) - }, constant_op.constant([[1], [0], [1]]) - - price_bucket = feature_column.bucketized_column( - feature_column.real_valued_column('price'), boundaries=[500.0, 700.0]) - sq_footage_bucket = feature_column.bucketized_column( - feature_column.real_valued_column('sq_footage'), boundaries=[650.0]) - - svm_classifier = svm.SVM(feature_columns=[price_bucket, sq_footage_bucket], - example_id_column='example_id', - l1_regularization=0.1, - l2_regularization=1.0) - svm_classifier.fit(input_fn=input_fn, steps=30) - accuracy = svm_classifier.evaluate(input_fn=input_fn, steps=1)['accuracy'] - self.assertAlmostEqual(accuracy, 1.0, places=3) - - def testMixedFeatures(self): - """Tests SVM classifier with a mix of features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([0.6, 0.8, 0.3]), - 'sq_footage': - constant_op.constant([[900.0], [700.0], [600.0]]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - 'weights': - constant_op.constant([[3.0], [1.0], [1.0]]) - }, constant_op.constant([[1], [0], [1]]) - - price = feature_column.real_valued_column('price') - sq_footage_bucket = feature_column.bucketized_column( - feature_column.real_valued_column('sq_footage'), - boundaries=[650.0, 800.0]) - country = feature_column.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - sq_footage_country = feature_column.crossed_column( - [sq_footage_bucket, country], hash_bucket_size=10) - svm_classifier = svm.SVM( - feature_columns=[price, sq_footage_bucket, country, sq_footage_country], - example_id_column='example_id', - weight_column_name='weights', - l1_regularization=0.1, - l2_regularization=1.0) - - svm_classifier.fit(input_fn=input_fn, steps=30) - accuracy = svm_classifier.evaluate(input_fn=input_fn, steps=1)['accuracy'] - self.assertAlmostEqual(accuracy, 1.0, places=3) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/tensor_signature.py b/tensorflow/contrib/learn/python/learn/estimators/tensor_signature.py deleted file mode 100644 index 9eccb2ed185..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/tensor_signature.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""TensorSignature class and utilities (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import parsing_ops - - -class TensorSignature( - collections.namedtuple("TensorSignature", ["dtype", "shape", "is_sparse"])): - """Signature of the `Tensor` object. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Useful to check compatibility of tensors. - - Example: - - ```python - examples = tf.compat.v1.placeholder(...) - inputs = {'a': var_a, 'b': var_b} - signatures = tensor_signature.create_signatures(inputs) - result = tensor_signature.create_example_parser_from_signatures( - signatures, examples) - self.assertTrue(tensor_signature.tensors_compatible(result, signatures)) - ``` - - Attributes: - dtype: `DType` object. - shape: `TensorShape` object. - """ - - def __new__(cls, tensor): - if isinstance(tensor, sparse_tensor.SparseTensor): - return super(TensorSignature, cls).__new__( - cls, dtype=tensor.values.dtype, shape=None, is_sparse=True) - return super(TensorSignature, cls).__new__( - cls, dtype=tensor.dtype, shape=tensor.get_shape(), is_sparse=False) - - def is_compatible_with(self, other): - """Returns True if signatures are compatible.""" - - def _shape_is_compatible_0dim(this, other): - """Checks that shapes are compatible skipping dim 0.""" - other = tensor_shape.as_shape(other) - # If shapes are None (unknown) they may be compatible. - if this.dims is None or other.dims is None: - return True - if this.ndims != other.ndims: - return False - for dim, (x_dim, y_dim) in enumerate(zip(this.dims, other.dims)): - if dim == 0: - continue - if not x_dim.is_compatible_with(y_dim): - return False - return True - - if other.is_sparse: - return self.is_sparse and self.dtype.is_compatible_with(other.dtype) - return (self.dtype.is_compatible_with(other.dtype) and - _shape_is_compatible_0dim(self.shape, other.shape) and - not self.is_sparse) - - def get_placeholder(self): - if self.is_sparse: - return array_ops.sparse_placeholder(dtype=self.dtype) - return array_ops.placeholder( - dtype=self.dtype, shape=[None] + list(self.shape[1:])) - - def get_feature_spec(self): - dtype = self.dtype - # Convert, because example parser only supports float32, int64 and string. - if dtype == dtypes.int32: - dtype = dtypes.int64 - if dtype == dtypes.float64: - dtype = dtypes.float32 - if self.is_sparse: - return parsing_ops.VarLenFeature(dtype=dtype) - return parsing_ops.FixedLenFeature(shape=self.shape[1:], dtype=dtype) - - -def tensors_compatible(tensors, signatures): - """Check that tensors are compatible with signatures. - - Args: - tensors: Dict of `Tensor` objects or single `Tensor` object. - signatures: Dict of `TensorSignature` objects or single `TensorSignature` - object. - - Returns: - True if all tensors are compatible, False otherwise. - """ - # Dict of Tensors as input. - if tensors is None: - return signatures is None - - if isinstance(tensors, dict): - if not isinstance(signatures, dict): - return False - for key in signatures: - if key not in tensors: - return False - if not TensorSignature(tensors[key]).is_compatible_with(signatures[key]): - return False - return True - - # Single tensor as input. - if signatures is None or isinstance(signatures, dict): - return False - return TensorSignature(tensors).is_compatible_with(signatures) - - -def create_signatures(tensors): - """Creates TensorSignature objects for given tensors. - - Args: - tensors: Dict of `Tensor` objects or single `Tensor`. - - Returns: - Dict of `TensorSignature` objects or single `TensorSignature`. - """ - if isinstance(tensors, dict): - return {key: TensorSignature(tensors[key]) for key in tensors} - if tensors is None: - return None - return TensorSignature(tensors) - - -def create_placeholders_from_signatures(signatures): - """Creates placeholders from given signatures. - - Args: - signatures: Dict of `TensorSignature` objects or single `TensorSignature`, - or `None`. - - Returns: - Dict of `tf.compat.v1.placeholder` objects or single - `tf.compat.v1.placeholder`, or `None`. - """ - if signatures is None: - return None - if not isinstance(signatures, dict): - return signatures.get_placeholder() - return {key: signatures[key].get_placeholder() for key in signatures} - - -def create_example_parser_from_signatures(signatures, - examples_batch, - single_feature_name="feature"): - """Creates example parser from given signatures. - - Args: - signatures: Dict of `TensorSignature` objects or single `TensorSignature`. - examples_batch: string `Tensor` of serialized `Example` proto. - single_feature_name: string, single feature name. - - Returns: - features: `Tensor` or `dict` of `Tensor` objects. - """ - feature_spec = {} - if not isinstance(signatures, dict): - feature_spec[single_feature_name] = signatures.get_feature_spec() - else: - feature_spec = { - key: signatures[key].get_feature_spec() for key in signatures - } - features = parsing_ops.parse_example(examples_batch, feature_spec) - if not isinstance(signatures, dict): - # Returns single feature, casts if needed. - features = features[single_feature_name] - if not signatures.dtype.is_compatible_with(features.dtype): - features = math_ops.cast(features, signatures.dtype) - return features - # Returns dict of features, casts if needed. - for name in features: - if not signatures[name].dtype.is_compatible_with(features[name].dtype): - features[name] = math_ops.cast(features[name], signatures[name].dtype) - return features diff --git a/tensorflow/contrib/learn/python/learn/estimators/tensor_signature_test.py b/tensorflow/contrib/learn/python/learn/estimators/tensor_signature_test.py deleted file mode 100644 index 26c2b7840f8..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/tensor_signature_test.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for learn.estimators.tensor_signature.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.estimators import tensor_signature -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class TensorSignatureTest(test.TestCase): - - def testTensorPlaceholderNone(self): - self.assertEqual(None, - tensor_signature.create_placeholders_from_signatures(None)) - - def testTensorSignatureNone(self): - self.assertEqual(None, tensor_signature.create_signatures(None)) - - def testTensorSignatureCompatible(self): - placeholder_a = array_ops.placeholder( - name='test', shape=[None, 100], dtype=dtypes.int32) - placeholder_b = array_ops.placeholder( - name='another', shape=[256, 100], dtype=dtypes.int32) - placeholder_c = array_ops.placeholder( - name='mismatch', shape=[256, 100], dtype=dtypes.float32) - placeholder_d = array_ops.placeholder( - name='mismatch', shape=[128, 100], dtype=dtypes.int32) - signatures = tensor_signature.create_signatures(placeholder_a) - self.assertTrue(tensor_signature.tensors_compatible(None, None)) - self.assertFalse(tensor_signature.tensors_compatible(None, signatures)) - self.assertFalse(tensor_signature.tensors_compatible(placeholder_a, None)) - self.assertTrue( - tensor_signature.tensors_compatible(placeholder_a, signatures)) - self.assertTrue( - tensor_signature.tensors_compatible(placeholder_b, signatures)) - self.assertFalse( - tensor_signature.tensors_compatible(placeholder_c, signatures)) - self.assertTrue( - tensor_signature.tensors_compatible(placeholder_d, signatures)) - - inputs = {'a': placeholder_a} - signatures = tensor_signature.create_signatures(inputs) - self.assertTrue(tensor_signature.tensors_compatible(inputs, signatures)) - self.assertFalse( - tensor_signature.tensors_compatible(placeholder_a, signatures)) - self.assertFalse( - tensor_signature.tensors_compatible(placeholder_b, signatures)) - self.assertFalse( - tensor_signature.tensors_compatible({ - 'b': placeholder_b - }, signatures)) - self.assertTrue( - tensor_signature.tensors_compatible({ - 'a': placeholder_b, - 'c': placeholder_c - }, signatures)) - self.assertFalse( - tensor_signature.tensors_compatible({ - 'a': placeholder_c - }, signatures)) - - def testSparseTensorCompatible(self): - t = sparse_tensor.SparseTensor( - indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4]) - signatures = tensor_signature.create_signatures(t) - self.assertTrue(tensor_signature.tensors_compatible(t, signatures)) - - def testTensorSignaturePlaceholders(self): - placeholder_a = array_ops.placeholder( - name='test', shape=[None, 100], dtype=dtypes.int32) - signatures = tensor_signature.create_signatures(placeholder_a) - placeholder_out = tensor_signature.create_placeholders_from_signatures( - signatures) - self.assertEqual(placeholder_out.dtype, placeholder_a.dtype) - self.assertTrue(placeholder_out.get_shape().is_compatible_with( - placeholder_a.get_shape())) - self.assertTrue( - tensor_signature.tensors_compatible(placeholder_out, signatures)) - - inputs = {'a': placeholder_a} - signatures = tensor_signature.create_signatures(inputs) - placeholders_out = tensor_signature.create_placeholders_from_signatures( - signatures) - self.assertEqual(placeholders_out['a'].dtype, placeholder_a.dtype) - self.assertTrue(placeholders_out['a'].get_shape().is_compatible_with( - placeholder_a.get_shape())) - self.assertTrue( - tensor_signature.tensors_compatible(placeholders_out, signatures)) - - def testSparseTensorSignaturePlaceholders(self): - tensor = sparse_tensor.SparseTensor( - values=[1.0, 2.0], indices=[[0, 2], [0, 3]], dense_shape=[5, 5]) - signature = tensor_signature.create_signatures(tensor) - placeholder = tensor_signature.create_placeholders_from_signatures( - signature) - self.assertTrue(isinstance(placeholder, sparse_tensor.SparseTensor)) - self.assertEqual(placeholder.values.dtype, tensor.values.dtype) - - def testTensorSignatureExampleParserSingle(self): - examples = array_ops.placeholder( - name='example', shape=[None], dtype=dtypes.string) - placeholder_a = array_ops.placeholder( - name='test', shape=[None, 100], dtype=dtypes.int32) - signatures = tensor_signature.create_signatures(placeholder_a) - result = tensor_signature.create_example_parser_from_signatures(signatures, - examples) - self.assertTrue(tensor_signature.tensors_compatible(result, signatures)) - new_signatures = tensor_signature.create_signatures(result) - self.assertTrue(new_signatures.is_compatible_with(signatures)) - - def testTensorSignatureExampleParserDict(self): - examples = array_ops.placeholder( - name='example', shape=[None], dtype=dtypes.string) - placeholder_a = array_ops.placeholder( - name='test', shape=[None, 100], dtype=dtypes.int32) - placeholder_b = array_ops.placeholder( - name='bb', shape=[None, 100], dtype=dtypes.float64) - inputs = {'a': placeholder_a, 'b': placeholder_b} - signatures = tensor_signature.create_signatures(inputs) - result = tensor_signature.create_example_parser_from_signatures(signatures, - examples) - self.assertTrue(tensor_signature.tensors_compatible(result, signatures)) - new_signatures = tensor_signature.create_signatures(result) - self.assertTrue(new_signatures['a'].is_compatible_with(signatures['a'])) - self.assertTrue(new_signatures['b'].is_compatible_with(signatures['b'])) - - def testUnknownShape(self): - placeholder_unk = array_ops.placeholder( - name='unk', shape=None, dtype=dtypes.string) - placeholder_a = array_ops.placeholder( - name='a', shape=[None], dtype=dtypes.string) - placeholder_b = array_ops.placeholder( - name='b', shape=[128, 2], dtype=dtypes.string) - placeholder_c = array_ops.placeholder( - name='c', shape=[128, 2], dtype=dtypes.int32) - unk_signature = tensor_signature.create_signatures(placeholder_unk) - # Tensors of same dtype match unk shape signature. - self.assertTrue( - tensor_signature.tensors_compatible(placeholder_unk, unk_signature)) - self.assertTrue( - tensor_signature.tensors_compatible(placeholder_a, unk_signature)) - self.assertTrue( - tensor_signature.tensors_compatible(placeholder_b, unk_signature)) - self.assertFalse( - tensor_signature.tensors_compatible(placeholder_c, unk_signature)) - - string_signature = tensor_signature.create_signatures(placeholder_a) - int_signature = tensor_signature.create_signatures(placeholder_c) - # Unk shape Tensor matche signatures same dtype. - self.assertTrue( - tensor_signature.tensors_compatible(placeholder_unk, string_signature)) - self.assertFalse( - tensor_signature.tensors_compatible(placeholder_unk, int_signature)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/estimators/test_data.py b/tensorflow/contrib/learn/python/learn/estimators/test_data.py deleted file mode 100644 index e4b057b4f5a..00000000000 --- a/tensorflow/contrib/learn/python/learn/estimators/test_data.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Test data utilities (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes - - -def get_quantile_based_buckets(feature_values, num_buckets): - quantiles = np.percentile( - np.array(feature_values), - ([100 * (i + 1.) / (num_buckets + 1.) for i in range(num_buckets)])) - return list(quantiles) - - -def prepare_iris_data_for_logistic_regression(): - # Converts iris data to a logistic regression problem. - iris = base.load_iris() - ids = np.where((iris.target == 0) | (iris.target == 1)) - return base.Dataset(data=iris.data[ids], target=iris.target[ids]) - - -def iris_input_multiclass_fn(): - iris = base.load_iris() - return { - 'feature': constant_op.constant( - iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=(150, 1), dtype=dtypes.int32) - - -def iris_input_logistic_fn(): - iris = prepare_iris_data_for_logistic_regression() - return { - 'feature': constant_op.constant( - iris.data, dtype=dtypes.float32) - }, constant_op.constant( - iris.target, shape=(100, 1), dtype=dtypes.int32) diff --git a/tensorflow/contrib/learn/python/learn/evaluable.py b/tensorflow/contrib/learn/python/learn/evaluable.py deleted file mode 100644 index 5dedf548f73..00000000000 --- a/tensorflow/contrib/learn/python/learn/evaluable.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""`Evaluable` interface (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - - -@six.add_metaclass(abc.ABCMeta) -class Evaluable(object): - """Interface for objects that are evaluatable by, e.g., `Experiment`. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - @abc.abstractproperty - def model_dir(self): - """Returns a path in which the eval process will look for checkpoints.""" - raise NotImplementedError - - @abc.abstractmethod - def evaluate(self, - x=None, - y=None, - input_fn=None, - feed_fn=None, - batch_size=None, - steps=None, - metrics=None, - name=None, - checkpoint_path=None, - hooks=None): - """Evaluates given model with provided evaluation data. - - Stop conditions - we evaluate on the given input data until one of the - following: - - If `steps` is provided, and `steps` batches of size `batch_size` are - processed. - - If `input_fn` is provided, and it raises an end-of-input - exception (`OutOfRangeError` or `StopIteration`). - - If `x` is provided, and all items in `x` have been processed. - - The return value is a dict containing the metrics specified in `metrics`, as - well as an entry `global_step` which contains the value of the global step - for which this evaluation was performed. - - Args: - x: Matrix of shape [n_samples, n_features...] or dictionary of many - matrices - containing the input samples for fitting the model. Can be iterator that - returns - arrays of features or dictionary of array of features. If set, - `input_fn` must - be `None`. - y: Vector or matrix [n_samples] or [n_samples, n_outputs] containing the - label values (class labels in classification, real numbers in - regression) or dictionary of multiple vectors/matrices. Can be iterator - that returns array of targets or dictionary of array of targets. If set, - `input_fn` must be `None`. Note: For classification, label values must - be integers representing the class index (i.e. values from 0 to - n_classes-1). - input_fn: Input function returning a tuple of: - features - Dictionary of string feature name to `Tensor` or `Tensor`. - labels - `Tensor` or dictionary of `Tensor` with labels. - If input_fn is set, `x`, `y`, and `batch_size` must be `None`. If - `steps` is not provided, this should raise `OutOfRangeError` or - `StopIteration` after the desired amount of data (e.g., one epoch) has - been provided. See "Stop conditions" above for specifics. - feed_fn: Function creating a feed dict every time it is called. Called - once per iteration. Must be `None` if `input_fn` is provided. - batch_size: minibatch size to use on the input, defaults to first - dimension of `x`, if specified. Must be `None` if `input_fn` is - provided. - steps: Number of steps for which to evaluate model. If `None`, evaluate - until `x` is consumed or `input_fn` raises an end-of-input exception. - See "Stop conditions" above for specifics. - metrics: Dict of metrics to run. If None, the default metric functions - are used; if {}, no metrics are used. Otherwise, `metrics` should map - friendly names for the metric to a `MetricSpec` object defining which - model outputs to evaluate against which labels with which metric - function. - Metric ops should support streaming, e.g., returning `update_op` and - `value` tensors. For example, see the options defined in - `../../../metrics/python/ops/metrics_ops.py`. - name: Name of the evaluation if user needs to run multiple evaluations on - different data sets, such as on training data vs test data. - checkpoint_path: Path of a specific checkpoint to evaluate. If `None`, the - latest checkpoint in `model_dir` is used. - hooks: List of `SessionRunHook` subclass instances. Used for callbacks - inside the evaluation call. - - Returns: - Returns `dict` with evaluation results. - """ - raise NotImplementedError diff --git a/tensorflow/contrib/learn/python/learn/experiment.py b/tensorflow/contrib/learn/python/learn/experiment.py deleted file mode 100644 index 4e64efdd959..00000000000 --- a/tensorflow/contrib/learn/python/learn/experiment.py +++ /dev/null @@ -1,945 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Experiment class collecting information for a single training run (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import contextlib -import functools -import math -import os -import time - -from tensorflow.contrib.framework import deprecated -from tensorflow.contrib.framework.python.framework import experimental -from tensorflow.contrib.learn.python.learn import evaluable -from tensorflow.contrib.learn.python.learn import export_strategy -from tensorflow.contrib.learn.python.learn import monitors -from tensorflow.contrib.learn.python.learn import trainable -from tensorflow.contrib.learn.python.learn.estimators import run_config -from tensorflow.contrib.tpu.python.tpu import tpu_estimator -from tensorflow.python.estimator import estimator as core_estimator -from tensorflow.python.framework import ops -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.training import checkpoint_management -from tensorflow.python.training import server_lib -from tensorflow.python.util import compat -from tensorflow.python.util import function_utils - -__all__ = ["Experiment"] - - -def _get_standardized_predicate_fn(predicate_fn): - pred_fn_args = function_utils.fn_args(predicate_fn) - if "checkpoint_path" not in pred_fn_args: - # pylint: disable=unused-argument - def _pred_fn_wrapper(eval_results, checkpoint_path): - return predicate_fn(eval_results) - - return _pred_fn_wrapper - else: - return predicate_fn - - -class _EvalAndExportListener(basic_session_run_hooks.CheckpointSaverListener): - """Listener that evaluates and exports a model after creating a checkpoint. - - The `EvalAndExportListener` waits for the associated `CheckpointSaverHook` - to save a checkpoint. It then uses the provided `eval_fn` and `export_fn` to - first evaluate the model using the newly-created checkpoint, and then export - the model according to the `export_strategies` provided in the `Experiment`. - - This listener is experimental and may be changed or removed in the future. - """ - - def __init__(self, eval_fn, export_fn, model_dir): - """Initializes an `EvalAndExportListener`. - - Args: - eval_fn: function which evaluates the model with the following signature: - `(name, checkpoint_path) -> eval_result` - export_fn: function which exports the model according to a set of export - strategies. Has the following signature: - `(eval_result, checkpoint_path) -> export_results` - model_dir: directory which contains estimator parameters and checkpoints. - """ - self._eval_fn = eval_fn - self._export_fn = export_fn - self._model_dir = model_dir - self._latest_path = None - self._eval_result = None - self._export_results = None - - def after_save(self, session, global_step_value): - """Evaluates and exports the model after a checkpoint is created.""" - # Load and cache the path of the most recent checkpoint to avoid duplicate - # searches on GCS. - logging.info("Checking for checkpoint in %s", self._model_dir) - latest_path = checkpoint_management.latest_checkpoint(self._model_dir) - - if not latest_path: - logging.warning("Skipping evaluation and export since model has not been " - "saved yet.") - elif latest_path == self._latest_path: - logging.warning("Skipping evaluation due to same latest checkpoint %s.", - latest_path) - else: - self._latest_path = latest_path - self._eval_result = self._eval_fn( - name="intermediate_export", checkpoint_path=latest_path) - self._export_results = self._export_fn( - self._eval_result, checkpoint_path=latest_path) - - @property - def eval_result(self): - return self._eval_result - - @property - def export_results(self): - return self._export_results - - -class Experiment(object): - """Experiment is a class containing all information needed to train a model. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - After an experiment is created (by passing an Estimator and inputs for - training and evaluation), an Experiment instance knows how to invoke training - and eval loops in a sensible fashion for distributed training. - """ - - # TODO(ispir): remove delay_workers_by_global_step and make global step based - # waiting as only behavior. - @deprecated(None, "Please switch to tf.estimator.train_and_evaluate. You will" - " also have to convert to a tf.estimator.Estimator.") - def __init__(self, - estimator, - train_input_fn, - eval_input_fn, - eval_metrics=None, - train_steps=None, - eval_steps=100, - train_monitors=None, - eval_hooks=None, - local_eval_frequency=None, - eval_delay_secs=120, - continuous_eval_throttle_secs=60, - min_eval_frequency=None, - delay_workers_by_global_step=False, - export_strategies=None, - train_steps_per_iteration=None, - checkpoint_and_export=False, - saving_listeners=None, - check_interval_secs=5): - """Constructor for `Experiment`. - - Creates an Experiment instance. None of the functions passed to this - constructor are executed at construction time. They are stored and used - when a method is executed which requires it. - - Args: - estimator: Object implementing Estimator interface, which could be a - combination of `tf.contrib.learn.Trainable` and - `tf.contrib.learn.Evaluable` (deprecated), or - `tf.estimator.Estimator`. - train_input_fn: function, returns features and labels for training. - eval_input_fn: function, returns features and labels for evaluation. If - `eval_steps` is `None`, this should be configured only to produce for a - finite number of batches (generally, 1 epoch over the evaluation data). - eval_metrics: `dict` of string, metric function. If `None`, default set - is used. This should be `None` if the `estimator` is - `tf.estimator.Estimator`. If metrics are provided they will be - *appended* to the default set. - train_steps: Perform this many steps of training. `None`, the default, - means train forever. - eval_steps: `evaluate` runs until input is exhausted (or another exception - is raised), or for `eval_steps` steps, if specified. - train_monitors: A list of monitors to pass to the `Estimator`'s `fit` - function. - eval_hooks: A list of `SessionRunHook` hooks to pass to the - `Estimator`'s `evaluate` function. - local_eval_frequency: (applies only to local_run) Frequency of running - eval in steps. If `None`, runs evaluation only at the end of training. - eval_delay_secs: Start evaluating after waiting for this many seconds. - continuous_eval_throttle_secs: Do not re-evaluate unless the last - evaluation was started at least this many seconds ago for - continuous_eval(). - min_eval_frequency: (applies only to train_and_evaluate). the minimum - number of steps between evaluations. Of course, evaluation does not - occur if no new snapshot is available, hence, this is the minimum. - If 0, the evaluation will only happen after training. - If None, defaults to 1. To avoid checking for new checkpoints too - frequent, the interval is further limited to be at least - check_interval_secs between checks. - delay_workers_by_global_step: if `True` delays training workers - based on global step instead of time. - export_strategies: Iterable of `ExportStrategy`s, or a single one, or - `None`. - train_steps_per_iteration: (applies only to continuous_train_and_eval). - Perform this many (integer) number of train steps for each - training-evaluation iteration. With a small value, the model will be - evaluated more frequently with more checkpoints saved. If `None`, will - use a default value (which is smaller than `train_steps` if provided). - checkpoint_and_export: (applies only to train_and_evaluate). If `True`, - performs intermediate model checkpoints and exports during the training - process, rather than only once model training is complete. This - parameter is experimental and may be changed or removed in the future. - Setting this parameter leads to the following: the value of - `min_eval_frequency` will be ignored, and the number of steps between - evaluations and exports will instead be determined by the Estimator - configuration parameters `save_checkpoints_secs` and - `save_checkpoints_steps`. Also, this parameter leads to the creation of - a default `CheckpointSaverHook` instead of a `ValidationMonitor`, so the - provided `train_monitors` will need to be adjusted accordingly. - saving_listeners: list of `CheckpointSaverListener` objects. Used by - tf.estimator.Estimator for callbacks that run immediately before or - after checkpoint savings. - check_interval_secs: - Minimum time between subsequent checks for a new checkpoint. This - mostly applies if both min_eval_frequency and the time spent per - training step is low. - Raises: - ValueError: if `estimator` does not implement Estimator interface, - or if export_strategies has the wrong type. - """ - if isinstance(estimator, core_estimator.Estimator): - self._core_estimator_used = True - if eval_metrics is not None: - raise ValueError( - "`eval_metrics` must be `None` with `tf.estimator.Estimator`. " - "Use `eval_metric_ops` in `tf.estimator.EstimatorSpec` instead.") - else: - self._core_estimator_used = False - if not isinstance(estimator, evaluable.Evaluable): - raise ValueError( - "`estimator` must implement `tf.contrib.learn.Evaluable` " - "or `tf.estimator.Estimator`.") - if not isinstance(estimator, trainable.Trainable): - raise ValueError( - "`estimator` must implement `tf.contrib.learn.Trainable`" - "or `tf.estimator.`Estimator`.") - if saving_listeners is not None: - raise ValueError("`saving_listeners` must be `None` with " - "`tf.contrib.learn.Estimator`.") - - if isinstance(estimator, tpu_estimator.TPUEstimator): - logging.warn( - "`Experiment` class cannot work with `tf.contrib.tpu.TPUEstimator`. " - "Please call `TPUEstimator` train/evaluate directly. \n" - "Details: `Experiment` class is designed for between-graph " - "distributed training, while `TPUEstimator` is working in in-graph " - "distributed mode. Use with care.") - - super(Experiment, self).__init__() - # Immutable fields. - self._estimator = estimator - self._train_input_fn = train_input_fn - self._eval_input_fn = eval_input_fn - self._eval_metrics = eval_metrics - self._train_steps = train_steps - self._eval_steps = eval_steps - self._local_eval_frequency = local_eval_frequency - self._eval_delay_secs = eval_delay_secs - self._continuous_eval_throttle_secs = continuous_eval_throttle_secs - self._checkpoint_and_export = checkpoint_and_export - self._saving_listeners = saving_listeners - self._min_eval_frequency = min_eval_frequency if ( - min_eval_frequency is not None) else 1 - self._check_interval_secs = check_interval_secs - self._delay_workers_by_global_step = delay_workers_by_global_step - self._train_monitors = train_monitors[:] if train_monitors else [] - self._eval_hooks = eval_hooks[:] if eval_hooks else [] - self._set_export_strategies(export_strategies) - - self._train_steps_per_iteration = train_steps_per_iteration - if (self._train_steps_per_iteration is not None and - not isinstance(self._train_steps_per_iteration, int)): - raise ValueError("`train_steps_per_iteration` must be an integer.") - - @property - def estimator(self): - return self._estimator - - @property - def eval_metrics(self): - return self._eval_metrics - - @property - def train_steps(self): - return self._train_steps - - @property - def eval_steps(self): - return self._eval_steps - - def _set_export_strategies(self, values): # pylint: disable=missing-docstring - export_strategies = [] - if values: - if isinstance(values, export_strategy.ExportStrategy): - export_strategies.append(values) - else: - for value in values: - if not isinstance(value, export_strategy.ExportStrategy): - raise ValueError("`export_strategies` must be an ExportStrategy," - " an iterable of ExportStrategy, or `None`," - " found %s." % value) - export_strategies.append(value) - self._export_strategies = tuple(export_strategies) - - def extend_train_hooks(self, additional_hooks): - """Extends the hooks for training.""" - self._train_monitors.extend(additional_hooks) - - def reset_export_strategies(self, new_export_strategies=None): - """Resets the export strategies with the `new_export_strategies`. - - Args: - new_export_strategies: A new list of `ExportStrategy`s, or a single one, - or None. - - Returns: - The old export strategies. - """ - old_export_strategies = self._export_strategies - self._set_export_strategies(new_export_strategies) - return old_export_strategies - - def train(self, delay_secs=None): - """Fit the estimator using the training data. - - Train the estimator for `self._train_steps` steps, after waiting for - `delay_secs` seconds. If `self._train_steps` is `None`, train forever. - - Args: - delay_secs: Start training after this many seconds. - - Returns: - The trained estimator. - """ - start = time.time() - - # Start the server, if needed. It's important to start the server before - # we (optionally) sleep for the case where no device_filters are set. - # Otherwise, the servers will wait to connect to each other before starting - # to train. We might as well start as soon as we can. - config = self._estimator.config - if isinstance(config, run_config.RunConfig): - if (config.cluster_spec and config.master and - config.environment == run_config.Environment.LOCAL): - logging.warn("ClusterSpec and master are provided, but environment is " - "set to 'local'. Set environment to 'cloud' if you intend " - "to use the distributed runtime.") - if (config.environment != run_config.Environment.LOCAL and - config.environment != run_config.Environment.GOOGLE and - config.cluster_spec and config.master): - self._start_server() - elif config.cluster_spec and config.master: - raise ValueError( - "For distributed runtime, Experiment class only works with " - "tf.contrib.learn.RunConfig for now, but provided {}".format( - type(config))) - - extra_hooks = [] - if delay_secs is None: - task_id = self._estimator.config.task_id or 0 - if self._delay_workers_by_global_step: - # Wait 5500 global steps for the second worker. Each worker waits more - # then previous one but with a diminishing number of steps. - extra_hooks.append( - basic_session_run_hooks.GlobalStepWaiterHook( - int(8000.0 * math.log(task_id + 1)))) - delay_secs = 0 - else: - # Wait 5 secs more for each new worker up to 60 secs. - delay_secs = min(60, task_id * 5) - - if delay_secs > 0: - elapsed_secs = time.time() - start - remaining = delay_secs - elapsed_secs - logging.info("Waiting %d secs before starting training.", remaining) - time.sleep(delay_secs) - - return self._call_train( - input_fn=self._train_input_fn, - max_steps=self._train_steps, - hooks=self._train_monitors + extra_hooks, - saving_listeners=self._saving_listeners) - - def evaluate(self, delay_secs=None, name=None): - """Evaluate on the evaluation data. - - Runs evaluation on the evaluation data and returns the result. Runs for - `self._eval_steps` steps, or if it's `None`, then run until input is - exhausted or another exception is raised. Start the evaluation after - `delay_secs` seconds, or if it's `None`, defaults to using - `self._eval_delay_secs` seconds. - - Args: - delay_secs: Start evaluating after this many seconds. If `None`, defaults - to using `self._eval_delays_secs`. - name: Gives the name to the evauation for the case multiple evaluation is - run for the same experiment. - - Returns: - The result of the `evaluate` call to the `Estimator`. - """ - if delay_secs is None: - delay_secs = self._eval_delay_secs - - if delay_secs: - logging.info("Waiting %d secs before starting eval.", delay_secs) - time.sleep(delay_secs) - - return self._call_evaluate( - input_fn=self._eval_input_fn, - steps=self._eval_steps, - metrics=self._eval_metrics, - name=(name or "one_pass"), - hooks=self._eval_hooks) - - @deprecated( - "2016-10-23", - "local_run will be renamed to train_and_evaluate and the new default " - "behavior will be to run evaluation every time there is a new " - "checkpoint.") - def local_run(self): - with _new_attr_context(self, "_min_eval_frequency"): - self._min_eval_frequency = self._local_eval_frequency - return self.train_and_evaluate() - - # TODO(xiejw): Allow continuous_eval_predicate_fn to be passed via constructor - # once stopping all jobs is implemented. - def _continuous_eval(self, - input_fn, - name, - delay_secs, - throttle_delay_secs, - evaluate_checkpoint_only_once=True, - continuous_eval_predicate_fn=None, - export=True): - """Run continuous eval. - - Runs infinite eval on the evaluation data set. This function starts - evaluating after `delay_secs` seconds and then runs no more than one - evaluation (with `self._eval_steps` steps each time) per - `throttle_delay_secs`. If `train_steps` is not None, will return after - global_step reaches `train_steps`. - - Args: - input_fn: The input to use for this eval. - name: A string appended to the folder name of evaluation results. - delay_secs: Start evaluating after this many seconds. If None, defaults to - self._eval_delay_secs. - throttle_delay_secs: Do not re-evaluate unless the last evaluation was - started at least this many seconds ago. If None, defaults to - self._continuous_eval_throttle_secs. - evaluate_checkpoint_only_once: Whether to skip evaluation of checkpoints - that have already been evaluated. Default is `True`. - continuous_eval_predicate_fn: A predicate function determining whether to - continue eval after each iteration. A `predicate_fn` has one of the - following signatures: - * (eval_results) -> boolean - * (eval_results, checkpoint_path) -> boolean - Where `eval_results` is the dictionary of metric evaluations and - checkpoint_path is the path to the checkpoint containing the parameters - on which that evaluation was based. - At the beginning of evaluation, the passed `eval_results` will be None - so it's expected that the predicate function handles that gracefully. - Continuous eval behavior under different conditions: - * When `predicate_fn` is specified: - + if `train_steps` is None, run until `predicate_fn` returns False. - + if `train_steps` is specified, run until either global step - reaches `train_steps` or `predicate_fn` returns False. - * When `predicate_fn` is not specified: - + if `train_steps` is None, run in an infinite loop. - + if `train_steps` is specified, run until global step reaches - `train_steps`. - export: Whether to export from this step. Default is 'True'. - - Raises: - ValueError: if `continuous_eval_predicate_fn` is neither None nor - callable. - """ - if continuous_eval_predicate_fn is not None: - if not callable(continuous_eval_predicate_fn): - raise ValueError( - "`continuous_eval_predicate_fn` must be a callable, or None.") - predicate_fn = _get_standardized_predicate_fn( - continuous_eval_predicate_fn) - else: - predicate_fn = None - - if delay_secs is None: - delay_secs = self._eval_delay_secs - if throttle_delay_secs is None: - throttle_delay_secs = self._continuous_eval_throttle_secs - - if delay_secs: - logging.info("Waiting %f secs before starting eval.", delay_secs) - time.sleep(delay_secs) - - previous_path = None - eval_result = None - last_warning_time = 0 - while (not predicate_fn or predicate_fn( - eval_result, checkpoint_path=previous_path)): - # Exit if we have already reached number of steps to train. - if self._has_training_stopped(eval_result): - logging.info("Exiting continuous eval, global_step=%s >= " - "train_step=%s", eval_result[ops.GraphKeys.GLOBAL_STEP], - self._train_steps) - return - - start = time.time() - - error_msg = None - latest_path = checkpoint_management.latest_checkpoint( - self._estimator.model_dir) - if not latest_path: - error_msg = ("Estimator is not fitted yet. " - "Will start an evaluation when a checkpoint is ready.") - elif evaluate_checkpoint_only_once and latest_path == previous_path: - error_msg = "No new checkpoint ready for evaluation." - - if error_msg: - # Print warning message every 10 mins. - eval_result = {} - if time.time() - last_warning_time > 600: - logging.warning(error_msg) - last_warning_time = time.time() - else: - eval_result = self._call_evaluate( - input_fn=input_fn, - steps=self._eval_steps, - metrics=self._eval_metrics, - name=name, - checkpoint_path=latest_path, - hooks=self._eval_hooks) - # Ensure eval result is not None for next round of evaluation. - if not eval_result: - eval_result = {} - - if export: - self._maybe_export(eval_result, checkpoint_path=latest_path) - - # Clear warning timer and update last evaluated checkpoint - last_warning_time = 0 - previous_path = latest_path - - duration = time.time() - start - if duration < throttle_delay_secs: - difference = throttle_delay_secs - duration - logging.info("Waiting %f secs before starting next eval run.", - difference) - time.sleep(difference) - - def _has_training_stopped(self, eval_result): - """Determines whether the training has stopped.""" - if not eval_result: - return False - - global_step = eval_result.get(ops.GraphKeys.GLOBAL_STEP) - return global_step and self._train_steps and (global_step >= - self._train_steps) - - def continuous_eval(self, - delay_secs=None, - throttle_delay_secs=None, - evaluate_checkpoint_only_once=True, - continuous_eval_predicate_fn=None, - name="continuous"): - self._continuous_eval( - self._eval_input_fn, - name=name, - delay_secs=delay_secs, - throttle_delay_secs=throttle_delay_secs, - evaluate_checkpoint_only_once=evaluate_checkpoint_only_once, - continuous_eval_predicate_fn=continuous_eval_predicate_fn) - - def continuous_eval_on_train_data(self, - delay_secs=None, - throttle_delay_secs=None, - continuous_eval_predicate_fn=None, - name="continuous_on_train_data"): - self._continuous_eval( - self._train_input_fn, - name=name, - delay_secs=delay_secs, - throttle_delay_secs=throttle_delay_secs, - continuous_eval_predicate_fn=continuous_eval_predicate_fn, - export=False) - - def train_and_evaluate(self): - """Interleaves training and evaluation. - - The frequency of evaluation is controlled by the constructor arg - `min_eval_frequency`. When this parameter is 0, evaluation happens - only after training has completed. Note that evaluation cannot happen - more frequently than checkpoints are taken. If no new snapshots are - available when evaluation is supposed to occur, then evaluation doesn't - happen for another `min_eval_frequency` steps (assuming a checkpoint is - available at that point). Thus, settings `min_eval_frequency` to 1 means - that the model will be evaluated everytime there is a new checkpoint. - - This is particular useful for a "Master" task in the cloud, whose - responsibility it is to take checkpoints, evaluate those checkpoints, - and write out summaries. Participating in training as the supervisor - allows such a task to accomplish the first and last items, while - performing evaluation allows for the second. - - Returns: - The result of the `evaluate` call to the `Estimator` as well as the - export results using the specified `ExportStrategy`. - """ - # The directory to which evaluation summaries are written are determined - # by adding a suffix to 'eval'; that suffix is the 'name' parameter to - # the various evaluate(...) methods. By setting it to None, we force - # the directory name to simply be 'eval'. - eval_dir_suffix = None - - # We set every_n_steps to 1, but evaluation only occurs when a new - # snapshot is available. If, by the time we finish evaluation - # there is a new snapshot, then we just evaluate again. Otherwise, - # we keep training until one becomes available. - with _new_attr_context(self, "_train_monitors"): - self._train_monitors = self._train_monitors or [] - config = self._estimator.config - intermediate_export = self._checkpoint_and_export and ( - config.save_checkpoints_secs or config.save_checkpoints_steps) - if intermediate_export: - # Create a partially specified evaluate function with the desired - # arguments. This will be executed by the _EvalAndExportListener, - # which will specify the latest checkpoint path. - eval_fn = functools.partial( - self._call_evaluate, - input_fn=self._eval_input_fn, - steps=self._eval_steps, - metrics=self._eval_metrics, - hooks=self._eval_hooks) - - export_listener = _EvalAndExportListener( - eval_fn=eval_fn, - export_fn=self._maybe_export, - model_dir=self._estimator.model_dir) - - saver_hook = basic_session_run_hooks.CheckpointSaverHook( - checkpoint_dir=self._estimator.model_dir, - save_secs=config.save_checkpoints_secs, - save_steps=config.save_checkpoints_steps, - listeners=[export_listener]) - self._train_monitors += [saver_hook] - else: - if self._min_eval_frequency: - # Using low min_eval_frequency (default is 1) on a non-cached file - # system requires a lot of overhead to read the checkpoint state file. - # This is particular bad on GCS and CNS. See also b/36498507 for - # context. `check_interval_secs = 5` avoids polling a remote - # fileystem too often. - - self._train_monitors += [ - monitors.ValidationMonitor( - input_fn=self._eval_input_fn, - eval_steps=self._eval_steps, - metrics=self._eval_metrics, - every_n_steps=self._min_eval_frequency, - check_interval_secs=self._check_interval_secs, - name=eval_dir_suffix, - hooks=self._eval_hooks) - ] - self.train(delay_secs=0) - - # If the checkpoint_and_export flag and appropriate estimator configuration - # parameters are set, then model evaluations and exports are done during the - # training process. In particular, this will always occur at the end of - # training, so we return the most recent results to avoid performing a - # duplicate evaluation and model export. - if intermediate_export: - return export_listener.eval_result, export_listener.export_results - else: - eval_result = self._call_evaluate( - input_fn=self._eval_input_fn, - steps=self._eval_steps, - metrics=self._eval_metrics, - name=eval_dir_suffix, - hooks=self._eval_hooks) - export_results = self._maybe_export(eval_result) - return eval_result, export_results - - @experimental - def continuous_train_and_eval(self, continuous_eval_predicate_fn=None): - """Interleaves training and evaluation. - - The frequency of evaluation is controlled by the `train_steps_per_iteration` - (via constructor). The model will be first trained for - `train_steps_per_iteration`, and then be evaluated in turns. - - This method is intended for single machine usage. - - This differs from `train_and_evaluate` as follows: - - 1. The procedure will have train and evaluation in turns. The model - will be trained for a number of steps (usually smaller than `train_steps` - if provided) and then be evaluated. `train_and_evaluate` will train the - model for `train_steps` (no small training iterations). - - 2. Due to the different approach this schedule takes, it leads to two - differences in resource control. First, the resources (e.g., memory) used - by training will be released before evaluation (`train_and_evaluate` takes - double resources). Second, more checkpoints will be saved as a checkpoint - is generated at the end of each training iteration. - - 3. As the estimator.train starts from scratch (new graph, new states for - input, etc) at each iteration, it is recommended to have the - `train_steps_per_iteration` larger. It is also recommended to shuffle your - input. - - Args: - continuous_eval_predicate_fn: A predicate function determining whether to - continue eval after each iteration. A `predicate_fn` has one of the - following signatures: - * (eval_results) -> boolean - * (eval_results, checkpoint_path) -> boolean - Where `eval_results` is the dictionary of metric evaluations and - checkpoint_path is the path to the checkpoint containing the parameters - on which that evaluation was based. - At the beginning of evaluation, the passed `eval_results` and - `checkpoint_path` will be None so it's expected that the predicate - function handles that gracefully. - When `predicate_fn` is not specified, continuous eval will run in an - infinite loop (if `train_steps` is None). or exit once global step - reaches `train_steps`. - - Returns: - A tuple of the result of the `evaluate` call to the `Estimator` and the - export results using the specified `ExportStrategy`. - - Raises: - ValueError: if `continuous_eval_predicate_fn` is neither None nor - callable. - """ - - if continuous_eval_predicate_fn is not None: - if not callable(continuous_eval_predicate_fn): - raise ValueError( - "`continuous_eval_predicate_fn` must be a callable, or None.") - predicate_fn = _get_standardized_predicate_fn( - continuous_eval_predicate_fn) - else: - predicate_fn = None - - export_results = None - latest_checkpoint = None - eval_result = None - - # Set the default value for train_steps_per_iteration, which will be - # overridden by other settings. - train_steps_per_iteration = 1000 - if self._train_steps_per_iteration is not None: - train_steps_per_iteration = self._train_steps_per_iteration - elif self._train_steps is not None: - train_steps_per_iteration = int(self._train_steps / 10) - - while (not predicate_fn or predicate_fn( - eval_result, checkpoint_path=latest_checkpoint - if eval_result else None)): - - if self._has_training_stopped(eval_result): - # Exits once max steps of training is satisfied. - logging.info("Stop training model as max steps reached") - break - - logging.info("Training model for %s steps", train_steps_per_iteration) - self._call_train( - input_fn=self._train_input_fn, - steps=train_steps_per_iteration, - hooks=self._train_monitors, - saving_listeners=self._saving_listeners) - - logging.info("Evaluating model now.") - latest_checkpoint = checkpoint_management.latest_checkpoint( - self._estimator.model_dir) - eval_result = self._call_evaluate( - input_fn=self._eval_input_fn, - steps=self._eval_steps, - metrics=self._eval_metrics, - name="one_pass", - checkpoint_path=latest_checkpoint, - hooks=self._eval_hooks) - export_results = self._maybe_export(eval_result) - - return eval_result, export_results - - def _maybe_export(self, eval_result, checkpoint_path=None): - """Export the Estimator using export_fn, if defined.""" - export_dir_base = os.path.join( - compat.as_bytes(self._estimator.model_dir), compat.as_bytes("export")) - - export_results = [] - for strategy in self._export_strategies: - export_results.append( - strategy.export( - self._estimator, - os.path.join( - compat.as_bytes(export_dir_base), - compat.as_bytes(strategy.name)), - checkpoint_path=checkpoint_path, - eval_result=eval_result)) - - return export_results - - def run_std_server(self): - """Starts a TensorFlow server and joins the serving thread. - - Typically used for parameter servers. - - Raises: - ValueError: if not enough information is available in the estimator's - config to create a server. - """ - self._start_server().join() - - def test(self): - """Tests training, evaluating and exporting the estimator for a single step. - - Returns: - The result of the `evaluate` call to the `Estimator`. - """ - self._call_train( - input_fn=self._train_input_fn, - steps=1, - hooks=self._train_monitors, - saving_listeners=self._saving_listeners) - - eval_result = self._call_evaluate( - input_fn=self._eval_input_fn, - steps=1, - metrics=self._eval_metrics, - name="one_pass") - _ = self._maybe_export(eval_result) - - return eval_result - - def _start_server(self): - """Creates, starts, and returns a server_lib.Server.""" - config = self._estimator.config - if (not config.cluster_spec or not config.task_type or not config.master or - config.task_id is None): - raise ValueError("Could not start server; be sure to specify " - "cluster_spec, task_type, master, and task in " - "RunConfig or set the TF_CONFIG environment variable.") - server = server_lib.Server( - config.cluster_spec, - job_name=config.task_type, - task_index=config.task_id, - config=config.tf_config, - start=False) - server.start() - return server - - def _call_train( - self, - _sentinel=None, # pylint: disable=invalid-name, - input_fn=None, - steps=None, - hooks=None, - max_steps=None, - saving_listeners=None): - if _sentinel is not None: - raise ValueError("_call_train should be called with keyword args only") - - # Estimator in core cannot work with monitors. We need to convert them - # to hooks. For Estimator in contrib, it is converted internally. So, it is - # safe to convert for both cases. - hooks = monitors.replace_monitors_with_hooks(hooks, self._estimator) - if self._core_estimator_used: - return self._estimator.train( - input_fn=input_fn, - steps=steps, - max_steps=max_steps, - hooks=hooks, - saving_listeners=saving_listeners) - else: - return self._estimator.fit( - input_fn=input_fn, steps=steps, max_steps=max_steps, monitors=hooks) - - def _call_evaluate( - self, - _sentinel=None, # pylint: disable=invalid-name, - input_fn=None, - steps=None, - metrics=None, - name=None, - checkpoint_path=None, - hooks=None): - if _sentinel is not None: - raise ValueError("_call_evaluate should be called with keyword args only") - - if self._core_estimator_used: - if metrics is not None: - raise ValueError( - "`eval_metrics` must be `None` with `tf.estimator.Estimator`") - return self._estimator.evaluate( - input_fn=input_fn, - steps=steps, - name=name, - checkpoint_path=checkpoint_path, - hooks=hooks) - else: - return self._estimator.evaluate( - input_fn=input_fn, - steps=steps, - metrics=metrics, - name=name, - checkpoint_path=checkpoint_path, - hooks=hooks) - - -@contextlib.contextmanager -def _new_attr_context(obj, attr): - """Creates a new context in which an object's attribute can be changed. - - This creates a context in which an object's attribute can be changed. - Once the context is exited, the attribute reverts to its original value. - - Args: - obj: An object whose attribute to restore at the end of the context. - attr: An attribute to remember and restore at the end of the context. - - Yields: - Context. - - Example: - my_obj.x = 1 - with _new_attr_context(my_obj, "x"): - my_obj.x = 2 - print(my_obj.x) - print(my_obj.x) - """ - saved = getattr(obj, attr) - try: - yield - finally: - setattr(obj, attr, saved) diff --git a/tensorflow/contrib/learn/python/learn/experiment_test.py b/tensorflow/contrib/learn/python/learn/experiment_test.py deleted file mode 100644 index fb16c94c296..00000000000 --- a/tensorflow/contrib/learn/python/learn/experiment_test.py +++ /dev/null @@ -1,962 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for TaskRunner and Experiment class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import json -import os -import tempfile -import time - -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.learn.python.learn import estimator as estimator_lib -from tensorflow.contrib.learn.python.learn import evaluable -from tensorflow.contrib.learn.python.learn import experiment -from tensorflow.contrib.learn.python.learn import run_config -from tensorflow.contrib.learn.python.learn import trainable -from tensorflow.contrib.learn.python.learn.estimators import dnn -from tensorflow.contrib.learn.python.learn.estimators import run_config as run_config_lib -from tensorflow.contrib.learn.python.learn.estimators import test_data -from tensorflow.contrib.learn.python.learn.utils import saved_model_export_utils -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session -from tensorflow.python.estimator import estimator as core_estimator -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging -from tensorflow.python.training import saver -from tensorflow.python.training import server_lib -from tensorflow.python.training import session_run_hook -from tensorflow.python.util import compat -from tensorflow.python.util import tf_inspect - - -class SheepCounter(object): - """To be patched in for the time module, replacing sleep() and time().""" - - def __init__(self): - self._total_time = 0 - self._sleeptimes = [] - self._time_calls = 0 - - def sleep(self, t): - self._total_time += t - self._sleeptimes += [t] - - def time(self): - self._time_calls += 1 - return self._total_time - - @property - def sleep_times(self): - return self._sleeptimes - - @property - def time_calls(self): - return self._time_calls - - -class TestBaseEstimator(object): - - def __init__(self, config, max_evals, eval_dict): - self.eval_count = 0 - self.fit_count = 0 - self._max_evals = max_evals - self.export_count = 0 - self.monitors = [] - self.eval_hooks = [] - self._config = config or run_config.RunConfig() - self._model_dir = tempfile.mkdtemp() - self._eval_dict = eval_dict - - @property - def model_dir(self): - return self._model_dir - - @property - def config(self): - return self._config - - def evaluate(self, **kwargs): - tf_logging.info('evaluate called with args: %s' % kwargs) - if 'hooks' in kwargs: - self.eval_hooks = kwargs['hooks'] - self.eval_count += 1 - if self.eval_count > self._max_evals: - tf_logging.info('Ran %d evals. Done.' % self.eval_count) - raise StopIteration() - return self._eval_dict - - def fake_checkpoint(self): - save_path = os.path.join(self.model_dir, 'model.ckpt') - with session.Session() as sess: - var = variables.Variable(1.0, name='var0') - save = saver.Saver({var.op.name: var}) - var.initializer.run() - save.save(sess, save_path, global_step=0) - - def train(self, **kwargs): - self.fake_checkpoint() - tf_logging.info('fit called with args: %s' % kwargs) - self.fit_count += 1 - - return [(key, kwargs[key]) for key in sorted(kwargs.keys())] - - def export_savedmodel(self, export_dir_base, serving_input_fn, **kwargs): - tf_logging.info('export_savedmodel called with args: %s, %s, %s' % - (export_dir_base, serving_input_fn, kwargs)) - self.export_count += 1 - return os.path.join( - compat.as_bytes(export_dir_base), compat.as_bytes('bogus_timestamp')) - - -def _check_method_supports_args(method, kwargs): - """Checks that the given method supports the given args.""" - supported_args = tuple(tf_inspect.getargspec(method).args) - for kwarg in kwargs: - if kwarg not in supported_args: - raise ValueError( - 'Argument `{}` is not supported in method {}.'.format(kwarg, method)) - - -class TestEstimator( - TestBaseEstimator, evaluable.Evaluable, trainable.Trainable): - - def __init__(self, config=None, max_evals=5, eval_dict=None): - super(TestEstimator, self).__init__(config, max_evals, eval_dict) - tf_logging.info('Create Estimator') - - def evaluate(self, **kwargs): - _check_method_supports_args(evaluable.Evaluable.evaluate, kwargs) - return super(TestEstimator, self).evaluate(**kwargs) - - def fit(self, **kwargs): - _check_method_supports_args(trainable.Trainable.fit, kwargs) - if 'monitors' in kwargs: - self.monitors = kwargs['monitors'] - return super(TestEstimator, self).train(**kwargs) - - def train(self, **kwargs): - raise ValueError('`train` is not defined in Estimator.') - - def export_savedmodel( - self, export_dir_base, serving_input_fn, **kwargs): - _check_method_supports_args( - estimator_lib.Estimator.export_savedmodel, kwargs) - return super(TestEstimator, self).export_savedmodel( - export_dir_base, serving_input_fn, **kwargs) - - -class TestCoreEstimator(TestBaseEstimator, core_estimator.Estimator): - - def __init__(self, config=None, max_evals=5, eval_dict=None): - super(TestCoreEstimator, self).__init__(config, max_evals, eval_dict) - tf_logging.info('Create Core Estimator') - - def evaluate(self, **kwargs): - _check_method_supports_args(core_estimator.Estimator.evaluate, kwargs) - return super(TestCoreEstimator, self).evaluate(**kwargs) - - def train(self, **kwargs): - _check_method_supports_args(core_estimator.Estimator.train, kwargs) - if 'hooks' in kwargs: - self.monitors = kwargs['hooks'] - return super(TestCoreEstimator, self).train(**kwargs) - - def export_savedmodel( - self, export_dir_base, serving_input_receiver_fn, **kwargs): - _check_method_supports_args( - core_estimator.Estimator.export_savedmodel, kwargs) - return super(TestCoreEstimator, self).export_savedmodel( - export_dir_base, serving_input_receiver_fn, **kwargs) - - -class _NoopHook(session_run_hook.SessionRunHook): - pass - - -class ExperimentTest(test.TestCase): - - def _cluster_spec(self): - return { - run_config_lib.TaskType.PS: ['host1:2222', 'host2:2222'], - run_config_lib.TaskType.WORKER: - ['host3:2222', 'host4:2222', 'host5:2222'] - } - - def _estimators_for_tests(self, config=None, eval_dict=None): - return [TestEstimator(config=config, eval_dict=eval_dict), - TestCoreEstimator(config=config, eval_dict=eval_dict)] - - def test_eval_metrcis_for_core_estimator(self): - est = TestCoreEstimator() - with self.assertRaisesRegexp( - ValueError, '`eval_metrics` must be `None`'): - experiment.Experiment( - est, - train_input_fn='train_input', - train_steps='train_steps', - eval_input_fn='eval_input', - eval_metrics='eval_metrics') - - def test_default_output_alternative_key_core_estimator(self): - est = TestCoreEstimator() - export_strategy = saved_model_export_utils.make_export_strategy( - est, - default_output_alternative_key='export_key', - exports_to_keep=None) - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - train_steps=100, - eval_steps=100, - export_strategies=export_strategy) - with self.assertRaisesRegexp( - ValueError, 'default_output_alternative_key is not supported'): - ex.train_and_evaluate() - - def test_train(self): - for est in self._estimators_for_tests(): - if isinstance(est, core_estimator.Estimator): - eval_metrics = None - saving_listeners = 'saving_listeners' - else: - eval_metrics = 'eval_metrics' - saving_listeners = None - ex = experiment.Experiment( - est, - train_input_fn='train_input', - train_steps='train_steps', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - saving_listeners=saving_listeners) - fit_args = ex.train(delay_secs=0) - self.assertEqual(1, est.fit_count) - self.assertIn(('max_steps', 'train_steps'), fit_args) - self.assertEqual(0, est.eval_count) - - def test_train_delay(self): - for est in self._estimators_for_tests(): - ex = experiment.Experiment( - est, train_input_fn='train_input', eval_input_fn='eval_input') - for delay in [0, 1, 3]: - sheep = SheepCounter() - with test.mock.patch.object(time, 'time', sheep.time): - with test.mock.patch.object(time, 'sleep', sheep.sleep): - ex.train(delay_secs=delay) - self.assertAlmostEqual(delay, sheep.time(), delta=1e-4) - - def test_train_default_delay(self): - for task_id in [0, 1, 3]: - tf_config = {'task': {'index': task_id}} - with test.mock.patch.dict('os.environ', - {'TF_CONFIG': json.dumps(tf_config)}): - config = run_config.RunConfig() - for est in self._estimators_for_tests(config): - ex = experiment.Experiment( - est, train_input_fn='train_input', eval_input_fn='eval_input') - - sheep = SheepCounter() - with test.mock.patch.object(time, 'time', sheep.time): - with test.mock.patch.object(time, 'sleep', sheep.sleep): - ex.train() - self.assertAlmostEqual(task_id * 5, sheep.time(), delta=1e-4) - - @test.mock.patch.object(server_lib, 'Server') - def test_train_starts_server(self, mock_server): - # Arrange. - tf_config = { - 'cluster': self._cluster_spec(), - 'environment': run_config_lib.Environment.CLOUD, - 'task': { - 'type': run_config_lib.TaskType.WORKER, - 'index': 1 - } - } - with test.mock.patch.dict('os.environ', - {'TF_CONFIG': json.dumps(tf_config)}): - config = run_config_lib.RunConfig( - master='host4:2222', num_cores=15, gpu_memory_fraction=0.314) - - for est in self._estimators_for_tests(config): - ex = experiment.Experiment( - est, train_input_fn='train_input', eval_input_fn='eval_input') - - # Act. - # We want to make sure we discount the time it takes to start the server - # in our accounting of the delay, so we set a small delay here. - sheep = SheepCounter() - with test.mock.patch.object(time, 'time', sheep.time): - with test.mock.patch.object(time, 'sleep', sheep.sleep): - ex.train(delay_secs=1) - # Ensure that the delay takes into account the time to start server. - self.assertAlmostEqual(1, sheep.time(), delta=1e-4) - - # Assert. - expected_config_proto = config_pb2.ConfigProto() - expected_config_proto.inter_op_parallelism_threads = 15 - expected_config_proto.intra_op_parallelism_threads = 15 - expected_config_proto.gpu_options.per_process_gpu_memory_fraction = 0.314 - mock_server.assert_called_with( - config.cluster_spec, - job_name=run_config_lib.TaskType.WORKER, - task_index=1, - config=expected_config_proto, - start=False) - mock_server.assert_has_calls([test.mock.call().start()]) - - @test.mock.patch.object(server_lib, 'Server') - def test_train_server_does_not_start_without_cluster_spec(self, mock_server): - config = run_config_lib.RunConfig(master='host4:2222') - for est in self._estimators_for_tests(config): - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input') - ex.train() - - # The server should not have started because there was no ClusterSpec. - self.assertFalse(mock_server.called) - - @test.mock.patch.object(server_lib, 'Server') - def test_train_server_does_not_start_with_empty_master(self, mock_server): - tf_config = {'cluster': self._cluster_spec()} - with test.mock.patch.dict('os.environ', - {'TF_CONFIG': json.dumps(tf_config)}): - config = run_config_lib.RunConfig(master='') - for est in self._estimators_for_tests(config): - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input') - ex.train() - # The server should not have started because master was the empty string. - self.assertFalse(mock_server.called) - - def test_train_raises_if_job_name_is_missing(self): - tf_config = { - 'cluster': self._cluster_spec(), - 'environment': run_config_lib.Environment.CLOUD, - 'task': { - 'index': 1 - } - } - with test.mock.patch.dict( - 'os.environ', - {'TF_CONFIG': json.dumps(tf_config)}), self.assertRaises(ValueError): - config = run_config_lib.RunConfig( - master='host3:2222' # Normally selected by task type. - ) - for est in self._estimators_for_tests(config): - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input') - ex.train() - - def test_evaluate(self): - for est in self._estimators_for_tests(): - eval_metrics = 'eval_metrics' if not isinstance( - est, core_estimator.Estimator) else None - est.fake_checkpoint() - noop_hook = _NoopHook() - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - eval_hooks=[noop_hook], - eval_steps='steps', - eval_delay_secs=0) - ex.evaluate() - self.assertEqual(0, est.fit_count) - self.assertEqual(1, est.eval_count) - self.assertEqual([noop_hook], est.eval_hooks) - - def test_evaluate_delay(self): - for est in self._estimators_for_tests(): - est.fake_checkpoint() - noop_hook = _NoopHook() - ex = experiment.Experiment( - est, train_input_fn='train_input', eval_input_fn='eval_input', - eval_hooks=[noop_hook]) - - for delay in [0, 1, 3]: - sheep = SheepCounter() - with test.mock.patch.object(time, 'time', sheep.time): - with test.mock.patch.object(time, 'sleep', sheep.sleep): - ex.evaluate(delay_secs=delay) - self.assertAlmostEqual(delay, sheep.time(), delta=1e-4) - self.assertEqual([noop_hook], est.eval_hooks) - - def test_continuous_eval(self): - for est in self._estimators_for_tests(eval_dict={'global_step': 100}): - eval_metrics = 'eval_metrics' if not isinstance( - est, core_estimator.Estimator) else None - est.fake_checkpoint() - noop_hook = _NoopHook() - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - eval_hooks=[noop_hook], - eval_delay_secs=0, - continuous_eval_throttle_secs=0) - self.assertRaises(StopIteration, ex.continuous_eval, - evaluate_checkpoint_only_once=False) - self.assertEqual(0, est.fit_count) - self.assertEqual(6, est.eval_count) - self.assertEqual([noop_hook], est.eval_hooks) - - def test_continuous_eval_ends_after_train_step(self): - for est in self._estimators_for_tests(eval_dict={'global_step': 100}): - eval_metrics = 'eval_metrics' if not isinstance( - est, core_estimator.Estimator) else None - est.fake_checkpoint() - noop_hook = _NoopHook() - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - eval_hooks=[noop_hook], - eval_delay_secs=0, - continuous_eval_throttle_secs=0, - train_steps=100) - ex.continuous_eval() - self.assertEqual(0, est.fit_count) - self.assertEqual(1, est.eval_count) - self.assertEqual([noop_hook], est.eval_hooks) - - def test_continuous_eval_throttle_delay(self): - for delay in [0, 1, 2]: - for est in self._estimators_for_tests(): - eval_metrics = 'eval_metrics' if not isinstance( - est, core_estimator.Estimator) else None - est.fake_checkpoint() - noop_hook = _NoopHook() - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - eval_hooks=[noop_hook], - continuous_eval_throttle_secs=delay, - eval_delay_secs=0) - sheep = SheepCounter() - with test.mock.patch.object(time, 'time', sheep.time): - with test.mock.patch.object(time, 'sleep', sheep.sleep): - self.assertRaises( - StopIteration, - ex.continuous_eval, - evaluate_checkpoint_only_once=False) - self.assertAlmostEqual(5 * delay, sheep.time(), delta=1e-4) - - def test_continuous_eval_predicate_fn(self): - for est in self._estimators_for_tests(): - eval_metrics = 'eval_metrics' if not isinstance( - est, core_estimator.Estimator) else None - est.fake_checkpoint() - noop_hook = _NoopHook() - - def _predicate_fn(unused_eval_result): - return est.eval_count < 3 # pylint: disable=cell-var-from-loop - - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - eval_hooks=[noop_hook], - eval_delay_secs=0, - continuous_eval_throttle_secs=0) - ex.continuous_eval(evaluate_checkpoint_only_once=False, - continuous_eval_predicate_fn=_predicate_fn) - self.assertEqual(0, est.fit_count) - self.assertEqual(3, est.eval_count) - self.assertEqual([noop_hook], est.eval_hooks) - - def test_continuous_eval_predicate_fn_with_checkpoint(self): - for est in self._estimators_for_tests(): - eval_metrics = 'eval_metrics' if not isinstance( - est, core_estimator.Estimator) else None - est.fake_checkpoint() - noop_hook = _NoopHook() - - def _predicate_fn(eval_result, checkpoint_path): - self.assertEqual(eval_result is None, - checkpoint_path is None) - return est.eval_count < 3 # pylint: disable=cell-var-from-loop - - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - eval_hooks=[noop_hook], - eval_delay_secs=0, - continuous_eval_throttle_secs=0) - ex.continuous_eval( - evaluate_checkpoint_only_once=False, - continuous_eval_predicate_fn=_predicate_fn) - self.assertEqual(0, est.fit_count) - self.assertEqual(3, est.eval_count) - self.assertEqual([noop_hook], est.eval_hooks) - - def test_run_local(self): - for est in self._estimators_for_tests(): - eval_metrics = 'eval_metrics' if not isinstance( - est, core_estimator.Estimator) else None - noop_hook = _NoopHook() - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - eval_hooks=[noop_hook], - train_steps=100, - eval_steps=100, - local_eval_frequency=10) - ex.local_run() - self.assertEqual(1, est.fit_count) - self.assertEqual(1, est.eval_count) - self.assertEqual(1, len(est.monitors)) - self.assertEqual([noop_hook], est.eval_hooks) - self.assertTrue(isinstance(est.monitors[0], - session_run_hook.SessionRunHook)) - - def test_train_hooks_extend_does_not_mutate_input_hooks(self): - for est in self._estimators_for_tests(): - eval_metrics = 'eval_metrics' if not isinstance( - est, core_estimator.Estimator) else None - noop_hook = _NoopHook() - input_hooks = [noop_hook] - - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - train_monitors=input_hooks) - self.assertAllEqual([noop_hook], ex._train_monitors) - - another_noop_hook = _NoopHook() - # Assert that the extend API mutates the hooks, but not the input hooks - ex.extend_train_hooks([another_noop_hook]) - self.assertAllEqual([noop_hook, another_noop_hook], ex._train_monitors) - self.assertAllEqual([noop_hook], input_hooks) - - def test_invalid_export_strategies(self): - for est in self._estimators_for_tests(): - with self.assertRaisesRegexp(ValueError, 'ExportStrategy'): - experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - train_steps=100, - eval_steps=100, - export_strategies='not_an_export_strategy') - with self.assertRaisesRegexp(ValueError, 'ExportStrategy'): - experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - train_steps=100, - eval_steps=100, - export_strategies=['not_an_export_srategy']) - - def test_export_strategies_reset(self): - for est in self._estimators_for_tests(): - eval_metrics = 'eval_metrics' if not isinstance( - est, core_estimator.Estimator) else None - export_strategy_1 = saved_model_export_utils.make_export_strategy( - est, - None if isinstance(est, core_estimator.Estimator) else 'export_1', - exports_to_keep=None) - - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - train_steps=100, - eval_steps=100, - export_strategies=(export_strategy_1,)) - ex.train_and_evaluate() - self.assertEqual(1, est.export_count) - - # After reset with empty list (None), the count does not change and the - # user provided export strategy list should remain intact. - old_es = ex.reset_export_strategies() - ex.train_and_evaluate() - self.assertAllEqual([export_strategy_1], old_es) - self.assertEqual(1, est.export_count) - - # After reset with list, the count should increase with the number of - # items. - export_strategy_2 = saved_model_export_utils.make_export_strategy( - est, - None if isinstance(est, core_estimator.Estimator) else 'export_2', - exports_to_keep=None) - export_strategy_3 = saved_model_export_utils.make_export_strategy( - est, - None if isinstance(est, core_estimator.Estimator) else 'export_3', - exports_to_keep=None) - - old_es = ex.reset_export_strategies( - [export_strategy_2, export_strategy_3]) - ex.train_and_evaluate() - self.assertAllEqual([], old_es) - self.assertEqual(3, est.export_count) - - def test_train_and_evaluate(self): - for est in self._estimators_for_tests(): - eval_metrics = 'eval_metrics' if not isinstance( - est, core_estimator.Estimator) else None - noop_hook = _NoopHook() - export_strategy = saved_model_export_utils.make_export_strategy( - est, - None if isinstance(est, core_estimator.Estimator) else 'export_input', - exports_to_keep=None) - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - eval_hooks=[noop_hook], - train_steps=100, - eval_steps=100, - export_strategies=export_strategy) - ex.train_and_evaluate() - self.assertEqual(1, est.fit_count) - self.assertEqual(1, est.eval_count) - self.assertEqual(1, est.export_count) - self.assertEqual(1, len(est.monitors)) - self.assertEqual([noop_hook], est.eval_hooks) - self.assertTrue(isinstance(est.monitors[0], - session_run_hook.SessionRunHook)) - - def test_train_and_evaluate_with_no_eval_during_training(self): - for est in self._estimators_for_tests(): - eval_metrics = 'eval_metrics' if not isinstance( - est, core_estimator.Estimator) else None - noop_hook = _NoopHook() - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - eval_hooks=[noop_hook], - train_steps=100, - eval_steps=100, - min_eval_frequency=0) - ex.train_and_evaluate() - self.assertEqual(1, est.fit_count) - self.assertEqual(1, est.eval_count) - self.assertEqual(0, len(est.monitors)) - - def test_min_eval_frequency_defaults(self): - def dummy_model_fn(features, labels): # pylint: disable=unused-argument - pass - estimator = core_estimator.Estimator(dummy_model_fn, '/tmp/dummy') - ex = experiment.Experiment( - estimator, train_input_fn=None, eval_input_fn=None) - self.assertEquals(ex._min_eval_frequency, 1) - - def test_continuous_train_and_eval(self): - for est in self._estimators_for_tests(eval_dict={'global_step': 100}): - if isinstance(est, core_estimator.Estimator): - eval_metrics = None - saving_listeners = 'saving_listeners' - else: - eval_metrics = 'eval_metrics' - saving_listeners = None - noop_hook = _NoopHook() - export_strategy = saved_model_export_utils.make_export_strategy( - est, - None if isinstance(est, core_estimator.Estimator) else 'export_input', - exports_to_keep=None) - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - eval_hooks=[noop_hook], - train_steps=100, - eval_steps=100, - export_strategies=export_strategy, - saving_listeners=saving_listeners) - ex.continuous_train_and_eval() - self.assertEqual(1, est.fit_count) - self.assertEqual(1, est.eval_count) - self.assertEqual(1, est.export_count) - self.assertEqual([noop_hook], est.eval_hooks) - - def test_continuous_train_and_eval_with_predicate_fn(self): - for est in self._estimators_for_tests(eval_dict={'global_step': 100}): - eval_metrics = 'eval_metrics' if not isinstance( - est, core_estimator.Estimator) else None - export_strategy = saved_model_export_utils.make_export_strategy( - est, - None if isinstance(est, core_estimator.Estimator) else 'export_input', - exports_to_keep=None) - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - train_steps=100000000000, # a value will make `ex` never stops. - eval_steps=100, - export_strategies=export_strategy) - - def predicate_fn(eval_result): - del eval_result # unused. for fn signature. - return False - - ex.continuous_train_and_eval(continuous_eval_predicate_fn=predicate_fn) - self.assertEqual(0, est.fit_count) - self.assertEqual(0, est.eval_count) - self.assertEqual(0, est.export_count) - - def test_continuous_train_and_eval_with_adapted_steps_per_iteration(self): - mock_estimator = test.mock.Mock(core_estimator.Estimator) - type(mock_estimator).model_dir = test.mock.PropertyMock( - return_value='test_dir') - - total_steps = 100000000000000 - ex = experiment.Experiment( - mock_estimator, - train_input_fn='train_input', - eval_input_fn='eval_input', - train_steps=total_steps) - - def predicate_fn(eval_result): - # Allows the first invoke only. - return eval_result is None - - ex.continuous_train_and_eval(continuous_eval_predicate_fn=predicate_fn) - mock_estimator.train.assert_called_once_with( - input_fn='train_input', - steps=int(total_steps / 10), - max_steps=test.mock.ANY, - hooks=test.mock.ANY, - saving_listeners=test.mock.ANY) - - def test_continuous_train_and_eval_with_steps_per_iteration_from_user(self): - mock_estimator = test.mock.Mock(core_estimator.Estimator) - type(mock_estimator).model_dir = test.mock.PropertyMock( - return_value='test_dir') - - total_steps = 100000000000000 - ex = experiment.Experiment( - mock_estimator, - train_input_fn='train_input', - eval_input_fn='eval_input', - train_steps_per_iteration=1234, - train_steps=total_steps) - - def predicate_fn(eval_result): - # Allows the first invoke only. - return eval_result is None - - ex.continuous_train_and_eval(continuous_eval_predicate_fn=predicate_fn) - mock_estimator.train.assert_called_once_with( - input_fn='train_input', - steps=1234, - max_steps=test.mock.ANY, - hooks=test.mock.ANY, - saving_listeners=test.mock.ANY) - - def test_continuous_train_and_eval_with_default_steps_per_iteration(self): - mock_estimator = test.mock.Mock(core_estimator.Estimator) - type(mock_estimator).model_dir = test.mock.PropertyMock( - return_value='test_dir') - - ex = experiment.Experiment( - mock_estimator, - train_input_fn='train_input', - eval_input_fn='eval_input', - train_steps_per_iteration=None, - train_steps=None) - - def predicate_fn(eval_result): - # Allows the first invoke only. - return eval_result is None - - ex.continuous_train_and_eval(continuous_eval_predicate_fn=predicate_fn) - mock_estimator.train.assert_called_once_with( - input_fn='train_input', - steps=1000, - max_steps=test.mock.ANY, - hooks=test.mock.ANY, - saving_listeners=test.mock.ANY) - - def test_continuous_train_and_eval_with_invalid_predicate_fn(self): - for est in self._estimators_for_tests(): - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input') - with self.assertRaisesRegexp( - ValueError, '`continuous_eval_predicate_fn` must be a callable'): - ex.continuous_train_and_eval(continuous_eval_predicate_fn='fn') - - def test_continuous_train_and_eval_with_invalid_train_steps_iterations(self): - for est in self._estimators_for_tests(): - with self.assertRaisesRegexp( - ValueError, '`train_steps_per_iteration` must be an integer.'): - experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - train_steps_per_iteration='123') - - @test.mock.patch.object(server_lib, 'Server') - def test_run_std_server(self, mock_server): - # Arrange. - tf_config = { - 'cluster': self._cluster_spec(), - 'task': { - 'type': run_config_lib.TaskType.PS, - 'index': 1 - } - } - with test.mock.patch.dict('os.environ', - {'TF_CONFIG': json.dumps(tf_config)}): - config = run_config_lib.RunConfig( - master='host2:2222', - num_cores=15, - gpu_memory_fraction=0.314,) - for est in self._estimators_for_tests(config): - ex = experiment.Experiment( - est, train_input_fn='train_input', eval_input_fn='eval_input') - - # Act. - ex.run_std_server() - - # Assert. - mock_server.assert_has_calls( - [test.mock.call().start(), test.mock.call().join()]) - - @test.mock.patch.object(server_lib, 'Server') - def test_run_std_server_raises_without_cluster_spec(self, mock_server): - config = run_config_lib.RunConfig(master='host4:2222') - for est in self._estimators_for_tests(config): - with self.assertRaises(ValueError): - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input') - ex.run_std_server() - - def test_test(self): - for est in self._estimators_for_tests(): - exp_strategy = saved_model_export_utils.make_export_strategy( - est, - None if isinstance(est, core_estimator.Estimator) else 'export_input', - exports_to_keep=None) - if isinstance(est, core_estimator.Estimator): - eval_metrics = None - saving_listeners = 'saving_listeners' - else: - eval_metrics = 'eval_metrics' - saving_listeners = None - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - export_strategies=(exp_strategy,), - eval_metrics=eval_metrics, - saving_listeners=saving_listeners) - ex.test() - self.assertEqual(1, est.fit_count) - self.assertEqual(1, est.eval_count) - self.assertEqual(1, est.export_count) - - def test_continuous_eval_evaluates_checkpoint_once(self): - for est in self._estimators_for_tests(eval_dict={'global_step': 100}): - eval_metrics = 'eval_metrics' if not isinstance( - est, core_estimator.Estimator) else None - est.fake_checkpoint() - - result = { - 'called': 0, - 'called_with_eval_result': 0, - } - # pylint: disable=cell-var-from-loop - def _predicate_fn(eval_result): - result['called'] += 1 - if eval_result: - # If eval_result is not empty nor None, the checkpoint has been - # evaluated. - result['called_with_eval_result'] += 1 - # With 300 times of evaluation, this should prove something. - return result['called'] < 300 - # pylint: enable=cell-var-from-loop - - ex = experiment.Experiment( - est, - train_input_fn='train_input', - eval_input_fn='eval_input', - eval_metrics=eval_metrics, - eval_delay_secs=0, - continuous_eval_throttle_secs=0) - ex.continuous_eval(evaluate_checkpoint_only_once=True, - continuous_eval_predicate_fn=_predicate_fn) - - self.assertEqual(0, est.fit_count) - self.assertEqual(1, est.eval_count) - self.assertEqual(300, result['called']) - self.assertEqual(1, result['called_with_eval_result']) - - def test_checkpoint_and_export(self): - model_dir = tempfile.mkdtemp() - config = run_config_lib.RunConfig(save_checkpoints_steps=3) - est = dnn.DNNClassifier( - n_classes=3, - feature_columns=[ - feature_column.real_valued_column('feature', dimension=4) - ], - hidden_units=[3, 3], - model_dir=model_dir, - config=config) - - exp_strategy = saved_model_export_utils.make_export_strategy( - est, 'export_input', exports_to_keep=None) - - ex = experiment.Experiment( - est, - train_input_fn=test_data.iris_input_multiclass_fn, - eval_input_fn=test_data.iris_input_multiclass_fn, - export_strategies=(exp_strategy,), - train_steps=8, - checkpoint_and_export=True, - eval_delay_secs=0) - - with test.mock.patch.object(ex, '_maybe_export'): - with test.mock.patch.object(ex, '_call_evaluate'): - ex.train_and_evaluate() - # Eval and export are called after steps 1, 4, 7, and 8 (after training - # is completed). - self.assertEqual(ex._maybe_export.call_count, 4) - self.assertEqual(ex._call_evaluate.call_count, 4) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/export_strategy.py b/tensorflow/contrib/learn/python/learn/export_strategy.py deleted file mode 100644 index 075cab536ec..00000000000 --- a/tensorflow/contrib/learn/python/learn/export_strategy.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ExportStrategy class represents different flavors of model export (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from tensorflow.python.util import tf_inspect -from tensorflow.python.util.deprecation import deprecated - -__all__ = ['ExportStrategy'] - - -class ExportStrategy( - collections.namedtuple('ExportStrategy', - ['name', 'export_fn', 'strip_default_attrs'])): - """A class representing a type of model export. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Typically constructed by a utility function specific to the exporter, such as - `saved_model_export_utils.make_export_strategy()`. - - Attributes: - name: The directory name under the export base directory where exports of - this type will be written. - export_fn: A function that writes an export, given an estimator, a - destination path, and optionally a checkpoint path and an evaluation - result for that checkpoint. This export_fn() may be run repeatedly during - continuous training, or just once at the end of fixed-length training. - Note the export_fn() may choose whether or not to export based on the eval - result or based on an internal timer or any other criterion, if exports - are not desired for every checkpoint. - - The signature of this function must be one of: - - * `(estimator, export_path) -> export_path` - * `(estimator, export_path, checkpoint_path) -> export_path` - * `(estimator, export_path, checkpoint_path, eval_result) -> export_path` - * `(estimator, export_path, checkpoint_path, eval_result, - strip_default_attrs) -> export_path` - strip_default_attrs: (Optional) Boolean. If set as True, default attrs in - the `GraphDef` will be stripped on write. This is recommended for better - forward compatibility of the resulting `SavedModel`. - """ - - @deprecated(None, 'Please switch to tf.estimator.train_and_evaluate, and use ' - 'tf.estimator.Exporter.') - def __new__(cls, name, export_fn, strip_default_attrs=None): - return super(ExportStrategy, cls).__new__( - cls, name, export_fn, strip_default_attrs) - - def export(self, - estimator, - export_path, - checkpoint_path=None, - eval_result=None): - """Exports the given Estimator to a specific format. - - Args: - estimator: the Estimator to export. - export_path: A string containing a directory where to write the export. - checkpoint_path: The checkpoint path to export. If None (the default), - the strategy may locate a checkpoint (e.g. the most recent) by itself. - eval_result: The output of Estimator.evaluate on this checkpoint. This - should be set only if checkpoint_path is provided (otherwise it is - unclear which checkpoint this eval refers to). - - Returns: - The string path to the exported directory. - - Raises: - ValueError: if the export_fn does not have the required signature - """ - # don't break existing export_fns that don't accept checkpoint_path and - # eval_result - export_fn_args = tf_inspect.getargspec(self.export_fn).args - kwargs = {} - if 'checkpoint_path' in export_fn_args: - kwargs['checkpoint_path'] = checkpoint_path - if 'eval_result' in export_fn_args: - if 'checkpoint_path' not in export_fn_args: - raise ValueError('An export_fn accepting eval_result must also accept ' - 'checkpoint_path.') - kwargs['eval_result'] = eval_result - if 'strip_default_attrs' in export_fn_args: - kwargs['strip_default_attrs'] = self.strip_default_attrs - return self.export_fn(estimator, export_path, **kwargs) diff --git a/tensorflow/contrib/learn/python/learn/export_strategy_test.py b/tensorflow/contrib/learn/python/learn/export_strategy_test.py deleted file mode 100644 index 43c3551cccc..00000000000 --- a/tensorflow/contrib/learn/python/learn/export_strategy_test.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for ExportStrategy.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn import export_strategy -from tensorflow.python.platform import test - - -class ExportStrategyTest(test.TestCase): - - def test_no_optional_args_export(self): - model_path = '/path/to/model' - def _export_fn(estimator, export_path): - self.assertTupleEqual((estimator, export_path), (None, None)) - return model_path - - strategy = export_strategy.ExportStrategy('foo', _export_fn) - self.assertTupleEqual(strategy, ('foo', _export_fn, None)) - self.assertIs(strategy.export(None, None), model_path) - - def test_checkpoint_export(self): - ckpt_model_path = '/path/to/checkpoint_model' - def _ckpt_export_fn(estimator, export_path, checkpoint_path): - self.assertTupleEqual((estimator, export_path), (None, None)) - self.assertEqual(checkpoint_path, 'checkpoint') - return ckpt_model_path - - strategy = export_strategy.ExportStrategy('foo', _ckpt_export_fn) - self.assertTupleEqual(strategy, ('foo', _ckpt_export_fn, None)) - self.assertIs(strategy.export(None, None, 'checkpoint'), ckpt_model_path) - - def test_checkpoint_eval_export(self): - ckpt_eval_model_path = '/path/to/checkpoint_eval_model' - def _ckpt_eval_export_fn(estimator, export_path, checkpoint_path, - eval_result): - self.assertTupleEqual((estimator, export_path), (None, None)) - self.assertEqual(checkpoint_path, 'checkpoint') - self.assertEqual(eval_result, 'eval') - return ckpt_eval_model_path - - strategy = export_strategy.ExportStrategy('foo', _ckpt_eval_export_fn) - self.assertTupleEqual(strategy, ('foo', _ckpt_eval_export_fn, None)) - self.assertIs(strategy.export(None, None, 'checkpoint', 'eval'), - ckpt_eval_model_path) - - def test_eval_only_export(self): - def _eval_export_fn(estimator, export_path, eval_result): - del estimator, export_path, eval_result - - strategy = export_strategy.ExportStrategy('foo', _eval_export_fn) - self.assertTupleEqual(strategy, ('foo', _eval_export_fn, None)) - with self.assertRaisesRegexp(ValueError, 'An export_fn accepting ' - 'eval_result must also accept ' - 'checkpoint_path'): - strategy.export(None, None, eval_result='eval') - - def test_strip_default_attr_export(self): - strip_default_attrs_model_path = '/path/to/strip_default_attrs_model' - def _strip_default_attrs_export_fn(estimator, export_path, - strip_default_attrs): - self.assertTupleEqual((estimator, export_path), (None, None)) - self.assertTrue(strip_default_attrs) - return strip_default_attrs_model_path - - strategy = export_strategy.ExportStrategy('foo', - _strip_default_attrs_export_fn, - True) - self.assertTupleEqual(strategy, - ('foo', _strip_default_attrs_export_fn, True)) - self.assertIs(strategy.export(None, None), strip_default_attrs_model_path) - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/graph_actions.py b/tensorflow/contrib/learn/python/learn/graph_actions.py deleted file mode 100644 index 244cb3fd438..00000000000 --- a/tensorflow/contrib/learn/python/learn/graph_actions.py +++ /dev/null @@ -1,735 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""High level operations on graphs (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools -import sys -import threading -import time - -import numpy as np - -from six import reraise - -from tensorflow.contrib.framework import load_variable -from tensorflow.contrib.framework.python.ops import ops as contrib_ops -from tensorflow.contrib.framework.python.ops import variables as contrib_variables -from tensorflow.contrib.learn.python.learn import monitors as monitors_lib -from tensorflow.core.framework import summary_pb2 -from tensorflow.python.client import session as tf_session -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import resources -from tensorflow.python.ops import variables -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import coordinator -from tensorflow.python.training import queue_runner -from tensorflow.python.training import saver as tf_saver -from tensorflow.python.training import session_manager as session_manager_lib -from tensorflow.python.training import summary_io -from tensorflow.python.training import supervisor as tf_supervisor -from tensorflow.python.util.deprecation import deprecated - -# Singleton for SummaryWriter per logdir folder. -_SUMMARY_WRITERS = {} - -# Lock protecting _SUMMARY_WRITERS -_summary_writer_lock = threading.Lock() - -_graph_action_deprecation = deprecated( - '2017-02-15', - 'graph_actions.py will be deleted. Use tf.train.* utilities instead. ' - 'You can use learn/estimators/estimator.py as an example.') - - -@_graph_action_deprecation -def clear_summary_writers(): - """Clear cached summary writers. Currently only used for unit tests.""" - return summary_io.SummaryWriterCache.clear() - - -@deprecated(None, 'Use `SummaryWriterCache.get` directly.') -def get_summary_writer(logdir): - """Returns single SummaryWriter per logdir in current run. - - Args: - logdir: str, folder to write summaries. - - Returns: - Existing `SummaryWriter` object or new one if never wrote to given - directory. - """ - return summary_io.SummaryWriterCache.get(logdir) - - -def _make_saver(graph, keep_checkpoint_max=5): - vars_to_save = (graph.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + - graph.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS)) - if vars_to_save: - return tf_saver.Saver(vars_to_save, - sharded=True, - max_to_keep=keep_checkpoint_max) - else: - return None - - -def _restore_from_checkpoint(session, graph, checkpoint_path, saver=None): - logging.info('Loading model from checkpoint: %s.', checkpoint_path) - saver = saver or _make_saver(graph) - if saver: - saver.restore(session, checkpoint_path) - else: - logging.info('No variables found in graph, not creating Saver() object.') - - -def _run_with_monitors(session, step, tensors, feed_dict, monitors): - """Runs session for given tensors with monitor callbacks.""" - for monitor in monitors: - tensors += monitor.step_begin(step) - tensors = list(set(tensors)) - - outputs = session.run(tensors, feed_dict=feed_dict) - outputs = dict(zip( - [t.name if isinstance(t, ops.Tensor) else t for t in tensors], - outputs)) - - should_stop = False - for monitor in monitors: - induce_stop = monitor.step_end(step, outputs) - should_stop = should_stop or induce_stop - return outputs, should_stop - - -@_graph_action_deprecation -def train(graph, - output_dir, - train_op, - loss_op, - global_step_tensor=None, - init_op=None, - init_feed_dict=None, - init_fn=None, - log_every_steps=10, - supervisor_is_chief=True, - supervisor_master='', - supervisor_save_model_secs=600, - keep_checkpoint_max=5, - supervisor_save_summaries_steps=100, - feed_fn=None, - steps=None, - fail_on_nan_loss=True, - monitors=None, - max_steps=None): - """Train a model. - - Given `graph`, a directory to write outputs to (`output_dir`), and some ops, - run a training loop. The given `train_op` performs one step of training on the - model. The `loss_op` represents the objective function of the training. It is - expected to increment the `global_step_tensor`, a scalar integer tensor - counting training steps. This function uses `Supervisor` to initialize the - graph (from a checkpoint if one is available in `output_dir`), write summaries - defined in the graph, and write regular checkpoints as defined by - `supervisor_save_model_secs`. - - Training continues until `global_step_tensor` evaluates to `max_steps`, or, if - `fail_on_nan_loss`, until `loss_op` evaluates to `NaN`. In that case the - program is terminated with exit code 1. - - Args: - graph: A graph to train. It is expected that this graph is not in use - elsewhere. - output_dir: A directory to write outputs to. - train_op: An op that performs one training step when run. - loss_op: A scalar loss tensor. - global_step_tensor: A tensor representing the global step. If none is given, - one is extracted from the graph using the same logic as in `Supervisor`. - init_op: An op that initializes the graph. If `None`, use `Supervisor`'s - default. - init_feed_dict: A dictionary that maps `Tensor` objects to feed values. - This feed dictionary will be used when `init_op` is evaluated. - init_fn: Optional callable passed to Supervisor to initialize the model. - log_every_steps: Output logs regularly. The logs contain timing data and the - current loss. - supervisor_is_chief: Whether the current process is the chief supervisor in - charge of restoring the model and running standard services. - supervisor_master: The master string to use when preparing the session. - supervisor_save_model_secs: Save a checkpoint every - `supervisor_save_model_secs` seconds when training. - keep_checkpoint_max: The maximum number of recent checkpoint files to - keep. As new files are created, older files are deleted. If None or 0, - all checkpoint files are kept. This is simply passed as the max_to_keep - arg to tf.compat.v1.train.Saver constructor. - supervisor_save_summaries_steps: Save summaries every - `supervisor_save_summaries_steps` seconds when training. - feed_fn: A function that is called every iteration to produce a `feed_dict` - passed to `session.run` calls. Optional. - steps: Trains for this many steps (e.g. current global step + `steps`). - fail_on_nan_loss: If true, raise `NanLossDuringTrainingError` if `loss_op` - evaluates to `NaN`. If false, continue training as if nothing happened. - monitors: List of `BaseMonitor` subclass instances. Used for callbacks - inside the training loop. - max_steps: Number of total steps for which to train model. If `None`, - train forever. Two calls fit(steps=100) means 200 training iterations. - On the other hand two calls of fit(max_steps=100) means, second call - will not do any iteration since first call did all 100 steps. - - Returns: - The final loss value. - - Raises: - ValueError: If `output_dir`, `train_op`, `loss_op`, or `global_step_tensor` - is not provided. See `tf.contrib.framework.get_global_step` for how we - look up the latter if not provided explicitly. - NanLossDuringTrainingError: If `fail_on_nan_loss` is `True`, and loss ever - evaluates to `NaN`. - ValueError: If both `steps` and `max_steps` are not `None`. - """ - while True: - try: - return _train_internal(graph, - output_dir, - train_op, - loss_op, - global_step_tensor, - init_op, - init_feed_dict, - init_fn, - log_every_steps, - supervisor_is_chief, - supervisor_master, - supervisor_save_model_secs, - keep_checkpoint_max, - supervisor_save_summaries_steps, - feed_fn, - steps, - fail_on_nan_loss, - monitors, - max_steps) - except errors.AbortedError: - # Happens when PS restarts, keep training. - logging.warning('Training got Aborted error. Keep training.') - - -def _train_internal(graph, - output_dir, - train_op, - loss_op, - global_step_tensor, - init_op, - init_feed_dict, - init_fn, - log_every_steps, - supervisor_is_chief, - supervisor_master, - supervisor_save_model_secs, - keep_checkpoint_max, - supervisor_save_summaries_steps, - feed_fn, - steps, - fail_on_nan_loss, - monitors, - max_steps): - """See train.""" - if (steps is not None) and (max_steps is not None): - raise ValueError('Can not provide both steps and max_steps.') - if not output_dir: - raise ValueError('Output directory should be non-empty %s.' % output_dir) - if train_op is None: - raise ValueError('Missing train_op.') - if loss_op is None: - raise ValueError('Missing loss_op.') - - with graph.as_default(): - global_step_tensor = contrib_variables.assert_or_get_global_step( - graph, global_step_tensor) - if global_step_tensor is None: - raise ValueError('No "global_step" was provided or found in the graph.') - - # Get current step. - try: - start_step = load_variable(output_dir, global_step_tensor.name) - except (errors.NotFoundError, ValueError): - start_step = 0 - - summary_writer = (get_summary_writer(output_dir) - if supervisor_is_chief else None) - - # Add default chief monitors if none were provided. - if not monitors: - monitors = monitors_lib.get_default_monitors( - loss_op=loss_op, - summary_op=logging_ops.get_summary_op(), - save_summary_steps=supervisor_save_summaries_steps, - summary_writer=summary_writer) if supervisor_is_chief else [] - - # TODO(ipolosukhin): Replace all functionality of Supervisor - # with Chief-Exclusive Monitors. - if not supervisor_is_chief: - # Prune list of monitor to the ones runnable on all workers. - monitors = [monitor for monitor in monitors if monitor.run_on_all_workers] - - if max_steps is None: - max_steps = (start_step + steps) if steps else None - # Start monitors, can create graph parts. - for monitor in monitors: - monitor.begin(max_steps=max_steps) - - supervisor = tf_supervisor.Supervisor( - graph, - init_op=init_op or tf_supervisor.Supervisor.USE_DEFAULT, - init_feed_dict=init_feed_dict, - is_chief=supervisor_is_chief, - logdir=output_dir, - saver=_make_saver(graph, keep_checkpoint_max), - global_step=global_step_tensor, - summary_op=None, - summary_writer=summary_writer, - save_model_secs=supervisor_save_model_secs, - init_fn=init_fn) - session = supervisor.PrepareSession(master=supervisor_master, - start_standard_services=True) - supervisor.StartQueueRunners(session) - - with session: - get_current_step = lambda: session.run(global_step_tensor) - - start_step = get_current_step() - last_step = start_step - last_log_step = start_step - loss_value = None - logging.info('Training steps [%d,%s)', last_step, 'inf' - if max_steps is None else str(max_steps)) - - excinfo = None - try: - while not supervisor.ShouldStop() and ( - (max_steps is None) or (last_step < max_steps)): - start_time = time.time() - feed_dict = feed_fn() if feed_fn is not None else None - - outputs, should_stop = _run_with_monitors( - session, last_step + 1, [train_op, loss_op], feed_dict, monitors) - - loss_value = outputs[loss_op.name] - if np.isnan(loss_value): - failure_message = 'Model diverged with loss = NaN.' - if fail_on_nan_loss: - logging.error(failure_message) - raise monitors_lib.NanLossDuringTrainingError() - else: - logging.warning(failure_message) - - if should_stop: - break - - this_step = get_current_step() - - if this_step <= last_step: - logging.error( - 'Global step was not incremented by train op at step %s' - ': new step %d', last_step, this_step) - - last_step = this_step - is_last_step = (max_steps is not None) and (last_step >= max_steps) - if is_last_step or (last_step - last_log_step >= log_every_steps): - logging.info( - 'training step %d, loss = %.5f (%.3f sec/batch).', - last_step, loss_value, float(time.time() - start_time)) - last_log_step = last_step - except errors.OutOfRangeError as e: - logging.warn('Got exception during tf.learn training loop possibly ' - 'due to exhausted input queue %s.', e) - except StopIteration: - logging.info('Exhausted input iterarator.') - except BaseException as e: # pylint: disable=broad-except - # Hold on to any other exceptions while we try recording a final - # checkpoint and summary. - excinfo = sys.exc_info() - finally: - try: - # Call supervisor.Stop() from within a try block because it re-raises - # exceptions thrown by the supervised threads. - supervisor.Stop(close_summary_writer=False) - - # Save one last checkpoint and summaries - # TODO(wicke): This should be handled by Supervisor - - # In case we encountered an exception in the try block before we updated - # last_step, update it here (again). - last_step = get_current_step() - if supervisor_is_chief: - ckpt_path = supervisor.save_path - logging.info('Saving checkpoint for step %d to checkpoint: %s.', - last_step, ckpt_path) - supervisor.saver.save(session, ckpt_path, global_step=last_step) - - # Finish monitors. - for monitor in monitors: - monitor.end() - - # catch OutOfRangeError which is thrown when queue is out of data (and for - # other reasons as well). - except errors.OutOfRangeError as e: - logging.warn('OutOfRangeError in tf.learn final checkpoint possibly ' - 'due to exhausted input queue. Note: summary_op is not ' - 'expected to trigger dequeues. %s.', e) - except BaseException as e: # pylint: disable=broad-except - # If we don't already have an exception to re-raise, raise this one. - if not excinfo: - raise - # Otherwise, log this one and raise the other in the finally block. - logging.error('Got exception during tf.learn final checkpoint %s.', e) - finally: - if excinfo: - reraise(*excinfo) - return loss_value - - -def _get_first_op_from_collection(collection_name): - elements = ops.get_collection(collection_name) - if elements: - return elements[0] - return None - - -def _get_saver(): - """Lazy init and return saver.""" - saver = _get_first_op_from_collection(ops.GraphKeys.SAVERS) - if saver is None and variables.global_variables(): - saver = tf_saver.Saver() - ops.add_to_collection(ops.GraphKeys.SAVERS, saver) - return saver - - -def _get_ready_op(): - ready_op = _get_first_op_from_collection(ops.GraphKeys.READY_OP) - if ready_op is None: - ready_op = variables.report_uninitialized_variables() - ops.add_to_collection(ops.GraphKeys.READY_OP, ready_op) - return ready_op - - -def _get_local_init_op(): - """Returns the local init ops to initialize tables and local variables.""" - local_init_op = _get_first_op_from_collection( - ops.GraphKeys.LOCAL_INIT_OP) - if local_init_op is None: - op_list = [ - variables.local_variables_initializer(), - lookup_ops.tables_initializer() - ] - if op_list: - local_init_op = control_flow_ops.group(*op_list) - ops.add_to_collection(ops.GraphKeys.LOCAL_INIT_OP, local_init_op) - return local_init_op - - -def _eval_results_to_str(eval_results): - return ', '.join('%s = %s' % (k, v) for k, v in sorted(eval_results.items())) - - -def _write_summary_results(output_dir, eval_results, current_global_step): - """Writes eval results into summary file in given dir.""" - logging.info('Saving evaluation summary for step %d: %s', current_global_step, - _eval_results_to_str(eval_results)) - summary_writer = get_summary_writer(output_dir) - summary = summary_pb2.Summary() - for key in eval_results: - if eval_results[key] is None: - continue - value = summary.value.add() - value.tag = key - if (isinstance(eval_results[key], np.float32) or - isinstance(eval_results[key], float)): - value.simple_value = float(eval_results[key]) - else: - logging.warn('Skipping summary for %s, must be a float or np.float32.', - key) - summary_writer.add_summary(summary, current_global_step) - summary_writer.flush() - - -@_graph_action_deprecation -def evaluate(graph, - output_dir, - checkpoint_path, - eval_dict, - update_op=None, - global_step_tensor=None, - supervisor_master='', - log_every_steps=10, - feed_fn=None, - max_steps=None): - """Evaluate a model loaded from a checkpoint. - - Given `graph`, a directory to write summaries to (`output_dir`), a checkpoint - to restore variables from, and a `dict` of `Tensor`s to evaluate, run an eval - loop for `max_steps` steps, or until an exception (generally, an - end-of-input signal from a reader operation) is raised from running - `eval_dict`. - - In each step of evaluation, all tensors in the `eval_dict` are evaluated, and - every `log_every_steps` steps, they are logged. At the very end of evaluation, - a summary is evaluated (finding the summary ops using `Supervisor`'s logic) - and written to `output_dir`. - - Args: - graph: A `Graph` to train. It is expected that this graph is not in use - elsewhere. - output_dir: A string containing the directory to write a summary to. - checkpoint_path: A string containing the path to a checkpoint to restore. - Can be `None` if the graph doesn't require loading any variables. - eval_dict: A `dict` mapping string names to tensors to evaluate. It is - evaluated in every logging step. The result of the final evaluation is - returned. If `update_op` is None, then it's evaluated in every step. If - `max_steps` is `None`, this should depend on a reader that will raise an - end-of-input exception when the inputs are exhausted. - update_op: A `Tensor` which is run in every step. - global_step_tensor: A `Variable` containing the global step. If `None`, - one is extracted from the graph using the same logic as in `Supervisor`. - Used to place eval summaries on training curves. - supervisor_master: The master string to use when preparing the session. - log_every_steps: Integer. Output logs every `log_every_steps` evaluation - steps. The logs contain the `eval_dict` and timing information. - feed_fn: A function that is called every iteration to produce a `feed_dict` - passed to `session.run` calls. Optional. - max_steps: Integer. Evaluate `eval_dict` this many times. - - Returns: - A tuple `(eval_results, global_step)`: - eval_results: A `dict` mapping `string` to numeric values (`int`, `float`) - that are the result of running eval_dict in the last step. `None` if no - eval steps were run. - global_step: The global step this evaluation corresponds to. - - Raises: - ValueError: if `output_dir` is empty. - """ - if not output_dir: - raise ValueError('Output directory should be non-empty %s.' % output_dir) - with graph.as_default(): - global_step_tensor = contrib_variables.assert_or_get_global_step( - graph, global_step_tensor) - - # Create or get summary op, global_step and saver. - saver = _get_saver() - local_init_op = _get_local_init_op() - ready_for_local_init_op = _get_first_op_from_collection( - ops.GraphKeys.READY_FOR_LOCAL_INIT_OP) - ready_op = _get_ready_op() - - session_manager = session_manager_lib.SessionManager( - local_init_op=local_init_op, - ready_op=ready_op, - ready_for_local_init_op=ready_for_local_init_op) - session, initialized = session_manager.recover_session( - master=supervisor_master, - saver=saver, - checkpoint_dir=checkpoint_path) - - # Start queue runners. - coord = coordinator.Coordinator() - threads = queue_runner.start_queue_runners(session, coord) - - with session: - if not initialized: - logging.warning('Failed to initialize from %s.', checkpoint_path) - # TODO(ipolosukhin): This should be failing, but old code relies on that. - session.run(variables.global_variables_initializer()) - if checkpoint_path: - _restore_from_checkpoint(session, graph, checkpoint_path, saver) - - current_global_step = session.run(global_step_tensor) - eval_results = None - # TODO(amodei): Fix this to run through the eval set exactly once. - step = 0 - eval_step = None - feed_dict = None - logging.info('Eval steps [%d,%s) for training step %d.', step, - 'inf' if max_steps is None - else str(max_steps), current_global_step) - try: - try: - while (max_steps is None) or (step < max_steps): - step += 1 - start_time = time.time() - feed_dict = feed_fn() if feed_fn is not None else None - if update_op is not None: - session.run(update_op, feed_dict=feed_dict) - else: - eval_results = session.run(eval_dict, feed_dict=feed_dict) - eval_step = step - - # TODO(wicke): We should assert that the global step hasn't changed. - if step % log_every_steps == 0: - if eval_step is None or step != eval_step: - eval_results = session.run(eval_dict, feed_dict=feed_dict) - eval_step = step - duration = time.time() - start_time - logging.info('Results after %d steps (%.3f sec/batch): %s.', - step, float(duration), - _eval_results_to_str(eval_results)) - finally: - if eval_results is None or step != eval_step: - eval_results = session.run(eval_dict, feed_dict=feed_dict) - eval_step = step - # Stop session first, before queue runners. - session.close() - - # Stop queue runners. - try: - coord.request_stop() - coord.join(threads, stop_grace_period_secs=120) - except (RuntimeError, errors.CancelledError) as e: - logging.warning('Coordinator didn\'t stop cleanly: %s', e) - - # catch OutOfRangeError which is thrown when queue is out of data (and for - # other reasons as well). - except errors.OutOfRangeError as e: - if max_steps is None: - logging.info('Input queue is exhausted.') - else: - logging.warn('Input queue is exhausted: %s.', e) - # catch StopIteration which is thrown is DataReader is out of data. - except StopIteration as e: - if max_steps is None: - logging.info('Input iterator is exhausted.') - else: - logging.warn('Input iterator is exhausted: %s.', e) - - # Save summaries for this evaluation. - _write_summary_results(output_dir, eval_results, current_global_step) - - return eval_results, current_global_step - - -@_graph_action_deprecation -def run_n(output_dict, feed_dict=None, restore_checkpoint_path=None, n=1): - """Run `output_dict` tensors `n` times, with the same `feed_dict` each run. - - Args: - output_dict: A `dict` mapping string names to tensors to run. Must all be - from the same graph. - feed_dict: `dict` of input values to feed each run. - restore_checkpoint_path: A string containing the path to a checkpoint to - restore. - n: Number of times to repeat. - - Returns: - A list of `n` `dict` objects, each containing values read from `output_dict` - tensors. - """ - return run_feeds( - output_dict=output_dict, - feed_dicts=itertools.repeat(feed_dict, n), - restore_checkpoint_path=restore_checkpoint_path) - - -@_graph_action_deprecation -def run_feeds_iter(output_dict, feed_dicts, restore_checkpoint_path=None): - """Run `output_dict` tensors with each input in `feed_dicts`. - - If `restore_checkpoint_path` is supplied, restore from checkpoint. Otherwise, - init all variables. - - Args: - output_dict: A `dict` mapping string names to `Tensor` objects to run. - Tensors must all be from the same graph. - feed_dicts: Iterable of `dict` objects of input values to feed. - restore_checkpoint_path: A string containing the path to a checkpoint to - restore. - - Yields: - A sequence of dicts of values read from `output_dict` tensors, one item - yielded for each item in `feed_dicts`. Keys are the same as `output_dict`, - values are the results read from the corresponding `Tensor` in - `output_dict`. - - Raises: - ValueError: if `output_dict` or `feed_dicts` is None or empty. - """ - if not output_dict: - raise ValueError('output_dict is invalid: %s.' % output_dict) - if not feed_dicts: - raise ValueError('feed_dicts is invalid: %s.' % feed_dicts) - - graph = contrib_ops.get_graph_from_inputs(output_dict.values()) - with graph.as_default() as g: - with tf_session.Session('') as session: - session.run( - resources.initialize_resources(resources.shared_resources() + - resources.local_resources())) - if restore_checkpoint_path: - _restore_from_checkpoint(session, g, restore_checkpoint_path) - else: - session.run(variables.global_variables_initializer()) - session.run(variables.local_variables_initializer()) - session.run(lookup_ops.tables_initializer()) - coord = coordinator.Coordinator() - threads = None - try: - threads = queue_runner.start_queue_runners(session, coord=coord) - for f in feed_dicts: - yield session.run(output_dict, f) - finally: - coord.request_stop() - if threads: - coord.join(threads, stop_grace_period_secs=120) - - -@_graph_action_deprecation -def run_feeds(*args, **kwargs): - """See run_feeds_iter(). Returns a `list` instead of an iterator.""" - return list(run_feeds_iter(*args, **kwargs)) - - -@_graph_action_deprecation -def infer(restore_checkpoint_path, output_dict, feed_dict=None): - """Restore graph from `restore_checkpoint_path` and run `output_dict` tensors. - - If `restore_checkpoint_path` is supplied, restore from checkpoint. Otherwise, - init all variables. - - Args: - restore_checkpoint_path: A string containing the path to a checkpoint to - restore. - output_dict: A `dict` mapping string names to `Tensor` objects to run. - Tensors must all be from the same graph. - feed_dict: `dict` object mapping `Tensor` objects to input values to feed. - - Returns: - Dict of values read from `output_dict` tensors. Keys are the same as - `output_dict`, values are the results read from the corresponding `Tensor` - in `output_dict`. - - Raises: - ValueError: if `output_dict` or `feed_dicts` is None or empty. - """ - return run_feeds(output_dict=output_dict, - feed_dicts=[feed_dict] if feed_dict is not None else [None], - restore_checkpoint_path=restore_checkpoint_path)[0] diff --git a/tensorflow/contrib/learn/python/learn/graph_actions_test.py b/tensorflow/contrib/learn/python/learn/graph_actions_test.py deleted file mode 100644 index a160cb54a39..00000000000 --- a/tensorflow/contrib/learn/python/learn/graph_actions_test.py +++ /dev/null @@ -1,686 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Graph actions tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import shutil -import tempfile - -from tensorflow.contrib import testing -from tensorflow.contrib.framework.python.framework import checkpoint_utils -from tensorflow.contrib.framework.python.ops import variables as variables_lib -from tensorflow.contrib.learn.python import learn -from tensorflow.contrib.learn.python.learn.monitors import BaseMonitor -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import resources -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.summary import summary -from tensorflow.python.training import checkpoint_management -from tensorflow.python.training import saver as saver_lib - - -class _Feeder(object): - """Simple generator for `feed_fn`, returning 10 * step.""" - - def __init__(self, tensor, max_step): - self._step = 0 - self._tensor = tensor - self._max_step = max_step - - @property - def step(self): - return self._step - - def feed_fn(self): - if self._step >= self._max_step: - raise StopIteration - value = self._step * 10.0 - self._step += 1 - return {self._tensor: value} - - -class _BaseMonitorWrapper(BaseMonitor): - """Base monitor wrapper to facilitate testing. - - This monitor can act as either chief-exclusive or non-exclusive. - """ - - def __init__(self, run_on_all_workers): - super(_BaseMonitorWrapper, self).__init__() - self._run_on_all_workers = run_on_all_workers - self._is_active = False - self._has_step = False - - @property - def run_on_all_workers(self): - return self._run_on_all_workers - - @property - def is_active(self): - return self._is_active - - @property - def has_step(self): - return self._has_step - - def begin(self, max_steps=None): - self._is_active = True - return super(_BaseMonitorWrapper, self).begin(max_steps) - - def step_begin(self, step): - self._has_step = True - return super(_BaseMonitorWrapper, self).step_begin(step) - - -class GraphActionsTest(test.TestCase): - """Graph actions tests.""" - - def setUp(self): - learn.graph_actions.clear_summary_writers() - self._output_dir = tempfile.mkdtemp() - testing.FakeSummaryWriter.install() - - def tearDown(self): - testing.FakeSummaryWriter.uninstall() - if self._output_dir: - shutil.rmtree(self._output_dir) - learn.graph_actions.clear_summary_writers() - - def _assert_summaries(self, - output_dir, - writer, - expected_summaries=None, - expected_graphs=None, - expected_meta_graphs=None, - expected_session_logs=None): - self.assertTrue(isinstance(writer, testing.FakeSummaryWriter)) - writer.assert_summaries( - self, - expected_logdir=output_dir, - expected_graph=ops.get_default_graph(), - expected_summaries=expected_summaries, - expected_added_graphs=expected_graphs, - expected_added_meta_graphs=expected_meta_graphs, - expected_session_logs=expected_session_logs) - - # TODO(ptucker): Test number and contents of checkpoint files. - def _assert_ckpt(self, output_dir, expected=True): - ckpt_state = checkpoint_management.get_checkpoint_state(output_dir) - if expected: - pattern = '%s/model.ckpt-.*' % output_dir - primary_ckpt_path = ckpt_state.model_checkpoint_path - self.assertRegexpMatches(primary_ckpt_path, pattern) - all_ckpt_paths = ckpt_state.all_model_checkpoint_paths - self.assertTrue(primary_ckpt_path in all_ckpt_paths) - for ckpt_path in all_ckpt_paths: - self.assertRegexpMatches(ckpt_path, pattern) - else: - self.assertTrue(ckpt_state is None) - - # TODO(ptucker): Test lock, multi-threaded access? - def test_summary_writer(self): - writer = learn.graph_actions.get_summary_writer('log/dir/0') - self._assert_summaries('log/dir/0', writer) - self.assertTrue( - learn.graph_actions.get_summary_writer('log/dir/0') is - learn.graph_actions.get_summary_writer('log/dir/0')) - self.assertTrue( - learn.graph_actions.get_summary_writer('log/dir/0') is - not learn.graph_actions.get_summary_writer('log/dir/1')) - - # TODO(ptucker): Test restore_checkpoint_path for eval; this should obsolete - # test_evaluate_with_saver(). - # TODO(ptucker): Test start_queue_runners for both eval & train. - # TODO(ptucker): Test coord.request_stop & coord.join for eval. - - def _build_inference_graph(self): - """Build simple inference graph. - - This includes a regular variable, local variable, and fake table. - - Returns: - Tuple of 3 `Tensor` objects, 2 input and 1 output. - """ - variables_lib.create_global_step() - in0 = variables.VariableV1(1.0) - in1 = variables_lib.local_variable(2.0) - fake_table = variables.VariableV1( - 3.0, - trainable=False, - collections=['fake_tables'], - name='fake_table_var') - in0.graph.add_to_collections([ops.GraphKeys.TABLE_INITIALIZERS], - fake_table.initializer) - out = in0 + in1 + fake_table - return in0, in1, out - - def test_infer(self): - with ops.Graph().as_default() as g, self.session(g): - self._assert_ckpt(self._output_dir, False) - in0, in1, out = self._build_inference_graph() - self.assertEqual({ - 'a': 1.0, - 'b': 2.0, - 'c': 6.0 - }, learn.graph_actions.infer(None, {'a': in0, - 'b': in1, - 'c': out})) - self._assert_ckpt(self._output_dir, False) - - @test.mock.patch.object( - learn.graph_actions.coordinator.Coordinator, - 'request_stop', - side_effect=learn.graph_actions.coordinator.Coordinator.request_stop, - autospec=True) - def test_coordinator_request_stop_called(self, request_stop): - with ops.Graph().as_default() as g, self.session(g): - in0, in1, out = self._build_inference_graph() - learn.graph_actions.infer(None, {'a': in0, 'b': in1, 'c': out}) - self.assertTrue(request_stop.called) - - @test.mock.patch.object( - learn.graph_actions.coordinator.Coordinator, - 'request_stop', - side_effect=learn.graph_actions.coordinator.Coordinator.request_stop, - autospec=True) - def test_run_feeds_iter_cleanup_with_exceptions(self, request_stop): - with ops.Graph().as_default() as g, self.session(g): - in0, in1, out = self._build_inference_graph() - try: - for _ in learn.graph_actions.run_feeds_iter({ - 'a': in0, - 'b': in1, - 'c': out - }, [None] * 3): - self.assertFalse(request_stop.called) - raise ValueError('Fake exception') - except ValueError: - pass - self.assertTrue(request_stop.called) - - def test_run_feeds_iter_calls_resources_init(self): - with ops.Graph().as_default(): - in0, _, _ = self._build_inference_graph() - handle = test_ops.stub_resource_handle_op(container='a', shared_name='b') - resources.register_resource( - handle=handle, - create_op=test_ops.resource_create_op(handle), - is_initialized_op=test_ops.resource_initialized_op(handle)) - - for _ in learn.graph_actions.run_feeds_iter( - { - 'in0': in0 - }, feed_dicts=[{}]): - self.assertTrue(test_ops.resource_initialized_op(handle).eval()) - - def test_infer_different_default_graph(self): - with self.cached_session(): - self._assert_ckpt(self._output_dir, False) - with ops.Graph().as_default(): - in0, in1, out = self._build_inference_graph() - with ops.Graph().as_default(): - self.assertEqual({ - 'a': 1.0, - 'b': 2.0, - 'c': 6.0 - }, learn.graph_actions.infer(None, {'a': in0, - 'b': in1, - 'c': out})) - self._assert_ckpt(self._output_dir, False) - - def test_infer_invalid_feed(self): - with ops.Graph().as_default() as g, self.session(g): - self._assert_ckpt(self._output_dir, False) - in0, _, _ = self._build_inference_graph() - with self.assertRaisesRegexp(TypeError, 'Can not convert a NoneType'): - learn.graph_actions.infer(None, {'a': in0}, feed_dict={None: 4.0}) - self._assert_ckpt(self._output_dir, False) - - def test_infer_feed(self): - with ops.Graph().as_default() as g, self.session(g): - self._assert_ckpt(self._output_dir, False) - in0, _, out = self._build_inference_graph() - self.assertEqual( - { - 'c': 9.0 - }, - learn.graph_actions.infer( - None, {'c': out}, feed_dict={in0: 4.0})) - self._assert_ckpt(self._output_dir, False) - - # TODO(ptucker): Test eval for 1 epoch. - - def test_evaluate_invalid_args(self): - with ops.Graph().as_default() as g, self.session(g): - self._assert_ckpt(self._output_dir, False) - with self.assertRaisesRegexp(ValueError, 'utput directory'): - learn.graph_actions.evaluate( - g, - output_dir=None, - checkpoint_path=None, - eval_dict={'a': constant_op.constant(1.0)}) - with self.assertRaisesRegexp(ValueError, 'utput directory'): - learn.graph_actions.evaluate( - g, - output_dir='', - checkpoint_path=None, - eval_dict={'a': constant_op.constant(1.0)}) - self._assert_ckpt(self._output_dir, False) - - def test_evaluate(self): - with ops.Graph().as_default() as g, self.session(g): - _, _, out = self._build_inference_graph() - writer = learn.graph_actions.get_summary_writer(self._output_dir) - self._assert_summaries(self._output_dir, writer, expected_session_logs=[]) - self._assert_ckpt(self._output_dir, False) - results = learn.graph_actions.evaluate( - g, - output_dir=self._output_dir, - checkpoint_path=None, - eval_dict={'a': out}, - max_steps=1) - self.assertEqual(({'a': 6.0}, 0), results) - self._assert_summaries( - self._output_dir, - writer, - expected_summaries={0: { - 'a': 6.0 - }}, - expected_session_logs=[]) - self._assert_ckpt(self._output_dir, False) - - def test_evaluate_ready_for_local_init(self): - with ops.Graph().as_default() as g, self.session(g): - variables_lib.create_global_step() - v = variables.VariableV1(1.0) - variables.VariableV1( - v + 1, collections=[ops.GraphKeys.LOCAL_VARIABLES], trainable=False) - ready_for_local_init_op = variables.report_uninitialized_variables( - variables.global_variables()) - ops.add_to_collection(ops.GraphKeys.READY_FOR_LOCAL_INIT_OP, - ready_for_local_init_op) - _ = learn.graph_actions.evaluate( - g, - output_dir=self._output_dir, - checkpoint_path=None, - eval_dict={'a': v}, - max_steps=1) - - def test_evaluate_feed_fn(self): - with ops.Graph().as_default() as g, self.session(g): - in0, _, out = self._build_inference_graph() - writer = learn.graph_actions.get_summary_writer(self._output_dir) - self._assert_summaries(self._output_dir, writer, expected_session_logs=[]) - self._assert_ckpt(self._output_dir, False) - feeder = _Feeder(in0, 3) - results = learn.graph_actions.evaluate( - g, - output_dir=self._output_dir, - checkpoint_path=None, - eval_dict={'a': out}, - feed_fn=feeder.feed_fn, - max_steps=3) - self.assertEqual(3, feeder.step) - self.assertEqual(({'a': 25.0}, 0), results) - self._assert_summaries( - self._output_dir, - writer, - expected_summaries={0: { - 'a': 25.0 - }}, - expected_session_logs=[]) - self._assert_ckpt(self._output_dir, False) - - def test_evaluate_feed_fn_with_exhaustion(self): - with ops.Graph().as_default() as g, self.session(g): - in0, _, out = self._build_inference_graph() - writer = learn.graph_actions.get_summary_writer(self._output_dir) - self._assert_summaries(self._output_dir, writer, expected_session_logs=[]) - feeder = _Feeder(in0, 2) - results = learn.graph_actions.evaluate( - g, - output_dir=self._output_dir, - checkpoint_path=None, - eval_dict={'a': out}, - feed_fn=feeder.feed_fn, - max_steps=3) - self.assertEqual(2, feeder.step) - self.assertEqual(({'a': 15.0}, 0), results) - self._assert_summaries( - self._output_dir, - writer, - expected_summaries={0: { - 'a': 15.0 - }}, - expected_session_logs=[]) - - def test_evaluate_with_saver(self): - with ops.Graph().as_default() as g, self.session(g): - _, _, out = self._build_inference_graph() - ops.add_to_collection(ops.GraphKeys.SAVERS, saver_lib.Saver()) - writer = learn.graph_actions.get_summary_writer(self._output_dir) - self._assert_summaries(self._output_dir, writer, expected_session_logs=[]) - results = learn.graph_actions.evaluate( - g, - output_dir=self._output_dir, - checkpoint_path=None, - eval_dict={'a': out}, - max_steps=1) - self.assertEqual(({'a': 6.0}, 0), results) - self._assert_summaries( - self._output_dir, - writer, - expected_summaries={0: { - 'a': 6.0 - }}, - expected_session_logs=[]) - - # TODO(ptucker): Resume training from previous ckpt. - # TODO(ptucker): !supervisor_is_chief - # TODO(ptucker): Custom init op for training. - # TODO(ptucker): Mock supervisor, and assert all interactions. - - -# TODO(ispir): remove following tests after deprecated train. -class GraphActionsTrainTest(test.TestCase): - """Tests for train.""" - - def setUp(self): - learn.graph_actions.clear_summary_writers() - self._output_dir = tempfile.mkdtemp() - testing.FakeSummaryWriter.install() - - def tearDown(self): - testing.FakeSummaryWriter.uninstall() - if self._output_dir: - shutil.rmtree(self._output_dir) - learn.graph_actions.clear_summary_writers() - - def _assert_summaries(self, - output_dir, - expected_summaries=None, - expected_graphs=None, - expected_meta_graphs=None, - expected_session_logs=None): - writer = learn.graph_actions.get_summary_writer(output_dir) - self.assertTrue(isinstance(writer, testing.FakeSummaryWriter)) - writer.assert_summaries( - self, - expected_logdir=output_dir, - expected_graph=ops.get_default_graph(), - expected_summaries=expected_summaries, - expected_added_graphs=expected_graphs, - expected_added_meta_graphs=expected_meta_graphs, - expected_session_logs=expected_session_logs) - - # TODO(ptucker): Test number and contents of checkpoint files. - def _assert_ckpt(self, output_dir, expected=True): - ckpt_state = checkpoint_management.get_checkpoint_state(output_dir) - if expected: - pattern = '%s/model.ckpt-.*' % output_dir - primary_ckpt_path = ckpt_state.model_checkpoint_path - self.assertRegexpMatches(primary_ckpt_path, pattern) - all_ckpt_paths = ckpt_state.all_model_checkpoint_paths - self.assertTrue(primary_ckpt_path in all_ckpt_paths) - for ckpt_path in all_ckpt_paths: - self.assertRegexpMatches(ckpt_path, pattern) - else: - self.assertTrue(ckpt_state is None) - - def _build_inference_graph(self): - """Build simple inference graph. - - This includes a regular variable, local variable, and fake table. - - Returns: - Tuple of 3 `Tensor` objects, 2 input and 1 output. - """ - variables_lib.create_global_step() - in0 = variables.VariableV1(1.0) - in1 = variables_lib.local_variable(2.0) - fake_table = variables.VariableV1( - 3.0, - trainable=False, - collections=['fake_tables'], - name='fake_table_var') - in0.graph.add_to_collections([ops.GraphKeys.TABLE_INITIALIZERS], - fake_table.initializer) - out = in0 + in1 + fake_table - return in0, in1, out - - def test_train_invalid_args(self): - with ops.Graph().as_default() as g, self.session(g): - train_op = constant_op.constant(1.0) - loss_op = constant_op.constant(2.0) - with self.assertRaisesRegexp(ValueError, 'utput directory'): - learn.graph_actions.train( - g, output_dir=None, train_op=train_op, loss_op=loss_op) - with self.assertRaisesRegexp(ValueError, 'utput directory'): - learn.graph_actions.train( - g, - output_dir='', - train_op=constant_op.constant(1.0), - loss_op=constant_op.constant(2.0)) - with self.assertRaisesRegexp(ValueError, 'train_op'): - learn.graph_actions.train( - g, output_dir=self._output_dir, train_op=None, loss_op=loss_op) - with self.assertRaisesRegexp(ValueError, 'loss_op'): - learn.graph_actions.train( - g, - output_dir=self._output_dir, - train_op=constant_op.constant(1.0), - loss_op=None) - with self.assertRaisesRegexp(ValueError, 'global_step'): - learn.graph_actions.train( - g, - output_dir=self._output_dir, - train_op=constant_op.constant(1.0), - loss_op=loss_op) - - # TODO(ptucker): Resume training from previous ckpt. - # TODO(ptucker): !supervisor_is_chief - # TODO(ptucker): Custom init op for training. - # TODO(ptucker): Mock supervisor, and assert all interactions. - - def test_train(self): - with ops.Graph().as_default() as g, self.session(g): - with ops.control_dependencies(self._build_inference_graph()): - train_op = state_ops.assign_add(variables_lib.get_global_step(), 1) - self._assert_summaries(self._output_dir) - self._assert_ckpt(self._output_dir, False) - loss = learn.graph_actions.train( - g, - output_dir=self._output_dir, - train_op=train_op, - loss_op=constant_op.constant(2.0), - steps=1) - # TODO(ebrevdo,ptucker,ispir): this meta_graph_def lacks the - # SaverDef, so we can't add it to the summary assertion test below. - # meta_graph_def = meta_graph.create_meta_graph_def() - self.assertEqual(2.0, loss) - self._assert_summaries(self._output_dir, expected_graphs=[g]) - self._assert_ckpt(self._output_dir, True) - - def test_train_steps_is_incremental(self): - with ops.Graph().as_default() as g, self.session(g): - with ops.control_dependencies(self._build_inference_graph()): - train_op = state_ops.assign_add(variables_lib.get_global_step(), 1) - learn.graph_actions.train( - g, - output_dir=self._output_dir, - train_op=train_op, - loss_op=constant_op.constant(2.0), - steps=10) - step = checkpoint_utils.load_variable( - self._output_dir, variables_lib.get_global_step().name) - self.assertEqual(10, step) - - with ops.Graph().as_default() as g, self.session(g): - with ops.control_dependencies(self._build_inference_graph()): - train_op = state_ops.assign_add(variables_lib.get_global_step(), 1) - learn.graph_actions.train( - g, - output_dir=self._output_dir, - train_op=train_op, - loss_op=constant_op.constant(2.0), - steps=15) - step = checkpoint_utils.load_variable( - self._output_dir, variables_lib.get_global_step().name) - self.assertEqual(25, step) - - def test_train_max_steps_is_not_incremental(self): - with ops.Graph().as_default() as g, self.session(g): - with ops.control_dependencies(self._build_inference_graph()): - train_op = state_ops.assign_add(variables_lib.get_global_step(), 1) - learn.graph_actions.train( - g, - output_dir=self._output_dir, - train_op=train_op, - loss_op=constant_op.constant(2.0), - max_steps=10) - step = checkpoint_utils.load_variable( - self._output_dir, variables_lib.get_global_step().name) - self.assertEqual(10, step) - - with ops.Graph().as_default() as g, self.session(g): - with ops.control_dependencies(self._build_inference_graph()): - train_op = state_ops.assign_add(variables_lib.get_global_step(), 1) - learn.graph_actions.train( - g, - output_dir=self._output_dir, - train_op=train_op, - loss_op=constant_op.constant(2.0), - max_steps=15) - step = checkpoint_utils.load_variable( - self._output_dir, variables_lib.get_global_step().name) - self.assertEqual(15, step) - - def test_train_loss(self): - with ops.Graph().as_default() as g, self.session(g): - variables_lib.create_global_step() - loss_var = variables_lib.local_variable(10.0) - train_op = control_flow_ops.group( - state_ops.assign_add(variables_lib.get_global_step(), 1), - state_ops.assign_add(loss_var, -1.0)) - self._assert_summaries(self._output_dir) - self._assert_ckpt(self._output_dir, False) - loss = learn.graph_actions.train( - g, - output_dir=self._output_dir, - train_op=train_op, - loss_op=loss_var.value(), - steps=6) - # TODO(ebrevdo,ptucker,ispir): this meta_graph_def lacks the - # SaverDef, so we can't add it to the summary assertion test below. - # meta_graph_def = meta_graph.create_meta_graph_def() - self.assertEqual(4.0, loss) - self._assert_summaries(self._output_dir, expected_graphs=[g]) - self._assert_ckpt(self._output_dir, True) - - def test_train_summaries(self): - with ops.Graph().as_default() as g, self.session(g): - with ops.control_dependencies(self._build_inference_graph()): - train_op = state_ops.assign_add(variables_lib.get_global_step(), 1) - loss_op = constant_op.constant(2.0) - summary.scalar('loss', loss_op) - self._assert_summaries(self._output_dir) - self._assert_ckpt(self._output_dir, False) - loss = learn.graph_actions.train( - g, - output_dir=self._output_dir, - train_op=train_op, - loss_op=loss_op, - steps=1) - # TODO(ebrevdo,ptucker,ispir): this meta_graph_def lacks the - # SaverDef, so we can't add it to the summary assertion test below. - # meta_graph_def = meta_graph.create_meta_graph_def() - self.assertEqual(2.0, loss) - self._assert_summaries( - self._output_dir, - expected_graphs=[g], - expected_summaries={1: { - 'loss': 2.0 - }}) - self._assert_ckpt(self._output_dir, True) - - def test_train_chief_monitor(self): - with ops.Graph().as_default() as g, self.session(g): - with ops.control_dependencies(self._build_inference_graph()): - train_op = state_ops.assign_add(variables_lib.get_global_step(), 1) - loss_op = constant_op.constant(2.0) - summary.scalar('loss', loss_op) - chief_exclusive_monitor = _BaseMonitorWrapper(False) - all_workers_monitor = _BaseMonitorWrapper(True) - loss = learn.graph_actions.train( - g, - output_dir=self._output_dir, - train_op=train_op, - loss_op=loss_op, - supervisor_is_chief=True, - steps=1, - monitors=[chief_exclusive_monitor, all_workers_monitor]) - self.assertEqual(2.0, loss) - self.assertTrue(chief_exclusive_monitor.is_active and - all_workers_monitor.is_active, - 'All monitors must have been active.') - self.assertTrue(chief_exclusive_monitor.has_step and - all_workers_monitor.has_step, - 'All monitors must have a step.') - - def test_train_worker_monitor(self): - # We need to explicitly set device due to check on non-chief workers - # requiring all variables to have a device assigned. - with ops.Graph().as_default() as g, g.device('/cpu:0'): - global_step = variables_lib.create_global_step(g) - train_op = state_ops.assign_add(global_step, 1) - loss_op = constant_op.constant(2.0) - summary.scalar('loss', loss_op) - # Add explicit "local" init op to initialize all variables - # as there's no chief to init here. - init_op = variables.global_variables_initializer() - ops.add_to_collection(ops.GraphKeys.LOCAL_INIT_OP, init_op) - # Create worker monitors where one should be active on the worker - # and the other chief exclusive. - chief_exclusive_monitor = _BaseMonitorWrapper(False) - all_workers_monitor = _BaseMonitorWrapper(True) - with self.session(g): - loss = learn.graph_actions.train( - g, - output_dir=self._output_dir, - global_step_tensor=global_step, - train_op=train_op, - loss_op=loss_op, - supervisor_is_chief=False, - steps=1, - monitors=[chief_exclusive_monitor, all_workers_monitor]) - self.assertEqual(2.0, loss) - self.assertTrue(not chief_exclusive_monitor.is_active and - all_workers_monitor.is_active, - 'Only non-chief runnable monitor must have been active.') - self.assertTrue(not chief_exclusive_monitor.has_step and - all_workers_monitor.has_step, - 'Only non-chief runnable monitor must have a step.') - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/grid_search_test.py b/tensorflow/contrib/learn/python/learn/grid_search_test.py deleted file mode 100644 index b7c3e21dee7..00000000000 --- a/tensorflow/contrib/learn/python/learn/grid_search_test.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Grid search tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import random - -from tensorflow.contrib.learn.python import learn -from tensorflow.python.platform import test - -HAS_SKLEARN = os.environ.get('TENSORFLOW_SKLEARN', False) -if HAS_SKLEARN: - try: - # pylint: disable=g-import-not-at-top - from sklearn import datasets - from sklearn.grid_search import GridSearchCV - from sklearn.metrics import accuracy_score - except ImportError: - HAS_SKLEARN = False - - -class GridSearchTest(test.TestCase): - """Grid search tests.""" - - def testIrisDNN(self): - if HAS_SKLEARN: - random.seed(42) - iris = datasets.load_iris() - feature_columns = learn.infer_real_valued_columns_from_input(iris.data) - classifier = learn.DNNClassifier( - feature_columns=feature_columns, - hidden_units=[10, 20, 10], - n_classes=3) - grid_search = GridSearchCV( - classifier, {'hidden_units': [[5, 5], [10, 10]]}, - scoring='accuracy', - fit_params={'steps': [50]}) - grid_search.fit(iris.data, iris.target) - score = accuracy_score(iris.target, grid_search.predict(iris.data)) - self.assertGreater(score, 0.5, 'Failed with score = {0}'.format(score)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/learn_io/__init__.py b/tensorflow/contrib/learn/python/learn/learn_io/__init__.py deleted file mode 100644 index 8b133a4440d..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_io/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tools to allow different io formats (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.learn_io.dask_io import extract_dask_data -from tensorflow.contrib.learn.python.learn.learn_io.dask_io import extract_dask_labels -from tensorflow.contrib.learn.python.learn.learn_io.dask_io import HAS_DASK -from tensorflow.contrib.learn.python.learn.learn_io.graph_io import queue_parsed_features -from tensorflow.contrib.learn.python.learn.learn_io.graph_io import read_batch_examples -from tensorflow.contrib.learn.python.learn.learn_io.graph_io import read_batch_features -from tensorflow.contrib.learn.python.learn.learn_io.graph_io import read_batch_record_features -from tensorflow.contrib.learn.python.learn.learn_io.graph_io import read_keyed_batch_examples -from tensorflow.contrib.learn.python.learn.learn_io.graph_io import read_keyed_batch_examples_shared_queue -from tensorflow.contrib.learn.python.learn.learn_io.graph_io import read_keyed_batch_features -from tensorflow.contrib.learn.python.learn.learn_io.graph_io import read_keyed_batch_features_shared_queue -from tensorflow.contrib.learn.python.learn.learn_io.numpy_io import numpy_input_fn -from tensorflow.contrib.learn.python.learn.learn_io.pandas_io import extract_pandas_data -from tensorflow.contrib.learn.python.learn.learn_io.pandas_io import extract_pandas_labels -from tensorflow.contrib.learn.python.learn.learn_io.pandas_io import extract_pandas_matrix -from tensorflow.contrib.learn.python.learn.learn_io.pandas_io import HAS_PANDAS -from tensorflow.contrib.learn.python.learn.learn_io.pandas_io import pandas_input_fn -from tensorflow.contrib.learn.python.learn.learn_io.generator_io import generator_input_fn diff --git a/tensorflow/contrib/learn/python/learn/learn_io/dask_io.py b/tensorflow/contrib/learn/python/learn/learn_io/dask_io.py deleted file mode 100644 index e0a1948d95a..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_io/dask_io.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Methods to allow dask.DataFrame (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.util.deprecation import deprecated - -try: - # pylint: disable=g-import-not-at-top - import dask.dataframe as dd - allowed_classes = (dd.Series, dd.DataFrame) - HAS_DASK = True -except ImportError: - HAS_DASK = False - - -def _add_to_index(df, start): - """New dask.dataframe with values added to index of each subdataframe.""" - df = df.copy() - df.index += start - return df - - -def _get_divisions(df): - """Number of rows in each sub-dataframe.""" - lengths = df.map_partitions(len).compute() - divisions = np.cumsum(lengths).tolist() - divisions.insert(0, 0) - return divisions - - -def _construct_dask_df_with_divisions(df): - """Construct the new task graph and make a new dask.dataframe around it.""" - divisions = _get_divisions(df) - # pylint: disable=protected-access - name = 'csv-index' + df._name - dsk = {(name, i): (_add_to_index, (df._name, i), divisions[i]) - for i in range(df.npartitions)} - # pylint: enable=protected-access - from toolz import merge # pylint: disable=g-import-not-at-top - if isinstance(df, dd.DataFrame): - return dd.DataFrame(merge(dsk, df.dask), name, df.columns, divisions) - elif isinstance(df, dd.Series): - return dd.Series(merge(dsk, df.dask), name, df.name, divisions) - - -@deprecated(None, 'Please feed input to tf.data to support dask.') -def extract_dask_data(data): - """Extract data from dask.Series or dask.DataFrame for predictors. - - Given a distributed dask.DataFrame or dask.Series containing columns or names - for one or more predictors, this operation returns a single dask.DataFrame or - dask.Series that can be iterated over. - - Args: - data: A distributed dask.DataFrame or dask.Series. - - Returns: - A dask.DataFrame or dask.Series that can be iterated over. - If the supplied argument is neither a dask.DataFrame nor a dask.Series this - operation returns it without modification. - """ - if isinstance(data, allowed_classes): - return _construct_dask_df_with_divisions(data) - else: - return data - - -@deprecated(None, 'Please feed input to tf.data to support dask.') -def extract_dask_labels(labels): - """Extract data from dask.Series or dask.DataFrame for labels. - - Given a distributed dask.DataFrame or dask.Series containing exactly one - column or name, this operation returns a single dask.DataFrame or dask.Series - that can be iterated over. - - Args: - labels: A distributed dask.DataFrame or dask.Series with exactly one - column or name. - - Returns: - A dask.DataFrame or dask.Series that can be iterated over. - If the supplied argument is neither a dask.DataFrame nor a dask.Series this - operation returns it without modification. - - Raises: - ValueError: If the supplied dask.DataFrame contains more than one - column or the supplied dask.Series contains more than - one name. - """ - if isinstance(labels, dd.DataFrame): - ncol = labels.columns - elif isinstance(labels, dd.Series): - ncol = labels.name - if isinstance(labels, allowed_classes): - if len(ncol) > 1: - raise ValueError('Only one column for labels is allowed.') - return _construct_dask_df_with_divisions(labels) - else: - return labels diff --git a/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py b/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py deleted file mode 100644 index b7dba0f2775..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py +++ /dev/null @@ -1,919 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implementations of different data feeders to provide data for TF trainer (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -# TODO(ipolosukhin): Replace this module with feed-dict queue runners & queues. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools -import math - -import numpy as np -import six -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util.deprecation import deprecated - -# pylint: disable=g-multiple-import,g-bad-import-order -from .pandas_io import HAS_PANDAS, extract_pandas_data, extract_pandas_matrix, extract_pandas_labels -from .dask_io import HAS_DASK, extract_dask_data, extract_dask_labels - -# pylint: enable=g-multiple-import,g-bad-import-order - - -def _get_in_out_shape(x_shape, y_shape, n_classes, batch_size=None): - """Returns shape for input and output of the data feeder.""" - x_is_dict, y_is_dict = isinstance( - x_shape, dict), y_shape is not None and isinstance(y_shape, dict) - if y_is_dict and n_classes is not None: - assert isinstance(n_classes, dict) - - if batch_size is None: - batch_size = list(x_shape.values())[0][0] if x_is_dict else x_shape[0] - elif batch_size <= 0: - raise ValueError('Invalid batch_size %d.' % batch_size) - - if x_is_dict: - input_shape = {} - for k, v in list(x_shape.items()): - input_shape[k] = [batch_size] + (list(v[1:]) if len(v) > 1 else [1]) - else: - x_shape = list(x_shape[1:]) if len(x_shape) > 1 else [1] - input_shape = [batch_size] + x_shape - - if y_shape is None: - return input_shape, None, batch_size - - def out_el_shape(out_shape, num_classes): - out_shape = list(out_shape[1:]) if len(out_shape) > 1 else [] - # Skip first dimension if it is 1. - if out_shape and out_shape[0] == 1: - out_shape = out_shape[1:] - if num_classes is not None and num_classes > 1: - return [batch_size] + out_shape + [num_classes] - else: - return [batch_size] + out_shape - - if not y_is_dict: - output_shape = out_el_shape(y_shape, n_classes) - else: - output_shape = dict([(k, - out_el_shape(v, n_classes[k] - if n_classes is not None and - k in n_classes else None)) - for k, v in list(y_shape.items())]) - - return input_shape, output_shape, batch_size - - -def _data_type_filter(x, y): - """Filter data types into acceptable format.""" - if HAS_DASK: - x = extract_dask_data(x) - if y is not None: - y = extract_dask_labels(y) - if HAS_PANDAS: - x = extract_pandas_data(x) - if y is not None: - y = extract_pandas_labels(y) - return x, y - - -def _is_iterable(x): - return hasattr(x, 'next') or hasattr(x, '__next__') - - -@deprecated(None, 'Please use tensorflow/transform or tf.data.') -def setup_train_data_feeder(x, - y, - n_classes, - batch_size=None, - shuffle=True, - epochs=None): - """Create data feeder, to sample inputs from dataset. - - If `x` and `y` are iterators, use `StreamingDataFeeder`. - - Args: - x: numpy, pandas or Dask matrix or dictionary of aforementioned. Also - supports iterables. - y: numpy, pandas or Dask array or dictionary of aforementioned. Also - supports - iterables. - n_classes: number of classes. Must be None or same type as y. In case, `y` - is `dict` - (or iterable which returns dict) such that `n_classes[key] = n_classes for - y[key]` - batch_size: size to split data into parts. Must be >= 1. - shuffle: Whether to shuffle the inputs. - epochs: Number of epochs to run. - - Returns: - DataFeeder object that returns training data. - - Raises: - ValueError: if one of `x` and `y` is iterable and the other is not. - """ - x, y = _data_type_filter(x, y) - if HAS_DASK: - # pylint: disable=g-import-not-at-top - import dask.dataframe as dd - if (isinstance(x, (dd.Series, dd.DataFrame)) and - (y is None or isinstance(y, (dd.Series, dd.DataFrame)))): - data_feeder_cls = DaskDataFeeder - else: - data_feeder_cls = DataFeeder - else: - data_feeder_cls = DataFeeder - - if _is_iterable(x): - if y is not None and not _is_iterable(y): - raise ValueError('Both x and y should be iterators for ' - 'streaming learning to work.') - return StreamingDataFeeder(x, y, n_classes, batch_size) - return data_feeder_cls( - x, y, n_classes, batch_size, shuffle=shuffle, epochs=epochs) - - -def _batch_data(x, batch_size=None): - if (batch_size is not None) and (batch_size <= 0): - raise ValueError('Invalid batch_size %d.' % batch_size) - - x_first_el = six.next(x) - x = itertools.chain([x_first_el], x) - - chunk = dict([(k, []) for k in list(x_first_el.keys())]) if isinstance( - x_first_el, dict) else [] - chunk_filled = False - for data in x: - if isinstance(data, dict): - for k, v in list(data.items()): - chunk[k].append(v) - if (batch_size is not None) and (len(chunk[k]) >= batch_size): - chunk[k] = np.matrix(chunk[k]) - chunk_filled = True - if chunk_filled: - yield chunk - chunk = dict([(k, []) for k in list(x_first_el.keys())]) if isinstance( - x_first_el, dict) else [] - chunk_filled = False - else: - chunk.append(data) - if (batch_size is not None) and (len(chunk) >= batch_size): - yield np.matrix(chunk) - chunk = [] - - if isinstance(x_first_el, dict): - for k, v in list(data.items()): - chunk[k] = np.matrix(chunk[k]) - yield chunk - else: - yield np.matrix(chunk) - - -@deprecated(None, 'Please use tensorflow/transform or tf.data.') -def setup_predict_data_feeder(x, batch_size=None): - """Returns an iterable for feeding into predict step. - - Args: - x: numpy, pandas, Dask array or dictionary of aforementioned. Also supports - iterable. - batch_size: Size of batches to split data into. If `None`, returns one - batch of full size. - - Returns: - List or iterator (or dictionary thereof) of parts of data to predict on. - - Raises: - ValueError: if `batch_size` <= 0. - """ - if HAS_DASK: - x = extract_dask_data(x) - if HAS_PANDAS: - x = extract_pandas_data(x) - if _is_iterable(x): - return _batch_data(x, batch_size) - if len(x.shape) == 1: - x = np.reshape(x, (-1, 1)) - if batch_size is not None: - if batch_size <= 0: - raise ValueError('Invalid batch_size %d.' % batch_size) - n_batches = int(math.ceil(float(len(x)) / batch_size)) - return [x[i * batch_size:(i + 1) * batch_size] for i in xrange(n_batches)] - return [x] - - -@deprecated(None, 'Please use tensorflow/transform or tf.data.') -def setup_processor_data_feeder(x): - """Sets up processor iterable. - - Args: - x: numpy, pandas or iterable. - - Returns: - Iterable of data to process. - """ - if HAS_PANDAS: - x = extract_pandas_matrix(x) - return x - - -@deprecated(None, 'Please convert numpy dtypes explicitly.') -def check_array(array, dtype): - """Checks array on dtype and converts it if different. - - Args: - array: Input array. - dtype: Expected dtype. - - Returns: - Original array or converted. - """ - # skip check if array is instance of other classes, e.g. h5py.Dataset - # to avoid copying array and loading whole data into memory - if isinstance(array, (np.ndarray, list)): - array = np.array(array, dtype=dtype, order=None, copy=False) - return array - - -def _access(data, iloc): - """Accesses an element from collection, using integer location based indexing. - - Args: - data: array-like. The collection to access - iloc: `int` or `list` of `int`s. Location(s) to access in `collection` - - Returns: - The element of `a` found at location(s) `iloc`. - """ - if HAS_PANDAS: - import pandas as pd # pylint: disable=g-import-not-at-top - if isinstance(data, pd.Series) or isinstance(data, pd.DataFrame): - return data.iloc[iloc] - return data[iloc] - - -def _check_dtype(dtype): - if dtypes.as_dtype(dtype) == dtypes.float64: - logging.warn( - 'float64 is not supported by many models, consider casting to float32.') - return dtype - - -class DataFeeder(object): - """Data feeder is an example class to sample data for TF trainer. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - @deprecated(None, 'Please use tensorflow/transform or tf.data.') - def __init__(self, - x, - y, - n_classes, - batch_size=None, - shuffle=True, - random_state=None, - epochs=None): - """Initializes a DataFeeder instance. - - Args: - x: One feature sample which can either Nd numpy matrix of shape - `[n_samples, n_features, ...]` or dictionary of Nd numpy matrix. - y: label vector, either floats for regression or class id for - classification. If matrix, will consider as a sequence of labels. - Can be `None` for unsupervised setting. Also supports dictionary of - labels. - n_classes: Number of classes, 0 and 1 are considered regression, `None` - will pass through the input labels without one-hot conversion. Also, if - `y` is `dict`, then `n_classes` must be `dict` such that - `n_classes[key] = n_classes for label y[key]`, `None` otherwise. - batch_size: Mini-batch size to accumulate samples in one mini batch. - shuffle: Whether to shuffle `x`. - random_state: Numpy `RandomState` object to reproduce sampling. - epochs: Number of times to iterate over input data before raising - `StopIteration` exception. - - Attributes: - x: Input features (ndarray or dictionary of ndarrays). - y: Input label (ndarray or dictionary of ndarrays). - n_classes: Number of classes (if `None`, pass through indices without - one-hot conversion). - batch_size: Mini-batch size to accumulate. - input_shape: Shape of the input (or dictionary of shapes). - output_shape: Shape of the output (or dictionary of shapes). - input_dtype: DType of input (or dictionary of shapes). - output_dtype: DType of output (or dictionary of shapes. - """ - x_is_dict, y_is_dict = isinstance( - x, dict), y is not None and isinstance(y, dict) - if isinstance(y, list): - y = np.array(y) - - self._x = dict([(k, check_array(v, v.dtype)) for k, v in list(x.items()) - ]) if x_is_dict else check_array(x, x.dtype) - self._y = None if y is None else (dict( - [(k, check_array(v, v.dtype)) for k, v in list(y.items())]) - if y_is_dict else check_array(y, y.dtype)) - - # self.n_classes is not None means we're converting raw target indices - # to one-hot. - if n_classes is not None: - if not y_is_dict: - y_dtype = ( - np.int64 if n_classes is not None and n_classes > 1 else np.float32) - self._y = (None if y is None else check_array(y, dtype=y_dtype)) - - self.n_classes = n_classes - self.max_epochs = epochs - - x_shape = dict([(k, v.shape) for k, v in list(self._x.items()) - ]) if x_is_dict else self._x.shape - y_shape = dict([(k, v.shape) for k, v in list(self._y.items()) - ]) if y_is_dict else None if y is None else self._y.shape - - self.input_shape, self.output_shape, self._batch_size = _get_in_out_shape( - x_shape, y_shape, n_classes, batch_size) - - # Input dtype matches dtype of x. - self._input_dtype = ( - dict([(k, _check_dtype(v.dtype)) for k, v in list(self._x.items())]) - if x_is_dict else _check_dtype(self._x.dtype)) - - # self._output_dtype == np.float32 when y is None - self._output_dtype = ( - dict([(k, _check_dtype(v.dtype)) for k, v in list(self._y.items())]) - if y_is_dict else (_check_dtype(self._y.dtype) - if y is not None else np.float32)) - - # self.n_classes is None means we're passing in raw target indices - if n_classes is not None and y_is_dict: - for key in list(n_classes.keys()): - if key in self._output_dtype: - self._output_dtype[key] = np.float32 - - self._shuffle = shuffle - self.random_state = np.random.RandomState( - 42) if random_state is None else random_state - - if x_is_dict: - num_samples = list(self._x.values())[0].shape[0] - elif tensor_util.is_tensor(self._x): - num_samples = self._x.shape[ - 0].value # shape will be a Dimension, extract an int - else: - num_samples = self._x.shape[0] - - if self._shuffle: - self.indices = self.random_state.permutation(num_samples) - else: - self.indices = np.array(range(num_samples)) - self.offset = 0 - self.epoch = 0 - self._epoch_placeholder = None - - @property - def x(self): - return self._x - - @property - def y(self): - return self._y - - @property - def shuffle(self): - return self._shuffle - - @property - def input_dtype(self): - return self._input_dtype - - @property - def output_dtype(self): - return self._output_dtype - - @property - def batch_size(self): - return self._batch_size - - def make_epoch_variable(self): - """Adds a placeholder variable for the epoch to the graph. - - Returns: - The epoch placeholder. - """ - self._epoch_placeholder = array_ops.placeholder( - dtypes.int32, [1], name='epoch') - return self._epoch_placeholder - - def input_builder(self): - """Builds inputs in the graph. - - Returns: - Two placeholders for inputs and outputs. - """ - - def get_placeholder(shape, dtype, name_prepend): - if shape is None: - return None - if isinstance(shape, dict): - placeholder = {} - for key in list(shape.keys()): - placeholder[key] = array_ops.placeholder( - dtypes.as_dtype(dtype[key]), [None] + shape[key][1:], - name=name_prepend + '_' + key) - else: - placeholder = array_ops.placeholder( - dtypes.as_dtype(dtype), [None] + shape[1:], name=name_prepend) - return placeholder - - self._input_placeholder = get_placeholder(self.input_shape, - self._input_dtype, 'input') - self._output_placeholder = get_placeholder(self.output_shape, - self._output_dtype, 'output') - return self._input_placeholder, self._output_placeholder - - def set_placeholders(self, input_placeholder, output_placeholder): - """Sets placeholders for this data feeder. - - Args: - input_placeholder: Placeholder for `x` variable. Should match shape - of the examples in the x dataset. - output_placeholder: Placeholder for `y` variable. Should match - shape of the examples in the y dataset. Can be `None`. - """ - self._input_placeholder = input_placeholder - self._output_placeholder = output_placeholder - - def get_feed_params(self): - """Function returns a `dict` with data feed params while training. - - Returns: - A `dict` with data feed params while training. - """ - return { - 'epoch': self.epoch, - 'offset': self.offset, - 'batch_size': self._batch_size - } - - def get_feed_dict_fn(self): - """Returns a function that samples data into given placeholders. - - Returns: - A function that when called samples a random subset of batch size - from `x` and `y`. - """ - x_is_dict, y_is_dict = isinstance( - self._x, dict), self._y is not None and isinstance(self._y, dict) - - # Assign input features from random indices. - def extract(data, indices): - return (np.array(_access(data, indices)).reshape((indices.shape[0], 1)) - if len(data.shape) == 1 else _access(data, indices)) - - # assign labels from random indices - def assign_label(data, shape, dtype, n_classes, indices): - shape[0] = indices.shape[0] - out = np.zeros(shape, dtype=dtype) - for i in xrange(out.shape[0]): - sample = indices[i] - # self.n_classes is None means we're passing in raw target indices - if n_classes is None: - out[i] = _access(data, sample) - else: - if n_classes > 1: - if len(shape) == 2: - out.itemset((i, int(_access(data, sample))), 1.0) - else: - for idx, value in enumerate(_access(data, sample)): - out.itemset(tuple([i, idx, value]), 1.0) - else: - out[i] = _access(data, sample) - return out - - def _feed_dict_fn(): - """Function that samples data into given placeholders.""" - if self.max_epochs is not None and self.epoch + 1 > self.max_epochs: - raise StopIteration - assert self._input_placeholder is not None - feed_dict = {} - if self._epoch_placeholder is not None: - feed_dict[self._epoch_placeholder.name] = [self.epoch] - - # Take next batch of indices. - x_len = list( - self._x.values())[0].shape[0] if x_is_dict else self._x.shape[0] - end = min(x_len, self.offset + self._batch_size) - batch_indices = self.indices[self.offset:end] - - # adding input placeholder - feed_dict.update( - dict([(self._input_placeholder[k].name, extract(v, batch_indices)) - for k, v in list(self._x.items())]) if x_is_dict else { - self._input_placeholder.name: - extract(self._x, batch_indices) - }) - - # move offset and reset it if necessary - self.offset += self._batch_size - if self.offset >= x_len: - self.indices = self.random_state.permutation( - x_len) if self._shuffle else np.array(range(x_len)) - self.offset = 0 - self.epoch += 1 - - # return early if there are no labels - if self._output_placeholder is None: - return feed_dict - - # adding output placeholders - if y_is_dict: - for k, v in list(self._y.items()): - n_classes = (self.n_classes[k] if k in self.n_classes else - None) if self.n_classes is not None else None - shape, dtype = self.output_shape[k], self._output_dtype[k] - feed_dict.update({ - self._output_placeholder[k].name: - assign_label(v, shape, dtype, n_classes, batch_indices) - }) - else: - shape, dtype, n_classes = (self.output_shape, self._output_dtype, - self.n_classes) - feed_dict.update({ - self._output_placeholder.name: - assign_label(self._y, shape, dtype, n_classes, batch_indices) - }) - - return feed_dict - - return _feed_dict_fn - - -class StreamingDataFeeder(DataFeeder): - """Data feeder for TF trainer that reads data from iterator. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Streaming data feeder allows to read data as it comes it from disk or - somewhere else. It's custom to have this iterators rotate infinetly over - the dataset, to allow control of how much to learn on the trainer side. - """ - - def __init__(self, x, y, n_classes, batch_size): - """Initializes a StreamingDataFeeder instance. - - Args: - x: iterator each element of which returns one feature sample. Sample can - be a Nd numpy matrix or dictionary of Nd numpy matrices. - y: iterator each element of which returns one label sample. Sample can be - a Nd numpy matrix or dictionary of Nd numpy matrices with 1 or many - classes regression values. - n_classes: indicator of how many classes the corresponding label sample - has for the purposes of one-hot conversion of label. In case where `y` - is a dictionary, `n_classes` must be dictionary (with same keys as `y`) - of how many classes there are in each label in `y`. If key is - present in `y` and missing in `n_classes`, the value is assumed `None` - and no one-hot conversion will be applied to the label with that key. - batch_size: Mini batch size to accumulate samples in one batch. If set - `None`, then assumes that iterator to return already batched element. - - Attributes: - x: input features (or dictionary of input features). - y: input label (or dictionary of output features). - n_classes: number of classes. - batch_size: mini batch size to accumulate. - input_shape: shape of the input (can be dictionary depending on `x`). - output_shape: shape of the output (can be dictionary depending on `y`). - input_dtype: dtype of input (can be dictionary depending on `x`). - output_dtype: dtype of output (can be dictionary depending on `y`). - """ - # pylint: disable=invalid-name,super-init-not-called - x_first_el = six.next(x) - self._x = itertools.chain([x_first_el], x) - if y is not None: - y_first_el = six.next(y) - self._y = itertools.chain([y_first_el], y) - else: - y_first_el = None - self._y = None - self.n_classes = n_classes - - x_is_dict = isinstance(x_first_el, dict) - y_is_dict = y is not None and isinstance(y_first_el, dict) - if y_is_dict and n_classes is not None: - assert isinstance(n_classes, dict) - - # extract shapes for first_elements - if x_is_dict: - x_first_el_shape = dict( - [(k, [1] + list(v.shape)) for k, v in list(x_first_el.items())]) - else: - x_first_el_shape = [1] + list(x_first_el.shape) - - if y_is_dict: - y_first_el_shape = dict( - [(k, [1] + list(v.shape)) for k, v in list(y_first_el.items())]) - elif y is None: - y_first_el_shape = None - else: - y_first_el_shape = ( - [1] + list(y_first_el[0].shape - if isinstance(y_first_el, list) else y_first_el.shape)) - - self.input_shape, self.output_shape, self._batch_size = _get_in_out_shape( - x_first_el_shape, y_first_el_shape, n_classes, batch_size) - - # Input dtype of x_first_el. - if x_is_dict: - self._input_dtype = dict( - [(k, _check_dtype(v.dtype)) for k, v in list(x_first_el.items())]) - else: - self._input_dtype = _check_dtype(x_first_el.dtype) - - # Output dtype of y_first_el. - def check_y_dtype(el): - if isinstance(el, np.ndarray): - return el.dtype - elif isinstance(el, list): - return check_y_dtype(el[0]) - else: - return _check_dtype(np.dtype(type(el))) - - # Output types are floats, due to both softmaxes and regression req. - if n_classes is not None and (y is None or not y_is_dict) and n_classes > 0: - self._output_dtype = np.float32 - elif y_is_dict: - self._output_dtype = dict( - [(k, check_y_dtype(v)) for k, v in list(y_first_el.items())]) - elif y is None: - self._output_dtype = None - else: - self._output_dtype = check_y_dtype(y_first_el) - - def get_feed_params(self): - """Function returns a `dict` with data feed params while training. - - Returns: - A `dict` with data feed params while training. - """ - return {'batch_size': self._batch_size} - - def get_feed_dict_fn(self): - """Returns a function, that will sample data and provide it to placeholders. - - Returns: - A function that when called samples a random subset of batch size - from x and y. - """ - self.stopped = False - - def _feed_dict_fn(): - """Samples data and provides it to placeholders. - - Returns: - `dict` of input and output tensors. - """ - - def init_array(shape, dtype): - """Initialize array of given shape or dict of shapes and dtype.""" - if shape is None: - return None - elif isinstance(shape, dict): - return dict( - [(k, np.zeros(shape[k], dtype[k])) for k in list(shape.keys())]) - else: - return np.zeros(shape, dtype=dtype) - - def put_data_array(dest, index, source=None, n_classes=None): - """Puts data array into container.""" - if source is None: - dest = dest[:index] - elif n_classes is not None and n_classes > 1: - if len(self.output_shape) == 2: - dest.itemset((index, source), 1.0) - else: - for idx, value in enumerate(source): - dest.itemset(tuple([index, idx, value]), 1.0) - else: - if len(dest.shape) > 1: - dest[index, :] = source - else: - dest[index] = source[0] if isinstance(source, list) else source - return dest - - def put_data_array_or_dict(holder, index, data=None, n_classes=None): - """Puts data array or data dictionary into container.""" - if holder is None: - return None - if isinstance(holder, dict): - if data is None: - data = {k: None for k in holder.keys()} - assert isinstance(data, dict) - for k in holder.keys(): - num_classes = n_classes[k] if (n_classes is not None and - k in n_classes) else None - holder[k] = put_data_array(holder[k], index, data[k], num_classes) - else: - holder = put_data_array(holder, index, data, n_classes) - return holder - - if self.stopped: - raise StopIteration - - inp = init_array(self.input_shape, self._input_dtype) - out = init_array(self.output_shape, self._output_dtype) - - for i in xrange(self._batch_size): - # Add handling when queue ends. - try: - next_inp = six.next(self._x) - inp = put_data_array_or_dict(inp, i, next_inp, None) - except StopIteration: - self.stopped = True - if i == 0: - raise - inp = put_data_array_or_dict(inp, i, None, None) - out = put_data_array_or_dict(out, i, None, None) - break - - if self._y is not None: - next_out = six.next(self._y) - out = put_data_array_or_dict(out, i, next_out, self.n_classes) - - # creating feed_dict - if isinstance(inp, dict): - feed_dict = dict([(self._input_placeholder[k].name, inp[k]) - for k in list(self._input_placeholder.keys())]) - else: - feed_dict = {self._input_placeholder.name: inp} - if self._y is not None: - if isinstance(out, dict): - feed_dict.update( - dict([(self._output_placeholder[k].name, out[k]) - for k in list(self._output_placeholder.keys())])) - else: - feed_dict.update({self._output_placeholder.name: out}) - - return feed_dict - - return _feed_dict_fn - - -class DaskDataFeeder(object): - """Data feeder for that reads data from dask.Series and dask.DataFrame. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Numpy arrays can be serialized to disk and it's possible to do random seeks - into them. DaskDataFeeder will remove requirement to have full dataset in the - memory and still do random seeks for sampling of batches. - """ - - @deprecated(None, 'Please feed input to tf.data to support dask.') - def __init__(self, - x, - y, - n_classes, - batch_size, - shuffle=True, - random_state=None, - epochs=None): - """Initializes a DaskDataFeeder instance. - - Args: - x: iterator that returns for each element, returns features. - y: iterator that returns for each element, returns 1 or many classes / - regression values. - n_classes: indicator of how many classes the label has. - batch_size: Mini batch size to accumulate. - shuffle: Whether to shuffle the inputs. - random_state: random state for RNG. Note that it will mutate so use a - int value for this if you want consistent sized batches. - epochs: Number of epochs to run. - - Attributes: - x: input features. - y: input label. - n_classes: number of classes. - batch_size: mini batch size to accumulate. - input_shape: shape of the input. - output_shape: shape of the output. - input_dtype: dtype of input. - output_dtype: dtype of output. - - Raises: - ValueError: if `x` or `y` are `dict`, as they are not supported currently. - """ - - if isinstance(x, dict) or isinstance(y, dict): - raise ValueError( - 'DaskDataFeeder does not support dictionaries at the moment.') - - # pylint: disable=invalid-name,super-init-not-called - import dask.dataframe as dd # pylint: disable=g-import-not-at-top - # TODO(terrytangyuan): check x and y dtypes in dask_io like pandas - self._x = x - self._y = y - # save column names - self._x_columns = list(x.columns) - if isinstance(y.columns[0], str): - self._y_columns = list(y.columns) - else: - # deal with cases where two DFs have overlapped default numeric colnames - self._y_columns = len(self._x_columns) + 1 - self._y = self._y.rename(columns={y.columns[0]: self._y_columns}) - - # TODO(terrytangyuan): deal with unsupervised cases - # combine into a data frame - self.df = dd.multi.concat([self._x, self._y], axis=1) - self.n_classes = n_classes - - x_count = x.count().compute()[0] - x_shape = (x_count, len(self._x.columns)) - y_shape = (x_count, len(self._y.columns)) - # TODO(terrytangyuan): Add support for shuffle and epochs. - self._shuffle = shuffle - self.epochs = epochs - self.input_shape, self.output_shape, self._batch_size = _get_in_out_shape( - x_shape, y_shape, n_classes, batch_size) - self.sample_fraction = self._batch_size / float(x_count) - self._input_dtype = _check_dtype(self._x.dtypes[0]) - self._output_dtype = _check_dtype(self._y.dtypes[self._y_columns]) - if random_state is None: - self.random_state = 66 - else: - self.random_state = random_state - - def get_feed_params(self): - """Function returns a `dict` with data feed params while training. - - Returns: - A `dict` with data feed params while training. - """ - return {'batch_size': self._batch_size} - - def get_feed_dict_fn(self, input_placeholder, output_placeholder): - """Returns a function, that will sample data and provide it to placeholders. - - Args: - input_placeholder: tf.compat.v1.placeholder for input features mini batch. - output_placeholder: tf.compat.v1.placeholder for output labels. - - Returns: - A function that when called samples a random subset of batch size - from x and y. - """ - - def _feed_dict_fn(): - """Samples data and provides it to placeholders.""" - # TODO(ipolosukhin): option for with/without replacement (dev version of - # dask) - sample = self.df.random_split( - [self.sample_fraction, 1 - self.sample_fraction], - random_state=self.random_state) - inp = extract_pandas_matrix(sample[0][self._x_columns].compute()).tolist() - out = extract_pandas_matrix(sample[0][self._y_columns].compute()) - # convert to correct dtype - inp = np.array(inp, dtype=self._input_dtype) - # one-hot encode out for each class for cross entropy loss - if HAS_PANDAS: - import pandas as pd # pylint: disable=g-import-not-at-top - if not isinstance(out, pd.Series): - out = out.flatten() - out_max = self._y.max().compute().values[0] - encoded_out = np.zeros((out.size, out_max + 1), dtype=self._output_dtype) - encoded_out[np.arange(out.size), out] = 1 - return {input_placeholder.name: inp, output_placeholder.name: encoded_out} - - return _feed_dict_fn diff --git a/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py b/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py deleted file mode 100644 index 284a4f45f63..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py +++ /dev/null @@ -1,366 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for `DataFeeder`.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os.path -import numpy as np -import six -from six.moves import xrange # pylint: disable=redefined-builtin - -# pylint: disable=wildcard-import -from tensorflow.contrib.learn.python.learn.learn_io import * -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.lib.io import file_io -from tensorflow.python.platform import test - -# pylint: enable=wildcard-import - - -class DataFeederTest(test.TestCase): - # pylint: disable=undefined-variable - """Tests for `DataFeeder`.""" - - def setUp(self): - self._base_dir = os.path.join(self.get_temp_dir(), 'base_dir') - file_io.create_dir(self._base_dir) - - def tearDown(self): - file_io.delete_recursively(self._base_dir) - - def _wrap_dict(self, data, prepend=''): - return {prepend + '1': data, prepend + '2': data} - - def _assert_raises(self, input_data): - with self.assertRaisesRegexp(TypeError, 'annot convert'): - data_feeder.DataFeeder(input_data, None, n_classes=0, batch_size=1) - - def _assert_dtype(self, expected_np_dtype, expected_tf_dtype, input_data): - feeder = data_feeder.DataFeeder(input_data, None, n_classes=0, batch_size=1) - if isinstance(input_data, dict): - for v in list(feeder.input_dtype.values()): - self.assertEqual(expected_np_dtype, v) - else: - self.assertEqual(expected_np_dtype, feeder.input_dtype) - with ops.Graph().as_default() as g, self.session(g): - inp, _ = feeder.input_builder() - if isinstance(inp, dict): - for v in list(inp.values()): - self.assertEqual(expected_tf_dtype, v.dtype) - else: - self.assertEqual(expected_tf_dtype, inp.dtype) - - def test_input_int8(self): - data = np.matrix([[1, 2], [3, 4]], dtype=np.int8) - self._assert_dtype(np.int8, dtypes.int8, data) - self._assert_dtype(np.int8, dtypes.int8, self._wrap_dict(data)) - - def test_input_int16(self): - data = np.matrix([[1, 2], [3, 4]], dtype=np.int16) - self._assert_dtype(np.int16, dtypes.int16, data) - self._assert_dtype(np.int16, dtypes.int16, self._wrap_dict(data)) - - def test_input_int32(self): - data = np.matrix([[1, 2], [3, 4]], dtype=np.int32) - self._assert_dtype(np.int32, dtypes.int32, data) - self._assert_dtype(np.int32, dtypes.int32, self._wrap_dict(data)) - - def test_input_int64(self): - data = np.matrix([[1, 2], [3, 4]], dtype=np.int64) - self._assert_dtype(np.int64, dtypes.int64, data) - self._assert_dtype(np.int64, dtypes.int64, self._wrap_dict(data)) - - def test_input_uint32(self): - data = np.matrix([[1, 2], [3, 4]], dtype=np.uint32) - self._assert_dtype(np.uint32, dtypes.uint32, data) - self._assert_dtype(np.uint32, dtypes.uint32, self._wrap_dict(data)) - - def test_input_uint64(self): - data = np.matrix([[1, 2], [3, 4]], dtype=np.uint64) - self._assert_dtype(np.uint64, dtypes.uint64, data) - self._assert_dtype(np.uint64, dtypes.uint64, self._wrap_dict(data)) - - def test_input_uint8(self): - data = np.matrix([[1, 2], [3, 4]], dtype=np.uint8) - self._assert_dtype(np.uint8, dtypes.uint8, data) - self._assert_dtype(np.uint8, dtypes.uint8, self._wrap_dict(data)) - - def test_input_uint16(self): - data = np.matrix([[1, 2], [3, 4]], dtype=np.uint16) - self._assert_dtype(np.uint16, dtypes.uint16, data) - self._assert_dtype(np.uint16, dtypes.uint16, self._wrap_dict(data)) - - def test_input_float16(self): - data = np.matrix([[1, 2], [3, 4]], dtype=np.float16) - self._assert_dtype(np.float16, dtypes.float16, data) - self._assert_dtype(np.float16, dtypes.float16, self._wrap_dict(data)) - - def test_input_float32(self): - data = np.matrix([[1, 2], [3, 4]], dtype=np.float32) - self._assert_dtype(np.float32, dtypes.float32, data) - self._assert_dtype(np.float32, dtypes.float32, self._wrap_dict(data)) - - def test_input_float64(self): - data = np.matrix([[1, 2], [3, 4]], dtype=np.float64) - self._assert_dtype(np.float64, dtypes.float64, data) - self._assert_dtype(np.float64, dtypes.float64, self._wrap_dict(data)) - - def test_input_bool(self): - data = np.array([[False for _ in xrange(2)] for _ in xrange(2)]) - self._assert_dtype(np.bool, dtypes.bool, data) - self._assert_dtype(np.bool, dtypes.bool, self._wrap_dict(data)) - - def test_input_string(self): - input_data = np.array([['str%d' % i for i in xrange(2)] for _ in xrange(2)]) - self._assert_dtype(input_data.dtype, dtypes.string, input_data) - self._assert_dtype(input_data.dtype, dtypes.string, - self._wrap_dict(input_data)) - - def _assertAllClose(self, src, dest, src_key_of=None, src_prop=None): - - def func(x): - val = getattr(x, src_prop) if src_prop else x - return val if src_key_of is None else src_key_of[val] - - if isinstance(src, dict): - for k in list(src.keys()): - self.assertAllClose(func(src[k]), dest) - else: - self.assertAllClose(func(src), dest) - - def test_unsupervised(self): - - def func(feeder): - with self.cached_session(): - inp, _ = feeder.input_builder() - feed_dict_fn = feeder.get_feed_dict_fn() - feed_dict = feed_dict_fn() - self._assertAllClose(inp, [[1, 2]], feed_dict, 'name') - - data = np.matrix([[1, 2], [2, 3], [3, 4]]) - func(data_feeder.DataFeeder(data, None, n_classes=0, batch_size=1)) - func( - data_feeder.DataFeeder( - self._wrap_dict(data), None, n_classes=0, batch_size=1)) - - def test_data_feeder_regression(self): - - def func(df): - inp, out = df.input_builder() - feed_dict_fn = df.get_feed_dict_fn() - feed_dict = feed_dict_fn() - self._assertAllClose(inp, [[3, 4], [1, 2]], feed_dict, 'name') - self._assertAllClose(out, [2, 1], feed_dict, 'name') - - x = np.matrix([[1, 2], [3, 4]]) - y = np.array([1, 2]) - func(data_feeder.DataFeeder(x, y, n_classes=0, batch_size=3)) - func( - data_feeder.DataFeeder( - self._wrap_dict(x, 'in'), - self._wrap_dict(y, 'out'), - n_classes=self._wrap_dict(0, 'out'), - batch_size=3)) - - def test_epoch(self): - - def func(feeder): - with self.cached_session(): - feeder.input_builder() - epoch = feeder.make_epoch_variable() - feed_dict_fn = feeder.get_feed_dict_fn() - # First input - feed_dict = feed_dict_fn() - self.assertAllClose(feed_dict[epoch.name], [0]) - # Second input - feed_dict = feed_dict_fn() - self.assertAllClose(feed_dict[epoch.name], [0]) - # Third input - feed_dict = feed_dict_fn() - self.assertAllClose(feed_dict[epoch.name], [0]) - # Back to the first input again, so new epoch. - feed_dict = feed_dict_fn() - self.assertAllClose(feed_dict[epoch.name], [1]) - - data = np.matrix([[1, 2], [2, 3], [3, 4]]) - labels = np.array([0, 0, 1]) - func(data_feeder.DataFeeder(data, labels, n_classes=0, batch_size=1)) - func( - data_feeder.DataFeeder( - self._wrap_dict(data, 'in'), - self._wrap_dict(labels, 'out'), - n_classes=self._wrap_dict(0, 'out'), - batch_size=1)) - - def test_data_feeder_multioutput_regression(self): - - def func(df): - inp, out = df.input_builder() - feed_dict_fn = df.get_feed_dict_fn() - feed_dict = feed_dict_fn() - self._assertAllClose(inp, [[3, 4], [1, 2]], feed_dict, 'name') - self._assertAllClose(out, [[3, 4], [1, 2]], feed_dict, 'name') - - x = np.matrix([[1, 2], [3, 4]]) - y = np.array([[1, 2], [3, 4]]) - func(data_feeder.DataFeeder(x, y, n_classes=0, batch_size=2)) - func( - data_feeder.DataFeeder( - self._wrap_dict(x, 'in'), - self._wrap_dict(y, 'out'), - n_classes=self._wrap_dict(0, 'out'), - batch_size=2)) - - def test_data_feeder_multioutput_classification(self): - - def func(df): - inp, out = df.input_builder() - feed_dict_fn = df.get_feed_dict_fn() - feed_dict = feed_dict_fn() - self._assertAllClose(inp, [[3, 4], [1, 2]], feed_dict, 'name') - self._assertAllClose( - out, [[[0, 0, 1, 0, 0], [0, 0, 0, 1, 0], [0, 0, 0, 0, 1]], - [[1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0]]], feed_dict, - 'name') - - x = np.matrix([[1, 2], [3, 4]]) - y = np.array([[0, 1, 2], [2, 3, 4]]) - func(data_feeder.DataFeeder(x, y, n_classes=5, batch_size=2)) - func( - data_feeder.DataFeeder( - self._wrap_dict(x, 'in'), - self._wrap_dict(y, 'out'), - n_classes=self._wrap_dict(5, 'out'), - batch_size=2)) - - def test_streaming_data_feeder(self): - - def func(df): - inp, out = df.input_builder() - feed_dict_fn = df.get_feed_dict_fn() - feed_dict = feed_dict_fn() - self._assertAllClose(inp, [[[1, 2]], [[3, 4]]], feed_dict, 'name') - self._assertAllClose(out, [[[1], [2]], [[2], [2]]], feed_dict, 'name') - - def x_iter(wrap_dict=False): - yield np.array([[1, 2]]) if not wrap_dict else self._wrap_dict( - np.array([[1, 2]]), 'in') - yield np.array([[3, 4]]) if not wrap_dict else self._wrap_dict( - np.array([[3, 4]]), 'in') - - def y_iter(wrap_dict=False): - yield np.array([[1], [2]]) if not wrap_dict else self._wrap_dict( - np.array([[1], [2]]), 'out') - yield np.array([[2], [2]]) if not wrap_dict else self._wrap_dict( - np.array([[2], [2]]), 'out') - - func( - data_feeder.StreamingDataFeeder( - x_iter(), y_iter(), n_classes=0, batch_size=2)) - func( - data_feeder.StreamingDataFeeder( - x_iter(True), - y_iter(True), - n_classes=self._wrap_dict(0, 'out'), - batch_size=2)) - # Test non-full batches. - func( - data_feeder.StreamingDataFeeder( - x_iter(), y_iter(), n_classes=0, batch_size=10)) - func( - data_feeder.StreamingDataFeeder( - x_iter(True), - y_iter(True), - n_classes=self._wrap_dict(0, 'out'), - batch_size=10)) - - def test_dask_data_feeder(self): - if HAS_PANDAS and HAS_DASK: - x = pd.DataFrame( - dict( - a=np.array([.1, .3, .4, .6, .2, .1, .6]), - b=np.array([.7, .8, .1, .2, .5, .3, .9]))) - x = dd.from_pandas(x, npartitions=2) - y = pd.DataFrame(dict(labels=np.array([1, 0, 2, 1, 0, 1, 2]))) - y = dd.from_pandas(y, npartitions=2) - # TODO(ipolosukhin): Remove or restore this. - # x = extract_dask_data(x) - # y = extract_dask_labels(y) - df = data_feeder.DaskDataFeeder(x, y, n_classes=2, batch_size=2) - inp, out = df.input_builder() - feed_dict_fn = df.get_feed_dict_fn() - feed_dict = feed_dict_fn() - self.assertAllClose(feed_dict[inp.name], [[0.40000001, 0.1], - [0.60000002, 0.2]]) - self.assertAllClose(feed_dict[out.name], [[0., 0., 1.], [0., 1., 0.]]) - - # TODO(rohanj): Fix this test by fixing data_feeder. Currently, h5py doesn't - # support permutation based indexing lookups (More documentation at - # http://docs.h5py.org/en/latest/high/dataset.html#fancy-indexing) - def DISABLED_test_hdf5_data_feeder(self): - - def func(df): - inp, out = df.input_builder() - feed_dict_fn = df.get_feed_dict_fn() - feed_dict = feed_dict_fn() - self._assertAllClose(inp, [[3, 4], [1, 2]], feed_dict, 'name') - self.assertAllClose(out, [2, 1], feed_dict, 'name') - - try: - import h5py # pylint: disable=g-import-not-at-top - x = np.matrix([[1, 2], [3, 4]]) - y = np.array([1, 2]) - file_path = os.path.join(self._base_dir, 'test_hdf5.h5') - h5f = h5py.File(file_path, 'w') - h5f.create_dataset('x', data=x) - h5f.create_dataset('y', data=y) - h5f.close() - h5f = h5py.File(file_path, 'r') - x = h5f['x'] - y = h5f['y'] - func(data_feeder.DataFeeder(x, y, n_classes=0, batch_size=3)) - func( - data_feeder.DataFeeder( - self._wrap_dict(x, 'in'), - self._wrap_dict(y, 'out'), - n_classes=self._wrap_dict(0, 'out'), - batch_size=3)) - except ImportError: - print("Skipped test for hdf5 since it's not installed.") - - -class SetupPredictDataFeederTest(DataFeederTest): - """Tests for `DataFeeder.setup_predict_data_feeder`.""" - - def test_iterable_data(self): - # pylint: disable=undefined-variable - - def func(df): - self._assertAllClose(six.next(df), [[1, 2], [3, 4]]) - self._assertAllClose(six.next(df), [[5, 6]]) - - data = [[1, 2], [3, 4], [5, 6]] - x = iter(data) - x_dict = iter([self._wrap_dict(v) for v in iter(data)]) - func(data_feeder.setup_predict_data_feeder(x, batch_size=2)) - func(data_feeder.setup_predict_data_feeder(x_dict, batch_size=2)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/learn_io/generator_io.py b/tensorflow/contrib/learn/python/learn/learn_io/generator_io.py deleted file mode 100644 index 20fb6a5fd9d..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_io/generator_io.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Methods to allow generator of dict with numpy arrays (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from collections import Container -from types import FunctionType -from types import GeneratorType - -from tensorflow.python.estimator.inputs.queues.feeding_functions import _enqueue_data as enqueue_data -from tensorflow.python.util.deprecation import deprecated - - -@deprecated(None, 'Please use tf.data.') -def generator_input_fn(x, - target_key=None, - batch_size=128, - num_epochs=1, - shuffle=True, - queue_capacity=1000, - num_threads=1, - pad_value=None): - """Returns input function that returns dicts of numpy arrays - yielded from a generator. - - It is assumed that every dict of numpy arrays yielded from the dictionary - represents a single sample. The generator should consume a single epoch of the - data. - - This returns a function outputting `features` and `target` based on the dict - of numpy arrays. The dict `features` has the same keys as an element yielded - from x. - - Example: - ```python - def generator(): - for index in range(10): - yield {'height': np.random.randint(32,36), - 'age': np.random.randint(18, 80), - 'label': np.ones(1)} - - with tf.compat.v1.Session() as session: - input_fn = generator_io.generator_input_fn( - generator, target_key="label", batch_size=2, shuffle=False, - num_epochs=1) - ``` - - Args: - x: Generator Function, returns a `Generator` that will yield the data - in `dict` of numpy arrays - target_key: String or Container of Strings, the key or Container of keys of - the numpy arrays in x dictionaries to use as target. - batch_size: Integer, size of batches to return. - num_epochs: Integer, number of epochs to iterate over data. If `None` will - run forever. - shuffle: Boolean, if True shuffles the queue. Avoid shuffle at prediction - time. - queue_capacity: Integer, size of queue to accumulate. - num_threads: Integer, number of threads used for reading and enqueueing. - pad_value: default value for dynamic padding of data samples, if provided. - - Returns: - Function, that returns a feature `dict` with `Tensors` and an optional - label `dict` with `Tensors`, or if target_key is `str` label is a `Tensor` - - Raises: - TypeError: `x` is not `FunctionType`. - TypeError: `x()` is not `GeneratorType`. - TypeError: `next(x())` is not `dict`. - TypeError: `target_key` is not `str` or `target_key` is not `Container` - of `str`. - KeyError: `target_key` not a key or `target_key[index]` not in next(`x()`). - KeyError: `key` mismatch between dicts emitted from `x()` - """ - if not isinstance(x, FunctionType): - raise TypeError( - 'x must be generator function; got {}'.format(type(x).__name__)) - generator = x() - if not isinstance(generator, GeneratorType): - raise TypeError( - 'x() must be generator; got {}'.format(type(generator).__name__)) - data = next(generator) - if not isinstance(data, dict): - raise TypeError('x() must yield dict; got {}'.format(type(data).__name__)) - input_keys = sorted(next(x()).keys()) - if target_key is not None: - if isinstance(target_key, str): - target_key = [target_key] - elif isinstance(target_key, Container): - for item in target_key: - if not isinstance(item, str): - raise TypeError('target_key must be str or Container of str; got {}'. - format(type(item).__name__)) - if item not in input_keys: - raise KeyError( - 'target_key not in yielded dict. Expected {} keys; got {}'.format( - input_keys, item)) - else: - raise TypeError('target_key must be str or Container of str; got {}'. - format(type(target_key).__name__)) - - def _generator_input_fn(): - """generator input function.""" - queue = enqueue_data( - x, - queue_capacity, - shuffle=shuffle, - num_threads=num_threads, - enqueue_size=batch_size, - num_epochs=num_epochs, - pad_value=pad_value) - - features = (queue.dequeue_many(batch_size) - if num_epochs is None else queue.dequeue_up_to(batch_size)) - if not isinstance(features, list): - features = [features] - features = dict(zip(input_keys, features)) - if target_key is not None: - if len(target_key) > 1: - target = {key: features.pop(key) for key in target_key} - else: - target = features.pop(target_key[0]) - return features, target - return features - - return _generator_input_fn diff --git a/tensorflow/contrib/learn/python/learn/learn_io/generator_io_test.py b/tensorflow/contrib/learn/python/learn/learn_io/generator_io_test.py deleted file mode 100644 index 318046733bf..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_io/generator_io_test.py +++ /dev/null @@ -1,340 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for numpy_io.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib.learn.python.learn.learn_io import generator_io -from tensorflow.python.framework import errors -from tensorflow.python.platform import test -from tensorflow.python.training import coordinator -from tensorflow.python.training import queue_runner_impl - - -class GeneratorIoTest(test.TestCase): - - def testGeneratorInputFn(self): - - def generator(): - for index in range(2): - yield { - 'a': np.ones(1) * index, - 'b': np.ones(1) * index + 32, - 'label': np.ones(1) * index - 32 - } - - with self.cached_session() as session: - input_fn = generator_io.generator_input_fn( - generator, - target_key='label', - batch_size=2, - shuffle=False, - num_epochs=1) - features, target = input_fn() - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - res = session.run([features, target]) - self.assertAllEqual(res[0]['a'], np.asarray([0, 1]).reshape(-1, 1)) - self.assertAllEqual(res[0]['b'], np.asarray([32, 33]).reshape(-1, 1)) - self.assertAllEqual(res[1], np.asarray([-32, -31]).reshape(-1, 1)) - - session.run([features]) - with self.assertRaises(errors.OutOfRangeError): - session.run([features, target]) - - coord.request_stop() - coord.join(threads) - - def testGeneratorSingleInputFn(self): - - def generator(): - for index in range(2): - yield {'a': np.ones(1) * index} - - with self.cached_session() as session: - input_fn = generator_io.generator_input_fn( - generator, target_key=None, batch_size=2, shuffle=False, num_epochs=1) - features = input_fn() - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - res = session.run([features]) - self.assertAllEqual(res[0]['a'], np.asarray([0, 1]).reshape(-1, 1)) - - session.run([features]) - with self.assertRaises(errors.OutOfRangeError): - session.run([features]) - - coord.request_stop() - coord.join(threads) - - def testGeneratorInputFnLabelDict(self): - - def generator(): - for index in range(2): - yield { - 'a': np.ones(1) * index, - 'b': np.ones(1) * index + 32, - 'label': np.ones(1) * index - 32, - 'label2': np.ones(1) * index - 64, - } - - with self.cached_session() as session: - input_fn = generator_io.generator_input_fn( - generator, - target_key=['label', 'label2'], - batch_size=2, - shuffle=False, - num_epochs=1) - features, target = input_fn() - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - res = session.run([features, target]) - self.assertAllEqual(res[0]['a'], np.asarray([0, 1]).reshape(-1, 1)) - self.assertAllEqual(res[0]['b'], np.asarray([32, 33]).reshape(-1, 1)) - self.assertAllEqual(res[1]['label'], np.asarray([-32, -31]).reshape( - -1, 1)) - self.assertAllEqual(res[1]['label2'], - np.asarray([-64, -63]).reshape(-1, 1)) - - session.run([features]) - with self.assertRaises(errors.OutOfRangeError): - session.run([features, target]) - - coord.request_stop() - coord.join(threads) - - def testGeneratorInputFnWithDifferentDimensionsOfFeatures(self): - - def generator(): - for index in range(100): - yield { - 'a': np.ones((10, 10)) * index, - 'b': np.ones((5, 5)) * index + 32, - 'label': np.ones((3, 3)) * index - 32 - } - - with self.cached_session() as session: - input_fn = generator_io.generator_input_fn( - generator, - target_key='label', - batch_size=2, - shuffle=False, - num_epochs=1) - features, target = input_fn() - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - res = session.run([features, target]) - self.assertAllEqual(res[0]['a'], - np.vstack((np.zeros((10, 10)), np.ones( - (10, 10)))).reshape(2, 10, 10)) - self.assertAllEqual(res[0]['b'], - np.vstack((np.zeros((5, 5)), np.ones( - (5, 5)))).reshape(2, 5, 5) + 32) - self.assertAllEqual(res[1], - np.vstack((np.zeros((3, 3)), np.ones( - (3, 3)))).reshape(2, 3, 3) - 32) - - coord.request_stop() - coord.join(threads) - - def testGeneratorInputFnWithXAsNonGeneratorFunction(self): - x = np.arange(32, 36) - with self.cached_session(): - with self.assertRaisesRegexp(TypeError, 'x must be generator function'): - failing_input_fn = generator_io.generator_input_fn( - x, batch_size=2, shuffle=False, num_epochs=1) - failing_input_fn() - - def testGeneratorInputFnWithXAsNonGenerator(self): - - def generator(): - return np.arange(32, 36) - - with self.cached_session(): - with self.assertRaisesRegexp(TypeError, r'x\(\) must be generator'): - failing_input_fn = generator_io.generator_input_fn( - generator, batch_size=2, shuffle=False, num_epochs=1) - failing_input_fn() - - def testGeneratorInputFnWithXAsNonGeneratorYieldingDicts(self): - - def generator(): - yield np.arange(32, 36) - - with self.cached_session(): - with self.assertRaisesRegexp(TypeError, r'x\(\) must yield dict'): - failing_input_fn = generator_io.generator_input_fn( - generator, batch_size=2, shuffle=False, num_epochs=1) - failing_input_fn() - - def testGeneratorInputFNWithTargetLabelNotString(self): - - def generator(): - for index in range(2): - yield { - 'a': np.ones((10, 10)) * index, - 'b': np.ones((5, 5)) * index + 32, - 'label': np.ones((3, 3)) * index - 32 - } - - y = np.arange(32, 36) - with self.cached_session(): - with self.assertRaisesRegexp(TypeError, 'target_key must be str or' - ' Container of str'): - failing_input_fn = generator_io.generator_input_fn( - generator, target_key=y, batch_size=2, shuffle=False, num_epochs=1) - failing_input_fn() - - def testGeneratorInputFNWithTargetLabelListNotString(self): - - def generator(): - for index in range(2): - yield { - 'a': np.ones((10, 10)) * index, - 'b': np.ones((5, 5)) * index + 32, - 'label': np.ones((3, 3)) * index - 32 - } - - y = ['label', np.arange(10)] - with self.cached_session(): - with self.assertRaisesRegexp(TypeError, 'target_key must be str or' - ' Container of str'): - failing_input_fn = generator_io.generator_input_fn( - generator, target_key=y, batch_size=2, shuffle=False, num_epochs=1) - failing_input_fn() - - def testGeneratorInputFNWithTargetLabelNotInDict(self): - - def generator(): - for index in range(2): - yield { - 'a': np.ones((10, 10)) * index, - 'b': np.ones((5, 5)) * index + 32, - 'label': np.ones((3, 3)) * index - 32 - } - - y = ['label', 'target'] - with self.cached_session(): - with self.assertRaisesRegexp(KeyError, 'target_key not in yielded dict'): - failing_input_fn = generator_io.generator_input_fn( - generator, target_key=y, batch_size=2, shuffle=False, num_epochs=1) - failing_input_fn() - - def testGeneratorInputFnWithNoTargetKey(self): - - def generator(): - for index in range(2): - yield { - 'a': np.ones(1) * index, - 'b': np.ones(1) * index + 32, - 'label': np.ones(1) * index - 32 - } - - with self.cached_session() as session: - input_fn = generator_io.generator_input_fn( - generator, target_key=None, batch_size=2, shuffle=False, num_epochs=1) - features = input_fn() - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - res = session.run(features) - self.assertAllEqual(res['a'], np.asarray([0, 1]).reshape(-1, 1)) - self.assertAllEqual(res['b'], np.asarray([32, 33]).reshape(-1, 1)) - self.assertAllEqual(res['label'], np.asarray([-32, -31]).reshape(-1, 1)) - - session.run([features]) - with self.assertRaises(errors.OutOfRangeError): - session.run([features]) - - coord.request_stop() - coord.join(threads) - - def testGeneratorInputFnWithBatchLargerthanData(self): - - def generator(): - for index in range(2): - yield { - 'a': np.ones(1) * index, - 'b': np.ones(1) * index + 32, - 'label': np.ones(1) * index - 32 - } - - with self.cached_session() as session: - input_fn = generator_io.generator_input_fn( - generator, target_key=None, batch_size=4, shuffle=False, num_epochs=1) - features = input_fn() - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - res = session.run(features) - self.assertAllEqual(res['a'], np.asarray([0, 1, 0, 1]).reshape(-1, 1)) - self.assertAllEqual(res['b'], np.asarray([32, 33, 32, 33]).reshape(-1, 1)) - self.assertAllEqual(res['label'], - np.asarray([-32, -31, -32, -31]).reshape(-1, 1)) - - with self.assertRaises(errors.OutOfRangeError): - session.run([features]) - - coord.request_stop() - coord.join(threads) - - def testGeneratorInputFnWithMismatchinGeneratorKeys(self): - - def generator(): - index = 0 - yield { - 'a': np.ones(1) * index, - 'b': np.ones(1) * index + 32, - 'label': np.ones(1) * index - 32 - } - index = 1 - yield { - 'a': np.ones(1) * index, - 'c': np.ones(1) * index + 32, - 'label': np.ones(1) * index - 32 - } - - with self.cached_session() as session: - input_fn = generator_io.generator_input_fn( - generator, target_key=None, batch_size=2, shuffle=False, num_epochs=1) - features = input_fn() - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - with self.assertRaises(errors.OutOfRangeError): - session.run([features]) - - with self.assertRaisesRegex(KeyError, 'key mismatch between dicts emitted' - ' by GenFunExpected'): - coord.request_stop() - coord.join(threads) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py b/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py deleted file mode 100644 index 4017929bc5f..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py +++ /dev/null @@ -1,882 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Methods to read data in the graph (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.input_pipeline.python.ops import input_pipeline_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.layers import utils -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import io_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.platform import gfile -from tensorflow.python.summary import summary -from tensorflow.python.training import input as input_ops -from tensorflow.python.training import queue_runner -from tensorflow.python.util.deprecation import deprecated - -# Default name for key in the feature dict. -KEY_FEATURE_NAME = '__key__' - - -@deprecated(None, 'Use tf.data.') -def read_batch_examples(file_pattern, - batch_size, - reader, - randomize_input=True, - num_epochs=None, - queue_capacity=10000, - num_threads=1, - read_batch_size=1, - parse_fn=None, - name=None, - seed=None): - """Adds operations to read, queue, batch `Example` protos. - - Given file pattern (or list of files), will setup a queue for file names, - read `Example` proto using provided `reader`, use batch queue to create - batches of examples of size `batch_size`. - - All queue runners are added to the queue runners collection, and may be - started via `start_queue_runners`. - - All ops are added to the default graph. - - Use `parse_fn` if you need to do parsing / processing on single examples. - - Args: - file_pattern: List of files or patterns of file paths containing - `Example` records. See `tf.io.gfile.glob` for pattern rules. - batch_size: An int or scalar `Tensor` specifying the batch size to use. - reader: A function or class that returns an object with - `read` method, (filename tensor) -> (example tensor). - randomize_input: Whether the input should be randomized. - num_epochs: Integer specifying the number of times to read through the - dataset. If `None`, cycles through the dataset forever. - NOTE - If specified, creates a variable that must be initialized, so call - `tf.compat.v1.local_variables_initializer()` and run the op in a session. - queue_capacity: Capacity for input queue. - num_threads: The number of threads enqueuing examples. In order to have - predictable and repeatable order of reading and enqueueing, such as in - prediction and evaluation mode, `num_threads` should be 1. - read_batch_size: An int or scalar `Tensor` specifying the number of - records to read at once. - parse_fn: Parsing function, takes `Example` Tensor returns parsed - representation. If `None`, no parsing is done. - name: Name of resulting op. - seed: An integer (optional). Seed used if randomize_input == True. - - Returns: - String `Tensor` of batched `Example` proto. - - Raises: - ValueError: for invalid inputs. - """ - _, examples = read_keyed_batch_examples( - file_pattern=file_pattern, - batch_size=batch_size, - reader=reader, - randomize_input=randomize_input, - num_epochs=num_epochs, - queue_capacity=queue_capacity, - num_threads=num_threads, - read_batch_size=read_batch_size, - parse_fn=parse_fn, - name=name, - seed=seed) - return examples - - -@deprecated(None, 'Use tf.data.') -def read_keyed_batch_examples(file_pattern, - batch_size, - reader, - randomize_input=True, - num_epochs=None, - queue_capacity=10000, - num_threads=1, - read_batch_size=1, - parse_fn=None, - name=None, - seed=None): - """Adds operations to read, queue, batch `Example` protos. - - Given file pattern (or list of files), will setup a queue for file names, - read `Example` proto using provided `reader`, use batch queue to create - batches of examples of size `batch_size`. - - All queue runners are added to the queue runners collection, and may be - started via `start_queue_runners`. - - All ops are added to the default graph. - - Use `parse_fn` if you need to do parsing / processing on single examples. - - Args: - file_pattern: List of files or patterns of file paths containing - `Example` records. See `tf.io.gfile.glob` for pattern rules. - batch_size: An int or scalar `Tensor` specifying the batch size to use. - reader: A function or class that returns an object with - `read` method, (filename tensor) -> (example tensor). - randomize_input: Whether the input should be randomized. - num_epochs: Integer specifying the number of times to read through the - dataset. If `None`, cycles through the dataset forever. - NOTE - If specified, creates a variable that must be initialized, so call - `tf.compat.v1.local_variables_initializer()` and run the op in a session. - queue_capacity: Capacity for input queue. - num_threads: The number of threads enqueuing examples. In order to have - predictable and repeatable order of reading and enqueueing, such as in - prediction and evaluation mode, `num_threads` should be 1. - read_batch_size: An int or scalar `Tensor` specifying the number of - records to read at once. - parse_fn: Parsing function, takes `Example` Tensor returns parsed - representation. If `None`, no parsing is done. - name: Name of resulting op. - seed: An integer (optional). Seed used if randomize_input == True. - - Returns: - Returns tuple of: - - `Tensor` of string keys. - - String `Tensor` of batched `Example` proto. - - Raises: - ValueError: for invalid inputs. - """ - return _read_keyed_batch_examples_helper( - file_pattern, - batch_size, - reader, - randomize_input=randomize_input, - num_epochs=num_epochs, - queue_capacity=queue_capacity, - num_threads=num_threads, - read_batch_size=read_batch_size, - parse_fn=parse_fn, - setup_shared_queue=False, - name=name, - seed=seed) - - -@deprecated(None, 'Use tf.data.') -def read_keyed_batch_examples_shared_queue(file_pattern, - batch_size, - reader, - randomize_input=True, - num_epochs=None, - queue_capacity=10000, - num_threads=1, - read_batch_size=1, - parse_fn=None, - name=None, - seed=None): - """Adds operations to read, queue, batch `Example` protos. - - Given file pattern (or list of files), will setup a shared queue for file - names, setup a worker queue that pulls from the shared queue, read `Example` - protos using provided `reader`, use batch queue to create batches of examples - of size `batch_size`. This provides at most once visit guarantees. Note that - this only works if the parameter servers are not pre-empted or restarted or - the session is not restored from a checkpoint since the state of a queue - is not checkpointed and we will end up restarting from the entire list of - files. - - All queue runners are added to the queue runners collection, and may be - started via `start_queue_runners`. - - All ops are added to the default graph. - - Use `parse_fn` if you need to do parsing / processing on single examples. - - Args: - file_pattern: List of files or patterns of file paths containing - `Example` records. See `tf.io.gfile.glob` for pattern rules. - batch_size: An int or scalar `Tensor` specifying the batch size to use. - reader: A function or class that returns an object with - `read` method, (filename tensor) -> (example tensor). - randomize_input: Whether the input should be randomized. - num_epochs: Integer specifying the number of times to read through the - dataset. If `None`, cycles through the dataset forever. - NOTE - If specified, creates a variable that must be initialized, so call - `tf.compat.v1.local_variables_initializer()` and run the op in a session. - queue_capacity: Capacity for input queue. - num_threads: The number of threads enqueuing examples. - read_batch_size: An int or scalar `Tensor` specifying the number of - records to read at once. - parse_fn: Parsing function, takes `Example` Tensor returns parsed - representation. If `None`, no parsing is done. - name: Name of resulting op. - seed: An integer (optional). Seed used if randomize_input == True. - - Returns: - Returns tuple of: - - `Tensor` of string keys. - - String `Tensor` of batched `Example` proto. - - Raises: - ValueError: for invalid inputs. - """ - return _read_keyed_batch_examples_helper( - file_pattern, - batch_size, - reader, - randomize_input=randomize_input, - num_epochs=num_epochs, - queue_capacity=queue_capacity, - num_threads=num_threads, - read_batch_size=read_batch_size, - parse_fn=parse_fn, - setup_shared_queue=True, - name=name, - seed=seed) - - -def _get_file_names(file_pattern, randomize_input): - """Parse list of file names from pattern, optionally shuffled. - - Args: - file_pattern: File glob pattern, or list of glob patterns. - randomize_input: Whether to shuffle the order of file names. - - Returns: - List of file names matching `file_pattern`. - - Raises: - ValueError: If `file_pattern` is empty, or pattern matches no files. - """ - if isinstance(file_pattern, list): - if not file_pattern: - raise ValueError('No files given to dequeue_examples.') - file_names = [] - for entry in file_pattern: - file_names.extend(gfile.Glob(entry)) - else: - file_names = list(gfile.Glob(file_pattern)) - - if not file_names: - raise ValueError('No files match %s.' % file_pattern) - - # Sort files so it will be deterministic for unit tests. They'll be shuffled - # in `string_input_producer` if `randomize_input` is enabled. - if not randomize_input: - file_names = sorted(file_names) - return file_names - - -def _get_examples(file_name_queue, reader, num_threads, read_batch_size, - filter_fn, parse_fn): - """Get example filenames matching. - - Args: - file_name_queue: A queue implementation that dequeues elements in - first-in first-out order. - reader: A function or class that returns an object with - `read` method, (filename tensor) -> (example tensor). - num_threads: The number of threads enqueuing examples. - read_batch_size: An int or scalar `Tensor` specifying the number of - records to read at once. - filter_fn: Filtering function, takes both keys as well as an `Example` - Tensors and returns a boolean mask of the same shape as the input Tensors - to be applied for filtering. If `None`, no filtering is done. - parse_fn: Parsing function, takes `Example` Tensor returns parsed - representation. If `None`, no parsing is done. - - Returns: - List of example file names matching `file_name_queue`. - """ - with ops.name_scope('read'): - example_list = [] - for _ in range(num_threads): - keys, examples_proto = utils.smart_cond( - read_batch_size > 1, - lambda: reader().read_up_to(file_name_queue, read_batch_size), - lambda: reader().read(file_name_queue)) - - if filter_fn: - mask = filter_fn(keys, examples_proto) - keys = array_ops.boolean_mask(keys, mask) - examples_proto = array_ops.boolean_mask(examples_proto, mask) - if parse_fn: - parsed_examples = parse_fn(examples_proto) - # Map keys into example map because batch_join doesn't support - # tuple of Tensor + dict. - if isinstance(parsed_examples, dict): - parsed_examples[KEY_FEATURE_NAME] = keys - example_list.append(parsed_examples) - else: - example_list.append((keys, parsed_examples)) - else: - example_list.append((keys, examples_proto)) - return example_list - - -def _read_keyed_batch_examples_helper(file_pattern, - batch_size, - reader, - randomize_input=True, - num_epochs=None, - queue_capacity=10000, - num_threads=1, - read_batch_size=1, - filter_fn=None, - parse_fn=None, - setup_shared_queue=False, - name=None, - seed=None): - """Adds operations to read, queue, batch `Example` protos. - - Args: - file_pattern: List of files or patterns of file paths containing - `Example` records. See `tf.io.gfile.glob` for pattern rules. - batch_size: An int or scalar `Tensor` specifying the batch size to use. - reader: A function or class that returns an object with - `read` method, (filename tensor) -> (example tensor). - randomize_input: Whether the input should be randomized. - num_epochs: Integer specifying the number of times to read through the - dataset. If `None`, cycles through the dataset forever. - NOTE - If specified, creates a variable that must be initialized, so call - `tf.compat.v1.local_variables_initializer()` and run the op in a session. - queue_capacity: Capacity for input queue. - num_threads: The number of threads enqueuing examples. - read_batch_size: An int or scalar `Tensor` specifying the number of - records to read at once. - filter_fn: Filtering function, takes both keys as well `Example` Tensors - and returns a boolean mask of the same shape as the input Tensors to - be applied for filtering. If `None`, no filtering is done. - parse_fn: Parsing function, takes `Example` Tensor returns parsed - representation. If `None`, no parsing is done. - setup_shared_queue: Whether to set up a shared queue for file names. - name: Name of resulting op. - seed: An integer (optional). Seed used if randomize_input == True. - - Returns: - Returns tuple of: - - `Tensor` of string keys. - - String `Tensor` of batched `Example` proto. - - Raises: - ValueError: for invalid inputs. - """ - # Retrieve files to read. - file_names = _get_file_names(file_pattern, randomize_input) - - # Check input parameters are given and reasonable. - if (not queue_capacity) or (queue_capacity <= 0): - raise ValueError('Invalid queue_capacity %s.' % queue_capacity) - if (batch_size is None) or ( - (not isinstance(batch_size, ops.Tensor)) and - (batch_size <= 0 or batch_size >= queue_capacity)): - raise ValueError('Invalid batch_size %s, with queue_capacity %s.' % - (batch_size, queue_capacity)) - if (read_batch_size is None) or ( - (not isinstance(read_batch_size, ops.Tensor)) and (read_batch_size <= 0)): - raise ValueError('Invalid read_batch_size %s.' % read_batch_size) - if (not num_threads) or (num_threads <= 0): - raise ValueError('Invalid num_threads %s.' % num_threads) - if (num_epochs is not None) and (num_epochs <= 0): - raise ValueError('Invalid num_epochs %s.' % num_epochs) - - with ops.name_scope(name, 'read_batch_examples', [file_pattern]) as scope: - with ops.name_scope('file_name_queue') as file_name_queue_scope: - if setup_shared_queue: - file_name_queue = data_flow_ops.FIFOQueue( - capacity=1, dtypes=[dtypes.string], shapes=[[]]) - enqueue_op = file_name_queue.enqueue( - input_pipeline_ops.seek_next( - file_names, - shuffle=randomize_input, - num_epochs=num_epochs, - seed=seed)) - queue_runner.add_queue_runner( - queue_runner.QueueRunner(file_name_queue, [enqueue_op])) - else: - file_name_queue = input_ops.string_input_producer( - constant_op.constant(file_names, name='input'), - shuffle=randomize_input, - num_epochs=num_epochs, - name=file_name_queue_scope, - seed=seed) - - example_list = _get_examples(file_name_queue, reader, num_threads, - read_batch_size, filter_fn, parse_fn) - - enqueue_many = read_batch_size > 1 - - if num_epochs is None: - allow_smaller_final_batch = False - else: - allow_smaller_final_batch = True - - # Setup batching queue given list of read example tensors. - if randomize_input: - if isinstance(batch_size, ops.Tensor): - min_after_dequeue = int(queue_capacity * 0.4) - else: - min_after_dequeue = max(queue_capacity - (3 * batch_size), batch_size) - queued_examples_with_keys = input_ops.shuffle_batch_join( - example_list, - batch_size, - capacity=queue_capacity, - min_after_dequeue=min_after_dequeue, - enqueue_many=enqueue_many, - name=scope, - allow_smaller_final_batch=allow_smaller_final_batch, - seed=seed) - else: - queued_examples_with_keys = input_ops.batch_join( - example_list, - batch_size, - capacity=queue_capacity, - enqueue_many=enqueue_many, - name=scope, - allow_smaller_final_batch=allow_smaller_final_batch) - if parse_fn and isinstance(queued_examples_with_keys, dict): - queued_keys = queued_examples_with_keys.pop(KEY_FEATURE_NAME) - return queued_keys, queued_examples_with_keys - return queued_examples_with_keys - - -@deprecated(None, 'Use tf.data.') -def read_keyed_batch_features(file_pattern, - batch_size, - features, - reader, - randomize_input=True, - num_epochs=None, - queue_capacity=10000, - reader_num_threads=1, - feature_queue_capacity=100, - num_enqueue_threads=2, - parse_fn=None, - name=None, - read_batch_size=None): - """Adds operations to read, queue, batch and parse `Example` protos. - - Given file pattern (or list of files), will setup a queue for file names, - read `Example` proto using provided `reader`, use batch queue to create - batches of examples of size `batch_size` and parse example given `features` - specification. - - All queue runners are added to the queue runners collection, and may be - started via `start_queue_runners`. - - All ops are added to the default graph. - - Args: - file_pattern: List of files or patterns of file paths containing - `Example` records. See `tf.io.gfile.glob` for pattern rules. - batch_size: An int or scalar `Tensor` specifying the batch size to use. - features: A `dict` mapping feature keys to `FixedLenFeature` or - `VarLenFeature` values. - reader: A function or class that returns an object with - `read` method, (filename tensor) -> (example tensor). - randomize_input: Whether the input should be randomized. - num_epochs: Integer specifying the number of times to read through the - dataset. If None, cycles through the dataset forever. NOTE - If specified, - creates a variable that must be initialized, so call - tf.compat.v1.local_variables_initializer() and run the op in a session. - queue_capacity: Capacity for input queue. - reader_num_threads: The number of threads to read examples. In order to have - predictable and repeatable order of reading and enqueueing, such as in - prediction and evaluation mode, `reader_num_threads` should be 1. - feature_queue_capacity: Capacity of the parsed features queue. - num_enqueue_threads: Number of threads to enqueue the parsed example queue. - Using multiple threads to enqueue the parsed example queue helps maintain - a full queue when the subsequent computations overall are cheaper than - parsing. In order to have predictable and repeatable order of reading and - enqueueing, such as in prediction and evaluation mode, - `num_enqueue_threads` should be 1. - parse_fn: Parsing function, takes `Example` Tensor returns parsed - representation. If `None`, no parsing is done. - name: Name of resulting op. - read_batch_size: An int or scalar `Tensor` specifying the number of - records to read at once. If `None`, defaults to `batch_size`. - - Returns: - Returns tuple of: - - `Tensor` of string keys. - - A dict of `Tensor` or `SparseTensor` objects for each in `features`. - - Raises: - ValueError: for invalid inputs. - """ - - with ops.name_scope(name, 'read_batch_features', [file_pattern]) as scope: - if read_batch_size is None: - read_batch_size = batch_size - keys, examples = read_keyed_batch_examples( - file_pattern, - batch_size, - reader, - randomize_input=randomize_input, - num_epochs=num_epochs, - queue_capacity=queue_capacity, - num_threads=reader_num_threads, - read_batch_size=read_batch_size, - parse_fn=parse_fn, - name=scope) - # Parse the example. - feature_map = parsing_ops.parse_example(examples, features) - return queue_parsed_features( - feature_map, - keys=keys, - feature_queue_capacity=feature_queue_capacity, - num_enqueue_threads=num_enqueue_threads, - name=scope) - - -@deprecated(None, 'Use tf.data.') -def read_keyed_batch_features_shared_queue(file_pattern, - batch_size, - features, - reader, - randomize_input=True, - num_epochs=None, - queue_capacity=10000, - reader_num_threads=1, - feature_queue_capacity=100, - num_queue_runners=2, - parse_fn=None, - name=None): - """Adds operations to read, queue, batch and parse `Example` protos. - - Given file pattern (or list of files), will setup a shared queue for file - names, setup a worker queue that gets filenames from the shared queue, - read `Example` proto using provided `reader`, use batch queue to create - batches of examples of size `batch_size` and parse example given `features` - specification. - - All queue runners are added to the queue runners collection, and may be - started via `start_queue_runners`. - - All ops are added to the default graph. - - Args: - file_pattern: List of files or patterns of file paths containing - `Example` records. See `tf.io.gfile.glob` for pattern rules. - batch_size: An int or scalar `Tensor` specifying the batch size to use. - features: A `dict` mapping feature keys to `FixedLenFeature` or - `VarLenFeature` values. - reader: A function or class that returns an object with - `read` method, (filename tensor) -> (example tensor). - randomize_input: Whether the input should be randomized. - num_epochs: Integer specifying the number of times to read through the - dataset. If None, cycles through the dataset forever. NOTE - If specified, - creates a variable that must be initialized, so call - tf.compat.v1.local_variables_initializer() and run the op in a session. - queue_capacity: Capacity for input queue. - reader_num_threads: The number of threads to read examples. - feature_queue_capacity: Capacity of the parsed features queue. - num_queue_runners: Number of threads to enqueue the parsed example queue. - Using multiple threads to enqueue the parsed example queue helps maintain - a full queue when the subsequent computations overall are cheaper than - parsing. - parse_fn: Parsing function, takes `Example` Tensor returns parsed - representation. If `None`, no parsing is done. - name: Name of resulting op. - - Returns: - Returns tuple of: - - `Tensor` of string keys. - - A dict of `Tensor` or `SparseTensor` objects for each in `features`. - - Raises: - ValueError: for invalid inputs. - """ - - with ops.name_scope(name, 'read_batch_features', [file_pattern]) as scope: - keys, examples = read_keyed_batch_examples_shared_queue( - file_pattern, - batch_size, - reader, - randomize_input=randomize_input, - num_epochs=num_epochs, - queue_capacity=queue_capacity, - num_threads=reader_num_threads, - read_batch_size=batch_size, - parse_fn=parse_fn, - name=scope) - # Parse the example. - feature_map = parsing_ops.parse_example(examples, features) - return queue_parsed_features( - feature_map, - keys=keys, - feature_queue_capacity=feature_queue_capacity, - num_enqueue_threads=num_queue_runners, - name=scope) - - -@deprecated(None, 'Use tf.data.') -def queue_parsed_features(parsed_features, - keys=None, - feature_queue_capacity=100, - num_enqueue_threads=2, - name=None): - """Speeds up parsing by using queues to do it asynchronously. - - This function adds the tensors in `parsed_features` to a queue, which allows - the parsing (or any other expensive op before this) to be asynchronous wrt the - rest of the training graph. This greatly improves read latency and speeds up - training since the data will already be parsed and ready when each step of - training needs it. - - All queue runners are added to the queue runners collection, and may be - started via `start_queue_runners`. - - All ops are added to the default graph. - - Args: - parsed_features: A dict of string key to `Tensor` or `SparseTensor` objects. - keys: `Tensor` of string keys. - feature_queue_capacity: Capacity of the parsed features queue. - num_enqueue_threads: Number of threads to enqueue the parsed example queue. - Using multiple threads to enqueue the parsed example queue helps maintain - a full queue when the subsequent computations overall are cheaper than - parsing. In order to have predictable and repeatable order of reading and - enqueueing, such as in prediction and evaluation mode, - `num_enqueue_threads` should be 1. - name: Name of resulting op. - - Returns: - Returns tuple of: - - `Tensor` corresponding to `keys` if provided, otherwise `None`. - - A dict of string key to `Tensor` or `SparseTensor` objects corresponding - to `parsed_features`. - Raises: - ValueError: for invalid inputs. - """ - - args = list(parsed_features.values()) - if keys is not None: - args += [keys] - - with ops.name_scope(name, 'queue_parsed_features', args): - # Lets also add preprocessed tensors into the queue types for each item of - # the queue. - tensors_to_enqueue = [] - # Each entry contains the key, and a boolean which indicates whether the - # tensor was a sparse tensor. - tensors_mapping = [] - # TODO(sibyl-Aix6ihai): Most of the functionality here is about pushing sparse - # tensors into a queue. This could be taken care in somewhere else so others - # can reuse it. Also, QueueBase maybe extended to handle sparse tensors - # directly. - for key in sorted(parsed_features.keys()): - tensor = parsed_features[key] - if isinstance(tensor, sparse_tensor.SparseTensor): - tensors_mapping.append((key, True)) - tensors_to_enqueue.extend( - [tensor.indices, tensor.values, tensor.dense_shape]) - else: - tensors_mapping.append((key, False)) - tensors_to_enqueue.append(tensor) - - if keys is not None: - tensors_to_enqueue.append(keys) - - queue_dtypes = [x.dtype for x in tensors_to_enqueue] - input_queue = data_flow_ops.FIFOQueue(feature_queue_capacity, queue_dtypes) - - # Add a summary op to debug if our feature queue is full or not. - summary.scalar('queue/parsed_features/%s/fraction_of_%d_full' % - (input_queue.name, feature_queue_capacity), - math_ops.cast(input_queue.size(), dtypes.float32) * - (1. / feature_queue_capacity)) - - # Use a single QueueRunner with multiple threads to enqueue so the queue is - # always full. The threads are coordinated so the last batch will not be - # lost. - enqueue_ops = [ - input_queue.enqueue(tensors_to_enqueue) - for _ in range(num_enqueue_threads) - ] - queue_runner.add_queue_runner( - queue_runner.QueueRunner( - input_queue, - enqueue_ops, - queue_closed_exception_types=(errors.OutOfRangeError, - errors.CancelledError))) - - dequeued_tensors = input_queue.dequeue() - if not isinstance(dequeued_tensors, list): - # input_queue.dequeue() returns a single tensor instead of a list of - # tensors if there is only one tensor to dequeue, which breaks the - # assumption of a list below. - dequeued_tensors = [dequeued_tensors] - - # Reset shapes on dequeued tensors. - for i in range(len(tensors_to_enqueue)): - dequeued_tensors[i].set_shape(tensors_to_enqueue[i].get_shape()) - - # Recreate feature mapping according to the original dictionary. - dequeued_parsed_features = {} - index = 0 - for key, is_sparse_tensor in tensors_mapping: - if is_sparse_tensor: - # Three tensors are (indices, values, shape). - dequeued_parsed_features[key] = sparse_tensor.SparseTensor( - dequeued_tensors[index], dequeued_tensors[index + 1], - dequeued_tensors[index + 2]) - index += 3 - else: - dequeued_parsed_features[key] = dequeued_tensors[index] - index += 1 - - dequeued_keys = None - if keys is not None: - dequeued_keys = dequeued_tensors[-1] - - return dequeued_keys, dequeued_parsed_features - - -@deprecated(None, 'Use tf.data.') -def read_batch_features(file_pattern, - batch_size, - features, - reader, - randomize_input=True, - num_epochs=None, - queue_capacity=10000, - feature_queue_capacity=100, - reader_num_threads=1, - num_enqueue_threads=2, - parse_fn=None, - name=None, - read_batch_size=None): - """Adds operations to read, queue, batch and parse `Example` protos. - - Given file pattern (or list of files), will setup a queue for file names, - read `Example` proto using provided `reader`, use batch queue to create - batches of examples of size `batch_size` and parse example given `features` - specification. - - All queue runners are added to the queue runners collection, and may be - started via `start_queue_runners`. - - All ops are added to the default graph. - - Args: - file_pattern: List of files or patterns of file paths containing - `Example` records. See `tf.io.gfile.glob` for pattern rules. - batch_size: An int or scalar `Tensor` specifying the batch size to use. - features: A `dict` mapping feature keys to `FixedLenFeature` or - `VarLenFeature` values. - reader: A function or class that returns an object with - `read` method, (filename tensor) -> (example tensor). - randomize_input: Whether the input should be randomized. - num_epochs: Integer specifying the number of times to read through the - dataset. If None, cycles through the dataset forever. NOTE - If specified, - creates a variable that must be initialized, so call - tf.compat.v1.local_variables_initializer() and run the op in a session. - queue_capacity: Capacity for input queue. - feature_queue_capacity: Capacity of the parsed features queue. Set this - value to a small number, for example 5 if the parsed features are large. - reader_num_threads: The number of threads to read examples. In order to have - predictable and repeatable order of reading and enqueueing, such as in - prediction and evaluation mode, `reader_num_threads` should be 1. - num_enqueue_threads: Number of threads to enqueue the parsed example queue. - Using multiple threads to enqueue the parsed example queue helps maintain - a full queue when the subsequent computations overall are cheaper than - parsing. In order to have predictable and repeatable order of reading and - enqueueing, such as in prediction and evaluation mode, - `num_enqueue_threads` should be 1. - parse_fn: Parsing function, takes `Example` Tensor returns parsed - representation. If `None`, no parsing is done. - name: Name of resulting op. - read_batch_size: An int or scalar `Tensor` specifying the number of - records to read at once. If `None`, defaults to `batch_size`. - - Returns: - A dict of `Tensor` or `SparseTensor` objects for each in `features`. - - Raises: - ValueError: for invalid inputs. - """ - _, features = read_keyed_batch_features( - file_pattern, - batch_size, - features, - reader, - randomize_input=randomize_input, - num_epochs=num_epochs, - queue_capacity=queue_capacity, - reader_num_threads=reader_num_threads, - feature_queue_capacity=feature_queue_capacity, - num_enqueue_threads=num_enqueue_threads, - read_batch_size=read_batch_size, - parse_fn=parse_fn, - name=name) - return features - - -@deprecated(None, 'Use tf.data.') -def read_batch_record_features(file_pattern, - batch_size, - features, - randomize_input=True, - num_epochs=None, - queue_capacity=10000, - reader_num_threads=1, - name='dequeue_record_examples'): - """Reads TFRecord, queues, batches and parses `Example` proto. - - See more detailed description in `read_examples`. - - Args: - file_pattern: List of files or patterns of file paths containing - `Example` records. See `tf.io.gfile.glob` for pattern rules. - batch_size: An int or scalar `Tensor` specifying the batch size to use. - features: A `dict` mapping feature keys to `FixedLenFeature` or - `VarLenFeature` values. - randomize_input: Whether the input should be randomized. - num_epochs: Integer specifying the number of times to read through the - dataset. If None, cycles through the dataset forever. NOTE - If specified, - creates a variable that must be initialized, so call - tf.compat.v1.local_variables_initializer() and run the op in a session. - queue_capacity: Capacity for input queue. - reader_num_threads: The number of threads to read examples. In order to have - predictable and repeatable order of reading and enqueueing, such as in - prediction and evaluation mode, `reader_num_threads` should be 1. - name: Name of resulting op. - - Returns: - A dict of `Tensor` or `SparseTensor` objects for each in `features`. - - Raises: - ValueError: for invalid inputs. - """ - return read_batch_features( - file_pattern=file_pattern, - batch_size=batch_size, - features=features, - reader=io_ops.TFRecordReader, - randomize_input=randomize_input, - num_epochs=num_epochs, - queue_capacity=queue_capacity, - reader_num_threads=reader_num_threads, - name=name) diff --git a/tensorflow/contrib/learn/python/learn/learn_io/graph_io_test.py b/tensorflow/contrib/learn/python/learn/learn_io/graph_io_test.py deleted file mode 100644 index 8e68a17e478..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_io/graph_io_test.py +++ /dev/null @@ -1,852 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for learn.io.graph_io.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import base64 -import os -import random -import tempfile - -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib.learn.python.learn.learn_io import graph_io -from tensorflow.python.client import session as session_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes as dtypes_lib -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import io_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test -from tensorflow.python.training import coordinator -from tensorflow.python.training import queue_runner_impl -from tensorflow.python.training import server_lib - -_VALID_FILE_PATTERN = "VALID" -_VALID_FILE_PATTERN_2 = "VALID_2" -_FILE_NAMES = [b"abc", b"def", b"ghi", b"jkl"] -_FILE_NAMES_2 = [b"mno", b"pqr"] -_INVALID_FILE_PATTERN = "INVALID" - - -class GraphIOTest(test.TestCase): - - def _mock_glob(self, pattern): - if _VALID_FILE_PATTERN == pattern: - return _FILE_NAMES - if _VALID_FILE_PATTERN_2 == pattern: - return _FILE_NAMES_2 - self.assertEqual(_INVALID_FILE_PATTERN, pattern) - return [] - - def setUp(self): - super(GraphIOTest, self).setUp() - random.seed(42) - self._orig_glob = gfile.Glob - gfile.Glob = self._mock_glob - - def tearDown(self): - gfile.Glob = self._orig_glob - super(GraphIOTest, self).tearDown() - - def test_dequeue_batch_value_errors(self): - default_batch_size = 17 - queue_capacity = 1234 - num_threads = 3 - name = "my_batch" - - self.assertRaisesRegexp( - ValueError, - "No files match", - graph_io.read_batch_examples, - _INVALID_FILE_PATTERN, - default_batch_size, - io_ops.TFRecordReader, - False, - num_epochs=None, - queue_capacity=queue_capacity, - num_threads=num_threads, - name=name) - self.assertRaisesRegexp( - ValueError, - "Invalid batch_size", - graph_io.read_batch_examples, - _VALID_FILE_PATTERN, - None, - io_ops.TFRecordReader, - False, - num_epochs=None, - queue_capacity=queue_capacity, - num_threads=num_threads, - name=name) - self.assertRaisesRegexp( - ValueError, - "Invalid batch_size", - graph_io.read_batch_examples, - _VALID_FILE_PATTERN, - -1, - io_ops.TFRecordReader, - False, - num_epochs=None, - queue_capacity=queue_capacity, - num_threads=num_threads, - name=name) - self.assertRaisesRegexp( - ValueError, - "Invalid batch_size", - graph_io.read_batch_examples, - _VALID_FILE_PATTERN, - default_batch_size, - io_ops.TFRecordReader, - False, - num_epochs=None, - queue_capacity=default_batch_size, - num_threads=num_threads, - name=name) - self.assertRaisesRegexp( - ValueError, - "Invalid queue_capacity", - graph_io.read_batch_examples, - _VALID_FILE_PATTERN, - default_batch_size, - io_ops.TFRecordReader, - False, - num_epochs=None, - queue_capacity=None, - num_threads=num_threads, - name=name) - self.assertRaisesRegexp( - ValueError, - "Invalid num_threads", - graph_io.read_batch_examples, - _VALID_FILE_PATTERN, - default_batch_size, - io_ops.TFRecordReader, - False, - num_epochs=None, - queue_capacity=queue_capacity, - num_threads=None, - name=name) - self.assertRaisesRegexp( - ValueError, - "Invalid num_threads", - graph_io.read_batch_examples, - _VALID_FILE_PATTERN, - default_batch_size, - io_ops.TFRecordReader, - False, - num_epochs=None, - queue_capacity=queue_capacity, - num_threads=-1, - name=name) - self.assertRaisesRegexp( - ValueError, - "Invalid batch_size", - graph_io.read_batch_examples, - _VALID_FILE_PATTERN, - queue_capacity + 1, - io_ops.TFRecordReader, - False, - num_epochs=None, - queue_capacity=queue_capacity, - num_threads=1, - name=name) - self.assertRaisesRegexp( - ValueError, - "Invalid num_epochs", - graph_io.read_batch_examples, - _VALID_FILE_PATTERN, - default_batch_size, - io_ops.TFRecordReader, - False, - num_epochs=-1, - queue_capacity=queue_capacity, - num_threads=1, - name=name) - self.assertRaisesRegexp( - ValueError, - "Invalid read_batch_size", - graph_io.read_batch_examples, - _VALID_FILE_PATTERN, - default_batch_size, - io_ops.TFRecordReader, - False, - num_epochs=None, - queue_capacity=queue_capacity, - num_threads=1, - read_batch_size=0, - name=name) - - def test_batch_record_features(self): - batch_size = 17 - queue_capacity = 1234 - name = "my_batch" - shape = (0,) - features = { - "feature": - parsing_ops.FixedLenFeature(shape=shape, dtype=dtypes_lib.float32) - } - - with ops.Graph().as_default() as g, self.session(graph=g) as sess: - features = graph_io.read_batch_record_features( - _VALID_FILE_PATTERN, - batch_size, - features, - randomize_input=False, - queue_capacity=queue_capacity, - reader_num_threads=2, - name=name) - self.assertTrue("feature" in features, - "'feature' missing from %s." % features.keys()) - feature = features["feature"] - self.assertEqual("%s/fifo_queue_1_Dequeue:0" % name, feature.name) - self.assertAllEqual((batch_size,) + shape, feature.get_shape().as_list()) - file_name_queue_name = "%s/file_name_queue" % name - file_names_name = "%s/input" % file_name_queue_name - example_queue_name = "%s/fifo_queue" % name - parse_example_queue_name = "%s/fifo_queue" % name - op_nodes = test_util.assert_ops_in_graph({ - file_names_name: "Const", - file_name_queue_name: "FIFOQueueV2", - "%s/read/TFRecordReaderV2" % name: "TFRecordReaderV2", - example_queue_name: "FIFOQueueV2", - parse_example_queue_name: "FIFOQueueV2", - name: "QueueDequeueManyV2" - }, g) - self.assertAllEqual(_FILE_NAMES, sess.run(["%s:0" % file_names_name])[0]) - self.assertEqual(queue_capacity, - op_nodes[example_queue_name].attr["capacity"].i) - - def test_one_epoch(self): - batch_size = 17 - queue_capacity = 1234 - name = "my_batch" - - with ops.Graph().as_default() as g, self.session(graph=g) as sess: - inputs = graph_io.read_batch_examples( - _VALID_FILE_PATTERN, - batch_size, - reader=io_ops.TFRecordReader, - randomize_input=True, - num_epochs=1, - queue_capacity=queue_capacity, - name=name) - self.assertAllEqual((None,), inputs.get_shape().as_list()) - self.assertEqual("%s:1" % name, inputs.name) - file_name_queue_name = "%s/file_name_queue" % name - file_name_queue_limit_name = ( - "%s/limit_epochs/epochs" % file_name_queue_name) - file_names_name = "%s/input" % file_name_queue_name - example_queue_name = "%s/random_shuffle_queue" % name - op_nodes = test_util.assert_ops_in_graph({ - file_names_name: "Const", - file_name_queue_name: "FIFOQueueV2", - "%s/read/TFRecordReaderV2" % name: "TFRecordReaderV2", - example_queue_name: "RandomShuffleQueueV2", - name: "QueueDequeueUpToV2", - file_name_queue_limit_name: "VariableV2" - }, g) - self.assertEqual( - set(_FILE_NAMES), set(sess.run(["%s:0" % file_names_name])[0])) - self.assertEqual(queue_capacity, - op_nodes[example_queue_name].attr["capacity"].i) - - def test_batch_randomized_multiple_globs(self): - batch_size = 17 - queue_capacity = 1234 - name = "my_batch" - - with ops.Graph().as_default() as g, self.session(graph=g) as sess: - inputs = graph_io.read_batch_examples( - [_VALID_FILE_PATTERN, _VALID_FILE_PATTERN_2], - batch_size, - reader=io_ops.TFRecordReader, - randomize_input=True, - queue_capacity=queue_capacity, - name=name) - self.assertAllEqual((batch_size,), inputs.get_shape().as_list()) - self.assertEqual("%s:1" % name, inputs.name) - file_name_queue_name = "%s/file_name_queue" % name - file_names_name = "%s/input" % file_name_queue_name - example_queue_name = "%s/random_shuffle_queue" % name - op_nodes = test_util.assert_ops_in_graph({ - file_names_name: "Const", - file_name_queue_name: "FIFOQueueV2", - "%s/read/TFRecordReaderV2" % name: "TFRecordReaderV2", - example_queue_name: "RandomShuffleQueueV2", - name: "QueueDequeueManyV2" - }, g) - self.assertEqual( - set(_FILE_NAMES + _FILE_NAMES_2), - set(sess.run(["%s:0" % file_names_name])[0])) - self.assertEqual(queue_capacity, - op_nodes[example_queue_name].attr["capacity"].i) - - def _create_temp_file(self, lines): - tempdir = tempfile.mkdtemp() - filename = os.path.join(tempdir, "temp_file") - gfile.Open(filename, "w").write(lines) - return filename - - def _create_sorted_temp_files(self, lines_list): - tempdir = tempfile.mkdtemp() - filenames = [] - for i, lines in enumerate(lines_list): - filename = os.path.join(tempdir, "temp_file%05d" % i) - gfile.Open(filename, "w").write(lines) - filenames.append(filename) - return filenames - - def test_read_text_lines(self): - gfile.Glob = self._orig_glob - filename = self._create_temp_file("ABC\nDEF\nGHK\n") - - batch_size = 1 - queue_capacity = 5 - name = "my_batch" - - with ops.Graph().as_default() as g, self.session(graph=g) as session: - inputs = graph_io.read_batch_examples( - filename, - batch_size, - reader=io_ops.TextLineReader, - randomize_input=False, - num_epochs=1, - queue_capacity=queue_capacity, - name=name) - self.assertAllEqual((None,), inputs.get_shape().as_list()) - session.run(variables.local_variables_initializer()) - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - self.assertAllEqual(session.run(inputs), [b"ABC"]) - self.assertAllEqual(session.run(inputs), [b"DEF"]) - self.assertAllEqual(session.run(inputs), [b"GHK"]) - with self.assertRaises(errors.OutOfRangeError): - session.run(inputs) - - coord.request_stop() - coord.join(threads) - - def _create_file_from_list_of_features(self, lines): - json_lines = [ - "".join([ - '{"features": { "feature": { "sequence": {', - '"bytes_list": { "value": ["', - base64.b64encode(l).decode("ascii"), '"]}}}}}\n' - ]) for l in lines - ] - return self._create_temp_file("".join(json_lines)) - - def test_read_text_lines_large(self): - gfile.Glob = self._orig_glob - sequence_prefix = "abcdefghijklmnopqrstuvwxyz123456789" - num_records = 49999 - lines = [ - "".join([sequence_prefix, str(l)]).encode("ascii") - for l in xrange(num_records) - ] - filename = self._create_file_from_list_of_features(lines) - batch_size = 10000 - queue_capacity = 100000 - name = "my_large_batch" - - features = {"sequence": parsing_ops.FixedLenFeature([], dtypes_lib.string)} - - with ops.Graph().as_default() as g, self.session(graph=g) as session: - keys, result = graph_io.read_keyed_batch_features( - filename, - batch_size, - features, - io_ops.TextLineReader, - randomize_input=False, - num_epochs=1, - queue_capacity=queue_capacity, - num_enqueue_threads=2, - parse_fn=parsing_ops.decode_json_example, - name=name) - self.assertAllEqual((None,), keys.get_shape().as_list()) - self.assertEqual(1, len(result)) - self.assertAllEqual((None,), result["sequence"].get_shape().as_list()) - session.run(variables.local_variables_initializer()) - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - data = [] - try: - while not coord.should_stop(): - data.append(session.run(result)) - except errors.OutOfRangeError: - pass - finally: - coord.request_stop() - - coord.join(threads) - - parsed_records = [ - item for sublist in [d["sequence"] for d in data] for item in sublist - ] - # Check that the number of records matches expected and all records - # are present. - self.assertEqual(len(parsed_records), num_records) - self.assertEqual(set(parsed_records), set(lines)) - - def test_read_batch_features_maintains_order(self): - """Make sure that examples are read in the right order. - - When randomize_input=False, num_enqueue_threads=1 and reader_num_threads=1 - read_keyed_batch_features() should read the examples in the same order as - they appear in the file. - """ - gfile.Glob = self._orig_glob - num_records = 1000 - lines = ["".join(str(l)).encode("ascii") for l in xrange(num_records)] - filename = self._create_file_from_list_of_features(lines) - batch_size = 10 - queue_capacity = 1000 - name = "my_large_batch" - - features = {"sequence": parsing_ops.FixedLenFeature([], dtypes_lib.string)} - - with ops.Graph().as_default() as g, self.session(graph=g) as session: - result = graph_io.read_batch_features( - filename, - batch_size, - features, - io_ops.TextLineReader, - randomize_input=False, - num_epochs=1, - queue_capacity=queue_capacity, - reader_num_threads=1, - num_enqueue_threads=1, - parse_fn=parsing_ops.decode_json_example, - name=name) - self.assertEqual(1, len(result)) - self.assertAllEqual((None,), result["sequence"].get_shape().as_list()) - session.run(variables.local_variables_initializer()) - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - data = [] - try: - while not coord.should_stop(): - data.append(session.run(result)) - except errors.OutOfRangeError: - pass - finally: - coord.request_stop() - - coord.join(threads) - - parsed_records = [ - item for sublist in [d["sequence"] for d in data] for item in sublist - ] - # Check that the number of records matches expected and all records - # are present in the right order. - self.assertEqual(len(parsed_records), num_records) - self.assertEqual(parsed_records, lines) - - def test_read_text_lines_multifile(self): - gfile.Glob = self._orig_glob - filenames = self._create_sorted_temp_files(["ABC\n", "DEF\nGHK\n"]) - - batch_size = 1 - queue_capacity = 5 - name = "my_batch" - - with ops.Graph().as_default() as g, self.session(graph=g) as session: - inputs = graph_io.read_batch_examples( - filenames, - batch_size, - reader=io_ops.TextLineReader, - randomize_input=False, - num_epochs=1, - queue_capacity=queue_capacity, - name=name) - self.assertAllEqual((None,), inputs.get_shape().as_list()) - session.run(variables.local_variables_initializer()) - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - self.assertEqual("%s:1" % name, inputs.name) - file_name_queue_name = "%s/file_name_queue" % name - file_names_name = "%s/input" % file_name_queue_name - example_queue_name = "%s/fifo_queue" % name - test_util.assert_ops_in_graph({ - file_names_name: "Const", - file_name_queue_name: "FIFOQueueV2", - "%s/read/TextLineReaderV2" % name: "TextLineReaderV2", - example_queue_name: "FIFOQueueV2", - name: "QueueDequeueUpToV2" - }, g) - - self.assertAllEqual(session.run(inputs), [b"ABC"]) - self.assertAllEqual(session.run(inputs), [b"DEF"]) - self.assertAllEqual(session.run(inputs), [b"GHK"]) - with self.assertRaises(errors.OutOfRangeError): - session.run(inputs) - - coord.request_stop() - coord.join(threads) - - def test_read_text_lines_multifile_with_shared_queue(self): - gfile.Glob = self._orig_glob - filenames = self._create_sorted_temp_files(["ABC\n", "DEF\nGHK\n"]) - - batch_size = 1 - queue_capacity = 5 - name = "my_batch" - - with ops.Graph().as_default() as g, self.session(graph=g) as session: - keys, inputs = graph_io.read_keyed_batch_examples_shared_queue( - filenames, - batch_size, - reader=io_ops.TextLineReader, - randomize_input=False, - num_epochs=1, - queue_capacity=queue_capacity, - name=name) - self.assertAllEqual((None,), keys.get_shape().as_list()) - self.assertAllEqual((None,), inputs.get_shape().as_list()) - session.run([ - variables.local_variables_initializer(), - variables.global_variables_initializer() - ]) - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - self.assertEqual("%s:1" % name, inputs.name) - example_queue_name = "%s/fifo_queue" % name - worker_file_name_queue_name = "%s/file_name_queue/fifo_queue" % name - test_util.assert_ops_in_graph({ - "%s/read/TextLineReaderV2" % name: "TextLineReaderV2", - example_queue_name: "FIFOQueueV2", - worker_file_name_queue_name: "FIFOQueueV2", - name: "QueueDequeueUpToV2" - }, g) - - self.assertAllEqual(session.run(inputs), [b"ABC"]) - self.assertAllEqual(session.run(inputs), [b"DEF"]) - self.assertAllEqual(session.run(inputs), [b"GHK"]) - with self.assertRaises(errors.OutOfRangeError): - session.run(inputs) - - coord.request_stop() - coord.join(threads) - - def _get_qr(self, name): - for qr in ops.get_collection(ops.GraphKeys.QUEUE_RUNNERS): - if qr.name == name: - return qr - - def _run_queue(self, name, session): - qr = self._get_qr(name) - for op in qr.enqueue_ops: - session.run(op) - - def test_multiple_workers_with_shared_queue(self): - gfile.Glob = self._orig_glob - filenames = self._create_sorted_temp_files([ - "ABC\n", "DEF\n", "GHI\n", "JKL\n", "MNO\n", "PQR\n", "STU\n", "VWX\n", - "YZ\n" - ]) - - batch_size = 1 - queue_capacity = 5 - name = "my_batch" - example_queue_name = "%s/fifo_queue" % name - worker_file_name_queue_name = "%s/file_name_queue/fifo_queue" % name - - server = server_lib.Server.create_local_server() - - with ops.Graph().as_default() as g1, session_lib.Session( - server.target, graph=g1) as session: - keys, inputs = graph_io.read_keyed_batch_examples_shared_queue( - filenames, - batch_size, - reader=io_ops.TextLineReader, - randomize_input=False, - num_epochs=1, - queue_capacity=queue_capacity, - name=name) - self.assertAllEqual((None,), keys.get_shape().as_list()) - self.assertAllEqual((None,), inputs.get_shape().as_list()) - session.run([ - variables.local_variables_initializer(), - variables.global_variables_initializer() - ]) - - # Run the two queues once manually. - self._run_queue(worker_file_name_queue_name, session) - self._run_queue(example_queue_name, session) - - self.assertAllEqual(session.run(inputs), [b"ABC"]) - - # Run the worker and the example queue. - self._run_queue(worker_file_name_queue_name, session) - self._run_queue(example_queue_name, session) - - self.assertAllEqual(session.run(inputs), [b"DEF"]) - - with ops.Graph().as_default() as g2, session_lib.Session( - server.target, graph=g2) as session: - keys, inputs = graph_io.read_keyed_batch_examples_shared_queue( - filenames, - batch_size, - reader=io_ops.TextLineReader, - randomize_input=False, - num_epochs=1, - queue_capacity=queue_capacity, - name=name) - self.assertAllEqual((None,), keys.get_shape().as_list()) - self.assertAllEqual((None,), inputs.get_shape().as_list()) - - # Run the worker and the example queue. - self._run_queue(worker_file_name_queue_name, session) - self._run_queue(example_queue_name, session) - - self.assertAllEqual(session.run(inputs), [b"GHI"]) - - self.assertTrue(g1 is not g2) - - def test_batch_text_lines(self): - gfile.Glob = self._orig_glob - filename = self._create_temp_file("A\nB\nC\nD\nE\n") - - batch_size = 3 - queue_capacity = 10 - name = "my_batch" - - with ops.Graph().as_default() as g, self.session(graph=g) as session: - inputs = graph_io.read_batch_examples( - [filename], - batch_size, - reader=io_ops.TextLineReader, - randomize_input=False, - num_epochs=1, - queue_capacity=queue_capacity, - read_batch_size=10, - name=name) - self.assertAllEqual((None,), inputs.get_shape().as_list()) - session.run(variables.local_variables_initializer()) - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - self.assertAllEqual(session.run(inputs), [b"A", b"B", b"C"]) - self.assertAllEqual(session.run(inputs), [b"D", b"E"]) - with self.assertRaises(errors.OutOfRangeError): - session.run(inputs) - - coord.request_stop() - coord.join(threads) - - def test_keyed_read_text_lines(self): - gfile.Glob = self._orig_glob - filename = self._create_temp_file("ABC\nDEF\nGHK\n") - - batch_size = 1 - queue_capacity = 5 - name = "my_batch" - - with ops.Graph().as_default() as g, self.session(graph=g) as session: - keys, inputs = graph_io.read_keyed_batch_examples( - filename, - batch_size, - reader=io_ops.TextLineReader, - randomize_input=False, - num_epochs=1, - queue_capacity=queue_capacity, - name=name) - self.assertAllEqual((None,), keys.get_shape().as_list()) - self.assertAllEqual((None,), inputs.get_shape().as_list()) - session.run(variables.local_variables_initializer()) - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - self.assertAllEqual( - session.run([keys, inputs]), - [[filename.encode("utf-8") + b":1"], [b"ABC"]]) - self.assertAllEqual( - session.run([keys, inputs]), - [[filename.encode("utf-8") + b":2"], [b"DEF"]]) - self.assertAllEqual( - session.run([keys, inputs]), - [[filename.encode("utf-8") + b":3"], [b"GHK"]]) - with self.assertRaises(errors.OutOfRangeError): - session.run(inputs) - - coord.request_stop() - coord.join(threads) - - def test_keyed_parse_json(self): - gfile.Glob = self._orig_glob - filename = self._create_temp_file( - '{"features": {"feature": {"age": {"int64_list": {"value": [0]}}}}}\n' - '{"features": {"feature": {"age": {"int64_list": {"value": [1]}}}}}\n' - '{"features": {"feature": {"age": {"int64_list": {"value": [2]}}}}}\n') - - batch_size = 1 - queue_capacity = 5 - name = "my_batch" - - with ops.Graph().as_default() as g, self.session(graph=g) as session: - dtypes = {"age": parsing_ops.FixedLenFeature([1], dtypes_lib.int64)} - parse_fn = lambda example: parsing_ops.parse_single_example( # pylint: disable=g-long-lambda - parsing_ops.decode_json_example(example), dtypes) - keys, inputs = graph_io.read_keyed_batch_examples( - filename, - batch_size, - reader=io_ops.TextLineReader, - randomize_input=False, - num_epochs=1, - queue_capacity=queue_capacity, - parse_fn=parse_fn, - name=name) - self.assertAllEqual((None,), keys.get_shape().as_list()) - self.assertEqual(1, len(inputs)) - self.assertAllEqual((None, 1), inputs["age"].get_shape().as_list()) - session.run(variables.local_variables_initializer()) - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - key, age = session.run([keys, inputs["age"]]) - self.assertAllEqual(age, [[0]]) - self.assertAllEqual(key, [filename.encode("utf-8") + b":1"]) - key, age = session.run([keys, inputs["age"]]) - self.assertAllEqual(age, [[1]]) - self.assertAllEqual(key, [filename.encode("utf-8") + b":2"]) - key, age = session.run([keys, inputs["age"]]) - self.assertAllEqual(age, [[2]]) - self.assertAllEqual(key, [filename.encode("utf-8") + b":3"]) - with self.assertRaises(errors.OutOfRangeError): - session.run(inputs) - - coord.request_stop() - coord.join(threads) - - def test_keyed_features_filter(self): - gfile.Glob = self._orig_glob - lines = [ - '{"features": {"feature": {"age": {"int64_list": {"value": [2]}}}}}', - '{"features": {"feature": {"age": {"int64_list": {"value": [0]}}}}}', - '{"features": {"feature": {"age": {"int64_list": {"value": [1]}}}}}', - '{"features": {"feature": {"age": {"int64_list": {"value": [0]}}}}}', - '{"features": {"feature": {"age": {"int64_list": {"value": [3]}}}}}', - '{"features": {"feature": {"age": {"int64_list": {"value": [5]}}}}}' - ] - filename = self._create_temp_file("\n".join(lines)) - - batch_size = 2 - queue_capacity = 4 - name = "my_batch" - features = {"age": parsing_ops.FixedLenFeature([], dtypes_lib.int64)} - - def filter_fn(keys, examples_json): - del keys - serialized = parsing_ops.decode_json_example(examples_json) - examples = parsing_ops.parse_example(serialized, features) - return math_ops.less(examples["age"], 2) - - with ops.Graph().as_default() as g, self.session(graph=g) as session: - keys, inputs = graph_io._read_keyed_batch_examples_helper( - filename, - batch_size, - reader=io_ops.TextLineReader, - randomize_input=False, - num_epochs=1, - read_batch_size=batch_size, - queue_capacity=queue_capacity, - filter_fn=filter_fn, - name=name) - self.assertAllEqual((None,), keys.get_shape().as_list()) - self.assertAllEqual((None,), inputs.get_shape().as_list()) - session.run(variables.local_variables_initializer()) - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - # First batch of two filtered examples. - out_keys, out_vals = session.run((keys, inputs)) - self.assertAllEqual( - [filename.encode("utf-8") + b":2", filename.encode("utf-8") + b":3"], - out_keys) - self.assertAllEqual([lines[1].encode("utf-8"), lines[2].encode("utf-8")], - out_vals) - - # Second batch will only have one filtered example as that's the only - # remaining example that satisfies the filtering criterion. - out_keys, out_vals = session.run((keys, inputs)) - self.assertAllEqual([filename.encode("utf-8") + b":4"], out_keys) - self.assertAllEqual([lines[3].encode("utf-8")], out_vals) - - # Exhausted input. - with self.assertRaises(errors.OutOfRangeError): - session.run((keys, inputs)) - - coord.request_stop() - coord.join(threads) - - def test_queue_parsed_features_single_tensor(self): - with ops.Graph().as_default() as g, self.session(graph=g) as session: - features = {"test": constant_op.constant([1, 2, 3])} - _, queued_features = graph_io.queue_parsed_features(features) - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - out_features = session.run(queued_features["test"]) - self.assertAllEqual([1, 2, 3], out_features) - coord.request_stop() - coord.join(threads) - - def test_read_keyed_batch_features_shared_queue(self): - batch_size = 17 - shape = (0,) - fixed_feature = parsing_ops.FixedLenFeature( - shape=shape, dtype=dtypes_lib.float32) - feature = {"feature": fixed_feature} - reader = io_ops.TFRecordReader - - _, queued_feature = graph_io.read_keyed_batch_features_shared_queue( - _VALID_FILE_PATTERN, batch_size, feature, reader) - - with ops.Graph().as_default() as g, self.session(graph=g) as session: - features_result = graph_io.read_batch_features( - _VALID_FILE_PATTERN, batch_size, feature, reader) - session.run(variables.local_variables_initializer()) - - self.assertAllEqual( - queued_feature.get("feature").get_shape().as_list(), - features_result.get("feature").get_shape().as_list()) - - def test_get_file_names_errors(self): - # Raise bad file_pattern. - with self.assertRaises(ValueError): - graph_io._get_file_names([], True) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/learn_io/io_test.py b/tensorflow/contrib/learn/python/learn/learn_io/io_test.py deleted file mode 100644 index 678f80c45c7..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_io/io_test.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""tf.learn IO operation tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random - -# pylint: disable=wildcard-import -from tensorflow.contrib.learn.python import learn -from tensorflow.contrib.learn.python.learn import datasets -from tensorflow.contrib.learn.python.learn.estimators._sklearn import accuracy_score -from tensorflow.contrib.learn.python.learn.learn_io import * -from tensorflow.python.platform import test - -# pylint: enable=wildcard-import - - -class IOTest(test.TestCase): - # pylint: disable=undefined-variable - """tf.learn IO operation tests.""" - - def test_pandas_dataframe(self): - if HAS_PANDAS: - import pandas as pd # pylint: disable=g-import-not-at-top - random.seed(42) - iris = datasets.load_iris() - data = pd.DataFrame(iris.data) - labels = pd.DataFrame(iris.target) - classifier = learn.LinearClassifier( - feature_columns=learn.infer_real_valued_columns_from_input(data), - n_classes=3) - classifier.fit(data, labels, steps=100) - score = accuracy_score(labels[0], list(classifier.predict_classes(data))) - self.assertGreater(score, 0.5, "Failed with score = {0}".format(score)) - else: - print("No pandas installed. pandas-related tests are skipped.") - - def test_pandas_series(self): - if HAS_PANDAS: - import pandas as pd # pylint: disable=g-import-not-at-top - random.seed(42) - iris = datasets.load_iris() - data = pd.DataFrame(iris.data) - labels = pd.Series(iris.target) - classifier = learn.LinearClassifier( - feature_columns=learn.infer_real_valued_columns_from_input(data), - n_classes=3) - classifier.fit(data, labels, steps=100) - score = accuracy_score(labels, list(classifier.predict_classes(data))) - self.assertGreater(score, 0.5, "Failed with score = {0}".format(score)) - - def test_string_data_formats(self): - if HAS_PANDAS: - import pandas as pd # pylint: disable=g-import-not-at-top - with self.assertRaises(ValueError): - learn.io.extract_pandas_data(pd.DataFrame({"Test": ["A", "B"]})) - with self.assertRaises(ValueError): - learn.io.extract_pandas_labels(pd.DataFrame({"Test": ["A", "B"]})) - - def test_dask_io(self): - if HAS_DASK and HAS_PANDAS: - import pandas as pd # pylint: disable=g-import-not-at-top - import dask.dataframe as dd # pylint: disable=g-import-not-at-top - # test dask.dataframe - df = pd.DataFrame( - dict( - a=list("aabbcc"), b=list(range(6))), - index=pd.date_range( - start="20100101", periods=6)) - ddf = dd.from_pandas(df, npartitions=3) - extracted_ddf = extract_dask_data(ddf) - self.assertEqual( - extracted_ddf.divisions, (0, 2, 4, 6), - "Failed with divisions = {0}".format(extracted_ddf.divisions)) - self.assertEqual( - extracted_ddf.columns.tolist(), ["a", "b"], - "Failed with columns = {0}".format(extracted_ddf.columns)) - # test dask.series - labels = ddf["a"] - extracted_labels = extract_dask_labels(labels) - self.assertEqual( - extracted_labels.divisions, (0, 2, 4, 6), - "Failed with divisions = {0}".format(extracted_labels.divisions)) - # labels should only have one column - with self.assertRaises(ValueError): - extract_dask_labels(ddf) - else: - print("No dask installed. dask-related tests are skipped.") - - def test_dask_iris_classification(self): - if HAS_DASK and HAS_PANDAS: - import pandas as pd # pylint: disable=g-import-not-at-top - import dask.dataframe as dd # pylint: disable=g-import-not-at-top - random.seed(42) - iris = datasets.load_iris() - data = pd.DataFrame(iris.data) - data = dd.from_pandas(data, npartitions=2) - labels = pd.DataFrame(iris.target) - labels = dd.from_pandas(labels, npartitions=2) - classifier = learn.LinearClassifier( - feature_columns=learn.infer_real_valued_columns_from_input(data), - n_classes=3) - classifier.fit(data, labels, steps=100) - predictions = data.map_partitions(classifier.predict).compute() - score = accuracy_score(labels.compute(), predictions) - self.assertGreater(score, 0.5, "Failed with score = {0}".format(score)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/learn_io/numpy_io.py b/tensorflow/contrib/learn/python/learn/learn_io/numpy_io.py deleted file mode 100644 index 59a67636ae2..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_io/numpy_io.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Methods to allow dict of numpy arrays (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.estimator.inputs.numpy_io import numpy_input_fn as core_numpy_input_fn -from tensorflow.python.util.deprecation import deprecated - - -@deprecated(None, 'Use tf.compat.v1.estimator.inputs.numpy_input_fn.') -def numpy_input_fn(x, - y=None, - batch_size=128, - num_epochs=1, - shuffle=True, - queue_capacity=1000, - num_threads=1): - """This input_fn diffs from the core version with default `shuffle`.""" - return core_numpy_input_fn(x=x, - y=y, - batch_size=batch_size, - shuffle=shuffle, - num_epochs=num_epochs, - queue_capacity=queue_capacity, - num_threads=num_threads) diff --git a/tensorflow/contrib/learn/python/learn/learn_io/pandas_io.py b/tensorflow/contrib/learn/python/learn/learn_io/pandas_io.py deleted file mode 100644 index e9df7258a35..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_io/pandas_io.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Methods to allow pandas.DataFrame (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.estimator.inputs.pandas_io import pandas_input_fn as core_pandas_input_fn -from tensorflow.python.util.deprecation import deprecated - -try: - # pylint: disable=g-import-not-at-top - import pandas as pd - HAS_PANDAS = True -except IOError: - # Pandas writes a temporary file during import. If it fails, don't use pandas. - HAS_PANDAS = False -except ImportError: - HAS_PANDAS = False - -PANDAS_DTYPES = { - 'int8': 'int', - 'int16': 'int', - 'int32': 'int', - 'int64': 'int', - 'uint8': 'int', - 'uint16': 'int', - 'uint32': 'int', - 'uint64': 'int', - 'float16': 'float', - 'float32': 'float', - 'float64': 'float', - 'bool': 'i' -} - - -@deprecated(None, 'Please use tf.compat.v1.estimator.inputs.pandas_input_fn') -def pandas_input_fn(x, - y=None, - batch_size=128, - num_epochs=1, - shuffle=True, - queue_capacity=1000, - num_threads=1, - target_column='target'): - """This input_fn diffs from the core version with default `shuffle`.""" - return core_pandas_input_fn(x=x, - y=y, - batch_size=batch_size, - shuffle=shuffle, - num_epochs=num_epochs, - queue_capacity=queue_capacity, - num_threads=num_threads, - target_column=target_column) - - -@deprecated(None, 'Please access pandas data directly.') -def extract_pandas_data(data): - """Extract data from pandas.DataFrame for predictors. - - Given a DataFrame, will extract the values and cast them to float. The - DataFrame is expected to contain values of type int, float or bool. - - Args: - data: `pandas.DataFrame` containing the data to be extracted. - - Returns: - A numpy `ndarray` of the DataFrame's values as floats. - - Raises: - ValueError: if data contains types other than int, float or bool. - """ - if not isinstance(data, pd.DataFrame): - return data - - bad_data = [column for column in data - if data[column].dtype.name not in PANDAS_DTYPES] - - if not bad_data: - return data.values.astype('float') - else: - error_report = [("'" + str(column) + "' type='" + - data[column].dtype.name + "'") for column in bad_data] - raise ValueError('Data types for extracting pandas data must be int, ' - 'float, or bool. Found: ' + ', '.join(error_report)) - - -@deprecated(None, 'Please access pandas data directly.') -def extract_pandas_matrix(data): - """Extracts numpy matrix from pandas DataFrame. - - Args: - data: `pandas.DataFrame` containing the data to be extracted. - - Returns: - A numpy `ndarray` of the DataFrame's values. - """ - if not isinstance(data, pd.DataFrame): - return data - - return data.as_matrix() - - -@deprecated(None, 'Please access pandas data directly.') -def extract_pandas_labels(labels): - """Extract data from pandas.DataFrame for labels. - - Args: - labels: `pandas.DataFrame` or `pandas.Series` containing one column of - labels to be extracted. - - Returns: - A numpy `ndarray` of labels from the DataFrame. - - Raises: - ValueError: if more than one column is found or type is not int, float or - bool. - """ - if isinstance(labels, - pd.DataFrame): # pandas.Series also belongs to DataFrame - if len(labels.columns) > 1: - raise ValueError('Only one column for labels is allowed.') - - bad_data = [column for column in labels - if labels[column].dtype.name not in PANDAS_DTYPES] - if not bad_data: - return labels.values - else: - error_report = ["'" + str(column) + "' type=" - + str(labels[column].dtype.name) for column in bad_data] - raise ValueError('Data types for extracting labels must be int, ' - 'float, or bool. Found: ' + ', '.join(error_report)) - else: - return labels diff --git a/tensorflow/contrib/learn/python/learn/learn_io/pandas_io_test.py b/tensorflow/contrib/learn/python/learn/learn_io/pandas_io_test.py deleted file mode 100644 index 396539a76ae..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_io/pandas_io_test.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for pandas_io.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.learn.python.learn.learn_io import pandas_io -from tensorflow.python.framework import errors -from tensorflow.python.platform import test -from tensorflow.python.training import coordinator -from tensorflow.python.training import queue_runner_impl - -# pylint: disable=g-import-not-at-top -try: - import pandas as pd - HAS_PANDAS = True -except ImportError: - HAS_PANDAS = False - - -class PandasIoTest(test.TestCase): - - def makeTestDataFrame(self): - index = np.arange(100, 104) - a = np.arange(4) - b = np.arange(32, 36) - x = pd.DataFrame({'a': a, 'b': b}, index=index) - y = pd.Series(np.arange(-32, -28), index=index) - return x, y - - def callInputFnOnce(self, input_fn, session): - results = input_fn() - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - result_values = session.run(results) - coord.request_stop() - coord.join(threads) - return result_values - - def testPandasInputFn_IndexMismatch(self): - if not HAS_PANDAS: - return - x, _ = self.makeTestDataFrame() - y_noindex = pd.Series(np.arange(-32, -28)) - with self.assertRaises(ValueError): - pandas_io.pandas_input_fn( - x, y_noindex, batch_size=2, shuffle=False, num_epochs=1) - - def testPandasInputFn_ProducesExpectedOutputs(self): - if not HAS_PANDAS: - return - with self.cached_session() as session: - x, y = self.makeTestDataFrame() - input_fn = pandas_io.pandas_input_fn( - x, y, batch_size=2, shuffle=False, num_epochs=1) - - features, target = self.callInputFnOnce(input_fn, session) - - self.assertAllEqual(features['a'], [0, 1]) - self.assertAllEqual(features['b'], [32, 33]) - self.assertAllEqual(target, [-32, -31]) - - def testPandasInputFn_ProducesOutputsForLargeBatchAndMultipleEpochs(self): - if not HAS_PANDAS: - return - with self.cached_session() as session: - index = np.arange(100, 102) - a = np.arange(2) - b = np.arange(32, 34) - x = pd.DataFrame({'a': a, 'b': b}, index=index) - y = pd.Series(np.arange(-32, -30), index=index) - input_fn = pandas_io.pandas_input_fn( - x, y, batch_size=128, shuffle=False, num_epochs=2) - - results = input_fn() - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - features, target = session.run(results) - self.assertAllEqual(features['a'], [0, 1, 0, 1]) - self.assertAllEqual(features['b'], [32, 33, 32, 33]) - self.assertAllEqual(target, [-32, -31, -32, -31]) - - with self.assertRaises(errors.OutOfRangeError): - session.run(results) - - coord.request_stop() - coord.join(threads) - - def testPandasInputFn_ProducesOutputsWhenDataSizeNotDividedByBatchSize(self): - if not HAS_PANDAS: - return - with self.cached_session() as session: - index = np.arange(100, 105) - a = np.arange(5) - b = np.arange(32, 37) - x = pd.DataFrame({'a': a, 'b': b}, index=index) - y = pd.Series(np.arange(-32, -27), index=index) - - input_fn = pandas_io.pandas_input_fn( - x, y, batch_size=2, shuffle=False, num_epochs=1) - - results = input_fn() - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - - features, target = session.run(results) - self.assertAllEqual(features['a'], [0, 1]) - self.assertAllEqual(features['b'], [32, 33]) - self.assertAllEqual(target, [-32, -31]) - - features, target = session.run(results) - self.assertAllEqual(features['a'], [2, 3]) - self.assertAllEqual(features['b'], [34, 35]) - self.assertAllEqual(target, [-30, -29]) - - features, target = session.run(results) - self.assertAllEqual(features['a'], [4]) - self.assertAllEqual(features['b'], [36]) - self.assertAllEqual(target, [-28]) - - with self.assertRaises(errors.OutOfRangeError): - session.run(results) - - coord.request_stop() - coord.join(threads) - - def testPandasInputFn_OnlyX(self): - if not HAS_PANDAS: - return - with self.cached_session() as session: - x, _ = self.makeTestDataFrame() - input_fn = pandas_io.pandas_input_fn( - x, y=None, batch_size=2, shuffle=False, num_epochs=1) - - features = self.callInputFnOnce(input_fn, session) - - self.assertAllEqual(features['a'], [0, 1]) - self.assertAllEqual(features['b'], [32, 33]) - - def testPandasInputFn_ExcludesIndex(self): - if not HAS_PANDAS: - return - with self.cached_session() as session: - x, y = self.makeTestDataFrame() - input_fn = pandas_io.pandas_input_fn( - x, y, batch_size=2, shuffle=False, num_epochs=1) - - features, _ = self.callInputFnOnce(input_fn, session) - - self.assertFalse('index' in features) - - def assertInputsCallableNTimes(self, input_fn, session, n): - inputs = input_fn() - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(session, coord=coord) - for _ in range(n): - session.run(inputs) - with self.assertRaises(errors.OutOfRangeError): - session.run(inputs) - coord.request_stop() - coord.join(threads) - - def testPandasInputFn_RespectsEpoch_NoShuffle(self): - if not HAS_PANDAS: - return - with self.cached_session() as session: - x, y = self.makeTestDataFrame() - input_fn = pandas_io.pandas_input_fn( - x, y, batch_size=4, shuffle=False, num_epochs=1) - - self.assertInputsCallableNTimes(input_fn, session, 1) - - def testPandasInputFn_RespectsEpoch_WithShuffle(self): - if not HAS_PANDAS: - return - with self.cached_session() as session: - x, y = self.makeTestDataFrame() - input_fn = pandas_io.pandas_input_fn( - x, y, batch_size=4, shuffle=True, num_epochs=1) - - self.assertInputsCallableNTimes(input_fn, session, 1) - - def testPandasInputFn_RespectsEpoch_WithShuffleAutosize(self): - if not HAS_PANDAS: - return - with self.cached_session() as session: - x, y = self.makeTestDataFrame() - input_fn = pandas_io.pandas_input_fn( - x, y, batch_size=2, shuffle=True, queue_capacity=None, num_epochs=2) - - self.assertInputsCallableNTimes(input_fn, session, 4) - - def testPandasInputFn_RespectsEpochUnevenBatches(self): - if not HAS_PANDAS: - return - x, y = self.makeTestDataFrame() - with self.cached_session() as session: - input_fn = pandas_io.pandas_input_fn( - x, y, batch_size=3, shuffle=False, num_epochs=1) - - # Before the last batch, only one element of the epoch should remain. - self.assertInputsCallableNTimes(input_fn, session, 2) - - def testPandasInputFn_Idempotent(self): - if not HAS_PANDAS: - return - x, y = self.makeTestDataFrame() - for _ in range(2): - pandas_io.pandas_input_fn( - x, y, batch_size=2, shuffle=False, num_epochs=1)() - for _ in range(2): - pandas_io.pandas_input_fn( - x, y, batch_size=2, shuffle=True, num_epochs=1)() - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/learn_runner.py b/tensorflow/contrib/learn/python/learn/learn_runner.py deleted file mode 100644 index d719a3e488b..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_runner.py +++ /dev/null @@ -1,297 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Runs an Experiment (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.estimators import run_config as run_config_lib -from tensorflow.contrib.learn.python.learn.experiment import Experiment -from tensorflow.contrib.training.python.training import hparam as hparam_lib -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util.deprecation import deprecated - - -# TODO(xiejw): Refactor the learn_runner to make code reusable. -def _execute_schedule(experiment, schedule): - """Execute the method named `schedule` of `experiment`.""" - if not hasattr(experiment, schedule): - logging.error('Schedule references non-existent task %s', schedule) - valid_tasks = [x for x in dir(experiment) - if not x.startswith('_') - and callable(getattr(experiment, x))] - logging.error('Allowed values for this experiment are: %s', valid_tasks) - raise ValueError('Schedule references non-existent task %s' % schedule) - - task = getattr(experiment, schedule) - if not callable(task): - logging.error('Schedule references non-callable member %s', schedule) - valid_tasks = [x for x in dir(experiment) - if not x.startswith('_') - and callable(getattr(experiment, x))] - logging.error('Allowed values for this experiment are: %s', valid_tasks) - raise TypeError('Schedule references non-callable member %s' % schedule) - return task() - - -def _wrapped_experiment_fn_with_uid_check(experiment_fn, require_hparams=False): - """Wraps the `RunConfig` uid check with `experiment_fn`. - - For `experiment_fn` which takes `run_config`, it is expected that the - `run_config` is passed to the Estimator correctly. Toward that, the wrapped - `experiment_fn` compares the `uid` of the `RunConfig` instance. - - Args: - experiment_fn: The original `experiment_fn` which takes `run_config` and - `hparams`. - require_hparams: If True, the `hparams` passed to `experiment_fn` cannot be - `None`. - - Returns: - A experiment_fn with same signature. - """ - def wrapped_experiment_fn(run_config, hparams): - """Calls experiment_fn and checks the uid of `RunConfig`.""" - if not isinstance(run_config, run_config_lib.RunConfig): - raise ValueError( - '`run_config` must be `tf.contrib.learn.RunConfig` instance') - if not run_config.model_dir: - raise ValueError( - 'Must specify a model directory `model_dir` in `run_config`.') - if hparams is not None and not isinstance(hparams, hparam_lib.HParams): - raise ValueError('`hparams` must be `HParams` instance') - if require_hparams and hparams is None: - raise ValueError('`hparams` cannot be `None`.') - - expected_uid = run_config.uid() - experiment = experiment_fn(run_config, hparams) - - if not isinstance(experiment, Experiment): - raise TypeError('Experiment builder did not return an Experiment ' - 'instance, got %s instead.' % type(experiment)) - - config_from_estimator = experiment.estimator.config - if not hasattr(config_from_estimator, 'uid'): - raise RuntimeError( - 'Pass `run_config` argument of the `experiment_fn` to the Estimator ' - 'in Experiment. It is likely a different `RunConfig` is passed to ' - '`Estimator` or the `config` constructor argument in `Estimator` ' - 'is not set.') - - if config_from_estimator.uid() != expected_uid: - raise RuntimeError( - '`RunConfig` instance is expected to be used by the `Estimator` ' - 'inside the `Experiment`. expected {}, but got {}'.format( - expected_uid, experiment.estimator.config.uid())) - return experiment - return wrapped_experiment_fn - - -@deprecated(None, 'Use tf.estimator.train_and_evaluate.') -def run(experiment_fn, output_dir=None, schedule=None, run_config=None, - hparams=None): - """Make and run an experiment. - - It creates an Experiment by calling `experiment_fn`. Then it calls the - function named as `schedule` of the Experiment. - - If schedule is not provided, then the default schedule for the current task - type is used. The defaults are as follows: - - * 'ps' maps to 'serve' - * 'worker' maps to 'train' - * 'master' maps to 'local_run' - - If the experiment's config does not include a task type, then an exception - is raised. - - Example with `run_config` (Recommended): - ``` - def _create_my_experiment(run_config, hparams): - - # You can change a subset of the run_config properties as - # run_config = run_config.replace(save_checkpoints_steps=500) - - return tf.contrib.learn.Experiment( - estimator=my_estimator(config=run_config, hparams=hparams), - train_input_fn=my_train_input, - eval_input_fn=my_eval_input) - - learn_runner.run( - experiment_fn=_create_my_experiment, - run_config=run_config_lib.RunConfig(model_dir="some/output/dir"), - schedule="train_and_evaluate", - hparams=_create_default_hparams()) - ``` - or simply as - ``` - learn_runner.run( - experiment_fn=_create_my_experiment, - run_config=run_config_lib.RunConfig(model_dir="some/output/dir")) - ``` - if `hparams` is not used by the `Estimator`. On a single machine, `schedule` - defaults to `train_and_evaluate`. - - Example with `output_dir` (deprecated): - ``` - def _create_my_experiment(output_dir): - return tf.contrib.learn.Experiment( - estimator=my_estimator(model_dir=output_dir), - train_input_fn=my_train_input, - eval_input_fn=my_eval_input) - - learn_runner.run( - experiment_fn=_create_my_experiment, - output_dir="some/output/dir", - schedule="train") - ``` - Args: - experiment_fn: A function that creates an `Experiment`. It could be one of - the two following signatures: - 1) [Deprecated] It accepts an argument `output_dir` which should be used - to create the `Estimator` (passed as `model_dir` to its constructor). It - must return an `Experiment`. For this case, `run_config` and `hparams` - must be None. - 2) It accepts two arguments `run_config` and `hparams`, which should be - used to create the `Estimator` (`run_config` passed as `config` to its - constructor; `hparams` used as the hyper-parameters of the model). - It must return an `Experiment`. For this case, `output_dir` must be None. - output_dir: Base output directory [Deprecated]. - schedule: The name of the method in the `Experiment` to run. - run_config: `RunConfig` instance. The `run_config.model_dir` must be - non-empty. If `run_config` is set, `output_dir` must be None. - hparams: `HParams` instance. The default hyper-parameters, which will be - passed to the `experiment_fn` if `run_config` is not None. - - Returns: - The return value of function `schedule`. - - Raises: - ValueError: If both `output_dir` and `run_config` are empty or set, - `schedule` is None but no task type is set in the built experiment's - config, the task type has no default, `run_config.model_dir` is empty or - `schedule` doesn't reference a member of `Experiment`. - TypeError: `schedule` references non-callable member. - """ - - if output_dir is not None and run_config is not None: - raise ValueError('Cannot provide both `output_dir` and `run_config`') - - if output_dir is None and run_config is None: - raise ValueError('Must set value for `output_dir` or `run_config`') - - if not callable(experiment_fn): - raise TypeError('Experiment builder "%s" is not callable.' % - experiment_fn) - - experiment = None - if run_config is not None: - wrapped_experiment_fn = _wrapped_experiment_fn_with_uid_check(experiment_fn) - experiment = wrapped_experiment_fn(run_config=run_config, hparams=hparams) - else: - if not output_dir: - raise ValueError('Must specify an output directory') - if hparams is not None: - raise ValueError( - 'Must set `hparams` as None for `experiment_fn` with `output_dir`.') - # Call the builder - experiment = experiment_fn(output_dir=output_dir) - if not isinstance(experiment, Experiment): - raise TypeError('Experiment builder did not return an Experiment ' - 'instance, got %s instead.' % type(experiment)) - - # Get the schedule - run_config = run_config or experiment.estimator.config - schedule = schedule or _get_default_schedule(run_config) - - return _execute_schedule(experiment, schedule) - - -@deprecated(None, 'Use tf.estimator.train_and_evaluate.') -def tune(experiment_fn, tuner): - """Tune an experiment with hyper-parameters. - - It iterates trials by running the Experiment for each trial with the - corresponding hyper-parameters. For each trial, it retrieves the - hyper-parameters from `tuner`, creates an Experiment by calling experiment_fn, - and then reports the measure back to `tuner`. - - Example: - ``` - def _create_my_experiment(run_config, hparams): - hidden_units = [hparams.unit_per_layer] * hparams.num_hidden_layers - - return tf.contrib.learn.Experiment( - estimator=DNNClassifier(config=run_config, hidden_units=hidden_units), - train_input_fn=my_train_input, - eval_input_fn=my_eval_input) - - tuner = create_tuner(study_configuration, objective_key) - - learn_runner.tune(experiment_fn=_create_my_experiment, tuner) - ``` - Args: - experiment_fn: A function that creates an `Experiment`. It should accept an - argument `run_config` which should be used to create the `Estimator` ( - passed as `config` to its constructor), and an argument `hparams`, which - should be used for hyper-parameters tuning. It must return an - `Experiment`. - tuner: A `Tuner` instance. - """ - while tuner.next_trial(): - tuner.run_experiment( - _wrapped_experiment_fn_with_uid_check( - experiment_fn, require_hparams=True)) - - -def _is_distributed(config): - """Returns true if this is a distributed job.""" - if not config.cluster_spec: - return False - - # This is considered a distributed job if there is more than one task - # in the cluster spec. - task_count = 0 - for job in config.cluster_spec.jobs: - for _ in config.cluster_spec.job_tasks(job): - task_count += 1 - - return task_count > 1 - - -def _get_default_schedule(config): - """Returns the default schedule for the provided RunConfig.""" - if not config or not _is_distributed(config): - return 'train_and_evaluate' - - if not config.task_type: - raise ValueError('Must specify a schedule') - - if config.task_type == run_config_lib.TaskType.MASTER: - # TODO(rhaertel): handle the case where there is more than one master - # or explicitly disallow such a case. - return 'train_and_evaluate' - elif config.task_type == run_config_lib.TaskType.PS: - return 'run_std_server' - elif config.task_type == run_config_lib.TaskType.WORKER: - return 'train' - - raise ValueError('No default schedule for task type: %s' % (config.task_type)) diff --git a/tensorflow/contrib/learn/python/learn/learn_runner_lib.py b/tensorflow/contrib/learn/python/learn/learn_runner_lib.py deleted file mode 100644 index ba2d067787c..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_runner_lib.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities to run and tune an Experiment (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. - -@@run -@@tune -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.learn_runner import * # pylint: disable=wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/learn/python/learn/learn_runner_test.py b/tensorflow/contrib/learn/python/learn/learn_runner_test.py deleted file mode 100644 index ea96b661863..00000000000 --- a/tensorflow/contrib/learn/python/learn/learn_runner_test.py +++ /dev/null @@ -1,421 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""learn_main tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import json -import os - -from tensorflow.contrib.learn.python.learn import evaluable # pylint: disable=g-import-not-at-top -from tensorflow.contrib.learn.python.learn import experiment -from tensorflow.contrib.learn.python.learn import learn_runner -from tensorflow.contrib.learn.python.learn import trainable - -from tensorflow.contrib.learn.python.learn.estimators import run_config as run_config_lib -from tensorflow.contrib.training.python.training import hparam as hparam_lib -from tensorflow.python.estimator import run_config as core_run_config_lib -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging - -patch = test.mock.patch - -_MODIR_DIR = "/tmp" -_HPARAMS = hparam_lib.HParams(learning_rate=0.01) -_MUST_SPECIFY_OUTPUT_DIR_MSG = "Must specify an output directory" -_MISSING_MODEL_DIR_ERR_MSG = ( - "Must specify a model directory `model_dir` in `run_config`.") -_EXP_NOT_CALLABLE_MSG = "Experiment builder .* is not callable" -_INVALID_HPARAMS_ERR_MSG = "`hparams` must be `HParams` instance" -_NOT_EXP_TYPE_MSG = "Experiment builder did not return an Experiment" -_NON_EXIST_TASK_MSG = "Schedule references non-existent task" -_NON_CALLABLE_MSG = "Schedule references non-callable member" -_MUST_SPECIFY_OUTPUT_DIR_OR_CONFIG_MSG = ( - "Must set value for `output_dir` or `run_config`") -_HPARAMS_CANNOT_BE_SET_FOR_OUTPUT_DIR_MSG = ( - "Must set `hparams` as None for `experiment_fn` with `output_dir`.") -_CANNOT_SET_BOTH_OUTPUT_DIR_AND_CONFIG_MSG = ( - "Cannot provide both `output_dir` and `run_config`") -_INVALID_RUN_CONFIG_TYPE_MSG = ( - "`run_config` must be `tf.contrib.learn.RunConfig` instance") -_RUN_CONFIG_UID_CHECK_ERR_MSG = ( - "`RunConfig` instance is expected to be used by the `Estimator`") -_MISSING_RUN_CONFIG_UID_ERR_MSG = ( - "Pass `run_config` argument of the `experiment_fn` to the Estimator") - - -class TestExperiment(experiment.Experiment): - - def __init__(self, default=None, config=None, model_dir=None): - self.default = default - self.config = config - internal_model_dir = model_dir or config.model_dir - self._model_dir = internal_model_dir - - class Estimator(evaluable.Evaluable, trainable.Trainable): - config = self.config - - @property - def model_dir(self): - return internal_model_dir - - def fit(self, x=None, y=None, input_fn=None, steps=None, batch_size=None, - monitors=None, max_steps=None): - raise NotImplementedError - - def evaluate(self, x=None, y=None, input_fn=None, feed_fn=None, - batch_size=None, steps=None, metrics=None, name=None, - checkpoint_path=None, hooks=None): - raise NotImplementedError - - super(TestExperiment, self).__init__(Estimator(), None, None) - - def local_run(self): - return "local_run-{}".format(self._model_dir) - - def train(self): - return "train-{}".format(self._model_dir) - - def run_std_server(self): - return "run_std_server-{}".format(self._model_dir) - - def train_and_evaluate(self): - return "train_and_evaluate-{}".format(self._model_dir) - - def simple_task(self): - return "simple_task, default=%s." % self.default - - -# pylint: disable=unused-argument -def build_experiment(output_dir): - tf_logging.info("In default build_experiment.") - return TestExperiment(model_dir=output_dir) - - -def build_experiment_fn_for_output_dir(run_config=None): - def _build_experiment(output_dir): - tf_logging.info("In default build_experiment.") - return TestExperiment(config=run_config, model_dir=output_dir) - return _build_experiment - - -def build_experiment_for_run_config(run_config, hparams): - if hparams is not None and hparams != _HPARAMS: - raise ValueError("hparams is not set correctly") - return TestExperiment(config=run_config) - - -def build_non_experiment(output_dir): - return "Ceci n'est pas un Experiment." - - -# pylint: enable=unused-argument - - -def build_distributed_cluster_spec(): - return { - run_config_lib.TaskType.PS: ["localhost:1234", "localhost:1235"], - run_config_lib.TaskType.WORKER: ["localhost:1236", "localhost:1237"], - run_config_lib.TaskType.MASTER: ["localhost:1238"], - "foo_has_no_default_schedule": ["localhost:1239"] - } - - -def build_non_distributed_cluster_spec(): - return {"foo": ["localhost:1234"]} - - -class LearnRunnerRunWithOutputDirTest(test.TestCase): - - def setUp(self): - # Ensure the TF_CONFIG environment variable is unset for all tests. - os.environ.pop("TF_CONFIG", None) - - def test_run_with_custom_schedule(self): - self.assertEqual( - "simple_task, default=None.", - learn_runner.run(build_experiment, - output_dir=_MODIR_DIR, - schedule="simple_task")) - - def test_run_with_explicit_local_run(self): - self.assertEqual( - "local_run-" + _MODIR_DIR, - learn_runner.run(build_experiment, - output_dir=_MODIR_DIR, - schedule="local_run")) - - def test_fail_output_dir_and_run_config_are_both_set(self): - with self.assertRaisesRegexp( - ValueError, _CANNOT_SET_BOTH_OUTPUT_DIR_AND_CONFIG_MSG): - learn_runner.run(build_experiment, - output_dir=_MODIR_DIR, - schedule="simple_task", - run_config=run_config_lib.RunConfig()) - - def test_fail_empty_output_dir(self): - with self.assertRaisesRegexp(ValueError, _MUST_SPECIFY_OUTPUT_DIR_MSG): - learn_runner.run(build_experiment, output_dir="", schedule="simple_task") - - def test_fail_no_output_dir(self): - with self.assertRaisesRegexp( - ValueError, _MUST_SPECIFY_OUTPUT_DIR_OR_CONFIG_MSG): - learn_runner.run(build_experiment, None, "simple_task") - - def test_fail_hparams_are_set(self): - hparams = _HPARAMS - with self.assertRaisesRegexp( - ValueError, _HPARAMS_CANNOT_BE_SET_FOR_OUTPUT_DIR_MSG): - learn_runner.run( - build_experiment, _MODIR_DIR, schedule="simple_task", hparams=hparams) - - def test_fail_non_callable(self): - with self.assertRaisesRegexp(TypeError, _EXP_NOT_CALLABLE_MSG): - learn_runner.run("not callable", _MODIR_DIR, "simple_test") - - def test_fail_not_experiment(self): - with self.assertRaisesRegexp(TypeError, _NOT_EXP_TYPE_MSG): - learn_runner.run(build_non_experiment, _MODIR_DIR, "simple_test") - - def test_fail_non_existent_task(self): - with self.assertRaisesRegexp(ValueError, _NON_EXIST_TASK_MSG): - learn_runner.run(build_experiment, _MODIR_DIR, "mirage") - - def test_fail_non_callable_task(self): - with self.assertRaisesRegexp(TypeError, _NON_CALLABLE_MSG): - learn_runner.run(build_experiment, _MODIR_DIR, "default") - - -class LearnRunnerRunWithRunConfigTest(test.TestCase): - - def setUp(self): - # Ensure the TF_CONFIG environment variable is unset for all tests. - os.environ.pop("TF_CONFIG", None) - - def test_run_with_custom_schedule(self): - run_config = run_config_lib.RunConfig(model_dir=_MODIR_DIR) - self.assertEqual( - "simple_task, default=None.", - learn_runner.run(build_experiment_for_run_config, - run_config=run_config, - schedule="simple_task")) - - def test_run_with_hparams(self): - run_config = run_config_lib.RunConfig(model_dir=_MODIR_DIR) - self.assertEqual( - "simple_task, default=None.", - learn_runner.run(build_experiment_for_run_config, - run_config=run_config, - schedule="simple_task", - hparams=_HPARAMS)) - - def test_run_with_explicit_local_run(self): - run_config = run_config_lib.RunConfig(model_dir=_MODIR_DIR) - self.assertEqual( - "local_run-" + _MODIR_DIR, - learn_runner.run(build_experiment_for_run_config, - run_config=run_config, - schedule="local_run")) - - def test_fail_empty_output_dir(self): - run_config = run_config_lib.RunConfig(model_dir="") - with self.assertRaisesRegexp(ValueError, _MISSING_MODEL_DIR_ERR_MSG): - learn_runner.run(build_experiment_for_run_config, - run_config=run_config, - schedule="local_run") - - def test_fail_no_output_dir(self): - run_config = run_config_lib.RunConfig() - with self.assertRaisesRegexp(ValueError, _MISSING_MODEL_DIR_ERR_MSG): - learn_runner.run(build_experiment_for_run_config, - run_config=run_config, - schedule="local_run") - - def test_fail_invalid_run_config_type(self): - run_config = "invalid_run_config" - with self.assertRaisesRegexp(ValueError, _INVALID_RUN_CONFIG_TYPE_MSG): - learn_runner.run(build_experiment_for_run_config, - run_config=run_config, - schedule="local_run") - - def test_fail_invalid_hparams_type(self): - run_config = run_config_lib.RunConfig(model_dir=_MODIR_DIR) - with self.assertRaisesRegexp(ValueError, _INVALID_HPARAMS_ERR_MSG): - learn_runner.run(build_experiment_for_run_config, - run_config=run_config, - schedule="local_run", - hparams=["hparams"]) - - def test_fail_non_callable(self): - run_config = run_config_lib.RunConfig(model_dir=_MODIR_DIR) - with self.assertRaisesRegexp(TypeError, _EXP_NOT_CALLABLE_MSG): - learn_runner.run("not callable", - run_config=run_config, - schedule="simple_task") - - def test_fail_not_experiment(self): - def _experiment_fn(run_config, hparams): - del run_config, hparams # unused. - return "not experiment" - - run_config = run_config_lib.RunConfig(model_dir=_MODIR_DIR) - with self.assertRaisesRegexp(TypeError, _NOT_EXP_TYPE_MSG): - learn_runner.run(_experiment_fn, - run_config=run_config, - schedule="simple_task") - - def test_fail_non_existent_task(self): - run_config = run_config_lib.RunConfig(model_dir=_MODIR_DIR) - with self.assertRaisesRegexp(ValueError, _NON_EXIST_TASK_MSG): - learn_runner.run(build_experiment_for_run_config, - run_config=run_config, - schedule="mirage") - - def test_fail_non_callable_task(self): - run_config = run_config_lib.RunConfig(model_dir=_MODIR_DIR) - with self.assertRaisesRegexp(TypeError, _NON_CALLABLE_MSG): - learn_runner.run(build_experiment_for_run_config, - run_config=run_config, - schedule="default") - - def test_basic_run_config_uid_check(self): - expected_run_config = run_config_lib.RunConfig(model_dir=_MODIR_DIR) - - def _experiment_fn(run_config, hparams): - del run_config, hparams # unused. - # Explicitly use a new run_config. - new_config = run_config_lib.RunConfig(model_dir=_MODIR_DIR + "/123") - - return TestExperiment(config=new_config) - - with self.assertRaisesRegexp(RuntimeError, _RUN_CONFIG_UID_CHECK_ERR_MSG): - learn_runner.run(experiment_fn=_experiment_fn, - run_config=expected_run_config) - - def test_fail_invalid_experiment_config_type(self): - expected_run_config = run_config_lib.RunConfig(model_dir=_MODIR_DIR) - - def _experiment_fn(run_config, hparams): - del run_config, hparams # unused. - # Explicitly use a new run_config without `uid` method. - new_config = core_run_config_lib.RunConfig( - model_dir=_MODIR_DIR + "/123") - - return TestExperiment(config=new_config) - - with self.assertRaisesRegexp(RuntimeError, - _MISSING_RUN_CONFIG_UID_ERR_MSG): - learn_runner.run(experiment_fn=_experiment_fn, - run_config=expected_run_config) - - -class LearnRunnerDefaultScheduleTest(test.TestCase): - - def setUp(self): - # Ensure the TF_CONFIG environment variable is unset for all tests. - os.environ.pop("TF_CONFIG", None) - - def test_schedule_from_tf_config_runs_train_on_worker(self): - os.environ["TF_CONFIG"] = json.dumps({ - "cluster": build_distributed_cluster_spec(), - "task": { - "type": run_config_lib.TaskType.WORKER - } - }) - # RunConfig constructor will set job_name from TF_CONFIG. - config = run_config_lib.RunConfig() - self.assertEqual( - "train-" + _MODIR_DIR, - learn_runner.run( - build_experiment_fn_for_output_dir(config), - output_dir=_MODIR_DIR)) - - def test_schedule_from_tf_config_runs_train_and_evaluate_on_master(self): - tf_config = { - "cluster": build_distributed_cluster_spec(), - "task": { - "type": run_config_lib.TaskType.MASTER - } - } - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - config = run_config_lib.RunConfig() - self.assertEqual( - "train_and_evaluate-" + _MODIR_DIR, - learn_runner.run( - build_experiment_fn_for_output_dir(config), - output_dir=_MODIR_DIR)) - - def test_schedule_from_tf_config_runs_serve_on_ps(self): - tf_config = { - "cluster": build_distributed_cluster_spec(), - "task": { - "type": run_config_lib.TaskType.PS - } - } - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - config = run_config_lib.RunConfig() - self.assertEqual( - "run_std_server-" + _MODIR_DIR, - learn_runner.run( - build_experiment_fn_for_output_dir(config), - output_dir=_MODIR_DIR)) - - def test_no_schedule_and_no_config_runs_train_and_evaluate(self): - self.assertEqual( - "train_and_evaluate-" + _MODIR_DIR, - learn_runner.run(build_experiment, output_dir=_MODIR_DIR)) - - def test_no_schedule_and_non_distributed_runs_train_and_evaluate(self): - tf_config = {"cluster": build_non_distributed_cluster_spec()} - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - config = run_config_lib.RunConfig() - self.assertEqual( - "train_and_evaluate-" + _MODIR_DIR, - learn_runner.run( - build_experiment_fn_for_output_dir(config), - output_dir=_MODIR_DIR)) - - def test_fail_task_type_with_no_default_schedule(self): - tf_config = { - "cluster": build_distributed_cluster_spec(), - "task": { - "type": "foo_has_no_default_schedule" - } - } - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - config = run_config_lib.RunConfig() - create_experiment_fn = lambda output_dir: TestExperiment(config=config) - self.assertRaisesRegexp(ValueError, - "No default schedule", - learn_runner.run, - create_experiment_fn, - _MODIR_DIR) - - def test_fail_schedule_from_config_with_no_task_type(self): - tf_config = {"cluster": build_distributed_cluster_spec()} - with patch.dict("os.environ", {"TF_CONFIG": json.dumps(tf_config)}): - config = run_config_lib.RunConfig() - self.assertRaisesRegexp( - ValueError, - "Must specify a schedule", - learn_runner.run, - lambda output_dir: TestExperiment(config=config), - output_dir=_MODIR_DIR) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/metric_spec.py b/tensorflow/contrib/learn/python/learn/metric_spec.py deleted file mode 100644 index 97220365d5d..00000000000 --- a/tensorflow/contrib/learn/python/learn/metric_spec.py +++ /dev/null @@ -1,442 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""The metric spec class to flexibly connect models and metrics (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import six - -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util import tf_inspect -from tensorflow.python.util.deprecation import deprecated - - -def _assert_named_args(sentinel): - if sentinel is not None: - raise ValueError( - '`metric_fn` requires named args: ' - '`labels`, `predictions`, and optionally `weights`.') - - -def _args(fn): - """Get argument names for function-like object. - - Args: - fn: Function, or function-like object (e.g., result of `functools.partial`). - - Returns: - `tuple` of string argument names. - """ - if hasattr(fn, 'func') and hasattr(fn, 'keywords'): - # Handle functools.partial and similar objects. - return tuple( - [arg for arg in _args(fn.func) if arg not in set(fn.keywords.keys())]) - # Handle function. - return tuple(tf_inspect.getargspec(fn).args) - - -_CANONICAL_LABELS_ARG = 'labels' -_LABELS_ARGS = set((_CANONICAL_LABELS_ARG, 'label', 'targets', 'target')) -_CANONICAL_PREDICTIONS_ARG = 'predictions' -_PREDICTIONS_ARGS = set((_CANONICAL_PREDICTIONS_ARG, 'prediction', - 'logits', 'logit')) -_CANONICAL_WEIGHTS_ARG = 'weights' -_WEIGHTS_ARGS = set((_CANONICAL_WEIGHTS_ARG, 'weight')) - - -def _matching_arg( - fn_name, fn_args, candidate_args, canonical_arg, is_required=False): - """Find single argument in `args` from `candidate_args`. - - Args: - fn_name: Function name, only used for error string. - fn_args: String argument names to `fn_name` function. - candidate_args: Candidate argument names to find in `args`. - canonical_arg: Canonical argument name in `candidate_args`. This is only - used to log a warning if a non-canonical match is found. - is_required: Whether function is required to have an arg in - `candidate_args`. - - Returns: - String argument name if found, or `None` if not found. - - Raises: - ValueError: if 2 candidates are found, or 0 are found and `is_required` is - set. - """ - assert canonical_arg in candidate_args # Sanity check. - matching_args = candidate_args.intersection(fn_args) - if len(matching_args) > 1: - raise ValueError( - 'Ambiguous arguments %s, must provide only one of %s.' % ( - matching_args, candidate_args)) - matching_arg = matching_args.pop() if matching_args else None - if matching_arg: - if matching_arg != canonical_arg: - logging.warning( - 'Canonical arg %s missing from %s(%s), using %s.', - canonical_arg, fn_name, fn_args, matching_arg) - elif is_required: - raise ValueError( - '%s missing from %s(%s).' % (candidate_args, fn_name, fn_args)) - return matching_arg - - -def _fn_name(fn): - if hasattr(fn, '__name__'): - return fn.__name__ - if hasattr(fn, 'func') and hasattr(fn.func, '__name__'): - return fn.func.__name__ # If it's a functools.partial. - return str(fn) - - -def _adapt_metric_fn( - metric_fn, metric_fn_name, is_labels_required, is_weights_required): - """Adapt `metric_fn` to take only named args. - - This returns a function that takes only named args `labels`, `predictions`, - and `weights`, and invokes `metric_fn` according to the following rules: - - If `metric_fn` args include exactly one of `_LABELS_ARGS`, that arg is - passed (usually by name, but positionally if both it and `predictions` need - to be passed positionally). Otherwise, `labels` are omitted. - - If `metric_fn` args include exactly one of `_PREDICTIONS_ARGS`, that arg is - passed by name. Otherwise, `predictions` are passed positionally as the - first non-label argument. - - If exactly one of `_WEIGHTS_ARGS` is provided, that arg is passed by - name. - - Args: - metric_fn: Metric function to be wrapped. - metric_fn_name: `metric_fn` name, only used for logging. - is_labels_required: Whether `labels` is a required arg. - is_weights_required: Whether `weights` is a required arg. - - Returns: - Function accepting only named args `labels, `predictions`, and `weights`, - and passing those to `metric_fn`. - - Raises: - ValueError: if one of the following is true: - - `metric_fn` has more than one arg of `_LABELS_ARGS`, `_PREDICTIONS_ARGS`, - or `_WEIGHTS_ARGS` - - `is_labels_required` is true, and `metric_fn` has no arg from - `_LABELS_ARGS`. - - `is_weights_required` is true, and `metric_fn` has no arg from - `_WEIGHTS_ARGS`. - """ - args = _args(metric_fn) - - labels_arg = _matching_arg( - metric_fn_name, args, _LABELS_ARGS, _CANONICAL_LABELS_ARG, - is_labels_required) - predictions_arg = _matching_arg( - metric_fn_name, args, _PREDICTIONS_ARGS, _CANONICAL_PREDICTIONS_ARG) - weights_arg = _matching_arg( - metric_fn_name, args, _WEIGHTS_ARGS, _CANONICAL_WEIGHTS_ARG, - is_weights_required) - - # pylint: disable=invalid-name - if labels_arg: - if predictions_arg: - # Both labels and predictions are named args. - def _named_metric_fn( - _sentinel=None, labels=None, predictions=None, weights=None): - _assert_named_args(_sentinel) - kwargs = { - labels_arg: labels, - predictions_arg: predictions, - } - if weights is not None: - kwargs[weights_arg] = weights - return metric_fn(**kwargs) - return _named_metric_fn - - if labels_arg == args[0]: - # labels is a named arg, and first. predictions is not a named arg, so we - # want to pass it as the 2nd positional arg (i.e., the first non-labels - # position), which means passing both positionally. - def _positional_metric_fn( - _sentinel=None, labels=None, predictions=None, weights=None): - _assert_named_args(_sentinel) - # TODO(ptucker): Should we support metrics that take only labels? - # Currently, if you want streaming mean of a label, you have to wrap it - # in a fn that takes discards predictions. - if weights is None: - return metric_fn(labels, predictions) - return metric_fn(labels, predictions, **{weights_arg: weights}) - return _positional_metric_fn - - # labels is a named arg, and not first, so we pass predictions positionally - # and labels by name. - def _positional_predictions_metric_fn( - _sentinel=None, labels=None, predictions=None, weights=None): - _assert_named_args(_sentinel) - kwargs = { - labels_arg: labels, - } - if weights is not None: - kwargs[weights_arg] = weights - return metric_fn(predictions, **kwargs) - return _positional_predictions_metric_fn - - if predictions_arg: - # No labels, and predictions is named, so we pass the latter as a named arg. - def _named_no_labels_metric_fn( - _sentinel=None, labels=None, predictions=None, weights=None): - del labels - _assert_named_args(_sentinel) - kwargs = { - predictions_arg: predictions, - } - # TODO(ptucker): Should we allow weights with no labels? - if weights is not None: - kwargs[weights_arg] = weights - return metric_fn(**kwargs) - return _named_no_labels_metric_fn - - # Neither labels nor predictions are named, so we just pass predictions as the - # first arg. - def _positional_no_labels_metric_fn( - _sentinel=None, labels=None, predictions=None, weights=None): - del labels - _assert_named_args(_sentinel) - if weights is None: - return metric_fn(predictions) - # TODO(ptucker): Should we allow weights with no labels? - return metric_fn(predictions, **{weights_arg: weights}) - return _positional_no_labels_metric_fn - - -class MetricSpec(object): - """MetricSpec connects a model to metric functions. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - The MetricSpec class contains all information necessary to connect the - output of a `model_fn` to the metrics (usually, streaming metrics) that are - used in evaluation. - - It is passed in the `metrics` argument of `Estimator.evaluate`. The - `Estimator` then knows which predictions, labels, and weight to use to call a - given metric function. - - When building the ops to run in evaluation, an `Estimator` will call - `create_metric_ops`, which will connect the given `metric_fn` to the model - as detailed in the docstring for `create_metric_ops`, and return the metric. - - Example: - - Assuming a model has an input function which returns inputs containing - (among other things) a tensor with key "input_key", and a labels dictionary - containing "label_key". Let's assume that the `model_fn` for this model - returns a prediction with key "prediction_key". - - In order to compute the accuracy of the "prediction_key" prediction, we - would add - - ``` - "prediction accuracy": MetricSpec(metric_fn=prediction_accuracy_fn, - prediction_key="prediction_key", - label_key="label_key") - ``` - - to the metrics argument to `evaluate`. `prediction_accuracy_fn` can be either - a predefined function in metric_ops (e.g., `streaming_accuracy`) or a custom - function you define. - - If we would like the accuracy to be weighted by "input_key", we can add that - as the `weight_key` argument. - - ``` - "prediction accuracy": MetricSpec(metric_fn=prediction_accuracy_fn, - prediction_key="prediction_key", - label_key="label_key", - weight_key="input_key") - ``` - - An end-to-end example is as follows: - - ``` - estimator = tf.contrib.learn.Estimator(...) - estimator.fit(...) - _ = estimator.evaluate( - input_fn=input_fn, - steps=1, - metrics={ - 'prediction accuracy': - metric_spec.MetricSpec( - metric_fn=prediction_accuracy_fn, - prediction_key="prediction_key", - label_key="label_key") - }) - ``` - - """ - - @deprecated(None, 'Use tf.estimator.EstimatorSpec.eval_metric_ops.') - def __init__(self, - metric_fn, - prediction_key=None, - label_key=None, - weight_key=None): - """Constructor. - - Creates a MetricSpec. - - Args: - metric_fn: A function to use as a metric. See `_adapt_metric_fn` for - rules on how `predictions`, `labels`, and `weights` are passed to this - function. This must return either a single `Tensor`, which is - interpreted as a value of this metric, or a pair - `(value_op, update_op)`, where `value_op` is the op to call to - obtain the value of the metric, and `update_op` should be run for - each batch to update internal state. - prediction_key: The key for a tensor in the `predictions` dict (output - from the `model_fn`) to use as the `predictions` input to the - `metric_fn`. Optional. If `None`, the `model_fn` must return a single - tensor or a dict with only a single entry as `predictions`. - label_key: The key for a tensor in the `labels` dict (output from the - `input_fn`) to use as the `labels` input to the `metric_fn`. - Optional. If `None`, the `input_fn` must return a single tensor or a - dict with only a single entry as `labels`. - weight_key: The key for a tensor in the `inputs` dict (output from the - `input_fn`) to use as the `weights` input to the `metric_fn`. - Optional. If `None`, no weights will be passed to the `metric_fn`. - """ - self._metric_fn_name = _fn_name(metric_fn) - self._metric_fn = _adapt_metric_fn( - metric_fn=metric_fn, - metric_fn_name=self._metric_fn_name, - is_labels_required=label_key is not None, - is_weights_required=weight_key is not None) - self._prediction_key = prediction_key - self._label_key = label_key - self._weight_key = weight_key - - @property - def prediction_key(self): - return self._prediction_key - - @property - def label_key(self): - return self._label_key - - @property - def weight_key(self): - return self._weight_key - - @property - def metric_fn(self): - """Metric function. - - This function accepts named args: `predictions`, `labels`, `weights`. It - returns a single `Tensor` or `(value_op, update_op)` pair. See `metric_fn` - constructor argument for more details. - - Returns: - Function, see `metric_fn` constructor argument for more details. - """ - return self._metric_fn - - def __str__(self): - return ('MetricSpec(metric_fn=%s, ' % self._metric_fn_name + - 'prediction_key=%s, ' % self.prediction_key + - 'label_key=%s, ' % self.label_key + - 'weight_key=%s)' % self.weight_key - ) - - def create_metric_ops(self, inputs, labels, predictions): - """Connect our `metric_fn` to the specified members of the given dicts. - - This function will call the `metric_fn` given in our constructor as follows: - - ``` - metric_fn(predictions[self.prediction_key], - labels[self.label_key], - weights=weights[self.weight_key]) - ``` - - And returns the result. The `weights` argument is only passed if - `self.weight_key` is not `None`. - - `predictions` and `labels` may be single tensors as well as dicts. If - `predictions` is a single tensor, `self.prediction_key` must be `None`. If - `predictions` is a single element dict, `self.prediction_key` is allowed to - be `None`. Conversely, if `labels` is a single tensor, `self.label_key` must - be `None`. If `labels` is a single element dict, `self.label_key` is allowed - to be `None`. - - Args: - inputs: A dict of inputs produced by the `input_fn` - labels: A dict of labels or a single label tensor produced by the - `input_fn`. - predictions: A dict of predictions or a single tensor produced by the - `model_fn`. - - Returns: - The result of calling `metric_fn`. - - Raises: - ValueError: If `predictions` or `labels` is a single `Tensor` and - `self.prediction_key` or `self.label_key` is not `None`; or if - `self.label_key` is `None` but `labels` is a dict with more than one - element, or if `self.prediction_key` is `None` but `predictions` is a - dict with more than one element. - """ - def _get_dict(name, dict_or_tensor, key): - """Get a single tensor or an element of a dict or raise ValueError.""" - if key: - if not isinstance(dict_or_tensor, dict): - raise ValueError('MetricSpec with ' + name + '_key specified' - ' requires ' + - name + 's dict, got %s.\n' % dict_or_tensor + - 'You must not provide a %s_key if you ' % name + - 'only have a single Tensor as %ss.' % name) - if key not in dict_or_tensor: - raise KeyError( - 'Key \'%s\' missing from %s.' % (key, dict_or_tensor.keys())) - return dict_or_tensor[key] - else: - if isinstance(dict_or_tensor, dict): - if len(dict_or_tensor) != 1: - raise ValueError('MetricSpec without specified ' + name + '_key' - ' requires ' + name + 's tensor or single element' - ' dict, got %s' % dict_or_tensor) - return six.next(six.itervalues(dict_or_tensor)) - return dict_or_tensor - - # Get the predictions. - prediction = _get_dict('prediction', predictions, self.prediction_key) - - # Get the labels. - label = _get_dict('label', labels, self.label_key) - - try: - return self.metric_fn( - labels=label, - predictions=prediction, - weights=inputs[self.weight_key] if self.weight_key else None) - except Exception as ex: - logging.error('Could not create metric ops for %s, %s.' % (self, ex)) - raise diff --git a/tensorflow/contrib/learn/python/learn/metric_spec_test.py b/tensorflow/contrib/learn/python/learn/metric_spec_test.py deleted file mode 100644 index 8d578174ad3..00000000000 --- a/tensorflow/contrib/learn/python/learn/metric_spec_test.py +++ /dev/null @@ -1,433 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for MetricSpec.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -# pylint: disable=g-bad-todo,g-import-not-at-top -from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec -from tensorflow.python.platform import test - - -class MetricSpecTest(test.TestCase): - - def test_named_args_with_weights(self): - features = {"f1": "f1_value", "f2": "f2_value"} - labels_ = {"l1": "l1_value", "l2": "l2_value"} - predictions_ = {"p1": "p1_value", "p2": "p2_value"} - - def _fn0(predictions, labels, weights=None): - self.assertEqual("p1_value", predictions) - self.assertEqual("l1_value", labels) - self.assertEqual("f2_value", weights) - return "metric_fn_result" - - def _fn1(predictions, targets, weights=None): - self.assertEqual("p1_value", predictions) - self.assertEqual("l1_value", targets) - self.assertEqual("f2_value", weights) - return "metric_fn_result" - - def _fn2(prediction, label, weight=None): - self.assertEqual("p1_value", prediction) - self.assertEqual("l1_value", label) - self.assertEqual("f2_value", weight) - return "metric_fn_result" - - def _fn3(prediction, target, weight=None): - self.assertEqual("p1_value", prediction) - self.assertEqual("l1_value", target) - self.assertEqual("f2_value", weight) - return "metric_fn_result" - - for fn in (_fn0, _fn1, _fn2, _fn3): - spec = MetricSpec( - metric_fn=fn, prediction_key="p1", label_key="l1", weight_key="f2") - self.assertEqual( - "metric_fn_result", - spec.create_metric_ops(features, labels_, predictions_)) - - def test_no_args(self): - def _fn(): - self.fail("Expected failure before metric_fn.") - - spec = MetricSpec(metric_fn=_fn) - with self.assertRaises(TypeError): - spec.create_metric_ops( - {"f1": "f1_value"}, "labels_value", "predictions_value") - - def test_kwargs(self): - features = {"f1": "f1_value"} - labels_ = "labels_value" - predictions_ = "predictions_value" - - def _fn(**kwargs): - self.assertEqual({}, kwargs) - return "metric_fn_result" - - spec = MetricSpec(metric_fn=_fn) - with self.assertRaises(TypeError): - spec.create_metric_ops(features, labels_, predictions_) - - def test_named_labels_no_predictions(self): - features = {"f1": "f1_value"} - labels_ = "labels_value" - predictions_ = "predictions_value" - - def _fn(labels): - self.assertEqual(labels_, labels) - return "metric_fn_result" - - spec = MetricSpec(metric_fn=_fn) - with self.assertRaises(TypeError): - spec.create_metric_ops(features, labels_, predictions_) - - def test_named_labels_no_predictions_with_kwargs(self): - features = {"f1": "f1_value"} - labels_ = "labels_value" - predictions_ = "predictions_value" - - def _fn(labels, **kwargs): - self.assertEqual(labels_, labels) - self.assertEqual({}, kwargs) - return "metric_fn_result" - - spec = MetricSpec(metric_fn=_fn) - with self.assertRaises(TypeError): - spec.create_metric_ops(features, labels_, predictions_) - - def test_no_named_predictions_named_labels_first_arg(self): - features = {"f1": "f1_value"} - labels_ = "labels_value" - predictions_ = "predictions_value" - - def _fn(labels, predictions_by_another_name): - self.assertEqual(predictions_, predictions_by_another_name) - self.assertEqual(labels_, labels) - return "metric_fn_result" - - spec = MetricSpec(metric_fn=_fn) - self.assertEqual( - "metric_fn_result", - spec.create_metric_ops(features, labels_, predictions_)) - - def test_no_named_predictions_named_labels_second_arg(self): - features = {"f1": "f1_value"} - labels_ = "labels_value" - predictions_ = "predictions_value" - - def _fn(predictions_by_another_name, labels): - self.assertEqual(predictions_, predictions_by_another_name) - self.assertEqual(labels_, labels) - return "metric_fn_result" - - spec = MetricSpec(metric_fn=_fn) - self.assertEqual( - "metric_fn_result", - spec.create_metric_ops(features, labels_, predictions_)) - - def test_no_named_labels(self): - features = {"f1": "f1_value"} - labels_ = "labels_value" - predictions_ = "predictions_value" - - def _fn(predictions): - self.assertEqual(predictions_, predictions) - return "metric_fn_result" - - spec = MetricSpec(metric_fn=_fn) - self.assertEqual( - "metric_fn_result", - spec.create_metric_ops(features, labels_, predictions_)) - - def test_no_named_labels_or_predictions_1arg(self): - features = {"f1": "f1_value"} - labels_ = "labels_value" - predictions_ = "predictions_value" - - def _fn(a): - self.assertEqual(predictions_, a) - return "metric_fn_result" - - spec = MetricSpec(metric_fn=_fn) - self.assertEqual( - "metric_fn_result", - spec.create_metric_ops(features, labels_, predictions_)) - - def test_no_named_labels_or_predictions_2args(self): - features = {"f1": "f1_value"} - labels_ = "labels_value" - predictions_ = "predictions_value" - - def _fn(a, b): - del a, b - self.fail("Expected failure before metric_fn.") - - spec = MetricSpec(metric_fn=_fn) - with self.assertRaises(TypeError): - spec.create_metric_ops(features, labels_, predictions_) - - def test_named_args_no_weights(self): - features = {"f1": "f1_value", "f2": "f2_value"} - labels_ = {"l1": "l1_value", "l2": "l2_value"} - predictions_ = {"p1": "p1_value", "p2": "p2_value"} - - def _fn0(predictions, labels): - self.assertEqual("p1_value", predictions) - self.assertEqual("l1_value", labels) - return "metric_fn_result" - - def _fn1(predictions, targets): - self.assertEqual("p1_value", predictions) - self.assertEqual("l1_value", targets) - return "metric_fn_result" - - def _fn2(prediction, label): - self.assertEqual("p1_value", prediction) - self.assertEqual("l1_value", label) - return "metric_fn_result" - - def _fn3(prediction, target): - self.assertEqual("p1_value", prediction) - self.assertEqual("l1_value", target) - return "metric_fn_result" - - for fn in (_fn0, _fn1, _fn2, _fn3): - spec = MetricSpec(metric_fn=fn, prediction_key="p1", label_key="l1") - self.assertEqual( - "metric_fn_result", - spec.create_metric_ops(features, labels_, predictions_)) - - def test_predictions_dict_no_key(self): - features = {"f1": "f1_value", "f2": "f2_value"} - labels = {"l1": "l1_value", "l2": "l2_value"} - predictions = {"p1": "p1_value", "p2": "p2_value"} - - def _fn(predictions, labels, weights=None): - del labels, predictions, weights - self.fail("Expected failure before metric_fn.") - - spec = MetricSpec(metric_fn=_fn, label_key="l1", weight_key="f2") - with self.assertRaisesRegexp( - ValueError, - "MetricSpec without specified prediction_key requires predictions" - " tensor or single element dict"): - spec.create_metric_ops(features, labels, predictions) - - def test_labels_dict_no_key(self): - features = {"f1": "f1_value", "f2": "f2_value"} - labels = {"l1": "l1_value", "l2": "l2_value"} - predictions = {"p1": "p1_value", "p2": "p2_value"} - - def _fn(labels, predictions, weights=None): - del labels, predictions, weights - self.fail("Expected failure before metric_fn.") - - spec = MetricSpec(metric_fn=_fn, prediction_key="p1", weight_key="f2") - with self.assertRaisesRegexp( - ValueError, - "MetricSpec without specified label_key requires labels tensor or" - " single element dict"): - spec.create_metric_ops(features, labels, predictions) - - def test_single_prediction(self): - features = {"f1": "f1_value", "f2": "f2_value"} - labels_ = {"l1": "l1_value", "l2": "l2_value"} - predictions_ = "p1_value" - - def _fn(predictions, labels, weights=None): - self.assertEqual(predictions_, predictions) - self.assertEqual("l1_value", labels) - self.assertEqual("f2_value", weights) - return "metric_fn_result" - - spec = MetricSpec(metric_fn=_fn, label_key="l1", weight_key="f2") - self.assertEqual( - "metric_fn_result", - spec.create_metric_ops(features, labels_, predictions_)) - - def test_single_label(self): - features = {"f1": "f1_value", "f2": "f2_value"} - labels_ = "l1_value" - predictions_ = {"p1": "p1_value", "p2": "p2_value"} - - def _fn(predictions, labels, weights=None): - self.assertEqual("p1_value", predictions) - self.assertEqual(labels_, labels) - self.assertEqual("f2_value", weights) - return "metric_fn_result" - - spec = MetricSpec(metric_fn=_fn, prediction_key="p1", weight_key="f2") - self.assertEqual( - "metric_fn_result", - spec.create_metric_ops(features, labels_, predictions_)) - - def test_single_predictions_with_key(self): - features = {"f1": "f1_value", "f2": "f2_value"} - labels = {"l1": "l1_value", "l2": "l2_value"} - predictions = "p1_value" - - def _fn(predictions, labels, weights=None): - del labels, predictions, weights - self.fail("Expected failure before metric_fn.") - - spec = MetricSpec( - metric_fn=_fn, prediction_key="p1", label_key="l1", weight_key="f2") - with self.assertRaisesRegexp( - ValueError, - "MetricSpec with prediction_key specified requires predictions dict"): - spec.create_metric_ops(features, labels, predictions) - - def test_single_labels_with_key(self): - features = {"f1": "f1_value", "f2": "f2_value"} - labels = "l1_value" - predictions = {"p1": "p1_value", "p2": "p2_value"} - - def _fn(predictions, labels, weights=None): - del labels, predictions, weights - self.fail("Expected failure before metric_fn.") - - spec = MetricSpec( - metric_fn=_fn, prediction_key="p1", label_key="l1", weight_key="f2") - with self.assertRaisesRegexp( - ValueError, "MetricSpec with label_key specified requires labels dict"): - spec.create_metric_ops(features, labels, predictions) - - def test_str(self): - def _metric_fn(labels, predictions, weights=None): - return predictions, labels, weights - - string = str(MetricSpec( - metric_fn=_metric_fn, - label_key="my_label", - prediction_key="my_prediction", - weight_key="my_weight")) - self.assertIn("_metric_fn", string) - self.assertIn("my_label", string) - self.assertIn("my_prediction", string) - self.assertIn("my_weight", string) - - def test_partial_str(self): - - def custom_metric(predictions, labels, stuff, weights=None): - return predictions, labels, weights, stuff - - string = str(MetricSpec( - metric_fn=functools.partial(custom_metric, stuff=5), - label_key="my_label", - prediction_key="my_prediction", - weight_key="my_weight")) - self.assertIn("custom_metric", string) - self.assertIn("my_label", string) - self.assertIn("my_prediction", string) - self.assertIn("my_weight", string) - - def test_partial(self): - features = {"f1": "f1_value", "f2": "f2_value"} - labels = {"l1": "l1_value"} - predictions = {"p1": "p1_value", "p2": "p2_value"} - - def custom_metric(predictions, labels, stuff, weights=None): - self.assertEqual("p1_value", predictions) - self.assertEqual("l1_value", labels) - self.assertEqual("f2_value", weights) - if stuff: - return "metric_fn_result" - raise ValueError("No stuff.") - - spec = MetricSpec( - metric_fn=functools.partial(custom_metric, stuff=5), - label_key="l1", - prediction_key="p1", - weight_key="f2") - self.assertEqual( - "metric_fn_result", - spec.create_metric_ops(features, labels, predictions)) - - spec = MetricSpec( - metric_fn=functools.partial(custom_metric, stuff=None), - prediction_key="p1", label_key="l1", weight_key="f2") - with self.assertRaisesRegexp(ValueError, "No stuff."): - spec.create_metric_ops(features, labels, predictions) - - def test_label_key_without_label_arg(self): - def _fn0(predictions, weights=None): - del predictions, weights - self.fail("Expected failure before metric_fn.") - - def _fn1(prediction, weight=None): - del prediction, weight - self.fail("Expected failure before metric_fn.") - - for fn in (_fn0, _fn1): - with self.assertRaisesRegexp(ValueError, "label.*missing"): - MetricSpec(metric_fn=fn, label_key="l1") - - def test_weight_key_without_weight_arg(self): - def _fn0(predictions, labels): - del predictions, labels - self.fail("Expected failure before metric_fn.") - - def _fn1(prediction, label): - del prediction, label - self.fail("Expected failure before metric_fn.") - - def _fn2(predictions, targets): - del predictions, targets - self.fail("Expected failure before metric_fn.") - - def _fn3(prediction, target): - del prediction, target - self.fail("Expected failure before metric_fn.") - - for fn in (_fn0, _fn1, _fn2, _fn3): - with self.assertRaisesRegexp(ValueError, "weight.*missing"): - MetricSpec(metric_fn=fn, weight_key="f2") - - def test_multiple_label_args(self): - def _fn0(predictions, labels, targets): - del predictions, labels, targets - self.fail("Expected failure before metric_fn.") - - def _fn1(prediction, label, target): - del prediction, label, target - self.fail("Expected failure before metric_fn.") - - for fn in (_fn0, _fn1): - with self.assertRaisesRegexp(ValueError, "provide only one of.*label"): - MetricSpec(metric_fn=fn) - - def test_multiple_prediction_args(self): - def _fn(predictions, prediction, labels): - del predictions, prediction, labels - self.fail("Expected failure before metric_fn.") - - with self.assertRaisesRegexp(ValueError, "provide only one of.*prediction"): - MetricSpec(metric_fn=_fn) - - def test_multiple_weight_args(self): - def _fn(predictions, labels, weights=None, weight=None): - del predictions, labels, weights, weight - self.fail("Expected failure before metric_fn.") - - with self.assertRaisesRegexp(ValueError, "provide only one of.*weight"): - MetricSpec(metric_fn=_fn) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/models.py b/tensorflow/contrib/learn/python/learn/models.py deleted file mode 100644 index bd4bbf9f8c9..00000000000 --- a/tensorflow/contrib/learn/python/learn/models.py +++ /dev/null @@ -1,407 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Various high level TF models (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -from tensorflow.contrib import rnn as contrib_rnn -from tensorflow.contrib.learn.python.learn.ops import losses_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops as array_ops_ -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.summary import summary -from tensorflow.python.util.deprecation import deprecated - - -@deprecated(None, 'Consider using a tf.estimator.LinearRegressor') -def linear_regression_zero_init(x, y): - """Linear regression subgraph with zero-value initial weights and bias. - - Args: - x: tensor or placeholder for input features. - y: tensor or placeholder for labels. - - Returns: - Predictions and loss tensors. - """ - return linear_regression(x, y, init_mean=0.0, init_stddev=0.0) - - -@deprecated(None, 'Consider using a class from tf.estimator.LinearClassifier') -def logistic_regression_zero_init(x, y): - """Logistic regression subgraph with zero-value initial weights and bias. - - Args: - x: tensor or placeholder for input features. - y: tensor or placeholder for labels. - - Returns: - Predictions and loss tensors. - """ - return logistic_regression(x, y, init_mean=0.0, init_stddev=0.0) - - -@deprecated(None, 'Consider using a class from tf.estimator.') -def linear_regression(x, y, init_mean=None, init_stddev=1.0): - """Creates linear regression TensorFlow subgraph. - - Args: - x: tensor or placeholder for input features. - y: tensor or placeholder for labels. - init_mean: the mean value to use for initialization. - init_stddev: the standard deviation to use for initialization. - - Returns: - Predictions and loss tensors. - - Side effects: - The variables linear_regression.weights and linear_regression.bias are - initialized as follows. If init_mean is not None, then initialization - will be done using a random normal initializer with the given init_mean - and init_stddv. (These may be set to 0.0 each if a zero initialization - is desirable for convex use cases.) If init_mean is None, then the - uniform_unit_scaling_initialzer will be used. - """ - with vs.variable_scope('linear_regression'): - scope_name = vs.get_variable_scope().name - summary.histogram('%s.x' % scope_name, x) - summary.histogram('%s.y' % scope_name, y) - dtype = x.dtype.base_dtype - y_shape = y.get_shape() - if len(y_shape) == 1: - output_shape = 1 - else: - output_shape = y_shape[1] - # Set up the requested initialization. - if init_mean is None: - weights = vs.get_variable( - 'weights', [x.get_shape()[1], output_shape], dtype=dtype) - bias = vs.get_variable('bias', [output_shape], dtype=dtype) - else: - weights = vs.get_variable( - 'weights', [x.get_shape()[1], output_shape], - initializer=init_ops.random_normal_initializer( - init_mean, init_stddev, dtype=dtype), - dtype=dtype) - bias = vs.get_variable( - 'bias', [output_shape], - initializer=init_ops.random_normal_initializer( - init_mean, init_stddev, dtype=dtype), - dtype=dtype) - summary.histogram('%s.weights' % scope_name, weights) - summary.histogram('%s.bias' % scope_name, bias) - return losses_ops.mean_squared_error_regressor(x, y, weights, bias) - - -@deprecated(None, 'Consider using a class from tf.estimator.') -def logistic_regression(x, - y, - class_weight=None, - init_mean=None, - init_stddev=1.0): - """Creates logistic regression TensorFlow subgraph. - - Args: - x: tensor or placeholder for input features, - shape should be [batch_size, n_features]. - y: tensor or placeholder for labels (one-hot), - shape should be [batch_size, n_classes]. - class_weight: tensor, [n_classes], where for each class - it has weight of the class. If not provided - will check if graph contains tensor `class_weight:0`. - If that is not provided either all ones are used. - init_mean: the mean value to use for initialization. - init_stddev: the standard deviation to use for initialization. - - Returns: - Predictions and loss tensors. - - Side effects: - The variables linear_regression.weights and linear_regression.bias are - initialized as follows. If init_mean is not None, then initialization - will be done using a random normal initializer with the given init_mean - and init_stddv. (These may be set to 0.0 each if a zero initialization - is desirable for convex use cases.) If init_mean is None, then the - uniform_unit_scaling_initialzer will be used. - """ - with vs.variable_scope('logistic_regression'): - scope_name = vs.get_variable_scope().name - summary.histogram('%s.x' % scope_name, x) - summary.histogram('%s.y' % scope_name, y) - dtype = x.dtype.base_dtype - # Set up the requested initialization. - if init_mean is None: - weights = vs.get_variable( - 'weights', [x.get_shape()[1], y.get_shape()[-1]], dtype=dtype) - bias = vs.get_variable('bias', [y.get_shape()[-1]], dtype=dtype) - else: - weights = vs.get_variable( - 'weights', [x.get_shape()[1], y.get_shape()[-1]], - initializer=init_ops.random_normal_initializer( - init_mean, init_stddev, dtype=dtype), - dtype=dtype) - bias = vs.get_variable( - 'bias', [y.get_shape()[-1]], - initializer=init_ops.random_normal_initializer( - init_mean, init_stddev, dtype=dtype), - dtype=dtype) - summary.histogram('%s.weights' % scope_name, weights) - summary.histogram('%s.bias' % scope_name, bias) - # If no class weight provided, try to retrieve one from pre-defined - # tensor name in the graph. - if not class_weight: - try: - class_weight = ops.get_default_graph().get_tensor_by_name( - 'class_weight:0') - except KeyError: - pass - - return losses_ops.softmax_classifier( - x, y, weights, bias, class_weight=class_weight) - - -## This will be in TensorFlow 0.7. -## TODO(ilblackdragon): Clean this up when it's released -def _reverse_seq(input_seq, lengths): - """Reverse a list of Tensors up to specified lengths. - - Args: - input_seq: Sequence of seq_len tensors of dimension (batch_size, depth) - lengths: A tensor of dimension batch_size, containing lengths for each - sequence in the batch. If "None" is specified, simply - reverses the list. - - Returns: - time-reversed sequence - """ - if lengths is None: - return list(reversed(input_seq)) - - for input_ in input_seq: - input_.set_shape(input_.get_shape().with_rank(2)) - - # Join into (time, batch_size, depth) - s_joined = array_ops_.pack(input_seq) - - # Reverse along dimension 0 - s_reversed = array_ops_.reverse_sequence(s_joined, lengths, 0, 1) - # Split again into list - result = array_ops_.unpack(s_reversed) - return result - - -@deprecated(None, 'Please consider `tf.nn.bidirectional_dynamic_rnn`.') -def bidirectional_rnn(cell_fw, - cell_bw, - inputs, - initial_state_fw=None, - initial_state_bw=None, - dtype=None, - sequence_length=None, - scope=None): - """Creates a bidirectional recurrent neural network. - - Similar to the unidirectional case (rnn) but takes input and builds - independent forward and backward RNNs with the final forward and backward - outputs depth-concatenated, such that the output will have the format - [time][batch][cell_fw.output_size + cell_bw.output_size]. The input_size of - forward and backward cell must match. The initial state for both directions - is zero by default (but can be set optionally) and no intermediate states - are ever returned -- the network is fully unrolled for the given (passed in) - length(s) of the sequence(s) or completely unrolled if length(s) is not - given. - Args: - cell_fw: An instance of RNNCell, to be used for forward direction. - cell_bw: An instance of RNNCell, to be used for backward direction. - inputs: A length T list of inputs, each a tensor of shape - [batch_size, cell.input_size]. - initial_state_fw: (optional) An initial state for the forward RNN. - This must be a tensor of appropriate type and shape - [batch_size x cell.state_size]. - initial_state_bw: (optional) Same as for initial_state_fw. - dtype: (optional) The data type for the initial state. Required if - either of the initial states are not provided. - sequence_length: (optional) An int64 vector (tensor) of size - [batch_size], - containing the actual lengths for each of the sequences. - scope: VariableScope for the created subgraph; defaults to "BiRNN" - - Returns: - A pair (outputs, state) where: - outputs is a length T list of outputs (one for each input), which - are depth-concatenated forward and backward outputs - state is the concatenated final state of the forward and backward RNN - - Raises: - TypeError: If "cell_fw" or "cell_bw" is not an instance of RNNCell. - ValueError: If inputs is None or an empty list. - """ - - if not isinstance(cell_fw, contrib_rnn.RNNCell): - raise TypeError('cell_fw must be an instance of RNNCell') - if not isinstance(cell_bw, contrib_rnn.RNNCell): - raise TypeError('cell_bw must be an instance of RNNCell') - if not isinstance(inputs, list): - raise TypeError('inputs must be a list') - if not inputs: - raise ValueError('inputs must not be empty') - - name = scope or 'BiRNN' - # Forward direction - with vs.variable_scope(name + '_FW'): - output_fw, state_fw = contrib_rnn.static_rnn(cell_fw, inputs, - initial_state_fw, dtype, - sequence_length) - - # Backward direction - with vs.variable_scope(name + '_BW'): - tmp, state_bw = contrib_rnn.static_rnn( - cell_bw, - _reverse_seq(inputs, sequence_length), initial_state_bw, dtype, - sequence_length) - output_bw = _reverse_seq(tmp, sequence_length) - # Concat each of the forward/backward outputs - outputs = [ - array_ops_.concat([fw, bw], 1) for fw, bw in zip(output_fw, output_bw) - ] - - return outputs, array_ops_.concat([state_fw, state_bw], 1) - - -# End of TensorFlow 0.7 - - -@deprecated(None, 'Please consider tensorflow/tensor2tensor.') -def get_rnn_model(rnn_size, cell_type, num_layers, input_op_fn, bidirectional, - target_predictor_fn, sequence_length, initial_state, - attn_length, attn_size, attn_vec_size): - """Returns a function that creates a RNN TensorFlow subgraph. - - Args: - rnn_size: The size for rnn cell, e.g. size of your word embeddings. - cell_type: The type of rnn cell, including rnn, gru, and lstm. - num_layers: The number of layers of the rnn model. - input_op_fn: Function that will transform the input tensor, such as - creating word embeddings, byte list, etc. This takes - an argument `x` for input and returns transformed `x`. - bidirectional: boolean, Whether this is a bidirectional rnn. - target_predictor_fn: Function that will predict target from input - features. This can be logistic regression, - linear regression or any other model, - that takes `x`, `y` and returns predictions and loss - tensors. - sequence_length: If sequence_length is provided, dynamic calculation is - performed. This saves computational time when unrolling past max sequence - length. Required for bidirectional RNNs. - initial_state: An initial state for the RNN. This must be a tensor of - appropriate type and shape [batch_size x cell.state_size]. - attn_length: integer, the size of attention vector attached to rnn cells. - attn_size: integer, the size of an attention window attached to rnn cells. - attn_vec_size: integer, the number of convolutional features calculated on - attention state and the size of the hidden layer built from base cell - state. - - Returns: - A function that creates the subgraph. - """ - - def rnn_estimator(x, y): - """RNN estimator with target predictor function on top.""" - x = input_op_fn(x) - if cell_type == 'rnn': - cell_fn = contrib_rnn.BasicRNNCell - elif cell_type == 'gru': - cell_fn = contrib_rnn.GRUCell - elif cell_type == 'lstm': - cell_fn = functools.partial( - contrib_rnn.BasicLSTMCell, state_is_tuple=False) - else: - raise ValueError('cell_type {} is not supported. '.format(cell_type)) - # TODO(ipolosukhin): state_is_tuple=False is deprecated - if bidirectional: - # forward direction cell - fw_cell = lambda: cell_fn(rnn_size) - bw_cell = lambda: cell_fn(rnn_size) - # attach attention cells if specified - if attn_length is not None: - def attn_fw_cell(): - return contrib_rnn.AttentionCellWrapper( - fw_cell(), - attn_length=attn_length, - attn_size=attn_size, - attn_vec_size=attn_vec_size, - state_is_tuple=False) - - def attn_bw_cell(): - return contrib_rnn.AttentionCellWrapper( - bw_cell(), - attn_length=attn_length, - attn_size=attn_size, - attn_vec_size=attn_vec_size, - state_is_tuple=False) - else: - attn_fw_cell = fw_cell - attn_bw_cell = bw_cell - - rnn_fw_cell = contrib_rnn.MultiRNNCell( - [attn_fw_cell() for _ in range(num_layers)], state_is_tuple=False) - # backward direction cell - rnn_bw_cell = contrib_rnn.MultiRNNCell( - [attn_bw_cell() for _ in range(num_layers)], state_is_tuple=False) - # pylint: disable=unexpected-keyword-arg, no-value-for-parameter - _, encoding = bidirectional_rnn( - rnn_fw_cell, - rnn_bw_cell, - x, - dtype=dtypes.float32, - sequence_length=sequence_length, - initial_state_fw=initial_state, - initial_state_bw=initial_state) - else: - rnn_cell = lambda: cell_fn(rnn_size) - - if attn_length is not None: - def attn_rnn_cell(): - return contrib_rnn.AttentionCellWrapper( - rnn_cell(), - attn_length=attn_length, - attn_size=attn_size, - attn_vec_size=attn_vec_size, - state_is_tuple=False) - else: - attn_rnn_cell = rnn_cell - - cell = contrib_rnn.MultiRNNCell( - [attn_rnn_cell() for _ in range(num_layers)], state_is_tuple=False) - _, encoding = contrib_rnn.static_rnn( - cell, - x, - dtype=dtypes.float32, - sequence_length=sequence_length, - initial_state=initial_state) - return target_predictor_fn(encoding, y) - - return rnn_estimator diff --git a/tensorflow/contrib/learn/python/learn/monitored_session.py b/tensorflow/contrib/learn/python/learn/monitored_session.py deleted file mode 100644 index ac0433f1775..00000000000 --- a/tensorflow/contrib/learn/python/learn/monitored_session.py +++ /dev/null @@ -1,34 +0,0 @@ -# pylint: disable=g-bad-file-header -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A wrapper of Session API which runs hooks (deprecated). - -These are deprecated aliases for classes and functions in `tf.train`. Please use -those directly. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.training import monitored_session - -# pylint: disable=invalid-name -Scaffold = monitored_session.Scaffold -SessionCreator = monitored_session.SessionCreator -ChiefSessionCreator = monitored_session.ChiefSessionCreator -WorkerSessionCreator = monitored_session.WorkerSessionCreator -MonitoredSession = monitored_session.MonitoredSession -# pylint: disable=invalid-name diff --git a/tensorflow/contrib/learn/python/learn/monitors.py b/tensorflow/contrib/learn/python/learn/monitors.py deleted file mode 100644 index 22b85cb034b..00000000000 --- a/tensorflow/contrib/learn/python/learn/monitors.py +++ /dev/null @@ -1,1375 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Monitors instrument the training process (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. - -@@get_default_monitors -@@BaseMonitor -@@CaptureVariable -@@CheckpointSaver -@@EveryN -@@ExportMonitor -@@GraphDump -@@LoggingTrainable -@@NanLoss -@@PrintTensor -@@StepCounter -@@StopAtStep -@@SummarySaver -@@ValidationMonitor -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import copy -import os -import time - -import numpy as np -import six - -from tensorflow.core.framework.summary_pb2 import Summary -from tensorflow.core.util.event_pb2 import SessionLog -from tensorflow.python.estimator import estimator as core_estimator -from tensorflow.python.framework import ops -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.summary import summary as core_summary -from tensorflow.python.training import checkpoint_management -from tensorflow.python.training import session_run_hook -from tensorflow.python.training import training_util -from tensorflow.python.util import deprecation -from tensorflow.python.util import tf_inspect - - -# TODO(ptucker): Split each monitor class into a separate file. -# TODO(ptucker): Fail if epoch or step does not monotonically increase? -class BaseMonitor(object): - """Base class for Monitors. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Defines basic interfaces of Monitors. - Monitors can either be run on all workers or, more commonly, restricted - to run exclusively on the elected chief worker. - """ - - @deprecation.deprecated( - "2016-12-05", - "Monitors are deprecated. Please use tf.train.SessionRunHook.") - def __init__(self): - self._begun = False - self._current_epoch = None - self._current_step = None - self._max_steps = None - self._estimator = None - - @property - def run_on_all_workers(self): - return False - - def set_estimator(self, estimator): - """A setter called automatically by the target estimator. - - If the estimator is locked, this method does nothing. - - Args: - estimator: the estimator that this monitor monitors. - - Raises: - ValueError: if the estimator is None. - """ - if estimator is None: - raise ValueError("Missing estimator.") - # TODO(mdan): This should fail if called twice with the same estimator. - self._estimator = estimator - - def begin(self, max_steps=None): - """Called at the beginning of training. - - When called, the default graph is the one we are executing. - - Args: - max_steps: `int`, the maximum global step this training will run until. - - Raises: - ValueError: if we've already begun a run. - """ - if self._begun: - raise ValueError("begin called twice without end.") - self._max_steps = max_steps - self._begun = True - - def end(self, session=None): - """Callback at the end of training/evaluation. - - Args: - session: A `tf.compat.v1.Session` object that can be used to run ops. - - Raises: - ValueError: if we've not begun a run. - """ - _ = session - if not self._begun: - raise ValueError("end called without begin.") - self._max_steps = None - self._begun = False - - def epoch_begin(self, epoch): - """Begin epoch. - - Args: - epoch: `int`, the epoch number. - - Raises: - ValueError: if we've already begun an epoch, or `epoch` < 0. - """ - if self._current_epoch is not None: - raise ValueError("epoch_begin called twice without epoch_end.") - if epoch < 0: - raise ValueError("Invalid epoch %s." % epoch) - self._current_epoch = epoch - - def epoch_end(self, epoch): - """End epoch. - - Args: - epoch: `int`, the epoch number. - - Raises: - ValueError: if we've not begun an epoch, or `epoch` number does not match. - """ - if self._current_epoch != epoch: - raise ValueError("epoch_end expected %s but got %s.", self._current_epoch, - epoch) - self._current_epoch = None - - def step_begin(self, step): - """Callback before training step begins. - - You may use this callback to request evaluation of additional tensors - in the graph. - - Args: - step: `int`, the current value of the global step. - - Returns: - List of `Tensor` objects or string tensor names to be run. - - Raises: - ValueError: if we've already begun a step, or `step` < 0, or - `step` > `max_steps`. - """ - if (step < 0) or ((self._max_steps is not None) and - (step > self._max_steps)): - raise ValueError("Invalid step %s." % step) - self._current_step = step - return [] - - def step_end(self, step, output): # pylint: disable=unused-argument - """Callback after training step finished. - - This callback provides access to the tensors/ops evaluated at this step, - including the additional tensors for which evaluation was requested in - `step_begin`. - - In addition, the callback has the opportunity to stop training by returning - `True`. This is useful for early stopping, for example. - - Note that this method is not called if the call to `Session.run()` that - followed the last call to `step_begin()` failed. - - Args: - step: `int`, the current value of the global step. - output: `dict` mapping `string` values representing tensor names to - the value resulted from running these tensors. Values may be either - scalars, for scalar tensors, or Numpy `array`, for non-scalar tensors. - - Returns: - `bool`. True if training should stop. - - Raises: - ValueError: if we've not begun a step, or `step` number does not match. - """ - if self._current_step != step: - raise ValueError("step_end expected %s but got %s.", self._current_step, - step) - self._current_step = None - return False - - def post_step(self, step, session): # pylint: disable=unused-argument - """Callback after the step is finished. - - Called after step_end and receives session to perform extra session.run - calls. If failure occurred in the process, will be called as well. - - Args: - step: `int`, global step of the model. - session: `Session` object. - """ - _ = step, session - - -def _extract_output(outputs, request): - if request in outputs: - return outputs[request] - return outputs[request.name] - - -class EveryN(BaseMonitor): - """Base class for monitors that execute callbacks every N steps. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - This class adds three new callbacks: - - every_n_step_begin - - every_n_step_end - - every_n_post_step - - The callbacks are executed every n steps, or optionally every step for the - first m steps, where m and n can both be user-specified. - - When extending this class, note that if you wish to use any of the - `BaseMonitor` callbacks, you must call their respective super implementation: - - def step_begin(self, step): - super(ExampleMonitor, self).step_begin(step) - return [] - - Failing to call the super implementation will cause unpredictable behavior. - - The `every_n_post_step()` callback is also called after the last step if it - was not already called through the regular conditions. Note that - `every_n_step_begin()` and `every_n_step_end()` do not receive that special - treatment. - - """ - - # TODO(ipolosukhin): Add also every n seconds. - - def __init__(self, every_n_steps=100, first_n_steps=1): - """Initializes an `EveryN` monitor. - - Args: - every_n_steps: `int`, the number of steps to allow between callbacks. - first_n_steps: `int`, specifying the number of initial steps during - which the callbacks will always be executed, regardless of the value - of `every_n_steps`. Note that this value is relative to the global step - """ - super(EveryN, self).__init__() - self._every_n_steps = every_n_steps - self._first_n_steps = first_n_steps - # Last step in the model. - self._last_successful_step = None - # Last step at which we called one of the every_n methods - self._last_active_step = 0 - self._every_n_step_begin_called = False - - def every_n_step_begin(self, step): # pylint: disable=unused-argument - """Callback before every n'th step begins. - - Args: - step: `int`, the current value of the global step. - - Returns: - A `list` of tensors that will be evaluated at this step. - """ - return [] - - def every_n_step_end(self, step, outputs): # pylint: disable=unused-argument - """Callback after every n'th step finished. - - This callback provides access to the tensors/ops evaluated at this step, - including the additional tensors for which evaluation was requested in - `step_begin`. - - In addition, the callback has the opportunity to stop training by returning - `True`. This is useful for early stopping, for example. - - Args: - step: `int`, the current value of the global step. - outputs: `dict` mapping `string` values representing tensor names to - the value resulted from running these tensors. Values may be either - scalars, for scalar tensors, or Numpy `array`, for non-scalar tensors. - - Returns: - `bool`. True if training should stop. - """ - return False - - def every_n_post_step(self, step, session): - """Callback after a step is finished or `end()` is called. - - Args: - step: `int`, the current value of the global step. - session: `Session` object. - """ - pass - - def step_begin(self, step): - """Overrides `BaseMonitor.step_begin`. - - When overriding this method, you must call the super implementation. - - Args: - step: `int`, the current value of the global step. - Returns: - A `list`, the result of every_n_step_begin, if that was called this step, - or an empty list otherwise. - - Raises: - ValueError: if called more than once during a step. - """ - super(EveryN, self).step_begin(step) - if (step <= self._first_n_steps or - step >= (self._every_n_steps + self._last_active_step) or - step == self._max_steps): # Note: max_steps can be None here. - self._every_n_step_begin_called = True - return self.every_n_step_begin(step) - self._every_n_step_begin_called = False - return [] - - def step_end(self, step, output): - """Overrides `BaseMonitor.step_end`. - - When overriding this method, you must call the super implementation. - - Args: - step: `int`, the current value of the global step. - output: `dict` mapping `string` values representing tensor names to - the value resulted from running these tensors. Values may be either - scalars, for scalar tensors, or Numpy `array`, for non-scalar tensors. - Returns: - `bool`, the result of every_n_step_end, if that was called this step, - or `False` otherwise. - """ - super(EveryN, self).step_end(step, output) - if self._every_n_step_begin_called: - return self.every_n_step_end(step, output) - return False - - def post_step(self, step, session): - super(EveryN, self).post_step(step, session) - if self._every_n_step_begin_called: - self.every_n_post_step(step, session) - self._last_active_step = step - self._last_successful_step = step - - def end(self, session=None): - super(EveryN, self).end(session=session) - if self._last_successful_step != self._last_active_step: - self.every_n_post_step(self._last_successful_step, session) - - -class StopAtStep(BaseMonitor): - """Monitor to request stop at a specified step.""" - - def __init__(self, num_steps=None, last_step=None): - """Create a StopAtStep monitor. - - This monitor requests stop after either a number of steps have been - executed or a last step has been reached. Only of the two options can be - specified. - - if `num_steps` is specified, it indicates the number of steps to execute - after `begin()` is called. If instead `last_step` is specified, it - indicates the last step we want to execute, as passed to the `step_begin()` - call. - - Args: - num_steps: Number of steps to execute. - last_step: Step after which to stop. - - Raises: - ValueError: If one of the arguments is invalid. - """ - super(StopAtStep, self).__init__() - if num_steps is None and last_step is None: - raise ValueError("One of num_steps or last_step must be specified.") - if num_steps is not None and last_step is not None: - raise ValueError("Only one of num_steps or last_step can be specified.") - self._num_steps = num_steps - self._last_step = last_step - - @property - def run_on_all_workers(self): - return True - - def step_begin(self, step): - super(StopAtStep, self).step_begin(step) - if self._last_step is None: - self._last_step = step + self._num_steps - 1 - return [] - - def step_end(self, step, output): - super(StopAtStep, self).step_end(step, output) - return step >= self._last_step - - -# TODO(ptucker): Rename to LoggingTensor since it's not writing to stdout. -class PrintTensor(EveryN): - """Prints given tensors every N steps. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - This is an `EveryN` monitor and has consistent semantic for `every_n` - and `first_n`. - - The tensors will be printed to the log, with `INFO` severity. - """ - - def __init__(self, tensor_names, every_n=100, first_n=1): - """Initializes a PrintTensor monitor. - - Args: - tensor_names: `dict` of tag to tensor names or - `iterable` of tensor names (strings). - every_n: `int`, print every N steps. See `PrintN.` - first_n: `int`, also print the first N steps. See `PrintN.` - """ - super(PrintTensor, self).__init__(every_n, first_n) - if not isinstance(tensor_names, dict): - tensor_names = {item: item for item in tensor_names} - self._tensor_names = tensor_names - - def every_n_step_begin(self, step): - super(PrintTensor, self).every_n_step_begin(step) - return list(self._tensor_names.values()) - - def every_n_step_end(self, step, outputs): - super(PrintTensor, self).every_n_step_end(step, outputs) - stats = [] - for tag, tensor_name in six.iteritems(self._tensor_names): - if tensor_name in outputs: - stats.append("%s = %s" % (tag, - str(_extract_output(outputs, tensor_name)))) - logging.info("Step %d: %s", step, ", ".join(stats)) - - -class LoggingTrainable(EveryN): - """Writes trainable variable values into log every N steps. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Write the tensors in trainable variables `every_n` steps, - starting with the `first_n`th step. - """ - - def __init__(self, scope=None, every_n=100, first_n=1): - """Initializes LoggingTrainable monitor. - - Args: - scope: An optional string to match variable names using re.match. - every_n: Print every N steps. - first_n: Print first N steps. - """ - super(LoggingTrainable, self).__init__(every_n, first_n) - self._scope = scope - - def every_n_step_begin(self, step): - super(LoggingTrainable, self).every_n_step_begin(step) - # Get a list of trainable variables at the beginning of every N steps. - # We cannot get this in __init__ because train_op has not been generated. - trainables = ops.get_collection( - ops.GraphKeys.TRAINABLE_VARIABLES, scope=self._scope) - self._names = {} - for var in trainables: - self._names[var.name] = var.value().name - return list(self._names.values()) - - def every_n_step_end(self, step, outputs): - super(LoggingTrainable, self).every_n_step_end(step, outputs) - stats = [] - for tag, tensor_name in six.iteritems(self._names): - if tensor_name in outputs: - stats.append("%s = %s" % (tag, - str(_extract_output(outputs, tensor_name)))) - logging.info("Logging Trainable: Step %d: %s", step, ", ".join(stats)) - - -class SummarySaver(EveryN): - """Saves summaries every N steps. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - def __init__(self, - summary_op, - save_steps=100, - output_dir=None, - summary_writer=None, - scaffold=None): - """Initializes a `SummarySaver` monitor. - - Args: - summary_op: `Tensor` of type `string`. A serialized `Summary` protocol - buffer, as output by TF summary methods like `summary.scalar` or - `summary.merge_all`. - save_steps: `int`, save summaries every N steps. See `EveryN`. - output_dir: `string`, the directory to save the summaries to. Only used - if no `summary_writer` is supplied. - summary_writer: `SummaryWriter`. If `None` and an `output_dir` was passed, - one will be created accordingly. - scaffold: `Scaffold` to get summary_op if it's not provided. - """ - # TODO(ipolosukhin): Implement every N seconds. - super(SummarySaver, self).__init__(every_n_steps=save_steps) - self._summary_op = summary_op - self._summary_writer = summary_writer - if summary_writer is None and output_dir: - self._summary_writer = core_summary.FileWriter(output_dir) - self._scaffold = scaffold - # TODO(mdan): Throw an error if output_dir and summary_writer are None. - - def set_estimator(self, estimator): - super(SummarySaver, self).set_estimator(estimator) - # TODO(mdan): This line looks redundant. - if self._summary_writer is None: - self._summary_writer = core_summary.FileWriter(estimator.model_dir) - - def every_n_step_begin(self, step): - super(SummarySaver, self).every_n_step_begin(step) - if self._summary_op is None and self._scaffold is not None: - self._summary_op = self._scaffold.summary_op - if self._summary_op is not None: - return [self._summary_op] - return [] - - def every_n_step_end(self, step, outputs): - super(SummarySaver, self).every_n_step_end(step, outputs) - if self._summary_op is not None: - summary_strs = _extract_output(outputs, self._summary_op) - if self._summary_writer: - self._summary_writer.add_summary(summary_strs, step) - return False - - def end(self, session=None): - super(SummarySaver, self).end(session=session) - if self._summary_writer: - self._summary_writer.flush() - - -class ValidationMonitor(EveryN): - """Runs evaluation of a given estimator, at most every N steps. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Note that the evaluation is done based on the saved checkpoint, which will - usually be older than the current step. - - Can do early stopping on validation metrics if `early_stopping_rounds` is - provided. - """ - - def __init__(self, - x=None, - y=None, - input_fn=None, - batch_size=None, - eval_steps=None, - every_n_steps=100, - metrics=None, - hooks=None, - early_stopping_rounds=None, - early_stopping_metric="loss", - early_stopping_metric_minimize=True, - name=None, - check_interval_secs=5): - """Initializes a ValidationMonitor. - - Args: - x: See `BaseEstimator.evaluate`. - y: See `BaseEstimator.evaluate`. - input_fn: See `BaseEstimator.evaluate`. - batch_size: See `BaseEstimator.evaluate`. - eval_steps: See `BaseEstimator.evaluate`. - every_n_steps: Check for new checkpoints to evaluate every N steps. If a - new checkpoint is found, it is evaluated. See `EveryN`. - metrics: See `BaseEstimator.evaluate`. - hooks: A list of `SessionRunHook` hooks to pass to the - `Estimator`'s `evaluate` function. - early_stopping_rounds: `int`. If the metric indicated by - `early_stopping_metric` does not change according to - `early_stopping_metric_minimize` for this many steps, then training - will be stopped. - early_stopping_metric: `string`, name of the metric to check for early - stopping. - early_stopping_metric_minimize: `bool`, True if `early_stopping_metric` is - expected to decrease (thus early stopping occurs when this metric - stops decreasing), False if `early_stopping_metric` is expected to - increase. Typically, `early_stopping_metric_minimize` is True for - loss metrics like mean squared error, and False for performance - metrics like accuracy. - name: See `BaseEstimator.evaluate`. - check_interval_secs: Only check for new checkpoint if at least - `check_interval_secs` have passed. Ignore if None. Default is 5 secs. - - - Raises: - ValueError: If both x and input_fn are provided. - """ - super(ValidationMonitor, self).__init__( - every_n_steps=every_n_steps, first_n_steps=-1) - # TODO(mdan): Checks like this are already done by evaluate. - if x is None and input_fn is None: - raise ValueError("Either x or input_fn should be provided.") - self.x = x - self.y = y - self.input_fn = input_fn - self.batch_size = batch_size - self.eval_steps = eval_steps - self.metrics = metrics - self.hooks = hooks - self.early_stopping_rounds = early_stopping_rounds - self.early_stopping_metric = early_stopping_metric - self.early_stopping_metric_minimize = early_stopping_metric_minimize - self.name = name - self._best_value_step = None - self._best_value = None - self._best_metrics = None - self._early_stopped = False - self._latest_path = None - self._latest_path_step = None - self._last_checkpoint_check_time = None - self._check_interval_secs = check_interval_secs - - @property - def early_stopped(self): - """Returns True if this monitor caused an early stop.""" - return self._early_stopped - - @property - def best_step(self): - """Returns the step at which the best early stopping metric was found.""" - return self._best_value_step - - @property - def best_value(self): - """Returns the best early stopping metric value found so far.""" - return self._best_value - - @property - def best_metrics(self): - """Returns all eval metrics computed with the best early stopping metric. - - For instance, if the metrics computed in two successive evals are - 1. {'loss':40, 'auc':0.5} - 2. {'loss':50, 'auc':0.6} - this function would return the first dict {'loss':40, 'auc':0.5} after both - first and second eval (if `early_stopping_metric` is 'loss' and - `early_stopping_metric_minimize` is True). - - Returns: - The output dict of estimator.evaluate which contains the best value of - the early stopping metric seen so far. - """ - return self._best_metrics - - def _evaluate_estimator(self): - if isinstance(self._estimator, core_estimator.Estimator): - if any((x is not None - for x in [self.x, self.y, self.batch_size, self.metrics])): - raise ValueError( - "tf.estimator.Estimator does not support following " - "arguments: x, y, batch_size, metrics. Should set as `None` " - "in ValidationMonitor") - return self._estimator.evaluate( - input_fn=self.input_fn, - steps=self.eval_steps, - hooks=self.hooks, - name=self.name) - else: - return self._estimator.evaluate( - x=self.x, - y=self.y, - input_fn=self.input_fn, - batch_size=self.batch_size, - steps=self.eval_steps, - metrics=self.metrics, - hooks=self.hooks, - name=self.name) - - def every_n_step_end(self, step, outputs): - super(ValidationMonitor, self).every_n_step_end(step, outputs) - # TODO(mdan): The use of step below is probably misleading. - # The code should probably use the step from the checkpoint, because - # that's what is being evaluated. - if self._estimator is None: - raise ValueError("Missing call to set_estimator.") - current_time = time.time() - if (self._check_interval_secs is not None and - self._last_checkpoint_check_time is not None and - current_time - self._last_checkpoint_check_time <= - self._check_interval_secs): - logging.debug( - "Skipping evaluation since less than %d seconds have passed since " - "last check for a new checkpoint.", self._check_interval_secs) - return False - self._last_checkpoint_check_time = current_time - # Check that we are not running evaluation on the same checkpoint. - latest_path = checkpoint_management.latest_checkpoint( - self._estimator.model_dir) - if latest_path is None: - logging.debug("Skipping evaluation since model has not been saved yet " - "at step %d.", step) - return False - if latest_path is not None and latest_path == self._latest_path: - logging.debug("Skipping evaluation due to same checkpoint %s for step %d " - "as for step %d.", latest_path, step, - self._latest_path_step) - return False - self._latest_path = latest_path - self._latest_path_step = step - - # Run evaluation and log it. - validation_outputs = self._evaluate_estimator() - stats = [] - for name in validation_outputs: - stats.append("%s = %s" % (name, str(validation_outputs[name]))) - logging.info("Validation (step %d): %s", step, ", ".join(stats)) - - # Early stopping logic. - if self.early_stopping_rounds is not None: - if self.early_stopping_metric not in validation_outputs: - raise ValueError("Metric %s missing from outputs %s." % - (self.early_stopping_metric, - set(validation_outputs.keys()))) - current_value = validation_outputs[self.early_stopping_metric] - if (self._best_value is None or (self.early_stopping_metric_minimize and - (current_value < self._best_value)) or - (not self.early_stopping_metric_minimize and - (current_value > self._best_value))): - self._best_value = current_value - self._best_metrics = copy.deepcopy(validation_outputs) - self._best_value_step = step - stop_now = (step - self._best_value_step >= self.early_stopping_rounds) - if stop_now: - logging.info("Stopping. Best step: {} with {} = {}.".format( - self._best_value_step, self.early_stopping_metric, - self._best_value)) - self._early_stopped = True - return True - return False - - -# TODO(ptucker): This really reads any tensor, not just vars, and requires the -# ':0' suffix on var_name. -class CaptureVariable(EveryN): - """Captures a variable's values into a collection. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - This monitor is useful for unit testing. You should exercise caution when - using this monitor in production, since it never discards values. - - This is an `EveryN` monitor and has consistent semantic for `every_n` - and `first_n`. - """ - - def __init__(self, var_name, every_n=100, first_n=1): - """Initializes a CaptureVariable monitor. - - Args: - var_name: `string`. The variable name, including suffix (typically ":0"). - every_n: `int`, print every N steps. See `PrintN.` - first_n: `int`, also print the first N steps. See `PrintN.` - """ - super(CaptureVariable, self).__init__(every_n, first_n) - self._var_name = var_name - self._var_values = {} - - @property - def values(self): - """Returns the values captured so far. - - Returns: - `dict` mapping `int` step numbers to that values of the variable at the - respective step. - """ - return self._var_values - - def every_n_step_begin(self, step): - super(CaptureVariable, self).every_n_step_begin(step) - return [self._var_name] - - def every_n_step_end(self, step, outputs): - super(CaptureVariable, self).every_n_step_end(step, outputs) - self._var_values[step] = _extract_output(outputs, self._var_name) - - -@deprecation.deprecated(None, "Use tf.train.MonitoredTrainingSession.") -def get_default_monitors(loss_op=None, - summary_op=None, - save_summary_steps=100, - output_dir=None, - summary_writer=None): - """Returns a default set of typically-used monitors. - - Args: - loss_op: `Tensor`, the loss tensor. This will be printed using `PrintTensor` - at the default interval. - summary_op: See `SummarySaver`. - save_summary_steps: See `SummarySaver`. - output_dir: See `SummarySaver`. - summary_writer: See `SummarySaver`. - Returns: - `list` of monitors. - """ - - monitors = [] - if loss_op is not None: - monitors.append(PrintTensor(tensor_names={"loss": loss_op.name})) - if summary_op is not None: - monitors.append( - SummarySaver( - summary_op, - save_steps=save_summary_steps, - output_dir=output_dir, - summary_writer=summary_writer)) - return monitors - - -class GraphDump(BaseMonitor): - """Dumps almost all tensors in the graph at every step. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Note, this is very expensive, prefer `PrintTensor` in production. - """ - - IGNORE_OPS = [ - "Const", "Assign", "Identity", "Placeholder", "RandomUniform", "Cast", - "RestoreSlice" - ] - - def __init__(self, ignore_ops=None): - """Initializes GraphDump monitor. - - Args: - ignore_ops: `list` of `string`. Names of ops to ignore. - If None, `GraphDump.IGNORE_OPS` is used. - """ - super(GraphDump, self).__init__() - self._ignore_ops = ignore_ops or GraphDump.IGNORE_OPS - self._data = {} - - def begin(self, max_steps=None): - super(GraphDump, self).begin(max_steps=max_steps) - self._tensors = [] - graph = ops.get_default_graph() - graph_def = graph.as_graph_def() - for node in graph_def.node: - if node.op in self._ignore_ops: - continue - logging.info("op=%s name=%s.", node.op, node.name) - try: - self._tensors.append(graph.get_tensor_by_name(node.name + ":0")) - except KeyError: - pass - - def step_begin(self, step): - super(GraphDump, self).step_begin(step) - return self._tensors - - def step_end(self, step, output): - super(GraphDump, self).step_end(step, output) - self._data[step] = output - - @property - def data(self): - return self._data - - # TODO(ptucker): Handle keys that are in one but not the other. - def compare(self, other_dump, step, atol=1e-06): - """Compares two `GraphDump` monitors and returns differences. - - Args: - other_dump: Another `GraphDump` monitor. - step: `int`, step to compare on. - atol: `float`, absolute tolerance in comparison of floating arrays. - - Returns: - Returns tuple: - matched: `list` of keys that matched. - non_matched: `dict` of keys to tuple of 2 mismatched values. - - Raises: - ValueError: if a key in `data` is missing from `other_dump` at `step`. - """ - non_matched = {} - matched = [] - this_output = self.data[step] if step in self.data else {} - other_output = other_dump.data[step] if step in other_dump.data else {} - for key in this_output: - if not isinstance(key, six.string_types): - continue - if key not in other_output: - raise ValueError("%s missing at step %s.", (key, step)) - value1 = _extract_output(this_output, key) - value2 = _extract_output(other_output, key) - if isinstance(value1, str): - continue - if isinstance(value1, np.ndarray): - if not np.allclose(value1, value2, atol=atol): - non_matched[key] = value1 - value2 - else: - matched.append(key) - else: - if value1 != value2: - non_matched[key] = (value1, value2) - else: - matched.append(key) - return matched, non_matched - - -class ExportMonitor(EveryN): - """Monitor that exports Estimator every N steps. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - @deprecation.deprecated("2017-03-25", - "ExportMonitor is deprecated. Please pass an " - "ExportStrategy to Experiment instead.") - def __init__(self, - every_n_steps, - export_dir, - input_fn=None, - input_feature_key=None, - exports_to_keep=5, - signature_fn=None, - default_batch_size=1): - """Initializes ExportMonitor. - - Args: - every_n_steps: Run monitor every N steps. - export_dir: str, folder to export. - input_fn: A function that takes no argument and returns a tuple of - (features, labels), where features is a dict of string key to `Tensor` - and labels is a `Tensor` that's currently not used (and so can be - `None`). - input_feature_key: String key into the features dict returned by - `input_fn` that corresponds to the raw `Example` strings `Tensor` that - the exported model will take as input. Should be `None` if and only if - you're passing in a `signature_fn` that does not use the first arg - (`Tensor` of `Example` strings). - exports_to_keep: int, number of exports to keep. - signature_fn: Function that returns a default signature and a named - signature map, given `Tensor` of `Example` strings, `dict` of `Tensor`s - for features and `dict` of `Tensor`s for predictions. - default_batch_size: Default batch size of the `Example` placeholder. - - Raises: - ValueError: If `input_fn` and `input_feature_key` are not both defined or - are not both `None`. - """ - super(ExportMonitor, self).__init__(every_n_steps=every_n_steps) - self._export_dir = export_dir - self._input_fn = input_fn - self._input_feature_key = input_feature_key - self._use_deprecated_input_fn = input_fn is None - self._exports_to_keep = exports_to_keep - self._signature_fn = signature_fn - self._default_batch_size = default_batch_size - self._last_export_dir = None - - @property - def export_dir(self): - return self._export_dir - - @property - def exports_to_keep(self): - return self._exports_to_keep - - @property - def signature_fn(self): - return self._signature_fn - - @property - def last_export_dir(self): - """Returns the directory containing the last completed export. - - Returns: - The string path to the exported directory. NB: this functionality was - added on 2016/09/25; clients that depend on the return value may need - to handle the case where this function returns None because the - estimator being fitted does not yet return a value during export. - """ - return self._last_export_dir - - def every_n_step_end(self, step, outputs): - super(ExportMonitor, self).every_n_step_end(step, outputs) - try: - if isinstance(self._estimator, core_estimator.Estimator): - raise ValueError( - "ExportMonitor does not support `tf.estimator.Estimator. `. " - "Please pass an ExportStrategy to Experiment instead.") - self._last_export_dir = self._estimator.export( - self.export_dir, - exports_to_keep=self.exports_to_keep, - signature_fn=self.signature_fn, - input_fn=self._input_fn, - default_batch_size=self._default_batch_size, - input_feature_key=self._input_feature_key, - use_deprecated_input_fn=self._use_deprecated_input_fn) - except RuntimeError: - # Currently we are not syncronized with saving checkpoints, which leads to - # runtime errors when we are calling export on the same global step. - # Exports depend on saved checkpoints for constructing the graph and - # getting the global step from the graph instance saved in the checkpoint. - # If the checkpoint is stale with respect to current step, the global step - # is taken to be the last saved checkpoint's global step and exporter - # doesn't export the same checkpoint again with the following error. - logging.info("Skipping exporting because the existing checkpoint has " - "already been exported. " - "Consider exporting less frequently.") - - def end(self, session=None): - super(ExportMonitor, self).end(session=session) - latest_path = checkpoint_management.latest_checkpoint( - self._estimator.model_dir) - if latest_path is None: - logging.info("Skipping export at the end since model has not been saved " - "yet.") - return - if isinstance(self._estimator, core_estimator.Estimator): - raise ValueError( - "ExportMonitor does not support `tf.estimator.Estimator. `. " - "Please pass an ExportStrategy to Experiment instead.") - try: - self._last_export_dir = self._estimator.export( - self.export_dir, - exports_to_keep=self.exports_to_keep, - signature_fn=self.signature_fn, - input_fn=self._input_fn, - default_batch_size=self._default_batch_size, - input_feature_key=self._input_feature_key, - use_deprecated_input_fn=self._use_deprecated_input_fn) - except RuntimeError: - logging.info("Skipping exporting for the same step.") - - -class CheckpointSaver(BaseMonitor): - """Saves checkpoints every N steps or N seconds. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - def __init__(self, - checkpoint_dir, - save_secs=None, - save_steps=None, - saver=None, - checkpoint_basename="model.ckpt", - scaffold=None): - """Initialize CheckpointSaver monitor. - - Args: - checkpoint_dir: `str`, base directory for the checkpoint files. - save_secs: `int`, save every N secs. - save_steps: `int`, save every N steps. - saver: `Saver` object, used for saving. - checkpoint_basename: `str`, base name for the checkpoint files. - scaffold: `Scaffold`, use to get saver object. - - Raises: - ValueError: If both `save_steps` and `save_secs` are not `None`. - ValueError: If both `save_steps` and `save_secs` are `None`. - """ - logging.info("Create CheckpointSaver.") - super(CheckpointSaver, self).__init__() - self._saver = saver - self._summary_writer = core_summary.FileWriterCache.get(checkpoint_dir) - self._save_path = os.path.join(checkpoint_dir, checkpoint_basename) - self._scaffold = scaffold - self._save_secs = save_secs - self._save_steps = save_steps - self._last_saved_time = None - self._last_begin_step = None - self._last_saved_step = None - - if save_steps is None and save_secs is None: - raise ValueError("Either save_steps or save_secs should be provided") - if (save_steps is not None) and (save_secs is not None): - raise ValueError("Can not provide both save_steps and save_secs.") - - def begin(self, max_steps=None): - super(CheckpointSaver, self).begin(max_steps) - self._last_saved_time = None - self._last_begin_step = None - self._last_saved_step = None - - def step_begin(self, step): - super(CheckpointSaver, self).step_begin(step) - self._last_begin_step = step - - def post_step(self, step, session): - super(CheckpointSaver, self).post_step(step, session) - if self._last_saved_time is None: - self._save(step, session) - - if self._save_steps is not None: - if step >= self._last_saved_step + self._save_steps: - self._save(step, session) - - if self._save_secs is not None: - if time.time() >= self._last_saved_time + self._save_secs: - self._save(step, session) - - def end(self, session=None): - super(CheckpointSaver, self).end(session) - self._save(self._last_begin_step, session) - - def _save(self, step, session): - """Saves the latest checkpoint.""" - if step == self._last_saved_step: - return - logging.info("Saving checkpoints for %d into %s.", step, self._save_path) - self._last_saved_time = time.time() - self._last_saved_step = step - if self._saver is None: - self._scaffold.saver.save(session, self._save_path, global_step=step) - else: - self._saver.save(session, self._save_path, global_step=step) - self._summary_writer.add_session_log( - SessionLog( - status=SessionLog.CHECKPOINT, checkpoint_path=self._save_path), - step) - - -class StepCounter(EveryN): - """Steps per second monitor. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - def __init__(self, every_n_steps=100, output_dir=None, summary_writer=None): - super(StepCounter, self).__init__(every_n_steps=every_n_steps) - self._summary_tag = "global_step/sec" - self._last_reported_step = None - self._last_reported_time = None - self._summary_writer = summary_writer - if summary_writer is None and output_dir: - self._summary_writer = core_summary.FileWriterCache.get(output_dir) - - def set_estimator(self, estimator): - super(StepCounter, self).set_estimator(estimator) - if self._summary_writer is None: - self._summary_writer = core_summary.FileWriterCache.get( - estimator.model_dir) - - def every_n_step_end(self, current_step, outputs): - current_time = time.time() - if self._last_reported_time is not None and self._summary_writer: - added_steps = current_step - self._last_reported_step - elapsed_time = current_time - self._last_reported_time - steps_per_sec = added_steps / elapsed_time - summary = Summary(value=[ - Summary.Value(tag=self._summary_tag, simple_value=steps_per_sec) - ]) - self._summary_writer.add_summary(summary, current_step) - self._last_reported_step = current_step - self._last_reported_time = current_time - - -class NanLossDuringTrainingError(RuntimeError): - - def __str__(self): - return "NaN loss during training." - - -class NanLoss(EveryN): - """NaN Loss monitor. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Monitors loss and stops training if loss is NaN. - Can either fail with exception or just stop training. - """ - - def __init__(self, loss_tensor, every_n_steps=100, fail_on_nan_loss=True): - """Initializes NanLoss monitor. - - Args: - loss_tensor: `Tensor`, the loss tensor. - every_n_steps: `int`, run check every this many steps. - fail_on_nan_loss: `bool`, whether to raise exception when loss is NaN. - """ - super(NanLoss, self).__init__(every_n_steps=every_n_steps) - self._loss_tensor = loss_tensor - self._fail_on_nan_loss = fail_on_nan_loss - - def every_n_step_begin(self, step): - super(NanLoss, self).every_n_step_begin(step) - return [self._loss_tensor] - - def every_n_step_end(self, step, outputs): - super(NanLoss, self).every_n_step_end(step, outputs) - if np.isnan(_extract_output(outputs, self._loss_tensor)): - failure_message = "Model diverged with loss = NaN." - if self._fail_on_nan_loss: - logging.error(failure_message) - raise NanLossDuringTrainingError - else: - logging.warning(failure_message) - # We don't raise an error but we return "should stop" so we stop, but - # without an exception. - return True - - -class RunHookAdapterForMonitors(session_run_hook.SessionRunHook): - """Wraps monitors into a SessionRunHook.""" - - def __init__(self, monitors): - self._monitors = monitors - - def begin(self): - self._last_step = None - self._global_step_tensor = training_util.get_global_step() - for m in self._monitors: - m.begin(max_steps=None) - - def before_run(self, run_context): - if self._last_step is None: - self._last_step = run_context.session.run(self._global_step_tensor) + 1 - - request = {self._global_step_tensor: self._global_step_tensor} - monitor_fetches = [] - for m in self._monitors: - monitor_requests = m.step_begin(self._last_step) - if monitor_requests: - if not isinstance(monitor_requests, list): - raise ValueError("Monitor.step_begin should return a list.") - monitor_fetches.extend(monitor_requests) - if monitor_fetches: - request["monitors"] = dict( - zip(monitor_fetches, [_as_graph_element(f) for f in monitor_fetches])) - - return session_run_hook.SessionRunArgs(request) - - def after_run(self, run_context, run_values): - result = run_values.results[ - "monitors"] if "monitors" in run_values.results else {} - for m in self._monitors: - induce_stop = m.step_end(self._last_step, result) - if induce_stop: - run_context.request_stop() - - for m in self._monitors: - m.post_step(self._last_step, run_context.session) - - self._last_step = run_values.results[self._global_step_tensor] + 1 - - def end(self, session): - self._last_step = None - for m in self._monitors: - if "session" in tf_inspect.getargspec(m.end).args: - m.end(session=session) - else: - m.end() - - -def replace_monitors_with_hooks(monitors_or_hooks, estimator): - """Wraps monitors with a hook. - - `Monitor` is deprecated in favor of `SessionRunHook`. If you're using a - monitor, you can wrap it with a hook using function. It is recommended to - implement hook version of your monitor. - - Args: - monitors_or_hooks: A `list` may contain both monitors and hooks. - estimator: An `Estimator` that monitor will be used with. - - Returns: - Returns a list of hooks. If there is any monitor in the given list, it is - replaced by a hook. - """ - monitors_or_hooks = monitors_or_hooks or [] - hooks = [ - m for m in monitors_or_hooks - if isinstance(m, session_run_hook.SessionRunHook) - ] - - deprecated_monitors = [ - m for m in monitors_or_hooks - if not isinstance(m, session_run_hook.SessionRunHook) - ] - - if not estimator.config.is_chief: - # Prune list of monitor to the ones runnable on all workers. - deprecated_monitors = [ - m for m in deprecated_monitors if m.run_on_all_workers - ] - - # Setup monitors. - for monitor in deprecated_monitors: - monitor.set_estimator(estimator) - - if deprecated_monitors: - hooks.append(RunHookAdapterForMonitors(deprecated_monitors)) - - return hooks - - -def _as_graph_element(obj): - """Retrieves Graph element.""" - graph = ops.get_default_graph() - if not isinstance(obj, six.string_types): - if not hasattr(obj, "graph") or obj.graph != graph: - raise ValueError("Passed %s should have graph attribute that is equal " - "to current graph %s." % (obj, graph)) - return obj - if ":" in obj: - element = graph.as_graph_element(obj) - else: - element = graph.as_graph_element(obj + ":0") - # Check that there is no :1 (e.g. it's single output). - try: - graph.as_graph_element(obj + ":1") - except (KeyError, ValueError): - pass - else: - raise ValueError("Name %s is ambiguous, " - "as this `Operation` has multiple outputs " - "(at least 2)." % obj) - return element diff --git a/tensorflow/contrib/learn/python/learn/monitors_test.py b/tensorflow/contrib/learn/python/learn/monitors_test.py deleted file mode 100644 index d4a7169bb63..00000000000 --- a/tensorflow/contrib/learn/python/learn/monitors_test.py +++ /dev/null @@ -1,851 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Monitors tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import shutil -import tempfile -import time - -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib import testing -from tensorflow.contrib.framework.python.framework import checkpoint_utils -from tensorflow.contrib.learn.python import learn -from tensorflow.contrib.learn.python.learn import estimators -from tensorflow.python.client import session as session_lib -from tensorflow.python.estimator import estimator as core_estimator -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.summary import summary -from tensorflow.python.training import checkpoint_management -from tensorflow.python.training import gradient_descent -from tensorflow.python.training import monitored_session -from tensorflow.python.training import training_util - - -class _MyEveryN(learn.monitors.EveryN): - - def __init__(self, every_n_steps=100, first_n_steps=1): - super(_MyEveryN, self).__init__( - every_n_steps=every_n_steps, first_n_steps=first_n_steps) - self._steps_begun = [] - self._steps_ended = [] - self._post_steps = [] - - @property - def steps_begun(self): - return self._steps_begun - - @property - def steps_ended(self): - return self._steps_ended - - @property - def post_steps(self): - return self._post_steps - - def every_n_step_begin(self, step): - super(_MyEveryN, self).every_n_step_begin(step) - self._steps_begun.append(step) - return [] - - def every_n_step_end(self, step, outputs): - super(_MyEveryN, self).every_n_step_end(step, outputs) - self._steps_ended.append(step) - return False - - def every_n_post_step(self, step, session): - super(_MyEveryN, self).every_n_post_step(step, session) - self._post_steps.append(step) - return False - - -class MonitorsTest(test.TestCase): - """Monitors tests.""" - - def setUp(self): - # Mock out logging calls so we can verify whether correct tensors are being - # monitored. - self._actual_log = logging.info - - def mockLog(*args, **kwargs): # pylint: disable=invalid-name - self.logged_message = args - self._actual_log(*args, **kwargs) - - logging.info = mockLog - - def tearDown(self): - logging.info = self._actual_log - - def _run_monitor(self, - monitor, - num_epochs=3, - num_steps_per_epoch=10, - pass_max_steps=True): - if pass_max_steps: - max_steps = num_epochs * num_steps_per_epoch - 1 - else: - max_steps = None - monitor.begin(max_steps=max_steps) - for epoch in xrange(num_epochs): - monitor.epoch_begin(epoch) - should_stop = False - step = epoch * num_steps_per_epoch - next_epoch_step = step + num_steps_per_epoch - while (not should_stop) and (step < next_epoch_step): - tensors = monitor.step_begin(step) - output = ops.get_default_session().run(tensors) if tensors else {} - output = dict( - zip([t.name if isinstance(t, ops.Tensor) else t for t in tensors], - output)) - should_stop = monitor.step_end(step=step, output=output) - monitor.post_step(step=step, session=None) - step += 1 - monitor.epoch_end(epoch) - monitor.end() - - def test_base_monitor(self): - with ops.Graph().as_default() as g, self.session(g): - self._run_monitor(learn.monitors.BaseMonitor()) - - def test_every_0(self): - monitor = _MyEveryN(every_n_steps=0, first_n_steps=-1) - with ops.Graph().as_default() as g, self.session(g): - self._run_monitor(monitor, num_epochs=3, num_steps_per_epoch=10) - expected_steps = list(range(30)) - self.assertAllEqual(expected_steps, monitor.steps_begun) - self.assertAllEqual(expected_steps, monitor.steps_ended) - self.assertAllEqual(expected_steps, monitor.post_steps) - - def test_every_1(self): - monitor = _MyEveryN(every_n_steps=1, first_n_steps=-1) - with ops.Graph().as_default() as g, self.session(g): - self._run_monitor(monitor, num_epochs=3, num_steps_per_epoch=10) - expected_steps = list(range(1, 30)) - self.assertEqual(expected_steps, monitor.steps_begun) - self.assertEqual(expected_steps, monitor.steps_ended) - self.assertEqual(expected_steps, monitor.post_steps) - - def test_every_2(self): - monitor = _MyEveryN(every_n_steps=2, first_n_steps=-1) - with ops.Graph().as_default() as g, self.session(g): - self._run_monitor(monitor, num_epochs=3, num_steps_per_epoch=10) - expected_steps = list(range(2, 29, 2)) + [29] - self.assertEqual(expected_steps, monitor.steps_begun) - self.assertEqual(expected_steps, monitor.steps_ended) - self.assertEqual(expected_steps, monitor.post_steps) - - def test_every_8(self): - monitor = _MyEveryN(every_n_steps=8, first_n_steps=2) - with ops.Graph().as_default() as g, self.session(g): - self._run_monitor(monitor, num_epochs=3, num_steps_per_epoch=10) - expected_steps = [0, 1, 2, 10, 18, 26, 29] - self.assertEqual(expected_steps, monitor.steps_begun) - self.assertEqual(expected_steps, monitor.steps_ended) - self.assertEqual(expected_steps, monitor.post_steps) - - def test_every_8_no_max_steps(self): - monitor = _MyEveryN(every_n_steps=8, first_n_steps=2) - with ops.Graph().as_default() as g, self.session(g): - self._run_monitor( - monitor, num_epochs=3, num_steps_per_epoch=10, pass_max_steps=False) - begin_end_steps = [0, 1, 2, 10, 18, 26] - post_steps = [0, 1, 2, 10, 18, 26, 29] - self.assertEqual(begin_end_steps, monitor.steps_begun) - self.assertEqual(begin_end_steps, monitor.steps_ended) - self.assertEqual(post_steps, monitor.post_steps) - - def test_every_8_recovered_after_step_begin(self): - monitor = _MyEveryN(every_n_steps=8) - with ops.Graph().as_default() as g, self.session(g): - for step in [8, 16]: - monitor.step_begin(step) - monitor.step_begin(step) - monitor.step_end(step, output=None) - monitor.post_step(step, session=None) - # It should call begin again since, end was not called - self.assertEqual([8, 8, 16, 16], monitor.steps_begun) - self.assertEqual([8, 16], monitor.steps_ended) - self.assertEqual([8, 16], monitor.post_steps) - - def test_every_8_recovered_after_step_end(self): - monitor = _MyEveryN(every_n_steps=8) - with ops.Graph().as_default() as g, self.session(g): - for step in [8, 16]: - monitor.step_begin(step) - monitor.step_end(step, output=None) - monitor.post_step(step, session=None) - monitor.step_begin(step) - monitor.step_end(step, output=None) - monitor.post_step(step, session=None) - # It should not call begin twice since end was called - self.assertEqual([8, 16], monitor.steps_begun) - self.assertEqual([8, 16], monitor.steps_ended) - self.assertEqual([8, 16], monitor.post_steps) - - def test_every_8_call_post_step_at_the_end(self): - monitor = _MyEveryN(every_n_steps=8) - with ops.Graph().as_default() as g, self.session(g): - monitor.begin() - for step in [8, 16]: - monitor.step_begin(step) - monitor.step_end(step, output=None) - monitor.post_step(step, session=None) - monitor.step_begin(19) - monitor.step_end(19, output=None) - monitor.post_step(19, session=None) - monitor.end(session=None) - # It should not call begin twice since end was called - self.assertEqual([8, 16], monitor.steps_begun) - self.assertEqual([8, 16], monitor.steps_ended) - self.assertEqual([8, 16, 19], monitor.post_steps) - - def test_every_8_call_post_step_should_not_be_called_twice(self): - monitor = _MyEveryN(every_n_steps=8) - with ops.Graph().as_default() as g, self.session(g): - monitor.begin() - for step in [8, 16]: - monitor.step_begin(step) - monitor.step_end(step, output=None) - monitor.post_step(step, session=None) - monitor.step_begin(16) - monitor.step_end(16, output=None) - monitor.post_step(16, session=None) - monitor.end(session=None) - # It should not call begin twice since end was called - self.assertEqual([8, 16], monitor.steps_begun) - self.assertEqual([8, 16], monitor.steps_ended) - self.assertEqual([8, 16], monitor.post_steps) - - def test_print(self): - with ops.Graph().as_default() as g, self.session(g): - t = constant_op.constant(42.0, name='foo') - self._run_monitor(learn.monitors.PrintTensor(tensor_names=[t.name])) - self.assertRegexpMatches(str(self.logged_message), t.name) - - def test_logging_trainable(self): - with ops.Graph().as_default() as g, self.session(g): - var = variables.VariableV1(constant_op.constant(42.0), name='foo') - var.initializer.run() - cof = constant_op.constant(1.0) - loss = math_ops.subtract( - math_ops.multiply(var, cof), constant_op.constant(1.0)) - train_step = gradient_descent.GradientDescentOptimizer(0.5).minimize(loss) - ops.get_default_session().run(train_step) - self._run_monitor(learn.monitors.LoggingTrainable('foo')) - self.assertRegexpMatches(str(self.logged_message), var.name) - - def test_summary_saver(self): - with ops.Graph().as_default() as g, self.session(g): - log_dir = 'log/dir' - summary_writer = testing.FakeSummaryWriter(log_dir, g) - var = variables.VariableV1(0.0) - var.initializer.run() - tensor = state_ops.assign_add(var, 1.0) - summary_op = summary.scalar('my_summary', tensor) - self._run_monitor( - learn.monitors.SummarySaver( - summary_op=summary_op, - save_steps=8, - summary_writer=summary_writer), - num_epochs=3, - num_steps_per_epoch=10) - summary_writer.assert_summaries( - test_case=self, - expected_logdir=log_dir, - expected_graph=g, - expected_summaries={ - 0: { - 'my_summary': 1.0 - }, - 1: { - 'my_summary': 2.0 - }, - 9: { - 'my_summary': 3.0 - }, - 17: { - 'my_summary': 4.0 - }, - 25: { - 'my_summary': 5.0 - }, - 29: { - 'my_summary': 6.0 - }, - }) - - def _assert_validation_monitor(self, - monitor, - expected_early_stopped=False, - expected_best_step=None, - expected_best_value=None, - expected_best_metrics=None): - self.assertEqual(expected_early_stopped, monitor.early_stopped) - self.assertEqual(expected_best_step, monitor.best_step) - self.assertEqual(expected_best_value, monitor.best_value) - self.assertEqual(expected_best_metrics, monitor.best_metrics) - - def test_validation_monitor_no_estimator(self): - monitor = learn.monitors.ValidationMonitor( - x=constant_op.constant(2.0), every_n_steps=0) - self._assert_validation_monitor(monitor) - with ops.Graph().as_default() as g, self.session(g): - with self.assertRaisesRegexp(ValueError, 'set_estimator'): - self._run_monitor(monitor) - - @test.mock.patch.object(estimators, 'Estimator', autospec=True) - @test.mock.patch.object(checkpoint_management, 'latest_checkpoint') - def test_validation_monitor_no_ckpt(self, mock_latest_checkpoint, - mock_estimator_class): - estimator = mock_estimator_class() - model_dir = 'model/dir' - estimator.model_dir = model_dir - mock_latest_checkpoint.return_value = None - - # Do nothing with no checkpoint. - monitor = learn.monitors.ValidationMonitor( - x=constant_op.constant(2.0), every_n_steps=0) - self._assert_validation_monitor(monitor) - monitor.set_estimator(estimator) - with ops.Graph().as_default() as g, self.session(g): - self._run_monitor(monitor) - self._assert_validation_monitor(monitor) - mock_latest_checkpoint.assert_called_with(model_dir) - - @test.mock.patch.object(estimators, 'Estimator', autospec=True) - @test.mock.patch.object(checkpoint_management, 'latest_checkpoint') - def test_validation_monitor_no_early_stopping_rounds(self, - mock_latest_checkpoint, - mock_estimator_class): - estimator = mock_estimator_class() - model_dir = 'model/dir' - estimator.model_dir = model_dir - estimator.evaluate.return_value = {} - mock_latest_checkpoint.return_value = '%s/ckpt' % model_dir - - # Do nothing with early_stopping_rounds=None. - monitor = learn.monitors.ValidationMonitor( - x=constant_op.constant(2.0), every_n_steps=0) - self._assert_validation_monitor(monitor) - monitor.set_estimator(estimator) - with ops.Graph().as_default() as g, self.session(g): - self._run_monitor(monitor) - self._assert_validation_monitor(monitor) - - @test.mock.patch.object(estimators, 'Estimator', autospec=True) - @test.mock.patch.object(checkpoint_management, 'latest_checkpoint') - def test_validation_monitor_invalid_metric(self, mock_latest_checkpoint, - mock_estimator_class): - estimator = mock_estimator_class() - model_dir = 'model/dir' - estimator.model_dir = model_dir - estimator.evaluate.return_value = {} - mock_latest_checkpoint.return_value = '%s/ckpt' % model_dir - - # Fail for missing metric. - monitor = learn.monitors.ValidationMonitor( - x=constant_op.constant(2.0), every_n_steps=0, early_stopping_rounds=1) - self._assert_validation_monitor(monitor) - monitor.set_estimator(estimator) - with ops.Graph().as_default() as g, self.session(g): - with self.assertRaisesRegexp(ValueError, 'missing from outputs'): - self._run_monitor(monitor, num_epochs=1, num_steps_per_epoch=1) - - @test.mock.patch.object(estimators, 'Estimator', autospec=True) - @test.mock.patch.object(checkpoint_management, 'latest_checkpoint') - def test_validation_monitor(self, mock_latest_checkpoint, - mock_estimator_class): - estimator = mock_estimator_class() - model_dir = 'model/dir' - estimator.model_dir = model_dir - validation_outputs = {'loss': None, 'auc': None} - estimator.evaluate.return_value = validation_outputs - - monitor = learn.monitors.ValidationMonitor( - x=constant_op.constant(2.0), - every_n_steps=0, - early_stopping_rounds=2, - check_interval_secs=None) - - self._assert_validation_monitor(monitor) - monitor.set_estimator(estimator) - with ops.Graph().as_default() as g, self.session(g): - monitor.begin(max_steps=100) - monitor.epoch_begin(epoch=0) - self.assertEqual(0, estimator.evaluate.call_count) - - # Step 0, initial loss. - step = 0 - mock_latest_checkpoint.return_value = '%s/ckpt.%s' % (model_dir, step) - validation_outputs['loss'] = 42.0 - validation_outputs['auc'] = 0.5 - self.assertEqual(0, len(monitor.step_begin(step=step))) - self.assertFalse(monitor.step_end(step=step, output={})) - self.assertEqual(1, estimator.evaluate.call_count) - self._assert_validation_monitor( - monitor, expected_best_step=0, expected_best_value=42.0, - expected_best_metrics={'loss': 42.0, 'auc': 0.5}) - monitor.post_step(step=step, session=None) - - # Step 1, same checkpoint, no eval. - step = 1 - self.assertEqual(0, len(monitor.step_begin(step=step))) - self.assertFalse(monitor.step_end(step=step, output={})) - self.assertEqual(1, estimator.evaluate.call_count) - self._assert_validation_monitor( - monitor, expected_best_step=0, expected_best_value=42.0, - expected_best_metrics={'loss': 42.0, 'auc': 0.5}) - monitor.post_step(step=step, session=None) - - # Step 2, lower loss. - step = 2 - mock_latest_checkpoint.return_value = '%s/ckpt.%s' % (model_dir, step) - validation_outputs['loss'] = 40.0 - validation_outputs['auc'] = 0.6 - self.assertEqual(0, len(monitor.step_begin(step=step))) - self.assertFalse(monitor.step_end(step=step, output={})) - self.assertEqual(2, estimator.evaluate.call_count) - self._assert_validation_monitor( - monitor, expected_best_step=2, expected_best_value=40.0, - expected_best_metrics={'loss': 40.0, 'auc': 0.6}) - monitor.post_step(step=step, session=None) - - # Step 3, higher loss. - step = 3 - mock_latest_checkpoint.return_value = '%s/ckpt.%s' % (model_dir, step) - validation_outputs['loss'] = 44.0 - validation_outputs['auc'] = 0.7 - self.assertEqual(0, len(monitor.step_begin(step=step))) - self.assertFalse(monitor.step_end(step=step, output={})) - self.assertEqual(3, estimator.evaluate.call_count) - self._assert_validation_monitor( - monitor, expected_best_step=2, expected_best_value=40.0, - expected_best_metrics={'loss': 40.0, 'auc': 0.6}) - monitor.post_step(step=step, session=None) - - # Step 4, higher loss for 2 steps, early stopping. - step = 4 - mock_latest_checkpoint.return_value = '%s/ckpt.%s' % (model_dir, step) - validation_outputs['loss'] = 43.0 - self.assertEqual(0, len(monitor.step_begin(step=step))) - self.assertTrue(monitor.step_end(step=step, output={})) - self.assertEqual(4, estimator.evaluate.call_count) - self._assert_validation_monitor( - monitor, - expected_early_stopped=True, - expected_best_step=2, - expected_best_value=40.0, - expected_best_metrics={'loss': 40.0, 'auc': 0.6}) - monitor.post_step(step=step, session=None) - - monitor.epoch_end(epoch=0) - monitor.end() - - @test.mock.patch.object(checkpoint_management, 'latest_checkpoint') - def test_validation_monitor_with_core_estimator(self, mock_latest_checkpoint): - estimator = test.mock.Mock(spec=core_estimator.Estimator) - model_dir = 'model/dir' - estimator.model_dir = model_dir - validation_outputs = {'loss': None, 'auc': None} - estimator.evaluate.return_value = validation_outputs - - monitor = learn.monitors.ValidationMonitor( - input_fn=lambda: constant_op.constant(2.0), - every_n_steps=0, early_stopping_rounds=2) - self._assert_validation_monitor(monitor) - monitor.set_estimator(estimator) - with ops.Graph().as_default() as g, self.session(g): - monitor.begin(max_steps=100) - monitor.epoch_begin(epoch=0) - self.assertEqual(0, estimator.evaluate.call_count) - - # Step 0, initial loss. - step = 0 - mock_latest_checkpoint.return_value = '%s/ckpt.%s' % (model_dir, step) - validation_outputs['loss'] = 42.0 - validation_outputs['auc'] = 0.5 - self.assertEqual(0, len(monitor.step_begin(step=step))) - self.assertFalse(monitor.step_end(step=step, output={})) - self.assertEqual(1, estimator.evaluate.call_count) - self._assert_validation_monitor( - monitor, expected_best_step=0, expected_best_value=42.0, - expected_best_metrics={'loss': 42.0, 'auc': 0.5}) - monitor.post_step(step=step, session=None) - - @test.mock.patch.object(checkpoint_management, 'latest_checkpoint') - def test_validation_monitor_fail_with_core_estimator_and_metrics( - self, mock_latest_checkpoint): - estimator = test.mock.Mock(spec=core_estimator.Estimator) - model_dir = 'model/dir' - estimator.model_dir = model_dir - validation_outputs = {'loss': None} - estimator.evaluate.return_value = validation_outputs - - monitor = learn.monitors.ValidationMonitor( - input_fn=lambda: constant_op.constant(2.0), - metrics=constant_op.constant(2.0), - every_n_steps=0, early_stopping_rounds=2) - monitor.set_estimator(estimator) - with ops.Graph().as_default() as g, self.session(g): - monitor.begin(max_steps=100) - monitor.epoch_begin(epoch=0) - - with self.assertRaisesRegexp( - ValueError, - 'tf.estimator.Estimator does not support .* metrics'): - step = 0 - mock_latest_checkpoint.return_value = '%s/ckpt.%s' % (model_dir, step) - validation_outputs['loss'] = 42.0 - self.assertEqual(0, len(monitor.step_begin(step=step))) - self.assertFalse(monitor.step_end(step=step, output={})) - - def test_graph_dump(self): - monitor0 = learn.monitors.GraphDump() - monitor1 = learn.monitors.GraphDump() - with ops.Graph().as_default() as g, self.session(g): - const_var = variables.VariableV1(42.0, name='my_const') - counter_var = variables.VariableV1(0.0, name='my_counter') - assign_add = state_ops.assign_add(counter_var, 1.0, name='my_assign_add') - variables.global_variables_initializer().run() - - self._run_monitor(monitor0, num_epochs=3, num_steps_per_epoch=10) - self.assertEqual({ - step: { - const_var.name: 42.0, - counter_var.name: step + 1.0, - assign_add.name: step + 1.0, - } - for step in xrange(30) - }, monitor0.data) - - self._run_monitor(monitor1, num_epochs=3, num_steps_per_epoch=10) - self.assertEqual({ - step: { - const_var.name: 42.0, - counter_var.name: step + 31.0, - assign_add.name: step + 31.0, - } - for step in xrange(30) - }, monitor1.data) - - for step in xrange(30): - matched, non_matched = monitor1.compare(monitor0, step=step) - self.assertEqual([const_var.name], matched) - self.assertEqual({ - assign_add.name: (step + 31.0, step + 1.0), - counter_var.name: (step + 31.0, step + 1.0), - }, non_matched) - matched, non_matched = monitor0.compare(monitor1, step=step) - self.assertEqual([const_var.name], matched) - self.assertEqual({ - assign_add.name: (step + 1.0, step + 31.0), - counter_var.name: (step + 1.0, step + 31.0), - }, non_matched) - - def test_capture_variable(self): - monitor = learn.monitors.CaptureVariable( - var_name='my_assign_add:0', every_n=8, first_n=2) - with ops.Graph().as_default() as g, self.session(g): - var = variables.VariableV1(0.0, name='my_var') - var.initializer.run() - state_ops.assign_add(var, 1.0, name='my_assign_add') - self._run_monitor(monitor, num_epochs=3, num_steps_per_epoch=10) - self.assertEqual({ - 0: 1.0, - 1: 2.0, - 2: 3.0, - 10: 4.0, - 18: 5.0, - 26: 6.0, - 29: 7.0, - }, monitor.values) - - -class StopAtStepTest(test.TestCase): - - def test_raise_in_both_last_step_and_num_steps(self): - with self.assertRaises(ValueError): - learn.monitors.StopAtStep(num_steps=10, last_step=20) - - def test_stop_based_on_last_step(self): - m = learn.monitors.StopAtStep(last_step=10) - m.step_begin(5) - self.assertFalse(m.step_end(5, None)) - m.step_begin(9) - self.assertFalse(m.step_end(9, None)) - m.step_begin(10) - self.assertTrue(m.step_end(10, None)) - m.step_begin(11) - self.assertTrue(m.step_end(11, None)) - - def test_stop_based_on_num_step(self): - m = learn.monitors.StopAtStep(num_steps=10) - m.step_begin(5) - self.assertFalse(m.step_end(5, None)) - m.step_begin(13) - self.assertFalse(m.step_end(13, None)) - m.step_begin(14) - self.assertTrue(m.step_end(14, None)) - m.step_begin(15) - self.assertTrue(m.step_end(15, None)) - - -class CheckpointSaverTest(test.TestCase): - - def setUp(self): - self.model_dir = tempfile.mkdtemp() - self.graph = ops.Graph() - with self.graph.as_default(): - self.scaffold = monitored_session.Scaffold() - self.global_step = training_util.get_or_create_global_step() - self.train_op = state_ops.assign_add(self.global_step, 1) - - def tearDown(self): - shutil.rmtree(self.model_dir, ignore_errors=True) - - def _run(self, monitor, step, train_op, sess): - monitor.step_begin(step) - sess.run(train_op) - monitor.post_step(step, sess) - - def test_raise_in_both_secs_and_steps(self): - with self.assertRaises(ValueError): - learn.monitors.CheckpointSaver( - self.model_dir, save_secs=10, save_steps=20) - - def test_raise_in_none_secs_and_steps(self): - with self.assertRaises(ValueError): - learn.monitors.CheckpointSaver(self.model_dir) - - def test_save_secs_saves_in_first_step(self): - with self.graph.as_default(): - monitor = learn.monitors.CheckpointSaver( - self.model_dir, save_secs=2, scaffold=self.scaffold) - monitor.begin() - self.scaffold.finalize() - with session_lib.Session() as sess: - sess.run(self.scaffold.init_op) - self._run(monitor, 1, self.train_op, sess) - self.assertEqual(1, - checkpoint_utils.load_variable(self.model_dir, - self.global_step.name)) - - # TODO(gunan): Reenable this test after b/32446874 is fixed. - def disabled_test_save_secs_saves_periodically(self): - with self.graph.as_default(): - monitor = learn.monitors.CheckpointSaver( - self.model_dir, save_secs=2, scaffold=self.scaffold) - monitor.begin() - self.scaffold.finalize() - with session_lib.Session() as sess: - sess.run(self.scaffold.init_op) - self._run(monitor, 1, self.train_op, sess) - self._run(monitor, 2, self.train_op, sess) - # Not saved - self.assertEqual(1, - checkpoint_utils.load_variable(self.model_dir, - self.global_step.name)) - time.sleep(2.5) - self._run(monitor, 3, self.train_op, sess) - # saved - self.assertEqual(3, - checkpoint_utils.load_variable(self.model_dir, - self.global_step.name)) - self._run(monitor, 4, self.train_op, sess) - self._run(monitor, 5, self.train_op, sess) - # Not saved - self.assertEqual(3, - checkpoint_utils.load_variable(self.model_dir, - self.global_step.name)) - time.sleep(2.5) - self._run(monitor, 6, self.train_op, sess) - # saved - self.assertEqual(6, - checkpoint_utils.load_variable(self.model_dir, - self.global_step.name)) - - def test_save_steps_saves_in_first_step(self): - with self.graph.as_default(): - monitor = learn.monitors.CheckpointSaver( - self.model_dir, save_steps=2, scaffold=self.scaffold) - monitor.begin() - self.scaffold.finalize() - with session_lib.Session() as sess: - sess.run(self.scaffold.init_op) - self._run(monitor, 1, self.train_op, sess) - self.assertEqual(1, - checkpoint_utils.load_variable(self.model_dir, - self.global_step.name)) - - def test_save_steps_saves_periodically(self): - with self.graph.as_default(): - monitor = learn.monitors.CheckpointSaver( - self.model_dir, save_steps=2, scaffold=self.scaffold) - monitor.begin() - self.scaffold.finalize() - with session_lib.Session() as sess: - sess.run(self.scaffold.init_op) - self._run(monitor, 1, self.train_op, sess) - self._run(monitor, 2, self.train_op, sess) - # Not saved - self.assertEqual(1, - checkpoint_utils.load_variable(self.model_dir, - self.global_step.name)) - self._run(monitor, 3, self.train_op, sess) - # saved - self.assertEqual(3, - checkpoint_utils.load_variable(self.model_dir, - self.global_step.name)) - self._run(monitor, 4, self.train_op, sess) - # Not saved - self.assertEqual(3, - checkpoint_utils.load_variable(self.model_dir, - self.global_step.name)) - self._run(monitor, 5, self.train_op, sess) - # saved - self.assertEqual(5, - checkpoint_utils.load_variable(self.model_dir, - self.global_step.name)) - - def test_save_saves_at_end(self): - with self.graph.as_default(): - monitor = learn.monitors.CheckpointSaver( - self.model_dir, save_secs=2, scaffold=self.scaffold) - monitor.begin() - self.scaffold.finalize() - with session_lib.Session() as sess: - sess.run(self.scaffold.init_op) - self._run(monitor, 1, self.train_op, sess) - self._run(monitor, 2, self.train_op, sess) - monitor.end(sess) - self.assertEqual(2, - checkpoint_utils.load_variable(self.model_dir, - self.global_step.name)) - - -class FakeMonitor(learn.monitors.BaseMonitor): - - def __init__(self): - learn.monitors.BaseMonitor.__init__(self) - self.should_stop = False - self.requested_tensors = [] - self.call_counter = collections.Counter() - self.last_begin_step = None - self.last_end_step = None - self.last_post_step = None - - def begin(self, max_steps): - self.call_counter['begin'] += 1 - - def end(self, session): - self.call_counter['end'] += 1 - - def step_begin(self, step): - self.call_counter['step_begin'] += 1 - self.last_begin_step = step - return self.requested_tensors - - def step_end(self, step, output): - self.call_counter['step_end'] += 1 - self.last_end_step = step - self.output = output - return self.should_stop - - def post_step(self, step, session): - self.call_counter['post_step'] += 1 - self.last_post_step = step - self.session = session - - -class RunHookAdapterForMonitorsTest(test.TestCase): - - def test_calls_and_steps(self): - with ops.Graph().as_default(), session_lib.Session() as sess: - global_step_tensor = training_util.create_global_step() - inc_5 = state_ops.assign_add(global_step_tensor, 5) - mock_mon = FakeMonitor() - mock_mon2 = FakeMonitor() - - hook = learn.monitors.RunHookAdapterForMonitors([mock_mon, mock_mon2]) - hook.begin() - for mon in [mock_mon, mock_mon2]: - self.assertEqual(mon.call_counter['begin'], 1) - - sess.run(variables.global_variables_initializer()) - sess.run(global_step_tensor.assign(10)) - - mon_sess = monitored_session._HookedSession(sess=sess, hooks=[hook]) - - mon_sess.run(inc_5) - for mon in [mock_mon, mock_mon2]: - self.assertEqual(mon.output, {}) - self.assertEqual(mon.last_begin_step, 11) - self.assertEqual(mon.last_end_step, 11) - self.assertEqual(mon.last_post_step, 11) - self.assertEqual(mon.call_counter['step_end'], 1) - self.assertEqual(mon.call_counter['step_begin'], 1) - self.assertEqual(mon.call_counter['post_step'], 1) - - mon_sess.run(inc_5) - for mon in [mock_mon, mock_mon2]: - self.assertEqual(mon.output, {}) - self.assertEqual(mon.last_begin_step, 16) - self.assertEqual(mon.last_end_step, 16) - self.assertEqual(mon.last_post_step, 16) - self.assertEqual(mon.call_counter['step_end'], 2) - self.assertEqual(mon.call_counter['step_begin'], 2) - self.assertEqual(mon.call_counter['post_step'], 2) - - hook.end(sess) - for mon in [mock_mon, mock_mon2]: - self.assertEqual(mon.call_counter['end'], 1) - - def test_requests(self): - with ops.Graph().as_default(), session_lib.Session() as sess: - training_util.create_global_step() - mock_mon = FakeMonitor() - mock_mon2 = FakeMonitor() - - hook = learn.monitors.RunHookAdapterForMonitors([mock_mon, mock_mon2]) - hook.begin() - - mon_sess = monitored_session._HookedSession(sess=sess, hooks=[hook]) - - a_tensor = constant_op.constant([0], name='a_tensor') - constant_op.constant([5], name='another_tensor') - constant_op.constant([10], name='third_tensor') - mock_mon.requested_tensors = ['another_tensor'] - mock_mon2.requested_tensors = ['third_tensor'] - sess.run(variables.global_variables_initializer()) - - output = mon_sess.run(a_tensor) - self.assertEqual(output, [0]) - self.assertEqual(mock_mon.output['another_tensor'], [5]) - self.assertEqual(mock_mon2.output['third_tensor'], [10]) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/ops/__init__.py b/tensorflow/contrib/learn/python/learn/ops/__init__.py deleted file mode 100644 index efb1f47cf5b..00000000000 --- a/tensorflow/contrib/learn/python/learn/ops/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Various TensorFlow Ops (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.learn.python.learn.ops.embeddings_ops import * -from tensorflow.contrib.learn.python.learn.ops.losses_ops import * -from tensorflow.contrib.learn.python.learn.ops.seq2seq_ops import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py b/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py deleted file mode 100644 index 8f9811cf251..00000000000 --- a/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""TensorFlow Ops to work with embeddings (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. - -Note: categorical variables are handled via embeddings in many cases. -For example, in case of words. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework import deprecated -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops as array_ops_ -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import variable_scope as vs - - -@deprecated('2016-12-01', 'Use `tf.embedding_lookup` instead.') -def embedding_lookup(params, ids, name='embedding_lookup'): - """Provides a N dimensional version of tf.embedding_lookup. - - Ids are flattened to a 1d tensor before being passed to embedding_lookup - then, they are unflattend to match the original ids shape plus an extra - leading dimension of the size of the embeddings. - - Args: - params: List of tensors of size D0 x D1 x ... x Dn-2 x Dn-1. - ids: N-dimensional tensor of B0 x B1 x .. x Bn-2 x Bn-1. - Must contain indexes into params. - name: Optional name for the op. - - Returns: - A tensor of size B0 x B1 x .. x Bn-2 x Bn-1 x D1 x ... x Dn-2 x Dn-1 - containing the values from the params tensor(s) for indecies in ids. - - Raises: - ValueError: if some parameters are invalid. - """ - with ops.name_scope(name, 'embedding_lookup', [params, ids]): - params = ops.convert_to_tensor(params) - ids = ops.convert_to_tensor(ids) - shape = array_ops_.shape(ids) - ids_flat = array_ops_.reshape( - ids, math_ops.reduce_prod(shape, keepdims=True)) - embeds_flat = nn.embedding_lookup(params, ids_flat, name) - embed_shape = array_ops_.concat([shape, [-1]], 0) - embeds = array_ops_.reshape(embeds_flat, embed_shape) - embeds.set_shape(ids.get_shape().concatenate(params.get_shape()[1:])) - return embeds - - -@deprecated('2016-12-01', 'Use `tf.contrib.layers.embed_sequence` instead.') -def categorical_variable(tensor_in, n_classes, embedding_size, name): - """Creates an embedding for categorical variable with given number of classes. - - Args: - tensor_in: Input tensor with class identifier (can be batch or - N-dimensional). - n_classes: Number of classes. - embedding_size: Size of embedding vector to represent each class. - name: Name of this categorical variable. - Returns: - Tensor of input shape, with additional dimension for embedding. - - Example: - Calling categorical_variable([1, 2], 5, 10, "my_cat"), will return 2 x 10 - tensor, where each row is representation of the class. - """ - with vs.variable_scope(name): - embeddings = vs.get_variable(name + '_embeddings', - [n_classes, embedding_size]) - return embedding_lookup(embeddings, tensor_in) diff --git a/tensorflow/contrib/learn/python/learn/ops/losses_ops.py b/tensorflow/contrib/learn/python/learn/ops/losses_ops.py deleted file mode 100644 index 9f2cadb0174..00000000000 --- a/tensorflow/contrib/learn/python/learn/ops/losses_ops.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""TensorFlow Ops for loss computation (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework import deprecated -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops as array_ops_ -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops.losses import losses - - -@deprecated('2016-12-01', 'Use `tf.losses.mean_squared_error` ' - 'and explicit logits computation.') -def mean_squared_error_regressor(tensor_in, labels, weights, biases, name=None): - """Returns prediction and loss for mean squared error regression.""" - with ops.name_scope(name, 'mean_squared_error_regressor', - [tensor_in, labels]): - predictions = nn.xw_plus_b(tensor_in, weights, biases) - if len(labels.get_shape()) == 1 and len(predictions.get_shape()) == 2: - predictions = array_ops_.squeeze(predictions, axis=[1]) - return predictions, losses.mean_squared_error(labels, predictions) - - -@deprecated('2016-12-01', 'Use `tf.losses.softmax_cross_entropy` ' - 'and explicit logits computation.') -def softmax_classifier(tensor_in, - labels, - weights, - biases, - class_weight=None, - name=None): - """Returns prediction and loss for softmax classifier. - - This function returns "probabilities" and a cross entropy loss. To obtain - predictions, use `tf.argmax` on the returned probabilities. - - This function requires labels to be passed in one-hot encoding. - - Args: - tensor_in: Input tensor, [batch_size, feature_size], features. - labels: Tensor, [batch_size, n_classes], one-hot labels of the output - classes. - weights: Tensor, [batch_size, feature_size], linear transformation - matrix. - biases: Tensor, [batch_size], biases. - class_weight: Tensor, optional, [n_classes], weight for each class. - If not given, all classes are supposed to have weight one. - name: Operation name. - - Returns: - `tuple` of softmax predictions and loss `Tensor`s. - """ - with ops.name_scope(name, 'softmax_classifier', [tensor_in, labels]): - logits = nn.xw_plus_b(tensor_in, weights, biases) - if class_weight is not None: - logits = math_ops.multiply(logits, class_weight) - return nn.softmax(logits), losses.softmax_cross_entropy(labels, logits) diff --git a/tensorflow/contrib/learn/python/learn/ops/ops_test.py b/tensorflow/contrib/learn/python/learn/ops/ops_test.py deleted file mode 100644 index ff190110c14..00000000000 --- a/tensorflow/contrib/learn/python/learn/ops/ops_test.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.learn.python.learn import ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import variables -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class OpsTest(test.TestCase): - """Ops tests.""" - - def test_softmax_classifier(self): - with self.cached_session() as session: - features = array_ops.placeholder(dtypes.float32, [None, 3]) - labels = array_ops.placeholder(dtypes.float32, [None, 2]) - weights = constant_op.constant([[0.1, 0.1], [0.1, 0.1], [0.1, 0.1]]) - biases = constant_op.constant([0.2, 0.3]) - class_weight = constant_op.constant([0.1, 0.9]) - prediction, loss = ops.softmax_classifier(features, labels, weights, - biases, class_weight) - self.assertEqual(prediction.get_shape()[1], 2) - self.assertEqual(loss.get_shape(), []) - value = session.run(loss, {features: [[0.2, 0.3, 0.2]], labels: [[0, 1]]}) - self.assertAllClose(value, 0.55180627) - - def test_embedding_lookup(self): - d_embed = 5 - n_embed = 10 - ids_shape = (2, 3, 4) - embeds = np.random.randn(n_embed, d_embed) - ids = np.random.randint(0, n_embed, ids_shape) - with self.cached_session(): - embed_np = embeds[ids] - embed_tf = ops.embedding_lookup(embeds, ids).eval() - self.assertEqual(embed_np.shape, embed_tf.shape) - self.assertAllClose(embed_np, embed_tf) - - def test_categorical_variable(self): - random_seed.set_random_seed(42) - with self.cached_session() as sess: - cat_var_idx = array_ops.placeholder(dtypes.int64, [2, 2]) - embeddings = ops.categorical_variable( - cat_var_idx, n_classes=5, embedding_size=10, name="my_cat_var") - sess.run(variables.global_variables_initializer()) - emb1 = sess.run(embeddings, - feed_dict={cat_var_idx.name: [[0, 1], [2, 3]]}) - emb2 = sess.run(embeddings, - feed_dict={cat_var_idx.name: [[0, 2], [1, 3]]}) - self.assertEqual(emb1.shape, emb2.shape) - self.assertAllEqual(np.transpose(emb2, axes=[1, 0, 2]), emb1) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/ops/seq2seq_ops.py b/tensorflow/contrib/learn/python/learn/ops/seq2seq_ops.py deleted file mode 100644 index aa37cb4a76e..00000000000 --- a/tensorflow/contrib/learn/python/learn/ops/seq2seq_ops.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""TensorFlow Ops for Sequence to Sequence models (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import rnn -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.util.deprecation import deprecated - - -@deprecated(None, 'Please use tf.nn/tf.layers directly.') -def sequence_classifier(decoding, labels, sampling_decoding=None, name=None): - """Returns predictions and loss for sequence of predictions. - - Args: - decoding: List of Tensors with predictions. - labels: List of Tensors with labels. - sampling_decoding: Optional, List of Tensor with predictions to be used - in sampling. E.g. they shouldn't have dependncy on outputs. - If not provided, decoding is used. - name: Operation name. - - Returns: - Predictions and losses tensors. - """ - with ops.name_scope(name, "sequence_classifier", [decoding, labels]): - predictions, xent_list = [], [] - for i, pred in enumerate(decoding): - xent_list.append(nn.softmax_cross_entropy_with_logits( - labels=labels[i], logits=pred, - name="sequence_loss/xent_raw{0}".format(i))) - if sampling_decoding: - predictions.append(nn.softmax(sampling_decoding[i])) - else: - predictions.append(nn.softmax(pred)) - xent = math_ops.add_n(xent_list, name="sequence_loss/xent") - loss = math_ops.reduce_sum(xent, name="sequence_loss") - return array_ops.stack(predictions, axis=1), loss - - -@deprecated(None, 'Please use tf.nn/tf.layers directly.') -def seq2seq_inputs(x, y, input_length, output_length, sentinel=None, name=None): - """Processes inputs for Sequence to Sequence models. - - Args: - x: Input Tensor [batch_size, input_length, embed_dim]. - y: Output Tensor [batch_size, output_length, embed_dim]. - input_length: length of input x. - output_length: length of output y. - sentinel: optional first input to decoder and final output expected. - If sentinel is not provided, zeros are used. Due to fact that y is not - available in sampling time, shape of sentinel will be inferred from x. - name: Operation name. - - Returns: - Encoder input from x, and decoder inputs and outputs from y. - """ - with ops.name_scope(name, "seq2seq_inputs", [x, y]): - in_x = array_ops.unstack(x, axis=1) - y = array_ops.unstack(y, axis=1) - if not sentinel: - # Set to zeros of shape of y[0], using x for batch size. - sentinel_shape = array_ops.stack( - [array_ops.shape(x)[0], y[0].get_shape()[1]]) - sentinel = array_ops.zeros(sentinel_shape) - sentinel.set_shape(y[0].get_shape()) - in_y = [sentinel] + y - out_y = y + [sentinel] - return in_x, in_y, out_y - - -@deprecated(None, 'Please use tf.nn/tf.layers directly.') -def rnn_decoder(decoder_inputs, initial_state, cell, scope=None): - """RNN Decoder that creates training and sampling sub-graphs. - - Args: - decoder_inputs: Inputs for decoder, list of tensors. - This is used only in training sub-graph. - initial_state: Initial state for the decoder. - cell: RNN cell to use for decoder. - scope: Scope to use, if None new will be produced. - - Returns: - List of tensors for outputs and states for training and sampling sub-graphs. - """ - with vs.variable_scope(scope or "dnn_decoder"): - states, sampling_states = [initial_state], [initial_state] - outputs, sampling_outputs = [], [] - with ops.name_scope("training", values=[decoder_inputs, initial_state]): - for i, inp in enumerate(decoder_inputs): - if i > 0: - vs.get_variable_scope().reuse_variables() - output, new_state = cell(inp, states[-1]) - outputs.append(output) - states.append(new_state) - with ops.name_scope("sampling", values=[initial_state]): - for i, _ in enumerate(decoder_inputs): - if i == 0: - sampling_outputs.append(outputs[i]) - sampling_states.append(states[i]) - else: - sampling_output, sampling_state = cell(sampling_outputs[-1], - sampling_states[-1]) - sampling_outputs.append(sampling_output) - sampling_states.append(sampling_state) - return outputs, states, sampling_outputs, sampling_states - - -@deprecated(None, 'Please use tf.nn/tf.layers directly.') -def rnn_seq2seq(encoder_inputs, - decoder_inputs, - encoder_cell, - decoder_cell=None, - dtype=dtypes.float32, - scope=None): - """RNN Sequence to Sequence model. - - Args: - encoder_inputs: List of tensors, inputs for encoder. - decoder_inputs: List of tensors, inputs for decoder. - encoder_cell: RNN cell to use for encoder. - decoder_cell: RNN cell to use for decoder, if None encoder_cell is used. - dtype: Type to initialize encoder state with. - scope: Scope to use, if None new will be produced. - - Returns: - List of tensors for outputs and states for training and sampling sub-graphs. - """ - with vs.variable_scope(scope or "rnn_seq2seq"): - _, last_enc_state = rnn.static_rnn( - encoder_cell, encoder_inputs, dtype=dtype) - return rnn_decoder(decoder_inputs, last_enc_state, decoder_cell or - encoder_cell) diff --git a/tensorflow/contrib/learn/python/learn/ops/seq2seq_ops_test.py b/tensorflow/contrib/learn/python/learn/ops/seq2seq_ops_test.py deleted file mode 100644 index 5a7e4ebfeae..00000000000 --- a/tensorflow/contrib/learn/python/learn/ops/seq2seq_ops_test.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Sequence-to-sequence tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.learn.python.learn import ops -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import rnn_cell -from tensorflow.python.platform import test - - -class Seq2SeqOpsTest(test.TestCase): - """Sequence-to-sequence tests.""" - - def test_sequence_classifier(self): - with self.cached_session() as session: - decoding = [ - array_ops.placeholder(dtypes.float32, [2, 2]) for _ in range(3) - ] - labels = [array_ops.placeholder(dtypes.float32, [2, 2]) for _ in range(3)] - sampling_decoding = [ - array_ops.placeholder(dtypes.float32, [2, 2]) for _ in range(3) - ] - predictions, loss = ops.sequence_classifier(decoding, labels, - sampling_decoding) - pred, cost = session.run( - [predictions, loss], - feed_dict={ - decoding[0].name: [[0.1, 0.9], [0.7, 0.3]], - decoding[1].name: [[0.9, 0.1], [0.8, 0.2]], - decoding[2].name: [[0.5, 0.5], [0.4, 0.6]], - labels[0].name: [[1, 0], [0, 1]], - labels[1].name: [[1, 0], [0, 1]], - labels[2].name: [[1, 0], [0, 1]], - sampling_decoding[0].name: [[0.1, 0.9], [0.7, 0.3]], - sampling_decoding[1].name: [[0.9, 0.1], [0.8, 0.2]], - sampling_decoding[2].name: [[0.5, 0.5], [0.4, 0.6]], - }) - self.assertAllEqual(pred.argmax(axis=2), [[1, 0, 0], [0, 0, 1]]) - self.assertAllClose(cost, 4.7839908599) - - def test_seq2seq_inputs(self): - inp = np.array([[[1, 0], [0, 1], [1, 0]], [[0, 1], [1, 0], [0, 1]]]) - out = np.array([[[0, 1, 0], [1, 0, 0]], [[1, 0, 0], [0, 1, 0]]]) - with self.cached_session() as session: - x = array_ops.placeholder(dtypes.float32, [2, 3, 2]) - y = array_ops.placeholder(dtypes.float32, [2, 2, 3]) - in_x, in_y, out_y = ops.seq2seq_inputs(x, y, 3, 2) - enc_inp = session.run(in_x, feed_dict={x.name: inp}) - dec_inp = session.run(in_y, feed_dict={x.name: inp, y.name: out}) - dec_out = session.run(out_y, feed_dict={x.name: inp, y.name: out}) - # Swaps from batch x len x height to list of len of batch x height. - self.assertAllEqual(enc_inp, np.swapaxes(inp, 0, 1)) - self.assertAllEqual(dec_inp, [[[0, 0, 0], [0, 0, 0]], - [[0, 1, 0], [1, 0, 0]], - [[1, 0, 0], [0, 1, 0]]]) - self.assertAllEqual(dec_out, [[[0, 1, 0], [1, 0, 0]], - [[1, 0, 0], [0, 1, 0]], - [[0, 0, 0], [0, 0, 0]]]) - - def test_rnn_decoder(self): - with self.cached_session(): - decoder_inputs = [ - array_ops.placeholder(dtypes.float32, [2, 2]) for _ in range(3) - ] - encoding = array_ops.placeholder(dtypes.float32, [2, 2]) - cell = rnn_cell.GRUCell(2) - outputs, states, sampling_outputs, sampling_states = ( - ops.rnn_decoder(decoder_inputs, encoding, cell)) - self.assertEqual(len(outputs), 3) - self.assertEqual(outputs[0].get_shape(), [2, 2]) - self.assertEqual(len(states), 4) - self.assertEqual(states[0].get_shape(), [2, 2]) - self.assertEqual(len(sampling_outputs), 3) - self.assertEqual(sampling_outputs[0].get_shape(), [2, 2]) - self.assertEqual(len(sampling_states), 4) - self.assertEqual(sampling_states[0].get_shape(), [2, 2]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/preprocessing/__init__.py b/tensorflow/contrib/learn/python/learn/preprocessing/__init__.py deleted file mode 100644 index e8c6e1acf80..00000000000 --- a/tensorflow/contrib/learn/python/learn/preprocessing/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Preprocessing tools useful for building models (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.learn.python.learn.preprocessing.categorical import * -from tensorflow.contrib.learn.python.learn.preprocessing.text import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/learn/python/learn/preprocessing/categorical.py b/tensorflow/contrib/learn/python/learn/preprocessing/categorical.py deleted file mode 100644 index faba3b2025e..00000000000 --- a/tensorflow/contrib/learn/python/learn/preprocessing/categorical.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Implements preprocessing transformers for categorical variables (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import numpy as np - -from tensorflow.python.util.deprecation import deprecated - -# pylint: disable=g-bad-import-order -from . import categorical_vocabulary -from ..learn_io.data_feeder import setup_processor_data_feeder -# pylint: enable=g-bad-import-order - - -class CategoricalProcessor(object): - """Maps documents to sequences of word ids. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - As a common convention, Nan values are handled as unknown tokens. - Both float('nan') and np.nan are accepted. - """ - - @deprecated(None, 'Please use tensorflow/transform or tf.data for sequence ' - 'processing.') - def __init__(self, min_frequency=0, share=False, vocabularies=None): - """Initializes a CategoricalProcessor instance. - - Args: - min_frequency: Minimum frequency of categories in the vocabulary. - share: Share vocabulary between variables. - vocabularies: list of CategoricalVocabulary objects for each variable in - the input dataset. - - Attributes: - vocabularies_: list of CategoricalVocabulary objects. - """ - self.min_frequency = min_frequency - self.share = share - self.vocabularies_ = vocabularies - - def freeze(self, freeze=True): - """Freeze or unfreeze all vocabularies. - - Args: - freeze: Boolean, indicate if vocabularies should be frozen. - """ - for vocab in self.vocabularies_: - vocab.freeze(freeze) - - def fit(self, x, unused_y=None): - """Learn a vocabulary dictionary of all categories in `x`. - - Args: - x: numpy matrix or iterable of lists/numpy arrays. - unused_y: to match fit format signature of estimators. - - Returns: - self - """ - x = setup_processor_data_feeder(x) - for row in x: - # Create vocabularies if not given. - if self.vocabularies_ is None: - # If not share, one per column, else one shared across. - if not self.share: - self.vocabularies_ = [ - categorical_vocabulary.CategoricalVocabulary() for _ in row - ] - else: - vocab = categorical_vocabulary.CategoricalVocabulary() - self.vocabularies_ = [vocab for _ in row] - for idx, value in enumerate(row): - # Nans are handled as unknowns. - if (isinstance(value, float) and math.isnan(value)) or value == np.nan: - continue - self.vocabularies_[idx].add(value) - if self.min_frequency > 0: - for vocab in self.vocabularies_: - vocab.trim(self.min_frequency) - self.freeze() - return self - - def fit_transform(self, x, unused_y=None): - """Learn the vocabulary dictionary and return indexies of categories. - - Args: - x: numpy matrix or iterable of lists/numpy arrays. - unused_y: to match fit_transform signature of estimators. - - Returns: - x: iterable, [n_samples]. Category-id matrix. - """ - self.fit(x) - return self.transform(x) - - def transform(self, x): - """Transform documents to category-id matrix. - - Converts categories to ids give fitted vocabulary from `fit` or - one provided in the constructor. - - Args: - x: numpy matrix or iterable of lists/numpy arrays. - - Yields: - x: iterable, [n_samples]. Category-id matrix. - """ - self.freeze() - x = setup_processor_data_feeder(x) - for row in x: - output_row = [] - for idx, value in enumerate(row): - # Return when it's Nan. - if (isinstance(value, float) and math.isnan(value)) or value == np.nan: - output_row.append(0) - continue - output_row.append(self.vocabularies_[idx].get(value)) - yield np.array(output_row, dtype=np.int64) diff --git a/tensorflow/contrib/learn/python/learn/preprocessing/categorical_vocabulary.py b/tensorflow/contrib/learn/python/learn/preprocessing/categorical_vocabulary.py deleted file mode 100644 index 3ac370a6ab4..00000000000 --- a/tensorflow/contrib/learn/python/learn/preprocessing/categorical_vocabulary.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Categorical vocabulary classes to map categories to indexes (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. - -Can be used for categorical variables, sparse variables and words. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import six - -from tensorflow.python.util.deprecation import deprecated - - -class CategoricalVocabulary(object): - """Categorical variables vocabulary class. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - - Accumulates and provides mapping from classes to indexes. - Can be easily used for words. - """ - - @deprecated(None, 'Please use tensorflow/transform or tf.data.') - def __init__(self, unknown_token="", support_reverse=True): - self._unknown_token = unknown_token - self._mapping = {unknown_token: 0} - self._support_reverse = support_reverse - if support_reverse: - self._reverse_mapping = [unknown_token] - self._freq = collections.defaultdict(int) - self._freeze = False - - def __len__(self): - """Returns total count of mappings. Including unknown token.""" - return len(self._mapping) - - def freeze(self, freeze=True): - """Freezes the vocabulary, after which new words return unknown token id. - - Args: - freeze: True to freeze, False to unfreeze. - """ - self._freeze = freeze - - def get(self, category): - """Returns word's id in the vocabulary. - - If category is new, creates a new id for it. - - Args: - category: string or integer to lookup in vocabulary. - - Returns: - interger, id in the vocabulary. - """ - if category not in self._mapping: - if self._freeze: - return 0 - self._mapping[category] = len(self._mapping) - if self._support_reverse: - self._reverse_mapping.append(category) - return self._mapping[category] - - def add(self, category, count=1): - """Adds count of the category to the frequency table. - - Args: - category: string or integer, category to add frequency to. - count: optional integer, how many to add. - """ - category_id = self.get(category) - if category_id <= 0: - return - self._freq[category] += count - - def trim(self, min_frequency, max_frequency=-1): - """Trims vocabulary for minimum frequency. - - Remaps ids from 1..n in sort frequency order. - where n - number of elements left. - - Args: - min_frequency: minimum frequency to keep. - max_frequency: optional, maximum frequency to keep. - Useful to remove very frequent categories (like stop words). - """ - # Sort by alphabet then reversed frequency. - self._freq = sorted( - sorted( - six.iteritems(self._freq), - key=lambda x: (isinstance(x[0], str), x[0])), - key=lambda x: x[1], - reverse=True) - self._mapping = {self._unknown_token: 0} - if self._support_reverse: - self._reverse_mapping = [self._unknown_token] - idx = 1 - for category, count in self._freq: - if max_frequency > 0 and count >= max_frequency: - continue - if count <= min_frequency: - break - self._mapping[category] = idx - idx += 1 - if self._support_reverse: - self._reverse_mapping.append(category) - self._freq = dict(self._freq[:idx - 1]) - - def reverse(self, class_id): - """Given class id reverse to original class name. - - Args: - class_id: Id of the class. - - Returns: - Class name. - - Raises: - ValueError: if this vocabulary wasn't initialized with support_reverse. - """ - if not self._support_reverse: - raise ValueError("This vocabulary wasn't initialized with " - "support_reverse to support reverse() function.") - return self._reverse_mapping[class_id] diff --git a/tensorflow/contrib/learn/python/learn/preprocessing/tests/__init__.py b/tensorflow/contrib/learn/python/learn/preprocessing/tests/__init__.py deleted file mode 100644 index 4a059f1e4c8..00000000000 --- a/tensorflow/contrib/learn/python/learn/preprocessing/tests/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Preprocessing tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/learn/python/learn/preprocessing/tests/categorical_test.py b/tensorflow/contrib/learn/python/learn/preprocessing/tests/categorical_test.py deleted file mode 100644 index 4e9cb9df62e..00000000000 --- a/tensorflow/contrib/learn/python/learn/preprocessing/tests/categorical_test.py +++ /dev/null @@ -1,56 +0,0 @@ -# encoding: utf-8 -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Categorical tests.""" - -# limitations under the License. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.learn.python.learn.learn_io import HAS_PANDAS -from tensorflow.contrib.learn.python.learn.preprocessing import categorical -from tensorflow.python.platform import test - - -class CategoricalTest(test.TestCase): - """Categorical tests.""" - - def testSingleCategoricalProcessor(self): - cat_processor = categorical.CategoricalProcessor(min_frequency=1) - x = cat_processor.fit_transform([["0"], [1], [float("nan")], ["C"], ["C"], - [1], ["0"], [np.nan], [3]]) - self.assertAllEqual(list(x), [[2], [1], [0], [3], [3], [1], [2], [0], [0]]) - - def testSingleCategoricalProcessorPandasSingleDF(self): - if HAS_PANDAS: - import pandas as pd # pylint: disable=g-import-not-at-top - cat_processor = categorical.CategoricalProcessor() - data = pd.DataFrame({"Gender": ["Male", "Female", "Male"]}) - x = list(cat_processor.fit_transform(data)) - self.assertAllEqual(list(x), [[1], [2], [1]]) - - def testMultiCategoricalProcessor(self): - cat_processor = categorical.CategoricalProcessor( - min_frequency=0, share=False) - x = cat_processor.fit_transform([["0", "Male"], [1, "Female"], - ["3", "Male"]]) - self.assertAllEqual(list(x), [[1, 1], [2, 2], [3, 1]]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/preprocessing/tests/categorical_vocabulary_test.py b/tensorflow/contrib/learn/python/learn/preprocessing/tests/categorical_vocabulary_test.py deleted file mode 100644 index 54e90ddd704..00000000000 --- a/tensorflow/contrib/learn/python/learn/preprocessing/tests/categorical_vocabulary_test.py +++ /dev/null @@ -1,64 +0,0 @@ -# encoding: utf-8 -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Categorical vocabulary tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.preprocessing import categorical_vocabulary -from tensorflow.python.platform import test - - -class CategoricalVocabularyTest(test.TestCase): - """Categorical vocabulary tests.""" - - def testIntVocabulary(self): - vocab = categorical_vocabulary.CategoricalVocabulary() - self.assertEqual(vocab.get(1), 1) - self.assertEqual(vocab.get(3), 2) - self.assertEqual(vocab.get(2), 3) - self.assertEqual(vocab.get(3), 2) - # This vocab doesn't handle nan specially. - self.assertEqual(vocab.get(float('nan')), 4) - self.assertEqual(len(vocab), 5) - - def testWordVocabulary(self): - vocab = categorical_vocabulary.CategoricalVocabulary() - self.assertEqual(vocab.get('a'), 1) - self.assertEqual(vocab.get('b'), 2) - self.assertEqual(vocab.get('a'), 1) - self.assertEqual(vocab.get('b'), 2) - - def testCountsTrim(self): - vocab = categorical_vocabulary.CategoricalVocabulary() - vocab.get('c') - vocab.add('c', 5) - vocab.get('a') - vocab.add('a', 10) - # not in vocab yet, skips. - vocab.add('b', 5) - vocab.add('d', 12) - vocab.trim(7, 11) - vocab.freeze() - self.assertEqual(vocab.get('b'), 0) - self.assertEqual(vocab.get('c'), 0) - self.assertEqual(len(vocab), 2) - self.assertEqual(vocab.get('a'), 1) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/preprocessing/tests/text_test.py b/tensorflow/contrib/learn/python/learn/preprocessing/tests/text_test.py deleted file mode 100644 index e9555140d08..00000000000 --- a/tensorflow/contrib/learn/python/learn/preprocessing/tests/text_test.py +++ /dev/null @@ -1,79 +0,0 @@ -# encoding: utf-8 -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Text processor tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from tensorflow.contrib.learn.python.learn.preprocessing import CategoricalVocabulary -from tensorflow.contrib.learn.python.learn.preprocessing import text -from tensorflow.python.platform import test - - -class TextTest(test.TestCase): - """Text processor tests.""" - - def testTokenizer(self): - words = text.tokenizer( - ["a b c", "a\nb\nc", "a, b - c", "фыв выф", "你好 怎么样"]) - self.assertEqual( - list(words), [["a", "b", "c"], ["a", "b", "c"], ["a", "b", "-", "c"], - ["фыв", "выф"], ["你好", "怎么样"]]) - - def testByteProcessor(self): - processor = text.ByteProcessor(max_document_length=8) - inp = ["abc", "фыва", "фыва", b"abc", "12345678901234567890"] - res = list(processor.fit_transform(inp)) - self.assertAllEqual(res, [[97, 98, 99, 0, 0, 0, 0, 0], - [209, 132, 209, 139, 208, 178, 208, 176], - [209, 132, 209, 139, 208, 178, 208, 176], - [97, 98, 99, 0, 0, 0, 0, 0], - [49, 50, 51, 52, 53, 54, 55, 56]]) - res = list(processor.reverse(res)) - self.assertAllEqual(res, ["abc", "фыва", "фыва", "abc", "12345678"]) - - def testVocabularyProcessor(self): - vocab_processor = text.VocabularyProcessor( - max_document_length=4, min_frequency=1) - tokens = vocab_processor.fit_transform(["a b c", "a\nb\nc", "a, b - c"]) - self.assertAllEqual( - list(tokens), [[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 3]]) - - def testVocabularyProcessorSaveRestore(self): - filename = test.get_temp_dir() + "test.vocab" - vocab_processor = text.VocabularyProcessor( - max_document_length=4, min_frequency=1) - tokens = vocab_processor.fit_transform(["a b c", "a\nb\nc", "a, b - c"]) - vocab_processor.save(filename) - new_vocab = text.VocabularyProcessor.restore(filename) - tokens = new_vocab.transform(["a b c"]) - self.assertAllEqual(list(tokens), [[1, 2, 3, 0]]) - - def testExistingVocabularyProcessor(self): - vocab = CategoricalVocabulary() - vocab.get("A") - vocab.get("B") - vocab.freeze() - vocab_processor = text.VocabularyProcessor( - max_document_length=4, vocabulary=vocab, tokenizer_fn=list) - tokens = vocab_processor.fit_transform(["ABC", "CBABAF"]) - self.assertAllEqual(list(tokens), [[1, 2, 0, 0], [0, 2, 1, 2]]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/preprocessing/text.py b/tensorflow/contrib/learn/python/learn/preprocessing/text.py deleted file mode 100644 index f2b6776be77..00000000000 --- a/tensorflow/contrib/learn/python/learn/preprocessing/text.py +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Implements a number of text preprocessing utilities (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re -import numpy as np -import six - -from tensorflow.python.platform import gfile -from tensorflow.python.util.deprecation import deprecated - -from .categorical_vocabulary import CategoricalVocabulary # pylint: disable=g-bad-import-order - -try: - # pylint: disable=g-import-not-at-top - import cPickle as pickle -except ImportError: - # pylint: disable=g-import-not-at-top - import pickle - -TOKENIZER_RE = re.compile(r"[A-Z]{2,}(?![a-z])|[A-Z][a-z]+(?=[A-Z])|[\'\w\-]+", - re.UNICODE) - - -@deprecated(None, 'Please use tensorflow/transform or tf.data.') -def tokenizer(iterator): - """Tokenizer generator. - - Args: - iterator: Input iterator with strings. - - Yields: - array of tokens per each value in the input. - """ - for value in iterator: - yield TOKENIZER_RE.findall(value) - - -@deprecated(None, 'Please use tensorflow/transform or tf.data.') -class ByteProcessor(object): - """Maps documents into sequence of ids for bytes. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - @deprecated(None, 'Please use tensorflow/transform or tf.data.') - def __init__(self, max_document_length): - self.max_document_length = max_document_length - - def fit(self, x): - """Does nothing. No fitting required.""" - pass - - def fit_transform(self, x): - """Calls transform.""" - return self.transform(x) - - # pylint: disable=no-self-use - def reverse(self, x): - """Reverses output of transform back to text. - - Args: - x: iterator or matrix of integers. Document representation in bytes. - - Yields: - Iterators of utf-8 strings. - """ - for data in x: - document = np.trim_zeros(data.astype(np.int8), trim='b').tostring() - try: - yield document.decode('utf-8') - except UnicodeDecodeError: - yield '' - - def transform(self, x): - """Transforms input documents into sequence of ids. - - Args: - x: iterator or list of input documents. - Documents can be bytes or unicode strings, which will be encoded as - utf-8 to map to bytes. Note, in Python2 str and bytes is the same type. - - Yields: - iterator of byte ids. - """ - if six.PY3: - # For Python3 defined buffer as memoryview. - buffer_or_memoryview = memoryview - else: - buffer_or_memoryview = buffer # pylint: disable=undefined-variable - for document in x: - if isinstance(document, six.text_type): - document = document.encode('utf-8') - document_mv = buffer_or_memoryview(document) - buff = np.frombuffer(document_mv[:self.max_document_length], - dtype=np.uint8) - yield np.pad(buff, (0, self.max_document_length - len(buff)), 'constant') - - -class VocabularyProcessor(object): - """Maps documents to sequences of word ids. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - @deprecated(None, 'Please use tensorflow/transform or tf.data.') - def __init__(self, - max_document_length, - min_frequency=0, - vocabulary=None, - tokenizer_fn=None): - """Initializes a VocabularyProcessor instance. - - Args: - max_document_length: Maximum length of documents. - if documents are longer, they will be trimmed, if shorter - padded. - min_frequency: Minimum frequency of words in the vocabulary. - vocabulary: CategoricalVocabulary object. - - Attributes: - vocabulary_: CategoricalVocabulary object. - """ - self.max_document_length = max_document_length - self.min_frequency = min_frequency - if vocabulary: - self.vocabulary_ = vocabulary - else: - self.vocabulary_ = CategoricalVocabulary() - if tokenizer_fn: - self._tokenizer = tokenizer_fn - else: - self._tokenizer = tokenizer - - def fit(self, raw_documents, unused_y=None): - """Learn a vocabulary dictionary of all tokens in the raw documents. - - Args: - raw_documents: An iterable which yield either str or unicode. - unused_y: to match fit format signature of estimators. - - Returns: - self - """ - for tokens in self._tokenizer(raw_documents): - for token in tokens: - self.vocabulary_.add(token) - if self.min_frequency > 0: - self.vocabulary_.trim(self.min_frequency) - self.vocabulary_.freeze() - return self - - def fit_transform(self, raw_documents, unused_y=None): - """Learn the vocabulary dictionary and return indexies of words. - - Args: - raw_documents: An iterable which yield either str or unicode. - unused_y: to match fit_transform signature of estimators. - - Returns: - x: iterable, [n_samples, max_document_length]. Word-id matrix. - """ - self.fit(raw_documents) - return self.transform(raw_documents) - - def transform(self, raw_documents): - """Transform documents to word-id matrix. - - Convert words to ids with vocabulary fitted with fit or the one - provided in the constructor. - - Args: - raw_documents: An iterable which yield either str or unicode. - - Yields: - x: iterable, [n_samples, max_document_length]. Word-id matrix. - """ - for tokens in self._tokenizer(raw_documents): - word_ids = np.zeros(self.max_document_length, np.int64) - for idx, token in enumerate(tokens): - if idx >= self.max_document_length: - break - word_ids[idx] = self.vocabulary_.get(token) - yield word_ids - - def reverse(self, documents): - """Reverses output of vocabulary mapping to words. - - Args: - documents: iterable, list of class ids. - - Yields: - Iterator over mapped in words documents. - """ - for item in documents: - output = [] - for class_id in item: - output.append(self.vocabulary_.reverse(class_id)) - yield ' '.join(output) - - def save(self, filename): - """Saves vocabulary processor into given file. - - Args: - filename: Path to output file. - """ - with gfile.Open(filename, 'wb') as f: - f.write(pickle.dumps(self)) - - @classmethod - def restore(cls, filename): - """Restores vocabulary processor from given file. - - Args: - filename: Path to file to load from. - - Returns: - VocabularyProcessor object. - """ - with gfile.Open(filename, 'rb') as f: - return pickle.loads(f.read()) diff --git a/tensorflow/contrib/learn/python/learn/session_run_hook.py b/tensorflow/contrib/learn/python/learn/session_run_hook.py deleted file mode 100644 index 87edc9b720b..00000000000 --- a/tensorflow/contrib/learn/python/learn/session_run_hook.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""This file is deprecated. Use `tensorflow.python.training.session_run_hook`. - -See [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.training import session_run_hook - -# pylint: disable=invalid-name -SessionRunHook = session_run_hook.SessionRunHook -SessionRunArgs = session_run_hook.SessionRunArgs -SessionRunContext = session_run_hook.SessionRunContext -SessionRunValues = session_run_hook.SessionRunValues -# pylint: enable=invalid-name diff --git a/tensorflow/contrib/learn/python/learn/summary_writer_cache.py b/tensorflow/contrib/learn/python/learn/summary_writer_cache.py deleted file mode 100644 index d663cf5fb79..00000000000 --- a/tensorflow/contrib/learn/python/learn/summary_writer_cache.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Wrapper for a Session-like object that handles threads and recovery (deprecated). - -These are deprecated aliases for classes and functions in `tf.train`. Please use -those directly. - -Based on an original design of Illia Polosukhin. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.training import summary_io - - -SummaryWriterCache = summary_io.SummaryWriterCache # pylint: disable=invalid-name - -# Backward compatible interface. Remove? -clear_summary_writers = SummaryWriterCache.clear -get_summary_writer = SummaryWriterCache.get diff --git a/tensorflow/contrib/learn/python/learn/trainable.py b/tensorflow/contrib/learn/python/learn/trainable.py deleted file mode 100644 index 1ea9e5d67a9..00000000000 --- a/tensorflow/contrib/learn/python/learn/trainable.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""`Trainable` interface (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - - -@six.add_metaclass(abc.ABCMeta) -class Trainable(object): - """Interface for objects that are trainable by, e.g., `Experiment`. - - THIS CLASS IS DEPRECATED. - """ - - @abc.abstractmethod - def fit(self, - x=None, - y=None, - input_fn=None, - steps=None, - batch_size=None, - monitors=None, - max_steps=None): - """Trains a model given training data `x` predictions and `y` labels. - - Args: - x: Matrix of shape [n_samples, n_features...] or the dictionary of - Matrices. - Can be iterator that returns arrays of features or dictionary of arrays - of features. - The training input samples for fitting the model. If set, `input_fn` - must be `None`. - y: Vector or matrix [n_samples] or [n_samples, n_outputs] or the - dictionary of same. - Can be iterator that returns array of labels or dictionary of array of - labels. - The training label values (class labels in classification, real numbers - in regression). - If set, `input_fn` must be `None`. Note: For classification, label - values must - be integers representing the class index (i.e. values from 0 to - n_classes-1). - input_fn: Input function returning a tuple of: - features - `Tensor` or dictionary of string feature name to `Tensor`. - labels - `Tensor` or dictionary of `Tensor` with labels. - If input_fn is set, `x`, `y`, and `batch_size` must be `None`. - steps: Number of steps for which to train model. If `None`, train forever. - 'steps' works incrementally. If you call two times fit(steps=10) then - training occurs in total 20 steps. If you don't want to have incremental - behavior please set `max_steps` instead. If set, `max_steps` must be - `None`. - batch_size: minibatch size to use on the input, defaults to first - dimension of `x`. Must be `None` if `input_fn` is provided. - monitors: List of `BaseMonitor` subclass instances. Used for callbacks - inside the training loop. - max_steps: Number of total steps for which to train model. If `None`, - train forever. If set, `steps` must be `None`. - - Two calls to `fit(steps=100)` means 200 training - iterations. On the other hand, two calls to `fit(max_steps=100)` means - that the second call will not do any iteration since first call did - all 100 steps. - - Returns: - `self`, for chaining. - """ - raise NotImplementedError diff --git a/tensorflow/contrib/learn/python/learn/utils/__init__.py b/tensorflow/contrib/learn/python/learn/utils/__init__.py deleted file mode 100644 index 66d8dc6fd43..00000000000 --- a/tensorflow/contrib/learn/python/learn/utils/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""TensorFlow Learn Utils (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.utils.export import export_estimator -from tensorflow.contrib.learn.python.learn.utils.input_fn_utils import build_default_serving_input_fn -from tensorflow.contrib.learn.python.learn.utils.input_fn_utils import build_parsing_serving_input_fn -from tensorflow.contrib.learn.python.learn.utils.input_fn_utils import InputFnOps -from tensorflow.contrib.learn.python.learn.utils.saved_model_export_utils import make_export_strategy - diff --git a/tensorflow/contrib/learn/python/learn/utils/export.py b/tensorflow/contrib/learn/python/learn/utils/export.py deleted file mode 100644 index 0144b93814a..00000000000 --- a/tensorflow/contrib/learn/python/learn/utils/export.py +++ /dev/null @@ -1,361 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Export utilities (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework import deprecated -from tensorflow.contrib.session_bundle import exporter -from tensorflow.contrib.session_bundle import gc -from tensorflow.python.client import session as tf_session -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import checkpoint_management -from tensorflow.python.training import saver as tf_saver -from tensorflow.python.training import training_util - - -@deprecated('2017-03-25', 'Please use Estimator.export_savedmodel() instead.') -def _get_first_op_from_collection(collection_name): - """Get first element from the collection.""" - elements = ops.get_collection(collection_name) - if elements is not None: - if elements: - return elements[0] - return None - - -@deprecated('2017-03-25', 'Please use Estimator.export_savedmodel() instead.') -def _get_saver(): - """Lazy init and return saver.""" - saver = _get_first_op_from_collection(ops.GraphKeys.SAVERS) - if saver is not None: - if saver: - saver = saver[0] - else: - saver = None - if saver is None and variables.global_variables(): - saver = tf_saver.Saver() - ops.add_to_collection(ops.GraphKeys.SAVERS, saver) - return saver - - -@deprecated('2017-03-25', 'Please use Estimator.export_savedmodel() instead.') -def _export_graph(graph, saver, checkpoint_path, export_dir, - default_graph_signature, named_graph_signatures, - exports_to_keep): - """Exports graph via session_bundle, by creating a Session.""" - with graph.as_default(): - with tf_session.Session('') as session: - variables.local_variables_initializer() - lookup_ops.tables_initializer() - saver.restore(session, checkpoint_path) - - export = exporter.Exporter(saver) - export.init( - init_op=control_flow_ops.group( - variables.local_variables_initializer(), - lookup_ops.tables_initializer()), - default_graph_signature=default_graph_signature, - named_graph_signatures=named_graph_signatures, - assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS)) - return export.export(export_dir, training_util.get_global_step(), - session, exports_to_keep=exports_to_keep) - - -@deprecated('2017-03-25', - 'signature_fns are deprecated. For canned Estimators they are no ' - 'longer needed. For custom Estimators, please return ' - 'output_alternatives from your model_fn via ModelFnOps.') -def generic_signature_fn(examples, unused_features, predictions): - """Creates generic signature from given examples and predictions. - - This is needed for backward compatibility with default behavior of - export_estimator. - - Args: - examples: `Tensor`. - unused_features: `dict` of `Tensor`s. - predictions: `Tensor` or `dict` of `Tensor`s. - - Returns: - Tuple of default signature and empty named signatures. - - Raises: - ValueError: If examples is `None`. - """ - if examples is None: - raise ValueError('examples cannot be None when using this signature fn.') - - tensors = {'inputs': examples} - if not isinstance(predictions, dict): - predictions = {'outputs': predictions} - tensors.update(predictions) - default_signature = exporter.generic_signature(tensors) - return default_signature, {} - - -@deprecated('2017-03-25', - 'signature_fns are deprecated. For canned Estimators they are no ' - 'longer needed. For custom Estimators, please return ' - 'output_alternatives from your model_fn via ModelFnOps.') -def classification_signature_fn(examples, unused_features, predictions): - """Creates classification signature from given examples and predictions. - - Args: - examples: `Tensor`. - unused_features: `dict` of `Tensor`s. - predictions: `Tensor` or dict of tensors that contains the classes tensor - as in {'classes': `Tensor`}. - - Returns: - Tuple of default classification signature and empty named signatures. - - Raises: - ValueError: If examples is `None`. - """ - if examples is None: - raise ValueError('examples cannot be None when using this signature fn.') - - if isinstance(predictions, dict): - default_signature = exporter.classification_signature( - examples, classes_tensor=predictions['classes']) - else: - default_signature = exporter.classification_signature( - examples, classes_tensor=predictions) - return default_signature, {} - - -@deprecated('2017-03-25', - 'signature_fns are deprecated. For canned Estimators they are no ' - 'longer needed. For custom Estimators, please return ' - 'output_alternatives from your model_fn via ModelFnOps.') -def classification_signature_fn_with_prob( - examples, unused_features, predictions): - """Classification signature from given examples and predicted probabilities. - - Args: - examples: `Tensor`. - unused_features: `dict` of `Tensor`s. - predictions: `Tensor` of predicted probabilities or dict that contains the - probabilities tensor as in {'probabilities', `Tensor`}. - - Returns: - Tuple of default classification signature and empty named signatures. - - Raises: - ValueError: If examples is `None`. - """ - if examples is None: - raise ValueError('examples cannot be None when using this signature fn.') - - if isinstance(predictions, dict): - default_signature = exporter.classification_signature( - examples, scores_tensor=predictions['probabilities']) - else: - default_signature = exporter.classification_signature( - examples, scores_tensor=predictions) - return default_signature, {} - - -@deprecated('2017-03-25', - 'signature_fns are deprecated. For canned Estimators they are no ' - 'longer needed. For custom Estimators, please return ' - 'output_alternatives from your model_fn via ModelFnOps.') -def regression_signature_fn(examples, unused_features, predictions): - """Creates regression signature from given examples and predictions. - - Args: - examples: `Tensor`. - unused_features: `dict` of `Tensor`s. - predictions: `Tensor`. - - Returns: - Tuple of default regression signature and empty named signatures. - - Raises: - ValueError: If examples is `None`. - """ - if examples is None: - raise ValueError('examples cannot be None when using this signature fn.') - - default_signature = exporter.regression_signature( - input_tensor=examples, output_tensor=predictions) - return default_signature, {} - - -@deprecated('2017-03-25', - 'signature_fns are deprecated. For canned Estimators they are no ' - 'longer needed. For custom Estimators, please return ' - 'output_alternatives from your model_fn via ModelFnOps.') -def logistic_regression_signature_fn(examples, unused_features, predictions): - """Creates logistic regression signature from given examples and predictions. - - Args: - examples: `Tensor`. - unused_features: `dict` of `Tensor`s. - predictions: `Tensor` of shape [batch_size, 2] of predicted probabilities or - dict that contains the probabilities tensor as in - {'probabilities', `Tensor`}. - - Returns: - Tuple of default regression signature and named signature. - - Raises: - ValueError: If examples is `None`. - """ - if examples is None: - raise ValueError('examples cannot be None when using this signature fn.') - - if isinstance(predictions, dict): - predictions_tensor = predictions['probabilities'] - else: - predictions_tensor = predictions - # predictions should have shape [batch_size, 2] where first column is P(Y=0|x) - # while second column is P(Y=1|x). We are only interested in the second - # column for inference. - predictions_shape = predictions_tensor.get_shape() - predictions_rank = len(predictions_shape) - if predictions_rank != 2: - logging.fatal( - 'Expected predictions to have rank 2, but received predictions with ' - 'rank: {} and shape: {}'.format(predictions_rank, predictions_shape)) - if predictions_shape[1] != 2: - logging.fatal( - 'Expected predictions to have 2nd dimension: 2, but received ' - 'predictions with 2nd dimension: {} and shape: {}. Did you mean to use ' - 'regression_signature_fn or classification_signature_fn_with_prob ' - 'instead?'.format(predictions_shape[1], predictions_shape)) - - positive_predictions = predictions_tensor[:, 1] - default_signature = exporter.regression_signature( - input_tensor=examples, output_tensor=positive_predictions) - return default_signature, {} - - -# pylint: disable=protected-access -@deprecated('2017-03-25', 'Please use Estimator.export_savedmodel() instead.') -def _default_input_fn(estimator, examples): - """Creates default input parsing using Estimator's feature signatures.""" - return estimator._get_feature_ops_from_example(examples) - - -@deprecated('2016-09-23', 'Please use Estimator.export_savedmodel() instead.') -def export_estimator(estimator, - export_dir, - signature_fn=None, - input_fn=_default_input_fn, - default_batch_size=1, - exports_to_keep=None): - """Deprecated, please use Estimator.export_savedmodel().""" - _export_estimator(estimator=estimator, - export_dir=export_dir, - signature_fn=signature_fn, - input_fn=input_fn, - default_batch_size=default_batch_size, - exports_to_keep=exports_to_keep) - - -@deprecated('2017-03-25', 'Please use Estimator.export_savedmodel() instead.') -def _export_estimator(estimator, - export_dir, - signature_fn, - input_fn, - default_batch_size, - exports_to_keep, - input_feature_key=None, - use_deprecated_input_fn=True, - prediction_key=None, - checkpoint_path=None): - if use_deprecated_input_fn: - input_fn = input_fn or _default_input_fn - elif input_fn is None: - raise ValueError('input_fn must be defined.') - - # If checkpoint_path is specified, use the specified checkpoint path. - checkpoint_path = (checkpoint_path or - checkpoint_management.latest_checkpoint( - estimator._model_dir)) - with ops.Graph().as_default() as g: - training_util.create_global_step(g) - - if use_deprecated_input_fn: - examples = array_ops.placeholder(dtype=dtypes.string, - shape=[default_batch_size], - name='input_example_tensor') - features = input_fn(estimator, examples) - else: - features, _ = input_fn() - examples = None - if input_feature_key is not None: - examples = features.pop(input_feature_key) - - if (not features) and (examples is None): - raise ValueError('Either features or examples must be defined.') - - predictions = estimator._get_predict_ops(features).predictions - - if prediction_key is not None: - predictions = predictions[prediction_key] - - # Explicit signature_fn takes priority - if signature_fn: - default_signature, named_graph_signatures = signature_fn(examples, - features, - predictions) - else: - try: - # Some estimators provide a signature function. - # TODO(zakaria): check if the estimator has this function, - # raise helpful error if not - signature_fn = estimator._create_signature_fn() - - default_signature, named_graph_signatures = ( - signature_fn(examples, features, predictions)) - except AttributeError: - logging.warn( - 'Change warning: `signature_fn` will be required after' - '2016-08-01.\n' - 'Using generic signatures for now. To maintain this behavior, ' - 'pass:\n' - ' signature_fn=export.generic_signature_fn\n' - 'Also consider passing a regression or classification signature; ' - 'see cl/126430915 for an example.') - default_signature, named_graph_signatures = generic_signature_fn( - examples, features, predictions) - if exports_to_keep is not None: - exports_to_keep = gc.largest_export_versions(exports_to_keep) - return _export_graph( - g, - _get_saver(), - checkpoint_path, - export_dir, - default_graph_signature=default_signature, - named_graph_signatures=named_graph_signatures, - exports_to_keep=exports_to_keep) -# pylint: enable=protected-access diff --git a/tensorflow/contrib/learn/python/learn/utils/export_test.py b/tensorflow/contrib/learn/python/learn/utils/export_test.py deleted file mode 100644 index 9bfb1fc952c..00000000000 --- a/tensorflow/contrib/learn/python/learn/utils/export_test.py +++ /dev/null @@ -1,255 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for export tools.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import random -import tempfile -import numpy as np -import six - -from tensorflow.contrib import learn -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.learn.python.learn.utils import export -from tensorflow.contrib.session_bundle import exporter -from tensorflow.contrib.session_bundle import manifest_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test -from tensorflow.python.training import saver - -_X_KEY = 'my_x_key' - -_X_COLUMN = feature_column.real_valued_column(_X_KEY, dimension=1) - - -def _training_input_fn(): - x = random_ops.random_uniform(shape=(1,), minval=0.0, maxval=1000.0) - y = 2 * x + 3 - return {_X_KEY: x}, y - - -class ExportTest(test.TestCase): - - def _get_default_signature(self, export_meta_filename): - """ Gets the default signature from the export.meta file. """ - with session.Session(): - save = saver.import_meta_graph(export_meta_filename) - meta_graph_def = save.export_meta_graph() - collection_def = meta_graph_def.collection_def - - signatures_any = collection_def['serving_signatures'].any_list.value - self.assertEquals(len(signatures_any), 1) - signatures = manifest_pb2.Signatures() - signatures_any[0].Unpack(signatures) - default_signature = signatures.default_signature - return default_signature - - def _assert_export(self, export_monitor, export_dir, expected_signature): - self.assertTrue(gfile.Exists(export_dir)) - # Only the written checkpoints are exported. - self.assertTrue( - saver.checkpoint_exists(os.path.join(export_dir, '00000001', 'export')), - 'Exported checkpoint expected but not found: %s' % os.path.join( - export_dir, '00000001', 'export')) - self.assertTrue( - saver.checkpoint_exists(os.path.join(export_dir, '00000010', 'export')), - 'Exported checkpoint expected but not found: %s' % os.path.join( - export_dir, '00000010', 'export')) - self.assertEquals( - six.b(os.path.join(export_dir, '00000010')), - export_monitor.last_export_dir) - # Validate the signature - signature = self._get_default_signature( - os.path.join(export_dir, '00000010', 'export.meta')) - self.assertTrue(signature.HasField(expected_signature)) - - def testExportMonitor_EstimatorProvidesSignature(self): - random.seed(42) - x = np.random.rand(1000) - y = 2 * x + 3 - cont_features = [feature_column.real_valued_column('', dimension=1)] - regressor = learn.LinearRegressor(feature_columns=cont_features) - export_dir = os.path.join(tempfile.mkdtemp(), 'export') - export_monitor = learn.monitors.ExportMonitor( - every_n_steps=1, export_dir=export_dir, exports_to_keep=2) - regressor.fit(x, y, steps=10, monitors=[export_monitor]) - self._assert_export(export_monitor, export_dir, 'regression_signature') - - def testExportMonitor(self): - random.seed(42) - x = np.random.rand(1000) - y = 2 * x + 3 - cont_features = [feature_column.real_valued_column('', dimension=1)] - export_dir = os.path.join(tempfile.mkdtemp(), 'export') - export_monitor = learn.monitors.ExportMonitor( - every_n_steps=1, - export_dir=export_dir, - exports_to_keep=2, - signature_fn=export.generic_signature_fn) - regressor = learn.LinearRegressor(feature_columns=cont_features) - regressor.fit(x, y, steps=10, monitors=[export_monitor]) - self._assert_export(export_monitor, export_dir, 'generic_signature') - - def testExportMonitorInputFeatureKeyMissing(self): - random.seed(42) - - def _serving_input_fn(): - return { - _X_KEY: - random_ops.random_uniform(shape=(1,), minval=0.0, maxval=1000.0) - }, None - - input_feature_key = 'my_example_key' - monitor = learn.monitors.ExportMonitor( - every_n_steps=1, - export_dir=os.path.join(tempfile.mkdtemp(), 'export'), - input_fn=_serving_input_fn, - input_feature_key=input_feature_key, - exports_to_keep=2, - signature_fn=export.generic_signature_fn) - regressor = learn.LinearRegressor(feature_columns=[_X_COLUMN]) - with self.assertRaisesRegexp(KeyError, input_feature_key): - regressor.fit(input_fn=_training_input_fn, steps=10, monitors=[monitor]) - - def testExportMonitorInputFeatureKeyNoneNoFeatures(self): - random.seed(42) - input_feature_key = 'my_example_key' - - def _serving_input_fn(): - return {input_feature_key: None}, None - - monitor = learn.monitors.ExportMonitor( - every_n_steps=1, - export_dir=os.path.join(tempfile.mkdtemp(), 'export'), - input_fn=_serving_input_fn, - input_feature_key=input_feature_key, - exports_to_keep=2, - signature_fn=export.generic_signature_fn) - regressor = learn.LinearRegressor(feature_columns=[_X_COLUMN]) - with self.assertRaisesRegexp(ValueError, - 'features or examples must be defined'): - regressor.fit(input_fn=_training_input_fn, steps=10, monitors=[monitor]) - - def testExportMonitorInputFeatureKeyNone(self): - random.seed(42) - input_feature_key = 'my_example_key' - - def _serving_input_fn(): - return { - input_feature_key: - None, - _X_KEY: - random_ops.random_uniform(shape=(1,), minval=0.0, maxval=1000.0) - }, None - - monitor = learn.monitors.ExportMonitor( - every_n_steps=1, - export_dir=os.path.join(tempfile.mkdtemp(), 'export'), - input_fn=_serving_input_fn, - input_feature_key=input_feature_key, - exports_to_keep=2, - signature_fn=export.generic_signature_fn) - regressor = learn.LinearRegressor(feature_columns=[_X_COLUMN]) - with self.assertRaisesRegexp(ValueError, 'examples cannot be None'): - regressor.fit(input_fn=_training_input_fn, steps=10, monitors=[monitor]) - - def testExportMonitorInputFeatureKeyNoFeatures(self): - random.seed(42) - input_feature_key = 'my_example_key' - - def _serving_input_fn(): - return { - input_feature_key: - array_ops.placeholder(dtype=dtypes.string, shape=(1,)) - }, None - - monitor = learn.monitors.ExportMonitor( - every_n_steps=1, - export_dir=os.path.join(tempfile.mkdtemp(), 'export'), - input_fn=_serving_input_fn, - input_feature_key=input_feature_key, - exports_to_keep=2, - signature_fn=export.generic_signature_fn) - regressor = learn.LinearRegressor(feature_columns=[_X_COLUMN]) - with self.assertRaisesRegexp(KeyError, _X_KEY): - regressor.fit(input_fn=_training_input_fn, steps=10, monitors=[monitor]) - - def testExportMonitorInputFeature(self): - random.seed(42) - input_feature_key = 'my_example_key' - - def _serving_input_fn(): - return { - input_feature_key: - array_ops.placeholder(dtype=dtypes.string, shape=(1,)), - _X_KEY: - random_ops.random_uniform(shape=(1,), minval=0.0, maxval=1000.0) - }, None - - export_dir = os.path.join(tempfile.mkdtemp(), 'export') - monitor = learn.monitors.ExportMonitor( - every_n_steps=1, - export_dir=export_dir, - input_fn=_serving_input_fn, - input_feature_key=input_feature_key, - exports_to_keep=2, - signature_fn=export.generic_signature_fn) - regressor = learn.LinearRegressor(feature_columns=[_X_COLUMN]) - regressor.fit(input_fn=_training_input_fn, steps=10, monitors=[monitor]) - self._assert_export(monitor, export_dir, 'generic_signature') - - def testExportMonitorRegressionSignature(self): - - def _regression_signature(examples, unused_features, predictions): - signatures = {} - signatures['regression'] = ( - exporter.regression_signature(examples, predictions)) - return signatures['regression'], signatures - - random.seed(42) - x = np.random.rand(1000) - y = 2 * x + 3 - cont_features = [feature_column.real_valued_column('', dimension=1)] - regressor = learn.LinearRegressor(feature_columns=cont_features) - export_dir = os.path.join(tempfile.mkdtemp(), 'export') - export_monitor = learn.monitors.ExportMonitor( - every_n_steps=1, - export_dir=export_dir, - exports_to_keep=1, - signature_fn=_regression_signature) - regressor.fit(x, y, steps=10, monitors=[export_monitor]) - - self.assertTrue(gfile.Exists(export_dir)) - with self.assertRaises(errors.NotFoundError): - saver.checkpoint_exists(os.path.join(export_dir, '00000000', 'export')) - self.assertTrue( - saver.checkpoint_exists(os.path.join(export_dir, '00000010', 'export'))) - # Validate the signature - signature = self._get_default_signature( - os.path.join(export_dir, '00000010', 'export.meta')) - self.assertTrue(signature.HasField('regression_signature')) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/utils/gc.py b/tensorflow/contrib/learn/python/learn/utils/gc.py deleted file mode 100644 index f69a4dd6ad5..00000000000 --- a/tensorflow/contrib/learn/python/learn/utils/gc.py +++ /dev/null @@ -1,220 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -r"""System for specifying garbage collection (GC) of path based data (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. - -This framework allows for GC of data specified by path names, for example files -on disk. gc.Path objects each represent a single item stored at a path and may -be a base directory, - /tmp/exports/0/... - /tmp/exports/1/... - ... -or a fully qualified file, - /tmp/train-1.ckpt - /tmp/train-2.ckpt - ... - -A gc filter function takes and returns a list of gc.Path items. Filter -functions are responsible for selecting Path items for preservation or deletion. -Note that functions should always return a sorted list. - -For example, - base_dir = "/tmp" - # Create the directories. - for e in xrange(10): - os.mkdir("%s/%d" % (base_dir, e), 0o755) - - # Create a simple parser that pulls the export_version from the directory. - path_regex = "^" + re.escape(base_dir) + "/(\\d+)$" - def parser(path): - match = re.match(path_regex, path.path) - if not match: - return None - return path._replace(export_version=int(match.group(1))) - - path_list = gc.get_paths("/tmp", parser) # contains all ten Paths - - every_fifth = gc.mod_export_version(5) - print(every_fifth(path_list)) # shows ["/tmp/0", "/tmp/5"] - - largest_three = gc.largest_export_versions(3) - print(largest_three(all_paths)) # shows ["/tmp/7", "/tmp/8", "/tmp/9"] - - both = gc.union(every_fifth, largest_three) - print(both(all_paths)) # shows ["/tmp/0", "/tmp/5", - # "/tmp/7", "/tmp/8", "/tmp/9"] - # Delete everything not in 'both'. - to_delete = gc.negation(both) - for p in to_delete(all_paths): - gfile.rmtree(p.path) # deletes: "/tmp/1", "/tmp/2", - # "/tmp/3", "/tmp/4", "/tmp/6", -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import heapq -import math -import os - -from tensorflow.python.platform import gfile -from tensorflow.python.util import compat -from tensorflow.python.util.deprecation import deprecated - -Path = collections.namedtuple('Path', 'path export_version') - - -@deprecated(None, 'Please implement your own file management or use Saver.') -def largest_export_versions(n): - """Creates a filter that keeps the largest n export versions. - - Args: - n: number of versions to keep. - - Returns: - A filter function that keeps the n largest paths. - """ - def keep(paths): - heap = [] - for idx, path in enumerate(paths): - if path.export_version is not None: - heapq.heappush(heap, (path.export_version, idx)) - keepers = [paths[i] for _, i in heapq.nlargest(n, heap)] - return sorted(keepers) - - return keep - - -@deprecated(None, 'Please implement your own file management or use Saver.') -def one_of_every_n_export_versions(n): - """Creates a filter that keeps one of every n export versions. - - Args: - n: interval size. - - Returns: - A filter function that keeps exactly one path from each interval - [0, n], (n, 2n], (2n, 3n], etc... If more than one path exists in an - interval the largest is kept. - """ - def keep(paths): - """A filter function that keeps exactly one out of every n paths.""" - - keeper_map = {} # map from interval to largest path seen in that interval - for p in paths: - if p.export_version is None: - # Skip missing export_versions. - continue - # Find the interval (with a special case to map export_version = 0 to - # interval 0. - interval = math.floor( - (p.export_version - 1) / n) if p.export_version else 0 - existing = keeper_map.get(interval, None) - if (not existing) or (existing.export_version < p.export_version): - keeper_map[interval] = p - return sorted(keeper_map.values()) - - return keep - - -@deprecated(None, 'Please implement your own file management or use Saver.') -def mod_export_version(n): - """Creates a filter that keeps every export that is a multiple of n. - - Args: - n: step size. - - Returns: - A filter function that keeps paths where export_version % n == 0. - """ - def keep(paths): - keepers = [] - for p in paths: - if p.export_version % n == 0: - keepers.append(p) - return sorted(keepers) - return keep - - -@deprecated(None, 'Please implement your own file management or use Saver.') -def union(lf, rf): - """Creates a filter that keeps the union of two filters. - - Args: - lf: first filter - rf: second filter - - Returns: - A filter function that keeps the n largest paths. - """ - def keep(paths): - l = set(lf(paths)) - r = set(rf(paths)) - return sorted(list(l|r)) - return keep - - -@deprecated(None, 'Please implement your own file management or use Saver.') -def negation(f): - """Negate a filter. - - Args: - f: filter function to invert - - Returns: - A filter function that returns the negation of f. - """ - def keep(paths): - l = set(paths) - r = set(f(paths)) - return sorted(list(l-r)) - return keep - - -@deprecated(None, 'Please implement your own file name management.') -def get_paths(base_dir, parser): - """Gets a list of Paths in a given directory. - - Args: - base_dir: directory. - parser: a function which gets the raw Path and can augment it with - information such as the export_version, or ignore the path by returning - None. An example parser may extract the export version from a path - such as "/tmp/exports/100" an another may extract from a full file - name such as "/tmp/checkpoint-99.out". - - Returns: - A list of Paths contained in the base directory with the parsing function - applied. - By default the following fields are populated, - - Path.path - The parsing function is responsible for populating, - - Path.export_version - """ - raw_paths = gfile.ListDirectory(base_dir) - paths = [] - for r in raw_paths: - p = parser(Path(os.path.join(compat.as_str_any(base_dir), - compat.as_str_any(r)), - None)) - if p: - paths.append(p) - return sorted(paths) diff --git a/tensorflow/contrib/learn/python/learn/utils/gc_test.py b/tensorflow/contrib/learn/python/learn/utils/gc_test.py deleted file mode 100644 index af93e517f51..00000000000 --- a/tensorflow/contrib/learn/python/learn/utils/gc_test.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for learn.utils.gc.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import re - -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib.learn.python.learn.utils import gc -from tensorflow.python.framework import test_util -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test -from tensorflow.python.util import compat - - -def _create_parser(base_dir): - # create a simple parser that pulls the export_version from the directory. - def parser(path): - # Modify the path object for RegEx match for Windows Paths - if os.name == "nt": - match = re.match( - r"^" + compat.as_str_any(base_dir).replace("\\", "/") + r"/(\d+)$", - compat.as_str_any(path.path).replace("\\", "/")) - else: - match = re.match(r"^" + compat.as_str_any(base_dir) + r"/(\d+)$", - compat.as_str_any(path.path)) - if not match: - return None - return path._replace(export_version=int(match.group(1))) - - return parser - - -class GcTest(test_util.TensorFlowTestCase): - - def testLargestExportVersions(self): - paths = [gc.Path("/foo", 8), gc.Path("/foo", 9), gc.Path("/foo", 10)] - newest = gc.largest_export_versions(2) - n = newest(paths) - self.assertEqual(n, [gc.Path("/foo", 9), gc.Path("/foo", 10)]) - - def testLargestExportVersionsDoesNotDeleteZeroFolder(self): - paths = [gc.Path("/foo", 0), gc.Path("/foo", 3)] - newest = gc.largest_export_versions(2) - n = newest(paths) - self.assertEqual(n, [gc.Path("/foo", 0), gc.Path("/foo", 3)]) - - def testModExportVersion(self): - paths = [ - gc.Path("/foo", 4), - gc.Path("/foo", 5), - gc.Path("/foo", 6), - gc.Path("/foo", 9) - ] - mod = gc.mod_export_version(2) - self.assertEqual(mod(paths), [gc.Path("/foo", 4), gc.Path("/foo", 6)]) - mod = gc.mod_export_version(3) - self.assertEqual(mod(paths), [gc.Path("/foo", 6), gc.Path("/foo", 9)]) - - def testOneOfEveryNExportVersions(self): - paths = [ - gc.Path("/foo", 0), - gc.Path("/foo", 1), - gc.Path("/foo", 3), - gc.Path("/foo", 5), - gc.Path("/foo", 6), - gc.Path("/foo", 7), - gc.Path("/foo", 8), - gc.Path("/foo", 33) - ] - one_of = gc.one_of_every_n_export_versions(3) - self.assertEqual( - one_of(paths), [ - gc.Path("/foo", 3), - gc.Path("/foo", 6), - gc.Path("/foo", 8), - gc.Path("/foo", 33) - ]) - - def testOneOfEveryNExportVersionsZero(self): - # Zero is a special case since it gets rolled into the first interval. - # Test that here. - paths = [gc.Path("/foo", 0), gc.Path("/foo", 4), gc.Path("/foo", 5)] - one_of = gc.one_of_every_n_export_versions(3) - self.assertEqual(one_of(paths), [gc.Path("/foo", 0), gc.Path("/foo", 5)]) - - def testUnion(self): - paths = [] - for i in xrange(10): - paths.append(gc.Path("/foo", i)) - f = gc.union(gc.largest_export_versions(3), gc.mod_export_version(3)) - self.assertEqual( - f(paths), [ - gc.Path("/foo", 0), - gc.Path("/foo", 3), - gc.Path("/foo", 6), - gc.Path("/foo", 7), - gc.Path("/foo", 8), - gc.Path("/foo", 9) - ]) - - def testNegation(self): - paths = [ - gc.Path("/foo", 4), - gc.Path("/foo", 5), - gc.Path("/foo", 6), - gc.Path("/foo", 9) - ] - mod = gc.negation(gc.mod_export_version(2)) - self.assertEqual(mod(paths), [gc.Path("/foo", 5), gc.Path("/foo", 9)]) - mod = gc.negation(gc.mod_export_version(3)) - self.assertEqual(mod(paths), [gc.Path("/foo", 4), gc.Path("/foo", 5)]) - - def testPathsWithParse(self): - base_dir = os.path.join(test.get_temp_dir(), "paths_parse") - self.assertFalse(gfile.Exists(base_dir)) - for p in xrange(3): - gfile.MakeDirs(os.path.join(base_dir, "%d" % p)) - # add a base_directory to ignore - gfile.MakeDirs(os.path.join(base_dir, "ignore")) - - self.assertEqual( - gc.get_paths(base_dir, _create_parser(base_dir)), [ - gc.Path(os.path.join(base_dir, "0"), 0), - gc.Path(os.path.join(base_dir, "1"), 1), - gc.Path(os.path.join(base_dir, "2"), 2) - ]) - - def testMixedStrTypes(self): - temp_dir = compat.as_bytes(test.get_temp_dir()) - - for sub_dir in ["str", b"bytes", u"unicode"]: - base_dir = os.path.join( - (temp_dir - if isinstance(sub_dir, bytes) else temp_dir.decode()), sub_dir) - self.assertFalse(gfile.Exists(base_dir)) - gfile.MakeDirs(os.path.join(compat.as_str_any(base_dir), "42")) - gc.get_paths(base_dir, _create_parser(base_dir)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/learn/python/learn/utils/input_fn_utils.py b/tensorflow/contrib/learn/python/learn/utils/input_fn_utils.py deleted file mode 100644 index b92eb9fea8b..00000000000 --- a/tensorflow/contrib/learn/python/learn/utils/input_fn_utils.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities for creating input_fns (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. - -Contents of this file are moved to tensorflow/python/estimator/export.py. -InputFnOps is renamed to ServingInputReceiver. -build_parsing_serving_input_fn is renamed to - build_parsing_serving_input_receiver_fn. -build_default_serving_input_fn is renamed to - build_raw_serving_input_receiver_fn. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.util.deprecation import deprecated - - -class InputFnOps(collections.namedtuple('InputFnOps', - ['features', - 'labels', - 'default_inputs'])): - """A return type for an input_fn (deprecated). - - THIS CLASS IS DEPRECATED. Please use tf.estimator.export.ServingInputReceiver - instead. - - This return type is currently only supported for serving input_fn. - Training and eval input_fn should return a `(features, labels)` tuple. - - The expected return values are: - features: A dict of string to `Tensor` or `SparseTensor`, specifying the - features to be passed to the model. - labels: A `Tensor`, `SparseTensor`, or a dict of string to `Tensor` or - `SparseTensor`, specifying labels for training or eval. For serving, set - `labels` to `None`. - default_inputs: a dict of string to `Tensor` or `SparseTensor`, specifying - the input placeholders (if any) that this input_fn expects to be fed. - Typically, this is used by a serving input_fn, which expects to be fed - serialized `tf.Example` protos. - """ - - -@deprecated(None, 'Please use ' - 'tf.estimator.export.build_parsing_serving_input_receiver_fn.') -def build_parsing_serving_input_fn(feature_spec, default_batch_size=None): - """Build an input_fn appropriate for serving, expecting fed tf.Examples. - - Creates an input_fn that expects a serialized tf.Example fed into a string - placeholder. The function parses the tf.Example according to the provided - feature_spec, and returns all parsed Tensors as features. This input_fn is - for use at serving time, so the labels return value is always None. - - Args: - feature_spec: a dict of string to `VarLenFeature`/`FixedLenFeature`. - default_batch_size: the number of query examples expected per batch. - Leave unset for variable batch size (recommended). - - Returns: - An input_fn suitable for use in serving. - """ - def input_fn(): - """An input_fn that expects a serialized tf.Example.""" - serialized_tf_example = array_ops.placeholder(dtype=dtypes.string, - shape=[default_batch_size], - name='input_example_tensor') - inputs = {'examples': serialized_tf_example} - features = parsing_ops.parse_example(serialized_tf_example, feature_spec) - labels = None # these are not known in serving! - return InputFnOps(features, labels, inputs) - return input_fn - - -@deprecated(None, 'Please use ' - 'tf.estimator.export.build_raw_serving_input_receiver_fn.') -def build_default_serving_input_fn(features, default_batch_size=None): - """Build an input_fn appropriate for serving, expecting feature Tensors. - - Creates an input_fn that expects all features to be fed directly. - This input_fn is for use at serving time, so the labels return value is always - None. - - Args: - features: a dict of string to `Tensor`. - default_batch_size: the number of query examples expected per batch. - Leave unset for variable batch size (recommended). - - Returns: - An input_fn suitable for use in serving. - """ - def input_fn(): - """an input_fn that expects all features to be fed directly.""" - features_placeholders = {} - for name, t in features.items(): - shape_list = t.get_shape().as_list() - shape_list[0] = default_batch_size - shape = tensor_shape.TensorShape(shape_list) - - features_placeholders[name] = array_ops.placeholder( - dtype=t.dtype, shape=shape, name=t.op.name) - labels = None # these are not known in serving! - return InputFnOps(features_placeholders, labels, features_placeholders) - return input_fn diff --git a/tensorflow/contrib/learn/python/learn/utils/input_fn_utils_test.py b/tensorflow/contrib/learn/python/learn/utils/input_fn_utils_test.py deleted file mode 100644 index e9dc6a68759..00000000000 --- a/tensorflow/contrib/learn/python/learn/utils/input_fn_utils_test.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests of utilities for creating input_fns.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.utils import input_fn_utils -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class InputFnTest(test.TestCase): - - def test_build_default_serving_input_fn_name(self): - """Test case for issue #12755.""" - f = { - 'feature': - array_ops.placeholder( - name='feature', shape=[32], dtype=dtypes.float32) - } - serving_input = input_fn_utils.build_default_serving_input_fn(f) - v = serving_input() - self.assertTrue(isinstance(v, input_fn_utils.InputFnOps)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/learn/python/learn/utils/inspect_checkpoint.py b/tensorflow/contrib/learn/python/learn/utils/inspect_checkpoint.py deleted file mode 100644 index 6dbaa15f839..00000000000 --- a/tensorflow/contrib/learn/python/learn/utils/inspect_checkpoint.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A simple script for inspect checkpoint files (deprecated).""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import sys - -from tensorflow.contrib.framework.python.framework import checkpoint_utils -from tensorflow.python.platform import app - -FLAGS = None - - -def print_tensors_in_checkpoint_file(file_name, tensor_name): - """Prints tensors in a checkpoint file. - - If no `tensor_name` is provided, prints the tensor names and shapes - in the checkpoint file. - - If `tensor_name` is provided, prints the content of the tensor. - - Args: - file_name: Name of the checkpoint file. - tensor_name: Name of the tensor in the checkpoint file to print. - """ - try: - if not tensor_name: - variables = checkpoint_utils.list_variables(file_name) - for name, shape in variables: - print("%s\t%s" % (name, str(shape))) - else: - print("tensor_name: ", tensor_name) - print(checkpoint_utils.load_variable(file_name, tensor_name)) - except Exception as e: # pylint: disable=broad-except - print(str(e)) - if "corrupted compressed block contents" in str(e): - print("It's likely that your checkpoint file has been compressed " - "with SNAPPY.") - - -def main(unused_argv): - if not FLAGS.file_name: - print("Usage: inspect_checkpoint --file_name= [--tensor_name=tensor_to_print]") - sys.exit(1) - else: - print_tensors_in_checkpoint_file(FLAGS.file_name, FLAGS.tensor_name) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.register("type", "bool", lambda v: v.lower() == "true") - parser.add_argument( - "--file_name", - type=str, - default="", - help="Checkpoint filename" - ) - parser.add_argument( - "--tensor_name", - type=str, - default="", - help="Name of the tensor to inspect" - ) - FLAGS, unparsed = parser.parse_known_args() - app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py b/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py deleted file mode 100644 index 4f22054af30..00000000000 --- a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py +++ /dev/null @@ -1,805 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities supporting export to SavedModel (deprecated). - -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. - -Some contents of this file are moved to tensorflow/python/estimator/export.py: - -get_input_alternatives() -> obsolete -get_output_alternatives() -> obsolete, but see _get_default_export_output() -build_all_signature_defs() -> build_all_signature_defs() -get_timestamped_export_directory() -> get_timestamped_export_directory() -_get_* -> obsolete -_is_* -> obsolete - -Functionality of build_standardized_signature_def() is moved to -tensorflow/python/estimator/export_output.py as ExportOutput.as_signature_def(). - -Anything to do with ExportStrategies or garbage collection is not moved. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import time - -from tensorflow.contrib.layers.python.layers import feature_column -from tensorflow.contrib.learn.python.learn import export_strategy -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import metric_key -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.contrib.learn.python.learn.utils import gc -from tensorflow.contrib.learn.python.learn.utils import input_fn_utils -from tensorflow.python.estimator import estimator as core_estimator -from tensorflow.python.estimator.export import export as core_export -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.platform import gfile -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.saved_model import signature_constants -from tensorflow.python.saved_model import signature_def_utils -from tensorflow.python.summary import summary_iterator -from tensorflow.python.training import checkpoint_management -from tensorflow.python.util import compat -from tensorflow.python.util.deprecation import deprecated - - -# A key for use in the input_alternatives dict indicating the default input. -# This is the input that will be expected when a serving request does not -# specify a specific signature. -# The default input alternative specifies placeholders that the input_fn -# requires to be fed (in the typical case, a single placeholder for a -# serialized tf.Example). -DEFAULT_INPUT_ALTERNATIVE_KEY = 'default_input_alternative' - -# A key for use in the input_alternatives dict indicating the features input. -# The features inputs alternative specifies the feature Tensors provided as -# input to the model_fn, i.e. the outputs of the input_fn. -FEATURES_INPUT_ALTERNATIVE_KEY = 'features_input_alternative' - -# A key for use in the output_alternatives dict indicating the default output. -# This is the output that will be provided when a serving request does not -# specify a specific signature. -# In a single-headed model, the single output is automatically the default. -# In a multi-headed model, the name of the desired default head should be -# provided to get_output_alternatives. -_FALLBACK_DEFAULT_OUTPUT_ALTERNATIVE_KEY = 'default_output_alternative' - - -@deprecated(None, 'Switch to tf.estimator.Exporter and associated utilities.') -def build_standardized_signature_def(input_tensors, output_tensors, - problem_type): - """Build a SignatureDef using problem type and input and output Tensors. - - Note that this delegates the actual creation of the signatures to methods in - //third_party/tensorflow/python/saved_model/signature_def_utils.py, which may - assign names to the input and output tensors (depending on the problem type) - that are standardized in the context of SavedModel. - - Args: - input_tensors: a dict of string key to `Tensor` - output_tensors: a dict of string key to `Tensor` - problem_type: an instance of constants.ProblemType, specifying - classification, regression, etc. - - Returns: - A SignatureDef using SavedModel standard keys where possible. - - Raises: - ValueError: if input_tensors or output_tensors is None or empty. - """ - - if not input_tensors: - raise ValueError('input_tensors must be provided.') - if not output_tensors: - raise ValueError('output_tensors must be provided.') - - # Per-method signature_def functions will standardize the keys if possible - if _is_classification_problem(problem_type, input_tensors, output_tensors): - (_, examples), = input_tensors.items() - classes = _get_classification_classes(output_tensors) - scores = _get_classification_scores(output_tensors) - if classes is None and scores is None: - items = list(output_tensors.items()) - if items[0][1].dtype == dtypes.string: - (_, classes), = items - else: - (_, scores), = items - return signature_def_utils.classification_signature_def( - examples, classes, scores) - elif _is_regression_problem(problem_type, input_tensors, output_tensors): - (_, examples), = input_tensors.items() - (_, predictions), = output_tensors.items() - return signature_def_utils.regression_signature_def(examples, predictions) - else: - return signature_def_utils.predict_signature_def(input_tensors, - output_tensors) - - -def _get_classification_scores(output_tensors): - scores = output_tensors.get(prediction_key.PredictionKey.SCORES) - if scores is None: - scores = output_tensors.get(prediction_key.PredictionKey.PROBABILITIES) - return scores - - -def _get_classification_classes(output_tensors): - classes = output_tensors.get(prediction_key.PredictionKey.CLASSES) - if classes is not None and classes.dtype != dtypes.string: - # Servo classification can only serve string classes. - return None - return classes - - -def _is_classification_problem(problem_type, input_tensors, output_tensors): - classes = _get_classification_classes(output_tensors) - scores = _get_classification_scores(output_tensors) - return ((problem_type == constants.ProblemType.CLASSIFICATION or - problem_type == constants.ProblemType.LOGISTIC_REGRESSION) and - len(input_tensors) == 1 and - (classes is not None or scores is not None or - len(output_tensors) == 1)) - - -def _is_regression_problem(problem_type, input_tensors, output_tensors): - return (problem_type == constants.ProblemType.LINEAR_REGRESSION and - len(input_tensors) == 1 and len(output_tensors) == 1) - - -@deprecated(None, 'Switch to tf.estimator.Exporter and associated utilities.') -def get_input_alternatives(input_ops): - """Obtain all input alternatives using the input_fn output and heuristics.""" - input_alternatives = {} - if isinstance(input_ops, input_fn_utils.InputFnOps): - features, unused_labels, default_inputs = input_ops - input_alternatives[DEFAULT_INPUT_ALTERNATIVE_KEY] = default_inputs - else: - features, unused_labels = input_ops - - if not features: - raise ValueError('Features must be defined.') - - # TODO(b/34253951): reinstate the "features" input_signature. - # The "features" input_signature, as written, does not work with - # SparseTensors. It is simply commented out as a stopgap, pending discussion - # on the bug as to the correct solution. - - # Add the "features" input_signature in any case. - # Note defensive copy because model_fns alter the features dict. - # input_alternatives[FEATURES_INPUT_ALTERNATIVE_KEY] = ( - # copy.copy(features)) - - return input_alternatives, features - - -@deprecated(None, 'Switch to tf.estimator.Exporter and associated utilities.') -def get_output_alternatives(model_fn_ops, default_output_alternative_key=None): - """Obtain all output alternatives using the model_fn output and heuristics. - - Args: - model_fn_ops: a `ModelFnOps` object produced by a `model_fn`. This may or - may not have output_alternatives populated. - default_output_alternative_key: the name of the head to serve when an - incoming serving request does not explicitly request a specific head. - Not needed for single-headed models. - - Returns: - A tuple of (output_alternatives, actual_default_output_alternative_key), - where the latter names the head that will actually be served by default. - This may differ from the requested default_output_alternative_key when - a) no output_alternatives are provided at all, so one must be generated, or - b) there is exactly one head, which is used regardless of the requested - default. - - Raises: - ValueError: if the requested default_output_alternative_key is not available - in output_alternatives, or if there are multiple output_alternatives and - no default is specified. - """ - output_alternatives = model_fn_ops.output_alternatives - - if not output_alternatives: - if default_output_alternative_key: - raise ValueError('Requested default_output_alternative: {}, ' - 'but available output_alternatives are: []'.format( - default_output_alternative_key)) - - # Lacking provided output alternatives, the best we can do is to - # interpret the model as single-headed of unknown type. - default_problem_type = constants.ProblemType.UNSPECIFIED - default_outputs = model_fn_ops.predictions - if not isinstance(default_outputs, dict): - default_outputs = {prediction_key.PredictionKey.GENERIC: default_outputs} - actual_default_output_alternative_key = ( - _FALLBACK_DEFAULT_OUTPUT_ALTERNATIVE_KEY) - output_alternatives = { - actual_default_output_alternative_key: (default_problem_type, - default_outputs) - } - return output_alternatives, actual_default_output_alternative_key - - if default_output_alternative_key: - # If a default head is provided, use it. - if default_output_alternative_key in output_alternatives: - return output_alternatives, default_output_alternative_key - - raise ValueError('Requested default_output_alternative: {}, ' - 'but available output_alternatives are: {}'.format( - default_output_alternative_key, - sorted(output_alternatives.keys()))) - - if len(output_alternatives) == 1: - # If there is only one head, use it as the default regardless of its name. - (actual_default_output_alternative_key, _), = output_alternatives.items() - return output_alternatives, actual_default_output_alternative_key - - raise ValueError('Please specify a default_output_alternative. ' - 'Available output_alternatives are: {}'.format( - sorted(output_alternatives.keys()))) - - -@deprecated(None, 'Switch to tf.estimator.Exporter and associated utilities.') -def build_all_signature_defs(input_alternatives, output_alternatives, - actual_default_output_alternative_key): - """Build `SignatureDef`s from all pairs of input and output alternatives.""" - - signature_def_map = {('%s:%s' % (input_key, output_key or 'None')): - build_standardized_signature_def(inputs, outputs, - problem_type) - for input_key, inputs in input_alternatives.items() - for output_key, (problem_type, - outputs) in output_alternatives.items()} - - # Add the default SignatureDef - default_inputs = input_alternatives.get(DEFAULT_INPUT_ALTERNATIVE_KEY) - if not default_inputs: - raise ValueError('A default input_alternative must be provided.') - # default_inputs = input_alternatives[FEATURES_INPUT_ALTERNATIVE_KEY] - # default outputs are guaranteed to exist above - (default_problem_type, default_outputs) = ( - output_alternatives[actual_default_output_alternative_key]) - signature_def_map[signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] = ( - build_standardized_signature_def(default_inputs, default_outputs, - default_problem_type)) - - return signature_def_map - - -# When we create a timestamped directory, there is a small chance that the -# directory already exists because another worker is also writing exports. -# In this case we just wait one second to get a new timestamp and try again. -# If this fails several times in a row, then something is seriously wrong. -MAX_DIRECTORY_CREATION_ATTEMPTS = 10 - - -@deprecated(None, 'Switch to tf.estimator.Exporter and associated utilities.') -def get_timestamped_export_dir(export_dir_base): - """Builds a path to a new subdirectory within the base directory. - - Each export is written into a new subdirectory named using the - current time. This guarantees monotonically increasing version - numbers even across multiple runs of the pipeline. - The timestamp used is the number of seconds since epoch UTC. - - Args: - export_dir_base: A string containing a directory to write the exported - graph and checkpoints. - Returns: - The full path of the new subdirectory (which is not actually created yet). - - Raises: - RuntimeError: if repeated attempts fail to obtain a unique timestamped - directory name. - """ - attempts = 0 - while attempts < MAX_DIRECTORY_CREATION_ATTEMPTS: - export_timestamp = int(time.time()) - - export_dir = os.path.join( - compat.as_bytes(export_dir_base), - compat.as_bytes(str(export_timestamp))) - if not gfile.Exists(export_dir): - # Collisions are still possible (though extremely unlikely): this - # directory is not actually created yet, but it will be almost - # instantly on return from this function. - return export_dir - time.sleep(1) - attempts += 1 - logging.warn('Export directory {} already exists; retrying (attempt {}/{})'. - format(export_dir, attempts, MAX_DIRECTORY_CREATION_ATTEMPTS)) - raise RuntimeError('Failed to obtain a unique export directory name after ' - '{} attempts.'.format(MAX_DIRECTORY_CREATION_ATTEMPTS)) - - -@deprecated(None, 'Switch to tf.estimator.Exporter and associated utilities.') -def get_temp_export_dir(timestamped_export_dir): - """Builds a directory name based on the argument but starting with 'temp-'. - - This relies on the fact that TensorFlow Serving ignores subdirectories of - the base directory that can't be parsed as integers. - - Args: - timestamped_export_dir: the name of the eventual export directory, e.g. - /foo/bar/ - - Returns: - A sister directory prefixed with 'temp-', e.g. /foo/bar/temp-. - """ - (dirname, basename) = os.path.split(timestamped_export_dir) - temp_export_dir = os.path.join( - compat.as_bytes(dirname), - compat.as_bytes('temp-{}'.format(compat.as_text(basename)))) - return temp_export_dir - - -# create a simple parser that pulls the export_version from the directory. -def _export_version_parser(path): - filename = os.path.basename(path.path) - if not (len(filename) == 10 and filename.isdigit()): - return None - return path._replace(export_version=int(filename)) - - -@deprecated(None, 'Switch to tf.estimator.Exporter and associated utilities.') -def get_most_recent_export(export_dir_base): - """Locate the most recent SavedModel export in a directory of many exports. - - This method assumes that SavedModel subdirectories are named as a timestamp - (seconds from epoch), as produced by get_timestamped_export_dir(). - - Args: - export_dir_base: A base directory containing multiple timestamped - directories. - - Returns: - A gc.Path, with is just a namedtuple of (path, export_version). - """ - select_filter = gc.largest_export_versions(1) - results = select_filter( - gc.get_paths(export_dir_base, parser=_export_version_parser)) - return next(iter(results or []), None) - - -@deprecated(None, 'Switch to tf.estimator.Exporter and associated utilities.') -def garbage_collect_exports(export_dir_base, exports_to_keep): - """Deletes older exports, retaining only a given number of the most recent. - - Export subdirectories are assumed to be named with monotonically increasing - integers; the most recent are taken to be those with the largest values. - - Args: - export_dir_base: the base directory under which each export is in a - versioned subdirectory. - exports_to_keep: the number of recent exports to retain. - """ - if exports_to_keep is None: - return - - keep_filter = gc.largest_export_versions(exports_to_keep) - delete_filter = gc.negation(keep_filter) - for p in delete_filter( - gc.get_paths(export_dir_base, parser=_export_version_parser)): - try: - gfile.DeleteRecursively(p.path) - except errors_impl.NotFoundError as e: - logging.warn('Can not delete %s recursively: %s', p.path, e) - - -@deprecated(None, 'Switch to tf.estimator.Exporter and associated utilities.') -def make_export_strategy(serving_input_fn, - default_output_alternative_key=None, - assets_extra=None, - as_text=False, - exports_to_keep=5, - strip_default_attrs=None): - """Create an ExportStrategy for use with Experiment. - - Args: - serving_input_fn: A function that takes no arguments and returns an - `InputFnOps`. - default_output_alternative_key: the name of the head to serve when an - incoming serving request does not explicitly request a specific head. - Must be `None` if the estimator inherits from `tf.estimator.Estimator` - or for single-headed models. - assets_extra: A dict specifying how to populate the assets.extra directory - within the exported SavedModel. Each key should give the destination - path (including the filename) relative to the assets.extra directory. - The corresponding value gives the full path of the source file to be - copied. For example, the simple case of copying a single file without - renaming it is specified as - `{'my_asset_file.txt': '/path/to/my_asset_file.txt'}`. - as_text: whether to write the SavedModel proto in text format. - exports_to_keep: Number of exports to keep. Older exports will be - garbage-collected. Defaults to 5. Set to None to disable garbage - collection. - strip_default_attrs: Boolean. If True, default attrs in the - `GraphDef` will be stripped on write. This is recommended for better - forward compatibility of the resulting `SavedModel`. - - Returns: - An ExportStrategy that can be passed to the Experiment constructor. - """ - - def export_fn(estimator, export_dir_base, checkpoint_path=None, - strip_default_attrs=False): - """Exports the given Estimator as a SavedModel. - - Args: - estimator: the Estimator to export. - export_dir_base: A string containing a directory to write the exported - graph and checkpoints. - checkpoint_path: The checkpoint path to export. If None (the default), - the most recent checkpoint found within the model directory is chosen. - strip_default_attrs: Boolean. If `True`, default-valued attributes will - be removed from the NodeDefs. - - Returns: - The string path to the exported directory. - - Raises: - ValueError: If `estimator` is a `tf.estimator.Estimator` instance - and `default_output_alternative_key` was specified. - """ - if isinstance(estimator, core_estimator.Estimator): - if default_output_alternative_key is not None: - raise ValueError( - 'default_output_alternative_key is not supported in core ' - 'Estimator. Given: {}'.format(default_output_alternative_key)) - export_result = estimator.export_savedmodel( - export_dir_base, - serving_input_fn, - assets_extra=assets_extra, - as_text=as_text, - checkpoint_path=checkpoint_path, - strip_default_attrs=strip_default_attrs) - else: - export_result = estimator.export_savedmodel( - export_dir_base, - serving_input_fn, - default_output_alternative_key=default_output_alternative_key, - assets_extra=assets_extra, - as_text=as_text, - checkpoint_path=checkpoint_path, - strip_default_attrs=strip_default_attrs) - - garbage_collect_exports(export_dir_base, exports_to_keep) - return export_result - - return export_strategy.ExportStrategy('Servo', export_fn, strip_default_attrs) - - -@deprecated(None, - 'Use tf.estimator.export.build_parsing_serving_input_receiver_fn') -def make_parsing_export_strategy(feature_columns, - default_output_alternative_key=None, - assets_extra=None, - as_text=False, - exports_to_keep=5, - target_core=False, - strip_default_attrs=None): - """Create an ExportStrategy for use with Experiment, using `FeatureColumn`s. - - Creates a SavedModel export that expects to be fed with a single string - Tensor containing serialized tf.Examples. At serving time, incoming - tf.Examples will be parsed according to the provided `FeatureColumn`s. - - Args: - feature_columns: An iterable of `FeatureColumn`s representing the features - that must be provided at serving time (excluding labels!). - default_output_alternative_key: the name of the head to serve when an - incoming serving request does not explicitly request a specific head. - Must be `None` if the estimator inherits from `tf.estimator.Estimator` - or for single-headed models. - assets_extra: A dict specifying how to populate the assets.extra directory - within the exported SavedModel. Each key should give the destination - path (including the filename) relative to the assets.extra directory. - The corresponding value gives the full path of the source file to be - copied. For example, the simple case of copying a single file without - renaming it is specified as - `{'my_asset_file.txt': '/path/to/my_asset_file.txt'}`. - as_text: whether to write the SavedModel proto in text format. - exports_to_keep: Number of exports to keep. Older exports will be - garbage-collected. Defaults to 5. Set to None to disable garbage - collection. - target_core: If True, prepare an ExportStrategy for use with - tensorflow.python.estimator.*. If False (default), prepare an - ExportStrategy for use with tensorflow.contrib.learn.python.learn.*. - strip_default_attrs: Boolean. If True, default attrs in the - `GraphDef` will be stripped on write. This is recommended for better - forward compatibility of the resulting `SavedModel`. - - Returns: - An ExportStrategy that can be passed to the Experiment constructor. - """ - feature_spec = feature_column.create_feature_spec_for_parsing(feature_columns) - if target_core: - serving_input_fn = ( - core_export.build_parsing_serving_input_receiver_fn(feature_spec)) - else: - serving_input_fn = ( - input_fn_utils.build_parsing_serving_input_fn(feature_spec)) - return make_export_strategy( - serving_input_fn, - default_output_alternative_key=default_output_alternative_key, - assets_extra=assets_extra, - as_text=as_text, - exports_to_keep=exports_to_keep, - strip_default_attrs=strip_default_attrs) - - -def _default_compare_fn(curr_best_eval_result, cand_eval_result): - """Compares two evaluation results and returns true if the 2nd one is better. - - Both evaluation results should have the values for MetricKey.LOSS, which are - used for comparison. - - Args: - curr_best_eval_result: current best eval metrics. - cand_eval_result: candidate eval metrics. - - Returns: - True if cand_eval_result is better. - - Raises: - ValueError: If input eval result is None or no loss is available. - """ - default_key = metric_key.MetricKey.LOSS - if not curr_best_eval_result or default_key not in curr_best_eval_result: - raise ValueError( - 'curr_best_eval_result cannot be empty or no loss is found in it.') - - if not cand_eval_result or default_key not in cand_eval_result: - raise ValueError( - 'cand_eval_result cannot be empty or no loss is found in it.') - - return curr_best_eval_result[default_key] > cand_eval_result[default_key] - - -class BestModelSelector(object): - """A helper that keeps track of export selection candidates. - - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. - """ - - @deprecated(None, 'Switch to tf.estimator.Exporter and associated utilities.') - def __init__(self, event_file_pattern=None, compare_fn=None): - """Constructor of this class. - - Args: - event_file_pattern: absolute event file name pattern. - compare_fn: a function that returns true if the candidate is better than - the current best model. - """ - self._compare_fn = compare_fn or _default_compare_fn - self._best_eval_result = self._get_best_eval_result(event_file_pattern) - - def update(self, checkpoint_path, eval_result): - """Records a given checkpoint and exports if this is the best model. - - Args: - checkpoint_path: the checkpoint path to export. - eval_result: a dictionary which is usually generated in evaluation runs. - By default, eval_results contains 'loss' field. - - Returns: - A string representing the path to the checkpoint to be exported. - A dictionary of the same type of eval_result. - - Raises: - ValueError: if checkpoint path is empty. - ValueError: if eval_results is None object. - """ - if not checkpoint_path: - raise ValueError('Checkpoint path is empty.') - if eval_result is None: - raise ValueError('%s has empty evaluation results.', checkpoint_path) - - if (self._best_eval_result is None or - self._compare_fn(self._best_eval_result, eval_result)): - self._best_eval_result = eval_result - return checkpoint_path, eval_result - else: - return '', None - - def _get_best_eval_result(self, event_files): - """Get the best eval result from event files. - - Args: - event_files: Absolute pattern of event files. - - Returns: - The best eval result. - """ - if not event_files: - return None - - best_eval_result = None - for event_file in gfile.Glob(os.path.join(event_files)): - for event in summary_iterator.summary_iterator(event_file): - if event.HasField('summary'): - event_eval_result = {} - for value in event.summary.value: - if value.HasField('simple_value'): - event_eval_result[value.tag] = value.simple_value - if best_eval_result is None or self._compare_fn( - best_eval_result, event_eval_result): - best_eval_result = event_eval_result - return best_eval_result - - -@deprecated(None, 'Switch to tf.estimator.Exporter and associated utilities.') -def make_best_model_export_strategy( - serving_input_fn, - exports_to_keep=1, - model_dir=None, - event_file_pattern=None, - compare_fn=None, - default_output_alternative_key=None, - strip_default_attrs=None): - """Creates an custom ExportStrategy for use with tf.contrib.learn.Experiment. - - Args: - serving_input_fn: a function that takes no arguments and returns an - `InputFnOps`. - exports_to_keep: an integer indicating how many historical best models need - to be preserved. - model_dir: Directory where model parameters, graph etc. are saved. This will - be used to load eval metrics from the directory when the export strategy - is created. So the best metrics would not be lost even if the export - strategy got preempted, which guarantees that only the best model would - be exported regardless of preemption. If None, however, the export - strategy would not be preemption-safe. To be preemption-safe, both - model_dir and event_file_pattern would be needed. - event_file_pattern: event file name pattern relative to model_dir, e.g. - "eval_continuous/*.tfevents.*". If None, however, the export strategy - would not be preemption-safe. To be preemption-safe, both - model_dir and event_file_pattern would be needed. - compare_fn: a function that select the 'best' candidate from a dictionary - of evaluation result keyed by corresponding checkpoint path. - default_output_alternative_key: the key for default serving signature for - multi-headed inference graphs. - strip_default_attrs: Boolean. If True, default attrs in the - `GraphDef` will be stripped on write. This is recommended for better - forward compatibility of the resulting `SavedModel`. - - Returns: - An ExportStrategy that can be passed to the Experiment constructor. - """ - best_model_export_strategy = make_export_strategy( - serving_input_fn, - exports_to_keep=exports_to_keep, - default_output_alternative_key=default_output_alternative_key, - strip_default_attrs=strip_default_attrs) - - full_event_file_pattern = os.path.join( - model_dir, - event_file_pattern) if model_dir and event_file_pattern else None - best_model_selector = BestModelSelector(full_event_file_pattern, compare_fn) - - def export_fn(estimator, export_dir_base, checkpoint_path, eval_result=None): - """Exports the given Estimator as a SavedModel. - - Args: - estimator: the Estimator to export. - export_dir_base: A string containing a directory to write the exported - graph and checkpoints. - checkpoint_path: The checkpoint path to export. If None (the default), - the most recent checkpoint found within the model directory is chosen. - eval_result: placehold args matching the call signature of ExportStrategy. - - Returns: - The string path to the exported directory. - """ - if not checkpoint_path: - # TODO(b/67425018): switch to - # checkpoint_path = estimator.latest_checkpoint() - # as soon as contrib is cleaned up and we can thus be sure that - # estimator is a tf.estimator.Estimator and not a - # tf.contrib.learn.Estimator - checkpoint_path = checkpoint_management.latest_checkpoint( - estimator.model_dir) - export_checkpoint_path, export_eval_result = best_model_selector.update( - checkpoint_path, eval_result) - - if export_checkpoint_path and export_eval_result is not None: - checkpoint_base = os.path.basename(export_checkpoint_path) - export_dir = os.path.join(export_dir_base, checkpoint_base) - return best_model_export_strategy.export( - estimator, export_dir, export_checkpoint_path, export_eval_result) - else: - return '' - - return export_strategy.ExportStrategy('best_model', export_fn) - - -# TODO(b/67013778): Revisit this approach when corresponding changes to -# TF Core are finalized. -@deprecated(None, 'Switch to tf.estimator.Exporter and associated utilities.') -def extend_export_strategy(base_export_strategy, - post_export_fn, - post_export_name=None): - """Extend ExportStrategy, calling post_export_fn after export. - - Args: - base_export_strategy: An ExportStrategy that can be passed to the Experiment - constructor. - post_export_fn: A user-specified function to call after exporting the - SavedModel. Takes two arguments - the path to the SavedModel exported by - base_export_strategy and the directory where to export the SavedModel - modified by the post_export_fn. Returns the path to the exported - SavedModel. - post_export_name: The directory name under the export base directory where - SavedModels generated by the post_export_fn will be written. If None, the - directory name of base_export_strategy is used. - - Returns: - An ExportStrategy that can be passed to the Experiment constructor. - """ - def export_fn(estimator, export_dir_base, checkpoint_path=None): - """Exports the given Estimator as a SavedModel and invokes post_export_fn. - - Args: - estimator: the Estimator to export. - export_dir_base: A string containing a directory to write the exported - graphs and checkpoint. - checkpoint_path: The checkpoint path to export. If None (the default), - the most recent checkpoint found within the model directory is chosen. - - Returns: - The string path to the SavedModel indicated by post_export_fn. - - Raises: - ValueError: If `estimator` is a `tf.estimator.Estimator` instance - and `default_output_alternative_key` was specified or if post_export_fn - does not return a valid directory. - RuntimeError: If unable to create temporary or final export directory. - """ - tmp_base_export_folder = 'temp-base-export-' + str(int(time.time())) - tmp_base_export_dir = os.path.join(export_dir_base, tmp_base_export_folder) - if gfile.Exists(tmp_base_export_dir): - raise RuntimeError('Failed to obtain base export directory') - gfile.MakeDirs(tmp_base_export_dir) - tmp_base_export = base_export_strategy.export( - estimator, tmp_base_export_dir, checkpoint_path) - - tmp_post_export_folder = 'temp-post-export-' + str(int(time.time())) - tmp_post_export_dir = os.path.join(export_dir_base, tmp_post_export_folder) - if gfile.Exists(tmp_post_export_dir): - raise RuntimeError('Failed to obtain temp export directory') - - gfile.MakeDirs(tmp_post_export_dir) - tmp_post_export = post_export_fn(tmp_base_export, tmp_post_export_dir) - - if not tmp_post_export.startswith(tmp_post_export_dir): - raise ValueError('post_export_fn must return a sub-directory of {}' - .format(tmp_post_export_dir)) - post_export_relpath = os.path.relpath(tmp_post_export, tmp_post_export_dir) - post_export = os.path.join(export_dir_base, post_export_relpath) - if gfile.Exists(post_export): - raise RuntimeError('Failed to obtain final export directory') - gfile.Rename(tmp_post_export, post_export) - - gfile.DeleteRecursively(tmp_base_export_dir) - gfile.DeleteRecursively(tmp_post_export_dir) - return post_export - - name = post_export_name if post_export_name else base_export_strategy.name - return export_strategy.ExportStrategy(name, export_fn) diff --git a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils_test.py b/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils_test.py deleted file mode 100644 index 14bf1136e8e..00000000000 --- a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils_test.py +++ /dev/null @@ -1,897 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests of utilities supporting export to SavedModel.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import tempfile -import time - -from tensorflow.contrib.layers.python.layers import feature_column as fc -from tensorflow.contrib.learn.python.learn import export_strategy as export_strategy_lib -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.utils import input_fn_utils -from tensorflow.contrib.learn.python.learn.utils import saved_model_export_utils -from tensorflow.core.framework import tensor_shape_pb2 -from tensorflow.core.framework import types_pb2 -from tensorflow.core.protobuf import meta_graph_pb2 -from tensorflow.python.estimator import estimator as core_estimator -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test -from tensorflow.python.saved_model import signature_constants -from tensorflow.python.saved_model import signature_def_utils -from tensorflow.python.util import compat - - -class TestEstimator(estimator.Estimator): - - def __init__(self, *args, **kwargs): - super(TestEstimator, self).__init__(*args, **kwargs) - self.last_exported_checkpoint = "" - self.last_exported_dir = "" - - # @Override - def export_savedmodel(self, - export_dir, - serving_input_fn, - default_output_alternative_key=None, - assets_extra=None, - as_text=False, - checkpoint_path=None, - strip_default_attrs=False): - - if not os.path.exists(export_dir): - os.makedirs(export_dir) - - open(os.path.join(export_dir, "placeholder.txt"), "a").close() - - self.last_exported_checkpoint = checkpoint_path - self.last_exported_dir = export_dir - - return export_dir - - -class SavedModelExportUtilsTest(test.TestCase): - - def test_build_standardized_signature_def_regression(self): - input_tensors = { - "input-1": - array_ops.placeholder(dtypes.string, 1, name="input-tensor-1") - } - output_tensors = { - "output-1": - array_ops.placeholder(dtypes.float32, 1, name="output-tensor-1") - } - problem_type = constants.ProblemType.LINEAR_REGRESSION - actual_signature_def = ( - saved_model_export_utils.build_standardized_signature_def( - input_tensors, output_tensors, problem_type)) - expected_signature_def = meta_graph_pb2.SignatureDef() - shape = tensor_shape_pb2.TensorShapeProto( - dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) - dtype_float = types_pb2.DataType.Value("DT_FLOAT") - dtype_string = types_pb2.DataType.Value("DT_STRING") - expected_signature_def.inputs[signature_constants.REGRESS_INPUTS].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype_string, tensor_shape=shape)) - expected_signature_def.outputs[ - signature_constants.REGRESS_OUTPUTS].CopyFrom( - meta_graph_pb2.TensorInfo( - name="output-tensor-1:0", dtype=dtype_float, - tensor_shape=shape)) - - expected_signature_def.method_name = signature_constants.REGRESS_METHOD_NAME - self.assertEqual(actual_signature_def, expected_signature_def) - - def test_build_standardized_signature_def_classification(self): - """Tests classification with one output tensor.""" - input_tensors = { - "input-1": - array_ops.placeholder(dtypes.string, 1, name="input-tensor-1") - } - output_tensors = { - "output-1": - array_ops.placeholder(dtypes.string, 1, name="output-tensor-1") - } - problem_type = constants.ProblemType.CLASSIFICATION - actual_signature_def = ( - saved_model_export_utils.build_standardized_signature_def( - input_tensors, output_tensors, problem_type)) - expected_signature_def = meta_graph_pb2.SignatureDef() - shape = tensor_shape_pb2.TensorShapeProto( - dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) - dtype_string = types_pb2.DataType.Value("DT_STRING") - expected_signature_def.inputs[signature_constants.CLASSIFY_INPUTS].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype_string, tensor_shape=shape)) - expected_signature_def.outputs[ - signature_constants.CLASSIFY_OUTPUT_CLASSES].CopyFrom( - meta_graph_pb2.TensorInfo( - name="output-tensor-1:0", - dtype=dtype_string, - tensor_shape=shape)) - - expected_signature_def.method_name = ( - signature_constants.CLASSIFY_METHOD_NAME) - self.assertEqual(actual_signature_def, expected_signature_def) - - def test_build_standardized_signature_def_classification2(self): - """Tests multiple output tensors that include classes and probabilities.""" - input_tensors = { - "input-1": - array_ops.placeholder(dtypes.string, 1, name="input-tensor-1") - } - output_tensors = { - "classes": - array_ops.placeholder( - dtypes.string, 1, name="output-tensor-classes"), - # Will be used for CLASSIFY_OUTPUT_SCORES. - "probabilities": - array_ops.placeholder( - dtypes.float32, 1, name="output-tensor-proba"), - "logits": - array_ops.placeholder( - dtypes.float32, 1, name="output-tensor-logits-unused"), - } - problem_type = constants.ProblemType.CLASSIFICATION - actual_signature_def = ( - saved_model_export_utils.build_standardized_signature_def( - input_tensors, output_tensors, problem_type)) - expected_signature_def = meta_graph_pb2.SignatureDef() - shape = tensor_shape_pb2.TensorShapeProto( - dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) - dtype_float = types_pb2.DataType.Value("DT_FLOAT") - dtype_string = types_pb2.DataType.Value("DT_STRING") - expected_signature_def.inputs[signature_constants.CLASSIFY_INPUTS].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype_string, tensor_shape=shape)) - expected_signature_def.outputs[ - signature_constants.CLASSIFY_OUTPUT_CLASSES].CopyFrom( - meta_graph_pb2.TensorInfo( - name="output-tensor-classes:0", - dtype=dtype_string, - tensor_shape=shape)) - expected_signature_def.outputs[ - signature_constants.CLASSIFY_OUTPUT_SCORES].CopyFrom( - meta_graph_pb2.TensorInfo( - name="output-tensor-proba:0", - dtype=dtype_float, - tensor_shape=shape)) - - expected_signature_def.method_name = ( - signature_constants.CLASSIFY_METHOD_NAME) - self.assertEqual(actual_signature_def, expected_signature_def) - - def test_build_standardized_signature_def_classification3(self): - """Tests multiple output tensors that include classes and scores.""" - input_tensors = { - "input-1": - array_ops.placeholder(dtypes.string, 1, name="input-tensor-1") - } - output_tensors = { - "classes": - array_ops.placeholder( - dtypes.string, 1, name="output-tensor-classes"), - "scores": - array_ops.placeholder( - dtypes.float32, 1, name="output-tensor-scores"), - "logits": - array_ops.placeholder( - dtypes.float32, 1, name="output-tensor-logits-unused"), - } - problem_type = constants.ProblemType.CLASSIFICATION - actual_signature_def = ( - saved_model_export_utils.build_standardized_signature_def( - input_tensors, output_tensors, problem_type)) - expected_signature_def = meta_graph_pb2.SignatureDef() - shape = tensor_shape_pb2.TensorShapeProto( - dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) - dtype_float = types_pb2.DataType.Value("DT_FLOAT") - dtype_string = types_pb2.DataType.Value("DT_STRING") - expected_signature_def.inputs[signature_constants.CLASSIFY_INPUTS].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype_string, tensor_shape=shape)) - expected_signature_def.outputs[ - signature_constants.CLASSIFY_OUTPUT_CLASSES].CopyFrom( - meta_graph_pb2.TensorInfo( - name="output-tensor-classes:0", - dtype=dtype_string, - tensor_shape=shape)) - expected_signature_def.outputs[ - signature_constants.CLASSIFY_OUTPUT_SCORES].CopyFrom( - meta_graph_pb2.TensorInfo( - name="output-tensor-scores:0", - dtype=dtype_float, - tensor_shape=shape)) - - expected_signature_def.method_name = ( - signature_constants.CLASSIFY_METHOD_NAME) - self.assertEqual(actual_signature_def, expected_signature_def) - - def test_build_standardized_signature_def_classification4(self): - """Tests classification without classes tensor.""" - input_tensors = { - "input-1": - array_ops.placeholder(dtypes.string, 1, name="input-tensor-1") - } - output_tensors = { - "probabilities": - array_ops.placeholder( - dtypes.float32, 1, name="output-tensor-proba"), - "logits": - array_ops.placeholder( - dtypes.float32, 1, name="output-tensor-logits-unused"), - } - problem_type = constants.ProblemType.CLASSIFICATION - actual_signature_def = ( - saved_model_export_utils.build_standardized_signature_def( - input_tensors, output_tensors, problem_type)) - expected_signature_def = meta_graph_pb2.SignatureDef() - shape = tensor_shape_pb2.TensorShapeProto( - dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) - dtype_float = types_pb2.DataType.Value("DT_FLOAT") - dtype_string = types_pb2.DataType.Value("DT_STRING") - expected_signature_def.inputs[signature_constants.CLASSIFY_INPUTS].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype_string, tensor_shape=shape)) - expected_signature_def.outputs[ - signature_constants.CLASSIFY_OUTPUT_SCORES].CopyFrom( - meta_graph_pb2.TensorInfo( - name="output-tensor-proba:0", - dtype=dtype_float, - tensor_shape=shape)) - - expected_signature_def.method_name = ( - signature_constants.CLASSIFY_METHOD_NAME) - self.assertEqual(actual_signature_def, expected_signature_def) - - def test_build_standardized_signature_def_classification5(self): - """Tests multiple output tensors that include integer classes and scores. - - Integer classes are dropped out, because Servo classification can only serve - string classes. So, only scores are present in the signature. - """ - input_tensors = { - "input-1": - array_ops.placeholder(dtypes.string, 1, name="input-tensor-1") - } - output_tensors = { - "classes": - array_ops.placeholder( - dtypes.int64, 1, name="output-tensor-classes"), - "scores": - array_ops.placeholder( - dtypes.float32, 1, name="output-tensor-scores"), - "logits": - array_ops.placeholder( - dtypes.float32, 1, name="output-tensor-logits-unused"), - } - problem_type = constants.ProblemType.CLASSIFICATION - actual_signature_def = ( - saved_model_export_utils.build_standardized_signature_def( - input_tensors, output_tensors, problem_type)) - expected_signature_def = meta_graph_pb2.SignatureDef() - shape = tensor_shape_pb2.TensorShapeProto( - dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) - dtype_float = types_pb2.DataType.Value("DT_FLOAT") - dtype_string = types_pb2.DataType.Value("DT_STRING") - expected_signature_def.inputs[signature_constants.CLASSIFY_INPUTS].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype_string, tensor_shape=shape)) - expected_signature_def.outputs[ - signature_constants.CLASSIFY_OUTPUT_SCORES].CopyFrom( - meta_graph_pb2.TensorInfo( - name="output-tensor-scores:0", - dtype=dtype_float, - tensor_shape=shape)) - - expected_signature_def.method_name = ( - signature_constants.CLASSIFY_METHOD_NAME) - self.assertEqual(actual_signature_def, expected_signature_def) - - def test_build_standardized_signature_def_classification6(self): - """Tests multiple output tensors that with integer classes and no scores. - - Servo classification cannot serve integer classes, but no scores are - available. So, we fall back to predict signature. - """ - input_tensors = { - "input-1": - array_ops.placeholder(dtypes.string, 1, name="input-tensor-1") - } - output_tensors = { - "classes": - array_ops.placeholder( - dtypes.int64, 1, name="output-tensor-classes"), - "logits": - array_ops.placeholder( - dtypes.float32, 1, name="output-tensor-logits"), - } - problem_type = constants.ProblemType.CLASSIFICATION - actual_signature_def = ( - saved_model_export_utils.build_standardized_signature_def( - input_tensors, output_tensors, problem_type)) - expected_signature_def = meta_graph_pb2.SignatureDef() - shape = tensor_shape_pb2.TensorShapeProto( - dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) - dtype_int64 = types_pb2.DataType.Value("DT_INT64") - dtype_float = types_pb2.DataType.Value("DT_FLOAT") - dtype_string = types_pb2.DataType.Value("DT_STRING") - expected_signature_def.inputs["input-1"].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype_string, tensor_shape=shape)) - expected_signature_def.outputs["classes"].CopyFrom( - meta_graph_pb2.TensorInfo( - name="output-tensor-classes:0", - dtype=dtype_int64, - tensor_shape=shape)) - expected_signature_def.outputs["logits"].CopyFrom( - meta_graph_pb2.TensorInfo( - name="output-tensor-logits:0", - dtype=dtype_float, - tensor_shape=shape)) - - expected_signature_def.method_name = ( - signature_constants.PREDICT_METHOD_NAME) - self.assertEqual(actual_signature_def, expected_signature_def) - - def test_get_input_alternatives(self): - input_ops = input_fn_utils.InputFnOps("bogus features dict", None, - "bogus default input dict") - - input_alternatives, _ = saved_model_export_utils.get_input_alternatives( - input_ops) - self.assertEqual(input_alternatives[ - saved_model_export_utils.DEFAULT_INPUT_ALTERNATIVE_KEY], - "bogus default input dict") - # self.assertEqual(input_alternatives[ - # saved_model_export_utils.FEATURES_INPUT_ALTERNATIVE_KEY], - # "bogus features dict") - - def test_get_output_alternatives_explicit_default(self): - provided_output_alternatives = { - "head-1": (constants.ProblemType.LINEAR_REGRESSION, - "bogus output dict"), - "head-2": (constants.ProblemType.CLASSIFICATION, "bogus output dict 2"), - "head-3": (constants.ProblemType.UNSPECIFIED, "bogus output dict 3"), - } - model_fn_ops = model_fn.ModelFnOps( - model_fn.ModeKeys.INFER, - predictions={"some_output": "bogus_tensor"}, - output_alternatives=provided_output_alternatives) - - output_alternatives, _ = saved_model_export_utils.get_output_alternatives( - model_fn_ops, "head-1") - - self.assertEqual(provided_output_alternatives, output_alternatives) - - def test_get_output_alternatives_wrong_default(self): - provided_output_alternatives = { - "head-1": (constants.ProblemType.LINEAR_REGRESSION, - "bogus output dict"), - "head-2": (constants.ProblemType.CLASSIFICATION, "bogus output dict 2"), - "head-3": (constants.ProblemType.UNSPECIFIED, "bogus output dict 3"), - } - model_fn_ops = model_fn.ModelFnOps( - model_fn.ModeKeys.INFER, - predictions={"some_output": "bogus_tensor"}, - output_alternatives=provided_output_alternatives) - - with self.assertRaises(ValueError) as e: - saved_model_export_utils.get_output_alternatives(model_fn_ops, "WRONG") - - self.assertEqual("Requested default_output_alternative: WRONG, but " - "available output_alternatives are: ['head-1', 'head-2', " - "'head-3']", str(e.exception)) - - def test_get_output_alternatives_single_no_default(self): - prediction_tensor = constant_op.constant(["bogus"]) - provided_output_alternatives = { - "head-1": (constants.ProblemType.LINEAR_REGRESSION, { - "output": prediction_tensor - }), - } - model_fn_ops = model_fn.ModelFnOps( - model_fn.ModeKeys.INFER, - predictions=prediction_tensor, - output_alternatives=provided_output_alternatives) - - output_alternatives, _ = saved_model_export_utils.get_output_alternatives( - model_fn_ops) - - self.assertEqual({ - "head-1": (constants.ProblemType.LINEAR_REGRESSION, { - "output": prediction_tensor - }) - }, output_alternatives) - - def test_get_output_alternatives_multi_no_default(self): - provided_output_alternatives = { - "head-1": (constants.ProblemType.LINEAR_REGRESSION, - "bogus output dict"), - "head-2": (constants.ProblemType.CLASSIFICATION, "bogus output dict 2"), - "head-3": (constants.ProblemType.UNSPECIFIED, "bogus output dict 3"), - } - model_fn_ops = model_fn.ModelFnOps( - model_fn.ModeKeys.INFER, - predictions={"some_output": "bogus_tensor"}, - output_alternatives=provided_output_alternatives) - - with self.assertRaises(ValueError) as e: - saved_model_export_utils.get_output_alternatives(model_fn_ops) - - self.assertEqual("Please specify a default_output_alternative. Available " - "output_alternatives are: ['head-1', 'head-2', 'head-3']", - str(e.exception)) - - def test_get_output_alternatives_none_provided(self): - prediction_tensor = constant_op.constant(["bogus"]) - model_fn_ops = model_fn.ModelFnOps( - model_fn.ModeKeys.INFER, - predictions={"some_output": prediction_tensor}, - output_alternatives=None) - - output_alternatives, _ = saved_model_export_utils.get_output_alternatives( - model_fn_ops) - - self.assertEqual({ - "default_output_alternative": (constants.ProblemType.UNSPECIFIED, { - "some_output": prediction_tensor - }) - }, output_alternatives) - - def test_get_output_alternatives_empty_provided_with_default(self): - prediction_tensor = constant_op.constant(["bogus"]) - model_fn_ops = model_fn.ModelFnOps( - model_fn.ModeKeys.INFER, - predictions={"some_output": prediction_tensor}, - output_alternatives={}) - - with self.assertRaises(ValueError) as e: - saved_model_export_utils.get_output_alternatives(model_fn_ops, "WRONG") - - self.assertEqual("Requested default_output_alternative: WRONG, but " - "available output_alternatives are: []", str(e.exception)) - - def test_get_output_alternatives_empty_provided_no_default(self): - prediction_tensor = constant_op.constant(["bogus"]) - model_fn_ops = model_fn.ModelFnOps( - model_fn.ModeKeys.INFER, - predictions={"some_output": prediction_tensor}, - output_alternatives={}) - - output_alternatives, _ = saved_model_export_utils.get_output_alternatives( - model_fn_ops) - - self.assertEqual({ - "default_output_alternative": (constants.ProblemType.UNSPECIFIED, { - "some_output": prediction_tensor - }) - }, output_alternatives) - - def test_get_output_alternatives_implicit_single(self): - prediction_tensor = constant_op.constant(["bogus"]) - model_fn_ops = model_fn.ModelFnOps( - model_fn.ModeKeys.INFER, - predictions=prediction_tensor, - output_alternatives=None) - - output_alternatives, _ = saved_model_export_utils.get_output_alternatives( - model_fn_ops) - self.assertEqual({ - "default_output_alternative": (constants.ProblemType.UNSPECIFIED, { - "output": prediction_tensor - }) - }, output_alternatives) - - def test_build_all_signature_defs(self): - input_features = constant_op.constant(["10"]) - input_example = constant_op.constant(["input string"]) - input_ops = input_fn_utils.InputFnOps({ - "features": input_features - }, None, { - "default input": input_example - }) - input_alternatives, _ = ( - saved_model_export_utils.get_input_alternatives(input_ops)) - output_1 = constant_op.constant([1.0]) - output_2 = constant_op.constant(["2"]) - output_3 = constant_op.constant(["3"]) - provided_output_alternatives = { - "head-1": (constants.ProblemType.LINEAR_REGRESSION, { - "some_output_1": output_1 - }), - "head-2": (constants.ProblemType.CLASSIFICATION, { - "some_output_2": output_2 - }), - "head-3": (constants.ProblemType.UNSPECIFIED, { - "some_output_3": output_3 - }), - } - model_fn_ops = model_fn.ModelFnOps( - model_fn.ModeKeys.INFER, - predictions={"some_output": constant_op.constant(["4"])}, - output_alternatives=provided_output_alternatives) - output_alternatives, _ = ( - saved_model_export_utils.get_output_alternatives( - model_fn_ops, "head-1")) - - signature_defs = saved_model_export_utils.build_all_signature_defs( - input_alternatives, output_alternatives, "head-1") - - expected_signature_defs = { - "serving_default": - signature_def_utils.regression_signature_def( - input_example, output_1), - "default_input_alternative:head-1": - signature_def_utils.regression_signature_def( - input_example, output_1), - "default_input_alternative:head-2": - signature_def_utils.classification_signature_def( - input_example, output_2, None), - "default_input_alternative:head-3": - signature_def_utils.predict_signature_def({ - "default input": input_example - }, { - "some_output_3": output_3 - }), - # "features_input_alternative:head-1": - # signature_def_utils.regression_signature_def(input_features, - # output_1), - # "features_input_alternative:head-2": - # signature_def_utils.classification_signature_def(input_features, - # output_2, None), - # "features_input_alternative:head-3": - # signature_def_utils.predict_signature_def({ - # "input": input_features - # }, {"output": output_3}), - } - - self.assertDictEqual(expected_signature_defs, signature_defs) - - def test_build_all_signature_defs_legacy_input_fn_not_supported(self): - """Tests that legacy input_fn returning (features, labels) raises error. - - serving_input_fn must return InputFnOps including a default input - alternative. - """ - input_features = constant_op.constant(["10"]) - input_ops = ({"features": input_features}, None) - input_alternatives, _ = ( - saved_model_export_utils.get_input_alternatives(input_ops)) - output_1 = constant_op.constant(["1"]) - output_2 = constant_op.constant(["2"]) - output_3 = constant_op.constant(["3"]) - provided_output_alternatives = { - "head-1": (constants.ProblemType.LINEAR_REGRESSION, { - "some_output_1": output_1 - }), - "head-2": (constants.ProblemType.CLASSIFICATION, { - "some_output_2": output_2 - }), - "head-3": (constants.ProblemType.UNSPECIFIED, { - "some_output_3": output_3 - }), - } - model_fn_ops = model_fn.ModelFnOps( - model_fn.ModeKeys.INFER, - predictions={"some_output": constant_op.constant(["4"])}, - output_alternatives=provided_output_alternatives) - output_alternatives, _ = ( - saved_model_export_utils.get_output_alternatives( - model_fn_ops, "head-1")) - - with self.assertRaisesRegexp( - ValueError, "A default input_alternative must be provided"): - saved_model_export_utils.build_all_signature_defs( - input_alternatives, output_alternatives, "head-1") - - def test_get_timestamped_export_dir(self): - export_dir_base = tempfile.mkdtemp() + "export/" - export_dir_1 = saved_model_export_utils.get_timestamped_export_dir( - export_dir_base) - time.sleep(2) - export_dir_2 = saved_model_export_utils.get_timestamped_export_dir( - export_dir_base) - time.sleep(2) - export_dir_3 = saved_model_export_utils.get_timestamped_export_dir( - export_dir_base) - - # Export directories should be named using a timestamp that is seconds - # since epoch. Such a timestamp is 10 digits long. - time_1 = os.path.basename(export_dir_1) - self.assertEqual(10, len(time_1)) - time_2 = os.path.basename(export_dir_2) - self.assertEqual(10, len(time_2)) - time_3 = os.path.basename(export_dir_3) - self.assertEqual(10, len(time_3)) - - self.assertTrue(int(time_1) < int(time_2)) - self.assertTrue(int(time_2) < int(time_3)) - - def test_garbage_collect_exports(self): - export_dir_base = tempfile.mkdtemp() + "export/" - gfile.MkDir(export_dir_base) - export_dir_1 = _create_test_export_dir(export_dir_base) - export_dir_2 = _create_test_export_dir(export_dir_base) - export_dir_3 = _create_test_export_dir(export_dir_base) - export_dir_4 = _create_test_export_dir(export_dir_base) - - self.assertTrue(gfile.Exists(export_dir_1)) - self.assertTrue(gfile.Exists(export_dir_2)) - self.assertTrue(gfile.Exists(export_dir_3)) - self.assertTrue(gfile.Exists(export_dir_4)) - - # Garbage collect all but the most recent 2 exports, - # where recency is determined based on the timestamp directory names. - saved_model_export_utils.garbage_collect_exports(export_dir_base, 2) - - self.assertFalse(gfile.Exists(export_dir_1)) - self.assertFalse(gfile.Exists(export_dir_2)) - self.assertTrue(gfile.Exists(export_dir_3)) - self.assertTrue(gfile.Exists(export_dir_4)) - - def test_get_most_recent_export(self): - export_dir_base = tempfile.mkdtemp() + "export/" - gfile.MkDir(export_dir_base) - _create_test_export_dir(export_dir_base) - _create_test_export_dir(export_dir_base) - _create_test_export_dir(export_dir_base) - export_dir_4 = _create_test_export_dir(export_dir_base) - - (most_recent_export_dir, most_recent_export_version) = ( - saved_model_export_utils.get_most_recent_export(export_dir_base)) - - self.assertEqual( - compat.as_bytes(export_dir_4), compat.as_bytes(most_recent_export_dir)) - self.assertEqual( - compat.as_bytes(export_dir_4), - os.path.join( - compat.as_bytes(export_dir_base), - compat.as_bytes(str(most_recent_export_version)))) - - def test_make_export_strategy(self): - """Only tests that an ExportStrategy instance is created.""" - - def _serving_input_fn(): - return array_ops.constant([1]), None - - export_strategy = saved_model_export_utils.make_export_strategy( - serving_input_fn=_serving_input_fn, - default_output_alternative_key="default", - assets_extra={"from/path": "to/path"}, - as_text=False, - exports_to_keep=5) - self.assertTrue( - isinstance(export_strategy, export_strategy_lib.ExportStrategy)) - - def test_make_parsing_export_strategy(self): - """Only tests that an ExportStrategy instance is created.""" - sparse_col = fc.sparse_column_with_hash_bucket( - "sparse_column", hash_bucket_size=100) - embedding_col = fc.embedding_column( - fc.sparse_column_with_hash_bucket( - "sparse_column_for_embedding", hash_bucket_size=10), - dimension=4) - real_valued_col1 = fc.real_valued_column("real_valued_column1") - bucketized_col1 = fc.bucketized_column( - fc.real_valued_column("real_valued_column_for_bucketization1"), [0, 4]) - feature_columns = [ - sparse_col, embedding_col, real_valued_col1, bucketized_col1 - ] - - export_strategy = saved_model_export_utils.make_parsing_export_strategy( - feature_columns=feature_columns) - self.assertTrue( - isinstance(export_strategy, export_strategy_lib.ExportStrategy)) - - def test_make_best_model_export_strategy(self): - export_dir_base = tempfile.mkdtemp() + "export/" - gfile.MkDir(export_dir_base) - - test_estimator = TestEstimator() - export_strategy = saved_model_export_utils.make_best_model_export_strategy( - serving_input_fn=None, exports_to_keep=3, compare_fn=None) - - self.assertNotEqual("", - export_strategy.export(test_estimator, export_dir_base, - "fake_ckpt_0", { - "loss": 100 - })) - self.assertNotEqual("", test_estimator.last_exported_dir) - self.assertNotEqual("", test_estimator.last_exported_checkpoint) - - self.assertEqual("", - export_strategy.export(test_estimator, export_dir_base, - "fake_ckpt_1", { - "loss": 101 - })) - self.assertEqual(test_estimator.last_exported_dir, - os.path.join(export_dir_base, "fake_ckpt_0")) - - self.assertNotEqual("", - export_strategy.export(test_estimator, export_dir_base, - "fake_ckpt_2", { - "loss": 10 - })) - self.assertEqual(test_estimator.last_exported_dir, - os.path.join(export_dir_base, "fake_ckpt_2")) - - self.assertEqual("", - export_strategy.export(test_estimator, export_dir_base, - "fake_ckpt_3", { - "loss": 20 - })) - self.assertEqual(test_estimator.last_exported_dir, - os.path.join(export_dir_base, "fake_ckpt_2")) - - def test_make_best_model_export_strategy_with_preemption(self): - model_dir = self.get_temp_dir() - eval_dir_base = os.path.join(model_dir, "eval_continuous") - core_estimator._write_dict_to_summary(eval_dir_base, {"loss": 50}, 1) - core_estimator._write_dict_to_summary(eval_dir_base, {"loss": 60}, 2) - - test_estimator = TestEstimator() - export_strategy = saved_model_export_utils.make_best_model_export_strategy( - serving_input_fn=None, - exports_to_keep=3, - model_dir=model_dir, - event_file_pattern="eval_continuous/*.tfevents.*", - compare_fn=None) - - export_dir_base = os.path.join(self.get_temp_dir(), "export") - self.assertEqual("", - export_strategy.export(test_estimator, export_dir_base, - "fake_ckpt_0", { - "loss": 100 - })) - self.assertEqual("", test_estimator.last_exported_dir) - self.assertEqual("", test_estimator.last_exported_checkpoint) - - self.assertNotEqual("", - export_strategy.export(test_estimator, export_dir_base, - "fake_ckpt_2", { - "loss": 10 - })) - self.assertEqual(test_estimator.last_exported_dir, - os.path.join(export_dir_base, "fake_ckpt_2")) - - self.assertEqual("", - export_strategy.export(test_estimator, export_dir_base, - "fake_ckpt_3", { - "loss": 20 - })) - self.assertEqual(test_estimator.last_exported_dir, - os.path.join(export_dir_base, "fake_ckpt_2")) - - def test_make_best_model_export_strategy_exceptions(self): - export_dir_base = tempfile.mkdtemp() + "export/" - - test_estimator = TestEstimator() - export_strategy = saved_model_export_utils.make_best_model_export_strategy( - serving_input_fn=None, exports_to_keep=3, compare_fn=None) - - with self.assertRaises(ValueError): - export_strategy.export(test_estimator, export_dir_base, "", {"loss": 200}) - - with self.assertRaises(ValueError): - export_strategy.export(test_estimator, export_dir_base, "fake_ckpt_1", - None) - - def test_extend_export_strategy(self): - - def _base_export_fn(unused_estimator, - export_dir_base, - unused_checkpoint_path=None): - base_path = os.path.join(export_dir_base, "e1") - gfile.MkDir(base_path) - return base_path - - def _post_export_fn(orig_path, new_path): - assert orig_path.endswith("/e1") - post_export_path = os.path.join(new_path, "rewrite") - gfile.MkDir(post_export_path) - return post_export_path - - base_export_strategy = export_strategy_lib.ExportStrategy( - "Servo", _base_export_fn) - - final_export_strategy = saved_model_export_utils.extend_export_strategy( - base_export_strategy, _post_export_fn, "Servo2") - self.assertEqual(final_export_strategy.name, "Servo2") - - test_estimator = TestEstimator() - tmpdir = tempfile.mkdtemp() - export_model_dir = os.path.join(tmpdir, "model") - checkpoint_path = os.path.join(tmpdir, "checkpoint") - final_path = final_export_strategy.export(test_estimator, export_model_dir, - checkpoint_path) - self.assertEqual(os.path.join(export_model_dir, "rewrite"), final_path) - - def test_extend_export_strategy_same_name(self): - - def _base_export_fn(unused_estimator, - export_dir_base, - unused_checkpoint_path=None): - base_path = os.path.join(export_dir_base, "e1") - gfile.MkDir(base_path) - return base_path - - def _post_export_fn(orig_path, new_path): - assert orig_path.endswith("/e1") - post_export_path = os.path.join(new_path, "rewrite") - gfile.MkDir(post_export_path) - return post_export_path - - base_export_strategy = export_strategy_lib.ExportStrategy( - "Servo", _base_export_fn) - - final_export_strategy = saved_model_export_utils.extend_export_strategy( - base_export_strategy, _post_export_fn) - self.assertEqual(final_export_strategy.name, "Servo") - - test_estimator = TestEstimator() - tmpdir = tempfile.mkdtemp() - export_model_dir = os.path.join(tmpdir, "model") - checkpoint_path = os.path.join(tmpdir, "checkpoint") - final_path = final_export_strategy.export(test_estimator, export_model_dir, - checkpoint_path) - self.assertEqual(os.path.join(export_model_dir, "rewrite"), final_path) - - def test_extend_export_strategy_raises_error(self): - - def _base_export_fn(unused_estimator, - export_dir_base, - unused_checkpoint_path=None): - base_path = os.path.join(export_dir_base, "e1") - gfile.MkDir(base_path) - return base_path - - def _post_export_fn(unused_orig_path, unused_new_path): - return tempfile.mkdtemp() - - base_export_strategy = export_strategy_lib.ExportStrategy( - "Servo", _base_export_fn) - - final_export_strategy = saved_model_export_utils.extend_export_strategy( - base_export_strategy, _post_export_fn) - - test_estimator = TestEstimator() - tmpdir = tempfile.mkdtemp() - with self.assertRaises(ValueError) as ve: - final_export_strategy.export(test_estimator, tmpdir, - os.path.join(tmpdir, "checkpoint")) - - self.assertTrue( - "post_export_fn must return a sub-directory" in str(ve.exception)) - - -def _create_test_export_dir(export_dir_base): - export_dir = saved_model_export_utils.get_timestamped_export_dir( - export_dir_base) - gfile.MkDir(export_dir) - time.sleep(2) - return export_dir - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/legacy_seq2seq/BUILD b/tensorflow/contrib/legacy_seq2seq/BUILD deleted file mode 100644 index d31557e56ff..00000000000 --- a/tensorflow/contrib/legacy_seq2seq/BUILD +++ /dev/null @@ -1,66 +0,0 @@ -# Description: -# Contains library to create sequence-to-sequence models on top of TensorFlow. -# APIs here are meant to evolve over time. - -load("//tensorflow:tensorflow.bzl", "cuda_py_tests") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "seq2seq_py", - srcs = [ - "__init__.py", - "python/__init__.py", - ] + glob( - ["python/ops/*.py"], - exclude = ["python/ops/**/*_test.py"], - ), - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:rnn", - "//tensorflow/python:rnn_cell", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - ], -) - -cuda_py_tests( - name = "seq2seq_test", - size = "medium", - srcs = ["python/kernel_tests/seq2seq_test.py"], - additional_deps = [ - ":seq2seq_py", - "//third_party/py/numpy", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:clip_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:gradients", - "//tensorflow/python:init_ops", - "//tensorflow/python:nn", - "//tensorflow/python:rnn", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], - tags = [ - "noasan", # times out b/63678675 - "optonly", # times out (flaky) - ], -) diff --git a/tensorflow/contrib/legacy_seq2seq/__init__.py b/tensorflow/contrib/legacy_seq2seq/__init__.py deleted file mode 100644 index 75069fe9505..00000000000 --- a/tensorflow/contrib/legacy_seq2seq/__init__.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Deprecated library for creating sequence-to-sequence models in TensorFlow. - -@@attention_decoder -@@basic_rnn_seq2seq -@@embedding_attention_decoder -@@embedding_attention_seq2seq -@@embedding_rnn_decoder -@@embedding_rnn_seq2seq -@@embedding_tied_rnn_seq2seq -@@model_with_buckets -@@one2many_rnn_seq2seq -@@rnn_decoder -@@sequence_loss -@@sequence_loss_by_example -@@tied_rnn_seq2seq -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import attention_decoder -from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import basic_rnn_seq2seq -from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import embedding_attention_decoder -from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import embedding_attention_seq2seq -from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import embedding_rnn_decoder -from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import embedding_rnn_seq2seq -from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import embedding_tied_rnn_seq2seq -from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import model_with_buckets -from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import one2many_rnn_seq2seq -from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import rnn_decoder -from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import sequence_loss -from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import sequence_loss_by_example -from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import tied_rnn_seq2seq - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/legacy_seq2seq/python/__init__.py b/tensorflow/contrib/legacy_seq2seq/python/__init__.py deleted file mode 100644 index 52e83069cb0..00000000000 --- a/tensorflow/contrib/legacy_seq2seq/python/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/legacy_seq2seq/python/kernel_tests/__init__.py b/tensorflow/contrib/legacy_seq2seq/python/kernel_tests/__init__.py deleted file mode 100644 index 52e83069cb0..00000000000 --- a/tensorflow/contrib/legacy_seq2seq/python/kernel_tests/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/legacy_seq2seq/python/kernel_tests/seq2seq_test.py b/tensorflow/contrib/legacy_seq2seq/python/kernel_tests/seq2seq_test.py deleted file mode 100644 index 0e5ea6b9f71..00000000000 --- a/tensorflow/contrib/legacy_seq2seq/python/kernel_tests/seq2seq_test.py +++ /dev/null @@ -1,1100 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for functional style sequence-to-sequence models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import math -import random - -import numpy as np - -from tensorflow.contrib.legacy_seq2seq.python.ops import seq2seq as seq2seq_lib -from tensorflow.contrib.rnn.python.ops import core_rnn_cell -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import rnn -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import adam - - -class Seq2SeqTest(test.TestCase): - - def testRNNDecoder(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - inp = [constant_op.constant(0.5, shape=[2, 2])] * 2 - _, enc_state = rnn.static_rnn( - rnn_cell.GRUCell(2), inp, dtype=dtypes.float32) - dec_inp = [constant_op.constant(0.4, shape=[2, 2])] * 3 - cell = core_rnn_cell.OutputProjectionWrapper(rnn_cell.GRUCell(2), 4) - dec, mem = seq2seq_lib.rnn_decoder(dec_inp, enc_state, cell) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 4), res[0].shape) - - res = sess.run([mem]) - self.assertEqual((2, 2), res[0].shape) - - def testBasicRNNSeq2Seq(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - inp = [constant_op.constant(0.5, shape=[2, 2])] * 2 - dec_inp = [constant_op.constant(0.4, shape=[2, 2])] * 3 - cell = core_rnn_cell.OutputProjectionWrapper(rnn_cell.GRUCell(2), 4) - dec, mem = seq2seq_lib.basic_rnn_seq2seq(inp, dec_inp, cell) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 4), res[0].shape) - - res = sess.run([mem]) - self.assertEqual((2, 2), res[0].shape) - - def testTiedRNNSeq2Seq(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - inp = [constant_op.constant(0.5, shape=[2, 2])] * 2 - dec_inp = [constant_op.constant(0.4, shape=[2, 2])] * 3 - cell = core_rnn_cell.OutputProjectionWrapper(rnn_cell.GRUCell(2), 4) - dec, mem = seq2seq_lib.tied_rnn_seq2seq(inp, dec_inp, cell) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 4), res[0].shape) - - res = sess.run([mem]) - self.assertEqual(1, len(res)) - self.assertEqual((2, 2), res[0].shape) - - def testEmbeddingRNNDecoder(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - inp = [constant_op.constant(0.5, shape=[2, 2])] * 2 - cell_fn = lambda: rnn_cell.BasicLSTMCell(2) - cell = cell_fn() - _, enc_state = rnn.static_rnn(cell, inp, dtype=dtypes.float32) - dec_inp = [ - constant_op.constant( - i, dtypes.int32, shape=[2]) for i in range(3) - ] - # Use a new cell instance since the attention decoder uses a - # different variable scope. - dec, mem = seq2seq_lib.embedding_rnn_decoder( - dec_inp, enc_state, cell_fn(), num_symbols=4, embedding_size=2) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 2), res[0].shape) - - res = sess.run([mem]) - self.assertEqual(1, len(res)) - self.assertEqual((2, 2), res[0].c.shape) - self.assertEqual((2, 2), res[0].h.shape) - - def testEmbeddingRNNSeq2Seq(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - enc_inp = [ - constant_op.constant( - 1, dtypes.int32, shape=[2]) for i in range(2) - ] - dec_inp = [ - constant_op.constant( - i, dtypes.int32, shape=[2]) for i in range(3) - ] - cell_fn = lambda: rnn_cell.BasicLSTMCell(2) - cell = cell_fn() - dec, mem = seq2seq_lib.embedding_rnn_seq2seq( - enc_inp, - dec_inp, - cell, - num_encoder_symbols=2, - num_decoder_symbols=5, - embedding_size=2) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 5), res[0].shape) - - res = sess.run([mem]) - self.assertEqual((2, 2), res[0].c.shape) - self.assertEqual((2, 2), res[0].h.shape) - - # Test with state_is_tuple=False. - with variable_scope.variable_scope("no_tuple"): - cell_nt = rnn_cell.BasicLSTMCell(2, state_is_tuple=False) - dec, mem = seq2seq_lib.embedding_rnn_seq2seq( - enc_inp, - dec_inp, - cell_nt, - num_encoder_symbols=2, - num_decoder_symbols=5, - embedding_size=2) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 5), res[0].shape) - - res = sess.run([mem]) - self.assertEqual((2, 4), res[0].shape) - - # Test externally provided output projection. - w = variable_scope.get_variable("proj_w", [2, 5]) - b = variable_scope.get_variable("proj_b", [5]) - with variable_scope.variable_scope("proj_seq2seq"): - dec, _ = seq2seq_lib.embedding_rnn_seq2seq( - enc_inp, - dec_inp, - cell_fn(), - num_encoder_symbols=2, - num_decoder_symbols=5, - embedding_size=2, - output_projection=(w, b)) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 2), res[0].shape) - - # Test that previous-feeding model ignores inputs after the first. - dec_inp2 = [ - constant_op.constant( - 0, dtypes.int32, shape=[2]) for _ in range(3) - ] - with variable_scope.variable_scope("other"): - d3, _ = seq2seq_lib.embedding_rnn_seq2seq( - enc_inp, - dec_inp2, - cell_fn(), - num_encoder_symbols=2, - num_decoder_symbols=5, - embedding_size=2, - feed_previous=constant_op.constant(True)) - with variable_scope.variable_scope("other_2"): - d1, _ = seq2seq_lib.embedding_rnn_seq2seq( - enc_inp, - dec_inp, - cell_fn(), - num_encoder_symbols=2, - num_decoder_symbols=5, - embedding_size=2, - feed_previous=True) - with variable_scope.variable_scope("other_3"): - d2, _ = seq2seq_lib.embedding_rnn_seq2seq( - enc_inp, - dec_inp2, - cell_fn(), - num_encoder_symbols=2, - num_decoder_symbols=5, - embedding_size=2, - feed_previous=True) - sess.run([variables.global_variables_initializer()]) - res1 = sess.run(d1) - res2 = sess.run(d2) - res3 = sess.run(d3) - self.assertAllClose(res1, res2) - self.assertAllClose(res1, res3) - - def testEmbeddingTiedRNNSeq2Seq(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - enc_inp = [ - constant_op.constant( - 1, dtypes.int32, shape=[2]) for i in range(2) - ] - dec_inp = [ - constant_op.constant( - i, dtypes.int32, shape=[2]) for i in range(3) - ] - cell = functools.partial(rnn_cell.BasicLSTMCell, 2, state_is_tuple=True) - dec, mem = seq2seq_lib.embedding_tied_rnn_seq2seq( - enc_inp, dec_inp, cell(), num_symbols=5, embedding_size=2) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 5), res[0].shape) - - res = sess.run([mem]) - self.assertEqual((2, 2), res[0].c.shape) - self.assertEqual((2, 2), res[0].h.shape) - - # Test when num_decoder_symbols is provided, the size of decoder output - # is num_decoder_symbols. - with variable_scope.variable_scope("decoder_symbols_seq2seq"): - dec, mem = seq2seq_lib.embedding_tied_rnn_seq2seq( - enc_inp, - dec_inp, - cell(), - num_symbols=5, - num_decoder_symbols=3, - embedding_size=2) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 3), res[0].shape) - - # Test externally provided output projection. - w = variable_scope.get_variable("proj_w", [2, 5]) - b = variable_scope.get_variable("proj_b", [5]) - with variable_scope.variable_scope("proj_seq2seq"): - dec, _ = seq2seq_lib.embedding_tied_rnn_seq2seq( - enc_inp, - dec_inp, - cell(), - num_symbols=5, - embedding_size=2, - output_projection=(w, b)) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 2), res[0].shape) - - # Test that previous-feeding model ignores inputs after the first. - dec_inp2 = [constant_op.constant(0, dtypes.int32, shape=[2])] * 3 - with variable_scope.variable_scope("other"): - d3, _ = seq2seq_lib.embedding_tied_rnn_seq2seq( - enc_inp, - dec_inp2, - cell(), - num_symbols=5, - embedding_size=2, - feed_previous=constant_op.constant(True)) - with variable_scope.variable_scope("other_2"): - d1, _ = seq2seq_lib.embedding_tied_rnn_seq2seq( - enc_inp, - dec_inp, - cell(), - num_symbols=5, - embedding_size=2, - feed_previous=True) - with variable_scope.variable_scope("other_3"): - d2, _ = seq2seq_lib.embedding_tied_rnn_seq2seq( - enc_inp, - dec_inp2, - cell(), - num_symbols=5, - embedding_size=2, - feed_previous=True) - sess.run([variables.global_variables_initializer()]) - res1 = sess.run(d1) - res2 = sess.run(d2) - res3 = sess.run(d3) - self.assertAllClose(res1, res2) - self.assertAllClose(res1, res3) - - def testAttentionDecoder1(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - cell_fn = lambda: rnn_cell.GRUCell(2) - cell = cell_fn() - inp = [constant_op.constant(0.5, shape=[2, 2])] * 2 - enc_outputs, enc_state = rnn.static_rnn(cell, inp, dtype=dtypes.float32) - attn_states = array_ops.concat([ - array_ops.reshape(e, [-1, 1, cell.output_size]) for e in enc_outputs - ], 1) - dec_inp = [constant_op.constant(0.4, shape=[2, 2])] * 3 - - # Create a new cell instance for the decoder, since it uses a - # different variable scope - dec, mem = seq2seq_lib.attention_decoder( - dec_inp, enc_state, attn_states, cell_fn(), output_size=4) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 4), res[0].shape) - - res = sess.run([mem]) - self.assertEqual((2, 2), res[0].shape) - - def testAttentionDecoder2(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - cell_fn = lambda: rnn_cell.GRUCell(2) - cell = cell_fn() - inp = [constant_op.constant(0.5, shape=[2, 2])] * 2 - enc_outputs, enc_state = rnn.static_rnn(cell, inp, dtype=dtypes.float32) - attn_states = array_ops.concat([ - array_ops.reshape(e, [-1, 1, cell.output_size]) for e in enc_outputs - ], 1) - dec_inp = [constant_op.constant(0.4, shape=[2, 2])] * 3 - - # Use a new cell instance since the attention decoder uses a - # different variable scope. - dec, mem = seq2seq_lib.attention_decoder( - dec_inp, enc_state, attn_states, cell_fn(), - output_size=4, num_heads=2) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 4), res[0].shape) - - res = sess.run([mem]) - self.assertEqual((2, 2), res[0].shape) - - def testDynamicAttentionDecoder1(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - cell_fn = lambda: rnn_cell.GRUCell(2) - cell = cell_fn() - inp = constant_op.constant(0.5, shape=[2, 2, 2]) - enc_outputs, enc_state = rnn.dynamic_rnn( - cell, inp, dtype=dtypes.float32) - attn_states = enc_outputs - dec_inp = [constant_op.constant(0.4, shape=[2, 2])] * 3 - - # Use a new cell instance since the attention decoder uses a - # different variable scope. - dec, mem = seq2seq_lib.attention_decoder( - dec_inp, enc_state, attn_states, cell_fn(), output_size=4) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 4), res[0].shape) - - res = sess.run([mem]) - self.assertEqual((2, 2), res[0].shape) - - def testDynamicAttentionDecoder2(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - cell_fn = lambda: rnn_cell.GRUCell(2) - cell = cell_fn() - inp = constant_op.constant(0.5, shape=[2, 2, 2]) - enc_outputs, enc_state = rnn.dynamic_rnn( - cell, inp, dtype=dtypes.float32) - attn_states = enc_outputs - dec_inp = [constant_op.constant(0.4, shape=[2, 2])] * 3 - - # Use a new cell instance since the attention decoder uses a - # different variable scope. - dec, mem = seq2seq_lib.attention_decoder( - dec_inp, enc_state, attn_states, cell_fn(), - output_size=4, num_heads=2) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 4), res[0].shape) - - res = sess.run([mem]) - self.assertEqual((2, 2), res[0].shape) - - def testAttentionDecoderStateIsTuple(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - single_cell = lambda: rnn_cell.BasicLSTMCell( # pylint: disable=g-long-lambda - 2, state_is_tuple=True) - cell_fn = lambda: rnn_cell.MultiRNNCell( # pylint: disable=g-long-lambda - cells=[single_cell() for _ in range(2)], state_is_tuple=True) - cell = cell_fn() - inp = [constant_op.constant(0.5, shape=[2, 2])] * 2 - enc_outputs, enc_state = rnn.static_rnn(cell, inp, dtype=dtypes.float32) - attn_states = array_ops.concat([ - array_ops.reshape(e, [-1, 1, cell.output_size]) for e in enc_outputs - ], 1) - dec_inp = [constant_op.constant(0.4, shape=[2, 2])] * 3 - - # Use a new cell instance since the attention decoder uses a - # different variable scope. - dec, mem = seq2seq_lib.attention_decoder( - dec_inp, enc_state, attn_states, cell_fn(), output_size=4) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 4), res[0].shape) - - res = sess.run([mem]) - self.assertEqual(2, len(res[0])) - self.assertEqual((2, 2), res[0][0].c.shape) - self.assertEqual((2, 2), res[0][0].h.shape) - self.assertEqual((2, 2), res[0][1].c.shape) - self.assertEqual((2, 2), res[0][1].h.shape) - - def testDynamicAttentionDecoderStateIsTuple(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - cell_fn = lambda: rnn_cell.MultiRNNCell( # pylint: disable=g-long-lambda - cells=[rnn_cell.BasicLSTMCell(2) for _ in range(2)]) - cell = cell_fn() - inp = [constant_op.constant(0.5, shape=[2, 2])] * 2 - enc_outputs, enc_state = rnn.static_rnn(cell, inp, dtype=dtypes.float32) - attn_states = array_ops.concat([ - array_ops.reshape(e, [-1, 1, cell.output_size]) - for e in enc_outputs - ], 1) - dec_inp = [constant_op.constant(0.4, shape=[2, 2])] * 3 - - # Use a new cell instance since the attention decoder uses a - # different variable scope. - dec, mem = seq2seq_lib.attention_decoder( - dec_inp, enc_state, attn_states, cell_fn(), output_size=4) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 4), res[0].shape) - - res = sess.run([mem]) - self.assertEqual(2, len(res[0])) - self.assertEqual((2, 2), res[0][0].c.shape) - self.assertEqual((2, 2), res[0][0].h.shape) - self.assertEqual((2, 2), res[0][1].c.shape) - self.assertEqual((2, 2), res[0][1].h.shape) - - def testEmbeddingAttentionDecoder(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - inp = [constant_op.constant(0.5, shape=[2, 2])] * 2 - cell_fn = lambda: rnn_cell.GRUCell(2) - cell = cell_fn() - enc_outputs, enc_state = rnn.static_rnn(cell, inp, dtype=dtypes.float32) - attn_states = array_ops.concat([ - array_ops.reshape(e, [-1, 1, cell.output_size]) for e in enc_outputs - ], 1) - dec_inp = [ - constant_op.constant( - i, dtypes.int32, shape=[2]) for i in range(3) - ] - - # Use a new cell instance since the attention decoder uses a - # different variable scope. - dec, mem = seq2seq_lib.embedding_attention_decoder( - dec_inp, - enc_state, - attn_states, - cell_fn(), - num_symbols=4, - embedding_size=2, - output_size=3) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 3), res[0].shape) - - res = sess.run([mem]) - self.assertEqual((2, 2), res[0].shape) - - def testEmbeddingAttentionSeq2Seq(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - enc_inp = [ - constant_op.constant( - 1, dtypes.int32, shape=[2]) for i in range(2) - ] - dec_inp = [ - constant_op.constant( - i, dtypes.int32, shape=[2]) for i in range(3) - ] - cell_fn = lambda: rnn_cell.BasicLSTMCell(2) - cell = cell_fn() - dec, mem = seq2seq_lib.embedding_attention_seq2seq( - enc_inp, - dec_inp, - cell, - num_encoder_symbols=2, - num_decoder_symbols=5, - embedding_size=2) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 5), res[0].shape) - - res = sess.run([mem]) - self.assertEqual((2, 2), res[0].c.shape) - self.assertEqual((2, 2), res[0].h.shape) - - # Test with state_is_tuple=False. - with variable_scope.variable_scope("no_tuple"): - cell_fn = functools.partial( - rnn_cell.BasicLSTMCell, 2, state_is_tuple=False) - cell_nt = cell_fn() - dec, mem = seq2seq_lib.embedding_attention_seq2seq( - enc_inp, - dec_inp, - cell_nt, - num_encoder_symbols=2, - num_decoder_symbols=5, - embedding_size=2) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 5), res[0].shape) - - res = sess.run([mem]) - self.assertEqual((2, 4), res[0].shape) - - # Test externally provided output projection. - w = variable_scope.get_variable("proj_w", [2, 5]) - b = variable_scope.get_variable("proj_b", [5]) - with variable_scope.variable_scope("proj_seq2seq"): - dec, _ = seq2seq_lib.embedding_attention_seq2seq( - enc_inp, - dec_inp, - cell_fn(), - num_encoder_symbols=2, - num_decoder_symbols=5, - embedding_size=2, - output_projection=(w, b)) - sess.run([variables.global_variables_initializer()]) - res = sess.run(dec) - self.assertEqual(3, len(res)) - self.assertEqual((2, 2), res[0].shape) - - # TODO(ebrevdo, lukaszkaiser): Re-enable once RNNCells allow reuse - # within a variable scope that already has a weights tensor. - # - # # Test that previous-feeding model ignores inputs after the first. - # dec_inp2 = [ - # constant_op.constant( - # 0, dtypes.int32, shape=[2]) for _ in range(3) - # ] - # with variable_scope.variable_scope("other"): - # d3, _ = seq2seq_lib.embedding_attention_seq2seq( - # enc_inp, - # dec_inp2, - # cell_fn(), - # num_encoder_symbols=2, - # num_decoder_symbols=5, - # embedding_size=2, - # feed_previous=constant_op.constant(True)) - # sess.run([variables.global_variables_initializer()]) - # variable_scope.get_variable_scope().reuse_variables() - # cell = cell_fn() - # d1, _ = seq2seq_lib.embedding_attention_seq2seq( - # enc_inp, - # dec_inp, - # cell, - # num_encoder_symbols=2, - # num_decoder_symbols=5, - # embedding_size=2, - # feed_previous=True) - # d2, _ = seq2seq_lib.embedding_attention_seq2seq( - # enc_inp, - # dec_inp2, - # cell, - # num_encoder_symbols=2, - # num_decoder_symbols=5, - # embedding_size=2, - # feed_previous=True) - # res1 = sess.run(d1) - # res2 = sess.run(d2) - # res3 = sess.run(d3) - # self.assertAllClose(res1, res2) - # self.assertAllClose(res1, res3) - - def testOne2ManyRNNSeq2Seq(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - enc_inp = [ - constant_op.constant( - 1, dtypes.int32, shape=[2]) for i in range(2) - ] - dec_inp_dict = {} - dec_inp_dict["0"] = [ - constant_op.constant( - i, dtypes.int32, shape=[2]) for i in range(3) - ] - dec_inp_dict["1"] = [ - constant_op.constant( - i, dtypes.int32, shape=[2]) for i in range(4) - ] - dec_symbols_dict = {"0": 5, "1": 6} - def EncCellFn(): - return rnn_cell.BasicLSTMCell(2, state_is_tuple=True) - def DecCellsFn(): - return dict((k, rnn_cell.BasicLSTMCell(2, state_is_tuple=True)) - for k in dec_symbols_dict) - outputs_dict, state_dict = (seq2seq_lib.one2many_rnn_seq2seq( - enc_inp, dec_inp_dict, EncCellFn(), DecCellsFn(), - 2, dec_symbols_dict, embedding_size=2)) - - sess.run([variables.global_variables_initializer()]) - res = sess.run(outputs_dict["0"]) - self.assertEqual(3, len(res)) - self.assertEqual((2, 5), res[0].shape) - res = sess.run(outputs_dict["1"]) - self.assertEqual(4, len(res)) - self.assertEqual((2, 6), res[0].shape) - res = sess.run([state_dict["0"]]) - self.assertEqual((2, 2), res[0].c.shape) - self.assertEqual((2, 2), res[0].h.shape) - res = sess.run([state_dict["1"]]) - self.assertEqual((2, 2), res[0].c.shape) - self.assertEqual((2, 2), res[0].h.shape) - - # Test that previous-feeding model ignores inputs after the first, i.e. - # dec_inp_dict2 has different inputs from dec_inp_dict after the first - # time-step. - dec_inp_dict2 = {} - dec_inp_dict2["0"] = [ - constant_op.constant( - 0, dtypes.int32, shape=[2]) for _ in range(3) - ] - dec_inp_dict2["1"] = [ - constant_op.constant( - 0, dtypes.int32, shape=[2]) for _ in range(4) - ] - with variable_scope.variable_scope("other"): - outputs_dict3, _ = seq2seq_lib.one2many_rnn_seq2seq( - enc_inp, - dec_inp_dict2, - EncCellFn(), - DecCellsFn(), - 2, - dec_symbols_dict, - embedding_size=2, - feed_previous=constant_op.constant(True)) - with variable_scope.variable_scope("other_2"): - outputs_dict1, _ = seq2seq_lib.one2many_rnn_seq2seq( - enc_inp, - dec_inp_dict, - EncCellFn(), - DecCellsFn(), - 2, - dec_symbols_dict, - embedding_size=2, - feed_previous=True) - with variable_scope.variable_scope("other_3"): - outputs_dict2, _ = seq2seq_lib.one2many_rnn_seq2seq( - enc_inp, - dec_inp_dict2, - EncCellFn(), - DecCellsFn(), - 2, - dec_symbols_dict, - embedding_size=2, - feed_previous=True) - sess.run([variables.global_variables_initializer()]) - res1 = sess.run(outputs_dict1["0"]) - res2 = sess.run(outputs_dict2["0"]) - res3 = sess.run(outputs_dict3["0"]) - self.assertAllClose(res1, res2) - self.assertAllClose(res1, res3) - - def testSequenceLoss(self): - with self.cached_session() as sess: - logits = [constant_op.constant(i + 0.5, shape=[2, 5]) for i in range(3)] - targets = [ - constant_op.constant( - i, dtypes.int32, shape=[2]) for i in range(3) - ] - weights = [constant_op.constant(1.0, shape=[2]) for i in range(3)] - - average_loss_per_example = seq2seq_lib.sequence_loss( - logits, - targets, - weights, - average_across_timesteps=True, - average_across_batch=True) - res = sess.run(average_loss_per_example) - self.assertAllClose(1.60944, res) - - average_loss_per_sequence = seq2seq_lib.sequence_loss( - logits, - targets, - weights, - average_across_timesteps=False, - average_across_batch=True) - res = sess.run(average_loss_per_sequence) - self.assertAllClose(4.828314, res) - - total_loss = seq2seq_lib.sequence_loss( - logits, - targets, - weights, - average_across_timesteps=False, - average_across_batch=False) - res = sess.run(total_loss) - self.assertAllClose(9.656628, res) - - def testSequenceLossByExample(self): - with self.cached_session() as sess: - output_classes = 5 - logits = [ - constant_op.constant( - i + 0.5, shape=[2, output_classes]) for i in range(3) - ] - targets = [ - constant_op.constant( - i, dtypes.int32, shape=[2]) for i in range(3) - ] - weights = [constant_op.constant(1.0, shape=[2]) for i in range(3)] - - average_loss_per_example = (seq2seq_lib.sequence_loss_by_example( - logits, targets, weights, average_across_timesteps=True)) - res = sess.run(average_loss_per_example) - self.assertAllClose(np.asarray([1.609438, 1.609438]), res) - - loss_per_sequence = seq2seq_lib.sequence_loss_by_example( - logits, targets, weights, average_across_timesteps=False) - res = sess.run(loss_per_sequence) - self.assertAllClose(np.asarray([4.828314, 4.828314]), res) - - # TODO(ebrevdo, lukaszkaiser): Re-enable once RNNCells allow reuse - # within a variable scope that already has a weights tensor. - # - # def testModelWithBucketsScopeAndLoss(self): - # """Test variable scope reuse is not reset after model_with_buckets.""" - # classes = 10 - # buckets = [(4, 4), (8, 8)] - - # with self.cached_session(): - # # Here comes a sample Seq2Seq model using GRU cells. - # def SampleGRUSeq2Seq(enc_inp, dec_inp, weights, per_example_loss): - # """Example sequence-to-sequence model that uses GRU cells.""" - - # def GRUSeq2Seq(enc_inp, dec_inp): - # cell = rnn_cell.MultiRNNCell( - # [rnn_cell.GRUCell(24) for _ in range(2)]) - # return seq2seq_lib.embedding_attention_seq2seq( - # enc_inp, - # dec_inp, - # cell, - # num_encoder_symbols=classes, - # num_decoder_symbols=classes, - # embedding_size=24) - - # targets = [dec_inp[i + 1] for i in range(len(dec_inp) - 1)] + [0] - # return seq2seq_lib.model_with_buckets( - # enc_inp, - # dec_inp, - # targets, - # weights, - # buckets, - # GRUSeq2Seq, - # per_example_loss=per_example_loss) - - # # Now we construct the copy model. - # inp = [ - # array_ops.placeholder( - # dtypes.int32, shape=[None]) for _ in range(8) - # ] - # out = [ - # array_ops.placeholder( - # dtypes.int32, shape=[None]) for _ in range(8) - # ] - # weights = [ - # array_ops.ones_like( - # inp[0], dtype=dtypes.float32) for _ in range(8) - # ] - # with variable_scope.variable_scope("root"): - # _, losses1 = SampleGRUSeq2Seq( - # inp, out, weights, per_example_loss=False) - # # Now check that we did not accidentally set reuse. - # self.assertEqual(False, variable_scope.get_variable_scope().reuse) - # with variable_scope.variable_scope("new"): - # _, losses2 = SampleGRUSeq2Seq - # inp, out, weights, per_example_loss=True) - # # First loss is scalar, the second one is a 1-dimensional tensor. - # self.assertEqual([], losses1[0].get_shape().as_list()) - # self.assertEqual([None], losses2[0].get_shape().as_list()) - - def testModelWithBuckets(self): - """Larger tests that does full sequence-to-sequence model training.""" - # We learn to copy 10 symbols in 2 buckets: length 4 and length 8. - classes = 10 - buckets = [(4, 4), (8, 8)] - perplexities = [[], []] # Results for each bucket. - random_seed.set_random_seed(111) - random.seed(111) - np.random.seed(111) - - with self.cached_session() as sess: - # We use sampled softmax so we keep output projection separate. - w = variable_scope.get_variable("proj_w", [24, classes]) - w_t = array_ops.transpose(w) - b = variable_scope.get_variable("proj_b", [classes]) - - # Here comes a sample Seq2Seq model using GRU cells. - def SampleGRUSeq2Seq(enc_inp, dec_inp, weights): - """Example sequence-to-sequence model that uses GRU cells.""" - - def GRUSeq2Seq(enc_inp, dec_inp): - cell = rnn_cell.MultiRNNCell( - [rnn_cell.GRUCell(24) for _ in range(2)], state_is_tuple=True) - return seq2seq_lib.embedding_attention_seq2seq( - enc_inp, - dec_inp, - cell, - num_encoder_symbols=classes, - num_decoder_symbols=classes, - embedding_size=24, - output_projection=(w, b)) - - targets = [dec_inp[i + 1] for i in range(len(dec_inp) - 1)] + [0] - - def SampledLoss(labels, logits): - labels = array_ops.reshape(labels, [-1, 1]) - return nn_impl.sampled_softmax_loss( - weights=w_t, - biases=b, - labels=labels, - inputs=logits, - num_sampled=8, - num_classes=classes) - - return seq2seq_lib.model_with_buckets( - enc_inp, - dec_inp, - targets, - weights, - buckets, - GRUSeq2Seq, - softmax_loss_function=SampledLoss) - - # Now we construct the copy model. - batch_size = 8 - inp = [ - array_ops.placeholder( - dtypes.int32, shape=[None]) for _ in range(8) - ] - out = [ - array_ops.placeholder( - dtypes.int32, shape=[None]) for _ in range(8) - ] - weights = [ - array_ops.ones_like( - inp[0], dtype=dtypes.float32) for _ in range(8) - ] - with variable_scope.variable_scope("root"): - _, losses = SampleGRUSeq2Seq(inp, out, weights) - updates = [] - params = variables.global_variables() - optimizer = adam.AdamOptimizer(0.03, epsilon=1e-5) - for i in range(len(buckets)): - full_grads = gradients_impl.gradients(losses[i], params) - grads, _ = clip_ops.clip_by_global_norm(full_grads, 30.0) - update = optimizer.apply_gradients(zip(grads, params)) - updates.append(update) - sess.run([variables.global_variables_initializer()]) - steps = 6 - for _ in range(steps): - bucket = random.choice(np.arange(len(buckets))) - length = buckets[bucket][0] - i = [ - np.array( - [np.random.randint(9) + 1 for _ in range(batch_size)], - dtype=np.int32) for _ in range(length) - ] - # 0 is our "GO" symbol here. - o = [np.array([0] * batch_size, dtype=np.int32)] + i - feed = {} - for i1, i2, o1, o2 in zip(inp[:length], i[:length], out[:length], - o[:length]): - feed[i1.name] = i2 - feed[o1.name] = o2 - if length < 8: # For the 4-bucket, we need the 5th as target. - feed[out[length].name] = o[length] - res = sess.run([updates[bucket], losses[bucket]], feed) - perplexities[bucket].append(math.exp(float(res[1]))) - for bucket in range(len(buckets)): - if len(perplexities[bucket]) > 1: # Assert that perplexity went down. - self.assertLess(perplexities[bucket][-1], # 20% margin of error. - 1.2 * perplexities[bucket][0]) - - def testModelWithBooleanFeedPrevious(self): - """Test the model behavior when feed_previous is True. - - For example, the following two cases have the same effect: - - Train `embedding_rnn_seq2seq` with `feed_previous=True`, which contains - a `embedding_rnn_decoder` with `feed_previous=True` and - `update_embedding_for_previous=True`. The decoder is fed with "" - and outputs "A, B, C". - - Train `embedding_rnn_seq2seq` with `feed_previous=False`. The decoder - is fed with ", A, B". - """ - num_encoder_symbols = 3 - num_decoder_symbols = 5 - batch_size = 2 - num_enc_timesteps = 2 - num_dec_timesteps = 3 - - def TestModel(seq2seq): - with self.session(graph=ops.Graph()) as sess: - random_seed.set_random_seed(111) - random.seed(111) - np.random.seed(111) - - enc_inp = [ - constant_op.constant( - i + 1, dtypes.int32, shape=[batch_size]) - for i in range(num_enc_timesteps) - ] - dec_inp_fp_true = [ - constant_op.constant( - i, dtypes.int32, shape=[batch_size]) - for i in range(num_dec_timesteps) - ] - dec_inp_holder_fp_false = [ - array_ops.placeholder( - dtypes.int32, shape=[batch_size]) - for _ in range(num_dec_timesteps) - ] - targets = [ - constant_op.constant( - i + 1, dtypes.int32, shape=[batch_size]) - for i in range(num_dec_timesteps) - ] - weights = [ - constant_op.constant( - 1.0, shape=[batch_size]) for i in range(num_dec_timesteps) - ] - - def ForwardBackward(enc_inp, dec_inp, feed_previous): - scope_name = "fp_{}".format(feed_previous) - with variable_scope.variable_scope(scope_name): - dec_op, _ = seq2seq(enc_inp, dec_inp, feed_previous=feed_previous) - net_variables = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES, - scope_name) - optimizer = adam.AdamOptimizer(0.03, epsilon=1e-5) - update_op = optimizer.minimize( - seq2seq_lib.sequence_loss(dec_op, targets, weights), - var_list=net_variables) - return dec_op, update_op, net_variables - - dec_op_fp_true, update_fp_true, variables_fp_true = ForwardBackward( - enc_inp, dec_inp_fp_true, feed_previous=True) - _, update_fp_false, variables_fp_false = ForwardBackward( - enc_inp, dec_inp_holder_fp_false, feed_previous=False) - - sess.run(variables.global_variables_initializer()) - - # We only check consistencies between the variables existing in both - # the models with True and False feed_previous. Variables created by - # the loop_function in the model with True feed_previous are ignored. - v_false_name_dict = { - v.name.split("/", 1)[-1]: v - for v in variables_fp_false - } - matched_variables = [(v, v_false_name_dict[v.name.split("/", 1)[-1]]) - for v in variables_fp_true] - for v_true, v_false in matched_variables: - sess.run(state_ops.assign(v_false, v_true)) - - # Take the symbols generated by the decoder with feed_previous=True as - # the true input symbols for the decoder with feed_previous=False. - dec_fp_true = sess.run(dec_op_fp_true) - output_symbols_fp_true = np.argmax(dec_fp_true, axis=2) - dec_inp_fp_false = np.vstack((dec_inp_fp_true[0].eval(), - output_symbols_fp_true[:-1])) - sess.run(update_fp_true) - sess.run(update_fp_false, { - holder: inp - for holder, inp in zip(dec_inp_holder_fp_false, dec_inp_fp_false) - }) - - for v_true, v_false in matched_variables: - self.assertAllClose(v_true.eval(), v_false.eval()) - - def EmbeddingRNNSeq2SeqF(enc_inp, dec_inp, feed_previous): - cell = rnn_cell.BasicLSTMCell(2, state_is_tuple=True) - return seq2seq_lib.embedding_rnn_seq2seq( - enc_inp, - dec_inp, - cell, - num_encoder_symbols, - num_decoder_symbols, - embedding_size=2, - feed_previous=feed_previous) - - def EmbeddingRNNSeq2SeqNoTupleF(enc_inp, dec_inp, feed_previous): - cell = rnn_cell.BasicLSTMCell(2, state_is_tuple=False) - return seq2seq_lib.embedding_rnn_seq2seq( - enc_inp, - dec_inp, - cell, - num_encoder_symbols, - num_decoder_symbols, - embedding_size=2, - feed_previous=feed_previous) - - def EmbeddingTiedRNNSeq2Seq(enc_inp, dec_inp, feed_previous): - cell = rnn_cell.BasicLSTMCell(2, state_is_tuple=True) - return seq2seq_lib.embedding_tied_rnn_seq2seq( - enc_inp, - dec_inp, - cell, - num_decoder_symbols, - embedding_size=2, - feed_previous=feed_previous) - - def EmbeddingTiedRNNSeq2SeqNoTuple(enc_inp, dec_inp, feed_previous): - cell = rnn_cell.BasicLSTMCell(2, state_is_tuple=False) - return seq2seq_lib.embedding_tied_rnn_seq2seq( - enc_inp, - dec_inp, - cell, - num_decoder_symbols, - embedding_size=2, - feed_previous=feed_previous) - - def EmbeddingAttentionSeq2Seq(enc_inp, dec_inp, feed_previous): - cell = rnn_cell.BasicLSTMCell(2, state_is_tuple=True) - return seq2seq_lib.embedding_attention_seq2seq( - enc_inp, - dec_inp, - cell, - num_encoder_symbols, - num_decoder_symbols, - embedding_size=2, - feed_previous=feed_previous) - - def EmbeddingAttentionSeq2SeqNoTuple(enc_inp, dec_inp, feed_previous): - cell = rnn_cell.BasicLSTMCell(2, state_is_tuple=False) - return seq2seq_lib.embedding_attention_seq2seq( - enc_inp, - dec_inp, - cell, - num_encoder_symbols, - num_decoder_symbols, - embedding_size=2, - feed_previous=feed_previous) - - for model in (EmbeddingRNNSeq2SeqF, EmbeddingRNNSeq2SeqNoTupleF, - EmbeddingTiedRNNSeq2Seq, EmbeddingTiedRNNSeq2SeqNoTuple, - EmbeddingAttentionSeq2Seq, EmbeddingAttentionSeq2SeqNoTuple): - TestModel(model) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py b/tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py deleted file mode 100644 index 9555bb2fe8b..00000000000 --- a/tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py +++ /dev/null @@ -1,1227 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Library for creating sequence-to-sequence models in TensorFlow. - -Sequence-to-sequence recurrent neural networks can learn complex functions -that map input sequences to output sequences. These models yield very good -results on a number of tasks, such as speech recognition, parsing, machine -translation, or even constructing automated replies to emails. - -Before using this module, it is recommended to read the TensorFlow tutorial -on sequence-to-sequence models. It explains the basic concepts of this module -and shows an end-to-end example of how to build a translation model. - https://www.tensorflow.org/versions/master/tutorials/seq2seq/index.html - -Here is an overview of functions available in this module. They all use -a very similar interface, so after reading the above tutorial and using -one of them, others should be easy to substitute. - -* Full sequence-to-sequence models. - - basic_rnn_seq2seq: The most basic RNN-RNN model. - - tied_rnn_seq2seq: The basic model with tied encoder and decoder weights. - - embedding_rnn_seq2seq: The basic model with input embedding. - - embedding_tied_rnn_seq2seq: The tied model with input embedding. - - embedding_attention_seq2seq: Advanced model with input embedding and - the neural attention mechanism; recommended for complex tasks. - -* Multi-task sequence-to-sequence models. - - one2many_rnn_seq2seq: The embedding model with multiple decoders. - -* Decoders (when you write your own encoder, you can use these to decode; - e.g., if you want to write a model that generates captions for images). - - rnn_decoder: The basic decoder based on a pure RNN. - - attention_decoder: A decoder that uses the attention mechanism. - -* Losses. - - sequence_loss: Loss for a sequence model returning average log-perplexity. - - sequence_loss_by_example: As above, but not averaging over all examples. - -* model_with_buckets: A convenience function to create models with bucketing - (see the tutorial above for an explanation of why and how to use it). -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import copy - -# We disable pylint because we need python3 compatibility. -from six.moves import xrange # pylint: disable=redefined-builtin -from six.moves import zip # pylint: disable=redefined-builtin - -from tensorflow.contrib.rnn.python.ops import core_rnn_cell -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import rnn -from tensorflow.python.ops import rnn_cell_impl -from tensorflow.python.ops import variable_scope -from tensorflow.python.util import nest - -# TODO(ebrevdo): Remove once _linear is fully deprecated. -Linear = core_rnn_cell._Linear # pylint: disable=protected-access,invalid-name - - -def _extract_argmax_and_embed(embedding, - output_projection=None, - update_embedding=True): - """Get a loop_function that extracts the previous symbol and embeds it. - - Args: - embedding: embedding tensor for symbols. - output_projection: None or a pair (W, B). If provided, each fed previous - output will first be multiplied by W and added B. - update_embedding: Boolean; if False, the gradients will not propagate - through the embeddings. - - Returns: - A loop function. - """ - - def loop_function(prev, _): - if output_projection is not None: - prev = nn_ops.xw_plus_b(prev, output_projection[0], output_projection[1]) - prev_symbol = math_ops.argmax(prev, 1) - # Note that gradients will not propagate through the second parameter of - # embedding_lookup. - emb_prev = embedding_ops.embedding_lookup(embedding, prev_symbol) - if not update_embedding: - emb_prev = array_ops.stop_gradient(emb_prev) - return emb_prev - - return loop_function - - -def rnn_decoder(decoder_inputs, - initial_state, - cell, - loop_function=None, - scope=None): - """RNN decoder for the sequence-to-sequence model. - - Args: - decoder_inputs: A list of 2D Tensors [batch_size x input_size]. - initial_state: 2D Tensor with shape [batch_size x cell.state_size]. - cell: rnn_cell.RNNCell defining the cell function and size. - loop_function: If not None, this function will be applied to the i-th output - in order to generate the i+1-st input, and decoder_inputs will be ignored, - except for the first element ("GO" symbol). This can be used for decoding, - but also for training to emulate http://arxiv.org/abs/1506.03099. - Signature -- loop_function(prev, i) = next * prev is a 2D Tensor of - shape [batch_size x output_size], * i is an integer, the step number - (when advanced control is needed), * next is a 2D Tensor of shape - [batch_size x input_size]. - scope: VariableScope for the created subgraph; defaults to "rnn_decoder". - - Returns: - A tuple of the form (outputs, state), where: - outputs: A list of the same length as decoder_inputs of 2D Tensors with - shape [batch_size x output_size] containing generated outputs. - state: The state of each cell at the final time-step. - It is a 2D Tensor of shape [batch_size x cell.state_size]. - (Note that in some cases, like basic RNN cell or GRU cell, outputs and - states can be the same. They are different for LSTM cells though.) - """ - with variable_scope.variable_scope(scope or "rnn_decoder"): - state = initial_state - outputs = [] - prev = None - for i, inp in enumerate(decoder_inputs): - if loop_function is not None and prev is not None: - with variable_scope.variable_scope("loop_function", reuse=True): - inp = loop_function(prev, i) - if i > 0: - variable_scope.get_variable_scope().reuse_variables() - output, state = cell(inp, state) - outputs.append(output) - if loop_function is not None: - prev = output - return outputs, state - - -def basic_rnn_seq2seq(encoder_inputs, - decoder_inputs, - cell, - dtype=dtypes.float32, - scope=None): - """Basic RNN sequence-to-sequence model. - - This model first runs an RNN to encode encoder_inputs into a state vector, - then runs decoder, initialized with the last encoder state, on decoder_inputs. - Encoder and decoder use the same RNN cell type, but don't share parameters. - - Args: - encoder_inputs: A list of 2D Tensors [batch_size x input_size]. - decoder_inputs: A list of 2D Tensors [batch_size x input_size]. - cell: tf.compat.v1.nn.rnn_cell.RNNCell defining the cell function and size. - dtype: The dtype of the initial state of the RNN cell (default: tf.float32). - scope: VariableScope for the created subgraph; default: "basic_rnn_seq2seq". - - Returns: - A tuple of the form (outputs, state), where: - outputs: A list of the same length as decoder_inputs of 2D Tensors with - shape [batch_size x output_size] containing the generated outputs. - state: The state of each decoder cell in the final time-step. - It is a 2D Tensor of shape [batch_size x cell.state_size]. - """ - with variable_scope.variable_scope(scope or "basic_rnn_seq2seq"): - enc_cell = copy.deepcopy(cell) - _, enc_state = rnn.static_rnn(enc_cell, encoder_inputs, dtype=dtype) - return rnn_decoder(decoder_inputs, enc_state, cell) - - -def tied_rnn_seq2seq(encoder_inputs, - decoder_inputs, - cell, - loop_function=None, - dtype=dtypes.float32, - scope=None): - """RNN sequence-to-sequence model with tied encoder and decoder parameters. - - This model first runs an RNN to encode encoder_inputs into a state vector, and - then runs decoder, initialized with the last encoder state, on decoder_inputs. - Encoder and decoder use the same RNN cell and share parameters. - - Args: - encoder_inputs: A list of 2D Tensors [batch_size x input_size]. - decoder_inputs: A list of 2D Tensors [batch_size x input_size]. - cell: tf.compat.v1.nn.rnn_cell.RNNCell defining the cell function and size. - loop_function: If not None, this function will be applied to i-th output in - order to generate i+1-th input, and decoder_inputs will be ignored, except - for the first element ("GO" symbol), see rnn_decoder for details. - dtype: The dtype of the initial state of the rnn cell (default: tf.float32). - scope: VariableScope for the created subgraph; default: "tied_rnn_seq2seq". - - Returns: - A tuple of the form (outputs, state), where: - outputs: A list of the same length as decoder_inputs of 2D Tensors with - shape [batch_size x output_size] containing the generated outputs. - state: The state of each decoder cell in each time-step. This is a list - with length len(decoder_inputs) -- one item for each time-step. - It is a 2D Tensor of shape [batch_size x cell.state_size]. - """ - with variable_scope.variable_scope("combined_tied_rnn_seq2seq"): - scope = scope or "tied_rnn_seq2seq" - _, enc_state = rnn.static_rnn( - cell, encoder_inputs, dtype=dtype, scope=scope) - variable_scope.get_variable_scope().reuse_variables() - return rnn_decoder( - decoder_inputs, - enc_state, - cell, - loop_function=loop_function, - scope=scope) - - -def embedding_rnn_decoder(decoder_inputs, - initial_state, - cell, - num_symbols, - embedding_size, - output_projection=None, - feed_previous=False, - update_embedding_for_previous=True, - scope=None): - """RNN decoder with embedding and a pure-decoding option. - - Args: - decoder_inputs: A list of 1D batch-sized int32 Tensors (decoder inputs). - initial_state: 2D Tensor [batch_size x cell.state_size]. - cell: tf.compat.v1.nn.rnn_cell.RNNCell defining the cell function. - num_symbols: Integer, how many symbols come into the embedding. - embedding_size: Integer, the length of the embedding vector for each symbol. - output_projection: None or a pair (W, B) of output projection weights and - biases; W has shape [output_size x num_symbols] and B has shape - [num_symbols]; if provided and feed_previous=True, each fed previous - output will first be multiplied by W and added B. - feed_previous: Boolean; if True, only the first of decoder_inputs will be - used (the "GO" symbol), and all other decoder inputs will be generated by: - next = embedding_lookup(embedding, argmax(previous_output)), In effect, - this implements a greedy decoder. It can also be used - during training to emulate http://arxiv.org/abs/1506.03099. If False, - decoder_inputs are used as given (the standard decoder case). - update_embedding_for_previous: Boolean; if False and feed_previous=True, - only the embedding for the first symbol of decoder_inputs (the "GO" - symbol) will be updated by back propagation. Embeddings for the symbols - generated from the decoder itself remain unchanged. This parameter has no - effect if feed_previous=False. - scope: VariableScope for the created subgraph; defaults to - "embedding_rnn_decoder". - - Returns: - A tuple of the form (outputs, state), where: - outputs: A list of the same length as decoder_inputs of 2D Tensors. The - output is of shape [batch_size x cell.output_size] when - output_projection is not None (and represents the dense representation - of predicted tokens). It is of shape [batch_size x num_decoder_symbols] - when output_projection is None. - state: The state of each decoder cell in each time-step. This is a list - with length len(decoder_inputs) -- one item for each time-step. - It is a 2D Tensor of shape [batch_size x cell.state_size]. - - Raises: - ValueError: When output_projection has the wrong shape. - """ - with variable_scope.variable_scope(scope or "embedding_rnn_decoder") as scope: - if output_projection is not None: - dtype = scope.dtype - proj_weights = ops.convert_to_tensor(output_projection[0], dtype=dtype) - proj_weights.get_shape().assert_is_compatible_with([None, num_symbols]) - proj_biases = ops.convert_to_tensor(output_projection[1], dtype=dtype) - proj_biases.get_shape().assert_is_compatible_with([num_symbols]) - - embedding = variable_scope.get_variable("embedding", - [num_symbols, embedding_size]) - loop_function = _extract_argmax_and_embed( - embedding, output_projection, - update_embedding_for_previous) if feed_previous else None - emb_inp = ( - embedding_ops.embedding_lookup(embedding, i) for i in decoder_inputs) - return rnn_decoder( - emb_inp, initial_state, cell, loop_function=loop_function) - - -def embedding_rnn_seq2seq(encoder_inputs, - decoder_inputs, - cell, - num_encoder_symbols, - num_decoder_symbols, - embedding_size, - output_projection=None, - feed_previous=False, - dtype=None, - scope=None): - """Embedding RNN sequence-to-sequence model. - - This model first embeds encoder_inputs by a newly created embedding (of shape - [num_encoder_symbols x input_size]). Then it runs an RNN to encode - embedded encoder_inputs into a state vector. Next, it embeds decoder_inputs - by another newly created embedding (of shape [num_decoder_symbols x - input_size]). Then it runs RNN decoder, initialized with the last - encoder state, on embedded decoder_inputs. - - Args: - encoder_inputs: A list of 1D int32 Tensors of shape [batch_size]. - decoder_inputs: A list of 1D int32 Tensors of shape [batch_size]. - cell: tf.compat.v1.nn.rnn_cell.RNNCell defining the cell function and size. - num_encoder_symbols: Integer; number of symbols on the encoder side. - num_decoder_symbols: Integer; number of symbols on the decoder side. - embedding_size: Integer, the length of the embedding vector for each symbol. - output_projection: None or a pair (W, B) of output projection weights and - biases; W has shape [output_size x num_decoder_symbols] and B has shape - [num_decoder_symbols]; if provided and feed_previous=True, each fed - previous output will first be multiplied by W and added B. - feed_previous: Boolean or scalar Boolean Tensor; if True, only the first of - decoder_inputs will be used (the "GO" symbol), and all other decoder - inputs will be taken from previous outputs (as in embedding_rnn_decoder). - If False, decoder_inputs are used as given (the standard decoder case). - dtype: The dtype of the initial state for both the encoder and encoder - rnn cells (default: tf.float32). - scope: VariableScope for the created subgraph; defaults to - "embedding_rnn_seq2seq" - - Returns: - A tuple of the form (outputs, state), where: - outputs: A list of the same length as decoder_inputs of 2D Tensors. The - output is of shape [batch_size x cell.output_size] when - output_projection is not None (and represents the dense representation - of predicted tokens). It is of shape [batch_size x num_decoder_symbols] - when output_projection is None. - state: The state of each decoder cell in each time-step. This is a list - with length len(decoder_inputs) -- one item for each time-step. - It is a 2D Tensor of shape [batch_size x cell.state_size]. - """ - with variable_scope.variable_scope(scope or "embedding_rnn_seq2seq") as scope: - if dtype is not None: - scope.set_dtype(dtype) - else: - dtype = scope.dtype - - # Encoder. - encoder_cell = copy.deepcopy(cell) - encoder_cell = core_rnn_cell.EmbeddingWrapper( - encoder_cell, - embedding_classes=num_encoder_symbols, - embedding_size=embedding_size) - _, encoder_state = rnn.static_rnn(encoder_cell, encoder_inputs, dtype=dtype) - - # Decoder. - if output_projection is None: - cell = core_rnn_cell.OutputProjectionWrapper(cell, num_decoder_symbols) - - if isinstance(feed_previous, bool): - return embedding_rnn_decoder( - decoder_inputs, - encoder_state, - cell, - num_decoder_symbols, - embedding_size, - output_projection=output_projection, - feed_previous=feed_previous) - - # If feed_previous is a Tensor, we construct 2 graphs and use cond. - def decoder(feed_previous_bool): - reuse = None if feed_previous_bool else True - with variable_scope.variable_scope( - variable_scope.get_variable_scope(), reuse=reuse): - outputs, state = embedding_rnn_decoder( - decoder_inputs, - encoder_state, - cell, - num_decoder_symbols, - embedding_size, - output_projection=output_projection, - feed_previous=feed_previous_bool, - update_embedding_for_previous=False) - state_list = [state] - if nest.is_sequence(state): - state_list = nest.flatten(state) - return outputs + state_list - - outputs_and_state = control_flow_ops.cond( - feed_previous, lambda: decoder(True), lambda: decoder(False)) - outputs_len = len(decoder_inputs) # Outputs length same as decoder inputs. - state_list = outputs_and_state[outputs_len:] - state = state_list[0] - if nest.is_sequence(encoder_state): - state = nest.pack_sequence_as( - structure=encoder_state, flat_sequence=state_list) - return outputs_and_state[:outputs_len], state - - -def embedding_tied_rnn_seq2seq(encoder_inputs, - decoder_inputs, - cell, - num_symbols, - embedding_size, - num_decoder_symbols=None, - output_projection=None, - feed_previous=False, - dtype=None, - scope=None): - """Embedding RNN sequence-to-sequence model with tied (shared) parameters. - - This model first embeds encoder_inputs by a newly created embedding (of shape - [num_symbols x input_size]). Then it runs an RNN to encode embedded - encoder_inputs into a state vector. Next, it embeds decoder_inputs using - the same embedding. Then it runs RNN decoder, initialized with the last - encoder state, on embedded decoder_inputs. The decoder output is over symbols - from 0 to num_decoder_symbols - 1 if num_decoder_symbols is none; otherwise it - is over 0 to num_symbols - 1. - - Args: - encoder_inputs: A list of 1D int32 Tensors of shape [batch_size]. - decoder_inputs: A list of 1D int32 Tensors of shape [batch_size]. - cell: tf.compat.v1.nn.rnn_cell.RNNCell defining the cell function and size. - num_symbols: Integer; number of symbols for both encoder and decoder. - embedding_size: Integer, the length of the embedding vector for each symbol. - num_decoder_symbols: Integer; number of output symbols for decoder. If - provided, the decoder output is over symbols 0 to num_decoder_symbols - 1. - Otherwise, decoder output is over symbols 0 to num_symbols - 1. Note that - this assumes that the vocabulary is set up such that the first - num_decoder_symbols of num_symbols are part of decoding. - output_projection: None or a pair (W, B) of output projection weights and - biases; W has shape [output_size x num_symbols] and B has shape - [num_symbols]; if provided and feed_previous=True, each fed previous - output will first be multiplied by W and added B. - feed_previous: Boolean or scalar Boolean Tensor; if True, only the first of - decoder_inputs will be used (the "GO" symbol), and all other decoder - inputs will be taken from previous outputs (as in embedding_rnn_decoder). - If False, decoder_inputs are used as given (the standard decoder case). - dtype: The dtype to use for the initial RNN states (default: tf.float32). - scope: VariableScope for the created subgraph; defaults to - "embedding_tied_rnn_seq2seq". - - Returns: - A tuple of the form (outputs, state), where: - outputs: A list of the same length as decoder_inputs of 2D Tensors with - shape [batch_size x output_symbols] containing the generated - outputs where output_symbols = num_decoder_symbols if - num_decoder_symbols is not None otherwise output_symbols = num_symbols. - state: The state of each decoder cell at the final time-step. - It is a 2D Tensor of shape [batch_size x cell.state_size]. - - Raises: - ValueError: When output_projection has the wrong shape. - """ - with variable_scope.variable_scope( - scope or "embedding_tied_rnn_seq2seq", dtype=dtype) as scope: - dtype = scope.dtype - - if output_projection is not None: - proj_weights = ops.convert_to_tensor(output_projection[0], dtype=dtype) - proj_weights.get_shape().assert_is_compatible_with([None, num_symbols]) - proj_biases = ops.convert_to_tensor(output_projection[1], dtype=dtype) - proj_biases.get_shape().assert_is_compatible_with([num_symbols]) - - embedding = variable_scope.get_variable( - "embedding", [num_symbols, embedding_size], dtype=dtype) - - emb_encoder_inputs = [ - embedding_ops.embedding_lookup(embedding, x) for x in encoder_inputs - ] - emb_decoder_inputs = [ - embedding_ops.embedding_lookup(embedding, x) for x in decoder_inputs - ] - - output_symbols = num_symbols - if num_decoder_symbols is not None: - output_symbols = num_decoder_symbols - if output_projection is None: - cell = core_rnn_cell.OutputProjectionWrapper(cell, output_symbols) - - if isinstance(feed_previous, bool): - loop_function = _extract_argmax_and_embed(embedding, output_projection, - True) if feed_previous else None - return tied_rnn_seq2seq( - emb_encoder_inputs, - emb_decoder_inputs, - cell, - loop_function=loop_function, - dtype=dtype) - - # If feed_previous is a Tensor, we construct 2 graphs and use cond. - def decoder(feed_previous_bool): - loop_function = _extract_argmax_and_embed( - embedding, output_projection, False) if feed_previous_bool else None - reuse = None if feed_previous_bool else True - with variable_scope.variable_scope( - variable_scope.get_variable_scope(), reuse=reuse): - outputs, state = tied_rnn_seq2seq( - emb_encoder_inputs, - emb_decoder_inputs, - cell, - loop_function=loop_function, - dtype=dtype) - state_list = [state] - if nest.is_sequence(state): - state_list = nest.flatten(state) - return outputs + state_list - - outputs_and_state = control_flow_ops.cond( - feed_previous, lambda: decoder(True), lambda: decoder(False)) - outputs_len = len(decoder_inputs) # Outputs length same as decoder inputs. - state_list = outputs_and_state[outputs_len:] - state = state_list[0] - # Calculate zero-state to know it's structure. - static_batch_size = encoder_inputs[0].get_shape()[0] - for inp in encoder_inputs[1:]: - static_batch_size.merge_with(inp.get_shape()[0]) - batch_size = static_batch_size.value - if batch_size is None: - batch_size = array_ops.shape(encoder_inputs[0])[0] - zero_state = cell.zero_state(batch_size, dtype) - if nest.is_sequence(zero_state): - state = nest.pack_sequence_as( - structure=zero_state, flat_sequence=state_list) - return outputs_and_state[:outputs_len], state - - -def attention_decoder(decoder_inputs, - initial_state, - attention_states, - cell, - output_size=None, - num_heads=1, - loop_function=None, - dtype=None, - scope=None, - initial_state_attention=False): - """RNN decoder with attention for the sequence-to-sequence model. - - In this context "attention" means that, during decoding, the RNN can look up - information in the additional tensor attention_states, and it does this by - focusing on a few entries from the tensor. This model has proven to yield - especially good results in a number of sequence-to-sequence tasks. This - implementation is based on http://arxiv.org/abs/1412.7449 (see below for - details). It is recommended for complex sequence-to-sequence tasks. - - Args: - decoder_inputs: A list of 2D Tensors [batch_size x input_size]. - initial_state: 2D Tensor [batch_size x cell.state_size]. - attention_states: 3D Tensor [batch_size x attn_length x attn_size]. - cell: tf.compat.v1.nn.rnn_cell.RNNCell defining the cell function and size. - output_size: Size of the output vectors; if None, we use cell.output_size. - num_heads: Number of attention heads that read from attention_states. - loop_function: If not None, this function will be applied to i-th output in - order to generate i+1-th input, and decoder_inputs will be ignored, except - for the first element ("GO" symbol). This can be used for decoding, - but also for training to emulate http://arxiv.org/abs/1506.03099. - Signature -- loop_function(prev, i) = next * prev is a 2D Tensor of - shape [batch_size x output_size], * i is an integer, the step number - (when advanced control is needed), * next is a 2D Tensor of shape - [batch_size x input_size]. - dtype: The dtype to use for the RNN initial state (default: tf.float32). - scope: VariableScope for the created subgraph; default: "attention_decoder". - initial_state_attention: If False (default), initial attentions are zero. If - True, initialize the attentions from the initial state and attention - states -- useful when we wish to resume decoding from a previously stored - decoder state and attention states. - - Returns: - A tuple of the form (outputs, state), where: - outputs: A list of the same length as decoder_inputs of 2D Tensors of - shape [batch_size x output_size]. These represent the generated outputs. - Output i is computed from input i (which is either the i-th element - of decoder_inputs or loop_function(output {i-1}, i)) as follows. - First, we run the cell on a combination of the input and previous - attention masks: - cell_output, new_state = cell(linear(input, prev_attn), prev_state). - Then, we calculate new attention masks: - new_attn = softmax(V^T * tanh(W * attention_states + U * new_state)) - and then we calculate the output: - output = linear(cell_output, new_attn). - state: The state of each decoder cell the final time-step. - It is a 2D Tensor of shape [batch_size x cell.state_size]. - - Raises: - ValueError: when num_heads is not positive, there are no inputs, shapes - of attention_states are not set, or input size cannot be inferred - from the input. - """ - if not decoder_inputs: - raise ValueError("Must provide at least 1 input to attention decoder.") - if num_heads < 1: - raise ValueError("With less than 1 heads, use a non-attention decoder.") - if attention_states.get_shape()[2].value is None: - raise ValueError("Shape[2] of attention_states must be known: %s" % - attention_states.get_shape()) - if output_size is None: - output_size = cell.output_size - - with variable_scope.variable_scope( - scope or "attention_decoder", dtype=dtype) as scope: - dtype = scope.dtype - - batch_size = array_ops.shape(decoder_inputs[0])[0] # Needed for reshaping. - attn_length = attention_states.get_shape()[1].value - if attn_length is None: - attn_length = array_ops.shape(attention_states)[1] - attn_size = attention_states.get_shape()[2].value - - # To calculate W1 * h_t we use a 1-by-1 convolution, need to reshape before. - hidden = array_ops.reshape(attention_states, - [-1, attn_length, 1, attn_size]) - hidden_features = [] - v = [] - attention_vec_size = attn_size # Size of query vectors for attention. - for a in xrange(num_heads): - k = variable_scope.get_variable( - "AttnW_%d" % a, [1, 1, attn_size, attention_vec_size], dtype=dtype) - hidden_features.append(nn_ops.conv2d(hidden, k, [1, 1, 1, 1], "SAME")) - v.append( - variable_scope.get_variable( - "AttnV_%d" % a, [attention_vec_size], dtype=dtype)) - - state = initial_state - - def attention(query): - """Put attention masks on hidden using hidden_features and query.""" - ds = [] # Results of attention reads will be stored here. - if nest.is_sequence(query): # If the query is a tuple, flatten it. - query_list = nest.flatten(query) - for q in query_list: # Check that ndims == 2 if specified. - ndims = q.get_shape().ndims - if ndims: - assert ndims == 2 - query = array_ops.concat(query_list, 1) - for a in xrange(num_heads): - with variable_scope.variable_scope("Attention_%d" % a): - y = Linear(query, attention_vec_size, True)(query) - y = array_ops.reshape(y, [-1, 1, 1, attention_vec_size]) - y = math_ops.cast(y, dtype) - # Attention mask is a softmax of v^T * tanh(...). - s = math_ops.reduce_sum(v[a] * math_ops.tanh(hidden_features[a] + y), - [2, 3]) - a = nn_ops.softmax(math_ops.cast(s, dtype=dtypes.float32)) - # Now calculate the attention-weighted vector d. - a = math_ops.cast(a, dtype) - d = math_ops.reduce_sum( - array_ops.reshape(a, [-1, attn_length, 1, 1]) * hidden, [1, 2]) - ds.append(array_ops.reshape(d, [-1, attn_size])) - return ds - - outputs = [] - prev = None - batch_attn_size = array_ops.stack([batch_size, attn_size]) - attns = [ - array_ops.zeros(batch_attn_size, dtype=dtype) for _ in xrange(num_heads) - ] - for a in attns: # Ensure the second shape of attention vectors is set. - a.set_shape([None, attn_size]) - if initial_state_attention: - attns = attention(initial_state) - for i, inp in enumerate(decoder_inputs): - if i > 0: - variable_scope.get_variable_scope().reuse_variables() - # If loop_function is set, we use it instead of decoder_inputs. - if loop_function is not None and prev is not None: - with variable_scope.variable_scope("loop_function", reuse=True): - inp = loop_function(prev, i) - # Merge input and previous attentions into one vector of the right size. - input_size = inp.get_shape().with_rank(2)[1] - if input_size.value is None: - raise ValueError("Could not infer input size from input: %s" % inp.name) - - inputs = [inp] + attns - inputs = [math_ops.cast(e, dtype) for e in inputs] - x = Linear(inputs, input_size, True)(inputs) - # Run the RNN. - cell_output, state = cell(x, state) - # Run the attention mechanism. - if i == 0 and initial_state_attention: - with variable_scope.variable_scope( - variable_scope.get_variable_scope(), reuse=True): - attns = attention(state) - else: - attns = attention(state) - - with variable_scope.variable_scope("AttnOutputProjection"): - cell_output = math_ops.cast(cell_output, dtype) - inputs = [cell_output] + attns - output = Linear(inputs, output_size, True)(inputs) - if loop_function is not None: - prev = output - outputs.append(output) - - return outputs, state - - -def embedding_attention_decoder(decoder_inputs, - initial_state, - attention_states, - cell, - num_symbols, - embedding_size, - num_heads=1, - output_size=None, - output_projection=None, - feed_previous=False, - update_embedding_for_previous=True, - dtype=None, - scope=None, - initial_state_attention=False): - """RNN decoder with embedding and attention and a pure-decoding option. - - Args: - decoder_inputs: A list of 1D batch-sized int32 Tensors (decoder inputs). - initial_state: 2D Tensor [batch_size x cell.state_size]. - attention_states: 3D Tensor [batch_size x attn_length x attn_size]. - cell: tf.compat.v1.nn.rnn_cell.RNNCell defining the cell function. - num_symbols: Integer, how many symbols come into the embedding. - embedding_size: Integer, the length of the embedding vector for each symbol. - num_heads: Number of attention heads that read from attention_states. - output_size: Size of the output vectors; if None, use output_size. - output_projection: None or a pair (W, B) of output projection weights and - biases; W has shape [output_size x num_symbols] and B has shape - [num_symbols]; if provided and feed_previous=True, each fed previous - output will first be multiplied by W and added B. - feed_previous: Boolean; if True, only the first of decoder_inputs will be - used (the "GO" symbol), and all other decoder inputs will be generated by: - next = embedding_lookup(embedding, argmax(previous_output)), In effect, - this implements a greedy decoder. It can also be used - during training to emulate http://arxiv.org/abs/1506.03099. If False, - decoder_inputs are used as given (the standard decoder case). - update_embedding_for_previous: Boolean; if False and feed_previous=True, - only the embedding for the first symbol of decoder_inputs (the "GO" - symbol) will be updated by back propagation. Embeddings for the symbols - generated from the decoder itself remain unchanged. This parameter has no - effect if feed_previous=False. - dtype: The dtype to use for the RNN initial states (default: tf.float32). - scope: VariableScope for the created subgraph; defaults to - "embedding_attention_decoder". - initial_state_attention: If False (default), initial attentions are zero. If - True, initialize the attentions from the initial state and attention - states -- useful when we wish to resume decoding from a previously stored - decoder state and attention states. - - Returns: - A tuple of the form (outputs, state), where: - outputs: A list of the same length as decoder_inputs of 2D Tensors with - shape [batch_size x output_size] containing the generated outputs. - state: The state of each decoder cell at the final time-step. - It is a 2D Tensor of shape [batch_size x cell.state_size]. - - Raises: - ValueError: When output_projection has the wrong shape. - """ - if output_size is None: - output_size = cell.output_size - if output_projection is not None: - proj_biases = ops.convert_to_tensor(output_projection[1], dtype=dtype) - proj_biases.get_shape().assert_is_compatible_with([num_symbols]) - - with variable_scope.variable_scope( - scope or "embedding_attention_decoder", dtype=dtype) as scope: - - embedding = variable_scope.get_variable("embedding", - [num_symbols, embedding_size]) - loop_function = _extract_argmax_and_embed( - embedding, output_projection, - update_embedding_for_previous) if feed_previous else None - emb_inp = [ - embedding_ops.embedding_lookup(embedding, i) for i in decoder_inputs - ] - return attention_decoder( - emb_inp, - initial_state, - attention_states, - cell, - output_size=output_size, - num_heads=num_heads, - loop_function=loop_function, - initial_state_attention=initial_state_attention) - - -def embedding_attention_seq2seq(encoder_inputs, - decoder_inputs, - cell, - num_encoder_symbols, - num_decoder_symbols, - embedding_size, - num_heads=1, - output_projection=None, - feed_previous=False, - dtype=None, - scope=None, - initial_state_attention=False): - """Embedding sequence-to-sequence model with attention. - - This model first embeds encoder_inputs by a newly created embedding (of shape - [num_encoder_symbols x input_size]). Then it runs an RNN to encode - embedded encoder_inputs into a state vector. It keeps the outputs of this - RNN at every step to use for attention later. Next, it embeds decoder_inputs - by another newly created embedding (of shape [num_decoder_symbols x - input_size]). Then it runs attention decoder, initialized with the last - encoder state, on embedded decoder_inputs and attending to encoder outputs. - - Warning: when output_projection is None, the size of the attention vectors - and variables will be made proportional to num_decoder_symbols, can be large. - - Args: - encoder_inputs: A list of 1D int32 Tensors of shape [batch_size]. - decoder_inputs: A list of 1D int32 Tensors of shape [batch_size]. - cell: tf.compat.v1.nn.rnn_cell.RNNCell defining the cell function and size. - num_encoder_symbols: Integer; number of symbols on the encoder side. - num_decoder_symbols: Integer; number of symbols on the decoder side. - embedding_size: Integer, the length of the embedding vector for each symbol. - num_heads: Number of attention heads that read from attention_states. - output_projection: None or a pair (W, B) of output projection weights and - biases; W has shape [output_size x num_decoder_symbols] and B has shape - [num_decoder_symbols]; if provided and feed_previous=True, each fed - previous output will first be multiplied by W and added B. - feed_previous: Boolean or scalar Boolean Tensor; if True, only the first of - decoder_inputs will be used (the "GO" symbol), and all other decoder - inputs will be taken from previous outputs (as in embedding_rnn_decoder). - If False, decoder_inputs are used as given (the standard decoder case). - dtype: The dtype of the initial RNN state (default: tf.float32). - scope: VariableScope for the created subgraph; defaults to - "embedding_attention_seq2seq". - initial_state_attention: If False (default), initial attentions are zero. If - True, initialize the attentions from the initial state and attention - states. - - Returns: - A tuple of the form (outputs, state), where: - outputs: A list of the same length as decoder_inputs of 2D Tensors with - shape [batch_size x num_decoder_symbols] containing the generated - outputs. - state: The state of each decoder cell at the final time-step. - It is a 2D Tensor of shape [batch_size x cell.state_size]. - """ - with variable_scope.variable_scope( - scope or "embedding_attention_seq2seq", dtype=dtype) as scope: - dtype = scope.dtype - # Encoder. - encoder_cell = copy.deepcopy(cell) - encoder_cell = core_rnn_cell.EmbeddingWrapper( - encoder_cell, - embedding_classes=num_encoder_symbols, - embedding_size=embedding_size) - encoder_outputs, encoder_state = rnn.static_rnn( - encoder_cell, encoder_inputs, dtype=dtype) - - # First calculate a concatenation of encoder outputs to put attention on. - top_states = [ - array_ops.reshape(e, [-1, 1, cell.output_size]) for e in encoder_outputs - ] - attention_states = array_ops.concat(top_states, 1) - - # Decoder. - output_size = None - if output_projection is None: - cell = core_rnn_cell.OutputProjectionWrapper(cell, num_decoder_symbols) - output_size = num_decoder_symbols - - if isinstance(feed_previous, bool): - return embedding_attention_decoder( - decoder_inputs, - encoder_state, - attention_states, - cell, - num_decoder_symbols, - embedding_size, - num_heads=num_heads, - output_size=output_size, - output_projection=output_projection, - feed_previous=feed_previous, - initial_state_attention=initial_state_attention) - - # If feed_previous is a Tensor, we construct 2 graphs and use cond. - def decoder(feed_previous_bool): - reuse = None if feed_previous_bool else True - with variable_scope.variable_scope( - variable_scope.get_variable_scope(), reuse=reuse): - outputs, state = embedding_attention_decoder( - decoder_inputs, - encoder_state, - attention_states, - cell, - num_decoder_symbols, - embedding_size, - num_heads=num_heads, - output_size=output_size, - output_projection=output_projection, - feed_previous=feed_previous_bool, - update_embedding_for_previous=False, - initial_state_attention=initial_state_attention) - state_list = [state] - if nest.is_sequence(state): - state_list = nest.flatten(state) - return outputs + state_list - - outputs_and_state = control_flow_ops.cond( - feed_previous, lambda: decoder(True), lambda: decoder(False)) - outputs_len = len(decoder_inputs) # Outputs length same as decoder inputs. - state_list = outputs_and_state[outputs_len:] - state = state_list[0] - if nest.is_sequence(encoder_state): - state = nest.pack_sequence_as( - structure=encoder_state, flat_sequence=state_list) - return outputs_and_state[:outputs_len], state - - -def one2many_rnn_seq2seq(encoder_inputs, - decoder_inputs_dict, - enc_cell, - dec_cells_dict, - num_encoder_symbols, - num_decoder_symbols_dict, - embedding_size, - feed_previous=False, - dtype=None, - scope=None): - """One-to-many RNN sequence-to-sequence model (multi-task). - - This is a multi-task sequence-to-sequence model with one encoder and multiple - decoders. Reference to multi-task sequence-to-sequence learning can be found - here: http://arxiv.org/abs/1511.06114 - - Args: - encoder_inputs: A list of 1D int32 Tensors of shape [batch_size]. - decoder_inputs_dict: A dictionary mapping decoder name (string) to the - corresponding decoder_inputs; each decoder_inputs is a list of 1D Tensors - of shape [batch_size]; num_decoders is defined as - len(decoder_inputs_dict). - enc_cell: tf.compat.v1.nn.rnn_cell.RNNCell defining the encoder cell - function and size. - dec_cells_dict: A dictionary mapping encoder name (string) to an instance of - tf.nn.rnn_cell.RNNCell. - num_encoder_symbols: Integer; number of symbols on the encoder side. - num_decoder_symbols_dict: A dictionary mapping decoder name (string) to an - integer specifying number of symbols for the corresponding decoder; - len(num_decoder_symbols_dict) must be equal to num_decoders. - embedding_size: Integer, the length of the embedding vector for each symbol. - feed_previous: Boolean or scalar Boolean Tensor; if True, only the first of - decoder_inputs will be used (the "GO" symbol), and all other decoder - inputs will be taken from previous outputs (as in embedding_rnn_decoder). - If False, decoder_inputs are used as given (the standard decoder case). - dtype: The dtype of the initial state for both the encoder and encoder - rnn cells (default: tf.float32). - scope: VariableScope for the created subgraph; defaults to - "one2many_rnn_seq2seq" - - Returns: - A tuple of the form (outputs_dict, state_dict), where: - outputs_dict: A mapping from decoder name (string) to a list of the same - length as decoder_inputs_dict[name]; each element in the list is a 2D - Tensors with shape [batch_size x num_decoder_symbol_list[name]] - containing the generated outputs. - state_dict: A mapping from decoder name (string) to the final state of the - corresponding decoder RNN; it is a 2D Tensor of shape - [batch_size x cell.state_size]. - - Raises: - TypeError: if enc_cell or any of the dec_cells are not instances of RNNCell. - ValueError: if len(dec_cells) != len(decoder_inputs_dict). - """ - outputs_dict = {} - state_dict = {} - - if not isinstance(enc_cell, rnn_cell_impl.RNNCell): - raise TypeError("enc_cell is not an RNNCell: %s" % type(enc_cell)) - if set(dec_cells_dict) != set(decoder_inputs_dict): - raise ValueError("keys of dec_cells_dict != keys of decodre_inputs_dict") - for dec_cell in dec_cells_dict.values(): - if not isinstance(dec_cell, rnn_cell_impl.RNNCell): - raise TypeError("dec_cell is not an RNNCell: %s" % type(dec_cell)) - - with variable_scope.variable_scope( - scope or "one2many_rnn_seq2seq", dtype=dtype) as scope: - dtype = scope.dtype - - # Encoder. - enc_cell = core_rnn_cell.EmbeddingWrapper( - enc_cell, - embedding_classes=num_encoder_symbols, - embedding_size=embedding_size) - _, encoder_state = rnn.static_rnn(enc_cell, encoder_inputs, dtype=dtype) - - # Decoder. - for name, decoder_inputs in decoder_inputs_dict.items(): - num_decoder_symbols = num_decoder_symbols_dict[name] - dec_cell = dec_cells_dict[name] - - with variable_scope.variable_scope("one2many_decoder_" + - str(name)) as scope: - dec_cell = core_rnn_cell.OutputProjectionWrapper( - dec_cell, num_decoder_symbols) - if isinstance(feed_previous, bool): - outputs, state = embedding_rnn_decoder( - decoder_inputs, - encoder_state, - dec_cell, - num_decoder_symbols, - embedding_size, - feed_previous=feed_previous) - else: - # If feed_previous is a Tensor, we construct 2 graphs and use cond. - def filled_embedding_rnn_decoder(feed_previous): - """The current decoder with a fixed feed_previous parameter.""" - # pylint: disable=cell-var-from-loop - reuse = None if feed_previous else True - vs = variable_scope.get_variable_scope() - with variable_scope.variable_scope(vs, reuse=reuse): - outputs, state = embedding_rnn_decoder( - decoder_inputs, - encoder_state, - dec_cell, - num_decoder_symbols, - embedding_size, - feed_previous=feed_previous) - # pylint: enable=cell-var-from-loop - state_list = [state] - if nest.is_sequence(state): - state_list = nest.flatten(state) - return outputs + state_list - - outputs_and_state = control_flow_ops.cond( - feed_previous, lambda: filled_embedding_rnn_decoder(True), lambda: - filled_embedding_rnn_decoder(False)) - # Outputs length is the same as for decoder inputs. - outputs_len = len(decoder_inputs) - outputs = outputs_and_state[:outputs_len] - state_list = outputs_and_state[outputs_len:] - state = state_list[0] - if nest.is_sequence(encoder_state): - state = nest.pack_sequence_as( - structure=encoder_state, flat_sequence=state_list) - outputs_dict[name] = outputs - state_dict[name] = state - - return outputs_dict, state_dict - - -def sequence_loss_by_example(logits, - targets, - weights, - average_across_timesteps=True, - softmax_loss_function=None, - name=None): - """Weighted cross-entropy loss for a sequence of logits (per example). - - Args: - logits: List of 2D Tensors of shape [batch_size x num_decoder_symbols]. - targets: List of 1D batch-sized int32 Tensors of the same length as logits. - weights: List of 1D batch-sized float-Tensors of the same length as logits. - average_across_timesteps: If set, divide the returned cost by the total - label weight. - softmax_loss_function: Function (labels, logits) -> loss-batch to be used - instead of the standard softmax (the default if this is None). **Note that - to avoid confusion, it is required for the function to accept named - arguments.** - name: Optional name for this operation, default: "sequence_loss_by_example". - - Returns: - 1D batch-sized float Tensor: The log-perplexity for each sequence. - - Raises: - ValueError: If len(logits) is different from len(targets) or len(weights). - """ - if len(targets) != len(logits) or len(weights) != len(logits): - raise ValueError("Lengths of logits, weights, and targets must be the same " - "%d, %d, %d." % (len(logits), len(weights), len(targets))) - with ops.name_scope(name, "sequence_loss_by_example", - logits + targets + weights): - log_perp_list = [] - for logit, target, weight in zip(logits, targets, weights): - if softmax_loss_function is None: - # TODO(irving,ebrevdo): This reshape is needed because - # sequence_loss_by_example is called with scalars sometimes, which - # violates our general scalar strictness policy. - target = array_ops.reshape(target, [-1]) - crossent = nn_ops.sparse_softmax_cross_entropy_with_logits( - labels=target, logits=logit) - else: - crossent = softmax_loss_function(labels=target, logits=logit) - log_perp_list.append(crossent * weight) - log_perps = math_ops.add_n(log_perp_list) - if average_across_timesteps: - total_size = math_ops.add_n(weights) - total_size += 1e-12 # Just to avoid division by 0 for all-0 weights. - log_perps /= total_size - return log_perps - - -def sequence_loss(logits, - targets, - weights, - average_across_timesteps=True, - average_across_batch=True, - softmax_loss_function=None, - name=None): - """Weighted cross-entropy loss for a sequence of logits, batch-collapsed. - - Args: - logits: List of 2D Tensors of shape [batch_size x num_decoder_symbols]. - targets: List of 1D batch-sized int32 Tensors of the same length as logits. - weights: List of 1D batch-sized float-Tensors of the same length as logits. - average_across_timesteps: If set, divide the returned cost by the total - label weight. - average_across_batch: If set, divide the returned cost by the batch size. - softmax_loss_function: Function (labels, logits) -> loss-batch to be used - instead of the standard softmax (the default if this is None). **Note that - to avoid confusion, it is required for the function to accept named - arguments.** - name: Optional name for this operation, defaults to "sequence_loss". - - Returns: - A scalar float Tensor: The average log-perplexity per symbol (weighted). - - Raises: - ValueError: If len(logits) is different from len(targets) or len(weights). - """ - with ops.name_scope(name, "sequence_loss", logits + targets + weights): - cost = math_ops.reduce_sum( - sequence_loss_by_example( - logits, - targets, - weights, - average_across_timesteps=average_across_timesteps, - softmax_loss_function=softmax_loss_function)) - if average_across_batch: - batch_size = array_ops.shape(targets[0])[0] - return cost / math_ops.cast(batch_size, cost.dtype) - else: - return cost - - -def model_with_buckets(encoder_inputs, - decoder_inputs, - targets, - weights, - buckets, - seq2seq, - softmax_loss_function=None, - per_example_loss=False, - name=None): - """Create a sequence-to-sequence model with support for bucketing. - - The seq2seq argument is a function that defines a sequence-to-sequence model, - e.g., seq2seq = lambda x, y: basic_rnn_seq2seq( - x, y, rnn_cell.GRUCell(24)) - - Args: - encoder_inputs: A list of Tensors to feed the encoder; first seq2seq input. - decoder_inputs: A list of Tensors to feed the decoder; second seq2seq input. - targets: A list of 1D batch-sized int32 Tensors (desired output sequence). - weights: List of 1D batch-sized float-Tensors to weight the targets. - buckets: A list of pairs of (input size, output size) for each bucket. - seq2seq: A sequence-to-sequence model function; it takes 2 input that agree - with encoder_inputs and decoder_inputs, and returns a pair consisting of - outputs and states (as, e.g., basic_rnn_seq2seq). - softmax_loss_function: Function (labels, logits) -> loss-batch to be used - instead of the standard softmax (the default if this is None). **Note that - to avoid confusion, it is required for the function to accept named - arguments.** - per_example_loss: Boolean. If set, the returned loss will be a batch-sized - tensor of losses for each sequence in the batch. If unset, it will be a - scalar with the averaged loss from all examples. - name: Optional name for this operation, defaults to "model_with_buckets". - - Returns: - A tuple of the form (outputs, losses), where: - outputs: The outputs for each bucket. Its j'th element consists of a list - of 2D Tensors. The shape of output tensors can be either - [batch_size x output_size] or [batch_size x num_decoder_symbols] - depending on the seq2seq model used. - losses: List of scalar Tensors, representing losses for each bucket, or, - if per_example_loss is set, a list of 1D batch-sized float Tensors. - - Raises: - ValueError: If length of encoder_inputs, targets, or weights is smaller - than the largest (last) bucket. - """ - if len(encoder_inputs) < buckets[-1][0]: - raise ValueError("Length of encoder_inputs (%d) must be at least that of la" - "st bucket (%d)." % (len(encoder_inputs), buckets[-1][0])) - if len(targets) < buckets[-1][1]: - raise ValueError("Length of targets (%d) must be at least that of last " - "bucket (%d)." % (len(targets), buckets[-1][1])) - if len(weights) < buckets[-1][1]: - raise ValueError("Length of weights (%d) must be at least that of last " - "bucket (%d)." % (len(weights), buckets[-1][1])) - - all_inputs = encoder_inputs + decoder_inputs + targets + weights - losses = [] - outputs = [] - with ops.name_scope(name, "model_with_buckets", all_inputs): - for j, bucket in enumerate(buckets): - with variable_scope.variable_scope( - variable_scope.get_variable_scope(), reuse=True if j > 0 else None): - bucket_outputs, _ = seq2seq(encoder_inputs[:bucket[0]], - decoder_inputs[:bucket[1]]) - outputs.append(bucket_outputs) - if per_example_loss: - losses.append( - sequence_loss_by_example( - outputs[-1], - targets[:bucket[1]], - weights[:bucket[1]], - softmax_loss_function=softmax_loss_function)) - else: - losses.append( - sequence_loss( - outputs[-1], - targets[:bucket[1]], - weights[:bucket[1]], - softmax_loss_function=softmax_loss_function)) - - return outputs, losses diff --git a/tensorflow/contrib/libsvm/BUILD b/tensorflow/contrib/libsvm/BUILD deleted file mode 100644 index 51498791211..00000000000 --- a/tensorflow/contrib/libsvm/BUILD +++ /dev/null @@ -1,89 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "tf_custom_op_library") -load("//tensorflow:tensorflow.bzl", "tf_gen_op_libs") -load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py") -load("//tensorflow:tensorflow.bzl", "tf_kernel_library") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") -load("//tensorflow:tensorflow.bzl", "tf_py_test") - -package( - default_visibility = ["//visibility:private"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_custom_op_library( - name = "python/ops/_libsvm_ops.so", - srcs = [ - "kernels/decode_libsvm_op.cc", - "ops/libsvm_ops.cc", - ], - deps = [ - "//tensorflow/core/kernels:bounds_check_lib", - ], -) - -tf_kernel_library( - name = "libsvm_kernels", - srcs = ["kernels/decode_libsvm_op.cc"], - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//tensorflow/core/kernels:bounds_check_lib", - ], -) - -tf_gen_op_libs( - op_lib_names = ["libsvm_ops"], - deps = [ - "//tensorflow/core:lib", - ], -) - -tf_gen_op_wrapper_py( - name = "libsvm_ops", - deps = [":libsvm_ops_op_lib"], -) - -tf_custom_op_py_library( - name = "libsvm", - srcs = [ - "__init__.py", - "python/ops/libsvm_ops.py", - ], - dso = [ - ":python/ops/_libsvm_ops.so", - ], - kernels = [ - ":libsvm_kernels", - ":libsvm_ops_op_lib", - ], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":libsvm_ops", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - ], -) - -tf_py_test( - name = "decode_libsvm_op_test", - srcs = ["python/kernel_tests/decode_libsvm_op_test.py"], - additional_deps = [ - ":libsvm", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) diff --git a/tensorflow/contrib/libsvm/__init__.py b/tensorflow/contrib/libsvm/__init__.py deleted file mode 100644 index a875863caab..00000000000 --- a/tensorflow/contrib/libsvm/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Libsvm decoder. - -@@decode_libsvm -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.libsvm.python.ops.libsvm_ops import decode_libsvm - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - "decode_libsvm", -] - -remove_undocumented(__name__) diff --git a/tensorflow/contrib/libsvm/kernels/decode_libsvm_op.cc b/tensorflow/contrib/libsvm/kernels/decode_libsvm_op.cc deleted file mode 100644 index f35453f267e..00000000000 --- a/tensorflow/contrib/libsvm/kernels/decode_libsvm_op.cc +++ /dev/null @@ -1,168 +0,0 @@ -/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/strings/numbers.h" -#include "tensorflow/core/lib/strings/str_util.h" - -namespace tensorflow { - -template -class DecodeLibsvmOp : public OpKernel { - public: - explicit DecodeLibsvmOp(OpKernelConstruction* ctx) : OpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("num_features", &num_features_)); - OP_REQUIRES(ctx, (num_features_ >= 1), - errors::InvalidArgument("Invalid number of features \"", - num_features_, "\"")); - } - - void Compute(OpKernelContext* ctx) override { - const Tensor* input_tensor; - OP_REQUIRES_OK(ctx, ctx->input("input", &input_tensor)); - const auto& input_flat = input_tensor->flat(); - - Tensor* label_tensor; - OP_REQUIRES_OK( - ctx, ctx->allocate_output(0, input_tensor->shape(), &label_tensor)); - auto label = label_tensor->flat(); - - std::vector out_values; - std::vector> out_indices; - for (int i = 0; i < input_flat.size(); ++i) { - StringPiece line(input_flat(i)); - str_util::RemoveWhitespaceContext(&line); - - StringPiece piece; - OP_REQUIRES(ctx, str_util::ConsumeNonWhitespace(&line, &piece), - errors::InvalidArgument("No label found for input[", i, - "]: \"", input_flat(i), "\"")); - - Tlabel label_value; - OP_REQUIRES(ctx, - strings::SafeStringToNumeric(piece, &label_value), - errors::InvalidArgument("Label format incorrect: ", piece)); - - label(i) = label_value; - - str_util::RemoveLeadingWhitespace(&line); - while (str_util::ConsumeNonWhitespace(&line, &piece)) { - size_t p = piece.find(':'); - OP_REQUIRES(ctx, (p != StringPiece::npos), - errors::InvalidArgument("Invalid feature \"", piece, "\"")); - - int64 feature_index; - OP_REQUIRES( - ctx, strings::safe_strto64(piece.substr(0, p), &feature_index), - errors::InvalidArgument("Feature format incorrect: ", piece)); - OP_REQUIRES(ctx, (feature_index >= 0), - errors::InvalidArgument( - "Feature index should be >= 0, got ", feature_index)); - - T feature_value; - OP_REQUIRES( - - ctx, - strings::SafeStringToNumeric(piece.substr(p + 1), - &feature_value), - errors::InvalidArgument("Feature format incorrect: ", piece)); - - out_values.emplace_back(feature_value); - out_indices.emplace_back(std::pair(i, feature_index)); - - str_util::RemoveLeadingWhitespace(&line); - } - } - - Tensor* indices_tensor; - OP_REQUIRES_OK(ctx, ctx->allocate_output( - 1, - TensorShape({static_cast(out_indices.size()), - input_tensor->shape().dims() + 1}), - &indices_tensor)); - auto indices = indices_tensor->matrix(); - // Translate flat index to shaped index like np.unravel_index - // Calculate factors for each dimension - std::vector factors(input_tensor->shape().dims()); - factors[input_tensor->shape().dims() - 1] = 1; - for (int j = input_tensor->shape().dims() - 2; j >= 0; j--) { - factors[j] = factors[j + 1] * input_tensor->shape().dim_size(j + 1); - } - for (int i = 0; i < out_indices.size(); i++) { - indices(i, 0) = out_indices[i].first; - int64 value = out_indices[i].first; - for (int j = 0; j < input_tensor->shape().dims(); j++) { - indices(i, j) = value / factors[j]; - value = value % factors[j]; - } - indices(i, input_tensor->shape().dims()) = out_indices[i].second; - } - - Tensor* values_tensor; - OP_REQUIRES_OK(ctx, - ctx->allocate_output( - 2, TensorShape({static_cast(out_values.size())}), - &values_tensor)); - auto values = values_tensor->vec(); - std::copy_n(out_values.begin(), out_values.size(), &values(0)); - - Tensor* shape_tensor; - OP_REQUIRES_OK(ctx, ctx->allocate_output( - 3, TensorShape({input_tensor->shape().dims() + 1}), - &shape_tensor)); - auto shape = shape_tensor->flat(); - for (int i = 0; i < input_tensor->shape().dims(); i++) { - shape(i) = input_tensor->shape().dim_size(i); - } - shape(input_tensor->shape().dims()) = num_features_; - } - - private: - int64 num_features_; -}; - -#define REGISTER_KERNEL(type) \ - REGISTER_KERNEL_BUILDER(Name("DecodeLibsvm") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("dtype") \ - .TypeConstraint("label_dtype"), \ - DecodeLibsvmOp); \ - REGISTER_KERNEL_BUILDER(Name("DecodeLibsvm") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("dtype") \ - .TypeConstraint("label_dtype"), \ - DecodeLibsvmOp); \ - REGISTER_KERNEL_BUILDER(Name("DecodeLibsvm") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("dtype") \ - .TypeConstraint("label_dtype"), \ - DecodeLibsvmOp); \ - REGISTER_KERNEL_BUILDER(Name("DecodeLibsvm") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("dtype") \ - .TypeConstraint("label_dtype"), \ - DecodeLibsvmOp); - -REGISTER_KERNEL(float); -REGISTER_KERNEL(double); -REGISTER_KERNEL(int32); -REGISTER_KERNEL(int64); -#undef REGISTER_KERNEL - -} // namespace tensorflow diff --git a/tensorflow/contrib/libsvm/ops/libsvm_ops.cc b/tensorflow/contrib/libsvm/ops/libsvm_ops.cc deleted file mode 100644 index dec946189e3..00000000000 --- a/tensorflow/contrib/libsvm/ops/libsvm_ops.cc +++ /dev/null @@ -1,58 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { - -using shape_inference::InferenceContext; - -REGISTER_OP("DecodeLibsvm") - .Input("input: string") - .Output("label: label_dtype") - .Output("feature_indices: int64") - .Output("feature_values: dtype") - .Output("feature_shape: int64") - .Attr("dtype: {float, double, int32, int64} = DT_FLOAT") - .Attr("label_dtype: {float, double, int32, int64} = DT_INT64") - .Attr("num_features: int >= 1") - .SetShapeFn([](InferenceContext* c) { - c->set_output(0, c->input(0)); - - c->set_output(1, c->Matrix(InferenceContext::kUnknownDim, - InferenceContext::kUnknownDim)); - c->set_output(2, c->Vector(InferenceContext::kUnknownDim)); - c->set_output(3, c->Vector(InferenceContext::kUnknownDim)); - - return Status::OK(); - }) - - .Doc(R"doc( -Convert LibSVM input to tensors. The output consists of -a label and a feature tensor. The shape of the label tensor -is the same as input and the shape of the feature tensor is -`[input_shape, num_features]`. - -input: Each string is a record in the LibSVM. -label: A tensor of the same shape as input. -feature_indices: A 2-D int64 tensor of dense_shape [N, ndims]. -feature_values: A 1-D tensor of any type and dense_shape [N]. -feature_shape: A 1-D int64 tensor of dense_shape [ndims]. -num_features: The number of features. -)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/libsvm/python/kernel_tests/decode_libsvm_op_test.py b/tensorflow/contrib/libsvm/python/kernel_tests/decode_libsvm_op_test.py deleted file mode 100644 index 8390ddda902..00000000000 --- a/tensorflow/contrib/libsvm/python/kernel_tests/decode_libsvm_op_test.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for DecodeLibsvm op.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.libsvm.python.ops import libsvm_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import sparse_ops -from tensorflow.python.platform import test - - -class DecodeLibsvmOpTest(test.TestCase): - - def testBasic(self): - with self.cached_session() as sess: - content = [ - "1 1:3.4 2:0.5 4:0.231", "1 2:2.5 3:inf 5:0.503", - "2 3:2.5 2:nan 1:0.105" - ] - sparse_features, labels = libsvm_ops.decode_libsvm( - content, num_features=6) - features = sparse_ops.sparse_tensor_to_dense( - sparse_features, validate_indices=False) - - self.assertAllEqual(labels.get_shape().as_list(), [3]) - - features, labels = sess.run([features, labels]) - self.assertAllEqual(labels, [1, 1, 2]) - self.assertAllClose( - features, [[0, 3.4, 0.5, 0, 0.231, 0], [0, 0, 2.5, np.inf, 0, 0.503], - [0, 0.105, np.nan, 2.5, 0, 0]]) - - def testNDimension(self): - with self.cached_session() as sess: - content = [["1 1:3.4 2:0.5 4:0.231", "1 1:3.4 2:0.5 4:0.231"], - ["1 2:2.5 3:inf 5:0.503", "1 2:2.5 3:inf 5:0.503"], - ["2 3:2.5 2:nan 1:0.105", "2 3:2.5 2:nan 1:0.105"]] - sparse_features, labels = libsvm_ops.decode_libsvm( - content, num_features=6, label_dtype=dtypes.float64) - features = sparse_ops.sparse_tensor_to_dense( - sparse_features, validate_indices=False) - - self.assertAllEqual(labels.get_shape().as_list(), [3, 2]) - - features, labels = sess.run([features, labels]) - self.assertAllEqual(labels, [[1, 1], [1, 1], [2, 2]]) - self.assertAllClose( - features, [[[0, 3.4, 0.5, 0, 0.231, 0], [0, 3.4, 0.5, 0, 0.231, 0]], [ - [0, 0, 2.5, np.inf, 0, 0.503], [0, 0, 2.5, np.inf, 0, 0.503] - ], [[0, 0.105, np.nan, 2.5, 0, 0], [0, 0.105, np.nan, 2.5, 0, 0]]]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/libsvm/python/ops/libsvm_ops.py b/tensorflow/contrib/libsvm/python/ops/libsvm_ops.py deleted file mode 100644 index 937a49395e2..00000000000 --- a/tensorflow/contrib/libsvm/python/ops/libsvm_ops.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Libsvm decoder.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.libsvm.ops import gen_libsvm_ops -from tensorflow.contrib.util import loader -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.platform import resource_loader -from tensorflow.python.util.deprecation import deprecated - - -_libsvm_ops_so = loader.load_op_library( - resource_loader.get_path_to_datafile("_libsvm_ops.so")) - - -@deprecated(None, - 'tf.contrib.libsvm will be removed in 2.0, the support for libsvm ' - 'format will continue to be provided in tensorflow-io: ' - 'https://github.com/tensorflow/io') -def decode_libsvm(content, num_features, dtype=None, label_dtype=None): - """Convert Libsvm records to a tensor of label and a tensor of feature. - - Args: - content: A `Tensor` of type `string`. Each string is a record/row in - the Libsvm format. - num_features: The number of features. - dtype: The type of the output feature tensor. Default to tf.float32. - label_dtype: The type of the output label tensor. Default to tf.int64. - - Returns: - features: A `SparseTensor` of the shape `[input_shape, num_features]`. - labels: A `Tensor` of the same shape as content. - """ - labels, indices, values, shape = gen_libsvm_ops.decode_libsvm( - content, num_features, dtype=dtype, label_dtype=label_dtype) - return sparse_tensor.SparseTensor(indices, values, shape), labels - - -ops.NotDifferentiable("DecodeLibSVM") diff --git a/tensorflow/contrib/linear_optimizer/BUILD b/tensorflow/contrib/linear_optimizer/BUILD deleted file mode 100644 index c25602a36e5..00000000000 --- a/tensorflow/contrib/linear_optimizer/BUILD +++ /dev/null @@ -1,149 +0,0 @@ -# Description: -# Contains ops to train linear models on top of TensorFlow. -# APIs here are meant to evolve over time. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "sdca_ops_py", - srcs = [ - "__init__.py", - "python/ops/sdca_ops.py", - "python/sdca_optimizer.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":sharded_mutable_dense_hashtable_py", - ":sparse_feature_column_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:nn_ops", - "//tensorflow/python:sdca_ops_gen", - "//tensorflow/python:state_ops", - "//tensorflow/python:summary", - "//tensorflow/python:util", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "sdca_ops_test", - size = "medium", - srcs = ["python/kernel_tests/sdca_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_gpu", - "no_pip_gpu", - ], - deps = [ - ":sdca_ops_py", - ":sparse_feature_column_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:sdca_ops_gen", - "//tensorflow/python:variables", - ], -) - -py_library( - name = "sharded_mutable_dense_hashtable_py", - srcs = ["python/ops/sharded_mutable_dense_hashtable.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/lookup:lookup_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - ], -) - -py_test( - name = "sharded_mutable_dense_hashtable_test", - size = "small", - srcs = ["python/ops/sharded_mutable_dense_hashtable_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":sharded_mutable_dense_hashtable_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -py_library( - name = "sparse_feature_column_py", - srcs = ["python/ops/sparse_feature_column.py"], - srcs_version = "PY2AND3", - deps = ["//tensorflow/python:framework_for_generated_wrappers"], -) - -py_test( - name = "sparse_feature_column_test", - size = "small", - srcs = ["python/ops/sparse_feature_column_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":sparse_feature_column_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -py_library( - name = "sdca_estimator_py", - srcs = ["python/sdca_estimator.py"], - srcs_version = "PY2AND3", - deps = [ - ":sdca_ops_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:tensor_util", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - ], -) - -py_test( - name = "sdca_estimator_test", - srcs = ["python/sdca_estimator_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - deps = [ - ":sdca_estimator_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:sparse_tensor", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/linear_optimizer/__init__.py b/tensorflow/contrib/linear_optimizer/__init__.py deleted file mode 100644 index d447487b4a5..00000000000 --- a/tensorflow/contrib/linear_optimizer/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for training linear models. - -## This package provides optimizers to train linear models. - -@@SdcaModel -@@SparseFeatureColumn -@@SDCAOptimizer -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.linear_optimizer.python.ops.sdca_ops import SdcaModel -from tensorflow.contrib.linear_optimizer.python.ops.sparse_feature_column import SparseFeatureColumn -from tensorflow.contrib.linear_optimizer.python.sdca_optimizer import SDCAOptimizer - -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/linear_optimizer/kernels/g3doc/mod_newton.png b/tensorflow/contrib/linear_optimizer/kernels/g3doc/mod_newton.png deleted file mode 100644 index 8e3b7c30b6b..00000000000 Binary files a/tensorflow/contrib/linear_optimizer/kernels/g3doc/mod_newton.png and /dev/null differ diff --git a/tensorflow/contrib/linear_optimizer/kernels/g3doc/newton.png b/tensorflow/contrib/linear_optimizer/kernels/g3doc/newton.png deleted file mode 100644 index 5b6b46ba1db..00000000000 Binary files a/tensorflow/contrib/linear_optimizer/kernels/g3doc/newton.png and /dev/null differ diff --git a/tensorflow/contrib/linear_optimizer/kernels/g3doc/newton_compare_experiment.png b/tensorflow/contrib/linear_optimizer/kernels/g3doc/newton_compare_experiment.png deleted file mode 100644 index a19f4520953..00000000000 Binary files a/tensorflow/contrib/linear_optimizer/kernels/g3doc/newton_compare_experiment.png and /dev/null differ diff --git a/tensorflow/contrib/linear_optimizer/kernels/g3doc/readme.md b/tensorflow/contrib/linear_optimizer/kernels/g3doc/readme.md deleted file mode 100644 index 5fe883d647e..00000000000 --- a/tensorflow/contrib/linear_optimizer/kernels/g3doc/readme.md +++ /dev/null @@ -1,287 +0,0 @@ -$$ \def\a{\alpha} \def\d{\Delta\a} \def\l{\ell} \def\P{\mathcal{P}}$$ - -# Distributed SDCA - -We want to minimize on $$K$$ machines the following objective - -$$ P(w) = \frac{1}{n}\sum_{i=1}^n \l_i(x_i^T w)+\lambda g(w) $$ - -By Fenchel duality, this is equivalent to maximizing its dual - -$$ D(\a) = \frac{1}{n} \left(\sum_{i=1}^n -\l_i^\star(-\a_i)\right) -\lambda g^\star\left(\tfrac{1}{\lambda n} X\a\right) $$ - -which can be done very efficiently on a single machine with SDCA [3]. - -Here, $$f^\star$$ denotes the convex dual of a convex function $$f$$, $$\l_i$$ -is the loss for the example $$i$$, $$n$$ is the total number of examples and -$$\lambda n$$ is the L2 parameter. - -Following [1,2], we use a data partition $$\P_1,\dots,\P_K$$ of -$$\{1,2,\dots,n\}$$ such that $$\P_k$$ contains the examples on machine $$k$$. -For an $$n$$-dimensional vector $$h$$, we denote by $$h_{[k]}$$ the -$$n$$-dimensional vector restricted to the machine $$k$$: $$(h_{[k]})_i = h_i$$ -if $$i\in\P_k$$ and $$0$$ otherwise. - -## CoCoA+ Local Solver - -The local subproblem on machine $$k$$ is [1, 2] - -$$ \max_{\d_{[k]}} \mathcal{G}^{\sigma}_k (\d_{[k]}) $$ - -with - -$$ -\mathcal{G}^{\sigma}_k (\d_{[k]}) = --\frac{1}{n} \sum_{i\in\P_k}\l_i^\star(-\a_i-(\d_{[k]})_i) -\frac{1}{n} w^T X -\d_{[k]}- \frac{\lambda}{2}\sigma \left\| \frac{1}{\lambda n} X \d_{[k]} -\right\|^2 $$ - -$$\sigma$$ is a parameter the measures the difficulty of the data partition. -CoCoA+ makes the choice - -$$ \sigma = K $$ - -This decision is motivated in [2] and shown to be more efficient than the -previous CoCoA choice ($$\sigma = 1$$). - -For one example, the problem is simply - -$$ \max_{\d} \left\{ D_i(\d) = -\l_i^\star(-(\a_i+\d)) - \bar{y}_i \d - \frac{A}{2} \d^2 \right\} $$ - -where we have defined $$A=\sigma X_i^2/(\lambda n) $$ and $$ \bar{y}_i = w^T X_i -$$. - -To take into account example weights, it suffices to replace $$\frac{1}{n}$$ by -$$\frac{s_i}{S}$$ where $$s_i$$ is the weight of the i-th example and $$S=\sum -s_i$$. For our problem, this will only change $$A$$ to $$\sigma -X_i^2s_i/(\lambda S) $$. - -### Hinge Loss - -Hinge loss is given by $$ \l_i(u) = \max(0,1-y u) $$. Its convex dual is $$ -\l_i^\star(-a) = -a y $$ with the constraint $$ a y\in [0,1] $$. - -The solution for the update is given explicitly in [3]. To derive the CoCoA+ -formulation, we replace $$\lambda$$ by $$\frac{\lambda}{\sigma}$$. This gives - -$$ \d = \frac{y - \bar{y}}{A} $$ - -with the restriction that $$y(\a+\d)\in(0,1)$$. - -### Smooth Hinge Loss - -Smooth hinge loss is given by - -$$ \l_i(u) = -\begin{cases} -0 \:\:\: & y_i u \geq 1\\ -1-y_i u -\gamma/2 \:\:\:& y_i u \leq1-\gamma \\ -\frac{(1-y_i u)^2}{2\gamma} & \text{otherwise} -\end{cases} $$ - -The optimal $$\d$$ is computed to be - -$$\d = \frac{y-\bar{y}-\gamma\a}{A+\gamma} $$ - -with the restriction that $$y(\a+\d)\in(0,1)$$. We see that we recover standard -hinge update for $$\gamma = 0$$. The details of the computation can be found in -Appendix. - -### Squared Loss - -Squared loss is $$ \l_i(u) = \frac{1}{2}(u-y)^2 $$ with dual $$ \l_i^\star(v) = -\frac{1}{2}v^2+y v$$. - -The closed form solution for squared loss is given in [4]. By replacing again -$$\lambda$$ by $$\frac{\lambda}{\sigma}$$ we obtain - -$$ \d = -\frac{\a + w^T X_i - y}{1 + \frac{\sigma X_i^2}{2 \lambda n}} $$ - -### Logistic loss - -Logistic loss is $$ \l_i(u) = \log (1+e^{-uy_i}) $$ and its dual is - -$$ \l_i^\star(v) = -vy_i\log(-vy_i) + (1+vy_i) -\log(1+vy_i) $$ - -The label $$y_i$$ is $$\pm 1$$ and the dual loss is only defined for $$ -y_i -v\in (0,1) $$. We then have the constraint - -$$ y_i (\a+\d) \in (0,1) $$ - -The problem of finding the maximum of $$ D(\d) $$ can be reformulated as the -problem of finding the unique zero of its derivative. Newton method works well -for finding the zero of $$ D'(\d) $$ but can be a bit unstable due to the -constraint requiring $$y_i(\a+\d)$$ be in the range $$(0,1)$$ (more on this -below). - -To avoid this problem, we make the following change of variable - -$$ y(\a+\d) = \frac{1}{2}(1+\tanh x) $$ - -This enforces the constraint and is well suited because the objective derivative -has the following simple form: - -$$ D' = H(x) = -2y x - \bar{y} + A\a -\frac{A}{2y}(1+\tanh x) $$ - -with derivative - -$$ H'(x) = -2y - \frac{A}{2y}(1-\tanh^2 x) $$ - -This function is always positive or always negative so that $$H$$ is strictly -monotonic. - -We can start Newton algorithm at $$x_0=0$$ which corresponds to $$ y(\a+\d) = -0.5 $$. A Newton step is given by - -$$x_{k+1} = x_k - \frac{H(x_k)}{H'(x_k)} $$ - -The convergence is very fast with the modified function and 5 Newton steps -should be largely enough. - -##### Plots and experiments - -Below is a plot where standard Newton will be unstable. The starting point is -$$x=0.5$$ and we leave the definition interval after the first step (the orange -line is the tangent at the starting point). Newton algorithm will return 0 where -the real zero is at 0.10. - -![newton](newton.png) - -The same plot for the modified function (with the same parameters) is almost a -straight line. The starting point is $$x=0$$ and we see that the convergence is -fast. - -![mod_newton](mod_newton.png) - -We see that the result will be $$ \frac{1+\tanh(-1.1)}{2} \sim 0.10$$ as -expected. - -On criteo dataset, the usual Newton method goes out of range for a small (but -non negligible) fraction of the examples. The returned dual in these cases will -be $$0$$ or $$\pm 1$$. The modified Newton algorithm always find the true zero -and achieves a better log loss. - -The blue lines represent the modified Newton (evaluation and training) and the -orange line is the normal Newton algorithm (training). - -![experiment](newton_compare_experiment.png) - -TODO(vgodet): Update the plot with eval_continuous for Newton - -Note: Newton seems to converge faster at the beginning because it is taking more -aggressive steps when going out-of-range. - -##### Proof of convergence - -The second derivative of $$H$$ - -$$ H''(x) = \frac{A}{y} \tanh x (1-\tanh^2 x) $$ - -is bounded and quadratic convergence should be guaranteed if we are close enough -to the solution (see -https://en.wikipedia.org/wiki/Newton%27s_method#Proof_of_quadratic_convergence_for_Newton.27s_iterative_method -for the proof). - -However we can't really know if we are close to the zero. To prove the -convergence in any cases, we can use Kantovitch Theorem (reviewed in [5]). The -sufficient condition to have convergence is that we start at a point $$ x_0 $$ -such that - -$$ -\left|\frac{4A H(x_0)}{H'(x_0)^2} \right|\leq 1 -$$ - -If $$ A$$ is not small, the starting point $$x_0 = 0$$ doesn't satisfy this -condition and we may solve the above inequality to find a starting point which -does. - -However, in practice, convergence with $$x_0 = 0$$ always happens (tested for a -sample of generic values for the parameters). - -### Poisson log loss - -Poisson log loss is defined as $$ \l(u) = e^u - uy $$ for label $$y \geq 0.$$ -Its dual is - -$$ \l^\star(v) = (y+v) (\log(y+v) - 1) $$ - -and is only defined for $$ y+v > 0 $$. We then have the constraint - -$$ y > \a+\d. $$ - -The dual is - -$$ D(\d) = -(y-\a-\d) (\log(y-\a-\d) - 1) - \bar{y} \d - \frac{A}{2} \d^2 $$ - -and its derivative is, - -$$ D'(\d) = \log(y-\a-\d) - \bar{y} - A\d $$ - -Similar to the logistic loss, we perform a change of variable to handle the -constraint on $$ \d $$ - -$$ y - (\a+\d) = e^x $$ - -After this change of variable, the goal is to find the zero of this function - -$$ H(x) = x - \bar{y} -A(y-\a-e^x) $$ - -whose first derivative is - -$$ H'(x) = 1+Ae^x $$ - -Since this function is always positive, $$H$$ is increasing and has a unique -zero. - -We can start Newton algorithm at $$\d=0$$ which corresponds to $$ x = -\log(y-\a)$$. As before the Newton step is given by - -$$x_{k+1} = x_k - \frac{H(x_k)}{H'(x_k)}. $$ - -### References - -[1] C. Ma et al., Adding vs. Averaging in Distributed Primal-Dual Optimization, -arXiv:1502.03508, 2015 - -[2] C. Ma et al., Distributed Optimization with Arbitrary Local Solvers, -arxiv:1512.04039, 2015 - -[3] S. Shalev-Shwartz, T. Zhang, Stochastic Dual Coordinate Ascent Methods for -Regularized Loss Minimization, 2013 - -[4] S. Shalev-Shwartz, T. Zhang, Accelerated Proximal Stochastic Dual Coordinate -Ascent for Regularized Loss Minimization, 2013 - -[5] A. Galantai, The theory of Newton’s method, 2000 - -## Appendix - -##### Dual computation for smooth hinge loss - -We want to compute $$\l^\star(v) = \max_u [ uv-\l(u) ] $$ where $$\l$$ is smooth -hinge loss. We thus have to solve $$v=\l'(u)$$. The derivative of smooth hinge -loss is given by - -$$ \l'(u) = -\begin{cases} -0 \:\:\: & y_i u \geq 1\\ --y \:\:\:& y_i u \leq1-\gamma \\ -\frac{u-y}{\gamma} & \text{otherwise} -\end{cases} $$ - -By solving for $$v$$, we find the dual of smooth hinge loss as - -$$ \l^\star(v) = yv + \frac{\gamma}{2}v^2 $$ - -with the restriction $$ yv \in (0,1) $$. - -Now, we can now minimize the dual objective with respect to $$\d$$ - -$$ D(\a+\d) = -\l^\star(-\a-\d)-\bar{y}\d-\frac{A}{2} \d^2 $$ - -which gives the expected result - -$$\d = \frac{y-\bar{y}-\gamma\a}{A+\gamma} $$ - -with the constraint $$ y(\a+\d) \in (0,1)$$. diff --git a/tensorflow/contrib/linear_optimizer/python/kernel_tests/sdca_ops_test.py b/tensorflow/contrib/linear_optimizer/python/kernel_tests/sdca_ops_test.py deleted file mode 100644 index 9dea5eff337..00000000000 --- a/tensorflow/contrib/linear_optimizer/python/kernel_tests/sdca_ops_test.py +++ /dev/null @@ -1,1339 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for SdcaModel (deprecated). - -This module and all its submodules are deprecated. To UPDATE or USE linear -optimizers, please check its latest version in core: -tensorflow_estimator/python/estimator/canned/linear_optimizer/. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random -import threading - -from tensorflow.contrib.linear_optimizer.python.ops.sdca_ops import SdcaModel -from tensorflow.contrib.linear_optimizer.python.ops.sparse_feature_column import SparseFeatureColumn -from tensorflow.core.example import example_pb2 -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.framework.test_util import TensorFlowTestCase -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_sdca_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.platform import googletest - -_MAX_ITERATIONS = 100 -_SHARD_NUMBERS = [None, 1, 3] -_NUM_LOSS_PARTITIONS = [4] - - -def make_example_proto(feature_dict, target, value=1.0): - e = example_pb2.Example() - features = e.features - - features.feature['target'].float_list.value.append(target) - - for key, values in feature_dict.items(): - features.feature[key + '_indices'].int64_list.value.extend(values) - features.feature[key + '_values'].float_list.value.extend([value] * - len(values)) - - return e - - -def make_example_dict(example_protos, example_weights): - - def parse_examples(example_protos): - features = { - 'target': - parsing_ops.FixedLenFeature( - shape=[1], dtype=dtypes.float32, default_value=0), - 'age_indices': - parsing_ops.VarLenFeature(dtype=dtypes.int64), - 'age_values': - parsing_ops.VarLenFeature(dtype=dtypes.float32), - 'gender_indices': - parsing_ops.VarLenFeature(dtype=dtypes.int64), - 'gender_values': - parsing_ops.VarLenFeature(dtype=dtypes.float32) - } - return parsing_ops.parse_example( - [e.SerializeToString() for e in example_protos], features) - - parsed = parse_examples(example_protos) - sparse_features = [ - SparseFeatureColumn( - array_ops.reshape( - array_ops.split( - value=parsed['age_indices'].indices, - num_or_size_splits=2, - axis=1)[0], [-1]), - array_ops.reshape(parsed['age_indices'].values, [-1]), - array_ops.reshape(parsed['age_values'].values, [-1])), - SparseFeatureColumn( - array_ops.reshape( - array_ops.split( - value=parsed['gender_indices'].indices, - num_or_size_splits=2, - axis=1)[0], [-1]), - array_ops.reshape(parsed['gender_indices'].values, [-1]), - array_ops.reshape(parsed['gender_values'].values, [-1])) - ] - return dict( - sparse_features=sparse_features, - dense_features=[], - example_weights=example_weights, - example_labels=array_ops.reshape(parsed['target'], [-1]), - example_ids=['%d' % i for i in range(0, len(example_protos))]) - - -def make_random_examples_and_variables_dicts(num_examples, dim, num_non_zero): - random.seed(1) - - sparse_features = [ - SparseFeatureColumn( - [i for i in range(num_examples) for _ in range(num_non_zero)], [ - i for _ in range(num_examples) - for i in random.sample(range(dim), num_non_zero) - ], - [num_non_zero**(-0.5) for _ in range(num_examples * num_non_zero)]) - ] - examples_dict = dict( - sparse_features=sparse_features, - dense_features=[], - example_weights=[random.random() for _ in range(num_examples)], - example_labels=[ - 1. if random.random() > 0.5 else 0. for _ in range(num_examples) - ], - example_ids=[str(i) for i in range(num_examples)]) - - weights = variables_lib.VariableV1( - array_ops.zeros([dim], dtype=dtypes.float32)) - variables_dict = dict( - sparse_features_weights=[weights], - dense_features_weights=[]) - - return examples_dict, variables_dict - - -def make_variable_dict(max_age, max_gender, num_shards=None, partitioned=False): - # TODO(sibyl-toe9oF2e): Figure out how to derive max_age & max_gender from - # examples_dict. - partitioner = None - if partitioned: - partitioner = partitioned_variables.fixed_size_partitioner(num_shards=2, - axis=0) - with variable_scope.variable_scope( - name_or_scope=('variables/shard_{}'.format(num_shards) - if num_shards else 'variables'), - partitioner=partitioner): - age_weights = variable_scope.get_variable( - name='age', - initializer=array_ops.zeros([max_age + 1], dtype=dtypes.float32)) - gender_weights = variable_scope.get_variable( - name='gender', - initializer=array_ops.zeros([max_gender + 1], dtype=dtypes.float32)) - return dict( - sparse_features_weights=[age_weights, gender_weights], - dense_features_weights=[]) - - -def make_dense_examples_and_variables_dicts(dense_features_values, weights, - labels): - """Creates examples and variables dictionaries for dense features. - - Variables shapes are inferred from the list of dense feature values passed as - argument. - - Args: - dense_features_values: The values of the dense features - weights: The example weights. - labels: The example labels. - Returns: - One dictionary for the examples and one for the variables. - """ - dense_tensors = [] - dense_weights = [] - for dense_feature in dense_features_values: - dense_tensor = ops.convert_to_tensor(dense_feature, dtype=dtypes.float32) - check_shape_op = control_flow_ops.Assert( - math_ops.less_equal(array_ops.rank(dense_tensor), 2), - ['dense_tensor shape must be [batch_size, dimension] or [batch_size]']) - # Reshape to [batch_size, dense_column_dimension]. - with ops.control_dependencies([check_shape_op]): - dense_tensor = array_ops.reshape( - dense_tensor, [dense_tensor.get_shape().as_list()[0], -1]) - dense_tensors.append(dense_tensor) - # Add variables of shape [feature_column_dimension]. - dense_weights.append( - variables_lib.VariableV1( - array_ops.zeros( - [dense_tensor.get_shape().as_list()[1]], dtype=dtypes.float32))) - - examples_dict = dict( - sparse_features=[], - dense_features=dense_tensors, - example_weights=weights, - example_labels=labels, - example_ids=['%d' % i for i in range(0, len(labels))]) - variables_dict = dict( - sparse_features_weights=[], dense_features_weights=dense_weights) - - return examples_dict, variables_dict - - -def get_binary_predictions_for_logistic(predictions, cutoff=0.5): - return math_ops.cast( - math_ops.greater_equal(predictions, - array_ops.ones_like(predictions) * cutoff), - dtype=dtypes.int32) - - -def get_binary_predictions_for_hinge(predictions): - return math_ops.cast( - math_ops.greater_equal(predictions, array_ops.zeros_like(predictions)), - dtype=dtypes.int32) - - -# TODO(sibyl-Mooth6ku): Add tests that exercise L1 and Shrinking. -# TODO(sibyl-vie3Poto): Refactor tests to avoid repetition of boilerplate code. -class SdcaModelTest(TensorFlowTestCase): - """Base SDCA optimizer test class for any loss type.""" - - def _single_threaded_test_session(self): - config = config_pb2.ConfigProto( - inter_op_parallelism_threads=1, intra_op_parallelism_threads=1) - return self.test_session(use_gpu=False, config=config) - - -class SdcaWithLogisticLossTest(SdcaModelTest): - """SDCA optimizer test class for logistic loss.""" - - def testSimple(self): - # Setup test data - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 1), - ] - example_weights = [1.0, 1.0] - for num_shards in _SHARD_NUMBERS: - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1, num_shards) - options = dict( - symmetric_l2_regularization=1, - symmetric_l1_regularization=0, - num_table_shards=num_shards, - loss_type='logistic_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - unregularized_loss = lr.unregularized_loss(examples) - loss = lr.regularized_loss(examples) - predictions = lr.predictions(examples) - self.assertAllClose(0.693147, unregularized_loss.eval()) - self.assertAllClose(0.693147, loss.eval()) - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - # The high tolerance in unregularized_loss comparisons is due to the - # fact that it's possible to trade off unregularized_loss vs. - # regularization and still have a sum that is quite close to the - # optimal regularized_loss value. SDCA's duality gap only ensures that - # the regularized_loss is within 0.01 of optimal. - # 0.525457 is the optimal regularized_loss. - # 0.411608 is the unregularized_loss at that optimum. - self.assertAllClose(0.411608, unregularized_loss.eval(), atol=0.05) - self.assertAllClose(0.525457, loss.eval(), atol=0.01) - predicted_labels = get_binary_predictions_for_logistic(predictions) - self.assertAllEqual([0, 1], predicted_labels.eval()) - self.assertAllClose( - 0.01, lr.approximate_duality_gap().eval(), rtol=1e-2, atol=1e-2) - - def testPartitionedPrimals(self): - # Setup test data - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 1), - ] - example_weights = [1.0, 1.0] - for num_shards in _SHARD_NUMBERS: - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1, num_shards, partitioned=True) - options = dict( - symmetric_l2_regularization=1, - symmetric_l1_regularization=0, - num_table_shards=num_shards, - loss_type='logistic_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - unregularized_loss = lr.unregularized_loss(examples) - loss = lr.regularized_loss(examples) - predictions = lr.predictions(examples) - self.assertAllClose(0.693147, unregularized_loss.eval()) - self.assertAllClose(0.693147, loss.eval()) - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - # The high tolerance in unregularized_loss comparisons is due to the - # fact that it's possible to trade off unregularized_loss vs. - # regularization and still have a sum that is quite close to the - # optimal regularized_loss value. SDCA's duality gap only ensures that - # the regularized_loss is within 0.01 of optimal. - # 0.525457 is the optimal regularized_loss. - # 0.411608 is the unregularized_loss at that optimum. - self.assertAllClose(0.411608, unregularized_loss.eval(), atol=0.05) - self.assertAllClose(0.525457, loss.eval(), atol=0.01) - predicted_labels = get_binary_predictions_for_logistic(predictions) - self.assertAllEqual([0, 1], predicted_labels.eval()) - self.assertAllClose( - 0.01, lr.approximate_duality_gap().eval(), rtol=1e-2, atol=1e-2) - - def testSomePartitionedPrimals(self): - # Setup test data - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [0], - 'gender': [1] - }, 1), - ] - example_weights = [1.0, 1.0] - for num_shards in _SHARD_NUMBERS: - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - # Explicitly make age a [1]-shaped Variable (which cannot be - # partitioned), while making gender a PartitionedVariable. - age_weights = variables_lib.VariableV1( - array_ops.zeros([1], dtype=dtypes.float32)) - with variable_scope.variable_scope( - name_or_scope=('variables/shard_{}'.format(num_shards) - if num_shards else 'variables'), - partitioner=partitioned_variables.fixed_size_partitioner( - num_shards=2, axis=0)): - gender_weights = variable_scope.get_variable( - name='gender', - initializer=array_ops.zeros([2], dtype=dtypes.float32)) - variables = dict( - sparse_features_weights=[age_weights, gender_weights], - dense_features_weights=[]) - options = dict( - symmetric_l2_regularization=1, - symmetric_l1_regularization=0, - num_table_shards=num_shards, - loss_type='logistic_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - unregularized_loss = lr.unregularized_loss(examples) - loss = lr.regularized_loss(examples) - predictions = lr.predictions(examples) - self.assertAllClose(0.693147, unregularized_loss.eval()) - self.assertAllClose(0.693147, loss.eval()) - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - # The high tolerance in unregularized_loss comparisons is due to the - # fact that it's possible to trade off unregularized_loss vs. - # regularization and still have a sum that is quite close to the - # optimal regularized_loss value. SDCA's duality gap only ensures that - # the regularized_loss is within 0.01 of optimal. - # 0.525457 is the optimal regularized_loss. - # 0.593014 is the unregularized_loss at that optimum. - self.assertAllClose(0.512591, unregularized_loss.eval(), atol=0.05) - self.assertAllClose(0.593014, loss.eval(), atol=0.01) - predicted_labels = get_binary_predictions_for_logistic(predictions) - self.assertAllEqual([0, 1], predicted_labels.eval()) - self.assertAllClose( - 0.01, lr.approximate_duality_gap().eval(), rtol=1e-2, atol=1e-2) - - def testSparseRandom(self): - dim = 20 - num_examples = 1000 - # Number of non-zero features per example. - non_zeros = 10 - # Setup test data. - with self._single_threaded_test_session(): - examples, variables = make_random_examples_and_variables_dicts( - num_examples, dim, non_zeros) - options = dict( - symmetric_l2_regularization=.1, - symmetric_l1_regularization=0, - num_table_shards=1, - adaptive=False, - loss_type='logistic_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - train_op = lr.minimize() - for _ in range(4): - train_op.run() - lr.update_weights(train_op).run() - # Duality gap is 1.4e-5. - # It would be 0.01 without shuffling and 0.02 with adaptive sampling. - self.assertNear(0.0, lr.approximate_duality_gap().eval(), err=1e-3) - - def testSparseDuplicate(self): - # Setup test data - example_protos = [ - make_example_proto({ - 'age': [0] * 5, - 'gender': [0] * 5 - }, 0), - make_example_proto({ - 'age': [1] * 5, - 'gender': [1] * 5 - }, 1), - ] - example_weights = [1.0, 1.0] - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1) - options = dict( - symmetric_l2_regularization=1, - symmetric_l1_regularization=0, - loss_type='logistic_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - train_op = lr.minimize() - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - 'Duplicate'): - train_op.run() - - def testDistributedSimple(self): - # Distributed SDCA may not converge if the workers update concurrently the - # same example. In this test the examples are partitioned across workers. - # The examples are the same for all workers, just the example_ids are - # different. - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 1), - ] - example_weights = [1.0, 1.0] - examples = make_example_dict(example_protos, example_weights) - example_ids = array_ops.placeholder( - dtypes.string, shape=(len(example_weights),)) - examples['example_ids'] = example_ids - variables = make_variable_dict(1, 1) - # We need each thread to keep its own device stack or the device scopes - # won't be properly nested. - ops.get_default_graph().switch_to_thread_local() - for num_shards in _SHARD_NUMBERS: - for num_loss_partitions in _NUM_LOSS_PARTITIONS: - with self._single_threaded_test_session(): - options = dict( - # Keep the same solution as for TestSimple: since the number of - # examples is multplied by num_loss_partitions, multiply also - # L2 by the same value. - symmetric_l2_regularization=num_loss_partitions, - symmetric_l1_regularization=0, - loss_type='logistic_loss', - num_table_shards=num_shards, - num_loss_partitions=num_loss_partitions) - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - unregularized_loss = lr.unregularized_loss(examples) - loss = lr.regularized_loss(examples) - predictions = lr.predictions(examples) - self.assertAllClose(0.693147, unregularized_loss.eval()) - self.assertAllClose(0.693147, loss.eval()) - - train_op = lr.minimize() - - def minimize(worker_id): - with self._single_threaded_test_session(): - feed_dict = {example_ids: [ - str(i + worker_id*len(example_weights)) for i in range( - len(example_weights))]} - for _ in range(_MAX_ITERATIONS): - train_op.run(feed_dict=feed_dict) # pylint: disable=cell-var-from-loop - - threads = [] - for worker_id in range(num_loss_partitions): - threads.append(threading.Thread(target=minimize, args=(worker_id,))) - threads[-1].start() - - for t in threads: - t.join() - lr.update_weights(train_op).run(feed_dict={ - example_ids: [str(i) for i in range(len(example_weights))]}) - - # Test only the unregularized loss because the optimal value of the - # regularized loss depends on num_loss_partitions. - self.assertAllClose(0.411608, unregularized_loss.eval(), atol=0.02) - predicted_labels = get_binary_predictions_for_logistic(predictions) - self.assertAllEqual([0, 1], predicted_labels.eval()) - self.assertNear(0.0, lr.approximate_duality_gap().eval(), 0.02) - - def testSimpleNoL2(self): - # Same as test above (so comments from above apply) but without an L2. - # The algorithm should behave as if we have an L2 of 1 in optimization but - # 0 in regularized_loss. - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 1), - ] - example_weights = [1.0, 1.0] - for num_shards in _SHARD_NUMBERS: - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1, num_shards) - options = dict( - symmetric_l2_regularization=0, - symmetric_l1_regularization=0, - num_table_shards=num_shards, - loss_type='logistic_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - unregularized_loss = lr.unregularized_loss(examples) - loss = lr.regularized_loss(examples) - predictions = lr.predictions(examples) - self.assertAllClose(0.693147, unregularized_loss.eval()) - self.assertAllClose(0.693147, loss.eval()) - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - - # There is neither L1 nor L2 loss, so regularized and unregularized - # losses should be exactly the same. - self.assertAllClose(0.40244, unregularized_loss.eval(), atol=0.01) - self.assertAllClose(0.40244, loss.eval(), atol=0.01) - predicted_labels = get_binary_predictions_for_logistic(predictions) - self.assertAllEqual([0, 1], predicted_labels.eval()) - self.assertAllClose( - 0.01, lr.approximate_duality_gap().eval(), rtol=1e-2, atol=1e-2) - - def testSomeUnweightedExamples(self): - # Setup test data with 4 examples, but should produce the same - # results as testSimple. - example_protos = [ - # Will be used. - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0), - # Will be ignored. - make_example_proto({ - 'age': [1], - 'gender': [0] - }, 0), - # Will be used. - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 1), - # Will be ignored. - make_example_proto({ - 'age': [1], - 'gender': [0] - }, 1), - ] - example_weights = [1.0, 0.0, 1.0, 0.0] - for num_shards in _SHARD_NUMBERS: - with self._single_threaded_test_session(): - # Only use examples 0 and 2 - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1, num_shards) - options = dict( - symmetric_l2_regularization=1, - symmetric_l1_regularization=0, - num_table_shards=num_shards, - loss_type='logistic_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - unregularized_loss = lr.unregularized_loss(examples) - loss = lr.regularized_loss(examples) - predictions = lr.predictions(examples) - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - - self.assertAllClose(0.411608, unregularized_loss.eval(), atol=0.05) - self.assertAllClose(0.525457, loss.eval(), atol=0.01) - predicted_labels = get_binary_predictions_for_logistic(predictions) - self.assertAllClose([0, 1, 1, 1], predicted_labels.eval()) - self.assertAllClose( - 0.0, lr.approximate_duality_gap().eval(), rtol=1e-2, atol=1e-2) - - def testFractionalExampleLabel(self): - # Setup test data with 1 positive, and 1 mostly-negative example. - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0.1), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 0.9), - ] - example_weights = [1.0, 1.0] - for num_shards in _SHARD_NUMBERS: - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1, num_shards) - options = dict( - symmetric_l2_regularization=1, - symmetric_l1_regularization=0, - num_table_shards=num_shards, - loss_type='logistic_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - with self.assertRaisesOpError( - 'Only labels of 0.0 or 1.0 are supported right now.'): - lr.minimize().run() - - def testImbalanced(self): - # Setup test data with 1 positive, and 3 negative examples. - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [2], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [3], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 1), - ] - example_weights = [1.0, 1.0, 1.0, 1.0] - for num_shards in _SHARD_NUMBERS: - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(3, 1, num_shards) - options = dict( - symmetric_l2_regularization=1, - symmetric_l1_regularization=0, - num_table_shards=num_shards, - loss_type='logistic_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - unregularized_loss = lr.unregularized_loss(examples) - loss = lr.regularized_loss(examples) - predictions = lr.predictions(examples) - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - - self.assertAllClose( - 0.226487 + 0.102902, unregularized_loss.eval(), atol=0.08) - self.assertAllClose(0.328394 + 0.131364, loss.eval(), atol=0.01) - predicted_labels = get_binary_predictions_for_logistic(predictions) - self.assertAllEqual([0, 0, 0, 1], predicted_labels.eval()) - self.assertAllClose( - 0.0, lr.approximate_duality_gap().eval(), rtol=2e-2, atol=1e-2) - - def testImbalancedWithExampleWeights(self): - # Setup test data with 1 positive, and 1 negative example. - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 1), - ] - example_weights = [3.0, 1.0] - for num_shards in _SHARD_NUMBERS: - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1, num_shards) - options = dict( - symmetric_l2_regularization=1, - symmetric_l1_regularization=0, - num_table_shards=num_shards, - loss_type='logistic_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - unregularized_loss = lr.unregularized_loss(examples) - loss = lr.regularized_loss(examples) - predictions = lr.predictions(examples) - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - - self.assertAllClose(0.284860, unregularized_loss.eval(), atol=0.08) - self.assertAllClose(0.408044, loss.eval(), atol=0.012) - predicted_labels = get_binary_predictions_for_logistic(predictions) - self.assertAllEqual([0, 1], predicted_labels.eval()) - self.assertAllClose( - 0.0, lr.approximate_duality_gap().eval(), rtol=2e-2, atol=1e-2) - - def testInstancesOfOneClassOnly(self): - # Setup test data with 1 positive (ignored), and 1 negative example. - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [1], - 'gender': [0] - }, 1), # Shares gender with the instance above. - ] - example_weights = [1.0, 0.0] # Second example "omitted" from training. - for num_shards in _SHARD_NUMBERS: - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1, num_shards) - options = dict( - symmetric_l2_regularization=1, - symmetric_l1_regularization=0, - num_table_shards=num_shards, - loss_type='logistic_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - unregularized_loss = lr.unregularized_loss(examples) - loss = lr.regularized_loss(examples) - predictions = lr.predictions(examples) - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - self.assertAllClose(0.411608, unregularized_loss.eval(), atol=0.05) - self.assertAllClose(0.525457, loss.eval(), atol=0.01) - predicted_labels = get_binary_predictions_for_logistic(predictions) - self.assertAllEqual([0, 0], predicted_labels.eval()) - self.assertAllClose( - 0.01, lr.approximate_duality_gap().eval(), rtol=1e-2, atol=1e-2) - - def testOutOfRangeSparseFeatures(self): - # Setup test data - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 1), - ] - example_weights = [1.0, 1.0] - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(0, 0) - options = dict( - symmetric_l2_regularization=1, - symmetric_l1_regularization=0, - loss_type='logistic_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - train_op = lr.minimize() - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - 'indices.*'): - train_op.run() - - def testOutOfRangeDenseFeatures(self): - with self._single_threaded_test_session(): - examples, variables = make_dense_examples_and_variables_dicts( - dense_features_values=[[[1.0, 0.0], [0.0, 1.0]]], - weights=[20.0, 10.0], - labels=[1.0, 0.0]) - # Replace with a variable of size 1 instead of 2. - variables['dense_features_weights'] = [ - variables_lib.VariableV1(array_ops.zeros( - [1], dtype=dtypes.float32)) - ] - options = dict( - symmetric_l2_regularization=1.0, - symmetric_l1_regularization=0, - loss_type='logistic_loss') - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - train_op = lr.minimize() - with self.assertRaisesRegexp( - errors_impl.InvalidArgumentError, - 'More dense features than we have parameters for.*'): - train_op.run() - - # TODO(katsiaspis): add a test for the case when examples at the end of an - # epoch are repeated, since example id may be duplicated. - - -class SdcaWithLinearLossTest(SdcaModelTest): - """SDCA optimizer test class for linear (squared) loss.""" - - def testSimple(self): - # Setup test data - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, -10.0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 14.0), - ] - example_weights = [1.0, 1.0] - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1) - options = dict( - symmetric_l2_regularization=1, - symmetric_l1_regularization=0, - loss_type='squared_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - predictions = lr.predictions(examples) - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - - # Predictions should be 2/3 of label due to minimizing regularized loss: - # (label - 2 * weight)^2 / 2 + L2 * 2 * weight^2 - self.assertAllClose( - [-20.0 / 3.0, 28.0 / 3.0], predictions.eval(), rtol=0.005) - # Approximate gap should be very close to 0.0. (In fact, because the gap - # is only approximate, it is likely that upon convergence the duality gap - # can have a tiny negative value). - self.assertAllClose(0.0, lr.approximate_duality_gap().eval(), atol=1e-2) - - def testL2Regularization(self): - # Setup test data - example_protos = [ - # 2 identical examples - make_example_proto({ - 'age': [0], - 'gender': [0] - }, -10.0), - make_example_proto({ - 'age': [0], - 'gender': [0] - }, -10.0), - # 2 more identical examples - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 14.0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 14.0), - ] - example_weights = [1.0, 1.0, 1.0, 1.0] - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1) - options = dict( - symmetric_l2_regularization=16, - symmetric_l1_regularization=0, - loss_type='squared_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - predictions = lr.predictions(examples) - - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - - # Predictions should be 1/5 of label due to minimizing regularized loss: - # (label - 2 * weight)^2 + L2 * 16 * weight^2 - optimal1 = -10.0 / 5.0 - optimal2 = 14.0 / 5.0 - self.assertAllClose( - [optimal1, optimal1, optimal2, optimal2], - predictions.eval(), - rtol=0.01) - - def testL1Regularization(self): - # Setup test data - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, -10.0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 14.0), - ] - example_weights = [1.0, 1.0] - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1) - options = dict( - symmetric_l2_regularization=1.0, - symmetric_l1_regularization=4.0, - loss_type='squared_loss') - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - prediction = lr.predictions(examples) - loss = lr.regularized_loss(examples) - - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - - # Predictions should be -4.0, 48/5 due to minimizing regularized loss: - # (label - 2 * weight)^2 / 2 + L2 * 2 * weight^2 + L1 * 4 * weight - self.assertAllClose([-4.0, 20.0 / 3.0], prediction.eval(), rtol=0.08) - - # Loss should be the sum of the regularized loss value from above per - # example after plugging in the optimal weights. - self.assertAllClose(308.0 / 6.0, loss.eval(), atol=0.01) - - def testFeatureValues(self): - # Setup test data - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, -10.0, -2.0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 14.0, 2.0), - ] - example_weights = [5.0, 3.0] - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - - variables = make_variable_dict(1, 1) - options = dict( - symmetric_l2_regularization=1, - symmetric_l1_regularization=0, - loss_type='squared_loss') - - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - predictions = lr.predictions(examples) - - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - - # There are 4 (sparse) variable weights to be learned. 2 for age and 2 for - # gender. Let w_1, w_2 be age weights, w_3, w_4 be gender weights, y_1, - # y_2 be the labels for examples 1 and 2 respectively and s_1, s_2 the - # corresponding *example* weights. With the given feature values, the loss - # function is given by: - # s_1/2(y_1 + 2w_1 + 2w_3)^2 + s_2/2(y_2 - 2w_2 - 2w_4)^2 - # + \lambda/2 (w_1^2 + w_2^2 + w_3^2 + w_4^2). Solving for the optimal, it - # can be verified that: - # w_1* = w_3* = -2.0 s_1 y_1/(\lambda + 8 s_1) and - # w_2* = w_4* = 2 \cdot s_2 y_2/(\lambda + 8 s_2). Equivalently, due to - # regularization and example weights, the predictions are within: - # 8 \cdot s_i /(\lambda + 8 \cdot s_i) of the labels. - self.assertAllClose( - [-10 * 40.0 / 41.0, 14.0 * 24 / 25.0], predictions.eval(), atol=0.01) - - def testDenseFeaturesWithDefaultWeights(self): - with self._single_threaded_test_session(): - examples, variables = make_dense_examples_and_variables_dicts( - dense_features_values=[[[1.0], [0.0]], [0.0, 1.0]], - weights=[1.0, 1.0], - labels=[10.0, -5.0]) - options = dict( - symmetric_l2_regularization=1.0, - symmetric_l1_regularization=0, - loss_type='squared_loss') - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - predictions = lr.predictions(examples) - - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - - # The loss function for these particular features is given by: - # 1/2(label_1-w_1)^2 + 1/2(label_2-w_2)^2 + \lambda/2 (w_1^2 + w_2^2). So, - # differentiating wrt to w_1, w_2 yields the following optimal values: - # w_1* = label_1/(\lambda + 1)= 10/2, w_2* =label_2/(\lambda + 1)= -5/2. - # In this case the (unnormalized regularized) loss will be: - # 1/2(10-5)^2 + 1/2(5-5/2)^2 + 1/2(5^2 + (5/2)^2) = 125.0/4. The actual - # loss should be further normalized by the sum of example weights. - self.assertAllClose([5.0, -2.5], predictions.eval(), rtol=0.01) - loss = lr.regularized_loss(examples) - self.assertAllClose(125.0 / 8.0, loss.eval(), atol=0.01) - - def testDenseFeaturesWithArbitraryWeights(self): - with self._single_threaded_test_session(): - examples, variables = make_dense_examples_and_variables_dicts( - dense_features_values=[[[1.0, 0.0], [0.0, 1.0]]], - weights=[20.0, 10.0], - labels=[10.0, -5.0]) - options = dict( - symmetric_l2_regularization=5.0, - symmetric_l1_regularization=0, - loss_type='squared_loss') - lr = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - predictions = lr.predictions(examples) - - train_op = lr.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - lr.update_weights(train_op).run() - - # The loss function for these particular features is given by: - # 1/2 s_1 (label_1-w_1)^2 + 1/2 s_2(label_2-w_2)^2 + - # \lambda/2 (w_1^2 + w_2^2) where s_1, s_2 are the *example weights. It - # turns out that the optimal (variable) weights are given by: - # w_1* = label_1 \cdot s_1/(\lambda + s_1)= 8.0 and - # w_2* =label_2 \cdot s_2/(\lambda + s_2)= -10/3. - # In this case the (unnormalized regularized) loss will be: - # s_1/2(8-10)^2 + s_2/2(5-10/3)^2 + 5.0/2(8^2 + (10/3)^2) = 2175.0/9. The - # actual loss should be further normalized by the sum of example weights. - self.assertAllClose([8.0, -10.0 / 3], predictions.eval(), rtol=0.01) - loss = lr.regularized_loss(examples) - self.assertAllClose(2175.0 / 270.0, loss.eval(), atol=0.01) - - -class SdcaWithHingeLossTest(SdcaModelTest): - """SDCA optimizer test class for hinge loss.""" - - def testSimple(self): - # Setup test data - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 1), - ] - example_weights = [1.0, 1.0] - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1) - options = dict( - symmetric_l2_regularization=1.0, - symmetric_l1_regularization=0, - loss_type='hinge_loss') - model = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - - # Before minimization, the weights default to zero. There is no loss due - # to regularization, only unregularized loss which is 0.5 * (1+1) = 1.0. - predictions = model.predictions(examples) - self.assertAllClose([0.0, 0.0], predictions.eval()) - unregularized_loss = model.unregularized_loss(examples) - regularized_loss = model.regularized_loss(examples) - self.assertAllClose(1.0, unregularized_loss.eval()) - self.assertAllClose(1.0, regularized_loss.eval()) - - # After minimization, the model separates perfectly the data points. There - # are 4 sparse weights: 2 for age (say w1, w2) and 2 for gender (say w3 - # and w4). Solving the system w1 + w3 = 1.0, w2 + w4 = -1.0 and minimizing - # wrt to \|\vec{w}\|_2, gives w1=w3=1/2 and w2=w4=-1/2. This gives 0.0 - # unregularized loss and 0.25 L2 loss. - train_op = model.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - model.update_weights(train_op).run() - - binary_predictions = get_binary_predictions_for_hinge(predictions) - self.assertAllEqual([-1.0, 1.0], predictions.eval()) - self.assertAllEqual([0, 1], binary_predictions.eval()) - self.assertAllClose(0.0, unregularized_loss.eval()) - self.assertAllClose(0.25, regularized_loss.eval(), atol=0.05) - - def testDenseFeaturesPerfectlySeparable(self): - with self._single_threaded_test_session(): - examples, variables = make_dense_examples_and_variables_dicts( - dense_features_values=[[1.0, 1.0], [1.0, -1.0]], - weights=[1.0, 1.0], - labels=[1.0, 0.0]) - options = dict( - symmetric_l2_regularization=1.0, - symmetric_l1_regularization=0, - loss_type='hinge_loss') - model = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - predictions = model.predictions(examples) - binary_predictions = get_binary_predictions_for_hinge(predictions) - - train_op = model.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - model.update_weights(train_op).run() - - self.assertAllClose([1.0, -1.0], predictions.eval(), atol=0.05) - self.assertAllEqual([1, 0], binary_predictions.eval()) - - # (1.0, 1.0) and (1.0, -1.0) are perfectly separable by x-axis (that is, - # the SVM's functional margin >=1), so the unregularized loss is ~0.0. - # There is only loss due to l2-regularization. For these datapoints, it - # turns out that w_1~=0.0 and w_2~=1.0 which means that l2 loss is ~0.25. - unregularized_loss = model.unregularized_loss(examples) - regularized_loss = model.regularized_loss(examples) - self.assertAllClose(0.0, unregularized_loss.eval(), atol=0.02) - self.assertAllClose(0.25, regularized_loss.eval(), atol=0.02) - - def testDenseFeaturesSeparableWithinMargins(self): - with self._single_threaded_test_session(): - examples, variables = make_dense_examples_and_variables_dicts( - dense_features_values=[[[1.0, 0.5], [1.0, -0.5]]], - weights=[1.0, 1.0], - labels=[1.0, 0.0]) - options = dict( - symmetric_l2_regularization=1.0, - symmetric_l1_regularization=0, - loss_type='hinge_loss') - model = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - predictions = model.predictions(examples) - binary_predictions = get_binary_predictions_for_hinge(predictions) - - train_op = model.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - model.update_weights(train_op).run() - - # (1.0, 0.5) and (1.0, -0.5) are separable by x-axis but the datapoints - # are within the margins so there is unregularized loss (1/2 per example). - # For these datapoints, optimal weights are w_1~=0.0 and w_2~=1.0 which - # gives an L2 loss of ~0.25. - self.assertAllClose([0.5, -0.5], predictions.eval(), rtol=0.05) - self.assertAllEqual([1, 0], binary_predictions.eval()) - unregularized_loss = model.unregularized_loss(examples) - regularized_loss = model.regularized_loss(examples) - self.assertAllClose(0.5, unregularized_loss.eval(), atol=0.02) - self.assertAllClose(0.75, regularized_loss.eval(), atol=0.02) - - def testDenseFeaturesWeightedExamples(self): - with self._single_threaded_test_session(): - examples, variables = make_dense_examples_and_variables_dicts( - dense_features_values=[[[1.0], [1.0]], [[0.5], [-0.5]]], - weights=[3.0, 1.0], - labels=[1.0, 0.0]) - options = dict( - symmetric_l2_regularization=1.0, - symmetric_l1_regularization=0, - loss_type='hinge_loss') - model = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - predictions = model.predictions(examples) - binary_predictions = get_binary_predictions_for_hinge(predictions) - train_op = model.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - model.update_weights(train_op).run() - - # Point (1.0, 0.5) has higher weight than (1.0, -0.5) so the model will - # try to increase the margin from (1.0, 0.5). Due to regularization, - # (1.0, -0.5) will be within the margin. For these points and example - # weights, the optimal weights are w_1~=0.4 and w_2~=1.2 which give an L2 - # loss of 0.5 * 0.25 * 0.25 * 1.6 = 0.2. The binary predictions will be - # correct, but the boundary will be much closer to the 2nd point than the - # first one. - self.assertAllClose([1.0, -0.2], predictions.eval(), atol=0.05) - self.assertAllEqual([1, 0], binary_predictions.eval()) - unregularized_loss = model.unregularized_loss(examples) - regularized_loss = model.regularized_loss(examples) - self.assertAllClose(0.2, unregularized_loss.eval(), atol=0.02) - self.assertAllClose(0.4, regularized_loss.eval(), atol=0.02) - - -class SdcaWithSmoothHingeLossTest(SdcaModelTest): - """SDCA optimizer test class for smooth hinge loss.""" - - def testSimple(self): - # Setup test data - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 1), - ] - example_weights = [1.0, 1.0] - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1) - options = dict( - symmetric_l2_regularization=1.0, - symmetric_l1_regularization=0, - loss_type='smooth_hinge_loss') - model = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - - # Before minimization, the weights default to zero. There is no loss due - # to regularization, only unregularized loss which is 0.5 * (1+1) = 1.0. - predictions = model.predictions(examples) - self.assertAllClose([0.0, 0.0], predictions.eval()) - unregularized_loss = model.unregularized_loss(examples) - regularized_loss = model.regularized_loss(examples) - self.assertAllClose(1.0, unregularized_loss.eval()) - self.assertAllClose(1.0, regularized_loss.eval()) - - # After minimization, the model separates perfectly the data points. There - # are 4 sparse weights: 2 for age (say w1, w2) and 2 for gender (say w3 - # and w4). The minimization leads to w1=w3=1/3 and w2=w4=-1/3. This gives - # an unregularized hinge loss of 0.33 and a 0.11 L2 loss - train_op = model.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - model.update_weights(train_op).run() - - binary_predictions = get_binary_predictions_for_hinge(predictions) - self.assertAllClose([-0.67, 0.67], predictions.eval(), atol=0.05) - self.assertAllEqual([0, 1], binary_predictions.eval()) - self.assertAllClose(0.33, unregularized_loss.eval(), atol=0.02) - self.assertAllClose(0.44, regularized_loss.eval(), atol=0.02) - -class SdcaWithPoissonLossTest(SdcaModelTest): - """SDCA optimizer test class for poisson loss.""" - - def testSimple(self): - # Setup test data - example_protos = [ - make_example_proto({ - 'age': [0], - 'gender': [0] - }, 0), - make_example_proto({ - 'age': [1], - 'gender': [1] - }, 2), - ] - example_weights = [100.0, 100.0] - with self._single_threaded_test_session(): - examples = make_example_dict(example_protos, example_weights) - variables = make_variable_dict(1, 1) - options = dict( - symmetric_l2_regularization=1.0, - symmetric_l1_regularization=0, - loss_type='poisson_loss') - model = SdcaModel(examples, variables, options) - variables_lib.global_variables_initializer().run() - - # Before minimization, the weights default to zero. There is no loss due - # to regularization, only unregularized loss which is 1 for each example. - predictions = model.predictions(examples) - self.assertAllClose([1.0, 1.0], predictions.eval()) - unregularized_loss = model.unregularized_loss(examples) - regularized_loss = model.regularized_loss(examples) - approximate_duality_gap = model.approximate_duality_gap() - self.assertAllClose(1.0, unregularized_loss.eval()) - self.assertAllClose(1.0, regularized_loss.eval()) - - # There are 4 sparse weights: 2 for age (say w1, w2) and 2 for gender - # (say w3 and w4). The minimization leads to: - # w1=w3=-1.96487, argmin of 100*(exp(2*w)-2*w*0)+w**2. - # w2=w4=0.345708, argmin of 100*(exp(2*w)-2*w*2)+w**2. - # This gives an unregularized loss of .3167 and .3366 with regularization. - train_op = model.minimize() - for _ in range(_MAX_ITERATIONS): - train_op.run() - model.update_weights(train_op).run() - - self.assertAllClose([0.0196, 1.9965], predictions.eval(), atol=1e-4) - self.assertAllClose(0.3167, unregularized_loss.eval(), atol=1e-4) - self.assertAllClose(0.3366, regularized_loss.eval(), atol=1e-4) - self.assertAllClose(0., approximate_duality_gap.eval(), atol=1e-6) - - -class SdcaFprintTest(SdcaModelTest): - """Tests for the SdcaFprint op. - - This is one way of enforcing the platform-agnostic nature of SdcaFprint. - Basically we are checking against exact values and this test could be running - across different platforms. Note that it is fine for expected values to change - in the future, if the implementation of SdcaFprint changes (ie this is *not* a - frozen test). - """ - - def testFprint(self): - with self._single_threaded_test_session(): - in_data = constant_op.constant(['abc', 'very looooooong string', 'def']) - out_data = gen_sdca_ops.sdca_fprint(in_data) - self.assertAllEqual([[4143508125394299908, -6879828354153669051], - [5849691694103072671, -4874542629849009556], - [603227410218889250, 8762207001949257490]], - out_data.eval()) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/linear_optimizer/python/ops/sdca_ops.py b/tensorflow/contrib/linear_optimizer/python/ops/sdca_ops.py deleted file mode 100644 index 950840c6b77..00000000000 --- a/tensorflow/contrib/linear_optimizer/python/ops/sdca_ops.py +++ /dev/null @@ -1,724 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# pylint: disable=line-too-long -"""Proximal stochastic dual coordinate ascent optimizer for linear models (deprecated). - -This module and all its submodules are deprecated. To UPDATE or USE linear -optimizers, please check its latest version in core: -tensorflow_estimator/python/estimator/canned/linear_optimizer/. -""" -# pylint: enable=line-too-long -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from six.moves import range - -from tensorflow.contrib.linear_optimizer.python.ops.sharded_mutable_dense_hashtable import ShardedMutableDenseHashTable -from tensorflow.python.compat import compat -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework.ops import internal_convert_to_tensor -from tensorflow.python.framework.ops import name_scope -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import gen_sdca_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variables as var_ops -from tensorflow.python.ops.nn import log_poisson_loss -from tensorflow.python.ops.nn import sigmoid_cross_entropy_with_logits -from tensorflow.python.summary import summary -from tensorflow.python.util import deprecation - -__all__ = ['SdcaModel'] - - -# TODO(sibyl-Aix6ihai): add name_scope to appropriate methods. -class SdcaModel(object): - """Stochastic dual coordinate ascent solver for linear models. - - Loss functions supported: - - * Binary logistic loss - * Squared loss - * Hinge loss - * Smooth hinge loss - * Poisson log loss - - This class defines an optimizer API to train a linear model. - - ### Usage - - ```python - # Create a solver with the desired parameters. - lr = tf.contrib.linear_optimizer.SdcaModel(examples, variables, options) - min_op = lr.minimize() - opt_op = lr.update_weights(min_op) - - predictions = lr.predictions(examples) - # Primal loss + L1 loss + L2 loss. - regularized_loss = lr.regularized_loss(examples) - # Primal loss only - unregularized_loss = lr.unregularized_loss(examples) - - examples: { - sparse_features: list of SparseFeatureColumn. - dense_features: list of dense tensors of type float32. - example_labels: a tensor of type float32 and shape [Num examples] - example_weights: a tensor of type float32 and shape [Num examples] - example_ids: a tensor of type string and shape [Num examples] - } - variables: { - sparse_features_weights: list of tensors of shape [vocab size] - dense_features_weights: list of tensors of shape [dense_feature_dimension] - } - options: { - symmetric_l1_regularization: 0.0 - symmetric_l2_regularization: 1.0 - loss_type: "logistic_loss" - num_loss_partitions: 1 (Optional, with default value of 1. Number of - partitions of the global loss function, 1 means single machine solver, - and >1 when we have more than one optimizer working concurrently.) - num_table_shards: 1 (Optional, with default value of 1. Number of shards - of the internal state table, typically set to match the number of - parameter servers for large data sets. - } - ``` - - In the training program you will just have to run the returned Op from - minimize(). - - ```python - # Execute opt_op and train for num_steps. - for _ in range(num_steps): - opt_op.run() - - # You can also check for convergence by calling - lr.approximate_duality_gap() - ``` - """ - - @deprecation.deprecated( - None, 'This class is deprecated. To UPDATE or USE linear optimizers, ' - 'please check its latest version in core: ' - 'tensorflow_estimator/python/estimator/canned/linear_optimizer/.') - def __init__(self, examples, variables, options): - """Create a new sdca optimizer.""" - - if not examples or not variables or not options: - raise ValueError('examples, variables and options must all be specified.') - - supported_losses = ('logistic_loss', 'squared_loss', 'hinge_loss', - 'smooth_hinge_loss', 'poisson_loss') - if options['loss_type'] not in supported_losses: - raise ValueError('Unsupported loss_type: ', options['loss_type']) - - self._assertSpecified([ - 'example_labels', 'example_weights', 'example_ids', 'sparse_features', - 'dense_features' - ], examples) - self._assertList(['sparse_features', 'dense_features'], examples) - - self._assertSpecified(['sparse_features_weights', 'dense_features_weights'], - variables) - self._assertList(['sparse_features_weights', 'dense_features_weights'], - variables) - - self._assertSpecified([ - 'loss_type', 'symmetric_l2_regularization', - 'symmetric_l1_regularization' - ], options) - - for name in ['symmetric_l1_regularization', 'symmetric_l2_regularization']: - value = options[name] - if value < 0.0: - raise ValueError('%s should be non-negative. Found (%f)' % - (name, value)) - - self._examples = examples - self._variables = variables - self._options = options - self._create_slots() - self._hashtable = ShardedMutableDenseHashTable( - key_dtype=dtypes.int64, - value_dtype=dtypes.float32, - num_shards=self._num_table_shards(), - default_value=[0.0, 0.0, 0.0, 0.0], - # SdcaFprint never returns 0 or 1 for the low64 bits, so this a safe - # empty_key (that will never collide with actual payloads). - empty_key=[0, 0], - deleted_key=[1, 1]) - - summary.scalar('approximate_duality_gap', self.approximate_duality_gap()) - summary.scalar('examples_seen', self._hashtable.size()) - - def _symmetric_l1_regularization(self): - return self._options['symmetric_l1_regularization'] - - def _symmetric_l2_regularization(self): - # Algorithmic requirement (for now) is to have minimal l2 of 1.0. - return max(self._options['symmetric_l2_regularization'], 1.0) - - def _num_loss_partitions(self): - # Number of partitions of the global objective. - # TODO(andreasst): set num_loss_partitions automatically based on the number - # of workers - return self._options.get('num_loss_partitions', 1) - - def _adaptive(self): - # Perform adaptive sampling. - return self._options.get('adaptive', True) - - def _num_table_shards(self): - # Number of hash table shards. - # Return 1 if not specified or if the value is 'None' - # TODO(andreasst): set num_table_shards automatically based on the number - # of parameter servers - num_shards = self._options.get('num_table_shards') - return 1 if num_shards is None else num_shards - - # TODO(sibyl-Aix6ihai): Use optimizer interface to make use of slot creation logic. - def _create_slots(self): - """Make unshrinked internal variables (slots).""" - # Unshrinked variables have the updates before applying L1 regularization. - # Each unshrinked slot variable is either a `Variable` or list of - # `Variable`, depending on the value of its corresponding primary variable. - # We avoid using `PartitionedVariable` for the unshrinked slots since we do - # not need any of the extra information. - self._slots = collections.defaultdict(list) - for name in ['sparse_features_weights', 'dense_features_weights']: - for var in self._variables[name]: - # Our primary variable may be either a PartitionedVariable, or a list - # of Variables (each representing a partition). - if (isinstance(var, var_ops.PartitionedVariable) or - isinstance(var, list)): - var_list = [] - # pylint: disable=protected-access - for v in var: - with ops.colocate_with(v): - # TODO(andreasst): remove SDCAOptimizer suffix once bug 30843109 - # is fixed. - slot_var = var_ops.VariableV1( - initial_value=array_ops.zeros_like(v.initialized_value(), - dtypes.float32), - name=v.op.name + '_unshrinked/SDCAOptimizer') - var_list.append(slot_var) - self._slots['unshrinked_' + name].append(var_list) - # pylint: enable=protected-access - else: - with ops.device(var.device): - # TODO(andreasst): remove SDCAOptimizer suffix once bug 30843109 is - # fixed. - self._slots['unshrinked_' + name].append( - var_ops.VariableV1( - array_ops.zeros_like(var.initialized_value(), - dtypes.float32), - name=var.op.name + '_unshrinked/SDCAOptimizer')) - - def _assertSpecified(self, items, check_in): - for x in items: - if check_in[x] is None: - raise ValueError(check_in[x] + ' must be specified.') - - def _assertList(self, items, check_in): - for x in items: - if not isinstance(check_in[x], list): - raise ValueError(x + ' must be a list.') - - def _var_to_list(self, var): - """Wraps var in a list if it is not a list or PartitionedVariable.""" - if not (isinstance(var, list) or - isinstance(var, var_ops.PartitionedVariable)): - var = [var] - return var - - def _l1_loss(self): - """Computes the (un-normalized) l1 loss of the model.""" - with name_scope('sdca/l1_loss'): - sums = [] - for name in ['sparse_features_weights', 'dense_features_weights']: - for var in self._variables[name]: - for v in self._var_to_list(var): - weights = internal_convert_to_tensor(v) - with ops.device(weights.device): - sums.append( - math_ops.reduce_sum( - math_ops.abs(math_ops.cast(weights, dtypes.float64)))) - # SDCA L1 regularization cost is: l1 * sum(|weights|) - return self._options['symmetric_l1_regularization'] * math_ops.add_n(sums) - - def _l2_loss(self, l2): - """Computes the (un-normalized) l2 loss of the model.""" - with name_scope('sdca/l2_loss'): - sums = [] - for name in ['sparse_features_weights', 'dense_features_weights']: - for var in self._variables[name]: - for v in self._var_to_list(var): - weights = internal_convert_to_tensor(v) - with ops.device(weights.device): - sums.append(math_ops.reduce_sum(math_ops.square(math_ops.cast( - weights, dtypes.float64)))) - # SDCA L2 regularization cost is: l2 * sum(weights^2) / 2 - return l2 * math_ops.add_n(sums) / 2.0 - - def _convert_n_to_tensor(self, input_list, as_ref=False): - """Converts input list to a set of tensors.""" - # input_list can be a list of Variables (that are implicitly partitioned), - # in which case the underlying logic in internal_convert_to_tensor will not - # concatenate the partitions together. This method takes care of the - # concatenating (we only allow partitioning on the first axis). - output_list = [] - for x in input_list: - tensor_to_convert = x - if isinstance(x, list) or isinstance(x, var_ops.PartitionedVariable): - # We only allow for partitioning on the first axis. - tensor_to_convert = array_ops.concat(x, axis=0) - output_list.append(internal_convert_to_tensor( - tensor_to_convert, as_ref=as_ref)) - return output_list - - def _get_first_dimension_size_statically(self, w, num_partitions): - """Compute the static size of the first dimension for a sharded variable.""" - dim_0_size = w[0].get_shape()[0] - for p in range(1, num_partitions): - dim_0_size += w[p].get_shape()[0] - return dim_0_size - - def _linear_predictions(self, examples): - """Returns predictions of the form w*x.""" - with name_scope('sdca/prediction'): - sparse_variables = self._convert_n_to_tensor(self._variables[ - 'sparse_features_weights']) - result_sparse = 0.0 - for sfc, sv in zip(examples['sparse_features'], sparse_variables): - # TODO(sibyl-Aix6ihai): following does not take care of missing features. - result_sparse += math_ops.segment_sum( - math_ops.multiply( - array_ops.gather(sv, sfc.feature_indices), sfc.feature_values), - sfc.example_indices) - dense_features = self._convert_n_to_tensor(examples['dense_features']) - dense_variables = self._convert_n_to_tensor(self._variables[ - 'dense_features_weights']) - - result_dense = 0.0 - for i in range(len(dense_variables)): - result_dense += math_ops.matmul(dense_features[i], - array_ops.expand_dims( - dense_variables[i], -1)) - - # Reshaping to allow shape inference at graph construction time. - return array_ops.reshape(result_dense, [-1]) + result_sparse - - def predictions(self, examples): - """Add operations to compute predictions by the model. - - If logistic_loss is being used, predicted probabilities are returned. - If poisson_loss is being used, predictions are exponentiated. - Otherwise, (raw) linear predictions (w*x) are returned. - - Args: - examples: Examples to compute predictions on. - - Returns: - An Operation that computes the predictions for examples. - - Raises: - ValueError: if examples are not well defined. - """ - self._assertSpecified( - ['example_weights', 'sparse_features', 'dense_features'], examples) - self._assertList(['sparse_features', 'dense_features'], examples) - - result = self._linear_predictions(examples) - if self._options['loss_type'] == 'logistic_loss': - # Convert logits to probability for logistic loss predictions. - with name_scope('sdca/logistic_prediction'): - result = math_ops.sigmoid(result) - elif self._options['loss_type'] == 'poisson_loss': - # Exponeniate the prediction for poisson loss predictions. - with name_scope('sdca/poisson_prediction'): - result = math_ops.exp(result) - return result - - def _get_partitioned_update_ops(self, - v_num, - num_partitions_by_var, - p_assignments_by_var, - gather_ids_by_var, - weights, - full_update, - p_assignments, - num_partitions): - """Get updates for partitioned variables.""" - num_partitions = num_partitions_by_var[v_num] - p_assignments = p_assignments_by_var[v_num] - gather_ids = gather_ids_by_var[v_num] - updates = data_flow_ops.dynamic_partition( - full_update, p_assignments, num_partitions) - update_ops = [] - for p in range(num_partitions): - with ops.colocate_with(weights[p]): - result = state_ops.scatter_add(weights[p], gather_ids[p], updates[p]) - update_ops.append(result) - return update_ops - - def minimize(self, global_step=None, name=None): - """Add operations to train a linear model by minimizing the loss function. - - Args: - global_step: Optional `Variable` to increment by one after the - variables have been updated. - name: Optional name for the returned operation. - - Returns: - An Operation that updates the variables passed in the constructor. - """ - # Technically, the op depends on a lot more than the variables, - # but we'll keep the list short. - with name_scope(name, 'sdca/minimize'): - sparse_example_indices = [] - sparse_feature_indices = [] - sparse_features_values = [] - for sf in self._examples['sparse_features']: - sparse_example_indices.append(sf.example_indices) - sparse_feature_indices.append(sf.feature_indices) - # If feature values are missing, sdca assumes a value of 1.0f. - if sf.feature_values is not None: - sparse_features_values.append(sf.feature_values) - - # pylint: disable=protected-access - example_ids_hashed = gen_sdca_ops.sdca_fprint( - internal_convert_to_tensor(self._examples['example_ids'])) - # pylint: enable=protected-access - example_state_data = self._hashtable.lookup(example_ids_hashed) - # Solver returns example_state_update, new delta sparse_feature_weights - # and delta dense_feature_weights. - - sparse_weights = [] - sparse_indices = [] - # If we have partitioned variables, keep a few dictionaries of Tensors - # around that we need for the assign_add after the op call to - # gen_sdca_ops.sdca_optimizer(). These are keyed because we may have a - # mix of partitioned and un-partitioned variables. - num_partitions_by_var = {} - p_assignments_by_var = {} - gather_ids_by_var = {} - for v_num, (w, i) in enumerate( - zip(self._slots['unshrinked_sparse_features_weights'], - sparse_feature_indices)): - # Append the sparse_indices (in full-variable space). - sparse_idx = math_ops.cast( - array_ops.unique(math_ops.cast(i, dtypes.int32))[0], - dtypes.int64) - sparse_indices.append(sparse_idx) - if isinstance(w, list) or isinstance(w, var_ops.PartitionedVariable): - num_partitions = len(w) - flat_ids = array_ops.reshape(sparse_idx, [-1]) - # We use div partitioning, which is easiest to support downstream. - # Compute num_total_ids as the sum of dim-0 of w, then assign - # to partitions based on a constant number of ids per partition. - # Optimize if we already know the full shape statically. - dim_0_size = self._get_first_dimension_size_statically( - w, num_partitions) - - if tensor_shape.dimension_value(dim_0_size): - num_total_ids = constant_op.constant( - tensor_shape.dimension_value(dim_0_size), - flat_ids.dtype) - else: - dim_0_sizes = [] - for p in range(num_partitions): - if tensor_shape.dimension_value(w[p].shape[0]) is not None: - dim_0_sizes.append(tensor_shape.dimension_value(w[p].shape[0])) - else: - with ops.colocate_with(w[p]): - dim_0_sizes.append(array_ops.shape(w[p])[0]) - num_total_ids = math_ops.reduce_sum( - math_ops.cast(array_ops.stack(dim_0_sizes), flat_ids.dtype)) - ids_per_partition = num_total_ids // num_partitions - extras = num_total_ids % num_partitions - - p_assignments = math_ops.maximum( - flat_ids // (ids_per_partition + 1), - (flat_ids - extras) // ids_per_partition) - - # Emulate a conditional using a boolean indicator tensor - new_ids = array_ops.where(p_assignments < extras, - flat_ids % (ids_per_partition + 1), - (flat_ids - extras) % ids_per_partition) - - # Cast partition assignments to int32 for use in dynamic_partition. - # There really should not be more than 2^32 partitions. - p_assignments = math_ops.cast(p_assignments, dtypes.int32) - # Partition list of ids based on assignments into num_partitions - # separate lists. - gather_ids = data_flow_ops.dynamic_partition(new_ids, - p_assignments, - num_partitions) - # Add these into the dictionaries for use in the later update. - num_partitions_by_var[v_num] = num_partitions - p_assignments_by_var[v_num] = p_assignments - gather_ids_by_var[v_num] = gather_ids - - # Gather the weights from each partition. - partition_gathered_weights = [] - for p in range(num_partitions): - with ops.colocate_with(w[p]): - partition_gathered_weights.append( - array_ops.gather(w[p], gather_ids[p])) - - # Stitch the weights back together in the same order they were before - # we dynamic_partitioned them. - condition_indices = data_flow_ops.dynamic_partition( - math_ops.range(array_ops.shape(new_ids)[0]), - p_assignments, num_partitions) - batch_gathered_weights = data_flow_ops.dynamic_stitch( - condition_indices, partition_gathered_weights) - else: - w_as_tensor = internal_convert_to_tensor(w) - with ops.device(w_as_tensor.device): - batch_gathered_weights = array_ops.gather( - w_as_tensor, sparse_idx) - sparse_weights.append(batch_gathered_weights) - - # pylint: disable=protected-access - if compat.forward_compatible(year=2018, month=10, day=30): - esu, sfw, dfw = gen_sdca_ops.sdca_optimizer_v2( - sparse_example_indices, - sparse_feature_indices, - sparse_features_values, - self._convert_n_to_tensor(self._examples['dense_features']), - internal_convert_to_tensor(self._examples['example_weights']), - internal_convert_to_tensor(self._examples['example_labels']), - sparse_indices, - sparse_weights, - self._convert_n_to_tensor(self._slots[ - 'unshrinked_dense_features_weights']), - example_state_data, - loss_type=self._options['loss_type'], - l1=self._options['symmetric_l1_regularization'], - l2=self._symmetric_l2_regularization(), - num_loss_partitions=self._num_loss_partitions(), - num_inner_iterations=1, - adaptive=self._adaptive()) - else: - esu, sfw, dfw = gen_sdca_ops.sdca_optimizer( - sparse_example_indices, - sparse_feature_indices, - sparse_features_values, - self._convert_n_to_tensor(self._examples['dense_features']), - internal_convert_to_tensor(self._examples['example_weights']), - internal_convert_to_tensor(self._examples['example_labels']), - sparse_indices, - sparse_weights, - self._convert_n_to_tensor(self._slots[ - 'unshrinked_dense_features_weights']), - example_state_data, - loss_type=self._options['loss_type'], - l1=self._options['symmetric_l1_regularization'], - l2=self._symmetric_l2_regularization(), - num_loss_partitions=self._num_loss_partitions(), - num_inner_iterations=1, - adaptative=self._adaptive()) - # pylint: enable=protected-access - - with ops.control_dependencies([esu]): - update_ops = [self._hashtable.insert(example_ids_hashed, esu)] - # Update the weights before the proximal step. - for v_num, (w, i, u) in enumerate( - zip(self._slots['unshrinked_sparse_features_weights'], - sparse_indices, sfw)): - if (isinstance(w, var_ops.PartitionedVariable) or - isinstance(w, list)): - update_ops += self._get_partitioned_update_ops( - v_num, num_partitions_by_var, p_assignments_by_var, - gather_ids_by_var, w, u, p_assignments, num_partitions) - else: - update_ops.append(state_ops.scatter_add(w, i, u)) - for w, u in zip(self._slots['unshrinked_dense_features_weights'], dfw): - if (isinstance(w, var_ops.PartitionedVariable) or - isinstance(w, list)): - split_updates = array_ops.split( - u, num_or_size_splits=[v.shape.as_list()[0] for v in w]) - for v, split_update in zip(w, split_updates): - update_ops.append(state_ops.assign_add(v, split_update)) - else: - update_ops.append(state_ops.assign_add(w, u)) - if not global_step: - return control_flow_ops.group(*update_ops) - with ops.control_dependencies(update_ops): - return state_ops.assign_add(global_step, 1, name=name).op - - def update_weights(self, train_op): - """Updates the model weights. - - This function must be called on at least one worker after `minimize`. - In distributed training this call can be omitted on non-chief workers to - speed up training. - - Args: - train_op: The operation returned by the `minimize` call. - - Returns: - An Operation that updates the model weights. - """ - with ops.control_dependencies([train_op]): - update_ops = [] - # Copy over unshrinked weights to user provided variables. - for name in ['sparse_features_weights', 'dense_features_weights']: - for var, slot_var in zip(self._variables[name], - self._slots['unshrinked_' + name]): - for v, sv in zip(self._var_to_list(var), self._var_to_list(slot_var)): - update_ops.append(v.assign(sv)) - - # Apply proximal step. - with ops.control_dependencies(update_ops): - update_ops = [] - for name in ['sparse_features_weights', 'dense_features_weights']: - for var in self._variables[name]: - for v in self._var_to_list(var): - with ops.device(v.device): - # pylint: disable=protected-access - update_ops.append( - gen_sdca_ops.sdca_shrink_l1( - self._convert_n_to_tensor([v], as_ref=True), - l1=self._symmetric_l1_regularization(), - l2=self._symmetric_l2_regularization())) - return control_flow_ops.group(*update_ops) - - def approximate_duality_gap(self): - """Add operations to compute the approximate duality gap. - - Returns: - An Operation that computes the approximate duality gap over all - examples. - """ - with name_scope('sdca/approximate_duality_gap'): - _, values_list = self._hashtable.export_sharded() - shard_sums = [] - for values in values_list: - with ops.device(values.device): - # For large tables to_double() below allocates a large temporary - # tensor that is freed once the sum operation completes. To reduce - # peak memory usage in cases where we have multiple large tables on a - # single device, we serialize these operations. - # Note that we need double precision to get accurate results. - with ops.control_dependencies(shard_sums): - shard_sums.append( - math_ops.reduce_sum(math_ops.cast(values, dtypes.float64), 0)) - summed_values = math_ops.add_n(shard_sums) - - primal_loss = summed_values[1] - dual_loss = summed_values[2] - example_weights = summed_values[3] - # Note: we return NaN if there are no weights or all weights are 0, e.g. - # if no examples have been processed - return (primal_loss + dual_loss + self._l1_loss() + - (2.0 * self._l2_loss(self._symmetric_l2_regularization())) - ) / example_weights - - def unregularized_loss(self, examples): - """Add operations to compute the loss (without the regularization loss). - - Args: - examples: Examples to compute unregularized loss on. - - Returns: - An Operation that computes mean (unregularized) loss for given set of - examples. - - Raises: - ValueError: if examples are not well defined. - """ - self._assertSpecified([ - 'example_labels', 'example_weights', 'sparse_features', 'dense_features' - ], examples) - self._assertList(['sparse_features', 'dense_features'], examples) - with name_scope('sdca/unregularized_loss'): - predictions = math_ops.cast( - self._linear_predictions(examples), dtypes.float64) - labels = math_ops.cast( - internal_convert_to_tensor(examples['example_labels']), - dtypes.float64) - weights = math_ops.cast( - internal_convert_to_tensor(examples['example_weights']), - dtypes.float64) - - if self._options['loss_type'] == 'logistic_loss': - return math_ops.reduce_sum(math_ops.multiply( - sigmoid_cross_entropy_with_logits(labels=labels, - logits=predictions), - weights)) / math_ops.reduce_sum(weights) - - if self._options['loss_type'] == 'poisson_loss': - return math_ops.reduce_sum(math_ops.multiply( - log_poisson_loss(targets=labels, log_input=predictions), - weights)) / math_ops.reduce_sum(weights) - - if self._options['loss_type'] in ['hinge_loss', 'smooth_hinge_loss']: - # hinge_loss = max{0, 1 - y_i w*x} where y_i \in {-1, 1}. So, we need to - # first convert 0/1 labels into -1/1 labels. - all_ones = array_ops.ones_like(predictions) - adjusted_labels = math_ops.subtract(2 * labels, all_ones) - # Tensor that contains (unweighted) error (hinge loss) per - # example. - error = nn_ops.relu( - math_ops.subtract(all_ones, - math_ops.multiply(adjusted_labels, predictions))) - weighted_error = math_ops.multiply(error, weights) - return math_ops.reduce_sum(weighted_error) / math_ops.reduce_sum( - weights) - - # squared loss - err = math_ops.subtract(labels, predictions) - - weighted_squared_err = math_ops.multiply(math_ops.square(err), weights) - # SDCA squared loss function is sum(err^2) / (2*sum(weights)) - return (math_ops.reduce_sum(weighted_squared_err) / - (2.0 * math_ops.reduce_sum(weights))) - - def regularized_loss(self, examples): - """Add operations to compute the loss with regularization loss included. - - Args: - examples: Examples to compute loss on. - - Returns: - An Operation that computes mean (regularized) loss for given set of - examples. - Raises: - ValueError: if examples are not well defined. - """ - self._assertSpecified([ - 'example_labels', 'example_weights', 'sparse_features', 'dense_features' - ], examples) - self._assertList(['sparse_features', 'dense_features'], examples) - with name_scope('sdca/regularized_loss'): - weights = internal_convert_to_tensor(examples['example_weights']) - return (( - self._l1_loss() + - # Note that here we are using the raw regularization - # (as specified by the user) and *not* - # self._symmetric_l2_regularization(). - self._l2_loss(self._options['symmetric_l2_regularization'])) / - math_ops.reduce_sum(math_ops.cast(weights, dtypes.float64)) + - self.unregularized_loss(examples)) diff --git a/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable.py b/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable.py deleted file mode 100644 index 8fda828e994..00000000000 --- a/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Sharded mutable dense hash table (deprecated). - -This module and all its submodules are deprecated. To UPDATE or USE linear -optimizers, please check its latest version in core: -tensorflow_estimator/python/estimator/canned/linear_optimizer/. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from six.moves import range - -from tensorflow.contrib import lookup -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.util import deprecation - - -# TODO(rohanj): This should subclass Trackable and implement -# _gather_saveables_for_checkpoint. -class ShardedMutableDenseHashTable(object): - """A sharded version of MutableDenseHashTable. - - It is designed to be interface compatible with LookupInterface and - MutableDenseHashTable, with the exception of the export method, which is - replaced by an export_sharded method. - - The _ShardedMutableDenseHashTable keeps `num_shards` MutableDenseHashTable - internally. The shard is computed via the modulo operation on the key. - """ - - # TODO(andreasst): consider moving this to lookup module - - @deprecation.deprecated( - None, 'This class is deprecated. To UPDATE or USE linear optimizers, ' - 'please check its latest version in core: ' - 'tensorflow_estimator/python/estimator/canned/linear_optimizer/.') - def __init__(self, - key_dtype, - value_dtype, - default_value, - empty_key, - deleted_key, - num_shards=1, - checkpoint=True, - name='ShardedMutableHashTable'): - self._key_dtype = key_dtype - self._value_dtype = value_dtype - with ops.name_scope(name, 'sharded_mutable_hash_table') as scope: - self._table_name = scope - table_shards = [] - for i in range(num_shards): - table_shards.append( - lookup.MutableDenseHashTable( - key_dtype=key_dtype, - value_dtype=value_dtype, - default_value=default_value, - empty_key=empty_key, - deleted_key=deleted_key, - checkpoint=checkpoint, - name='%s-%d-of-%d' % (name, i + 1, num_shards))) - self._table_shards = table_shards - # TODO(andreasst): add a value_shape() method to LookupInterface - # pylint: disable=protected-access - self._value_shape = self._table_shards[0]._value_shape - # pylint: enable=protected-access - - @property - def name(self): - return self._table_name - - @property - def _num_shards(self): - return len(self._table_shards) - - @property - def table_shards(self): - return self._table_shards - - def size(self, name=None): - with ops.name_scope(name, 'sharded_mutable_hash_table_size'): - sizes = [ - self._table_shards[i].size() for i in range(self._num_shards) - ] - return math_ops.add_n(sizes) - - def _shard_indices(self, keys): - key_shape = keys.get_shape() - if key_shape.ndims > 1: - # If keys are a matrix (i.e. a single key is a vector), we use the first - # element of each key vector to determine the shard. - keys = array_ops.slice(keys, [0, 0], [key_shape.dims[0].value, 1]) - keys = array_ops.reshape(keys, [-1]) - indices = math_ops.mod(math_ops.abs(keys), self._num_shards) - return math_ops.cast(indices, dtypes.int32) - - def _check_keys(self, keys): - if not keys.get_shape().is_fully_defined(): - raise ValueError('Key shape must be fully defined, got %s.' % - keys.get_shape()) - if keys.get_shape().ndims != 1 and keys.get_shape().ndims != 2: - raise ValueError('Expected a vector or matrix for keys, got %s.' % - keys.get_shape()) - - def lookup(self, keys, name=None): - """Looks up `keys` in a table, outputs the corresponding values.""" - if keys.dtype.base_dtype != self._key_dtype: - raise TypeError('Signature mismatch. Keys must be dtype %s, got %s.' % - (self._key_dtype, keys.dtype)) - self._check_keys(keys) - num_shards = self._num_shards - if num_shards == 1: - return self._table_shards[0].lookup(keys, name=name) - - shard_indices = self._shard_indices(keys) - # TODO(andreasst): support 'keys' that are not vectors - key_shards = data_flow_ops.dynamic_partition(keys, shard_indices, - num_shards) - value_shards = [ - self._table_shards[i].lookup(key_shards[i], name=name) - for i in range(num_shards) - ] - - num_keys = keys.get_shape().dims[0] - original_indices = math_ops.range(num_keys) - partitioned_indices = data_flow_ops.dynamic_partition(original_indices, - shard_indices, - num_shards) - result = data_flow_ops.dynamic_stitch(partitioned_indices, value_shards) - result.set_shape( - tensor_shape.TensorShape([num_keys]).concatenate(self._value_shape)) - return result - - def insert(self, keys, values, name=None): - """Inserts `keys` in a table.""" - self._check_keys(keys) - num_shards = self._num_shards - if num_shards == 1: - return self._table_shards[0].insert(keys, values, name=name) - - shard_indices = self._shard_indices(keys) - # TODO(andreasst): support 'keys' that are not vectors - key_shards = data_flow_ops.dynamic_partition(keys, shard_indices, - num_shards) - value_shards = data_flow_ops.dynamic_partition(values, shard_indices, - num_shards) - return_values = [ - self._table_shards[i].insert(key_shards[i], value_shards[i], name=name) - for i in range(num_shards) - ] - - return control_flow_ops.group(*return_values) - - def export_sharded(self, name=None): - """Returns lists of the keys and values tensors in the sharded table. - - Args: - name: name of the table. - - Returns: - A pair of lists with the first list containing the key tensors and the - second list containing the value tensors from each shard. - """ - keys_list = [] - values_list = [] - for table_shard in self._table_shards: - exported_keys, exported_values = table_shard.export(name=name) - keys_list.append(exported_keys) - values_list.append(exported_values) - return keys_list, values_list diff --git a/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable_test.py b/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable_test.py deleted file mode 100644 index 2d1457f9e4c..00000000000 --- a/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable_test.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for sharded_mutable_dense_hashtable.py (deprecated). - -This module and all its submodules are deprecated. To UPDATE or USE linear -optimizers, please check its latest version in core: -tensorflow_estimator/python/estimator/canned/linear_optimizer/. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.linear_optimizer.python.ops.sharded_mutable_dense_hashtable import ShardedMutableDenseHashTable -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework.test_util import TensorFlowTestCase -from tensorflow.python.platform import googletest - - -class ShardedMutableDenseHashTableTest(TensorFlowTestCase): - """Tests for the ShardedMutableHashTable class.""" - - def testShardedMutableHashTable(self): - for num_shards in [1, 3, 10]: - with self.cached_session(): - default_val = -1 - empty_key = 0 - deleted_key = -1 - keys = constant_op.constant([11, 12, 13], dtypes.int64) - values = constant_op.constant([0, 1, 2], dtypes.int64) - table = ShardedMutableDenseHashTable( - dtypes.int64, - dtypes.int64, - default_val, - empty_key, - deleted_key, - num_shards=num_shards) - self.assertAllEqual(0, table.size().eval()) - - table.insert(keys, values).run() - self.assertAllEqual(3, table.size().eval()) - - input_string = constant_op.constant([11, 12, 14], dtypes.int64) - output = table.lookup(input_string) - self.assertAllEqual([3], output.get_shape()) - self.assertAllEqual([0, 1, -1], output.eval()) - - def testShardedMutableHashTableVectors(self): - for num_shards in [1, 3, 10]: - with self.cached_session(): - default_val = [-0.1, 0.2] - empty_key = [0, 1] - deleted_key = [1, 0] - keys = constant_op.constant([[11, 12], [13, 14], [15, 16]], - dtypes.int64) - values = constant_op.constant([[0.5, 0.6], [1.5, 1.6], [2.5, 2.6]], - dtypes.float32) - table = ShardedMutableDenseHashTable( - dtypes.int64, - dtypes.float32, - default_val, - empty_key, - deleted_key, - num_shards=num_shards) - self.assertAllEqual(0, table.size().eval()) - - table.insert(keys, values).run() - self.assertAllEqual(3, table.size().eval()) - - input_string = constant_op.constant([[11, 12], [13, 14], [11, 14]], - dtypes.int64) - output = table.lookup(input_string) - self.assertAllEqual([3, 2], output.get_shape()) - self.assertAllClose([[0.5, 0.6], [1.5, 1.6], [-0.1, 0.2]], - output.eval()) - - def testExportSharded(self): - with self.cached_session(): - empty_key = -2 - deleted_key = -3 - default_val = -1 - num_shards = 2 - keys = constant_op.constant([10, 11, 12], dtypes.int64) - values = constant_op.constant([2, 3, 4], dtypes.int64) - table = ShardedMutableDenseHashTable( - dtypes.int64, - dtypes.int64, - default_val, - empty_key, - deleted_key, - num_shards=num_shards) - self.assertAllEqual(0, table.size().eval()) - - table.insert(keys, values).run() - self.assertAllEqual(3, table.size().eval()) - - keys_list, values_list = table.export_sharded() - self.assertAllEqual(num_shards, len(keys_list)) - self.assertAllEqual(num_shards, len(values_list)) - - # Exported keys include empty key buckets set to the empty_key - self.assertAllEqual(set([-2, 10, 12]), set(keys_list[0].eval().flatten())) - self.assertAllEqual(set([-2, 11]), set(keys_list[1].eval().flatten())) - # Exported values include empty value buckets set to 0 - self.assertAllEqual(set([0, 2, 4]), set(values_list[0].eval().flatten())) - self.assertAllEqual(set([0, 3]), set(values_list[1].eval().flatten())) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column.py b/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column.py deleted file mode 100644 index 64730f8eed1..00000000000 --- a/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Sparse feature column (deprecated). - -This module and all its submodules are deprecated. To UPDATE or USE linear -optimizers, please check its latest version in core: -tensorflow_estimator/python/estimator/canned/linear_optimizer/. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework.ops import internal_convert_to_tensor -from tensorflow.python.framework.ops import name_scope -from tensorflow.python.util import deprecation - - -class SparseFeatureColumn(object): - """Represents a sparse feature column. - - Contains three tensors representing a sparse feature column, they are - example indices (`int64`), feature indices (`int64`), and feature - values (`float`). - Feature weights are optional, and are treated as `1.0f` if missing. - - For example, consider a batch of 4 examples, which contains the following - features in a particular `SparseFeatureColumn`: - - * Example 0: feature 5, value 1 - * Example 1: feature 6, value 1 and feature 10, value 0.5 - * Example 2: no features - * Example 3: two copies of feature 2, value 1 - - This SparseFeatureColumn will be represented as follows: - - ``` - <0, 5, 1> - <1, 6, 1> - <1, 10, 0.5> - <3, 2, 1> - <3, 2, 1> - ``` - - For a batch of 2 examples below: - - * Example 0: feature 5 - * Example 1: feature 6 - - is represented by `SparseFeatureColumn` as: - - ``` - <0, 5, 1> - <1, 6, 1> - - ``` - - @@__init__ - @@example_indices - @@feature_indices - @@feature_values - """ - - @deprecation.deprecated( - None, 'This class is deprecated. To UPDATE or USE linear optimizers, ' - 'please check its latest version in core: ' - 'tensorflow_estimator/python/estimator/canned/linear_optimizer/.') - def __init__(self, example_indices, feature_indices, feature_values): - """Creates a `SparseFeatureColumn` representation. - - Args: - example_indices: A 1-D int64 tensor of shape `[N]`. Also, accepts - python lists, or numpy arrays. - feature_indices: A 1-D int64 tensor of shape `[N]`. Also, accepts - python lists, or numpy arrays. - feature_values: An optional 1-D tensor float tensor of shape `[N]`. Also, - accepts python lists, or numpy arrays. - - Returns: - A `SparseFeatureColumn` - """ - with name_scope(None, 'SparseFeatureColumn', - [example_indices, feature_indices]): - self._example_indices = internal_convert_to_tensor( - example_indices, name='example_indices', dtype=dtypes.int64) - self._feature_indices = internal_convert_to_tensor( - feature_indices, name='feature_indices', dtype=dtypes.int64) - self._feature_values = None - if feature_values is not None: - with name_scope(None, 'SparseFeatureColumn', [feature_values]): - self._feature_values = internal_convert_to_tensor( - feature_values, name='feature_values', dtype=dtypes.float32) - - @property - def example_indices(self): - """The example indices represented as a dense tensor. - - Returns: - A 1-D Tensor of int64 with shape `[N]`. - """ - return self._example_indices - - @property - def feature_indices(self): - """The feature indices represented as a dense tensor. - - Returns: - A 1-D Tensor of int64 with shape `[N]`. - """ - return self._feature_indices - - @property - def feature_values(self): - """The feature values represented as a dense tensor. - - Returns: - May return None, or a 1-D Tensor of float32 with shape `[N]`. - """ - return self._feature_values diff --git a/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column_test.py b/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column_test.py deleted file mode 100644 index 0ae780e1a10..00000000000 --- a/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column_test.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for sparse_feature_column.py (deprecated). - -This module and all its submodules are deprecated. To UPDATE or USE linear -optimizers, please check its latest version in core: -tensorflow_estimator/python/estimator/canned/linear_optimizer/. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.linear_optimizer.python.ops.sparse_feature_column import SparseFeatureColumn -from tensorflow.python.framework import ops -from tensorflow.python.framework.test_util import TensorFlowTestCase -from tensorflow.python.platform import googletest - - -class SparseFeatureColumnTest(TensorFlowTestCase): - """Tests for SparseFeatureColumn. - """ - - def testBasic(self): - expected_example_indices = [1, 1, 1, 2] - expected_feature_indices = [0, 1, 2, 0] - sfc = SparseFeatureColumn(expected_example_indices, - expected_feature_indices, None) - self.assertTrue(isinstance(sfc.example_indices, ops.Tensor)) - self.assertTrue(isinstance(sfc.feature_indices, ops.Tensor)) - self.assertEqual(sfc.feature_values, None) - with self.cached_session(): - self.assertAllEqual(expected_example_indices, sfc.example_indices.eval()) - self.assertAllEqual(expected_feature_indices, sfc.feature_indices.eval()) - expected_feature_values = [1.0, 2.0, 3.0, 4.0] - sfc = SparseFeatureColumn([1, 1, 1, 2], [0, 1, 2, 0], - expected_feature_values) - with self.cached_session(): - self.assertAllEqual(expected_feature_values, sfc.feature_values.eval()) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py b/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py deleted file mode 100644 index 200e7de6b95..00000000000 --- a/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py +++ /dev/null @@ -1,535 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Linear Estimators.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.python.training import training_util -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.contrib.linear_optimizer.python import sdca_optimizer -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.training import session_run_hook - - -def _head_is_valid_for_sdca(head): - """Returns true if the provided head is supported by SDCAOptimizer.""" - # pylint: disable=protected-access - return isinstance(head, head_lib._BinaryLogisticHead) or isinstance( - head, head_lib._BinarySvmHead) or isinstance(head, - head_lib._RegressionHead) - # pylint: enable=protected-access - - -def _add_bias_column(feature_columns, columns_to_tensors, bias_variable, - columns_to_variables): - """Adds a fake bias feature column filled with all 1s.""" - # TODO(b/31008490): Move definition to a common constants place. - bias_column_name = "tf_virtual_bias_column" - if any(col.name is bias_column_name for col in feature_columns): - raise ValueError("%s is a reserved column name." % bias_column_name) - if not feature_columns: - raise ValueError("feature_columns can't be empty.") - - # Loop through input tensors until we can figure out batch_size. - batch_size = None - for column in columns_to_tensors.values(): - if isinstance(column, tuple): - column = column[0] - if isinstance(column, sparse_tensor.SparseTensor): - shape = tensor_util.constant_value(column.dense_shape) - if shape is not None: - batch_size = shape[0] - break - else: - batch_size = array_ops.shape(column)[0] - break - if batch_size is None: - raise ValueError("Could not infer batch size from input features.") - - bias_column = layers.real_valued_column(bias_column_name) - columns_to_tensors[bias_column] = array_ops.ones( - [batch_size, 1], dtype=dtypes.float32) - columns_to_variables[bias_column] = [bias_variable] - - -def sdca_model_fn(features, labels, mode, params, config=None): - """A model_fn for linear models that use the SDCA optimizer. - - Args: - features: A dict of `Tensor` keyed by column name. - labels: `Tensor` of shape [batch_size, 1] or [batch_size] labels of - dtype `int32` or `int64` with values in the set {0, 1}. - mode: Defines whether this is training, evaluation or prediction. - See `ModeKeys`. - params: A dict of hyperparameters. - The following hyperparameters are expected: - * head: A `Head` instance. Type must be one of `_BinarySvmHead`, - `_RegressionHead` or `_BinaryLogisticHead`. - * feature_columns: An iterable containing all the feature columns used by - the model. - * l1_regularization: Global (across all examples) L1-regularization - parameter. - * l2_regularization: Global (across all examples) L2-regularization - parameter. - * num_loss_partitions: Number of partitions of the global loss function - optimized by `SDCAOptimizer`. - * weight_column_name: A string defining the weight feature column, or - None if there are no weights. - * update_weights_hook: A `SessionRunHook` object or None. Used to update - model weights. - config: `RunConfig` object to configure the runtime settings. - - Returns: - A `ModelFnOps` instance. - - Raises: - ValueError: If the type of head is not one of `_BinarySvmHead`, - `_RegressionHead` or `_MultiClassHead`. - ValueError: If mode is not any of the `ModeKeys`. - """ - head = params["head"] - feature_columns = params["feature_columns"] - example_id_column = params["example_id_column"] - l1_regularization = params["l1_regularization"] - l2_regularization = params["l2_regularization"] - num_loss_partitions = params["num_loss_partitions"] - weight_column_name = params["weight_column_name"] - update_weights_hook = params.get("update_weights_hook", None) - partitioner = params["partitioner"] - - loss_type = None - if isinstance(head, head_lib._BinarySvmHead): # pylint: disable=protected-access - loss_type = "hinge_loss" - elif isinstance(head, head_lib._BinaryLogisticHead): # pylint: disable=protected-access - loss_type = "logistic_loss" - elif isinstance(head, head_lib._RegressionHead): # pylint: disable=protected-access - loss_type = "squared_loss" - else: - raise ValueError("Unsupported head type: {}".format(type(head))) - - assert head.logits_dimension == 1, ( - "SDCA only applies to logits_dimension=1.") - - # Update num_loss_partitions based on number of workers. - n_loss_partitions = num_loss_partitions or max(1, config.num_worker_replicas) - optimizer = sdca_optimizer.SDCAOptimizer( - example_id_column=example_id_column, - num_loss_partitions=n_loss_partitions, - symmetric_l1_regularization=l1_regularization, - symmetric_l2_regularization=l2_regularization, - partitioner=partitioner) - - parent_scope = "linear" - - with variable_scope.variable_scope( - values=features.values(), name_or_scope=parent_scope, - partitioner=partitioner) as scope: - features = features.copy() - features.update(layers.transform_features(features, feature_columns)) - logits, columns_to_variables, bias = ( - layers.weighted_sum_from_feature_columns( - columns_to_tensors=features, - feature_columns=feature_columns, - num_outputs=1, - scope=scope)) - - _add_bias_column(feature_columns, features, bias, columns_to_variables) - - def _train_op_fn(unused_loss): - global_step = training_util.get_global_step() - sdca_model, train_op = optimizer.get_train_step( - columns_to_variables, weight_column_name, loss_type, features, labels, - global_step) - if update_weights_hook is not None: - update_weights_hook.set_parameters(sdca_model, train_op) - return train_op - - model_fn_ops = head.create_model_fn_ops( - features=features, - labels=labels, - mode=mode, - train_op_fn=_train_op_fn, - logits=logits) - if update_weights_hook is not None: - return model_fn_ops._replace(training_chief_hooks=( - model_fn_ops.training_chief_hooks + [update_weights_hook])) - return model_fn_ops - - -class _SdcaUpdateWeightsHook(session_run_hook.SessionRunHook): - """SessionRunHook to update and shrink SDCA model weights.""" - - def __init__(self): - pass - - def set_parameters(self, sdca_model, train_op): - self._sdca_model = sdca_model - self._train_op = train_op - - def begin(self): - """Construct the update_weights op. - - The op is implicitly added to the default graph. - """ - self._update_op = self._sdca_model.update_weights(self._train_op) - - def before_run(self, run_context): - """Return the update_weights op so that it is executed during this run.""" - return session_run_hook.SessionRunArgs(self._update_op) - - -class _SDCAEstimator(estimator.Estimator): - """Base estimator class for linear models using the SDCA optimizer. - - This class should not be used directly. Rather, users should call one of the - derived estimators. - """ - - def __init__(self, - example_id_column, - feature_columns, - weight_column_name=None, - model_dir=None, - head=None, - l1_regularization=0.0, - l2_regularization=1.0, - num_loss_partitions=None, - config=None, - feature_engineering_fn=None, - partitioner=None): - """Construct a `_SDCAEstimator` estimator object. - - Args: - example_id_column: A string defining the feature column name representing - example ids. Used to initialize the underlying SDCA optimizer. - feature_columns: An iterable containing all the feature columns used by - the model. All items in the set should be instances of classes derived - from `FeatureColumn`. - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - model_dir: Directory to save model parameters, graph etc. This can also be - used to load checkpoints from the directory into an estimator to - continue training a previously saved model. - head: type of head. Currently, _BinaryLogisticHead and _BinarySvmHead are - supported for classification and _RegressionHead for regression. It - should be a subclass of _SingleHead. - l1_regularization: L1-regularization parameter. Refers to global L1 - regularization (across all examples). - l2_regularization: L2-regularization parameter. Refers to global L2 - regularization (across all examples). - num_loss_partitions: number of partitions of the (global) loss function - optimized by the underlying optimizer (SDCAOptimizer). - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - partitioner: Variable partitioner for the primal weights (`div` - partitioning strategy will be used). - - Returns: - A `_SDCAEstimator` estimator. - - Raises: - ValueError: if head is not supported by SDCA. - """ - self._feature_columns = tuple(feature_columns or []) - assert self._feature_columns - - if not _head_is_valid_for_sdca(head): - raise ValueError( - "head type: {} is not supported. Supported head types: " - "_BinaryLogisticHead, _BinarySvmHead and _RegressionHead.".format( - type(head))) - assert head.logits_dimension == 1 - - params = { - "head": head, - "feature_columns": feature_columns, - "example_id_column": example_id_column, - "num_loss_partitions": num_loss_partitions, - "l1_regularization": l1_regularization, - "l2_regularization": l2_regularization, - "weight_column_name": weight_column_name, - "update_weights_hook": _SdcaUpdateWeightsHook(), - "partitioner": partitioner, - } - - super(_SDCAEstimator, self).__init__( - model_fn=sdca_model_fn, - model_dir=model_dir, - config=config, - params=params, - feature_engineering_fn=feature_engineering_fn) - - -class SDCALogisticClassifier(_SDCAEstimator): - """Logistic regression binary classifier using the SDCA optimizer. - - Example usage: - - ```python - sparse_column_a = sparse_column_with_hash_bucket(...) - sparse_column_b = sparse_column_with_hash_bucket(...) - - sparse_feature_a_x_sparse_feature_b = crossed_column(...) - - classifier = SDCALogisticClassifier( - example_id_column='example_id', - feature_columns=[sparse_column_a, sparse_feature_a_x_sparse_feature_b]), - weight_column_name=..., - l2_regularization=..., - num_loss_partitions=..., - ) - - # Input builders - # returns x, y (where y is the label Tensor (with 0/1 values) - def input_fn_{train, eval}: - - # returns x (features dict) - def input_fn_test: - ... - classifier.fit(input_fn=input_fn_train) - classifier.evaluate(input_fn=input_fn_eval) - # Returns predicted classes. - classifier.predict_classes(input_fn=input_fn_test) - # Returns predicted probabilities. - classifier.predict_proba(input_fn=input_fn_test) - ``` - - The input_fn provided to `fit`, `evaluate` and predict_* methods should return - the following features, otherwise there will be a `KeyError`: - * A feature with `key=example_id_column` whose value is a `Tensor` of dtype - string. - * If `weight_column_name` is not `None`, a feature with - `key=weight_column_name` whose value is a `Tensor`. - * For each `column` in `feature_columns`: - - if `column` is a `SparseColumn`, a feature with `key=column.name` whose - `value` is a `SparseTensor` - - if `column` is a `RealValuedColumn, a feature with `key=column.name` - whose `value` is a `Tensor` - - if `column` is a `WeightedSparseColumn`, two features: the first with - `key` the id column name, the second with `key` the weight column name. - Both features' `value` must be a `SparseTensor` - """ - - def __init__(self, - example_id_column, - feature_columns, - weight_column_name=None, - model_dir=None, - l1_regularization=0.0, - l2_regularization=1.0, - num_loss_partitions=None, - config=None, - feature_engineering_fn=None, - partitioner=None): - """Construct a `SDCALogisticClassifier` object. - - Args: - example_id_column: A string defining the feature column name representing - example ids. Used to initialize the underlying SDCA optimizer. - feature_columns: An iterable containing all the feature columns used by - the model. All items in the iterable should derive from `FeatureColumn`. - Note that the order of the items is ignored at model construction time. - weight_column_name: A string defining feature column name representing - weights. It is used to downweight or boost examples during training. It - will be multiplied by the loss of the example. - model_dir: Directory to save model parameters, graph etc. This can also be - used to load checkpoints from the directory into an estimator to - continue training a previously saved model. - l1_regularization: L1-regularization parameter. Refers to global L1 - regularization (across all examples). - l2_regularization: L2-regularization parameter. Refers to global L2 - regularization (across all examples). - num_loss_partitions: Number of partitions of the global loss function - optimized by the underlying optimizer (SDCAOptimizer). - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - partitioner: Variable partitioner for the primal weights (`div` - partitioning strategy will be used). - - Returns: - A `SDCALogisiticClassifier` estimator. - """ - super(SDCALogisticClassifier, self).__init__( - example_id_column=example_id_column, - feature_columns=feature_columns, - weight_column_name=weight_column_name, - model_dir=model_dir, - head=head_lib.multi_class_head( - n_classes=2, weight_column_name=weight_column_name), - l1_regularization=l1_regularization, - l2_regularization=l2_regularization, - num_loss_partitions=num_loss_partitions, - config=config, - feature_engineering_fn=None, - partitioner=partitioner) - - def predict_classes(self, input_fn=None): - """Runs inference to determine the predicted class. - - Args: - input_fn: The input function providing features. - - Returns: - A generator of predicted classes for the features provided by input_fn. - """ - key = prediction_key.PredictionKey.CLASSES - predictions = super(SDCALogisticClassifier, self).predict( - input_fn=input_fn, outputs=[key]) - return (pred[key] for pred in predictions) - - def predict_proba(self, input_fn=None): - """Runs inference to determine the class probability predictions. - - Args: - input_fn: The input function providing features. - - Returns: - A generator of predicted class probabilities for the features provided by - input_fn. - """ - key = prediction_key.PredictionKey.PROBABILITIES - predictions = super(SDCALogisticClassifier, self).predict( - input_fn=input_fn, outputs=[key]) - return (pred[key] for pred in predictions) - - -class SDCALinearRegressor(_SDCAEstimator): - """Linear regression model using SDCA to solve the underlying optimization. - - Example usage: - - ```python - real_column_a = real_valued_column(...) - sparse_column_b = sparse_column_with_hash_bucket(...) - - regressor = SDCALinearRegressor( - example_id_column='example_id', - feature_columns=[real_column_a, sparse_column_b]), - weight_column_name=..., - l2_regularization=..., - num_loss_partitions=..., - ) - - # Input builders - # returns x, y (where y is the label Tensor (with 0/1 values) - def input_fn_{train, eval}: - - # returns x (features dict) - def input_fn_test: - ... - regressor.fit(input_fn=input_fn_train) - regressor.evaluate(input_fn=input_fn_eval) - regressor.predict_scores(input_fn=input_fn_test) # returns predicted scores. - ``` - - The input_fn provided to `fit`, `evaluate` and predict_* methods should return - the following features, otherwise there will be a `KeyError`: - * A feature with `key=example_id_column` whose value is a `Tensor` of dtype - string. - * If `weight_column_name` is not `None`, a feature with - `key=weight_column_name` whose value is a `Tensor`. - * For each `column` in `feature_columns`: - - if `column` is a `SparseColumn`, a feature with `key=column.name` whose - `value` is a `SparseTensor` - - if `column` is a `RealValuedColumn, a feature with `key=column.name` - whose `value` is a `Tensor` - - if `column` is a `WeightedSparseColumn`, two features: the first with - `key` the id column name, the second with `key` the weight column name. - Both features' `value` must be a `SparseTensor` - - """ - - def __init__(self, - example_id_column, - feature_columns, - weight_column_name=None, - model_dir=None, - l1_regularization=0.0, - l2_regularization=1.0, - num_loss_partitions=None, - config=None, - feature_engineering_fn=None, - partitioner=None): - """Construct a `SDCALinearRegressor` estimator object. - - - Args: - example_id_column: A string defining the feature column name representing - example ids. Used to initialize the underlying SDCA optimizer. - feature_columns: An iterable containing all the feature columns used by - the model. All items in the iterable should derive from `FeatureColumn`. - Note that the order of the items is ignored at model construction time. - weight_column_name: A string defining feature column name representing - weights. It is used to down weight or boost examples during training. It - will be multiplied by the loss of the example. - model_dir: Directory to save model parameters, graph etc. This can also be - used to load checkpoints from the directory into an estimator to - continue training a previously saved model. - l1_regularization: L1-regularization parameter. Refers to global L1 - regularization (across all examples). - l2_regularization: L2-regularization parameter. Refers to global L2 - regularization (across all examples). - num_loss_partitions: number of partitions of the (global) loss function - optimized by the underlying optimizer (SDCAOptimizer). - config: `RunConfig` object to configure the runtime settings. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - partitioner: Variable partitioner for the primal weights (`div` - partitioning strategy will be used). - - Returns: - A `SDCALinearRegressor` estimator. - """ - super(SDCALinearRegressor, self).__init__( - example_id_column=example_id_column, - feature_columns=feature_columns, - weight_column_name=weight_column_name, - model_dir=model_dir, - head=head_lib.regression_head(weight_column_name=weight_column_name), - l1_regularization=l1_regularization, - l2_regularization=l2_regularization, - num_loss_partitions=num_loss_partitions, - config=config, - feature_engineering_fn=None, - partitioner=partitioner) - - def predict_scores(self, input_fn): - """Returns predicted scores for given features. - - Args: - input_fn: The input function providing features. - - Returns: - A generator of predicted scores for the features provided by input_fn. - """ - key = prediction_key.PredictionKey.SCORES - predictions = super(SDCALinearRegressor, self).predict( - input_fn=input_fn, outputs=[key]) - return (pred[key] for pred in predictions) diff --git a/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py b/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py deleted file mode 100644 index 7a5354222f1..00000000000 --- a/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py +++ /dev/null @@ -1,651 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for linear_optimizer.sdca_estimator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.layers.python.layers import feature_column as feature_column_lib -from tensorflow.contrib.linear_optimizer.python import sdca_estimator -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.platform import test - - -class SDCALogisticClassifierTest(test.TestCase): - - def _single_threaded_test_session(self): - # TODO(andreasst): figure out why SDCALinearRegressor needs a single - # threaded session to pass in tsan mode but SDCALogisticClassifier does not. - config = config_pb2.ConfigProto( - inter_op_parallelism_threads=1, intra_op_parallelism_threads=1) - return self.test_session(config=config) - - def testRealValuedFeatures(self): - """Tests SDCALogisticClassifier works with real valued features.""" - - def input_fn(): - return { - 'example_id': constant_op.constant(['1', '2']), - 'maintenance_cost': constant_op.constant([500.0, 200.0]), - 'sq_footage': constant_op.constant([[800.0], [600.0]]), - 'weights': constant_op.constant([[1.0], [1.0]]) - }, constant_op.constant([[0], [1]]) - - with self._single_threaded_test_session(): - maintenance_cost = feature_column_lib.real_valued_column( - 'maintenance_cost') - sq_footage = feature_column_lib.real_valued_column('sq_footage') - classifier = sdca_estimator.SDCALogisticClassifier( - example_id_column='example_id', - feature_columns=[maintenance_cost, sq_footage], - weight_column_name='weights') - classifier.fit(input_fn=input_fn, steps=100) - loss = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss, 0.05) - - def testRealValuedFeatureWithHigherDimension(self): - """Tests SDCALogisticClassifier with high-dimension real valued features.""" - - # input_fn is identical to the one in testRealValuedFeatures where 2 - # 1-dimensional dense features are replaced by a 2-dimensional feature. - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2']), - 'dense_feature': - constant_op.constant([[500.0, 800.0], [200.0, 600.0]]) - }, constant_op.constant([[0], [1]]) - - with self._single_threaded_test_session(): - dense_feature = feature_column_lib.real_valued_column( - 'dense_feature', dimension=2) - classifier = sdca_estimator.SDCALogisticClassifier( - example_id_column='example_id', feature_columns=[dense_feature]) - classifier.fit(input_fn=input_fn, steps=100) - loss = classifier.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss, 0.05) - - def testBucketizedFeatures(self): - """Tests SDCALogisticClassifier with bucketized features.""" - - def input_fn(): - return { - 'example_id': constant_op.constant(['1', '2', '3']), - 'price': constant_op.constant([600.0, 1000.0, 400.0]), - 'sq_footage': constant_op.constant([[1000.0], [600.0], [700.0]]), - 'weights': constant_op.constant([[1.0], [1.0], [1.0]]) - }, constant_op.constant([[1], [0], [1]]) - - with self._single_threaded_test_session(): - price_bucket = feature_column_lib.bucketized_column( - feature_column_lib.real_valued_column('price'), - boundaries=[500.0, 700.0]) - sq_footage_bucket = feature_column_lib.bucketized_column( - feature_column_lib.real_valued_column('sq_footage'), - boundaries=[650.0]) - classifier = sdca_estimator.SDCALogisticClassifier( - example_id_column='example_id', - feature_columns=[price_bucket, sq_footage_bucket], - weight_column_name='weights', - l2_regularization=1.0) - classifier.fit(input_fn=input_fn, steps=50) - metrics = classifier.evaluate(input_fn=input_fn, steps=1) - self.assertGreater(metrics['accuracy'], 0.9) - - def testSparseFeatures(self): - """Tests SDCALogisticClassifier with sparse features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([[0.4], [0.6], [0.3]]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - 'weights': - constant_op.constant([[1.0], [1.0], [1.0]]) - }, constant_op.constant([[1], [0], [1]]) - - with self._single_threaded_test_session(): - price = feature_column_lib.real_valued_column('price') - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - classifier = sdca_estimator.SDCALogisticClassifier( - example_id_column='example_id', - feature_columns=[price, country], - weight_column_name='weights') - classifier.fit(input_fn=input_fn, steps=50) - metrics = classifier.evaluate(input_fn=input_fn, steps=1) - self.assertGreater(metrics['accuracy'], 0.9) - - def testWeightedSparseFeatures(self): - """Tests SDCALogisticClassifier with weighted sparse features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - sparse_tensor.SparseTensor( - values=[2., 3., 1.], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 5]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 5]) - }, constant_op.constant([[1], [0], [1]]) - - with self._single_threaded_test_session(): - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - country_weighted_by_price = feature_column_lib.weighted_sparse_column( - country, 'price') - classifier = sdca_estimator.SDCALogisticClassifier( - example_id_column='example_id', - feature_columns=[country_weighted_by_price]) - classifier.fit(input_fn=input_fn, steps=50) - metrics = classifier.evaluate(input_fn=input_fn, steps=1) - self.assertGreater(metrics['accuracy'], 0.9) - - def testSparseFeaturesWithDuplicates(self): - """Tests SDCALogisticClassifier with duplicated sparse features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2']), - 'age': - sparse_tensor.SparseTensor( - values=['20-29'] * 5 + ['31-40'] * 5, - indices=[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [1, 0], - [1, 0], [1, 0], [1, 0], [1, 0]], - dense_shape=[2, 1]), - 'gender': - sparse_tensor.SparseTensor( - values=['m'] * 5 + ['f'] * 5, - indices=[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [1, 0], - [1, 0], [1, 0], [1, 0], [1, 0]], - dense_shape=[2, 1]), - }, constant_op.constant([[1], [0]]) - - with self._single_threaded_test_session(): - age = feature_column_lib.sparse_column_with_hash_bucket( - 'age', hash_bucket_size=10) - gender = feature_column_lib.sparse_column_with_hash_bucket( - 'gender', hash_bucket_size=10) - classifier = sdca_estimator.SDCALogisticClassifier( - example_id_column='example_id', feature_columns=[age, gender]) - classifier.fit(input_fn=input_fn, steps=50) - metrics = classifier.evaluate(input_fn=input_fn, steps=1) - self.assertLess(metrics['loss'], 0.060) - - def testCrossedFeatures(self): - """Tests SDCALogisticClassifier with crossed features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'language': - sparse_tensor.SparseTensor( - values=['english', 'italian', 'spanish'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]), - 'country': - sparse_tensor.SparseTensor( - values=['US', 'IT', 'MX'], - indices=[[0, 0], [1, 0], [2, 0]], - dense_shape=[3, 1]) - }, constant_op.constant([[0], [0], [1]]) - - with self._single_threaded_test_session(): - language = feature_column_lib.sparse_column_with_hash_bucket( - 'language', hash_bucket_size=5) - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - country_language = feature_column_lib.crossed_column( - [language, country], hash_bucket_size=10) - classifier = sdca_estimator.SDCALogisticClassifier( - example_id_column='example_id', feature_columns=[country_language]) - classifier.fit(input_fn=input_fn, steps=10) - metrics = classifier.evaluate(input_fn=input_fn, steps=1) - self.assertGreater(metrics['accuracy'], 0.9) - - def testMixedFeatures(self): - """Tests SDCALogisticClassifier with a mix of features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([[0.6], [0.8], [0.3]]), - 'sq_footage': - constant_op.constant([900.0, 700.0, 600.0]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - 'weights': - constant_op.constant([[3.0], [1.0], [1.0]]) - }, constant_op.constant([[1], [0], [1]]) - - with self._single_threaded_test_session(): - price = feature_column_lib.real_valued_column('price') - sq_footage_bucket = feature_column_lib.bucketized_column( - feature_column_lib.real_valued_column('sq_footage'), - boundaries=[650.0, 800.0]) - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - sq_footage_country = feature_column_lib.crossed_column( - [sq_footage_bucket, country], hash_bucket_size=10) - classifier = sdca_estimator.SDCALogisticClassifier( - example_id_column='example_id', - feature_columns=[ - price, sq_footage_bucket, country, sq_footage_country - ], - weight_column_name='weights') - classifier.fit(input_fn=input_fn, steps=50) - metrics = classifier.evaluate(input_fn=input_fn, steps=1) - self.assertGreater(metrics['accuracy'], 0.9) - - def testPartitionedMixedFeatures(self): - """Tests SDCALogisticClassifier with a mix of features (partitioned).""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([[0.6], [0.8], [0.3]]), - 'sq_footage': - constant_op.constant([900.0, 700.0, 600.0]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - 'weights': - constant_op.constant([[3.0], [1.0], [1.0]]) - }, constant_op.constant([[1], [0], [1]]) - - with self._single_threaded_test_session(): - price = feature_column_lib.real_valued_column('price') - sq_footage_bucket = feature_column_lib.bucketized_column( - feature_column_lib.real_valued_column('sq_footage'), - boundaries=[650.0, 800.0]) - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - sq_footage_country = feature_column_lib.crossed_column( - [sq_footage_bucket, country], hash_bucket_size=10) - classifier = sdca_estimator.SDCALogisticClassifier( - example_id_column='example_id', - feature_columns=[ - price, sq_footage_bucket, country, sq_footage_country - ], - weight_column_name='weights', - partitioner=partitioned_variables.fixed_size_partitioner( - num_shards=2, axis=0)) - classifier.fit(input_fn=input_fn, steps=50) - metrics = classifier.evaluate(input_fn=input_fn, steps=1) - self.assertGreater(metrics['accuracy'], 0.9) - - -class SDCALinearRegressorTest(test.TestCase): - - def _single_threaded_test_session(self): - # TODO(andreasst): figure out why SDCALinearRegressor needs a single - # threaded session to pass in tsan mode but SDCALogisticClassifier does not. - config = config_pb2.ConfigProto( - inter_op_parallelism_threads=1, intra_op_parallelism_threads=1) - return self.test_session(config=config) - - def testRealValuedLinearFeatures(self): - """Tests SDCALinearRegressor works with real valued features.""" - x = [[1.2, 2.0, -1.5], [-2.0, 3.0, -0.5], [1.0, -0.5, 4.0]] - weights = [[3.0], [-1.2], [0.5]] - y = np.dot(x, weights) - - def input_fn(): - return { - 'example_id': constant_op.constant(['1', '2', '3']), - 'x': constant_op.constant(x), - 'weights': constant_op.constant([[10.0], [10.0], [10.0]]) - }, constant_op.constant(y) - - with self._single_threaded_test_session(): - x_column = feature_column_lib.real_valued_column('x', dimension=3) - regressor = sdca_estimator.SDCALinearRegressor( - example_id_column='example_id', - feature_columns=[x_column], - weight_column_name='weights') - regressor.fit(input_fn=input_fn, steps=20) - loss = regressor.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss, 0.01) - self.assertIn('linear/x/weight', regressor.get_variable_names()) - regressor_weights = regressor.get_variable_value('linear/x/weight') - self.assertAllClose( - [w[0] for w in weights], regressor_weights.flatten(), rtol=0.1) - - def testMixedFeaturesArbitraryWeights(self): - """Tests SDCALinearRegressor works with a mix of features.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([[0.6], [0.8], [0.3]]), - 'sq_footage': - constant_op.constant([[900.0], [700.0], [600.0]]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - 'weights': - constant_op.constant([[3.0], [5.0], [7.0]]) - }, constant_op.constant([[1.55], [-1.25], [-3.0]]) - - with self._single_threaded_test_session(): - price = feature_column_lib.real_valued_column('price') - sq_footage_bucket = feature_column_lib.bucketized_column( - feature_column_lib.real_valued_column('sq_footage'), - boundaries=[650.0, 800.0]) - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - sq_footage_country = feature_column_lib.crossed_column( - [sq_footage_bucket, country], hash_bucket_size=10) - regressor = sdca_estimator.SDCALinearRegressor( - example_id_column='example_id', - feature_columns=[ - price, sq_footage_bucket, country, sq_footage_country - ], - l2_regularization=1.0, - weight_column_name='weights') - regressor.fit(input_fn=input_fn, steps=20) - loss = regressor.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss, 0.05) - - def testMixedFeaturesArbitraryWeightsPartitioned(self): - """Tests SDCALinearRegressor works with a mix of features (partitioned).""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([[0.6], [0.8], [0.3]]), - 'sq_footage': - constant_op.constant([[900.0], [700.0], [600.0]]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - 'weights': - constant_op.constant([[3.0], [5.0], [7.0]]) - }, constant_op.constant([[1.55], [-1.25], [-3.0]]) - - with self._single_threaded_test_session(): - price = feature_column_lib.real_valued_column('price') - sq_footage_bucket = feature_column_lib.bucketized_column( - feature_column_lib.real_valued_column('sq_footage'), - boundaries=[650.0, 800.0]) - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - sq_footage_country = feature_column_lib.crossed_column( - [sq_footage_bucket, country], hash_bucket_size=10) - regressor = sdca_estimator.SDCALinearRegressor( - example_id_column='example_id', - feature_columns=[ - price, sq_footage_bucket, country, sq_footage_country - ], - l2_regularization=1.0, - weight_column_name='weights', - partitioner=partitioned_variables.fixed_size_partitioner( - num_shards=2, axis=0)) - regressor.fit(input_fn=input_fn, steps=20) - loss = regressor.evaluate(input_fn=input_fn, steps=1)['loss'] - self.assertLess(loss, 0.05) - - def testSdcaOptimizerSparseFeaturesWithL1Reg(self): - """SDCALinearRegressor works with sparse features and L1 regularization.""" - - def input_fn(): - return { - 'example_id': - constant_op.constant(['1', '2', '3']), - 'price': - constant_op.constant([0.4, 0.6, 0.3]), - 'country': - sparse_tensor.SparseTensor( - values=['IT', 'US', 'GB'], - indices=[[0, 0], [1, 3], [2, 1]], - dense_shape=[3, 5]), - 'weights': - constant_op.constant([[10.0], [10.0], [10.0]]) - }, constant_op.constant([[1.4], [-0.8], [2.6]]) - - with self._single_threaded_test_session(): - price = feature_column_lib.real_valued_column('price') - country = feature_column_lib.sparse_column_with_hash_bucket( - 'country', hash_bucket_size=5) - # Regressor with no L1 regularization. - regressor = sdca_estimator.SDCALinearRegressor( - example_id_column='example_id', - feature_columns=[price, country], - weight_column_name='weights') - regressor.fit(input_fn=input_fn, steps=20) - no_l1_reg_loss = regressor.evaluate(input_fn=input_fn, steps=1)['loss'] - variable_names = regressor.get_variable_names() - self.assertIn('linear/price/weight', variable_names) - self.assertIn('linear/country/weights', variable_names) - no_l1_reg_weights = { - 'linear/price/weight': - regressor.get_variable_value('linear/price/weight'), - 'linear/country/weights': - regressor.get_variable_value('linear/country/weights'), - } - - # Regressor with L1 regularization. - regressor = sdca_estimator.SDCALinearRegressor( - example_id_column='example_id', - feature_columns=[price, country], - l1_regularization=1.0, - weight_column_name='weights') - regressor.fit(input_fn=input_fn, steps=20) - l1_reg_loss = regressor.evaluate(input_fn=input_fn, steps=1)['loss'] - l1_reg_weights = { - 'linear/price/weight': - regressor.get_variable_value('linear/price/weight'), - 'linear/country/weights': - regressor.get_variable_value('linear/country/weights'), - } - - # Unregularized loss is lower when there is no L1 regularization. - self.assertLess(no_l1_reg_loss, l1_reg_loss) - self.assertLess(no_l1_reg_loss, 0.05) - - # But weights returned by the regressor with L1 regularization have - # smaller L1 norm. - l1_reg_weights_norm, no_l1_reg_weights_norm = 0.0, 0.0 - for var_name in sorted(l1_reg_weights): - l1_reg_weights_norm += sum( - np.absolute(l1_reg_weights[var_name].flatten())) - no_l1_reg_weights_norm += sum( - np.absolute(no_l1_reg_weights[var_name].flatten())) - print('Var name: %s, value: %s' % - (var_name, no_l1_reg_weights[var_name].flatten())) - self.assertLess(l1_reg_weights_norm, no_l1_reg_weights_norm) - - def testBiasOnly(self): - """Tests SDCALinearRegressor has a valid bias weight.""" - - def input_fn(): - """Testing the bias weight when it's the only feature present. - - All of the instances in this input only have the bias feature, and a - 1/4 of the labels are positive. This means that the expected weight for - the bias should be close to the average prediction, i.e 0.25. - Returns: - Training data for the test. - """ - num_examples = 40 - return { - 'example_id': - constant_op.constant([str(x + 1) for x in range(num_examples)]), - # place_holder is an empty column which is always 0 (absent), because - # LinearClassifier requires at least one column. - 'place_holder': - constant_op.constant([[0.0]] * num_examples), - }, constant_op.constant([[1 if i % 4 == 0 else 0] - for i in range(num_examples)]) - - with self._single_threaded_test_session(): - place_holder = feature_column_lib.real_valued_column('place_holder') - regressor = sdca_estimator.SDCALinearRegressor( - example_id_column='example_id', feature_columns=[place_holder]) - regressor.fit(input_fn=input_fn, steps=100) - self.assertNear( - regressor.get_variable_value('linear/bias_weight')[0], 0.25, err=0.1) - - def testBiasAndOtherColumns(self): - """SDCALinearRegressor has valid bias weight with other columns present.""" - - def input_fn(): - """Testing the bias weight when there are other features present. - - 1/2 of the instances in this input have feature 'a', the rest have - feature 'b', and we expect the bias to be added to each instance as well. - 0.4 of all instances that have feature 'a' are positive, and 0.2 of all - instances that have feature 'b' are positive. The labels in the dataset - are ordered to appear shuffled since SDCA expects shuffled data, and - converges faster with this pseudo-random ordering. - If the bias was centered we would expect the weights to be: - bias: 0.3 - a: 0.1 - b: -0.1 - Until b/29339026 is resolved, the bias gets regularized with the same - global value for the other columns, and so the expected weights get - shifted and are: - bias: 0.2 - a: 0.2 - b: 0.0 - Returns: - The test dataset. - """ - num_examples = 200 - half = int(num_examples / 2) - return { - 'example_id': - constant_op.constant([str(x + 1) for x in range(num_examples)]), - 'a': - constant_op.constant([[1]] * int(half) + [[0]] * int(half)), - 'b': - constant_op.constant([[0]] * int(half) + [[1]] * int(half)), - }, constant_op.constant( - [[x] - for x in [1, 0, 0, 1, 1, 0, 0, 0, 1, 0] * int(half / 10) + - [0, 1, 0, 0, 0, 0, 0, 0, 1, 0] * int(half / 10)]) - - with self._single_threaded_test_session(): - regressor = sdca_estimator.SDCALinearRegressor( - example_id_column='example_id', - feature_columns=[ - feature_column_lib.real_valued_column('a'), - feature_column_lib.real_valued_column('b') - ]) - - regressor.fit(input_fn=input_fn, steps=200) - - variable_names = regressor.get_variable_names() - self.assertIn('linear/bias_weight', variable_names) - self.assertIn('linear/a/weight', variable_names) - self.assertIn('linear/b/weight', variable_names) - # TODO(b/29339026): Change the expected results to expect a centered bias. - self.assertNear( - regressor.get_variable_value('linear/bias_weight')[0], 0.2, err=0.05) - self.assertNear( - regressor.get_variable_value('linear/a/weight')[0], 0.2, err=0.05) - self.assertNear( - regressor.get_variable_value('linear/b/weight')[0], 0.0, err=0.05) - - def testBiasAndOtherColumnsFabricatedCentered(self): - """SDCALinearRegressor has valid bias weight when instances are centered.""" - - def input_fn(): - """Testing the bias weight when there are other features present. - - 1/2 of the instances in this input have feature 'a', the rest have - feature 'b', and we expect the bias to be added to each instance as well. - 0.1 of all instances that have feature 'a' have a label of 1, and 0.1 of - all instances that have feature 'b' have a label of -1. - We can expect the weights to be: - bias: 0.0 - a: 0.1 - b: -0.1 - Returns: - The test dataset. - """ - num_examples = 200 - half = int(num_examples / 2) - return { - 'example_id': - constant_op.constant([str(x + 1) for x in range(num_examples)]), - 'a': - constant_op.constant([[1]] * int(half) + [[0]] * int(half)), - 'b': - constant_op.constant([[0]] * int(half) + [[1]] * int(half)), - }, constant_op.constant([[1 if x % 10 == 0 else 0] for x in range(half)] + - [[-1 if x % 10 == 0 else 0] for x in range(half)]) - - with self._single_threaded_test_session(): - regressor = sdca_estimator.SDCALinearRegressor( - example_id_column='example_id', - feature_columns=[ - feature_column_lib.real_valued_column('a'), - feature_column_lib.real_valued_column('b') - ]) - - regressor.fit(input_fn=input_fn, steps=100) - - variable_names = regressor.get_variable_names() - self.assertIn('linear/bias_weight', variable_names) - self.assertIn('linear/a/weight', variable_names) - self.assertIn('linear/b/weight', variable_names) - self.assertNear( - regressor.get_variable_value('linear/bias_weight')[0], 0.0, err=0.05) - self.assertNear( - regressor.get_variable_value('linear/a/weight')[0], 0.1, err=0.05) - self.assertNear( - regressor.get_variable_value('linear/b/weight')[0], -0.1, err=0.05) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/linear_optimizer/python/sdca_optimizer.py b/tensorflow/contrib/linear_optimizer/python/sdca_optimizer.py deleted file mode 100644 index 58ab3aec664..00000000000 --- a/tensorflow/contrib/linear_optimizer/python/sdca_optimizer.py +++ /dev/null @@ -1,278 +0,0 @@ -"""Linear Estimators.""" -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.linear_optimizer.python.ops import sdca_ops -from tensorflow.contrib.linear_optimizer.python.ops.sparse_feature_column import SparseFeatureColumn -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops - - -# TODO(sibyl-vie3Poto, sibyl-Aix6ihai): Add proper testing to this wrapper once the API is -# stable. -class SDCAOptimizer(object): - """Wrapper class for SDCA optimizer. - - The wrapper is currently meant for use as an optimizer within a tf.learn - Estimator. - - Example usage: - - ```python - real_feature_column = real_valued_column(...) - sparse_feature_column = sparse_column_with_hash_bucket(...) - sdca_optimizer = linear.SDCAOptimizer(example_id_column='example_id', - num_loss_partitions=1, - num_table_shards=1, - symmetric_l2_regularization=2.0) - classifier = tf.contrib.learn.LinearClassifier( - feature_columns=[real_feature_column, sparse_feature_column], - weight_column_name=..., - optimizer=sdca_optimizer) - classifier.fit(input_fn_train, steps=50) - classifier.evaluate(input_fn=input_fn_eval) - ``` - - Here the expectation is that the `input_fn_*` functions passed to train and - evaluate return a pair (dict, label_tensor) where dict has `example_id_column` - as `key` whose value is a `Tensor` of shape [batch_size] and dtype string. - num_loss_partitions defines the number of partitions of the global loss - function and should be set to `(#concurrent train ops/per worker) - x (#workers)`. - Convergence of (global) loss is guaranteed if `num_loss_partitions` is larger - or equal to the above product. Larger values for `num_loss_partitions` lead to - slower convergence. The recommended value for `num_loss_partitions` in - `tf.learn` (where currently there is one process per worker) is the number - of workers running the train steps. It defaults to 1 (single machine). - `num_table_shards` defines the number of shards for the internal state - table, typically set to match the number of parameter servers for large - data sets. You can also specify a `partitioner` object to partition the primal - weights during training (`div` partitioning strategy will be used). - """ - - def __init__(self, - example_id_column, - num_loss_partitions=1, - num_table_shards=None, - symmetric_l1_regularization=0.0, - symmetric_l2_regularization=1.0, - adaptive=True, - partitioner=None): - self._example_id_column = example_id_column - self._num_loss_partitions = num_loss_partitions - self._num_table_shards = num_table_shards - self._symmetric_l1_regularization = symmetric_l1_regularization - self._symmetric_l2_regularization = symmetric_l2_regularization - self._adaptive = adaptive - self._partitioner = partitioner - - def get_name(self): - return 'SDCAOptimizer' - - @property - def example_id_column(self): - return self._example_id_column - - @property - def num_loss_partitions(self): - return self._num_loss_partitions - - @property - def num_table_shards(self): - return self._num_table_shards - - @property - def symmetric_l1_regularization(self): - return self._symmetric_l1_regularization - - @property - def symmetric_l2_regularization(self): - return self._symmetric_l2_regularization - - @property - def adaptive(self): - return self._adaptive - - @property - def partitioner(self): - return self._partitioner - - def get_train_step(self, columns_to_variables, weight_column_name, loss_type, - features, targets, global_step): - """Returns the training operation of an SdcaModel optimizer.""" - - def _dense_tensor_to_sparse_feature_column(dense_tensor): - """Returns SparseFeatureColumn for the input dense_tensor.""" - ignore_value = 0.0 - sparse_indices = array_ops.where( - math_ops.not_equal(dense_tensor, - math_ops.cast(ignore_value, dense_tensor.dtype))) - sparse_values = array_ops.gather_nd(dense_tensor, sparse_indices) - # TODO(sibyl-Aix6ihai, sibyl-vie3Poto): Makes this efficient, as now SDCA supports - # very sparse features with weights and not weights. - return SparseFeatureColumn( - array_ops.reshape( - array_ops.split( - value=sparse_indices, num_or_size_splits=2, axis=1)[0], [-1]), - array_ops.reshape( - array_ops.split( - value=sparse_indices, num_or_size_splits=2, axis=1)[1], [-1]), - array_ops.reshape(math_ops.cast(sparse_values, dtypes.float32), [-1])) - - def _training_examples_and_variables(): - """Returns dictionaries for training examples and variables.""" - batch_size = targets.get_shape()[0] - - # Iterate over all feature columns and create appropriate lists for dense - # and sparse features as well as dense and sparse weights (variables) for - # SDCA. - # TODO(sibyl-vie3Poto): Reshape variables stored as values in column_to_variables - # dict as 1-dimensional tensors. - dense_features, sparse_features, sparse_feature_with_values = [], [], [] - dense_feature_weights = [] - sparse_feature_weights, sparse_feature_with_values_weights = [], [] - for column in sorted(columns_to_variables.keys(), key=lambda x: x.key): - transformed_tensor = features[column] - if isinstance(column, layers.feature_column._RealValuedColumn): # pylint: disable=protected-access - # A real-valued column corresponds to a dense feature in SDCA. A - # transformed tensor corresponding to a RealValuedColumn should have - # rank at most 2. In order to be passed to SDCA, its rank needs to be - # exactly 2 (i.e., its shape should be [batch_size, column.dim]). - check_rank_op = control_flow_ops.Assert( - math_ops.less_equal(array_ops.rank(transformed_tensor), 2), - ['transformed_tensor should have rank at most 2.']) - # Reshape to [batch_size, dense_column_dimension]. - with ops.control_dependencies([check_rank_op]): - transformed_tensor = array_ops.reshape(transformed_tensor, [ - array_ops.shape(transformed_tensor)[0], -1 - ]) - - dense_features.append(transformed_tensor) - # For real valued columns, the variables list contains exactly one - # element. - dense_feature_weights.append(columns_to_variables[column][0]) - elif isinstance(column, layers.feature_column._BucketizedColumn): # pylint: disable=protected-access - # A bucketized column corresponds to a sparse feature in SDCA. The - # bucketized feature is "sparsified" for SDCA by converting it to a - # SparseFeatureColumn representing the one-hot encoding of the - # bucketized feature. - # - # TODO(sibyl-vie3Poto): Explore whether it is more efficient to translate a - # bucketized feature column to a dense feature in SDCA. This will - # likely depend on the number of buckets. - dense_bucket_tensor = column._to_dnn_input_layer(transformed_tensor) # pylint: disable=protected-access - sparse_feature_column = _dense_tensor_to_sparse_feature_column( - dense_bucket_tensor) - sparse_feature_with_values.append(sparse_feature_column) - # If a partitioner was used during variable creation, we will have a - # list of Variables here larger than 1. - vars_to_append = columns_to_variables[column][0] - if len(columns_to_variables[column]) > 1: - vars_to_append = columns_to_variables[column] - sparse_feature_with_values_weights.append(vars_to_append) - elif isinstance( - column, - ( - layers.feature_column._WeightedSparseColumn, # pylint: disable=protected-access - layers.feature_column._CrossedColumn, # pylint: disable=protected-access - layers.feature_column._SparseColumn)): # pylint: disable=protected-access - - if isinstance(column, layers.feature_column._WeightedSparseColumn): # pylint: disable=protected-access - id_tensor = column.id_tensor(transformed_tensor) - weight_tensor = array_ops.reshape( - column.weight_tensor(transformed_tensor).values, [-1]) - else: - id_tensor = transformed_tensor - weight_tensor = array_ops.ones( - [array_ops.shape(id_tensor.indices)[0]], dtypes.float32) - - example_ids = array_ops.reshape(id_tensor.indices[:, 0], [-1]) - - flat_ids = array_ops.reshape(id_tensor.values, [-1]) - # Prune invalid IDs (< 0) from the flat_ids, example_ids, and - # weight_tensor. These can come from looking up an OOV entry in the - # vocabulary (default value being -1). - is_id_valid = math_ops.greater_equal(flat_ids, 0) - flat_ids = array_ops.boolean_mask(flat_ids, is_id_valid) - example_ids = array_ops.boolean_mask(example_ids, is_id_valid) - weight_tensor = array_ops.boolean_mask(weight_tensor, is_id_valid) - - projection_length = math_ops.reduce_max(flat_ids) + 1 - # project ids based on example ids so that we can dedup ids that - # occur multiple times for a single example. - projected_ids = projection_length * example_ids + flat_ids - - # Remove any redundant ids. - ids, idx = array_ops.unique(projected_ids) - # Keep only one example id per duplicated ids. - example_ids_filtered = math_ops.unsorted_segment_min( - example_ids, idx, - array_ops.shape(ids)[0]) - - # reproject ids back feature id space. - reproject_ids = (ids - projection_length * example_ids_filtered) - - weights = array_ops.reshape( - math_ops.unsorted_segment_sum(weight_tensor, idx, - array_ops.shape(ids)[0]), [-1]) - sparse_feature_with_values.append( - SparseFeatureColumn(example_ids_filtered, reproject_ids, weights)) - # If a partitioner was used during variable creation, we will have a - # list of Variables here larger than 1. - vars_to_append = columns_to_variables[column][0] - if len(columns_to_variables[column]) > 1: - vars_to_append = columns_to_variables[column] - sparse_feature_with_values_weights.append(vars_to_append) - else: - raise ValueError('SDCAOptimizer does not support column type %s.' % - type(column).__name__) - - example_weights = array_ops.reshape( - features[weight_column_name], - shape=[-1]) if weight_column_name else array_ops.ones([batch_size]) - example_ids = features[self._example_id_column] - sparse_feature_with_values.extend(sparse_features) - sparse_feature_with_values_weights.extend(sparse_feature_weights) - examples = dict( - sparse_features=sparse_feature_with_values, - dense_features=dense_features, - example_labels=math_ops.cast( - array_ops.reshape(targets, shape=[-1]), dtypes.float32), - example_weights=example_weights, - example_ids=example_ids) - sdca_variables = dict( - sparse_features_weights=sparse_feature_with_values_weights, - dense_features_weights=dense_feature_weights) - return examples, sdca_variables - - training_examples, training_variables = _training_examples_and_variables() - sdca_model = sdca_ops.SdcaModel( - examples=training_examples, - variables=training_variables, - options=dict( - symmetric_l1_regularization=self._symmetric_l1_regularization, - symmetric_l2_regularization=self._symmetric_l2_regularization, - adaptive=self._adaptive, - num_loss_partitions=self._num_loss_partitions, - num_table_shards=self._num_table_shards, - loss_type=loss_type)) - train_op = sdca_model.minimize(global_step=global_step) - return sdca_model, train_op diff --git a/tensorflow/contrib/lookup/BUILD b/tensorflow/contrib/lookup/BUILD deleted file mode 100644 index 111b9ea76a1..00000000000 --- a/tensorflow/contrib/lookup/BUILD +++ /dev/null @@ -1,52 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -load("//tensorflow:tensorflow.bzl", "tf_py_test") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -# TODO(yleon): Refactor after one we switching to the V2 kernels. -py_library( - name = "lookup_py", - srcs = [ - "__init__.py", - "lookup_ops.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:lookup_ops", - "//tensorflow/python:lookup_ops_gen", - "//tensorflow/python:training", - "//tensorflow/python:util", - ], -) - -tf_py_test( - name = "lookup_ops_test", - size = "medium", - srcs = ["lookup_ops_test.py"], - additional_deps = [ - ":lookup_py", - "//third_party/py/numpy", - "@six_archive//:six", - "//tensorflow/contrib/data", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:lookup_ops", - "//tensorflow/python:session", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], - grpc_enabled = True, - tags = ["no_windows"], # TODO: needs investigation on Windows -) diff --git a/tensorflow/contrib/lookup/__init__.py b/tensorflow/contrib/lookup/__init__.py deleted file mode 100644 index 17eafeb2dad..00000000000 --- a/tensorflow/contrib/lookup/__init__.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for lookup operations. - -@@string_to_index -@@string_to_index_table_from_file -@@string_to_index_table_from_tensor -@@index_table_from_file -@@index_table_from_tensor -@@index_to_string -@@index_to_string_table_from_file -@@index_to_string_table_from_tensor -@@LookupInterface -@@InitializableLookupTableBase -@@IdTableWithHashBuckets -@@HashTable -@@MutableHashTable -@@MutableDenseHashTable -@@TableInitializerBase -@@KeyValueTensorInitializer -@@TextFileIndex -@@TextFileInitializer -@@TextFileIdTableInitializer -@@TextFileStringTableInitializer - -@@HasherSpec -@@StrongHashSpec -@@FastHashSpec -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.lookup.lookup_ops import * -# pylint: enable=unused-import,wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/lookup/lookup_ops.py b/tensorflow/contrib/lookup/lookup_ops.py deleted file mode 100644 index 60aaae7e4c9..00000000000 --- a/tensorflow/contrib/lookup/lookup_ops.py +++ /dev/null @@ -1,376 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Lookup table operations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import gen_lookup_ops -from tensorflow.python.ops import lookup_ops -# pylint: disable=unused-import -from tensorflow.python.ops.lookup_ops import FastHashSpec -from tensorflow.python.ops.lookup_ops import HasherSpec -from tensorflow.python.ops.lookup_ops import IdTableWithHashBuckets -from tensorflow.python.ops.lookup_ops import index_table_from_file -from tensorflow.python.ops.lookup_ops import index_to_string_table_from_file -from tensorflow.python.ops.lookup_ops import InitializableLookupTableBase -from tensorflow.python.ops.lookup_ops import InitializableLookupTableBaseV1 -from tensorflow.python.ops.lookup_ops import KeyValueTensorInitializer -from tensorflow.python.ops.lookup_ops import LookupInterface -from tensorflow.python.ops.lookup_ops import StrongHashSpec -from tensorflow.python.ops.lookup_ops import TableInitializerBase -from tensorflow.python.ops.lookup_ops import TextFileIdTableInitializer -from tensorflow.python.ops.lookup_ops import TextFileIndex -from tensorflow.python.ops.lookup_ops import TextFileInitializer -from tensorflow.python.ops.lookup_ops import TextFileStringTableInitializer -# pylint: enable=unused-import -from tensorflow.python.util.deprecation import deprecated - - -@deprecated("2017-04-10", "Use `index_table_from_file`.") -def string_to_index_table_from_file(vocabulary_file=None, - num_oov_buckets=0, - vocab_size=None, - default_value=-1, - hasher_spec=FastHashSpec, - name=None): - return index_table_from_file( - vocabulary_file, - num_oov_buckets, - vocab_size, - default_value, - hasher_spec, - key_dtype=dtypes.string, - name=name) - - -@deprecated("2017-04-10", "Use `index_table_from_tensor`.") -def string_to_index_table_from_tensor(mapping, - num_oov_buckets=0, - default_value=-1, - hasher_spec=FastHashSpec, - name=None): - with ops.name_scope(name, "string_to_index") as scope: - mapping = ops.convert_to_tensor(mapping) - if dtypes.string != mapping.dtype.base_dtype: - raise ValueError("string_to_index_table_from_tensor requires string.") - return index_table_from_tensor( - mapping, num_oov_buckets, default_value, hasher_spec, name=scope) - - -def index_table_from_tensor(mapping, - num_oov_buckets=0, - default_value=-1, - hasher_spec=FastHashSpec, - dtype=dtypes.string, - name=None): - """Returns a lookup table that converts a string tensor into int64 IDs. - - This operation constructs a lookup table to convert tensor of strings into - int64 IDs. The mapping can be initialized from a string `mapping` 1-D tensor - where each element is a key and corresponding index within the tensor is the - value. - - Any lookup of an out-of-vocabulary token will return a bucket ID based on its - hash if `num_oov_buckets` is greater than zero. Otherwise it is assigned the - `default_value`. - The bucket ID range is `[mapping size, mapping size + num_oov_buckets - 1]`. - - The underlying table must be initialized by calling - `session.run(tf.compat.v1.tables_initializer)` or `session.run(table.init)` - once. - - Elements in `mapping` cannot have duplicates, otherwise when executing the - table initializer op, it will throw a `FailedPreconditionError`. - - Sample Usages: - - ```python - mapping_strings = tf.constant(["emerson", "lake", "palmer"]) - table = tf.contrib.lookup.index_table_from_tensor( - mapping=mapping_strings, num_oov_buckets=1, default_value=-1) - features = tf.constant(["emerson", "lake", "and", "palmer"]) - ids = table.lookup(features) - ... - tf.compat.v1.tables_initializer().run() - - ids.eval() ==> [0, 1, 3, 2] - ``` - - Args: - mapping: A 1-D `Tensor` that specifies the mapping of keys to indices. The - type of this object must be castable to `dtype`. - num_oov_buckets: The number of out-of-vocabulary buckets. - default_value: The value to use for out-of-vocabulary feature values. - Defaults to -1. - hasher_spec: A `HasherSpec` to specify the hash function to use for - assignment of out-of-vocabulary buckets. - dtype: The type of values passed to `lookup`. Only string and integers are - supported. - name: A name for this op (optional). - - Returns: - The lookup table to map an input `Tensor` to index `int64` `Tensor`. - - Raises: - ValueError: If `mapping` is invalid. - ValueError: If `num_oov_buckets` is negative. - """ - if mapping is None: - raise ValueError("mapping must be specified.") - return lookup_ops.index_table_from_tensor( - vocabulary_list=mapping, - num_oov_buckets=num_oov_buckets, - default_value=default_value, - hasher_spec=hasher_spec, - dtype=dtype, - name=name) - - -@deprecated("2017-01-07", "This op will be removed after the deprecation date. " - "Please switch to index_table_from_tensor and call the lookup " - "method of the returned table.") -def string_to_index(tensor, mapping, default_value=-1, name=None): - """Maps `tensor` of strings into `int64` indices based on `mapping`. - - This operation converts `tensor` of strings into `int64` indices. - The mapping is initialized from a string `mapping` tensor where each element - is a key and corresponding index within the tensor is the value. - - Any entry in the input which does not have a corresponding entry in 'mapping' - (an out-of-vocabulary entry) is assigned the `default_value` - - Elements in `mapping` cannot be duplicated, otherwise the initialization - will throw a FailedPreconditionError. - - The underlying table must be initialized by calling - `session.run(tf.compat.v1.tables_initializer)` once. - - For example: - - ```python - mapping_strings = tf.constant(["emerson", "lake", "palmer"]) - feats = tf.constant(["emerson", "lake", "and", "palmer"]) - ids = tf.contrib.lookup.string_to_index( - feats, mapping=mapping_strings, default_value=-1) - ... - tf.compat.v1.tables_initializer().run() - - ids.eval() ==> [0, 1, -1, 2] - ``` - - Args: - tensor: A 1-D input `Tensor` with the strings to map to indices. - mapping: A 1-D string `Tensor` that specifies the mapping of strings to - indices. - default_value: The `int64` value to use for out-of-vocabulary strings. - Defaults to -1. - name: A name for this op (optional). - - Returns: - The mapped indices. It has the same shape and tensor type (dense or sparse) - as `tensor`. - """ - table = index_table_from_tensor( - mapping=mapping, default_value=default_value, name=name) - return table.lookup(tensor) - - -def index_to_string_table_from_tensor(mapping, default_value="UNK", name=None): - """Returns a lookup table that maps a `Tensor` of indices into strings. - - This operation constructs a lookup table to map int64 indices into string - values. The mapping is initialized from a string `mapping` 1-D `Tensor` where - each element is a value and the corresponding index within the tensor is the - key. - - Any input which does not have a corresponding index in 'mapping' - (an out-of-vocabulary entry) is assigned the `default_value` - - The underlying table must be initialized by calling - `session.run(tf.compat.v1.tables_initializer)` or `session.run(table.init)` - once. - - Elements in `mapping` cannot have duplicates, otherwise when executing the - table initializer op, it will throw a `FailedPreconditionError`. - - Sample Usages: - - ```python - mapping_string = tf.constant(["emerson", "lake", "palmer"]) - indices = tf.constant([1, 5], tf.int64) - table = tf.contrib.lookup.index_to_string_table_from_tensor( - mapping_string, default_value="UNKNOWN") - values = table.lookup(indices) - ... - tf.compat.v1.tables_initializer().run() - - values.eval() ==> ["lake", "UNKNOWN"] - ``` - - Args: - mapping: A 1-D string `Tensor` that specifies the strings to map from - indices. - default_value: The value to use for out-of-vocabulary indices. - name: A name for this op (optional). - - Returns: - The lookup table to map a string values associated to a given index `int64` - `Tensors`. - - Raises: - ValueError: when `mapping` is not set. - """ - - if mapping is None: - raise ValueError("mapping must be specified.") - - return lookup_ops.index_to_string_table_from_tensor( - vocabulary_list=mapping, default_value=default_value, name=name) - - -@deprecated( - "2017-01-07", "This op will be removed after the deprecation date. " - "Please switch to index_to_string_table_from_tensor and call the lookup " - "method of the returned table.") -def index_to_string(tensor, mapping, default_value="UNK", name=None): - """Maps `tensor` of indices into string values based on `mapping`. - - This operation converts `int64` indices into string values. The mapping is - initialized from a string `mapping` tensor where each element is a value and - the corresponding index within the tensor is the key. - - Any input which does not have a corresponding index in 'mapping' - (an out-of-vocabulary entry) is assigned the `default_value` - - The underlying table must be initialized by calling - `session.run(tf.compat.v1.tables_initializer)` once. - - For example: - - ```python - mapping_string = tf.constant(["emerson", "lake", "palmer"]) - indices = tf.constant([1, 5], tf.int64) - values = tf.contrib.lookup.index_to_string( - indices, mapping=mapping_string, default_value="UNKNOWN") - ... - tf.compat.v1.tables_initializer().run() - - values.eval() ==> ["lake", "UNKNOWN"] - ``` - - Args: - tensor: A `int64` `Tensor` with the indices to map to strings. - mapping: A 1-D string `Tensor` that specifies the strings to map from - indices. - default_value: The string value to use for out-of-vocabulary indices. - name: A name for this op (optional). - - Returns: - The strings values associated to the indices. The resultant dense - feature value tensor has the same shape as the corresponding `indices`. - """ - table = index_to_string_table_from_tensor( - mapping=mapping, default_value=default_value, name=name) - return table.lookup(tensor) - - -class HashTable(InitializableLookupTableBaseV1): - """A generic hash table implementation. - - Example usage: - - ```python - table = tf.HashTable( - tf.KeyValueTensorInitializer(keys, values), -1) - out = table.lookup(input_tensor) - table.init.run() - print(out.eval()) - ``` - """ - - def __init__(self, initializer, default_value, shared_name=None, name=None): - """Creates a non-initialized `HashTable` object. - - Creates a table, the type of its keys and values are specified by the - initializer. - Before using the table you will have to initialize it. After initialization - the table will be immutable. - - Args: - initializer: The table initializer to use. See `HashTable` kernel for - supported key and value types. - default_value: The value to use if a key is missing in the table. - shared_name: If non-empty, this table will be shared under the given name - across multiple sessions. - name: A name for the operation (optional). - - Returns: - A `HashTable` object. - """ - self._initializer = initializer - self._default_value = default_value - self._shared_name = shared_name - self._name = name or "hash_table" - self._table_name = None - super(HashTable, self).__init__(default_value, initializer) - self._value_shape = self._default_value.get_shape() - - def _create_resource(self): - table_ref = gen_lookup_ops.hash_table_v2( - shared_name=self._shared_name, - key_dtype=self._initializer.key_dtype, - value_dtype=self._initializer.value_dtype, - name=self._name) - if context.executing_eagerly(): - self._table_name = None - else: - self._table_name = table_ref.op.name.split("/")[-1] - return table_ref - - @property - def init(self): - return self.initializer - - @property - def name(self): - return self._table_name - - def export(self, name=None): - """Returns tensors of all keys and values in the table. - - Args: - name: A name for the operation (optional). - - Returns: - A pair of tensors with the first tensor containing all keys and the - second tensors containing all values in the table. - """ - with ops.name_scope(name, "%s_Export" % self.name, - [self.resource_handle]) as name: - exported_keys, exported_values = gen_lookup_ops.lookup_table_export_v2( - self.resource_handle, self._key_dtype, self._value_dtype, name=name) - - exported_values.set_shape(exported_keys.get_shape().concatenate( - self._value_shape)) - return exported_keys, exported_values - - -MutableHashTable = lookup_ops.MutableHashTable -MutableDenseHashTable = lookup_ops.DenseHashTable diff --git a/tensorflow/contrib/lookup/lookup_ops_test.py b/tensorflow/contrib/lookup/lookup_ops_test.py deleted file mode 100644 index ae5fcf8182b..00000000000 --- a/tensorflow/contrib/lookup/lookup_ops_test.py +++ /dev/null @@ -1,1482 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.contrib.lookup.lookup.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import numpy as np - -from tensorflow.contrib import lookup -from tensorflow.python.client import session -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import server_lib - - -class HashTableOpTest(test.TestCase): - - def testHashTable(self): - with self.cached_session(): - default_val = -1 - keys = constant_op.constant(["brain", "salad", "surgery"]) - values = constant_op.constant([0, 1, 2], dtypes.int64) - table = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), default_val) - table.initializer.run() - - self.assertAllEqual(3, table.size().eval()) - - input_string = constant_op.constant(["brain", "salad", "tank"]) - output = table.lookup(input_string) - self.assertAllEqual([3], output.get_shape()) - - result = output.eval() - self.assertAllEqual([0, 1, -1], result) - - exported_keys_tensor, exported_values_tensor = table.export() - - self.assertItemsEqual([b"brain", b"salad", b"surgery"], - exported_keys_tensor.eval()) - self.assertItemsEqual([0, 1, 2], exported_values_tensor.eval()) - - def testHashTableFindHighRank(self): - with self.cached_session(): - default_val = -1 - keys = constant_op.constant(["brain", "salad", "surgery"]) - values = constant_op.constant([0, 1, 2], dtypes.int64) - table = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), default_val) - table.initializer.run() - - self.assertAllEqual(3, table.size().eval()) - - input_string = constant_op.constant( - [["brain", "salad"], ["tank", "tarkus"]]) - output = table.lookup(input_string) - - result = output.eval() - self.assertAllEqual([[0, 1], [-1, -1]], result) - - def testHashTableInitWithPythonArrays(self): - with self.cached_session(): - default_val = -1 - keys = ["brain", "salad", "surgery"] - values = [0, 1, 2] - table = lookup.HashTable( - lookup.KeyValueTensorInitializer( - keys, values, value_dtype=dtypes.int64), - default_val) - table.initializer.run() - - self.assertAllEqual(3, table.size().eval()) - - input_string = constant_op.constant(["brain", "salad", "tank"]) - output = table.lookup(input_string) - - result = output.eval() - self.assertAllEqual([0, 1, -1], result) - - def testHashTableInitWithNumPyArrays(self): - with self.cached_session(): - default_val = -1 - keys = np.array(["brain", "salad", "surgery"], dtype=np.str) - values = np.array([0, 1, 2], dtype=np.int64) - table = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), default_val) - table.initializer.run() - - self.assertAllEqual(3, table.size().eval()) - - input_string = constant_op.constant(["brain", "salad", "tank"]) - output = table.lookup(input_string) - - result = output.eval() - self.assertAllEqual([0, 1, -1], result) - - def testMultipleHashTables(self): - with self.cached_session() as sess: - default_val = -1 - keys = constant_op.constant(["brain", "salad", "surgery"]) - values = constant_op.constant([0, 1, 2], dtypes.int64) - - table1 = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), default_val) - table2 = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), default_val) - table3 = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), default_val) - - lookup_ops.tables_initializer().run() - self.assertAllEqual(3, table1.size().eval()) - self.assertAllEqual(3, table2.size().eval()) - self.assertAllEqual(3, table3.size().eval()) - - input_string = constant_op.constant(["brain", "salad", "tank"]) - output1 = table1.lookup(input_string) - output2 = table2.lookup(input_string) - output3 = table3.lookup(input_string) - - out1, out2, out3 = sess.run([output1, output2, output3]) - self.assertAllEqual([0, 1, -1], out1) - self.assertAllEqual([0, 1, -1], out2) - self.assertAllEqual([0, 1, -1], out3) - - def testHashTableWithTensorDefault(self): - with self.cached_session(): - default_val = constant_op.constant(-1, dtypes.int64) - keys = constant_op.constant(["brain", "salad", "surgery"]) - values = constant_op.constant([0, 1, 2], dtypes.int64) - table = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), default_val) - table.initializer.run() - - input_string = constant_op.constant(["brain", "salad", "tank"]) - output = table.lookup(input_string) - - result = output.eval() - self.assertAllEqual([0, 1, -1], result) - - def testHashTableWithSparseTensorInput(self): - with self.cached_session() as sess: - default_val = constant_op.constant(-1, dtypes.int64) - keys = constant_op.constant(["brain", "salad", "surgery"]) - values = constant_op.constant([0, 1, 2], dtypes.int64) - table = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), default_val) - table.initializer.run() - - sp_indices = [[0, 0], [0, 1], [1, 0]] - sp_shape = [2, 2] - input_tensor = sparse_tensor.SparseTensor( - constant_op.constant(sp_indices, dtypes.int64), - constant_op.constant(["brain", "salad", "tank"]), - constant_op.constant(sp_shape, dtypes.int64)) - output = table.lookup(input_tensor) - - out_indices, out_values, out_shape = sess.run(output) - - self.assertAllEqual([0, 1, -1], out_values) - self.assertAllEqual(sp_indices, out_indices) - self.assertAllEqual(sp_shape, out_shape) - - def testSignatureMismatch(self): - with self.cached_session(): - default_val = -1 - keys = constant_op.constant(["brain", "salad", "surgery"]) - values = constant_op.constant([0, 1, 2], dtypes.int64) - table = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), default_val) - table.initializer.run() - - # Ref types do not produce a lookup signature mismatch. - input_string_ref = variables.Variable("brain") - variables.global_variables_initializer().run() - self.assertEqual(0, table.lookup(input_string_ref).eval()) - - input_string = constant_op.constant([1, 2, 3], dtypes.int64) - with self.assertRaises(TypeError): - table.lookup(input_string) - - with self.assertRaises(TypeError): - lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), "UNK") - - def testDTypes(self): - with self.cached_session(): - default_val = -1 - with self.assertRaises(TypeError): - lookup.HashTable( - lookup.KeyValueTensorInitializer(["a"], [1], [dtypes.string], - dtypes.int64), default_val) - - def testNotInitialized(self): - with self.cached_session(): - default_val = -1 - table = lookup.HashTable( - lookup.KeyValueTensorInitializer( - ["a"], [1], value_dtype=dtypes.int64), - default_val) - - input_string = constant_op.constant(["brain", "salad", "surgery"]) - output = table.lookup(input_string) - - with self.assertRaisesOpError("Table not initialized"): - output.eval() - - def testInitializeTwice(self): - with self.cached_session(): - default_val = -1 - keys = constant_op.constant(["brain", "salad", "surgery"]) - values = constant_op.constant([0, 1, 2], dtypes.int64) - table = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), default_val) - table.initializer.run() - # Re-initializing should not throw an error. - table.initializer.run() - - def testInitializationWithInvalidDimensions(self): - with self.cached_session(): - default_val = -1 - keys = constant_op.constant(["brain", "salad", "surgery"]) - values = constant_op.constant([0, 1, 2, 3, 4], dtypes.int64) - - with self.assertRaises(ValueError): - lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), default_val) - - def testMultipleSessions(self): - # Start a server - server = server_lib.Server( - { - "local0": ["localhost:0"] - }, protocol="grpc", start=True) - # Create two sessions sharing the same state - session1 = session.Session(server.target) - session2 = session.Session(server.target) - - default_val = -1 - keys = constant_op.constant(["brain", "salad", "surgery"]) - values = constant_op.constant([0, 1, 2], dtypes.int64) - table = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), - default_val, - name="t1") - - # Init the table in the first session. - with session1: - table.initializer.run() - self.assertAllEqual(3, table.size().eval()) - - # Init the table in the second session and verify that we do not get a - # "Table already initialized" error. - with session2: - table.initializer.run() - self.assertAllEqual(3, table.size().eval()) - - def testHashTableInt32String(self): - with self.cached_session(): - default_val = "n/a" - keys = constant_op.constant([0, 1, 2], dtypes.int32) - values = constant_op.constant(["brain", "salad", "surgery"]) - table = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), default_val) - table.initializer.run() - - input_tensor = constant_op.constant([0, 1, -1]) - output = table.lookup(input_tensor) - - result = output.eval() - self.assertAllEqual([b"brain", b"salad", b"n/a"], result) - - -class IndexTableFromFile(test.TestCase): - - def _createVocabFile(self, basename, values=("brain", "salad", "surgery")): - vocabulary_file = os.path.join(self.get_temp_dir(), basename) - with open(vocabulary_file, "w") as f: - f.write("\n".join(values) + "\n") - return vocabulary_file - - def test_string_index_table_from_file(self): - vocabulary_file = self._createVocabFile("f2i_vocab1.txt") - with self.cached_session(): - table = lookup.index_table_from_file( - vocabulary_file=vocabulary_file, num_oov_buckets=1) - ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) - - self.assertRaises(errors_impl.OpError, ids.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) - - def test_string_index_table_from_file_tensor_filename(self): - vocabulary_file = self._createVocabFile("f2i_vocab1.txt") - with self.cached_session(): - vocabulary_file = constant_op.constant(vocabulary_file) - table = lookup.index_table_from_file( - vocabulary_file=vocabulary_file, num_oov_buckets=1) - ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) - - self.assertRaises(errors_impl.OpError, ids.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) - self.assertEqual(1, - len(ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS))) - - def test_string_index_table_from_file_placeholder_filename(self): - vocabulary_file = self._createVocabFile("f2i_vocab1.txt") - with self.cached_session(): - vocabulary_placeholder = array_ops.placeholder(dtypes.string, []) - table = lookup.index_table_from_file( - vocabulary_file=vocabulary_placeholder, num_oov_buckets=1) - ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) - - self.assertRaises(errors_impl.OpError, ids.eval) - - feed_dict = {vocabulary_placeholder.name: vocabulary_file} - lookup_ops.tables_initializer().run(feed_dict=feed_dict) - self.assertAllEqual((1, 2, 3), ids.eval()) - self.assertEqual(0, - len(ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS))) - - def test_int32_index_table_from_file(self): - vocabulary_file = self._createVocabFile( - "f2i_vocab2.txt", values=("42", "1", "-1000")) - with self.cached_session(): - table = lookup.index_table_from_file( - vocabulary_file=vocabulary_file, num_oov_buckets=1, - key_dtype=dtypes.int32) - ids = table.lookup( - constant_op.constant((1, -1000, 11), dtype=dtypes.int32)) - - self.assertRaises(errors_impl.OpError, ids.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) - - def test_int64_index_table_from_file(self): - vocabulary_file = self._createVocabFile( - "f2i_vocab3.txt", values=("42", "1", "-1000")) - with self.cached_session(): - table = lookup.index_table_from_file( - vocabulary_file=vocabulary_file, num_oov_buckets=1, - key_dtype=dtypes.int64) - ids = table.lookup( - constant_op.constant((1, -1000, 11), dtype=dtypes.int64)) - - self.assertRaises(errors_impl.OpError, ids.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) - - def test_index_table_from_file_with_default_value(self): - default_value = -42 - vocabulary_file = self._createVocabFile("f2i_vocab4.txt") - with self.cached_session(): - table = lookup.index_table_from_file( - vocabulary_file=vocabulary_file, default_value=default_value) - ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) - - self.assertRaises(errors_impl.OpError, ids.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, default_value), ids.eval()) - - def test_index_table_from_file_with_oov_buckets(self): - vocabulary_file = self._createVocabFile("f2i_vocab5.txt") - with self.cached_session(): - table = lookup.index_table_from_file( - vocabulary_file=vocabulary_file, num_oov_buckets=1000) - ids = table.lookup( - constant_op.constant(["salad", "surgery", "tarkus", "toccata"])) - - self.assertRaises(errors_impl.OpError, ids.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual( - ( - 1, # From vocabulary file. - 2, # From vocabulary file. - 867, # 3 + fingerprint("tarkus") mod 300. - 860), # 3 + fingerprint("toccata") mod 300. - ids.eval()) - - def test_index_table_from_file_fails_with_empty_vocabulary_file_name(self): - self.assertRaises( - ValueError, - lookup.index_table_from_file, - vocabulary_file="") - - def test_index_table_from_file_fails_with_empty_vocabulary(self): - self.assertRaises( - ValueError, - lookup.index_table_from_file, - vocabulary_file=None) - - def test_index_table_from_file_with_vocab_size_too_small(self): - vocabulary_file = self._createVocabFile("f2i_vocab6.txt") - with self.cached_session(): - table = lookup.index_table_from_file( - vocabulary_file=vocabulary_file, vocab_size=2) - ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) - - self.assertRaises(errors_impl.OpError, ids.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((1, -1, -1), ids.eval()) - self.assertEqual(2, table.size().eval()) - - def test_index_table_from_file_with_vocab_size_too_large(self): - vocabulary_file = self._createVocabFile("f2i_vocab7.txt") - with self.cached_session(): - table = lookup.index_table_from_file( - vocabulary_file=vocabulary_file, vocab_size=4) - self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - "Invalid vocab_size", table.initializer.run) - - def test_index_table_from_file_with_vocab_size(self): - vocabulary_file = self._createVocabFile("f2i_vocab8.txt") - - self.assertRaises( - ValueError, - lookup.index_table_from_file, - vocabulary_file=vocabulary_file, - vocab_size=0) - - with self.cached_session(): - table = lookup.index_table_from_file( - vocabulary_file=vocabulary_file, vocab_size=3) - ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) - - self.assertRaises(errors_impl.OpError, ids.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, -1), ids.eval()) - self.assertEqual(3, table.size().eval()) - - def test_index_table_from_file_with_invalid_hashers(self): - vocabulary_file = self._createVocabFile("invalid_hasher.txt") - with self.cached_session(): - with self.assertRaises(TypeError): - lookup.index_table_from_file( - vocabulary_file=vocabulary_file, - vocab_size=3, - num_oov_buckets=1, - hasher_spec=1) - - table = lookup.index_table_from_file( - vocabulary_file=vocabulary_file, - vocab_size=3, - num_oov_buckets=1, - hasher_spec=lookup.HasherSpec("my-awesome-hash", None)) - - self.assertRaises(ValueError, table.lookup, - constant_op.constant(["salad", "surgery", "tarkus"])) - - -class KeyValueTensorInitializerTest(test.TestCase): - - def test_string(self): - with ops.Graph().as_default(), self.cached_session(): - init = lookup.KeyValueTensorInitializer( - ("brain", "salad", "surgery"), (0, 1, 2), dtypes.string, dtypes.int64) - table = lookup.HashTable(init, default_value=-1) - table.initializer.run() - - def test_int64(self): - with ops.Graph().as_default(), self.cached_session(): - init = lookup.KeyValueTensorInitializer( - (42, 1, -1000), (0, 1, 2), dtypes.int64, dtypes.int64) - table = lookup.HashTable(init, default_value=-1) - table.initializer.run() - - def test_int32(self): - with ops.Graph().as_default(), self.cached_session(): - init = lookup.KeyValueTensorInitializer( - (42, 1, -1000), (0, 1, 2), dtypes.int32, dtypes.int64) - table = lookup.HashTable(init, default_value=-1) - with self.assertRaisesRegexp( - errors_impl.OpError, "No OpKernel was registered"): - table.initializer.run() - - -class IndexTableFromTensor(test.TestCase): - - @test_util.run_in_graph_and_eager_modes - def test_index_table_from_tensor_with_tensor_init(self): - table = lookup.index_table_from_tensor( - mapping=("brain", "salad", "surgery"), num_oov_buckets=1) - - if not context.executing_eagerly(): - with self.assertRaises(errors_impl.OpError): - self.evaluate(table.lookup( - constant_op.constant(("salad", "surgery", "tarkus")))) - else: - # Reinitializing a table in eager should work. - table = lookup.index_table_from_tensor( - mapping=("brain", "salad", "surgery"), num_oov_buckets=1) - self.evaluate(lookup_ops.tables_initializer()) - ids = table.lookup(constant_op.constant(("salad", "surgery", "tarkus"))) - self.assertAllEqual((1, 2, 3), self.evaluate(ids)) - - def test_int32_index_table_from_tensor_with_tensor_init(self): - with self.cached_session(): - table = lookup.index_table_from_tensor( - mapping=(42, 1, -1000), num_oov_buckets=1, dtype=dtypes.int32) - ids = table.lookup( - constant_op.constant((1, -1000, 11), dtype=dtypes.int32)) - - self.assertRaises(errors_impl.OpError, ids.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) - - def test_int64_index_table_from_tensor_with_tensor_init(self): - with self.cached_session(): - table = lookup.index_table_from_tensor( - mapping=(42, 1, -1000), num_oov_buckets=1, dtype=dtypes.int64) - ids = table.lookup( - constant_op.constant((1, -1000, 11), dtype=dtypes.int64)) - - self.assertRaises(errors_impl.OpError, ids.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) - - def test_index_table_from_tensor_with_default_value(self): - default_value = -42 - with self.cached_session(): - table = lookup.index_table_from_tensor( - mapping=["brain", "salad", "surgery"], default_value=default_value) - ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) - - self.assertRaises(errors_impl.OpError, ids.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, default_value), ids.eval()) - - def test_index_table_from_tensor_missing_mapping(self): - with self.cached_session(): - with self.assertRaisesRegexp(ValueError, "mapping must be specified"): - lookup.index_table_from_tensor(mapping=None, num_oov_buckets=1) - - def test_index_table_from_tensor_empty_mapping(self): - with self.cached_session(): - table = lookup.index_table_from_tensor( - mapping=np.array([], dtype=np.str_), num_oov_buckets=1) - ids = table.lookup(constant_op.constant(["salad", "surgery", "brain"])) - self.assertRaises(errors_impl.OpError, ids.eval) - with self.assertRaisesRegexp( - errors_impl.OpError, "keys and values cannot be empty"): - lookup_ops.tables_initializer().run() - - def test_index_table_from_tensor_with_invalid_hashers(self): - with self.cached_session(): - with self.assertRaises(TypeError): - lookup.index_table_from_tensor( - mapping=["brain", "salad", "surgery"], - num_oov_buckets=1, - hasher_spec=1) - - table = lookup.index_table_from_tensor( - mapping=["brain", "salad", "surgery"], - num_oov_buckets=1, - hasher_spec=lookup.HasherSpec("my-awesome-hash", None)) - - self.assertRaises(ValueError, table.lookup, - constant_op.constant(["salad", "surgery", "tarkus"])) - - -class StringToIndexTest(test.TestCase): - - def test_string_to_index(self): - with self.cached_session(): - mapping_strings = constant_op.constant(["brain", "salad", "surgery"]) - feats = constant_op.constant(["salad", "surgery", "tarkus"]) - indices = lookup.string_to_index(feats, mapping=mapping_strings) - - self.assertRaises(errors_impl.OpError, indices.eval) - lookup_ops.tables_initializer().run() - - self.assertAllEqual((1, 2, -1), indices.eval()) - - def test_duplicate_entries(self): - with self.cached_session(): - mapping_strings = constant_op.constant(["hello", "hello"]) - feats = constant_op.constant(["hello", "hola"]) - _ = lookup.string_to_index(feats, mapping=mapping_strings) - - self.assertRaises(errors_impl.OpError, - lookup_ops.tables_initializer().run) - - def test_string_to_index_with_default_value(self): - default_value = -42 - with self.cached_session(): - mapping_strings = constant_op.constant(["brain", "salad", "surgery"]) - feats = constant_op.constant(["salad", "surgery", "tarkus"]) - indices = lookup.string_to_index( - feats, mapping=mapping_strings, default_value=default_value) - self.assertRaises(errors_impl.OpError, indices.eval) - - lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, default_value), indices.eval()) - - -class IndexToStringTableFromFileTest(test.TestCase): - - def _createVocabFile(self, basename): - vocabulary_file = os.path.join(self.get_temp_dir(), basename) - with open(vocabulary_file, "w") as f: - f.write("\n".join(["brain", "salad", "surgery"]) + "\n") - return vocabulary_file - - def test_index_to_string_table(self): - vocabulary_file = self._createVocabFile("i2f_vocab1.txt") - with self.cached_session(): - table = lookup.index_to_string_table_from_file( - vocabulary_file=vocabulary_file) - features = table.lookup(constant_op.constant([0, 1, 2, 3], dtypes.int64)) - self.assertRaises(errors_impl.OpError, features.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"), - features.eval()) - - def test_index_to_string_table_with_default_value(self): - default_value = b"NONE" - vocabulary_file = self._createVocabFile("f2i_vocab2.txt") - with self.cached_session(): - table = lookup.index_to_string_table_from_file( - vocabulary_file=vocabulary_file, default_value=default_value) - features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64)) - self.assertRaises(errors_impl.OpError, features.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((b"salad", b"surgery", default_value), - features.eval()) - - def test_index_to_string_table_with_vocab_size_too_small(self): - default_value = b"NONE" - vocabulary_file = self._createVocabFile("f2i_vocab2.txt") - with self.cached_session(): - table = lookup.index_to_string_table_from_file( - vocabulary_file=vocabulary_file, - vocab_size=2, - default_value=default_value) - features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64)) - self.assertRaises(errors_impl.OpError, features.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((b"salad", default_value, default_value), - features.eval()) - - def test_index_to_string_table_with_vocab_size_too_large(self): - vocabulary_file = self._createVocabFile("f2i_vocab6.txt") - with self.cached_session(): - table = lookup.index_to_string_table_from_file( - vocabulary_file=vocabulary_file, vocab_size=4) - features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64)) - - self.assertRaises(errors_impl.OpError, features.eval) - init = lookup_ops.tables_initializer() - self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - "Invalid vocab_size", init.run) - - def test_index_to_string_table_with_vocab_size(self): - vocabulary_file = self._createVocabFile("f2i_vocab7.txt") - with self.cached_session(): - table = lookup.index_to_string_table_from_file( - vocabulary_file=vocabulary_file, vocab_size=3) - features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64)) - - self.assertRaises(errors_impl.OpError, features.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((b"salad", b"surgery", b"UNK"), features.eval()) - - -class IndexToStringTableFromTensorTest(test.TestCase): - - def test_index_to_string_table_from_tensor(self): - with self.cached_session(): - mapping_strings = constant_op.constant(["brain", "salad", "surgery"]) - table = lookup.index_to_string_table_from_tensor( - mapping=mapping_strings) - - indices = constant_op.constant([0, 1, 2, 3], dtypes.int64) - features = table.lookup(indices) - self.assertRaises(errors_impl.OpError, features.eval) - lookup_ops.tables_initializer().run() - - self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"), - features.eval()) - - def test_duplicate_entries(self): - with self.cached_session(): - mapping_strings = constant_op.constant(["hello", "hello"]) - table = lookup.index_to_string_table_from_tensor( - mapping=mapping_strings) - indices = constant_op.constant([0, 1, 4], dtypes.int64) - features = table.lookup(indices) - lookup_ops.tables_initializer().run() - self.assertAllEqual((b"hello", b"hello", b"UNK"), features.eval()) - - def test_index_to_string_with_default_value(self): - default_value = b"NONE" - with self.cached_session(): - mapping_strings = constant_op.constant(["brain", "salad", "surgery"]) - table = lookup.index_to_string_table_from_tensor( - mapping=mapping_strings, default_value=default_value) - indices = constant_op.constant([1, 2, 4], dtypes.int64) - features = table.lookup(indices) - self.assertRaises(errors_impl.OpError, features.eval) - - lookup_ops.tables_initializer().run() - self.assertAllEqual((b"salad", b"surgery", default_value), - features.eval()) - - -class IndexToStringTest(test.TestCase): - - def test_index_to_string(self): - with self.cached_session(): - mapping_strings = constant_op.constant(["brain", "salad", "surgery"]) - indices = constant_op.constant([0, 1, 2, 3], dtypes.int64) - feats = lookup.index_to_string(indices, mapping=mapping_strings) - - self.assertRaises(errors_impl.OpError, feats.eval) - lookup_ops.tables_initializer().run() - - self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"), - feats.eval()) - - def test_duplicate_entries(self): - with self.cached_session(): - mapping_strings = constant_op.constant(["hello", "hello"]) - indices = constant_op.constant([0, 1, 4], dtypes.int64) - feats = lookup.index_to_string(indices, mapping=mapping_strings) - lookup_ops.tables_initializer().run() - self.assertAllEqual((b"hello", b"hello", b"UNK"), feats.eval()) - - def test_index_to_string_with_default_value(self): - default_value = b"NONE" - with self.cached_session(): - mapping_strings = constant_op.constant(["brain", "salad", "surgery"]) - indices = constant_op.constant([1, 2, 4], dtypes.int64) - feats = lookup.index_to_string( - indices, mapping=mapping_strings, default_value=default_value) - self.assertRaises(errors_impl.OpError, feats.eval) - - lookup_ops.tables_initializer().run() - self.assertAllEqual((b"salad", b"surgery", default_value), feats.eval()) - - -class InitializeTableFromFileOpTest(test.TestCase): - - def _createVocabFile(self, basename, values=("brain", "salad", "surgery")): - vocabulary_file = os.path.join(self.get_temp_dir(), basename) - with open(vocabulary_file, "w") as f: - f.write("\n".join(values) + "\n") - return vocabulary_file - - @test_util.run_in_graph_and_eager_modes - def testInitializeStringTable(self): - vocabulary_file = self._createVocabFile("one_column_1.txt") - default_value = -1 - table = lookup.HashTable( - lookup.TextFileInitializer(vocabulary_file, dtypes.string, - lookup.TextFileIndex.WHOLE_LINE, - dtypes.int64, - lookup.TextFileIndex.LINE_NUMBER), - default_value) - self.evaluate(table.initializer) - - output = table.lookup(constant_op.constant(["brain", "salad", "tank"])) - - result = self.evaluate(output) - self.assertAllEqual([0, 1, -1], result) - - def testInitializeInt64Table(self): - vocabulary_file = self._createVocabFile( - "one_column_int64.txt", values=("42", "1", "-1000")) - - with self.cached_session(): - default_value = -1 - table = lookup.HashTable( - lookup.TextFileInitializer(vocabulary_file, dtypes.int64, - lookup.TextFileIndex.WHOLE_LINE, - dtypes.int64, - lookup.TextFileIndex.LINE_NUMBER), - default_value) - table.initializer.run() - - output = table.lookup( - constant_op.constant((42, 1, 11), dtype=dtypes.int64)) - - result = output.eval() - self.assertAllEqual([0, 1, -1], result) - - def testInitializeIndexTable(self): - vocabulary_file = self._createVocabFile("one_column_2.txt") - - with self.cached_session(): - default_value = "UNK" - key_index = lookup.TextFileIndex.LINE_NUMBER - value_index = lookup.TextFileIndex.WHOLE_LINE - table = lookup.HashTable( - lookup.TextFileInitializer(vocabulary_file, dtypes.int64, - key_index, dtypes.string, value_index), - default_value) - table.initializer.run() - - input_values = constant_op.constant([0, 1, 2, 3], dtypes.int64) - output = table.lookup(input_values) - - result = output.eval() - self.assertAllEqual([b"brain", b"salad", b"surgery", b"UNK"], result) - - def testMultiColumn(self): - vocabulary_file = os.path.join(self.get_temp_dir(), "three_columns.txt") - with open(vocabulary_file, "w") as f: - f.write("\n".join(["0\tbrain\t1", "1\tsalad\t5", "2\tsurgery\t6"]) + "\n") - - with self.cached_session(): - default_value = -1 - key_index = 1 - value_index = 2 - - table = lookup.HashTable( - lookup.TextFileInitializer(vocabulary_file, dtypes.string, - key_index, dtypes.int64, value_index), - default_value) - table.initializer.run() - - input_string = constant_op.constant(["brain", "salad", "surgery"]) - output = table.lookup(input_string) - - result = output.eval() - self.assertAllEqual([1, 5, 6], result) - - def testInvalidDataTypeInMultiColumn(self): - vocabulary_file = os.path.join(self.get_temp_dir(), "three_columns.txt") - with open(vocabulary_file, "w") as f: - f.write("\n".join(["0\tbrain\t1", "1\tsalad\t5", "2\tsurgery\t6"]) + "\n") - - with self.cached_session(): - default_value = -1 - key_index = 2 - value_index = 1 - table = lookup.HashTable( - lookup.TextFileInitializer(vocabulary_file, dtypes.string, - key_index, dtypes.int64, value_index), - default_value) - with self.assertRaisesOpError("is not a valid"): - table.initializer.run() - - def testInvalidDataType(self): - vocabulary_file = self._createVocabFile("one_column_3.txt") - - with self.cached_session(): - default_value = "UNK" - key_index = lookup.TextFileIndex.WHOLE_LINE - value_index = lookup.TextFileIndex.LINE_NUMBER - - with self.assertRaises(ValueError): - lookup.HashTable( - lookup.TextFileInitializer(vocabulary_file, dtypes.int64, - key_index, dtypes.string, - value_index), default_value) - - def testInvalidIndex(self): - vocabulary_file = self._createVocabFile("one_column_4.txt") - with self.cached_session(): - default_value = -1 - key_index = 1 # second column of the line - value_index = lookup.TextFileIndex.LINE_NUMBER - table = lookup.HashTable( - lookup.TextFileInitializer(vocabulary_file, dtypes.string, - key_index, dtypes.int64, value_index), - default_value) - - with self.assertRaisesOpError("Invalid number of columns"): - table.initializer.run() - - def testInitializeSameTableWithMultipleNodes(self): - vocabulary_file = self._createVocabFile("one_column_5.txt") - - with self.cached_session() as sess: - shared_name = "shared-one-columm" - default_value = -1 - table1 = lookup.HashTable( - lookup.TextFileInitializer(vocabulary_file, dtypes.string, - lookup.TextFileIndex.WHOLE_LINE, - dtypes.int64, - lookup.TextFileIndex.LINE_NUMBER), - default_value, - shared_name=shared_name) - table2 = lookup.HashTable( - lookup.TextFileInitializer(vocabulary_file, dtypes.string, - lookup.TextFileIndex.WHOLE_LINE, - dtypes.int64, - lookup.TextFileIndex.LINE_NUMBER), - default_value, - shared_name=shared_name) - table3 = lookup.HashTable( - lookup.TextFileInitializer(vocabulary_file, dtypes.string, - lookup.TextFileIndex.WHOLE_LINE, - dtypes.int64, - lookup.TextFileIndex.LINE_NUMBER), - default_value, - shared_name=shared_name) - - lookup_ops.tables_initializer().run() - - input_string = constant_op.constant(["brain", "salad", "tank"]) - - output1 = table1.lookup(input_string) - output2 = table2.lookup(input_string) - output3 = table3.lookup(input_string) - - out1, out2, out3 = sess.run([output1, output2, output3]) - self.assertAllEqual([0, 1, -1], out1) - self.assertAllEqual([0, 1, -1], out2) - self.assertAllEqual([0, 1, -1], out3) - - def testInitializeTableWithNoFilename(self): - with self.cached_session(): - default_value = -1 - with self.assertRaises(ValueError): - lookup.HashTable( - lookup.TextFileInitializer( - "", dtypes.string, lookup.TextFileIndex.WHOLE_LINE, - dtypes.int64, lookup.TextFileIndex.LINE_NUMBER), - default_value) - - def testInitializeWithVocabSize(self): - with self.cached_session(): - default_value = -1 - vocab_size = 3 - vocabulary_file1 = self._createVocabFile("one_column6.txt") - table1 = lookup.HashTable( - lookup.TextFileInitializer( - vocabulary_file1, - dtypes.string, - lookup.TextFileIndex.WHOLE_LINE, - dtypes.int64, - lookup.TextFileIndex.LINE_NUMBER, - vocab_size=vocab_size), - default_value) - - # Initialize from file. - table1.initializer.run() - self.assertEquals(vocab_size, table1.size().eval()) - - vocabulary_file2 = self._createVocabFile("one_column7.txt") - vocab_size = 5 - table2 = lookup.HashTable( - lookup.TextFileInitializer( - vocabulary_file2, - dtypes.string, - lookup.TextFileIndex.WHOLE_LINE, - dtypes.int64, - lookup.TextFileIndex.LINE_NUMBER, - vocab_size=vocab_size), - default_value) - with self.assertRaisesOpError("Invalid vocab_size"): - table2.initializer.run() - - vocab_size = 1 - vocabulary_file3 = self._createVocabFile("one_column3.txt") - table3 = lookup.HashTable( - lookup.TextFileInitializer( - vocabulary_file3, - dtypes.string, - lookup.TextFileIndex.WHOLE_LINE, - dtypes.int64, - lookup.TextFileIndex.LINE_NUMBER, - vocab_size=vocab_size), - default_value) - - # Smaller vocab size reads only vocab_size records. - table3.initializer.run() - self.assertEquals(vocab_size, table3.size().eval()) - - def testFeedVocabularyName(self): - vocabulary_file = self._createVocabFile("feed_vocabulary.txt") - - with self.cached_session(): - default_value = -1 - table = lookup.HashTable( - lookup.TextFileInitializer("old_file.txt", dtypes.string, - lookup.TextFileIndex.WHOLE_LINE, - dtypes.int64, - lookup.TextFileIndex.LINE_NUMBER), - default_value) - - # Initialize with non existing file (old_file.txt) should fail. - # TODO(yleon): Update message, which might change per FileSystem. - with self.assertRaisesOpError("old_file.txt"): - table.initializer.run() - - # Initialize the model feeding the vocabulary file. - filenames = ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS) - table.initializer.run(feed_dict={filenames[0]: vocabulary_file}) - - input_string = constant_op.constant(["brain", "salad", "tank"]) - output = table.lookup(input_string) - - result = output.eval() - self.assertAllEqual([0, 1, -1], result) - - def testInvalidFilenames(self): - vocabulary_file = self._createVocabFile("filename_shape.txt") - - with self.cached_session(): - default_value = -1 - - # Invalid data type - other_type = constant_op.constant(1) - with self.assertRaises(ValueError): - lookup.HashTable( - lookup.TextFileInitializer( - other_type, dtypes.string, lookup.TextFileIndex.WHOLE_LINE, - dtypes.int64, lookup.TextFileIndex.LINE_NUMBER), - default_value) - - # Non-scalar filename - filenames = constant_op.constant([vocabulary_file, vocabulary_file]) - with self.assertRaises(ValueError): - lookup.HashTable( - lookup.TextFileInitializer( - filenames, dtypes.string, lookup.TextFileIndex.WHOLE_LINE, - dtypes.int64, lookup.TextFileIndex.LINE_NUMBER), - default_value) - - def testIdToStringTable(self): - vocab_file = self._createVocabFile("feat_to_id_1.txt") - with self.cached_session(): - default_value = "UNK" - vocab_size = 3 - table = lookup.HashTable( - lookup.TextFileStringTableInitializer( - vocab_file, vocab_size=vocab_size), - default_value) - - table.initializer.run() - - input_values = constant_op.constant([0, 1, 2, 3], dtypes.int64) - - out = table.lookup(input_values) - self.assertAllEqual([b"brain", b"salad", b"surgery", b"UNK"], out.eval()) - self.assertEquals(vocab_size, table.size().eval()) - - def testStringToIdTable(self): - vocab_file = self._createVocabFile("feat_to_id_2.txt") - with self.cached_session(): - default_value = -1 - vocab_size = 3 - table = lookup.HashTable( - lookup.TextFileIdTableInitializer( - vocab_file, vocab_size=vocab_size), - default_value) - table.initializer.run() - - input_string = constant_op.constant(["brain", "salad", "surgery", "UNK"]) - - out = table.lookup(input_string) - self.assertAllEqual([0, 1, 2, -1], out.eval()) - self.assertEquals(vocab_size, table.size().eval()) - - def testInt64ToIdTable(self): - vocab_file = self._createVocabFile( - "feat_to_id_3.txt", values=("42", "1", "-1000")) - with self.cached_session(): - default_value = -1 - vocab_size = 3 - table = lookup.HashTable( - lookup.TextFileIdTableInitializer( - vocab_file, vocab_size=vocab_size, key_dtype=dtypes.int64), - default_value) - table.initializer.run() - - out = table.lookup( - constant_op.constant((42, 1, -1000, 11), dtype=dtypes.int64)) - self.assertAllEqual((0, 1, 2, -1), out.eval()) - self.assertEquals(vocab_size, table.size().eval()) - - -class IdTableWithHashBucketsTest(test.TestCase): - - def _createVocabFile(self, basename, values=("brain", "salad", "surgery")): - vocabulary_file = os.path.join(self.get_temp_dir(), basename) - with open(vocabulary_file, "w") as f: - f.write("\n".join(values) + "\n") - return vocabulary_file - - def testStringIdTableWithHashBuckets(self): - vocab_file = self._createVocabFile("feat_to_id_1.txt") - with self.cached_session(): - default_value = -1 - vocab_size = 3 - oov_buckets = 1 - table = lookup.IdTableWithHashBuckets( - lookup.HashTable( - lookup.TextFileIdTableInitializer( - vocab_file, vocab_size=vocab_size), - default_value), - oov_buckets) - - table.initializer.run() - - input_string = constant_op.constant(["brain", "salad", "surgery", "UNK"]) - - out = table.lookup(input_string) - self.assertAllEqual([0, 1, 2, 3], out.eval()) - self.assertEquals(vocab_size + oov_buckets, table.size().eval()) - - def testInt32IdTableWithHashBuckets(self): - vocab_file = self._createVocabFile("feat_to_id_2.txt", ("42", "1", "-1000")) - with self.cached_session(): - default_value = -1 - vocab_size = 3 - oov_buckets = 1 - table = lookup.IdTableWithHashBuckets( - lookup.HashTable( - lookup.TextFileIdTableInitializer( - vocab_file, vocab_size=vocab_size, key_dtype=dtypes.int64), - default_value), - oov_buckets, - key_dtype=dtypes.int32) - - table.initializer.run() - - values = constant_op.constant((42, 1, -1000, 11), dtype=dtypes.int32) - - out = table.lookup(values) - self.assertAllEqual([0, 1, 2, 3], out.eval()) - self.assertEquals(vocab_size + oov_buckets, table.size().eval()) - - def testInt64IdTableWithHashBuckets(self): - vocab_file = self._createVocabFile("feat_to_id_3.txt", ("42", "1", "-1000")) - with self.cached_session(): - default_value = -1 - vocab_size = 3 - oov_buckets = 1 - table = lookup.IdTableWithHashBuckets( - lookup.HashTable( - lookup.TextFileIdTableInitializer( - vocab_file, vocab_size=vocab_size, key_dtype=dtypes.int64), - default_value), - oov_buckets) - - table.initializer.run() - - values = constant_op.constant((42, 1, -1000, 11), dtype=dtypes.int64) - - out = table.lookup(values) - self.assertAllEqual([0, 1, 2, 3], out.eval()) - self.assertEquals(vocab_size + oov_buckets, table.size().eval()) - - def testStringIdTableWithOnlyHashBucket(self): - with self.cached_session(): - oov_buckets = 5 - - # Set a table that only uses hash buckets, for each input value returns - # an id calculated by fingerprint("input") mod oov_buckets. - table = lookup.IdTableWithHashBuckets(None, oov_buckets) - table.initializer.run() - - values = constant_op.constant(("brain", "salad", "surgery")) - - out = table.lookup(values) - self.assertAllEqual( - [ - 3, # fingerprint("brain") mod 5. - 1, # fingerprint("salad") mod 5. - 4 # fingerprint("surgery") mod 5 - ], - out.eval()) - self.assertEquals(oov_buckets, table.size().eval()) - - def testInt32IdTableWithOnlyHashBucket(self): - with self.cached_session(): - oov_buckets = 5 - - # Set a table that only uses hash buckets, for each input value returns - # an id calculated by fingerprint("input") mod oov_buckets. - table = lookup.IdTableWithHashBuckets( - None, oov_buckets, key_dtype=dtypes.int32) - table.initializer.run() - - input_string = constant_op.constant([42, 1, -1000], dtype=dtypes.int32) - - out = table.lookup(input_string) - self.assertAllEqual( - [ - 1, # fingerprint("42") mod 5. - 4, # fingerprint("1") mod 5. - 2 # fingerprint("-1000") mod 5 - ], - out.eval()) - self.assertEquals(oov_buckets, table.size().eval()) - - def testFloat64IdTableWithOnlyHashBucket(self): - with self.cached_session(): - with self.assertRaisesRegexp(TypeError, "Invalid key_dtype"): - lookup.IdTableWithHashBuckets( - None, num_oov_buckets=5, key_dtype=dtypes.float64) - - def testBoolIdTableWithOnlyHashBucket(self): - with self.cached_session(): - with self.assertRaisesRegexp(TypeError, "Invalid key_dtype"): - lookup.IdTableWithHashBuckets( - None, num_oov_buckets=5, key_dtype=dtypes.bool) - - def testIdTableWithHashBucketsWithMultipleInitializers(self): - vocab_file = self._createVocabFile("feat_to_id_4.txt") - with self.cached_session() as sess: - default_value = -1 - vocab_size = 3 - oov_buckets = 3 - - vocab_table = lookup.HashTable( - lookup.TextFileIdTableInitializer( - vocab_file, vocab_size=vocab_size), - default_value) - table1 = lookup.IdTableWithHashBuckets( - vocab_table, - oov_buckets, - hasher_spec=lookup.FastHashSpec, - name="table1") - - table2 = lookup.IdTableWithHashBuckets( - vocab_table, - oov_buckets, - hasher_spec=lookup.StrongHashSpec((1, 2)), - name="table2") - - lookup_ops.tables_initializer().run() - - input_string = constant_op.constant( - ["fruit", "brain", "salad", "surgery", "UNK"]) - - out1 = table1.lookup(input_string) - out2 = table2.lookup(input_string) - - out1, out2 = sess.run([out1, out2]) - self.assertAllEqual([5, 0, 1, 2, 5], out1) - self.assertAllEqual([5, 0, 1, 2, 3], out2) - self.assertEquals(vocab_size + oov_buckets, table1.size().eval()) - self.assertEquals(vocab_size + oov_buckets, table2.size().eval()) - test_util.assert_ops_in_graph({ - "table1_Lookup/hash_bucket": "StringToHashBucketFast", - "table2_Lookup/hash_bucket": "StringToHashBucketStrong", - }, sess.graph) - - def testIdTableWithHashBucketsInitializationAcrossSessions(self): - vocab_file = self._createVocabFile("feat_to_id_5.txt") - shared_name = "across-sessions" - with self.cached_session(): - default_value = -1 - vocab_size = 3 - oov_buckets = 1 - table1 = lookup.IdTableWithHashBuckets( - lookup.HashTable( - lookup.TextFileIdTableInitializer( - vocab_file, vocab_size=vocab_size), - default_value, - shared_name=shared_name), - oov_buckets) - - table1.initializer.run() - - input_string_1 = constant_op.constant( - ["brain", "salad", "surgery", "UNK"]) - - out1 = table1.lookup(input_string_1) - - self.assertAllEqual([0, 1, 2, 3], out1.eval()) - self.assertEquals(vocab_size + oov_buckets, table1.size().eval()) - - with self.cached_session(): - default_value = -1 - vocab_size = 3 - oov_buckets = 1 - - # Underlying lookup table already initialized in previous session. - # No need to call table2.initializer.run() - table2 = lookup.IdTableWithHashBuckets( - lookup.HashTable( - lookup.TextFileIdTableInitializer( - vocab_file, vocab_size=vocab_size), - default_value, - shared_name=shared_name), - oov_buckets) - - input_string_2 = constant_op.constant(["fruit", "salad", "UNK"]) - - out2 = table2.lookup(input_string_2) - - self.assertAllEqual([3, 1, 3], out2.eval()) - self.assertEquals(vocab_size + oov_buckets, table2.size().eval()) - - def testIdTableWithHashBucketsWithMultipleInitializersDifferentDefault(self): - vocab_file = self._createVocabFile("feat_to_id_6.txt") - with self.cached_session() as sess: - default_value1 = -1 - vocab_size = 3 - oov_buckets = 0 - table1 = lookup.IdTableWithHashBuckets( - lookup.HashTable( - lookup.TextFileIdTableInitializer( - vocab_file, vocab_size=vocab_size), - default_value1), - oov_buckets) - - default_value2 = -2 - table2 = lookup.IdTableWithHashBuckets( - lookup.HashTable( - lookup.TextFileIdTableInitializer( - vocab_file, vocab_size=vocab_size), - default_value2), - oov_buckets) - - lookup_ops.tables_initializer().run() - - input_string_1 = constant_op.constant( - ["brain", "salad", "surgery", "UNK"]) - input_string_2 = constant_op.constant(["fruit", "salad", "UNK"]) - - out1 = table1.lookup(input_string_1) - out2 = table2.lookup(input_string_2) - - out1, out2 = sess.run([out1, out2]) - self.assertAllEqual([0, 1, 2, -1], out1) - self.assertAllEqual([-2, 1, -2], out2) - self.assertEquals(vocab_size + oov_buckets, table1.size().eval()) - self.assertEquals(vocab_size + oov_buckets, table2.size().eval()) - - def testSparseTensor(self): - vocab_file = self._createVocabFile("feat_to_id_7.txt") - input_indices = [[0, 0], [0, 1], [2, 0], [2, 2], [3, 0]] - input_shape = [4, 4] - with self.cached_session() as sess: - sp_features = sparse_tensor.SparseTensor( - constant_op.constant(input_indices, dtypes.int64), - constant_op.constant(["brain", "salad", "brain", "surgery", "tarkus"], - dtypes.string), - constant_op.constant(input_shape, dtypes.int64)) - - table = lookup.IdTableWithHashBuckets( - lookup.HashTable( - lookup.TextFileIdTableInitializer( - vocab_file, vocab_size=3), - -1), - 1) - table.initializer.run() - - sp_ids = table.lookup(sp_features) - - self.assertAllEqual([5], sp_ids.values._shape_as_list()) - - sp_ids_ind, sp_ids_val, sp_ids_shape = sess.run( - [sp_ids.indices, sp_ids.values, sp_ids.dense_shape]) - - self.assertAllEqual(input_indices, sp_ids_ind) - self.assertAllEqual([0, 1, 0, 2, 3], sp_ids_val) - self.assertAllEqual(input_shape, sp_ids_shape) - - def testInt32SparseTensor(self): - input_indices = [[0, 0], [0, 1], [2, 0], [2, 2], [3, 0]] - input_shape = [4, 4] - with self.cached_session() as sess: - sp_features = sparse_tensor.SparseTensor( - constant_op.constant(input_indices, dtypes.int64), - constant_op.constant([42, 1, 42, -1000, 11], dtypes.int32), - constant_op.constant(input_shape, dtypes.int64)) - - table = lookup.IdTableWithHashBuckets( - lookup.HashTable( - lookup.KeyValueTensorInitializer( - (42, 1, -1000), (0, 1, 2), dtypes.int64, dtypes.int64), - -1), - 1, - key_dtype=dtypes.int32) - table.initializer.run() - - sp_ids = table.lookup(sp_features) - - self.assertAllEqual([5], sp_ids.values._shape_as_list()) - - sp_ids_ind, sp_ids_val, sp_ids_shape = sess.run( - [sp_ids.indices, sp_ids.values, sp_ids.dense_shape]) - - self.assertAllEqual(input_indices, sp_ids_ind) - self.assertAllEqual([0, 1, 0, 2, 3], sp_ids_val) - self.assertAllEqual(input_shape, sp_ids_shape) - - def testInt64SparseTensor(self): - input_indices = [[0, 0], [0, 1], [2, 0], [2, 2], [3, 0]] - input_shape = [4, 4] - with self.cached_session() as sess: - sp_features = sparse_tensor.SparseTensor( - constant_op.constant(input_indices, dtypes.int64), - constant_op.constant([42, 1, 42, -1000, 11], dtypes.int64), - constant_op.constant(input_shape, dtypes.int64)) - - table = lookup.IdTableWithHashBuckets( - lookup.HashTable( - lookup.KeyValueTensorInitializer( - (42, 1, -1000), (0, 1, 2), dtypes.int64, dtypes.int64), - -1), - 1, - key_dtype=dtypes.int64) - table.initializer.run() - - sp_ids = table.lookup(sp_features) - - self.assertAllEqual([5], sp_ids.values._shape_as_list()) - - sp_ids_ind, sp_ids_val, sp_ids_shape = sess.run( - [sp_ids.indices, sp_ids.values, sp_ids.dense_shape]) - - self.assertAllEqual(input_indices, sp_ids_ind) - self.assertAllEqual([0, 1, 0, 2, 3], sp_ids_val) - self.assertAllEqual(input_shape, sp_ids_shape) - - def testIdTableWithHashBucketsWithInvalidHashers(self): - vocab_file = self._createVocabFile("feat_to_id_4.txt") - with self.cached_session(): - default_value = -1 - vocab_size = 3 - oov_buckets = 1 - lookup_table = lookup.HashTable( - lookup.TextFileIdTableInitializer( - vocab_file, vocab_size=vocab_size), - default_value) - - with self.assertRaises(TypeError): - lookup.IdTableWithHashBuckets( - lookup_table, oov_buckets, hasher_spec=1) - - table = lookup.IdTableWithHashBuckets( - lookup_table, - oov_buckets, - hasher_spec=lookup.HasherSpec("my-awesome-hash", None)) - - input_string = constant_op.constant(["brain", "salad", "surgery", "UNK"]) - - with self.assertRaises(ValueError): - table.lookup(input_string) - - with self.assertRaises(ValueError): - table = lookup.IdTableWithHashBuckets( - lookup_table, - oov_buckets, - hasher_spec=lookup.StrongHashSpec([])) - - with self.assertRaises(ValueError): - table = lookup.IdTableWithHashBuckets( - lookup_table, - oov_buckets, - hasher_spec=lookup.StrongHashSpec([1, 2, 3])) - - with self.assertRaises(TypeError): - table = lookup.IdTableWithHashBuckets( - lookup_table, - oov_buckets, - hasher_spec=lookup.StrongHashSpec([None, 2])) - - -if __name__ == "__main__": - test.main() - diff --git a/tensorflow/contrib/losses/BUILD b/tensorflow/contrib/losses/BUILD deleted file mode 100644 index f19ccc967fa..00000000000 --- a/tensorflow/contrib/losses/BUILD +++ /dev/null @@ -1,107 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "losses_py", - srcs = [ - "__init__.py", - "python/losses/__init__.py", - "python/losses/loss_ops.py", - "python/metric_learning/metric_loss_ops.py", - ], - srcs_version = "PY2AND3", - visibility = [ - "//tensorflow:__subpackages__", - "//third_party/py/tf_slim:__subpackages__", - ], - deps = [ - ":metric_learning_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:logging_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:nn_ops", - "//tensorflow/python:script_ops", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:summary", - "//tensorflow/python:util", - ], -) - -py_test( - name = "loss_ops_test", - srcs = glob(["python/losses/loss_ops_test.py"]), - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":losses_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_library( - name = "metric_learning_py", - srcs = [ - "python/metric_learning/__init__.py", - "python/metric_learning/metric_loss_ops.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:logging_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:script_ops", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:summary", - "//tensorflow/python:util", - ], -) - -py_test( - name = "metric_loss_ops_test", - size = "medium", - srcs = [ - "python/metric_learning/metric_loss_ops_test.py", - ], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - deps = [ - ":metric_learning_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:sparse_tensor", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/losses/README.md b/tensorflow/contrib/losses/README.md deleted file mode 100644 index dcda898ed85..00000000000 --- a/tensorflow/contrib/losses/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# TensorFlow contrib losses. - -## Deprecated - -This module is deprecated. Instructions for updating: Use tf.losses instead. - -## losses - -Note: By default all the losses are collected into the GraphKeys.LOSSES collection. - -Loss operations for use in training models, typically with signature like the -following: - -`sum_of_squares(predictions, labels, weight, scope) : Tensor` - -All loss functions take a pair of tensors, `predictions` and ground truth -`labels`. It is assumed that the shape of both these tensors is of the form -`[batch_size, d1, ... dN]` where `batch_size` is the number -of samples in the batch and `d1` ... `dN` are the remaining dimensions. - -The `weight` parameter can be used to adjust the relative weight samples within -the batch. The result of each loss is a scalar average of all sample losses with -non-zero weights. - -Any parameter named `logit` should be the raw model outputs, not a normalized -probability distribution (i.e., `[0.0, 1.0]`). `target` for losses taking -`logit` _should_ be a normalized probability distribution. diff --git a/tensorflow/contrib/losses/__init__.py b/tensorflow/contrib/losses/__init__.py deleted file mode 100644 index 92b380df53b..00000000000 --- a/tensorflow/contrib/losses/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Ops for building neural network losses. - -See [Contrib Losses](https://tensorflow.org/api_guides/python/contrib.losses). -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.losses.python import metric_learning -# pylint: disable=wildcard-import -from tensorflow.contrib.losses.python.losses import * -# pylint: enable=wildcard-import -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'absolute_difference', - 'add_loss', - 'hinge_loss', - 'compute_weighted_loss', - 'cosine_distance', - 'get_losses', - 'get_regularization_losses', - 'get_total_loss', - 'log_loss', - 'mean_pairwise_squared_error', - 'mean_squared_error', - 'sigmoid_cross_entropy', - 'softmax_cross_entropy', - 'sparse_softmax_cross_entropy', - 'metric_learning' -] -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/losses/python/losses/__init__.py b/tensorflow/contrib/losses/python/losses/__init__.py deleted file mode 100644 index 1675387227b..00000000000 --- a/tensorflow/contrib/losses/python/losses/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for building neural network losses. - -See [Contrib Losses](https://tensorflow.org/api_guides/python/contrib.losses). -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.losses.python.losses.loss_ops import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/losses/python/losses/loss_ops.py b/tensorflow/contrib/losses/python/losses/loss_ops.py deleted file mode 100644 index dea111f9a0f..00000000000 --- a/tensorflow/contrib/losses/python/losses/loss_ops.py +++ /dev/null @@ -1,652 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Loss operations for use in neural networks. - -Note: All the losses are added to the `GraphKeys.LOSSES` collection. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework.python.ops import add_arg_scope -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.util.deprecation import deprecated -from tensorflow.python.util.deprecation import deprecated_args -from tensorflow.python.util.deprecation import deprecated_argument_lookup - -__all__ = [ - "absolute_difference", "add_loss", "cosine_distance", - "compute_weighted_loss", "get_losses", "get_regularization_losses", - "get_total_loss", "hinge_loss", "log_loss", "mean_pairwise_squared_error", - "mean_squared_error", "sigmoid_cross_entropy", "softmax_cross_entropy", - "sparse_softmax_cross_entropy" -] - - -def _scale_losses(losses, weights): - """Computes the scaled loss. - - Args: - losses: A `Tensor` of size [batch_size, d1, ... dN]. - weights: A `Tensor` of size [1], [batch_size] or [batch_size, d1, ... dN]. - The `losses` are reduced (tf.reduce_sum) until its dimension matches - that of `weights` at which point the reduced `losses` are element-wise - multiplied by `weights` and a final reduce_sum is computed on the result. - Conceptually, this operation is equivalent to broadcasting (tiling) - `weights` to be the same size as `losses`, performing an element-wise - multiplication, and summing the result. - - Returns: - A scalar tf.float32 `Tensor` whose value represents the sum of the scaled - `losses`. - """ - # First, compute the sum of the losses over all elements: - start_index = max(0, weights.get_shape().ndims) - axis = list(range(start_index, losses.get_shape().ndims)) - reduced_losses = math_ops.reduce_sum(losses, axis=axis) - reduced_losses = math_ops.multiply(reduced_losses, weights) - return math_ops.reduce_sum(reduced_losses) - - -def _safe_mean(losses, num_present): - """Computes a safe mean of the losses. - - Args: - losses: A tensor whose elements contain individual loss measurements. - num_present: The number of measurable losses in the tensor. - - Returns: - A scalar representing the mean of the losses. If `num_present` is zero, - then zero is returned. - """ - total_loss = math_ops.reduce_sum(losses) - return math_ops.div_no_nan(total_loss, num_present, name="value") - - -@deprecated("2016-12-30", "Use tf.losses.compute_weighted_loss instead.") -def compute_weighted_loss(losses, weights=1.0, scope=None): - """Computes the weighted loss. - - Args: - losses: A tensor of size [batch_size, d1, ... dN]. - weights: A tensor of size [1] or [batch_size, d1, ... dK] where K < N. - scope: the scope for the operations performed in computing the loss. - - Returns: - A scalar `Tensor` that returns the weighted loss. - - Raises: - ValueError: If `weights` is `None` or the shape is not compatible with - `losses`, or if the number of dimensions (rank) of either `losses` or - `weights` is missing. - """ - with ops.name_scope(scope, "weighted_loss", [losses, weights]): - losses = ops.convert_to_tensor(losses) - input_dtype = losses.dtype - losses = math_ops.cast(losses, dtypes.float32) - weights = math_ops.cast(ops.convert_to_tensor(weights), dtypes.float32) - - if losses.get_shape().ndims is None: - raise ValueError("losses.get_shape().ndims cannot be None") - weights_shape = weights.get_shape() - if weights_shape.ndims is None: - raise ValueError("weights.get_shape().ndims cannot be None") - - if weights_shape.ndims > 1 and weights_shape.dims[-1].is_compatible_with(1): - weights = array_ops.squeeze(weights, [-1]) - - total_loss = _scale_losses(losses, weights) - num_present = _num_present(losses, weights) - mean_loss = _safe_mean(total_loss, num_present) - # convert the result back to the input type - mean_loss = math_ops.cast(mean_loss, input_dtype) - add_loss(mean_loss) - return mean_loss - - -def _num_present(losses, weights, per_batch=False): - """Computes the number of elements in the loss function induced by `weights`. - - A given weights tensor induces different numbers of usable elements in the - `losses` tensor. The `weights` tensor is broadcast across `losses` for all - possible dimensions. For example, if `losses` is a tensor of dimension - [4, 5, 6, 3] and `weights` is a tensor of size [4, 5], then `weights` is, in - effect, tiled to match the size of `losses`. Following this effective tile, - the total number of present elements is the number of non-zero weights. - - Args: - losses: A tensor of size [batch_size, d1, ... dN]. - weights: A tensor of size [1] or [batch_size, d1, ... dK] where K < N. - per_batch: Whether to return the number of elements per batch or as a sum - total. - - Returns: - The number of present (non-zero) elements in the losses tensor. If - `per_batch` is True, the value is returned as a tensor of size - [batch_size]. Otherwise, a single scalar tensor is returned. - """ - # If weights is a scalar, its easy to compute: - if weights.get_shape().ndims == 0: - batch_size = array_ops.reshape( - array_ops.slice(array_ops.shape(losses), [0], [1]), []) - num_per_batch = math_ops.div( - math_ops.cast(array_ops.size(losses), dtypes.float32), - math_ops.cast(batch_size, dtypes.float32)) - num_per_batch = array_ops.where( - math_ops.equal(weights, 0), 0.0, num_per_batch) - num_per_batch = math_ops.multiply( - array_ops.ones(array_ops.reshape(batch_size, [1])), num_per_batch) - return num_per_batch if per_batch else math_ops.reduce_sum(num_per_batch) - - # First, count the number of nonzero weights: - if weights.get_shape().ndims >= 1: - axis = list(range(1, weights.get_shape().ndims)) - num_nonzero_per_batch = math_ops.reduce_sum( - math_ops.cast(math_ops.not_equal(weights, 0), dtypes.float32), - axis=axis) - - # Next, determine the number of elements that weights would broadcast to: - broadcast_dims = array_ops.slice( - array_ops.shape(losses), [weights.get_shape().ndims], [-1]) - num_to_broadcast = math_ops.cast(math_ops.reduce_prod(broadcast_dims), - dtypes.float32) - - num_per_batch = math_ops.multiply(num_nonzero_per_batch, num_to_broadcast) - return num_per_batch if per_batch else math_ops.reduce_sum(num_per_batch) - - -@deprecated("2016-12-30", "Use tf.losses.add_loss instead.") -@add_arg_scope -def add_loss(loss, loss_collection=ops.GraphKeys.LOSSES): - """Adds a externally defined loss to the collection of losses. - - Args: - loss: A loss `Tensor`. - loss_collection: Optional collection to add the loss to. - """ - if loss_collection: - ops.add_to_collection(loss_collection, loss) - - -@deprecated("2016-12-30", "Use tf.losses.get_losses instead.") -def get_losses(scope=None, loss_collection=ops.GraphKeys.LOSSES): - """Gets the list of losses from the loss_collection. - - Args: - scope: an optional scope for filtering the losses to return. - loss_collection: Optional losses collection. - - Returns: - a list of loss tensors. - """ - return ops.get_collection(loss_collection, scope) - - -@deprecated("2016-12-30", "Use tf.losses.get_regularization_losses instead.") -def get_regularization_losses(scope=None): - """Gets the regularization losses. - - Args: - scope: an optional scope for filtering the losses to return. - - Returns: - A list of regularization losses as Tensors. - """ - return ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES, scope) - - -@deprecated("2016-12-30", "Use tf.losses.get_total_loss instead.") -def get_total_loss(add_regularization_losses=True, name="total_loss"): - """Returns a tensor whose value represents the total loss. - - Notice that the function adds the given losses to the regularization losses. - - Args: - add_regularization_losses: A boolean indicating whether or not to use the - regularization losses in the sum. - name: The name of the returned tensor. - - Returns: - A `Tensor` whose value represents the total loss. - - Raises: - ValueError: if `losses` is not iterable. - """ - losses = get_losses() - if add_regularization_losses: - losses += get_regularization_losses() - return math_ops.add_n(losses, name=name) - - -@deprecated("2016-12-30", "Use tf.losses.absolute_difference instead.") -def absolute_difference(predictions, labels=None, weights=1.0, scope=None): - """Adds an Absolute Difference loss to the training procedure. - - `weights` acts as a coefficient for the loss. If a scalar is provided, then - the loss is simply scaled by the given value. If `weights` is a tensor of size - [batch_size], then the total loss for each sample of the batch is rescaled - by the corresponding element in the `weights` vector. If the shape of - `weights` matches the shape of `predictions`, then the loss of each - measurable element of `predictions` is scaled by the corresponding value of - `weights`. - - Args: - predictions: The predicted outputs. - labels: The ground truth output tensor, same dimensions as 'predictions'. - weights: Coefficients for the loss a scalar, a tensor of shape - [batch_size] or a tensor whose shape matches `predictions`. - scope: The scope for the operations performed in computing the loss. - - Returns: - A scalar `Tensor` representing the loss value. - - Raises: - ValueError: If the shape of `predictions` doesn't match that of `labels` or - if the shape of `weights` is invalid. - """ - with ops.name_scope(scope, "absolute_difference", - [predictions, labels, weights]) as scope: - predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - predictions = math_ops.cast(predictions, dtypes.float32) - labels = math_ops.cast(labels, dtypes.float32) - losses = math_ops.abs(math_ops.subtract(predictions, labels)) - return compute_weighted_loss(losses, weights, scope=scope) - - -@deprecated("2016-12-30", - "Use tf.losses.sigmoid_cross_entropy instead. Note that the order " - "of the predictions and labels arguments has been changed.") -def sigmoid_cross_entropy(logits, - multi_class_labels, - weights=1.0, - label_smoothing=0, - scope=None): - """Creates a cross-entropy loss using tf.nn.sigmoid_cross_entropy_with_logits. - - `weights` acts as a coefficient for the loss. If a scalar is provided, - then the loss is simply scaled by the given value. If `weights` is a - tensor of size [`batch_size`], then the loss weights apply to each - corresponding sample. - - If `label_smoothing` is nonzero, smooth the labels towards 1/2: - - new_multiclass_labels = multiclass_labels * (1 - label_smoothing) - + 0.5 * label_smoothing - - Args: - logits: [batch_size, num_classes] logits outputs of the network . - multi_class_labels: [batch_size, num_classes] labels in (0, 1). - weights: Coefficients for the loss. The tensor must be a scalar, a tensor of - shape [batch_size] or shape [batch_size, num_classes]. - label_smoothing: If greater than 0 then smooth the labels. - scope: The scope for the operations performed in computing the loss. - - Returns: - A scalar `Tensor` representing the loss value. - - Raises: - ValueError: If the shape of `logits` doesn't match that of - `multi_class_labels` or if the shape of `weights` is invalid, or if - `weights` is None. - """ - with ops.name_scope(scope, "sigmoid_cross_entropy_loss", - [logits, multi_class_labels, weights]) as scope: - logits.get_shape().assert_is_compatible_with(multi_class_labels.get_shape()) - - multi_class_labels = math_ops.cast(multi_class_labels, logits.dtype) - - if label_smoothing > 0: - multi_class_labels = ( - multi_class_labels * (1 - label_smoothing) + 0.5 * label_smoothing) - - losses = nn.sigmoid_cross_entropy_with_logits( - labels=multi_class_labels, logits=logits, name="xentropy") - return compute_weighted_loss(losses, weights, scope=scope) - - -@deprecated("2016-12-30", - "Use tf.losses.softmax_cross_entropy instead. Note that the order " - "of the logits and labels arguments has been changed.") -def softmax_cross_entropy(logits, - onehot_labels, - weights=1.0, - label_smoothing=0, - scope=None): - """Creates a cross-entropy loss using tf.nn.softmax_cross_entropy_with_logits. - - `weights` acts as a coefficient for the loss. If a scalar is provided, - then the loss is simply scaled by the given value. If `weights` is a - tensor of size [`batch_size`], then the loss weights apply to each - corresponding sample. - - If `label_smoothing` is nonzero, smooth the labels towards 1/num_classes: - new_onehot_labels = onehot_labels * (1 - label_smoothing) - + label_smoothing / num_classes - - Args: - logits: [batch_size, num_classes] logits outputs of the network . - onehot_labels: [batch_size, num_classes] one-hot-encoded labels. - weights: Coefficients for the loss. The tensor must be a scalar or a tensor - of shape [batch_size]. - label_smoothing: If greater than 0 then smooth the labels. - scope: the scope for the operations performed in computing the loss. - - Returns: - A scalar `Tensor` representing the mean loss value. - - Raises: - ValueError: If the shape of `logits` doesn't match that of `onehot_labels` - or if the shape of `weights` is invalid or if `weights` is None. - """ - with ops.name_scope(scope, "softmax_cross_entropy_loss", - [logits, onehot_labels, weights]) as scope: - logits.get_shape().assert_is_compatible_with(onehot_labels.get_shape()) - - onehot_labels = math_ops.cast(onehot_labels, logits.dtype) - - if label_smoothing > 0: - num_classes = math_ops.cast( - array_ops.shape(onehot_labels)[1], logits.dtype) - smooth_positives = 1.0 - label_smoothing - smooth_negatives = label_smoothing / num_classes - onehot_labels = onehot_labels * smooth_positives + smooth_negatives - - losses = nn.softmax_cross_entropy_with_logits( - labels=onehot_labels, logits=logits, name="xentropy") - return compute_weighted_loss(losses, weights, scope=scope) - - -@deprecated("2016-12-30", - "Use tf.losses.sparse_softmax_cross_entropy instead. Note that " - "the order of the logits and labels arguments has been changed.") -def sparse_softmax_cross_entropy(logits, labels, weights=1.0, scope=None): - """Cross-entropy loss using `tf.nn.sparse_softmax_cross_entropy_with_logits`. - - `weights` acts as a coefficient for the loss. If a scalar is provided, - then the loss is simply scaled by the given value. If `weights` is a - tensor of size [`batch_size`], then the loss weights apply to each - corresponding sample. - - Args: - logits: [batch_size, num_classes] logits outputs of the network . - labels: [batch_size, 1] or [batch_size] labels of dtype `int32` or `int64` - in the range `[0, num_classes)`. - weights: Coefficients for the loss. The tensor must be a scalar or a tensor - of shape [batch_size] or [batch_size, 1]. - scope: the scope for the operations performed in computing the loss. - - Returns: - A scalar `Tensor` representing the mean loss value. - - Raises: - ValueError: If the shapes of `logits`, `labels`, and `weights` are - incompatible, or if `weights` is None. - """ - with ops.name_scope(scope, "sparse_softmax_cross_entropy_loss", - [logits, labels, weights]) as scope: - labels = array_ops.reshape(labels, shape=[array_ops.shape(labels)[0]]) - - losses = nn.sparse_softmax_cross_entropy_with_logits( - labels=labels, logits=logits, name="xentropy") - return compute_weighted_loss(losses, weights, scope=scope) - - -@deprecated("2016-12-30", - "Use tf.losses.log_loss instead. Note that the order of the " - "predictions and labels arguments has been changed.") -def log_loss(predictions, labels=None, weights=1.0, epsilon=1e-7, scope=None): - """Adds a Log Loss term to the training procedure. - - `weights` acts as a coefficient for the loss. If a scalar is provided, then - the loss is simply scaled by the given value. If `weights` is a tensor of size - [batch_size], then the total loss for each sample of the batch is rescaled - by the corresponding element in the `weights` vector. If the shape of - `weights` matches the shape of `predictions`, then the loss of each - measurable element of `predictions` is scaled by the corresponding value of - `weights`. - - Args: - predictions: The predicted outputs. - labels: The ground truth output tensor, same dimensions as 'predictions'. - weights: Coefficients for the loss a scalar, a tensor of shape - [batch_size] or a tensor whose shape matches `predictions`. - epsilon: A small increment to add to avoid taking a log of zero. - scope: The scope for the operations performed in computing the loss. - - Returns: - A scalar `Tensor` representing the loss value. - - Raises: - ValueError: If the shape of `predictions` doesn't match that of `labels` or - if the shape of `weights` is invalid. - """ - with ops.name_scope(scope, "log_loss", - [predictions, labels, weights]) as scope: - predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - predictions = math_ops.cast(predictions, dtypes.float32) - labels = math_ops.cast(labels, dtypes.float32) - losses = -math_ops.multiply( - labels, math_ops.log(predictions + epsilon)) - math_ops.multiply( - (1 - labels), math_ops.log(1 - predictions + epsilon)) - return compute_weighted_loss(losses, weights, scope=scope) - - -@deprecated("2016-12-30", - "Use tf.losses.hinge_loss instead. Note that the order of the " - "logits and labels arguments has been changed, and to stay " - "unweighted, reduction=Reduction.NONE") -def hinge_loss(logits, labels=None, scope=None): - """Method that returns the loss tensor for hinge loss. - - Args: - logits: The logits, a float tensor. Note that logits are assumed to be - unbounded and 0-centered. A value > 0 (resp. < 0) is considered a positive - (resp. negative) binary prediction. - labels: The ground truth output tensor. Its shape should match the shape of - logits. The values of the tensor are expected to be 0.0 or 1.0. Internally - the {0,1} labels are converted to {-1,1} when calculating the hinge loss. - scope: The scope for the operations performed in computing the loss. - - Returns: - An unweighted `Tensor` of same shape as `logits` and `labels` representing - the - loss values across the batch. - - Raises: - ValueError: If the shapes of `logits` and `labels` don't match. - """ - with ops.name_scope(scope, "hinge_loss", [logits, labels]) as scope: - logits.get_shape().assert_is_compatible_with(labels.get_shape()) - # We first need to convert binary labels to -1/1 labels (as floats). - labels = math_ops.cast(labels, dtypes.float32) - all_ones = array_ops.ones_like(labels) - labels = math_ops.subtract(2 * labels, all_ones) - return nn_ops.relu( - math_ops.subtract(all_ones, math_ops.multiply(labels, logits))) - - -@deprecated("2016-12-30", "Use tf.losses.mean_squared_error instead.") -def mean_squared_error(predictions, labels=None, weights=1.0, scope=None): - """Adds a Sum-of-Squares loss to the training procedure. - - `weights` acts as a coefficient for the loss. If a scalar is provided, then - the loss is simply scaled by the given value. If `weights` is a tensor of size - [batch_size], then the total loss for each sample of the batch is rescaled - by the corresponding element in the `weights` vector. If the shape of - `weights` matches the shape of `predictions`, then the loss of each - measurable element of `predictions` is scaled by the corresponding value of - `weights`. - - Args: - predictions: The predicted outputs. - labels: The ground truth output tensor, same dimensions as 'predictions'. - weights: Coefficients for the loss a scalar, a tensor of shape - [batch_size] or a tensor whose shape matches `predictions`. - scope: The scope for the operations performed in computing the loss. - - Returns: - A scalar `Tensor` representing the loss value. - - Raises: - ValueError: If the shape of `predictions` doesn't match that of `labels` or - if the shape of `weights` is invalid. - """ - with ops.name_scope(scope, "mean_squared_error", - [predictions, labels, weights]) as scope: - predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - predictions = math_ops.cast(predictions, dtypes.float32) - labels = math_ops.cast(labels, dtypes.float32) - losses = math_ops.squared_difference(predictions, labels) - return compute_weighted_loss(losses, weights, scope=scope) - - -@deprecated("2016-12-30", - "Use tf.losses.mean_pairwise_squared_error instead. Note that the " - "order of the predictions and labels arguments has been changed.") -def mean_pairwise_squared_error(predictions, - labels=None, - weights=1.0, - scope=None): - """Adds a pairwise-errors-squared loss to the training procedure. - - Unlike `mean_squared_error`, which is a measure of the differences between - corresponding elements of `predictions` and `labels`, - `mean_pairwise_squared_error` is a measure of the differences between pairs of - corresponding elements of `predictions` and `labels`. - - For example, if `labels`=[a, b, c] and `predictions`=[x, y, z], there are - three pairs of differences are summed to compute the loss: - loss = [ ((a-b) - (x-y)).^2 + ((a-c) - (x-z)).^2 + ((b-c) - (y-z)).^2 ] / 3 - - Note that since the inputs are of size [batch_size, d0, ... dN], the - corresponding pairs are computed within each batch sample but not across - samples within a batch. For example, if `predictions` represents a batch of - 16 grayscale images of dimension [batch_size, 100, 200], then the set of pairs - is drawn from each image, but not across images. - - `weights` acts as a coefficient for the loss. If a scalar is provided, then - the loss is simply scaled by the given value. If `weights` is a tensor of size - [batch_size], then the total loss for each sample of the batch is rescaled - by the corresponding element in the `weights` vector. - - Args: - predictions: The predicted outputs, a tensor of size [batch_size, d0, .. dN] - where N+1 is the total number of dimensions in `predictions`. - labels: The ground truth output tensor, whose shape must match the shape of - the `predictions` tensor. - weights: Coefficients for the loss a scalar, a tensor of shape [batch_size] - or a tensor whose shape matches `predictions`. - scope: The scope for the operations performed in computing the loss. - - Returns: - A scalar `Tensor` representing the loss value. - - Raises: - ValueError: If the shape of `predictions` doesn't match that of `labels` or - if the shape of `weights` is invalid. - """ - with ops.name_scope(scope, "mean_pairwise_squared_error", - [predictions, labels, weights]) as scope: - predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - predictions = math_ops.cast(predictions, dtypes.float32) - labels = math_ops.cast(labels, dtypes.float32) - weights = math_ops.cast(ops.convert_to_tensor(weights), dtypes.float32) - - diffs = math_ops.subtract(predictions, labels) - - # Need to verify here since the function doesn't use compute_weighted_loss - if diffs.get_shape().ndims is None: - raise ValueError("diffs.get_shape().ndims cannot be None") - if weights.get_shape().ndims is None: - raise ValueError("weights.get_shape().ndims cannot be None") - - axis = list(range(1, diffs.get_shape().ndims)) - - sum_squares_diff_per_batch = math_ops.reduce_sum( - math_ops.square(diffs), axis=axis) - num_present_per_batch = _num_present(diffs, weights, per_batch=True) - - term1 = 2.0 * math_ops.div_no_nan( - sum_squares_diff_per_batch, num_present_per_batch, name="value") - - sum_diff = math_ops.reduce_sum(diffs, axis=axis) - term2 = 2.0 * math_ops.div_no_nan( - math_ops.square(sum_diff), - math_ops.square(num_present_per_batch), - name="value") - - loss = _scale_losses(term1 - term2, weights) - - mean_loss = array_ops.where( - math_ops.reduce_sum(num_present_per_batch) > 0, - loss, - array_ops.zeros_like(loss), - name="value") - add_loss(mean_loss) - return mean_loss - - -@deprecated("2016-12-30", "Use tf.losses.cosine_distance instead.") -@deprecated_args(None, "dim is deprecated, use axis instead", "dim") -def cosine_distance(predictions, - labels=None, - axis=None, - weights=1.0, - scope=None, - dim=None): - """Adds a cosine-distance loss to the training procedure. - - Note that the function assumes that `predictions` and `labels` are already - unit-normalized. - - Args: - predictions: An arbitrary matrix. - labels: A `Tensor` whose shape matches 'predictions' - axis: The dimension along which the cosine distance is computed. - weights: Coefficients for the loss a scalar, a tensor of shape - [batch_size] or a tensor whose shape matches `predictions`. - scope: The scope for the operations performed in computing the loss. - dim: The old (deprecated) name for `axis`. - - Returns: - A scalar `Tensor` representing the loss value. - - Raises: - ValueError: If `predictions` shape doesn't match `labels` shape, or - `weights` is `None`. - """ - axis = deprecated_argument_lookup( - "axis", axis, "dim", dim) - if axis is None: - raise ValueError("You must specify 'axis'.") - with ops.name_scope(scope, "cosine_distance_loss", - [predictions, labels, weights]) as scope: - predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - - predictions = math_ops.cast(predictions, dtypes.float32) - labels = math_ops.cast(labels, dtypes.float32) - - radial_diffs = math_ops.multiply(predictions, labels) - losses = 1 - math_ops.reduce_sum( - radial_diffs, axis=[ - axis, - ]) - return compute_weighted_loss(losses, weights, scope=scope) diff --git a/tensorflow/contrib/losses/python/losses/loss_ops_test.py b/tensorflow/contrib/losses/python/losses/loss_ops_test.py deleted file mode 100644 index 14c2f6665e2..00000000000 --- a/tensorflow/contrib/losses/python/losses/loss_ops_test.py +++ /dev/null @@ -1,1286 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for contrib.losses.python.losses.loss_ops.""" -# pylint: disable=unused-import,g-bad-import-order -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -# pylint: enable=unused-import - -import numpy as np -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.losses.python.losses import loss_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import momentum as momentum_lib - - -class AbsoluteDifferenceLossTest(test.TestCase): - - def setUp(self): - self._predictions = constant_op.constant([4, 8, 12, 8, 1, 3], shape=(2, 3)) - self._labels = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) - - def testValueErrorThrownWhenWeightIsNone(self): - with self.cached_session(): - with self.assertRaises(ValueError): - loss_ops.absolute_difference( - self._predictions, self._predictions, weights=None) - - def testAllCorrectNoLossWeight(self): - loss = loss_ops.absolute_difference(self._predictions, self._predictions) - with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) - - def testNonZeroLoss(self): - loss = loss_ops.absolute_difference(self._predictions, self._labels) - with self.cached_session(): - self.assertAlmostEqual(5.5, loss.eval(), 3) - - def testNonZeroLossWithPythonScalarWeight(self): - weights = 2.3 - loss = loss_ops.absolute_difference(self._predictions, self._labels, - weights) - with self.cached_session(): - self.assertAlmostEqual(5.5 * weights, loss.eval(), 3) - - def testNonZeroLossWithScalarTensorWeight(self): - weights = 2.3 - loss = loss_ops.absolute_difference(self._predictions, self._labels, - constant_op.constant(weights)) - with self.cached_session(): - self.assertAlmostEqual(5.5 * weights, loss.eval(), 3) - - def testNonZeroLossWithOneDimBatchSpecificWeights(self): - weights = constant_op.constant([1.2, 0.0], shape=[2,]) - loss = loss_ops.absolute_difference(self._predictions, self._labels, - weights) - with self.cached_session(): - self.assertAlmostEqual(5.6, loss.eval(), 3) - - def testNonZeroLossWithTwoDimBatchSpecificWeights(self): - weights = constant_op.constant([1.2, 0.0], shape=[2, 1]) - loss = loss_ops.absolute_difference(self._predictions, self._labels, - weights) - with self.cached_session(): - self.assertAlmostEqual(5.6, loss.eval(), 3) - - def testNonZeroLossWithSampleSpecificWeights(self): - weights = constant_op.constant([3, 6, 5, 0, 4, 2], shape=[2, 3]) - loss = loss_ops.absolute_difference(self._predictions, self._labels, - weights) - with self.cached_session(): - self.assertAlmostEqual(16.6, loss.eval(), 3) - - def testNonZeroLossWithSampleSpecificWeightsMostZero(self): - weights = constant_op.constant([0, 0, 0, 0, 0, 2], shape=[2, 3]) - loss = loss_ops.absolute_difference(self._predictions, self._labels, - weights) - with self.cached_session(): - self.assertAlmostEqual(6.0, loss.eval(), 3) - - def testLossWithSampleSpecificWeightsAllZero(self): - weights = array_ops.zeros((2, 3)) - loss = loss_ops.absolute_difference(self._predictions, self._labels, - weights) - with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) - - -class SoftmaxCrossEntropyLossTest(test.TestCase): - - def testNoneWeightRaisesValueError(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[1, 0, 0], - [0, 1, 0], - [0, 0, 1]]) - with self.cached_session(): - with self.assertRaises(ValueError): - loss_ops.softmax_cross_entropy(logits, labels, weights=None) - - def testAllCorrect(self): - with self.cached_session(): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[1, 0, 0], - [0, 1, 0], - [0, 0, 1]]) - loss = loss_ops.softmax_cross_entropy(logits, labels) - self.assertEquals('softmax_cross_entropy_loss/value', loss.op.name) - self.assertAlmostEqual(loss.eval(), 0.0, 3) - - def testAllWrong(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[0, 0, 1], - [1, 0, 0], - [0, 1, 0]]) - - with self.cached_session(): - loss = loss_ops.softmax_cross_entropy(logits, labels) - self.assertEquals(loss.op.name, 'softmax_cross_entropy_loss/value') - self.assertAlmostEqual(loss.eval(), 10.0, 3) - - def testNonZeroLossWithPythonScalarWeight(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[0, 0, 1], - [1, 0, 0], - [0, 1, 0]]) - weights = 2.3 - with self.cached_session(): - loss = loss_ops.softmax_cross_entropy(logits, labels, weights) - self.assertAlmostEqual(weights * 10.0, loss.eval(), 3) - - def testNonZeroLossWithScalarTensorWeight(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[0, 0, 1], - [1, 0, 0], - [0, 1, 0]]) - weights = 2.3 - with self.cached_session(): - loss = loss_ops.softmax_cross_entropy(logits, labels, - constant_op.constant(weights)) - self.assertAlmostEqual(weights * 10.0, loss.eval(), 3) - - def testNonZeroLossWithOneDimBatchSpecificWeights(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[0, 0, 1], - [1, 0, 0], - [0, 1, 0]]) - weights = constant_op.constant([1.2, 3.4, 5.6], shape=[3]) - with self.cached_session(): - loss = loss_ops.softmax_cross_entropy(logits, labels, weights) - self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, loss.eval(), 3) - - def testAllWrongAllWeightsMissing(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[0, 0, 1], - [1, 0, 0], - [0, 1, 0]]) - weights = constant_op.constant([0, 0, 0], shape=[3]) - with self.cached_session(): - loss = loss_ops.softmax_cross_entropy(logits, labels, weights) - self.assertAlmostEqual(0.0, loss.eval(), 3) - - def testSomeWeightsMissing(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[0, 0, 1], - [1, 0, 0], - [0, 1, 0]]) - weights = constant_op.constant([1.2, 0, 0], shape=[3]) - with self.cached_session(): - loss = loss_ops.softmax_cross_entropy(logits, labels, weights) - self.assertAlmostEqual(12.0, loss.eval(), 3) - - def testSoftmaxWithMeasurementSpecificWeightsRaisesException(self): - with self.cached_session(): - logits = constant_op.constant([[100.0, -100.0, -100.0], - [-100.0, 100.0, -100.0], - [-100.0, -100.0, 100.0]]) - labels = constant_op.constant([[1, 0, 0], - [0, 1, 0], - [0, 0, 1]]) - weights = constant_op.constant([[3, 4, 5], - [2, 6, 0], - [8, 0, 1]]) - - with self.assertRaises(ValueError): - loss_ops.softmax_cross_entropy(logits, labels, weights=weights).eval() - - def testSoftmaxLabelSmoothing(self): - with self.cached_session(): - # Softmax Cross Entropy Loss is: - # -\sum_i p_i \log q_i - # where for a softmax activation - # \log q_i = x_i - \log \sum_j \exp x_j - # = x_i - x_max - \log \sum_j \exp (x_j - x_max) - # For our activations, [100, -100, -100] the log partition function - # becomes \log ( exp(0) + exp(-200) + exp(-200) ) = 0 - # so our log softmaxes become: [0, -200, -200] - # so our cross entropy loss is: - # -(1 - L + L/n) * 0 + 400 * L/n = 400 L/n - logits = constant_op.constant([[100.0, -100.0, -100.0]]) - labels = constant_op.constant([[1, 0, 0]]) - label_smoothing = 0.1 - loss = loss_ops.softmax_cross_entropy( - logits, labels, label_smoothing=label_smoothing) - self.assertEquals(loss.op.name, 'softmax_cross_entropy_loss/value') - expected_value = 400.0 * label_smoothing / 3.0 - self.assertAlmostEqual(loss.eval(), expected_value, 3) - - def testLossWithDynamicallyShapedWeights1D(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[0, 0, 1], - [1, 0, 0], - [0, 1, 0]]) - weights = [2.3, 2.4, 2.5] - weights_placeholder = array_ops.placeholder(dtypes.float32, shape=[None]) - loss = loss_ops.softmax_cross_entropy(logits, labels, weights_placeholder) - with self.cached_session() as sess: - loss = sess.run(loss, {weights_placeholder: weights}) - self.assertAlmostEqual(np.average(weights) * 10.0, loss, 3) - - def testLossWithDynamicallyShapedWeights2D(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[0, 0, 1], - [1, 0, 0], - [0, 1, 0]]) - weights = [[2.3], [2.4], [2.5]] - weights_placeholder = array_ops.placeholder( - dtypes.float32, shape=[None, None]) - loss = loss_ops.softmax_cross_entropy(logits, labels, weights_placeholder) - with self.cached_session() as sess: - loss = sess.run(loss, {weights_placeholder: weights}) - self.assertAlmostEqual(np.average(weights) * 10.0, loss, 3) - - -class SparseSoftmaxCrossEntropyLossTest(test.TestCase): - - def testNoneWeightRaisesValueError(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[0], [1], [2]]) - with self.cached_session(): - with self.assertRaises(ValueError): - loss_ops.sparse_softmax_cross_entropy(logits, labels, weights=None) - - def testAllCorrectInt32Labels(self): - with self.cached_session(): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[0], [1], [2]], dtype=dtypes.int32) - loss = loss_ops.sparse_softmax_cross_entropy(logits, labels) - self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value') - self.assertAlmostEqual(loss.eval(), 0.0, 3) - - def testAllCorrectInt64Labels(self): - with self.cached_session(): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[0], [1], [2]], dtype=dtypes.int64) - loss = loss_ops.sparse_softmax_cross_entropy(logits, labels) - self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value') - self.assertAlmostEqual(loss.eval(), 0.0, 3) - - def testAllCorrectNonColumnLabels(self): - with self.cached_session(): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([0, 1, 2]) - loss = loss_ops.sparse_softmax_cross_entropy(logits, labels) - self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value') - self.assertAlmostEqual(loss.eval(), 0.0, 3) - - def testAllWrongInt32Labels(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[2], [0], [1]], dtype=dtypes.int32) - - with self.cached_session(): - loss = loss_ops.sparse_softmax_cross_entropy(logits, labels) - self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value') - self.assertAlmostEqual(loss.eval(), 10.0, 3) - - def testAllWrongInt64Labels(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[2], [0], [1]], dtype=dtypes.int64) - - with self.cached_session(): - loss = loss_ops.sparse_softmax_cross_entropy(logits, labels) - self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value') - self.assertAlmostEqual(loss.eval(), 10.0, 3) - - def testAllWrongNonColumnLabels(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([2, 0, 1]) - - with self.cached_session(): - loss = loss_ops.sparse_softmax_cross_entropy(logits, labels) - self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value') - self.assertAlmostEqual(loss.eval(), 10.0, 3) - - def testNonZeroLossWithPythonScalarWeight(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[2], [0], [1]]) - weights = 2.3 - with self.cached_session(): - loss = loss_ops.sparse_softmax_cross_entropy(logits, labels, weights) - self.assertAlmostEqual(weights * 10.0, loss.eval(), 3) - - def testNonZeroLossWithScalarTensorWeight(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[2], [0], [1]]) - weights = 2.3 - with self.cached_session(): - loss = loss_ops.sparse_softmax_cross_entropy( - logits, labels, constant_op.constant(weights)) - self.assertAlmostEqual(weights * 10.0, loss.eval(), 3) - - def testNonZeroLossWithOneDimBatchSpecificWeights(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[2], [0], [1]]) - weights = constant_op.constant([1.2, 3.4, 5.6], shape=[3]) - with self.cached_session(): - loss = loss_ops.sparse_softmax_cross_entropy(logits, labels, weights) - self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, loss.eval(), 3) - - def testNonZeroLossWithColumnWeights(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[2], [0], [1]]) - weights = constant_op.constant([[1.2], [3.4], [5.6]]) - with self.cached_session(): - loss = loss_ops.sparse_softmax_cross_entropy(logits, labels, weights) - self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, loss.eval(), 3) - - def testAllWrongAllWeightsMissing(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[2], [0], [1]]) - weights = constant_op.constant([0, 0, 0], shape=[3]) - with self.cached_session(): - loss = loss_ops.sparse_softmax_cross_entropy(logits, labels, weights) - self.assertAlmostEqual(0.0, loss.eval(), 3) - - def testSomeWeightsMissing(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([[2], [0], [1]]) - weights = constant_op.constant([1.2, 0, 0], shape=[3]) - with self.cached_session(): - loss = loss_ops.sparse_softmax_cross_entropy(logits, labels, weights) - self.assertAlmostEqual(12.0, loss.eval(), 3) - - def testMeasurementSpecificWeightsRaisesException(self): - with self.cached_session(): - logits = constant_op.constant([[100.0, -100.0, -100.0], - [-100.0, 100.0, -100.0], - [-100.0, -100.0, 100.0]]) - labels = constant_op.constant([[0], [1], [2]]) - weights = constant_op.constant([[3, 4, 5], [2, 6, 0], [8, 0, 1]]) - - with self.assertRaises(ValueError): - loss_ops.sparse_softmax_cross_entropy( - logits, labels, weights=weights).eval() - - def testInconsistentWeightSizeRaisesException(self): - """The weight tensor has incorrect number of elements.""" - with self.cached_session(): - logits = constant_op.constant([[100.0, -100.0, -100.0], - [-100.0, 100.0, -100.0], - [-100.0, -100.0, 100.0]]) - labels = constant_op.constant([[0], [1], [2]]) - weights = constant_op.constant([1.2, 3.4, 5.6, 7.8]) - - with self.assertRaises(ValueError): - loss_ops.sparse_softmax_cross_entropy( - logits, labels, weights=weights).eval() - - def testInconsistentLabelSizeRaisesException(self): - """The label tensor has incorrect number of elements.""" - with self.cached_session(): - logits = constant_op.constant([[100.0, -100.0, -100.0], - [-100.0, 100.0, -100.0], - [-100.0, -100.0, 100.0]]) - labels = constant_op.constant([[0], [1], [2], [3]]) - weights = constant_op.constant([1.2, 3.4, 5.6]) - - with self.assertRaises(ValueError): - loss_ops.sparse_softmax_cross_entropy( - logits, labels, weights=weights).eval() - - def testInconsistentWeightShapeRaisesException(self): - """The weight tensor has incorrect shape.""" - with self.cached_session(): - logits = constant_op.constant([[100.0, -100.0, -100.0, -100.0], - [-100.0, 100.0, -100.0, -100.0], - [-100.0, -100.0, 100.0, -100.0], - [-100.0, -100.0, -100.0, 100.0]]) - labels = constant_op.constant([[0], [1], [2], [3]]) - weights = constant_op.constant([[1.2, 3.4], [5.6, 7.8]]) - - with self.assertRaises(ValueError): - loss_ops.sparse_softmax_cross_entropy( - logits, labels, weights=weights).eval() - - def testInconsistentLabelShapeRaisesException(self): - """The label tensor has incorrect shape.""" - with self.cached_session(): - logits = constant_op.constant([[100.0, -100.0, -100.0, -100.0], - [-100.0, 100.0, -100.0, -100.0], - [-100.0, -100.0, 100.0, -100.0], - [-100.0, -100.0, -100.0, 100.0]]) - labels = constant_op.constant([[0, 1], [2, 3]]) - weights = constant_op.constant([1.2, 3.4, 5.6, 7.8]) - - with self.assertRaises(ValueError): - loss_ops.sparse_softmax_cross_entropy( - logits, labels, weights=weights).eval() - - def testLossWithDynamicallyShapedWeights1D(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([2, 0, 1]) - weights = [2.3, 2.4, 2.5] - weights_placeholder = array_ops.placeholder( - dtypes.float32, shape=[None]) - loss = loss_ops.sparse_softmax_cross_entropy( - logits, labels, weights_placeholder) - with self.cached_session() as sess: - loss = sess.run(loss, {weights_placeholder: weights}) - self.assertAlmostEqual(np.average(weights) * 10.0, loss, 3) - - def testLossWithDynamicallyShapedWeights2D(self): - logits = constant_op.constant([[10.0, 0.0, 0.0], - [0.0, 10.0, 0.0], - [0.0, 0.0, 10.0]]) - labels = constant_op.constant([2, 0, 1]) - weights = [[2.3], [2.4], [2.5]] - weights_placeholder = array_ops.placeholder( - dtypes.float32, shape=[None, None]) - loss = loss_ops.sparse_softmax_cross_entropy( - logits, labels, weights_placeholder) - with self.cached_session() as sess: - loss = sess.run(loss, {weights_placeholder: weights}) - self.assertAlmostEqual(np.average(weights) * 10.0, loss, 3) - - -class SigmoidCrossEntropyLossTest(test.TestCase): - - def testAllCorrectSigmoid(self): - with self.cached_session(): - logits = constant_op.constant([[100.0, -100.0, -100.0], - [-100.0, 100.0, -100.0], - [-100.0, -100.0, 100.0]]) - labels = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) - loss = loss_ops.sigmoid_cross_entropy(logits, labels) - self.assertEquals(loss.op.name, 'sigmoid_cross_entropy_loss/value') - self.assertAlmostEqual(0.0, loss.eval(), 3) - - def testLossWithSingleDimPlaceholderForLogitsAndWeights1(self): - logits = array_ops.placeholder(dtypes.float32, shape=(None, 1)) - labels = array_ops.placeholder(dtypes.float32, shape=(None, 1)) - weights = array_ops.ones_like(logits, dtype=dtypes.float32) - - loss = loss_ops.sigmoid_cross_entropy(logits, labels, weights) - - with self.cached_session() as sess: - loss = sess.run(loss, - feed_dict={ - logits: np.ones((32, 1)), - labels: np.ones((32, 1)), - }) - self.assertAlmostEqual(0.313, loss, 3) - - def testLossWithSingleDimPlaceholderForLogitsAndWeights2(self): - logits = array_ops.placeholder(dtypes.float32, shape=(None, 2)) - labels = array_ops.placeholder(dtypes.float32, shape=(None, 2)) - weights = array_ops.ones_like(logits, dtype=dtypes.float32) - - loss = loss_ops.sigmoid_cross_entropy(logits, labels, weights) - - with self.cached_session() as sess: - loss = sess.run(loss, - feed_dict={ - logits: np.ones((32, 2)), - labels: np.ones((32, 2)), - }) - self.assertAlmostEqual(0.313, loss, 3) - - def testAllWrongSigmoid(self): - with self.cached_session(): - logits = constant_op.constant([[100.0, -100.0, -100.0], - [-100.0, 100.0, -100.0], - [-100.0, -100.0, 100.0]]) - labels = constant_op.constant([[0, 0, 1], - [1, 0, 0], - [0, 1, 0]]) - loss = loss_ops.sigmoid_cross_entropy(logits, labels) - self.assertEquals(loss.op.name, 'sigmoid_cross_entropy_loss/value') - self.assertAlmostEqual(loss.eval(), 600.0 / 9.0, 3) - - def testAllWrongSigmoidWithMeasurementSpecificWeights(self): - with self.cached_session(): - logits = constant_op.constant([[100.0, -100.0, -100.0], - [-100.0, 100.0, -100.0], - [-100.0, -100.0, 100.0]]) - labels = constant_op.constant([[0, 0, 1], - [1, 0, 0], - [0, 1, 0]]) - weights = constant_op.constant([[3, 4, 5], - [2, 6, 0], - [8, 0, 1]]) - loss = loss_ops.sigmoid_cross_entropy(logits, labels, weights) - self.assertEquals(loss.op.name, 'sigmoid_cross_entropy_loss/value') - self.assertAlmostEqual(1700.0 / 7.0, loss.eval(), 3) - - def testMultiCorrectSigmoid(self): - logits = constant_op.constant([[100.0, -100.0, 100.0], - [100.0, 100.0, -100.0], - [-100.0, 100.0, 100.0]]) - labels = constant_op.constant([[1, 0, 1], - [1, 1, 0], - [0, 1, 1]]) - loss = loss_ops.sigmoid_cross_entropy(logits, labels) - self.assertEquals(loss.op.name, 'sigmoid_cross_entropy_loss/value') - - with self.cached_session(): - self.assertAlmostEqual(loss.eval(), 0.0, 3) - - def testSigmoidLabelSmoothingCorrect(self): - with self.cached_session(): - logits = constant_op.constant([[100.0, -100.0, -100.0]]) - labels = constant_op.constant([[1, 0, 1]]) - # Sigmoid cross entropy loss is: - # max(x,0) - x*z + log(1 + exp(-abs(x))) - # The new labels are: - # z' = z * (1 - L) + 0.5 L - # 1 -> 1 - 0.5 L - # 0 -> 0.5 L - # here we expect: - # 1/3 * (100 - 100 * (1 - 0.5 L) + 0 - # + 0 + 100 * (0.5 L) + 0 - # + 0 + 100 * (1 - 0.5 L) + 0) - # = 1/3 * (100 + 50 L) - label_smoothing = 0.1 - loss = loss_ops.sigmoid_cross_entropy( - logits, labels, label_smoothing=label_smoothing) - self.assertEquals(loss.op.name, 'sigmoid_cross_entropy_loss/value') - expected_value = (100.0 + 50.0 * label_smoothing) / 3.0 - self.assertAlmostEqual(loss.eval(), expected_value, 3) - - def testSigmoidLabelSmoothingEqualsSoftmaxTwoLabel(self): - with self.cached_session(): - label_smoothing = 0.1 - sigmoid_logits = constant_op.constant([[100.0, -100.0, -100.0]]) - sigmoid_labels = constant_op.constant([[1, 0, 1]]) - sigmoid_loss = loss_ops.sigmoid_cross_entropy( - sigmoid_logits, sigmoid_labels, label_smoothing=label_smoothing) - - softmax_logits = constant_op.constant( - [[0.0, 100.0], [100.0, 0.0], [100.0, 0.0]]) - softmax_labels = constant_op.constant([[0, 1], [1, 0], [0, 1]]) - softmax_loss = loss_ops.softmax_cross_entropy( - softmax_logits, softmax_labels, label_smoothing=label_smoothing) - self.assertAlmostEqual(sigmoid_loss.eval(), softmax_loss.eval(), 3) - - -class LogLossTest(test.TestCase): - - def setUp(self): - predictions = np.asarray([.9, .2, .2, .8, .4, .6]).reshape((2, 3)) - labels = np.asarray([1.0, 0.0, 1.0, 1.0, 0.0, 0.0]).reshape((2, 3)) - - self._np_predictions = predictions - self._np_labels = labels - - epsilon = 1e-7 - self._expected_losses = np.multiply( - labels, np.log(predictions + epsilon)) + np.multiply( - 1 - labels, np.log(1 - predictions + epsilon)) - - self._predictions = constant_op.constant(predictions) - self._labels = constant_op.constant(labels) - - def testValueErrorThrownWhenWeightIsNone(self): - with self.cached_session(): - with self.assertRaises(ValueError): - loss_ops.log_loss(self._labels, self._labels, weights=None) - - def testAllCorrectNoLossWeight(self): - loss = loss_ops.log_loss(self._labels, self._labels) - with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) - - def testAllCorrectNoLossWeightWithPlaceholder(self): - tf_predictions = array_ops.placeholder( - dtypes.float32, shape=self._np_labels.shape) - loss = loss_ops.log_loss(tf_predictions, self._labels) - with self.cached_session(): - self.assertAlmostEqual( - 0.0, loss.eval(feed_dict={tf_predictions: self._np_labels}), 3) - - def testNonZeroLoss(self): - loss = loss_ops.log_loss(self._predictions, self._labels) - with self.cached_session(): - self.assertAlmostEqual(-np.sum(self._expected_losses) / 6.0, - loss.eval(), 3) - - def testNonZeroLossWithPythonScalarWeight(self): - weights = 2.3 - loss = loss_ops.log_loss(self._predictions, self._labels, weights) - with self.cached_session(): - self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0, - loss.eval(), 3) - - def testNonZeroLossWithScalarTensorWeight(self): - weights = 2.3 - loss = loss_ops.log_loss(self._predictions, self._labels, - constant_op.constant(weights)) - with self.cached_session(): - self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0, - loss.eval(), 3) - - def testNonZeroLossWithScalarTensorWeightAndPlaceholder(self): - tf_predictions = array_ops.placeholder( - dtypes.float32, shape=self._np_predictions.shape) - weights = 2.3 - loss = loss_ops.log_loss(tf_predictions, self._labels, - constant_op.constant(weights)) - with self.cached_session() as sess: - loss = sess.run(loss, feed_dict={tf_predictions: self._np_predictions}) - self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0, - loss, 3) - - def testNonZeroLossWithScalarTensorWeightAndPlaceholderWithRankOnly(self): - tf_predictions = array_ops.placeholder(dtypes.float32, shape=[None, None]) - weights = 2.3 - loss = loss_ops.log_loss(tf_predictions, self._labels, - constant_op.constant(weights)) - with self.cached_session() as sess: - loss = sess.run(loss, feed_dict={tf_predictions: self._np_predictions}) - self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0, - loss, 3) - - def testNonZeroLossWithOneDimBatchSpecificWeights(self): - weights = constant_op.constant([1.2, 3.4], shape=[2]) - expected_losses = np.multiply( - self._expected_losses, - np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3))) - loss = loss_ops.log_loss(self._predictions, self._labels, weights) - with self.cached_session(): - self.assertAlmostEqual(-np.sum(expected_losses) / 6.0, loss.eval(), 3) - - def testNonZeroLossWithOneDimBatchSpecificWeightsSomeZero(self): - weights = constant_op.constant([1.2, 0], shape=[2]) - expected_losses = np.multiply(self._expected_losses, - np.asarray([1.2, 1.2, 1.2, 0, 0, 0]).reshape( - (2, 3))) - loss = loss_ops.log_loss(self._predictions, self._labels, weights) - with self.cached_session(): - self.assertAlmostEqual(-np.sum(expected_losses) / 3.0, loss.eval(), 3) - - def testNonZeroLossWithTwoDimBatchSpecificWeightsSomeZero(self): - weights = constant_op.constant([1.2, 0], shape=[2, 1]) - expected_losses = np.multiply(self._expected_losses, - np.asarray([1.2, 1.2, 1.2, 0, 0, 0]).reshape( - (2, 3))) - loss = loss_ops.log_loss(self._predictions, self._labels, weights) - with self.cached_session(): - self.assertAlmostEqual(-np.sum(expected_losses) / 3.0, loss.eval(), 3) - - def testWeightsWithSameNumDimsButWrongShapeThrowsException(self): - weights = constant_op.constant(np.random.normal(size=(2, 4)), shape=[2, 4]) - with self.cached_session(): - with self.assertRaises(ValueError): - loss_ops.log_loss(self._predictions, self._labels, weights) - - def testNonZeroLossWithMeasurementSpecificWeights(self): - weights = np.array([3, 6, 5, 0, 4, 2]).reshape((2, 3)) - expected_losses = np.multiply(self._expected_losses, weights) - - loss = loss_ops.log_loss( - self._predictions, - self._labels, - constant_op.constant( - weights, shape=(2, 3))) - with self.cached_session(): - self.assertAlmostEqual(-np.sum(expected_losses) / 5.0, loss.eval(), 3) - - def testNonZeroLossWithMeasurementSpecificWeightsWithPlaceholder(self): - weights = np.array([3, 6, 5, 0, 4, 2]).reshape((2, 3)) - expected_losses = np.multiply(self._expected_losses, weights) - - tf_predictions = array_ops.placeholder(dtypes.float32, shape=[2, 3]) - loss = loss_ops.log_loss( - tf_predictions, - self._labels, - constant_op.constant( - weights, shape=(2, 3))) - - with self.cached_session() as sess: - loss = sess.run(loss, feed_dict={tf_predictions: self._np_predictions}) - self.assertAlmostEqual(-np.sum(expected_losses) / 5.0, loss, 3) - - def testNonZeroLossWithSampleSpecificWeightsMostZero(self): - weights = np.array([0, 0, 0, 0, 0, 2]).reshape((2, 3)) - expected_losses = np.multiply(self._expected_losses, weights) - - loss = loss_ops.log_loss( - self._predictions, - self._labels, - constant_op.constant( - weights, shape=(2, 3))) - with self.cached_session(): - self.assertAlmostEqual(-np.sum(expected_losses), loss.eval(), 3) - - def testNonZeroLossWithSampleSpecificWeightsMostZeroWithPlaceholder(self): - weights = np.array([0, 0, 0, 0, 0, 2]).reshape((2, 3)) - expected_losses = np.multiply(self._expected_losses, weights) - - tf_predictions = array_ops.placeholder(dtypes.float32, shape=[2, 3]) - tf_weights = constant_op.constant(weights, shape=(2, 3)) - loss = loss_ops.log_loss(tf_predictions, self._labels, tf_weights) - - with self.cached_session() as sess: - loss = sess.run(loss, feed_dict={tf_predictions: self._np_predictions}) - self.assertAlmostEqual(-np.sum(expected_losses), loss, 3) - - def testLossWithSampleSpecificWeightsAllZero(self): - tf_weights = array_ops.zeros(shape=(2, 3)) - loss = loss_ops.log_loss(self._predictions, self._labels, tf_weights) - with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) - - -class HingeLossTest(test.TestCase): - - def testIncompatibleShapes(self): - with self.cached_session(): - logits = constant_op.constant([[-1.0], [2.1]]) - labels = constant_op.constant([0.0, 1.0]) - with self.assertRaises(ValueError): - _ = loss_ops.hinge_loss(logits, labels).eval() - - def testAllOutsideMargin(self): - with self.cached_session(): - logits = constant_op.constant([1.2, -1.4, -1.0, 2.1]) - labels = constant_op.constant([1.0, 0.0, 0.0, 1.0]) - loss = loss_ops.hinge_loss(logits, labels) - self.assertAllClose(loss.eval(), [0.0, 0.0, 0.0, 0.0], atol=1e-3) - - def testSomeInsideMargin(self): - with self.cached_session(): - logits = constant_op.constant([[-0.7], [-1.4], [1.4], [0.6]]) - labels = constant_op.constant([[0.0], [0.0], [1.0], [1.0]]) - loss = loss_ops.hinge_loss(logits, labels) - # Examples 1 and 4 are on the correct side of the hyperplane but within - # the margin so they incur some (small) loss. - self.assertAllClose(loss.eval(), [[0.3], [0.0], [0.0], [0.4]], atol=1e-3) - - def testSomeMisclassified(self): - with self.cached_session(): - logits = constant_op.constant([[[1.2], [0.4], [-1.0], [-1.1]]]) - labels = constant_op.constant([[[1.0], [0.0], [0.0], [1.0]]]) - loss = loss_ops.hinge_loss(logits, labels) - # Examples 2 and 4 are on the wrong side of the hyperplane so they incur - # some (fairly large) loss. - self.assertAllClose( - loss.eval(), [[[0.0], [1.4], [0.0], [2.1]]], atol=1e-3) - - -class MeanSquaredErrorTest(test.TestCase): - - def setUp(self): - self._predictions = constant_op.constant([4, 8, 12, 8, 1, 3], shape=(2, 3)) - self._labels = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) - - def testValueErrorThrownWhenWeightIsNone(self): - with self.cached_session(): - with self.assertRaises(ValueError): - loss_ops.mean_squared_error( - self._predictions, self._predictions, weights=None) - - def testAllCorrectNoLossWeight(self): - loss = loss_ops.mean_squared_error(self._predictions, self._predictions) - with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) - - def testNonZeroLoss(self): - loss = loss_ops.mean_squared_error(self._predictions, self._labels) - with self.cached_session(): - self.assertAlmostEqual(49.5, loss.eval(), 3) - - def testNonZeroLossWithPythonScalarWeight(self): - weights = 2.3 - loss = loss_ops.mean_squared_error(self._predictions, self._labels, weights) - with self.cached_session(): - self.assertAlmostEqual(49.5 * weights, loss.eval(), 3) - - def testNonZeroLossWithScalarTensorWeight(self): - weights = 2.3 - loss = loss_ops.mean_squared_error(self._predictions, self._labels, - constant_op.constant(weights)) - with self.cached_session(): - self.assertAlmostEqual(49.5 * weights, loss.eval(), 3) - - def testNonZeroLossWithOneDimBatchSpecificWeights(self): - weights = constant_op.constant([1.2, 3.4], shape=[2,]) - loss = loss_ops.mean_squared_error(self._predictions, self._labels, weights) - with self.cached_session(): - self.assertAlmostEqual(767.8 / 6.0, loss.eval(), 3) - - def testNonZeroLossWithTwoDimBatchSpecificWeights(self): - weights = constant_op.constant([1.2, 3.4], shape=[2, 1]) - loss = loss_ops.mean_squared_error(self._predictions, self._labels, weights) - with self.cached_session(): - self.assertAlmostEqual(767.8 / 6.0, loss.eval(), 3) - - def testNonZeroLossWithSampleSpecificWeights(self): - weights = constant_op.constant([3, 6, 5, 0, 4, 2], shape=[2, 3]) - loss = loss_ops.mean_squared_error(self._predictions, self._labels, weights) - with self.cached_session(): - self.assertAlmostEqual(587 / 5.0, loss.eval(), 3) - - def testNonZeroLossWithSampleSpecificWeightsMostZero(self): - weights = constant_op.constant([0, 0, 0, 0, 0, 2], shape=[2, 3]) - loss = loss_ops.mean_squared_error(self._predictions, self._labels, weights) - with self.cached_session(): - self.assertAlmostEqual(18.0, loss.eval(), 3) - - def testLossWithSampleSpecificWeightsAllZero(self): - weights = array_ops.zeros((2, 3)) - loss = loss_ops.mean_squared_error(self._predictions, self._labels, weights) - with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) - - -class MeanPairwiseSquaresErrorTest(test.TestCase): - - def setUp(self): - self._predictions = np.array([[4, 8, 12], [8, 1, 3]]) - self._labels = np.array([[1, 9, 2], [-5, -5, 7]]) - - batch_size, dims = self._labels.shape - - # Compute the expected loss 'manually'. - total = np.zeros((batch_size, 1)) - for b in range(batch_size): - for i in range(dims): - for j in range(dims): - x = self._predictions[b, i].item() - self._predictions[b, j].item() - y = self._labels[b, i].item() - self._labels[b, j].item() - tmp = (x - y) * (x - y) - total[b] += tmp - - self._expected_losses = np.divide(total, 9.0) - - def testValueErrorThrownWhenWeightIsNone(self): - with self.cached_session(): - with self.assertRaises(ValueError): - loss_ops.mean_pairwise_squared_error( - predictions=constant_op.constant(self._labels), - labels=constant_op.constant(self._labels), - weights=None) - - def testAllCorrectNoLossWeight(self): - loss = loss_ops.mean_pairwise_squared_error( - predictions=constant_op.constant(self._labels), - labels=constant_op.constant(self._labels)) - with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) - - def testNonZeroLoss(self): - loss = loss_ops.mean_pairwise_squared_error( - predictions=constant_op.constant(self._predictions), - labels=constant_op.constant(self._labels)) - with self.cached_session(): - self.assertAlmostEqual(np.sum(self._expected_losses), loss.eval(), 3) - - def testGradientWithZeroWeight(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - - inputs = array_ops.ones((2, 3)) - weights = variable_scope.get_variable( - 'weights', - shape=[3, 4], - initializer=init_ops.truncated_normal_initializer()) - predictions = math_ops.matmul(inputs, weights) - - optimizer = momentum_lib.MomentumOptimizer( - learning_rate=0.001, momentum=0.9) - loss = loss_ops.mean_pairwise_squared_error(predictions, predictions, 0) - - gradients_to_variables = optimizer.compute_gradients(loss) - - init_op = variables.global_variables_initializer() - - with self.cached_session() as sess: - sess.run(init_op) - for grad, _ in gradients_to_variables: - np_grad = sess.run(grad) - self.assertFalse(np.isnan(np_grad).any()) - - def testNonZeroLossWithPythonScalarWeight(self): - weights = 2.3 - loss = loss_ops.mean_pairwise_squared_error( - predictions=constant_op.constant(self._predictions), - labels=constant_op.constant(self._labels), - weights=weights) - with self.cached_session(): - self.assertAlmostEqual(weights * np.sum(self._expected_losses), - loss.eval(), 3) - - def testNonZeroLossWithScalarTensorWeight(self): - weights = 2.3 - loss = loss_ops.mean_pairwise_squared_error( - predictions=constant_op.constant(self._predictions), - labels=constant_op.constant(self._labels), - weights=constant_op.constant(weights)) - with self.cached_session(): - self.assertAlmostEqual(weights * np.sum(self._expected_losses), - loss.eval(), 3) - - def testNonZeroLossWithScalarZeroWeight(self): - weights = 0 - loss = loss_ops.mean_pairwise_squared_error( - predictions=constant_op.constant(self._predictions), - labels=constant_op.constant(self._labels), - weights=constant_op.constant(weights)) - with self.cached_session(): - self.assertAlmostEqual(0, loss.eval(), 3) - - def testNonZeroLossWithScalarTensorWeightWithPlaceholder(self): - weights = 2.3 - tf_predictions = array_ops.placeholder( - dtypes.float32, shape=self._predictions.shape) - tf_labels = array_ops.placeholder(dtypes.float32, shape=self._labels.shape) - loss = loss_ops.mean_pairwise_squared_error( - predictions=tf_predictions, - labels=tf_labels, - weights=constant_op.constant(weights)) - with self.cached_session() as sess: - loss = sess.run(loss, - feed_dict={ - tf_predictions: self._predictions, - tf_labels: self._labels, - }) - self.assertAlmostEqual(weights * np.sum(self._expected_losses), loss, 3) - - def testNonZeroLossWithOneDimBatchSpecificWeights(self): - weights = np.asarray([2.0, 1.0]).reshape((2, 1)) - expected_losses = np.multiply(weights, self._expected_losses) - - loss = loss_ops.mean_pairwise_squared_error( - predictions=constant_op.constant(self._predictions), - labels=constant_op.constant(self._labels), - weights=constant_op.constant( - weights, shape=[2])) - with self.cached_session(): - self.assertAlmostEqual(np.sum(expected_losses), loss.eval(), 3) - - def testZeroLossWithOneDimBatchZeroWeights(self): - weights = np.asarray([0.0, 0.0]).reshape((2, 1)) - loss = loss_ops.mean_pairwise_squared_error( - predictions=constant_op.constant(self._predictions), - labels=constant_op.constant(self._labels), - weights=constant_op.constant( - weights, shape=[2])) - with self.cached_session(): - self.assertAlmostEqual(0, loss.eval(), 3) - - def testNonZeroLossWithOneDimBatchSpecificWeightsAndPlaceholders(self): - weights = np.asarray([1.2, 3.4]).reshape((2, 1)) - expected_losses = np.multiply(weights, self._expected_losses) - - tf_predictions = array_ops.placeholder( - dtypes.float32, shape=self._predictions.shape) - tf_labels = array_ops.placeholder(dtypes.int32, shape=self._labels.shape) - loss = loss_ops.mean_pairwise_squared_error( - predictions=tf_predictions, - labels=tf_labels, - weights=constant_op.constant( - weights, shape=[2])) - - with self.cached_session() as sess: - loss = sess.run(loss, - feed_dict={ - tf_predictions: self._predictions, - tf_labels: self._labels, - }) - self.assertAlmostEqual(np.sum(expected_losses), loss, 3) - - def testLossWithAllZeroBatchSpecificWeights(self): - weights = np.zeros((2, 1)) - loss = loss_ops.mean_pairwise_squared_error( - predictions=constant_op.constant(self._predictions), - labels=constant_op.constant(self._labels), - weights=constant_op.constant( - weights, shape=[2])) - with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) - - def testLossIsAssociativeAcrossBatchElements(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - - height = 3 - width = 4 - shape = (1, height, width, 1) - - labels0 = random_ops.random_uniform( - shape, minval=0, maxval=1, dtype=dtypes.float32) - predictions0 = random_ops.random_uniform( - shape, minval=0, maxval=1, dtype=dtypes.float32) - - labels1 = random_ops.random_uniform( - shape, minval=0, maxval=1, dtype=dtypes.float32) - predictions1 = random_ops.random_uniform( - shape, minval=0, maxval=1, dtype=dtypes.float32) - - loss0 = loss_ops.mean_pairwise_squared_error( - predictions=predictions0, - labels=labels0) - loss1 = loss_ops.mean_pairwise_squared_error( - predictions=predictions1, - labels=labels1) - loss0_1 = loss_ops.mean_pairwise_squared_error( - predictions=array_ops.concat([predictions0, predictions1], 0), - labels=array_ops.concat([labels0, labels1], 0)) - - with self.cached_session() as session: - loss0, loss1, loss0_1 = session.run([loss0, loss1, loss0_1]) - - self.assertTrue(loss0 > 0) - self.assertTrue(loss1 > 0) - self.assertAlmostEqual(loss0 + loss1, loss0_1, 5) - - -class CosineDistanceLossTest(test.TestCase): - - def setUp(self): - self._predictions = np.asarray([ - [1, 0, 0], # Batch 1 - [0, 0, -1], - [1, 0, 0], # Batch 2 - [1, 0, 0], - [0, 0, -1], # Batch 3 - [1, 0, 0] - ]).reshape((3, 2, 3)) - - self._labels = np.asarray([[1, 0, 0], - [0, 0, 1], - [0, 1, 0], - [1, 0, 0], - [0, 0, 1], - [0, 1, 0]]).reshape((3, 2, 3)) - - def testValueErrorThrownWhenWeightIsNone(self): - with self.cached_session(): - with self.assertRaises(ValueError): - loss_ops.cosine_distance( - predictions=constant_op.constant(self._labels), - labels=constant_op.constant(self._labels), - dim=2, - weights=None) - - def testAllCorrectNoWeights(self): - loss = loss_ops.cosine_distance( - predictions=constant_op.constant(self._labels), - labels=constant_op.constant(self._labels), - dim=2) - with self.cached_session(): - self.assertAlmostEqual(0, loss.eval(), 5) - - def testPartiallyCorrectWithIntegerValues(self): - loss = loss_ops.cosine_distance( - predictions=constant_op.constant(self._predictions), - labels=constant_op.constant(self._labels), - dim=2) - with self.cached_session(): - self.assertAlmostEqual(1, loss.eval(), 5) - - def testPartiallyCorrectFloatingPointValues(self): - predictions = np.matrix( - ('0.819031913261206 0.567041924552012 0.087465312324590;' - '-0.665139432070255 -0.739487441769973 -0.103671883216994;' - '0.707106781186548 -0.707106781186548 0')) - labels = np.matrix(('0.819031913261206 0.567041924552012 0.087465312324590;' - '0.665139432070255 0.739487441769973 0.103671883216994;' - '0.707106781186548 0.707106781186548 0')) - - tf_preds = constant_op.constant( - predictions, shape=(3, 1, 3), dtype=dtypes.float32) - tf_labels = constant_op.constant( - labels, shape=(3, 1, 3), dtype=dtypes.float32) - loss = loss_ops.cosine_distance(tf_preds, tf_labels, dim=2) - - with self.cached_session(): - self.assertAlmostEqual(1.0, loss.eval(), 5) - - def testSampleSpecificWeights(self): - loss = loss_ops.cosine_distance( - predictions=constant_op.constant(self._predictions), - labels=constant_op.constant(self._labels), - dim=2, - weights=constant_op.constant([1, 0, 0])) - with self.cached_session(): - self.assertEqual(1.0, loss.eval()) - - def testMeasurementSpecificWeights(self): - loss = loss_ops.cosine_distance( - predictions=constant_op.constant(self._predictions), - labels=constant_op.constant(self._labels), - dim=2, - weights=constant_op.constant( - [1, 0, 0, 1, 1, 1], shape=(3, 2))) - with self.cached_session(): - self.assertEqual(3.0 / 4.0, loss.eval()) - - def testValueErrorThrownWithShapelessPlaceholder(self): - tf_predictions = array_ops.placeholder(dtypes.float32) - with self.cached_session(): - with self.assertRaises(ValueError): - loss_ops.cosine_distance( - predictions=tf_predictions, - labels=constant_op.constant(self._labels), - dim=2, - weights=constant_op.constant( - [1, 0, 0, 1, 1, 1], shape=(3, 2))) - - def testMeasurementSpecificWeightsWithPlaceholderWithShape(self): - tf_predictions = array_ops.placeholder( - dtypes.float32, shape=self._labels.shape) - loss = loss_ops.cosine_distance( - predictions=tf_predictions, - labels=constant_op.constant(self._labels), - dim=2, - weights=constant_op.constant( - [1, 0, 0, 1, 1, 1], shape=(3, 2))) - with self.cached_session() as sess: - loss = sess.run(loss, feed_dict={tf_predictions: self._predictions}) - self.assertEqual(3.0 / 4.0, loss) - - def testZeroLossWhenAllSampleSpecificWeightsAreZero(self): - loss = loss_ops.cosine_distance( - predictions=constant_op.constant(self._predictions), - labels=constant_op.constant(self._labels), - dim=2, - weights=array_ops.zeros((3,))) - with self.cached_session(): - self.assertEqual(0, loss.eval()) - - def testZeroLossWhenAllMeasurementSpecificWeightsAreZero(self): - loss = loss_ops.cosine_distance( - predictions=constant_op.constant(self._predictions), - labels=constant_op.constant(self._labels), - dim=2, - weights=array_ops.zeros((3, 2))) - with self.cached_session(): - self.assertEqual(0, loss.eval()) - - -class ComputeWeightedLossTest(test.TestCase): - - def testHingeLoss(self): - logits = constant_op.constant([1.2, 0.4, -1.0, -1.1]) - labels = constant_op.constant([1.0, 0.0, 0.0, 1.0]) - losses = loss_ops.hinge_loss(logits, labels) - self.assertFalse(loss_ops.get_losses()) - loss = loss_ops.compute_weighted_loss(losses) - self.assertTrue(loss_ops.get_losses()) - with self.cached_session(): - self.assertAllClose(losses.eval(), [0.0, 1.4, 0.0, 2.1], atol=1e-3) - self.assertAllClose(loss.eval(), 3.5 / 4.0, atol=1e-3) - - -class AddLossTest(test.TestCase): - - def testAddExternalLoss(self): - logits = constant_op.constant([[1.2, 0.4, -1.0, -1.1]]) - labels = constant_op.constant([[1.0, 0.0, 0.0, 1.0]]) - losses = loss_ops.hinge_loss(logits, labels) - self.assertFalse(loss_ops.get_losses()) - loss_ops.add_loss(math_ops.reduce_mean(losses)) - self.assertTrue(loss_ops.get_losses()) - total_loss = loss_ops.get_total_loss() - with self.cached_session(): - self.assertAllClose(losses.eval(), [[0.0, 1.4, 0.0, 2.1]], atol=1e-3) - self.assertAllClose(total_loss.eval(), 3.5 / 4.0, atol=1e-3) - - def testNoneLossCollection(self): - logits = constant_op.constant([[1.2, 0.4, -1.0, -1.1]]) - labels = constant_op.constant([[1.0, 0.0, 0.0, 1.0]]) - losses = loss_ops.hinge_loss(logits, labels) - self.assertFalse(loss_ops.get_losses()) - loss_ops.add_loss(math_ops.reduce_mean(losses), loss_collection=None) - self.assertFalse(loss_ops.get_losses()) - with self.cached_session(): - self.assertAllClose(losses.eval(), [[0.0, 1.4, 0.0, 2.1]], atol=1e-3) - - def testNoCollectLosses(self): - logits = constant_op.constant([[1.2, 0.4, -1.0, -1.1]]) - labels = constant_op.constant([[1.0, 0.0, 0.0, 1.0]]) - self.assertFalse(loss_ops.get_losses()) - with arg_scope([loss_ops.add_loss], loss_collection=None): - loss_ops.absolute_difference(logits, labels) - loss_ops.log_loss(logits, labels) - loss_ops.mean_squared_error(logits, labels) - loss_ops.sigmoid_cross_entropy(logits, labels) - loss_ops.softmax_cross_entropy(logits, labels) - self.assertFalse(loss_ops.get_losses()) - - def testNoCollectLossesBatch2(self): - logits = constant_op.constant([[1.2, 0.4, -1.0, -1.1]] * 2) - labels = constant_op.constant([[1.0, 0.0, 0.0, 1.0]] * 2) - self.assertFalse(loss_ops.get_losses()) - with arg_scope([loss_ops.add_loss], loss_collection=None): - loss_ops.absolute_difference(logits, labels) - loss_ops.log_loss(logits, labels) - loss_ops.mean_squared_error(logits, labels) - loss_ops.sigmoid_cross_entropy(logits, labels) - loss_ops.softmax_cross_entropy(logits, labels) - self.assertFalse(loss_ops.get_losses()) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/losses/python/metric_learning/__init__.py b/tensorflow/contrib/losses/python/metric_learning/__init__.py deleted file mode 100644 index 3d93a4d0ac6..00000000000 --- a/tensorflow/contrib/losses/python/metric_learning/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for building neural network losses. - -See [Contrib Losses](https://tensorflow.org/api_guides/python/contrib.losses). -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import * -# pylint: enable=wildcard-import -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'contrastive_loss', - 'cluster_loss', - 'lifted_struct_loss', - 'npairs_loss', - 'npairs_loss_multilabel', - 'triplet_semihard_loss', -] -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops.py b/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops.py deleted file mode 100644 index 9bf2b683a02..00000000000 --- a/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops.py +++ /dev/null @@ -1,1038 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implements various metric learning losses.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import script_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.summary import summary -try: - # pylint: disable=g-import-not-at-top - from sklearn import metrics - HAS_SKLEARN = True -except ImportError: - HAS_SKLEARN = False - - -def pairwise_distance(feature, squared=False): - """Computes the pairwise distance matrix with numerical stability. - - output[i, j] = || feature[i, :] - feature[j, :] ||_2 - - Args: - feature: 2-D Tensor of size [number of data, feature dimension]. - squared: Boolean, whether or not to square the pairwise distances. - - Returns: - pairwise_distances: 2-D Tensor of size [number of data, number of data]. - """ - pairwise_distances_squared = math_ops.add( - math_ops.reduce_sum(math_ops.square(feature), axis=[1], keepdims=True), - math_ops.reduce_sum( - math_ops.square(array_ops.transpose(feature)), - axis=[0], - keepdims=True)) - 2.0 * math_ops.matmul(feature, - array_ops.transpose(feature)) - - # Deal with numerical inaccuracies. Set small negatives to zero. - pairwise_distances_squared = math_ops.maximum(pairwise_distances_squared, 0.0) - # Get the mask where the zero distances are at. - error_mask = math_ops.less_equal(pairwise_distances_squared, 0.0) - - # Optionally take the sqrt. - if squared: - pairwise_distances = pairwise_distances_squared - else: - pairwise_distances = math_ops.sqrt( - pairwise_distances_squared + - math_ops.cast(error_mask, dtypes.float32) * 1e-16) - - # Undo conditionally adding 1e-16. - pairwise_distances = math_ops.multiply( - pairwise_distances, - math_ops.cast(math_ops.logical_not(error_mask), dtypes.float32)) - - num_data = array_ops.shape(feature)[0] - # Explicitly set diagonals to zero. - mask_offdiagonals = array_ops.ones_like(pairwise_distances) - array_ops.diag( - array_ops.ones([num_data])) - pairwise_distances = math_ops.multiply(pairwise_distances, mask_offdiagonals) - return pairwise_distances - - -def contrastive_loss(labels, embeddings_anchor, embeddings_positive, - margin=1.0): - """Computes the contrastive loss. - - This loss encourages the embedding to be close to each other for - the samples of the same label and the embedding to be far apart at least - by the margin constant for the samples of different labels. - See: http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf - - Args: - labels: 1-D tf.int32 `Tensor` with shape [batch_size] of - binary labels indicating positive vs negative pair. - embeddings_anchor: 2-D float `Tensor` of embedding vectors for the anchor - images. Embeddings should be l2 normalized. - embeddings_positive: 2-D float `Tensor` of embedding vectors for the - positive images. Embeddings should be l2 normalized. - margin: margin term in the loss definition. - - Returns: - contrastive_loss: tf.float32 scalar. - """ - # Get per pair distances - distances = math_ops.sqrt( - math_ops.reduce_sum( - math_ops.squared_difference(embeddings_anchor, embeddings_positive), - 1)) - - # Add contrastive loss for the siamese network. - # label here is {0,1} for neg, pos. - return math_ops.reduce_mean( - math_ops.cast(labels, distances.dtype) * math_ops.square(distances) + - (1. - math_ops.cast(labels, distances.dtype)) * - math_ops.square(math_ops.maximum(margin - distances, 0.)), - name='contrastive_loss') - - -def masked_maximum(data, mask, dim=1): - """Computes the axis wise maximum over chosen elements. - - Args: - data: 2-D float `Tensor` of size [n, m]. - mask: 2-D Boolean `Tensor` of size [n, m]. - dim: The dimension over which to compute the maximum. - - Returns: - masked_maximums: N-D `Tensor`. - The maximized dimension is of size 1 after the operation. - """ - axis_minimums = math_ops.reduce_min(data, dim, keepdims=True) - masked_maximums = math_ops.reduce_max( - math_ops.multiply(data - axis_minimums, mask), dim, - keepdims=True) + axis_minimums - return masked_maximums - - -def masked_minimum(data, mask, dim=1): - """Computes the axis wise minimum over chosen elements. - - Args: - data: 2-D float `Tensor` of size [n, m]. - mask: 2-D Boolean `Tensor` of size [n, m]. - dim: The dimension over which to compute the minimum. - - Returns: - masked_minimums: N-D `Tensor`. - The minimized dimension is of size 1 after the operation. - """ - axis_maximums = math_ops.reduce_max(data, dim, keepdims=True) - masked_minimums = math_ops.reduce_min( - math_ops.multiply(data - axis_maximums, mask), dim, - keepdims=True) + axis_maximums - return masked_minimums - - -def triplet_semihard_loss(labels, embeddings, margin=1.0): - """Computes the triplet loss with semi-hard negative mining. - - The loss encourages the positive distances (between a pair of embeddings with - the same labels) to be smaller than the minimum negative distance among - which are at least greater than the positive distance plus the margin constant - (called semi-hard negative) in the mini-batch. If no such negative exists, - uses the largest negative distance instead. - See: https://arxiv.org/abs/1503.03832. - - Args: - labels: 1-D tf.int32 `Tensor` with shape [batch_size] of - multiclass integer labels. - embeddings: 2-D float `Tensor` of embedding vectors. Embeddings should - be l2 normalized. - margin: Float, margin term in the loss definition. - - Returns: - triplet_loss: tf.float32 scalar. - """ - # Reshape [batch_size] label tensor to a [batch_size, 1] label tensor. - lshape = array_ops.shape(labels) - assert lshape.shape == 1 - labels = array_ops.reshape(labels, [lshape[0], 1]) - - # Build pairwise squared distance matrix. - pdist_matrix = pairwise_distance(embeddings, squared=True) - # Build pairwise binary adjacency matrix. - adjacency = math_ops.equal(labels, array_ops.transpose(labels)) - # Invert so we can select negatives only. - adjacency_not = math_ops.logical_not(adjacency) - - batch_size = array_ops.size(labels) - - # Compute the mask. - pdist_matrix_tile = array_ops.tile(pdist_matrix, [batch_size, 1]) - mask = math_ops.logical_and( - array_ops.tile(adjacency_not, [batch_size, 1]), - math_ops.greater( - pdist_matrix_tile, array_ops.reshape( - array_ops.transpose(pdist_matrix), [-1, 1]))) - mask_final = array_ops.reshape( - math_ops.greater( - math_ops.reduce_sum( - math_ops.cast(mask, dtype=dtypes.float32), 1, keepdims=True), - 0.0), [batch_size, batch_size]) - mask_final = array_ops.transpose(mask_final) - - adjacency_not = math_ops.cast(adjacency_not, dtype=dtypes.float32) - mask = math_ops.cast(mask, dtype=dtypes.float32) - - # negatives_outside: smallest D_an where D_an > D_ap. - negatives_outside = array_ops.reshape( - masked_minimum(pdist_matrix_tile, mask), [batch_size, batch_size]) - negatives_outside = array_ops.transpose(negatives_outside) - - # negatives_inside: largest D_an. - negatives_inside = array_ops.tile( - masked_maximum(pdist_matrix, adjacency_not), [1, batch_size]) - semi_hard_negatives = array_ops.where( - mask_final, negatives_outside, negatives_inside) - - loss_mat = math_ops.add(margin, pdist_matrix - semi_hard_negatives) - - mask_positives = math_ops.cast( - adjacency, dtype=dtypes.float32) - array_ops.diag( - array_ops.ones([batch_size])) - - # In lifted-struct, the authors multiply 0.5 for upper triangular - # in semihard, they take all positive pairs except the diagonal. - num_positives = math_ops.reduce_sum(mask_positives) - - triplet_loss = math_ops.truediv( - math_ops.reduce_sum( - math_ops.maximum( - math_ops.multiply(loss_mat, mask_positives), 0.0)), - num_positives, - name='triplet_semihard_loss') - - return triplet_loss - - -# pylint: disable=line-too-long -def npairs_loss(labels, embeddings_anchor, embeddings_positive, - reg_lambda=0.002, print_losses=False): - """Computes the npairs loss. - - Npairs loss expects paired data where a pair is composed of samples from the - same labels and each pairs in the minibatch have different labels. The loss - has two components. The first component is the L2 regularizer on the - embedding vectors. The second component is the sum of cross entropy loss - which takes each row of the pair-wise similarity matrix as logits and - the remapped one-hot labels as labels. - - See: http://www.nec-labs.com/uploads/images/Department-Images/MediaAnalytics/papers/nips16_npairmetriclearning.pdf - - Args: - labels: 1-D tf.int32 `Tensor` of shape [batch_size/2]. - embeddings_anchor: 2-D Tensor of shape [batch_size/2, embedding_dim] for the - embedding vectors for the anchor images. Embeddings should not be - l2 normalized. - embeddings_positive: 2-D Tensor of shape [batch_size/2, embedding_dim] for the - embedding vectors for the positive images. Embeddings should not be - l2 normalized. - reg_lambda: Float. L2 regularization term on the embedding vectors. - print_losses: Boolean. Option to print the xent and l2loss. - - Returns: - npairs_loss: tf.float32 scalar. - """ - # pylint: enable=line-too-long - # Add the regularizer on the embedding. - reg_anchor = math_ops.reduce_mean( - math_ops.reduce_sum(math_ops.square(embeddings_anchor), 1)) - reg_positive = math_ops.reduce_mean( - math_ops.reduce_sum(math_ops.square(embeddings_positive), 1)) - l2loss = math_ops.multiply( - 0.25 * reg_lambda, reg_anchor + reg_positive, name='l2loss') - - # Get per pair similarities. - similarity_matrix = math_ops.matmul( - embeddings_anchor, embeddings_positive, transpose_a=False, - transpose_b=True) - - # Reshape [batch_size] label tensor to a [batch_size, 1] label tensor. - lshape = array_ops.shape(labels) - assert lshape.shape == 1 - labels = array_ops.reshape(labels, [lshape[0], 1]) - - labels_remapped = math_ops.cast( - math_ops.equal(labels, array_ops.transpose(labels)), dtypes.float32) - labels_remapped /= math_ops.reduce_sum(labels_remapped, 1, keepdims=True) - - # Add the softmax loss. - xent_loss = nn.softmax_cross_entropy_with_logits( - logits=similarity_matrix, labels=labels_remapped) - xent_loss = math_ops.reduce_mean(xent_loss, name='xentropy') - - if print_losses: - xent_loss = logging_ops.Print( - xent_loss, ['cross entropy:', xent_loss, 'l2loss:', l2loss]) - - return l2loss + xent_loss - - -def _build_multilabel_adjacency(sparse_labels): - """Builds multilabel adjacency matrix. - - As of March 14th, 2017, there's no op for the dot product between - two sparse tensors in TF. However, there is `sparse_minimum` op which is - equivalent to an AND op between two sparse boolean tensors. - This computes the dot product between two sparse boolean inputs. - - Args: - sparse_labels: List of 1-D boolean sparse tensors. - - Returns: - adjacency_matrix: 2-D dense `Tensor`. - """ - num_pairs = len(sparse_labels) - adjacency_matrix = array_ops.zeros([num_pairs, num_pairs]) - for i in range(num_pairs): - for j in range(num_pairs): - sparse_dot_product = math_ops.cast( - sparse_ops.sparse_reduce_sum(sparse_ops.sparse_minimum( - sparse_labels[i], sparse_labels[j])), - dtypes.float32) - sparse_dot_product = array_ops.expand_dims(sparse_dot_product, 0) - sparse_dot_product = array_ops.expand_dims(sparse_dot_product, 1) - one_hot_matrix = array_ops.pad(sparse_dot_product, - [[i, num_pairs-i-1], - [j, num_pairs-j-1]], 'CONSTANT') - adjacency_matrix += one_hot_matrix - - return adjacency_matrix - - -def npairs_loss_multilabel(sparse_labels, embeddings_anchor, - embeddings_positive, reg_lambda=0.002, - print_losses=False): - r"""Computes the npairs loss with multilabel data. - - Npairs loss expects paired data where a pair is composed of samples from the - same labels and each pairs in the minibatch have different labels. The loss - has two components. The first component is the L2 regularizer on the - embedding vectors. The second component is the sum of cross entropy loss - which takes each row of the pair-wise similarity matrix as logits and - the remapped one-hot labels as labels. Here, the similarity is defined by the - dot product between two embedding vectors. S_{i,j} = f(x_i)^T f(x_j) - - To deal with multilabel inputs, we use the count of label intersection - i.e. L_{i,j} = | set_of_labels_for(i) \cap set_of_labels_for(j) | - Then we normalize each rows of the count based label matrix so that each row - sums to one. - - Args: - sparse_labels: List of 1-D Boolean `SparseTensor` of dense_shape - [batch_size/2, num_classes] labels for the anchor-pos pairs. - embeddings_anchor: 2-D `Tensor` of shape [batch_size/2, embedding_dim] for - the embedding vectors for the anchor images. Embeddings should not be - l2 normalized. - embeddings_positive: 2-D `Tensor` of shape [batch_size/2, embedding_dim] for - the embedding vectors for the positive images. Embeddings should not be - l2 normalized. - reg_lambda: Float. L2 regularization term on the embedding vectors. - print_losses: Boolean. Option to print the xent and l2loss. - - Returns: - npairs_loss: tf.float32 scalar. - Raises: - TypeError: When the specified sparse_labels is not a `SparseTensor`. - """ - if False in [isinstance( - l, sparse_tensor.SparseTensor) for l in sparse_labels]: - raise TypeError( - 'sparse_labels must be a list of SparseTensors, but got %s' % str( - sparse_labels)) - - with ops.name_scope('NpairsLossMultiLabel'): - # Add the regularizer on the embedding. - reg_anchor = math_ops.reduce_mean( - math_ops.reduce_sum(math_ops.square(embeddings_anchor), 1)) - reg_positive = math_ops.reduce_mean( - math_ops.reduce_sum(math_ops.square(embeddings_positive), 1)) - l2loss = math_ops.multiply(0.25 * reg_lambda, - reg_anchor + reg_positive, name='l2loss') - - # Get per pair similarities. - similarity_matrix = math_ops.matmul( - embeddings_anchor, embeddings_positive, transpose_a=False, - transpose_b=True) - - # TODO(coreylynch): need to check the sparse values - # TODO(coreylynch): are composed only of 0's and 1's. - - multilabel_adjacency_matrix = _build_multilabel_adjacency(sparse_labels) - labels_remapped = math_ops.cast(multilabel_adjacency_matrix, dtypes.float32) - labels_remapped /= math_ops.reduce_sum(labels_remapped, 1, keepdims=True) - - # Add the softmax loss. - xent_loss = nn.softmax_cross_entropy_with_logits( - logits=similarity_matrix, labels=labels_remapped) - xent_loss = math_ops.reduce_mean(xent_loss, name='xentropy') - - if print_losses: - xent_loss = logging_ops.Print( - xent_loss, ['cross entropy:', xent_loss, 'l2loss:', l2loss]) - - return l2loss + xent_loss - - -def lifted_struct_loss(labels, embeddings, margin=1.0): - """Computes the lifted structured loss. - - The loss encourages the positive distances (between a pair of embeddings - with the same labels) to be smaller than any negative distances (between a - pair of embeddings with different labels) in the mini-batch in a way - that is differentiable with respect to the embedding vectors. - See: https://arxiv.org/abs/1511.06452. - - Args: - labels: 1-D tf.int32 `Tensor` with shape [batch_size] of - multiclass integer labels. - embeddings: 2-D float `Tensor` of embedding vectors. Embeddings should not - be l2 normalized. - margin: Float, margin term in the loss definition. - - Returns: - lifted_loss: tf.float32 scalar. - """ - # Reshape [batch_size] label tensor to a [batch_size, 1] label tensor. - lshape = array_ops.shape(labels) - assert lshape.shape == 1 - labels = array_ops.reshape(labels, [lshape[0], 1]) - - # Build pairwise squared distance matrix. - pairwise_distances = pairwise_distance(embeddings) - - # Build pairwise binary adjacency matrix. - adjacency = math_ops.equal(labels, array_ops.transpose(labels)) - # Invert so we can select negatives only. - adjacency_not = math_ops.logical_not(adjacency) - - batch_size = array_ops.size(labels) - - diff = margin - pairwise_distances - mask = math_ops.cast(adjacency_not, dtype=dtypes.float32) - # Safe maximum: Temporarily shift negative distances - # above zero before taking max. - # this is to take the max only among negatives. - row_minimums = math_ops.reduce_min(diff, 1, keepdims=True) - row_negative_maximums = math_ops.reduce_max( - math_ops.multiply(diff - row_minimums, mask), 1, - keepdims=True) + row_minimums - - # Compute the loss. - # Keep track of matrix of maximums where M_ij = max(m_i, m_j) - # where m_i is the max of alpha - negative D_i's. - # This matches the Caffe loss layer implementation at: - # https://github.com/rksltnl/Caffe-Deep-Metric-Learning-CVPR16/blob/0efd7544a9846f58df923c8b992198ba5c355454/src/caffe/layers/lifted_struct_similarity_softmax_layer.cpp # pylint: disable=line-too-long - - max_elements = math_ops.maximum( - row_negative_maximums, array_ops.transpose(row_negative_maximums)) - diff_tiled = array_ops.tile(diff, [batch_size, 1]) - mask_tiled = array_ops.tile(mask, [batch_size, 1]) - max_elements_vect = array_ops.reshape( - array_ops.transpose(max_elements), [-1, 1]) - - loss_exp_left = array_ops.reshape( - math_ops.reduce_sum( - math_ops.multiply( - math_ops.exp(diff_tiled - max_elements_vect), mask_tiled), - 1, - keepdims=True), [batch_size, batch_size]) - - loss_mat = max_elements + math_ops.log( - loss_exp_left + array_ops.transpose(loss_exp_left)) - # Add the positive distance. - loss_mat += pairwise_distances - - mask_positives = math_ops.cast( - adjacency, dtype=dtypes.float32) - array_ops.diag( - array_ops.ones([batch_size])) - - # *0.5 for upper triangular, and another *0.5 for 1/2 factor for loss^2. - num_positives = math_ops.reduce_sum(mask_positives) / 2.0 - - lifted_loss = math_ops.truediv( - 0.25 * math_ops.reduce_sum( - math_ops.square( - math_ops.maximum( - math_ops.multiply(loss_mat, mask_positives), 0.0))), - num_positives, - name='liftedstruct_loss') - return lifted_loss - - -def update_1d_tensor(y, index, value): - """Updates 1d tensor y so that y[index] = value. - - Args: - y: 1-D Tensor. - index: index of y to modify. - value: new value to write at y[index]. - - Returns: - y_mod: 1-D Tensor. Tensor y after the update. - """ - value = array_ops.squeeze(value) - # modify the 1D tensor x at index with value. - # ex) chosen_ids = update_1D_tensor(chosen_ids, cluster_idx, best_medoid) - y_before = array_ops.slice(y, [0], [index]) - y_after = array_ops.slice(y, [index + 1], [-1]) - y_mod = array_ops.concat([y_before, [value], y_after], 0) - return y_mod - - -def get_cluster_assignment(pairwise_distances, centroid_ids): - """Assign data points to the neareset centroids. - - Tensorflow has numerical instability and doesn't always choose - the data point with theoretically zero distance as it's nearest neighbor. - Thus, for each centroid in centroid_ids, explicitly assign - the centroid itself as the nearest centroid. - This is done through the mask tensor and the constraint_vect tensor. - - Args: - pairwise_distances: 2-D Tensor of pairwise distances. - centroid_ids: 1-D Tensor of centroid indices. - - Returns: - y_fixed: 1-D tensor of cluster assignment. - """ - predictions = math_ops.argmin( - array_ops.gather(pairwise_distances, centroid_ids), dimension=0) - batch_size = array_ops.shape(pairwise_distances)[0] - - # Deal with numerical instability - mask = math_ops.reduce_any(array_ops.one_hot( - centroid_ids, batch_size, True, False, axis=-1, dtype=dtypes.bool), - axis=0) - constraint_one_hot = math_ops.multiply( - array_ops.one_hot(centroid_ids, - batch_size, - array_ops.constant(1, dtype=dtypes.int64), - array_ops.constant(0, dtype=dtypes.int64), - axis=0, - dtype=dtypes.int64), - math_ops.cast(math_ops.range(array_ops.shape(centroid_ids)[0]), - dtypes.int64)) - constraint_vect = math_ops.reduce_sum( - array_ops.transpose(constraint_one_hot), axis=0) - - y_fixed = array_ops.where(mask, constraint_vect, predictions) - return y_fixed - - -def compute_facility_energy(pairwise_distances, centroid_ids): - """Compute the average travel distance to the assigned centroid. - - Args: - pairwise_distances: 2-D Tensor of pairwise distances. - centroid_ids: 1-D Tensor of indices. - - Returns: - facility_energy: dtypes.float32 scalar. - """ - return -1.0 * math_ops.reduce_sum( - math_ops.reduce_min( - array_ops.gather(pairwise_distances, centroid_ids), axis=0)) - - -def compute_clustering_score(labels, predictions, margin_type): - """Computes the clustering score via sklearn.metrics functions. - - There are various ways to compute the clustering score. Intuitively, - we want to measure the agreement of two clustering assignments (labels vs - predictions) ignoring the permutations and output a score from zero to one. - (where the values close to one indicate significant agreement). - This code supports following scoring functions: - nmi: normalized mutual information - ami: adjusted mutual information - ari: adjusted random index - vmeasure: v-measure - const: indicator checking whether the two clusterings are the same. - See http://scikit-learn.org/stable/modules/classes.html#clustering-metrics - for the detailed descriptions. - Args: - labels: 1-D Tensor. ground truth cluster assignment. - predictions: 1-D Tensor. predicted cluster assignment. - margin_type: Type of structured margin to use. Default is nmi. - Returns: - clustering_score: dtypes.float32 scalar. - The possible valid values are from zero to one. - Zero means the worst clustering and one means the perfect clustering. - Raises: - ValueError: margin_type is not recognized. - """ - margin_type_to_func = { - 'nmi': _compute_nmi_score, - 'ami': _compute_ami_score, - 'ari': _compute_ari_score, - 'vmeasure': _compute_vmeasure_score, - 'const': _compute_zeroone_score - } - - if margin_type not in margin_type_to_func: - raise ValueError('Unrecognized margin_type: %s' % margin_type) - clustering_score_fn = margin_type_to_func[margin_type] - return array_ops.squeeze(clustering_score_fn(labels, predictions)) - - -def _compute_nmi_score(labels, predictions): - return math_ops.cast( - script_ops.py_func( - metrics.normalized_mutual_info_score, [labels, predictions], - [dtypes.float64], - name='nmi'), - dtypes.float32) - - -def _compute_ami_score(labels, predictions): - ami_score = math_ops.cast( - script_ops.py_func( - metrics.adjusted_mutual_info_score, [labels, predictions], - [dtypes.float64], - name='ami'), - dtypes.float32) - return math_ops.maximum(0.0, ami_score) - - -def _compute_ari_score(labels, predictions): - ari_score = math_ops.cast( - script_ops.py_func( - metrics.adjusted_rand_score, [labels, predictions], [dtypes.float64], - name='ari'), - dtypes.float32) - # ari score can go below 0 - # http://scikit-learn.org/stable/modules/clustering.html#adjusted-rand-score - return math_ops.maximum(0.0, ari_score) - - -def _compute_vmeasure_score(labels, predictions): - vmeasure_score = math_ops.cast( - script_ops.py_func( - metrics.v_measure_score, [labels, predictions], [dtypes.float64], - name='vmeasure'), - dtypes.float32) - return math_ops.maximum(0.0, vmeasure_score) - - -def _compute_zeroone_score(labels, predictions): - zeroone_score = math_ops.cast( - math_ops.equal( - math_ops.reduce_sum( - math_ops.cast(math_ops.equal(labels, predictions), dtypes.int32)), - array_ops.shape(labels)[0]), - dtypes.float32) - return zeroone_score - - -def _find_loss_augmented_facility_idx(pairwise_distances, labels, chosen_ids, - candidate_ids, margin_multiplier, - margin_type): - """Find the next centroid that maximizes the loss augmented inference. - - This function is a subroutine called from compute_augmented_facility_locations - - Args: - pairwise_distances: 2-D Tensor of pairwise distances. - labels: 1-D Tensor of ground truth cluster assignment. - chosen_ids: 1-D Tensor of current centroid indices. - candidate_ids: 1-D Tensor of candidate indices. - margin_multiplier: multiplication constant. - margin_type: Type of structured margin to use. Default is nmi. - - Returns: - integer index. - """ - num_candidates = array_ops.shape(candidate_ids)[0] - - pairwise_distances_chosen = array_ops.gather(pairwise_distances, chosen_ids) - pairwise_distances_candidate = array_ops.gather( - pairwise_distances, candidate_ids) - pairwise_distances_chosen_tile = array_ops.tile( - pairwise_distances_chosen, [1, num_candidates]) - - candidate_scores = -1.0 * math_ops.reduce_sum( - array_ops.reshape( - math_ops.reduce_min( - array_ops.concat([ - pairwise_distances_chosen_tile, - array_ops.reshape(pairwise_distances_candidate, [1, -1]) - ], 0), - axis=0, - keepdims=True), [num_candidates, -1]), - axis=1) - - nmi_scores = array_ops.zeros([num_candidates]) - iteration = array_ops.constant(0) - - def func_cond(iteration, nmi_scores): - del nmi_scores # Unused in func_cond() - return iteration < num_candidates - - def func_body(iteration, nmi_scores): - predictions = get_cluster_assignment( - pairwise_distances, - array_ops.concat([chosen_ids, [candidate_ids[iteration]]], 0)) - nmi_score_i = compute_clustering_score(labels, predictions, margin_type) - pad_before = array_ops.zeros([iteration]) - pad_after = array_ops.zeros([num_candidates - 1 - iteration]) - # return 1 - NMI score as the structured loss. - # because NMI is higher the better [0,1]. - return iteration + 1, nmi_scores + array_ops.concat( - [pad_before, [1.0 - nmi_score_i], pad_after], 0) - - _, nmi_scores = control_flow_ops.while_loop( - func_cond, func_body, [iteration, nmi_scores]) - - candidate_scores = math_ops.add( - candidate_scores, margin_multiplier * nmi_scores) - - argmax_index = math_ops.cast( - math_ops.argmax(candidate_scores, axis=0), dtypes.int32) - - return candidate_ids[argmax_index] - - -def compute_augmented_facility_locations(pairwise_distances, labels, all_ids, - margin_multiplier, margin_type): - """Computes the centroid locations. - - Args: - pairwise_distances: 2-D Tensor of pairwise distances. - labels: 1-D Tensor of ground truth cluster assignment. - all_ids: 1-D Tensor of all data indices. - margin_multiplier: multiplication constant. - margin_type: Type of structured margin to use. Default is nmi. - - Returns: - chosen_ids: 1-D Tensor of chosen centroid indices. - """ - - def func_cond_augmented(iteration, chosen_ids): - del chosen_ids # Unused argument in func_cond_augmented. - return iteration < num_classes - - def func_body_augmented(iteration, chosen_ids): - # find a new facility location to add - # based on the clustering score and the NMI score - candidate_ids = array_ops.setdiff1d(all_ids, chosen_ids)[0] - new_chosen_idx = _find_loss_augmented_facility_idx(pairwise_distances, - labels, chosen_ids, - candidate_ids, - margin_multiplier, - margin_type) - chosen_ids = array_ops.concat([chosen_ids, [new_chosen_idx]], 0) - return iteration + 1, chosen_ids - - num_classes = array_ops.size(array_ops.unique(labels)[0]) - chosen_ids = array_ops.constant(0, dtype=dtypes.int32, shape=[0]) - - # num_classes get determined at run time based on the sampled batch. - iteration = array_ops.constant(0) - - _, chosen_ids = control_flow_ops.while_loop( - func_cond_augmented, - func_body_augmented, [iteration, chosen_ids], - shape_invariants=[iteration.get_shape(), tensor_shape.TensorShape( - [None])]) - return chosen_ids - - -def update_medoid_per_cluster(pairwise_distances, pairwise_distances_subset, - labels, chosen_ids, cluster_member_ids, - cluster_idx, margin_multiplier, margin_type): - """Updates the cluster medoid per cluster. - - Args: - pairwise_distances: 2-D Tensor of pairwise distances. - pairwise_distances_subset: 2-D Tensor of pairwise distances for one cluster. - labels: 1-D Tensor of ground truth cluster assignment. - chosen_ids: 1-D Tensor of cluster centroid indices. - cluster_member_ids: 1-D Tensor of cluster member indices for one cluster. - cluster_idx: Index of this one cluster. - margin_multiplier: multiplication constant. - margin_type: Type of structured margin to use. Default is nmi. - - Returns: - chosen_ids: Updated 1-D Tensor of cluster centroid indices. - """ - - def func_cond(iteration, scores_margin): - del scores_margin # Unused variable scores_margin. - return iteration < num_candidates - - def func_body(iteration, scores_margin): - # swap the current medoid with the candidate cluster member - candidate_medoid = math_ops.cast(cluster_member_ids[iteration], dtypes.int32) - tmp_chosen_ids = update_1d_tensor(chosen_ids, cluster_idx, candidate_medoid) - predictions = get_cluster_assignment(pairwise_distances, tmp_chosen_ids) - metric_score = compute_clustering_score(labels, predictions, margin_type) - pad_before = array_ops.zeros([iteration]) - pad_after = array_ops.zeros([num_candidates - 1 - iteration]) - return iteration + 1, scores_margin + array_ops.concat( - [pad_before, [1.0 - metric_score], pad_after], 0) - - # pairwise_distances_subset is of size [p, 1, 1, p], - # the intermediate dummy dimensions at - # [1, 2] makes this code work in the edge case where p=1. - # this happens if the cluster size is one. - scores_fac = -1.0 * math_ops.reduce_sum( - array_ops.squeeze(pairwise_distances_subset, [1, 2]), axis=0) - - iteration = array_ops.constant(0) - num_candidates = array_ops.size(cluster_member_ids) - scores_margin = array_ops.zeros([num_candidates]) - - _, scores_margin = control_flow_ops.while_loop(func_cond, func_body, - [iteration, scores_margin]) - candidate_scores = math_ops.add(scores_fac, margin_multiplier * scores_margin) - - argmax_index = math_ops.cast( - math_ops.argmax(candidate_scores, axis=0), dtypes.int32) - - best_medoid = math_ops.cast(cluster_member_ids[argmax_index], dtypes.int32) - chosen_ids = update_1d_tensor(chosen_ids, cluster_idx, best_medoid) - return chosen_ids - - -def update_all_medoids(pairwise_distances, predictions, labels, chosen_ids, - margin_multiplier, margin_type): - """Updates all cluster medoids a cluster at a time. - - Args: - pairwise_distances: 2-D Tensor of pairwise distances. - predictions: 1-D Tensor of predicted cluster assignment. - labels: 1-D Tensor of ground truth cluster assignment. - chosen_ids: 1-D Tensor of cluster centroid indices. - margin_multiplier: multiplication constant. - margin_type: Type of structured margin to use. Default is nmi. - - Returns: - chosen_ids: Updated 1-D Tensor of cluster centroid indices. - """ - - def func_cond_augmented_pam(iteration, chosen_ids): - del chosen_ids # Unused argument. - return iteration < num_classes - - def func_body_augmented_pam(iteration, chosen_ids): - """Call the update_medoid_per_cluster subroutine.""" - mask = math_ops.equal( - math_ops.cast(predictions, dtypes.int64), - math_ops.cast(iteration, dtypes.int64)) - this_cluster_ids = array_ops.where(mask) - - pairwise_distances_subset = array_ops.transpose( - array_ops.gather( - array_ops.transpose( - array_ops.gather(pairwise_distances, this_cluster_ids)), - this_cluster_ids)) - - chosen_ids = update_medoid_per_cluster(pairwise_distances, - pairwise_distances_subset, labels, - chosen_ids, this_cluster_ids, - iteration, margin_multiplier, - margin_type) - return iteration + 1, chosen_ids - - unique_class_ids = array_ops.unique(labels)[0] - num_classes = array_ops.size(unique_class_ids) - iteration = array_ops.constant(0) - - _, chosen_ids = control_flow_ops.while_loop( - func_cond_augmented_pam, func_body_augmented_pam, [iteration, chosen_ids]) - return chosen_ids - - -def compute_augmented_facility_locations_pam(pairwise_distances, - labels, - margin_multiplier, - margin_type, - chosen_ids, - pam_max_iter=5): - """Refine the cluster centroids with PAM local search. - - For fixed iterations, alternate between updating the cluster assignment - and updating cluster medoids. - - Args: - pairwise_distances: 2-D Tensor of pairwise distances. - labels: 1-D Tensor of ground truth cluster assignment. - margin_multiplier: multiplication constant. - margin_type: Type of structured margin to use. Default is nmi. - chosen_ids: 1-D Tensor of initial estimate of cluster centroids. - pam_max_iter: Number of refinement iterations. - - Returns: - chosen_ids: Updated 1-D Tensor of cluster centroid indices. - """ - for _ in range(pam_max_iter): - # update the cluster assignment given the chosen_ids (S_pred) - predictions = get_cluster_assignment(pairwise_distances, chosen_ids) - - # update the medoids per each cluster - chosen_ids = update_all_medoids(pairwise_distances, predictions, labels, - chosen_ids, margin_multiplier, margin_type) - - return chosen_ids - - -def compute_gt_cluster_score(pairwise_distances, labels): - """Compute ground truth facility location score. - - Loop over each unique classes and compute average travel distances. - - Args: - pairwise_distances: 2-D Tensor of pairwise distances. - labels: 1-D Tensor of ground truth cluster assignment. - - Returns: - gt_cluster_score: dtypes.float32 score. - """ - unique_class_ids = array_ops.unique(labels)[0] - num_classes = array_ops.size(unique_class_ids) - iteration = array_ops.constant(0) - gt_cluster_score = array_ops.constant(0.0, dtype=dtypes.float32) - - def func_cond(iteration, gt_cluster_score): - del gt_cluster_score # Unused argument. - return iteration < num_classes - - def func_body(iteration, gt_cluster_score): - """Per each cluster, compute the average travel distance.""" - mask = math_ops.equal(labels, unique_class_ids[iteration]) - this_cluster_ids = array_ops.where(mask) - pairwise_distances_subset = array_ops.transpose( - array_ops.gather( - array_ops.transpose( - array_ops.gather(pairwise_distances, this_cluster_ids)), - this_cluster_ids)) - this_cluster_score = -1.0 * math_ops.reduce_min( - math_ops.reduce_sum( - pairwise_distances_subset, axis=0)) - return iteration + 1, gt_cluster_score + this_cluster_score - - _, gt_cluster_score = control_flow_ops.while_loop( - func_cond, func_body, [iteration, gt_cluster_score]) - return gt_cluster_score - - -def cluster_loss(labels, - embeddings, - margin_multiplier, - enable_pam_finetuning=True, - margin_type='nmi', - print_losses=False): - """Computes the clustering loss. - - The following structured margins are supported: - nmi: normalized mutual information - ami: adjusted mutual information - ari: adjusted random index - vmeasure: v-measure - const: indicator checking whether the two clusterings are the same. - - Args: - labels: 2-D Tensor of labels of shape [batch size, 1] - embeddings: 2-D Tensor of embeddings of shape - [batch size, embedding dimension]. Embeddings should be l2 normalized. - margin_multiplier: float32 scalar. multiplier on the structured margin term - See section 3.2 of paper for discussion. - enable_pam_finetuning: Boolean, Whether to run local pam refinement. - See section 3.4 of paper for discussion. - margin_type: Type of structured margin to use. See section 3.2 of - paper for discussion. Can be 'nmi', 'ami', 'ari', 'vmeasure', 'const'. - print_losses: Boolean. Option to print the loss. - - Paper: https://arxiv.org/abs/1612.01213. - - Returns: - clustering_loss: A float32 scalar `Tensor`. - Raises: - ImportError: If sklearn dependency is not installed. - """ - if not HAS_SKLEARN: - raise ImportError('Cluster loss depends on sklearn.') - pairwise_distances = pairwise_distance(embeddings) - labels = array_ops.squeeze(labels) - all_ids = math_ops.range(array_ops.shape(embeddings)[0]) - - # Compute the loss augmented inference and get the cluster centroids. - chosen_ids = compute_augmented_facility_locations(pairwise_distances, labels, - all_ids, margin_multiplier, - margin_type) - # Given the predicted centroids, compute the clustering score. - score_pred = compute_facility_energy(pairwise_distances, chosen_ids) - - # Branch whether to use PAM finetuning. - if enable_pam_finetuning: - # Initialize with augmented facility solution. - chosen_ids = compute_augmented_facility_locations_pam(pairwise_distances, - labels, - margin_multiplier, - margin_type, - chosen_ids) - score_pred = compute_facility_energy(pairwise_distances, chosen_ids) - - # Given the predicted centroids, compute the cluster assignments. - predictions = get_cluster_assignment(pairwise_distances, chosen_ids) - - # Compute the clustering (i.e. NMI) score between the two assignments. - clustering_score_pred = compute_clustering_score(labels, predictions, - margin_type) - - # Compute the clustering score from labels. - score_gt = compute_gt_cluster_score(pairwise_distances, labels) - - # Compute the hinge loss. - clustering_loss = math_ops.maximum( - score_pred + margin_multiplier * (1.0 - clustering_score_pred) - score_gt, - 0.0, - name='clustering_loss') - clustering_loss.set_shape([]) - - if print_losses: - clustering_loss = logging_ops.Print( - clustering_loss, - ['clustering_loss: ', clustering_loss, array_ops.shape( - clustering_loss)]) - - # Clustering specific summary. - summary.scalar('losses/score_pred', score_pred) - summary.scalar('losses/' + margin_type, clustering_score_pred) - summary.scalar('losses/score_gt', score_gt) - - return clustering_loss diff --git a/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops_test.py b/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops_test.py deleted file mode 100644 index 9c389144ff6..00000000000 --- a/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops_test.py +++ /dev/null @@ -1,562 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for triplet_semihard_loss.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib.losses.python import metric_learning as metric_loss_ops -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.platform import test -try: - # pylint: disable=g-import-not-at-top - from sklearn import datasets - from sklearn import metrics - HAS_SKLEARN = True -except ImportError: - HAS_SKLEARN = False - - -def pairwise_distance_np(feature, squared=False): - """Computes the pairwise distance matrix in numpy. - - Args: - feature: 2-D numpy array of size [number of data, feature dimension] - squared: Boolean. If true, output is the pairwise squared euclidean - distance matrix; else, output is the pairwise euclidean distance matrix. - - Returns: - pairwise_distances: 2-D numpy array of size - [number of data, number of data]. - """ - triu = np.triu_indices(feature.shape[0], 1) - upper_tri_pdists = np.linalg.norm(feature[triu[1]] - feature[triu[0]], axis=1) - if squared: - upper_tri_pdists **= 2. - num_data = feature.shape[0] - pairwise_distances = np.zeros((num_data, num_data)) - pairwise_distances[np.triu_indices(num_data, 1)] = upper_tri_pdists - # Make symmetrical. - pairwise_distances = pairwise_distances + pairwise_distances.T - np.diag( - pairwise_distances.diagonal()) - return pairwise_distances - - -class ContrastiveLossTest(test.TestCase): - - def testContrastive(self): - with self.cached_session(): - num_data = 10 - feat_dim = 6 - margin = 1.0 - - embeddings_anchor = np.random.rand(num_data, feat_dim).astype(np.float32) - embeddings_positive = np.random.rand(num_data, feat_dim).astype( - np.float32) - labels = np.random.randint(0, 2, size=(num_data,)).astype(np.float32) - - # Compute the loss in NP - dist = np.sqrt( - np.sum(np.square(embeddings_anchor - embeddings_positive), axis=1)) - loss_np = np.mean( - labels * np.square(dist) + - (1.0 - labels) * np.square(np.maximum(margin - dist, 0.0))) - # Compute the loss with TF - loss_tf = metric_loss_ops.contrastive_loss( - labels=ops.convert_to_tensor(labels), - embeddings_anchor=ops.convert_to_tensor(embeddings_anchor), - embeddings_positive=ops.convert_to_tensor(embeddings_positive), - margin=margin) - loss_tf = loss_tf.eval() - self.assertAllClose(loss_np, loss_tf) - - -class TripletSemiHardLossTest(test.TestCase): - - def testTripletSemiHard(self): - with self.cached_session(): - num_data = 10 - feat_dim = 6 - margin = 1.0 - num_classes = 4 - - embedding = np.random.rand(num_data, feat_dim).astype(np.float32) - labels = np.random.randint( - 0, num_classes, size=(num_data)).astype(np.float32) - - # Reshape labels to compute adjacency matrix. - labels_reshaped = np.reshape(labels, (labels.shape[0], 1)) - # Compute the loss in NP. - adjacency = np.equal(labels_reshaped, labels_reshaped.T) - - pdist_matrix = pairwise_distance_np(embedding, squared=True) - loss_np = 0.0 - num_positives = 0.0 - for i in range(num_data): - for j in range(num_data): - if adjacency[i][j] > 0.0 and i != j: - num_positives += 1.0 - - pos_distance = pdist_matrix[i][j] - neg_distances = [] - - for k in range(num_data): - if adjacency[i][k] == 0: - neg_distances.append(pdist_matrix[i][k]) - - # Sort by distance. - neg_distances.sort() - chosen_neg_distance = neg_distances[0] - - for l in range(len(neg_distances)): - chosen_neg_distance = neg_distances[l] - if chosen_neg_distance > pos_distance: - break - - loss_np += np.maximum( - 0.0, margin - chosen_neg_distance + pos_distance) - - loss_np /= num_positives - - # Compute the loss in TF. - loss_tf = metric_loss_ops.triplet_semihard_loss( - labels=ops.convert_to_tensor(labels), - embeddings=ops.convert_to_tensor(embedding), - margin=margin) - loss_tf = loss_tf.eval() - self.assertAllClose(loss_np, loss_tf) - - -class LiftedStructLossTest(test.TestCase): - - def testLiftedStruct(self): - with self.cached_session(): - num_data = 10 - feat_dim = 6 - margin = 1.0 - num_classes = 4 - - embedding = np.random.rand(num_data, feat_dim).astype(np.float32) - labels = np.random.randint( - 0, num_classes, size=(num_data)).astype(np.float32) - # Reshape labels to compute adjacency matrix. - labels_reshaped = np.reshape(labels, (labels.shape[0], 1)) - - # Compute the loss in NP - adjacency = np.equal(labels_reshaped, labels_reshaped.T) - pdist_matrix = pairwise_distance_np(embedding) - loss_np = 0.0 - num_constraints = 0.0 - for i in range(num_data): - for j in range(num_data): - if adjacency[i][j] > 0.0 and i != j: - d_pos = pdist_matrix[i][j] - negs = [] - for k in range(num_data): - if not adjacency[i][k]: - negs.append(margin - pdist_matrix[i][k]) - for l in range(num_data): - if not adjacency[j][l]: - negs.append(margin - pdist_matrix[j][l]) - - negs = np.array(negs) - max_elem = np.max(negs) - negs -= max_elem - negs = np.exp(negs) - soft_maximum = np.log(np.sum(negs)) + max_elem - - num_constraints += 1.0 - this_loss = max(soft_maximum + d_pos, 0) - loss_np += this_loss * this_loss - - loss_np = loss_np / num_constraints / 2.0 - - # Compute the loss in TF - loss_tf = metric_loss_ops.lifted_struct_loss( - labels=ops.convert_to_tensor(labels), - embeddings=ops.convert_to_tensor(embedding), - margin=margin) - loss_tf = loss_tf.eval() - self.assertAllClose(loss_np, loss_tf) - - -def convert_to_list_of_sparse_tensor(np_matrix): - list_of_sparse_tensors = [] - nrows, ncols = np_matrix.shape - for i in range(nrows): - sp_indices = [] - for j in range(ncols): - if np_matrix[i][j] == 1: - sp_indices.append([j]) - - num_non_zeros = len(sp_indices) - list_of_sparse_tensors.append(sparse_tensor.SparseTensor( - indices=np.array(sp_indices), - values=np.ones((num_non_zeros,)), - dense_shape=np.array([ncols,]))) - - return list_of_sparse_tensors - - -class NpairsLossTest(test.TestCase): - - def testNpairs(self): - with self.cached_session(): - num_data = 15 - feat_dim = 6 - num_classes = 5 - reg_lambda = 0.02 - - embeddings_anchor = np.random.rand(num_data, feat_dim).astype(np.float32) - embeddings_positive = np.random.rand(num_data, feat_dim).astype( - np.float32) - - labels = np.random.randint( - 0, num_classes, size=(num_data)).astype(np.float32) - # Reshape labels to compute adjacency matrix. - labels_reshaped = np.reshape(labels, (labels.shape[0], 1)) - - # Compute the loss in NP - reg_term = np.mean(np.sum(np.square(embeddings_anchor), 1)) - reg_term += np.mean(np.sum(np.square(embeddings_positive), 1)) - reg_term *= 0.25 * reg_lambda - - similarity_matrix = np.matmul(embeddings_anchor, embeddings_positive.T) - - labels_remapped = np.equal( - labels_reshaped, labels_reshaped.T).astype(np.float32) - labels_remapped /= np.sum(labels_remapped, axis=1, keepdims=True) - - xent_loss = math_ops.reduce_mean(nn.softmax_cross_entropy_with_logits( - logits=ops.convert_to_tensor(similarity_matrix), - labels=ops.convert_to_tensor(labels_remapped))).eval() - loss_np = xent_loss + reg_term - - # Compute the loss in TF - loss_tf = metric_loss_ops.npairs_loss( - labels=ops.convert_to_tensor(labels), - embeddings_anchor=ops.convert_to_tensor(embeddings_anchor), - embeddings_positive=ops.convert_to_tensor(embeddings_positive), - reg_lambda=reg_lambda) - loss_tf = loss_tf.eval() - self.assertAllClose(loss_np, loss_tf) - - -class NpairsLossMultiLabelTest(test.TestCase): - - def testNpairsMultiLabelLossWithSingleLabelEqualsNpairsLoss(self): - with self.cached_session(): - num_data = 15 - feat_dim = 6 - reg_lambda = 0.02 - - embeddings_anchor = np.random.rand(num_data, feat_dim).astype(np.float32) - embeddings_positive = np.random.rand(num_data, feat_dim).astype( - np.float32) - labels = np.arange(num_data) - labels = np.reshape(labels, -1) - - # Compute vanila npairs loss. - loss_npairs = metric_loss_ops.npairs_loss( - labels=ops.convert_to_tensor(labels), - embeddings_anchor=ops.convert_to_tensor(embeddings_anchor), - embeddings_positive=ops.convert_to_tensor(embeddings_positive), - reg_lambda=reg_lambda).eval() - - # Compute npairs multilabel loss. - labels_one_hot = np.identity(num_data) - loss_npairs_multilabel = metric_loss_ops.npairs_loss_multilabel( - sparse_labels=convert_to_list_of_sparse_tensor(labels_one_hot), - embeddings_anchor=ops.convert_to_tensor(embeddings_anchor), - embeddings_positive=ops.convert_to_tensor(embeddings_positive), - reg_lambda=reg_lambda).eval() - - self.assertAllClose(loss_npairs, loss_npairs_multilabel) - - def testNpairsMultiLabel(self): - with self.cached_session(): - num_data = 15 - feat_dim = 6 - num_classes = 10 - reg_lambda = 0.02 - - embeddings_anchor = np.random.rand(num_data, feat_dim).astype(np.float32) - embeddings_positive = np.random.rand(num_data, feat_dim).astype( - np.float32) - - labels = np.random.randint(0, 2, (num_data, num_classes)) - # set entire column to one so that each row has at least one bit set. - labels[:, -1] = 1 - - # Compute the loss in NP - reg_term = np.mean(np.sum(np.square(embeddings_anchor), 1)) - reg_term += np.mean(np.sum(np.square(embeddings_positive), 1)) - reg_term *= 0.25 * reg_lambda - - similarity_matrix = np.matmul(embeddings_anchor, embeddings_positive.T) - - labels_remapped = np.dot(labels, labels.T).astype(np.float) - labels_remapped /= np.sum(labels_remapped, 1, keepdims=True) - - xent_loss = math_ops.reduce_mean(nn.softmax_cross_entropy_with_logits( - logits=ops.convert_to_tensor(similarity_matrix), - labels=ops.convert_to_tensor(labels_remapped))).eval() - loss_np = xent_loss + reg_term - - # Compute the loss in TF - loss_tf = metric_loss_ops.npairs_loss_multilabel( - sparse_labels=convert_to_list_of_sparse_tensor(labels), - embeddings_anchor=ops.convert_to_tensor(embeddings_anchor), - embeddings_positive=ops.convert_to_tensor(embeddings_positive), - reg_lambda=reg_lambda) - loss_tf = loss_tf.eval() - - self.assertAllClose(loss_np, loss_tf) - - -def compute_ground_truth_cluster_score(feat, y): - y_unique = np.unique(y) - score_gt_np = 0.0 - for c in y_unique: - feat_subset = feat[y == c, :] - pdist_subset = pairwise_distance_np(feat_subset) - score_gt_np += -1.0 * np.min(np.sum(pdist_subset, axis=0)) - score_gt_np = score_gt_np.astype(np.float32) - return score_gt_np - - -def compute_cluster_loss_numpy(feat, - y, - margin_multiplier=1.0, - enable_pam_finetuning=True): - if enable_pam_finetuning: - facility = ForwardGreedyFacility( - n_clusters=np.unique(y).size).pam_augmented_fit(feat, y, - margin_multiplier) - else: - facility = ForwardGreedyFacility( - n_clusters=np.unique(y).size).loss_augmented_fit(feat, y, - margin_multiplier) - - score_augmented = facility.score_aug_ - score_gt = compute_ground_truth_cluster_score(feat, y) - return np.maximum(np.float32(0.0), score_augmented - score_gt) - - -class ForwardGreedyFacility(object): - - def __init__(self, n_clusters=8): - self.n_clusters = n_clusters - self.center_ics_ = None - - def _check_init_args(self): - # Check n_clusters. - if (self.n_clusters is None or self.n_clusters <= 0 or - not isinstance(self.n_clusters, int)): - raise ValueError('n_clusters has to be nonnegative integer.') - - def loss_augmented_fit(self, feat, y, loss_mult): - """Fit K-Medoids to the provided data.""" - self._check_init_args() - # Check that the array is good and attempt to convert it to - # Numpy array if possible. - feat = self._check_array(feat) - # Apply distance metric to get the distance matrix. - pdists = pairwise_distance_np(feat) - - num_data = feat.shape[0] - candidate_ids = list(range(num_data)) - candidate_scores = np.zeros(num_data,) - subset = [] - - k = 0 - while k < self.n_clusters: - candidate_scores = [] - for i in candidate_ids: - # push i to subset. - subset.append(i) - marginal_cost = -1.0 * np.sum(np.min(pdists[:, subset], axis=1)) - loss = 1.0 - metrics.normalized_mutual_info_score( - y, self._get_cluster_ics(pdists, subset)) - candidate_scores.append(marginal_cost + loss_mult * loss) - # remove i from subset. - subset.pop() - - # push i_star to subset. - i_star = candidate_ids[np.argmax(candidate_scores)] - subset.append(i_star) - # remove i_star from candidate indices. - candidate_ids.remove(i_star) - k += 1 - - # Expose labels_ which are the assignments of - # the training data to clusters. - self.labels_ = self._get_cluster_ics(pdists, subset) - # Expose cluster centers, i.e. medoids. - self.cluster_centers_ = feat.take(subset, axis=0) - # Expose indices of chosen cluster centers. - self.center_ics_ = subset - # Expose the score = -\sum_{i \in V} min_{j \in S} || x_i - x_j || - self.score_ = np.float32(-1.0) * self._get_facility_distance(pdists, subset) - self.score_aug_ = self.score_ + loss_mult * ( - 1.0 - metrics.normalized_mutual_info_score( - y, self._get_cluster_ics(pdists, subset))) - self.score_aug_ = self.score_aug_.astype(np.float32) - # Expose the chosen cluster indices. - self.subset_ = subset - return self - - def _augmented_update_medoid_ics_in_place(self, pdists, y_gt, cluster_ics, - medoid_ics, loss_mult): - for cluster_idx in range(self.n_clusters): - # y_pred = self._get_cluster_ics(D, medoid_ics) - # Don't prematurely do the assignment step. - # Do this after we've updated all cluster medoids. - y_pred = cluster_ics - - if sum(y_pred == cluster_idx) == 0: - # Cluster is empty. - continue - - curr_score = ( - -1.0 * np.sum( - pdists[medoid_ics[cluster_idx], y_pred == cluster_idx]) + - loss_mult * (1.0 - metrics.normalized_mutual_info_score( - y_gt, y_pred))) - - pdist_in = pdists[y_pred == cluster_idx, :] - pdist_in = pdist_in[:, y_pred == cluster_idx] - - all_scores_fac = np.sum(-1.0 * pdist_in, axis=1) - all_scores_loss = [] - for i in range(y_pred.size): - if y_pred[i] != cluster_idx: - continue - # remove this cluster's current centroid - medoid_ics_i = medoid_ics[:cluster_idx] + medoid_ics[cluster_idx + 1:] - # add this new candidate to the centroid list - medoid_ics_i += [i] - y_pred_i = self._get_cluster_ics(pdists, medoid_ics_i) - all_scores_loss.append(loss_mult * ( - 1.0 - metrics.normalized_mutual_info_score(y_gt, y_pred_i))) - - all_scores = all_scores_fac + all_scores_loss - max_score_idx = np.argmax(all_scores) - max_score = all_scores[max_score_idx] - - if max_score > curr_score: - medoid_ics[cluster_idx] = np.where( - y_pred == cluster_idx)[0][max_score_idx] - - def pam_augmented_fit(self, feat, y, loss_mult): - pam_max_iter = 5 - self._check_init_args() - feat = self._check_array(feat) - pdists = pairwise_distance_np(feat) - self.loss_augmented_fit(feat, y, loss_mult) - print('PAM -1 (before PAM): score: %f, score_aug: %f' % ( - self.score_, self.score_aug_)) - # Initialize from loss augmented facility location - subset = self.center_ics_ - for iter_ in range(pam_max_iter): - # update the cluster assignment - cluster_ics = self._get_cluster_ics(pdists, subset) - # update the medoid for each clusters - self._augmented_update_medoid_ics_in_place(pdists, y, cluster_ics, subset, - loss_mult) - self.score_ = np.float32(-1.0) * self._get_facility_distance( - pdists, subset) - self.score_aug_ = self.score_ + loss_mult * ( - 1.0 - metrics.normalized_mutual_info_score( - y, self._get_cluster_ics(pdists, subset))) - self.score_aug_ = self.score_aug_.astype(np.float32) - print('PAM iter: %d, score: %f, score_aug: %f' % (iter_, self.score_, - self.score_aug_)) - - self.center_ics_ = subset - self.labels_ = cluster_ics - return self - - def _check_array(self, feat): - # Check that the number of clusters is less than or equal to - # the number of samples - if self.n_clusters > feat.shape[0]: - raise ValueError('The number of medoids ' + '({}) '.format( - self.n_clusters) + 'must be larger than the number ' + - 'of samples ({})'.format(feat.shape[0])) - return feat - - def _get_cluster_ics(self, pdists, subset): - """Returns cluster indices for pdist and current medoid indices.""" - # Assign data points to clusters based on - # which cluster assignment yields - # the smallest distance` - cluster_ics = np.argmin(pdists[subset, :], axis=0) - return cluster_ics - - def _get_facility_distance(self, pdists, subset): - return np.sum(np.min(pdists[subset, :], axis=0)) - - -class ClusterLossTest(test.TestCase): - - def _genClusters(self, n_samples, n_clusters): - blobs = datasets.make_blobs( - n_samples=n_samples, centers=n_clusters) - embedding, labels = blobs - embedding = (embedding - embedding.mean(axis=0)) / embedding.std(axis=0) - embedding = embedding.astype(np.float32) - return embedding, labels - - def testClusteringLossPAMOff(self): - if not HAS_SKLEARN: - return - with self.cached_session(): - margin_multiplier = 10.0 - embeddings, labels = self._genClusters(n_samples=128, n_clusters=64) - - loss_np = compute_cluster_loss_numpy( - embeddings, labels, margin_multiplier, enable_pam_finetuning=False) - loss_tf = metric_loss_ops.cluster_loss( - labels=ops.convert_to_tensor(labels), - embeddings=ops.convert_to_tensor(embeddings), - margin_multiplier=margin_multiplier, - enable_pam_finetuning=False) - loss_tf = loss_tf.eval() - self.assertAllClose(loss_np, loss_tf) - - def testClusteringLossPAMOn(self): - if not HAS_SKLEARN: - return - with self.cached_session(): - margin_multiplier = 10.0 - embeddings, labels = self._genClusters(n_samples=128, n_clusters=64) - - loss_np = compute_cluster_loss_numpy( - embeddings, labels, margin_multiplier, enable_pam_finetuning=True) - loss_tf = metric_loss_ops.cluster_loss( - labels=ops.convert_to_tensor(labels), - embeddings=ops.convert_to_tensor(embeddings), - margin_multiplier=margin_multiplier, - enable_pam_finetuning=True) - loss_tf = loss_tf.eval() - self.assertAllClose(loss_np, loss_tf) - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/makefile/.gitignore b/tensorflow/contrib/makefile/.gitignore deleted file mode 100644 index 3ff6e609f4e..00000000000 --- a/tensorflow/contrib/makefile/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -gen/ -downloads/ diff --git a/tensorflow/contrib/makefile/BUILD b/tensorflow/contrib/makefile/BUILD deleted file mode 100644 index afd6a785705..00000000000 --- a/tensorflow/contrib/makefile/BUILD +++ /dev/null @@ -1,6 +0,0 @@ -# Necessary build rules for makefile build in our CI. - -package( - default_visibility = ["//visibility:private"], - licenses = ["notice"], # Apache 2.0 -) diff --git a/tensorflow/contrib/makefile/Dockerfile b/tensorflow/contrib/makefile/Dockerfile deleted file mode 100644 index 64d571a4edf..00000000000 --- a/tensorflow/contrib/makefile/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:16.04 - -LABEL maintainer="Gunhan Gulsoy " - -# Install make build dependencies for TensorFlow. -RUN apt-get update -RUN apt-get install -y \ - autoconf \ - automake \ - curl \ - g++ \ - git \ - libtool \ - make \ - python \ - unzip \ - wget \ - zlib1g-dev diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile deleted file mode 100644 index 8e75fcb666a..00000000000 --- a/tensorflow/contrib/makefile/Makefile +++ /dev/null @@ -1,960 +0,0 @@ -# This Makefile compiles a library containing the C++ runtime for the TensorFlow -# library. It's designed for use on platforms with limited resources where -# running a full Bazel build would be prohibitive, or for cross-compilation onto -# embedded systems. It includes only a bare-bones set of functionality. -# -# The default setup below is aimed at Unix-like devices, and should work on -# modern Linux and OS X distributions without changes. -# -# If you have another platform, you'll need to take a careful look at the -# compiler flags and folders defined below. They're separated into two sections, -# the first for the host (the machine you're compiling on) and the second for -# the target (the machine you want the program to run on). - -SHELL := /bin/bash - -# Host compilation settings - -# Find where we're running from, so we can store generated files here. -ifeq ($(origin MAKEFILE_DIR), undefined) - MAKEFILE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) -endif - -HAS_GEN_HOST_PROTOC := \ -$(shell test -f $(MAKEFILE_DIR)/gen/protobuf-host/bin/protoc && echo "true" ||\ -echo "false") - -# Hexagon integration -ifdef HEXAGON_LIBS - LIBGEMM_WRAPPER := $(HEXAGON_LIBS)/libhexagon_controller.so - ifeq ($(shell test -f $(LIBGEMM_WRAPPER) 2> /dev/null; echo $$?), 0) - $(info "Use hexagon libs at " $(LIBGEMM_WRAPPER)) - else - $(error "hexagon libs not found at " $(LIBGEMM_WRAPPER)) - endif - ifdef HEXAGON_INCLUDE - ifeq ($(shell test -d $(HEXAGON_INCLUDE) 2> /dev/null; echo $$?), 0) - $(info "Use hexagon libs at " $(HEXAGON_INCLUDE)) - else - $(error "hexagon libs not found at " $(HEXAGON_INCLUDE)) - endif - else - $(error "HEXAGON_INCLUDE is not set.") - endif - ifneq ($(TARGET),ANDROID) - $(error "hexagon is only supported on Android") - endif -endif # HEXAGON_LIBS - -# If ANDROID_TYPES is not set assume __ANDROID_TYPES_SLIM__ -ifeq ($(ANDROID_TYPES),) - ANDROID_TYPES := -D__ANDROID_TYPES_SLIM__ -endif - -# Try to figure out the host system -HOST_OS := -ifeq ($(OS),Windows_NT) - HOST_OS = WINDOWS -else - UNAME_S := $(shell uname -s) - ifeq ($(UNAME_S),Linux) - HOST_OS := LINUX - endif - ifeq ($(UNAME_S),Darwin) - HOST_OS := OSX - endif -endif - -HOST_ARCH := $(shell if [[ $(shell uname -m) =~ i[345678]86 ]]; then echo x86_32; else echo $(shell uname -m); fi) - -# Where compiled objects are stored. -HOST_OBJDIR := $(MAKEFILE_DIR)/gen/host_obj/ -HOST_BINDIR := $(MAKEFILE_DIR)/gen/host_bin/ -HOST_GENDIR := $(MAKEFILE_DIR)/gen/host_obj/ - -# Settings for the host compiler. -HOST_CXX := $(CC_PREFIX) gcc -HOST_CXXFLAGS := --std=c++14 -HOST_LDOPTS := -ifeq ($(HAS_GEN_HOST_PROTOC),true) - HOST_LDOPTS += -L$(MAKEFILE_DIR)/gen/protobuf-host/lib -endif -HOST_LDOPTS += -L/usr/local/lib - -HOST_INCLUDES := \ --I. \ --I$(MAKEFILE_DIR)/../../../ \ --I$(MAKEFILE_DIR)/downloads/ \ --I$(MAKEFILE_DIR)/downloads/eigen \ --I$(MAKEFILE_DIR)/downloads/gemmlowp \ --I$(MAKEFILE_DIR)/downloads/nsync/public \ --I$(MAKEFILE_DIR)/downloads/fft2d \ --I$(MAKEFILE_DIR)/downloads/double_conversion \ --I$(MAKEFILE_DIR)/downloads/absl \ --I$(HOST_GENDIR) -ifeq ($(HAS_GEN_HOST_PROTOC),true) - HOST_INCLUDES += -I$(MAKEFILE_DIR)/gen/protobuf-host/include -endif -# This is at the end so any globally-installed frameworks like protobuf don't -# override local versions in the source tree. -HOST_INCLUDES += -I/usr/local/include - -HOST_LIBS := \ -$(HOST_NSYNC_LIB) \ --lstdc++ \ --lprotobuf \ --lpthread \ --lm \ --lz - -# If we're on Linux, also link in the dl library. -ifeq ($(HOST_OS),LINUX) - HOST_LIBS += -ldl -lpthread -lrt -endif - -# If we're on a Pi, link in pthreads and dl -ifeq ($(HOST_OS),PI) - HOST_LIBS += -ldl -lpthread -endif - -# Abseil sources. -ABSL_CC_ALL_SRCS := \ -$(wildcard tensorflow/contrib/makefile/downloads/absl/absl/*/*.cc) \ -$(wildcard tensorflow/contrib/makefile/downloads/absl/absl/*/*/*.cc) \ -$(wildcard tensorflow/contrib/makefile/downloads/absl/absl/*/*/*/*.cc) \ -$(wildcard tensorflow/contrib/makefile/downloads/absl/absl/*/*/*/*/*.cc) - -ABSL_CC_EXCLUDE_SRCS := \ -$(wildcard tensorflow/contrib/makefile/downloads/absl/absl/*/*test*.cc) \ -$(wildcard tensorflow/contrib/makefile/downloads/absl/absl/*/*/*test*.cc) \ -$(wildcard tensorflow/contrib/makefile/downloads/absl/absl/*/*/*/*test*.cc) \ -$(wildcard tensorflow/contrib/makefile/downloads/absl/absl/*/*/*/*/*test*.cc) \ -$(wildcard tensorflow/contrib/makefile/downloads/absl/absl/*/*benchmark*.cc) \ -$(wildcard tensorflow/contrib/makefile/downloads/absl/absl/*/*/*benchmark*.cc) \ -$(wildcard tensorflow/contrib/makefile/downloads/absl/absl/*/*/*/*benchmark*.cc) \ -$(wildcard tensorflow/contrib/makefile/downloads/absl/absl/*/*/*/*/*benchmark*.cc) \ -tensorflow/contrib/makefile/downloads/absl/absl/synchronization/internal/mutex_nonprod.cc \ -tensorflow/contrib/makefile/downloads/absl/absl/hash/internal/print_hash_of.cc - -ABSL_CC_SRCS := $(filter-out $(ABSL_CC_EXCLUDE_SRCS), $(ABSL_CC_ALL_SRCS)) - -# proto_text is a tool that converts protobufs into a form we can use more -# compactly within TensorFlow. It's a bit like protoc, but is designed to -# produce a much more minimal result so we can save binary space. -# We have to build it on the host system first so that we can create files -# that are needed for the runtime building. -PROTO_TEXT := $(HOST_BINDIR)proto_text -# The list of dependencies is derived from the Bazel build file by running -# the gen_file_lists.sh script on a system with a working Bazel setup. -PROTO_TEXT_CC_FILES := \ - $(ABSL_CC_SRCS) \ - $(shell cat $(MAKEFILE_DIR)/proto_text_cc_files.txt) -PROTO_TEXT_PB_CC_LIST := \ - $(shell cat $(MAKEFILE_DIR)/proto_text_pb_cc_files.txt) \ - $(wildcard tensorflow/contrib/makefile/downloads/double_conversion/double-conversion/*.cc) -PROTO_TEXT_PB_H_LIST := $(shell cat $(MAKEFILE_DIR)/proto_text_pb_h_files.txt) - -# Locations of the intermediate files proto_text generates. -PROTO_TEXT_PB_H_FILES := $(addprefix $(HOST_GENDIR), $(PROTO_TEXT_PB_H_LIST)) -PROTO_TEXT_CC_OBJS := $(addprefix $(HOST_OBJDIR), $(PROTO_TEXT_CC_FILES:.cc=.o)) -PROTO_TEXT_PB_OBJS := $(addprefix $(HOST_OBJDIR), $(PROTO_TEXT_PB_CC_LIST:.cc=.o)) -PROTO_TEXT_OBJS := $(PROTO_TEXT_CC_OBJS) $(PROTO_TEXT_PB_OBJS) - -# Target device settings. - -# Default to running on the same system we're compiling on. -# You should override TARGET on the command line if you're cross-compiling, e.g. -# make -f tensorflow/contrib/makefile/Makefile TARGET=ANDROID -TARGET := $(HOST_OS) - -# Where compiled objects are stored. -GENDIR := $(MAKEFILE_DIR)/gen/ -OBJDIR := $(GENDIR)obj/ -LIBDIR := $(GENDIR)lib/ -BINDIR := $(GENDIR)bin/ -PBTGENDIR := $(GENDIR)proto_text/ -PROTOGENDIR := $(GENDIR)proto/ -DEPDIR := $(GENDIR)dep/ -$(shell mkdir -p $(DEPDIR) >/dev/null) - -# Settings for the target compiler. -CXX := $(CC_PREFIX) gcc -OPTFLAGS := -O2 - -ifneq ($(TARGET),ANDROID) - OPTFLAGS += -march=native -endif - -CXXFLAGS := --std=c++14 -DIS_SLIM_BUILD -fno-exceptions -DNDEBUG $(OPTFLAGS) -LDFLAGS := \ --L/usr/local/lib -DEPFLAGS = -MT $@ -MMD -MP -MF $(DEPDIR)/$*.Td - -INCLUDES := \ --I. \ --I$(MAKEFILE_DIR)/downloads/ \ --I$(MAKEFILE_DIR)/downloads/eigen \ --I$(MAKEFILE_DIR)/downloads/gemmlowp \ --I$(MAKEFILE_DIR)/downloads/nsync/public \ --I$(MAKEFILE_DIR)/downloads/fft2d \ --I$(MAKEFILE_DIR)/downloads/double_conversion \ --I$(MAKEFILE_DIR)/downloads/absl \ --I$(PROTOGENDIR) \ --I$(PBTGENDIR) -ifeq ($(HAS_GEN_HOST_PROTOC),true) - INCLUDES += -I$(MAKEFILE_DIR)/gen/protobuf-host/include -endif -# This is at the end so any globally-installed frameworks like protobuf don't -# override local versions in the source tree. -INCLUDES += -I/usr/local/include - -# If `$(WITH_TFLITE_FLEX)` is `true`, this Makefile will build a library -# for TensorFlow Lite Flex runtime. -# Farmhash and Flatbuffer is required for TensorFlow Lite Flex runtime. -ifeq ($(WITH_TFLITE_FLEX), true) - HOST_INCLUDES += -I$(MAKEFILE_DIR)/downloads/farmhash/src - HOST_INCLUDES += -I$(MAKEFILE_DIR)/downloads/flatbuffers/include - INCLUDES += -I$(MAKEFILE_DIR)/downloads/farmhash/src - INCLUDES += -I$(MAKEFILE_DIR)/downloads/flatbuffers/include -endif - -LIBS := \ -$(TARGET_NSYNC_LIB) \ --lstdc++ \ --lprotobuf \ --lz \ --lm - -ifeq ($(HAS_GEN_HOST_PROTOC),true) - PROTOC := $(MAKEFILE_DIR)/gen/protobuf-host/bin/protoc -else - PROTOC := protoc -endif - -$(info PROTOC = "$(PROTOC)") -$(info CC_PREFIX = "$(CC_PREFIX)") - -PROTOCFLAGS := -AR := ar -ARFLAGS := -r -LIBFLAGS := - -# If we're on OS X, make sure that globals aren't stripped out. -ifeq ($(TARGET),OSX) -ifeq ($(HAS_GEN_HOST_PROTOC),true) - LIBFLAGS += -L$(MAKEFILE_DIR)/gen/protobuf-host/lib - export LD_LIBRARY_PATH=$(MAKEFILE_DIR)/gen/protobuf-host/lib -endif - LDFLAGS += -all_load -endif -# Make sure that we don't strip global constructors on Linux. -ifeq ($(TARGET),LINUX) -ifeq ($(HAS_GEN_HOST_PROTOC),true) - LIBFLAGS += -L$(MAKEFILE_DIR)/gen/protobuf-host/lib - export LD_LIBRARY_PATH=$(MAKEFILE_DIR)/gen/protobuf-host/lib -endif - CXXFLAGS += -fPIC - LIBFLAGS += -Wl,--allow-multiple-definition -Wl,--whole-archive - LDFLAGS := -Wl,--no-whole-archive -endif -# If we're on Linux, also link in the dl library. -ifeq ($(TARGET),LINUX) - LIBS += -ldl -lpthread -lrt -endif -# If we're cross-compiling for the Raspberry Pi, use the right gcc. -ifeq ($(TARGET),PI) - CXXFLAGS += $(ANDROID_TYPES) -DRASPBERRY_PI - LDFLAGS := -Wl,--no-whole-archive - LIBS += -ldl -lpthread - LIBFLAGS += -Wl,--allow-multiple-definition -Wl,--whole-archive -endif - -# Set up Android building -ifeq ($(TARGET),ANDROID) -# Override NDK_ROOT on the command line with your own NDK location, e.g. -# make -f tensorflow/contrib/makefile/Makefile TARGET=ANDROID \ -# NDK_ROOT=/path/to/your/ndk -# You need to have an Android version of the protobuf libraries compiled to link -# in. The compile_android_protobuf.sh script may help. - - ANDROID_HOST_OS_ARCH := - ifeq ($(HOST_OS),LINUX) - ANDROID_HOST_OS_ARCH=linux - endif - ifeq ($(HOST_OS),OSX) - ANDROID_HOST_OS_ARCH=darwin - endif - ifeq ($(HOST_OS),WINDOWS) - $(error "windows is not supported.") - endif - - ifeq ($(HOST_ARCH),x86_32) - ANDROID_HOST_OS_ARCH := $(ANDROID_HOST_OS_ARCH)-x86 - else - ANDROID_HOST_OS_ARCH := $(ANDROID_HOST_OS_ARCH)-$(HOST_ARCH) - endif - - ifndef ANDROID_ARCH - ANDROID_ARCH := armeabi-v7a - endif - - ifeq ($(ANDROID_ARCH),arm64-v8a) - TOOLCHAIN := aarch64-linux-android-4.9 - SYSROOT_ARCH := arm64 - BIN_PREFIX := aarch64-linux-android - MARCH_OPTION := - endif - ifeq ($(ANDROID_ARCH),armeabi) - TOOLCHAIN := arm-linux-androideabi-4.9 - SYSROOT_ARCH := arm - BIN_PREFIX := arm-linux-androideabi - MARCH_OPTION := - endif - ifeq ($(ANDROID_ARCH),armeabi-v7a) - TOOLCHAIN := arm-linux-androideabi-4.9 - SYSROOT_ARCH := arm - BIN_PREFIX := arm-linux-androideabi - MARCH_OPTION := -march=armv7-a -mfloat-abi=softfp -mfpu=neon - endif - ifeq ($(ANDROID_ARCH),mips) - TOOLCHAIN := mipsel-linux-android-4.9 - SYSROOT_ARCH := mips - BIN_PREFIX := mipsel-linux-android - MARCH_OPTION := - endif - ifeq ($(ANDROID_ARCH),mips64) - TOOLCHAIN := mips64el-linux-android-4.9 - SYSROOT_ARCH := mips64 - BIN_PREFIX := mips64el-linux-android - MARCH_OPTION := - endif - ifeq ($(ANDROID_ARCH),x86) - TOOLCHAIN := x86-4.9 - SYSROOT_ARCH := x86 - BIN_PREFIX := i686-linux-android - MARCH_OPTION := - endif - ifeq ($(ANDROID_ARCH),x86_64) - TOOLCHAIN := x86_64-4.9 - SYSROOT_ARCH := x86_64 - BIN_PREFIX := x86_64-linux-android - MARCH_OPTION := - endif - - ifndef NDK_ROOT - $(error "NDK_ROOT is not defined.") - endif - CXX := $(CC_PREFIX) $(NDK_ROOT)/toolchains/$(TOOLCHAIN)/prebuilt/$(ANDROID_HOST_OS_ARCH)/bin/$(BIN_PREFIX)-g++ - CC := $(CC_PREFIX) $(NDK_ROOT)/toolchains/$(TOOLCHAIN)/prebuilt/$(ANDROID_HOST_OS_ARCH)/bin/$(BIN_PREFIX)-gcc - CXXFLAGS +=\ ---sysroot $(NDK_ROOT)/platforms/android-21/arch-$(SYSROOT_ARCH) \ --Wno-narrowing \ --fomit-frame-pointer \ -$(MARCH_OPTION) \ --fPIE \ --fPIC - INCLUDES = \ --I$(NDK_ROOT)/sources/android/support/include \ --I$(NDK_ROOT)/sources/cxx-stl/gnu-libstdc++/4.9/include \ --I$(NDK_ROOT)/sources/cxx-stl/gnu-libstdc++/4.9/libs/$(ANDROID_ARCH)/include \ --I. \ --I$(MAKEFILE_DIR)/downloads/ \ --I$(MAKEFILE_DIR)/downloads/eigen \ --I$(MAKEFILE_DIR)/downloads/gemmlowp \ --I$(MAKEFILE_DIR)/downloads/nsync/public \ --I$(MAKEFILE_DIR)/downloads/fft2d \ --I$(MAKEFILE_DIR)/downloads/double_conversion \ --I$(MAKEFILE_DIR)/downloads/absl \ --I$(MAKEFILE_DIR)/gen/protobuf_android/$(ANDROID_ARCH)/include \ --I$(PROTOGENDIR) \ --I$(PBTGENDIR) - - LIBS := \ -$(TARGET_NSYNC_LIB) \ --lgnustl_static \ --lprotobuf \ --llog \ --lz \ --lm \ --ldl \ --latomic - - LD := $(NDK_ROOT)/toolchains/$(TOOLCHAIN)/prebuilt/$(ANDROID_HOST_OS_ARCH)/$(BIN_PREFIX)/bin/ld - - LDFLAGS := \ -$(MARCH_OPTION) \ --L$(MAKEFILE_DIR)/gen/protobuf_android/$(ANDROID_ARCH)/lib \ --L$(NDK_ROOT)/sources/cxx-stl/gnu-libstdc++/4.9/libs/$(ANDROID_ARCH) \ --fPIE \ --pie \ --v - - AR := $(NDK_ROOT)/toolchains/$(TOOLCHAIN)/prebuilt/$(ANDROID_HOST_OS_ARCH)/bin/$(BIN_PREFIX)-ar - ARFLAGS := r - LIBFLAGS += -Wl,--allow-multiple-definition -Wl,--whole-archive - - ifdef HEXAGON_LIBS - INCLUDES += -I$(HEXAGON_INCLUDE) - LIBS += -lhexagon_controller - LDFLAGS += -L$(HEXAGON_LIBS) - CXXFLAGS += -DUSE_HEXAGON_LIBS - -# CAVEAT: We should disable TENSORFLOW_DISABLE_META while running -# quantized_matmul on Android because it crashes in -# MultiThreadGemm in tensorflow/core/kernels/meta_support.cc -# See http://b/33270149 -# TODO(satok): Remove once it's fixed - CXXFLAGS += -DTENSORFLOW_DISABLE_META - -# Declare __ANDROID_TYPES_FULL__ to enable required types for hvx - CXXFLAGS += -D__ANDROID_TYPES_FULL__ - endif - - ifdef ENABLE_EXPERIMENTAL_HEXNN_OPS - CXXFLAGS += -DENABLE_EXPERIMENTAL_HEXNN_OPS - endif - - ifeq ($(BUILD_FOR_TEGRA),1) - NVCC := $(JETPACK)/cuda/bin/nvcc - NVCCFLAGS := -x=cu -D__CUDACC__ -DNVCC -DANDROID_TEGRA -ccbin $(NDK_ROOT)/toolchains/$(TOOLCHAIN)/prebuilt/$(ANDROID_HOST_OS_ARCH)/bin/$(BIN_PREFIX)-g++ --std c++14 --expt-relaxed-constexpr -m64 -gencode arch=compute_53,\"code=sm_53\" -gencode arch=compute_62,\"code=sm_62\" -DEIGEN_AVOID_STL_ARRAY -DTENSORFLOW_USE_EIGEN_THREADPOOL -DLANG_CXX14 -DEIGEN_HAS_C99_MATH -DGOOGLE_CUDA=1 -DTF_EXTRA_CUDA_CAPABILITIES=5.3 - CXXFLAGS4NVCC =\ --DIS_SLIM_BUILD \ --DANDROID_TEGRA \ --fno-exceptions \ --DNDEBUG $(OPTFLAGS) \ --march=armv8-a \ --fPIE \ --D__ANDROID_TYPES_FULL__ \ ---sysroot $(NDK_ROOT)/platforms/android-21/arch-arm64 - - CXXFLAGS +=\ --DGOOGLE_CUDA=1 \ --D__ANDROID_TYPES_FULL__ \ --DANDROID_TEGRA \ --DEIGEN_AVOID_STL_ARRAY \ --DEIGEN_HAS_C99_MATH \ --DLANG_CXX14 -DTENSORFLOW_USE_EIGEN_THREADPOOL -DTF_EXTRA_CUDA_CAPABILITIES=5.3 - - INCLUDES += \ --Itensorflow/core/kernels \ --I$(MAKEFILE_DIR)/downloads/cub \ --I$(MAKEFILE_DIR)/downloads/cub/cub_archive/cub/device \ --I$(JETPACK)/cuda/include \ --I$(JETPACK) \ --I$(JETPACK)/cuDNN/aarch64 \ --I$(JETPACK)/cuda/extras/CUPTI/include - - - CUDA_LIBS := \ --ltfcuda \ --lcudart_static \ --lcudnn \ --lcublas_static \ --lcufftw_static \ --lcusolver_static \ --lcusparse_static \ --lcufft \ --lcuda \ --lculibos \ --lcurand_static - - OBJDIR := $(OBJDIR)android_arm64-v8a/ - LIBDIR := $(LIBDIR)android_arm64-v8a/ - BINDIR := $(BINDIR)android_arm64-v8a/ - DEPDIR := $(DEPDIR)android_arm64-v8a/ - - TEGRA_LIBS := \ --L$(JETPACK)/cuda/targets/aarch64-linux-androideabi/lib \ --L$(JETPACK)/cuda/targets/aarch64-linux-androideabi/lib/stubs \ --L$(JETPACK)/cuda/targets/aarch64-linux-androideabi/lib64 \ --L$(JETPACK)/cuda/targets/aarch64-linux-androideabi/lib64/stubs \ --L$(JETPACK)/cuDNN/aarch64/cuda/lib64 \ --L$(LIBDIR) - - CUDA_LIB_DEPS := $(LIBDIR)libtfcuda.a - else - OBJDIR := $(OBJDIR)android_$(ANDROID_ARCH)/ - LIBDIR := $(LIBDIR)android_$(ANDROID_ARCH)/ - BINDIR := $(BINDIR)android_$(ANDROID_ARCH)/ - DEPDIR := $(DEPDIR)android_$(ANDROID_ARCH)/ - endif # ifeq ($(BUILD_FOR_TEGRA),1) -endif # ANDROID - -# Settings for iOS. -ifeq ($(TARGET),IOS) - IPHONEOS_PLATFORM := $(shell xcrun --sdk iphoneos --show-sdk-platform-path) - IPHONEOS_SYSROOT := $(shell xcrun --sdk iphoneos --show-sdk-path) - IPHONESIMULATOR_PLATFORM := $(shell xcrun --sdk iphonesimulator \ - --show-sdk-platform-path) - IPHONESIMULATOR_SYSROOT := $(shell xcrun --sdk iphonesimulator \ - --show-sdk-path) - IOS_SDK_VERSION := $(shell xcrun --sdk iphoneos --show-sdk-version) - MIN_SDK_VERSION := 9.0 -# Override IOS_ARCH with ARMV7, ARMV7S, ARM64, or I386. - IOS_ARCH := X86_64 - ifeq ($(IOS_ARCH),ARMV7) - CXXFLAGS += -miphoneos-version-min=$(MIN_SDK_VERSION) \ - -arch armv7 \ - -fembed-bitcode \ - -D__thread=thread_local \ - -DUSE_GEMM_FOR_CONV \ - -Wno-c++11-narrowing \ - -mno-thumb \ - -DTF_LEAN_BINARY \ - $(ANDROID_TYPES) \ - -fno-exceptions \ - -isysroot \ - ${IPHONEOS_SYSROOT} - LDFLAGS := -arch armv7 \ - -fembed-bitcode \ - -miphoneos-version-min=${MIN_SDK_VERSION} \ - -framework Accelerate \ - -framework CoreFoundation \ - -Xlinker -S \ - -Xlinker -x \ - -Xlinker -dead_strip \ - -all_load \ - -L$(GENDIR)protobuf_ios/lib \ - -lz - endif - ifeq ($(IOS_ARCH),ARMV7S) - CXXFLAGS += -miphoneos-version-min=$(MIN_SDK_VERSION) \ - -arch armv7s \ - -fembed-bitcode \ - -D__thread=thread_local \ - -DUSE_GEMM_FOR_CONV \ - -Wno-c++11-narrowing \ - -mno-thumb \ - -DTF_LEAN_BINARY \ - $(ANDROID_TYPES) \ - -fno-exceptions \ - -isysroot \ - ${IPHONEOS_SYSROOT} - LDFLAGS := -arch armv7s \ - -fembed-bitcode \ - -miphoneos-version-min=${MIN_SDK_VERSION} \ - -framework Accelerate \ - -framework CoreFoundation \ - -Xlinker -S \ - -Xlinker -x \ - -Xlinker -dead_strip \ - -all_load \ - -L$(GENDIR)protobuf_ios/lib \ - -lz - endif - ifeq ($(IOS_ARCH),ARM64) - CXXFLAGS += -miphoneos-version-min=$(MIN_SDK_VERSION) \ - -arch arm64 \ - -fembed-bitcode \ - -D__thread=thread_local \ - -DUSE_GEMM_FOR_CONV \ - -Wno-c++11-narrowing \ - -DTF_LEAN_BINARY \ - $(ANDROID_TYPES) \ - -fno-exceptions \ - -isysroot \ - ${IPHONEOS_SYSROOT} - LDFLAGS := -arch arm64 \ - -fembed-bitcode \ - -miphoneos-version-min=${MIN_SDK_VERSION} \ - -framework Accelerate \ - -framework CoreFoundation \ - -Xlinker -S \ - -Xlinker -x \ - -Xlinker -dead_strip \ - -all_load \ - -L$(GENDIR)protobuf_ios/lib \ - -lz - endif - ifeq ($(IOS_ARCH),I386) - CXXFLAGS += -mios-simulator-version-min=$(MIN_SDK_VERSION) \ - -arch i386 \ - -mno-sse \ - -fembed-bitcode \ - -D__thread=thread_local \ - -DUSE_GEMM_FOR_CONV \ - -Wno-c++11-narrowing \ - -DTF_LEAN_BINARY \ - $(ANDROID_TYPES) \ - -fno-exceptions \ - -isysroot \ - ${IPHONESIMULATOR_SYSROOT} - LDFLAGS := -arch i386 \ - -fembed-bitcode \ - -mios-simulator-version-min=${MIN_SDK_VERSION} \ - -framework Accelerate \ - -framework CoreFoundation \ - -Xlinker -S \ - -Xlinker -x \ - -Xlinker -dead_strip \ - -all_load \ - -L$(GENDIR)protobuf_ios/lib \ - -lz - endif - ifeq ($(IOS_ARCH),X86_64) - CXXFLAGS += -mios-simulator-version-min=$(MIN_SDK_VERSION) \ - -arch x86_64 \ - -fembed-bitcode \ - -D__thread=thread_local \ - -DUSE_GEMM_FOR_CONV \ - -Wno-c++11-narrowing \ - -DTF_LEAN_BINARY \ - $(ANDROID_TYPES) \ - -fno-exceptions \ - -isysroot \ - ${IPHONESIMULATOR_SYSROOT} - LDFLAGS := -arch x86_64 \ - -fembed-bitcode \ - -mios-simulator-version-min=${MIN_SDK_VERSION} \ - -framework Accelerate \ - -framework CoreFoundation \ - -Xlinker -S \ - -Xlinker -x \ - -Xlinker -dead_strip \ - -all_load \ - -L$(GENDIR)protobuf_ios/lib \ - -lz - endif - OBJDIR := $(OBJDIR)ios_$(IOS_ARCH)/ - LIBDIR := $(LIBDIR)ios_$(IOS_ARCH)/ - BINDIR := $(BINDIR)ios_$(IOS_ARCH)/ - DEPDIR := $(DEPDIR)ios_$(IOS_ARCH)/ -endif - -# This library is the main target for this makefile. It will contain a minimal -# runtime that can be linked in to other programs. -LIB_NAME := libtensorflow-core.a -LIB_PATH := $(LIBDIR)$(LIB_NAME) - -# A small example program that shows how to link against the library. -BENCHMARK_NAME := $(BINDIR)benchmark - -# What sources we want to compile, derived from the main Bazel build using the -# gen_file_lists.sh script. - -CORE_CC_ALL_SRCS := \ -$(ABSL_CC_SRCS) \ -tensorflow/c/c_api.cc \ -tensorflow/c/kernels.cc \ -tensorflow/c/tf_datatype.cc \ -tensorflow/c/tf_status.cc \ -tensorflow/c/tf_status_helper.cc \ -tensorflow/c/tf_tensor.cc \ -$(wildcard tensorflow/core/*.cc) \ -$(wildcard tensorflow/core/common_runtime/*.cc) \ -$(wildcard tensorflow/core/framework/*.cc) \ -$(wildcard tensorflow/core/graph/*.cc) \ -$(wildcard tensorflow/core/grappler/*.cc) \ -$(wildcard tensorflow/core/grappler/*/*.cc) \ -$(wildcard tensorflow/core/lib/*/*.cc) \ -$(wildcard tensorflow/core/platform/*.cc) \ -$(wildcard tensorflow/core/platform/*/*.cc) \ -$(wildcard tensorflow/core/platform/*/*/*.cc) \ -$(wildcard tensorflow/core/util/*.cc) \ -$(wildcard tensorflow/core/util/*/*.cc) \ -$(wildcard tensorflow/contrib/makefile/downloads/double_conversion/double-conversion/*.cc) \ -tensorflow/core/profiler/internal/profiler_interface.cc \ -tensorflow/core/profiler/internal/traceme_recorder.cc \ -$(wildcard tensorflow/core/profiler/lib/*.cc) \ -tensorflow/core/util/version_info.cc -# Remove duplicates (for version_info.cc) -CORE_CC_ALL_SRCS := $(sort $(CORE_CC_ALL_SRCS)) - -CORE_CC_EXCLUDE_SRCS_NON_GPU := \ -$(wildcard tensorflow/core/*/*test.cc) \ -$(wildcard tensorflow/core/*/*testutil*) \ -$(wildcard tensorflow/core/*/*testlib*) \ -$(wildcard tensorflow/core/*/*main.cc) \ -$(wildcard tensorflow/core/*/*/*test.cc) \ -$(wildcard tensorflow/core/*/*/*testutil*) \ -$(wildcard tensorflow/core/*/*/*testlib*) \ -$(wildcard tensorflow/core/*/*/*main.cc) \ -$(wildcard tensorflow/core/debug/*.cc) \ -$(wildcard tensorflow/core/framework/op_gen_lib.cc) \ -$(wildcard tensorflow/core/graph/dot.*) \ -$(wildcard tensorflow/core/lib/db/*) \ -$(wildcard tensorflow/core/lib/gif/*) \ -$(wildcard tensorflow/core/lib/io/zlib*) \ -$(wildcard tensorflow/core/lib/io/record*) \ -$(wildcard tensorflow/core/lib/jpeg/*) \ -$(wildcard tensorflow/core/lib/png/*) \ -$(wildcard tensorflow/core/util/events_writer.*) \ -$(wildcard tensorflow/core/util/reporter.*) \ -$(wildcard tensorflow/core/platform/default/test_benchmark.*) \ -$(wildcard tensorflow/core/platform/cloud/*) \ -$(wildcard tensorflow/core/platform/google/*) \ -$(wildcard tensorflow/core/platform/google/*/*) \ -$(wildcard tensorflow/core/platform/jpeg.*) \ -$(wildcard tensorflow/core/platform/png.*) \ -$(wildcard tensorflow/core/platform/s3/*) \ -$(wildcard tensorflow/core/platform/windows/*) \ -$(wildcard tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.*) \ -$(wildcard tensorflow/core/grappler/inputs/file_input_yielder.*) \ -$(wildcard tensorflow/core/grappler/clusters/single_machine.*) \ -tensorflow/core/util/gpu_kernel_helper_test.cu.cc - -CORE_CC_EXCLUDE_SRCS := \ -$(CORE_CC_EXCLUDE_SRCS_NON_GPU) \ -$(wildcard tensorflow/core/platform/stream_executor.*) \ -$(wildcard tensorflow/core/platform/default/cuda_libdevice_path.*) \ -$(wildcard tensorflow/core/platform/cuda.h) \ -$(wildcard tensorflow/core/platform/cuda_libdevice_path.*) \ -$(wildcard tensorflow/core/user_ops/*.cu.cc) \ -$(wildcard tensorflow/core/common_runtime/gpu/*) \ -$(wildcard tensorflow/core/common_runtime/gpu_device_factory.*) - -ifeq ($(BUILD_FOR_TEGRA),1) -CORE_CC_ALL_SRCS := $(CORE_CC_ALL_SRCS) \ -tensorflow/core/kernels/concat_lib_gpu.cc \ -tensorflow/core/kernels/cuda_solvers.cc \ -tensorflow/core/kernels/cudnn_pooling_gpu.cc \ -tensorflow/core/kernels/dense_update_functor.cc \ -tensorflow/core/kernels/fractional_avg_pool_op.cc \ -tensorflow/core/kernels/fractional_max_pool_op.cc \ -tensorflow/core/kernels/fractional_pool_common.cc \ -tensorflow/core/kernels/pooling_ops_3d.cc \ -tensorflow/core/kernels/sparse_fill_empty_rows_op.cc \ -tensorflow/core/kernels/list_kernels.cc \ -$(wildcard tensorflow/core/common_runtime/gpu/*.cc) \ -$(wildcard tensorflow/stream_executor/*.cc) \ -$(wildcard tensorflow/stream_executor/*/*.cc) - -CORE_CC_EXCLUDE_SRCS := \ -$(CORE_CC_EXCLUDE_SRCS_NON_GPU) - -CUDA_CC_SRCS := $(wildcard tensorflow/core/kernels/*.cu.cc) -CUDA_CC_OBJS := $(addprefix $(OBJDIR), $(CUDA_CC_SRCS:.cc=.o)) -endif # TEGRA - -# Filter out all the excluded files. -TF_CC_SRCS := $(filter-out $(CORE_CC_EXCLUDE_SRCS), $(CORE_CC_ALL_SRCS)) -# Add in any extra files that don't fit the patterns easily -TF_CC_SRCS += tensorflow/contrib/makefile/downloads/fft2d/fftsg.c -TF_CC_SRCS += tensorflow/core/common_runtime/gpu/gpu_id_manager.cc -# Also include the op and kernel definitions. -TF_CC_SRCS += $(shell cat $(MAKEFILE_DIR)/tf_op_files.txt) -PBT_CC_SRCS := $(shell cat $(MAKEFILE_DIR)/tf_pb_text_files.txt) -PROTO_SRCS := $(shell cat $(MAKEFILE_DIR)/tf_proto_files.txt) -BENCHMARK_SRCS := \ -tensorflow/core/util/reporter.cc \ -tensorflow/tools/benchmark/benchmark_model.cc \ -tensorflow/tools/benchmark/benchmark_model_main.cc - -# If `$(WITH_TFLITE_FLEX)` is `true`, this Makefile will build a library -# for TensorFlow Lite Flex runtime. -# Adding the following dependencies> -# * TensorFlow Eager Runtime. -# * TensorFlow Lite Runtime. -# * TensorFlow Lite Flex Delegate. -ifeq ($(WITH_TFLITE_FLEX), true) - EAGER_CC_ALL_SRCS += $(wildcard tensorflow/core/common_runtime/eager/*.cc) - EAGER_CC_EXCLUDE_SRCS := $(wildcard tensorflow/core/common_runtime/eager/*test.cc) - EAGER_CC_SRCS := $(filter-out $(EAGER_CC_EXCLUDE_SRCS), $(EAGER_CC_ALL_SRCS)) - TF_CC_SRCS += $(EAGER_CC_SRCS) - - TF_LITE_CORE_CC_ALL_SRCS := \ - $(wildcard tensorflow/lite/*.cc) \ - $(wildcard tensorflow/lite/*.c) \ - $(wildcard tensorflow/lite/c/*.c) \ - $(wildcard tensorflow/lite/core/api/*.cc) - - TF_LITE_CORE_CC_ALL_SRCS += \ - $(wildcard tensorflow/lite/kernels/*.cc) \ - $(wildcard tensorflow/lite/kernels/internal/*.cc) \ - $(wildcard tensorflow/lite/kernels/internal/optimized/*.cc) \ - $(wildcard tensorflow/lite/kernels/internal/reference/*.cc) \ - $(PROFILER_SRCS) \ - $(wildcard tensorflow/lite/kernels/*.c) \ - $(wildcard tensorflow/lite/kernels/internal/*.c) \ - $(wildcard tensorflow/lite/kernels/internal/optimized/*.c) \ - $(wildcard tensorflow/lite/kernels/internal/reference/*.c) \ - $(wildcard tensorflow/lite/delegates/flex/*.cc) - - # Hack. This shouldn't be here? - TF_LITE_CORE_CC_ALL_SRCS += \ - $(wildcard tensorflow/contrib/makefile/downloads/farmhash/src/farmhash.cc) \ - - # Remove any duplicates. - TF_LITE_CORE_CC_ALL_SRCS := $(sort $(TF_LITE_CORE_CC_ALL_SRCS)) - TF_LITE_CORE_CC_EXCLUDE_SRCS := \ - $(wildcard tensorflow/lite/*test.cc) \ - $(wildcard tensorflow/lite/*/*test.cc) \ - $(wildcard tensorflow/lite/*/*/*test.cc) \ - $(wildcard tensorflow/lite/*/*/*/*test.cc) \ - $(wildcard tensorflow/lite/kernels/test_util.cc) \ - $(wildcard tensorflow/lite/delegates/flex/test_util.cc) \ - $(wildcard tensorflow/lite/nnapi_delegate.cc) \ - $(wildcard tensorflow/lite/mmap_allocation_disabled.cc) - - # Filter out all the excluded files. - TF_LITE_CC_SRCS := $(filter-out $(TF_LITE_CORE_CC_EXCLUDE_SRCS), $(TF_LITE_CORE_CC_ALL_SRCS)) - TF_CC_SRCS += $(TF_LITE_CC_SRCS) -endif - -ifdef HEXAGON_LIBS - TF_CC_SRCS += \ -tensorflow/cc/framework/scope.cc \ -tensorflow/cc/framework/ops.cc \ -tensorflow/cc/ops/const_op.cc \ -tensorflow/core/kernels/hexagon/graph_transfer_utils.cc \ -tensorflow/core/kernels/hexagon/graph_transferer.cc \ -tensorflow/core/kernels/hexagon/hexagon_control_wrapper.cc \ -tensorflow/core/kernels/hexagon/hexagon_ops_definitions.cc \ -tensorflow/core/kernels/hexagon/hexagon_remote_fused_graph_executor_build.cc -endif - -# File names of the intermediate files target compilation generates. -TF_CC_OBJS := $(addprefix $(OBJDIR), \ -$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(TF_CC_SRCS)))) -PBT_GEN_FILES := $(addprefix $(PBTGENDIR), $(PBT_CC_SRCS)) -PBT_OBJS := $(addprefix $(OBJDIR), $(PBT_CC_SRCS:.cc=.o)) -PROTO_CC_SRCS := $(addprefix $(PROTOGENDIR), $(PROTO_SRCS:.proto=.pb.cc)) -PROTO_OBJS := $(addprefix $(OBJDIR), $(PROTO_SRCS:.proto=.pb.o)) -LIB_OBJS := $(PROTO_OBJS) $(TF_CC_OBJS) $(PBT_OBJS) -BENCHMARK_OBJS := $(addprefix $(OBJDIR), $(BENCHMARK_SRCS:.cc=.o)) - -.PHONY: clean cleantarget - -# The target that's compiled if there's no command-line arguments. -all: $(LIB_PATH) $(BENCHMARK_NAME) - -# Rules for target compilation. - - -.phony_version_info: -tensorflow/core/util/version_info.cc: .phony_version_info - tensorflow/tools/git/gen_git_source.sh $@ - -# Gathers together all the objects we've compiled into a single '.a' archive. -$(LIB_PATH): $(LIB_OBJS) - @mkdir -p $(dir $@) - $(AR) $(ARFLAGS) $(LIB_PATH) $(LIB_OBJS) - -$(BENCHMARK_NAME): $(BENCHMARK_OBJS) $(LIB_PATH) $(CUDA_LIB_DEPS) - @mkdir -p $(dir $@) - $(CXX) $(CXXFLAGS) $(INCLUDES) \ - -o $(BENCHMARK_NAME) $(BENCHMARK_OBJS) \ - $(LIBFLAGS) $(TEGRA_LIBS) $(LIB_PATH) $(LDFLAGS) $(LIBS) $(CUDA_LIBS) - -# NVCC compilation rules for Tegra -ifeq ($(BUILD_FOR_TEGRA),1) -$(OBJDIR)%.cu.o: %.cu.cc - @mkdir -p $(dir $@) - @mkdir -p $(dir $(DEPDIR)$*) - $(NVCC) $(NVCCFLAGS) -Xcompiler "$(CXXFLAGS4NVCC) $(DEPFLAGS)" $(INCLUDES) -c $< -o $@ - -$(LIBDIR)libtfcuda.a: $(CUDA_CC_OBJS) - @mkdir -p $(dir $@) - $(AR) $(ARFLAGS) $@ $(CUDA_CC_OBJS) -endif - -# Matches on the normal hand-written TensorFlow C++ source files. -$(OBJDIR)%.o: %.cc | $(PBT_GEN_FILES) - @mkdir -p $(dir $@) - @mkdir -p $(dir $(DEPDIR)$*) - $(CXX) $(CXXFLAGS) $(DEPFLAGS) $(INCLUDES) -c $< -o $@ - @mv -f $(DEPDIR)/$*.Td $(DEPDIR)/$*.d - -# Matches on plain C files. -$(OBJDIR)%.o: %.c - @mkdir -p $(dir $@) - @mkdir -p $(dir $(DEPDIR)$*) - $(CXX) $(patsubst --std=c++14,--std=c99, $(CXXFLAGS)) -x c $(DEPFLAGS) \ -$(INCLUDES) -c $< -o $@ - @mv -f $(DEPDIR)/$*.Td $(DEPDIR)/$*.d - -# Compiles C++ source files that have been generated by protoc. -$(OBJDIR)%.pb.o: $(PROTOGENDIR)%.pb.cc - @mkdir -p $(dir $@) - $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ - -# Builds C++ code from proto files using protoc. -$(PROTOGENDIR)%.pb.cc $(PROTOGENDIR)%.pb.h: %.proto - @mkdir -p $(dir $@) - $(PROTOC) $(PROTOCFLAGS) $< --cpp_out $(PROTOGENDIR) - -# Uses proto_text to generate minimal pb_text C++ files from protos. -$(PBTGENDIR)%.pb_text.cc $(PBTGENDIR)%.pb_text.h $(PBTGENDIR)%.pb_text-impl.h: %.proto | $(PROTO_TEXT) - @mkdir -p $(dir $@) - $(PROTO_TEXT) \ - $(PBTGENDIR)tensorflow/core \ - tensorflow/core/ \ - tensorflow/tools/proto_text/placeholder.txt \ - $< - -# Compiles the C++ source files created by proto_text. -$(OBJDIR)%.pb_text.o: $(PBTGENDIR)%.pb_text.cc - @mkdir -p $(dir $@) - $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ - -# Makes sure that we don't compile the protoc-generated C++ sources before they -# and the proto_text files have been created. -$(PROTO_OBJS): $(PROTO_CC_SRCS) $(PBT_GEN_FILES) - -# Host compilation rules. - -# For normal manually-created TensorFlow C++ source files. -$(HOST_OBJDIR)%.o: %.cc - @mkdir -p $(dir $@) - $(HOST_CXX) $(HOST_CXXFLAGS) $(HOST_INCLUDES) -c $< -o $@ - -# Compiles object code from protoc-built C++ source files. -$(HOST_OBJDIR)%.pb.o: $(HOST_GENDIR)%.pb.cc - @mkdir -p $(dir $@) - $(HOST_CXX) $(HOST_CXXFLAGS) $(HOST_INCLUDES) -c $< -o $@ - -# Ensures we wait until proto_text has generated the .h files from protos before -# we compile the C++. -$(PROTO_TEXT_OBJS) : $(PROTO_TEXT_PB_H_FILES) - -# Ensures we link CoreFoundation as it is used for time library when building -# for Mac and iOS -ifeq ($(TARGET),OSX) - ifeq ($(HOST_ARCH),x86_64) - HOST_LDOPTS += -framework CoreFoundation - LIBS += -framework CoreFoundation - endif -endif -ifeq ($(TARGET),IOS) - HOST_LDOPTS += -framework CoreFoundation -endif - -# Runs proto_text to generate C++ source files from protos. -$(PROTO_TEXT): $(PROTO_TEXT_OBJS) $(PROTO_TEXT_PB_H_FILES) - @mkdir -p $(dir $@) - $(HOST_CXX) $(HOST_CXXFLAGS) $(HOST_INCLUDES) \ - -o $(PROTO_TEXT) $(PROTO_TEXT_OBJS) $(HOST_LDOPTS) $(HOST_LIBS) - -# Compiles the C++ source files from protos using protoc. -$(HOST_GENDIR)%.pb.cc $(HOST_GENDIR)%.pb.h: %.proto - @mkdir -p $(dir $@) - $(PROTOC) $(PROTOCFLAGS) $< --cpp_out $(HOST_GENDIR) - -# Gets rid of all generated files. -clean: - rm -rf $(MAKEFILE_DIR)/gen - rm -rf tensorflow/core/util/version_info.cc - -# Gets rid of all generated files except protobuf libs generated -# before calling make. This allows users not to recompile proto libs everytime. -clean_except_protobuf_libs: - find $(MAKEFILE_DIR)/gen -mindepth 1 -maxdepth 1 ! -name "protobuf*" -exec rm -r "{}" \; - rm -rf tensorflow/core/util/version_info.cc - -# Gets rid of target files only, leaving the host alone. Also leaves the lib -# directory untouched deliberately, so we can persist multiple architectures -# across builds for iOS and Android. -cleantarget: - rm -rf $(OBJDIR) - rm -rf $(BINDIR) - rm -rf $(LIBDIR) - -$(DEPDIR)/%.d: ; -.PRECIOUS: $(DEPDIR)/%.d - --include $(patsubst %,$(DEPDIR)/%.d,$(basename $(TF_CC_SRCS))) - -ifdef SUB_MAKEFILES - $(warning "include sub makefiles, must not contain white spaces in the path:" $(SUB_MAKEFILES)) - include $(SUB_MAKEFILES) -endif diff --git a/tensorflow/contrib/makefile/README.md b/tensorflow/contrib/makefile/README.md deleted file mode 100644 index a7b8113f597..00000000000 --- a/tensorflow/contrib/makefile/README.md +++ /dev/null @@ -1,501 +0,0 @@ -### TensorFlow Makefile - -The recommended way to build TensorFlow from source is using the Bazel -open-source build system. Sometimes this isn't possible. For example, -if you are building for iOS, you currently need to use the Makefile. - - - The build system may not have the RAM or processing power to support Bazel. - - Bazel or its dependencies may not be available. - - You may want to cross-compile for an unsupported target system. - -This experimental project supplies a Makefile automatically derived from the -dependencies listed in the Bazel project that can be used with GNU's make tool. -With it, you can compile the core C++ runtime into a static library. - -This static library will not contain: - - - Python or other language bindings - - GPU support - -You can target: -- iOS -- OS X (macOS) -- Android -- Raspberry-PI - -You will compile tensorflow and protobuf libraries that you can link into other -applications. You will also compile the [benchmark](../../tools/benchmark/) -application that will let you check your application. - -## Before you start (all platforms) - -First, clone this TensorFlow repository. - -You will need to download all dependencies as well. We have provided a script -that does so, to be run (as with all commands) **at the root of the repository**: - -```bash -tensorflow/contrib/makefile/download_dependencies.sh -``` - -You should only need to do this step once. It downloads the required libraries -like Eigen in the `tensorflow/contrib/makefile/downloads/` folder. - -You should download the example graph from [https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip](https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip). - -## Building on Linux - -_Note: This has only been tested on Ubuntu._ - -As a first step, you need to make sure the required packages are installed: -```bash -sudo apt-get install autoconf automake libtool curl make g++ unzip zlib1g-dev \ -git python -``` - -You should then be able to run the `build_all_linux.sh` script to compile: -```bash -tensorflow/contrib/makefile/build_all_linux.sh -``` - -This should compile a static library in -`tensorflow/contrib/makefile/gen/lib/libtensorflow-core.a`, -and create an example executable at `tensorflow/contrib/makefile/gen/bin/benchmark`. - -Get the graph file, if you have not already: - -```bash -mkdir -p ~/graphs -curl -o ~/graphs/inception.zip \ - https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip \ - && unzip ~/graphs/inception.zip -d ~/graphs/inception -``` - -To run the executable, use: - -```bash -tensorflow/contrib/makefile/gen/bin/benchmark \ - --graph=$HOME/graphs/inception/tensorflow_inception_graph.pb -``` - -## Android - -First, you will need to download and unzip the -[Native Development Kit (NDK)](https://developer.android.com/ndk/). You will not -need to install the standalone toolchain, however. - -Assign your NDK location to $NDK_ROOT: - -```bash -export NDK_ROOT=/absolute/path/to/NDK/android-ndk-r14b -``` - -Note : libtensorflow-core.a cannot be compiled with any ndk version above r14b. - -Download the graph if you haven't already: - -```bash -mkdir -p ~/graphs -curl -o ~/graphs/inception.zip \ - https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip \ - && unzip ~/graphs/inception.zip -d ~/graphs/inception -``` - -Then, execute the following: - -```bash -tensorflow/contrib/makefile/download_dependencies.sh -tensorflow/contrib/makefile/compile_android_protobuf.sh -c -export HOST_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh` -export TARGET_NSYNC_LIB=`CC_PREFIX="${CC_PREFIX}" NDK_ROOT="${NDK_ROOT}" \ - tensorflow/contrib/makefile/compile_nsync.sh -t android -a armeabi-v7a` -make -f tensorflow/contrib/makefile/Makefile TARGET=ANDROID -``` - -At this point, you will have compiled libraries in `gen/lib/*` and the -[benchmark app](../../tools/benchmark) compiled for Android. - -Run the benchmark by pushing both the benchmark and the graph file to your -attached Android device: - -```bash -adb push ~/graphs/inception/tensorflow_inception_graph.pb /data/local/tmp/ -adb push tensorflow/contrib/makefile/gen/bin/benchmark /data/local/tmp/ -adb shell '/data/local/tmp/benchmark \ - --graph=/data/local/tmp/tensorflow_inception_graph.pb \ - --input_layer="input:0" \ - --input_layer_shape="1,224,224,3" \ - --input_layer_type="float" \ - --output_layer="output:0" -' -``` - -For more details, see the [benchmark documentation](../../tools/benchmark). - -## CUDA support for Tegra devices running Android (Nvidia Shield TV, etc) - -With the release of TF 1.6 and JetPack for Android 3.2 (currently pending), you can now build a version of TensorFlow for compatible devices according to the following instructions which will receive the full benefits of GPU acceleration. - -#### Environment setup: - -First, download and install JetPack for Android version 3.2 or greater from [Nvidia](https://developers.nvidia.com). Note that as of the TF 1.6 release the JetPack for Android 3.2 release is still pending, and regular JetPack for L4T will not work. - -```bash -git clone https://github.com/tensorflow/tensorflow.git -cd tensorflow -JETPACK=$HOME/JetPack_Android_3.2 -TEGRA_LIBS="$JETPACK/cuDNN/aarch64/cuda/lib64/libcudnn.so $JETPACK/cuda/extras/CUPTI/lib64/libcupti.so $JETPACK/cuda/targets/aarch64-linux-androideabi/lib64/libcufft.so" -``` - -#### Building all CUDA-enabled native binaries: -This will build CUDA-enabled versions of libtensorflow_inference.so and the benchmark binary. (libtensorflow_demo.so will also be built incidentally, but it does not support CUDA) - -```bash -NDK_ROOT=$JETPACK/android-ndk-r13b -CC_PREFIX=ccache tensorflow/contrib/makefile/build_all_android.sh -s tensorflow/contrib/makefile/sub_makefiles/android/Makefile.in -t "libtensorflow_inference.so libtensorflow_demo.so all" -a tegra -``` -(add -T on subsequent builds to skip protobuf downloading/building) - - -#### Testing the CUDA-enabled benchmark via adb: -Build binaries first as above, then run: - -```bash -adb shell mkdir -p /data/local/tmp/lib64 -adb push $TEGRA_LIBS /data/local/tmp/lib64 -adb push tensorflow/contrib/makefile/gen/bin/android_arm64-v8a/benchmark /data/local/tmp -wget https://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/tensorflow_demo.apk -unzip tensorflow_demo.apk -d /tmp/tensorflow_demo -adb push /tmp/tensorflow_demo/assets/*.pb /data/local/tmp -adb shell "LD_LIBRARY_PATH=/data/local/tmp/lib64 /data/local/tmp/benchmark --graph=/data/local/tmp/tensorflow_inception_graph.pb" -``` - -#### Building the CUDA-enabled TensorFlow AAR with Bazel: -Build the native binaries first as above. Then, build the aar and package the native libs by executing the following: -```bash -mkdir -p /tmp/tf/jni/arm64-v8a -cp tensorflow/contrib/makefile/gen/lib/android_tegra/libtensorflow_*.so /tmp/tf/jni/arm64-v8a/ -cp $TEGRA_LIBS /tmp/tf/jni/arm64-v8a -bazel build //tensorflow/tools/android/inference_interface:android_tensorflow_inference_java.aar -cp bazel-bin/tensorflow/tools/android/inference_interface/android_tensorflow_inference_java.aar /tmp/tf/tensorflow.aar -cd /tmp/tf -chmod +w tensorflow.aar -zip -ur tensorflow.aar $(find jni -name *.so) -``` - -#### Building the CUDA-enabled TensorFlow Android demo with Bazel: -Build binaries first as above, then edit tensorflow/examples/android/BUILD and replace: -``` - srcs = [ - ":libtensorflow_demo.so", - "//tensorflow/tools/android/inference_interface:libtensorflow_inference.so", - ], -``` -with: -``` -srcs = glob(["libs/arm64-v8a/*.so"]), -``` - -If you are building for Android TV (Shield TV devices), replace "portrait" with "landscape" for android:screenOrientation in all four activities in tensorflow/examples/android/AndroidManifest.xml - -Then run: -```bash -# Create dir for native libs -mkdir -p tensorflow/examples/android/libs/arm64-v8a - -# Copy JetPack libs -cp $TEGRA_LIBS tensorflow/examples/android/libs/arm64-v8a - -# Copy native TensorFlow libraries -cp tensorflow/contrib/makefile/gen/lib/android_arm64-v8a/libtensorflow_*.so tensorflow/examples/android/libs/arm64-v8a/ - -# Build APK -bazel build -c opt --fat_apk_cpu=arm64-v8a tensorflow/android:tensorflow_demo - -# Install -adb install -r -f bazel-bin/tensorflow/examples/android/tensorflow_demo.apk -``` - -#### Building the CUDA-enabled Android demo with gradle/Android Studio: - -Add tensorflow/examples/android as an Android project in Android Studio as normal. - -Edit build.gradle and: -* set nativeBuildSystem = 'makefile' -* set cpuType = 'arm64-v8a' -* in "buildNativeMake", replace cpuType with 'tegra' (optional speedups like -T and ccache also work) -* set the environment "NDK_ROOT" var to $JETPACK/android-ndk-r13b - -Click "build apk" to build. - -Install: -```bash -adb install -r -f tensorflow/examples/android/gradleBuild/outputs/apk/debug/android-debug.apk -``` - -## iOS - -_Note: To use this library in an iOS application, see related instructions in -the [iOS examples](../../examples/ios/) directory._ - -Install XCode 7.3 or more recent. If you have not already, you will need to -install the command-line tools using `xcode-select`: - -```bash -xcode-select --install -``` - -If this is a new install, you will need to run XCode once to agree to the -license before continuing. - -(You will also need to have [Homebrew](http://brew.sh/) installed.) - -Then install [automake](https://en.wikipedia.org/wiki/Automake)/[libtool](https://en.wikipedia.org/wiki/GNU_Libtool): - -```bash -brew install automake -brew install libtool -``` - -Also, download the graph if you haven't already: - -```bash -mkdir -p ~/graphs -curl -o ~/graphs/inception.zip \ - https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip \ - && unzip ~/graphs/inception.zip -d ~/graphs/inception -``` - -### Building all at once - -If you just want to get the libraries compiled in a hurry, you can run this -from the root of your TensorFlow source folder: - -```bash -tensorflow/contrib/makefile/build_all_ios.sh -``` - -This process will take around twenty minutes on a modern MacBook Pro. - -When it completes, you will have a unified library for all architectures -(i386sim, x86_64sim, armv7, armv7s and arm64) and the benchmark program. -Although successfully compiling the benchmark program is a -sign of success, the program is not a complete iOS app. - -If you would only like to build only one architecture to save time: -(iOS 11+ only supports 64bit so you can get away with arm64) - -```bash -tensorflow/contrib/makefile/build_all_ios.sh -a arm64 -``` - -After the first build if you would like to just build the tensorflow -library you can pass the -T flag to avoid a clean & rebuild. This should -take you just a few seconds to generate the library if you modified one file. - -```bash -tensorflow/contrib/makefile/build_all_ios.sh -a arm64 -T -``` - -To see TensorFlow running on iOS, the example Xcode project in -[tensorflow/examples/ios](../../examples/ios/) shows how to use the static -library in a simple app. - -### Building by hand - -This section covers each step of building. For all the code in one place, see -[build_all_ios.sh](build_all_ios.sh). - -If you have not already, you will need to download dependencies: - -```bash -tensorflow/contrib/makefile/download_dependencies.sh -``` - -Next, you will need to compile protobufs for iOS (optionally takes the -a $ARCH flag): - -```bash -tensorflow/contrib/makefile/compile_ios_protobuf.sh -``` - -Then, you will need to compile the nsync library for iOS (optionally takes -a $ARCH flag): - -```bash -export HOST_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh` -export TARGET_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh -t ios` -``` -Then, you can run the makefile specifying iOS as the target, along with the -architecture you want to build for: - -```bash -make -f tensorflow/contrib/makefile/Makefile \ - TARGET=IOS \ - IOS_ARCH=ARM64 -``` - -This creates a library in -`tensorflow/contrib/makefile/gen/lib/libtensorflow-core.a` that you can link any -xcode project against. - -To see TensorFlow running on iOS, the example Xcode project in -[tensorflow/examples/ios](../../examples/ios/) shows how to use the static -library in a simple app. - -#### Universal binaries - -In some situations, you will need a universal library. In that case, you will -still need to run `compile_ios_protobuf.sh` and `compile_nsync.sh`, but this -time follow it with: - -```bash -compile_ios_tensorflow.sh -``` - -`compile_ios_tensorflow.sh` takes the -a flag to build only for one architecture. -In case you run into issues with unresolved symbols with nsync you can also pass --h ${HOST_NSYNC_LIB} and -n {TARGET_NSYNC_LIB} so it would look like: - -```bash -tensorflow/contrib/makefile/compile_ios_tensorflow.sh -f "-O3" -h tensorflow/contrib/makefile/downloads/nsync/builds/default.macos.c++11/nsync.a -n tensorflow/contrib/makefile/downloads/nsync/builds/lipo.ios.c++11/nsync.a -a arm64 -``` - -In XCode, you will need to use -force_load in the linker flags -section of the build settings to pull in the global constructors that are used -to register ops and kernels. - -#### Optimization - -The `build_all_ios.sh` script can take optional command-line arguments to -selectively register only for the operators used in your graph. - -```bash -tensorflow/contrib/makefile/build_all_ios.sh -a arm64 -g $HOME/graphs/inception/tensorflow_inception_graph.pb -``` -Please note this is an aggressive optimization of the operators and the resulting library may not work with other graphs but will reduce the size of the final library. - -The `compile_ios_tensorflow.sh` script can take optional command-line arguments. -The first argument will be passed as a C++ optimization flag and defaults to -debug mode. If you are concerned about performance or are working on a release -build, you would likely want a higher optimization setting, like so: - -```bash -compile_ios_tensorflow.sh -f "-Os" -``` - -For other variations of valid optimization flags, see [clang optimization levels](http://stackoverflow.com/questions/15548023/clang-optimization-levels). - -## Raspberry Pi - -Building on the Raspberry Pi is similar to a normal Linux system. First -download the dependencies, install the required packages and build protobuf: - -```bash -tensorflow/contrib/makefile/download_dependencies.sh -sudo apt-get install -y autoconf automake libtool gcc-4.8 g++-4.8 -cd tensorflow/contrib/makefile/downloads/protobuf/ -./autogen.sh -./configure -make -sudo make install -sudo ldconfig # refresh shared library cache -cd ../../../../.. -export HOST_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh` -export TARGET_NSYNC_LIB="$HOST_NSYNC_LIB" -``` - -Once that's done, you can use make to build the library and example: - -```bash -make -f tensorflow/contrib/makefile/Makefile HOST_OS=PI TARGET=PI OPTFLAGS="-Os" CXX=g++-4.8 -``` - -If you're only interested in building for Raspberry Pi's 2 and 3, you can supply -some extra optimization flags to give you code that will run faster: - -```bash -make -f tensorflow/contrib/makefile/Makefile HOST_OS=PI TARGET=PI \ - OPTFLAGS="-Os -mfpu=neon-vfpv4 -funsafe-math-optimizations -ftree-vectorize" CXX=g++-4.8 -``` - -One thing to be careful of is that the gcc version 4.9 currently installed on -Jessie by default will hit an error mentioning `__atomic_compare_exchange`. This -is why the examples above specify `CXX=g++-4.8` explicitly, and why we install -it using apt-get. If you have partially built using the default gcc 4.9, hit the -error and switch to 4.8, you need to do a -`make -f tensorflow/contrib/makefile/Makefile clean` before you build. If you -don't, the build will appear to succeed but you'll encounter [malloc(): memory corruption errors](https://github.com/tensorflow/tensorflow/issues/3442) -when you try to run any programs using the library. - -For more examples, look at the tensorflow/contrib/pi_examples folder in the -source tree, which contains code samples aimed at the Raspberry Pi. - -# Other notes - -## Supported Systems - -The Make script has been tested on Ubuntu and OS X. If you look in the Makefile -itself, you'll see it's broken up into host and target sections. If you are -cross-compiling, you should look at customizing the target settings to match -what you need for your desired system. - -## Dependency Management - -The Makefile loads in a list of dependencies stored in text files. These files -are generated from the main Bazel build by running -`tensorflow/contrib/makefile/gen_file_lists.sh`. You'll need to re-run this i -you make changes to the files that are included in the build. - -Header dependencies are not automatically tracked by the Makefile, so if you -make header changes you will need to run this command to recompile cleanly: - -```bash -make -f tensorflow/contrib/makefile/Makefile clean -``` - -### Cleaning up - -In some situations, you may want to completely clean up. The dependencies, -intermediate stages, and generated files are stored in: - -```bash -tensorflow/contrib/makefile/downloads -tensorflow/contrib/makefile/gen -``` - -Those directories can safely be removed, but you will have to start over with -`download_dependencies.sh` once you delete them. - -### Fixing Makefile Issues - -Because the main development of TensorFlow is done using Bazel, changes to the -codebase can sometimes break the makefile build process. If you find that tests -relying on this makefile are failing with a change you're involved in, here are -some trouble-shooting steps: - - - Try to reproduce the issue on your platform. If you're on Linux, running - `make -f tensorflow/contrib/makefile/Makefile` should be enough to recreate - most issues. For other platforms, see the sections earlier in this document. - - - The most common cause of breakages are files that have been added to the - Bazel build scripts, but that the makefile isn't aware of. Typical symptoms - of this include linker errors mentioning missing symbols or protobuf headers - that aren't found. To address these problems, take a look at the *.txt files - in `tensorflow/contrib/makefile`. If you have a new operator, you may need to - add it to `tf_op_files.txt`, or for a new proto to `tf_proto_files.txt`. - - - There's also a wildcard system in `Makefile` that defines what core C++ files - are included in the library. This is designed to match the equivalent rule in - `tensorflow/core/BUILD`, so if you change the wildcards there to include new - files you'll need to also update `CORE_CC_ALL_SRCS` and `CORE_CC_EXCLUDE_SRCS` - in the makefile. - - - Some of the supported platforms use clang instead of gcc as their compiler, - so if you're hitting compile errors you may need to tweak your code to be more - friendly to different compilers by avoiding gcc extensions or idioms. - -These are the most common reasons for makefile breakages, but it's also -possible you may hit something unusual, like a platform incompatibility. For -those, you'll need to see if you can reproduce the issue on that particular -platform and debug it there. You can also reach out to the broader TensorFlow -team by [filing a Github issue](https://github.com/tensorflow/tensorflow/issues) -to ask for help. diff --git a/tensorflow/contrib/makefile/build_all_android.sh b/tensorflow/contrib/makefile/build_all_android.sh deleted file mode 100755 index dc296944497..00000000000 --- a/tensorflow/contrib/makefile/build_all_android.sh +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# This is a composite script to build all for Android OS - -set -e - -usage() { - echo "Usage: NDK_ROOT= $(basename "$0") [-Es:t:Tx:a]" - echo "-E enable experimental hexnn ops" - echo "-s [sub_makefiles] sub makefiles separated by white space" - echo "-t [build_target] build target for Android makefile [default=all]" - echo "-T only build tensorflow" - echo "-x [hexagon library path] copy and hexagon libraries in the specified path" - echo "-a [architecture] Architecture of target android [default=armeabi-v7a] \ -(supported architecture list: \ -arm64-v8a armeabi armeabi-v7a mips mips64 x86 x86_64 tegra)" - exit 1 -} - -echo "********************************************************************" -echo "TensorFlow Lite is the recommended library for mobile and embedded machine learning inference." -echo "You are currently using an older version. Please switch over to TensorFlow Lite." -echo "" -echo "Link to the code: https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite" -echo "********************************************************************" -echo "" - -if [[ -z "${NDK_ROOT}" ]]; then - echo "NDK_ROOT should be set as an environment variable" 1>&2 - exit 1 -fi - -ARCH=armeabi-v7a - -while getopts "Es:t:Tx:a:" opt_name; do - case "$opt_name" in - E) ENABLE_EXPERIMENTAL_HEXNN_OPS="true";; - s) SUB_MAKEFILES="${OPTARG}";; - t) BUILD_TARGET="${OPTARG}";; - T) ONLY_MAKE_TENSORFLOW="true";; - x) HEXAGON_LIB_PATH="${OPTARG}";; - a) ARCH="${OPTARG}";; - *) usage;; - esac -done -shift $((OPTIND - 1)) - -if [ "$ARCH" == "tegra" ]; then - if [[ -z "${JETPACK}" ]]; then - export JETPACK="$HOME/JetPack_Android_3.2" - fi - if [ ! -d ${JETPACK} ]; then - echo "Can't find Jetpack at ${JETPACK}" - echo "Set JETPACK= to specify a non-default Jetpack path" - exit -1 - fi - if [ ! -d ${JETPACK}/cuda ]; then - ln -s $(ls -d ${JETPACK}/cuda-*/|sort -r|head -n1) ${JETPACK}/cuda - fi - if [ ! -d ${JETPACK}/cuda ]; then - ln -s $(ls -d ${JETPACK}/cuda-*/|sort -r|head -n1) ${JETPACK}/cuda - fi - - export BUILD_FOR_TEGRA=1 - ARCH="arm64-v8a" -fi - -# Make sure we're in the correct directory, at the root of the source tree. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -cd "${SCRIPT_DIR}"/../../../ - -source "${SCRIPT_DIR}/build_helper.subr" -JOB_COUNT="${JOB_COUNT:-$(get_job_count)}" - -HEXAGON_DOWNLOAD_PATH="tensorflow/contrib/makefile/downloads/hexagon" - -# Remove any old files first. -make -f tensorflow/contrib/makefile/Makefile cleantarget - -if [[ "${ONLY_MAKE_TENSORFLOW}" != "true" ]]; then - rm -rf tensorflow/contrib/makefile/downloads - # Pull down the required versions of the frameworks we need. - tensorflow/contrib/makefile/download_dependencies.sh - # Compile protobuf for the target Android device architectures. - CC_PREFIX="${CC_PREFIX}" NDK_ROOT="${NDK_ROOT}" \ -tensorflow/contrib/makefile/compile_android_protobuf.sh -c -a ${ARCH} -fi - -# Compile nsync for the host and the target Android device architecture. -# Don't use export var=`something` syntax; it swallows the exit status. -HOST_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh` -TARGET_NSYNC_LIB=`CC_PREFIX="${CC_PREFIX}" NDK_ROOT="${NDK_ROOT}" \ - tensorflow/contrib/makefile/compile_nsync.sh -t android -a ${ARCH}` -export HOST_NSYNC_LIB TARGET_NSYNC_LIB - -if [[ ! -z "${HEXAGON_LIB_PATH}" ]]; then - echo "Copy hexagon libraries from ${HEXAGON_LIB_PATH}" - - mkdir -p "${HEXAGON_DOWNLOAD_PATH}/libs" - cp -fv "${HEXAGON_LIB_PATH}/libhexagon_controller.so" \ -"${HEXAGON_DOWNLOAD_PATH}/libs/libhexagon_controller.so" - cp -fv "${HEXAGON_LIB_PATH}/libhexagon_nn_skel.so" \ -"${HEXAGON_DOWNLOAD_PATH}/libs/libhexagon_nn_skel.so" - - USE_HEXAGON="true" -fi - -if [[ "${USE_HEXAGON}" == "true" ]]; then - HEXAGON_PARENT_DIR=$(cd "${HEXAGON_DOWNLOAD_PATH}" >/dev/null && pwd) - HEXAGON_LIBS="${HEXAGON_PARENT_DIR}/libs" - HEXAGON_INCLUDE=$(cd "tensorflow/core/kernels/hexagon" >/dev/null && pwd) -fi - -if [[ "${ENABLE_EXPERIMENTAL_HEXNN_OPS}" == "true" ]]; then - EXTRA_MAKE_ARGS+=("ENABLE_EXPERIMENTAL_HEXNN_OPS=true") -fi - -if [[ -z "${BUILD_TARGET}" ]]; then - make -j"${JOB_COUNT}" -f tensorflow/contrib/makefile/Makefile \ - TARGET=ANDROID NDK_ROOT="${NDK_ROOT}" ANDROID_ARCH="${ARCH}" \ - CC_PREFIX="${CC_PREFIX}" \ - HOST_NSYNC_LIB="$HOST_NSYNC_LIB" TARGET_NSYNC_LIB="$TARGET_NSYNC_LIB" \ -HEXAGON_LIBS="${HEXAGON_LIBS}" HEXAGON_INCLUDE="${HEXAGON_INCLUDE}" \ -SUB_MAKEFILES="${SUB_MAKEFILES}" ${EXTRA_MAKE_ARGS[@]} -else - # BUILD_TARGET explicitly uncommented to allow multiple targets to be - # passed to make in a single build_all_android.sh invocation. - make -j"${JOB_COUNT}" -f tensorflow/contrib/makefile/Makefile \ - TARGET=ANDROID NDK_ROOT="${NDK_ROOT}" ANDROID_ARCH="${ARCH}" \ - CC_PREFIX="${CC_PREFIX}" \ - HOST_NSYNC_LIB="$HOST_NSYNC_LIB" TARGET_NSYNC_LIB="$TARGET_NSYNC_LIB" \ -HEXAGON_LIBS="${HEXAGON_LIBS}" HEXAGON_INCLUDE="${HEXAGON_INCLUDE}" \ -SUB_MAKEFILES="${SUB_MAKEFILES}" ${EXTRA_MAKE_ARGS[@]} ${BUILD_TARGET} -fi diff --git a/tensorflow/contrib/makefile/build_all_ios.sh b/tensorflow/contrib/makefile/build_all_ios.sh deleted file mode 100755 index 9a8059ce500..00000000000 --- a/tensorflow/contrib/makefile/build_all_ios.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -set -e - -# Make sure we're on OS X. -if [[ $(uname) != "Darwin" ]]; then - echo "ERROR: This makefile build requires macOS, which the current system "\ - "is not." - exit 1 -fi - -usage() { - echo "Usage: $(basename "$0") [-a:T]" - echo "-a [build_arch] build only for specified arch x86_64 [default=all]" - echo "-g [graph] optimize and selectively register ops only for this graph" - echo "-T only build tensorflow (dont download other deps etc)" - exit 1 -} - -echo "********************************************************************" -echo "TensorFlow Lite is the recommended library for mobile and embedded machine learning inference." -echo "You are currently using an older version. Please switch over to TensorFlow Lite." -echo "" -echo "Link to the code: https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite" -echo "********************************************************************" -echo "" - -while getopts "a:g:T" opt_name; do - case "$opt_name" in - a) BUILD_ARCH="${OPTARG}";; - g) OPTIMIZE_FOR_GRAPH="${OPTARG}";; - T) ONLY_MAKE_TENSORFLOW="true";; - *) usage;; - esac -done -shift $((OPTIND - 1)) - - -# Make sure we're in the correct directory, at the root of the source tree. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -TOP_SRCDIR="${SCRIPT_DIR}/../../../" -cd ${TOP_SRCDIR} - -source "${SCRIPT_DIR}/build_helper.subr" -JOB_COUNT="${JOB_COUNT:-$(get_job_count)}" - -# Setting a deployment target is required for building with bitcode, -# otherwise linking will fail with: -# -# ld: -bind_at_load and -bitcode_bundle (Xcode setting ENABLE_BITCODE=YES) cannot be used together -# -if [[ -n MACOSX_DEPLOYMENT_TARGET ]]; then - export MACOSX_DEPLOYMENT_TARGET=$(sw_vers -productVersion) -fi - -PRNT_SLCTV_BIN="${TOP_SRCDIR}bazel-bin/tensorflow/python/tools/print_selective_registration_header" - -if [[ ! -z "${OPTIMIZE_FOR_GRAPH}" ]]; then - echo "Request to optimize for graph: ${OPTIMIZE_FOR_GRAPH}" - #Request to trim the OPs by selectively registering - if [ ! -f ${PRNT_SLCTV_BIN} ]; then - #Build bazel build tensorflow/python/tools:print_selective_registration_header - echo "${PRNT_SLCTV_BIN} not found. Trying to build it" - cd ${TOP_SRCDIR} - bazel build --copt="-DUSE_GEMM_FOR_CONV" tensorflow/python/tools:print_selective_registration_header - if [ ! -f ${PRNT_SLCTV_BIN} ]; then - echo "Building print_selective_registration_header failed" - echo "You may want to build TensorFlow with: " - echo "./configure" - echo "bazel build --copt="-DUSE_GEMM_FOR_CONV" tensorflow/python/tools:print_selective_registration_header" - echo "and then run this script again" - exit 1 - fi - else - echo "${PRNT_SLCTV_BIN} found. Using it" - fi - - ${PRNT_SLCTV_BIN} --graphs=${OPTIMIZE_FOR_GRAPH} > ${TOP_SRCDIR}/tensorflow/core/framework/ops_to_register.h -fi - -if [[ "${ONLY_MAKE_TENSORFLOW}" != "true" ]]; then - # Remove any old files first. - make -f tensorflow/contrib/makefile/Makefile clean - rm -rf tensorflow/contrib/makefile/downloads - - # Pull down the required versions of the frameworks we need. - tensorflow/contrib/makefile/download_dependencies.sh - - if [[ -z "${BUILD_ARCH}" ]]; then - # Compile protobuf for the target iOS device architectures. - tensorflow/contrib/makefile/compile_ios_protobuf.sh - else - # Compile protobuf for the target iOS device architectures. - tensorflow/contrib/makefile/compile_ios_protobuf.sh -a ${BUILD_ARCH} - fi -fi - -# Compile nsync for the target iOS device architectures. -# Don't use export var=`something` syntax; it swallows the exit status. -HOST_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh` -if [[ -z "${BUILD_ARCH}" ]]; then - # No arch specified so build all architectures - TARGET_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh -t ios` -else - # arch specified so build just that - TARGET_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh -t ios -a "${BUILD_ARCH}"` -fi -export HOST_NSYNC_LIB TARGET_NSYNC_LIB - -TF_CC_FLAGS="-O3" -TF_SCRIPT_FLAGS="-h ${HOST_NSYNC_LIB} -n ${TARGET_NSYNC_LIB}" - -if [[ ! -z "${OPTIMIZE_FOR_GRAPH}" ]]; then - # arch specified so build just that - TF_CC_FLAGS="${TF_CC_FLAGS} -DANDROID_TYPES=__ANDROID_TYPES_FULL__ -DSELECTIVE_REGISTRATION -DSUPPORT_SELECTIVE_REGISTRATION" - # The Makefile checks the env var to decide which ANDROID_TYPES to build - export ANDROID_TYPES="-D__ANDROID_TYPES_FULL__" -fi - -if [[ ! -z "${BUILD_ARCH}" ]]; then - # arch specified so build just that - TF_SCRIPT_FLAGS="${TF_SCRIPT_FLAGS} -a ${BUILD_ARCH}" -fi - -# build the ios tensorflow libraries. -echo "Building TensorFlow with command: ${TF_SCRIPT_FLAGS} -f ${TF_CC_FLAGS}" -tensorflow/contrib/makefile/compile_ios_tensorflow.sh ${TF_SCRIPT_FLAGS} -f "${TF_CC_FLAGS}" - -# Creates a static universal library in -# tensorflow/contrib/makefile/gen/lib/libtensorflow-core.a diff --git a/tensorflow/contrib/makefile/build_all_ios_with_tflite.sh b/tensorflow/contrib/makefile/build_all_ios_with_tflite.sh deleted file mode 100755 index 8d34911f154..00000000000 --- a/tensorflow/contrib/makefile/build_all_ios_with_tflite.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# This shell script is used to build TensorFlow Lite Flex runtime for iOS. -# It compiles TensorFlow Lite and TensorFlow codebases together, and enable a -# route to use TensorFlow kernels in TensorFlow Lite. -# -# After the script is executed, the multi-architecture static libraries will be -# created under: `tensorflow/contrib/makefile/gen/lib/`. - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -TOP_SRCDIR="${SCRIPT_DIR}/../../../" -cd ${TOP_SRCDIR} - -# Exporting `WITH_TFLITE_FLEX`. The flag will be propagated all the way -# down to Makefile. -export WITH_TFLITE_FLEX="true" -# Execute `build_all_ios.sh` and propagate all parameters. -tensorflow/contrib/makefile/build_all_ios.sh $* - -# Copy all the libraries required for TFLite Flex runtime together. -cd "${TOP_SRCDIR}/tensorflow/contrib/makefile" -cp 'downloads/nsync/builds/lipo.ios.c++11/nsync.a' 'gen/lib/' -cp 'gen/protobuf_ios/lib/libprotobuf.a' 'gen/lib/' -cp 'gen/lib/libtensorflow-core.a' 'gen/lib/libtensorflow-lite.a' diff --git a/tensorflow/contrib/makefile/build_all_linux.sh b/tensorflow/contrib/makefile/build_all_linux.sh deleted file mode 100755 index a440633cfc2..00000000000 --- a/tensorflow/contrib/makefile/build_all_linux.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# Downloads and builds all of TensorFlow's dependencies for Linux, and compiles -# the TensorFlow library itself. Supported on Ubuntu 14.04 and 16.04. - -set -e - -# Make sure we're in the correct directory, at the root of the source tree. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd ${SCRIPT_DIR}/../../../ - -source "${SCRIPT_DIR}/build_helper.subr" -JOB_COUNT="${JOB_COUNT:-$(get_job_count)}" - -# Remove any old files first. -make -f tensorflow/contrib/makefile/Makefile clean -rm -rf tensorflow/contrib/makefile/downloads - -# Pull down the required versions of the frameworks we need. -tensorflow/contrib/makefile/download_dependencies.sh - -# Compile nsync. -# Don't use export var=`something` syntax; it swallows the exit status. -HOST_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh` -TARGET_NSYNC_LIB="$HOST_NSYNC_LIB" -export HOST_NSYNC_LIB TARGET_NSYNC_LIB - -# Compile protobuf. -tensorflow/contrib/makefile/compile_linux_protobuf.sh - -# Build TensorFlow. -make -j"${JOB_COUNT}" -f tensorflow/contrib/makefile/Makefile \ - OPTFLAGS="-O3 -march=native" \ - HOST_CXXFLAGS="--std=c++11 -march=native" \ - MAKEFILE_DIR=$SCRIPT_DIR diff --git a/tensorflow/contrib/makefile/build_helper.subr b/tensorflow/contrib/makefile/build_helper.subr deleted file mode 100644 index d58b2c0a9be..00000000000 --- a/tensorflow/contrib/makefile/build_helper.subr +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -e -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# Utility functions for scripts to build tensorflow by Makefile - -get_cpu_count() { - case "${OSTYPE}" in - linux*) - grep processor /proc/cpuinfo | wc -l ;; - darwin*) - sysctl hw.ncpu | awk '{print $2}' ;; - cygwin*) - grep processor /proc/cpuinfo | wc -l ;; - *) - echo "1" - exit 1 ;; - esac - exit 0 -} - -get_job_count() { - echo $(($(get_cpu_count))) -} - -make_host_protoc() { - if [[ ! $1 ]]; then - echo "needs 1 argument (HOST_GENDIR)" - exit 1 - fi - HOST_GENDIR="$1" - ./autogen.sh - if [ $? -ne 0 ] - then - echo "./autogen.sh command failed." - exit 1 - fi - rm -rf "${HOST_GENDIR}" - mkdir -p "${HOST_GENDIR}" - ./configure --disable-shared --prefix="${HOST_GENDIR}" - make clean - make -j"$(get_job_count)" - if [ $? -ne 0 ]; then - echo "make failed" - exit 1 - fi - make install -} - -download_and_push() { - URL="$1" - LOCAL_DEST="$2" - ANDROID_DEST="$3" - SKIP_DOWNLOAD_IF_EXIST="$4" - if [[ "${SKIP_DOWNLOAD_IF_EXIST}" == "true" ]]; then - ANDROID_DEST_FILE_PATH="${ANDROID_DEST}/$(basename "${LOCAL_DEST}")" - if adb shell test -f "${ANDROID_DEST_FILE_PATH}"; then - echo "${ANDROID_DEST_FILE_PATH} already existins, skip download" 1>&2 - return 0 - fi - fi - - curl -Ls "${URL}" -o "${LOCAL_DEST}" - - if [[ ! -z "${ANDROID_DEST}" ]]; then - adb shell mkdir -p "${ANDROID_DEST}" - adb push "${LOCAL_DEST}" "${ANDROID_DEST}" - fi -} diff --git a/tensorflow/contrib/makefile/build_with_docker.sh b/tensorflow/contrib/makefile/build_with_docker.sh deleted file mode 100755 index 4a17576810c..00000000000 --- a/tensorflow/contrib/makefile/build_with_docker.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# -# Usage: build_with_docker.sh -# -# Will build a docker container, and run the makefile build inside that -# container. -# - -# Make sure we're in the correct directory, at the root of the source tree. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -WORKSPACE="${SCRIPT_DIR}/../../../" -cd ${WORKSPACE} || exit 1 - -COMMAND="tensorflow/contrib/makefile/build_all_linux.sh" - -echo "Running ${COMMAND} inside Android docker image..." -tensorflow/tools/ci_build/ci_build.sh android ${COMMAND} diff --git a/tensorflow/contrib/makefile/compile_android_protobuf.sh b/tensorflow/contrib/makefile/compile_android_protobuf.sh deleted file mode 100755 index 4355e3e5974..00000000000 --- a/tensorflow/contrib/makefile/compile_android_protobuf.sh +++ /dev/null @@ -1,190 +0,0 @@ -#!/bin/bash -e -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# Builds protobuf 3 for Android. Pass in the location of your NDK as the first -# argument to the script, for example: -# tensorflow/contrib/makefile/compile_android_protobuf.sh \ -# ${HOME}/toolchains/clang-21-stl-gnu - -# Pass ANDROID_API_VERSION as an environment variable to support -# a different version of API. -android_api_version="${ANDROID_API_VERSION:-21}" -# Pass cc prefix to set the prefix for cc (e.g. ccache) -cc_prefix="${CC_PREFIX}" - -usage() { - echo "Usage: $(basename "$0") [-a:c]" - echo "-a [Architecture] Architecture of target android [default=armeabi-v7a] \ -(supported architecture list: \ -arm64-v8a armeabi armeabi-v7a armeabi-v7a-hard mips mips64 x86 x86_64)" - echo "-c Clean before building protobuf for target" - echo "\"NDK_ROOT\" should be defined as an environment variable." - exit 1 -} - -SCRIPT_DIR=$(dirname $0) -ARCHITECTURE=armeabi-v7a - -# debug options -while getopts "a:c" opt_name; do - case "$opt_name" in - a) ARCHITECTURE=$OPTARG;; - c) clean=true;; - *) usage;; - esac -done -shift $((OPTIND - 1)) - -source "${SCRIPT_DIR}/build_helper.subr" -JOB_COUNT="${JOB_COUNT:-$(get_job_count)}" - -if [[ -z "${NDK_ROOT}" ]] -then - echo "You need to pass in the Android NDK location as the environment \ -variable" - echo "e.g. NDK_ROOT=${HOME}/android_ndk/android-ndk-rXXx \ -tensorflow/contrib/makefile/compile_android_protobuf.sh" - exit 1 -fi - -if [[ ! -f "${SCRIPT_DIR}/Makefile" ]]; then - echo "Makefile not found in ${SCRIPT_DIR}" 1>&2 - exit 1 -fi - -cd "${SCRIPT_DIR}" -if [ $? -ne 0 ] -then - echo "cd to ${SCRIPT_DIR} failed." 1>&2 - exit 1 -fi - -GENDIR="$(pwd)/gen/protobuf_android" -HOST_GENDIR="$(pwd)/gen/protobuf-host" -mkdir -p "${GENDIR}" -mkdir -p "${GENDIR}/${ARCHITECTURE}" - -if [[ ! -f "./downloads/protobuf/autogen.sh" ]]; then - echo "You need to download dependencies before running this script." 1>&2 - echo "tensorflow/contrib/makefile/download_dependencies.sh" 1>&2 - exit 1 -fi - -cd downloads/protobuf - -PROTOC_PATH="${HOST_GENDIR}/bin/protoc" -if [[ ! -f "${PROTOC_PATH}" || ${clean} == true ]]; then - # Try building compatible protoc first on host - echo "protoc not found at ${PROTOC_PATH}. Build it first." - make_host_protoc "${HOST_GENDIR}" -else - echo "protoc found. Skip building host tools." -fi - - -echo $OSTYPE | grep -q "darwin" && os_type="darwin" || os_type="linux" -if [[ ${ARCHITECTURE} == "arm64-v8a" ]]; then - toolchain="aarch64-linux-android-4.9" - sysroot_arch="arm64" - bin_prefix="aarch64-linux-android" -elif [[ ${ARCHITECTURE} == "armeabi" ]]; then - toolchain="arm-linux-androideabi-4.9" - sysroot_arch="arm" - bin_prefix="arm-linux-androideabi" -elif [[ ${ARCHITECTURE} == "armeabi-v7a" ]]; then - toolchain="arm-linux-androideabi-4.9" - sysroot_arch="arm" - bin_prefix="arm-linux-androideabi" - march_option="-march=armv7-a" -elif [[ ${ARCHITECTURE} == "armeabi-v7a-hard" ]]; then - toolchain="arm-linux-androideabi-4.9" - sysroot_arch="arm" - bin_prefix="arm-linux-androideabi" - march_option="-march=armv7-a" -elif [[ ${ARCHITECTURE} == "mips" ]]; then - toolchain="mipsel-linux-android-4.9" - sysroot_arch="mips" - bin_prefix="mipsel-linux-android" -elif [[ ${ARCHITECTURE} == "mips64" ]]; then - toolchain="mips64el-linux-android-4.9" - sysroot_arch="mips64" - bin_prefix="mips64el-linux-android" -elif [[ ${ARCHITECTURE} == "x86" ]]; then - toolchain="x86-4.9" - sysroot_arch="x86" - bin_prefix="i686-linux-android" -elif [[ ${ARCHITECTURE} == "x86_64" ]]; then - toolchain="x86_64-4.9" - sysroot_arch="x86_64" - bin_prefix="x86_64-linux-android" -else - echo "architecture ${ARCHITECTURE} is not supported." 1>&2 - usage - exit 1 -fi - -echo "Android api version = ${android_api_version} cc_prefix = ${cc_prefix}" - -export PATH=\ -"${NDK_ROOT}/toolchains/${toolchain}/prebuilt/${os_type}-x86_64/bin:$PATH" -export SYSROOT=\ -"${NDK_ROOT}/platforms/android-${android_api_version}/arch-${sysroot_arch}" -export CC="${cc_prefix} ${bin_prefix}-gcc --sysroot ${SYSROOT}" -export CXX="${cc_prefix} ${bin_prefix}-g++ --sysroot ${SYSROOT}" -export CXXSTL=\ -"${NDK_ROOT}/sources/cxx-stl/gnu-libstdc++/4.9/libs/${ARCHITECTURE}" - -./autogen.sh -if [ $? -ne 0 ] -then - echo "./autogen.sh command failed." - exit 1 -fi - -./configure --prefix="${GENDIR}/${ARCHITECTURE}" \ ---host="${bin_prefix}" \ ---with-sysroot="${SYSROOT}" \ ---disable-shared \ ---enable-cross-compile \ ---with-protoc="${PROTOC_PATH}" \ -CFLAGS="${march_option}" \ -CXXFLAGS="-frtti -fexceptions ${march_option} \ --I${NDK_ROOT}/sources/android/support/include \ --I${NDK_ROOT}/sources/cxx-stl/gnu-libstdc++/4.9/include \ --I${NDK_ROOT}/sources/cxx-stl/gnu-libstdc++/4.9/libs/${ARCHITECTURE}/include" \ -LDFLAGS="-L${NDK_ROOT}/sources/cxx-stl/gnu-libstdc++/4.9/libs/${ARCHITECTURE}" \ -LIBS="-llog -lz -lgnustl_static" - -if [ $? -ne 0 ] -then - echo "./configure command failed." - exit 1 -fi - -if [[ ${clean} == true ]]; then - echo "clean before build" - make clean -fi - -make -j"${JOB_COUNT}" -if [ $? -ne 0 ] -then - echo "make command failed." - exit 1 -fi - -make install - -echo "$(basename $0) finished successfully!!!" diff --git a/tensorflow/contrib/makefile/compile_ios_protobuf.sh b/tensorflow/contrib/makefile/compile_ios_protobuf.sh deleted file mode 100755 index d2fbf696f8f..00000000000 --- a/tensorflow/contrib/makefile/compile_ios_protobuf.sh +++ /dev/null @@ -1,251 +0,0 @@ -#!/bin/bash -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# Builds protobuf 3 for iOS. - -set -e - -if [[ -n MACOSX_DEPLOYMENT_TARGET ]]; then - export MACOSX_DEPLOYMENT_TARGET=$(sw_vers -productVersion) -fi - -usage() { - echo "Usage: $(basename "$0") [-a]" - echo "-a [build_arch] build for specified arch comma separate for multiple archs (eg: x86_64,arm64)" - echo "default arch x86_64, armv7, armv7s, arm64" - exit 1 -} - -BUILD_TARGET="x86_64 armv7 armv7s arm64" -while getopts "a:" opt_name; do - case "$opt_name" in - a) BUILD_TARGET="${OPTARG}";; - *) usage;; - esac -done -shift $((OPTIND - 1)) - -IFS=' ' read -r -a build_targets <<< "${BUILD_TARGET}" - -SCRIPT_DIR=$(cd `dirname $0` && pwd) -source "${SCRIPT_DIR}/build_helper.subr" - -cd ${SCRIPT_DIR} - -HOST_GENDIR="$(pwd)/gen/protobuf-host" -mkdir -p "${HOST_GENDIR}" -if [[ ! -f "./downloads/protobuf/autogen.sh" ]]; then - echo "You need to download dependencies before running this script." 1>&2 - echo "tensorflow/contrib/makefile/download_dependencies.sh" 1>&2 - exit 1 -fi - -JOB_COUNT="${JOB_COUNT:-$(get_job_count)}" - -GENDIR=$(pwd)/gen/protobuf_ios/ -LIBDIR=${GENDIR}lib -mkdir -p ${LIBDIR} - -OSX_VERSION=darwin14.0.0 - -IPHONEOS_PLATFORM=$(xcrun --sdk iphoneos --show-sdk-platform-path) -IPHONEOS_SYSROOT=$(xcrun --sdk iphoneos --show-sdk-path) -IPHONESIMULATOR_PLATFORM=$(xcrun --sdk iphonesimulator --show-sdk-platform-path) -IPHONESIMULATOR_SYSROOT=$(xcrun --sdk iphonesimulator --show-sdk-path) -IOS_SDK_VERSION=$(xcrun --sdk iphoneos --show-sdk-version) -MIN_SDK_VERSION=8.0 - -CFLAGS="-DNDEBUG -Os -pipe -fPIC -fno-exceptions" -CXXFLAGS="${CFLAGS} -std=c++11 -stdlib=libc++" -LDFLAGS="-stdlib=libc++" -LIBS="-lc++ -lc++abi" - -cd downloads/protobuf -PROTOC_PATH="${HOST_GENDIR}/bin/protoc" -if [[ ! -f "${PROTOC_PATH}" || ${clean} == true ]]; then - # Try building compatible protoc first on host - echo "protoc not found at ${PROTOC_PATH}. Build it first." - make_host_protoc "${HOST_GENDIR}" -else - echo "protoc found. Skip building host tools." -fi - -# Remove old libs -rm -f ${LIBDIR}/libprotobuf.a -rm -f ${LIBDIR}/libprotobuf-lite.a - -./autogen.sh -if [ $? -ne 0 ] -then - echo "./autogen.sh command failed." - exit 1 -fi - -package_pb_library() { - pb_libs="${LIBDIR}/${1}/lib/libprotobuf.a" - if [ -f "${LIBDIR}/libprotobuf.a" ]; then - pb_libs="$pb_libs ${LIBDIR}/libprotobuf.a" - fi - lipo \ - $pb_libs \ - -create \ - -output ${LIBDIR}/libprotobuf.a - - pblite_libs="${LIBDIR}/${1}/lib/libprotobuf-lite.a" - if [ -f "${LIBDIR}/libprotobuf-lite.a" ]; then - pblite_libs="$pblite_libs ${LIBDIR}/libprotobuf-lite.a" - fi - lipo \ - $pblite_libs \ - -create \ - -output ${LIBDIR}/libprotobuf-lite.a -} - -build_target() { -case "$1" in - x86_64) make distclean - ./configure \ - --host=x86_64-apple-${OSX_VERSION} \ - --disable-shared \ - --enable-cross-compile \ - --with-protoc="${PROTOC_PATH}" \ - --prefix=${LIBDIR}/iossim_x86_64 \ - --exec-prefix=${LIBDIR}/iossim_x86_64 \ - "CFLAGS=${CFLAGS} \ - -mios-simulator-version-min=${MIN_SDK_VERSION} \ - -arch x86_64 \ - -fembed-bitcode \ - -isysroot ${IPHONESIMULATOR_SYSROOT}" \ - "CXX=${CXX}" \ - "CXXFLAGS=${CXXFLAGS} \ - -mios-simulator-version-min=${MIN_SDK_VERSION} \ - -arch x86_64 \ - -fembed-bitcode \ - -isysroot \ - ${IPHONESIMULATOR_SYSROOT}" \ - LDFLAGS="-arch x86_64 \ - -fembed-bitcode \ - -mios-simulator-version-min=${MIN_SDK_VERSION} \ - ${LDFLAGS} \ - -L${IPHONESIMULATOR_SYSROOT}/usr/lib/ \ - -L${IPHONESIMULATOR_SYSROOT}/usr/lib/system" \ - "LIBS=${LIBS}" - make -j"${JOB_COUNT}" - make install - - package_pb_library "iossim_x86_64" - ;; - - armv7) make distclean - ./configure \ - --host=armv7-apple-${OSX_VERSION} \ - --with-protoc="${PROTOC_PATH}" \ - --disable-shared \ - --prefix=${LIBDIR}/ios_arm7 \ - --exec-prefix=${LIBDIR}/ios_arm7 \ - "CFLAGS=${CFLAGS} \ - -miphoneos-version-min=${MIN_SDK_VERSION} \ - -arch armv7 \ - -fembed-bitcode \ - -isysroot ${IPHONEOS_SYSROOT}" \ - "CXX=${CXX}" \ - "CXXFLAGS=${CXXFLAGS} \ - -miphoneos-version-min=${MIN_SDK_VERSION} \ - -arch armv7 \ - -fembed-bitcode \ - -isysroot ${IPHONEOS_SYSROOT}" \ - LDFLAGS="-arch armv7 \ - -fembed-bitcode \ - -miphoneos-version-min=${MIN_SDK_VERSION} \ - ${LDFLAGS}" \ - "LIBS=${LIBS}" - make -j"${JOB_COUNT}" - make install - - package_pb_library "ios_arm7" - ;; - - armv7s) make distclean - ./configure \ - --host=armv7s-apple-${OSX_VERSION} \ - --with-protoc="${PROTOC_PATH}" \ - --disable-shared \ - --prefix=${LIBDIR}/ios_arm7s \ - --exec-prefix=${LIBDIR}/ios_arm7s \ - "CFLAGS=${CFLAGS} \ - -miphoneos-version-min=${MIN_SDK_VERSION} \ - -arch armv7s \ - -fembed-bitcode \ - -isysroot ${IPHONEOS_SYSROOT}" \ - "CXX=${CXX}" \ - "CXXFLAGS=${CXXFLAGS} \ - -miphoneos-version-min=${MIN_SDK_VERSION} \ - -arch armv7s \ - -fembed-bitcode \ - -isysroot ${IPHONEOS_SYSROOT}" \ - LDFLAGS="-arch armv7s \ - -fembed-bitcode \ - -miphoneos-version-min=${MIN_SDK_VERSION} \ - ${LDFLAGS}" \ - "LIBS=${LIBS}" - make -j"${JOB_COUNT}" - make install - - package_pb_library "ios_arm7s" - ;; - - arm64) make distclean - ./configure \ - --host=arm \ - --with-protoc="${PROTOC_PATH}" \ - --disable-shared \ - --prefix=${LIBDIR}/ios_arm64 \ - --exec-prefix=${LIBDIR}/ios_arm64 \ - "CFLAGS=${CFLAGS} \ - -miphoneos-version-min=${MIN_SDK_VERSION} \ - -arch arm64 \ - -fembed-bitcode \ - -isysroot ${IPHONEOS_SYSROOT}" \ - "CXXFLAGS=${CXXFLAGS} \ - -miphoneos-version-min=${MIN_SDK_VERSION} \ - -arch arm64 \ - -fembed-bitcode \ - -isysroot ${IPHONEOS_SYSROOT}" \ - LDFLAGS="-arch arm64 \ - -fembed-bitcode \ - -miphoneos-version-min=${MIN_SDK_VERSION} \ - ${LDFLAGS}" \ - "LIBS=${LIBS}" - make -j"${JOB_COUNT}" - make install - - package_pb_library "ios_arm64" - ;; - *) - echo "Unknown ARCH" - exit 1 - ;; -esac -} - -for build_element in "${build_targets[@]}" -do - echo "$build_element" - build_target "$build_element" -done - -file ${LIBDIR}/libprotobuf.a -file ${LIBDIR}/libprotobuf-lite.a -echo "Done building and packaging the libraries" diff --git a/tensorflow/contrib/makefile/compile_ios_tensorflow.sh b/tensorflow/contrib/makefile/compile_ios_tensorflow.sh deleted file mode 100755 index 3822f0d7da7..00000000000 --- a/tensorflow/contrib/makefile/compile_ios_tensorflow.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash -x -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# Builds the TensorFlow core library with ARM and x86 architectures for iOS, and -# packs them into a fat file. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/build_helper.subr" -JOB_COUNT="${JOB_COUNT:-$(get_job_count)}" - -function less_than_required_version() { - echo $1 | (IFS=. read -r major minor micro - if [ $major -ne $2 ]; then - [ $major -lt $2 ] - elif [ $minor -ne $3 ]; then - [ $minor -lt $3 ] - else - [ ${micro:-0} -lt $4 ] - fi - ) -} - -if [[ -n MACOSX_DEPLOYMENT_TARGET ]]; then - export MACOSX_DEPLOYMENT_TARGET=$(sw_vers -productVersion) -fi - -ACTUAL_XCODE_VERSION=$(xcodebuild -version | head -n 1 | sed 's/Xcode //') -REQUIRED_XCODE_VERSION=7.3.0 -if less_than_required_version $ACTUAL_XCODE_VERSION 7 3 0 -then - echo "error: Xcode ${REQUIRED_XCODE_VERSION} or later is required." - exit 1 -fi - -usage() { - echo "Usage: $(basename "$0") [-a]" - echo "-a [build_arch] build for specified arch comma separate for multiple archs (eg: x86_64,arm64)" - echo "default is [x86_64, armv7, armv7s, arm64]" - exit 1 -} - -BUILD_TARGET="x86_64 armv7 armv7s arm64" -while getopts "a:f:h:n:" opt_name; do - case "$opt_name" in - a) BUILD_TARGET="${OPTARG}";; - f) BUILD_OPT="${OPTARG}";; - h) NSYNC_HOST="${OPTARG}";; - n) NSYNC_TARGET="${OPTARG}";; - *) usage;; - esac -done -shift $((OPTIND - 1)) - -IFS=' ' read -r -a build_targets <<< "${BUILD_TARGET}" - -SCRIPT_DIR=$(cd `dirname $0` && pwd) -source "${SCRIPT_DIR}/build_helper.subr" - - -GENDIR=tensorflow/contrib/makefile/gen/ -LIBDIR=${GENDIR}lib -LIB_PREFIX=libtensorflow-core - -#remove any old artifacts -rm -rf ${LIBDIR}/${LIB_PREFIX}.a - -package_tf_library() { - CAP_DIR=`echo $1 | tr 'a-z' 'A-Z'` - tf_libs="${LIBDIR}/ios_${CAP_DIR}/${LIB_PREFIX}-${1}.a" - if [ -f "${LIBDIR}/${LIB_PREFIX}.a" ]; then - tf_libs="$tf_libs ${LIBDIR}/${LIB_PREFIX}.a" - fi - lipo \ - $tf_libs \ - -create \ - -output ${LIBDIR}/${LIB_PREFIX}.a -} - -build_tf_target() { -case "$1" in - armv7) - make -j"${JOB_COUNT}" -f tensorflow/contrib/makefile/Makefile \ - TARGET=IOS IOS_ARCH=ARMV7 LIB_NAME=${LIB_PREFIX}-armv7.a \ - OPTFLAGS="${BUILD_OPT}" HOST_NSYNC_LIB="${NSYNC_HOST}" \ - TARGET_NSYNC_LIB="${NSYNC_TARGET}" - if [ $? -ne 0 ] - then - echo "armv7 compilation failed." - exit 1 - fi - package_tf_library "armv7" - ;; - armv7s) - make -j"${JOB_COUNT}" -f tensorflow/contrib/makefile/Makefile \ - TARGET=IOS IOS_ARCH=ARMV7S LIB_NAME=${LIB_PREFIX}-armv7s.a \ - OPTFLAGS="${BUILD_OPT}" HOST_NSYNC_LIB="${NSYNC_HOST}" \ - TARGET_NSYNC_LIB="${NSYNC_TARGET}" - - if [ $? -ne 0 ] - then - echo "arm7vs compilation failed." - exit 1 - fi - package_tf_library "armv7s" - ;; - arm64) - make -j"${JOB_COUNT}" -f tensorflow/contrib/makefile/Makefile \ - TARGET=IOS IOS_ARCH=ARM64 LIB_NAME=${LIB_PREFIX}-arm64.a \ - OPTFLAGS="${BUILD_OPT}" HOST_NSYNC_LIB="${NSYNC_HOST}" \ - TARGET_NSYNC_LIB="${NSYNC_TARGET}" - if [ $? -ne 0 ] - then - echo "arm64 compilation failed." - exit 1 - fi - package_tf_library "arm64" - ;; - x86_64) - make -j"${JOB_COUNT}" -f tensorflow/contrib/makefile/Makefile \ - TARGET=IOS IOS_ARCH=X86_64 LIB_NAME=${LIB_PREFIX}-x86_64.a \ - OPTFLAGS="${BUILD_OPT}" HOST_NSYNC_LIB="${NSYNC_HOST}" \ - TARGET_NSYNC_LIB="${NSYNC_TARGET}" - if [ $? -ne 0 ] - then - echo "x86_64 compilation failed." - exit 1 - fi - package_tf_library "x86_64" - ;; - *) - echo "Unknown ARCH" - exit 1 -esac -} - -for build_tf_element in "${build_targets[@]}" -do - echo "$build_tf_element" - build_tf_target "$build_tf_element" -done - -echo "Done building and packaging TF" -file ${LIBDIR}/${LIB_PREFIX}.a diff --git a/tensorflow/contrib/makefile/compile_linux_protobuf.sh b/tensorflow/contrib/makefile/compile_linux_protobuf.sh deleted file mode 100755 index 6eb061a3c96..00000000000 --- a/tensorflow/contrib/makefile/compile_linux_protobuf.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -e -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# Builds protobuf 3 for Linux inside the local build tree. - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GENDIR="$(pwd)/tensorflow/contrib/makefile/gen/protobuf" -HOST_GENDIR="$(pwd)/tensorflow/contrib/makefile/gen/protobuf-host" -mkdir -p "${GENDIR}" -ln -s "${GENDIR}" "${HOST_GENDIR}" - -if [[ ! -f "tensorflow/contrib/makefile/downloads/protobuf/autogen.sh" ]]; then - echo "You need to download dependencies before running this script." 1>&2 - echo "tensorflow/contrib/makefile/download_dependencies.sh" 1>&2 - exit 1 -fi -source "${SCRIPT_DIR}"/build_helper.subr -JOB_COUNT="${JOB_COUNT:-$(get_job_count)}" - -cd tensorflow/contrib/makefile/downloads/protobuf - -./autogen.sh -if [ $? -ne 0 ] -then - echo "./autogen.sh command failed." - exit 1 -fi - -./configure --prefix="${GENDIR}" --with-pic -if [ $? -ne 0 ] -then - echo "./configure command failed." - exit 1 -fi - -make clean - -make -j"${JOB_COUNT}" -if [ $? -ne 0 ] -then - echo "make command failed." - exit 1 -fi - -make install - -echo "$(basename $0) finished successfully!!!" diff --git a/tensorflow/contrib/makefile/compile_nsync.sh b/tensorflow/contrib/makefile/compile_nsync.sh deleted file mode 100755 index e154b8223c6..00000000000 --- a/tensorflow/contrib/makefile/compile_nsync.sh +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# ============================================================================== - -# Compile the nsync library for the platforms given as arguments. - -set -e - -prog=compile_nsync.sh -android_api_version=21 -default_android_arch=armeabi-v7a -default_ios_arch="x86_64 armv7 armv7s arm64" - -usage="usage: $prog [-t linux|ios|android|macos|native] - [-a architecture] [-v android_api_version] - -A script to build nsync for tensorflow. -This script can be run on Linux or MacOS host platforms, and can target -Linux, MacOS, iOS, or Android. - -Options: --t target_platform -The default target platform is the native host platform. - --a architecture -For Android and iOS target platforms, specify which architecture -to target. -For iOS, the default is: $default_ios_arch. -For Android, the default is: $default_android_arch. - --v android_api_version -Specify the Android API version; the default is $android_api_version." - -# Deduce host platform. -host_platform= -nsync_path= -case `uname -s` in -Linux) host_platform=linux android_host=linux;; -Darwin) host_platform=macos android_host=darwin;; -*) echo "$prog: can't deduce host platform" >&2; exit 2;; -esac -host_arch=`uname -m` -case "$host_arch" in i[345678]86) host_arch=x86_32;; esac - -# Parse command line. -target_platform=native # Default is to build for the host. -target_arch=default -while - arg="${1-}" - case "$arg" in - -*) case "$arg" in -*t*) target_platform="${2?"$usage"}"; shift; esac - case "$arg" in -*a*) target_arch="${2?"$usage"}"; shift; esac - case "$arg" in -*v*) android_api_version="${2?"$usage"}"; shift; esac - case "$arg" in -*[!atv]*) echo "$usage" >&2; exit 2;; esac;; - "") break;; - *) echo "$usage" >&2; exit 2;; - esac -do - shift -done - -# Sanity check the target platform. -case "$target_platform" in -native) target_platform="$host_platform";; -esac - -# Change directory to the root of the source tree. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "${SCRIPT_DIR}/../../.." - -nsync_builds_dir=tensorflow/contrib/makefile/downloads/nsync/builds - -case "$target_platform" in -ios) case "$target_arch" in - default) archs="$default_ios_arch";; - *) archs="$target_arch";; - esac - ;; -android) case "$target_arch" in - default) archs="$default_android_arch";; - *) archs="$target_arch";; - esac - ;; -*) archs="$target_arch";; -esac - -# For ios, the library names for the CPU types accumulate in $platform_libs -platform_libs= - -# Compile nsync. -for arch in $archs; do - nsync_platform_dir="$nsync_builds_dir/$arch.$target_platform.c++11" - - # Get Makefile for target. - case "$target_platform" in - linux) makefile=' - CC=${CC_PREFIX} g++ - PLATFORM_CPPFLAGS=-DNSYNC_USE_CPP11_TIMEPOINT -DNSYNC_ATOMIC_CPP11 \ - -I../../platform/c++11.futex \ - -I../../platform/c++11 -I../../platform/gcc \ - -I../../platform/posix -pthread - PLATFORM_CFLAGS=-std=c++11 -Werror -Wall -Wextra -pedantic - PLATFORM_LDFLAGS=-pthread - MKDEP=${CC} -M -std=c++11 - PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c \ - ../../platform/c++11/src/per_thread_waiter.cc \ - ../../platform/c++11/src/yield.cc \ - ../../platform/c++11/src/time_rep_timespec.cc \ - ../../platform/c++11/src/nsync_panic.cc - PLATFORM_OBJS=nsync_semaphore_futex.o per_thread_waiter.o yield.o \ - time_rep_timespec.o nsync_panic.o - TEST_PLATFORM_C=../../platform/c++11/src/start_thread.cc - TEST_PLATFORM_OBJS=start_thread.o - include ../../platform/posix/make.common - include dependfile - ';; - - ios) arch_flags= - case "$arch" in - x86_64) - arch_flags="$arch_flags -mios-simulator-version-min=8.0" - arch_flags="$arch_flags -isysroot $(xcrun --sdk iphonesimulator --show-sdk-path)" - ;; - *) - arch_flags="$arch_flags -miphoneos-version-min=8.0" - arch_flags="$arch_flags -isysroot $(xcrun --sdk iphoneos --show-sdk-path)" - ;; - esac - makefile=' - CC=${CC_PREFIX} clang++ - PLATFORM_CPPFLAGS=-DNSYNC_USE_CPP11_TIMEPOINT -DNSYNC_ATOMIC_CPP11 \ - -I../../platform/c++11 -I../../platform/gcc_no_tls \ - -I../../platform/macos -I../../platform/posix -pthread - PLATFORM_CFLAGS=-arch '"$arch"' -fno-exceptions -stdlib=libc++ \ - -fembed-bitcode '"$arch_flags"' -fPIC -x c++ \ - -std=c++11 -Werror -Wall -Wextra -pedantic - PLATFORM_LDFLAGS=-pthread - MKDEP=${CC} -x c++ -M -std=c++11 - PLATFORM_C=../../platform/posix/src/clock_gettime.c \ - ../../platform/c++11/src/nsync_semaphore_mutex.cc \ - ../../platform/posix/src/per_thread_waiter.c \ - ../../platform/c++11/src/yield.cc \ - ../../platform/c++11/src/time_rep_timespec.cc \ - ../../platform/c++11/src/nsync_panic.cc - PLATFORM_OBJS=clock_gettime.o nsync_semaphore_mutex.o per_thread_waiter.o \ - yield.o time_rep_timespec.o nsync_panic.o - TEST_PLATFORM_C=../../platform/c++11/src/start_thread.cc - TEST_PLATFORM_OBJS=start_thread.o - include ../../platform/posix/make.common - include dependfile - ';; - - macos) makefile=' - CC=${CC_PREFIX} clang++ - PLATFORM_CPPFLAGS=-DNSYNC_USE_CPP11_TIMEPOINT -DNSYNC_ATOMIC_CPP11 \ - -I../../platform/c++11 -I../../platform/gcc \ - -I../../platform/macos -I../../platform/posix -pthread - PLATFORM_CFLAGS=-x c++ -std=c++11 -Werror -Wall -Wextra -pedantic - PLATFORM_LDFLAGS=-pthread - MKDEP=${CC} -x c++ -M -std=c++11 - PLATFORM_C=../../platform/posix/src/clock_gettime.c \ - ../../platform/c++11/src/nsync_semaphore_mutex.cc \ - ../../platform/posix/src/per_thread_waiter.c \ - ../../platform/c++11/src/yield.cc \ - ../../platform/c++11/src/time_rep_timespec.cc \ - ../../platform/c++11/src/nsync_panic.cc - PLATFORM_OBJS=clock_gettime.o nsync_semaphore_mutex.o per_thread_waiter.o \ - yield.o time_rep_timespec.o nsync_panic.o - TEST_PLATFORM_C=../../platform/c++11/src/start_thread.cc - TEST_PLATFORM_OBJS=start_thread.o - include ../../platform/posix/make.common - include dependfile - ';; - - android) - # The Android build uses many different names for the same - # platform in different parts of the tree, so things get messy here. - - # Make $android_os_arch be the OS-arch name for the host - # binaries used in the NDK tree. - case "$host_platform" in - linux) android_os_arch=linux;; - macos) android_os_arch=darwin;; - *) android_os_arch="$host_platform";; - esac - case "$host_arch" in - x86_32) android_os_arch="$android_os_arch"-x86;; - *) android_os_arch="$android_os_arch-$host_arch";; - esac - - case "$arch" in - arm64-v8a) toolchain="aarch64-linux-android-4.9" - sysroot_arch="arm64" - bin_prefix="aarch64-linux-android" - march_option= - ;; - armeabi) toolchain="arm-linux-androideabi-4.9" - sysroot_arch="arm" - bin_prefix="arm-linux-androideabi" - march_option= - ;; - armeabi-v7a) toolchain="arm-linux-androideabi-4.9" - sysroot_arch="arm" - bin_prefix="arm-linux-androideabi" - march_option="-march=armv7-a -mfloat-abi=softfp -mfpu=neon" - ;; - armeabi-v7a-hard) toolchain="arm-linux-androideabi-4.9" - sysroot_arch="arm" - bin_prefix="arm-linux-androideabi" - march_option="-march=armv7-a -mfpu=neon" - ;; - mips) toolchain="mipsel-linux-android-4.9" - sysroot_arch="mips" - bin_prefix="mipsel-linux-android" - march_option= - ;; - mips64) toolchain="mips64el-linux-android-4.9" - sysroot_arch="mips64" - bin_prefix="mips64el-linux-android" - march_option= - ;; - x86) toolchain="x86-4.9" - sysroot_arch="x86" - bin_prefix="i686-linux-android" - march_option= - ;; - x86_64) toolchain="x86_64-4.9" - sysroot_arch="x86_64" - bin_prefix="x86_64-linux-android" - march_option= - ;; - *) echo "android is not supported for $arch" >&2 - echo "$usage" >&2 - exit 2 - ;; - esac - - - android_target_platform=armeabi - case "$NDK_ROOT" in - "") echo "$prog: requires \$NDK_ROOT for android build" >&2 - exit 2;; - esac - - makefile=' - AR := ${NDK_ROOT}/toolchains/'"$toolchain"'/prebuilt/'"$android_os_arch"'/bin/'"$bin_prefix"'-ar - CC=${CC_PREFIX} \ - ${NDK_ROOT}/toolchains/'"$toolchain"'/prebuilt/'"$android_os_arch"'/bin/'"$bin_prefix"'-g++ - PLATFORM_CPPFLAGS=--sysroot \ - $(NDK_ROOT)/platforms/android-'"$android_api_version"'/arch-'"$sysroot_arch"' \ - -DNSYNC_USE_CPP11_TIMEPOINT -DNSYNC_ATOMIC_CPP11 \ - -I$(NDK_ROOT)/sources/android/support/include \ - -I$(NDK_ROOT)/sources/cxx-stl/gnu-libstdc++/4.9/include \ - -I$(NDK_ROOT)/sources/cxx-stl/gnu-libstdc++/4.9/libs/'"$arch"'/include \ - -I../../platform/c++11 -I../../platform/gcc \ - -I../../platform/posix -pthread - PLATFORM_CFLAGS=-std=c++11 -Wno-narrowing '"$march_option"' -fPIE -fPIC - PLATFORM_LDFLAGS=-pthread - MKDEP=${CC} -M -std=c++11 - PLATFORM_C=../../platform/c++11/src/nsync_semaphore_mutex.cc \ - ../../platform/posix/src/per_thread_waiter.c \ - ../../platform/c++11/src/yield.cc \ - ../../platform/c++11/src/time_rep_timespec.cc \ - ../../platform/c++11/src/nsync_panic.cc - PLATFORM_OBJS=nsync_semaphore_mutex.o per_thread_waiter.o yield.o \ - time_rep_timespec.o nsync_panic.o - TEST_PLATFORM_C=../../platform/c++11/src/start_thread.cc - TEST_PLATFORM_OBJS=start_thread.o - include ../../platform/posix/make.common - include dependfile - ';; - - *) echo "$usage" >&2; exit 2;; - esac - - if [ ! -d "$nsync_platform_dir" ]; then - mkdir "$nsync_platform_dir" - echo "$makefile" | sed $'s,^[ \t]*,,' > "$nsync_platform_dir/Makefile" - touch "$nsync_platform_dir/dependfile" - fi - if (cd "$nsync_platform_dir" && make depend nsync.a >&2); then - case "$target_platform" in - ios) platform_libs="$platform_libs '$nsync_platform_dir/nsync.a'";; - *) echo "$nsync_platform_dir/nsync.a";; - esac - else - exit 2 # The if-statement suppresses the "set -e" on the "make". - fi -done - -case "$target_platform" in -ios) nsync_platform_dir="$nsync_builds_dir/lipo.$target_platform.c++11" - if [ -d "$nsync_platform_dir" ]; then - rm -rf "$nsync_platform_dir" - fi - mkdir "$nsync_platform_dir" - eval lipo $platform_libs -create -output '$nsync_platform_dir/nsync.a' - echo "$nsync_platform_dir/nsync.a" - ;; -esac diff --git a/tensorflow/contrib/makefile/compile_pi_protobuf.sh b/tensorflow/contrib/makefile/compile_pi_protobuf.sh deleted file mode 100755 index bc0978a4b4a..00000000000 --- a/tensorflow/contrib/makefile/compile_pi_protobuf.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -x -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# Builds protobuf 3 for Raspberry Pi. - -cd tensorflow/contrib/makefile || exit 1 - -GENDIR=$(pwd)/gen/protobuf_pi/ -LIBDIR=${GENDIR} -mkdir -p ${LIBDIR} - -CXX=arm-linux-gnueabihf-g++ - -cd downloads/protobuf || exit 1 - -./autogen.sh -if [ $? -ne 0 ] -then - echo "./autogen.sh command failed." - exit 1 -fi - -make distclean -./configure \ ---build=i686-pc-linux-gnu \ ---host=arm-linux \ ---target=arm-linux \ ---disable-shared \ ---enable-cross-compile \ ---with-protoc=protoc \ ---prefix=${LIBDIR} \ ---exec-prefix=${LIBDIR} \ -"CXX=${CXX}" \ -make -make install diff --git a/tensorflow/contrib/makefile/create_ios_frameworks.sh b/tensorflow/contrib/makefile/create_ios_frameworks.sh deleted file mode 100644 index 2bbde6aa885..00000000000 --- a/tensorflow/contrib/makefile/create_ios_frameworks.sh +++ /dev/null @@ -1,112 +0,0 @@ -#! /bin/sh - -#!/usr/bin/env bash -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# Must be run after: build_all_ios.sh -# Creates an iOS framework which is placed under: -# gen/ios_frameworks/tensorflow_experimental.framework.zip - -set -e -pushd . - -echo "Starting" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -TMP_DIR=$(mktemp -d) -echo "Package dir: " $TMP_DIR -FW_DIR=$TMP_DIR/tensorflow_ios_frameworks -FW_DIR_TFCORE=$FW_DIR/tensorflow_experimental.framework -FW_DIR_TFCORE_HDRS=$FW_DIR_TFCORE/Headers - -echo "Creating target Headers directories" -mkdir -p $FW_DIR_TFCORE_HDRS - -echo "Generate master LICENSE file and copy to target" -bazel build //tensorflow/tools/lib_package:clicenses_generate -cp $SCRIPT_DIR/../../../bazel-genfiles/tensorflow/tools/lib_package/include/tensorflow/c/LICENSE \ - $FW_DIR_TFCORE - -echo "Copying static libraries" -cp $SCRIPT_DIR/gen/lib/libtensorflow-core.a \ - $FW_DIR_TFCORE/tensorflow_experimental -cp $SCRIPT_DIR/gen/protobuf_ios/lib/libprotobuf.a \ - $FW_DIR_TFCORE/libprotobuf_experimental.a - -echo "Headers, populating: tensorflow (core)" -cd $SCRIPT_DIR/../../.. -find tensorflow -name "*.h" | tar -cf $FW_DIR_TFCORE_HDRS/tmp.tar -T - -cd $FW_DIR_TFCORE_HDRS -tar xf tmp.tar -rm -f tmp.tar - -echo "Headers, populating: third_party" -cd $SCRIPT_DIR/../../.. -tar cf $FW_DIR_TFCORE_HDRS/tmp.tar third_party -cd $FW_DIR_TFCORE_HDRS -tar xf tmp.tar -rm -f tmp.tar - -echo "Headers, populating: unsupported" -cd $SCRIPT_DIR/downloads/eigen -tar cf $FW_DIR_TFCORE_HDRS/third_party/eigen3/tmp.tar unsupported -cd $FW_DIR_TFCORE_HDRS/third_party/eigen3 -tar xf tmp.tar -rm -f tmp.tar - -echo "Headers, populating: Eigen" -cd $SCRIPT_DIR/downloads/eigen -tar cf $FW_DIR_TFCORE_HDRS/third_party/eigen3/tmp.tar Eigen -cd $FW_DIR_TFCORE_HDRS/third_party/eigen3 -tar xf tmp.tar -rm -f tmp.tar - -echo "Headers, populating: tensorflow (protos)" -cd $SCRIPT_DIR/gen/proto -tar cf $FW_DIR_TFCORE_HDRS/tmp.tar tensorflow -cd $FW_DIR_TFCORE_HDRS -tar xf tmp.tar -# Don't include the auto downloaded/generated to build this library -rm -rf tensorflow/contrib/makefile -rm -f tmp.tar - -echo "Headers, populating: google (proto src)" -cd $SCRIPT_DIR/downloads/protobuf/src -tar cf $FW_DIR_TFCORE_HDRS/tmp.tar google -cd $FW_DIR_TFCORE_HDRS -tar xf tmp.tar -rm -f tmp.tar - -# This is required, otherwise they interfere with the documentation of the -# pod at cocoapods.org -echo "Remove all README files" -cd $FW_DIR_TFCORE_HDRS -find . -type f -name README\* -exec rm -f {} \; -find . -type f -name readme\* -exec rm -f {} \; - -TARGET_GEN_LOCATION="$SCRIPT_DIR/gen/ios_frameworks" -echo "Moving results to target: " $TARGET_GEN_LOCATION -cd $FW_DIR -zip -q -r tensorflow_experimental.framework.zip tensorflow_experimental.framework -x .DS_Store -rm -rf $TARGET_GEN_LOCATION -mkdir -p $TARGET_GEN_LOCATION -cp -r tensorflow_experimental.framework.zip $TARGET_GEN_LOCATION - -echo "Cleaning up" -popd -rm -rf $TMP_DIR - -echo "Finished" diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh deleted file mode 100755 index 6cf1145021c..00000000000 --- a/tensorflow/contrib/makefile/download_dependencies.sh +++ /dev/null @@ -1,147 +0,0 @@ -#!/bin/bash -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -set -e - -DOWNLOADS_DIR=tensorflow/contrib/makefile/downloads -BZL_FILE_PATH=tensorflow/workspace.bzl - -# Ensure it is being run from repo root -if [ ! -f $BZL_FILE_PATH ]; then - echo "Could not find ${BZL_FILE_PATH}": - echo "Likely you are not running this from the root directory of the repository."; - exit 1; -fi - -EIGEN_URL="$(grep -o 'https://bitbucket.org/eigen/eigen/get/.*tar\.gz' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)" -GEMMLOWP_URL="$(grep -o 'https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/gemmlowp/.*zip' "${BZL_FILE_PATH}" | head -n1)" -GOOGLETEST_URL="https://github.com/google/googletest/archive/release-1.8.0.tar.gz" -NSYNC_URL="$(grep -o 'https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/nsync/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" - -# Note: The protobuf repo needs to be cloned due to its submodules. -# These variables contain the GitHub repo and the sha, from `tensorflow/workspace.bzl`, -# from which to clone it from and checkout to. -readonly PROTOBUF_REPO="https://github.com/protocolbuffers/protobuf.git" -readonly PROTOBUF_TAG="$(grep -o 'https://github.com/protocolbuffers/protobuf/archive/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1 | awk '{print substr($0, index($0, "archive") + 8, index($0, "tar") - index($0, "archive") - 9) }')" - -# TODO (yongtang): Replace the following with 'https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/re2/.*tar\.gz' once -# the archive has been propagated in mirror.tensorflow.org. -RE2_URL="$(grep -o 'https://github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" -FFT2D_URL="$(grep -o 'http.*fft2d\.tgz' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)" -DOUBLE_CONVERSION_URL="$(grep -o "https.*google/double-conversion.*\.zip" "${BZL_FILE_PATH}" | head -n1)" -ABSL_URL="$(grep -o 'https://github.com/abseil/abseil-cpp/.*tar.gz' "${BZL_FILE_PATH}" | head -n1)" -CUB_URL="$(grep -o 'https.*cub/archive.*zip' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)" - -# Required for TensorFlow Lite Flex runtime. -FARMHASH_URL="https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz" -FLATBUFFERS_URL="https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/flatbuffers/archive/v1.11.0.tar.gz" - -# TODO(petewarden): Some new code in Eigen triggers a clang bug with iOS arm64, -# so work around it by patching the source. -replace_by_sed() { - local regex="${1}" - shift - # Detect the version of sed by the return value of "--version" flag. GNU-sed - # supports "--version" while BSD-sed doesn't. - if ! sed --version >/dev/null 2>&1; then - # BSD-sed. - sed -i '' -e "${regex}" "$@" - else - # GNU-sed. - sed -i -e "${regex}" "$@" - fi -} - -download_and_extract() { - local usage="Usage: download_and_extract URL DIR" - local url="${1:?${usage}}" - local dir="${2:?${usage}}" - echo "downloading ${url}" >&2 - mkdir -p "${dir}" - if [[ "${url}" == *gz ]]; then - curl -Ls "${url}" | tar -C "${dir}" --strip-components=1 -xz - elif [[ "${url}" == *zip ]]; then - tempdir=$(mktemp -d) - tempdir2=$(mktemp -d) - if [[ "$OSTYPE" == "darwin"* ]]; then - # macOS (AKA darwin) doesn't have wget. - (cd "${tempdir}"; curl --remote-name --silent --location "${url}") - else - wget -P "${tempdir}" "${url}" - fi - unzip "${tempdir}"/* -d "${tempdir2}" - # unzip has no strip components, so unzip to a temp dir, and move the files - # we want from the tempdir to destination. - cp -R "${tempdir2}"/*/* "${dir}"/ - rm -rf "${tempdir2}" "${tempdir}" - fi - - # Delete any potential BUILD files, which would interfere with Bazel builds. - find "${dir}" -type f -name '*BUILD' -delete -} - -function clone_repository() { - local repo_url="${1}" - local destination_directory="${2}" - local commit_sha="${3}" - - if [[ -d "${destination_directory}" ]]; then - rm -rf "${destination_directory}" - fi - - git clone "${repo_url}" "${destination_directory}" - - pushd "$(pwd)" 1>/dev/null - - cd "${destination_directory}" - - if [[ -n "${commit_sha}" ]]; then - git checkout "${PROTOBUF_TAG}" - fi - - git submodule update --init - - popd 1>/dev/null -} - -download_and_extract "${EIGEN_URL}" "${DOWNLOADS_DIR}/eigen" -download_and_extract "${GEMMLOWP_URL}" "${DOWNLOADS_DIR}/gemmlowp" -download_and_extract "${GOOGLETEST_URL}" "${DOWNLOADS_DIR}/googletest" -download_and_extract "${NSYNC_URL}" "${DOWNLOADS_DIR}/nsync" -download_and_extract "${RE2_URL}" "${DOWNLOADS_DIR}/re2" -download_and_extract "${FFT2D_URL}" "${DOWNLOADS_DIR}/fft2d" -download_and_extract "${DOUBLE_CONVERSION_URL}" "${DOWNLOADS_DIR}/double_conversion" -download_and_extract "${ABSL_URL}" "${DOWNLOADS_DIR}/absl" -download_and_extract "${CUB_URL}" "${DOWNLOADS_DIR}/cub/external/cub_archive" - -# Required for TensorFlow Lite Flex runtime. -download_and_extract "${FARMHASH_URL}" "${DOWNLOADS_DIR}/farmhash" -download_and_extract "${FLATBUFFERS_URL}" "${DOWNLOADS_DIR}/flatbuffers" - -clone_repository "${PROTOBUF_REPO}" "${DOWNLOADS_DIR}/protobuf" "${PROTOBUF_TAG}" - -replace_by_sed 's#static uint32x4_t p4ui_CONJ_XOR = vld1q_u32( conj_XOR_DATA );#static uint32x4_t p4ui_CONJ_XOR; // = vld1q_u32( conj_XOR_DATA ); - Removed by script#' \ - "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h" -replace_by_sed 's#static uint32x2_t p2ui_CONJ_XOR = vld1_u32( conj_XOR_DATA );#static uint32x2_t p2ui_CONJ_XOR;// = vld1_u32( conj_XOR_DATA ); - Removed by scripts#' \ - "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h" -replace_by_sed 's#static uint64x2_t p2ul_CONJ_XOR = vld1q_u64( p2ul_conj_XOR_DATA );#static uint64x2_t p2ul_CONJ_XOR;// = vld1q_u64( p2ul_conj_XOR_DATA ); - Removed by script#' \ - "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h" -# TODO(satok): Remove this once protobuf/autogen.sh is fixed. -replace_by_sed 's#https://googlemock.googlecode.com/files/gmock-1.7.0.zip#https://storage.googleapis.com/download.tensorflow.org/deps/gmock-1.7.0.zip#' \ - "${DOWNLOADS_DIR}/protobuf/autogen.sh" -cat "third_party/eigen3/gebp_neon.patch" | patch "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/products/GeneralBlockPanelKernel.h" - -echo "download_dependencies.sh completed successfully." >&2 diff --git a/tensorflow/contrib/makefile/gen_file_lists.sh b/tensorflow/contrib/makefile/gen_file_lists.sh deleted file mode 100755 index 68a6fdf9094..00000000000 --- a/tensorflow/contrib/makefile/gen_file_lists.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -x -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# This script generates the source file lists needed by the makefile by querying -# the master Bazel build configuration. - -bazel query 'kind("source file", deps(//tensorflow/core:android_tensorflow_lib))' | \ -grep "//tensorflow/.*\.proto$" | \ -sed -E 's#^//##g' | \ -sed -E 's#:#/#g' \ -> tensorflow/contrib/makefile/tf_proto_files.txt - -bazel query 'kind("generated file", deps(//tensorflow/core:proto_text))' | \ -grep "pb_text\.cc$" | \ -sed -E 's#^//##g' | \ -sed -E 's#:#/#g' \ -> tensorflow/contrib/makefile/tf_pb_text_files.txt - -bazel query 'kind("source file", deps(//tensorflow/tools/proto_text:gen_proto_text_functions))' | \ -grep -E "//tensorflow/.*\.cc$" | \ -grep -E -v "jpeg" | \ -grep -E -v "png" | \ -sed -E 's#^//##g' | \ -sed -E 's#:#/#g' \ -> tensorflow/contrib/makefile/proto_text_cc_files.txt - -bazel query 'kind("generated file", deps(//tensorflow/tools/proto_text:gen_proto_text_functions))' | \ -grep -E "//tensorflow/.*\.cc$" | \ -sed -E 's#^//##g' | \ -sed -E 's#:#/#g' \ -> tensorflow/contrib/makefile/proto_text_pb_cc_files.txt - -bazel query 'kind("generated file", deps(//tensorflow/tools/proto_text:gen_proto_text_functions))' | \ -grep -E "//tensorflow/.*\.h$" | \ -sed -E 's#^//##g' | \ -sed -E 's#:#/#g' \ -> tensorflow/contrib/makefile/proto_text_pb_h_files.txt diff --git a/tensorflow/contrib/makefile/proto_text_cc_files.txt b/tensorflow/contrib/makefile/proto_text_cc_files.txt deleted file mode 100644 index c07a97933b7..00000000000 --- a/tensorflow/contrib/makefile/proto_text_cc_files.txt +++ /dev/null @@ -1,61 +0,0 @@ -tensorflow/core/framework/resource_handle.cc -tensorflow/core/framework/tensor_shape.cc -tensorflow/core/lib/core/arena.cc -tensorflow/core/lib/core/coding.cc -tensorflow/core/lib/core/status.cc -tensorflow/core/lib/hash/crc32c.cc -tensorflow/core/lib/hash/crc32c_accelerate.cc -tensorflow/core/lib/hash/hash.cc -tensorflow/core/lib/histogram/histogram.cc -tensorflow/core/lib/io/block.cc -tensorflow/core/lib/io/block_builder.cc -tensorflow/core/lib/io/buffered_inputstream.cc -tensorflow/core/lib/io/compression.cc -tensorflow/core/lib/io/format.cc -tensorflow/core/lib/io/inputbuffer.cc -tensorflow/core/lib/io/inputstream_interface.cc -tensorflow/core/lib/io/iterator.cc -tensorflow/core/lib/io/path.cc -tensorflow/core/lib/io/random_inputstream.cc -tensorflow/core/lib/io/record_reader.cc -tensorflow/core/lib/io/record_writer.cc -tensorflow/core/lib/io/table.cc -tensorflow/core/lib/io/table_builder.cc -tensorflow/core/lib/io/two_level_iterator.cc -tensorflow/core/lib/io/zlib_compression_options.cc -tensorflow/core/lib/io/zlib_inputstream.cc -tensorflow/core/lib/io/zlib_outputbuffer.cc -tensorflow/core/lib/random/distribution_sampler.cc -tensorflow/core/lib/random/random.cc -tensorflow/core/lib/random/simple_philox.cc -tensorflow/core/lib/random/weighted_picker.cc -tensorflow/core/lib/strings/ordered_code.cc -tensorflow/core/lib/strings/proto_text_util.cc -tensorflow/core/lib/wav/wav_io.cc -tensorflow/core/platform/cpu_info.cc -tensorflow/core/platform/default/env.cc -tensorflow/core/platform/default/env_time.cc -tensorflow/core/platform/default/load_library.cc -tensorflow/core/platform/default/logging.cc -tensorflow/core/platform/default/mutex.cc -tensorflow/core/platform/default/port.cc -tensorflow/core/platform/default/posix_file_system.cc -tensorflow/core/platform/default/tracing.cc -tensorflow/core/platform/denormal.cc -tensorflow/core/platform/env.cc -tensorflow/core/platform/error.cc -tensorflow/core/platform/file_system.cc -tensorflow/core/platform/file_system_helper.cc -tensorflow/core/platform/numbers.cc -tensorflow/core/platform/protobuf.cc -tensorflow/core/platform/protobuf_util.cc -tensorflow/core/platform/scanner.cc -tensorflow/core/platform/setround.cc -tensorflow/core/platform/strcat.cc -tensorflow/core/platform/stringprintf.cc -tensorflow/core/platform/str_util.cc -tensorflow/core/platform/tensor_coding.cc -tensorflow/core/platform/threadpool.cc -tensorflow/core/platform/tracing.cc -tensorflow/tools/proto_text/gen_proto_text_functions.cc -tensorflow/tools/proto_text/gen_proto_text_functions_lib.cc diff --git a/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt b/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt deleted file mode 100644 index eece1897c68..00000000000 --- a/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt +++ /dev/null @@ -1,48 +0,0 @@ -tensorflow/core/example/example.pb.cc -tensorflow/core/example/feature.pb.cc -tensorflow/core/framework/allocation_description.pb.cc -tensorflow/core/framework/api_def.pb.cc -tensorflow/core/framework/attr_value.pb.cc -tensorflow/core/framework/cost_graph.pb.cc -tensorflow/core/framework/device_attributes.pb.cc -tensorflow/core/framework/function.pb.cc -tensorflow/core/framework/graph.pb.cc -tensorflow/core/framework/graph_transfer_info.pb.cc -tensorflow/core/framework/kernel_def.pb.cc -tensorflow/core/framework/log_memory.pb.cc -tensorflow/core/framework/node_def.pb.cc -tensorflow/core/framework/op_def.pb.cc -tensorflow/core/framework/remote_fused_graph_execute_info.pb.cc -tensorflow/core/framework/resource_handle.pb.cc -tensorflow/core/framework/step_stats.pb.cc -tensorflow/core/framework/summary.pb.cc -tensorflow/core/framework/tensor.pb.cc -tensorflow/core/framework/tensor_description.pb.cc -tensorflow/core/framework/tensor_shape.pb.cc -tensorflow/core/framework/tensor_slice.pb.cc -tensorflow/core/framework/types.pb.cc -tensorflow/core/framework/variable.pb.cc -tensorflow/core/framework/versions.pb.cc -tensorflow/core/grappler/costs/op_performance_data.pb.cc -tensorflow/core/lib/core/error_codes.pb.cc -tensorflow/core/protobuf/trackable_object_graph.pb.cc -tensorflow/core/protobuf/cluster.pb.cc -tensorflow/core/protobuf/config.pb.cc -tensorflow/core/protobuf/eager_service.pb.cc -tensorflow/core/protobuf/debug.pb.cc -tensorflow/core/protobuf/device_properties.pb.cc -tensorflow/core/protobuf/meta_graph.pb.cc -tensorflow/core/protobuf/named_tensor.pb.cc -tensorflow/core/protobuf/queue_runner.pb.cc -tensorflow/core/protobuf/remote_tensor_handle.pb.cc -tensorflow/core/protobuf/rewriter_config.pb.cc -tensorflow/core/protobuf/saved_object_graph.pb.cc -tensorflow/core/protobuf/saver.pb.cc -tensorflow/core/protobuf/struct.pb.cc -tensorflow/core/protobuf/tensorflow_server.pb.cc -tensorflow/core/protobuf/verifier_config.pb.cc -tensorflow/core/protobuf/trace_events.pb.cc -tensorflow/core/util/event.pb.cc -tensorflow/core/util/memmapped_file_system.pb.cc -tensorflow/core/util/saved_tensor_slice.pb.cc -tensorflow/core/util/test_log.pb.cc diff --git a/tensorflow/contrib/makefile/proto_text_pb_h_files.txt b/tensorflow/contrib/makefile/proto_text_pb_h_files.txt deleted file mode 100644 index ee3f74cb3ee..00000000000 --- a/tensorflow/contrib/makefile/proto_text_pb_h_files.txt +++ /dev/null @@ -1,50 +0,0 @@ -tensorflow/core/example/example.pb.h -tensorflow/core/example/feature.pb.h -tensorflow/core/framework/allocation_description.pb.h -tensorflow/core/framework/api_def.pb.h -tensorflow/core/framework/attr_value.pb.h -tensorflow/core/framework/cost_graph.pb.h -tensorflow/core/framework/device_attributes.pb.h -tensorflow/core/framework/function.pb.h -tensorflow/core/framework/graph.pb.h -tensorflow/core/framework/graph_transfer_info.pb.h -tensorflow/core/framework/kernel_def.pb.h -tensorflow/core/framework/log_memory.pb.h -tensorflow/core/framework/node_def.pb.h -tensorflow/core/framework/op_def.pb.h -tensorflow/core/framework/remote_fused_graph_execute_info.pb.h -tensorflow/core/framework/resource_handle.pb.h -tensorflow/core/framework/step_stats.pb.h -tensorflow/core/framework/summary.pb.h -tensorflow/core/framework/tensor.pb.h -tensorflow/core/framework/tensor_description.pb.h -tensorflow/core/framework/tensor_shape.pb.h -tensorflow/core/framework/tensor_slice.pb.h -tensorflow/core/framework/types.pb.h -tensorflow/core/framework/variable.pb.h -tensorflow/core/framework/versions.pb.h -tensorflow/core/grappler/costs/op_performance_data.pb.h -tensorflow/core/lib/core/error_codes.pb.h -tensorflow/core/protobuf/trackable_object_graph.pb.h -tensorflow/core/protobuf/cluster.pb.h -tensorflow/core/protobuf/config.pb.h -tensorflow/core/protobuf/debug.pb.h -tensorflow/core/protobuf/device_properties.pb.h -tensorflow/core/protobuf/eager_service.pb.h -tensorflow/core/protobuf/meta_graph.pb.h -tensorflow/core/protobuf/named_tensor.pb.h -tensorflow/core/protobuf/queue_runner.pb.h -tensorflow/core/protobuf/remote_tensor_handle.pb.h -tensorflow/core/protobuf/rewriter_config.pb.h -tensorflow/core/protobuf/saved_object_graph.pb.h -tensorflow/core/protobuf/saver.pb.h -tensorflow/core/protobuf/struct.pb.h -tensorflow/core/protobuf/tensor_bundle.pb.h -tensorflow/core/protobuf/tensorflow_server.pb.h -tensorflow/core/protobuf/verifier_config.pb.h -tensorflow/core/protobuf/trace_events.pb.h -tensorflow/core/util/event.pb.h -tensorflow/core/util/memmapped_file_system.pb.h -tensorflow/core/util/saved_tensor_slice.pb.h -tensorflow/core/util/test_log.pb.h - diff --git a/tensorflow/contrib/makefile/rename_protobuf.sh b/tensorflow/contrib/makefile/rename_protobuf.sh deleted file mode 100755 index 8d52c1a1694..00000000000 --- a/tensorflow/contrib/makefile/rename_protobuf.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# This script modifies the downloaded protobuf library and the TensorFlow source -# to put all protobuf-related symbols into the google::protobuf3 namespace -# instead of the default google::protobuf. This is necessary to work around -# linking issues for applications that use protobuf v2 already, and want to -# adopt TensorFlow, since otherwise the two libraries have duplicate function -# symbols that clash. It also renames all the include paths to google/protobuf3 -# throughout the protobuf and TensorFlow code. -# This is a massive hack, and if possible it's recommended that you switch your -# whole application to protobuf v3 so there's no mismatch with TensorFlow. There -# are also no guarantees that this will continue to work with future versions of -# protobuf or TensorFlow, or that it's a bulletproof solution for every -# application. -# -# To use this script, run the following sequence: -# tensorflow/contrib/makefile/download_dependencies.sh -# tensorflow/contrib/makefile/rename_protobuf.sh -# -# You can then build the source as normal. For example on iOS: -# tensorflow/contrib/makefile/compile_ios_protobuf.sh -# tensorflow/contrib/makefile/compile_ios_tensorflow.sh -# -# Note that this script modifies the source code in-place, so once it's been run -# it's no longer suitable for further manual modifications, since the difference -# with the top of tree will already be large. - -mv tensorflow/contrib/makefile/downloads/protobuf/src/google/protobuf \ - tensorflow/contrib/makefile/downloads/protobuf//src/google/protobuf3 - -# Rename protobuf #includes to use protobuf3. -find . \ - -type f \ - \( -name "*.cc" -or -name "*.h" \) \ - -exec sed -i '' \ - 's%#include \([<"]\)google/protobuf/%#include \1google/protobuf3/%' {} \; -find . \ - -type f \ - -name "*.proto" \ - -exec sed -i '' \ - 's%import \(.*\)\([<"]\)google/protobuf/%import \1\2google/protobuf3/%' {} \; - -# Rename the namespace mentions. -find . \ - -type f \ - \( -name "*.cc" -or -name "*.h" \) \ - -exec sed -i '' \ - 's%namespace protobuf\([^3]\)%namespace protobuf3\1%' {} \; -find . \ - -type f \ - \( -name "*.cc" -or -name "*.h" \) \ - -exec sed -i '' \ - 's%protobuf::%protobuf3::%g' {} \; -sed -i '' 's%::google::protobuf;%google::protobuf3;%' \ - tensorflow/core/platform/default/protobuf.h - -# Fix up a couple of special build scripts that look for particular files. -sed -i '' 's%src/google/protobuf/message.cc%src/google/protobuf3/message.cc%' \ - tensorflow/contrib/makefile/downloads/protobuf/configure.ac -sed -i '' 's%src/google/protobuf/stubs/common.h%src/google/protobuf3/stubs/common.h%' \ - tensorflow/contrib/makefile/downloads/protobuf/autogen.sh - -# Update the locations within the protobuf makefile. -sed -i '' 's%google/protobuf/%google/protobuf3/%g' \ - tensorflow/contrib/makefile/downloads/protobuf/src/Makefile.am - -# Make sure protoc can find the new google/protobuf3 paths by putting them at -# the root directory. -cp -r tensorflow/contrib/makefile/downloads/protobuf/src/google . - -# Update the protobuf commands used in the makefile. -sed -i '' 's%$(PROTOC) $(PROTOCFLAGS) $< --cpp_out $(PROTOGENDIR)%tensorflow/contrib/makefile/rename_protoc.sh $(PROTOC) $(PROTOCFLAGS) $< --cpp_out $(PROTOGENDIR)%' tensorflow/contrib/makefile/Makefile -sed -i '' 's%$(PROTOC) $(PROTOCFLAGS) $< --cpp_out $(HOST_GENDIR)%tensorflow/contrib/makefile/rename_protoc.sh $(PROTOC) $(PROTOCFLAGS) $< --cpp_out $(HOST_GENDIR)%' tensorflow/contrib/makefile/Makefile -sed -i '' 's%$(PROTO_TEXT) \\%tensorflow/contrib/makefile/rename_prototext.sh $(PROTO_TEXT) \\%' tensorflow/contrib/makefile/Makefile diff --git a/tensorflow/contrib/makefile/rename_protoc.sh b/tensorflow/contrib/makefile/rename_protoc.sh deleted file mode 100755 index 7eaa60271a6..00000000000 --- a/tensorflow/contrib/makefile/rename_protoc.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -e -x - -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -PROTO_C_COMMAND=$1 -shift -${PROTO_C_COMMAND} $* - -# Assumes that the order is always *.protofile --cpp_out dir -PROTO_LAST_THREE_ARGS=(${@: -3}) -PROTO_FILE=${PROTO_LAST_THREE_ARGS[0]} -CC_FILE=${PROTO_FILE%.proto}.pb.cc -H_FILE=${PROTO_FILE%.proto}.pb.h -GEN_DIR=${PROTO_LAST_THREE_ARGS[2]} -GEN_CC=${GEN_DIR}/${CC_FILE} -GEN_H=${GEN_DIR}/${H_FILE} - -sed -i '' 's%protobuf::%protobuf3::%g' ${GEN_CC} -sed -i '' 's%protobuf::%protobuf3::%g' ${GEN_H} -sed -i '' 's%google_2fprotobuf3_2f%google_2fprotobuf_2f%g' ${GEN_CC} -sed -i '' 's%google_2fprotobuf3_2f%google_2fprotobuf_2f%g' ${GEN_H} diff --git a/tensorflow/contrib/makefile/rename_prototext.sh b/tensorflow/contrib/makefile/rename_prototext.sh deleted file mode 100755 index 0b285aae40e..00000000000 --- a/tensorflow/contrib/makefile/rename_prototext.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -e -x - -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -PROTO_TEXT_COMMAND=$1 -shift -${PROTO_TEXT_COMMAND} $* - -# Assumes a fixed order for the arguments. -PROTO_LAST_FOUR_ARGS=(${@: -4}) -PROTO_FILE=${PROTO_LAST_FOUR_ARGS[3]} -CC_FILE=${PROTO_FILE%.proto}.pb_text.cc -H_FILE=${PROTO_FILE%.proto}.pb_text.h -GEN_DIR=${PROTO_LAST_FOUR_ARGS[0]} -GEN_CC=${GEN_DIR}/${CC_FILE#tensorflow/core} -GEN_H=${GEN_DIR}/${H_FILE#tensorflow/core} - -sed -i '' 's%protobuf::%protobuf3::%g' ${GEN_CC} -sed -i '' 's%protobuf::%protobuf3::%g' ${GEN_H} -sed -i '' 's%google_2fprotobuf3_2f%google_2fprotobuf_2f%g' ${GEN_CC} -sed -i '' 's%google_2fprotobuf3_2f%google_2fprotobuf_2f%g' ${GEN_H} diff --git a/tensorflow/contrib/makefile/samples/build_and_run_inception_hexagon.sh b/tensorflow/contrib/makefile/samples/build_and_run_inception_hexagon.sh deleted file mode 100755 index 421ddd210fd..00000000000 --- a/tensorflow/contrib/makefile/samples/build_and_run_inception_hexagon.sh +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# This is a composite script to build and run inception with hexagon on Android - -set -e - -usage() { - echo "Usage: QUALCOMM_SDK= NDK_ROOT= $(basename "$0")" - echo "Optional: NNLIB_DIR=" - echo "-b build only" - echo "-c test count" - echo "-E enable experimental hexnn ops" - echo "-p use prebuilt hexagon binaries" - echo "-s skip download if files already exist" - exit 1 -} - -TEST_COUNT=1 -SKIP_DOWNLOAD_IF_EXIST=false - -while getopts "bc:Eps" opt_name; do - case "$opt_name" in - b) BUILD_ONLY="true";; - c) TEST_COUNT="${OPTARG}";; - E) ENABLE_EXPERIMENTAL_HEXNN_OPS="true";; - p) USE_PREBUILT_HEXAGON_BINARIES="true";; - s) SKIP_DOWNLOAD_IF_EXIST="true";; - *) usage;; - esac -done -shift $((OPTIND - 1)) - -if [[ -z "${NDK_ROOT}" ]]; then - echo "NDK_ROOT is empty" 1>&2 - usage - exit 1 -fi - -if [[ "${USE_PREBUILT_HEXAGON_BINARIES}" != "true" && - -z "${QUALCOMM_SDK}" ]]; then - echo "QUALCOMM_SDK is empty" 1>&2 - usage - exit 1 -fi - -if [[ "${BUILD_ONLY}" != "true" ]]; then - if ! type adb >/dev/null 2>&1; then - echo "adb is not in your path ${PATH}." - exit 1 - fi - if ! adb shell ls /system/lib/rfsa/adsp/testsig* >/dev/null 2>&1; then - echo "test signature not found. Unlock your phone first" - echo "See ${QUALCOMM_SDK}/tools/elfsigner/README.txt" - exit 1 - fi -fi - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -TF_ROOT_DIR="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" -BUILD_ALL_ANDROID_PATH="${SCRIPT_DIR}/../build_all_android.sh" -GEN_DIR="${SCRIPT_DIR}/gen" -GEN_LIBS_DIR="${GEN_DIR}/libs" -GEN_DOWNLOAD_DIR="${GEN_DIR}/downloads" -URL_BASE="https://storage.googleapis.com/download.tensorflow.org" - -ARCH="armeabi-v7a" - -source "${SCRIPT_DIR}/../build_helper.subr" - -rm -rf "${GEN_DIR}" -mkdir -p "${GEN_LIBS_DIR}" -mkdir -p "${GEN_DOWNLOAD_DIR}" - -if [[ "${USE_PREBUILT_HEXAGON_BINARIES}" == "true" ]]; then - echo "Download prebuilt hexagon binaries" - if [[ "${BUILD_ONLY}" != "true" ]]; then - CONTROLLER_PUSH_DEST="/data/local/tmp" - NN_LIB_PUSH_DEST="/vendor/lib/rfsa/adsp" - fi - download_and_push "${URL_BASE}/deps/hexagon/libhexagon_controller.so" \ -"${GEN_LIBS_DIR}/libhexagon_controller.so" "${CONTROLLER_PUSH_DEST}" \ -"${SKIP_DOWNLOAD_IF_EXIST}" - - download_and_push "${URL_BASE}/deps/hexagon/libhexagon_nn_skel.so" \ -"${GEN_LIBS_DIR}/libhexagon_nn_skel.so" "${NN_LIB_PUSH_DEST}" \ -"${SKIP_DOWNLOAD_IF_EXIST}" -else - echo "Build hexagon binaries from source code" - cd "${GEN_DIR}" - if [[ -z "${NNLIB_DIR}" ]]; then - git clone https://source.codeaurora.org/quic/hexagon_nn/nnlib - else - if [[ ! -f "${NNLIB_DIR}/Makefile" ]]; then - echo "Couldn't locate ${NNLIB_DIR}/Makefile" 1>&2 - exit 1 - fi - echo "Use nnlib in ${NNLIB_DIR}" 1>&2 - GEN_NNLIB_DIR="${GEN_DIR}/nnlib" - mkdir -p "${GEN_NNLIB_DIR}" - cp -af "${NNLIB_DIR}/"* "${GEN_NNLIB_DIR}" - fi - - cd "${QUALCOMM_SDK}" - source "${QUALCOMM_SDK}/setup_sdk_env.sh" - - GENERATED_NNLIB_DIRECTORY="${QUALCOMM_SDK}/examples/common/generated_nnlib" - if [[ -d "${GENERATED_NNLIB_DIRECTORY}" ]]; then - echo "Existing nnlib found. Remove" - rm -rf "${GENERATED_NNLIB_DIRECTORY}" - fi - - cp -af "${GEN_DIR}/nnlib" "${GENERATED_NNLIB_DIRECTORY}" - cd "${GENERATED_NNLIB_DIRECTORY}" - - make clean V=hexagon_Release_dynamic_toolv72_v60 - rm -rf hexagon_Release_dynamic_toolv72_v60 - make tree VERBOSE=1 V=hexagon_Release_dynamic_toolv72_v60 - - GENERATED_HEXAGON_CONTROLLER_DIRECTORY=\ -"${QUALCOMM_SDK}/examples/common/generated_hexagon_controller" - - if [[ -d "${GENERATED_HEXAGON_CONTROLLER_DIRECTORY}" ]]; then - echo "Existing hexagon controller found. Remove" - rm -rf "${GENERATED_HEXAGON_CONTROLLER_DIRECTORY}" - fi - - cp -af "${TF_ROOT_DIR}/tensorflow/contrib/hvx/hexagon_controller" \ - "${GENERATED_HEXAGON_CONTROLLER_DIRECTORY}" - - echo "Copy interface directory" - cp -afv "${GENERATED_NNLIB_DIRECTORY}/interface" \ - "${GENERATED_HEXAGON_CONTROLLER_DIRECTORY}/" - - echo "Copy glue directory" - cp -afv "${GENERATED_NNLIB_DIRECTORY}/glue" \ - "${GENERATED_HEXAGON_CONTROLLER_DIRECTORY}/" - - cd "${GENERATED_HEXAGON_CONTROLLER_DIRECTORY}" - make clean V=android_Release - rm -rf android_Release - make tree VERBOSE=1 V=android_Release - - cp -v "${GENERATED_HEXAGON_CONTROLLER_DIRECTORY}/android_Release/ship/libhexagon_controller.so" \ - "${GEN_LIBS_DIR}" - cp -v "${GENERATED_NNLIB_DIRECTORY}/hexagon_Release_dynamic_toolv72_v60/ship/libhexagon_nn_skel.so" \ - "${GEN_LIBS_DIR}" -fi - -if [[ -d "${TF_ROOT_DIR}/tensorflow/contrib/makefile/gen/protobuf" && - -d "${TF_ROOT_DIR}/tensorflow/contrib/makefile/gen/protobuf-host" ]]; then - echo "generated protobuf and protobuf-host found." - EXTRA_ARGS+=("-T") -fi - -if [[ "${ENABLE_EXPERIMENTAL_HEXNN_OPS}" == "true" ]]; then - EXTRA_ARGS+=("-E") -fi - -if [[ -z "${CC_PREFIX}" ]]; then - echo "HINT: Installing ccache and specifying CC_PREFIX=ccache accelerate build time" -fi - -CC_PREFIX=${CC_PREFIX} NDK_ROOT=${NDK_ROOT} "${BUILD_ALL_ANDROID_PATH}" \ --x "${GEN_LIBS_DIR}" \ --s "${TF_ROOT_DIR}/tensorflow/contrib/makefile/sub_makefiles/hexagon_graph_execution/Makefile.in" \ --t "hexagon_graph_execution" ${EXTRA_ARGS[@]} - -echo "Download and push inception image" -HEXAGON_DOWNLOAD_PATH=\ -"${TF_ROOT_DIR}/tensorflow/contrib/makefile/downloads/hexagon" -rm -rf "${HEXAGON_DOWNLOAD_PATH}" -mkdir -p "${HEXAGON_DOWNLOAD_PATH}/libs" - -if [[ "${BUILD_ONLY}" != "true" ]]; then - BIN_PUSH_DEST="/data/local/tmp" -fi - -download_and_push "${URL_BASE}/example_images/img_299x299.bmp" \ -"${GEN_DOWNLOAD_DIR}/img_299x299.bmp" "${BIN_PUSH_DEST}" \ -"${SKIP_DOWNLOAD_IF_EXIST}" - -download_and_push \ -"${URL_BASE}/models/tensorflow_inception_v3_stripped_optimized_quantized.pb" \ -"${GEN_DOWNLOAD_DIR}/tensorflow_inception_v3_stripped_optimized_quantized.pb" \ -"${BIN_PUSH_DEST}" \ -"${SKIP_DOWNLOAD_IF_EXIST}" - -download_and_push "${URL_BASE}/models/imagenet_comp_graph_label_strings.txt" \ -"${GEN_DOWNLOAD_DIR}/imagenet_comp_graph_label_strings.txt" "${BIN_PUSH_DEST}" \ -"${SKIP_DOWNLOAD_IF_EXIST}" - -# By default this script runs a test to fuse and run the model -gtest_args+=("--gtest_filter=GraphTransferer.RunInceptionV3OnHexagonExampleWithTfRuntime") -# Uncomment this block if you want to run the fused model -#gtest_args+=("--gtest_filter=GraphTransferer.RunInceptionV3OnHexagonExampleWithFusedGraph") -# Uncomment this block if you want to run the model with hexagon wrapper -#gtest_args+=( -# "--gtest_also_run_disabled_tests" -# "--gtest_filter=GraphTransferer.DISABLED_RunInceptionV3OnHexagonExampleWithHexagonWrapper") -# Uncomment this block if you want to get the list of tests -#gtest_args+=("--gtest_list_tests") - -if [[ "${BUILD_ONLY}" != "true" ]]; then - echo "Run hexagon_graph_execution" - ANDROID_EXEC_FILE_MODE=755 - - adb push "${GEN_LIBS_DIR}/libhexagon_controller.so" "/data/local/tmp" - adb push "${GEN_LIBS_DIR}/libhexagon_nn_skel.so" "/vendor/lib/rfsa/adsp" - - adb push -p \ - "${TF_ROOT_DIR}/tensorflow/contrib/makefile/gen/bin/android_${ARCH}/hexagon_graph_execution" \ - "/data/local/tmp/" - adb wait-for-device - adb shell chmod "${ANDROID_EXEC_FILE_MODE}" \ - "/data/local/tmp/hexagon_graph_execution" - adb wait-for-device - - for i in $(seq 1 "${TEST_COUNT}"); do - adb shell 'LD_LIBRARY_PATH=/data/local/tmp:$LD_LIBRARY_PATH' \ - "/data/local/tmp/hexagon_graph_execution" ${gtest_args[@]} - done -fi diff --git a/tensorflow/contrib/makefile/sub_makefiles/android/Makefile.in b/tensorflow/contrib/makefile/sub_makefiles/android/Makefile.in deleted file mode 100644 index f7bb4934946..00000000000 --- a/tensorflow/contrib/makefile/sub_makefiles/android/Makefile.in +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# This sub Makefile compiles the TensorFlow Android Inference library. This is -# designed to be used as a sub Makefile with tensorflow/contrib/makefile/Makefile. -# -# You can build targets in this file by including this sub makefile like: -# $ make -f tensorflow/contrib/makefile/Makefile TARGET=ANDROID \ -# SUB_MAKEFILES=\ -# $(pwd)/tensorflow/contrib/makefile/sub_makefiles/android/Makefile.in \ -# (optional: NDK_ROOT=) libtensorflow_inference.so \ -# libtensorflow_demo.so - - -# libtensorflow_inference.so: -# This library provides TensorFlow support on Android via the Java API and -# TensorFlowInferenceInterface. -# It should be packaged into the Android APK along with the Java counterparts -# under tensorflow/java and tensorflow/tools/android/inference_interface/java. -INFERENCE_SRCS := \ -tensorflow/c/tf_status_helper.cc \ -tensorflow/c/checkpoint_reader.cc \ -tensorflow/c/test_op.cc \ -tensorflow/c/c_api.cc \ -tensorflow/java/src/main/native/exception_jni.cc \ -tensorflow/java/src/main/native/graph_jni.cc \ -tensorflow/java/src/main/native/operation_builder_jni.cc \ -tensorflow/java/src/main/native/operation_jni.cc \ -tensorflow/java/src/main/native/session_jni.cc \ -tensorflow/java/src/main/native/tensorflow_jni.cc \ -tensorflow/java/src/main/native/tensor_jni.cc \ -tensorflow/tools/android/inference_interface/jni/run_stats_jni.cc - -INFERENCE_OBJS := $(addprefix $(OBJDIR), $(INFERENCE_SRCS:.cc=.o)) - -INFERENCE_SO_NAME := libtensorflow_inference.so -INFERENCE_SO_PATH := $(LIBDIR)$(INFERENCE_SO_NAME) - -$(INFERENCE_SO_PATH): $(LIB_OBJS) $(INFERENCE_OBJS) $(CUDA_LIB_DEPS) - @mkdir -p $(dir $@) - $(CXX) $(CXXFLAGS) $(INCLUDES) \ - -o $@ $(INFERENCE_OBJS) $(LIB_OBJS) $(TEGRA_LIBS) \ - $(LIBFLAGS) $(LDFLAGS) \ - -shared -Wl,-soname,$(INFERENCE_SO_NAME) \ - $(LIBS) $(CUDA_LIBS) - -$(INFERENCE_SO_NAME): $(INFERENCE_SO_PATH) - - -# libtensorflow_demo.so: -# This library provides the additional native support necessary to run the -# Android TensorFlow demo. This includes image colorspace conversion and object -# tracking code. It does not provide any TensorFlow functionality itself. -DEMO_SRCS := \ -tensorflow/examples/android/jni/imageutils_jni.cc \ -tensorflow/examples/android/jni/object_tracking/frame_pair.cc \ -tensorflow/examples/android/jni/object_tracking/image_neon.cc \ -tensorflow/examples/android/jni/object_tracking/keypoint_detector.cc \ -tensorflow/examples/android/jni/object_tracking/logging.cc \ -tensorflow/examples/android/jni/object_tracking/object_detector.cc \ -tensorflow/examples/android/jni/object_tracking/object_tracker.cc \ -tensorflow/examples/android/jni/object_tracking/object_tracker_jni.cc \ -tensorflow/examples/android/jni/object_tracking/optical_flow.cc \ -tensorflow/examples/android/jni/object_tracking/time_log.cc \ -tensorflow/examples/android/jni/object_tracking/tracked_object.cc \ -tensorflow/examples/android/jni/object_tracking/utils_neon.cc \ -tensorflow/examples/android/jni/rgb2yuv.cc \ -tensorflow/examples/android/jni/yuv2rgb.cc - -DEMO_OBJS := $(addprefix $(OBJDIR), $(DEMO_SRCS:.cc=.o)) - -DEMO_SO_NAME := libtensorflow_demo.so -DEMO_SO_PATH := $(LIBDIR)$(DEMO_SO_NAME) - -CXXFLAGS += -DSTANDALONE_DEMO_LIB -$(DEMO_SO_PATH): $(DEMO_OBJS) - @mkdir -p $(dir $@) - $(CXX) $(CXXFLAGS) \ - -o $@ $(DEMO_OBJS) \ - $(LIBFLAGS) $(LDFLAGS) -shared $(LIBS) - -$(DEMO_SO_NAME): $(DEMO_SO_PATH) diff --git a/tensorflow/contrib/makefile/sub_makefiles/hexagon_graph_execution/Makefile.in b/tensorflow/contrib/makefile/sub_makefiles/hexagon_graph_execution/Makefile.in deleted file mode 100644 index 9aa81144fd2..00000000000 --- a/tensorflow/contrib/makefile/sub_makefiles/hexagon_graph_execution/Makefile.in +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# This sub Makefile compiles libraries under this directory. This is designed to -# be used as a sub Makefile with tensorflow/contrib/makefile/Makefile. -# You can build targets in this file by including this sub makefile like: -# $ make -f tensorflow/contrib/makefile/Makefile TARGET= \ -# SUB_MAKEFILES=\ -# $(pwd)/tensorflow/contrib/makefile/sub_makefiles/hexagon_graph_execution/Makefile.in \ -# (optional: NDK_ROOT=) hexagon_graph_execution -# TODO(satok): Support more targets - -GTEST_DIR := \ -$(MAKEFILE_DIR)/downloads/googletest/googletest - -GTEST_HEADERS = \ -$(wildcard $(GTEST_DIR)/include/gtest/*.h) \ -$(wildcard $(GTEST_DIR)/include/gtest/internal/*.h) - -GTEST_SRCS := \ -$(wildcard $(GTEST_DIR)/src/*.cc) \ -$(wildcard $(GTEST_DIR)/src/*.h) \ -$(GTEST_HEADERS) - -GRAPH_TRANSFER_SRCS := \ -tensorflow/core/platform/posix/test.cc - -GRAPH_EXECUTION_SRCS := \ -$(GRAPH_TRANSFER_SRCS) \ -tensorflow/core/kernels/hexagon/hexagon_graph_execution_test.cc \ -tensorflow/contrib/makefile/test/test_main.cc - -GRAPH_EXECUTION_OBJS := $(addprefix $(OBJDIR), $(GRAPH_EXECUTION_SRCS:.cc=.o)) - -GRAPH_EXECUTION_NAME := hexagon_graph_execution -GRAPH_EXECUTION_BIN_PATH := $(BINDIR)$(GRAPH_EXECUTION_NAME) - -INCLUDES += \ --I$(MAKEFILE_DIR)/downloads/googletest/googletest/include - -GRAPH_EXECUTION_INCLUDES := $(INCLUDES) - -$(OBJDIR)gtest-all.o: $(GTEST_SRCS) $(GRAPH_EXECUTION_OBJS) - $(CXX) $(CXXFLAGS) $(GRAPH_EXECUTION_INCLUDES) -I $(GTEST_DIR) -c \ - $(GTEST_DIR)/src/gtest-all.cc -o $@ - -$(LIBDIR)gtest.a: $(OBJDIR)gtest-all.o - @mkdir -p $(dir $@) - $(AR) $(ARFLAGS) $@ $^ - -$(GRAPH_EXECUTION_BIN_PATH): $(LIB_PATH) $(LIBDIR)gtest.a $(GRAPH_EXECUTION_OBJS) - @mkdir -p $(dir $@) - $(CXX) $(CXXFLAGS) $(GRAPH_EXECUTION_INCLUDES) \ - -o $(GRAPH_EXECUTION_BIN_PATH) $(GRAPH_EXECUTION_OBJS) \ - $(LIBFLAGS) $(LIB_PATH) $(LIBDIR)gtest.a $(LDFLAGS) $(LIBS) - -$(GRAPH_EXECUTION_NAME): $(GRAPH_EXECUTION_BIN_PATH) diff --git a/tensorflow/contrib/makefile/sub_makefiles/quantization/Makefile.in b/tensorflow/contrib/makefile/sub_makefiles/quantization/Makefile.in deleted file mode 100644 index 362ccedfc2f..00000000000 --- a/tensorflow/contrib/makefile/sub_makefiles/quantization/Makefile.in +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# This sub Makefile compiles libraries under this directory. This is designed to -# be used as a sub Makefile with tensorflow/contrib/makefile/Makefile. -# You can build targets in this file by including this sub makefile like: -# $ make -f tensorflow/contrib/makefile/Makefile TARGET= \ -# SUB_MAKEFILES=\ -# $(pwd)/tensorflow/contrib/makefile/sub_makefiles/quantization/Makefile.in \ -# (optional: NDK_ROOT=) contrib_quantization_tests -# TODO(satok): Support more targets - -GTEST_DIR := \ -$(MAKEFILE_DIR)/downloads/googletest/googletest - -GTEST_HEADERS = \ -$(wildcard $(GTEST_DIR)/include/gtest/*.h) \ -$(wildcard $(GTEST_DIR)/include/gtest/internal/*.h) - -GTEST_SRCS := \ -$(wildcard $(GTEST_DIR)/src/*.cc) \ -$(wildcard $(GTEST_DIR)/src/*.h) \ -$(GTEST_HEADERS) - -# CAVEAT: We should disable TENSORFLOW_DISABLE_META while running -# quantized_matmul on Android because it crashes in -# MultiThreadGemm in tensorflow/core/kernels/meta_support.cc -# TODO(satok): Remove once it's fixed -CXXFLAGS += -DTENSORFLOW_DISABLE_META - -GRAPH_TRANSFER_SRCS := \ -tensorflow/cc/framework/scope.cc \ -tensorflow/cc/framework/ops.cc \ -tensorflow/cc/ops/const_op.cc \ -tensorflow/core/kernels/hexagon/graph_transfer_utils.cc \ -tensorflow/core/kernels/hexagon/graph_transferer.cc \ -tensorflow/core/kernels/hexagon/graph_transferer_test.cc \ -tensorflow/core/kernels/hexagon/hexagon_control_wrapper.cc \ -tensorflow/core/kernels/hexagon/hexagon_ops_definitions.cc \ -tensorflow/core/kernels/remote_fused_graph_execute_op.cc \ -tensorflow/core/kernels/remote_fused_graph_execute_utils.cc \ -tensorflow/core/ops/remote_fused_graph_ops.cc \ -tensorflow/core/platform/posix/test.cc - -QUANTIZATION_TEST_SRCS := \ -$(GRAPH_TRANSFER_SRCS) \ -tensorflow/core/kernels/hexagon/graph_transferer_test.cc \ -tensorflow/contrib/makefile/test/test_main.cc - -QUANTIZATION_TEST_OBJS := $(addprefix $(OBJDIR), $(QUANTIZATION_TEST_SRCS:.cc=.o)) - -QUANTIZATION_TEST_NAME := quantization_tests -QUANTIZATION_TEST_BIN_PATH := $(BINDIR)$(QUANTIZATION_TEST_NAME) - -INCLUDES += \ --I$(MAKEFILE_DIR)/downloads/gemmlowp \ --I$(MAKEFILE_DIR)/downloads/googletest/googletest/include - -QUANTIZATION_TEST_INCLUDES := $(INCLUDES) - -$(OBJDIR)gtest-all.o: $(GTEST_SRCS) $(QUANTIZATION_TEST_OBJS) - $(CXX) $(CXXFLAGS) $(QUANTIZATION_TEST_INCLUDES) -I $(GTEST_DIR) -c \ - $(GTEST_DIR)/src/gtest-all.cc -o $@ - -$(LIBDIR)gtest.a: $(OBJDIR)gtest-all.o - @mkdir -p $(dir $@) - $(AR) $(ARFLAGS) $@ $^ - -$(QUANTIZATION_TEST_BIN_PATH): $(LIB_PATH) $(LIBDIR)gtest.a $(QUANTIZATION_TEST_OBJS) - @mkdir -p $(dir $@) - $(CXX) $(CXXFLAGS) $(QUANTIZATION_TEST_INCLUDES) \ - -o $(QUANTIZATION_TEST_BIN_PATH) $(QUANTIZATION_TEST_OBJS) \ - $(LIBFLAGS) $(LIB_PATH) $(LIBDIR)gtest.a $(LDFLAGS) $(LIBS) - -$(QUANTIZATION_TEST_NAME): $(QUANTIZATION_TEST_BIN_PATH) diff --git a/tensorflow/contrib/makefile/sub_makefiles/so/Makefile.in b/tensorflow/contrib/makefile/sub_makefiles/so/Makefile.in deleted file mode 100644 index a20fbcd8fa2..00000000000 --- a/tensorflow/contrib/makefile/sub_makefiles/so/Makefile.in +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# This sub Makefile compiles libraries under this directory. This is designed to -# be used as a sub Makefile with tensorflow/contrib/makefile/Makefile. -# You can build targets in this file by including this sub makefile like: -# $ make -f tensorflow/contrib/makefile/Makefile TARGET= \ -# SUB_MAKEFILES=\ -# $(pwd)/tensorflow/contrib/makefile/sub_makefiles/so/Makefile.in \ -# (optional: NDK_ROOT=) tensorflow-core.so - -SO_NAME := tensorflow-core.so -SO_PATH := $(LIBDIR)$(SO_NAME) - -$(SO_PATH): $(LIB_OBJS) - @mkdir -p $(dir $@) - $(CXX) $(CXXFLAGS) $(INCLUDES) \ - -o $@ $(LIB_OBJS) \ - $(LIBFLAGS) $(LDFLAGS) -shared $(LIBS) - -$(SO_NAME): $(SO_PATH) diff --git a/tensorflow/contrib/makefile/test/test_main.cc b/tensorflow/contrib/makefile/test/test_main.cc deleted file mode 100644 index 0432faa9154..00000000000 --- a/tensorflow/contrib/makefile/test/test_main.cc +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// A program with a main that is suitable for unittests -// This is designed to be used unittests built by Makefile - -#include -#include "tensorflow/core/platform/test.h" - -GTEST_API_ int main(int argc, char** argv) { - std::cout << "Running main() from test_main.cc" << std::endl; - testing::InitGoogleTest(&argc, argv); - - return RUN_ALL_TESTS(); -} diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt deleted file mode 100644 index 299db8bc871..00000000000 --- a/tensorflow/contrib/makefile/tf_op_files.txt +++ /dev/null @@ -1,383 +0,0 @@ -tensorflow/c/kernels/bitcast_op.cc -tensorflow/contrib/boosted_trees/ops/model_ops.cc -tensorflow/contrib/boosted_trees/ops/prediction_ops.cc -tensorflow/contrib/boosted_trees/ops/quantile_ops.cc -tensorflow/contrib/boosted_trees/ops/split_handler_ops.cc -tensorflow/contrib/boosted_trees/ops/stats_accumulator_ops.cc -tensorflow/contrib/boosted_trees/ops/training_ops.cc -tensorflow/core/kernels/aggregate_ops.cc -tensorflow/core/kernels/argmax_op.cc -tensorflow/core/kernels/avgpooling_op.cc -tensorflow/core/kernels/batch_matmul_op_real.cc -tensorflow/core/kernels/batch_norm_op.cc -tensorflow/core/kernels/batchtospace_op.cc -tensorflow/core/kernels/bcast_ops.cc -tensorflow/core/kernels/bias_op.cc -tensorflow/core/kernels/boosted_trees/prediction_ops.cc -tensorflow/core/kernels/boosted_trees/resource_ops.cc -tensorflow/core/kernels/boosted_trees/resources.cc -tensorflow/core/kernels/boosted_trees/stats_ops.cc -tensorflow/core/kernels/boosted_trees/training_ops.cc -tensorflow/core/kernels/cast_op.cc -tensorflow/core/kernels/cast_op_impl_bfloat.cc -tensorflow/core/kernels/cast_op_impl_bool.cc -tensorflow/core/kernels/cast_op_impl_complex128.cc -tensorflow/core/kernels/cast_op_impl_complex64.cc -tensorflow/core/kernels/cast_op_impl_double.cc -tensorflow/core/kernels/cast_op_impl_float.cc -tensorflow/core/kernels/cast_op_impl_half.cc -tensorflow/core/kernels/cast_op_impl_int16.cc -tensorflow/core/kernels/cast_op_impl_int32.cc -tensorflow/core/kernels/cast_op_impl_int64.cc -tensorflow/core/kernels/cast_op_impl_int8.cc -tensorflow/core/kernels/cast_op_impl_uint16.cc -tensorflow/core/kernels/cast_op_impl_uint32.cc -tensorflow/core/kernels/cast_op_impl_uint64.cc -tensorflow/core/kernels/cast_op_impl_uint8.cc -tensorflow/core/kernels/check_numerics_op.cc -tensorflow/core/kernels/concat_lib_cpu.cc -tensorflow/core/kernels/concat_op.cc -tensorflow/core/kernels/constant_op.cc -tensorflow/core/kernels/control_flow_ops.cc -tensorflow/core/kernels/conv_grad_filter_ops.cc -tensorflow/core/kernels/conv_grad_input_ops.cc -tensorflow/core/kernels/conv_grad_ops.cc -tensorflow/core/kernels/conv_ops.cc -tensorflow/core/kernels/conv_ops_3d.cc -tensorflow/core/kernels/conv_ops_fused_double.cc -tensorflow/core/kernels/conv_ops_fused_float.cc -tensorflow/core/kernels/conv_ops_fused_half.cc -tensorflow/core/kernels/conv_ops_using_gemm.cc -tensorflow/core/kernels/crop_and_resize_op.cc -tensorflow/core/kernels/ctc_decoder_ops.cc -tensorflow/core/kernels/cwise_op_abs.cc -tensorflow/core/kernels/cwise_op_add_1.cc -tensorflow/core/kernels/cwise_op_add_2.cc -tensorflow/core/kernels/cwise_op_bitwise_and.cc -tensorflow/core/kernels/cwise_op_bitwise_or.cc -tensorflow/core/kernels/cwise_op_bitwise_xor.cc -tensorflow/core/kernels/cwise_op_cos.cc -tensorflow/core/kernels/cwise_op_cosh.cc -tensorflow/core/kernels/cwise_op_div.cc -tensorflow/core/kernels/cwise_op_equal_to_1.cc -tensorflow/core/kernels/cwise_op_equal_to_2.cc -tensorflow/core/kernels/cwise_op_erf.cc -tensorflow/core/kernels/cwise_op_exp.cc -tensorflow/core/kernels/cwise_op_floor.cc -tensorflow/core/kernels/cwise_op_floor_div.cc -tensorflow/core/kernels/cwise_op_floor_mod.cc -tensorflow/core/kernels/cwise_op_greater.cc -tensorflow/core/kernels/cwise_op_greater_equal.cc -tensorflow/core/kernels/cwise_op_invert.cc -tensorflow/core/kernels/cwise_op_isfinite.cc -tensorflow/core/kernels/cwise_op_isnan.cc -tensorflow/core/kernels/cwise_op_left_shift.cc -tensorflow/core/kernels/cwise_op_less.cc -tensorflow/core/kernels/cwise_op_less_equal.cc -tensorflow/core/kernels/cwise_op_log.cc -tensorflow/core/kernels/cwise_op_logical_and.cc -tensorflow/core/kernels/cwise_op_logical_not.cc -tensorflow/core/kernels/cwise_op_logical_or.cc -tensorflow/core/kernels/cwise_op_maximum.cc -tensorflow/core/kernels/cwise_op_minimum.cc -tensorflow/core/kernels/cwise_op_mul_1.cc -tensorflow/core/kernels/cwise_op_mul_2.cc -tensorflow/core/kernels/cwise_op_neg.cc -tensorflow/core/kernels/cwise_op_not_equal_to_1.cc -tensorflow/core/kernels/cwise_op_not_equal_to_2.cc -tensorflow/core/kernels/cwise_op_pow.cc -tensorflow/core/kernels/cwise_op_reciprocal.cc -tensorflow/core/kernels/cwise_op_right_shift.cc -tensorflow/core/kernels/cwise_op_round.cc -tensorflow/core/kernels/cwise_op_rsqrt.cc -tensorflow/core/kernels/cwise_op_select.cc -tensorflow/core/kernels/cwise_op_sigmoid.cc -tensorflow/core/kernels/cwise_op_sign.cc -tensorflow/core/kernels/cwise_op_sin.cc -tensorflow/core/kernels/cwise_op_sinh.cc -tensorflow/core/kernels/cwise_op_sqrt.cc -tensorflow/core/kernels/cwise_op_square.cc -tensorflow/core/kernels/cwise_op_squared_difference.cc -tensorflow/core/kernels/cwise_op_sub.cc -tensorflow/core/kernels/cwise_op_tan.cc -tensorflow/core/kernels/cwise_op_tanh.cc -tensorflow/core/kernels/cwise_op_xdivy.cc -tensorflow/core/kernels/cwise_op_xlogy.cc -tensorflow/core/kernels/cwise_ops_common.cc -tensorflow/core/kernels/data_format_ops.cc -tensorflow/core/kernels/decode_bmp_op.cc -tensorflow/core/kernels/decode_proto_op.cc -tensorflow/core/kernels/decode_wav_op.cc -tensorflow/core/kernels/deep_conv2d.cc -tensorflow/core/kernels/dense_update_functor.cc -tensorflow/core/kernels/dense_update_ops.cc -tensorflow/core/kernels/depthtospace_op.cc -tensorflow/core/kernels/depthwise_conv_op.cc -tensorflow/core/kernels/dequantize_op.cc -tensorflow/core/kernels/dynamic_partition_op.cc -tensorflow/core/kernels/dynamic_stitch_op.cc -tensorflow/core/kernels/einsum_op_impl_half.cc -tensorflow/core/kernels/einsum_op_impl_bfloat16.cc -tensorflow/core/kernels/einsum_op_impl_int32.cc -tensorflow/core/kernels/einsum_op_impl_int64.cc -tensorflow/core/kernels/einsum_op_impl_float.cc -tensorflow/core/kernels/einsum_op_impl_double.cc -tensorflow/core/kernels/einsum_op_impl_complex64.cc -tensorflow/core/kernels/einsum_op_impl_complex128.cc -tensorflow/core/kernels/encode_proto_op.cc -tensorflow/core/kernels/encode_wav_op.cc -tensorflow/core/kernels/example_parsing_ops.cc -tensorflow/core/kernels/fake_quant_ops.cc -tensorflow/core/kernels/fifo_queue.cc -tensorflow/core/kernels/fifo_queue_op.cc -tensorflow/core/kernels/fill_functor.cc -tensorflow/core/kernels/fft_ops.cc -tensorflow/core/kernels/function_ops.cc -tensorflow/core/kernels/fused_batch_norm_op.cc -tensorflow/core/kernels/fused_eigen_output_kernels.cc -tensorflow/core/kernels/gather_functor.cc -tensorflow/core/kernels/gather_functor_batched.cc -tensorflow/core/kernels/gather_nd_op.cc -tensorflow/core/kernels/gather_nd_op_cpu_impl_0.cc -tensorflow/core/kernels/gather_nd_op_cpu_impl_1.cc -tensorflow/core/kernels/gather_nd_op_cpu_impl_2.cc -tensorflow/core/kernels/gather_nd_op_cpu_impl_3.cc -tensorflow/core/kernels/gather_nd_op_cpu_impl_4.cc -tensorflow/core/kernels/gather_nd_op_cpu_impl_5.cc -tensorflow/core/kernels/gather_nd_op_cpu_impl_6.cc -tensorflow/core/kernels/gather_nd_op_cpu_impl_7.cc -tensorflow/core/kernels/gather_op.cc -tensorflow/core/kernels/identity_n_op.cc -tensorflow/core/kernels/identity_op.cc -tensorflow/core/kernels/immutable_constant_op.cc -tensorflow/core/kernels/in_topk_op.cc -tensorflow/core/kernels/initializable_lookup_table.c -tensorflow/core/kernels/inplace_ops.cc -tensorflow/core/kernels/listdiff_op.cc -tensorflow/core/kernels/logging_ops.cc -tensorflow/core/kernels/lookup_table_init_op.cc -tensorflow/core/kernels/lookup_table_op.cc -tensorflow/core/kernels/lookup_util.cc -tensorflow/core/kernels/lrn_op.cc -tensorflow/core/kernels/matmul_op.cc -tensorflow/core/kernels/matrix_diag_op.cc -tensorflow/core/kernels/matrix_set_diag_op.cc -tensorflow/core/kernels/maxpooling_op.cc -tensorflow/core/kernels/meta_support.cc -tensorflow/core/kernels/mfcc.cc -tensorflow/core/kernels/mfcc_dct.cc -tensorflow/core/kernels/mfcc_mel_filterbank.cc -tensorflow/core/kernels/mfcc_op.cc -tensorflow/core/kernels/mirror_pad_op.cc -tensorflow/core/kernels/mirror_pad_op_cpu_impl_1.cc -tensorflow/core/kernels/mirror_pad_op_cpu_impl_2.cc -tensorflow/core/kernels/mirror_pad_op_cpu_impl_3.cc -tensorflow/core/kernels/mirror_pad_op_cpu_impl_4.cc -tensorflow/core/kernels/mirror_pad_op_cpu_impl_5.cc -tensorflow/core/kernels/multinomial_op.cc -tensorflow/core/kernels/no_op.cc -tensorflow/core/kernels/non_max_suppression_op.cc -tensorflow/core/kernels/one_hot_op.cc -tensorflow/core/kernels/pack_op.cc -tensorflow/core/kernels/pad_op.cc -tensorflow/core/kernels/padding_fifo_queue.cc -tensorflow/core/kernels/padding_fifo_queue_op.cc -tensorflow/core/kernels/pooling_ops_3d.cc -tensorflow/core/kernels/pooling_ops_common.cc -tensorflow/core/kernels/population_count_op.cc -tensorflow/core/kernels/quantization_utils.cc -tensorflow/core/kernels/quantize_down_and_shrink_range.cc -tensorflow/core/kernels/quantize_op.cc -tensorflow/core/kernels/quantized_activation_ops.cc -tensorflow/core/kernels/quantized_add_op.cc -tensorflow/core/kernels/quantized_batch_norm_op.cc -tensorflow/core/kernels/quantized_bias_add_op.cc -tensorflow/core/kernels/quantized_concat_op.cc -tensorflow/core/kernels/quantized_conv_ops.cc -tensorflow/core/kernels/quantized_instance_norm.cc -tensorflow/core/kernels/quantized_matmul_op.cc -tensorflow/core/kernels/quantized_mul_op.cc -tensorflow/core/kernels/quantized_pooling_ops.cc -tensorflow/core/kernels/quantized_reshape_op.cc -tensorflow/core/kernels/quantized_resize_bilinear_op.cc -tensorflow/core/kernels/queue_base.cc -tensorflow/core/kernels/queue_op.cc -tensorflow/core/kernels/queue_ops.cc -tensorflow/core/kernels/random_op.cc -tensorflow/core/kernels/reduction_ops_all.cc -tensorflow/core/kernels/reduction_ops_any.cc -tensorflow/core/kernels/reduction_ops_common.cc -tensorflow/core/kernels/reduction_ops_max.cc -tensorflow/core/kernels/reduction_ops_mean.cc -tensorflow/core/kernels/reduction_ops_min.cc -tensorflow/core/kernels/reduction_ops_prod.cc -tensorflow/core/kernels/reduction_ops_sum.cc -tensorflow/core/kernels/relu_op.cc -tensorflow/core/kernels/remote_fused_graph_execute_op.cc -tensorflow/core/kernels/remote_fused_graph_execute_utils.cc -tensorflow/core/kernels/requantization_range_op.cc -tensorflow/core/kernels/requantize.cc -tensorflow/core/kernels/reshape_op.cc -tensorflow/core/kernels/reshape_util.cc -tensorflow/core/kernels/resize_bilinear_op.cc -tensorflow/core/kernels/resize_nearest_neighbor_op.cc -tensorflow/core/kernels/restore_op.cc -tensorflow/core/kernels/reverse_op.cc -tensorflow/core/kernels/reverse_sequence_op.cc -tensorflow/core/kernels/roll_op.cc -tensorflow/core/kernels/save_op.cc -tensorflow/core/kernels/save_restore_tensor.cc -tensorflow/core/kernels/save_restore_v2_ops.cc -tensorflow/core/kernels/scatter_functor.cc -tensorflow/core/kernels/scatter_nd_op.cc -tensorflow/core/kernels/scatter_nd_op_cpu_impl_0.cc -tensorflow/core/kernels/scatter_nd_op_cpu_impl_1.cc -tensorflow/core/kernels/scatter_nd_op_cpu_impl_2.cc -tensorflow/core/kernels/scatter_nd_op_cpu_impl_3.cc -tensorflow/core/kernels/scatter_nd_op_cpu_impl_4.cc -tensorflow/core/kernels/scatter_nd_op_cpu_impl_5.cc -tensorflow/core/kernels/scatter_nd_op_cpu_impl_6.cc -tensorflow/core/kernels/scatter_nd_op_cpu_impl_7.cc -tensorflow/core/kernels/scatter_op.cc -tensorflow/core/kernels/segment_reduction_ops_impl_1.cc -tensorflow/core/kernels/segment_reduction_ops_impl_2.cc -tensorflow/core/kernels/segment_reduction_ops_impl_3.cc -tensorflow/core/kernels/segment_reduction_ops_impl_4.cc -tensorflow/core/kernels/segment_reduction_ops_impl_5.cc -tensorflow/core/kernels/sendrecv_ops.cc -tensorflow/core/kernels/sequence_ops.cc -tensorflow/core/kernels/session_ops.cc -tensorflow/core/kernels/shape_ops.cc -tensorflow/core/kernels/slice_op.cc -tensorflow/core/kernels/slice_op_cpu_impl_1.cc -tensorflow/core/kernels/slice_op_cpu_impl_2.cc -tensorflow/core/kernels/slice_op_cpu_impl_3.cc -tensorflow/core/kernels/slice_op_cpu_impl_4.cc -tensorflow/core/kernels/slice_op_cpu_impl_5.cc -tensorflow/core/kernels/slice_op_cpu_impl_6.cc -tensorflow/core/kernels/slice_op_cpu_impl_7.cc -tensorflow/core/kernels/slice_op_cpu_impl_8.cc -tensorflow/core/kernels/softmax_op.cc -tensorflow/core/kernels/softplus_op.cc -tensorflow/core/kernels/softsign_op.cc -tensorflow/core/kernels/spacetobatch_functor.cc -tensorflow/core/kernels/spacetobatch_op.cc -tensorflow/core/kernels/spacetodepth_op.cc -tensorflow/core/kernels/sparse_fill_empty_rows_op.cc -tensorflow/core/kernels/sparse_matmul_op.cc -tensorflow/core/kernels/sparse_reshape_op.c -tensorflow/core/kernels/sparse_to_dense_op.cc -tensorflow/core/kernels/spectrogram.cc -tensorflow/core/kernels/spectrogram_op.cc -tensorflow/core/kernels/split_lib_cpu.cc -tensorflow/core/kernels/split_op.cc -tensorflow/core/kernels/split_v_op.cc -tensorflow/core/kernels/stack.cc -tensorflow/core/kernels/stack_ops.cc -tensorflow/core/kernels/stateful_random_ops.cc -tensorflow/core/kernels/stateless_random_ops.cc -tensorflow/core/kernels/strided_slice_op.cc -tensorflow/core/kernels/strided_slice_op_inst_0.cc -tensorflow/core/kernels/strided_slice_op_inst_1.cc -tensorflow/core/kernels/strided_slice_op_inst_2.cc -tensorflow/core/kernels/strided_slice_op_inst_3.cc -tensorflow/core/kernels/strided_slice_op_inst_4.cc -tensorflow/core/kernels/strided_slice_op_inst_5.cc -tensorflow/core/kernels/strided_slice_op_inst_6.cc -tensorflow/core/kernels/strided_slice_op_inst_7.cc -tensorflow/core/kernels/strided_slice_op_inst_8.cc -tensorflow/core/kernels/string_join_op.cc -tensorflow/core/kernels/string_util.cc -tensorflow/core/kernels/tensor_array.cc -tensorflow/core/kernels/tensor_array_ops.cc -tensorflow/core/kernels/tile_functor_cpu.cc -tensorflow/core/kernels/tile_ops.cc -tensorflow/core/kernels/tile_ops_cpu_impl_1.cc -tensorflow/core/kernels/tile_ops_cpu_impl_2.cc -tensorflow/core/kernels/tile_ops_cpu_impl_3.cc -tensorflow/core/kernels/tile_ops_cpu_impl_4.cc -tensorflow/core/kernels/tile_ops_cpu_impl_5.cc -tensorflow/core/kernels/tile_ops_cpu_impl_6.cc -tensorflow/core/kernels/tile_ops_cpu_impl_7.cc -tensorflow/core/kernels/topk_op.cc -tensorflow/core/kernels/training_op_helpers.cc -tensorflow/core/kernels/training_ops.cc -tensorflow/core/kernels/transpose_functor_cpu.cc -tensorflow/core/kernels/transpose_op.cc -tensorflow/core/kernels/unique_op.cc -tensorflow/core/kernels/unpack_op.cc -tensorflow/core/kernels/variable_ops.cc -tensorflow/core/kernels/where_op.cc -tensorflow/core/kernels/xent_op.cc -tensorflow/core/kernels/xsmm_conv2d.cc -tensorflow/core/kernels/data/batch_dataset_op.cc -tensorflow/core/kernels/data/cache_dataset_ops.cc -tensorflow/core/kernels/data/cache_ops.cc -tensorflow/core/kernels/data/captured_function.cc -tensorflow/core/kernels/data/concatenate_dataset_op.cc -tensorflow/core/kernels/data/dataset_utils.cc -tensorflow/core/kernels/data/filter_dataset_op.cc -tensorflow/core/kernels/data/flat_map_dataset_op.cc -tensorflow/core/kernels/data/generator_dataset_op.cc -tensorflow/core/kernels/data/interleave_dataset_op.cc -tensorflow/core/kernels/data/iterator_ops.cc -tensorflow/core/kernels/data/map_dataset_op.cc -tensorflow/core/kernels/data/map_defun_op.cc -tensorflow/core/kernels/data/model_dataset_op.cc -tensorflow/core/kernels/data/multi_device_iterator_ops.cc -tensorflow/core/kernels/data/name_utils.cc -tensorflow/core/kernels/data/optional_ops.cc -tensorflow/core/kernels/data/optional_ops.cu.cc -tensorflow/core/kernels/data/padded_batch_dataset_op.cc -tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc -tensorflow/core/kernels/data/parallel_map_dataset_op.cc -tensorflow/core/kernels/data/parallel_map_iterator.cc -tensorflow/core/kernels/data/prefetch_autotuner.cc -tensorflow/core/kernels/data/prefetch_dataset_op.cc -tensorflow/core/kernels/data/random_seed_ops.cc -tensorflow/core/kernels/data/range_dataset_op.cc -tensorflow/core/kernels/data/repeat_dataset_op.cc -tensorflow/core/kernels/data/shard_dataset_op.cc -tensorflow/core/kernels/data/shuffle_dataset_op.cc -tensorflow/core/kernels/data/single_threaded_executor.cc -tensorflow/core/kernels/data/skip_dataset_op.cc -tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc -tensorflow/core/kernels/data/stats_utils.cc -tensorflow/core/kernels/data/take_dataset_op.cc -tensorflow/core/kernels/data/tensor_dataset_op.cc -tensorflow/core/kernels/data/tensor_slice_dataset_op.cc -tensorflow/core/kernels/data/unbounded_thread_pool.cc -tensorflow/core/kernels/data/window_dataset.cc -tensorflow/core/kernels/data/window_dataset_op.cc -tensorflow/core/kernels/data/zip_dataset_op.cc -tensorflow/core/ops/array_grad.cc -tensorflow/core/ops/array_ops.cc -tensorflow/core/ops/audio_ops.cc -tensorflow/core/ops/boosted_trees_ops.cc -tensorflow/core/ops/candidate_sampling_ops.cc -tensorflow/core/ops/control_flow_ops.cc -tensorflow/core/ops/ctc_ops.cc -tensorflow/core/ops/data_flow_ops.cc -tensorflow/core/ops/function_ops.cc -tensorflow/core/ops/functional_grad.cc -tensorflow/core/ops/functional_ops.cc -tensorflow/core/ops/image_ops.cc -tensorflow/core/ops/io_ops.cc -tensorflow/core/ops/linalg_ops.cc -tensorflow/core/ops/logging_ops.cc -tensorflow/core/ops/manip_ops.cc -tensorflow/core/ops/math_grad.cc -tensorflow/core/ops/math_ops.cc -tensorflow/core/ops/nn_grad.cc -tensorflow/core/ops/nn_ops.cc -tensorflow/core/ops/no_op.cc -tensorflow/core/ops/parsing_ops.cc -tensorflow/core/ops/random_grad.cc -tensorflow/core/ops/random_ops.cc -tensorflow/core/ops/remote_fused_graph_ops.cc -tensorflow/core/ops/script_ops.cc -tensorflow/core/ops/sendrecv_ops.cc -tensorflow/core/ops/sparse_ops.cc -tensorflow/core/ops/state_ops.cc -tensorflow/core/ops/string_ops.cc -tensorflow/core/ops/training_ops.cc diff --git a/tensorflow/contrib/makefile/tf_pb_text_files.txt b/tensorflow/contrib/makefile/tf_pb_text_files.txt deleted file mode 100644 index 57c977bed4e..00000000000 --- a/tensorflow/contrib/makefile/tf_pb_text_files.txt +++ /dev/null @@ -1,35 +0,0 @@ -tensorflow/core/example/example.pb_text.cc -tensorflow/core/example/feature.pb_text.cc -tensorflow/core/framework/allocation_description.pb_text.cc -tensorflow/core/framework/api_def.pb_text.cc -tensorflow/core/framework/attr_value.pb_text.cc -tensorflow/core/framework/cost_graph.pb_text.cc -tensorflow/core/framework/device_attributes.pb_text.cc -tensorflow/core/framework/function.pb_text.cc -tensorflow/core/framework/graph.pb_text.cc -tensorflow/core/framework/graph_transfer_info.pb_text.cc -tensorflow/core/framework/kernel_def.pb_text.cc -tensorflow/core/framework/log_memory.pb_text.cc -tensorflow/core/framework/node_def.pb_text.cc -tensorflow/core/framework/op_def.pb_text.cc -tensorflow/core/framework/remote_fused_graph_execute_info.pb_text.cc -tensorflow/core/framework/resource_handle.pb_text.cc -tensorflow/core/framework/step_stats.pb_text.cc -tensorflow/core/framework/summary.pb_text.cc -tensorflow/core/framework/tensor.pb_text.cc -tensorflow/core/framework/tensor_description.pb_text.cc -tensorflow/core/framework/tensor_shape.pb_text.cc -tensorflow/core/framework/tensor_slice.pb_text.cc -tensorflow/core/framework/types.pb_text.cc -tensorflow/core/framework/versions.pb_text.cc -tensorflow/core/lib/core/error_codes.pb_text.cc -tensorflow/core/protobuf/cluster.pb_text.cc -tensorflow/core/protobuf/config.pb_text.cc -tensorflow/core/protobuf/debug.pb_text.cc -tensorflow/core/protobuf/rewriter_config.pb_text.cc -tensorflow/core/protobuf/saver.pb_text.cc -tensorflow/core/protobuf/tensor_bundle.pb_text.cc -tensorflow/core/protobuf/trace_events.pb_text.cc -tensorflow/core/protobuf/verifier_config.pb_text.cc -tensorflow/core/util/memmapped_file_system.pb_text.cc -tensorflow/core/util/saved_tensor_slice.pb_text.cc diff --git a/tensorflow/contrib/makefile/tf_proto_files.txt b/tensorflow/contrib/makefile/tf_proto_files.txt deleted file mode 100644 index 27b60ecd6b5..00000000000 --- a/tensorflow/contrib/makefile/tf_proto_files.txt +++ /dev/null @@ -1,55 +0,0 @@ -tensorflow/contrib/boosted_trees/proto/learner.proto -tensorflow/contrib/boosted_trees/proto/quantiles.proto -tensorflow/contrib/boosted_trees/proto/split_info.proto -tensorflow/contrib/boosted_trees/proto/tree_config.proto -tensorflow/core/example/example.proto -tensorflow/core/example/feature.proto -tensorflow/core/framework/allocation_description.proto -tensorflow/core/framework/api_def.proto -tensorflow/core/framework/attr_value.proto -tensorflow/core/framework/cost_graph.proto -tensorflow/core/framework/device_attributes.proto -tensorflow/core/framework/function.proto -tensorflow/core/framework/graph.proto -tensorflow/core/framework/graph_transfer_info.proto -tensorflow/core/framework/kernel_def.proto -tensorflow/core/framework/log_memory.proto -tensorflow/core/framework/node_def.proto -tensorflow/core/framework/op_def.proto -tensorflow/core/framework/reader_base.proto -tensorflow/core/framework/remote_fused_graph_execute_info.proto -tensorflow/core/framework/resource_handle.proto -tensorflow/core/framework/step_stats.proto -tensorflow/core/framework/summary.proto -tensorflow/core/framework/tensor.proto -tensorflow/core/framework/tensor_description.proto -tensorflow/core/framework/tensor_shape.proto -tensorflow/core/framework/tensor_slice.proto -tensorflow/core/framework/types.proto -tensorflow/core/framework/variable.proto -tensorflow/core/framework/versions.proto -tensorflow/core/grappler/costs/op_performance_data.proto -tensorflow/core/kernels/boosted_trees/boosted_trees.proto -tensorflow/core/lib/core/error_codes.proto -tensorflow/core/protobuf/trackable_object_graph.proto -tensorflow/core/protobuf/cluster.proto -tensorflow/core/protobuf/config.proto -tensorflow/core/protobuf/debug.proto -tensorflow/core/protobuf/eager_service.proto -tensorflow/core/protobuf/device_properties.proto -tensorflow/core/protobuf/meta_graph.proto -tensorflow/core/protobuf/named_tensor.proto -tensorflow/core/protobuf/queue_runner.proto -tensorflow/core/protobuf/remote_tensor_handle.proto -tensorflow/core/protobuf/rewriter_config.proto -tensorflow/core/protobuf/saved_object_graph.proto -tensorflow/core/protobuf/saver.proto -tensorflow/core/protobuf/struct.proto -tensorflow/core/protobuf/tensor_bundle.proto -tensorflow/core/protobuf/tensorflow_server.proto -tensorflow/core/protobuf/trace_events.proto -tensorflow/core/protobuf/verifier_config.proto -tensorflow/core/util/event.proto -tensorflow/core/util/memmapped_file_system.proto -tensorflow/core/util/saved_tensor_slice.proto -tensorflow/core/util/test_log.proto diff --git a/tensorflow/contrib/memory_stats/BUILD b/tensorflow/contrib/memory_stats/BUILD deleted file mode 100644 index 765c93b06e5..00000000000 --- a/tensorflow/contrib/memory_stats/BUILD +++ /dev/null @@ -1,106 +0,0 @@ -# Description: -# Ops that get statistics on memory allocators. - -load("//tensorflow:tensorflow.bzl", "tf_custom_op_library") -load("//tensorflow:tensorflow.bzl", "tf_gen_op_libs") -load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py") -load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_cc") -load("//tensorflow:tensorflow.bzl", "tf_kernel_library") -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_custom_op_library( - name = "python/ops/_memory_stats_ops.so", - srcs = [ - "kernels/memory_stats_ops.cc", - "ops/memory_stats_ops.cc", - ], -) - -tf_kernel_library( - name = "memory_stats_kernels", - srcs = [ - "kernels/memory_stats_ops.cc", - ], - deps = [ - "//tensorflow/core:framework", - ], - alwayslink = 1, -) - -tf_gen_op_libs( - op_lib_names = ["memory_stats_ops"], - deps = [ - "//tensorflow/core:lib", - ], -) - -tf_gen_op_wrapper_py( - name = "memory_stats_ops", - deps = [":memory_stats_ops_op_lib"], -) - -tf_gen_op_wrapper_cc( - name = "memory_stats_ops", - out_ops_file = "memory_stats_ops", -) - -cc_library( - name = "memory_stats_cc", - srcs = ["memory_stats_ops.cc"], - hdrs = ["memory_stats_ops.h"], - visibility = ["//visibility:public"], - deps = [ - ":memory_stats_kernels", - ":memory_stats_ops_op_lib", - "//tensorflow/cc:const_op", - "//tensorflow/cc:ops", - "//tensorflow/cc:scope", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - ], - alwayslink = 1, -) - -tf_custom_op_py_library( - name = "memory_stats_py", - srcs = [ - "__init__.py", - "python/ops/memory_stats_ops.py", - ], - dso = [ - ":python/ops/_memory_stats_ops.so", - ], - kernels = [ - ":memory_stats_kernels", - ":memory_stats_ops_op_lib", - ], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":memory_stats_ops", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:platform", - "//tensorflow/python:util", - ], -) - -cuda_py_test( - name = "memory_stats_ops_test", - size = "small", - srcs = ["python/kernel_tests/memory_stats_ops_test.py"], - additional_deps = [ - ":memory_stats_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - ], - xla_enable_strict_auto_jit = False, -) diff --git a/tensorflow/contrib/memory_stats/__init__.py b/tensorflow/contrib/memory_stats/__init__.py deleted file mode 100644 index 2ce849ca660..00000000000 --- a/tensorflow/contrib/memory_stats/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for memory statistics. - -@@BytesInUse -@@BytesLimit -@@MaxBytesInUse -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.memory_stats.python.ops.memory_stats_ops import BytesInUse -from tensorflow.contrib.memory_stats.python.ops.memory_stats_ops import BytesLimit -from tensorflow.contrib.memory_stats.python.ops.memory_stats_ops import MaxBytesInUse - -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/memory_stats/kernels/memory_stats_ops.cc b/tensorflow/contrib/memory_stats/kernels/memory_stats_ops.cc deleted file mode 100644 index 7ae1dbeaa2d..00000000000 --- a/tensorflow/contrib/memory_stats/kernels/memory_stats_ops.cc +++ /dev/null @@ -1,114 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include "tensorflow/core/framework/op_kernel.h" - -namespace tensorflow { - -// Base class of ops that collects statistics of the allocator of a device. -class MemoryStatsOp : public OpKernel { - public: - explicit MemoryStatsOp(OpKernelConstruction* context) : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - Allocator* allocator = - context->device()->GetAllocator(AllocatorAttributes()); - absl::optional allocator_stats = allocator->GetStats(); - if (!allocator_stats) { - *allocator_stats = AllocatorStats(); - } - - Tensor* output_tensor = nullptr; - OP_REQUIRES_OK( - context, context->allocate_output(0, TensorShape({}), &output_tensor)); - output_tensor->scalar()() = ExtractAllocatorStats(*allocator_stats); - } - - protected: - // Extracts a certain field (determined by subclasses) from an allocator - // stats. - virtual int64 ExtractAllocatorStats( - const AllocatorStats& allocator_stats) const = 0; -}; - -// Op that measures current memory in bytes. -class BytesInUseOp : public MemoryStatsOp { - public: - explicit BytesInUseOp(OpKernelConstruction* context) - : MemoryStatsOp(context) {} - - private: - int64 ExtractAllocatorStats( - const AllocatorStats& allocator_stats) const override { - return allocator_stats.bytes_in_use; - } -}; - -// Register this op on GPU only, see comment for MaxBytesInUse for reason -REGISTER_KERNEL_BUILDER(Name("BytesInUse").Device(DEVICE_GPU).HostMemory("out"), - BytesInUseOp); - -#ifdef TENSORFLOW_USE_SYCL -REGISTER_KERNEL_BUILDER( - Name("BytesInUse").Device(DEVICE_SYCL).HostMemory("out"), BytesInUseOp); -#endif // TENSORFLOW_USE_SYCL - -// Op that measures the total memory (in bytes) of a device. -class BytesLimitOp : public MemoryStatsOp { - public: - explicit BytesLimitOp(OpKernelConstruction* context) - : MemoryStatsOp(context) {} - - private: - int64 ExtractAllocatorStats( - const AllocatorStats& allocator_stats) const override { - return allocator_stats.bytes_limit ? *allocator_stats.bytes_limit : -1; - } -}; - -REGISTER_KERNEL_BUILDER(Name("BytesLimit").Device(DEVICE_CPU), BytesLimitOp); -REGISTER_KERNEL_BUILDER(Name("BytesLimit").Device(DEVICE_GPU).HostMemory("out"), - BytesLimitOp); - -#ifdef TENSORFLOW_USE_SYCL -REGISTER_KERNEL_BUILDER( - Name("BytesLimit").Device(DEVICE_SYCL).HostMemory("out"), BytesLimitOp); -#endif // TENSORFLOW_USE_SYCL - -// Op that measures the peak memory in bytes. -class MaxBytesInUseOp : public MemoryStatsOp { - public: - explicit MaxBytesInUseOp(OpKernelConstruction* context) - : MemoryStatsOp(context) {} - - private: - int64 ExtractAllocatorStats( - const AllocatorStats& allocator_stats) const override { - return allocator_stats.peak_bytes_in_use; - } -}; - -// MallocExtension_GetAllocatedSize doesn't return the allocated size reliably -// for CPU allocators, so we register this op on GPU only. -REGISTER_KERNEL_BUILDER( - Name("MaxBytesInUse").Device(DEVICE_GPU).HostMemory("out"), - MaxBytesInUseOp); - -#ifdef TENSORFLOW_USE_SYCL -REGISTER_KERNEL_BUILDER( - Name("MaxBytesInUse").Device(DEVICE_SYCL).HostMemory("out"), - MaxBytesInUseOp); -#endif // TENSORFLOW_USE_SYCL - -} // namespace tensorflow diff --git a/tensorflow/contrib/memory_stats/ops/memory_stats_ops.cc b/tensorflow/contrib/memory_stats/ops/memory_stats_ops.cc deleted file mode 100644 index 42020cf7f6b..00000000000 --- a/tensorflow/contrib/memory_stats/ops/memory_stats_ops.cc +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" - -namespace tensorflow { - -REGISTER_OP("BytesInUse") - .Output("out: int64") - .SetIsStateful() - .SetShapeFn(shape_inference::ScalarShape); -REGISTER_OP("BytesLimit") - .Output("out: int64") - .SetIsStateful() - .SetShapeFn(shape_inference::ScalarShape); -REGISTER_OP("MaxBytesInUse") - .Output("out: int64") - .SetIsStateful() - .SetShapeFn(shape_inference::ScalarShape); - -} // namespace tensorflow diff --git a/tensorflow/contrib/memory_stats/python/kernel_tests/memory_stats_ops_test.py b/tensorflow/contrib/memory_stats/python/kernel_tests/memory_stats_ops_test.py deleted file mode 100644 index 02c2ac06fb7..00000000000 --- a/tensorflow/contrib/memory_stats/python/kernel_tests/memory_stats_ops_test.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for memory statistics ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.memory_stats.python.ops import memory_stats_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import test_util -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import test - - -class MemoryStatsOpsTest(test_util.TensorFlowTestCase): - - def testBytesLimit(self): - # AllocatorStats.bytes_limit is set to zero for CPU allocators, so we skip - # the check. - if not test.is_gpu_available(): - return - - with self.test_session(use_gpu=True) as sess: - bytes_limit = sess.run(memory_stats_ops.BytesLimit()) - self.assertLess(0, bytes_limit) - - # Tests the peak memory usage of the following computation. - # a b - # | / | - # c | - # \ | - # \ | - # d - # The memory for matrix "a" can be reused for matrix "d". Therefore, this - # computation needs space for only three matrix plus some small overhead. - def testChainOfMatmul(self): - # MaxBytesInUse is registered on GPU only. See kernels/memory_stats_ops.cc. - if not test.is_gpu_available(): - return - - with self.test_session(use_gpu=True) as sess: - matrix_size = 64 - matrix_shape = tensor_shape.TensorShape([matrix_size, matrix_size]) - dtype = dtypes.float32 - matrix_size_in_bytes = matrix_shape.num_elements() * dtype.size - a = random_ops.random_uniform(matrix_shape, dtype=dtype) - b = random_ops.random_uniform(matrix_shape, dtype=dtype) - c = math_ops.matmul(a, b) - d = math_ops.matmul(c, b) - sess.run(d) - - max_bytes_in_use_op = memory_stats_ops.MaxBytesInUse() - max_bytes_in_use = sess.run(max_bytes_in_use_op) - self.assertGreaterEqual(max_bytes_in_use, matrix_size_in_bytes * 3) - self.assertLess(max_bytes_in_use, matrix_size_in_bytes * 4) - - # run chain with 2 ops, make sure BytesInUse captures intermediate - # memory usage - a = random_ops.random_uniform(matrix_shape, dtype=dtype) - with ops.control_dependencies([a]): - bytes_in_use_op = memory_stats_ops.BytesInUse() - with ops.control_dependencies([bytes_in_use_op]): - b = random_ops.random_uniform(matrix_shape, dtype=dtype) - c = math_ops.matmul(a, b) - - _, bytes_in_use, max_bytes_in_use = sess.run([c, bytes_in_use_op, - max_bytes_in_use_op]) - - # intermediate result allocates 1 matrix, max usage is at least 2 - self.assertGreaterEqual(bytes_in_use, matrix_size_in_bytes * 1) - self.assertLess(bytes_in_use, matrix_size_in_bytes * 2) - - # max usage is still 3 because it reflects maxium from previous .run call - self.assertGreaterEqual(max_bytes_in_use, matrix_size_in_bytes * 3) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/memory_stats/python/ops/memory_stats_ops.py b/tensorflow/contrib/memory_stats/python/ops/memory_stats_ops.py deleted file mode 100644 index c0f7788c1c6..00000000000 --- a/tensorflow/contrib/memory_stats/python/ops/memory_stats_ops.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for memory statistics.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.memory_stats.ops import gen_memory_stats_ops -from tensorflow.contrib.util import loader -from tensorflow.python.platform import resource_loader - -_memory_stats_ops_so = loader.load_op_library( - resource_loader.get_path_to_datafile("_memory_stats_ops.so")) - - -def BytesInUse(): - """Generates an op that computes the current memory of a device.""" - return gen_memory_stats_ops.bytes_in_use() - - -def BytesLimit(): - """Generates an op that measures the total memory (in bytes) of a device.""" - return gen_memory_stats_ops.bytes_limit() - - -def MaxBytesInUse(): - """Generates an op that computes the peak memory of a device.""" - return gen_memory_stats_ops.max_bytes_in_use() diff --git a/tensorflow/contrib/meta_graph_transform/BUILD b/tensorflow/contrib/meta_graph_transform/BUILD deleted file mode 100644 index f3d2202db49..00000000000 --- a/tensorflow/contrib/meta_graph_transform/BUILD +++ /dev/null @@ -1,63 +0,0 @@ -# Description: -# Utility for applying the Graph Transform tool to a MetaGraphDef. - -load( - "//tensorflow:tensorflow.bzl", - "py_test", -) - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "meta_graph_transform", - srcs = [ - "__init__.py", - "meta_graph_transform.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/core:protos_all_py", - "//tensorflow/python:framework", - "//tensorflow/python:framework_ops", - "//tensorflow/python:graph_util", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python/saved_model:constants", - "//tensorflow/tools/graph_transforms:transform_graph_py", - ], -) - -py_test( - name = "meta_graph_transform_test", - size = "small", - srcs = ["meta_graph_transform_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - visibility = ["//visibility:private"], - deps = [ - ":meta_graph_transform", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python/saved_model:constants", - "//tensorflow/tools/graph_transforms:transform_graph_py", - ], -) - -filegroup( - name = "py_srcs", - data = glob([ - "**/*.py", - ]), -) diff --git a/tensorflow/contrib/meta_graph_transform/__init__.py b/tensorflow/contrib/meta_graph_transform/__init__.py deleted file mode 100644 index a58ac243a58..00000000000 --- a/tensorflow/contrib/meta_graph_transform/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Utility for applying the Graph Transform tool to a MetaGraphDef.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.meta_graph_transform import meta_graph_transform -from tensorflow.python.util.all_util import remove_undocumented - - -_allowed_symbols = ['meta_graph_transform'] - -remove_undocumented(__name__, allowed_exception_list=_allowed_symbols) diff --git a/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py b/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py deleted file mode 100644 index b1c852c2c69..00000000000 --- a/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py +++ /dev/null @@ -1,781 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Apply graph_transforms tool to MetaGraphDefs. - -@@meta_graph_transform -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -import re as _re - -from tensorflow.core.framework import graph_pb2 as _graph_pb2 -from tensorflow.core.protobuf import meta_graph_pb2 as _meta_graph_pb2 -from tensorflow.python.client import session as _session -from tensorflow.python.framework import graph_util as _graph_util -from tensorflow.python.framework import importer as _importer -from tensorflow.python.framework import ops as _ops -from tensorflow.python.platform import tf_logging as _logging -from tensorflow.python.saved_model import constants as _saved_model_constants -from tensorflow.python.training import saver as _saver_lib -from tensorflow.python.util import compat as _compat -from tensorflow.tools import graph_transforms as _graph_transforms - - -_FREEZE_GRAPH_TRANSFORM = 'freeze_graph' -_SPARSIFY_GATHER_TRANSFORM = 'sparsify_gather' - - -def _op_name(tensor_name): - """Get the op name from a tensor name.""" - # control dependency inputs start with ^ - if tensor_name[0] == '^': - tensor_name = tensor_name[1:] - if ':' in tensor_name: - op_name, _ = tensor_name.split(':') - return op_name - return tensor_name - - -def _get_shared_init_op(initializer_names): - """Obtain the shared init op name, if it exists. - - Args: - initializer_names: Dictionary of the "infrastructural" nodes (initializers, - save and restore ops, etc.). The keys in this dictionary - indicate the collection where these nodes were obtained from. - - Returns: - A string indicating the shared init op name or none if None if none exists. - """ - return_value = initializer_names.get(_saved_model_constants.MAIN_OP_KEY, None) - if not return_value: - return_value = initializer_names.get( - _saved_model_constants.LEGACY_INIT_OP_KEY, None) - return str(return_value[0]) if return_value else None - - -def _gtt_transforms(graph_def, input_names, output_names, initializer_names, - transforms): - """Pass through gtt transforms, applying them to the graph_def. - - Args: - graph_def: A GraphDef proto to be transformed. - input_names: Names of input nodes. - output_names: Names of output nodes. - initializer_names: Dictionary of the "infrastructural" nodes (initializers, - save and restore ops, etc.) that should be retained even if they are not - transitively reachable from output nodes. The keys in this dictionary - indicate the collection where these nodes were obtained from. - transforms: A list of strings naming the graph transforms to be applied in - order. - Returns: - The transformed GraphDef. - """ - if not transforms: - transformed_graph_def = _graph_pb2.GraphDef() - transformed_graph_def.CopyFrom(graph_def) - return transformed_graph_def - - initializer_names_flat = sorted( - [k for l in initializer_names.values() for k in l]) - all_output_names = output_names + initializer_names_flat - return _graph_transforms.TransformGraph(graph_def, input_names, - all_output_names, transforms) - - -def _freeze_transform(graph_def, output_names, initializer_names, saver_def, - checkpoint_path): - """Handle the freeze transform. - - Determine which initializer nodes should be retained by the freeze transform. - Retain those nodes and return an updated dictionary containing them. - - Args: - graph_def: A GraphDef proto to be transformed. - output_names: Names of output nodes. - initializer_names: Dictionary of the "infrastructural" nodes (initializers, - save and restore ops, etc.). The keys in this dictionary - indicate the collection where these nodes were obtained from. - saver_def: A SaverDef proto used for restoring a checkpoint during freezing, - if needed (default None). - checkpoint_path: A path to a checkpoint to restore during freezing, - if needed (default None). - - Returns: - A tuple containing the GraphDef and a Dict of pruned initializer nodes. - """ - table_initializers = initializer_names.get(_ops.GraphKeys.TABLE_INITIALIZERS, - []) - shared_init_op = _get_shared_init_op(initializer_names) - - graph_def = _freeze_graph_with_def_protos(graph_def, output_names, - table_initializers, shared_init_op, - saver_def, checkpoint_path) - pruned_initializer_names = {} - # Freeze graph prunes all initializers and shared init nodes that are not - # explicitly maintained. Create new initializer_names dictionary to reflect - # this. - if table_initializers: - pruned_initializer_names[_ops.GraphKeys.TABLE_INITIALIZERS] = ( - table_initializers) - if _saved_model_constants.LEGACY_INIT_OP_KEY in initializer_names: - pruned_initializer_names[_saved_model_constants.LEGACY_INIT_OP_KEY] = ( - initializer_names[_saved_model_constants.LEGACY_INIT_OP_KEY]) - if _saved_model_constants.MAIN_OP_KEY in initializer_names: - pruned_initializer_names[_saved_model_constants.MAIN_OP_KEY] = ( - initializer_names[_saved_model_constants.MAIN_OP_KEY]) - return (graph_def, pruned_initializer_names) - - -def _clean_save_and_restore(graph_def, op, removed_op_names): - """Clean the specified save and restore op. - - Updates the dtypes attribute of the save / restore op and the associated name - and shape tensors to remove entries for variables that have been removed. - - Args: - graph_def: A GraphDef proto to be transformed. - op: The save or restore op to update. - removed_op_names: List of op names that have been removed. - """ - name = op.name + '/tensor_names' - shape = op.name + '/shape_and_slices' - name_op = _find_op(graph_def, name) - shape_op = _find_op(graph_def, shape) - name_op_value_tensor = name_op.attr['value'].tensor - shape_op_value_tensor = shape_op.attr['value'].tensor - names = [] - shapes = [] - dtypes = [] - for index, value in enumerate(name_op_value_tensor.string_val): - if not _is_removed(_compat.as_str(value), removed_op_names): - names.append(value) - shapes.append(shape_op_value_tensor.string_val[index]) - dtypes.append(op.attr['dtypes'].list.type[index]) - name_op_value_tensor.string_val[:] = names - name_op_value_tensor.tensor_shape.dim[0].size = len(names) - shape_op_value_tensor.string_val[:] = shapes - shape_op_value_tensor.tensor_shape.dim[0].size = len(shapes) - op.attr['dtypes'].list.type[:] = dtypes - - if not name_op.attr['_output_shapes'].list.shape: - name_op.attr['_output_shapes'].list.shape.add() - name_op.attr['_output_shapes'].list.shape[0].dim.add() - name_op.attr['_output_shapes'].list.shape[0].dim[0].size = len(names) - - if not shape_op.attr['_output_shapes'].list.shape: - shape_op.attr['_output_shapes'].list.shape.add() - shape_op.attr['_output_shapes'].list.shape[0].dim.add() - shape_op.attr['_output_shapes'].list.shape[0].dim[0].size = len(shapes) - - -def _sparsify_gather_transform(graph_def, input_names, output_names, - initializer_names, checkpoint_path): - """Handle the sparsify gather transform. - - Provides the transform the checkpoint and keeps track of the newly created - initializer nodes. - - Args: - graph_def: A GraphDef proto to be transformed. - input_names: Names of input nodes. - output_names: Names of output nodes. - initializer_names: Dictionary of the "infrastructural" nodes (initializers, - save and restore ops, etc.). The keys in this dictionary - indicate the collection where these nodes were obtained from. - checkpoint_path: A path to a checkpoint. - - Returns: - A tuple containing the GraphDef and a Dict of updated initializer nodes. - Raises: - ValueError: if the restore_op_name does not have the expected format. - """ - # Ensure that sparsify_shared_init_op is unique. - sparsify_shared_init_op = 'sparify_gather_init_op' - while _find_op(graph_def, sparsify_shared_init_op): - sparsify_shared_init_op += '_1' - - input_flag = '' - if checkpoint_path: - input_flag = 'input_checkpoint="%s", ' % checkpoint_path - - sparsify_cmd = [ - 'sparsify_gather(%sgroup_init_node="%s")' % (input_flag, - sparsify_shared_init_op) - ] - - starting_op_names = [node.name for node in graph_def.node] - - graph_def = _gtt_transforms(graph_def, input_names, output_names, - initializer_names, sparsify_cmd) - ending_op_names = [node.name for node in graph_def.node] - removed_op_names = list(set(starting_op_names) - set(ending_op_names)) - removed_op_names.sort() - - for op_index, op_name in enumerate(removed_op_names): - op_name_parts = op_name.rsplit('/', 1) - # Remove part to get the checkpoint names used by the saver. - if len(op_name_parts) == 2 and op_name_parts[1].startswith('part_'): - removed_op_names[op_index] = op_name_parts[0] - else: - removed_op_names[op_index] = op_name - - # Obtain newly created table inits from gtt sparsify transform. - added_table_inits = [] - for index, node in enumerate(graph_def.node): - if node.name == sparsify_shared_init_op: - added_table_inits = [n.lstrip('^') for n in node.input] - - table_initializers = initializer_names.get( - _ops.GraphKeys.TABLE_INITIALIZERS, []) - table_initializers.extend(added_table_inits) - initializer_names[_ops.GraphKeys.TABLE_INITIALIZERS] = table_initializers - - del graph_def.node[index] - break - - # Add inits to existing shared init op. - node = _find_op(graph_def, _get_shared_init_op(initializer_names)) - for init in added_table_inits: - node.input.append('^' + init) - - # Update saver. - for node in graph_def.node: - if node.name.endswith('SaveV2'): - _clean_save_and_restore(graph_def, node, removed_op_names) - - return (graph_def, initializer_names) - - -def _do_transforms(graph_def, - input_names, - output_names, - initializer_names, - transforms, - saver_def=None, - checkpoint_path=None): - """Apply requested transforms to a GraphDef, including freezing. - - Args: - graph_def: A GraphDef proto to be transformed. - input_names: Names of input nodes. - output_names: Names of output nodes. - initializer_names: Dictionary of the "infrastructural" nodes (initializers, - save and restore ops, etc.) that should be retained even if they are not - transitively reachable from output nodes. The keys in this dictionary - indicate the collection where these nodes were obtained from. - transforms: A list of strings naming the graph transforms to be applied in - order. These transform names are exactly those supported by the Graph - Transform Tool, with the addition of the 'freeze_graph' and - 'sparsify_gather' transforms. - saver_def: A SaverDef proto used for restoring a checkpoint during freezing, - if needed (default None). - checkpoint_path: A path to a checkpoint to restore during freezing, - if needed (default None). - Returns: - A tuple containing the GraphDef and a Dict of updated initializer nodes. - """ - transformed_graph_def = _graph_pb2.GraphDef() - transformed_graph_def.CopyFrom(graph_def) - transformed_initializer_names = initializer_names.copy() - - if not transforms: - return transformed_graph_def, transformed_initializer_names - - current_gtt_transforms = [] - for t in transforms: - if t == _FREEZE_GRAPH_TRANSFORM: - transformed_graph_def = _gtt_transforms( - transformed_graph_def, input_names, output_names, - transformed_initializer_names, current_gtt_transforms) - output_node_names = [_op_name(x) for x in output_names] - transformed_graph_def, transformed_initializer_names = _freeze_transform( - transformed_graph_def, output_node_names, - transformed_initializer_names, saver_def, checkpoint_path) - current_gtt_transforms = [] - elif t == _SPARSIFY_GATHER_TRANSFORM: - transformed_graph_def = _gtt_transforms( - transformed_graph_def, input_names, output_names, - transformed_initializer_names, current_gtt_transforms) - transformed_graph_def, transformed_initializer_names = ( - _sparsify_gather_transform( - transformed_graph_def, input_names, output_names, - transformed_initializer_names, checkpoint_path)) - current_gtt_transforms = [] - else: - current_gtt_transforms.append(t) - - transformed_graph_def = _gtt_transforms( - transformed_graph_def, input_names, output_names, - transformed_initializer_names, current_gtt_transforms) - return transformed_graph_def, transformed_initializer_names - - -def _connect_to_shared_init_op(graph_def, shared_init_op_name, - nodes_to_connect): - """Creates a new shared init node that is connected to via control deps. - - Args: - graph_def: The GraphDef proto to add the shared init node to. - shared_init_op_name: A string specifying the name of the shared init node to - create. - nodes_to_connect: A list of strings specifying the names of nodes to connect - to the shared node via control dependencies. - """ - if nodes_to_connect: - init_op = graph_def.node.add() - init_op.name = shared_init_op_name - init_op.op = 'NoOp' - init_op.input.extend(['^' + i for i in nodes_to_connect]) - - -# forked and modified from freeze_graph.py -def _freeze_graph_with_def_protos(input_graph_def, output_node_names, - initializer_names, shared_init_op_name, - input_saver_def, input_checkpoint): - """Converts all variables in a graph and checkpoint into constants. - - During this process, we need to retain certain initializer nodes (e.g. table - initializer nodes). Instead of determining which dependencies - of the shared initializer node (e.g. group_deps) to keep, we - reconstruct the connections between the individual initializer nodes and - the shared node after freezing the graph. - - Args: - input_graph_def: A GraphDef proto to be frozen. - output_node_names: Names of output nodes. - initializer_names: Names of initializer nodes to keep. - shared_init_op_name: The name of the shared initializer node to connect the - nodes in initializer names to. - input_saver_def: A SaverDef proto used for restoring a checkpoint. - input_checkpoint: A path to a checkpoint to restore. - - Returns: - A frozen GraphDef. - """ - - with _ops.Graph().as_default(): - _ = _importer.import_graph_def(input_graph_def, name='') - - with _session.Session() as sess: - saver = _saver_lib.Saver(saver_def=input_saver_def) - saver.restore(sess, input_checkpoint) - output_graph_def = _graph_util.convert_variables_to_constants( - sess, input_graph_def, output_node_names + initializer_names) - _connect_to_shared_init_op(output_graph_def, shared_init_op_name, - initializer_names) - return output_graph_def - - -def _find_all_mandatory_retain_ops(base_meta_graph_def): - """Identify all infrastructural Ops, to ensure that they are retained. - - We need to retain infrastructural Ops (init and saver stuff), in addition - to the desired outputs. - - For now we retain *all* save and restore ops, variable initializers, - table initializers, and main init ops. - This means that strip_unused_nodes will not remove unused variables. - - Args: - base_meta_graph_def: a GraphDef proto in which to identify nodes to retain. - - Returns: - A dictionary corresponding to the nodes associated with each collection - that are to be retained. - """ - # TODO(b/63447631): implement variable stripping. - - initializer_names = {} - - # Primary SaverDef and SAVERS collection - saver_defs = [] - if base_meta_graph_def.HasField('saver_def'): - saver_defs.append(base_meta_graph_def.saver_def) - saver_defs.extend(_get_all_protos_from_collection( - base_meta_graph_def, _ops.GraphKeys.SAVERS)) - for saver_def in saver_defs: - savers = initializer_names.get(_ops.GraphKeys.SAVERS, []) - savers.extend([ - saver_def.filename_tensor_name, saver_def.save_tensor_name, - saver_def.restore_op_name - ]) - initializer_names[_ops.GraphKeys.SAVERS] = savers - - # Variable initializers - variable_collections = [ - _ops.GraphKeys.GLOBAL_VARIABLES, - _ops.GraphKeys.TRAINABLE_VARIABLES, - _ops.GraphKeys.MOVING_AVERAGE_VARIABLES, - _ops.GraphKeys.LOCAL_VARIABLES, - _ops.GraphKeys.MODEL_VARIABLES] - for var_coll in variable_collections: - variables = _get_all_protos_from_collection(base_meta_graph_def, var_coll) - var_init_names = [v.initializer_name for v in variables] - if var_init_names: - # Sanity check to ensure we don't overwrite dictionary entries. - assert var_coll not in initializer_names - initializer_names[var_coll] = var_init_names - - # Table initializers - op_names = _get_all_node_names_from_collection( - base_meta_graph_def, _ops.GraphKeys.TABLE_INITIALIZERS) - if op_names: - # Sanity check to ensure we don't overwrite dictionary entries. - assert _ops.GraphKeys.TABLE_INITIALIZERS not in initializer_names - table_initializers = [t for t in op_names] - initializer_names[_ops.GraphKeys.TABLE_INITIALIZERS] = table_initializers - - # Various init ops - various_init_op_collections = [_saved_model_constants.LEGACY_INIT_OP_KEY, - _saved_model_constants.MAIN_OP_KEY, - _ops.GraphKeys.INIT_OP, - _ops.GraphKeys.LOCAL_INIT_OP, - _ops.GraphKeys.READY_OP, - _ops.GraphKeys.READY_FOR_LOCAL_INIT_OP] - for op_coll in various_init_op_collections: - op_name = _get_single_node_name_from_collection( - base_meta_graph_def, op_coll) - if op_name: - # Sanity check to ensure we don't overwrite dictionary entries. - assert op_coll not in initializer_names - initializer_names[op_coll] = [op_name] - return initializer_names - - -def _add_pruned_collection(base_meta_graph_def, meta_graph_def, - collection_name, removed_op_names): - """Copy collection to the transformed MetaGraphDef, omitting removed items.""" - - base_collection = base_meta_graph_def.collection_def[collection_name] - collection = meta_graph_def.collection_def[collection_name] - - if base_collection.HasField('any_list'): - for any_value in base_collection.any_list.value: - # just search the serialized proto as a string - if not _is_removed_mentioned(any_value.value, removed_op_names): - copied_any = collection.any_list.value.add() - copied_any.CopyFrom(any_value) - elif base_collection.HasField('bytes_list'): - collection.bytes_list.value[:] = [ - s for s in base_collection.bytes_list.value - if not _is_removed_mentioned(s, removed_op_names)] - _logging.info( - 'In collection %s, nodes excluded are: %s', collection_name, - sorted([ - s for s in base_collection.bytes_list.value - if _is_removed_mentioned(s, removed_op_names) - ])) - elif base_collection.HasField('node_list'): - collection.node_list.value[:] = [ - s for s in base_collection.node_list.value - if not _is_removed(s, removed_op_names)] - else: - collection.CopyFrom(base_collection) - - -def _add_pruned_saver(base_meta_graph_def, meta_graph_def, removed_op_names): - """Copy the Saver into the transformed MetaGraphDef, if valid. - - Currently this copies the Saver as is, after verifying that none of the - referenced Save & Restore ops were removed. A future version will modify - the Save and Restore ops themselves as needed to account for removed - Variables. - - Args: - base_meta_graph_def: The untransformed MetaGraphDef. - meta_graph_def: The transformed MetaGraphDef being built. - removed_op_names: An iterable of names of ops that were removed. - """ - - # Note this does surgery on meta_graph_def.graph_def too, so that should have - # been copied already. - if base_meta_graph_def.HasField('saver_def'): - filename_tensor_name = base_meta_graph_def.saver_def.filename_tensor_name - save_tensor_name = base_meta_graph_def.saver_def.save_tensor_name - restore_op_name = base_meta_graph_def.saver_def.restore_op_name - - _check_tensor_not_removed(filename_tensor_name, removed_op_names) - _check_tensor_not_removed(save_tensor_name, removed_op_names) - _check_tensor_not_removed(restore_op_name, removed_op_names) - - # TODO(b/63447631): Once we strip unused variables, remove references to - # them from save and restore ops. Retain those ops only if they also refer - # to retained Variables. See if we can use _clean_save_and_restore() for - # this. - - # saver_name, restore_all = restore_op_name.rsplit('/', 1) - # if restore_all != 'restore_all': - # raise ValueError( - # 'SaverDef restore_op_name did not have expected form */restore_all') - - # save_tensor_names_op_name = '{}/SaveV2/tensor_names'.format(saver_name) - # restore_tensor_names_op_name = ( - # '{}/RestoreV2/tensor_names'.format(saver_name)) - - # save_tensor_names_op = _find_op(meta_graph_def.graph_def, - # save_tensor_names_op_name) - # save_tensor_names_value_tensor = save_tensor_names_op.attr['value'].tensor - # save_tensor_names_value_tensor.string_val[:] = [ - # s for s in save_tensor_names_value_tensor.string_val - # if not _is_removed(s, removed_op_names)] - - # restore_tensor_names_op = _find_op( - # meta_graph_def.graph_def, restore_tensor_names_op_name) - # restore_tensor_names_value_tensor = ( - # restore_tensor_names_op.attr['value'].tensor) - # restore_tensor_names_value_tensor.string_val[:] = [ - # s for s in restore_tensor_names_value_tensor.string_val - # if not _is_removed(s, removed_op_names)] - - # if (save_tensor_names_value_tensor.string_val - # or restore_tensor_names_value_tensor.string_val): - meta_graph_def.saver_def.CopyFrom(base_meta_graph_def.saver_def) - - -def _find_op(graph_def, op_name): - """Fetch a node from a GraphDef proto by name.""" - for node_def in graph_def.node: - if node_def.name == op_name: - return node_def - return None - - -def _add_pruned_signature(base_meta_graph_def, meta_graph_def, - signature_name, removed_op_names): - """Copy the named signature into the transformed MetaGraphDef, if valid. - - If any input or output mentioned in the signature was removed by the graph - transform, the signature is silently omitted from the transformed - MetaGraphDef. - - Args: - base_meta_graph_def: The untransformed MetaGraphDef. - meta_graph_def: The transformed MetaGraphDef being built. - signature_name: The name of the signature to copy. - removed_op_names: An iterable of names of ops that were removed. - """ - try: - base_signature = base_meta_graph_def.signature_def[signature_name] - for key in base_signature.inputs: - _check_tensor_not_removed(base_signature.inputs[key].name, - removed_op_names) - for key in base_signature.outputs: - _check_tensor_not_removed(base_signature.outputs[key].name, - removed_op_names) - meta_graph_def.signature_def[signature_name].CopyFrom(base_signature) - except ValueError: - # exclude any signature that mentions a removed node - pass - - -def _get_single_node_name_from_collection(meta_graph_def, collection_key): - """Obtain a node name that is the single element of a collection.""" - if collection_key not in meta_graph_def.collection_def: - return None - collection = meta_graph_def.collection_def[collection_key] - if not collection.node_list.value: - raise ValueError( - 'Collection {} is present but type is not node_list.'.format( - collection_key)) - if len(collection.node_list.value) != 1: - raise ValueError( - 'Collection {} is has {} elements; expected exactly one.'.format( - collection_key, collection.bytes_list)) - return collection.node_list.value[0] - - -def _get_all_node_names_from_collection(meta_graph_def, collection_key): - """Obtain node names from a collection.""" - if collection_key not in meta_graph_def.collection_def: - return None - collection = meta_graph_def.collection_def[collection_key] - if not collection.node_list.value: - raise ValueError( - 'Collection {} is present but type is not node_list.'.format( - collection_key)) - return collection.node_list.value - - -def _get_all_protos_from_collection(meta_graph_def, collection_key): - """Obtain node names from a collection.""" - if collection_key not in meta_graph_def.collection_def: - return [] - collection = meta_graph_def.collection_def[collection_key] - if not collection.bytes_list.value: - raise ValueError( - 'Collection {} is present but type is not bytes_list.'.format( - collection_key)) - proto_type = _ops.get_collection_proto_type(collection_key) - result = [] - for value in collection.bytes_list.value: - proto = proto_type() - proto.ParseFromString(value) - result.append(proto) - return result - - -def _is_removed(tensor_name, removed_op_names): - """Determine whether the named tensor is an output of a removed op.""" - for removed_op_name in removed_op_names: - if tensor_name.split(':')[0] == removed_op_name: - return True - return False - - -def _is_removed_mentioned(s, removed_op_names): - """Determine whether any removed op is mentioned in the given object. - - This relies on the string representation of the object. This is used for - proto messages that may mention ops by name in nested fields. The string - representation of the proto includes those field values, so this string - search approach is sufficient. - - Args: - s: an object to search for removed op names. - removed_op_names: An iterable of names of ops that were removed. - - Returns: - True if any removed op is mentioned in the given object, False otherwise. - """ - # A common approach taken by some of the transforms in gtt is to add new nodes - # that have the same prefix as the node they are removing. For example, if - # the original node name was /foo, they may remove that node and add in - # /foo/bar. This regex ensures that we handle these two nodes - # as separate entities. It matches on nodes having names in the form of - # '/foo/bar_x' as well as nodes having names in the form of 'foo.' - s_names = _re.findall(r'((?:[\/]?[a-zA-Z0-9\_]*)*)', _compat.as_str_any(s)) - for removed_op_name in removed_op_names: - for s_name in s_names: - if s_name.endswith(removed_op_name): - return True - return False - - -def _check_tensor_not_removed(tensor_name, removed_op_names): - """Verify that the named tensor was not removed. - - Args: - tensor_name: the name of a tensor to check. - removed_op_names: An iterable of names of ops that were removed. - - Raises: - ValueError: if the tensor was removed. - """ - if not tensor_name: - raise ValueError('Tensor name should not be empty') - if _is_removed(tensor_name, removed_op_names): - raise ValueError( - 'Expected Tensor, but it was removed: {}'.format(tensor_name)) - - -def _add_new_inits_to_collection(meta_graph_def, updated_initializer_names): - """Add new inits to collection. - - Args: - meta_graph_def: The MetaGraphDef protocol buffer to update. - updated_initializer_names: Dictionary of the updated "infrastructural" nodes - (initializers, save and restore ops, etc.). The keys in this dictionary - indicate the collection where these nodes were obtained from. - - Raises: - ValueError: if the tensor was removed. - """ - # TODO(dzats): Extend this to support all collections. - if _ops.GraphKeys.TABLE_INITIALIZERS in updated_initializer_names: - orig_table_inits = _get_all_node_names_from_collection( - meta_graph_def, _ops.GraphKeys.TABLE_INITIALIZERS) - orig_table_inits = orig_table_inits if orig_table_inits else [] - updated_table_inits = updated_initializer_names[ - _ops.GraphKeys.TABLE_INITIALIZERS] - new_table_inits = list(set(updated_table_inits) - set(orig_table_inits)) - new_table_inits.sort() - meta_graph_def.collection_def[ - _ops.GraphKeys.TABLE_INITIALIZERS].node_list.value.extend( - new_table_inits) - - -def meta_graph_transform( - base_meta_graph_def, input_names, output_names, transforms, tags, - checkpoint_path=None): - """Apply the Graph Transform tool to a MetaGraphDef. - - Args: - base_meta_graph_def: A MetaGraphDef protocol buffer to transform. - input_names: Names of input nodes. - output_names: Names of output nodes. - transforms: A list of strings naming the graph transforms to be applied in - order. These transform names are exactly those supported by the Graph - Transform Tool, with the addition of the 'freeze_graph' and - 'sparsify_gather' transforms. - tags: A list of tags with which to annotate the transformed MetaGraphDef. - checkpoint_path: A path to a checkpoint to restore during freezing, - if needed (default None). - - Returns: - A new transformed MetaGraphDef protocol buffer. - """ - meta_graph_def = _meta_graph_pb2.MetaGraphDef() - - initializer_names = _find_all_mandatory_retain_ops(base_meta_graph_def) - - transformed_graph_def, updated_initializer_names = _do_transforms( - base_meta_graph_def.graph_def, input_names, output_names, - initializer_names, transforms, base_meta_graph_def.saver_def, - checkpoint_path) - - meta_graph_def.graph_def.CopyFrom(transformed_graph_def) - meta_graph_def.meta_info_def.CopyFrom(base_meta_graph_def.meta_info_def) - meta_graph_def.meta_info_def.ClearField('tags') - for tag in tags: - meta_graph_def.meta_info_def.tags.append(tag) - - base_op_names = [_compat.as_str(node.name) - for node in base_meta_graph_def.graph_def.node] - retained_op_names = [_compat.as_str(node.name) - for node in meta_graph_def.graph_def.node] - removed_op_names = set(base_op_names) - set(retained_op_names) - _logging.info('Node names in base graph: %s', sorted(base_op_names)) - _logging.info('Node names retained: %s', sorted(retained_op_names)) - _logging.info('Node names removed: %s', sorted(removed_op_names)) - - # Copy saver, excluding any pruned nodes if graph was not frozen. - # TODO(b/63447631): Revisit this once the problem is addressed. Currently - # _add_pruned_saver assumes that the save and restore nodes have not been - # removed but freeze_graph (correctly) removes them. - if _FREEZE_GRAPH_TRANSFORM not in transforms: - _add_pruned_saver(base_meta_graph_def, meta_graph_def, removed_op_names) - - # Copy collections, excluding any pruned nodes - for collection_name in base_meta_graph_def.collection_def: - _add_pruned_collection( - base_meta_graph_def, meta_graph_def, collection_name, - removed_op_names) - - # Append newly added initializers to collection. - _add_new_inits_to_collection(meta_graph_def, updated_initializer_names) - - # Copy signature_defs, excluding any pruned nodes - for signature_name in base_meta_graph_def.signature_def: - _add_pruned_signature( - base_meta_graph_def, meta_graph_def, signature_name, - removed_op_names) - - return meta_graph_def diff --git a/tensorflow/contrib/meta_graph_transform/meta_graph_transform_test.py b/tensorflow/contrib/meta_graph_transform/meta_graph_transform_test.py deleted file mode 100644 index f3aec86be15..00000000000 --- a/tensorflow/contrib/meta_graph_transform/meta_graph_transform_test.py +++ /dev/null @@ -1,584 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for MetaGraphDef Transform Tool.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from google.protobuf.any_pb2 import Any -from tensorflow.contrib.meta_graph_transform import meta_graph_transform -from tensorflow.core.framework import function_pb2 -from tensorflow.core.framework import graph_pb2 -from tensorflow.core.framework import node_def_pb2 -from tensorflow.core.framework import types_pb2 -from tensorflow.core.protobuf import meta_graph_pb2 -from tensorflow.core.protobuf import saver_pb2 -from tensorflow.python.client import session as tf_session -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test -from tensorflow.python.saved_model import constants as saved_model_constants -from tensorflow.python.training import saver -from tensorflow.python.util import compat -from tensorflow.tools import graph_transforms - - -def _make_asset_file_def_any(node_name): - asset_file_def = meta_graph_pb2.AssetFileDef() - asset_file_def.tensor_info.name = node_name - any_message = Any() - any_message.Pack(asset_file_def) - return any_message - - -class MetaGraphTransformTest(test.TestCase): - - def test_meta_graph_transform(self): - - with ops.Graph().as_default(): - with tf_session.Session(''): - a = array_ops.placeholder(dtypes.int64, [1], name='a') - b = array_ops.placeholder(dtypes.int64, [1], name='b') - c = array_ops.placeholder(dtypes.int64, [1], name='c') - _ = a * b - _ = b * c - base_meta_graph_def = saver.export_meta_graph() - - with ops.Graph().as_default(): - with tf_session.Session(''): - a = array_ops.placeholder(dtypes.int64, [1], name='a') - b = array_ops.placeholder(dtypes.int64, [1], name='b') - _ = a * b - meta_info_def = meta_graph_pb2.MetaGraphDef.MetaInfoDef() - meta_info_def.tags.append('tag_ab') - - expected_meta_graph_def = saver.export_meta_graph( - meta_info_def=meta_info_def) - # Graph rewriter clears versions field, so we expect that. - expected_meta_graph_def.graph_def.ClearField('versions') - # Graph rewriter adds an empty library field, so we expect that. - expected_meta_graph_def.graph_def.library.CopyFrom( - function_pb2.FunctionDefLibrary()) - - input_names = ['a', 'b'] - output_names = ['mul:0'] - transforms = ['strip_unused_nodes'] - tags = ['tag_ab'] - print('AAAAAA: {}'.format(base_meta_graph_def)) - transformed_meta_graph_def = meta_graph_transform.meta_graph_transform( - base_meta_graph_def, input_names, output_names, transforms, tags) - - self.assertEqual(expected_meta_graph_def, transformed_meta_graph_def) - - def test_get_shared_init_op(self): - main_op = 'main_op' - legacy_op = 'legacy_op' - - legacy_only = {saved_model_constants.LEGACY_INIT_OP_KEY: [legacy_op]} - main_and_legacy = { - saved_model_constants.MAIN_OP_KEY: [main_op], - saved_model_constants.LEGACY_INIT_OP_KEY: [legacy_op] - } - self.assertEqual(meta_graph_transform._get_shared_init_op({}), None) - self.assertEqual( - meta_graph_transform._get_shared_init_op(main_and_legacy), main_op) - self.assertEqual( - meta_graph_transform._get_shared_init_op(legacy_only), legacy_op) - - @test.mock.patch.object(graph_transforms, 'TransformGraph') - def test_gtt_transforms(self, graph_transform_mock): - graph_def = graph_pb2.GraphDef() - graph_def.node.extend([node_def_pb2.NodeDef(name='z1', op='NoOp')]) - input_names = ['i1', 'i2'] - output_names = ['o1', 'o2'] - init_nodes = ['init1', 'init2'] - initializer_names = {'init': init_nodes} - transforms = ['t1', 't2'] - - expected_graph = graph_pb2.GraphDef() - expected_graph.node.extend([node_def_pb2.NodeDef(name='n1', op='NoOp')]) - graph_transform_mock.return_value = expected_graph - transformed_graph_def = (meta_graph_transform._gtt_transforms( - graph_def, input_names, output_names, initializer_names, transforms)) - - self.assertEqual(transformed_graph_def, expected_graph) - graph_transform_mock.assert_called_once_with( - graph_def, input_names, output_names + init_nodes, transforms) - - @test.mock.patch.object(meta_graph_transform, '_freeze_graph_with_def_protos') - def test_freeze_transform(self, freeze_mock): - graph_def = graph_pb2.GraphDef() - graph_def.node.extend([node_def_pb2.NodeDef(name='z1', op='NoOp')]) - output_names = ['o1', 'o2'] - table_init_names = ['t1', 't2'] - main_op = 'main_op' - legacy_op = 'legacy_op' - initializer_names = { - 'foo_init': ['init1', 'init2'], - ops.GraphKeys.TABLE_INITIALIZERS: table_init_names, - saved_model_constants.MAIN_OP_KEY: [main_op], - saved_model_constants.LEGACY_INIT_OP_KEY: [legacy_op] - } - expected_graph_def = graph_pb2.GraphDef() - graph_def.node.extend([node_def_pb2.NodeDef(name='n1', op='NoOp')]) - freeze_mock.return_value = expected_graph_def - saver_def = saver_pb2.SaverDef() - saver_def.filename_tensor_name = 'f1' - checkpoint_path = '/checkpoint/path' - transformed_graph_def, transformed_initializer_names = ( - meta_graph_transform._freeze_transform(graph_def, output_names, - initializer_names, saver_def, - checkpoint_path)) - self.assertEqual(transformed_graph_def, expected_graph_def) - expected_initializer_names = { - ops.GraphKeys.TABLE_INITIALIZERS: table_init_names, - saved_model_constants.MAIN_OP_KEY: [main_op], - saved_model_constants.LEGACY_INIT_OP_KEY: [legacy_op] - } - self.assertEqual(transformed_initializer_names, expected_initializer_names) - freeze_mock.assert_called_once_with(graph_def, output_names, - table_init_names, main_op, saver_def, - checkpoint_path) - - def test_clean_save_and_restore(self): - graph_def = graph_pb2.GraphDef() - save_name = 'save_1/SaveV2' - save_tensor_name = save_name + '/tensor_names' - save_tensor_shape = save_name + '/shape_and_slices' - save_op = graph_def.node.add() - save_op.name = save_name - save_op.op = 'NoOp' - save_name_op = graph_def.node.add() - save_name_op.name = save_tensor_name - save_name_op.op = 'NoOp' - save_shape_op = graph_def.node.add() - save_shape_op.name = save_tensor_shape - save_shape_op.op = 'NoOp' - - types = [types_pb2.DT_INT32, types_pb2.DT_FLOAT, types_pb2.DT_INT32] - names = [ - compat.as_bytes('/foo'), - compat.as_bytes('/bar'), - compat.as_bytes('/baz') - ] - shapes = [ - compat.as_bytes('100 10 0,100:0,10'), - compat.as_bytes('150 11 0,150:0,11'), - compat.as_bytes('101 12 0,101:0,12') - ] - - expected_types = [types[0], types[2]] - expected_names = [names[0], names[2]] - expected_shapes = [shapes[0], shapes[2]] - - save_op.attr['dtypes'].list.type[:] = types - save_name_op.attr['value'].tensor.string_val[:] = names - save_name_op.attr['value'].tensor.tensor_shape.dim.add().size = len(names) - save_name_op.attr['_output_shapes'].list.shape.add().dim.add().size = len( - names) - - save_shape_op.attr['value'].tensor.string_val[:] = shapes - save_shape_op.attr['value'].tensor.tensor_shape.dim.add().size = len(shapes) - save_shape_op.attr['_output_shapes'].list.shape.add().dim.add().size = len( - shapes) - - meta_graph_transform._clean_save_and_restore(graph_def, save_op, ['/bar']) - self.assertEqual(save_op.attr['dtypes'].list.type[:], expected_types) - self.assertEqual(save_name_op.attr['value'].tensor.string_val[:], - expected_names) - self.assertEqual(save_name_op.attr['value'].tensor.tensor_shape.dim[0].size, - len(expected_names)) - self.assertEqual( - save_name_op.attr['_output_shapes'].list.shape[0].dim[0].size, - len(expected_names)) - - self.assertEqual(save_shape_op.attr['value'].tensor.string_val[:], - expected_shapes) - self.assertEqual( - save_shape_op.attr['value'].tensor.tensor_shape.dim[0].size, - len(expected_shapes)) - self.assertEqual( - save_shape_op.attr['_output_shapes'].list.shape[0].dim[0].size, - len(expected_shapes)) - - @test.mock.patch.object(meta_graph_transform, '_clean_save_and_restore') - @test.mock.patch.object(meta_graph_transform, '_gtt_transforms') - def test_sparsify_gather_transform(self, gtt_mock, clean_save_restore_mock): - # Initial graph def. - graph_def = graph_pb2.GraphDef() - variable_op = graph_def.node.add() - variable_op.name = '/foo/part_1' - - constant_op = graph_def.node.add() - constant_op.name = '/bar' - - # Transformed graph def. - transformed_graph_def = graph_pb2.GraphDef() - constant_op = transformed_graph_def.node.add() - constant_op.name = '/foo' - - sparsify_shared_init_op_name = 'sparify_gather_init_op' - new_table_init_names = ['table1', 'table2'] - init_op = transformed_graph_def.node.add() - init_op.name = sparsify_shared_init_op_name - init_op.input.extend(['^' + f for f in new_table_init_names]) - - saver_op = transformed_graph_def.node.add() - saver_op.name = 'save_1/SaveV2' - - orig_table_init_names = ['orig_table_init_1', 'orig_table_init_2'] - - legacy_op_name = 'legacy_op' - legacy_op = transformed_graph_def.node.add() - legacy_op.name = legacy_op_name - legacy_op.input.extend(['^' + f for f in orig_table_init_names]) - - input_names = ['i1', 'i2'] - output_names = ['o1', 'o2'] - - initializer_names = { - 'foo_init': ['init1', 'init2'], - ops.GraphKeys.TABLE_INITIALIZERS: orig_table_init_names, - saved_model_constants.LEGACY_INIT_OP_KEY: [legacy_op_name] - } - checkpoint_path = '/path/to/checkpoint' - - expected_initializer_names = { - 'foo_init': ['init1', 'init2'], - ops.GraphKeys.TABLE_INITIALIZERS: ( - orig_table_init_names + new_table_init_names), - saved_model_constants.LEGACY_INIT_OP_KEY: [legacy_op_name] - } - - expected_sparsify_cmd = [ - 'sparsify_gather(input_checkpoint="%s", group_init_node="%s")' % - (checkpoint_path, sparsify_shared_init_op_name) - ] - - # Expected graph def. - expected_graph_def = graph_pb2.GraphDef() - constant_op = expected_graph_def.node.add() - constant_op.name = '/foo' - - saver_op = expected_graph_def.node.add() - saver_op.name = 'save_1/SaveV2' - - legacy_op_name = 'legacy_op' - legacy_op = expected_graph_def.node.add() - legacy_op.name = legacy_op_name - legacy_op.input.extend( - ['^' + f for f in orig_table_init_names + new_table_init_names]) - - gtt_mock.return_value = transformed_graph_def - graph_def_result, init_names_result = ( - meta_graph_transform._sparsify_gather_transform( - graph_def, input_names, output_names, initializer_names, - checkpoint_path)) - - gtt_mock.assert_called_once_with(graph_def, input_names, output_names, - initializer_names, expected_sparsify_cmd) - - clean_save_restore_mock.assert_called_once_with(transformed_graph_def, - saver_op, ['/bar', '/foo']) - - self.assertEqual(expected_graph_def, graph_def_result) - self.assertEqual(expected_initializer_names, init_names_result) - - @test.mock.patch.object(meta_graph_transform, '_gtt_transforms') - @test.mock.patch.object(meta_graph_transform, '_freeze_transform') - @test.mock.patch.object(meta_graph_transform, '_sparsify_gather_transform') - def test_do_transforms(self, sparsify_mock, freeze_mock, gtt_mock): - graph_def = graph_pb2.GraphDef() - constant_op = graph_def.node.add() - constant_op.name = 'c1' - - input_names = ['i1', 'i2'] - output_names = ['o1', 'o2'] - initializer_names = { - 'foo_init': ['init1', 'init2'], - ops.GraphKeys.TABLE_INITIALIZERS: ['table1'], - saved_model_constants.LEGACY_INIT_OP_KEY: ['legacy_op'] - } - - transforms = ['foo', 'freeze_graph', 'bar', 'sparsify_gather', 'baz'] - - sparsify_mock.return_value = (graph_def, initializer_names) - freeze_mock.return_value = (graph_def, initializer_names) - gtt_mock.return_value = graph_def - - graph_def_result, initializer_names_result = ( - meta_graph_transform._do_transforms(graph_def, input_names, - output_names, initializer_names, - transforms)) - - sparsify_mock.assert_called_once_with(graph_def, input_names, output_names, - initializer_names, None) - - freeze_mock.assert_called_once_with(graph_def, output_names, - initializer_names, None, None) - - gtt_mock.assert_has_calls([ - test.mock.call(graph_def, input_names, output_names, initializer_names, - ['foo']), - test.mock.call(graph_def, input_names, output_names, initializer_names, - ['bar']), - test.mock.call(graph_def, input_names, output_names, initializer_names, - ['baz']) - ]) - self.assertEqual(graph_def_result, graph_def) - self.assertEqual(initializer_names, initializer_names_result) - - def test_add_new_inits_to_collection(self): - meta_graph_def = meta_graph_pb2.MetaGraphDef() - - orig_table_inits = ['t1', 't2'] - new_table_inits = ['t3', 't4'] - - meta_graph_def.collection_def[ - ops.GraphKeys.TABLE_INITIALIZERS].node_list.value.extend( - orig_table_inits) - updated_init_names = { - ops.GraphKeys.TABLE_INITIALIZERS: orig_table_inits + new_table_inits - } - - meta_graph_transform._add_new_inits_to_collection(meta_graph_def, - updated_init_names) - - self.assertEqual(meta_graph_def.collection_def[ - ops.GraphKeys.TABLE_INITIALIZERS].node_list.value, - orig_table_inits + new_table_inits) - - @test.mock.patch.object(graph_transforms, 'TransformGraph') - @test.mock.patch.object(meta_graph_transform, '_freeze_graph_with_def_protos') - def test_freeze_then_sparsify(self, freeze_mock, graph_transform_mock): - tag_name = 'tag' - input_nodes = 'input_nodes' - output_nodes = 'output_nodes' - freeze_transform = 'freeze_graph' - sparsify_transform = 'sparsify_gather' - - base_meta_graph_def = meta_graph_pb2.MetaGraphDef() - - # Add a table initializer. - table_init_name = 'table_init' - node_def = node_def_pb2.NodeDef( - name=table_init_name, op='InitializeTableV2') - base_meta_graph_def.graph_def.node.extend([node_def]) - - # Add a group_deps node. - group_deps_name = 'group_deps' - node_def = node_def_pb2.NodeDef(name=group_deps_name, op='NoOp') - node_def.input.extend(['^table_init']) - base_meta_graph_def.graph_def.node.extend([node_def]) - - base_meta_graph_def.collection_def[ - ops.GraphKeys.TABLE_INITIALIZERS].node_list.value.extend( - [table_init_name]) - base_meta_graph_def.collection_def[ - saved_model_constants.LEGACY_INIT_OP_KEY].node_list.value.extend( - [group_deps_name]) - - # Expected metagraphdef. - expected_meta_graph_def = meta_graph_pb2.MetaGraphDef() - expected_meta_graph_def.CopyFrom(base_meta_graph_def) - expected_meta_graph_def.meta_info_def.tags.append(tag_name) - - transformed_graph_def = graph_pb2.GraphDef() - transformed_graph_def.CopyFrom(expected_meta_graph_def.graph_def) - freeze_mock.return_value = transformed_graph_def - graph_transform_mock.return_value = transformed_graph_def - - # Add unsaved init node. - unsaved_init_name = 'unsaved_node' - node_def = node_def_pb2.NodeDef(name=unsaved_init_name, op='NoOp') - base_meta_graph_def.graph_def.node.extend([node_def]) - - # Add a saver. - base_meta_graph_def.saver_def.filename_tensor_name = 'node1' - base_meta_graph_def.saver_def.save_tensor_name = 'node3' - base_meta_graph_def.saver_def.restore_op_name = 'node6' - - transformed_meta_graph_def = meta_graph_transform.meta_graph_transform( - base_meta_graph_def, [input_nodes], [output_nodes], - [freeze_transform, sparsify_transform], [tag_name]) - - self.assertEqual(expected_meta_graph_def, transformed_meta_graph_def) - freeze_mock.assert_called_once_with( - base_meta_graph_def.graph_def, [output_nodes], [table_init_name], - group_deps_name, base_meta_graph_def.saver_def, None) - graph_transform_mock.assert_called_once_with( - transformed_graph_def, [input_nodes], [ - output_nodes, group_deps_name, table_init_name - ], [sparsify_transform + '(group_init_node="sparify_gather_init_op")']) - - def test_connect_to_shared_init_op(self): - group_deps_name = 'group_deps' - init_node_1 = 'table_init_1' - init_node_2 = 'table_init_2' - - orig_graph_def = graph_pb2.GraphDef() - expected_graph_def_1 = graph_pb2.GraphDef() - - meta_graph_transform._connect_to_shared_init_op(orig_graph_def, - group_deps_name, []) - self.assertEqual(expected_graph_def_1, orig_graph_def) - - expected_graph_def_2 = graph_pb2.GraphDef() - node_def = node_def_pb2.NodeDef(name=group_deps_name, op='NoOp') - node_def.input.extend(['^' + init_node_1, '^' + init_node_2]) - expected_graph_def_2.node.extend([node_def]) - - meta_graph_transform._connect_to_shared_init_op( - orig_graph_def, group_deps_name, [init_node_1, init_node_2]) - self.assertEqual(expected_graph_def_2, orig_graph_def) - - def test_add_pruned_collection_node(self): - # Note: This also tests _is_removed(). - collection_name = 'node_collection' - base_meta_graph_def = meta_graph_pb2.MetaGraphDef() - base_meta_graph_def.collection_def[collection_name].node_list.value.extend( - ['node1', 'node2', 'node3', 'node4', '/a/a_1', '/b/b_1']) - - meta_graph_def = meta_graph_pb2.MetaGraphDef() - removed_op_names = ['node2', 'node4', 'node5', '/a', '/b/b_1'] - meta_graph_transform._add_pruned_collection( - base_meta_graph_def, meta_graph_def, collection_name, removed_op_names) - - collection = meta_graph_def.collection_def[collection_name] - - expected_nodes = ['node1', 'node3', '/a/a_1'] - self.assertEqual(expected_nodes, collection.node_list.value) - - def test_add_pruned_collection_int(self): - collection_name = 'int_collection' - base_meta_graph_def = meta_graph_pb2.MetaGraphDef() - base_meta_graph_def.collection_def[collection_name].int64_list.value[:] = ( - [10, 20, 30, 40]) - - meta_graph_def = meta_graph_pb2.MetaGraphDef() - removed_op_names = ['node2', 'node4', 'node5', '/a', '/b/b_1'] - meta_graph_transform._add_pruned_collection( - base_meta_graph_def, meta_graph_def, collection_name, removed_op_names) - - collection = meta_graph_def.collection_def[collection_name] - - expected_ints = [10, 20, 30, 40] - self.assertEqual(expected_ints, collection.int64_list.value) - - def test_add_pruned_collection_proto_in_any_list(self): - # Note: This also tests _is_removed_mentioned(). - collection_name = 'proto_collection' - base_meta_graph_def = meta_graph_pb2.MetaGraphDef() - base_meta_graph_def.collection_def[collection_name].any_list.value.extend([ - _make_asset_file_def_any('node1'), - _make_asset_file_def_any('node2'), - _make_asset_file_def_any('node3'), - _make_asset_file_def_any('node4'), - _make_asset_file_def_any('/a/a_1'), - _make_asset_file_def_any('/b/b_1') - ]) - - meta_graph_def = meta_graph_pb2.MetaGraphDef() - removed_op_names = ['node2', 'node4', 'node5', '/a', '/b/b_1'] - meta_graph_transform._add_pruned_collection( - base_meta_graph_def, meta_graph_def, collection_name, removed_op_names) - - collection = meta_graph_def.collection_def[collection_name] - - expected_protos = [ - _make_asset_file_def_any('node1'), - _make_asset_file_def_any('node3'), - _make_asset_file_def_any('/a/a_1'), - ] - self.assertEqual(expected_protos, collection.any_list.value[:]) - - def test_add_pruned_collection_proto_in_bytes_list(self): - # Note: This also tests _is_removed_mentioned(). - collection_name = 'proto_collection' - base_meta_graph_def = meta_graph_pb2.MetaGraphDef() - base_meta_graph_def.collection_def[collection_name].bytes_list.value.extend( - [compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('node1'))), - compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('node2'))), - compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('node3'))), - compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('node4'))), - compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('/a/a_1'))), - compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('/b/b_1'))) - ]) - - meta_graph_def = meta_graph_pb2.MetaGraphDef() - removed_op_names = ['node2', 'node4', 'node5', '/a', '/b/b_1'] - meta_graph_transform._add_pruned_collection( - base_meta_graph_def, meta_graph_def, collection_name, removed_op_names) - - collection = meta_graph_def.collection_def[collection_name] - - expected_values = [ - compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('node1'))), - compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('node3'))), - compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('/a/a_1'))), - ] - self.assertEqual(expected_values, collection.bytes_list.value[:]) - - def test_add_pruned_saver(self): - base_meta_graph_def = meta_graph_pb2.MetaGraphDef() - - base_meta_graph_def.saver_def.filename_tensor_name = 'node1' - base_meta_graph_def.saver_def.save_tensor_name = 'node3' - base_meta_graph_def.saver_def.restore_op_name = 'node6' - - meta_graph_def = meta_graph_pb2.MetaGraphDef() - removed_op_names = ['node2', 'node4', 'node5'] - meta_graph_transform._add_pruned_saver(base_meta_graph_def, - meta_graph_def, - removed_op_names) - - # TODO(b/63447631): For now the saver is just copied unchanged - self.assertEqual(base_meta_graph_def.saver_def, meta_graph_def.saver_def) - - def test_add_pruned_signature(self): - base_meta_graph_def = meta_graph_pb2.MetaGraphDef() - - signature_name_keep = 'test_signature_keep' - base_sig_keep = base_meta_graph_def.signature_def[signature_name_keep] - base_sig_keep.inputs['input_1'].name = 'input_1' - base_sig_keep.outputs['output_1'].name = 'output_1' - - signature_name_remove = 'test_signature_remove' - base_sig_remove = base_meta_graph_def.signature_def[signature_name_remove] - base_sig_remove.inputs['node2'].name = 'node2' - base_sig_remove.outputs['output_1'].name = 'output_1' - - meta_graph_def = meta_graph_pb2.MetaGraphDef() - removed_op_names = ['node2', 'node4', 'node5'] - meta_graph_transform._add_pruned_signature(base_meta_graph_def, - meta_graph_def, - signature_name_keep, - removed_op_names) - meta_graph_transform._add_pruned_signature(base_meta_graph_def, - meta_graph_def, - signature_name_remove, - removed_op_names) - - self.assertTrue(signature_name_keep in meta_graph_def.signature_def) - sig_keep = meta_graph_def.signature_def[signature_name_keep] - self.assertEqual(base_sig_keep, sig_keep) - - self.assertFalse(signature_name_remove in meta_graph_def.signature_def) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/metrics/BUILD b/tensorflow/contrib/metrics/BUILD deleted file mode 100644 index fffe8861b56..00000000000 --- a/tensorflow/contrib/metrics/BUILD +++ /dev/null @@ -1,133 +0,0 @@ -# Description: -# Contains ops for evaluation metrics and summary statistics. -# APIs here are meant to evolve over time. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = [ - "//engedu/ml/tf_from_scratch:__pkg__", - "//tensorflow:internal", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "metrics_py", - srcs = [ - "__init__.py", - "python/metrics/__init__.py", - "python/metrics/classification.py", - "python/ops/confusion_matrix_ops.py", - "python/ops/histogram_ops.py", - "python/ops/metric_ops.py", - "python/ops/set_ops.py", - ], - srcs_version = "PY2AND3", - visibility = [ - "//tensorflow:internal", - "//third_party/py/tf_slim:__subpackages__", - ], - deps = [ - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:confusion_matrix", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:histogram_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:metrics", - "//tensorflow/python:nn", - "//tensorflow/python:nn_ops", - "//tensorflow/python:sets", - "//tensorflow/python:state_ops", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:weights_broadcast_ops", - "//tensorflow/python/distribute:distribute_lib", - "//tensorflow/python/ops/distributions", - ], -) - -py_test( - name = "classification_test", - srcs = ["python/metrics/classification_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":metrics_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - ], -) - -py_test( - name = "histogram_ops_test", - size = "medium", - srcs = ["python/kernel_tests/histogram_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":metrics_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "metric_ops_test", - srcs = ["python/ops/metric_ops_test.py"], - python_version = "PY2", - shard_count = 30, - srcs_version = "PY2AND3", - tags = ["noasan"], # times out b/63678675 - deps = [ - ":metrics_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "metric_ops_large_test", - size = "large", - srcs = ["python/ops/metric_ops_large_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["noasan"], # times out b/63678675 - deps = [ - ":metrics_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/metrics/README.md b/tensorflow/contrib/metrics/README.md deleted file mode 100644 index e0f2d74fa32..00000000000 --- a/tensorflow/contrib/metrics/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# TensorFlow evaluation metrics and summary statistics - -## Evaluation metrics - -Metrics are used in evaluation to assess the quality of a model. Most are -"streaming" ops, meaning they create variables to accumulate a running total, -and return an update tensor to update these variables, and a value tensor to -read the accumulated value. Example: - -value, update_op = metrics.streaming_mean_squared_error( - predictions, targets, weight) - -Most metric functions take a pair of tensors, `predictions` and ground truth -`targets` (`streaming_mean` is an exception, it takes a single value tensor, -usually a loss). It is assumed that the shape of both these tensors is of the -form `[batch_size, d1, ... dN]` where `batch_size` is the number of samples in -the batch and `d1` ... `dN` are the remaining dimensions. - -The `weight` parameter can be used to adjust the relative weight of samples -within the batch. The result of each loss is a scalar average of all sample -losses with non-zero weights. - -The result is 2 tensors that should be used like the following for each eval -run: - -```python -predictions = ... -labels = ... -value, update_op = some_metric(predictions, labels) - -for step_num in range(max_steps): - update_op.run() - -print "evaluation score: ", value.eval() -``` diff --git a/tensorflow/contrib/metrics/__init__.py b/tensorflow/contrib/metrics/__init__.py deleted file mode 100644 index 5645784f8de..00000000000 --- a/tensorflow/contrib/metrics/__init__.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for evaluation metrics and summary statistics. - -See the -[Contrib Metrics](https://tensorflow.org/api_guides/python/contrib.metrics) -guide. - -@@auc_with_confidence_intervals -@@streaming_accuracy -@@streaming_mean -@@streaming_recall -@@streaming_recall_at_thresholds -@@streaming_precision -@@streaming_precision_at_thresholds -@@streaming_false_positive_rate -@@streaming_false_positive_rate_at_thresholds -@@streaming_false_negative_rate -@@streaming_false_negative_rate_at_thresholds -@@streaming_auc -@@streaming_dynamic_auc -@@streaming_curve_points -@@streaming_recall_at_k -@@streaming_mean_absolute_error -@@streaming_mean_iou -@@streaming_mean_relative_error -@@streaming_mean_squared_error -@@streaming_mean_tensor -@@streaming_root_mean_squared_error -@@streaming_covariance -@@streaming_pearson_correlation -@@streaming_mean_cosine_distance -@@streaming_percentage_less -@@streaming_sensitivity_at_specificity -@@streaming_sparse_average_precision_at_k -@@streaming_sparse_average_precision_at_top_k -@@streaming_sparse_precision_at_k -@@streaming_sparse_precision_at_top_k -@@streaming_sparse_recall_at_k -@@streaming_specificity_at_sensitivity -@@streaming_concat -@@streaming_false_negatives -@@streaming_false_negatives_at_thresholds -@@streaming_false_positives -@@streaming_false_positives_at_thresholds -@@streaming_true_negatives -@@streaming_true_negatives_at_thresholds -@@streaming_true_positives -@@streaming_true_positives_at_thresholds -@@sparse_recall_at_top_k -@@auc_using_histogram -@@accuracy -@@aggregate_metrics -@@aggregate_metric_map -@@confusion_matrix -@@f1_score -@@set_difference -@@set_intersection -@@set_size -@@set_union -@@cohen_kappa -@@count -@@precision_recall_at_equal_thresholds -@@recall_at_precision -@@precision_at_recall - -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,line-too-long,g-importing-member,wildcard-import -from tensorflow.contrib.metrics.python.metrics import * -# pylint: enable=wildcard-import -from tensorflow.contrib.metrics.python.ops.confusion_matrix_ops import confusion_matrix -from tensorflow.contrib.metrics.python.ops.histogram_ops import auc_using_histogram -from tensorflow.contrib.metrics.python.ops.metric_ops import aggregate_metric_map -from tensorflow.contrib.metrics.python.ops.metric_ops import aggregate_metrics -from tensorflow.contrib.metrics.python.ops.metric_ops import auc_with_confidence_intervals -from tensorflow.contrib.metrics.python.ops.metric_ops import cohen_kappa -from tensorflow.contrib.metrics.python.ops.metric_ops import count -from tensorflow.contrib.metrics.python.ops.metric_ops import precision_at_recall -from tensorflow.contrib.metrics.python.ops.metric_ops import precision_recall_at_equal_thresholds -from tensorflow.contrib.metrics.python.ops.metric_ops import recall_at_precision -from tensorflow.contrib.metrics.python.ops.metric_ops import sparse_recall_at_top_k -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_accuracy -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_auc -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_concat -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_covariance -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_curve_points -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_dynamic_auc -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negative_rate -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negative_rate_at_thresholds -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negatives -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negatives_at_thresholds -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positive_rate -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positive_rate_at_thresholds -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positives -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positives_at_thresholds -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean_absolute_error -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean_cosine_distance -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean_iou -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean_relative_error -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean_squared_error -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean_tensor -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_pearson_correlation -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_percentage_less -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_precision -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_precision_at_thresholds -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_recall -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_recall_at_k -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_recall_at_thresholds -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_root_mean_squared_error -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_sensitivity_at_specificity -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_sparse_average_precision_at_k -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_sparse_average_precision_at_top_k -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_sparse_precision_at_k -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_sparse_precision_at_top_k -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_sparse_recall_at_k -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_specificity_at_sensitivity -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_true_negatives -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_true_negatives_at_thresholds -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_true_positives -from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_true_positives_at_thresholds -from tensorflow.contrib.metrics.python.ops.set_ops import set_difference -from tensorflow.contrib.metrics.python.ops.set_ops import set_intersection -from tensorflow.contrib.metrics.python.ops.set_ops import set_size -from tensorflow.contrib.metrics.python.ops.set_ops import set_union -# pylint: enable=unused-import,line-too-long - -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/metrics/python/kernel_tests/histogram_ops_test.py b/tensorflow/contrib/metrics/python/kernel_tests/histogram_ops_test.py deleted file mode 100644 index bed1ecb71c2..00000000000 --- a/tensorflow/contrib/metrics/python/kernel_tests/histogram_ops_test.py +++ /dev/null @@ -1,255 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for histogram_ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.metrics.python.ops import histogram_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class Strict1dCumsumTest(test.TestCase): - """Test this private function.""" - - def test_empty_tensor_returns_empty(self): - with self.cached_session(): - tensor = constant_op.constant([]) - result = histogram_ops._strict_1d_cumsum(tensor, 0) - expected = constant_op.constant([]) - np.testing.assert_array_equal(expected.eval(), result.eval()) - - def test_length_1_tensor_works(self): - with self.cached_session(): - tensor = constant_op.constant([3], dtype=dtypes.float32) - result = histogram_ops._strict_1d_cumsum(tensor, 1) - expected = constant_op.constant([3], dtype=dtypes.float32) - np.testing.assert_array_equal(expected.eval(), result.eval()) - - def test_length_3_tensor_works(self): - with self.cached_session(): - tensor = constant_op.constant([1, 2, 3], dtype=dtypes.float32) - result = histogram_ops._strict_1d_cumsum(tensor, 3) - expected = constant_op.constant([1, 3, 6], dtype=dtypes.float32) - np.testing.assert_array_equal(expected.eval(), result.eval()) - - -class AUCUsingHistogramTest(test.TestCase): - - def setUp(self): - self.rng = np.random.RandomState(0) - - def test_empty_labels_and_scores_gives_nan_auc(self): - with self.cached_session(): - labels = constant_op.constant([], shape=[0], dtype=dtypes.bool) - scores = constant_op.constant([], shape=[0], dtype=dtypes.float32) - score_range = [0, 1.] - auc, update_op = histogram_ops.auc_using_histogram(labels, scores, - score_range) - variables.local_variables_initializer().run() - update_op.run() - self.assertTrue(np.isnan(auc.eval())) - - def test_perfect_scores_gives_auc_1(self): - self._check_auc( - nbins=100, - desired_auc=1.0, - score_range=[0, 1.], - num_records=50, - frac_true=0.5, - atol=0.05, - num_updates=1) - - def test_terrible_scores_gives_auc_0(self): - self._check_auc( - nbins=100, - desired_auc=0.0, - score_range=[0, 1.], - num_records=50, - frac_true=0.5, - atol=0.05, - num_updates=1) - - def test_many_common_conditions(self): - for nbins in [50]: - for desired_auc in [0.3, 0.5, 0.8]: - for score_range in [[-1, 1], [-10, 0]]: - for frac_true in [0.3, 0.8]: - # Tests pass with atol = 0.03. Moved up to 0.05 to avoid flakes. - self._check_auc( - nbins=nbins, - desired_auc=desired_auc, - score_range=score_range, - num_records=100, - frac_true=frac_true, - atol=0.05, - num_updates=50) - - def test_large_class_imbalance_still_ok(self): - # With probability frac_true ** num_records, each batch contains only True - # records. In this case, ~ 95%. - # Tests pass with atol = 0.02. Increased to 0.05 to avoid flakes. - self._check_auc( - nbins=100, - desired_auc=0.8, - score_range=[-1, 1.], - num_records=10, - frac_true=0.995, - atol=0.05, - num_updates=1000) - - def test_super_accuracy_with_many_bins_and_records(self): - # Test passes with atol = 0.0005. Increased atol to avoid flakes. - self._check_auc( - nbins=1000, - desired_auc=0.75, - score_range=[0, 1.], - num_records=1000, - frac_true=0.5, - atol=0.005, - num_updates=100) - - def _check_auc(self, - nbins=100, - desired_auc=0.75, - score_range=None, - num_records=50, - frac_true=0.5, - atol=0.05, - num_updates=10): - """Check auc accuracy against synthetic data. - - Args: - nbins: nbins arg from contrib.metrics.auc_using_histogram. - desired_auc: Number in [0, 1]. The desired auc for synthetic data. - score_range: 2-tuple, (low, high), giving the range of the resultant - scores. Defaults to [0, 1.]. - num_records: Positive integer. The number of records to return. - frac_true: Number in (0, 1). Expected fraction of resultant labels that - will be True. This is just in expectation...more or less may actually - be True. - atol: Absolute tolerance for final AUC estimate. - num_updates: Update internal histograms this many times, each with a new - batch of synthetic data, before computing final AUC. - - Raises: - AssertionError: If resultant AUC is not within atol of theoretical AUC - from synthetic data. - """ - score_range = [0, 1.] or score_range - with self.cached_session(): - labels = array_ops.placeholder(dtypes.bool, shape=[num_records]) - scores = array_ops.placeholder(dtypes.float32, shape=[num_records]) - auc, update_op = histogram_ops.auc_using_histogram( - labels, scores, score_range, nbins=nbins) - variables.local_variables_initializer().run() - # Updates, then extract auc. - for _ in range(num_updates): - labels_a, scores_a = synthetic_data(desired_auc, score_range, - num_records, self.rng, frac_true) - update_op.run(feed_dict={labels: labels_a, scores: scores_a}) - labels_a, scores_a = synthetic_data(desired_auc, score_range, num_records, - self.rng, frac_true) - # Fetch current auc, and verify that fetching again doesn't change it. - auc_eval = auc.eval() - self.assertAlmostEqual(auc_eval, auc.eval(), places=5) - - msg = ('nbins: %s, desired_auc: %s, score_range: %s, ' - 'num_records: %s, frac_true: %s, num_updates: %s') % (nbins, - desired_auc, - score_range, - num_records, - frac_true, - num_updates) - np.testing.assert_allclose(desired_auc, auc_eval, atol=atol, err_msg=msg) - - -def synthetic_data(desired_auc, score_range, num_records, rng, frac_true): - """Create synthetic boolean_labels and scores with adjustable auc. - - Args: - desired_auc: Number in [0, 1], the theoretical AUC of resultant data. - score_range: 2-tuple, (low, high), giving the range of the resultant scores - num_records: Positive integer. The number of records to return. - rng: Initialized np.random.RandomState random number generator - frac_true: Number in (0, 1). Expected fraction of resultant labels that - will be True. This is just in expectation...more or less may actually be - True. - - Returns: - boolean_labels: np.array, dtype=bool. - scores: np.array, dtype=np.float32 - """ - # We prove here why the method (below) for computing AUC works. Of course we - # also checked this against sklearn.metrics.roc_auc_curve. - # - # First do this for score_range = [0, 1], then rescale. - # WLOG assume AUC >= 0.5, otherwise we will solve for AUC >= 0.5 then swap - # the labels. - # So for AUC in [0, 1] we create False and True labels - # and corresponding scores drawn from: - # F ~ U[0, 1], T ~ U[x, 1] - # We have, - # AUC - # = P[T > F] - # = P[T > F | F < x] P[F < x] + P[T > F | F > x] P[F > x] - # = (1 * x) + (0.5 * (1 - x)). - # Inverting, we have: - # x = 2 * AUC - 1, when AUC >= 0.5. - assert 0 <= desired_auc <= 1 - assert 0 < frac_true < 1 - - if desired_auc < 0.5: - flip_labels = True - desired_auc = 1 - desired_auc - frac_true = 1 - frac_true - else: - flip_labels = False - x = 2 * desired_auc - 1 - - labels = rng.binomial(1, frac_true, size=num_records).astype(bool) - num_true = labels.sum() - num_false = num_records - labels.sum() - - # Draw F ~ U[0, 1], and T ~ U[x, 1] - false_scores = rng.rand(num_false) - true_scores = x + rng.rand(num_true) * (1 - x) - - # Reshape [0, 1] to score_range. - def reshape(scores): - return score_range[0] + scores * (score_range[1] - score_range[0]) - - false_scores = reshape(false_scores) - true_scores = reshape(true_scores) - - # Place into one array corresponding with the labels. - scores = np.nan * np.ones(num_records, dtype=np.float32) - scores[labels] = true_scores - scores[~labels] = false_scores - - if flip_labels: - labels = ~labels - - return labels, scores - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/metrics/python/metrics/__init__.py b/tensorflow/contrib/metrics/python/metrics/__init__.py deleted file mode 100644 index beba5b5f7e7..00000000000 --- a/tensorflow/contrib/metrics/python/metrics/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A module containing TensorFlow ops whose API may change in the future.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.metrics.python.metrics.classification import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/metrics/python/metrics/classification.py b/tensorflow/contrib/metrics/python/metrics/classification.py deleted file mode 100644 index 9aabc4bec30..00000000000 --- a/tensorflow/contrib/metrics/python/metrics/classification.py +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Classification metrics library.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.distribute import distribution_strategy_context -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import metrics_impl -from tensorflow.python.ops import variable_scope - -# TODO(nsilberman): move into metrics/python/ops/ - - -def accuracy(predictions, labels, weights=None, name=None): - """Computes the percentage of times that predictions matches labels. - - Args: - predictions: the predicted values, a `Tensor` whose dtype and shape - matches 'labels'. - labels: the ground truth values, a `Tensor` of any shape and - bool, integer, or string dtype. - weights: None or `Tensor` of float values to reweight the accuracy. - name: A name for the operation (optional). - - Returns: - Accuracy `Tensor`. - - Raises: - ValueError: if dtypes don't match or - if dtype is not bool, integer, or string. - """ - if not (labels.dtype.is_integer or - labels.dtype in (dtypes.bool, dtypes.string)): - raise ValueError( - 'Labels should have bool, integer, or string dtype, not %r' % - labels.dtype) - if not labels.dtype.is_compatible_with(predictions.dtype): - raise ValueError('Dtypes of predictions and labels should match. ' - 'Given: predictions (%r) and labels (%r)' % - (predictions.dtype, labels.dtype)) - with ops.name_scope(name, 'accuracy', values=[predictions, labels]): - is_correct = math_ops.cast( - math_ops.equal(predictions, labels), dtypes.float32) - if weights is not None: - is_correct = math_ops.multiply(is_correct, weights) - num_values = math_ops.multiply(weights, array_ops.ones_like(is_correct)) - return math_ops.div(math_ops.reduce_sum(is_correct), - math_ops.reduce_sum(num_values)) - return math_ops.reduce_mean(is_correct) - - -def f1_score(labels, predictions, weights=None, num_thresholds=200, - metrics_collections=None, updates_collections=None, name=None): - """Computes the approximately best F1-score across different thresholds. - - The f1_score function applies a range of thresholds to the predictions to - convert them from [0, 1] to bool. Precision and recall are computed by - comparing them to the labels. The F1-Score is then defined as - 2 * precision * recall / (precision + recall). The best one across the - thresholds is returned. - - Disclaimer: In practice it may be desirable to choose the best threshold on - the validation set and evaluate the F1 score with this threshold on a - separate test set. Or it may be desirable to use a fixed threshold (e.g. 0.5). - - This function internally creates four local variables, `true_positives`, - `true_negatives`, `false_positives` and `false_negatives` that are used to - compute the pairs of recall and precision values for a linearly spaced set of - thresholds from which the best f1-score is derived. - - This value is ultimately returned as `f1-score`, an idempotent operation that - computes the F1-score (computed using the aforementioned variables). The - `num_thresholds` variable controls the degree of discretization with larger - numbers of thresholds more closely approximating the true best F1-score. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the F1-score. - - Example usage with a custom estimator: - def model_fn(features, labels, mode): - predictions = make_predictions(features) - loss = make_loss(predictions, labels) - train_op = tf.contrib.training.create_train_op( - total_loss=loss, - optimizer='Adam') - eval_metric_ops = {'f1': f1_score(labels, predictions)} - return tf.estimator.EstimatorSpec( - mode=mode, - predictions=predictions, - loss=loss, - train_op=train_op, - eval_metric_ops=eval_metric_ops, - export_outputs=export_outputs) - estimator = tf.estimator.Estimator(model_fn=model_fn) - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - labels: A `Tensor` whose shape matches `predictions`. Will be cast to - `bool`. - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions must - be either `1`, or the same as the corresponding `labels` dimension). - num_thresholds: The number of thresholds to use when discretizing the roc - curve. - metrics_collections: An optional list of collections that `f1_score` should - be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - f1_score: A scalar `Tensor` representing the current best f1-score across - different thresholds. - update_op: An operation that increments the `true_positives`, - `true_negatives`, `false_positives` and `false_negatives` variables - appropriately and whose value matches the `f1_score`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - with variable_scope.variable_scope( - name, 'f1', (labels, predictions, weights)): - predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access - predictions=predictions, labels=labels, weights=weights) - # To account for floating point imprecisions / avoid division by zero. - epsilon = 1e-7 - thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) - for i in range(num_thresholds - 2)] - thresholds = [0.0 - epsilon] + thresholds + [1.0 + epsilon] - - # Confusion matrix. - values, update_ops = metrics_impl._confusion_matrix_at_thresholds( # pylint: disable=protected-access - labels, predictions, thresholds, weights, includes=('tp', 'fp', 'fn')) - - # Compute precision and recall at various thresholds. - def compute_best_f1_score(tp, fp, fn, name): - precision_at_t = math_ops.div(tp, epsilon + tp + fp, - name='precision_' + name) - recall_at_t = math_ops.div(tp, epsilon + tp + fn, name='recall_' + name) - # Compute F1 score. - f1_at_thresholds = ( - 2.0 * precision_at_t * recall_at_t / - (precision_at_t + recall_at_t + epsilon)) - return math_ops.reduce_max(f1_at_thresholds) - - def f1_across_replicas(_, values): - best_f1 = compute_best_f1_score(tp=values['tp'], fp=values['fp'], - fn=values['fn'], name='value') - if metrics_collections: - ops.add_to_collections(metrics_collections, best_f1) - return best_f1 - - best_f1 = distribution_strategy_context.get_replica_context().merge_call( - f1_across_replicas, args=(values,)) - - update_op = compute_best_f1_score(tp=update_ops['tp'], fp=update_ops['fp'], - fn=update_ops['fn'], name='update') - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - - return best_f1, update_op diff --git a/tensorflow/contrib/metrics/python/metrics/classification_test.py b/tensorflow/contrib/metrics/python/metrics/classification_test.py deleted file mode 100644 index e789d2cb9df..00000000000 --- a/tensorflow/contrib/metrics/python/metrics/classification_test.py +++ /dev/null @@ -1,313 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for metrics.classification.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.metrics.python.metrics import classification -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class ClassificationTest(test.TestCase): - - def testAccuracy1D(self): - with self.cached_session() as session: - pred = array_ops.placeholder(dtypes.int32, shape=[None]) - labels = array_ops.placeholder(dtypes.int32, shape=[None]) - acc = classification.accuracy(pred, labels) - result = session.run(acc, - feed_dict={pred: [1, 0, 1, 0], - labels: [1, 1, 0, 0]}) - self.assertEqual(result, 0.5) - - def testAccuracy1DBool(self): - with self.cached_session() as session: - pred = array_ops.placeholder(dtypes.bool, shape=[None]) - labels = array_ops.placeholder(dtypes.bool, shape=[None]) - acc = classification.accuracy(pred, labels) - result = session.run(acc, - feed_dict={pred: [1, 0, 1, 0], - labels: [1, 1, 0, 0]}) - self.assertEqual(result, 0.5) - - def testAccuracy1DInt64(self): - with self.cached_session() as session: - pred = array_ops.placeholder(dtypes.int64, shape=[None]) - labels = array_ops.placeholder(dtypes.int64, shape=[None]) - acc = classification.accuracy(pred, labels) - result = session.run(acc, - feed_dict={pred: [1, 0, 1, 0], - labels: [1, 1, 0, 0]}) - self.assertEqual(result, 0.5) - - def testAccuracy1DString(self): - with self.cached_session() as session: - pred = array_ops.placeholder(dtypes.string, shape=[None]) - labels = array_ops.placeholder(dtypes.string, shape=[None]) - acc = classification.accuracy(pred, labels) - result = session.run( - acc, - feed_dict={pred: ['a', 'b', 'a', 'c'], - labels: ['a', 'c', 'b', 'c']}) - self.assertEqual(result, 0.5) - - def testAccuracyDtypeMismatch(self): - with self.assertRaises(ValueError): - pred = array_ops.placeholder(dtypes.int32, shape=[None]) - labels = array_ops.placeholder(dtypes.int64, shape=[None]) - classification.accuracy(pred, labels) - - def testAccuracyFloatLabels(self): - with self.assertRaises(ValueError): - pred = array_ops.placeholder(dtypes.int32, shape=[None]) - labels = array_ops.placeholder(dtypes.float32, shape=[None]) - classification.accuracy(pred, labels) - - def testAccuracy1DWeighted(self): - with self.cached_session() as session: - pred = array_ops.placeholder(dtypes.int32, shape=[None]) - labels = array_ops.placeholder(dtypes.int32, shape=[None]) - weights = array_ops.placeholder(dtypes.float32, shape=[None]) - acc = classification.accuracy(pred, labels) - result = session.run(acc, - feed_dict={ - pred: [1, 0, 1, 1], - labels: [1, 1, 0, 1], - weights: [3.0, 1.0, 2.0, 0.0] - }) - self.assertEqual(result, 0.5) - - def testAccuracy1DWeightedBroadcast(self): - with self.cached_session() as session: - pred = array_ops.placeholder(dtypes.int32, shape=[None]) - labels = array_ops.placeholder(dtypes.int32, shape=[None]) - weights = array_ops.placeholder(dtypes.float32, shape=[]) - acc = classification.accuracy(pred, labels) - result = session.run(acc, - feed_dict={ - pred: [1, 0, 1, 0], - labels: [1, 1, 0, 0], - weights: 3.0, - }) - self.assertEqual(result, 0.5) - - -class F1ScoreTest(test.TestCase): - - def setUp(self): - super(F1ScoreTest, self).setUp() - np.random.seed(1) - - def testVars(self): - classification.f1_score( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - num_thresholds=3) - expected = {'f1/true_positives:0', 'f1/false_positives:0', - 'f1/false_negatives:0'} - self.assertEquals( - expected, set(v.name for v in variables.local_variables())) - self.assertEquals( - set(expected), set(v.name for v in variables.local_variables())) - self.assertEquals( - set(expected), - set(v.name for v in ops.get_collection(ops.GraphKeys.METRIC_VARIABLES))) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - f1, _ = classification.f1_score( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - num_thresholds=3, - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [f1]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, f1_op = classification.f1_score( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - num_thresholds=3, - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [f1_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes.float32, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes.int64, seed=2) - f1, f1_op = classification.f1_score(predictions, labels, num_thresholds=3) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run([f1_op]) - - # Then verify idempotency. - initial_f1 = f1.eval() - for _ in range(10): - self.assertAllClose(initial_f1, f1.eval()) - - def testAllCorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - with self.cached_session() as sess: - predictions = constant_op.constant(inputs, dtype=dtypes.float32) - labels = constant_op.constant(inputs) - f1, f1_op = classification.f1_score(predictions, labels, num_thresholds=3) - - sess.run(variables.local_variables_initializer()) - sess.run([f1_op]) - - self.assertEqual(1, f1.eval()) - - def testSomeCorrect(self): - predictions = constant_op.constant( - [1, 0, 1, 0], shape=(1, 4), dtype=dtypes.float32) - labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) - f1, f1_op = classification.f1_score(predictions, labels, num_thresholds=1) - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run([f1_op]) - # Threshold 0 will have around 0.5 precision and 1 recall yielding an F1 - # score of 2 * 0.5 * 1 / (1 + 0.5). - self.assertAlmostEqual(2 * 0.5 * 1 / (1 + 0.5), f1.eval()) - - def testAllIncorrect(self): - inputs = np.random.randint(0, 2, size=(10000, 1)) - - with self.cached_session() as sess: - predictions = constant_op.constant(inputs, dtype=dtypes.float32) - labels = constant_op.constant(1 - inputs, dtype=dtypes.float32) - f1, f1_op = classification.f1_score(predictions, labels, num_thresholds=3) - - sess.run(variables.local_variables_initializer()) - sess.run([f1_op]) - - # Threshold 0 will have around 0.5 precision and 1 recall yielding an F1 - # score of 2 * 0.5 * 1 / (1 + 0.5). - self.assertAlmostEqual(2 * 0.5 * 1 / (1 + 0.5), f1.eval(), places=2) - - def testWeights1d(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes.float32) - labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) - weights = constant_op.constant( - [[0], [1]], shape=(2, 1), dtype=dtypes.float32) - f1, f1_op = classification.f1_score(predictions, labels, weights, - num_thresholds=3) - sess.run(variables.local_variables_initializer()) - sess.run([f1_op]) - - self.assertAlmostEqual(1.0, f1.eval(), places=5) - - def testWeights2d(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes.float32) - labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) - weights = constant_op.constant( - [[0, 0], [1, 1]], shape=(2, 2), dtype=dtypes.float32) - f1, f1_op = classification.f1_score(predictions, labels, weights, - num_thresholds=3) - sess.run(variables.local_variables_initializer()) - sess.run([f1_op]) - - self.assertAlmostEqual(1.0, f1.eval(), places=5) - - def testZeroLabelsPredictions(self): - with self.cached_session() as sess: - predictions = array_ops.zeros([4], dtype=dtypes.float32) - labels = array_ops.zeros([4]) - f1, f1_op = classification.f1_score(predictions, labels, num_thresholds=3) - sess.run(variables.local_variables_initializer()) - sess.run([f1_op]) - - self.assertAlmostEqual(0.0, f1.eval(), places=5) - - def testWithMultipleUpdates(self): - num_samples = 1000 - batch_size = 10 - num_batches = int(num_samples / batch_size) - - # Create the labels and data. - labels = np.random.randint(0, 2, size=(num_samples, 1)) - noise = np.random.normal(0.0, scale=0.2, size=(num_samples, 1)) - predictions = 0.4 + 0.2 * labels + noise - predictions[predictions > 1] = 1 - predictions[predictions < 0] = 0 - thresholds = [-0.01, 0.5, 1.01] - - expected_max_f1 = -1.0 - for threshold in thresholds: - tp = 0 - fp = 0 - fn = 0 - tn = 0 - for i in range(num_samples): - if predictions[i] >= threshold: - if labels[i] == 1: - tp += 1 - else: - fp += 1 - else: - if labels[i] == 1: - fn += 1 - else: - tn += 1 - epsilon = 1e-7 - expected_prec = tp / (epsilon + tp + fp) - expected_rec = tp / (epsilon + tp + fn) - expected_f1 = (2 * expected_prec * expected_rec / - (epsilon + expected_prec + expected_rec)) - if expected_f1 > expected_max_f1: - expected_max_f1 = expected_f1 - - labels = labels.astype(np.float32) - predictions = predictions.astype(np.float32) - tf_predictions, tf_labels = dataset_ops.make_one_shot_iterator( - dataset_ops.Dataset - .from_tensor_slices((predictions, labels)) - .repeat() - .batch(batch_size)).get_next() - f1, f1_op = classification.f1_score(tf_labels, tf_predictions, - num_thresholds=3) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - for _ in range(num_batches): - sess.run([f1_op]) - # Since this is only approximate, we can't expect a 6 digits match. - # Although with higher number of samples/thresholds we should see the - # accuracy improving - self.assertAlmostEqual(expected_max_f1, f1.eval(), 2) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/metrics/python/ops/confusion_matrix_ops.py b/tensorflow/contrib/metrics/python/ops/confusion_matrix_ops.py deleted file mode 100644 index 1fb15bfcd6d..00000000000 --- a/tensorflow/contrib/metrics/python/ops/confusion_matrix_ops.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Confusion matrix related metrics.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import confusion_matrix as cm - - -def confusion_matrix(labels, predictions, num_classes=None, dtype=dtypes.int32, - name=None, weights=None): - """Deprecated. Use tf.math.confusion_matrix instead.""" - return cm.confusion_matrix(labels=labels, predictions=predictions, - num_classes=num_classes, dtype=dtype, name=name, - weights=weights) diff --git a/tensorflow/contrib/metrics/python/ops/histogram_ops.py b/tensorflow/contrib/metrics/python/ops/histogram_ops.py deleted file mode 100644 index d3d74d28a3a..00000000000 --- a/tensorflow/contrib/metrics/python/ops/histogram_ops.py +++ /dev/null @@ -1,243 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# pylint: disable=g-short-docstring-punctuation -"""Metrics that use histograms. - -Module documentation, including "@@" callouts, should be put in -third_party/tensorflow/contrib/metrics/__init__.py -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework.python.framework import tensor_util -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import histogram_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope - - -def auc_using_histogram(boolean_labels, - scores, - score_range, - nbins=100, - collections=None, - check_shape=True, - name=None): - """AUC computed by maintaining histograms. - - Rather than computing AUC directly, this Op maintains Variables containing - histograms of the scores associated with `True` and `False` labels. By - comparing these the AUC is generated, with some discretization error. - See: "Efficient AUC Learning Curve Calculation" by Bouckaert. - - This AUC Op updates in `O(batch_size + nbins)` time and works well even with - large class imbalance. The accuracy is limited by discretization error due - to finite number of bins. If scores are concentrated in a fewer bins, - accuracy is lower. If this is a concern, we recommend trying different - numbers of bins and comparing results. - - Args: - boolean_labels: 1-D boolean `Tensor`. Entry is `True` if the corresponding - record is in class. - scores: 1-D numeric `Tensor`, same shape as boolean_labels. - score_range: `Tensor` of shape `[2]`, same dtype as `scores`. The min/max - values of score that we expect. Scores outside range will be clipped. - nbins: Integer number of bins to use. Accuracy strictly increases as the - number of bins increases. - collections: List of graph collections keys. Internal histogram Variables - are added to these collections. Defaults to `[GraphKeys.LOCAL_VARIABLES]`. - check_shape: Boolean. If `True`, do a runtime shape check on the scores - and labels. - name: A name for this Op. Defaults to "auc_using_histogram". - - Returns: - auc: `float32` scalar `Tensor`. Fetching this converts internal histograms - to auc value. - update_op: `Op`, when run, updates internal histograms. - """ - if collections is None: - collections = [ops.GraphKeys.LOCAL_VARIABLES] - with variable_scope.variable_scope( - name, 'auc_using_histogram', [boolean_labels, scores, score_range]): - scores, boolean_labels = tensor_util.remove_squeezable_dimensions( - scores, boolean_labels) - score_range = ops.convert_to_tensor(score_range, name='score_range') - boolean_labels, scores = _check_labels_and_scores( - boolean_labels, scores, check_shape) - hist_true, hist_false = _make_auc_histograms(boolean_labels, scores, - score_range, nbins) - hist_true_acc, hist_false_acc, update_op = _auc_hist_accumulate(hist_true, - hist_false, - nbins, - collections) - auc = _auc_convert_hist_to_auc(hist_true_acc, hist_false_acc, nbins) - return auc, update_op - - -def _check_labels_and_scores(boolean_labels, scores, check_shape): - """Check the rank of labels/scores, return tensor versions.""" - with ops.name_scope('_check_labels_and_scores', - values=[boolean_labels, scores]): - boolean_labels = ops.convert_to_tensor(boolean_labels, - name='boolean_labels') - scores = ops.convert_to_tensor(scores, name='scores') - - if boolean_labels.dtype != dtypes.bool: - raise ValueError( - 'Argument boolean_labels should have dtype bool. Found: %s', - boolean_labels.dtype) - - if check_shape: - labels_rank_1 = control_flow_ops.Assert( - math_ops.equal(1, array_ops.rank(boolean_labels)), - ['Argument boolean_labels should have rank 1. Found: ', - boolean_labels.name, array_ops.shape(boolean_labels)]) - - scores_rank_1 = control_flow_ops.Assert( - math_ops.equal(1, array_ops.rank(scores)), - ['Argument scores should have rank 1. Found: ', scores.name, - array_ops.shape(scores)]) - - with ops.control_dependencies([labels_rank_1, scores_rank_1]): - return boolean_labels, scores - else: - return boolean_labels, scores - - -def _make_auc_histograms(boolean_labels, scores, score_range, nbins): - """Create histogram tensors from one batch of labels/scores.""" - - with variable_scope.variable_scope( - None, 'make_auc_histograms', [boolean_labels, scores, nbins]): - # Histogram of scores for records in this batch with True label. - hist_true = histogram_ops.histogram_fixed_width( - array_ops.boolean_mask(scores, boolean_labels), - score_range, - nbins=nbins, - dtype=dtypes.int64, - name='hist_true') - # Histogram of scores for records in this batch with False label. - hist_false = histogram_ops.histogram_fixed_width( - array_ops.boolean_mask(scores, math_ops.logical_not(boolean_labels)), - score_range, - nbins=nbins, - dtype=dtypes.int64, - name='hist_false') - return hist_true, hist_false - - -def _auc_hist_accumulate(hist_true, hist_false, nbins, collections): - """Accumulate histograms in new variables.""" - with variable_scope.variable_scope( - None, 'hist_accumulate', [hist_true, hist_false]): - # Holds running total histogram of scores for records labeled True. - hist_true_acc = variable_scope.get_variable( - 'hist_true_acc', - shape=[nbins], - dtype=hist_true.dtype, - initializer=init_ops.zeros_initializer(), - collections=collections, - trainable=False) - # Holds running total histogram of scores for records labeled False. - hist_false_acc = variable_scope.get_variable( - 'hist_false_acc', - shape=[nbins], - dtype=hist_true.dtype, - initializer=init_ops.zeros_initializer(), - collections=collections, - trainable=False) - - update_op = control_flow_ops.group( - hist_true_acc.assign_add(hist_true), - hist_false_acc.assign_add(hist_false), - name='update_op') - - return hist_true_acc, hist_false_acc, update_op - - -def _auc_convert_hist_to_auc(hist_true_acc, hist_false_acc, nbins): - """Convert histograms to auc. - - Args: - hist_true_acc: `Tensor` holding accumulated histogram of scores for records - that were `True`. - hist_false_acc: `Tensor` holding accumulated histogram of scores for - records that were `False`. - nbins: Integer number of bins in the histograms. - - Returns: - Scalar `Tensor` estimating AUC. - """ - # Note that this follows the "Approximating AUC" section in: - # Efficient AUC learning curve calculation, R. R. Bouckaert, - # AI'06 Proceedings of the 19th Australian joint conference on Artificial - # Intelligence: advances in Artificial Intelligence - # Pages 181-191. - # Note that the above paper has an error, and we need to re-order our bins to - # go from high to low score. - - # Normalize histogram so we get fraction in each bin. - normed_hist_true = math_ops.truediv(hist_true_acc, - math_ops.reduce_sum(hist_true_acc)) - normed_hist_false = math_ops.truediv(hist_false_acc, - math_ops.reduce_sum(hist_false_acc)) - - # These become delta x, delta y from the paper. - delta_y_t = array_ops.reverse_v2(normed_hist_true, [0], name='delta_y_t') - delta_x_t = array_ops.reverse_v2(normed_hist_false, [0], name='delta_x_t') - - # strict_1d_cumsum requires float32 args. - delta_y_t = math_ops.cast(delta_y_t, dtypes.float32) - delta_x_t = math_ops.cast(delta_x_t, dtypes.float32) - - # Trapezoidal integration, \int_0^1 0.5 * (y_t + y_{t-1}) dx_t - y_t = _strict_1d_cumsum(delta_y_t, nbins) - first_trap = delta_x_t[0] * y_t[0] / 2.0 - other_traps = delta_x_t[1:] * (y_t[1:] + y_t[:nbins - 1]) / 2.0 - return math_ops.add(first_trap, math_ops.reduce_sum(other_traps), name='auc') - - -# TODO(langmore) Remove once a faster cumsum (accumulate_sum) Op is available. -# Also see if cast to float32 above can be removed with new cumsum. -# See: https://github.com/tensorflow/tensorflow/issues/813 -def _strict_1d_cumsum(tensor, len_tensor): - """Cumsum of a 1D tensor with defined shape by padding and convolving.""" - # Assumes tensor shape is fully defined. - with ops.name_scope('strict_1d_cumsum', values=[tensor]): - if len_tensor == 0: - return constant_op.constant([]) - len_pad = len_tensor - 1 - x = array_ops.pad(tensor, [[len_pad, 0]]) - h = array_ops.ones_like(x) - return _strict_conv1d(x, h)[:len_tensor] - - -# TODO(langmore) Remove once a faster cumsum (accumulate_sum) Op is available. -# See: https://github.com/tensorflow/tensorflow/issues/813 -def _strict_conv1d(x, h): - """Return x * h for rank 1 tensors x and h.""" - with ops.name_scope('strict_conv1d', values=[x, h]): - x = array_ops.reshape(x, (1, -1, 1, 1)) - h = array_ops.reshape(h, (-1, 1, 1, 1)) - result = nn_ops.conv2d(x, h, [1, 1, 1, 1], 'SAME') - return array_ops.reshape(result, [-1]) diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py deleted file mode 100644 index e46263b48a6..00000000000 --- a/tensorflow/contrib/metrics/python/ops/metric_ops.py +++ /dev/null @@ -1,3970 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains metric-computing operations on streamed tensors. - -Module documentation, including "@@" callouts, should be put in -third_party/tensorflow/contrib/metrics/__init__.py -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections as collections_lib - -from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import confusion_matrix -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import metrics -from tensorflow.python.ops import metrics_impl -from tensorflow.python.ops import nn -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import weights_broadcast_ops -from tensorflow.python.ops.distributions.normal import Normal -from tensorflow.python.util.deprecation import deprecated - -# Epsilon constant used to represent extremely small quantity. -_EPSILON = 1e-7 - - -@deprecated(None, 'Please switch to tf.metrics.true_positives. Note that the ' - 'order of the labels and predictions arguments has been switched.') -def streaming_true_positives(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Sum the weights of true_positives. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will - be cast to `bool`. - labels: The ground truth values, a `Tensor` whose dimensions must match - `predictions`. Will be cast to `bool`. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions must - be either `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that the metric value - variable should be added to. - updates_collections: An optional list of collections that the metric update - ops should be added to. - name: An optional variable_scope name. - - Returns: - value_tensor: A `Tensor` representing the current value of the metric. - update_op: An operation that accumulates the error from a batch of data. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.true_positives( - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -@deprecated(None, 'Please switch to tf.metrics.true_negatives. Note that the ' - 'order of the labels and predictions arguments has been switched.') -def streaming_true_negatives(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Sum the weights of true_negatives. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will - be cast to `bool`. - labels: The ground truth values, a `Tensor` whose dimensions must match - `predictions`. Will be cast to `bool`. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions must - be either `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that the metric value - variable should be added to. - updates_collections: An optional list of collections that the metric update - ops should be added to. - name: An optional variable_scope name. - - Returns: - value_tensor: A `Tensor` representing the current value of the metric. - update_op: An operation that accumulates the error from a batch of data. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.true_negatives( - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -@deprecated(None, 'Please switch to tf.metrics.false_positives. Note that the ' - 'order of the labels and predictions arguments has been switched.') -def streaming_false_positives(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Sum the weights of false positives. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will - be cast to `bool`. - labels: The ground truth values, a `Tensor` whose dimensions must match - `predictions`. Will be cast to `bool`. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions must - be either `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that the metric value - variable should be added to. - updates_collections: An optional list of collections that the metric update - ops should be added to. - name: An optional variable_scope name. - - Returns: - value_tensor: A `Tensor` representing the current value of the metric. - update_op: An operation that accumulates the error from a batch of data. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.false_positives( - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -@deprecated(None, 'Please switch to tf.metrics.false_negatives. Note that the ' - 'order of the labels and predictions arguments has been switched.') -def streaming_false_negatives(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the total number of false negatives. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will - be cast to `bool`. - labels: The ground truth values, a `Tensor` whose dimensions must match - `predictions`. Will be cast to `bool`. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions must - be either `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that the metric value - variable should be added to. - updates_collections: An optional list of collections that the metric update - ops should be added to. - name: An optional variable_scope name. - - Returns: - value_tensor: A `Tensor` representing the current value of the metric. - update_op: An operation that accumulates the error from a batch of data. - - Raises: - ValueError: If `weights` is not `None` and its shape doesn't match `values`, - or if either `metrics_collections` or `updates_collections` are not a list - or tuple. - """ - return metrics.false_negatives( - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -@deprecated(None, 'Please switch to tf.metrics.mean') -def streaming_mean(values, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the (weighted) mean of the given values. - - The `streaming_mean` function creates two local variables, `total` and `count` - that are used to compute the average of `values`. This average is ultimately - returned as `mean` which is an idempotent operation that simply divides - `total` by `count`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the `mean`. - `update_op` increments `total` with the reduced sum of the product of `values` - and `weights`, and it increments `count` with the reduced sum of `weights`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - values: A `Tensor` of arbitrary dimensions. - weights: `Tensor` whose rank is either 0, or the same rank as `values`, and - must be broadcastable to `values` (i.e., all dimensions must be either - `1`, or the same as the corresponding `values` dimension). - metrics_collections: An optional list of collections that `mean` should be - added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - mean: A `Tensor` representing the current mean, the value of `total` divided - by `count`. - update_op: An operation that increments the `total` and `count` variables - appropriately and whose value matches `mean`. - - Raises: - ValueError: If `weights` is not `None` and its shape doesn't match `values`, - or if either `metrics_collections` or `updates_collections` are not a list - or tuple. - """ - return metrics.mean( - values=values, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -@deprecated(None, 'Please switch to tf.metrics.mean_tensor') -def streaming_mean_tensor(values, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the element-wise (weighted) mean of the given tensors. - - In contrast to the `streaming_mean` function which returns a scalar with the - mean, this function returns an average tensor with the same shape as the - input tensors. - - The `streaming_mean_tensor` function creates two local variables, - `total_tensor` and `count_tensor` that are used to compute the average of - `values`. This average is ultimately returned as `mean` which is an idempotent - operation that simply divides `total` by `count`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the `mean`. - `update_op` increments `total` with the reduced sum of the product of `values` - and `weights`, and it increments `count` with the reduced sum of `weights`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - values: A `Tensor` of arbitrary dimensions. - weights: `Tensor` whose rank is either 0, or the same rank as `values`, and - must be broadcastable to `values` (i.e., all dimensions must be either - `1`, or the same as the corresponding `values` dimension). - metrics_collections: An optional list of collections that `mean` should be - added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - mean: A float `Tensor` representing the current mean, the value of `total` - divided by `count`. - update_op: An operation that increments the `total` and `count` variables - appropriately and whose value matches `mean`. - - Raises: - ValueError: If `weights` is not `None` and its shape doesn't match `values`, - or if either `metrics_collections` or `updates_collections` are not a list - or tuple. - """ - return metrics.mean_tensor( - values=values, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -@deprecated(None, 'Please switch to tf.metrics.accuracy. Note that the order ' - 'of the labels and predictions arguments has been switched.') -def streaming_accuracy(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Calculates how often `predictions` matches `labels`. - - The `streaming_accuracy` function creates two local variables, `total` and - `count` that are used to compute the frequency with which `predictions` - matches `labels`. This frequency is ultimately returned as `accuracy`: an - idempotent operation that simply divides `total` by `count`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the `accuracy`. - Internally, an `is_correct` operation computes a `Tensor` with elements 1.0 - where the corresponding elements of `predictions` and `labels` match and 0.0 - otherwise. Then `update_op` increments `total` with the reduced sum of the - product of `weights` and `is_correct`, and it increments `count` with the - reduced sum of `weights`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: The predicted values, a `Tensor` of any shape. - labels: The ground truth values, a `Tensor` whose shape matches - `predictions`. - weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and - must be broadcastable to `labels` (i.e., all dimensions must be either - `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that `accuracy` should - be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - accuracy: A `Tensor` representing the accuracy, the value of `total` divided - by `count`. - update_op: An operation that increments the `total` and `count` variables - appropriately and whose value matches `accuracy`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.accuracy( - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -@deprecated(None, 'Please switch to tf.metrics.precision. Note that the order ' - 'of the labels and predictions arguments has been switched.') -def streaming_precision(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the precision of the predictions with respect to the labels. - - The `streaming_precision` function creates two local variables, - `true_positives` and `false_positives`, that are used to compute the - precision. This value is ultimately returned as `precision`, an idempotent - operation that simply divides `true_positives` by the sum of `true_positives` - and `false_positives`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `precision`. `update_op` weights each prediction by the corresponding value in - `weights`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: The predicted values, a `bool` `Tensor` of arbitrary shape. - labels: The ground truth values, a `bool` `Tensor` whose dimensions must - match `predictions`. - weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and - must be broadcastable to `labels` (i.e., all dimensions must be either - `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that `precision` should - be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - precision: Scalar float `Tensor` with the value of `true_positives` - divided by the sum of `true_positives` and `false_positives`. - update_op: `Operation` that increments `true_positives` and - `false_positives` variables appropriately and whose value matches - `precision`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.precision( - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -@deprecated(None, 'Please switch to tf.metrics.recall. Note that the order ' - 'of the labels and predictions arguments has been switched.') -def streaming_recall(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the recall of the predictions with respect to the labels. - - The `streaming_recall` function creates two local variables, `true_positives` - and `false_negatives`, that are used to compute the recall. This value is - ultimately returned as `recall`, an idempotent operation that simply divides - `true_positives` by the sum of `true_positives` and `false_negatives`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` that updates these variables and returns the `recall`. `update_op` - weights each prediction by the corresponding value in `weights`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: The predicted values, a `bool` `Tensor` of arbitrary shape. - labels: The ground truth values, a `bool` `Tensor` whose dimensions must - match `predictions`. - weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and - must be broadcastable to `labels` (i.e., all dimensions must be either - `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that `recall` should be - added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - recall: Scalar float `Tensor` with the value of `true_positives` divided - by the sum of `true_positives` and `false_negatives`. - update_op: `Operation` that increments `true_positives` and - `false_negatives` variables appropriately and whose value matches - `recall`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.recall( - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -def streaming_false_positive_rate(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the false positive rate of predictions with respect to labels. - - The `false_positive_rate` function creates two local variables, - `false_positives` and `true_negatives`, that are used to compute the - false positive rate. This value is ultimately returned as - `false_positive_rate`, an idempotent operation that simply divides - `false_positives` by the sum of `false_positives` and `true_negatives`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `false_positive_rate`. `update_op` weights each prediction by the - corresponding value in `weights`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will - be cast to `bool`. - labels: The ground truth values, a `Tensor` whose dimensions must match - `predictions`. Will be cast to `bool`. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions must - be either `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that - `false_positive_rate` should be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - false_positive_rate: Scalar float `Tensor` with the value of - `false_positives` divided by the sum of `false_positives` and - `true_negatives`. - update_op: `Operation` that increments `false_positives` and - `true_negatives` variables appropriately and whose value matches - `false_positive_rate`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - with variable_scope.variable_scope(name, 'false_positive_rate', - (predictions, labels, weights)): - predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) - - false_p, false_positives_update_op = metrics.false_positives( - labels=labels, - predictions=predictions, - weights=weights, - metrics_collections=None, - updates_collections=None, - name=None) - true_n, true_negatives_update_op = metrics.true_negatives( - labels=labels, - predictions=predictions, - weights=weights, - metrics_collections=None, - updates_collections=None, - name=None) - - def compute_fpr(fp, tn, name): - return array_ops.where( - math_ops.greater(fp + tn, 0), math_ops.div(fp, fp + tn), 0, name) - - fpr = compute_fpr(false_p, true_n, 'value') - update_op = compute_fpr(false_positives_update_op, true_negatives_update_op, - 'update_op') - - if metrics_collections: - ops.add_to_collections(metrics_collections, fpr) - - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - - return fpr, update_op - - -def streaming_false_negative_rate(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the false negative rate of predictions with respect to labels. - - The `false_negative_rate` function creates two local variables, - `false_negatives` and `true_positives`, that are used to compute the - false positive rate. This value is ultimately returned as - `false_negative_rate`, an idempotent operation that simply divides - `false_negatives` by the sum of `false_negatives` and `true_positives`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `false_negative_rate`. `update_op` weights each prediction by the - corresponding value in `weights`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will - be cast to `bool`. - labels: The ground truth values, a `Tensor` whose dimensions must match - `predictions`. Will be cast to `bool`. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions must - be either `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that - `false_negative_rate` should be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - false_negative_rate: Scalar float `Tensor` with the value of - `false_negatives` divided by the sum of `false_negatives` and - `true_positives`. - update_op: `Operation` that increments `false_negatives` and - `true_positives` variables appropriately and whose value matches - `false_negative_rate`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - with variable_scope.variable_scope(name, 'false_negative_rate', - (predictions, labels, weights)): - predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access - predictions=math_ops.cast(predictions, dtype=dtypes.bool), - labels=math_ops.cast(labels, dtype=dtypes.bool), - weights=weights) - - false_n, false_negatives_update_op = metrics.false_negatives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None) - true_p, true_positives_update_op = metrics.true_positives( - labels, - predictions, - weights, - metrics_collections=None, - updates_collections=None, - name=None) - - def compute_fnr(fn, tp, name): - return array_ops.where( - math_ops.greater(fn + tp, 0), math_ops.div(fn, fn + tp), 0, name) - - fnr = compute_fnr(false_n, true_p, 'value') - update_op = compute_fnr(false_negatives_update_op, true_positives_update_op, - 'update_op') - - if metrics_collections: - ops.add_to_collections(metrics_collections, fnr) - - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - - return fnr, update_op - - -def _streaming_confusion_matrix_at_thresholds(predictions, - labels, - thresholds, - weights=None, - includes=None): - """Computes true_positives, false_negatives, true_negatives, false_positives. - - This function creates up to four local variables, `true_positives`, - `true_negatives`, `false_positives` and `false_negatives`. - `true_positive[i]` is defined as the total weight of values in `predictions` - above `thresholds[i]` whose corresponding entry in `labels` is `True`. - `false_negatives[i]` is defined as the total weight of values in `predictions` - at most `thresholds[i]` whose corresponding entry in `labels` is `True`. - `true_negatives[i]` is defined as the total weight of values in `predictions` - at most `thresholds[i]` whose corresponding entry in `labels` is `False`. - `false_positives[i]` is defined as the total weight of values in `predictions` - above `thresholds[i]` whose corresponding entry in `labels` is `False`. - - For estimation of these metrics over a stream of data, for each metric the - function respectively creates an `update_op` operation that updates the - variable and returns its value. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - labels: A `Tensor` whose shape matches `predictions`. `labels` will be cast - to `bool`. - thresholds: A python list or tuple of float thresholds in `[0, 1]`. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions must - be either `1`, or the same as the corresponding `labels` dimension). - includes: Tuple of keys to return, from 'tp', 'fn', 'tn', fp'. If `None`, - default to all four. - - Returns: - values: Dict of variables of shape `[len(thresholds)]`. Keys are from - `includes`. - update_ops: Dict of operations that increments the `values`. Keys are from - `includes`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - `includes` contains invalid keys. - """ - all_includes = ('tp', 'fn', 'tn', 'fp') - if includes is None: - includes = all_includes - else: - for include in includes: - if include not in all_includes: - raise ValueError('Invalid key: %s.' % include) - - predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access - predictions, labels, weights) - predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - - num_thresholds = len(thresholds) - - # Reshape predictions and labels. - predictions_2d = array_ops.reshape(predictions, [-1, 1]) - labels_2d = array_ops.reshape( - math_ops.cast(labels, dtype=dtypes.bool), [1, -1]) - - # Use static shape if known. - num_predictions = predictions_2d.get_shape().as_list()[0] - - # Otherwise use dynamic shape. - if num_predictions is None: - num_predictions = array_ops.shape(predictions_2d)[0] - thresh_tiled = array_ops.tile( - array_ops.expand_dims(array_ops.constant(thresholds), [1]), - array_ops.stack([1, num_predictions])) - - # Tile the predictions after thresholding them across different thresholds. - pred_is_pos = math_ops.greater( - array_ops.tile(array_ops.transpose(predictions_2d), [num_thresholds, 1]), - thresh_tiled) - if ('fn' in includes) or ('tn' in includes): - pred_is_neg = math_ops.logical_not(pred_is_pos) - - # Tile labels by number of thresholds - label_is_pos = array_ops.tile(labels_2d, [num_thresholds, 1]) - if ('fp' in includes) or ('tn' in includes): - label_is_neg = math_ops.logical_not(label_is_pos) - - if weights is not None: - broadcast_weights = weights_broadcast_ops.broadcast_weights( - math_ops.cast(weights, dtypes.float32), predictions) - weights_tiled = array_ops.tile( - array_ops.reshape(broadcast_weights, [1, -1]), [num_thresholds, 1]) - thresh_tiled.get_shape().assert_is_compatible_with( - weights_tiled.get_shape()) - else: - weights_tiled = None - - values = {} - update_ops = {} - - if 'tp' in includes: - true_positives = metrics_impl.metric_variable([num_thresholds], - dtypes.float32, - name='true_positives') - is_true_positive = math_ops.cast( - math_ops.logical_and(label_is_pos, pred_is_pos), dtypes.float32) - if weights_tiled is not None: - is_true_positive *= weights_tiled - update_ops['tp'] = state_ops.assign_add( - true_positives, math_ops.reduce_sum(is_true_positive, 1)) - values['tp'] = true_positives - - if 'fn' in includes: - false_negatives = metrics_impl.metric_variable([num_thresholds], - dtypes.float32, - name='false_negatives') - is_false_negative = math_ops.cast( - math_ops.logical_and(label_is_pos, pred_is_neg), dtypes.float32) - if weights_tiled is not None: - is_false_negative *= weights_tiled - update_ops['fn'] = state_ops.assign_add( - false_negatives, math_ops.reduce_sum(is_false_negative, 1)) - values['fn'] = false_negatives - - if 'tn' in includes: - true_negatives = metrics_impl.metric_variable([num_thresholds], - dtypes.float32, - name='true_negatives') - is_true_negative = math_ops.cast( - math_ops.logical_and(label_is_neg, pred_is_neg), dtypes.float32) - if weights_tiled is not None: - is_true_negative *= weights_tiled - update_ops['tn'] = state_ops.assign_add( - true_negatives, math_ops.reduce_sum(is_true_negative, 1)) - values['tn'] = true_negatives - - if 'fp' in includes: - false_positives = metrics_impl.metric_variable([num_thresholds], - dtypes.float32, - name='false_positives') - is_false_positive = math_ops.cast( - math_ops.logical_and(label_is_neg, pred_is_pos), dtypes.float32) - if weights_tiled is not None: - is_false_positive *= weights_tiled - update_ops['fp'] = state_ops.assign_add( - false_positives, math_ops.reduce_sum(is_false_positive, 1)) - values['fp'] = false_positives - - return values, update_ops - - -def streaming_true_positives_at_thresholds(predictions, - labels, - thresholds, - weights=None): - values, update_ops = _streaming_confusion_matrix_at_thresholds( - predictions, labels, thresholds, weights=weights, includes=('tp',)) - return values['tp'], update_ops['tp'] - - -def streaming_false_negatives_at_thresholds(predictions, - labels, - thresholds, - weights=None): - values, update_ops = _streaming_confusion_matrix_at_thresholds( - predictions, labels, thresholds, weights=weights, includes=('fn',)) - return values['fn'], update_ops['fn'] - - -def streaming_false_positives_at_thresholds(predictions, - labels, - thresholds, - weights=None): - values, update_ops = _streaming_confusion_matrix_at_thresholds( - predictions, labels, thresholds, weights=weights, includes=('fp',)) - return values['fp'], update_ops['fp'] - - -def streaming_true_negatives_at_thresholds(predictions, - labels, - thresholds, - weights=None): - values, update_ops = _streaming_confusion_matrix_at_thresholds( - predictions, labels, thresholds, weights=weights, includes=('tn',)) - return values['tn'], update_ops['tn'] - - -def streaming_curve_points(labels=None, - predictions=None, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - curve='ROC', - name=None): - """Computes curve (ROC or PR) values for a prespecified number of points. - - The `streaming_curve_points` function creates four local variables, - `true_positives`, `true_negatives`, `false_positives` and `false_negatives` - that are used to compute the curve values. To discretize the curve, a linearly - spaced set of thresholds is used to compute pairs of recall and precision - values. - - For best results, `predictions` should be distributed approximately uniformly - in the range [0, 1] and not peaked around 0 or 1. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - labels: A `Tensor` whose shape matches `predictions`. Will be cast to - `bool`. - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions must - be either `1`, or the same as the corresponding `labels` dimension). - num_thresholds: The number of thresholds to use when discretizing the roc - curve. - metrics_collections: An optional list of collections that `auc` should be - added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - curve: Specifies the name of the curve to be computed, 'ROC' [default] or - 'PR' for the Precision-Recall-curve. - name: An optional variable_scope name. - - Returns: - points: A `Tensor` with shape [num_thresholds, 2] that contains points of - the curve. - update_op: An operation that increments the `true_positives`, - `true_negatives`, `false_positives` and `false_negatives` variables. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - - TODO(chizeng): Consider rewriting this method to make use of logic within the - precision_recall_at_equal_thresholds method (to improve run time). - """ - with variable_scope.variable_scope(name, 'curve_points', - (labels, predictions, weights)): - if curve != 'ROC' and curve != 'PR': - raise ValueError('curve must be either ROC or PR, %s unknown' % (curve)) - kepsilon = _EPSILON # to account for floating point imprecisions - thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) - ] - thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] - - values, update_ops = _streaming_confusion_matrix_at_thresholds( - labels=labels, - predictions=predictions, - thresholds=thresholds, - weights=weights) - - # Add epsilons to avoid dividing by 0. - epsilon = 1.0e-6 - - def compute_points(tp, fn, tn, fp): - """Computes the roc-auc or pr-auc based on confusion counts.""" - rec = math_ops.div(tp + epsilon, tp + fn + epsilon) - if curve == 'ROC': - fp_rate = math_ops.div(fp, fp + tn + epsilon) - return fp_rate, rec - else: # curve == 'PR'. - prec = math_ops.div(tp + epsilon, tp + fp + epsilon) - return rec, prec - - xs, ys = compute_points(values['tp'], values['fn'], values['tn'], - values['fp']) - points = array_ops.stack([xs, ys], axis=1) - update_op = control_flow_ops.group(*update_ops.values()) - - if metrics_collections: - ops.add_to_collections(metrics_collections, points) - - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - - return points, update_op - - -@deprecated(None, 'Please switch to tf.metrics.auc. Note that the order of ' - 'the labels and predictions arguments has been switched.') -def streaming_auc(predictions, - labels, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - curve='ROC', - name=None): - """Computes the approximate AUC via a Riemann sum. - - The `streaming_auc` function creates four local variables, `true_positives`, - `true_negatives`, `false_positives` and `false_negatives` that are used to - compute the AUC. To discretize the AUC curve, a linearly spaced set of - thresholds is used to compute pairs of recall and precision values. The area - under the ROC-curve is therefore computed using the height of the recall - values by the false positive rate, while the area under the PR-curve is the - computed using the height of the precision values by the recall. - - This value is ultimately returned as `auc`, an idempotent operation that - computes the area under a discretized curve of precision versus recall values - (computed using the aforementioned variables). The `num_thresholds` variable - controls the degree of discretization with larger numbers of thresholds more - closely approximating the true AUC. The quality of the approximation may vary - dramatically depending on `num_thresholds`. - - For best results, `predictions` should be distributed approximately uniformly - in the range [0, 1] and not peaked around 0 or 1. The quality of the AUC - approximation may be poor if this is not the case. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the `auc`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - labels: A `bool` `Tensor` whose shape matches `predictions`. - weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and - must be broadcastable to `labels` (i.e., all dimensions must be either - `1`, or the same as the corresponding `labels` dimension). - num_thresholds: The number of thresholds to use when discretizing the roc - curve. - metrics_collections: An optional list of collections that `auc` should be - added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - curve: Specifies the name of the curve to be computed, 'ROC' [default] or - 'PR' for the Precision-Recall-curve. - name: An optional variable_scope name. - - Returns: - auc: A scalar `Tensor` representing the current area-under-curve. - update_op: An operation that increments the `true_positives`, - `true_negatives`, `false_positives` and `false_negatives` variables - appropriately and whose value matches `auc`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.auc( - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - num_thresholds=num_thresholds, - curve=curve, - updates_collections=updates_collections, - name=name) - - -def _compute_dynamic_auc(labels, predictions, curve='ROC', weights=None): - """Computes the apporixmate AUC by a Riemann sum with data-derived thresholds. - - Computes the area under the ROC or PR curve using each prediction as a - threshold. This could be slow for large batches, but has the advantage of not - having its results degrade depending on the distribution of predictions. - - Args: - labels: A `Tensor` of ground truth labels with the same shape as - `predictions` with values of 0 or 1 and type `int64`. - predictions: A 1-D `Tensor` of predictions whose values are `float64`. - curve: The name of the curve to be computed, 'ROC' for the Receiving - Operating Characteristic or 'PR' for the Precision-Recall curve. - weights: A 1-D `Tensor` of weights whose values are `float64`. - - Returns: - A scalar `Tensor` containing the area-under-curve value for the input. - """ - # Compute the total weight and the total positive weight. - size = array_ops.size(predictions) - if weights is None: - weights = array_ops.ones_like(labels, dtype=dtypes.float64) - labels, predictions, weights = metrics_impl._remove_squeezable_dimensions( - labels, predictions, weights) - total_weight = math_ops.reduce_sum(weights) - total_positive = math_ops.reduce_sum( - array_ops.where( - math_ops.greater(labels, 0), weights, - array_ops.zeros_like(labels, dtype=dtypes.float64))) - - def continue_computing_dynamic_auc(): - """Continues dynamic auc computation, entered if labels are not all equal. - - Returns: - A scalar `Tensor` containing the area-under-curve value. - """ - # Sort the predictions descending, keeping the same order for the - # corresponding labels and weights. - ordered_predictions, indices = nn.top_k(predictions, k=size) - ordered_labels = array_ops.gather(labels, indices) - ordered_weights = array_ops.gather(weights, indices) - - # Get the counts of the unique ordered predictions. - _, _, counts = array_ops.unique_with_counts(ordered_predictions) - - # Compute the indices of the split points between different predictions. - splits = math_ops.cast( - array_ops.pad(math_ops.cumsum(counts), paddings=[[1, 0]]), dtypes.int32) - - # Count the positives to the left of the split indices. - true_positives = array_ops.gather( - array_ops.pad( - math_ops.cumsum( - array_ops.where( - math_ops.greater(ordered_labels, 0), ordered_weights, - array_ops.zeros_like(ordered_labels, - dtype=dtypes.float64))), - paddings=[[1, 0]]), splits) - if curve == 'ROC': - # Compute the weight of the negatives to the left of every split point and - # the total weight of the negatives number of negatives for computing the - # FPR. - false_positives = array_ops.gather( - array_ops.pad( - math_ops.cumsum( - array_ops.where( - math_ops.less(ordered_labels, 1), ordered_weights, - array_ops.zeros_like( - ordered_labels, dtype=dtypes.float64))), - paddings=[[1, 0]]), splits) - total_negative = total_weight - total_positive - x_axis_values = math_ops.truediv(false_positives, total_negative) - y_axis_values = math_ops.truediv(true_positives, total_positive) - elif curve == 'PR': - x_axis_values = math_ops.truediv(true_positives, total_positive) - # For conformance, set precision to 1 when the number of positive - # classifications is 0. - positives = array_ops.gather( - array_ops.pad(math_ops.cumsum(ordered_weights), paddings=[[1, 0]]), - splits) - y_axis_values = array_ops.where( - math_ops.greater(splits, 0), - math_ops.truediv(true_positives, positives), - array_ops.ones_like(true_positives, dtype=dtypes.float64)) - - # Calculate trapezoid areas. - heights = math_ops.add(y_axis_values[1:], y_axis_values[:-1]) / 2.0 - widths = math_ops.abs( - math_ops.subtract(x_axis_values[1:], x_axis_values[:-1])) - return math_ops.reduce_sum(math_ops.multiply(heights, widths)) - - # If all the labels are the same, AUC isn't well-defined (but raising an - # exception seems excessive) so we return 0, otherwise we finish computing. - return control_flow_ops.cond( - math_ops.logical_or( - math_ops.equal(total_positive, 0), - math_ops.equal(total_positive, total_weight)), - true_fn=lambda: array_ops.constant(0, dtypes.float64), - false_fn=continue_computing_dynamic_auc) - - -def streaming_dynamic_auc(labels, - predictions, - curve='ROC', - metrics_collections=(), - updates_collections=(), - name=None, - weights=None): - """Computes the apporixmate AUC by a Riemann sum with data-derived thresholds. - - USAGE NOTE: this approach requires storing all of the predictions and labels - for a single evaluation in memory, so it may not be usable when the evaluation - batch size and/or the number of evaluation steps is very large. - - Computes the area under the ROC or PR curve using each prediction as a - threshold. This has the advantage of being resilient to the distribution of - predictions by aggregating across batches, accumulating labels and predictions - and performing the final calculation using all of the concatenated values. - - Args: - labels: A `Tensor` of ground truth labels with the same shape as - `predictions` and with values of 0 or 1 whose values are castable to - `int64`. - predictions: A `Tensor` of predictions whose values are castable to - `float64`. Will be flattened into a 1-D `Tensor`. - curve: The name of the curve for which to compute AUC, 'ROC' for the - Receiving Operating Characteristic or 'PR' for the Precision-Recall curve. - metrics_collections: An optional iterable of collections that `auc` should - be added to. - updates_collections: An optional iterable of collections that `update_op` - should be added to. - name: An optional name for the variable_scope that contains the metric - variables. - weights: A 'Tensor' of non-negative weights whose values are castable to - `float64`. Will be flattened into a 1-D `Tensor`. - - Returns: - auc: A scalar `Tensor` containing the current area-under-curve value. - update_op: An operation that concatenates the input labels and predictions - to the accumulated values. - - Raises: - ValueError: If `labels` and `predictions` have mismatched shapes or if - `curve` isn't a recognized curve type. - """ - - if curve not in ['PR', 'ROC']: - raise ValueError('curve must be either ROC or PR, %s unknown' % curve) - - with variable_scope.variable_scope(name, default_name='dynamic_auc'): - labels.get_shape().assert_is_compatible_with(predictions.get_shape()) - predictions = array_ops.reshape( - math_ops.cast(predictions, dtypes.float64), [-1]) - labels = array_ops.reshape(math_ops.cast(labels, dtypes.int64), [-1]) - with ops.control_dependencies([ - check_ops.assert_greater_equal( - labels, - array_ops.zeros_like(labels, dtypes.int64), - message='labels must be 0 or 1, at least one is <0'), - check_ops.assert_less_equal( - labels, - array_ops.ones_like(labels, dtypes.int64), - message='labels must be 0 or 1, at least one is >1'), - ]): - preds_accum, update_preds = streaming_concat( - predictions, name='concat_preds') - labels_accum, update_labels = streaming_concat( - labels, name='concat_labels') - if weights is not None: - weights = array_ops.reshape( - math_ops.cast(weights, dtypes.float64), [-1]) - weights_accum, update_weights = streaming_concat( - weights, name='concat_weights') - update_op = control_flow_ops.group(update_labels, update_preds, - update_weights) - else: - weights_accum = None - update_op = control_flow_ops.group(update_labels, update_preds) - auc = _compute_dynamic_auc( - labels_accum, preds_accum, curve=curve, weights=weights_accum) - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - if metrics_collections: - ops.add_to_collections(metrics_collections, auc) - return auc, update_op - - -def _compute_placement_auc(labels, predictions, weights, alpha, - logit_transformation, is_valid): - """Computes the AUC and asymptotic normally distributed confidence interval. - - The calculations are achieved using the fact that AUC = P(Y_1>Y_0) and the - concept of placement values for each labeled group, as presented by Delong and - Delong (1988). The actual algorithm used is a more computationally efficient - approach presented by Sun and Xu (2014). This could be slow for large batches, - but has the advantage of not having its results degrade depending on the - distribution of predictions. - - Args: - labels: A `Tensor` of ground truth labels with the same shape as - `predictions` with values of 0 or 1 and type `int64`. - predictions: A 1-D `Tensor` of predictions whose values are `float64`. - weights: `Tensor` whose rank is either 0, or the same rank as `labels`. - alpha: Confidence interval level desired. - logit_transformation: A boolean value indicating whether the estimate should - be logit transformed prior to calculating the confidence interval. Doing - so enforces the restriction that the AUC should never be outside the - interval [0,1]. - is_valid: A bool tensor describing whether the input is valid. - - Returns: - A 1-D `Tensor` containing the area-under-curve, lower, and upper confidence - interval values. - """ - # Disable the invalid-name checker so that we can capitalize the name. - # pylint: disable=invalid-name - AucData = collections_lib.namedtuple('AucData', ['auc', 'lower', 'upper']) - # pylint: enable=invalid-name - - # If all the labels are the same or if number of observations are too few, - # AUC isn't well-defined - size = array_ops.size(predictions, out_type=dtypes.int32) - - # Count the total number of positive and negative labels in the input. - total_0 = math_ops.reduce_sum( - math_ops.cast(1 - labels, weights.dtype) * weights) - total_1 = math_ops.reduce_sum(math_ops.cast(labels, weights.dtype) * weights) - - # Sort the predictions ascending, as well as - # (i) the corresponding labels and - # (ii) the corresponding weights. - ordered_predictions, indices = nn.top_k(predictions, k=size, sorted=True) - ordered_predictions = array_ops.reverse( - ordered_predictions, axis=array_ops.zeros(1, dtypes.int32)) - indices = array_ops.reverse(indices, axis=array_ops.zeros(1, dtypes.int32)) - ordered_labels = array_ops.gather(labels, indices) - ordered_weights = array_ops.gather(weights, indices) - - # We now compute values required for computing placement values. - - # We generate a list of indices (segmented_indices) of increasing order. An - # index is assigned for each unique prediction float value. Prediction - # values that are the same share the same index. - _, segmented_indices = array_ops.unique(ordered_predictions) - - # We create 2 tensors of weights. weights_for_true is non-zero for true - # labels. weights_for_false is non-zero for false labels. - float_labels_for_true = math_ops.cast(ordered_labels, dtypes.float32) - float_labels_for_false = 1.0 - float_labels_for_true - weights_for_true = ordered_weights * float_labels_for_true - weights_for_false = ordered_weights * float_labels_for_false - - # For each set of weights with the same segmented indices, we add up the - # weight values. Note that for each label, we deliberately rely on weights - # for the opposite label. - weight_totals_for_true = math_ops.segment_sum(weights_for_false, - segmented_indices) - weight_totals_for_false = math_ops.segment_sum(weights_for_true, - segmented_indices) - - # These cumulative sums of weights importantly exclude the current weight - # sums. - cum_weight_totals_for_true = math_ops.cumsum( - weight_totals_for_true, exclusive=True) - cum_weight_totals_for_false = math_ops.cumsum( - weight_totals_for_false, exclusive=True) - - # Compute placement values using the formula. Values with the same segmented - # indices and labels share the same placement values. - placements_for_true = ( - (cum_weight_totals_for_true + weight_totals_for_true / 2.0) / - (math_ops.reduce_sum(weight_totals_for_true) + _EPSILON)) - placements_for_false = ( - (cum_weight_totals_for_false + weight_totals_for_false / 2.0) / - (math_ops.reduce_sum(weight_totals_for_false) + _EPSILON)) - - # We expand the tensors of placement values (for each label) so that their - # shapes match that of predictions. - placements_for_true = array_ops.gather(placements_for_true, segmented_indices) - placements_for_false = array_ops.gather(placements_for_false, - segmented_indices) - - # Select placement values based on the label for each index. - placement_values = ( - placements_for_true * float_labels_for_true + - placements_for_false * float_labels_for_false) - - # Split placement values by labeled groups. - placement_values_0 = placement_values * math_ops.cast(1 - ordered_labels, - weights.dtype) - weights_0 = ordered_weights * math_ops.cast(1 - ordered_labels, weights.dtype) - placement_values_1 = placement_values * math_ops.cast(ordered_labels, - weights.dtype) - weights_1 = ordered_weights * math_ops.cast(ordered_labels, weights.dtype) - - # Calculate AUC using placement values - auc_0 = ( - math_ops.reduce_sum(weights_0 * (1. - placement_values_0)) / - (total_0 + _EPSILON)) - auc_1 = ( - math_ops.reduce_sum(weights_1 * (placement_values_1)) / - (total_1 + _EPSILON)) - auc = array_ops.where(math_ops.less(total_0, total_1), auc_1, auc_0) - - # Calculate variance and standard error using the placement values. - var_0 = ( - math_ops.reduce_sum( - weights_0 * math_ops.square(1. - placement_values_0 - auc_0)) / - (total_0 - 1. + _EPSILON)) - var_1 = ( - math_ops.reduce_sum( - weights_1 * math_ops.squared_difference(placement_values_1, auc_1)) / - (total_1 - 1. + _EPSILON)) - auc_std_err = math_ops.sqrt((var_0 / (total_0 + _EPSILON)) + - (var_1 / (total_1 + _EPSILON))) - - # Calculate asymptotic normal confidence intervals - std_norm_dist = Normal(loc=0., scale=1.) - z_value = std_norm_dist.quantile((1.0 - alpha) / 2.0) - if logit_transformation: - estimate = math_ops.log(auc / (1. - auc + _EPSILON)) - std_err = auc_std_err / (auc * (1. - auc + _EPSILON)) - transformed_auc_lower = estimate + (z_value * std_err) - transformed_auc_upper = estimate - (z_value * std_err) - - def inverse_logit_transformation(x): - exp_negative = math_ops.exp(math_ops.negative(x)) - return 1. / (1. + exp_negative + _EPSILON) - - auc_lower = inverse_logit_transformation(transformed_auc_lower) - auc_upper = inverse_logit_transformation(transformed_auc_upper) - else: - estimate = auc - std_err = auc_std_err - auc_lower = estimate + (z_value * std_err) - auc_upper = estimate - (z_value * std_err) - - ## If estimate is 1 or 0, no variance is present so CI = 1 - ## n.b. This can be misleading, since number obs can just be too low. - lower = array_ops.where( - math_ops.logical_or( - math_ops.equal(auc, array_ops.ones_like(auc)), - math_ops.equal(auc, array_ops.zeros_like(auc))), auc, auc_lower) - upper = array_ops.where( - math_ops.logical_or( - math_ops.equal(auc, array_ops.ones_like(auc)), - math_ops.equal(auc, array_ops.zeros_like(auc))), auc, auc_upper) - - # If all the labels are the same, AUC isn't well-defined (but raising an - # exception seems excessive) so we return 0, otherwise we finish computing. - trivial_value = array_ops.constant(0.0) - - return AucData(*control_flow_ops.cond( - is_valid, lambda: [auc, lower, upper], lambda: [trivial_value] * 3)) - - -def auc_with_confidence_intervals(labels, - predictions, - weights=None, - alpha=0.95, - logit_transformation=True, - metrics_collections=(), - updates_collections=(), - name=None): - """Computes the AUC and asymptotic normally distributed confidence interval. - - USAGE NOTE: this approach requires storing all of the predictions and labels - for a single evaluation in memory, so it may not be usable when the evaluation - batch size and/or the number of evaluation steps is very large. - - Computes the area under the ROC curve and its confidence interval using - placement values. This has the advantage of being resilient to the - distribution of predictions by aggregating across batches, accumulating labels - and predictions and performing the final calculation using all of the - concatenated values. - - Args: - labels: A `Tensor` of ground truth labels with the same shape as `labels` - and with values of 0 or 1 whose values are castable to `int64`. - predictions: A `Tensor` of predictions whose values are castable to - `float64`. Will be flattened into a 1-D `Tensor`. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`. - alpha: Confidence interval level desired. - logit_transformation: A boolean value indicating whether the estimate should - be logit transformed prior to calculating the confidence interval. Doing - so enforces the restriction that the AUC should never be outside the - interval [0,1]. - metrics_collections: An optional iterable of collections that `auc` should - be added to. - updates_collections: An optional iterable of collections that `update_op` - should be added to. - name: An optional name for the variable_scope that contains the metric - variables. - - Returns: - auc: A 1-D `Tensor` containing the current area-under-curve, lower, and - upper confidence interval values. - update_op: An operation that concatenates the input labels and predictions - to the accumulated values. - - Raises: - ValueError: If `labels`, `predictions`, and `weights` have mismatched shapes - or if `alpha` isn't in the range (0,1). - """ - if not (alpha > 0 and alpha < 1): - raise ValueError('alpha must be between 0 and 1; currently %.02f' % alpha) - - if weights is None: - weights = array_ops.ones_like(predictions) - - with variable_scope.variable_scope( - name, - default_name='auc_with_confidence_intervals', - values=[labels, predictions, weights]): - - predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access - predictions=predictions, - labels=labels, - weights=weights) - - total_weight = math_ops.reduce_sum(weights) - - weights = array_ops.reshape(weights, [-1]) - predictions = array_ops.reshape( - math_ops.cast(predictions, dtypes.float64), [-1]) - labels = array_ops.reshape(math_ops.cast(labels, dtypes.int64), [-1]) - - with ops.control_dependencies([ - check_ops.assert_greater_equal( - labels, - array_ops.zeros_like(labels, dtypes.int64), - message='labels must be 0 or 1, at least one is <0'), - check_ops.assert_less_equal( - labels, - array_ops.ones_like(labels, dtypes.int64), - message='labels must be 0 or 1, at least one is >1'), - ]): - preds_accum, update_preds = streaming_concat( - predictions, name='concat_preds') - labels_accum, update_labels = streaming_concat( - labels, name='concat_labels') - weights_accum, update_weights = streaming_concat( - weights, name='concat_weights') - update_op_for_valid_case = control_flow_ops.group(update_labels, - update_preds, - update_weights) - - # Only perform updates if this case is valid. - all_labels_positive_or_0 = math_ops.logical_and( - math_ops.equal(math_ops.reduce_min(labels), 0), - math_ops.equal(math_ops.reduce_max(labels), 1)) - sums_of_weights_at_least_1 = math_ops.greater_equal(total_weight, 1.0) - is_valid = math_ops.logical_and(all_labels_positive_or_0, - sums_of_weights_at_least_1) - - update_op = control_flow_ops.cond( - sums_of_weights_at_least_1, lambda: update_op_for_valid_case, - control_flow_ops.no_op) - - auc = _compute_placement_auc( - labels_accum, - preds_accum, - weights_accum, - alpha=alpha, - logit_transformation=logit_transformation, - is_valid=is_valid) - - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - if metrics_collections: - ops.add_to_collections(metrics_collections, auc) - return auc, update_op - - -def precision_recall_at_equal_thresholds(labels, - predictions, - weights=None, - num_thresholds=None, - use_locking=None, - name=None): - """A helper method for creating metrics related to precision-recall curves. - - These values are true positives, false negatives, true negatives, false - positives, precision, and recall. This function returns a data structure that - contains ops within it. - - Unlike _streaming_confusion_matrix_at_thresholds (which exhibits O(T * N) - space and run time), this op exhibits O(T + N) space and run time, where T is - the number of thresholds and N is the size of the predictions tensor. Hence, - it may be advantageous to use this function when `predictions` is big. - - For instance, prefer this method for per-pixel classification tasks, for which - the predictions tensor may be very large. - - Each number in `predictions`, a float in `[0, 1]`, is compared with its - corresponding label in `labels`, and counts as a single tp/fp/tn/fn value at - each threshold. This is then multiplied with `weights` which can be used to - reweight certain values, or more commonly used for masking values. - - Args: - labels: A bool `Tensor` whose shape matches `predictions`. - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - weights: Optional; If provided, a `Tensor` that has the same dtype as, and - broadcastable to, `predictions`. This tensor is multiplied by counts. - num_thresholds: Optional; Number of thresholds, evenly distributed in `[0, - 1]`. Should be `>= 2`. Defaults to 201. Note that the number of bins is 1 - less than `num_thresholds`. Using an even `num_thresholds` value instead - of an odd one may yield unfriendly edges for bins. - use_locking: Optional; If True, the op will be protected by a lock. - Otherwise, the behavior is undefined, but may exhibit less contention. - Defaults to True. - name: Optional; variable_scope name. If not provided, the string - 'precision_recall_at_equal_threshold' is used. - - Returns: - result: A named tuple (See PrecisionRecallData within the implementation of - this function) with properties that are variables of shape - `[num_thresholds]`. The names of the properties are tp, fp, tn, fn, - precision, recall, thresholds. Types are same as that of predictions. - update_op: An op that accumulates values. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - `includes` contains invalid keys. - """ - # Disable the invalid-name checker so that we can capitalize the name. - # pylint: disable=invalid-name - PrecisionRecallData = collections_lib.namedtuple( - 'PrecisionRecallData', - ['tp', 'fp', 'tn', 'fn', 'precision', 'recall', 'thresholds']) - # pylint: enable=invalid-name - - if num_thresholds is None: - num_thresholds = 201 - - if weights is None: - weights = 1.0 - - if use_locking is None: - use_locking = True - - check_ops.assert_type(labels, dtypes.bool) - - with variable_scope.variable_scope(name, - 'precision_recall_at_equal_thresholds', - (labels, predictions, weights)): - # Make sure that predictions are within [0.0, 1.0]. - with ops.control_dependencies([ - check_ops.assert_greater_equal( - predictions, - math_ops.cast(0.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]'), - check_ops.assert_less_equal( - predictions, - math_ops.cast(1.0, dtype=predictions.dtype), - message='predictions must be in [0, 1]') - ]): - predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access - predictions=predictions, - labels=labels, - weights=weights) - - predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - - # It's important we aggregate using float64 since we're accumulating a lot - # of 1.0's for the true/false labels, and accumulating to float32 will - # be quite inaccurate even with just a modest amount of values (~20M). - # We use float64 instead of integer primarily since GPU scatter kernel - # only support floats. - agg_dtype = dtypes.float64 - - f_labels = math_ops.cast(labels, agg_dtype) - weights = math_ops.cast(weights, agg_dtype) - true_labels = f_labels * weights - false_labels = (1.0 - f_labels) * weights - - # Flatten predictions and labels. - predictions = array_ops.reshape(predictions, [-1]) - true_labels = array_ops.reshape(true_labels, [-1]) - false_labels = array_ops.reshape(false_labels, [-1]) - - # To compute TP/FP/TN/FN, we are measuring a binary classifier - # C(t) = (predictions >= t) - # at each threshold 't'. So we have - # TP(t) = sum( C(t) * true_labels ) - # FP(t) = sum( C(t) * false_labels ) - # - # But, computing C(t) requires computation for each t. To make it fast, - # observe that C(t) is a cumulative integral, and so if we have - # thresholds = [t_0, ..., t_{n-1}]; t_0 < ... < t_{n-1} - # where n = num_thresholds, and if we can compute the bucket function - # B(i) = Sum( (predictions == t), t_i <= t < t{i+1} ) - # then we get - # C(t_i) = sum( B(j), j >= i ) - # which is the reversed cumulative sum in tf.cumsum(). - # - # We can compute B(i) efficiently by taking advantage of the fact that - # our thresholds are evenly distributed, in that - # width = 1.0 / (num_thresholds - 1) - # thresholds = [0.0, 1*width, 2*width, 3*width, ..., 1.0] - # Given a prediction value p, we can map it to its bucket by - # bucket_index(p) = floor( p * (num_thresholds - 1) ) - # so we can use tf.compat.v1.scatter_add() to update the buckets in one pass. - # - # This implementation exhibits a run time and space complexity of O(T + N), - # where T is the number of thresholds and N is the size of predictions. - # Metrics that rely on _streaming_confusion_matrix_at_thresholds instead - # exhibit a complexity of O(T * N). - - # Compute the bucket indices for each prediction value. - bucket_indices = math_ops.cast( - math_ops.floor(predictions * (num_thresholds - 1)), dtypes.int32) - - with ops.name_scope('variables'): - tp_buckets_v = metrics_impl.metric_variable([num_thresholds], - agg_dtype, - name='tp_buckets') - fp_buckets_v = metrics_impl.metric_variable([num_thresholds], - agg_dtype, - name='fp_buckets') - - with ops.name_scope('update_op'): - update_tp = state_ops.scatter_add( - tp_buckets_v, bucket_indices, true_labels, use_locking=use_locking) - update_fp = state_ops.scatter_add( - fp_buckets_v, bucket_indices, false_labels, use_locking=use_locking) - - # Set up the cumulative sums to compute the actual metrics. - tp = math_ops.cumsum(tp_buckets_v, reverse=True, name='tp') - fp = math_ops.cumsum(fp_buckets_v, reverse=True, name='fp') - # fn = sum(true_labels) - tp - # = sum(tp_buckets) - tp - # = tp[0] - tp - # Similarly, - # tn = fp[0] - fp - tn = fp[0] - fp - fn = tp[0] - tp - - # We use a minimum to prevent division by 0. - epsilon = ops.convert_to_tensor(1e-7, dtype=agg_dtype) - precision = tp / math_ops.maximum(epsilon, tp + fp) - recall = tp / math_ops.maximum(epsilon, tp + fn) - - # Convert all tensors back to predictions' dtype (as per function contract). - out_dtype = predictions.dtype - _convert = lambda tensor: math_ops.cast(tensor, out_dtype) - result = PrecisionRecallData( - tp=_convert(tp), - fp=_convert(fp), - tn=_convert(tn), - fn=_convert(fn), - precision=_convert(precision), - recall=_convert(recall), - thresholds=_convert(math_ops.lin_space(0.0, 1.0, num_thresholds))) - update_op = control_flow_ops.group(update_tp, update_fp) - return result, update_op - - -def streaming_specificity_at_sensitivity(predictions, - labels, - sensitivity, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the specificity at a given sensitivity. - - The `streaming_specificity_at_sensitivity` function creates four local - variables, `true_positives`, `true_negatives`, `false_positives` and - `false_negatives` that are used to compute the specificity at the given - sensitivity value. The threshold for the given sensitivity value is computed - and used to evaluate the corresponding specificity. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `specificity`. `update_op` increments the `true_positives`, `true_negatives`, - `false_positives` and `false_negatives` counts with the weight of each case - found in the `predictions` and `labels`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - For additional information about specificity and sensitivity, see the - following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity - - Args: - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - labels: A `bool` `Tensor` whose shape matches `predictions`. - sensitivity: A scalar value in range `[0, 1]`. - weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and - must be broadcastable to `labels` (i.e., all dimensions must be either - `1`, or the same as the corresponding `labels` dimension). - num_thresholds: The number of thresholds to use for matching the given - sensitivity. - metrics_collections: An optional list of collections that `specificity` - should be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - specificity: A scalar `Tensor` representing the specificity at the given - `specificity` value. - update_op: An operation that increments the `true_positives`, - `true_negatives`, `false_positives` and `false_negatives` variables - appropriately and whose value matches `specificity`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, if - `weights` is not `None` and its shape doesn't match `predictions`, or if - `sensitivity` is not between 0 and 1, or if either `metrics_collections` - or `updates_collections` are not a list or tuple. - """ - return metrics.specificity_at_sensitivity( - sensitivity=sensitivity, - num_thresholds=num_thresholds, - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -def streaming_sensitivity_at_specificity(predictions, - labels, - specificity, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the sensitivity at a given specificity. - - The `streaming_sensitivity_at_specificity` function creates four local - variables, `true_positives`, `true_negatives`, `false_positives` and - `false_negatives` that are used to compute the sensitivity at the given - specificity value. The threshold for the given specificity value is computed - and used to evaluate the corresponding sensitivity. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `sensitivity`. `update_op` increments the `true_positives`, `true_negatives`, - `false_positives` and `false_negatives` counts with the weight of each case - found in the `predictions` and `labels`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - For additional information about specificity and sensitivity, see the - following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity - - Args: - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - labels: A `bool` `Tensor` whose shape matches `predictions`. - specificity: A scalar value in range `[0, 1]`. - weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and - must be broadcastable to `labels` (i.e., all dimensions must be either - `1`, or the same as the corresponding `labels` dimension). - num_thresholds: The number of thresholds to use for matching the given - specificity. - metrics_collections: An optional list of collections that `sensitivity` - should be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - sensitivity: A scalar `Tensor` representing the sensitivity at the given - `specificity` value. - update_op: An operation that increments the `true_positives`, - `true_negatives`, `false_positives` and `false_negatives` variables - appropriately and whose value matches `sensitivity`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, if - `weights` is not `None` and its shape doesn't match `predictions`, or if - `specificity` is not between 0 and 1, or if either `metrics_collections` - or `updates_collections` are not a list or tuple. - """ - return metrics.sensitivity_at_specificity( - specificity=specificity, - num_thresholds=num_thresholds, - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -@deprecated(None, - 'Please switch to tf.metrics.precision_at_thresholds. Note that ' - 'the order of the labels and predictions arguments are switched.') -def streaming_precision_at_thresholds(predictions, - labels, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes precision values for different `thresholds` on `predictions`. - - The `streaming_precision_at_thresholds` function creates four local variables, - `true_positives`, `true_negatives`, `false_positives` and `false_negatives` - for various values of thresholds. `precision[i]` is defined as the total - weight of values in `predictions` above `thresholds[i]` whose corresponding - entry in `labels` is `True`, divided by the total weight of values in - `predictions` above `thresholds[i]` (`true_positives[i] / (true_positives[i] + - false_positives[i])`). - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `precision`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - labels: A `bool` `Tensor` whose shape matches `predictions`. - thresholds: A python list or tuple of float thresholds in `[0, 1]`. - weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and - must be broadcastable to `labels` (i.e., all dimensions must be either - `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that `precision` should - be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - precision: A float `Tensor` of shape `[len(thresholds)]`. - update_op: An operation that increments the `true_positives`, - `true_negatives`, `false_positives` and `false_negatives` variables that - are used in the computation of `precision`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.precision_at_thresholds( - thresholds=thresholds, - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -@deprecated(None, - 'Please switch to tf.metrics.recall_at_thresholds. Note that the ' - 'order of the labels and predictions arguments has been switched.') -def streaming_recall_at_thresholds(predictions, - labels, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes various recall values for different `thresholds` on `predictions`. - - The `streaming_recall_at_thresholds` function creates four local variables, - `true_positives`, `true_negatives`, `false_positives` and `false_negatives` - for various values of thresholds. `recall[i]` is defined as the total weight - of values in `predictions` above `thresholds[i]` whose corresponding entry in - `labels` is `True`, divided by the total weight of `True` values in `labels` - (`true_positives[i] / (true_positives[i] + false_negatives[i])`). - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the `recall`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - labels: A `bool` `Tensor` whose shape matches `predictions`. - thresholds: A python list or tuple of float thresholds in `[0, 1]`. - weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and - must be broadcastable to `labels` (i.e., all dimensions must be either - `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that `recall` should be - added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - recall: A float `Tensor` of shape `[len(thresholds)]`. - update_op: An operation that increments the `true_positives`, - `true_negatives`, `false_positives` and `false_negatives` variables that - are used in the computation of `recall`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.recall_at_thresholds( - thresholds=thresholds, - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -def streaming_false_positive_rate_at_thresholds(predictions, - labels, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes various fpr values for different `thresholds` on `predictions`. - - The `streaming_false_positive_rate_at_thresholds` function creates two - local variables, `false_positives`, `true_negatives`, for various values of - thresholds. `false_positive_rate[i]` is defined as the total weight - of values in `predictions` above `thresholds[i]` whose corresponding entry in - `labels` is `False`, divided by the total weight of `False` values in `labels` - (`false_positives[i] / (false_positives[i] + true_negatives[i])`). - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `false_positive_rate`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - labels: A `bool` `Tensor` whose shape matches `predictions`. - thresholds: A python list or tuple of float thresholds in `[0, 1]`. - weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and - must be broadcastable to `labels` (i.e., all dimensions must be either - `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that - `false_positive_rate` should be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - false_positive_rate: A float `Tensor` of shape `[len(thresholds)]`. - update_op: An operation that increments the `false_positives` and - `true_negatives` variables that are used in the computation of - `false_positive_rate`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - with variable_scope.variable_scope(name, 'false_positive_rate_at_thresholds', - (predictions, labels, weights)): - values, update_ops = _streaming_confusion_matrix_at_thresholds( - predictions, labels, thresholds, weights, includes=('fp', 'tn')) - - # Avoid division by zero. - epsilon = _EPSILON - - def compute_fpr(fp, tn, name): - return math_ops.div(fp, epsilon + fp + tn, name='fpr_' + name) - - fpr = compute_fpr(values['fp'], values['tn'], 'value') - update_op = compute_fpr(update_ops['fp'], update_ops['tn'], 'update_op') - - if metrics_collections: - ops.add_to_collections(metrics_collections, fpr) - - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - - return fpr, update_op - - -def streaming_false_negative_rate_at_thresholds(predictions, - labels, - thresholds, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes various fnr values for different `thresholds` on `predictions`. - - The `streaming_false_negative_rate_at_thresholds` function creates two - local variables, `false_negatives`, `true_positives`, for various values of - thresholds. `false_negative_rate[i]` is defined as the total weight - of values in `predictions` above `thresholds[i]` whose corresponding entry in - `labels` is `False`, divided by the total weight of `True` values in `labels` - (`false_negatives[i] / (false_negatives[i] + true_positives[i])`). - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `false_positive_rate`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - labels: A `bool` `Tensor` whose shape matches `predictions`. - thresholds: A python list or tuple of float thresholds in `[0, 1]`. - weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and - must be broadcastable to `labels` (i.e., all dimensions must be either - `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that - `false_negative_rate` should be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - false_negative_rate: A float `Tensor` of shape `[len(thresholds)]`. - update_op: An operation that increments the `false_negatives` and - `true_positives` variables that are used in the computation of - `false_negative_rate`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - with variable_scope.variable_scope(name, 'false_negative_rate_at_thresholds', - (predictions, labels, weights)): - values, update_ops = _streaming_confusion_matrix_at_thresholds( - predictions, labels, thresholds, weights, includes=('fn', 'tp')) - - # Avoid division by zero. - epsilon = _EPSILON - - def compute_fnr(fn, tp, name): - return math_ops.div(fn, epsilon + fn + tp, name='fnr_' + name) - - fnr = compute_fnr(values['fn'], values['tp'], 'value') - update_op = compute_fnr(update_ops['fn'], update_ops['tp'], 'update_op') - - if metrics_collections: - ops.add_to_collections(metrics_collections, fnr) - - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - - return fnr, update_op - - -def _at_k_name(name, k=None, class_id=None): - if k is not None: - name = '%s_at_%d' % (name, k) - else: - name = '%s_at_k' % (name) - if class_id is not None: - name = '%s_class%d' % (name, class_id) - return name - - -@deprecated('2016-11-08', 'Please use `streaming_sparse_recall_at_k`, ' - 'and reshape labels from [batch_size] to [batch_size, 1].') -def streaming_recall_at_k(predictions, - labels, - k, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the recall@k of the predictions with respect to dense labels. - - The `streaming_recall_at_k` function creates two local variables, `total` and - `count`, that are used to compute the recall@k frequency. This frequency is - ultimately returned as `recall_at_`: an idempotent operation that simply - divides `total` by `count`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `recall_at_`. Internally, an `in_top_k` operation computes a `Tensor` with - shape [batch_size] whose elements indicate whether or not the corresponding - label is in the top `k` `predictions`. Then `update_op` increments `total` - with the reduced sum of `weights` where `in_top_k` is `True`, and it - increments `count` with the reduced sum of `weights`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: A float `Tensor` of dimension [batch_size, num_classes]. - labels: A `Tensor` of dimension [batch_size] whose type is in `int32`, - `int64`. - k: The number of top elements to look at for computing recall. - weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and - must be broadcastable to `labels` (i.e., all dimensions must be either - `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that `recall_at_k` - should be added to. - updates_collections: An optional list of collections `update_op` should be - added to. - name: An optional variable_scope name. - - Returns: - recall_at_k: A `Tensor` representing the recall@k, the fraction of labels - which fall into the top `k` predictions. - update_op: An operation that increments the `total` and `count` variables - appropriately and whose value matches `recall_at_k`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - in_top_k = math_ops.cast(nn.in_top_k(predictions, labels, k), dtypes.float32) - return streaming_mean(in_top_k, weights, metrics_collections, - updates_collections, name or _at_k_name('recall', k)) - - -# TODO(ptucker): Validate range of values in labels? -def streaming_sparse_recall_at_k(predictions, - labels, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes recall@k of the predictions with respect to sparse labels. - - If `class_id` is not specified, we'll calculate recall as the ratio of true - positives (i.e., correct predictions, items in the top `k` highest - `predictions` that are found in the corresponding row in `labels`) to - actual positives (the full `labels` row). - If `class_id` is specified, we calculate recall by considering only the rows - in the batch for which `class_id` is in `labels`, and computing the - fraction of them for which `class_id` is in the corresponding row in - `labels`. - - `streaming_sparse_recall_at_k` creates two local variables, - `true_positive_at_` and `false_negative_at_`, that are used to compute - the recall_at_k frequency. This frequency is ultimately returned as - `recall_at_`: an idempotent operation that simply divides - `true_positive_at_` by total (`true_positive_at_` + - `false_negative_at_`). - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `recall_at_`. Internally, a `top_k` operation computes a `Tensor` - indicating the top `k` `predictions`. Set operations applied to `top_k` and - `labels` calculate the true positives and false negatives weighted by - `weights`. Then `update_op` increments `true_positive_at_` and - `false_negative_at_` using these values. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: Float `Tensor` with shape [D1, ... DN, num_classes] where N >= - 1. Commonly, N=1 and predictions has shape [batch size, num_classes]. The - final dimension contains the logit values for each class. [D1, ... DN] - must match `labels`. - labels: `int64` `Tensor` or `SparseTensor` with shape [D1, ... DN, - num_labels], where N >= 1 and num_labels is the number of target classes - for the associated prediction. Commonly, N=1 and `labels` has shape - [batch_size, num_labels]. [D1, ... DN] must match `predictions`. Values - should be in range [0, num_classes), where num_classes is the last - dimension of `predictions`. Values outside this range always count towards - `false_negative_at_`. - k: Integer, k for @k metric. - class_id: Integer class ID for which we want binary metrics. This should be - in range [0, num_classes), where num_classes is the last dimension of - `predictions`. If class_id is outside this range, the method returns NAN. - weights: `Tensor` whose rank is either 0, or n-1, where n is the rank of - `labels`. If the latter, it must be broadcastable to `labels` (i.e., all - dimensions must be either `1`, or the same as the corresponding `labels` - dimension). - metrics_collections: An optional list of collections that values should be - added to. - updates_collections: An optional list of collections that updates should be - added to. - name: Name of new update operation, and namespace for other dependent ops. - - Returns: - recall: Scalar `float64` `Tensor` with the value of `true_positives` divided - by the sum of `true_positives` and `false_negatives`. - update_op: `Operation` that increments `true_positives` and - `false_negatives` variables appropriately, and whose value matches - `recall`. - - Raises: - ValueError: If `weights` is not `None` and its shape doesn't match - `predictions`, or if either `metrics_collections` or `updates_collections` - are not a list or tuple. - """ - return metrics.recall_at_k( - k=k, - class_id=class_id, - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -# TODO(ptucker): Validate range of values in labels? -def streaming_sparse_precision_at_k(predictions, - labels, - k, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes precision@k of the predictions with respect to sparse labels. - - If `class_id` is not specified, we calculate precision as the ratio of true - positives (i.e., correct predictions, items in the top `k` highest - `predictions` that are found in the corresponding row in `labels`) to - positives (all top `k` `predictions`). - If `class_id` is specified, we calculate precision by considering only the - rows in the batch for which `class_id` is in the top `k` highest - `predictions`, and computing the fraction of them for which `class_id` is - in the corresponding row in `labels`. - - We expect precision to decrease as `k` increases. - - `streaming_sparse_precision_at_k` creates two local variables, - `true_positive_at_` and `false_positive_at_`, that are used to compute - the precision@k frequency. This frequency is ultimately returned as - `precision_at_`: an idempotent operation that simply divides - `true_positive_at_` by total (`true_positive_at_` + - `false_positive_at_`). - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `precision_at_`. Internally, a `top_k` operation computes a `Tensor` - indicating the top `k` `predictions`. Set operations applied to `top_k` and - `labels` calculate the true positives and false positives weighted by - `weights`. Then `update_op` increments `true_positive_at_` and - `false_positive_at_` using these values. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: Float `Tensor` with shape [D1, ... DN, num_classes] where N >= - 1. Commonly, N=1 and predictions has shape [batch size, num_classes]. The - final dimension contains the logit values for each class. [D1, ... DN] - must match `labels`. - labels: `int64` `Tensor` or `SparseTensor` with shape [D1, ... DN, - num_labels], where N >= 1 and num_labels is the number of target classes - for the associated prediction. Commonly, N=1 and `labels` has shape - [batch_size, num_labels]. [D1, ... DN] must match `predictions`. Values - should be in range [0, num_classes), where num_classes is the last - dimension of `predictions`. Values outside this range are ignored. - k: Integer, k for @k metric. - class_id: Integer class ID for which we want binary metrics. This should be - in range [0, num_classes], where num_classes is the last dimension of - `predictions`. If `class_id` is outside this range, the method returns - NAN. - weights: `Tensor` whose rank is either 0, or n-1, where n is the rank of - `labels`. If the latter, it must be broadcastable to `labels` (i.e., all - dimensions must be either `1`, or the same as the corresponding `labels` - dimension). - metrics_collections: An optional list of collections that values should be - added to. - updates_collections: An optional list of collections that updates should be - added to. - name: Name of new update operation, and namespace for other dependent ops. - - Returns: - precision: Scalar `float64` `Tensor` with the value of `true_positives` - divided by the sum of `true_positives` and `false_positives`. - update_op: `Operation` that increments `true_positives` and - `false_positives` variables appropriately, and whose value matches - `precision`. - - Raises: - ValueError: If `weights` is not `None` and its shape doesn't match - `predictions`, or if either `metrics_collections` or `updates_collections` - are not a list or tuple. - """ - return metrics.precision_at_k( - k=k, - class_id=class_id, - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -# TODO(ptucker): Validate range of values in labels? -def streaming_sparse_precision_at_top_k(top_k_predictions, - labels, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes precision@k of top-k predictions with respect to sparse labels. - - If `class_id` is not specified, we calculate precision as the ratio of - true positives (i.e., correct predictions, items in `top_k_predictions` - that are found in the corresponding row in `labels`) to positives (all - `top_k_predictions`). - If `class_id` is specified, we calculate precision by considering only the - rows in the batch for which `class_id` is in the top `k` highest - `predictions`, and computing the fraction of them for which `class_id` is - in the corresponding row in `labels`. - - We expect precision to decrease as `k` increases. - - `streaming_sparse_precision_at_top_k` creates two local variables, - `true_positive_at_k` and `false_positive_at_k`, that are used to compute - the precision@k frequency. This frequency is ultimately returned as - `precision_at_k`: an idempotent operation that simply divides - `true_positive_at_k` by total (`true_positive_at_k` + `false_positive_at_k`). - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `precision_at_k`. Internally, set operations applied to `top_k_predictions` - and `labels` calculate the true positives and false positives weighted by - `weights`. Then `update_op` increments `true_positive_at_k` and - `false_positive_at_k` using these values. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - top_k_predictions: Integer `Tensor` with shape [D1, ... DN, k] where N >= 1. - Commonly, N=1 and top_k_predictions has shape [batch size, k]. The final - dimension contains the indices of top-k labels. [D1, ... DN] must match - `labels`. - labels: `int64` `Tensor` or `SparseTensor` with shape [D1, ... DN, - num_labels], where N >= 1 and num_labels is the number of target classes - for the associated prediction. Commonly, N=1 and `labels` has shape - [batch_size, num_labels]. [D1, ... DN] must match `top_k_predictions`. - Values should be in range [0, num_classes), where num_classes is the last - dimension of `predictions`. Values outside this range are ignored. - class_id: Integer class ID for which we want binary metrics. This should be - in range [0, num_classes), where num_classes is the last dimension of - `predictions`. If `class_id` is outside this range, the method returns - NAN. - weights: `Tensor` whose rank is either 0, or n-1, where n is the rank of - `labels`. If the latter, it must be broadcastable to `labels` (i.e., all - dimensions must be either `1`, or the same as the corresponding `labels` - dimension). - metrics_collections: An optional list of collections that values should be - added to. - updates_collections: An optional list of collections that updates should be - added to. - name: Name of new update operation, and namespace for other dependent ops. - - Returns: - precision: Scalar `float64` `Tensor` with the value of `true_positives` - divided by the sum of `true_positives` and `false_positives`. - update_op: `Operation` that increments `true_positives` and - `false_positives` variables appropriately, and whose value matches - `precision`. - - Raises: - ValueError: If `weights` is not `None` and its shape doesn't match - `predictions`, or if either `metrics_collections` or `updates_collections` - are not a list or tuple. - ValueError: If `top_k_predictions` has rank < 2. - """ - default_name = _at_k_name('precision', class_id=class_id) - with ops.name_scope(name, default_name, - (top_k_predictions, labels, weights)) as name_scope: - return metrics_impl.precision_at_top_k( - labels=labels, - predictions_idx=top_k_predictions, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name_scope) - - -def sparse_recall_at_top_k(labels, - top_k_predictions, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes recall@k of top-k predictions with respect to sparse labels. - - If `class_id` is specified, we calculate recall by considering only the - entries in the batch for which `class_id` is in the label, and computing - the fraction of them for which `class_id` is in the top-k `predictions`. - If `class_id` is not specified, we'll calculate recall as how often on - average a class among the labels of a batch entry is in the top-k - `predictions`. - - `sparse_recall_at_top_k` creates two local variables, `true_positive_at_` - and `false_negative_at_`, that are used to compute the recall_at_k - frequency. This frequency is ultimately returned as `recall_at_`: an - idempotent operation that simply divides `true_positive_at_` by total - (`true_positive_at_` + `false_negative_at_`). - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `recall_at_`. Set operations applied to `top_k` and `labels` calculate the - true positives and false negatives weighted by `weights`. Then `update_op` - increments `true_positive_at_` and `false_negative_at_` using these - values. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - labels: `int64` `Tensor` or `SparseTensor` with shape [D1, ... DN, - num_labels], where N >= 1 and num_labels is the number of target classes - for the associated prediction. Commonly, N=1 and `labels` has shape - [batch_size, num_labels]. [D1, ... DN] must match `top_k_predictions`. - Values should be in range [0, num_classes), where num_classes is the last - dimension of `predictions`. Values outside this range always count towards - `false_negative_at_`. - top_k_predictions: Integer `Tensor` with shape [D1, ... DN, k] where N >= 1. - Commonly, N=1 and top_k_predictions has shape [batch size, k]. The final - dimension contains the indices of top-k labels. [D1, ... DN] must match - `labels`. - class_id: Integer class ID for which we want binary metrics. This should be - in range [0, num_classes), where num_classes is the last dimension of - `predictions`. If class_id is outside this range, the method returns NAN. - weights: `Tensor` whose rank is either 0, or n-1, where n is the rank of - `labels`. If the latter, it must be broadcastable to `labels` (i.e., all - dimensions must be either `1`, or the same as the corresponding `labels` - dimension). - metrics_collections: An optional list of collections that values should be - added to. - updates_collections: An optional list of collections that updates should be - added to. - name: Name of new update operation, and namespace for other dependent ops. - - Returns: - recall: Scalar `float64` `Tensor` with the value of `true_positives` divided - by the sum of `true_positives` and `false_negatives`. - update_op: `Operation` that increments `true_positives` and - `false_negatives` variables appropriately, and whose value matches - `recall`. - - Raises: - ValueError: If `weights` is not `None` and its shape doesn't match - `predictions`, or if either `metrics_collections` or `updates_collections` - are not a list or tuple. - """ - default_name = _at_k_name('recall', class_id=class_id) - with ops.name_scope(name, default_name, - (top_k_predictions, labels, weights)) as name_scope: - return metrics_impl.recall_at_top_k( - labels=labels, - predictions_idx=top_k_predictions, - class_id=class_id, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name_scope) - - -def _compute_recall_at_precision(tp, fp, fn, precision, name, - strict_mode=False): - """Helper function to compute recall at a given `precision`. - - Args: - tp: The number of true positives. - fp: The number of false positives. - fn: The number of false negatives. - precision: The precision for which the recall will be calculated. - name: An optional variable_scope name. - strict_mode: If true and there exists a threshold where the precision is no - smaller than the target precision, return the corresponding recall at the - threshold. Otherwise, return 0. If false, find the threshold where the - precision is closest to the target precision and return the recall at the - threshold. - - Returns: - The recall at a given `precision`. - """ - precisions = math_ops.div(tp, tp + fp + _EPSILON) - if not strict_mode: - tf_index = math_ops.argmin( - math_ops.abs(precisions - precision), 0, output_type=dtypes.int32) - # Now, we have the implicit threshold, so compute the recall: - return math_ops.div(tp[tf_index], tp[tf_index] + fn[tf_index] + _EPSILON, - name) - else: - # We aim to find the threshold where the precision is minimum but no smaller - # than the target precision. - # The rationale: - # 1. Compute the difference between precisions (by different thresholds) and - # the target precision. - # 2. Take the reciprocal of the values by the above step. The intention is - # to make the positive values rank before negative values and also the - # smaller positives rank before larger positives. - tf_index = math_ops.argmax( - math_ops.div(1.0, precisions - precision + _EPSILON), - 0, - output_type=dtypes.int32) - - def _return_good_recall(): - return math_ops.div(tp[tf_index], tp[tf_index] + fn[tf_index] + _EPSILON, - name) - - return control_flow_ops.cond(precisions[tf_index] >= precision, - _return_good_recall, lambda: .0) - - -def recall_at_precision(labels, - predictions, - precision, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - name=None, - strict_mode=False): - """Computes `recall` at `precision`. - - The `recall_at_precision` function creates four local variables, - `tp` (true positives), `fp` (false positives) and `fn` (false negatives) - that are used to compute the `recall` at the given `precision` value. The - threshold for the given `precision` value is computed and used to evaluate the - corresponding `recall`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `recall`. `update_op` increments the `tp`, `fp` and `fn` counts with the - weight of each case found in the `predictions` and `labels`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - labels: The ground truth values, a `Tensor` whose dimensions must match - `predictions`. Will be cast to `bool`. - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - precision: A scalar value in range `[0, 1]`. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions must - be either `1`, or the same as the corresponding `labels` dimension). - num_thresholds: The number of thresholds to use for matching the given - `precision`. - metrics_collections: An optional list of collections that `recall` should be - added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - strict_mode: If true and there exists a threshold where the precision is - above the target precision, return the corresponding recall at the - threshold. Otherwise, return 0. If false, find the threshold where the - precision is closest to the target precision and return the recall at the - threshold. - - Returns: - recall: A scalar `Tensor` representing the recall at the given - `precision` value. - update_op: An operation that increments the `tp`, `fp` and `fn` - variables appropriately and whose value matches `recall`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, if - `weights` is not `None` and its shape doesn't match `predictions`, or if - `precision` is not between 0 and 1, or if either `metrics_collections` - or `updates_collections` are not a list or tuple. - - """ - if not 0 <= precision <= 1: - raise ValueError('`precision` must be in the range [0, 1].') - - with variable_scope.variable_scope(name, 'recall_at_precision', - (predictions, labels, weights)): - thresholds = [ - i * 1.0 / (num_thresholds - 1) for i in range(1, num_thresholds - 1) - ] - thresholds = [0.0 - _EPSILON] + thresholds + [1.0 + _EPSILON] - - values, update_ops = _streaming_confusion_matrix_at_thresholds( - predictions, labels, thresholds, weights) - - recall = _compute_recall_at_precision(values['tp'], values['fp'], - values['fn'], precision, 'value', - strict_mode) - update_op = _compute_recall_at_precision(update_ops['tp'], update_ops['fp'], - update_ops['fn'], precision, - 'update_op', strict_mode) - - if metrics_collections: - ops.add_to_collections(metrics_collections, recall) - - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - - return recall, update_op - - -def precision_at_recall(labels, - predictions, - target_recall, - weights=None, - num_thresholds=200, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the precision at a given recall. - - This function creates variables to track the true positives, false positives, - true negatives, and false negatives at a set of thresholds. Among those - thresholds where recall is at least `target_recall`, precision is computed - at the threshold where recall is closest to `target_recall`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - precision at `target_recall`. `update_op` increments the counts of true - positives, false positives, true negatives, and false negatives with the - weight of each case found in the `predictions` and `labels`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - For additional information about precision and recall, see - http://en.wikipedia.org/wiki/Precision_and_recall - - Args: - labels: The ground truth values, a `Tensor` whose dimensions must match - `predictions`. Will be cast to `bool`. - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - target_recall: A scalar value in range `[0, 1]`. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions must - be either `1`, or the same as the corresponding `labels` dimension). - num_thresholds: The number of thresholds to use for matching the given - recall. - metrics_collections: An optional list of collections to which `precision` - should be added. - updates_collections: An optional list of collections to which `update_op` - should be added. - name: An optional variable_scope name. - - Returns: - precision: A scalar `Tensor` representing the precision at the given - `target_recall` value. - update_op: An operation that increments the variables for tracking the - true positives, false positives, true negatives, and false negatives and - whose value matches `precision`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, if - `weights` is not `None` and its shape doesn't match `predictions`, or if - `target_recall` is not between 0 and 1, or if either `metrics_collections` - or `updates_collections` are not a list or tuple. - RuntimeError: If eager execution is enabled. - """ - if context.executing_eagerly(): - raise RuntimeError('tf.metrics.precision_at_recall is not ' - 'supported when eager execution is enabled.') - - if target_recall < 0 or target_recall > 1: - raise ValueError('`target_recall` must be in the range [0, 1].') - - with variable_scope.variable_scope(name, 'precision_at_recall', - (predictions, labels, weights)): - kepsilon = 1e-7 # Used to avoid division by zero. - thresholds = [ - (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) - ] - thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon] - - values, update_ops = _streaming_confusion_matrix_at_thresholds( - predictions, labels, thresholds, weights) - - def compute_precision_at_recall(tp, fp, fn, name): - """Computes the precision at a given recall. - - Args: - tp: True positives. - fp: False positives. - fn: False negatives. - name: A name for the operation. - - Returns: - The precision at the desired recall. - """ - recalls = math_ops.div(tp, tp + fn + kepsilon) - - # Because recall is monotone decreasing as a function of the threshold, - # the smallest recall exceeding target_recall occurs at the largest - # threshold where recall >= target_recall. - admissible_recalls = math_ops.cast( - math_ops.greater_equal(recalls, target_recall), dtypes.int64) - tf_index = math_ops.reduce_sum(admissible_recalls) - 1 - - # Now we have the threshold at which to compute precision: - return math_ops.div(tp[tf_index] + kepsilon, - tp[tf_index] + fp[tf_index] + kepsilon, name) - - precision_value = compute_precision_at_recall(values['tp'], values['fp'], - values['fn'], 'value') - update_op = compute_precision_at_recall(update_ops['tp'], update_ops['fp'], - update_ops['fn'], 'update_op') - - if metrics_collections: - ops.add_to_collections(metrics_collections, precision_value) - - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - - return precision_value, update_op - - -def streaming_sparse_average_precision_at_k(predictions, - labels, - k, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes average precision@k of predictions with respect to sparse labels. - - See `sparse_average_precision_at_k` for details on formula. `weights` are - applied to the result of `sparse_average_precision_at_k` - - `streaming_sparse_average_precision_at_k` creates two local variables, - `average_precision_at_/total` and `average_precision_at_/max`, that - are used to compute the frequency. This frequency is ultimately returned as - `average_precision_at_`: an idempotent operation that simply divides - `average_precision_at_/total` by `average_precision_at_/max`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `precision_at_`. Internally, a `top_k` operation computes a `Tensor` - indicating the top `k` `predictions`. Set operations applied to `top_k` and - `labels` calculate the true positives and false positives weighted by - `weights`. Then `update_op` increments `true_positive_at_` and - `false_positive_at_` using these values. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: Float `Tensor` with shape [D1, ... DN, num_classes] where N >= - 1. Commonly, N=1 and `predictions` has shape [batch size, num_classes]. - The final dimension contains the logit values for each class. [D1, ... DN] - must match `labels`. - labels: `int64` `Tensor` or `SparseTensor` with shape [D1, ... DN, - num_labels], where N >= 1 and num_labels is the number of target classes - for the associated prediction. Commonly, N=1 and `labels` has shape - [batch_size, num_labels]. [D1, ... DN] must match `predictions_`. Values - should be in range [0, num_classes), where num_classes is the last - dimension of `predictions`. Values outside this range are ignored. - k: Integer, k for @k metric. This will calculate an average precision for - range `[1,k]`, as documented above. - weights: `Tensor` whose rank is either 0, or n-1, where n is the rank of - `labels`. If the latter, it must be broadcastable to `labels` (i.e., all - dimensions must be either `1`, or the same as the corresponding `labels` - dimension). - metrics_collections: An optional list of collections that values should be - added to. - updates_collections: An optional list of collections that updates should be - added to. - name: Name of new update operation, and namespace for other dependent ops. - - Returns: - mean_average_precision: Scalar `float64` `Tensor` with the mean average - precision values. - update: `Operation` that increments variables appropriately, and whose - value matches `metric`. - """ - return metrics.average_precision_at_k( - k=k, - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -def streaming_sparse_average_precision_at_top_k(top_k_predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes average precision@k of predictions with respect to sparse labels. - - `streaming_sparse_average_precision_at_top_k` creates two local variables, - `average_precision_at_/total` and `average_precision_at_/max`, that - are used to compute the frequency. This frequency is ultimately returned as - `average_precision_at_`: an idempotent operation that simply divides - `average_precision_at_/total` by `average_precision_at_/max`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `precision_at_`. Set operations applied to `top_k` and `labels` calculate - the true positives and false positives weighted by `weights`. Then `update_op` - increments `true_positive_at_` and `false_positive_at_` using these - values. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - top_k_predictions: Integer `Tensor` with shape [D1, ... DN, k] where N >= 1. - Commonly, N=1 and `predictions_idx` has shape [batch size, k]. The final - dimension must be set and contains the top `k` predicted class indices. - [D1, ... DN] must match `labels`. Values should be in range [0, - num_classes). - labels: `int64` `Tensor` or `SparseTensor` with shape [D1, ... DN, - num_labels] or [D1, ... DN], where the latter implies num_labels=1. N >= 1 - and num_labels is the number of target classes for the associated - prediction. Commonly, N=1 and `labels` has shape [batch_size, num_labels]. - [D1, ... DN] must match `top_k_predictions`. Values should be in range [0, - num_classes). - weights: `Tensor` whose rank is either 0, or n-1, where n is the rank of - `labels`. If the latter, it must be broadcastable to `labels` (i.e., all - dimensions must be either `1`, or the same as the corresponding `labels` - dimension). - metrics_collections: An optional list of collections that values should be - added to. - updates_collections: An optional list of collections that updates should be - added to. - name: Name of new update operation, and namespace for other dependent ops. - - Returns: - mean_average_precision: Scalar `float64` `Tensor` with the mean average - precision values. - update: `Operation` that increments variables appropriately, and whose - value matches `metric`. - - Raises: - ValueError: if the last dimension of top_k_predictions is not set. - """ - return metrics_impl._streaming_sparse_average_precision_at_top_k( # pylint: disable=protected-access - predictions_idx=top_k_predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -@deprecated(None, - 'Please switch to tf.metrics.mean_absolute_error. Note that the ' - 'order of the labels and predictions arguments has been switched.') -def streaming_mean_absolute_error(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the mean absolute error between the labels and predictions. - - The `streaming_mean_absolute_error` function creates two local variables, - `total` and `count` that are used to compute the mean absolute error. This - average is weighted by `weights`, and it is ultimately returned as - `mean_absolute_error`: an idempotent operation that simply divides `total` by - `count`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `mean_absolute_error`. Internally, an `absolute_errors` operation computes the - absolute value of the differences between `predictions` and `labels`. Then - `update_op` increments `total` with the reduced sum of the product of - `weights` and `absolute_errors`, and it increments `count` with the reduced - sum of `weights` - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: A `Tensor` of arbitrary shape. - labels: A `Tensor` of the same shape as `predictions`. - weights: Optional `Tensor` indicating the frequency with which an example is - sampled. Rank must be 0, or the same rank as `labels`, and must be - broadcastable to `labels` (i.e., all dimensions must be either `1`, or the - same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that - `mean_absolute_error` should be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - mean_absolute_error: A `Tensor` representing the current mean, the value of - `total` divided by `count`. - update_op: An operation that increments the `total` and `count` variables - appropriately and whose value matches `mean_absolute_error`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.mean_absolute_error( - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -def streaming_mean_relative_error(predictions, - labels, - normalizer, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the mean relative error by normalizing with the given values. - - The `streaming_mean_relative_error` function creates two local variables, - `total` and `count` that are used to compute the mean relative absolute error. - This average is weighted by `weights`, and it is ultimately returned as - `mean_relative_error`: an idempotent operation that simply divides `total` by - `count`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `mean_reative_error`. Internally, a `relative_errors` operation divides the - absolute value of the differences between `predictions` and `labels` by the - `normalizer`. Then `update_op` increments `total` with the reduced sum of the - product of `weights` and `relative_errors`, and it increments `count` with the - reduced sum of `weights`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: A `Tensor` of arbitrary shape. - labels: A `Tensor` of the same shape as `predictions`. - normalizer: A `Tensor` of the same shape as `predictions`. - weights: Optional `Tensor` indicating the frequency with which an example is - sampled. Rank must be 0, or the same rank as `labels`, and must be - broadcastable to `labels` (i.e., all dimensions must be either `1`, or the - same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that - `mean_relative_error` should be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - mean_relative_error: A `Tensor` representing the current mean, the value of - `total` divided by `count`. - update_op: An operation that increments the `total` and `count` variables - appropriately and whose value matches `mean_relative_error`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.mean_relative_error( - normalizer=normalizer, - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -@deprecated(None, - 'Please switch to tf.metrics.mean_squared_error. Note that the ' - 'order of the labels and predictions arguments has been switched.') -def streaming_mean_squared_error(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the mean squared error between the labels and predictions. - - The `streaming_mean_squared_error` function creates two local variables, - `total` and `count` that are used to compute the mean squared error. - This average is weighted by `weights`, and it is ultimately returned as - `mean_squared_error`: an idempotent operation that simply divides `total` by - `count`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `mean_squared_error`. Internally, a `squared_error` operation computes the - element-wise square of the difference between `predictions` and `labels`. Then - `update_op` increments `total` with the reduced sum of the product of - `weights` and `squared_error`, and it increments `count` with the reduced sum - of `weights`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: A `Tensor` of arbitrary shape. - labels: A `Tensor` of the same shape as `predictions`. - weights: Optional `Tensor` indicating the frequency with which an example is - sampled. Rank must be 0, or the same rank as `labels`, and must be - broadcastable to `labels` (i.e., all dimensions must be either `1`, or the - same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that - `mean_squared_error` should be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - mean_squared_error: A `Tensor` representing the current mean, the value of - `total` divided by `count`. - update_op: An operation that increments the `total` and `count` variables - appropriately and whose value matches `mean_squared_error`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.mean_squared_error( - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -@deprecated( - None, 'Please switch to tf.metrics.root_mean_squared_error. Note that the ' - 'order of the labels and predictions arguments has been switched.') -def streaming_root_mean_squared_error(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the root mean squared error between the labels and predictions. - - The `streaming_root_mean_squared_error` function creates two local variables, - `total` and `count` that are used to compute the root mean squared error. - This average is weighted by `weights`, and it is ultimately returned as - `root_mean_squared_error`: an idempotent operation that takes the square root - of the division of `total` by `count`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `root_mean_squared_error`. Internally, a `squared_error` operation computes - the element-wise square of the difference between `predictions` and `labels`. - Then `update_op` increments `total` with the reduced sum of the product of - `weights` and `squared_error`, and it increments `count` with the reduced sum - of `weights`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: A `Tensor` of arbitrary shape. - labels: A `Tensor` of the same shape as `predictions`. - weights: Optional `Tensor` indicating the frequency with which an example is - sampled. Rank must be 0, or the same rank as `labels`, and must be - broadcastable to `labels` (i.e., all dimensions must be either `1`, or the - same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that - `root_mean_squared_error` should be added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - root_mean_squared_error: A `Tensor` representing the current mean, the value - of `total` divided by `count`. - update_op: An operation that increments the `total` and `count` variables - appropriately and whose value matches `root_mean_squared_error`. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.root_mean_squared_error( - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -def streaming_covariance(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the unbiased sample covariance between `predictions` and `labels`. - - The `streaming_covariance` function creates four local variables, - `comoment`, `mean_prediction`, `mean_label`, and `count`, which are used to - compute the sample covariance between predictions and labels across multiple - batches of data. The covariance is ultimately returned as an idempotent - operation that simply divides `comoment` by `count` - 1. We use `count` - 1 - in order to get an unbiased estimate. - - The algorithm used for this online computation is described in - https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance. - Specifically, the formula used to combine two sample comoments is - `C_AB = C_A + C_B + (E[x_A] - E[x_B]) * (E[y_A] - E[y_B]) * n_A * n_B / n_AB` - The comoment for a single batch of data is simply - `sum((x - E[x]) * (y - E[y]))`, optionally weighted. - - If `weights` is not None, then it is used to compute weighted comoments, - means, and count. NOTE: these weights are treated as "frequency weights", as - opposed to "reliability weights". See discussion of the difference on - https://wikipedia.org/wiki/Weighted_arithmetic_mean#Weighted_sample_variance - - To facilitate the computation of covariance across multiple batches of data, - the function creates an `update_op` operation, which updates underlying - variables and returns the updated covariance. - - Args: - predictions: A `Tensor` of arbitrary size. - labels: A `Tensor` of the same size as `predictions`. - weights: Optional `Tensor` indicating the frequency with which an example is - sampled. Rank must be 0, or the same rank as `labels`, and must be - broadcastable to `labels` (i.e., all dimensions must be either `1`, or the - same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that the metric value - variable should be added to. - updates_collections: An optional list of collections that the metric update - ops should be added to. - name: An optional variable_scope name. - - Returns: - covariance: A `Tensor` representing the current unbiased sample covariance, - `comoment` / (`count` - 1). - update_op: An operation that updates the local variables appropriately. - - Raises: - ValueError: If labels and predictions are of different sizes or if either - `metrics_collections` or `updates_collections` are not a list or tuple. - """ - with variable_scope.variable_scope(name, 'covariance', - (predictions, labels, weights)): - predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access - predictions, labels, weights) - predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - count_ = metrics_impl.metric_variable([], dtypes.float32, name='count') - mean_prediction = metrics_impl.metric_variable([], - dtypes.float32, - name='mean_prediction') - mean_label = metrics_impl.metric_variable([], - dtypes.float32, - name='mean_label') - comoment = metrics_impl.metric_variable( # C_A in update equation - [], dtypes.float32, name='comoment') - - if weights is None: - batch_count = math_ops.cast(array_ops.size(labels), - dtypes.float32) # n_B in eqn - weighted_predictions = predictions - weighted_labels = labels - else: - weights = weights_broadcast_ops.broadcast_weights(weights, labels) - batch_count = math_ops.reduce_sum(weights) # n_B in eqn - weighted_predictions = math_ops.multiply(predictions, weights) - weighted_labels = math_ops.multiply(labels, weights) - - update_count = state_ops.assign_add(count_, batch_count) # n_AB in eqn - prev_count = update_count - batch_count # n_A in update equation - - # We update the means by Delta=Error*BatchCount/(BatchCount+PrevCount) - # batch_mean_prediction is E[x_B] in the update equation - batch_mean_prediction = math_ops.div_no_nan( - math_ops.reduce_sum(weighted_predictions), batch_count) - delta_mean_prediction = math_ops.div_no_nan( - (batch_mean_prediction - mean_prediction) * batch_count, update_count) - update_mean_prediction = state_ops.assign_add(mean_prediction, - delta_mean_prediction) - # prev_mean_prediction is E[x_A] in the update equation - prev_mean_prediction = update_mean_prediction - delta_mean_prediction - - # batch_mean_label is E[y_B] in the update equation - batch_mean_label = math_ops.div_no_nan( - math_ops.reduce_sum(weighted_labels), batch_count) - delta_mean_label = math_ops.div_no_nan( - (batch_mean_label - mean_label) * batch_count, update_count) - update_mean_label = state_ops.assign_add(mean_label, delta_mean_label) - # prev_mean_label is E[y_A] in the update equation - prev_mean_label = update_mean_label - delta_mean_label - - unweighted_batch_coresiduals = ((predictions - batch_mean_prediction) * - (labels - batch_mean_label)) - # batch_comoment is C_B in the update equation - if weights is None: - batch_comoment = math_ops.reduce_sum(unweighted_batch_coresiduals) - else: - batch_comoment = math_ops.reduce_sum(unweighted_batch_coresiduals * - weights) - - # View delta_comoment as = C_AB - C_A in the update equation above. - # Since C_A is stored in a var, by how much do we need to increment that var - # to make the var = C_AB? - delta_comoment = ( - batch_comoment + (prev_mean_prediction - batch_mean_prediction) * - (prev_mean_label - batch_mean_label) * - (prev_count * batch_count / update_count)) - update_comoment = state_ops.assign_add(comoment, delta_comoment) - - covariance = array_ops.where( - math_ops.less_equal(count_, 1.), - float('nan'), - math_ops.truediv(comoment, count_ - 1), - name='covariance') - with ops.control_dependencies([update_comoment]): - update_op = array_ops.where( - math_ops.less_equal(count_, 1.), - float('nan'), - math_ops.truediv(comoment, count_ - 1), - name='update_op') - - if metrics_collections: - ops.add_to_collections(metrics_collections, covariance) - - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - - return covariance, update_op - - -def streaming_pearson_correlation(predictions, - labels, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes Pearson correlation coefficient between `predictions`, `labels`. - - The `streaming_pearson_correlation` function delegates to - `streaming_covariance` the tracking of three [co]variances: - - - `streaming_covariance(predictions, labels)`, i.e. covariance - - `streaming_covariance(predictions, predictions)`, i.e. variance - - `streaming_covariance(labels, labels)`, i.e. variance - - The product-moment correlation ultimately returned is an idempotent operation - `cov(predictions, labels) / sqrt(var(predictions) * var(labels))`. To - facilitate correlation computation across multiple batches, the function - groups the `update_op`s of the underlying streaming_covariance and returns an - `update_op`. - - If `weights` is not None, then it is used to compute a weighted correlation. - NOTE: these weights are treated as "frequency weights", as opposed to - "reliability weights". See discussion of the difference on - https://wikipedia.org/wiki/Weighted_arithmetic_mean#Weighted_sample_variance - - Args: - predictions: A `Tensor` of arbitrary size. - labels: A `Tensor` of the same size as predictions. - weights: Optional `Tensor` indicating the frequency with which an example is - sampled. Rank must be 0, or the same rank as `labels`, and must be - broadcastable to `labels` (i.e., all dimensions must be either `1`, or the - same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that the metric value - variable should be added to. - updates_collections: An optional list of collections that the metric update - ops should be added to. - name: An optional variable_scope name. - - Returns: - pearson_r: A `Tensor` representing the current Pearson product-moment - correlation coefficient, the value of - `cov(predictions, labels) / sqrt(var(predictions) * var(labels))`. - update_op: An operation that updates the underlying variables appropriately. - - Raises: - ValueError: If `labels` and `predictions` are of different sizes, or if - `weights` is the wrong size, or if either `metrics_collections` or - `updates_collections` are not a `list` or `tuple`. - """ - with variable_scope.variable_scope(name, 'pearson_r', - (predictions, labels, weights)): - predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access - predictions, labels, weights) - predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - # Broadcast weights here to avoid duplicate broadcasting in each call to - # `streaming_covariance`. - if weights is not None: - weights = weights_broadcast_ops.broadcast_weights(weights, labels) - cov, update_cov = streaming_covariance( - predictions, labels, weights=weights, name='covariance') - var_predictions, update_var_predictions = streaming_covariance( - predictions, predictions, weights=weights, name='variance_predictions') - var_labels, update_var_labels = streaming_covariance( - labels, labels, weights=weights, name='variance_labels') - - pearson_r = math_ops.truediv( - cov, - math_ops.multiply( - math_ops.sqrt(var_predictions), math_ops.sqrt(var_labels)), - name='pearson_r') - update_op = math_ops.truediv( - update_cov, - math_ops.multiply( - math_ops.sqrt(update_var_predictions), - math_ops.sqrt(update_var_labels)), - name='update_op') - - if metrics_collections: - ops.add_to_collections(metrics_collections, pearson_r) - - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - - return pearson_r, update_op - - -# TODO(nsilberman): add a 'normalized' flag so that the user can request -# normalization if the inputs are not normalized. -def streaming_mean_cosine_distance(predictions, - labels, - dim, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the cosine distance between the labels and predictions. - - The `streaming_mean_cosine_distance` function creates two local variables, - `total` and `count` that are used to compute the average cosine distance - between `predictions` and `labels`. This average is weighted by `weights`, - and it is ultimately returned as `mean_distance`, which is an idempotent - operation that simply divides `total` by `count`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `mean_distance`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: A `Tensor` of the same shape as `labels`. - labels: A `Tensor` of arbitrary shape. - dim: The dimension along which the cosine distance is computed. - weights: An optional `Tensor` whose shape is broadcastable to `predictions`, - and whose dimension `dim` is 1. - metrics_collections: An optional list of collections that the metric value - variable should be added to. - updates_collections: An optional list of collections that the metric update - ops should be added to. - name: An optional variable_scope name. - - Returns: - mean_distance: A `Tensor` representing the current mean, the value of - `total` divided by `count`. - update_op: An operation that increments the `total` and `count` variables - appropriately. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access - predictions, labels, weights) - predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - radial_diffs = math_ops.multiply(predictions, labels) - radial_diffs = math_ops.reduce_sum( - radial_diffs, axis=[ - dim, - ], keepdims=True) - mean_distance, update_op = streaming_mean(radial_diffs, weights, None, None, - name or 'mean_cosine_distance') - mean_distance = math_ops.subtract(1.0, mean_distance) - update_op = math_ops.subtract(1.0, update_op) - - if metrics_collections: - ops.add_to_collections(metrics_collections, mean_distance) - - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - - return mean_distance, update_op - - -def streaming_percentage_less(values, - threshold, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the percentage of values less than the given threshold. - - The `streaming_percentage_less` function creates two local variables, - `total` and `count` that are used to compute the percentage of `values` that - fall below `threshold`. This rate is weighted by `weights`, and it is - ultimately returned as `percentage` which is an idempotent operation that - simply divides `total` by `count`. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `percentage`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - values: A numeric `Tensor` of arbitrary size. - threshold: A scalar threshold. - weights: An optional `Tensor` whose shape is broadcastable to `values`. - metrics_collections: An optional list of collections that the metric value - variable should be added to. - updates_collections: An optional list of collections that the metric update - ops should be added to. - name: An optional variable_scope name. - - Returns: - percentage: A `Tensor` representing the current mean, the value of `total` - divided by `count`. - update_op: An operation that increments the `total` and `count` variables - appropriately. - - Raises: - ValueError: If `weights` is not `None` and its shape doesn't match `values`, - or if either `metrics_collections` or `updates_collections` are not a list - or tuple. - """ - return metrics.percentage_below( - values=values, - threshold=threshold, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -def streaming_mean_iou(predictions, - labels, - num_classes, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Calculate per-step mean Intersection-Over-Union (mIOU). - - Mean Intersection-Over-Union is a common evaluation metric for - semantic image segmentation, which first computes the IOU for each - semantic class and then computes the average over classes. - IOU is defined as follows: - IOU = true_positive / (true_positive + false_positive + false_negative). - The predictions are accumulated in a confusion matrix, weighted by `weights`, - and mIOU is then calculated from it. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the `mean_iou`. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - predictions: A `Tensor` of prediction results for semantic labels, whose - shape is [batch size] and type `int32` or `int64`. The tensor will be - flattened, if its rank > 1. - labels: A `Tensor` of ground truth labels with shape [batch size] and of - type `int32` or `int64`. The tensor will be flattened, if its rank > 1. - num_classes: The possible number of labels the prediction task can have. - This value must be provided, since a confusion matrix of dimension = - [num_classes, num_classes] will be allocated. - weights: An optional `Tensor` whose shape is broadcastable to `predictions`. - metrics_collections: An optional list of collections that `mean_iou` should - be added to. - updates_collections: An optional list of collections `update_op` should be - added to. - name: An optional variable_scope name. - - Returns: - mean_iou: A `Tensor` representing the mean intersection-over-union. - update_op: An operation that increments the confusion matrix. - - Raises: - ValueError: If `predictions` and `labels` have mismatched shapes, or if - `weights` is not `None` and its shape doesn't match `predictions`, or if - either `metrics_collections` or `updates_collections` are not a list or - tuple. - """ - return metrics.mean_iou( - num_classes=num_classes, - predictions=predictions, - labels=labels, - weights=weights, - metrics_collections=metrics_collections, - updates_collections=updates_collections, - name=name) - - -def _next_array_size(required_size, growth_factor=1.5): - """Calculate the next size for reallocating a dynamic array. - - Args: - required_size: number or tf.Tensor specifying required array capacity. - growth_factor: optional number or tf.Tensor specifying the growth factor - between subsequent allocations. - - Returns: - tf.Tensor with dtype=int32 giving the next array size. - """ - exponent = math_ops.ceil( - math_ops.log(math_ops.cast(required_size, dtypes.float32)) / - math_ops.log(math_ops.cast(growth_factor, dtypes.float32))) - return math_ops.cast(math_ops.ceil(growth_factor**exponent), dtypes.int32) - - -def streaming_concat(values, - axis=0, - max_size=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Concatenate values along an axis across batches. - - The function `streaming_concat` creates two local variables, `array` and - `size`, that are used to store concatenated values. Internally, `array` is - used as storage for a dynamic array (if `maxsize` is `None`), which ensures - that updates can be run in amortized constant time. - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that appends the values of a tensor and returns the - length of the concatenated axis. - - This op allows for evaluating metrics that cannot be updated incrementally - using the same framework as other streaming metrics. - - Args: - values: `Tensor` to concatenate. Rank and the shape along all axes other - than the axis to concatenate along must be statically known. - axis: optional integer axis to concatenate along. - max_size: optional integer maximum size of `value` along the given axis. - Once the maximum size is reached, further updates are no-ops. By default, - there is no maximum size: the array is resized as necessary. - metrics_collections: An optional list of collections that `value` should be - added to. - updates_collections: An optional list of collections `update_op` should be - added to. - name: An optional variable_scope name. - - Returns: - value: A `Tensor` representing the concatenated values. - update_op: An operation that concatenates the next values. - - Raises: - ValueError: if `values` does not have a statically known rank, `axis` is - not in the valid range or the size of `values` is not statically known - along any axis other than `axis`. - """ - with variable_scope.variable_scope(name, 'streaming_concat', (values,)): - # pylint: disable=invalid-slice-index - values_shape = values.get_shape() - if values_shape.dims is None: - raise ValueError('`values` must have known statically known rank') - - ndim = len(values_shape) - if axis < 0: - axis += ndim - if not 0 <= axis < ndim: - raise ValueError('axis = %r not in [0, %r)' % (axis, ndim)) - - fixed_shape = [dim.value for n, dim in enumerate(values_shape) if n != axis] - if any(value is None for value in fixed_shape): - raise ValueError('all dimensions of `values` other than the dimension to ' - 'concatenate along must have statically known size') - - # We move `axis` to the front of the internal array so assign ops can be - # applied to contiguous slices - init_size = 0 if max_size is None else max_size - init_shape = [init_size] + fixed_shape - array = metrics_impl.metric_variable( - init_shape, values.dtype, validate_shape=False, name='array') - size = metrics_impl.metric_variable([], dtypes.int32, name='size') - - perm = [0 if n == axis else n + 1 if n < axis else n for n in range(ndim)] - valid_array = array[:size] - valid_array.set_shape([None] + fixed_shape) - value = array_ops.transpose(valid_array, perm, name='concat') - - values_size = array_ops.shape(values)[axis] - if max_size is None: - batch_size = values_size - else: - batch_size = math_ops.minimum(values_size, max_size - size) - - perm = [axis] + [n for n in range(ndim) if n != axis] - batch_values = array_ops.transpose(values, perm)[:batch_size] - - def reallocate(): - next_size = _next_array_size(new_size) - next_shape = array_ops.stack([next_size] + fixed_shape) - new_value = array_ops.zeros(next_shape, dtype=values.dtype) - old_value = array.value() - with ops.control_dependencies([old_value]): - assign_op = state_ops.assign(array, new_value, validate_shape=False) - with ops.control_dependencies([assign_op]): - copy_op = array[:size].assign(old_value[:size]) - # return value needs to be the same dtype as no_op() for cond - with ops.control_dependencies([copy_op]): - return control_flow_ops.no_op() - - new_size = size + batch_size - array_size = array_ops.shape_internal(array, optimize=False)[0] - maybe_reallocate_op = control_flow_ops.cond(new_size > array_size, - reallocate, - control_flow_ops.no_op) - with ops.control_dependencies([maybe_reallocate_op]): - append_values_op = array[size:new_size].assign(batch_values) - with ops.control_dependencies([append_values_op]): - update_op = size.assign(new_size) - - if metrics_collections: - ops.add_to_collections(metrics_collections, value) - - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - - return value, update_op - # pylint: enable=invalid-slice-index - - -def aggregate_metrics(*value_update_tuples): - """Aggregates the metric value tensors and update ops into two lists. - - Args: - *value_update_tuples: a variable number of tuples, each of which contain the - pair of (value_tensor, update_op) from a streaming metric. - - Returns: - A list of value `Tensor` objects and a list of update ops. - - Raises: - ValueError: if `value_update_tuples` is empty. - """ - if not value_update_tuples: - raise ValueError('Expected at least one value_tensor/update_op pair') - value_ops, update_ops = zip(*value_update_tuples) - return list(value_ops), list(update_ops) - - -def aggregate_metric_map(names_to_tuples): - """Aggregates the metric names to tuple dictionary. - - This function is useful for pairing metric names with their associated value - and update ops when the list of metrics is long. For example: - - ```python - metrics_to_values, metrics_to_updates = slim.metrics.aggregate_metric_map({ - 'Mean Absolute Error': new_slim.metrics.streaming_mean_absolute_error( - predictions, labels, weights), - 'Mean Relative Error': new_slim.metrics.streaming_mean_relative_error( - predictions, labels, labels, weights), - 'RMSE Linear': new_slim.metrics.streaming_root_mean_squared_error( - predictions, labels, weights), - 'RMSE Log': new_slim.metrics.streaming_root_mean_squared_error( - predictions, labels, weights), - }) - ``` - - Args: - names_to_tuples: a map of metric names to tuples, each of which contain the - pair of (value_tensor, update_op) from a streaming metric. - - Returns: - A dictionary from metric names to value ops and a dictionary from metric - names to update ops. - """ - metric_names = names_to_tuples.keys() - value_ops, update_ops = zip(*names_to_tuples.values()) - return dict(zip(metric_names, value_ops)), dict(zip(metric_names, update_ops)) - - -def count(values, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Computes the number of examples, or sum of `weights`. - - This metric keeps track of the denominator in `tf.compat.v1.metrics.mean`. - When evaluating some metric (e.g. mean) on one or more subsets of the data, - this auxiliary metric is useful for keeping track of how many examples there - are in each subset. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - Args: - values: A `Tensor` of arbitrary dimensions. Only it's shape is used. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions must - be either `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that the metric value - variable should be added to. - updates_collections: An optional list of collections that the metric update - ops should be added to. - name: An optional variable_scope name. - - Returns: - count: A `Tensor` representing the current value of the metric. - update_op: An operation that accumulates the metric from a batch of data. - - Raises: - ValueError: If `weights` is not `None` and its shape doesn't match `values`, - or if either `metrics_collections` or `updates_collections` are not a list - or tuple. - RuntimeError: If eager execution is enabled. - """ - if context.executing_eagerly(): - raise RuntimeError('tf.contrib.metrics.count is not supported when eager ' - 'execution is enabled.') - - with variable_scope.variable_scope(name, 'count', (values, weights)): - - count_ = metrics_impl.metric_variable([], dtypes.float32, name='count') - - if weights is None: - num_values = math_ops.cast(array_ops.size(values), dtypes.float32) - else: - values = math_ops.cast(values, dtypes.float32) - values, _, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access - predictions=values, - labels=None, - weights=weights) - weights = weights_broadcast_ops.broadcast_weights( - math_ops.cast(weights, dtypes.float32), values) - num_values = math_ops.reduce_sum(weights) - - with ops.control_dependencies([values]): - update_count_op = state_ops.assign_add(count_, num_values) - - count_ = metrics_impl._aggregate_variable(count_, metrics_collections) # pylint: disable=protected-access - - if updates_collections: - ops.add_to_collections(updates_collections, update_count_op) - - return count_, update_count_op - - -def cohen_kappa(labels, - predictions_idx, - num_classes, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): - """Calculates Cohen's kappa. - - [Cohen's kappa](https://en.wikipedia.org/wiki/Cohen's_kappa) is a statistic - that measures inter-annotator agreement. - - The `cohen_kappa` function calculates the confusion matrix, and creates three - local variables to compute the Cohen's kappa: `po`, `pe_row`, and `pe_col`, - which refer to the diagonal part, rows and columns totals of the confusion - matrix, respectively. This value is ultimately returned as `kappa`, an - idempotent operation that is calculated by - - pe = (pe_row * pe_col) / N - k = (sum(po) - sum(pe)) / (N - sum(pe)) - - For estimation of the metric over a stream of data, the function creates an - `update_op` operation that updates these variables and returns the - `kappa`. `update_op` weights each prediction by the corresponding value in - `weights`. - - Class labels are expected to start at 0. E.g., if `num_classes` - was three, then the possible labels would be [0, 1, 2]. - - If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. - - NOTE: Equivalent to `sklearn.metrics.cohen_kappa_score`, but the method - doesn't support weighted matrix yet. - - Args: - labels: 1-D `Tensor` of real labels for the classification task. Must be - one of the following types: int16, int32, int64. - predictions_idx: 1-D `Tensor` of predicted class indices for a given - classification. Must have the same type as `labels`. - num_classes: The possible number of labels. - weights: Optional `Tensor` whose shape matches `predictions`. - metrics_collections: An optional list of collections that `kappa` should be - added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - name: An optional variable_scope name. - - Returns: - kappa: Scalar float `Tensor` representing the current Cohen's kappa. - update_op: `Operation` that increments `po`, `pe_row` and `pe_col` - variables appropriately and whose value matches `kappa`. - - Raises: - ValueError: If `num_classes` is less than 2, or `predictions` and `labels` - have mismatched shapes, or if `weights` is not `None` and its shape - doesn't match `predictions`, or if either `metrics_collections` or - `updates_collections` are not a list or tuple. - RuntimeError: If eager execution is enabled. - """ - if context.executing_eagerly(): - raise RuntimeError('tf.contrib.metrics.cohen_kappa is not supported ' - 'when eager execution is enabled.') - if num_classes < 2: - raise ValueError('`num_classes` must be >= 2.' - 'Found: {}'.format(num_classes)) - with variable_scope.variable_scope(name, 'cohen_kappa', - (labels, predictions_idx, weights)): - # Convert 2-dim (num, 1) to 1-dim (num,) - labels.get_shape().with_rank_at_most(2) - if labels.get_shape().ndims == 2: - labels = array_ops.squeeze(labels, axis=[-1]) - predictions_idx, labels, weights = ( - metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access - predictions=predictions_idx, - labels=labels, - weights=weights)) - predictions_idx.get_shape().assert_is_compatible_with(labels.get_shape()) - - stat_dtype = ( - dtypes.int64 - if weights is None or weights.dtype.is_integer else dtypes.float32) - po = metrics_impl.metric_variable((num_classes,), stat_dtype, name='po') - pe_row = metrics_impl.metric_variable((num_classes,), - stat_dtype, - name='pe_row') - pe_col = metrics_impl.metric_variable((num_classes,), - stat_dtype, - name='pe_col') - - # Table of the counts of agreement: - counts_in_table = confusion_matrix.confusion_matrix( - labels, - predictions_idx, - num_classes=num_classes, - weights=weights, - dtype=stat_dtype, - name='counts_in_table') - - po_t = array_ops.diag_part(counts_in_table) - pe_row_t = math_ops.reduce_sum(counts_in_table, axis=0) - pe_col_t = math_ops.reduce_sum(counts_in_table, axis=1) - update_po = state_ops.assign_add(po, po_t) - update_pe_row = state_ops.assign_add(pe_row, pe_row_t) - update_pe_col = state_ops.assign_add(pe_col, pe_col_t) - - def _calculate_k(po, pe_row, pe_col, name): - po_sum = math_ops.reduce_sum(po) - total = math_ops.reduce_sum(pe_row) - pe_sum = math_ops.reduce_sum( - math_ops.div_no_nan( - math_ops.cast(pe_row * pe_col, dtypes.float64), - math_ops.cast(total, dtypes.float64))) - po_sum, pe_sum, total = (math_ops.cast(po_sum, dtypes.float64), - math_ops.cast(pe_sum, dtypes.float64), - math_ops.cast(total, dtypes.float64)) - # kappa = (po - pe) / (N - pe) - k = metrics_impl._safe_scalar_div( # pylint: disable=protected-access - po_sum - pe_sum, - total - pe_sum, - name=name) - return k - - kappa = _calculate_k(po, pe_row, pe_col, name='value') - update_op = _calculate_k( - update_po, update_pe_row, update_pe_col, name='update_op') - - if metrics_collections: - ops.add_to_collections(metrics_collections, kappa) - - if updates_collections: - ops.add_to_collections(updates_collections, update_op) - - return kappa, update_op - - -__all__ = [ - 'auc_with_confidence_intervals', - 'aggregate_metric_map', - 'aggregate_metrics', - 'cohen_kappa', - 'count', - 'precision_recall_at_equal_thresholds', - 'recall_at_precision', - 'sparse_recall_at_top_k', - 'streaming_accuracy', - 'streaming_auc', - 'streaming_curve_points', - 'streaming_dynamic_auc', - 'streaming_false_negative_rate', - 'streaming_false_negative_rate_at_thresholds', - 'streaming_false_negatives', - 'streaming_false_negatives_at_thresholds', - 'streaming_false_positive_rate', - 'streaming_false_positive_rate_at_thresholds', - 'streaming_false_positives', - 'streaming_false_positives_at_thresholds', - 'streaming_mean', - 'streaming_mean_absolute_error', - 'streaming_mean_cosine_distance', - 'streaming_mean_iou', - 'streaming_mean_relative_error', - 'streaming_mean_squared_error', - 'streaming_mean_tensor', - 'streaming_percentage_less', - 'streaming_precision', - 'streaming_precision_at_thresholds', - 'streaming_recall', - 'streaming_recall_at_k', - 'streaming_recall_at_thresholds', - 'streaming_root_mean_squared_error', - 'streaming_sensitivity_at_specificity', - 'streaming_sparse_average_precision_at_k', - 'streaming_sparse_average_precision_at_top_k', - 'streaming_sparse_precision_at_k', - 'streaming_sparse_precision_at_top_k', - 'streaming_sparse_recall_at_k', - 'streaming_specificity_at_sensitivity', - 'streaming_true_negatives', - 'streaming_true_negatives_at_thresholds', - 'streaming_true_positives', - 'streaming_true_positives_at_thresholds', -] diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_large_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_large_test.py deleted file mode 100644 index 5777e64c294..00000000000 --- a/tensorflow/contrib/metrics/python/ops/metric_ops_large_test.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Large tests for metric_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from six.moves import xrange # pylint: disable=redefined-builtin -from tensorflow.contrib.metrics.python.ops import metric_ops -from tensorflow.python.framework import dtypes as dtypes_lib -from tensorflow.python.framework import ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class StreamingPrecisionRecallAtEqualThresholdsLargeTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testLargeCase(self): - shape = [32, 512, 256, 1] - predictions = random_ops.random_uniform( - shape, 0.0, 1.0, dtype=dtypes_lib.float32) - labels = math_ops.greater(random_ops.random_uniform(shape, 0.0, 1.0), 0.5) - - result, update_op = metric_ops.precision_recall_at_equal_thresholds( - labels=labels, predictions=predictions, num_thresholds=201) - # Run many updates, enough to cause highly inaccurate values if the - # code used float32 for accumulation. - num_updates = 71 - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - for _ in xrange(num_updates): - sess.run(update_op) - - prdata = sess.run(result) - - # Since we use random values, we won't know the tp/fp/tn/fn values, but - # tp and fp at threshold 0 should be the total number of positive and - # negative labels, hence their sum should be total number of pixels. - expected_value = 1.0 * np.product(shape) * num_updates - got_value = prdata.tp[0] + prdata.fp[0] - # They should be at least within 1. - self.assertNear(got_value, expected_value, 1.0) - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py deleted file mode 100644 index 906bebe3b82..00000000000 --- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py +++ /dev/null @@ -1,7276 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for metric_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -import numpy as np -from six.moves import xrange # pylint: disable=redefined-builtin -from tensorflow.contrib import metrics as metrics_lib -from tensorflow.contrib.metrics.python.ops import metric_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes as dtypes_lib -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - -NAN = float('nan') - -metrics = metrics_lib - - -def _enqueue_vector(sess, queue, values, shape=None): - if not shape: - shape = (1, len(values)) - dtype = queue.dtypes[0] - sess.run( - queue.enqueue(constant_op.constant(values, dtype=dtype, shape=shape))) - - -def _binary_2d_label_to_sparse_value(labels): - """Convert dense 2D binary indicator tensor to sparse tensor. - - Only 1 values in `labels` are included in result. - - Args: - labels: Dense 2D binary indicator tensor. - - Returns: - `SparseTensorValue` whose values are indices along the last dimension of - `labels`. - """ - indices = [] - values = [] - batch = 0 - for row in labels: - label = 0 - xi = 0 - for x in row: - if x == 1: - indices.append([batch, xi]) - values.append(label) - xi += 1 - else: - assert x == 0 - label += 1 - batch += 1 - shape = [len(labels), len(labels[0])] - return sparse_tensor.SparseTensorValue( - np.array(indices, np.int64), np.array(values, np.int64), - np.array(shape, np.int64)) - - -def _binary_2d_label_to_sparse(labels): - """Convert dense 2D binary indicator tensor to sparse tensor. - - Only 1 values in `labels` are included in result. - - Args: - labels: Dense 2D binary indicator tensor. - - Returns: - `SparseTensor` whose values are indices along the last dimension of - `labels`. - """ - return sparse_tensor.SparseTensor.from_value( - _binary_2d_label_to_sparse_value(labels)) - - -def _binary_3d_label_to_sparse_value(labels): - """Convert dense 3D binary indicator tensor to sparse tensor. - - Only 1 values in `labels` are included in result. - - Args: - labels: Dense 2D binary indicator tensor. - - Returns: - `SparseTensorValue` whose values are indices along the last dimension of - `labels`. - """ - indices = [] - values = [] - for d0, labels_d0 in enumerate(labels): - for d1, labels_d1 in enumerate(labels_d0): - d2 = 0 - for class_id, label in enumerate(labels_d1): - if label == 1: - values.append(class_id) - indices.append([d0, d1, d2]) - d2 += 1 - else: - assert label == 0 - shape = [len(labels), len(labels[0]), len(labels[0][0])] - return sparse_tensor.SparseTensorValue( - np.array(indices, np.int64), np.array(values, np.int64), - np.array(shape, np.int64)) - - -def _binary_3d_label_to_sparse(labels): - """Convert dense 3D binary indicator tensor to sparse tensor. - - Only 1 values in `labels` are included in result. - - Args: - labels: Dense 2D binary indicator tensor. - - Returns: - `SparseTensor` whose values are indices along the last dimension of - `labels`. - """ - return sparse_tensor.SparseTensor.from_value( - _binary_3d_label_to_sparse_value(labels)) - - -def _assert_nan(test_case, actual): - test_case.assertTrue(math.isnan(actual), 'Expected NAN, got %s.' % actual) - - -def _assert_metric_variables(test_case, expected): - test_case.assertEquals( - set(expected), set(v.name for v in variables.local_variables())) - test_case.assertEquals( - set(expected), - set(v.name for v in ops.get_collection(ops.GraphKeys.METRIC_VARIABLES))) - - -class StreamingMeanTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_mean(array_ops.ones([4, 3])) - _assert_metric_variables(self, ('mean/count:0', 'mean/total:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_mean( - array_ops.ones([4, 3]), metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_mean( - array_ops.ones([4, 3]), updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testBasic(self): - with self.cached_session() as sess: - values_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, values_queue, [0, 1]) - _enqueue_vector(sess, values_queue, [-4.2, 9.1]) - _enqueue_vector(sess, values_queue, [6.5, 0]) - _enqueue_vector(sess, values_queue, [-3.2, 4.0]) - values = values_queue.dequeue() - - mean, update_op = metrics.streaming_mean(values) - - sess.run(variables.local_variables_initializer()) - for _ in range(4): - sess.run(update_op) - self.assertAlmostEqual(1.65, sess.run(mean), 5) - - def testUpdateOpsReturnsCurrentValue(self): - with self.cached_session() as sess: - values_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, values_queue, [0, 1]) - _enqueue_vector(sess, values_queue, [-4.2, 9.1]) - _enqueue_vector(sess, values_queue, [6.5, 0]) - _enqueue_vector(sess, values_queue, [-3.2, 4.0]) - values = values_queue.dequeue() - - mean, update_op = metrics.streaming_mean(values) - - sess.run(variables.local_variables_initializer()) - - self.assertAlmostEqual(0.5, sess.run(update_op), 5) - self.assertAlmostEqual(1.475, sess.run(update_op), 5) - self.assertAlmostEqual(12.4 / 6.0, sess.run(update_op), 5) - self.assertAlmostEqual(1.65, sess.run(update_op), 5) - - self.assertAlmostEqual(1.65, sess.run(mean), 5) - - def test1dWeightedValues(self): - with self.cached_session() as sess: - # Create the queue that populates the values. - values_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, values_queue, [0, 1]) - _enqueue_vector(sess, values_queue, [-4.2, 9.1]) - _enqueue_vector(sess, values_queue, [6.5, 0]) - _enqueue_vector(sess, values_queue, [-3.2, 4.0]) - values = values_queue.dequeue() - - # Create the queue that populates the weighted labels. - weights_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 1)) - _enqueue_vector(sess, weights_queue, [1]) - _enqueue_vector(sess, weights_queue, [0]) - _enqueue_vector(sess, weights_queue, [0]) - _enqueue_vector(sess, weights_queue, [1]) - weights = weights_queue.dequeue() - - mean, update_op = metrics.streaming_mean(values, weights) - - variables.local_variables_initializer().run() - for _ in range(4): - update_op.eval() - self.assertAlmostEqual((0 + 1 - 3.2 + 4.0) / 4.0, mean.eval(), 5) - - def test1dWeightedValues_placeholders(self): - with self.cached_session() as sess: - # Create the queue that populates the values. - feed_values = ((0, 1), (-4.2, 9.1), (6.5, 0), (-3.2, 4.0)) - values = array_ops.placeholder(dtype=dtypes_lib.float32) - - # Create the queue that populates the weighted labels. - weights_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1,)) - _enqueue_vector(sess, weights_queue, 1, shape=(1,)) - _enqueue_vector(sess, weights_queue, 0, shape=(1,)) - _enqueue_vector(sess, weights_queue, 0, shape=(1,)) - _enqueue_vector(sess, weights_queue, 1, shape=(1,)) - weights = weights_queue.dequeue() - - mean, update_op = metrics.streaming_mean(values, weights) - - variables.local_variables_initializer().run() - for i in range(4): - update_op.eval(feed_dict={values: feed_values[i]}) - self.assertAlmostEqual((0 + 1 - 3.2 + 4.0) / 4.0, mean.eval(), 5) - - def test2dWeightedValues(self): - with self.cached_session() as sess: - # Create the queue that populates the values. - values_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, values_queue, [0, 1]) - _enqueue_vector(sess, values_queue, [-4.2, 9.1]) - _enqueue_vector(sess, values_queue, [6.5, 0]) - _enqueue_vector(sess, values_queue, [-3.2, 4.0]) - values = values_queue.dequeue() - - # Create the queue that populates the weighted labels. - weights_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, weights_queue, [1, 1]) - _enqueue_vector(sess, weights_queue, [1, 0]) - _enqueue_vector(sess, weights_queue, [0, 1]) - _enqueue_vector(sess, weights_queue, [0, 0]) - weights = weights_queue.dequeue() - - mean, update_op = metrics.streaming_mean(values, weights) - - variables.local_variables_initializer().run() - for _ in range(4): - update_op.eval() - self.assertAlmostEqual((0 + 1 - 4.2 + 0) / 4.0, mean.eval(), 5) - - def test2dWeightedValues_placeholders(self): - with self.cached_session() as sess: - # Create the queue that populates the values. - feed_values = ((0, 1), (-4.2, 9.1), (6.5, 0), (-3.2, 4.0)) - values = array_ops.placeholder(dtype=dtypes_lib.float32) - - # Create the queue that populates the weighted labels. - weights_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(2,)) - _enqueue_vector(sess, weights_queue, [1, 1], shape=(2,)) - _enqueue_vector(sess, weights_queue, [1, 0], shape=(2,)) - _enqueue_vector(sess, weights_queue, [0, 1], shape=(2,)) - _enqueue_vector(sess, weights_queue, [0, 0], shape=(2,)) - weights = weights_queue.dequeue() - - mean, update_op = metrics.streaming_mean(values, weights) - - variables.local_variables_initializer().run() - for i in range(4): - update_op.eval(feed_dict={values: feed_values[i]}) - self.assertAlmostEqual((0 + 1 - 4.2 + 0) / 4.0, mean.eval(), 5) - - -class StreamingMeanTensorTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_mean_tensor(array_ops.ones([4, 3])) - _assert_metric_variables(self, - ('mean/total_tensor:0', 'mean/count_tensor:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_mean_tensor( - array_ops.ones([4, 3]), metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_mean_tensor( - array_ops.ones([4, 3]), updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testBasic(self): - with self.cached_session() as sess: - values_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, values_queue, [0, 1]) - _enqueue_vector(sess, values_queue, [-4.2, 9.1]) - _enqueue_vector(sess, values_queue, [6.5, 0]) - _enqueue_vector(sess, values_queue, [-3.2, 4.0]) - values = values_queue.dequeue() - - mean, update_op = metrics.streaming_mean_tensor(values) - - sess.run(variables.local_variables_initializer()) - for _ in range(4): - sess.run(update_op) - self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(mean)) - - def testMultiDimensional(self): - with self.cached_session() as sess: - values_queue = data_flow_ops.FIFOQueue( - 2, dtypes=dtypes_lib.float32, shapes=(2, 2, 2)) - _enqueue_vector( - sess, - values_queue, [[[1, 2], [1, 2]], [[1, 2], [1, 2]]], - shape=(2, 2, 2)) - _enqueue_vector( - sess, - values_queue, [[[1, 2], [1, 2]], [[3, 4], [9, 10]]], - shape=(2, 2, 2)) - values = values_queue.dequeue() - - mean, update_op = metrics.streaming_mean_tensor(values) - - sess.run(variables.local_variables_initializer()) - for _ in range(2): - sess.run(update_op) - self.assertAllClose([[[1, 2], [1, 2]], [[2, 3], [5, 6]]], sess.run(mean)) - - def testUpdateOpsReturnsCurrentValue(self): - with self.cached_session() as sess: - values_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, values_queue, [0, 1]) - _enqueue_vector(sess, values_queue, [-4.2, 9.1]) - _enqueue_vector(sess, values_queue, [6.5, 0]) - _enqueue_vector(sess, values_queue, [-3.2, 4.0]) - values = values_queue.dequeue() - - mean, update_op = metrics.streaming_mean_tensor(values) - - sess.run(variables.local_variables_initializer()) - - self.assertAllClose([[0, 1]], sess.run(update_op), 5) - self.assertAllClose([[-2.1, 5.05]], sess.run(update_op), 5) - self.assertAllClose([[2.3 / 3., 10.1 / 3.]], sess.run(update_op), 5) - self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(update_op), 5) - - self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(mean), 5) - - def testWeighted1d(self): - with self.cached_session() as sess: - # Create the queue that populates the values. - values_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, values_queue, [0, 1]) - _enqueue_vector(sess, values_queue, [-4.2, 9.1]) - _enqueue_vector(sess, values_queue, [6.5, 0]) - _enqueue_vector(sess, values_queue, [-3.2, 4.0]) - values = values_queue.dequeue() - - # Create the queue that populates the weights. - weights_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 1)) - _enqueue_vector(sess, weights_queue, [[1]]) - _enqueue_vector(sess, weights_queue, [[0]]) - _enqueue_vector(sess, weights_queue, [[1]]) - _enqueue_vector(sess, weights_queue, [[0]]) - weights = weights_queue.dequeue() - - mean, update_op = metrics.streaming_mean_tensor(values, weights) - - sess.run(variables.local_variables_initializer()) - for _ in range(4): - sess.run(update_op) - self.assertAllClose([[3.25, 0.5]], sess.run(mean), 5) - - def testWeighted2d_1(self): - with self.cached_session() as sess: - # Create the queue that populates the values. - values_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, values_queue, [0, 1]) - _enqueue_vector(sess, values_queue, [-4.2, 9.1]) - _enqueue_vector(sess, values_queue, [6.5, 0]) - _enqueue_vector(sess, values_queue, [-3.2, 4.0]) - values = values_queue.dequeue() - - # Create the queue that populates the weights. - weights_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, weights_queue, [1, 1]) - _enqueue_vector(sess, weights_queue, [1, 0]) - _enqueue_vector(sess, weights_queue, [0, 1]) - _enqueue_vector(sess, weights_queue, [0, 0]) - weights = weights_queue.dequeue() - - mean, update_op = metrics.streaming_mean_tensor(values, weights) - - sess.run(variables.local_variables_initializer()) - for _ in range(4): - sess.run(update_op) - self.assertAllClose([[-2.1, 0.5]], sess.run(mean), 5) - - def testWeighted2d_2(self): - with self.cached_session() as sess: - # Create the queue that populates the values. - values_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, values_queue, [0, 1]) - _enqueue_vector(sess, values_queue, [-4.2, 9.1]) - _enqueue_vector(sess, values_queue, [6.5, 0]) - _enqueue_vector(sess, values_queue, [-3.2, 4.0]) - values = values_queue.dequeue() - - # Create the queue that populates the weights. - weights_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, weights_queue, [0, 1]) - _enqueue_vector(sess, weights_queue, [0, 0]) - _enqueue_vector(sess, weights_queue, [0, 1]) - _enqueue_vector(sess, weights_queue, [0, 0]) - weights = weights_queue.dequeue() - - mean, update_op = metrics.streaming_mean_tensor(values, weights) - - sess.run(variables.local_variables_initializer()) - for _ in range(4): - sess.run(update_op) - self.assertAllClose([[0, 0.5]], sess.run(mean), 5) - - -class StreamingAccuracyTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_accuracy( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - name='my_accuracy') - _assert_metric_variables(self, - ('my_accuracy/count:0', 'my_accuracy/total:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_accuracy( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_accuracy( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testPredictionsAndLabelsOfDifferentSizeRaisesValueError(self): - predictions = array_ops.ones((10, 3)) - labels = array_ops.ones((10, 4)) - with self.assertRaises(ValueError): - metrics.streaming_accuracy(predictions, labels) - - def testPredictionsAndWeightsOfDifferentSizeRaisesValueError(self): - predictions = array_ops.ones((10, 3)) - labels = array_ops.ones((10, 3)) - weights = array_ops.ones((9, 3)) - with self.assertRaises(ValueError): - metrics.streaming_accuracy(predictions, labels, weights) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=3, dtype=dtypes_lib.int64, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=3, dtype=dtypes_lib.int64, seed=2) - accuracy, update_op = metrics.streaming_accuracy(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_accuracy = accuracy.eval() - for _ in range(10): - self.assertEqual(initial_accuracy, accuracy.eval()) - - def testMultipleUpdates(self): - with self.cached_session() as sess: - # Create the queue that populates the predictions. - preds_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 1)) - _enqueue_vector(sess, preds_queue, [0]) - _enqueue_vector(sess, preds_queue, [1]) - _enqueue_vector(sess, preds_queue, [2]) - _enqueue_vector(sess, preds_queue, [1]) - predictions = preds_queue.dequeue() - - # Create the queue that populates the labels. - labels_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 1)) - _enqueue_vector(sess, labels_queue, [0]) - _enqueue_vector(sess, labels_queue, [1]) - _enqueue_vector(sess, labels_queue, [1]) - _enqueue_vector(sess, labels_queue, [2]) - labels = labels_queue.dequeue() - - accuracy, update_op = metrics.streaming_accuracy(predictions, labels) - - sess.run(variables.local_variables_initializer()) - for _ in xrange(3): - sess.run(update_op) - self.assertEqual(0.5, sess.run(update_op)) - self.assertEqual(0.5, accuracy.eval()) - - def testEffectivelyEquivalentSizes(self): - predictions = array_ops.ones((40, 1)) - labels = array_ops.ones((40,)) - with self.cached_session() as sess: - accuracy, update_op = metrics.streaming_accuracy(predictions, labels) - - sess.run(variables.local_variables_initializer()) - self.assertEqual(1.0, update_op.eval()) - self.assertEqual(1.0, accuracy.eval()) - - def testEffectivelyEquivalentSizesWithStaicShapedWeight(self): - predictions = ops.convert_to_tensor([1, 1, 1]) # shape 3, - labels = array_ops.expand_dims(ops.convert_to_tensor([1, 0, 0]), - 1) # shape 3, 1 - weights = array_ops.expand_dims(ops.convert_to_tensor([100, 1, 1]), - 1) # shape 3, 1 - - with self.cached_session() as sess: - accuracy, update_op = metrics.streaming_accuracy(predictions, labels, - weights) - - sess.run(variables.local_variables_initializer()) - # if streaming_accuracy does not flatten the weight, accuracy would be - # 0.33333334 due to an intended broadcast of weight. Due to flattening, - # it will be higher than .95 - self.assertGreater(update_op.eval(), .95) - self.assertGreater(accuracy.eval(), .95) - - def testEffectivelyEquivalentSizesWithDynamicallyShapedWeight(self): - predictions = ops.convert_to_tensor([1, 1, 1]) # shape 3, - labels = array_ops.expand_dims(ops.convert_to_tensor([1, 0, 0]), - 1) # shape 3, 1 - - weights = [[100], [1], [1]] # shape 3, 1 - weights_placeholder = array_ops.placeholder( - dtype=dtypes_lib.int32, name='weights') - feed_dict = {weights_placeholder: weights} - - with self.cached_session() as sess: - accuracy, update_op = metrics.streaming_accuracy(predictions, labels, - weights_placeholder) - - sess.run(variables.local_variables_initializer()) - # if streaming_accuracy does not flatten the weight, accuracy would be - # 0.33333334 due to an intended broadcast of weight. Due to flattening, - # it will be higher than .95 - self.assertGreater(update_op.eval(feed_dict=feed_dict), .95) - self.assertGreater(accuracy.eval(feed_dict=feed_dict), .95) - - def testMultipleUpdatesWithWeightedValues(self): - with self.cached_session() as sess: - # Create the queue that populates the predictions. - preds_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 1)) - _enqueue_vector(sess, preds_queue, [0]) - _enqueue_vector(sess, preds_queue, [1]) - _enqueue_vector(sess, preds_queue, [2]) - _enqueue_vector(sess, preds_queue, [1]) - predictions = preds_queue.dequeue() - - # Create the queue that populates the labels. - labels_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 1)) - _enqueue_vector(sess, labels_queue, [0]) - _enqueue_vector(sess, labels_queue, [1]) - _enqueue_vector(sess, labels_queue, [1]) - _enqueue_vector(sess, labels_queue, [2]) - labels = labels_queue.dequeue() - - # Create the queue that populates the weights. - weights_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.int64, shapes=(1, 1)) - _enqueue_vector(sess, weights_queue, [1]) - _enqueue_vector(sess, weights_queue, [1]) - _enqueue_vector(sess, weights_queue, [0]) - _enqueue_vector(sess, weights_queue, [0]) - weights = weights_queue.dequeue() - - accuracy, update_op = metrics.streaming_accuracy(predictions, labels, - weights) - - sess.run(variables.local_variables_initializer()) - for _ in xrange(3): - sess.run(update_op) - self.assertEqual(1.0, sess.run(update_op)) - self.assertEqual(1.0, accuracy.eval()) - - -class StreamingTruePositivesTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_true_positives((0, 1, 0), (0, 1, 1)) - _assert_metric_variables(self, ('true_positives/count:0',)) - - def testUnweighted(self): - for expand_predictions in [True, False]: - for expand_labels in [True, False]: - for dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32): - predictions = math_ops.cast( - constant_op.constant(((1, 0, 1, 0), (0, 1, 1, 1), (0, 0, 0, 0))), - dtype=dtype) - if expand_predictions: - predictions = array_ops.expand_dims(predictions, 2) - labels = math_ops.cast( - constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))), - dtype=dtype) - if expand_labels: - labels = array_ops.expand_dims(labels, 2) - tp, tp_update_op = metrics.streaming_true_positives( - predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, tp.eval()) - self.assertEqual(1, tp_update_op.eval()) - self.assertEqual(1, tp.eval()) - - def testWeighted(self): - for dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32): - predictions = math_ops.cast( - constant_op.constant(((1, 0, 1, 0), (0, 1, 1, 1), (0, 0, 0, 0))), - dtype=dtype) - labels = math_ops.cast( - constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))), - dtype=dtype) - tp, tp_update_op = metrics.streaming_true_positives( - predictions, labels, weights=37.0) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, tp.eval()) - self.assertEqual(37.0, tp_update_op.eval()) - self.assertEqual(37.0, tp.eval()) - - -class StreamingFalseNegativesTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_false_negatives((0, 1, 0), (0, 1, 1)) - _assert_metric_variables(self, ('false_negatives/count:0',)) - - def testUnweighted(self): - for expand_predictions in [True, False]: - for expand_labels in [True, False]: - for dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32): - predictions = math_ops.cast( - constant_op.constant(((1, 0, 1, 0), (0, 1, 1, 1), (0, 0, 0, 0))), - dtype=dtype) - if expand_predictions: - predictions = array_ops.expand_dims(predictions, 2) - labels = math_ops.cast( - constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))), - dtype=dtype) - if expand_labels: - labels = array_ops.expand_dims(labels, 2) - fn, fn_update_op = metrics.streaming_false_negatives( - predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, fn.eval()) - self.assertEqual(2, fn_update_op.eval()) - self.assertEqual(2, fn.eval()) - - def testWeighted(self): - for dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32): - predictions = math_ops.cast( - constant_op.constant(((1, 0, 1, 0), (0, 1, 1, 1), (0, 0, 0, 0))), - dtype=dtype) - labels = math_ops.cast( - constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))), - dtype=dtype) - fn, fn_update_op = metrics.streaming_false_negatives( - predictions, labels, weights=((3.0,), (5.0,), (7.0,))) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, fn.eval()) - self.assertEqual(8.0, fn_update_op.eval()) - self.assertEqual(8.0, fn.eval()) - - -class StreamingFalsePositivesTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_false_positives((0, 1, 0), (0, 1, 1)) - _assert_metric_variables(self, ('false_positives/count:0',)) - - def testUnweighted(self): - for expand_predictions in [True, False]: - for expand_labels in [True, False]: - for dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32): - predictions = math_ops.cast( - constant_op.constant(((1, 0, 1, 0), (0, 1, 1, 1), (0, 0, 0, 0))), - dtype=dtype) - if expand_predictions: - predictions = array_ops.expand_dims(predictions, 2) - labels = math_ops.cast( - constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))), - dtype=dtype) - if expand_labels: - labels = array_ops.expand_dims(labels, 2) - fp, fp_update_op = metrics.streaming_false_positives( - predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, fp.eval()) - self.assertEqual(4, fp_update_op.eval()) - self.assertEqual(4, fp.eval()) - - def testWeighted(self): - for dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32): - predictions = math_ops.cast( - constant_op.constant(((1, 0, 1, 0), (0, 1, 1, 1), (0, 0, 0, 0))), - dtype=dtype) - labels = math_ops.cast( - constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))), - dtype=dtype) - fp, fp_update_op = metrics.streaming_false_positives( - predictions, - labels, - weights=((1.0, 2.0, 3.0, 5.0), (7.0, 11.0, 13.0, 17.0), (19.0, 23.0, - 29.0, 31.0))) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, fp.eval()) - self.assertEqual(42.0, fp_update_op.eval()) - self.assertEqual(42.0, fp.eval()) - - -class StreamingTrueNegativesTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_true_negatives((0, 1, 0), (0, 1, 1)) - _assert_metric_variables(self, ('true_negatives/count:0',)) - - def testUnweighted(self): - for expand_predictions in [True, False]: - for expand_labels in [True, False]: - for dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32): - predictions = math_ops.cast( - constant_op.constant(((1, 0, 1, 0), (0, 1, 1, 1), (0, 0, 0, 0))), - dtype=dtype) - if expand_predictions: - predictions = array_ops.expand_dims(predictions, 2) - labels = math_ops.cast( - constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))), - dtype=dtype) - if expand_labels: - labels = array_ops.expand_dims(labels, 2) - tn, tn_update_op = metrics.streaming_true_negatives( - predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, tn.eval()) - self.assertEqual(5, tn_update_op.eval()) - self.assertEqual(5, tn.eval()) - - def testWeighted(self): - for dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32): - predictions = math_ops.cast( - constant_op.constant(((1, 0, 1, 0), (0, 1, 1, 1), (0, 0, 0, 0))), - dtype=dtype) - labels = math_ops.cast( - constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))), - dtype=dtype) - tn, tn_update_op = metrics.streaming_true_negatives( - predictions, labels, weights=((0.0, 2.0, 3.0, 5.0),)) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, tn.eval()) - self.assertEqual(15.0, tn_update_op.eval()) - self.assertEqual(15.0, tn.eval()) - - -class StreamingTruePositivesAtThresholdsTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_true_positives_at_thresholds( - (0.0, 1.0, 0.0), (0, 1, 1), thresholds=(0.15, 0.5, 0.85)) - _assert_metric_variables(self, ('true_positives:0',)) - - def testUnweighted(self): - predictions = constant_op.constant( - ((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3))) - labels = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))) - tp, tp_update_op = metrics.streaming_true_positives_at_thresholds( - predictions, labels, thresholds=(0.15, 0.5, 0.85)) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAllEqual((0, 0, 0), tp.eval()) - self.assertAllEqual((3, 1, 0), tp_update_op.eval()) - self.assertAllEqual((3, 1, 0), tp.eval()) - - def testWeighted(self): - predictions = constant_op.constant( - ((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3))) - labels = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))) - tp, tp_update_op = metrics.streaming_true_positives_at_thresholds( - predictions, labels, weights=37.0, thresholds=(0.15, 0.5, 0.85)) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAllEqual((0.0, 0.0, 0.0), tp.eval()) - self.assertAllEqual((111.0, 37.0, 0.0), tp_update_op.eval()) - self.assertAllEqual((111.0, 37.0, 0.0), tp.eval()) - - -class StreamingFalseNegativesAtThresholdsTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_false_negatives_at_thresholds( - (0.0, 1.0, 0.0), (0, 1, 1), thresholds=( - 0.15, - 0.5, - 0.85, - )) - _assert_metric_variables(self, ('false_negatives:0',)) - - def testUnweighted(self): - predictions = constant_op.constant( - ((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3))) - labels = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))) - fn, fn_update_op = metrics.streaming_false_negatives_at_thresholds( - predictions, labels, thresholds=(0.15, 0.5, 0.85)) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAllEqual((0, 0, 0), fn.eval()) - self.assertAllEqual((0, 2, 3), fn_update_op.eval()) - self.assertAllEqual((0, 2, 3), fn.eval()) - - def testWeighted(self): - predictions = constant_op.constant( - ((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3))) - labels = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))) - fn, fn_update_op = metrics.streaming_false_negatives_at_thresholds( - predictions, - labels, - weights=((3.0,), (5.0,), (7.0,)), - thresholds=(0.15, 0.5, 0.85)) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAllEqual((0.0, 0.0, 0.0), fn.eval()) - self.assertAllEqual((0.0, 8.0, 11.0), fn_update_op.eval()) - self.assertAllEqual((0.0, 8.0, 11.0), fn.eval()) - - -class StreamingFalsePositivesAtThresholdsTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_false_positives_at_thresholds( - (0.0, 1.0, 0.0), (0, 1, 1), thresholds=(0.15, 0.5, 0.85)) - _assert_metric_variables(self, ('false_positives:0',)) - - def testUnweighted(self): - predictions = constant_op.constant( - ((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3))) - labels = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))) - fp, fp_update_op = metrics.streaming_false_positives_at_thresholds( - predictions, labels, thresholds=(0.15, 0.5, 0.85)) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAllEqual((0, 0, 0), fp.eval()) - self.assertAllEqual((7, 4, 2), fp_update_op.eval()) - self.assertAllEqual((7, 4, 2), fp.eval()) - - def testWeighted(self): - predictions = constant_op.constant( - ((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3))) - labels = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))) - fp, fp_update_op = metrics.streaming_false_positives_at_thresholds( - predictions, - labels, - weights=((1.0, 2.0, 3.0, 5.0), (7.0, 11.0, 13.0, 17.0), (19.0, 23.0, - 29.0, 31.0)), - thresholds=(0.15, 0.5, 0.85)) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAllEqual((0.0, 0.0, 0.0), fp.eval()) - self.assertAllEqual((125.0, 42.0, 12.0), fp_update_op.eval()) - self.assertAllEqual((125.0, 42.0, 12.0), fp.eval()) - - -class StreamingTrueNegativesAtThresholdsTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_true_negatives_at_thresholds( - (0.0, 1.0, 0.0), (0, 1, 1), thresholds=(0.15, 0.5, 0.85)) - _assert_metric_variables(self, ('true_negatives:0',)) - - def testUnweighted(self): - predictions = constant_op.constant( - ((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3))) - labels = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))) - tn, tn_update_op = metrics.streaming_true_negatives_at_thresholds( - predictions, labels, thresholds=(0.15, 0.5, 0.85)) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAllEqual((0, 0, 0), tn.eval()) - self.assertAllEqual((2, 5, 7), tn_update_op.eval()) - self.assertAllEqual((2, 5, 7), tn.eval()) - - def testWeighted(self): - predictions = constant_op.constant( - ((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3))) - labels = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0))) - tn, tn_update_op = metrics.streaming_true_negatives_at_thresholds( - predictions, - labels, - weights=((0.0, 2.0, 3.0, 5.0),), - thresholds=(0.15, 0.5, 0.85)) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAllEqual((0.0, 0.0, 0.0), tn.eval()) - self.assertAllEqual((5.0, 15.0, 23.0), tn_update_op.eval()) - self.assertAllEqual((5.0, 15.0, 23.0), tn.eval()) - - -class StreamingPrecisionTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_precision( - predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) - _assert_metric_variables(self, ('precision/false_positives/count:0', - 'precision/true_positives/count:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_precision( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_precision( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) - precision, update_op = metrics.streaming_precision(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_precision = precision.eval() - for _ in range(10): - self.assertEqual(initial_precision, precision.eval()) - - def testAllCorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - predictions = constant_op.constant(inputs) - labels = constant_op.constant(inputs) - precision, update_op = metrics.streaming_precision(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(1, sess.run(update_op)) - self.assertAlmostEqual(1, precision.eval()) - - def testSomeCorrect(self): - predictions = constant_op.constant([1, 0, 1, 0], shape=(1, 4)) - labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) - precision, update_op = metrics.streaming_precision(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.5, update_op.eval()) - self.assertAlmostEqual(0.5, precision.eval()) - - def testWeighted1d(self): - predictions = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]]) - labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) - precision, update_op = metrics.streaming_precision( - predictions, labels, weights=constant_op.constant([[2], [5]])) - - with self.cached_session(): - variables.local_variables_initializer().run() - weighted_tp = 2.0 + 5.0 - weighted_positives = (2.0 + 2.0) + (5.0 + 5.0) - expected_precision = weighted_tp / weighted_positives - self.assertAlmostEqual(expected_precision, update_op.eval()) - self.assertAlmostEqual(expected_precision, precision.eval()) - - def testWeighted1d_placeholders(self): - predictions = array_ops.placeholder(dtype=dtypes_lib.float32) - labels = array_ops.placeholder(dtype=dtypes_lib.float32) - feed_dict = { - predictions: ((1, 0, 1, 0), (1, 0, 1, 0)), - labels: ((0, 1, 1, 0), (1, 0, 0, 1)) - } - precision, update_op = metrics.streaming_precision( - predictions, labels, weights=constant_op.constant([[2], [5]])) - - with self.cached_session(): - variables.local_variables_initializer().run() - weighted_tp = 2.0 + 5.0 - weighted_positives = (2.0 + 2.0) + (5.0 + 5.0) - expected_precision = weighted_tp / weighted_positives - self.assertAlmostEqual( - expected_precision, update_op.eval(feed_dict=feed_dict)) - self.assertAlmostEqual( - expected_precision, precision.eval(feed_dict=feed_dict)) - - def testWeighted2d(self): - predictions = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]]) - labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) - precision, update_op = metrics.streaming_precision( - predictions, - labels, - weights=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]])) - - with self.cached_session(): - variables.local_variables_initializer().run() - weighted_tp = 3.0 + 4.0 - weighted_positives = (1.0 + 3.0) + (4.0 + 2.0) - expected_precision = weighted_tp / weighted_positives - self.assertAlmostEqual(expected_precision, update_op.eval()) - self.assertAlmostEqual(expected_precision, precision.eval()) - - def testWeighted2d_placeholders(self): - predictions = array_ops.placeholder(dtype=dtypes_lib.float32) - labels = array_ops.placeholder(dtype=dtypes_lib.float32) - feed_dict = { - predictions: ((1, 0, 1, 0), (1, 0, 1, 0)), - labels: ((0, 1, 1, 0), (1, 0, 0, 1)) - } - precision, update_op = metrics.streaming_precision( - predictions, - labels, - weights=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]])) - - with self.cached_session(): - variables.local_variables_initializer().run() - weighted_tp = 3.0 + 4.0 - weighted_positives = (1.0 + 3.0) + (4.0 + 2.0) - expected_precision = weighted_tp / weighted_positives - self.assertAlmostEqual( - expected_precision, update_op.eval(feed_dict=feed_dict)) - self.assertAlmostEqual( - expected_precision, precision.eval(feed_dict=feed_dict)) - - def testAllIncorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - predictions = constant_op.constant(inputs) - labels = constant_op.constant(1 - inputs) - precision, update_op = metrics.streaming_precision(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertAlmostEqual(0, precision.eval()) - - def testZeroTrueAndFalsePositivesGivesZeroPrecision(self): - predictions = constant_op.constant([0, 0, 0, 0]) - labels = constant_op.constant([0, 0, 0, 0]) - precision, update_op = metrics.streaming_precision(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertEqual(0.0, precision.eval()) - - -class StreamingRecallTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_recall( - predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) - _assert_metric_variables( - self, - ('recall/false_negatives/count:0', 'recall/true_positives/count:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_recall( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_recall( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) - recall, update_op = metrics.streaming_recall(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_recall = recall.eval() - for _ in range(10): - self.assertEqual(initial_recall, recall.eval()) - - def testAllCorrect(self): - np_inputs = np.random.randint(0, 2, size=(100, 1)) - - predictions = constant_op.constant(np_inputs) - labels = constant_op.constant(np_inputs) - recall, update_op = metrics.streaming_recall(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertEqual(1, recall.eval()) - - def testSomeCorrect(self): - predictions = constant_op.constant([1, 0, 1, 0], shape=(1, 4)) - labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) - recall, update_op = metrics.streaming_recall(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.5, update_op.eval()) - self.assertAlmostEqual(0.5, recall.eval()) - - def testWeighted1d(self): - predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]]) - labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) - weights = constant_op.constant([[2], [5]]) - recall, update_op = metrics.streaming_recall( - predictions, labels, weights=weights) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - weighted_tp = 2.0 + 5.0 - weighted_t = (2.0 + 2.0) + (5.0 + 5.0) - expected_precision = weighted_tp / weighted_t - self.assertAlmostEqual(expected_precision, update_op.eval()) - self.assertAlmostEqual(expected_precision, recall.eval()) - - def testWeighted2d(self): - predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]]) - labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) - weights = constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]) - recall, update_op = metrics.streaming_recall( - predictions, labels, weights=weights) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - weighted_tp = 3.0 + 1.0 - weighted_t = (2.0 + 3.0) + (4.0 + 1.0) - expected_precision = weighted_tp / weighted_t - self.assertAlmostEqual(expected_precision, update_op.eval()) - self.assertAlmostEqual(expected_precision, recall.eval()) - - def testAllIncorrect(self): - np_inputs = np.random.randint(0, 2, size=(100, 1)) - - predictions = constant_op.constant(np_inputs) - labels = constant_op.constant(1 - np_inputs) - recall, update_op = metrics.streaming_recall(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertEqual(0, recall.eval()) - - def testZeroTruePositivesAndFalseNegativesGivesZeroRecall(self): - predictions = array_ops.zeros((1, 4)) - labels = array_ops.zeros((1, 4)) - recall, update_op = metrics.streaming_recall(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertEqual(0, recall.eval()) - - -class StreamingFPRTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_false_positive_rate( - predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) - _assert_metric_variables(self, - ('false_positive_rate/false_positives/count:0', - 'false_positive_rate/true_negatives/count:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_false_positive_rate( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_false_positive_rate( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) - fpr, update_op = metrics.streaming_false_positive_rate(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_fpr = fpr.eval() - for _ in range(10): - self.assertEqual(initial_fpr, fpr.eval()) - - def testAllCorrect(self): - np_inputs = np.random.randint(0, 2, size=(100, 1)) - - predictions = constant_op.constant(np_inputs) - labels = constant_op.constant(np_inputs) - fpr, update_op = metrics.streaming_false_positive_rate(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertEqual(0, fpr.eval()) - - def testSomeCorrect(self): - predictions = constant_op.constant([1, 0, 1, 0], shape=(1, 4)) - labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) - fpr, update_op = metrics.streaming_false_positive_rate(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.5, update_op.eval()) - self.assertAlmostEqual(0.5, fpr.eval()) - - def testWeighted1d(self): - predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]]) - labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) - weights = constant_op.constant([[2], [5]]) - fpr, update_op = metrics.streaming_false_positive_rate( - predictions, labels, weights=weights) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - weighted_fp = 2.0 + 5.0 - weighted_f = (2.0 + 2.0) + (5.0 + 5.0) - expected_fpr = weighted_fp / weighted_f - self.assertAlmostEqual(expected_fpr, update_op.eval()) - self.assertAlmostEqual(expected_fpr, fpr.eval()) - - def testWeighted2d(self): - predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]]) - labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) - weights = constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]) - fpr, update_op = metrics.streaming_false_positive_rate( - predictions, labels, weights=weights) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - weighted_fp = 1.0 + 3.0 - weighted_f = (1.0 + 4.0) + (2.0 + 3.0) - expected_fpr = weighted_fp / weighted_f - self.assertAlmostEqual(expected_fpr, update_op.eval()) - self.assertAlmostEqual(expected_fpr, fpr.eval()) - - def testAllIncorrect(self): - np_inputs = np.random.randint(0, 2, size=(100, 1)) - - predictions = constant_op.constant(np_inputs) - labels = constant_op.constant(1 - np_inputs) - fpr, update_op = metrics.streaming_false_positive_rate(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertEqual(1, fpr.eval()) - - def testZeroFalsePositivesAndTrueNegativesGivesZeroFPR(self): - predictions = array_ops.ones((1, 4)) - labels = array_ops.ones((1, 4)) - fpr, update_op = metrics.streaming_false_positive_rate(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertEqual(0, fpr.eval()) - - -class StreamingFNRTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_false_negative_rate( - predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) - _assert_metric_variables(self, - ('false_negative_rate/false_negatives/count:0', - 'false_negative_rate/true_positives/count:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_false_negative_rate( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_false_negative_rate( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) - fnr, update_op = metrics.streaming_false_negative_rate(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_fnr = fnr.eval() - for _ in range(10): - self.assertEqual(initial_fnr, fnr.eval()) - - def testAllCorrect(self): - np_inputs = np.random.randint(0, 2, size=(100, 1)) - - predictions = constant_op.constant(np_inputs) - labels = constant_op.constant(np_inputs) - fnr, update_op = metrics.streaming_false_negative_rate(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertEqual(0, fnr.eval()) - - def testSomeCorrect(self): - predictions = constant_op.constant([1, 0, 1, 0], shape=(1, 4)) - labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) - fnr, update_op = metrics.streaming_false_negative_rate(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.5, update_op.eval()) - self.assertAlmostEqual(0.5, fnr.eval()) - - def testWeighted1d(self): - predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]]) - labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) - weights = constant_op.constant([[2], [5]]) - fnr, update_op = metrics.streaming_false_negative_rate( - predictions, labels, weights=weights) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - weighted_fn = 2.0 + 5.0 - weighted_t = (2.0 + 2.0) + (5.0 + 5.0) - expected_fnr = weighted_fn / weighted_t - self.assertAlmostEqual(expected_fnr, update_op.eval()) - self.assertAlmostEqual(expected_fnr, fnr.eval()) - - def testWeighted2d(self): - predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]]) - labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) - weights = constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]) - fnr, update_op = metrics.streaming_false_negative_rate( - predictions, labels, weights=weights) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - weighted_fn = 2.0 + 4.0 - weighted_t = (2.0 + 3.0) + (1.0 + 4.0) - expected_fnr = weighted_fn / weighted_t - self.assertAlmostEqual(expected_fnr, update_op.eval()) - self.assertAlmostEqual(expected_fnr, fnr.eval()) - - def testAllIncorrect(self): - np_inputs = np.random.randint(0, 2, size=(100, 1)) - - predictions = constant_op.constant(np_inputs) - labels = constant_op.constant(1 - np_inputs) - fnr, update_op = metrics.streaming_false_negative_rate(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertEqual(1, fnr.eval()) - - def testZeroFalseNegativesAndTruePositivesGivesZeroFNR(self): - predictions = array_ops.zeros((1, 4)) - labels = array_ops.zeros((1, 4)) - fnr, update_op = metrics.streaming_false_negative_rate(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertEqual(0, fnr.eval()) - - -class StreamingCurvePointsTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metric_ops.streaming_curve_points( - predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) - _assert_metric_variables( - self, - ('curve_points/true_positives:0', 'curve_points/false_negatives:0', - 'curve_points/false_positives:0', 'curve_points/true_negatives:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - points, _ = metric_ops.streaming_curve_points( - labels=array_ops.ones((10, 1)), - predictions=array_ops.ones((10, 1)), - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [points]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metric_ops.streaming_curve_points( - labels=array_ops.ones((10, 1)), - predictions=array_ops.ones((10, 1)), - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def _testValueTensorIsIdempotent(self, curve): - predictions = constant_op.constant( - np.random.uniform(size=(10, 3)), dtype=dtypes_lib.float32) - labels = constant_op.constant( - np.random.uniform(high=2, size=(10, 3)), dtype=dtypes_lib.float32) - - points, update_op = metric_ops.streaming_curve_points( - labels, predictions=predictions, curve=curve) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - sess.run(update_op) - initial_points = points.eval() - - sess.run(update_op) - self.assertAllClose(initial_points, points.eval()) - - def testValueTensorIsIdempotentROC(self): - self._testValueTensorIsIdempotent(curve='ROC') - - def testValueTensorIsIdempotentPR(self): - self._testValueTensorIsIdempotent(curve='PR') - - def _testCase(self, labels, predictions, curve, expected_points): - with self.cached_session() as sess: - predictions_tensor = constant_op.constant( - predictions, dtype=dtypes_lib.float32) - labels_tensor = constant_op.constant(labels, dtype=dtypes_lib.float32) - points, update_op = metric_ops.streaming_curve_points( - labels=labels_tensor, - predictions=predictions_tensor, - num_thresholds=3, - curve=curve) - - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - - self.assertAllClose(expected_points, points.eval()) - - def testEdgeCasesROC(self): - self._testCase([[1]], [[1]], 'ROC', [[0, 1], [0, 1], [0, 0]]) - self._testCase([[0]], [[0]], 'ROC', [[1, 1], [0, 1], [0, 1]]) - self._testCase([[0]], [[1]], 'ROC', [[1, 1], [1, 1], [0, 1]]) - self._testCase([[1]], [[0]], 'ROC', [[0, 1], [0, 0], [0, 0]]) - - def testManyValuesROC(self): - self._testCase([[1.0, 0.0, 0.0, 1.0, 1.0, 1.0]], - [[0.2, 0.3, 0.4, 0.6, 0.7, 0.8]], 'ROC', - [[1.0, 1.0], [0.0, 0.75], [0.0, 0.0]]) - - def testEdgeCasesPR(self): - self._testCase([[1]], [[1]], 'PR', [[1, 1], [1, 1], [0, 1]]) - self._testCase([[0]], [[0]], 'PR', [[1, 0], [1, 1], [1, 1]]) - self._testCase([[0]], [[1]], 'PR', [[1, 0], [1, 0], [1, 1]]) - self._testCase([[1]], [[0]], 'PR', [[1, 1], [0, 1], [0, 1]]) - - def testManyValuesPR(self): - self._testCase([[1.0, 0.0, 0.0, 1.0, 1.0, 1.0]], - [[0.2, 0.3, 0.4, 0.6, 0.7, 0.8]], 'PR', - [[1.0, 4.0 / 6.0], [0.75, 1.0], [0.0, 1.0]]) - - -def _np_auc(predictions, labels, weights=None): - """Computes the AUC explicitly using Numpy. - - Args: - predictions: an ndarray with shape [N]. - labels: an ndarray with shape [N]. - weights: an ndarray with shape [N]. - - Returns: - the area under the ROC curve. - """ - if weights is None: - weights = np.ones(np.size(predictions)) - is_positive = labels > 0 - num_positives = np.sum(weights[is_positive]) - num_negatives = np.sum(weights[~is_positive]) - - # Sort descending: - inds = np.argsort(-predictions) - - sorted_labels = labels[inds] - sorted_weights = weights[inds] - is_positive = sorted_labels > 0 - - tp = np.cumsum(sorted_weights * is_positive) / num_positives - return np.sum((sorted_weights * tp)[~is_positive]) / num_negatives - - -class StreamingAUCTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_auc( - predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) - _assert_metric_variables(self, - ('auc/true_positives:0', 'auc/false_negatives:0', - 'auc/false_positives:0', 'auc/true_negatives:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_auc( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_auc( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) - auc, update_op = metrics.streaming_auc(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_auc = auc.eval() - for _ in range(10): - self.assertAlmostEqual(initial_auc, auc.eval(), 5) - - def testPredictionsOutOfRange(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [1, -1, 1, -1], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - r'predictions must be in \[0, 1\]'): - _, _ = metrics.streaming_auc(predictions, labels) - # Error detected statically; no need to run the op. - - def testAllCorrect(self): - self.allCorrectAsExpected('ROC') - - def allCorrectAsExpected(self, curve): - inputs = np.random.randint(0, 2, size=(100, 1)) - - with self.cached_session() as sess: - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(inputs) - auc, update_op = metrics.streaming_auc(predictions, labels, curve=curve) - - sess.run(variables.local_variables_initializer()) - self.assertEqual(1, sess.run(update_op)) - - self.assertEqual(1, auc.eval()) - - def testSomeCorrect(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) - auc, update_op = metrics.streaming_auc(predictions, labels) - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.5, sess.run(update_op)) - - self.assertAlmostEqual(0.5, auc.eval()) - - def testWeighted1d(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) - weights = constant_op.constant([2], shape=(1, 1)) - auc, update_op = metrics.streaming_auc( - predictions, labels, weights=weights) - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.5, sess.run(update_op), 5) - - self.assertAlmostEqual(0.5, auc.eval(), 5) - - def testWeighted2d(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) - weights = constant_op.constant([1, 2, 3, 4], shape=(1, 4)) - auc, update_op = metrics.streaming_auc( - predictions, labels, weights=weights) - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.7, sess.run(update_op), 5) - - self.assertAlmostEqual(0.7, auc.eval(), 5) - - def testAUCPRSpecialCase(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [0.1, 0.4, 0.35, 0.8], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 0, 1, 1], shape=(1, 4)) - auc, update_op = metrics.streaming_auc(predictions, labels, curve='PR') - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.79166, sess.run(update_op), delta=1e-3) - - self.assertAlmostEqual(0.79166, auc.eval(), delta=1e-3) - - def testAnotherAUCPRSpecialCase(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [0.1, 0.4, 0.35, 0.8, 0.1, 0.135, 0.81], - shape=(1, 7), - dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 0, 1, 0, 1, 0, 1], shape=(1, 7)) - auc, update_op = metrics.streaming_auc(predictions, labels, curve='PR') - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.610317, sess.run(update_op), delta=1e-3) - - self.assertAlmostEqual(0.610317, auc.eval(), delta=1e-3) - - def testThirdAUCPRSpecialCase(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [0.0, 0.1, 0.2, 0.33, 0.3, 0.4, 0.5], - shape=(1, 7), - dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 0, 0, 0, 1, 1, 1], shape=(1, 7)) - auc, update_op = metrics.streaming_auc(predictions, labels, curve='PR') - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.90277, sess.run(update_op), delta=1e-3) - - self.assertAlmostEqual(0.90277, auc.eval(), delta=1e-3) - - def testAllIncorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - with self.cached_session() as sess: - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32) - auc, update_op = metrics.streaming_auc(predictions, labels) - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0, sess.run(update_op)) - - self.assertAlmostEqual(0, auc.eval()) - - def testZeroTruePositivesAndFalseNegativesGivesOneAUC(self): - with self.cached_session() as sess: - predictions = array_ops.zeros([4], dtype=dtypes_lib.float32) - labels = array_ops.zeros([4]) - auc, update_op = metrics.streaming_auc(predictions, labels) - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(1, sess.run(update_op), 6) - - self.assertAlmostEqual(1, auc.eval(), 6) - - def testRecallOneAndPrecisionOneGivesOnePRAUC(self): - with self.cached_session() as sess: - predictions = array_ops.ones([4], dtype=dtypes_lib.float32) - labels = array_ops.ones([4]) - auc, update_op = metrics.streaming_auc(predictions, labels, curve='PR') - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(1, sess.run(update_op), 6) - - self.assertAlmostEqual(1, auc.eval(), 6) - - def testWithMultipleUpdates(self): - num_samples = 1000 - batch_size = 10 - num_batches = int(num_samples / batch_size) - - # Create the labels and data. - labels = np.random.randint(0, 2, size=num_samples) - noise = np.random.normal(0.0, scale=0.2, size=num_samples) - predictions = 0.4 + 0.2 * labels + noise - predictions[predictions > 1] = 1 - predictions[predictions < 0] = 0 - - def _enqueue_as_batches(x, enqueue_ops): - x_batches = x.astype(np.float32).reshape((num_batches, batch_size)) - x_queue = data_flow_ops.FIFOQueue( - num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,)) - for i in range(num_batches): - enqueue_ops[i].append(x_queue.enqueue(x_batches[i, :])) - return x_queue.dequeue() - - for weights in (None, np.ones(num_samples), - np.random.exponential(scale=1.0, size=num_samples)): - expected_auc = _np_auc(predictions, labels, weights) - - with self.cached_session() as sess: - enqueue_ops = [[] for i in range(num_batches)] - tf_predictions = _enqueue_as_batches(predictions, enqueue_ops) - tf_labels = _enqueue_as_batches(labels, enqueue_ops) - tf_weights = ( - _enqueue_as_batches(weights, enqueue_ops) - if weights is not None else None) - - for i in range(num_batches): - sess.run(enqueue_ops[i]) - - auc, update_op = metrics.streaming_auc( - tf_predictions, - tf_labels, - curve='ROC', - num_thresholds=500, - weights=tf_weights) - - sess.run(variables.local_variables_initializer()) - for i in range(num_batches): - sess.run(update_op) - - # Since this is only approximate, we can't expect a 6 digits match. - # Although with higher number of samples/thresholds we should see the - # accuracy improving - self.assertAlmostEqual(expected_auc, auc.eval(), 2) - - -class StreamingDynamicAUCTest(test.TestCase): - - def setUp(self): - super(StreamingDynamicAUCTest, self).setUp() - np.random.seed(1) - ops.reset_default_graph() - - def testUnknownCurve(self): - with self.assertRaisesRegexp( - ValueError, 'curve must be either ROC or PR, TEST_CURVE unknown'): - metrics.streaming_dynamic_auc( - labels=array_ops.ones((10, 1)), - predictions=array_ops.ones((10, 1)), - curve='TEST_CURVE') - - def testVars(self): - metrics.streaming_dynamic_auc( - labels=array_ops.ones((10, 1)), predictions=array_ops.ones((10, 1))) - _assert_metric_variables(self, [ - 'dynamic_auc/concat_labels/array:0', 'dynamic_auc/concat_labels/size:0', - 'dynamic_auc/concat_preds/array:0', 'dynamic_auc/concat_preds/size:0' - ]) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - auc, _ = metrics.streaming_dynamic_auc( - labels=array_ops.ones((10, 1)), - predictions=array_ops.ones((10, 1)), - metrics_collections=[my_collection_name]) - self.assertEqual(ops.get_collection(my_collection_name), [auc]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_dynamic_auc( - labels=array_ops.ones((10, 1)), - predictions=array_ops.ones((10, 1)), - updates_collections=[my_collection_name]) - self.assertEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) - auc, update_op = metrics.streaming_dynamic_auc(labels, predictions) - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - # Run several updates. - for _ in xrange(10): - sess.run(update_op) - # Then verify idempotency. - initial_auc = auc.eval() - for _ in xrange(10): - self.assertAlmostEqual(initial_auc, auc.eval(), 5) - - def testAllLabelsOnes(self): - with self.cached_session() as sess: - predictions = constant_op.constant([1., 1., 1.]) - labels = constant_op.constant([1, 1, 1]) - auc, update_op = metrics.streaming_dynamic_auc(labels, predictions) - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertEqual(0, auc.eval()) - - def testAllLabelsZeros(self): - with self.cached_session() as sess: - predictions = constant_op.constant([1., 1., 1.]) - labels = constant_op.constant([0, 0, 0]) - auc, update_op = metrics.streaming_dynamic_auc(labels, predictions) - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertEqual(0, auc.eval()) - - def testNonZeroOnePredictions(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [2.5, -2.5, 2.5, -2.5], dtype=dtypes_lib.float32) - labels = constant_op.constant([1, 0, 1, 0]) - auc, update_op = metrics.streaming_dynamic_auc(labels, predictions) - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertAlmostEqual(auc.eval(), 1.0) - - def testAllCorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - with self.cached_session() as sess: - predictions = constant_op.constant(inputs) - labels = constant_op.constant(inputs) - auc, update_op = metrics.streaming_dynamic_auc(labels, predictions) - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertEqual(1, auc.eval()) - - def testSomeCorrect(self): - with self.cached_session() as sess: - predictions = constant_op.constant([1, 0, 1, 0]) - labels = constant_op.constant([0, 1, 1, 0]) - auc, update_op = metrics.streaming_dynamic_auc(labels, predictions) - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertAlmostEqual(0.5, auc.eval()) - - def testAllIncorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - with self.cached_session() as sess: - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32) - auc, update_op = metrics.streaming_dynamic_auc(labels, predictions) - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertAlmostEqual(0, auc.eval()) - - def testExceptionOnIncompatibleShapes(self): - with self.cached_session() as sess: - predictions = array_ops.ones([5]) - labels = array_ops.zeros([6]) - with self.assertRaisesRegexp(ValueError, 'Shapes .* are incompatible'): - _, update_op = metrics.streaming_dynamic_auc(labels, predictions) - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - - def testExceptionOnGreaterThanOneLabel(self): - with self.cached_session() as sess: - predictions = constant_op.constant([1, 0.5, 0], dtypes_lib.float32) - labels = constant_op.constant([2, 1, 0]) - _, update_op = metrics.streaming_dynamic_auc(labels, predictions) - sess.run(variables.local_variables_initializer()) - with self.assertRaisesRegexp( - errors_impl.InvalidArgumentError, - '.*labels must be 0 or 1, at least one is >1.*'): - sess.run(update_op) - - def testExceptionOnNegativeLabel(self): - with self.cached_session() as sess: - predictions = constant_op.constant([1, 0.5, 0], dtypes_lib.float32) - labels = constant_op.constant([1, 0, -1]) - _, update_op = metrics.streaming_dynamic_auc(labels, predictions) - sess.run(variables.local_variables_initializer()) - with self.assertRaisesRegexp( - errors_impl.InvalidArgumentError, - '.*labels must be 0 or 1, at least one is <0.*'): - sess.run(update_op) - - def testWithMultipleUpdates(self): - batch_size = 10 - num_batches = 100 - labels = np.array([]) - predictions = np.array([]) - tf_labels = variables.VariableV1( - array_ops.ones(batch_size, dtypes_lib.int32), - collections=[ops.GraphKeys.LOCAL_VARIABLES], - dtype=dtypes_lib.int32) - tf_predictions = variables.VariableV1( - array_ops.ones(batch_size), - collections=[ops.GraphKeys.LOCAL_VARIABLES], - dtype=dtypes_lib.float32) - auc, update_op = metrics.streaming_dynamic_auc(tf_labels, tf_predictions) - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - for _ in xrange(num_batches): - new_labels = np.random.randint(0, 2, size=batch_size) - noise = np.random.normal(0.0, scale=0.2, size=batch_size) - new_predictions = 0.4 + 0.2 * new_labels + noise - labels = np.concatenate([labels, new_labels]) - predictions = np.concatenate([predictions, new_predictions]) - sess.run(tf_labels.assign(new_labels)) - sess.run(tf_predictions.assign(new_predictions)) - sess.run(update_op) - expected_auc = _np_auc(predictions, labels) - self.assertAlmostEqual(expected_auc, auc.eval()) - - def testAUCPRReverseIncreasingPredictions(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [0.1, 0.4, 0.35, 0.8], dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 0, 1, 1]) - auc, update_op = metrics.streaming_dynamic_auc( - labels, predictions, curve='PR') - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertAlmostEqual(0.79166, auc.eval(), delta=1e-5) - - def testAUCPRJumbledPredictions(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [0.1, 0.4, 0.35, 0.8, 0.1, 0.135, 0.81], dtypes_lib.float32) - labels = constant_op.constant([0, 0, 1, 0, 1, 0, 1]) - auc, update_op = metrics.streaming_dynamic_auc( - labels, predictions, curve='PR') - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertAlmostEqual(0.610317, auc.eval(), delta=1e-6) - - def testAUCPRPredictionsLessThanHalf(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [0.0, 0.1, 0.2, 0.33, 0.3, 0.4, 0.5], - shape=(1, 7), - dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 0, 0, 0, 1, 1, 1], shape=(1, 7)) - auc, update_op = metrics.streaming_dynamic_auc( - labels, predictions, curve='PR') - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertAlmostEqual(0.90277, auc.eval(), delta=1e-5) - - def testWithWeights(self): - batch_size = 10 - num_batches = 100 - labels = np.array([]) - predictions = np.array([]) - weights = np.array([]) - tf_labels = variables.VariableV1( - array_ops.ones(batch_size, dtypes_lib.int32), - collections=[ops.GraphKeys.LOCAL_VARIABLES], - dtype=dtypes_lib.int32) - tf_predictions = variables.VariableV1( - array_ops.ones(batch_size), - collections=[ops.GraphKeys.LOCAL_VARIABLES], - dtype=dtypes_lib.float32) - tf_weights = variables.VariableV1( - array_ops.ones(batch_size), - collections=[ops.GraphKeys.LOCAL_VARIABLES], - dtype=dtypes_lib.float32) - auc, update_op = metrics.streaming_dynamic_auc(tf_labels, - tf_predictions, - weights=tf_weights) - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - for _ in xrange(num_batches): - new_labels = np.random.randint(0, 2, size=batch_size) - noise = np.random.uniform(-0.2, 0.2, size=batch_size) - new_predictions = 0.4 + 0.2 * new_labels + noise - new_weights = np.random.uniform(0.0, 3.0, size=batch_size) - labels = np.concatenate([labels, new_labels]) - predictions = np.concatenate([predictions, new_predictions]) - weights = np.concatenate([weights, new_weights]) - sess.run([tf_labels.assign(new_labels), - tf_predictions.assign(new_predictions), - tf_weights.assign(new_weights)]) - sess.run(update_op) - expected_auc = _np_auc(predictions, labels, weights) - self.assertAlmostEqual(expected_auc, auc.eval()) - - -class AucWithConfidenceIntervalsTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def _testResultsEqual(self, expected_dict, gotten_result): - """Tests that 2 results (dicts) represent the same data. - - Args: - expected_dict: A dictionary with keys that are the names of properties - of PrecisionRecallData and whose values are lists of floats. - gotten_result: A AucWithConfidenceIntervalData object. - """ - gotten_dict = {k: t.eval() for k, t in gotten_result._asdict().items()} - self.assertItemsEqual( - list(expected_dict.keys()), list(gotten_dict.keys())) - - for key, expected_values in expected_dict.items(): - self.assertAllClose(expected_values, gotten_dict[key]) - - def _testCase(self, predictions, labels, expected_result, weights=None): - """Performs a test given a certain scenario of labels, predictions, weights. - - Args: - predictions: The predictions tensor. Of type float32. - labels: The labels tensor. Of type bool. - expected_result: The expected result (dict) that maps to tensors. - weights: Optional weights tensor. - """ - with self.cached_session() as sess: - predictions_tensor = constant_op.constant( - predictions, dtype=dtypes_lib.float32) - labels_tensor = constant_op.constant(labels, dtype=dtypes_lib.int64) - weights_tensor = None - if weights: - weights_tensor = constant_op.constant(weights, dtype=dtypes_lib.float32) - gotten_result, update_op = ( - metric_ops.auc_with_confidence_intervals( - labels=labels_tensor, - predictions=predictions_tensor, - weights=weights_tensor)) - - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - - self._testResultsEqual(expected_result, gotten_result) - - def testAucAllCorrect(self): - self._testCase( - predictions=[0., 0.2, 0.3, 0.3, 0.4, 0.5, 0.6, 0.6, 0.8, 1.0], - labels=[0, 0, 1, 0, 0, 1, 0, 1, 1, 0], - expected_result={ - 'auc': 0.66666667, - 'lower': 0.27826795, - 'upper': 0.91208512, - }) - - def testAucUnorderedInput(self): - self._testCase( - predictions=[1.0, 0.6, 0., 0.3, 0.4, 0.2, 0.5, 0.3, 0.6, 0.8], - labels=[0, 1, 0, 1, 0, 0, 1, 0, 0, 1], - expected_result={ - 'auc': 0.66666667, - 'lower': 0.27826795, - 'upper': 0.91208512, - }) - - def testAucWithWeights(self): - self._testCase( - predictions=[0., 0.2, 0.3, 0.3, 0.4, 0.5, 0.6, 0.6, 0.8, 1.0], - labels=[0, 0, 1, 0, 0, 1, 0, 1, 1, 0], - weights=[0.5, 0.6, 1.2, 1.5, 2.0, 2.0, 1.5, 1.2, 0.6, 0.5], - expected_result={ - 'auc': 0.65151515, - 'lower': 0.28918604, - 'upper': 0.89573906, - }) - - def testAucEqualOne(self): - self._testCase( - predictions=[0, 0.2, 0.3, 0.3, 0.4, 0.5, 0.6, 0.6, 0.8, 1.0], - labels=[0, 0, 0, 0, 0, 1, 1, 1, 1, 1], - expected_result={ - 'auc': 1.0, - 'lower': 1.0, - 'upper': 1.0, - }) - - def testAucEqualZero(self): - self._testCase( - predictions=[0, 0.2, 0.3, 0.3, 0.4, 0.5, 0.6, 0.6, 0.8, 1.0], - labels=[1, 1, 1, 1, 1, 0, 0, 0, 0, 0], - expected_result={ - 'auc': 0.0, - 'lower': 0.0, - 'upper': 0.0, - }) - - def testNonZeroOnePredictions(self): - self._testCase( - predictions=[2.5, -2.5, .5, -.5, 1], - labels=[1, 0, 1, 0, 0], - expected_result={ - 'auc': 0.83333333, - 'lower': 0.15229267, - 'upper': 0.99286517, - }) - - def testAllLabelsOnes(self): - self._testCase( - predictions=[1., 1., 1., 1., 1.], - labels=[1, 1, 1, 1, 1], - expected_result={ - 'auc': 0., - 'lower': 0., - 'upper': 0., - }) - - def testAllLabelsZeros(self): - self._testCase( - predictions=[0., 0., 0., 0., 0.], - labels=[0, 0, 0, 0, 0], - expected_result={ - 'auc': 0., - 'lower': 0., - 'upper': 0., - }) - - def testWeightSumLessThanOneAll(self): - self._testCase( - predictions=[1., 1., 0., 1., 0., 0.], - labels=[1, 1, 1, 0, 0, 0], - weights=[0.1, 0.1, 0.1, 0.1, 0.1, 0.1], - expected_result={ - 'auc': 0., - 'lower': 0., - 'upper': 0., - }) - - def testWithMultipleUpdates(self): - batch_size = 50 - num_batches = 100 - labels = np.array([]) - predictions = np.array([]) - tf_labels = variables.VariableV1( - array_ops.ones(batch_size, dtypes_lib.int32), - collections=[ops.GraphKeys.LOCAL_VARIABLES], - dtype=dtypes_lib.int32) - tf_predictions = variables.VariableV1( - array_ops.ones(batch_size), - collections=[ops.GraphKeys.LOCAL_VARIABLES], - dtype=dtypes_lib.float32) - auc, update_op = metrics.auc_with_confidence_intervals(tf_labels, - tf_predictions) - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - for _ in xrange(num_batches): - new_labels = np.random.randint(0, 2, size=batch_size) - noise = np.random.normal(0.0, scale=0.2, size=batch_size) - new_predictions = 0.4 + 0.2 * new_labels + noise - labels = np.concatenate([labels, new_labels]) - predictions = np.concatenate([predictions, new_predictions]) - sess.run(tf_labels.assign(new_labels)) - sess.run(tf_predictions.assign(new_predictions)) - sess.run(update_op) - expected_auc = _np_auc(predictions, labels) - self.assertAllClose(expected_auc, auc.auc.eval()) - - def testExceptionOnFloatLabels(self): - with self.cached_session() as sess: - predictions = constant_op.constant([1, 0.5, 0, 1, 0], dtypes_lib.float32) - labels = constant_op.constant([0.7, 0, 1, 0, 1]) - _, update_op = metrics.auc_with_confidence_intervals(labels, predictions) - sess.run(variables.local_variables_initializer()) - self.assertRaises(TypeError, sess.run(update_op)) - - def testExceptionOnGreaterThanOneLabel(self): - with self.cached_session() as sess: - predictions = constant_op.constant([1, 0.5, 0, 1, 0], dtypes_lib.float32) - labels = constant_op.constant([2, 1, 0, 1, 0]) - _, update_op = metrics.auc_with_confidence_intervals(labels, predictions) - sess.run(variables.local_variables_initializer()) - with self.assertRaisesRegexp( - errors_impl.InvalidArgumentError, - '.*labels must be 0 or 1, at least one is >1.*'): - sess.run(update_op) - - def testExceptionOnNegativeLabel(self): - with self.cached_session() as sess: - predictions = constant_op.constant([1, 0.5, 0, 1, 0], dtypes_lib.float32) - labels = constant_op.constant([1, 0, -1, 1, 0]) - _, update_op = metrics.auc_with_confidence_intervals(labels, predictions) - sess.run(variables.local_variables_initializer()) - with self.assertRaisesRegexp( - errors_impl.InvalidArgumentError, - '.*labels must be 0 or 1, at least one is <0.*'): - sess.run(update_op) - - -class StreamingPrecisionRecallAtEqualThresholdsTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def _testResultsEqual(self, expected_dict, gotten_result, eps=None): - """Tests that 2 results (dicts) represent the same data. - - Args: - expected_dict: A dictionary with keys that are the names of properties - of PrecisionRecallData and whose values are lists of floats. - gotten_result: A PrecisionRecallData object. - eps: Epsilon value to use for testing output values. If unspecified, use - default from assertAllClose. - """ - gotten_dict = {k: t.eval() for k, t in gotten_result._asdict().items()} - self.assertItemsEqual(list(expected_dict.keys()), list(gotten_dict.keys())) - - for key, expected_values in expected_dict.items(): - if eps is not None: - self.assertAllClose(expected_values, gotten_dict[key], atol=eps) - else: - self.assertAllClose(expected_values, gotten_dict[key]) - - def testVars(self): - metric_ops.precision_recall_at_equal_thresholds( - labels=constant_op.constant([True], dtype=dtypes_lib.bool), - predictions=constant_op.constant([0.42], dtype=dtypes_lib.float32)) - _assert_metric_variables( - self, ('precision_recall_at_equal_thresholds/variables/tp_buckets:0', - 'precision_recall_at_equal_thresholds/variables/fp_buckets:0')) - - def testVarsWithName(self): - metric_ops.precision_recall_at_equal_thresholds( - labels=constant_op.constant([True], dtype=dtypes_lib.bool), - predictions=constant_op.constant([0.42], dtype=dtypes_lib.float32), - name='foo') - _assert_metric_variables( - self, ('foo/variables/tp_buckets:0', 'foo/variables/fp_buckets:0')) - - def testValuesAreIdempotent(self): - predictions = constant_op.constant( - np.random.uniform(size=(10, 3)), dtype=dtypes_lib.float32) - labels = constant_op.constant( - np.random.uniform(size=(10, 3)) > 0.5, dtype=dtypes_lib.bool) - - result, update_op = metric_ops.precision_recall_at_equal_thresholds( - labels=labels, predictions=predictions) - - with self.cached_session() as sess: - # Run several updates. - sess.run(variables.local_variables_initializer()) - for _ in range(3): - sess.run(update_op) - - # Then verify idempotency. - initial_result = { - k: value.eval().tolist() - for k, value in result._asdict().items() - } - for _ in range(3): - self._testResultsEqual(initial_result, result) - - def _testCase(self, - predictions, - labels, - expected_result, - dtype=dtypes_lib.float32, - eps=None, - weights=None): - """Performs a test given a certain scenario of labels, predictions, weights. - - Args: - predictions: The predictions tensor. Of type dtype. - labels: The labels tensor. Of type bool. - expected_result: The expected result (dict) that maps to tensors. - dtype: Data type to use for predictions and weights tensor. Default - is float32. - eps: Epsilon value to use for testing output values. If unspecified, use - default from assertAllClose. - weights: Optional weights tensor. - """ - with self.cached_session() as sess: - predictions_tensor = constant_op.constant(predictions, dtype=dtype) - labels_tensor = constant_op.constant(labels, dtype=dtypes_lib.bool) - weights_tensor = None - if weights: - weights_tensor = constant_op.constant(weights, dtype=dtype) - gotten_result, update_op = ( - metric_ops.precision_recall_at_equal_thresholds( - labels=labels_tensor, - predictions=predictions_tensor, - weights=weights_tensor, - num_thresholds=3)) - self.assertEqual(gotten_result.tp.dtype, dtype) - self.assertEqual(gotten_result.fp.dtype, dtype) - self.assertEqual(gotten_result.tn.dtype, dtype) - self.assertEqual(gotten_result.fn.dtype, dtype) - self.assertEqual(gotten_result.precision.dtype, dtype) - self.assertEqual(gotten_result.recall.dtype, dtype) - self.assertEqual(gotten_result.thresholds.dtype, dtype) - - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - - self._testResultsEqual(expected_result, gotten_result, eps=eps) - - def testAllTruePositives(self): - self._testCase( - [[1]], [[True]], { - 'tp': [1, 1, 1], - 'fp': [0, 0, 0], - 'tn': [0, 0, 0], - 'fn': [0, 0, 0], - 'precision': [1.0, 1.0, 1.0], - 'recall': [1.0, 1.0, 1.0], - 'thresholds': [0.0, 0.5, 1.0], - }) - - def testAllTrueNegatives(self): - self._testCase( - [[0]], [[False]], { - 'tp': [0, 0, 0], - 'fp': [1, 0, 0], - 'tn': [0, 1, 1], - 'fn': [0, 0, 0], - 'precision': [0.0, 0.0, 0.0], - 'recall': [0.0, 0.0, 0.0], - 'thresholds': [0.0, 0.5, 1.0], - }) - - def testAllFalsePositives(self): - self._testCase( - [[1]], [[False]], { - 'tp': [0, 0, 0], - 'fp': [1, 1, 1], - 'tn': [0, 0, 0], - 'fn': [0, 0, 0], - 'precision': [0.0, 0.0, 0.0], - 'recall': [0.0, 0.0, 0.0], - 'thresholds': [0.0, 0.5, 1.0], - }) - - def testAllFalseNegatives(self): - self._testCase( - [[0]], [[True]], { - 'tp': [1, 0, 0], - 'fp': [0, 0, 0], - 'tn': [0, 0, 0], - 'fn': [0, 1, 1], - 'precision': [1.0, 0.0, 0.0], - 'recall': [1.0, 0.0, 0.0], - 'thresholds': [0.0, 0.5, 1.0], - }) - - def testManyValues(self): - self._testCase( - [[0.2, 0.3, 0.4, 0.6, 0.7, 0.8]], - [[True, False, False, True, True, True]], { - 'tp': [4, 3, 0], - 'fp': [2, 0, 0], - 'tn': [0, 2, 2], - 'fn': [0, 1, 4], - 'precision': [2.0 / 3.0, 1.0, 0.0], - 'recall': [1.0, 0.75, 0.0], - 'thresholds': [0.0, 0.5, 1.0], - }) - - def testManyValuesWithWeights(self): - self._testCase( - [[0.2, 0.3, 0.4, 0.6, 0.7, 0.8]], - [[True, False, False, True, True, True]], { - 'tp': [1.5, 1.5, 0.0], - 'fp': [2.5, 0.0, 0.0], - 'tn': [0.0, 2.5, 2.5], - 'fn': [0.0, 0.0, 1.5], - 'precision': [0.375, 1.0, 0.0], - 'recall': [1.0, 1.0, 0.0], - 'thresholds': [0.0, 0.5, 1.0], - }, - weights=[[0.0, 0.5, 2.0, 0.0, 0.5, 1.0]]) - - def testFloat64(self): - self._testCase( - [[0.2, 0.3, 0.4, 0.6, 0.7, 0.8]], - [[True, False, False, True, True, True]], { - 'tp': [4, 3, 0], - 'fp': [2, 0, 0], - 'tn': [0, 2, 2], - 'fn': [0, 1, 4], - 'precision': [2.0 / 3.0, 1.0, 0.0], - 'recall': [1.0, 0.75, 0.0], - 'thresholds': [0.0, 0.5, 1.0], - }, - dtype=dtypes_lib.float64) - - def testFloat16(self): - self._testCase( - [[0.2, 0.3, 0.4, 0.6, 0.7, 0.8]], - [[True, False, False, True, True, True]], { - 'tp': [4, 3, 0], - 'fp': [2, 0, 0], - 'tn': [0, 2, 2], - 'fn': [0, 1, 4], - 'precision': [2.0 / 3.0, 1.0, 0.0], - 'recall': [1.0, 0.75, 0.0], - 'thresholds': [0.0, 0.5, 1.0], - }, - dtype=dtypes_lib.float16, - eps=1e-3) - - -class StreamingSpecificityAtSensitivityTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_specificity_at_sensitivity( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - sensitivity=0.7) - _assert_metric_variables(self, - ('specificity_at_sensitivity/true_positives:0', - 'specificity_at_sensitivity/false_negatives:0', - 'specificity_at_sensitivity/false_positives:0', - 'specificity_at_sensitivity/true_negatives:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_specificity_at_sensitivity( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - sensitivity=0.7, - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_specificity_at_sensitivity( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - sensitivity=0.7, - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) - specificity, update_op = metrics.streaming_specificity_at_sensitivity( - predictions, labels, sensitivity=0.7) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_specificity = specificity.eval() - for _ in range(10): - self.assertAlmostEqual(initial_specificity, specificity.eval(), 5) - - def testAllCorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(inputs) - specificity, update_op = metrics.streaming_specificity_at_sensitivity( - predictions, labels, sensitivity=0.7) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(1, sess.run(update_op)) - self.assertEqual(1, specificity.eval()) - - def testSomeCorrectHighSensitivity(self): - predictions_values = [0.1, 0.2, 0.4, 0.3, 0.0, 0.1, 0.45, 0.5, 0.8, 0.9] - labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] - - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels_values) - specificity, update_op = metrics.streaming_specificity_at_sensitivity( - predictions, labels, sensitivity=0.8) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(1.0, sess.run(update_op)) - self.assertAlmostEqual(1.0, specificity.eval()) - - def testSomeCorrectLowSensitivity(self): - predictions_values = [0.1, 0.2, 0.4, 0.3, 0.0, 0.1, 0.2, 0.2, 0.26, 0.26] - labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] - - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels_values) - specificity, update_op = metrics.streaming_specificity_at_sensitivity( - predictions, labels, sensitivity=0.4) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - self.assertAlmostEqual(0.6, sess.run(update_op)) - self.assertAlmostEqual(0.6, specificity.eval()) - - def testWeighted1d(self): - predictions_values = [0.1, 0.2, 0.4, 0.3, 0.0, 0.1, 0.2, 0.2, 0.26, 0.26] - labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] - weights_values = [3] - - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels_values) - weights = constant_op.constant(weights_values) - specificity, update_op = metrics.streaming_specificity_at_sensitivity( - predictions, labels, weights=weights, sensitivity=0.4) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - self.assertAlmostEqual(0.6, sess.run(update_op)) - self.assertAlmostEqual(0.6, specificity.eval()) - - def testWeighted2d(self): - predictions_values = [0.1, 0.2, 0.4, 0.3, 0.0, 0.1, 0.2, 0.2, 0.26, 0.26] - labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] - weights_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels_values) - weights = constant_op.constant(weights_values) - specificity, update_op = metrics.streaming_specificity_at_sensitivity( - predictions, labels, weights=weights, sensitivity=0.4) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - self.assertAlmostEqual(8.0 / 15.0, sess.run(update_op)) - self.assertAlmostEqual(8.0 / 15.0, specificity.eval()) - - -class StreamingSensitivityAtSpecificityTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_sensitivity_at_specificity( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - specificity=0.7) - _assert_metric_variables(self, - ('sensitivity_at_specificity/true_positives:0', - 'sensitivity_at_specificity/false_negatives:0', - 'sensitivity_at_specificity/false_positives:0', - 'sensitivity_at_specificity/true_negatives:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_sensitivity_at_specificity( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - specificity=0.7, - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_sensitivity_at_specificity( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - specificity=0.7, - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) - sensitivity, update_op = metrics.streaming_sensitivity_at_specificity( - predictions, labels, specificity=0.7) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_sensitivity = sensitivity.eval() - for _ in range(10): - self.assertAlmostEqual(initial_sensitivity, sensitivity.eval(), 5) - - def testAllCorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(inputs) - specificity, update_op = metrics.streaming_sensitivity_at_specificity( - predictions, labels, specificity=0.7) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(1, sess.run(update_op)) - self.assertEqual(1, specificity.eval()) - - def testSomeCorrectHighSpecificity(self): - predictions_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9] - labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] - - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels_values) - specificity, update_op = metrics.streaming_sensitivity_at_specificity( - predictions, labels, specificity=0.8) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.8, sess.run(update_op)) - self.assertAlmostEqual(0.8, specificity.eval()) - - def testSomeCorrectLowSpecificity(self): - predictions_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26] - labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] - - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels_values) - specificity, update_op = metrics.streaming_sensitivity_at_specificity( - predictions, labels, specificity=0.4) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.6, sess.run(update_op)) - self.assertAlmostEqual(0.6, specificity.eval()) - - def testWeighted(self): - predictions_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26] - labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] - weights_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels_values) - weights = constant_op.constant(weights_values) - specificity, update_op = metrics.streaming_sensitivity_at_specificity( - predictions, labels, weights=weights, specificity=0.4) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.675, sess.run(update_op)) - self.assertAlmostEqual(0.675, specificity.eval()) - - -# TODO(nsilberman): Break this up into two sets of tests. -class StreamingPrecisionRecallThresholdsTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_precision_at_thresholds( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - thresholds=[0, 0.5, 1.0]) - _assert_metric_variables(self, ( - 'precision_at_thresholds/true_positives:0', - 'precision_at_thresholds/false_positives:0', - )) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - prec, _ = metrics.streaming_precision_at_thresholds( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - thresholds=[0, 0.5, 1.0], - metrics_collections=[my_collection_name]) - rec, _ = metrics.streaming_recall_at_thresholds( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - thresholds=[0, 0.5, 1.0], - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [prec, rec]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, precision_op = metrics.streaming_precision_at_thresholds( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - thresholds=[0, 0.5, 1.0], - updates_collections=[my_collection_name]) - _, recall_op = metrics.streaming_recall_at_thresholds( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - thresholds=[0, 0.5, 1.0], - updates_collections=[my_collection_name]) - self.assertListEqual( - ops.get_collection(my_collection_name), [precision_op, recall_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) - thresholds = [0, 0.5, 1.0] - prec, prec_op = metrics.streaming_precision_at_thresholds( - predictions, labels, thresholds) - rec, rec_op = metrics.streaming_recall_at_thresholds( - predictions, labels, thresholds) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run([prec_op, rec_op]) - - # Then verify idempotency. - initial_prec = prec.eval() - initial_rec = rec.eval() - for _ in range(10): - self.assertAllClose(initial_prec, prec.eval()) - self.assertAllClose(initial_rec, rec.eval()) - - # TODO(nsilberman): fix tests (passing but incorrect). - def testAllCorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - with self.cached_session() as sess: - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(inputs) - thresholds = [0.5] - prec, prec_op = metrics.streaming_precision_at_thresholds( - predictions, labels, thresholds) - rec, rec_op = metrics.streaming_recall_at_thresholds( - predictions, labels, thresholds) - - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) - - self.assertEqual(1, prec.eval()) - self.assertEqual(1, rec.eval()) - - def testSomeCorrect(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) - thresholds = [0.5] - prec, prec_op = metrics.streaming_precision_at_thresholds( - predictions, labels, thresholds) - rec, rec_op = metrics.streaming_recall_at_thresholds( - predictions, labels, thresholds) - - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) - - self.assertAlmostEqual(0.5, prec.eval()) - self.assertAlmostEqual(0.5, rec.eval()) - - def testAllIncorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - with self.cached_session() as sess: - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32) - thresholds = [0.5] - prec, prec_op = metrics.streaming_precision_at_thresholds( - predictions, labels, thresholds) - rec, rec_op = metrics.streaming_recall_at_thresholds( - predictions, labels, thresholds) - - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) - - self.assertAlmostEqual(0, prec.eval()) - self.assertAlmostEqual(0, rec.eval()) - - def testWeights1d(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32) - labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) - weights = constant_op.constant( - [[0], [1]], shape=(2, 1), dtype=dtypes_lib.float32) - thresholds = [0.5, 1.1] - prec, prec_op = metrics.streaming_precision_at_thresholds( - predictions, labels, thresholds, weights=weights) - rec, rec_op = metrics.streaming_recall_at_thresholds( - predictions, labels, thresholds, weights=weights) - - prec_low = prec[0] - prec_high = prec[1] - rec_low = rec[0] - rec_high = rec[1] - - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) - - self.assertAlmostEqual(1.0, prec_low.eval(), places=5) - self.assertAlmostEqual(0.0, prec_high.eval(), places=5) - self.assertAlmostEqual(1.0, rec_low.eval(), places=5) - self.assertAlmostEqual(0.0, rec_high.eval(), places=5) - - def testWeights2d(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32) - labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) - weights = constant_op.constant( - [[0, 0], [1, 1]], shape=(2, 2), dtype=dtypes_lib.float32) - thresholds = [0.5, 1.1] - prec, prec_op = metrics.streaming_precision_at_thresholds( - predictions, labels, thresholds, weights=weights) - rec, rec_op = metrics.streaming_recall_at_thresholds( - predictions, labels, thresholds, weights=weights) - - prec_low = prec[0] - prec_high = prec[1] - rec_low = rec[0] - rec_high = rec[1] - - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) - - self.assertAlmostEqual(1.0, prec_low.eval(), places=5) - self.assertAlmostEqual(0.0, prec_high.eval(), places=5) - self.assertAlmostEqual(1.0, rec_low.eval(), places=5) - self.assertAlmostEqual(0.0, rec_high.eval(), places=5) - - def testExtremeThresholds(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 1, 1, 1], shape=(1, 4)) - thresholds = [-1.0, 2.0] # lower/higher than any values - prec, prec_op = metrics.streaming_precision_at_thresholds( - predictions, labels, thresholds) - rec, rec_op = metrics.streaming_recall_at_thresholds( - predictions, labels, thresholds) - - prec_low = prec[0] - prec_high = prec[1] - rec_low = rec[0] - rec_high = rec[1] - - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) - - self.assertAlmostEqual(0.75, prec_low.eval()) - self.assertAlmostEqual(0.0, prec_high.eval()) - self.assertAlmostEqual(1.0, rec_low.eval()) - self.assertAlmostEqual(0.0, rec_high.eval()) - - def testZeroLabelsPredictions(self): - with self.cached_session() as sess: - predictions = array_ops.zeros([4], dtype=dtypes_lib.float32) - labels = array_ops.zeros([4]) - thresholds = [0.5] - prec, prec_op = metrics.streaming_precision_at_thresholds( - predictions, labels, thresholds) - rec, rec_op = metrics.streaming_recall_at_thresholds( - predictions, labels, thresholds) - - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) - - self.assertAlmostEqual(0, prec.eval(), 6) - self.assertAlmostEqual(0, rec.eval(), 6) - - def testWithMultipleUpdates(self): - num_samples = 1000 - batch_size = 10 - num_batches = int(num_samples / batch_size) - - # Create the labels and data. - labels = np.random.randint(0, 2, size=(num_samples, 1)) - noise = np.random.normal(0.0, scale=0.2, size=(num_samples, 1)) - predictions = 0.4 + 0.2 * labels + noise - predictions[predictions > 1] = 1 - predictions[predictions < 0] = 0 - thresholds = [0.3] - - tp = 0 - fp = 0 - fn = 0 - tn = 0 - for i in range(num_samples): - if predictions[i] > thresholds[0]: - if labels[i] == 1: - tp += 1 - else: - fp += 1 - else: - if labels[i] == 1: - fn += 1 - else: - tn += 1 - epsilon = 1e-7 - expected_prec = tp / (epsilon + tp + fp) - expected_rec = tp / (epsilon + tp + fn) - - labels = labels.astype(np.float32) - predictions = predictions.astype(np.float32) - - with self.cached_session() as sess: - # Reshape the data so its easy to queue up: - predictions_batches = predictions.reshape((batch_size, num_batches)) - labels_batches = labels.reshape((batch_size, num_batches)) - - # Enqueue the data: - predictions_queue = data_flow_ops.FIFOQueue( - num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,)) - labels_queue = data_flow_ops.FIFOQueue( - num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,)) - - for i in range(int(num_batches)): - tf_prediction = constant_op.constant(predictions_batches[:, i]) - tf_label = constant_op.constant(labels_batches[:, i]) - sess.run([ - predictions_queue.enqueue(tf_prediction), - labels_queue.enqueue(tf_label) - ]) - - tf_predictions = predictions_queue.dequeue() - tf_labels = labels_queue.dequeue() - - prec, prec_op = metrics.streaming_precision_at_thresholds( - tf_predictions, tf_labels, thresholds) - rec, rec_op = metrics.streaming_recall_at_thresholds( - tf_predictions, tf_labels, thresholds) - - sess.run(variables.local_variables_initializer()) - for _ in range(int(num_samples / batch_size)): - sess.run([prec_op, rec_op]) - # Since this is only approximate, we can't expect a 6 digits match. - # Although with higher number of samples/thresholds we should see the - # accuracy improving - self.assertAlmostEqual(expected_prec, prec.eval(), 2) - self.assertAlmostEqual(expected_rec, rec.eval(), 2) - - -class StreamingFPRThresholdsTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_false_positive_rate_at_thresholds( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - thresholds=[0, 0.5, 1.0]) - _assert_metric_variables(self, ( - 'false_positive_rate_at_thresholds/false_positives:0', - 'false_positive_rate_at_thresholds/true_negatives:0', - )) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - fpr, _ = metrics.streaming_false_positive_rate_at_thresholds( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - thresholds=[0, 0.5, 1.0], - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [fpr]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_false_positive_rate_at_thresholds( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - thresholds=[0, 0.5, 1.0], - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) - thresholds = [0, 0.5, 1.0] - fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds( - predictions, labels, thresholds) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(fpr_op) - - # Then verify idempotency. - initial_fpr = fpr.eval() - for _ in range(10): - self.assertAllClose(initial_fpr, fpr.eval()) - - def testAllCorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - with self.cached_session() as sess: - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(inputs) - thresholds = [0.5] - fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds( - predictions, labels, thresholds) - - sess.run(variables.local_variables_initializer()) - sess.run(fpr_op) - - self.assertEqual(0, fpr.eval()) - - def testSomeCorrect(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) - thresholds = [0.5] - fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds( - predictions, labels, thresholds) - - sess.run(variables.local_variables_initializer()) - sess.run(fpr_op) - - self.assertAlmostEqual(0.5, fpr.eval()) - - def testAllIncorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - with self.cached_session() as sess: - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32) - thresholds = [0.5] - fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds( - predictions, labels, thresholds) - - sess.run(variables.local_variables_initializer()) - sess.run(fpr_op) - - self.assertAlmostEqual(1, fpr.eval()) - - def testWeights1d(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32) - labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) - weights = constant_op.constant( - [[0], [1]], shape=(2, 1), dtype=dtypes_lib.float32) - thresholds = [0.5, 1.1] - fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds( - predictions, labels, thresholds, weights=weights) - - fpr_low = fpr[0] - fpr_high = fpr[1] - - sess.run(variables.local_variables_initializer()) - sess.run(fpr_op) - - self.assertAlmostEqual(0.0, fpr_low.eval(), places=5) - self.assertAlmostEqual(0.0, fpr_high.eval(), places=5) - - def testWeights2d(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32) - labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) - weights = constant_op.constant( - [[0, 0], [1, 1]], shape=(2, 2), dtype=dtypes_lib.float32) - thresholds = [0.5, 1.1] - fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds( - predictions, labels, thresholds, weights=weights) - - fpr_low = fpr[0] - fpr_high = fpr[1] - - sess.run(variables.local_variables_initializer()) - sess.run(fpr_op) - - self.assertAlmostEqual(0.0, fpr_low.eval(), places=5) - self.assertAlmostEqual(0.0, fpr_high.eval(), places=5) - - def testExtremeThresholds(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 1, 1, 1], shape=(1, 4)) - thresholds = [-1.0, 2.0] # lower/higher than any values - fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds( - predictions, labels, thresholds) - - fpr_low = fpr[0] - fpr_high = fpr[1] - - sess.run(variables.local_variables_initializer()) - sess.run(fpr_op) - - self.assertAlmostEqual(1.0, fpr_low.eval(), places=5) - self.assertAlmostEqual(0.0, fpr_high.eval(), places=5) - - def testZeroLabelsPredictions(self): - with self.cached_session() as sess: - predictions = array_ops.zeros([4], dtype=dtypes_lib.float32) - labels = array_ops.zeros([4]) - thresholds = [0.5] - fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds( - predictions, labels, thresholds) - - sess.run(variables.local_variables_initializer()) - sess.run(fpr_op) - - self.assertAlmostEqual(0, fpr.eval(), 6) - - def testWithMultipleUpdates(self): - num_samples = 1000 - batch_size = 10 - num_batches = int(num_samples / batch_size) - - # Create the labels and data. - labels = np.random.randint(0, 2, size=(num_samples, 1)) - noise = np.random.normal(0.0, scale=0.2, size=(num_samples, 1)) - predictions = 0.4 + 0.2 * labels + noise - predictions[predictions > 1] = 1 - predictions[predictions < 0] = 0 - thresholds = [0.3] - - fp = 0 - tn = 0 - for i in range(num_samples): - if predictions[i] > thresholds[0]: - if labels[i] == 0: - fp += 1 - else: - if labels[i] == 0: - tn += 1 - epsilon = 1e-7 - expected_fpr = fp / (epsilon + fp + tn) - - labels = labels.astype(np.float32) - predictions = predictions.astype(np.float32) - - with self.cached_session() as sess: - # Reshape the data so its easy to queue up: - predictions_batches = predictions.reshape((batch_size, num_batches)) - labels_batches = labels.reshape((batch_size, num_batches)) - - # Enqueue the data: - predictions_queue = data_flow_ops.FIFOQueue( - num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,)) - labels_queue = data_flow_ops.FIFOQueue( - num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,)) - - for i in range(int(num_batches)): - tf_prediction = constant_op.constant(predictions_batches[:, i]) - tf_label = constant_op.constant(labels_batches[:, i]) - sess.run([ - predictions_queue.enqueue(tf_prediction), - labels_queue.enqueue(tf_label) - ]) - - tf_predictions = predictions_queue.dequeue() - tf_labels = labels_queue.dequeue() - - fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds( - tf_predictions, tf_labels, thresholds) - - sess.run(variables.local_variables_initializer()) - for _ in range(int(num_samples / batch_size)): - sess.run(fpr_op) - # Since this is only approximate, we can't expect a 6 digits match. - # Although with higher number of samples/thresholds we should see the - # accuracy improving - self.assertAlmostEqual(expected_fpr, fpr.eval(), 2) - - -class RecallAtPrecisionTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.recall_at_precision( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - precision=0.7) - _assert_metric_variables(self, ('recall_at_precision/true_positives:0', - 'recall_at_precision/false_negatives:0', - 'recall_at_precision/false_positives:0', - 'recall_at_precision/true_negatives:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.recall_at_precision( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - precision=0.7, - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.recall_at_precision( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - precision=0.7, - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) - recall, update_op = metrics.recall_at_precision( - labels, predictions, precision=0.7) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_recall = recall.eval() - for _ in range(10): - self.assertAlmostEqual(initial_recall, recall.eval(), 5) - - def testAllCorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(inputs) - recall, update_op = metrics.recall_at_precision( - labels, predictions, precision=1.0) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(1, sess.run(update_op)) - self.assertEqual(1, recall.eval()) - - def testSomeCorrectHighPrecision(self): - predictions_values = [1, .9, .8, .7, .6, .5, .4, .3] - labels_values = [1, 1, 1, 1, 0, 0, 0, 1] - - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels_values) - recall, update_op = metrics.recall_at_precision( - labels, predictions, precision=0.8) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.8, sess.run(update_op)) - self.assertAlmostEqual(0.8, recall.eval()) - - def testSomeCorrectLowPrecision(self): - predictions_values = [1, .9, .8, .7, .6, .5, .4, .3, .2, .1] - labels_values = [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] - - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels_values) - recall, update_op = metrics.recall_at_precision( - labels, predictions, precision=0.4) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - target_recall = 2.0 / 3.0 - self.assertAlmostEqual(target_recall, sess.run(update_op)) - self.assertAlmostEqual(target_recall, recall.eval()) - - def testWeighted(self): - predictions_values = [1, .9, .8, .7, .6] - labels_values = [1, 1, 0, 0, 1] - weights_values = [1, 1, 3, 4, 1] - - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels_values) - weights = constant_op.constant(weights_values) - recall, update_op = metrics.recall_at_precision( - labels, predictions, weights=weights, precision=0.4) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - target_recall = 2.0 / 3.0 - self.assertAlmostEqual(target_recall, sess.run(update_op)) - self.assertAlmostEqual(target_recall, recall.eval()) - - def _test_strict_mode(self, strict_mode, target_precision, expected_recall): - num_thresholds = 11 - predictions_values = [.2, .3, .5, .6, .7, .8, .9, .9, .9, .1] - labels_values = [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] - # Resulting thresholds and the corresponding precision and recall values at - # each threshold: - # Thresholds [0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9] - # precisions: [0.3 0.2 0.1 0 0 0 0 0 0] - # recalls: [1.0 0.7 0.3 0 0 0 0 0 0] - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels_values) - recall, update_op = metrics.recall_at_precision( - labels, - predictions, - num_thresholds=num_thresholds, - precision=target_precision, - strict_mode=strict_mode) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(expected_recall, sess.run(update_op)) - self.assertAlmostEqual(expected_recall, recall.eval()) - - def testStrictMode_Off(self): - # strict_mode is turned off and return the recall at the threshold where the - # precision (0.3) is closest to target precision (0.9). The recall - # corresponding to the threshold is 1.0. - self._test_strict_mode( - strict_mode=False, target_precision=0.9, expected_recall=1.0) - - def testStrictMode_OnAndFail(self): - # strict_mode is turned on and we fail to reach the target precision at any - # threshold. - # Target precision: 0.9 - # Diff: [-0.6 -0.7 -0.8 -0.9 -0.9 -0.9 -0.9 -0.9 -0.9] - # Reciprocal: [-1.6 -1.4 -1.3 -1.1 -1.1 -1.1 -1.1 -1.1 -1.1] - # Max index: 3 and corresponding precision is: 0 which is smaller than - # target precsion 0.9. As a result, the expected recall is 0. - self._test_strict_mode( - strict_mode=True, target_precision=0.9, expected_recall=.0) - - def testStrictMode_OnAndSucceed(self): - # strict_mode is on and we can reach the target precision at certain - # threshold. - # Target precision: 0.2 - # Diff: [0.1 0 -0.1 -0.2 -0.2 -0.2 -0.2 -0.2 -0.2] - # Reciprocal: [10 infty -10.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0] - # Max index: 1 and corresponding precision is: 0.2 which is no smaller than - # target precsion 0.2. In this case, we return the recall at index 1, which - # is 2.0/3 (0.7). - self._test_strict_mode( - strict_mode=True, target_precision=0.2, expected_recall=2.0 / 3) - - -class PrecisionAtRecallTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.precision_at_recall( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - target_recall=0.7) - _assert_metric_variables(self, - ('precision_at_recall/true_positives:0', - 'precision_at_recall/false_negatives:0', - 'precision_at_recall/false_positives:0', - 'precision_at_recall/true_negatives:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.precision_at_recall( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - target_recall=0.7, - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.precision_at_recall( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - target_recall=0.7, - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=1) - precision, update_op = metrics.precision_at_recall( - labels, predictions, target_recall=0.7) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_precision = precision.eval() - for _ in range(10): - self.assertAlmostEqual(initial_precision, precision.eval(), places=5) - - def testAllCorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(inputs) - precision, update_op = metrics.precision_at_recall( - labels, predictions, target_recall=0.7) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(1, sess.run(update_op)) - self.assertEqual(1, precision.eval()) - - def testAllIncorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = 1.0 - predictions - label_prior = math_ops.reduce_mean(labels) - precision, update_op = metrics.precision_at_recall( - labels, predictions, target_recall=0.2) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(sess.run(label_prior), sess.run(update_op)) - self.assertEqual(sess.run(label_prior), precision.eval()) - - def testSomeCorrectHighRecall(self): - predictions_values = [0.1, 0.2, 0.5, 0.3, 0.0, 0.1, 0.45, 0.5, 0.8, 0.9] - labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] - - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels_values) - precision, update_op = metrics.precision_at_recall( - labels, predictions, target_recall=0.8) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.8, sess.run(update_op)) - self.assertAlmostEqual(0.8, precision.eval()) - - def testSomeCorrectLowRecall(self): - predictions_values = [0.1, 0.2, 0.7, 0.3, 0.0, 0.1, 0.45, 0.5, 0.6, 0.9] - labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] - - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels_values) - precision, update_op = metrics.precision_at_recall( - labels, predictions, target_recall=0.4) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(2.0/3, sess.run(update_op)) - self.assertAlmostEqual(2.0/3, precision.eval()) - - def testWeighted_multipleLabelDtypes(self): - for label_dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32): - predictions_values = [ - 0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.22, 0.25, 0.31, 0.35] - labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] - weights_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - - predictions = constant_op.constant( - predictions_values, dtype=dtypes_lib.float32) - labels = math_ops.cast(labels_values, dtype=label_dtype) - weights = constant_op.constant(weights_values) - precision, update_op = metrics.precision_at_recall( - labels, predictions, target_recall=0.8, weights=weights) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(34.0/43, sess.run(update_op)) - self.assertAlmostEqual(34.0/43, precision.eval()) - - -class StreamingFNRThresholdsTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_false_negative_rate_at_thresholds( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - thresholds=[0, 0.5, 1.0]) - _assert_metric_variables(self, ( - 'false_negative_rate_at_thresholds/false_negatives:0', - 'false_negative_rate_at_thresholds/true_positives:0', - )) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - fnr, _ = metrics.streaming_false_negative_rate_at_thresholds( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - thresholds=[0, 0.5, 1.0], - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [fnr]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_false_negative_rate_at_thresholds( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - thresholds=[0, 0.5, 1.0], - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) - labels = random_ops.random_uniform( - (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) - thresholds = [0, 0.5, 1.0] - fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds( - predictions, labels, thresholds) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(fnr_op) - - # Then verify idempotency. - initial_fnr = fnr.eval() - for _ in range(10): - self.assertAllClose(initial_fnr, fnr.eval()) - - def testAllCorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - with self.cached_session() as sess: - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(inputs) - thresholds = [0.5] - fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds( - predictions, labels, thresholds) - - sess.run(variables.local_variables_initializer()) - sess.run(fnr_op) - - self.assertEqual(0, fnr.eval()) - - def testSomeCorrect(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) - thresholds = [0.5] - fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds( - predictions, labels, thresholds) - - sess.run(variables.local_variables_initializer()) - sess.run(fnr_op) - - self.assertAlmostEqual(0.5, fnr.eval()) - - def testAllIncorrect(self): - inputs = np.random.randint(0, 2, size=(100, 1)) - - with self.cached_session() as sess: - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32) - thresholds = [0.5] - fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds( - predictions, labels, thresholds) - - sess.run(variables.local_variables_initializer()) - sess.run(fnr_op) - - self.assertAlmostEqual(1, fnr.eval()) - - def testWeights1d(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32) - labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) - weights = constant_op.constant( - [[0], [1]], shape=(2, 1), dtype=dtypes_lib.float32) - thresholds = [0.5, 1.1] - fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds( - predictions, labels, thresholds, weights=weights) - - fnr_low = fnr[0] - fnr_high = fnr[1] - - sess.run(variables.local_variables_initializer()) - sess.run(fnr_op) - - self.assertAlmostEqual(0.0, fnr_low.eval(), places=5) - self.assertAlmostEqual(1.0, fnr_high.eval(), places=5) - - def testWeights2d(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32) - labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) - weights = constant_op.constant( - [[0, 0], [1, 1]], shape=(2, 2), dtype=dtypes_lib.float32) - thresholds = [0.5, 1.1] - fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds( - predictions, labels, thresholds, weights=weights) - - fnr_low = fnr[0] - fnr_high = fnr[1] - - sess.run(variables.local_variables_initializer()) - sess.run(fnr_op) - - self.assertAlmostEqual(0.0, fnr_low.eval(), places=5) - self.assertAlmostEqual(1.0, fnr_high.eval(), places=5) - - def testExtremeThresholds(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant([0, 1, 1, 1], shape=(1, 4)) - thresholds = [-1.0, 2.0] # lower/higher than any values - fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds( - predictions, labels, thresholds) - - fnr_low = fnr[0] - fnr_high = fnr[1] - - sess.run(variables.local_variables_initializer()) - sess.run(fnr_op) - - self.assertAlmostEqual(0.0, fnr_low.eval()) - self.assertAlmostEqual(1.0, fnr_high.eval()) - - def testZeroLabelsPredictions(self): - with self.cached_session() as sess: - predictions = array_ops.zeros([4], dtype=dtypes_lib.float32) - labels = array_ops.zeros([4]) - thresholds = [0.5] - fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds( - predictions, labels, thresholds) - - sess.run(variables.local_variables_initializer()) - sess.run(fnr_op) - - self.assertAlmostEqual(0, fnr.eval(), 6) - - def testWithMultipleUpdates(self): - num_samples = 1000 - batch_size = 10 - num_batches = int(num_samples / batch_size) - - # Create the labels and data. - labels = np.random.randint(0, 2, size=(num_samples, 1)) - noise = np.random.normal(0.0, scale=0.2, size=(num_samples, 1)) - predictions = 0.4 + 0.2 * labels + noise - predictions[predictions > 1] = 1 - predictions[predictions < 0] = 0 - thresholds = [0.3] - - fn = 0 - tp = 0 - for i in range(num_samples): - if predictions[i] > thresholds[0]: - if labels[i] == 1: - tp += 1 - else: - if labels[i] == 1: - fn += 1 - epsilon = 1e-7 - expected_fnr = fn / (epsilon + fn + tp) - - labels = labels.astype(np.float32) - predictions = predictions.astype(np.float32) - - with self.cached_session() as sess: - # Reshape the data so its easy to queue up: - predictions_batches = predictions.reshape((batch_size, num_batches)) - labels_batches = labels.reshape((batch_size, num_batches)) - - # Enqueue the data: - predictions_queue = data_flow_ops.FIFOQueue( - num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,)) - labels_queue = data_flow_ops.FIFOQueue( - num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,)) - - for i in range(int(num_batches)): - tf_prediction = constant_op.constant(predictions_batches[:, i]) - tf_label = constant_op.constant(labels_batches[:, i]) - sess.run([ - predictions_queue.enqueue(tf_prediction), - labels_queue.enqueue(tf_label) - ]) - - tf_predictions = predictions_queue.dequeue() - tf_labels = labels_queue.dequeue() - - fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds( - tf_predictions, tf_labels, thresholds) - - sess.run(variables.local_variables_initializer()) - for _ in range(int(num_samples / batch_size)): - sess.run(fnr_op) - # Since this is only approximate, we can't expect a 6 digits match. - # Although with higher number of samples/thresholds we should see the - # accuracy improving - self.assertAlmostEqual(expected_fnr, fnr.eval(), 2) - - -# TODO(ptucker): Remove when we remove `streaming_recall_at_k`. -# This op will be deprecated soon in favor of `streaming_sparse_recall_at_k`. -# Until then, this test validates that both ops yield the same results. -class StreamingRecallAtKTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - self._batch_size = 4 - self._num_classes = 3 - self._np_predictions = np.matrix(('0.1 0.2 0.7;' - '0.6 0.2 0.2;' - '0.0 0.9 0.1;' - '0.2 0.0 0.8')) - self._np_labels = [0, 0, 0, 0] - - def testVars(self): - metrics.streaming_recall_at_k( - predictions=array_ops.ones((self._batch_size, self._num_classes)), - labels=array_ops.ones((self._batch_size,), dtype=dtypes_lib.int32), - k=1) - _assert_metric_variables(self, - ('recall_at_1/count:0', 'recall_at_1/total:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_recall_at_k( - predictions=array_ops.ones((self._batch_size, self._num_classes)), - labels=array_ops.ones((self._batch_size,), dtype=dtypes_lib.int32), - k=1, - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_recall_at_k( - predictions=array_ops.ones((self._batch_size, self._num_classes)), - labels=array_ops.ones((self._batch_size,), dtype=dtypes_lib.int32), - k=1, - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testSingleUpdateKIs1(self): - predictions = constant_op.constant( - self._np_predictions, - shape=(self._batch_size, self._num_classes), - dtype=dtypes_lib.float32) - labels = constant_op.constant( - self._np_labels, shape=(self._batch_size,), dtype=dtypes_lib.int64) - recall, update_op = metrics.streaming_recall_at_k(predictions, labels, k=1) - sp_recall, sp_update_op = metrics.streaming_sparse_recall_at_k( - predictions, array_ops.reshape(labels, (self._batch_size, 1)), k=1) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0.25, sess.run(update_op)) - self.assertEqual(0.25, recall.eval()) - self.assertEqual(0.25, sess.run(sp_update_op)) - self.assertEqual(0.25, sp_recall.eval()) - - def testSingleUpdateKIs2(self): - predictions = constant_op.constant( - self._np_predictions, - shape=(self._batch_size, self._num_classes), - dtype=dtypes_lib.float32) - labels = constant_op.constant( - self._np_labels, shape=(self._batch_size,), dtype=dtypes_lib.int64) - recall, update_op = metrics.streaming_recall_at_k(predictions, labels, k=2) - sp_recall, sp_update_op = metrics.streaming_sparse_recall_at_k( - predictions, array_ops.reshape(labels, (self._batch_size, 1)), k=2) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0.5, sess.run(update_op)) - self.assertEqual(0.5, recall.eval()) - self.assertEqual(0.5, sess.run(sp_update_op)) - self.assertEqual(0.5, sp_recall.eval()) - - def testSingleUpdateKIs3(self): - predictions = constant_op.constant( - self._np_predictions, - shape=(self._batch_size, self._num_classes), - dtype=dtypes_lib.float32) - labels = constant_op.constant( - self._np_labels, shape=(self._batch_size,), dtype=dtypes_lib.int64) - recall, update_op = metrics.streaming_recall_at_k(predictions, labels, k=3) - sp_recall, sp_update_op = metrics.streaming_sparse_recall_at_k( - predictions, array_ops.reshape(labels, (self._batch_size, 1)), k=3) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(1.0, sess.run(update_op)) - self.assertEqual(1.0, recall.eval()) - self.assertEqual(1.0, sess.run(sp_update_op)) - self.assertEqual(1.0, sp_recall.eval()) - - def testSingleUpdateSomeMissingKIs2(self): - predictions = constant_op.constant( - self._np_predictions, - shape=(self._batch_size, self._num_classes), - dtype=dtypes_lib.float32) - labels = constant_op.constant( - self._np_labels, shape=(self._batch_size,), dtype=dtypes_lib.int64) - weights = constant_op.constant( - [0, 1, 0, 1], shape=(self._batch_size,), dtype=dtypes_lib.float32) - recall, update_op = metrics.streaming_recall_at_k( - predictions, labels, k=2, weights=weights) - sp_recall, sp_update_op = metrics.streaming_sparse_recall_at_k( - predictions, - array_ops.reshape(labels, (self._batch_size, 1)), - k=2, - weights=weights) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(1.0, sess.run(update_op)) - self.assertEqual(1.0, recall.eval()) - self.assertEqual(1.0, sess.run(sp_update_op)) - self.assertEqual(1.0, sp_recall.eval()) - - -class StreamingSparsePrecisionTest(test.TestCase): - - def _test_streaming_sparse_precision_at_k(self, - predictions, - labels, - k, - expected, - class_id=None, - weights=None): - with ops.Graph().as_default() as g, self.session(g): - if weights is not None: - weights = constant_op.constant(weights, dtypes_lib.float32) - metric, update = metrics.streaming_sparse_precision_at_k( - predictions=constant_op.constant(predictions, dtypes_lib.float32), - labels=labels, - k=k, - class_id=class_id, - weights=weights) - - # Fails without initialized vars. - self.assertRaises(errors_impl.OpError, metric.eval) - self.assertRaises(errors_impl.OpError, update.eval) - variables.variables_initializer(variables.local_variables()).run() - - # Run per-step op and assert expected values. - if math.isnan(expected): - _assert_nan(self, update.eval()) - _assert_nan(self, metric.eval()) - else: - self.assertEqual(expected, update.eval()) - self.assertEqual(expected, metric.eval()) - - def _test_streaming_sparse_precision_at_top_k(self, - top_k_predictions, - labels, - expected, - class_id=None, - weights=None): - with ops.Graph().as_default() as g, self.session(g): - if weights is not None: - weights = constant_op.constant(weights, dtypes_lib.float32) - metric, update = metrics.streaming_sparse_precision_at_top_k( - top_k_predictions=constant_op.constant(top_k_predictions, - dtypes_lib.int32), - labels=labels, - class_id=class_id, - weights=weights) - - # Fails without initialized vars. - self.assertRaises(errors_impl.OpError, metric.eval) - self.assertRaises(errors_impl.OpError, update.eval) - variables.variables_initializer(variables.local_variables()).run() - - # Run per-step op and assert expected values. - if math.isnan(expected): - self.assertTrue(math.isnan(update.eval())) - self.assertTrue(math.isnan(metric.eval())) - else: - self.assertEqual(expected, update.eval()) - self.assertEqual(expected, metric.eval()) - - def _test_streaming_sparse_average_precision_at_k(self, - predictions, - labels, - k, - expected, - weights=None): - with ops.Graph().as_default() as g, self.session(g): - if weights is not None: - weights = constant_op.constant(weights, dtypes_lib.float32) - predictions = constant_op.constant(predictions, dtypes_lib.float32) - metric, update = metrics.streaming_sparse_average_precision_at_k( - predictions, labels, k, weights=weights) - - # Fails without initialized vars. - self.assertRaises(errors_impl.OpError, metric.eval) - self.assertRaises(errors_impl.OpError, update.eval) - local_variables = variables.local_variables() - variables.variables_initializer(local_variables).run() - - # Run per-step op and assert expected values. - if math.isnan(expected): - _assert_nan(self, update.eval()) - _assert_nan(self, metric.eval()) - else: - self.assertAlmostEqual(expected, update.eval()) - self.assertAlmostEqual(expected, metric.eval()) - - def _test_streaming_sparse_average_precision_at_top_k(self, - top_k_predictions, - labels, - expected, - weights=None): - with ops.Graph().as_default() as g, self.session(g): - if weights is not None: - weights = constant_op.constant(weights, dtypes_lib.float32) - metric, update = metrics.streaming_sparse_average_precision_at_top_k( - top_k_predictions, labels, weights=weights) - - # Fails without initialized vars. - self.assertRaises(errors_impl.OpError, metric.eval) - self.assertRaises(errors_impl.OpError, update.eval) - local_variables = variables.local_variables() - variables.variables_initializer(local_variables).run() - - # Run per-step op and assert expected values. - if math.isnan(expected): - _assert_nan(self, update.eval()) - _assert_nan(self, metric.eval()) - else: - self.assertAlmostEqual(expected, update.eval()) - self.assertAlmostEqual(expected, metric.eval()) - - def test_top_k_rank_invalid(self): - with self.cached_session(): - # top_k_predictions has rank < 2. - top_k_predictions = [9, 4, 6, 2, 0] - sp_labels = sparse_tensor.SparseTensorValue( - indices=np.array([[ - 0, - ], [ - 1, - ], [ - 2, - ]], np.int64), - values=np.array([2, 7, 8], np.int64), - dense_shape=np.array([ - 10, - ], np.int64)) - - with self.assertRaises(ValueError): - precision, _ = metrics.streaming_sparse_precision_at_top_k( - top_k_predictions=constant_op.constant(top_k_predictions, - dtypes_lib.int64), - labels=sp_labels) - variables.variables_initializer(variables.local_variables()).run() - precision.eval() - - def test_average_precision(self): - # Example 1. - # Matches example here: - # fastml.com/what-you-wanted-to-know-about-mean-average-precision - labels_ex1 = (0, 1, 2, 3, 4) - labels = np.array([labels_ex1], dtype=np.int64) - predictions_ex1 = (0.2, 0.1, 0.0, 0.4, 0.0, 0.5, 0.3) - predictions = (predictions_ex1,) - predictions_top_k_ex1 = (5, 3, 6, 0, 1, 2) - precision_ex1 = (0.0 / 1, 1.0 / 2, 1.0 / 3, 2.0 / 4) - avg_precision_ex1 = (0.0 / 1, precision_ex1[1] / 2, precision_ex1[1] / 3, - (precision_ex1[1] + precision_ex1[3]) / 4) - for i in xrange(4): - k = i + 1 - self._test_streaming_sparse_precision_at_k( - predictions, labels, k, expected=precision_ex1[i]) - self._test_streaming_sparse_precision_at_top_k( - (predictions_top_k_ex1[:k],), labels, expected=precision_ex1[i]) - self._test_streaming_sparse_average_precision_at_k( - predictions, labels, k, expected=avg_precision_ex1[i]) - self._test_streaming_sparse_average_precision_at_top_k( - (predictions_top_k_ex1[:k],), labels, expected=avg_precision_ex1[i]) - - # Example 2. - labels_ex2 = (0, 2, 4, 5, 6) - labels = np.array([labels_ex2], dtype=np.int64) - predictions_ex2 = (0.3, 0.5, 0.0, 0.4, 0.0, 0.1, 0.2) - predictions = (predictions_ex2,) - predictions_top_k_ex2 = (1, 3, 0, 6, 5) - precision_ex2 = (0.0 / 1, 0.0 / 2, 1.0 / 3, 2.0 / 4) - avg_precision_ex2 = (0.0 / 1, 0.0 / 2, precision_ex2[2] / 3, - (precision_ex2[2] + precision_ex2[3]) / 4) - for i in xrange(4): - k = i + 1 - self._test_streaming_sparse_precision_at_k( - predictions, labels, k, expected=precision_ex2[i]) - self._test_streaming_sparse_precision_at_top_k( - (predictions_top_k_ex2[:k],), labels, expected=precision_ex2[i]) - self._test_streaming_sparse_average_precision_at_k( - predictions, labels, k, expected=avg_precision_ex2[i]) - self._test_streaming_sparse_average_precision_at_top_k( - (predictions_top_k_ex2[:k],), labels, expected=avg_precision_ex2[i]) - - # Both examples, we expect both precision and average precision to be the - # average of the 2 examples. - labels = np.array([labels_ex1, labels_ex2], dtype=np.int64) - predictions = (predictions_ex1, predictions_ex2) - streaming_precision = [ - (ex1 + ex2) / 2 for ex1, ex2 in zip(precision_ex1, precision_ex2) - ] - streaming_average_precision = [ - (ex1 + ex2) / 2 - for ex1, ex2 in zip(avg_precision_ex1, avg_precision_ex2) - ] - for i in xrange(4): - k = i + 1 - self._test_streaming_sparse_precision_at_k( - predictions, labels, k, expected=streaming_precision[i]) - predictions_top_k = (predictions_top_k_ex1[:k], predictions_top_k_ex2[:k]) - self._test_streaming_sparse_precision_at_top_k( - predictions_top_k, labels, expected=streaming_precision[i]) - self._test_streaming_sparse_average_precision_at_k( - predictions, labels, k, expected=streaming_average_precision[i]) - self._test_streaming_sparse_average_precision_at_top_k( - predictions_top_k, labels, expected=streaming_average_precision[i]) - - # Weighted examples, we expect streaming average precision to be the - # weighted average of the 2 examples. - weights = (0.3, 0.6) - streaming_average_precision = [ - (weights[0] * ex1 + weights[1] * ex2) / (weights[0] + weights[1]) - for ex1, ex2 in zip(avg_precision_ex1, avg_precision_ex2) - ] - for i in xrange(4): - k = i + 1 - self._test_streaming_sparse_average_precision_at_k( - predictions, - labels, - k, - expected=streaming_average_precision[i], - weights=weights) - self._test_streaming_sparse_average_precision_at_top_k( - (predictions_top_k_ex1[:k], predictions_top_k_ex2[:k]), - labels, - expected=streaming_average_precision[i], - weights=weights) - - def test_average_precision_some_labels_out_of_range(self): - """Tests that labels outside the [0, n_classes) range are ignored.""" - labels_ex1 = (-1, 0, 1, 2, 3, 4, 7) - labels = np.array([labels_ex1], dtype=np.int64) - predictions_ex1 = (0.2, 0.1, 0.0, 0.4, 0.0, 0.5, 0.3) - predictions = (predictions_ex1,) - predictions_top_k_ex1 = (5, 3, 6, 0, 1, 2) - precision_ex1 = (0.0 / 1, 1.0 / 2, 1.0 / 3, 2.0 / 4) - avg_precision_ex1 = (0.0 / 1, precision_ex1[1] / 2, precision_ex1[1] / 3, - (precision_ex1[1] + precision_ex1[3]) / 4) - for i in xrange(4): - k = i + 1 - self._test_streaming_sparse_precision_at_k( - predictions, labels, k, expected=precision_ex1[i]) - self._test_streaming_sparse_precision_at_top_k( - (predictions_top_k_ex1[:k],), labels, expected=precision_ex1[i]) - self._test_streaming_sparse_average_precision_at_k( - predictions, labels, k, expected=avg_precision_ex1[i]) - self._test_streaming_sparse_average_precision_at_top_k( - (predictions_top_k_ex1[:k],), labels, expected=avg_precision_ex1[i]) - - def test_average_precision_at_top_k_static_shape_check(self): - predictions_top_k = array_ops.placeholder( - shape=(2, None), dtype=dtypes_lib.int64) - labels = np.array(((1,), (2,)), dtype=np.int64) - # Fails due to non-static predictions_idx shape. - with self.assertRaises(ValueError): - metric_ops.streaming_sparse_average_precision_at_top_k( - predictions_top_k, labels) - - predictions_top_k = (2, 1) - # Fails since rank of predictions_idx is less than one. - with self.assertRaises(ValueError): - metric_ops.streaming_sparse_average_precision_at_top_k( - predictions_top_k, labels) - predictions_top_k = ((2,), (1,)) - # Valid static shape. - metric_ops.streaming_sparse_average_precision_at_top_k( - predictions_top_k, labels) - - def test_one_label_at_k1_nan(self): - predictions = [[0.1, 0.3, 0.2, 0.4], [0.1, 0.2, 0.3, 0.4]] - top_k_predictions = [[3], [3]] - sparse_labels = _binary_2d_label_to_sparse_value([[0, 0, 0, 1], - [0, 0, 1, 0]]) - dense_labels = np.array([[3], [2]], dtype=np.int64) - - for labels in (sparse_labels, dense_labels): - # Classes 0,1,2 have 0 predictions, classes -1 and 4 are out of range. - for class_id in (-1, 0, 1, 2, 4): - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=1, expected=NAN, class_id=class_id) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=NAN, class_id=class_id) - - def test_one_label_at_k1(self): - predictions = [[0.1, 0.3, 0.2, 0.4], [0.1, 0.2, 0.3, 0.4]] - top_k_predictions = [[3], [3]] - sparse_labels = _binary_2d_label_to_sparse_value([[0, 0, 0, 1], - [0, 0, 1, 0]]) - dense_labels = np.array([[3], [2]], dtype=np.int64) - - for labels in (sparse_labels, dense_labels): - # Class 3: 1 label, 2 predictions, 1 correct. - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=1, expected=1.0 / 2, class_id=3) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=1.0 / 2, class_id=3) - - # All classes: 2 labels, 2 predictions, 1 correct. - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=1, expected=1.0 / 2) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=1.0 / 2) - - def test_three_labels_at_k5_no_predictions(self): - predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] - top_k_predictions = [ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ] - sparse_labels = _binary_2d_label_to_sparse_value( - [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]]) - dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64) - - for labels in (sparse_labels, dense_labels): - # Classes 1,3,8 have 0 predictions, classes -1 and 10 are out of range. - for class_id in (-1, 1, 3, 8, 10): - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=NAN, class_id=class_id) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=NAN, class_id=class_id) - - def test_three_labels_at_k5_no_labels(self): - predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] - top_k_predictions = [ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ] - sparse_labels = _binary_2d_label_to_sparse_value( - [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]]) - dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64) - - for labels in (sparse_labels, dense_labels): - # Classes 0,4,6,9: 0 labels, >=1 prediction. - for class_id in (0, 4, 6, 9): - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=0.0, class_id=class_id) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=0.0, class_id=class_id) - - def test_three_labels_at_k5(self): - predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] - top_k_predictions = [ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ] - sparse_labels = _binary_2d_label_to_sparse_value( - [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]]) - dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64) - - for labels in (sparse_labels, dense_labels): - # Class 2: 2 labels, 2 correct predictions. - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=2.0 / 2, class_id=2) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=2.0 / 2, class_id=2) - - # Class 5: 1 label, 1 correct prediction. - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=1.0 / 1, class_id=5) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=1.0 / 1, class_id=5) - - # Class 7: 1 label, 1 incorrect prediction. - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=0.0 / 1, class_id=7) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=0.0 / 1, class_id=7) - - # All classes: 10 predictions, 3 correct. - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=3.0 / 10) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=3.0 / 10) - - def test_three_labels_at_k5_some_out_of_range(self): - """Tests that labels outside the [0, n_classes) range are ignored.""" - predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] - top_k_predictions = [ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ] - sp_labels = sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1], [0, 2], [0, 3], [1, 0], [1, 1], [1, 2], [1, - 3]], - # values -1 and 10 are outside the [0, n_classes) range and are ignored. - values=np.array([2, 7, -1, 8, 1, 2, 5, 10], np.int64), - dense_shape=[2, 4]) - - # Class 2: 2 labels, 2 correct predictions. - self._test_streaming_sparse_precision_at_k( - predictions, sp_labels, k=5, expected=2.0 / 2, class_id=2) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, sp_labels, expected=2.0 / 2, class_id=2) - - # Class 5: 1 label, 1 correct prediction. - self._test_streaming_sparse_precision_at_k( - predictions, sp_labels, k=5, expected=1.0 / 1, class_id=5) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, sp_labels, expected=1.0 / 1, class_id=5) - - # Class 7: 1 label, 1 incorrect prediction. - self._test_streaming_sparse_precision_at_k( - predictions, sp_labels, k=5, expected=0.0 / 1, class_id=7) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, sp_labels, expected=0.0 / 1, class_id=7) - - # All classes: 10 predictions, 3 correct. - self._test_streaming_sparse_precision_at_k( - predictions, sp_labels, k=5, expected=3.0 / 10) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, sp_labels, expected=3.0 / 10) - - def test_3d_nan(self): - predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], - [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], - [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] - top_k_predictions = [[ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ], [ - [5, 7, 2, 9, 6], - [9, 4, 6, 2, 0], - ]] - labels = _binary_3d_label_to_sparse_value( - [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], - [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]]) - - # Classes 1,3,8 have 0 predictions, classes -1 and 10 are out of range. - for class_id in (-1, 1, 3, 8, 10): - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=NAN, class_id=class_id) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=NAN, class_id=class_id) - - def test_3d_no_labels(self): - predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], - [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], - [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] - top_k_predictions = [[ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ], [ - [5, 7, 2, 9, 6], - [9, 4, 6, 2, 0], - ]] - labels = _binary_3d_label_to_sparse_value( - [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], - [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]]) - - # Classes 0,4,6,9: 0 labels, >=1 prediction. - for class_id in (0, 4, 6, 9): - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=0.0, class_id=class_id) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=0.0, class_id=class_id) - - def test_3d(self): - predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], - [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], - [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] - top_k_predictions = [[ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ], [ - [5, 7, 2, 9, 6], - [9, 4, 6, 2, 0], - ]] - labels = _binary_3d_label_to_sparse_value( - [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], - [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]]) - - # Class 2: 4 predictions, all correct. - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=4.0 / 4, class_id=2) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=4.0 / 4, class_id=2) - - # Class 5: 2 predictions, both correct. - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=2.0 / 2, class_id=5) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=2.0 / 2, class_id=5) - - # Class 7: 2 predictions, 1 correct. - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=1.0 / 2, class_id=7) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=1.0 / 2, class_id=7) - - # All classes: 20 predictions, 7 correct. - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=7.0 / 20) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=7.0 / 20) - - def test_3d_ignore_all(self): - predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], - [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], - [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] - top_k_predictions = [[ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ], [ - [5, 7, 2, 9, 6], - [9, 4, 6, 2, 0], - ]] - labels = _binary_3d_label_to_sparse_value( - [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], - [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]]) - - for class_id in xrange(10): - self._test_streaming_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=NAN, - class_id=class_id, - weights=[[0], [0]]) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, - labels, - expected=NAN, - class_id=class_id, - weights=[[0], [0]]) - self._test_streaming_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=NAN, - class_id=class_id, - weights=[[0, 0], [0, 0]]) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, - labels, - expected=NAN, - class_id=class_id, - weights=[[0, 0], [0, 0]]) - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=NAN, weights=[[0], [0]]) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=NAN, weights=[[0], [0]]) - self._test_streaming_sparse_precision_at_k( - predictions, labels, k=5, expected=NAN, weights=[[0, 0], [0, 0]]) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, labels, expected=NAN, weights=[[0, 0], [0, 0]]) - - def test_3d_ignore_some(self): - predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], - [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], - [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] - top_k_predictions = [[ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ], [ - [5, 7, 2, 9, 6], - [9, 4, 6, 2, 0], - ]] - labels = _binary_3d_label_to_sparse_value( - [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], - [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]]) - - # Class 2: 2 predictions, both correct. - self._test_streaming_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=2.0 / 2.0, - class_id=2, - weights=[[1], [0]]) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, - labels, - expected=2.0 / 2.0, - class_id=2, - weights=[[1], [0]]) - - # Class 2: 2 predictions, both correct. - self._test_streaming_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=2.0 / 2.0, - class_id=2, - weights=[[0], [1]]) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, - labels, - expected=2.0 / 2.0, - class_id=2, - weights=[[0], [1]]) - - # Class 7: 1 incorrect prediction. - self._test_streaming_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=0.0 / 1.0, - class_id=7, - weights=[[1], [0]]) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, - labels, - expected=0.0 / 1.0, - class_id=7, - weights=[[1], [0]]) - - # Class 7: 1 correct prediction. - self._test_streaming_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=1.0 / 1.0, - class_id=7, - weights=[[0], [1]]) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, - labels, - expected=1.0 / 1.0, - class_id=7, - weights=[[0], [1]]) - - # Class 7: no predictions. - self._test_streaming_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=NAN, - class_id=7, - weights=[[1, 0], [0, 1]]) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, - labels, - expected=NAN, - class_id=7, - weights=[[1, 0], [0, 1]]) - - # Class 7: 2 predictions, 1 correct. - self._test_streaming_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=1.0 / 2.0, - class_id=7, - weights=[[0, 1], [1, 0]]) - self._test_streaming_sparse_precision_at_top_k( - top_k_predictions, - labels, - expected=1.0 / 2.0, - class_id=7, - weights=[[0, 1], [1, 0]]) - - def test_sparse_tensor_value(self): - predictions = [[0.1, 0.3, 0.2, 0.4], [0.1, 0.2, 0.3, 0.4]] - labels = [[0, 0, 0, 1], [0, 0, 1, 0]] - expected_precision = 0.5 - with self.cached_session(): - _, precision = metrics.streaming_sparse_precision_at_k( - predictions=constant_op.constant(predictions, dtypes_lib.float32), - labels=_binary_2d_label_to_sparse_value(labels), - k=1) - - variables.variables_initializer(variables.local_variables()).run() - - self.assertEqual(expected_precision, precision.eval()) - - -class StreamingSparseRecallTest(test.TestCase): - - def _test_streaming_sparse_recall_at_k(self, - predictions, - labels, - k, - expected, - class_id=None, - weights=None): - with ops.Graph().as_default() as g, self.session(g): - if weights is not None: - weights = constant_op.constant(weights, dtypes_lib.float32) - metric, update = metrics.streaming_sparse_recall_at_k( - predictions=constant_op.constant(predictions, dtypes_lib.float32), - labels=labels, - k=k, - class_id=class_id, - weights=weights) - - # Fails without initialized vars. - self.assertRaises(errors_impl.OpError, metric.eval) - self.assertRaises(errors_impl.OpError, update.eval) - variables.variables_initializer(variables.local_variables()).run() - - # Run per-step op and assert expected values. - if math.isnan(expected): - _assert_nan(self, update.eval()) - _assert_nan(self, metric.eval()) - else: - self.assertEqual(expected, update.eval()) - self.assertEqual(expected, metric.eval()) - - def _test_sparse_recall_at_top_k(self, - labels, - top_k_predictions, - expected, - class_id=None, - weights=None): - with ops.Graph().as_default() as g, self.session(g): - if weights is not None: - weights = constant_op.constant(weights, dtypes_lib.float32) - metric, update = metric_ops.sparse_recall_at_top_k( - labels=labels, - top_k_predictions=constant_op.constant(top_k_predictions, - dtypes_lib.int32), - class_id=class_id, - weights=weights) - - # Fails without initialized vars. - self.assertRaises(errors_impl.OpError, metric.eval) - self.assertRaises(errors_impl.OpError, update.eval) - variables.variables_initializer(variables.local_variables()).run() - - # Run per-step op and assert expected values. - if math.isnan(expected): - self.assertTrue(math.isnan(update.eval())) - self.assertTrue(math.isnan(metric.eval())) - else: - self.assertEqual(expected, update.eval()) - self.assertEqual(expected, metric.eval()) - - def test_one_label_at_k1_nan(self): - predictions = [[0.1, 0.3, 0.2, 0.4], [0.1, 0.2, 0.3, 0.4]] - top_k_predictions = [[3], [3]] - sparse_labels = _binary_2d_label_to_sparse_value([[0, 0, 0, 1], - [0, 0, 1, 0]]) - dense_labels = np.array([[3], [2]], dtype=np.int64) - - # Classes 0,1 have 0 labels, 0 predictions, classes -1 and 4 are out of - # range. - for labels in (sparse_labels, dense_labels): - for class_id in (-1, 0, 1, 4): - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=NAN, class_id=class_id) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=NAN, class_id=class_id) - - def test_one_label_at_k1_no_predictions(self): - predictions = [[0.1, 0.3, 0.2, 0.4], [0.1, 0.2, 0.3, 0.4]] - top_k_predictions = [[3], [3]] - sparse_labels = _binary_2d_label_to_sparse_value([[0, 0, 0, 1], - [0, 0, 1, 0]]) - dense_labels = np.array([[3], [2]], dtype=np.int64) - - for labels in (sparse_labels, dense_labels): - # Class 2: 0 predictions. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=0.0, class_id=2) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=0.0, class_id=2) - - def test_one_label_at_k1(self): - predictions = [[0.1, 0.3, 0.2, 0.4], [0.1, 0.2, 0.3, 0.4]] - top_k_predictions = [[3], [3]] - sparse_labels = _binary_2d_label_to_sparse_value([[0, 0, 0, 1], - [0, 0, 1, 0]]) - dense_labels = np.array([[3], [2]], dtype=np.int64) - - for labels in (sparse_labels, dense_labels): - # Class 3: 1 label, 2 predictions, 1 correct. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=1.0 / 1, class_id=3) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=1.0 / 1, class_id=3) - - # All classes: 2 labels, 2 predictions, 1 correct. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=1.0 / 2) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=1.0 / 2) - - def _test_one_label_at_k1_weighted(self, labels): - predictions = [[0.1, 0.3, 0.2, 0.4], [0.1, 0.2, 0.3, 0.4]] - top_k_predictions = [[3], [3]] - - # Class 3: 1 label, 2 predictions, 1 correct. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=NAN, class_id=3, weights=(0.0,)) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=NAN, class_id=3, weights=(0.0,)) - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=1, - expected=1.0 / 1, - class_id=3, - weights=(1.0,)) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=1.0 / 1, - class_id=3, - weights=(1.0,)) - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=1, - expected=1.0 / 1, - class_id=3, - weights=(2.0,)) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=1.0 / 1, - class_id=3, - weights=(2.0,)) - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=1, - expected=NAN, - class_id=3, - weights=(0.0, 0.0)) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=NAN, - class_id=3, - weights=(0.0, 0.0)) - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=1, - expected=NAN, - class_id=3, - weights=(0.0, 1.0)) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=NAN, - class_id=3, - weights=(0.0, 1.0)) - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=1, - expected=1.0 / 1, - class_id=3, - weights=(1.0, 0.0)) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=1.0 / 1, - class_id=3, - weights=(1.0, 0.0)) - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=1, - expected=1.0 / 1, - class_id=3, - weights=(1.0, 1.0)) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=1.0 / 1, - class_id=3, - weights=(1.0, 1.0)) - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=1, - expected=2.0 / 2, - class_id=3, - weights=(2.0, 3.0)) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=2.0 / 2, - class_id=3, - weights=(2.0, 3.0)) - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=1, - expected=3.0 / 3, - class_id=3, - weights=(3.0, 2.0)) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=3.0 / 3, - class_id=3, - weights=(3.0, 2.0)) - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=1, - expected=0.3 / 0.3, - class_id=3, - weights=(0.3, 0.6)) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=0.3 / 0.3, - class_id=3, - weights=(0.3, 0.6)) - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=1, - expected=0.6 / 0.6, - class_id=3, - weights=(0.6, 0.3)) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=0.6 / 0.6, - class_id=3, - weights=(0.6, 0.3)) - - # All classes: 2 labels, 2 predictions, 1 correct. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=NAN, weights=(0.0,)) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=NAN, weights=(0.0,)) - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=1.0 / 2, weights=(1.0,)) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=1.0 / 2, weights=(1.0,)) - - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=1.0 / 2, weights=(2.0,)) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=1.0 / 2, weights=(2.0,)) - - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=1.0 / 1, weights=(1.0, 0.0)) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=1.0 / 1, weights=(1.0, 0.0)) - - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=0.0 / 1, weights=(0.0, 1.0)) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=0.0 / 1, weights=(0.0, 1.0)) - - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=1.0 / 2, weights=(1.0, 1.0)) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=1.0 / 2, weights=(1.0, 1.0)) - - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=2.0 / 5, weights=(2.0, 3.0)) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=2.0 / 5, weights=(2.0, 3.0)) - - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=3.0 / 5, weights=(3.0, 2.0)) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=3.0 / 5, weights=(3.0, 2.0)) - - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=0.3 / 0.9, weights=(0.3, 0.6)) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=0.3 / 0.9, weights=(0.3, 0.6)) - - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=1, expected=0.6 / 0.9, weights=(0.6, 0.3)) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=0.6 / 0.9, weights=(0.6, 0.3)) - - def test_one_label_at_k1_weighted_sparse_labels(self): - sparse_labels = _binary_2d_label_to_sparse_value([[0, 0, 0, 1], - [0, 0, 1, 0]]) - self._test_one_label_at_k1_weighted(sparse_labels) - - def test_one_label_at_k1_weighted_dense_labels(self): - dense_labels = np.array([[3], [2]], dtype=np.int64) - self._test_one_label_at_k1_weighted(dense_labels) - - def test_three_labels_at_k5_nan(self): - predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] - top_k_predictions = [ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ] - sparse_labels = _binary_2d_label_to_sparse_value( - [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]]) - dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64) - - for labels in (sparse_labels, dense_labels): - # Classes 0,3,4,6,9 have 0 labels, class 10 is out of range. - for class_id in (0, 3, 4, 6, 9, 10): - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=NAN, class_id=class_id) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=NAN, class_id=class_id) - - def test_three_labels_at_k5_no_predictions(self): - predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] - top_k_predictions = [ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ] - sparse_labels = _binary_2d_label_to_sparse_value( - [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]]) - dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64) - - for labels in (sparse_labels, dense_labels): - # Class 8: 1 label, no predictions. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=0.0 / 1, class_id=8) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=0.0 / 1, class_id=8) - - def test_three_labels_at_k5(self): - predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] - top_k_predictions = [ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ] - sparse_labels = _binary_2d_label_to_sparse_value( - [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]]) - dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64) - - for labels in (sparse_labels, dense_labels): - # Class 2: 2 labels, both correct. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=2.0 / 2, class_id=2) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=2.0 / 2, class_id=2) - - # Class 5: 1 label, incorrect. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=1.0 / 1, class_id=5) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=1.0 / 1, class_id=5) - - # Class 7: 1 label, incorrect. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=0.0 / 1, class_id=7) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=0.0 / 1, class_id=7) - - # All classes: 6 labels, 3 correct. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=3.0 / 6) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=3.0 / 6) - - def test_three_labels_at_k5_some_out_of_range(self): - """Tests that labels outside the [0, n_classes) count in denominator.""" - predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] - top_k_predictions = [ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ] - sp_labels = sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1], [0, 2], [0, 3], [1, 0], [1, 1], [1, 2], [1, - 3]], - # values -1 and 10 are outside the [0, n_classes) range. - values=np.array([2, 7, -1, 8, 1, 2, 5, 10], np.int64), - dense_shape=[2, 4]) - - # Class 2: 2 labels, both correct. - self._test_streaming_sparse_recall_at_k( - predictions=predictions, - labels=sp_labels, - k=5, - expected=2.0 / 2, - class_id=2) - self._test_sparse_recall_at_top_k( - sp_labels, top_k_predictions, expected=2.0 / 2, class_id=2) - - # Class 5: 1 label, incorrect. - self._test_streaming_sparse_recall_at_k( - predictions=predictions, - labels=sp_labels, - k=5, - expected=1.0 / 1, - class_id=5) - self._test_sparse_recall_at_top_k( - sp_labels, top_k_predictions, expected=1.0 / 1, class_id=5) - - # Class 7: 1 label, incorrect. - self._test_streaming_sparse_recall_at_k( - predictions=predictions, - labels=sp_labels, - k=5, - expected=0.0 / 1, - class_id=7) - self._test_sparse_recall_at_top_k( - sp_labels, top_k_predictions, expected=0.0 / 1, class_id=7) - - # All classes: 8 labels, 3 correct. - self._test_streaming_sparse_recall_at_k( - predictions=predictions, labels=sp_labels, k=5, expected=3.0 / 8) - self._test_sparse_recall_at_top_k( - sp_labels, top_k_predictions, expected=3.0 / 8) - - def test_3d_nan(self): - predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], - [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], - [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] - top_k_predictions = [[ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ], [ - [5, 7, 2, 9, 6], - [9, 4, 6, 2, 0], - ]] - sparse_labels = _binary_3d_label_to_sparse_value( - [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], - [[0, 1, 1, 0, 0, 1, 0, 0, 0, 0], [0, 0, 1, 0, 0, 0, 0, 1, 1, 0]]]) - dense_labels = np.array( - [[[2, 7, 8], [1, 2, 5]], [ - [1, 2, 5], - [2, 7, 8], - ]], dtype=np.int64) - - for labels in (sparse_labels, dense_labels): - # Classes 0,3,4,6,9 have 0 labels, class 10 is out of range. - for class_id in (0, 3, 4, 6, 9, 10): - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=NAN, class_id=class_id) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=NAN, class_id=class_id) - - def test_3d_no_predictions(self): - predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], - [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], - [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] - top_k_predictions = [[ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ], [ - [5, 7, 2, 9, 6], - [9, 4, 6, 2, 0], - ]] - sparse_labels = _binary_3d_label_to_sparse_value( - [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], - [[0, 1, 1, 0, 0, 1, 0, 0, 0, 0], [0, 0, 1, 0, 0, 0, 0, 1, 1, 0]]]) - dense_labels = np.array( - [[[2, 7, 8], [1, 2, 5]], [ - [1, 2, 5], - [2, 7, 8], - ]], dtype=np.int64) - - for labels in (sparse_labels, dense_labels): - # Classes 1,8 have 0 predictions, >=1 label. - for class_id in (1, 8): - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=0.0, class_id=class_id) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=0.0, class_id=class_id) - - def test_3d(self): - predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], - [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], - [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] - top_k_predictions = [[ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ], [ - [5, 7, 2, 9, 6], - [9, 4, 6, 2, 0], - ]] - labels = _binary_3d_label_to_sparse_value( - [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], - [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]]) - - # Class 2: 4 labels, all correct. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=4.0 / 4, class_id=2) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=4.0 / 4, class_id=2) - - # Class 5: 2 labels, both correct. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=2.0 / 2, class_id=5) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=2.0 / 2, class_id=5) - - # Class 7: 2 labels, 1 incorrect. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=1.0 / 2, class_id=7) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=1.0 / 2, class_id=7) - - # All classes: 12 labels, 7 correct. - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=7.0 / 12) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=7.0 / 12) - - def test_3d_ignore_all(self): - predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], - [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], - [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] - top_k_predictions = [[ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ], [ - [5, 7, 2, 9, 6], - [9, 4, 6, 2, 0], - ]] - labels = _binary_3d_label_to_sparse_value( - [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], - [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]]) - - for class_id in xrange(10): - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=5, - expected=NAN, - class_id=class_id, - weights=[[0], [0]]) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=NAN, - class_id=class_id, - weights=[[0], [0]]) - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=5, - expected=NAN, - class_id=class_id, - weights=[[0, 0], [0, 0]]) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=NAN, - class_id=class_id, - weights=[[0, 0], [0, 0]]) - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=NAN, weights=[[0], [0]]) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=NAN, weights=[[0], [0]]) - self._test_streaming_sparse_recall_at_k( - predictions, labels, k=5, expected=NAN, weights=[[0, 0], [0, 0]]) - self._test_sparse_recall_at_top_k( - labels, top_k_predictions, expected=NAN, weights=[[0, 0], [0, 0]]) - - def test_3d_ignore_some(self): - predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], - [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], - [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], - [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] - top_k_predictions = [[ - [9, 4, 6, 2, 0], - [5, 7, 2, 9, 6], - ], [ - [5, 7, 2, 9, 6], - [9, 4, 6, 2, 0], - ]] - labels = _binary_3d_label_to_sparse_value( - [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], - [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]]) - - # Class 2: 2 labels, both correct. - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=5, - expected=2.0 / 2.0, - class_id=2, - weights=[[1], [0]]) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=2.0 / 2.0, - class_id=2, - weights=[[1], [0]]) - - # Class 2: 2 labels, both correct. - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=5, - expected=2.0 / 2.0, - class_id=2, - weights=[[0], [1]]) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=2.0 / 2.0, - class_id=2, - weights=[[0], [1]]) - - # Class 7: 1 label, correct. - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=5, - expected=1.0 / 1.0, - class_id=7, - weights=[[0], [1]]) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=1.0 / 1.0, - class_id=7, - weights=[[0], [1]]) - - # Class 7: 1 label, incorrect. - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=5, - expected=0.0 / 1.0, - class_id=7, - weights=[[1], [0]]) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=0.0 / 1.0, - class_id=7, - weights=[[1], [0]]) - - # Class 7: 2 labels, 1 correct. - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=5, - expected=1.0 / 2.0, - class_id=7, - weights=[[1, 0], [1, 0]]) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=1.0 / 2.0, - class_id=7, - weights=[[1, 0], [1, 0]]) - - # Class 7: No labels. - self._test_streaming_sparse_recall_at_k( - predictions, - labels, - k=5, - expected=NAN, - class_id=7, - weights=[[0, 1], [0, 1]]) - self._test_sparse_recall_at_top_k( - labels, - top_k_predictions, - expected=NAN, - class_id=7, - weights=[[0, 1], [0, 1]]) - - def test_sparse_tensor_value(self): - predictions = [[0.1, 0.3, 0.2, 0.4], [0.1, 0.2, 0.3, 0.4]] - labels = [[0, 0, 1, 0], [0, 0, 0, 1]] - expected_recall = 0.5 - with self.cached_session(): - _, recall = metrics.streaming_sparse_recall_at_k( - predictions=constant_op.constant(predictions, dtypes_lib.float32), - labels=_binary_2d_label_to_sparse_value(labels), - k=1) - - variables.variables_initializer(variables.local_variables()).run() - - self.assertEqual(expected_recall, recall.eval()) - - -class StreamingMeanAbsoluteErrorTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_mean_absolute_error( - predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) - _assert_metric_variables( - self, ('mean_absolute_error/count:0', 'mean_absolute_error/total:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_mean_absolute_error( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_mean_absolute_error( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_normal((10, 3), seed=1) - labels = random_ops.random_normal((10, 3), seed=2) - error, update_op = metrics.streaming_mean_absolute_error( - predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_error = error.eval() - for _ in range(10): - self.assertEqual(initial_error, error.eval()) - - def testSingleUpdateWithErrorAndWeights(self): - predictions = constant_op.constant( - [2, 4, 6, 8], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant( - [1, 3, 2, 3], shape=(1, 4), dtype=dtypes_lib.float32) - weights = constant_op.constant([0, 1, 0, 1], shape=(1, 4)) - - error, update_op = metrics.streaming_mean_absolute_error( - predictions, labels, weights) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(3, sess.run(update_op)) - self.assertEqual(3, error.eval()) - - -class StreamingMeanRelativeErrorTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_mean_relative_error( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - normalizer=array_ops.ones((10, 1))) - _assert_metric_variables( - self, ('mean_relative_error/count:0', 'mean_relative_error/total:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_mean_relative_error( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - normalizer=array_ops.ones((10, 1)), - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_mean_relative_error( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - normalizer=array_ops.ones((10, 1)), - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_normal((10, 3), seed=1) - labels = random_ops.random_normal((10, 3), seed=2) - normalizer = random_ops.random_normal((10, 3), seed=3) - error, update_op = metrics.streaming_mean_relative_error( - predictions, labels, normalizer) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_error = error.eval() - for _ in range(10): - self.assertEqual(initial_error, error.eval()) - - def testSingleUpdateNormalizedByLabels(self): - np_predictions = np.asarray([2, 4, 6, 8], dtype=np.float32) - np_labels = np.asarray([1, 3, 2, 3], dtype=np.float32) - expected_error = np.mean( - np.divide(np.absolute(np_predictions - np_labels), np_labels)) - - predictions = constant_op.constant( - np_predictions, shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant(np_labels, shape=(1, 4)) - - error, update_op = metrics.streaming_mean_relative_error( - predictions, labels, normalizer=labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(expected_error, sess.run(update_op)) - self.assertEqual(expected_error, error.eval()) - - def testSingleUpdateNormalizedByZeros(self): - np_predictions = np.asarray([2, 4, 6, 8], dtype=np.float32) - - predictions = constant_op.constant( - np_predictions, shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant( - [1, 3, 2, 3], shape=(1, 4), dtype=dtypes_lib.float32) - - error, update_op = metrics.streaming_mean_relative_error( - predictions, labels, normalizer=array_ops.zeros_like(labels)) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0.0, sess.run(update_op)) - self.assertEqual(0.0, error.eval()) - - -class StreamingMeanSquaredErrorTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_mean_squared_error( - predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) - _assert_metric_variables( - self, ('mean_squared_error/count:0', 'mean_squared_error/total:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_mean_squared_error( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_mean_squared_error( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_normal((10, 3), seed=1) - labels = random_ops.random_normal((10, 3), seed=2) - error, update_op = metrics.streaming_mean_squared_error(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_error = error.eval() - for _ in range(10): - self.assertEqual(initial_error, error.eval()) - - def testSingleUpdateZeroError(self): - predictions = array_ops.zeros((1, 3), dtype=dtypes_lib.float32) - labels = array_ops.zeros((1, 3), dtype=dtypes_lib.float32) - - error, update_op = metrics.streaming_mean_squared_error(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, sess.run(update_op)) - self.assertEqual(0, error.eval()) - - def testSingleUpdateWithError(self): - predictions = constant_op.constant( - [2, 4, 6], shape=(1, 3), dtype=dtypes_lib.float32) - labels = constant_op.constant( - [1, 3, 2], shape=(1, 3), dtype=dtypes_lib.float32) - - error, update_op = metrics.streaming_mean_squared_error(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(6, sess.run(update_op)) - self.assertEqual(6, error.eval()) - - def testSingleUpdateWithErrorAndWeights(self): - predictions = constant_op.constant( - [2, 4, 6, 8], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant( - [1, 3, 2, 3], shape=(1, 4), dtype=dtypes_lib.float32) - weights = constant_op.constant([0, 1, 0, 1], shape=(1, 4)) - - error, update_op = metrics.streaming_mean_squared_error( - predictions, labels, weights) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(13, sess.run(update_op)) - self.assertEqual(13, error.eval()) - - def testMultipleBatchesOfSizeOne(self): - with self.cached_session() as sess: - # Create the queue that populates the predictions. - preds_queue = data_flow_ops.FIFOQueue( - 2, dtypes=dtypes_lib.float32, shapes=(1, 3)) - _enqueue_vector(sess, preds_queue, [10, 8, 6]) - _enqueue_vector(sess, preds_queue, [-4, 3, -1]) - predictions = preds_queue.dequeue() - - # Create the queue that populates the labels. - labels_queue = data_flow_ops.FIFOQueue( - 2, dtypes=dtypes_lib.float32, shapes=(1, 3)) - _enqueue_vector(sess, labels_queue, [1, 3, 2]) - _enqueue_vector(sess, labels_queue, [2, 4, 6]) - labels = labels_queue.dequeue() - - error, update_op = metrics.streaming_mean_squared_error( - predictions, labels) - - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertAlmostEqual(208.0 / 6, sess.run(update_op), 5) - - self.assertAlmostEqual(208.0 / 6, error.eval(), 5) - - def testMetricsComputedConcurrently(self): - with self.cached_session() as sess: - # Create the queue that populates one set of predictions. - preds_queue0 = data_flow_ops.FIFOQueue( - 2, dtypes=dtypes_lib.float32, shapes=(1, 3)) - _enqueue_vector(sess, preds_queue0, [10, 8, 6]) - _enqueue_vector(sess, preds_queue0, [-4, 3, -1]) - predictions0 = preds_queue0.dequeue() - - # Create the queue that populates one set of predictions. - preds_queue1 = data_flow_ops.FIFOQueue( - 2, dtypes=dtypes_lib.float32, shapes=(1, 3)) - _enqueue_vector(sess, preds_queue1, [0, 1, 1]) - _enqueue_vector(sess, preds_queue1, [1, 1, 0]) - predictions1 = preds_queue1.dequeue() - - # Create the queue that populates one set of labels. - labels_queue0 = data_flow_ops.FIFOQueue( - 2, dtypes=dtypes_lib.float32, shapes=(1, 3)) - _enqueue_vector(sess, labels_queue0, [1, 3, 2]) - _enqueue_vector(sess, labels_queue0, [2, 4, 6]) - labels0 = labels_queue0.dequeue() - - # Create the queue that populates another set of labels. - labels_queue1 = data_flow_ops.FIFOQueue( - 2, dtypes=dtypes_lib.float32, shapes=(1, 3)) - _enqueue_vector(sess, labels_queue1, [-5, -3, -1]) - _enqueue_vector(sess, labels_queue1, [5, 4, 3]) - labels1 = labels_queue1.dequeue() - - mse0, update_op0 = metrics.streaming_mean_squared_error( - predictions0, labels0, name='msd0') - mse1, update_op1 = metrics.streaming_mean_squared_error( - predictions1, labels1, name='msd1') - - sess.run(variables.local_variables_initializer()) - sess.run([update_op0, update_op1]) - sess.run([update_op0, update_op1]) - - mse0, mse1 = sess.run([mse0, mse1]) - self.assertAlmostEqual(208.0 / 6, mse0, 5) - self.assertAlmostEqual(79.0 / 6, mse1, 5) - - def testMultipleMetricsOnMultipleBatchesOfSizeOne(self): - with self.cached_session() as sess: - # Create the queue that populates the predictions. - preds_queue = data_flow_ops.FIFOQueue( - 2, dtypes=dtypes_lib.float32, shapes=(1, 3)) - _enqueue_vector(sess, preds_queue, [10, 8, 6]) - _enqueue_vector(sess, preds_queue, [-4, 3, -1]) - predictions = preds_queue.dequeue() - - # Create the queue that populates the labels. - labels_queue = data_flow_ops.FIFOQueue( - 2, dtypes=dtypes_lib.float32, shapes=(1, 3)) - _enqueue_vector(sess, labels_queue, [1, 3, 2]) - _enqueue_vector(sess, labels_queue, [2, 4, 6]) - labels = labels_queue.dequeue() - - mae, ma_update_op = metrics.streaming_mean_absolute_error( - predictions, labels) - mse, ms_update_op = metrics.streaming_mean_squared_error( - predictions, labels) - - sess.run(variables.local_variables_initializer()) - sess.run([ma_update_op, ms_update_op]) - sess.run([ma_update_op, ms_update_op]) - - self.assertAlmostEqual(32.0 / 6, mae.eval(), 5) - self.assertAlmostEqual(208.0 / 6, mse.eval(), 5) - - -class StreamingRootMeanSquaredErrorTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_root_mean_squared_error( - predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) - _assert_metric_variables( - self, - ('root_mean_squared_error/count:0', 'root_mean_squared_error/total:0')) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_root_mean_squared_error( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_root_mean_squared_error( - predictions=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_normal((10, 3), seed=1) - labels = random_ops.random_normal((10, 3), seed=2) - error, update_op = metrics.streaming_root_mean_squared_error( - predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_error = error.eval() - for _ in range(10): - self.assertEqual(initial_error, error.eval()) - - def testSingleUpdateZeroError(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - 0.0, shape=(1, 3), dtype=dtypes_lib.float32) - labels = constant_op.constant(0.0, shape=(1, 3), dtype=dtypes_lib.float32) - - rmse, update_op = metrics.streaming_root_mean_squared_error( - predictions, labels) - - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, sess.run(update_op)) - - self.assertEqual(0, rmse.eval()) - - def testSingleUpdateWithError(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [2, 4, 6], shape=(1, 3), dtype=dtypes_lib.float32) - labels = constant_op.constant( - [1, 3, 2], shape=(1, 3), dtype=dtypes_lib.float32) - - rmse, update_op = metrics.streaming_root_mean_squared_error( - predictions, labels) - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(math.sqrt(6), update_op.eval(), 5) - self.assertAlmostEqual(math.sqrt(6), rmse.eval(), 5) - - def testSingleUpdateWithErrorAndWeights(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [2, 4, 6, 8], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant( - [1, 3, 2, 3], shape=(1, 4), dtype=dtypes_lib.float32) - weights = constant_op.constant([0, 1, 0, 1], shape=(1, 4)) - - rmse, update_op = metrics.streaming_root_mean_squared_error( - predictions, labels, weights) - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(math.sqrt(13), sess.run(update_op)) - - self.assertAlmostEqual(math.sqrt(13), rmse.eval(), 5) - - -class StreamingCovarianceTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_covariance( - predictions=math_ops.cast(math_ops.range(10), dtypes_lib.float32) + - array_ops.ones([10, 10]), - labels=(math_ops.cast(math_ops.range(10), dtypes_lib.float32) + - array_ops.ones([10, 10]))) - _assert_metric_variables(self, ( - 'covariance/comoment:0', - 'covariance/count:0', - 'covariance/mean_label:0', - 'covariance/mean_prediction:0', - )) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - cov, _ = metrics.streaming_covariance( - predictions=math_ops.cast(math_ops.range(10), dtypes_lib.float32) + - array_ops.ones([10, 10]), - labels=(math_ops.cast(math_ops.range(10), dtypes_lib.float32) + - array_ops.ones([10, 10])), - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [cov]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_covariance( - predictions=math_ops.cast(math_ops.range(10), dtypes_lib.float32) + - array_ops.ones([10, 10]), - labels=(math_ops.cast(math_ops.range(10), dtypes_lib.float32) + - array_ops.ones([10, 10])), - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - labels = random_ops.random_normal((10, 3), seed=2) - predictions = labels * 0.5 + random_ops.random_normal((10, 3), seed=1) * 0.5 - cov, update_op = metrics.streaming_covariance(predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_cov = cov.eval() - for _ in range(10): - self.assertEqual(initial_cov, cov.eval()) - - def testSingleUpdateIdentical(self): - with self.cached_session() as sess: - predictions = math_ops.cast(math_ops.range(10), dtypes_lib.float32) - labels = math_ops.cast(math_ops.range(10), dtypes_lib.float32) - - cov, update_op = metrics.streaming_covariance(predictions, labels) - - expected_cov = np.cov(np.arange(10), np.arange(10))[0, 1] - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(expected_cov, sess.run(update_op), 5) - self.assertAlmostEqual(expected_cov, cov.eval(), 5) - - def testSingleUpdateNonIdentical(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [2, 4, 6], shape=(1, 3), dtype=dtypes_lib.float32) - labels = constant_op.constant( - [1, 3, 2], shape=(1, 3), dtype=dtypes_lib.float32) - - cov, update_op = metrics.streaming_covariance(predictions, labels) - - expected_cov = np.cov([2, 4, 6], [1, 3, 2])[0, 1] - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(expected_cov, update_op.eval()) - self.assertAlmostEqual(expected_cov, cov.eval()) - - def testSingleUpdateWithErrorAndWeights(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [2, 4, 6, 8], shape=(1, 4), dtype=dtypes_lib.float32) - labels = constant_op.constant( - [1, 3, 2, 7], shape=(1, 4), dtype=dtypes_lib.float32) - weights = constant_op.constant( - [0, 1, 3, 1], shape=(1, 4), dtype=dtypes_lib.float32) - - cov, update_op = metrics.streaming_covariance( - predictions, labels, weights=weights) - - expected_cov = np.cov( - [2, 4, 6, 8], [1, 3, 2, 7], fweights=[0, 1, 3, 1])[0, 1] - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(expected_cov, sess.run(update_op)) - self.assertAlmostEqual(expected_cov, cov.eval()) - - def testMultiUpdateWithErrorNoWeights(self): - with self.cached_session() as sess: - np.random.seed(123) - n = 100 - predictions = np.random.randn(n) - labels = 0.5 * predictions + np.random.randn(n) - - stride = 10 - predictions_t = array_ops.placeholder(dtypes_lib.float32, [stride]) - labels_t = array_ops.placeholder(dtypes_lib.float32, [stride]) - - cov, update_op = metrics.streaming_covariance(predictions_t, labels_t) - - sess.run(variables.local_variables_initializer()) - prev_expected_cov = NAN - for i in range(n // stride): - feed_dict = { - predictions_t: predictions[stride * i:stride * (i + 1)], - labels_t: labels[stride * i:stride * (i + 1)] - } - self.assertEqual( - np.isnan(prev_expected_cov), - np.isnan(sess.run(cov, feed_dict=feed_dict))) - if not np.isnan(prev_expected_cov): - self.assertAlmostEqual(prev_expected_cov, - sess.run(cov, feed_dict=feed_dict), 5) - expected_cov = np.cov(predictions[:stride * (i + 1)], - labels[:stride * (i + 1)])[0, 1] - self.assertAlmostEqual(expected_cov, - sess.run(update_op, feed_dict=feed_dict), 5) - self.assertAlmostEqual(expected_cov, sess.run(cov, feed_dict=feed_dict), - 5) - prev_expected_cov = expected_cov - - def testMultiUpdateWithErrorAndWeights(self): - with self.cached_session() as sess: - np.random.seed(123) - n = 100 - predictions = np.random.randn(n) - labels = 0.5 * predictions + np.random.randn(n) - weights = np.tile(np.arange(n // 10), n // 10) - np.random.shuffle(weights) - - stride = 10 - predictions_t = array_ops.placeholder(dtypes_lib.float32, [stride]) - labels_t = array_ops.placeholder(dtypes_lib.float32, [stride]) - weights_t = array_ops.placeholder(dtypes_lib.float32, [stride]) - - cov, update_op = metrics.streaming_covariance( - predictions_t, labels_t, weights=weights_t) - - sess.run(variables.local_variables_initializer()) - prev_expected_cov = NAN - for i in range(n // stride): - feed_dict = { - predictions_t: predictions[stride * i:stride * (i + 1)], - labels_t: labels[stride * i:stride * (i + 1)], - weights_t: weights[stride * i:stride * (i + 1)] - } - self.assertEqual( - np.isnan(prev_expected_cov), - np.isnan(sess.run(cov, feed_dict=feed_dict))) - if not np.isnan(prev_expected_cov): - self.assertAlmostEqual(prev_expected_cov, - sess.run(cov, feed_dict=feed_dict), 5) - expected_cov = np.cov( - predictions[:stride * (i + 1)], - labels[:stride * (i + 1)], - fweights=weights[:stride * (i + 1)])[0, 1] - self.assertAlmostEqual(expected_cov, - sess.run(update_op, feed_dict=feed_dict), 5) - self.assertAlmostEqual(expected_cov, sess.run(cov, feed_dict=feed_dict), - 5) - prev_expected_cov = expected_cov - - -class StreamingPearsonRTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_pearson_correlation( - predictions=math_ops.cast(math_ops.range(10), dtypes_lib.float32) + - array_ops.ones([10, 10]), - labels=(math_ops.cast(math_ops.range(10), dtypes_lib.float32) + - array_ops.ones([10, 10]))) - _assert_metric_variables(self, ( - 'pearson_r/covariance/comoment:0', - 'pearson_r/covariance/count:0', - 'pearson_r/covariance/mean_label:0', - 'pearson_r/covariance/mean_prediction:0', - 'pearson_r/variance_labels/count:0', - 'pearson_r/variance_labels/comoment:0', - 'pearson_r/variance_labels/mean_label:0', - 'pearson_r/variance_labels/mean_prediction:0', - 'pearson_r/variance_predictions/comoment:0', - 'pearson_r/variance_predictions/count:0', - 'pearson_r/variance_predictions/mean_label:0', - 'pearson_r/variance_predictions/mean_prediction:0', - )) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - pearson_r, _ = metrics.streaming_pearson_correlation( - predictions=math_ops.cast(math_ops.range(10), dtypes_lib.float32) + - array_ops.ones([10, 10]), - labels=(math_ops.cast(math_ops.range(10), dtypes_lib.float32) + - array_ops.ones([10, 10])), - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [pearson_r]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_pearson_correlation( - predictions=math_ops.cast(math_ops.range(10), dtypes_lib.float32) + - array_ops.ones([10, 10]), - labels=(math_ops.cast(math_ops.range(10), dtypes_lib.float32) + - array_ops.ones([10, 10])), - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - labels = random_ops.random_normal((10, 3), seed=2) - predictions = labels * 0.5 + random_ops.random_normal((10, 3), seed=1) * 0.5 - pearson_r, update_op = metrics.streaming_pearson_correlation( - predictions, labels) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_r = pearson_r.eval() - for _ in range(10): - self.assertEqual(initial_r, pearson_r.eval()) - - def testSingleUpdateIdentical(self): - with self.cached_session() as sess: - predictions = math_ops.cast(math_ops.range(10), dtypes_lib.float32) - labels = math_ops.cast(math_ops.range(10), dtypes_lib.float32) - - pearson_r, update_op = metrics.streaming_pearson_correlation( - predictions, labels) - - expected_r = np.corrcoef(np.arange(10), np.arange(10))[0, 1] - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(expected_r, sess.run(update_op), 5) - self.assertAlmostEqual(expected_r, pearson_r.eval(), 5) - - def testSingleUpdateNonIdentical(self): - with self.cached_session() as sess: - predictions = constant_op.constant( - [2, 4, 6], shape=(1, 3), dtype=dtypes_lib.float32) - labels = constant_op.constant( - [1, 3, 2], shape=(1, 3), dtype=dtypes_lib.float32) - - pearson_r, update_op = metrics.streaming_pearson_correlation( - predictions, labels) - - expected_r = np.corrcoef([2, 4, 6], [1, 3, 2])[0, 1] - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(expected_r, update_op.eval()) - self.assertAlmostEqual(expected_r, pearson_r.eval()) - - def testSingleUpdateWithErrorAndWeights(self): - with self.cached_session() as sess: - predictions = np.array([2, 4, 6, 8]) - labels = np.array([1, 3, 2, 7]) - weights = np.array([0, 1, 3, 1]) - predictions_t = constant_op.constant( - predictions, shape=(1, 4), dtype=dtypes_lib.float32) - labels_t = constant_op.constant( - labels, shape=(1, 4), dtype=dtypes_lib.float32) - weights_t = constant_op.constant( - weights, shape=(1, 4), dtype=dtypes_lib.float32) - - pearson_r, update_op = metrics.streaming_pearson_correlation( - predictions_t, labels_t, weights=weights_t) - - cmat = np.cov(predictions, labels, fweights=weights) - expected_r = cmat[0, 1] / np.sqrt(cmat[0, 0] * cmat[1, 1]) - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(expected_r, sess.run(update_op)) - self.assertAlmostEqual(expected_r, pearson_r.eval()) - - def testMultiUpdateWithErrorNoWeights(self): - with self.cached_session() as sess: - np.random.seed(123) - n = 100 - predictions = np.random.randn(n) - labels = 0.5 * predictions + np.random.randn(n) - - stride = 10 - predictions_t = array_ops.placeholder(dtypes_lib.float32, [stride]) - labels_t = array_ops.placeholder(dtypes_lib.float32, [stride]) - - pearson_r, update_op = metrics.streaming_pearson_correlation( - predictions_t, labels_t) - - sess.run(variables.local_variables_initializer()) - prev_expected_r = NAN - for i in range(n // stride): - feed_dict = { - predictions_t: predictions[stride * i:stride * (i + 1)], - labels_t: labels[stride * i:stride * (i + 1)] - } - self.assertEqual( - np.isnan(prev_expected_r), - np.isnan(sess.run(pearson_r, feed_dict=feed_dict))) - if not np.isnan(prev_expected_r): - self.assertAlmostEqual(prev_expected_r, - sess.run(pearson_r, feed_dict=feed_dict), 5) - expected_r = np.corrcoef(predictions[:stride * (i + 1)], - labels[:stride * (i + 1)])[0, 1] - self.assertAlmostEqual(expected_r, - sess.run(update_op, feed_dict=feed_dict), 5) - self.assertAlmostEqual(expected_r, - sess.run(pearson_r, feed_dict=feed_dict), 5) - prev_expected_r = expected_r - - def testMultiUpdateWithErrorAndWeights(self): - with self.cached_session() as sess: - np.random.seed(123) - n = 100 - predictions = np.random.randn(n) - labels = 0.5 * predictions + np.random.randn(n) - weights = np.tile(np.arange(n // 10), n // 10) - np.random.shuffle(weights) - - stride = 10 - predictions_t = array_ops.placeholder(dtypes_lib.float32, [stride]) - labels_t = array_ops.placeholder(dtypes_lib.float32, [stride]) - weights_t = array_ops.placeholder(dtypes_lib.float32, [stride]) - - pearson_r, update_op = metrics.streaming_pearson_correlation( - predictions_t, labels_t, weights=weights_t) - - sess.run(variables.local_variables_initializer()) - prev_expected_r = NAN - for i in range(n // stride): - feed_dict = { - predictions_t: predictions[stride * i:stride * (i + 1)], - labels_t: labels[stride * i:stride * (i + 1)], - weights_t: weights[stride * i:stride * (i + 1)] - } - self.assertEqual( - np.isnan(prev_expected_r), - np.isnan(sess.run(pearson_r, feed_dict=feed_dict))) - if not np.isnan(prev_expected_r): - self.assertAlmostEqual(prev_expected_r, - sess.run(pearson_r, feed_dict=feed_dict), 5) - cmat = np.cov( - predictions[:stride * (i + 1)], - labels[:stride * (i + 1)], - fweights=weights[:stride * (i + 1)]) - expected_r = cmat[0, 1] / np.sqrt(cmat[0, 0] * cmat[1, 1]) - self.assertAlmostEqual(expected_r, - sess.run(update_op, feed_dict=feed_dict), 5) - self.assertAlmostEqual(expected_r, - sess.run(pearson_r, feed_dict=feed_dict), 5) - prev_expected_r = expected_r - - def testMultiUpdateWithErrorAndSingletonBatches(self): - with self.cached_session() as sess: - np.random.seed(123) - n = 100 - predictions = np.random.randn(n) - labels = 0.5 * predictions + np.random.randn(n) - stride = 10 - weights = (np.arange(n).reshape(n // stride, stride) % stride == 0) - for row in weights: - np.random.shuffle(row) - # Now, weights is one-hot by row - one item per batch has non-zero weight. - weights = weights.reshape((n,)) - - predictions_t = array_ops.placeholder(dtypes_lib.float32, [stride]) - labels_t = array_ops.placeholder(dtypes_lib.float32, [stride]) - weights_t = array_ops.placeholder(dtypes_lib.float32, [stride]) - - pearson_r, update_op = metrics.streaming_pearson_correlation( - predictions_t, labels_t, weights=weights_t) - - sess.run(variables.local_variables_initializer()) - for i in range(n // stride): - feed_dict = { - predictions_t: predictions[stride * i:stride * (i + 1)], - labels_t: labels[stride * i:stride * (i + 1)], - weights_t: weights[stride * i:stride * (i + 1)] - } - cmat = np.cov( - predictions[:stride * (i + 1)], - labels[:stride * (i + 1)], - fweights=weights[:stride * (i + 1)]) - expected_r = cmat[0, 1] / np.sqrt(cmat[0, 0] * cmat[1, 1]) - actual_r = sess.run(update_op, feed_dict=feed_dict) - self.assertEqual(np.isnan(expected_r), np.isnan(actual_r)) - self.assertEqual( - np.isnan(expected_r), - np.isnan(sess.run(pearson_r, feed_dict=feed_dict))) - if not np.isnan(expected_r): - self.assertAlmostEqual(expected_r, actual_r, 5) - self.assertAlmostEqual(expected_r, - sess.run(pearson_r, feed_dict=feed_dict), 5) - - -class StreamingMeanCosineDistanceTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_mean_cosine_distance( - predictions=array_ops.ones((10, 3)), - labels=array_ops.ones((10, 3)), - dim=1) - _assert_metric_variables(self, ( - 'mean_cosine_distance/count:0', - 'mean_cosine_distance/total:0', - )) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_mean_cosine_distance( - predictions=array_ops.ones((10, 3)), - labels=array_ops.ones((10, 3)), - dim=1, - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_mean_cosine_distance( - predictions=array_ops.ones((10, 3)), - labels=array_ops.ones((10, 3)), - dim=1, - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_normal((10, 3), seed=1) - labels = random_ops.random_normal((10, 3), seed=2) - error, update_op = metrics.streaming_mean_cosine_distance( - predictions, labels, dim=1) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_error = error.eval() - for _ in range(10): - self.assertEqual(initial_error, error.eval()) - - def testSingleUpdateZeroError(self): - np_labels = np.matrix(('1 0 0;' '0 0 1;' '0 1 0')) - - predictions = constant_op.constant( - np_labels, shape=(1, 3, 3), dtype=dtypes_lib.float32) - labels = constant_op.constant( - np_labels, shape=(1, 3, 3), dtype=dtypes_lib.float32) - - error, update_op = metrics.streaming_mean_cosine_distance( - predictions, labels, dim=2) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, sess.run(update_op)) - self.assertEqual(0, error.eval()) - - def testSingleUpdateWithError1(self): - np_labels = np.matrix(('1 0 0;' '0 0 1;' '0 1 0')) - np_predictions = np.matrix(('1 0 0;' '0 0 -1;' '1 0 0')) - - predictions = constant_op.constant( - np_predictions, shape=(3, 1, 3), dtype=dtypes_lib.float32) - labels = constant_op.constant( - np_labels, shape=(3, 1, 3), dtype=dtypes_lib.float32) - - error, update_op = metrics.streaming_mean_cosine_distance( - predictions, labels, dim=2) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(1, sess.run(update_op), 5) - self.assertAlmostEqual(1, error.eval(), 5) - - def testSingleUpdateWithError2(self): - np_predictions = np.matrix( - ('0.819031913261206 0.567041924552012 0.087465312324590;' - '-0.665139432070255 -0.739487441769973 -0.103671883216994;' - '0.707106781186548 -0.707106781186548 0')) - np_labels = np.matrix( - ('0.819031913261206 0.567041924552012 0.087465312324590;' - '0.665139432070255 0.739487441769973 0.103671883216994;' - '0.707106781186548 0.707106781186548 0')) - - predictions = constant_op.constant( - np_predictions, shape=(3, 1, 3), dtype=dtypes_lib.float32) - labels = constant_op.constant( - np_labels, shape=(3, 1, 3), dtype=dtypes_lib.float32) - error, update_op = metrics.streaming_mean_cosine_distance( - predictions, labels, dim=2) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(1.0, sess.run(update_op), 5) - self.assertAlmostEqual(1.0, error.eval(), 5) - - def testSingleUpdateWithErrorAndWeights1(self): - np_predictions = np.matrix(('1 0 0;' '0 0 -1;' '1 0 0')) - np_labels = np.matrix(('1 0 0;' '0 0 1;' '0 1 0')) - - predictions = constant_op.constant( - np_predictions, shape=(3, 1, 3), dtype=dtypes_lib.float32) - labels = constant_op.constant( - np_labels, shape=(3, 1, 3), dtype=dtypes_lib.float32) - weights = constant_op.constant( - [1, 0, 0], shape=(3, 1, 1), dtype=dtypes_lib.float32) - - error, update_op = metrics.streaming_mean_cosine_distance( - predictions, labels, dim=2, weights=weights) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, sess.run(update_op)) - self.assertEqual(0, error.eval()) - - def testSingleUpdateWithErrorAndWeights2(self): - np_predictions = np.matrix(('1 0 0;' '0 0 -1;' '1 0 0')) - np_labels = np.matrix(('1 0 0;' '0 0 1;' '0 1 0')) - - predictions = constant_op.constant( - np_predictions, shape=(3, 1, 3), dtype=dtypes_lib.float32) - labels = constant_op.constant( - np_labels, shape=(3, 1, 3), dtype=dtypes_lib.float32) - weights = constant_op.constant( - [0, 1, 1], shape=(3, 1, 1), dtype=dtypes_lib.float32) - - error, update_op = metrics.streaming_mean_cosine_distance( - predictions, labels, dim=2, weights=weights) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(1.5, update_op.eval()) - self.assertEqual(1.5, error.eval()) - - -class PcntBelowThreshTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_percentage_less(values=array_ops.ones((10,)), threshold=2) - _assert_metric_variables(self, ( - 'percentage_below_threshold/count:0', - 'percentage_below_threshold/total:0', - )) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.streaming_percentage_less( - values=array_ops.ones((10,)), - threshold=2, - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_percentage_less( - values=array_ops.ones((10,)), - threshold=2, - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testOneUpdate(self): - with self.cached_session() as sess: - values = constant_op.constant( - [2, 4, 6, 8], shape=(1, 4), dtype=dtypes_lib.float32) - - pcnt0, update_op0 = metrics.streaming_percentage_less( - values, 100, name='high') - pcnt1, update_op1 = metrics.streaming_percentage_less( - values, 7, name='medium') - pcnt2, update_op2 = metrics.streaming_percentage_less( - values, 1, name='low') - - sess.run(variables.local_variables_initializer()) - sess.run([update_op0, update_op1, update_op2]) - - pcnt0, pcnt1, pcnt2 = sess.run([pcnt0, pcnt1, pcnt2]) - self.assertAlmostEqual(1.0, pcnt0, 5) - self.assertAlmostEqual(0.75, pcnt1, 5) - self.assertAlmostEqual(0.0, pcnt2, 5) - - def testSomePresentOneUpdate(self): - with self.cached_session() as sess: - values = constant_op.constant( - [2, 4, 6, 8], shape=(1, 4), dtype=dtypes_lib.float32) - weights = constant_op.constant( - [1, 0, 0, 1], shape=(1, 4), dtype=dtypes_lib.float32) - - pcnt0, update_op0 = metrics.streaming_percentage_less( - values, 100, weights=weights, name='high') - pcnt1, update_op1 = metrics.streaming_percentage_less( - values, 7, weights=weights, name='medium') - pcnt2, update_op2 = metrics.streaming_percentage_less( - values, 1, weights=weights, name='low') - - sess.run(variables.local_variables_initializer()) - self.assertListEqual([1.0, 0.5, 0.0], - sess.run([update_op0, update_op1, update_op2])) - - pcnt0, pcnt1, pcnt2 = sess.run([pcnt0, pcnt1, pcnt2]) - self.assertAlmostEqual(1.0, pcnt0, 5) - self.assertAlmostEqual(0.5, pcnt1, 5) - self.assertAlmostEqual(0.0, pcnt2, 5) - - -class StreamingMeanIOUTest(test.TestCase): - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.streaming_mean_iou( - predictions=array_ops.ones([10, 1]), - labels=array_ops.ones([10, 1]), - num_classes=2) - _assert_metric_variables(self, ('mean_iou/total_confusion_matrix:0',)) - - def testMetricsCollections(self): - my_collection_name = '__metrics__' - mean_iou, _ = metrics.streaming_mean_iou( - predictions=array_ops.ones([10, 1]), - labels=array_ops.ones([10, 1]), - num_classes=2, - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean_iou]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_mean_iou( - predictions=array_ops.ones([10, 1]), - labels=array_ops.ones([10, 1]), - num_classes=2, - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testPredictionsAndLabelsOfDifferentSizeRaisesValueError(self): - predictions = array_ops.ones([10, 3]) - labels = array_ops.ones([10, 4]) - with self.assertRaises(ValueError): - metrics.streaming_mean_iou(predictions, labels, num_classes=2) - - def testLabelsAndWeightsOfDifferentSizeRaisesValueError(self): - predictions = array_ops.ones([10]) - labels = array_ops.ones([10]) - weights = array_ops.zeros([9]) - with self.assertRaises(ValueError): - metrics.streaming_mean_iou( - predictions, labels, num_classes=2, weights=weights) - - def testValueTensorIsIdempotent(self): - num_classes = 3 - predictions = random_ops.random_uniform( - [10], maxval=num_classes, dtype=dtypes_lib.int64, seed=1) - labels = random_ops.random_uniform( - [10], maxval=num_classes, dtype=dtypes_lib.int64, seed=2) - miou, update_op = metrics.streaming_mean_iou( - predictions, labels, num_classes=num_classes) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_miou = miou.eval() - for _ in range(10): - self.assertEqual(initial_miou, miou.eval()) - - def testMultipleUpdates(self): - num_classes = 3 - with self.cached_session() as sess: - # Create the queue that populates the predictions. - preds_queue = data_flow_ops.FIFOQueue( - 5, dtypes=dtypes_lib.int32, shapes=(1, 1)) - _enqueue_vector(sess, preds_queue, [0]) - _enqueue_vector(sess, preds_queue, [1]) - _enqueue_vector(sess, preds_queue, [2]) - _enqueue_vector(sess, preds_queue, [1]) - _enqueue_vector(sess, preds_queue, [0]) - predictions = preds_queue.dequeue() - - # Create the queue that populates the labels. - labels_queue = data_flow_ops.FIFOQueue( - 5, dtypes=dtypes_lib.int32, shapes=(1, 1)) - _enqueue_vector(sess, labels_queue, [0]) - _enqueue_vector(sess, labels_queue, [1]) - _enqueue_vector(sess, labels_queue, [1]) - _enqueue_vector(sess, labels_queue, [2]) - _enqueue_vector(sess, labels_queue, [1]) - labels = labels_queue.dequeue() - - miou, update_op = metrics.streaming_mean_iou(predictions, labels, - num_classes) - - sess.run(variables.local_variables_initializer()) - for _ in range(5): - sess.run(update_op) - desired_output = np.mean([1.0 / 2.0, 1.0 / 4.0, 0.]) - self.assertEqual(desired_output, miou.eval()) - - def testMultipleUpdatesWithWeights(self): - num_classes = 2 - with self.cached_session() as sess: - # Create the queue that populates the predictions. - preds_queue = data_flow_ops.FIFOQueue( - 6, dtypes=dtypes_lib.int32, shapes=(1, 1)) - _enqueue_vector(sess, preds_queue, [0]) - _enqueue_vector(sess, preds_queue, [1]) - _enqueue_vector(sess, preds_queue, [0]) - _enqueue_vector(sess, preds_queue, [1]) - _enqueue_vector(sess, preds_queue, [0]) - _enqueue_vector(sess, preds_queue, [1]) - predictions = preds_queue.dequeue() - - # Create the queue that populates the labels. - labels_queue = data_flow_ops.FIFOQueue( - 6, dtypes=dtypes_lib.int32, shapes=(1, 1)) - _enqueue_vector(sess, labels_queue, [0]) - _enqueue_vector(sess, labels_queue, [1]) - _enqueue_vector(sess, labels_queue, [1]) - _enqueue_vector(sess, labels_queue, [0]) - _enqueue_vector(sess, labels_queue, [0]) - _enqueue_vector(sess, labels_queue, [1]) - labels = labels_queue.dequeue() - - # Create the queue that populates the weights. - weights_queue = data_flow_ops.FIFOQueue( - 6, dtypes=dtypes_lib.float32, shapes=(1, 1)) - _enqueue_vector(sess, weights_queue, [1.0]) - _enqueue_vector(sess, weights_queue, [1.0]) - _enqueue_vector(sess, weights_queue, [1.0]) - _enqueue_vector(sess, weights_queue, [0.0]) - _enqueue_vector(sess, weights_queue, [1.0]) - _enqueue_vector(sess, weights_queue, [0.0]) - weights = weights_queue.dequeue() - - miou, update_op = metrics.streaming_mean_iou( - predictions, labels, num_classes, weights=weights) - - sess.run(variables.local_variables_initializer()) - for _ in range(6): - sess.run(update_op) - desired_output = np.mean([2.0 / 3.0, 1.0 / 2.0]) - self.assertAlmostEqual(desired_output, miou.eval()) - - def testMultipleUpdatesWithMissingClass(self): - # Test the case where there are no predicions and labels for - # one class, and thus there is one row and one column with - # zero entries in the confusion matrix. - num_classes = 3 - with self.cached_session() as sess: - # Create the queue that populates the predictions. - # There is no prediction for class 2. - preds_queue = data_flow_ops.FIFOQueue( - 5, dtypes=dtypes_lib.int32, shapes=(1, 1)) - _enqueue_vector(sess, preds_queue, [0]) - _enqueue_vector(sess, preds_queue, [1]) - _enqueue_vector(sess, preds_queue, [1]) - _enqueue_vector(sess, preds_queue, [1]) - _enqueue_vector(sess, preds_queue, [0]) - predictions = preds_queue.dequeue() - - # Create the queue that populates the labels. - # There is label for class 2. - labels_queue = data_flow_ops.FIFOQueue( - 5, dtypes=dtypes_lib.int32, shapes=(1, 1)) - _enqueue_vector(sess, labels_queue, [0]) - _enqueue_vector(sess, labels_queue, [1]) - _enqueue_vector(sess, labels_queue, [1]) - _enqueue_vector(sess, labels_queue, [0]) - _enqueue_vector(sess, labels_queue, [1]) - labels = labels_queue.dequeue() - - miou, update_op = metrics.streaming_mean_iou(predictions, labels, - num_classes) - - sess.run(variables.local_variables_initializer()) - for _ in range(5): - sess.run(update_op) - desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0]) - self.assertAlmostEqual(desired_output, miou.eval()) - - def testUpdateOpEvalIsAccumulatedConfusionMatrix(self): - predictions = array_ops.concat([ - constant_op.constant(0, shape=[5]), - constant_op.constant(1, shape=[5]) - ], 0) - labels = array_ops.concat([ - constant_op.constant(0, shape=[3]), - constant_op.constant(1, shape=[7]) - ], 0) - num_classes = 2 - with self.cached_session() as sess: - miou, update_op = metrics.streaming_mean_iou(predictions, labels, - num_classes) - sess.run(variables.local_variables_initializer()) - confusion_matrix = update_op.eval() - self.assertAllEqual([[3, 0], [2, 5]], confusion_matrix) - desired_miou = np.mean([3. / 5., 5. / 7.]) - self.assertAlmostEqual(desired_miou, miou.eval()) - - def testAllCorrect(self): - predictions = array_ops.zeros([40]) - labels = array_ops.zeros([40]) - num_classes = 1 - with self.cached_session() as sess: - miou, update_op = metrics.streaming_mean_iou(predictions, labels, - num_classes) - sess.run(variables.local_variables_initializer()) - self.assertEqual(40, update_op.eval()[0]) - self.assertEqual(1.0, miou.eval()) - - def testAllWrong(self): - predictions = array_ops.zeros([40]) - labels = array_ops.ones([40]) - num_classes = 2 - with self.cached_session() as sess: - miou, update_op = metrics.streaming_mean_iou(predictions, labels, - num_classes) - sess.run(variables.local_variables_initializer()) - self.assertAllEqual([[0, 0], [40, 0]], update_op.eval()) - self.assertEqual(0., miou.eval()) - - def testResultsWithSomeMissing(self): - predictions = array_ops.concat([ - constant_op.constant(0, shape=[5]), - constant_op.constant(1, shape=[5]) - ], 0) - labels = array_ops.concat([ - constant_op.constant(0, shape=[3]), - constant_op.constant(1, shape=[7]) - ], 0) - num_classes = 2 - weights = array_ops.concat([ - constant_op.constant(0, shape=[1]), - constant_op.constant(1, shape=[8]), - constant_op.constant(0, shape=[1]) - ], 0) - with self.cached_session() as sess: - miou, update_op = metrics.streaming_mean_iou( - predictions, labels, num_classes, weights=weights) - sess.run(variables.local_variables_initializer()) - self.assertAllEqual([[2, 0], [2, 4]], update_op.eval()) - desired_miou = np.mean([2. / 4., 4. / 6.]) - self.assertAlmostEqual(desired_miou, miou.eval()) - - def testMissingClassInLabels(self): - labels = constant_op.constant([[[0, 0, 1, 1, 0, 0], [1, 0, 0, 0, 0, 1]], - [[1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0]]]) - predictions = constant_op.constant( - [[[0, 0, 2, 1, 1, 0], [0, 1, 2, 2, 0, 1]], [[0, 0, 2, 1, 1, 1], - [1, 1, 2, 0, 0, 0]]]) - num_classes = 3 - with self.cached_session() as sess: - miou, update_op = metrics.streaming_mean_iou(predictions, labels, - num_classes) - sess.run(variables.local_variables_initializer()) - self.assertAllEqual([[7, 4, 3], [3, 5, 2], [0, 0, 0]], update_op.eval()) - self.assertAlmostEqual(1 / 3 * (7 / (7 + 3 + 7) + 5 / (5 + 4 + 5) + 0 / - (0 + 5 + 0)), miou.eval()) - - def testMissingClassOverallSmall(self): - labels = constant_op.constant([0]) - predictions = constant_op.constant([0]) - num_classes = 2 - with self.cached_session() as sess: - miou, update_op = metrics.streaming_mean_iou(predictions, labels, - num_classes) - sess.run(variables.local_variables_initializer()) - self.assertAllEqual([[1, 0], [0, 0]], update_op.eval()) - self.assertAlmostEqual(1, miou.eval()) - - def testMissingClassOverallLarge(self): - labels = constant_op.constant([[[0, 0, 1, 1, 0, 0], [1, 0, 0, 0, 0, 1]], - [[1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0]]]) - predictions = constant_op.constant( - [[[0, 0, 1, 1, 0, 0], [1, 1, 0, 0, 1, 1]], [[0, 0, 0, 1, 1, 1], - [1, 1, 1, 0, 0, 0]]]) - num_classes = 3 - with self.cached_session() as sess: - miou, update_op = metrics.streaming_mean_iou(predictions, labels, - num_classes) - sess.run(variables.local_variables_initializer()) - self.assertAllEqual([[9, 5, 0], [3, 7, 0], [0, 0, 0]], update_op.eval()) - self.assertAlmostEqual(1 / 2 * (9 / (9 + 3 + 5) + 7 / (7 + 5 + 3)), - miou.eval()) - - -class StreamingConcatTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - variable_scope.enable_resource_variables() - - def testVars(self): - metrics.streaming_concat(values=array_ops.ones((10,))) - _assert_metric_variables(self, ( - 'streaming_concat/array:0', - 'streaming_concat/size:0', - )) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - value, _ = metrics.streaming_concat( - values=array_ops.ones((10,)), metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [value]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.streaming_concat( - values=array_ops.ones((10,)), updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testNextArraySize(self): - next_array_size = metric_ops._next_array_size # pylint: disable=protected-access - with self.cached_session(): - self.assertEqual(next_array_size(2, growth_factor=2).eval(), 2) - self.assertEqual(next_array_size(3, growth_factor=2).eval(), 4) - self.assertEqual(next_array_size(4, growth_factor=2).eval(), 4) - self.assertEqual(next_array_size(5, growth_factor=2).eval(), 8) - self.assertEqual(next_array_size(6, growth_factor=2).eval(), 8) - - def testStreamingConcat(self): - with self.cached_session() as sess: - values = array_ops.placeholder(dtypes_lib.int32, [None]) - concatenated, update_op = metrics.streaming_concat(values) - sess.run(variables.local_variables_initializer()) - - self.assertAllEqual([], concatenated.eval()) - - sess.run([update_op], feed_dict={values: [0, 1, 2]}) - self.assertAllEqual([0, 1, 2], concatenated.eval()) - - sess.run([update_op], feed_dict={values: [3, 4]}) - self.assertAllEqual([0, 1, 2, 3, 4], concatenated.eval()) - - sess.run([update_op], feed_dict={values: [5, 6, 7, 8, 9]}) - self.assertAllEqual(np.arange(10), concatenated.eval()) - - def testStreamingConcatStringValues(self): - with self.cached_session() as sess: - values = array_ops.placeholder(dtypes_lib.string, [None]) - concatenated, update_op = metrics.streaming_concat(values) - sess.run(variables.local_variables_initializer()) - - self.assertItemsEqual([], concatenated.eval()) - - sess.run([update_op], feed_dict={values: ['a', 'b', 'c']}) - self.assertItemsEqual([b'a', b'b', b'c'], concatenated.eval()) - - sess.run([update_op], feed_dict={values: ['d', 'e']}) - self.assertItemsEqual([b'a', b'b', b'c', b'd', b'e'], concatenated.eval()) - - sess.run([update_op], feed_dict={values: ['f', 'g', 'h', 'i', 'j']}) - self.assertItemsEqual( - [b'a', b'b', b'c', b'd', b'e', b'f', b'g', b'h', b'i', b'j'], - concatenated.eval()) - - def testStreamingConcatMaxSize(self): - with self.cached_session() as sess: - values = math_ops.range(3) - concatenated, update_op = metrics.streaming_concat(values, max_size=5) - sess.run(variables.local_variables_initializer()) - - self.assertAllEqual([], concatenated.eval()) - - sess.run([update_op]) - self.assertAllEqual([0, 1, 2], concatenated.eval()) - - sess.run([update_op]) - self.assertAllEqual([0, 1, 2, 0, 1], concatenated.eval()) - - sess.run([update_op]) - self.assertAllEqual([0, 1, 2, 0, 1], concatenated.eval()) - - def testStreamingConcat2D(self): - with self.cached_session() as sess: - values = array_ops.reshape(math_ops.range(3), (3, 1)) - concatenated, update_op = metrics.streaming_concat(values, axis=-1) - sess.run(variables.local_variables_initializer()) - for _ in range(10): - sess.run([update_op]) - self.assertAllEqual([[0] * 10, [1] * 10, [2] * 10], concatenated.eval()) - - def testStreamingConcatErrors(self): - with self.assertRaises(ValueError): - metrics.streaming_concat(array_ops.placeholder(dtypes_lib.float32)) - - values = array_ops.zeros((2, 3)) - with self.assertRaises(ValueError): - metrics.streaming_concat(values, axis=-3, max_size=3) - with self.assertRaises(ValueError): - metrics.streaming_concat(values, axis=2, max_size=3) - - with self.assertRaises(ValueError): - metrics.streaming_concat( - array_ops.placeholder(dtypes_lib.float32, [None, None])) - - def testStreamingConcatReset(self): - with self.cached_session() as sess: - values = array_ops.placeholder(dtypes_lib.int32, [None]) - concatenated, update_op = metrics.streaming_concat(values) - sess.run(variables.local_variables_initializer()) - - self.assertAllEqual([], concatenated.eval()) - - sess.run([update_op], feed_dict={values: [0, 1, 2]}) - self.assertAllEqual([0, 1, 2], concatenated.eval()) - - sess.run(variables.local_variables_initializer()) - - sess.run([update_op], feed_dict={values: [3, 4]}) - self.assertAllEqual([3, 4], concatenated.eval()) - - -class AggregateMetricsTest(test.TestCase): - - def testAggregateNoMetricsRaisesValueError(self): - with self.assertRaises(ValueError): - metrics.aggregate_metrics() - - def testAggregateSingleMetricReturnsOneItemLists(self): - values = array_ops.ones((10, 4)) - value_tensors, update_ops = metrics.aggregate_metrics( - metrics.streaming_mean(values)) - self.assertEqual(len(value_tensors), 1) - self.assertEqual(len(update_ops), 1) - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(1, update_ops[0].eval()) - self.assertEqual(1, value_tensors[0].eval()) - - def testAggregateMultipleMetricsReturnsListsInOrder(self): - predictions = array_ops.ones((10, 4)) - labels = array_ops.ones((10, 4)) * 3 - value_tensors, update_ops = metrics.aggregate_metrics( - metrics.streaming_mean_absolute_error(predictions, labels), - metrics.streaming_mean_squared_error(predictions, labels)) - self.assertEqual(len(value_tensors), 2) - self.assertEqual(len(update_ops), 2) - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(2, update_ops[0].eval()) - self.assertEqual(4, update_ops[1].eval()) - self.assertEqual(2, value_tensors[0].eval()) - self.assertEqual(4, value_tensors[1].eval()) - - -class AggregateMetricMapTest(test.TestCase): - - def testAggregateMultipleMetricsReturnsListsInOrder(self): - predictions = array_ops.ones((10, 4)) - labels = array_ops.ones((10, 4)) * 3 - names_to_values, names_to_updates = metrics.aggregate_metric_map({ - 'm1': metrics.streaming_mean_absolute_error(predictions, labels), - 'm2': metrics.streaming_mean_squared_error(predictions, labels), - }) - - self.assertEqual(2, len(names_to_values)) - self.assertEqual(2, len(names_to_updates)) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(2, names_to_updates['m1'].eval()) - self.assertEqual(4, names_to_updates['m2'].eval()) - self.assertEqual(2, names_to_values['m1'].eval()) - self.assertEqual(4, names_to_values['m2'].eval()) - - -class CountTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testVars(self): - metrics.count(array_ops.ones([4, 3])) - _assert_metric_variables(self, ['count/count:0']) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - mean, _ = metrics.count( - array_ops.ones([4, 3]), metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [mean]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.count( - array_ops.ones([4, 3]), updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testReturnType(self): - c, op = metrics.count(array_ops.ones([4, 3])) - self.assertTrue(isinstance(c, ops.Tensor)) - self.assertTrue(isinstance(op, ops.Operation) or isinstance(op, ops.Tensor)) - - def testBasic(self): - with self.cached_session() as sess: - values_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, values_queue, [0, 1]) - _enqueue_vector(sess, values_queue, [-4.2, 9.1]) - _enqueue_vector(sess, values_queue, [6.5, 0]) - _enqueue_vector(sess, values_queue, [-3.2, 4.0]) - values = values_queue.dequeue() - - result, update_op = metrics.count(values) - - sess.run(variables.local_variables_initializer()) - for _ in range(4): - sess.run(update_op) - self.assertAlmostEqual(8.0, sess.run(result), 5) - - def testUpdateOpsReturnsCurrentValue(self): - with self.cached_session() as sess: - values_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, values_queue, [0, 1]) - _enqueue_vector(sess, values_queue, [-4.2, 9.1]) - _enqueue_vector(sess, values_queue, [6.5, 0]) - _enqueue_vector(sess, values_queue, [-3.2, 4.0]) - values = values_queue.dequeue() - - result, update_op = metrics.count(values) - - sess.run(variables.local_variables_initializer()) - - self.assertAlmostEqual(2.0, sess.run(update_op), 5) - self.assertAlmostEqual(4.0, sess.run(update_op), 5) - self.assertAlmostEqual(6.0, sess.run(update_op), 5) - self.assertAlmostEqual(8.0, sess.run(update_op), 5) - - self.assertAlmostEqual(8.0, sess.run(result), 5) - - def test1dWeightedValues(self): - with self.cached_session() as sess: - # Create the queue that populates the values. - values_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, values_queue, [0, 1]) - _enqueue_vector(sess, values_queue, [-4.2, 9.1]) - _enqueue_vector(sess, values_queue, [6.5, 0]) - _enqueue_vector(sess, values_queue, [-3.2, 4.0]) - values = values_queue.dequeue() - - # Create the queue that populates the weighted labels. - weights_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 1)) - _enqueue_vector(sess, weights_queue, [0.5]) - _enqueue_vector(sess, weights_queue, [0]) - _enqueue_vector(sess, weights_queue, [0]) - _enqueue_vector(sess, weights_queue, [1.2]) - weights = weights_queue.dequeue() - - result, update_op = metrics.count(values, weights) - - variables.local_variables_initializer().run() - for _ in range(4): - update_op.eval() - self.assertAlmostEqual(3.4, result.eval(), 5) - - def test1dWeightedValues_placeholders(self): - with self.cached_session() as sess: - # Create the queue that populates the values. - feed_values = ((0, 1), (-4.2, 9.1), (6.5, 0), (-3.2, 4.0)) - values = array_ops.placeholder(dtype=dtypes_lib.float32) - - # Create the queue that populates the weighted labels. - weights_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1,)) - _enqueue_vector(sess, weights_queue, 0.5, shape=(1,)) - _enqueue_vector(sess, weights_queue, 0, shape=(1,)) - _enqueue_vector(sess, weights_queue, 0, shape=(1,)) - _enqueue_vector(sess, weights_queue, 1.2, shape=(1,)) - weights = weights_queue.dequeue() - - result, update_op = metrics.count(values, weights) - - variables.local_variables_initializer().run() - for i in range(4): - update_op.eval(feed_dict={values: feed_values[i]}) - self.assertAlmostEqual(3.4, result.eval(), 5) - - def test2dWeightedValues(self): - with self.cached_session() as sess: - # Create the queue that populates the values. - values_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, values_queue, [0, 1]) - _enqueue_vector(sess, values_queue, [-4.2, 9.1]) - _enqueue_vector(sess, values_queue, [6.5, 0]) - _enqueue_vector(sess, values_queue, [-3.2, 4.0]) - values = values_queue.dequeue() - - # Create the queue that populates the weighted labels. - weights_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) - _enqueue_vector(sess, weights_queue, [1.1, 1]) - _enqueue_vector(sess, weights_queue, [1, 0]) - _enqueue_vector(sess, weights_queue, [0, 1]) - _enqueue_vector(sess, weights_queue, [0, 0]) - weights = weights_queue.dequeue() - - result, update_op = metrics.count(values, weights) - - variables.local_variables_initializer().run() - for _ in range(4): - update_op.eval() - self.assertAlmostEqual(4.1, result.eval(), 5) - - def test2dWeightedValues_placeholders(self): - with self.cached_session() as sess: - # Create the queue that populates the values. - feed_values = ((0, 1), (-4.2, 9.1), (6.5, 0), (-3.2, 4.0)) - values = array_ops.placeholder(dtype=dtypes_lib.float32) - - # Create the queue that populates the weighted labels. - weights_queue = data_flow_ops.FIFOQueue( - 4, dtypes=dtypes_lib.float32, shapes=(2,)) - _enqueue_vector(sess, weights_queue, [1.1, 1], shape=(2,)) - _enqueue_vector(sess, weights_queue, [1, 0], shape=(2,)) - _enqueue_vector(sess, weights_queue, [0, 1], shape=(2,)) - _enqueue_vector(sess, weights_queue, [0, 0], shape=(2,)) - weights = weights_queue.dequeue() - - result, update_op = metrics.count(values, weights) - - variables.local_variables_initializer().run() - for i in range(4): - update_op.eval(feed_dict={values: feed_values[i]}) - self.assertAlmostEqual(4.1, result.eval(), 5) - - -class CohenKappaTest(test.TestCase): - - def _confusion_matrix_to_samples(self, confusion_matrix): - x, y = confusion_matrix.shape - pairs = [] - for label in range(x): - for feature in range(y): - pairs += [label, feature] * confusion_matrix[label, feature] - pairs = np.array(pairs).reshape((-1, 2)) - return pairs[:, 0], pairs[:, 1] - - def setUp(self): - np.random.seed(1) - ops.reset_default_graph() - - def testVars(self): - metrics.cohen_kappa( - predictions_idx=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - num_classes=2) - _assert_metric_variables(self, ( - 'cohen_kappa/po:0', - 'cohen_kappa/pe_row:0', - 'cohen_kappa/pe_col:0', - )) - - def testMetricsCollection(self): - my_collection_name = '__metrics__' - kappa, _ = metrics.cohen_kappa( - predictions_idx=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - num_classes=2, - metrics_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [kappa]) - - def testUpdatesCollection(self): - my_collection_name = '__updates__' - _, update_op = metrics.cohen_kappa( - predictions_idx=array_ops.ones((10, 1)), - labels=array_ops.ones((10, 1)), - num_classes=2, - updates_collections=[my_collection_name]) - self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) - - def testValueTensorIsIdempotent(self): - predictions = random_ops.random_uniform( - (10, 1), maxval=3, dtype=dtypes_lib.int64, seed=1) - labels = random_ops.random_uniform( - (10, 1), maxval=3, dtype=dtypes_lib.int64, seed=2) - kappa, update_op = metrics.cohen_kappa(labels, predictions, 3) - - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - # Run several updates. - for _ in range(10): - sess.run(update_op) - - # Then verify idempotency. - initial_kappa = kappa.eval() - for _ in range(10): - self.assertAlmostEqual(initial_kappa, kappa.eval(), 5) - - def testBasic(self): - confusion_matrix = np.array([[9, 3, 1], [4, 8, 2], [2, 1, 6]]) - # overall total = 36 - # po = [9, 8, 6], sum(po) = 23 - # pe_row = [15, 12, 9], pe_col = [13, 14, 9], so pe = [5.42, 4.67, 2.25] - # finally, kappa = (sum(po) - sum(pe)) / (N - sum(pe)) - # = (23 - 12.34) / (36 - 12.34) - # = 0.45 - # see: http://psych.unl.edu/psycrs/handcomp/hckappa.PDF - expect = 0.45 - labels, predictions = self._confusion_matrix_to_samples(confusion_matrix) - - dtypes = [dtypes_lib.int16, dtypes_lib.int32, dtypes_lib.int64] - shapes = [ - (len(labels,)), # 1-dim - (len(labels), 1) - ] # 2-dim - weights = [None, np.ones_like(labels)] - - for dtype in dtypes: - for shape in shapes: - for weight in weights: - with self.cached_session() as sess: - predictions_tensor = constant_op.constant( - np.reshape(predictions, shape), dtype=dtype) - labels_tensor = constant_op.constant( - np.reshape(labels, shape), dtype=dtype) - kappa, update_op = metrics.cohen_kappa( - labels_tensor, predictions_tensor, 3, weights=weight) - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(expect, sess.run(update_op), 2) - self.assertAlmostEqual(expect, kappa.eval(), 2) - - def testAllCorrect(self): - inputs = np.arange(0, 100) % 4 - # confusion matrix - # [[25, 0, 0], - # [0, 25, 0], - # [0, 0, 25]] - # Calculated by v0.19: sklearn.metrics.cohen_kappa_score(inputs, inputs) - expect = 1.0 - - with self.cached_session() as sess: - predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32) - labels = constant_op.constant(inputs) - kappa, update_op = metrics.cohen_kappa(labels, predictions, 4) - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(expect, sess.run(update_op), 5) - self.assertAlmostEqual(expect, kappa.eval(), 5) - - def testAllIncorrect(self): - labels = np.arange(0, 100) % 4 - predictions = (labels + 1) % 4 - # confusion matrix - # [[0, 25, 0], - # [0, 0, 25], - # [25, 0, 0]] - # Calculated by v0.19: sklearn.metrics.cohen_kappa_score(labels, predictions) - expect = -0.333333333333 - - with self.cached_session() as sess: - predictions = constant_op.constant(predictions, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels) - kappa, update_op = metrics.cohen_kappa(labels, predictions, 4) - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(expect, sess.run(update_op), 5) - self.assertAlmostEqual(expect, kappa.eval(), 5) - - def testWeighted(self): - confusion_matrix = np.array([[9, 3, 1], [4, 8, 2], [2, 1, 6]]) - labels, predictions = self._confusion_matrix_to_samples(confusion_matrix) - num_samples = np.sum(confusion_matrix, dtype=np.int32) - weights = (np.arange(0, num_samples) % 5) / 5.0 - # Calculated by v0.19: sklearn.metrics.cohen_kappa_score( - # labels, predictions, sample_weight=weights) - expect = 0.453466583385 - - with self.cached_session() as sess: - predictions = constant_op.constant(predictions, dtype=dtypes_lib.float32) - labels = constant_op.constant(labels) - kappa, update_op = metrics.cohen_kappa( - labels, predictions, 4, weights=weights) - - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(expect, sess.run(update_op), 5) - self.assertAlmostEqual(expect, kappa.eval(), 5) - - def testWithMultipleUpdates(self): - confusion_matrix = np.array([[90, 30, 10, 20], [40, 80, 20, 30], - [20, 10, 60, 35], [15, 25, 30, 25]]) - labels, predictions = self._confusion_matrix_to_samples(confusion_matrix) - num_samples = np.sum(confusion_matrix, dtype=np.int32) - weights = (np.arange(0, num_samples) % 5) / 5.0 - num_classes = confusion_matrix.shape[0] - - batch_size = num_samples // 10 - predictions_t = array_ops.placeholder( - dtypes_lib.float32, shape=(batch_size,)) - labels_t = array_ops.placeholder(dtypes_lib.int32, shape=(batch_size,)) - weights_t = array_ops.placeholder(dtypes_lib.float32, shape=(batch_size,)) - kappa, update_op = metrics.cohen_kappa( - labels_t, predictions_t, num_classes, weights=weights_t) - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - - for idx in range(0, num_samples, batch_size): - batch_start, batch_end = idx, idx + batch_size - sess.run( - update_op, - feed_dict={ - labels_t: labels[batch_start:batch_end], - predictions_t: predictions[batch_start:batch_end], - weights_t: weights[batch_start:batch_end] - }) - # Calculated by v0.19: sklearn.metrics.cohen_kappa_score( - # labels_np, predictions_np, sample_weight=weights_np) - expect = 0.289965397924 - self.assertAlmostEqual(expect, kappa.eval(), 5) - - def testInvalidNumClasses(self): - predictions = array_ops.placeholder(dtypes_lib.float32, shape=(4, 1)) - labels = array_ops.placeholder(dtypes_lib.int32, shape=(4, 1)) - with self.assertRaisesRegexp(ValueError, 'num_classes'): - metrics.cohen_kappa(labels, predictions, 1) - - def testInvalidDimension(self): - predictions = array_ops.placeholder(dtypes_lib.float32, shape=(4, 1)) - invalid_labels = array_ops.placeholder(dtypes_lib.int32, shape=(4, 2)) - with self.assertRaises(ValueError): - metrics.cohen_kappa(invalid_labels, predictions, 3) - - invalid_predictions = array_ops.placeholder( - dtypes_lib.float32, shape=(4, 2)) - labels = array_ops.placeholder(dtypes_lib.int32, shape=(4, 1)) - with self.assertRaises(ValueError): - metrics.cohen_kappa(labels, invalid_predictions, 3) - - def testConditionalPackingOptimization(self): - placeholder = array_ops.placeholder(dtypes_lib.float32, [None]) - values, update_op = metric_ops.streaming_concat(placeholder) - with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - for feed in range(10): - sess.run(update_op, feed_dict={placeholder: [feed]}) - print(sess.run(values)) - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/metrics/python/ops/set_ops.py b/tensorflow/contrib/metrics/python/ops/set_ops.py deleted file mode 100644 index bca8334110c..00000000000 --- a/tensorflow/contrib/metrics/python/ops/set_ops.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python layer for set_ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import sets - -set_size = sets.set_size - -set_intersection = sets.set_intersection - -set_difference = sets.set_difference - -set_union = sets.set_union diff --git a/tensorflow/contrib/mixed_precision/BUILD b/tensorflow/contrib/mixed_precision/BUILD deleted file mode 100644 index 5b41eed73f3..00000000000 --- a/tensorflow/contrib/mixed_precision/BUILD +++ /dev/null @@ -1,31 +0,0 @@ -# Mixed precision training optimizers - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -filegroup( - name = "all_files", - srcs = glob( - ["**/*"], - exclude = [ - "**/METADATA", - "**/OWNERS", - ], - ), - visibility = ["//tensorflow:__subpackages__"], -) - -py_library( - name = "mixed_precision", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/contrib/mixed_precision/python:loss_scale_manager", - "//tensorflow/contrib/mixed_precision/python:loss_scale_optimizer", - ], -) diff --git a/tensorflow/contrib/mixed_precision/__init__.py b/tensorflow/contrib/mixed_precision/__init__.py deleted file mode 100644 index 43e98cdda09..00000000000 --- a/tensorflow/contrib/mixed_precision/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# mixed_precisiond under the License is mixed_precisiond on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Library for mixed precision training.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.mixed_precision.python.loss_scale_manager import * -from tensorflow.contrib.mixed_precision.python.loss_scale_optimizer import * - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - "LossScaleManager", - "FixedLossScaleManager", - "ExponentialUpdateLossScaleManager", - "LossScaleOptimizer", -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/mixed_precision/python/BUILD b/tensorflow/contrib/mixed_precision/python/BUILD deleted file mode 100644 index a63268b8412..00000000000 --- a/tensorflow/contrib/mixed_precision/python/BUILD +++ /dev/null @@ -1,75 +0,0 @@ -# Mixed precision training optimizers - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "loss_scale_manager", - srcs = ["loss_scale_manager.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:state_ops", - "//tensorflow/python:variable_scope", - ], -) - -py_test( - name = "loss_scale_manager_test", - size = "small", - srcs = ["loss_scale_manager_test.py"], - python_version = "PY2", - deps = [ - ":loss_scale_manager", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python/data/ops:dataset_ops", - "//third_party/py/numpy", - ], -) - -py_library( - name = "loss_scale_optimizer", - srcs = ["loss_scale_optimizer.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":loss_scale_manager", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:util", - ], -) - -py_test( - name = "loss_scale_optimizer_test", - size = "small", - srcs = ["loss_scale_optimizer_test.py"], - python_version = "PY2", - deps = [ - ":loss_scale_optimizer", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/mixed_precision/python/loss_scale_manager.py b/tensorflow/contrib/mixed_precision/python/loss_scale_manager.py deleted file mode 100644 index eba505881fb..00000000000 --- a/tensorflow/contrib/mixed_precision/python/loss_scale_manager.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""LossScaleManager classes for mixed precision training.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc -import six - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_control_flow_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope - - -@six.add_metaclass(abc.ABCMeta) -class LossScaleManager(object): - """Abstract loss scale manager class. - - Loss scale managers with a different strategy should subclass this class. - Loss scaling is a process that: - - 1) Applies a multiplier on the loss before computing gradients, and - 2) Applies the reciprocal of the multiplier on the gradients before they are - applied on variables. - - This class is used together with - `tf.contrib.mixed_precision.LossScaleOptimizer` for mixed precision training - (float32 variables and float16 ops) on Nvidia GPUs in order to achieve the - same model quality as single precision training, with the benefits of - potential higher throughput. - - See `tf.contrib.mixed_precision.LossScaleOptimizer` for more details. - """ - - @abc.abstractmethod - def get_loss_scale(self): - """Returns the loss scale as a scalar `float32` tensor.""" - pass - - @abc.abstractmethod - def update_loss_scale(self, finite_grads): - """Updates loss scale based on if gradients are finite in current step. - - Args: - finite_grads: bool scalar tensor indicating if all gradients are - finite (i.e., not inf or nan). - - Returns: - An op, when executed updates the loss scale. If eager execution is - enabled, does not return anything. - """ - del finite_grads - return - - -class FixedLossScaleManager(LossScaleManager): - """Loss scale manager with a fixed loss scale. - - The loss scale is not updated for the lifetime of the class. - """ - - def __init__(self, loss_scale): - """Creates the fixed loss scale manager. - - Args: - loss_scale: A Python float. Its ideal value varies depending on models to - run. Choosing a too small loss_scale might affect model quality; a too - big loss_scale might cause inf or nan. There is no single right - loss_scale to apply. There is no harm choosing a relatively big number - as long as no nan or inf is encountered in training. - - Raises: - ValueError: If loss_scale is less than 1. - """ - if loss_scale < 1: - raise ValueError("loss scale must be at least 1.") - self._loss_scale = ops.convert_to_tensor(loss_scale, dtype=dtypes.float32) - - def get_loss_scale(self): - return self._loss_scale - - def update_loss_scale(self, finite_grads): - del finite_grads - return gen_control_flow_ops.no_op() - - -class ExponentialUpdateLossScaleManager(LossScaleManager): - """Loss scale manager uses an exponential update strategy. - - In general, the strategy increases loss scale by a greater-than-one factor - after encountering a consecutive series of steps with finite gradients; - Similarly, it decreases the loss scale by a factor when the accumulated number - of steps with non-finite (nan or inf) gradients are met. An update is not - applied if its result is less than 1 or overflows the float32 dynamic range. - - The number of finite and non-finite steps are cleared every time the loss - scale is changed. The condition to decrease the loss scale is looser than to - increase it since the former does not require the steps to be consecutive. - """ - - def __init__(self, - init_loss_scale, - incr_every_n_steps, - decr_every_n_nan_or_inf=2, - incr_ratio=2, - decr_ratio=0.8): - """Constructor of exponential-update loss scale manager. - - Args: - init_loss_scale: A Python float. The loss scale to use at the beginning. - incr_every_n_steps: Increases loss scale every n consecutive steps with - finite gradients. - decr_every_n_nan_or_inf: Decreases loss scale every n accumulated steps - with nan or inf gradients. - incr_ratio: The multiplier to use when increasing the loss scale. - decr_ratio: The less-than-one-multiplier to use when decreasing the loss - scale. - """ - self._incr_every_n_steps = incr_every_n_steps - self._decr_every_n_nan_or_inf = decr_every_n_nan_or_inf - self._incr_ratio = incr_ratio - self._decr_ratio = decr_ratio - self._loss_scale = variable_scope.variable( - name="loss_scale", - initial_value=ops.convert_to_tensor(init_loss_scale, dtypes.float32), - dtype=dtypes.float32, - trainable=False) - self._num_good_steps = variable_scope.variable( - name="good_steps", initial_value=0, dtype=dtypes.int32, trainable=False) - self._num_bad_steps = variable_scope.variable( - name="bad_steps", initial_value=0, dtype=dtypes.int32, trainable=False) - - def _reset_stats(self): - return control_flow_ops.group( - state_ops.assign(self._num_good_steps, 0), - state_ops.assign(self._num_bad_steps, 0)) - - def get_loss_scale(self): - """Returns the loss scale.""" - return self._loss_scale - - def update_loss_scale(self, finite_grads): - """Updates loss scale based on if gradients are finite in current step.""" - - def update_if_finite_grads(): - """Branch function when grads are all finite.""" - - def incr_loss_scale(): - new_loss_scale = control_flow_ops.cond( - gen_math_ops.is_finite(self._loss_scale * self._incr_ratio), - lambda: self._loss_scale * self._incr_ratio, - lambda: self._loss_scale) - update_op = state_ops.assign(self._loss_scale, new_loss_scale) - # When loss_scale is updated, both good and bad steps are reset. - return control_flow_ops.group(update_op, self._reset_stats()) - - return control_flow_ops.cond( - self._num_good_steps + 1 >= self._incr_every_n_steps, - incr_loss_scale, - lambda: state_ops.assign_add(self._num_good_steps, 1).op) - - def update_if_not_finite_grads(): - """Branch function when any grad is not finite.""" - - def decr_loss_scale(): - update_op = state_ops.assign( - self._loss_scale, - gen_math_ops.maximum(1., self._loss_scale * self._decr_ratio)) - # When loss_scale is updated, both good and bad steps are reset. - return control_flow_ops.group(update_op, self._reset_stats()) - - def just_update_steps(): - # When bad_steps is incremented, good_step is reset. - return control_flow_ops.group( - state_ops.assign_add(self._num_bad_steps, 1), - state_ops.assign(self._num_good_steps, 0)) - - return control_flow_ops.cond( - self._num_bad_steps + 1 >= self._decr_every_n_nan_or_inf, - decr_loss_scale, just_update_steps) - - return control_flow_ops.cond(finite_grads, update_if_finite_grads, - update_if_not_finite_grads) diff --git a/tensorflow/contrib/mixed_precision/python/loss_scale_manager_test.py b/tensorflow/contrib/mixed_precision/python/loss_scale_manager_test.py deleted file mode 100644 index c922d0cd11f..00000000000 --- a/tensorflow/contrib/mixed_precision/python/loss_scale_manager_test.py +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for LossScaleManager classes..""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.mixed_precision.python import loss_scale_manager as lsm_lib -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.eager import context -from tensorflow.python.framework import test_util -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def _GetExampleIter(inputs): - dataset = dataset_ops.Dataset.from_tensor_slices(inputs) - return dataset_ops.make_one_shot_iterator(dataset) - - -class FixedLossScaleManagerTest(test.TestCase): - - @test_util.run_in_graph_and_eager_modes - def test_basic(self): - itr = _GetExampleIter([True] * 10 + [False] * 10) - - loss_scale = 1000 - lsm = lsm_lib.FixedLossScaleManager(loss_scale) - update_fn = lambda: lsm.update_loss_scale(itr.get_next()) - - self.evaluate(variables.global_variables_initializer()) - if not context.executing_eagerly(): - update_op = update_fn() - for _ in range(10): - if context.executing_eagerly(): - update_fn() - else: - self.evaluate(update_op) - self.assertEqual(loss_scale, self.evaluate(lsm.get_loss_scale())) - - -class ExponentialUpdateLossScaleManagerTest(test.TestCase): - - def _test_helper(self, - inputs, - expected_outputs, - init_loss_scale=1, - incr_every_n_step=2, - decr_every_n_nan_or_inf=2): - ratio = 2 - lsm = lsm_lib.ExponentialUpdateLossScaleManager( - init_loss_scale=init_loss_scale, - incr_every_n_steps=incr_every_n_step, - decr_every_n_nan_or_inf=decr_every_n_nan_or_inf, - incr_ratio=ratio, - decr_ratio=1. / ratio) - itr = _GetExampleIter(inputs) - update_fn = lambda: lsm.update_loss_scale(itr.get_next()) - - self.evaluate(variables.global_variables_initializer()) - actual_outputs = [] - - if not context.executing_eagerly(): - update_op = update_fn() - for _ in range(len(inputs)): - if context.executing_eagerly(): - update_fn() - else: - self.evaluate(update_op) - actual_outputs.append(self.evaluate(lsm.get_loss_scale())) - self.assertEqual(actual_outputs, expected_outputs) - - @test_util.run_in_graph_and_eager_modes - def test_increase_every_n_steps(self): - inputs = [True] * 6 - expected_outputs = [1, 2, 2, 4, 4, 8] - self._test_helper(inputs, expected_outputs) - - @test_util.run_in_graph_and_eager_modes - def test_keep_increasing_until_capped(self): - init_loss_scale = np.finfo(np.float32).max / 4 + 10 - max_float = np.finfo(np.float32).max - - inputs = [True] * 6 - # Output is capped the 2nd time it doubles. - expected_outputs = [ - init_loss_scale, init_loss_scale * 2, init_loss_scale * 2, max_float, - max_float, max_float - ] - - self._test_helper(inputs, expected_outputs, init_loss_scale) - - @test_util.run_in_graph_and_eager_modes - def test_decrease_every_n_steps(self): - inputs = [False] * 6 - init_loss_scale = 1024 - expected_outputs = [1024, 512, 512, 256, 256, 128] - - self._test_helper(inputs, expected_outputs, init_loss_scale) - - @test_util.run_in_graph_and_eager_modes - def test_keep_decreasing_until_one(self): - inputs = [False] * 10 - init_loss_scale = 16 - expected_outputs = [16, 8, 8, 4, 4, 2, 2, 1, 1, 1] - - self._test_helper(inputs, expected_outputs, init_loss_scale) - - @test_util.run_in_graph_and_eager_modes - def test_incr_bad_step_clear_good_step(self): - inputs = [True, True, True, False, True] - expected_outputs = [1, 2, 2, 2, 2] - self._test_helper(inputs, expected_outputs) - - @test_util.run_in_graph_and_eager_modes - def test_incr_good_step_does_not_clear_bad_step(self): - inputs = [True, True, True, False, True, False] - expected_outputs = [1, 2, 2, 2, 2, 1] - self._test_helper(inputs, expected_outputs) - - @test_util.run_in_graph_and_eager_modes - def test_trigger_loss_scale_update_each_step(self): - """Test when incr_every_n_step and decr_every_n_nan_or_inf is 1.""" - init_loss_scale = 1 - incr_every_n_step = 1 - decr_every_n_nan_or_inf = 1 - - inputs = [True] * 3 + [False, True, True] - expected_outputs = [2, 4, 8, 4, 8, 16] - - self._test_helper(inputs, expected_outputs, init_loss_scale, - incr_every_n_step, decr_every_n_nan_or_inf) - - @test_util.run_in_graph_and_eager_modes - def test_alternating_good_and_bad_gradients_trigger_each_step(self): - init_loss_scale = 1 - incr_every_n_step = 1 - decr_every_n_nan_or_inf = 1 - - inputs = [True, False] * 4 + [True] - expected_outputs = [2, 1, 2, 1, 2, 1, 2, 1, 2] - self._test_helper(inputs, expected_outputs, init_loss_scale, - incr_every_n_step, decr_every_n_nan_or_inf) - - @test_util.run_in_graph_and_eager_modes - def test_alternating_good_and_bad_gradients_trigger_incr_every_2steps(self): - init_loss_scale = 32 - incr_every_n_step = 2 - decr_every_n_nan_or_inf = 1 - - inputs = [True, False] * 3 + [True] - expected_outputs = [32, 16, 16, 8, 8, 4, 4] - self._test_helper(inputs, expected_outputs, init_loss_scale, - incr_every_n_step, decr_every_n_nan_or_inf) - - @test_util.run_in_graph_and_eager_modes - def test_random_mix_good_and_bad_gradients(self): - init_loss_scale = 4 - inputs = [ - False, False, True, True, True, False, True, False, True, True, True, - False - ] - expected_outputs = [4, 2, 2, 4, 4, 4, 4, 2, 2, 4, 4, 4] - self._test_helper(inputs, expected_outputs, init_loss_scale) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer.py b/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer.py deleted file mode 100644 index 86306050560..00000000000 --- a/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer.py +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Loss scaling optimizer.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.eager import context -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_control_flow_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.training import optimizer - - -class LossScaleOptimizer(optimizer.Optimizer): - # TODO(jamesqin): move mixed precision training explanation to __init__ - # docstring. - """An optimizer that applies loss scaling in backprop. - - This class is useful for "mixed precision training" on GPUs (or other - potential accelerators), an approach to improve compute throughput without - compromising model quality. - - The canonical way to perform mixed precision training is the following: - * Model variables are kept in high precision (e.g. float32). - * Computations are done in lower precision (e.g. float16), which enjoys - performance speedup by virtue of hardware support. Variables are casted to - lower precision before they're used. - * Final gradients are casted back to high precision dtype, then used to update - variables. - - The side-effect of performing computation in lower precision, is that it comes - with smaller numerical range. During backproping, small gradients might - underflow in the reduced numerical range, causing a model to converge at - suboptimal level. - - To prevent underflow, this optimizer multiplies the loss by a factor before - backprop starts. Consequently, the gradients are linearly scaled up by the - same factor, thus not falling into the underflow zone. After that, to perserve - the correctness of backprop, the gradients are down-scaled by the same factor, - casted to the (higher) variable precision, then applied on the variables. - - See [Nvidia's manual on mixed precision training]( - https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html) - for more details. - - To use loss scale optimizer, one only needs choose a loss scale strategy and - wrap a regular optimizer. See examples below. - - ``` - loss = loss_fn() - opt = tf.AdamOptimizer(learning_rate=...) - - # Choose a loss scale manager which decides how to pick the right loss scale - # throughout the training process. - loss_scale_manager = tf.contrib.mixed_precision.FixedLossScaleManager(5000) - - # Wraps the original optimizer in a LossScaleOptimizer. - loss_scale_optimizer = - tf.contrib.mixed_precision.LossScaleOptimizer(opt, loss_scale_manager) - - # Call minimize() on the loss scale optimizer. - train_op = loss_scale_optimizer.minimize(loss) - ``` - - If gradients clipping is applied, one can call - `optimizer.compute_gradients()` and `optimizer.apply_gradients()` - separately. - - Notice the following way of using LossScaleOptimizer is not intended. Always - use `loss_scale_optimizer.compute_gradients()` to compute gradients instead of - `tf.gradients()` if doing mixed precision training. - - ``` - # The following is a wrong way to use LossScaleOptimizer along with - # tf.gradients(). - - # Always use loss_scale_optimizer.compute_gradients() to compute grads, or - # loss scale is not correctly applied. - grads = tf.gradients(loss, ...) - - # Do some custom grad clipping. - grads = clip_grads(grads, ...) - - loss_scale_optimizer.apply(grads_and_vars) - ``` - """ - - def __init__(self, opt, loss_scale_manager): - """Construct a loss scaling optimizer. - - Args: - opt: The actual optimizer that will be used to compute and apply the - gradients. Must be an implementation of the - `tf.compat.v1.train.Optimizer` interface. - loss_scale_manager: A LossScaleManager object. - """ - self._opt = opt - self._loss_scale_manager = loss_scale_manager - - def compute_gradients(self, - loss, - var_list=None, - gate_gradients=optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - grad_loss=None): - """Compute gradients. See base class `tf.compat.v1.train.Optimizer`.""" - loss_scale = self._loss_scale_manager.get_loss_scale() - if context.executing_eagerly(): - - def scaled_loss(): - loss_val = loss() - return loss_val * math_ops.cast(loss_scale, loss_val.dtype.base_dtype) - else: - if callable(loss): - loss_val = loss() - else: - loss_val = loss - scaled_loss = loss_val * math_ops.cast(loss_scale, - loss_val.dtype.base_dtype) - grads_and_vars = self._opt.compute_gradients( - scaled_loss, - var_list=var_list, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, - grad_loss=grad_loss) - return self._down_scale(grads_and_vars, loss_scale) - - def apply_gradients(self, grads_and_vars, global_step=None, name=None): - """Apply gradients. See base class `tf.compat.v1.train.Optimizer`.""" - grads = [g for (g, _) in grads_and_vars] - - is_finite_grad = [] - for g in grads: - is_finite_grad.append(math_ops.reduce_all(gen_math_ops.is_finite(g))) - is_overall_finite = math_ops.reduce_all(is_finite_grad) - - # Only update gradients when all grads are finite. - def true_apply_gradients_fn(): - return self._opt.apply_gradients(grads_and_vars, global_step, name) - - update_vars = control_flow_ops.cond(is_overall_finite, - true_apply_gradients_fn, - gen_control_flow_ops.no_op) - # Potentially adjust gradient scale in case of finite gradients. - return control_flow_ops.group( - update_vars, - self._loss_scale_manager.update_loss_scale(is_overall_finite)) - - def _down_scale(self, grads_vars, loss_scale): - # Down scale grads by the loss_scale. - gv = [] - inv_loss_scale = gen_math_ops.reciprocal(loss_scale) - for g, v in grads_vars: - if g is not None: - gv.append((g * math_ops.cast(inv_loss_scale, g.dtype.base_dtype), v)) - else: - gv.append((g, v)) - return gv diff --git a/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer_test.py b/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer_test.py deleted file mode 100644 index 33f9a43e803..00000000000 --- a/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer_test.py +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for LossScaleOptimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.mixed_precision.python import loss_scale_manager as lsm_lib -from tensorflow.contrib.mixed_precision.python import loss_scale_optimizer as lso -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import test_util -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import gradient_descent as gd - - -class LossScaleOptimizerTest(test.TestCase): - - def _build_graph(self, lr, init_val, loss_scale_opt_fn=None): - x = variable_scope.get_variable( - "x", initializer=init_val, dtype=dtypes.float32) - c1 = constant_op.constant(1e4, dtype=dtypes.float16) - c2 = constant_op.constant(1e-4, dtype=dtypes.float16) - c3 = constant_op.constant(1e-4, dtype=dtypes.float16) - if context.executing_eagerly(): - loss = lambda: math_ops.cast(x, dtypes.float16) * c1 * c2 * c3 - else: - loss = math_ops.cast(x, dtypes.float16) * c1 * c2 * c3 - - opt = gd.GradientDescentOptimizer(lr) - if loss_scale_opt_fn: - opt = loss_scale_opt_fn(opt) - return x, loss, opt - - @test_util.run_in_graph_and_eager_modes - def test_float16_underflow_without_loss_scale(self): - lr = 1 - init_val = 1. - x, loss, opt = self._build_graph(lr, init_val) - - self.evaluate(variables.global_variables_initializer()) - self.evaluate(opt.minimize(loss, var_list=[x])) - - # Symbolic grad is c1 * c2 * c3 = 1e-4 and actual grad is 0, since in - # backprop, c2 * c3 underflows in fp16 range. So variable isn't updated. - expected_update = 0 - symbolic_update = 1e-4 * lr - self.assertAllClose( - init_val - expected_update, - self.evaluate(x), - rtol=0, - atol=min(symbolic_update, 1e-6)) - - @test_util.run_in_graph_and_eager_modes - def test_float16_with_loss_scale(self): - lr = 1. - init_val = 1. - - def loss_scale_opt_fn(opt): - return lso.LossScaleOptimizer(opt, lsm_lib.FixedLossScaleManager(1e4)) - - x, loss, opt = self._build_graph(lr, init_val, loss_scale_opt_fn) - - self.evaluate(variables.global_variables_initializer()) - self.evaluate(opt.minimize(loss, var_list=[x])) - - # Symbolic grad is c1 * c2 * c3 = 1e-4 and actual grad is the same, due to - # up-scaled loss before backprop starts. - expected_update = 1.e-4 * lr - self.assertAllClose( - init_val - expected_update, - self.evaluate(x), - rtol=0, - atol=min(expected_update, 1e-6)) - - @test_util.run_in_graph_and_eager_modes - def test_compute_gradients_with_loss_scale(self): - lr = 1 - init_val = 1. - - def loss_scale_opt_fn(opt): - return lso.LossScaleOptimizer(opt, lsm_lib.FixedLossScaleManager(1e4)) - - x, loss, opt = self._build_graph(lr, init_val, loss_scale_opt_fn) - grads_and_vars = opt.compute_gradients(loss, var_list=[x]) - - self.assertEqual(len(grads_and_vars), 1) - - self.evaluate(variables.global_variables_initializer()) - g_v = self.evaluate(grads_and_vars[0][0]) - self.assertAllClose(g_v, 1e-4) - self.assertIs(grads_and_vars[0][1], x) - # Gradients aren't applied. - self.assertAllClose(init_val, self.evaluate(x), rtol=0, atol=1e-6) - - @test_util.run_in_graph_and_eager_modes - def test_compute_gradients_without_loss_scale(self): - lr = 1 - init_val = 1. - x, loss, opt = self._build_graph(lr, init_val) - grads_and_vars = opt.compute_gradients(loss, var_list=[x]) - - self.assertEqual(len(grads_and_vars), 1) - self.evaluate(variables.global_variables_initializer()) - g_v = self.evaluate(grads_and_vars[0][0]) - self.assertAllClose(g_v, 0) - - @test_util.run_in_graph_and_eager_modes - def test_apply_gradients(self): - - x = variable_scope.get_variable("x", initializer=1., dtype=dtypes.float32) - dataset = dataset_ops.Dataset.from_tensor_slices([np.nan, np.inf, 0.1]) - itr = dataset_ops.make_one_shot_iterator(dataset) - - lr = 1 - opt = gd.GradientDescentOptimizer(lr) - lsm = lsm_lib.FixedLossScaleManager(1.e4) - opt = lso.LossScaleOptimizer(opt, lsm) - train_fn = lambda: opt.apply_gradients([(itr.get_next(), x)]) - if not context.executing_eagerly(): - train_op = train_fn() - - expected_output = [1, 1, 1 - 0.1] - actual_output = [] - - self.evaluate(variables.global_variables_initializer()) - for _ in range(3): - # nan or inf is not applied. - if context.executing_eagerly(): - train_fn() - else: - self.evaluate(train_op) - actual_output.append(self.evaluate(x)) - self.assertAllClose(expected_output, actual_output) - - @test_util.run_in_graph_and_eager_modes - def test_apply_gradients_loss_scale_is_updated(self): - - class SimpleLossScaleManager(lsm_lib.LossScaleManager): - """A simple loss scale manager for easier testing. - - It increments loss scale by 1 if grads are finite, and decreases loss - scale by 1 if otherwise. - """ - - def __init__(self, loss_scale): - self._loss_scale = variable_scope.variable( - name="loss_scale", - initial_value=loss_scale, - dtype=dtypes.float32, - trainable=False) - - def get_loss_scale(self): - return self._loss_scale - - def update_loss_scale(self, if_finite_grads): - return control_flow_ops.cond( - if_finite_grads, lambda: state_ops.assign_add(self._loss_scale, 1), - lambda: state_ops.assign_sub(self._loss_scale, 1)) - - x = variable_scope.get_variable("x", initializer=1., dtype=dtypes.float32) - dataset = dataset_ops.Dataset.from_tensor_slices([np.nan, np.inf, 0.1]) - itr = dataset_ops.make_one_shot_iterator(dataset) - - lr = 1 - init_loss_scale = 8 - opt = gd.GradientDescentOptimizer(lr) - lsm = SimpleLossScaleManager(init_loss_scale) - opt = lso.LossScaleOptimizer(opt, lsm) - train_fn = lambda: opt.apply_gradients([(itr.get_next(), x)]) - if not context.executing_eagerly(): - train_op = train_fn() - - self.evaluate(variables.global_variables_initializer()) - - expected_loss_scale = [ - init_loss_scale - 1, init_loss_scale - 2, init_loss_scale - 2 + 1 - ] - expected_output = [1, 1, 1 - 0.1] - actual_output = [] - for i in range(3): - # nan or inf is not applied. - if context.executing_eagerly(): - train_fn() - else: - self.evaluate(train_op) - actual_output.append(self.evaluate(x)) - self.assertAllClose(expected_loss_scale[i], - self.evaluate(lsm._loss_scale)) - self.assertAllClose(expected_output, actual_output) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/model_pruning/BUILD b/tensorflow/contrib/model_pruning/BUILD deleted file mode 100644 index a42e01e9a9c..00000000000 --- a/tensorflow/contrib/model_pruning/BUILD +++ /dev/null @@ -1,202 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "py_test") - -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "core_layers", - srcs = ["python/layers/core_layers.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/python:layers", - "//tensorflow/python:ops", - "//tensorflow/python:platform", - ], -) - -py_library( - name = "layers", - srcs = ["python/layers/layers.py"], - srcs_version = "PY2AND3", - deps = [ - ":core_layers", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//third_party/py/numpy", - ], -) - -py_test( - name = "layers_test", - size = "small", - srcs = ["python/layers/layers_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":layers", - "//tensorflow/python:client_testlib", - ], -) - -py_library( - name = "learning", - srcs = ["python/learning.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/slim", - ], -) - -py_library( - name = "rnn_cells", - srcs = ["python/layers/rnn_cells.py"], - srcs_version = "PY2AND3", - deps = [ - ":core_layers", - ], -) - -py_library( - name = "pruning_utils", - srcs = ["python/pruning_utils.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/python:platform", - "//third_party/py/numpy", - ], -) - -py_library( - name = "pruning", - srcs = ["python/pruning.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":core_layers", - ":pruning_utils", - "//tensorflow/contrib/training:training_py", - "//tensorflow/python:platform", - ], -) - -py_library( - name = "strip_pruning_vars_lib", - srcs = ["python/strip_pruning_vars_lib.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":pruning", - "//tensorflow/python:client", - "//tensorflow/python:framework", - "//tensorflow/python:platform", - "//tensorflow/python:training", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_test( - name = "pruning_utils_test", - size = "medium", - srcs = ["python/pruning_utils_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":pruning_utils", - "//tensorflow/python:client_testlib", - "@absl_py//absl/testing:parameterized", - ], -) - -py_test( - name = "pruning_test", - size = "small", - srcs = ["python/pruning_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":pruning", - "//tensorflow/python:client_testlib", - ], -) - -py_test( - name = "rnn_cells_test", - size = "small", - srcs = ["python/layers/rnn_cells_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":pruning", - ":rnn_cells", - "//tensorflow/python:client_testlib", - ], -) - -py_test( - name = "strip_pruning_vars_test", - size = "small", - srcs = ["python/strip_pruning_vars_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_oss", # b/132443370 - ], - deps = [ - ":layers", - ":pruning", - ":rnn_cells", - ":strip_pruning_vars_lib", - "//tensorflow/python:client_testlib", - ], -) - -py_binary( - name = "strip_pruning_vars", - srcs = ["python/strip_pruning_vars.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":strip_pruning_vars_lib", - "//tensorflow/python:platform", - ], -) - -py_library( - name = "init_py", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", -) - -# Top-level library -py_library( - name = "model_pruning", - srcs_version = "PY2AND3", - deps = [ - ":init_py", - ":layers", - ":learning", - ":pruning", - ":rnn_cells", - ":strip_pruning_vars_lib", - ], -) diff --git a/tensorflow/contrib/model_pruning/README.md b/tensorflow/contrib/model_pruning/README.md deleted file mode 100644 index 7154d441625..00000000000 --- a/tensorflow/contrib/model_pruning/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# Model pruning: Training tensorflow models to have masked connections - -This document describes the API that facilitates magnitude-based pruning of -neural network's weight tensors. The API helps inject necessary tensorflow op -into the training graph so the model can be pruned while it is being trained. - -## Table of contents -1. [Model creation](#model-creation) -2. [Hyperparameters for pruning](#hyperparameters) - - [Block sparsity](#block-sparsity) -3. [Adding pruning ops to the training graph](#adding-pruning-ops) -4. [Removing pruning ops from trained model](#remove) -5. [Example](#example) - -### Model creation - -The first step involves adding mask and threshold variables to the layers that -need to undergo pruning. The variable mask is the same shape as the layer's -weight tensor and determines which of the weights participate in the forward -execution of the graph. This can be achieved by wrapping the weight tensor of -the layer with the `apply_mask` function provided in -[pruning.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/model_pruning/python/pruning.py). -For example: - -```python -conv = tf.nn.conv2d(images, pruning.apply_mask(weights), stride, padding) -``` - -This creates a convolutional layer with additional variables mask and threshold -as shown below: ![Convolutional layer with mask and -threshold](https://storage.googleapis.com/download.tensorflow.org/example_images/mask.png "Convolutional layer with mask and threshold") - -Alternatively, the API also provides variant of tensorflow layers with these -auxiliary variables built-in (see -[layers](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/model_pruning/python/layers)) -. Layers currently supported: - -* [layers.masked_conv2d](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/model_pruning/python/layers/layers.py?l=83) - -* [layers.masked_fully_connected](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/model_pruning/python/layers/layers.py?l=241) - -* [rnn_cells.MaskedLSTMCell](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/model_pruning/python/layers/rnn_cells.py?l=154) - -### Pruning-related hyperparameters - -The pruning library allows for specification of the following hyper parameters: - -|Hyperparameter | Type | Default | Description | -|:----------------------------|:-------:|:-------------:|:--------------| -| name | string | model_pruning | Name of the pruning specification. Used for adding summaries and ops under a common tensorflow name_scope | -| begin_pruning_step | integer | 0 | The global step at which to begin pruning | -| end_pruning_step | integer | -1 | The global step at which to terminate pruning. Defaults to -1 implying that pruning continues till the training stops | -| weight_sparsity_map | list of strings | [""] | list of weight variable name regex (or layer name regex):target sparsity pairs. Eg. [conv1:0.9,conv.*/kernel:0.8]. For layers/weights not in this list, sparsity as specified by the target_sparsity hyperparameter is used. | -| block_dims_map | list of strings | [""] | list of weight variable name regex (or layer name regex):block_heightxblock_width pairs. Eg. [dense1:4x4,dense2:1x16,dense3:1x1]. For layers/weights not in this list, block dims are specified by the block_height, block_width hyperparameters are used. | -| threshold_decay | float | 0.0 | The decay factor to use for exponential decay of the thresholds | -| pruning_frequency | integer | 10 | How often should the masks be updated? (in # of global_steps) | -| block_height|integer | 1 | Number of rows in a block for block sparse matrices| -| block_width |integer | 1 | Number of cols in a block for block sparse matrices| -| block_pooling_function| string | AVG | The function to use to pool weight values in a block: average (AVG) or max (MAX)| -| initial_sparsity | float | 0.0 | Initial sparsity value | -| target_sparsity | float | 0.5 | Target sparsity value | -| sparsity_function_begin_step | integer | 0 | The global step at this which the gradual sparsity function begins to take effect | -| sparsity_function_end_step | integer | 100 | The global step used as the end point for the gradual sparsity function | -| sparsity_function_exponent | float | 3.0 | exponent = 1 is linearly varying sparsity between initial and final. exponent > 1 varies more slowly towards the end than the beginning | -| use_tpu | bool | False | Training using TPUs? | - -The sparsity $$s_t$$ at global step $$t$$ is given by: - -$$s_{t}=s_{f}+\left(s_{i}-s_{f}\right)\left(1-\frac{t-t_{0}}{n\Delta t}\right)^{3}$$ - -The interval between sparsity_function_begin_step and sparsity_function_end_step -is divided into $$n$$ intervals of size equal to the pruning_frequency ($$\Delta -t$$). $$s_f$$ is the target_sparsity, $$s_i$$ is the initial_sparsity, $$t_0$$ -is the sparsity_function_begin_step. In this equation, the -sparsity_function_exponent is set to 3. - -#### Block Sparsity - -For some hardware architectures, it may be beneficial to induce spatially correlated sparsity. To train models in which the weight tensors have block sparse structure, set *block_height* and *block_width* hyperparameters to the desired block configuration (2x2, 4x4, 4x1, 1x8, etc). Currently, block sparsity is only supported for weight tensors which can be squeezed to rank 2. The matrix is partitioned into non-overlapping blocks of size *[block_height, block_dim]* and the either the average or max absolute value in this block is taken as a proxy for the entire block (set by *block_pooling_function* hyperparameter). -The convolution layer tensors are always pruned used block dimensions of [1,1]. - -### Adding pruning ops to the training graph - -The final step involves adding ops to the training graph that monitor the -distribution of the layer's weight magnitudes and determine the layer threshold, -such that masking all the weights below this threshold achieves the sparsity -level desired for the current training step. This can be achieved as follows: - -```python -tf.app.flags.DEFINE_string( - 'pruning_hparams', '', - """Comma separated list of pruning-related hyperparameters""") - -with tf.graph.as_default(): - - # Create global step variable - global_step = tf.train.get_or_create_global_step() - - # Parse pruning hyperparameters - pruning_hparams = pruning.get_pruning_hparams().parse(FLAGS.pruning_hparams) - - # Create a pruning object using the pruning specification - p = pruning.Pruning(pruning_hparams, global_step=global_step) - - # Add conditional mask update op. Executing this op will update all - # the masks in the graph if the current global step is in the range - # [begin_pruning_step, end_pruning_step] as specified by the pruning spec - mask_update_op = p.conditional_mask_update_op() - - # Add summaries to keep track of the sparsity in different layers during training - p.add_pruning_summaries() - - with tf.train.MonitoredTrainingSession(...) as mon_sess: - # Run the usual training op in the tf session - mon_sess.run(train_op) - - # Update the masks by running the mask_update_op - mon_sess.run(mask_update_op) - -``` -Ensure that `global_step` is being [incremented](https://www.tensorflow.org/api_docs/python/tf/train/Optimizer#minimize), otherwise pruning will not work! - -### Removing pruning ops from the trained graph -Once the model is trained, it is necessary to remove the auxiliary variables (mask, threshold) and pruning ops added to the graph in the steps above. This can be accomplished using the `strip_pruning_vars` utility. - -This utility generates a binary GraphDef in which the variables have been converted to constants. In particular, the threshold variables are removed from the graph and the mask variable is fused with the corresponding weight tensor to produce a `masked_weight` tensor. This tensor is sparse, has the same size as the weight tensor, and the sparsity is as set by the `target_sparsity` or the `weight_sparsity_map` hyperparameters above. - -```shell -$ bazel build -c opt contrib/model_pruning:strip_pruning_vars -$ bazel-bin/contrib/model_pruning/strip_pruning_vars --checkpoint_dir=/path/to/checkpoints/ --output_node_names=graph_node1,graph_node2 --output_dir=/tmp --filename=pruning_stripped.pb -``` - -For now, it is assumed that the underlying hardware platform will provide mechanisms for compressing the sparse tensors and/or accelerating the sparse tensor computations. - -## Example: Pruning and training deep CNNs on the cifar10 dataset - -Please see -[Advanced Convolutional Neural Networks](https://www.tensorflow.org/tutorials/images/deep_cnn) -for details on neural network architecture, setting up inputs etc. The -additional changes needed to incorporate pruning are captured in the following: - -* [cifar10_pruning.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_pruning.py) - creates a deep CNN with the same architecture, but adds mask and threshold - variables for each of the weight tensors in the convolutional and - locally-connected layers. - -* [cifar10_train.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_train.py) - add pruning ops to the training graph as described above. - -To train the pruned version of cifar10: - -```shell -$ examples_dir=contrib/model_pruning/examples -$ bazel build -c opt $examples_dir/cifar10:cifar10_{train,eval} -$ bazel-bin/$examples_dir/cifar10/cifar10_train --pruning_hparams=name=cifar10_pruning,begin_pruning_step=10000,end_pruning_step=100000,target_sparsity=0.9,sparsity_function_begin_step=10000,sparsity_function_end_step=100000 -``` - -Eval: - -```shell -$ bazel-bin/$examples_dir/cifar10/cifar10_eval --run_once -``` - -Removing pruning nodes from the trained graph: - -```shell -$ bazel build -c opt contrib/model_pruning:strip_pruning_vars -$ bazel-bin/contrib/model_pruning/strip_pruning_vars --checkpoint_path=/tmp/cifar10_train --output_node_names=softmax_linear/softmax_linear_2 --filename=cifar_pruned.pb -``` - -The generated GraphDef (cifar_pruned.pb) may be visualized using the [`import_pb_to_tensorboard`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python/tools/import_pb_to_tensorboard.py) utility - -## References - -Michael Zhu and Suyog Gupta, “To prune, or not to prune: exploring the efficacy of pruning for model compressionâ€, *2017 NIPS Workshop on Machine Learning of Phones and other Consumer Devices* (https://arxiv.org/pdf/1710.01878.pdf) diff --git a/tensorflow/contrib/model_pruning/__init__.py b/tensorflow/contrib/model_pruning/__init__.py deleted file mode 100644 index 6eca54aaee1..00000000000 --- a/tensorflow/contrib/model_pruning/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model pruning implementation in tensorflow.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.contrib.model_pruning.python.layers.layers import masked_conv2d -from tensorflow.contrib.model_pruning.python.layers.layers import masked_convolution -from tensorflow.contrib.model_pruning.python.layers.layers import masked_fully_connected -from tensorflow.contrib.model_pruning.python.layers.rnn_cells import MaskedBasicLSTMCell -from tensorflow.contrib.model_pruning.python.layers.rnn_cells import MaskedLSTMCell -from tensorflow.contrib.model_pruning.python.learning import train -from tensorflow.contrib.model_pruning.python.pruning import apply_mask -from tensorflow.contrib.model_pruning.python.pruning import get_masked_weights -from tensorflow.contrib.model_pruning.python.pruning import get_masks -from tensorflow.contrib.model_pruning.python.pruning import get_pruning_hparams -from tensorflow.contrib.model_pruning.python.pruning import get_thresholds -from tensorflow.contrib.model_pruning.python.pruning import get_weight_sparsity -from tensorflow.contrib.model_pruning.python.pruning import get_weights -from tensorflow.contrib.model_pruning.python.pruning import Pruning -from tensorflow.contrib.model_pruning.python.strip_pruning_vars_lib import graph_def_from_checkpoint -from tensorflow.contrib.model_pruning.python.strip_pruning_vars_lib import strip_pruning_vars_fn - -# pylint: enable=unused-import - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'masked_convolution', 'masked_conv2d', 'masked_fully_connected', - 'MaskedBasicLSTMCell', 'MaskedLSTMCell', 'train', 'apply_mask', - 'get_masked_weights', 'get_masks', 'get_pruning_hparams', 'get_thresholds', - 'get_weights', 'get_weight_sparsity', 'Pruning', 'strip_pruning_vars_fn', - 'graph_def_from_checkpoint' -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/model_pruning/examples/cifar10/BUILD b/tensorflow/contrib/model_pruning/examples/cifar10/BUILD deleted file mode 100644 index d75211086e6..00000000000 --- a/tensorflow/contrib/model_pruning/examples/cifar10/BUILD +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# Description: -# Example TensorFlow models for CIFAR-10 - -package( - default_visibility = [ - "//tensorflow:internal", - ], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "cifar10_input", - srcs = ["cifar10_input.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - ], -) - -py_library( - name = "cifar10_pruning", - srcs = ["cifar10_pruning.py"], - srcs_version = "PY2AND3", - deps = [ - ":cifar10_input", - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/model_pruning:pruning", - ], -) - -py_binary( - name = "cifar10_eval", - srcs = [ - "cifar10_eval.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":cifar10_pruning", - "//tensorflow:tensorflow_py", - "//third_party/py/numpy", - ], -) - -py_binary( - name = "cifar10_train", - srcs = [ - "cifar10_train.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":cifar10_pruning", - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/model_pruning:pruning", - ], -) diff --git a/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_eval.py b/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_eval.py deleted file mode 100644 index d72b2a1dca5..00000000000 --- a/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_eval.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Evaluation for CIFAR-10. - -Accuracy: -cifar10_train.py achieves 83.0% accuracy after 100K steps (256 epochs -of data) as judged by cifar10_eval.py. - -Speed: -On a single Tesla K40, cifar10_train.py processes a single batch of 128 images -in 0.25-0.35 sec (i.e. 350 - 600 images /sec). The model reaches ~86% -accuracy after 100K steps in 8 hours of training time. - -Usage: -Please see the tutorial and website for how to download the CIFAR-10 -data set, compile the program and train the model. - -http://tensorflow.org/tutorials/deep_cnn/ -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import datetime -import math -import sys -import time - -import numpy as np -import tensorflow as tf - -from tensorflow.contrib.model_pruning.examples.cifar10 import cifar10_pruning as cifar10 - -FLAGS = None - - -def eval_once(saver, summary_writer, top_k_op, summary_op): - """Run Eval once. - - Args: - saver: Saver. - summary_writer: Summary writer. - top_k_op: Top K op. - summary_op: Summary op. - """ - with tf.Session() as sess: - ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir) - if ckpt and ckpt.model_checkpoint_path: - # Restores from checkpoint - saver.restore(sess, ckpt.model_checkpoint_path) - # Assuming model_checkpoint_path looks something like: - # /my-favorite-path/cifar10_train/model.ckpt-0, - # extract global_step from it. - global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] - else: - print('No checkpoint file found') - return - - # Start the queue runners. - coord = tf.train.Coordinator() - try: - threads = [] - for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS): - threads.extend(qr.create_threads(sess, coord=coord, daemon=True, - start=True)) - - num_iter = int(math.ceil(FLAGS.num_examples / 128)) - true_count = 0 # Counts the number of correct predictions. - total_sample_count = num_iter * 128 - step = 0 - while step < num_iter and not coord.should_stop(): - predictions = sess.run([top_k_op]) - true_count += np.sum(predictions) - step += 1 - - # Compute precision @ 1. - precision = true_count / total_sample_count - print('%s: precision @ 1 = %.3f' % (datetime.datetime.now(), precision)) - - summary = tf.Summary() - summary.ParseFromString(sess.run(summary_op)) - summary.value.add(tag='Precision @ 1', simple_value=precision) - summary_writer.add_summary(summary, global_step) - except Exception as e: # pylint: disable=broad-except - coord.request_stop(e) - - coord.request_stop() - coord.join(threads, stop_grace_period_secs=10) - - -def evaluate(): - """Eval CIFAR-10 for a number of steps.""" - with tf.Graph().as_default() as g: - # Get images and labels for CIFAR-10. - eval_data = FLAGS.eval_data == 'test' - images, labels = cifar10.inputs(eval_data=eval_data) - - # Build a Graph that computes the logits predictions from the - # inference model. - logits = cifar10.inference(images) - - # Calculate predictions. - top_k_op = tf.nn.in_top_k(logits, labels, 1) - - # Restore the moving average version of the learned variables for eval. - variable_averages = tf.train.ExponentialMovingAverage( - cifar10.MOVING_AVERAGE_DECAY) - variables_to_restore = variable_averages.variables_to_restore() - saver = tf.train.Saver(variables_to_restore) - - # Build the summary operation based on the TF collection of Summaries. - summary_op = tf.summary.merge_all() - - summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, g) - - while True: - eval_once(saver, summary_writer, top_k_op, summary_op) - if FLAGS.run_once: - break - time.sleep(FLAGS.eval_interval_secs) - - -def main(argv=None): # pylint: disable=unused-argument - cifar10.maybe_download_and_extract() - if tf.gfile.Exists(FLAGS.eval_dir): - tf.gfile.DeleteRecursively(FLAGS.eval_dir) - tf.gfile.MakeDirs(FLAGS.eval_dir) - evaluate() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument( - '--eval_dir', - type=str, - default='/tmp/cifar10_eval', - help='Directory where to write event logs.') - parser.add_argument( - '--eval_data', - type=str, - default='test', - help="""Either 'test' or 'train_eval'.""") - parser.add_argument( - '--checkpoint_dir', - type=str, - default='/tmp/cifar10_train', - help="""Directory where to read model checkpoints.""") - parser.add_argument( - '--eval_interval_secs', - type=int, - default=60 * 5, - help='How often to run the eval.') - parser.add_argument( - '--num_examples', - type=int, - default=10000, - help='Number of examples to run.') - parser.add_argument( - '--run_once', - type=bool, - default=False, - help='Whether to run eval only once.') - - FLAGS, unparsed = parser.parse_known_args() - tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_input.py b/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_input.py deleted file mode 100644 index 6a3b535eb44..00000000000 --- a/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_input.py +++ /dev/null @@ -1,264 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Routine for decoding the CIFAR-10 binary file format.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow as tf - -# Process images of this size. Note that this differs from the original CIFAR -# image size of 32 x 32. If one alters this number, then the entire model -# architecture will change and any model would need to be retrained. -IMAGE_SIZE = 24 - -# Global constants describing the CIFAR-10 data set. -NUM_CLASSES = 10 -NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000 -NUM_EXAMPLES_PER_EPOCH_FOR_EVAL = 10000 - - -def read_cifar10(filename_queue): - """Reads and parses examples from CIFAR10 data files. - - Recommendation: if you want N-way read parallelism, call this function - N times. This will give you N independent Readers reading different - files & positions within those files, which will give better mixing of - examples. - - Args: - filename_queue: A queue of strings with the filenames to read from. - - Returns: - An object representing a single example, with the following fields: - height: number of rows in the result (32) - width: number of columns in the result (32) - depth: number of color channels in the result (3) - key: a scalar string Tensor describing the filename & record number - for this example. - label: an int32 Tensor with the label in the range 0..9. - uint8image: a [height, width, depth] uint8 Tensor with the image data - """ - - class CIFAR10Record(object): - pass - - result = CIFAR10Record() - - # Dimensions of the images in the CIFAR-10 dataset. - # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the - # input format. - label_bytes = 1 # 2 for CIFAR-100 - result.height = 32 - result.width = 32 - result.depth = 3 - image_bytes = result.height * result.width * result.depth - # Every record consists of a label followed by the image, with a - # fixed number of bytes for each. - record_bytes = label_bytes + image_bytes - - # Read a record, getting filenames from the filename_queue. No - # header or footer in the CIFAR-10 format, so we leave header_bytes - # and footer_bytes at their default of 0. - reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) - result.key, value = reader.read(filename_queue) - - # Convert from a string to a vector of uint8 that is record_bytes long. - record_bytes = tf.decode_raw(value, tf.uint8) - - # The first bytes represent the label, which we convert from uint8->int32. - result.label = tf.cast( - tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) - - # The remaining bytes after the label represent the image, which we reshape - # from [depth * height * width] to [depth, height, width]. - depth_major = tf.reshape( - tf.strided_slice(record_bytes, [label_bytes], - [label_bytes + image_bytes]), - [result.depth, result.height, result.width]) - # Convert from [depth, height, width] to [height, width, depth]. - result.uint8image = tf.transpose(depth_major, [1, 2, 0]) - - return result - - -def _generate_image_and_label_batch(image, label, min_queue_examples, - batch_size, shuffle): - """Construct a queued batch of images and labels. - - Args: - image: 3-D Tensor of [height, width, 3] of type.float32. - label: 1-D Tensor of type.int32 - min_queue_examples: int32, minimum number of samples to retain - in the queue that provides of batches of examples. - batch_size: Number of images per batch. - shuffle: boolean indicating whether to use a shuffling queue. - - Returns: - images: Images. 4D tensor of [batch_size, height, width, 3] size. - labels: Labels. 1D tensor of [batch_size] size. - """ - # Create a queue that shuffles the examples, and then - # read 'batch_size' images + labels from the example queue. - num_preprocess_threads = 16 - if shuffle: - images, label_batch = tf.train.shuffle_batch( - [image, label], - batch_size=batch_size, - num_threads=num_preprocess_threads, - capacity=min_queue_examples + 3 * batch_size, - min_after_dequeue=min_queue_examples) - else: - images, label_batch = tf.train.batch( - [image, label], - batch_size=batch_size, - num_threads=num_preprocess_threads, - capacity=min_queue_examples + 3 * batch_size) - - # Display the training images in the visualizer. - tf.summary.image('images', images) - - return images, tf.reshape(label_batch, [batch_size]) - - -def distorted_inputs(data_dir, batch_size): - """Construct distorted input for CIFAR training using the Reader ops. - - Args: - data_dir: Path to the CIFAR-10 data directory. - batch_size: Number of images per batch. - - Returns: - images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size. - labels: Labels. 1D tensor of [batch_size] size. - """ - filenames = [ - os.path.join(data_dir, 'data_batch_%d.bin' % i) for i in xrange(1, 6) - ] - for f in filenames: - if not tf.gfile.Exists(f): - raise ValueError('Failed to find file: ' + f) - - # Create a queue that produces the filenames to read. - filename_queue = tf.train.string_input_producer(filenames) - - # Read examples from files in the filename queue. - read_input = read_cifar10(filename_queue) - reshaped_image = tf.cast(read_input.uint8image, tf.float32) - - height = IMAGE_SIZE - width = IMAGE_SIZE - - # Image processing for training the network. Note the many random - # distortions applied to the image. - - # Randomly crop a [height, width] section of the image. - distorted_image = tf.random_crop(reshaped_image, [height, width, 3]) - - # Randomly flip the image horizontally. - distorted_image = tf.image.random_flip_left_right(distorted_image) - - # Because these operations are not commutative, consider randomizing - # the order their operation. - distorted_image = tf.image.random_brightness(distorted_image, max_delta=63) - distorted_image = tf.image.random_contrast( - distorted_image, lower=0.2, upper=1.8) - - # Subtract off the mean and divide by the variance of the pixels. - float_image = tf.image.per_image_standardization(distorted_image) - - # Set the shapes of tensors. - float_image.set_shape([height, width, 3]) - read_input.label.set_shape([1]) - - # Ensure that the random shuffling has good mixing properties. - min_fraction_of_examples_in_queue = 0.4 - min_queue_examples = int( - NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * min_fraction_of_examples_in_queue) - print('Filling queue with %d CIFAR images before starting to train. ' - 'This will take a few minutes.' % min_queue_examples) - - # Generate a batch of images and labels by building up a queue of examples. - return _generate_image_and_label_batch( - float_image, - read_input.label, - min_queue_examples, - batch_size, - shuffle=True) - - -def inputs(eval_data, data_dir, batch_size): - """Construct input for CIFAR evaluation using the Reader ops. - - Args: - eval_data: bool, indicating if one should use the train or eval data set. - data_dir: Path to the CIFAR-10 data directory. - batch_size: Number of images per batch. - - Returns: - images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size. - labels: Labels. 1D tensor of [batch_size] size. - """ - if not eval_data: - filenames = [ - os.path.join(data_dir, 'data_batch_%d.bin' % i) for i in xrange(1, 6) - ] - num_examples_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN - else: - filenames = [os.path.join(data_dir, 'test_batch.bin')] - num_examples_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_EVAL - - for f in filenames: - if not tf.gfile.Exists(f): - raise ValueError('Failed to find file: ' + f) - - # Create a queue that produces the filenames to read. - filename_queue = tf.train.string_input_producer(filenames) - - # Read examples from files in the filename queue. - read_input = read_cifar10(filename_queue) - reshaped_image = tf.cast(read_input.uint8image, tf.float32) - - height = IMAGE_SIZE - width = IMAGE_SIZE - - # Image processing for evaluation. - # Crop the central [height, width] of the image. - resized_image = tf.image.resize_image_with_crop_or_pad( - reshaped_image, width, height) - - # Subtract off the mean and divide by the variance of the pixels. - float_image = tf.image.per_image_standardization(resized_image) - - # Set the shapes of tensors. - float_image.set_shape([height, width, 3]) - read_input.label.set_shape([1]) - - # Ensure that the random shuffling has good mixing properties. - min_fraction_of_examples_in_queue = 0.4 - min_queue_examples = int( - num_examples_per_epoch * min_fraction_of_examples_in_queue) - - # Generate a batch of images and labels by building up a queue of examples. - return _generate_image_and_label_batch( - float_image, - read_input.label, - min_queue_examples, - batch_size, - shuffle=False) diff --git a/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_pruning.py b/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_pruning.py deleted file mode 100644 index 96303f3984c..00000000000 --- a/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_pruning.py +++ /dev/null @@ -1,403 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Builds the CIFAR-10 network with additional variables to support pruning. - -Summary of available functions: - - # Compute input images and labels for training. If you would like to run - # evaluations, use inputs() instead. - inputs, labels = distorted_inputs() - - # Compute inference on the model inputs to make a prediction. - predictions = inference(inputs) - - # Compute the total loss of the prediction with respect to the labels. - loss = loss(predictions, labels) - - # Create a graph to run one step of training with respect to the loss. - train_op = train(loss, global_step) -""" -# pylint: disable=missing-docstring -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import re -import sys -import tarfile - -from six.moves import urllib -import tensorflow as tf - -from tensorflow.contrib.model_pruning.examples.cifar10 import cifar10_input -from tensorflow.contrib.model_pruning.python import pruning - -# Global constants describing the CIFAR-10 data set. -IMAGE_SIZE = cifar10_input.IMAGE_SIZE -NUM_CLASSES = cifar10_input.NUM_CLASSES -NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = cifar10_input.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN # pylint: disable=line-too-long -NUM_EXAMPLES_PER_EPOCH_FOR_EVAL = cifar10_input.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL -BATCH_SIZE = 128 -DATA_DIR = '/tmp/cifar10_data' - -# Constants describing the training process. -MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average. -NUM_EPOCHS_PER_DECAY = 350.0 # Epochs after which learning rate decays. -LEARNING_RATE_DECAY_FACTOR = 0.1 # Learning rate decay factor. -INITIAL_LEARNING_RATE = 0.1 # Initial learning rate. - -# If a model is trained with multiple GPUs, prefix all Op names with tower_name -# to differentiate the operations. Note that this prefix is removed from the -# names of the summaries when visualizing a model. -TOWER_NAME = 'tower' - -DATA_URL = 'http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz' - - -def _activation_summary(x): - """Helper to create summaries for activations. - - Creates a summary that provides a histogram of activations. - Creates a summary that measures the sparsity of activations. - - Args: - x: Tensor - - Returns: - nothing - """ - # Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training - # session. This helps the clarity of presentation on tensorboard. - tensor_name = re.sub('%s_[0-9]*/' % TOWER_NAME, '', x.op.name) - tf.summary.histogram(tensor_name + '/activations', x) - tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) - - -def _variable_on_cpu(name, shape, initializer): - """Helper to create a Variable stored on CPU memory. - - Args: - name: name of the variable - shape: list of ints - initializer: initializer for Variable - - Returns: - Variable Tensor - """ - with tf.device('/cpu:0'): - dtype = tf.float32 - var = tf.get_variable(name, shape, initializer=initializer, dtype=dtype) - return var - - -def _variable_with_weight_decay(name, shape, stddev, wd): - """Helper to create an initialized Variable with weight decay. - - Note that the Variable is initialized with a truncated normal distribution. - A weight decay is added only if one is specified. - - Args: - name: name of the variable - shape: list of ints - stddev: standard deviation of a truncated Gaussian - wd: add L2Loss weight decay multiplied by this float. If None, weight decay - is not added for this Variable. - - Returns: - Variable Tensor - """ - dtype = tf.float32 - var = _variable_on_cpu( - name, shape, tf.truncated_normal_initializer(stddev=stddev, dtype=dtype)) - if wd is not None: - weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss') - tf.add_to_collection('losses', weight_decay) - return var - - -def distorted_inputs(): - """Construct distorted input for CIFAR training using the Reader ops. - - Returns: - images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size. - labels: Labels. 1D tensor of [batch_size] size. - - Raises: - ValueError: If no data_dir - """ - if not DATA_DIR: - raise ValueError('Please supply a data_dir') - data_dir = os.path.join(DATA_DIR, 'cifar-10-batches-bin') - images, labels = cifar10_input.distorted_inputs( - data_dir=data_dir, batch_size=BATCH_SIZE) - return images, labels - - -def inputs(eval_data): - """Construct input for CIFAR evaluation using the Reader ops. - - Args: - eval_data: bool, indicating if one should use the train or eval data set. - - Returns: - images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size. - labels: Labels. 1D tensor of [batch_size] size. - - Raises: - ValueError: If no data_dir - """ - if not DATA_DIR: - raise ValueError('Please supply a data_dir') - data_dir = os.path.join(DATA_DIR, 'cifar-10-batches-bin') - images, labels = cifar10_input.inputs( - eval_data=eval_data, data_dir=data_dir, batch_size=BATCH_SIZE) - return images, labels - - -def inference(images): - """Build the CIFAR-10 model. - - Args: - images: Images returned from distorted_inputs() or inputs(). - - Returns: - Logits. - """ - # We instantiate all variables using tf.compat.v1.get_variable() instead of - # tf.Variable() in order to share variables across multiple GPU training runs. - # If we only ran this model on a single GPU, we could simplify this function - # by replacing all instances of tf.compat.v1.get_variable() with tf.Variable(). - # - # While instantiating conv and local layers, we add mask and threshold - # variables to the layer by calling the pruning.apply_mask() function. - # Note that the masks are applied only to the weight tensors - # conv1 - with tf.variable_scope('conv1') as scope: - kernel = _variable_with_weight_decay( - 'weights', shape=[5, 5, 3, 64], stddev=5e-2, wd=0.0) - - conv = tf.nn.conv2d( - images, pruning.apply_mask(kernel, scope), [1, 1, 1, 1], padding='SAME') - biases = _variable_on_cpu('biases', [64], tf.constant_initializer(0.0)) - pre_activation = tf.nn.bias_add(conv, biases) - conv1 = tf.nn.relu(pre_activation, name=scope.name) - _activation_summary(conv1) - - # pool1 - pool1 = tf.nn.max_pool( - conv1, - ksize=[1, 3, 3, 1], - strides=[1, 2, 2, 1], - padding='SAME', - name='pool1') - # norm1 - norm1 = tf.nn.lrn( - pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm1') - - # conv2 - with tf.variable_scope('conv2') as scope: - kernel = _variable_with_weight_decay( - 'weights', shape=[5, 5, 64, 64], stddev=5e-2, wd=0.0) - conv = tf.nn.conv2d( - norm1, pruning.apply_mask(kernel, scope), [1, 1, 1, 1], padding='SAME') - biases = _variable_on_cpu('biases', [64], tf.constant_initializer(0.1)) - pre_activation = tf.nn.bias_add(conv, biases) - conv2 = tf.nn.relu(pre_activation, name=scope.name) - _activation_summary(conv2) - - # norm2 - norm2 = tf.nn.lrn( - conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm2') - # pool2 - pool2 = tf.nn.max_pool( - norm2, - ksize=[1, 3, 3, 1], - strides=[1, 2, 2, 1], - padding='SAME', - name='pool2') - - # local3 - with tf.variable_scope('local3') as scope: - # Move everything into depth so we can perform a single matrix multiply. - reshape = tf.reshape(pool2, [BATCH_SIZE, -1]) - dim = reshape.get_shape()[1].value - weights = _variable_with_weight_decay( - 'weights', shape=[dim, 384], stddev=0.04, wd=0.004) - biases = _variable_on_cpu('biases', [384], tf.constant_initializer(0.1)) - local3 = tf.nn.relu( - tf.matmul(reshape, pruning.apply_mask(weights, scope)) + biases, - name=scope.name) - _activation_summary(local3) - - # local4 - with tf.variable_scope('local4') as scope: - weights = _variable_with_weight_decay( - 'weights', shape=[384, 192], stddev=0.04, wd=0.004) - biases = _variable_on_cpu('biases', [192], tf.constant_initializer(0.1)) - local4 = tf.nn.relu( - tf.matmul(local3, pruning.apply_mask(weights, scope)) + biases, - name=scope.name) - _activation_summary(local4) - - # linear layer(WX + b), - # We don't apply softmax here because - # tf.nn.sparse_softmax_cross_entropy_with_logits accepts the unscaled logits - # and performs the softmax internally for efficiency. - with tf.variable_scope('softmax_linear') as scope: - weights = _variable_with_weight_decay( - 'weights', [192, NUM_CLASSES], stddev=1 / 192.0, wd=0.0) - biases = _variable_on_cpu('biases', [NUM_CLASSES], - tf.constant_initializer(0.0)) - softmax_linear = tf.add( - tf.matmul(local4, pruning.apply_mask(weights, scope)), - biases, - name=scope.name) - _activation_summary(softmax_linear) - - return softmax_linear - - -def loss(logits, labels): - """Add L2Loss to all the trainable variables. - - Add summary for "Loss" and "Loss/avg". - Args: - logits: Logits from inference(). - labels: Labels from distorted_inputs or inputs(). 1-D tensor of shape - [batch_size] - - Returns: - Loss tensor of type float. - """ - # Calculate the average cross entropy loss across the batch. - labels = tf.cast(labels, tf.int64) - cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( - labels=labels, logits=logits, name='cross_entropy_per_example') - cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy') - tf.add_to_collection('losses', cross_entropy_mean) - - # The total loss is defined as the cross entropy loss plus all of the weight - # decay terms (L2 loss). - return tf.add_n(tf.get_collection('losses'), name='total_loss') - - -def _add_loss_summaries(total_loss): - """Add summaries for losses in CIFAR-10 model. - - Generates moving average for all losses and associated summaries for - visualizing the performance of the network. - - Args: - total_loss: Total loss from loss(). - - Returns: - loss_averages_op: op for generating moving averages of losses. - """ - # Compute the moving average of all individual losses and the total loss. - loss_averages = tf.train.ExponentialMovingAverage(0.9, name='avg') - losses = tf.get_collection('losses') - loss_averages_op = loss_averages.apply(losses + [total_loss]) - - # Attach a scalar summary to all individual losses and the total loss; do the - # same for the averaged version of the losses. - for l in losses + [total_loss]: - # Name each loss as '(raw)' and name the moving average version of the loss - # as the original loss name. - tf.summary.scalar(l.op.name + ' (raw)', l) - tf.summary.scalar(l.op.name, loss_averages.average(l)) - - return loss_averages_op - - -def train(total_loss, global_step): - """Train CIFAR-10 model. - - Create an optimizer and apply to all trainable variables. Add moving - average for all trainable variables. - - Args: - total_loss: Total loss from loss(). - global_step: Integer Variable counting the number of training steps - processed. - - Returns: - train_op: op for training. - """ - # Variables that affect learning rate. - num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / BATCH_SIZE - decay_steps = int(num_batches_per_epoch * NUM_EPOCHS_PER_DECAY) - - # Decay the learning rate exponentially based on the number of steps. - lr = tf.train.exponential_decay( - INITIAL_LEARNING_RATE, - global_step, - decay_steps, - LEARNING_RATE_DECAY_FACTOR, - staircase=True) - tf.summary.scalar('learning_rate', lr) - - # Generate moving averages of all losses and associated summaries. - loss_averages_op = _add_loss_summaries(total_loss) - - # Compute gradients. - with tf.control_dependencies([loss_averages_op]): - opt = tf.train.GradientDescentOptimizer(lr) - grads = opt.compute_gradients(total_loss) - - # Apply gradients. - apply_gradient_op = opt.apply_gradients(grads, global_step=global_step) - - # Add histograms for trainable variables. - for var in tf.trainable_variables(): - tf.summary.histogram(var.op.name, var) - - # Add histograms for gradients. - for grad, var in grads: - if grad is not None: - tf.summary.histogram(var.op.name + '/gradients', grad) - - # Track the moving averages of all trainable variables. - variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, - global_step) - variables_averages_op = variable_averages.apply(tf.trainable_variables()) - - with tf.control_dependencies([apply_gradient_op, variables_averages_op]): - train_op = tf.no_op(name='train') - - return train_op - - -def maybe_download_and_extract(): - """Download and extract the tarball from Alex's website.""" - dest_directory = DATA_DIR - if not os.path.exists(dest_directory): - os.makedirs(dest_directory) - filename = DATA_URL.split('/')[-1] - filepath = os.path.join(dest_directory, filename) - if not os.path.exists(filepath): - - def _progress(count, block_size, total_size): - sys.stdout.write( - '\r>> Downloading %s %.1f%%' % - (filename, float(count * block_size) / float(total_size) * 100.0)) - sys.stdout.flush() - - filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress) - print() - statinfo = os.stat(filepath) - print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') - - tarfile.open(filepath, 'r:gz').extractall(dest_directory) diff --git a/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_train.py b/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_train.py deleted file mode 100644 index a1064a3b6ab..00000000000 --- a/tensorflow/contrib/model_pruning/examples/cifar10/cifar10_train.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A binary to train pruned CIFAR-10 using a single GPU. - -Accuracy: -cifar10_train.py achieves ~86% accuracy after 100K steps (256 epochs of -data) as judged by cifar10_eval.py when target sparsity in -cifar10_pruning_spec.pbtxt is set to zero - -Results: -Sparsity | Accuracy after 150K steps --------- | ------------------------- -0% | 86% -50% | 86% -75% | TODO(suyoggupta) -90% | TODO(suyoggupta) -95% | 77% - -Usage: -Please see the tutorial and website for how to download the CIFAR-10 -data set, compile the program and train the model. - - -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import datetime -import sys -import time - - -import tensorflow as tf - -from tensorflow.contrib.model_pruning.examples.cifar10 import cifar10_pruning as cifar10 -from tensorflow.contrib.model_pruning.python import pruning - -FLAGS = None - - -def train(): - """Train CIFAR-10 for a number of steps.""" - with tf.Graph().as_default(): - global_step = tf.contrib.framework.get_or_create_global_step() - - # Get images and labels for CIFAR-10. - images, labels = cifar10.distorted_inputs() - - # Build a Graph that computes the logits predictions from the - # inference model. - logits = cifar10.inference(images) - - # Calculate loss. - loss = cifar10.loss(logits, labels) - - # Build a Graph that trains the model with one batch of examples and - # updates the model parameters. - train_op = cifar10.train(loss, global_step) - - # Parse pruning hyperparameters - pruning_hparams = pruning.get_pruning_hparams().parse(FLAGS.pruning_hparams) - - # Create a pruning object using the pruning hyperparameters - pruning_obj = pruning.Pruning(pruning_hparams, global_step=global_step) - - # Use the pruning_obj to add ops to the training graph to update the masks - # The conditional_mask_update_op will update the masks only when the - # training step is in [begin_pruning_step, end_pruning_step] specified in - # the pruning spec proto - mask_update_op = pruning_obj.conditional_mask_update_op() - - # Use the pruning_obj to add summaries to the graph to track the sparsity - # of each of the layers - pruning_obj.add_pruning_summaries() - - class _LoggerHook(tf.train.SessionRunHook): - """Logs loss and runtime.""" - - def begin(self): - self._step = -1 - - def before_run(self, run_context): - self._step += 1 - self._start_time = time.time() - return tf.train.SessionRunArgs(loss) # Asks for loss value. - - def after_run(self, run_context, run_values): - duration = time.time() - self._start_time - loss_value = run_values.results - if self._step % 10 == 0: - num_examples_per_step = 128 - examples_per_sec = num_examples_per_step / duration - sec_per_batch = float(duration) - - format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f ' - 'sec/batch)') - print(format_str % (datetime.datetime.now(), self._step, loss_value, - examples_per_sec, sec_per_batch)) - - with tf.train.MonitoredTrainingSession( - checkpoint_dir=FLAGS.train_dir, - hooks=[tf.train.StopAtStepHook(last_step=FLAGS.max_steps), - tf.train.NanTensorHook(loss), - _LoggerHook()], - config=tf.ConfigProto( - log_device_placement=FLAGS.log_device_placement)) as mon_sess: - while not mon_sess.should_stop(): - mon_sess.run(train_op) - # Update the masks - mon_sess.run(mask_update_op) - - -def main(argv=None): # pylint: disable=unused-argument - cifar10.maybe_download_and_extract() - if tf.gfile.Exists(FLAGS.train_dir): - tf.gfile.DeleteRecursively(FLAGS.train_dir) - tf.gfile.MakeDirs(FLAGS.train_dir) - train() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument( - '--train_dir', - type=str, - default='/tmp/cifar10_train', - help='Directory where to write event logs and checkpoint.') - parser.add_argument( - '--pruning_hparams', - type=str, - default='', - help="""Comma separated list of pruning-related hyperparameters""") - parser.add_argument( - '--max_steps', - type=int, - default=1000000, - help='Number of batches to run.') - parser.add_argument( - '--log_device_placement', - type=bool, - default=False, - help='Whether to log device placement.') - - FLAGS, unparsed = parser.parse_known_args() - tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/model_pruning/python/layers/core_layers.py b/tensorflow/contrib/model_pruning/python/layers/core_layers.py deleted file mode 100644 index 1fa5c8cb485..00000000000 --- a/tensorflow/contrib/model_pruning/python/layers/core_layers.py +++ /dev/null @@ -1,478 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains the core layer classes for model pruning and its functional aliases. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.keras.engine import input_spec -from tensorflow.python.layers import base -from tensorflow.python.layers import utils -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import standard_ops - -MASK_COLLECTION = 'masks' -THRESHOLD_COLLECTION = 'thresholds' -MASKED_WEIGHT_COLLECTION = 'masked_weights' -WEIGHT_COLLECTION = 'kernel' -# The 'weights' part of the name is needed for the quantization library -# to recognize that the kernel should be quantized. -MASKED_WEIGHT_NAME = 'weights/masked_weight' - - -class _MaskedConv(base.Layer): - """Abstract nD convolution layer (private, used as implementation base). - - This layer creates a convolution kernel that is convolved - (actually cross-correlated) with the layer input to produce a tensor of - outputs. The weight tensor of this layer is masked. - If `use_bias` is True (and a `bias_initializer` is provided), - a bias vector is created and added to the outputs. Finally, if - `activation` is not `None`, it is applied to the outputs as well. - - Arguments: - rank: An integer, the rank of the convolution, e.g. "2" for 2D convolution. - filters: Integer, the dimensionality of the output space (i.e. the number - of filters in the convolution). - kernel_size: An integer or tuple/list of n integers, specifying the - length of the convolution window. - strides: An integer or tuple/list of n integers, - specifying the stride length of the convolution. - Specifying any stride value != 1 is incompatible with specifying - any `dilation_rate` value != 1. - padding: One of `"valid"` or `"same"` (case-insensitive). - data_format: A string, one of `channels_last` (default) or `channels_first`. - The ordering of the dimensions in the inputs. - `channels_last` corresponds to inputs with shape - `(batch, ..., channels)` while `channels_first` corresponds to - inputs with shape `(batch, channels, ...)`. - dilation_rate: An integer or tuple/list of n integers, specifying - the dilation rate to use for dilated convolution. - Currently, specifying any `dilation_rate` value != 1 is - incompatible with specifying any `strides` value != 1. - activation: Activation function. Set it to None to maintain a - linear activation. - use_bias: Boolean, whether the layer uses a bias. - kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, the default - initializer will be used. - kernel_regularizer: Optional regularizer for the convolution kernel. - bias_regularizer: Optional regularizer for the bias vector. - activity_regularizer: Regularizer function for the output. - trainable: Boolean, if `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - name: A string, the name of the layer. - """ - - def __init__(self, - rank, - filters, - kernel_size, - strides=1, - padding='valid', - data_format='channels_last', - dilation_rate=1, - activation=None, - use_bias=True, - kernel_initializer=None, - bias_initializer=init_ops.zeros_initializer(), - kernel_regularizer=None, - bias_regularizer=None, - activity_regularizer=None, - trainable=True, - name=None, - **kwargs): - super(_MaskedConv, self).__init__( - trainable=trainable, - name=name, - activity_regularizer=activity_regularizer, - **kwargs) - self.rank = rank - self.filters = filters - self.kernel_size = utils.normalize_tuple(kernel_size, rank, 'kernel_size') - self.strides = utils.normalize_tuple(strides, rank, 'strides') - self.padding = utils.normalize_padding(padding) - self.data_format = utils.normalize_data_format(data_format) - self.dilation_rate = utils.normalize_tuple(dilation_rate, rank, - 'dilation_rate') - self.activation = activation - self.use_bias = use_bias - self.kernel_initializer = kernel_initializer - self.bias_initializer = bias_initializer - self.kernel_regularizer = kernel_regularizer - self.bias_regularizer = bias_regularizer - self.input_spec = input_spec.InputSpec(ndim=self.rank + 2) - - def build(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape) - channel_axis = 1 if self.data_format == 'channels_first' else -1 - if tensor_shape.dimension_value(input_shape[channel_axis]) is None: - raise ValueError('The channel dimension of the inputs ' - 'should be defined. Found `None`.') - input_dim = tensor_shape.dimension_value(input_shape[channel_axis]) - kernel_shape = self.kernel_size + (input_dim, self.filters) - self.mask = self.add_variable( - name='mask', - shape=kernel_shape, - initializer=init_ops.ones_initializer(), - trainable=False, - dtype=self.dtype) - - self.kernel = self.add_variable( - name='kernel', - shape=kernel_shape, - initializer=self.kernel_initializer, - regularizer=self.kernel_regularizer, - trainable=True, - dtype=self.dtype) - - self.threshold = self.add_variable( - name='threshold', - shape=[], - initializer=init_ops.zeros_initializer(), - trainable=False, - dtype=self.dtype) - - # Add masked_weights in the weights namescope so as to make it easier - # for the quantization library to add quant ops. - self.masked_kernel = math_ops.multiply(self.mask, self.kernel, - MASKED_WEIGHT_NAME) - - ops.add_to_collection(MASK_COLLECTION, self.mask) - ops.add_to_collection(MASKED_WEIGHT_COLLECTION, self.masked_kernel) - ops.add_to_collection(THRESHOLD_COLLECTION, self.threshold) - ops.add_to_collection(WEIGHT_COLLECTION, self.kernel) - - if self.use_bias: - self.bias = self.add_variable( - name='bias', - shape=(self.filters,), - initializer=self.bias_initializer, - regularizer=self.bias_regularizer, - trainable=True, - dtype=self.dtype) - else: - self.bias = None - self.input_spec = input_spec.InputSpec( - ndim=self.rank + 2, axes={channel_axis: input_dim}) - self.built = True - - def call(self, inputs): - outputs = nn.convolution( - input=inputs, - filter=self.masked_kernel, - dilation_rate=self.dilation_rate, - strides=self.strides, - padding=self.padding.upper(), - data_format=utils.convert_data_format(self.data_format, self.rank + 2)) - - if self.bias is not None: - if self.data_format == 'channels_first': - if self.rank == 1: - # nn.bias_add does not accept a 1D input tensor. - bias = array_ops.reshape(self.bias, (1, self.filters, 1)) - outputs += bias - if self.rank == 2: - outputs = nn.bias_add(outputs, self.bias, data_format='NCHW') - if self.rank == 3: - # As of Mar 2017, direct addition is significantly slower than - # bias_add when computing gradients. To use bias_add, we collapse Z - # and Y into a single dimension to obtain a 4D input tensor. - outputs_shape = outputs.shape.as_list() - outputs_4d = array_ops.reshape(outputs, [ - outputs_shape[0], outputs_shape[1], - outputs_shape[2] * outputs_shape[3], outputs_shape[4] - ]) - outputs_4d = nn.bias_add(outputs_4d, self.bias, data_format='NCHW') - outputs = array_ops.reshape(outputs_4d, outputs_shape) - else: - outputs = nn.bias_add(outputs, self.bias, data_format='NHWC') - - if self.activation is not None: - return self.activation(outputs) - return outputs - - def compute_output_shape(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape).as_list() - if self.data_format == 'channels_last': - space = input_shape[1:-1] - new_space = [] - for i in range(len(space)): - new_dim = utils.conv_output_length( - space[i], - self.kernel_size[i], - padding=self.padding, - stride=self.strides[i], - dilation=self.dilation_rate[i]) - new_space.append(new_dim) - return tensor_shape.TensorShape([input_shape[0]] + new_space + - [self.filters]) - else: - space = input_shape[2:] - new_space = [] - for i in range(len(space)): - new_dim = utils.conv_output_length( - space[i], - self.kernel_size[i], - padding=self.padding, - stride=self.strides[i], - dilation=self.dilation_rate[i]) - new_space.append(new_dim) - return tensor_shape.TensorShape([input_shape[0], self.filters] + - new_space) - - -class MaskedConv2D(_MaskedConv): - """2D convolution layer (e.g. spatial convolution over images). - - This layer creates a convolution kernel that is convolved - (actually cross-correlated) with the layer input to produce a tensor of - outputs. If `use_bias` is True (and a `bias_initializer` is provided), - a bias vector is created and added to the outputs. Finally, if - `activation` is not `None`, it is applied to the outputs as well. - - Arguments: - filters: Integer, the dimensionality of the output space (i.e. the number - of filters in the convolution). - kernel_size: An integer or tuple/list of 2 integers, specifying the - height and width of the 2D convolution window. - Can be a single integer to specify the same value for - all spatial dimensions. - strides: An integer or tuple/list of 2 integers, - specifying the strides of the convolution along the height and width. - Can be a single integer to specify the same value for - all spatial dimensions. - Specifying any stride value != 1 is incompatible with specifying - any `dilation_rate` value != 1. - padding: One of `"valid"` or `"same"` (case-insensitive). - data_format: A string, one of `channels_last` (default) or `channels_first`. - The ordering of the dimensions in the inputs. - `channels_last` corresponds to inputs with shape - `(batch, height, width, channels)` while `channels_first` corresponds to - inputs with shape `(batch, channels, height, width)`. - - dilation_rate: An integer or tuple/list of 2 integers, specifying - the dilation rate to use for dilated convolution. - Can be a single integer to specify the same value for - all spatial dimensions. - Currently, specifying any `dilation_rate` value != 1 is - incompatible with specifying any stride value != 1. - activation: Activation function. Set it to None to maintain a - linear activation. - use_bias: Boolean, whether the layer uses a bias. - kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, the default - initializer will be used. - kernel_regularizer: Optional regularizer for the convolution kernel. - bias_regularizer: Optional regularizer for the bias vector. - activity_regularizer: Regularizer function for the output. - trainable: Boolean, if `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - name: A string, the name of the layer. - """ - - def __init__(self, - filters, - kernel_size, - strides=(1, 1), - padding='valid', - data_format='channels_last', - dilation_rate=(1, 1), - activation=None, - use_bias=True, - kernel_initializer=None, - bias_initializer=init_ops.zeros_initializer(), - kernel_regularizer=None, - bias_regularizer=None, - activity_regularizer=None, - trainable=True, - name=None, - **kwargs): - super(MaskedConv2D, self).__init__( - rank=2, - filters=filters, - kernel_size=kernel_size, - strides=strides, - padding=padding, - data_format=data_format, - dilation_rate=dilation_rate, - activation=activation, - use_bias=use_bias, - kernel_initializer=kernel_initializer, - bias_initializer=bias_initializer, - kernel_regularizer=kernel_regularizer, - bias_regularizer=bias_regularizer, - activity_regularizer=activity_regularizer, - trainable=trainable, - name=name, - **kwargs) - - -class MaskedFullyConnected(base.Layer): - """Fully-connected layer class with masked weights. - - This layer implements the operation: - `outputs = activation(inputs.kernel + bias)` - Where `activation` is the activation function passed as the `activation` - argument (if not `None`), `kernel` is a weights matrix created by the layer, - and `bias` is a bias vector created by the layer - (only if `use_bias` is `True`). - - Note: if the input to the layer has a rank greater than 2, then it is - flattened prior to the initial matrix multiply by `kernel`. - - Arguments: - units: Integer or Long, dimensionality of the output space. - activation: Activation function (callable). Set it to None to maintain a - linear activation. - use_bias: Boolean, whether the layer uses a bias. - kernel_initializer: Initializer function for the weight matrix. - bias_initializer: Initializer function for the bias. - kernel_regularizer: Regularizer function for the weight matrix. - bias_regularizer: Regularizer function for the bias. - activity_regularizer: Regularizer function for the output. - trainable: Boolean, if `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - name: String, the name of the layer. Layers with the same name will - share weights, but to avoid mistakes we require reuse=True in such cases. - reuse: Boolean, whether to reuse the weights of a previous layer - by the same name. - - Properties: - units: Python integer, dimensionality of the output space. - activation: Activation function (callable). - use_bias: Boolean, whether the layer uses a bias. - kernel_initializer: Initializer instance (or name) for the weight matrix. - bias_initializer: Initializer instance (or name) for the bias. - kernel_regularizer: Regularizer instance for the weight matrix (callable) - bias_regularizer: Regularizer instance for the bias (callable). - activity_regularizer: Regularizer instance for the output (callable) - kernel: Weight matrix (TensorFlow variable or tensor). - bias: Bias vector, if applicable (TensorFlow variable or tensor). - """ - - def __init__(self, - units, - activation=None, - use_bias=True, - kernel_initializer=None, - bias_initializer=init_ops.zeros_initializer(), - kernel_regularizer=None, - bias_regularizer=None, - activity_regularizer=None, - trainable=True, - name=None, - **kwargs): - super(MaskedFullyConnected, self).__init__( - trainable=trainable, - name=name, - activity_regularizer=activity_regularizer, - **kwargs) - self.units = units - self.activation = activation - self.use_bias = use_bias - self.kernel_initializer = kernel_initializer - self.bias_initializer = bias_initializer - self.kernel_regularizer = kernel_regularizer - self.bias_regularizer = bias_regularizer - self.input_spec = input_spec.InputSpec(min_ndim=2) - - def build(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape) - if tensor_shape.dimension_value(input_shape[-1]) is None: - raise ValueError('The last dimension of the inputs to `Dense` ' - 'should be defined. Found `None`.') - self.input_spec = input_spec.InputSpec( - min_ndim=2, axes={-1: tensor_shape.dimension_value(input_shape[-1])}) - - self.kernel = self.add_variable( - 'kernel', - shape=[tensor_shape.dimension_value(input_shape[-1]), self.units], - initializer=self.kernel_initializer, - regularizer=self.kernel_regularizer, - dtype=self.dtype, - trainable=True) - - self.mask = self.add_variable( - name='mask', - shape=[tensor_shape.dimension_value(input_shape[-1]), self.units], - initializer=init_ops.ones_initializer(), - trainable=False, - dtype=self.dtype) - - self.threshold = self.add_variable( - name='threshold', - shape=[], - initializer=init_ops.zeros_initializer(), - trainable=False, - dtype=self.dtype) - - # Add masked_weights in the weights namescope so as to make it easier - # for the quantization library to add quant ops. - self.masked_kernel = math_ops.multiply(self.mask, self.kernel, - MASKED_WEIGHT_NAME) - - ops.add_to_collection(MASK_COLLECTION, self.mask) - ops.add_to_collection(MASKED_WEIGHT_COLLECTION, self.masked_kernel) - ops.add_to_collection(THRESHOLD_COLLECTION, self.threshold) - ops.add_to_collection(WEIGHT_COLLECTION, self.kernel) - - if self.use_bias: - self.bias = self.add_variable( - 'bias', - shape=[ - self.units, - ], - initializer=self.bias_initializer, - regularizer=self.bias_regularizer, - dtype=self.dtype, - trainable=True) - else: - self.bias = None - self.built = True - - def call(self, inputs): - inputs = ops.convert_to_tensor(inputs, dtype=self.dtype) - shape = inputs.get_shape().as_list() - output_shape = shape[:-1] + [self.units] - if len(output_shape) > 2: - # Broadcasting is required for the inputs. - outputs = standard_ops.tensordot(inputs, self.masked_kernel, - [[len(shape) - 1], [0]]) - # Reshape the output back to the original ndim of the input. - outputs.set_shape(output_shape) - else: - outputs = standard_ops.matmul(inputs, self.masked_kernel) - if self.use_bias: - outputs = nn.bias_add(outputs, self.bias) - if self.activation is not None: - return self.activation(outputs) # pylint: disable=not-callable - return outputs - - def compute_output_shape(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape) - input_shape = input_shape.with_rank_at_least(2) - if tensor_shape.dimension_value(input_shape[-1]) is None: - raise ValueError( - 'The innermost dimension of input_shape must be defined, but saw: %s' - % input_shape) - return input_shape[:-1].concatenate(self.units) diff --git a/tensorflow/contrib/model_pruning/python/layers/layers.py b/tensorflow/contrib/model_pruning/python/layers/layers.py deleted file mode 100644 index d453e350f05..00000000000 --- a/tensorflow/contrib/model_pruning/python/layers/layers.py +++ /dev/null @@ -1,363 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tensorflow layers with added variables for parameter masking. - -Branched from tensorflow/contrib/layers/python/layers/layers.py -""" -# pylint: disable=missing-docstring -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import six - -from tensorflow.contrib.framework.python.ops import add_arg_scope -from tensorflow.contrib.framework.python.ops import variables -from tensorflow.contrib.layers.python.layers import initializers -from tensorflow.contrib.layers.python.layers import utils -from tensorflow.contrib.model_pruning.python.layers import core_layers as core -from tensorflow.python.framework import ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables as tf_variables - - -def _model_variable_getter(getter, - name, - shape=None, - dtype=None, - initializer=None, - regularizer=None, - trainable=True, - collections=None, - caching_device=None, - partitioner=None, - rename=None, - use_resource=None, - **_): - """Getter that uses model_variable for compatibility with core layers.""" - short_name = name.split('/')[-1] - if rename and short_name in rename: - name_components = name.split('/') - name_components[-1] = rename[short_name] - name = '/'.join(name_components) - return variables.model_variable( - name, - shape=shape, - dtype=dtype, - initializer=initializer, - regularizer=regularizer, - collections=collections, - trainable=trainable, - caching_device=caching_device, - partitioner=partitioner, - custom_getter=getter, - use_resource=use_resource) - - -def _build_variable_getter(rename=None): - """Build a model variable getter that respects scope getter and renames.""" - - # VariableScope will nest the getters - def layer_variable_getter(getter, *args, **kwargs): - kwargs['rename'] = rename - return _model_variable_getter(getter, *args, **kwargs) - - return layer_variable_getter - - -def _add_variable_to_collections(variable, collections_set, collections_name): - """Adds variable (or all its parts) to all collections with that name.""" - collections = utils.get_variable_collections(collections_set, - collections_name) or [] - variables_list = [variable] - if isinstance(variable, tf_variables.PartitionedVariable): - variables_list = [v for v in variable] - for collection in collections: - for var in variables_list: - if var not in ops.get_collection(collection): - ops.add_to_collection(collection, var) - - -@add_arg_scope -def masked_convolution(inputs, - num_outputs, - kernel_size, - stride=1, - padding='SAME', - data_format=None, - rate=1, - activation_fn=nn.relu, - normalizer_fn=None, - normalizer_params=None, - weights_initializer=initializers.xavier_initializer(), - weights_regularizer=None, - biases_initializer=init_ops.zeros_initializer(), - biases_regularizer=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - scope=None): - """Adds an 2D convolution followed by an optional batch_norm layer. - The layer creates a mask variable on top of the weight variable. The input to - the convolution operation is the elementwise multiplication of the mask - variable and the weigh - - It is required that 1 <= N <= 3. - - `convolution` creates a variable called `weights`, representing the - convolutional kernel, that is convolved (actually cross-correlated) with the - `inputs` to produce a `Tensor` of activations. If a `normalizer_fn` is - provided (such as `batch_norm`), it is then applied. Otherwise, if - `normalizer_fn` is None and a `biases_initializer` is provided then a `biases` - variable would be created and added the activations. Finally, if - `activation_fn` is not `None`, it is applied to the activations as well. - - Performs atrous convolution with input stride/dilation rate equal to `rate` - if a value > 1 for any dimension of `rate` is specified. In this case - `stride` values != 1 are not supported. - - Args: - inputs: A Tensor of rank N+2 of shape - `[batch_size] + input_spatial_shape + [in_channels]` if data_format does - not start with "NC" (default), or - `[batch_size, in_channels] + input_spatial_shape` if data_format starts - with "NC". - num_outputs: Integer, the number of output filters. - kernel_size: A sequence of N positive integers specifying the spatial - dimensions of the filters. Can be a single integer to specify the same - value for all spatial dimensions. - stride: A sequence of N positive integers specifying the stride at which to - compute output. Can be a single integer to specify the same value for all - spatial dimensions. Specifying any `stride` value != 1 is incompatible - with specifying any `rate` value != 1. - padding: One of `"VALID"` or `"SAME"`. - data_format: A string or None. Specifies whether the channel dimension of - the `input` and output is the last dimension (default, or if `data_format` - does not start with "NC"), or the second dimension (if `data_format` - starts with "NC"). For N=1, the valid values are "NWC" (default) and - "NCW". For N=2, the valid values are "NHWC" (default) and "NCHW". - For N=3, the valid values are "NDHWC" (default) and "NCDHW". - rate: A sequence of N positive integers specifying the dilation rate to use - for atrous convolution. Can be a single integer to specify the same - value for all spatial dimensions. Specifying any `rate` value != 1 is - incompatible with specifying any `stride` value != 1. - activation_fn: Activation function. The default value is a ReLU function. - Explicitly set it to None to skip it and maintain a linear activation. - normalizer_fn: Normalization function to use instead of `biases`. If - `normalizer_fn` is provided then `biases_initializer` and - `biases_regularizer` are ignored and `biases` are not created nor added. - default set to None for no normalizer function - normalizer_params: Normalization function parameters. - weights_initializer: An initializer for the weights. - weights_regularizer: Optional regularizer for the weights. - biases_initializer: An initializer for the biases. If None skip biases. - biases_regularizer: Optional regularizer for the biases. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional list of collections for all the variables or - a dictionary containing a different list of collection per variable. - outputs_collections: Collection to add the outputs. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). - scope: Optional scope for `variable_scope`. - - Returns: - A tensor representing the output of the operation. - - Raises: - ValueError: If `data_format` is invalid. - ValueError: Both 'rate' and `stride` are not uniformly 1. - """ - if data_format not in [None, 'NWC', 'NCW', 'NHWC', 'NCHW', 'NDHWC', 'NCDHW']: - raise ValueError('Invalid data_format: %r' % (data_format,)) - - layer_variable_getter = _build_variable_getter({ - 'bias': 'biases', - 'kernel': 'weights' - }) - - with variable_scope.variable_scope( - scope, 'Conv', [inputs], reuse=reuse, - custom_getter=layer_variable_getter) as sc: - inputs = ops.convert_to_tensor(inputs) - input_rank = inputs.get_shape().ndims - - if input_rank == 3: - raise ValueError('Sparse Convolution not supported for input with rank', - input_rank) - elif input_rank == 4: - layer_class = core.MaskedConv2D - elif input_rank == 5: - raise ValueError('Sparse Convolution not supported for input with rank', - input_rank) - else: - raise ValueError('Sparse Convolution not supported for input with rank', - input_rank) - - if data_format is None or data_format == 'NHWC': - df = 'channels_last' - elif data_format == 'NCHW': - df = 'channels_first' - else: - raise ValueError('Unsupported data format', data_format) - - layer = layer_class( - filters=num_outputs, - kernel_size=kernel_size, - strides=stride, - padding=padding, - data_format=df, - dilation_rate=rate, - activation=None, - use_bias=not normalizer_fn and biases_initializer, - kernel_initializer=weights_initializer, - bias_initializer=biases_initializer, - kernel_regularizer=weights_regularizer, - bias_regularizer=biases_regularizer, - activity_regularizer=None, - trainable=trainable, - name=sc.name, - dtype=inputs.dtype.base_dtype, - _scope=sc, - _reuse=reuse) - outputs = layer.apply(inputs) - - # Add variables to collections. - _add_variable_to_collections(layer.kernel, variables_collections, 'weights') - if layer.use_bias: - _add_variable_to_collections(layer.bias, variables_collections, 'biases') - - if normalizer_fn is not None: - normalizer_params = normalizer_params or {} - outputs = normalizer_fn(outputs, **normalizer_params) - - if activation_fn is not None: - outputs = activation_fn(outputs) - return utils.collect_named_outputs(outputs_collections, - sc.original_name_scope, outputs) - - -masked_conv2d = masked_convolution - - -@add_arg_scope -def masked_fully_connected( - inputs, - num_outputs, - activation_fn=nn.relu, - normalizer_fn=None, - normalizer_params=None, - weights_initializer=initializers.xavier_initializer(), - weights_regularizer=None, - biases_initializer=init_ops.zeros_initializer(), - biases_regularizer=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - scope=None): - """Adds a sparse fully connected layer. The weight matrix is masked. - - `fully_connected` creates a variable called `weights`, representing a fully - connected weight matrix, which is multiplied by the `inputs` to produce a - `Tensor` of hidden units. If a `normalizer_fn` is provided (such as - `batch_norm`), it is then applied. Otherwise, if `normalizer_fn` is - None and a `biases_initializer` is provided then a `biases` variable would be - created and added the hidden units. Finally, if `activation_fn` is not `None`, - it is applied to the hidden units as well. - - Note: that if `inputs` have a rank greater than 2, then `inputs` is flattened - prior to the initial matrix multiply by `weights`. - - Args: - inputs: A tensor of at least rank 2 and static value for the last dimension; - i.e. `[batch_size, depth]`, `[None, None, None, channels]`. - num_outputs: Integer or long, the number of output units in the layer. - activation_fn: Activation function. The default value is a ReLU function. - Explicitly set it to None to skip it and maintain a linear activation. - normalizer_fn: Normalization function to use instead of `biases`. If - `normalizer_fn` is provided then `biases_initializer` and - `biases_regularizer` are ignored and `biases` are not created nor added. - default set to None for no normalizer function - normalizer_params: Normalization function parameters. - weights_initializer: An initializer for the weights. - weights_regularizer: Optional regularizer for the weights. - biases_initializer: An initializer for the biases. If None skip biases. - biases_regularizer: Optional regularizer for the biases. - reuse: Whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - variables_collections: Optional list of collections for all the variables or - a dictionary containing a different list of collections per variable. - outputs_collections: Collection to add the outputs. - trainable: If `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). - scope: Optional scope for variable_scope. - - Returns: - The tensor variable representing the result of the series of operations. - - Raises: - ValueError: If x has rank less than 2 or if its last dimension is not set. - """ - if not isinstance(num_outputs, six.integer_types): - raise ValueError('num_outputs should be int or long, got %s.' % - (num_outputs,)) - - layer_variable_getter = _build_variable_getter({ - 'bias': 'biases', - 'kernel': 'weights' - }) - - with variable_scope.variable_scope( - scope, - 'fully_connected', [inputs], - reuse=reuse, - custom_getter=layer_variable_getter) as sc: - inputs = ops.convert_to_tensor(inputs) - layer = core.MaskedFullyConnected( - units=num_outputs, - activation=None, - use_bias=not normalizer_fn and biases_initializer, - kernel_initializer=weights_initializer, - bias_initializer=biases_initializer, - kernel_regularizer=weights_regularizer, - bias_regularizer=biases_regularizer, - activity_regularizer=None, - trainable=trainable, - name=sc.name, - dtype=inputs.dtype.base_dtype, - _scope=sc, - _reuse=reuse) - outputs = layer.apply(inputs) - - # Add variables to collections. - _add_variable_to_collections(layer.kernel, variables_collections, 'weights') - if layer.bias is not None: - _add_variable_to_collections(layer.bias, variables_collections, 'biases') - - # Apply normalizer function / layer. - if normalizer_fn is not None: - if not normalizer_params: - normalizer_params = {} - outputs = normalizer_fn(outputs, **normalizer_params) - - if activation_fn is not None: - outputs = activation_fn(outputs) - - return utils.collect_named_outputs(outputs_collections, - sc.original_name_scope, outputs) diff --git a/tensorflow/contrib/model_pruning/python/layers/layers_test.py b/tensorflow/contrib/model_pruning/python/layers/layers_test.py deleted file mode 100644 index 97a2c978509..00000000000 --- a/tensorflow/contrib/model_pruning/python/layers/layers_test.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for imagingvision.intelligence.tensorflow.model_pruning.layers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.model_pruning.python.layers import core_layers -from tensorflow.contrib.model_pruning.python.layers import layers -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class MaskedConvolutionLayerTest(test.TestCase): - - def setUp(self): - super(MaskedConvolutionLayerTest, self).setUp() - self.height, self.width = 7, 9 - - def testInvalidRank3(self): - input_tensor = array_ops.ones((self.height, self.width, 3)) - with self.assertRaisesRegexp(ValueError, 'rank'): - layers.masked_conv2d(input_tensor, 32, 3) - - def testInvalidRank5(self): - input_tensor = array_ops.ones((8, 8, self.height, self.width, 3)) - with self.assertRaisesRegexp(ValueError, 'rank'): - layers.masked_conv2d(input_tensor, 32, 3) - - def testSingleConvMaskAdded(self): - kernel_size = 3 - input_depth, output_depth = 8, 32 - input_tensor = array_ops.ones((8, self.height, self.width, input_depth)) - layers.masked_conv2d(input_tensor, output_depth, kernel_size) - - masks = ops.get_collection(core_layers.MASK_COLLECTION) - self.assertEqual(len(masks), 1) - self.assertListEqual(masks[0].get_shape().as_list(), - [kernel_size, kernel_size, input_depth, output_depth]) - - masked_weight = ops.get_collection(core_layers.MASKED_WEIGHT_COLLECTION) - self.assertEqual(len(masked_weight), 1) - self.assertListEqual(masked_weight[0].get_shape().as_list(), - [kernel_size, kernel_size, input_depth, output_depth]) - - def testMultipleConvMaskAdded(self): - number_of_layers = 5 - - kernel_size = 3 - base_depth = 4 - depth_step = 7 - - input_tensor = array_ops.ones((8, self.height, self.width, base_depth)) - - top_layer = input_tensor - - for ix in range(number_of_layers): - top_layer = layers.masked_conv2d(top_layer, base_depth + - (ix + 1) * depth_step, kernel_size) - - masks = ops.get_collection(core_layers.MASK_COLLECTION) - self.assertEqual(len(masks), number_of_layers) - for ix in range(number_of_layers): - self.assertListEqual(masks[ix].get_shape().as_list(), [ - kernel_size, kernel_size, base_depth + ix * depth_step, - base_depth + (ix + 1) * depth_step - ]) - - masked_weight = ops.get_collection(core_layers.MASKED_WEIGHT_COLLECTION) - self.assertEqual(len(masked_weight), number_of_layers) - for ix in range(number_of_layers): - self.assertListEqual(masked_weight[ix].get_shape().as_list(), [ - kernel_size, kernel_size, base_depth + ix * depth_step, - base_depth + (ix + 1) * depth_step - ]) - - -class MaskedFullyConnectedLayerTest(test.TestCase): - - def testSingleFCMaskAdded(self): - input_depth, output_depth = 8, 32 - input_tensor = array_ops.ones((5, input_depth)) - layers.masked_fully_connected(input_tensor, output_depth) - - masks = ops.get_collection(core_layers.MASK_COLLECTION) - self.assertEqual(len(masks), 1) - self.assertListEqual(masks[0].get_shape().as_list(), - [input_depth, output_depth]) - - masked_weight = ops.get_collection(core_layers.MASKED_WEIGHT_COLLECTION) - self.assertEqual(len(masked_weight), 1) - self.assertListEqual(masked_weight[0].get_shape().as_list(), - [input_depth, output_depth]) - - def testMultipleConvMaskAdded(self): - number_of_layers = 5 - - base_depth = 4 - depth_step = 7 - - input_tensor = array_ops.ones((8, base_depth)) - - top_layer = input_tensor - - for ix in range(number_of_layers): - top_layer = layers.masked_fully_connected(top_layer, base_depth + - (ix + 1) * depth_step) - - masks = ops.get_collection(core_layers.MASK_COLLECTION) - self.assertEqual(len(masks), number_of_layers) - for ix in range(number_of_layers): - self.assertListEqual(masks[ix].get_shape().as_list(), [ - base_depth + ix * depth_step, base_depth + (ix + 1) * depth_step - ]) - - masked_weight = ops.get_collection(core_layers.MASKED_WEIGHT_COLLECTION) - self.assertEqual(len(masked_weight), number_of_layers) - for ix in range(number_of_layers): - self.assertListEqual(masked_weight[ix].get_shape().as_list(), [ - base_depth + ix * depth_step, base_depth + (ix + 1) * depth_step - ]) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/model_pruning/python/layers/rnn_cells.py b/tensorflow/contrib/model_pruning/python/layers/rnn_cells.py deleted file mode 100644 index abbc7d01e2d..00000000000 --- a/tensorflow/contrib/model_pruning/python/layers/rnn_cells.py +++ /dev/null @@ -1,348 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Module implementing RNN Cells with pruning. - -This module implements BasicLSTMCell and LSTMCell with pruning. -Code adapted from third_party/tensorflow/python/ops/rnn_cell_impl.py -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.model_pruning.python.layers import core_layers -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import rnn_cell as tf_rnn - - -class MaskedBasicLSTMCell(tf_rnn.BasicLSTMCell): - """Basic LSTM recurrent network cell with pruning. - - Overrides the call method of tensorflow BasicLSTMCell and injects the weight - masks - - The implementation is based on: http://arxiv.org/abs/1409.2329. - - We add forget_bias (default: 1) to the biases of the forget gate in order to - reduce the scale of forgetting in the beginning of the training. - - It does not allow cell clipping, a projection layer, and does not - use peep-hole connections: it is the basic baseline. - - For advanced models, please use the full `tf.compat.v1.nn.rnn_cell.LSTMCell` - that follows. - """ - - def __init__(self, - num_units, - forget_bias=1.0, - state_is_tuple=True, - activation=None, - reuse=None, - name=None): - """Initialize the basic LSTM cell with pruning. - - Args: - num_units: int, The number of units in the LSTM cell. - forget_bias: float, The bias added to forget gates (see above). - Must set to `0.0` manually when restoring from CudnnLSTM-trained - checkpoints. - state_is_tuple: If True, accepted and returned states are 2-tuples of - the `c_state` and `m_state`. If False, they are concatenated - along the column axis. The latter behavior will soon be deprecated. - activation: Activation function of the inner states. Default: `tanh`. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - name: String, the name of the layer. Layers with the same name will - share weights, but to avoid mistakes we require reuse=True in such - cases. - - When restoring from CudnnLSTM-trained checkpoints, must use - CudnnCompatibleLSTMCell instead. - """ - super(MaskedBasicLSTMCell, self).__init__( - num_units, - forget_bias=forget_bias, - state_is_tuple=state_is_tuple, - activation=activation, - reuse=reuse, - name=name) - - def build(self, inputs_shape): - # Call the build method of the parent class. - super(MaskedBasicLSTMCell, self).build(inputs_shape) - - self.built = False - - input_depth = inputs_shape.dims[1].value - h_depth = self._num_units - self._mask = self.add_variable( - name="mask", - shape=[input_depth + h_depth, 4 * h_depth], - initializer=init_ops.ones_initializer(), - trainable=False, - dtype=self.dtype) - self._threshold = self.add_variable( - name="threshold", - shape=[], - initializer=init_ops.zeros_initializer(), - trainable=False, - dtype=self.dtype) - # Add masked_weights in the weights namescope so as to make it easier - # for the quantization library to add quant ops. - self._masked_kernel = math_ops.multiply(self._mask, self._kernel, - core_layers.MASKED_WEIGHT_NAME) - if self._mask not in ops.get_collection_ref(core_layers.MASK_COLLECTION): - ops.add_to_collection(core_layers.MASK_COLLECTION, self._mask) - ops.add_to_collection(core_layers.MASKED_WEIGHT_COLLECTION, - self._masked_kernel) - ops.add_to_collection(core_layers.THRESHOLD_COLLECTION, self._threshold) - ops.add_to_collection(core_layers.WEIGHT_COLLECTION, self._kernel) - - self.built = True - - def call(self, inputs, state): - """Long short-term memory cell (LSTM) with masks for pruning. - - Args: - inputs: `2-D` tensor with shape `[batch_size, input_size]`. - state: An `LSTMStateTuple` of state tensors, each shaped - `[batch_size, self.state_size]`, if `state_is_tuple` has been set to - `True`. Otherwise, a `Tensor` shaped - `[batch_size, 2 * self.state_size]`. - - Returns: - A pair containing the new hidden state, and the new state (either a - `LSTMStateTuple` or a concatenated state, depending on - `state_is_tuple`). - """ - sigmoid = math_ops.sigmoid - one = constant_op.constant(1, dtype=dtypes.int32) - # Parameters of gates are concatenated into one multiply for efficiency. - if self._state_is_tuple: - c, h = state - else: - c, h = array_ops.split(value=state, num_or_size_splits=2, axis=one) - - gate_inputs = math_ops.matmul( - array_ops.concat([inputs, h], 1), self._masked_kernel) - gate_inputs = nn_ops.bias_add(gate_inputs, self._bias) - - # i = input_gate, j = new_input, f = forget_gate, o = output_gate - i, j, f, o = array_ops.split( - value=gate_inputs, num_or_size_splits=4, axis=one) - - forget_bias_tensor = constant_op.constant(self._forget_bias, dtype=f.dtype) - # Note that using `add` and `multiply` instead of `+` and `*` gives a - # performance improvement. So using those at the cost of readability. - add = math_ops.add - multiply = math_ops.multiply - new_c = add( - multiply(c, sigmoid(add(f, forget_bias_tensor))), - multiply(sigmoid(i), self._activation(j))) - new_h = multiply(self._activation(new_c), sigmoid(o)) - - if self._state_is_tuple: - new_state = tf_rnn.LSTMStateTuple(new_c, new_h) - else: - new_state = array_ops.concat([new_c, new_h], 1) - return new_h, new_state - - -class MaskedLSTMCell(tf_rnn.LSTMCell): - """LSTMCell with pruning. - - Overrides the call method of tensorflow LSTMCell and injects the weight masks. - Masks are applied to only the weight matrix of the LSTM and not the - projection matrix. - """ - - def __init__(self, - num_units, - use_peepholes=False, - cell_clip=None, - initializer=None, - num_proj=None, - proj_clip=None, - num_unit_shards=None, - num_proj_shards=None, - forget_bias=1.0, - state_is_tuple=True, - activation=None, - reuse=None): - """Initialize the parameters for an LSTM cell with masks for pruning. - - Args: - num_units: int, The number of units in the LSTM cell - use_peepholes: bool, set True to enable diagonal/peephole connections. - cell_clip: (optional) A float value, if provided the cell state is clipped - by this value prior to the cell output activation. - initializer: (optional) The initializer to use for the weight and - projection matrices. - num_proj: (optional) int, The output dimensionality for the projection - matrices. If None, no projection is performed. - proj_clip: (optional) A float value. If `num_proj > 0` and `proj_clip` is - provided, then the projected values are clipped elementwise to within - `[-proj_clip, proj_clip]`. - num_unit_shards: Deprecated, will be removed by Jan. 2017. - Use a variable_scope partitioner instead. - num_proj_shards: Deprecated, will be removed by Jan. 2017. - Use a variable_scope partitioner instead. - forget_bias: Biases of the forget gate are initialized by default to 1 - in order to reduce the scale of forgetting at the beginning of - the training. Must set it manually to `0.0` when restoring from - CudnnLSTM trained checkpoints. - state_is_tuple: If True, accepted and returned states are 2-tuples of - the `c_state` and `m_state`. If False, they are concatenated - along the column axis. This latter behavior will soon be deprecated. - activation: Activation function of the inner states. Default: `tanh`. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - - When restoring from CudnnLSTM-trained checkpoints, must use - CudnnCompatibleLSTMCell instead. - """ - super(MaskedLSTMCell, self).__init__( - num_units, - use_peepholes=use_peepholes, - cell_clip=cell_clip, - initializer=initializer, - num_proj=num_proj, - proj_clip=proj_clip, - num_unit_shards=num_unit_shards, - num_proj_shards=num_proj_shards, - forget_bias=forget_bias, - state_is_tuple=state_is_tuple, - activation=activation, - reuse=reuse) - - def build(self, inputs_shape): - # Call the build method of the parent class. - super(MaskedLSTMCell, self).build(inputs_shape) - - self.built = False - - input_depth = inputs_shape.dims[1].value - h_depth = self._num_units - self._mask = self.add_variable( - name="mask", - shape=[input_depth + h_depth, 4 * h_depth], - initializer=init_ops.ones_initializer(), - trainable=False, - dtype=self.dtype) - self._threshold = self.add_variable( - name="threshold", - shape=[], - initializer=init_ops.zeros_initializer(), - trainable=False, - dtype=self.dtype) - # Add masked_weights in the weights namescope so as to make it easier - # for the quantization library to add quant ops. - self._masked_kernel = math_ops.multiply(self._mask, self._kernel, - core_layers.MASKED_WEIGHT_NAME) - if self._mask not in ops.get_collection_ref(core_layers.MASK_COLLECTION): - ops.add_to_collection(core_layers.MASK_COLLECTION, self._mask) - ops.add_to_collection(core_layers.MASKED_WEIGHT_COLLECTION, - self._masked_kernel) - ops.add_to_collection(core_layers.THRESHOLD_COLLECTION, self._threshold) - ops.add_to_collection(core_layers.WEIGHT_COLLECTION, self._kernel) - - self.built = True - - def call(self, inputs, state): - """Run one step of LSTM. - - Args: - inputs: input Tensor, 2D, `[batch, num_units]. - state: if `state_is_tuple` is False, this must be a state Tensor, - `2-D, [batch, state_size]`. If `state_is_tuple` is True, this must be a - tuple of state Tensors, both `2-D`, with column sizes `c_state` and - `m_state`. - - Returns: - A tuple containing: - - - A `2-D, [batch, output_dim]`, Tensor representing the output of the - LSTM after reading `inputs` when previous state was `state`. - Here output_dim is: - num_proj if num_proj was set, - num_units otherwise. - - Tensor(s) representing the new state of LSTM after reading `inputs` when - the previous state was `state`. Same type and shape(s) as `state`. - - Raises: - ValueError: If input size cannot be inferred from inputs via - static shape inference. - """ - num_proj = self._num_units if self._num_proj is None else self._num_proj - sigmoid = math_ops.sigmoid - - if self._state_is_tuple: - (c_prev, m_prev) = state - else: - c_prev = array_ops.slice(state, [0, 0], [-1, self._num_units]) - m_prev = array_ops.slice(state, [0, self._num_units], [-1, num_proj]) - - input_size = inputs.get_shape().with_rank(2).dims[1] - if input_size.value is None: - raise ValueError("Could not infer input size from inputs.get_shape()[-1]") - - # i = input_gate, j = new_input, f = forget_gate, o = output_gate - lstm_matrix = math_ops.matmul( - array_ops.concat([inputs, m_prev], 1), self._masked_kernel) - lstm_matrix = nn_ops.bias_add(lstm_matrix, self._bias) - - i, j, f, o = array_ops.split( - value=lstm_matrix, num_or_size_splits=4, axis=1) - # Diagonal connections - if self._use_peepholes: - c = ( - sigmoid(f + self._forget_bias + self._w_f_diag * c_prev) * c_prev + - sigmoid(i + self._w_i_diag * c_prev) * self._activation(j)) - else: - c = ( - sigmoid(f + self._forget_bias) * c_prev + - sigmoid(i) * self._activation(j)) - - if self._cell_clip is not None: - # pylint: disable=invalid-unary-operand-type - c = clip_ops.clip_by_value(c, -self._cell_clip, self._cell_clip) - # pylint: enable=invalid-unary-operand-type - if self._use_peepholes: - m = sigmoid(o + self._w_o_diag * c) * self._activation(c) - else: - m = sigmoid(o) * self._activation(c) - - if self._num_proj is not None: - m = math_ops.matmul(m, self._proj_kernel) - - if self._proj_clip is not None: - # pylint: disable=invalid-unary-operand-type - m = clip_ops.clip_by_value(m, -self._proj_clip, self._proj_clip) - # pylint: enable=invalid-unary-operand-type - - new_state = ( - tf_rnn.LSTMStateTuple(c, m) - if self._state_is_tuple else array_ops.concat([c, m], 1)) - return m, new_state diff --git a/tensorflow/contrib/model_pruning/python/layers/rnn_cells_test.py b/tensorflow/contrib/model_pruning/python/layers/rnn_cells_test.py deleted file mode 100644 index 586c6c7bfcb..00000000000 --- a/tensorflow/contrib/model_pruning/python/layers/rnn_cells_test.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for creating different number of masks in rnn_cells.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.model_pruning.python import pruning -from tensorflow.contrib.model_pruning.python.layers import rnn_cells -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import rnn_cell as tf_rnn_cells -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class RnnCellsTest(test.TestCase): - - def setUp(self): - super(RnnCellsTest, self).setUp() - self.batch_size = 8 - self.dim = 10 - - def testMaskedBasicLSTMCell(self): - expected_num_masks = 1 - expected_num_rows = 2 * self.dim - expected_num_cols = 4 * self.dim - with self.cached_session(): - inputs = variables.Variable( - random_ops.random_normal([self.batch_size, self.dim])) - c = variables.Variable( - random_ops.random_normal([self.batch_size, self.dim])) - h = variables.Variable( - random_ops.random_normal([self.batch_size, self.dim])) - state = tf_rnn_cells.LSTMStateTuple(c, h) - lstm_cell = rnn_cells.MaskedBasicLSTMCell(self.dim) - lstm_cell(inputs, state) - self.assertEqual(len(pruning.get_masks()), expected_num_masks) - self.assertEqual(len(pruning.get_masked_weights()), expected_num_masks) - self.assertEqual(len(pruning.get_thresholds()), expected_num_masks) - self.assertEqual(len(pruning.get_weights()), expected_num_masks) - - for mask in pruning.get_masks(): - self.assertEqual(mask.shape, (expected_num_rows, expected_num_cols)) - for weight in pruning.get_weights(): - self.assertEqual(weight.shape, (expected_num_rows, expected_num_cols)) - - def testMaskedLSTMCell(self): - expected_num_masks = 1 - expected_num_rows = 2 * self.dim - expected_num_cols = 4 * self.dim - with self.cached_session(): - inputs = variables.Variable( - random_ops.random_normal([self.batch_size, self.dim])) - c = variables.Variable( - random_ops.random_normal([self.batch_size, self.dim])) - h = variables.Variable( - random_ops.random_normal([self.batch_size, self.dim])) - state = tf_rnn_cells.LSTMStateTuple(c, h) - lstm_cell = rnn_cells.MaskedLSTMCell(self.dim) - lstm_cell(inputs, state) - self.assertEqual(len(pruning.get_masks()), expected_num_masks) - self.assertEqual(len(pruning.get_masked_weights()), expected_num_masks) - self.assertEqual(len(pruning.get_thresholds()), expected_num_masks) - self.assertEqual(len(pruning.get_weights()), expected_num_masks) - - for mask in pruning.get_masks(): - self.assertEqual(mask.shape, (expected_num_rows, expected_num_cols)) - for weight in pruning.get_weights(): - self.assertEqual(weight.shape, (expected_num_rows, expected_num_cols)) - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/model_pruning/python/learning.py b/tensorflow/contrib/model_pruning/python/learning.py deleted file mode 100644 index ca34de61a7c..00000000000 --- a/tensorflow/contrib/model_pruning/python/learning.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Wrapper around tf-slim's training code contrib/slim/python/slim/learning.py - -to support training of pruned models - -******************************************************************* -* A simple working training script with support for model pruning * -******************************************************************* - - # Load data and create the model: - images, labels = LoadData(...) - predictions = MyModel(images) - - # Define the loss: - slim.losses.log_loss(predictions, labels) - total_loss = slim.losses.get_total_loss() - - # Define the optimizer: - optimizer = tf.compat.v1.train.MomentumOptimizer(FLAGS.learning_rate, - FLAGS.momentum) - - # Create the train_op - train_op = slim.learning.create_train_op(total_loss, optimizer) - - # Parse pruning hyperparameters - pruning_hparams = pruning.get_pruning_hparams().parse(FLAGS.pruning_hparams) - - # Create a pruning object using the pruning_hparams - p = pruning.Pruning(pruning_hparams) - - # Add mask update ops to the graph - mask_update_op = p.conditional_mask_update_op() - - # Run training. - learning.train(train_op, - my_log_dir, - mask_update_op) - see contrib/slim/python/slim/learning.py for additional examples -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import slim as _slim - -_USE_DEFAULT = 0 -train_step = _slim.learning.train_step - - -def train(train_op, - logdir, - mask_update_op, - train_step_fn=train_step, - train_step_kwargs=_USE_DEFAULT, - log_every_n_steps=1, - graph=None, - master='', - is_chief=True, - global_step=None, - number_of_steps=None, - init_op=_USE_DEFAULT, - init_feed_dict=None, - local_init_op=_USE_DEFAULT, - init_fn=None, - ready_op=_USE_DEFAULT, - summary_op=_USE_DEFAULT, - save_summaries_secs=600, - summary_writer=_USE_DEFAULT, - startup_delay_steps=0, - saver=None, - save_interval_secs=600, - sync_optimizer=None, - session_config=None, - trace_every_n_steps=None): - """Wrapper around tf-slim's train function. - - Runs a training loop using a TensorFlow supervisor. - When the sync_optimizer is supplied, gradient updates are applied - synchronously. Otherwise, gradient updates are applied asynchronous. - - Args: - train_op: A `Tensor` that, when executed, will apply the gradients and - return the loss value. - logdir: The directory where training logs are written to. If None, model - checkpoints and summaries will not be written. - mask_update_op: Operation that upon execution updates the weight masks and - thresholds. - train_step_fn: The function to call in order to execute a single gradient - step. The function must have take exactly four arguments: the current - session, the `train_op` `Tensor`, a global step `Tensor` and a - dictionary. - train_step_kwargs: A dictionary which is passed to the `train_step_fn`. By - default, two `Boolean`, scalar ops called "should_stop" and "should_log" - are provided. - log_every_n_steps: The frequency, in terms of global steps, that the loss - and global step and logged. - graph: The graph to pass to the supervisor. If no graph is supplied the - default graph is used. - master: The address of the tensorflow master. - is_chief: Specifies whether or not the training is being run by the primary - replica during replica training. - global_step: The `Tensor` representing the global step. If left as `None`, - then slim.variables.get_or_create_global_step() is used. - number_of_steps: The max number of gradient steps to take during training, - as measured by 'global_step': training will stop if global_step is greater - than 'number_of_steps'. If the value is left as None, training proceeds - indefinitely. - init_op: The initialization operation. If left to its default value, then - the session is initialized by calling - `tf.compat.v1.global_variables_initializer()`. - init_feed_dict: A feed dictionary to use when executing the `init_op`. - local_init_op: The local initialization operation. If left to its default - value, then the session is initialized by calling - `tf.compat.v1.local_variables_initializer()` and - `tf.compat.v1.tables_initializer()`. - init_fn: An optional callable to be executed after `init_op` is called. The - callable must accept one argument, the session being initialized. - ready_op: Operation to check if the model is ready to use. If left to its - default value, then the session checks for readiness by calling - `tf.compat.v1.report_uninitialized_variables()`. - summary_op: The summary operation. - save_summaries_secs: How often, in seconds, to save summaries. - summary_writer: `SummaryWriter` to use. Can be `None` to indicate that no - summaries should be written. If unset, we create a SummaryWriter. - startup_delay_steps: The number of steps to wait for before beginning. Note - that this must be 0 if a sync_optimizer is supplied. - saver: Saver to save checkpoints. If None, a default one will be created and - used. - save_interval_secs: How often, in seconds, to save the model to `logdir`. - sync_optimizer: an instance of tf.compat.v1.train.SyncReplicasOptimizer, or - a list of them. If the argument is supplied, gradient updates will be - synchronous. If left as `None`, gradient updates will be asynchronous. - session_config: An instance of `tf.compat.v1.ConfigProto` that will be used - to configure the `Session`. If left as `None`, the default will be used. - trace_every_n_steps: produce and save a `Timeline` in Chrome trace format - and add it to the summaries every `trace_every_n_steps`. If None, no trace - information will be produced or saved. - - Returns: - the value of the loss function after training. - - Raises: - ValueError: if `train_op` is empty or if `startup_delay_steps` is - non-zero when `sync_optimizer` is supplied, if `number_of_steps` is - negative, or if `trace_every_n_steps` is not `None` and no `logdir` is - provided. - """ - - def train_step_with_pruning_fn(sess, train_op, global_step, - train_step_kwargs): - total_loss, should_stop = train_step_fn(sess, train_op, global_step, - train_step_kwargs) - sess.run(mask_update_op) - return total_loss, should_stop - - total_loss, _ = _slim.learning.train( - train_op, - logdir, - train_step_fn=train_step_with_pruning_fn, - train_step_kwargs=train_step_kwargs, - log_every_n_steps=log_every_n_steps, - graph=graph, - master=master, - is_chief=is_chief, - global_step=global_step, - number_of_steps=number_of_steps, - init_op=init_op, - init_feed_dict=init_feed_dict, - local_init_op=local_init_op, - init_fn=init_fn, - ready_op=ready_op, - summary_op=summary_op, - save_summaries_secs=save_summaries_secs, - summary_writer=summary_writer, - startup_delay_steps=startup_delay_steps, - saver=saver, - save_interval_secs=save_interval_secs, - sync_optimizer=sync_optimizer, - session_config=session_config, - trace_every_n_steps=trace_every_n_steps) - - return total_loss diff --git a/tensorflow/contrib/model_pruning/python/pruning.py b/tensorflow/contrib/model_pruning/python/pruning.py deleted file mode 100644 index 30375c7f56e..00000000000 --- a/tensorflow/contrib/model_pruning/python/pruning.py +++ /dev/null @@ -1,622 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Helper functions to add support for magnitude-based model pruning. - - # Adds variables and ops to the graph to enable - # elementwise masking of weights - apply_mask(weights) - - # Returns a list containing the sparsity of each of the weight tensors - get_weight_sparsity() - - # Returns a list of all the masked weight tensorflow variables - get_masked_weights() - - # Returns a list of all the mask tensorflow variables - get_masks() - - # Returns a list of all the thresholds - get_thresholds() - - # Returns a list of all the weight tensors that have been masked - get_weights() - - The Pruning class uses a tf.hparams object to set up the - parameters for a model pruning. Here's a typical usage: - - # Parse pruning hyperparameters - pruning_hparams = pruning.get_pruning_hparams().parse(FLAGS.pruning_hparams) - - # Create a pruning object using the pruning_hparams - p = pruning.Pruning(pruning_hparams) - - # Add mask update ops to the graph - mask_update_op = p.conditional_mask_update_op() - - # Add the summaries - p.add_pruning_summaries() - - # Run the op - session.run(mask_update_op) - - # An object of the pruning also accepts externally defined sparsity: - sparsity = tf.Variable(0.5, name = "ConstantSparsity") - p = pruning.Pruning(pruning_hparams, sparsity=sparsity) -""" -# pylint: disable=missing-docstring -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re - -from tensorflow.contrib.model_pruning.python import pruning_utils -from tensorflow.contrib.model_pruning.python.layers import core_layers as core -from tensorflow.contrib.training.python.training import hparam -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.summary import summary -from tensorflow.python.training import training_util - -_MASK_COLLECTION = core.MASK_COLLECTION -_THRESHOLD_COLLECTION = core.THRESHOLD_COLLECTION -_MASKED_WEIGHT_COLLECTION = core.MASKED_WEIGHT_COLLECTION -_WEIGHT_COLLECTION = core.WEIGHT_COLLECTION -_MASKED_WEIGHT_NAME = core.MASKED_WEIGHT_NAME - - -def apply_mask(x, scope=''): - """Apply mask to a given weight tensor. - - Args: - x: Input weight tensor - scope: The current variable scope. Defaults to "". - Returns: - Tensor representing masked_weights - """ - - mask = pruning_utils.weight_mask_variable(x, scope) - threshold = pruning_utils.weight_threshold_variable(x, scope) - # Add masked_weights in the weights namescope so as to make it easier - # for the quantization library to add quant ops. - masked_weights = math_ops.multiply(mask, x, _MASKED_WEIGHT_NAME) - - # Make sure the mask for a given variable are not added multiple times to the - # collection. This is particularly important when applying mask to RNN's - # weight variables - if mask not in ops.get_collection_ref(_MASK_COLLECTION): - ops.add_to_collection(_THRESHOLD_COLLECTION, threshold) - ops.add_to_collection(_MASK_COLLECTION, mask) - ops.add_to_collection(_MASKED_WEIGHT_COLLECTION, masked_weights) - ops.add_to_collection(_WEIGHT_COLLECTION, x) - return masked_weights - - -def get_masked_weights(): - return ops.get_collection(_MASKED_WEIGHT_COLLECTION) - - -def get_masks(): - return ops.get_collection(_MASK_COLLECTION) - - -def get_thresholds(): - return ops.get_collection(_THRESHOLD_COLLECTION) - - -def get_weights(): - return ops.get_collection(_WEIGHT_COLLECTION) - - -def get_weight_sparsity(): - """Get sparsity of the weights. - - Args: - None - - Returns: - A list containing the sparsity of each of the weight tensors - """ - masks = get_masks() - return [nn_impl.zero_fraction(mask) for mask in masks] - - -def get_pruning_hparams(): - """Get a tf.HParams object with the default values for the hyperparameters. - - name: string - name of the pruning specification. Used for adding summaries and ops under - a common tensorflow name_scope - begin_pruning_step: integer - the global step at which to begin pruning - end_pruning_step: integer - the global step at which to terminate pruning. Defaults to -1 implying - that pruning continues till the training stops - weight_sparsity_map: list of strings - comma separed list of {weight_variable_name:target sparsity} or - {regex:target sparsity} pairs. - For layers/weights not in this list, sparsity as specified by the - target_sparsity hyperparameter is used. - Eg. [conv1:0.9,conv2/kernel:0.8] - block_dims_map: list of strings - comma separated list of {weight variable name:block_height x block_width} - or {regex:block_height x block_width} pairs. For layers/weights not in - this list, block dims are specified by the block_height, block_width - hyperparameters are used Eg. [dense1:4x4,dense2:1x16,dense3:1x1] - threshold_decay: float - the decay factor to use for exponential decay of the thresholds - pruning_frequency: integer - How often should the masks be updated? (in # of global_steps) - nbins: integer - number of bins to use for histogram computation - block_height: integer - number of rows in a block (defaults to 1), can be -1 in which - case it is set to the size of the corresponding weight tensor. - block_width: integer - number of cols in a block (defaults to 1), can be -1 in which - case it is set to the size of the corresponding weight tensor. - block_pooling_function: string - Whether to perform average (AVG) or max (MAX) pooling in the block - (default: AVG) - initial_sparsity: float - initial sparsity value - target_sparsity: float - target sparsity value - sparsity_function_begin_step: integer - the global step at this which the gradual sparsity function begins to - take effect - sparsity_function_end_step: integer - the global step used as the end point for the gradual sparsity function - sparsity_function_exponent: float - exponent = 1 is linearly varying sparsity between initial and final. - exponent > 1 varies more slowly towards the end than the beginning - use_tpu: False - Indicates whether to use TPU - - We use the following sparsity function: - - num_steps = (sparsity_function_end_step - - sparsity_function_begin_step)/pruning_frequency - sparsity(step) = (initial_sparsity - target_sparsity)* - [1-step/(num_steps -1)]**exponent + target_sparsity - - Args: - None - - Returns: - tf.HParams object initialized to default values - - """ - return hparam.HParams( - name='model_pruning', - begin_pruning_step=0, - end_pruning_step=-1, - weight_sparsity_map=[''], - block_dims_map=[''], - threshold_decay=0.0, - pruning_frequency=10, - nbins=256, - block_height=1, - block_width=1, - block_pooling_function='AVG', - initial_sparsity=0.0, - target_sparsity=0.5, - sparsity_function_begin_step=0, - sparsity_function_end_step=100, - sparsity_function_exponent=3.0, - use_tpu=False) - - -class Pruning(object): - - def __init__(self, spec=None, global_step=None, sparsity=None): - """Set up the specification for model pruning. - - If a spec is provided, the sparsity is set up based on the sparsity_function - in the spec. The effect of sparsity_function is overridden if the sparsity - variable is passed to the constructor. This enables setting up arbitrary - sparsity profiles externally and passing it to this pruning functions. - - Args: - spec: Pruning spec as defined in pruning.proto - global_step: A tensorflow variable that is used while setting up the - sparsity function - sparsity: A tensorflow scalar variable storing the sparsity - """ - # Pruning specification - self._spec = spec if spec else get_pruning_hparams() - - # Sanity check for pruning hparams - self._validate_spec() - - # A tensorflow variable that tracks the sparsity function. - # If not provided as input, the graph must already contain the global_step - # variable before calling this constructor. - self._global_step = self._setup_global_step(global_step) - - # Stores the tensorflow sparsity variable. - # Built using self._setup_sparsity() or provided externally - self._sparsity = (sparsity - if sparsity is not None else self._setup_sparsity()) - - # List of tensorflow assignments ops for new masks and thresholds - self._assign_ops = [] - - # Tensorflow variable keeping track of the last global step when the masks - # were updated - self._last_update_step = self._setup_last_update_step() - - # Block dimensions - self._block_dims = [self._spec.block_height, self._spec.block_width] - - # Block pooling function - self._block_pooling_function = self._spec.block_pooling_function - - # Mapping of layer/weight names and block dims - self._block_dims_map = self._get_block_dims_map() - - # Mapping of weight names and target sparsity - self._weight_sparsity_map = self._get_weight_sparsity_map() - - def _validate_spec(self): - spec = self._spec - if spec.begin_pruning_step < 0: - raise ValueError('Illegal value for begin_pruning_step') - - if spec.begin_pruning_step >= spec.end_pruning_step: - if spec.end_pruning_step != -1: - raise ValueError( - 'Pruning must begin before it can end. begin_step=%d, end_step=%d.' - 'Set end_pruning_step to -1 if pruning is required till training' - 'stops' % (spec.begin_pruning_step, spec.end_pruning_step)) - - if spec.sparsity_function_begin_step < 0: - raise ValueError('Illegal value for sparsity_function_begin_step') - - if spec.sparsity_function_begin_step >= spec.sparsity_function_end_step: - raise ValueError( - 'Sparsity function requires begin_step < end_step') - - if not 0.0 <= spec.threshold_decay < 1.0: - raise ValueError('threshold_decay must be in range [0,1)') - - if not 0.0 <= spec.initial_sparsity < 1.0: - raise ValueError('initial_sparsity must be in range [0,1)') - - if not 0.0 <= spec.target_sparsity < 1.0: - raise ValueError('target_sparsity must be in range [0,1)') - - def _setup_global_step(self, global_step): - graph_global_step = global_step - if graph_global_step is None: - graph_global_step = training_util.get_global_step() - - return math_ops.cast(graph_global_step, dtypes.int32) - - def _setup_sparsity(self): - begin_step = self._spec.sparsity_function_begin_step - end_step = self._spec.sparsity_function_end_step - initial_sparsity = self._spec.initial_sparsity - target_sparsity = self._spec.target_sparsity - exponent = self._spec.sparsity_function_exponent - - with ops.name_scope(self._spec.name): - p = math_ops.minimum( - 1.0, - math_ops.maximum( - 0.0, - math_ops.div( - math_ops.cast(self._global_step - begin_step, dtypes.float32), - end_step - begin_step))) - sparsity = math_ops.add( - math_ops.multiply(initial_sparsity - target_sparsity, - math_ops.pow(1 - p, exponent)), - target_sparsity, - name='sparsity') - - return sparsity - - def _setup_last_update_step(self): - with variable_scope.variable_scope( - self._spec.name, use_resource=self._spec.use_tpu) as scope: - try: - last_update_step = variable_scope.get_variable( - 'last_mask_update_step', [], - initializer=init_ops.zeros_initializer(), - trainable=False, - dtype=dtypes.int32) - except ValueError: - scope.reuse_variables() - last_update_step = variable_scope.get_variable( - 'last_mask_update_step', dtype=dtypes.int32) - return last_update_step - - def _get_block_dims_map(self): - """Returns the map of layer name: block dims.""" - block_dims_map = {} - val_list = self._spec.block_dims_map - filtered_val_list = [l for l in val_list if l] - for val in filtered_val_list: - weight_name, block_dims_str = val.split(':') - block_dims_str = block_dims_str.split('x') - if len(block_dims_str) != 2: - raise ValueError('Expected 2 values for block dim for %s, got %s' % - (weight_name, block_dims_str)) - block_dims = [int(block_dims_str[0]), int(block_dims_str[1])] - block_dims_map[re.compile(weight_name)] = block_dims - - return block_dims_map - - def _get_block_dims(self, weight_name): - """Returns the block dims for the given layer/weight name.""" - block_dims_list = [ - block_dims for regexp, block_dims in self._block_dims_map.items() - if regexp.search(weight_name) - ] - if not block_dims_list: - return self._block_dims - - if len(block_dims_list) > 1: - raise ValueError('Multiple matches in block_dims_map for weight %s' % - weight_name) - - return block_dims_list[0] - - def _get_weight_sparsity_map(self): - """Returns the map of weight_name:sparsity parsed from the hparams.""" - weight_sparsity_map = {} - val_list = self._spec.weight_sparsity_map - filtered_val_list = [l for l in val_list if l] - for val in filtered_val_list: - weight_name, sparsity = val.split(':') - if float(sparsity) >= 1.0: - raise ValueError('Weight sparsity can not exceed 1.0') - weight_sparsity_map[re.compile(weight_name)] = float(sparsity) - - return weight_sparsity_map - - def _get_sparsity(self, weight_name): - """Returns target sparsity for the given layer/weight name.""" - target_sparsity = [ - sparsity for regexp, sparsity in self._weight_sparsity_map.items() - if regexp.search(weight_name) - ] - if not target_sparsity: - return self._sparsity - - if len(target_sparsity) > 1: - raise ValueError( - 'Multiple matches in weight_sparsity_map for weight %s' % weight_name) - # TODO(suyoggupta): This will work when initial_sparsity = 0. Generalize - # to handle other cases as well. - return math_ops.mul( - self._sparsity, - math_ops.div(target_sparsity[0], self._spec.target_sparsity)) - - def _update_mask(self, weights, threshold): - """Updates the mask for a given weight tensor. - - This functions first computes the cdf of the weight tensor, and estimates - the threshold value such that 'desired_sparsity' fraction of weights - have magnitude less than the threshold. - - Args: - weights: The weight tensor that needs to be masked. - threshold: The current threshold value. The function will compute a new - threshold and return the exponential moving average using the current - value of threshold - - Returns: - new_threshold: The new value of the threshold based on weights, and - sparsity at the current global_step - new_mask: A numpy array of the same size and shape as weights containing - 0 or 1 to indicate which of the values in weights falls below - the threshold - - Raises: - ValueError: if sparsity is not defined - """ - if self._sparsity is None: - raise ValueError('Sparsity variable undefined') - - sparsity = self._get_sparsity(weights.op.name) - with ops.name_scope(weights.op.name + '_pruning_ops'): - abs_weights = math_ops.abs(weights) - k = math_ops.cast( - math_ops.round( - math_ops.cast(array_ops.size(abs_weights), dtypes.float32) * - (1 - sparsity)), dtypes.int32) - # Sort the entire array - values, _ = nn_ops.top_k( - array_ops.reshape(abs_weights, [-1]), k=array_ops.size(abs_weights)) - # Grab the (k-1) th value - current_threshold = array_ops.gather(values, k - 1) - smoothed_threshold = math_ops.add_n([ - math_ops.multiply(current_threshold, 1 - self._spec.threshold_decay), - math_ops.multiply(threshold, self._spec.threshold_decay) - ]) - - new_mask = math_ops.cast( - math_ops.greater_equal(abs_weights, smoothed_threshold), - dtypes.float32) - - return smoothed_threshold, new_mask - - def _maybe_update_block_mask(self, weights, threshold): - """Performs block-granular masking of the weights. - - Block pruning occurs only if the block_height or block_width is > 1 and - if the weight tensor, when squeezed, has ndims = 2. Otherwise, elementwise - pruning occurs. - Args: - weights: The weight tensor that needs to be masked. - threshold: The current threshold value. The function will compute a new - threshold and return the exponential moving average using the current - value of threshold - - Returns: - new_threshold: The new value of the threshold based on weights, and - sparsity at the current global_step - new_mask: A numpy array of the same size and shape as weights containing - 0 or 1 to indicate which of the values in weights falls below - the threshold - - Raises: - ValueError: if block pooling function is not AVG or MAX - """ - block_dims = self._get_block_dims(weights.op.name) - squeezed_weights = array_ops.squeeze(weights) - if squeezed_weights.get_shape().ndims != 2 or block_dims == [1, 1]: - return self._update_mask(weights, threshold) - - for i in range(2): - if block_dims[i] == -1: - block_dims[i] = squeezed_weights.get_shape()[i] - - if self._block_pooling_function not in ['AVG', 'MAX']: - raise ValueError('Unknown pooling function for block sparsity: %s' % - self._block_pooling_function) - - with ops.name_scope(weights.op.name + '_pruning_ops'): - abs_weights = math_ops.abs(squeezed_weights) - - pool_window = block_dims - pool_fn = pruning_utils.factorized_pool - squeeze_axis = None - if not self._spec.use_tpu: - pool_fn = nn_ops.pool - abs_weights = array_ops.reshape( - abs_weights, - [1, abs_weights.get_shape()[0], - abs_weights.get_shape()[1], 1]) - squeeze_axis = [0, 3] - - pooled_weights = pool_fn( - abs_weights, - window_shape=pool_window, - pooling_type=self._block_pooling_function, - strides=pool_window, - padding='SAME', - name=weights.op.name + '_pooled') - - if pooled_weights.get_shape().ndims != 2: - pooled_weights = array_ops.squeeze(pooled_weights, axis=squeeze_axis) - - smoothed_threshold, new_mask = self._update_mask(pooled_weights, - threshold) - - updated_mask = pruning_utils.expand_tensor(new_mask, block_dims) - sliced_mask = array_ops.slice( - updated_mask, [0, 0], - [squeezed_weights.get_shape()[0], - squeezed_weights.get_shape()[1]]) - - return smoothed_threshold, array_ops.reshape(sliced_mask, - array_ops.shape(weights)) - - def _get_mask_assign_ops(self): - # Make sure the assignment ops have not already been added to the list - if self._assign_ops: - raise ValueError( - 'Assign op list not empty. _get_mask_assign_ops() called twice?') - - masks = get_masks() - weights = get_weights() - thresholds = get_thresholds() - - if len(masks) != len(thresholds): - raise ValueError( - 'Number of masks %s and number of thresholds %s mismatch' % - (len(masks), len(thresholds))) - - for index, mask in enumerate(masks): - threshold = thresholds[index] - weight = weights[index] - is_partitioned = isinstance(weight, variables.PartitionedVariable) - if is_partitioned: - weight = weight.as_tensor() - - new_threshold, new_mask = self._maybe_update_block_mask(weight, threshold) - self._assign_ops.append( - pruning_utils.variable_assign(threshold, new_threshold)) - - self._assign_ops.append( - pruning_utils.partitioned_variable_assign(mask, new_mask) - if is_partitioned else pruning_utils.variable_assign(mask, new_mask)) - - def mask_update_op(self): - with ops.name_scope(self._spec.name): - if not self._assign_ops: - self._get_mask_assign_ops() - with ops.control_dependencies([ - state_ops.assign( - self._last_update_step, - self._global_step, - name='last_mask_update_step_assign') - ]): - with ops.control_dependencies(self._assign_ops): - logging.info('Updating masks.') - return control_flow_ops.no_op('mask_update') - - def conditional_mask_update_op(self): - - def maybe_update_masks(): - with ops.name_scope(self._spec.name): - is_step_within_pruning_range = math_ops.logical_and( - math_ops.greater_equal(self._global_step, - self._spec.begin_pruning_step), - # If end_pruning_step is negative, keep pruning forever! - math_ops.logical_or( - math_ops.less_equal(self._global_step, - self._spec.end_pruning_step), - math_ops.less(self._spec.end_pruning_step, 0))) - is_pruning_step = math_ops.less_equal( - math_ops.add(self._last_update_step, self._spec.pruning_frequency), - self._global_step) - return math_ops.logical_and(is_step_within_pruning_range, - is_pruning_step) - - def mask_update_op(): - return self.mask_update_op() - - def no_update_op(): - return control_flow_ops.no_op() - - return control_flow_ops.cond(maybe_update_masks(), mask_update_op, - no_update_op) - - def add_pruning_summaries(self): - """Adds summaries of weight sparsities and thresholds.""" - with ops.name_scope(self._spec.name + '_summaries'): - summary.scalar('sparsity', self._sparsity) - summary.scalar('last_mask_update_step', self._last_update_step) - masks = get_masks() - thresholds = get_thresholds() - for mask, threshold in zip(masks, thresholds): - summary.scalar(mask.op.name + '/sparsity', nn_impl.zero_fraction(mask)) - summary.scalar(threshold.op.name + '/threshold', threshold) - - def print_hparams(self): - logging.info(self._spec.to_json()) diff --git a/tensorflow/contrib/model_pruning/python/pruning_test.py b/tensorflow/contrib/model_pruning/python/pruning_test.py deleted file mode 100644 index 1a925caab96..00000000000 --- a/tensorflow/contrib/model_pruning/python/pruning_test.py +++ /dev/null @@ -1,305 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the key functions in pruning library.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.model_pruning.python import pruning -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import training_util - - -class PruningHParamsTest(test.TestCase): - PARAM_LIST = [ - "name=test", "threshold_decay=0.9", "pruning_frequency=10", - "sparsity_function_end_step=100", "target_sparsity=0.9", - "weight_sparsity_map=[conv1:0.8,conv2/kernel:0.8]", - "block_dims_map=[dense1:4x4,dense2:1x4]" - ] - TEST_HPARAMS = ",".join(PARAM_LIST) - - def setUp(self): - super(PruningHParamsTest, self).setUp() - # Add global step variable to the graph - self.global_step = training_util.get_or_create_global_step() - # Add sparsity - self.sparsity = variables.VariableV1(0.5, name="sparsity") - # Parse hparams - self.pruning_hparams = pruning.get_pruning_hparams().parse( - self.TEST_HPARAMS) - - def testInit(self): - p = pruning.Pruning(self.pruning_hparams) - self.assertEqual(p._spec.name, "test") - self.assertAlmostEqual(p._spec.threshold_decay, 0.9) - self.assertEqual(p._spec.pruning_frequency, 10) - self.assertEqual(p._spec.sparsity_function_end_step, 100) - self.assertAlmostEqual(p._spec.target_sparsity, 0.9) - - def testInitWithExternalSparsity(self): - with self.cached_session(): - p = pruning.Pruning(spec=self.pruning_hparams, sparsity=self.sparsity) - variables.global_variables_initializer().run() - sparsity = p._sparsity.eval() - self.assertAlmostEqual(sparsity, 0.5) - - def testInitWithVariableReuse(self): - with self.cached_session(): - p = pruning.Pruning(spec=self.pruning_hparams, sparsity=self.sparsity) - p_copy = pruning.Pruning( - spec=self.pruning_hparams, sparsity=self.sparsity) - variables.global_variables_initializer().run() - sparsity = p._sparsity.eval() - self.assertAlmostEqual(sparsity, 0.5) - self.assertEqual(p._sparsity.eval(), p_copy._sparsity.eval()) - - -class PruningTest(test.TestCase): - - def setUp(self): - super(PruningTest, self).setUp() - self.global_step = training_util.get_or_create_global_step() - - def testCreateMask2D(self): - width = 10 - height = 20 - with self.cached_session(): - weights = variables.VariableV1( - random_ops.random_normal([width, height], stddev=1), name="weights") - masked_weights = pruning.apply_mask(weights, - variable_scope.get_variable_scope()) - variables.global_variables_initializer().run() - weights_val = weights.eval() - masked_weights_val = masked_weights.eval() - self.assertAllEqual(weights_val, masked_weights_val) - - def testUpdateSingleMask(self): - with self.cached_session() as session: - weights = variables.VariableV1( - math_ops.linspace(1.0, 100.0, 100), name="weights") - masked_weights = pruning.apply_mask(weights) - sparsity = variables.VariableV1(0.95, name="sparsity") - p = pruning.Pruning(sparsity=sparsity) - p._spec.threshold_decay = 0.0 - mask_update_op = p.mask_update_op() - variables.global_variables_initializer().run() - masked_weights_val = masked_weights.eval() - self.assertAllEqual(np.count_nonzero(masked_weights_val), 100) - session.run(mask_update_op) - masked_weights_val = masked_weights.eval() - self.assertAllEqual(np.count_nonzero(masked_weights_val), 5) - - def _blockMasking(self, hparams, weights, expected_mask): - - threshold = variables.VariableV1(0.0, name="threshold") - sparsity = variables.VariableV1(0.5, name="sparsity") - test_spec = ",".join(hparams) - pruning_hparams = pruning.get_pruning_hparams().parse(test_spec) - - # Set up pruning - p = pruning.Pruning(pruning_hparams, sparsity=sparsity) - with self.cached_session(): - variables.global_variables_initializer().run() - _, new_mask = p._maybe_update_block_mask(weights, threshold) - # Check if the mask is the same size as the weights - self.assertAllEqual(new_mask.get_shape(), weights.get_shape()) - mask_val = new_mask.eval() - self.assertAllEqual(mask_val, expected_mask) - - def testBlockMaskingWithNonnegativeBlockDimensions(self): - param_list = ["block_height=2", "block_width=2", "threshold_decay=0"] - - weights_avg = constant_op.constant( - [[0.1, 0.1, 0.2, 0.2], [0.1, 0.1, 0.2, 0.2], [0.3, 0.3, 0.4, 0.4], - [0.3, 0.3, 0.4, 0.4]]) - weights_max = constant_op.constant( - [[0.1, 0.0, 0.2, 0.0], [0.0, -0.1, 0.0, -0.2], [0.3, 0.0, 0.4, 0.0], - [0.0, -0.3, 0.0, -0.4]]) - expected_mask = [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], - [1., 1., 1., 1.], [1., 1., 1., 1.]] - - self._blockMasking(param_list + ["block_pooling_function=MAX"], weights_max, - expected_mask) - self._blockMasking(param_list + ["block_pooling_function=AVG"], weights_avg, - expected_mask) - - def testBlockMaskingWithNegativeBlockDimensions(self): - param_list = ["block_height=1", "block_width=-1", "threshold_decay=0"] - - weights_avg = constant_op.constant([[0.1, 0.1, 0.1, 0.1], - [0.2, 0.2, 0.2, 0.2], - [0.3, 0.3, 0.3, 0.3], - [0.3, 0.3, 0.4, 0.4]]) - weights_max = constant_op.constant([[0.1, 0.0, 0.1, 0.0], - [0.0, 0.1, 0.0, 0.2], - [0.3, 0.0, 0.3, 0.0], - [0.0, -0.3, 0.0, 0.4]]) - expected_mask = [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], - [1., 1., 1., 1.], [1., 1., 1., 1.]] - - self._blockMasking(param_list + ["block_pooling_function=MAX"], weights_max, - expected_mask) - self._blockMasking(param_list + ["block_pooling_function=AVG"], weights_avg, - expected_mask) - - def testBlockMaskingWithHigherDimensions(self): - param_list = ["block_height=2", "block_width=2", "threshold_decay=0"] - - # Weights as in testBlockMasking, but with one extra dimension. - weights_avg = constant_op.constant( - [[[0.1, 0.1, 0.2, 0.2], [0.1, 0.1, 0.2, 0.2], [0.3, 0.3, 0.4, 0.4], - [0.3, 0.3, 0.4, 0.4]]]) - weights_max = constant_op.constant( - [[[0.1, 0.0, 0.2, 0.0], [0.0, -0.1, 0.0, -0.2], [0.3, 0.0, 0.4, 0.0], - [0.0, -0.3, 0.0, -0.4]]]) - expected_mask = [[[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], - [1., 1., 1., 1.], [1., 1., 1., 1.]]] - - self._blockMasking(param_list + ["block_pooling_function=MAX"], weights_max, - expected_mask) - self._blockMasking(param_list + ["block_pooling_function=AVG"], - weights_avg, expected_mask) - - def testPartitionedVariableMasking(self): - partitioner = partitioned_variables.variable_axis_size_partitioner(40) - with self.cached_session() as session: - with variable_scope.variable_scope("", partitioner=partitioner): - sparsity = variables.VariableV1(0.5, name="Sparsity") - weights = variable_scope.get_variable( - "weights", initializer=math_ops.linspace(1.0, 100.0, 100)) - masked_weights = pruning.apply_mask( - weights, scope=variable_scope.get_variable_scope()) - p = pruning.Pruning(sparsity=sparsity) - p._spec.threshold_decay = 0.0 - mask_update_op = p.mask_update_op() - variables.global_variables_initializer().run() - masked_weights_val = masked_weights.eval() - session.run(mask_update_op) - masked_weights_val = masked_weights.eval() - self.assertAllEqual(np.count_nonzero(masked_weights_val), 50) - - def testConditionalMaskUpdate(self): - param_list = [ - "pruning_frequency=2", "begin_pruning_step=1", "end_pruning_step=6", - "nbins=100" - ] - test_spec = ",".join(param_list) - pruning_hparams = pruning.get_pruning_hparams().parse(test_spec) - weights = variables.VariableV1( - math_ops.linspace(1.0, 100.0, 100), name="weights") - masked_weights = pruning.apply_mask(weights) - sparsity = variables.VariableV1(0.00, name="sparsity") - # Set up pruning - p = pruning.Pruning(pruning_hparams, sparsity=sparsity) - p._spec.threshold_decay = 0.0 - mask_update_op = p.conditional_mask_update_op() - sparsity_val = math_ops.linspace(0.0, 0.9, 10) - increment_global_step = state_ops.assign_add(self.global_step, 1) - non_zero_count = [] - with self.cached_session() as session: - variables.global_variables_initializer().run() - for i in range(10): - session.run(state_ops.assign(sparsity, sparsity_val[i])) - session.run(mask_update_op) - session.run(increment_global_step) - non_zero_count.append(np.count_nonzero(masked_weights.eval())) - # Weights pruned at steps 0,2,4,and,6 - expected_non_zero_count = [100, 100, 80, 80, 60, 60, 40, 40, 40, 40] - self.assertAllEqual(expected_non_zero_count, non_zero_count) - - def testWeightSpecificSparsity(self): - param_list = [ - "begin_pruning_step=1", "pruning_frequency=1", "end_pruning_step=100", - "target_sparsity=0.5", - "weight_sparsity_map=[layer1:0.6,layer2/weights:0.75,.*kernel:0.6]", - "threshold_decay=0.0" - ] - test_spec = ",".join(param_list) - pruning_hparams = pruning.get_pruning_hparams().parse(test_spec) - - with variable_scope.variable_scope("layer1"): - w1 = variables.VariableV1( - math_ops.linspace(1.0, 100.0, 100), name="weights") - _ = pruning.apply_mask(w1) - with variable_scope.variable_scope("layer2"): - w2 = variables.VariableV1( - math_ops.linspace(1.0, 100.0, 100), name="weights") - _ = pruning.apply_mask(w2) - with variable_scope.variable_scope("layer3"): - w3 = variables.VariableV1( - math_ops.linspace(1.0, 100.0, 100), name="kernel") - _ = pruning.apply_mask(w3) - - p = pruning.Pruning(pruning_hparams) - mask_update_op = p.conditional_mask_update_op() - increment_global_step = state_ops.assign_add(self.global_step, 1) - - with self.cached_session() as session: - variables.global_variables_initializer().run() - for _ in range(110): - session.run(mask_update_op) - session.run(increment_global_step) - - self.assertAllClose( - session.run(pruning.get_weight_sparsity()), [0.6, 0.75, 0.6]) - - def testPerLayerBlockSparsity(self): - param_list = [ - "block_dims_map=[layer1/weights:1x1,layer2/weights:1x2]", - "block_pooling_function=AVG", "threshold_decay=0.0" - ] - - test_spec = ",".join(param_list) - pruning_hparams = pruning.get_pruning_hparams().parse(test_spec) - - with variable_scope.variable_scope("layer1"): - w1 = constant_op.constant([[-0.1, 0.1], [-0.2, 0.2]], name="weights") - pruning.apply_mask(w1) - - with variable_scope.variable_scope("layer2"): - w2 = constant_op.constant([[0.1, 0.1, 0.3, 0.3], [0.2, 0.2, 0.4, 0.4]], - name="weights") - pruning.apply_mask(w2) - - sparsity = variables.VariableV1(0.5, name="sparsity") - - p = pruning.Pruning(pruning_hparams, sparsity=sparsity) - mask_update_op = p.mask_update_op() - with self.cached_session() as session: - variables.global_variables_initializer().run() - session.run(mask_update_op) - mask1_eval = session.run(pruning.get_masks()[0]) - mask2_eval = session.run(pruning.get_masks()[1]) - - self.assertAllEqual( - session.run(pruning.get_weight_sparsity()), [0.5, 0.5]) - - self.assertAllEqual(mask1_eval, [[0.0, 0.0], [1., 1.]]) - self.assertAllEqual(mask2_eval, [[0, 0, 1., 1.], [0, 0, 1., 1.]]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/model_pruning/python/pruning_utils.py b/tensorflow/contrib/model_pruning/python/pruning_utils.py deleted file mode 100644 index 8f2ba036469..00000000000 --- a/tensorflow/contrib/model_pruning/python/pruning_utils.py +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utility functions for adding pruning related ops to the graph. -""" -# pylint: disable=missing-docstring -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope - - -def weight_mask_variable(var, scope): - """Create a mask for the weights. - - This function adds a variable 'mask' to the graph. - - Args: - var: the weight variable that needs to be masked - scope: The variable scope of the variable var - - Returns: - the mask variable of the same size and shape as var, initialized to all 1s. - """ - with variable_scope.variable_scope(scope): - mask = variable_scope.get_variable( - 'mask', - var.get_shape(), - initializer=init_ops.ones_initializer(), - trainable=False, - dtype=var.dtype) - return mask - - -def weight_threshold_variable(var, scope): - """Create a scalar threshold for the weights. - - This function adds a variable - 'threshold' to the graph. - - Args: - var: The weight variable that needs to be masked - scope: The variable scope of the variable var - - Returns: - A scalar threshold variable initialized to 0. - """ - with variable_scope.variable_scope(scope): - threshold = variable_scope.get_variable( - 'threshold', [], - initializer=init_ops.zeros_initializer(), - trainable=False, - dtype=var.dtype) - return threshold - - -def kronecker_product(mat1, mat2): - """Computes the Kronecker product of two matrices mat1 and mat2. - - Args: - mat1: A matrix of size m x n - mat2: A matrix of size p x q - Returns: - Kronecker product of matrices mat1 and mat2 of size mp x nq - """ - - m1, n1 = mat1.get_shape().as_list() - mat1_rsh = array_ops.reshape(mat1, [m1, 1, n1, 1]) - m2, n2 = mat2.get_shape().as_list() - mat2_rsh = array_ops.reshape(mat2, [1, m2, 1, n2]) - return array_ops.reshape(mat1_rsh * mat2_rsh, [m1 * m2, n1 * n2]) - - -def expand_tensor(tensor, block_dims): - """Expands a 2D tensor by replicating the tensor values. - - This is equivalent to the kronecker product of the tensor and a matrix of - ones of size block_dims. - - Example: - - tensor = [[1,2] - [3,4]] - block_dims = [2,2] - - result = [[1 1 2 2] - [1 1 2 2] - [3 3 4 4] - [3 3 4 4]] - - Args: - tensor: A 2D tensor that needs to be expanded. - block_dims: List of integers specifying the expansion factor. - - Returns: - The expanded tensor - - Raises: - ValueError: if tensor is not rank-2 or block_dims is does not have 2 - elements. - """ - if tensor.get_shape().ndims != 2: - raise ValueError('Input tensor must be rank 2') - - if len(block_dims) != 2: - raise ValueError('block_dims must have 2 elements') - - block_height, block_width = block_dims - - def _tile_rows(tensor, multiple): - """Create a new tensor by tiling the tensor along rows.""" - return array_ops.tile(tensor, [multiple, 1]) - - def _generate_indices(num_rows, block_dim): - indices = np.zeros(shape=[num_rows * block_dim, 1], dtype=np.int32) - for k in range(block_dim): - for r in range(num_rows): - indices[k * num_rows + r] = r * block_dim + k - return indices - - def _replicate_rows(tensor, multiple): - tensor_shape = tensor.shape.as_list() - expanded_shape = [tensor_shape[0] * multiple, tensor_shape[1]] - indices = constant_op.constant(_generate_indices(tensor_shape[0], multiple)) - return array_ops.scatter_nd(indices, _tile_rows(tensor, multiple), - expanded_shape) - - expanded_tensor = tensor - - # Expand rows by factor block_height. - if block_height > 1: - expanded_tensor = _replicate_rows(tensor, block_height) - - # Transpose and expand by factor block_width. Transpose the result. - if block_width > 1: - expanded_tensor = array_ops.transpose( - _replicate_rows(array_ops.transpose(expanded_tensor), block_width)) - - return expanded_tensor - - -def factorized_pool(input_tensor, - window_shape, - pooling_type, - strides, - padding, - name=None): - """Performs m x n pooling through a combination of 1xm and 1xn pooling. - - Args: - input_tensor: Input tensor. Must be rank 2 - window_shape: Pooling window shape - pooling_type: Either 'MAX' or 'AVG' - strides: The stride of the pooling window - padding: 'SAME' or 'VALID'. - name: Name of the op - - Returns: - A rank 2 tensor containing the pooled output - - Raises: - ValueError: if the input tensor is not rank 2 - """ - if input_tensor.get_shape().ndims != 2: - raise ValueError('factorized_pool() accepts tensors of rank 2 only') - - [height, width] = input_tensor.get_shape() - with ops.name_scope(name, 'factorized_pool'): - input_tensor_aligned = array_ops.reshape( - input_tensor, [1, 1, height, width], - name=input_tensor.op.name + '_aligned') - - height_pooling = nn_ops.pool( - input_tensor_aligned, - window_shape=[1, window_shape[0]], - pooling_type=pooling_type, - strides=[1, strides[0]], - padding=padding) - swap_height_width = array_ops.transpose(height_pooling, perm=[0, 1, 3, 2]) - - width_pooling = nn_ops.pool( - swap_height_width, - window_shape=[1, window_shape[1]], - pooling_type=pooling_type, - strides=[1, strides[1]], - padding=padding) - - return array_ops.squeeze( - array_ops.transpose(width_pooling, perm=[0, 1, 3, 2]), axis=[0, 1]) - - -def determine_partitioned_axis(partitioned_variable): - partitioned_axis = 0 - concatenated_variable_shape = partitioned_variable.get_shape() - for partition in partitioned_variable: - partition_shape = partition.get_shape() - maybe_partitioned_axis = np.less(partition_shape, - concatenated_variable_shape) - # Sanity check: make sure number of partitioned axis == 1 - if np.count_nonzero(maybe_partitioned_axis) != 1: - raise ValueError('Number of partitioned axes %s not equal to 1' % - np.count_nonzero(maybe_partitioned_axis)) - partitioned_axis = np.where(maybe_partitioned_axis)[0][0] - return partitioned_axis - - -def variable_assign(var, new_value): - return state_ops.assign(var, new_value, name=var.op.name + '_assign') - - -def partitioned_variable_assign(partitioned_var, new_value): - """Assign op for partitioned variables. - - Args: - partitioned_var: A partitioned tensorflow variable - new_value: Value to be assigned to the variable var - - Returns: - A tensorflow op that groups the assign ops for each of the variable slices - """ - # Determine which axis was used to partition the variable. Currently - # tensorflow allows partitioning variable only along 1 axis. - axis = 0 if len(partitioned_var) == 1 else determine_partitioned_axis( - partitioned_var) - - partition_sizes = np.array( - [partition.get_shape()[axis] for partition in partitioned_var]) - new_partitioned_values = array_ops.split( - new_value, - ops.convert_to_tensor(partition_sizes, dtype=dtypes.int32), - axis=axis) - op_list = [] - for partition in partitioned_var: - op_list.append( - variable_assign(partition, new_partitioned_values[len(op_list)])) - return control_flow_ops.group( - *op_list, name=partitioned_var.name + '_group_assign') diff --git a/tensorflow/contrib/model_pruning/python/pruning_utils_test.py b/tensorflow/contrib/model_pruning/python/pruning_utils_test.py deleted file mode 100644 index b85bc413155..00000000000 --- a/tensorflow/contrib/model_pruning/python/pruning_utils_test.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for utility functions in pruning_utils.py.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized - -from tensorflow.contrib.model_pruning.python import pruning_utils -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -@parameterized.named_parameters( - ("Input_32x32_block_1x1", [32, 32], [1, 1]), - # block size 6x6 - ("Input_3x3_block_6x6", [3, 3], [6, 6]), - ("Input_32x32_block_6x6", [32, 32], [6, 6]), - ("Input_2x32_block_6x6", [2, 32], [6, 6]), - ("Input_32x2_block_6x6", [32, 2], [6, 6]), - ("Input_30x30_block_6x6", [30, 30], [6, 6]), - # block size 4x4 - ("Input_32x32_block_4x4", [32, 32], [4, 4]), - ("Input_2x32_block_4x4", [2, 32], [4, 4]), - ("Input_32x2_block_4x4", [32, 2], [4, 4]), - ("Input_30x30_block_4x4", [30, 30], [4, 4]), - # block size 1x4 - ("Input_32x32_block_1x4", [32, 32], [1, 4]), - ("Input_2x32_block_1x4", [2, 32], [1, 4]), - ("Input_32x2_block_1x4", [32, 2], [1, 4]), - ("Input_30x30_block_1x4", [30, 30], [1, 4]), - # block size 4x1 - ("Input_32x32_block_4x1", [32, 32], [4, 1]), - ("Input_2x32_block_4x1", [2, 32], [4, 1]), - ("Input_32x2_block_4x1", [32, 2], [4, 1]), - ("Input_30x30_block_4x1", [30, 30], [4, 1])) -class PruningUtilsParameterizedTest(test.TestCase, parameterized.TestCase): - - def _compare_pooling_methods(self, weights, pooling_kwargs): - with self.cached_session(): - variables.global_variables_initializer().run() - pooled_weights_tf = array_ops.squeeze( - nn_ops.pool( - array_ops.reshape( - weights, - [1, weights.get_shape()[0], - weights.get_shape()[1], 1]), **pooling_kwargs), - axis=[0, 3]) - pooled_weights_factorized_pool = pruning_utils.factorized_pool( - weights, **pooling_kwargs) - - self.assertAllClose(pooled_weights_tf.eval(), - pooled_weights_factorized_pool.eval()) - - def _compare_expand_tensor_with_kronecker_product(self, tensor, block_dim): - with self.cached_session() as session: - variables.global_variables_initializer().run() - expanded_tensor = pruning_utils.expand_tensor(tensor, block_dim) - kronecker_product = pruning_utils.kronecker_product( - tensor, array_ops.ones(block_dim)) - expanded_tensor_val, kronecker_product_val = session.run( - [expanded_tensor, kronecker_product]) - self.assertAllEqual(expanded_tensor_val, kronecker_product_val) - - def testFactorizedAvgPool(self, input_shape, window_shape): - weights = variable_scope.get_variable("weights", shape=input_shape) - pooling_kwargs = { - "window_shape": window_shape, - "pooling_type": "AVG", - "strides": window_shape, - "padding": "SAME" - } - self._compare_pooling_methods(weights, pooling_kwargs) - - def testFactorizedMaxPool(self, input_shape, window_shape): - weights = variable_scope.get_variable("weights", shape=input_shape) - pooling_kwargs = { - "window_shape": window_shape, - "pooling_type": "MAX", - "strides": window_shape, - "padding": "SAME" - } - self._compare_pooling_methods(weights, pooling_kwargs) - - def testExpandTensor(self, input_shape, block_dim): - weights = random_ops.random_normal(shape=input_shape) - self._compare_expand_tensor_with_kronecker_product(weights, block_dim) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/model_pruning/python/strip_pruning_vars.py b/tensorflow/contrib/model_pruning/python/strip_pruning_vars.py deleted file mode 100644 index 3385103807f..00000000000 --- a/tensorflow/contrib/model_pruning/python/strip_pruning_vars.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -r"""Removes the auxiliary variables and ops added by the pruning library. - -Usage: - -bazel build tensorflow/contrib/model_pruning:strip_pruning_vars && \ -bazel-bin/tensorflow/contrib/model_pruning/strip_pruning_vars \ ---checkpoint_dir=/tmp/model_ckpts \ ---output_node_names=softmax \ ---output_dir=/tmp \ ---filename=pruning_stripped.pb -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import os -import sys - -from tensorflow.contrib.model_pruning.python import strip_pruning_vars_lib -from tensorflow.python.framework import graph_io -from tensorflow.python.platform import app -from tensorflow.python.platform import tf_logging as logging - -FLAGS = None - - -def strip_pruning_vars(checkpoint_dir, output_node_names, output_dir, filename): - """Remove pruning-related auxiliary variables and ops from the graph. - - Accepts training checkpoints and produces a GraphDef in which the pruning vars - and ops have been removed. - - Args: - checkpoint_dir: Path to the checkpoints. - output_node_names: The name of the output nodes, comma separated. - output_dir: Directory where to write the graph. - filename: Output GraphDef file name. - - Returns: - None - - Raises: - ValueError: if output_nodes_names are not provided. - """ - if not output_node_names: - raise ValueError( - 'Need to specify atleast 1 output node through output_node_names flag') - output_node_names = output_node_names.replace(' ', '').split(',') - - initial_graph_def = strip_pruning_vars_lib.graph_def_from_checkpoint( - checkpoint_dir, output_node_names) - - final_graph_def = strip_pruning_vars_lib.strip_pruning_vars_fn( - initial_graph_def, output_node_names) - graph_io.write_graph(final_graph_def, output_dir, filename, as_text=False) - logging.info('\nFinal graph written to %s', os.path.join( - output_dir, filename)) - - -def main(unused_args): - return strip_pruning_vars(FLAGS.checkpoint_dir, FLAGS.output_node_names, - FLAGS.output_dir, FLAGS.filename) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.register('type', 'bool', lambda v: v.lower() == 'true') - parser.add_argument( - '--checkpoint_dir', type=str, default='', help='Path to the checkpoints.') - parser.add_argument( - '--output_node_names', - type=str, - default='', - help='The name of the output nodes, comma separated.') - parser.add_argument( - '--output_dir', - type=str, - default='/tmp', - help='Directory where to write the graph.') - parser.add_argument( - '--filename', - type=str, - default='pruning_stripped.pb', - help='Output \'GraphDef\' file name.') - - FLAGS, unparsed = parser.parse_known_args() - app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/model_pruning/python/strip_pruning_vars_lib.py b/tensorflow/contrib/model_pruning/python/strip_pruning_vars_lib.py deleted file mode 100644 index fc4b10863f7..00000000000 --- a/tensorflow/contrib/model_pruning/python/strip_pruning_vars_lib.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities to remove pruning-related ops and variables from a GraphDef. -""" - -# pylint: disable=missing-docstring -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.core.framework import attr_value_pb2 -from tensorflow.core.framework import graph_pb2 -from tensorflow.core.framework import node_def_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import graph_util -from tensorflow.python.framework import importer -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_util -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import saver as saver_lib - - -def _node_name(tensor_name): - """Remove the trailing ':0' from the variable name.""" - if ':' not in tensor_name: - return tensor_name - - return tensor_name.split(':')[0] - - -def _tensor_name(node_name): - """Appends the :0 in the op name to get the canonical tensor name.""" - if ':' in node_name: - return node_name - - return node_name + ':0' - - -def _get_masked_weights(input_graph_def): - """Extracts masked_weights from the graph as a dict of {var_name:ndarray}.""" - input_graph = ops.Graph() - with input_graph.as_default(): - importer.import_graph_def(input_graph_def, name='') - - with session.Session(graph=input_graph) as sess: - masked_weights_dict = {} - for node in input_graph_def.node: - if 'masked_weight' in node.name: - masked_weight_val = sess.run( - sess.graph.get_tensor_by_name(_tensor_name(node.name))) - logging.info( - '%s has %d values, %1.2f%% zeros \n', node.name, - np.size(masked_weight_val), - 100 - float(100 * np.count_nonzero(masked_weight_val)) / - np.size(masked_weight_val)) - masked_weights_dict.update({node.name: masked_weight_val}) - return masked_weights_dict - - -def strip_pruning_vars_fn(input_graph_def, output_node_names): - """Removes mask variable from the graph. - - Replaces the masked_weight tensor with element-wise multiplication of mask - and the corresponding weight variable. - - Args: - input_graph_def: A GraphDef in which the variables have been converted to - constants. This is typically the output of - tf.graph_util.convert_variables_to_constant() - output_node_names: List of name strings for the result nodes of the graph - - Returns: - A GraphDef in which pruning-related variables have been removed - """ - masked_weights_dict = _get_masked_weights(input_graph_def) - pruned_graph_def = graph_pb2.GraphDef() - - # Replace masked_weight with a const op containing the - # result of tf.multiply(mask,weight) - for node in input_graph_def.node: - output_node = node_def_pb2.NodeDef() - if 'masked_weight' in node.name: - output_node.op = 'Const' - output_node.name = node.name - dtype = node.attr['T'] - data = masked_weights_dict[node.name] - output_node.attr['dtype'].CopyFrom(dtype) - output_node.attr['value'].CopyFrom( - attr_value_pb2.AttrValue(tensor=tensor_util.make_tensor_proto(data))) - - else: - output_node.CopyFrom(node) - pruned_graph_def.node.extend([output_node]) - - # Remove stranded nodes: mask and weights - return graph_util.extract_sub_graph(pruned_graph_def, output_node_names) - - -def graph_def_from_checkpoint(checkpoint_dir, output_node_names): - """Converts checkpoint data to GraphDef. - - Reads the latest checkpoint data and produces a GraphDef in which the - variables have been converted to constants. - - Args: - checkpoint_dir: Path to the checkpoints. - output_node_names: List of name strings for the result nodes of the graph. - - Returns: - A GraphDef from the latest checkpoint - - Raises: - ValueError: if no checkpoint is found - """ - checkpoint_path = saver_lib.latest_checkpoint(checkpoint_dir) - if checkpoint_path is None: - raise ValueError('Could not find a checkpoint at: {0}.' - .format(checkpoint_dir)) - - saver_for_restore = saver_lib.import_meta_graph( - checkpoint_path + '.meta', clear_devices=True) - with session.Session() as sess: - saver_for_restore.restore(sess, checkpoint_path) - graph_def = ops.get_default_graph().as_graph_def() - output_graph_def = graph_util.convert_variables_to_constants( - sess, graph_def, output_node_names) - - return output_graph_def diff --git a/tensorflow/contrib/model_pruning/python/strip_pruning_vars_test.py b/tensorflow/contrib/model_pruning/python/strip_pruning_vars_test.py deleted file mode 100644 index 237510cb0c8..00000000000 --- a/tensorflow/contrib/model_pruning/python/strip_pruning_vars_test.py +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for strip_pruning_vars.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re - -from tensorflow.contrib.model_pruning.python import pruning -from tensorflow.contrib.model_pruning.python import strip_pruning_vars_lib -from tensorflow.contrib.model_pruning.python.layers import layers -from tensorflow.contrib.model_pruning.python.layers import rnn_cells -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import graph_util -from tensorflow.python.framework import importer -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import rnn -from tensorflow.python.ops import rnn_cell as tf_rnn_cells -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import training_util - - -def _get_number_pruning_vars(graph_def): - number_vars = 0 - for node in graph_def.node: - if re.match(r"^.*(mask$)|(threshold$)", node.name): - number_vars += 1 - return number_vars - - -def _get_node_names(tensor_names): - return [ - strip_pruning_vars_lib._node_name(tensor_name) - for tensor_name in tensor_names - ] - - -class StripPruningVarsTest(test.TestCase): - - def setUp(self): - param_list = [ - "pruning_frequency=1", "begin_pruning_step=1", "end_pruning_step=10", - "nbins=2048", "threshold_decay=0.0" - ] - self.initial_graph = ops.Graph() - self.initial_graph_def = None - self.final_graph = ops.Graph() - self.final_graph_def = None - self.pruning_spec = ",".join(param_list) - with self.initial_graph.as_default(): - self.sparsity = variables.Variable(0.5, name="sparsity") - self.global_step = training_util.get_or_create_global_step() - self.increment_global_step = state_ops.assign_add(self.global_step, 1) - self.mask_update_op = None - - def _build_convolutional_model(self, number_of_layers): - # Create a graph with several conv2d layers - kernel_size = 3 - base_depth = 4 - depth_step = 7 - height, width = 7, 9 - with variable_scope.variable_scope("conv_model"): - input_tensor = array_ops.ones((8, height, width, base_depth)) - top_layer = input_tensor - for ix in range(number_of_layers): - top_layer = layers.masked_conv2d( - top_layer, - base_depth + (ix + 1) * depth_step, - kernel_size, - scope="Conv_" + str(ix)) - - return top_layer - - def _build_fully_connected_model(self, number_of_layers): - base_depth = 4 - depth_step = 7 - - input_tensor = array_ops.ones((8, base_depth)) - - top_layer = input_tensor - - with variable_scope.variable_scope("fc_model"): - for ix in range(number_of_layers): - top_layer = layers.masked_fully_connected( - top_layer, base_depth + (ix + 1) * depth_step) - - return top_layer - - def _build_lstm_model(self, number_of_layers): - batch_size = 8 - dim = 10 - inputs = variables.Variable(random_ops.random_normal([batch_size, dim])) - - def lstm_cell(): - return rnn_cells.MaskedBasicLSTMCell( - dim, forget_bias=0.0, state_is_tuple=True, reuse=False) - - cell = tf_rnn_cells.MultiRNNCell( - [lstm_cell() for _ in range(number_of_layers)], state_is_tuple=True) - - outputs = rnn.static_rnn( - cell, [inputs], - initial_state=cell.zero_state(batch_size, dtypes.float32)) - - return outputs - - def _prune_model(self, session): - pruning_hparams = pruning.get_pruning_hparams().parse(self.pruning_spec) - p = pruning.Pruning(pruning_hparams, sparsity=self.sparsity) - self.mask_update_op = p.conditional_mask_update_op() - - variables.global_variables_initializer().run() - for _ in range(20): - session.run(self.mask_update_op) - session.run(self.increment_global_step) - - def _get_outputs(self, session, input_graph, tensors_list, graph_prefix=None): - outputs = [] - - for output_tensor in tensors_list: - if graph_prefix: - output_tensor = graph_prefix + "/" + output_tensor - outputs.append( - session.run(session.graph.get_tensor_by_name(output_tensor))) - - return outputs - - def _get_initial_outputs(self, output_tensor_names_list): - with self.session(graph=self.initial_graph) as sess1: - self._prune_model(sess1) - reference_outputs = self._get_outputs(sess1, self.initial_graph, - output_tensor_names_list) - - self.initial_graph_def = graph_util.convert_variables_to_constants( - sess1, sess1.graph.as_graph_def(), - _get_node_names(output_tensor_names_list)) - return reference_outputs - - def _get_final_outputs(self, output_tensor_names_list): - self.final_graph_def = strip_pruning_vars_lib.strip_pruning_vars_fn( - self.initial_graph_def, _get_node_names(output_tensor_names_list)) - _ = importer.import_graph_def(self.final_graph_def, name="final") - - with self.test_session(self.final_graph) as sess2: - final_outputs = self._get_outputs( - sess2, - self.final_graph, - output_tensor_names_list, - graph_prefix="final") - return final_outputs - - def _check_removal_of_pruning_vars(self, number_masked_layers): - self.assertEqual( - _get_number_pruning_vars(self.initial_graph_def), number_masked_layers) - self.assertEqual(_get_number_pruning_vars(self.final_graph_def), 0) - - def _check_output_equivalence(self, initial_outputs, final_outputs): - for initial_output, final_output in zip(initial_outputs, final_outputs): - self.assertAllEqual(initial_output, final_output) - - def testConvolutionalModel(self): - with self.initial_graph.as_default(): - number_masked_conv_layers = 5 - top_layer = self._build_convolutional_model(number_masked_conv_layers) - output_tensor_names = [top_layer.name] - initial_outputs = self._get_initial_outputs(output_tensor_names) - - # Remove pruning-related nodes. - with self.final_graph.as_default(): - final_outputs = self._get_final_outputs(output_tensor_names) - - # Check that the final graph has no pruning-related vars - self._check_removal_of_pruning_vars(number_masked_conv_layers) - - # Check that outputs remain the same after removal of pruning-related nodes - self._check_output_equivalence(initial_outputs, final_outputs) - - def testFullyConnectedModel(self): - with self.initial_graph.as_default(): - number_masked_fc_layers = 3 - top_layer = self._build_fully_connected_model(number_masked_fc_layers) - output_tensor_names = [top_layer.name] - initial_outputs = self._get_initial_outputs(output_tensor_names) - - # Remove pruning-related nodes. - with self.final_graph.as_default(): - final_outputs = self._get_final_outputs(output_tensor_names) - - # Check that the final graph has no pruning-related vars - self._check_removal_of_pruning_vars(number_masked_fc_layers) - - # Check that outputs remain the same after removal of pruning-related nodes - self._check_output_equivalence(initial_outputs, final_outputs) - - def testLSTMModel(self): - with self.initial_graph.as_default(): - number_masked_lstm_layers = 2 - outputs = self._build_lstm_model(number_masked_lstm_layers) - output_tensor_names = [outputs[0][0].name] - initial_outputs = self._get_initial_outputs(output_tensor_names) - - # Remove pruning-related nodes. - with self.final_graph.as_default(): - final_outputs = self._get_final_outputs(output_tensor_names) - - # Check that the final graph has no pruning-related vars - self._check_removal_of_pruning_vars(number_masked_lstm_layers) - - # Check that outputs remain the same after removal of pruning-related nodes - self._check_output_equivalence(initial_outputs, final_outputs) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/nearest_neighbor/BUILD b/tensorflow/contrib/nearest_neighbor/BUILD deleted file mode 100644 index 22be1e2f9d7..00000000000 --- a/tensorflow/contrib/nearest_neighbor/BUILD +++ /dev/null @@ -1,114 +0,0 @@ -# Description: -# Tensorflow ops for nearest neighbor queries etc. - -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") -load( - "//tensorflow:tensorflow.bzl", - "tf_cc_test", - "tf_custom_op_library", - "tf_gen_op_libs", - "tf_gen_op_wrapper_py", - "tf_kernel_library", - "tf_py_test", -) - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_custom_op_library( - name = "python/ops/_nearest_neighbor_ops.so", - srcs = [ - "kernels/hyperplane_lsh_probes.cc", - "ops/nearest_neighbor_ops.cc", - ], - deps = [ - ":hyperplane_lsh_probes", - ], -) - -tf_gen_op_libs( - op_lib_names = ["nearest_neighbor_ops"], -) - -tf_gen_op_wrapper_py( - name = "nearest_neighbor_ops_pywrapper", - deps = ["nearest_neighbor_ops_op_lib"], -) - -tf_custom_op_py_library( - name = "nearest_neighbor_py", - srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - dso = [":python/ops/_nearest_neighbor_ops.so"], - kernels = [":nearest_neighbor_ops_kernels"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - ], -) - -tf_kernel_library( - name = "nearest_neighbor_ops_kernels", - srcs = ["kernels/hyperplane_lsh_probes.cc"], - deps = [ - ":hyperplane_lsh_probes", - ":nearest_neighbor_ops_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//third_party/eigen3", - ], -) - -cc_library( - name = "heap", - hdrs = ["kernels/heap.h"], -) - -tf_cc_test( - name = "heap_test", - size = "small", - srcs = ["kernels/heap_test.cc"], - deps = [ - ":heap", - "//tensorflow/core:test_main", - "//tensorflow/core/kernels:ops_testutil", - ], -) - -cc_library( - name = "hyperplane_lsh_probes", - hdrs = ["kernels/hyperplane_lsh_probes.h"], - deps = [ - ":heap", - "//third_party/eigen3", - ], -) - -tf_cc_test( - name = "hyperplane_lsh_probes_test_cc", - size = "small", - srcs = ["kernels/hyperplane_lsh_probes_test.cc"], - deps = [ - ":hyperplane_lsh_probes", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - "//tensorflow/core/kernels:ops_testutil", - ], -) - -tf_py_test( - name = "hyperplane_lsh_probes_test", - size = "small", - srcs = ["python/kernel_tests/hyperplane_lsh_probes_test.py"], - additional_deps = [ - ":nearest_neighbor_py", - "//tensorflow/python:client_testlib", - ], -) diff --git a/tensorflow/contrib/nearest_neighbor/__init__.py b/tensorflow/contrib/nearest_neighbor/__init__.py deleted file mode 100644 index 9ba124c6d61..00000000000 --- a/tensorflow/contrib/nearest_neighbor/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for (approximate) nearest neighbor look-ups. - -## Ops for (approximate) nearest neighbor look-ups - -This package provides several ops for efficient (approximate) nearest -neighbor look-ups. - -### LSH multiprobe ops - -The following ops generate multiprobe sequences for various hash families. - -@@hyperplane_lsh_hash - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import, line-too-long -from tensorflow.contrib.nearest_neighbor.python.ops.nearest_neighbor_ops import * -# pylint: enable=unused-import,wildcard-import,line-too-long diff --git a/tensorflow/contrib/nearest_neighbor/kernels/heap.h b/tensorflow/contrib/nearest_neighbor/kernels/heap.h deleted file mode 100644 index a2dbb8052bf..00000000000 --- a/tensorflow/contrib/nearest_neighbor/kernels/heap.h +++ /dev/null @@ -1,208 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_NEAREST_NEIGHBOR_KERNELS_HEAP_H_ -#define TENSORFLOW_CONTRIB_NEAREST_NEIGHBOR_KERNELS_HEAP_H_ - -#include -#include -#include -#include - -namespace tensorflow { -namespace nearest_neighbor { - -// A simple binary heap. We use our own implementation because multiprobe for -// the cross-polytope hash interacts with the heap in a way so that about half -// of the insertion operations are guaranteed to be on top of the heap. We make -// use of this fact in the AugmentedHeap below. - -// HeapBase is a base class for both the SimpleHeap and AugmentedHeap below. -template -class HeapBase { - public: - class Item { - public: - KeyType key; - DataType data; - - Item() {} - Item(const KeyType& k, const DataType& d) : key(k), data(d) {} - - bool operator<(const Item& i2) const { return key < i2.key; } - }; - - void ExtractMin(KeyType* key, DataType* data) { - *key = v_[0].key; - *data = v_[0].data; - num_elements_ -= 1; - v_[0] = v_[num_elements_]; - HeapDown(0); - } - - bool IsEmpty() { return num_elements_ == 0; } - - // This method adds an element at the end of the internal array without - // "heapifying" the array afterwards. This is useful for setting up a heap - // where a single call to heapify at the end of the initial insertion - // operations suffices. - void InsertUnsorted(const KeyType& key, const DataType& data) { - if (v_.size() == static_cast(num_elements_)) { - v_.push_back(Item(key, data)); - } else { - v_[num_elements_].key = key; - v_[num_elements_].data = data; - } - num_elements_ += 1; - } - - void Insert(const KeyType& key, const DataType& data) { - if (v_.size() == static_cast(num_elements_)) { - v_.push_back(Item(key, data)); - } else { - v_[num_elements_].key = key; - v_[num_elements_].data = data; - } - num_elements_ += 1; - HeapUp(num_elements_ - 1); - } - - void Heapify() { - int_fast32_t rightmost = parent(num_elements_ - 1); - for (int_fast32_t cur_loc = rightmost; cur_loc >= 0; --cur_loc) { - HeapDown(cur_loc); - } - } - - void Reset() { num_elements_ = 0; } - - void Resize(size_t new_size) { v_.resize(new_size); } - - protected: - int_fast32_t lchild(int_fast32_t x) { return 2 * x + 1; } - - int_fast32_t rchild(int_fast32_t x) { return 2 * x + 2; } - - int_fast32_t parent(int_fast32_t x) { return (x - 1) / 2; } - - void SwapEntries(int_fast32_t a, int_fast32_t b) { - Item tmp = v_[a]; - v_[a] = v_[b]; - v_[b] = tmp; - } - - void HeapUp(int_fast32_t cur_loc) { - int_fast32_t p = parent(cur_loc); - while (cur_loc > 0 && v_[p].key > v_[cur_loc].key) { - SwapEntries(p, cur_loc); - cur_loc = p; - p = parent(cur_loc); - } - } - - void HeapDown(int_fast32_t cur_loc) { - while (true) { - int_fast32_t lc = lchild(cur_loc); - int_fast32_t rc = rchild(cur_loc); - if (lc >= num_elements_) { - return; - } - - if (v_[cur_loc].key <= v_[lc].key) { - if (rc >= num_elements_ || v_[cur_loc].key <= v_[rc].key) { - return; - } else { - SwapEntries(cur_loc, rc); - cur_loc = rc; - } - } else { - if (rc >= num_elements_ || v_[lc].key <= v_[rc].key) { - SwapEntries(cur_loc, lc); - cur_loc = lc; - } else { - SwapEntries(cur_loc, rc); - cur_loc = rc; - } - } - } - } - - std::vector v_; - int_fast32_t num_elements_ = 0; -}; - -// A "simple" binary heap. -template -class SimpleHeap : public HeapBase { - public: - void ReplaceTop(const KeyType& key, const DataType& data) { - this->v_[0].key = key; - this->v_[0].data = data; - this->HeapDown(0); - } - - KeyType MinKey() { return this->v_[0].key; } - - std::vector::Item>& GetData() { - return this->v_; - } -}; - -// An "augmented" heap that can hold an extra element that is guaranteed to -// be at the top of the heap. This is useful if a significant fraction of the -// insertion operations are guaranteed insertions at the top. However, the heap -// only stores at most one such special top element, i.e., the heap assumes -// that extract_min() is called at least once between successive calls to -// insert_guaranteed_top(). -template -class AugmentedHeap : public HeapBase { - public: - void ExtractMin(KeyType* key, DataType* data) { - if (has_guaranteed_top_) { - has_guaranteed_top_ = false; - *key = guaranteed_top_.key; - *data = guaranteed_top_.data; - } else { - *key = this->v_[0].key; - *data = this->v_[0].data; - this->num_elements_ -= 1; - this->v_[0] = this->v_[this->num_elements_]; - this->HeapDown(0); - } - } - - bool IsEmpty() { return this->num_elements_ == 0 && !has_guaranteed_top_; } - - void InsertGuaranteedTop(const KeyType& key, const DataType& data) { - assert(!has_guaranteed_top_); - has_guaranteed_top_ = true; - guaranteed_top_.key = key; - guaranteed_top_.data = data; - } - - void Reset() { - this->num_elements_ = 0; - has_guaranteed_top_ = false; - } - - protected: - typename HeapBase::Item guaranteed_top_; - bool has_guaranteed_top_ = false; -}; - -} // namespace nearest_neighbor -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_NEAREST_NEIGHBOR_KERNELS_HEAP_H_ diff --git a/tensorflow/contrib/nearest_neighbor/kernels/heap_test.cc b/tensorflow/contrib/nearest_neighbor/kernels/heap_test.cc deleted file mode 100644 index c95f2613058..00000000000 --- a/tensorflow/contrib/nearest_neighbor/kernels/heap_test.cc +++ /dev/null @@ -1,188 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/nearest_neighbor/kernels/heap.h" - -#include "tensorflow/core/kernels/ops_testutil.h" - -namespace tensorflow { -namespace nearest_neighbor { -namespace { - -TEST(HeapTest, SimpleHeapTest1) { - SimpleHeap h; - h.Resize(10); - h.InsertUnsorted(2.0, 2); - h.InsertUnsorted(1.0, 1); - h.InsertUnsorted(5.0, 5); - h.InsertUnsorted(3.0, 3); - h.Heapify(); - - float k; - int d; - h.ExtractMin(&k, &d); - ASSERT_EQ(1.0, k); - ASSERT_EQ(1, d); - h.ExtractMin(&k, &d); - ASSERT_EQ(2.0, k); - ASSERT_EQ(2, d); - - h.Insert(4.0, 4); - h.ExtractMin(&k, &d); - ASSERT_EQ(3.0, k); - ASSERT_EQ(3, d); - h.ExtractMin(&k, &d); - ASSERT_EQ(4.0, k); - ASSERT_EQ(4, d); - h.ExtractMin(&k, &d); - ASSERT_EQ(5.0, k); - ASSERT_EQ(5, d); - - h.Reset(); - h.InsertUnsorted(2.0, 2); - h.InsertUnsorted(10.0, 10); - h.InsertUnsorted(8.0, 8); - h.Heapify(); - h.ExtractMin(&k, &d); - ASSERT_EQ(2.0, k); - ASSERT_EQ(2, d); - h.ExtractMin(&k, &d); - ASSERT_EQ(8.0, k); - ASSERT_EQ(8, d); - - h.Insert(9.5, 9); - h.ExtractMin(&k, &d); - ASSERT_EQ(9.5, k); - ASSERT_EQ(9, d); - - h.ExtractMin(&k, &d); - ASSERT_EQ(10.0, k); - ASSERT_EQ(10, d); -} - -TEST(HeapTest, SimpleHeapTest2) { - // Same as above, but without initial resize - SimpleHeap h; - h.InsertUnsorted(2.0, 2); - h.InsertUnsorted(1.0, 1); - h.InsertUnsorted(5.0, 5); - h.InsertUnsorted(3.0, 3); - h.Heapify(); - - float k; - int d; - h.ExtractMin(&k, &d); - ASSERT_EQ(1.0, k); - ASSERT_EQ(1, d); - h.ExtractMin(&k, &d); - ASSERT_EQ(2.0, k); - ASSERT_EQ(2, d); - - h.Insert(4.0, 4); - h.ExtractMin(&k, &d); - ASSERT_EQ(3.0, k); - ASSERT_EQ(3, d); - h.ExtractMin(&k, &d); - ASSERT_EQ(4.0, k); - ASSERT_EQ(4, d); - h.ExtractMin(&k, &d); - ASSERT_EQ(5.0, k); - ASSERT_EQ(5, d); - - h.Reset(); - h.InsertUnsorted(2.0, 2); - h.InsertUnsorted(10.0, 10); - h.InsertUnsorted(8.0, 8); - h.Heapify(); - h.ExtractMin(&k, &d); - ASSERT_EQ(2.0, k); - ASSERT_EQ(2, d); - h.ExtractMin(&k, &d); - ASSERT_EQ(8.0, k); - ASSERT_EQ(8, d); - - h.Insert(9.5, 9); - h.ExtractMin(&k, &d); - ASSERT_EQ(9.5, k); - ASSERT_EQ(9, d); - - h.ExtractMin(&k, &d); - ASSERT_EQ(10.0, k); - ASSERT_EQ(10, d); -} - -TEST(HeapTest, SimpleHeapTest3) { - SimpleHeap h; - h.InsertUnsorted(2.0, 2); - h.InsertUnsorted(1.0, 1); - h.InsertUnsorted(5.0, 5); - h.InsertUnsorted(3.0, 3); - h.Heapify(); - - EXPECT_EQ(1.0, h.MinKey()); - - h.ReplaceTop(0.5, 0); - float k; - int d; - h.ExtractMin(&k, &d); - EXPECT_EQ(0.5, k); - EXPECT_EQ(0, d); - - h.ExtractMin(&k, &d); - EXPECT_EQ(2.0, k); - EXPECT_EQ(2, d); -} - -TEST(HeapTest, AugmentedHeapTest1) { - AugmentedHeap h; - h.InsertUnsorted(2.0, 2); - h.InsertUnsorted(1.0, 1); - h.InsertUnsorted(5.0, 5); - h.InsertUnsorted(3.0, 3); - h.Heapify(); - - float k; - int d; - h.ExtractMin(&k, &d); - ASSERT_EQ(1.0, k); - ASSERT_EQ(1, d); - - h.InsertGuaranteedTop(1.0, 10); - h.ExtractMin(&k, &d); - ASSERT_EQ(1.0, k); - ASSERT_EQ(10, d); - - h.ExtractMin(&k, &d); - ASSERT_EQ(2.0, k); - ASSERT_EQ(2, d); - - h.Insert(4.0, 4); - - h.ExtractMin(&k, &d); - ASSERT_EQ(3.0, k); - ASSERT_EQ(3, d); - - h.ExtractMin(&k, &d); - ASSERT_EQ(4.0, k); - ASSERT_EQ(4, d); - - h.ExtractMin(&k, &d); - ASSERT_EQ(5.0, k); - ASSERT_EQ(5, d); -} - -} // namespace -} // namespace nearest_neighbor -} // namespace tensorflow diff --git a/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes.cc b/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes.cc deleted file mode 100644 index 13db6f62f52..00000000000 --- a/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes.cc +++ /dev/null @@ -1,157 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include - -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/lib/core/threadpool.h" - -#include "tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes.h" - -namespace tensorflow { - -using errors::Internal; -using errors::InvalidArgument; - -using nearest_neighbor::HyperplaneMultiprobe; - -// This class wraps the multiprobe LSH code in hyperplane_lsh_probes in a -// TensorFlow op implementation. -template -class HyperplaneLSHProbesOp : public OpKernel { - public: - using Matrix = Eigen::Matrix; - using ConstMatrixMap = Eigen::Map; - using MatrixMap = Eigen::Map; - - explicit HyperplaneLSHProbesOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - // Get the input tensors and check their shapes. - const Tensor& products_tensor = context->input(0); - OP_REQUIRES(context, products_tensor.dims() == 2, - InvalidArgument("Need a two-dimensional products tensor, got ", - products_tensor.dims(), " dimensions.")); - - const Tensor& num_tables_tensor = context->input(1); - OP_REQUIRES(context, num_tables_tensor.dims() == 0, - InvalidArgument("Need a scalar num_tables tensor, got ", - num_tables_tensor.dims(), " dimensions.")); - int num_tables = num_tables_tensor.scalar()(); - OP_REQUIRES(context, num_tables >= 1, - InvalidArgument("num_tables must be at least 1 but got ", - num_tables, ".")); - OP_REQUIRES(context, num_tables <= 1000, - InvalidArgument("Need num_tables <= 1000, got ", num_tables, - ". This is mostly to protect against incorrect " - "use of this Op. If you really need more tables" - ", change the code.")); - - const Tensor& num_hyperplanes_per_table_tensor = context->input(2); - OP_REQUIRES(context, num_hyperplanes_per_table_tensor.dims() == 0, - InvalidArgument("Need a scalar num_hyperplanes_per_table " - "tensor, got ", - num_hyperplanes_per_table_tensor.dims(), - " dimensions.")); - int num_hyperplanes_per_table = - num_hyperplanes_per_table_tensor.scalar()(); - OP_REQUIRES(context, num_hyperplanes_per_table >= 1, - InvalidArgument("num_hyperplanes_per_table must be at least 1 " - "but got ", - num_hyperplanes_per_table, ".")); - OP_REQUIRES(context, num_hyperplanes_per_table <= 30, - InvalidArgument("Need num_hyperplanes_per_table <= 30, got ", - num_hyperplanes_per_table, - ". " - "If you need more hyperplanes, change this Op" - " to work for larger integer types (int64).")); - - const Tensor& num_probes_tensor = context->input(3); - OP_REQUIRES(context, num_probes_tensor.dims() == 0, - InvalidArgument("Need a scalar num_probes tensor, got ", - num_probes_tensor.dims(), " dimensions.")); - int num_probes = num_probes_tensor.scalar()(); - OP_REQUIRES(context, num_probes >= 1, - InvalidArgument("num_probes must be at least 1.")); - - int expected_num_hyperplanes = num_tables * num_hyperplanes_per_table; - OP_REQUIRES(context, - products_tensor.dim_size(1) == expected_num_hyperplanes, - InvalidArgument("Expected number of hyperplanes is ", - expected_num_hyperplanes, " but received ", - products_tensor.dim_size(1), - " inner products per " - "point.")); - - auto products_eigen_tensor = products_tensor.matrix(); - ConstMatrixMap products_matrix(products_eigen_tensor.data(), - products_tensor.dim_size(0), - products_tensor.dim_size(1)); - - int batch_size = products_tensor.dim_size(0); - - Tensor* probes_tensor = nullptr; - Tensor* tables_tensor = nullptr; - TensorShape output_shape({batch_size, num_probes}); - OP_REQUIRES_OK(context, - context->allocate_output(0, output_shape, &probes_tensor)); - OP_REQUIRES_OK(context, - context->allocate_output(1, output_shape, &tables_tensor)); - auto probes_eigen_tensor = probes_tensor->matrix(); - auto tables_eigen_tensor = tables_tensor->matrix(); - - // Constants (cycles per hyperplane and table) were measured on - // lschmidt's workstation. - int64 cost_per_unit = 21 * num_hyperplanes_per_table * num_tables; - if (num_probes > num_tables) { - cost_per_unit += - 110 * num_hyperplanes_per_table * (num_probes - num_tables); - } - context->device()->tensorflow_cpu_worker_threads()->workers->ParallelFor( - batch_size, cost_per_unit, [&](int64 start, int64 end) { - HyperplaneMultiprobe multiprobe( - num_hyperplanes_per_table, num_tables); - - for (int point_index = start; point_index < end; ++point_index) { - multiprobe.SetupProbing(products_matrix.row(point_index), - num_probes); - for (int ii = 0; ii < num_probes; ++ii) { - int32 cur_probe; - int_fast32_t cur_table; - OP_REQUIRES(context, - multiprobe.GetNextProbe(&cur_probe, &cur_table), - Internal("Failed to get probe number ", ii, - " for point number ", point_index, ".")); - probes_eigen_tensor(point_index, ii) = cur_probe; - tables_eigen_tensor(point_index, ii) = cur_table; - } - } - }); - } -}; - -REGISTER_KERNEL_BUILDER(Name("HyperplaneLSHProbes") - .Device(DEVICE_CPU) - .TypeConstraint("CoordinateType"), - HyperplaneLSHProbesOp); - -REGISTER_KERNEL_BUILDER(Name("HyperplaneLSHProbes") - .Device(DEVICE_CPU) - .TypeConstraint("CoordinateType"), - HyperplaneLSHProbesOp); - -} // namespace tensorflow diff --git a/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes.h b/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes.h deleted file mode 100644 index c53205e1a40..00000000000 --- a/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes.h +++ /dev/null @@ -1,235 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_NEAREST_NEIGHBOR_KERNELS_HYPERPLANE_LSH_PROBES_H_ -#define TENSORFLOW_CONTRIB_NEAREST_NEIGHBOR_KERNELS_HYPERPLANE_LSH_PROBES_H_ - -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" - -#include "tensorflow/contrib/nearest_neighbor/kernels/heap.h" - -namespace tensorflow { -namespace nearest_neighbor { - -// This class implements hyperplane multiprobe LSH as described in the -// following paper: -// -// Multi-probe LSH: efficient indexing for high-dimensional similarity search -// Qin Lv, William Josephson, Zhe Wang, Moses Charikar, Kai Li -// -// The class is only responsible for generating the probing sequence of given -// length for a given batch of points. The actual hash table lookups are -// implemented in other classes. -template -class HyperplaneMultiprobe { - public: - using Matrix = Eigen::Matrix; - using ConstMatrixMap = Eigen::Map; - using MatrixMap = Eigen::Map; - using Vector = - Eigen::Matrix; - - HyperplaneMultiprobe(int num_hyperplanes_per_table, int num_tables) - : num_hyperplanes_per_table_(num_hyperplanes_per_table), - num_tables_(num_tables), - num_probes_(0), - cur_probe_counter_(0), - sorted_hyperplane_indices_(0), - main_table_probe_(num_tables) {} - - // The first input hash_vector is the matrix-vector product between the - // hyperplane matrix and the vector for which we want to generate a probing - // sequence. We assume that each index in hash_vector is proportional to the - // distance between vector and hyperplane (i.e., the hyperplane vectors should - // all have the same norm). - // - // The second input is the number of probes we want to retrieve. If this - // number is fixed in advance, it should be passed in here in order to enable - // some (minor) internal optimizations. If the number of probes it not known - // in advance, the multiprobe sequence can still produce an arbitrary length - // probing sequence (up to the maximum number of probes) by calling - // get_next_probe multiple times. - // - // If num_probes is at most num_tables, it is not necessary to generate an - // actual multiprobe sequence and the multiprobe object will simply return - // the "standard" LSH probes without incurring any multiprobe overhead. - void SetupProbing(const Vector& hash_vector, int_fast64_t num_probes) { - // We accept a copy here for now. - hash_vector_ = hash_vector; - num_probes_ = num_probes; - cur_probe_counter_ = -1; - - // Compute the initial probes for each table, i.e., the "true" hash - // locations LSH without multiprobe would give. - for (int_fast32_t ii = 0; ii < num_tables_; ++ii) { - main_table_probe_[ii] = 0; - for (int_fast32_t jj = 0; jj < num_hyperplanes_per_table_; ++jj) { - main_table_probe_[ii] = main_table_probe_[ii] << 1; - main_table_probe_[ii] = - main_table_probe_[ii] | - (hash_vector_[ii * num_hyperplanes_per_table_ + jj] >= 0.0); - } - } - - if (num_probes_ >= 0 && num_probes_ <= num_tables_) { - return; - } - - if (sorted_hyperplane_indices_.size() == 0) { - sorted_hyperplane_indices_.resize(num_tables_); - for (int_fast32_t ii = 0; ii < num_tables_; ++ii) { - sorted_hyperplane_indices_[ii].resize(num_hyperplanes_per_table_); - for (int_fast32_t jj = 0; jj < num_hyperplanes_per_table_; ++jj) { - sorted_hyperplane_indices_[ii][jj] = jj; - } - } - } - - for (int_fast32_t ii = 0; ii < num_tables_; ++ii) { - HyperplaneComparator comp(hash_vector_, ii * num_hyperplanes_per_table_); - std::sort(sorted_hyperplane_indices_[ii].begin(), - sorted_hyperplane_indices_[ii].end(), comp); - } - - if (num_probes_ >= 0) { - heap_.Resize(2 * num_probes_); - } - heap_.Reset(); - for (int_fast32_t ii = 0; ii < num_tables_; ++ii) { - int_fast32_t best_index = sorted_hyperplane_indices_[ii][0]; - CoordinateType score = - hash_vector_[ii * num_hyperplanes_per_table_ + best_index]; - score = score * score; - HashType hash_mask = 1; - hash_mask = hash_mask << (num_hyperplanes_per_table_ - best_index - 1); - heap_.InsertUnsorted(score, ProbeCandidate(ii, hash_mask, 0)); - } - heap_.Heapify(); - } - - // This method stores the current probe (= hash table location) and - // corresponding table in the output parameters. The return value indicates - // whether this succeeded (true) or the current probing sequence is exhausted - // (false). Here, we say a probing sequence is exhausted if one of the - // following two conditions occurs: - // - We have used a non-negative value for num_probes in setup_probing, and - // we have produced this many number of probes in the current sequence. - // - We have used a negative value for num_probes in setup_probing, and we - // have produced all possible probes in the probing sequence. - bool GetNextProbe(HashType* cur_probe, int_fast32_t* cur_table) { - cur_probe_counter_ += 1; - - if (num_probes_ >= 0 && cur_probe_counter_ >= num_probes_) { - // We are out of probes in the current probing sequence. - return false; - } - - // For the first num_tables_ probes, we directly return the "standard LSH" - // probes to guarantee that they always come first and we avoid any - // multiprobe overhead. - if (cur_probe_counter_ < num_tables_) { - *cur_probe = main_table_probe_[cur_probe_counter_]; - *cur_table = cur_probe_counter_; - return true; - } - - // If the heap is empty, the current probing sequence is exhausted. - if (heap_.IsEmpty()) { - return false; - } - - CoordinateType cur_score; - ProbeCandidate cur_candidate; - heap_.ExtractMin(&cur_score, &cur_candidate); - *cur_table = cur_candidate.table_; - int_fast32_t cur_index = - sorted_hyperplane_indices_[*cur_table][cur_candidate.last_index_]; - *cur_probe = main_table_probe_[*cur_table] ^ cur_candidate.hash_mask_; - - if (cur_candidate.last_index_ != num_hyperplanes_per_table_ - 1) { - // swapping out the last flipped index - int_fast32_t next_index = - sorted_hyperplane_indices_[*cur_table][cur_candidate.last_index_ + 1]; - - // xor out previous bit, xor in new bit. - HashType next_mask = - cur_candidate.hash_mask_ ^ - (HashType(1) << (num_hyperplanes_per_table_ - cur_index - 1)) ^ - (HashType(1) << (num_hyperplanes_per_table_ - next_index - 1)); - - CoordinateType cur_coord = - hash_vector_[*cur_table * num_hyperplanes_per_table_ + cur_index]; - CoordinateType next_coord = - hash_vector_[*cur_table * num_hyperplanes_per_table_ + next_index]; - CoordinateType next_score = - cur_score - cur_coord * cur_coord + next_coord * next_coord; - - heap_.Insert(next_score, ProbeCandidate(*cur_table, next_mask, - cur_candidate.last_index_ + 1)); - - // adding a new flipped index - next_mask = - cur_candidate.hash_mask_ ^ - (HashType(1) << (num_hyperplanes_per_table_ - next_index - 1)); - next_score = cur_score + next_coord * next_coord; - - heap_.Insert(next_score, ProbeCandidate(*cur_table, next_mask, - cur_candidate.last_index_ + 1)); - } - - return true; - } - - private: - class ProbeCandidate { - public: - ProbeCandidate(int_fast32_t table = 0, HashType hash_mask = 0, - int_fast32_t last_index = 0) - : table_(table), hash_mask_(hash_mask), last_index_(last_index) {} - - int_fast32_t table_; - HashType hash_mask_; - int_fast32_t last_index_; - }; - - class HyperplaneComparator { - public: - HyperplaneComparator(const Vector& values, int_fast32_t offset) - : values_(values), offset_(offset) {} - - bool operator()(int_fast32_t ii, int_fast32_t jj) const { - return std::abs(values_[offset_ + ii]) < std::abs(values_[offset_ + jj]); - } - - private: - const Vector& values_; - int_fast32_t offset_; - }; - - int_fast32_t num_hyperplanes_per_table_; - int_fast32_t num_tables_; - int_fast64_t num_probes_; - int_fast64_t cur_probe_counter_; - std::vector> sorted_hyperplane_indices_; - std::vector main_table_probe_; - SimpleHeap heap_; - Vector hash_vector_; -}; - -} // namespace nearest_neighbor -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_NEAREST_NEIGHBOR_KERNELS_HYPERPLANE_LSH_PROBES_H_ diff --git a/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes_test.cc b/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes_test.cc deleted file mode 100644 index 87d807dcba5..00000000000 --- a/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes_test.cc +++ /dev/null @@ -1,130 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes.h" - -#include - -#include "tensorflow/core/kernels/ops_testutil.h" - -namespace { - -using tensorflow::uint32; - -typedef tensorflow::nearest_neighbor::HyperplaneMultiprobe - Multiprobe; - -void CheckSequenceSingleTable(Multiprobe* multiprobe, - const std::vector& expected_probes) { - uint32 cur_probe; - int_fast32_t cur_table; - for (int ii = 0; ii < expected_probes.size(); ++ii) { - ASSERT_TRUE(multiprobe->GetNextProbe(&cur_probe, &cur_table)); - EXPECT_EQ(expected_probes[ii], cur_probe); - EXPECT_EQ(0, cur_table); - } -} - -void CheckSequenceMultipleTables( - Multiprobe* multiprobe, - const std::vector>& expected_result) { - uint32 cur_probe; - int_fast32_t cur_table; - for (int ii = 0; ii < expected_result.size(); ++ii) { - ASSERT_TRUE(multiprobe->GetNextProbe(&cur_probe, &cur_table)); - EXPECT_EQ(expected_result[ii].first, cur_probe); - EXPECT_EQ(expected_result[ii].second, cur_table); - } -} - -// Just the first two probes for two tables and two hyperplanes pro table. -TEST(HyperplaneMultiprobeTest, SimpleTest1) { - Multiprobe multiprobe(2, 2); - Multiprobe::Vector hash_vector(4); - hash_vector << -1.0, 1.0, 1.0, -1.0; - std::vector> expected_result = {{1, 0}, - {2, 1}}; - multiprobe.SetupProbing(hash_vector, expected_result.size()); - CheckSequenceMultipleTables(&multiprobe, expected_result); -} - -// Checking that the beginning of a probing sequence for a single table is -// correct. -TEST(HyperplaneMultiprobeTest, SimpleTest2) { - Multiprobe multiprobe(4, 1); - Multiprobe::Vector hash_vector(4); - hash_vector << -2.0, -0.9, -0.8, -0.7; - std::vector expected_result = {0, 1, 2, 4, 3}; - multiprobe.SetupProbing(hash_vector, expected_result.size()); - CheckSequenceSingleTable(&multiprobe, expected_result); -} - -// Checking that the probing sequence for a single table is exhaustive. -TEST(HyperplaneMultiprobeTest, SimpleTest3) { - Multiprobe multiprobe(3, 1); - Multiprobe::Vector hash_vector(3); - hash_vector << -1.0, -10.0, -0.1; - std::vector expected_result = {0, 1, 4, 5, 2, 3, 6, 7}; - multiprobe.SetupProbing(hash_vector, expected_result.size()); - CheckSequenceSingleTable(&multiprobe, expected_result); -} - -// Checking that the probing sequence is generated correctly across tables. -TEST(HyperplaneMultiprobeTest, SimpleTest4) { - Multiprobe multiprobe(2, 2); - Multiprobe::Vector hash_vector(4); - hash_vector << -0.2, 0.9, 0.1, -1.0; - std::vector> expected_result = { - {1, 0}, {2, 1}, {0, 1}, {3, 0}, {0, 0}, {2, 0}, {3, 1}, {1, 1}}; - multiprobe.SetupProbing(hash_vector, expected_result.size()); - CheckSequenceMultipleTables(&multiprobe, expected_result); -} - -// Slightly larger test that checks whether we have an exhaustive probing -// sequence (but this test does not check the order). -TEST(HyperplaneMultiprobeTest, ExhaustiveTest1) { - int dim = 8; - int num_tables = 10; - Multiprobe multiprobe(dim, num_tables); - Multiprobe::Vector hash_vector(dim * num_tables); - - std::mt19937 random_generator(487344882); - std::normal_distribution distribution(0.0, 1.0); - for (int ii = 0; ii < dim * num_tables; ++ii) { - hash_vector[ii] = distribution(random_generator); - } - - std::vector> checked_cell(num_tables); - for (int ii = 0; ii < num_tables; ++ii) { - checked_cell[ii].resize(1 << dim); - std::fill(checked_cell[ii].begin(), checked_cell[ii].end(), false); - } - - int num_probes = (1 << dim) * num_tables; - multiprobe.SetupProbing(hash_vector, num_probes); - uint32 cur_probe; - int_fast32_t cur_table; - for (int ii = 0; ii < num_probes; ++ii) { - ASSERT_TRUE(multiprobe.GetNextProbe(&cur_probe, &cur_table)); - ASSERT_LE(0, cur_probe); - ASSERT_LT(cur_probe, 1 << dim); - ASSERT_LE(0, cur_table); - ASSERT_LT(cur_table, num_tables); - EXPECT_FALSE(checked_cell[cur_table][cur_probe]); - checked_cell[cur_table][cur_probe] = true; - } -} - -} // namespace diff --git a/tensorflow/contrib/nearest_neighbor/ops/nearest_neighbor_ops.cc b/tensorflow/contrib/nearest_neighbor/ops/nearest_neighbor_ops.cc deleted file mode 100644 index 00a3191074b..00000000000 --- a/tensorflow/contrib/nearest_neighbor/ops/nearest_neighbor_ops.cc +++ /dev/null @@ -1,49 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/op.h" - -namespace tensorflow { - -REGISTER_OP("HyperplaneLSHProbes") - .Attr("CoordinateType: {float, double}") - .Input("point_hyperplane_product: CoordinateType") - .Input("num_tables: int32") - .Input("num_hyperplanes_per_table: int32") - .Input("num_probes: int32") - .Output("probes: int32") - .Output("table_ids: int32") - .Doc(R"doc( -Computes probes for the hyperplane hash. - -The op supports multiprobing, i.e., the number of requested probes can be -larger than the number of tables. In that case, the same table can be probed -multiple times. - -The first `num_tables` probes are always the primary hashes for each table. - -point_hyperplane_product: a matrix of inner products between the hyperplanes - and the points to be hashed. These values should not be quantized so that we - can correctly compute the probing sequence. The expected shape is - `batch_size` times `num_tables * num_hyperplanes_per_table`, i.e., each - element of the batch corresponds to one row of the matrix. -num_tables: the number of tables to compute probes for. -num_hyperplanes_per_table: the number of hyperplanes per table. -num_probes: the requested number of probes per table. -probes: the output matrix of probes. Size `batch_size` times `num_probes`. -table_ids: the output matrix of tables ids. Size `batch_size` times `num_probes`. -)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/nearest_neighbor/python/kernel_tests/hyperplane_lsh_probes_test.py b/tensorflow/contrib/nearest_neighbor/python/kernel_tests/hyperplane_lsh_probes_test.py deleted file mode 100644 index d0955cbe115..00000000000 --- a/tensorflow/contrib/nearest_neighbor/python/kernel_tests/hyperplane_lsh_probes_test.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for hyperplane_lsh_probes.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.nearest_neighbor.python.ops.nearest_neighbor_ops import hyperplane_lsh_probes -from tensorflow.python.platform import test - - -class HyperplaneLshProbesTest(test.TestCase): - - # We only test the batch functionality of the op here because the multiprobe - # tests in hyperplane_lsh_probes_test.cc already cover most of the LSH - # functionality. - def simple_batch_test(self): - with self.cached_session(): - hyperplanes = np.eye(4) - points = np.array([[1.2, 0.5, -0.9, -1.0], [2.0, -3.0, 1.0, -1.5]]) - product = np.dot(points, hyperplanes) - num_tables = 2 - num_hyperplanes_per_table = 2 - num_probes = 4 - hashes, tables = hyperplane_lsh_probes(product, - num_tables, - num_hyperplanes_per_table, - num_probes) - - self.assertAllEqual(hashes.eval(), [[3, 0, 2, 2], [2, 2, 0, 3]]) - self.assertAllEqual(tables.eval(), [[0, 1, 0, 1], [0, 1, 1, 1]]) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/nearest_neighbor/python/ops/nearest_neighbor_ops.py b/tensorflow/contrib/nearest_neighbor/python/ops/nearest_neighbor_ops.py deleted file mode 100644 index 2b4bc55f5b9..00000000000 --- a/tensorflow/contrib/nearest_neighbor/python/ops/nearest_neighbor_ops.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Wrappers for nearest neighbor operations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.util import loader -from tensorflow.python.framework import ops -from tensorflow.python.platform import resource_loader - -_nearest_neighbor_ops = loader.load_op_library( - resource_loader.get_path_to_datafile("_nearest_neighbor_ops.so")) - - -def hyperplane_lsh_probes(point_hyperplane_product, - num_tables, - num_hyperplanes_per_table, - num_probes, - name=None): - """Computes probes for the hyperplane hash. - - The op supports multiprobing, i.e., the number of requested probes can be - larger than the number of tables. In that case, the same table can be probed - multiple times. - - The first `num_tables` probes are always the primary hashes for each table. - - Args: - point_hyperplane_product: a matrix of inner products between the hyperplanes - and the points to be hashed. These values should not be quantized so that - we can correctly compute the probing sequence. The expected shape is - `batch_size` times `num_tables * num_hyperplanes_per_table`, i.e., each - element of the batch corresponds to one row of the matrix. - num_tables: the number of tables to compute probes for. - num_hyperplanes_per_table: the number of hyperplanes per table. - num_probes: the requested number of probes per table. - name: A name prefix for the returned tensors (optional). - - Returns: - probes: the output matrix of probes. Size `batch_size` times `num_probes`. - table_ids: the output matrix of tables ids. Size `batch_size` times - `num_probes`. - """ - return _nearest_neighbor_ops.hyperplane_lsh_probes(point_hyperplane_product, - num_tables, - num_hyperplanes_per_table, - num_probes, - name=name) - -ops.NotDifferentiable("HyperplaneLSHProbes") diff --git a/tensorflow/contrib/nn/BUILD b/tensorflow/contrib/nn/BUILD deleted file mode 100644 index bb7d6b54e30..00000000000 --- a/tensorflow/contrib/nn/BUILD +++ /dev/null @@ -1,105 +0,0 @@ -# Description: -# Contains deprecated ops to calculate cross entropy. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "nn_py", - srcs = [ - "__init__.py", - "python/__init__.py", - "python/ops/__init__.py", - "python/ops/alpha_dropout.py", - "python/ops/cross_entropy.py", - "python/ops/fwd_gradients.py", - "python/ops/sampling_ops.py", - "python/ops/scaled_softplus.py", - ], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:function", - "//tensorflow/python:gradients", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:nn_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:tensor_util", - "//tensorflow/python:util", - ], -) - -py_test( - name = "alpha_dropout_test", - size = "small", - srcs = ["python/ops/alpha_dropout_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":nn_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:nn", - "//tensorflow/python:random_ops", - ], -) - -py_test( - name = "fwd_gradients_test", - size = "small", - srcs = ["python/ops/fwd_gradients_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":nn_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:math_ops", - ], -) - -py_test( - name = "sampling_ops_test", - size = "small", - srcs = ["python/ops/sampling_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":nn_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:nn", - ], -) - -py_test( - name = "scaled_softplus_test", - size = "small", - srcs = ["python/ops/scaled_softplus_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":nn_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:gradient_checker", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/nn/__init__.py b/tensorflow/contrib/nn/__init__.py deleted file mode 100644 index 96d60e14980..00000000000 --- a/tensorflow/contrib/nn/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Module for variants of ops in tf.nn. - -@@alpha_dropout -@@conv1d_transpose -@@deprecated_flipped_softmax_cross_entropy_with_logits -@@deprecated_flipped_sparse_softmax_cross_entropy_with_logits -@@deprecated_flipped_sigmoid_cross_entropy_with_logits -@@nth_element -@@rank_sampled_softmax_loss -@@sampled_sparse_softmax_loss -@@scaled_softplus -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.nn.python.ops.alpha_dropout import * -from tensorflow.contrib.nn.python.ops.cross_entropy import * -from tensorflow.contrib.nn.python.ops.sampling_ops import * -from tensorflow.contrib.nn.python.ops.scaled_softplus import * -from tensorflow.python.ops.nn_ops import conv1d_transpose -from tensorflow.python.ops.nn_ops import nth_element -# pylint: enable=unused-import,wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/nn/python/__init__.py b/tensorflow/contrib/nn/python/__init__.py deleted file mode 100644 index 73dd4d40151..00000000000 --- a/tensorflow/contrib/nn/python/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Module for variants of ops in tf.nn.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/nn/python/ops/__init__.py b/tensorflow/contrib/nn/python/ops/__init__.py deleted file mode 100644 index 73dd4d40151..00000000000 --- a/tensorflow/contrib/nn/python/ops/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Module for variants of ops in tf.nn.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/nn/python/ops/alpha_dropout.py b/tensorflow/contrib/nn/python/ops/alpha_dropout.py deleted file mode 100644 index ad9f223f302..00000000000 --- a/tensorflow/contrib/nn/python/ops/alpha_dropout.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numbers - -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops - - -def alpha_dropout(x, keep_prob, noise_shape=None, seed=None, name=None): # pylint: disable=invalid-name - """Computes alpha dropout. - - Alpha Dropout is a dropout that maintains the self-normalizing property. For - an input with zero mean and unit standard deviation, the output of - Alpha Dropout maintains the original mean and standard deviation of the input. - - See [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515) - - Args: - x: A tensor. - keep_prob: A scalar `Tensor` with the same type as x. The probability - that each element is kept. - noise_shape: A 1-D `Tensor` of type `int32`, representing the - shape for randomly generated keep/drop flags. - seed: A Python integer. Used to create random seeds. See - `tf.compat.v1.set_random_seed` for behavior. - name: A name for this operation (optional). - - Returns: - A Tensor of the same shape of `x`. - - Raises: - ValueError: If `keep_prob` is not in `(0, 1]`. - - """ - with ops.name_scope(name, "alpha_dropout", [x]) as name: - x = ops.convert_to_tensor(x, name="x") - if isinstance(keep_prob, numbers.Real) and not 0 < keep_prob <= 1.: - raise ValueError("keep_prob must be a scalar tensor or a float in the " - "range (0, 1], got %g" % keep_prob) - keep_prob = ops.convert_to_tensor(keep_prob, - dtype=x.dtype, - name="keep_prob") - keep_prob.get_shape().assert_has_rank(0) - - # Do nothing if we know keep_prob == 1 - if tensor_util.constant_value(keep_prob) == 1: - return x - - alpha = -1.7580993408473766 - - noise_shape = noise_shape if noise_shape is not None else array_ops.shape(x) - random_tensor = random_ops.random_uniform(noise_shape, - seed=seed, - dtype=x.dtype) - kept_idx = gen_math_ops.greater_equal(random_tensor, 1 - keep_prob) - kept_idx = math_ops.cast(kept_idx, x.dtype) - # Mask - x = x * kept_idx + alpha * (1 - kept_idx) - - # Affine transformation parameters - a = (keep_prob + keep_prob * (1 - keep_prob) * alpha ** 2) ** -0.5 - b = -a * alpha * (1 - keep_prob) - - # Affine transformation - return a * x + b diff --git a/tensorflow/contrib/nn/python/ops/alpha_dropout_test.py b/tensorflow/contrib/nn/python/ops/alpha_dropout_test.py deleted file mode 100644 index 3aec88bcbfe..00000000000 --- a/tensorflow/contrib/nn/python/ops/alpha_dropout_test.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for sampling_ops.py.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.nn.python.ops.alpha_dropout import alpha_dropout -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.platform import test - - -class AlphaDropoutTest(test.TestCase): - - def testAlphaDropout(self): - x_dim, y_dim = 40, 30 - for keep_prob in [0.1, 0.5, 0.8]: - with self.cached_session(): - t = random_ops.random_normal([x_dim, y_dim]) - output = alpha_dropout(t, keep_prob) - self.assertEqual([x_dim, y_dim], output.get_shape()) - t_mean, t_std = nn_impl.moments(t, axes=[0, 1]) - output_mean, output_std = nn_impl.moments(output, axes=[0, 1]) - self.assertLess(abs(t_mean.eval() - output_mean.eval()), 0.1) - self.assertLess(abs(t_std.eval() - output_std.eval()), 0.1) - - def testShapedDropoutShapeError(self): - # Runs shaped dropout and verifies an error is thrown on misshapen noise. - x_dim = 40 - y_dim = 30 - keep_prob = 0.5 - t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) - with self.assertRaises(ValueError): - _ = alpha_dropout(t, keep_prob, noise_shape=[x_dim, y_dim + 10]) - with self.assertRaises(ValueError): - _ = alpha_dropout(t, keep_prob, noise_shape=[x_dim, y_dim, 5]) - with self.assertRaises(ValueError): - _ = alpha_dropout(t, keep_prob, noise_shape=[x_dim + 3]) - with self.assertRaises(ValueError): - _ = alpha_dropout(t, keep_prob, noise_shape=[x_dim]) - - # test that broadcasting proceeds - _ = alpha_dropout(t, keep_prob, noise_shape=[y_dim]) - _ = alpha_dropout(t, keep_prob, noise_shape=[1, y_dim]) - _ = alpha_dropout(t, keep_prob, noise_shape=[x_dim, 1]) - _ = alpha_dropout(t, keep_prob, noise_shape=[1, 1]) - - def testInvalidKeepProb(self): - x_dim, y_dim = 40, 30 - t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) - with self.assertRaises(ValueError): - alpha_dropout(t, -1.0) - with self.assertRaises(ValueError): - alpha_dropout(t, 1.1) - with self.assertRaises(ValueError): - alpha_dropout(t, [0.0, 1.0]) - with self.assertRaises(ValueError): - alpha_dropout(t, array_ops.placeholder(dtypes.float64)) - with self.assertRaises(ValueError): - alpha_dropout(t, array_ops.placeholder(dtypes.float32, shape=[2])) - - def testNoDropoutFast(self): - x = array_ops.zeros((5,)) - for p in 1, constant_op.constant(1.0): - y = alpha_dropout(x, keep_prob=p) - self.assertTrue(x is y) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/nn/python/ops/cross_entropy.py b/tensorflow/contrib/nn/python/ops/cross_entropy.py deleted file mode 100644 index 5045f2c957f..00000000000 --- a/tensorflow/contrib/nn/python/ops/cross_entropy.py +++ /dev/null @@ -1,177 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Deprecated Wrappers for Neural Net (NN) Cross Entropy Operations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import nn - - -# TODO(b/33392402): Formally deprecate this API. -# After LSC (see b/33392402#comment1), this API will be deprecated and callers -# will be suggested to use the (updated version of) -# tf.nn.softmax_cross_entropy_with_logits. -def deprecated_flipped_softmax_cross_entropy_with_logits(logits, - labels, - dim=-1, - name=None): - """Computes softmax cross entropy between `logits` and `labels`. - - This function diffs from tf.nn.softmax_cross_entropy_with_logits only in the - argument order. - - Measures the probability error in discrete classification tasks in which the - classes are mutually exclusive (each entry is in exactly one class). For - example, each CIFAR-10 image is labeled with one and only one label: an image - can be a dog or a truck, but not both. - - **NOTE:** While the classes are mutually exclusive, their probabilities - need not be. All that is required is that each row of `labels` is - a valid probability distribution. If they are not, the computation of the - gradient will be incorrect. - - If using exclusive `labels` (wherein one and only - one class is true at a time), see `sparse_softmax_cross_entropy_with_logits`. - - **WARNING:** This op expects unscaled logits, since it performs a `softmax` - on `logits` internally for efficiency. Do not call this op with the - output of `softmax`, as it will produce incorrect results. - - `logits` and `labels` must have the same shape `[batch_size, num_classes]` - and the same dtype (either `float16`, `float32`, or `float64`). - - Args: - logits: Unscaled log probabilities. - labels: Each row `labels[i]` must be a valid probability distribution. - dim: The class dimension. Defaulted to -1 which is the last dimension. - name: A name for the operation (optional). - - Returns: - A 1-D `Tensor` of length `batch_size` of the same type as `logits` with the - softmax cross entropy loss. - """ - return nn.softmax_cross_entropy_with_logits( - labels=labels, logits=logits, dim=dim, name=name) - - -# TODO(b/33392402): Formally deprecate this API. -# After LSC (see b/33392402#comment1), this API will be deprecated and callers -# will be suggested to use the (updated version of) -# tf.nn.sparse_softmax_cross_entropy_with_logits. -def deprecated_flipped_sparse_softmax_cross_entropy_with_logits(logits, - labels, - name=None): - """Computes sparse softmax cross entropy between `logits` and `labels`. - - This function diffs from tf.nn.sparse_softmax_cross_entropy_with_logits only - in the argument order. - - Measures the probability error in discrete classification tasks in which the - classes are mutually exclusive (each entry is in exactly one class). For - example, each CIFAR-10 image is labeled with one and only one label: an image - can be a dog or a truck, but not both. - - **NOTE:** For this operation, the probability of a given label is considered - exclusive. That is, soft classes are not allowed, and the `labels` vector - must provide a single specific index for the true class for each row of - `logits` (each minibatch entry). For soft softmax classification with - a probability distribution for each entry, see - `softmax_cross_entropy_with_logits`. - - **WARNING:** This op expects unscaled logits, since it performs a softmax - on `logits` internally for efficiency. Do not call this op with the - output of `softmax`, as it will produce incorrect results. - - A common use case is to have logits of shape `[batch_size, num_classes]` and - labels of shape `[batch_size]`. But higher dimensions are supported. - - Args: - - logits: Unscaled log probabilities of rank `r` and shape - `[d_0, d_1, ..., d_{r-2}, num_classes]` and dtype `float32` or `float64`. - labels: `Tensor` of shape `[d_0, d_1, ..., d_{r-2}]` and dtype `int32` or - `int64`. Each entry in `labels` must be an index in `[0, num_classes)`. - Other values will raise an exception when this op is run on CPU, and - return `NaN` for corresponding corresponding loss and gradient rows - on GPU. - name: A name for the operation (optional). - - Returns: - A `Tensor` of the same shape as `labels` and of the same type as `logits` - with the softmax cross entropy loss. - - Raises: - ValueError: If logits are scalars (need to have rank >= 1) or if the rank - of the labels is not equal to the rank of the logits minus one. - """ - return nn.sparse_softmax_cross_entropy_with_logits( - labels=labels, logits=logits, name=name) - - -# TODO(b/33392402): Formally deprecate this API. -# After LSC (see b/33392402#comment1), this API will be deprecated and callers -# will be suggested to use the (updated version of) -# tf.nn.sigmoid_cross_entropy_with_logits. -def deprecated_flipped_sigmoid_cross_entropy_with_logits(logits, - targets, - name=None): - """Computes sigmoid cross entropy given `logits`. - - This function diffs from tf.nn.sigmoid_cross_entropy_with_logits only in the - argument order. - - Measures the probability error in discrete classification tasks in which each - class is independent and not mutually exclusive. For instance, one could - perform multilabel classification where a picture can contain both an elephant - and a dog at the same time. - - For brevity, let `x = logits`, `z = targets`. The logistic loss is - - z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - = z * -log(1 / (1 + exp(-x))) + (1 - z) * -log(exp(-x) / (1 + exp(-x))) - = z * log(1 + exp(-x)) + (1 - z) * (-log(exp(-x)) + log(1 + exp(-x))) - = z * log(1 + exp(-x)) + (1 - z) * (x + log(1 + exp(-x)) - = (1 - z) * x + log(1 + exp(-x)) - = x - x * z + log(1 + exp(-x)) - - For x < 0, to avoid overflow in exp(-x), we reformulate the above - - x - x * z + log(1 + exp(-x)) - = log(exp(x)) - x * z + log(1 + exp(-x)) - = - x * z + log(1 + exp(x)) - - Hence, to ensure stability and avoid overflow, the implementation uses this - equivalent formulation - - max(x, 0) - x * z + log(1 + exp(-abs(x))) - - `logits` and `targets` must have the same type and shape. - - Args: - logits: A `Tensor` of type `float32` or `float64`. - targets: A `Tensor` of the same type and shape as `logits`. - name: A name for the operation (optional). - - Returns: - A `Tensor` of the same shape as `logits` with the componentwise - logistic losses. - - Raises: - ValueError: If `logits` and `targets` do not have the same shape. - """ - return nn.sigmoid_cross_entropy_with_logits( - labels=targets, logits=logits, name=name) diff --git a/tensorflow/contrib/nn/python/ops/fwd_gradients.py b/tensorflow/contrib/nn/python/ops/fwd_gradients.py deleted file mode 100644 index 922497779b1..00000000000 --- a/tensorflow/contrib/nn/python/ops/fwd_gradients.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Forward-mode derivatives.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops.gradients_impl import gradients - - -def fwd_gradients(ys, xs, grad_xs=None, assert_unused=False): - """Computes forward-mode derivatives. - - This is accomplished in pure-python using tensorflow's existing (reverse-mode) - gradients. There is additional overhead on graph construction, but runtime - performance should be equal to a manual implementation [citation needed]. - - See https://j-towns.github.io/2017/06/12/A-new-trick.html and - https://github.com/HIPS/autograd/pull/175 for the original discussion of this - method, and https://github.com/renmengye/tensorflow-forward-ad for a "direct" - implementation. - - Args: - ys: A list of tensors. - xs: A list of tensors. - grad_xs: An optional list of tensors. If provided, must have the same length - and shapes compatible with xs. - assert_unused: Add assertions that intermediate values are not computed. - Returns: - A list of tensors of the same shapes as ys. The directional derivatives of - ys with respect to xs in the direction grad_xs. Leaving grad_xs unspecified - is equivalent to passing in 1s for each x in xs. - """ - # This version of forward-mode autodiff is based on code by Tim Cooijmans - # and handles list arguments and certain special cases such as when the - # ys doesn't depend on one or more of the xs, and when tf.IndexedSlices are - # generated by the first tf.gradients call. - - us = [array_ops.zeros_like(y) + float('nan') for y in ys] - - dydxs = gradients(ys, xs, grad_ys=us) - - # deal with strange types that tf.gradients returns but can't deal with - dydxs = [ops.convert_to_tensor(dydx) if isinstance(dydx, ops.IndexedSlices) - else dydx for dydx in dydxs] - - if assert_unused: - with ops.control_dependencies(dydxs): - assert_unused = control_flow_ops.Assert(False, [1], name='fwd_gradients') - with ops.control_dependencies([assert_unused]): - dydxs = array_ops.identity_n(dydxs) - - dydxs = [array_ops.zeros_like(x) if dydx is None else dydx - for x, dydx in zip(xs, dydxs)] - for x, dydx in zip(xs, dydxs): - dydx.set_shape(x.shape) - - dysdx = gradients(dydxs, us, grad_ys=grad_xs) - - return dysdx diff --git a/tensorflow/contrib/nn/python/ops/fwd_gradients_test.py b/tensorflow/contrib/nn/python/ops/fwd_gradients_test.py deleted file mode 100644 index 4cdac6a7429..00000000000 --- a/tensorflow/contrib/nn/python/ops/fwd_gradients_test.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for forward_ad.py.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.nn.python.ops import fwd_gradients -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class ForwardAdTest(test.TestCase): - - def testSquare(self): - x = constant_op.constant(1.) - y = math_ops.square(x) - grad_x = 3. - - dydx_tf = fwd_gradients.fwd_gradients([y], [x], [grad_x])[0] - dydx_py = 2. * grad_x - - with self.cached_session() as sess: - self.assertAllClose(sess.run(dydx_tf), dydx_py, 1e-6) - - def testGather(self): - x = constant_op.constant([1., 2., 3.]) - y = array_ops.gather(x, [0, 1]) - y.set_shape([2]) - dydx = fwd_gradients.fwd_gradients([y], [x], assert_unused=True) - - with self.cached_session() as sess: - sess.run(dydx) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/nn/python/ops/sampling_ops.py b/tensorflow/contrib/nn/python/ops/sampling_ops.py deleted file mode 100644 index de71b0845e2..00000000000 --- a/tensorflow/contrib/nn/python/ops/sampling_ops.py +++ /dev/null @@ -1,342 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops related to candidate sampling.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import nn_ops - - -def _rank_resample(weights, biases, inputs, sampled_values, num_resampled, - resampling_temperature, partition_strategy): - """A helper function for rank_sampled_softmax_loss. - - This computes, for each i in `sampled_values`, - - log(sum_j exp((w_i * x_j + b_i) / resampling_temperature)) - - where w_i, b_i are the weight and bias of the i-th class, respectively, - and j ranges over the rows of `inputs`. For efficiency, we rearrange the - computation to - - log(sum_j exp(w_i * (x_j / resampling_temperature))) + - b_i / resampling_temperature. - - This translates to the following batched computation using tensorflow ops: - - reduce_logsumexp(matmul(embeddings, - transpose(inputs / resampling_temperature))) + - biases / resampling_temperature - - The computation of the first term is colocated with the embeddings using - `transform_fn` in `embedding_ops._embedding_lookup_and_transform`. The second - term, not the bottleneck, is computed at the worker. - - Args: - weights: From `rank_sampled_softmax_loss`. - biases: From `rank_sampled_softmax_loss`. - inputs: From `rank_sampled_softmax_loss`. - sampled_values: A tuple of (`sampled_candidates`, `true_expected_count`, - `sampled_expected_count`) returned by a `*_candidate_sampler` function. - num_resampled: An `int`. This many values are selected from - `sampled_values` using the adaptive resampling algorithm. The caller - must ensure that `num_resampled` is less than the size of - `sampled_values`. - resampling_temperature: A scalar `Tensor` with the temperature parameter - for the adaptive resampling algorithm. - partition_strategy: From `rank_sampled_softmax_loss`. - - Returns: - A tuple of (`resampled_candidates`, `true_expected_count`, - `resampled_expected_count`), similar to `sampled_values` but sampled - down to `num_resampled` values. - """ - # This code supports passing a Tensor for num_resampled, but since it is only - # called with an int, that's what we specify in the arg list. If this - # function is ever externalized, we should change the doc to support Tensor. - - sampled, true_expected_count, sampled_expected_count = sampled_values - - sampled = math_ops.cast(array_ops.stop_gradient(sampled), dtypes.int64) - true_expected_count = array_ops.stop_gradient(true_expected_count) - sampled_expected_count = array_ops.stop_gradient(sampled_expected_count) - - reweighted_inputs = inputs / resampling_temperature - - def logsumexp_logit(embeddings): - return math_ops.reduce_logsumexp( - math_ops.matmul(embeddings, reweighted_inputs, transpose_b=True), - axis=1, - keepdims=False) - - # Calling this protected form of embedding_lookup allows co-locating - # the logsumexp computation with the partitioned weights, which yields - # a large speedup in practice. - sampled_logits = embedding_ops._embedding_lookup_and_transform( # pylint: disable=protected-access - weights, sampled, partition_strategy, transform_fn=logsumexp_logit) - sampled_b = array_ops.reshape( - embedding_ops.embedding_lookup(biases, sampled, partition_strategy), [-1]) - sampled_logits += sampled_b / resampling_temperature - - _, resampled_indices = nn.top_k(sampled_logits, k=num_resampled, sorted=False) - resampled = array_ops.gather(sampled, indices=resampled_indices) - resampled_expected_count = array_ops.gather( - sampled_expected_count, indices=resampled_indices) - - return resampled, true_expected_count, resampled_expected_count - - -def rank_sampled_softmax_loss(weights, - biases, - labels, - inputs, - num_sampled, - num_resampled, - num_classes, - num_true, - sampled_values, - resampling_temperature, - remove_accidental_hits, - partition_strategy, - name=None): - """Computes softmax loss using rank-based adaptive resampling. - - This has been shown to improve rank loss after training compared to - `tf.nn.sampled_softmax_loss`. For a description of the algorithm and some - experimental results, please see: [TAPAS: Two-pass Approximate Adaptive - Sampling for Softmax](https://arxiv.org/abs/1707.03073). - - Sampling follows two phases: - * In the first phase, `num_sampled` classes are selected using - `tf.nn.learned_unigram_candidate_sampler` or supplied `sampled_values`. - The logits are calculated on those sampled classes. This phases is - similar to `tf.nn.sampled_softmax_loss`. - * In the second phase, the `num_resampled` classes with highest predicted - probability are kept. Probabilities are - `LogSumExp(logits / resampling_temperature)`, where the sum is over - `inputs`. - - The `resampling_temperature` parameter controls the "adaptiveness" of the - resampling. At lower temperatures, resampling is more adaptive because it - picks more candidates close to the predicted classes. A common strategy is - to decrease the temperature as training proceeds. - - See `tf.nn.sampled_softmax_loss` for more documentation on sampling and - for typical default values for some of the parameters. - - This operation is for training only. It is generally an underestimate of - the full softmax loss. - - A common use case is to use this method for training, and calculate the full - softmax loss for evaluation or inference. In this case, you must set - `partition_strategy="div"` for the two losses to be consistent, as in the - following example: - - ```python - if mode == "train": - loss = rank_sampled_softmax_loss( - weights=weights, - biases=biases, - labels=labels, - inputs=inputs, - ..., - partition_strategy="div") - elif mode == "eval": - logits = tf.matmul(inputs, tf.transpose(weights)) - logits = tf.nn.bias_add(logits, biases) - labels_one_hot = tf.one_hot(labels, n_classes) - loss = tf.nn.softmax_cross_entropy_with_logits( - labels=labels_one_hot, - logits=logits) - ``` - - Args: - weights: A `Tensor` or `PartitionedVariable` of shape `[num_classes, dim]`, - or a list of `Tensor` objects whose concatenation along dimension 0 - has shape [num_classes, dim]. The (possibly-sharded) class embeddings. - biases: A `Tensor` or `PartitionedVariable` of shape `[num_classes]`. - The (possibly-sharded) class biases. - labels: A `Tensor` of type `int64` and shape `[batch_size, - num_true]`. The target classes. Note that this format differs from - the `labels` argument of `nn.softmax_cross_entropy_with_logits`. - inputs: A `Tensor` of shape `[batch_size, dim]`. The forward - activations of the input network. - num_sampled: An `int`. The number of classes to randomly sample per batch. - num_resampled: An `int`. The number of classes to select from the - `num_sampled` classes using the adaptive resampling algorithm. Must be - less than `num_sampled`. - num_classes: An `int`. The number of possible classes. - num_true: An `int`. The number of target classes per training example. - sampled_values: A tuple of (`sampled_candidates`, `true_expected_count`, - `sampled_expected_count`) returned by a `*_candidate_sampler` function. - If None, default to `nn.learned_unigram_candidate_sampler`. - resampling_temperature: A scalar `Tensor` with the temperature parameter - for the adaptive resampling algorithm. - remove_accidental_hits: A `bool`. Whether to remove "accidental hits" - where a sampled class equals one of the target classes. - partition_strategy: A string specifying the partitioning strategy, relevant - if `len(weights) > 1`. Currently `"div"` and `"mod"` are supported. - See `tf.nn.embedding_lookup` for more details. - name: A name for the operation (optional). - - Returns: - A `batch_size` 1-D tensor of per-example sampled softmax losses. - - Raises: - ValueError: If `num_sampled <= num_resampled`. - """ - if num_sampled > num_classes: - raise ValueError("num_sampled ({}) cannot be greater than num_classes ({})". - format(num_sampled, num_classes)) - if num_sampled <= num_resampled: - raise ValueError("num_resampled ({}) must be less than num_sampled ({})". - format(num_resampled, num_sampled)) - if partition_strategy not in ("div", "mod"): - raise ValueError( - "unsupported partition_strategy ({})".format(partition_strategy)) - with ops.name_scope(name, "rank_sampled_softmax_loss", [ - weights, biases, labels, inputs, sampled_values, resampling_temperature - ]) as name: - if not sampled_values: - sampled_values = nn.learned_unigram_candidate_sampler( - true_classes=labels, - num_true=num_true, - num_sampled=num_sampled, - unique=True, - range_max=num_classes) - # From sampled_values, select the top num_resampled values using the - # adaptive rank resampling strategy. - resampled_values = _rank_resample(weights, biases, inputs, sampled_values, - num_resampled, resampling_temperature, - partition_strategy) - return nn.sampled_softmax_loss( - weights=weights, - biases=biases, - labels=labels, - inputs=inputs, - num_sampled=num_resampled, - num_classes=num_classes, - num_true=num_true, - sampled_values=resampled_values, - remove_accidental_hits=remove_accidental_hits, - partition_strategy=partition_strategy, - name=name) - - -def sampled_sparse_softmax_loss(weights, - biases, - labels, - inputs, - num_sampled, - num_classes, - sampled_values=None, - remove_accidental_hits=True, - partition_strategy="mod", - name="sampled_sparse_softmax_loss"): - """Computes and returns the sampled sparse softmax training loss. - - This is a faster way to train a softmax classifier over a huge number of - classes. - - This operation is for training only. It is generally an underestimate of - the full softmax loss. - - A common use case is to use this method for training, and calculate the full - softmax loss for evaluation or inference. In this case, you must set - `partition_strategy="div"` for the two losses to be consistent, as in the - following example: - - ```python - if mode == "train": - loss = tf.nn.sampled_sparse_softmax_loss( - weights=weights, - biases=biases, - labels=labels, - inputs=inputs, - ..., - partition_strategy="div") - elif mode == "eval": - logits = tf.matmul(inputs, tf.transpose(weights)) - logits = tf.nn.bias_add(logits, biases) - loss = tf.nn.sparse_softmax_cross_entropy_with_logits( - labels=tf.squeeze(labels), - logits=logits) - ``` - - See our [Candidate Sampling Algorithms Reference] - (https://www.tensorflow.org/extras/candidate_sampling.pdf) - - Also see Section 3 of [Jean et al., 2014](http://arxiv.org/abs/1412.2007) - ([pdf](http://arxiv.org/pdf/1412.2007.pdf)) for the math. - - Args: - weights: A `Tensor` of shape `[num_classes, dim]`, or a list of `Tensor` - objects whose concatenation along dimension 0 has shape - [num_classes, dim]. The (possibly-sharded) class embeddings. - biases: A `Tensor` of shape `[num_classes]`. The class biases. - labels: A `Tensor` of type `int64` and shape `[batch_size, 1]`. - The index of the single target class for each row of logits. Note that - this format differs from the `labels` argument of - `nn.sparse_softmax_cross_entropy_with_logits`. - inputs: A `Tensor` of shape `[batch_size, dim]`. The forward - activations of the input network. - num_sampled: An `int`. The number of classes to randomly sample per batch. - num_classes: An `int`. The number of possible classes. - sampled_values: a tuple of (`sampled_candidates`, `true_expected_count`, - `sampled_expected_count`) returned by a `*_candidate_sampler` function. - (if None, we default to `log_uniform_candidate_sampler`) - remove_accidental_hits: A `bool`. whether to remove "accidental hits" - where a sampled class equals one of the target classes. Default is - True. - partition_strategy: A string specifying the partitioning strategy, relevant - if `len(weights) > 1`. Currently `"div"` and `"mod"` are supported. - Default is `"mod"`. See `tf.nn.embedding_lookup` for more details. - name: A name for the operation (optional). - - Returns: - A `batch_size` 1-D tensor of per-example sampled softmax losses. - - """ - logits, _ = nn_impl._compute_sampled_logits( - weights=weights, - biases=biases, - labels=labels, - inputs=inputs, - num_sampled=num_sampled, - num_classes=num_classes, - num_true=1, - sampled_values=sampled_values, - subtract_log_q=True, - remove_accidental_hits=remove_accidental_hits, - partition_strategy=partition_strategy, - name=name) - - # There is only one true label. _compute_sampled_logits puts the true logit - # at index 0. - labels = array_ops.zeros([array_ops.shape(logits)[0], 1], dtype=dtypes.int64) - - sampled_losses = nn_ops.sparse_softmax_cross_entropy_with_logits( - labels=array_ops.squeeze(labels), logits=logits) - # sampled_losses is a [batch_size] tensor. - return sampled_losses diff --git a/tensorflow/contrib/nn/python/ops/sampling_ops_test.py b/tensorflow/contrib/nn/python/ops/sampling_ops_test.py deleted file mode 100644 index 11738bb215c..00000000000 --- a/tensorflow/contrib/nn/python/ops/sampling_ops_test.py +++ /dev/null @@ -1,322 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for sampling_ops.py.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.nn.python.ops import sampling_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import nn -from tensorflow.python.platform import test - - -class RankSampledSoftmaxLossTest(test.TestCase): - - def setUp(self): - self._sampled = [3, 4, 5, 6, 7] - self._num_sampled = len(self._sampled) - # Because values of all matrices increase with indices, logits increase with - # class id. So, for the above sampled classes, adaptive sampling will select - # these resampled classes. - self._resampled = [5, 6, 7] - self._num_resampled = len(self._resampled) - self._num_classes = 10 - self._num_true = 2 - self._sampled_values = (self._sampled, [[0.5], [0.5]], - [0.5, 0.5, 0.5, 0.5, 0.5]) - self._resampled_values = (self._resampled, [[0.5], [0.5]], [0.5, 0.5, 0.5]) - self._remove_accidental_hits = False - self._embed_dim = 5 - self._batch_size = 2 - - def _weights(self): - return constant_op.constant([ - [0.0, 0.1, 0.2, 0.3, 0.4], - [1.0, 1.1, 1.2, 1.3, 1.4], - [2.0, 2.1, 2.2, 2.3, 2.4], - [3.0, 3.1, 3.2, 3.3, 3.4], - [4.0, 4.1, 4.2, 4.3, 4.4], - [5.0, 5.1, 5.2, 5.3, 5.4], - [6.0, 6.1, 6.2, 6.3, 6.4], - [7.0, 7.1, 7.2, 7.3, 7.4], - [8.0, 8.1, 8.2, 8.3, 8.4], - [9.0, 9.1, 9.2, 9.3, 9.4], - ]) - - def _div_sharded_weights(self): - return [ - constant_op.constant([ - [0.0, 0.1, 0.2, 0.3, 0.4], - [1.0, 1.1, 1.2, 1.3, 1.4], - ]), - constant_op.constant([ - [2.0, 2.1, 2.2, 2.3, 2.4], - [3.0, 3.1, 3.2, 3.3, 3.4], - ]), - constant_op.constant([ - [4.0, 4.1, 4.2, 4.3, 4.4], - [5.0, 5.1, 5.2, 5.3, 5.4], - ]), - constant_op.constant([ - [6.0, 6.1, 6.2, 6.3, 6.4], - [7.0, 7.1, 7.2, 7.3, 7.4], - ]), - constant_op.constant([ - [8.0, 8.1, 8.2, 8.3, 8.4], - [9.0, 9.1, 9.2, 9.3, 9.4], - ]), - ] - - def _mod_sharded_weights(self): - return [ - constant_op.constant([ - [0.0, 0.1, 0.2, 0.3, 0.4], - [5.0, 5.1, 5.2, 5.3, 5.4], - ]), - constant_op.constant([ - [1.0, 1.1, 1.2, 1.3, 1.4], - [6.0, 6.1, 6.2, 6.3, 6.4], - ]), - constant_op.constant([ - [2.0, 2.1, 2.2, 2.3, 2.4], - [7.0, 7.1, 7.2, 7.3, 7.4], - ]), - constant_op.constant([ - [3.0, 3.1, 3.2, 3.3, 3.4], - [8.0, 8.1, 8.2, 8.3, 8.4], - ]), - constant_op.constant([ - [4.0, 4.1, 4.2, 4.3, 4.4], - [9.0, 9.1, 9.2, 9.3, 9.4], - ]), - ] - - def _biases(self): - return constant_op.constant( - [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]) - - def _div_sharded_biases(self): - return [ - constant_op.constant([0.0, 0.1]), - constant_op.constant([0.2, 0.3]), - constant_op.constant([0.4, 0.5]), - constant_op.constant([0.6, 0.7]), - constant_op.constant([0.8, 0.9]), - ] - - def _mod_sharded_biases(self): - return [ - constant_op.constant([0.0, 0.5]), - constant_op.constant([0.1, 0.6]), - constant_op.constant([0.2, 0.7]), - constant_op.constant([0.3, 0.8]), - constant_op.constant([0.4, 0.9]), - ] - - def _labels(self): - return constant_op.constant( - [[0, 1], [1, 2]], - shape=(self._batch_size, self._num_true), - name='labels', - dtype=dtypes.int64) - - def _inputs(self): - return constant_op.constant( - [ - [0., 1., 2., 3., 4.], - [10., 11., 12., 13., 14.], - ], - shape=(self._batch_size, self._embed_dim), - name='inputs') - - def testInvalidNumSampled0(self): - with ops.Graph().as_default(): - with self.assertRaisesRegexp( - ValueError, - r'num_resampled \(3\) must be less than num_sampled \(3\)'): - sampling_ops.rank_sampled_softmax_loss( - weights=self._weights(), - biases=self._biases(), - labels=self._labels(), - inputs=self._inputs(), - num_sampled=3, - num_resampled=3, - num_classes=self._num_classes, - num_true=self._num_true, - sampled_values=None, - resampling_temperature=1., - remove_accidental_hits=True, - partition_strategy='div') - - def testInvalidNumSampled1(self): - with ops.Graph().as_default(): - with self.assertRaisesRegexp( - ValueError, - r'num_resampled \(3\) must be less than num_sampled \(2\)'): - sampling_ops.rank_sampled_softmax_loss( - weights=self._weights(), - biases=self._biases(), - labels=self._labels(), - inputs=self._inputs(), - num_sampled=2, - num_resampled=3, - num_classes=self._num_classes, - num_true=self._num_true, - sampled_values=None, - resampling_temperature=1., - remove_accidental_hits=True, - partition_strategy='div') - - def testMissingPartitionStrategy(self): - with ops.Graph().as_default(): - with self.assertRaisesRegexp(ValueError, - r'unsupported partition_strategy \(None\)'): - sampling_ops.rank_sampled_softmax_loss( - weights=self._weights(), - biases=self._biases(), - labels=self._labels(), - inputs=self._inputs(), - num_sampled=2, - num_resampled=1, - num_classes=self._num_classes, - num_true=self._num_true, - sampled_values=None, - resampling_temperature=1., - remove_accidental_hits=True, - partition_strategy=None) - - def _testCompareWithNN(self, weights, biases, partition_strategy): - with ops.Graph().as_default(): - loss = sampling_ops.rank_sampled_softmax_loss( - weights=weights(), - biases=biases(), - labels=self._labels(), - inputs=self._inputs(), - num_sampled=self._num_sampled, - num_resampled=self._num_resampled, - num_classes=self._num_classes, - num_true=self._num_true, - sampled_values=self._sampled_values, - resampling_temperature=1., - remove_accidental_hits=self._remove_accidental_hits, - partition_strategy=partition_strategy) - loss_nn = nn.sampled_softmax_loss( - weights=weights(), - biases=biases(), - labels=self._labels(), - inputs=self._inputs(), - num_sampled=self._num_resampled, - num_classes=self._num_classes, - num_true=self._num_true, - sampled_values=self._resampled_values, - remove_accidental_hits=self._remove_accidental_hits, - partition_strategy=partition_strategy) - with self.cached_session() as sess: - loss_val = sess.run(loss) - loss_nn_val = sess.run(loss_nn) - - self.assertAllClose(loss_val, loss_nn_val) - - def testCompareWithNNUnsharded(self): - self._testCompareWithNN(self._weights, self._biases, 'div') - - def testCompareWithNNShardWeightsDiv(self): - self._testCompareWithNN(self._div_sharded_weights, self._biases, 'div') - - def testCompareWithNNShardWeightsAndBiasesDiv(self): - self._testCompareWithNN(self._div_sharded_weights, self._div_sharded_biases, - 'div') - - def testCompareWithNNShardWeightsMod(self): - self._testCompareWithNN(self._mod_sharded_weights, self._biases, 'mod') - - def testCompareWithNNShardWeightsAndBiasesMod(self): - self._testCompareWithNN(self._mod_sharded_weights, self._mod_sharded_biases, - 'mod') - - def _testCompareWithNNTemperature(self, temperature, resampled): - weights = [[1., 2.], [3., 4.]] # two sampled classes - inputs = [[6., -5. / 2.], [-11., 21. / 2.]] - # Let w0, w1 = weights of sampled classes (biases set to 0 for simplicity) - # Let x0, x1 = inputs - # logits: - # w0.x0 = 1 - # w0.x1 = 10 - # w1.x0 = 8 - # w1.x1 = 9 - # Resampling 1 class with temperature = t will pick the larger of: - # exp(1/t) + exp(10/t) ==> w0, for values of t < 2.12 - # exp(8/t) + exp(9/t) ==> w1, for values of t > 2.13 - num_sampled = 2 - num_resampled = 1 - num_classes = 2 - num_true = 1 - sampled_values = [0, 1], [[1.], [1.]], [1., 1.] - resampled_values = [resampled], [[1.], [1.]], [1.] - remove_accidental_hits = False - with ops.Graph().as_default(): - weights = constant_op.constant(weights) - biases = constant_op.constant([0., 0.]) - labels = constant_op.constant([[0], [1]], dtype=dtypes.int64) - inputs = constant_op.constant(inputs) - loss = sampling_ops.rank_sampled_softmax_loss( - weights=weights, - biases=biases, - labels=labels, - inputs=inputs, - num_sampled=num_sampled, - num_resampled=num_resampled, - num_classes=num_classes, - num_true=num_true, - sampled_values=sampled_values, - resampling_temperature=constant_op.constant(temperature), - remove_accidental_hits=remove_accidental_hits, - partition_strategy='div') - loss_nn = nn.sampled_softmax_loss( - weights=weights, - biases=biases, - labels=labels, - inputs=inputs, - num_sampled=num_resampled, - num_classes=num_classes, - num_true=num_true, - sampled_values=resampled_values, - remove_accidental_hits=remove_accidental_hits, - partition_strategy='div') - with self.cached_session() as sess: - loss_val = sess.run(loss) - loss_nn_val = sess.run(loss_nn) - - self.assertAllClose(loss_val, loss_nn_val) - - def testCompareWithNNTemperatureLo1(self): - self._testCompareWithNNTemperature(1., 0) - - def testCompareWithNNTemperatureLo2(self): - self._testCompareWithNNTemperature(2.12, 0) - - def testCompareWithNNTemperatureHi1(self): - self._testCompareWithNNTemperature(2.13, 1) - - def testCompareWithNNTemperatureHi2(self): - self._testCompareWithNNTemperature(3., 1) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/nn/python/ops/scaled_softplus.py b/tensorflow/contrib/nn/python/ops/scaled_softplus.py deleted file mode 100644 index 7184ef2b66e..00000000000 --- a/tensorflow/contrib/nn/python/ops/scaled_softplus.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Support for scaled softplus, a smoothed version of ReLU.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import function -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn - - -def _reduce_and_reshape_grad(g, t): - """Returns the gradient, sum-reduced and reshaped to `t`'s shape.""" - shape = array_ops.shape(t) - g_shape = array_ops.shape(g) - bcast_dims, _ = gen_array_ops.broadcast_gradient_args(shape, g_shape) - return array_ops.reshape(math_ops.reduce_sum(g, bcast_dims), shape) - - -def scaled_softplus(x, alpha, clip=None, name=None): - """Returns `y = alpha * ln(1 + exp(x / alpha))` or `min(y, clip)`. - - This can be seen as a softplus applied to the scaled input, with the output - appropriately scaled. As `alpha` tends to 0, `scaled_softplus(x, alpha)` tends - to `relu(x)`. The clipping is optional. As alpha->0, scaled_softplus(x, alpha) - tends to relu(x), and scaled_softplus(x, alpha, clip=6) tends to relu6(x). - - Note: the gradient for this operation is defined to depend on the backprop - inputs as well as the outputs of this operation. - - Args: - x: A `Tensor` of inputs. - alpha: A `Tensor`, indicating the amount of smoothness. The caller - must ensure that `alpha > 0`. - clip: (optional) A `Tensor`, the upper bound to clip the values. - name: A name for the scope of the operations (optional). - - Returns: - A tensor of the size and type determined by broadcasting of the inputs. - - """ - clipping = clip is not None - with ops.name_scope(name, 'scaled_softplus', - [x, alpha] + ([clip] if clipping else [])): - x = ops.convert_to_tensor(x, name='x') - dtype = x.dtype - alpha = ops.convert_to_tensor(alpha, dtype=dtype, name='alpha') - # Compute the forward value. - y = alpha * nn.softplus(x / alpha) - if clipping: - clip = ops.convert_to_tensor(clip, dtype=dtype, name='clip') - y = math_ops.minimum(y, clip) - - def _grad(op, g): - """Backprop for scaled softplus, with optional clipping.""" - y, x, alpha = op.inputs[:3] - # Prevent the memory-expensive computations from happening before g is - # available. - with ops.control_dependencies([g]): - y = array_ops.identity(y) - clip_grad = [] - if clipping: - clip = op.inputs[3] - unclipped = math_ops.cast(y < clip, g.dtype) - clip_grad = [_reduce_and_reshape_grad(g * (1. - unclipped), clip)] - g *= unclipped - y /= alpha - emy = math_ops.exp(-y) - dy_dx = 1. - emy - # The eps below avoids log(0). Note that t*log(t) -> 0 as t->0. - eps = 1e-8 - dy_dalpha = y * emy - dy_dx * math_ops.log(dy_dx + eps) - # Backprop to the actual inputs, but not to the output. - return [None, - _reduce_and_reshape_grad(g * dy_dx, x), - _reduce_and_reshape_grad(g * dy_dalpha, alpha)] + clip_grad - - if clipping: - @function.Defun(dtype, dtype, dtype, dtype, - func_name='ScaledSoftplusHelper_clip_%s' % dtype.name, - shape_func=lambda op: [op.inputs[0].shape], - python_grad_func=_grad) - def _forward_helper_clip(y, x, alpha, clip): - del x, alpha, clip # Unused. - return y - return _forward_helper_clip(y, x, alpha, clip) - # No clipping. - @function.Defun(dtype, dtype, dtype, - func_name='ScaledSoftplusHelper_%s' % dtype.name, - shape_func=lambda op: [op.inputs[0].shape], - python_grad_func=_grad) - def _forward_helper(y, x, alpha): - del x, alpha # Unused. - return y - return _forward_helper(y, x, alpha) - diff --git a/tensorflow/contrib/nn/python/ops/scaled_softplus_test.py b/tensorflow/contrib/nn/python/ops/scaled_softplus_test.py deleted file mode 100644 index b978343c6a7..00000000000 --- a/tensorflow/contrib/nn/python/ops/scaled_softplus_test.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for scaled_softplus.py.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.nn.python.ops.scaled_softplus import scaled_softplus -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import gradient_checker -from tensorflow.python.platform import test - - -class ScaledSoftplusTest(test.TestCase): - - def test(self): - np.random.seed(1) # Make it reproducible. - x = np.random.randn(3, 4).astype(np.float32) - x64 = np.random.randn(3, 4).astype(np.float64) - alpha = np.random.rand() + 0.01 - clip = np.float32(0.1) - y = np.minimum(alpha * np.log(1. + np.exp(x / alpha)), clip) - y64 = alpha * np.log(1. + np.exp(x64 / alpha)) - with self.test_session(use_gpu=True) as sess: - z = scaled_softplus(constant_op.constant(x), alpha, clip) - z64 = scaled_softplus(constant_op.constant(x64), alpha) - z, z64 = sess.run([z, z64]) - eps = 1e-6 - self.assertAllClose(y, z, eps) - self.assertAllClose(y64, z64, eps) - - def testGradient(self): - np.random.seed(1) # Make it reproducible. - x_shape = [5, 10] - x_np = np.random.randn(*x_shape).astype(np.float32) - alpha_np = np.float32(np.random.rand(1, x_shape[1]) + 0.01) - clip_np = np.float32(np.random.rand(x_shape[0], 1) * 5.) - with self.test_session(use_gpu=True): - x_tf = constant_op.constant(x_np) - alpha_tf = constant_op.constant(alpha_np) - clip_tf = constant_op.constant(clip_np) - y_tf = scaled_softplus(x_tf, alpha_tf) - z_tf = scaled_softplus(x_tf, alpha_tf, clip_tf * 0.1) - err = gradient_checker.compute_gradient_error([x_tf, alpha_tf], - [x_shape, alpha_np.shape], - y_tf, x_shape, - [x_np, alpha_np], - delta=0.002) - err_clip = gradient_checker.compute_gradient_error( - [x_tf, alpha_tf, clip_tf], - [x_shape, alpha_np.shape, clip_np.shape], - z_tf, x_shape, - [x_np, alpha_np, clip_np], - delta=0.002) - eps = 2e-4 - self.assertLess(err, eps) - self.assertLess(err_clip, eps) - - -if __name__ == '__main__': - test.main() - - diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD deleted file mode 100644 index 34c1ce22886..00000000000 --- a/tensorflow/contrib/opt/BUILD +++ /dev/null @@ -1,477 +0,0 @@ -# Description: -# Optimization routines. - -load("//tensorflow:tensorflow.bzl", "py_test", "tf_py_test") -load("//tensorflow:tensorflow.bzl", "cuda_py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "opt_py", - srcs = [ - "__init__.py", - "python/training/adam_gs_optimizer.py", - "python/training/adamax.py", - "python/training/addsign.py", - "python/training/agn_optimizer.py", - "python/training/drop_stale_gradient_optimizer.py", - "python/training/elastic_average_optimizer.py", - "python/training/external_optimizer.py", - "python/training/ggt.py", - "python/training/lars_optimizer.py", - "python/training/lazy_adam_gs_optimizer.py", - "python/training/lazy_adam_optimizer.py", - "python/training/matrix_functions.py", - "python/training/model_average_optimizer.py", - "python/training/moving_average_optimizer.py", - "python/training/multitask_optimizer_wrapper.py", - "python/training/nadam_optimizer.py", - "python/training/powersign.py", - "python/training/reg_adagrad_optimizer.py", - "python/training/shampoo.py", - "python/training/sign_decay.py", - "python/training/variable_clipping_optimizer.py", - "python/training/weight_decay_optimizers.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/optimizer_v2:optimizer_v2_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:clip_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_ops", - "//tensorflow/python:gradients", - "//tensorflow/python:init_ops", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:state_ops", - "//tensorflow/python:summary", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/eager:context", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_test( - name = "adam_gs_optimizer_test", - srcs = ["python/training/adam_gs_optimizer_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) - -py_test( - name = "adamax_test", - srcs = ["python/training/adamax_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) - -py_test( - name = "external_optimizer_test", - srcs = ["python/training/external_optimizer_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no-internal-py3", - ], - deps = [ - ":opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:extra_py_tests_deps", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "moving_average_optimizer_test", - srcs = ["python/training/moving_average_optimizer_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "notsan", # b/31055119 - ], - deps = [ - ":opt_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "@six_archive//:six", - ], -) - -tf_py_test( - name = "variable_clipping_optimizer_test", - srcs = ["python/training/variable_clipping_optimizer_test.py"], - additional_deps = [ - ":opt_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], - grpc_enabled = True, - tags = [ - "manual", # Flaky: b/29892493 - "notap", # data race due to b/62910646 - ], -) - -py_test( - name = "multitask_optimizer_wrapper_test", - srcs = ["python/training/multitask_optimizer_wrapper_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":opt_py", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_test( - name = "lazy_adam_gs_optimizer_test", - srcs = ["python/training/lazy_adam_gs_optimizer_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - "@absl_py//absl/testing:parameterized", - ], -) - -py_test( - name = "lazy_adam_optimizer_test", - srcs = ["python/training/lazy_adam_optimizer_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - "@absl_py//absl/testing:parameterized", - ], -) - -cuda_py_test( - name = "reg_adagrad_optimizer_test", - srcs = ["python/training/reg_adagrad_optimizer_test.py"], - additional_deps = [ - ":opt_py", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "nadam_optimizer_test", - srcs = ["python/training/nadam_optimizer_test.py"], - deps = [ - ":opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:session", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "weight_decay_optimizers_test", - srcs = ["python/training/weight_decay_optimizers_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:session", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -tf_py_test( - name = "drop_stale_gradient_optimizer_test", - srcs = ["python/training/drop_stale_gradient_optimizer_test.py"], - additional_deps = [ - ":opt_py", - "//third_party/py/numpy", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], - grpc_enabled = True, - tags = [ - "no_oss", # Flaky due to port collisions - ], -) - -tf_py_test( - name = "agn_optimizer_test", - srcs = ["python/training/agn_optimizer_test.py"], - additional_deps = [ - ":opt_py", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:array_ops", - "//tensorflow/python:variables", - "//tensorflow/python:framework", - "//tensorflow/python:platform", - "//tensorflow/python:training", - "//tensorflow/python:ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//third_party/py/numpy", - ], - tags = [ - "notap", # this test launches a local server - ], -) - -tf_py_test( - name = "elastic_average_optimizer_test", - srcs = ["python/training/elastic_average_optimizer_test.py"], - additional_deps = [ - ":opt_py", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:array_ops", - "//tensorflow/python:variables", - "//tensorflow/python:framework", - "//tensorflow/python:platform", - "//tensorflow/python:training", - "//tensorflow/python:ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//third_party/py/numpy", - ], - tags = [ - "oss_serial", - ], -) - -tf_py_test( - name = "model_average_optimizer_test", - srcs = ["python/training/model_average_optimizer_test.py"], - additional_deps = [ - ":opt_py", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:array_ops", - "//tensorflow/python:variables", - "//tensorflow/python:framework", - "//tensorflow/python:platform", - "//tensorflow/python:training", - "//tensorflow/python:ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//third_party/py/numpy", - ], - tags = [ - "notap", # This test launches local server. - ], -) - -py_test( - name = "sign_decay_test", - srcs = ["python/training/sign_decay_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":opt_py", - "//tensorflow/python:client_testlib", - ], -) - -py_test( - name = "addsign_test", - srcs = ["python/training/addsign_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:session", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "powersign_test", - srcs = ["python/training/powersign_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:session", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "ggt_test", - srcs = ["python/training/ggt_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "shampoo_test", - size = "medium", - srcs = ["python/training/shampoo_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - deps = [ - ":opt_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - "@absl_py//absl/testing:parameterized", - ], -) - -py_test( - name = "lars_optimizer_test", - srcs = ["python/training/lars_optimizer_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":opt_py", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:variables", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_test( - name = "matrix_functions_test", - srcs = ["python/training/matrix_functions_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":opt_py", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:variables", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) diff --git a/tensorflow/contrib/opt/README.md b/tensorflow/contrib/opt/README.md deleted file mode 100644 index f93aef94dd1..00000000000 --- a/tensorflow/contrib/opt/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Optimization package - -Contains the following experimental optimization functionality: - - * Support for controlling a TensorFlow session using external optimization - algorithms. - * L2-norm clipping of weights - -Maintainer: joshburkart diff --git a/tensorflow/contrib/opt/__init__.py b/tensorflow/contrib/opt/__init__.py deleted file mode 100644 index e8fc52342ce..00000000000 --- a/tensorflow/contrib/opt/__init__.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A module containing optimization routines.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.opt.python.training.adam_gs_optimizer import * -from tensorflow.contrib.opt.python.training.adamax import * -from tensorflow.contrib.opt.python.training.addsign import * -from tensorflow.contrib.opt.python.training.agn_optimizer import * -from tensorflow.contrib.opt.python.training.drop_stale_gradient_optimizer import * -from tensorflow.contrib.opt.python.training.elastic_average_optimizer import * -from tensorflow.contrib.opt.python.training.external_optimizer import * -from tensorflow.contrib.opt.python.training.lars_optimizer import * -from tensorflow.contrib.opt.python.training.ggt import * -from tensorflow.contrib.opt.python.training.lazy_adam_optimizer import * -from tensorflow.contrib.opt.python.training.lazy_adam_gs_optimizer import * -from tensorflow.contrib.opt.python.training.model_average_optimizer import * -from tensorflow.contrib.opt.python.training.moving_average_optimizer import * -from tensorflow.contrib.opt.python.training.multitask_optimizer_wrapper import * -from tensorflow.contrib.opt.python.training.nadam_optimizer import * -from tensorflow.contrib.opt.python.training.reg_adagrad_optimizer import * -from tensorflow.contrib.opt.python.training.shampoo import * -from tensorflow.contrib.opt.python.training.weight_decay_optimizers import * -from tensorflow.contrib.opt.python.training.powersign import * -from tensorflow.contrib.opt.python.training.variable_clipping_optimizer import * -# pylint: enable=wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented - - -_allowed_symbols = [ - 'AdaMaxOptimizer', - 'AdamGSOptimizer', - 'PowerSignOptimizer', - 'AddSignOptimizer', - 'DelayCompensatedGradientDescentOptimizer', - 'DropStaleGradientOptimizer', - 'ExternalOptimizerInterface', - 'LARSOptimizer', - 'LazyAdamGSOptimizer', - 'LazyAdamOptimizer', - 'NadamOptimizer', - 'MovingAverageOptimizer', - 'MomentumWOptimizer', - 'AdamWOptimizer', - 'DecoupledWeightDecayExtension', - 'extend_with_decoupled_weight_decay', - 'ScipyOptimizerInterface', - 'VariableClippingOptimizer', - 'MultitaskOptimizerWrapper', - 'clip_gradients_by_global_norm', - 'AGNOptimizer', - 'AGNCustomGetter', - 'ElasticAverageOptimizer', - 'ElasticAverageCustomGetter', - 'ModelAverageOptimizer', - 'ModelAverageCustomGetter', - 'GGTOptimizer', - 'ShampooOptimizer', - 'RegAdagradOptimizer', -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py b/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py deleted file mode 100644 index 0b149ed1753..00000000000 --- a/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Adam rewrite to use global step for computing beta1 & beta2 accumulation.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.training import optimizer -from tensorflow.python.training import training_ops -from tensorflow.python.util.tf_export import tf_export - - -@tf_export("train.AdamOptimizer") -class AdamGSOptimizer(optimizer.Optimizer): - """Optimizer that implements the Adam algorithm. - - See [Kingma et al., 2014](http://arxiv.org/abs/1412.6980) - ([pdf](http://arxiv.org/pdf/1412.6980.pdf)). - """ - - def __init__(self, - global_step=0, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - use_locking=False, - name="Adam"): - r"""Construct a new Adam optimizer. - - Branched from tf.train.AdamOptimizer. The only difference is to pass - global step for computing beta1 and beta2 accumulators, instead of having - optimizer keep its own independent beta1 and beta2 accumulators as non-slot - variables. - - Initialization: - - $$m_0 := 0 \text{(Initialize initial 1st moment vector)}$$ - $$v_0 := 0 \text{(Initialize initial 2nd moment vector)}$$ - $$t := 0 \text{(Initialize timestep)}$$ - - The update rule for `variable` with gradient `g` uses an optimization - described at the end of section2 of the paper: - - $$t := t + 1$$ - $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$ - - $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$ - $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ - $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ - - The default value of 1e-8 for epsilon might not be a good default in - general. For example, when training an Inception network on ImageNet a - current good choice is 1.0 or 0.1. Note that since AdamOptimizer uses the - formulation just before Section 2.1 of the Kingma and Ba paper rather than - the formulation in Algorithm 1, the "epsilon" referred to here is "epsilon - hat" in the paper. - - The sparse implementation of this algorithm (used when the gradient is an - IndexedSlices object, typically because of `tf.gather` or an embedding - lookup in the forward pass) does apply momentum to variable slices even if - they were not used in the forward pass (meaning they have a gradient equal - to zero). Momentum decay (beta1) is also applied to the entire momentum - accumulator. This means that the sparse behavior is equivalent to the dense - behavior (in contrast to some momentum implementations which ignore momentum - unless a variable slice was actually used). - - Args: - global_step: tensorflow variable indicating the step. - learning_rate: A Tensor or a floating point value. The learning rate. - beta1: A float value or a constant float tensor. The exponential decay - rate for the 1st moment estimates. - beta2: A float value or a constant float tensor. The exponential decay - rate for the 2nd moment estimates. - epsilon: A small constant for numerical stability. This epsilon is - "epsilon hat" in the Kingma and Ba paper (in the formula just before - Section 2.1), not the epsilon in Algorithm 1 of the paper. - use_locking: If True use locks for update operations. - name: Optional name for the operations created when applying gradients. - Defaults to "Adam". @compatibility(eager) When eager execution is - enabled, `learning_rate`, `beta1`, `beta2`, and `epsilon` can each be a - callable that takes no arguments and returns the actual value to use. - This can be useful for changing these values across different - invocations of optimizer functions. @end_compatibility - """ - super(AdamGSOptimizer, self).__init__(use_locking, name) - self._lr = learning_rate - self._beta1 = beta1 - self._beta2 = beta2 - self._epsilon = epsilon - self._global_step = global_step - self._global_step_on_worker = None - - # Tensor versions of the constructor arguments, created in _prepare(). - self._lr_t = None - self._beta1_t = None - self._beta2_t = None - self._epsilon_t = None - - def _get_beta_accumulators(self): - return (math_ops.pow(self._beta1_t, self._global_step_on_worker), - math_ops.pow(self._beta2_t, self._global_step_on_worker)) - - def _create_slots(self, var_list): - # Create slots for the first and second moments. - for v in var_list: - self._zeros_slot(v, "m", self._name) - self._zeros_slot(v, "v", self._name) - - def _prepare(self): - lr = self._call_if_callable(self._lr) - beta1 = self._call_if_callable(self._beta1) - beta2 = self._call_if_callable(self._beta2) - epsilon = self._call_if_callable(self._epsilon) - - self._lr_t = ops.convert_to_tensor(lr, name="learning_rate") - self._beta1_t = ops.convert_to_tensor(beta1, name="beta1") - self._beta2_t = ops.convert_to_tensor(beta2, name="beta2") - self._epsilon_t = ops.convert_to_tensor(epsilon, name="epsilon") - - # Performance optimization so that worker creates a copy of the global step - # to avoid overloading the parameter server holding the global step. - self._global_step_on_worker = math_ops.cast( - array_ops.identity(self._global_step) + 1, dtypes.float32) - - def _apply_dense(self, grad, var): - m = self.get_slot(var, "m") - v = self.get_slot(var, "v") - beta1_power, beta2_power = self._get_beta_accumulators() - return training_ops.apply_adam( - var, - m, - v, - math_ops.cast(beta1_power, var.dtype.base_dtype), - math_ops.cast(beta2_power, var.dtype.base_dtype), - math_ops.cast(self._lr_t, var.dtype.base_dtype), - math_ops.cast(self._beta1_t, var.dtype.base_dtype), - math_ops.cast(self._beta2_t, var.dtype.base_dtype), - math_ops.cast(self._epsilon_t, var.dtype.base_dtype), - grad, - use_locking=self._use_locking).op - - def _resource_apply_dense(self, grad, var): - m = self.get_slot(var, "m") - v = self.get_slot(var, "v") - beta1_power, beta2_power = self._get_beta_accumulators() - return training_ops.resource_apply_adam( - var.handle, - m.handle, - v.handle, - math_ops.cast(beta1_power, grad.dtype.base_dtype), - math_ops.cast(beta2_power, grad.dtype.base_dtype), - math_ops.cast(self._lr_t, grad.dtype.base_dtype), - math_ops.cast(self._beta1_t, grad.dtype.base_dtype), - math_ops.cast(self._beta2_t, grad.dtype.base_dtype), - math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), - grad, - use_locking=self._use_locking) - - def _apply_sparse_shared(self, grad, var, indices, scatter_add): - beta1_power, beta2_power = self._get_beta_accumulators() - beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) - beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype) - lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) - beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype) - beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype) - epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype) - lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power)) - # m_t = beta1 * m + (1 - beta1) * g_t - m = self.get_slot(var, "m") - m_scaled_g_values = grad * (1 - beta1_t) - m_t = state_ops.assign(m, m * beta1_t, use_locking=self._use_locking) - with ops.control_dependencies([m_t]): - m_t = scatter_add(m, indices, m_scaled_g_values) - # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) - v = self.get_slot(var, "v") - v_scaled_g_values = (grad * grad) * (1 - beta2_t) - v_t = state_ops.assign(v, v * beta2_t, use_locking=self._use_locking) - with ops.control_dependencies([v_t]): - v_t = scatter_add(v, indices, v_scaled_g_values) - v_sqrt = math_ops.sqrt(v_t) - var_update = state_ops.assign_sub( - var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking) - return control_flow_ops.group(*[var_update, m_t, v_t]) - - def _apply_sparse(self, grad, var): - return self._apply_sparse_shared( - grad.values, - var, - grad.indices, - lambda x, i, v: state_ops.scatter_add( # pylint: disable=g-long-lambda - x, - i, - v, - use_locking=self._use_locking)) - - def _resource_scatter_add(self, x, i, v): - with ops.control_dependencies( - [resource_variable_ops.resource_scatter_add(x.handle, i, v)]): - return x.value() - - def _resource_apply_sparse(self, grad, var, indices): - return self._apply_sparse_shared(grad, var, indices, - self._resource_scatter_add) diff --git a/tensorflow/contrib/opt/python/training/adam_gs_optimizer_test.py b/tensorflow/contrib/opt/python/training/adam_gs_optimizer_test.py deleted file mode 100644 index c68c965aef3..00000000000 --- a/tensorflow/contrib/opt/python/training/adam_gs_optimizer_test.py +++ /dev/null @@ -1,382 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for AdamGS.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.opt.python.training import adam_gs_optimizer -from tensorflow.python.client import session -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def adam_update_numpy(param, - g_t, - t, - m, - v, - alpha=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8): - alpha_t = alpha * np.sqrt(1 - beta2**t) / (1 - beta1**t) - - m_t = beta1 * m + (1 - beta1) * g_t - v_t = beta2 * v + (1 - beta2) * g_t * g_t - - param_t = param - alpha_t * m_t / (np.sqrt(v_t) + epsilon) - return param_t, m_t, v_t - - -class AdamGSOptimizerTest(test.TestCase): - - def doTestSparse(self, use_resource=False): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - global_step = resource_variable_ops.ResourceVariable( - array_ops.zeros([], dtypes.int64)) - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - else: - global_step = variables.Variable(array_ops.zeros([], dtypes.int64)) - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0_np_indices = np.array([0, 1], dtype=np.int32) - grads0 = ops.IndexedSlices( - constant_op.constant(grads0_np), - constant_op.constant(grads0_np_indices), constant_op.constant([2])) - grads1_np_indices = np.array([0, 1], dtype=np.int32) - grads1 = ops.IndexedSlices( - constant_op.constant(grads1_np), - constant_op.constant(grads1_np_indices), constant_op.constant([2])) - opt = adam_gs_optimizer.AdamGSOptimizer(global_step=global_step) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Run 3 steps of Adam - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power)) - self.assertAllCloseAccordingToType(0.999**t, - self.evaluate(beta2_power)) - update.run() - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - - def testSparse(self): - self.doTestSparse(use_resource=False) - - def testResourceSparse(self): - self.doTestSparse(use_resource=True) - - def testSparseDevicePlacement(self): - for index_dtype in [dtypes.int32, dtypes.int64]: - with self.cached_session(force_gpu=test.is_gpu_available()): - # If a GPU is available, tests that all optimizer ops can be placed on - # it (i.e. they have GPU kernels). - var = variables.Variable([[1.0], [2.0]]) - indices = constant_op.constant([0, 1], dtype=index_dtype) - gathered_sum = math_ops.reduce_sum(array_ops.gather(var, indices)) - optimizer = adam_gs_optimizer.AdamGSOptimizer(3.0) - minimize_op = optimizer.minimize(gathered_sum) - variables.global_variables_initializer().run() - minimize_op.run() - - def testSparseRepeatedIndices(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - repeated_index_global_step = variables.Variable( - array_ops.zeros([], dtypes.int64)) - aggregated_global_step = variables.Variable( - array_ops.zeros([], dtypes.int64)) - repeated_index_update_var = variables.Variable( - [[1.0], [2.0]], dtype=dtype) - aggregated_update_var = variables.Variable( - [[1.0], [2.0]], dtype=dtype) - grad_repeated_index = ops.IndexedSlices( - constant_op.constant( - [0.1, 0.1], shape=[2, 1], dtype=dtype), - constant_op.constant([1, 1]), - constant_op.constant([2, 1])) - grad_aggregated = ops.IndexedSlices( - constant_op.constant( - [0.2], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), - constant_op.constant([2, 1])) - repeated_update = adam_gs_optimizer.AdamGSOptimizer( - global_step=repeated_index_global_step).apply_gradients( - [(grad_repeated_index, repeated_index_update_var)], - global_step=repeated_index_global_step) - aggregated_update = adam_gs_optimizer.AdamGSOptimizer( - global_step=aggregated_global_step).apply_gradients( - [(grad_aggregated, aggregated_update_var)], - global_step=aggregated_global_step) - variables.global_variables_initializer().run() - self.assertAllClose(aggregated_update_var.eval(), - self.evaluate(repeated_index_update_var)) - for _ in range(3): - repeated_update.run() - aggregated_update.run() - self.assertAllClose(aggregated_update_var.eval(), - self.evaluate(repeated_index_update_var)) - - def doTestBasic(self, use_resource=False, use_callable_params=False): - for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - with self.session(graph=ops.Graph()): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - global_step = resource_variable_ops.ResourceVariable( - array_ops.zeros([], dtypes.int64), name="global_step_%d" % i) - var0 = resource_variable_ops.ResourceVariable( - var0_np, name="var0_%d" % i) - var1 = resource_variable_ops.ResourceVariable( - var1_np, name="var1_%d" % i) - else: - global_step = variables.Variable(array_ops.zeros([], dtypes.int64)) - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - learning_rate = lambda: 0.001 - beta1 = lambda: 0.9 - beta2 = lambda: 0.999 - epsilon = lambda: 1e-8 - if not use_callable_params: - learning_rate = learning_rate() - beta1 = beta1() - beta2 = beta2() - epsilon = epsilon() - - opt = adam_gs_optimizer.AdamGSOptimizer(global_step=global_step, - learning_rate=learning_rate) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - opt_variables = opt.variables() - beta1_power, beta2_power = opt._get_beta_accumulators() - self.assertTrue(beta1_power is not None) - self.assertTrue(beta2_power is not None) - self.assertNotIn(beta1_power, opt_variables) - self.assertNotIn(beta2_power, opt_variables) - - if not context.executing_eagerly(): - with ops.Graph().as_default(): - # Shouldn't return non-slot variables from other graphs. - self.assertEqual(0, len(opt.variables())) - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - # Run 3 steps of Adam - for t in range(1, 4): - if not context.executing_eagerly(): - self.evaluate(update) - self.assertAllCloseAccordingToType( - 0.9**(t + 1), self.evaluate(beta1_power)) - self.assertAllCloseAccordingToType( - 0.999**(t + 1), self.evaluate(beta2_power)) - else: - if t > 1: - opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - beta1_power, beta2_power = opt._get_beta_accumulators() - self.assertAllCloseAccordingToType( - 0.9**t, self.evaluate(beta1_power)) - self.assertAllCloseAccordingToType( - 0.999**t, self.evaluate(beta2_power)) - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - if use_resource: - self.assertEqual("var0_%d/Adam:0" % (i,), - opt.get_slot(var=var0, name="m").name) - - def testBasic(self): - with self.cached_session(): - self.doTestBasic(use_resource=False) - - @test_util.run_in_graph_and_eager_modes(reset_test=True) - def testResourceBasic(self): - self.doTestBasic(use_resource=True) - - def testBasicCallableParams(self): - with context.eager_mode(): - self.doTestBasic(use_resource=True, use_callable_params=True) - - def testTensorLearningRate(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - global_step = variables.Variable(array_ops.zeros([], dtypes.int64)) - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - opt = adam_gs_optimizer.AdamGSOptimizer( - global_step=global_step, learning_rate=constant_op.constant(0.001)) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Run 3 steps of Adam - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power)) - self.assertAllCloseAccordingToType(0.999**t, - self.evaluate(beta2_power)) - update.run() - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - - def testSharing(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - global_step = variables.Variable(array_ops.zeros([], dtypes.int64)) - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - opt = adam_gs_optimizer.AdamGSOptimizer(global_step=global_step) - update1 = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - variables.global_variables_initializer().run() - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - # Run 3 steps of intertwined Adam1 and Adam2. - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power)) - self.assertAllCloseAccordingToType(0.999**t, - self.evaluate(beta2_power)) - if t % 2 == 0: - update1.run() - else: - update2.run() - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - - def testTwoSessions(self): - optimizer = adam_gs_optimizer.AdamGSOptimizer() - - with context.eager_mode(): - var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") - grads0 = constant_op.constant(np.array([0.1, 0.1])) - optimizer.apply_gradients([(grads0, var0)]) - - g = ops.Graph() - with g.as_default(): - with session.Session(): - var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") - grads0 = constant_op.constant(np.array([0.1, 0.1])) - optimizer.apply_gradients([(grads0, var0)]) - - gg = ops.Graph() - with gg.as_default(): - with session.Session(): - var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") - grads0 = constant_op.constant(np.array([0.1, 0.1])) - - # If the optimizer saves any state not keyed by graph the following line - # fails. - optimizer.apply_gradients([(grads0, var0)]) - - def testSlotsUniqueEager(self): - with context.eager_mode(): - v1 = resource_variable_ops.ResourceVariable(1.) - v2 = resource_variable_ops.ResourceVariable(1.) - opt = adam_gs_optimizer.AdamGSOptimizer(1.) - opt.minimize(lambda: v1 + v2) - # There should be two unique slot variables for v1 and v2 respectively. - self.assertEqual(4, len(set(opt.variables()))) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/opt/python/training/adamax.py b/tensorflow/contrib/opt/python/training/adamax.py deleted file mode 100644 index 686bac0d840..00000000000 --- a/tensorflow/contrib/opt/python/training/adamax.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""AdaMax for TensorFlow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.eager import context -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.training import adam -from tensorflow.python.training import training_ops - - -class AdaMaxOptimizer(adam.AdamOptimizer): - """Optimizer that implements the AdaMax algorithm. - - Adamax is sometimes superior to adam, specially in models with embeddings, - see [Kingma et al., 2014](http://arxiv.org/abs/1412.6980) - ([pdf](http://arxiv.org/pdf/1412.6980.pdf)). - """ - - def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8, - use_locking=False, name="AdaMax"): - """Construct a new AdaMax optimizer. - - Initialization: - - ``` - m_0 <- 0 (Initialize initial 1st moment vector) - v_0 <- 0 (Initialize the exponentially weighted infinity norm) - t <- 0 (Initialize timestep) - ``` - - The update rule for `variable` with gradient `g` uses an optimization - described at the end of section 7.1 of the paper: - - ``` - t <- t + 1 - - m_t <- beta1 * m_{t-1} + (1 - beta1) * g - v_t <- max(beta2 * v_{t-1}, abs(g)) - variable <- variable - learning_rate / (1 - beta1^t) * m_t / (v_t + epsilon) - ``` - - Similar to AdamOptimizer, the epsilon is added for numerical stability - (especially to get rid of division by zero when v_t = 0). - - Contrast to AdamOptimizer, the sparse implementation of this algorithm - (used when the gradient is an IndexedSlices object, typically because of - `tf.gather` or an embedding lookup in the forward pass) only updates - variable slices and corresponding `m_t`, `v_t` terms when that part of - the variable was used in the forward pass. This means that the sparse - behavior is contrast to the dense behavior (similar to some momentum - implementations which ignore momentum unless a variable slice was actually - used). - - Args: - learning_rate: A Tensor or a floating point value. The learning rate. - beta1: A float value or a constant float tensor. - The exponential decay rate for the 1st moment estimates. - beta2: A float value or a constant float tensor. - The exponential decay rate for the exponentially weighted infinity norm. - epsilon: A small constant for numerical stability. - use_locking: If True use locks for update operations. - name: Optional name for the operations created when applying gradients. - Defaults to "AdaMax". - """ - super(AdaMaxOptimizer, self).__init__(learning_rate, beta1, beta2, - epsilon, use_locking, name) - - def _get_beta_accumulators(self): - if context.executing_eagerly(): - graph = None - else: - graph = ops.get_default_graph() - return self._get_non_slot_variable("beta1_power", graph=graph) - - def _create_slots(self, var_list): - # Create the beta1 accumulators on the same device as the first - # variable. Sort the var_list to make sure this device is consistent across - # workers (these need to go on the same PS, otherwise some updates are - # silently ignored). - first_var = min(var_list, key=lambda x: x.name) - self._create_non_slot_variable(initial_value=self._beta1, - name="beta1_power", - colocate_with=first_var) - - # Create slots for the first and second moments. - for v in var_list: - self._zeros_slot(v, "m", self._name) - self._zeros_slot(v, "v", self._name) - - def _apply_dense(self, grad, var): - m = self.get_slot(var, "m") - v = self.get_slot(var, "v") - beta1_power = self._get_beta_accumulators() - return training_ops.apply_ada_max( - var, m, v, - math_ops.cast(beta1_power, var.dtype.base_dtype), - math_ops.cast(self._lr_t, var.dtype.base_dtype), - math_ops.cast(self._beta1_t, var.dtype.base_dtype), - math_ops.cast(self._beta2_t, var.dtype.base_dtype), - math_ops.cast(self._epsilon_t, var.dtype.base_dtype), - grad, use_locking=self._use_locking).op - - def _resource_apply_dense(self, grad, var): - m = self.get_slot(var, "m") - v = self.get_slot(var, "v") - beta1_power = self._get_beta_accumulators() - return training_ops.resource_apply_ada_max( - var.handle, m.handle, v.handle, - math_ops.cast(beta1_power, grad.dtype.base_dtype), - math_ops.cast(self._lr_t, grad.dtype.base_dtype), - math_ops.cast(self._beta1_t, grad.dtype.base_dtype), - math_ops.cast(self._beta2_t, grad.dtype.base_dtype), - math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), - grad, use_locking=self._use_locking) - - def _apply_sparse_shared(self, grad, var, indices, - scatter_add, scatter_update): - beta1_power = self._get_beta_accumulators() - beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) - lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) - beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype) - beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype) - epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype) - # m_t = beta1 * m + (1 - beta1) * g_t - m = self.get_slot(var, "m") - m_slice = array_ops.gather(m, indices) - m_t_slice = m_slice * beta1_t + grad * (1 - beta1_t) - with ops.control_dependencies([m_t_slice]): - m_t = scatter_update(m, indices, m_t_slice) - # u_t = max(beta2 * u, abs(g_t)) - v = self.get_slot(var, "v") - v_slice = array_ops.gather(v, indices) - v_t_slice = math_ops.maximum(v_slice * beta2_t, math_ops.abs(grad)) - with ops.control_dependencies([v_t_slice]): - v_t = scatter_update(v, indices, v_t_slice) - # theta_t = theta - lr / (1 - beta1^t) * m_t / u_t - var_slice = -lr_t / (1 - beta1_power) * (m_t_slice / - (v_t_slice + epsilon_t)) - with ops.control_dependencies([var_slice]): - var_update = scatter_add(var, indices, var_slice) - return control_flow_ops.group(*[var_update, m_t, v_t]) - - def _apply_sparse(self, grad, var): - return self._apply_sparse_shared( - grad.values, var, grad.indices, - lambda x, i, v: state_ops.scatter_add( # pylint: disable=g-long-lambda - x, i, v, use_locking=self._use_locking), - lambda x, i, v: state_ops.scatter_update( # pylint: disable=g-long-lambda - x, i, v, use_locking=self._use_locking)) - - def _resource_scatter_update(self, x, i, v): - with ops.control_dependencies( - [resource_variable_ops.resource_scatter_update( - x.handle, i, v)]): - return x.value() - - def _resource_apply_sparse(self, grad, var, indices): - return self._apply_sparse_shared( - grad, var, indices, - self._resource_scatter_add, self._resource_scatter_update) - - def _finish(self, update_ops, name_scope): - # Update the power accumulators. - with ops.control_dependencies(update_ops): - beta1_power = self._get_beta_accumulators() - with ops.colocate_with(beta1_power): - update_beta1 = beta1_power.assign( - beta1_power * self._beta1_t, use_locking=self._use_locking) - return control_flow_ops.group(*update_ops + [update_beta1], - name=name_scope) diff --git a/tensorflow/contrib/opt/python/training/adamax_test.py b/tensorflow/contrib/opt/python/training/adamax_test.py deleted file mode 100644 index a1e220924f3..00000000000 --- a/tensorflow/contrib/opt/python/training/adamax_test.py +++ /dev/null @@ -1,350 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for AdaMax.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.opt.python.training import adamax -from tensorflow.python.client import session -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def adamax_update_numpy(param, - g_t, - t, - m, - v, - alpha=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8): - m_t = beta1 * m + (1 - beta1) * g_t - v_t = np.maximum(beta2 * v, np.abs(g_t)) - param_t = param - (alpha / (1 - beta1**t)) * (m_t / (v_t + epsilon)) - return param_t, m_t, v_t - - -def adamax_sparse_update_numpy(param, - indices, - g_t, - t, - m, - v, - alpha=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8): - m_t, v_t, param_t = np.copy(m), np.copy(v), np.copy(param) - m_t_slice = beta1 * m[indices] + (1 - beta1) * g_t - v_t_slice = np.maximum(beta2 * v[indices], np.abs(g_t)) - param_t_slice = param[indices] - ((alpha / (1 - beta1**t)) * - (m_t_slice / (v_t_slice + epsilon))) - m_t[indices] = m_t_slice - v_t[indices] = v_t_slice - param_t[indices] = param_t_slice - return param_t, m_t, v_t - - -class AdaMaxOptimizerTest(test.TestCase): - - def doTestSparse(self, use_resource=False): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - # Initialize variables for numpy implementation. - zero_slots = lambda: np.zeros((3), dtype=dtype.as_numpy_dtype) - m0, v0, m1, v1 = zero_slots(), zero_slots(), zero_slots(), zero_slots() - var0_np = np.array([1.0, 2.0, 3.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([4.0, 5.0, 6.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - else: - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0_np_indices = np.array([0, 1], dtype=np.int32) - grads0 = ops.IndexedSlices( - constant_op.constant(grads0_np), - constant_op.constant(grads0_np_indices), constant_op.constant([2])) - grads1_np_indices = np.array([2, 1], dtype=np.int32) - grads1 = ops.IndexedSlices( - constant_op.constant(grads1_np), - constant_op.constant(grads1_np_indices), constant_op.constant([2])) - opt = adamax.AdaMaxOptimizer() - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0, 3.0], var0.eval()) - self.assertAllClose([4.0, 5.0, 6.0], var1.eval()) - - beta1_power = opt._get_beta_accumulators() - - # Run 3 steps of AdaMax - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - update.run() - - var0_np, m0, v0 = adamax_sparse_update_numpy( - var0_np, grads0_np_indices, grads0_np, t, m0, v0) - var1_np, m1, v1 = adamax_sparse_update_numpy( - var1_np, grads1_np_indices, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testSparse(self): - self.doTestSparse(use_resource=False) - - def testResourceSparse(self): - self.doTestSparse(use_resource=True) - - def testSparseDevicePlacement(self): - for index_dtype in [dtypes.int32, dtypes.int64]: - with self.cached_session(force_gpu=test.is_gpu_available()): - # If a GPU is available, tests that all optimizer ops can be placed on - # it (i.e. they have GPU kernels). - var = variables.Variable([[1.0], [2.0]]) - indices = constant_op.constant([0, 1], dtype=index_dtype) - gathered_sum = math_ops.reduce_sum(array_ops.gather(var, indices)) - optimizer = adamax.AdaMaxOptimizer(3.0) - minimize_op = optimizer.minimize(gathered_sum) - variables.global_variables_initializer().run() - minimize_op.run() - - def testSparseRepeatedIndices(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - repeated_index_update_var = variables.Variable( - [[1.0], [2.0]], dtype=dtype) - aggregated_update_var = variables.Variable( - [[1.0], [2.0]], dtype=dtype) - grad_repeated_index = ops.IndexedSlices( - constant_op.constant( - [0.1, 0.1], shape=[2, 1], dtype=dtype), - constant_op.constant([1, 1]), - constant_op.constant([2, 1])) - grad_aggregated = ops.IndexedSlices( - constant_op.constant( - [0.2], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), - constant_op.constant([2, 1])) - repeated_update = adamax.AdaMaxOptimizer().apply_gradients( - [(grad_repeated_index, repeated_index_update_var)]) - aggregated_update = adamax.AdaMaxOptimizer().apply_gradients( - [(grad_aggregated, aggregated_update_var)]) - variables.global_variables_initializer().run() - self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) - for _ in range(3): - repeated_update.run() - aggregated_update.run() - self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) - - def doTestBasic(self, use_resource=False): - for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - with self.session(graph=ops.Graph()): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable( - var0_np, name="var0_%d" % i) - var1 = resource_variable_ops.ResourceVariable( - var1_np, name="var1_%d" % i) - else: - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - opt = adamax.AdaMaxOptimizer() - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - opt_variables = opt.variables() - beta1_power = opt._get_beta_accumulators() - self.assertTrue(beta1_power is not None) - self.assertIn(beta1_power, opt_variables) - - if not context.executing_eagerly(): - with ops.Graph().as_default(): - # Shouldn't return non-slot variables from other graphs. - self.assertEqual(0, len(opt.variables())) - - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - beta1_power = opt._get_beta_accumulators() - - # Run 3 steps of AdaMax - for t in range(1, 4): - if not context.executing_eagerly(): - self.evaluate(update) - elif t > 1: - opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - - self.assertAllCloseAccordingToType(0.9**(t + 1), - self.evaluate(beta1_power)) - - var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0), - rtol=1e-2) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1), - rtol=1e-2) - if use_resource: - self.assertEqual("var0_%d/AdaMax:0" % (i,), - opt.get_slot(var=var0, name="m").name) - - def testBasic(self): - with self.cached_session(): - self.doTestBasic(use_resource=False) - - @test_util.run_in_graph_and_eager_modes(reset_test=True) - def testResourceBasic(self): - self.doTestBasic(use_resource=True) - - def testTensorLearningRate(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - opt = adamax.AdaMaxOptimizer(constant_op.constant(0.001)) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - beta1_power = opt._get_beta_accumulators() - - # Run 3 steps of AdaMax - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - update.run() - - var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testSharing(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - opt = adamax.AdaMaxOptimizer() - update1 = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - beta1_power = opt._get_beta_accumulators() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - # Run 3 steps of intertwined AdaMax1 and AdaMax2. - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - if t % 2 == 0: - update1.run() - else: - update2.run() - - var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testTwoSessions(self): - optimizer = adamax.AdaMaxOptimizer() - g = ops.Graph() - with g.as_default(): - with session.Session(): - var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") - grads0 = constant_op.constant(np.array([0.1, 0.1])) - optimizer.apply_gradients([(grads0, var0)]) - - gg = ops.Graph() - with gg.as_default(): - with session.Session(): - var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") - grads0 = constant_op.constant(np.array([0.1, 0.1])) - - # If the optimizer saves any state not keyed by graph the following line - # fails. - optimizer.apply_gradients([(grads0, var0)]) - - def testSlotsUniqueEager(self): - with context.eager_mode(): - v1 = resource_variable_ops.ResourceVariable(1.) - v2 = resource_variable_ops.ResourceVariable(1.) - opt = adamax.AdaMaxOptimizer(1.) - opt.minimize(lambda: v1 + v2) - # There should be two non-slot variables, and two unique slot variables - # for v1 and v2 respectively. - self.assertEqual(5, len(set(opt.variables()))) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/opt/python/training/addsign.py b/tensorflow/contrib/opt/python/training/addsign.py deleted file mode 100644 index 22da4453e20..00000000000 --- a/tensorflow/contrib/opt/python/training/addsign.py +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implementation of AddSign.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.training import optimizer -from tensorflow.python.training import training_ops - - -class AddSignOptimizer(optimizer.Optimizer): - """Optimizer that implements the AddSign update. - - See [Bello et al., ICML2017], - [Neural Optimizer Search with RL](https://arxiv.org/abs/1709.07417). - """ - - def __init__(self, - learning_rate=0.1, - alpha=1.0, - beta=0.9, - sign_decay_fn=None, - use_locking=False, - name='AddSignOptimizer'): - """Constructs a new AddSignOptimizer object. - - Initialization: - - ``` - m_0 <- 0 (Initialize initial 1st moment vector) - t <- 0 (Initialize timestep) - ``` - - Update: - - ``` - t <- t + 1 - m_t <- beta1 * m_{t-1} + (1 - beta1) * g - sign_decay <- sign_decay_fn(t) - update <- (alpha + sign_decay * sign(g) *sign(m)) * g - variable <- variable - lr_t * update - ``` - - Example for AddSign-ld (AddSign with linear sign decay) - ``` - decay_steps = 1000 - linear_decay_fn = sign_decays.get_linear_decay_fn(decay_steps) - opt = AddSignOptimizer(learning_rate=0.1, sign_decay_fn=linear_decay_fn) - ``` - - Args: - learning_rate: learning_rate used when taking a step. - alpha: alpha used in optimizer. - beta: decay used for computing the moving average m. - sign_decay_fn: decay function applied to the sign(g) sign(m) quantity. - Takes global_step as an argument. See sign_decay.py for some examples. - use_locking: If True, use locks for update operations. - name: Optional name for the operations created when applying gradients. - Defaults to "AddSignOptimizer". - """ - super(AddSignOptimizer, self).__init__(use_locking, name) - self._lr = learning_rate - self._alpha = alpha - self._beta = beta - - self._sign_decay_fn = sign_decay_fn - - # Tensor versions of the constructor arguments, created in _prepare(). - self._lr_t = None - self._alpha_t = None - self._beta_t = None - - def apply_gradients(self, grads_and_vars, global_step=None, name=None): - if self._sign_decay_fn is not None: - self._sign_decay_t = ops.convert_to_tensor( - self._sign_decay_fn(global_step), name='sign_decay') - return super(AddSignOptimizer, self).apply_gradients( - grads_and_vars, global_step=global_step, name=name) - - def _create_slots(self, var_list): - # Create slots for the first moment. - for v in var_list: - self._zeros_slot(v, 'm', self._name) - - def _prepare(self): - self._lr_t = ops.convert_to_tensor(self._lr, name='learning_rate') - self._beta_t = ops.convert_to_tensor(self._beta, name='beta') - self._alpha_t = ops.convert_to_tensor(self._alpha, name='alpha') - if self._sign_decay_fn is None: - self._sign_decay_t = ops.convert_to_tensor(1.0, name='sign_decay') - - def _apply_dense(self, grad, var): - m = self.get_slot(var, 'm') - return training_ops.apply_add_sign( - var, - m, - math_ops.cast(self._lr_t, var.dtype.base_dtype), - math_ops.cast(self._alpha_t, var.dtype.base_dtype), - math_ops.cast(self._sign_decay_t, var.dtype.base_dtype), - math_ops.cast(self._beta_t, var.dtype.base_dtype), - grad, - use_locking=self._use_locking).op - - def _resource_apply_dense(self, grad, var): - m = self.get_slot(var, 'm') - return training_ops.resource_apply_add_sign( - var.handle, - m.handle, - math_ops.cast(self._lr_t, var.dtype.base_dtype), - math_ops.cast(self._alpha_t, var.dtype.base_dtype), - math_ops.cast(self._sign_decay_t, var.dtype.base_dtype), - math_ops.cast(self._beta_t, var.dtype.base_dtype), - grad, - use_locking=self._use_locking) - - def _apply_sparse(self, grad, var): - lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) - alpha_t = math_ops.cast(self._alpha_t, var.dtype.base_dtype) - beta_t = math_ops.cast(self._beta_t, var.dtype.base_dtype) - - m = self.get_slot(var, 'm') - m_t = state_ops.assign( - m, (m * beta_t) + (grad * (1 - beta_t)), use_locking=self._use_locking) - - sign_g = ops.IndexedSlices( - math_ops.sign(grad.values), grad.indices, dense_shape=grad.dense_shape) - sign_gm = ops.IndexedSlices( - array_ops.gather(math_ops.sign(m_t), sign_g.indices) * sign_g.values, - sign_g.indices, - dense_shape=sign_g.dense_shape) - - sign_decayed = math_ops.cast( - self._sign_decay_t, var.dtype.base_dtype) - multiplier_values = alpha_t + sign_decayed * sign_gm.values - multiplier = ops.IndexedSlices( - multiplier_values, sign_gm.indices, dense_shape=sign_gm.dense_shape) - - final_update = ops.IndexedSlices( - lr_t * multiplier.values * grad.values, - multiplier.indices, - dense_shape=multiplier.dense_shape) - - var_update = state_ops.scatter_sub( - var, - final_update.indices, - final_update.values, - use_locking=self._use_locking) - - return control_flow_ops.group(* [var_update, m_t]) diff --git a/tensorflow/contrib/opt/python/training/addsign_test.py b/tensorflow/contrib/opt/python/training/addsign_test.py deleted file mode 100644 index 2c74acd9fff..00000000000 --- a/tensorflow/contrib/opt/python/training/addsign_test.py +++ /dev/null @@ -1,262 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for AddSign.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.opt.python.training import addsign -from tensorflow.contrib.opt.python.training import sign_decay -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def py_linear_decay_fn(decay_steps): - def linear_decay(step): - step = min(step, decay_steps) - return float(decay_steps - step) / decay_steps - return linear_decay - - -def addsign_update_numpy(params, - g_t, - m, - lr, - alpha=1.0, - beta=0.9, - py_sign_decay_fn=None, - t=None): - m_t = beta * m + (1 - beta) * g_t - if py_sign_decay_fn is None: - sign_decayed = 1.0 - else: - sign_decayed = py_sign_decay_fn(t-1) - multiplier = alpha + sign_decayed * np.sign(g_t) * np.sign(m_t) - params_t = params - lr * multiplier * g_t - return params_t, m_t - - -class AddSignTest(test.TestCase): - - def _testDense(self, - use_resource=False, - learning_rate=0.1, - sign_decay_fn=None, - py_sign_decay_fn=None, - alpha=1.0, - beta=0.9): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(use_gpu=True): - # Initialize variables for numpy implementation. - m0, m1 = 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - global_step = resource_variable_ops.ResourceVariable( - 0, trainable=False) - else: - var0 = variables.VariableV1(var0_np) - var1 = variables.VariableV1(var1_np) - global_step = variables.VariableV1( - 0, trainable=False) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - opt = addsign.AddSignOptimizer( - learning_rate=learning_rate, - alpha=alpha, - beta=beta, - sign_decay_fn=sign_decay_fn, - ) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - neg_update = opt.apply_gradients(zip([-grads0, -grads1], [var0, var1]), - global_step=global_step) - if not context.executing_eagerly(): - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - # Run 7 steps of AddSign - # first 4 steps with positive gradient - # last 3 steps with negative gradient (sign(gm) should be -1) - for t in range(1, 8): - if t < 5: - if not context.executing_eagerly(): - self.evaluate(update) - elif t > 1: - opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - else: - if not context.executing_eagerly(): - self.evaluate(neg_update) - elif t > 1: - opt.apply_gradients(zip([-grads0, -grads1], [var0, var1]), - global_step=global_step) - - var0_np, m0 = addsign_update_numpy( - var0_np, - grads0_np if t < 5 else -grads0_np, - m0, - learning_rate, - alpha=alpha, - beta=beta, - py_sign_decay_fn=py_sign_decay_fn, - t=t, - ) - var1_np, m1 = addsign_update_numpy( - var1_np, - grads1_np if t < 5 else -grads1_np, - m1, - learning_rate, - alpha=alpha, - beta=beta, - py_sign_decay_fn=py_sign_decay_fn, - t=t, - ) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - - def testDense(self): - decay_steps = 10 - sign_decay_fn = sign_decay.get_linear_decay_fn(decay_steps) - py_sign_decay_fn = py_linear_decay_fn(decay_steps) - self._testDense(use_resource=False) - self._testDense(use_resource=False, learning_rate=0.01, alpha=0.1, beta=0.8) - self._testDense(use_resource=False, - sign_decay_fn=sign_decay_fn, - py_sign_decay_fn=py_sign_decay_fn) - - self._testDense(use_resource=True) - self._testDense(use_resource=True, learning_rate=0.01, alpha=0.1, beta=0.8) - self._testDense(use_resource=True, - sign_decay_fn=sign_decay_fn, - py_sign_decay_fn=py_sign_decay_fn) - - def _testSparse(self, - use_resource=False, - learning_rate=0.1, - sign_decay_fn=None, - py_sign_decay_fn=None, - alpha=1.0, - beta=0.9): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(use_gpu=True): - # Initialize variables for numpy implementation. - m0, m1 = 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - global_step = resource_variable_ops.ResourceVariable( - 0, trainable=False) - else: - var0 = variables.VariableV1(var0_np) - var1 = variables.VariableV1(var1_np) - global_step = variables.VariableV1( - 0, trainable=False) - grads0_np_indices = np.array([0, 1], dtype=np.int32) - grads0 = ops.IndexedSlices( - constant_op.constant(grads0_np), - constant_op.constant(grads0_np_indices), constant_op.constant([2])) - grads1_np_indices = np.array([0, 1], dtype=np.int32) - grads1 = ops.IndexedSlices( - constant_op.constant(grads1_np), - constant_op.constant(grads1_np_indices), constant_op.constant([2])) - opt = addsign.AddSignOptimizer( - learning_rate=learning_rate, - alpha=alpha, - beta=beta, - sign_decay_fn=sign_decay_fn, - ) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - neg_update = opt.apply_gradients(zip([-grads0, -grads1], [var0, var1]), - global_step=global_step) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - # Run 7 steps of AddSign - # first 4 steps with positive gradient - # last 3 steps with negative gradient (sign(gm) should be -1) - for t in range(1, 8): - if t < 5: - update.run() - else: - neg_update.run() - - var0_np, m0 = addsign_update_numpy( - var0_np, - grads0_np if t < 5 else -grads0_np, - m0, - learning_rate, - alpha=alpha, - beta=beta, - py_sign_decay_fn=py_sign_decay_fn, - t=t, - ) - var1_np, m1 = addsign_update_numpy( - var1_np, - grads1_np if t < 5 else -grads1_np, - m1, - learning_rate, - alpha=alpha, - beta=beta, - py_sign_decay_fn=py_sign_decay_fn, - t=t, - ) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testSparse(self): - decay_steps = 10 - sign_decay_fn = sign_decay.get_linear_decay_fn(decay_steps) - py_sign_decay_fn = py_linear_decay_fn(decay_steps) - self._testSparse(use_resource=False) - self._testSparse(use_resource=False, - learning_rate=0.01, - alpha=0.1, - beta=0.8) - self._testSparse(use_resource=False, - sign_decay_fn=sign_decay_fn, - py_sign_decay_fn=py_sign_decay_fn) - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/opt/python/training/agn_optimizer.py b/tensorflow/contrib/opt/python/training/agn_optimizer.py deleted file mode 100644 index 9d8bab8d334..00000000000 --- a/tensorflow/contrib/opt/python/training/agn_optimizer.py +++ /dev/null @@ -1,262 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.training import optimizer -from tensorflow.python.training import session_run_hook - -GLOBAL_VARIABLE_NAME = 'global_center_variable' -GRAD_VARIABLE_NAME = 'grad_variable' - - -class AGNCustomGetter(object): - """Custom_getter class is used to do: - - 1. Change trainable variables to local collection and place them at worker - device - 2. Generate global variables(global center variables) - 3. Generate grad variables(gradients) which record the gradients sum - and place them at worker device - Notice that the class should be used with tf.replica_device_setter, - so that the global center variables and global step variable can be placed - at ps device. - """ - - def __init__(self, worker_device): - """ - Args: - worker_device: put the grad_variables on worker device - """ - self._worker_device = worker_device - self._global_map = {} - self._grad_map = {} - - def __call__(self, getter, name, trainable, collections, *args, **kwargs): - if trainable: - with ops.device(self._worker_device): - local_var = getter( - name, - trainable=True, - collections=[ops.GraphKeys.LOCAL_VARIABLES], - *args, - **kwargs) - if kwargs['reuse'] == True: - return local_var - global_center_variable = getter( - name='%s/%s' % (GLOBAL_VARIABLE_NAME, name), - trainable=False, - collections=[ops.GraphKeys.GLOBAL_VARIABLES], - *args, - **kwargs) - - with ops.device(self._worker_device): - grad_variable = getter( - name='%s/%s' % (GRAD_VARIABLE_NAME, name), - trainable=False, - collections=[ops.GraphKeys.LOCAL_VARIABLES], - *args, - **kwargs) - if kwargs['partitioner'] is None: - self._grad_map[local_var] = grad_variable - self._global_map[local_var] = global_center_variable - else: - v_list = list(local_var) - for i in range(len(v_list)): - self._grad_map[v_list[i]] = list(grad_variable)[i] - self._global_map[v_list[i]] = list(global_center_variable)[i] - return local_var - else: - return getter( - name, trainable=trainable, collections=collections, *args, **kwargs) - - -class AGNOptimizer(optimizer.Optimizer): - """Wrapper that implements the Accumulated GradientNormalization algorithm. - - Reference: - Accumulated Gradient Normalization: Joeri Hermans ACML2017 - https://arxiv.org/abs/1710.02368 - """ - - def __init__(self, - optimizer, - num_worker, - custom_getter, - communication_period=10, - use_locking=True, - name='AGNOptimizer'): - """Construct a new AGN optimizer. - - Args: - optimizer: input optimizer, can be sgd/momentum/adam etc. - num_worker: The number of workers - custom_getter: The AGNCustomGetter - communication_period: An int point value to controls the frequency of the - communication between every worker and the ps. - use_locking: If True use locks for update operations. - name: Optional name prefix for the operations created when applying - gradients. Defaults to "AGNOptimizer". - """ - super(AGNOptimizer, self).__init__(use_locking, name) - self._opt = optimizer - self._num_worker = num_worker - self._period = communication_period - self._global_map = custom_getter._global_map - self._grad_map = custom_getter._grad_map - self._local_step = variable_scope.get_variable( - initializer=0, - trainable=False, - collections=[ops.GraphKeys.LOCAL_VARIABLES], - name='local_step') - self._opt._prepare() - - def apply_gradients(self, grads_and_vars, global_step=None, name=None): - """Apply gradients to global variables. - - This is the second part of `minimize()`. It returns an `Operation` that - applies gradients. - - Args: - grads_and_vars: List of (gradient, variable) pairs as returned by - `compute_gradients()`. - global_step: Optional `Variable` to increment by one after the variables - have been updated. - name: Optional name for the returned operation. Default to the name - passed to the `Optimizer` constructor. - - Returns: - An `Operation` that applies the specified gradients. If `global_step` - was not None, that operation also increments `global_step`. - """ - local_vars = [v for g, v in grads_and_vars if g is not None] - grads = [g for g, v in grads_and_vars if g is not None] - - def _variable_creator(next_creator, collections, **kwargs): - if not collections: - collections = [ops.GraphKeys.LOCAL_VARIABLES] - elif ops.GraphKeys.GLOBAL_VARIABLES in collections: - collections = list(collections) - collections.append(ops.GraphKeys.LOCAL_VARIABLES) - collections.remove(ops.GraphKeys.GLOBAL_VARIABLES) - return next_creator(collections=collections, **kwargs) - - # theta = theta - lr * grad - with variable_scope.variable_creator_scope(_variable_creator): - local_update_op = self._opt.apply_gradients(grads_and_vars) - - # a = a + grad - update_ops = [] - update_ops.append(local_update_op) - grad_vars = [self._grad_map[var] for var in local_vars] - for g, grad_var in zip(grads, grad_vars): - update_ops.append(state_ops.assign_add(grad_var, g)) - - global_center_vars = [self._global_map[var] for var in local_vars] - - # update global variables. - def _Update_global_variables(): - global_norm = [] - # a = a / t - for g in grad_vars: - global_norm.append(state_ops.assign(g, g / self._period)) - # apply - with ops.control_dependencies(global_norm): - apply_global_op = self._opt.apply_gradients( - zip(grad_vars, global_center_vars)) - - # pull - with ops.control_dependencies([apply_global_op]): - update_ops = [] - if global_step: - with ops.colocate_with(global_step): - update_ops.append(state_ops.assign_add(global_step, 1)) - - for lvar in local_vars: - g_val = self._global_map[lvar].read_value() - update_ops.append(state_ops.assign(lvar, g_val)) - for grad_var in grad_vars: - update_ops.append( - state_ops.assign(grad_var, array_ops.zeros_like(grad_var))) - variable_update = control_flow_ops.group(*(update_ops)) - return variable_update - - local_update = state_ops.assign_add( - self._local_step, 1, name='local_step_update').op - - with ops.control_dependencies([local_update]): - condition = math_ops.equal( - math_ops.mod(self._local_step, self._period), 0) - with ops.control_dependencies(update_ops): - conditional_update = control_flow_ops.cond( - condition, _Update_global_variables, control_flow_ops.no_op) - return conditional_update - - def get_init_op(self, task_index): - """Returns the op to let all the local variables and local center - - variables equal to the global center variables before the training begins - """ - init_ops = [] - local_vars = variables.trainable_variables() - global_center_vars = [self._global_map[var] for var in local_vars] - grad_vars = [self._grad_map[var] for var in local_vars] - if not (local_vars and global_center_vars and grad_vars): - raise ValueError('The lists of local_variables, global_center_variables,' - 'grad_center_variables should not be empty') - for lvar, gc_var in zip(local_vars, global_center_vars): - init_ops.append(state_ops.assign(lvar, gc_var)) - for g in grad_vars: - init_ops.append(state_ops.assign(g, array_ops.zeros_like(g))) - init_op = control_flow_ops.group(*(init_ops)) - return init_op - - def make_session_run_hook(self, is_chief, task_index): - """Creates a hook to handle AGNOptimizerHook ops such as initialization.""" - return _AGNOptimizerHook(self, is_chief, task_index) - - -class _AGNOptimizerHook(session_run_hook.SessionRunHook): - - def __init__(self, agn_optimizer, is_chief, task_index): - """Creates hook to handle AGNOptimizer initialization ops. - - Args: - agn_optimizer: `AGNOptimizer` which this hook will initialize. - is_chief: `Bool`, whether is this a chief replica or not. - task_index: int, task_index of worker - """ - self._agn_optimizer = agn_optimizer - self._is_chief = is_chief - self._task_index = task_index - - def begin(self): - self._local_init_op = variables.local_variables_initializer() - self._global_init_op = None - if self._is_chief: - self._global_init_op = variables.global_variables_initializer() - self._variable_init_op = self._agn_optimizer.get_init_op(self._task_index) - - def after_create_session(self, session, coord): - """Run initialization ops""" - session.run(self._variable_init_op) diff --git a/tensorflow/contrib/opt/python/training/agn_optimizer_test.py b/tensorflow/contrib/opt/python/training/agn_optimizer_test.py deleted file mode 100644 index d3da290bdbd..00000000000 --- a/tensorflow/contrib/opt/python/training/agn_optimizer_test.py +++ /dev/null @@ -1,281 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Tests for EAOptimizer.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import portpicker - -from tensorflow.contrib.opt.python.training import agn_optimizer -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import adam -from tensorflow.python.training import device_setter -from tensorflow.python.training import server_lib -from tensorflow.python.training import training -from tensorflow.python.training import training_util - - - -def create_local_cluster(num_workers, num_ps, protocol="grpc"): - """Create local GRPC servers and return them.""" - worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)] - ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)] - cluster_dict = { - "worker": ["localhost:%s" % port for port in worker_ports], - "ps": ["localhost:%s" % port for port in ps_ports] - } - cs = server_lib.ClusterSpec(cluster_dict) - - workers = [ - server_lib.Server( - cs, job_name="worker", protocol=protocol, task_index=ix, start=True) - for ix in range(num_workers) - ] - ps_servers = [ - server_lib.Server( - cs, job_name="ps", protocol=protocol, task_index=ix, start=True) - for ix in range(num_ps) - ] - - return cluster_dict, workers, ps_servers - - -# Creates the workers and return their sessions, graphs, train_ops. -# Cheif worker will update at last -def _get_workers(num_workers, period, workers, num_ps=1): - sessions = [] - graphs = [] - train_ops = [] - for worker_id in range(num_workers): - graph = ops.Graph() - is_chief = (worker_id == 0) - with graph.as_default(): - worker_device = "/job:worker/task:%d/cpu:0" % (worker_id) - ps_device = device_setter.replica_device_setter( - worker_device=worker_device, - ps_device="/job:ps/task:0/cpu:0", - ps_tasks=1) - agn_getter = agn_optimizer.AGNCustomGetter(worker_device=worker_device) - with variable_scope.variable_scope( - "", custom_getter=agn_getter), ops.device(ps_device): - global_step = training_util.get_or_create_global_step() - var_0 = variable_scope.get_variable(initializer=0.0, name="v0") - var_1 = variable_scope.get_variable(initializer=0.5, name="v1") - if num_ps > 1: - with variable_scope.variable_scope( - "", - partitioner=partitioned_variables.fixed_size_partitioner( - num_ps, axis=0), - custom_getter=agn_getter), ops.device(ps_device): - - partition_var = variable_scope.get_variable( - "partition_var", - shape=[2, 4], - initializer=init_ops.zeros_initializer) - part_0 = list(partition_var)[0] - part_1 = list(partition_var)[1] - - with ops.device("/job:worker/task:" + str(worker_id)): - grads_0 = constant_op.constant(-1.0) - grads_1 = constant_op.constant(-1.0) - grads_part_0 = constant_op.constant([[-1., -1., -1., -1.]]) - grads_part_1 = constant_op.constant([[-1., -1., -1., -1.]]) - - optimizer = \ - adam.AdamOptimizer(learning_rate=0.1, beta1=0.0, beta2=0.0) - opt = agn_optimizer.AGNOptimizer( - optimizer, - num_worker=num_workers, - communication_period=period, - custom_getter=agn_getter) - if num_ps == 1: - train_op = [ - opt.apply_gradients(([grads_0, var_0], [grads_1, var_1]), - global_step) - ] - else: - train_op = [ - opt.apply_gradients( - ([grads_0, var_0], [grads_1, var_1], [grads_part_0, part_0], - [grads_part_1, part_1]), global_step) - ] - hook = opt.make_session_run_hook(is_chief, worker_id) - # Creates MonitoredSession - sess = training.MonitoredTrainingSession( - workers[worker_id].target, hooks=[hook]) - - sessions.append(sess) - graphs.append(graph) - train_ops.append(train_op) - - return sessions, graphs, train_ops - - -class AGNOptimizerTest(test.TestCase): - - def _run(self, train_op, sess): - sess.run(train_op) - - def test1Workers2Period(self): - num_workers = 1 - communication_period = 4 - num_ps = 1 - _, workers, _ = create_local_cluster(num_workers=num_workers, num_ps=num_ps) - - sessions, graphs, train_ops = _get_workers(num_workers, - communication_period, workers) - - var_0 = graphs[0].get_tensor_by_name("v0:0") - var_1 = graphs[0].get_tensor_by_name("v1:0") - global_step = training_util.get_global_step(graphs[0]) - var_0_g = graphs[0].get_tensor_by_name( - agn_optimizer.GLOBAL_VARIABLE_NAME + "/v0:0") - var_1_g = graphs[0].get_tensor_by_name( - agn_optimizer.GLOBAL_VARIABLE_NAME + "/v1:0") - - # verify adam/beta variables not in global collection - with graphs[0].as_default(): - for ele in variables.global_variables(): - self.assertTrue(ele.op.name.find("beta") < 0) - if ele.op.name.find("global_center_variable") < 0: - self.assertTrue(ele.op.name.find("Adam") < 0) - - # Verify the initialized value. - self.assertAllEqual(0.0, sessions[0].run(var_0)) - self.assertAllEqual(0.5, sessions[0].run(var_1)) - self.assertAllEqual(0.0, sessions[0].run(var_0_g)) - self.assertAllEqual(0.5, sessions[0].run(var_1_g)) - self.assertAllEqual(0, sessions[0].run(global_step)) - # step 0 - sessions[0].run(train_ops[0]) - self.assertNear(0.1, sessions[0].run(var_0), 1e-6) - self.assertNear(0.6, sessions[0].run(var_1), 1e-6) - self.assertAllEqual(0.0, sessions[0].run(var_0_g)) - self.assertAllEqual(0.5, sessions[0].run(var_1_g)) - self.assertAllEqual(0, sessions[0].run(global_step)) - - # 2 & 3 - sessions[0].run(train_ops[0]) - sessions[0].run(train_ops[0]) - self.assertNear(0.3, sessions[0].run(var_0), 1e-6) - self.assertNear(0.8, sessions[0].run(var_1), 1e-6) - - # 4 - sessions[0].run(train_ops[0]) - # pull - self.assertAllEqual(sessions[0].run(var_0), sessions[0].run(var_0_g)) - self.assertAllEqual(sessions[0].run(var_1), sessions[0].run(var_1_g)) - self.assertNear(0.1, sessions[0].run(var_0), 1e-6) - self.assertNear(0.6, sessions[0].run(var_1), 1e-6) - - sessions[0].run(train_ops[0]) - sessions[0].run(train_ops[0]) - sessions[0].run(train_ops[0]) - sessions[0].run(train_ops[0]) - self.assertAllEqual(sessions[0].run(var_0), sessions[0].run(var_0_g)) - self.assertAllEqual(sessions[0].run(var_1), sessions[0].run(var_1_g)) - self.assertNear(0.2, sessions[0].run(var_0), 1e-6) - self.assertNear(0.7, sessions[0].run(var_1), 1e-6) - - def test2Worker1Period(self): - num_workers = 2 - communication_period = 1 - num_ps = 2 - _, workers, _ = create_local_cluster(num_workers=num_workers, num_ps=num_ps) - - sessions, graphs, train_ops = _get_workers( - num_workers, communication_period, workers, num_ps=2) - - var_0 = graphs[0].get_tensor_by_name("v0:0") - var_1 = graphs[0].get_tensor_by_name("v1:0") - - var_0_1 = graphs[1].get_tensor_by_name("v0:0") - var_1_1 = graphs[1].get_tensor_by_name("v1:0") - - var_0_g = graphs[0].get_tensor_by_name( - agn_optimizer.GLOBAL_VARIABLE_NAME + "/v0:0") - var_1_g = graphs[0].get_tensor_by_name( - agn_optimizer.GLOBAL_VARIABLE_NAME + "/v1:0") - part_0_g = graphs[0].get_tensor_by_name( - agn_optimizer.GLOBAL_VARIABLE_NAME + - "/partition_var/part_0:0") - part_1_g = graphs[0].get_tensor_by_name( - agn_optimizer.GLOBAL_VARIABLE_NAME + - "/partition_var/part_1:0") - - # Verify the initialized value. - self.assertAllEqual(0.0, sessions[0].run(var_0)) - self.assertAllEqual(0.5, sessions[0].run(var_1)) - self.assertAllEqual(0.0, sessions[1].run(var_0_1)) - self.assertAllEqual(0.5, sessions[1].run(var_1_1)) - self.assertAllEqual(0.0, sessions[0].run(var_0_g)) - self.assertAllEqual(0.5, sessions[0].run(var_1_g)) - - # verify each step - sessions[0].run(train_ops[0]) - self.assertNear(0.1, sessions[0].run(var_0_g), 1e-6) - self.assertNDArrayNear([0.1, 0.1, 0.1, 0.1], sessions[0].run(part_0_g), - 1e-6) - self.assertNDArrayNear([0.1, 0.1, 0.1, 0.1], sessions[0].run(part_1_g), - 1e-6) - - sessions[1].run(train_ops[1]) - self.assertNear(0.2, sessions[0].run(var_0_g), 1e-6) - self.assertNDArrayNear([0.2, 0.2, 0.2, 0.2], sessions[0].run(part_0_g), - 1e-6) - self.assertNDArrayNear([0.2, 0.2, 0.2, 0.2], sessions[0].run(part_1_g), - 1e-6) - - sessions[0].run(train_ops[0]) - sessions[1].run(train_ops[1]) - - sessions[0].run(train_ops[0]) - sessions[1].run(train_ops[1]) - self.assertNear(0.6, sessions[0].run(var_0_g), 1e-6) - self.assertNDArrayNear([0.6, 0.6, 0.6, 0.6], sessions[0].run(part_0_g), - 1e-6) - self.assertNDArrayNear([0.6, 0.6, 0.6, 0.6], sessions[0].run(part_1_g), - 1e-6) - - def testAGNCustomGetter(self): - cluster_spec = server_lib.ClusterSpec({ - "ps": ["ps0:2222", "ps1:2222"], - "worker": ["worker0:2222", "worker1:2222", "worker2:2222"] - }) - agn_getter = agn_optimizer.AGNCustomGetter( - worker_device="/job:worker/task:0") - with ops.device( - device_setter.replica_device_setter(cluster=cluster_spec, - worker_device="/job:worker/task:0", - ps_device="/job:ps")), \ - variable_scope.variable_scope("", custom_getter=agn_getter): - v = variable_scope.get_variable(initializer=[1, 2], name="v") - w = variable_scope.get_variable(initializer=[2, 1], name="w") - v_g, w_g = agn_getter._global_map[v], agn_getter._global_map[w] - self.assertDeviceEqual("/job:worker/task:0", v.device) - self.assertDeviceEqual("job:ps/task:0", v_g.device) - self.assertDeviceEqual("/job:worker/task:0", w.device) - self.assertDeviceEqual("job:ps/task:1", w_g.device) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/opt/python/training/drop_stale_gradient_optimizer.py b/tensorflow/contrib/opt/python/training/drop_stale_gradient_optimizer.py deleted file mode 100644 index 4a905b1b2a0..00000000000 --- a/tensorflow/contrib/opt/python/training/drop_stale_gradient_optimizer.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Wrapper optimizer for checking and dropping stale gradients.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.summary import summary -from tensorflow.python.training import optimizer -from tensorflow.python.training import training_util - - -class DropStaleGradientOptimizer(optimizer.Optimizer): - """Wrapper optimizer that checks and drops stale gradient. - - This optimizer records the global step for each worker before computing - gradients and compares it with the global step at the time of applying the - gradients. If the difference is larger than a threshold, it will drop all - the computed gradients. - """ - - def __init__(self, - opt, - staleness, - use_locking=False, - name="DropStaleGradient"): - """Constructs a new DropStaleGradientOptimizer. - - Args: - opt: The actual optimizer that will be used to compute and apply the - gradients. Must be one of the Optimizer classes. - staleness: The maximum staleness allowed for the optimizer. - use_locking: If `True` use locks for clip update operations. - name: Optional name prefix for the operations created when applying - gradients. Defaults to "DropStaleGradient". - """ - super(DropStaleGradientOptimizer, self).__init__(use_locking, name) - self._opt = opt - self._staleness = staleness - - def compute_gradients(self, loss, *args, **kwargs): - # Record current global step for worker. - with ops.colocate_with(loss): - self._local_step = training_util.get_global_step() + 0 - - with ops.control_dependencies([self._local_step]): - loss = gen_array_ops.identity(loss) - return self._opt.compute_gradients(loss, *args, **kwargs) - - def get_slot(self, *args, **kwargs): - return self._opt.get_slot(*args, **kwargs) - - def get_slot_names(self, *args, **kwargs): - return self._opt.get_slot_names(*args, **kwargs) - - def apply_gradients(self, grads_and_vars, global_step=None, name=None): - gradients = [] - # Number of stale gradients. - with ops.colocate_with(global_step): - stale_counter = variable_scope.get_variable( - "stale_counter", [], - initializer=init_ops.zeros_initializer(), - trainable=False) - - def _AcceptGradientOp(): - with ops.control_dependencies( - [self._opt.apply_gradients( - grads_and_vars, global_step=global_step, name=name)]): - return gen_array_ops.identity(0.0) - - def _DropGradientOp(): - return gen_array_ops.identity(1.0) - - for grad_and_var in grads_and_vars: - grad = grad_and_var[0] - if isinstance(grad, ops.Tensor): - gradients.append(grad) - elif grad is not None: - gradients.append(grad.op) - - with ops.control_dependencies(gradients), ops.colocate_with(global_step): - staleness = gen_array_ops.reshape( - global_step - self._local_step, shape=()) - - conditional_update = stale_counter.assign_add(control_flow_ops.cond( - gen_math_ops.less_equal(staleness, self._staleness), - _AcceptGradientOp, _DropGradientOp)) - - summary.scalar( - "Gradient staleness percentage", - stale_counter / (math_ops.cast(global_step + 1, dtypes.float32))) - return conditional_update diff --git a/tensorflow/contrib/opt/python/training/drop_stale_gradient_optimizer_test.py b/tensorflow/contrib/opt/python/training/drop_stale_gradient_optimizer_test.py deleted file mode 100644 index 0a69096768a..00000000000 --- a/tensorflow/contrib/opt/python/training/drop_stale_gradient_optimizer_test.py +++ /dev/null @@ -1,297 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for DropStaleGradientOptimizer.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import portpicker - -from tensorflow.contrib.opt.python.training import drop_stale_gradient_optimizer -from tensorflow.python.client import session -from tensorflow.python.framework import ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import gradient_descent -from tensorflow.python.training import server_lib -from tensorflow.python.training import training_util - - -# Creates the workers and return their sessions, graphs, train_ops. -def _get_workers(num_workers, staleness): - worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)] - cluster_dict = { - 'worker': ['localhost:%s' % port for port in worker_ports], - 'ps': ['localhost:%s' % portpicker.pick_unused_port()] - } - cs = server_lib.ClusterSpec(cluster_dict) - workers = [ - server_lib.Server( - cs, job_name='worker', task_index=ix, start=True) - for ix in range(num_workers) - ] - server_lib.Server(cs, job_name='ps', task_index=0, start=True) - - sessions = [] - graphs = [] - train_ops = [] - - # To simulate stale cases, maintaining two queues for computing and - # applying gradients respectively. In the phase of computing gradients, - # all workers except chief worker compute gradients together and chief worker - # computes after all other worers' computing finished. In the phase of - # applying gradients, chief worker will first apply gradients, then all other - # workers will apply gradients one by one. Therefore, the chief worker will - # always have 0 staleness, each of all other workers will have a unique - # staleness value from [1, num_workers). - for worker_id in range(num_workers): - graph = ops.Graph() - with graph.as_default(): - global_step = training_util.create_global_step() - var_0 = variables.VariableV1(0.0, name='v0') - var_1 = variables.VariableV1(1.0, name='v1') - compute_gradients_queue = data_flow_ops.FIFOQueue( - -1, global_step.dtype.base_dtype, shapes=(), - name='compute_gradients_queue', shared_name='compute_gradients_queue') - apply_gradients_queue = data_flow_ops.FIFOQueue( - -1, global_step.dtype.base_dtype, shapes=(), - name='apply_gradients_queue', shared_name='apply_gradients_queue') - - # Gradients for loss on var_0 and var_1 will be 1.0. - loss = 0 - var_0 - var_1 - sgd_opt = gradient_descent.GradientDescentOptimizer(1.0) - stale_check_opt = ( - drop_stale_gradient_optimizer.DropStaleGradientOptimizer( - sgd_opt, staleness)) - - # Compute gradients. - if worker_id == 0: - with ops.control_dependencies( - [compute_gradients_queue.dequeue_many(num_workers - 1)]): - grad_and_vars = stale_check_opt.compute_gradients(loss) - else: - grad_and_vars = stale_check_opt.compute_gradients(loss) - with ops.control_dependencies([t[0] for t in grad_and_vars]): - worker_enqueue_op = compute_gradients_queue.enqueue(global_step) - - # Apply gradients. - if worker_id == 0: - with ops.control_dependencies( - [stale_check_opt.apply_gradients(grad_and_vars, global_step)]): - train_op = apply_gradients_queue.enqueue(global_step) - else: - with ops.control_dependencies([worker_enqueue_op]): - with ops.control_dependencies([apply_gradients_queue.dequeue()]): - with ops.control_dependencies( - [stale_check_opt.apply_gradients( - grad_and_vars, global_step)]): - train_op = apply_gradients_queue.enqueue(global_step) - - sess = session.Session(workers[worker_id].target) - - sessions.append(sess) - graphs.append(graph) - train_ops.append(train_op) - - return sessions, graphs, train_ops - - -class DropStaleGradientOptimizerTest(test.TestCase): - - def _run(self, train_op, sess): - sess.run(train_op) - - def test1Worker(self): - num_workers = 1 - sessions, graphs, train_ops = _get_workers(num_workers, 0) - with graphs[0].as_default(): - sessions[0].run(variables.global_variables_initializer()) - global_step = training_util.get_global_step(graphs[0]) - var_0 = graphs[0].get_tensor_by_name('v0:0') - var_1 = graphs[0].get_tensor_by_name('v1:0') - stale_counter = graphs[0].get_tensor_by_name('stale_counter:0') - # Verify the initialized value. - self.assertAllEqual(0.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0, sessions[0].run(var_1)) - self.assertAllEqual(0.0, sessions[0].run(stale_counter)) - self.assertAllEqual(0, sessions[0].run(global_step)) - - sessions[0].run(train_ops[0]) - - # Verify the updated value after 1 step. - self.assertAllEqual(1, sessions[0].run(global_step)) - self.assertAllEqual(0.0 + 1.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0 + 1.0, sessions[0].run(var_1)) - self.assertAllEqual(1, sessions[0].run(global_step)) - - def test1WorkerNegativeStaleness(self): - num_workers = 1 - sessions, graphs, train_ops = _get_workers(num_workers, -1) - with graphs[0].as_default(): - sessions[0].run(variables.global_variables_initializer()) - global_step = training_util.get_global_step(graphs[0]) - var_0 = graphs[0].get_tensor_by_name('v0:0') - var_1 = graphs[0].get_tensor_by_name('v1:0') - stale_counter = graphs[0].get_tensor_by_name('stale_counter:0') - # Verify the initialized value. - self.assertAllEqual(0.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0, sessions[0].run(var_1)) - self.assertAllEqual(0.0, sessions[0].run(stale_counter)) - self.assertAllEqual(0, sessions[0].run(global_step)) - - sessions[0].run(train_ops[0]) - - # Verify no updates because max staleness is negative. - self.assertAllEqual(0, sessions[0].run(global_step)) - self.assertAllEqual(1.0, sessions[0].run(stale_counter)) - self.assertAllEqual(0.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0, sessions[0].run(var_1)) - - def test2WorkersStaleness0(self): - num_workers = 2 - sessions, graphs, train_ops = _get_workers(num_workers, 0) - with graphs[0].as_default(): - sessions[0].run(variables.global_variables_initializer()) - global_step = training_util.get_global_step(graphs[0]) - var_0 = graphs[0].get_tensor_by_name('v0:0') - var_1 = graphs[0].get_tensor_by_name('v1:0') - stale_counter = graphs[0].get_tensor_by_name('stale_counter:0') - # Verify the initialized value. - self.assertAllEqual(0.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0, sessions[0].run(var_1)) - self.assertAllEqual(0.0, sessions[0].run(stale_counter)) - self.assertAllEqual(0, sessions[0].run(global_step)) - - thread_0 = self.checkedThread( - target=self._run, args=(train_ops[0], sessions[0])) - thread_1 = self.checkedThread( - target=self._run, args=(train_ops[1], sessions[1])) - thread_0.start() - thread_1.start() - thread_0.join() - thread_1.join() - - # With 2 workers and max staleness set to 0, only chief worker will update - # var_0 and var_1. - self.assertAllEqual(1, sessions[0].run(global_step)) - self.assertAllEqual(1.0, sessions[0].run(stale_counter)) - self.assertAllEqual(0.0 + 1.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0 + 1.0, sessions[0].run(var_1)) - - def test2WorkersStaleness1(self): - num_workers = 2 - sessions, graphs, train_ops = _get_workers(num_workers, 1) - with graphs[0].as_default(): - sessions[0].run(variables.global_variables_initializer()) - global_step = training_util.get_global_step(graphs[0]) - var_0 = graphs[0].get_tensor_by_name('v0:0') - var_1 = graphs[0].get_tensor_by_name('v1:0') - stale_counter = graphs[0].get_tensor_by_name('stale_counter:0') - # Verify the initialized value. - self.assertAllEqual(0.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0, sessions[0].run(var_1)) - self.assertAllEqual(0.0, sessions[0].run(stale_counter)) - self.assertAllEqual(0, sessions[0].run(global_step)) - - thread_0 = self.checkedThread( - target=self._run, args=(train_ops[0], sessions[0])) - thread_1 = self.checkedThread( - target=self._run, args=(train_ops[1], sessions[1])) - thread_0.start() - thread_1.start() - thread_0.join() - thread_1.join() - - # With 2 workers and max staleness set to 1, both workers will update - # var_0 and var_1. - self.assertAllEqual(2, sessions[0].run(global_step)) - self.assertAllEqual(0.0, sessions[0].run(stale_counter)) - self.assertAllEqual(0.0 + 2.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0 + 2.0, sessions[0].run(var_1)) - - def test3WorkersStaleness0(self): - num_workers = 3 - sessions, graphs, train_ops = _get_workers(num_workers, 0) - with graphs[0].as_default(): - sessions[0].run(variables.global_variables_initializer()) - global_step = training_util.get_global_step(graphs[0]) - var_0 = graphs[0].get_tensor_by_name('v0:0') - var_1 = graphs[0].get_tensor_by_name('v1:0') - stale_counter = graphs[0].get_tensor_by_name('stale_counter:0') - # Verify the initialized value. - self.assertAllEqual(0.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0, sessions[0].run(var_1)) - self.assertAllEqual(0.0, sessions[0].run(stale_counter)) - self.assertAllEqual(0, sessions[0].run(global_step)) - - thread_0 = self.checkedThread( - target=self._run, args=(train_ops[0], sessions[0])) - thread_1 = self.checkedThread( - target=self._run, args=(train_ops[1], sessions[1])) - thread_2 = self.checkedThread( - target=self._run, args=(train_ops[2], sessions[2])) - thread_0.start() - thread_1.start() - thread_2.start() - thread_0.join() - thread_1.join() - thread_2.join() - - # With 3 workers and max staleness set to 0, only chief worker will update - # var_0 and var_1. - self.assertAllEqual(1, sessions[0].run(global_step)) - self.assertAllEqual(2.0, sessions[0].run(stale_counter)) - self.assertAllEqual(0.0 + 1.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0 + 1.0, sessions[0].run(var_1)) - - def test3WorkersStaleness1(self): - num_workers = 3 - sessions, graphs, train_ops = _get_workers(num_workers, 1) - with graphs[0].as_default(): - sessions[0].run(variables.global_variables_initializer()) - global_step = training_util.get_global_step(graphs[0]) - var_0 = graphs[0].get_tensor_by_name('v0:0') - var_1 = graphs[0].get_tensor_by_name('v1:0') - stale_counter = graphs[0].get_tensor_by_name('stale_counter:0') - # Verify the initialized value. - self.assertAllEqual(0.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0, sessions[0].run(var_1)) - self.assertAllEqual(0.0, sessions[0].run(stale_counter)) - self.assertAllEqual(0, sessions[0].run(global_step)) - - thread_0 = self.checkedThread( - target=self._run, args=(train_ops[0], sessions[0])) - thread_1 = self.checkedThread( - target=self._run, args=(train_ops[1], sessions[1])) - thread_2 = self.checkedThread( - target=self._run, args=(train_ops[2], sessions[2])) - thread_0.start() - thread_1.start() - thread_2.start() - thread_0.join() - thread_1.join() - thread_2.join() - - # With 3 workers and max staleness set to 1, chief worker and only one of - # the two other workers will update var_0 and var_1. - self.assertAllEqual(2, sessions[0].run(global_step)) - self.assertAllEqual(1.0, sessions[0].run(stale_counter)) - self.assertAllEqual(0.0 + 2.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0 + 2.0, sessions[0].run(var_1)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/opt/python/training/elastic_average_optimizer.py b/tensorflow/contrib/opt/python/training/elastic_average_optimizer.py deleted file mode 100644 index d24de9efeec..00000000000 --- a/tensorflow/contrib/opt/python/training/elastic_average_optimizer.py +++ /dev/null @@ -1,475 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Wrapper optimizer for Elastic Average SGD """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import gen_nn_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.training import optimizer -from tensorflow.python.training import saver -from tensorflow.python.training import session_run_hook -from tensorflow.python.training.saving import saveable_object_util - -LOCAL_VARIABLE_NAME = 'local_center_variable' -GLOBAL_VARIABLE_NAME = 'global_center_variable' -GLOBAL_STEP = 'global_step' - - -class ElasticAverageCustomGetter(object): - """Custom_getter class is used to do: - - 1. Change trainable variables to local collection and place them at worker - device - 2. Generate global variables(global center variables) - 3. Generate local variables(local center variables) which record the global - variables and place them at worker device - Notice that the class should be used with tf.replica_device_setter, - so that the global center variables and global step variable can be placed - at ps device. Besides, use 'tf.compat.v1.get_variable' instead of - 'tf.Variable' to - use this custom getter. - - For example, - ea_custom_getter = ElasticAverageCustomGetter(worker_device) - with tf.device( - tf.compat.v1.train.replica_device_setter( - worker_device=worker_device, - ps_device="/job:ps", - cluster=cluster)), - tf.compat.v1.variable_scope('',custom_getter=ea_custom_getter): - ... - create your model here - ... - with tf.device(worker_device): - opt = tf.compat.v1.train.MomentumOptimizer(...) - optimizer = ElasticAverageOptimizer( - opt, - num_worker=2, - moving_rate=0.01, # or use default value - communication_period=20, - ea_custom_getter=ea_custom_getter) - ... - train_op = optimizer.apply_gradients( - grads_vars, - global_step=global_step) - ... - hooks = [optimizer.make_session_run_hook(is_chief, task_index)] - ... - with tf.compat.v1.train.MonitoredTrainingSession(master=server.target, - is_chief=is_chief, - checkpoint_dir=("...), - save_checkpoint_secs=600, - hooks=hooks) as mon_sess: - """ - - def __init__(self, worker_device): - """Create a new `ElasticAverageCustomGetter`. - - Args: - worker_device: String. Name of the `worker` job. - """ - self._worker_device = worker_device - self._local_map = {} - self._global_map = {} - - def __call__(self, getter, name, trainable, collections, *args, **kwargs): - if trainable: - with ops.device(self._worker_device): - local_var = getter( - name, - trainable=True, - collections=[ops.GraphKeys.LOCAL_VARIABLES], - *args, - **kwargs) - if kwargs['reuse'] == True: - return local_var - global_center_variable = getter( - name='%s/%s' % (GLOBAL_VARIABLE_NAME, name), - trainable=False, - collections=[ops.GraphKeys.GLOBAL_VARIABLES], - *args, - **kwargs) - - with ops.device(self._worker_device): - local_center_variable = getter( - name='%s/%s' % (LOCAL_VARIABLE_NAME, name), - trainable=False, - collections=[ops.GraphKeys.LOCAL_VARIABLES], - *args, - **kwargs) - if kwargs['partitioner'] is None: - self._local_map[local_var] = local_center_variable - self._global_map[local_var] = global_center_variable - else: - v_list = list(local_var) - for i in range(len(v_list)): - self._local_map[v_list[i]] \ - = list(local_center_variable)[i] - self._global_map[v_list[i]] \ - = list(global_center_variable)[i] - return local_var - else: - kwargs['trainable'] = trainable - kwargs['collections'] = collections - if ops.GraphKeys.LOCAL_VARIABLES in collections: - with ops.device(self._worker_device): - return getter(name, *args, **kwargs) - else: - return getter(name, *args, **kwargs) - - -class ElasticAverageOptimizer(optimizer.Optimizer): - """Wrapper optimizer that implements the Elastic Average SGD algorithm. - - This is an async optimizer. During the training, Each worker will update - the local variables and maintains its own local_step, which starts from 0 - and is incremented by 1 after each update of local variables. Whenever - the communication period divides the local step, the worker requests - the current global center variables and then computed the elastic difference - between global center variables and local variables. The elastic difference - then be used to update both local variables and global variables. - """ - - # Default value as paper described - BETA = 0.9 - - def __init__(self, - opt, - num_worker, - ea_custom_getter, - communication_period=10, - moving_rate=None, - rho=None, - use_locking=True, - synchronous=False, - name='ElasticAverageOptimizer'): - """Construct a new gradient descent optimizer. - - Args: - opt: The actual optimizer that will be used to update local variables. - Must be one of the Optimizer classes. - num_worker: The number of workers - ea_custom_getter: The ElasticAverageCustomGetter - communication_period: An int point value to controls the frequency of the - communication between every worker and the ps. - moving_rate: A floating point value to control the elastic difference. - rho: the amount of exploration we allow in the model. The default value is - moving_rate/learning_rate rho=0.0 is suggested in async mode. - use_locking: If True use locks for update operations. - synchronous: Add_sync_queues_and_barrier or not. - True: all workers will wait for each other before start training - False: worker can start training when its initilization is done, - no need to wait for everyone is ready. in case one worker is - restarted, it can join and continue training without being - blocked. - name: Optional name prefix for the operations created when applying - gradients. Defaults to "ElasticAverageOptimizer". - """ - super(ElasticAverageOptimizer, self).__init__(use_locking, name) - self._opt = opt - self._num_worker = num_worker - self._period = communication_period - self._local_map = ea_custom_getter._local_map - self._global_map = ea_custom_getter._global_map - self._synchronous = synchronous - - if moving_rate is None: - self._moving_rate = self.BETA / communication_period / num_worker - else: - self._moving_rate = moving_rate - if rho is None: - self._rho = self._moving_rate / self._opt._learning_rate - else: - self._rho = rho - - self._local_step = variable_scope.get_variable( - initializer=0, - trainable=False, - collections=[ops.GraphKeys.LOCAL_VARIABLES], - name='local_step') - self._opt._prepare() - - def compute_gradients(self, - loss, - var_list=None, - gate_gradients=optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - grad_loss=None): - """Compute gradients of `loss` for the variables in `var_list`. - - Add rho*elastic_difference to loss to control the exploration - This is the first part of `minimize()`. It returns a list - of (gradient, variable) pairs where "gradient" is the gradient - for "variable". Note that "gradient" can be a `Tensor`, an - `IndexedSlices`, or `None` if there is no gradient for the - given variable. - - Args: - loss: A Tensor containing the value to minimize. - var_list: Optional list or tuple of `tf.Variable` to update to minimize - `loss`. Defaults to the list of variables collected in the graph under - the key `GraphKey.TRAINABLE_VARIABLES`. - gate_gradients: How to gate the computation of gradients. Can be - `GATE_NONE`, `GATE_OP`, or `GATE_GRAPH`. - aggregation_method: Specifies the method used to combine gradient terms. - Valid values are defined in the class `AggregationMethod`. - colocate_gradients_with_ops: If True, try colocating gradients with the - corresponding op. - grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`. - - Returns: - A list of (gradient, variable) pairs. Variable is always present, but - gradient can be `None`. - - Raises: - TypeError: If `var_list` contains anything else than `Variable` objects. - ValueError: If some arguments are invalid. - """ - if not var_list: - var_list = variables.trainable_variables() - - elastic_difference = [ - math_ops.subtract(v, lv) - for v, lv in zip(variables.trainable_variables(), - [self._local_map[var] for var in var_list]) - ] - - distance_loss = self._rho * math_ops.add_n( - [gen_nn_ops.l2_loss(ed) for ed in elastic_difference]) - - total_loss = loss + distance_loss - return self._opt.compute_gradients(total_loss, var_list, gate_gradients, - aggregation_method, - colocate_gradients_with_ops, grad_loss) - - def apply_gradients(self, grads_and_vars, global_step=None, name=None): - """Apply gradients to global variables. - - This is the second part of `minimize()`. It returns an `Operation` that - applies gradients. - - Args: - grads_and_vars: List of (gradient, variable) pairs as returned by - `compute_gradients()`. - global_step: Optional `Variable` to increment by one after the variables - have been updated. - name: Optional name for the returned operation. Default to the name - passed to the `Optimizer` constructor. - - Returns: - An `Operation` that applies the specified gradients. If `global_step` - was not None, that operation also increments `global_step`. - - Raises: - TypeError: If `grads_and_vars` is malformed. - ValueError: If none of the variables have gradients. - """ - global_old = set(n.op.name for n in variables.global_variables()) - apply_updates = self._opt.apply_gradients(grads_and_vars) - global_new = set(n.op.name for n in variables.global_variables()) - with ops.control_dependencies([apply_updates]): - local_update = state_ops.assign_add( - self._local_step, 1, name='local_step_update').op - - # this is for place the variables created by optimizer to local collection - # e.g., AdamOptimizer will create beta as global variables - def _adjust_optimizer_variable_collection(opt_vars): - g = ops.get_default_graph() - idx = 0 - for _ in range(len(g._collections[ops.GraphKeys.GLOBAL_VARIABLES])): - var = g.get_collection_ref(ops.GraphKeys.GLOBAL_VARIABLES)[idx] - name = var.op.name - if name in opt_vars: - ops.add_to_collection(ops.GraphKeys.LOCAL_VARIABLES, var) - del g.get_collection_ref(ops.GraphKeys.GLOBAL_VARIABLES)[idx] - else: - idx += 1 - - _adjust_optimizer_variable_collection(global_new - global_old) - - # update global variables. - def _Update_global_variables(): - local_vars = [v for g, v in grads_and_vars if g is not None] - global_center_vars = [self._global_map[var] for var in local_vars] - local_center_vars = [self._local_map[var] for var in local_vars] - local_center_vars_update = [] - for lvar, var in zip(local_center_vars, global_center_vars): - local_center_vars_update.append(lvar.assign(var)) - update_ops = [] - differences = [] - with ops.control_dependencies(local_center_vars_update): - for v, lv in zip(local_vars, local_center_vars): - with ops.device(v.device): - differences.append(math_ops.subtract(v, lv)) - for lvar, diff in zip(local_vars, differences): - with ops.device(lvar.device): - update_ops.append( - state_ops.assign_sub(lvar, - math_ops.multiply(self._moving_rate, - diff))) - for var, diff in zip(global_center_vars, differences): - with ops.device(var.device): - update_ops.append( - state_ops.assign_add(var, - math_ops.multiply(self._moving_rate, - diff))) - if global_step: - with ops.colocate_with(global_step): - update_ops.append(state_ops.assign_add(global_step, 1)) - variable_update = control_flow_ops.group(*(update_ops)) - return variable_update - - with ops.control_dependencies([local_update]): - condition = math_ops.equal( - math_ops.mod(self._local_step, self._period), 0) - conditional_update = control_flow_ops.cond(condition, - _Update_global_variables, - control_flow_ops.no_op) - return conditional_update - - def get_init_op(self, task_index): - """Returns the op to let all the local variables and local center - - variables equal to the global center variables before the training begins - """ - - def _Add_sync_queues_and_barrier(enqueue_after_list): - """Adds ops to enqueue on all worker queues""" - sync_queues = [ - data_flow_ops.FIFOQueue( - self._num_worker, [dtypes.bool], - shapes=[[]], - shared_name='%s%s' % ('variable_init_sync_queue', i)) - for i in range(self._num_worker) - ] - queue_ops = [] - # For each other worker, add an entry in a queue - token = constant_op.constant(False) - with ops.control_dependencies(enqueue_after_list): - for i, q in enumerate(sync_queues): - if i == task_index: - queue_ops.append(control_flow_ops.no_op()) - else: - queue_ops.append(q.enqueue(token)) - queue_ops.append( - sync_queues[task_index].dequeue_many(len(sync_queues) - 1)) - return control_flow_ops.group(*queue_ops) - - init_ops = [] - local_vars = variables.trainable_variables() - global_center_vars = [self._global_map[var] for var in local_vars] - local_center_vars = [self._local_map[var] for var in local_vars] - if not (local_vars and global_center_vars and local_center_vars): - raise ValueError('The lists of local_variables, global_center_variables, ' - 'local_center_variables should not be empty ') - for lvar, gc_var, lc_var in zip(local_vars, global_center_vars, - local_center_vars): - init_ops.append(state_ops.assign(lvar, gc_var)) - init_ops.append(state_ops.assign(lc_var, gc_var)) - - init_op = control_flow_ops.group(*(init_ops)) - if self._synchronous == False: - return init_op - - sync_queue_op = _Add_sync_queues_and_barrier([init_op]) - return sync_queue_op - - def make_session_run_hook(self, is_chief, task_index): - """Creates a hook to handle ElasticAverageOptimizerHook ops such as initialization.""" - return _ElasticAverageOptimizerHook(self, is_chief, task_index) - - def swapping_saver(self, var_list=None, name='swapping_saver', **kwargs): - """Create a saver copy global_center_variable to trainable variables - - Please call this function after all your variables created with - ElasticAverageCustomGetter. For evaluations or inference, use this saver - during training. It will save the global_center_variable of the trained - parameters under the original parameter names. - Args: - var_list: List of variables to save, as per `Saver()`. If set to None, - save all the trainable_variables that have been created before this - call. - name: The name of the saver. - **kwargs: Keyword arguments of `Saver()`. - - Returns: - A `tf.compat.v1.train.Saver` object. - Raises: - RuntimeError: global_center_variable is empty, please make sure - this is called after model created and - ElasticAverageCustomGetter is used when declaring you model - """ - if not self._global_map: - raise RuntimeError('global_center_variable is empty, please make sure ' - 'this is called after model created and ' - 'ElasticAverageCustomGetter is used when declaring ' - 'you model') - - if var_list is None: - var_list = variables.trainable_variables() - if not isinstance(var_list, dict): - var_list = saveable_object_util.op_list_to_dict(var_list) - - swapped_var_list = {} - for key, var in var_list.items(): - tensor = var - - if not isinstance(var, list): - for tvar in variables.trainable_variables(): - if tvar.op.name == var.op.name: - tensor = self._global_map.get(tvar, var) - break - else: #partitioned variable - tensor = [self._global_map.get(lvar, lvar) for lvar in var] - - swapped_var_list[key] = tensor - - return saver.Saver(swapped_var_list, name=name, **kwargs) - - -class _ElasticAverageOptimizerHook(session_run_hook.SessionRunHook): - - def __init__(self, ea_optimizer, is_chief, task_index): - """Creates hook to handle ElasticAverageOptimizer initialization ops. - - Args: - ea_optimizer: `ElasticAverageOptimizer` which this hook will initialize. - is_chief: `Bool`, whether is this a chief replica or not. - """ - self._ea_optimizer = ea_optimizer - self._is_chief = is_chief - self._task_index = task_index - - def begin(self): - self._local_init_op = variables.local_variables_initializer() - self._global_init_op = None - if self._is_chief: - self._global_init_op = variables.global_variables_initializer() - self._variable_init_op = self._ea_optimizer.get_init_op(self._task_index) - - def after_create_session(self, session, coord): - """Run initialization ops""" - session.run(self._variable_init_op) diff --git a/tensorflow/contrib/opt/python/training/elastic_average_optimizer_test.py b/tensorflow/contrib/opt/python/training/elastic_average_optimizer_test.py deleted file mode 100644 index 5bf6a08de12..00000000000 --- a/tensorflow/contrib/opt/python/training/elastic_average_optimizer_test.py +++ /dev/null @@ -1,297 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for ElasticAverageOptimizer.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import portpicker -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import device_setter -from tensorflow.python.training import gradient_descent -from tensorflow.python.training import saver -from tensorflow.python.training import server_lib -from tensorflow.python.training import training -from tensorflow.python.training import training_util - -from tensorflow.contrib.opt.python.training.elastic_average_optimizer import \ - ElasticAverageOptimizer, ElasticAverageCustomGetter, GLOBAL_VARIABLE_NAME - - -def create_local_cluster(num_workers, num_ps, protocol="grpc"): - """Create local GRPC servers and return them.""" - worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)] - ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)] - cluster_dict = { - "worker": ["localhost:%s" % port for port in worker_ports], - "ps": ["localhost:%s" % port for port in ps_ports] - } - cs = server_lib.ClusterSpec(cluster_dict) - - workers = [ - server_lib.Server( - cs, job_name="worker", protocol=protocol, task_index=ix, start=True) - for ix in range(num_workers) - ] - ps_servers = [ - server_lib.Server( - cs, job_name="ps", protocol=protocol, task_index=ix, start=True) - for ix in range(num_ps) - ] - - return cluster_dict, workers, ps_servers - - -# Creates the workers and return their sessions, graphs, train_ops. -# Chief worker will update at last -def _get_workers(num_workers, period, workers, moving_rate, num_ps=1): - sessions = [] - graphs = [] - train_ops = [] - savers = [] - for worker_id in range(num_workers): - graph = ops.Graph() - is_chief = (worker_id == 0) - with graph.as_default(): - worker_device = "/job:worker/task:%d/cpu:0" % (worker_id) - ea_custom = ElasticAverageCustomGetter(worker_device=worker_device) - with variable_scope.variable_scope( - "", custom_getter=ea_custom), ops.device( - device_setter.replica_device_setter( - worker_device=worker_device, - ps_device="/job:ps/task:0/cpu:0", - ps_tasks=1)): - global_step = training_util.get_or_create_global_step() - var_0 = variable_scope.get_variable(initializer=0.0, name="v0") - var_1 = variable_scope.get_variable(initializer=1.0, name="v1") - if num_ps > 1: - with variable_scope.variable_scope( - "", - partitioner=partitioned_variables.fixed_size_partitioner( - num_ps, axis=0), - custom_getter=ea_custom), ops.device( - device_setter.replica_device_setter( - worker_device=worker_device, - ps_device="/job:ps/task:0/cpu:0", - ps_tasks=num_ps)): - - partition_var = variable_scope.get_variable( - 'partition_var', - shape=[2, 4], - initializer=init_ops.ones_initializer) - part_0 = list(partition_var)[0] - part_1 = list(partition_var)[1] - - with ops.device("/job:worker/task:" + str(worker_id)): - grads_0 = constant_op.constant(-1.0) - grads_1 = constant_op.constant(-1.0) - grads_part_0 = constant_op.constant([[-1., -1., -1., -1.]]) - grads_part_1 = constant_op.constant([[-1., -1., -1., -1.]]) - - sgd_opt = gradient_descent.GradientDescentOptimizer(1.0) - opt = ElasticAverageOptimizer( - opt=sgd_opt, - num_worker=num_workers, - moving_rate=moving_rate, - communication_period=period, - ea_custom_getter=ea_custom) - if num_ps == 1: - train_op = [ - opt.apply_gradients(([grads_0, var_0], [grads_1, var_1]), - global_step) - ] - else: - train_op = [ - opt.apply_gradients(([grads_0, var_0], - [grads_1, var_1], - [grads_part_0, part_0], - [grads_part_1, part_1]), - global_step) - ] - easgd_hook = opt.make_session_run_hook(is_chief, worker_id) - saver = opt.swapping_saver() - # Creates MonitoredSession - sess = training.MonitoredTrainingSession( - workers[worker_id].target, hooks=[easgd_hook]) - - sessions.append(sess) - graphs.append(graph) - train_ops.append(train_op) - savers.append(saver) - - return sessions, graphs, train_ops, savers - - -class ElasticAverageOptimizerTest(test.TestCase): - - def _run(self, train_op, sess): - sess.run(train_op) - - def test1Workers2Period(self): - num_workers = 1 - communication_period = 2 - num_ps = 1 - cluster, workers, _ = create_local_cluster( - num_workers=num_workers, num_ps=num_ps) - - sessions, graphs, train_ops, savers = _get_workers( - num_workers, communication_period, workers, 1.0) - - var_0 = graphs[0].get_tensor_by_name("v0:0") - var_1 = graphs[0].get_tensor_by_name("v1:0") - global_step = training_util.get_global_step(graphs[0]) - var_0_g = graphs[0].get_tensor_by_name(GLOBAL_VARIABLE_NAME + "/v0:0") - var_1_g = graphs[0].get_tensor_by_name(GLOBAL_VARIABLE_NAME + "/v1:0") - # Verify the initialized value. - self.assertAllEqual(0.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0, sessions[0].run(var_1)) - self.assertAllEqual(0.0, sessions[0].run(var_0_g)) - self.assertAllEqual(1.0, sessions[0].run(var_1_g)) - self.assertAllEqual(0, sessions[0].run(global_step)) - - sessions[0].run(train_ops[0]) - - self.assertAllEqual(1.0, sessions[0].run(var_0)) - self.assertAllEqual(2.0, sessions[0].run(var_1)) - self.assertAllEqual(0.0, sessions[0].run(var_0_g)) - self.assertAllEqual(1.0, sessions[0].run(var_1_g)) - self.assertAllEqual(0, sessions[0].run(global_step)) - - # iteration 2, global variable update - sessions[0].run(train_ops[0]) - - self.assertAllEqual(0.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0, sessions[0].run(var_1)) - self.assertAllEqual(2.0, sessions[0].run(var_0_g)) - self.assertAllEqual(3.0, sessions[0].run(var_1_g)) - self.assertAllEqual(1, sessions[0].run(global_step)) - - # iteration 3 - sessions[0].run(train_ops[0]) - - self.assertAllEqual(1.0, sessions[0].run(var_0)) - self.assertAllEqual(2.0, sessions[0].run(var_1)) - self.assertAllEqual(2.0, sessions[0].run(var_0_g)) - self.assertAllEqual(3.0, sessions[0].run(var_1_g)) - self.assertAllEqual(1, sessions[0].run(global_step)) - sessions[0].run(train_ops[0]) - - # save, data will be global value - outfile = os.path.join(test.get_temp_dir(), "model") - savers[0].save(sessions[0]._sess._sess._sess._sess, - save_path=outfile) - ops.reset_default_graph() # restore on a new graph - with session.Session() as sess: - v0 = variable_scope.get_variable(initializer=0.0, name="v0") - v1 = variable_scope.get_variable(initializer=1.0, name="v1") - sess.run(variables.local_variables_initializer()) - saver_opt = saver.Saver(var_list=[v1, v0]) - saver_opt.restore(sess, outfile) - self.assertAllEqual(2.0, sess.run(v0)) - self.assertAllEqual(3.0, sess.run(v1)) - - def test2Worker1Period(self): - num_workers = 2 - communication_period = 1 - num_ps = 2 - cluster, workers, _ = create_local_cluster( - num_workers=num_workers, num_ps=num_ps) - - sessions, graphs, train_ops, savers = _get_workers( - num_workers, communication_period, workers, 0.5, num_ps=2) - - var_0 = graphs[0].get_tensor_by_name("v0:0") - var_1 = graphs[0].get_tensor_by_name("v1:0") - - var_0_1 = graphs[1].get_tensor_by_name("v0:0") - var_1_1 = graphs[1].get_tensor_by_name("v1:0") - - var_0_g = graphs[0].get_tensor_by_name(GLOBAL_VARIABLE_NAME + "/v0:0") - var_1_g = graphs[0].get_tensor_by_name(GLOBAL_VARIABLE_NAME + "/v1:0") - part_0_g = graphs[0].get_tensor_by_name( - GLOBAL_VARIABLE_NAME + "/partition_var/part_0:0") - - # Verify the initialized value. - self.assertAllEqual(0.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0, sessions[0].run(var_1)) - self.assertAllEqual(0.0, sessions[1].run(var_0_1)) - self.assertAllEqual(1.0, sessions[1].run(var_1_1)) - self.assertAllEqual(0.0, sessions[0].run(var_0_g)) - self.assertAllEqual(1.0, sessions[0].run(var_1_g)) - - sessions[0].run(train_ops[0]) - sessions[1].run(train_ops[1]) - - self.assertAllEqual(0.5, sessions[0].run(var_0)) - self.assertAllEqual(1.5, sessions[0].run(var_1)) - self.assertAllEqual(0.75, sessions[0].run(var_0_g)) - self.assertAllEqual(1.75, sessions[0].run(var_1_g)) - self.assertAllEqual(0.75, sessions[1].run(var_0_1)) - self.assertAllEqual(1.75, sessions[1].run(var_1_1)) - # part_0 of global_center copy - part_0_g = sessions[0].run(part_0_g) - - outfile = os.path.join(test.get_temp_dir(), "model") - savers[0].save(sessions[0]._sess._sess._sess._sess, - save_path=outfile) - - # verify restore of partitioned_variables - ops.reset_default_graph() # restore on a new graph - g = ops.get_default_graph() - with session.Session() as sess, g.as_default(): - with variable_scope.variable_scope( - "", - partitioner=partitioned_variables.fixed_size_partitioner( - num_ps, axis=0)): - partition_var = variable_scope.get_variable( - 'partition_var', - shape=[2, 4], - initializer=init_ops.ones_initializer) - s = saver.Saver(var_list=[partition_var]) - s.restore(sess, outfile) - part_0 = g.get_tensor_by_name('partition_var/part_0:0') - self.assertAllEqual(part_0_g, sess.run(part_0)) - - def testPS2TasksWithClusterSpecClass(self): - cluster_spec = server_lib.ClusterSpec({ - "ps": ["ps0:2222", "ps1:2222"], - "worker": ["worker0:2222", "worker1:2222", "worker2:2222"] - }) - ea_custom = ElasticAverageCustomGetter(worker_device="/job:worker/task:0") - from tensorflow.python.training import device_setter - with ops.device( - device_setter.replica_device_setter(cluster=cluster_spec, - worker_device="/job:worker/task:0", - ps_device="/job:ps")), \ - variable_scope.variable_scope("", custom_getter=ea_custom): - v = variable_scope.get_variable(initializer=[1, 2], name="v") - w = variable_scope.get_variable(initializer=[2, 1], name="w") - v_g, w_g = ea_custom._global_map[v], ea_custom._global_map[w] - self.assertDeviceEqual("/job:worker/task:0", v.device) - self.assertDeviceEqual("job:ps/task:0", v_g.device) - self.assertDeviceEqual("/job:worker/task:0", w.device) - self.assertDeviceEqual("job:ps/task:1", w_g.device) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/opt/python/training/external_optimizer.py b/tensorflow/contrib/opt/python/training/external_optimizer.py deleted file mode 100644 index 814f980c791..00000000000 --- a/tensorflow/contrib/opt/python/training/external_optimizer.py +++ /dev/null @@ -1,448 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""TensorFlow interface for third-party optimizers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients -from tensorflow.python.ops import variables -from tensorflow.python.platform import tf_logging as logging - -__all__ = ['ExternalOptimizerInterface', 'ScipyOptimizerInterface'] - - -class ExternalOptimizerInterface(object): - """Base class for interfaces with external optimization algorithms. - - Subclass this and implement `_minimize` in order to wrap a new optimization - algorithm. - - `ExternalOptimizerInterface` should not be instantiated directly; instead use - e.g. `ScipyOptimizerInterface`. - - @@__init__ - - @@minimize - """ - - def __init__(self, - loss, - var_list=None, - equalities=None, - inequalities=None, - var_to_bounds=None, - **optimizer_kwargs): - """Initialize a new interface instance. - - Args: - loss: A scalar `Tensor` to be minimized. - var_list: Optional `list` of `Variable` objects to update to minimize - `loss`. Defaults to the list of variables collected in the graph - under the key `GraphKeys.TRAINABLE_VARIABLES`. - equalities: Optional `list` of equality constraint scalar `Tensor`s to be - held equal to zero. - inequalities: Optional `list` of inequality constraint scalar `Tensor`s - to be held nonnegative. - var_to_bounds: Optional `dict` where each key is an optimization - `Variable` and each corresponding value is a length-2 tuple of - `(low, high)` bounds. Although enforcing this kind of simple constraint - could be accomplished with the `inequalities` arg, not all optimization - algorithms support general inequality constraints, e.g. L-BFGS-B. Both - `low` and `high` can either be numbers or anything convertible to a - NumPy array that can be broadcast to the shape of `var` (using - `np.broadcast_to`). To indicate that there is no bound, use `None` (or - `+/- np.infty`). For example, if `var` is a 2x3 matrix, then any of - the following corresponding `bounds` could be supplied: - * `(0, np.infty)`: Each element of `var` held positive. - * `(-np.infty, [1, 2])`: First column less than 1, second column less - than 2. - * `(-np.infty, [[1], [2], [3]])`: First row less than 1, second row less - than 2, etc. - * `(-np.infty, [[1, 2, 3], [4, 5, 6]])`: Entry `var[0, 0]` less than 1, - `var[0, 1]` less than 2, etc. - **optimizer_kwargs: Other subclass-specific keyword arguments. - """ - self._loss = loss - self._equalities = equalities or [] - self._inequalities = inequalities or [] - - if var_list is None: - self._vars = variables.trainable_variables() - else: - self._vars = list(var_list) - - packed_bounds = None - if var_to_bounds is not None: - left_packed_bounds = [] - right_packed_bounds = [] - for var in self._vars: - shape = var.get_shape().as_list() - bounds = (-np.infty, np.infty) - if var in var_to_bounds: - bounds = var_to_bounds[var] - left_packed_bounds.extend(list(np.broadcast_to(bounds[0], shape).flat)) - right_packed_bounds.extend(list(np.broadcast_to(bounds[1], shape).flat)) - packed_bounds = list(zip(left_packed_bounds, right_packed_bounds)) - self._packed_bounds = packed_bounds - - self._update_placeholders = [ - array_ops.placeholder(var.dtype) for var in self._vars - ] - self._var_updates = [ - var.assign(array_ops.reshape(placeholder, _get_shape_tuple(var))) - for var, placeholder in zip(self._vars, self._update_placeholders) - ] - - loss_grads = _compute_gradients(loss, self._vars) - equalities_grads = [ - _compute_gradients(equality, self._vars) - for equality in self._equalities - ] - inequalities_grads = [ - _compute_gradients(inequality, self._vars) - for inequality in self._inequalities - ] - - self.optimizer_kwargs = optimizer_kwargs - - self._packed_var = self._pack(self._vars) - self._packed_loss_grad = self._pack(loss_grads) - self._packed_equality_grads = [ - self._pack(equality_grads) for equality_grads in equalities_grads - ] - self._packed_inequality_grads = [ - self._pack(inequality_grads) for inequality_grads in inequalities_grads - ] - - dims = [_prod(_get_shape_tuple(var)) for var in self._vars] - accumulated_dims = list(_accumulate(dims)) - self._packing_slices = [ - slice(start, end) - for start, end in zip(accumulated_dims[:-1], accumulated_dims[1:]) - ] - - def minimize(self, - session=None, - feed_dict=None, - fetches=None, - step_callback=None, - loss_callback=None, - **run_kwargs): - """Minimize a scalar `Tensor`. - - Variables subject to optimization are updated in-place at the end of - optimization. - - Note that this method does *not* just return a minimization `Op`, unlike - `Optimizer.minimize()`; instead it actually performs minimization by - executing commands to control a `Session`. - - Args: - session: A `Session` instance. - feed_dict: A feed dict to be passed to calls to `session.run`. - fetches: A list of `Tensor`s to fetch and supply to `loss_callback` - as positional arguments. - step_callback: A function to be called at each optimization step; - arguments are the current values of all optimization variables - flattened into a single vector. - loss_callback: A function to be called every time the loss and gradients - are computed, with evaluated fetches supplied as positional arguments. - **run_kwargs: kwargs to pass to `session.run`. - """ - session = session or ops.get_default_session() - feed_dict = feed_dict or {} - fetches = fetches or [] - - loss_callback = loss_callback or (lambda *fetches: None) - step_callback = step_callback or (lambda xk: None) - - # Construct loss function and associated gradient. - loss_grad_func = self._make_eval_func([self._loss, - self._packed_loss_grad], session, - feed_dict, fetches, loss_callback) - - # Construct equality constraint functions and associated gradients. - equality_funcs = self._make_eval_funcs(self._equalities, session, feed_dict, - fetches) - equality_grad_funcs = self._make_eval_funcs(self._packed_equality_grads, - session, feed_dict, fetches) - - # Construct inequality constraint functions and associated gradients. - inequality_funcs = self._make_eval_funcs(self._inequalities, session, - feed_dict, fetches) - inequality_grad_funcs = self._make_eval_funcs(self._packed_inequality_grads, - session, feed_dict, fetches) - - # Get initial value from TF session. - initial_packed_var_val = session.run(self._packed_var) - - # Perform minimization. - packed_var_val = self._minimize( - initial_val=initial_packed_var_val, - loss_grad_func=loss_grad_func, - equality_funcs=equality_funcs, - equality_grad_funcs=equality_grad_funcs, - inequality_funcs=inequality_funcs, - inequality_grad_funcs=inequality_grad_funcs, - packed_bounds=self._packed_bounds, - step_callback=step_callback, - optimizer_kwargs=self.optimizer_kwargs) - var_vals = [ - packed_var_val[packing_slice] for packing_slice in self._packing_slices - ] - - # Set optimization variables to their new values. - session.run( - self._var_updates, - feed_dict=dict(zip(self._update_placeholders, var_vals)), - **run_kwargs) - - def _minimize(self, initial_val, loss_grad_func, equality_funcs, - equality_grad_funcs, inequality_funcs, inequality_grad_funcs, - packed_bounds, step_callback, optimizer_kwargs): - """Wrapper for a particular optimization algorithm implementation. - - It would be appropriate for a subclass implementation of this method to - raise `NotImplementedError` if unsupported arguments are passed: e.g. if an - algorithm does not support constraints but `len(equality_funcs) > 0`. - - Args: - initial_val: A NumPy vector of initial values. - loss_grad_func: A function accepting a NumPy packed variable vector and - returning two outputs, a loss value and the gradient of that loss with - respect to the packed variable vector. - equality_funcs: A list of functions each of which specifies a scalar - quantity that an optimizer should hold exactly zero. - equality_grad_funcs: A list of gradients of equality_funcs. - inequality_funcs: A list of functions each of which specifies a scalar - quantity that an optimizer should hold >= 0. - inequality_grad_funcs: A list of gradients of inequality_funcs. - packed_bounds: A list of bounds for each index, or `None`. - step_callback: A callback function to execute at each optimization step, - supplied with the current value of the packed variable vector. - optimizer_kwargs: Other key-value arguments available to the optimizer. - - Returns: - The optimal variable vector as a NumPy vector. - """ - raise NotImplementedError( - 'To use ExternalOptimizerInterface, subclass from it and implement ' - 'the _minimize() method.') - - @classmethod - def _pack(cls, tensors): - """Pack a list of `Tensor`s into a single, flattened, rank-1 `Tensor`.""" - if not tensors: - return None - elif len(tensors) == 1: - return array_ops.reshape(tensors[0], [-1]) - else: - flattened = [array_ops.reshape(tensor, [-1]) for tensor in tensors] - return array_ops.concat(flattened, 0) - - def _make_eval_func(self, tensors, session, feed_dict, fetches, - callback=None): - """Construct a function that evaluates a `Tensor` or list of `Tensor`s.""" - if not isinstance(tensors, list): - tensors = [tensors] - num_tensors = len(tensors) - - def eval_func(x): - """Function to evaluate a `Tensor`.""" - augmented_feed_dict = { - var: x[packing_slice].reshape(_get_shape_tuple(var)) - for var, packing_slice in zip(self._vars, self._packing_slices) - } - augmented_feed_dict.update(feed_dict) - augmented_fetches = tensors + fetches - - augmented_fetch_vals = session.run( - augmented_fetches, feed_dict=augmented_feed_dict) - - if callable(callback): - callback(*augmented_fetch_vals[num_tensors:]) - - return augmented_fetch_vals[:num_tensors] - - return eval_func - - def _make_eval_funcs(self, - tensors, - session, - feed_dict, - fetches, - callback=None): - return [ - self._make_eval_func(tensor, session, feed_dict, fetches, callback) - for tensor in tensors - ] - - -class ScipyOptimizerInterface(ExternalOptimizerInterface): - """Wrapper allowing `scipy.optimize.minimize` to operate a `tf.compat.v1.Session`. - - Example: - - ```python - vector = tf.Variable([7., 7.], 'vector') - - # Make vector norm as small as possible. - loss = tf.reduce_sum(tf.square(vector)) - - optimizer = ScipyOptimizerInterface(loss, options={'maxiter': 100}) - - with tf.compat.v1.Session() as session: - optimizer.minimize(session) - - # The value of vector should now be [0., 0.]. - ``` - - Example with simple bound constraints: - - ```python - vector = tf.Variable([7., 7.], 'vector') - - # Make vector norm as small as possible. - loss = tf.reduce_sum(tf.square(vector)) - - optimizer = ScipyOptimizerInterface( - loss, var_to_bounds={vector: ([1, 2], np.infty)}) - - with tf.compat.v1.Session() as session: - optimizer.minimize(session) - - # The value of vector should now be [1., 2.]. - ``` - - Example with more complicated constraints: - - ```python - vector = tf.Variable([7., 7.], 'vector') - - # Make vector norm as small as possible. - loss = tf.reduce_sum(tf.square(vector)) - # Ensure the vector's y component is = 1. - equalities = [vector[1] - 1.] - # Ensure the vector's x component is >= 1. - inequalities = [vector[0] - 1.] - - # Our default SciPy optimization algorithm, L-BFGS-B, does not support - # general constraints. Thus we use SLSQP instead. - optimizer = ScipyOptimizerInterface( - loss, equalities=equalities, inequalities=inequalities, method='SLSQP') - - with tf.compat.v1.Session() as session: - optimizer.minimize(session) - - # The value of vector should now be [1., 1.]. - ``` - """ - - _DEFAULT_METHOD = 'L-BFGS-B' - - def _minimize(self, initial_val, loss_grad_func, equality_funcs, - equality_grad_funcs, inequality_funcs, inequality_grad_funcs, - packed_bounds, step_callback, optimizer_kwargs): - - def loss_grad_func_wrapper(x): - # SciPy's L-BFGS-B Fortran implementation requires gradients as doubles. - loss, gradient = loss_grad_func(x) - return loss, gradient.astype('float64') - - optimizer_kwargs = dict(optimizer_kwargs.items()) - method = optimizer_kwargs.pop('method', self._DEFAULT_METHOD) - - constraints = [] - for func, grad_func in zip(equality_funcs, equality_grad_funcs): - constraints.append({'type': 'eq', 'fun': func, 'jac': grad_func}) - for func, grad_func in zip(inequality_funcs, inequality_grad_funcs): - constraints.append({'type': 'ineq', 'fun': func, 'jac': grad_func}) - - minimize_args = [loss_grad_func_wrapper, initial_val] - minimize_kwargs = { - 'jac': True, - 'callback': step_callback, - 'method': method, - 'constraints': constraints, - 'bounds': packed_bounds, - } - - for kwarg in minimize_kwargs: - if kwarg in optimizer_kwargs: - if kwarg == 'bounds': - # Special handling for 'bounds' kwarg since ability to specify bounds - # was added after this module was already publicly released. - raise ValueError( - 'Bounds must be set using the var_to_bounds argument') - raise ValueError( - 'Optimizer keyword arg \'{}\' is set ' - 'automatically and cannot be injected manually'.format(kwarg)) - - minimize_kwargs.update(optimizer_kwargs) - - import scipy.optimize # pylint: disable=g-import-not-at-top - result = scipy.optimize.minimize(*minimize_args, **minimize_kwargs) - - message_lines = [ - 'Optimization terminated with:', - ' Message: %s', - ' Objective function value: %f', - ] - message_args = [result.message, result.fun] - if hasattr(result, 'nit'): - # Some optimization methods might not provide information such as nit and - # nfev in the return. Logs only available information. - message_lines.append(' Number of iterations: %d') - message_args.append(result.nit) - if hasattr(result, 'nfev'): - message_lines.append(' Number of functions evaluations: %d') - message_args.append(result.nfev) - logging.info('\n'.join(message_lines), *message_args) - - return result['x'] - - -def _accumulate(list_): - total = 0 - yield total - for x in list_: - total += x - yield total - - -def _get_shape_tuple(tensor): - return tuple(tensor.get_shape().as_list()) - - -def _prod(array): - prod = 1 - for value in array: - prod *= value - return prod - - -def _compute_gradients(tensor, var_list): - grads = gradients.gradients(tensor, var_list) - # tf.gradients sometimes returns `None` when it should return 0. - return [ - grad if grad is not None else array_ops.zeros_like(var) - for var, grad in zip(var_list, grads) - ] diff --git a/tensorflow/contrib/opt/python/training/external_optimizer_test.py b/tensorflow/contrib/opt/python/training/external_optimizer_test.py deleted file mode 100644 index 70c5f8ff19c..00000000000 --- a/tensorflow/contrib/opt/python/training/external_optimizer_test.py +++ /dev/null @@ -1,343 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for external_optimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.opt.python.training import external_optimizer -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - -# pylint: disable=g-import-not-at-top,unused-import -try: - import __builtin__ as builtins -except ImportError: - import builtins - - -class MockOptimizerInterface(external_optimizer.ExternalOptimizerInterface): - - NUM_STEP_CALLS = 5 - NUM_LOSS_CALLS = 2 - - def _minimize(self, initial_val, loss_grad_func, step_callback, - optimizer_kwargs, **unused_kwargs): - """Minimize (x - x0)**2 / 2 with respect to x.""" - for _ in range(self.NUM_LOSS_CALLS): - loss_grad_func(initial_val) - for _ in range(self.NUM_STEP_CALLS): - step_callback(initial_val) - - _, grad = loss_grad_func(initial_val) - return initial_val - grad - - -class TestCase(test.TestCase): - - def assertAllClose(self, array1, array2, rtol=1e-5, atol=1e-5): - array1 = np.asarray(array1) - array2 = np.asarray(array2) - if not array1.shape: - array1 = np.array([array1]) - if not array2.shape: - array2 = np.array([array2]) - - super(TestCase, self).assertAllClose(array1, array2, rtol=rtol, atol=atol) - - -class ExternalOptimizerInterfaceTest(TestCase): - - def test_optimize(self): - scalar = variables.VariableV1(random_ops.random_normal([]), 'scalar') - vector = variables.VariableV1(random_ops.random_normal([2]), 'vector') - matrix = variables.VariableV1(random_ops.random_normal([2, 3]), 'matrix') - - minimum_location = constant_op.constant(np.arange(9), dtype=dtypes.float32) - - loss = math_ops.reduce_sum( - math_ops.square(vector - minimum_location[:2])) / 2. - loss += math_ops.reduce_sum( - math_ops.square(scalar - minimum_location[2])) / 2. - loss += math_ops.reduce_sum( - math_ops.square( - matrix - array_ops.reshape(minimum_location[3:], [2, 3]))) / 2. - - optimizer = MockOptimizerInterface(loss) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - - optimizer.minimize(sess) - - self.assertAllClose(np.arange(2), sess.run(vector)) - self.assertAllClose(np.arange(1) + 2, sess.run(scalar)) - self.assertAllClose(np.arange(6).reshape(2, 3) + 3, sess.run(matrix)) - - def test_callbacks(self): - vector_val = np.array([7., -2.], dtype=np.float32) - vector = variables.VariableV1(vector_val, 'vector') - - minimum_location_val = np.arange(2) - minimum_location = constant_op.constant( - minimum_location_val, dtype=dtypes.float32) - - loss = math_ops.reduce_sum(math_ops.square(vector - minimum_location)) / 2. - loss_val = ((vector_val - minimum_location_val)**2).sum() / 2. - - optimizer = MockOptimizerInterface(loss) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - - initial_vector_val = sess.run(vector) - - extra_fetches = [loss] - - step_callback = test.mock.Mock() - loss_callback = test.mock.Mock() - - optimizer.minimize( - sess, - fetches=extra_fetches, - loss_callback=loss_callback, - step_callback=step_callback) - - call = test.mock.call(loss_val) - loss_calls = [call] * MockOptimizerInterface.NUM_LOSS_CALLS - loss_callback.assert_has_calls(loss_calls) - - args, _ = step_callback.call_args - self.assertAllClose(initial_vector_val, args[0]) - - -class ScipyOptimizerInterfaceTest(TestCase): - - def _objective(self, x): - """Rosenbrock function. (Carl Edward Rasmussen, 2001-07-21). - - f(x) = sum_{i=1:D-1} 100*(x(i+1) - x(i)^2)^2 + (1-x(i))^2 - - Args: - x: a Variable - Returns: - f: a tensor (objective value) - """ - - d = array_ops.size(x) - s = math_ops.add( - 100 * math_ops.square( - math_ops.subtract( - array_ops.strided_slice(x, [1], [d]), - math_ops.square(array_ops.strided_slice(x, [0], [d - 1])))), - math_ops.square( - math_ops.subtract(1.0, array_ops.strided_slice(x, [0], [d - 1])))) - return math_ops.reduce_sum(s) - - def _test_optimization_method(self, - method, - options, - rtol=1e-5, - atol=1e-5, - dimension=5): - x = variables.VariableV1(array_ops.zeros(dimension)) - optimizer = external_optimizer.ScipyOptimizerInterface( - self._objective(x), method=method, options=options) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - optimizer.minimize(sess) - - self.assertAllClose(np.ones(dimension), sess.run(x), rtol=rtol, atol=atol) - - def test_unconstrained(self): - - dimension = 5 - x = variables.VariableV1(array_ops.zeros(dimension)) - optimizer = external_optimizer.ScipyOptimizerInterface(self._objective(x)) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - optimizer.minimize(sess) - - self.assertAllClose(np.ones(dimension), sess.run(x)) - - def test_nelder_mead_method2(self): - self._test_optimization_method( - method='Nelder-Mead', options={}, rtol=1e-4, atol=1e-4) - - def test_newton_cg_method(self): - self._test_optimization_method( - method='Newton-CG', - options={'eps': 1e-03, - 'xtol': 1e-05}, - rtol=1e-3, - atol=1e-3) - - def test_newton_tnc_method(self): - self._test_optimization_method( - method='TNC', - options={'gtol': -5, - 'maxiter': 1000}, - rtol=1e-1, - atol=1e-1) - - def test_cobyla_method(self): - # COBYLA does not reach the global optima - self._test_optimization_method( - method='COBYLA', - options={ - 'maxiter': 9000, - }, - rtol=1e-1, - atol=1e-1, - dimension=2) - - def test_slsqp_method(self): - self._test_optimization_method( - method='SLSQP', options={}, rtol=1e-3, atol=1e-3) - - def test_cg_method(self): - self._test_optimization_method( - method='CG', options={'gtol': 1e-03}, rtol=1e-3, atol=1e-3) - - def test_other_optimization_methods(self): - # These methods do not require special options to converge on rosenbrock - methods = ['Powell', 'BFGS', 'L-BFGS-B'] - - for method in methods: - self._test_optimization_method(method=method, options={}) - - def test_nonlinear_programming(self): - vector_initial_value = [7., 7.] - vector = variables.VariableV1(vector_initial_value, 'vector') - - # Make norm as small as possible. - loss = math_ops.reduce_sum(math_ops.square(vector)) - # Ensure y = 1. - equalities = [vector[1] - 1.] - # Ensure x >= 1. Thus optimum should be at (1, 1). - inequalities = [vector[0] - 1.] - - optimizer = external_optimizer.ScipyOptimizerInterface( - loss, equalities=equalities, inequalities=inequalities, method='SLSQP') - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - optimizer.minimize(sess) - self.assertAllClose(np.ones(2), sess.run(vector)) - - def test_scalar_bounds(self): - vector_initial_value = [7., 7.] - vector = variables.VariableV1(vector_initial_value, 'vector') - - # Make norm as small as possible. - loss = math_ops.reduce_sum(math_ops.square(vector)) - - # Make the minimum value of each component be 1. - var_to_bounds = {vector: (1., np.infty)} - - optimizer = external_optimizer.ScipyOptimizerInterface( - loss, var_to_bounds=var_to_bounds) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - optimizer.minimize(sess) - self.assertAllClose(np.ones(2), sess.run(vector)) - - def test_vector_bounds(self): - vector_initial_value = [7., 7.] - vector = variables.VariableV1(vector_initial_value, 'vector') - - # Make norm as small as possible. - loss = math_ops.reduce_sum(math_ops.square(vector)) - - var_to_bounds = {vector: ([None, 2.], None)} - - optimizer = external_optimizer.ScipyOptimizerInterface( - loss, var_to_bounds=var_to_bounds) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - optimizer.minimize(sess) - self.assertAllClose([0., 2.], sess.run(vector)) - - def test_optimizer_kwargs(self): - # Checks that the 'method' argument is stil present - # after running optimizer.minimize(). - # Bug reference: b/64065260 - vector_initial_value = [7., 7.] - vector = variables.VariableV1(vector_initial_value, 'vector') - loss = math_ops.reduce_sum(math_ops.square(vector)) - - optimizer = external_optimizer.ScipyOptimizerInterface( - loss, method='SLSQP') - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - optimizer.minimize(sess) - method = optimizer.optimizer_kwargs.get('method') - self.assertEqual('SLSQP', method) - - def test_callbacks(self): - vector_val = np.array([7., -2.], dtype=np.float32) - vector = variables.VariableV1(vector_val, 'vector') - - minimum_location_val = np.arange(2) - minimum_location = constant_op.constant( - minimum_location_val, dtype=dtypes.float32) - - loss = math_ops.reduce_sum(math_ops.square(vector - minimum_location)) / 2. - loss_val_first = ((vector_val - minimum_location_val)**2).sum() / 2. - - optimizer = external_optimizer.ScipyOptimizerInterface(loss, method='SLSQP') - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - - initial_vector_val = sess.run(vector) - - extra_fetches = [loss] - - step_callback = test.mock.Mock() - loss_callback = test.mock.Mock() - - optimizer.minimize( - sess, - fetches=extra_fetches, - loss_callback=loss_callback, - step_callback=step_callback) - - loss_val_last = sess.run(loss) - - call_first = test.mock.call(loss_val_first) - call_last = test.mock.call(loss_val_last) - loss_calls = [call_first, call_last] - loss_callback.assert_has_calls(loss_calls, any_order=True) - - args, _ = step_callback.call_args - self.assertAllClose(minimum_location_val, args[0]) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/opt/python/training/ggt.py b/tensorflow/contrib/opt/python/training/ggt.py deleted file mode 100644 index df0cb2b0071..00000000000 --- a/tensorflow/contrib/opt/python/training/ggt.py +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""GGT for Tensorflow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import numpy as np -from tensorflow.contrib.optimizer_v2 import optimizer_v2 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops - - -class GGTOptimizer(optimizer_v2.OptimizerV2): - """Optimizer that implements the GGT algorithm. - - GGT has an advantage over sgd and adam on large models with poor conditioning, - for example language models and CNNs, - see [[ABCHSZZ 2018]](https://arxiv.org/pdf/1806.02958.pdf). - """ - - def __init__(self, - learning_rate=0.001, - beta1=0.9, - use_locking=False, - name="GGT", - window=10, - eps=1e-4, - svd_eps=1e-6, - sigma_eps=1e-2): - """Construct a new GGT optimizer. - - Initialization: - - ``` - t <- 0 (Initialize timestep) - grad_buffer <- 0 (Initialize buffer for keeping past gradients) - flat_grad <- 0 (Initialize flattened gradient that contains gradients of all - variables) - m_0 <- 0 (Initialize 1st moment vector) - ``` - - Suppose all variables and their gradients are concatenated into vectors - `flat_vars` and `flat_grad`. The update rule for `flat_vars` - uses an optimization described at the beginning of section 2 of the paper: - - ``` - t <- t + 1 - - m_t <- beta1 * m_{t-1} + (1 - beta1) * flat_grad - grad_buffer[(t-1) % window, :] <- m_t - - M <- grad_buffer^T / sqrt(min(t, window)) - U, sigma, _ <- SVD(M^TM + I * svd_eps) - - sigma_sqrt_inv <- (sqrt(sigma) + sigma_eps)^(-3) - sigma_sqrt_min <- min(sqrt(sigma)) - - if sigma_sqrt_min > eps: - new_step <- M U diag(sigma_sqrt_inv) U^T M^T m_t + - (m_t - M U diag(1/sigma) U^T M^T m_t) / sigma_sqrt_min - else: - new_step <- M U diag(sigma_sqrt_inv) U^T M^T m_t - - flat_vars <- flat_vars - learning_rate * new_step - ``` - - GGT provides the power of full-matrix adaptive regularization at a cost not - much larger than SGD. As a result it is suited for large models where the - gradient covariance matrix has a poor condition number that slows down first - order methods. - GGT uses the preconditioner from full-matrix AdaGrad, with gradient history - attenuated exponentially as in Adam, and truncated to a window parameter. - It has provable guarantees even for non-convex optimization that is never - significantly worse than SGD and in some cases better. - - Args: - learning_rate: A float hyperparameter. The learning rate. - beta1: A float hyperparameter. The exponential decay rate for the 1st - moment estimates. - use_locking: If True use locks for update operations. - name: Optional name for the operations created when applying gradients. - Defaults to "GGT". - window: An integer hyperparameter. The number of first moments to keep in - computing the adaptive preconditioner. - eps: A float hyperparameter. Used to truncate small eigenvalues of the - gradient covariance matrix. - svd_eps: A float hyperparameter. Used to stabilize SVD. - sigma_eps: A float hyperparameter. Used to regularize matrix inversion. - """ - super(GGTOptimizer, self).__init__(use_locking, name) - self._set_hyper("lr", learning_rate) - self._set_hyper("beta1", beta1) - self._set_hyper("window", window) - self._set_hyper("eps", eps) - self._set_hyper("svd_eps", svd_eps) - self._set_hyper("sigma_eps", sigma_eps) - - self.index_dict = {} - self.shape_dict = {} - - def _create_vars(self, var_list, state): - # Construct ordered dictionary for variable dimensions, sorted by name. - shape_dict = {} - for v in var_list: - shape_dict[v.name] = tensor_shape.dimension_value(np.prod(v.get_shape())) - self.shape_dict = collections.OrderedDict( - sorted(shape_dict.items(), key=lambda t: t[0])) - - # Assign each variable its location in flat_grad. The locations are based on - # the order of sorted names. - idx = 0 - for v_name, v_dim in self.shape_dict.items(): - self.index_dict[v_name] = idx - idx += v_dim - - state.create_non_slot( - initial_value=math_ops.cast(0., dtype=var_list[0].dtype.base_dtype), - name="global_step") - - # Buffer for keeping past gradients. - window = state.get_hyper("window") - grad_buffer_init = array_ops.zeros( - [window, idx], dtype=var_list[0].dtype.base_dtype) - state.create_non_slot(initial_value=grad_buffer_init, name="grad_buffer") - - state.create_non_slot( - initial_value=array_ops.zeros( - (idx,), dtype=var_list[0].dtype.base_dtype), - name="moment1") - - # Flattened gradient that contains gradients for all variables in the model. - state.create_non_slot( - initial_value=array_ops.zeros( - (idx,), dtype=var_list[0].dtype.base_dtype), - name="flat_grad") - - def _get_global_step(self, state=None): - if state is None: - state = self._get_per_graph_state() - return state.get_non_slot("global_step") - - def _get_moment1(self, state=None): - if state is None: - state = self._get_per_graph_state() - return state.get_non_slot("moment1") - - def _get_grad_buffer(self, state=None): - if state is None: - state = self._get_per_graph_state() - return state.get_non_slot("grad_buffer") - - def _get_flat_grad(self, state=None): - if state is None: - state = self._get_per_graph_state() - return state.get_non_slot("flat_grad") - - def _apply_sparse(self, grad, var): - raise NotImplementedError("Sparse gradient updates are not supported.") - - def _prepare(self, state): - self._variables = [] - - def _apply_dense(self, grad, var, state): - self._variables.append(var) - dim = self.shape_dict[var.name] - start_index = self.index_dict[var.name] - end_index = start_index + dim - - # Update flat_gradient at the index associated with the variable. - flat_grad = self._get_flat_grad(state) - new_flat_grad = array_ops.reshape(grad, [-1]) - flat_grad_updated = state_ops.scatter_update( - flat_grad, math_ops.range(start_index, end_index), new_flat_grad) - - return flat_grad_updated - - def _resource_apply_dense(self, grad, var, state): - self._variables.append(var) - dim = self.shape_dict[var.name] - start_index = self.index_dict[var.name] - end_index = start_index + dim - - # Update flat_gradient at the index associated with the variable. - flat_grad = self._get_flat_grad(state) - new_flat_grad = array_ops.reshape(grad, [-1]) - flat_grad_updated = state_ops.scatter_update( - flat_grad, math_ops.range(start_index, end_index), new_flat_grad) - - return flat_grad_updated - - def _finish(self, state): - var_dtype = self._variables[0].dtype.base_dtype - # Update global step. - global_step = self._get_global_step(state) - update_global_step = state_ops.assign_add(global_step, 1.) - - # Update the first moment estimate. - beta1 = state.get_hyper("beta1", dtype=var_dtype) - moment1 = self._get_moment1(state) - flat_grad = self._get_flat_grad(state) - # moment1_t := beta1 * moment1_{t-1} + (1 - beta1) * flat_grad_t - update_moment1 = moment1.assign(beta1 * moment1 + (1. - beta1) * flat_grad) - - # Update the gradient buffer. - window = state.get_hyper("window") - grad_buffer = self._get_grad_buffer(state) - next_grad_index = math_ops.floormod( - math_ops.cast(update_global_step - 1., dtypes.int32), window) - # grad_buffer[(t-1) % window] := moment1_t - update_grad_buffer = state_ops.scatter_update(grad_buffer, next_grad_index, - update_moment1) - - # Compute the update step. - eps = state.get_hyper("eps", dtype=var_dtype) - svd_eps = state.get_hyper("svd_eps", dtype=var_dtype) - sigma_eps = state.get_hyper("sigma_eps", dtype=var_dtype) - lr = state.get_hyper("lr", dtype=var_dtype) - denom = math_ops.sqrt( - math_ops.minimum( - ops.convert_to_tensor(update_global_step), - ops.convert_to_tensor(math_ops.cast(window, dtype=var_dtype)))) - moment1_2d = array_ops.expand_dims(update_moment1, -1) - - # m = grad_buffer^T / sqrt(min(t, window)) - # m has shape [model dimension, window], where model dimension is the sum - # of the dimensions of the flattened variables. - m = array_ops.transpose(math_ops.divide(update_grad_buffer, denom)) - - # sigma, u, _ = SVD(m^Tm + I * svd_eps) - mm = math_ops.matmul(m, m, transpose_a=True) - damping = math_ops.cast(linalg_ops.eye(window), dtype=var_dtype) * svd_eps - sigma, u, _ = linalg_ops.svd(mm + damping) - sigma_sqrt = math_ops.sqrt(sigma) - sigma_sqrt_min = math_ops.reduce_min(sigma_sqrt) - - # sigma_sqrt_inv = 1 / (\sqrt{sigma} + sigma_eps) ^ 3 - # We add sigma_eps to alleviate numerical instability. - # Note that (m^Tm)^(-3/2) = u diag(sigma_sqrt_inv) u^T. - sigma_sqrt_inv = math_ops.divide( - math_ops.cast(1.0, dtype=var_dtype), - math_ops.pow(sigma_sqrt + sigma_eps, 3)) - - # In full matrix AdaGrad, the update step computes (mm^T)^(-1/2)g, where the - # inversion of a model dimension by model dimension matrix is needed. To - # speed up this computation we calculate the following instead: - # m(m^Tm)^(-3/2)m^T moment1 = m u diag(sigma_sqrt_inv) u^T m^T moment1. - new_step = array_ops.expand_dims( - array_ops.zeros(flat_grad.get_shape(), dtype=var_dtype), -1) - head = math_ops.matmul( - m, - math_ops.matmul( - u, - math_ops.matmul( - array_ops.diag(sigma_sqrt_inv), - math_ops.matmul( - u, - math_ops.matmul(m, moment1_2d, transpose_a=True), - transpose_a=True)))) - - # When inverting (mm^t)^(1/2), we also add epsilon * I regularization for - # degenerate cases. We expand ((mm^t)^(1/2) + epsilon * I)^(-1) using - # Woodbury's identity. - # For full derivation please see paper at - # https://arxiv.org/pdf/1806.02958.pdf - tail = moment1_2d - math_ops.matmul( - m, - math_ops.matmul( - u, - math_ops.matmul( - array_ops.diag( - math_ops.divide(math_ops.cast(1.0, dtype=var_dtype), - sigma)), - math_ops.matmul( - u, - math_ops.matmul(m, moment1_2d, transpose_a=True), - transpose_a=True)))) - scaled_tail = math_ops.divide(tail, sigma_sqrt_min) - - update_new_step = control_flow_ops.cond( - sigma_sqrt_min > eps, lambda: math_ops.add(head, scaled_tail), - lambda: math_ops.add(new_step, head)) - - # Update each variable. - update_step = [] - for var in self._variables: - dim = self.shape_dict[var.name] - start_index = self.index_dict[var.name] - end_index = start_index + dim - var_update_correct_shape = array_ops.reshape( - update_new_step[start_index:end_index], var.get_shape()) - var_updated = state_ops.assign_sub(var, lr * var_update_correct_shape) - update_step.append(var_updated) - - return control_flow_ops.group(update_step) diff --git a/tensorflow/contrib/opt/python/training/ggt_test.py b/tensorflow/contrib/opt/python/training/ggt_test.py deleted file mode 100644 index 1775edabb33..00000000000 --- a/tensorflow/contrib/opt/python/training/ggt_test.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for GGTOptimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib.opt.python.training.ggt import GGTOptimizer -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def ggt_update_numpy(param, - g_t, - lr, - grad_buffer, - m, - window, - t, - beta1=0.9, - eps=1e-4, - svd_eps=1e-6, - sigma_eps=1e-2): - """Tests the correctness of one step of GGT.""" - m_t = m * beta1 + (1 - beta1) * g_t - grad_buffer[((t - 1) % window), :] = m_t - m_matrix = np.transpose(grad_buffer / np.sqrt(np.minimum(t, window))) - mm = np.dot(np.transpose(m_matrix), m_matrix) - damping = np.eye(window) * svd_eps - u, sigma, _ = np.linalg.svd(mm + damping) - - sigma_sqrt_inv = np.power(np.sqrt(sigma) + sigma_eps, -3) - new_step = np.linalg.multi_dot([ - m_matrix, u, - np.diag(sigma_sqrt_inv), - np.transpose(u), - np.transpose(m_matrix), m_t - ]) - - sigma_sqrt_min = np.sqrt(sigma).min() - - if sigma_sqrt_min > eps: - new_step += (m_t - np.linalg.multi_dot([ - m_matrix, u, - np.diag(1.0 / sigma), - np.transpose(u), - np.transpose(m_matrix), m_t - ])) * (1.0 / sigma_sqrt_min) - - param_t = param - lr * new_step - return param_t, m_t, grad_buffer - - -class GGTOptimizerTest(test.TestCase): - - def doTestBasic(self, use_resource=False): - # SVD does not support float16 - for i, dtype in enumerate([dtypes.float32, dtypes.float64]): - with self.session(graph=ops.Graph()): - # Initialize variables for numpy implementation. - m0 = 0.0 - window = 3 - grad_buffer = np.zeros((window, 4), dtype=dtype.as_numpy_dtype) - lr = 0.001 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable( - var0_np, name="var0_%d" % i) - var1 = resource_variable_ops.ResourceVariable( - var1_np, name="var1_%d" % i) - else: - var0 = variables.Variable(var0_np, name="var0") - var1 = variables.Variable(var1_np, name="var1") - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - opt = GGTOptimizer(learning_rate=lr, window=window) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - opt_variables = opt.variables() - - m_t = opt._get_moment1() - grad_buffer_t = opt._get_grad_buffer() - g_t = opt._get_flat_grad() - self.assertTrue(m_t is not None) - self.assertTrue(grad_buffer_t is not None) - self.assertTrue(g_t is not None) - self.assertIn(m_t, opt_variables) - self.assertIn(grad_buffer_t, opt_variables) - self.assertIn(g_t, opt_variables) - - with ops.Graph().as_default(): - # Shouldn't return non-slot variables from other graphs. - self.assertEqual(0, len(opt.variables())) - - if not context.executing_eagerly(): - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - m_t = opt._get_moment1() - grad_buffer_t = opt._get_grad_buffer() - g_t = opt._get_flat_grad() - - # Run 3 steps of GGT - for t in range(1, 4): - if not context.executing_eagerly(): - self.evaluate(update) - elif t > 1: - opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - - if t == 1: - self.assertAllCloseAccordingToType( - np.array([0.01, 0.01, 0.001, 0.001]), self.evaluate(m_t)) - self.assertAllCloseAccordingToType( - np.array([[0.01, 0.01, 0.001, 0.001], [0., 0., 0., 0.], - [0., 0., 0., 0.]]), self.evaluate(grad_buffer_t)) - elif t == 2: - self.assertAllCloseAccordingToType( - np.array([0.019, 0.019, 0.0019, 0.0019]), self.evaluate(m_t)) - self.assertAllCloseAccordingToType( - np.array([[0.01, 0.01, 0.001, 0.001], - [0.019, 0.019, 0.0019, 0.0019], [0., 0., 0., 0.]]), - self.evaluate(grad_buffer_t)) - else: - self.assertAllCloseAccordingToType( - np.array([0.0271, 0.0271, 0.00271, 0.00271]), - self.evaluate(m_t)) - self.assertAllCloseAccordingToType( - np.array([[0.01, 0.01, 0.001, - 0.001], [0.019, 0.019, 0.0019, 0.0019], - [0.0271, 0.0271, 0.00271, 0.00271]]), - self.evaluate(grad_buffer_t)) - - self.assertAllCloseAccordingToType([0.1, 0.1, 0.01, 0.01], - self.evaluate(g_t)) - - var_np = np.append(var0_np, var1_np) - grads_np = np.append(grads0_np, grads1_np) - var_np, m0, grad_buffer = ggt_update_numpy(var_np, grads_np, lr, - grad_buffer, m0, window, t) - - var0_np = var_np[:2] - var1_np = var_np[2:] - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - - def testBasic(self): - with self.cached_session(): - self.doTestBasic(use_resource=False) - - @test_util.run_in_graph_and_eager_modes(reset_test=True) - def testResourceBasic(self): - self.doTestBasic(use_resource=True) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/opt/python/training/lars_optimizer.py b/tensorflow/contrib/opt/python/training/lars_optimizer.py deleted file mode 100644 index 0c06f4d7f36..00000000000 --- a/tensorflow/contrib/opt/python/training/lars_optimizer.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Layer-wise Adaptive Rate Scaling optimizer for large-batch training.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.training import optimizer -from tensorflow.python.training import training_ops - - -class LARSOptimizer(optimizer.Optimizer): - """Layer-wise Adaptive Rate Scaling for large batch training. - - Introduced by "Large Batch Training of Convolutional Networks" by Y. You, - I. Gitman, and B. Ginsburg. (https://arxiv.org/abs/1708.03888) - - Implements the LARS learning rate scheme presented in the paper above. This - optimizer is useful when scaling the batch size to up to 32K without - significant performance degradation. It is recommended to use the optimizer - in conjunction with: - - Gradual learning rate warm-up - - Linear learning rate scaling - - Poly rule learning rate decay - - Note, LARS scaling is currently only enabled for dense tensors. Sparse tensors - use the default momentum optimizer. - """ - - def __init__( - self, - learning_rate, - momentum=0.9, - weight_decay=0.0001, - # The LARS coefficient is a hyperparameter - eeta=0.001, - epsilon=0.0, - name="LARSOptimizer", - # Enable skipping variables from LARS scaling. - # TODO(sameerkm): Enable a direct mechanism to pass a - # subset of variables to the optimizer. - skip_list=None, - use_nesterov=False): - """Construct a new LARS Optimizer. - - Args: - learning_rate: A `Tensor` or floating point value. The base learning rate. - momentum: A floating point value. Momentum hyperparameter. - weight_decay: A floating point value. Weight decay hyperparameter. - eeta: LARS coefficient as used in the paper. Dfault set to LARS - coefficient from the paper. (eeta / weight_decay) determines the highest - scaling factor in LARS. - epsilon: Optional epsilon parameter to be set in models that have very - small gradients. Default set to 0.0. - name: Optional name prefix for variables and ops created by LARSOptimizer. - skip_list: List of strings to enable skipping variables from LARS scaling. - If any of the strings in skip_list is a subset of var.name, variable - 'var' is skipped from LARS scaling. For a typical classification model - with batch normalization, the skip_list is ['batch_normalization', - 'bias'] - use_nesterov: when set to True, nesterov momentum will be enabled - - Raises: - ValueError: If a hyperparameter is set to a non-sensical value. - """ - if momentum < 0.0: - raise ValueError("momentum should be positive: %s" % momentum) - if weight_decay < 0.0: - raise ValueError("weight_decay should be positive: %s" % weight_decay) - super(LARSOptimizer, self).__init__(use_locking=False, name=name) - - self._learning_rate = learning_rate - self._momentum = momentum - self._weight_decay = weight_decay - self._eeta = eeta - self._epsilon = epsilon - self._name = name - self._skip_list = skip_list - self._use_nesterov = use_nesterov - - def _create_slots(self, var_list): - for v in var_list: - self._zeros_slot(v, "momentum", self._name) - - def compute_lr(self, grad, var): - scaled_lr = self._learning_rate - if self._skip_list is None or not any(v in var.name - for v in self._skip_list): - w_norm = linalg_ops.norm(var, ord=2) - g_norm = linalg_ops.norm(grad, ord=2) - trust_ratio = array_ops.where( - math_ops.greater(w_norm, 0), - array_ops.where( - math_ops.greater(g_norm, 0), - (self._eeta * w_norm / - (g_norm + self._weight_decay * w_norm + self._epsilon)), 1.0), - 1.0) - scaled_lr = self._learning_rate * trust_ratio - # Add the weight regularization gradient - grad = grad + self._weight_decay * var - return scaled_lr, grad - - def _apply_dense(self, grad, var): - scaled_lr, grad = self.compute_lr(grad, var) - mom = self.get_slot(var, "momentum") - return training_ops.apply_momentum( - var, - mom, - math_ops.cast(1.0, var.dtype.base_dtype), - grad * scaled_lr, - self._momentum, - use_locking=False, - use_nesterov=self._use_nesterov) - - def _resource_apply_dense(self, grad, var): - scaled_lr, grad = self.compute_lr(grad, var) - mom = self.get_slot(var, "momentum") - return training_ops.resource_apply_momentum( - var.handle, - mom.handle, - math_ops.cast(1.0, var.dtype.base_dtype), - grad * scaled_lr, - self._momentum, - use_locking=False, - use_nesterov=self._use_nesterov) - - # Fallback to momentum optimizer for sparse tensors - def _apply_sparse(self, grad, var): - mom = self.get_slot(var, "momentum") - return training_ops.sparse_apply_momentum( - var, - mom, - math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype), - grad.values, - grad.indices, - math_ops.cast(self._momentum_tensor, var.dtype.base_dtype), - use_locking=self._use_locking, - use_nesterov=self._use_nesterov).op - - def _resource_apply_sparse(self, grad, var, indices): - mom = self.get_slot(var, "momentum") - return training_ops.resource_sparse_apply_momentum( - var.handle, - mom.handle, - math_ops.cast(self._learning_rate_tensor, grad.dtype), - grad, - indices, - math_ops.cast(self._momentum_tensor, grad.dtype), - use_locking=self._use_locking, - use_nesterov=self._use_nesterov) - - def _prepare(self): - learning_rate = self._learning_rate - if callable(learning_rate): - learning_rate = learning_rate() - self._learning_rate_tensor = ops.convert_to_tensor( - learning_rate, name="learning_rate") - momentum = self._momentum - if callable(momentum): - momentum = momentum() - self._momentum_tensor = ops.convert_to_tensor(momentum, name="momentum") diff --git a/tensorflow/contrib/opt/python/training/lars_optimizer_test.py b/tensorflow/contrib/opt/python/training/lars_optimizer_test.py deleted file mode 100644 index 8c135a21bc2..00000000000 --- a/tensorflow/contrib/opt/python/training/lars_optimizer_test.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0. Licensed to the Apache -# Software Foundation. You may not use this file except in compliance with the -# License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Test for Layer-wise Adaptive Rate Scaling optimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.opt.python.training import lars_optimizer as lo -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class LARSOptimizerTest(test.TestCase): - - def testLARSGradientOneStep(self): - for _ in range(10): - for dtype in [dtypes.float32, dtypes.float64]: - with self.cached_session() as sess: - shape = [3, 3] - var_np = np.ones(shape) - grad_np = np.ones(shape) - lr_np = 0.1 - m_np = 0.9 - wd_np = 0.1 - ep_np = 1e-5 - eeta = 0.1 - vel_np = np.zeros(shape) - - var = variables.Variable(var_np, dtype=dtype) - grad = variables.Variable(grad_np, dtype=dtype) - opt = lo.LARSOptimizer( - learning_rate=lr_np, - momentum=m_np, - weight_decay=wd_np, - eeta=eeta, - epsilon=ep_np) - - step = opt.apply_gradients([(grad, var)]) - variables.global_variables_initializer().run() - - pre_var = sess.run(var) - pre_vel = sess.run(opt.get_slot(var, 'momentum')) - self.assertAllClose(var_np, pre_var) - self.assertAllClose(vel_np, pre_vel) - - step.run() - post_var = sess.run(var) - post_vel = sess.run(opt.get_slot(var, 'momentum')) - - w_norm = np.linalg.norm(var_np.flatten(), ord=2) - g_norm = np.linalg.norm(grad_np.flatten(), ord=2) - trust_ratio = eeta * w_norm / (g_norm + wd_np * w_norm + ep_np) - scaled_lr = lr_np * trust_ratio - grad_np = grad_np + wd_np * var_np - - vel_np = m_np * vel_np + scaled_lr * grad_np - var_np -= vel_np - - self.assertAllClose(var_np, post_var) - self.assertAllClose(vel_np, post_vel) - - def testLARSGradientMultiStep(self): - for _ in range(10): - for dtype in [dtypes.float32, dtypes.float64]: - with self.cached_session() as sess: - shape = [3, 3] - var_np = np.ones(shape) - grad_np = np.ones(shape) - lr_np = 0.1 - m_np = 0.9 - wd_np = 0.1 - ep_np = 1e-5 - eeta = 0.1 - vel_np = np.zeros(shape) - - var = variables.Variable(var_np, dtype=dtype) - grad = variables.Variable(grad_np, dtype=dtype) - opt = lo.LARSOptimizer( - learning_rate=lr_np, - momentum=m_np, - eeta=eeta, - weight_decay=wd_np, - epsilon=ep_np) - - step = opt.apply_gradients([(grad, var)]) - variables.global_variables_initializer().run() - - pre_var = sess.run(var) - pre_vel = sess.run(opt.get_slot(var, 'momentum')) - self.assertAllClose(var_np, pre_var) - self.assertAllClose(vel_np, pre_vel) - - for _ in range(10): - step.run() - - post_var = sess.run(var) - post_vel = sess.run(opt.get_slot(var, 'momentum')) - - w_norm = np.linalg.norm(var_np.flatten(), ord=2) - g_norm = np.linalg.norm(grad_np.flatten(), ord=2) - trust_ratio = eeta * w_norm / (g_norm + wd_np * w_norm + ep_np) - scaled_lr = lr_np * trust_ratio - grad_np = grad_np + wd_np * var_np - - vel_np = m_np * vel_np + scaled_lr * grad_np - var_np -= vel_np - - self.assertAllClose(var_np, post_var) - self.assertAllClose(vel_np, post_vel) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/opt/python/training/lazy_adam_gs_optimizer.py b/tensorflow/contrib/opt/python/training/lazy_adam_gs_optimizer.py deleted file mode 100644 index 8827007e4d7..00000000000 --- a/tensorflow/contrib/opt/python/training/lazy_adam_gs_optimizer.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""LazyAdam rewrite to use global step for computing beta1 & beta2 accumulation. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.opt.python.training import adam_gs_optimizer -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops - - -class LazyAdamGSOptimizer(adam_gs_optimizer.AdamGSOptimizer): - """Variant of the Adam optimizer that handles sparse updates more efficiently. - - Branched from tf.contrib.opt.LazyAdamGSOptimizer. The only difference is to - pass global step for computing beta1 and beta2 accumulators, instead of having - optimizer keep its own independent beta1 and beta2 accumulators as non-slot - variables. - - The original Adam algorithm maintains two moving-average accumulators for - each trainable variable; the accumulators are updated at every step. - This class provides lazier handling of gradient updates for sparse variables. - It only updates moving-average accumulators for sparse variable indices that - appear in the current batch, rather than updating the accumulators for all - indices. Compared with the original Adam optimizer, it can provide large - improvements in model training throughput for some applications. However, it - provides slightly different semantics than the original Adam algorithm, and - may lead to different empirical results. - """ - - def _apply_sparse(self, grad, var): - beta1_power, beta2_power = self._get_beta_accumulators() - beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) - beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype) - lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) - beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype) - beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype) - epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype) - lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power)) - - # \\(m := beta1 * m + (1 - beta1) * g_t\\) - m = self.get_slot(var, "m") - m_t = state_ops.scatter_update(m, grad.indices, - beta1_t * array_ops.gather(m, grad.indices) + - (1 - beta1_t) * grad.values, - use_locking=self._use_locking) - - # \\(v := beta2 * v + (1 - beta2) * (g_t * g_t)\\) - v = self.get_slot(var, "v") - v_t = state_ops.scatter_update(v, grad.indices, - beta2_t * array_ops.gather(v, grad.indices) + - (1 - beta2_t) * math_ops.square(grad.values), - use_locking=self._use_locking) - - # \\(variable -= learning_rate * m_t / (epsilon_t + sqrt(v_t))\\) - m_t_slice = array_ops.gather(m_t, grad.indices) - v_t_slice = array_ops.gather(v_t, grad.indices) - denominator_slice = math_ops.sqrt(v_t_slice) + epsilon_t - var_update = state_ops.scatter_sub(var, grad.indices, - lr * m_t_slice / denominator_slice, - use_locking=self._use_locking) - return control_flow_ops.group(var_update, m_t, v_t) - - def _resource_apply_sparse(self, grad, var, indices): - beta1_power, beta2_power = self._get_beta_accumulators() - beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) - beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype) - lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) - beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype) - beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype) - epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype) - lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power)) - - # \\(m := beta1 * m + (1 - beta1) * g_t\\) - m = self.get_slot(var, "m") - m_t_slice = beta1_t * array_ops.gather(m, indices) + (1 - beta1_t) * grad - m_update_op = resource_variable_ops.resource_scatter_update(m.handle, - indices, - m_t_slice) - - # \\(v := beta2 * v + (1 - beta2) * (g_t * g_t)\\) - v = self.get_slot(var, "v") - v_t_slice = (beta2_t * array_ops.gather(v, indices) + - (1 - beta2_t) * math_ops.square(grad)) - v_update_op = resource_variable_ops.resource_scatter_update(v.handle, - indices, - v_t_slice) - - # \\(variable -= learning_rate * m_t / (epsilon_t + sqrt(v_t))\\) - var_slice = lr * m_t_slice / (math_ops.sqrt(v_t_slice) + epsilon_t) - var_update_op = resource_variable_ops.resource_scatter_sub(var.handle, - indices, - var_slice) - - return control_flow_ops.group(var_update_op, m_update_op, v_update_op) diff --git a/tensorflow/contrib/opt/python/training/lazy_adam_gs_optimizer_test.py b/tensorflow/contrib/opt/python/training/lazy_adam_gs_optimizer_test.py deleted file mode 100644 index bdc9a02a546..00000000000 --- a/tensorflow/contrib/opt/python/training/lazy_adam_gs_optimizer_test.py +++ /dev/null @@ -1,402 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for LazyAdamGSOptimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy as np - -from tensorflow.contrib.opt.python.training import lazy_adam_gs_optimizer -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def adam_update_numpy(param, - g_t, - t, - m, - v, - alpha=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8): - alpha_t = alpha * np.sqrt(1 - beta2**t) / (1 - beta1**t) - - m_t = beta1 * m + (1 - beta1) * g_t - v_t = beta2 * v + (1 - beta2) * g_t * g_t - - param_t = param - alpha_t * m_t / (np.sqrt(v_t) + epsilon) - return param_t, m_t, v_t - - -class LazyAdamGSOptimizerTest(test.TestCase, parameterized.TestCase): - - @parameterized.parameters([False, True]) - def testSparse(self, use_resource): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - global_step = resource_variable_ops.ResourceVariable( - array_ops.zeros([], dtypes.int64)) - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - else: - global_step = variables.Variable(array_ops.zeros([], dtypes.int64)) - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - - grads0_np_indices = np.array([0, 1], dtype=np.int32) - grads0 = ops.IndexedSlices( - constant_op.constant(grads0_np), - constant_op.constant(grads0_np_indices), constant_op.constant([2])) - grads1_np_indices = np.array([0, 1], dtype=np.int32) - grads1 = ops.IndexedSlices( - constant_op.constant(grads1_np), - constant_op.constant(grads1_np_indices), constant_op.constant([2])) - opt = lazy_adam_gs_optimizer.LazyAdamGSOptimizer( - global_step=global_step) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Run 3 steps of Adam - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) - update.run() - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - @parameterized.parameters([False, True]) - def testSparseDevicePlacement(self, use_resource): - for index_dtype in [dtypes.int32, dtypes.int64]: - with self.cached_session(force_gpu=test.is_gpu_available()): - # If a GPU is available, tests that all optimizer ops can be placed on - # it (i.e. they have GPU kernels). - if use_resource: - global_step = resource_variable_ops.ResourceVariable( - array_ops.zeros([], dtypes.int64)) - var = resource_variable_ops.ResourceVariable([[1.0], [2.0]]) - else: - global_step = variables.Variable(array_ops.zeros([], dtypes.int64)) - var = variables.Variable([[1.0], [2.0]]) - - indices = constant_op.constant([0, 1], dtype=index_dtype) - gathered_sum = math_ops.reduce_sum(array_ops.gather(var, indices)) - optimizer = lazy_adam_gs_optimizer.LazyAdamGSOptimizer( - global_step=global_step, learning_rate=3.0) - minimize_op = optimizer.minimize(gathered_sum, global_step=global_step) - variables.global_variables_initializer().run() - minimize_op.run() - - @parameterized.parameters([False, True]) - def testSparseRepeatedIndices(self, use_resource): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - if use_resource: - repeated_index_global_step = resource_variable_ops.ResourceVariable( - array_ops.zeros([], dtypes.int64)) - aggregated_global_step = resource_variable_ops.ResourceVariable( - array_ops.zeros([], dtypes.int64)) - repeated_index_update_var = resource_variable_ops.ResourceVariable( - [[1.0], [2.0]], dtype=dtype) - aggregated_update_var = resource_variable_ops.ResourceVariable( - [[1.0], [2.0]], dtype=dtype) - else: - repeated_index_global_step = variables.Variable( - array_ops.zeros([], dtypes.int64)) - aggregated_global_step = variables.Variable( - array_ops.zeros([], dtypes.int64)) - repeated_index_update_var = variables.Variable( - [[1.0], [2.0]], dtype=dtype) - aggregated_update_var = variables.Variable( - [[1.0], [2.0]], dtype=dtype) - - grad_repeated_index = ops.IndexedSlices( - constant_op.constant( - [0.1, 0.1], shape=[2, 1], dtype=dtype), - constant_op.constant([1, 1]), - constant_op.constant([2, 1])) - grad_aggregated = ops.IndexedSlices( - constant_op.constant( - [0.2], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), - constant_op.constant([2, 1])) - repeated_update_opt = lazy_adam_gs_optimizer.LazyAdamGSOptimizer( - global_step=repeated_index_global_step) - repeated_update = repeated_update_opt.apply_gradients( - [(grad_repeated_index, repeated_index_update_var)], - global_step=repeated_index_global_step) - aggregated_update_opt = lazy_adam_gs_optimizer.LazyAdamGSOptimizer( - global_step=aggregated_global_step) - aggregated_update = aggregated_update_opt.apply_gradients( - [(grad_aggregated, aggregated_update_var)], - global_step=aggregated_global_step) - variables.global_variables_initializer().run() - self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) - for _ in range(3): - repeated_update.run() - aggregated_update.run() - self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) - - def doTestBasic(self, use_resource=False, use_callable_params=False): - for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - with self.session(graph=ops.Graph()): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - global_step = resource_variable_ops.ResourceVariable( - array_ops.zeros([], dtypes.int64), name="global_step_%d" % i) - var0 = resource_variable_ops.ResourceVariable( - var0_np, name="var0_%d" % i) - var1 = resource_variable_ops.ResourceVariable( - var1_np, name="var1_%d" % i) - else: - global_step = variables.Variable(array_ops.zeros([], dtypes.int64)) - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - learning_rate = lambda: 0.001 - beta1 = lambda: 0.9 - beta2 = lambda: 0.999 - epsilon = lambda: 1e-8 - if not use_callable_params: - learning_rate = learning_rate() - beta1 = beta1() - beta2 = beta2() - epsilon = epsilon() - - opt = lazy_adam_gs_optimizer.LazyAdamGSOptimizer( - global_step=global_step, learning_rate=learning_rate) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - opt_variables = opt.variables() - beta1_power, beta2_power = opt._get_beta_accumulators() - self.assertIsNotNone(beta1_power) - self.assertIsNotNone(beta2_power is not None) - self.assertNotIn(beta1_power, opt_variables) - self.assertNotIn(beta2_power, opt_variables) - - if not context.executing_eagerly(): - with ops.Graph().as_default(): - # Shouldn't return non-slot variables from other graphs. - self.assertEqual(0, len(opt.variables())) - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - # Run 3 steps of Adam - for t in range(1, 4): - if not context.executing_eagerly(): - self.evaluate(update) - self.assertAllCloseAccordingToType( - 0.9**(t + 1), self.evaluate(beta1_power)) - self.assertAllCloseAccordingToType( - 0.999**(t + 1), self.evaluate(beta2_power)) - else: - if t > 1: - opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - beta1_power, beta2_power = opt._get_beta_accumulators() - self.assertAllCloseAccordingToType( - 0.9**t, self.evaluate(beta1_power)) - self.assertAllCloseAccordingToType( - 0.999**t, self.evaluate(beta2_power)) - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - if use_resource: - self.assertEqual("var0_%d/Adam:0" % (i,), - opt.get_slot(var=var0, name="m").name) - - def testBasic(self): - with self.cached_session(): - self.doTestBasic(use_resource=False) - - @test_util.run_in_graph_and_eager_modes(reset_test=True) - def testResourceBasic(self): - self.doTestBasic(use_resource=True) - - def testBasicCallableParams(self): - with context.eager_mode(): - self.doTestBasic(use_resource=True, use_callable_params=True) - - def testTensorLearningRate(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - global_step = variables.Variable(array_ops.zeros([], dtypes.int64)) - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - opt = lazy_adam_gs_optimizer.LazyAdamGSOptimizer( - global_step=global_step, learning_rate=constant_op.constant(0.001)) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Run 3 steps of Adam - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) - update.run() - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testSharing(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - global_step = variables.Variable(array_ops.zeros([], dtypes.int64)) - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - opt = lazy_adam_gs_optimizer.LazyAdamGSOptimizer( - global_step=global_step) - update1 = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - variables.global_variables_initializer().run() - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - # Run 3 steps of intertwined Adam1 and Adam2. - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) - if t % 2 == 0: - update1.run() - else: - update2.run() - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testTwoSessions(self): - optimizer = lazy_adam_gs_optimizer.LazyAdamGSOptimizer() - - with context.eager_mode(): - var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") - grads0 = constant_op.constant(np.array([0.1, 0.1])) - optimizer.apply_gradients([(grads0, var0)]) - - g = ops.Graph() - with g.as_default(): - with self.session(graph=g): - var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") - grads0 = constant_op.constant(np.array([0.1, 0.1])) - optimizer.apply_gradients([(grads0, var0)]) - - gg = ops.Graph() - with gg.as_default(): - with self.session(graph=gg): - var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") - grads0 = constant_op.constant(np.array([0.1, 0.1])) - - # If the optimizer saves any state not keyed by graph the following line - # fails. - optimizer.apply_gradients([(grads0, var0)]) - - def testSlotsUniqueEager(self): - with context.eager_mode(): - v1 = resource_variable_ops.ResourceVariable(1.) - v2 = resource_variable_ops.ResourceVariable(1.) - opt = lazy_adam_gs_optimizer.LazyAdamGSOptimizer(1.) - opt.minimize(lambda: v1 + v2) - # There should be two non-slot variables, and two unique slot variables - # for v1 and v2 respectively. - self.assertLen(set(opt.variables()), 4) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/opt/python/training/lazy_adam_optimizer.py b/tensorflow/contrib/opt/python/training/lazy_adam_optimizer.py deleted file mode 100644 index f55209ec49f..00000000000 --- a/tensorflow/contrib/opt/python/training/lazy_adam_optimizer.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Variant of the Adam optimizer that handles sparse updates more efficiently. - -Compared with the original Adam optimizer, the one in this file can provide a -large improvement in model training throughput for some applications. However, -it provides slightly different semantics than the original Adam algorithm, and -may lead to different empirical results. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.training import adam - - -class LazyAdamOptimizer(adam.AdamOptimizer): - """Variant of the Adam optimizer that handles sparse updates more efficiently. - - The original Adam algorithm maintains two moving-average accumulators for - each trainable variable; the accumulators are updated at every step. - This class provides lazier handling of gradient updates for sparse variables. - It only updates moving-average accumulators for sparse variable indices that - appear in the current batch, rather than updating the accumulators for all - indices. Compared with the original Adam optimizer, it can provide large - improvements in model training throughput for some applications. However, it - provides slightly different semantics than the original Adam algorithm, and - may lead to different empirical results. - """ - - def _apply_sparse(self, grad, var): - beta1_power, beta2_power = self._get_beta_accumulators() - beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) - beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype) - lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) - beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype) - beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype) - epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype) - lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power)) - - # \\(m := beta1 * m + (1 - beta1) * g_t\\) - m = self.get_slot(var, "m") - m_t = state_ops.scatter_update(m, grad.indices, - beta1_t * array_ops.gather(m, grad.indices) + - (1 - beta1_t) * grad.values, - use_locking=self._use_locking) - - # \\(v := beta2 * v + (1 - beta2) * (g_t * g_t)\\) - v = self.get_slot(var, "v") - v_t = state_ops.scatter_update(v, grad.indices, - beta2_t * array_ops.gather(v, grad.indices) + - (1 - beta2_t) * math_ops.square(grad.values), - use_locking=self._use_locking) - - # \\(variable -= learning_rate * m_t / (epsilon_t + sqrt(v_t))\\) - m_t_slice = array_ops.gather(m_t, grad.indices) - v_t_slice = array_ops.gather(v_t, grad.indices) - denominator_slice = math_ops.sqrt(v_t_slice) + epsilon_t - var_update = state_ops.scatter_sub(var, grad.indices, - lr * m_t_slice / denominator_slice, - use_locking=self._use_locking) - return control_flow_ops.group(var_update, m_t, v_t) - - def _resource_apply_sparse(self, grad, var, indices): - beta1_power, beta2_power = self._get_beta_accumulators() - beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) - beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype) - lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) - beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype) - beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype) - epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype) - lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power)) - - # \\(m := beta1 * m + (1 - beta1) * g_t\\) - m = self.get_slot(var, "m") - m_t_slice = beta1_t * array_ops.gather(m, indices) + (1 - beta1_t) * grad - m_update_op = resource_variable_ops.resource_scatter_update(m.handle, - indices, - m_t_slice) - - # \\(v := beta2 * v + (1 - beta2) * (g_t * g_t)\\) - v = self.get_slot(var, "v") - v_t_slice = (beta2_t * array_ops.gather(v, indices) + - (1 - beta2_t) * math_ops.square(grad)) - v_update_op = resource_variable_ops.resource_scatter_update(v.handle, - indices, - v_t_slice) - - # \\(variable -= learning_rate * m_t / (epsilon_t + sqrt(v_t))\\) - var_slice = lr * m_t_slice / (math_ops.sqrt(v_t_slice) + epsilon_t) - var_update_op = resource_variable_ops.resource_scatter_sub(var.handle, - indices, - var_slice) - - return control_flow_ops.group(var_update_op, m_update_op, v_update_op) diff --git a/tensorflow/contrib/opt/python/training/lazy_adam_optimizer_test.py b/tensorflow/contrib/opt/python/training/lazy_adam_optimizer_test.py deleted file mode 100644 index 65ad724b3c3..00000000000 --- a/tensorflow/contrib/opt/python/training/lazy_adam_optimizer_test.py +++ /dev/null @@ -1,365 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for LazyAdamOptimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy as np - -from tensorflow.contrib.opt.python.training import lazy_adam_optimizer -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def adam_update_numpy(param, - g_t, - t, - m, - v, - alpha=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8): - alpha_t = alpha * np.sqrt(1 - beta2**t) / (1 - beta1**t) - - m_t = beta1 * m + (1 - beta1) * g_t - v_t = beta2 * v + (1 - beta2) * g_t * g_t - - param_t = param - alpha_t * m_t / (np.sqrt(v_t) + epsilon) - return param_t, m_t, v_t - - -class AdamOptimizerTest(test.TestCase, parameterized.TestCase): - - @parameterized.parameters([False, True]) - def testSparse(self, use_resource): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - else: - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - - grads0_np_indices = np.array([0, 1], dtype=np.int32) - grads0 = ops.IndexedSlices( - constant_op.constant(grads0_np), - constant_op.constant(grads0_np_indices), constant_op.constant([2])) - grads1_np_indices = np.array([0, 1], dtype=np.int32) - grads1 = ops.IndexedSlices( - constant_op.constant(grads1_np), - constant_op.constant(grads1_np_indices), constant_op.constant([2])) - opt = lazy_adam_optimizer.LazyAdamOptimizer() - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Run 3 steps of Adam - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) - update.run() - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - @parameterized.parameters([False, True]) - def testSparseDevicePlacement(self, use_resource): - for index_dtype in [dtypes.int32, dtypes.int64]: - with self.cached_session(force_gpu=test.is_gpu_available()): - # If a GPU is available, tests that all optimizer ops can be placed on - # it (i.e. they have GPU kernels). - if use_resource: - var = resource_variable_ops.ResourceVariable([[1.0], [2.0]]) - else: - var = variables.Variable([[1.0], [2.0]]) - - indices = constant_op.constant([0, 1], dtype=index_dtype) - gathered_sum = math_ops.reduce_sum(array_ops.gather(var, indices)) - optimizer = lazy_adam_optimizer.LazyAdamOptimizer(3.0) - minimize_op = optimizer.minimize(gathered_sum) - variables.global_variables_initializer().run() - minimize_op.run() - - @parameterized.parameters([False, True]) - def testSparseRepeatedIndices(self, use_resource): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - if use_resource: - repeated_index_update_var = resource_variable_ops.ResourceVariable( - [[1.0], [2.0]], dtype=dtype) - aggregated_update_var = resource_variable_ops.ResourceVariable( - [[1.0], [2.0]], dtype=dtype) - else: - repeated_index_update_var = variables.Variable( - [[1.0], [2.0]], dtype=dtype) - aggregated_update_var = variables.Variable( - [[1.0], [2.0]], dtype=dtype) - - grad_repeated_index = ops.IndexedSlices( - constant_op.constant( - [0.1, 0.1], shape=[2, 1], dtype=dtype), - constant_op.constant([1, 1]), - constant_op.constant([2, 1])) - grad_aggregated = ops.IndexedSlices( - constant_op.constant( - [0.2], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), - constant_op.constant([2, 1])) - repeated_update_opt = lazy_adam_optimizer.LazyAdamOptimizer() - repeated_update = repeated_update_opt.apply_gradients( - [(grad_repeated_index, repeated_index_update_var)]) - aggregated_update_opt = lazy_adam_optimizer.LazyAdamOptimizer() - aggregated_update = aggregated_update_opt.apply_gradients( - [(grad_aggregated, aggregated_update_var)]) - variables.global_variables_initializer().run() - self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) - for _ in range(3): - repeated_update.run() - aggregated_update.run() - self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) - - def doTestBasic(self, use_resource=False, use_callable_params=False): - for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - with self.session(graph=ops.Graph()): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable( - var0_np, name="var0_%d" % i) - var1 = resource_variable_ops.ResourceVariable( - var1_np, name="var1_%d" % i) - else: - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - learning_rate = lambda: 0.001 - beta1 = lambda: 0.9 - beta2 = lambda: 0.999 - epsilon = lambda: 1e-8 - if not use_callable_params: - learning_rate = learning_rate() - beta1 = beta1() - beta2 = beta2() - epsilon = epsilon() - - opt = lazy_adam_optimizer.LazyAdamOptimizer(learning_rate=learning_rate) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - opt_variables = opt.variables() - beta1_power, beta2_power = opt._get_beta_accumulators() - self.assertIsNotNone(beta1_power) - self.assertIsNotNone(beta2_power is not None) - self.assertIn(beta1_power, opt_variables) - self.assertIn(beta2_power, opt_variables) - - if not context.executing_eagerly(): - with ops.Graph().as_default(): - # Shouldn't return non-slot variables from other graphs. - self.assertEqual(0, len(opt.variables())) - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Run 3 steps of Adam - for t in range(1, 4): - if not context.executing_eagerly(): - self.evaluate(update) - elif t > 1: - opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - - self.assertAllCloseAccordingToType(0.9**(t + 1), - self.evaluate(beta1_power)) - self.assertAllCloseAccordingToType(0.999**(t + 1), - self.evaluate(beta2_power)) - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - if use_resource: - self.assertEqual("var0_%d/Adam:0" % (i,), - opt.get_slot(var=var0, name="m").name) - - def testBasic(self): - with self.cached_session(): - self.doTestBasic(use_resource=False) - - @test_util.run_in_graph_and_eager_modes(reset_test=True) - def testResourceBasic(self): - self.doTestBasic(use_resource=True) - - def testBasicCallableParams(self): - with context.eager_mode(): - self.doTestBasic(use_resource=True, use_callable_params=True) - - def testTensorLearningRate(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - opt = lazy_adam_optimizer.LazyAdamOptimizer(constant_op.constant(0.001)) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Run 3 steps of Adam - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) - update.run() - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testSharing(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - opt = lazy_adam_optimizer.LazyAdamOptimizer() - update1 = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - # Run 3 steps of intertwined Adam1 and Adam2. - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) - if t % 2 == 0: - update1.run() - else: - update2.run() - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testTwoSessions(self): - optimizer = lazy_adam_optimizer.LazyAdamOptimizer() - - with context.eager_mode(): - var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") - grads0 = constant_op.constant(np.array([0.1, 0.1])) - optimizer.apply_gradients([(grads0, var0)]) - - g = ops.Graph() - with g.as_default(): - with self.session(graph=g): - var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") - grads0 = constant_op.constant(np.array([0.1, 0.1])) - optimizer.apply_gradients([(grads0, var0)]) - - gg = ops.Graph() - with gg.as_default(): - with self.session(graph=gg): - var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") - grads0 = constant_op.constant(np.array([0.1, 0.1])) - - # If the optimizer saves any state not keyed by graph the following line - # fails. - optimizer.apply_gradients([(grads0, var0)]) - - def testSlotsUniqueEager(self): - with context.eager_mode(): - v1 = resource_variable_ops.ResourceVariable(1.) - v2 = resource_variable_ops.ResourceVariable(1.) - opt = lazy_adam_optimizer.LazyAdamOptimizer(1.) - opt.minimize(lambda: v1 + v2) - # There should be two non-slot variables, and two unique slot variables - # for v1 and v2 respectively. - self.assertEqual(6, len(set(opt.variables()))) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/opt/python/training/matrix_functions.py b/tensorflow/contrib/opt/python/training/matrix_functions.py deleted file mode 100644 index 1c5d2fe1787..00000000000 --- a/tensorflow/contrib/opt/python/training/matrix_functions.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Matrix functions contains iterative methods for M^p.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops - - -def matrix_square_root(mat_a, mat_a_size, iter_count=100, ridge_epsilon=1e-4): - """Iterative method to get matrix square root. - - Stable iterations for the matrix square root, Nicholas J. Higham - - Page 231, Eq 2.6b - http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.6.8799&rep=rep1&type=pdf - - Args: - mat_a: the symmetric PSD matrix whose matrix square root be computed - mat_a_size: size of mat_a. - iter_count: Maximum number of iterations. - ridge_epsilon: Ridge epsilon added to make the matrix positive definite. - - Returns: - mat_a^0.5 - """ - - def _iter_condition(i, unused_mat_y, unused_old_mat_y, unused_mat_z, - unused_old_mat_z, err, old_err): - # This method require that we check for divergence every step. - return math_ops.logical_and(i < iter_count, err < old_err) - - def _iter_body(i, mat_y, unused_old_mat_y, mat_z, unused_old_mat_z, err, - unused_old_err): - current_iterate = 0.5 * (3.0 * identity - math_ops.matmul(mat_z, mat_y)) - current_mat_y = math_ops.matmul(mat_y, current_iterate) - current_mat_z = math_ops.matmul(current_iterate, mat_z) - # Compute the error in approximation. - mat_sqrt_a = current_mat_y * math_ops.sqrt(norm) - mat_a_approx = math_ops.matmul(mat_sqrt_a, mat_sqrt_a) - residual = mat_a - mat_a_approx - current_err = math_ops.sqrt(math_ops.reduce_sum(residual * residual)) / norm - return i + 1, current_mat_y, mat_y, current_mat_z, mat_z, current_err, err - - identity = linalg_ops.eye(math_ops.cast(mat_a_size, dtypes.int32)) - mat_a = mat_a + ridge_epsilon * identity - norm = math_ops.sqrt(math_ops.reduce_sum(mat_a * mat_a)) - mat_init_y = mat_a / norm - mat_init_z = identity - init_err = norm - - _, _, prev_mat_y, _, _, _, _ = control_flow_ops.while_loop( - _iter_condition, _iter_body, [ - 0, mat_init_y, mat_init_y, mat_init_z, mat_init_z, init_err, - init_err + 1.0 - ]) - return prev_mat_y * math_ops.sqrt(norm) - - -def matrix_inverse_pth_root(mat_g, - mat_g_size, - alpha, - iter_count=100, - epsilon=1e-6, - ridge_epsilon=1e-6): - """Computes mat_g^alpha, where alpha = -1/p, p a positive integer. - - We use an iterative Schur-Newton method from equation 3.2 on page 9 of: - - A Schur-Newton Method for the Matrix p-th Root and its Inverse - by Chun-Hua Guo and Nicholas J. Higham - SIAM Journal on Matrix Analysis and Applications, - 2006, Vol. 28, No. 3 : pp. 788-804 - https://pdfs.semanticscholar.org/0abe/7f77433cf5908bfe2b79aa91af881da83858.pdf - - Args: - mat_g: the symmetric PSD matrix whose power it to be computed - mat_g_size: size of mat_g. - alpha: exponent, must be -1/p for p a positive integer. - iter_count: Maximum number of iterations. - epsilon: accuracy indicator, useful for early termination. - ridge_epsilon: Ridge epsilon added to make the matrix positive definite. - - Returns: - mat_g^alpha - """ - - identity = linalg_ops.eye(math_ops.cast(mat_g_size, dtypes.int32)) - - def mat_power(mat_m, p): - """Computes mat_m^p, for p a positive integer. - - Power p is known at graph compile time, so no need for loop and cond. - Args: - mat_m: a square matrix - p: a positive integer - - Returns: - mat_m^p - """ - assert p == int(p) and p > 0 - power = None - while p > 0: - if p % 2 == 1: - power = math_ops.matmul(mat_m, power) if power is not None else mat_m - p //= 2 - mat_m = math_ops.matmul(mat_m, mat_m) - return power - - def _iter_condition(i, mat_m, _): - return math_ops.logical_and( - i < iter_count, - math_ops.reduce_max(math_ops.abs(mat_m - identity)) > epsilon) - - def _iter_body(i, mat_m, mat_x): - mat_m_i = (1 - alpha) * identity + alpha * mat_m - return (i + 1, math_ops.matmul(mat_power(mat_m_i, -1.0 / alpha), mat_m), - math_ops.matmul(mat_x, mat_m_i)) - - if mat_g_size == 1: - mat_h = math_ops.pow(mat_g + ridge_epsilon, alpha) - else: - damped_mat_g = mat_g + ridge_epsilon * identity - z = (1 - 1 / alpha) / (2 * linalg_ops.norm(damped_mat_g)) - # The best value for z is - # (1 - 1/alpha) * (c_max^{-alpha} - c_min^{-alpha}) / - # (c_max^{1-alpha} - c_min^{1-alpha}) - # where c_max and c_min are the largest and smallest singular values of - # damped_mat_g. - # The above estimate assumes that c_max > c_min * 2^p. (p = -1/alpha) - # Can replace above line by the one below, but it is less accurate, - # hence needs more iterations to converge. - # z = (1 - 1/alpha) / math_ops.trace(damped_mat_g) - # If we want the method to always converge, use z = 1 / norm(damped_mat_g) - # or z = 1 / math_ops.trace(damped_mat_g), but these can result in many - # extra iterations. - _, _, mat_h = control_flow_ops.while_loop( - _iter_condition, _iter_body, - [0, damped_mat_g * z, identity * math_ops.pow(z, -alpha)]) - return mat_h diff --git a/tensorflow/contrib/opt/python/training/matrix_functions_test.py b/tensorflow/contrib/opt/python/training/matrix_functions_test.py deleted file mode 100644 index 518fa382339..00000000000 --- a/tensorflow/contrib/opt/python/training/matrix_functions_test.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functional tests for Matrix functions.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.opt.python.training import matrix_functions -from tensorflow.python.platform import test - -TOLERANCE = 1e-3 - - -def np_power(mat_g, alpha): - """Computes mat_g^alpha for a square symmetric matrix mat_g.""" - - mat_u, diag_d, mat_v = np.linalg.svd(mat_g) - diag_d = np.power(diag_d, alpha) - return np.dot(np.dot(mat_u, np.diag(diag_d)), mat_v) - - -class MatrixFunctionTests(test.TestCase): - - def testMatrixSquareRootFunction(self): - """Tests for matrix square roots.""" - - size = 20 - mat_a = np.random.rand(size, size) - mat = np.dot(mat_a, mat_a.T) - expected_mat = np_power(mat, 0.5) - mat_root = matrix_functions.matrix_square_root(mat, size) - self.assertAllCloseAccordingToType( - expected_mat, mat_root, atol=TOLERANCE, rtol=TOLERANCE) - - def testMatrixInversePthRootFunction(self): - """Tests for matrix inverse pth roots.""" - - size = 20 - mat_a = np.random.rand(size, size) - mat = np.dot(mat_a, mat_a.T) - expected_mat = np_power(mat, -0.125) - mat_root = matrix_functions.matrix_inverse_pth_root(mat, size, -0.125) - self.assertAllCloseAccordingToType( - expected_mat, mat_root, atol=TOLERANCE, rtol=TOLERANCE) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/opt/python/training/model_average_optimizer.py b/tensorflow/contrib/opt/python/training/model_average_optimizer.py deleted file mode 100644 index 669d83a3a49..00000000000 --- a/tensorflow/contrib/opt/python/training/model_average_optimizer.py +++ /dev/null @@ -1,317 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Wrapper optimizer for Model Average.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.training import optimizer -from tensorflow.python.training import session_run_hook - -GLOBAL_VARIABLE_NAME = "global_center_variable" - - -class ModelAverageCustomGetter(object): - """Custom_getter class is used to do. - - 1. Change trainable variables to local collection and place them at worker - device - 2. Generate global variables - Notice that the class should be used with tf.replica_device_setter, - so that the global center variables and global step variable can be placed - at ps device. Besides, use 'tf.compat.v1.get_variable' instead of - 'tf.Variable' to - use this custom getter. - - For example, - ma_custom_getter = ModelAverageCustomGetter(worker_device) - with tf.device( - tf.compat.v1.train.replica_device_setter( - worker_device=worker_device, - ps_device="/job:ps/cpu:0", - cluster=cluster)), - tf.compat.v1.variable_scope('',custom_getter=ma_custom_getter): - hid_w = tf.compat.v1.get_variable( - initializer=tf.random.truncated_normal( - [IMAGE_PIXELS * IMAGE_PIXELS, FLAGS.hidden_units], - stddev=1.0 / IMAGE_PIXELS), - name="hid_w") - hid_b = - tf.compat.v1.get_variable(initializer=tf.zeros([FLAGS.hidden_units]), - name="hid_b") - """ - - def __init__(self, worker_device): - """Create a new `ModelAverageCustomGetter`. - - Args: - worker_device: String. Name of the `worker` job. - """ - self._worker_device = worker_device - self._local_2_global = {} - - def __call__(self, getter, name, trainable, collections, *args, **kwargs): - if trainable: - with ops.device(self._worker_device): - local_var = getter( - name, - trainable=True, - collections=[ops.GraphKeys.LOCAL_VARIABLES], - *args, - **kwargs) - - global_variable = variable_scope.variable( - name="%s/%s" % (GLOBAL_VARIABLE_NAME, name), - initial_value=local_var.initialized_value(), - trainable=False, - collections=[ops.GraphKeys.GLOBAL_VARIABLES]) - - self._local_2_global[local_var] = global_variable - return local_var - else: - kwargs["trainable"] = trainable - kwargs["collections"] = collections - if ops.GraphKeys.LOCAL_VARIABLES in collections: - with ops.device(self._worker_device): - return getter(name, *args, **kwargs) - else: - return getter(name, *args, **kwargs) - - -class ModelAverageOptimizer(optimizer.Optimizer): - """Wrapper optimizer that implements the Model Average algorithm. - - This is a sync optimizer. During the training, each worker will update - the local variables and maintains its own local_step, which starts from 0 - and is incremented by 1 after each update of local variables. Whenever the - interval_steps divides the local step, the local variables from all the - workers will be averaged and assigned to global center variables. Then the - local variables will be assigned by global center variables. - """ - - def __init__(self, - opt, - num_worker, - is_chief, - ma_custom_getter, - interval_steps=100, - use_locking=True, - name="ModelAverageOptimizer"): - """Construct a new model average optimizer. - - Args: - opt: The actual optimizer that will be used to update local variables - num_worker: The number of workers - is_chief: whether chief worker - ma_custom_getter: ModelAverageCustomGetter - interval_steps: An int point value to controls the frequency of the - average of local variables - use_locking: If True use locks for update operations - name: string. Optional name of the returned operation - """ - super(ModelAverageOptimizer, self).__init__(use_locking, name) - self._opt = opt - self._num_worker = num_worker - self._is_chief = is_chief - self._local_2_global = ma_custom_getter._local_2_global # pylint:disable=protected-access - self._interval_steps = interval_steps - self._accumulator_list = [] - self._chief_init_op = None - - self._local_step = variable_scope.get_variable( - initializer=0, - trainable=False, - collections=[ops.GraphKeys.LOCAL_VARIABLES], - name="local_step") - - self._opt._prepare() # pylint:disable=protected-access - - def compute_gradients(self, *args, **kwargs): - """Compute gradients of "loss" for the variables in "var_list". - - This simply wraps the compute_gradients() from the real optimizer. - - Args: - *args: Arguments for compute_gradients(). - **kwargs: Keyword arguments for compute_gradients(). - - Returns: - A list of (gradient, variable) pairs. - """ - return self._opt.compute_gradients(*args, **kwargs) - - def _local_vars_update(self, var_list): - """Get the update ops for the local variables in "var_list". - - Args: - var_list: Optional list or tuple of 'tf.Variable' to update - - Returns: - An update op - - Raises: - ValueError: if var_list is empty. - """ - if not var_list: - raise ValueError("The list of local_variables should not be empty") - update_ops = [] - global_center_vars = [self._local_2_global[var] for var in var_list] - for lvar, gvar in zip(var_list, global_center_vars): - with ops.device(lvar.device): - update_ops.append(state_ops.assign(lvar, gvar.read_value())) - return control_flow_ops.group(*(update_ops)) - - def apply_gradients(self, grads_and_vars, global_step=None, name=None): - """Apply gradients to variables. - - This contains most of the synchronization implementation and also wraps the - apply_gradients() from the real optimizer. The chief work updates global - variables. - - Args: - grads_and_vars: List of (gradient, variable) pairs as returned by - compute_gradients(). - global_step: Optional Variable to increment by one after the variables - have been updated. - name: Optional name for the returned operation. Default to the name - passed to the Optimizer constructor. - - Returns: - A conditional 'Operation' that update both local and global variables or - just local variables - - Raises: - ValueError: If the grads_and_vars is empty. - ValueError: If global step is not provided, the staleness cannot be - checked. - """ - - # update local variables - if not grads_and_vars: - raise ValueError("Must supply at least one variable") - if global_step is None: - raise ValueError("Global step is required") - - apply_updates = self._opt.apply_gradients(grads_and_vars) - with ops.control_dependencies([apply_updates]): - local_update = state_ops.assign_add( - self._local_step, 1, name="local_step_update").op - - # update global variables. - def _update_global_variables(): # pylint: disable=missing-docstring - local_vars = [v for g, v in grads_and_vars if g is not None] - global_vars = [self._local_2_global[v] for v in local_vars] - # sync queue - with ops.colocate_with(global_step): - sync_queue = data_flow_ops.FIFOQueue( - -1, [dtypes.bool], shapes=[[]], shared_name="sync_queue") - train_ops = [] - aggregated_vars = [] - with ops.name_scope(None, self._name + "/global"): - for var, gvar in zip(local_vars, global_vars): - # pylint: disable=protected-access - with ops.device(gvar.device): - if isinstance(var._ref(), ops.Tensor): - var_accum = data_flow_ops.ConditionalAccumulator( - var.dtype, - shape=var.get_shape(), - shared_name=gvar.name + "/var_accum") - train_ops.append( - var_accum.apply_grad(var._ref(), local_step=global_step)) - aggregated_vars.append(var_accum.take_grad(self._num_worker)) - else: - raise ValueError("Unknown local variable type!") - self._accumulator_list.append((var_accum, gvar.device)) - # chief worker updates global vars and enqueues tokens to the sync queue - if self._is_chief: - update_ops = [] - with ops.control_dependencies(train_ops): - for avg_var, gvar in zip(aggregated_vars, global_vars): - with ops.device(gvar.device): - update_ops.append(state_ops.assign(gvar, avg_var)) - with ops.device(global_step.device): - update_ops.append(state_ops.assign_add(global_step, 1)) - with ops.control_dependencies(update_ops), ops.device( - global_step.device): - tokens = array_ops.fill([self._num_worker - 1], - constant_op.constant(False)) - sync_op = sync_queue.enqueue_many(tokens) - else: - with ops.control_dependencies(train_ops), ops.device( - global_step.device): - sync_op = sync_queue.dequeue() - - with ops.control_dependencies([sync_op]): - local_update_op = self._local_vars_update(local_vars) - return local_update_op - - with ops.control_dependencies([local_update]): - condition = math_ops.equal( - math_ops.mod(self._local_step, self._interval_steps), 0) - conditional_update = control_flow_ops.cond(condition, - _update_global_variables, - control_flow_ops.no_op) - - chief_init_ops = [] - for accum, dev in self._accumulator_list: - with ops.device(dev): - chief_init_ops.append( - accum.set_global_step(global_step, name="SetGlobalStep")) - self._chief_init_op = control_flow_ops.group(*(chief_init_ops)) - - return conditional_update - - def get_init_op(self): - """Returns the op. - - This method lets all the local variables equal to the global - variables before the training begins. - """ - return self._local_vars_update(variables.trainable_variables()) - - def make_session_run_hook(self): - """Creates a hook to handle ModelAverage ops such as initialization.""" - return _ModelAverageOptimizerHook(self, self._is_chief) - - -class _ModelAverageOptimizerHook(session_run_hook.SessionRunHook): # pylint: disable=missing-docstring - - def __init__(self, ma_optimizer, is_chief): - """Creates hook to handle ModelAverageOptimizer initialization ops. - - Args: - ma_optimizer: `ModelAverageOptimizer` which this hook will initialize. - is_chief: `Bool`, whether is this a chief replica or not. - """ - self._ma_optimizer = ma_optimizer - self._is_chief = is_chief - - def begin(self): - self._local_init_op = variables.local_variables_initializer() - self._global_init_op = None - if self._is_chief: - self._global_init_op = variables.global_variables_initializer() - self._chief_init_op = self._ma_optimizer._chief_init_op # pylint: disable=protected-access - self._variable_init_op = self._ma_optimizer.get_init_op() diff --git a/tensorflow/contrib/opt/python/training/model_average_optimizer_test.py b/tensorflow/contrib/opt/python/training/model_average_optimizer_test.py deleted file mode 100644 index a25455e95d0..00000000000 --- a/tensorflow/contrib/opt/python/training/model_average_optimizer_test.py +++ /dev/null @@ -1,199 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for ModelAverageOptimizer.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import portpicker - -from tensorflow.contrib.opt.python.training import model_average_optimizer -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import device_setter -from tensorflow.python.training import gradient_descent -from tensorflow.python.training import server_lib -from tensorflow.python.training import training -from tensorflow.python.training import training_util - - -def create_local_cluster(num_workers, num_ps, protocol="grpc"): - """Create local GRPC servers and return them.""" - worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)] - ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)] - cluster_dict = { - "worker": ["localhost:%s" % port for port in worker_ports], - "ps": ["localhost:%s" % port for port in ps_ports] - } - cs = server_lib.ClusterSpec(cluster_dict) - - workers = [ - server_lib.Server( - cs, job_name="worker", protocol=protocol, task_index=ix, start=True) - for ix in range(num_workers) - ] - ps_servers = [ - server_lib.Server( - cs, job_name="ps", protocol=protocol, task_index=ix, start=True) - for ix in range(num_ps) - ] - - return cluster_dict, workers, ps_servers - - -# Creates the workers and return their sessions, graphs, train_ops. -# Chief worker will update at last -def _get_workers(num_workers, steps, workers): - sessions = [] - graphs = [] - train_ops = [] - for worker_id in range(num_workers): - graph = ops.Graph() - is_chief = (worker_id == 0) - with graph.as_default(): - worker_device = "/job:worker/task:%d/cpu:0" % (worker_id) - ma_coustom = model_average_optimizer.ModelAverageCustomGetter( - worker_device=worker_device) - with variable_scope.variable_scope( - "", custom_getter=ma_coustom), ops.device( - device_setter.replica_device_setter( - worker_device=worker_device, - ps_device="/job:ps/task:0/cpu:0", - ps_tasks=1)): - - global_step = variables.Variable(0, name="global_step", trainable=False) - var_0 = variable_scope.get_variable(initializer=0.0, name="v0") - var_1 = variable_scope.get_variable(initializer=1.0, name="v1") - - with ops.device("/job:worker/task:" + str(worker_id)): - if worker_id == 0: - grads_0 = constant_op.constant(-1.0) - grads_1 = constant_op.constant(-1.0) - else: - grads_0 = constant_op.constant(-2.0) - grads_1 = constant_op.constant(-2.0) - sgd_opt = gradient_descent.GradientDescentOptimizer(1.0) - opt = model_average_optimizer.ModelAverageOptimizer( - opt=sgd_opt, - num_worker=num_workers, - ma_custom_getter=ma_coustom, - is_chief=is_chief, - interval_steps=steps) - train_op = [ - opt.apply_gradients([[grads_0, var_0], [grads_1, var_1]], - global_step) - ] - ma_hook = opt.make_session_run_hook() - # Creates MonitoredSession - sess = training.MonitoredTrainingSession( - workers[worker_id].target, hooks=[ma_hook]) - - sessions.append(sess) - graphs.append(graph) - train_ops.append(train_op) - return sessions, graphs, train_ops - - -class ModelAverageOptimizerTest(test.TestCase): - - def _run(self, train_op, sess): - sess.run(train_op) - - def disabled_test1Workers2Period(self): - num_workers = 2 - steps = 2 - num_ps = 1 - _, workers, _ = create_local_cluster( - num_workers=num_workers, num_ps=num_ps) - - sessions, graphs, train_ops = _get_workers(num_workers, steps, workers) - - var_0 = graphs[0].get_tensor_by_name("v0:0") - var_1 = graphs[0].get_tensor_by_name("v1:0") - global_step = training_util.get_global_step(graphs[0]) - global_var_0 = graphs[0].get_tensor_by_name( - model_average_optimizer.GLOBAL_VARIABLE_NAME + "/v0:0") - global_var_1 = graphs[0].get_tensor_by_name( - model_average_optimizer.GLOBAL_VARIABLE_NAME + "/v1:0") - - # Verify the initialized value. - self.assertAllEqual(0.0, sessions[0].run(var_0)) - self.assertAllEqual(1.0, sessions[0].run(var_1)) - self.assertAllEqual(0.0, sessions[0].run(global_var_0)) - self.assertAllEqual(1.0, sessions[0].run(global_var_1)) - self.assertAllEqual(0, sessions[0].run(global_step)) - - sessions[0].run(train_ops[0]) - sessions[1].run(train_ops[1]) - - self.assertAllEqual(1.0, sessions[0].run(var_0)) - self.assertAllEqual(2.0, sessions[0].run(var_1)) - self.assertAllEqual(0.0, sessions[0].run(global_var_0)) - self.assertAllEqual(1.0, sessions[0].run(global_var_1)) - self.assertAllEqual(0, sessions[0].run(global_step)) - - # iteration 2, global variable update - thread_0 = self.checkedThread( - target=self._run, args=(train_ops[0], sessions[0])) - thread_1 = self.checkedThread( - target=self._run, args=(train_ops[1], sessions[1])) - thread_0.start() - thread_1.start() - thread_0.join() - thread_1.join() - - self.assertAllEqual(3.0, sessions[0].run(var_0)) - self.assertAllEqual(4.0, sessions[0].run(var_1)) - self.assertAllEqual(3.0, sessions[0].run(global_var_0)) - self.assertAllEqual(4.0, sessions[0].run(global_var_1)) - self.assertAllEqual(1, sessions[0].run(global_step)) - - # iteration 3 - sessions[0].run(train_ops[0]) - - self.assertAllEqual(4.0, sessions[0].run(var_0)) - self.assertAllEqual(5.0, sessions[0].run(var_1)) - self.assertAllEqual(3.0, sessions[0].run(global_var_0)) - self.assertAllEqual(4.0, sessions[0].run(global_var_1)) - self.assertAllEqual(1, sessions[0].run(global_step)) - - def testPS2TasksWithClusterSpecClass(self): - cluster_spec = server_lib.ClusterSpec({ - "ps": ["ps0:2222", "ps1:2222"], - "worker": ["worker0:2222", "worker1:2222", "worker2:2222"] - }) - worker_device = "/job:worker/task:0" - ma_coustom = model_average_optimizer.ModelAverageCustomGetter( - worker_device=worker_device) - from tensorflow.python.training import device_setter - with ops.device( - device_setter.replica_device_setter(cluster=cluster_spec, - worker_device=worker_device, - ps_device="/job:ps")), \ - variable_scope.variable_scope("", custom_getter=ma_coustom): - v = variable_scope.get_variable(initializer=[1, 2], name="v") - w = variable_scope.get_variable(initializer=[2, 1], name="w") - v_g, w_g = ma_coustom._local_2_global[v], ma_coustom._local_2_global[w] - self.assertDeviceEqual("/job:worker/task:0", v.device) - self.assertDeviceEqual("job:ps/task:0", v_g.device) - self.assertDeviceEqual("/job:worker/task:0", w.device) - self.assertDeviceEqual("job:ps/task:1", w_g.device) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/opt/python/training/moving_average_optimizer.py b/tensorflow/contrib/opt/python/training/moving_average_optimizer.py deleted file mode 100644 index 8b5a49f8ec7..00000000000 --- a/tensorflow/contrib/opt/python/training/moving_average_optimizer.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Moving average optimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import six - -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import variables -from tensorflow.python.training import moving_averages -from tensorflow.python.training import optimizer -from tensorflow.python.training import saver -from tensorflow.python.training.saving import saveable_object_util - - -class MovingAverageOptimizer(optimizer.Optimizer): - """Optimizer that computes a moving average of the variables. - - Empirically it has been found that using the moving average of the trained - parameters of a deep network is better than using its trained parameters - directly. This optimizer allows you to compute this moving average and swap - the variables at save time so that any code outside of the training loop will - use by default the averaged values instead of the original ones. - - Example of usage: - - ```python - - // Encapsulate your favorite optimizer (here the momentum one) - // inside the MovingAverageOptimizer. - opt = tf.compat.v1.train.MomentumOptimizer(learning_rate, FLAGS.momentum) - opt = tf.contrib.opt.MovingAverageOptimizer(opt) - // Then create your model and all its variables. - model = build_model() - // Add the training op that optimizes using opt. - // This needs to be called before swapping_saver(). - opt.minimize(cost, var_list) - // Then create your saver like this: - saver = opt.swapping_saver() - // Pass it to your training loop. - slim.learning.train( - model, - ... - saver=saver) - ``` - - Note that for evaluation, the normal saver should be used instead of - swapping_saver(). - """ - - def __init__(self, opt, average_decay=0.9999, num_updates=None, - sequential_update=True): - """Construct a new MovingAverageOptimizer. - - Args: - opt: A tf.Optimizer that will be used to compute and apply gradients. - average_decay: Float. Decay to use to maintain the moving averages - of trained variables. - See tf.train.ExponentialMovingAverage for details. - num_updates: Optional count of number of updates applied to variables. - See tf.train.ExponentialMovingAverage for details. - sequential_update: Bool. If False, will compute the moving average at the - same time as the model is updated, potentially doing - benign data races. - If True, will update the moving average after gradient - updates. - """ - self._optimizer = opt - self._ema = moving_averages.ExponentialMovingAverage( - average_decay, num_updates=num_updates) - self._swapped_variable_name_map = None - self._sequential_update = sequential_update - - def compute_gradients(self, *args, **kwargs): - return self._optimizer.compute_gradients(*args, **kwargs) - - def apply_gradients(self, grads_and_vars, global_step=None, name=None): - train_op = self._optimizer.apply_gradients( - grads_and_vars, global_step=global_step, name=name) - var_list = [x[1] for x in grads_and_vars if x[0] is not None] - self._swapped_variable_name_map = {} - if self._sequential_update: - with ops.control_dependencies([train_op]): - ma_op = self._ema.apply(var_list) - else: - ma_op = self._ema.apply(var_list) - - for v in var_list: - v_avg = self._ema.average(v) - self._swapped_variable_name_map[v.op.name] = v_avg.op.name - self._swapped_variable_name_map[v_avg.op.name] = v.op.name - return control_flow_ops.group(train_op, ma_op, name='train_with_avg') - - def _find_swapped_variable(self, v_name_to_tensor, v_name, tensor): - """Returns name of swapped variable for given tensor. - - Args: - v_name_to_tensor: Mapping from variable names to tensors. - v_name: name of the variable for which swapped variable should be returned - tensor: Tensor which correspond to variable for which swapped variable - should be returned. - - Returns: - Tensor which correspond to swapped variable. - - Raises: - ValueError: If swapped variable could not be found in v_name_to_tensor. - """ - swapped_v_name = self._swapped_variable_name_map.get(v_name, None) - if swapped_v_name is None: - return tensor - else: - if swapped_v_name in v_name_to_tensor: - return v_name_to_tensor[swapped_v_name] - else: - raise ValueError( - ('Variable to swap %s is not part of variables to save. ' - 'This breaks MovingAverageOptimizer.') % swapped_v_name) - - def swapping_saver(self, var_list=None, name='swapping_saver', **kwargs): - """Create a saver swapping moving averages and variables. - - You should use this saver during training. It will save the moving averages - of the trained parameters under the original parameter names. For - evaluations or inference you should use a regular saver and it will - automatically use the moving averages for the trained variable. - - You must call this function after all variables have been created and after - you have called Optimizer.minimize(). - - Args: - var_list: List of variables to save, as per `Saver()`. - If set to None, will save all the variables that have been - created before this call. - name: The name of the saver. - **kwargs: Keyword arguments of `Saver()`. - - Returns: - A `tf.compat.v1.train.Saver` object. - - Raises: - RuntimeError: If apply_gradients or minimize has not been called before. - ValueError: If var_list is provided and contains some variables but not - their moving average counterpart. - """ - - if self._swapped_variable_name_map is None: - raise RuntimeError('Must call apply_gradients or minimize before ' - 'creating the swapping_saver') - if var_list is None: - var_list = variables.global_variables() - if not isinstance(var_list, dict): - var_list = saveable_object_util.op_list_to_dict(var_list) - - v_name_to_tensor = {} - for k, tensor_or_list in six.iteritems(var_list): - # For each partitioned variable OpListToDict returns list of constituent - # parts instead of single tensor. - if (isinstance(tensor_or_list, list) - or isinstance(tensor_or_list, variables.PartitionedVariable)): - for tensor in tensor_or_list: - v_name = tensor.op.name - v_name_to_tensor[v_name] = tensor - else: - v_name_to_tensor[k] = tensor_or_list - - # Now swap variables and moving averages - swapped_var_list = {} - for k, tensor_or_list in six.iteritems(var_list): - if isinstance(tensor_or_list, list): - tensor_list_to_save = [] - for tensor in tensor_or_list: - v_name = tensor.op.name - swapped_variable = self._find_swapped_variable(v_name_to_tensor, - v_name, - tensor) - tensor_list_to_save.append(swapped_variable) - swapped_var_list[k] = tensor_list_to_save - else: - swapped_var_list[k] = self._find_swapped_variable( - v_name_to_tensor, k, tensor_or_list) - - # Build the swapping saver. - return saver.Saver(swapped_var_list, name=name, **kwargs) diff --git a/tensorflow/contrib/opt/python/training/moving_average_optimizer_test.py b/tensorflow/contrib/opt/python/training/moving_average_optimizer_test.py deleted file mode 100644 index 643403eea6f..00000000000 --- a/tensorflow/contrib/opt/python/training/moving_average_optimizer_test.py +++ /dev/null @@ -1,266 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for moving_average_optimizer.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os.path -import tempfile - -import six -from tensorflow.contrib.opt.python.training import moving_average_optimizer -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import gradient_descent -from tensorflow.python.training import saver - - -class MovingAverageOptimizerTest(test.TestCase): - - def testRun(self): - self._helpTestRun(use_resource=False) - - def testRunUseResource(self): - # Test that MovingAverageOptimizer works with resource variables. - self._helpTestRun(use_resource=True) - - def testRunUsePartitionedVars(self): - # Test that MovingAverageOptimizer works with partitioned variables. - self._helpTestRun(use_partitioned_vars=True) - - def testRunUseResourcePartitionedVars(self): - # Test that MovingAverageOptimizer works with resource and partitioned - # variables. - self._helpTestRun(use_partitioned_vars=True, use_resource=True) - - def _helpTestRun(self, use_resource=False, use_partitioned_vars=False): - # Partitioned variables are represented as a "collection" of partitions. - # To simplify the test and reuse as much code as possible we employ - # following test strategy for partitioned variables. - # - # In the case of non-partitioned variables test runs on variables with - # shape [2]. - # - # In the case of partitioned variables we use shape [4] with two partitions, - # thus each partition has shape [2]. - # For partitioned variables the test is run twice (for loop over - # variable_part_names), first time on the first partition of each variable, - # second time on the second partition of each variable. - variable_part_names = ['part_0', 'part_1'] if use_partitioned_vars else [''] - for sequential_update in [True, False]: - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - for var_part_name in variable_part_names: - with self.session(graph=ops.Graph()) as sess: - orig_val0 = [1.0, 2.0] - orig_val1 = [3.0, 4.0] - grads0 = [0.1, 0.1] - grads1 = [0.01, 0.01] - if use_partitioned_vars: - # Use partitioned variables. - # Create partitioned and duplicate each value used as initial - # value of variables. - partitioner = partitioned_variables.fixed_size_partitioner( - num_shards=2) - orig_val0 = orig_val0 * 2 - orig_val1 = orig_val1 * 2 - grads0 = grads0 * 2 - grads1 = grads1 * 2 - else: - # Regular (non-partitioned) variables. - partitioner = None - var0 = variable_scope.get_variable( - 'var0', - initializer=constant_op.constant(orig_val0, dtype=dtype), - use_resource=use_resource, - partitioner=partitioner) - var1 = variable_scope.get_variable( - 'var1', - initializer=constant_op.constant(orig_val1, dtype=dtype), - use_resource=use_resource, - partitioner=partitioner) - # Make a fake loss, such that gradient(loss, var0) == grads0 - # and gradient(loss, var1) == grads1 - grads0 = constant_op.constant(grads0, dtype=dtype) - grads1 = constant_op.constant(grads1, dtype=dtype) - loss = (math_ops.reduce_sum(grads0 * var0) - + math_ops.reduce_sum(grads1 * var1)) - - opt = moving_average_optimizer.MovingAverageOptimizer( - gradient_descent.GradientDescentOptimizer(learning_rate=2.0), - average_decay=0.5, - sequential_update=sequential_update) - save_dir = tempfile.mkdtemp( - prefix=os.path.join(self.get_temp_dir(), 'run_1')) - save_path = os.path.join(save_dir, 'model') - - update = opt.minimize(loss) - - # Get variables and their EMAs. In case of partitioned variables - # get proper part of each variable. - def _get_variable(var_name, part_name, ema): - """Returns variable of it's moving average by name.""" - matches = [ - v for v in variables.global_variables() - if ((var_name in v.op.name) - and (part_name in v.op.name) - and (('ExponentialMovingAverage' in v.op.name) == ema)) - ] - self.assertEqual(len(matches), 1) - return matches[0] - var0 = _get_variable('var0', var_part_name, ema=False) - var1 = _get_variable('var1', var_part_name, ema=False) - ema_var0 = _get_variable('var0', var_part_name, ema=True) - ema_var1 = _get_variable('var1', var_part_name, ema=True) - - perturb = control_flow_ops.group([ - state_ops.assign_add(var0, [1.0, 1.0]), - state_ops.assign_add(var1, [2.0, 2.0]), - state_ops.assign_add(ema_var0, [3.0, 3.0]), - state_ops.assign_add(ema_var1, [4.0, 4.0]) - ]) - - # Test that saver with missing ema variables will fail. - with self.assertRaisesRegexp(ValueError, r'Variable to swap'): - opt.swapping_saver(var_list=[var0]) - - train_saver = opt.swapping_saver() - train_saver_subset = opt.swapping_saver(var_list=[var0, ema_var0]) - inference_saver = saver.Saver() - variables.global_variables_initializer().run() - # Step 1. - update.run() - self.assertAllCloseAccordingToType([0.8, 1.8], var0.eval()) - self.assertAllCloseAccordingToType([2.98, 3.98], var1.eval()) - if sequential_update: - self.assertAllCloseAccordingToType([0.9, 1.9], ema_var0.eval()) - self.assertAllCloseAccordingToType([2.99, 3.99], ema_var1.eval()) - # Test that the swapping saver save/restore operation is identity. - train_saver.save(sess, save_path) - train_saver.restore(sess, save_path) - self.assertAllCloseAccordingToType([0.8, 1.8], var0.eval()) - self.assertAllCloseAccordingToType([2.98, 3.98], var1.eval()) - if sequential_update: - self.assertAllCloseAccordingToType([0.9, 1.9], ema_var0.eval()) - self.assertAllCloseAccordingToType([2.99, 3.99], ema_var1.eval()) - # Test that the subset saver saves the EMA variable as well. - if sequential_update: - subset_save_path = save_path + '_subset' - train_saver_subset.save(sess, subset_save_path) - perturb.run() - self.assertAllCloseAccordingToType([1.8, 2.8], var0.eval()) - self.assertAllCloseAccordingToType([3.9, 4.9], ema_var0.eval()) - self.assertAllCloseAccordingToType([4.98, 5.98], var1.eval()) - self.assertAllCloseAccordingToType([6.99, 7.99], ema_var1.eval()) - # Restoring should only restore var0 and ema_var0. - train_saver_subset.restore(sess, subset_save_path) - self.assertAllCloseAccordingToType([0.8, 1.8], var0.eval()) - self.assertAllCloseAccordingToType([0.9, 1.9], ema_var0.eval()) - self.assertAllCloseAccordingToType([4.98, 5.98], var1.eval()) - self.assertAllCloseAccordingToType([6.99, 7.99], ema_var1.eval()) - # Restore back to previous state. - train_saver.restore(sess, save_path) - - # If updates are parallel, - # this is not always true after the 1st step. - if sequential_update: - # Test that the normal saver will have the averaged variables. - # We test that the average values are between the original value - # and the most recent variable values (since they are an average - # of the two). - val0 = var0.eval() - val1 = var1.eval() - train_saver.save(sess, save_path) - inference_saver.restore(sess, save_path) - avg_val0 = var0.eval() - avg_val1 = var1.eval() - for i in six.moves.range(len(val0)): - self.assertLess(val0[i], avg_val0[i]) - self.assertLess(avg_val0[i], orig_val0[i]) - self.assertLess(val1[i], avg_val1[i]) - self.assertLess(avg_val1[i], orig_val1[i]) - train_saver.restore(sess, save_path) - # Step 2. - update.run() - # Test that the normal saver will have the averaged variables. - # We test that the average values are between the original value and - # the most recent variable values (since they are an average of the - # two). - val0 = var0.eval() - val1 = var1.eval() - self.assertAllCloseAccordingToType([0.6, 1.6], val0) - self.assertAllCloseAccordingToType([2.96, 3.96], val1) - train_saver.save(sess, save_path) - inference_saver.restore(sess, save_path) - avg_val0 = var0.eval() - avg_val1 = var1.eval() - for i in six.moves.range(len(val0)): - self.assertLess(val0[i], avg_val0[i]) - self.assertLess(avg_val0[i], orig_val0[i]) - self.assertLess(val1[i], avg_val1[i]) - self.assertLess(avg_val1[i], orig_val1[i]) - - def testFailWhenSaverCreatedBeforeInitialized(self): - with self.cached_session(): - var = variables.Variable([1.0], name='var', dtype=dtypes.float32) - opt = moving_average_optimizer.MovingAverageOptimizer( - gradient_descent.GradientDescentOptimizer(learning_rate=2.0)) - # We didn't call apply_gradients yet. - # This will raise an exception. - with self.assertRaises(RuntimeError): - _ = opt.swapping_saver([var]) - - def testCorrectOverride(self): - - class WrapperOptimizer(gradient_descent.GradientDescentOptimizer): - - def compute_gradients(self, *args, **kwargs): - self.compute_gradients_called = True - return super(WrapperOptimizer, self).compute_gradients( - *args, **kwargs) - - def apply_gradients(self, *args, **kwargs): - self.apply_gradients_called = True - return super(WrapperOptimizer, self).apply_gradients(*args, **kwargs) - - with self.cached_session() as sess: - var = variables.Variable([1.2], name='var', dtype=dtypes.float32) - loss = var ** 2 - wrapper_opt = WrapperOptimizer(learning_rate=2.0) - opt = moving_average_optimizer.MovingAverageOptimizer(wrapper_opt) - train_op = opt.minimize(loss) - - # Check that both methods are called on the underlying optimizer. - self.assertTrue(wrapper_opt.compute_gradients_called) - self.assertTrue(wrapper_opt.apply_gradients_called) - - # Run train_op once, and verify that we've updated the variable. - variables.global_variables_initializer().run() - sess.run(train_op) - var_value = sess.run(var) - # Started at 1.2, gradient is 2*1.2=2.4, lr=2, so should now be -3.6. - self.assertNear(-3.6, var_value, 1e-6) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/opt/python/training/multitask_optimizer_wrapper.py b/tensorflow/contrib/opt/python/training/multitask_optimizer_wrapper.py deleted file mode 100644 index 30d96e16be0..00000000000 --- a/tensorflow/contrib/opt/python/training/multitask_optimizer_wrapper.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""An optimizer wrapper for stateful optimizers with multitask loss.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import types -import six - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.training import optimizer - -__all__ = ['MultitaskOptimizerWrapper', 'clip_gradients_by_global_norm'] - - -def _is_all_zeros(grad): - all_zeros = math_ops.equal(math_ops.count_nonzero(grad), 0) - return all_zeros - - -def _get_wrapper(fn, opt): - - def wrapper(self, grad, *args, **kwargs): # pylint: disable=unused-argument - all_zeros = _is_all_zeros(grad) - def call_fn(): - with ops.control_dependencies([fn(grad, *args, **kwargs)]): - return control_flow_ops.no_op() - return control_flow_ops.cond(all_zeros, control_flow_ops.no_op, call_fn) - - wrapper = types.MethodType(wrapper, opt) - return wrapper - - -class MultitaskOptimizerWrapper(object): - """Optimizer wrapper making all-zero gradients harmless. - - This might be useful when a multi-task loss is used, - and some components of the loss might be - not present (e.g. masked out) in some training batches. - Technically their gradient would be zero, - which would normally affect the optimizer state - (e.g. push running average to zero). - However this is not the desired behaviour, - since the missing loss component - should be treated as unknown rather than zero. - - This wrapper filters out all-zero gradient tensors, - therefore preserving the optimizer state. - - If gradient clipping by global norm is used, - the provided function clip_gradients_by_global_norm - should be used (and specified explicitly by the user). - Otherwise the global norm would be underestimated - because of all-zero tensors that should be ignored. - - The gradient calculation and application - are delegated to an underlying optimizer. - The gradient application is altered only for all-zero tensors. - - Example: - ```python - momentum_optimizer = tf.compat.v1.train.MomentumOptimizer( - learning_rate, momentum=0.9) - multitask_momentum_optimizer = tf.contrib.opt.MultitaskOptimizerWrapper( - momentum_optimizer) - gradvars = multitask_momentum_optimizer.compute_gradients( - loss) - gradvars_clipped, _ = tf.contrib.opt.clip_gradients_by_global_norm( - gradvars, 15.0) - train_op = multitask_momentum_optimizer.apply_gradients( - gradvars_clipped, global_step=batch) - ``` - """ - - def __init__(self, opt): - """Constructor. - - Args: - opt: an instance of a class that implements tf.train.Optimizer. - """ - if not isinstance(opt, optimizer.Optimizer): - raise TypeError( - 'Supplied optimizer must be an instance of tf.train.Optimizer') - self._opt = opt - overridden_methods = ('_apply_dense', '_resource_apply_dense', - '_apply_sparse', '_resource_apply_sparse') - for name in overridden_methods: - fn = getattr(self._opt, name) - wrapper = _get_wrapper(fn, self._opt) - setattr(self._opt, name, wrapper) - - def __getattr__(self, name): - return getattr(self._opt, name) - - -def clip_gradients_by_global_norm(gradients_variables, clip_norm=20.): - """Clips gradients of a multitask loss by their global norm. - - Ignores all-zero tensors when computing the global norm. - - Args: - gradients_variables: a list of pairs (gradient, variable). - clip_norm: a float Tensor, the global norm to clip on. Default is 20.0. - - Returns: - list: A list of pairs of the same type as gradients_variables,. - fixed_global_norm: A 0-D (scalar) Tensor representing the global norm. - """ - gradients, variables = six.moves.zip(*gradients_variables) - - def _replace_nonexisting_grad(grad): - if grad is None: - return grad - all_zeros = _is_all_zeros(grad) - return control_flow_ops.cond( - all_zeros, - lambda: array_ops.zeros([], dtype=dtypes.as_dtype(grad.dtype)), - lambda: grad) - - nonzero_gradients = [_replace_nonexisting_grad(g) for g in gradients] - fixed_global_norm = clip_ops.global_norm(nonzero_gradients) - gradients, _ = clip_ops.clip_by_global_norm( - gradients, clip_norm, use_norm=fixed_global_norm) - return list(six.moves.zip(gradients, variables)), fixed_global_norm diff --git a/tensorflow/contrib/opt/python/training/multitask_optimizer_wrapper_test.py b/tensorflow/contrib/opt/python/training/multitask_optimizer_wrapper_test.py deleted file mode 100644 index 904aa9ab13c..00000000000 --- a/tensorflow/contrib/opt/python/training/multitask_optimizer_wrapper_test.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for MultitaskOptimizerWrapper.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import six - -from tensorflow.contrib.opt.python.training import multitask_optimizer_wrapper -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import momentum - - -class MultitaskOptimizerWrapperTest(test.TestCase): - """Tests for the multitask optimizer wrapper. - """ - - def testWrapper(self): - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtypes.float32) - var1 = variables.Variable([3.0, 4.0], dtype=dtypes.float32) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtypes.float32) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtypes.float32) - grads_allzero = constant_op.constant([0.0, 0.0], dtype=dtypes.float32) - mom_opt_impl = momentum.MomentumOptimizer(learning_rate=2.0, momentum=0.9) - mom_opt = multitask_optimizer_wrapper.MultitaskOptimizerWrapper( - mom_opt_impl) - mom_update = mom_opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - mom_update_partial = mom_opt.apply_gradients( - zip([grads_allzero, grads1], [var0, var1])) - mom_update_no_action = mom_opt.apply_gradients( - zip([grads_allzero, grads_allzero], [var0, var1])) - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - self.assertEqual(["momentum"], mom_opt.get_slot_names()) - slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEquals(slot0.get_shape(), var0.get_shape()) - slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEquals(slot1.get_shape(), var1.get_shape()) - - # Step 1: normal momentum update. - self.evaluate(mom_update) - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([0.1, 0.1]), self.evaluate(slot0)) - self.assertAllCloseAccordingToType( - np.array([0.01, 0.01]), self.evaluate(slot1)) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), self.evaluate(var0)) - self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), - self.evaluate(var1)) - - # Step 2: momentum update that changes only slot1 but not slot0. - self.evaluate(mom_update_partial) - # Check that only the relevant momentum accumulator has been updated. - self.assertAllCloseAccordingToType( - np.array([0.1, 0.1]), self.evaluate(slot0)) - self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), - self.evaluate(slot1)) - - # Step 3: momentum update that does not change anything. - self.evaluate(mom_update_no_action) - # Check that the momentum accumulators have *NOT* been updated. - self.assertAllCloseAccordingToType( - np.array([0.1, 0.1]), self.evaluate(slot0)) - self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), - self.evaluate(slot1)) - - def testGradientClipping(self): - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtypes.float32) - var1 = variables.Variable([3.0, 4.0], dtype=dtypes.float32) - var2 = variables.Variable([3.0, 4.0], dtype=dtypes.float32) - var3 = variables.Variable([3.0, 4.0], dtype=dtypes.float32) - grads0 = constant_op.constant([10.0, 15.0], dtype=dtypes.float32) - grads1 = constant_op.constant([0.0, 5.0], dtype=dtypes.float32) - grads2 = constant_op.constant([0.0, 0.0], dtype=dtypes.float32) - grads3 = None - varlist = [var0, var1, var2, var3] - gradients = [grads0, grads1, grads2, grads3] - clipped_gradvars, global_norm = ( - multitask_optimizer_wrapper.clip_gradients_by_global_norm( - six.moves.zip(gradients, varlist), clip_norm=1.0)) - clipped_grads = list(six.moves.zip(*clipped_gradvars))[0] - reference_global_norm = np.sqrt(np.sum(np.square([10.0, 15.0, 0.0, 5.0]))) - self.assertAllCloseAccordingToType( - self.evaluate(global_norm), reference_global_norm) - self.assertAllCloseAccordingToType( - self.evaluate(clipped_grads[2]), np.array([0., 0.])) - self.assertEqual(clipped_grads[3], None) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/opt/python/training/nadam_optimizer.py b/tensorflow/contrib/opt/python/training/nadam_optimizer.py deleted file mode 100644 index 046c6ee83fd..00000000000 --- a/tensorflow/contrib/opt/python/training/nadam_optimizer.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Nadam for TensorFlow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.training import adam -from tensorflow.python.training import training_ops -from tensorflow.python.util import deprecation - - -class NadamOptimizer(adam.AdamOptimizer): - """Optimizer that implements the Nadam algorithm. - - See [Dozat, T., 2015](http://cs229.stanford.edu/proj2015/054_report.pdf). - - WARNING: due to a known issue this optimizer does not use nesterov momentum - on TPUs or when using XLA in general. This is deprecated; instead prefer - tf.keras.optimizers.Nadam which does the right thing. - """ - - @deprecation.deprecated( - None, "WARNING: wrong behavior with XLA. Use tf.keras.optimizers.Nadam.") - def __init__( - self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - use_locking=False, - name="Adam"): - super(NadamOptimizer, self).__init__( - learning_rate=learning_rate, - beta1=beta1, - beta2=beta2, - epsilon=epsilon, - use_locking=use_locking, - name=name) - - def _apply_dense(self, grad, var): - m = self.get_slot(var, "m") - v = self.get_slot(var, "v") - beta1_power, beta2_power = self._get_beta_accumulators() - return training_ops.apply_adam( - var, - m, - v, - math_ops.cast(beta1_power, var.dtype.base_dtype), - math_ops.cast(beta2_power, var.dtype.base_dtype), - math_ops.cast(self._lr_t, var.dtype.base_dtype), - math_ops.cast(self._beta1_t, var.dtype.base_dtype), - math_ops.cast(self._beta2_t, var.dtype.base_dtype), - math_ops.cast(self._epsilon_t, var.dtype.base_dtype), - grad, - use_locking=self._use_locking, - use_nesterov=True).op - - def _resource_apply_dense(self, grad, var): - m = self.get_slot(var, "m") - v = self.get_slot(var, "v") - beta1_power, beta2_power = self._get_beta_accumulators() - return training_ops.resource_apply_adam( - var.handle, - m.handle, - v.handle, - math_ops.cast(beta1_power, grad.dtype.base_dtype), - math_ops.cast(beta2_power, grad.dtype.base_dtype), - math_ops.cast(self._lr_t, grad.dtype.base_dtype), - math_ops.cast(self._beta1_t, grad.dtype.base_dtype), - math_ops.cast(self._beta2_t, grad.dtype.base_dtype), - math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), - grad, - use_locking=self._use_locking, - use_nesterov=True) - - def _apply_sparse_shared(self, grad, var, indices, scatter_add): - beta1_power, beta2_power = self._get_beta_accumulators() - beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) - beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype) - lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) - beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype) - beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype) - epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype) - lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power)) - # m_t = beta1 * m + (1 - beta1) * g_t - m = self.get_slot(var, "m") - m_scaled_g_values = grad * (1 - beta1_t) - m_t = state_ops.assign(m, m * beta1_t, use_locking=self._use_locking) - with ops.control_dependencies([m_t]): - m_t = scatter_add(m, indices, m_scaled_g_values) - # m_bar = (1 - beta1) * g_t + beta1 * m_t - m_bar = m_scaled_g_values + beta1_t * array_ops.gather(m_t, indices) - # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) - v = self.get_slot(var, "v") - v_scaled_g_values = (grad * grad) * (1 - beta2_t) - v_t = state_ops.assign(v, v * beta2_t, use_locking=self._use_locking) - with ops.control_dependencies([v_t]): - v_t = scatter_add(v, indices, v_scaled_g_values) - v_t_slice = array_ops.gather(v_t, indices) - v_sqrt = math_ops.sqrt(v_t_slice) - var_update = scatter_add(var, indices, -lr * m_bar / (v_sqrt + epsilon_t)) - return control_flow_ops.group(*[var_update, m_bar, v_t]) diff --git a/tensorflow/contrib/opt/python/training/nadam_optimizer_test.py b/tensorflow/contrib/opt/python/training/nadam_optimizer_test.py deleted file mode 100644 index a4372f64874..00000000000 --- a/tensorflow/contrib/opt/python/training/nadam_optimizer_test.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Nadam.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.opt.python.training import nadam_optimizer -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def nadam_update_numpy(param, - g_t, - t, - m, - v, - alpha=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8): - alpha_t = alpha * np.sqrt(1 - beta2**t) / (1 - beta1**t) - - m_t = beta1 * m + (1 - beta1) * g_t - v_t = beta2 * v + (1 - beta2) * g_t * g_t - - m_bar = (1 - beta1) * g_t + beta1 * m_t - - param_t = param - alpha_t * m_bar / (np.sqrt(v_t) + epsilon) - return param_t, m_t, v_t - - -class NadamOptimizerTest(test.TestCase): - - def doTestSparse(self, use_resource=False): - # need to use a larger value of epsilon here so that - # np.sqrt(v_t) + epsilon doesn't get rounded to 0 when - # the dtype is half and np.sqrt(v_t) = 0, as is the case - # when the gradient is 0 - sparse_epsilon = 1e-7 - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - else: - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0_np_indices = np.array([0, 2], dtype=np.int32) - grads0 = ops.IndexedSlices( - constant_op.constant(grads0_np[grads0_np_indices]), - constant_op.constant(grads0_np_indices), constant_op.constant([3])) - grads1_np_indices = np.array([0, 2], dtype=np.int32) - grads1 = ops.IndexedSlices( - constant_op.constant(grads1_np[grads1_np_indices]), - constant_op.constant(grads1_np_indices), constant_op.constant([3])) - opt = nadam_optimizer.NadamOptimizer(epsilon=sparse_epsilon) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 3.0, 4.0], var1.eval()) - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Run 3 steps of Nadam - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) - update.run() - - var0_np, m0, v0 = nadam_update_numpy(var0_np, grads0_np, t, m0, v0, - epsilon=sparse_epsilon) - var1_np, m1, v1 = nadam_update_numpy(var1_np, grads1_np, t, m1, v1, - epsilon=sparse_epsilon) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testSparse(self): - self.doTestSparse(use_resource=False) - - def testResourceSparse(self): - self.doTestSparse(use_resource=True) - - def doTestBasic(self, use_resource=False): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - else: - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - opt = nadam_optimizer.NadamOptimizer() - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Run 3 steps of Nadam - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) - update.run() - - var0_np, m0, v0 = nadam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = nadam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testBasic(self): - self.doTestBasic(use_resource=False) - - def testResourceBasic(self): - self.doTestBasic(use_resource=True) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/opt/python/training/powersign.py b/tensorflow/contrib/opt/python/training/powersign.py deleted file mode 100644 index b4aa19264de..00000000000 --- a/tensorflow/contrib/opt/python/training/powersign.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implementation of PowerSign.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.training import optimizer -from tensorflow.python.training import training_ops - - -class PowerSignOptimizer(optimizer.Optimizer): - """Optimizer that implements the PowerSign update. - - See [Bello et al., ICML2017], - [Neural Optimizer Search with RL](https://arxiv.org/abs/1709.07417). - """ - - def __init__(self, - learning_rate=0.1, - base=math.e, - beta=0.9, - sign_decay_fn=None, - use_locking=False, - name='PowerSignOptimizer'): - """Constructs a new PowerSignOptimizer object. - - Initialization: - - ``` - m_0 <- 0 (Initialize initial 1st moment vector) - t <- 0 (Initialize timestep) - ``` - - Update: - - ``` - t <- t + 1 - m_t <- beta1 * m_{t-1} + (1 - beta1) * g - sign_decay <- sign_decay_fn(t) - update <- base ** (sign_decay * sign(g) * sign(m)) * g - variable <- variable - lr_t * update - ``` - - Example usage for PowerSign-cd (PowerSign with cosine sign decay) - ``` - decay_steps = 1000 - linear_decay_fn = sign_decays.get_cosine_decay_fn(decay_steps) - opt = PowerSignOptimizer(learning_rate=0.1, sign_decay_fn=linear_decay_fn) - ``` - - Args: - learning_rate: learning_rate used when taking a step. - base: base used in optimizer. - beta: decay used for computing the moving average m. - sign_decay_fn: decay function applied to the sign(g) sign(m) quantity. - Takes global_step as an argument. See sign_decay.py for some examples. - use_locking: If True, use locks for update operations. - name: Optional name for the operations created iwhen applying gradients. - Defaults to "PowerSignOptimizer". - """ - super(PowerSignOptimizer, self).__init__(use_locking, name) - self._lr = learning_rate - self._beta = beta - self._logbase = math.log(base) - - self._sign_decay_fn = sign_decay_fn - - # Tensor versions of the constructor arguments, created in _prepare(). - self._lr_t = None - self._beta_t = None - self._logbase_t = None - - def apply_gradients(self, grads_and_vars, global_step=None, name=None): - if self._sign_decay_fn is not None: - self._sign_decay_t = ops.convert_to_tensor( - self._sign_decay_fn(global_step), name='sign_decay') - return super(PowerSignOptimizer, self).apply_gradients( - grads_and_vars, global_step=global_step, name=name) - - def _create_slots(self, var_list): - # Create slots for the first moment. - for v in var_list: - self._zeros_slot(v, 'm', self._name) - - def _prepare(self): - self._lr_t = ops.convert_to_tensor(self._lr, name='learning_rate') - self._beta_t = ops.convert_to_tensor(self._beta, name='beta') - self._logbase_t = ops.convert_to_tensor(self._logbase, name='logbase') - if self._sign_decay_fn is None: - self._sign_decay_t = ops.convert_to_tensor(1.0, name='sign_decay') - - def _apply_dense(self, grad, var): - m = self.get_slot(var, 'm') - return training_ops.apply_power_sign( - var, - m, - math_ops.cast(self._lr_t, var.dtype.base_dtype), - math_ops.cast(self._logbase_t, var.dtype.base_dtype), - math_ops.cast(self._sign_decay_t, var.dtype.base_dtype), - math_ops.cast(self._beta_t, var.dtype.base_dtype), - grad, - use_locking=self._use_locking).op - - def _resource_apply_dense(self, grad, var): - m = self.get_slot(var, 'm') - return training_ops.resource_apply_power_sign( - var.handle, - m.handle, - math_ops.cast(self._lr_t, var.dtype.base_dtype), - math_ops.cast(self._logbase_t, var.dtype.base_dtype), - math_ops.cast(self._sign_decay_t, var.dtype.base_dtype), - math_ops.cast(self._beta_t, var.dtype.base_dtype), - grad, - use_locking=self._use_locking) - - def _apply_sparse(self, grad, var): - lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) - beta_t = math_ops.cast(self._beta_t, var.dtype.base_dtype) - logbase_t = math_ops.cast(self._logbase_t, var.dtype.base_dtype) - e_t = math_ops.cast(math.e, var.dtype.base_dtype) - - m = self.get_slot(var, 'm') - m_t = state_ops.assign( - m, (m * beta_t) + (grad * (1 - beta_t)), use_locking=self._use_locking) - - sign_g = ops.IndexedSlices( - math_ops.sign(grad.values), grad.indices, dense_shape=grad.dense_shape) - sign_gm = ops.IndexedSlices( - array_ops.gather(math_ops.sign(m_t), sign_g.indices) * sign_g.values, - sign_g.indices, - dense_shape=sign_g.dense_shape) - - sign_decayed = math_ops.cast( - self._sign_decay_t, var.dtype.base_dtype) - multiplier_values = math_ops.pow( - e_t, logbase_t * sign_decayed * sign_gm.values) - multiplier = ops.IndexedSlices( - multiplier_values, sign_gm.indices, dense_shape=sign_gm.dense_shape) - - final_update = ops.IndexedSlices( - lr_t * multiplier.values * grad.values, - multiplier.indices, - dense_shape=multiplier.dense_shape) - - var_update = state_ops.scatter_sub( - var, - final_update.indices, - final_update.values, - use_locking=self._use_locking) - - return control_flow_ops.group(* [var_update, m_t]) diff --git a/tensorflow/contrib/opt/python/training/powersign_test.py b/tensorflow/contrib/opt/python/training/powersign_test.py deleted file mode 100644 index f2c87b58839..00000000000 --- a/tensorflow/contrib/opt/python/training/powersign_test.py +++ /dev/null @@ -1,268 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for PowerSign.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import numpy as np - -from tensorflow.contrib.opt.python.training import powersign -from tensorflow.contrib.opt.python.training import sign_decay -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def py_linear_decay_fn(decay_steps): - def linear_decay(step): - step = min(step, decay_steps) - return float(decay_steps - step) / decay_steps - return linear_decay - - -def powersign_update_numpy(params, - g_t, - m, - lr, - base=math.e, - beta=0.9, - py_sign_decay_fn=None, - t=None): - m_t = beta * m + (1 - beta) * g_t - if py_sign_decay_fn is None: - sign_decayed = 1.0 - else: - sign_decayed = py_sign_decay_fn(t-1) - multiplier = base ** (sign_decayed * np.sign(g_t) * np.sign(m_t)) - params_t = params - lr * multiplier * g_t - return params_t, m_t - - -class PowerSignTest(test.TestCase): - - def _testDense(self, - use_resource=False, - learning_rate=0.1, - sign_decay_fn=None, - py_sign_decay_fn=None, - base=math.e, - beta=0.9): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(use_gpu=True): - # Initialize variables for numpy implementation. - m0, m1 = 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - global_step = resource_variable_ops.ResourceVariable( - 0, trainable=False) - else: - var0 = variables.VariableV1(var0_np) - var1 = variables.VariableV1(var1_np) - global_step = variables.VariableV1( - 0, trainable=False) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - opt = powersign.PowerSignOptimizer( - learning_rate=learning_rate, - base=base, - beta=beta, - sign_decay_fn=sign_decay_fn, - ) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - neg_update = opt.apply_gradients(zip([-grads0, -grads1], [var0, var1]), - global_step=global_step) - - if not context.executing_eagerly(): - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - # Run 7 steps of powersign - # first 4 steps with positive gradient - # last 3 steps with negative gradient (sign(gm) should be -1) - for t in range(1, 8): - if t < 5: - if not context.executing_eagerly(): - self.evaluate(update) - elif t > 1: - opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - else: - if not context.executing_eagerly(): - self.evaluate(neg_update) - elif t > 1: - opt.apply_gradients(zip([-grads0, -grads1], [var0, var1]), - global_step=global_step) - - var0_np, m0 = powersign_update_numpy( - var0_np, - grads0_np if t < 5 else -grads0_np, - m0, - learning_rate, - base=base, - beta=beta, - py_sign_decay_fn=py_sign_decay_fn, - t=t, - ) - var1_np, m1 = powersign_update_numpy( - var1_np, - grads1_np if t < 5 else -grads1_np, - m1, - learning_rate, - base=base, - beta=beta, - py_sign_decay_fn=py_sign_decay_fn, - t=t, - ) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - - def testDense(self): - decay_steps = 10 - sign_decay_fn = sign_decay.get_linear_decay_fn(decay_steps) - py_sign_decay_fn = py_linear_decay_fn(decay_steps) - self._testDense(use_resource=False) - self._testDense(use_resource=False, - learning_rate=0.1, - base=10.0, - beta=0.8) - self._testDense(use_resource=False, - sign_decay_fn=sign_decay_fn, - py_sign_decay_fn=py_sign_decay_fn) - - self._testDense(use_resource=True) - self._testDense(use_resource=True, learning_rate=0.1, base=10.0, beta=0.8) - self._testDense(use_resource=True, - sign_decay_fn=sign_decay_fn, - py_sign_decay_fn=py_sign_decay_fn) - - def _testSparse(self, - use_resource=False, - learning_rate=0.1, - sign_decay_fn=None, - py_sign_decay_fn=None, - base=math.e, - beta=0.9): - with self.session(use_gpu=True): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - # Initialize variables for numpy implementation. - m0, m1 = 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - global_step = resource_variable_ops.ResourceVariable( - 0, trainable=False) - else: - var0 = variables.VariableV1(var0_np) - var1 = variables.VariableV1(var1_np) - global_step = variables.VariableV1( - 0, trainable=False) - grads0_np_indices = np.array([0, 1], dtype=np.int32) - grads0 = ops.IndexedSlices( - constant_op.constant(grads0_np), - constant_op.constant(grads0_np_indices), constant_op.constant([2])) - grads1_np_indices = np.array([0, 1], dtype=np.int32) - grads1 = ops.IndexedSlices( - constant_op.constant(grads1_np), - constant_op.constant(grads1_np_indices), constant_op.constant([2])) - opt = powersign.PowerSignOptimizer( - learning_rate=learning_rate, - base=base, - beta=beta, - sign_decay_fn=sign_decay_fn, - ) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), - global_step=global_step) - neg_update = opt.apply_gradients(zip([-grads0, -grads1], [var0, var1]), - global_step=global_step) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - # Run 7 steps of powersign - # first 4 steps with positive gradient - # last 3 steps with negative gradient (sign(gm) should be -1) - for t in range(1, 8): - if t < 5: - update.run() - else: - neg_update.run() - - var0_np, m0 = powersign_update_numpy( - var0_np, - grads0_np if t < 5 else -grads0_np, - m0, - learning_rate, - base=base, - beta=beta, - py_sign_decay_fn=py_sign_decay_fn, - t=t, - ) - var1_np, m1 = powersign_update_numpy( - var1_np, - grads1_np if t < 5 else -grads1_np, - m1, - learning_rate, - base=base, - beta=beta, - py_sign_decay_fn=py_sign_decay_fn, - t=t, - ) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testSparse(self): - decay_steps = 10 - sign_decay_fn = sign_decay.get_linear_decay_fn(decay_steps) - py_sign_decay_fn = py_linear_decay_fn(decay_steps) - self._testSparse(use_resource=False) - self._testSparse(use_resource=False, - learning_rate=0.01, - base=2.0, - beta=0.8) - self._testSparse(use_resource=False, - sign_decay_fn=sign_decay_fn, - py_sign_decay_fn=py_sign_decay_fn) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/opt/python/training/reg_adagrad_optimizer.py b/tensorflow/contrib/opt/python/training/reg_adagrad_optimizer.py deleted file mode 100644 index d0e0405a2c3..00000000000 --- a/tensorflow/contrib/opt/python/training/reg_adagrad_optimizer.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""RegAdagrad for TensorFlow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import math_ops -from tensorflow.python.training import adagrad -from tensorflow.python.training import training_ops -from tensorflow.python.util import tf_contextlib - - -class RegAdagradOptimizer(adagrad.AdagradOptimizer): - """RegAdagrad: Adagrad with updates that optionally skip updating the slots. - - This is meant to address the problem of additional regularization terms in the - loss function affecting learning rate decay and causing hyper-param - entanglement. Example usage: - - loss = tf.nn.cross_entropy(x, labels) - reg_loss = reg_strength * tf.reduce_sum(x * x) - opt = tf.contrib.opt.RegAdagradOptimizer(learning_rate) - loss_update = opt.minimize(loss) - with opt.avoid_updating_slots(): - reg_update = opt.minimize(reg_loss) - total_update = tf.group([loss_update, reg_update]) - - # ... - - sess.run(total_update, ...) - """ - - def __init__(self, - learning_rate, - initial_accumulator_value=0.1, - use_locking=False, - name="RegAdagrad"): - super(RegAdagradOptimizer, self).__init__( - learning_rate, - initial_accumulator_value=initial_accumulator_value, - use_locking=use_locking, - name=name) - self._should_update_slots = True - - @tf_contextlib.contextmanager - def avoid_updating_slots(self): - old = self._should_update_slots - self._should_update_slots = False - try: - yield - finally: - self._should_update_slots = old - - def _apply_dense(self, grad, var): - acc = self.get_slot(var, "accumulator") - return training_ops.apply_adagrad( - var, - acc, - math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype), - grad, - use_locking=self._use_locking, - update_slots=self._should_update_slots) - - def _resource_apply_dense(self, grad, var, update_slots=True): - acc = self.get_slot(var, "accumulator") - return training_ops.resource_apply_adagrad( - var.handle, - acc.handle, - math_ops.cast(self._learning_rate_tensor, grad.dtype.base_dtype), - grad, - use_locking=self._use_locking, - update_slots=self._should_update_slots) - - def _apply_sparse(self, grad, var, update_slots=True): - acc = self.get_slot(var, "accumulator") - return training_ops.sparse_apply_adagrad( - var, - acc, - math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype), - grad.values, - grad.indices, - use_locking=self._use_locking, - update_slots=self._should_update_slots) - - def _resource_apply_sparse(self, grad, var, indices, update_slots=True): - acc = self.get_slot(var, "accumulator") - return training_ops.resource_sparse_apply_adagrad( - var.handle, - acc.handle, - math_ops.cast(self._learning_rate_tensor, grad.dtype), - grad, - indices, - use_locking=self._use_locking, - update_slots=self._should_update_slots) diff --git a/tensorflow/contrib/opt/python/training/reg_adagrad_optimizer_test.py b/tensorflow/contrib/opt/python/training/reg_adagrad_optimizer_test.py deleted file mode 100644 index c09e2ac76d4..00000000000 --- a/tensorflow/contrib/opt/python/training/reg_adagrad_optimizer_test.py +++ /dev/null @@ -1,343 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functional tests for Regreg_adagrad_optimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.opt.python.training import reg_adagrad_optimizer -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class RegAdagradOptimizerTest(test.TestCase): - - def doTestBasic(self, use_locking=False, use_resource=False): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - if use_resource: - var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) - var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) - else: - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - ada_opt = reg_adagrad_optimizer.RegAdagradOptimizer( - 3.0, initial_accumulator_value=0.1, use_locking=use_locking) - ada_update = ada_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Run 3 steps of adagrad - for _ in range(3): - ada_update.run() - # Validate updated params - self.assertAllCloseAccordingToType( - np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([2.715679168701172, 3.715679168701172]), var1.eval()) - - def testBasic(self): - self.doTestBasic(use_locking=False) - - def testBasicResource(self): - self.doTestBasic(use_locking=False, use_resource=True) - - def testBasicLocked(self): - self.doTestBasic(use_locking=True) - - def testMinimizeSparseResourceVariable(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = resource_variable_ops.ResourceVariable( - [[1.0, 2.0], [3.0, 4.0]], dtype=dtype) - x = constant_op.constant([[4.0], [5.0]], dtype=dtype) - pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) - loss = pred * pred - sgd_op = reg_adagrad_optimizer.RegAdagradOptimizer(1.0).minimize(loss) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0], [3.0, 4.0]], - var0.eval()) - # Run 1 step of sgd - sgd_op.run() - # Validate updated params - self.assertAllCloseAccordingToType( - [[0, 1], [3, 4]], var0.eval(), atol=0.01) - - def testTensorLearningRate(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - ada_opt = reg_adagrad_optimizer.RegAdagradOptimizer( - constant_op.constant(3.0), initial_accumulator_value=0.1) - ada_update = ada_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Run 3 steps of adagrad - for _ in range(3): - ada_update.run() - # Validate updated params - self.assertAllCloseAccordingToType( - np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([2.715679168701172, 3.715679168701172]), var1.eval()) - - def testSparseBasic(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) - var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) - grads0 = ops.IndexedSlices( - constant_op.constant([0.1], shape=[1, 1], dtype=dtype), - constant_op.constant([0]), constant_op.constant([2, 1])) - grads1 = ops.IndexedSlices( - constant_op.constant([0.01], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), constant_op.constant([2, 1])) - ada_opt = reg_adagrad_optimizer.RegAdagradOptimizer( - 3.0, initial_accumulator_value=0.1) - ada_update = ada_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([[1.0], [2.0]], var0.eval()) - self.assertAllClose([[3.0], [4.0]], var1.eval()) - # Run 3 step of sgd - for _ in range(3): - ada_update.run() - # Validate updated params - self.assertAllCloseAccordingToType( - np.array([[-1.6026098728179932], [2.0]]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([[3.0], [3.715679168701172]]), var1.eval()) - - def testSparseRepeatedIndices(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - repeated_index_update_var = variables.Variable( - [[1.0], [2.0]], dtype=dtype) - aggregated_update_var = variables.Variable([[1.0], [2.0]], dtype=dtype) - grad_repeated_index = ops.IndexedSlices( - constant_op.constant([0.1, 0.1], shape=[2, 1], dtype=dtype), - constant_op.constant([1, 1]), constant_op.constant([2, 1])) - grad_aggregated = ops.IndexedSlices( - constant_op.constant([0.2], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), constant_op.constant([2, 1])) - repeated_update = reg_adagrad_optimizer.RegAdagradOptimizer( - 3.0).apply_gradients([(grad_repeated_index, - repeated_index_update_var)]) - aggregated_update = reg_adagrad_optimizer.RegAdagradOptimizer( - 3.0).apply_gradients([(grad_aggregated, aggregated_update_var)]) - variables.global_variables_initializer().run() - self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) - for _ in range(3): - repeated_update.run() - aggregated_update.run() - self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) - - def testSparseRepeatedIndicesResourceVariable(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var_repeated = resource_variable_ops.ResourceVariable( - [1.0, 2.0], dtype=dtype) - loss_repeated = math_ops.reduce_sum( - embedding_ops.embedding_lookup(var_repeated, [0, 0])) - var_aggregated = resource_variable_ops.ResourceVariable( - [1.0, 2.0], dtype=dtype) - loss_aggregated = 2 * math_ops.reduce_sum( - embedding_ops.embedding_lookup(var_aggregated, [0])) - update_op_repeated = reg_adagrad_optimizer.RegAdagradOptimizer( - 2.0).minimize(loss_repeated) - update_op_aggregated = reg_adagrad_optimizer.RegAdagradOptimizer( - 2.0).minimize(loss_aggregated) - variables.global_variables_initializer().run() - self.assertAllCloseAccordingToType(var_repeated.eval(), - var_aggregated.eval()) - for _ in range(3): - update_op_repeated.run() - update_op_aggregated.run() - self.assertAllCloseAccordingToType(var_repeated.eval(), - var_aggregated.eval()) - - def testSparseStability(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - shape = [1, 6] - var0 = variables.Variable( - [[ - 0.00872496, -0.106952, 0.110467, 0.226505, -0.0147257, - -0.0105945 - ]], - dtype=dtype) - grads0 = ops.IndexedSlices( - constant_op.constant( - [[ - -5.91278e-05, 5.31673e-05, -2.5779e-06, 4.29153e-05, - -8.4877e-05, -9.48906e-05 - ]], - shape=shape, - dtype=dtype), constant_op.constant([0]), - constant_op.constant(shape)) - ada_opt = reg_adagrad_optimizer.RegAdagradOptimizer( - 1.0, initial_accumulator_value=0.1) - ada_update = ada_opt.apply_gradients(zip([grads0], [var0])) - self.assertEqual(["accumulator"], ada_opt.get_slot_names()) - slot0 = ada_opt.get_slot(var0, "accumulator") - init = variables.global_variables_initializer() - for _ in range(100): - init.run() - ada_update.run() - self.assertAllCloseAccordingToType( - np.array([[0.1, 0.1, 0.1, 0.1, 0.1, 0.1]]), slot0.eval()) - self.assertAllCloseAccordingToType( - np.array([[ - 0.00891194, -0.10712013, 0.11047515, 0.22636929, -0.0144573, - -0.01029443 - ]]), var0.eval()) - - def testSharing(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - ada_opt = reg_adagrad_optimizer.RegAdagradOptimizer(3.0) - # Apply the optimizer twice. Both applications will use - # the same accums. - ada_update1 = ada_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - ada_update2 = ada_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - self.assertEqual(["accumulator"], ada_opt.get_slot_names()) - slot0 = ada_opt.get_slot(var0, "accumulator") - self.assertEquals(slot0.get_shape(), var0.get_shape()) - slot1 = ada_opt.get_slot(var1, "accumulator") - self.assertEquals(slot1.get_shape(), var1.get_shape()) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values. - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Mix the first and the second adagrad for 3 steps. - ada_update1.run() - ada_update2.run() - ada_update1.run() - # Validate updated params (the same as with only 1 RegAdagrad). - self.assertAllCloseAccordingToType( - np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([2.715679168701172, 3.715679168701172]), var1.eval()) - - def testDynamicShapeVariable_Ok(self): - with self.cached_session(): - v = variable_scope.get_variable( - "v", initializer=constant_op.constant(1.), validate_shape=False) - self.assertFalse(v.shape.is_fully_defined()) - # Creating optimizer should cause no exception. - reg_adagrad_optimizer.RegAdagradOptimizer( - 3.0, initial_accumulator_value=0.1) - - def testSkipUpdatingSlots(self): - iav = 0.130005 # A value that works with float16 - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - ada_opt = reg_adagrad_optimizer.RegAdagradOptimizer( - 3.0, initial_accumulator_value=iav) - # Apply the optimizer twice. Both applications will use - # the same accums. - with ada_opt.avoid_updating_slots(): - ada_update = ada_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - self.assertEqual(["accumulator"], ada_opt.get_slot_names()) - slot0 = ada_opt.get_slot(var0, "accumulator") - self.assertEquals(slot0.get_shape(), var0.get_shape()) - slot1 = ada_opt.get_slot(var1, "accumulator") - self.assertEquals(slot1.get_shape(), var1.get_shape()) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values. - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Mix the first and the second adagrad for 3 steps. - for _ in range(3): - ada_update.run() - # Validate that ada_opt's slots are not updated. - self.assertAllCloseAccordingToType(np.array([iav, iav]), slot0.eval()) - self.assertAllCloseAccordingToType(np.array([iav, iav]), slot1.eval()) - - def testSparseSkipUpdatingSlots(self): - iav = 0.130005 # A value that works with float16 - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) - var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) - grads0 = ops.IndexedSlices( - constant_op.constant([0.1], shape=[1, 1], dtype=dtype), - constant_op.constant([0]), constant_op.constant([2, 1])) - grads1 = ops.IndexedSlices( - constant_op.constant([0.01], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), constant_op.constant([2, 1])) - ada_opt = reg_adagrad_optimizer.RegAdagradOptimizer( - 3.0, initial_accumulator_value=iav) - with ada_opt.avoid_updating_slots(): - ada_update = ada_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - slot0 = ada_opt.get_slot(var0, "accumulator") - self.assertEquals(slot0.get_shape(), var0.get_shape()) - slot1 = ada_opt.get_slot(var1, "accumulator") - self.assertEquals(slot1.get_shape(), var1.get_shape()) - - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([[1.0], [2.0]], var0.eval()) - self.assertAllClose([[3.0], [4.0]], var1.eval()) - # Run 3 step of sgd - for _ in range(3): - ada_update.run() - # Validate that ada_opt's slots are not updated. - self.assertAllCloseAccordingToType( - np.array([[iav], [iav]]), slot0.eval()) - self.assertAllCloseAccordingToType( - np.array([[iav], [iav]]), slot1.eval()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/opt/python/training/shampoo.py b/tensorflow/contrib/opt/python/training/shampoo.py deleted file mode 100644 index efbafac662b..00000000000 --- a/tensorflow/contrib/opt/python/training/shampoo.py +++ /dev/null @@ -1,428 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""The Shampoo Optimizer. - -Variant of Adagrad using one preconditioner matrix per variable dimension. -For details, see https://arxiv.org/abs/1802.09568 -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib.opt.python.training import matrix_functions -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.platform import tf_logging -from tensorflow.python.training import optimizer - - -def GetParam(var, timestep): - if callable(var): - return var(timestep) - else: - return var - - -class ShampooOptimizer(optimizer.Optimizer): - """The Shampoo Optimizer - - Variant of Adagrad using one preconditioner matrix per variable dimension. - For details, see https://arxiv.org/abs/1802.09568 - - gbar is time-weighted accumulated gradient: - gbar[t] = gbar_decay[t] * gbar[t-1] + gbar_weight[t] * g[t] - - mat_gbar is time-weighted accumulated gradient square: - mat_gbar_j[t] = mat_gbar_decay[t] * mat_gbar_j[t-1] - + mat_gbar_weight[t] * gg_j[t] - where if g[t] = g_abcd then gg_a[t] = g_abcd g_a'bcd (Einstein notation) - - Update rule: - w[t+1] = w[t] - learning_rate[t] * Prod_j mat_gbar_j[t]^(-alpha/n) gbar[t] - Again, mat_gbar_j[t]^(-alpha) gbar[t] is a tensor contraction along the - j'th dimension of gbar[t] with the first dimension of - mat_gbar_j[t]^(-alpha/n), where alpha is a hyperparameter, - and n = rank of the variable. - Prod_j represents doing this contraction for all j in 0..n-1. - - Typically learning_rate is constant, but could be time dependent by passing - a lambda function that depends on step. - """ - - def __init__(self, - global_step=0, - max_matrix_size=768, - gbar_decay=0.0, - gbar_weight=1.0, - mat_gbar_decay=1.0, - mat_gbar_weight=1.0, - learning_rate=1.0, - svd_interval=1, - precond_update_interval=1, - epsilon=1e-4, - alpha=0.5, - use_iterative_root=False, - use_locking=False, - name="Shampoo"): - """Default values of the various hyper-parameters. - - gbar_decay, gbar_weight etc. can be a float or a time varying parameter. - For time-varying parameters use e.g. "lambda T: T / (T + 1.0)" - where the expression in the lambda is a tensorflow expression - - Args: - global_step: tensorflow variable indicating the step. - max_matrix_size: We do not perform SVD for matrices larger than this. - gbar_decay: - gbar_weight: Used to update gbar: - gbar[t] = gbar_decay[t] * gbar[t-1] + gbar_weight[t] * g[t] - mat_gbar_decay: - mat_gbar_weight: Used to update mat_gbar: - mat_gbar_j[t] = mat_gbar_decay[t] * mat_gbar_j[t-1] - + mat_gbar_weight[t] * gg_j[t] - learning_rate: Similar to SGD - svd_interval: We should do SVD after this many steps. Default = 1, i.e. - every step. Usually 20 leads to no loss of accuracy, and - 50 or 100 is also OK. May also want more often early, - and less often later - set in caller as for example: - "svd_interval = lambda(T): tf.cond( - T < 2000, lambda: 20.0, lambda: 1000.0)" - precond_update_interval: We should update the preconditioners after - this many steps. Default = 1. Usually less than - svd_interval. - epsilon: epsilon * I_n is added to each mat_gbar_j for stability for - non-diagonal version of shampoo. - alpha: total power of the preconditioners. - use_iterative_root: should the optimizer use SVD (faster) or the - iterative root method (for TPU) for finding the - roots of PSD matrices. - use_locking: - name: name of optimizer. - """ - - super(ShampooOptimizer, self).__init__(use_locking, name) - - self._global_step = math_ops.cast(global_step, dtypes.float32) - self._max_matrix_size = max_matrix_size - self._gbar_decay = gbar_decay - self._gbar_weight = gbar_weight - self._mat_gbar_decay = mat_gbar_decay - self._mat_gbar_weight = mat_gbar_weight - self._learning_rate = learning_rate - self._svd_interval = svd_interval - self._precond_update_interval = precond_update_interval - self._epsilon = epsilon - self._alpha = alpha - self._use_iterative_root = use_iterative_root - self._name = name - - def _create_slots(self, var_list): - for v in var_list: - with ops.colocate_with(v): - _ = self._zeros_slot(v, "gbar", self._name) - shape = np.array(v.get_shape()) - for i, d in enumerate(shape): - d_tensor = ops.convert_to_tensor(d) - if d <= self._max_matrix_size: - mat_g_init = array_ops.zeros_like(linalg_ops.eye(d_tensor)) - if self._svd_interval > 1: - _ = self._get_or_make_slot(v, linalg_ops.eye(d_tensor), - "H_" + str(i), self._name) - else: - mat_g_init = array_ops.zeros([d_tensor]) - - _ = self._get_or_make_slot(v, mat_g_init, "Gbar_" + str(i), - self._name) - - def _resource_apply_dense(self, grad, var): - return self._apply_dense(grad, var) - - def _apply_dense(self, grad, var): - return self._apply_gradient(grad, var) - - def _resource_apply_sparse(self, grad_values, var, grad_indices): - return self._apply_sparse_shared(grad_values, grad_indices, var) - - def _apply_sparse(self, grad, var): - return self._apply_sparse_shared(grad.values, grad.indices, var) - - def _apply_sparse_shared(self, grad_values, grad_indices, var): - if var.get_shape()[0] <= self._max_matrix_size or self._gbar_decay != 0.0: - # The dimension is small enough, we can make the variable dense and - # do a dense update - dense_grad = array_ops.scatter_nd( - array_ops.expand_dims(grad_indices, axis=1), grad_values, - array_ops.shape(var, out_type=grad_indices.dtype)) - return self._apply_gradient(dense_grad, var) - return self._apply_gradient(grad_values, var, grad_indices) - - def _weighted_average(self, var, weight, weight_t, rest): - """Computes exponential weighted average: var = weight_t * var + rest. - - Important to ensure that var does not occur in rest, otherwise - we can get race conditions in a distributed setting. - - Args: - var: variable to be updated - weight: parameter to be checked. If it is a constant, we can optimize. - weight_t: current value of parameter, used for weighting - rest: the remaining tensor to be added - - Returns: - updated variable. - """ - if weight == 0.0: - return rest # no need to update var, we will never use it. - if weight == 1.0: # common case - return state_ops.assign_add(var, rest) - # The op below can cause race conditions in a distributed setting, - # since computing weight_t * var + rest can take some time, during - # which var may be set by another worker. To prevent this, it should - # be implemented as a C++ op. - return var.assign_add((weight_t - 1) * var + rest) - - def _update_mat_g(self, mat_g, grad, axes, mat_gbar_decay, - mat_gbar_weight, i): - """Updates the cumulative outer products of the gradients. - - Args: - mat_g: the matrix to be updated - grad: the gradient of the variable - axes: a list of k-1 integers 0 to k-1, except i - mat_gbar_decay: constant for weighted average: - mat_g = mat_g * decay + grad * weight - mat_gbar_weight: constant for weighted average - i: index of dimension to be updated. - - Returns: - updated mat_g = mat_g * mat_gbar_decay + grad_outer * mat_gbar_weight - - In Einstein notation if i = 0: grad_outer_aa'= g_abcd g_a'bcd - thus grad_outer is a matrix d_i x d_i, where d_i is the size of the - i'th dimension of g. - Alternate view: If mat_i(grad) is the flattening of grad to a - d_i x (d_1d_2...d_{i-1}d_{i+1}...d_k) matrix, then - grad_outer = mat_i(grad) mat_i(grad).transpose - """ - grad_outer = math_ops.tensordot(grad, grad, axes=(axes, axes), - name="grad_outer_" + str(i)) - return self._weighted_average(mat_g, self._mat_gbar_decay, mat_gbar_decay, - mat_gbar_weight * grad_outer) - - def _compute_power_svd(self, var, mat_g, mat_g_size, alpha, mat_h_slot_name): - """Computes mat_h = mat_g^alpha using svd. mat_g is a symmetric PSD matrix. - - Args: - var: the variable we are updating. - mat_g: the symmetric PSD matrix whose power it to be computed - mat_g_size: size of mat_g - alpha: a real number - mat_h_slot_name: name of slot to store the power, if needed. - - Returns: - mat_h = mat_g^alpha - - Stores mat_h in the appropriate slot, if it exists. - Note that mat_g is PSD. So we could use linalg_ops.self_adjoint_eig. - """ - if mat_g_size == 1: - mat_h = math_ops.pow(mat_g + self._epsilon, alpha) - else: - damping = self._epsilon * linalg_ops.eye( - math_ops.cast(mat_g_size, dtypes.int32)) - diag_d, mat_u, mat_v = linalg_ops.svd(mat_g + damping, full_matrices=True) - mat_h = math_ops.matmul( - mat_v * math_ops.pow(math_ops.maximum(diag_d, self._epsilon), alpha), - array_ops.transpose(mat_u)) - if mat_h_slot_name is not None: - return state_ops.assign(self.get_slot(var, mat_h_slot_name), mat_h) - return mat_h - - def _compute_power_iter(self, var, mat_g, mat_g_size, alpha, mat_h_slot_name, - iter_count=100, epsilon=1e-6): - """Computes mat_g^alpha, where alpha = -1/p, p a positive integer.""" - - mat_g_sqrt = matrix_functions.matrix_square_root(mat_g, mat_g_size, - iter_count, self._epsilon) - mat_h = matrix_functions.matrix_inverse_pth_root( - mat_g_sqrt, - mat_g_size, - 2 * alpha, - iter_count, - epsilon, - ridge_epsilon=0.0) - - if mat_h_slot_name is not None: - return state_ops.assign(self.get_slot(var, mat_h_slot_name), mat_h) - return mat_h - - def _compute_power(self, var, mat_g, mat_g_size, alpha, mat_h_slot_name=None): - """Just a switch between the iterative power vs svd.""" - with ops.name_scope("matrix_iterative_power"): - if self._use_iterative_root: - return self._compute_power_iter(var, mat_g, mat_g_size, alpha, - mat_h_slot_name) - else: - return self._compute_power_svd(var, mat_g, mat_g_size, alpha, - mat_h_slot_name) - - def _apply_gradient(self, grad, var, indices=None): - """The main function to update a variable. - - Args: - grad: A Tensor containing gradient to apply. - var: A Tensor containing the variable to update. - indices: An array of integers, for sparse update. - - Returns: - Updated variable var = var - learning_rate * preconditioner * grad - - If the gradient is dense, var and grad have the same shape. - If the update is sparse, then the first dimension of the gradient and var - may differ, others are all the same. In this case the indices array - provides the set of indices of the variable which are to be updated with - each row of the gradient. - """ - global_step = self._global_step + 1 - - # Update accumulated weighted average of gradients - gbar = self.get_slot(var, "gbar") - gbar_decay_t = GetParam(self._gbar_decay, global_step) - gbar_weight_t = GetParam(self._gbar_weight, global_step) - if indices is not None: - # Note - the sparse update is not easily implemented, since the - # algorithm needs all indices of gbar to be updated - # if mat_gbar_decay != 1 or mat_gbar_decay != 0. - # One way to make mat_gbar_decay = 1 is by rescaling. - # If we want the update: - # G_{t+1} = a_{t+1} G_t + b_{t+1} w_t - # define: - # r_{t+1} = a_{t+1} * r_t - # h_t = G_t / r_t - # Then: - # h_{t+1} = h_t + (b_{t+1} / r_{t+1}) * w_t - # So we get the mat_gbar_decay = 1 as desired. - # We can implement this in a future version as needed. - # However we still need gbar_decay = 0, otherwise all indices - # of the variable will need to be updated. - if self._gbar_decay != 0.0: - tf_logging.warning("Not applying momentum for variable: %s" % var.name) - gbar_updated = grad - else: - gbar_updated = self._weighted_average(gbar, self._gbar_decay, - gbar_decay_t, - gbar_weight_t * grad) - - # Update the preconditioners and compute the preconditioned gradient - shape = var.get_shape() - mat_g_list = [] - for i in range(len(shape)): - mat_g_list.append(self.get_slot(var, "Gbar_" + str(i))) - mat_gbar_decay_t = GetParam(self._mat_gbar_decay, global_step) - mat_gbar_weight_t = GetParam(self._mat_gbar_weight, global_step) - - preconditioned_grad = gbar_updated - v_rank = len(mat_g_list) - neg_alpha = - GetParam(self._alpha, global_step) / v_rank - svd_interval = GetParam(self._svd_interval, global_step) - precond_update_interval = GetParam(self._precond_update_interval, - global_step) - for i, mat_g in enumerate(mat_g_list): - # axes is the list of indices to reduce - everything but the current i. - axes = list(range(i)) + list(range(i+1, v_rank)) - if shape[i] <= self._max_matrix_size: - # If the tensor size is sufficiently small perform full Shampoo update - # Note if precond_update_interval > 1 and mat_gbar_decay_t != 1, this - # is not strictly correct. However we will use it for now, and - # fix if needed. (G_1 = aG + bg ==> G_n = a^n G + (1+a+..+a^{n-1})bg) - - # pylint: disable=g-long-lambda,cell-var-from-loop - mat_g_updated = control_flow_ops.cond( - math_ops.mod(global_step, precond_update_interval) < 1, - lambda: self._update_mat_g( - mat_g, grad, axes, mat_gbar_decay_t, - mat_gbar_weight_t * precond_update_interval, i), - lambda: mat_g) - - mat_g_updated = mat_g_updated / float(shape[i].value) - - if self._svd_interval == 1: - mat_h = self._compute_power(var, mat_g_updated, shape[i], neg_alpha) - else: - mat_h = control_flow_ops.cond( - math_ops.mod(global_step, svd_interval) < 1, - lambda: self._compute_power(var, mat_g_updated, shape[i], - neg_alpha, "H_" + str(i)), - lambda: self.get_slot(var, "H_" + str(i))) - - # mat_h is a square matrix of size d_i x d_i - # preconditioned_grad is a d_i x ... x d_n x d_0 x ... d_{i-1} tensor - # After contraction with a d_i x d_i tensor - # it becomes a d_{i+1} x ... x d_n x d_0 x ... d_i tensor - # (the first dimension is contracted out, and the second dimension of - # mat_h is appended). After going through all the indices, it becomes - # a d_0 x ... x d_n tensor again. - preconditioned_grad = math_ops.tensordot(preconditioned_grad, mat_h, - axes=([0], [0]), - name="precond_" + str(i)) - else: - # Tensor size is too large -- perform diagonal Shampoo update - # Only normalize non-vector cases. - if axes: - normalizer = 1.0 if indices is not None else float(shape[i].value) - grad_outer = math_ops.reduce_sum(grad * grad, axis=axes) / normalizer - else: - grad_outer = grad * grad - - if i == 0 and indices is not None: - assert self._mat_gbar_decay == 1.0 - mat_g_updated = state_ops.scatter_add(mat_g, indices, - mat_gbar_weight_t * grad_outer) - mat_g_updated_slice = array_ops.gather(mat_g_updated, indices) - mat_h = array_ops.where( - math_ops.greater(mat_g_updated_slice, 0), - math_ops.pow(mat_g_updated_slice, neg_alpha), - array_ops.zeros_like(mat_g_updated_slice)) - else: - mat_g_updated = self._weighted_average(mat_g, - self._mat_gbar_decay, - mat_gbar_decay_t, - mat_gbar_weight_t * grad_outer) - mat_h = array_ops.where( - math_ops.greater(mat_g_updated, 0), - math_ops.pow(mat_g_updated, neg_alpha), - array_ops.zeros_like(mat_g_updated)) - - # Need to do the transpose to ensure that the tensor becomes - # a d_{i+1} x ... x d_n x d_0 x ... d_i tensor as described above. - preconditioned_grad = array_ops.transpose( - preconditioned_grad, perm=list(range(1, v_rank)) + [0]) * mat_h - - # Update the variable based on the Shampoo update - learning_rate_t = GetParam(self._learning_rate, global_step) - if indices is not None: - var_updated = state_ops.scatter_add( - var, indices, -learning_rate_t * preconditioned_grad) - else: - var_updated = state_ops.assign_sub(var, - learning_rate_t * preconditioned_grad) - return var_updated diff --git a/tensorflow/contrib/opt/python/training/shampoo_test.py b/tensorflow/contrib/opt/python/training/shampoo_test.py deleted file mode 100644 index e88c8221a0d..00000000000 --- a/tensorflow/contrib/opt/python/training/shampoo_test.py +++ /dev/null @@ -1,772 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Functional tests for AdaMoo optimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy as np - -from tensorflow.contrib.opt.python.training import shampoo -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - -TOLERANCE = 1e-3 -RIDGE_EPSILON = 1e-4 - - -def np_power(mat_g, alpha): - """Computes mat_g^alpha for a square symmetric matrix mat_g.""" - - mat_u, diag_d, mat_v = np.linalg.svd(mat_g) - diag_d = np.power(diag_d, alpha) - return np.dot(np.dot(mat_u, np.diag(diag_d)), mat_v) - - -class ShampooTest(test.TestCase, parameterized.TestCase): - - @parameterized.named_parameters(('Var', False), ('ResourceVar', True)) - def testBasicVector(self, use_resource_var): - """Similar to the full Adagrad update.""" - - size = 20 - init_var_np = np.zeros(size) - grad_np = np.random.rand(size) - grad_np_2 = np.random.rand(size) - - with self.cached_session() as sess: - global_step = variables.VariableV1( - 0, dtype=dtypes.int64, use_resource=use_resource_var) - var = variables.VariableV1( - init_var_np, dtype=dtypes.float32, use_resource=use_resource_var) - grad = constant_op.constant(grad_np, dtype=dtypes.float32) - grad_2 = constant_op.constant(grad_np_2, dtype=dtypes.float32) - - opt = shampoo.ShampooOptimizer(global_step) - update = opt.apply_gradients(zip([grad], [var]), - global_step=global_step) - update_2 = opt.apply_gradients(zip([grad_2], [var]), - global_step=global_step) - variables.global_variables_initializer().run() - - init_val = sess.run(var) - self.assertAllCloseAccordingToType(init_var_np, init_val) - - # Run a step of Shampoo - update.run() - new_val = sess.run(var) - - # let up compute this in numpy - # Update rule is var = var - lr * mat_g^{-0.5} * grad - # lr = 1 - mat_g = np.outer(grad_np, grad_np) / grad_np.shape[0] - mat_h = np_power(mat_g + RIDGE_EPSILON * np.eye(size), -0.5) - new_val_np = init_var_np - np.dot(mat_h, grad_np) - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - # Run another step of Shampoo - update_2.run() - new_val = sess.run(var) - - mat_g += np.outer(grad_np_2, grad_np_2) / grad_np.shape[0] - mat_h = np_power(mat_g + RIDGE_EPSILON * np.eye(size), -0.5) - new_val_np -= np.dot(mat_h, grad_np_2) - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - @parameterized.named_parameters(('Var', False), ('ResourceVar', True)) - def testBasicMatrix(self, use_resource_var): - """Check update when gradient is a matrix.""" - size = [10, 5] - init_var_np = np.zeros(size) - grad_np = np.random.rand(size[0], size[1]) - grad_np_2 = np.random.rand(size[0], size[1]) - - with self.cached_session() as sess: - global_step = variables.VariableV1( - 0, dtype=dtypes.int64, use_resource=use_resource_var) - var = variables.VariableV1( - init_var_np, dtype=dtypes.float32, use_resource=use_resource_var) - grad = constant_op.constant(grad_np, dtype=dtypes.float32) - grad_2 = constant_op.constant(grad_np_2, dtype=dtypes.float32) - - opt = shampoo.ShampooOptimizer(global_step) - update = opt.apply_gradients(zip([grad], [var]), - global_step=global_step) - update_2 = opt.apply_gradients(zip([grad_2], [var]), - global_step=global_step) - variables.global_variables_initializer().run() - - init_val = sess.run(var) - self.assertAllCloseAccordingToType(init_var_np, init_val) - - # Run a step of Shampoo - update.run() - new_val = sess.run(var) - - # let up compute this in numpy - # Update rule is var = var - lr * mat_g1^{-0.25} * grad * mat_g2^{-0.25} - # lr = 1 - mat_g1 = np.dot(grad_np, grad_np.transpose()) / grad_np.shape[0] - mat_left = np_power(mat_g1 + RIDGE_EPSILON * np.eye(size[0]), -0.25) - mat_g2 = np.dot(grad_np.transpose(), grad_np) / grad_np.shape[1] - mat_right = np_power(mat_g2 + RIDGE_EPSILON * np.eye(size[1]), -0.25) - new_val_np = init_var_np - np.dot(np.dot(mat_left, grad_np), mat_right) - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - # Run another step of Shampoo - update_2.run() - new_val = sess.run(var) - - mat_g1 += np.dot(grad_np_2, grad_np_2.transpose()) / grad_np_2.shape[0] - mat_left = np_power(mat_g1 + RIDGE_EPSILON * np.eye(size[0]), -0.25) - mat_g2 += np.dot(grad_np_2.transpose(), grad_np_2) / grad_np_2.shape[1] - mat_right = np_power(mat_g2 + RIDGE_EPSILON * np.eye(size[1]), -0.25) - new_val_np -= np.dot(np.dot(mat_left, grad_np_2), mat_right) - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - def _testBasicTensor(self, use_iterative_root, use_resource_var): - """Check update when gradient is a tensor. - - Args: - use_iterative_root: use iterative power method or SVD to find nth roots. - use_resource_var: use resource var as variables. - """ - size = [10, 5, 7] - init_var_np = np.zeros(size) - grad_np = np.random.rand(size[0], size[1], size[2]) - grad_np_2 = np.random.rand(size[0], size[1], size[2]) - - with self.cached_session() as sess: - global_step = variables.VariableV1( - 0, dtype=dtypes.int64, use_resource=use_resource_var) - var = variables.VariableV1( - init_var_np, dtype=dtypes.float32, use_resource=use_resource_var) - grad = constant_op.constant(grad_np, dtype=dtypes.float32) - grad_2 = constant_op.constant(grad_np_2, dtype=dtypes.float32) - - opt = shampoo.ShampooOptimizer(global_step, - use_iterative_root=use_iterative_root) - update = opt.apply_gradients(zip([grad], [var]), - global_step=global_step) - update_2 = opt.apply_gradients(zip([grad_2], [var]), - global_step=global_step) - variables.global_variables_initializer().run() - - init_val = sess.run(var) - self.assertAllCloseAccordingToType(init_var_np, init_val) - - # Run a step of Shampoo - update.run() - new_val = sess.run(var) - - # let up compute this in numpy - # Update rule is var = var - lr * Prod_i mat_g_i^{-0.5/3} grad - # lr = 1 - mat_g1 = ( - np.tensordot(grad_np, grad_np, axes=([1, 2], [1, 2])) / - grad_np.shape[0]) - mat_g1_a = np_power(mat_g1 + RIDGE_EPSILON * np.eye(size[0]), -0.5 / 3.0) - mat_g2 = ( - np.tensordot(grad_np, grad_np, axes=([0, 2], [0, 2])) / - grad_np.shape[1]) - mat_g2_a = np_power(mat_g2 + RIDGE_EPSILON * np.eye(size[1]), -0.5 / 3.0) - mat_g3 = ( - np.tensordot(grad_np, grad_np, axes=([0, 1], [0, 1])) / - grad_np.shape[2]) - mat_g3_a = np_power(mat_g3 + RIDGE_EPSILON * np.eye(size[2]), -0.5 / 3.0) - - precond_grad = np.tensordot(grad_np, mat_g1_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g2_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g3_a, axes=([0], [0])) - new_val_np = init_var_np - precond_grad - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - # Run another step of Shampoo - update_2.run() - new_val = sess.run(var) - - mat_g1 += ( - np.tensordot(grad_np_2, grad_np_2, axes=([1, 2], [1, 2])) / - grad_np_2.shape[0]) - mat_g1_a = np_power(mat_g1 + RIDGE_EPSILON * np.eye(size[0]), -0.5 / 3.0) - mat_g2 += ( - np.tensordot(grad_np_2, grad_np_2, axes=([0, 2], [0, 2])) / - grad_np_2.shape[1]) - mat_g2_a = np_power(mat_g2 + RIDGE_EPSILON * np.eye(size[1]), -0.5 / 3.0) - mat_g3 += ( - np.tensordot(grad_np_2, grad_np_2, axes=([0, 1], [0, 1])) / - grad_np_2.shape[2]) - mat_g3_a = np_power(mat_g3 + RIDGE_EPSILON * np.eye(size[2]), -0.5 / 3.0) - - precond_grad = np.tensordot(grad_np_2, mat_g1_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g2_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g3_a, axes=([0], [0])) - new_val_np -= precond_grad - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - @parameterized.named_parameters( - ('SVDWithVar', False, False), - ('SVDWithResourceVar', False, True), - ('IterRootWithVar', True, False), - ('IterRootWithResourceVar', True, True), - ) - def testBasicTensor(self, use_iterative_root, use_resource_var): - self._testBasicTensor(use_iterative_root, use_resource_var) - - @parameterized.named_parameters(('Var', False), ('ResourceVar', True)) - def testLargeVector(self, use_resource_var): - """This is just the diagonal Adagrad update.""" - - size = 2000 - init_var_np = np.zeros(size) - grad_np = np.random.rand(size) - grad_np_2 = np.random.rand(size) - - with self.cached_session() as sess: - global_step = variables.VariableV1( - 0, dtype=dtypes.int64, use_resource=use_resource_var) - var = variables.VariableV1( - init_var_np, dtype=dtypes.float32, use_resource=use_resource_var) - grad = constant_op.constant(grad_np, dtype=dtypes.float32) - grad_2 = constant_op.constant(grad_np_2, dtype=dtypes.float32) - - opt = shampoo.ShampooOptimizer(global_step) - update = opt.apply_gradients(zip([grad], [var]), - global_step=global_step) - update_2 = opt.apply_gradients(zip([grad_2], [var]), - global_step=global_step) - variables.global_variables_initializer().run() - - init_val = sess.run(var) - self.assertAllCloseAccordingToType(init_var_np, init_val) - - # Run a step of Shampoo - update.run() - new_val = sess.run(var) - - # let up compute this in numpy - # Update rule is var = var - lr * gg^{-0.5} * grad - # lr = 1 - mat_g = (grad_np * grad_np) - new_val_np = init_var_np - np.power(mat_g, -0.5) * grad_np - - self.assertAllCloseAccordingToType( - new_val_np, new_val, atol=TOLERANCE, rtol=TOLERANCE) - # Run another step of Shampoo - update_2.run() - new_val = sess.run(var) - - mat_g += (grad_np_2 * grad_np_2) - new_val_np -= np.power(mat_g, -0.5) * grad_np_2 - - self.assertAllCloseAccordingToType( - new_val_np, new_val, atol=TOLERANCE, rtol=TOLERANCE) - - - @parameterized.named_parameters(('Var', False), ('ResourceVar', True)) - def testLargeMatrix(self, use_resource_var): - """Gradient is a matrix, one of whose dimensions is large. - - We do diagonal updates for large dimensions. - - Args: - use_resource_var: use resource var as variables. - """ - - size = [2000, 3] - init_var_np = np.zeros(size) - grad_np = np.random.rand(size[0], size[1]) - grad_np_2 = np.random.rand(size[0], size[1]) - - with self.cached_session() as sess: - global_step = variables.VariableV1( - 0, dtype=dtypes.int64, use_resource=use_resource_var) - var = variables.VariableV1( - init_var_np, dtype=dtypes.float32, use_resource=use_resource_var) - grad = constant_op.constant(grad_np, dtype=dtypes.float32) - grad_2 = constant_op.constant(grad_np_2, dtype=dtypes.float32) - - opt = shampoo.ShampooOptimizer(global_step) - update = opt.apply_gradients(zip([grad], [var]), - global_step=global_step) - update_2 = opt.apply_gradients(zip([grad_2], [var]), - global_step=global_step) - variables.global_variables_initializer().run() - - init_val = sess.run(var) - self.assertAllCloseAccordingToType(init_var_np, init_val) - - # Run a step of Shampoo - update.run() - new_val = sess.run(var) - - # let up compute this in numpy - # Update rule is var = var - lr * mat_left * grad * mat_right - # where the mat_left * grad is just element-wise product, - # with broadcasting - # lr = 1 - - mat_g1 = np.sum( - grad_np * grad_np, axis=1, keepdims=True) / grad_np.shape[0] - mat_left = np.power(mat_g1, -0.25) - mat_g2 = np.dot(grad_np.transpose(), grad_np) / grad_np.shape[1] - mat_right = np_power(mat_g2 + RIDGE_EPSILON * np.eye(size[1]), -0.25) - new_val_np = init_var_np - np.dot(grad_np * mat_left, mat_right) - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - # Run another step of Shampoo - update_2.run() - new_val = sess.run(var) - - mat_g1 += np.sum( - grad_np_2 * grad_np_2, axis=1, keepdims=True) / grad_np_2.shape[0] - mat_left = np.power(mat_g1, -0.25) - mat_g2 += np.dot(grad_np_2.transpose(), grad_np_2) / grad_np_2.shape[1] - mat_right = np_power(mat_g2 + RIDGE_EPSILON * np.eye(size[1]), -0.25) - new_val_np -= np.dot(grad_np_2 * mat_left, mat_right) - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - @parameterized.named_parameters(('Var', False)) - def testSparseUpdateLarge(self, use_resource_var): - """Check update when gradient is of type IndexSlices. - - We do diagonal updates for the first dimension, unless it is very small. - - Args: - use_resource_var: use resource var as variables. - """ - size = [2000, 3] - sample_size_1 = 100 - init_var_np = np.zeros(size) - grad_indices = np.sort(np.random.choice(np.arange(size[0]), sample_size_1, - replace=False)) - grad_np = np.random.rand(sample_size_1, size[1]) - - sample_size_2 = 7 - grad_indices_2 = np.sort(np.random.choice(np.arange(size[0]), sample_size_2, - replace=False)) - grad_np_2 = np.random.rand(sample_size_2, size[1]) - - with self.cached_session() as sess: - global_step = variables.VariableV1( - 0, dtype=dtypes.int64, use_resource=use_resource_var) - var = variables.VariableV1( - init_var_np, dtype=dtypes.float32, use_resource=use_resource_var) - grad = ops.IndexedSlices( - constant_op.constant(grad_np, dtype=dtypes.float32), - constant_op.constant(grad_indices), - constant_op.constant(size)) - grad_2 = ops.IndexedSlices( - constant_op.constant(grad_np_2, dtype=dtypes.float32), - constant_op.constant(grad_indices_2), - constant_op.constant(size)) - - opt = shampoo.ShampooOptimizer(global_step) - update = opt.apply_gradients(zip([grad], [var]), - global_step=global_step) - update_2 = opt.apply_gradients(zip([grad_2], [var]), - global_step=global_step) - variables.global_variables_initializer().run() - - init_val = sess.run(var) - self.assertAllCloseAccordingToType(init_var_np, init_val) - - # Run a step of Shampoo - update.run() - new_val = sess.run(var) - - # let up compute this in numpy - # Update rule is var = var - lr * mat_left * grad * mat_right - # where the mat_left * grad is just element-wise product, - # with broadcasting - # lr = 1 - # In this case the update lr * mat_left * grad * mat_right is - # of size 10 x 2. - # So the correct indices of var need to be updated. - - mat_g1 = np.sum(grad_np * grad_np, axis=1, keepdims=True) - mat_g1_acc = np.zeros((size[0], 1)) - mat_g1_acc[grad_indices] += mat_g1 - mat_left = np.power(mat_g1 + RIDGE_EPSILON, -0.25) - mat_g2 = np.dot(grad_np.transpose(), grad_np) / grad_np.shape[1] - mat_right = np_power(mat_g2 + RIDGE_EPSILON * np.eye(size[1]), -0.25) - new_val_np = init_var_np - new_val_np[grad_indices, :] -= np.dot(grad_np * mat_left, mat_right) - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - # Run another step of Shampoo - update_2.run() - new_val = sess.run(var) - - mat_g1 = np.sum(grad_np_2 * grad_np_2, axis=1, keepdims=True) - mat_g1_acc[grad_indices_2] += mat_g1 - mat_left = np.power(mat_g1_acc[grad_indices_2] + RIDGE_EPSILON, -0.25) - mat_g2 += np.dot(grad_np_2.transpose(), grad_np_2) / grad_np_2.shape[1] - mat_right = np_power(mat_g2 + RIDGE_EPSILON * np.eye(size[1]), -0.25) - new_val_np[grad_indices_2, :] -= np.dot(grad_np_2 * mat_left, mat_right) - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - def _testSparseUpdateSmall(self, use_iterative_root, use_resource_var): - """Gradient is of type IndexSlices, but the first dimension is small. - - We create dense gradient and do the full update with SVD etc. - - Args: - use_iterative_root: use iterative power method or SVD to find nth roots. - use_resource_var: use resource var as variables. - """ - - size = [100, 3, 5] - sample_size = 10 - init_var_np = np.zeros(size) - grad_indices = np.sort(np.random.choice(np.arange(size[0]), sample_size, - replace=False)) - grad_np = np.random.rand(sample_size, size[1], size[2]) - - with self.cached_session() as sess: - global_step = variables.VariableV1( - 0, dtype=dtypes.int64, use_resource=use_resource_var) - var = variables.VariableV1( - init_var_np, dtype=dtypes.float32, use_resource=use_resource_var) - grad = ops.IndexedSlices( - constant_op.constant(grad_np, dtype=dtypes.float32), - constant_op.constant(grad_indices), - constant_op.constant(size)) - - opt = shampoo.ShampooOptimizer(global_step, - use_iterative_root=use_iterative_root) - update = opt.apply_gradients(zip([grad], [var]), - global_step=global_step) - variables.global_variables_initializer().run() - - init_val = sess.run(var) - self.assertAllCloseAccordingToType(init_var_np, init_val) - - # Run a step of Shampoo - update.run() - new_val = sess.run(var) - - # let up compute this in numpy - # Update rule is var = var - lr * Prod_i mat_g_i^{-0.125} grad - # lr = 1 - grad_dense = np.zeros_like(init_var_np) - grad_dense[grad_indices] = grad_np - - mat_g1 = np.tensordot( - grad_dense, grad_dense, axes=([1, 2], [1, 2])) / grad_dense.shape[0] - mat_g1_a = np_power(mat_g1 + RIDGE_EPSILON * np.eye(size[0]), -0.5 / 3.0) - mat_g2 = np.tensordot( - grad_dense, grad_dense, axes=([0, 2], [0, 2])) / grad_dense.shape[1] - mat_g2_a = np_power(mat_g2 + RIDGE_EPSILON * np.eye(size[1]), -0.5 / 3.0) - mat_g3 = np.tensordot( - grad_dense, grad_dense, axes=([0, 1], [0, 1])) / grad_dense.shape[2] - mat_g3_a = np_power(mat_g3 + RIDGE_EPSILON * np.eye(size[2]), -0.5 / 3.0) - - precond_grad = np.tensordot(grad_dense, mat_g1_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g2_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g3_a, axes=([0], [0])) - new_val_np = init_var_np - precond_grad - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - @parameterized.named_parameters( - ('SVDWithVar', False, False), - ('SVDWithResourceVar', False, True), - ('IterRootWithVar', True, False), - ('IterRootWithResourceVar', True, True), - ) - def testSparseUpdateSmall(self, use_iterative_root, use_resource_var): - self._testSparseUpdateSmall(use_iterative_root, use_resource_var) - - def _testBasicTensorWithMomentum(self, use_iterative_root, use_resource_var): - """Check update with momentum when gradient is a tensor. - - Args: - use_iterative_root: use iterative power method or SVD to find nth roots. - use_resource_var: use resource var as variables. - """ - size = [10, 5, 7] - init_var_np = np.zeros(size) - grad_np = np.random.rand(size[0], size[1], size[2]) - grad_np_2 = np.random.rand(size[0], size[1], size[2]) - gbar_decay = 0.9 - gbar_weight = 0.1 - - with self.cached_session() as sess: - global_step = variables.VariableV1( - 0, dtype=dtypes.int64, use_resource=use_resource_var) - var = variables.VariableV1( - init_var_np, dtype=dtypes.float32, use_resource=use_resource_var) - grad = constant_op.constant(grad_np, dtype=dtypes.float32) - grad_2 = constant_op.constant(grad_np_2, dtype=dtypes.float32) - - opt = shampoo.ShampooOptimizer(global_step, gbar_decay=gbar_decay, - gbar_weight=gbar_weight, - use_iterative_root=use_iterative_root) - update = opt.apply_gradients(zip([grad], [var]), - global_step=global_step) - update_2 = opt.apply_gradients(zip([grad_2], [var]), - global_step=global_step) - variables.global_variables_initializer().run() - - # Run a step of Shampoo - update.run() - new_val = sess.run(var) - - # let up compute this in numpy - # Update rule is var = var - lr * Prod_i mat_g_i^{-0.5/3} grad - # lr = 1 - mat_g1 = np.tensordot( - grad_np, grad_np, axes=([1, 2], [1, 2])) / grad_np.shape[0] - mat_g1_a = np_power(mat_g1 + RIDGE_EPSILON * np.eye(size[0]), -0.5 / 3.0) - mat_g2 = np.tensordot( - grad_np, grad_np, axes=([0, 2], [0, 2])) / grad_np.shape[1] - mat_g2_a = np_power(mat_g2 + RIDGE_EPSILON * np.eye(size[1]), -0.5 / 3.0) - mat_g3 = np.tensordot( - grad_np, grad_np, axes=([0, 1], [0, 1])) / grad_np.shape[2] - mat_g3_a = np_power(mat_g3 + RIDGE_EPSILON * np.eye(size[2]), -0.5 / 3.0) - - gbar_np = gbar_weight * grad_np - precond_grad = np.tensordot(gbar_np, mat_g1_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g2_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g3_a, axes=([0], [0])) - new_val_np = init_var_np - precond_grad - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - # Run another step of Shampoo - update_2.run() - new_val = sess.run(var) - - mat_g1 += np.tensordot( - grad_np_2, grad_np_2, axes=([1, 2], [1, 2])) / grad_np_2.shape[0] - mat_g1_a = np_power(mat_g1 + RIDGE_EPSILON * np.eye(size[0]), -0.5 / 3.0) - mat_g2 += np.tensordot( - grad_np_2, grad_np_2, axes=([0, 2], [0, 2])) / grad_np_2.shape[1] - mat_g2_a = np_power(mat_g2 + RIDGE_EPSILON * np.eye(size[1]), -0.5 / 3.0) - mat_g3 += np.tensordot( - grad_np_2, grad_np_2, axes=([0, 1], [0, 1])) / grad_np_2.shape[2] - mat_g3_a = np_power(mat_g3 + RIDGE_EPSILON * np.eye(size[2]), -0.5 / 3.0) - - gbar_np_2 = gbar_decay * gbar_np + gbar_weight * grad_np_2 - precond_grad = np.tensordot(gbar_np_2, mat_g1_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g2_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g3_a, axes=([0], [0])) - new_val_np -= precond_grad - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - @parameterized.named_parameters( - ('SVDWithVar', False, False), - ('SVDWithResourceVar', False, True), - ('IterRootWithVar', True, False), - ('IterRootWithResourceVar', True, True), - ) - def testBasicTensorWithMomentum(self, use_iterative_root, use_resource_var): - self._testBasicTensorWithMomentum(use_iterative_root, use_resource_var) - - def _testDelayedSVD(self, use_iterative_root, use_resource_var): - """Performing the SVD every nth step. - - Args: - use_iterative_root: use iterative power method or SVD to find nth roots. - use_resource_var: use resource var as variables. - """ - size = [10, 5, 7] - init_var_np = np.zeros(size).astype(np.float32) - iterations = 20 - svd_interval = 5 - grad_np = np.random.rand( - iterations, size[0], size[1], size[2]).astype(np.float32) - mat_g1_a = np.eye(size[0]) - mat_g1 = np.zeros_like(mat_g1_a) - mat_g2_a = np.eye(size[1]) - mat_g2 = np.zeros_like(mat_g2_a) - mat_g3_a = np.eye(size[2]) - mat_g3 = np.zeros_like(mat_g3_a) - - with self.cached_session() as sess: - global_step = variables.VariableV1( - 0, dtype=dtypes.int64, use_resource=use_resource_var) - var = variables.VariableV1( - init_var_np, dtype=dtypes.float32, use_resource=use_resource_var) - grad = array_ops.placeholder(dtypes.float32, shape=size) - - opt = shampoo.ShampooOptimizer(global_step, svd_interval=svd_interval, - use_iterative_root=use_iterative_root) - update = opt.apply_gradients(zip([grad], [var]), - global_step=global_step) - variables.global_variables_initializer().run() - - init_val = sess.run(var) - self.assertAllCloseAccordingToType(init_var_np, init_val) - new_val_np = init_var_np - - # Run n steps of Shampoo - for i in range(iterations): - _ = sess.run(update, feed_dict={grad: grad_np[i]}) - new_val = sess.run(var) - - # let up compute this in numpy - # Update rule is var = var - lr * Prod_i mat_g_i^{-0.5/3} grad - # lr = 1 - mat_g1 += np.tensordot( - grad_np[i], grad_np[i], axes=([1, 2], [1, 2])) / grad_np[i].shape[0] - mat_g2 += np.tensordot( - grad_np[i], grad_np[i], axes=([0, 2], [0, 2])) / grad_np[i].shape[1] - mat_g3 += np.tensordot( - grad_np[i], grad_np[i], axes=([0, 1], [0, 1])) / grad_np[i].shape[2] - if (i + 1) % svd_interval == 0: - mat_g1_a = np_power(mat_g1 + RIDGE_EPSILON * np.eye(size[0]), - -0.5 / 3.0) - mat_g2_a = np_power(mat_g2 + RIDGE_EPSILON * np.eye(size[1]), - -0.5 / 3.0) - mat_g3_a = np_power(mat_g3 + RIDGE_EPSILON * np.eye(size[2]), - -0.5 / 3.0) - - precond_grad = np.tensordot(grad_np[i], mat_g1_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g2_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g3_a, axes=([0], [0])) - new_val_np -= precond_grad - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - @parameterized.named_parameters( - ('SVDWithVar', False, False), - ('SVDWithResourceVar', False, True), - ('IterRootWithVar', True, False), - ('IterRootWithResourceVar', True, True), - ) - def testDelayedSVD(self, use_iterative_root, use_resource_var): - self._testDelayedSVD(use_iterative_root, use_resource_var) - - def _testDelayedPrecondUpdate(self, use_iterative_root, use_resource_var): - """Update the squared sum every nth step, drop the other steps. - - Args: - use_iterative_root: use iterative power method or SVD to find nth roots. - use_resource_var: use resource var as variables. - """ - size = [10, 5, 7] - init_var_np = np.zeros(size).astype(np.float32) - iterations = 100 - grad_np = np.random.rand( - iterations, size[0], size[1], size[2]).astype(np.float32) - svd_interval = 20 - precond_update_interval = 5 - mat_g1_a = np.eye(size[0]) - mat_g1 = np.zeros_like(mat_g1_a) - mat_g2_a = np.eye(size[1]) - mat_g2 = np.zeros_like(mat_g2_a) - mat_g3_a = np.eye(size[2]) - mat_g3 = np.zeros_like(mat_g3_a) - - with self.cached_session() as sess: - global_step = variables.VariableV1( - 0, dtype=dtypes.int64, use_resource=use_resource_var) - var = variables.VariableV1( - init_var_np, dtype=dtypes.float32, use_resource=use_resource_var) - grad = array_ops.placeholder(dtypes.float32, shape=size) - - opt = shampoo.ShampooOptimizer( - global_step, svd_interval=svd_interval, - precond_update_interval=precond_update_interval, - use_iterative_root=use_iterative_root) - update = opt.apply_gradients(zip([grad], [var]), - global_step=global_step) - variables.global_variables_initializer().run() - - init_val = sess.run(var) - self.assertAllCloseAccordingToType(init_var_np, init_val) - new_val_np = init_var_np - - # Run n steps of Shampoo - for i in range(iterations): - _ = sess.run(update, feed_dict={grad: grad_np[i]}) - new_val = sess.run(var) - - # let up compute this in numpy - # Update rule is var = var - lr * Prod_i mat_g_i^{-0.5/3} grad - # lr = 1 - if (i + 1) % precond_update_interval == 0: - mat_g1 += ( - np.tensordot(grad_np[i], grad_np[i], axes=([1, 2], [1, 2])) / - grad_np[i].shape[0] * precond_update_interval) - mat_g2 += ( - np.tensordot(grad_np[i], grad_np[i], axes=([0, 2], [0, 2])) / - grad_np[i].shape[1] * precond_update_interval) - mat_g3 += ( - np.tensordot(grad_np[i], grad_np[i], axes=([0, 1], [0, 1])) / - grad_np[i].shape[2] * precond_update_interval) - - if (i + 1) % svd_interval == 0: - mat_g1_a = np_power(mat_g1 + RIDGE_EPSILON * np.eye(size[0]), - -0.5 / 3.0) - mat_g2_a = np_power(mat_g2 + RIDGE_EPSILON * np.eye(size[1]), - -0.5 / 3.0) - mat_g3_a = np_power(mat_g3 + RIDGE_EPSILON * np.eye(size[2]), - -0.5 / 3.0) - - precond_grad = np.tensordot(grad_np[i], mat_g1_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g2_a, axes=([0], [0])) - precond_grad = np.tensordot(precond_grad, mat_g3_a, axes=([0], [0])) - new_val_np -= precond_grad - - self.assertAllCloseAccordingToType(new_val_np, new_val, - atol=TOLERANCE, rtol=TOLERANCE) - - @parameterized.named_parameters( - ('SVDWithVar', False, False), - ('SVDWithResourceVar', False, True), - ('IterRootWithVar', True, False), - ('IterRootWithResourceVar', True, True), - ) - def testDelayedPrecondUpdate(self, use_iterative_root, use_resource_var): - self._testDelayedPrecondUpdate(use_iterative_root, use_resource_var) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/opt/python/training/sign_decay.py b/tensorflow/contrib/opt/python/training/sign_decay.py deleted file mode 100644 index 99cd0f6e60e..00000000000 --- a/tensorflow/contrib/opt/python/training/sign_decay.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implementation of the sign decay functions used in PowerSign and AddSign. - -See [Bello et al., ICML 2017] Neural Optimizer Search with Reinforcement -Learning for details. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops - - -def get_linear_decay_fn(decay_steps): - """Returns a function that computes a linear decay. - - This decay computes linear annealing: - max(0, (decay_steps - global_step) / decay_steps) - - Example usage: - ``` - decay_steps = 1000 - linear_decay_fn = get_linear_decay_fn(decay_steps) - decayed = linear_decay_fn(global_step) - x *= decayed - ``` - Args: - decay_steps: number of steps to decay over. - Returns: - linear_decay_fn: a function that computes the linear decay. - """ - # pylint:disable=missing-docstring - def linear_decay_fn(global_step): - if global_step is None: - raise ValueError("global_step is required for linear_decay.") - global_step = math_ops.minimum(global_step, decay_steps) - remaining_steps = math_ops.cast( - decay_steps, dtypes.int32) - math_ops.cast(global_step, dtypes.int32) - decayed = (math_ops.cast(remaining_steps, dtypes.float32) / - math_ops.cast(decay_steps, dtypes.float32)) - return math_ops.maximum(0.0, decayed) - # pylint:enable=missing-docstring - return linear_decay_fn - - -def get_cosine_decay_fn(decay_steps, num_periods=0.5, zero_after=None): - """Returns a function that computes a cosine decay. - - This decay computes cosine annealing: - 0.5 * (1.0 + cos(2.0 * pi * num_periods * global_step / decay_steps)) - - This decay can be used to decay the sign quantity in the AddSign and PowerSign - optimizers discovered in - [Bello et al., ICML 2017] Neural Optimizer Search with RL. - - Example usage: - ``` - decay_steps = 1000 - num_periods = 2 - cosine_decay_fn = get_cosine_decay_fn(decay_steps, num_periods=num_periods) - decayed = cosine_decay_fn(global_step) - x *= decayed - ``` - Args: - decay_steps: number of steps to decay over. - num_periods: number of periods for cosine signal. 0.5 by default, - which maps the last decay step to 0. - zero_after: if not None, number after which the decay function - will just return 0. - Returns: - cosine_decay_fn: a function that computes the cosine decay. - """ - # pylint:disable=missing-docstring - def cosine_decay_fn(global_step): - if global_step is None: - raise ValueError("global_step is required for cosine_decay.") - global_step = math_ops.minimum(global_step, decay_steps) - completed_fraction = (math_ops.cast(global_step, dtypes.float32) / - math_ops.cast(decay_steps, dtypes.float32)) - fraction = 2.0 * num_periods * completed_fraction - decayed = 0.5 * ( - 1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction)) - if zero_after is not None: - decayed = array_ops.where( - math_ops.greater_equal(fraction, 2 * zero_after), 0.0, decayed) - return decayed - # pylint:enable=missing-docstring - return cosine_decay_fn - - -def get_restart_decay_fn(decay_steps, num_periods=1, zero_after=None): - """Returns a function that computes a restart decay. - - This decay computes - 0.5 * (1.0 + cos(pi * (num_periods * global_step) % num_training_steps)) - - This is a simplified version of the restart decay introduced in - "SGDR: Stochastic Gradient Descent with Warm Restarts" - by Ilya Loshchilov & Frank Hutter, Proceedings of - ICLR'2017, available at https://arxiv.org/pdf/1608.03983.pdf - - This decay can be used to decay the sign quantity in the AddSign and PowerSign - optimizers discovered in - [Bello et al., ICML 2017] Neural Optimizer Search with RL. - - Example usage: - ``` - decay_steps = 1000 - num_periods = 2.0 - restart_decay_fn = get_restart_decay_fn(decay_steps, - num_periods=num_periods) - decayed = restart_decay_fn(global_step) - x *= decayed - ``` - Args: - decay_steps: number of steps to decay over. - num_periods: number of periods for cosine signal. 1 by default, - which maps the last decay step to 0. - zero_after: if not None, number after which the decay function - will return 0. - Returns: - restart_decay_fn: a function that computes the restart decay. - """ - # pylint:disable=missing-docstring - def restart_decay_fn(global_step): - if global_step is None: - raise ValueError("global_step is required for cosine_decay.") - global_step = math_ops.minimum(global_step, decay_steps) - num = math_ops.mod(num_periods * math_ops.cast(global_step, dtypes.float32), - decay_steps) - fraction = num / math_ops.cast(decay_steps, dtypes.float32) - decayed = 0.5 * ( - 1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction)) - if zero_after is not None: - tmp = (math_ops.cast(num_periods * global_step, dtypes.float32) / - math_ops.cast(decay_steps, dtypes.float32)) - decayed = array_ops.where( - math_ops.greater_equal(tmp, zero_after), 0.0, decayed) - return decayed - # pylint:enable=missing-docstring - return restart_decay_fn diff --git a/tensorflow/contrib/opt/python/training/sign_decay_test.py b/tensorflow/contrib/opt/python/training/sign_decay_test.py deleted file mode 100644 index 3a84789afd7..00000000000 --- a/tensorflow/contrib/opt/python/training/sign_decay_test.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for sign_decay.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -from tensorflow.contrib.opt.python.training import sign_decay -from tensorflow.python.platform import test - - -def py_linear_decay_fn(decay_steps): - - def linear_decay(step): - step = min(step, decay_steps) - return float(decay_steps - step) / decay_steps - - return linear_decay - - -def py_cosine_decay_fn(decay_steps, num_periods=0.5, zero_after=None): - - def cosine_decay(step): - step = min(step, decay_steps) - fraction = 2.0 * num_periods * step / float(decay_steps) - if zero_after is not None and fraction >= 2 * zero_after: - return 0.0 - return 0.5 * (1.0 + math.cos(math.pi * fraction)) - - return cosine_decay - - -def py_restart_decay_fn(decay_steps, num_periods=1, zero_after=None): - - def restart_decay(step): - step = min(step, decay_steps) - tmp = num_periods * step / float(decay_steps) - fraction = ( - num_periods * step % decay_steps) / float(decay_steps) - if zero_after is not None and tmp >= zero_after: - return 0 - return 0.5 * (1.0 + math.cos(math.pi * fraction)) - - return restart_decay - - -class SignDecaysTest(test.TestCase): - - def testLinearDecay(self): - num_training_steps = 1000 - linear_decay_fn = sign_decay.get_linear_decay_fn(num_training_steps) - - for step in range(0, 1000, 100): - with self.cached_session(): - tf_decayed = linear_decay_fn(step).eval() - py_decayed = py_linear_decay_fn(num_training_steps)(step) - self.assertAlmostEqual(tf_decayed, py_decayed, places=4) - - def testCosineDecay(self): - num_training_steps = 1000 - cosine_decay_fn = sign_decay.get_cosine_decay_fn(num_training_steps) - cosine_decay_2_fn = sign_decay.get_cosine_decay_fn( - num_training_steps, num_periods=5, zero_after=2) - - for step in range(0, 1000, 100): - with self.cached_session(): - tf_decayed = cosine_decay_fn(step).eval() - py_decayed = py_cosine_decay_fn(num_training_steps)(step) - self.assertAlmostEqual(tf_decayed, py_decayed, places=4) - - tf_decayed = cosine_decay_2_fn(step).eval() - py_decayed = py_cosine_decay_fn( - num_training_steps, num_periods=5, zero_after=2)(step) - self.assertAlmostEqual(tf_decayed, py_decayed, places=4) - - def testRestartDecay(self): - num_training_steps = 1000 - restart_decay_fn = sign_decay.get_restart_decay_fn(num_training_steps) - restart_decay_2_fn = sign_decay.get_restart_decay_fn( - num_training_steps, num_periods=5, zero_after=2) - - for step in range(0, 1000, 100): - with self.cached_session(): - tf_decayed = restart_decay_fn(step).eval() - py_decayed = py_restart_decay_fn(num_training_steps)(step) - self.assertAlmostEqual(tf_decayed, py_decayed, places=4) - - tf_decayed = restart_decay_2_fn(step).eval() - py_decayed = py_restart_decay_fn( - num_training_steps, num_periods=5, zero_after=2)(step) - self.assertAlmostEqual(tf_decayed, py_decayed, places=4) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/opt/python/training/variable_clipping_optimizer.py b/tensorflow/contrib/opt/python/training/variable_clipping_optimizer.py deleted file mode 100644 index 3c0b8394be5..00000000000 --- a/tensorflow/contrib/opt/python/training/variable_clipping_optimizer.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Delegating optimizer to clip norm for specified variables.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import contextlib - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import optimizer - -__all__ = ["VariableClippingOptimizer"] - - -class VariableClippingOptimizer(optimizer.Optimizer): - """Wrapper optimizer that clips the norm of specified variables after update. - - This optimizer delegates all aspects of gradient calculation and application - to an underlying optimizer. After applying gradients, this optimizer then - clips the variable to have a maximum L2 norm along specified dimensions. - NB: this is quite different from clipping the norm of the gradients. - - Multiple instances of `VariableClippingOptimizer` may be chained to specify - different max norms for different subsets of variables. - - This is more efficient at serving-time than using normalization during - embedding lookup, at the expense of more expensive training and fewer - guarantees about the norms. - - @@__init__ - - """ - - def __init__(self, - opt, - vars_to_clip_dims, - max_norm, - use_locking=False, - colocate_clip_ops_with_vars=False, - name="VariableClipping"): - """Construct a new clip-norm optimizer. - - Args: - opt: The actual optimizer that will be used to compute and apply the - gradients. Must be one of the Optimizer classes. - vars_to_clip_dims: A dict with keys as Variables and values as lists - of dimensions along which to compute the L2-norm. See - `tf.clip_by_norm` for more details. - max_norm: The L2-norm to clip to, for all variables specified. - use_locking: If `True` use locks for clip update operations. - colocate_clip_ops_with_vars: If `True`, try colocating the clip norm - ops with the corresponding variable. - name: Optional name prefix for the operations created when applying - gradients. Defaults to "VariableClipping". - """ - super(VariableClippingOptimizer, self).__init__(use_locking, name) - self._opt = opt - # Defensive copy of input dict - self._vars_to_clip_dims = { - var: clip_dims[:] for var, clip_dims in vars_to_clip_dims.items()} - self._max_norm = max_norm - self._colocate_clip_ops_with_vars = colocate_clip_ops_with_vars - - def compute_gradients(self, *args, **kwargs): - return self._opt.compute_gradients(*args, **kwargs) - - def get_slot(self, *args, **kwargs): - return self._opt.get_slot(*args, **kwargs) - - def get_slot_names(self, *args, **kwargs): - return self._opt.get_slot_names(*args, **kwargs) - - def apply_gradients(self, grads_and_vars, global_step=None, name=None): - with ops.name_scope(name, self._name) as name: - update_op = self._opt.apply_gradients( - grads_and_vars, global_step=global_step) - clip_update_ops = [] - with ops.control_dependencies([update_op]): - for grad, var in grads_and_vars: - if grad is None or var not in self._vars_to_clip_dims: - continue - with ops.name_scope("clip_" + var.op.name): - if isinstance(grad, ops.Tensor): - clip_update_ops.append(self._clip_dense(var)) - else: - clip_update_ops.append(self._clip_sparse(grad, var)) - - # In case no var was clipped, still need to run the update_op. - return control_flow_ops.group(*([update_op] + clip_update_ops), name=name) - - def _clip_dense(self, var): - with self._maybe_colocate_with(var): - updated_var_value = var.read_value() - normalized_var = clip_ops.clip_by_norm( - updated_var_value, self._max_norm, self._vars_to_clip_dims[var]) - delta = updated_var_value - normalized_var - with ops.colocate_with(var): - return var.assign_sub(delta, use_locking=self._use_locking) - - def _clip_sparse(self, grad, var): - assert isinstance(grad, ops.IndexedSlices) - clip_dims = self._vars_to_clip_dims[var] - if 0 in clip_dims: - logging.warning("Clipping norm across dims %s for %s is inefficient " - "when including sparse dimension 0.", clip_dims, - var.op.name) - return self._clip_dense(var) - - with ops.colocate_with(var): - var_subset = array_ops.gather(var, grad.indices) - with self._maybe_colocate_with(var): - normalized_var_subset = clip_ops.clip_by_norm( - var_subset, self._max_norm, clip_dims) - delta = ops.IndexedSlices( - var_subset - normalized_var_subset, grad.indices, grad.dense_shape) - with ops.colocate_with(var): - return var.scatter_sub(delta, use_locking=self._use_locking) - - @contextlib.contextmanager - def _maybe_colocate_with(self, var): - """Context to colocate with `var` if `colocate_clip_ops_with_vars`.""" - if self._colocate_clip_ops_with_vars: - with ops.colocate_with(var): - yield - else: - yield diff --git a/tensorflow/contrib/opt/python/training/variable_clipping_optimizer_test.py b/tensorflow/contrib/opt/python/training/variable_clipping_optimizer_test.py deleted file mode 100644 index ff0ea8d7669..00000000000 --- a/tensorflow/contrib/opt/python/training/variable_clipping_optimizer_test.py +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for VariableClippingOptimizer.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import contextlib -import socket -import numpy as np -from tensorflow.contrib.opt.python.training import variable_clipping_optimizer -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import gradient_descent -from tensorflow.python.training import server_lib - - -class VariableClippingOptimizerTest(test.TestCase): - - def _setupCluster(self): - - def get_open_port(): - try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - except IOError: - s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - s.bind(("", 0)) - port = s.getsockname()[1] - s.close() - return port - - port1 = get_open_port() - port2 = get_open_port() - cs = server_lib.ClusterSpec({ - "worker": ["localhost:%s" % port1], - "ps": ["localhost:%s" % port2] - }) - - worker = server_lib.Server(cs, job_name="worker", start=True) - ps = server_lib.Server(cs, job_name="ps", start=True) - - return worker, ps - - @contextlib.contextmanager - def _maybeWithDevice(self, device): - if device is not None: - with ops.device(device): - yield - else: - yield - - def _setupDense(self, is_distributed, dtype): - with self._maybeWithDevice("/job:ps" if is_distributed else None): - var0 = variables.Variable([[0.0, 1.0], [2.0, 3.0]], dtype=dtype) - var1 = variables.Variable([4.0, 5.0], dtype=dtype) - with self._maybeWithDevice("/job:worker" if is_distributed else None): - grads0 = constant_op.constant([[0.1, 0.1], [0.1, 0.1]], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - sgd = gradient_descent.GradientDescentOptimizer(3.0) - clip_opt = variable_clipping_optimizer.VariableClippingOptimizer( - sgd, {var0: [1]}, 2.0) - - update_op = clip_opt.apply_gradients( - list(zip([grads0, grads1], [var0, var1]))) - variables.global_variables_initializer().run() - return var0, var1, update_op - - def _assertDenseCorrect(self, var0, var1, update_op): - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[0.0, 1.0], [2.0, 3.0]], var0.eval()) - self.assertAllCloseAccordingToType([4.0, 5.0], var1.eval()) - - # Run 1 step of sgd, clipping each var0[i] to max L2-norm 2.0 - update_op.run() - # Validate updated params - var0_out = var0.eval() - # var0[0] has norm < 2.0, so it is not clipped. - self.assertAllCloseAccordingToType([(0.0 - 3.0 * 0.1), (1.0 - 3.0 * 0.1)], - var0_out[0]) - # var0[1] has norm > 2.0, so it is clipped. - expected_unclipped = np.array([(2.0 - 3.0 * 0.1), (3.0 - 3.0 * 0.1)]) - self.assertAllCloseAccordingToType(2.0 * expected_unclipped / - np.linalg.norm(expected_unclipped), - var0_out[1]) - # var1 is not in the var list, so it should not be clipped - self.assertAllCloseAccordingToType([4.0 - 3.0 * 0.01, 5.0 - 3.0 * 0.01], - var1.eval()) - - def _setupSparse(self, is_distributed, dtype): - with self._maybeWithDevice("/job:ps" if is_distributed else None): - var0 = variables.Variable( - [[0.0, 1.0], [2.0, 3.0], [4.0, 5.0]], dtype=dtype) - var1 = variables.Variable( - [[0.0, 1.0], [0.0, 3.0], [0.0, 5.0]], dtype=dtype) - with self._maybeWithDevice("/job:worker" if is_distributed else None): - grads = ops.IndexedSlices( - constant_op.constant( - [[0.1, 0.1], [0.1, 0.1]], dtype=dtype), [0, 2], [3, 2]) - sgd = gradient_descent.GradientDescentOptimizer(3.0) - clip_opt = variable_clipping_optimizer.VariableClippingOptimizer( - sgd, {var0: [1], - var1: [0]}, 2.0) - update_op = clip_opt.apply_gradients( - list(zip([grads, grads], [var0, var1]))) - variables.global_variables_initializer().run() - return var0, var1, update_op - - def _assertSparseCorrect(self, var0, var1, update_op): - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[0.0, 1.0], [2.0, 3.0], [4.0, 5.0]], - var0.eval()) - self.assertAllCloseAccordingToType([[0.0, 1.0], [0.0, 3.0], [0.0, 5.0]], - var1.eval()) - - # Run 1 step of sgd - update_op.run() - - # var1 is clipped along the sparse dimension, so defaults to using dense - # calculations. There should be a warning logged, but the numerics - # should still be correct. - var1_out = var1.eval() - # var1[:, 0] has norm < 2.0, so it is not clipped. - self.assertAllCloseAccordingToType( - [(0.0 - 3.0 * 0.1), 0.0, (0.0 - 3.0 * 0.1)], var1_out[:, 0]) - # var1[:, 1] has norm > 2.0, so it is clipped. - expected_unclipped = np.array([(1.0 - 3.0 * 0.1), 3.0, (5.0 - 3.0 * 0.1)]) - self.assertAllCloseAccordingToType(2.0 * expected_unclipped / - np.linalg.norm(expected_unclipped), - var1_out[:, 1]) - - # Validate updated params - var0_out = var0.eval() - # var0[0] has norm < 2.0, so it is not clipped. - self.assertAllCloseAccordingToType([(0.0 - 3.0 * 0.1), (1.0 - 3.0 * 0.1)], - var0_out[0]) - # var0[1] has no gradients, so it should remain unchanged. - self.assertAllCloseAccordingToType([2.0, 3.0], var0_out[1]) - # var0[2] has norm > 2.0, so it is clipped. - expected_unclipped = np.array([(4.0 - 3.0 * 0.1), (5.0 - 3.0 * 0.1)]) - self.assertAllCloseAccordingToType(2.0 * expected_unclipped / - np.linalg.norm(expected_unclipped), - var0_out[2]) - - def testDenseLocal(self): - for dtype in [dtypes.float32, dtypes.float64, dtypes.half]: - with self.cached_session(): - var0, var1, update_op = self._setupDense(False, dtype) - self._assertDenseCorrect(var0, var1, update_op) - - def testDenseDistributed(self): - worker, unused_ps = self._setupCluster() - for dtype in [dtypes.float64, dtypes.half, dtypes.float32]: - with session.Session(worker.target): - var0, var1, update_op = self._setupDense(True, dtype) - self._assertDenseCorrect(var0, var1, update_op) - - def testSparseLocal(self): - for dtype in [dtypes.float64, dtypes.float32, dtypes.half]: - with self.cached_session(): - var0, var1, update_op = self._setupSparse(False, dtype) - self._assertSparseCorrect(var0, var1, update_op) - - def testSparseDistributed(self): - worker, unused_ps = self._setupCluster() - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with session.Session(worker.target): - var0, var1, update_op = self._setupSparse(True, dtype) - self._assertSparseCorrect(var0, var1, update_op) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/opt/python/training/weight_decay_optimizers.py b/tensorflow/contrib/opt/python/training/weight_decay_optimizers.py deleted file mode 100644 index 233503b911e..00000000000 --- a/tensorflow/contrib/opt/python/training/weight_decay_optimizers.py +++ /dev/null @@ -1,483 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Base class to make optimizers weight decay ready.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.opt.python.training import shampoo -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.training import adam -from tensorflow.python.training import momentum as momentum_opt -from tensorflow.python.training import optimizer -from tensorflow.python.util.tf_export import tf_export -from tensorflow.python.ops import array_ops - - -class DecoupledWeightDecayExtension(object): - """This class allows to extend optimizers with decoupled weight decay. - - It implements the decoupled weight decay described by Loshchilov & Hutter - (https://arxiv.org/pdf/1711.05101.pdf), in which the weight decay is - decoupled from the optimization steps w.r.t. to the loss function. - For SGD variants, this simplifies hyperparameter search since it decouples - the settings of weight decay and learning rate. - For adaptive gradient algorithms, it regularizes variables with large - gradients more than L2 regularization would, which was shown to yield better - training loss and generalization error in the paper above. - - This class alone is not an optimizer but rather extends existing - optimizers with decoupled weight decay. We explicitly define the two examples - used in the above paper (SGDW and AdamW), but in general this can extend - any OptimizerX by using - `extend_with_weight_decay(OptimizerX, weight_decay=weight_decay)`. - In order for it to work, it must be the first class the Optimizer with - weight decay inherits from, e.g. - - ```python - class AdamWOptimizer(DecoupledWeightDecayExtension, adam.AdamOptimizer): - def __init__(self, weight_decay, *args, **kwargs): - super(AdamWOptimizer, self).__init__(weight_decay, *args, **kwargs). - ``` - - Note that this extension decays weights BEFORE applying the update based - on the gradient, i.e. this extension only has the desired behaviour for - optimizers which do not depend on the value of'var' in the update step! - - Note: when applying a decay to the learning rate, be sure to manually apply - the decay to the `weight_decay` as well. For example: - - ```python - schedule = - tf.compat.v1.train.piecewise_constant(tf.compat.v1.train.get_global_step(), - [10000, 15000], [1e-0, 1e-1, 1e-2]) - lr = 1e-1 * schedule() - wd = lambda: 1e-4 * schedule() - - # ... - - optimizer = tf.contrib.opt.MomentumWOptimizer(learning_rate=lr, - weight_decay=wd, - momentum=0.9, - use_nesterov=True) - ``` - """ - - def __init__(self, weight_decay, **kwargs): - """Construct the extension class that adds weight decay to an optimizer. - - Args: - weight_decay: A `Tensor` or a floating point value, the factor by which a - variable is decayed in the update step. - **kwargs: Optional list or tuple or set of `Variable` objects to decay. - """ - self._decay_var_list = None # is set in minimize or apply_gradients - self._weight_decay = weight_decay - # The tensors are initialized in call to _prepare - self._weight_decay_tensor = None - super(DecoupledWeightDecayExtension, self).__init__(**kwargs) - - def minimize(self, - loss, - global_step=None, - var_list=None, - gate_gradients=optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - name=None, - grad_loss=None, - decay_var_list=None): - """Add operations to minimize `loss` by updating `var_list` with decay. - - This function is the same as Optimizer.minimize except that it allows to - specify the variables that should be decayed using decay_var_list. - If decay_var_list is None, all variables in var_list are decayed. - - For more information see the documentation of Optimizer.minimize. - - Args: - loss: A `Tensor` containing the value to minimize. - global_step: Optional `Variable` to increment by one after the variables - have been updated. - var_list: Optional list or tuple of `Variable` objects to update to - minimize `loss`. Defaults to the list of variables collected in the - graph under the key `GraphKeys.TRAINABLE_VARIABLES`. - gate_gradients: How to gate the computation of gradients. Can be - `GATE_NONE`, `GATE_OP`, or `GATE_GRAPH`. - aggregation_method: Specifies the method used to combine gradient terms. - Valid values are defined in the class `AggregationMethod`. - colocate_gradients_with_ops: If True, try colocating gradients with the - corresponding op. - name: Optional name for the returned operation. - grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`. - decay_var_list: Optional list of decay variables. - - Returns: - An Operation that updates the variables in `var_list`. If `global_step` - was not `None`, that operation also increments `global_step`. - - """ - self._decay_var_list = set(decay_var_list) if decay_var_list else False - return super(DecoupledWeightDecayExtension, self).minimize( - loss, - global_step=global_step, - var_list=var_list, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, - name=name, - grad_loss=grad_loss) - - def apply_gradients(self, - grads_and_vars, - global_step=None, - name=None, - decay_var_list=None): - """Apply gradients to variables and decay the variables. - - This function is the same as Optimizer.apply_gradients except that it - allows to specify the variables that should be decayed using - decay_var_list. If decay_var_list is None, all variables in var_list - are decayed. - - For more information see the documentation of Optimizer.apply_gradients. - - Args: - grads_and_vars: List of (gradient, variable) pairs as returned by - `compute_gradients()`. - global_step: Optional `Variable` to increment by one after the variables - have been updated. - name: Optional name for the returned operation. Default to the name - passed to the `Optimizer` constructor. - decay_var_list: Optional list of decay variables. - - Returns: - An `Operation` that applies the specified gradients. If `global_step` - was not None, that operation also increments `global_step`. - """ - self._decay_var_list = set(decay_var_list) if decay_var_list else False - return super(DecoupledWeightDecayExtension, self).apply_gradients( - grads_and_vars, global_step=global_step, name=name) - - def _prepare(self): - weight_decay = self._weight_decay - if callable(weight_decay): - weight_decay = weight_decay() - self._weight_decay_tensor = ops.convert_to_tensor( - weight_decay, name="weight_decay") - # Call the optimizers _prepare function. - super(DecoupledWeightDecayExtension, self)._prepare() - - def _decay_weights_op(self, var): - if not self._decay_var_list or var in self._decay_var_list: - return var.assign_sub(self._weight_decay * var, self._use_locking) - return control_flow_ops.no_op() - - def _decay_weights_sparse_op(self, var, indices, scatter_add): - if not self._decay_var_list or var in self._decay_var_list: - update = -self._weight_decay * array_ops.gather(var, indices) - return scatter_add(var, indices, update, self._use_locking) - return control_flow_ops.no_op() - - # Here, we overwrite the apply functions that the base optimizer calls. - # super().apply_x resolves to the apply_x function of the BaseOptimizer. - def _apply_dense(self, grad, var): - with ops.control_dependencies([self._decay_weights_op(var)]): - return super(DecoupledWeightDecayExtension, self)._apply_dense(grad, var) - - def _resource_apply_dense(self, grad, var): - with ops.control_dependencies([self._decay_weights_op(var)]): - return super(DecoupledWeightDecayExtension, - self)._resource_apply_dense(grad, var) - - def _apply_sparse(self, grad, var): - scatter_add = state_ops.scatter_add - decay_op = self._decay_weights_sparse_op(var, grad.indices, scatter_add) - with ops.control_dependencies([decay_op]): - return super(DecoupledWeightDecayExtension, self)._apply_sparse(grad, var) - - def _resource_scatter_add(self, x, i, v, _=None): - # last argument allows for one overflow argument, to have the same function - # signature as state_ops.scatter_add - with ops.control_dependencies( - [resource_variable_ops.resource_scatter_add(x.handle, i, v)]): - return x.value() - - def _resource_apply_sparse(self, grad, var, indices): - scatter_add = self._resource_scatter_add - decay_op = self._decay_weights_sparse_op(var, indices, scatter_add) - with ops.control_dependencies([decay_op]): - return super(DecoupledWeightDecayExtension, - self)._resource_apply_sparse(grad, var, indices) - - -def extend_with_decoupled_weight_decay(base_optimizer): - """Factory function returning an optimizer class with decoupled weight decay. - - Returns an optimizer class. An instance of the returned class computes the - update step of `base_optimizer` and additionally decays the weights. - E.g., the class returned by - `extend_with_decoupled_weight_decay(tf.compat.v1.train.AdamOptimizer)` is - equivalent to - `tf.contrib.opt.AdamWOptimizer`. - - The API of the new optimizer class slightly differs from the API of the - base optimizer: - - The first argument to the constructor is the weight decay rate. - - `minimize` and `apply_gradients` accept the optional keyword argument - `decay_var_list`, which specifies the variables that should be decayed. - If `None`, all variables that are optimized are decayed. - - Usage example: - ```python - # MyAdamW is a new class - MyAdamW = extend_with_decoupled_weight_decay(tf.compat.v1.train.AdamOptimizer) - # Create a MyAdamW object - optimizer = MyAdamW(weight_decay=0.001, learning_rate=0.001) - sess.run(optimizer.minimize(loss, decay_variables=[var1, var2])) - - Note that this extension decays weights BEFORE applying the update based - on the gradient, i.e. this extension only has the desired behaviour for - optimizers which do not depend on the value of'var' in the update step! - ``` - - Args: - base_optimizer: An optimizer class that inherits from tf.train.Optimizer. - - Returns: - A new optimizer class that inherits from DecoupledWeightDecayExtension - and base_optimizer. - """ - - class OptimizerWithDecoupledWeightDecay(DecoupledWeightDecayExtension, - base_optimizer): - """Base_optimizer with decoupled weight decay. - - This class computes the update step of `base_optimizer` and - additionally decays the variable with the weight decay being decoupled from - the optimization steps w.r.t. to the loss function, as described by - Loshchilov & Hutter (https://arxiv.org/pdf/1711.05101.pdf). - For SGD variants, this simplifies hyperparameter search since - it decouples the settings of weight decay and learning rate. - For adaptive gradient algorithms, it regularizes variables with large - gradients more than L2 regularization would, which was shown to yield - better training loss and generalization error in the paper above. - """ - - def __init__(self, weight_decay, *args, **kwargs): - # super delegation is necessary here - # pylint: disable=useless-super-delegation - super(OptimizerWithDecoupledWeightDecay, - self).__init__(weight_decay, *args, **kwargs) - # pylint: enable=useless-super-delegation - - return OptimizerWithDecoupledWeightDecay - - -@tf_export("contrib.opt.MomentumWOptimizer") -class MomentumWOptimizer(DecoupledWeightDecayExtension, - momentum_opt.MomentumOptimizer): - """Optimizer that implements the Momentum algorithm with weight_decay. - - This is an implementation of the SGDW optimizer described in "Fixing - Weight Decay Regularization in Adam" by Loshchilov & Hutter - (https://arxiv.org/abs/1711.05101) - ([pdf])(https://arxiv.org/pdf/1711.05101.pdf). - It computes the update step of `train.MomentumOptimizer` and additionally - decays the variable. Note that this is different from adding - L2 regularization on the variables to the loss. Decoupling the weight decay - from other hyperparameters (in particular the learning rate) simplifies - hyperparameter search. - - For further information see the documentation of the Momentum Optimizer. - - Note that this optimizer can also be instantiated as - ```python - extend_with_weight_decay(tf.compat.v1.train.MomentumOptimizer, - weight_decay=weight_decay) - ``` - """ - - def __init__(self, - weight_decay, - learning_rate, - momentum, - use_locking=False, - name="MomentumW", - use_nesterov=False): - """Construct a new MomentumW optimizer. - - For further information see the documentation of the Momentum Optimizer. - - Args: - weight_decay: A `Tensor` or a floating point value. The weight decay. - learning_rate: A `Tensor` or a floating point value. The learning rate. - momentum: A `Tensor` or a floating point value. The momentum. - use_locking: If `True` use locks for update operations. - name: Optional name prefix for the operations created when applying - gradients. Defaults to "Momentum". - use_nesterov: If `True` use Nesterov Momentum. See [Sutskever et al., - 2013]( - http://jmlr.org/proceedings/papers/v28/sutskever13.pdf). This - implementation always computes gradients at the value of the - variable(s) passed to the optimizer. Using Nesterov Momentum makes the - variable(s) track the values called `theta_t + mu*v_t` in the paper. - @compatibility(eager) When eager execution is enabled, learning_rate, - weight_decay and momentum can each be a callable that takes no - arguments and returns the actual value to use. This can be useful for - changing these values across different invocations of optimizer - functions. @end_compatibility - """ - super(MomentumWOptimizer, self).__init__( - weight_decay, - learning_rate=learning_rate, - momentum=momentum, - use_locking=use_locking, - name=name, - use_nesterov=use_nesterov) - - -@tf_export("contrib.opt.AdamWOptimizer") -class AdamWOptimizer(DecoupledWeightDecayExtension, adam.AdamOptimizer): - """Optimizer that implements the Adam algorithm with weight decay. - - This is an implementation of the AdamW optimizer described in ["Fixing - Weight Decay Regularization in Adam" by Loshchilov & Hutter] - (https://arxiv.org/abs/1711.05101) - ([pdf](https://arxiv.org/pdf/1711.05101.pdf)). - - It computes the update step of `train.AdamOptimizer` and additionally decays - the variable. Note that this is different from adding L2 regularization on - the variables to the loss: it regularizes variables with large - gradients more than L2 regularization would, which was shown to yield better - training loss and generalization error in the paper above. - - For further information see the documentation of the Adam Optimizer. - - Note that this optimizer can also be instantiated as - ```python - extend_with_weight_decay(tf.compat.v1.train.AdamOptimizer, - weight_decay=weight_decay) - ``` - """ - - def __init__(self, - weight_decay, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - use_locking=False, - name="AdamW"): - """Construct a new AdamW optimizer. - - For further information see the documentation of the Adam Optimizer. - - Args: - weight_decay: A `Tensor` or a floating point value. The weight decay. - learning_rate: A Tensor or a floating point value. The learning rate. - beta1: A float value or a constant float tensor. The exponential decay - rate for the 1st moment estimates. - beta2: A float value or a constant float tensor. The exponential decay - rate for the 2nd moment estimates. - epsilon: A small constant for numerical stability. This epsilon is - "epsilon hat" in the Kingma and Ba paper (in the formula just before - Section 2.1), not the epsilon in Algorithm 1 of the paper. - use_locking: If True use locks for update operations. - name: Optional name for the operations created when applying gradients. - Defaults to "Adam". - """ - super(AdamWOptimizer, self).__init__( - weight_decay, - learning_rate=learning_rate, - beta1=beta1, - beta2=beta2, - epsilon=epsilon, - use_locking=use_locking, - name=name) - - -@tf_export("contrib.opt.ShampooWOptimizer") -class ShampooWOptimizer(DecoupledWeightDecayExtension, - shampoo.ShampooOptimizer): - """Optimizer that implements the Shampoo algorithm with weight decay. - - For further information see the documentation of the Shampoo Optimizer. - """ - - def __init__(self, - weight_decay, - global_step, - max_matrix_size=768, - gbar_decay=0.0, - gbar_weight=1.0, - mat_gbar_decay=1.0, - mat_gbar_weight=1.0, - learning_rate=1.0, - svd_interval=1, - precond_update_interval=1, - epsilon=1e-4, - alpha=0.5, - use_iterative_root=False, - use_locking=False, - name="ShampooW"): - """Construct a new ShampooW optimizer. - - For further information see the documentation of the Shampoo Optimizer. - - Args: - weight_decay: A `Tensor` or a floating point value. The weight decay. - global_step: tensorflow variable indicating the step. - max_matrix_size: We do not perform SVD for matrices larger than this. - gbar_decay: - gbar_weight: Used to update gbar: gbar[t] = gbar_decay[t] * gbar[t-1] + - gbar_weight[t] * g[t] - mat_gbar_decay: - mat_gbar_weight: Used to update mat_gbar: mat_gbar_j[t] = - mat_gbar_decay[t] * mat_gbar_j[t-1] + mat_gbar_weight[t] * gg_j[t] - learning_rate: Similar to SGD - svd_interval: We should do SVD after this many steps. Default = 1, i.e. - every step. Usually 20 leads to no loss of accuracy, and 50 or 100 is - also OK. May also want more often early, - and less often later - set in caller as for example: - "svd_interval = lambda(T): tf.cond( - T < 2000, lambda: 20.0, lambda: 1000.0)" - precond_update_interval: We should update the preconditioners after this - many steps. Default = 1. Usually less than svd_interval. - epsilon: epsilon * I_n is added to each mat_gbar_j for stability - alpha: total power of the preconditioners. - use_iterative_root: should the optimizer use SVD (faster) or the iterative - root method (for TPU) for finding the roots of PSD matrices. - use_locking: If `True` use locks for update operations. - name: name of optimizer. - """ - super(ShampooWOptimizer, self).__init__( - weight_decay, - global_step=global_step, - max_matrix_size=max_matrix_size, - gbar_decay=gbar_decay, - gbar_weight=gbar_weight, - mat_gbar_decay=mat_gbar_weight, - learning_rate=learning_rate, - svd_interval=svd_interval, - precond_update_interval=precond_update_interval, - epsilon=epsilon, - alpha=alpha, - use_iterative_root=use_iterative_root, - use_locking=use_locking, - name=name) diff --git a/tensorflow/contrib/opt/python/training/weight_decay_optimizers_test.py b/tensorflow/contrib/opt/python/training/weight_decay_optimizers_test.py deleted file mode 100644 index 9c910783018..00000000000 --- a/tensorflow/contrib/opt/python/training/weight_decay_optimizers_test.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for optimizers with weight decay.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.opt.python.training import weight_decay_optimizers -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import adam - -WEIGHT_DECAY = 0.01 - - -def adamw_update_numpy(param, g_t, t, m, v, lr=0.001, beta1=0.9, - beta2=0.999, epsilon=1e-8): - lr_t = lr * np.sqrt(1 - beta2**t) / (1 - beta1**t) - - m_t = beta1 * m + (1 - beta1) * g_t - v_t = beta2 * v + (1 - beta2) * g_t * g_t - - param_t = (param - lr_t * m_t / (np.sqrt(v_t) + epsilon) - - (param * WEIGHT_DECAY)) - return param_t, m_t, v_t - - -def momentumw_update_numpy(param, g_t, m, lr=0.001, momentum=0.9, **_): - # v, t are not needed for momentum optimizer - m = momentum * m + g_t - param_t = param - lr * m - param * WEIGHT_DECAY - return param_t, m, None - - -class WeightDecayOptimizerTest(test.TestCase): - - def doTest(self, optimizer, update_fn, optimizer_name, slot_name, - use_resource=False, do_sparse=False): - for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - with self.session(graph=ops.Graph()): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable( - var0_np, name="var0_%d" % i) - var1 = resource_variable_ops.ResourceVariable( - var1_np, name="var1_%d" % i) - else: - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - - if do_sparse: - grads0_np_indices = np.array([0, 1], dtype=np.int32) - grads0 = ops.IndexedSlices(constant_op.constant(grads0_np), - constant_op.constant(grads0_np_indices), - constant_op.constant([2])) - grads1_np_indices = np.array([0, 1], dtype=np.int32) - grads1 = ops.IndexedSlices(constant_op.constant(grads1_np), - constant_op.constant(grads1_np_indices), - constant_op.constant([2])) - else: - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - opt = optimizer() - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - - if not context.executing_eagerly(): - with ops.Graph().as_default(): - # Shouldn't return non-slot variables from other graphs. - self.assertEqual(0, len(opt.variables())) - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - # Run 3 steps of the optimizer - for t in range(1, 4): - if not context.executing_eagerly(): - self.evaluate(update) - elif t > 1: - opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - - var0_np, m0, v0 = update_fn(var0_np, grads0_np, t=t, m=m0, v=v0) - var1_np, m1, v1 = update_fn(var1_np, grads1_np, t=t, m=m1, v=v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - if use_resource: - self.assertEqual("var0_%d/%s:0" % (i, optimizer_name), - opt.get_slot(var=var0, name=slot_name).name) - - -class AdamWOptimizerTest(WeightDecayOptimizerTest): - - @staticmethod - def get_optimizer(): - return weight_decay_optimizers.AdamWOptimizer(WEIGHT_DECAY) - - def testSparse(self): - self.doTest(self.get_optimizer, adamw_update_numpy, "AdamW", "m", - use_resource=False, do_sparse=True) - - def testResourceSparse(self): - self.doTest(self.get_optimizer, adamw_update_numpy, "AdamW", "m", - use_resource=True, do_sparse=True) - - def testBasic(self): - self.doTest(self.get_optimizer, adamw_update_numpy, "AdamW", "m", - use_resource=False) - - @test_util.run_in_graph_and_eager_modes(reset_test=True) - def testResourceBasic(self): - self.doTest(self.get_optimizer, adamw_update_numpy, "AdamW", "m", - use_resource=True) - - -class MomentumWOptimizerTest(WeightDecayOptimizerTest): - - @staticmethod - def get_optimizer(): - return weight_decay_optimizers.MomentumWOptimizer(WEIGHT_DECAY, 0.001, 0.9) - - def testSparse(self): - self.doTest(self.get_optimizer, momentumw_update_numpy, "MomentumW", - "momentum", use_resource=False, do_sparse=True) - - def testResourceSparse(self): - self.doTest(self.get_optimizer, momentumw_update_numpy, "MomentumW", - "momentum", use_resource=True, do_sparse=True) - - def testBasic(self): - self.doTest(self.get_optimizer, momentumw_update_numpy, "MomentumW", - "momentum", use_resource=False) - - @test_util.run_in_graph_and_eager_modes(reset_test=True) - def testResourceBasic(self): - self.doTest(self.get_optimizer, momentumw_update_numpy, "MomentumW", - "momentum", use_resource=True) - - -class ExtendWithWeightDecayTest(WeightDecayOptimizerTest): - - @staticmethod - def get_optimizer(): - adamw = weight_decay_optimizers.extend_with_decoupled_weight_decay( - adam.AdamOptimizer) - return adamw(WEIGHT_DECAY) - - def testBasic(self): - self.doTest(self.get_optimizer, adamw_update_numpy, "Adam", "m", - use_resource=False) - - @test_util.run_in_graph_and_eager_modes(reset_test=True) - def testResourceBasic(self): - self.doTest(self.get_optimizer, adamw_update_numpy, "Adam", "m", - use_resource=True) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/optimizer_v2/BUILD b/tensorflow/contrib/optimizer_v2/BUILD deleted file mode 100644 index c872bdab1c8..00000000000 --- a/tensorflow/contrib/optimizer_v2/BUILD +++ /dev/null @@ -1,206 +0,0 @@ -# Prototype of OptimizerV2. - -load("//tensorflow:tensorflow.bzl", "cuda_py_test") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -filegroup( - name = "all_files", - srcs = glob( - ["**/*"], - exclude = [ - "**/METADATA", - "**/OWNERS", - ], - ), - visibility = ["//tensorflow:__subpackages__"], -) - -py_library( - name = "optimizer_v2_py", - srcs = ["optimizer_v2_symbols.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":training", - "//tensorflow/python:util", - ], -) - -py_library( - name = "training", - srcs = [ - "adadelta.py", - "adagrad.py", - "adam.py", - "gradient_descent.py", - "momentum.py", - "optimizer_v2.py", - "rmsprop.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework", - "//tensorflow/python:math_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/distribute:distribute_lib", - "//tensorflow/python/distribute:reduce_util", - ], -) - -cuda_py_test( - name = "adadelta_test", - size = "medium", - srcs = ["adadelta_test.py"], - additional_deps = [ - ":training", - "//tensorflow/python:client_testlib", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -cuda_py_test( - name = "adagrad_test", - size = "small", - srcs = ["adagrad_test.py"], - additional_deps = [ - ":training", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:client_testlib", - "//third_party/py/numpy", - ], -) - -cuda_py_test( - name = "adam_test", - size = "small", - srcs = ["adam_test.py"], - additional_deps = [ - ":training", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:client_testlib", - "//third_party/py/numpy", - ], -) - -cuda_py_test( - name = "checkpointable_utils_test", - srcs = ["checkpointable_utils_test.py"], - additional_deps = [ - ":training", - "@six_archive//:six", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:layers", - "//tensorflow/python:layers_base", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:test", - "//tensorflow/python/keras", - ], - tags = ["notsan"], -) - -cuda_py_test( - name = "gradient_descent_test", - size = "medium", - srcs = ["gradient_descent_test.py"], - additional_deps = [ - ":training", - "//tensorflow/python:client_testlib", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:framework", - "//tensorflow/python:math_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:resources", - "//tensorflow/python:variables", - ], -) - -cuda_py_test( - name = "momentum_test", - size = "medium", - srcs = ["momentum_test.py"], - additional_deps = [ - ":training", - "//tensorflow/python:client_testlib", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:framework", - "//tensorflow/python:math_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:resources", - "//tensorflow/python:variables", - "//tensorflow/python/eager:context", - ], -) - -cuda_py_test( - name = "optimizer_v2_test", - size = "medium", - srcs = ["optimizer_v2_test.py"], - additional_deps = [ - ":training", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:array_ops", - "//tensorflow/python:clip_ops", - "//tensorflow/python:gradients", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:variables", - ], -) - -cuda_py_test( - name = "rmsprop_test", - size = "small", - srcs = ["rmsprop_test.py"], - additional_deps = [ - ":training", - "@absl_py//absl/testing:parameterized", - "//tensorflow/python:array_ops", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:client_testlib", - "//third_party/py/numpy", - ], - tags = ["optonly"], -) diff --git a/tensorflow/contrib/optimizer_v2/adadelta.py b/tensorflow/contrib/optimizer_v2/adadelta.py deleted file mode 100644 index b206f9f61bd..00000000000 --- a/tensorflow/contrib/optimizer_v2/adadelta.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Adadelta for TensorFlow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.optimizer_v2 import optimizer_v2 -from tensorflow.python.training import training_ops - - -class AdadeltaOptimizer(optimizer_v2.OptimizerV2): - """Optimizer that implements the Adadelta algorithm. - - See [M. D. Zeiler](http://arxiv.org/abs/1212.5701) - ([pdf](http://arxiv.org/pdf/1212.5701v1.pdf)) - """ - - def __init__(self, learning_rate=0.001, rho=0.95, epsilon=1e-8, - use_locking=False, name="Adadelta"): - """Construct a new Adadelta optimizer. - - Some of the args below are hyperparameters, where a hyperparameter is - defined as a scalar Tensor, a regular Python value or a callable (which - will be evaluated when `apply_gradients` is called) returning a scalar - Tensor or a Python value. - - Args: - learning_rate: A float hyperparameter. The learning rate. - To match the exact form in the original paper use 1.0. - rho: A float hyperparameter. The decay rate. - epsilon: A float hyperparameter. A constant epsilon used to better - condition the grad update. - use_locking: If `True` use locks for update operations. - name: Optional name prefix for the operations created when applying - gradients. Defaults to "Adadelta". - """ - super(AdadeltaOptimizer, self).__init__(use_locking, name) - self._set_hyper("learning_rate", learning_rate) - self._set_hyper("rho", rho) - self._set_hyper("epsilon", epsilon) - - def _create_vars(self, var_list, state): - for v in var_list: - state.zeros_slot(v, "accum") - state.zeros_slot(v, "accum_update") - - def _apply_dense(self, grad, var, state): - accum = state.get_slot(var, "accum") - accum_update = state.get_slot(var, "accum_update") - return training_ops.apply_adadelta( - var, - accum, - accum_update, - state.get_hyper("learning_rate", var.dtype.base_dtype), - state.get_hyper("rho", var.dtype.base_dtype), - state.get_hyper("epsilon", var.dtype.base_dtype), - grad, - use_locking=self._use_locking) - - def _resource_apply_dense(self, grad, var, state): - accum = state.get_slot(var, "accum") - accum_update = state.get_slot(var, "accum_update") - return training_ops.resource_apply_adadelta( - var.handle, - accum.handle, - accum_update.handle, - state.get_hyper("learning_rate", var.dtype.base_dtype), - state.get_hyper("rho", var.dtype.base_dtype), - state.get_hyper("epsilon", var.dtype.base_dtype), - grad, - use_locking=self._use_locking) - - def _apply_sparse(self, grad, var, state): - accum = state.get_slot(var, "accum") - accum_update = state.get_slot(var, "accum_update") - return training_ops.sparse_apply_adadelta( - var, - accum, - accum_update, - state.get_hyper("learning_rate", var.dtype.base_dtype), - state.get_hyper("rho", var.dtype.base_dtype), - state.get_hyper("epsilon", var.dtype.base_dtype), - grad.values, - grad.indices, - use_locking=self._use_locking) - - def _resource_apply_sparse(self, grad, var, indices, state): - accum = state.get_slot(var, "accum") - accum_update = state.get_slot(var, "accum_update") - return training_ops.resource_sparse_apply_adadelta( - var.handle, - accum.handle, - accum_update.handle, - state.get_hyper("learning_rate", var.dtype.base_dtype), - state.get_hyper("rho", var.dtype.base_dtype), - state.get_hyper("epsilon", var.dtype.base_dtype), - grad, - indices, - use_locking=self._use_locking) diff --git a/tensorflow/contrib/optimizer_v2/adadelta_test.py b/tensorflow/contrib/optimizer_v2/adadelta_test.py deleted file mode 100644 index 4c94b66679a..00000000000 --- a/tensorflow/contrib/optimizer_v2/adadelta_test.py +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Adadelta Optimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.optimizer_v2 import adadelta -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class AdadeltaOptimizerTest(test.TestCase): - - def doTestBasic(self, use_resource=False): - num_updates = 4 # number of ADADELTA steps to perform - for dtype in [dtypes.half, dtypes.float32]: - for grad in [0.2, 0.1, 0.01]: - for lr in [1.0, 0.5, 0.1]: - with self.cached_session(): - var0_init = [1.0, 2.0] - var1_init = [3.0, 4.0] - if use_resource: - var0 = resource_variable_ops.ResourceVariable( - var0_init, dtype=dtype) - var1 = resource_variable_ops.ResourceVariable( - var1_init, dtype=dtype) - else: - var0 = variables.Variable(var0_init, dtype=dtype) - var1 = variables.Variable(var1_init, dtype=dtype) - - grads = constant_op.constant([grad, grad], dtype=dtype) - - accum = 0.0 - accum_update = 0.0 - - # ADADELTA gradient optimizer - rho = 0.95 - epsilon = 1e-8 - adadelta_opt = adadelta.AdadeltaOptimizer(lr, rho, epsilon) - adadelta_update = adadelta_opt.apply_gradients( - zip([grads, grads], [var0, var1])) - - opt_vars = adadelta_opt.variables() - self.assertStartsWith(opt_vars[0].name, var0._shared_name) - self.assertStartsWith(opt_vars[1].name, var0._shared_name) - self.assertStartsWith(opt_vars[2].name, var1._shared_name) - self.assertStartsWith(opt_vars[3].name, var1._shared_name) - self.assertEqual(4, len(opt_vars)) - - variables.global_variables_initializer().run() - - # Assign slots - slot = [None] * 2 - slot_update = [None] * 2 - self.assertEqual(["accum", "accum_update"], - adadelta_opt.get_slot_names()) - slot[0] = adadelta_opt.get_slot(var0, "accum") - self.assertEquals(slot[0].get_shape(), var0.get_shape()) - self.assertFalse(slot[0] in variables.trainable_variables()) - - slot_update[0] = adadelta_opt.get_slot(var0, "accum_update") - self.assertEquals(slot_update[0].get_shape(), var0.get_shape()) - self.assertFalse(slot_update[0] in variables.trainable_variables()) - - slot[1] = adadelta_opt.get_slot(var1, "accum") - self.assertEquals(slot[1].get_shape(), var1.get_shape()) - self.assertFalse(slot[1] in variables.trainable_variables()) - - slot_update[1] = adadelta_opt.get_slot(var1, "accum_update") - self.assertEquals(slot_update[1].get_shape(), var1.get_shape()) - self.assertFalse(slot_update[1] in variables.trainable_variables()) - - # Fetch params to validate initial values - self.assertAllClose(var0_init, var0.eval()) - self.assertAllClose(var1_init, var1.eval()) - - update = [None] * num_updates - tot_update = 0 - for step in range(num_updates): - # Run adadelta update for comparison - adadelta_update.run() - - # Perform initial update without previous accum values - accum = accum * rho + (grad**2) * (1 - rho) - update[step] = (np.sqrt(accum_update + epsilon) * - (1. / np.sqrt(accum + epsilon)) * grad) - accum_update = (accum_update * rho + (update[step]**2) * - (1.0 - rho)) - tot_update += update[step] * lr - - # Check that the accumulators have been updated - for slot_idx in range(2): - self.assertAllCloseAccordingToType( - np.array([accum, accum], dtype=dtype.as_numpy_dtype()), - slot[slot_idx].eval(), - rtol=1e-5) - - self.assertAllCloseAccordingToType( - np.array( - [accum_update, accum_update], - dtype=dtype.as_numpy_dtype()), - slot_update[slot_idx].eval(), - rtol=1e-5) - - # Check that the parameters have been updated - self.assertAllCloseAccordingToType( - np.array( - [var0_init[0] - tot_update, var0_init[1] - tot_update], - dtype=dtype.as_numpy_dtype()), - var0.eval(), - rtol=1e-5) - - self.assertAllCloseAccordingToType( - np.array( - [var1_init[0] - tot_update, var1_init[1] - tot_update], - dtype=dtype.as_numpy_dtype()), - var1.eval(), - rtol=1e-5) - - def testBasic(self): - self.doTestBasic(use_resource=False) - - def testResourceBasic(self): - self.doTestBasic(use_resource=True) - - def testMinimizeSparseResourceVariable(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) - x = constant_op.constant([[4.0], [5.0]], dtype=dtype) - pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) - loss = pred * pred - sgd_op = adadelta.AdadeltaOptimizer( - 1.0, 1.0, 1.0).minimize(loss) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) - # Run 1 step of sgd - sgd_op.run() - # Validate updated params - self.assertAllCloseAccordingToType( - [[-111, -138]], var0.eval()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/optimizer_v2/adagrad.py b/tensorflow/contrib/optimizer_v2/adagrad.py deleted file mode 100644 index 346c3fbd2c8..00000000000 --- a/tensorflow/contrib/optimizer_v2/adagrad.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Adagrad optimizer for TensorFlow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.optimizer_v2 import optimizer_v2 -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.training import training_ops - - -class AdagradOptimizer(optimizer_v2.OptimizerV2): - """Optimizer that implements the Adagrad algorithm. - - See this [paper](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) - or this - [intro](https://ppasupat.github.io/a9online/uploads/proximal_notes.pdf). - """ - - def __init__(self, learning_rate, initial_accumulator_value=0.1, - use_locking=False, name="Adagrad"): - """Construct a new Adagrad optimizer. - - The learning_rate arg below is a hyperparameter, where a hyperparameter is - defined as a scalar Tensor, a regular Python value or a callable (which - will be evaluated when `apply_gradients` is called) returning a scalar - Tensor or a Python value. - - Args: - learning_rate: A float hyperparameter. The learning rate. - initial_accumulator_value: A floating point value. - Starting value for the accumulators, must be positive. - use_locking: If `True` use locks for update operations. - name: Optional name prefix for the operations created when applying - gradients. Defaults to "Adagrad". - - Raises: - ValueError: If the `initial_accumulator_value` is invalid. - """ - if initial_accumulator_value <= 0.0: - raise ValueError("initial_accumulator_value must be positive: %s" % - initial_accumulator_value) - super(AdagradOptimizer, self).__init__(use_locking, name) - self._set_hyper("learning_rate", learning_rate) - - self._initial_accumulator_value = initial_accumulator_value - - def _create_vars(self, var_list, state): - for v in var_list: - dtype = v.dtype.base_dtype - if v.get_shape().is_fully_defined(): - init = init_ops.constant_initializer( - self._initial_accumulator_value, dtype=dtype) - else: - - def init(v=v, dtype=dtype): - # Use a Tensor instead of initializer if variable does not have - # static shape. - init_constant = gen_array_ops.fill( - array_ops.shape(v), self._initial_accumulator_value) - return math_ops.cast(init_constant, dtype) - - state.create_slot_with_initializer(v, init, v.get_shape(), dtype, - "accumulator") - - def _apply_dense(self, grad, var, state): - acc = state.get_slot(var, "accumulator") - return training_ops.apply_adagrad( - var, - acc, - state.get_hyper("learning_rate", var.dtype.base_dtype), - grad, - use_locking=self._use_locking) - - def _resource_apply_dense(self, grad, var, state): - acc = state.get_slot(var, "accumulator") - return training_ops.resource_apply_adagrad( - var.handle, - acc.handle, - state.get_hyper("learning_rate", var.dtype.base_dtype), - grad, - use_locking=self._use_locking) - - def _apply_sparse(self, grad, var, state): - acc = state.get_slot(var, "accumulator") - return training_ops.sparse_apply_adagrad( - var, - acc, - state.get_hyper("learning_rate", var.dtype.base_dtype), - grad.values, - grad.indices, - use_locking=self._use_locking) - - def _resource_apply_sparse(self, grad, var, indices, state): - acc = state.get_slot(var, "accumulator") - return training_ops.resource_sparse_apply_adagrad( - var.handle, - acc.handle, - state.get_hyper("learning_rate", var.dtype.base_dtype), - grad, - indices, - use_locking=self._use_locking) diff --git a/tensorflow/contrib/optimizer_v2/adagrad_test.py b/tensorflow/contrib/optimizer_v2/adagrad_test.py deleted file mode 100644 index 320e41567ff..00000000000 --- a/tensorflow/contrib/optimizer_v2/adagrad_test.py +++ /dev/null @@ -1,279 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functional tests for aggregate operations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.optimizer_v2 import adagrad -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class AdagradOptimizerTest(test.TestCase): - - def doTestBasic(self, use_locking=False, use_resource=False): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - if use_resource: - var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) - var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) - else: - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - ada_opt = adagrad.AdagradOptimizer( - 3.0, initial_accumulator_value=0.1, use_locking=use_locking) - ada_update = ada_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Run 3 steps of adagrad - for _ in range(3): - ada_update.run() - # Validate updated params - self.assertAllCloseAccordingToType( - np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([2.715679168701172, 3.715679168701172]), var1.eval()) - - def testBasic(self): - self.doTestBasic(use_locking=False) - - def testBasicResource(self): - self.doTestBasic(use_locking=False, use_resource=True) - - def testMinimizeSparseResourceVariable(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = resource_variable_ops.ResourceVariable( - [[1.0, 2.0], [3.0, 4.0]], dtype=dtype) - x = constant_op.constant([[4.0], [5.0]], dtype=dtype) - pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) - loss = pred * pred - sgd_op = adagrad.AdagradOptimizer(1.0).minimize(loss) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllCloseAccordingToType( - [[1.0, 2.0], [3.0, 4.0]], var0.eval()) - # Run 1 step of sgd - sgd_op.run() - # Validate updated params - self.assertAllCloseAccordingToType( - [[0, 1], [3, 4]], var0.eval(), atol=0.01) - - def testTensorLearningRate(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - ada_opt = adagrad.AdagradOptimizer( - constant_op.constant(3.0), initial_accumulator_value=0.1) - ada_update = ada_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Run 3 steps of adagrad - for _ in range(3): - ada_update.run() - # Validate updated params - self.assertAllCloseAccordingToType( - np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([2.715679168701172, 3.715679168701172]), var1.eval()) - - def testSparseBasic(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) - var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) - grads0 = ops.IndexedSlices( - constant_op.constant( - [0.1], shape=[1, 1], dtype=dtype), - constant_op.constant([0]), - constant_op.constant([2, 1])) - grads1 = ops.IndexedSlices( - constant_op.constant( - [0.01], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), - constant_op.constant([2, 1])) - ada_opt = adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1) - ada_update = ada_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([[1.0], [2.0]], var0.eval()) - self.assertAllClose([[3.0], [4.0]], var1.eval()) - # Run 3 step of sgd - for _ in range(3): - ada_update.run() - # Validate updated params - self.assertAllCloseAccordingToType( - np.array([[-1.6026098728179932], [2.0]]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([[3.0], [3.715679168701172]]), var1.eval()) - - def testSparseRepeatedIndices(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - repeated_index_update_var = variables.Variable( - [[1.0], [2.0]], dtype=dtype) - aggregated_update_var = variables.Variable( - [[1.0], [2.0]], dtype=dtype) - grad_repeated_index = ops.IndexedSlices( - constant_op.constant( - [0.1, 0.1], shape=[2, 1], dtype=dtype), - constant_op.constant([1, 1]), - constant_op.constant([2, 1])) - grad_aggregated = ops.IndexedSlices( - constant_op.constant( - [0.2], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), - constant_op.constant([2, 1])) - repeated_update = adagrad.AdagradOptimizer(3.0).apply_gradients( - [(grad_repeated_index, repeated_index_update_var)]) - aggregated_update = adagrad.AdagradOptimizer(3.0).apply_gradients( - [(grad_aggregated, aggregated_update_var)]) - variables.global_variables_initializer().run() - self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) - for _ in range(3): - repeated_update.run() - aggregated_update.run() - self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) - - def testSparseRepeatedIndicesResourceVariable(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var_repeated = resource_variable_ops.ResourceVariable( - [1.0, 2.0], dtype=dtype) - loss_repeated = math_ops.reduce_sum( - embedding_ops.embedding_lookup(var_repeated, [0, 0])) - var_aggregated = resource_variable_ops.ResourceVariable( - [1.0, 2.0], dtype=dtype) - loss_aggregated = 2 * math_ops.reduce_sum( - embedding_ops.embedding_lookup(var_aggregated, [0])) - update_op_repeated = adagrad.AdagradOptimizer( - 2.0).minimize(loss_repeated) - update_op_aggregated = adagrad.AdagradOptimizer( - 2.0).minimize(loss_aggregated) - variables.global_variables_initializer().run() - self.assertAllCloseAccordingToType( - var_repeated.eval(), var_aggregated.eval()) - for _ in range(3): - update_op_repeated.run() - update_op_aggregated.run() - self.assertAllCloseAccordingToType( - var_repeated.eval(), var_aggregated.eval()) - - def testSparseStability(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - shape = [1, 6] - var0 = variables.Variable( - [[ - 0.00872496, -0.106952, 0.110467, 0.226505, -0.0147257, - -0.0105945 - ]], - dtype=dtype) - grads0 = ops.IndexedSlices( - constant_op.constant( - [[ - -5.91278e-05, 5.31673e-05, -2.5779e-06, 4.29153e-05, - -8.4877e-05, -9.48906e-05 - ]], - shape=shape, - dtype=dtype), - constant_op.constant([0]), - constant_op.constant(shape)) - ada_opt = adagrad.AdagradOptimizer(1.0, initial_accumulator_value=0.1) - ada_update = ada_opt.apply_gradients(zip([grads0], [var0])) - self.assertEqual(["accumulator"], ada_opt.get_slot_names()) - slot0 = ada_opt.get_slot(var0, "accumulator") - init = variables.global_variables_initializer() - for _ in range(100): - init.run() - ada_update.run() - self.assertAllCloseAccordingToType( - np.array([[0.1, 0.1, 0.1, 0.1, 0.1, 0.1]]), slot0.eval()) - self.assertAllCloseAccordingToType( - np.array([[ - 0.00891194, -0.10712013, 0.11047515, 0.22636929, -0.0144573, - -0.01029443 - ]]), var0.eval()) - - def testSharing(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - ada_opt = adagrad.AdagradOptimizer(3.0) - # Apply the optimizer twice. Both applications will use - # the same accums. - ada_update1 = ada_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - ada_update2 = ada_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - self.assertEqual(["accumulator"], ada_opt.get_slot_names()) - slot0 = ada_opt.get_slot(var0, "accumulator") - self.assertEquals(slot0.get_shape(), var0.get_shape()) - slot1 = ada_opt.get_slot(var1, "accumulator") - self.assertEquals(slot1.get_shape(), var1.get_shape()) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values. - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Mix the first and the second adagrad for 3 steps. - ada_update1.run() - ada_update2.run() - ada_update1.run() - # Validate updated params (the same as with only 1 Adagrad). - self.assertAllCloseAccordingToType( - np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([2.715679168701172, 3.715679168701172]), var1.eval()) - - def testDynamicShapeVariable_Ok(self): - with self.cached_session(): - v = variable_scope.get_variable("v", initializer=constant_op.constant(1.), - validate_shape=False) - self.assertFalse(v.shape.is_fully_defined()) - # Creating optimizer should cause no exception. - adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/optimizer_v2/adam.py b/tensorflow/contrib/optimizer_v2/adam.py deleted file mode 100644 index 1b7800f324b..00000000000 --- a/tensorflow/contrib/optimizer_v2/adam.py +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Adam optimizer for TensorFlow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.optimizer_v2 import optimizer_v2 -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.training import training_ops - - -class AdamOptimizer(optimizer_v2.OptimizerV2): - """Optimizer that implements the Adam algorithm. - - See [Kingma et al., 2014](http://arxiv.org/abs/1412.6980) - ([pdf](http://arxiv.org/pdf/1412.6980.pdf)). - """ - - def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8, - use_locking=False, name="Adam"): - r"""Construct a new Adam optimizer. - - Initialization: - - $$m_0 := 0 \text{(Initialize initial 1st moment vector)}$$ - $$v_0 := 0 \text{(Initialize initial 2nd moment vector)}$$ - $$t := 0 \text{(Initialize timestep)}$$ - The update rule for `variable` with gradient `g` uses an optimization - described at the end of section2 of the paper: - - $$t := t + 1$$ - $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$ - - $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$ - $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ - $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ - - The default value of 1e-8 for epsilon might not be a good default in - general. For example, when training an Inception network on ImageNet a - current good choice is 1.0 or 0.1. Note that since AdamOptimizer uses the - formulation just before Section 2.1 of the Kingma and Ba paper rather than - the formulation in Algorithm 1, the "epsilon" referred to here is "epsilon - hat" in the paper. - - The sparse implementation of this algorithm (used when the gradient is an - IndexedSlices object, typically because of `tf.gather` or an embedding - lookup in the forward pass) does apply momentum to variable slices even if - they were not used in the forward pass (meaning they have a gradient equal - to zero). Momentum decay (beta1) is also applied to the entire momentum - accumulator. This means that the sparse behavior is equivalent to the dense - behavior (in contrast to some momentum implementations which ignore momentum - unless a variable slice was actually used). - - Some of the args below are hyperparameters where a hyperparameter is - defined as a scalar Tensor, a regular Python value or a callable (which - will be evaluated when `apply_gradients` is called) returning a scalar - Tensor or a Python value. - - Args: - learning_rate: A float hyperparameter. The learning rate. - beta1: A float hyperparameter. The exponential decay rate for the 1st - moment estimates. - beta2: A float hyperparameter. The exponential decay rate for the 2nd - moment estimates. - epsilon: A float hyperparameter. This epsilon is "epsilon hat" in the - Kingma and Ba paper (in the formula just before Section 2.1), not the - epsilon in Algorithm 1 of the paper. - use_locking: If True use locks for update operations. - name: Optional name for the operations created when applying gradients. - Defaults to "Adam". - """ - super(AdamOptimizer, self).__init__(use_locking, name) - - self._set_hyper("learning_rate", learning_rate) - self._set_hyper("beta1", beta1) - self._set_hyper("beta2", beta2) - self._set_hyper("epsilon", epsilon) - - def _get_beta_accumulators(self, state=None): - if state is None: - state = self._get_per_graph_state() - return (state.get_non_slot("beta1_power"), - state.get_non_slot("beta2_power")) - - def _create_vars(self, var_list, state): - # Non-slot variables end up on the same device(s). - state.create_non_slot( - initial_value=lambda: state.get_hyper("beta1"), name="beta1_power") - state.create_non_slot( - initial_value=lambda: state.get_hyper("beta2"), name="beta2_power") - - # Create slots for the first and second moments. - for v in var_list: - state.zeros_slot(v, "m") - state.zeros_slot(v, "v") - - def _apply_dense(self, grad, var, state): - m = state.get_slot(var, "m") - v = state.get_slot(var, "v") - beta1_power, beta2_power = self._get_beta_accumulators(state) - return training_ops.apply_adam( - var, - m, - v, - math_ops.cast(beta1_power, var.dtype.base_dtype), - math_ops.cast(beta2_power, var.dtype.base_dtype), - state.get_hyper("learning_rate", var.dtype.base_dtype), - state.get_hyper("beta1", var.dtype.base_dtype), - state.get_hyper("beta2", var.dtype.base_dtype), - state.get_hyper("epsilon", var.dtype.base_dtype), - grad, - use_locking=self._use_locking).op - - def _resource_apply_dense(self, grad, var, state): - m = state.get_slot(var, "m") - v = state.get_slot(var, "v") - beta1_power, beta2_power = self._get_beta_accumulators(state) - return training_ops.resource_apply_adam( - var.handle, - m.handle, - v.handle, - math_ops.cast(beta1_power, grad.dtype.base_dtype), - math_ops.cast(beta2_power, grad.dtype.base_dtype), - state.get_hyper("learning_rate", grad.dtype.base_dtype), - state.get_hyper("beta1", grad.dtype.base_dtype), - state.get_hyper("beta2", grad.dtype.base_dtype), - state.get_hyper("epsilon", grad.dtype.base_dtype), - grad, - use_locking=self._use_locking) - - def _apply_sparse_shared(self, grad, var, indices, scatter_add, state): - beta1_power, beta2_power = self._get_beta_accumulators(state) - beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) - beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype) - lr_t = state.get_hyper("learning_rate", var.dtype.base_dtype) - beta1_t = state.get_hyper("beta1", var.dtype.base_dtype) - beta2_t = state.get_hyper("beta2", var.dtype.base_dtype) - epsilon_t = state.get_hyper("epsilon", var.dtype.base_dtype) - lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power)) - # m_t = beta1 * m + (1 - beta1) * g_t - m = state.get_slot(var, "m") - m_scaled_g_values = grad * (1 - beta1_t) - m_t = state_ops.assign(m, m * beta1_t, use_locking=self._use_locking) - with ops.control_dependencies([m_t]): - m_t = scatter_add(m, indices, m_scaled_g_values) - # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) - v = state.get_slot(var, "v") - v_scaled_g_values = (grad * grad) * (1 - beta2_t) - v_t = state_ops.assign(v, v * beta2_t, use_locking=self._use_locking) - with ops.control_dependencies([v_t]): - v_t = scatter_add(v, indices, v_scaled_g_values) - v_sqrt = math_ops.sqrt(v_t) - var_update = state_ops.assign_sub( - var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking) - return control_flow_ops.group(*[var_update, m_t, v_t]) - - def _apply_sparse(self, grad, var, state): - return self._apply_sparse_shared( - grad.values, var, grad.indices, - lambda x, i, v: state_ops.scatter_add( # pylint: disable=g-long-lambda - x, i, v, use_locking=self._use_locking), - state) - - def _resource_scatter_add(self, x, i, v): - with ops.control_dependencies( - [resource_variable_ops.resource_scatter_add(x.handle, i, v)]): - return x.value() - - def _resource_apply_sparse(self, grad, var, indices, state): - return self._apply_sparse_shared(grad, var, indices, - self._resource_scatter_add, state) - - def _finish(self, state): - # Update the power accumulators. - beta1_power, beta2_power = self._get_beta_accumulators(state) - update_beta1 = beta1_power.assign( - beta1_power * state.get_hyper("beta1"), use_locking=self._use_locking) - update_beta2 = beta2_power.assign( - beta2_power * state.get_hyper("beta2"), use_locking=self._use_locking) - return control_flow_ops.group(update_beta1, update_beta2) diff --git a/tensorflow/contrib/optimizer_v2/adam_test.py b/tensorflow/contrib/optimizer_v2/adam_test.py deleted file mode 100644 index b55739f788e..00000000000 --- a/tensorflow/contrib/optimizer_v2/adam_test.py +++ /dev/null @@ -1,333 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Adam optimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.optimizer_v2 import adam -from tensorflow.python.client import session -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def adam_update_numpy(param, - g_t, - t, - m, - v, - alpha=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8): - alpha_t = alpha * np.sqrt(1 - beta2**t) / (1 - beta1**t) - - m_t = beta1 * m + (1 - beta1) * g_t - v_t = beta2 * v + (1 - beta2) * g_t * g_t - - param_t = param - alpha_t * m_t / (np.sqrt(v_t) + epsilon) - return param_t, m_t, v_t - - -class AdamOptimizerTest(test.TestCase): - - def doTestSparse(self, use_resource=False): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - else: - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0_np_indices = np.array([0, 1], dtype=np.int32) - grads0 = ops.IndexedSlices( - constant_op.constant(grads0_np), - constant_op.constant(grads0_np_indices), constant_op.constant([2])) - grads1_np_indices = np.array([0, 1], dtype=np.int32) - grads1 = ops.IndexedSlices( - constant_op.constant(grads1_np), - constant_op.constant(grads1_np_indices), constant_op.constant([2])) - opt = adam.AdamOptimizer() - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Run 3 steps of Adam - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) - update.run() - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testSparse(self): - self.doTestSparse(use_resource=False) - - def testResourceSparse(self): - self.doTestSparse(use_resource=True) - - def testSparseDevicePlacement(self): - for index_dtype in [dtypes.int32, dtypes.int64]: - with self.cached_session(force_gpu=test.is_gpu_available()): - # If a GPU is available, tests that all optimizer ops can be placed on - # it (i.e. they have GPU kernels). - var = variables.Variable([[1.0], [2.0]]) - indices = constant_op.constant([0, 1], dtype=index_dtype) - gathered_sum = math_ops.reduce_sum(array_ops.gather(var, indices)) - optimizer = adam.AdamOptimizer(3.0) - minimize_op = optimizer.minimize(gathered_sum) - variables.global_variables_initializer().run() - minimize_op.run() - - def testSparseRepeatedIndices(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - repeated_index_update_var = variables.Variable( - [[1.0], [2.0]], dtype=dtype) - aggregated_update_var = variables.Variable( - [[1.0], [2.0]], dtype=dtype) - grad_repeated_index = ops.IndexedSlices( - constant_op.constant( - [0.1, 0.1], shape=[2, 1], dtype=dtype), - constant_op.constant([1, 1]), - constant_op.constant([2, 1])) - grad_aggregated = ops.IndexedSlices( - constant_op.constant( - [0.2], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), - constant_op.constant([2, 1])) - repeated_update = adam.AdamOptimizer().apply_gradients( - [(grad_repeated_index, repeated_index_update_var)]) - aggregated_update = adam.AdamOptimizer().apply_gradients( - [(grad_aggregated, aggregated_update_var)]) - variables.global_variables_initializer().run() - self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) - for _ in range(3): - repeated_update.run() - aggregated_update.run() - self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) - - def doTestBasic(self, use_resource=False): - for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - with self.session(graph=ops.Graph()): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable( - var0_np, name="var0_%d" % i) - var1 = resource_variable_ops.ResourceVariable( - var1_np, name="var1_%d" % i) - else: - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - opt = adam.AdamOptimizer() - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - opt_variables = opt.variables() - beta1_power, beta2_power = opt._get_beta_accumulators() - self.assertTrue(beta1_power is not None) - self.assertTrue(beta2_power is not None) - self.assertIn(beta1_power, opt_variables) - self.assertIn(beta2_power, opt_variables) - - with ops.Graph().as_default(): - # Shouldn't return non-slot variables from other graphs. - self.assertEqual(0, len(opt.variables())) - - if not context.executing_eagerly(): - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Run 3 steps of Adam - for t in range(1, 4): - if not context.executing_eagerly(): - self.evaluate(update) - elif t > 1: - opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - - self.assertAllCloseAccordingToType(0.9**(t + 1), - self.evaluate(beta1_power)) - self.assertAllCloseAccordingToType(0.999**(t + 1), - self.evaluate(beta2_power)) - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) - if use_resource: - self.assertEqual("var0_%d/Adam:0" % (i,), - opt.get_slot(var=var0, name="m").name) - - def testBasic(self): - with self.cached_session(): - self.doTestBasic(use_resource=False) - - @test_util.run_in_graph_and_eager_modes(reset_test=True) - def testResourceBasic(self): - self.doTestBasic(use_resource=True) - - def testTensorLearningRate(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - opt = adam.AdamOptimizer(constant_op.constant(0.001)) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Run 3 steps of Adam - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) - update.run() - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testSharing(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - opt = adam.AdamOptimizer() - update1 = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - beta1_power, beta2_power = opt._get_beta_accumulators() - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - # Run 3 steps of intertwined Adam1 and Adam2. - for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) - if t % 2 == 0: - update1.run() - else: - update2.run() - - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - def testTwoSessions(self): - optimizer = adam.AdamOptimizer() - g = ops.Graph() - with g.as_default(): - with session.Session(): - var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") - grads0 = constant_op.constant(np.array([0.1, 0.1])) - optimizer.apply_gradients([(grads0, var0)]) - - gg = ops.Graph() - with gg.as_default(): - with session.Session(): - var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") - grads0 = constant_op.constant(np.array([0.1, 0.1])) - - # If the optimizer saves any state not keyed by graph the following line - # fails. - optimizer.apply_gradients([(grads0, var0)]) - - def testSlotsUniqueEager(self): - with context.eager_mode(): - v1 = resource_variable_ops.ResourceVariable(1.) - v2 = resource_variable_ops.ResourceVariable(1.) - opt = adam.AdamOptimizer(1.) - opt.minimize(lambda: v1 + v2) - # There should be two non-slot variables, and two unique slot variables - # for v1 and v2 respectively. - self.assertEqual(6, len(set(opt.variables()))) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py deleted file mode 100644 index 1c8cdc5a42f..00000000000 --- a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py +++ /dev/null @@ -1,748 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# TODO(josh11b): Forked from contrib/eager/python to test OptimizerV2 the same way -# OptimizerV1 is tested. This file should be removed once the fork is resolved. - -import functools -import os - -import six - -from tensorflow.contrib.optimizer_v2 import adam -from tensorflow.python.client import session as session_lib -from tensorflow.python.eager import backprop -from tensorflow.python.eager import context -from tensorflow.python.eager import function -from tensorflow.python.eager import test -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.keras.engine import training -from tensorflow.python.keras.layers import core -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import template -from tensorflow.python.ops import variable_scope -from tensorflow.python.training import checkpoint_management -from tensorflow.python.training import saver as core_saver -from tensorflow.python.training import training_util -from tensorflow.python.training.tracking import graph_view -from tensorflow.python.training.tracking import tracking -from tensorflow.python.training.tracking import util - - -class NonLayerTrackable(tracking.AutoTrackable): - - def __init__(self): - super(NonLayerTrackable, self).__init__() - self.a_variable = util.add_variable( - self, name="a_variable", shape=[]) - - -# pylint: disable=not-callable -class MyModel(training.Model): - """A concrete Model for testing.""" - - def __init__(self): - super(MyModel, self).__init__() - self._named_dense = core.Dense(1, use_bias=True) - self._second = core.Dense(1, use_bias=False) - # We can still track Trackables which aren't Layers. - self._non_layer = NonLayerTrackable() - - def call(self, values): - ret = self._second(self._named_dense(values)) - return ret - - -class _MirroringSaveable( - core_saver.BaseSaverBuilder.ResourceVariableSaveable): - - def __init__(self, primary_variable, mirrored_variable, name): - self._primary_variable = primary_variable - self._mirrored_variable = mirrored_variable - super(_MirroringSaveable, self).__init__( - self._primary_variable, "", name) - - def restore(self, restored_tensors, restored_shapes): - """Restore the same value into both variables.""" - tensor, = restored_tensors - return control_flow_ops.group( - self._primary_variable.assign(tensor), - self._mirrored_variable.assign(tensor)) - - -class CheckpointingTests(test.TestCase): - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testNamingWithOptimizer(self): - input_value = constant_op.constant([[3.]]) - model = MyModel() - # A nuisance Model using the same optimizer. Its slot variables should not - # go in the checkpoint, since it is never depended on. - other_model = MyModel() - optimizer = adam.AdamOptimizer(0.001) - optimizer_step = training_util.get_or_create_global_step() - root_trackable = util.Checkpoint( - optimizer=optimizer, model=model, optimizer_step=optimizer_step) - if context.executing_eagerly(): - optimizer.minimize( - lambda: model(input_value), - global_step=optimizer_step) - optimizer.minimize( - lambda: other_model(input_value), - global_step=optimizer_step) - else: - train_op = optimizer.minimize( - model(input_value), global_step=optimizer_step) - optimizer.minimize( - other_model(input_value), - global_step=optimizer_step) - self.evaluate(util.gather_initializers( - root_trackable)) - self.evaluate(train_op) - named_variables, serialized_graph, _ = graph_view.ObjectGraphView( - root_trackable).serialize_object_graph() - expected_checkpoint_names = ( - # Created in the root node, so no prefix. - "optimizer_step", - "model/_second/kernel", - "model/_named_dense/kernel", - "model/_named_dense/bias", - # non-Layer dependency of the model - "model/_non_layer/a_variable", - # The optimizer creates two non-slot variables - "optimizer/beta1_power", - "optimizer/beta2_power", - # Slot variables - "model/_second/kernel/.OPTIMIZER_SLOT/optimizer/m", - "model/_second/kernel/.OPTIMIZER_SLOT/optimizer/v", - "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/m", - "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/v", - "model/_named_dense/bias/.OPTIMIZER_SLOT/optimizer/m", - "model/_named_dense/bias/.OPTIMIZER_SLOT/optimizer/v", - ) - suffix = "/.ATTRIBUTES/VARIABLE_VALUE" - expected_checkpoint_names = [ - name + suffix for name in expected_checkpoint_names] - named_variables = {v.name: v for v in named_variables} - six.assertCountEqual(self, expected_checkpoint_names, - named_variables.keys()) - # Check that we've mapped to the right variable objects (not exhaustive) - self.assertEqual( - "global_step", - named_variables["optimizer_step" + suffix].full_name) - self.assertEqual( - "my_model/dense_1/kernel", - named_variables["model/_second/kernel" + suffix].full_name) - self.assertEqual( - "my_model/dense/kernel", - named_variables["model/_named_dense/kernel" + suffix].full_name) - self.assertEqual( - "beta1_power", - named_variables["optimizer/beta1_power" + suffix].full_name) - self.assertEqual( - "beta2_power", - named_variables["optimizer/beta2_power" + suffix].full_name) - # Spot check the generated protocol buffers. - self.assertEqual("optimizer", - serialized_graph.nodes[0].children[1].local_name) - optimizer_node = serialized_graph.nodes[serialized_graph.nodes[0].children[ - 1].node_id] - self.assertEqual("beta1_power", optimizer_node.children[0].local_name) - self.assertEqual( - "beta1_power", serialized_graph.nodes[optimizer_node.children[0] - .node_id].attributes[0].full_name) - self.assertEqual( - "my_model/dense/kernel", - serialized_graph.nodes[optimizer_node.slot_variables[0] - .original_variable_node_id] - .attributes[0].full_name) - # We strip off the :0 suffix, as variable.name-based saving does. - self.assertEqual( - "my_model/dense/kernel/Adam", - serialized_graph.nodes[optimizer_node.slot_variables[0] - .slot_variable_node_id] - .attributes[0].full_name) - self.assertEqual( - "my_model/dense/kernel/Adam:0", - optimizer.get_slot( - var=model._named_dense.kernel, - name="m").name) - self.assertEqual( - "model/_named_dense/kernel" + suffix, - serialized_graph.nodes[ - optimizer_node.slot_variables[0] - .original_variable_node_id].attributes[0].checkpoint_key) - self.assertEqual("m", optimizer_node.slot_variables[0].slot_name) - self.assertEqual( - "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/m" + suffix, - serialized_graph.nodes[ - optimizer_node.slot_variables[0] - .slot_variable_node_id].attributes[0].checkpoint_key) - - @test_util.run_in_graph_and_eager_modes - def testSaveRestore(self): - model = MyModel() - optimizer = adam.AdamOptimizer(0.001) - root_trackable = util.Checkpoint( - optimizer=optimizer, model=model) - input_value = constant_op.constant([[3.]]) - if context.executing_eagerly(): - optimizer.minimize( - lambda: model(input_value)) - else: - train_op = optimizer.minimize(model(input_value)) - # TODO(allenl): Make initialization more pleasant when graph building. - root_trackable.save_counter # pylint: disable=pointless-statement - self.evaluate(util.gather_initializers( - root_trackable)) - self.evaluate(train_op) - prefix = os.path.join(self.get_temp_dir(), "ckpt") - self.evaluate(state_ops.assign(model._named_dense.variables[1], [42.])) - m_bias_slot = optimizer.get_slot(model._named_dense.variables[1], "m") - self.evaluate(state_ops.assign(m_bias_slot, [1.5])) - save_path = root_trackable.save(file_prefix=prefix) - self.evaluate(state_ops.assign(model._named_dense.variables[1], [43.])) - self.evaluate(state_ops.assign(root_trackable.save_counter, 3)) - optimizer_variables = self.evaluate(optimizer.variables()) - self.evaluate(state_ops.assign(m_bias_slot, [-2.])) - # Immediate restoration - status = root_trackable.restore(save_path=save_path).assert_consumed() - status.run_restore_ops() - self.assertAllEqual([42.], self.evaluate(model._named_dense.variables[1])) - self.assertAllEqual(1, self.evaluate(root_trackable.save_counter)) - self.assertAllEqual([1.5], self.evaluate(m_bias_slot)) - if not context.executing_eagerly(): - return # Restore-on-create is only supported when executing eagerly - on_create_model = MyModel() - on_create_optimizer = adam.AdamOptimizer( - 0.001, - # Preserve beta_1_power and beta_2_power when appying gradients - # so we can test that they've been restored correctly. - beta1=1.0, - beta2=1.0) - on_create_root = util.Checkpoint( - optimizer=on_create_optimizer, model=on_create_model) - # Deferred restoration - status = on_create_root.restore(save_path=save_path) - on_create_model(constant_op.constant([[3.]])) # create variables - self.assertAllEqual(1, self.evaluate(on_create_root.save_counter)) - self.assertAllEqual([42.], - self.evaluate( - on_create_model._named_dense.variables[1])) - on_create_m_bias_slot = on_create_optimizer.get_slot( - on_create_model._named_dense.variables[1], "m") - # Optimizer slot variables are created when the original variable is - # restored. - self.assertAllEqual([1.5], self.evaluate(on_create_m_bias_slot)) - self.assertAllEqual(optimizer_variables[2:], - self.evaluate(on_create_optimizer.variables())) - dummy_var = resource_variable_ops.ResourceVariable([1.]) - on_create_optimizer.minimize(loss=dummy_var.read_value) - status.assert_consumed() - beta_1_power, beta_2_power = on_create_optimizer._get_beta_accumulators() - self.assertAllEqual(optimizer_variables[0], self.evaluate(beta_1_power)) - self.assertAllEqual(optimizer_variables[1], self.evaluate(beta_2_power)) - - # TODO(allenl): Debug garbage created by this test in python3. - def testDeferredRestorationUsageEager(self): - """An idiomatic eager execution example.""" - num_training_steps = 10 - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - for training_continuation in range(3): - model = MyModel() - optimizer = adam.AdamOptimizer(0.001) - root = util.Checkpoint( - optimizer=optimizer, model=model, - optimizer_step=training_util.get_or_create_global_step()) - root.restore(checkpoint_management.latest_checkpoint( - checkpoint_directory)) - for _ in range(num_training_steps): - # TODO(allenl): Use a Dataset and serialize/checkpoint it. - input_value = constant_op.constant([[3.]]) - optimizer.minimize( - lambda: model(input_value), # pylint: disable=cell-var-from-loop - global_step=root.optimizer_step) - root.save(file_prefix=checkpoint_prefix) - self.assertEqual((training_continuation + 1) * num_training_steps, - root.optimizer_step.numpy()) - - def testUsageGraph(self): - """Expected usage when graph building.""" - with context.graph_mode(): - num_training_steps = 10 - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - for training_continuation in range(3): - with ops.Graph().as_default(): - model = MyModel() - optimizer = adam.AdamOptimizer(0.001) - root = util.CheckpointV1( - optimizer=optimizer, model=model, - global_step=training_util.get_or_create_global_step()) - input_value = constant_op.constant([[3.]]) - train_op = optimizer.minimize( - model(input_value), - global_step=root.global_step) - checkpoint_path = checkpoint_management.latest_checkpoint( - checkpoint_directory) - with self.session(graph=ops.get_default_graph()) as session: - status = root.restore(save_path=checkpoint_path) - status.initialize_or_restore(session=session) - if checkpoint_path is None: - self.assertEqual(0, training_continuation) - with self.assertRaises(AssertionError): - status.assert_consumed() - else: - status.assert_consumed() - for _ in range(num_training_steps): - session.run(train_op) - root.save(file_prefix=checkpoint_prefix, session=session) - self.assertEqual((training_continuation + 1) * num_training_steps, - session.run(root.global_step)) - self.assertEqual(training_continuation + 1, - session.run(root.save_counter)) - - @test_util.run_in_graph_and_eager_modes - def testAgnosticUsage(self): - """Graph/eager agnostic usage.""" - # Does create garbage when executing eagerly due to ops.Graph() creation. - num_training_steps = 10 - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - for training_continuation in range(3): - with ops.Graph().as_default(), self.test_session( - graph=ops.get_default_graph()), test_util.device(use_gpu=True): - model = MyModel() - optimizer = adam.AdamOptimizer(0.001) - root = util.Checkpoint( - optimizer=optimizer, model=model, - global_step=training_util.get_or_create_global_step()) - checkpoint_path = checkpoint_management.latest_checkpoint( - checkpoint_directory) - status = root.restore(save_path=checkpoint_path) - input_value = constant_op.constant([[3.]]) - train_fn = functools.partial( - optimizer.minimize, - functools.partial(model, input_value), - global_step=root.global_step) - if not context.executing_eagerly(): - train_fn = functools.partial(self.evaluate, train_fn()) - status.initialize_or_restore() - for _ in range(num_training_steps): - train_fn() - root.save(file_prefix=checkpoint_prefix) - self.assertEqual((training_continuation + 1) * num_training_steps, - self.evaluate(root.global_step)) - self.assertEqual(training_continuation + 1, - self.evaluate(root.save_counter)) - - # pylint: disable=cell-var-from-loop - @test_util.run_in_graph_and_eager_modes - def testWithDefun(self): - num_training_steps = 2 - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - for training_continuation in range(3): - with ops.Graph().as_default(), self.test_session( - graph=ops.get_default_graph()), test_util.device(use_gpu=True): - model = MyModel() - # Don't actually train so we can test variable values - optimizer = adam.AdamOptimizer(0.) - root = util.Checkpoint( - optimizer=optimizer, model=model, - global_step=training_util.get_or_create_global_step()) - checkpoint_path = checkpoint_management.latest_checkpoint( - checkpoint_directory) - status = root.restore(save_path=checkpoint_path) - def train_fn(): - @function.defun - def _call_model(x): - return model(x) - with backprop.GradientTape() as tape: - loss = _call_model(constant_op.constant([[3.]])) - gradients = tape.gradient(loss, model.variables) - return optimizer.apply_gradients(zip(gradients, model.variables), - global_step=root.global_step) - if not context.executing_eagerly(): - train_fn = functools.partial( - self.evaluate, train_fn()) - status.initialize_or_restore() - for _ in range(num_training_steps): - train_fn() - if training_continuation > 0: - status.assert_consumed() - self.assertAllClose([[42.]], self.evaluate(model.variables[0])) - else: - self.evaluate(model.variables[0].assign([[42.]])) - root.save(file_prefix=checkpoint_prefix) - self.assertEqual((training_continuation + 1) * num_training_steps, - self.evaluate(root.global_step)) - self.assertEqual(training_continuation + 1, - self.evaluate(root.save_counter)) - # pylint: enable=cell-var-from-loop - - def testAnonymousVarsInInit(self): - - class Model(training.Model): - - def __init__(self): - super(Model, self).__init__() - self.w = resource_variable_ops.ResourceVariable(0.0) - self.b = resource_variable_ops.ResourceVariable(0.0) - self.vars = [self.w, self.b] - - def call(self, x): - return x * self.w + self.b - - with context.eager_mode(): - model = Model() - optimizer = adam.AdamOptimizer(learning_rate=0.05) - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - checkpoint = util.Checkpoint( - model=model, optimizer=optimizer) - for _ in range(2): - checkpoint.save(checkpoint_prefix) - with backprop.GradientTape() as tape: - loss = (constant_op.constant(1.) - - model(constant_op.constant(1.))) ** 2 - grad = tape.gradient(loss, model.vars) - optimizer.apply_gradients( - [(g, v) for g, v in zip(grad, model.vars)]) - - @test_util.run_in_graph_and_eager_modes - def testDeferredSlotRestoration(self): - checkpoint_directory = self.get_temp_dir() - - root = util.Checkpoint() - root.var = util.add_variable( - root, name="var", initializer=0.) - optimizer = adam.AdamOptimizer(0.1) - if context.executing_eagerly(): - optimizer.minimize(root.var.read_value) - else: - train_op = optimizer.minimize(root.var) - # Note that `optimizer` has not been added as a dependency of - # `root`. Create a one-off grouping so that slot variables for `root.var` - # get initialized too. - self.evaluate(util.gather_initializers( - util.Checkpoint(root=root, optimizer=optimizer))) - self.evaluate(train_op) - self.evaluate(state_ops.assign(root.var, 12.)) - no_slots_path = root.save(os.path.join(checkpoint_directory, "no_slots")) - root.optimizer = optimizer - self.evaluate(state_ops.assign(root.var, 13.)) - self.evaluate(state_ops.assign(optimizer.get_slot(name="m", var=root.var), - 14.)) - slots_path = root.save(os.path.join(checkpoint_directory, "with_slots")) - new_root = util.Checkpoint() - # Load the slot-containing checkpoint (deferred), then immediately overwrite - # the non-slot variable (also deferred). - slot_status = new_root.restore(slots_path) - no_slot_status = new_root.restore(no_slots_path) - with self.assertRaises(AssertionError): - no_slot_status.assert_consumed() - new_root.var = util.add_variable( - new_root, name="var", shape=[]) - no_slot_status.assert_consumed() - no_slot_status.run_restore_ops() - self.assertEqual(12., self.evaluate(new_root.var)) - new_root.optimizer = adam.AdamOptimizer(0.1) - with self.assertRaisesRegexp(AssertionError, "beta1_power"): - slot_status.assert_consumed() - self.assertEqual(12., self.evaluate(new_root.var)) - if context.executing_eagerly(): - # Slot variables are only created with restoring initializers when - # executing eagerly. - self.assertEqual(14., self.evaluate( - new_root.optimizer.get_slot(name="m", var=new_root.var))) - else: - self.assertIs(new_root.optimizer.get_slot(name="m", var=new_root.var), - None) - if context.executing_eagerly(): - new_root.optimizer.minimize(new_root.var.read_value) - else: - train_op = new_root.optimizer.minimize(new_root.var) - # The slot variable now exists; restore() didn't create it, but we should - # now have a restore op for it. - slot_status.run_restore_ops() - self.assertEqual(14., self.evaluate( - new_root.optimizer.get_slot(name="m", var=new_root.var))) - self.evaluate(train_op) - slot_status.assert_consumed() - - def testManySavesGraph(self): - """Saves after the first should not modify the graph.""" - with context.graph_mode(): - graph = ops.Graph() - with graph.as_default(), self.session(graph): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - obj = util.Checkpoint() - obj.var = variable_scope.get_variable(name="v", initializer=0.) - obj.opt = adam.AdamOptimizer(0.1) - obj.opt.minimize(obj.var.read_value()) - self.evaluate(util.gather_initializers(obj)) - obj.save(checkpoint_prefix) - before_ops = graph.get_operations() - obj.save(checkpoint_prefix) - self.assertEqual(before_ops, graph.get_operations()) - - def testManyRestoresGraph(self): - """Restores after the first should not modify the graph.""" - with context.graph_mode(): - graph = ops.Graph() - with graph.as_default(), self.session(graph): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - obj = util.Checkpoint() - obj.var = variable_scope.get_variable(name="v", initializer=0.) - obj.opt = adam.AdamOptimizer(0.1) - obj.opt.minimize(obj.var.read_value()) - self.evaluate(util.gather_initializers(obj)) - save_path = obj.save(checkpoint_prefix) - obj.restore(save_path) - before_ops = graph.get_operations() - obj.restore(save_path) - self.assertEqual(before_ops, graph.get_operations()) - - def testMultipleGraphsNonSlotVariables(self): - with context.graph_mode(): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - optimizer = adam.AdamOptimizer(0.001) - # Construct a model in one graph - first_graph = ops.Graph() - first_session = session_lib.Session(graph=first_graph) - with first_graph.as_default(), first_session.as_default(): - first_variable = resource_variable_ops.ResourceVariable([1.]) - first_root_trackable = util.Checkpoint( - optimizer=optimizer, variable=first_variable) - train_op = optimizer.minimize(first_variable.read_value) - self.evaluate(util.gather_initializers( - first_root_trackable)) - self.evaluate(train_op) - self.evaluate(first_variable.assign([1.])) - self.evaluate(optimizer.get_slot( - var=first_variable, name="m").assign([2.])) - beta_1_power, _ = optimizer._get_beta_accumulators() - self.evaluate(beta_1_power.assign(3.)) - - # Save and load in a second graph - second_graph = ops.Graph() - with second_graph.as_default(), session_lib.Session(graph=second_graph): - second_variable = resource_variable_ops.ResourceVariable([1.]) - second_root_trackable = util.Checkpoint( - optimizer=optimizer, variable=second_variable) - train_op = optimizer.minimize(second_variable.read_value) - second_root_trackable.restore(None).initialize_or_restore() - self.evaluate(train_op) - self.evaluate(second_variable.assign([4.])) - self.evaluate(optimizer.get_slot( - var=second_variable, name="m").assign([5.])) - beta_1_power, _ = optimizer._get_beta_accumulators() - self.evaluate(beta_1_power.assign(6.)) - save_path = second_root_trackable.save(checkpoint_prefix) - self.evaluate(second_variable.assign([7.])) - self.evaluate(optimizer.get_slot( - var=second_variable, name="m").assign([8.])) - beta_1_power, _ = optimizer._get_beta_accumulators() - self.assertAllEqual(6., self.evaluate(beta_1_power)) - status = second_root_trackable.restore(save_path) - status.assert_consumed().run_restore_ops() - self.assertAllEqual([4.], self.evaluate(second_variable)) - self.assertAllEqual([5.], self.evaluate(optimizer.get_slot( - var=second_variable, name="m"))) - beta_1_power, _ = optimizer._get_beta_accumulators() - self.assertAllEqual(6., self.evaluate(beta_1_power)) - - # Check that the first graph is unmolested - with first_graph.as_default(), first_session.as_default(): - self.assertAllEqual([1.], self.evaluate(first_variable)) - self.assertAllEqual([2.], self.evaluate(optimizer.get_slot( - var=first_variable, name="m"))) - beta_1_power, _ = optimizer._get_beta_accumulators() - self.assertAllEqual(3., self.evaluate(beta_1_power)) - - -class TemplateTests(test.TestCase): - - @test_util.run_in_graph_and_eager_modes - def test_trackable_save_restore(self): - - def _templated(): - v = variable_scope.get_variable( - "v", shape=[1], initializer=init_ops.zeros_initializer(), - use_resource=True) - v2 = variable_scope.get_variable( - "v2", shape=[1], initializer=init_ops.zeros_initializer(), - use_resource=True) - return v, v + 1., v2 - - save_template = template.make_template("s1", _templated) - v1_save, _, v2_save = save_template() - optimizer = adam.AdamOptimizer(0.0) - save_root = util.Checkpoint( - my_template=save_template, optimizer=optimizer) - optimizer.minimize(v1_save.read_value) - self.evaluate([v.initializer for v in optimizer.variables()]) - self.evaluate(v1_save.assign([12.])) - self.evaluate(v2_save.assign([14.])) - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - save_path = save_root.save(checkpoint_prefix) - - load_template = template.make_template("s2", _templated) - load_optimizer = adam.AdamOptimizer(0.0) - load_root = util.Checkpoint( - my_template=load_template, optimizer=load_optimizer) - status = load_root.restore(save_path) - var, var_plus_one, var2 = load_template() - load_optimizer.minimize(var.read_value) - self.assertEqual(2, len(load_template._checkpoint_dependencies)) - self.assertEqual("v", load_template._checkpoint_dependencies[0].name) - self.assertEqual("v2", load_template._checkpoint_dependencies[1].name) - status.assert_consumed().run_restore_ops() - self.assertAllEqual([12.], self.evaluate(var)) - self.assertAllEqual([13.], self.evaluate(var_plus_one)) - self.assertAllEqual([14.], self.evaluate(var2)) - - -class CheckpointCompatibilityTests(test.TestCase): - - def _initialized_model(self): - input_value = constant_op.constant([[3.]]) - model = MyModel() - optimizer = adam.AdamOptimizer(0.001) - optimizer_step = training_util.get_or_create_global_step() - root_trackable = util.Checkpoint( - optimizer=optimizer, model=model, optimizer_step=optimizer_step) - train_op = optimizer.minimize( - functools.partial(model, input_value), - global_step=optimizer_step) - self.evaluate(util.gather_initializers( - root_trackable)) - self.evaluate(train_op) - # A regular variable, a slot variable, and a non-slot Optimizer variable - # with known values to check when loading. - self.evaluate(model._named_dense.bias.assign([1.])) - self.evaluate(optimizer.get_slot( - var=model._named_dense.bias, name="m").assign([2.])) - beta_1_power, _ = optimizer._get_beta_accumulators() - self.evaluate(beta_1_power.assign(3.)) - return root_trackable - - def _set_sentinels(self, root_trackable): - self.evaluate(root_trackable.model._named_dense.bias.assign([101.])) - self.evaluate( - root_trackable.optimizer.get_slot( - var=root_trackable.model._named_dense.bias, name="m") - .assign([102.])) - beta_1_power, _ = root_trackable.optimizer._get_beta_accumulators() - self.evaluate(beta_1_power.assign(103.)) - - def _check_sentinels(self, root_trackable): - self.assertAllEqual( - [1.], self.evaluate(root_trackable.model._named_dense.bias)) - self.assertAllEqual([2.], self.evaluate( - root_trackable.optimizer.get_slot( - var=root_trackable.model._named_dense.bias, name="m"))) - beta_1_power, _ = root_trackable.optimizer._get_beta_accumulators() - self.assertAllEqual(3., self.evaluate(beta_1_power)) - - def _write_name_based_checkpoint(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - with context.graph_mode(): - save_graph = ops.Graph() - with save_graph.as_default(), self.test_session( - graph=save_graph) as session: - root = self._initialized_model() - name_saver = core_saver.Saver() - return name_saver.save( - sess=session, save_path=checkpoint_prefix, - global_step=root.optimizer_step) - - @test_util.run_in_graph_and_eager_modes - def testLoadFromNameBasedSaver(self): - """Save a name-based checkpoint, load it using the object-based API.""" - with test_util.device(use_gpu=True): - save_path = self._write_name_based_checkpoint() - root = self._initialized_model() - self._set_sentinels(root) - with self.assertRaises(AssertionError): - self._check_sentinels(root) - object_saver = util.TrackableSaver(graph_view.ObjectGraphView(root)) - self._set_sentinels(root) - status = object_saver.restore(save_path) - if context.executing_eagerly(): - self._check_sentinels(root) - if context.executing_eagerly(): - status.assert_consumed() - else: - # When graph building, we haven't read any keys, so we don't know - # whether the restore will be complete. - with self.assertRaisesRegexp(AssertionError, "not restored"): - status.assert_consumed() - status.run_restore_ops() - self._check_sentinels(root) - self._set_sentinels(root) - status = object_saver.restore(save_path) - status.initialize_or_restore() - self._check_sentinels(root) - - # TODO(allenl): Test for the core name-based saver loading object-based - # checkpoints once object-based checkpointing is in core. - - def testSaveGraphLoadEager(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - with context.graph_mode(): - save_graph = ops.Graph() - with save_graph.as_default(), self.test_session( - graph=save_graph): - root = self._initialized_model() - save_path = root.save(file_prefix=checkpoint_prefix) - with context.eager_mode(): - root = self._initialized_model() - self._set_sentinels(root) - root.restore(save_path).assert_consumed() - self._check_sentinels(root) - - def testSaveEagerLoadGraph(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - with context.eager_mode(): - root = self._initialized_model() - save_path = root.save(file_prefix=checkpoint_prefix) - with context.graph_mode(): - save_graph = ops.Graph() - with save_graph.as_default(), self.test_session(graph=save_graph): - root = self._initialized_model() - self._set_sentinels(root) - root.restore(save_path).assert_consumed().run_restore_ops() - self._check_sentinels(root) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/optimizer_v2/gradient_descent.py b/tensorflow/contrib/optimizer_v2/gradient_descent.py deleted file mode 100644 index d103a55a357..00000000000 --- a/tensorflow/contrib/optimizer_v2/gradient_descent.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Momentum for TensorFlow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.optimizer_v2 import optimizer_v2 -from tensorflow.python.framework import ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.training import training_ops - - -class GradientDescentOptimizer(optimizer_v2.OptimizerV2): - """Optimizer that implements the gradient descent algorithm.""" - - def __init__(self, learning_rate, use_locking=False, name="GradientDescent"): - """Construct a new gradient descent optimizer. - - The learning rate arg below is a hyperparameter where a hyperparameter is - defined as a scalar Tensor, a regular Python value or a callable (which - will be evaluated when `apply_gradients` is called) returning a scalar - Tensor or a Python value. - - Args: - learning_rate: A float hyperparameter. The learning rate to use. - use_locking: If True use locks for update operations. - name: Optional name prefix for the operations created when applying - gradients. Defaults to "GradientDescent". - """ - super(GradientDescentOptimizer, self).__init__(use_locking, name) - self._set_hyper("learning_rate", learning_rate) - - def _apply_dense(self, grad, var, state): - return training_ops.apply_gradient_descent( - var, - state.get_hyper("learning_rate", var.dtype.base_dtype), - grad, - use_locking=self._use_locking).op - - def _resource_apply_dense(self, grad, handle, state): - lr = state.get_hyper("learning_rate", grad.dtype.base_dtype) - return training_ops.resource_apply_gradient_descent( - handle.handle, lr, grad, use_locking=self._use_locking) - - def _resource_apply_sparse_duplicate_indices(self, grad, handle, indices, - state): - lr = state.get_hyper("learning_rate", grad.dtype.base_dtype) - return resource_variable_ops.resource_scatter_add(handle.handle, indices, - -grad * lr) - - def _apply_sparse_duplicate_indices(self, grad, var, state): - delta = ops.IndexedSlices( - grad.values * state.get_hyper("learning_rate", var.dtype.base_dtype), - grad.indices, grad.dense_shape) - return var.scatter_sub(delta, use_locking=self._use_locking) diff --git a/tensorflow/contrib/optimizer_v2/gradient_descent_test.py b/tensorflow/contrib/optimizer_v2/gradient_descent_test.py deleted file mode 100644 index 4a77bce478c..00000000000 --- a/tensorflow/contrib/optimizer_v2/gradient_descent_test.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functional test for GradientDescent optimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.optimizer_v2 import gradient_descent -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import resources -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class GradientDescentOptimizerTest(test.TestCase): - - def testBasic(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - optimizer = gradient_descent.GradientDescentOptimizer(3.0) - sgd_op = optimizer.apply_gradients( - zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval()) - # Run 1 step of sgd - sgd_op.run() - # Validate updated params - self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], - var0.eval()) - self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], - var1.eval()) - self.assertEqual(0, len(optimizer.variables())) - - def testBasicResourceVariable(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) - var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - sgd_op = gradient_descent.GradientDescentOptimizer(3.0).apply_gradients( - zip([grads0, grads1], [var0, var1])) - # TODO(apassos) calling initialize_resources on all resources here - # doesn't work because the sessions and graph are reused across unit - # tests and this would mean trying to reinitialize variables. Figure out - # a long-term solution for this. - resources.initialize_resources([var0, var1]).run() - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval()) - # Run 1 step of sgd - sgd_op.run() - # Validate updated params - self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], - var0.eval()) - self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], - var1.eval()) - - def testMinimizeResourceVariable(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) - var1 = resource_variable_ops.ResourceVariable([3.0], dtype=dtype) - x = constant_op.constant([[4.0], [5.0]], dtype=dtype) - pred = math_ops.matmul(var0, x) + var1 - loss = pred * pred - sgd_op = gradient_descent.GradientDescentOptimizer(1.0).minimize(loss) - # TODO(apassos) calling initialize_resources on all resources here - # doesn't work because the sessions and graph are reused across unit - # tests and this would mean trying to reinitialize variables. Figure out - # a long-term solution for this. - resources.initialize_resources([var0, var1]).run() - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) - self.assertAllCloseAccordingToType([3.0], var1.eval()) - # Run 1 step of sgd - sgd_op.run() - # Validate updated params - np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0 - np_grad = 2 * np_pred - self.assertAllCloseAccordingToType( - [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], var0.eval()) - self.assertAllCloseAccordingToType([3.0 - np_grad], var1.eval()) - - def testMinimizeSparseResourceVariable(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) - var1 = resource_variable_ops.ResourceVariable([3.0], dtype=dtype) - x = constant_op.constant([[4.0], [5.0]], dtype=dtype) - pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) - pred += var1 - loss = pred * pred - sgd_op = gradient_descent.GradientDescentOptimizer(1.0).minimize(loss) - # TODO(apassos) calling initialize_resources on all resources here - # doesn't work because the sessions and graph are reused across unit - # tests and this would mean trying to reinitialize variables. Figure out - # a long-term solution for this. - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) - self.assertAllCloseAccordingToType([3.0], var1.eval()) - # Run 1 step of sgd - sgd_op.run() - # Validate updated params - np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0 - np_grad = 2 * np_pred - self.assertAllCloseAccordingToType( - [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], var0.eval()) - self.assertAllCloseAccordingToType([3.0 - np_grad], var1.eval()) - - def testTensorLearningRate(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - lrate = constant_op.constant(3.0) - sgd_op = gradient_descent.GradientDescentOptimizer( - lrate).apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval()) - # Run 1 step of sgd - sgd_op.run() - # Validate updated params - self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], - var0.eval()) - self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], - var1.eval()) - - def testGradWrtRef(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - opt = gradient_descent.GradientDescentOptimizer(3.0) - values = [1.0, 3.0] - vars_ = [variables.Variable([v], dtype=dtype) for v in values] - grads_and_vars = opt.compute_gradients(vars_[0] + vars_[1], vars_) - variables.global_variables_initializer().run() - for grad, _ in grads_and_vars: - self.assertAllCloseAccordingToType([1.0], grad.eval()) - - def testWithGlobalStep(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - global_step = variables.Variable(0, trainable=False) - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - sgd_op = gradient_descent.GradientDescentOptimizer(3.0).apply_gradients( - zip([grads0, grads1], [var0, var1]), global_step=global_step) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval()) - # Run 1 step of sgd - sgd_op.run() - # Validate updated params and global_step - self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], - var0.eval()) - self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], - var1.eval()) - self.assertAllCloseAccordingToType(1, global_step.eval()) - - def testSparseBasic(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) - var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) - grads0 = ops.IndexedSlices( - constant_op.constant( - [0.1], shape=[1, 1], dtype=dtype), - constant_op.constant([0]), - constant_op.constant([2, 1])) - grads1 = ops.IndexedSlices( - constant_op.constant( - [0.01], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), - constant_op.constant([2, 1])) - sgd_op = gradient_descent.GradientDescentOptimizer(3.0).apply_gradients( - zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0], [2.0]], var0.eval()) - self.assertAllCloseAccordingToType([[3.0], [4.0]], var1.eval()) - # Run 1 step of sgd - sgd_op.run() - # Validate updated params - self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], - var0.eval()) - self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], - var1.eval()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/optimizer_v2/momentum.py b/tensorflow/contrib/optimizer_v2/momentum.py deleted file mode 100644 index 0a5aadc2d13..00000000000 --- a/tensorflow/contrib/optimizer_v2/momentum.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Momentum for TensorFlow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.optimizer_v2 import optimizer_v2 -from tensorflow.python.training import training_ops - - -class MomentumOptimizer(optimizer_v2.OptimizerV2): - """Optimizer that implements the Momentum algorithm. - - Computes (if `use_nesterov = False`): - - ``` - accumulation = momentum * accumulation + gradient - variable -= learning_rate * accumulation - ``` - - Note that in the dense version of this algorithm, `accumulation` is updated - and applied regardless of a gradient's value, whereas the sparse version (when - the gradient is an `IndexedSlices`, typically because of `tf.gather` or an - embedding) only updates variable slices and corresponding `accumulation` terms - when that part of the variable was used in the forward pass. - """ - - def __init__(self, learning_rate, momentum, - use_locking=False, name="Momentum", use_nesterov=False): - """Construct a new Momentum optimizer. - - Some of the args below are hyperparameters, where a hyperparameter is - defined as a scalar Tensor, a regular Python value or a callable (which - will be evaluated when `apply_gradients` is called) returning a scalar - Tensor or a Python value. - - Args: - learning_rate: A float hyperparameter. The learning rate. - momentum: A float hyperparameter. The momentum. - use_locking: If `True` use locks for update operations. - name: Optional name prefix for the operations created when applying - gradients. Defaults to "Momentum". - use_nesterov: If `True` use Nesterov Momentum. - See [Sutskever et al., 2013]( - http://jmlr.org/proceedings/papers/v28/sutskever13.pdf). - This implementation always computes gradients at the value of the - variable(s) passed to the optimizer. Using Nesterov Momentum makes the - variable(s) track the values called `theta_t + mu*v_t` in the paper. - - @compatibility(eager) - When eager execution is enabled, learning_rate and momentum can each be a - callable that takes no arguments and returns the actual value to use. This - can be useful for changing these values across different invocations of - optimizer functions. - @end_compatibility - """ - super(MomentumOptimizer, self).__init__(use_locking, name) - self._set_hyper("learning_rate", learning_rate) - self._set_hyper("momentum", momentum) - self._use_nesterov = use_nesterov - - def _create_vars(self, var_list, state): - for v in var_list: - state.zeros_slot(v, "momentum") - - def _apply_dense(self, grad, var, state): - mom = state.get_slot(var, "momentum") - return training_ops.apply_momentum( - var, - mom, - state.get_hyper("learning_rate", var.dtype.base_dtype), - grad, - state.get_hyper("momentum", var.dtype.base_dtype), - use_locking=self._use_locking, - use_nesterov=self._use_nesterov).op - - def _resource_apply_dense(self, grad, var, state): - mom = state.get_slot(var, "momentum") - return training_ops.resource_apply_momentum( - var.handle, - mom.handle, - state.get_hyper("learning_rate", var.dtype.base_dtype), - grad, - state.get_hyper("momentum", var.dtype.base_dtype), - use_locking=self._use_locking, - use_nesterov=self._use_nesterov) - - def _apply_sparse(self, grad, var, state): - mom = state.get_slot(var, "momentum") - return training_ops.sparse_apply_momentum( - var, - mom, - state.get_hyper("learning_rate", var.dtype.base_dtype), - grad.values, - grad.indices, - state.get_hyper("momentum", var.dtype.base_dtype), - use_locking=self._use_locking, - use_nesterov=self._use_nesterov).op - - def _resource_apply_sparse(self, grad, var, indices, state): - mom = state.get_slot(var, "momentum") - return training_ops.resource_sparse_apply_momentum( - var.handle, - mom.handle, - state.get_hyper("learning_rate", var.dtype.base_dtype), - grad, - indices, - state.get_hyper("momentum", var.dtype.base_dtype), - use_locking=self._use_locking, - use_nesterov=self._use_nesterov) diff --git a/tensorflow/contrib/optimizer_v2/momentum_test.py b/tensorflow/contrib/optimizer_v2/momentum_test.py deleted file mode 100644 index e69f12839e9..00000000000 --- a/tensorflow/contrib/optimizer_v2/momentum_test.py +++ /dev/null @@ -1,575 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Momentum.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib.optimizer_v2 import momentum as momentum_lib -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class MomentumOptimizerTest(test.TestCase): - - def _update_nesterov_momentum_numpy(self, var, accum, g, lr, momentum): - var = var + accum * lr * momentum - accum = accum * momentum + g - var = var - lr * accum - var = var - accum * lr * momentum - return var, accum - - def doTestBasic(self, use_resource=False, use_callable_params=False): - for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - if use_resource: - var0 = resource_variable_ops.ResourceVariable( - [1.0, 2.0], dtype=dtype, name="var0_%d" % i) - var1 = resource_variable_ops.ResourceVariable( - [3.0, 4.0], dtype=dtype, name="var1_%d" % i) - else: - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - learning_rate = lambda: 2.0 - momentum = lambda: 0.9 - if not use_callable_params: - learning_rate = learning_rate() - momentum = momentum() - mom_opt = momentum_lib.MomentumOptimizer( - learning_rate=learning_rate, momentum=momentum) - mom_update = mom_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - - if not context.executing_eagerly(): - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - # Check we have slots - self.assertEqual(["momentum"], mom_opt.get_slot_names()) - slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEquals(slot0.get_shape(), var0.get_shape()) - slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEquals(slot1.get_shape(), var1.get_shape()) - if not context.executing_eagerly(): - self.assertFalse(slot0 in variables.trainable_variables()) - self.assertFalse(slot1 in variables.trainable_variables()) - - # Step 1: the momentum accumulators where 0. So we should see a normal - # update: v -= grad * learning_rate - if not context.executing_eagerly(): - self.evaluate(mom_update) - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), - self.evaluate(slot0)) - self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), - self.evaluate(slot1)) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), - self.evaluate(var0)) - self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), - self.evaluate(var1)) - # Step 2: the momentum accumulators contain the previous update. - if context.executing_eagerly(): - mom_opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - else: - self.evaluate(mom_update) - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), - self.evaluate(slot0)) - self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), - self.evaluate(slot1)) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([ - 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), - 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), self.evaluate(var0)) - self.assertAllCloseAccordingToType( - np.array([ - 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - ( - (0.9 * 0.01 + 0.01) * 2.0) - ]), self.evaluate(var1)) - - def testBasic(self): - with self.cached_session(): - self.doTestBasic(use_resource=False) - - @test_util.run_in_graph_and_eager_modes(reset_test=True) - def testResourceBasic(self): - self.doTestBasic(use_resource=True) - - def testBasicCallableParams(self): - with context.eager_mode(): - self.doTestBasic(use_resource=True, use_callable_params=True) - - def testVariablesAcrossGraphs(self): - optimizer = momentum_lib.MomentumOptimizer(0.01, 0.5) - with ops.Graph().as_default(): - var0 = resource_variable_ops.ResourceVariable( - [1.0, 2.0], dtype=dtypes.float32, name="var0") - var1 = resource_variable_ops.ResourceVariable( - [3.0, 4.0], dtype=dtypes.float32, name="var1") - loss = math_ops.reduce_sum(var0 + var1) - optimizer.minimize(loss) - optimizer_variables = optimizer.variables() - self.assertStartsWith(optimizer_variables[0].name, "var0") - self.assertStartsWith(optimizer_variables[1].name, "var1") - self.assertEquals(2, len(optimizer_variables)) - - with ops.Graph().as_default(): - var2 = resource_variable_ops.ResourceVariable( - [1.0, 2.0], dtype=dtypes.float32, name="var2") - var3 = resource_variable_ops.ResourceVariable( - [3.0, 4.0], dtype=dtypes.float32, name="var3") - loss = math_ops.reduce_sum(var2 + var3) - optimizer.minimize(loss) - optimizer_variables = optimizer.variables() - self.assertStartsWith(optimizer_variables[0].name, "var2") - self.assertStartsWith(optimizer_variables[1].name, "var3") - self.assertEquals(2, len(optimizer_variables)) - - def testNesterovMomentum(self): - for dtype in [dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - cost = 5 * var0 * var0 + 3 * var1 - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name="global_step") - mom_op = momentum_lib.MomentumOptimizer( - learning_rate=2.0, momentum=0.9, use_nesterov=True) - opt_op = mom_op.minimize(cost, global_step, [var0, var1]) - variables.global_variables_initializer().run() - for t in range(1, 5): - opt_op.run() - var0_np, accum0_np = self._update_nesterov_momentum_numpy( - var0_np, accum0_np, var0_np * 10, 2.0, 0.9) - var1_np, accum1_np = self._update_nesterov_momentum_numpy(var1_np, - accum1_np, - 3, 2.0, 0.9) - self.assertAllClose(var0_np, var0.eval()) - self.assertAllClose(var1_np, var1.eval()) - - def testSparseNesterovMomentum(self): - for dtype in [dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - grads = [] - for t in range(1, 5): - grads.append(var0_np * 10) - var0_np, accum0_np = self._update_nesterov_momentum_numpy( - var0_np, accum0_np, var0_np * 10, 2.0, 0.9) - var1_np, accum1_np = self._update_nesterov_momentum_numpy(var1_np, - accum1_np, - 3, 2.0, 0.9) - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - loss = 5 * var0 * var0 + 3 * var1 - mom_op = momentum_lib.MomentumOptimizer( - learning_rate=2.0, momentum=0.9, use_nesterov=True) - x_feed = array_ops.placeholder(dtype) - y_feed = ops.IndexedSlices( - x_feed, constant_op.constant([0, 1]), constant_op.constant([2])) - grads_and_vars = [(y_feed, var0), (constant_op.constant( - [3.0, 3.0], dtype=dtype), var1)] - opt_update = mom_op.apply_gradients(grads_and_vars) - variables.global_variables_initializer().run() - for t in range(1, 5): - opt_update.run(feed_dict={x_feed: grads[t - 1]}) - var0_np, accum0_np = self._update_nesterov_momentum_numpy( - var0_np, accum0_np, var0_np * 10, 2.0, 0.9) - var1_np, accum1_np = self._update_nesterov_momentum_numpy(var1_np, - accum1_np, - 3, 2.0, 0.9) - self.assertAllClose(var0_np, var0.eval()) - self.assertAllClose(var1_np, var1.eval()) - - @test_util.run_in_graph_and_eager_modes(reset_test=True) - def testMinimizeSparseResourceVariable(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - # This test invokes the ResourceSparseApplyMomentum operation, which - # did not have a registered GPU kernel as of April 2018. With graph - # execution, the placement algorithm notices this and automatically - # places the variable in CPU (host) memory. With eager execution, - # the variable would be placed in GPU memory if available, which - # would then conflict with the future invocation of the - # ResourceSparseApplyMomentum operation. - # To work around this discrepancy, for now we force the variable - # to be placed on CPU. - with ops.device("/cpu:0"): - var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) - - # pylint: disable=cell-var-from-loop - def loss(): - x = constant_op.constant([[4.0], [5.0]], dtype=dtype) - pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) - return pred * pred - # pylint: enable=cell-var-from-loop - - opt = momentum_lib.MomentumOptimizer(learning_rate=1.0, momentum=0.0) - sgd_op = opt.minimize(loss) - self.evaluate(variables.global_variables_initializer()) - # Run 1 step of sgd - self.evaluate(sgd_op) - # Validate updated params - self.assertAllCloseAccordingToType([[-111, -138]], self.evaluate(var0)) - - @test_util.run_in_graph_and_eager_modes(reset_test=True) - def testMinimizeWith2DIndiciesForEmbeddingLookup(self): - # This test invokes the ResourceSparseApplyMomentum operation, which - # did not have a registered GPU kernel as of April 2018. With graph - # execution, the placement algorithm notices this and automatically - # places the variable in CPU (host) memory. With eager execution, - # the variable would be placed in GPU memory if available, which - # would then conflict with the future invocation of the - # ResourceSparseApplyMomentum operation. - # To work around this discrepancy, for now we force the variable - # to be placed on CPU. - with ops.device("/cpu:0"): - var0 = resource_variable_ops.ResourceVariable(array_ops.ones([2, 2])) - - def loss(): - return math_ops.reduce_sum(embedding_ops.embedding_lookup(var0, [[1]])) - - opt = momentum_lib.MomentumOptimizer(learning_rate=1.0, momentum=0.0) - sgd_op = opt.minimize(loss) - self.evaluate(variables.global_variables_initializer()) - self.evaluate(sgd_op) - self.assertAllCloseAccordingToType([[1, 1], [0, 0]], self.evaluate(var0)) - - def testTensorLearningRateAndMomentum(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - mom_opt = momentum_lib.MomentumOptimizer( - learning_rate=constant_op.constant(2.0), - momentum=constant_op.constant(0.9)) - mom_update = mom_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - # Check we have slots - self.assertEqual(["momentum"], mom_opt.get_slot_names()) - slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEquals(slot0.get_shape(), var0.get_shape()) - self.assertFalse(slot0 in variables.trainable_variables()) - slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEquals(slot1.get_shape(), var1.get_shape()) - self.assertFalse(slot1 in variables.trainable_variables()) - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Step 1: the momentum accumulators where 0. So we should see a normal - # update: v -= grad * learning_rate - mom_update.run() - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval()) - self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval()) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval()) - # Step 2: the momentum accumulators contain the previous update. - mom_update.run() - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval()) - self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval()) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([ - 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), - 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([ - 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - ( - (0.9 * 0.01 + 0.01) * 2.0) - ]), var1.eval()) - - def _dbParamsMom01(self): - """Return dist-belief momentum values. - - Return values been generated from the dist-belief momentum unittest, - running with a learning rate of 0.1 and a momentum of 0.1. - - These values record how a parameter vector of size 10, initialized with 0.0, - gets updated with 10 consecutive momentum steps. It uses random gradients. - - Returns: - db_grad: The gradients to apply - db_out: The parameters after the momentum update. - """ - db_grad = [[]] * 10 - db_out = [[]] * 10 - # pylint: disable=line-too-long - db_grad[0] = [ - 0.00096264342, 0.17914793, 0.93945462, 0.41396621, 0.53037018, - 0.93197989, 0.78648776, 0.50036013, 0.55345792, 0.96722615 - ] - db_out[0] = [ - -9.6264346e-05, -0.017914793, -0.093945466, -0.041396622, -0.053037018, - -0.093197994, -0.078648776, -0.050036013, -0.055345792, -0.096722618 - ] - db_grad[1] = [ - 0.17075552, 0.88821375, 0.20873757, 0.25236958, 0.57578111, 0.15312378, - 0.5513742, 0.94687688, 0.16012503, 0.22159521 - ] - db_out[1] = [ - -0.017181443, -0.10852765, -0.12421377, -0.070773244, -0.11591884, - -0.11783017, -0.14165108, -0.14972731, -0.076892875, -0.1285544 - ] - db_grad[2] = [ - 0.35077485, 0.47304362, 0.44412705, 0.44368884, 0.078527533, 0.81223965, - 0.31168157, 0.43203235, 0.16792089, 0.24644311 - ] - db_out[2] = [ - -0.053967446, -0.1648933, -0.1716533, -0.1180798, -0.13005978, - -0.20151734, -0.17911947, -0.20289968, -0.095839672, -0.15638189 - ] - db_grad[3] = [ - 0.9694621, 0.75035888, 0.28171822, 0.83813518, 0.53807181, 0.3728098, - 0.81454384, 0.03848977, 0.89759839, 0.93665648 - ] - db_out[3] = [ - -0.15459226, -0.24556576, -0.20456907, -0.20662397, -0.18528105, - -0.24716705, -0.2643207, -0.21206589, -0.18749419, -0.2528303 - ] - db_grad[4] = [ - 0.38578293, 0.8536852, 0.88722926, 0.66276771, 0.13678469, 0.94036359, - 0.69107032, 0.81897682, 0.5433259, 0.67860287 - ] - db_out[4] = [ - -0.20323303, -0.33900154, -0.29658359, -0.28175515, -0.20448165, - -0.34576839, -0.34194785, -0.29488021, -0.25099224, -0.33033544 - ] - db_grad[5] = [ - 0.27885768, 0.76100707, 0.24625534, 0.81354135, 0.18959245, 0.48038563, - 0.84163809, 0.41172323, 0.83259648, 0.44941229 - ] - db_out[5] = [ - -0.23598288, -0.42444581, -0.33041057, -0.3706224, -0.22536094, - -0.40366709, -0.43387437, -0.34433398, -0.34060168, -0.38302717 - ] - db_grad[6] = [ - 0.27233034, 0.056316052, 0.5039115, 0.24105175, 0.35697976, 0.75913221, - 0.73577434, 0.16014607, 0.57500273, 0.071136251 - ] - db_out[6] = [ - -0.26649091, -0.43862185, -0.38418442, -0.40361428, -0.26314685, - -0.48537019, -0.51664448, -0.36529395, -0.40706289, -0.39540997 - ] - db_grad[7] = [ - 0.58697265, 0.2494842, 0.08106143, 0.39954534, 0.15892942, 0.12683646, - 0.74053431, 0.16033, 0.66625422, 0.73515922 - ] - db_out[7] = [ - -0.32823896, -0.46498787, -0.39766794, -0.446868, -0.28281838, - -0.50622416, -0.59897494, -0.38342294, -0.48033443, -0.47016418 - ] - db_grad[8] = [ - 0.8215279, 0.41994119, 0.95172721, 0.68000203, 0.79439718, 0.43384039, - 0.55561525, 0.22567581, 0.93331909, 0.29438227 - ] - db_out[8] = [ - -0.41656655, -0.50961858, -0.49418902, -0.51919359, -0.36422527, - -0.55169362, -0.6627695, -0.40780342, -0.58099347, -0.50707781 - ] - db_grad[9] = [ - 0.68297005, 0.67758518, 0.1748755, 0.13266537, 0.70697063, 0.055731893, - 0.68593478, 0.50580865, 0.12602448, 0.093537711 - ] - db_out[9] = [ - -0.49369633, -0.58184016, -0.52132869, -0.5396927, -0.44306302, - -0.56181377, -0.73774242, -0.46082234, -0.60366184, -0.52012295 - ] - # pylint: enable=line-too-long - return db_grad, db_out - - def testLikeDistBeliefMom01(self): - with self.cached_session(): - db_grad, db_out = self._dbParamsMom01() - num_samples = len(db_grad) - var0 = variables.Variable([0.0] * num_samples) - grads0 = constant_op.constant([0.0] * num_samples) - mom_opt = momentum_lib.MomentumOptimizer(learning_rate=0.1, momentum=0.1) - mom_update = mom_opt.apply_gradients(zip([grads0], [var0])) - variables.global_variables_initializer().run() - for i in xrange(num_samples): - mom_update.run(feed_dict={grads0: db_grad[i]}) - self.assertAllClose(np.array(db_out[i]), var0.eval()) - - def testSparse(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable(array_ops.zeros([4, 2], dtype=dtype)) - var1 = variables.Variable(constant_op.constant(1.0, dtype, [4, 2])) - grads0 = ops.IndexedSlices( - constant_op.constant( - [[.1, .1]], dtype=dtype), - constant_op.constant([1]), - constant_op.constant([4, 2])) - grads1 = ops.IndexedSlices( - constant_op.constant( - [[.01, .01], [.01, .01]], dtype=dtype), - constant_op.constant([2, 3]), - constant_op.constant([4, 2])) - mom_opt = momentum_lib.MomentumOptimizer( - learning_rate=2.0, momentum=0.9) - mom_update = mom_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - # Check we have slots - self.assertEqual(["momentum"], mom_opt.get_slot_names()) - slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEquals(slot0.get_shape(), var0.get_shape()) - slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEquals(slot1.get_shape(), var1.get_shape()) - - # Fetch params to validate initial values - self.assertAllClose([0, 0], var0.eval()[0]) - self.assertAllClose([0, 0], var0.eval()[1]) - self.assertAllClose([1, 1], var1.eval()[2]) - - # Step 1: the momentum accumulators are 0. So we should see a normal - # update: v -= grad * learning_rate - mom_update.run() - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType(np.array([0, 0]), slot0.eval()[0]) - self.assertAllCloseAccordingToType(np.array([.1, .1]), slot0.eval()[1]) - self.assertAllCloseAccordingToType( - np.array([.01, .01]), slot1.eval()[2]) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType(np.array([0, 0]), var0.eval()[0]) - self.assertAllCloseAccordingToType( - np.array([-(0.1 * 2.0), -(0.1 * 2.0)]), var0.eval()[1]) - self.assertAllCloseAccordingToType( - np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]), var1.eval()[2]) - # Step 2: the momentum accumulators contain the previous update. - mom_update.run() - # Check that the momentum accumulators have been updated. - self.assertAllClose(np.array([0, 0]), slot0.eval()[0]) - self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval()[1]) - self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), - slot1.eval()[2]) - # Check that the parameters have been updated. - self.assertAllClose(np.array([0, 0]), var0.eval()[0]) - self.assertAllCloseAccordingToType( - np.array([ - -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), -(0.1 * 2.0) - ( - (0.9 * 0.1 + 0.1) * 2.0) - ]), var0.eval()[1]) - self.assertAllCloseAccordingToType( - np.array([ - 0.98 - ((0.9 * 0.01 + 0.01) * 2.0), 0.98 - ( - (0.9 * 0.01 + 0.01) * 2.0) - ]), var1.eval()[2]) - - def testSharing(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - mom_opt = momentum_lib.MomentumOptimizer( - learning_rate=2.0, momentum=0.9) - mom_update1 = mom_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - mom_update2 = mom_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - self.assertEqual(["momentum"], mom_opt.get_slot_names()) - slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEquals(slot0.get_shape(), var0.get_shape()) - slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEquals(slot1.get_shape(), var1.get_shape()) - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Step 1: the momentum accumulators where 0. So we should see a normal - # update: v -= grad * learning_rate - mom_update1.run() - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval()) - self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval()) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval()) - # Step 2: the second momentum accumulators contain the previous update. - mom_update2.run() - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval()) - self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval()) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([ - 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), - 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([ - 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - ( - (0.9 * 0.01 + 0.01) * 2.0) - ]), var1.eval()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py deleted file mode 100644 index 7bcf07fbddc..00000000000 --- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py +++ /dev/null @@ -1,1359 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Version 2 of class Optimizer.""" -# pylint: disable=g-bad-name - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - -from tensorflow.python.distribute import distribution_strategy_context as distribute_ctx -from tensorflow.python.distribute import reduce_util as ds_reduce_util -from tensorflow.python.eager import backprop -from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gradients -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.training import optimizer as optimizer_v1 -from tensorflow.python.training import slot_creator -from tensorflow.python.training.tracking import base as trackable -from tensorflow.python.util import nest - - -@six.add_metaclass(abc.ABCMeta) -class _OptimizableVariable(object): - """Interface for abstracting over variables in the optimizers.""" - - @abc.abstractmethod - def target(self): - """Returns the optimization target for this variable.""" - raise NotImplementedError("Calling an abstract method.") - - @abc.abstractmethod - def update_op(self, optimizer, g, *args): - """Returns the update ops for updating the variable.""" - raise NotImplementedError("Calling an abstract method.") - - -class _RefVariableProcessor(_OptimizableVariable): - """Processor for Variable.""" - - def __init__(self, v): - self._v = v - - def target(self): - return self._v._ref() # pylint: disable=protected-access - - def update_op(self, optimizer, g, *args): - if isinstance(g, ops.Tensor): - update_op = optimizer._apply_dense(g, self._v, *args) # pylint: disable=protected-access - if self._v.constraint is not None: - with ops.control_dependencies([update_op]): - return self._v.assign(self._v.constraint(self._v)) - else: - return update_op - else: - assert isinstance(g, ops.IndexedSlices), ("Gradient ", g, " is neither a " - "tensor nor IndexedSlices.") - if self._v.constraint is not None: - raise RuntimeError( - "Cannot use a constraint function on a sparse variable.") - # pylint: disable=protected-access - return optimizer._apply_sparse_duplicate_indices(g, self._v, *args) - - -class _DenseReadResourceVariableProcessor(_OptimizableVariable): - """Processor for dense ResourceVariables.""" - - def __init__(self, v): - self._v = v - - def target(self): - return self._v - - def update_op(self, optimizer, g, *args): - # pylint: disable=protected-access - update_op = optimizer._resource_apply_dense(g, self._v.op.inputs[0], *args) - if self._v.constraint is not None: - with ops.control_dependencies([update_op]): - return self._v.assign(self._v.constraint(self._v)) - else: - return update_op - - -class _DenseResourceVariableProcessor(_OptimizableVariable): - """Processor for dense ResourceVariables.""" - - def __init__(self, v): - self._v = v - - def target(self): - return self._v - - def update_op(self, optimizer, g, *args): - # pylint: disable=protected-access - if isinstance(g, ops.IndexedSlices): - if self._v.constraint is not None: - raise RuntimeError( - "Cannot use a constraint function on a sparse variable.") - return optimizer._resource_apply_sparse_duplicate_indices( - g.values, self._v, g.indices, *args) - update_op = optimizer._resource_apply_dense(g, self._v, *args) - if self._v.constraint is not None: - with ops.control_dependencies([update_op]): - return self._v.assign(self._v.constraint(self._v)) - else: - return update_op - - -class _TensorProcessor(_OptimizableVariable): - """Processor for ordinary Tensors. - - Even though a Tensor can't really be updated, sometimes it is useful to - compute the gradients with respect to a Tensor using the optimizer. Updating - the Tensor is, of course, unsupported. - """ - - def __init__(self, v): - self._v = v - - def target(self): - return self._v - - def update_op(self, optimizer, g, *args): - raise NotImplementedError("Trying to update a Tensor ", self._v) - - -def _get_processor(v): - """The processor of v.""" - if context.executing_eagerly(): - if isinstance(v, ops.Tensor): - return _TensorProcessor(v) - else: - return _DenseResourceVariableProcessor(v) - if v.op.type == "VarHandleOp": - return _DenseResourceVariableProcessor(v) - if isinstance(v, variables.Variable): - return _RefVariableProcessor(v) - if isinstance(v, ops.Tensor): - return _TensorProcessor(v) - raise NotImplementedError("Trying to optimize unsupported type ", v) - - -def _var_key_v2(var): - """Key for representing a primary variable, for looking up slots.""" - # pylint: disable=protected-access - if hasattr(var, "_distributed_container"): - distributed_container = var._distributed_container() - assert distributed_container is not None - if context.executing_eagerly(): - return distributed_container._unique_id - return distributed_container._shared_name - if context.executing_eagerly(): - return var._unique_id - return var.op.name - - -def _resolve(value, name): - if callable(value): - value = value() - return ops.convert_to_tensor(value, name=name) - - -def _is_dynamic(value): - """Returns true if __init__ arg `value` should be re-evaluated each step.""" - if callable(value): - return True - # Don't need to do anything special in graph mode, since dynamic values - # will propagate correctly automatically. - # TODO(josh11b): Add per-replica caching across steps using variables for - # truly static values once we add distributed support. - if context.executing_eagerly() and isinstance( - value, resource_variable_ops.ResourceVariable): - return True - return False - - -class _OptimizerV2State(object): - """Holds per-graph and per-step optimizer state. - - Use _init_with_static_hyper() to create the state for a graph, and then - _copy_with_dynamic_hyper() to convert that to state for a particular step. - The difference between the two is that the former only has hyper - parameter values that are static and the latter also has values that - can change every step (according to _is_dynamic()). - """ - - def __init__(self, op_name): - self._op_name = op_name - - def _init_with_static_hyper(self, hyper): - """Initialize a fresh state object from hyper dict.""" - # self._hyper contains a dict from name to a dict with the Tensor values. - # This dict starts with a single item with key "None" with the hyper - # parameter value converted to a Tensor. Other items have dtype keys - # with that Tensor cast to that dtype. - with ops.init_scope(): - self._hyper = { - name: { - None: ops.convert_to_tensor(value, name=name) - } for name, (dynamic, value) in sorted(hyper.items()) if not dynamic - } - self._slots = {} - self._non_slot_dict = {} - # Extra state to help Optimizers implement Trackable. Holds information - # about variables which will be restored as soon as they're created. - self._deferred_dependencies = {} # Non-slot variables - self._deferred_slot_restorations = {} # Slot variables - - def _copy_with_dynamic_hyper(self, hyper, distribution, non_slot_devices): - """Create a new state object for a particular step.""" - ret = _OptimizerV2State(self._op_name) - # pylint: disable=protected-access - ret._slots = self._slots - ret._non_slot_dict = self._non_slot_dict - ret._deferred_dependencies = self._deferred_dependencies - ret._deferred_slot_restorations = self._deferred_slot_restorations - ret._hyper = { - name: { - None: _resolve(value, name) - } for name, (dynamic, value) in sorted(hyper.items()) if dynamic - } - ret._hyper.update(self._hyper) - ret._non_slot_devices = non_slot_devices - ret._distribution = distribution - return ret - - def _variables(self): - """Returns a list of all variables held by self.""" - optimizer_variables = list(self._non_slot_dict.values()) - for variable_dict in self._slots.values(): - for slot_for_variable in variable_dict.values(): - optimizer_variables.append(slot_for_variable) - # Sort variables by name so that the return is deterministic. - return sorted(optimizer_variables, key=lambda v: v.name) - - def _slot_dict(self, slot_name): - """Returns a dict for caching slots created under the given name. - - Args: - slot_name: Name for the slot. - - Returns: - A dict that maps primary `Variable` objects to the slot created - for that variable, under the given slot name. - """ - named_slots = self._slots.get(slot_name, None) - if named_slots is None: - named_slots = {} - self._slots[slot_name] = named_slots - return named_slots - - def create_slot(self, var, val, slot_name, optional_op_name=None): - """Find or create a slot for a variable. - - Args: - var: A `Variable` object. - val: A `Tensor`. The initial value of the slot. - slot_name: Name for the slot. - optional_op_name: Name to use when scoping the Variable that needs to be - created for the slot. - - Returns: - A `Variable` object. - """ - named_slots = self._slot_dict(slot_name) - var_key = _var_key_v2(var) - if var_key not in named_slots: - new_slot_variable = slot_creator.create_slot( - var, val, optional_op_name or self._op_name) - self._restore_slot_variable( - slot_name=slot_name, variable=var, slot_variable=new_slot_variable) - named_slots[var_key] = new_slot_variable - return named_slots[var_key] - - def create_slot_with_initializer(self, - var, - initializer, - shape, - dtype, - slot_name, - optional_op_name=None): - """Find or create a slot for a variable, using an Initializer. - - Args: - var: A `Variable` object. - initializer: An `Initializer`. The initial value of the slot. - shape: Shape of the initial value of the slot. - dtype: Type of the value of the slot. - slot_name: Name for the slot. - optional_op_name: Name to use when scoping the Variable that needs to be - created for the slot. - - Returns: - A `Variable` object. - """ - named_slots = self._slot_dict(slot_name) - var_key = _var_key_v2(var) - if var_key not in named_slots: - new_slot_variable = slot_creator.create_slot_with_initializer( - var, initializer, shape, dtype, optional_op_name or self._op_name) - self._restore_slot_variable( - slot_name=slot_name, variable=var, slot_variable=new_slot_variable) - named_slots[var_key] = new_slot_variable - return named_slots[var_key] - - def zeros_slot(self, var, slot_name, optional_op_name=None): - """Find or create a slot initialized with 0.0. - - Args: - var: A `Variable` object. - slot_name: Name for the slot. - optional_op_name: Name to use when scoping the Variable that needs to be - created for the slot. - - Returns: - A `Variable` object. - """ - named_slots = self._slot_dict(slot_name) - var_key = _var_key_v2(var) - if var_key not in named_slots: - new_slot_variable = slot_creator.create_zeros_slot( - var, optional_op_name or self._op_name) - self._restore_slot_variable( - slot_name=slot_name, variable=var, slot_variable=new_slot_variable) - named_slots[var_key] = new_slot_variable - return named_slots[var_key] - - def _create_or_restore_slot_variable(self, - slot_variable_position, - slot_name, - variable, - optional_op_name=None): - """Restore a slot variable's value, possibly creating it. - - Called when a variable which has an associated slot variable is created or - restored. When executing eagerly, we create the slot variable with a - restoring initializer. - - No new variables are created when graph building. Instead, - _restore_slot_variable catches these after normal creation and adds restore - ops to the graph. This method is nonetheless important when graph building - for the case when a slot variable has already been created but `variable` - has just been added to a dependency graph (causing us to realize that the - slot variable needs to be restored). - - Args: - slot_variable_position: A `trackable._CheckpointPosition` object - indicating the slot variable `Trackable` object to be restored. - slot_name: The name of this `Optimizer`'s slot to restore into. - variable: The variable object this slot is being created for. - optional_op_name: Name to use when scoping the Variable that needs to be - created for the slot. - """ - slot_variable = self.get_slot(var=variable, name=slot_name) - if (slot_variable is None and context.executing_eagerly() and - slot_variable_position.is_simple_variable() - # Defer slot variable creation if there is an active variable creator - # scope. Generally we'd like to eagerly create/restore slot variables - # when possible, but this may mean that scopes intended to catch - # `variable` also catch its eagerly created slot variable - # unintentionally (specifically make_template would add a dependency on - # a slot variable if not for this case). Deferring is mostly harmless - # (aside from double initialization), and makes variable creator scopes - # behave the same way they do when graph building. - and not ops.get_default_graph()._variable_creator_stack): # pylint: disable=protected-access - initializer = trackable.CheckpointInitialValue( - checkpoint_position=slot_variable_position) - slot_variable = self.create_slot( - var=variable, - val=initializer, - slot_name=slot_name, - optional_op_name=optional_op_name) - # Optimizers do not have unconditional dependencies on their slot - # variables (nor do any other objects). They are only saved if the - # variables they were created for are also saved. - if slot_variable is not None: - # If we've either made this slot variable, or if we've pulled out an - # existing slot variable, we should restore it. - slot_variable_position.restore(slot_variable) - else: - # We didn't make the slot variable. Defer restoring until it gets created - # normally. We keep a list rather than the one with the highest restore - # UID in case slot variables have their own dependencies, in which case - # those could differ between restores. - variable_key = _var_key_v2(variable) - self._deferred_slot_restorations.setdefault(slot_name, {}).setdefault( - variable_key, []).append(slot_variable_position) - - def get_slot(self, var, name): - """Return a slot named `name` created for `var` by the Optimizer. - - Some `Optimizer` subclasses use additional variables. For example - `Momentum` and `Adagrad` use variables to accumulate updates. This method - gives access to these `Variable` objects if for some reason you need them. - - Use `get_slot_names()` to get the list of slot names created by the - `Optimizer`. - - Args: - var: A variable passed to `minimize()` or `apply_gradients()`. - name: A string. - - Returns: - The `Variable` for the slot if it was created, `None` otherwise. - """ - named_slots = self._slots.get(name, None) - if not named_slots: - return None - return named_slots.get(_var_key_v2(var), None) - - def get_slot_names(self): - """Return a list of the names of slots created by the `Optimizer`. - - See `get_slot()`. - - Returns: - A list of strings. - """ - return sorted(self._slots.keys()) - - def create_non_slot(self, initial_value, name, colocate_with=None): - """Add an extra variable, not associated with a slot.""" - v = self._non_slot_dict.get(name, None) - if v is None: - if colocate_with is None: - colocate_with = self._non_slot_devices - with self._distribution.extended.colocate_vars_with(colocate_with): - # TODO(josh11b): Use get_variable() except for the legacy Adam use case. - v = variable_scope.variable(initial_value, name=name, trainable=False) - self._non_slot_dict[name] = v - deferred_dependencies_list = self._deferred_dependencies.pop(name, ()) - for checkpoint_position in sorted( - deferred_dependencies_list, - key=lambda restore: restore.checkpoint.restore_uid, - reverse=True): - checkpoint_position.restore(v) - return v - - def _restore_slot_variable(self, slot_name, variable, slot_variable): - """Restore a newly created slot variable's value.""" - variable_key = _var_key_v2(variable) - deferred_restorations = self._deferred_slot_restorations.get( - slot_name, {}).pop(variable_key, []) - # Iterate over restores, highest restore UID first to minimize the number - # of assignments. - deferred_restorations.sort( - key=lambda position: position.restore_uid, reverse=True) - for checkpoint_position in deferred_restorations: - checkpoint_position.restore(slot_variable) - - def get_non_slot(self, name): - """Returns the non-slot variable identified by `name`.""" - return self._non_slot_dict.get(name, None) - - def get_hyper(self, name, dtype=None): - """Returns the `name` hyper parameter, optionally cast to `dtype`.""" - dtype_dict = self._hyper[name] - # Do we have the value cast to dtype already cached? This should always - # succeed when dtype is None. - if dtype in dtype_dict: - return dtype_dict[dtype] - # Not cached, cast to dtype and save the result in the cache. - result = math_ops.cast(dtype_dict[None], dtype) - dtype_dict[dtype] = result - return result - - -class OptimizerV2(optimizer_v1.Optimizer): - """Updated base class for optimizers. - - This class defines the API to add Ops to train a model. You never use this - class directly, but instead instantiate one of its subclasses such as - `GradientDescentOptimizer`, `AdagradOptimizer`, or `MomentumOptimizer`. - - ### Usage - - ```python - # Create an optimizer with the desired parameters. - opt = GradientDescentOptimizer(learning_rate=0.1) - # Add Ops to the graph to minimize a cost by updating a list of variables. - # "cost" is a Tensor, and the list of variables contains tf.Variable - # objects. - opt_op = opt.minimize(cost, var_list=) - ``` - - In the training program you will just have to run the returned Op. - - ```python - # Execute opt_op to do one step of training: - opt_op.run() - ``` - - ### Processing gradients before applying them. - - Calling `minimize()` takes care of both computing the gradients and - applying them to the variables. If you want to process the gradients - before applying them you can instead use the optimizer in three steps: - - 1. Compute the gradients with `compute_gradients()`. - 2. Process the gradients as you wish. - 3. Apply the processed gradients with `apply_gradients()`. - - Example: - - ```python - # Create an optimizer. - opt = GradientDescentOptimizer(learning_rate=0.1) - - # Compute the gradients for a list of variables. - grads_and_vars = opt.compute_gradients(loss, ) - - # grads_and_vars is a list of tuples (gradient, variable). Do whatever you - # need to the 'gradient' part, for example cap them, etc. - capped_grads_and_vars = [(MyCapper(gv[0]), gv[1]) for gv in grads_and_vars] - - # Ask the optimizer to apply the capped gradients. - opt.apply_gradients(capped_grads_and_vars) - ``` - - ### Gating Gradients - - Both `minimize()` and `compute_gradients()` accept a `gate_gradients` - argument that controls the degree of parallelism during the application of - the gradients. - - The possible values are: `GATE_NONE`, `GATE_OP`, and `GATE_GRAPH`. - - `GATE_NONE`: Compute and apply gradients in parallel. This provides - the maximum parallelism in execution, at the cost of some non-reproducibility - in the results. For example the two gradients of `matmul` depend on the input - values: With `GATE_NONE` one of the gradients could be applied to one of the - inputs _before_ the other gradient is computed resulting in non-reproducible - results. - - `GATE_OP`: For each Op, make sure all gradients are computed before - they are used. This prevents race conditions for Ops that generate gradients - for multiple inputs where the gradients depend on the inputs. - - `GATE_GRAPH`: Make sure all gradients for all variables are computed - before any one of them is used. This provides the least parallelism but can - be useful if you want to process all gradients before applying any of them. - - ### Slots - - Some optimizer subclasses, such as `MomentumOptimizer` and `AdagradOptimizer` - allocate and manage additional variables associated with the variables to - train. These are called Slots. Slots have names and you can ask the - optimizer for the names of the slots that it uses. Once you have a slot name - you can ask the optimizer for the variable it created to hold the slot value. - - This can be useful if you want to log debug a training algorithm, report stats - about the slots, etc. - - ### Non-slot variables - - Some optimizer subclasses, such as `AdamOptimizer` have variables that - are not associated with the variables to train, just the step itself. - - ### Hyper parameters - - These are arguments passed to the optimizer subclass constructor - (the `__init__` method), and then passed to `self._set_hyper()`. - They can be either regular Python values (like 1.0), tensors, or - callables. If they are callable, the callable will be called during - `apply_gradients()` to get the value for the hyper parameter. - - ### State - - Internal methods are passed a `state` argument with the correct - values to use for the slot and non-slot variables, and the hyper - parameters. - """ - - # Values for gate_gradients. - GATE_NONE = 0 - GATE_OP = 1 - GATE_GRAPH = 2 - - def __init__(self, use_locking, name): - """Create a new Optimizer. - - This must be called by the constructors of subclasses. - Note that Optimizer instances should not bind to a single graph, - and so shouldn't keep Tensors as member variables. Generally - you should be able to use the _set_hyper()/state.get_hyper() - facility instead. - - Args: - use_locking: Bool. If True apply use locks to prevent concurrent updates - to variables. - name: A non-empty string. The name to use for accumulators created - for the optimizer. - - Raises: - ValueError: If name is malformed. - RuntimeError: If _create_slots has been overridden instead of - _create_vars. - """ - # Note: We intentionally don't call parent __init__. - - # Optimizer._create_slots was replaced by _create_vars in OptimizerV2. - if (self.__class__._create_slots.__code__ is not # pylint: disable=protected-access - OptimizerV2._create_slots.__code__): - raise RuntimeError( - "Override _create_vars instead of _create_slots when " - "descending from OptimizerV2 (class %s)" % self.__class__.__name__) - if not name: - raise ValueError("Must specify the optimizer name") - - self._use_locking = use_locking - self._name = name - # Map from graph_key to state for that graph. We use the graph_key - # since it works in both eager and graph mode, and gives the outer - # graph inside functions. - replica_context = distribute_ctx.get_replica_context() - if replica_context is None: - # In a cross-replica context for a DistributionStrategy, which means - # only one Optimizer will be created, not one per replica. - self._per_graph_state = {} - else: - # We use get_replica_context().merge_call() to get a single dict - # shared across all model replicas when running with a - # DistributionStrategy. - self._per_graph_state = replica_context.merge_call(lambda _: {}) - - # Hyper parameters, and whether they should be re-evaluated every step. - self._hyper = {} - - def _set_hyper(self, name, value): - self._hyper[name] = (_is_dynamic(value), value) - - def minimize(self, - loss, - global_step=None, - var_list=None, - gate_gradients=GATE_OP, - aggregation_method=None, - name=None, - grad_loss=None, - stop_gradients=None, - scale_loss_by_num_replicas=False): - """Add operations to minimize `loss` by updating `var_list`. - - This method simply combines calls `compute_gradients()` and - `apply_gradients()`. If you want to process the gradient before applying - them call `compute_gradients()` and `apply_gradients()` explicitly instead - of using this function. - - Args: - loss: A `Tensor` containing the value to minimize. - global_step: Optional `Variable` to increment by one after the variables - have been updated. - var_list: Optional list or tuple of `Variable` objects to update to - minimize `loss`. Defaults to the list of variables collected in the - graph under the key `GraphKeys.TRAINABLE_VARIABLES`. - gate_gradients: How to gate the computation of gradients. Can be - `GATE_NONE`, `GATE_OP`, or `GATE_GRAPH`. - aggregation_method: Specifies the method used to combine gradient terms. - Valid values are defined in the class `AggregationMethod`. - name: Optional name for the returned operation. - grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`. - stop_gradients: Optional. A Tensor or list of tensors not to differentiate - through. - scale_loss_by_num_replicas: Optional boolean. If true, scale the loss down - by the number of replicas. DEPRECATED and generally no longer needed. - - Returns: - An Operation that updates the variables in `var_list`. If `global_step` - was not `None`, that operation also increments `global_step`. - - Raises: - ValueError: If some of the variables are not `Variable` objects. - - @compatibility(eager) - When eager execution is enabled, `loss` should be a Python function that - takes elements of `var_list` as arguments and computes the value to be - minimized. If `var_list` is None, `loss` should take no arguments. - Minimization (and gradient computation) is done with respect to the - elements of `var_list` if not None, else with respect to any trainable - variables created during the execution of the `loss` function. - `gate_gradients`, `aggregation_method`, and `grad_loss` are ignored when - eager execution is enabled. - @end_compatibility - """ - grads_and_vars = self.compute_gradients( - loss, - var_list=var_list, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - grad_loss=grad_loss, - stop_gradients=stop_gradients, - scale_loss_by_num_replicas=scale_loss_by_num_replicas) - - vars_with_grad = [v for g, v in grads_and_vars if g is not None] - if not vars_with_grad: - raise ValueError( - "No gradients provided for any variable, check your graph for ops" - " that do not support gradients, between variables %s and loss %s." % - ([str(v) for _, v in grads_and_vars], loss)) - - return self.apply_gradients( - grads_and_vars, global_step=global_step, name=name) - - def compute_gradients(self, - loss, - var_list=None, - gate_gradients=GATE_OP, - aggregation_method=None, - grad_loss=None, - stop_gradients=None, - scale_loss_by_num_replicas=False): - """Compute gradients of `loss` for the variables in `var_list`. - - This is the first part of `minimize()`. It returns a list - of (gradient, variable) pairs where "gradient" is the gradient - for "variable". Note that "gradient" can be a `Tensor`, an - `IndexedSlices`, or `None` if there is no gradient for the - given variable. - - Args: - loss: A Tensor containing the value to minimize or a callable taking no - arguments which returns the value to minimize. When eager execution is - enabled it must be a callable. - var_list: Optional list or tuple of `tf.Variable` to update to minimize - `loss`. Defaults to the list of variables collected in the graph under - the key `GraphKeys.TRAINABLE_VARIABLES`. - gate_gradients: How to gate the computation of gradients. Can be - `GATE_NONE`, `GATE_OP`, or `GATE_GRAPH`. - aggregation_method: Specifies the method used to combine gradient terms. - Valid values are defined in the class `AggregationMethod`. - grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`. - stop_gradients: Optional. A Tensor or list of tensors not to differentiate - through. - scale_loss_by_num_replicas: Optional boolean. If true, scale the loss down - by the number of replicas. DEPRECATED and generally no longer needed. - - Returns: - A list of (gradient, variable) pairs. Variable is always present, but - gradient can be `None`. - - Raises: - TypeError: If `var_list` contains anything else than `Variable` objects. - ValueError: If some arguments are invalid. - RuntimeError: If called with eager execution enabled and `loss` is - not callable. - - @compatibility(eager) - When eager execution is enabled, `gate_gradients`, and `aggregation_method` - are ignored. - @end_compatibility - """ - # TODO(josh11b): Test that we handle weight decay in a reasonable way. - if callable(loss): - with backprop.GradientTape() as tape: - if var_list is not None: - tape.watch(var_list) - loss_value = loss() - - # Scale loss for number of replicas (callable-loss case). - loss_value = self._scale_loss(loss_value, scale_loss_by_num_replicas) - - if var_list is None: - var_list = tape.watched_variables() - grads = tape.gradient(loss_value, var_list, grad_loss) - return list(zip(grads, var_list)) - if context.executing_eagerly(): - raise RuntimeError("`loss` passed to Optimizer.compute_gradients should " - "be a function when eager execution is enabled.") - - # Scale loss for number of replicas (non-callable-loss case). - loss = self._scale_loss(loss, scale_loss_by_num_replicas) - - if gate_gradients not in [ - optimizer_v1.Optimizer.GATE_NONE, optimizer_v1.Optimizer.GATE_OP, - optimizer_v1.Optimizer.GATE_GRAPH - ]: - raise ValueError( - "gate_gradients must be one of: Optimizer.GATE_NONE, " - "Optimizer.GATE_OP, Optimizer.GATE_GRAPH. Not %s" % gate_gradients) - self._assert_valid_dtypes([loss]) - if grad_loss is not None: - self._assert_valid_dtypes([grad_loss]) - if var_list is None: - var_list = ( - variables.trainable_variables() + ops.get_collection( - ops.GraphKeys.TRAINABLE_RESOURCE_VARIABLES)) - else: - var_list = nest.flatten(var_list) - # pylint: disable=protected-access - var_list += ops.get_collection(ops.GraphKeys._STREAMING_MODEL_PORTS) - # pylint: enable=protected-access - processors = [_get_processor(v) for v in var_list] - if not var_list: - raise ValueError("No variables to optimize.") - var_refs = [p.target() for p in processors] - grads = gradients.gradients( - loss, - var_refs, - grad_ys=grad_loss, - gate_gradients=(gate_gradients == optimizer_v1.Optimizer.GATE_OP), - aggregation_method=aggregation_method, - stop_gradients=stop_gradients) - if gate_gradients == optimizer_v1.Optimizer.GATE_GRAPH: - grads = control_flow_ops.tuple(grads) - grads_and_vars = list(zip(grads, var_list)) - self._assert_valid_dtypes([ - v for g, v in grads_and_vars - if g is not None and v.dtype != dtypes.resource - ]) - return grads_and_vars - - @staticmethod - def _scale_loss(loss_value, scale_loss_by_num_replicas): - """Scale loss for the number of replicas.""" - if scale_loss_by_num_replicas: - num_replicas = distribute_ctx.get_strategy().num_replicas_in_sync - if num_replicas > 1: - loss_value *= 1. / num_replicas - return loss_value - - def apply_gradients(self, grads_and_vars, global_step=None, name=None): - """Apply gradients to variables. - - This is the second part of `minimize()`. It returns an `Operation` that - applies gradients. - - Args: - grads_and_vars: List of (gradient, variable) pairs as returned by - `compute_gradients()`. - global_step: Optional `Variable` to increment by one after the variables - have been updated. - name: Optional name for the returned operation. Default to the name - passed to the `Optimizer` constructor. - - Returns: - An `Operation` that applies the specified gradients. If `global_step` - was not None, that operation also increments `global_step`. - - Raises: - TypeError: If `grads_and_vars` is malformed. - ValueError: If none of the variables have gradients. - """ - # This is a default implementation of apply_gradients() that can be shared - # by most optimizers. It relies on the subclass implementing the following - # methods: _create_vars(), _prepare(), _apply_dense(), and _apply_sparse(). - - # Filter out variables with gradients of `None`. - grads_and_vars = tuple(grads_and_vars) # Make sure repeat iteration works. - if not grads_and_vars: - raise ValueError("No variables provided.") - filtered = tuple((g, v) for (g, v) in grads_and_vars if g is not None) - if not filtered: - raise ValueError("No gradients provided for any variable: %s." % - ([str(v) for _, v in grads_and_vars],)) - return distribute_ctx.get_replica_context().merge_call( - self._distributed_apply, args=(filtered,), - kwargs={"global_step": global_step, "name": name}) - - def _get_or_create_state(self, var_list=None): - """Either looks up or creates `_OptimizerV2State`. - - If any variables are available, they should be passed via the `var_list` - argument, and these will be used to determine the graph to create/retrieve - state for. Otherwise the returned state is for the current default graph. - - Args: - var_list: A list of variables to extract a graph from. - - Returns: - An `_OptimizerV2State` object. - """ - # Determine the graph_key from the current graph. - eager_execution = context.executing_eagerly() - if eager_execution or var_list is None: - graph = ops.get_default_graph() - else: - graph = ops._get_graph_from_inputs(var_list) # pylint: disable=protected-access - assert graph is not None - graph_key = graph._graph_key # pylint: disable=protected-access - - # Get the per graph state by looking up the graph_key. - if graph_key in self._per_graph_state: - per_graph_state = self._per_graph_state[graph_key] - else: - per_graph_state = _OptimizerV2State(self._name) - per_graph_state._init_with_static_hyper(self._hyper) # pylint: disable=protected-access - self._per_graph_state[graph_key] = per_graph_state - return per_graph_state - - def _distributed_apply(self, distribution, grads_and_vars, global_step, name): - """`apply_gradients` for use with a `DistributionStrategy`.""" - reduced_grads = distribution.extended.batch_reduce_to( - ds_reduce_util.ReduceOp.SUM, grads_and_vars) - var_list = [v for _, v in grads_and_vars] - grads_and_vars = zip(reduced_grads, var_list) - - unwrapped_var_list = [ - x for v in var_list for x in distribution.experimental_local_results(v)] - eager_execution = context.executing_eagerly() - if eager_execution: - # Give a clear error in this case instead of "name not supported - # for Eager Tensors" when we compute non_slot_devices. - for v in unwrapped_var_list: - if isinstance(v, ops.Tensor): - raise NotImplementedError("Trying to update a Tensor ", v) - - with ops.name_scope(name, self._name) as name: - per_graph_state = self._get_or_create_state(var_list=unwrapped_var_list) - # Include the current value of any dynamic hyper parameters in `state`. - non_slot_devices = distribution.extended.non_slot_devices(var_list) - state = per_graph_state._copy_with_dynamic_hyper( # pylint: disable=protected-access - self._hyper, distribution, non_slot_devices) - - # Create any slot and non-slot variables we need in `state`. - with ops.init_scope(): - self._create_vars(var_list, state) - - with ops.name_scope(name): # Re-enter name_scope created above - # Give the child class a chance to do something before we start - # applying gradients. - self._prepare(state) - - def update(v, g): - """Update variable `v` using gradient `g`.""" - assert v is not None - - # Convert the grad to Tensor or IndexedSlices if necessary, and - # look up a processor for each variable's type. - try: - g = ops.convert_to_tensor_or_indexed_slices(g) - except TypeError: - raise TypeError("Gradient must be convertible to a Tensor" - " or IndexedSlices, or None: %s" % g) - if not isinstance(g, (ops.Tensor, ops.IndexedSlices)): - raise TypeError( - "Gradient must be a Tensor, IndexedSlices, or None: %s" % g) - processor = _get_processor(v) - - # We colocate all ops created in _apply_dense or _apply_sparse - # on the same device as the variable. - # TODO(apassos): figure out how to get the variable name here. - scope_name = "" if eager_execution else v.op.name - # device_policy is set because non-mirrored tensors will be read in - # `update_op`. - # TODO(josh11b): Make different state objects for each device to - # avoid needing to set the device_policy. - device_policy = context.device_policy( - context.DEVICE_PLACEMENT_SILENT) - with ops.name_scope("update_" + scope_name), device_policy: - return processor.update_op(self, g, state) - - # Use the processors to update the variables. - update_ops = [] - for grad, var in grads_and_vars: - update_ops.extend(distribution.extended.update( - var, update, args=(grad,), group=False)) - - # Give the child class a chance to do something after applying - # gradients - def finish(): - # TODO(josh11b): Make different state objects for each device to - # avoid needing to set the device_policy. - with context.device_policy(context.DEVICE_PLACEMENT_SILENT): - return self._finish(state) - - update_ops = control_flow_ops.group(update_ops) - with ops.control_dependencies([update_ops]): - finish_updates = distribution.extended.update_non_slot( - non_slot_devices, finish, group=False) - # We said group=False, which means finish_updates is always a tuple. - # It will be (None,) when finish() returns None. - if finish_updates == (None,): - finish_updates = (update_ops,) - - # Update `global_step` (if any). - if global_step is None: - apply_updates = distribution.group(finish_updates, name=name) - else: - with ops.control_dependencies(finish_updates): - - def update_global_step(global_step, name): - return global_step.assign_add(1, read_value=False, name=name) - - apply_updates = distribution.extended.update( - global_step, update_global_step, args=(name,)) - - # Add the training op to the TRAIN_OP graph collection in graph mode. - if not eager_execution: - if isinstance(apply_updates, ops.Tensor): - apply_updates = apply_updates.op - train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) - if apply_updates not in train_op: - train_op.append(apply_updates) - - return apply_updates - - def get_slot(self, var, name): - """Return a slot named `name` created for `var` by the Optimizer. - - Some `Optimizer` subclasses use additional variables. For example - `Momentum` and `Adagrad` use variables to accumulate updates. This method - gives access to these `Variable` objects if for some reason you need them. - - Use `get_slot_names()` to get the list of slot names created by the - `Optimizer`. - - Args: - var: A variable passed to `minimize()` or `apply_gradients()`. - name: A string. - - Returns: - The `Variable` for the slot if it was created, `None` otherwise. - """ - state = self._get_state_for_var(var) - return state.get_slot(var, name) if state is not None else None - - def get_slot_names(self): - """Return a list of the names of slots created by the `Optimizer`. - - See `get_slot()`. - - Returns: - A list of strings. - """ - state = self._get_per_graph_state() - return state.get_slot_names() if state is not None else [] - - def variables(self): - """A list of variables which encode the current state of `Optimizer`. - - Includes slot variables and additional global variables created by the - optimizer in the current default graph. - - Returns: - A list of variables. - """ - state = self._get_per_graph_state() - return state._variables() if state is not None else [] # pylint: disable=protected-access - - # -------------- - # Methods to be implemented by subclasses if they want to use the - # inherited implementation of apply_gradients() or compute_gradients(). - # -------------- - def _create_vars(self, var_list, state): - """Create all slots needed by the variables and any non-slot variables. - - Args: - var_list: A list of `Variable` objects. - state: An object with these methods: `create_slot(var, val, slot_name, - optional_op_name)`, `create_slot_with_initializer(` `var, initializer, - shape, dtype, slot_name, optional_op_name)`, `zeros_slot(var, slot_name, - optional_op_name)`, `create_non_slot_variable(initial_value, name, - colocate_with)`, `get_hyper(name)` - """ - # No slots needed by default - pass - - def _prepare(self, state): - """Code to execute before applying gradients. - - Note that most uses of _prepare() in Optimizer have been subsumed - by explicit support for hyper parameters in OptimizerV2 - - Args: - state: An object with a `get_hyper(name)` method. - - Returns: - Return value will be ignored. - """ - pass - - def _apply_dense(self, grad, var, state): - """Add ops to apply dense gradients to `var`. - - Args: - grad: A `Tensor`. - var: A `Variable` object. - state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`, - and `get_hyper(name)` methods. - - Returns: - An `Operation`. - """ - raise NotImplementedError() - - def _resource_apply_dense(self, grad, handle, state): - """Add ops to apply dense gradients to the variable `handle`. - - Args: - grad: a `Tensor` representing the gradient. - handle: a `Tensor` of dtype `resource` which points to the variable to be - updated. - state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`, - and `get_hyper(name)` methods. - - Returns: - An `Operation` which updates the value of the variable. - """ - raise NotImplementedError() - - def _resource_apply_sparse_duplicate_indices(self, grad, handle, indices, - state): - """Add ops to apply sparse gradients to `handle`, with repeated indices. - - Optimizers which override this method must deal with repeated indices. See - the docstring of `_apply_sparse_duplicate_indices` for details. By default - the correct behavior, to sum non-unique indices and their associated - gradients, is enforced by first pre-processing `grad` and `indices` and - passing them on to `_resource_apply_sparse`. Optimizers which deal correctly - with duplicate indices may instead override this method to avoid the - overhead of summing. - - Args: - grad: a `Tensor` representing the gradient for the affected indices. - handle: a `Tensor` of dtype `resource` which points to the variable to be - updated. - indices: a `Tensor` of integral type representing the indices for which - the gradient is nonzero. Indices may be repeated. - state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`, - and `get_hyper(name)` methods. - - Returns: - An `Operation` which updates the value of the variable. - """ - # pylint: disable=protected-access - summed_grad, unique_indices = optimizer_v1._deduplicate_indexed_slices( - values=grad, indices=indices) - # pylint: enable=protected-access - return self._resource_apply_sparse(summed_grad, handle, unique_indices, - state) - - def _resource_apply_sparse(self, grad, handle, indices, state): - """Add ops to apply sparse gradients to the variable `handle`. - - Similar to `_apply_sparse`, the `indices` argument to this method has been - de-duplicated. Optimizers which deal correctly with non-unique indices may - instead override `_resource_apply_sparse_duplicate_indices` to avoid this - overhead. - - Args: - grad: a `Tensor` representing the gradient for the affected indices. - handle: a `Tensor` of dtype `resource` which points to the variable to be - updated. - indices: a `Tensor` of integral type representing the indices for which - the gradient is nonzero. Indices are unique. - state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`, - and `get_hyper(name)` methods. - - Returns: - An `Operation` which updates the value of the variable. - """ - raise NotImplementedError() - - def _apply_sparse_duplicate_indices(self, grad, var, state): - """Add ops to apply sparse gradients to `var`, with repeated sparse indices. - - Optimizers which override this method must deal with IndexedSlices objects - such as the following: - - IndexedSlicesValue(values=[1, 1], indices=[0, 0], dense_shape=[1]) - - The correct interpretation is: - - IndexedSlicesValue(values=[2], indices=[0], dense_shape=[1]) - - Many optimizers deal incorrectly with repeated indices when updating based - on sparse gradients (e.g. summing squares rather than squaring the sum, or - applying momentum terms multiple times). Adding first is always the correct - behavior, so this is enforced here by reconstructing the IndexedSlices to - have only unique indices, then calling _apply_sparse. - - Optimizers which deal correctly with repeated indices may instead override - this method to avoid the overhead of summing indices. - - Args: - grad: `IndexedSlices`. - var: A `Variable` object. - state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`, - and `get_hyper(name)` methods. - - Returns: - An `Operation`. - """ - # pylint: disable=protected-access - summed_values, unique_indices = optimizer_v1._deduplicate_indexed_slices( - values=grad.values, indices=grad.indices) - # pylint: enable=protected-access - gradient_no_duplicate_indices = ops.IndexedSlices( - indices=unique_indices, - values=summed_values, - dense_shape=grad.dense_shape) - return self._apply_sparse(gradient_no_duplicate_indices, var, state) - - def _apply_sparse(self, grad, var, state): - """Add ops to apply sparse gradients to `var`. - - The IndexedSlices object passed to `grad` in this function is by default - pre-processed in `_apply_sparse_duplicate_indices` to remove duplicate - indices (see its docstring for details). Optimizers which can tolerate or - have correct special cases for duplicate sparse indices may override - `_apply_sparse_duplicate_indices` instead of this function, avoiding that - overhead. - - Args: - grad: `IndexedSlices`, with no repeated indices. - var: A `Variable` object. - state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`, - and `get_hyper(name)` methods. - - Returns: - An `Operation`. - """ - raise NotImplementedError() - - def _finish(self, state): - """Do what is needed to finish the update. - - This is called inside a scope colocated with any non-slot variables. - - Args: - state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`, - and `get_hyper(name)` methods. - - Returns: - The operation to apply updates, or None if no updates. - """ - return None - - # -------------- - # Utility methods for subclasses. - # -------------- - def _get_per_graph_state(self): - # pylint: disable=protected-access - return self._per_graph_state.get(ops.get_default_graph()._graph_key, None) - - def _get_state_for_var(self, var): - # pylint: disable=protected-access - return self._per_graph_state.get(var._graph_key, None) - - # -------------- - # Overridden methods from Trackable. - # -------------- - - def _track_trackable(self, *args, **kwargs): - """Optimizers may not track dependencies. Raises an error.""" - raise NotImplementedError( - "Optimizers may not have dependencies. File a feature request if this " - "limitation bothers you.") - - @property - def _checkpoint_dependencies(self): - """From Trackable. Gather graph-specific non-slot variables to save.""" - current_graph_non_slot_variables = [] - state = self._get_per_graph_state() - if state is not None: - for name, variable_object in sorted( - state._non_slot_dict.items(), # pylint: disable=protected-access - # Avoid comparing variables - key=lambda item: item[0]): - current_graph_non_slot_variables.append( - trackable.TrackableReference( - name=name, ref=variable_object)) - # Note: ignores super(); Optimizers may not have any dependencies outside of - # state objects. - return current_graph_non_slot_variables - - def _lookup_dependency(self, name): - """From Trackable. Find a non-slot variable in the current graph.""" - state = self._get_per_graph_state() - if state is None: - return None - else: - return state.get_non_slot(name) - - @property - def _deferred_dependencies(self): - """Lets Trackable know where non-slot variables are created. - - If necessary, creates a new state object for the current default graph. - Trackable will then add entries to that state's deferred dependency - dictionary. The state object will check that dictionary when creating - non-slot variables, restoring their value if an entry is found. - - Returns: - A dictionary which holds deferred dependencies for the current default - graph. - """ - state = self._get_or_create_state() - return state._deferred_dependencies # pylint: disable=protected-access - - def _create_or_restore_slot_variable(self, slot_variable_position, slot_name, - variable): - """Trackable: Restore a slot variable's value, possibly creating it. - - Called when a variable which has an associated slot variable is created or - restored. - - Args: - slot_variable_position: A `trackable._CheckpointPosition` object - indicating the slot variable `Trackable` object to be restored. - slot_name: The name of this `Optimizer`'s slot to restore into. - variable: The variable object this slot is being created for. - """ - state = self._get_or_create_state(var_list=[variable]) - state._create_or_restore_slot_variable( # pylint: disable=protected-access - slot_variable_position=slot_variable_position, - slot_name=slot_name, - variable=variable, - optional_op_name=self._name) - - # -------------- - # Unsupported parent methods - # -------------- - def _slot_dict(self, slot_name): - raise NotImplementedError("_slot_dict() method unsupported in OptimizerV2") - - def _get_or_make_slot(self, var, val, slot_name, op_name): - raise NotImplementedError( - "_get_or_make_slot() method unsupported in OptimizerV2") - - def _get_or_make_slot_with_initializer(self, var, initializer, shape, dtype, - slot_name, op_name): - raise NotImplementedError( - "_get_or_make_slot_with_initializer() method unsupported in " - "OptimizerV2") - - def _create_non_slot_variable(self, initial_value, name, colocate_with): - raise NotImplementedError( - "_create_non_slot_variable() method unsupported in OptimizerV2") - - def _get_non_slot_variable(self, name, graph=None): - raise NotImplementedError( - "_get_non_slot_variable() method unsupported in OptimizerV2") - - def _non_slot_variables(self): - raise NotImplementedError( - "_non_slot_variables() method unsupported in OptimizerV2") diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2_symbols.py b/tensorflow/contrib/optimizer_v2/optimizer_v2_symbols.py deleted file mode 100644 index 24eada06ccd..00000000000 --- a/tensorflow/contrib/optimizer_v2/optimizer_v2_symbols.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Distribution-aware version of Optimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.contrib.optimizer_v2.adadelta import AdadeltaOptimizer -from tensorflow.contrib.optimizer_v2.adagrad import AdagradOptimizer -from tensorflow.contrib.optimizer_v2.adam import AdamOptimizer -from tensorflow.contrib.optimizer_v2.gradient_descent import GradientDescentOptimizer -from tensorflow.contrib.optimizer_v2.momentum import MomentumOptimizer -from tensorflow.contrib.optimizer_v2.optimizer_v2 import OptimizerV2 -from tensorflow.contrib.optimizer_v2.rmsprop import RMSPropOptimizer - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - 'AdadeltaOptimizer', - 'AdagradOptimizer', - 'AdamOptimizer', - 'GradientDescentOptimizer', - 'MomentumOptimizer', - 'OptimizerV2', - 'RMSPropOptimizer', -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2_test.py b/tensorflow/contrib/optimizer_v2/optimizer_v2_test.py deleted file mode 100644 index 2fc0b5ea4de..00000000000 --- a/tensorflow/contrib/optimizer_v2/optimizer_v2_test.py +++ /dev/null @@ -1,274 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functional test for OptimizerV2.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.optimizer_v2 import gradient_descent -from tensorflow.contrib.optimizer_v2 import optimizer_v2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import gradients_util -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class OptimizerTest(test.TestCase): - - @test_util.run_in_graph_and_eager_modes - def testBasic(self): - for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) - var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) - def loss(): - return 5 * var0 + 3 * var1 # pylint: disable=cell-var-from-loop - # Note that for eager execution, minimize expects a function instead of a - # Tensor. - global_step = resource_variable_ops.ResourceVariable( - array_ops.zeros([], dtypes.int64), name='global_step_%d' % i) - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - # Run 1 step of sgd through optimizer - opt_op = sgd_op.minimize(loss, global_step, [var0, var1]) - self.evaluate(opt_op) - # Validate updated params - self.assertAllClose([-14., -13.], self.evaluate(var0)) - self.assertAllClose([-6., -5.], self.evaluate(var1)) - - def testAggregationMethod(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - cost = 5 * var0 + 3 * var1 - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name='global_step') - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - opt_op = sgd_op.minimize( - cost, - global_step, [var0, var1], - aggregation_method=gradients_util.AggregationMethod. - EXPERIMENTAL_ACCUMULATE_N) - - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Run 1 step of sgd through optimizer - opt_op.run() - # Validate updated params - self.assertAllClose([-14., -13.], var0.eval()) - self.assertAllClose([-6., -5.], var1.eval()) - - def testPrecomputedGradient(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - cost = 5 * var0 + 3 * var1 - grad_loss = constant_op.constant([42, -42], dtype=dtype) - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name='global_step') - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - opt_op = sgd_op.minimize( - cost, global_step, [var0, var1], grad_loss=grad_loss) - - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Run 1 step of sgd through optimizer - opt_op.run() - # Validate updated params - self.assertAllClose([1.0 - 3 * 5 * 42.0, 2.0 - 3 * 5 * (-42.0)], - var0.eval()) - self.assertAllClose([3.0 - 3 * 3 * 42.0, 4.0 - 3 * 3 * (-42.0)], - var1.eval()) - - @test_util.run_in_graph_and_eager_modes - def testNoVariables(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - # pylint: disable=cell-var-from-loop - def loss(): - var0 = resource_variable_ops.ResourceVariable( - [1.0, 2.0], dtype=dtype, trainable=False, name='a') - var1 = resource_variable_ops.ResourceVariable( - [3.0, 4.0], dtype=dtype, trainable=False, name='b') - return 5 * var0 + var1 - # pylint: enable=cell-var-from-loop - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - with self.assertRaisesRegexp(ValueError, 'No.*variables'): - sgd_op.minimize(loss) - - @test_util.run_in_graph_and_eager_modes - def testNoGradients(self): - for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) - var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) - # pylint: disable=cell-var-from-loop - def loss(): - return 5 * var0 - # pylint: enable=cell-var-from-loop - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - with self.assertRaisesRegexp(ValueError, 'No gradients'): - # var1 has no gradient - sgd_op.minimize(loss, var_list=[var1]) - - @test_util.run_in_graph_and_eager_modes - def testNoGradientsForAnyVariables_Minimize(self): - for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) - var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) - def loss(): - return constant_op.constant(5.0) - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - with self.assertRaisesRegexp(ValueError, - 'No gradients provided for any variable'): - sgd_op.minimize(loss, var_list=[var0, var1]) - - @test_util.run_in_graph_and_eager_modes - def testNoGradientsForAnyVariables_ApplyGradients(self): - for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) - var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - with self.assertRaisesRegexp(ValueError, - 'No gradients provided for any variable'): - sgd_op.apply_gradients([(None, var0), (None, var1)]) - - @test_util.run_in_graph_and_eager_modes - def testGradientsAsVariables(self): - for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) - var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) - def loss(): - return 5 * var0 + 3 * var1 # pylint: disable=cell-var-from-loop - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - grads_and_vars = sgd_op.compute_gradients(loss, [var0, var1]) - # Convert gradients to tf.Variables - converted_grads = [ - resource_variable_ops.ResourceVariable(array_ops.zeros([2], dtype), - name='c_%d_%d' % (i, j)) - for j, gv in enumerate(grads_and_vars) - ] - convert_ops = [ - state_ops.assign(converted_grads[j], gv[0]) - for j, gv in enumerate(grads_and_vars) - ] - - self.evaluate(variables.global_variables_initializer()) - # Run convert_ops to achieve the gradietns converting - self.evaluate(convert_ops) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - # Run 1 step of sgd through optimizer - converted_grads_and_vars = list(zip(converted_grads, [var0, var1])) - opt_op = sgd_op.apply_gradients(converted_grads_and_vars) - self.evaluate(opt_op) - - # Validate updated params - self.assertAllClose([-14., -13.], self.evaluate(var0)) - self.assertAllClose([-6., -5.], self.evaluate(var1)) - - @test_util.run_in_graph_and_eager_modes - def testComputeGradientsWithTensors(self): - x = ops.convert_to_tensor(1.0) - def f(): - return x * x - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - grads_and_vars = sgd_op.compute_gradients(f, [x]) - self.assertEqual(1, len(grads_and_vars)) - grad, x_as_var = grads_and_vars[0] - self.assertIs(x, x_as_var) - self.assertEqual(2.0, self.evaluate(grad)) - - with self.assertRaises(NotImplementedError): - sgd_op.apply_gradients(grads_and_vars) - - def testTrainOp(self): - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0]) - var1 = variables.Variable([3.0, 4.0]) - cost = 5 * var0 + 3 * var1 - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name='global_step') - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - opt_op = sgd_op.minimize(cost, global_step, [var0, var1]) - self.assertTrue(opt_op in ops.get_collection(ops.GraphKeys.TRAIN_OP)) - - def testConstraint(self): - constraint_01 = lambda x: clip_ops.clip_by_value(x, -0.1, 0.) - constraint_0 = lambda x: clip_ops.clip_by_value(x, 0., 1.) - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], - constraint=constraint_01) - var1 = variables.Variable([3.0, 4.0], - constraint=constraint_0) - cost = 5 * var0 + 3 * var1 - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name='global_step') - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - opt_op = sgd_op.minimize(cost, global_step, [var0, var1]) - - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Run 1 step of sgd through optimizer - opt_op.run() - # Validate updated params - self.assertAllClose([-0.1, -0.1], var0.eval()) - self.assertAllClose([0., 0.], var1.eval()) - - def testStopGradients(self): - with self.cached_session(): - var0 = variables.Variable([1.0, 2.0], name='var0') - var1 = variables.Variable([3.0, 4.0], name='var1') - var0_id = array_ops.identity(var0) - cost = 5 * var0_id + 3 * var1 - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - grads_and_vars = sgd_op.compute_gradients(cost, [var0, var1], - stop_gradients=[var0_id]) - grad_dict = {var.op.name: grad for grad, var in grads_and_vars} - self.assertIsNone(grad_dict['var0']) - self.assertIsNotNone(grad_dict['var1']) - - def testDoNotOverrideCreateSlots(self): - class ShouldNotOverrideCreateSlots(optimizer_v2.OptimizerV2): - - def _create_slots(self, var_list): - """In OptimizerV2 _create_slots was renamed _create_vars.""" - return var_list - - with self.assertRaises(RuntimeError): - ShouldNotOverrideCreateSlots(True, 'name') - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/optimizer_v2/rmsprop.py b/tensorflow/contrib/optimizer_v2/rmsprop.py deleted file mode 100644 index 12175cedd36..00000000000 --- a/tensorflow/contrib/optimizer_v2/rmsprop.py +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""RMSprop optimizer for Tensorflow. - -rmsprop algorithm [tieleman2012rmsprop] - -A detailed description of rmsprop. - -- maintain a moving (discounted) average of the square of gradients -- divide gradient by the root of this average - -mean_square = rho * mean_square{t-1} + (1-rho) * gradient ** 2 -mom = momentum * mom{t-1} + learning_rate * g_t / sqrt(mean_square) -delta = - mom - -This implementation of RMSProp uses plain momentum, not Nesterov momentum. - -The centered version additionally maintains a moving (discounted) average of the -gradients, and uses that average to estimate the variance: - -mean_grad = rho * mean_square{t-1} + (1-rho) * gradient -mean_square = rho * mean_square{t-1} + (1-rho) * gradient ** 2 -mom = momentum * mom{t-1} + learning_rate * g_t / - sqrt(mean_square - mean_grad**2) -delta = - mom -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.optimizer_v2 import optimizer_v2 -from tensorflow.python.ops import array_ops - -from tensorflow.python.training import training_ops - - -class RMSPropOptimizer(optimizer_v2.OptimizerV2): - """Optimizer that implements the RMSProp algorithm. - - See the - [paper] - (http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf). - """ - - def __init__(self, - learning_rate, - decay=0.9, - momentum=0.0, - epsilon=1e-10, - use_locking=False, - centered=False, - name="RMSProp"): - """Construct a new RMSProp optimizer. - - Note that in the dense implementation of this algorithm, variables and their - corresponding accumulators (momentum, gradient moving average, square - gradient moving average) will be updated even if the gradient is zero - (i.e. accumulators will decay, momentum will be applied). The sparse - implementation (used when the gradient is an `IndexedSlices` object, - typically because of `tf.gather` or an embedding lookup in the forward pass) - will not update variable slices or their accumulators unless those slices - were used in the forward pass (nor is there an "eventual" correction to - account for these omitted updates). This leads to more efficient updates for - large embedding lookup tables (where most of the slices are not accessed in - a particular graph execution), but differs from the published algorithm. - - Some of the args below are hyperparameters, where a hyperparameter is - defined as a scalar Tensor, a regular Python value or a callable (which - will be evaluated when `apply_gradients` is called) returning a scalar - Tensor or a Python value. - - Args: - learning_rate: A float hyperparameter. The learning rate. - decay: A float hyperparameter. Discounting factor for the history/coming - gradient. - momentum: A float hyperparameter. - epsilon: A float hyperparameter. Small value to initialize the average - square gradient variable and avoid zero denominator. - use_locking: If True use locks for update operation. - centered: If True, gradients are normalized by the estimated variance of - the gradient; if False, by the uncentered second moment. Setting this to - True may help with training, but is slightly more expensive in terms of - computation and memory. Defaults to False. - name: Optional name prefix for the operations created when applying - gradients. Defaults to "RMSProp". - """ - super(RMSPropOptimizer, self).__init__(use_locking, name) - self._set_hyper("learning_rate", learning_rate) - self._set_hyper("decay", decay) - self._set_hyper("momentum", momentum) - self._set_hyper("epsilon", epsilon) - - self._centered = centered - - def _create_vars(self, var_list, state): - for v in var_list: - init_rms = state.get_hyper("epsilon", - v.dtype.base_dtype) * array_ops.ones_like(v) - state.create_slot_with_initializer(v, init_rms, v.get_shape(), - v.dtype.base_dtype, "rms") - if self._centered: - state.zeros_slot(v, "mg") - state.zeros_slot(v, "momentum") - - def _apply_dense(self, grad, var, state): - rms = state.get_slot(var, "rms") - mom = state.get_slot(var, "momentum") - if self._centered: - mg = state.get_slot(var, "mg") - return training_ops.apply_centered_rms_prop( - var, - mg, - rms, - mom, - state.get_hyper("learning_rate", var.dtype.base_dtype), - state.get_hyper("decay", var.dtype.base_dtype), - state.get_hyper("momentum", var.dtype.base_dtype), - # epsilon is now the rms initial value and is not added to the - # denominator anymore, hence calling the kernel op with epsilon=0. - 0, - grad, - use_locking=self._use_locking).op - else: - return training_ops.apply_rms_prop( - var, - rms, - mom, - state.get_hyper("learning_rate", var.dtype.base_dtype), - state.get_hyper("decay", var.dtype.base_dtype), - state.get_hyper("momentum", var.dtype.base_dtype), - 0, - grad, - use_locking=self._use_locking).op - - def _resource_apply_dense(self, grad, var, state): - rms = state.get_slot(var, "rms") - mom = state.get_slot(var, "momentum") - if self._centered: - mg = state.get_slot(var, "mg") - return training_ops.resource_apply_centered_rms_prop( - var.handle, - mg.handle, - rms.handle, - mom.handle, - state.get_hyper("learning_rate", var.dtype.base_dtype), - state.get_hyper("decay", var.dtype.base_dtype), - state.get_hyper("momentum", var.dtype.base_dtype), - 0, - grad, - use_locking=self._use_locking) - else: - return training_ops.resource_apply_rms_prop( - var.handle, - rms.handle, - mom.handle, - state.get_hyper("learning_rate", var.dtype.base_dtype), - state.get_hyper("decay", var.dtype.base_dtype), - state.get_hyper("momentum", var.dtype.base_dtype), - 0, - grad, - use_locking=self._use_locking) - - def _apply_sparse(self, grad, var, state): - rms = state.get_slot(var, "rms") - mom = state.get_slot(var, "momentum") - if self._centered: - mg = state.get_slot(var, "mg") - return training_ops.sparse_apply_centered_rms_prop( - var, - mg, - rms, - mom, - state.get_hyper("learning_rate", var.dtype.base_dtype), - state.get_hyper("decay", var.dtype.base_dtype), - state.get_hyper("momentum", var.dtype.base_dtype), - 0, - grad.values, - grad.indices, - use_locking=self._use_locking) - else: - return training_ops.sparse_apply_rms_prop( - var, - rms, - mom, - state.get_hyper("learning_rate", var.dtype.base_dtype), - state.get_hyper("decay", var.dtype.base_dtype), - state.get_hyper("momentum", var.dtype.base_dtype), - 0, - grad.values, - grad.indices, - use_locking=self._use_locking) - - def _resource_apply_sparse(self, grad, var, indices, state): - rms = state.get_slot(var, "rms") - mom = state.get_slot(var, "momentum") - if self._centered: - mg = self.get_slot(var, "mg") - return training_ops.resource_sparse_apply_centered_rms_prop( - var.handle, - mg.handle, - rms.handle, - mom.handle, - state.get_hyper("learning_rate", var.dtype.base_dtype), - state.get_hyper("decay", var.dtype.base_dtype), - state.get_hyper("momentum", var.dtype.base_dtype), - 0, - grad, - indices, - use_locking=self._use_locking) - else: - return training_ops.resource_sparse_apply_rms_prop( - var.handle, - rms.handle, - mom.handle, - state.get_hyper("learning_rate", var.dtype.base_dtype), - state.get_hyper("decay", var.dtype.base_dtype), - state.get_hyper("momentum", var.dtype.base_dtype), - 0, - grad, - indices, - use_locking=self._use_locking) diff --git a/tensorflow/contrib/optimizer_v2/rmsprop_test.py b/tensorflow/contrib/optimizer_v2/rmsprop_test.py deleted file mode 100644 index a161538d151..00000000000 --- a/tensorflow/contrib/optimizer_v2/rmsprop_test.py +++ /dev/null @@ -1,502 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for rmsprop optimizer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import copy -import math - -from absl.testing import parameterized -import numpy as np - -from tensorflow.contrib.optimizer_v2 import rmsprop -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - -_DATA_TYPES = [dtypes.half, dtypes.float32] - -_TEST_PARAM_VALUES = [ - # learning_rate, decay, momentum, epsilon, centered, use_resource - [0.5, 0.9, 0.0, 1.0, True, False], - [0.5, 0.9, 0.0, 1.0, False, False], - [0.5, 0.9, 0.0, 1.0, True, True], - [0.5, 0.9, 0.0, 1.0, False, True], - [0.1, 0.9, 0.0, 1.0, True, False], - [0.5, 0.95, 0.0, 1.0, False, False], - [0.5, 0.8, 0.0, 1e-3, True, False], - [0.5, 0.8, 0.9, 1e-3, True, False], -] - - -class RMSPropOptimizerTest(test.TestCase, parameterized.TestCase): - - def _rmsprop_update_numpy(self, var, g, mg, rms, mom, lr, decay, momentum, - centered): - rms_t = rms * decay + (1 - decay) * g * g - if centered: - mg_t = mg * decay + (1 - decay) * g - denom_t = rms_t - mg_t * mg_t - else: - mg_t = mg - denom_t = rms_t - mom_t = momentum * mom + lr * g / np.sqrt(denom_t, dtype=denom_t.dtype) - var_t = var - mom_t - return var_t, mg_t, rms_t, mom_t - - def _sparse_rmsprop_update_numpy(self, var, gindexs, gvalues, mg, rms, mom, - lr, decay, momentum, centered): - mg_t = copy.deepcopy(mg) - rms_t = copy.deepcopy(rms) - mom_t = copy.deepcopy(mom) - var_t = copy.deepcopy(var) - for i in range(len(gindexs)): - gindex = gindexs[i] - gvalue = gvalues[i] - rms_t[gindex] = rms[gindex] * decay + (1 - decay) * gvalue * gvalue - denom_t = rms_t[gindex] - if centered: - mg_t[gindex] = mg_t[gindex] * decay + (1 - decay) * gvalue - denom_t -= mg_t[gindex] * mg_t[gindex] - mom_t[gindex] = momentum * mom[gindex] + lr * gvalue / np.sqrt(denom_t) - var_t[gindex] = var[gindex] - mom_t[gindex] - return var_t, mg_t, rms_t, mom_t - - @parameterized.named_parameters( - *test_util.generate_combinations_with_testcase_name( - dtype=_DATA_TYPES, param_value=_TEST_PARAM_VALUES)) - def testDense(self, dtype, param_value): - (learning_rate, decay, momentum, epsilon, centered, use_resource) = tuple( - param_value) - with self.session(use_gpu=True): - # Initialize variables for numpy implementation. - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.2], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.2], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable(var0_np) - var1 = resource_variable_ops.ResourceVariable(var1_np) - else: - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - opt = rmsprop.RMSPropOptimizer( - learning_rate=learning_rate, - decay=decay, - momentum=momentum, - epsilon=epsilon, - centered=centered) - - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - mg0 = opt.get_slot(var0, "mg") - self.assertEqual(mg0 is not None, centered) - mg1 = opt.get_slot(var1, "mg") - self.assertEqual(mg1 is not None, centered) - rms0 = opt.get_slot(var0, "rms") - self.assertIsNotNone(rms0) - rms1 = opt.get_slot(var1, "rms") - self.assertIsNotNone(rms1) - mom0 = opt.get_slot(var0, "momentum") - self.assertIsNotNone(mom0) - mom1 = opt.get_slot(var1, "momentum") - self.assertIsNotNone(mom1) - - mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - rms0_np = np.array([epsilon, epsilon], dtype=dtype.as_numpy_dtype) - rms1_np = np.array([epsilon, epsilon], dtype=dtype.as_numpy_dtype) - mom0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - # Run 4 steps of RMSProp - for _ in range(4): - update.run() - - var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy( - var0_np, grads0_np, mg0_np, rms0_np, mom0_np, learning_rate, - decay, momentum, centered) - var1_np, mg1_np, rms1_np, mom1_np = self._rmsprop_update_numpy( - var1_np, grads1_np, mg1_np, rms1_np, mom1_np, learning_rate, - decay, momentum, centered) - - # Validate updated params - if centered: - self.assertAllCloseAccordingToType(mg0_np, mg0.eval()) - self.assertAllCloseAccordingToType(mg1_np, mg1.eval()) - self.assertAllCloseAccordingToType(rms0_np, rms0.eval()) - self.assertAllCloseAccordingToType(rms1_np, rms1.eval()) - self.assertAllCloseAccordingToType(mom0_np, mom0.eval()) - self.assertAllCloseAccordingToType(mom1_np, mom1.eval()) - # TODO(b/117393988): Reduce tolerances for float16. - self.assertAllCloseAccordingToType( - var0_np, var0.eval(), half_rtol=3e-3, half_atol=3e-3) - self.assertAllCloseAccordingToType( - var1_np, var1.eval(), half_rtol=3e-3, half_atol=3e-3) - - @parameterized.parameters([dtypes.float32, dtypes.float64]) - def testMinimizeSparseResourceVariable(self, dtype): - with self.cached_session(): - var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) - x = constant_op.constant([[4.0], [5.0]], dtype=dtype) - pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) - loss = pred * pred - sgd_op = rmsprop.RMSPropOptimizer( - learning_rate=1.0, - decay=0.0, - momentum=0.0, - epsilon=0.0, - centered=False).minimize(loss) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) - # Run 1 step of sgd - sgd_op.run() - # Validate updated params - self.assertAllCloseAccordingToType( - [[0., 1.]], var0.eval(), atol=0.01) - - @parameterized.parameters([dtypes.float32, dtypes.float64]) - def testMinimizeSparseResourceVariableCentered(self, dtype): - with self.cached_session(): - var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) - x = constant_op.constant([[4.0], [5.0]], dtype=dtype) - pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) - loss = pred * pred - sgd_op = rmsprop.RMSPropOptimizer( - learning_rate=1.0, - decay=0.1, - momentum=0.0, - epsilon=1.0, - centered=True).minimize(loss) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) - # Run 1 step of sgd - sgd_op.run() - # Validate updated params - self.assertAllCloseAccordingToType( - [[-7/3.0, -4/3.0]], var0.eval(), atol=0.01) - - @parameterized.named_parameters( - *test_util.generate_combinations_with_testcase_name( - dtype=_DATA_TYPES, param_value=_TEST_PARAM_VALUES)) - def testSparse(self, dtype, param_value): - (learning_rate, decay, momentum, epsilon, centered, _) = tuple( - param_value) - with self.session(use_gpu=True): - # Initialize variables for numpy implementation. - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01], dtype=dtype.as_numpy_dtype) - - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0_np_indices = np.array([0], dtype=np.int32) - grads0 = ops.IndexedSlices( - constant_op.constant(grads0_np), - constant_op.constant(grads0_np_indices), constant_op.constant([1])) - grads1_np_indices = np.array([1], dtype=np.int32) - grads1 = ops.IndexedSlices( - constant_op.constant(grads1_np), - constant_op.constant(grads1_np_indices), constant_op.constant([1])) - opt = rmsprop.RMSPropOptimizer( - learning_rate=learning_rate, - decay=decay, - momentum=momentum, - epsilon=epsilon, - centered=centered) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - mg0 = opt.get_slot(var0, "mg") - self.assertEqual(mg0 is not None, centered) - mg1 = opt.get_slot(var1, "mg") - self.assertEqual(mg1 is not None, centered) - rms0 = opt.get_slot(var0, "rms") - self.assertIsNotNone(rms0) - rms1 = opt.get_slot(var1, "rms") - self.assertIsNotNone(rms1) - mom0 = opt.get_slot(var0, "momentum") - self.assertIsNotNone(mom0) - mom1 = opt.get_slot(var1, "momentum") - self.assertIsNotNone(mom1) - - mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - rms0_np = np.array([epsilon, epsilon], dtype=dtype.as_numpy_dtype) - rms1_np = np.array([epsilon, epsilon], dtype=dtype.as_numpy_dtype) - mom0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - - # Run 4 steps of RMSProp - for _ in range(4): - update.run() - - var0_np, mg0_np, rms0_np, mom0_np = self._sparse_rmsprop_update_numpy( - var0_np, grads0_np_indices, grads0_np, mg0_np, rms0_np, mom0_np, - learning_rate, decay, momentum, centered) - var1_np, mg1_np, rms1_np, mom1_np = self._sparse_rmsprop_update_numpy( - var1_np, grads1_np_indices, grads1_np, mg1_np, rms1_np, mom1_np, - learning_rate, decay, momentum, centered) - - # Validate updated params - if centered: - self.assertAllCloseAccordingToType(mg0_np, mg0.eval()) - self.assertAllCloseAccordingToType(mg1_np, mg1.eval()) - self.assertAllCloseAccordingToType(rms0_np, rms0.eval()) - self.assertAllCloseAccordingToType(rms1_np, rms1.eval()) - self.assertAllCloseAccordingToType(mom0_np, mom0.eval()) - self.assertAllCloseAccordingToType(mom1_np, mom1.eval()) - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) - - @parameterized.parameters(_DATA_TYPES) - def testWithoutMomentum(self, dtype): - with self.session(use_gpu=True): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - opt = rmsprop.RMSPropOptimizer( - learning_rate=2.0, decay=0.9, momentum=0.0, epsilon=1.0) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - rms0 = opt.get_slot(var0, "rms") - self.assertIsNotNone(rms0) - rms1 = opt.get_slot(var1, "rms") - self.assertIsNotNone(rms1) - mom0 = opt.get_slot(var0, "momentum") - self.assertIsNotNone(mom0) - mom1 = opt.get_slot(var1, "momentum") - self.assertIsNotNone(mom1) - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Step 1: the rms accumulators where 1. So we should see a normal - # update: v -= grad * learning_rate - update.run() - # Check the root mean square accumulators. - self.assertAllCloseAccordingToType( - np.array([0.901, 0.901]), rms0.eval()) - self.assertAllCloseAccordingToType( - np.array([0.90001, 0.90001]), rms1.eval()) - # Check the parameters. - self.assertAllCloseAccordingToType( - np.array([ - 1.0 - (0.1 * 2.0 / math.sqrt(0.901)), - 2.0 - (0.1 * 2.0 / math.sqrt(0.901)) - ]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([ - 3.0 - (0.01 * 2.0 / math.sqrt(0.90001)), - 4.0 - (0.01 * 2.0 / math.sqrt(0.90001)) - ]), var1.eval()) - # Step 2: the root mean square accumulators contain the previous update. - update.run() - # Check the rms accumulators. - self.assertAllCloseAccordingToType( - np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), rms0.eval()) - self.assertAllCloseAccordingToType( - np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), rms1.eval()) - # Check the parameters. - self.assertAllCloseAccordingToType( - np.array([ - 1.0 - (0.1 * 2.0 / math.sqrt(0.901)) - - (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001)), - 2.0 - (0.1 * 2.0 / math.sqrt(0.901)) - - (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001)) - ]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([ - 3.0 - (0.01 * 2.0 / math.sqrt(0.90001)) - - (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5)), - 4.0 - (0.01 * 2.0 / math.sqrt(0.90001)) - - (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5)) - ]), var1.eval()) - - @parameterized.parameters(_DATA_TYPES) - def testWithMomentum(self, dtype): - with self.session(use_gpu=True): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - - opt = rmsprop.RMSPropOptimizer( - learning_rate=2.0, decay=0.9, momentum=0.5, epsilon=1.0) - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() - - rms0 = opt.get_slot(var0, "rms") - self.assertIsNotNone(rms0) - rms1 = opt.get_slot(var1, "rms") - self.assertIsNotNone(rms1) - mom0 = opt.get_slot(var0, "momentum") - self.assertIsNotNone(mom0) - mom1 = opt.get_slot(var1, "momentum") - self.assertIsNotNone(mom1) - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Step 1: rms = 1, mom = 0. So we should see a normal - # update: v -= grad * learning_rate - update.run() - # Check the root mean square accumulators. - self.assertAllCloseAccordingToType( - np.array([0.901, 0.901]), rms0.eval()) - self.assertAllCloseAccordingToType( - np.array([0.90001, 0.90001]), rms1.eval()) - # Check the momentum accumulators - self.assertAllCloseAccordingToType( - np.array([(0.1 * 2.0 / math.sqrt(0.901)), - (0.1 * 2.0 / math.sqrt(0.901))]), mom0.eval()) - self.assertAllCloseAccordingToType( - np.array([(0.01 * 2.0 / math.sqrt(0.90001)), - (0.01 * 2.0 / math.sqrt(0.90001))]), mom1.eval()) - - # Check that the parameters. - self.assertAllCloseAccordingToType( - np.array([ - 1.0 - (0.1 * 2.0 / math.sqrt(0.901)), - 2.0 - (0.1 * 2.0 / math.sqrt(0.901)) - ]), var0.eval()) - self.assertAllCloseAccordingToType( - np.array([ - 3.0 - (0.01 * 2.0 / math.sqrt(0.90001)), - 4.0 - (0.01 * 2.0 / math.sqrt(0.90001)) - ]), var1.eval()) - - # Step 2: the root mean square accumulators contain the previous update. - update.run() - # Check the rms accumulators. - self.assertAllCloseAccordingToType( - np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), rms0.eval()) - self.assertAllCloseAccordingToType( - np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), rms1.eval()) - self.assertAllCloseAccordingToType( - np.array([ - 0.5 * (0.1 * 2.0 / math.sqrt(0.901)) + - (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001)), - 0.5 * (0.1 * 2.0 / math.sqrt(0.901)) + - (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001)) - ]), mom0.eval()) - self.assertAllCloseAccordingToType( - np.array([ - 0.5 * (0.01 * 2.0 / math.sqrt(0.90001)) + - (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5)), - 0.5 * (0.01 * 2.0 / math.sqrt(0.90001)) + - (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5)) - ]), mom1.eval()) - - # Check the parameters. - self.assertAllCloseAccordingToType( - np.array([ - 1.0 - (0.1 * 2.0 / math.sqrt(0.901)) - - (0.5 * (0.1 * 2.0 / math.sqrt(0.901)) + - (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001))), - 2.0 - (0.1 * 2.0 / math.sqrt(0.901)) - - (0.5 * (0.1 * 2.0 / math.sqrt(0.901)) + - (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001))) - ]), var0.eval()) - - self.assertAllCloseAccordingToType( - np.array([ - 3.0 - (0.01 * 2.0 / math.sqrt(0.90001)) - - (0.5 * (0.01 * 2.0 / math.sqrt(0.90001)) + - (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5))), - 4.0 - (0.01 * 2.0 / math.sqrt(0.90001)) - - (0.5 * (0.01 * 2.0 / math.sqrt(0.90001)) + - (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5))) - ]), var1.eval()) - - -class SlotColocationTest(test.TestCase, parameterized.TestCase): - - @parameterized.parameters([True, False]) - @test_util.run_gpu_only - @test_util.run_in_graph_and_eager_modes - def testRunMinimizeOnGPUForCPUVariables(self, use_resource): - with ops.device("/device:CPU:0"): - if use_resource: - var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], - dtype=dtypes.float32) - var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], - dtype=dtypes.float32) - global_step = resource_variable_ops.ResourceVariable( - array_ops.zeros([], dtypes.int64), name="global_step") - else: - var0 = variables.Variable([1.0, 2.0], dtype=dtypes.float32) - var1 = variables.Variable([3.0, 4.0], dtype=dtypes.float32) - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name="global_step") - - def loss(): - return 5 * var0 + 3 * var1 - - opt = rmsprop.RMSPropOptimizer( - learning_rate=1.0, decay=0.9, momentum=0.5, epsilon=1.0) - - # Fetch params to validate initial values - self.evaluate(variables.global_variables_initializer()) - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - - # Run 1 step through optimizer on GPU. - # Slot variables are created the first time optimizer is used on some - # variable. This tests that slot variables will be colocated with the base - # variable. - with ops.device("/device:GPU:0"): - # Note that for eager execution, minimize expects a function instead of a - # Tensor. - opt_op = opt.minimize(loss, global_step, [var0, var1]) - self.evaluate(variables.global_variables_initializer()) - self.evaluate(opt_op) - - # Validate updated params, All variables should have decreased. - self.assertTrue(all(v < 0.0 for v in self.evaluate(var0)), - msg="updated variables: %s" % self.evaluate(var0)) - self.assertTrue(all(v < 2.0 for v in self.evaluate(var1)), - msg="updated variables: %s" % self.evaluate(var1)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/periodic_resample/BUILD b/tensorflow/contrib/periodic_resample/BUILD deleted file mode 100644 index 82e92f656f7..00000000000 --- a/tensorflow/contrib/periodic_resample/BUILD +++ /dev/null @@ -1,138 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "py_test") -load( - "//tensorflow:tensorflow.bzl", - "tf_cc_test", - "tf_custom_op_library", - "tf_gen_op_libs", - "tf_gen_op_wrapper_py", -) -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -cc_library( - name = "all_ops", - srcs = [":custom_op_sources"], - hdrs = [":custom_op_headers"], - deps = [ - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - "@com_google_protobuf//:protobuf_headers", - ], - alwayslink = 1, -) - -tf_custom_op_library( - name = "python/ops/_periodic_resample_op.so", - srcs = [ - ":custom_op_headers", - ":custom_op_sources", - ], -) - -tf_gen_op_libs( - op_lib_names = ["array_ops"], -) - -tf_gen_op_wrapper_py( - name = "gen_periodic_resample_op_py", - out = "python/ops/gen_periodic_resample_op.py", - deps = [":array_ops_op_lib"], -) - -tf_custom_op_py_library( - name = "periodic_resample_op_py", - srcs = ["python/ops/periodic_resample_op.py"], - dso = ["python/ops/_periodic_resample_op.so"], - kernels = [ - ":array_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":gen_periodic_resample_op_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:framework_for_generated_wrappers", - ], -) - -py_library( - name = "init_py", - srcs = [ - "__init__.py", - "python/__init__.py", - ], - srcs_version = "PY2AND3", - tags = [ - "notap", - ], - deps = [ - ":periodic_resample_op_py", - ], -) - -py_test( - name = "periodic_resample_op_test", - srcs = ["python/kernel_tests/periodic_resample_op_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "notap", - ], - deps = [ - ":init_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradient_checker", - ], -) - -tf_cc_test( - name = "periodic_resample_op_cc_test", - size = "small", - srcs = [ - "ops/array_ops_test.cc", - ], - deps = [ - ":all_ops", - "//tensorflow/core:framework", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -# py_library( -# name = "periodic_resample_op_py", -# srcs = ["python/ops/periodic_resample_op.py"], -# data = ["python/ops/_periodic_resample_op.so"], -# srcs_version = "PY2AND3", -# ) - -filegroup( - name = "custom_op_sources", - srcs = glob( - [ - "ops/*.cc", - "kernels/*.cc", - ], - exclude = [ - "ops/*_test.cc", - "kernels/*_test.cc", - ], - ), -) - -filegroup( - name = "custom_op_headers", - srcs = glob( - [ - "kernels/*.h", - "ops/*.h", - ], - ), -) diff --git a/tensorflow/contrib/periodic_resample/__init__.py b/tensorflow/contrib/periodic_resample/__init__.py deleted file mode 100644 index fde9091b88f..00000000000 --- a/tensorflow/contrib/periodic_resample/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# ============================================================================= -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Custom op used by periodic_resample.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.periodic_resample.python.ops.periodic_resample_op import periodic_resample -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = ["periodic_resample"] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/periodic_resample/kernels/periodic_resample_op.cc b/tensorflow/contrib/periodic_resample/kernels/periodic_resample_op.cc deleted file mode 100644 index 514689cf454..00000000000 --- a/tensorflow/contrib/periodic_resample/kernels/periodic_resample_op.cc +++ /dev/null @@ -1,30 +0,0 @@ -// ============================================================================= -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include "tensorflow/contrib/periodic_resample/kernels/periodic_resample_op.h" -#include "tensorflow/core/framework/register_types.h" - -namespace tensorflow { - -REGISTER_KERNEL_BUILDER(Name("PeriodicResample").Device(DEVICE_CPU), - PeriodicResampleOp); - - -REGISTER_KERNEL_BUILDER(Name("PeriodicResampleOpGrad") - .Device(DEVICE_CPU), - PeriodicResampleOpGrad); - -} // namespace tensorflow diff --git a/tensorflow/contrib/periodic_resample/kernels/periodic_resample_op.h b/tensorflow/contrib/periodic_resample/kernels/periodic_resample_op.h deleted file mode 100644 index 85b5a5a3b95..00000000000 --- a/tensorflow/contrib/periodic_resample/kernels/periodic_resample_op.h +++ /dev/null @@ -1,424 +0,0 @@ -// ============================================================================= -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#ifndef TENSORFLOW_CONTRIB_PERIODIC_RESAMPLE_KERNELS_PERIODIC_RESAMPLE_OP_H_ -#define TENSORFLOW_CONTRIB_PERIODIC_RESAMPLE_KERNELS_PERIODIC_RESAMPLE_OP_H_ - -#include -#include -#include -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace { - -// Computes input tensor index for given output index during forward -// propagation through periodic_resample operation. -class InputIndexer { - public: - InputIndexer(const std::vector& output_dimensions, - const tensorflow::TensorShape& input_shape, - int adjustable_dimension) - : output_dimensions_(output_dimensions), - adjustable_dimension_(adjustable_dimension), - rank_(input_shape.dims()), - linear_output_index_(0), - linear_input_index_(0), - adjustable_dimension_carriage_sum_(0) { - auto input_dimensions = TensorShapeToVector(input_shape); - // factors by which input_dimensions increases/decreases w.r.t. - // output_dimensions - dimension_ceiling_ = - ComputeDimensionCeiling(output_dimensions, input_dimensions); - cumulative_dimensions_ = ComputeCumulativeDimensions(); - - output_indices_.resize(output_dimensions_.size()); - input_indices_.resize(output_dimensions_.size()); - - // Compute index_factors - index_factors_.resize(rank_); - tensorflow::int64 last_index_factor = 1; - for (auto r = rank_ - 1; r >= 0; --r) { - index_factors_[r] = last_index_factor; - last_index_factor *= input_dimensions[r]; - } - } - - tensorflow::int64 linear_input_index() const { return linear_input_index_; } - - void MoveToOutputIndex(tensorflow::int64 output_index); - void IncrementOutputIndex(); - - private: - void RecomputeInputAdjustableDimensionIndex() { - tensorflow::int64 index = adjustable_dimension_carriage_sum_; - index *= output_dimensions_[adjustable_dimension_]; - index += output_indices_[adjustable_dimension_]; - input_indices_[adjustable_dimension_] = index; - } - - std::vector TensorShapeToVector( - const tensorflow::TensorShape& tensor_shape); - - std::vector ComputeDimensionCeiling( - const std::vector& output_dimensions, - const std::vector& input_dimensions); - - std::vector ComputeCumulativeDimensions(); - - const std::vector output_dimensions_; - std::vector dimension_ceiling_; - std::vector index_factors_; - std::vector cumulative_dimensions_; - std::vector output_indices_; - std::vector input_indices_; - - const int adjustable_dimension_; - const int rank_; - tensorflow::int64 linear_output_index_; - tensorflow::int64 linear_input_index_; - tensorflow::int64 adjustable_dimension_carriage_sum_; -}; - -void InputIndexer::MoveToOutputIndex(tensorflow::int64 output_index) { - linear_output_index_ = output_index; - linear_input_index_ = 0; - - // un-rasterize the output index - auto last_reduced_i = output_index; - for (auto r = rank_ - 1; r >= 0; --r) { - output_indices_[r] = last_reduced_i % output_dimensions_[r]; - last_reduced_i = - (last_reduced_i - output_indices_[r]) / output_dimensions_[r]; - } - - tensorflow::int64 carriage_sum = 0; - for (int qi = 0; qi < rank_; ++qi) { - if (qi == adjustable_dimension_) continue; - carriage_sum += cumulative_dimensions_[qi] * - (output_indices_[qi] % dimension_ceiling_[qi]); - } - adjustable_dimension_carriage_sum_ = carriage_sum; - - // rasterize the input index - for (auto r = rank_ - 1; r >= 0; --r) { - if (r != adjustable_dimension_) { - input_indices_[r] = output_indices_[r] / dimension_ceiling_[r]; - } else { - RecomputeInputAdjustableDimensionIndex(); - } - } - for (auto r = rank_ - 1; r >= 0; --r) { - linear_input_index_ += index_factors_[r] * input_indices_[r]; - } -} - -void InputIndexer::IncrementOutputIndex() { - linear_output_index_++; - for (auto r = rank_ - 1; r >= 0; --r) { - auto old_carriage_sum_increment = - cumulative_dimensions_[r] * - (output_indices_[r] % dimension_ceiling_[r]); - output_indices_[r] = (output_indices_[r] + 1) % output_dimensions_[r]; - if (r != adjustable_dimension_) { - auto new_input_index = output_indices_[r] / dimension_ceiling_[r]; - linear_input_index_ += - (new_input_index - input_indices_[r]) * index_factors_[r]; - - input_indices_[r] = new_input_index; - - auto new_carriage_sum_increment = - cumulative_dimensions_[r] * - (output_indices_[r] % dimension_ceiling_[r]); - - adjustable_dimension_carriage_sum_ = adjustable_dimension_carriage_sum_ - - old_carriage_sum_increment + - new_carriage_sum_increment; - } - - if (output_indices_[r] != 0) { - // No more carries to higher indices. - break; - } - } - auto old_adjustable_dimension_input_index = - input_indices_[adjustable_dimension_]; - RecomputeInputAdjustableDimensionIndex(); - linear_input_index_ += (input_indices_[adjustable_dimension_] - - old_adjustable_dimension_input_index) * - index_factors_[adjustable_dimension_]; -} - -std::vector InputIndexer::TensorShapeToVector( - const tensorflow::TensorShape& tensor_shape) { - std::vector result(tensor_shape.dims()); - int count = 0; - for (const auto dim_info : tensor_shape) { - result[count] = dim_info.size; - ++count; - } - return result; -} - -std::vector InputIndexer::ComputeDimensionCeiling( - const std::vector& output_dimensions, - const std::vector& input_dimensions) { - std::vector dimension_ceiling(input_dimensions.size()); - for (size_t i = 0; i < input_dimensions.size(); ++i) { - dimension_ceiling[i] = (output_dimensions[i] + input_dimensions[i] - 1) / - input_dimensions[i]; - } - return dimension_ceiling; -} - -std::vector InputIndexer::ComputeCumulativeDimensions() { - std::vector cumulative_dimensions(rank_); - int count = 0; - for (int i = 0; i < rank_; ++i) { - if (count == 0) { - cumulative_dimensions[count] = 1; - } else { - cumulative_dimensions[count] = - cumulative_dimensions[count - 1] * dimension_ceiling_[count - 1]; - } - ++count; - } - return cumulative_dimensions; -} - -template -void process_desired_shape(tensorflow::OpKernelContext* context, - const tensorflow::TensorShape& input_tensor_shape, - const IndexVecT& desired_shape, - int* adjustable_dimension, - std::vector* target_dimensions, - tensorflow::int64* output_size) { - tensorflow::int64 new_sliced_size = 1; - bool found = false; - const int rank = input_tensor_shape.dims(); - for (int i = 0; i < rank; ++i) { - if (desired_shape[i] < 1) { - // only one index can be adjustable - OP_REQUIRES(context, !found, - tensorflow::errors::InvalidArgument( - "periodic_resample expects only " - "one index to be marked as adjustable.")); - *adjustable_dimension = i; - found = true; - } else { - OP_REQUIRES( - context, desired_shape[i] >= input_tensor_shape.dim_size(i), - tensorflow::errors::InvalidArgument( - "periodic_resample expects the size of non-adjustable " - "dimensions be at least as large as size of input tensor." - " Dimension ", - i, " input tensor has size ", input_tensor_shape.dim_size(i), - ", desired shape has size ", desired_shape[i], ".")); - - (*target_dimensions)[i] = desired_shape[i]; - new_sliced_size *= (*target_dimensions)[i]; - } - } - // at least one index needs to be adjustable - OP_REQUIRES(context, found, - tensorflow::errors::InvalidArgument( - "periodic_resample expects at least " - "one index to be marked as adjustable.")); - (*target_dimensions)[*adjustable_dimension] = - input_tensor_shape.num_elements() / new_sliced_size; - - *output_size = new_sliced_size * (*target_dimensions)[*adjustable_dimension]; -} - -// Heuristic number based on measurements on -// Intel(R) Core(TM) i7-4930K CPU @ 3.40GHz -const tensorflow::int64 costPerFillIndex = 35; - -enum class Mode { - kForward, - kGradient -}; - -// Computes either periodic_resample operation output or gradients for it, -// depending on |mode|. -// |original_shape| is always shape of input to periodic_resample operation. -// |source_tensor| is either source for periodic_resample (for forward mode) -// or gradients tensor. -// |desired_shape| is always shape, provided by user, to which forward -// propagation attempts resample input tensor. -template -void -do_periodic_resample_op(tensorflow::OpKernelContext* context, - const tensorflow::TensorShape& original_shape, - const tensorflow::PartialTensorShape& desired_shape, - const tensorflow::Tensor& source_tensor) { - const int rank = source_tensor.dims(); - - // requires that the rank of the input tensor and length of the desired shape - // are equal - OP_REQUIRES(context, rank == desired_shape.dims(), - tensorflow::errors::InvalidArgument( - "periodic_resample expects the rank of the input tensor, ", - rank, ", to be the same as the length of the desired shape, ", - desired_shape.dims(), ".")); - - std::vector target_dimensions(rank); - tensorflow::int64 new_size = 0; - // index of adjustable dimension - int adjustable_dimension = 0; - process_desired_shape(context, original_shape, desired_shape.dim_sizes(), - &adjustable_dimension, &target_dimensions, &new_size); - - // ensure that the new dimension is greater than zero - OP_REQUIRES(context, target_dimensions[adjustable_dimension] > 0, - tensorflow::errors::InvalidArgument( - "periodic_resample found that the " - "adjustable dimension, ", - adjustable_dimension, ", isn't greater than zero, ", - target_dimensions[adjustable_dimension], ".")); - tensorflow::TensorShape output_shape; - if (mode == Mode::kForward) { - for (int i = 0; i < rank; ++i) { - output_shape.AddDim(target_dimensions[i]); - } - } else { - output_shape = original_shape; - } - - // Create an output tensor and attach it to the current context - tensorflow::Tensor* output_tensor = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(0, output_shape, &output_tensor)); - auto output = output_tensor->flat(); - - // input is a strided array (last index is fastest, C-ordered) - auto input = source_tensor.flat(); - - // Fill output tensor with periodically resampled input tensor values - InputIndexer input_indexer(target_dimensions, original_shape, - adjustable_dimension); - - auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads()); - auto fill_output_tensor = [&input_indexer, &output, &input]( - tensorflow::int64 start, tensorflow::int64 limit) { - InputIndexer local_indexer(input_indexer); - local_indexer.MoveToOutputIndex(start); - for (tensorflow::int64 output_index = start; output_index < limit; - ++output_index) { - if (mode == Mode::kForward) { - output(output_index) = input(local_indexer.linear_input_index()); - } else { - output(local_indexer.linear_input_index()) = input(output_index); - } - local_indexer.IncrementOutputIndex(); - } - }; - ::tensorflow::Shard(worker_threads.num_threads, worker_threads.workers, - new_size, costPerFillIndex, fill_output_tensor); -} - -#define DATA_TYPE_SWITCH(data_type, context, CASE) \ - switch (data_type) { \ - CASE(float) \ - CASE(double) \ - CASE(tensorflow::int32) \ - CASE(tensorflow::int64) \ - default: \ - context->CtxFailure(__FILE__, __LINE__, \ - tensorflow::errors::InvalidArgument( \ - "Unsuppored tensor elements type")); \ - break; \ - } - -void create_output_tensor( - tensorflow::OpKernelContext* context, - const tensorflow::Tensor& input_tensor, - const tensorflow::DataType& input_tensor_type, - const tensorflow::PartialTensorShape& desired_shape) { -#define CASE(type) \ - case tensorflow::DataTypeToEnum::value: \ - do_periodic_resample_op( \ - context, input_tensor.shape(), desired_shape, input_tensor); \ - break; - - DATA_TYPE_SWITCH(input_tensor_type, context, CASE); -#undef CASE -} - -void create_grad_tensor(tensorflow::OpKernelContext* context, - const tensorflow::Tensor& grad_tensor, - const tensorflow::DataType& grad_tensor_type, - const tensorflow::TensorShape& original_shape, - const tensorflow::PartialTensorShape& desired_shape) { -#define CASE(type) \ - case tensorflow::DataTypeToEnum::value: \ - do_periodic_resample_op( \ - context, original_shape, desired_shape, grad_tensor); \ - break; - - DATA_TYPE_SWITCH(grad_tensor_type, context, CASE); -#undef CASE -} - -} // namespace - -class PeriodicResampleOp : public tensorflow::OpKernel { - public: - explicit PeriodicResampleOp(tensorflow::OpKernelConstruction* context) - : tensorflow::OpKernel(context) { - // Get the desired shape - OP_REQUIRES_OK(context, context->GetAttr("shape", &desired_shape)); - } - - void Compute(tensorflow::OpKernelContext* context) override { - // Grab the input tensor - const tensorflow::Tensor& input_tensor = context->input(0); - const tensorflow::DataType input_tensor_type = context->input_dtype(0); - - create_output_tensor(context, input_tensor, input_tensor_type, - desired_shape); - } - - private: - tensorflow::PartialTensorShape desired_shape; -}; - -class PeriodicResampleOpGrad : public tensorflow::OpKernel { - public: - explicit PeriodicResampleOpGrad(tensorflow::OpKernelConstruction* context) - : tensorflow::OpKernel(context) { - OP_REQUIRES_OK(context, - context->GetAttr("original_shape", &original_shape)); - OP_REQUIRES_OK(context, context->GetAttr("desired_shape", &desired_shape)); - } - - void Compute(tensorflow::OpKernelContext* context) override { - const tensorflow::Tensor& grad_tensor = context->input(0); - const tensorflow::DataType grad_tensor_type = context->input_dtype(0); - create_grad_tensor(context, grad_tensor, grad_tensor_type, original_shape, - desired_shape); - } - - private: - tensorflow::TensorShape original_shape; - tensorflow::PartialTensorShape desired_shape; -}; - -#endif // TENSORFLOW_CONTRIB_PERIODIC_RESAMPLE_KERNELS_PERIODIC_RESAMPLE_OP_H_ diff --git a/tensorflow/contrib/periodic_resample/ops/array_ops.cc b/tensorflow/contrib/periodic_resample/ops/array_ops.cc deleted file mode 100644 index fd38cd09b4d..00000000000 --- a/tensorflow/contrib/periodic_resample/ops/array_ops.cc +++ /dev/null @@ -1,155 +0,0 @@ -// ============================================================================= -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { - -REGISTER_OP("PeriodicResample") - .Attr("T: numbertype") - .Input("values: T") - .Attr("shape: shape") - .Output("output: T") - .SetShapeFn([](shape_inference::InferenceContext* c) { - tensorflow::PartialTensorShape desired_shape; - TF_RETURN_IF_ERROR(c->GetAttr("shape", &desired_shape)); - shape_inference::ShapeHandle input_tensor_shape = c->input(0); - shape_inference::DimensionHandle num_input_elements = - c->NumElements(input_tensor_shape); - shape_inference::ShapeHandle result_shape_handle; - if (!shape_inference::InferenceContext::ValueKnown(num_input_elements)) { - TF_RETURN_IF_ERROR(c->MakeShapeFromPartialTensorShape( - desired_shape, &result_shape_handle)); - } else { - const int rank = c->Rank(input_tensor_shape); - std::vector target_dimensions(rank); - tensorflow::int64 new_sliced_size = 1; - int adjustable_dimension = 0; - for (int i = 0; i < rank; ++i) { - if (desired_shape.dim_size(i) < 1) { - adjustable_dimension = i; - } else { - target_dimensions[i] = desired_shape.dim_size(i); - new_sliced_size *= target_dimensions[i]; - } - } - target_dimensions[adjustable_dimension] = - shape_inference::InferenceContext::Value( - num_input_elements) / new_sliced_size; - tensorflow::TensorShape result_shape; - for (int i = 0; i < rank; ++i) { - result_shape.AddDim(target_dimensions[i]); - } - TF_RETURN_IF_ERROR(c->MakeShapeFromTensorShape( - result_shape, &result_shape_handle)); - } - c->set_output(0, result_shape_handle); - return Status::OK(); - }) - .Doc(R"doc( -Periodically resample elements of a tensor to conform to `shape`. - -This function implements a slightly more generic version of the subpixel -convolutions found in this [paper](https://arxiv.org/abs/1609.05158). - -The formula for computing the elements in the `output` tensor is as follows: - - `T` = `values` tensor of rank `R` - - `S` = desired `shape` of output tensor (vector of length `R`) - - `P` = `output` tensor of rank `R` - - \\((T_1,\\ldots,T_R)\\) = shape(`T`) - - \\([S_1,\\ldots,S_q,\\ldots,S_R]\\) = elements of vector `S` - - A single element in `S` is left unspecified (denoted \\(S_q=-1\\)). - - Let \\(f_i\\) denote the (possibly non-integer) factor that relates the original - dimension to the desired dimensions, \\(S_i=f_i T_i\\), for \\(i\\neq q\\) where - \\(f_i>0\\). - - Define the following: - - \\(g_i=\\lceil f_i\\rceil\\) - - \\(t=\\prod_i T_i\\) - - \\(s=\\prod_{i\\neq q} S_i\\) - - \\(S_q\\) can then be defined by \\(S_q=\\lfloor t/s\\rfloor\\). - The elements of the resulting tensor are defined as - - \\(P_{s_1,\\ldots,s_R}=T_{h_1,\\ldots,h_q,\\ldots,h_R}\\). - - The \\(h_i\\) (\\(i\\neq q\\)) are defined by \\(h_i=\\lfloor s_i/g_i\\rfloor\\). - - \\(h_q=S_q\\sum_{j\\neq q}^{q-1}G_j \\mathrm{mod}(s_j,g_j) + s_q\\), where - \\(G_j=\\prod_{i}^{j-1}g_i\\) (\\(G_0=1\\)). - -One drawback of this method is that whenever the output dimensions are slightly -less than integer multiples of the input dimensions, many of the tensor elements -are repeated in an inefficient way. This is resolved by specifying that all -desired dimensions are integer multiples of the input tensor. - -For example: - -```prettyprint -`input` is [[ 0 1 2 3] - [ 4 5 6 7] - [ 8 9 10 11]] - -tf.periodic_resample(input, [6, None]) ==> [[ 0 1] - [ 2 3] - [ 4 5] - [ 6 7] - [ 8 9] - [10 11]] -``` - -values: The tensor of rank `R` to periodic_resample -shape: A 1-D tensor representing the desired shape of the output tensor. - Exactly one element of this tensor must have the value `None` which represents - that this dimension of `values` can be adjusted downward in order to - accommodate increases in other dimensions. The specified sizes of the - non-adjustable dimensions must by at least as large as in the `values` tensor. -output: Periodically resampled tensor that has dimensions specified as in - `shape` except that the dimension specified as `None` will be minimally - decreased as necessary. - -)doc"); - - -REGISTER_OP("PeriodicResampleOpGrad") - .Attr("T: numbertype") - .Input("grad: T") - .Attr("original_shape: shape") - .Attr("desired_shape: shape") - .Output("grad_values: T") - .SetShapeFn([](shape_inference::InferenceContext* c) { - tensorflow::TensorShape original_shape; - TF_RETURN_IF_ERROR(c->GetAttr("original_shape", &original_shape)); - shape_inference::ShapeHandle s; - TF_RETURN_IF_ERROR(c->MakeShapeFromTensorShape(original_shape, &s)); - c->set_output(0, s); - return Status::OK(); -}); - -} // namespace tensorflow diff --git a/tensorflow/contrib/periodic_resample/ops/array_ops_test.cc b/tensorflow/contrib/periodic_resample/ops/array_ops_test.cc deleted file mode 100644 index 43b7c1799ff..00000000000 --- a/tensorflow/contrib/periodic_resample/ops/array_ops_test.cc +++ /dev/null @@ -1,41 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/node_def_builder.h" -#include "tensorflow/core/framework/shape_inference_testutil.h" -#include "tensorflow/core/framework/tensor_shape.pb.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { - -TEST(ArrayOpsTest, PeriodicResample_ShapeFn) { - ShapeInferenceTestOp op("PeriodicResample"); - // Case 1: output shape can be fully inferreed. - PartialTensorShape shape({4, 4, -1}); - TensorShapeProto shape_proto; - shape.AsProto(&shape_proto); - - TF_ASSERT_OK(NodeDefBuilder("test", "PeriodicResample") - .Input({"values", 0, DT_INT32}) - .Attr("shape", shape_proto) - .Finalize(&op.node_def)); - INFER_OK(op, "[2,2,4]", "[4,4,1]"); - // Case 2: output shape can not be inferred - report desired shape. - INFER_OK(op, "[2,2,?]", "[4,4,?]"); -} - -} // end namespace tensorflow diff --git a/tensorflow/contrib/periodic_resample/python/__init__.py b/tensorflow/contrib/periodic_resample/python/__init__.py deleted file mode 100644 index a8b6ead0f59..00000000000 --- a/tensorflow/contrib/periodic_resample/python/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# ============================================================================= -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Public API of periodic_resample.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/periodic_resample/python/kernel_tests/periodic_resample_op_test.py b/tensorflow/contrib/periodic_resample/python/kernel_tests/periodic_resample_op_test.py deleted file mode 100644 index 9a195022766..00000000000 --- a/tensorflow/contrib/periodic_resample/python/kernel_tests/periodic_resample_op_test.py +++ /dev/null @@ -1,133 +0,0 @@ -# ============================================================================= -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy - -from tensorflow.contrib.periodic_resample import periodic_resample -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradient_checker -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest - - -class PeriodicResampleTest(test_util.TensorFlowTestCase): - - def testPeriodicResampleBasic2D(self): - - input_tensor = numpy.arange(12).reshape((3, 4)) - desired_shape = numpy.array([6, None]) - output_tensor = input_tensor.reshape((6, 2)) - - with self.cached_session(): - variables.global_variables_initializer().run() - result = periodic_resample(input_tensor, desired_shape).eval() - self.assertAllEqual(result, output_tensor) - - def testPeriodicResampleTruncatedBasic2D(self): - - input_tensor = numpy.arange(12).reshape((3, 4)) - desired_shape = numpy.array([5, None]) - output_tensor = input_tensor.reshape((6, 2))[:-1] - - with self.cached_session(): - variables.global_variables_initializer().run() - result = periodic_resample(input_tensor, desired_shape).eval() - self.assertAllEqual(result, output_tensor) - - def testPeriodicResampleBasic3D(self): - - input_tensor = numpy.arange(2 * 2 * 4).reshape((2, 2, 4)) - desired_shape = numpy.array([4, 4, None]) - output_tensor = numpy.array([[[0], [2], [4], [6]], [[1], [3], [5], [7]], - [[8], [10], [12], [14]], [[9], [11], [13], - [15]]]) - - # NOTE: output_tensor != input_tensor.reshape((4, 4, -1)) - with self.cached_session(): - variables.global_variables_initializer().run() - result = periodic_resample(input_tensor, desired_shape).eval() - # input_tensor[0, 0, 0] == result[0, 0, 0] - # input_tensor[0, 0, 1] == result[1, 0, 0] - # input_tensor[0, 0, 2] == result[0, 1, 0] - # input_tensor[0, 0, 3] == result[1, 1, 0] - self.assertAllEqual(result, output_tensor) - - def testPeriodicResampleBasic4D(self): - - input_tensor = numpy.arange(2 * 2 * 2 * 8).reshape((2, 2, 2, 8)) - desired_shape = numpy.array([4, 4, 4, None]) - output_tensor = numpy.array( - [[[[0], [4], [8], [12]], [[2], [6], [10], [14]], - [[16], [20], [24], [28]], [[18], [22], [26], [30]]], - [[[1], [5], [9], [13]], [[3], [7], [11], [15]], [[17], [21], [25], - [29]], - [[19], [23], [27], - [31]]], [[[32], [36], [40], [44]], [[34], [38], [42], [46]], - [[48], [52], [56], [60]], [[50], [54], [58], [62]]], - [[[33], [37], [41], [45]], [[35], [39], [43], [47]], - [[49], [53], [57], [61]], [[51], [55], [59], [63]]]]) - - # NOTE: output_tensor != input_tensor.reshape((4, 4, 4, -1)) - with self.cached_session(): - variables.global_variables_initializer().run() - result = periodic_resample(input_tensor, desired_shape).eval() - self.assertAllEqual(result, output_tensor) - - def testPeriodicResampleErrors(self): - input_tensor = numpy.zeros(shape=[1, 2, 2, 4]) - with self.cached_session(): - with self.assertRaisesWithPredicateMatch( - errors_impl.InvalidArgumentError, - 'Dimension 3 input tensor has size 4, desired shape has size 1'): - periodic_resample(input_tensor, [None, 4, 4, 1]).eval() - with self.assertRaisesWithPredicateMatch( - errors_impl.InvalidArgumentError, - '4, to be the same as the length of the desired shape, 3'): - periodic_resample(input_tensor, [None, 4, 4]).eval() - - def testPeriodicResampleGradient(self): - desired_shape = numpy.array([4, 4, None]) - result_shape = (4, 4, 1) - input_shape = (2, 2, 4) - with self.cached_session() as sess: - x = array_ops.placeholder(dtypes.float32, shape=input_shape) - output = periodic_resample(x, desired_shape) - error = gradient_checker.compute_gradient_error( - x, input_shape, output, result_shape) - self.assertLess(error, 1e-4) - - def testPeriodicResampleShapeInference(self): - with self.cached_session() as sess: - # Case 1: output shape can be fully inferreed. - x = array_ops.placeholder(dtypes.float32, shape=(2, 2, 4)) - output = periodic_resample(x, [4, 4, None]) - self.assertEqual(output.shape, [4, 4, 1]) - # Case 2: output shape can not be inferred - report desired shape. - x = array_ops.placeholder(dtypes.float32, shape=(2, 2, None)) - output = periodic_resample(x, [4, 4, None]) - self.assertTrue(output.shape.is_compatible_with([4, 4, None])) - self.assertEqual(output.shape[2].value, None) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/periodic_resample/python/ops/periodic_resample_op.py b/tensorflow/contrib/periodic_resample/python/ops/periodic_resample_op.py deleted file mode 100644 index 470e300ccbe..00000000000 --- a/tensorflow/contrib/periodic_resample/python/ops/periodic_resample_op.py +++ /dev/null @@ -1,37 +0,0 @@ -# ============================================================================= -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.contrib.periodic_resample.python.ops import gen_periodic_resample_op - -from tensorflow.contrib.periodic_resample.python.ops.gen_periodic_resample_op import periodic_resample, periodic_resample_op_grad - -from tensorflow.contrib.util import loader -from tensorflow.python.framework import ops -from tensorflow.python.platform import resource_loader -# pylint: enable=unused-import - -_periodic_resample_op = loader.load_op_library( - resource_loader.get_path_to_datafile('_periodic_resample_op.so')) - -@ops.RegisterGradient("PeriodicResample") -def _periodic_resample_grad_cc(op, grad): - return periodic_resample_op_grad( - grad, op.inputs[0].shape, op.get_attr('shape')) diff --git a/tensorflow/contrib/pi_examples/.gitignore b/tensorflow/contrib/pi_examples/.gitignore deleted file mode 100644 index 8e0b1c6a024..00000000000 --- a/tensorflow/contrib/pi_examples/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -tensorflow_inception_graph.pb -imagenet_comp_graph_label_strings.txt -tensorflow_inception_stripped.pb -*/gen/ diff --git a/tensorflow/contrib/pi_examples/README.md b/tensorflow/contrib/pi_examples/README.md deleted file mode 100644 index 177357bca64..00000000000 --- a/tensorflow/contrib/pi_examples/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# TensorFlow Raspberry Pi Examples - -This folder contains examples of how to build applications for the Raspberry Pi using TensorFlow. - -## Building the Examples - - - Follow the Raspberry Pi section of the instructions at [tensorflow/contrib/makefile](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/makefile) to compile a static library containing the core TensorFlow code. - - - Install libjpeg, so we can load image files: - -``` -sudo apt-get install -y libjpeg-dev -``` - - - To download the example model you'll need, run these commands: - -```bash -curl https://storage.googleapis.com/download.tensorflow.org/models/inception_dec_2015_stripped.zip \ --o /tmp/inception_dec_2015_stripped.zip -unzip /tmp/inception_dec_2015_stripped.zip \ --d tensorflow/contrib/pi_examples/label_image/data/ -``` - - - From the root of the TensorFlow source tree, run `make -f tensorflow/contrib/pi_examples/label_image/Makefile` to build a basic example. - -## Usage - -Run `tensorflow/contrib/pi_examples/label_image/gen/bin/label_image` to try out image labeling with the default Grace Hopper image. You should several lines of output, with "Military Uniform" shown as the top result, something like this: - -```bash -I tensorflow/contrib/pi_examples/label_image/label_image.cc:384] Running model succeeded! -I tensorflow/contrib/pi_examples/label_image/label_image.cc:284] military uniform (866): 0.624293 -I tensorflow/contrib/pi_examples/label_image/label_image.cc:284] suit (794): 0.0473981 -I tensorflow/contrib/pi_examples/label_image/label_image.cc:284] academic gown (896): 0.0280926 -I tensorflow/contrib/pi_examples/label_image/label_image.cc:284] bolo tie (940): 0.0156956 -I tensorflow/contrib/pi_examples/label_image/label_image.cc:284] bearskin (849): 0.0143348 -``` - -Once you've verified that is working, you can supply your own images with `--image=your_image.jpg`, or even with graphs you've trained yourself with the TensorFlow for Poets tutorial using `--graph=your_graph.pb --input=Mul:0 --output=final_result:0`. - -## Camera Example - -Once you have the simple example running, you can try out a more complex version that -reads frames from a camera attached to the Pi. You'll need to install and set up your -camera module first. The example uses Video4Linux, so you'll need to install that first. -Here's some commands I found necessary to get that set up, and I found more information -at this blog post: http://www.richardmudhar.com/blog/2015/02/raspberry-pi-camera-and-motion-out-of-the-box-sparrowcam/ - -``` -sudo bash -c "echo 'bcm2835-v4l2' >> /etc/modules" -sudo apt-get install libv4l-dev -``` - -Once that's working, run the following commands to build and run the camera example: - -```bash -make -f tensorflow/contrib/pi_examples/camera/Makefile -tensorflow/contrib/pi_examples/camera/gen/bin/camera -``` - -You should see it looping over camera frames as they come in, and printing the top labels -to the command line. This is a great starting point for all sorts of fun image recognition -applications, especially when you combine it with a custom model you've built using -something like the TensorFlow for Poets tutorial. - -The example is designed to work with the Flite speech synthesis tool, so that your Pi -can speak any labels that have a high enough score. To enable this, just install the -Flite package and then pipe the output of the binary you've built, like this: - -``` -sudo apt-get install flite -tensorflow/contrib/pi_examples/camera/gen/bin/camera | xargs -n 1 flite -t -``` diff --git a/tensorflow/contrib/pi_examples/camera/Makefile b/tensorflow/contrib/pi_examples/camera/Makefile deleted file mode 100644 index b354c03b6e5..00000000000 --- a/tensorflow/contrib/pi_examples/camera/Makefile +++ /dev/null @@ -1,85 +0,0 @@ -# This Makefile compiles the label_image example for the Raspberry Pi. -# See tensorflow/contrib/pi_examples/README.md for full build instructions. - -# Find where we're running from, so we can store generated files here. -SCRIPT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) - -# The location of the tensorflow/contrib/makefile directory. -TFMAKEFILE_DIR := $(SCRIPT_DIR)/../../makefile - -# Where compiled objects are stored. -GENDIR := $(SCRIPT_DIR)/gen/ -OBJDIR := $(GENDIR)obj/ -LIBDIR := $(GENDIR)lib/ -BINDIR := $(GENDIR)bin/ - -# The expected locations of the TensorFlow library. -TFLIBDIR := $(TFMAKEFILE_DIR)/gen/lib -TFLIBS := $(TFLIBDIR)/libtensorflow-core.a - -# Where the downloads have been stored. -DOWNLOADSDIR := $(TFMAKEFILE_DIR)/downloads - -# The location of the compiled protobuf headers generated by TensorFlow. -PBTGENDIR := $(TFMAKEFILE_DIR)/gen/proto_text/ -PROTOGENDIR := $(TFMAKEFILE_DIR)/gen/proto/ - -# The name of the output program we're compiling. -EXECUTABLE_NAME := $(BINDIR)/camera - -# Settings for the target compiler. -CXX := gcc -OPTFLAGS := -O0 -CXXFLAGS := --std=c++11 $(OPTFLAGS) -LDFLAGS := \ --L/usr/local/lib \ --L$(TFLIBDIR) \ --Wl,--no-whole-archive -INCLUDES := \ --I/usr/local/include \ --I. \ --I$(DOWNLOADSDIR) \ --I$(DOWNLOADSDIR)/eigen/ \ --I$(PROTOGENDIR) \ --I$(PBTGENDIR) -LIBS := \ --Wl,--allow-multiple-definition \ --Wl,--whole-archive \ --ltensorflow-core \ --Wl,--no-whole-archive \ --lstdc++ \ --lprotobuf \ --lv4l2 \ --ldl \ --lpthread \ --lm \ --ljpeg \ --lz -LIBFLAGS := - -EXECUTABLE_SRCS := tensorflow/contrib/pi_examples/camera/camera.cc - -# File names of the intermediate files target compilation generates. -EXECUTABLE_OBJS := $(addprefix $(OBJDIR), $(EXECUTABLE_SRCS:.cc=.o)) - -.PHONY: clean - -# The target that's compiled if there's no command-line arguments. -all: $(EXECUTABLE_NAME) - -# Rules for target compilation. - -$(EXECUTABLE_NAME): $(EXECUTABLE_OBJS) $(TFLIBS) - @mkdir -p $(dir $@) - $(CXX) $(CXXFLAGS) $(INCLUDES) \ - -o $(EXECUTABLE_NAME) $(EXECUTABLE_OBJS) \ - $(LIBFLAGS) $(LIB_PATH) $(LDFLAGS) $(LIBS) - -# Matches on C++ source files. -$(OBJDIR)%.o: %.cc - @mkdir -p $(dir $@) - $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ - -# Gets rid of all generated files. -clean: - rm -rf $(GENDIR) diff --git a/tensorflow/contrib/pi_examples/camera/camera.cc b/tensorflow/contrib/pi_examples/camera/camera.cc deleted file mode 100644 index 8110185ea8d..00000000000 --- a/tensorflow/contrib/pi_examples/camera/camera.cc +++ /dev/null @@ -1,538 +0,0 @@ -/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Full build instructions are at tensorflow/contrib/pi_examples/README.md. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tensorflow/core/framework/graph.pb.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/graph/default_device.h" -#include "tensorflow/core/graph/graph_def_builder.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/lib/strings/stringprintf.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/public/session.h" -#include "tensorflow/core/util/command_line_flags.h" - -// These are all common classes it's handy to reference with no namespace. -using tensorflow::Flag; -using tensorflow::int32; -using tensorflow::Status; -using tensorflow::string; -using tensorflow::Tensor; - -// Used to store the memory-mapped buffers we use for capture. -struct CameraBuffer { - void* start; - size_t length; -}; - -// Wrapper around camera command sending. -Status SendCameraCommand(int fh, int request, void* arg) { - int r; - do { - r = v4l2_ioctl(fh, request, arg); - } while (r == -1 && ((errno == EINTR) || (errno == EAGAIN))); - if (r == -1) { - LOG(ERROR) << "SendCameraCommand error " << errno << " (" << strerror(errno) - << ")"; - return tensorflow::errors::Unknown("SendCameraCommand error ", errno, - strerror(errno)); - } - return Status::OK(); -} - -Status OpenCamera(int* camera_handle) { - const char* dev_name = "/dev/video0"; - int fd = v4l2_open(dev_name, O_RDWR | O_NONBLOCK, 0); - if (fd < 0) { - LOG(ERROR) << "Cannot open camera device"; - return tensorflow::errors::NotFound("V4L2 camera device not found"); - } - *camera_handle = fd; - return Status::OK(); -} - -Status CloseCamera(int camera_handle) { - v4l2_close(camera_handle); - return Status::OK(); -} - -Status SetCameraFormat(int camera_handle, int wanted_width, int wanted_height) { - struct v4l2_format fmt; - memset(&fmt, 0, sizeof(fmt)); - fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - fmt.fmt.pix.width = wanted_width; - fmt.fmt.pix.height = wanted_height; - fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_RGB24; - fmt.fmt.pix.field = V4L2_FIELD_INTERLACED; - Status set_format_status = - SendCameraCommand(camera_handle, VIDIOC_S_FMT, &fmt); - if (!set_format_status.ok()) { - LOG(ERROR) << "Setting format failed with " << set_format_status; - return set_format_status; - } - if (fmt.fmt.pix.pixelformat != V4L2_PIX_FMT_RGB24) { - LOG(ERROR) << "Libv4l didn't accept RGB24 format. Can't proceed."; - return tensorflow::errors::Unknown("Libv4l didn't accept RGB24 format"); - } - if ((fmt.fmt.pix.width != wanted_width) || - (fmt.fmt.pix.height != wanted_height)) { - LOG(WARNING) << "Warning: driver is sending image at " << fmt.fmt.pix.width - << "x" << fmt.fmt.pix.height; - } - return Status::OK(); -} - -Status StartCameraCapture(int camera_handle, int buffer_count, - CameraBuffer** buffers) { - struct v4l2_requestbuffers req; - memset(&req, 0, sizeof(req)); - req.count = buffer_count; - req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - req.memory = V4L2_MEMORY_MMAP; - Status request_buffers_status = - SendCameraCommand(camera_handle, VIDIOC_REQBUFS, &req); - if (!request_buffers_status.ok()) { - LOG(ERROR) << "Request buffers failed with " << request_buffers_status; - return request_buffers_status; - } - - *buffers = (CameraBuffer*)(calloc(buffer_count, sizeof(*buffers))); - for (int n_buffers = 0; n_buffers < buffer_count; ++n_buffers) { - struct v4l2_buffer buf; - memset(&buf, 0, sizeof(buf)); - buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - buf.memory = V4L2_MEMORY_MMAP; - buf.index = n_buffers; - Status query_buffer_status = - SendCameraCommand(camera_handle, VIDIOC_QUERYBUF, &buf); - if (!query_buffer_status.ok()) { - LOG(ERROR) << "Query buffer failed with " << query_buffer_status; - return query_buffer_status; - } - (*buffers)[n_buffers].length = buf.length; - (*buffers)[n_buffers].start = - v4l2_mmap(NULL, buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, - camera_handle, buf.m.offset); - - if (MAP_FAILED == (*buffers)[n_buffers].start) { - LOG(ERROR) << "Memory-mapping buffer failed"; - return tensorflow::errors::Unknown("Memory-mapping buffer failed"); - } - } - - for (int i = 0; i < buffer_count; ++i) { - struct v4l2_buffer buf; - memset(&buf, 0, sizeof(buf)); - buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - buf.memory = V4L2_MEMORY_MMAP; - buf.index = i; - Status set_buffer_status = - SendCameraCommand(camera_handle, VIDIOC_QBUF, &buf); - if (!set_buffer_status.ok()) { - LOG(ERROR) << "Set buffer failed with " << set_buffer_status; - return set_buffer_status; - } - } - - enum v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - Status stream_on_status = - SendCameraCommand(camera_handle, VIDIOC_STREAMON, &type); - if (!stream_on_status.ok()) { - LOG(ERROR) << "Turning stream on failed with " << stream_on_status; - return stream_on_status; - } - return Status::OK(); -} - -Status EndCameraCapture(int camera_handle, CameraBuffer* buffers, - int buffer_count) { - enum v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - Status stream_off_status = - SendCameraCommand(camera_handle, VIDIOC_STREAMOFF, &type); - if (!stream_off_status.ok()) { - LOG(ERROR) << "Turning stream off failed with " << stream_off_status; - return stream_off_status; - } - for (int i = 0; i < buffer_count; ++i) - v4l2_munmap(buffers[i].start, buffers[i].length); - return Status::OK(); -} - -Status CaptureNextFrame(int camera_handle, CameraBuffer* buffers, - uint8_t** frame_data, int* frame_data_size, - v4l2_buffer* buf) { - int r; - do { - fd_set fds; - FD_ZERO(&fds); - FD_SET(camera_handle, &fds); - struct timeval tv; - tv.tv_sec = 2; - tv.tv_usec = 0; - r = select(camera_handle + 1, &fds, NULL, NULL, &tv); - } while ((r == -1 && (errno = EINTR))); - if (r == -1) { - LOG(ERROR) << "select() failed while waiting for the camera with " << errno; - return tensorflow::errors::Unknown( - "CaptureCameraFrame: select() failed with", errno); - } - - memset(buf, 0, sizeof(*buf)); - buf->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - buf->memory = V4L2_MEMORY_MMAP; - Status get_buffer_status = - SendCameraCommand(camera_handle, VIDIOC_DQBUF, buf); - if (!get_buffer_status.ok()) { - LOG(ERROR) << "Get buffer failed with " << get_buffer_status; - return get_buffer_status; - } - - *frame_data = static_cast(buffers[buf->index].start); - *frame_data_size = buf->bytesused; - - return Status::OK(); -} - -Status ReleaseFrame(int camera_handle, v4l2_buffer* buf) { - Status release_buffer_status = - SendCameraCommand(camera_handle, VIDIOC_QBUF, buf); - if (!release_buffer_status.ok()) { - LOG(ERROR) << "Release buffer failed with " << release_buffer_status; - return release_buffer_status; - } -} - -// Reads a model graph definition from disk, and creates a session object you -// can use to run it. -Status LoadGraph(string graph_file_name, - std::unique_ptr* session) { - tensorflow::GraphDef graph_def; - Status load_graph_status = - ReadBinaryProto(tensorflow::Env::Default(), graph_file_name, &graph_def); - if (!load_graph_status.ok()) { - return tensorflow::errors::NotFound("Failed to load compute graph at '", - graph_file_name, "'"); - } - session->reset(tensorflow::NewSession(tensorflow::SessionOptions())); - Status session_create_status = (*session)->Create(graph_def); - if (!session_create_status.ok()) { - return session_create_status; - } - return Status::OK(); -} - -// Analyzes the output of the Inception graph to retrieve the highest scores and -// their positions in the tensor, which correspond to categories. -Status GetTopLabels(const std::vector& outputs, int how_many_labels, - Tensor* out_indices, Tensor* out_scores) { - const Tensor& unsorted_scores_tensor = outputs[0]; - auto unsorted_scores_flat = unsorted_scores_tensor.flat(); - std::vector> scores; - for (int i = 0; i < unsorted_scores_flat.size(); ++i) { - scores.push_back(std::pair({i, unsorted_scores_flat(i)})); - } - std::sort(scores.begin(), scores.end(), - [](const std::pair& left, - const std::pair& right) { - return left.second > right.second; - }); - scores.resize(how_many_labels); - Tensor sorted_indices(tensorflow::DT_INT32, {scores.size()}); - Tensor sorted_scores(tensorflow::DT_FLOAT, {scores.size()}); - for (int i = 0; i < scores.size(); ++i) { - sorted_indices.flat()(i) = scores[i].first; - sorted_scores.flat()(i) = scores[i].second; - } - *out_indices = sorted_indices; - *out_scores = sorted_scores; - return Status::OK(); -} - -// Takes a file name, and loads a list of labels from it, one per line, and -// returns a vector of the strings. It pads with empty strings so the length -// of the result is a multiple of 16, because our model expects that. -Status ReadLabelsFile(string file_name, std::vector* result, - size_t* found_label_count) { - std::ifstream file(file_name); - if (!file) { - return tensorflow::errors::NotFound("Labels file ", file_name, - " not found."); - } - result->clear(); - string line; - while (std::getline(file, line)) { - result->push_back(line); - } - *found_label_count = result->size(); - const int padding = 16; - while (result->size() % padding) { - result->emplace_back(); - } - return Status::OK(); -} - -// Given the output of a model run, and the name of a file containing the labels -// this prints out the top five highest-scoring values. -Status PrintTopLabels(const std::vector& outputs, - const std::vector& labels, int label_count, - float print_threshold) { - const int how_many_labels = std::min(5, static_cast(label_count)); - Tensor indices; - Tensor scores; - TF_RETURN_IF_ERROR(GetTopLabels(outputs, how_many_labels, &indices, &scores)); - tensorflow::TTypes::Flat scores_flat = scores.flat(); - tensorflow::TTypes::Flat indices_flat = indices.flat(); - for (int pos = 0; pos < how_many_labels; ++pos) { - const int label_index = indices_flat(pos); - const float score = scores_flat(pos); - LOG(INFO) << labels[label_index] << " (" << label_index << "): " << score; - // Print the top label to stdout if it's above a threshold. - if ((pos == 0) && (score > print_threshold)) { - std::cout << labels[label_index] << std::endl; - } - } - return Status::OK(); -} - -// Given an image buffer, resize it to the requested size, and then scale the -// values as desired. -Status TensorFromFrame(uint8_t* image_data, int image_width, int image_height, - int image_channels, const int wanted_height, - const int wanted_width, const float input_mean, - const float input_std, - std::vector* out_tensors) { - const int wanted_channels = 3; - if (image_channels < wanted_channels) { - return tensorflow::errors::FailedPrecondition( - "Image needs to have at least ", wanted_channels, " but only has ", - image_channels); - } - // In these loops, we convert the eight-bit data in the image into float, - // resize it using bilinear filtering, and scale it numerically to the float - // range that the model expects (given by input_mean and input_std). - tensorflow::Tensor image_tensor( - tensorflow::DT_FLOAT, - tensorflow::TensorShape( - {1, wanted_height, wanted_width, wanted_channels})); - auto image_tensor_mapped = image_tensor.tensor(); - tensorflow::uint8* in = image_data; - float* out = image_tensor_mapped.data(); - const size_t image_rowlen = image_width * image_channels; - const float width_scale = static_cast(image_width) / wanted_width; - const float height_scale = static_cast(image_height) / wanted_height; - for (int y = 0; y < wanted_height; ++y) { - const float in_y = y * height_scale; - const int top_y_index = static_cast(floorf(in_y)); - const int bottom_y_index = - std::min(static_cast(ceilf(in_y)), (image_height - 1)); - const float y_lerp = in_y - top_y_index; - tensorflow::uint8* in_top_row = in + (top_y_index * image_rowlen); - tensorflow::uint8* in_bottom_row = in + (bottom_y_index * image_rowlen); - float* out_row = out + (y * wanted_width * wanted_channels); - for (int x = 0; x < wanted_width; ++x) { - const float in_x = x * width_scale; - const int left_x_index = static_cast(floorf(in_x)); - const int right_x_index = - std::min(static_cast(ceilf(in_x)), (image_width - 1)); - tensorflow::uint8* in_top_left_pixel = - in_top_row + (left_x_index * wanted_channels); - tensorflow::uint8* in_top_right_pixel = - in_top_row + (right_x_index * wanted_channels); - tensorflow::uint8* in_bottom_left_pixel = - in_bottom_row + (left_x_index * wanted_channels); - tensorflow::uint8* in_bottom_right_pixel = - in_bottom_row + (right_x_index * wanted_channels); - const float x_lerp = in_x - left_x_index; - float* out_pixel = out_row + (x * wanted_channels); - for (int c = 0; c < wanted_channels; ++c) { - const float top_left((in_top_left_pixel[c] - input_mean) / input_std); - const float top_right((in_top_right_pixel[c] - input_mean) / input_std); - const float bottom_left((in_bottom_left_pixel[c] - input_mean) / - input_std); - const float bottom_right((in_bottom_right_pixel[c] - input_mean) / - input_std); - const float top = top_left + (top_right - top_left) * x_lerp; - const float bottom = - bottom_left + (bottom_right - bottom_left) * x_lerp; - out_pixel[c] = top + (bottom - top) * y_lerp; - } - } - } - - out_tensors->push_back(image_tensor); - return Status::OK(); -} - -int main(int argc, char** argv) { - string graph = - "tensorflow/contrib/pi_examples/label_image/data/" - "tensorflow_inception_stripped.pb"; - string labels_file_name = - "tensorflow/contrib/pi_examples/label_image/data/" - "imagenet_comp_graph_label_strings.txt"; - int32 input_width = 299; - int32 input_height = 299; - int32 input_mean = 128; - int32 input_std = 128; - string input_layer = "Mul"; - string output_layer = "softmax"; - int32 video_width = 640; - int32 video_height = 480; - int print_threshold = 50; - string root_dir = ""; - std::vector flag_list = { - Flag("graph", &graph, "graph file name"), - Flag("labels", &labels_file_name, "labels file name"), - Flag("input_width", &input_width, "image input width"), - Flag("input_height", &input_height, "image input height"), - Flag("input_mean", &input_mean, "transformed mean of input pixels"), - Flag("input_std", &input_std, "transformed std dev of input pixels"), - Flag("input_layer", &input_layer, "input layer name"), - Flag("output_layer", &output_layer, "output layer name"), - Flag("video_width", &video_width, "video width expected from device"), - Flag("video_height", &video_height, "video height expected from device"), - Flag("print_threshold", &print_threshold, - "print labels with scoe exceeding this"), - Flag("root_dir", &root_dir, - "interpret graph file name relative to this directory")}; - string usage = tensorflow::Flags::Usage(argv[0], flag_list); - const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); - - if (!parse_result || argc != 1) { - LOG(ERROR) << "\n" << usage; - return -1; - } - - // First we load and initialize the model. - std::unique_ptr session; - string graph_path = tensorflow::io::JoinPath(root_dir, graph); - Status load_graph_status = LoadGraph(graph_path, &session); - if (!load_graph_status.ok()) { - LOG(ERROR) << load_graph_status; - return -1; - } - - std::vector labels; - size_t label_count; - Status read_labels_status = - ReadLabelsFile(labels_file_name, &labels, &label_count); - if (!read_labels_status.ok()) { - LOG(ERROR) << read_labels_status; - return -1; - } - - int camera_handle; - Status open_status = OpenCamera(&camera_handle); - if (!open_status.ok()) { - LOG(ERROR) << "OpenCamera failed with " << open_status; - return -1; - } - - Status format_status = - SetCameraFormat(camera_handle, video_width, video_height); - if (!format_status.ok()) { - LOG(ERROR) << "SetCameraFormat failed with " << format_status; - return -1; - } - - const int how_many_buffers = 2; - CameraBuffer* buffers; - Status start_capture_status = - StartCameraCapture(camera_handle, how_many_buffers, &buffers); - if (!start_capture_status.ok()) { - LOG(ERROR) << "StartCameraCapture failed with " << start_capture_status; - return -1; - } - - for (int i = 0; i < 200; i++) { - uint8_t* frame_data; - int frame_data_size; - v4l2_buffer buf; - Status capture_next_status = CaptureNextFrame( - camera_handle, buffers, &frame_data, &frame_data_size, &buf); - if (!capture_next_status.ok()) { - LOG(ERROR) << "CaptureNextFrame failed with " << capture_next_status; - return -1; - } - - std::vector resized_tensors; - Status tensor_from_frame_status = - TensorFromFrame(frame_data, video_width, video_height, 3, input_height, - input_width, input_mean, input_std, &resized_tensors); - if (!tensor_from_frame_status.ok()) { - LOG(ERROR) << tensor_from_frame_status; - return -1; - } - const Tensor& resized_tensor = resized_tensors[0]; - - Status release_frame_status = ReleaseFrame(camera_handle, &buf); - if (!release_frame_status.ok()) { - LOG(ERROR) << "ReleaseFrame failed with " << release_frame_status; - return -1; - } - - // Actually run the image through the model. - std::vector outputs; - Status run_status = session->Run({{input_layer, resized_tensor}}, - {output_layer}, {}, &outputs); - if (!run_status.ok()) { - LOG(ERROR) << "Running model failed: " << run_status; - return -1; - } - - // Do something interesting with the results we've generated. - Status print_status = - PrintTopLabels(outputs, labels, label_count, print_threshold * 0.01f); - if (!print_status.ok()) { - LOG(ERROR) << "Running print failed: " << print_status; - return -1; - } - } - - Status end_capture_status = - EndCameraCapture(camera_handle, buffers, how_many_buffers); - if (!end_capture_status.ok()) { - LOG(ERROR) << "EndCameraCapture failed with " << end_capture_status; - return -1; - } - - Status close_status = CloseCamera(camera_handle); - if (!close_status.ok()) { - LOG(ERROR) << "CloseCamera failed with " << open_status; - return -1; - } - - return 0; -} diff --git a/tensorflow/contrib/pi_examples/label_image/Makefile b/tensorflow/contrib/pi_examples/label_image/Makefile deleted file mode 100644 index 58fbd18dc3a..00000000000 --- a/tensorflow/contrib/pi_examples/label_image/Makefile +++ /dev/null @@ -1,87 +0,0 @@ -# This Makefile compiles the label_image example for the Raspberry Pi. -# See tensorflow/contrib/pi_examples/README.md for full build instructions. - -# Find where we're running from, so we can store generated files here. -SCRIPT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) - -# The location of the tensorflow/contrib/makefile directory. -TFMAKEFILE_DIR := $(SCRIPT_DIR)/../../makefile - -# Where compiled objects are stored. -GENDIR := $(SCRIPT_DIR)/gen/ -OBJDIR := $(GENDIR)obj/ -LIBDIR := $(GENDIR)lib/ -BINDIR := $(GENDIR)bin/ - -# The expected locations of the TensorFlow library. -TFLIBDIR := $(TFMAKEFILE_DIR)/gen/lib -TFLIBS := $(TFLIBDIR)/libtensorflow-core.a - -# Where the downloads have been stored. -DOWNLOADSDIR := $(TFMAKEFILE_DIR)/downloads - -# The location of the compiled protobuf headers generated by TensorFlow. -PBTGENDIR := $(TFMAKEFILE_DIR)/gen/proto_text/ -PROTOGENDIR := $(TFMAKEFILE_DIR)/gen/proto/ - -# The name of the output program we're compiling. -EXECUTABLE_NAME := $(BINDIR)/label_image - -# Settings for the target compiler. -CXX := gcc -OPTFLAGS := -O0 -CXXFLAGS := --std=c++11 $(OPTFLAGS) -LDFLAGS := \ --L/usr/local/lib \ --L$(TFLIBDIR) \ --L$(DOWNLOADSDIR)/nsync/builds/default.linux.c++11/ \ --Wl,--no-whole-archive -INCLUDES := \ --I/usr/local/include \ --I. \ --I$(DOWNLOADSDIR) \ --I$(DOWNLOADSDIR)/eigen/ \ --I$(DOWNLOADSDIR)/absl/ \ --I$(PROTOGENDIR) \ --I$(PBTGENDIR) -LIBS := \ --Wl,--allow-multiple-definition \ --Wl,--whole-archive \ --ltensorflow-core \ --Wl,--no-whole-archive \ --lstdc++ \ --lprotobuf \ --lnsync \ --ldl \ --lpthread \ --lm \ --ljpeg \ --lz -LIBFLAGS := - -EXECUTABLE_SRCS := tensorflow/contrib/pi_examples/label_image/label_image.cc - -# File names of the intermediate files target compilation generates. -EXECUTABLE_OBJS := $(addprefix $(OBJDIR), $(EXECUTABLE_SRCS:.cc=.o)) - -.PHONY: clean - -# The target that's compiled if there's no command-line arguments. -all: $(EXECUTABLE_NAME) - -# Rules for target compilation. - -$(EXECUTABLE_NAME): $(EXECUTABLE_OBJS) $(TFLIBS) - @mkdir -p $(dir $@) - $(CXX) $(CXXFLAGS) $(INCLUDES) \ - -o $(EXECUTABLE_NAME) $(EXECUTABLE_OBJS) \ - $(LIBFLAGS) $(LIB_PATH) $(LDFLAGS) $(LIBS) - -# Matches on C++ source files. -$(OBJDIR)%.o: %.cc - @mkdir -p $(dir $@) - $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ - -# Gets rid of all generated files. -clean: - rm -rf $(GENDIR) diff --git a/tensorflow/contrib/pi_examples/label_image/data/grace_hopper.jpg b/tensorflow/contrib/pi_examples/label_image/data/grace_hopper.jpg deleted file mode 100644 index 478720d6694..00000000000 Binary files a/tensorflow/contrib/pi_examples/label_image/data/grace_hopper.jpg and /dev/null differ diff --git a/tensorflow/contrib/pi_examples/label_image/label_image.cc b/tensorflow/contrib/pi_examples/label_image/label_image.cc deleted file mode 100644 index 97a6e69ac03..00000000000 --- a/tensorflow/contrib/pi_examples/label_image/label_image.cc +++ /dev/null @@ -1,410 +0,0 @@ -/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// A minimal but useful C++ example showing how to load an Imagenet-style object -// recognition TensorFlow model, prepare input images for it, run them through -// the graph, and interpret the results. -// -// It has been stripped down from the tensorflow/examples/label_image sample -// code to remove features and ops not included in the mobile/embedded core -// library available on the Raspberry Pi. -// -// Full build instructions are at tensorflow/contrib/pi_examples/README.md. - -#include -#include -#include - -#include -#include - -#include "tensorflow/core/framework/graph.pb.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/graph/default_device.h" -#include "tensorflow/core/graph/graph_def_builder.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/lib/strings/stringprintf.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/public/session.h" -#include "tensorflow/core/util/command_line_flags.h" - -// These are all common classes it's handy to reference with no namespace. -using tensorflow::Flag; -using tensorflow::int32; -using tensorflow::Status; -using tensorflow::string; -using tensorflow::Tensor; - -// Takes a file name, and loads a list of labels from it, one per line, and -// returns a vector of the strings. It pads with empty strings so the length -// of the result is a multiple of 16, because our model expects that. -Status ReadLabelsFile(string file_name, std::vector* result, - size_t* found_label_count) { - std::ifstream file(file_name); - if (!file) { - return tensorflow::errors::NotFound("Labels file ", file_name, - " not found."); - } - result->clear(); - string line; - while (std::getline(file, line)) { - result->push_back(line); - } - *found_label_count = result->size(); - const int padding = 16; - while (result->size() % padding) { - result->emplace_back(); - } - return Status::OK(); -} - -// Error handling for JPEG decoding. -void CatchError(j_common_ptr cinfo) { - (*cinfo->err->output_message)(cinfo); - jmp_buf* jpeg_jmpbuf = reinterpret_cast(cinfo->client_data); - jpeg_destroy(cinfo); - longjmp(*jpeg_jmpbuf, 1); -} - -// Decompresses a JPEG file from disk. -Status LoadJpegFile(string file_name, std::vector* data, - int* width, int* height, int* channels) { - struct jpeg_decompress_struct cinfo; - FILE* infile; - JSAMPARRAY buffer; - int row_stride; - - if ((infile = fopen(file_name.c_str(), "rb")) == NULL) { - LOG(ERROR) << "Can't open " << file_name; - return tensorflow::errors::NotFound("JPEG file ", file_name, " not found"); - } - - struct jpeg_error_mgr jerr; - jmp_buf jpeg_jmpbuf; // recovery point in case of error - cinfo.err = jpeg_std_error(&jerr); - cinfo.client_data = &jpeg_jmpbuf; - jerr.error_exit = CatchError; - if (setjmp(jpeg_jmpbuf)) { - fclose(infile); - return tensorflow::errors::Unknown("JPEG decoding failed"); - } - - jpeg_create_decompress(&cinfo); - jpeg_stdio_src(&cinfo, infile); - jpeg_read_header(&cinfo, TRUE); - jpeg_start_decompress(&cinfo); - *width = cinfo.output_width; - *height = cinfo.output_height; - *channels = cinfo.output_components; - data->resize((*height) * (*width) * (*channels)); - - row_stride = cinfo.output_width * cinfo.output_components; - buffer = (*cinfo.mem->alloc_sarray)((j_common_ptr)&cinfo, JPOOL_IMAGE, - row_stride, 1); - while (cinfo.output_scanline < cinfo.output_height) { - tensorflow::uint8* row_address = - &((*data)[cinfo.output_scanline * row_stride]); - jpeg_read_scanlines(&cinfo, buffer, 1); - memcpy(row_address, buffer[0], row_stride); - } - - jpeg_finish_decompress(&cinfo); - jpeg_destroy_decompress(&cinfo); - fclose(infile); - return Status::OK(); -} - -// Given an image file name, read in the data, try to decode it as an image, -// resize it to the requested size, and then scale the values as desired. -Status ReadTensorFromImageFile(string file_name, const int wanted_height, - const int wanted_width, const float input_mean, - const float input_std, - std::vector* out_tensors) { - std::vector image_data; - int image_width; - int image_height; - int image_channels; - TF_RETURN_IF_ERROR(LoadJpegFile(file_name, &image_data, &image_width, - &image_height, &image_channels)); - LOG(INFO) << "Loaded JPEG: " << image_width << "x" << image_height << "x" - << image_channels; - const int wanted_channels = 3; - if (image_channels < wanted_channels) { - return tensorflow::errors::FailedPrecondition( - "Image needs to have at least ", wanted_channels, " but only has ", - image_channels); - } - // In these loops, we convert the eight-bit data in the image into float, - // resize it using bilinear filtering, and scale it numerically to the float - // range that the model expects (given by input_mean and input_std). - tensorflow::Tensor image_tensor( - tensorflow::DT_FLOAT, - tensorflow::TensorShape( - {1, wanted_height, wanted_width, wanted_channels})); - auto image_tensor_mapped = image_tensor.tensor(); - tensorflow::uint8* in = image_data.data(); - float* out = image_tensor_mapped.data(); - const size_t image_rowlen = image_width * image_channels; - const float width_scale = static_cast(image_width) / wanted_width; - const float height_scale = static_cast(image_height) / wanted_height; - for (int y = 0; y < wanted_height; ++y) { - const float in_y = y * height_scale; - const int top_y_index = static_cast(floorf(in_y)); - const int bottom_y_index = - std::min(static_cast(ceilf(in_y)), (image_height - 1)); - const float y_lerp = in_y - top_y_index; - tensorflow::uint8* in_top_row = in + (top_y_index * image_rowlen); - tensorflow::uint8* in_bottom_row = in + (bottom_y_index * image_rowlen); - float* out_row = out + (y * wanted_width * wanted_channels); - for (int x = 0; x < wanted_width; ++x) { - const float in_x = x * width_scale; - const int left_x_index = static_cast(floorf(in_x)); - const int right_x_index = - std::min(static_cast(ceilf(in_x)), (image_width - 1)); - tensorflow::uint8* in_top_left_pixel = - in_top_row + (left_x_index * wanted_channels); - tensorflow::uint8* in_top_right_pixel = - in_top_row + (right_x_index * wanted_channels); - tensorflow::uint8* in_bottom_left_pixel = - in_bottom_row + (left_x_index * wanted_channels); - tensorflow::uint8* in_bottom_right_pixel = - in_bottom_row + (right_x_index * wanted_channels); - const float x_lerp = in_x - left_x_index; - float* out_pixel = out_row + (x * wanted_channels); - for (int c = 0; c < wanted_channels; ++c) { - const float top_left((in_top_left_pixel[c] - input_mean) / input_std); - const float top_right((in_top_right_pixel[c] - input_mean) / input_std); - const float bottom_left((in_bottom_left_pixel[c] - input_mean) / - input_std); - const float bottom_right((in_bottom_right_pixel[c] - input_mean) / - input_std); - const float top = top_left + (top_right - top_left) * x_lerp; - const float bottom = - bottom_left + (bottom_right - bottom_left) * x_lerp; - out_pixel[c] = top + (bottom - top) * y_lerp; - } - } - } - - out_tensors->push_back(image_tensor); - return Status::OK(); -} - -// Reads a model graph definition from disk, and creates a session object you -// can use to run it. -Status LoadGraph(string graph_file_name, - std::unique_ptr* session) { - tensorflow::GraphDef graph_def; - Status load_graph_status = - ReadBinaryProto(tensorflow::Env::Default(), graph_file_name, &graph_def); - if (!load_graph_status.ok()) { - return tensorflow::errors::NotFound("Failed to load compute graph at '", - graph_file_name, "'"); - } - session->reset(tensorflow::NewSession(tensorflow::SessionOptions())); - Status session_create_status = (*session)->Create(graph_def); - if (!session_create_status.ok()) { - return session_create_status; - } - return Status::OK(); -} - -// Analyzes the output of the Inception graph to retrieve the highest scores and -// their positions in the tensor, which correspond to categories. -Status GetTopLabels(const std::vector& outputs, int how_many_labels, - Tensor* out_indices, Tensor* out_scores) { - const Tensor& unsorted_scores_tensor = outputs[0]; - auto unsorted_scores_flat = unsorted_scores_tensor.flat(); - std::vector> scores; - for (int i = 0; i < unsorted_scores_flat.size(); ++i) { - scores.push_back(std::pair({i, unsorted_scores_flat(i)})); - } - std::sort(scores.begin(), scores.end(), - [](const std::pair& left, - const std::pair& right) { - return left.second > right.second; - }); - scores.resize(how_many_labels); - Tensor sorted_indices(tensorflow::DT_INT32, {scores.size()}); - Tensor sorted_scores(tensorflow::DT_FLOAT, {scores.size()}); - for (int i = 0; i < scores.size(); ++i) { - sorted_indices.flat()(i) = scores[i].first; - sorted_scores.flat()(i) = scores[i].second; - } - *out_indices = sorted_indices; - *out_scores = sorted_scores; - return Status::OK(); -} - -// Given the output of a model run, and the name of a file containing the labels -// this prints out the top five highest-scoring values. -Status PrintTopLabels(const std::vector& outputs, - string labels_file_name) { - std::vector labels; - size_t label_count; - Status read_labels_status = - ReadLabelsFile(labels_file_name, &labels, &label_count); - if (!read_labels_status.ok()) { - LOG(ERROR) << read_labels_status; - return read_labels_status; - } - const int how_many_labels = std::min(5, static_cast(label_count)); - Tensor indices; - Tensor scores; - TF_RETURN_IF_ERROR(GetTopLabels(outputs, how_many_labels, &indices, &scores)); - tensorflow::TTypes::Flat scores_flat = scores.flat(); - tensorflow::TTypes::Flat indices_flat = indices.flat(); - for (int pos = 0; pos < how_many_labels; ++pos) { - const int label_index = indices_flat(pos); - const float score = scores_flat(pos); - LOG(INFO) << labels[label_index] << " (" << label_index << "): " << score; - } - return Status::OK(); -} - -// This is a testing function that returns whether the top label index is the -// one that's expected. -Status CheckTopLabel(const std::vector& outputs, int expected, - bool* is_expected) { - *is_expected = false; - Tensor indices; - Tensor scores; - const int how_many_labels = 1; - TF_RETURN_IF_ERROR(GetTopLabels(outputs, how_many_labels, &indices, &scores)); - tensorflow::TTypes::Flat indices_flat = indices.flat(); - if (indices_flat(0) != expected) { - LOG(ERROR) << "Expected label #" << expected << " but got #" - << indices_flat(0); - *is_expected = false; - } else { - *is_expected = true; - } - return Status::OK(); -} - -int main(int argc, char* argv[]) { - // These are the command-line flags the program can understand. - // They define where the graph and input data is located, and what kind of - // input the model expects. If you train your own model, or use something - // other than GoogLeNet you'll need to update these. - string image = - "tensorflow/contrib/pi_examples/label_image/data/" - "grace_hopper.jpg"; - string graph = - "tensorflow/contrib/pi_examples/label_image/data/" - "tensorflow_inception_stripped.pb"; - string labels = - "tensorflow/contrib/pi_examples/label_image/data/" - "imagenet_comp_graph_label_strings.txt"; - int32 input_width = 299; - int32 input_height = 299; - int32 input_mean = 128; - int32 input_std = 128; - string input_layer = "Mul"; - string output_layer = "softmax"; - bool self_test = false; - string root_dir = ""; - std::vector flag_list = { - Flag("image", &image, "image to be processed"), - Flag("graph", &graph, "graph to be executed"), - Flag("labels", &labels, "name of file containing labels"), - Flag("input_width", &input_width, "resize image to this width in pixels"), - Flag("input_height", &input_height, - "resize image to this height in pixels"), - Flag("input_mean", &input_mean, "scale pixel values to this mean"), - Flag("input_std", &input_std, "scale pixel values to this std deviation"), - Flag("input_layer", &input_layer, "name of input layer"), - Flag("output_layer", &output_layer, "name of output layer"), - Flag("self_test", &self_test, "run a self test"), - Flag("root_dir", &root_dir, - "interpret image and graph file names relative to this directory"), - }; - string usage = tensorflow::Flags::Usage(argv[0], flag_list); - const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); - if (!parse_result) { - LOG(ERROR) << "\n" << usage; - return -1; - } - - // We need to call this to set up global state for TensorFlow. - tensorflow::port::InitMain(usage.c_str(), &argc, &argv); - if (argc > 1) { - LOG(ERROR) << "Unknown argument " << argv[1] << "\n" << usage; - return -1; - } - - // First we load and initialize the model. - std::unique_ptr session; - string graph_path = tensorflow::io::JoinPath(root_dir, graph); - Status load_graph_status = LoadGraph(graph_path, &session); - if (!load_graph_status.ok()) { - LOG(ERROR) << load_graph_status; - return -1; - } - - // Get the image from disk as a float array of numbers, resized and normalized - // to the specifications the main graph expects. - std::vector resized_tensors; - string image_path = tensorflow::io::JoinPath(root_dir, image); - Status read_tensor_status = - ReadTensorFromImageFile(image_path, input_height, input_width, input_mean, - input_std, &resized_tensors); - if (!read_tensor_status.ok()) { - LOG(ERROR) << read_tensor_status; - return -1; - } - const Tensor& resized_tensor = resized_tensors[0]; - - // Actually run the image through the model. - std::vector outputs; - Status run_status = session->Run({{input_layer, resized_tensor}}, - {output_layer}, {}, &outputs); - if (!run_status.ok()) { - LOG(ERROR) << "Running model failed: " << run_status; - return -1; - } else { - LOG(INFO) << "Running model succeeded!"; - } - - // This is for automated testing to make sure we get the expected result with - // the default settings. We know that label 866 (military uniform) should be - // the top label for the Admiral Hopper image. - if (self_test) { - bool expected_matches; - Status check_status = CheckTopLabel(outputs, 866, &expected_matches); - if (!check_status.ok()) { - LOG(ERROR) << "Running check failed: " << check_status; - return -1; - } - if (!expected_matches) { - LOG(ERROR) << "Self-test failed!"; - return -1; - } - } - - // Do something interesting with the results we've generated. - Status print_status = PrintTopLabels(outputs, labels); - if (!print_status.ok()) { - LOG(ERROR) << "Running print failed: " << print_status; - return -1; - } - - return 0; -} diff --git a/tensorflow/contrib/predictor/BUILD b/tensorflow/contrib/predictor/BUILD deleted file mode 100644 index a32d3b3dffc..00000000000 --- a/tensorflow/contrib/predictor/BUILD +++ /dev/null @@ -1,172 +0,0 @@ -# `Predictor` classes provide an interface for efficient, repeated inference. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow/contrib/predictor:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "predictor", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":predictor_factories", - "//tensorflow/python:util", - ], -) - -py_library( - name = "predictor_factories", - srcs = ["predictor_factories.py"], - srcs_version = "PY2AND3", - deps = [ - ":contrib_estimator_predictor", - ":core_estimator_predictor", - ":saved_model_predictor", - "//tensorflow/python/estimator:estimator_py", - ], -) - -py_library( - name = "base_predictor", - srcs = ["predictor.py"], - srcs_version = "PY2AND3", - deps = ["@six_archive//:six"], -) - -py_library( - name = "saved_model_predictor", - srcs = ["saved_model_predictor.py"], - srcs_version = "PY2AND3", - visibility = ["//learning/brain/contrib/learn/tpu:__subpackages__"], - deps = [ - ":base_predictor", - "//tensorflow/contrib/saved_model:saved_model_py", - "//tensorflow/python:framework_ops", - "//tensorflow/python:session", - "//tensorflow/python/saved_model:loader", - "//tensorflow/python/saved_model:signature_constants", - ], -) - -py_library( - name = "core_estimator_predictor", - srcs = ["core_estimator_predictor.py"], - srcs_version = "PY2AND3", - deps = [ - ":base_predictor", - "//tensorflow/python:framework_ops", - "//tensorflow/python:training", - "//tensorflow/python/estimator:estimator_py", - "//tensorflow/python/saved_model:signature_constants", - ], -) - -py_library( - name = "contrib_estimator_predictor", - srcs = ["contrib_estimator_predictor.py"], - srcs_version = "PY2AND3", - deps = [ - ":base_predictor", - "//tensorflow/contrib/learn", - "//tensorflow/python:framework_ops", - "//tensorflow/python:training", - ], -) - -py_library( - name = "testing_common", - srcs = ["testing_common.py"], - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [ - "//tensorflow/contrib/learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python/estimator:estimator_py", - "//tensorflow/python/saved_model:signature_constants", - ], -) - -# Transitive dependencies of this target will be included in the pip package. -py_library( - name = "predictor_pip", - visibility = ["//visibility:public"], - deps = [ - ":contrib_estimator_predictor", - ":core_estimator_predictor", - ":saved_model_predictor", - ], -) - -py_test( - name = "saved_model_predictor_test", - srcs = ["saved_model_predictor_test.py"], - data = [":test_export_dir"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [ - ":saved_model_predictor", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_ops", - "//tensorflow/python/saved_model:signature_def_utils", - "//third_party/py/numpy", - ], -) - -py_test( - name = "predictor_factories_test", - srcs = ["predictor_factories_test.py"], - data = [":test_export_dir"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [ - ":predictor_factories", - ":testing_common", - ], -) - -py_test( - name = "core_estimator_predictor_test", - srcs = ["core_estimator_predictor_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [ - ":core_estimator_predictor", - ":testing_common", - "//tensorflow/python:client_testlib", - "//third_party/py/numpy", - ], -) - -py_test( - name = "contrib_estimator_predictor_test", - srcs = ["contrib_estimator_predictor_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [ - ":contrib_estimator_predictor", - ":testing_common", - "//tensorflow/python:client_testlib", - "//third_party/py/numpy", - ], -) - -filegroup( - name = "test_export_dir", - srcs = glob(["test_export_dir/**/*"]), - tags = ["no_pip"], -) diff --git a/tensorflow/contrib/predictor/README.md b/tensorflow/contrib/predictor/README.md deleted file mode 100644 index 16cdcf3e706..00000000000 --- a/tensorflow/contrib/predictor/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# Predictors - -The `Predictor` classes provide a simple interface for performing repeated, -efficient inference. A `Predictor` can be constructed from a `SavedModel` on -disk, a `tf.Estimator` or a `tf.contrib.Estimator`. - -To facilitate the examples below, let's define a trivial `Estimator` that just -calculates a sum: - -```python -def model_fn(features, labels, mode): - z = tf.add(features['x'], features['y'], name='z') - return tf.contrib.learn.ModelFnOps( - mode, {'z': z}, loss=tf.constant(0.0), train_op=tf.no_op()) - -estimator = tf.contrib.learn.Estimator(model_fn=model_fn) -``` - -We can then construct a `Predictor` in two different ways. - -## `Predictor` from a `SavedModel` - -Given a trained `Estimator`, we first export a `SavedModel`: - -```python -def serving_input_fn(): - x = tf.placeholder(dtype=tf.float32, shape=[None], name='x') - y = tf.placeholder(dtype=tf.float32, shape=[None], name='y') - - features = {'x': x, 'y': y} - return tf.contrib.learn.utils.input_fn_utils.InputFnOps( - features, None, default_inputs=features) - -saved_model_dir = estimator.export_savedmodel(my_export_dir, serving_input_fn) -``` - -We can then construct a `Predictor` as follows: - -```python -saved_model_predictor = predictor.from_saved_model(export_dir='test_export_dir') -output_dict = saved_model_predictor({'x': [1.0], 'y': [5.2]}) -# output_dict == {'sum': [6.2]} -``` - -By specifying a signature definition, we can feed and fetch any `Tensor`s in -the `Graph`. In this example, we feed and fetch the same `Tensor`, `z`: - -```python -inputs = outputs = {'z': tf.TensorInfo( - name='z:0', - dtype=types_pb2.DT_FLOAT, - tensor_shape=tensor_shape_pb2.TensorShapeProto())} - -signature_def = tf.saved_model.signature_def_utils.build_signature_def( - inputs=inputs, - outputs=outputs, - method_name='tensorflow/serving/regress') - -trivial_predictor = predictor.from_saved_model( - export_dir=saved_model_dir, - signature_def=signature_def) - -output_dict = trivial_predictor({'z': [32.]}) -# output_dict == {'z': [32.]} -``` - -You can also specify input and output `Tensor`s by name using the `input_names` -and `output_names` keywords: - -```python -saved_model_predictor = predictor.from_saved_model( - export_dir=saved_model_dir, - input_names={'x': 'x:0', 'y': 'y:0'}, - outputs={'z': 'z:0'}) - -output_dict = saved_model_predictor({'x': [6.], 'y': [11.]}) -# output_dict == {'z': [17.]} -``` - -This functionality is particularly useful for performing encoding once, but -doing multiple decoding iterations with e.g. seq2seq models. - -## `Predictor` from an `Estimator` - -We can also construct a `Predictor` directly from an `Estimator`. Defining -`serving_input_fn` as above, - -```python -estimator_predictor = predictor.from_contrib_estimator( - estimator, serving_input_fn) -output_dict = sum_predictor({'x': [1., 2.], 'y': [3., 4.]}) -# output_dict == {'z': [4., 6.]} -``` - -Construction from a `tf.Estimator` is almost identical. - diff --git a/tensorflow/contrib/predictor/__init__.py b/tensorflow/contrib/predictor/__init__.py deleted file mode 100644 index 68146aea174..00000000000 --- a/tensorflow/contrib/predictor/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Modules for `Predictor`s. - -@@from_contrib_estimator -@@from_estimator -@@from_saved_model -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.predictor.predictor_factories import from_contrib_estimator -from tensorflow.contrib.predictor.predictor_factories import from_estimator -from tensorflow.contrib.predictor.predictor_factories import from_saved_model - -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/predictor/contrib_estimator_predictor.py b/tensorflow/contrib/predictor/contrib_estimator_predictor.py deleted file mode 100644 index c2166594e59..00000000000 --- a/tensorflow/contrib/predictor/contrib_estimator_predictor.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""A `Predictor constructed from a `tf.contrib.learn.Estimator`.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.utils import saved_model_export_utils -from tensorflow.contrib.predictor import predictor -from tensorflow.python.framework import ops -from tensorflow.python.training import checkpoint_management -from tensorflow.python.training import monitored_session - - -class ContribEstimatorPredictor(predictor.Predictor): - """A `Predictor constructed from a `tf.contrib.learn.Estimator`.""" - - def __init__(self, - estimator, - prediction_input_fn, - input_alternative_key=None, - output_alternative_key=None, - graph=None, - config=None): - """Initialize a `ContribEstimatorPredictor`. - - Args: - estimator: an instance of `tf.contrib.learn.Estimator`. - prediction_input_fn: a function that takes no arguments and returns an - instance of `InputFnOps`. - input_alternative_key: Optional. Specify the input alternative used for - prediction. - output_alternative_key: Specify the output alternative used for - prediction. Not needed for single-headed models but required for - multi-headed models. - graph: Optional. The Tensorflow `graph` in which prediction should be - done. - config: `ConfigProto` proto used to configure the session. - """ - self._graph = graph or ops.Graph() - with self._graph.as_default(): - input_fn_ops = prediction_input_fn() - # pylint: disable=protected-access - model_fn_ops = estimator._get_predict_ops(input_fn_ops.features) - # pylint: enable=protected-access - checkpoint_path = checkpoint_management.latest_checkpoint( - estimator.model_dir) - self._session = monitored_session.MonitoredSession( - session_creator=monitored_session.ChiefSessionCreator( - config=config, - checkpoint_filename_with_path=checkpoint_path)) - - input_alternative_key = ( - input_alternative_key or - saved_model_export_utils.DEFAULT_INPUT_ALTERNATIVE_KEY) - input_alternatives, _ = saved_model_export_utils.get_input_alternatives( - input_fn_ops) - self._feed_tensors = input_alternatives[input_alternative_key] - - (output_alternatives, - output_alternative_key) = saved_model_export_utils.get_output_alternatives( - model_fn_ops, output_alternative_key) - _, fetch_tensors = output_alternatives[output_alternative_key] - self._fetch_tensors = fetch_tensors diff --git a/tensorflow/contrib/predictor/contrib_estimator_predictor_test.py b/tensorflow/contrib/predictor/contrib_estimator_predictor_test.py deleted file mode 100644 index 4b97a52b1a3..00000000000 --- a/tensorflow/contrib/predictor/contrib_estimator_predictor_test.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for predictor.contrib_estimator_predictor.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tempfile -import numpy as np - -from tensorflow.contrib.predictor import contrib_estimator_predictor -from tensorflow.contrib.predictor import testing_common -from tensorflow.python.platform import test - - -KEYS_AND_OPS = (('sum', lambda x, y: x + y), - ('product', lambda x, y: x * y,), - ('difference', lambda x, y: x - y)) - - -class ContribEstimatorPredictorTest(test.TestCase): - """Test fixture for `ContribEstimatorPredictor`.""" - - def setUp(self): - model_dir = tempfile.mkdtemp() - self._estimator = testing_common.get_arithmetic_estimator( - core=False, model_dir=model_dir) - self._prediction_input_fn = testing_common.get_arithmetic_input_fn( - core=False, train=False) - - def testSpecifiedSignatureKey(self): - """Test prediction with spedicified signatures.""" - np.random.seed(1234) - for key, op in KEYS_AND_OPS: - x = np.random.rand() - y = np.random.rand() - expected_output = op(x, y) - - predictor = contrib_estimator_predictor.ContribEstimatorPredictor( - estimator=self._estimator, - prediction_input_fn=self._prediction_input_fn, - output_alternative_key=key) - output_tensor_name = predictor.fetch_tensors[key].name - self.assertRegexpMatches( - output_tensor_name, - key, - msg='Unexpected fetch tensor.') - output = predictor({'x': x, 'y': y})[key] - self.assertAlmostEqual( - expected_output, output, places=3, - msg='Failed for output key "{}." ' - 'Got output {} for x = {} and y = {}'.format( - key, output, x, y)) - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/predictor/core_estimator_predictor.py b/tensorflow/contrib/predictor/core_estimator_predictor.py deleted file mode 100644 index a725072e72d..00000000000 --- a/tensorflow/contrib/predictor/core_estimator_predictor.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""A `Predictor` constructed from an `learn.python.estimator.Estimator`.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.predictor import predictor -from tensorflow.python.estimator import model_fn -from tensorflow.python.framework import ops -from tensorflow.python.saved_model import signature_constants -from tensorflow.python.training import monitored_session - - -def _get_signature_def( - serving_input_receiver, estimator, output_key=None): - """Construct a `SignatureDef` proto.""" - if output_key is None: - output_key = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY - # pylint: disable=protected-access - estimator_spec = estimator.model_fn( - serving_input_receiver.features, None, model_fn.ModeKeys.PREDICT, - estimator.config) - # pylint: enable=protected-access - export_outputs = estimator_spec.export_outputs - export_output = export_outputs.get(output_key) - if export_output is None: - raise KeyError('output_key must be one of {}; got {}'.format( - export_outputs.keys(), output_key)) - return export_output.as_signature_def(serving_input_receiver.receiver_tensors) - - -class CoreEstimatorPredictor(predictor.Predictor): - """A `Predictor` constructed from an `learn.python.estimator.Estimator`.""" - - def __init__(self, - estimator, - serving_input_receiver_fn, - output_key=None, - graph=None, - config=None): - """Initialize a `CoreEstimatorPredictor`. - - Args: - estimator: an instance of `learn.python.estimator.Estimator`. - serving_input_receiver_fn: a function that takes no arguments and returns - an instance of `ServingInputReceiver` compatible with `estimator`. - output_key: Optional string specifying the export output to use. If - `None`, then `DEFAULT_SERVING_SIGNATURE_DEF_KEY` is used. - graph: Optional. The Tensorflow `graph` in which prediction should be - done. - config: `ConfigProto` proto used to configure the session. - """ - self._graph = graph or ops.Graph() - with self._graph.as_default(): - serving_input_receiver = serving_input_receiver_fn() - signature_def = _get_signature_def( - serving_input_receiver, estimator, output_key) - checkpoint_dir = estimator.model_dir - self._session = monitored_session.MonitoredSession( - session_creator=monitored_session.ChiefSessionCreator( - config=config, - checkpoint_dir=checkpoint_dir)) - - feed_tensor_info = signature_def.inputs - self._feed_tensors = {k: self._graph.get_tensor_by_name(v.name) - for k, v in feed_tensor_info.items()} - fetch_tensor_info = signature_def.outputs - self._fetch_tensors = {k: self._graph.get_tensor_by_name(v.name) - for k, v in fetch_tensor_info.items()} diff --git a/tensorflow/contrib/predictor/core_estimator_predictor_test.py b/tensorflow/contrib/predictor/core_estimator_predictor_test.py deleted file mode 100644 index 42210867944..00000000000 --- a/tensorflow/contrib/predictor/core_estimator_predictor_test.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for predictor.core_estimator_predictor.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tempfile -import numpy as np - -from tensorflow.contrib.predictor import core_estimator_predictor -from tensorflow.contrib.predictor import testing_common -from tensorflow.python.platform import test - - -KEYS_AND_OPS = (('sum', lambda x, y: x + y), - ('product', lambda x, y: x * y,), - ('difference', lambda x, y: x - y)) - - -class CoreEstimatorPredictorTest(test.TestCase): - """Test fixture for `CoreEstimatorPredictor`.""" - - def setUp(self): - model_dir = tempfile.mkdtemp() - self._estimator = testing_common.get_arithmetic_estimator( - core=True, model_dir=model_dir) - self._serving_input_receiver_fn = testing_common.get_arithmetic_input_fn( - core=True, train=False) - - def testDefault(self): - """Test prediction with default signature.""" - np.random.seed(1111) - x = np.random.rand() - y = np.random.rand() - predictor = core_estimator_predictor.CoreEstimatorPredictor( - estimator=self._estimator, - serving_input_receiver_fn=self._serving_input_receiver_fn) - output = predictor({'x': x, 'y': y})['sum'] - self.assertAlmostEqual(output, x + y, places=3) - - def testSpecifiedSignatureKey(self): - """Test prediction with spedicified signatures.""" - np.random.seed(1234) - for output_key, op in KEYS_AND_OPS: - x = np.random.rand() - y = np.random.rand() - expected_output = op(x, y) - - predictor = core_estimator_predictor.CoreEstimatorPredictor( - estimator=self._estimator, - serving_input_receiver_fn=self._serving_input_receiver_fn, - output_key=output_key) - output_tensor_name = predictor.fetch_tensors[output_key].name - self.assertRegexpMatches( - output_tensor_name, - output_key, - msg='Unexpected fetch tensor.') - output = predictor({'x': x, 'y': y})[output_key] - self.assertAlmostEqual( - expected_output, output, places=3, - msg='Failed for output key "{}." ' - 'Got output {} for x = {} and y = {}'.format( - output_key, output, x, y)) - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/predictor/predictor.py b/tensorflow/contrib/predictor/predictor.py deleted file mode 100644 index 28fa815684d..00000000000 --- a/tensorflow/contrib/predictor/predictor.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Abstract base class for all predictors.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc -import six - - -@six.add_metaclass(abc.ABCMeta) -class Predictor(object): - """Abstract base class for all predictors.""" - - @property - def graph(self): - return self._graph - - @property - def session(self): - return self._session - - @property - def feed_tensors(self): - return self._feed_tensors - - @property - def fetch_tensors(self): - return self._fetch_tensors - - def __repr__(self): - return '{} with feed tensors {} and fetch_tensors {}'.format( - type(self).__name__, self._feed_tensors, self._fetch_tensors) - - def __call__(self, input_dict): - """Returns predictions based on `input_dict`. - - Args: - input_dict: a `dict` mapping strings to numpy arrays. These keys - must match `self._feed_tensors.keys()`. - - Returns: - A `dict` mapping strings to numpy arrays. The keys match - `self.fetch_tensors.keys()`. - - Raises: - ValueError: `input_dict` does not match `feed_tensors`. - """ - # TODO(jamieas): make validation optional? - input_keys = set(input_dict.keys()) - expected_keys = set(self.feed_tensors.keys()) - unexpected_keys = input_keys - expected_keys - if unexpected_keys: - raise ValueError( - 'Got unexpected keys in input_dict: {}\nexpected: {}'.format( - unexpected_keys, expected_keys)) - - feed_dict = {} - for key in self.feed_tensors.keys(): - value = input_dict.get(key) - if value is not None: - feed_dict[self.feed_tensors[key]] = value - return self._session.run(fetches=self.fetch_tensors, feed_dict=feed_dict) diff --git a/tensorflow/contrib/predictor/predictor_factories.py b/tensorflow/contrib/predictor/predictor_factories.py deleted file mode 100644 index 7886744b3ce..00000000000 --- a/tensorflow/contrib/predictor/predictor_factories.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Factory functions for `Predictor`s.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.predictor import contrib_estimator_predictor -from tensorflow.contrib.predictor import core_estimator_predictor -from tensorflow.contrib.predictor import saved_model_predictor - -from tensorflow.contrib.learn.python.learn.estimators import estimator as contrib_estimator -from tensorflow.python.estimator import estimator as core_estimator - - -def from_contrib_estimator(estimator, - prediction_input_fn, - input_alternative_key=None, - output_alternative_key=None, - graph=None, - config=None): - """Constructs a `Predictor` from a `tf.contrib.learn.Estimator`. - - Args: - estimator: an instance of `tf.contrib.learn.Estimator`. - prediction_input_fn: a function that takes no arguments and returns an - instance of `InputFnOps`. - input_alternative_key: Optional. Specify the input alternative used for - prediction. - output_alternative_key: Specify the output alternative used for - prediction. Not needed for single-headed models but required for - multi-headed models. - graph: Optional. The Tensorflow `graph` in which prediction should be - done. - config: `ConfigProto` proto used to configure the session. - - Returns: - An initialized `Predictor`. - - Raises: - TypeError: if `estimator` is a core `Estimator` instead of a contrib - `Estimator`. - """ - if isinstance(estimator, core_estimator.Estimator): - raise TypeError('Expected estimator to be of type ' - 'tf.contrib.learn.Estimator, but got type ' - 'tf.python.estimator.Estimator. You likely want to call ' - 'from_estimator.') - return contrib_estimator_predictor.ContribEstimatorPredictor( - estimator, - prediction_input_fn, - input_alternative_key=input_alternative_key, - output_alternative_key=output_alternative_key, - graph=graph, - config=config) - - -def from_estimator(estimator, - serving_input_receiver_fn, - output_key=None, - graph=None, - config=None): - """Constructs a `Predictor` from a `tf.python.estimator.Estimator`. - - Args: - estimator: an instance of `learn.python.estimator.Estimator`. - serving_input_receiver_fn: a function that takes no arguments and returns - an instance of `ServingInputReceiver` compatible with `estimator`. - output_key: Optional string specifying the export output to use. If - `None`, then `DEFAULT_SERVING_SIGNATURE_DEF_KEY` is used. - graph: Optional. The Tensorflow `graph` in which prediction should be - done. - config: `ConfigProto` proto used to configure the session. - - Returns: - An initialized `Predictor`. - - Raises: - TypeError: if `estimator` is a contrib `Estimator` instead of a core - `Estimator`. - """ - if isinstance(estimator, contrib_estimator.Estimator): - raise TypeError('Expected estimator to be of type ' - 'tf.python.estimator.Estimator, but got type ' - 'tf.contrib.learn.Estimator. You likely want to call ' - 'from_contrib_estimator.') - return core_estimator_predictor.CoreEstimatorPredictor( - estimator, - serving_input_receiver_fn, - output_key=output_key, - graph=graph, - config=config) - - -def from_saved_model(export_dir, - signature_def_key=None, - signature_def=None, - input_names=None, - output_names=None, - tags=None, - graph=None, - config=None): - """Constructs a `Predictor` from a `SavedModel` on disk. - - Args: - export_dir: a path to a directory containing a `SavedModel`. - signature_def_key: Optional string specifying the signature to use. If - `None`, then `DEFAULT_SERVING_SIGNATURE_DEF_KEY` is used. Only one of - `signature_def_key` and `signature_def` - signature_def: A `SignatureDef` proto specifying the inputs and outputs - for prediction. Only one of `signature_def_key` and `signature_def` - should be specified. - input_names: A dictionary mapping strings to `Tensor`s in the `SavedModel` - that represent the input. The keys can be any string of the user's - choosing. - output_names: A dictionary mapping strings to `Tensor`s in the - `SavedModel` that represent the output. The keys can be any string of - the user's choosing. - tags: Optional. Tags that will be used to retrieve the correct - `SignatureDef`. Defaults to `DEFAULT_TAGS`. - graph: Optional. The Tensorflow `graph` in which prediction should be - done. - config: `ConfigProto` proto used to configure the session. - - Returns: - An initialized `Predictor`. - - Raises: - ValueError: More than one of `signature_def_key` and `signature_def` is - specified. - """ - return saved_model_predictor.SavedModelPredictor( - export_dir, - signature_def_key=signature_def_key, - signature_def=signature_def, - input_names=input_names, - output_names=output_names, - tags=tags, - graph=graph, - config=config) diff --git a/tensorflow/contrib/predictor/predictor_factories_test.py b/tensorflow/contrib/predictor/predictor_factories_test.py deleted file mode 100644 index a2ef1dc3af0..00000000000 --- a/tensorflow/contrib/predictor/predictor_factories_test.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for predictor.predictor_factories.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.predictor import predictor_factories -from tensorflow.contrib.predictor import testing_common -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.platform import test - -MODEL_DIR_NAME = 'contrib/predictor/test_export_dir' - - -class PredictorFactoriesTest(test.TestCase): - - @classmethod - def setUpClass(cls): - # Load a saved model exported from the arithmetic `Estimator`. - # See `testing_common.py`. - cls._export_dir = test.test_src_dir_path(MODEL_DIR_NAME) - - def testFromSavedModel(self): - """Test loading from_saved_model.""" - predictor_factories.from_saved_model(self._export_dir) - - def testFromSavedModelWithTags(self): - """Test loading from_saved_model with tags.""" - predictor_factories.from_saved_model(self._export_dir, tags='serve') - - def testFromSavedModelWithSessionConfig(self): - """Test loading from_saved_model with session config.""" - predictor_factories.from_saved_model( - self._export_dir, config=config_pb2.ConfigProto()) - - def testFromSavedModelWithBadTags(self): - """Test that loading fails for bad tags.""" - bad_tags_regex = ('.*? could not be found in SavedModel') - with self.assertRaisesRegexp(RuntimeError, bad_tags_regex): - predictor_factories.from_saved_model(self._export_dir, tags='bad_tag') - - def testFromContribEstimator(self): - estimator = testing_common.get_arithmetic_estimator(core=False) - input_fn = testing_common.get_arithmetic_input_fn(core=False) - predictor_factories.from_contrib_estimator( - estimator, input_fn, output_alternative_key='sum') - - def testFromContribEstimatorWithSessionConfig(self): - estimator = testing_common.get_arithmetic_estimator(core=False) - input_fn = testing_common.get_arithmetic_input_fn(core=False) - predictor_factories.from_contrib_estimator( - estimator, input_fn, output_alternative_key='sum', - config=config_pb2.ConfigProto()) - - def testFromContribEstimatorWithCoreEstimatorRaises(self): - estimator = testing_common.get_arithmetic_estimator(core=True) - input_fn = testing_common.get_arithmetic_input_fn(core=True) - with self.assertRaises(TypeError): - predictor_factories.from_contrib_estimator(estimator, input_fn) - - def testFromCoreEstimator(self): - estimator = testing_common.get_arithmetic_estimator(core=True) - input_fn = testing_common.get_arithmetic_input_fn(core=True) - predictor_factories.from_estimator(estimator, input_fn) - - def testFromCoreEstimatorWithSessionConfig(self): - estimator = testing_common.get_arithmetic_estimator(core=True) - input_fn = testing_common.get_arithmetic_input_fn(core=True) - predictor_factories.from_estimator( - estimator, input_fn, config=config_pb2.ConfigProto()) - - def testFromCoreEstimatorWithContribEstimatorRaises(self): - estimator = testing_common.get_arithmetic_estimator(core=False) - input_fn = testing_common.get_arithmetic_input_fn(core=False) - with self.assertRaises(TypeError): - predictor_factories.from_estimator(estimator, input_fn) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/predictor/saved_model_predictor.py b/tensorflow/contrib/predictor/saved_model_predictor.py deleted file mode 100644 index 03399396df5..00000000000 --- a/tensorflow/contrib/predictor/saved_model_predictor.py +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""A `Predictor` constructed from a `SavedModel`.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import logging - -from tensorflow.contrib.predictor import predictor -from tensorflow.contrib.saved_model.python.saved_model import reader -from tensorflow.python.client import session -from tensorflow.python.framework import ops -from tensorflow.python.saved_model import loader -from tensorflow.python.saved_model import signature_constants - - -DEFAULT_TAGS = 'serve' - -_DEFAULT_INPUT_ALTERNATIVE_FORMAT = 'default_input_alternative:{}' - - -def get_meta_graph_def(saved_model_dir, tags): - """Gets `MetaGraphDef` from a directory containing a `SavedModel`. - - Returns the `MetaGraphDef` for the given tag-set and SavedModel directory. - - Args: - saved_model_dir: Directory containing the SavedModel. - tags: Comma separated list of tags used to identify the correct - `MetaGraphDef`. - - Raises: - ValueError: An error when the given tags cannot be found. - - Returns: - A `MetaGraphDef` corresponding to the given tags. - """ - saved_model = reader.read_saved_model(saved_model_dir) - set_of_tags = set([tag.strip() for tag in tags.split(',')]) - for meta_graph_def in saved_model.meta_graphs: - if set(meta_graph_def.meta_info_def.tags) == set_of_tags: - return meta_graph_def - raise ValueError('Could not find MetaGraphDef with tags {}'.format(tags)) - - -def _get_signature_def(signature_def_key, export_dir, tags): - """Construct a `SignatureDef` proto.""" - signature_def_key = ( - signature_def_key or - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY) - - metagraph_def = get_meta_graph_def(export_dir, tags) - - try: - signature_def = metagraph_def.signature_def[signature_def_key] - except KeyError as e: - formatted_key = _DEFAULT_INPUT_ALTERNATIVE_FORMAT.format( - signature_def_key) - try: - signature_def = metagraph_def.signature_def[formatted_key] - except KeyError: - raise ValueError( - 'Got signature_def_key "{}". Available signatures are {}. ' - 'Original error:\n{}'.format( - signature_def_key, list(metagraph_def.signature_def), e)) - logging.warning('Could not find signature def "%s". ' - 'Using "%s" instead', signature_def_key, formatted_key) - return signature_def - - -def _check_signature_arguments(signature_def_key, - signature_def, - input_names, - output_names): - """Validates signature arguments for `SavedModelPredictor`.""" - signature_def_key_specified = signature_def_key is not None - signature_def_specified = signature_def is not None - input_names_specified = input_names is not None - output_names_specified = output_names is not None - if input_names_specified != output_names_specified: - raise ValueError( - 'input_names and output_names must both be specified or both be ' - 'unspecified.' - ) - - if (signature_def_key_specified + signature_def_specified + - input_names_specified > 1): - raise ValueError( - 'You must specify at most one of signature_def_key OR signature_def OR' - '(input_names AND output_names).' - ) - - -class SavedModelPredictor(predictor.Predictor): - """A `Predictor` constructed from a `SavedModel`.""" - - def __init__(self, - export_dir, - signature_def_key=None, - signature_def=None, - input_names=None, - output_names=None, - tags=None, - graph=None, - config=None): - """Initialize a `CoreEstimatorPredictor`. - - Args: - export_dir: a path to a directory containing a `SavedModel`. - signature_def_key: Optional string specifying the signature to use. If - `None`, then `DEFAULT_SERVING_SIGNATURE_DEF_KEY` is used. Only one of - `signature_def_key` and `signature_def` should be specified. - signature_def: A `SignatureDef` proto specifying the inputs and outputs - for prediction. Only one of `signature_def_key` and `signature_def` - should be specified. - input_names: A dictionary mapping strings to `Tensor`s in the `SavedModel` - that represent the input. The keys can be any string of the user's - choosing. - output_names: A dictionary mapping strings to `Tensor`s in the - `SavedModel` that represent the output. The keys can be any string of - the user's choosing. - tags: Optional. Comma separated list of tags that will be used to retrieve - the correct `SignatureDef`. Defaults to `DEFAULT_TAGS`. - graph: Optional. The Tensorflow `graph` in which prediction should be - done. - config: `ConfigProto` proto used to configure the session. - Raises: - ValueError: If more than one of signature_def_key OR signature_def OR - (input_names AND output_names) is specified. - """ - _check_signature_arguments( - signature_def_key, signature_def, input_names, output_names) - tags = tags or DEFAULT_TAGS - self._graph = graph or ops.Graph() - - with self._graph.as_default(): - self._session = session.Session(config=config) - loader.load(self._session, tags.split(','), export_dir) - - if input_names is None: - if signature_def is None: - signature_def = _get_signature_def(signature_def_key, export_dir, tags) - input_names = {k: v.name for k, v in signature_def.inputs.items()} - output_names = {k: v.name for k, v in signature_def.outputs.items()} - - self._feed_tensors = {k: self._graph.get_tensor_by_name(v) - for k, v in input_names.items()} - self._fetch_tensors = {k: self._graph.get_tensor_by_name(v) - for k, v in output_names.items()} diff --git a/tensorflow/contrib/predictor/saved_model_predictor_test.py b/tensorflow/contrib/predictor/saved_model_predictor_test.py deleted file mode 100644 index f40e2e73d99..00000000000 --- a/tensorflow/contrib/predictor/saved_model_predictor_test.py +++ /dev/null @@ -1,170 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for predictor.saved_model_predictor.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.predictor import saved_model_predictor -from tensorflow.core.framework import tensor_shape_pb2 -from tensorflow.core.framework import types_pb2 -from tensorflow.core.protobuf import meta_graph_pb2 -from tensorflow.python.framework import ops -from tensorflow.python.platform import test -from tensorflow.python.saved_model import signature_def_utils - - -KEYS_AND_OPS = (('sum', lambda x, y: x + y), - ('product', lambda x, y: x * y,), - ('difference', lambda x, y: x - y)) - -MODEL_DIR_NAME = 'contrib/predictor/test_export_dir' - - -class SavedModelPredictorTest(test.TestCase): - - @classmethod - def setUpClass(cls): - # Load a saved model exported from the arithmetic `Estimator`. - # See `testing_common.py`. - cls._export_dir = test.test_src_dir_path(MODEL_DIR_NAME) - - def testDefault(self): - """Test prediction with default signature.""" - np.random.seed(1111) - x = np.random.rand() - y = np.random.rand() - predictor = saved_model_predictor.SavedModelPredictor( - export_dir=self._export_dir) - output = predictor({'x': x, 'y': y})['outputs'] - self.assertAlmostEqual(output, x + y, places=3) - - def testSpecifiedSignatureKey(self): - """Test prediction with spedicified signature key.""" - np.random.seed(1234) - for signature_def_key, op in KEYS_AND_OPS: - x = np.random.rand() - y = np.random.rand() - expected_output = op(x, y) - - predictor = saved_model_predictor.SavedModelPredictor( - export_dir=self._export_dir, - signature_def_key=signature_def_key) - - output_tensor_name = predictor.fetch_tensors['outputs'].name - self.assertRegexpMatches( - output_tensor_name, - signature_def_key, - msg='Unexpected fetch tensor.') - - output = predictor({'x': x, 'y': y})['outputs'] - self.assertAlmostEqual( - expected_output, output, places=3, - msg='Failed for signature "{}." ' - 'Got output {} for x = {} and y = {}'.format( - signature_def_key, output, x, y)) - - def testSpecifiedSignature(self): - """Test prediction with spedicified signature definition.""" - np.random.seed(4444) - for key, op in KEYS_AND_OPS: - x = np.random.rand() - y = np.random.rand() - expected_output = op(x, y) - - inputs = { - 'x': meta_graph_pb2.TensorInfo( - name='inputs/x:0', - dtype=types_pb2.DT_FLOAT, - tensor_shape=tensor_shape_pb2.TensorShapeProto()), - 'y': meta_graph_pb2.TensorInfo( - name='inputs/y:0', - dtype=types_pb2.DT_FLOAT, - tensor_shape=tensor_shape_pb2.TensorShapeProto())} - outputs = { - key: meta_graph_pb2.TensorInfo( - name='outputs/{}:0'.format(key), - dtype=types_pb2.DT_FLOAT, - tensor_shape=tensor_shape_pb2.TensorShapeProto())} - signature_def = signature_def_utils.build_signature_def( - inputs=inputs, - outputs=outputs, - method_name='tensorflow/serving/regress') - predictor = saved_model_predictor.SavedModelPredictor( - export_dir=self._export_dir, - signature_def=signature_def) - - output_tensor_name = predictor.fetch_tensors[key].name - self.assertRegexpMatches( - output_tensor_name, - key, - msg='Unexpected fetch tensor.') - - output = predictor({'x': x, 'y': y})[key] - self.assertAlmostEqual( - expected_output, output, places=3, - msg='Failed for signature "{}". ' - 'Got output {} for x = {} and y = {}'.format(key, output, x, y)) - - def testSpecifiedTensors(self): - """Test prediction with spedicified `Tensor`s.""" - np.random.seed(987) - for key, op in KEYS_AND_OPS: - x = np.random.rand() - y = np.random.rand() - expected_output = op(x, y) - input_names = {'x': 'inputs/x:0', - 'y': 'inputs/y:0'} - output_names = {key: 'outputs/{}:0'.format(key)} - predictor = saved_model_predictor.SavedModelPredictor( - export_dir=self._export_dir, - input_names=input_names, - output_names=output_names) - - output_tensor_name = predictor.fetch_tensors[key].name - self.assertRegexpMatches( - output_tensor_name, - key, - msg='Unexpected fetch tensor.') - - output = predictor({'x': x, 'y': y})[key] - self.assertAlmostEqual( - expected_output, output, places=3, - msg='Failed for signature "{}". ' - 'Got output {} for x = {} and y = {}'.format(key, output, x, y)) - - def testBadTagsFail(self): - """Test that predictor construction fails for bad tags.""" - bad_tags_regex = ('.* could not be found in SavedModel') - with self.assertRaisesRegexp(RuntimeError, bad_tags_regex): - _ = saved_model_predictor.SavedModelPredictor( - export_dir=self._export_dir, - tags=('zomg, bad, tags')) - - def testSpecifiedGraph(self): - """Test that the predictor remembers a specified `Graph`.""" - g = ops.Graph() - predictor = saved_model_predictor.SavedModelPredictor( - export_dir=self._export_dir, - graph=g) - self.assertEqual(predictor.graph, g) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/predictor/test_export_dir/saved_model.pb b/tensorflow/contrib/predictor/test_export_dir/saved_model.pb deleted file mode 100644 index 9100fefb720..00000000000 Binary files a/tensorflow/contrib/predictor/test_export_dir/saved_model.pb and /dev/null differ diff --git a/tensorflow/contrib/predictor/test_export_dir/variables/variables.data-00000-of-00001 b/tensorflow/contrib/predictor/test_export_dir/variables/variables.data-00000-of-00001 deleted file mode 100644 index 1b1cb4d44c5..00000000000 Binary files a/tensorflow/contrib/predictor/test_export_dir/variables/variables.data-00000-of-00001 and /dev/null differ diff --git a/tensorflow/contrib/predictor/test_export_dir/variables/variables.index b/tensorflow/contrib/predictor/test_export_dir/variables/variables.index deleted file mode 100644 index dd32e9b71b3..00000000000 Binary files a/tensorflow/contrib/predictor/test_export_dir/variables/variables.index and /dev/null differ diff --git a/tensorflow/contrib/predictor/testing_common.py b/tensorflow/contrib/predictor/testing_common.py deleted file mode 100644 index 1767704b993..00000000000 --- a/tensorflow/contrib/predictor/testing_common.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Common code used for testing `Predictor`s.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import estimator as contrib_estimator -from tensorflow.contrib.learn.python.learn.estimators import model_fn as contrib_model_fn -from tensorflow.contrib.learn.python.learn.utils import input_fn_utils -from tensorflow.python.estimator import estimator as core_estimator -from tensorflow.python.estimator import model_fn -from tensorflow.python.estimator.export import export_lib -from tensorflow.python.estimator.export import export_output -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.saved_model import signature_constants - - -def get_arithmetic_estimator(core=True, model_dir=None): - """Returns an `Estimator` that performs basic arithmetic. - - Args: - core: if `True`, returns a `tensorflow.python.estimator.Estimator`. - Otherwise, returns a `tensorflow.contrib.learn.Estimator`. - model_dir: directory in which to export checkpoints and saved models. - Returns: - An `Estimator` that performs arithmetic operations on its inputs. - """ - def _model_fn(features, labels, mode): - _ = labels - x = features['x'] - y = features['y'] - with ops.name_scope('outputs'): - predictions = {'sum': math_ops.add(x, y, name='sum'), - 'product': math_ops.multiply(x, y, name='product'), - 'difference': math_ops.subtract(x, y, name='difference')} - if core: - export_outputs = {k: export_output.PredictOutput({k: v}) - for k, v in predictions.items()} - export_outputs[signature_constants. - DEFAULT_SERVING_SIGNATURE_DEF_KEY] = export_outputs['sum'] - return model_fn.EstimatorSpec(mode=mode, - predictions=predictions, - export_outputs=export_outputs, - loss=constant_op.constant(0), - train_op=control_flow_ops.no_op()) - else: - output_alternatives = {k: (constants.ProblemType.UNSPECIFIED, {k: v}) - for k, v in predictions.items()} - return contrib_model_fn.ModelFnOps( - mode=mode, - predictions=predictions, - output_alternatives=output_alternatives, - loss=constant_op.constant(0), - train_op=control_flow_ops.no_op()) - if core: - return core_estimator.Estimator(_model_fn) - else: - return contrib_estimator.Estimator(_model_fn, model_dir=model_dir) - - -def get_arithmetic_input_fn(core=True, train=False): - """Returns a input functions or serving input receiver function.""" - def _input_fn(): - with ops.name_scope('inputs'): - x = array_ops.placeholder_with_default(0.0, shape=[], name='x') - y = array_ops.placeholder_with_default(0.0, shape=[], name='y') - label = constant_op.constant(0.0) - features = {'x': x, 'y': y} - if core: - if train: - return features, label - return export_lib.ServingInputReceiver( - features=features, - receiver_tensors=features) - else: - if train: - return features, label - return input_fn_utils.InputFnOps( - features=features, - labels={}, - default_inputs=features) - return _input_fn diff --git a/tensorflow/contrib/proto/BUILD b/tensorflow/contrib/proto/BUILD deleted file mode 100644 index 402e3241966..00000000000 --- a/tensorflow/contrib/proto/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "tf_py_test") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "proto", - srcs = [ - "__init__.py", - ], - deps = [ - "//tensorflow/contrib/proto/python/ops:decode_proto_op_py", - "//tensorflow/contrib/proto/python/ops:encode_proto_op_py", - "//tensorflow/python:proto_ops", - ], -) - -tf_py_test( - name = "import_test", - srcs = ["import_test.py"], - additional_deps = [ - ":proto", - "//tensorflow/python:client_testlib", - ], -) diff --git a/tensorflow/contrib/proto/__init__.py b/tensorflow/contrib/proto/__init__.py deleted file mode 100644 index 1fe17324cd9..00000000000 --- a/tensorflow/contrib/proto/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops and modules related to proto. - -@@decode_proto -@@encode_proto -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops.proto_ops import decode_proto -from tensorflow.python.ops.proto_ops import encode_proto - -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/proto/import_test.py b/tensorflow/contrib/proto/import_test.py deleted file mode 100644 index 5da74a8c578..00000000000 --- a/tensorflow/contrib/proto/import_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Backwards compatibility tests for imports of tf.contrib.proto.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import proto -from tensorflow.python.platform import test - - -class ProtoImportTest(test.TestCase): - - def testImport(self): - self.assertTrue(proto.decode_proto) # Should be accessible - self.assertTrue(proto.encode_proto) # Should be accessible - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/proto/python/ops/BUILD b/tensorflow/contrib/proto/python/ops/BUILD deleted file mode 100644 index cc5d319be27..00000000000 --- a/tensorflow/contrib/proto/python/ops/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -# Placeholders for folks with old dependencies. -py_library( - name = "encode_proto_op_py", - srcs = ["encode_proto_op.py"], - deps = [ - "//tensorflow/python:proto_ops", - ], -) - -py_library( - name = "decode_proto_op_py", - srcs = ["decode_proto_op.py"], - deps = [ - "//tensorflow/python:proto_ops", - ], -) diff --git a/tensorflow/contrib/proto/python/ops/decode_proto_op.py b/tensorflow/contrib/proto/python/ops/decode_proto_op.py deleted file mode 100644 index 1347ebe3346..00000000000 --- a/tensorflow/contrib/proto/python/ops/decode_proto_op.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.python.ops.proto_ops import decode_proto diff --git a/tensorflow/contrib/proto/python/ops/encode_proto_op.py b/tensorflow/contrib/proto/python/ops/encode_proto_op.py deleted file mode 100644 index 6c1fcf68566..00000000000 --- a/tensorflow/contrib/proto/python/ops/encode_proto_op.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.python.ops.proto_ops import encode_proto diff --git a/tensorflow/contrib/quantization/BUILD b/tensorflow/contrib/quantization/BUILD deleted file mode 100644 index 0f5c3e969b6..00000000000 --- a/tensorflow/contrib/quantization/BUILD +++ /dev/null @@ -1,52 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -load( - "//tensorflow:tensorflow.bzl", - "tf_custom_op_library", # @unused - "tf_gen_op_wrapper_py", # @unused -) - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "quantization_py", - srcs = [ - "__init__.py", - "python/__init__.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":ops", - "//tensorflow/python:array_ops_gen", - ], -) - -py_library( - name = "ops", - srcs = [ - "python/array_ops.py", - "python/math_ops.py", - "python/nn_ops.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops_gen", - "//tensorflow/python:common_shapes", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops_gen", - "//tensorflow/python:nn_ops_gen", - ], -) - -filegroup( - name = "py_srcs", - data = glob([ - "**/*.py", - ]), -) diff --git a/tensorflow/contrib/quantization/README.md b/tensorflow/contrib/quantization/README.md deleted file mode 100644 index 826e8db2d3a..00000000000 --- a/tensorflow/contrib/quantization/README.md +++ /dev/null @@ -1,7 +0,0 @@ -The contrib/quantization package exposes a few TensorFlow quantization operations. - -If you are looking for quantized training rewrites that allow for training -quantized models that work with -[TensorFlow Lite](https://www.tensorflow.org/lite/), you should look at -the [contrib/quantize](https://www.tensorflow.org/api_docs/python/tf/contrib/quantize) -package. diff --git a/tensorflow/contrib/quantization/__init__.py b/tensorflow/contrib/quantization/__init__.py deleted file mode 100644 index bbc7d91e5a1..00000000000 --- a/tensorflow/contrib/quantization/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Ops for building quantized models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import,g-bad-import-order -from tensorflow.contrib.quantization.python import array_ops as quantized_array_ops -from tensorflow.contrib.quantization.python.math_ops import * -from tensorflow.contrib.quantization.python.nn_ops import * - -from tensorflow.python.ops import gen_array_ops as quantized_gen_array_ops -from tensorflow.python.ops.gen_array_ops import dequantize -from tensorflow.python.ops.gen_array_ops import quantize_v2 -from tensorflow.python.ops.gen_array_ops import quantized_concat -# pylint: enable=unused-import,wildcard-import,g-bad-import-order diff --git a/tensorflow/contrib/quantization/python/__init__.py b/tensorflow/contrib/quantization/python/__init__.py deleted file mode 100644 index 903a53d8418..00000000000 --- a/tensorflow/contrib/quantization/python/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A module containing TensorFlow ops whose API may change in the future.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.quantization.python.array_ops import * -from tensorflow.contrib.quantization.python.math_ops import * -from tensorflow.contrib.quantization.python.nn_ops import * -# pylint: enable=unused-import,wildcard-import diff --git a/tensorflow/contrib/quantization/python/array_ops.py b/tensorflow/contrib/quantization/python/array_ops.py deleted file mode 100644 index 98ade5defe4..00000000000 --- a/tensorflow/contrib/quantization/python/array_ops.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Quantized Array Operations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.python.ops import gen_array_ops as quantized_gen_array_ops -from tensorflow.python.ops.gen_array_ops import dequantize -from tensorflow.python.ops.gen_array_ops import quantize_v2 -from tensorflow.python.ops.gen_array_ops import quantized_concat -# pylint: enable=unused-import diff --git a/tensorflow/contrib/quantization/python/math_ops.py b/tensorflow/contrib/quantization/python/math_ops.py deleted file mode 100644 index 79aa1013967..00000000000 --- a/tensorflow/contrib/quantization/python/math_ops.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Quantized Math Operations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.python.framework import common_shapes -from tensorflow.python.framework import ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops.gen_math_ops import * -# pylint: enable=unused-import,wildcard-import diff --git a/tensorflow/contrib/quantization/python/nn_ops.py b/tensorflow/contrib/quantization/python/nn_ops.py deleted file mode 100644 index 501438e4ccc..00000000000 --- a/tensorflow/contrib/quantization/python/nn_ops.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Wrappers for primitive Neural Net (NN) Operations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.python.framework import common_shapes -from tensorflow.python.framework import ops -from tensorflow.python.ops import gen_nn_ops -from tensorflow.python.ops.gen_nn_ops import * -# pylint: enable=unused-import,wildcard-import diff --git a/tensorflow/contrib/quantize/BUILD b/tensorflow/contrib/quantize/BUILD deleted file mode 100644 index 16e6f2dc005..00000000000 --- a/tensorflow/contrib/quantize/BUILD +++ /dev/null @@ -1,268 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "common", - srcs = ["python/common.py"], - srcs_version = "PY2AND3", - deps = [], -) - -py_test( - name = "common_test", - size = "small", - srcs = ["python/common_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":common", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:session", - "//tensorflow/python:variable_scope", - ], -) - -py_library( - name = "graph_matcher", - srcs = [ - "python/graph_matcher.py", - ], - srcs_version = "PY2AND3", - deps = [], -) - -py_test( - name = "graph_matcher_test", - size = "small", - srcs = ["python/graph_matcher_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":graph_matcher", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], -) - -py_library( - name = "input_to_ops", - srcs = ["python/input_to_ops.py"], - srcs_version = "PY2AND3", - deps = [ - ":common", - ], -) - -py_test( - name = "input_to_ops_test", - size = "small", - srcs = ["python/input_to_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":input_to_ops", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], -) - -py_library( - name = "fold_batch_norms", - srcs = ["python/fold_batch_norms.py"], - srcs_version = "PY2AND3", - deps = [ - ":common", - ":graph_matcher", - ":input_to_ops", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:layers", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:nn_ops", - "//tensorflow/python:ops", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "fold_batch_norms_test", - srcs = ["python/fold_batch_norms_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":fold_batch_norms", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], -) - -py_library( - name = "quant_ops", - srcs = ["python/quant_ops.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - ], -) - -py_test( - name = "quant_ops_test", - size = "small", - srcs = ["python/quant_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":quant_ops", - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:partitioned_variables", - "//tensorflow/python:platform_test", - "//tensorflow/python:session", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -py_library( - name = "quantize", - srcs = ["python/quantize.py"], - srcs_version = "PY2AND3", - deps = [ - ":graph_matcher", - ":input_to_ops", - ":quant_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:training", - ], -) - -py_test( - name = "quantize_test", - size = "small", - srcs = ["python/quantize_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":quantize", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], -) - -py_test( - name = "quantize_parameterized_test", - size = "medium", - srcs = ["python/quantize_parameterized_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - # TODO(b/118839526): Re-enable msan test. - tags = [ - "nomsan", - ], - deps = [ - ":fold_batch_norms", - ":quantize", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - ], -) - -py_library( - name = "quantize_graph", - srcs = [ - "__init__.py", - "python/quantize_graph.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":fold_batch_norms", - ":quantize", - "//tensorflow/python:util", - ], -) - -py_test( - name = "quantize_graph_test", - size = "small", - srcs = ["python/quantize_graph_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":quantize_graph", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:training", - ], -) diff --git a/tensorflow/contrib/quantize/README.md b/tensorflow/contrib/quantize/README.md deleted file mode 100644 index b335e1af69b..00000000000 --- a/tensorflow/contrib/quantize/README.md +++ /dev/null @@ -1,164 +0,0 @@ -# Quantization-aware training - -Quantization-aware model training ensures that the forward pass matches precision -for both training and inference. There are two aspects to this: - -* Operator fusion at inference time are accurately modeled at training time. -* Quantization effects at inference are modeled at training time. - -For efficient inference, TensorFlow combines batch normalization with the preceding -convolutional and fully-connected layers prior to quantization by -[folding batch norm layers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/quantize/python/fold_batch_norms.py){:.external}. - -The quantization error is modeled using [fake quantization](../../api_guides/python/array_ops.md#Fake_quantization) -nodes to simulate the effect of quantization in the forward and backward passes. The -forward-pass models quantization, while the backward-pass models quantization as a -straight-through estimator. Both the forward- and backward-pass simulate the quantization -of weights and activations. Note that during back propagation, the parameters are -updated at high precision as this is needed to ensure sufficient precision in -accumulating tiny adjustments to the parameters. - - -Additionally, the minimum and maximum values for activations are determined -during training. This allows a model trained with quantization in the loop to be -converted to a fixed point inference model with little effort, eliminating the -need for a separate calibration step. - -Since it's difficult to add these fake quantization operations to all the -required locations in the model, there's a function available that rewrites the -training graph. To create a fake quantized training graph: - -```python -# Build forward pass of model. -loss = tf.losses.get_total_loss() - -# Call the training rewrite which rewrites the graph in-place with -# FakeQuantization nodes and folds batchnorm for training. It is -# often needed to fine tune a floating point model for quantization -# with this training tool. When training from scratch, quant_delay -# can be used to activate quantization after training to converge -# with the float graph, effectively fine-tuning the model. -g = tf.get_default_graph() -tf.contrib.quantize.create_training_graph(input_graph=g, - quant_delay=2000000) - -# Call backward pass optimizer as usual. -optimizer = tf.train.GradientDescentOptimizer(learning_rate) -optimizer.minimize(loss) -``` - -The rewritten *eval graph* is non-trivially different from the *training graph* -since the quantization ops affect the batch normalization step. Because of this, -we've added a separate rewrite for the *eval graph*: - -```python -# Build eval model -logits = tf.nn.softmax_cross_entropy_with_logits_v2(...) - -# Call the eval rewrite which rewrites the graph in-place with -# FakeQuantization nodes and fold batchnorm for eval. -g = tf.get_default_graph() -tf.contrib.quantize.create_eval_graph(input_graph=g) - -# Save the checkpoint and eval graph proto to disk for freezing -# and providing to TFLite. -with open(eval_graph_file, ‘w’) as f: - f.write(str(g.as_graph_def())) -saver = tf.train.Saver() -saver.save(sess, checkpoint_name) -``` - -Methods to rewrite the training and eval graphs are an active area of research -and experimentation. Although rewrites and quantized training might not work or -improve performance for all models, we are working to generalize these techniques. - - -## Generating fully-quantized models - -The previously demonstrated after-rewrite eval graph only *simulates* -quantization. To generate real fixed-point computations from a trained -quantization model, convert it to a fixed-point kernel. TensorFlow Lite supports -this conversion from the graph resulting from `create_eval_graph`. - -First, create a frozen graph that will be the input for the TensorFlow Lite -toolchain: - -``` -freeze_graph \ - --input_graph=eval_graph_def.pb \ - --input_checkpoint=checkpoint \ - --output_graph=frozen_eval_graph.pb --output_node_names=outputs -``` - -Provide this to the TensorFlow Lite Optimizing Converter (TOCO) to get a -fully-quantized TensorFlow Lite model: - -``` -toco \ - --input_file=frozen_eval_graph.pb \ - --output_file=tflite_model.tflite \ - --input_format=TENSORFLOW_GRAPHDEF --output_format=TFLITE \ - --inference_type=QUANTIZED_UINT8 \ - --input_shape="1,224, 224,3" \ - --input_array=input \ - --output_array=outputs \ - --std_value=127.5 --mean_value=127.5 -``` - -See the documentation for `tf.contrib.quantize` and [TensorFlow Lite](../../lite/). - - -## Quantized accuracy results - -The following are results of training some popular CNN models (Mobilenet-v1, -Mobilenet-v2, and Inception-v3) using this tool: - -
- - - - - - - - - - - - - - - - - - - - - - - - -
ModelTop-1 Accuracy:
Floating point		Top-1 Accuracy:
Fixed point: 8 bit weights and activations
Mobilenet-v1-128-0.250.4150.399
Mobilenet-v1-128-0.50.5630.549
Mobilenet-v1-128-0.750.6210.598
Mobilenet-v1-128-10.6520.64
Mobilenet-v1-160-0.250.4550.435
Mobilenet-v1-160-0.50.5910.577
Mobilenet-v1-160-0.750.6530.639
Mobilenet-v1-160-10.680.673
Mobilenet-v1-192-0.250.4770.458
Mobilenet-v1-192-0.50.6170.604
Mobilenet-v1-192-0.750.6720.662
Mobilenet-v1-192-10.70.69
Mobilenet-v1-224-0.250.4980.482
Mobilenet-v1-224-0.50.6330.622
Mobilenet-v1-224-0.750.6840.679
Mobilenet-v1-224-10.7090.697
Mobilenet-v2-224-10.7180.708
Inception_v30.780.775
-
- Table 1: Top-1 accuracy of floating point and fully quantized CNNs on Imagenet Validation dataset. -
-
- -Our pre-trained models are available in the -TensorFlow Lite model repository. The code used to generate -these models is available. - - - -These rewrites are an active area of research and experimentation, so the -rewrites and quantized training will likely not work across all models, though -we hope to work towards generalizing these techniques. - -[1] B.Jacob et al., "Quantization and Training of Neural Networks for Efficient -Integer-Arithmetic-Only Inference", https://arxiv.org/abs/1712.05877 - -[2] P.Gysel et al., "HARDWARE-ORIENTED APPROXIMATION OF CONVOLUTIONAL -NEURAL NETWORKS", https://arxiv.org/pdf/1604.03168.pdf - -[3] Y.Bengio et al., "Estimating or Propagating Gradients Through Stochastic -Neurons for Conditional Computation", https://arxiv.org/abs/1308.3432 diff --git a/tensorflow/contrib/quantize/__init__.py b/tensorflow/contrib/quantize/__init__.py deleted file mode 100644 index 933200e6074..00000000000 --- a/tensorflow/contrib/quantize/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functions for rewriting graphs for quantized training.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import,line-too-long -from tensorflow.contrib.quantize.python.quantize_graph import * -# pylint: enable=unused-import,wildcard-import,line-too-long - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - "create_eval_graph", - "create_training_graph", - "experimental_create_eval_graph", - "experimental_create_training_graph", -] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/quantize/python/common.py b/tensorflow/contrib/quantize/python/common.py deleted file mode 100644 index 3c553d07102..00000000000 --- a/tensorflow/contrib/quantize/python/common.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Common utilities used across this package.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import re - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope - -# Skip all operations that are backprop related or export summaries. -SKIPPED_PREFIXES = ( - 'gradients/', 'RMSProp/', 'Adagrad/', 'Const_', 'HistogramSummary', - 'ScalarSummary') - -# Valid activation ops for quantization end points. -_ACTIVATION_OP_SUFFIXES = ['Relu6', 'Relu', 'Identity'] - -# Regular expression for recognizing nodes that are part of batch norm group. -_BATCHNORM_RE = re.compile(r'^(.*)BatchNorm/batchnorm') - - -def BatchNormGroups(graph): - """Finds batch norm layers, returns their prefixes as a list of strings. - - Args: - graph: Graph to inspect. - - Returns: - List of strings, prefixes of batch norm group names found. - """ - bns = [] - for op in graph.get_operations(): - match = _BATCHNORM_RE.search(op.name) - if match: - bn = match.group(1) - if not bn.startswith(SKIPPED_PREFIXES): - bns.append(bn) - # Filter out duplicates. - return list(collections.OrderedDict.fromkeys(bns)) - - -def GetEndpointActivationOp(graph, prefix): - """Returns an Operation with the given prefix and a valid end point suffix. - - Args: - graph: Graph where to look for the operation. - prefix: String, prefix of Operation to return. - - Returns: - The Operation with the given prefix and a valid end point suffix or None if - there are no matching operations in the graph for any valid suffix - """ - for suffix in _ACTIVATION_OP_SUFFIXES: - activation = _GetOperationByNameDontThrow(graph, prefix + suffix) - if activation: - return activation - return None - - -def _GetOperationByNameDontThrow(graph, name): - """Returns an Operation with the given name. - - Args: - graph: Graph where to look for the operation. - name: String, name of Operation to return. - - Returns: - The Operation with the given name. None if the name does not correspond to - any operation in the graph - """ - try: - return graph.get_operation_by_name(name) - except KeyError: - return None - - -def CreateOrGetQuantizationStep(): - """Returns a Tensor of the number of steps the quantized graph has run. - - Returns: - Quantization step Tensor. - """ - quantization_step_name = 'fake_quantization_step' - quantization_step_tensor_name = quantization_step_name + '/Identity:0' - g = ops.get_default_graph() - try: - return g.get_tensor_by_name(quantization_step_tensor_name) - except KeyError: - # Create in proper graph and base name_scope. - with g.name_scope(None): - quantization_step_tensor = variable_scope.get_variable( - quantization_step_name, - shape=[], - dtype=dtypes.int64, - initializer=init_ops.zeros_initializer(), - trainable=False, - collections=[ops.GraphKeys.GLOBAL_VARIABLES], - aggregation=variable_scope.VariableAggregation.ONLY_FIRST_REPLICA) - with g.name_scope(quantization_step_tensor.op.name + '/'): - # We return the incremented variable tensor. Since this is used in conds - # for quant_delay and freeze_bn_delay, it will run once per graph - # execution. We return an identity to force resource variables and - # normal variables to return a tensor of the same name. - return array_ops.identity( - state_ops.assign_add(quantization_step_tensor, 1)) - - -def DropStringPrefix(s, prefix): - """If the string starts with this prefix, drops it.""" - if s.startswith(prefix): - return s[len(prefix):] - else: - return s - - -def RerouteTensor(t0, t1, can_modify=None): - """Reroute the end of the tensor t0 to the ends of the tensor t1. - - Args: - t0: a tf.Tensor. - t1: a tf.Tensor. - can_modify: iterable of operations which can be modified. Any operation - outside within_ops will be left untouched by this function. - - Returns: - The number of individual modifications made by the function. - """ - nb_update_inputs = 0 - consumers = t1.consumers() - if can_modify is not None: - consumers = [c for c in consumers if c in can_modify] - consumers_indices = {} - for c in consumers: - consumers_indices[c] = [i for i, t in enumerate(c.inputs) if t is t1] - for c in consumers: - for i in consumers_indices[c]: - c._update_input(i, t0) # pylint: disable=protected-access - nb_update_inputs += 1 - return nb_update_inputs diff --git a/tensorflow/contrib/quantize/python/common_test.py b/tensorflow/contrib/quantize/python/common_test.py deleted file mode 100644 index a3ce041ceae..00000000000 --- a/tensorflow/contrib/quantize/python/common_test.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for common utilities in this package.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.quantize.python import common -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest - -batch_norm = layers.batch_norm -conv2d = layers.conv2d - - -class CommonTest(test_util.TensorFlowTestCase): - - def testCreateOrGetQuantizationStep(self): - self._TestCreateOrGetQuantizationStep(False) - - def testCreateOrGetQuantizationStepResourceVar(self): - self._TestCreateOrGetQuantizationStep(True) - - def _TestCreateOrGetQuantizationStep(self, use_resource): - g = ops.Graph() - with session.Session(graph=g) as sess: - variable_scope.get_variable_scope().set_use_resource(use_resource) - quantization_step_tensor = common.CreateOrGetQuantizationStep() - - # Check that operations are added to the graph. - num_nodes = len(g.get_operations()) - self.assertGreater(num_nodes, 0) - - # Check that getting the quantization step doesn't change the graph. - get_quantization_step_tensor = common.CreateOrGetQuantizationStep() - self.assertEqual(quantization_step_tensor, get_quantization_step_tensor) - self.assertEqual(num_nodes, len(g.get_operations())) - - # Ensure that running the graph increments the quantization step. - sess.run(variables.global_variables_initializer()) - step_val = sess.run(quantization_step_tensor) - self.assertEqual(step_val, 1) - - # Ensure that even running a graph that depends on the quantization step - # multiple times only executes it once. - a = quantization_step_tensor + 1 - b = a + quantization_step_tensor - _, step_val = sess.run([b, quantization_step_tensor]) - self.assertEqual(step_val, 2) - - def testRerouteTensor(self): - a = constant_op.constant(1, name='a') - b = constant_op.constant(2, name='b') - c = constant_op.constant(3, name='c') - d = constant_op.constant(4, name='d') - - add_ac = math_ops.add(a, c) - add_ad = math_ops.add(a, d) - - # Ensure that before rerouting the inputs are what we think. - self._CheckOpHasInputs(add_ac.op, [a, c]) - self._CheckOpHasInputs(add_ad.op, [a, d]) - - # references to tensor a should be replaced with b for all ops in - # can_modify. This means add_ac will be changed but add_ad will not. - common.RerouteTensor(b, a, can_modify=[add_ac.op]) - self._CheckOpHasInputs(add_ac.op, [b, c]) - self._CheckOpHasInputs(add_ad.op, [a, d]) - - def _CheckOpHasInputs(self, op, inputs): - for i in inputs: - self.assertIn(i, op.inputs) - - def testBatchNormScope(self): - batch_size, height, width, depth = 5, 128, 128, 3 - g = ops.Graph() - with g.as_default(): - inputs = array_ops.zeros((batch_size, height, width, depth)) - stride = 1 - out_depth = 32 - scope = '' - node = conv2d( - inputs, - out_depth, [2, 2], - stride=stride, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - normalizer_fn=batch_norm, - normalizer_params=self._BatchNormParams(False), - scope=scope) - - node = nn_ops.relu(node, name='Relu6') - bn_list = common.BatchNormGroups(g) - with open('/tmp/common_test.pbtxt', 'w') as f: - f.write(str(g.as_graph_def())) - - # Exactly one batch norm layer with empty scope should be found - self.assertEqual(len(bn_list), 1) - self.assertEqual(bn_list[0], '') - - def _BatchNormParams(self, fused=False, force_updates=False): - params = { - 'center': True, - 'scale': True, - 'decay': 1.0 - 0.003, - 'fused': fused - } - return params - - def _WeightInit(self, stddev): - """Returns a truncated normal variable initializer. - - Function is defined purely to shorten the name so that it stops wrapping. - - Args: - stddev: Standard deviation of normal variable. - - Returns: - An initializer that initializes with a truncated normal variable. - """ - return init_ops.truncated_normal_initializer(stddev=stddev, seed=1234) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py deleted file mode 100644 index 4d4e632e2c3..00000000000 --- a/tensorflow/contrib/quantize/python/fold_batch_norms.py +++ /dev/null @@ -1,1087 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Logic to fold batch norm into preceding convolution or FC layers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re -from tensorflow.contrib.quantize.python import common -from tensorflow.contrib.quantize.python import graph_matcher -from tensorflow.contrib.quantize.python import input_to_ops -from tensorflow.core.framework import attr_value_pb2 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.layers import utils -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.util import compat - - -def FoldBatchNorms(graph, is_training, freeze_batch_norm_delay=None): - """Finds batch norm layers and folds them into preceding layers. - - Folding only affects the following layers: Conv2D, fully connected, depthwise - convolution. - - Args: - graph: Graph to walk and modify. - is_training: Bool, true if training. - freeze_batch_norm_delay: How many steps to wait before freezing moving mean - and variance and using them for batch normalization. This value is used - only when is_training is True. - Raises: - ValueError: When batch norm folding fails. - """ - _FoldFusedBatchNorms( - graph, is_training, freeze_batch_norm_delay=freeze_batch_norm_delay) - _FoldUnfusedBatchNorms( - graph, - is_training=is_training, - freeze_batch_norm_delay=freeze_batch_norm_delay) - - -def _FoldFusedBatchNorms(graph, is_training, freeze_batch_norm_delay): - """Finds fused batch norm layers and folds them into preceding layers. - - Folding only affects the following layers: Conv2D, fully connected, depthwise - convolution. - - Args: - graph: Graph to walk and modify. - is_training: Bool, true if training. - freeze_batch_norm_delay: How many steps to wait before freezing moving mean - and variance and using them for batch normalization. - - Raises: - ValueError: When batch norm folding fails. - """ - for match in _FindFusedBatchNorms(graph): - scope, sep, _ = match.layer_op.name.rpartition('/') - # Make sure new ops are added to `graph` and put on the same device as - # `bn_op`. The '/' (i.e. `sep`) ensures that we reuse the existing scope - # named `scope`. Otherwise, TF creates a unique scope whose name starts with - # `scope`. - with graph.as_default(), graph.name_scope(scope + sep): - with graph.name_scope(scope + sep + 'BatchNorm_Fold' + sep): - # new weights = old weights * gamma / sqrt(variance + epsilon) - # new biases = -mean * gamma / sqrt(variance + epsilon) + beta - multiplier_tensor = match.gamma_tensor * math_ops.rsqrt( - match.variance_tensor + match.bn_op.get_attr('epsilon')) - bias_tensor = math_ops.subtract( - match.beta_tensor, - match.mean_tensor * multiplier_tensor, - name='bias') - - correction_scale, correction_recip, correction_offset = None, None, None - if is_training: - correction_scale, correction_recip, correction_offset = ( - _ComputeBatchNormCorrections( - context='', - match=match, - freeze_batch_norm_delay=freeze_batch_norm_delay)) - # The shape of depthwise weights is different, so we need to reshape the - # multiplier_tensor to ensure that the scaled_weight_tensor has the - # expected shape. - weights = match.weight_tensor - if match.layer_op.type == 'DepthwiseConv2dNative': - new_shape = [ - match.weight_tensor.get_shape().as_list()[2], - match.weight_tensor.get_shape().as_list()[3] - ] - multiplier_tensor = array_ops.reshape( - multiplier_tensor, new_shape, name='scale_reshape') - - if correction_scale is not None: - correction_scale = array_ops.reshape( - correction_scale, new_shape, name='correction_reshape') - - if correction_scale is not None: - weights = math_ops.multiply( - correction_scale, weights, name='correction_mult') - - scaled_weight_tensor = math_ops.multiply( - weights, multiplier_tensor, name='mul_fold') - - new_layer_tensor = _CloneWithNewOperands( - match.layer_op, match.input_tensor, scaled_weight_tensor, - match.batch_to_space_op) - - if correction_recip is not None: - new_layer_tensor = math_ops.multiply( - correction_recip, new_layer_tensor, name='post_conv_mul') - new_layer_tensor = math_ops.add(new_layer_tensor, (correction_offset), - 'correction_add') - - bias_add_tensor = math_ops.add( - new_layer_tensor, bias_tensor, name='add_fold') - - nodes_modified_count = common.RerouteTensor(bias_add_tensor, - match.output_tensor) - if nodes_modified_count == 0: - raise ValueError('Folding batch norms failed, %s had no outputs.' % - match.output_tensor.name) - - -def _FindFusedBatchNorms(graph): - """Finds all ops and tensors related to found FusedBatchNorms. - - Args: - graph: Graph to inspect. - - Returns: - _FusedBatchNormMatches. - """ - input_pattern = graph_matcher.OpTypePattern('*') - # In practice, the weight pattern can match a Variable or a SpaceToBatchND - # operation that follows a variable for atrous convolutions. - weight_pattern = graph_matcher.OpTypePattern('*') - gamma_pattern = graph_matcher.OpTypePattern('*') - beta_pattern = graph_matcher.OpTypePattern('*') - mean_pattern = graph_matcher.OpTypePattern('*') - variance_pattern = graph_matcher.OpTypePattern('*') - - moving_average_pattern = graph_matcher.OpTypePattern('*') - bn_decay_pattern = graph_matcher.OpTypePattern('*') - layer_pattern = graph_matcher.OpTypePattern( - 'Conv2D|DepthwiseConv2dNative|MatMul', - inputs=[input_pattern, weight_pattern]) - batch_to_space_pattern = graph_matcher.OpTypePattern( - 'BatchToSpaceND', - inputs=[ - layer_pattern, - graph_matcher.OpTypePattern('*'), - graph_matcher.OpTypePattern('*') - ]) - # Identity between conv/matmul and bn - layer_pattern_with_identity = graph_matcher.OpTypePattern( - 'Identity', - inputs=[ - graph_matcher.OneofPattern([batch_to_space_pattern, layer_pattern]) - ]) - layer_output_pattern = graph_matcher.OneofPattern( - [layer_pattern_with_identity, layer_pattern, batch_to_space_pattern]) - - # MatMul has a Reshape between it and FusedBatchNorm. - matmul_reshape_pattern = graph_matcher.OpTypePattern( - 'Reshape', - inputs=[layer_output_pattern, - graph_matcher.OpTypePattern('*')]) - - batch_norm_pattern = graph_matcher.OpTypePattern( - 'FusedBatchNorm|FusedBatchNormV3', - inputs=[ - graph_matcher.OneofPattern( - [matmul_reshape_pattern, layer_output_pattern]), gamma_pattern, - beta_pattern, mean_pattern, variance_pattern - ]) - matmul_bn_output_reshape_pattern = graph_matcher.OpTypePattern( - 'Reshape', inputs=[batch_norm_pattern, - graph_matcher.OpTypePattern('*')]) - - batch_norm_identity_pattern = graph_matcher.OpTypePattern( - 'Identity', inputs=[batch_norm_pattern, matmul_bn_output_reshape_pattern]) - - bn_identity_matcher = graph_matcher.GraphMatcher(batch_norm_identity_pattern) - - bn_matcher = graph_matcher.GraphMatcher( - graph_matcher.OneofPattern( - [matmul_bn_output_reshape_pattern, batch_norm_pattern])) - - moving_average_sub_pattern = graph_matcher.OpTypePattern( - 'Sub', inputs=[moving_average_pattern, batch_norm_pattern]) - moving_average_mul_pattern = graph_matcher.OpTypePattern( - 'Mul', inputs=[moving_average_sub_pattern, bn_decay_pattern]) - - moving_avg_mul_matcher = graph_matcher.GraphMatcher( - moving_average_mul_pattern) - - def _GetLayerMatch(match_result): - """Populates a layer match object containing ops/tensors for folding BNs. - - Args: - match_result: Matched result from graph matcher - - Returns: - layer_op: Matching conv/fc op prior to batch norm - BatchNormMatch: _BatchNormMatch containing all required batch norm - parameters. - """ - moving_mean_tensor = None - moving_variance_tensor = None - bn_decay_mean_tensor = None - bn_decay_var_tensor = None - batch_to_space_op = None - layer_op = match_result.get_op(layer_pattern) - layer_tensor = match_result.get_tensor(layer_pattern) - bn_id_op = match_result.get_op(batch_norm_identity_pattern) - bn_op = match_result.get_op(batch_norm_pattern) - if bn_id_op is None: - bn_id_op = bn_op - - batch_epsilon = bn_op.get_attr('epsilon') - - # In the MatMul case, the output of batch norm is reshaped back into a - # 2D tensor, so the output_tensor is the output of the Reshape op. - output_tensor = bn_op.outputs[0] - if layer_op.type == 'MatMul': - output_reshape_op = match_result.get_op(matmul_bn_output_reshape_pattern) - # If the matcher didn't match matmul_bn_output_reshape, there will be - # another match for this 'MatMul' later, so we can skip this one. - if output_reshape_op is None: - return None, None - output_tensor = output_reshape_op.outputs[0] - - # Ensure that the output tensor has consumers, otherwise this is a dangling - # node and not a match. - if not output_tensor.consumers(): - return None, None - - batch_to_space_op = match_result.get_op(batch_to_space_pattern) - input_tensor = match_result.get_tensor(input_pattern) - weight_tensor = match_result.get_tensor(weight_pattern) - gamma_tensor = match_result.get_tensor(gamma_pattern) - beta_tensor = match_result.get_tensor(beta_pattern) - # FusedBatchNorm in training is different from that in inference. It takes - # empty 'mean' and empty 'variance', and produces the mean and the variance - # of the batch. Therefore, when is_training is true, mean_tensor and - # variance_tensor point to 1st and 2nd (0-based) output of bn_op, - # respectively; when is_training is false, they point to bn_op's inputs. - is_training = bn_op.get_attr('is_training') - if is_training: - # FusedBatchNormGrad doesn't compute gradients of the batch_mean and - # batch_variance outputs, so we need to substitute our own custom - # gradient. - # TODO(suharshs, raghuramank): Find a way to avoid needing this hack. - # pylint: disable=protected-access - bn_op._set_attr( - '_gradient_op_type', - attr_value_pb2.AttrValue(s=compat.as_bytes('FoldFusedBatchNormGrad'))) - # pylint: enable=protected-access - mean_tensor = bn_op.outputs[1] - # The batch variance used during forward and backward prop is biased, - # i.e it is calculated as: V=sum(x(k)-mu)^2/N. For the moving average - # calculation, the variance is corrected by the term N/N-1 (Bessel's - # correction). The variance tensor read from FuseBatchNorm has Bessel's - # correction applied, so we undo it here. - scope, sep, _ = bn_op.name.rpartition('/') - g = ops.get_default_graph() - with g.as_default(), g.name_scope(scope + sep): - n = math_ops.cast( - array_ops.size(layer_tensor) / array_ops.size(mean_tensor), - dtypes.float32) - variance_tensor = math_ops.multiply( - bn_op.outputs[2], (n - 1) / n, name='Undo_Bessel_Correction') - # TODO(suharshs): Find a way to get rid of this inner match. - for mul_match_result in moving_avg_mul_matcher.match_graph(graph): - sub_op = mul_match_result.get_op(moving_average_sub_pattern) - if sub_op.inputs[1].name == bn_op.outputs[1].name: - # During training: Batch Mean is bn_op.outputs[1] - moving_mean_tensor = sub_op.inputs[0] - bn_decay_mean_tensor = mul_match_result.get_tensor(bn_decay_pattern) - if sub_op.inputs[1].name == bn_op.outputs[2].name: - # During training: Batch Var is bn_op.outputs[2] - moving_variance_tensor = sub_op.inputs[0] - bn_decay_var_tensor = mul_match_result.get_tensor(bn_decay_pattern) - else: - mean_tensor = match_result.get_tensor(mean_pattern) - variance_tensor = match_result.get_tensor(variance_pattern) - - return layer_op, _BatchNormMatch( - layer_op=layer_op, - bn_op=bn_op, - output_tensor=output_tensor, - input_tensor=input_tensor, - weight_tensor=weight_tensor, - gamma_tensor=gamma_tensor, - beta_tensor=beta_tensor, - mean_tensor=mean_tensor, - variance_tensor=variance_tensor, - moving_mean_tensor=moving_mean_tensor, - moving_variance_tensor=moving_variance_tensor, - bn_decay_mean_tensor=bn_decay_mean_tensor, - bn_decay_var_tensor=bn_decay_var_tensor, - batch_epsilon=batch_epsilon, - batch_to_space_op=batch_to_space_op) - - layer_matches = [] - # We use matched_layer_set to ensure that layers aren't matched multiple - # times. - matched_layer_set = set() - for match_result in bn_identity_matcher.match_graph(graph): - layer_op, layer_match = _GetLayerMatch(match_result) - if layer_op is not None: - if layer_op not in matched_layer_set: - matched_layer_set.add(layer_op) - layer_matches.append(layer_match) - - for match_result in bn_matcher.match_graph(graph): - layer_op, layer_match = _GetLayerMatch(match_result) - if layer_op is not None: - if layer_op not in matched_layer_set: - matched_layer_set.add(layer_op) - layer_matches.append(layer_match) - - return layer_matches - - -def _ComputeBatchNormCorrections(context, match, freeze_batch_norm_delay): - """Computes batch norm correction params. - - Before batch normalization is frozen: - We use batch statistics for batch norm. - correction_scale = sigma_b/sigma_mv - correction_recip = 1/correction_scale - correction_offset = 0 - - After batch normalization is frozen: - correction_scale = sigma_b/sigma_mv - correction_recip = 1 - correction_offset = gamma*(mu_b/sigma_b-mu_mv/sigma_mv). - - Batch norm is frozen if global_step > bn_freeze_delay. - The corrections ensure that: - a) The weights are quantized after scaling by gamma/sigma_mv. This enables - smoother training as the scaling on the weights changes slowly, rather than - jump across mini-batches - b) Changing the values of the corrections allows for one to switch between - using batch statistics to using moving mean and average, without requiring - changes to batch_norm - - - Args: - context: The scope under which we look for batch norm params - match: Object containing required batch norm tensors for correction - computation. - freeze_batch_norm_delay: Delay in steps at which computation switches - from regular batch norm to frozen mean and variance. - - - Returns: - A tuple of correction_scale, correction_recip, correction_offset - """ - - g = ops.get_default_graph() - prefix = '' if not context else context - with g.name_scope(prefix + 'batch_norm_correction'): - recip_sigma_mv = math_ops.rsqrt( - match.moving_variance_tensor + match.batch_epsilon) - recip_sigma = math_ops.rsqrt(match.variance_tensor + match.batch_epsilon) - correction_scale = math_ops.divide( - recip_sigma_mv, recip_sigma, name='scale_compute') - correction_scale = array_ops.identity( - correction_scale, name='correction_scale') - correction_recip = math_ops.reciprocal( - correction_scale, name='reciprocal_compute') - correction_offset = math_ops.multiply( - match.gamma_tensor, - match.mean_tensor * recip_sigma - - match.moving_mean_tensor * recip_sigma_mv, - name='offset_compute') - - if freeze_batch_norm_delay is not None: - use_mv_avg = math_ops.greater_equal( - common.CreateOrGetQuantizationStep(), - freeze_batch_norm_delay, - name='use_moving_average') - else: - use_mv_avg = False - - bn_decay_zero = 0.0 - bn_decay_mean_consumers = list(match.bn_decay_mean_tensor.consumers()) - bn_decay_var_consumers = list(match.bn_decay_mean_tensor.consumers()) - - bn_decay_mean_out = utils.smart_cond( - use_mv_avg, - lambda: bn_decay_zero, - lambda: match.bn_decay_mean_tensor, - name='freeze_moving_mean') - - common.RerouteTensor( - bn_decay_mean_out, - match.bn_decay_mean_tensor, - can_modify=bn_decay_mean_consumers) - - bn_decay_var_consumers = list(match.bn_decay_var_tensor.consumers()) - bn_decay_var_out = utils.smart_cond( - use_mv_avg, - lambda: bn_decay_zero, - lambda: match.bn_decay_var_tensor, - name='freeze_moving_var') - common.RerouteTensor( - bn_decay_var_out, - match.bn_decay_var_tensor, - can_modify=bn_decay_var_consumers) - - correction_recip = utils.smart_cond( - use_mv_avg, - lambda: array_ops.ones(correction_scale.shape), - lambda: correction_recip, - name='correction_recip') - - correction_offset = utils.smart_cond( - use_mv_avg, - lambda: correction_offset, - lambda: array_ops.zeros(correction_offset.shape), - name='correction_offset') - return correction_scale, correction_recip, correction_offset - - -def _CloneWithNewOperands(layer_op, input_tensor, weight_tensor, - batch_to_space_op): - """Clones layer_op with input_tensor and weight_tensor as new inputs.""" - new_layer_name = layer_op.name.split('/')[-1] + '_Fold' - if layer_op.type == 'Conv2D': - return nn_ops.conv2d( - input_tensor, - weight_tensor, - strides=layer_op.get_attr('strides'), - padding=layer_op.get_attr('padding'), - use_cudnn_on_gpu=layer_op.get_attr('use_cudnn_on_gpu'), - data_format=layer_op.get_attr('data_format').decode(), - name=new_layer_name) - elif layer_op.type == 'MatMul': - return math_ops.matmul( - input_tensor, - weight_tensor, - transpose_a=layer_op.get_attr('transpose_a'), - transpose_b=layer_op.get_attr('transpose_b'), - name=new_layer_name) - elif layer_op.type == 'DepthwiseConv2dNative': - # We don't copy dilation rate because we reuse the input SpaceToBatch - # and create our own BatchToSpace operation below. - conv = nn.depthwise_conv2d( - input_tensor, - weight_tensor, - strides=layer_op.get_attr('strides'), - padding=layer_op.get_attr('padding'), - name=new_layer_name) - # Copy the batch to space operation if we have a atrous convolution. - if batch_to_space_op: - batch_to_space_op = layer_op.outputs[0].consumers()[0] - # TODO(suharshs): It's hard to make this name match with the unfused name. - # Restructure this code to not rely on scope at all. - new_batch_to_space_name = batch_to_space_op.name.split('/')[-1] + '_Fold' - conv = array_ops.batch_to_space_nd( - conv, - batch_to_space_op.inputs[1], - batch_to_space_op.inputs[2], - name=new_batch_to_space_name) - return conv - else: - raise ValueError('Cannot handle operation of type: %s' % layer_op.type) - - -@ops.RegisterGradient('FoldFusedBatchNormGrad') -def _FoldFusedBatchNormGrad(op, - unused_grad_y, - grad_mean, - grad_var, - unused_1, - unused_2, - unused_3=None): - """Gradient function for the FusedBatchNorm ops matched by _GetLayerMatch.""" - x = op.inputs[0] - n = math_ops.cast( - array_ops.size(x) / array_ops.size(grad_mean), dtypes.float32) - dmean_dx = grad_mean / n - dvar_dx = 2 * grad_var * (x - op.outputs[1]) / (n - 1) - return (dmean_dx + dvar_dx), None, None, None, None - - -def _FoldUnfusedBatchNorms(graph, is_training, freeze_batch_norm_delay): - """Finds unfused batch norm layers and folds them into preceding layers. - - Folding only affects the following layers: Conv2D, fully connected, depthwise - convolution. - - Args: - graph: Graph to walk and modify. - is_training: Bool, True if training. - freeze_batch_norm_delay: How many steps to wait before freezing moving mean - and variance and using them for batch normalization. - - Raises: - ValueError: When batch norm folding fails. - """ - input_to_ops_map = input_to_ops.InputToOps(graph) - - for bn in common.BatchNormGroups(graph): - has_scaling = _HasScaling(graph, input_to_ops_map, bn) - - if not _IsValidUnfusedBatchNorm(graph, bn): - continue - - # The mangling code intimately depends on BatchNorm node's internals. - original_op, folded_op = _CreateFoldedOp( - graph, - bn, - has_scaling=has_scaling, - freeze_batch_norm_delay=freeze_batch_norm_delay, - is_training=is_training) - - activation = common.GetEndpointActivationOp(graph, bn) - if activation: - nodes_modified_count = common.RerouteTensor( - folded_op.outputs[0], original_op.outputs[0], can_modify=[activation]) - if nodes_modified_count != 1: - raise ValueError('Unexpected inputs to op: %s' % activation.name) - continue - - # Treat consumer ops in bypass modules differently since they have Add - # operations instead of Relu* above. - # Changes to make sure that the correct scope is selected for the bypass add - # The rule here is that if the scope is of the form: str1/str2 for the - # batch norm, - # the bypass add is at scope str1. If bn is of scope just str1, then the - # bypass add is at scope ''. - # If there is no batch norm, then there is no bypass add. - add_bypass_ctx = '' - if bn: - try: - add_bypass_ctx = re.search(r'^(.*)/([^/]+)', bn).group(1) - except AttributeError: - add_bypass_ctx = '' - - if add_bypass_ctx: - add_bypass_ctx = add_bypass_ctx + '/' - - add_bypass = graph.get_operation_by_name(add_bypass_ctx + 'AddV2') - nodes_modified_count = common.RerouteTensor( - folded_op.outputs[0], original_op.outputs[0], can_modify=[add_bypass]) - if nodes_modified_count != 1: - raise ValueError('Unexpected inputs to op: %s' % add_bypass.name) - - -def _IsValidUnfusedBatchNorm(graph, context): - """Checks that the output of the unfused batch norm has consumers.""" - add_shift = graph.get_operation_by_name(context + - 'BatchNorm/batchnorm_1/add_1') - # Ensure that the output tensor of batch norm has consumers, otherwise this - # is a dangling node and not a match. - return bool(add_shift.outputs[0].consumers()) - - -def _FindMatchingTensor(graph, match_pattern, scope): - """Finds best match of ops matching match_pattern with scope. - - Example: _FindMatchingTensor(graph,'/BatchNorm/moments/Squeeze', - 'MobilenetV1/MobilenetV1/Conv2d_0/') returns: - Tensor('MobilenetV1/Conv2d_0/BatchNorm/moments/Squeeze') - - Args: - graph: Graph to inspect. - match_pattern: Part of the name of the op that we need to match, should - be present in the op's name - scope: The scope of the op. All the elements of the scope need not be - present in the op's name. - - Returns: - Tensor from graph that provides the best match to the match_pattern and - scope - """ - - oplist = graph.get_operations() - split_context = set(scope.split('/')) - match_dict = {} - for op in oplist: - if op.name.endswith(match_pattern): - split_name = op.name.split('/') - num_matches = len(set(split_name) & split_context) - - if num_matches > 0 or not scope: - match_dict[op.name] = num_matches - # match_dict contains matching op names from graph with values being - # number of matches to scope. We pick the key with the most matches - if match_dict: - max_key = max(match_dict, key=match_dict.get) - return graph.get_tensor_by_name(max_key + ':0') - else: - return None - - -def _GetBatchNormParams(graph, context, has_scaling): - """Extracts relevant tensors for folding batch norms. - - Args: - graph: Graph to inspect. - context: The scope under which we look for batch norm params - has_scaling: Bool that specifies if scaling is done as part of batch norm. - - Returns: - _BatchNormMatch containing all required batch norm parameters. - """ - gamma_tensor = None - batch_mean_tensor = None - batch_variance_tensor = None - moving_mean_tensor = None - moving_variance_tensor = None - batch_epsilon = None - bn_decay_mean_tensor = None - bn_decay_var_tensor = None - - # TODO(raghuramank) This code relies on string matching and needs to be - # updated if unfused batch norm continues to be widely used - # Matching variable names is brittle and relies on scoping - # conventions. Fused batch norm folding is more robust. Support for unfused - # batch norms will be deprecated as we move forward. Fused batch norms allow - # for faster training and should be used whenever possible. - # context contains part of the names of the tensors we are interested in: - # For MobilenetV1, the context has repetitions: - # MobilenetV1/MobilenetV1/Conv2d_3_depthwise - # when the moving_mean tensor has the name: - # MobilenetV1/Conv2d_3_depthwise/BatchNorm/moving_mean/read - # To pick the correct variable name, it is necessary to ignore the repeating - # header. - - # For MobilenetV2, this problem does not exist: - # The context is: MobilenetV2/expanded_conv_3/depthwise - # and the names of the tensors start with a single MobilenetV2 - # The moving mean for example, has the name: - # MobilenetV2/expanded_conv_3/depthwise/BatchNorm/moving_mean/read - # We identify the best match for an op by checking for - # 1. The suffix of the op is exactly matched - # 2. Maximum number of matches with the context.The matching - # score is given by the number of parts of context (split by /) that - # are present in the parts of the tensor name (again split by /). - # For example: scope= MobilenetV2/MobilenetV2/expanded_conv_3 and - # op.name = MobilenetV2/expanded_conv_3/depthwise/BatchNorm/moving_mean/read - # will have 2 matches,scope with a different conv layer will have one match. - - op_suffix_mean = 'BatchNorm/moments/Squeeze' - op_suffix_variance = 'BatchNorm/moments/Squeeze_1' - op_suffix_epsilon = 'BatchNorm/batchnorm_1/add/y' - op_suffix_bn_decay_mean = 'BatchNorm/AssignMovingAvg/decay' - op_suffix_bn_decay_var = 'BatchNorm/AssignMovingAvg_1/decay' - - if variable_scope.get_variable_scope().use_resource: - op_suffix_gamma = 'BatchNorm/gamma/Read/ReadVariableOp' - op_suffix_moving_variance = ( - 'BatchNorm/moving_variance/Read/ReadVariableOp') - op_suffix_moving_mean = ('BatchNorm/moving_mean/Read/ReadVariableOp') - else: - op_suffix_gamma = 'BatchNorm/gamma' - op_suffix_moving_variance = 'BatchNorm/moving_variance/read' - op_suffix_moving_mean = 'BatchNorm/moving_mean/read' - # Parse through list of ops to find relevant ops - - batch_mean_tensor = _FindMatchingTensor(graph, op_suffix_mean, context) - batch_variance_tensor = _FindMatchingTensor(graph, op_suffix_variance, - context) - moving_mean_tensor = _FindMatchingTensor(graph, op_suffix_moving_mean, - context) - moving_variance_tensor = _FindMatchingTensor(graph, op_suffix_moving_variance, - context) - batch_epsilon = _FindMatchingTensor(graph, op_suffix_epsilon, context) - bn_decay_mean_tensor = _FindMatchingTensor(graph, op_suffix_bn_decay_mean, - context) - bn_decay_var_tensor = _FindMatchingTensor(graph, op_suffix_bn_decay_var, - context) - if batch_mean_tensor is None and moving_mean_tensor is None: - ValueError('Error folding unfused batch norms') - if has_scaling: - gamma_tensor = _FindMatchingTensor(graph, op_suffix_gamma, context) - - if not has_scaling: - gamma_tensor = array_ops.ones(moving_mean_tensor.shape) - - return _BatchNormMatch( - layer_op=None, - bn_op=None, - output_tensor=None, - input_tensor=None, - weight_tensor=None, - gamma_tensor=gamma_tensor, - beta_tensor=None, - mean_tensor=batch_mean_tensor, - variance_tensor=batch_variance_tensor, - moving_mean_tensor=moving_mean_tensor, - moving_variance_tensor=moving_variance_tensor, - bn_decay_mean_tensor=bn_decay_mean_tensor, - bn_decay_var_tensor=bn_decay_var_tensor, - batch_epsilon=batch_epsilon, - batch_to_space_op=None) - - -def _CreateFoldedOp(graph, context, has_scaling, freeze_batch_norm_delay, - is_training): - """Folds in batch norm layer into preceding convolution or FC layer. - - Creates 3 new nodes, connects their inputs and adds them to the graph: - mul is cloned into mul_fold, Conv2D or MatMul, or DepthwiseConv2d is cloned - into respective *_Fold, add is cloned into add_fold. - - Args: - graph: Graph to modify. - context: String, batch norm context, i.e. node into which BatchNorm is - nested. - has_scaling: Whether the batch norm has scaling enabled. - freeze_batch_norm_delay: How many steps to wait before freezing moving mean - and variance and using them for batch normalization. - is_training: Bool, true if training. - - Raises: - ValueError: When operation type is not supported, or input and output tensor - shapes mismatch for created operations: mul_fold, add_fold. - - Returns: - A pair of Operations, the first is the original consumer node of the batch - norm (../BatchNorm/batchnorm_1/add_1), the second is the consumer node of - the folded graph (add_fold). - """ - mul_scale_name = 'mul_1' if has_scaling else 'mul' - mul_scale = graph.get_operation_by_name(context + 'BatchNorm/batchnorm_1/' + - mul_scale_name) - op_below = mul_scale.inputs[0].op - # Skip over the BatchToSpace operation in the case of atrous convolutions. - batch_to_space_op = None - if op_below.type == 'BatchToSpaceND': - batch_to_space_op = op_below - op_below = op_below.inputs[0].op - weights = op_below.inputs[1] - match = _GetBatchNormParams( - graph=graph, context=context, has_scaling=has_scaling) - correction_scale, correction_recip, correction_offset = None, None, None - if is_training: - correction_scale, correction_recip, correction_offset = ( - _ComputeBatchNormCorrections( - context=context, - match=match, - freeze_batch_norm_delay=freeze_batch_norm_delay)) - # Special handling for weights of depthwise convolution. - if op_below.type == 'DepthwiseConv2dNative': - new_shape = [ - weights.get_shape().as_list()[2], - weights.get_shape().as_list()[3] - ] - scale_name = 'mul' if has_scaling else 'Rsqrt' - scale = graph.get_operation_by_name(context + 'BatchNorm/batchnorm_1/' + - scale_name) - scale = array_ops.reshape(scale.outputs[0], new_shape, - context + 'scale_reshape') - - if correction_scale is not None: - correction_scale = array_ops.reshape(correction_scale, new_shape, - context + 'correction_reshape') - with ops.device(mul_scale.device): - weights = math_ops.multiply(correction_scale, weights, - context + 'correction_mult') - - mul_fold = _CloneOp(mul_scale, context + 'mul_fold', [(0, weights), - (1, scale)]) - elif op_below.type in ['Conv2D', 'MatMul']: - - if correction_scale is not None: - with ops.device(mul_scale.device): - weights = math_ops.multiply(correction_scale, weights, - context + 'correction_mult') - mul_fold = _CloneOp(mul_scale, context + 'mul_fold', [(0, weights)]) - else: - raise ValueError('Cannot handle operation of type: %s' % op_below.type) - _AssertShapesMatch('mul_fold', mul_fold.inputs[0], mul_fold.outputs[0]) - - conv_or_fc_folded = _CloneOp(op_below, op_below.name + '_Fold', - [(1, mul_fold.outputs[0])]) - - add_shift = graph.get_operation_by_name(context + - 'BatchNorm/batchnorm_1/add_1') - - corrected_output = conv_or_fc_folded.outputs[0] - # Copy the batch to space operation if we have a atrous convolution. - if batch_to_space_op: - corrected_output = array_ops.batch_to_space_nd( - corrected_output, - batch_to_space_op.inputs[1], - batch_to_space_op.inputs[2], - name=batch_to_space_op.name + '_Fold') - if correction_offset is not None: - with ops.device(conv_or_fc_folded.device): - corrected_output = math_ops.multiply(correction_recip, corrected_output, - context + 'post_conv_mul') - corrected_output = math_ops.add(corrected_output, (correction_offset), - context + 'correction_add') - add_fold = _CloneOp(add_shift, context + 'add_fold', [(0, corrected_output)]) - _AssertShapesMatch('add_fold', add_fold.inputs[0], add_fold.outputs[0]) - return add_shift, add_fold - - -def _CloneOp(op, new_name, new_inputs): - """Clones a given op, replaces its name and some of its inputs. - - Args: - op: Operation to modify. - new_name: String, a new name to set on cloned op. - new_inputs: A list of tuples (idx, tensor), each input with corresponding - index will be replaced by the given Tensor in the cloned op. - - Returns: - Operation, the cloned op. - - Raises: - TypeError: When Operation type is not supported. - ValueError: When input shapes are incompatible. - """ - inputs = list(op.inputs) - for new_input in new_inputs: - inputs[new_input[0]] = new_input[1] - return _OP_CLONER.Clone(op, inputs, new_name) - - -class _OpCloner(object): - """Helper class that clones tf.Operations based on their type.""" - - def __init__(self): - self.op_type_to_action = { - 'Mul': self._CloneMul, - 'Add': self._CloneAdd, - 'AddV2': self._CloneAdd, - 'Conv2D': self._CloneConv2d, - 'DepthwiseConv2dNative': self._CloneDepthwiseConv2d, - 'MatMul': self._CloneMatMul, - } - - def _CloneMul(self, op, inputs, new_name): - del op # Unused. - return math_ops.multiply(inputs[0], inputs[1], name=new_name).op - - def _CloneAdd(self, op, inputs, new_name): - del op # Unused. - return math_ops.add(inputs[0], inputs[1], name=new_name).op - - def _CloneConv2d(self, op, inputs, new_name): - input_tensor = inputs[0] - weights = inputs[1] - self._AssertConvShapes(op.name, input_tensor, weights) - return nn_ops.conv2d( - input_tensor, - weights, - strides=op.get_attr('strides'), - padding=op.get_attr('padding'), - use_cudnn_on_gpu=op.get_attr('use_cudnn_on_gpu'), - data_format=op.get_attr('data_format').decode(), - name=new_name).op - - def _CloneDepthwiseConv2d(self, op, inputs, new_name): - input_tensor = inputs[0] - weights = inputs[1] - self._AssertConvShapes(op.name, input_tensor, weights) - return nn.depthwise_conv2d( - input_tensor, - weights, - strides=op.get_attr('strides'), - padding=op.get_attr('padding'), - name=new_name).op - - def _CloneMatMul(self, op, inputs, new_name): - weights = inputs[0] - input_tensor = inputs[1] - self._AssertFCShapes(op.name, weights, input_tensor) - return math_ops.matmul( - weights, - input_tensor, - transpose_a=op.get_attr('transpose_a'), - transpose_b=op.get_attr('transpose_b'), - name=new_name).op - - def Clone(self, op, inputs, new_name): - try: - return self.op_type_to_action[op.type](op, inputs, new_name) - except KeyError: - raise TypeError('Unsupported operation type: %s' % op.type) - - def _AssertConvShapes(self, op_name, input_tensor, weights): - """Makes sure that convolution inputs have compatible shapes. - - Args: - op_name: Operation name, only used in error message. - input_tensor: Input that is convolved. - weights: Weights of the convolution filter. - - Raises: - ValueError: When input shapes are incompatible. - """ - input_shape = input_tensor.get_shape() - weights_shape = weights.get_shape() - if (len(input_shape) != 4 or len(weights_shape) != 4 or - input_shape[3] != weights_shape[2]): - raise ValueError('Incompatible shapes for op %s inputs: %s and %s' % - (op_name, input_shape, weights_shape)) - - def _AssertFCShapes(self, op_name, weights, input_tensor): - """Makes sure that FC layer inputs have compatible shapes. - - Args: - op_name: Operation name, only used in error message. - weights: Weights used in FC layer. - input_tensor: Input into FC layer. - - Raises: - ValueError: When input shapes are incompatible. - """ - weights_shape = weights.get_shape() - input_shape = input_tensor.get_shape() - if (len(weights_shape) != 2 or len(input_shape) != 2 or - weights_shape[1] != input_shape[0]): - raise ValueError('Incompatible shapes for op %s inputs: %s and %s' % - (op_name, weights_shape, input_shape)) - -_OP_CLONER = _OpCloner() - - -def _AssertShapesMatch(op_name, in_tensor, out_tensor): - """Makes sure that shapes of input and output tensors are compatible. - - Args: - op_name: String, operation name, only used in error message. - in_tensor: Tensor, input tensor. - out_tensor: Tensor, output tensor. - - Raises: - ValueError: When input and output tensors have different shapes. - """ - in_shape = in_tensor.get_shape() - out_shape = out_tensor.get_shape() - - if not in_shape.is_compatible_with(out_shape): - raise ValueError('%s should not change tensor shape: input %s, ' - 'output %s' % (op_name, in_shape, out_shape)) - - -def _HasScaling(graph, input_to_ops_map, bn): - r"""Checks if batch norm has scaling enabled. - - Difference between batch norm with scaling and without is that with scaling: - - Rsqrt -> mul -> mul_1 - \-> mul_2 - - where - mul multiplies gamma by inverse square root of EMA of batch variance, - mul_1 multiplies output of mul with output from the base operation - (convolution, FC or depthwise convolution), - mul_2 multiplies output of mul with EMA of batch mean, - and without scaling: - - Rsqrt -> mul - \-> mul_1 - - where - mul multiplies the inverse square root of EMA of batch variance with output - from the base operation, - mul_1 multiplies inverse square root of EMA of batch variance with EMA - of batch mean. - - Args: - graph: Graph to inspect. - input_to_ops_map: InputToOps object containing mapping from tensor's name - to ops that take it as input. - bn: Batch norm layer prefix string. - - Returns: - A boolean indicating whether this batch norm layer has scaling enabled. - """ - rsqrt_op = graph.get_operation_by_name(bn + 'BatchNorm/batchnorm_1/Rsqrt') - rsqrt_consumers = input_to_ops_map.ConsumerOperations(rsqrt_op) - - return sum(1 for op in rsqrt_consumers if op.type == 'Mul') == 1 - - -class _BatchNormMatch(object): - """Contains all information related to a found Fused/UnfusedBatchNorm.""" - - def __init__(self, layer_op, bn_op, output_tensor, input_tensor, - weight_tensor, gamma_tensor, beta_tensor, mean_tensor, - variance_tensor, moving_mean_tensor, moving_variance_tensor, - bn_decay_mean_tensor, bn_decay_var_tensor, batch_epsilon, - batch_to_space_op): - self._layer_op = layer_op - self._bn_op = bn_op - self._output_tensor = output_tensor - self._input_tensor = input_tensor - self._weight_tensor = weight_tensor - self._gamma_tensor = gamma_tensor - self._beta_tensor = beta_tensor - self._mean_tensor = mean_tensor - self._variance_tensor = variance_tensor - self._moving_mean_tensor = moving_mean_tensor - self._moving_variance_tensor = moving_variance_tensor - self._bn_decay_mean_tensor = bn_decay_mean_tensor - self._bn_decay_var_tensor = bn_decay_var_tensor - self._batch_epsilon = batch_epsilon - self._batch_to_space_op = batch_to_space_op - - @property - def layer_op(self): - return self._layer_op - - @property - def bn_op(self): - return self._bn_op - - @property - def output_tensor(self): - return self._output_tensor - - @property - def input_tensor(self): - return self._input_tensor - - @property - def weight_tensor(self): - return self._weight_tensor - - @property - def gamma_tensor(self): - return self._gamma_tensor - - @property - def beta_tensor(self): - return self._beta_tensor - - @property - def mean_tensor(self): - return self._mean_tensor - - @property - def variance_tensor(self): - return self._variance_tensor - - @property - def moving_mean_tensor(self): - return self._moving_mean_tensor - - @property - def moving_variance_tensor(self): - return self._moving_variance_tensor - - @property - def batch_epsilon(self): - return self._batch_epsilon - - @property - def bn_decay_mean_tensor(self): - return self._bn_decay_mean_tensor - - @property - def bn_decay_var_tensor(self): - return self._bn_decay_var_tensor - - @property - def batch_to_space_op(self): - return self._batch_to_space_op diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py deleted file mode 100644 index c0eafbce299..00000000000 --- a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py +++ /dev/null @@ -1,828 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Unit tests for folding batch norm layers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.quantize.python import fold_batch_norms -from tensorflow.python.client import session -from tensorflow.python.compat import compat -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest -from tensorflow.python.training import saver as saver_lib - -batch_norm = layers.batch_norm -conv2d = layers.conv2d -fully_connected = layers.fully_connected -separable_conv2d = layers.separable_conv2d - - -# TODO(suharshs): Use parameterized test once OSS TF supports it. -class FoldBatchNormsTest(test_util.TensorFlowTestCase): - - def _RunTestOverParameters(self, test_fn): - parameters_list = [ - # (relu, relu_op_name, with_bypass, has_scaling, fused_batch_norm, - # freeze_batch_norm_delay, insert identity node) - (nn_ops.relu6, 'Relu6', False, False, False, 100, False), - (nn_ops.relu, 'Relu', False, False, False, None, False), - (nn_ops.relu6, 'Relu6', True, False, False, 100, False), - (nn_ops.relu, 'Relu', True, False, False, None, False), - (nn_ops.relu6, 'Relu6', False, True, False, 100, False), - (nn_ops.relu, 'Relu', False, True, False, None, False), - (nn_ops.relu6, 'Relu6', True, True, False, 100, False), - (nn_ops.relu, 'Relu', True, True, False, None, False), - # Fused batch norm always has scaling enabled. - (nn_ops.relu6, 'Relu6', False, True, True, None, False), - (nn_ops.relu, 'Relu', False, True, True, 100, False), - (nn_ops.relu6, 'Relu6', True, True, True, None, False), - (nn_ops.relu, 'Relu', True, True, True, 100, False), - (nn_ops.relu6, 'Relu6', False, True, True, None, True), - (nn_ops.relu, 'Relu', False, True, True, 100, True), - (nn_ops.relu6, 'Relu6', True, True, True, None, True), - (nn_ops.relu, 'Relu', True, True, True, 100, True), - ] - for params in parameters_list: - test_fn(params[0], params[1], params[2], params[3], params[4], params[5], - params[6]) - - def _TestFoldConv2d(self, relu, relu_op_name, with_bypass, has_scaling, - fused_batch_norm, freeze_batch_norm_delay, - insert_identity_node): - """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*. - - Args: - relu: Callable that returns an Operation, a factory method for the Relu*. - relu_op_name: String, name of the Relu* operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Relu*. - has_scaling: Bool, when true the batch norm has scaling. - fused_batch_norm: Bool, when true the batch norm is fused. - freeze_batch_norm_delay: None or the number of steps after which training - switches to using frozen mean and variance - insert_identity_node: Bool, insert identity node between conv and batch - norm - """ - g = ops.Graph() - with g.as_default(): - batch_size, height, width = 5, 128, 128 - inputs = array_ops.zeros((batch_size, height, width, 3)) - out_depth = 3 if with_bypass else 32 - stride = 1 if with_bypass else 2 - activation_fn = None if with_bypass else relu - name = 'test/test2' if with_bypass else 'test' - if insert_identity_node: - with g.name_scope(name): - node = conv2d( - inputs, - out_depth, [5, 5], - stride=stride, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - normalizer_fn=None, - biases_initializer=None) - conv_out = array_ops.identity(node, name='conv_out') - - node = batch_norm( - conv_out, - center=True, - scale=has_scaling, - decay=1.0 - 0.003, - fused=fused_batch_norm) - if activation_fn is not None: - node = activation_fn(node) - conv_name = name + '/Conv' - else: - node = conv2d( - inputs, - out_depth, [5, 5], - stride=stride, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=self._BatchNormParams( - scale=has_scaling, fused=fused_batch_norm), - scope=name) - conv_name = name - if with_bypass: - node = math_ops.add(inputs, node, name='test/AddV2') - relu(node, name='test/' + relu_op_name) - - fold_batch_norms.FoldBatchNorms( - g, is_training=True, freeze_batch_norm_delay=freeze_batch_norm_delay) - - folded_mul = g.get_operation_by_name(conv_name + '/mul_fold') - self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, [ - conv_name + '/correction_mult', - self._BatchNormMultiplierName(conv_name, has_scaling, fused_batch_norm) - ]) - self._AssertOutputGoesToOps(folded_mul, g, [conv_name + '/Conv2D_Fold']) - - folded_conv = g.get_operation_by_name(conv_name + '/Conv2D_Fold') - self.assertEqual(folded_conv.type, 'Conv2D') - self._AssertInputOpsAre(folded_conv, - [conv_name + '/mul_fold', inputs.op.name]) - self._AssertOutputGoesToOps(folded_conv, g, [conv_name + '/post_conv_mul']) - - folded_add = g.get_operation_by_name(conv_name + '/add_fold') - self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, [ - conv_name + '/correction_add', - self._BathNormBiasName(conv_name, fused_batch_norm) - ]) - output_op_names = ['test/AddV2' if with_bypass else 'test/' + relu_op_name] - self._AssertOutputGoesToOps(folded_add, g, output_op_names) - if freeze_batch_norm_delay is not None: - self._AssertMovingAveragesAreFrozen(g, name) - - for op in g.get_operations(): - self.assertFalse('//' in op.name, 'Double slash in op %s' % op.name) - - def testFoldConv2d(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - self._RunTestOverParameters(self._TestFoldConv2d) - - def testMultipleLayerConv2d(self, - relu=nn_ops.relu, - relu_op_name='Relu', - has_scaling=True, - fused_batch_norm=False, - freeze_batch_norm_delay=None, - insert_identity_node=False): - """Tests folding cases for a network with multiple layers. - - Args: - relu: Callable that returns an Operation, a factory method for the Relu*. - relu_op_name: String, name of the Relu* operation. - has_scaling: Bool, when true the batch norm has scaling. - fused_batch_norm: Bool, when true the batch norm is fused. - freeze_batch_norm_delay: None or the number of steps after which training - switches to using frozen mean and variance - insert_identity_node: Bool, insert identity node between conv and batch - norm - """ - g = ops.Graph() - with g.as_default(): - batch_size, height, width = 5, 128, 128 - inputs = array_ops.zeros((batch_size, height, width, 3)) - out_depth = 3 - stride = 1 - activation_fn = relu - scope = 'topnet/testnet' - with variable_scope.variable_scope(scope, [inputs]): - layer1 = conv2d( - inputs, - out_depth, [5, 5], - stride=stride, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - normalizer_fn=None, - scope='testnet/layer1') - # Add bn and relu with different scope - layer1 = batch_norm( - layer1, scale=has_scaling, fused=fused_batch_norm, scope='layer1') - layer1 = activation_fn(layer1) - layer2 = conv2d( - layer1, - 2 * out_depth, [5, 5], - stride=stride, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=self._BatchNormParams( - scale=has_scaling, fused=fused_batch_norm), - scope='testnet/layer2') - # Add bn and relu with different scope - layer2 = batch_norm( - layer2, scale=has_scaling, fused=fused_batch_norm, scope='layer2') - _ = activation_fn(layer2) - - scope = 'topnet/testnet/testnet/layer2' - - fold_batch_norms.FoldBatchNorms( - g, is_training=True, freeze_batch_norm_delay=freeze_batch_norm_delay) - folded_mul = g.get_operation_by_name(scope + '/mul_fold') - self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, [ - scope + '/correction_mult', - self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm) - ]) - self._AssertOutputGoesToOps(folded_mul, g, [scope + '/Conv2D_Fold']) - - folded_conv = g.get_operation_by_name(scope + '/Conv2D_Fold') - self.assertEqual(folded_conv.type, 'Conv2D') - # Remove :0 at end of name for tensor prior to comparison - self._AssertInputOpsAre(folded_conv, - [scope + '/mul_fold', layer1.name[:-2]]) - self._AssertOutputGoesToOps(folded_conv, g, [scope + '/post_conv_mul']) - - folded_add = g.get_operation_by_name(scope + '/add_fold') - self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, [ - scope + '/correction_add', - self._BathNormBiasName(scope, fused_batch_norm) - ]) - output_op_names = [scope + '/' + relu_op_name] - self._AssertOutputGoesToOps(folded_add, g, output_op_names) - if freeze_batch_norm_delay is not None: - self._AssertMovingAveragesAreFrozen(g, scope) - - for op in g.get_operations(): - self.assertFalse('//' in op.name, 'Double slash in op %s' % op.name) - - def _TestFoldConv2dUnknownShape(self, - relu, - relu_op_name, - with_bypass, - has_scaling, - fused_batch_norm, - freeze_batch_norm_delay, - insert_identity_node=False): - """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*. - - Tests that folding works even with an input shape where some dimensions are - not known (i.e. None). - - Args: - relu: Callable that returns an Operation, a factory method for the Relu*. - relu_op_name: String, name of the Relu* operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Relu*. - has_scaling: Bool, when true the batch norm has scaling. - fused_batch_norm: Bool, when true the batch norm is fused. - freeze_batch_norm_delay: None or the number of steps after which training - switches to using frozen mean and variance - insert_identity_node: Bool, insert identity node between conv and batch - norm - """ - g = ops.Graph() - with g.as_default(): - inputs = array_ops.placeholder(dtypes.float32, shape=(5, None, None, 3)) - out_depth = 3 if with_bypass else 32 - stride = 1 if with_bypass else 2 - activation_fn = None if with_bypass else relu - scope = 'test/test2' if with_bypass else 'test' - node = conv2d( - inputs, - out_depth, [5, 5], - stride=stride, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=self._BatchNormParams( - scale=has_scaling, fused=fused_batch_norm), - scope=scope) - if with_bypass: - node = math_ops.add(inputs, node, name='test/AddV2') - relu(node, name='test/' + relu_op_name) - - fold_batch_norms.FoldBatchNorms( - g, is_training=True, freeze_batch_norm_delay=freeze_batch_norm_delay) - - folded_mul = g.get_operation_by_name(scope + '/mul_fold') - self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, [ - scope + '/correction_mult', - self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm) - ]) - self._AssertOutputGoesToOps(folded_mul, g, [scope + '/Conv2D_Fold']) - - folded_conv = g.get_operation_by_name(scope + '/Conv2D_Fold') - self.assertEqual(folded_conv.type, 'Conv2D') - self._AssertInputOpsAre(folded_conv, [scope + '/mul_fold', inputs.op.name]) - self._AssertOutputGoesToOps(folded_conv, g, [scope + '/post_conv_mul']) - - folded_add = g.get_operation_by_name(scope + '/add_fold') - self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, [ - scope + '/correction_add', - self._BathNormBiasName(scope, fused_batch_norm) - ]) - output_op_names = ['test/AddV2' if with_bypass else 'test/' + relu_op_name] - self._AssertOutputGoesToOps(folded_add, g, output_op_names) - if freeze_batch_norm_delay is not None: - self._AssertMovingAveragesAreFrozen(g, scope) - - for op in g.get_operations(): - self.assertFalse('//' in op.name, 'Double slash in op %s' % op.name) - - def testFoldConv2dUnknownShape(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - self._RunTestOverParameters(self._TestFoldConv2dUnknownShape) - - def _TestFoldFullyConnectedLayer( - self, relu, relu_op_name, with_bypass, has_scaling, fused_batch_norm, - freeze_batch_norm_delay, insert_identity_node): - """Tests folding cases: inputs -> FC with batch norm -> Relu*. - - Args: - relu: Callable that returns an Operation, a factory method for the Relu*. - relu_op_name: String, name of the Relu* operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Relu*. - has_scaling: Bool, when true the batch norm has scaling. - fused_batch_norm: Bool, when true the batch norm is fused. - freeze_batch_norm_delay: None or the number of steps after which training - switches to using frozen mean and variance - insert_identity_node: Bool, insert identity node between conv and batch - norm - """ - g = ops.Graph() - with g.as_default(): - batch_size, depth = 5, 256 - inputs = array_ops.zeros((batch_size, depth)) - out_depth = 256 if with_bypass else 128 - activation_fn = None if with_bypass else relu - name = 'test/test2' if with_bypass else 'test' - insert_identity_node = fused_batch_norm - if insert_identity_node: - with g.name_scope(name): - node = fully_connected( - inputs, - out_depth, - weights_initializer=self._WeightInit(0.03), - activation_fn=None, - normalizer_fn=None, - biases_initializer=None) - node = array_ops.identity(node, name='fc_out') - - node = batch_norm( - node, - center=True, - scale=has_scaling, - decay=1.0 - 0.003, - fused=fused_batch_norm) - if activation_fn is not None: - node = activation_fn(node) - fc_name = name + '/fully_connected' - else: - - node = fully_connected( - inputs, - out_depth, - weights_initializer=self._WeightInit(0.03), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=self._BatchNormParams( - scale=has_scaling, fused=fused_batch_norm), - scope=name) - fc_name = name - if with_bypass: - node = math_ops.add(inputs, node, name='test/AddV2') - relu(node, name='test/' + relu_op_name) - - fold_batch_norms.FoldBatchNorms( - g, is_training=True, freeze_batch_norm_delay=freeze_batch_norm_delay) - - folded_mul = g.get_operation_by_name(fc_name + '/mul_fold') - self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, [ - fc_name + '/correction_mult', - self._BatchNormMultiplierName(fc_name, has_scaling, fused_batch_norm) - ]) - self._AssertOutputGoesToOps(folded_mul, g, [fc_name + '/MatMul_Fold']) - - folded_conv = g.get_operation_by_name(fc_name + '/MatMul_Fold') - self.assertEqual(folded_conv.type, 'MatMul') - self._AssertInputOpsAre(folded_conv, - [fc_name + '/mul_fold', inputs.op.name]) - self._AssertOutputGoesToOps(folded_conv, g, [fc_name + '/post_conv_mul']) - - folded_add = g.get_operation_by_name(fc_name + '/add_fold') - self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, [ - fc_name + '/correction_add', - self._BathNormBiasName(fc_name, fused_batch_norm) - ]) - output_op_names = ['test/AddV2' if with_bypass else 'test/' + relu_op_name] - self._AssertOutputGoesToOps(folded_add, g, output_op_names) - if freeze_batch_norm_delay is not None: - self._AssertMovingAveragesAreFrozen(g, name) - - for op in g.get_operations(): - self.assertFalse('//' in op.name, 'Double slash in op %s' % op.name) - - def testFoldFullyConnectedLayer(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - self._RunTestOverParameters(self._TestFoldFullyConnectedLayer) - - def _TestFoldDepthwiseConv2d(self, relu, relu_op_name, with_bypass, - has_scaling, fused_batch_norm, - freeze_batch_norm_delay, insert_identity_node): - """Tests folding: inputs -> DepthwiseConv2d with batch norm -> Relu*. - - Args: - relu: Callable that returns an Operation, a factory method for the Relu*. - relu_op_name: String, name of the Relu* operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Relu*. - has_scaling: Bool, when true the batch norm has scaling. - fused_batch_norm: Bool, when true the batch norm is fused. - freeze_batch_norm_delay: None or the number of steps after which training - insert_identity_node: Bool, insert identity node between conv and batch - norm switches to using frozen mean and variance - """ - g = ops.Graph() - with g.as_default(): - batch_size, height, width = 5, 128, 128 - inputs = array_ops.zeros((batch_size, height, width, 3)) - stride = 1 if with_bypass else 2 - activation_fn = None if with_bypass else relu - name = 'test/test2' if with_bypass else 'test' - if insert_identity_node: - with g.name_scope(name): - node = separable_conv2d( - inputs, - None, [5, 5], - stride=stride, - depth_multiplier=1.0, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - normalizer_fn=None, - biases_initializer=None) - node = array_ops.identity(node, name='sep_conv_out') - - node = batch_norm( - node, - center=True, - scale=has_scaling, - decay=1.0 - 0.003, - fused=fused_batch_norm) - if activation_fn is not None: - node = activation_fn(node) - sep_conv_name = name + '/SeparableConv2d' - else: - node = separable_conv2d( - inputs, - None, [5, 5], - stride=stride, - depth_multiplier=1.0, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=self._BatchNormParams( - scale=has_scaling, fused=fused_batch_norm), - scope=name) - sep_conv_name = name - if with_bypass: - node = math_ops.add(inputs, node, name='test/AddV2') - relu(node, name='test/' + relu_op_name) - - fold_batch_norms.FoldBatchNorms( - g, is_training=True, freeze_batch_norm_delay=freeze_batch_norm_delay) - - folded_mul = g.get_operation_by_name(sep_conv_name + '/mul_fold') - self.assertEqual(folded_mul.type, 'Mul') - if fused_batch_norm: - scale_reshape_op_name = sep_conv_name + '/BatchNorm_Fold/scale_reshape' - else: - scale_reshape_op_name = sep_conv_name + '/scale_reshape' - self._AssertInputOpsAre( - folded_mul, [sep_conv_name + '/correction_mult', scale_reshape_op_name]) - self._AssertOutputGoesToOps(folded_mul, g, - [sep_conv_name + '/depthwise_Fold']) - - scale_reshape = g.get_operation_by_name(scale_reshape_op_name) - self.assertEqual(scale_reshape.type, 'Reshape') - self._AssertInputOpsAre(scale_reshape, [ - self._BatchNormMultiplierName(sep_conv_name, has_scaling, - fused_batch_norm), - scale_reshape_op_name + '/shape' - ]) - self._AssertOutputGoesToOps(scale_reshape, g, [sep_conv_name + '/mul_fold']) - - folded_conv = g.get_operation_by_name(sep_conv_name + '/depthwise_Fold') - self.assertEqual(folded_conv.type, 'DepthwiseConv2dNative') - self._AssertInputOpsAre(folded_conv, - [sep_conv_name + '/mul_fold', inputs.op.name]) - self._AssertOutputGoesToOps(folded_conv, g, - [sep_conv_name + '/post_conv_mul']) - - folded_add = g.get_operation_by_name(sep_conv_name + '/add_fold') - self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, [ - sep_conv_name + '/correction_add', - self._BathNormBiasName(sep_conv_name, fused_batch_norm) - ]) - output_op_names = ['test/AddV2' if with_bypass else 'test/' + relu_op_name] - self._AssertOutputGoesToOps(folded_add, g, output_op_names) - if freeze_batch_norm_delay is not None: - self._AssertMovingAveragesAreFrozen(g, name) - - for op in g.get_operations(): - self.assertFalse('//' in op.name, 'Double slash in op %s' % op.name) - - def testFoldDepthwiseConv2d(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - self._RunTestOverParameters(self._TestFoldDepthwiseConv2d) - - def _TestFoldAtrousConv2d(self, relu, relu_op_name, with_bypass, has_scaling, - fused_batch_norm, freeze_batch_norm_delay, - insert_identity_node): - """Tests folding: inputs -> AtrousConv2d with batch norm -> Relu*. - - Args: - relu: Callable that returns an Operation, a factory method for the Relu*. - relu_op_name: String, name of the Relu* operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Relu*. - has_scaling: Bool, when true the batch norm has scaling. - fused_batch_norm: Bool, when true the batch norm is fused. - freeze_batch_norm_delay: None or the number of steps after which training - switches to using frozen mean and variance - insert_identity_node: Bool, insert identity node between conv and batch - norm - """ - g = ops.Graph() - with g.as_default(): - batch_size, height, width = 5, 128, 128 - inputs = array_ops.zeros((batch_size, height, width, 3)) - dilation_rate = 2 - activation_fn = None if with_bypass else relu - name = 'test/test2' if with_bypass else 'test' - if insert_identity_node: - with g.name_scope(name): - node = separable_conv2d( - inputs, - None, [3, 3], - rate=dilation_rate, - depth_multiplier=1.0, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - normalizer_fn=None, - biases_initializer=None) - node = array_ops.identity(node, name='sep_conv_out') - - node = batch_norm( - node, - center=True, - scale=has_scaling, - decay=1.0 - 0.003, - fused=fused_batch_norm) - if activation_fn is not None: - node = activation_fn(node) - sep_conv_name = name + '/SeparableConv2d' - else: - node = separable_conv2d( - inputs, - None, [3, 3], - rate=dilation_rate, - depth_multiplier=1.0, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=self._BatchNormParams( - scale=has_scaling, fused=fused_batch_norm), - scope=name) - sep_conv_name = name - if with_bypass: - node = math_ops.add(inputs, node, name='test/AddV2') - relu(node, name='test/' + relu_op_name) - - fold_batch_norms.FoldBatchNorms( - g, is_training=True, freeze_batch_norm_delay=freeze_batch_norm_delay) - - folded_mul = g.get_operation_by_name(sep_conv_name + '/mul_fold') - self.assertEqual(folded_mul.type, 'Mul') - if fused_batch_norm: - scale_reshape_op_name = sep_conv_name + '/BatchNorm_Fold/scale_reshape' - else: - scale_reshape_op_name = sep_conv_name + '/scale_reshape' - self._AssertInputOpsAre( - folded_mul, [sep_conv_name + '/correction_mult', scale_reshape_op_name]) - self._AssertOutputGoesToOps(folded_mul, g, - [sep_conv_name + '/depthwise_Fold']) - - scale_reshape = g.get_operation_by_name(scale_reshape_op_name) - self.assertEqual(scale_reshape.type, 'Reshape') - self._AssertInputOpsAre(scale_reshape, [ - self._BatchNormMultiplierName(sep_conv_name, has_scaling, - fused_batch_norm), - scale_reshape_op_name + '/shape' - ]) - self._AssertOutputGoesToOps(scale_reshape, g, [sep_conv_name + '/mul_fold']) - - folded_conv = g.get_operation_by_name(sep_conv_name + '/depthwise_Fold') - self.assertEqual(folded_conv.type, 'DepthwiseConv2dNative') - self._AssertInputOpsAre(folded_conv, [ - sep_conv_name + '/mul_fold', sep_conv_name + '/depthwise/SpaceToBatchND' - ]) - if fused_batch_norm: - self._AssertOutputGoesToOps(folded_conv, g, - [sep_conv_name + '/BatchToSpaceND_Fold']) - else: - self._AssertOutputGoesToOps( - folded_conv, g, [sep_conv_name + '/depthwise/BatchToSpaceND_Fold']) - - folded_add = g.get_operation_by_name(sep_conv_name + '/add_fold') - self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, [ - sep_conv_name + '/correction_add', - self._BathNormBiasName(sep_conv_name, fused_batch_norm) - ]) - output_op_names = ['test/AddV2' if with_bypass else 'test/' + relu_op_name] - self._AssertOutputGoesToOps(folded_add, g, output_op_names) - if freeze_batch_norm_delay is not None: - self._AssertMovingAveragesAreFrozen(g, name) - - for op in g.get_operations(): - self.assertFalse('//' in op.name, 'Double slash in op %s' % op.name) - - def testFoldAtrousConv2d(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - self._RunTestOverParameters(self._TestFoldAtrousConv2d) - - def _TestCompareFoldAndUnfolded(self, - relu, - relu_op_name, - with_bypass, - has_scaling, - fused_batch_norm, - freeze_batch_norm_delay, - insert_identity_node=False): - """Tests that running folded and unfolded BN returns the same results. - - Args: - relu: Callable that returns an Operation, a factory method for the Relu*. - relu_op_name: String, name of the Relu* operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Relu*. - has_scaling: Bool, when true the batch norm has scaling. - fused_batch_norm: Bool, when true the batch norm is fused. - freeze_batch_norm_delay: None or the number of steps after which training - switches to using frozen mean and variance - insert_identity_node: Bool, insert identity node between conv and batch - norm - """ - random_seed.set_random_seed(1234) - unfolded_g = ops.Graph() - with unfolded_g.as_default(): - batch_size, height, width = 5, 128, 128 - inputs = random_ops.random_uniform( - (batch_size, height, width, 3), dtype=dtypes.float32, seed=1234) - out_depth = 3 if with_bypass else 32 - stride = 1 if with_bypass else 2 - activation_fn = None if with_bypass else relu - scope = 'test/test2' if with_bypass else 'test' - node = conv2d( - inputs, - out_depth, [5, 5], - stride=stride, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=self._BatchNormParams( - scale=has_scaling, fused=fused_batch_norm), - scope=scope) - if with_bypass: - node = math_ops.add(inputs, node, name='test/AddV2') - relu_node = relu(node, name='test/' + relu_op_name) - folded_g = self._CopyGraph(unfolded_g) - with folded_g.as_default(): - fold_batch_norms.FoldBatchNorms( - folded_g, - is_training=True, - freeze_batch_norm_delay=freeze_batch_norm_delay) - with session.Session(graph=unfolded_g) as sess: - sess.run(variables.global_variables_initializer()) - grad_node = gradients.gradients(relu_node, inputs) - results = sess.run([relu_node, grad_node]) - unfolded_forward, unfolded_backward = results[0], results[1] - - with session.Session(graph=folded_g) as sess: - sess.run(variables.global_variables_initializer()) - relu_node = folded_g.get_tensor_by_name(relu_node.name) - inputs = folded_g.get_tensor_by_name(inputs.name) - grad_node = gradients.gradients(relu_node, inputs) - results = sess.run([relu_node, grad_node]) - folded_forward, folded_backward = results[0], results[1] - - # Check that the folded and unfolded results match. - self.assertAllClose(unfolded_forward, folded_forward, atol=1e-3) - self.assertAllClose(unfolded_backward, folded_backward, atol=1e-3) - - def testCompareFoldAndUnfolded(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - self._RunTestOverParameters(self._TestCompareFoldAndUnfolded) - - def _BatchNormParams(self, scale=True, fused=False): - return { - 'center': True, - 'scale': scale, - 'decay': 1.0 - 0.003, - 'fused': fused - } - - def _BatchNormMultiplierName(self, scope, has_scaling, fused): - if has_scaling: - if fused: - return scope + '/BatchNorm_Fold/mul' - return scope + '/BatchNorm/batchnorm_1/mul' - return scope + '/BatchNorm/batchnorm_1/Rsqrt' - - def _BathNormBiasName(self, scope, fused): - if fused: - return scope + '/BatchNorm_Fold/bias' - return scope + '/BatchNorm/batchnorm_1/sub' - - def _WeightInit(self, stddev): - """Returns a truncated normal variable initializer. - - Function is defined purely to shorten the name so that it stops wrapping. - - Args: - stddev: Standard deviation of normal variable. - - Returns: - An initializer that initializes with a truncated normal variable. - """ - return init_ops.truncated_normal_initializer(stddev=stddev, seed=1234) - - def _AssertInputOpsAre(self, op, in_op_names): - """Asserts that all inputs to op come from in_op_names (disregarding order). - - Args: - op: Operation to check inputs for. - in_op_names: List of strings, operations where all op's inputs should - come from. - """ - expected_inputs = [in_op_name + ':0' for in_op_name in in_op_names] - self.assertItemsEqual([t.name for t in op.inputs], expected_inputs) - - def _AssertOutputGoesToOps(self, op, graph, out_op_names): - """Asserts that outputs from op go to out_op_names (and perhaps others). - - Args: - op: Operation to check outputs for. - graph: Graph where output operations are located. - out_op_names: List of strings, operations where op's outputs should go. - """ - for out_op_name in out_op_names: - out_op = graph.get_operation_by_name(out_op_name) - self.assertIn(op.outputs[0].name, [str(t.name) for t in out_op.inputs]) - - def _AssertMovingAveragesAreFrozen(self, graph, scope): - """Asserts to check if moving mean and variance are frozen. - - Args: - graph: Graph where the operations are located. - scope: Scope of batch norm op - """ - moving_average_mult = graph.get_operation_by_name( - scope + '/BatchNorm/AssignMovingAvg/mul') - self.assertTrue( - moving_average_mult.inputs[1].name.find('freeze_moving_mean/Merge') > 0) - moving_var_mult = graph.get_operation_by_name( - scope + '/BatchNorm/AssignMovingAvg_1/mul') - self.assertTrue( - moving_var_mult.inputs[1].name.find('freeze_moving_var/Merge') > 0) - - def _CopyGraph(self, graph): - """Return a copy of graph.""" - meta_graph = saver_lib.export_meta_graph( - graph=graph, collection_list=graph.get_all_collection_keys()) - graph_copy = ops.Graph() - with graph_copy.as_default(): - _ = saver_lib.import_meta_graph(meta_graph) - return graph_copy - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/quantize/python/graph_matcher.py b/tensorflow/contrib/quantize/python/graph_matcher.py deleted file mode 100644 index cfbf5bf30f9..00000000000 --- a/tensorflow/contrib/quantize/python/graph_matcher.py +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities that match patterns in a tf.Graph.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc -import itertools - -import six - - -@six.add_metaclass(abc.ABCMeta) -class Pattern(object): - """The parent class of all patterns (e.g. OpTypePattern and OneofPattern).""" - - @abc.abstractmethod - def match(self, op, tensor): - """Returns the result of matching op/tensor against this pattern.""" - raise NotImplementedError('Method "match" not implemented.') - - -class OpTypePattern(Pattern): - """A tree pattern that matches TF expressions with certain op types.""" - - def __init__(self, op_type, name=None, inputs=None, ordered_inputs=True): - """Initializes an OpTypePattern. - - Args: - op_type: string that specifies the allowed types of the root. It can be - (1) an op type, e.g. 'Conv2D', - (2) '*', i.e. wildcard, or - (3) multiple op types separated by '|', e.g., 'Relu|Relu6'. - We could use regex strings, which might be worthwhile when we have many - similar TF op types. - name: Optional string. The name of the pattern that can be looked up in - MatchResult. - inputs: Optional list of `Pattern`s or strings that specify the - patterns for the inputs of a matching op. If None, this pattern accepts - any inputs of a matching op. - ordered_inputs: Defaults to True. If False, will match any op that - matches a permutation of the inputs. - - Raises: - ValueError: if too many inputs are provided when order_inputs is False. - """ - self._op_type = op_type - self._name = name - if inputs is None: - inputs = [] - if len(inputs) > 8: - raise ValueError( - 'Only < 8 inputs are allowed when ordered_inputs is False.') - self._inputs = [ - input_pattern - if isinstance(input_pattern, Pattern) else OpTypePattern(input_pattern) - for input_pattern in inputs - ] - self._ordered_inputs = ordered_inputs - - @property - def name(self): - return self._name - - def match(self, op, tensor): - if self._op_type != '*': - if op.type not in self._op_type.split('|'): - return None - - match_result = MatchResult() - match_result.add(self, op, tensor) - - if not self._inputs: - # If pattern.inputs is empty, skips the rest and accepts all the inputs. - return match_result - - if len(op.inputs) != len(self._inputs): - return None - - input_patterns_list = [self._inputs] - # If order doesn't matter for the inputs, then make sure we match at least - # one permutation of the inputs. - if not self._ordered_inputs: - input_patterns_list = list(itertools.permutations(self._inputs)) - - for input_patterns in input_patterns_list: - match_failed = False - for input_tensor, input_pattern in zip(op.inputs, input_patterns): - input_match_result = input_pattern.match(input_tensor.op, input_tensor) - if input_match_result is None: - match_failed = True - break - match_result.merge_from(input_match_result) - if not match_failed: - return match_result - return None - - -class OneofPattern(Pattern): - """Matches one of the given sub-patterns.""" - - def __init__(self, sub_patterns): - self._sub_patterns = sub_patterns - - def match(self, op, tensor): - for sub_pattern in self._sub_patterns: - match_result = sub_pattern.match(op, tensor) - if match_result is not None: - return match_result - return None - - -class MatchResult(object): - r"""Encapsulates the result of a match done by GraphMatcher. - - MatchResult contains a map from Pattern to the matching op and tensor. - When the matching op has multiple output tensors, the matching tensor is the - output tensor used by the matching op of the parent pattern. E.g., when we - match graph - - - + - / \y0 y1/ \ - x split z - | - y (nodes are ops; edges are going up) - - against add_pattern defined as - - y1_pattern = OpTypePattern('*') - z_pattern = OpTypePattern('*') - add_pattern = OpTypePattern('+', inputs=[y1_pattern, z_pattern]) - - the matching op of `y1_pattern` is `split`, and the matching tensor of - `y1_pattern` - is `y1` not `y0`. - """ - - def __init__(self): - self._pattern_to_op_tensor = {} - self._name_to_pattern = {} - - def add(self, pattern, op, tensor): - self._pattern_to_op_tensor[pattern] = op, tensor - if pattern.name is not None: - if pattern.name in self._name_to_pattern: - raise ValueError( - 'Name %s is already bound to another pattern' % pattern.name) - self._name_to_pattern[pattern.name] = pattern - - def _to_pattern(self, pattern_or_name): - if isinstance(pattern_or_name, Pattern): - return pattern_or_name - - if isinstance(pattern_or_name, str): - if pattern_or_name not in self._name_to_pattern: - return None - return self._name_to_pattern[pattern_or_name] - - raise ValueError('pattern_or_name has type %s. Expect Pattern or str.' % - type(pattern_or_name)) - - def _get_op_tensor(self, pattern_or_name): - pattern = self._to_pattern(pattern_or_name) - if pattern is None: - return None - - if pattern not in self._pattern_to_op_tensor: - return None - - return self._pattern_to_op_tensor[pattern] - - def get_op(self, pattern_or_name): - op_tensor = self._get_op_tensor(pattern_or_name) - return op_tensor[0] if op_tensor else None - - def get_tensor(self, pattern_or_name): - op_tensor = self._get_op_tensor(pattern_or_name) - return op_tensor[1] if op_tensor else None - - def merge_from(self, other_match_result): - # pylint: disable=protected-access - self._pattern_to_op_tensor.update(other_match_result._pattern_to_op_tensor) - self._name_to_pattern.update(other_match_result._name_to_pattern) - # pylint: enable=protected-access - - -class GraphMatcher(object): - """Checks if a particular subgraph matches a given pattern.""" - - def __init__(self, pattern): - """Initializes a GraphMatcher. - - Args: - pattern: The `Pattern` against which `GraphMatcher` matches - subgraphs. - """ - self._pattern = pattern - - def _match_pattern(self, pattern, op, tensor): - """Returns whether an TF expression rooted at `op` matches `pattern`. - - If there is a match, adds to `self._match_result` the matching op and tensor - with key `pattern`. - - Args: - pattern: An `Pattern`. - op: A `tf.Operation` to match against the pattern. - tensor: the output `tf.Tensor` of `op` that is used by the matching op of - `pattern`'s parent. Can be None if `pattern` is already the root of the - pattern tree. - - Returns: - True if an TF expression rooted at `op` matches `pattern`. - """ - match_result = pattern.match(op, tensor) - if match_result is None: - return False - self._match_result.merge_from(match_result) - return True - - def match_op(self, op): - """Matches `op` against `self._pattern`. - - Args: - op: `tf.Operation` to match against the pattern. - - Returns: - Returns a `MatchResult` if `op` matches the pattern; otherwise, returns - None. - """ - self._match_result = MatchResult() - if not self._match_pattern(self._pattern, op, tensor=None): - return None - return self._match_result - - def match_ops(self, ops): - """Matches each operation in `ops` against `self._pattern`. - - Args: - ops: collection of `tf.Operation` to match against the pattern. - - Yields: - `MatchResult` for each `tf.Operation` that matches the pattern. - """ - for op in ops: - match_result = self.match_op(op) - if match_result: - yield match_result - - def match_graph(self, graph): - """Matches each operation in `graph` against `self._pattern`. - - Args: - graph: `tf.Graph` containing operations to match. - - Yields: - `MatchResult` for each `tf.Operation` in `graph` that matches the pattern. - """ - # Python 3.3.2+ implements `yield from`, but for now: - for match_result in self.match_ops(graph.get_operations()): - yield match_result diff --git a/tensorflow/contrib/quantize/python/graph_matcher_test.py b/tensorflow/contrib/quantize/python/graph_matcher_test.py deleted file mode 100644 index bc916aaf174..00000000000 --- a/tensorflow/contrib/quantize/python/graph_matcher_test.py +++ /dev/null @@ -1,211 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for graph_matcher.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework.python import ops as contrib_ops -from tensorflow.contrib.layers.python.layers import initializers -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.quantize.python import graph_matcher -from tensorflow.python.compat import compat -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.platform import googletest - - -class GraphMatcherTest(test_util.TensorFlowTestCase): - - def test_conv_layer(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - g = ops.Graph() - with g.as_default(): - inputs = array_ops.placeholder(dtypes.float32, shape=[8, 5, 5, 3]) - - with contrib_ops.arg_scope([layers.batch_norm], - fused=True, - is_training=True, - trainable=True): - return layers.convolution( - inputs, - num_outputs=16, - kernel_size=3, - stride=1, - padding='VALID', - activation_fn=nn_ops.relu, - normalizer_fn=layers.batch_norm, - normalizer_params={}, - weights_initializer=initializers.xavier_initializer(), - weights_regularizer=None, - biases_initializer=init_ops.zeros_initializer(), - biases_regularizer=None, - reuse=None, - trainable=True, - scope=None) - - inputs_pattern = graph_matcher.OpTypePattern('*', name='inputs') - relu_pattern = graph_matcher.OpTypePattern( - 'Relu', - name='relu', - inputs=[ - graph_matcher.OpTypePattern( - 'FusedBatchNormV3', - inputs=[ - graph_matcher.OpTypePattern( - 'Conv2D', inputs=[inputs_pattern, '*']), '*', '*', - '*', '*' - ]) - ]) - matcher = graph_matcher.GraphMatcher(relu_pattern) - match_results = list(matcher.match_graph(g)) - self.assertEqual(1, len(match_results)) - match_result = match_results[0] - self.assertEqual(match_result.get_tensor(inputs_pattern), inputs) - self.assertEqual(match_result.get_tensor('inputs'), inputs) - - def test_multiple_outputs(self): - # - + - # / \y0 y1/ \ - # x split z - # | - # y (nodes are ops; edges are going up) - g = ops.Graph() - with g.as_default(): - x = array_ops.placeholder(dtypes.float32, shape=[1], name='x') - y = array_ops.placeholder(dtypes.float32, shape=[2], name='y') - y0, y1 = array_ops.split(y, num_or_size_splits=2, axis=0) - z = array_ops.placeholder(dtypes.float32, shape=[1], name='z') - math_ops.add(x, y0) - math_ops.subtract(y1, z) - - y1_pattern = graph_matcher.OpTypePattern('*') - minus_pattern = graph_matcher.OpTypePattern('Sub', inputs=[y1_pattern, '*']) - matcher = graph_matcher.GraphMatcher(minus_pattern) - - match_results = list(matcher.match_graph(g)) - self.assertEqual(1, len(match_results)) - match_result = match_results[0] - - self.assertEqual(y0.op, y1.op) - self.assertEqual(match_result.get_op(y1_pattern), y1.op) - self.assertEqual(match_result.get_tensor(y1_pattern), y1) - - def test_oneof_type_pattern(self): - # - + - # / \ / \ - # x y z - g = ops.Graph() - with g.as_default(): - x = array_ops.placeholder(dtypes.float32, shape=[], name='x') - y = array_ops.placeholder(dtypes.float32, shape=[], name='y') - z = array_ops.placeholder(dtypes.float32, shape=[], name='z') - plus = x + y - minus = y - z - - add_or_sub_pattern = graph_matcher.OpTypePattern( - 'AddV2|Add|Sub', inputs=['*', '*']) - matcher = graph_matcher.GraphMatcher(add_or_sub_pattern) - self.assertEqual([ - match_result.get_op(add_or_sub_pattern) - for match_result in matcher.match_graph(g) - ], [plus.op, minus.op]) - - def test_oneof_pattern(self): - reshape_pattern = graph_matcher.OpTypePattern('Reshape') - transpose_pattern = graph_matcher.OneofPattern([ - graph_matcher.OpTypePattern( - 'Transpose', - name='transpose', - inputs=[ - graph_matcher.OpTypePattern( - 'Slice', name='slice', inputs=[reshape_pattern, '*', '*']), - '*' - ]), - graph_matcher.OpTypePattern( - 'Transpose', name='transpose', inputs=[reshape_pattern, '*']) - ]) - - matcher = graph_matcher.GraphMatcher(transpose_pattern) - - g = ops.Graph() - with g.as_default(): - inputs = array_ops.placeholder(dtypes.float32, shape=[6]) - reshape = array_ops.reshape(inputs, [2, 3]) - transpose = array_ops.transpose(reshape) - [match_result] = list(matcher.match_graph(g)) - self.assertEqual(match_result.get_tensor(reshape_pattern), reshape) - self.assertEqual(match_result.get_tensor('slice'), None) - self.assertEqual(match_result.get_op('transpose'), transpose.op) - - g = ops.Graph() - with g.as_default(): - inputs = array_ops.placeholder(dtypes.float32, shape=[6]) - reshape = array_ops.reshape(inputs, [2, 3]) - slicing = array_ops.slice(reshape, [0, 0], [-1, -1]) - transpose = array_ops.transpose(slicing) - [match_result] = list(matcher.match_graph(g)) - self.assertEqual(match_result.get_tensor(reshape_pattern), reshape) - self.assertEqual(match_result.get_tensor('slice'), slicing) - self.assertEqual(match_result.get_op('transpose'), transpose.op) - - def test_ordered_pattern(self): - # + + - # / \ / \ - # x y and y x should both match when ordered inputs is False. - # Even when x and y are different operations. - g = ops.Graph() - with g.as_default(): - x = array_ops.placeholder(dtypes.float32, shape=[], name='x') - y = constant_op.constant(1.0, dtype=dtypes.float32) - plus = x + y - - add_pattern_a = graph_matcher.OpTypePattern( - 'Add|AddV2', inputs=['Const', 'Placeholder'], ordered_inputs=False) - add_pattern_b = graph_matcher.OpTypePattern( - 'Add|AddV2', inputs=['Placeholder', 'Const'], ordered_inputs=False) - add_pattern_fail = graph_matcher.OpTypePattern( - 'Add|AddV2', inputs=['Const', 'Placeholder'], ordered_inputs=True) - # Both add_pattern_a and add_pattern_b should match the graph since - # ordered_input was set False. - matcher_a = graph_matcher.GraphMatcher(add_pattern_a) - self.assertEqual([ - match_result.get_op(add_pattern_a) - for match_result in matcher_a.match_graph(g) - ], [plus.op]) - matcher_b = graph_matcher.GraphMatcher(add_pattern_b) - self.assertEqual([ - match_result.get_op(add_pattern_b) - for match_result in matcher_b.match_graph(g) - ], [plus.op]) - # But if ordered_inputs is True, the inputs list match should fail if not - # specified in the right order. - matcher_fail = graph_matcher.GraphMatcher(add_pattern_fail) - self.assertEqual( - len([ - match_result.get_op(add_pattern_fail) - for match_result in matcher_fail.match_graph(g) - ]), 0) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/quantize/python/input_to_ops.py b/tensorflow/contrib/quantize/python/input_to_ops.py deleted file mode 100644 index 98755607771..00000000000 --- a/tensorflow/contrib/quantize/python/input_to_ops.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Logic to update a Tensorflow model graph with quantization operations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -from tensorflow.contrib.quantize.python import common - - -class InputToOps(object): - """Holds a mapping from tensor's name to ops that take it as input.""" - - def __init__(self, graph): - """Initializes mapping from tensor's name to ops that take it. - - Helps find edges between ops faster and avoids iterating over the whole - graph. The mapping is of type Dict[str, Set[tf.Operation]]. - - Note: while inserting operations into the graph, we do not update the - mapping, assuming that insertion points in the graph are never adjacent. - With that restriction, an out of date mapping still works fine. - - Args: - graph: Graph to process. - """ - self.mapping = collections.defaultdict(set) - for op in (op for op in graph.get_operations()): - if op.name.startswith(common.SKIPPED_PREFIXES): - continue - for op_input in op.inputs: - self.mapping[op_input].add(op) - - def ConsumerOperations(self, producer_op): - """Looks through outputs of producer_op, finds ops that take them as input. - - Args: - producer_op: Operation containing outputs to process. - - Returns: - A Set[Operation] containing all operations taking input from producer_op - outputs. - """ - result = set() - for inp in producer_op.outputs: - result.update(self.mapping[inp]) - return result diff --git a/tensorflow/contrib/quantize/python/input_to_ops_test.py b/tensorflow/contrib/quantize/python/input_to_ops_test.py deleted file mode 100644 index 9dbd1eb7118..00000000000 --- a/tensorflow/contrib/quantize/python/input_to_ops_test.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Unit tests for InputToOps class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.quantize.python import input_to_ops -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.platform import googletest - - -class InputToOpsTest(test_util.TensorFlowTestCase): - - def testNoConsumerOperations(self): - graph = ops.Graph() - with graph.as_default(): - input_tensor = array_ops.zeros((1, 2, 3, 4)) - - input_to_ops_map = input_to_ops.InputToOps(graph) - consumer_operations = input_to_ops_map.ConsumerOperations(input_tensor.op) - - self.assertEqual(0, len(consumer_operations)) - - def testOneConsumerOperation(self): - graph = ops.Graph() - with graph.as_default(): - input_tensor = array_ops.zeros((1, 2, 3, 4)) - output_tensor = nn_ops.relu6(input_tensor) - - input_to_ops_map = input_to_ops.InputToOps(graph) - consumer_operations = input_to_ops_map.ConsumerOperations(input_tensor.op) - - self.assertEqual(consumer_operations, {output_tensor.op}) - - def testSeveralConsumerOperations(self): - graph = ops.Graph() - with graph.as_default(): - input_tensor = array_ops.zeros((1, 2, 3, 4)) - output_tensor_1 = nn_ops.relu6(input_tensor) - output_tensor_2 = input_tensor + output_tensor_1 - output_tensor_3 = input_tensor * output_tensor_2 - - input_to_ops_map = input_to_ops.InputToOps(graph) - consumer_operations = input_to_ops_map.ConsumerOperations(input_tensor.op) - - self.assertEqual(consumer_operations, - {output_tensor_1.op, output_tensor_2.op, - output_tensor_3.op}) - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/quantize/python/quant_ops.py b/tensorflow/contrib/quantize/python/quant_ops.py deleted file mode 100644 index ecee2549683..00000000000 --- a/tensorflow/contrib/quantize/python/quant_ops.py +++ /dev/null @@ -1,342 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python support for quantization operations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.training import moving_averages - - -def FixedQuantize(inputs, init_min=-6.0, init_max=6.0, scope=None): - """Adds a fake quantize layer with fixed quantization interval. - - Args: - inputs: a tensor containing values to be quantized. - init_min: the lower end of quantization interval. - init_max: the upper end of quantization interval. - scope: Optional scope for name_scope. - Returns: - a tensor containing quantized values. - """ - with ops.name_scope(scope, 'FixedQuantize', values=[inputs]): - return array_ops.fake_quant_with_min_max_args( - inputs, min=init_min, max=init_max) - - -def _ModelVariable(name, - shape=None, - initializer=None, - collections=None, - trainable=None): - collections = list(collections or []) - collections += [ops.GraphKeys.GLOBAL_VARIABLES] - return variable_scope.get_variable( - name, - shape=shape, - initializer=initializer, - collections=collections, - trainable=trainable, - aggregation=variable_scope.VariableAggregation.MEAN) - - -def LastValueQuantize(inputs, - per_channel=False, - init_min=-6.0, - init_max=6.0, - vars_collection=None, - name_prefix='LastValueQuant', - reuse=None, - is_training=True, - num_bits=8, - narrow_range=False, - symmetric=False): - """Adds a layer that collects quantization ranges as last input ranges. - - LastValueQuantize creates variables called 'min' and 'max', representing the - interval used for quantization and clamping. - - Args: - inputs: a tensor containing values to be quantized. - per_channel: (Optional) a boolean specifying whether to use different - quantization ranges per output channel. - init_min: a float scalar, the initial value for variable min. - init_max: a float scalar, the initial value for variable max. - vars_collection: (Optional) collection where to store variables for - quantization interval ends. - name_prefix: name_prefix for created nodes. - reuse: whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - is_training: Whether the op is applied to a training or eval graph. - num_bits: Number of bits to use for quantization, must be between 2 and 8. - narrow_range: Whether to use the narrow quantization range - [1; 2^num_bits - 1] or wide range [0; 2^num_bits - 1]. - symmetric: If true, use symmetric quantization limits instead of training - the minimum and maximum of each quantization range separately. - Returns: - a tensor containing quantized values. - """ - with variable_scope.variable_scope( - None, default_name=name_prefix, values=[inputs], reuse=reuse) as scope: - scope.set_partitioner(None) - input_shape = inputs.get_shape() - input_dim = len(input_shape) - if per_channel: - # Only support quantizing 1-, 2- and 4-dimensional tensors. - assert input_dim in [1, 2, 4], ('Expected 1D, 2D or 4D input, was: %s in ' - ' scope: %s' % (input_shape, name_prefix)) - min_max_shape = [input_shape[-1]] - else: - min_max_shape = [] - - vars_collections = [vars_collection] if vars_collection else [] - min_var = _ModelVariable( - 'min', - shape=min_max_shape, - initializer=init_ops.constant_initializer(init_min), - collections=vars_collections, - trainable=False) - max_var = _ModelVariable( - 'max', - shape=min_max_shape, - initializer=init_ops.constant_initializer(init_max), - collections=vars_collections, - trainable=False) - if not is_training: - return _FakeQuantWithMinMaxVars( - inputs, - min_var, - max_var, - per_channel=per_channel, - num_bits=num_bits, - narrow_range=narrow_range) - - if per_channel: - if input_dim == 2: - reduce_dims = [0] - elif input_dim == 4: - reduce_dims = [0, 1, 2] - - if per_channel: - if input_dim >= 2: - batch_min = math_ops.reduce_min( - inputs, axis=reduce_dims, name='BatchMin') - else: - batch_min = inputs - else: - batch_min = math_ops.reduce_min(inputs, name='BatchMin') - - if per_channel: - if input_dim >= 2: - batch_max = math_ops.reduce_max( - inputs, axis=reduce_dims, name='BatchMax') - else: - batch_max = inputs - else: - batch_max = math_ops.reduce_max(inputs, name='BatchMax') - - if symmetric: - if narrow_range: - min_max_ratio = -1 - else: - # In two's complement notation, the negative range is slightly larger - # than the positive range. - min_max_ratio = -((1 << num_bits) - 2) / (1 << num_bits) - - # TFLite requires that 0.0 is always in the [min; max] range. Because - # batch_min <= batch_max, it follows that range_min <= 0 <= range_max. - range_min = math_ops.minimum(batch_min, batch_max / min_max_ratio) - range_max = math_ops.maximum(batch_max, batch_min * min_max_ratio) - else: - # TFLite requires that 0.0 is always in the [min; max] range. - range_min = math_ops.minimum(batch_min, 0.0) - range_max = math_ops.maximum(batch_max, 0.0) - - assign_min = state_ops.assign(min_var, range_min, name='AssignMinLast') - assign_max = state_ops.assign(max_var, range_max, name='AssignMaxLast') - - return _FakeQuantWithMinMaxVars( - inputs, - assign_min, - assign_max, - per_channel=per_channel, - num_bits=num_bits, - narrow_range=narrow_range) - - -def MovingAvgQuantize(inputs, - per_channel=False, - init_min=-6.0, - init_max=6.0, - ema_decay=0.999, - vars_collection=ops.GraphKeys.MOVING_AVERAGE_VARIABLES, - name_prefix='MovingAvgQuantize', - reuse=None, - is_training=True, - num_bits=8, - narrow_range=False, - symmetric=False): - """Adds a layer that collects quantization ranges as EMAs of input ranges. - - MovingAvgQuantize creates variables called 'min' and 'max', representing the - interval used for quantization and clamping. - - Args: - inputs: a tensor containing values to be quantized. - per_channel: (default False) a boolean specifying whether to use different - quantization ranges per output channel. - init_min: a float scalar, the initial value for variable min. - init_max: a float scalar, the initial value for variable max. - ema_decay: EMA decay parameter. - vars_collection: (Optional) collection where to store variables for - quantization interval ends. - name_prefix: name_prefix for created nodes. - reuse: whether or not the layer and its variables should be reused. To be - able to reuse the layer scope must be given. - is_training: Whether the op is applied to a training or eval graph. - num_bits: Number of bits to use for quantization, must be between 2 and 8. - narrow_range: Whether to use the narrow quantization range - [1; 2^num_bits - 1] or wide range [0; 2^num_bits - 1]. - symmetric: If true, use symmetric quantization limits instead of training - the minimum and maximum of each quantization range separately. - Returns: - a tensor containing quantized values. - """ - with variable_scope.variable_scope( - None, default_name=name_prefix, values=[inputs], reuse=reuse) as scope: - scope.set_partitioner(None) - input_shape = inputs.get_shape() - if per_channel: - input_dim = len(input_shape) - # Only support quantizing 1-, 2- and 4-dimensional tensors. - assert input_dim in [1, 2, 4], ('Expected 1D, 2D or 4D input, was: %s in ' - ' scope: %s' % (input_shape, name_prefix)) - min_max_shape = [input_shape[-1]] - else: - min_max_shape = [] - - vars_collections = [vars_collection] if vars_collection else [] - min_var = _ModelVariable( - 'min', - shape=min_max_shape, - initializer=init_ops.constant_initializer(init_min), - collections=vars_collections, - trainable=False) - max_var = _ModelVariable( - 'max', - shape=min_max_shape, - initializer=init_ops.constant_initializer(init_max), - collections=vars_collections, - trainable=False) - if not is_training: - return _FakeQuantWithMinMaxVars( - inputs, - min_var, - max_var, - per_channel=per_channel, - num_bits=num_bits, - narrow_range=narrow_range) - if per_channel: - if input_dim == 2: - reduce_dims = [0] - elif input_dim == 4: - reduce_dims = [0, 1, 2] - - if per_channel: - if input_dim >= 2: - batch_min = math_ops.reduce_min( - inputs, axis=reduce_dims, name='BatchMin') - else: - batch_min = inputs - else: - batch_min = math_ops.reduce_min(inputs, name='BatchMin') - - if per_channel: - if input_dim >= 2: - batch_max = math_ops.reduce_max( - inputs, axis=reduce_dims, name='BatchMax') - else: - batch_max = inputs - else: - batch_max = math_ops.reduce_max(inputs, name='BatchMax') - - if symmetric: - if narrow_range: - min_max_ratio = -1 - else: - # In two's complement notation, the negative range is slightly larger - # than the positive range. - min_max_ratio = -((1 << num_bits) - 2) / (1 << num_bits) - - # TFLite requires that 0.0 is always in the [min; max] range. Because - # batch_min <= batch_max, it follows that range_min <= 0 <= range_max. - range_min = math_ops.minimum(batch_min, batch_max / min_max_ratio) - range_max = math_ops.maximum(batch_max, batch_min * min_max_ratio) - else: - # TFLite requires that 0.0 is always in the [min; max] range. - range_min = math_ops.minimum(batch_min, 0.0) - range_max = math_ops.maximum(batch_max, 0.0) - - assign_min = moving_averages.assign_moving_average( - min_var, range_min, ema_decay, name='AssignMinEma') - assign_max = moving_averages.assign_moving_average( - max_var, range_max, ema_decay, name='AssignMaxEma') - - return _FakeQuantWithMinMaxVars( - inputs, - assign_min, - assign_max, - per_channel=per_channel, - num_bits=num_bits, - narrow_range=narrow_range) - - -def _FakeQuantWithMinMaxVars(inputs, min_var, max_var, per_channel, num_bits, - narrow_range): - """Adds a fake quantization operation. - - Depending on value of per_channel, this operation may do global quantization - or per channel quantization. min_var and max_var should have corresponding - shapes: [1] when per_channel == False and [d] when per_channel == True. - - Args: - inputs: a tensor containing values to be quantized. - min_var: a variable containing quantization range lower end(s). - max_var: a variable containing quantization range upper end(s). - per_channel: a boolean specifying whether to use per-channel quantization. - num_bits: Number of bits to use for quantization, must be between 2 and 8. - narrow_range: Whether to use the narrow quantization range - [1; 2^num_bits - 1] or wide range [0; 2^num_bits - 1]. - Returns: - a tensor containing quantized values. - """ - - if per_channel: - assert len(min_var.get_shape()) == 1 - assert len(max_var.get_shape()) == 1 - return array_ops.fake_quant_with_min_max_vars_per_channel( - inputs, min_var, max_var, num_bits=num_bits, narrow_range=narrow_range) - else: - assert min_var.get_shape() == [] # pylint: disable=g-explicit-bool-comparison - assert max_var.get_shape() == [] # pylint: disable=g-explicit-bool-comparison - return array_ops.fake_quant_with_min_max_vars( - inputs, min_var, max_var, num_bits=num_bits, narrow_range=narrow_range) diff --git a/tensorflow/contrib/quantize/python/quant_ops_test.py b/tensorflow/contrib/quantize/python/quant_ops_test.py deleted file mode 100644 index c636c90d23a..00000000000 --- a/tensorflow/contrib/quantize/python/quant_ops_test.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for third_party.tensorflow.contrib.quantize.python.quant_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.quantize.python import quant_ops -from tensorflow.python.client import session -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest - -_MIN_MAX_VARS = 'min_max_vars' -_SYMMETRIC_RANGE_RATIO = 0.9921875 # 127 / 128 - - -class QuantOpsTest(googletest.TestCase): - - def testLastValueQuantizeTrainingAssign(self): - min_value, max_value = self._GetMinMaxValues(quant_ops.LastValueQuantize, - [[-1, 1]]) - self.assertEqual(min_value, -1.0) - self.assertEqual(max_value, 1.0) - - def testLastValueSymmetricQuantizeTrainingAssign(self): - min_value, max_value = self._GetMinMaxValues( - quant_ops.LastValueQuantize, - [[-_SYMMETRIC_RANGE_RATIO, _SYMMETRIC_RANGE_RATIO]], - symmetric=True, - narrow_range=False) - self.assertEqual(min_value, -1.0) - self.assertEqual(max_value, _SYMMETRIC_RANGE_RATIO) - - def testLastValueSymmetricQuantizeNarrowRangeTrainingAssign(self): - min_value, max_value = self._GetMinMaxValues( - quant_ops.LastValueQuantize, [[-1, 0.5]], - symmetric=True, - narrow_range=True) - self.assertEqual(min_value, -1.0) - self.assertEqual(max_value, 1) - - def testMovingAvgQuantizeTrainingAssign(self): - min_value, max_value = self._GetMinMaxValues(quant_ops.MovingAvgQuantize, - [[-1, 1], [0, 0]]) - self.assertAlmostEqual(min_value, -0.5, delta=1e-3) - self.assertAlmostEqual(max_value, 0.5, delta=1e-3) - - def testMovingAvgQuantizeTrainingAssignNoShape(self): - min_value, max_value = self._GetMinMaxValues( - quant_ops.MovingAvgQuantize, [[-1, 1], [0, 0]], shape=None) - self.assertAlmostEqual(min_value, -0.5, delta=1e-3) - self.assertAlmostEqual(max_value, 0.5, delta=1e-3) - - def testMovingAvgSymmetricQuantizeTrainingAssign(self): - min_value, max_value = self._GetMinMaxValues( - quant_ops.MovingAvgQuantize, [[-1, 0.5], [0, 0]], symmetric=True) - self.assertAlmostEqual(min_value, -0.5, delta=1e-3) - self.assertAlmostEqual(max_value, 0.5 * _SYMMETRIC_RANGE_RATIO, delta=1e-3) - self.assertAlmostEqual(max_value, min_value * -_SYMMETRIC_RANGE_RATIO) - - def testMovingAvgSymmetricQuantizeNarrowRangeTrainingAssign(self): - min_value, max_value = self._GetMinMaxValues( - quant_ops.MovingAvgQuantize, [[-1, 0.5], [0, 0]], - symmetric=True, - narrow_range=True) - self.assertAlmostEqual(min_value, -0.5, delta=1e-3) - self.assertAlmostEqual(max_value, 0.5, delta=1e-3) - self.assertAlmostEqual(max_value, -min_value) - - def testVariablesNotPartitioned_LastValue(self): - # Variables added should not use a default partiioner since they are - # scalar. There would be a tensorflow error thrown if the partitioner was - # respected by the rewrite. - with ops.Graph().as_default(): - with variable_scope.variable_scope( - 'part', partitioner=partitioned_variables.fixed_size_partitioner(2)): - x = array_ops.placeholder(dtypes.float32, shape=[2]) - _ = quant_ops.LastValueQuantize( - x, - init_min=0.0, - init_max=0.0, - is_training=True, - vars_collection=_MIN_MAX_VARS) - - def testVariablesNotPartitioned_MovingAvg(self): - # Variables added should not use a default partiioner since they are - # scalar. There would be a tensorflow error thrown if the partitioner was - # respected by the rewrite. - with ops.Graph().as_default(): - with variable_scope.variable_scope( - 'part', partitioner=partitioned_variables.fixed_size_partitioner(2)): - x = array_ops.placeholder(dtypes.float32, shape=[2]) - _ = quant_ops.MovingAvgQuantize( - x, - init_min=0.0, - init_max=0.0, - is_training=True, - vars_collection=_MIN_MAX_VARS) - - def _GetMinMaxValues(self, quantize_fn, input_values, shape=(2), **kwds): - g = ops.Graph() - with session.Session(graph=g) as sess: - x = array_ops.placeholder(dtypes.float32, shape=shape) - y = quantize_fn( - x, - init_min=0.0, - init_max=0.0, - is_training=True, - vars_collection=_MIN_MAX_VARS, - **kwds) - - # Run the step. - sess.run(variables.global_variables_initializer()) - for input_elem in input_values: - sess.run(y, feed_dict={x: input_elem}) - - # Now check that the min_max_vars were, in fact, updated. - min_max_vars = ops.get_collection(_MIN_MAX_VARS) - self.assertEqual(len(min_max_vars), 2) - min_idx = 0 if 'min' in min_max_vars[0].name else 1 - max_idx = (min_idx + 1) % 2 - min_var, max_var = min_max_vars[min_idx], min_max_vars[max_idx] - min_max_values = sess.run([min_var, max_var]) - return min_max_values[0], min_max_values[1] - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/quantize/python/quantize.py b/tensorflow/contrib/quantize/python/quantize.py deleted file mode 100644 index a90647deed0..00000000000 --- a/tensorflow/contrib/quantize/python/quantize.py +++ /dev/null @@ -1,794 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Logic to update a TensorFlow model graph with quantization operations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re -from tensorflow.contrib.quantize.python import common -from tensorflow.contrib.quantize.python import graph_matcher -from tensorflow.contrib.quantize.python import input_to_ops -from tensorflow.contrib.quantize.python import quant_ops -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import tf_logging as logging - -# Quantizable operation types that are supported by the quantization rewrite. -_QUANTIZABLE_TYPES = {'Conv2D', 'MatMul', 'DepthwiseConv2dNative'} - -# Activations that are supported by the quantization rewrite. -_ACTIVATION_TYPES = {'Relu', 'Relu6', 'Identity'} - -_RELU_TYPES = {'Relu', 'Relu6'} - -_QUANTIZATION_OP = {'FakeQuantWithMinMaxVars'} -_VALID_SRC_OP = {'Add', 'AddV2', 'Mul'} -_INTERMEDIATE_OP = {'Add', 'AddV2', 'Mul'} -_PASS_THROUGH_OP = {'Reshape', 'Identity', 'BatchToSpaceND', 'SpaceToBatchND', - 'MaxPool', 'Max'} -_VALID_ACTIVATION_OP = {'Relu', 'Relu6'} - - -def Quantize(graph, - is_training, - weight_bits=8, - activation_bits=8, - symmetric=False, - ema_decay=0.999, - quant_delay=None, - vars_collection=ops.GraphKeys.GLOBAL_VARIABLES, - scope=None): - """Updates graph with quantization operations. - - Currently we quantize the following tensors: - * Conv/MatMul: Quantize the weights if it matches. - * Activation: Quantize the output if it matches. - * Bypass/Post-activation Bypass: Quantize both input and output - if it matches. - - Args: - graph: Graph to modify. - is_training: Whether quantizing training graph or eval graph. - weight_bits: Number of bits to use for quantizing weights. - activation_bits: Number of bits to use for quantizing activations. - symmetric: (Optional) If true, use symmetric quantization limits instead of - training the minimum and maximum of each quantization range separately. - ema_decay: (Optional) Float, EMA decay parameter. EMA is used to update - quantization intervals for quantizing activations (see here about EMA: - https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average). - quant_delay: (Optional, default None) Int, count of global steps for which - to delay quantization. This helps weights stabilize at the start of - training. - vars_collection: (Optional) Collection where to store the variables for - quantization interval ends. - scope: The scope to be transformed. If it's not None, only the ops which - are in this scope will be transformed. - Raises: - ValueError: When quantization fails. - """ - if scope and not scope.endswith('/'): - scope += '/' - - input_to_ops_map = input_to_ops.InputToOps(graph) - quantized_ops = set() - for layer_match in _FindLayersToQuantize(graph): - # Quantize the weights. - context = _GetContextFromOp(layer_match.layer_op) - - # If `scope` is given, only quantize it if the consumer of weights - # (the layer op) is in the right scope. - if layer_match.weight_tensor is not None: - _InsertQuantOp( - context, - 'weights_quant', - layer_match.weight_tensor.op, - input_to_ops_map.ConsumerOperations(layer_match.weight_tensor.op), - is_training, - moving_avg=False, - ema_decay=ema_decay, - quant_delay=quant_delay, - narrow_range=True, - vars_collection=vars_collection, - bits=weight_bits, - symmetric=symmetric, - consumer_scope=scope) - - # Quantize the activations. - if layer_match.activation_op is not None: - consumer_ops = input_to_ops_map.ConsumerOperations( - layer_match.activation_op) - add_context = context - if layer_match.bypass_op: - pattern_match_result = re.search(r'^(.*)/([^/]+)', context) - if pattern_match_result is not None: - add_context = pattern_match_result.group(1) - else: - add_context = '' - # If `scope` is given, only quantize it if the producer of weights - # (usually it's the layer op) is in the right scope. - _InsertQuantOp( - add_context, - 'act_quant', - layer_match.activation_op, - consumer_ops, - is_training, - moving_avg=True, - ema_decay=ema_decay, - quant_delay=quant_delay, - vars_collection=vars_collection, - bits=activation_bits, - symmetric=symmetric, - init_min=0.0, - producer_scope=scope) - quantized_ops.add(layer_match.activation_op) - - # Quantize the inputs and output to the bypass (if it exists). The input to - # the bypass is the bias add, and the output is the activation. - if layer_match.bypass_op is not None: - # If `scope` is given, only quantize it if the both the producer and the - # consumer are in the right scope. - _InsertQuantOp( - context, - 'conv_quant', - layer_match.bias_add_op, - input_to_ops_map.ConsumerOperations(layer_match.bias_add_op), - is_training, - moving_avg=True, - ema_decay=ema_decay, - quant_delay=quant_delay, - vars_collection=vars_collection, - bits=activation_bits, - symmetric=symmetric, - producer_scope=scope, - consumer_scope=scope) - quantized_ops.add(layer_match.bias_add_op) - # Make sure the op following this isn't an activation. In which case, we - # shouldn't quantize it, since the activation will be Fused into the - # Add at inference time. - consumers = input_to_ops_map.ConsumerOperations(layer_match.bypass_op) - if any(consumer.type in _ACTIVATION_TYPES for consumer in consumers): - logging.info('Skipping %s, because its followed by an activation.', - layer_match.bypass_op.name) - else: - _InsertQuantOp( - add_context, - 'add_quant', - layer_match.bypass_op, - input_to_ops_map.ConsumerOperations(layer_match.bypass_op), - is_training, - moving_avg=True, - ema_decay=ema_decay, - quant_delay=quant_delay, - vars_collection=vars_collection, - bits=activation_bits, - symmetric=symmetric, - producer_scope=scope, - consumer_scope=scope) - quantized_ops.add(layer_match.bypass_op) - - # Quantize bypass ops that occur after the activation. - if layer_match.post_activation_bypass_op is not None: - pattern_match_result = re.search( - r'^(.*)/([^/]+)', layer_match.post_activation_bypass_op.name) - if pattern_match_result is not None: - post_activation_bypass_context = pattern_match_result.group(1) - else: - post_activation_bypass_context = '' - # If `scope` is given, only quantize it if the producer is in the right - # scope. - # Make sure the op following this isn't an activation. In which case, we - # shouldn't quantize it, since the activation will be Fused into the - # Add at inference time. - consumers = input_to_ops_map.ConsumerOperations( - layer_match.post_activation_bypass_op) - if any(consumer.type in _RELU_TYPES for consumer in consumers): - logging.info('Skipping %s, because its followed by an activation.', - layer_match.post_activation_bypass_op.name) - else: - _InsertQuantOp( - post_activation_bypass_context, - 'post_activation_bypass_quant', - layer_match.post_activation_bypass_op, - consumers, - is_training, - moving_avg=True, - ema_decay=ema_decay, - quant_delay=quant_delay, - vars_collection=vars_collection, - bits=activation_bits, - symmetric=symmetric, - producer_scope=scope) - quantized_ops.add(layer_match.post_activation_bypass_op) - - _QuantizeActivationLayers( - quantized_ops, - graph, - is_training, - activation_bits, - ema_decay, - quant_delay, - vars_collection, - scope=scope) - - -def _QuantizeActivationLayers(quantized_ops, - graph, - is_training, - activation_bits=8, - ema_decay=0.999, - quant_delay=None, - vars_collection=ops.GraphKeys.GLOBAL_VARIABLES, - scope=None): - """Quantize intermediate activation tensors after addition and multiplication. - - Args: - quantized_ops: Set of previously quantized activation ops. - graph: Graph to modify. - is_training: Whether quantizing training graph or eval graph. - activation_bits: Number of bits to use for quantizing activations. - ema_decay: (Optional) Float, EMA decay parameter. EMA is used to update - quantization intervals for quantizing activations (see here about EMA: - https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average). - quant_delay: (Optional, default None) Int, count of global steps for which - to delay quantization. This helps weights stabilize at the start of - training. - vars_collection: (Optional) Collection where to store the variables for - quantization interval ends. - scope: The scope to be transformed. If it's not None, only the ops which are - in this scope will be transformed. - - Raises: - ValueError: When quantization fails. - """ - input_to_ops_map = input_to_ops.InputToOps(graph) - for op in (op for op in graph.get_operations()): - if _CheckIfQuantizableOp(op, quantized_ops): - logging.info('Inserting fake quant op activation_%s_quant after %s', - op.type, op.name) - consumers = input_to_ops_map.ConsumerOperations(op) - _InsertQuantOp( - op.name, - 'activation_' + op.type + '_quant', - op, - consumers, - is_training, - moving_avg=True, - ema_decay=ema_decay, - quant_delay=quant_delay, - vars_collection=vars_collection, - bits=activation_bits, - producer_scope=scope) - - -def _CheckIfQuantizableOp(src_op, quantized_ops): - """Check if the output of an op should be quantized. - - Args: - src_op: op to be checked - quantized_ops: Set of previously quantized activation ops. - - Returns: - Boolean specifying if output should be quantized or not. - """ - src_op_name = set([src_op.type]) - if src_op in quantized_ops: - return False - if not src_op_name.intersection(_VALID_SRC_OP): - return False - - # If src op is an add or a mul and the output is immediately - # followed by an activation skip - if len(src_op.outputs) == 1 and len(src_op.outputs[0].consumers()) == 1: - op_consumers = src_op.outputs[0].consumers() - if set([op_consumers[0].type]).intersection(_VALID_ACTIVATION_OP): - logging.info('Skipping quant after %s', src_op.name) - return False - # Is an Add or a Mul - input_ops = src_op.inputs - - for op in input_ops: - curr_op = op.op - curr_op_type = set([curr_op.type]) - while curr_op_type.intersection(_PASS_THROUGH_OP): - # Walk back through pass through ops - curr_op = curr_op.inputs[0].op - curr_op_type = set([curr_op.type]) - # Now at a valid or quantizable op, need to check if - # atleast one of the inputs to a valid op is connected - # to a quantizable op via pass through ops - - if (curr_op_type.intersection(_QUANTIZATION_OP) or - curr_op.name.find('delayed_quant/Merge') > 0): - return True - - if curr_op_type.intersection(_INTERMEDIATE_OP): - # Check if atleast one input to intermediate_op are quantizable - for input_op in curr_op.inputs: - if _CheckIfQuantizableOp(input_op.op, quantized_ops): - return True - return False - - -def _FindLayersToQuantize(graph): - """Matches layers in graph to quantize. - - The following patterns get matched. Nodes surrounded by [] will be - optionally matched: - - weight|folded_weight - / - conv|fc - | - [batch_to_space_nd] - | - [post_conv_correction] - | - [biasadd|folded_bias] - | - [bypass] - | - activation - | - [post_activation_bypass] - - Match replacements: - If weight|folded_weight is found, FakeQuant is added afterwards. - If bypass is found, FakeQuant is added before and after. - If activation is found, FakeQuant is added afterwards. - If post_activation_bypass is found, FakeQuant is added afterwards. - - Args: - graph: Graph to perform match on. - - Returns: - list of _LayerMatches. - """ - input_pattern = graph_matcher.OpTypePattern('*') - weight_var_pattern = graph_matcher.OpTypePattern('Variable|VariableV2') - weight_partition_identity_pattern = graph_matcher.OpTypePattern( - 'Identity', inputs=[weight_var_pattern]) - weight_partition_concat_pattern = graph_matcher.OpTypePattern( - 'ConcatV2', inputs=[weight_partition_identity_pattern, '*', '*']) - weight_identity_pattern = graph_matcher.OpTypePattern( - 'Identity', - inputs=[ - graph_matcher.OneofPattern([ - weight_partition_identity_pattern, - weight_partition_concat_pattern, - weight_var_pattern, - ]) - ]) - weight_resource_var_pattern = graph_matcher.OpTypePattern('ReadVariableOp') - folded_weight_pattern = graph_matcher.OpTypePattern('Mul') - - # The weights inputs to the layer operation can either be from the Variable or - # the folded weight (Mul). - layer_pattern = graph_matcher.OpTypePattern( - '|'.join(_QUANTIZABLE_TYPES), - inputs=[ - input_pattern, - graph_matcher.OneofPattern([ - weight_identity_pattern, weight_resource_var_pattern, - folded_weight_pattern - ]) - ], - ordered_inputs=False) - - # For atrous convolutions a BatchToSpaceND will occur after the depthwise - # convolution. - batch_to_space_pattern = graph_matcher.OpTypePattern( - 'BatchToSpaceND', - inputs=[ - layer_pattern, - graph_matcher.OpTypePattern('*'), - graph_matcher.OpTypePattern('*') - ]) - - layer_output_pattern = graph_matcher.OneofPattern( - [batch_to_space_pattern, layer_pattern]) - - # For separable convolutions, we are looking for a conv, followed by a conv - # with no activations between the two. - sep_conv_pattern = graph_matcher.OpTypePattern( - '|'.join(_QUANTIZABLE_TYPES), - inputs=[ - graph_matcher.OneofPattern([layer_output_pattern]), - graph_matcher.OpTypePattern('*') - ], - ordered_inputs=False) - folded_bias_mul_pattern = graph_matcher.OpTypePattern( - 'Mul', - inputs=[graph_matcher.OpTypePattern('*'), layer_output_pattern], - ordered_inputs=False) - post_layer_op_correction_pattern = graph_matcher.OpTypePattern( - 'Add|AddV2', - inputs=[folded_bias_mul_pattern, - graph_matcher.OpTypePattern('*')], - ordered_inputs=False) - folded_bias_add_pattern = graph_matcher.OpTypePattern( - 'Add|AddV2', - inputs=[ - post_layer_op_correction_pattern, - graph_matcher.OpTypePattern('*') - ], - ordered_inputs=False) - - # batch_norms with forced updates have an Identity operation at the end. - # TODO(suharshs): Find a way to easily skip extra Identity operations. The - # current issue is that doing so can often match patterns across many layers - # incorrectly. - batch_norm_identity = graph_matcher.OpTypePattern( - 'Identity', inputs=[folded_bias_add_pattern]) - - bias_add_pattern = graph_matcher.OpTypePattern( - 'Add|AddV2|BiasAdd', - inputs=[layer_output_pattern, '*'], - ordered_inputs=False) - - # The bias can come from the bias add or the folded bias add. - bypass_pattern = graph_matcher.OpTypePattern( - 'Add|AddV2', - inputs=[ - graph_matcher.OneofPattern( - [bias_add_pattern, folded_bias_add_pattern, batch_norm_identity]), - '*' - ], - ordered_inputs=False) - - # The input to the activation can come from bias add, fold bias add, the - # bypasses. - # TODO(suharshs): We should ideally skip Identity operations instead of - # treating them as activations. - activation_pattern = graph_matcher.OpTypePattern( - '|'.join(_ACTIVATION_TYPES) + '|Identity', - inputs=[ - graph_matcher.OneofPattern([ - bias_add_pattern, - folded_bias_add_pattern, - batch_norm_identity, - bypass_pattern, - layer_pattern, - ]) - ]) - - post_activation_bypass_pattern = graph_matcher.OpTypePattern( - 'Add|AddV2', inputs=['*', activation_pattern], ordered_inputs=False) - - # The order of the following matching blocks is very important. Since matches - # aren't guaranteed to be disjoint, we structure matches from largest to - # smallest to guarantee that the largest match always wins. Additionally, we - # ensure that we don't match layers multiple times. - - layer_matches = [] - # We use matched_layer_set to ensure that layers aren't matched multiple - # times. - matched_layer_set = set() - - # First, we match layers that have a post activation bypass. We do this first - # to ensure we don't match only the first part of this layer, missing the - # post activation bypass node. - post_activation_bypass_layer_matcher = graph_matcher.GraphMatcher( - post_activation_bypass_pattern) - for match_result in post_activation_bypass_layer_matcher.match_graph(graph): - layer_op = match_result.get_op(layer_pattern) - weight_tensor = match_result.get_tensor(weight_identity_pattern) - if weight_tensor is None: - weight_tensor = match_result.get_tensor(weight_resource_var_pattern) - if weight_tensor is None: - weight_tensor = match_result.get_tensor(folded_weight_pattern) - activation_op = match_result.get_op(activation_pattern) - bias_add_op = match_result.get_op(bias_add_pattern) - if bias_add_op is None: - bias_add_op = match_result.get_op(folded_bias_add_pattern) - bypass_op = match_result.get_op(bypass_pattern) - post_activation_bypass_op = match_result.get_op( - post_activation_bypass_pattern) - if layer_op not in matched_layer_set: - matched_layer_set.add(layer_op) - layer_matches.append( - _LayerMatch(layer_op, weight_tensor, activation_op, bypass_op, - post_activation_bypass_op, bias_add_op)) - - # Now, we match the basic layer ending at an activation. We may get duplicate - # matches from above, but we don't add them to layer_matches. - layer_matcher = graph_matcher.GraphMatcher(activation_pattern) - for match_result in layer_matcher.match_graph(graph): - layer_op = match_result.get_op(layer_pattern) - weight_tensor = match_result.get_tensor(weight_identity_pattern) - if weight_tensor is None: - weight_tensor = match_result.get_tensor(weight_resource_var_pattern) - if weight_tensor is None: - weight_tensor = match_result.get_tensor(folded_weight_pattern) - activation_op = match_result.get_op(activation_pattern) - bias_add_op = match_result.get_op(bias_add_pattern) - if bias_add_op is None: - bias_add_op = match_result.get_op(folded_bias_add_pattern) - bypass_op = match_result.get_op(bypass_pattern) - if layer_op not in matched_layer_set: - if not _IsSkipLayer(activation_op): - matched_layer_set.add(layer_op) - layer_matches.append( - _LayerMatch(layer_op, weight_tensor, activation_op, bypass_op, None, - bias_add_op)) - - # Match the final layer, where there may not be an activation and instead - # the output of the final BiasAdd must be quantized. So we treat the BiasAdd - # as the 'activation_op' in the _LayerMatch, to ensure that it's output is - # quantized. - final_layer_matcher = graph_matcher.GraphMatcher( - graph_matcher.OneofPattern([bias_add_pattern, folded_bias_add_pattern])) - for match_result in final_layer_matcher.match_graph(graph): - layer_op = match_result.get_op(layer_pattern) - weight_tensor = match_result.get_tensor(weight_identity_pattern) - if weight_tensor is None: - weight_tensor = match_result.get_tensor(weight_resource_var_pattern) - if weight_tensor is None: - weight_tensor = match_result.get_tensor(folded_weight_pattern) - activation_op = match_result.get_op(bias_add_pattern) - if activation_op is None: - activation_op = match_result.get_op(folded_bias_add_pattern) - if layer_op not in matched_layer_set: - matched_layer_set.add(layer_op) - layer_matches.append( - _LayerMatch(layer_op, weight_tensor, activation_op, None, None, None)) - - # Look for separable convolutions here - sep_conv_matcher = graph_matcher.GraphMatcher(sep_conv_pattern) - for match_result in sep_conv_matcher.match_graph(graph): - layer_op = match_result.get_op(layer_pattern) - weight_tensor = match_result.get_tensor(weight_identity_pattern) - if weight_tensor is None: - weight_tensor = match_result.get_tensor(weight_resource_var_pattern) - activation_op = match_result.get_op(layer_pattern) - if layer_op not in matched_layer_set: - matched_layer_set.add(layer_op) - layer_matches.append( - _LayerMatch(layer_op, weight_tensor, activation_op, None, None, None)) - - return layer_matches - - -def _IsSkipLayer(activation_op): - """Skip quantizing conv->identity->Batch norm layers. - - Args: - activation_op: Activation op detected by layer matching pattern - - Returns: - skip_layer: boolean, true when conv->identity->batch norm is detected. - """ - - # Exclude quantization of conv->identity->BN, - # After folding, this part corresponds to estimation of mean and variance - # and should not be quantized. - skip_layer = False - if activation_op.type == 'Identity' and len(activation_op.outputs) == 1: - if len(activation_op.outputs[0].consumers()) == 1: - consumer = activation_op.outputs[0].consumers()[0] - if consumer.type in ['FusedBatchNorm', 'FusedBatchNormV3']: - skip_layer = True - logging.info( - 'Skipping quantizing %s, because it is the output of a conv/fc ' - 'followed by a identity, feeding a fused batch norm.', - activation_op.name) - return skip_layer - - -class _LayerMatch(object): - """Contains all information related to a matched Layer.""" - - def __init__(self, layer_op, weight_tensor, activation_op, bypass_op, - post_activation_bypass_op, bias_add_op): - self._layer_op = layer_op - self._weight_tensor = weight_tensor - self._activation_op = activation_op - self._bypass_op = bypass_op - self._post_activation_bypass_op = post_activation_bypass_op - self._bias_add_op = bias_add_op - - @property - def layer_op(self): - return self._layer_op - - @property - def weight_tensor(self): - return self._weight_tensor - - @property - def activation_op(self): - return self._activation_op - - @property - def bypass_op(self): - return self._bypass_op - - @property - def post_activation_bypass_op(self): - return self._post_activation_bypass_op - - @property - def bias_add_op(self): - return self._bias_add_op - - -def _FollowedByFakeQuant(tensor): - """Returns True if the tensor is followed by a FakeQuant.""" - fake_quant_ops = set([ - 'FakeQuantWithMinMaxVars', 'FakeQuantWithMinMaxArgs', - 'FakeQuantWithMinMaxVarsPerChannel' - ]) - pass_through_ops = set(['Reshape', 'Identity']) - consumers = tensor.consumers() - while consumers: - c = consumers.pop() - if c.type in fake_quant_ops: - return True - elif c.type in pass_through_ops: - for output in c.outputs: - consumers.extend(output.consumers()) - return False - - -def _InsertQuantOp(context, - name, - producer, - consumers, - is_training, - moving_avg=True, - init_min=-6.0, - init_max=6.0, - bits=8, - symmetric=False, - ema_decay=0.999, - quant_delay=None, - vars_collection=ops.GraphKeys.GLOBAL_VARIABLES, - narrow_range=False, - producer_scope=None, - consumer_scope=None): - """Inserts a quant op between a producer op and (multiple) consumer ops. - - Args: - context: Context where producer and consumer operations are nested. - name: Name for the new quantization op within the context. - producer: Producer operation of the pairs where quantization will be - inserted. - consumers: Consumer operations of the pairs. - is_training: Whether quantizing training graph or eval graph. - moving_avg: Specifies whether to use exponential moving average or just - the last value seen. - init_min: Starting minimum value for the new quantization op. - init_max: Starting maximum value for the new quantization op. - bits: Number of bits to use for quantization, must be between 2 and 8. - symmetric: (Optional) If true, use symmetric quantization limits instead of - training the minimum and maximum of each quantization range separately. - ema_decay: (Optional) Float, EMA decay parameter. EMA is used to update - quantization intervals for quantizing activations (see here about EMA: - https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average). - quant_delay: (Optional, default None) Int, count of global steps for which - to delay quantization. This helps weights stabilize at the start of - training. - vars_collection: (Optional) Collection where to store the variables for - quantization interval ends. - narrow_range: Whether to use the narrow quantization range - [1; 2^bits - 1] or wide range [0; 2^bits - 1]. - producer_scope: The restriction of producer scope. If not None, the new op - will be inserted only when the producer is in this scope. - consumer_scope: The restriction of consumer scope. If not None, the new op - will be inserted only when all the consumers are in this scope. - Raises: - ValueError: When producer operation is not directly connected to the - consumer operation. - """ - if producer_scope and not producer.name.startswith(producer_scope): - logging.info( - '_InsertQuantOp ignores context="%s" name="%s" ' - 'because producer "%s" is not in scope "%s"', - context, name, producer.name, producer_scope) - return - - if consumer_scope: - consumers_in_scope = [] - for consumer in consumers: - if consumer.name.startswith(consumer_scope): - consumers_in_scope.append(consumer) - else: - logging.info( - '_InsertQuantOp context="%s" name="%s" ignores ' - 'consumer "%s" because it is not in scope "%s"', - context, name, consumer.name, consumer_scope) - return - consumers = consumers_in_scope - - name_prefix = _AddContextToName(context, name) - # This is needed on TPU where name_scope == 'TPUReplicate/loop', and - # name_prefix starts with 'TPUReplicate/loop/'; without dropping it - # variables are created as TPUReplicate/loop/TPUReplicate/loop/..., which - # breaks things later. - name_scope = ops.get_name_scope() - if name_scope: - name_prefix = common.DropStringPrefix(name_prefix, name_scope + '/') - - inputs = producer.outputs[0] - # Prevent ops from being quantized multiple times. Bypass ops can sometimes - # overlap between multiple matches, so we need to ensure that we don't - # add duplicate FakeQuant operations. - if _FollowedByFakeQuant(inputs): - return - - if moving_avg: - quant = ( - quant_ops.MovingAvgQuantize( - inputs, - init_min=init_min, - init_max=init_max, - ema_decay=ema_decay, - is_training=is_training, - num_bits=bits, - symmetric=symmetric, - narrow_range=narrow_range, - vars_collection=vars_collection, - name_prefix=name_prefix)) - else: - quant = ( - quant_ops.LastValueQuantize( - inputs, - init_min=init_min, - init_max=init_max, - is_training=is_training, - num_bits=bits, - symmetric=symmetric, - narrow_range=narrow_range, - vars_collection=vars_collection, - name_prefix=name_prefix)) - - if quant_delay and quant_delay > 0: - activate_quant = math_ops.greater_equal( - common.CreateOrGetQuantizationStep(), - quant_delay, - name=name_prefix + '/activate_quant') - quant = control_flow_ops.cond( - activate_quant, - lambda: quant, - lambda: inputs, - name=name_prefix + '/delayed_quant') - - if consumers: - tensors_modified_count = common.RerouteTensor( - quant, inputs, can_modify=consumers) - # Some operations can have multiple output tensors going to the same - # consumer. Since consumers is a set, we need to ensure that - # tensors_modified_count is greater than or equal to the length of the set - # of consumers. - if tensors_modified_count < len(consumers): - raise ValueError('No inputs quantized for ops: [%s]' % ', '.join( - [consumer.name for consumer in consumers])) - - -def _GetContextFromOp(op): - """Gets the root context name from the op name.""" - context_re = re.search(r'^(.*)/([^/]+)', op.name) - if context_re: - return context_re.group(1) - return '' - - -def _AddContextToName(context, name): - """Adds the context to the name if it exists.""" - if not context: - return name - return context + '/' + name diff --git a/tensorflow/contrib/quantize/python/quantize_graph.py b/tensorflow/contrib/quantize/python/quantize_graph.py deleted file mode 100644 index 2a256a3c51c..00000000000 --- a/tensorflow/contrib/quantize/python/quantize_graph.py +++ /dev/null @@ -1,291 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""API to simulate quantization on a python graph.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.quantize.python import fold_batch_norms -from tensorflow.contrib.quantize.python import quantize -from tensorflow.python.framework import ops - - -def _create_graph(input_graph=None, - is_training=True, - weight_bits=8, - activation_bits=8, - symmetric=False, - quant_delay=None, - freeze_bn_delay=None, - scope=None): - """Rewrites an input_graph in place for simulated quantization. - - The graph has fake quantization ops inserted to simulate the error - introduced by quantization. Since the graph is transformed in place, - the expected behavior of previously held references to nodes and tensors may - change. - - Args: - input_graph: The tf.Graph to be transformed, if None then defaults to the - default graph. - is_training: Whether quantizing training or eval graph. - weight_bits: Number of bits to use for quantizing weights. - activation_bits: Number of bits to use for quantizing activations. - symmetric: If true, use symmetric quantization limits instead of training - the minimum and maximum of each quantization range separately. - quant_delay: Number of steps after which weights and activations are - quantized during training. - freeze_bn_delay: Number of steps after which moving mean and variance are - frozen and used instead of batch statistics during training. - freeze_bn_delay should be greater than quant_delay and should correspond - to the number of steps when training has almost converged - scope: The scope to be transformed. If it's not None, only the ops which - are in this scope will be transformed. - - Raises: - ValueError: If elements contains an element that isn't a tf.Tensor or - tf.Operation. - """ - - if input_graph is None: - input_graph = ops.get_default_graph() - - # Add check to see if graph has training ops, if so provide error message and - # exit - _check_for_training_ops(input_graph) - with input_graph.as_default(): - fold_batch_norms.FoldBatchNorms( - input_graph, - freeze_batch_norm_delay=freeze_bn_delay, - is_training=is_training) - quantize.Quantize( - input_graph, - is_training, - quant_delay=quant_delay, - weight_bits=weight_bits, - activation_bits=activation_bits, - symmetric=symmetric, - scope=scope) - - -def create_training_graph(input_graph=None, quant_delay=0): - """Rewrites a training input_graph in place for simulated quantization. - - Variables added by the rewrite get added to the global variables collection. - - This function must be invoked prior to insertion of gradient ops in a graph - as quantization should be modeled in both forward and backward passes. - - The graph has fake quantization ops inserted to simulate the error - introduced by quantization. Since the graph is transformed in place, - the expected behavior of previously held references to nodes and tensors may - change. - - The default value of quant_delay is suitable for finetuning an already trained - floating point model (recommended). - If one wants to train a quantized model from scratch, quant_delay should be - set to the number of steps it take the floating point model to converge. - Quantization will be activated at this point and effectively finetune the - model. If quant_delay is not provided when training from scratch, training can - often fail. - - Args: - input_graph: The tf.Graph to be transformed. - quant_delay: Number of steps after which weights and activations are - quantized during training. - - Raises: - ValueError: If elements contains an element that isn't a tf.Tensor or - tf.Operation. - """ - # TODO(raghuramank) Need to have freeze_bn_delay be a function of batch size - # Currently the values below are hardcoded for mobilenetV1 on imagenet - # Please use the experimental API if you need to tune these values. - freeze_bn_delay = None - _create_graph( - input_graph=input_graph, - is_training=True, - quant_delay=quant_delay, - freeze_bn_delay=freeze_bn_delay) - - -def create_eval_graph(input_graph=None): - """Rewrites an eval input_graph in place for simulated quantization. - - Variables added by the rewrite get added to the global variables collection. - - The graph has fake quantization ops inserted to simulate the error - introduced by quantization. Since the graph is transformed in place, - the expected behavior of previously held references to nodes and tensors may - change. - - Args: - input_graph: The tf.Graph to be transformed, if None then defaults to the - default graph. - - Raises: - ValueError: If elements contains an element that isn't a tf.Tensor or - tf.Operation. - """ - _create_graph(input_graph=input_graph, is_training=False) - - -def experimental_create_training_graph(input_graph=None, - weight_bits=8, - activation_bits=8, - symmetric=False, - quant_delay=0, - freeze_bn_delay=None, - scope=None): - """Rewrites a training input_graph in place for simulated quantization. - - This function must be invoked prior to insertion of gradient ops in a graph - as quantization should be modeled in both forward and backward passes. - - Variables added by the rewrite get added to the global variables collection. - - This function has additional experimental options not (yet) available to - create_training_graph. The resulting behavior may be undefined. - - The graph has fake quantization ops inserted to simulate the error - introduced by quantization. Since the graph is transformed in place, - the expected behavior of previously held references to nodes and tensors may - change. - - The default value of quant_delay is suitable for finetuning an already trained - floating point model (recommended). - If one wants to train a quantized model from scratch, quant_delay should be - set to the number of steps it take the floating point model to converge. - Quantization will be activated at this point and effectively finetune the - model. If quant_delay is not provided when training from scratch, training can - often fail. - - Args: - input_graph: The tf.Graph to be transformed, if None then defaults to the - default graph. - weight_bits: Number of bits to use for quantizing weights. - activation_bits: Number of bits to use for quantizing activations. - symmetric: If true, use symmetric quantization limits instead of training - the minimum and maximum of each quantization range separately. - quant_delay: Number of steps after which weights and activations are - quantized during training. - freeze_bn_delay: Number of steps after which moving mean and variance are - frozen and used instead of batch statistics during training. - freeze_bn_delay should be greater than quant_delay and should correspond - to when training has almost converged - scope: The scope to be transformed. If it's not None, only the ops which - are in this scope will be transformed. - - Raises: - ValueError: If elements contains an element that isn't a tf.Tensor or - tf.Operation. - """ - - _create_graph( - input_graph=input_graph, - is_training=True, - weight_bits=weight_bits, - activation_bits=activation_bits, - symmetric=symmetric, - quant_delay=quant_delay, - freeze_bn_delay=freeze_bn_delay, - scope=scope) - - -def experimental_create_eval_graph(input_graph=None, - weight_bits=8, - activation_bits=8, - symmetric=False, - quant_delay=None, - scope=None): - """Rewrites an eval input_graph in place for simulated quantization. - - Variables added by the rewrite get added to the global variables collection. - - This function has additional experimental options not (yet) available to - create_eval_graph. The resulting behavior may be undefined. - - The graph has fake quantization ops inserted to simulate the error - introduced by quantization. Since the graph is transformed in place, - the expected behavior of previously held references to nodes and tensors may - change. - - Args: - input_graph: The tf.Graph to be transformed, if None then defaults to the - default graph. - weight_bits: Number of bits to use for quantizing weights. - activation_bits: Number of bits to use for quantizing activations. - symmetric: If true, use symmetric quantization limits instead of training - the minimum and maximum of each quantization range separately. - quant_delay: Number of steps after which weights and activations are - quantized during eval. - scope: The scope to be transformed. If it's not None, only the ops which - are in this scope will be transformed. - - Raises: - ValueError: If elements contains an element that isn't a tf.Tensor or - tf.Operation. - """ - _create_graph( - input_graph=input_graph, - is_training=False, - weight_bits=weight_bits, - activation_bits=activation_bits, - symmetric=symmetric, - quant_delay=quant_delay, - scope=scope) - - -def _check_for_training_ops(g): - """Check if training ops are present in the graph. - - Args: - g: The tf.Graph on which the check for training ops needs to be - performed. - - Raises: - ValueError: If a training op is seen in the graph; - """ - - # The list here is obtained - # from https://www.tensorflow.org/api_docs/cc/group/training-ops - training_ops = frozenset([ - 'ApplyAdagrad', 'ApplyAdagradDA', 'ApplyAdam', 'ApplyAddSign', - 'ApplyCenteredRMSProp', 'ApplyFtrl', 'ApplyFtrlV2', - 'ApplyGradientDescent', 'ApplyMomentum', 'ApplyPowerSign', - 'ApplyProximalAdagrad', 'ApplyProximalGradientDescent', 'ApplyRMSProp', - 'ResourceApplyAdadelta', 'ResourceApplyAdagrad', 'ResourceApplyAdagradDA', - 'ResourceApplyAdam', 'ResourceApplyAddSign', - 'ResourceApplyCenteredRMSProp', 'ResourceApplyFtrl', - 'ResourceApplyFtrlV2', 'ResourceApplyGradientDescent', - 'ResourceApplyMomentum', 'ResourceApplyPowerSign', - 'ResourceApplyProximalAdagrad', 'ResourceApplyProximalGradientDescent', - 'ResourceApplyRMSProp', 'ResourceSparseApplyAdadelta', - 'ResourceSparseApplyAdagrad', 'ResourceSparseApplyAdagradDA', - 'ResourceSparseApplyCenteredRMSProp', 'ResourceSparseApplyFtrl', - 'ResourceSparseApplyFtrlV2', 'ResourceSparseApplyMomentum', - 'ResourceSparseApplyProximalAdagrad', - 'ResourceSparseApplyProximalGradientDescent', - 'ResourceSparseApplyRMSProp', 'SparseApplyAdadelta', 'SparseApplyAdagrad', - 'SparseApplyAdagradDA', 'SparseApplyCenteredRMSProp', 'SparseApplyFtrl', - 'SparseApplyFtrlV2', 'SparseApplyMomentum', 'SparseApplyProximalAdagrad', - 'SparseApplyProximalGradientDescent', 'SparseApplyRMSProp' - ]) - - op_types = set([op.type for op in g.get_operations()]) - train_op_list = op_types.intersection(training_ops) - if train_op_list: - raise ValueError('Training op found in graph, exiting %s' % train_op_list) diff --git a/tensorflow/contrib/quantize/python/quantize_graph_test.py b/tensorflow/contrib/quantize/python/quantize_graph_test.py deleted file mode 100644 index 4b28f84ec23..00000000000 --- a/tensorflow/contrib/quantize/python/quantize_graph_test.py +++ /dev/null @@ -1,548 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Unit tests for the quantize_graph graph rewriting API.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.quantize.python import quantize_graph -from tensorflow.python import training -from tensorflow.python.compat import compat -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import template -from tensorflow.python.platform import googletest - - -class QuantizeGraphTest(test_util.TensorFlowTestCase): - # We have a lot of other tests that test the details of the rewrite, here we - # just the specific features of the quantize_graph API. - - def _RunTestOverAllRewrites(self, test_fn): - rewrite_fns = [ - quantize_graph.create_training_graph, - quantize_graph.create_eval_graph, - quantize_graph.experimental_create_training_graph, - quantize_graph.experimental_create_eval_graph, - ] - for fn in rewrite_fns: - test_fn(fn) - - def _RunTestOverTrainingRewrites(self, test_fn): - rewrite_fns = [ - quantize_graph.create_training_graph, - quantize_graph.experimental_create_training_graph, - functools.partial( - quantize_graph.experimental_create_training_graph, symmetric=True), - ] - for fn in rewrite_fns: - test_fn(fn) - - def _RunTestOverEvalRewrites(self, test_fn): - rewrite_fns = [ - quantize_graph.create_eval_graph, - quantize_graph.experimental_create_eval_graph, - functools.partial( - quantize_graph.experimental_create_eval_graph, symmetric=True), - ] - for fn in rewrite_fns: - test_fn(fn) - - def _RunTestOverExperimentalRewrites(self, test_fn): - rewrite_fns = [ - quantize_graph.experimental_create_training_graph, - quantize_graph.experimental_create_eval_graph, - ] - for fn in rewrite_fns: - test_fn(fn) - - def _RunTestOverExperimentalRewritesWithScope(self, test_fn, scope): - def with_absent_scope(fn): - def fn_with_absent_scope(*args): - fn(*args, scope=scope) - return fn_with_absent_scope - rewrite_fns = [ - with_absent_scope( - quantize_graph.experimental_create_training_graph), - with_absent_scope( - quantize_graph.experimental_create_eval_graph), - ] - for fn in rewrite_fns: - test_fn(fn) - - def testRewrite(self): - self._RunTestOverAllRewrites(self._TestRewrite) - - def _TestRewrite(self, rewrite_fn): - graph = ops.Graph() - with graph.as_default(): - self._ConvLayer() - - orig_variable_names = set( - [v.name for v in graph.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)]) - - rewrite_fn(graph) - - q_variables = graph.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - # Ensure that variables were added. - self.assertTrue(len(orig_variable_names) < len(q_variables)) - - def testDefaultGraph(self): - self._RunTestOverAllRewrites(self._TestRewrite) - - def _TestDefaultGraph(self, rewrite_fn): - # Tests that the default graph is correctly used when no args are provided - # to rewrite_fn. - with ops.Graph().as_default() as g: - self._ConvLayer() - orig_variable_names = set( - [v.name for v in g.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)]) - rewrite_fn() - - q_variables = g.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - # Ensure that variables were added. - self.assertTrue(len(orig_variable_names) < len(q_variables)) - - def testWithPostActivationBypass(self): - self._RunTestOverAllRewrites(self._TestWithPostActivationBypass) - - def _TestWithPostActivationBypass(self, rewrite_fn): - # Tests that the default graph is correctly used when no args are provided - # to rewrite_fn. - with ops.Graph().as_default() as g: - self._ConvLayer(post_activation_bypass=True, scope='scope1') - rewrite_fn() - - op_names = [op.name for op in g.get_operations()] - self.assertTrue(any( - 'scope1/post_activation_bypass_quant/' in name for name in op_names)) - - def testQuantDelay(self): - self._RunTestOverTrainingRewrites(self._TestQuantDelay) - - def _TestQuantDelay(self, rewrite_fn): - with ops.Graph().as_default() as g: - self._ConvLayer() - quant_delay = 100 - rewrite_fn(quant_delay=quant_delay) - - quant_delay_found = False - for op in g.get_operations(): - # Check to see if the quant_delay is correctly set. - if 'activate_quant' in op.name and op.type == 'Const': - quant_delay_found = True - const_value = str(op.get_attr('value')) - self.assertTrue(('int64_val: %i' % quant_delay) in const_value) - self.assertTrue(quant_delay_found) - - def testTrainingOpsCheck(self): - self._RunTestOverTrainingRewrites(self._TestTrainingOpsCheck) - - def _TestTrainingOpsCheck(self, rewrite_fn): - with ops.Graph().as_default(): - output = self._ConvLayer() - output_scalar = math_ops.reduce_sum(output) - loss = math_ops.square(output_scalar - 1) - opt = training.gradient_descent.GradientDescentOptimizer(0.0001) - opt.minimize(loss) - with self.assertRaisesRegexp(ValueError, 'Training op found in graph'): - rewrite_fn() - - def testWeightBits(self): - self._RunTestOverExperimentalRewrites(self._TestWeightBits) - - def _TestWeightBits(self, rewrite_fn): - with ops.Graph().as_default() as g: - self._ConvLayer() - weight_bits = 4 - rewrite_fn(weight_bits=weight_bits) - - weights_quant_found = False - for op in g.get_operations(): - # Check to see if FakeQuant operations for weights have the right bits - # set. - if 'weights_quant' in op.name and op.type == 'FakeQuantWithMinMaxVars': - weights_quant_found = True - self.assertEqual(op.get_attr('num_bits'), weight_bits) - self.assertTrue(weights_quant_found) - - def testActivationBits(self): - self._RunTestOverExperimentalRewrites(self._TestActivationBits) - - def _TestActivationBits(self, rewrite_fn): - with ops.Graph().as_default() as g: - self._ConvLayer() - activation_bits = 4 - rewrite_fn(activation_bits=activation_bits) - - act_quant_found = False - for op in g.get_operations(): - # Check to see if FakeQuant operations for activations have the right bits - # set. - act_quant_names = ['act_quant', 'conv_quant', 'add_quant'] - if any(s in op.name - for s in act_quant_names) and op.type == 'FakeQuantWithMinMaxVars': - act_quant_found = True - self.assertEqual(op.get_attr('num_bits'), activation_bits) - self.assertTrue(act_quant_found) - - def testTrainingQuantization(self): - self._RunTestOverTrainingRewrites(self._TestTrainingQuantization) - - def _TestTrainingQuantization(self, rewrite_fn): - with ops.Graph().as_default() as g: - self._ConvLayer() - rewrite_fn() - - # Ensure that FakeQuant and variable update nodes were found. - quant_found = False - assign_min_last_found = False - assign_min_ema_found = False - assign_max_last_found = False - assign_max_ema_found = False - for op in g.get_operations(): - # Check that FakeQuant operations were added. - if op.type == 'FakeQuantWithMinMaxVars': - quant_found = True - # Check that update operations for the added min max variables exist in - # the graph. - if 'AssignMinLast' in op.name: - assign_min_last_found = True - elif 'AssignMinEma' in op.name: - assign_min_ema_found = True - elif 'AssignMaxLast' in op.name: - assign_max_last_found = True - elif 'AssignMaxEma' in op.name: - assign_max_ema_found = True - self.assertTrue(assign_min_last_found) - self.assertTrue(assign_min_ema_found) - self.assertTrue(assign_max_last_found) - self.assertTrue(assign_max_ema_found) - self.assertTrue(quant_found) - - def testEvalQuantization(self): - self._RunTestOverEvalRewrites(self._TestEvalQuantization) - - def _TestEvalQuantization(self, rewrite_fn): - with ops.Graph().as_default() as g: - self._ConvLayer() - rewrite_fn() - - # Ensure that FakeQuant and variable update nodes were found. - quant_found = False - for op in g.get_operations(): - # Check that FakeQuant operations were added. - if op.type == 'FakeQuantWithMinMaxVars': - quant_found = True - # Check that update operations for the added min max variables don't - # exist in the graph. - update_names = [ - 'AssignMinLast', 'AssignMinEma', 'AssignMaxLast', 'AssignMaxEma' - ] - self.assertFalse(any(s in op.name for s in update_names)) - self.assertTrue(quant_found) - - def testIdempotent(self): - self._RunTestOverAllRewrites(self._TestIdempotent) - - def _TestIdempotent(self, rewrite_fn): - with ops.Graph().as_default() as g: - self._ConvLayer() - rewrite_fn() - graph_def_before = str(g.as_graph_def()) - # Ensuring that calling the rewrite again doesn't add more nodes. - rewrite_fn() - graph_def_after = str(g.as_graph_def()) - self.assertEqual(graph_def_before, graph_def_after) - - def testIdentityNode(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - self._RunTestOverAllRewrites(self._TestIdentityNode) - - def _TestIdentityNode(self, rewrite_fn): - graph = ops.Graph() - with graph.as_default(): - self._LayerWithIdentity() - - rewrite_fn(graph) - op_names = [op.name for op in graph.get_operations()] - self.assertTrue(any('test/Conv/weights_quant' in name for name in op_names)) - self.assertTrue(any('test/Conv/act_quant' in name for name in op_names)) - bn_out_identity = graph.get_operation_by_name('test/bn_out') - self._AssertInputOpsAre(bn_out_identity, [ - 'test/Conv/add_fold', - ]) - - conv_out_identity = graph.get_operation_by_name('test/conv_out') - self._AssertOutputGoesToOps(conv_out_identity, graph, - ['test/BatchNorm/FusedBatchNormV3']) - - def testActivationQuantization(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - self._RunTestOverAllRewrites(self._TestActivationQuantization) - - def _TestActivationQuantization(self, rewrite_fn): - graph = ops.Graph() - with graph.as_default(): - _ = self._LayerWithActivationProcessing() - - rewrite_fn(graph) - # Check if outputs of multipliers and adds are quantized. - - mul_op = graph.get_operation_by_name('test/Mul') - self._AssertOutputGoesToOps( - mul_op, graph, - ['test/Mul/activation_Mul_quant/FakeQuantWithMinMaxVars']) - mul_op = graph.get_operation_by_name('test/Mul_1') - self._AssertOutputGoesToOps( - mul_op, graph, - ['test/Mul_1/activation_Mul_quant/FakeQuantWithMinMaxVars']) - add_op = graph.get_operation_by_name('test/add') - if compat.forward_compatible(2019, 6, 21): - self._AssertOutputGoesToOps( - add_op, graph, - ['test/add/activation_AddV2_quant/FakeQuantWithMinMaxVars']) - else: - self._AssertOutputGoesToOps( - add_op, graph, - ['test/add/activation_Add_quant/FakeQuantWithMinMaxVars']) - - def testRewriteWithScope(self): - self._RunTestOverExperimentalRewritesWithScope( - self._TestRewriteWithScope, 'scope1') - - def _TestRewriteWithScope(self, rewrite_fn): - graph = ops.Graph() - with graph.as_default(): - scope1_output = self._ConvLayer(scope='scope1') - self._ConvLayer(input_tensor=scope1_output, scope='scope2') - - rewrite_fn(graph) - - op_names = [op.name for op in graph.get_operations()] - # The weights and activation of scope1 is quantized, but not scope2. - self.assertTrue( - any('scope1/Conv/act_quant' in name for name in op_names)) - self.assertTrue( - any('scope1/Conv/weights_quant' in name for name in op_names)) - self.assertFalse( - any('scope2/Conv/act_quant' in name for name in op_names)) - self.assertFalse( - any('scope2/Conv/weights_quant' in name for name in op_names)) - - def testRewriteWithNonMatchingScope(self): - self._RunTestOverExperimentalRewritesWithScope( - self._TestRewriteWithNonMatchingScope, 'NonExistingScope') - - def _TestRewriteWithNonMatchingScope(self, rewrite_fn): - graph = ops.Graph() - with graph.as_default(): - self._ConvLayer() - - op_names_before_rewrite = set([op.name for op in graph.get_operations()]) - rewrite_fn(graph) - op_names_after_rewrite = set([op.name for op in graph.get_operations()]) - - # No ops should be inserted or removed. - self.assertEqual(op_names_before_rewrite, op_names_after_rewrite) - - def testActivationRewriteWithScope(self): - self._RunTestOverExperimentalRewritesWithScope( - self._TestActivationRewriteWithScope, 'scope1') - - def _TestActivationRewriteWithScope(self, rewrite_fn): - graph = ops.Graph() - with graph.as_default(): - output = self._LayerWithIdentity(scope='scope1') - with ops.name_scope('scope2'): - output = nn_ops.relu6(output) - scaled_output1 = math_ops.mul(2.0, output) - scaled_output2 = math_ops.mul(3.0, output) - output = scaled_output1 + scaled_output2 - rewrite_fn(graph) - - op_names = [op.name for op in graph.get_operations()] - # The weights and activation of scope1 is quantized, but not scope2. - self.assertTrue(any('scope1/Conv/act_quant' in name for name in op_names)) - self.assertTrue( - any('scope1/Conv/weights_quant' in name for name in op_names)) - - for op_name in op_names: - if op_name.startswith('scope2'): - self.assertTrue('FakeQuant' not in op_name) - - def testActivationRewriteWithNonMatchingScope(self): - self._RunTestOverExperimentalRewritesWithScope( - self._TestActivationRewriteWithNonMatchingScope, 'NonExistingScope') - - def _TestActivationRewriteWithNonMatchingScope(self, rewrite_fn): - graph = ops.Graph() - with graph.as_default(): - self._LayerWithActivationProcessing() - - rewrite_fn(graph) - op_types_after_rewrite = set([op.type for op in graph.get_operations()]) - self.assertFalse( - op_types_after_rewrite.intersection('FakeQuantWithMinMaxVars')) - # No fake quant ops should be inserted. - - def testWithSharedWeights(self): - - self._RunTestOverAllRewrites(self._TestWithSharedWeights) - self._RunTestOverTrainingRewrites(self._TestRewriteWithSharedWeights) - - def _TestRewriteWithSharedWeights(self, rewrite_fn, quant_delay=1): - self._TestWithSharedWeights(rewrite_fn, quant_delay) - - def _TestWithSharedWeights(self, rewrite_fn, quant_delay=None): - with ops.Graph().as_default() as g: - conv = template.make_template('shared_weights_conv', self._ConvLayer) - conv() - conv() - if quant_delay is None: - rewrite_fn() - else: - rewrite_fn(quant_delay=quant_delay) - - conv_ops = [op for op in g.get_operations() if op.type == 'Conv2D'] - weights_quants = [ - op for op in g.get_operations() - if 'weights_quant' in op.name and op.type == 'FakeQuantWithMinMaxVars' - ] - # Check that the shared weights variable is not quantized multiple times - self.assertTrue(len(weights_quants) == 1) - weights_quant_tensor = weights_quants[0].outputs[0] - if quant_delay: - delayed_weights_quants = [ - op for op in g.get_operations() - if 'weights_quant' in op.name and op.type == 'Merge' - ] - self.assertTrue(len(delayed_weights_quants) == 1) - weights_quant_tensor = delayed_weights_quants[0].outputs[0] - # Check that the Conv2D operations get the quantized weights - self.assertTrue(all(weights_quant_tensor in op.inputs for op in conv_ops)) - - def _ConvLayer( - self, input_tensor=None, scope='test', pre_activation_bypass=False, - post_activation_bypass=False): - """Add a basic convolution layer to the default graph.""" - batch_size, height, width, depth = 5, 128, 128, 3 - if input_tensor is None: - input_tensor = array_ops.zeros((batch_size, height, width, depth)) - weight_init = init_ops.truncated_normal_initializer - with ops.name_scope(scope): - output = layers.conv2d( - input_tensor, - depth, [5, 5], - padding='SAME', - weights_initializer=weight_init(0.09), - activation_fn=None) - if pre_activation_bypass: - output += input_tensor - output = nn_ops.relu6(output) - if post_activation_bypass: - output += input_tensor - return output - - def _LayerWithIdentity(self, - input_tensor=None, - scope='test', - post_activation_bypass=False): - """Add a basic conv, identity, batch norm with skip to the default graph.""" - batch_size, height, width, depth = 5, 128, 128, 3 - if input_tensor is None: - input_tensor = array_ops.zeros((batch_size, height, width, depth)) - weight_init = init_ops.truncated_normal_initializer - with ops.name_scope(scope): - output = layers.conv2d( - input_tensor, - depth, [5, 5], - padding='SAME', - weights_initializer=weight_init(0.09), - activation_fn=None, - normalizer_fn=None, - biases_initializer=None) - output = array_ops.identity(output, name='conv_out') - - output = layers.batch_norm( - output, center=True, scale=True, decay=1.0 - 0.003, fused=True) - - output = array_ops.identity(output, name='bn_out') - if post_activation_bypass: - output += input_tensor - return output - - def _LayerWithActivationProcessing(self, - input_tensor=None, - scope='test', - post_activation_bypass=False): - - batch_size, height, width, depth = 5, 128, 128, 3 - if input_tensor is None: - input_tensor = array_ops.zeros((batch_size, height, width, depth)) - weight_init = init_ops.truncated_normal_initializer - with ops.name_scope(scope): - output = layers.conv2d( - input_tensor, - depth, [5, 5], - padding='SAME', - weights_initializer=weight_init(0.09), - activation_fn=None, - normalizer_fn=None, - biases_initializer=None) - - output = layers.batch_norm( - output, center=True, scale=True, decay=1.0 - 0.003, fused=True) - - output = nn_ops.relu6(output) - scaled_output1 = math_ops.mul(2.0, output) - scaled_output2 = math_ops.mul(3.0, output) - output = scaled_output1 + scaled_output2 - return output - - def _AssertInputOpsAre(self, op, in_op_names): - """Asserts that all inputs to op come from in_op_names (disregarding order). - - Args: - op: Operation to check inputs for. - in_op_names: List of strings, operations where all op's inputs should come - from. - """ - expected_inputs = [in_op_name + ':0' for in_op_name in in_op_names] - self.assertItemsEqual([t.name for t in op.inputs], expected_inputs) - - def _AssertOutputGoesToOps(self, op, graph, out_op_names): - """Asserts that outputs from op go to out_op_names (and perhaps others). - - Args: - op: Operation to check outputs for. - graph: Graph where output operations are located. - out_op_names: List of strings, operations where op's outputs should go. - """ - for out_op_name in out_op_names: - out_op = graph.get_operation_by_name(out_op_name) - self.assertIn(op.outputs[0].name, [str(t.name) for t in out_op.inputs]) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py deleted file mode 100644 index 05993884f7c..00000000000 --- a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py +++ /dev/null @@ -1,857 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Parameterized unit tests for quantizing a Tensorflow graph.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.quantize.python import fold_batch_norms -from tensorflow.contrib.quantize.python import quantize -from tensorflow.python.compat import compat -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import googletest - -batch_norm = layers.batch_norm -conv2d = layers.conv2d -fully_connected = layers.fully_connected -separable_conv2d = layers.separable_conv2d - - -class QuantizeTest(test_util.TensorFlowTestCase): - - def _RunWithoutBatchNormTestOverParameters(self, test_fn): - # TODO(suharshs): Use parameterized test once OSS TF supports it. - parameters_list = [ - # (activation, activation_op_name, with_bypass, delay) - (nn_ops.relu6, 'Relu6', False, None), - (nn_ops.relu, 'Relu', False, None), - (array_ops.identity, 'Identity', False, None), - (nn_ops.relu6, 'Relu6', False, 5000), - (nn_ops.relu, 'Relu', False, 5000), - (array_ops.identity, 'Identity', False, 5000), - (nn_ops.relu6, 'Relu6', True, None), - (nn_ops.relu, 'Relu', True, None), - (array_ops.identity, 'Identity', True, None), - (nn_ops.relu6, 'Relu6', True, 5000), - (nn_ops.relu, 'Relu', True, 5000), - (array_ops.identity, 'Identity', True, 5000), - ] - for params in parameters_list: - # Test everything with resource variables and normal variables. - test_fn(params[0], params[1], params[2], params[3], False, None) - test_fn(params[0], params[1], params[2], params[3], True, None) - # Test with both empty scope and an example scope - test_fn(params[0], params[1], params[2], params[3], False, 'test') - test_fn(params[0], params[1], params[2], params[3], True, 'test') - - def _AssertCorrectQuantizedGraphWithoutBatchNorm( - self, graph, scope, layer, activation_op_name, with_bypass, delay, - use_resource): - quantization_node_name = 'FakeQuantWithMinMaxVars' - conv_scope = self._GetConvScope(scope, with_bypass) - delim = '/' if conv_scope else '' - - if scope: - scope = scope + '/' - weights_quant = graph.get_operation_by_name( - conv_scope + delim + 'weights_quant/' + quantization_node_name) - self.assertEqual(weights_quant.type, quantization_node_name) - - # Assemble the expected inputs. - if use_resource: - expected_inputs = [ - conv_scope + delim + - 'weights_quant/FakeQuantWithMinMaxVars/ReadVariableOp', - conv_scope + delim + - 'weights_quant/FakeQuantWithMinMaxVars/ReadVariableOp_1', - ] - if layer == 'DepthwiseConv2dNative': - expected_inputs.append(conv_scope + delim + 'depthwise/ReadVariableOp') - else: - expected_inputs.append(conv_scope + delim + layer + '/ReadVariableOp') - else: - expected_inputs = [ - conv_scope + delim + 'weights_quant/AssignMinLast', - conv_scope + delim + 'weights_quant/AssignMaxLast', - ] - if layer == 'DepthwiseConv2dNative': - expected_inputs.append(conv_scope + delim + 'depthwise_weights/read') - else: - expected_inputs.append(conv_scope + delim + 'weights/read') - - self._AssertInputOpsAre(weights_quant, expected_inputs) - if delay and delay > 0: - output_op_name = ( - conv_scope + delim + 'weights_quant/delayed_quant/Switch_1') - else: - if layer == 'DepthwiseConv2dNative': - output_op_name = conv_scope + delim + 'depthwise' - else: - output_op_name = conv_scope + delim + layer - - self._AssertOutputGoesToOps(weights_quant, graph, [output_op_name]) - - if with_bypass: - conv_quant = graph.get_operation_by_name( - conv_scope + delim + 'conv_quant/' + quantization_node_name) - self.assertEqual(conv_quant.type, quantization_node_name) - if use_resource: - expected_inputs = [ - conv_scope + delim + - 'conv_quant/FakeQuantWithMinMaxVars/ReadVariableOp', - conv_scope + delim + - 'conv_quant/FakeQuantWithMinMaxVars/ReadVariableOp_1', - conv_scope + delim + 'BiasAdd', - ] - else: - expected_inputs = [ - conv_scope + delim + 'conv_quant/AssignMinEma', - conv_scope + delim + 'conv_quant/AssignMaxEma', - conv_scope + delim + 'BiasAdd' - ] - self._AssertInputOpsAre(conv_quant, expected_inputs) - - output_op_name = ( - conv_scope + delim + - 'conv_quant/delayed_quant/Switch_1' if delay else scope + 'AddV2') - self._AssertOutputGoesToOps(conv_quant, graph, [output_op_name]) - - act_quant = graph.get_operation_by_name(scope + 'act_quant/' + - quantization_node_name) - self.assertEqual(act_quant.type, quantization_node_name) - if use_resource: - expected_inputs = [ - scope + 'act_quant/FakeQuantWithMinMaxVars/ReadVariableOp', - scope + 'act_quant/FakeQuantWithMinMaxVars/ReadVariableOp_1', - scope + activation_op_name, - ] - else: - expected_inputs = [ - scope + 'act_quant/AssignMinEma', scope + 'act_quant/AssignMaxEma', - scope + activation_op_name - ] - self._AssertInputOpsAre(act_quant, expected_inputs) - output_op_name = ( - scope + 'act_quant/delayed_quant/Switch_1' - if delay else 'control_dependency') - self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) - self._AssertIdempotent(graph) - - def testQuantize_Conv2dWithoutBatchNorm(self): - self._RunWithoutBatchNormTestOverParameters( - self._TestQuantize_Conv2dWithoutBatchNorm) - - def _TestQuantize_Conv2dWithoutBatchNorm(self, activation, activation_op_name, - with_bypass, delay, use_resource, - scope): - """Tests quantization: inputs -> Conv2d no batch norm -> Activation. - - Args: - activation: Callable that returns an Operation, a factory method for the - Activation. - activation_op_name: String, name of the Activation operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Activation. - delay: Int (optional), delay in number of steps until quantization starts. - use_resource: Bool, when true uses resource variables. - scope: String, specifies top level scope for the graph - """ - graph = ops.Graph() - with graph.as_default(): - variable_scope.get_variable_scope().set_use_resource(use_resource) - batch_size, height, width, depth = 5, 128, 128, 3 - inputs = array_ops.zeros((batch_size, height, width, depth)) - stride = 1 if with_bypass else 2 - out_depth = 3 if with_bypass else 32 - activation_fn = None if with_bypass else activation - conv_scope = self._GetConvScope(scope, with_bypass) - scope = '' if scope is None else scope - delim = '/' if scope else '' - node = conv2d( - inputs, - out_depth, [5, 5], - stride=stride, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - scope=conv_scope) - if with_bypass: - node = math_ops.add(inputs, node, name=scope + delim + 'AddV2') - node = activation(node, name=scope + delim + activation_op_name) - update_barrier = control_flow_ops.no_op(name='update_barrier') - with ops.control_dependencies([update_barrier]): - array_ops.identity(node, name='control_dependency') - - quantize.Quantize(graph, True, quant_delay=delay) - - if conv_scope is None: - conv_scope = '' - - self._AssertCorrectQuantizedGraphWithoutBatchNorm( - graph, scope, 'Conv2D', activation_op_name, with_bypass, delay, - use_resource) - - def testQuantize_FCWithoutBatchNorm(self): - self._RunWithoutBatchNormTestOverParameters( - self._TestQuantize_FCWithoutBatchNorm) - - def _TestQuantize_FCWithoutBatchNorm(self, activation, activation_op_name, - with_bypass, delay, use_resource, scope): - """Tests quantization: inputs -> FC no batch norm -> Activation. - - Args: - activation: Callable that returns an Operation, a factory method for the - Activation. - activation_op_name: String, name of the Activation operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Activation. - delay: Int (optional), delay in number of steps until quantization starts. - use_resource: Bool, when true uses resource variables. - scope: String, specifies top level scope for the graph - """ - graph = ops.Graph() - with graph.as_default(): - variable_scope.get_variable_scope().set_use_resource(use_resource) - batch_size, depth = 5, 256 - inputs = array_ops.zeros((batch_size, depth)) - out_depth = 256 if with_bypass else 128 - activation_fn = None if with_bypass else activation - fc_scope = self._GetConvScope(scope, with_bypass) - scope = '' if scope is None else scope - delim = '/' if scope else '' - node = fully_connected( - inputs, - out_depth, - weights_initializer=self._WeightInit(0.03), - activation_fn=activation_fn, - scope=fc_scope) - if with_bypass: - node = math_ops.add(inputs, node, name=scope + delim + 'AddV2') - node = activation(node, name=scope + delim + activation_op_name) - update_barrier = control_flow_ops.no_op(name='update_barrier') - with ops.control_dependencies([update_barrier]): - array_ops.identity(node, name='control_dependency') - quantize.Quantize(graph, True, quant_delay=delay) - - self._AssertCorrectQuantizedGraphWithoutBatchNorm( - graph, scope, 'MatMul', activation_op_name, with_bypass, delay, - use_resource) - - def testQuantize_DepthwiseConv2dWithoutBatchNorm(self): - self._RunWithoutBatchNormTestOverParameters( - self._TestQuantize_DepthwiseConv2dWithoutBatchNorm) - - def _TestQuantize_DepthwiseConv2dWithoutBatchNorm( - self, activation, activation_op_name, with_bypass, delay, use_resource, - scope): - """Tests quantization: inputs -> DWConv2d no batch norm -> Activation. - - Args: - activation: Callable that returns an Operation, a factory method for the - Activation. - activation_op_name: String, name of the Activation operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Activation. - delay: Int (optional), delay in number of steps until quantization starts. - use_resource: Bool, when true uses resource variables. - scope: String, specifies top level scope for the graph - """ - graph = ops.Graph() - with graph.as_default(): - variable_scope.get_variable_scope().set_use_resource(use_resource) - batch_size, height, width, depth = 5, 128, 128, 3 - inputs = array_ops.zeros((batch_size, height, width, depth)) - stride = 1 if with_bypass else 2 - activation_fn = None if with_bypass else activation - conv_scope = self._GetConvScope(scope, with_bypass) - scope = '' if scope is None else scope - delim = '/' if scope else '' - - node = separable_conv2d( - inputs, - None, [5, 5], - stride=stride, - depth_multiplier=1.0, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - scope=conv_scope) - if with_bypass: - node = math_ops.add(inputs, node, name=scope + delim + 'AddV2') - node = activation(node, name=scope + delim + activation_op_name) - update_barrier = control_flow_ops.no_op(name='update_barrier') - with ops.control_dependencies([update_barrier]): - array_ops.identity(node, name='control_dependency') - quantize.Quantize(graph, True, quant_delay=delay) - - self._AssertCorrectQuantizedGraphWithoutBatchNorm( - graph, scope, 'DepthwiseConv2dNative', activation_op_name, with_bypass, - delay, use_resource) - - def testQuantize_AtrousConvWithoutBatchNorm(self): - self._RunWithoutBatchNormTestOverParameters( - self._TestQuantize_AtrousConvWithoutBatchNorm) - - def _TestQuantize_AtrousConvWithoutBatchNorm(self, activation, - activation_op_name, with_bypass, - delay, use_resource, scope): - """Tests quantization: inputs -> atrous conv no batch norm -> Activation. - - Args: - activation: Callable that returns an Operation, a factory method for the - Activation. - activation_op_name: String, name of the Activation operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Activation. - delay: Int (optional), delay in number of steps until quantization starts. - use_resource: Bool, when true uses resource variables. - scope: String, specifies top level scope for the graph - """ - graph = ops.Graph() - with graph.as_default(): - variable_scope.get_variable_scope().set_use_resource(use_resource) - batch_size, height, width, depth = 5, 128, 128, 3 - inputs = array_ops.zeros((batch_size, height, width, depth)) - dilation_rate = 2 - activation_fn = None if with_bypass else activation - conv_scope = self._GetConvScope(scope, with_bypass) - scope = '' if scope is None else scope - delim = '/' if scope else '' - - node = separable_conv2d( - inputs, - None, [3, 3], - rate=dilation_rate, - depth_multiplier=1.0, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - scope=conv_scope) - if with_bypass: - node = math_ops.add(inputs, node, name=scope + delim + 'AddV2') - node = activation(node, name=scope + delim + activation_op_name) - update_barrier = control_flow_ops.no_op(name='update_barrier') - with ops.control_dependencies([update_barrier]): - array_ops.identity(node, name='control_dependency') - quantize.Quantize(graph, True, quant_delay=delay) - - self._AssertCorrectQuantizedGraphWithoutBatchNorm( - graph, scope, 'DepthwiseConv2dNative', activation_op_name, with_bypass, - delay, use_resource) - - def _RunBatchNormTestOverParameters(self, test_fn): - # TODO(suharshs): Use parameterized test once OSS TF supports it. - parameters_list = [ - # (activation, activation_op_name, with_bypass, delay, fused_batch_norm) - (nn_ops.relu6, 'Relu6', False, None, False), - (nn_ops.relu, 'Relu', False, None, False), - (array_ops.identity, 'Identity', False, None, False), - (nn_ops.relu6, 'Relu6', False, 5000, False), - (nn_ops.relu, 'Relu', False, 5000, False), - (array_ops.identity, 'Identity', False, 5000, False), - (nn_ops.relu6, 'Relu6', True, None, False), - (nn_ops.relu, 'Relu', True, None, False), - (array_ops.identity, 'Identity', True, None, False), - (nn_ops.relu6, 'Relu6', True, 5000, False), - (nn_ops.relu, 'Relu', True, 5000, False), - (array_ops.identity, 'Identity', True, 5000, False), - (nn_ops.relu6, 'Relu6', False, None, True), - (nn_ops.relu, 'Relu', False, None, True), - (array_ops.identity, 'Identity', False, None, True), - (nn_ops.relu6, 'Relu6', False, 5000, True), - (nn_ops.relu, 'Relu', False, 5000, True), - (array_ops.identity, 'Identity', False, 5000, True), - (nn_ops.relu6, 'Relu6', True, None, True), - (nn_ops.relu, 'Relu', True, None, True), - (array_ops.identity, 'Identity', True, None, True), - (nn_ops.relu6, 'Relu6', True, 5000, True), - (nn_ops.relu, 'Relu', True, 5000, True), - (array_ops.identity, 'Identity', True, 5000, True) - ] - for params in parameters_list: - # Test everything with resource variables and normal variables. - test_fn(params[0], params[1], params[2], params[3], params[4], False, - None) - test_fn(params[0], params[1], params[2], params[3], params[4], True, None) - test_fn(params[0], params[1], params[2], params[3], params[4], False, - 'test') - test_fn(params[0], params[1], params[2], params[3], params[4], True, - 'test') - - def _AssertCorrectQuantizedGraphWithBatchNorm(self, graph, scope, layer, - activation_op_name, with_bypass, - delay, use_resource): - quantization_node_name = 'FakeQuantWithMinMaxVars' - conv_scope = self._GetConvScope(scope, with_bypass) - delim = '/' if conv_scope else '' - - if scope: - scope = scope + '/' - - weights_quant = graph.get_operation_by_name( - conv_scope + delim + 'weights_quant/' + quantization_node_name) - - self.assertEqual(weights_quant.type, quantization_node_name) - if use_resource: - expected_inputs = [ - conv_scope + delim + - 'weights_quant/FakeQuantWithMinMaxVars/ReadVariableOp', - conv_scope + delim + - 'weights_quant/FakeQuantWithMinMaxVars/ReadVariableOp_1', - ] - else: - expected_inputs = [ - conv_scope + delim + 'weights_quant/' + 'AssignMinLast', - conv_scope + delim + 'weights_quant/' + 'AssignMaxLast' - ] - expected_inputs.append(conv_scope + delim + 'mul_fold') - - self._AssertInputOpsAre(weights_quant, expected_inputs) - if layer == 'DepthwiseConv2dNative': - output_op_name = conv_scope + delim + ( - 'weights_quant/delayed_quant/Switch_1' if delay else 'depthwise_Fold') - else: - output_op_name = conv_scope + delim + ( - 'weights_quant/delayed_quant/Switch_1' if delay else layer + '_Fold') - self._AssertOutputGoesToOps(weights_quant, graph, [output_op_name]) - - if with_bypass: - conv_quant = graph.get_operation_by_name( - conv_scope + delim + 'conv_quant/' + quantization_node_name) - self.assertEqual(conv_quant.type, quantization_node_name) - - if use_resource: - expected_inputs = [ - conv_scope + delim + - 'conv_quant/FakeQuantWithMinMaxVars/ReadVariableOp', - conv_scope + delim + - 'conv_quant/FakeQuantWithMinMaxVars/ReadVariableOp_1', - ] - else: - expected_inputs = [ - conv_scope + delim + 'conv_quant/AssignMinEma', - conv_scope + delim + 'conv_quant/AssignMaxEma', - ] - expected_inputs.append(conv_scope + delim + 'add_fold') - - self._AssertInputOpsAre(conv_quant, expected_inputs) - output_op_name = ( - conv_scope + delim + - 'conv_quant/delayed_quant/Switch_1' if delay else scope + 'AddV2') - self._AssertOutputGoesToOps(conv_quant, graph, [output_op_name]) - - act_quant = graph.get_operation_by_name(scope + 'act_quant/' + - quantization_node_name) - self.assertEqual(act_quant.type, quantization_node_name) - - if use_resource: - expected_inputs = [ - scope + 'act_quant/FakeQuantWithMinMaxVars/ReadVariableOp', - scope + 'act_quant/FakeQuantWithMinMaxVars/ReadVariableOp_1', - ] - else: - expected_inputs = [ - scope + 'act_quant/AssignMinEma', - scope + 'act_quant/AssignMaxEma', - ] - expected_inputs.append(scope + activation_op_name) - - self._AssertInputOpsAre(act_quant, expected_inputs) - output_op_name = ( - scope + 'act_quant/delayed_quant/Switch_1' - if delay else 'control_dependency') - self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) - self._AssertIdempotent(graph) - - def testQuantize_Conv2dWithBatchNorm(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - self._RunBatchNormTestOverParameters( - self._TestQuantize_Conv2dWithBatchNorm) - - def _TestQuantize_Conv2dWithBatchNorm(self, activation, activation_op_name, - with_bypass, delay, fused_batch_norm, - use_resource, scope): - """Tests quantization: inputs -> Conv2d with batch norm -> Activation. - - Args: - activation: Callable that returns an Operation, a factory method for the - Activation. - activation_op_name: String, name of the Activation operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Activation. - delay: Int (optional), delay in number of steps until quantization starts. - fused_batch_norm: Bool, when true use FusedBatchNorm. - use_resource: Bool, when true uses resource variables. - scope: String, specifies top level scope for the graph - """ - graph = ops.Graph() - with graph.as_default(): - variable_scope.get_variable_scope().set_use_resource(use_resource) - batch_size, height, width, depth = 5, 128, 128, 3 - inputs = array_ops.zeros((batch_size, height, width, depth)) - stride = 1 if with_bypass else 2 - out_depth = 3 if with_bypass else 32 - conv_scope = self._GetConvScope(scope, with_bypass) - scope = '' if scope is None else scope - delim = '/' if scope else '' - node = conv2d( - inputs, - out_depth, [5, 5], - stride=stride, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - normalizer_fn=batch_norm, - normalizer_params=self._BatchNormParams(fused_batch_norm), - scope=conv_scope) - - # Manually add a bypass (optional) and an activation. - if with_bypass: - node = math_ops.add(inputs, node, name=scope + delim + 'AddV2') - - node = activation(node, name=scope + delim + activation_op_name) - - update_barrier = control_flow_ops.no_op(name='update_barrier') - with ops.control_dependencies([update_barrier]): - array_ops.identity(node, name='control_dependency') - - fold_batch_norms.FoldBatchNorms(graph, is_training=True) - quantize.Quantize(graph, True, quant_delay=delay) - - self._AssertCorrectQuantizedGraphWithBatchNorm( - graph, scope, 'Conv2D', activation_op_name, with_bypass, delay, - use_resource) - - def testQuantize_FCWithBatchNorm(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - self._RunBatchNormTestOverParameters(self._TestQuantize_FCWithBatchNorm) - - def _TestQuantize_FCWithBatchNorm(self, activation, activation_op_name, - with_bypass, delay, fused_batch_norm, - use_resource, scope): - """Tests quantization: inputs -> FC with batch norm -> Activation. - - Args: - activation: Callable that returns an Operation, a factory method for the - Activation. - activation_op_name: String, name of the Activation operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Activation. - delay: Int (optional), delay in number of steps until quantization starts. - fused_batch_norm: Bool, when true use FusedBatchNorm. - use_resource: Bool, when true uses resource variables. - scope: String, specifies top level scope for the graph - """ - graph = ops.Graph() - with graph.as_default(): - variable_scope.get_variable_scope().set_use_resource(use_resource) - batch_size, depth = 5, 256 - inputs = array_ops.zeros((batch_size, depth)) - out_depth = 256 if with_bypass else 128 - conv_scope = self._GetConvScope(scope, with_bypass) - scope = '' if scope is None else scope - delim = '/' if scope else '' - node = fully_connected( - inputs, - out_depth, - weights_initializer=self._WeightInit(0.03), - activation_fn=None, - normalizer_fn=batch_norm, - normalizer_params=self._BatchNormParams(fused_batch_norm), - scope=conv_scope) - - # Manually add a bypass (optional) and an activation. - if with_bypass: - node = math_ops.add(inputs, node, name=scope + delim + 'AddV2') - - node = activation(node, name=scope + delim + activation_op_name) - - update_barrier = control_flow_ops.no_op(name='update_barrier') - with ops.control_dependencies([update_barrier]): - array_ops.identity(node, name='control_dependency') - - fold_batch_norms.FoldBatchNorms(graph, is_training=True) - - quantize.Quantize(graph, True, quant_delay=delay) - - self._AssertCorrectQuantizedGraphWithBatchNorm( - graph, scope, 'MatMul', activation_op_name, with_bypass, delay, - use_resource) - - def testQuantize_DepthwiseConv2dWithBatchNorm(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - self._RunBatchNormTestOverParameters( - self._TestQuantize_DepthwiseConv2dWithBatchNorm) - - def _TestQuantize_DepthwiseConv2dWithBatchNorm( - self, activation, activation_op_name, with_bypass, delay, - fused_batch_norm, use_resource, scope): - """Tests quantization: inputs -> DWConv2d with batch norm -> Activation. - - Args: - activation: Callable that returns an Operation, a factory method for the - Activation. - activation_op_name: String, name of the Activation operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Activation. - delay: Int (optional), delay in number of steps until quantization starts. - fused_batch_norm: Bool, when true use FusedBatchNorm. - use_resource: Bool, when true uses resource variables. - scope: String, specifies top level scope for the graph - """ - graph = ops.Graph() - with graph.as_default(): - variable_scope.get_variable_scope().set_use_resource(use_resource) - batch_size, height, width, depth = 5, 128, 128, 3 - inputs = array_ops.zeros((batch_size, height, width, depth)) - stride = 1 if with_bypass else 2 - conv_scope = self._GetConvScope(scope, with_bypass) - scope = '' if scope is None else scope - delim = '/' if scope else '' - node = separable_conv2d( - inputs, - None, [5, 5], - stride=stride, - depth_multiplier=1.0, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - normalizer_fn=batch_norm, - normalizer_params=self._BatchNormParams(fused_batch_norm), - scope=conv_scope) - - # Manually add a bypass (optional) and an activation. - if with_bypass: - node = math_ops.add(inputs, node, name=scope + delim + 'AddV2') - - node = activation(node, name=scope + delim + activation_op_name) - - update_barrier = control_flow_ops.no_op(name='update_barrier') - with ops.control_dependencies([update_barrier]): - array_ops.identity(node, name='control_dependency') - - fold_batch_norms.FoldBatchNorms(graph, is_training=True) - quantize.Quantize(graph, True, quant_delay=delay) - - self._AssertCorrectQuantizedGraphWithBatchNorm( - graph, scope, 'DepthwiseConv2dNative', activation_op_name, - with_bypass, delay, use_resource) - - def testQuantize_AtrousConvWithBatchNorm(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - self._RunBatchNormTestOverParameters( - self._TestQuantize_AtrousConvWithBatchNorm) - - def _TestQuantize_AtrousConvWithBatchNorm( - self, activation, activation_op_name, with_bypass, delay, - fused_batch_norm, use_resource, scope): - """Tests quantization: inputs -> atrous conv with batch norm -> Activation. - - Args: - activation: Callable that returns an Operation, a factory method for the - Activation. - activation_op_name: String, name of the Activation operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Activation. - delay: Int (optional), delay in number of steps until quantization starts. - fused_batch_norm: Bool, when true use FusedBatchNorm. - use_resource: Bool, when true uses resource variables. - scope: String, specifies top level scope for the graph - """ - graph = ops.Graph() - with graph.as_default(): - variable_scope.get_variable_scope().set_use_resource(use_resource) - batch_size, height, width, depth = 5, 128, 128, 3 - inputs = array_ops.zeros((batch_size, height, width, depth)) - dilation_rate = 2 - conv_scope = self._GetConvScope(scope, with_bypass) - scope = '' if scope is None else scope - delim = '/' if scope else '' - - node = separable_conv2d( - inputs, - None, [3, 3], - rate=dilation_rate, - depth_multiplier=1.0, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - normalizer_fn=batch_norm, - normalizer_params=self._BatchNormParams(fused_batch_norm), - scope=conv_scope) - - # Manually add a bypass (optional) and an activation. - if with_bypass: - node = math_ops.add(inputs, node, name=scope + delim + 'AddV2') - - node = activation(node, name=scope + delim + activation_op_name) - - update_barrier = control_flow_ops.no_op(name='update_barrier') - with ops.control_dependencies([update_barrier]): - array_ops.identity(node, name='control_dependency') - - fold_batch_norms.FoldBatchNorms(graph, is_training=True) - quantize.Quantize(graph, True, quant_delay=delay) - - self._AssertCorrectQuantizedGraphWithBatchNorm( - graph, scope, 'DepthwiseConv2dNative', activation_op_name, - with_bypass, delay, use_resource) - - def _AssertIdempotent(self, graph): - # Ensure that calling the rewrite again doesn't change the graph. - graph_def_before = str(graph.as_graph_def()) - with graph.as_default(): - # Ensuring that calling the rewrite again doesn't add more nodes. - fold_batch_norms.FoldBatchNorms(graph, is_training=True) - quantize.Quantize(graph, True) - graph_def_after = str(graph.as_graph_def()) - self.assertEqual(graph_def_before, graph_def_after) - - def testBatchNormForcedUpdates(self): - with compat.forward_compatibility_horizon(2019, 6, 7): - parameter_list = [ - # (activation, activation_op_name, fused_batch_norm) - (nn_ops.relu6, 'Relu6', False), - (nn_ops.relu, 'Relu', False), - (array_ops.identity, 'Identity', False), - (nn_ops.relu6, 'Relu6', True), - (nn_ops.relu, 'Relu', True), - (array_ops.identity, 'Identity', True), - ] - for params in parameter_list: - self._TestBatchNormForcedUpdates(params[0], params[1], params[2], False) - self._TestBatchNormForcedUpdates(params[0], params[1], params[2], True) - - def _TestBatchNormForcedUpdates(self, activation, activation_op_name, - fused_batch_norm, use_resource): - """post_activation bypass quantization should happen with forced updates.""" - graph = ops.Graph() - with graph.as_default(): - variable_scope.get_variable_scope().set_use_resource(use_resource) - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - input2 = array_ops.zeros((batch_size, height / 2, width / 2, 32)) - # Setting updates_collections to None forces updates adding an extra - # identity operation following batch norms. - bn_params = self._BatchNormParams( - fused=fused_batch_norm, force_updates=True) - conv = conv2d( - input1, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation, - normalizer_fn=batch_norm, - normalizer_params=bn_params, - scope='test/test') - bypass_tensor = math_ops.add(conv, input2, name='test/add') - # The output of the post_activation bypass will be another layer. - _ = conv2d( - bypass_tensor, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - normalizer_fn=batch_norm, - normalizer_params=bn_params, - activation_fn=activation, - scope='test/unused') - - fold_batch_norms.FoldBatchNorms(graph, is_training=True) - quantize.Quantize(graph, is_training=True) - - # Ensure that the bypass node is preceded by and followed by a - # FakeQuantWithMinMaxVar operation, since the output of the Add isn't an - # activation. - self.assertTrue('FakeQuantWithMinMaxVars' in - [c.type for c in bypass_tensor.consumers()]) - self.assertTrue('FakeQuantWithMinMaxVars' in - [i.op.type for i in bypass_tensor.op.inputs]) - - with open('/tmp/bn_quant_test.pbtxt', 'w') as f: - f.write(str(graph.as_graph_def())) - - def _GetConvScope(self, scope, with_bypass): - if scope is None: - scope = '' - delim = '/' if scope else '' - - if with_bypass: - conv_scope = scope + delim + 'test2' - else: - conv_scope = scope - - return conv_scope - - def _BatchNormParams(self, fused=False, force_updates=False): - params = { - 'center': True, - 'scale': True, - 'decay': 1.0 - 0.003, - 'fused': fused - } - if force_updates: - params['updates_collections'] = None - return params - - def _WeightInit(self, stddev): - """Returns truncated normal variable initializer. - - Function is defined purely to shorten the name so that it stops wrapping. - - Args: - stddev: Standard deviation of normal variable. - - Returns: - An initialized that initializes with a truncated normal variable. - """ - return init_ops.truncated_normal_initializer(stddev=stddev) - - def _AssertInputOpsAre(self, op, in_op_names): - """Asserts that all inputs to op come from in_op_names (disregarding order). - - Args: - op: Operation to check inputs for. - in_op_names: List of strings, operations where all op's inputs should - come from. - """ - expected_inputs = [in_op_name + ':0' for in_op_name in in_op_names] - self.assertItemsEqual([t.name for t in op.inputs], expected_inputs) - - def _AssertOutputGoesToOps(self, op, graph, out_op_names): - """Asserts that outputs from op go to out_op_names (and perhaps others). - - Args: - op: Operation to check outputs for. - graph: Graph where output operations are located. - out_op_names: List of strings, operations where op's outputs should go. - """ - for out_op_name in out_op_names: - out_op = graph.get_operation_by_name(out_op_name) - self.assertIn(op.outputs[0].name, [str(t.name) for t in out_op.inputs]) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/quantize/python/quantize_test.py b/tensorflow/contrib/quantize/python/quantize_test.py deleted file mode 100644 index 5681a213fe5..00000000000 --- a/tensorflow/contrib/quantize/python/quantize_test.py +++ /dev/null @@ -1,581 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Unit tests for quantizing a Tensorflow graph.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework.python.ops import variables -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.quantize.python import quantize -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import partitioned_variables -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import googletest - -conv2d = layers.conv2d -separable_conv2d = layers.separable_conv2d - - -class QuantizeTest(test_util.TensorFlowTestCase): - - def _RunTestOverParameters(self, test_fn): - params = [True, False] - for is_training in params: - test_fn(is_training) - - def testInsertQuantOpFailsWhenOpsNotConnected(self): - pass - - def _TestInsertQuantOpFailsWhenOpsNotConnected(self, is_training): - graph = ops.Graph() - with graph.as_default(): - batch_size, height, width, depth = 5, 128, 128, 3 - inputs = array_ops.zeros((batch_size, height, width, depth)) - conv = conv2d(inputs, 32, [5, 5], stride=2, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, scope='test') - relu = nn_ops.relu6(inputs) - - # Inserting a quantization op between two unconnected ops should fail with - # ValueError. - with self.assertRaises(ValueError) as err: - quantize._InsertQuantOp('test', is_training, conv.op, [relu.op], - 'FailingQuantOp') - self.assertEqual( - str(err.exception), 'Some inputs not quantized for ops: [Relu6]') - - def testInsertQuantOpForAddAfterConv2d(self): - self._RunTestOverParameters(self._TestInsertQuantOpForAddAfterConv2d) - - def _TestInsertQuantOpForAddAfterConv2d(self, is_training): - graph = ops.Graph() - with graph.as_default(): - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - input2 = array_ops.zeros((batch_size, height / 2, width / 2, 32)) - conv = conv2d(input1, 32, [5, 5], stride=2, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, scope='test/test') - node = math_ops.add(conv, input2, name='test/add') - node = nn_ops.relu6(node, name='test/relu6') - update_barrier = control_flow_ops.no_op(name='update_barrier') - with ops.control_dependencies([update_barrier]): - array_ops.identity(node, name='control_dependency') - - quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8) - - quantization_node_name = 'FakeQuantWithMinMaxVars' - conv_quant = graph.get_operation_by_name('test/test/conv_quant/' + - quantization_node_name) - self.assertEqual(conv_quant.type, quantization_node_name) - - # Scan through all FakeQuant operations, ensuring that the activation - # isn't in the consumers of the operation. Since activations are folded - # the preceding operation during inference, the FakeQuant operation after - # the activation is all that is needed. - for op in graph.get_operations(): - if op.type == quantization_node_name: - quant_op = graph.get_operation_by_name(op.name) - consumers = [] - for output in quant_op.outputs: - consumers.extend(output.consumers()) - - self.assertNotIn('test/relu6', [c.name for c in consumers]) - - def testInsertQuantOpForAddAfterSeparableConv2d(self): - self._RunTestOverParameters( - self._TestInsertQuantOpForAddAfterSeparableConv2d) - - def _TestInsertQuantOpForAddAfterSeparableConv2d(self, is_training): - graph = ops.Graph() - with graph.as_default(): - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - input2 = array_ops.zeros((batch_size, height / 2, width / 2, depth)) - conv = separable_conv2d(input1, None, [5, 5], stride=2, - depth_multiplier=1.0, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, scope='test/test') - node = math_ops.add(conv, input2, name='test/add') - node = nn_ops.relu6(node, name='test/relu6') - update_barrier = control_flow_ops.no_op(name='update_barrier') - with ops.control_dependencies([update_barrier]): - array_ops.identity(node, name='control_dependency') - - quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8) - # Check if output of bias add is quantized - quantization_node_name = 'FakeQuantWithMinMaxVars' - conv_quant = graph.get_operation_by_name('test/test/conv_quant/' + - quantization_node_name) - self.assertEqual(conv_quant.type, quantization_node_name) - - for op in graph.get_operations(): - if op.type == quantization_node_name: - quant_op = graph.get_operation_by_name(op.name) - # Scan through all FakeQuant operations, ensuring that the activation - # identity op isn't in the consumers of the operation. - consumers = [] - for output in quant_op.outputs: - consumers.extend(output.consumers()) - - self.assertNotIn('test/relu6', [c.name for c in consumers]) - - def testInsertQuantOpInSeparableConv2d(self): - self._RunTestOverParameters(self._TestInsertQuantOpInSeparableConv2d) - - def _TestInsertQuantOpInSeparableConv2d(self, is_training): - graph = ops.Graph() - with graph.as_default(): - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - input2 = array_ops.zeros((batch_size, height / 2, width / 2, depth)) - conv = separable_conv2d( - input1, - 3, [5, 5], - stride=2, - depth_multiplier=1.0, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - scope='test/test') - node = math_ops.add(conv, input2, name='test/add') - node = nn_ops.relu6(node, name='test/relu6') - update_barrier = control_flow_ops.no_op(name='update_barrier') - with ops.control_dependencies([update_barrier]): - array_ops.identity(node, name='control_dependency') - - quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8) - # Check if output of bias add is quantized - quantization_node_name = 'FakeQuantWithMinMaxVars' - conv_quant = graph.get_operation_by_name('test/test/conv_quant/' + - quantization_node_name) - self.assertEqual(conv_quant.type, quantization_node_name) - - # Check if weights for both convs inside seperable conv are quantized - pointwise_weight_quant = graph.get_operation_by_name( - 'test/test/weights_quant/' + quantization_node_name) - self.assertEqual(pointwise_weight_quant.type, quantization_node_name) - depthwise_weight_quant = graph.get_operation_by_name( - 'test/test/separable_conv2d/weights_quant/' + quantization_node_name) - self.assertEqual(depthwise_weight_quant.type, quantization_node_name) - - # Check if activations after first depthwise conv are quantized. - depthwise_act_quant = graph.get_operation_by_name( - 'test/test/separable_conv2d/act_quant/' + quantization_node_name) - self.assertEqual(depthwise_act_quant.type, quantization_node_name) - - for op in graph.get_operations(): - if op.type == quantization_node_name: - quant_op = graph.get_operation_by_name(op.name) - # Scan through all FakeQuant operations, ensuring that the activation - # identity op isn't in the consumers of the operation. - consumers = [] - for output in quant_op.outputs: - consumers.extend(output.consumers()) - - self.assertNotIn('test/relu6', [c.name for c in consumers]) - - def testLayerActivationQuantized(self): - self._RunTestOverParameters(self._TestLayerActivationQuantized) - - def _TestLayerActivationQuantized(self, is_training): - graph = ops.Graph() - with graph.as_default(): - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - _ = conv2d( - input1, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=nn_ops.relu6, - biases_initializer=None, - scope='test') - # Ensure that both weights and output of activations are quantized - # when we have a conv->relu6 with no bias add - quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8) - activation_op = graph.get_operation_by_name('test/Relu6') - conv_op = graph.get_operation_by_name('test/Conv2D') - self.assertTrue('test/weights_quant/FakeQuantWithMinMaxVars:0' in - [tensor_in.name for tensor_in in conv_op.inputs]) - self.assertTrue('FakeQuantWithMinMaxVars' in - [op.type for op in activation_op.outputs[0].consumers()]) - - def testFinalLayerQuantized(self): - self._RunTestOverParameters(self._TestFinalLayerQuantized) - - def _TestFinalLayerQuantized(self, is_training): - graph = ops.Graph() - with graph.as_default(): - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - _ = conv2d( - input1, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - scope='test') - # Ensure that the a FakeQuant operation is in the outputs of the BiasAdd. - bias_add_op = graph.get_operation_by_name('test/BiasAdd') - quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8) - self.assertTrue('FakeQuantWithMinMaxVars' in - [op.type for op in bias_add_op.outputs[0].consumers()]) - - def testPostActivationBypassQuantized(self): - self._RunTestOverParameters(self._TestPostActivationBypassQuantized) - - def _TestPostActivationBypassQuantized(self, is_training): - graph = ops.Graph() - with graph.as_default(): - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - input2 = array_ops.zeros((batch_size, height / 2, width / 2, 32)) - conv = conv2d( - input1, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=nn_ops.relu6, - scope='test/test') - bypass_tensor = math_ops.add(conv, input2, name='test/add') - # The output of the post_activation bypass will be another layer. - _ = conv2d( - bypass_tensor, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=nn_ops.relu6, - scope='test/unused') - - quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8) - - # Ensure that the bypass node is preceded by and followed by a - # FakeQuantWithMinMaxVar operation, since the output of the Add isn't an - # activation. - self.assertTrue('FakeQuantWithMinMaxVars' in - [c.type for c in bypass_tensor.consumers()]) - self.assertTrue('FakeQuantWithMinMaxVars' in - [i.op.type for i in bypass_tensor.op.inputs]) - - def testOverlappingPostActivationBypassQuantized(self): - self._RunTestOverParameters( - self._TestOverlappingPostActivationBypassQuantized) - - def _TestOverlappingPostActivationBypassQuantized(self, is_training): - graph = ops.Graph() - with graph.as_default(): - batch_size, height, width, depth = 5, 128, 128, 3 - conv_input = array_ops.zeros((batch_size, height, width, depth)) - conv1 = conv2d( - conv_input, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=nn_ops.relu6, - scope='test/test1') - - # The bypass of this conv is the post activation bypass of the previous - # conv. - conv2 = conv2d( - conv_input, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - scope='test/test2') - - bypass_tensor = math_ops.add(conv1, conv2, name='test/add') - _ = nn_ops.relu6(bypass_tensor, name='test/output') - - quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8) - - # Ensure that the bypass node is preceded by a FakeQuantWithMinMaxVar - # operation, and NOT followed by one. - self.assertTrue('FakeQuantWithMinMaxVars' not in - [c.type for c in bypass_tensor.consumers()]) - self.assertTrue('FakeQuantWithMinMaxVars' in - [i.op.type for i in bypass_tensor.op.inputs]) - - # Ensure that all the convs and activations are quantized. - op_names = [op.name for op in graph.get_operations()] - self.assertTrue( - 'test/test1/weights_quant/FakeQuantWithMinMaxVars' in op_names) - self.assertTrue( - 'test/test2/weights_quant/FakeQuantWithMinMaxVars' in op_names) - self.assertTrue( - 'test/test1/act_quant/FakeQuantWithMinMaxVars' in op_names) - self.assertTrue('test/act_quant/FakeQuantWithMinMaxVars' in op_names) - self.assertEqual( - 'Relu6', - graph.get_operation_by_name( - 'test/test1/act_quant/FakeQuantWithMinMaxVars').inputs[0].op.type) - self.assertEqual( - 'Relu6', - graph.get_operation_by_name( - 'test/act_quant/FakeQuantWithMinMaxVars').inputs[0].op.type) - - def testWithNameScope(self): - self._RunTestOverParameters(self._TestWithNameScope) - - def _TestWithNameScope(self, is_training): - graph = ops.Graph() - with graph.as_default(): - with graph.name_scope('name_scope'): - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - _ = conv2d( - input1, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - scope='test') - - quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8) - - for op in graph.get_operations(): - self.assertTrue(not op.name.startswith('name_scope/name_scope/'), - 'Broken op: %s' % op.name) - - def testWithNullNameScope(self): - self._RunTestOverParameters(self._TestWithNullNameScope) - - def _TestWithNullNameScope(self, is_training): - graph = ops.Graph() - with graph.as_default(): - with graph.name_scope(None): - batch_size, height, width, depth = 5, 128, 128, 32 - input1 = array_ops.zeros((batch_size, height, width, depth)) - _ = conv2d( - input1, - 32, [5, 5], - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - scope='test') - - quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8) - # Passes if Quantize() does not crash. - - def testWithNonMatchingNameScope(self): - self._RunTestOverParameters(self._testWithNonMatchingNameScope) - - def _testWithNonMatchingNameScope(self, is_training): - graph = ops.Graph() - with graph.as_default(): - with graph.name_scope('name_scope'): - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - _ = conv2d( - input1, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - scope='test') - - op_names_before_quantize = set([op.name for op in graph.get_operations()]) - quantize.Quantize( - graph, is_training, weight_bits=8, activation_bits=8, - scope='NonExisting/') - op_names_after_quantize = set([op.name for op in graph.get_operations()]) - - # No ops should be inserted or removed. - self.assertEqual(op_names_before_quantize, op_names_after_quantize) - - def testSinglePartitionedVariable(self): - self._RunTestOverParameters(self._testSinglePartitionedVariable) - - def _testSinglePartitionedVariable(self, is_training): - # When weights are partitioned into a single partition, the weights variable - # is followed by a identity -> identity (An additional identity node). - partitioner = partitioned_variables.fixed_size_partitioner(1) - graph = ops.Graph() - with graph.as_default(): - with variable_scope.variable_scope('part', partitioner=partitioner): - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - input2 = array_ops.zeros((batch_size, height / 2, width / 2, 32)) - conv = conv2d( - input1, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - scope='test/test') - node = math_ops.add(conv, input2, name='test/add') - node = nn_ops.relu6(node, name='test/relu6') - - quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8) - # Check that the weight's quant node was added. - op_names = [op.name for op in graph.get_operations()] - self.assertTrue( - 'part/test/test/weights_quant/FakeQuantWithMinMaxVars' in op_names) - - def testMultiplePartitionedVariables(self): - self._RunTestOverParameters(self._testMultiplePartitionedVariables) - - def _testMultiplePartitionedVariables(self, is_training): - # When weights are partitioned into multiple partitions the weights variable - # is followed by a identity -> concat -> identity to group the partitions. - partitioner = partitioned_variables.fixed_size_partitioner(2) - graph = ops.Graph() - with graph.as_default(): - with variable_scope.variable_scope('part', partitioner=partitioner): - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - input2 = array_ops.zeros((batch_size, height / 2, width / 2, 32)) - conv = conv2d( - input1, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - scope='test/test') - node = math_ops.add(conv, input2, name='test/add') - node = nn_ops.relu6(node, name='test/relu6') - - quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8) - # Check that the weight's quant node was added. - op_names = [op.name for op in graph.get_operations()] - self.assertTrue( - 'part/test/test/weights_quant/FakeQuantWithMinMaxVars' in op_names) - - def testSkipReshapeQuantization(self): - self._RunTestOverParameters(self._TestSkipReshapeQuantization) - - def _TestSkipReshapeQuantization(self, is_training): - graph = ops.Graph() - with graph.as_default(): - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - conv = conv2d( - input1, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=nn_ops.relu6, - scope='test/test') - - reshape = array_ops.reshape( - conv, (int(10), int(height / 2), int(width / 2), int(16))) - - # Insert a fake quant node after the reshape. We will check that one isn't - # insert before. - array_ops.fake_quant_with_min_max_vars(reshape, -1, 1) - - quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8) - - # Ensure that there isn't a FakeQuant added before the reshape. - self.assertFalse( - 'FakeQuantWithMinMaxVars' in [i.op.type for i in reshape.op.inputs]) - - graph = ops.Graph() - with graph.as_default(): - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - conv = conv2d( - input1, - 32, [5, 5], - stride=2, - padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=nn_ops.relu6, - scope='test/test') - - reshape = array_ops.reshape( - conv, (int(10), int(height / 2), int(width / 2), int(16))) - - # If no fake quant is added after the reshape, a FakeQuant should be added - # before the reshape. - quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8) - - # Ensure that there isn't a FakeQuant added before the reshape. - self.assertTrue( - 'FakeQuantWithMinMaxVars' in [i.op.type for i in reshape.op.inputs]) - - def testSeparableConvWithResourceVar(self): - graph = ops.Graph() - with graph.as_default(): - with variable_scope.variable_scope('', use_resource=True): - batch_size, height, width, depth = 5, 128, 128, 3 - input1 = array_ops.zeros((batch_size, height, width, depth)) - kernel_size, depth_multiplier = 3, 1 - depthwise_shape = [kernel_size, kernel_size, depth, depth_multiplier] - depthwise_weights = variables.model_variable( - 'depthwise_weights', shape=depthwise_shape) - strides = [1, 1, 1, 1] - with variable_scope.variable_scope('depthwise_conv_1'): - conv1 = nn.depthwise_conv2d( - input1, depthwise_weights, strides, padding='SAME') - with variable_scope.variable_scope('depthwise_conv_2'): - conv2 = nn.depthwise_conv2d( - conv1, depthwise_weights, strides, padding='SAME') - math_ops.add(conv2, input1, name='add') - - quantize.Quantize(graph, True) - - # Test that the weights and activations of all convs have been quantized. - quant_node_name = 'FakeQuantWithMinMaxVars' - weights_quant = graph.get_operation_by_name( - 'depthwise_conv_1/weights_quant/' + quant_node_name) - self.assertEqual(weights_quant.type, quant_node_name) - act_quant = graph.get_operation_by_name('depthwise_conv_1/act_quant/' + - quant_node_name) - self.assertEqual(act_quant.type, quant_node_name) - - weights_quant = graph.get_operation_by_name( - 'depthwise_conv_2/weights_quant/' + quant_node_name) - self.assertEqual(weights_quant.type, quant_node_name) - act_quant = graph.get_operation_by_name('depthwise_conv_2/act_quant/' + - quant_node_name) - self.assertEqual(act_quant.type, quant_node_name) - - def _WeightInit(self, stddev): - """Returns truncated normal variable initializer. - - Function is defined purely to shorten the name so that it stops wrapping. - - Args: - stddev: Standard deviation of normal variable. - - Returns: - An initialized that initializes with a truncated normal variable. - """ - return init_ops.truncated_normal_initializer(stddev=stddev) - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/rate/BUILD b/tensorflow/contrib/rate/BUILD deleted file mode 100644 index d3f2591b047..00000000000 --- a/tensorflow/contrib/rate/BUILD +++ /dev/null @@ -1,55 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "rate", - srcs = [ - "rate.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "rate_test", - size = "small", - srcs = ["rate_test.py"], - python_version = "PY2", - tags = [ - "manual", # TODO(b/120555555) - "no_oss", # TODO(b/120555555) - "notap", # TODO(b/120555555) - ], - deps = [ - ":rate", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:variables", - "//tensorflow/python/eager:test", - ], -) diff --git a/tensorflow/contrib/rate/rate.py b/tensorflow/contrib/rate/rate.py deleted file mode 100644 index d948066b364..00000000000 --- a/tensorflow/contrib/rate/rate.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implementation of tf.contrib.rate module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re - -from tensorflow.python.eager import context -from tensorflow.python.eager import function -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope - -_to_replace = re.compile("[^A-Za-z0-9.]") - - -class Rate(object): - """Computes the rate of change since the last rate call.""" - - def __init__(self, name=None): - self._built = False - self._vars = [] - self._initial_values = {} - name = name or self.__class__.__name__ - # Replace things like spaces in name to create a valid scope name. - scope_name = _to_replace.sub("_", name) - # We create the variable scope now to get the unique name that will - # be used as a variable prefix when build() calls _add_variable(). - with variable_scope.variable_scope( - scope_name, use_resource=True, reuse=False) as scope: - pos = scope.name.rfind(scope_name) - self._name = name + scope.name[pos + len(scope_name):] - self._scope = scope - - # Ensures that if the user calls build directly we still set self._built to - # True to prevent variables from being recreated. - self._build = self.build - if context.executing_eagerly(): - self._construction_scope = context.eager_mode - else: - # We make self.call() into a graph callable here, so that we can - # return a single op that performs all of the variable updates. - self._construction_scope = ops.get_default_graph().as_default - self.call = function.defun(self.call) - - def build(self, values, denominator): - """Method to create variables. - - Called by `__call__()` before `call()` for the first time. - - Args: - values: The numerator for rate. - denominator: Value to which the rate is taken with respect. - """ - self.numer = self._add_variable( - name="numer", shape=values.get_shape(), dtype=dtypes.float64) - self.denom = self._add_variable( - name="denom", shape=denominator.get_shape(), dtype=dtypes.float64) - self.prev_values = self._add_variable( - name="prev_values", shape=values.get_shape(), dtype=dtypes.float64) - self.prev_denominator = self._add_variable( - name="prev_denominator", - shape=denominator.get_shape(), - dtype=dtypes.float64) - self._built = True - - def __call__(self, *args, **kwargs): - """Returns op to execute to update. - - Returns None if eager execution is enabled. - Returns a graph-mode function if graph execution is enabled. - - Args: - *args: - **kwargs: A mini-batch of inputs to Rate, passed on to `call()`. - """ - if not self._built: - with variable_scope.variable_scope( - self._scope), self._construction_scope(): - self.build(*args, **kwargs) - self._built = True - return self.call(*args, **kwargs) - - @property - def name(self): - return self._name - - @property - def variables(self): - return self._vars - - def _add_variable(self, name, shape=None, dtype=None): - """Private method for adding variables to the graph.""" - if self._built: - raise RuntimeError("Can't call add_variable() except in build().") - v = resource_variable_ops.ResourceVariable( - lambda: array_ops.zeros(shape, dtype), - trainable=False, - validate_shape=True, - name=name, - collections=[ops.GraphKeys.LOCAL_VARIABLES]) - return v - - def call(self, values, denominator): - """Computes the rate since the last call. - - Args: - values: Tensor with the per-example value. - denominator: Measure to take the rate with respect to. - - Returns: - The rate or 0 if denominator is unchanged since last call. - """ - if denominator.dtype != dtypes.float64: - denominator = math_ops.cast(denominator, dtypes.float64) - if values.dtype != dtypes.float64: - values = math_ops.cast(values, dtypes.float64) - - state_ops.assign(self.numer, math_ops.subtract(values, self.prev_values)) - state_ops.assign(self.denom, - math_ops.subtract(denominator, self.prev_denominator)) - state_ops.assign(self.prev_values, values) - state_ops.assign(self.prev_denominator, denominator) - - return math_ops.div_no_nan(self.numer, - math_ops.maximum(self.denom, 0), - name="safe_rate") diff --git a/tensorflow/contrib/rate/rate_test.py b/tensorflow/contrib/rate/rate_test.py deleted file mode 100644 index 3dee1638818..00000000000 --- a/tensorflow/contrib/rate/rate_test.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Rate.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.rate import rate -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class RateTest(test.TestCase): - - @test_util.run_in_graph_and_eager_modes() - def testBuildRate(self): - m = rate.Rate() - m.build( - constant_op.constant([1], dtype=dtypes.float32), - constant_op.constant([2], dtype=dtypes.float32)) - old_numer = m.numer - m( - constant_op.constant([2], dtype=dtypes.float32), - constant_op.constant([2], dtype=dtypes.float32)) - self.assertTrue(old_numer is m.numer) - - @test_util.run_in_graph_and_eager_modes() - def testBasic(self): - with self.cached_session(): - r_ = rate.Rate() - a = r_(array_ops.ones([1]), denominator=array_ops.ones([1])) - self.evaluate(variables.global_variables_initializer()) - self.evaluate(variables.local_variables_initializer()) - self.assertEqual([[1]], self.evaluate(a)) - b = r_(constant_op.constant([2]), denominator=constant_op.constant([2])) - self.assertEqual([[1]], self.evaluate(b)) - c = r_(constant_op.constant([4]), denominator=constant_op.constant([3])) - self.assertEqual([[2]], self.evaluate(c)) - d = r_(constant_op.constant([16]), denominator=constant_op.constant([3])) - self.assertEqual([[0]], self.evaluate(d)) # divide by 0 - - def testNamesWithSpaces(self): - m1 = rate.Rate(name="has space") - m1(array_ops.ones([1]), array_ops.ones([1])) - self.assertEqual(m1.name, "has space") - self.assertEqual(m1.prev_values.name, "has_space_1/prev_values:0") - - @test_util.run_in_graph_and_eager_modes() - def testWhileLoop(self): - with self.cached_session(): - r_ = rate.Rate() - - def body(value, denom, i, ret_rate): - i += 1 - ret_rate = r_(value, denom) - with ops.control_dependencies([ret_rate]): - value = math_ops.add(value, 2) - denom = math_ops.add(denom, 1) - return [value, denom, i, ret_rate] - - def condition(v, d, i, r): - del v, d, r # unused vars by condition - return math_ops.less(i, 100) - - i = constant_op.constant(0) - value = constant_op.constant([1], dtype=dtypes.float64) - denom = constant_op.constant([1], dtype=dtypes.float64) - ret_rate = r_(value, denom) - self.evaluate(variables.global_variables_initializer()) - self.evaluate(variables.local_variables_initializer()) - loop = control_flow_ops.while_loop(condition, body, - [value, denom, i, ret_rate]) - self.assertEqual([[2]], self.evaluate(loop[3])) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/receptive_field/BUILD b/tensorflow/contrib/receptive_field/BUILD deleted file mode 100644 index f2ae0ce9ff4..00000000000 --- a/tensorflow/contrib/receptive_field/BUILD +++ /dev/null @@ -1,115 +0,0 @@ -# Description: -# Contains modules to compute receptive field parameters for CNN models. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -# Transitive dependencies of this target will be included in the pip package. -py_library( - name = "receptive_field_pip", - deps = [ - ":receptive_field_py", - ], -) - -py_library( - name = "graph_compute_order_py", - srcs = [ - "python/util/graph_compute_order.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":parse_layer_parameters_py", - "//tensorflow/python:platform", - ], -) - -py_library( - name = "parse_layer_parameters_py", - srcs = [ - "python/util/parse_layer_parameters.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:platform", - ], -) - -py_library( - name = "receptive_field_py", - srcs = [ - "python/util/parse_layer_parameters.py", - "python/util/receptive_field.py", - "receptive_field_api.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":graph_compute_order_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:framework_ops", - "//tensorflow/python:platform", - "//third_party/py/numpy", - ], -) - -py_test( - name = "graph_compute_order_test", - srcs = ["python/util/graph_compute_order_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":graph_compute_order_py", - ":receptive_field_py", - "//tensorflow/contrib/slim", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:nn", - ], -) - -py_test( - name = "parse_layer_parameters_test", - srcs = ["python/util/parse_layer_parameters_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":graph_compute_order_py", - ":parse_layer_parameters_py", - "//tensorflow/contrib/slim", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:graph_util", - "//tensorflow/python:nn", - "//tensorflow/python:session", - "//tensorflow/python:variables", - "@absl_py//absl/testing:parameterized", - ], -) - -py_test( - name = "receptive_field_test", - srcs = ["python/util/receptive_field_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":receptive_field_py", - "//tensorflow/contrib/slim", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:nn", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/receptive_field/README.md b/tensorflow/contrib/receptive_field/README.md deleted file mode 100644 index 6c74d0df38c..00000000000 --- a/tensorflow/contrib/receptive_field/README.md +++ /dev/null @@ -1,188 +0,0 @@ -# Receptive field computation for convnets - -This library enables you to easily compute the receptive field parameters of -your favorite convnet. You can use it to understand how big of an input image -region your output features depend on. Better yet, using the parameters computed -by the library, you can easily find the exact image region which is used to -compute each convnet feature. - -This library can be used to compute receptive field parameters of popular -convnets: - -
- -convnet model | receptive field | effective stride | effective padding | FLOPs (Billion) -:-----------------: | :------------------: | :-------------------: | :--------------------: | :------------------: -alexnet_v2 | 195 | 32 | 64 | 1.38 -vgg_16 | 212 | 32 | 90 | 30.71 -inception_v2 | 699 | 32 | 318 | 3.88 -inception_v3 | 1311 | 32 | 618 | 5.69 -inception_v4 | 2071 | 32 | 998 | 12.27 -inception_resnet_v2 | 3039 | 32 | 1482 | 12.96 -mobilenet_v1 | 315 | 32 | 126 | 1.14 -mobilenet_v1_075 | 315 | 32 | 126 | 0.65 -resnet_v1_50 | 483 | 32 | 239 | 6.97 -resnet_v1_101 | 1027 | 32 | 511 | 14.40 -resnet_v1_152 | 1507 | 32 | 751 | 21.82 -resnet_v1_200 | 1763 | 32 | 879 | 28.82 - -
- -A comprehensive table with pre-computed receptive field parameters for different -end-points, input resolutions, and other variants of these networks can be found -[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/receptive_field/RECEPTIVE_FIELD_TABLE.md). - -## Basic usage - -The main function to be called is `compute_receptive_field_from_graph_def`, -which will return the receptive field, effective stride and effective padding -for both horizontal and vertical directions. - -For example, if your model is constructed using the function -`my_model_construction()`, you can use the library as follows: - -```python -import tensorflow as tf - -# Construct graph. -g = tf.Graph() -with g.as_default(): - images = tf.placeholder(tf.float32, shape=(1, None, None, 3), name='input_image') - my_model_construction(images) - -# Compute receptive field parameters. -rf_x, rf_y, eff_stride_x, eff_stride_y, eff_pad_x, eff_pad_y = \ - tf.contrib.receptive_field.compute_receptive_field_from_graph_def( \ - g.as_graph_def(), 'input_image', 'my_output_endpoint') -``` - -Here's a simple example of computing the receptive field parameters for -Inception-Resnet-v2. To get this to work, be sure to checkout -[tensorflow/models](https://github.com/tensorflow/models), so that the Inception -models are available to you. This can be done in three simple commands: - -```sh -git clone https://github.com/tensorflow/models -cd models/research/slim -sudo python setup.py install_lib -``` - -You can then compute the receptive field parameters for Inception-Resnet-v2 as: - -```python -from nets import inception -import tensorflow as tf - -# Construct graph. -g = tf.Graph() -with g.as_default(): - images = tf.placeholder(tf.float32, shape=(1, None, None, 3), name='input_image') - inception.inception_resnet_v2_base(images) - -# Compute receptive field parameters. -rf_x, rf_y, eff_stride_x, eff_stride_y, eff_pad_x, eff_pad_y = \ - tf.contrib.receptive_field.compute_receptive_field_from_graph_def( \ - g.as_graph_def(), 'input_image', 'InceptionResnetV2/Conv2d_7b_1x1/Relu') -``` - -This will give you `rf_x = rf_y = 3039`, `eff_stride_x = eff_stride_y = 32`, and -`eff_pad_x = eff_pad_y = 1482`. This means that each feature that is output at -the node `'InceptionResnetV2/Conv2d_7b_1x1/Relu'` is computed from a region -which is of size `3039x3039`. Further, by using the expressions - -```python -center_x = -eff_pad_x + feature_x*eff_stride_x + (rf_x - 1)/2 -center_y = -eff_pad_y + feature_y*eff_stride_y + (rf_y - 1)/2 -``` - -one can compute the center of the region in the input image that is used to -compute the output feature at position `[feature_x, feature_y]`. For example, -the feature at position `[0, 2]` at the output of the layer -`'InceptionResnetV2/Conv2d_7b_1x1/Relu'` is centered in the original image in -the position `[37, 101]`. - -TODO: include link to derivations and definitions of different parameters. - -## Receptive field benchmark - -As you might expect, it is straightforward to run this library on the popular -convnets, and gather their receptive fields. We provide a python script which -does exactly that, available under `python/util/examples/rf_benchmark.py`. - -To get this to work, be sure to checkout -[tensorflow/models](https://github.com/tensorflow/models) (see the 3-command -instructions for this above). Then, simply: - -```sh -cd python/util/examples -python rf_benchmark.py --csv_path /tmp/rf_benchmark_results.csv -``` - -The script will write to stdout the receptive field parameters for many variants -of several popular convnets: AlexNet, VGG, ResNet, Inception, Mobilenet. They -are also written to the file `/tmp/rf_benchmark_results.csv`. - -A comprehensive table with pre-computed receptive field parameters for different -networks can be found -[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/receptive_field/RECEPTIVE_FIELD_TABLE.md). - -## Compute RF parameters from a graph pbtxt - -We also provide a utility to compute the receptive field parameters directly -from a graph protobuf file. - -Have a `graph.pbtxt` file and want to compute its receptive field parameters? We -got you covered. The only prerequisite is to install -[google/protobuf](https://github.com/google/protobuf), which you probably -already have if you're using tensorflow (otherwise, follow installation -instructions [here](https://github.com/google/protobuf/tree/master/python)). - -This should work: - -```sh -cd python/util/examples -python compute_rf.py \ - --graph_path /path/to/graph.pbtxt \ - --output_path /path/to/output/rf_info.txt \ - --input_node my_input_node \ - --output_node my_output_node -``` - -Don't know how to generate a graph protobuf file? Take a look at the -`write_inception_resnet_v2_graph.py` script, which shows how to save it for the -Inception-Resnet-v2 model: - -```sh -cd python/util/examples -python write_inception_resnet_v2_graph.py --graph_dir /tmp --graph_filename graph.pbtxt -``` - -This will write the Inception-Resnet-v2 graph protobuf to `/tmp/graph.pbtxt`. - -For completeness, here's how you would use this file to get the receptive field -parameters of the Inception-Resnet-v2 model: - -```sh -cd python/util/examples -python compute_rf.py \ - --graph_path /tmp/graph.pbtxt \ - --output_path /tmp/rf_info.txt \ - --input_node input_image \ - --output_node InceptionResnetV2/Conv2d_7b_1x1/Relu -``` - -This will write the receptive field parameters of the model to -`/tmp/rf_info.txt`, which will look like: - -```sh -Receptive field size (horizontal) = 3039 -Receptive field size (vertical) = 3039 -Effective stride (horizontal) = 32 -Effective stride (vertical) = 32 -Effective padding (horizontal) = 1482 -Effective padding (vertical) = 1482 -``` - -## Authors - -André Araujo (@andrefaraujo) and Mark Sandler (@marksandler) diff --git a/tensorflow/contrib/receptive_field/RECEPTIVE_FIELD_TABLE.md b/tensorflow/contrib/receptive_field/RECEPTIVE_FIELD_TABLE.md deleted file mode 100644 index 1c0c35c08c8..00000000000 --- a/tensorflow/contrib/receptive_field/RECEPTIVE_FIELD_TABLE.md +++ /dev/null @@ -1,654 +0,0 @@ -# Pre-computed receptive field parameters - -## Table with results - -The table below presents the receptive field parameters and cost (in terms of -floating point operations — FLOPs) for several popular convolutional -neural networks and their end-points. These are computed using the models from -the -[TF-Slim repository](https://github.com/tensorflow/models/tree/master/research/slim), -by using the -[rf_benchmark script](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/receptive_field/python/util/examples/rf_benchmark.py). - -Questions? See the [FAQ](#faq). - -CNN | resolution | end-point | FLOPs (Billion) | RF | effective stride | effective padding -:----------------------------: | :--------: | :------------------: | :-------------: | :--: | :--------------: | :---------------: -alexnet_v2 | None | alexnet_v2/conv1 | None | 11 | 4 | 0 -alexnet_v2 | None | alexnet_v2/pool1 | None | 19 | 8 | 0 -alexnet_v2 | None | alexnet_v2/conv2 | None | 51 | 8 | 16 -alexnet_v2 | None | alexnet_v2/conv3 | None | 99 | 16 | 32 -alexnet_v2 | None | alexnet_v2/conv4 | None | 131 | 16 | 48 -alexnet_v2 | None | alexnet_v2/conv5 | None | 163 | 16 | 64 -alexnet_v2 | None | alexnet_v2/pool5 | None | 195 | 32 | 64 -alexnet_v2 | 224 | alexnet_v2/conv1 | 0.136 | 11 | 4 | 0 -alexnet_v2 | 224 | alexnet_v2/pool1 | 0.136 | 19 | 8 | 0 -alexnet_v2 | 224 | alexnet_v2/conv2 | 0.552 | 51 | 8 | 16 -alexnet_v2 | 224 | alexnet_v2/conv3 | 0.743 | 99 | 16 | 32 -alexnet_v2 | 224 | alexnet_v2/conv4 | 1.125 | 131 | 16 | 48 -alexnet_v2 | 224 | alexnet_v2/conv5 | 1.380 | 163 | 16 | 64 -alexnet_v2 | 224 | alexnet_v2/pool5 | 1.380 | 195 | 32 | 64 -alexnet_v2 | 321 | alexnet_v2/conv1 | 0.283 | 11 | 4 | 0 -alexnet_v2 | 321 | alexnet_v2/pool1 | 0.284 | 19 | 8 | 0 -alexnet_v2 | 321 | alexnet_v2/conv2 | 1.171 | 51 | 8 | 16 -alexnet_v2 | 321 | alexnet_v2/conv3 | 1.602 | 99 | 16 | 32 -alexnet_v2 | 321 | alexnet_v2/conv4 | 2.462 | 131 | 16 | 48 -alexnet_v2 | 321 | alexnet_v2/conv5 | 3.036 | 163 | 16 | 64 -alexnet_v2 | 321 | alexnet_v2/pool5 | 3.036 | 195 | 32 | 64 -vgg_a | None | vgg_a/conv1/conv1_1 | None | 3 | 1 | 1 -vgg_a | None | vgg_a/pool1 | None | 4 | 2 | 1 -vgg_a | None | vgg_a/conv2/conv2_1 | None | 8 | 2 | 3 -vgg_a | None | vgg_a/pool2 | None | 10 | 4 | 3 -vgg_a | None | vgg_a/conv3/conv3_1 | None | 18 | 4 | 7 -vgg_a | None | vgg_a/conv3/conv3_2 | None | 26 | 4 | 11 -vgg_a | None | vgg_a/pool3 | None | 30 | 8 | 11 -vgg_a | None | vgg_a/conv4/conv4_1 | None | 46 | 8 | 19 -vgg_a | None | vgg_a/conv4/conv4_2 | None | 62 | 8 | 27 -vgg_a | None | vgg_a/pool4 | None | 70 | 16 | 27 -vgg_a | None | vgg_a/conv5/conv5_1 | None | 102 | 16 | 43 -vgg_a | None | vgg_a/conv5/conv5_2 | None | 134 | 16 | 59 -vgg_a | None | vgg_a/pool5 | None | 150 | 32 | 59 -vgg_a | 224 | vgg_a/conv1/conv1_1 | 0.177 | 3 | 1 | 1 -vgg_a | 224 | vgg_a/pool1 | 0.180 | 4 | 2 | 1 -vgg_a | 224 | vgg_a/conv2/conv2_1 | 2.031 | 8 | 2 | 3 -vgg_a | 224 | vgg_a/pool2 | 2.033 | 10 | 4 | 3 -vgg_a | 224 | vgg_a/conv3/conv3_1 | 3.883 | 18 | 4 | 7 -vgg_a | 224 | vgg_a/conv3/conv3_2 | 7.583 | 26 | 4 | 11 -vgg_a | 224 | vgg_a/pool3 | 7.584 | 30 | 8 | 11 -vgg_a | 224 | vgg_a/conv4/conv4_1 | 9.434 | 46 | 8 | 19 -vgg_a | 224 | vgg_a/conv4/conv4_2 | 13.134 | 62 | 8 | 27 -vgg_a | 224 | vgg_a/pool4 | 13.134 | 70 | 16 | 27 -vgg_a | 224 | vgg_a/conv5/conv5_1 | 14.059 | 102 | 16 | 43 -vgg_a | 224 | vgg_a/conv5/conv5_2 | 14.984 | 134 | 16 | 59 -vgg_a | 224 | vgg_a/pool5 | 14.984 | 150 | 32 | 59 -vgg_a | 321 | vgg_a/conv1/conv1_1 | 0.363 | 3 | 1 | 1 -vgg_a | 321 | vgg_a/pool1 | 0.369 | 4 | 2 | 1 -vgg_a | 321 | vgg_a/conv2/conv2_1 | 4.147 | 8 | 2 | 3 -vgg_a | 321 | vgg_a/pool2 | 4.151 | 10 | 4 | 3 -vgg_a | 321 | vgg_a/conv3/conv3_1 | 7.927 | 18 | 4 | 7 -vgg_a | 321 | vgg_a/conv3/conv3_2 | 15.479 | 26 | 4 | 11 -vgg_a | 321 | vgg_a/pool3 | 15.480 | 30 | 8 | 11 -vgg_a | 321 | vgg_a/conv4/conv4_1 | 19.256 | 46 | 8 | 19 -vgg_a | 321 | vgg_a/conv4/conv4_2 | 26.806 | 62 | 8 | 27 -vgg_a | 321 | vgg_a/pool4 | 26.807 | 70 | 16 | 27 -vgg_a | 321 | vgg_a/conv5/conv5_1 | 28.695 | 102 | 16 | 43 -vgg_a | 321 | vgg_a/conv5/conv5_2 | 30.583 | 134 | 16 | 59 -vgg_a | 321 | vgg_a/pool5 | 30.583 | 150 | 32 | 59 -vgg_16 | None | vgg_16/conv1/conv1_1 | None | 3 | 1 | 1 -vgg_16 | None | vgg_16/pool1 | None | 6 | 2 | 2 -vgg_16 | None | vgg_16/conv2/conv2_1 | None | 10 | 2 | 4 -vgg_16 | None | vgg_16/pool2 | None | 16 | 4 | 6 -vgg_16 | None | vgg_16/conv3/conv3_1 | None | 24 | 4 | 10 -vgg_16 | None | vgg_16/conv3/conv3_2 | None | 32 | 4 | 14 -vgg_16 | None | vgg_16/pool3 | None | 44 | 8 | 18 -vgg_16 | None | vgg_16/conv4/conv4_1 | None | 60 | 8 | 26 -vgg_16 | None | vgg_16/conv4/conv4_2 | None | 76 | 8 | 34 -vgg_16 | None | vgg_16/pool4 | None | 100 | 16 | 42 -vgg_16 | None | vgg_16/conv5/conv5_1 | None | 132 | 16 | 58 -vgg_16 | None | vgg_16/conv5/conv5_2 | None | 164 | 16 | 74 -vgg_16 | None | vgg_16/pool5 | None | 212 | 32 | 90 -vgg_16 | 224 | vgg_16/conv1/conv1_1 | 0.177 | 3 | 1 | 1 -vgg_16 | 224 | vgg_16/pool1 | 3.882 | 6 | 2 | 2 -vgg_16 | 224 | vgg_16/conv2/conv2_1 | 5.734 | 10 | 2 | 4 -vgg_16 | 224 | vgg_16/pool2 | 9.436 | 16 | 4 | 6 -vgg_16 | 224 | vgg_16/conv3/conv3_1 | 11.287 | 24 | 4 | 10 -vgg_16 | 224 | vgg_16/conv3/conv3_2 | 14.987 | 32 | 4 | 14 -vgg_16 | 224 | vgg_16/pool3 | 18.688 | 44 | 8 | 18 -vgg_16 | 224 | vgg_16/conv4/conv4_1 | 20.538 | 60 | 8 | 26 -vgg_16 | 224 | vgg_16/conv4/conv4_2 | 24.238 | 76 | 8 | 34 -vgg_16 | 224 | vgg_16/pool4 | 27.938 | 100 | 16 | 42 -vgg_16 | 224 | vgg_16/conv5/conv5_1 | 28.863 | 132 | 16 | 58 -vgg_16 | 224 | vgg_16/conv5/conv5_2 | 29.788 | 164 | 16 | 74 -vgg_16 | 224 | vgg_16/pool5 | 30.713 | 212 | 32 | 90 -vgg_16 | 321 | vgg_16/conv1/conv1_1 | 0.363 | 3 | 1 | 1 -vgg_16 | 321 | vgg_16/pool1 | 7.973 | 6 | 2 | 2 -vgg_16 | 321 | vgg_16/conv2/conv2_1 | 11.751 | 10 | 2 | 4 -vgg_16 | 321 | vgg_16/pool2 | 19.307 | 16 | 4 | 6 -vgg_16 | 321 | vgg_16/conv3/conv3_1 | 23.084 | 24 | 4 | 10 -vgg_16 | 321 | vgg_16/conv3/conv3_2 | 30.635 | 32 | 4 | 14 -vgg_16 | 321 | vgg_16/pool3 | 38.188 | 44 | 8 | 18 -vgg_16 | 321 | vgg_16/conv4/conv4_1 | 41.964 | 60 | 8 | 26 -vgg_16 | 321 | vgg_16/conv4/conv4_2 | 49.514 | 76 | 8 | 34 -vgg_16 | 321 | vgg_16/pool4 | 57.066 | 100 | 16 | 42 -vgg_16 | 321 | vgg_16/conv5/conv5_1 | 58.954 | 132 | 16 | 58 -vgg_16 | 321 | vgg_16/conv5/conv5_2 | 60.841 | 164 | 16 | 74 -vgg_16 | 321 | vgg_16/pool5 | 62.729 | 212 | 32 | 90 -inception_v2 | None | Conv2d_1a_7x7 | None | 7 | 2 | None -inception_v2 | None | MaxPool_2a_3x3 | None | 11 | 4 | None -inception_v2 | None | Conv2d_2b_1x1 | None | 11 | 4 | None -inception_v2 | None | Conv2d_2c_3x3 | None | 19 | 4 | None -inception_v2 | None | MaxPool_3a_3x3 | None | 27 | 8 | None -inception_v2 | None | Mixed_3b | None | 59 | 8 | None -inception_v2 | None | Mixed_3c | None | 91 | 8 | None -inception_v2 | None | Mixed_4a | None | 123 | 16 | None -inception_v2 | None | Mixed_4b | None | 187 | 16 | None -inception_v2 | None | Mixed_4c | None | 251 | 16 | None -inception_v2 | None | Mixed_4d | None | 315 | 16 | None -inception_v2 | None | Mixed_4e | None | 379 | 16 | None -inception_v2 | None | Mixed_5a | None | 443 | 32 | None -inception_v2 | None | Mixed_5b | None | 571 | 32 | None -inception_v2 | None | Mixed_5c | None | 699 | 32 | None -inception_v2 | 224 | Conv2d_1a_7x7 | 0.069 | 7 | 2 | 2 -inception_v2 | 224 | MaxPool_2a_3x3 | 0.071 | 11 | 4 | 2 -inception_v2 | 224 | Conv2d_2b_1x1 | 0.097 | 11 | 4 | 2 -inception_v2 | 224 | Conv2d_2c_3x3 | 0.791 | 19 | 4 | 6 -inception_v2 | 224 | MaxPool_3a_3x3 | 0.792 | 27 | 8 | 6 -inception_v2 | 224 | Mixed_3b | 1.136 | 59 | 8 | 22 -inception_v2 | 224 | Mixed_3c | 1.544 | 91 | 8 | 38 -inception_v2 | 224 | Mixed_4a | 1.833 | 123 | 16 | 46 -inception_v2 | 224 | Mixed_4b | 2.073 | 187 | 16 | 78 -inception_v2 | 224 | Mixed_4c | 2.334 | 251 | 16 | 110 -inception_v2 | 224 | Mixed_4d | 2.686 | 315 | 16 | 142 -inception_v2 | 224 | Mixed_4e | 3.120 | 379 | 16 | 174 -inception_v2 | 224 | Mixed_5a | 3.446 | 443 | 32 | 190 -inception_v2 | 224 | Mixed_5b | 3.660 | 571 | 32 | 254 -inception_v2 | 224 | Mixed_5c | 3.883 | 699 | 32 | 318 -inception_v2 | 321 | Conv2d_1a_7x7 | 0.142 | 7 | 2 | 3 -inception_v2 | 321 | MaxPool_2a_3x3 | 0.146 | 11 | 4 | 5 -inception_v2 | 321 | Conv2d_2b_1x1 | 0.200 | 11 | 4 | 5 -inception_v2 | 321 | Conv2d_2c_3x3 | 1.653 | 19 | 4 | 9 -inception_v2 | 321 | MaxPool_3a_3x3 | 1.656 | 27 | 8 | 13 -inception_v2 | 321 | Mixed_3b | 2.393 | 59 | 8 | 29 -inception_v2 | 321 | Mixed_3c | 3.268 | 91 | 8 | 45 -inception_v2 | 321 | Mixed_4a | 3.898 | 123 | 16 | 61 -inception_v2 | 321 | Mixed_4b | 4.438 | 187 | 16 | 93 -inception_v2 | 321 | Mixed_4c | 5.025 | 251 | 16 | 125 -inception_v2 | 321 | Mixed_4d | 5.817 | 315 | 16 | 157 -inception_v2 | 321 | Mixed_4e | 6.795 | 379 | 16 | 189 -inception_v2 | 321 | Mixed_5a | 7.545 | 443 | 32 | 221 -inception_v2 | 321 | Mixed_5b | 8.073 | 571 | 32 | 285 -inception_v2 | 321 | Mixed_5c | 8.626 | 699 | 32 | 349 -inception_v2-no-separable-conv | None | Conv2d_1a_7x7 | None | 7 | 2 | None -inception_v2-no-separable-conv | None | MaxPool_2a_3x3 | None | 11 | 4 | None -inception_v2-no-separable-conv | None | Conv2d_2b_1x1 | None | 11 | 4 | None -inception_v2-no-separable-conv | None | Conv2d_2c_3x3 | None | 19 | 4 | None -inception_v2-no-separable-conv | None | MaxPool_3a_3x3 | None | 27 | 8 | None -inception_v2-no-separable-conv | None | Mixed_3b | None | 59 | 8 | None -inception_v2-no-separable-conv | None | Mixed_3c | None | 91 | 8 | None -inception_v2-no-separable-conv | None | Mixed_4a | None | 123 | 16 | None -inception_v2-no-separable-conv | None | Mixed_4b | None | 187 | 16 | None -inception_v2-no-separable-conv | None | Mixed_4c | None | 251 | 16 | None -inception_v2-no-separable-conv | None | Mixed_4d | None | 315 | 16 | None -inception_v2-no-separable-conv | None | Mixed_4e | None | 379 | 16 | None -inception_v2-no-separable-conv | None | Mixed_5a | None | 443 | 32 | None -inception_v2-no-separable-conv | None | Mixed_5b | None | 571 | 32 | None -inception_v2-no-separable-conv | None | Mixed_5c | None | 699 | 32 | None -inception_v2-no-separable-conv | 224 | Conv2d_1a_7x7 | 0.237 | 7 | 2 | 2 -inception_v2-no-separable-conv | 224 | MaxPool_2a_3x3 | 0.239 | 11 | 4 | 2 -inception_v2-no-separable-conv | 224 | Conv2d_2b_1x1 | 0.265 | 11 | 4 | 2 -inception_v2-no-separable-conv | 224 | Conv2d_2c_3x3 | 0.959 | 19 | 4 | 6 -inception_v2-no-separable-conv | 224 | MaxPool_3a_3x3 | 0.960 | 27 | 8 | 6 -inception_v2-no-separable-conv | 224 | Mixed_3b | 1.304 | 59 | 8 | 22 -inception_v2-no-separable-conv | 224 | Mixed_3c | 1.712 | 91 | 8 | 38 -inception_v2-no-separable-conv | 224 | Mixed_4a | 2.001 | 123 | 16 | 46 -inception_v2-no-separable-conv | 224 | Mixed_4b | 2.241 | 187 | 16 | 78 -inception_v2-no-separable-conv | 224 | Mixed_4c | 2.502 | 251 | 16 | 110 -inception_v2-no-separable-conv | 224 | Mixed_4d | 2.854 | 315 | 16 | 142 -inception_v2-no-separable-conv | 224 | Mixed_4e | 3.288 | 379 | 16 | 174 -inception_v2-no-separable-conv | 224 | Mixed_5a | 3.614 | 443 | 32 | 190 -inception_v2-no-separable-conv | 224 | Mixed_5b | 3.828 | 571 | 32 | 254 -inception_v2-no-separable-conv | 224 | Mixed_5c | 4.051 | 699 | 32 | 318 -inception_v2-no-separable-conv | 321 | Conv2d_1a_7x7 | 0.489 | 7 | 2 | 3 -inception_v2-no-separable-conv | 321 | MaxPool_2a_3x3 | 0.493 | 11 | 4 | 5 -inception_v2-no-separable-conv | 321 | Conv2d_2b_1x1 | 0.547 | 11 | 4 | 5 -inception_v2-no-separable-conv | 321 | Conv2d_2c_3x3 | 2.000 | 19 | 4 | 9 -inception_v2-no-separable-conv | 321 | MaxPool_3a_3x3 | 2.003 | 27 | 8 | 13 -inception_v2-no-separable-conv | 321 | Mixed_3b | 2.740 | 59 | 8 | 29 -inception_v2-no-separable-conv | 321 | Mixed_3c | 3.615 | 91 | 8 | 45 -inception_v2-no-separable-conv | 321 | Mixed_4a | 4.246 | 123 | 16 | 61 -inception_v2-no-separable-conv | 321 | Mixed_4b | 4.785 | 187 | 16 | 93 -inception_v2-no-separable-conv | 321 | Mixed_4c | 5.373 | 251 | 16 | 125 -inception_v2-no-separable-conv | 321 | Mixed_4d | 6.164 | 315 | 16 | 157 -inception_v2-no-separable-conv | 321 | Mixed_4e | 7.142 | 379 | 16 | 189 -inception_v2-no-separable-conv | 321 | Mixed_5a | 7.892 | 443 | 32 | 221 -inception_v2-no-separable-conv | 321 | Mixed_5b | 8.421 | 571 | 32 | 285 -inception_v2-no-separable-conv | 321 | Mixed_5c | 8.973 | 699 | 32 | 349 -inception_v3 | None | Conv2d_1a_3x3 | None | 3 | 2 | 0 -inception_v3 | None | Conv2d_2a_3x3 | None | 7 | 2 | 0 -inception_v3 | None | Conv2d_2b_3x3 | None | 11 | 2 | 2 -inception_v3 | None | MaxPool_3a_3x3 | None | 15 | 4 | 2 -inception_v3 | None | Conv2d_3b_1x1 | None | 15 | 4 | 2 -inception_v3 | None | Conv2d_4a_3x3 | None | 23 | 4 | 2 -inception_v3 | None | MaxPool_5a_3x3 | None | 31 | 8 | 2 -inception_v3 | None | Mixed_5b | None | 63 | 8 | 18 -inception_v3 | None | Mixed_5c | None | 95 | 8 | 34 -inception_v3 | None | Mixed_5d | None | 127 | 8 | 50 -inception_v3 | None | Mixed_6a | None | 159 | 16 | 58 -inception_v3 | None | Mixed_6b | None | 351 | 16 | 154 -inception_v3 | None | Mixed_6c | None | 543 | 16 | 250 -inception_v3 | None | Mixed_6d | None | 735 | 16 | 346 -inception_v3 | None | Mixed_6e | None | 927 | 16 | 442 -inception_v3 | None | Mixed_7a | None | 1055 | 32 | 490 -inception_v3 | None | Mixed_7b | None | 1183 | 32 | 554 -inception_v3 | None | Mixed_7c | None | 1311 | 32 | 618 -inception_v3 | 224 | Conv2d_1a_3x3 | 0.022 | 3 | 2 | 0 -inception_v3 | 224 | Conv2d_2a_3x3 | 0.241 | 7 | 2 | 0 -inception_v3 | 224 | Conv2d_2b_3x3 | 0.680 | 11 | 2 | 2 -inception_v3 | 224 | MaxPool_3a_3x3 | 0.681 | 15 | 4 | 2 -inception_v3 | 224 | Conv2d_3b_1x1 | 0.712 | 15 | 4 | 2 -inception_v3 | 224 | Conv2d_4a_3x3 | 1.460 | 23 | 4 | 2 -inception_v3 | 224 | MaxPool_5a_3x3 | 1.461 | 31 | 8 | 2 -inception_v3 | 224 | Mixed_5b | 1.781 | 63 | 8 | 18 -inception_v3 | 224 | Mixed_5c | 2.128 | 95 | 8 | 34 -inception_v3 | 224 | Mixed_5d | 2.485 | 127 | 8 | 50 -inception_v3 | 224 | Mixed_6a | 2.889 | 159 | 16 | 58 -inception_v3 | 224 | Mixed_6b | 3.263 | 351 | 16 | 154 -inception_v3 | 224 | Mixed_6c | 3.750 | 543 | 16 | 250 -inception_v3 | 224 | Mixed_6d | 4.237 | 735 | 16 | 346 -inception_v3 | 224 | Mixed_6e | 4.854 | 927 | 16 | 442 -inception_v3 | 224 | Mixed_7a | 5.132 | 1055 | 32 | 490 -inception_v3 | 224 | Mixed_7b | 5.385 | 1183 | 32 | 554 -inception_v3 | 224 | Mixed_7c | 5.689 | 1311 | 32 | 618 -inception_v3 | 321 | Conv2d_1a_3x3 | 0.045 | 3 | 2 | 0 -inception_v3 | 321 | Conv2d_2a_3x3 | 0.506 | 7 | 2 | 0 -inception_v3 | 321 | Conv2d_2b_3x3 | 1.428 | 11 | 2 | 2 -inception_v3 | 321 | MaxPool_3a_3x3 | 1.431 | 15 | 4 | 2 -inception_v3 | 321 | Conv2d_3b_1x1 | 1.494 | 15 | 4 | 2 -inception_v3 | 321 | Conv2d_4a_3x3 | 3.092 | 23 | 4 | 2 -inception_v3 | 321 | MaxPool_5a_3x3 | 3.095 | 31 | 8 | 2 -inception_v3 | 321 | Mixed_5b | 3.796 | 63 | 8 | 18 -inception_v3 | 321 | Mixed_5c | 4.557 | 95 | 8 | 34 -inception_v3 | 321 | Mixed_5d | 5.339 | 127 | 8 | 50 -inception_v3 | 321 | Mixed_6a | 6.241 | 159 | 16 | 58 -inception_v3 | 321 | Mixed_6b | 7.082 | 351 | 16 | 154 -inception_v3 | 321 | Mixed_6c | 8.178 | 543 | 16 | 250 -inception_v3 | 321 | Mixed_6d | 9.275 | 735 | 16 | 346 -inception_v3 | 321 | Mixed_6e | 10.663 | 927 | 16 | 442 -inception_v3 | 321 | Mixed_7a | 11.303 | 1055 | 32 | 490 -inception_v3 | 321 | Mixed_7b | 11.948 | 1183 | 32 | 554 -inception_v3 | 321 | Mixed_7c | 12.727 | 1311 | 32 | 618 -inception_v4 | None | Conv2d_1a_3x3 | None | 3 | 2 | 0 -inception_v4 | None | Conv2d_2a_3x3 | None | 7 | 2 | 0 -inception_v4 | None | Conv2d_2b_3x3 | None | 11 | 2 | 2 -inception_v4 | None | Mixed_3a | None | 15 | 4 | 2 -inception_v4 | None | Mixed_4a | None | 47 | 4 | 14 -inception_v4 | None | Mixed_5a | None | 55 | 8 | 14 -inception_v4 | None | Mixed_5b | None | 87 | 8 | 30 -inception_v4 | None | Mixed_5c | None | 119 | 8 | 46 -inception_v4 | None | Mixed_5d | None | 151 | 8 | 62 -inception_v4 | None | Mixed_5e | None | 183 | 8 | 78 -inception_v4 | None | Mixed_6a | None | 215 | 16 | 86 -inception_v4 | None | Mixed_6b | None | 407 | 16 | 182 -inception_v4 | None | Mixed_6c | None | 599 | 16 | 278 -inception_v4 | None | Mixed_6d | None | 791 | 16 | 374 -inception_v4 | None | Mixed_6e | None | 983 | 16 | 470 -inception_v4 | None | Mixed_6f | None | 1175 | 16 | 566 -inception_v4 | None | Mixed_6g | None | 1367 | 16 | 662 -inception_v4 | None | Mixed_6h | None | 1559 | 16 | 758 -inception_v4 | None | Mixed_7a | None | 1687 | 32 | 806 -inception_v4 | None | Mixed_7b | None | 1815 | 32 | 870 -inception_v4 | None | Mixed_7c | None | 1943 | 32 | 934 -inception_v4 | None | Mixed_7d | None | 2071 | 32 | 998 -inception_v4 | 224 | Conv2d_1a_3x3 | 0.022 | 3 | 2 | 0 -inception_v4 | 224 | Conv2d_2a_3x3 | 0.241 | 7 | 2 | 0 -inception_v4 | 224 | Conv2d_2b_3x3 | 0.680 | 11 | 2 | 2 -inception_v4 | 224 | Mixed_3a | 1.004 | 15 | 4 | 2 -inception_v4 | 224 | Mixed_4a | 2.057 | 47 | 4 | 14 -inception_v4 | 224 | Mixed_5a | 2.473 | 55 | 8 | 14 -inception_v4 | 224 | Mixed_5b | 2.871 | 87 | 8 | 30 -inception_v4 | 224 | Mixed_5c | 3.269 | 119 | 8 | 46 -inception_v4 | 224 | Mixed_5d | 3.668 | 151 | 8 | 62 -inception_v4 | 224 | Mixed_5e | 4.066 | 183 | 8 | 78 -inception_v4 | 224 | Mixed_6a | 5.173 | 215 | 16 | 86 -inception_v4 | 224 | Mixed_6b | 6.019 | 407 | 16 | 182 -inception_v4 | 224 | Mixed_6c | 6.865 | 599 | 16 | 278 -inception_v4 | 224 | Mixed_6d | 7.711 | 791 | 16 | 374 -inception_v4 | 224 | Mixed_6e | 8.557 | 983 | 16 | 470 -inception_v4 | 224 | Mixed_6f | 9.403 | 1175 | 16 | 566 -inception_v4 | 224 | Mixed_6g | 10.249 | 1367 | 16 | 662 -inception_v4 | 224 | Mixed_6h | 11.095 | 1559 | 16 | 758 -inception_v4 | 224 | Mixed_7a | 11.588 | 1687 | 32 | 806 -inception_v4 | 224 | Mixed_7b | 11.815 | 1815 | 32 | 870 -inception_v4 | 224 | Mixed_7c | 12.043 | 1943 | 32 | 934 -inception_v4 | 224 | Mixed_7d | 12.271 | 2071 | 32 | 998 -inception_v4 | 321 | Conv2d_1a_3x3 | 0.045 | 3 | 2 | 0 -inception_v4 | 321 | Conv2d_2a_3x3 | 0.506 | 7 | 2 | 0 -inception_v4 | 321 | Conv2d_2b_3x3 | 1.428 | 11 | 2 | 2 -inception_v4 | 321 | Mixed_3a | 2.105 | 15 | 4 | 2 -inception_v4 | 321 | Mixed_4a | 4.332 | 47 | 4 | 14 -inception_v4 | 321 | Mixed_5a | 5.243 | 55 | 8 | 14 -inception_v4 | 321 | Mixed_5b | 6.115 | 87 | 8 | 30 -inception_v4 | 321 | Mixed_5c | 6.987 | 119 | 8 | 46 -inception_v4 | 321 | Mixed_5d | 7.859 | 151 | 8 | 62 -inception_v4 | 321 | Mixed_5e | 8.731 | 183 | 8 | 78 -inception_v4 | 321 | Mixed_6a | 11.189 | 215 | 16 | 86 -inception_v4 | 321 | Mixed_6b | 13.092 | 407 | 16 | 182 -inception_v4 | 321 | Mixed_6c | 14.996 | 599 | 16 | 278 -inception_v4 | 321 | Mixed_6d | 16.899 | 791 | 16 | 374 -inception_v4 | 321 | Mixed_6e | 18.802 | 983 | 16 | 470 -inception_v4 | 321 | Mixed_6f | 20.706 | 1175 | 16 | 566 -inception_v4 | 321 | Mixed_6g | 22.609 | 1367 | 16 | 662 -inception_v4 | 321 | Mixed_6h | 24.513 | 1559 | 16 | 758 -inception_v4 | 321 | Mixed_7a | 25.640 | 1687 | 32 | 806 -inception_v4 | 321 | Mixed_7b | 26.223 | 1815 | 32 | 870 -inception_v4 | 321 | Mixed_7c | 26.807 | 1943 | 32 | 934 -inception_v4 | 321 | Mixed_7d | 27.390 | 2071 | 32 | 998 -inception_resnet_v2 | None | Conv2d_1a_3x3 | None | 3 | 2 | 0 -inception_resnet_v2 | None | Conv2d_2a_3x3 | None | 7 | 2 | 0 -inception_resnet_v2 | None | Conv2d_2b_3x3 | None | 11 | 2 | 2 -inception_resnet_v2 | None | MaxPool_3a_3x3 | None | 15 | 4 | 2 -inception_resnet_v2 | None | Conv2d_3b_1x1 | None | 15 | 4 | 2 -inception_resnet_v2 | None | Conv2d_4a_3x3 | None | 23 | 4 | 2 -inception_resnet_v2 | None | MaxPool_5a_3x3 | None | 31 | 8 | 2 -inception_resnet_v2 | None | Mixed_5b | None | 63 | 8 | 18 -inception_resnet_v2 | None | Mixed_6a | None | 415 | 16 | 186 -inception_resnet_v2 | None | PreAuxLogits | None | 2335 | 16 | 1146 -inception_resnet_v2 | None | Mixed_7a | None | 2399 | 32 | 1162 -inception_resnet_v2 | None | Conv2d_7b_1x1 | None | 3039 | 32 | 1482 -inception_resnet_v2 | 224 | Conv2d_1a_3x3 | 0.022 | 3 | 2 | 0 -inception_resnet_v2 | 224 | Conv2d_2a_3x3 | 0.241 | 7 | 2 | 0 -inception_resnet_v2 | 224 | Conv2d_2b_3x3 | 0.680 | 11 | 2 | 2 -inception_resnet_v2 | 224 | MaxPool_3a_3x3 | 0.681 | 15 | 4 | 2 -inception_resnet_v2 | 224 | Conv2d_3b_1x1 | 0.712 | 15 | 4 | 2 -inception_resnet_v2 | 224 | Conv2d_4a_3x3 | 1.460 | 23 | 4 | 2 -inception_resnet_v2 | 224 | MaxPool_5a_3x3 | 1.461 | 31 | 8 | 2 -inception_resnet_v2 | 224 | Mixed_5b | 1.796 | 63 | 8 | 18 -inception_resnet_v2 | 224 | Mixed_6a | 4.747 | 415 | 16 | 186 -inception_resnet_v2 | 224 | PreAuxLogits | 11.235 | 2335 | 16 | 1146 -inception_resnet_v2 | 224 | Mixed_7a | 11.786 | 2399 | 32 | 1162 -inception_resnet_v2 | 224 | Conv2d_7b_1x1 | 12.963 | 3039 | 32 | 1482 -inception_resnet_v2 | 321 | Conv2d_1a_3x3 | 0.045 | 3 | 2 | 0 -inception_resnet_v2 | 321 | Conv2d_2a_3x3 | 0.506 | 7 | 2 | 0 -inception_resnet_v2 | 321 | Conv2d_2b_3x3 | 1.428 | 11 | 2 | 2 -inception_resnet_v2 | 321 | MaxPool_3a_3x3 | 1.431 | 15 | 4 | 2 -inception_resnet_v2 | 321 | Conv2d_3b_1x1 | 1.494 | 15 | 4 | 2 -inception_resnet_v2 | 321 | Conv2d_4a_3x3 | 3.092 | 23 | 4 | 2 -inception_resnet_v2 | 321 | MaxPool_5a_3x3 | 3.095 | 31 | 8 | 2 -inception_resnet_v2 | 321 | Mixed_5b | 3.829 | 63 | 8 | 18 -inception_resnet_v2 | 321 | Mixed_6a | 10.327 | 415 | 16 | 186 -inception_resnet_v2 | 321 | PreAuxLogits | 24.924 | 2335 | 16 | 1146 -inception_resnet_v2 | 321 | Mixed_7a | 26.201 | 2399 | 32 | 1162 -inception_resnet_v2 | 321 | Conv2d_7b_1x1 | 29.215 | 3039 | 32 | 1482 -inception_resnet_v2-same | None | Conv2d_1a_3x3 | None | 3 | 2 | None -inception_resnet_v2-same | None | Conv2d_2a_3x3 | None | 7 | 2 | None -inception_resnet_v2-same | None | Conv2d_2b_3x3 | None | 11 | 2 | None -inception_resnet_v2-same | None | MaxPool_3a_3x3 | None | 15 | 4 | None -inception_resnet_v2-same | None | Conv2d_3b_1x1 | None | 15 | 4 | None -inception_resnet_v2-same | None | Conv2d_4a_3x3 | None | 23 | 4 | None -inception_resnet_v2-same | None | MaxPool_5a_3x3 | None | 31 | 8 | None -inception_resnet_v2-same | None | Mixed_5b | None | 63 | 8 | None -inception_resnet_v2-same | None | Mixed_6a | None | 415 | 16 | None -inception_resnet_v2-same | None | PreAuxLogits | None | 2335 | 16 | None -inception_resnet_v2-same | None | Mixed_7a | None | 2399 | 32 | None -inception_resnet_v2-same | None | Conv2d_7b_1x1 | None | 3039 | 32 | None -inception_resnet_v2-same | 224 | Conv2d_1a_3x3 | 0.022 | 3 | 2 | 0 -inception_resnet_v2-same | 224 | Conv2d_2a_3x3 | 0.254 | 7 | 2 | 2 -inception_resnet_v2-same | 224 | Conv2d_2b_3x3 | 0.717 | 11 | 2 | 4 -inception_resnet_v2-same | 224 | MaxPool_3a_3x3 | 0.719 | 15 | 4 | 4 -inception_resnet_v2-same | 224 | Conv2d_3b_1x1 | 0.751 | 15 | 4 | 4 -inception_resnet_v2-same | 224 | Conv2d_4a_3x3 | 1.619 | 23 | 4 | 8 -inception_resnet_v2-same | 224 | MaxPool_5a_3x3 | 1.620 | 31 | 8 | 8 -inception_resnet_v2-same | 224 | Mixed_5b | 2.041 | 63 | 8 | 24 -inception_resnet_v2-same | 224 | Mixed_6a | 5.804 | 415 | 16 | 192 -inception_resnet_v2-same | 224 | PreAuxLogits | 14.634 | 2335 | 16 | 1152 -inception_resnet_v2-same | 224 | Mixed_7a | 15.456 | 2399 | 32 | 1168 -inception_resnet_v2-same | 224 | Conv2d_7b_1x1 | 17.763 | 3039 | 32 | 1488 -inception_resnet_v2-same | 321 | Conv2d_1a_3x3 | 0.046 | 3 | 2 | 1 -inception_resnet_v2-same | 321 | Conv2d_2a_3x3 | 0.524 | 7 | 2 | 3 -inception_resnet_v2-same | 321 | Conv2d_2b_3x3 | 1.481 | 11 | 2 | 5 -inception_resnet_v2-same | 321 | MaxPool_3a_3x3 | 1.485 | 15 | 4 | 7 -inception_resnet_v2-same | 321 | Conv2d_3b_1x1 | 1.553 | 15 | 4 | 7 -inception_resnet_v2-same | 321 | Conv2d_4a_3x3 | 3.368 | 23 | 4 | 11 -inception_resnet_v2-same | 321 | MaxPool_5a_3x3 | 3.371 | 31 | 8 | 15 -inception_resnet_v2-same | 321 | Mixed_5b | 4.273 | 63 | 8 | 31 -inception_resnet_v2-same | 321 | Mixed_6a | 12.424 | 415 | 16 | 207 -inception_resnet_v2-same | 321 | PreAuxLogits | 32.293 | 2335 | 16 | 1167 -inception_resnet_v2-same | 321 | Mixed_7a | 34.192 | 2399 | 32 | 1199 -inception_resnet_v2-same | 321 | Conv2d_7b_1x1 | 39.890 | 3039 | 32 | 1519 -mobilenet_v1 | None | Conv2d_0 | None | 3 | 2 | None -mobilenet_v1 | None | Conv2d_1_pointwise | None | 7 | 2 | None -mobilenet_v1 | None | Conv2d_2_pointwise | None | 11 | 4 | None -mobilenet_v1 | None | Conv2d_3_pointwise | None | 19 | 4 | None -mobilenet_v1 | None | Conv2d_4_pointwise | None | 27 | 8 | None -mobilenet_v1 | None | Conv2d_5_pointwise | None | 43 | 8 | None -mobilenet_v1 | None | Conv2d_6_pointwise | None | 59 | 16 | None -mobilenet_v1 | None | Conv2d_7_pointwise | None | 91 | 16 | None -mobilenet_v1 | None | Conv2d_8_pointwise | None | 123 | 16 | None -mobilenet_v1 | None | Conv2d_9_pointwise | None | 155 | 16 | None -mobilenet_v1 | None | Conv2d_10_pointwise | None | 187 | 16 | None -mobilenet_v1 | None | Conv2d_11_pointwise | None | 219 | 16 | None -mobilenet_v1 | None | Conv2d_12_pointwise | None | 251 | 32 | None -mobilenet_v1 | None | Conv2d_13_pointwise | None | 315 | 32 | None -mobilenet_v1 | 224 | Conv2d_0 | 0.022 | 3 | 2 | 0 -mobilenet_v1 | 224 | Conv2d_1_pointwise | 0.082 | 7 | 2 | 2 -mobilenet_v1 | 224 | Conv2d_2_pointwise | 0.137 | 11 | 4 | 2 -mobilenet_v1 | 224 | Conv2d_3_pointwise | 0.248 | 19 | 4 | 6 -mobilenet_v1 | 224 | Conv2d_4_pointwise | 0.302 | 27 | 8 | 6 -mobilenet_v1 | 224 | Conv2d_5_pointwise | 0.409 | 43 | 8 | 14 -mobilenet_v1 | 224 | Conv2d_6_pointwise | 0.461 | 59 | 16 | 14 -mobilenet_v1 | 224 | Conv2d_7_pointwise | 0.566 | 91 | 16 | 30 -mobilenet_v1 | 224 | Conv2d_8_pointwise | 0.671 | 123 | 16 | 46 -mobilenet_v1 | 224 | Conv2d_9_pointwise | 0.775 | 155 | 16 | 62 -mobilenet_v1 | 224 | Conv2d_10_pointwise | 0.880 | 187 | 16 | 78 -mobilenet_v1 | 224 | Conv2d_11_pointwise | 0.985 | 219 | 16 | 94 -mobilenet_v1 | 224 | Conv2d_12_pointwise | 1.037 | 251 | 32 | 94 -mobilenet_v1 | 224 | Conv2d_13_pointwise | 1.140 | 315 | 32 | 126 -mobilenet_v1 | 321 | Conv2d_0 | 0.046 | 3 | 2 | 1 -mobilenet_v1 | 321 | Conv2d_1_pointwise | 0.169 | 7 | 2 | 3 -mobilenet_v1 | 321 | Conv2d_2_pointwise | 0.286 | 11 | 4 | 5 -mobilenet_v1 | 321 | Conv2d_3_pointwise | 0.517 | 19 | 4 | 9 -mobilenet_v1 | 321 | Conv2d_4_pointwise | 0.632 | 27 | 8 | 13 -mobilenet_v1 | 321 | Conv2d_5_pointwise | 0.861 | 43 | 8 | 21 -mobilenet_v1 | 321 | Conv2d_6_pointwise | 0.979 | 59 | 16 | 29 -mobilenet_v1 | 321 | Conv2d_7_pointwise | 1.215 | 91 | 16 | 45 -mobilenet_v1 | 321 | Conv2d_8_pointwise | 1.450 | 123 | 16 | 61 -mobilenet_v1 | 321 | Conv2d_9_pointwise | 1.686 | 155 | 16 | 77 -mobilenet_v1 | 321 | Conv2d_10_pointwise | 1.922 | 187 | 16 | 93 -mobilenet_v1 | 321 | Conv2d_11_pointwise | 2.158 | 219 | 16 | 109 -mobilenet_v1 | 321 | Conv2d_12_pointwise | 2.286 | 251 | 32 | 125 -mobilenet_v1 | 321 | Conv2d_13_pointwise | 2.542 | 315 | 32 | 157 -mobilenet_v1_075 | None | Conv2d_0 | None | 3 | 2 | None -mobilenet_v1_075 | None | Conv2d_1_pointwise | None | 7 | 2 | None -mobilenet_v1_075 | None | Conv2d_2_pointwise | None | 11 | 4 | None -mobilenet_v1_075 | None | Conv2d_3_pointwise | None | 19 | 4 | None -mobilenet_v1_075 | None | Conv2d_4_pointwise | None | 27 | 8 | None -mobilenet_v1_075 | None | Conv2d_5_pointwise | None | 43 | 8 | None -mobilenet_v1_075 | None | Conv2d_6_pointwise | None | 59 | 16 | None -mobilenet_v1_075 | None | Conv2d_7_pointwise | None | 91 | 16 | None -mobilenet_v1_075 | None | Conv2d_8_pointwise | None | 123 | 16 | None -mobilenet_v1_075 | None | Conv2d_9_pointwise | None | 155 | 16 | None -mobilenet_v1_075 | None | Conv2d_10_pointwise | None | 187 | 16 | None -mobilenet_v1_075 | None | Conv2d_11_pointwise | None | 219 | 16 | None -mobilenet_v1_075 | None | Conv2d_12_pointwise | None | 251 | 32 | None -mobilenet_v1_075 | None | Conv2d_13_pointwise | None | 315 | 32 | None -mobilenet_v1_075 | 224 | Conv2d_0 | 0.017 | 3 | 2 | 0 -mobilenet_v1_075 | 224 | Conv2d_1_pointwise | 0.052 | 7 | 2 | 2 -mobilenet_v1_075 | 224 | Conv2d_2_pointwise | 0.084 | 11 | 4 | 2 -mobilenet_v1_075 | 224 | Conv2d_3_pointwise | 0.148 | 19 | 4 | 6 -mobilenet_v1_075 | 224 | Conv2d_4_pointwise | 0.178 | 27 | 8 | 6 -mobilenet_v1_075 | 224 | Conv2d_5_pointwise | 0.239 | 43 | 8 | 14 -mobilenet_v1_075 | 224 | Conv2d_6_pointwise | 0.269 | 59 | 16 | 14 -mobilenet_v1_075 | 224 | Conv2d_7_pointwise | 0.328 | 91 | 16 | 30 -mobilenet_v1_075 | 224 | Conv2d_8_pointwise | 0.387 | 123 | 16 | 46 -mobilenet_v1_075 | 224 | Conv2d_9_pointwise | 0.447 | 155 | 16 | 62 -mobilenet_v1_075 | 224 | Conv2d_10_pointwise | 0.506 | 187 | 16 | 78 -mobilenet_v1_075 | 224 | Conv2d_11_pointwise | 0.565 | 219 | 16 | 94 -mobilenet_v1_075 | 224 | Conv2d_12_pointwise | 0.594 | 251 | 32 | 94 -mobilenet_v1_075 | 224 | Conv2d_13_pointwise | 0.653 | 315 | 32 | 126 -mobilenet_v1_075 | 321 | Conv2d_0 | 0.034 | 3 | 2 | 1 -mobilenet_v1_075 | 321 | Conv2d_1_pointwise | 0.107 | 7 | 2 | 3 -mobilenet_v1_075 | 321 | Conv2d_2_pointwise | 0.174 | 11 | 4 | 5 -mobilenet_v1_075 | 321 | Conv2d_3_pointwise | 0.308 | 19 | 4 | 9 -mobilenet_v1_075 | 321 | Conv2d_4_pointwise | 0.373 | 27 | 8 | 13 -mobilenet_v1_075 | 321 | Conv2d_5_pointwise | 0.503 | 43 | 8 | 21 -mobilenet_v1_075 | 321 | Conv2d_6_pointwise | 0.570 | 59 | 16 | 29 -mobilenet_v1_075 | 321 | Conv2d_7_pointwise | 0.704 | 91 | 16 | 45 -mobilenet_v1_075 | 321 | Conv2d_8_pointwise | 0.837 | 123 | 16 | 61 -mobilenet_v1_075 | 321 | Conv2d_9_pointwise | 0.970 | 155 | 16 | 77 -mobilenet_v1_075 | 321 | Conv2d_10_pointwise | 1.104 | 187 | 16 | 93 -mobilenet_v1_075 | 321 | Conv2d_11_pointwise | 1.237 | 219 | 16 | 109 -mobilenet_v1_075 | 321 | Conv2d_12_pointwise | 1.310 | 251 | 32 | 125 -mobilenet_v1_075 | 321 | Conv2d_13_pointwise | 1.454 | 315 | 32 | 157 -resnet_v1_50 | None | resnet_v1_50/block1 | None | 35 | 8 | None -resnet_v1_50 | None | resnet_v1_50/block2 | None | 99 | 16 | None -resnet_v1_50 | None | resnet_v1_50/block3 | None | 291 | 32 | None -resnet_v1_50 | None | resnet_v1_50/block4 | None | 483 | 32 | None -resnet_v1_50 | 224 | resnet_v1_50/block1 | 1.325 | 35 | 8 | 15 -resnet_v1_50 | 224 | resnet_v1_50/block2 | 2.977 | 99 | 16 | 47 -resnet_v1_50 | 224 | resnet_v1_50/block3 | 5.502 | 291 | 32 | 143 -resnet_v1_50 | 224 | resnet_v1_50/block4 | 6.967 | 483 | 32 | 239 -resnet_v1_50 | 321 | resnet_v1_50/block1 | 2.771 | 35 | 8 | 17 -resnet_v1_50 | 321 | resnet_v1_50/block2 | 6.322 | 99 | 16 | 49 -resnet_v1_50 | 321 | resnet_v1_50/block3 | 12.022 | 291 | 32 | 145 -resnet_v1_50 | 321 | resnet_v1_50/block4 | 15.639 | 483 | 32 | 241 -resnet_v1_101 | None | resnet_v1_101/block1 | None | 35 | 8 | None -resnet_v1_101 | None | resnet_v1_101/block2 | None | 99 | 16 | None -resnet_v1_101 | None | resnet_v1_101/block3 | None | 835 | 32 | None -resnet_v1_101 | None | resnet_v1_101/block4 | None | 1027 | 32 | None -resnet_v1_101 | 224 | resnet_v1_101/block1 | 1.325 | 35 | 8 | 15 -resnet_v1_101 | 224 | resnet_v1_101/block2 | 2.977 | 99 | 16 | 47 -resnet_v1_101 | 224 | resnet_v1_101/block3 | 12.930 | 835 | 32 | 415 -resnet_v1_101 | 224 | resnet_v1_101/block4 | 14.395 | 1027 | 32 | 511 -resnet_v1_101 | 321 | resnet_v1_101/block1 | 2.771 | 35 | 8 | 17 -resnet_v1_101 | 321 | resnet_v1_101/block2 | 6.322 | 99 | 16 | 49 -resnet_v1_101 | 321 | resnet_v1_101/block3 | 28.734 | 835 | 32 | 417 -resnet_v1_101 | 321 | resnet_v1_101/block4 | 32.351 | 1027 | 32 | 513 -resnet_v1_152 | None | resnet_v1_152/block1 | None | 35 | 8 | None -resnet_v1_152 | None | resnet_v1_152/block2 | None | 163 | 16 | None -resnet_v1_152 | None | resnet_v1_152/block3 | None | 1315 | 32 | None -resnet_v1_152 | None | resnet_v1_152/block4 | None | 1507 | 32 | None -resnet_v1_152 | 224 | resnet_v1_152/block1 | 1.325 | 35 | 8 | 15 -resnet_v1_152 | 224 | resnet_v1_152/block2 | 4.726 | 163 | 16 | 79 -resnet_v1_152 | 224 | resnet_v1_152/block3 | 20.359 | 1315 | 32 | 655 -resnet_v1_152 | 224 | resnet_v1_152/block4 | 21.824 | 1507 | 32 | 751 -resnet_v1_152 | 321 | resnet_v1_152/block1 | 2.771 | 35 | 8 | 17 -resnet_v1_152 | 321 | resnet_v1_152/block2 | 10.071 | 163 | 16 | 81 -resnet_v1_152 | 321 | resnet_v1_152/block3 | 45.264 | 1315 | 32 | 657 -resnet_v1_152 | 321 | resnet_v1_152/block4 | 48.881 | 1507 | 32 | 753 -resnet_v1_200 | None | resnet_v1_200/block1 | None | 35 | 8 | None -resnet_v1_200 | None | resnet_v1_200/block2 | None | 419 | 16 | None -resnet_v1_200 | None | resnet_v1_200/block3 | None | 1571 | 32 | None -resnet_v1_200 | None | resnet_v1_200/block4 | None | 1763 | 32 | None -resnet_v1_200 | 224 | resnet_v1_200/block1 | 1.325 | 35 | 8 | 15 -resnet_v1_200 | 224 | resnet_v1_200/block2 | 11.720 | 419 | 16 | 207 -resnet_v1_200 | 224 | resnet_v1_200/block3 | 27.353 | 1571 | 32 | 783 -resnet_v1_200 | 224 | resnet_v1_200/block4 | 28.818 | 1763 | 32 | 879 -resnet_v1_200 | 321 | resnet_v1_200/block1 | 2.771 | 35 | 8 | 17 -resnet_v1_200 | 321 | resnet_v1_200/block2 | 25.067 | 419 | 16 | 209 -resnet_v1_200 | 321 | resnet_v1_200/block3 | 60.260 | 1571 | 32 | 785 -resnet_v1_200 | 321 | resnet_v1_200/block4 | 63.877 | 1763 | 32 | 881 -resnet_v2_50 | None | resnet_v2_50/block1 | None | 35 | 8 | None -resnet_v2_50 | None | resnet_v2_50/block2 | None | 99 | 16 | None -resnet_v2_50 | None | resnet_v2_50/block3 | None | 291 | 32 | None -resnet_v2_50 | None | resnet_v2_50/block4 | None | 483 | 32 | None -resnet_v2_50 | 224 | resnet_v2_50/block1 | 1.329 | 35 | 8 | 15 -resnet_v2_50 | 224 | resnet_v2_50/block2 | 2.982 | 99 | 16 | 47 -resnet_v2_50 | 224 | resnet_v2_50/block3 | 5.509 | 291 | 32 | 143 -resnet_v2_50 | 224 | resnet_v2_50/block4 | 6.974 | 483 | 32 | 239 -resnet_v2_50 | 321 | resnet_v2_50/block1 | 2.778 | 35 | 8 | 17 -resnet_v2_50 | 321 | resnet_v2_50/block2 | 6.333 | 99 | 16 | 49 -resnet_v2_50 | 321 | resnet_v2_50/block3 | 12.035 | 291 | 32 | 145 -resnet_v2_50 | 321 | resnet_v2_50/block4 | 15.653 | 483 | 32 | 241 -resnet_v2_101 | None | resnet_v2_101/block1 | None | 35 | 8 | None -resnet_v2_101 | None | resnet_v2_101/block2 | None | 99 | 16 | None -resnet_v2_101 | None | resnet_v2_101/block3 | None | 835 | 32 | None -resnet_v2_101 | None | resnet_v2_101/block4 | None | 1027 | 32 | None -resnet_v2_101 | 224 | resnet_v2_101/block1 | 1.329 | 35 | 8 | 15 -resnet_v2_101 | 224 | resnet_v2_101/block2 | 2.982 | 99 | 16 | 47 -resnet_v2_101 | 224 | resnet_v2_101/block3 | 12.940 | 835 | 32 | 415 -resnet_v2_101 | 224 | resnet_v2_101/block4 | 14.405 | 1027 | 32 | 511 -resnet_v2_101 | 321 | resnet_v2_101/block1 | 2.778 | 35 | 8 | 17 -resnet_v2_101 | 321 | resnet_v2_101/block2 | 6.333 | 99 | 16 | 49 -resnet_v2_101 | 321 | resnet_v2_101/block3 | 28.756 | 835 | 32 | 417 -resnet_v2_101 | 321 | resnet_v2_101/block4 | 32.374 | 1027 | 32 | 513 -resnet_v2_152 | None | resnet_v2_152/block1 | None | 35 | 8 | None -resnet_v2_152 | None | resnet_v2_152/block2 | None | 163 | 16 | None -resnet_v2_152 | None | resnet_v2_152/block3 | None | 1315 | 32 | None -resnet_v2_152 | None | resnet_v2_152/block4 | None | 1507 | 32 | None -resnet_v2_152 | 224 | resnet_v2_152/block1 | 1.329 | 35 | 8 | 15 -resnet_v2_152 | 224 | resnet_v2_152/block2 | 4.732 | 163 | 16 | 79 -resnet_v2_152 | 224 | resnet_v2_152/block3 | 20.373 | 1315 | 32 | 655 -resnet_v2_152 | 224 | resnet_v2_152/block4 | 21.838 | 1507 | 32 | 751 -resnet_v2_152 | 321 | resnet_v2_152/block1 | 2.778 | 35 | 8 | 17 -resnet_v2_152 | 321 | resnet_v2_152/block2 | 10.085 | 163 | 16 | 81 -resnet_v2_152 | 321 | resnet_v2_152/block3 | 45.294 | 1315 | 32 | 657 -resnet_v2_152 | 321 | resnet_v2_152/block4 | 48.912 | 1507 | 32 | 753 -resnet_v2_200 | None | resnet_v2_200/block1 | None | 35 | 8 | None -resnet_v2_200 | None | resnet_v2_200/block2 | None | 419 | 16 | None -resnet_v2_200 | None | resnet_v2_200/block3 | None | 1571 | 32 | None -resnet_v2_200 | None | resnet_v2_200/block4 | None | 1763 | 32 | None -resnet_v2_200 | 224 | resnet_v2_200/block1 | 1.329 | 35 | 8 | 15 -resnet_v2_200 | 224 | resnet_v2_200/block2 | 11.733 | 419 | 16 | 207 -resnet_v2_200 | 224 | resnet_v2_200/block3 | 27.373 | 1571 | 32 | 783 -resnet_v2_200 | 224 | resnet_v2_200/block4 | 28.839 | 1763 | 32 | 879 -resnet_v2_200 | 321 | resnet_v2_200/block1 | 2.778 | 35 | 8 | 17 -resnet_v2_200 | 321 | resnet_v2_200/block2 | 25.095 | 419 | 16 | 209 -resnet_v2_200 | 321 | resnet_v2_200/block3 | 60.305 | 1571 | 32 | 785 -resnet_v2_200 | 321 | resnet_v2_200/block4 | 63.922 | 1763 | 32 | 881 - -## FAQ - -### What does a resolution of 'None' mean? - -In this case, the input resolution is undefined. For most models, the receptive -field parameters can be computed even without knowing the input resolution. The -number of FLOPs cannot be computed in this case. - -### For some networks, effective_padding shows as 'None' (eg, for Inception_v2 or Mobilenet_v1 when input size is not specified). Why is that? - -This means that the padding for these networks depends on the input size. So, -unless we know exactly the input image dimensionality to be used, it is not -possible to determine the padding applied at the different layers. Look at the -other entries where the input size is fixed; for those cases, effective_padding -is not None. - -This happens due to Tensorflow's implementation of the 'SAME' padding mode, -which may depend on the input feature map size to a given layer. For background -on this, see -[these notes from the TF documentation](https://www.tensorflow.org/versions/master/api_guides/python/nn#Notes_on_SAME_Convolution_Padding). - -Also, note that in this case the program is not able to check if the network is -aligned (ie, it could be that the different paths from input to output have -receptive fields which are not consistently centered at the same position in the -input image). - -So you should be aware that such networks might not be aligned -- the program -has no way of checking it when the padding cannot be determined. - -### The receptive field parameters for network X seem different from what I expected... maybe your calculation is incorrect? - -First, note that the results presented here are based on the tensorflow -implementations from the -[TF-Slim model library](https://github.com/tensorflow/models/tree/master/research/slim). -So, it is possible that due to some implementation details the RF parameters are -different. - -One common case of confusion is the TF-Slim Resnet implementation, which applies -stride in the last residual unit of each block, instead of at the input -activations in the first residual unit of each block (which is what is described -in the Resnet paper) -- see -[this comment](https://github.com/tensorflow/models/blob/master/research/slim/nets/resnet_utils.py#L30). -This makes the stride with respect to each convolution block potentially -different. In this case, though, note that a -[flag](https://github.com/tensorflow/models/blob/master/research/slim/nets/resnet_v1.py#L150) -may be used to recover the original striding convention. - -Second, it could be that we have a bug somewhere. While we include -[many tests](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py) -in our library, it is always possible that we missed something. If you suspect -this is the case, please file a GitHub issue -[here](https://github.com/tensorflow/tensorflow/issues). - -### The number of FLOPs for network X seem different from what I expected... maybe your calculation is incorrect? - -First, note that the results presented here are based on the tensorflow -implementations from the -[TF-Slim model library](https://github.com/tensorflow/models/tree/master/research/slim). -So, it is possible that due to some implementation details the number of FLOPs -is different. - -Second, one common confusion arises since some papers refer to FLOPs as the -number of Multiply-Add operations; in other words, some papers count a -Multiply-Add as one floating point operation while others count as two. Here, we -follow the `tensorflow.profiler` convention and count a Multiply-Add as two -operations. One noticeable counter-example is the -[ResNet paper](https://arxiv.org/abs/1512.03385), where the FLOPs mentioned in -Table 1 therein actually mean the number of Multiply-Add's (see Section 3.3 in -their paper). So there is roughly a factor of two between their paper and our -numbers. - -Finally, we rely on `tensorflow.profiler` for estimating the number of floating -point operations. It could be that they have a bug somewhere, or that we are -using their library incorrectly, or that we simply have a bug somewhere else. If -you suspect this is the case, please file a GitHub issue -[here](https://github.com/tensorflow/tensorflow/issues). diff --git a/tensorflow/contrib/receptive_field/python/__init__.py b/tensorflow/contrib/receptive_field/python/__init__.py deleted file mode 100644 index 217047f92d3..00000000000 --- a/tensorflow/contrib/receptive_field/python/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Module to compute receptive field parameters for CNN tensorflow models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/receptive_field/python/util/examples/compute_rf.py b/tensorflow/contrib/receptive_field/python/util/examples/compute_rf.py deleted file mode 100644 index 72f98ccc32e..00000000000 --- a/tensorflow/contrib/receptive_field/python/util/examples/compute_rf.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Computes Receptive Field (RF) information given a graph protobuf.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import sys - -from google.protobuf import text_format - -from tensorflow.contrib.receptive_field import receptive_field_api as receptive_field -from tensorflow.core.framework import graph_pb2 -from tensorflow.python.platform import app -from tensorflow.python.platform import gfile -from tensorflow.python.platform import tf_logging as logging - -cmd_args = None - - -def _load_graphdef(path): - """Helper function to load GraphDef from file. - - Args: - path: Path to pbtxt file. - - Returns: - graph_def: A GraphDef object. - """ - graph_def = graph_pb2.GraphDef() - pbstr = gfile.Open(path).read() - text_format.Parse(pbstr, graph_def) - return graph_def - - -def main(unused_argv): - - graph_def = _load_graphdef(cmd_args.graph_path) - - (receptive_field_x, receptive_field_y, effective_stride_x, effective_stride_y, - effective_padding_x, effective_padding_y - ) = receptive_field.compute_receptive_field_from_graph_def( - graph_def, cmd_args.input_node, cmd_args.output_node) - - logging.info('Receptive field size (horizontal) = %s', receptive_field_x) - logging.info('Receptive field size (vertical) = %s', receptive_field_y) - logging.info('Effective stride (horizontal) = %s', effective_stride_x) - logging.info('Effective stride (vertical) = %s', effective_stride_y) - logging.info('Effective padding (horizontal) = %s', effective_padding_x) - logging.info('Effective padding (vertical) = %s', effective_padding_y) - - f = gfile.GFile('%s' % cmd_args.output_path, 'w') - f.write('Receptive field size (horizontal) = %s\n' % receptive_field_x) - f.write('Receptive field size (vertical) = %s\n' % receptive_field_y) - f.write('Effective stride (horizontal) = %s\n' % effective_stride_x) - f.write('Effective stride (vertical) = %s\n' % effective_stride_y) - f.write('Effective padding (horizontal) = %s\n' % effective_padding_x) - f.write('Effective padding (vertical) = %s\n' % effective_padding_y) - f.close() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.register('type', 'bool', lambda v: v.lower() == 'true') - parser.add_argument( - '--graph_path', type=str, default='', help='Graph path (pbtxt format).') - parser.add_argument( - '--output_path', - type=str, - default='', - help='Path to output text file where RF information will be written to.') - parser.add_argument( - '--input_node', type=str, default='', help='Name of input node.') - parser.add_argument( - '--output_node', type=str, default='', help='Name of output node.') - cmd_args, unparsed = parser.parse_known_args() - app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/receptive_field/python/util/examples/csv_to_markdown_table.py b/tensorflow/contrib/receptive_field/python/util/examples/csv_to_markdown_table.py deleted file mode 100644 index 8294dbde5f7..00000000000 --- a/tensorflow/contrib/receptive_field/python/util/examples/csv_to_markdown_table.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Simple script to convert CSV output from rf_benchmark to Markdown format. - -The input CSV should have the following fields: -- CNN -- input resolution -- end_point -- FLOPS (Billion) -- RF size hor -- RF size ver -- effective stride hor -- effective stride ver -- effective padding hor -- effective padding ver - -Since usually in all cases the parameters in the horizontal and vertical -directions are the same, this is assumed by this script, which only prints one -of them to the Markdown file. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import csv -import sys - -from tensorflow.python.platform import app - -cmd_args = None - - -def main(unused_argv): - with open(cmd_args.markdown_path, 'w') as f: - # Write table header and field size. - f.write('CNN | resolution | end-point | FLOPs (Billion) | RF | ' - 'effective stride | effective padding\n') - f.write(':--------------------: | :----------: | :---------------: | ' - ':---------------: | :-----: | :----: | :----:\n') - with open(cmd_args.csv_path) as csvfile: - reader = csv.DictReader(csvfile) - for row in reader: - # Make sure horizontal and parameters are the same. - assert row['RF size hor'] == row['RF size ver'] - assert row['effective stride hor'] == row['effective stride ver'] - assert row['effective padding hor'] == row['effective padding ver'] - - f.write('%s|%s|%s|%s|%s|%s|%s\n' % - (row['CNN'], row['input resolution'], row['end_point'], - row['FLOPs (Billion)'], row['RF size hor'], - row['effective stride hor'], row['effective padding hor'])) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.register('type', 'bool', lambda v: v.lower() == 'true') - parser.add_argument( - '--csv_path', - type=str, - default='/tmp/rf.csv', - help='Path where CSV output of rf_benchmark was saved.') - parser.add_argument( - '--markdown_path', - type=str, - default='/tmp/rf.md', - help='Path where Markdown output will be saved.') - cmd_args, unparsed = parser.parse_known_args() - app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/receptive_field/python/util/examples/rf_benchmark.py b/tensorflow/contrib/receptive_field/python/util/examples/rf_benchmark.py deleted file mode 100644 index f274e32e635..00000000000 --- a/tensorflow/contrib/receptive_field/python/util/examples/rf_benchmark.py +++ /dev/null @@ -1,583 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Computes Receptive Field (RF) information for different models. - -The receptive field (and related parameters) for the different models are -printed to stdout, and may also optionally be written to a CSV file. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import csv -import sys - -from tensorflow.contrib import framework -from tensorflow.contrib import slim -from tensorflow.contrib.receptive_field import receptive_field_api as receptive_field -from tensorflow.python.client import session -from tensorflow.python.framework import graph_util -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import importer -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import app -from tensorflow.python.profiler import profiler -from nets import alexnet -from nets import inception -from nets import mobilenet_v1 -from nets import resnet_v1 -from nets import resnet_v2 -from nets import vgg - -cmd_args = None - -# Input node name for all architectures. -_INPUT_NODE = 'input_image' - -# Variants of different network architectures. - -# - resnet: different versions and sizes. -_SUPPORTED_RESNET_VARIANTS = [ - 'resnet_v1_50', 'resnet_v1_101', 'resnet_v1_152', 'resnet_v1_200', - 'resnet_v2_50', 'resnet_v2_101', 'resnet_v2_152', 'resnet_v2_200' -] - -# - inception_resnet_v2: default, and version with SAME padding. -_SUPPORTED_INCEPTIONRESNETV2_VARIANTS = [ - 'inception_resnet_v2', 'inception_resnet_v2-same' -] - -# - inception_v2: default, and version with no separable conv. -_SUPPORTED_INCEPTIONV2_VARIANTS = [ - 'inception_v2', 'inception_v2-no-separable-conv' -] - -# - inception_v3: default version. -_SUPPORTED_INCEPTIONV3_VARIANTS = ['inception_v3'] - -# - inception_v4: default version. -_SUPPORTED_INCEPTIONV4_VARIANTS = ['inception_v4'] - -# - alexnet_v2: default version. -_SUPPORTED_ALEXNETV2_VARIANTS = ['alexnet_v2'] - -# - vgg: vgg_a (with 11 layers) and vgg_16 (version D). -_SUPPORTED_VGG_VARIANTS = ['vgg_a', 'vgg_16'] - -# - mobilenet_v1: 100% and 75%. -_SUPPORTED_MOBILENETV1_VARIANTS = ['mobilenet_v1', 'mobilenet_v1_075'] - - -def _construct_model(model_type='resnet_v1_50', placeholder_resolution=None): - """Constructs model for the desired type of CNN. - - Args: - model_type: Type of model to be used. - placeholder_resolution: Placeholder image resolution to use. - - Returns: - end_points: A dictionary from components of the network to the corresponding - activations. - - Raises: - ValueError: If the model_type is not supported. - """ - # Placeholder input. - images = array_ops.placeholder( - dtypes.float32, - shape=(1, placeholder_resolution, placeholder_resolution, 3), - name=_INPUT_NODE) - - # Construct model. - if model_type == 'inception_resnet_v2': - _, end_points = inception.inception_resnet_v2_base(images) - elif model_type == 'inception_resnet_v2-same': - _, end_points = inception.inception_resnet_v2_base( - images, align_feature_maps=True) - elif model_type == 'inception_v2': - _, end_points = inception.inception_v2_base(images) - elif model_type == 'inception_v2-no-separable-conv': - _, end_points = inception.inception_v2_base( - images, use_separable_conv=False) - elif model_type == 'inception_v3': - _, end_points = inception.inception_v3_base(images) - elif model_type == 'inception_v4': - _, end_points = inception.inception_v4_base(images) - elif model_type == 'alexnet_v2': - _, end_points = alexnet.alexnet_v2( - images, num_classes=None, is_training=False, global_pool=False) - elif model_type == 'vgg_a': - _, end_points = vgg.vgg_a( - images, num_classes=None, is_training=False, global_pool=False) - elif model_type == 'vgg_16': - _, end_points = vgg.vgg_16( - images, num_classes=None, is_training=False, global_pool=False) - elif model_type == 'mobilenet_v1': - _, end_points = mobilenet_v1.mobilenet_v1_base(images) - elif model_type == 'mobilenet_v1_075': - _, end_points = mobilenet_v1.mobilenet_v1_base( - images, depth_multiplier=0.75) - elif model_type == 'resnet_v1_50': - _, end_points = resnet_v1.resnet_v1_50( - images, num_classes=None, is_training=False, global_pool=False) - elif model_type == 'resnet_v1_101': - _, end_points = resnet_v1.resnet_v1_101( - images, num_classes=None, is_training=False, global_pool=False) - elif model_type == 'resnet_v1_152': - _, end_points = resnet_v1.resnet_v1_152( - images, num_classes=None, is_training=False, global_pool=False) - elif model_type == 'resnet_v1_200': - _, end_points = resnet_v1.resnet_v1_200( - images, num_classes=None, is_training=False, global_pool=False) - elif model_type == 'resnet_v2_50': - _, end_points = resnet_v2.resnet_v2_50( - images, num_classes=None, is_training=False, global_pool=False) - elif model_type == 'resnet_v2_101': - _, end_points = resnet_v2.resnet_v2_101( - images, num_classes=None, is_training=False, global_pool=False) - elif model_type == 'resnet_v2_152': - _, end_points = resnet_v2.resnet_v2_152( - images, num_classes=None, is_training=False, global_pool=False) - elif model_type == 'resnet_v2_200': - _, end_points = resnet_v2.resnet_v2_200( - images, num_classes=None, is_training=False, global_pool=False) - else: - raise ValueError('Unsupported model_type %s.' % model_type) - - return end_points - - -def _get_desired_end_point_keys(model_type='resnet_v1_50'): - """Gets list of desired end point keys for a type of CNN. - - Args: - model_type: Type of model to be used. - - Returns: - desired_end_point_types: A list containing the desired end-points. - - Raises: - ValueError: If the model_type is not supported. - """ - if model_type in _SUPPORTED_RESNET_VARIANTS: - blocks = ['block1', 'block2', 'block3', 'block4'] - desired_end_point_keys = ['%s/%s' % (model_type, i) for i in blocks] - elif model_type in _SUPPORTED_INCEPTIONRESNETV2_VARIANTS: - desired_end_point_keys = [ - 'Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', 'MaxPool_3a_3x3', - 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', 'MaxPool_5a_3x3', 'Mixed_5b', - 'Mixed_6a', 'PreAuxLogits', 'Mixed_7a', 'Conv2d_7b_1x1' - ] - elif model_type in _SUPPORTED_INCEPTIONV2_VARIANTS: - desired_end_point_keys = [ - 'Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', 'Conv2d_2c_3x3', - 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', 'Mixed_4a', 'Mixed_4b', - 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', 'Mixed_5a', 'Mixed_5b', 'Mixed_5c' - ] - elif model_type in _SUPPORTED_INCEPTIONV3_VARIANTS: - desired_end_point_keys = [ - 'Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', 'MaxPool_3a_3x3', - 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', 'MaxPool_5a_3x3', 'Mixed_5b', - 'Mixed_5c', 'Mixed_5d', 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', - 'Mixed_6e', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c' - ] - elif model_type in _SUPPORTED_INCEPTIONV4_VARIANTS: - desired_end_point_keys = [ - 'Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', 'Mixed_3a', - 'Mixed_4a', 'Mixed_5a', 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', 'Mixed_5e', - 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', 'Mixed_6e', 'Mixed_6f', - 'Mixed_6g', 'Mixed_6h', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c', 'Mixed_7d' - ] - elif model_type in _SUPPORTED_ALEXNETV2_VARIANTS: - ep = ['conv1', 'pool1', 'conv2', 'conv3', 'conv4', 'conv5', 'pool5'] - desired_end_point_keys = ['%s/%s' % (model_type, i) for i in ep] - elif model_type in _SUPPORTED_VGG_VARIANTS: - ep = [ - 'conv1/conv1_1', 'pool1', 'conv2/conv2_1', 'pool2', 'conv3/conv3_1', - 'conv3/conv3_2', 'pool3', 'conv4/conv4_1', 'conv4/conv4_2', 'pool4', - 'conv5/conv5_1', 'conv5/conv5_2', 'pool5' - ] - desired_end_point_keys = ['%s/%s' % (model_type, i) for i in ep] - elif model_type in _SUPPORTED_MOBILENETV1_VARIANTS: - desired_end_point_keys = [ - 'Conv2d_0', 'Conv2d_1_pointwise', 'Conv2d_2_pointwise', - 'Conv2d_3_pointwise', 'Conv2d_4_pointwise', 'Conv2d_5_pointwise', - 'Conv2d_6_pointwise', 'Conv2d_7_pointwise', 'Conv2d_8_pointwise', - 'Conv2d_9_pointwise', 'Conv2d_10_pointwise', 'Conv2d_11_pointwise', - 'Conv2d_12_pointwise', 'Conv2d_13_pointwise' - ] - else: - raise ValueError('Unsupported model_type %s.' % model_type) - - return desired_end_point_keys - - -def _model_graph_def(model_type, - desired_end_point_keys, - placeholder_resolution=None, - arg_sc=None): - """Constructs a model graph, returning GraphDef's and end-points. - - Args: - model_type: Type of model to be used. - desired_end_point_keys: List of desired end points for which receptive field - information will be computed. - placeholder_resolution: Placeholder resolution to use when constructing the - graph. - arg_sc: Optional arg scope to use in constructing the graph. - - Returns: - graphdefs: List of GraphDef's, one per desired end point. - end_points: A dictionary from components of the network to the corresponding - activations. - """ - if arg_sc is None: - arg_sc = {} - - g = ops.Graph() - sess = session.Session(graph=g) - with g.as_default(): - with framework.arg_scope(arg_sc): - end_points = _construct_model(model_type, placeholder_resolution) - sess.run(variables.global_variables_initializer()) - - # Produce a graphdef for each desired end point. While this is not required - # for receptive field computation, it helps to provide a better estimate of - # the number of floating point operations, since it removes initialization - # layers. - graphdefs = [] - for desired_end_point_key in desired_end_point_keys: - end_point_node_name = end_points[desired_end_point_key].name.split(':')[0] - graphdefs.append( - graph_util.convert_variables_to_constants(sess, g.as_graph_def(), - [end_point_node_name])) - - return graphdefs, end_points - - -def _model_rf_and_flops(graphdefs, - end_points, - desired_end_point_keys, - model_type='resnet_v1_50', - csv_writer=None, - input_resolution=None): - """Computes receptive field and FLOPs for a given CNN model. - - The information will be printed to stdout. If the RF parameters are the same - for the horizontal and vertical directions, it will be printed only once. - Otherwise, they are printed once for the horizontal and once for the vertical - directions. - - Args: - graphdefs: List of GraphDef's, one per desired end point. - end_points: A dictionary from components of the model to the corresponding - activations. - desired_end_point_keys: List of desired end points for which receptive field - information will be computed. - model_type: Type of model to be used, used only for printing purposes. - csv_writer: A CSV writer for RF parameters, which is used if it is not None. - input_resolution: Input resolution to use when computing RF parameters. This - is important for the case where padding can only be defined if the input - resolution is known, which may happen if using SAME padding. This is - assumed the resolution for both height and width. If None, we consider the - resolution is unknown. - """ - # Configuration of profiler. Avoid verbose output. - profiler_options = profiler.ProfileOptionBuilder.float_operation() - profiler_options['output'] = 'file:outfile=/dev/null' - - for i, desired_end_point_key in enumerate(desired_end_point_keys): - print('- %s:' % desired_end_point_key) - output_node_with_colon = end_points[desired_end_point_key].name - pos = output_node_with_colon.rfind(':') - output_node = output_node_with_colon[:pos] - try: - # Compute receptive field parameters. - (receptive_field_x, receptive_field_y, effective_stride_x, - effective_stride_y, effective_padding_x, effective_padding_y - ) = receptive_field.compute_receptive_field_from_graph_def( - graphdefs[i], - _INPUT_NODE, - output_node, - input_resolution=input_resolution) - - # Compute FLOPs. Can only be done if input resolution is known. - if input_resolution is None: - billion_flops_str = 'None' - else: - g = ops.Graph() - with g.as_default(): - importer.import_graph_def(graphdefs[i], name='') - flops = profiler.profile(g, options=profiler_options) - billion_flops = flops.total_float_ops / 1e9 - billion_flops_str = '%.3f' % billion_flops - - # If values are the same in horizontal/vertical directions, just report - # one of them. Otherwise, report both. - if (receptive_field_x == receptive_field_y) and ( - effective_stride_x == effective_stride_y) and ( - effective_padding_x == effective_padding_y): - print('Receptive field size = %5s, effective stride = %5s, effective ' - 'padding = %5s, FLOPs (Billion) = %7s' % - (str(receptive_field_x), str(effective_stride_x), - str(effective_padding_x), billion_flops_str)) - else: - print('Receptive field size: horizontal = %5s, vertical = %5s. ' - 'Effective stride: horizontal = %5s, vertical = %5s. Effective ' - 'padding: horizontal = %5s, vertical = %5s, ' - 'FLOPs (Billion) = %7s' % - (str(receptive_field_x), str(receptive_field_y), - str(effective_stride_x), str(effective_stride_y), - str(effective_padding_x), str(effective_padding_y), - billion_flops_str)) - if csv_writer is not None: - csv_writer.writerow({ - 'CNN': - model_type, - 'input resolution': - str(input_resolution[0]) - if input_resolution is not None else 'None', - 'end_point': - desired_end_point_key, - 'FLOPs (Billion)': - billion_flops_str, - 'RF size hor': - str(receptive_field_x), - 'RF size ver': - str(receptive_field_y), - 'effective stride hor': - str(effective_stride_x), - 'effective stride ver': - str(effective_stride_y), - 'effective padding hor': - str(effective_padding_x), - 'effective padding ver': - str(effective_padding_y) - }) - except ValueError as e: - print('---->ERROR: Computing RF parameters for model %s with final end ' - 'point %s and input resolution %s did not work' % - (model_type, desired_end_point_key, input_resolution)) - print('---->The returned error is: %s' % e) - if csv_writer is not None: - csv_writer.writerow({ - 'CNN': - model_type, - 'input resolution': - str(input_resolution[0]) - if input_resolution is not None else 'None', - 'end_point': - desired_end_point_key, - 'FLOPs': - 'None', - 'RF size hor': - 'None', - 'RF size ver': - 'None', - 'effective stride hor': - 'None', - 'effective stride ver': - 'None', - 'effective padding hor': - 'None', - 'effective padding ver': - 'None' - }) - - -def _process_model_rf_and_flops(model_type='resnet_v1_50', - csv_writer=None, - arg_sc=None, - input_resolutions=None): - """Contructs model graph and desired end-points, and compute RF. - - The computed RF parameters are printed to stdout by the _model_rf_and_flops - function. - - Args: - model_type: Type of model to be used. - csv_writer: A CSV writer for RF parameters, which is used if it is not None. - arg_sc: Optional arg scope to use in constructing the graph. - input_resolutions: List of 1D input resolutions to use when computing RF - parameters. This is important for the case where padding can only be - defined if the input resolution is known, which may happen if using SAME - padding. The entries in the list are assumed the resolution for both - height and width. If one of the elements in the list is None, we consider - it to mean that the resolution is unknown. If the list itself is None, we - use the default list [None, 224, 321]. - """ - # Process default value for this list. - if input_resolutions is None: - input_resolutions = [None, 224, 321] - - desired_end_point_keys = _get_desired_end_point_keys(model_type) - for n in input_resolutions: - print('********************%s, input resolution = %s' % (model_type, n)) - graphdefs, end_points = _model_graph_def(model_type, desired_end_point_keys, - n, arg_sc) - _model_rf_and_flops( - graphdefs, - end_points, - desired_end_point_keys, - model_type, - csv_writer, - input_resolution=[n, n] if n is not None else None) - - -def _resnet_rf(csv_writer=None): - """Computes RF and associated parameters for resnet models. - - The computed values are written to stdout. - - Args: - csv_writer: A CSV writer for RF parameters, which is used if it is not None. - """ - for model_type in _SUPPORTED_RESNET_VARIANTS: - arg_sc = resnet_v1.resnet_arg_scope() - _process_model_rf_and_flops(model_type, csv_writer, arg_sc) - - -def _inception_resnet_v2_rf(csv_writer=None): - """Computes RF and associated parameters for the inception_resnet_v2 model. - - The computed values are written to stdout. - - Args: - csv_writer: A CSV writer for RF parameters, which is used if it is not None. - """ - for model_type in _SUPPORTED_INCEPTIONRESNETV2_VARIANTS: - _process_model_rf_and_flops(model_type, csv_writer) - - -def _inception_v2_rf(csv_writer=None): - """Computes RF and associated parameters for the inception_v2 model. - - The computed values are written to stdout. - - Args: - csv_writer: A CSV writer for RF parameters, which is used if it is not None. - """ - for model_type in _SUPPORTED_INCEPTIONV2_VARIANTS: - _process_model_rf_and_flops(model_type, csv_writer) - - -def _inception_v3_rf(csv_writer=None): - """Computes RF and associated parameters for the inception_v3 model. - - The computed values are written to stdout. - - Args: - csv_writer: A CSV writer for RF parameters, which is used if it is not None. - """ - for model_type in _SUPPORTED_INCEPTIONV3_VARIANTS: - _process_model_rf_and_flops(model_type, csv_writer) - - -def _inception_v4_rf(csv_writer=None): - """Computes RF and associated parameters for the inception_v4 model. - - The computed values are written to stdout. - - Args: - csv_writer: A CSV writer for RF parameters, which is used if it is not None. - """ - for model_type in _SUPPORTED_INCEPTIONV4_VARIANTS: - _process_model_rf_and_flops(model_type, csv_writer) - - -def _alexnet_v2_rf(csv_writer=None): - """Computes RF and associated parameters for the alexnet_v2 model. - - The computed values are written to stdout. - - Args: - csv_writer: A CSV writer for RF parameters, which is used if it is not None. - """ - for model_type in _SUPPORTED_ALEXNETV2_VARIANTS: - _process_model_rf_and_flops(model_type, csv_writer) - - -def _vgg_rf(csv_writer=None): - """Computes RF and associated parameters for the vgg model. - - The computed values are written to stdout. - - Args: - csv_writer: A CSV writer for RF parameters, which is used if it is not None. - """ - for model_type in _SUPPORTED_VGG_VARIANTS: - _process_model_rf_and_flops(model_type, csv_writer) - - -def _mobilenet_v1_rf(csv_writer=None): - """Computes RF and associated parameters for the mobilenet_v1 model. - - The computed values are written to stdout. - - Args: - csv_writer: A CSV writer for RF parameters, which is used if it is not None. - """ - for model_type in _SUPPORTED_MOBILENETV1_VARIANTS: - with slim.arg_scope([slim.batch_norm, slim.dropout], - is_training=False) as arg_sc: - _process_model_rf_and_flops(model_type, csv_writer, arg_sc) - - -def main(unused_argv): - # Configure CSV file which will be written, if desired. - if cmd_args.csv_path: - csv_file = open(cmd_args.csv_path, 'w') - field_names = [ - 'CNN', 'input resolution', 'end_point', 'FLOPs (Billion)', - 'RF size hor', 'RF size ver', 'effective stride hor', - 'effective stride ver', 'effective padding hor', 'effective padding ver' - ] - rf_writer = csv.DictWriter(csv_file, fieldnames=field_names) - rf_writer.writeheader() - else: - rf_writer = None - - # Compute RF parameters for each network architecture. - _alexnet_v2_rf(rf_writer) - _vgg_rf(rf_writer) - _inception_v2_rf(rf_writer) - _inception_v3_rf(rf_writer) - _inception_v4_rf(rf_writer) - _inception_resnet_v2_rf(rf_writer) - _mobilenet_v1_rf(rf_writer) - _resnet_rf(rf_writer) - - # Close CSV file, if it was opened. - if cmd_args.csv_path: - csv_file.close() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.register('type', 'bool', lambda v: v.lower() == 'true') - parser.add_argument( - '--csv_path', - type=str, - default='', - help="""\ - Path to CSV file that will be written with RF parameters.If empty, no - file will be written.\ - """) - cmd_args, unparsed = parser.parse_known_args() - app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/receptive_field/python/util/examples/write_inception_resnet_v2_graph.py b/tensorflow/contrib/receptive_field/python/util/examples/write_inception_resnet_v2_graph.py deleted file mode 100644 index a4948833966..00000000000 --- a/tensorflow/contrib/receptive_field/python/util/examples/write_inception_resnet_v2_graph.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Simple script to write Inception-ResNet-v2 model to graph file. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import sys - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import graph_io -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import app -from nets import inception - -cmd_args = None - - -def main(unused_argv): - # Model definition. - g = ops.Graph() - with g.as_default(): - images = array_ops.placeholder( - dtypes.float32, shape=(1, None, None, 3), name='input_image') - inception.inception_resnet_v2_base(images) - - graph_io.write_graph(g.as_graph_def(), cmd_args.graph_dir, - cmd_args.graph_filename) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.register('type', 'bool', lambda v: v.lower() == 'true') - parser.add_argument( - '--graph_dir', - type=str, - default='/tmp', - help='Directory where graph will be saved.') - parser.add_argument( - '--graph_filename', - type=str, - default='graph.pbtxt', - help='Filename of graph that will be saved.') - cmd_args, unparsed = parser.parse_known_args() - app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/receptive_field/python/util/graph_compute_order.py b/tensorflow/contrib/receptive_field/python/util/graph_compute_order.py deleted file mode 100644 index 0388079f20d..00000000000 --- a/tensorflow/contrib/receptive_field/python/util/graph_compute_order.py +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Library to compute order of computations in a graph. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import math -from tensorflow.contrib.receptive_field.python.util import parse_layer_parameters -from tensorflow.python.platform import tf_logging as logging - - -def parse_graph_nodes(graph_def): - """Helper function to parse GraphDef's nodes. - - It returns a dict mapping from node name to NodeDef. - - Args: - graph_def: A GraphDef object. - - Returns: - name_to_node: Dict keyed by node name, each entry containing the node's - NodeDef. - """ - name_to_node = {} - for node_def in graph_def.node: - name_to_node[node_def.name] = node_def - return name_to_node - - -# Named tuple used to collect information from each node in a computation graph. -_node_info = collections.namedtuple( - 'NodeInfo', field_names=['order', 'node', 'input_size', 'output_size']) - - -def _compute_output_resolution(input_spatial_resolution, kernel_size, stride, - total_padding): - """Computes output resolution, given input resolution and layer parameters. - - Note that this computation is done only over one dimension (eg, x or y). - If any of the inputs is None, returns None. - - Args: - input_spatial_resolution: Input spatial resolution (int). - kernel_size: Kernel size (int). - stride: Stride (int). - total_padding: Total padding to be applied (int). - Returns: - output_resolution: Output dimension (int) or None. - """ - if (input_spatial_resolution is None) or (kernel_size is None) or ( - stride is None) or (total_padding is None): - return None - return int( - math.ceil(( - input_spatial_resolution + total_padding - kernel_size + 1) / stride)) - - -def _get_computed_nodes(name_to_node, - current, - node_info, - input_node_name='', - input_node_size=None): - """Traverses the graph recursively to compute its topological order. - - Optionally, the function may also compute the input and output feature map - resolutions at each node. In this case, input_node_name and input_node_size - must be set. Note that if a node's op type is unknown, the input and output - resolutions are ignored and set to None. - - Args: - name_to_node: Dict keyed by node name, each entry containing the node's - NodeDef. - current: Current node name. - node_info: Map of nodes we've already traversed, containing their _node_info - information. - input_node_name: Name of node with fixed input resolution (optional). - input_node_size: Fixed input resolution to use (optional). - Returns: - order: Order in topological sort for 'current'. - input_size: Tensor spatial resolution at input of current node. - output_size: Tensor spatial resolution at output of current node. - """ - if current in node_info: - return (node_info[current].order, node_info[current].input_size, - node_info[current].output_size) - - node_def = name_to_node[current] - - if current == input_node_name: - order = 0 - input_size = None - output_size = input_node_size - node_info[current] = _node_info(order, node_def, input_size, output_size) - return (order, input_size, output_size) - - input_size = None - output_size = None - - order = 0 - number_inputs = 0 - for each in node_def.input: - # Parses name of input node. - if each.startswith('^'): - # The character '^' denotes a control dependency, so this input node can - # be safely ignored. - continue - each = each.split(':')[0] - # Recursively computes ordering. - (parent_order, _, parent_output_size) = _get_computed_nodes( - name_to_node, each, node_info, input_node_name, input_node_size) - order = max(order, parent_order + 1) - if number_inputs == 0: - # For all the types of nodes we consider, the first input corresponds to - # the feature map. - input_size = parent_output_size - number_inputs += 1 - - # Figure out output size for this layer. - logging.vlog(3, 'input_size = %s', input_size) - if input_size is None: - output_size = None - else: - (kernel_size_x, kernel_size_y, stride_x, stride_y, _, _, total_padding_x, - total_padding_y) = ( - parse_layer_parameters.get_layer_params( - node_def, name_to_node, input_size, force=True)) - logging.vlog(3, 'kernel_size_x = %s, kernel_size_y = %s, ' - 'stride_x = %s, stride_y = %s, ' - 'total_padding_x = %s, total_padding_y = %s' % - (kernel_size_x, kernel_size_y, stride_x, stride_y, - total_padding_x, total_padding_y)) - output_size = [None] * 2 - output_size[0] = _compute_output_resolution(input_size[0], kernel_size_x, - stride_x, total_padding_x) - output_size[1] = _compute_output_resolution(input_size[1], kernel_size_y, - stride_y, total_padding_y) - - logging.vlog(3, 'output_size = %s', output_size) - node_info[current] = _node_info(order, node_def, input_size, output_size) - - return order, input_size, output_size - - -def get_compute_order(graph_def, input_node_name='', input_node_size=None): - """Computes order of computation for a given CNN graph. - - Optionally, the function may also compute the input and output feature map - resolutions at each node. In this case, input_node_name and input_node_size - must be set. Note that if a node's op type is unknown, the input and output - resolutions are ignored and set to None. - - Args: - graph_def: GraphDef object. - input_node_name: Name of node with fixed input resolution (optional). This - is usually the node name for the input image in a CNN. - input_node_size: 2D list of integers, fixed input resolution to use - (optional). This is usually the input resolution used for the input image - in a CNN (common examples are: [224, 224], [299, 299], [321, 321]). - Returns: - node_info: Default dict keyed by node name, mapping to a named tuple with - the following fields: - - order: Integer denoting topological order; - - node: NodeDef for the given node; - - input_size: 2D list of integers, denoting the input spatial resolution - to the node; - - output_size: 2D list of integers, denoting the output spatial resolution - of the node. - name_to_node: Dict keyed by node name, each entry containing the node's - NodeDef. - """ - name_to_node = parse_graph_nodes(graph_def) - node_info = collections.defaultdict(_node_info) - for each in graph_def.node: - _get_computed_nodes(name_to_node, each.name, node_info, input_node_name, - input_node_size) - return node_info, name_to_node diff --git a/tensorflow/contrib/receptive_field/python/util/graph_compute_order_test.py b/tensorflow/contrib/receptive_field/python/util/graph_compute_order_test.py deleted file mode 100644 index 94c992ad215..00000000000 --- a/tensorflow/contrib/receptive_field/python/util/graph_compute_order_test.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for graph_compute_order module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import slim -from tensorflow.contrib.receptive_field import receptive_field_api as receptive_field -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import nn -from tensorflow.python.platform import test - - -def create_test_network(): - """Convolutional neural network for test. - - Returns: - g: Tensorflow graph object (Graph proto). - """ - g = ops.Graph() - with g.as_default(): - # An input test image with unknown spatial resolution. - x = array_ops.placeholder( - dtypes.float32, (None, None, None, 1), name='input_image') - # Left branch before first addition. - l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID') - # Right branch before first addition. - l2_pad = array_ops.pad(x, [[0, 0], [1, 0], [1, 0], [0, 0]], name='L2_pad') - l2 = slim.conv2d(l2_pad, 1, [3, 3], stride=2, scope='L2', padding='VALID') - l3 = slim.max_pool2d(l2, [3, 3], stride=2, scope='L3', padding='SAME') - # First addition. - l4 = nn.relu(l1 + l3, name='L4_relu') - # Left branch after first addition. - l5 = slim.conv2d(l4, 1, [1, 1], stride=2, scope='L5', padding='SAME') - # Right branch after first addition. - l6 = slim.conv2d(l4, 1, [3, 3], stride=2, scope='L6', padding='SAME') - # Final addition. - gen_math_ops.add(l5, l6, name='L7_add') - - return g - - -class GraphComputeOrderTest(test.TestCase): - - def check_topological_sort_and_sizes(self, - node_info, - expected_input_sizes=None, - expected_output_sizes=None): - """Helper function to check topological sorting and sizes are correct. - - The arguments expected_input_sizes and expected_output_sizes are used to - check that the sizes are correct, if they are given. - - Args: - node_info: Default dict keyed by node name, mapping to a named tuple with - the following keys: {order, node, input_size, output_size}. - expected_input_sizes: Dict mapping node names to expected input sizes - (optional). - expected_output_sizes: Dict mapping node names to expected output sizes - (optional). - """ - # Loop over nodes in sorted order, collecting those that were already seen. - # These will be used to make sure that the graph is topologically sorted. - # At the same time, we construct dicts from node name to input/output size, - # which will be used to check those. - already_seen_nodes = [] - input_sizes = {} - output_sizes = {} - for _, (_, node, input_size, output_size) in sorted( - node_info.items(), key=lambda x: x[1].order): - for inp_name in node.input: - # Since the graph is topologically sorted, the inputs to the current - # node must have been seen beforehand. - self.assertIn(inp_name, already_seen_nodes) - input_sizes[node.name] = input_size - output_sizes[node.name] = output_size - already_seen_nodes.append(node.name) - - # Check input sizes, if desired. - if expected_input_sizes is not None: - for k, v in expected_input_sizes.items(): - self.assertIn(k, input_sizes) - self.assertEqual(input_sizes[k], v) - - # Check output sizes, if desired. - if expected_output_sizes is not None: - for k, v in expected_output_sizes.items(): - self.assertIn(k, output_sizes) - self.assertEqual(output_sizes[k], v) - - def testGraphOrderIsCorrect(self): - """Tests that the order and sizes of create_test_network() are correct.""" - - graph_def = create_test_network().as_graph_def() - - # Case 1: Input node name/size are not given. - node_info, _ = receptive_field.get_compute_order(graph_def) - self.check_topological_sort_and_sizes(node_info) - - # Case 2: Input node name is given, but not size. - node_info, _ = receptive_field.get_compute_order( - graph_def, input_node_name='input_image') - self.check_topological_sort_and_sizes(node_info) - - # Case 3: Input node name and size (224) are given. - node_info, _ = receptive_field.get_compute_order( - graph_def, input_node_name='input_image', input_node_size=[224, 224]) - expected_input_sizes = { - 'input_image': None, - 'L1/Conv2D': [224, 224], - 'L2_pad': [224, 224], - 'L2/Conv2D': [225, 225], - 'L3/MaxPool': [112, 112], - 'L4_relu': [56, 56], - 'L5/Conv2D': [56, 56], - 'L6/Conv2D': [56, 56], - 'L7_add': [28, 28], - } - expected_output_sizes = { - 'input_image': [224, 224], - 'L1/Conv2D': [56, 56], - 'L2_pad': [225, 225], - 'L2/Conv2D': [112, 112], - 'L3/MaxPool': [56, 56], - 'L4_relu': [56, 56], - 'L5/Conv2D': [28, 28], - 'L6/Conv2D': [28, 28], - 'L7_add': [28, 28], - } - self.check_topological_sort_and_sizes(node_info, expected_input_sizes, - expected_output_sizes) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters.py b/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters.py deleted file mode 100644 index 0bdd9e62ef7..00000000000 --- a/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters.py +++ /dev/null @@ -1,337 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functions to parse RF-related parameters from TF layers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -from tensorflow.contrib.util import make_ndarray -from tensorflow.python.platform import tf_logging as logging - -# White-listed layer operations, which do not affect the receptive field -# computation. -_UNCHANGED_RF_LAYER_OPS = [ - "Add", "AddV2", "BiasAdd", "Cast", "Ceil", "ConcatV2", "Const", "Floor", - "FusedBatchNorm", "Identity", "Log", "Mul", "Pow", "RealDiv", "Relu", - "Relu6", "Round", "Rsqrt", "Softplus", "Sub", "VariableV2", "LRN", - "GreaterEqual" -] - -# Different ways in which padding modes may be spelled. -_VALID_PADDING = ["VALID", b"VALID"] -_SAME_PADDING = ["SAME", b"SAME"] - - -def _stride_size(node, name_to_node): - """Computes stride size given a TF node. - - Args: - node: Tensorflow node (NodeDef proto). - name_to_node: For MaxPoolV2, mapping from variable name Tensorflow node. - - Returns: - stride_x: Stride size for horizontal direction (integer). - stride_y: Stride size for vertical direction (integer). - - Raises: - ValueError: If stride input cannot be found in `name_to_node`. - """ - if node.op == "MaxPoolV2": - strides_input_name = node.input[2] - if not strides_input_name.endswith("/strides"): - raise ValueError("Strides name does not end with '/strides'") - strides_node = name_to_node[strides_input_name] - value = strides_node.attr["value"] - t = make_ndarray(value.tensor) - stride_y = t[1] - stride_x = t[2] - else: - strides_attr = node.attr["strides"] - logging.vlog(4, "strides_attr = %s", strides_attr) - stride_y = strides_attr.list.i[1] - stride_x = strides_attr.list.i[2] - return stride_x, stride_y - - -def _conv_kernel_size(node, name_to_node): - """Computes kernel size given a TF convolution or pooling node. - - Args: - node: Tensorflow node (NodeDef proto). - name_to_node: Dict keyed by node name, each entry containing the node's - NodeDef. - - Returns: - kernel_size_x: Kernel size for horizontal direction (integer). - kernel_size_y: Kernel size for vertical direction (integer). - - Raises: - ValueError: If the weight layer node is misconfigured. - """ - weights_layer_read_name = node.input[1] - if not weights_layer_read_name.endswith("/read"): - raise ValueError( - "Weight layer's name input to conv layer does not end with '/read'") - weights_layer_param_name = weights_layer_read_name[:-5] - weights_node = name_to_node[weights_layer_param_name] - if weights_node.op == "VariableV2": - shape_dim = weights_node.attr["shape"].shape.dim - elif weights_node.op == "Const": - shape_dim = weights_node.attr["value"].tensor.tensor_shape.dim - else: - raise ValueError( - "Weight layer {} is not of type VariableV2 or Const: {}".format( - weights_layer_param_name, weights_node.op)) - if len(shape_dim) != 4: - raise ValueError( - "Weight layer {} does not have rank 4. Instead, it has: {}".format( - weights_layer_param_name, len(shape_dim))) - logging.vlog(4, "weight shape = %s", shape_dim) - kernel_size_y = shape_dim[0].size - kernel_size_x = shape_dim[1].size - - return kernel_size_x, kernel_size_y - - -def _padding_size_conv_pool(node, kernel_size, stride, input_resolution=None): - """Computes padding size given a TF convolution or pooling node. - - Args: - node: Tensorflow node (NodeDef proto). - kernel_size: Kernel size of node (integer). - stride: Stride size of node (integer). - input_resolution: Input resolution to assume, if not None (integer). - - Returns: - total_padding: Total padding size (integer). - padding: Padding size, applied to the left or top (integer). - - Raises: - ValueError: If padding is invalid. - """ - # In this case, we need to carefully consider the different TF padding modes. - # The padding depends on kernel size, and may depend on input size. If it - # depends on input size and input_resolution is None, we raise an exception. - padding_attr = node.attr["padding"] - logging.vlog(4, "padding_attr = %s", padding_attr) - if padding_attr.s in _VALID_PADDING: - total_padding = 0 - padding = 0 - elif padding_attr.s in _SAME_PADDING: - if input_resolution is None: - # In this case, we do not know the input resolution, so we can only know - # the padding in some special cases. - if kernel_size == 1: - total_padding = 0 - padding = 0 - elif stride == 1: - total_padding = kernel_size - 1 - padding = int(math.floor(float(total_padding) / 2)) - elif stride == 2 and kernel_size % 2 == 0: - # In this case, we can be sure of the left/top padding, but not of the - # total padding. - total_padding = None - padding = int(math.floor((float(kernel_size) - 1) / 2)) - else: - total_padding = None - padding = None - logging.warning( - "Padding depends on input size, which means that the effective " - "padding may be different depending on the input image " - "dimensionality. In this case, alignment check will be skipped. If" - " you know the input resolution, please set it.") - else: - # First, compute total_padding based on documentation. - if input_resolution % stride == 0: - total_padding = int(max(float(kernel_size - stride), 0.0)) - else: - total_padding = int( - max(float(kernel_size - (input_resolution % stride)), 0.0)) - # Then, compute left/top padding. - padding = int(math.floor(float(total_padding) / 2)) - - else: - raise ValueError("Invalid padding operation %s" % padding_attr.s) - return total_padding, padding - - -def _pool_kernel_size(node, name_to_node): - """Computes kernel size given a TF pooling node. - - Args: - node: Tensorflow node (NodeDef proto). - name_to_node: For MaxPoolV2, mapping from node name to NodeDef. - - Returns: - kernel_size_x: Kernel size for horizontal direction (integer). - kernel_size_y: Kernel size for vertical direction (integer). - - Raises: - ValueError: If pooling is invalid. - """ - if node.op == "MaxPoolV2": - ksize_input_name = node.input[1] - if not ksize_input_name.endswith("/ksize"): - raise ValueError("Kernel size name does not end with '/ksize'") - ksize_node = name_to_node[ksize_input_name] - value = ksize_node.attr["value"] - t = make_ndarray(value.tensor) - kernel_size_y = t[1] - kernel_size_x = t[2] - if t[0] != 1: - raise ValueError("pool ksize for first dim is not 1") - if t[3] != 1: - raise ValueError("pool ksize for last dim is not 1") - else: - ksize = node.attr["ksize"] - kernel_size_y = ksize.list.i[1] - kernel_size_x = ksize.list.i[2] - if ksize.list.i[0] != 1: - raise ValueError("pool ksize for first dim is not 1") - if ksize.list.i[3] != 1: - raise ValueError("pool ksize for last dim is not 1") - return kernel_size_x, kernel_size_y - - -def _padding_size_pad_layer(node, name_to_node): - """Computes padding size given a TF padding node. - - Args: - node: Tensorflow node (NodeDef proto). - name_to_node: Dict keyed by node name, each entry containing the node's - NodeDef. - - Returns: - total_padding_x: Total padding size for horizontal direction (integer). - padding_x: Padding size for horizontal direction, left side (integer). - total_padding_y: Total padding size for vertical direction (integer). - padding_y: Padding size for vertical direction, top side (integer). - - Raises: - ValueError: If padding layer is invalid. - """ - paddings_layer_name = node.input[1] - if not paddings_layer_name.endswith("/paddings"): - raise ValueError("Padding layer name does not end with '/paddings'") - paddings_node = name_to_node[paddings_layer_name] - if paddings_node.op != "Const": - raise ValueError("Padding op is not Const") - value = paddings_node.attr["value"] - t = make_ndarray(value.tensor) - padding_y = t[1][0] - padding_x = t[2][0] - total_padding_y = padding_y + t[1][1] - total_padding_x = padding_x + t[2][1] - if (t[0][0] != 0) or (t[0][1] != 0): - raise ValueError("padding is not zero for first tensor dim") - if (t[3][0] != 0) or (t[3][1] != 0): - raise ValueError("padding is not zero for last tensor dim") - return total_padding_x, padding_x, total_padding_y, padding_y - - -def get_layer_params(node, name_to_node, input_resolution=None, force=False): - """Gets layer parameters relevant for RF computation. - - Currently, only these nodes are supported: - - Conv2D - - DepthwiseConv2dNative - - Pad - - MaxPool - - AvgPool - - all nodes listed in _UNCHANGED_RF_LAYER_OPS - - Args: - node: Tensorflow node (NodeDef proto). - name_to_node: Dict keyed by node name, each entry containing the node's - NodeDef. - input_resolution: List with 2 dimensions, denoting the height/width of the - input feature map to this layer. If set to None, then the padding may be - undefined (in tensorflow, SAME padding depends on input spatial - resolution). - force: If True, the function does not raise a ValueError if the layer op is - unknown. Instead, in this case it sets each of the returned parameters to - None. - - Returns: - kernel_size_x: Kernel size for horizontal direction (integer). - kernel_size_y: Kernel size for vertical direction (integer). - stride_x: Stride size for horizontal direction (integer). - stride_y: Stride size for vertical direction (integer). - padding_x: Padding size for horizontal direction, left side (integer). - padding_y: Padding size for vertical direction, top side (integer). - total_padding_x: Total padding size for horizontal direction (integer). - total_padding_y: Total padding size for vertical direction (integer). - - Raises: - ValueError: If layer op is unknown and force is False. - """ - logging.vlog(3, "node.name = %s", node.name) - logging.vlog(3, "node.op = %s", node.op) - logging.vlog(4, "node = %s", node) - if node.op == "Conv2D" or node.op == "DepthwiseConv2dNative": - stride_x, stride_y = _stride_size(node, name_to_node) - kernel_size_x, kernel_size_y = _conv_kernel_size(node, name_to_node) - # Compute the padding for this node separately for each direction. - total_padding_x, padding_x = _padding_size_conv_pool( - node, kernel_size_x, stride_x, - input_resolution[1] if input_resolution is not None else None) - total_padding_y, padding_y = _padding_size_conv_pool( - node, kernel_size_y, stride_y, - input_resolution[0] if input_resolution is not None else None) - elif node.op == "Pad": - # Kernel and stride are simply 1 in this case. - kernel_size_x = 1 - kernel_size_y = 1 - stride_x = 1 - stride_y = 1 - total_padding_x, padding_x, total_padding_y, padding_y = ( - _padding_size_pad_layer(node, name_to_node)) - elif node.op == "MaxPool" or node.op == "MaxPoolV2" or node.op == "AvgPool": - stride_x, stride_y = _stride_size(node, name_to_node) - kernel_size_x, kernel_size_y = _pool_kernel_size(node, name_to_node) - # Compute the padding for this node separately for each direction. - total_padding_x, padding_x = _padding_size_conv_pool( - node, kernel_size_x, stride_x, - input_resolution[1] if input_resolution is not None else None) - total_padding_y, padding_y = _padding_size_conv_pool( - node, kernel_size_y, stride_y, - input_resolution[0] if input_resolution is not None else None) - elif node.op in _UNCHANGED_RF_LAYER_OPS: - # These nodes do not modify the RF parameters. - kernel_size_x = 1 - kernel_size_y = 1 - stride_x = 1 - stride_y = 1 - total_padding_x = 0 - padding_x = 0 - total_padding_y = 0 - padding_y = 0 - else: - if force: - kernel_size_x = None - kernel_size_y = None - stride_x = None - stride_y = None - total_padding_x = None - padding_x = None - total_padding_y = None - padding_y = None - else: - raise ValueError("Unknown layer for operation '%s': %s" % - (node.name, node.op)) - return (kernel_size_x, kernel_size_y, stride_x, stride_y, padding_x, - padding_y, total_padding_x, total_padding_y) diff --git a/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters_test.py b/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters_test.py deleted file mode 100644 index ec34ea22e62..00000000000 --- a/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters_test.py +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for parse_layer_parameters module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized - -from tensorflow.contrib import slim -from tensorflow.contrib.receptive_field.python.util import graph_compute_order -from tensorflow.contrib.receptive_field.python.util import parse_layer_parameters -from tensorflow.python.client import session -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import graph_util -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def create_test_network(placeholder_resolution, convert_variables_to_constants): - """Convolutional neural network for test. - - Args: - placeholder_resolution: Resolution to use for input placeholder. Used for - height and width dimensions. - convert_variables_to_constants: Whether to convert variables to constants. - - Returns: - name_to_node: Dict keyed by node name, each entry containing the node's - NodeDef. - """ - g = ops.Graph() - sess = session.Session(graph=g) - with g.as_default(): - # An input test image with unknown spatial resolution. - x = array_ops.placeholder( - dtypes.float32, (1, placeholder_resolution, placeholder_resolution, 1), - name='input_image') - # Left branch before first addition. - l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID') - # Right branch before first addition. - l2_pad = array_ops.pad(x, [[0, 0], [1, 0], [1, 0], [0, 0]], name='L2_pad') - l2 = slim.conv2d(l2_pad, 1, [3, 3], stride=2, scope='L2', padding='VALID') - l3 = slim.max_pool2d(l2, [3, 3], stride=2, scope='L3', padding='SAME') - # First addition. - l4 = nn.relu(l1 + l3, name='L4_relu') - # Left branch after first addition. - l5 = slim.conv2d(l4, 1, [1, 1], stride=2, scope='L5', padding='SAME') - # Right branch after first addition. - l6 = slim.conv2d(l4, 1, [3, 3], stride=2, scope='L6', padding='SAME') - # Final addition. - gen_math_ops.add(l5, l6, name='L7_add') - - if convert_variables_to_constants: - sess.run(variables.global_variables_initializer()) - graph_def = graph_util.convert_variables_to_constants( - sess, g.as_graph_def(), ['L7_add']) - else: - graph_def = g.as_graph_def() - - name_to_node = graph_compute_order.parse_graph_nodes(graph_def) - return name_to_node - - -class ParseLayerParametersTest(test.TestCase, parameterized.TestCase): - - @parameterized.named_parameters(('NonePlaceholder', None, False), - ('224Placeholder', 224, False), - ('NonePlaceholderVarAsConst', None, True), - ('224PlaceholderVarAsConst', 224, True)) - def testParametersAreParsedCorrectly(self, placeholder_resolution, - convert_variables_to_constants): - """Checks parameters from create_test_network() are parsed correctly.""" - name_to_node = create_test_network(placeholder_resolution, - convert_variables_to_constants) - - # L1. - l1_node_name = 'L1/Conv2D' - l1_params = parse_layer_parameters.get_layer_params( - name_to_node[l1_node_name], name_to_node) - expected_l1_params = (1, 1, 4, 4, 0, 0, 0, 0) - self.assertEqual(l1_params, expected_l1_params) - - # L2 padding. - l2_pad_name = 'L2_pad' - l2_pad_params = parse_layer_parameters.get_layer_params( - name_to_node[l2_pad_name], name_to_node) - expected_l2_pad_params = (1, 1, 1, 1, 1, 1, 1, 1) - self.assertEqual(l2_pad_params, expected_l2_pad_params) - - # L2. - l2_node_name = 'L2/Conv2D' - l2_params = parse_layer_parameters.get_layer_params( - name_to_node[l2_node_name], name_to_node) - expected_l2_params = (3, 3, 2, 2, 0, 0, 0, 0) - self.assertEqual(l2_params, expected_l2_params) - - # L3. - l3_node_name = 'L3/MaxPool' - # - Without knowing input size. - l3_params = parse_layer_parameters.get_layer_params( - name_to_node[l3_node_name], name_to_node) - expected_l3_params = (3, 3, 2, 2, None, None, None, None) - self.assertEqual(l3_params, expected_l3_params) - # - Input size is even. - l3_even_params = parse_layer_parameters.get_layer_params( - name_to_node[l3_node_name], name_to_node, input_resolution=[4, 4]) - expected_l3_even_params = (3, 3, 2, 2, 0, 0, 1, 1) - self.assertEqual(l3_even_params, expected_l3_even_params) - # - Input size is odd. - l3_odd_params = parse_layer_parameters.get_layer_params( - name_to_node[l3_node_name], name_to_node, input_resolution=[5, 5]) - expected_l3_odd_params = (3, 3, 2, 2, 1, 1, 2, 2) - self.assertEqual(l3_odd_params, expected_l3_odd_params) - - # L4. - l4_node_name = 'L4_relu' - l4_params = parse_layer_parameters.get_layer_params( - name_to_node[l4_node_name], name_to_node) - expected_l4_params = (1, 1, 1, 1, 0, 0, 0, 0) - self.assertEqual(l4_params, expected_l4_params) - - # L5. - l5_node_name = 'L5/Conv2D' - l5_params = parse_layer_parameters.get_layer_params( - name_to_node[l5_node_name], name_to_node) - expected_l5_params = (1, 1, 2, 2, 0, 0, 0, 0) - self.assertEqual(l5_params, expected_l5_params) - - # L6. - l6_node_name = 'L6/Conv2D' - # - Without knowing input size. - l6_params = parse_layer_parameters.get_layer_params( - name_to_node[l6_node_name], name_to_node) - expected_l6_params = (3, 3, 2, 2, None, None, None, None) - self.assertEqual(l6_params, expected_l6_params) - # - Input size is even. - l6_even_params = parse_layer_parameters.get_layer_params( - name_to_node[l6_node_name], name_to_node, input_resolution=[4, 4]) - expected_l6_even_params = (3, 3, 2, 2, 0, 0, 1, 1) - self.assertEqual(l6_even_params, expected_l6_even_params) - # - Input size is odd. - l6_odd_params = parse_layer_parameters.get_layer_params( - name_to_node[l6_node_name], name_to_node, input_resolution=[5, 5]) - expected_l6_odd_params = (3, 3, 2, 2, 1, 1, 2, 2) - self.assertEqual(l6_odd_params, expected_l6_odd_params) - - # L7. - l7_node_name = 'L7_add' - l7_params = parse_layer_parameters.get_layer_params( - name_to_node[l7_node_name], name_to_node) - expected_l7_params = (1, 1, 1, 1, 0, 0, 0, 0) - self.assertEqual(l7_params, expected_l7_params) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/receptive_field/python/util/receptive_field.py b/tensorflow/contrib/receptive_field/python/util/receptive_field.py deleted file mode 100644 index 9127c772c75..00000000000 --- a/tensorflow/contrib/receptive_field/python/util/receptive_field.py +++ /dev/null @@ -1,391 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functions to compute receptive field of a fully-convolutional network.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib.receptive_field.python.util import graph_compute_order -from tensorflow.contrib.receptive_field.python.util import parse_layer_parameters -from tensorflow.python.framework import ops as framework_ops -from tensorflow.python.platform import tf_logging as logging - - -def _get_rf_size_node_input(stride, kernel_size, rf_size_output): - """Computes RF size at the input of a given layer. - - Args: - stride: Stride of given layer (integer). - kernel_size: Kernel size of given layer (integer). - rf_size_output: RF size at output of given layer (integer). - - Returns: - rf_size_input: RF size at input of given layer (integer). - """ - return stride * rf_size_output + kernel_size - stride - - -def _get_effective_stride_node_input(stride, effective_stride_output): - """Computes effective stride at the input of a given layer. - - Args: - stride: Stride of given layer (integer). - effective_stride_output: Effective stride at output of given layer - (integer). - - Returns: - effective_stride_input: Effective stride at input of given layer - (integer). - """ - return stride * effective_stride_output - - -def _get_effective_padding_node_input(stride, padding, - effective_padding_output): - """Computes effective padding at the input of a given layer. - - Args: - stride: Stride of given layer (integer). - padding: Padding of given layer (integer). - effective_padding_output: Effective padding at output of given layer - (integer). - - Returns: - effective_padding_input: Effective padding at input of given layer - (integer). - """ - return stride * effective_padding_output + padding - - -class ReceptiveField(object): - """Receptive field of a convolutional neural network. - - Args: - size: Receptive field size. - stride: Effective stride. - padding: Effective padding. - """ - - def __init__(self, size, stride, padding): - self.size = np.asarray(size) - self.stride = np.asarray(stride) - self.padding = np.asarray(padding) - - def compute_input_center_coordinates(self, y, axis=None): - """Computes the center of the receptive field that generated a feature. - - Args: - y: An array of feature coordinates with shape `(..., d)`, where `d` is the - number of dimensions of the coordinates. - axis: The dimensions for which to compute the input center coordinates. If - `None` (the default), compute the input center coordinates for all - dimensions. - - Returns: - x: Center of the receptive field that generated the features, at the input - of the network. - - Raises: - ValueError: If the number of dimensions of the feature coordinates does - not match the number of elements in `axis`. - """ - # Use all dimensions. - if axis is None: - axis = range(self.size.size) - # Ensure axis is a list because tuples have different indexing behavior. - axis = list(axis) - y = np.asarray(y) - if y.shape[-1] != len(axis): - raise ValueError("Dimensionality of the feature coordinates `y` (%d) " - "does not match dimensionality of `axis` (%d)" % - (y.shape[-1], len(axis))) - return -self.padding[axis] + y * self.stride[axis] + ( - self.size[axis] - 1) / 2 - - def compute_feature_coordinates(self, x, axis=None): - """Computes the position of a feature given the center of a receptive field. - - Args: - x: An array of input center coordinates with shape `(..., d)`, where `d` - is the number of dimensions of the coordinates. - axis: The dimensions for which to compute the feature coordinates. If - `None` (the default), compute the feature coordinates for all - dimensions. - - Returns: - y: Coordinates of the features. - - Raises: - ValueError: If the number of dimensions of the input center coordinates - does not match the number of elements in `axis`. - """ - # Use all dimensions. - if axis is None: - axis = range(self.size.size) - # Ensure axis is a list because tuples have different indexing behavior. - axis = list(axis) - x = np.asarray(x) - if x.shape[-1] != len(axis): - raise ValueError("Dimensionality of the input center coordinates `x` " - "(%d) does not match dimensionality of `axis` (%d)" % - (x.shape[-1], len(axis))) - return (x + self.padding[axis] + - (1 - self.size[axis]) / 2) / self.stride[axis] - - def __iter__(self): - return iter(np.concatenate([self.size, self.stride, self.padding])) - - -def compute_receptive_field_from_graph_def(graph_def, - input_node, - output_node, - stop_propagation=None, - input_resolution=None): - """Computes receptive field (RF) parameters from a Graph or GraphDef object. - - The algorithm stops the calculation of the receptive field whenever it - encounters an operation in the list `stop_propagation`. Stopping the - calculation early can be useful to calculate the receptive field of a - subgraph such as a single branch of the - [inception network](https://arxiv.org/abs/1512.00567). - - Args: - graph_def: Graph or GraphDef object. - input_node: Name of the input node or Tensor object from graph. - output_node: Name of the output node or Tensor object from graph. - stop_propagation: List of operations or scope names for which to stop the - propagation of the receptive field. - input_resolution: 2D list. If the input resolution to the model is fixed and - known, this may be set. This is helpful for cases where the RF parameters - vary depending on the input resolution (this happens since SAME padding in - tensorflow depends on input resolution in general). If this is None, it is - assumed that the input resolution is unknown, so some RF parameters may be - unknown (depending on the model architecture). - - Returns: - rf_size_x: Receptive field size of network in the horizontal direction, with - respect to specified input and output. - rf_size_y: Receptive field size of network in the vertical direction, with - respect to specified input and output. - effective_stride_x: Effective stride of network in the horizontal direction, - with respect to specified input and output. - effective_stride_y: Effective stride of network in the vertical direction, - with respect to specified input and output. - effective_padding_x: Effective padding of network in the horizontal - direction, with respect to specified input and output. - effective_padding_y: Effective padding of network in the vertical - direction, with respect to specified input and output. - - Raises: - ValueError: If network is not aligned or if either input or output nodes - cannot be found. For network criterion alignment, see - photos/vision/features/delf/g3doc/rf_computation.md - """ - # Convert a graph to graph_def if necessary. - if isinstance(graph_def, framework_ops.Graph): - graph_def = graph_def.as_graph_def() - - # Convert tensors to names. - if isinstance(input_node, framework_ops.Tensor): - input_node = input_node.op.name - if isinstance(output_node, framework_ops.Tensor): - output_node = output_node.op.name - - stop_propagation = stop_propagation or [] - - # Computes order of computation for a given graph. - node_info, name_to_node = graph_compute_order.get_compute_order( - graph_def=graph_def, - input_node_name=input_node, - input_node_size=input_resolution) - - # Sort in reverse topological order. - ordered_node_info = sorted(node_info.items(), key=lambda x: -x[1].order) - - # Dictionaries to keep track of receptive field, effective stride and - # effective padding of different nodes. - rf_sizes_x = {} - rf_sizes_y = {} - effective_strides_x = {} - effective_strides_y = {} - effective_paddings_x = {} - effective_paddings_y = {} - - # Initialize dicts for output_node. - rf_sizes_x[output_node] = 1 - rf_sizes_y[output_node] = 1 - effective_strides_x[output_node] = 1 - effective_strides_y[output_node] = 1 - effective_paddings_x[output_node] = 0 - effective_paddings_y[output_node] = 0 - - # Flag to denote if we found output node yet. If we have not, we skip nodes - # until the output node is found. - found_output_node = False - - # Flag to denote if padding is undefined. This happens when SAME padding mode - # is used in conjunction with stride and kernel sizes which make it such that - # the padding to be applied would depend on the input size. In this case, - # alignment checks are skipped, and the effective padding is None. - undefined_padding = False - - for _, (o, node, _, _) in ordered_node_info: - if node: - logging.vlog(3, "%10d %-100s %-20s" % (o, node.name[:90], node.op)) - else: - continue - - # When we find input node, we can stop. - if node.name == input_node: - break - - # Loop until we find the output node. All nodes before finding the output - # one are irrelevant, so they can be skipped. - if not found_output_node: - if node.name == output_node: - found_output_node = True - - if found_output_node: - if node.name not in rf_sizes_x: - assert node.name not in rf_sizes_y, ("Node %s is in rf_sizes_y, but " - "not in rf_sizes_x" % node.name) - # In this case, node is not relevant since it's not part of the - # computation we're interested in. - logging.vlog(3, "Irrelevant node %s, skipping it...", node.name) - continue - - # Get params for this layer. - (kernel_size_x, kernel_size_y, stride_x, stride_y, padding_x, padding_y, - _, _) = parse_layer_parameters.get_layer_params( - node, name_to_node, node_info[node.name].input_size) - logging.vlog( - 3, "kernel_size_x = %s, kernel_size_y = %s, " - "stride_x = %s, stride_y = %s, " - "padding_x = %s, padding_y = %s, input size = %s" % - (kernel_size_x, kernel_size_y, stride_x, stride_y, padding_x, - padding_y, node_info[node.name].input_size)) - if padding_x is None or padding_y is None: - undefined_padding = True - - # Get parameters at input of this layer which may or may not be propagated - # to the input layers. - rf_size_input_x = _get_rf_size_node_input(stride_x, kernel_size_x, - rf_sizes_x[node.name]) - rf_size_input_y = _get_rf_size_node_input(stride_y, kernel_size_y, - rf_sizes_y[node.name]) - effective_stride_input_x = _get_effective_stride_node_input( - stride_x, effective_strides_x[node.name]) - effective_stride_input_y = _get_effective_stride_node_input( - stride_y, effective_strides_y[node.name]) - if not undefined_padding: - effective_padding_input_x = _get_effective_padding_node_input( - stride_x, padding_x, effective_paddings_x[node.name]) - effective_padding_input_y = _get_effective_padding_node_input( - stride_y, padding_y, effective_paddings_y[node.name]) - else: - effective_padding_input_x = None - effective_padding_input_y = None - logging.vlog( - 4, "rf_size_input_x = %s, rf_size_input_y = %s, " - "effective_stride_input_x = %s, effective_stride_input_y = %s, " - "effective_padding_input_x = %s, effective_padding_input_y = %s" % - (rf_size_input_x, rf_size_input_y, effective_stride_input_x, - effective_stride_input_y, effective_padding_input_x, - effective_padding_input_y)) - - # Loop over this node's inputs and potentially propagate information down. - for inp_name in node.input: - # Stop the propagation of the receptive field. - if any(inp_name.startswith(stop) for stop in stop_propagation): - logging.vlog(3, "Skipping explicitly ignored node %s.", inp_name) - continue - - logging.vlog(4, "inp_name = %s", inp_name) - if inp_name.startswith("^"): - # The character "^" denotes a control dependency, so this input node - # can be safely ignored. - continue - - inp_node = name_to_node[inp_name] - logging.vlog(4, "inp_node = \n%s", inp_node) - if inp_name in rf_sizes_x: - assert inp_name in rf_sizes_y, ("Node %s is in rf_sizes_x, but " - "not in rf_sizes_y" % inp_name) - logging.vlog( - 4, "rf_sizes_x[inp_name] = %s," - " rf_sizes_y[inp_name] = %s, " - "effective_strides_x[inp_name] = %s," - " effective_strides_y[inp_name] = %s, " - "effective_paddings_x[inp_name] = %s," - " effective_paddings_y[inp_name] = %s" % - (rf_sizes_x[inp_name], rf_sizes_y[inp_name], - effective_strides_x[inp_name], effective_strides_y[inp_name], - effective_paddings_x[inp_name], effective_paddings_y[inp_name])) - # This node was already discovered through a previous path, so we need - # to make sure that graph is aligned. This alignment check is skipped - # if the padding is not defined, since in this case alignment cannot - # be checked. - if not undefined_padding: - if effective_strides_x[inp_name] != effective_stride_input_x: - raise ValueError( - "Graph is not aligned since effective stride from different " - "paths is different in horizontal direction") - if effective_strides_y[inp_name] != effective_stride_input_y: - raise ValueError( - "Graph is not aligned since effective stride from different " - "paths is different in vertical direction") - if (rf_sizes_x[inp_name] - - 1) / 2 - effective_paddings_x[inp_name] != ( - rf_size_input_x - 1) / 2 - effective_padding_input_x: - raise ValueError( - "Graph is not aligned since center shift from different " - "paths is different in horizontal direction") - if (rf_sizes_y[inp_name] - - 1) / 2 - effective_paddings_y[inp_name] != ( - rf_size_input_y - 1) / 2 - effective_padding_input_y: - raise ValueError( - "Graph is not aligned since center shift from different " - "paths is different in vertical direction") - # Keep track of path with largest RF, for both directions. - if rf_sizes_x[inp_name] < rf_size_input_x: - rf_sizes_x[inp_name] = rf_size_input_x - effective_strides_x[inp_name] = effective_stride_input_x - effective_paddings_x[inp_name] = effective_padding_input_x - if rf_sizes_y[inp_name] < rf_size_input_y: - rf_sizes_y[inp_name] = rf_size_input_y - effective_strides_y[inp_name] = effective_stride_input_y - effective_paddings_y[inp_name] = effective_padding_input_y - else: - assert inp_name not in rf_sizes_y, ("Node %s is in rf_sizes_y, but " - "not in rf_sizes_x" % inp_name) - # In this case, it is the first time we encounter this node. So we - # propagate the RF parameters. - rf_sizes_x[inp_name] = rf_size_input_x - rf_sizes_y[inp_name] = rf_size_input_y - effective_strides_x[inp_name] = effective_stride_input_x - effective_strides_y[inp_name] = effective_stride_input_y - effective_paddings_x[inp_name] = effective_padding_input_x - effective_paddings_y[inp_name] = effective_padding_input_y - - if not found_output_node: - raise ValueError("Output node was not found") - if input_node not in rf_sizes_x: - raise ValueError("Input node was not found") - return ReceptiveField( - (rf_sizes_x[input_node], rf_sizes_y[input_node]), - (effective_strides_x[input_node], effective_strides_y[input_node]), - (effective_paddings_x[input_node], effective_paddings_y[input_node])) diff --git a/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py b/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py deleted file mode 100644 index a42bbca6113..00000000000 --- a/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py +++ /dev/null @@ -1,457 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for receptive_fields module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib import slim -from tensorflow.contrib.receptive_field import receptive_field_api as receptive_field -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import nn -from tensorflow.python.platform import test - - -# TODO(andrearaujo): Rename the create_test_network_* functions in order to have -# more descriptive names. -def create_test_network_1(): - """Aligned network for test. - - The graph corresponds to the example from the second figure in - go/cnn-rf-computation#arbitrary-computation-graphs - - Returns: - g: Tensorflow graph object (Graph proto). - """ - g = ops.Graph() - with g.as_default(): - # An input test image with unknown spatial resolution. - x = array_ops.placeholder( - dtypes.float32, (None, None, None, 1), name='input_image') - # Left branch. - l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID') - # Right branch. - l2_pad = array_ops.pad(x, [[0, 0], [1, 0], [1, 0], [0, 0]]) - l2 = slim.conv2d(l2_pad, 1, [3, 3], stride=2, scope='L2', padding='VALID') - l3 = slim.conv2d(l2, 1, [1, 1], stride=2, scope='L3', padding='VALID') - # Addition. - nn.relu(l1 + l3, name='output') - return g - - -def create_test_network_2(): - """Aligned network for test. - - The graph corresponds to a variation to the example from the second figure in - go/cnn-rf-computation#arbitrary-computation-graphs. Layers 2 and 3 are changed - to max-pooling operations. Since the functionality is the same as convolution, - the network is aligned and the receptive field size is the same as from the - network created using create_test_network_1(). - - Returns: - g: Tensorflow graph object (Graph proto). - """ - g = ops.Graph() - with g.as_default(): - # An input test image with unknown spatial resolution. - x = array_ops.placeholder( - dtypes.float32, (None, None, None, 1), name='input_image') - # Left branch. - l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID') - # Right branch. - l2_pad = array_ops.pad(x, [[0, 0], [1, 0], [1, 0], [0, 0]]) - l2 = slim.max_pool2d(l2_pad, [3, 3], stride=2, scope='L2', padding='VALID') - l3 = slim.max_pool2d(l2, [1, 1], stride=2, scope='L3', padding='VALID') - # Addition. - nn.relu(l1 + l3, name='output') - return g - - -def create_test_network_3(): - """Misaligned network for test. - - The graph corresponds to the example from the first figure in - go/cnn-rf-computation#arbitrary-computation-graphs - - Returns: - g: Tensorflow graph object (Graph proto). - """ - g = ops.Graph() - with g.as_default(): - # An input test image with unknown spatial resolution. - x = array_ops.placeholder( - dtypes.float32, (None, None, None, 1), name='input_image') - # Left branch. - l1_pad = array_ops.pad(x, [[0, 0], [2, 1], [2, 1], [0, 0]]) - l1 = slim.conv2d(l1_pad, 1, [5, 5], stride=2, scope='L1', padding='VALID') - # Right branch. - l2 = slim.conv2d(x, 1, [3, 3], stride=1, scope='L2', padding='VALID') - l3 = slim.conv2d(l2, 1, [3, 3], stride=1, scope='L3', padding='VALID') - # Addition. - nn.relu(l1 + l3, name='output') - return g - - -def create_test_network_4(): - """Misaligned network for test. - - The graph corresponds to a variation from the example from the second figure - in go/cnn-rf-computation#arbitrary-computation-graphs. Layer 2 uses 'SAME' - padding, which makes its padding dependent on the input image dimensionality. - In this case, the effective padding will be undetermined, and the utility is - not able to check the network alignment. - - Returns: - g: Tensorflow graph object (Graph proto). - """ - g = ops.Graph() - with g.as_default(): - # An input test image with unknown spatial resolution. - x = array_ops.placeholder( - dtypes.float32, (None, None, None, 1), name='input_image') - # Left branch. - l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID') - # Right branch. - l2 = slim.conv2d(x, 1, [3, 3], stride=2, scope='L2', padding='SAME') - l3 = slim.conv2d(l2, 1, [1, 1], stride=2, scope='L3', padding='VALID') - # Addition. - nn.relu(l1 + l3, name='output') - return g - - -def create_test_network_5(): - """Single-path network for testing non-square kernels. - - The graph is similar to the right branch of the graph from - create_test_network_1(), except that the kernel sizes are changed to be - non-square. - - Returns: - g: Tensorflow graph object (Graph proto). - """ - g = ops.Graph() - with g.as_default(): - # An input test image with unknown spatial resolution. - x = array_ops.placeholder( - dtypes.float32, (None, None, None, 1), name='input_image') - # Two convolutional layers, where the first one has non-square kernel. - l1 = slim.conv2d(x, 1, [3, 5], stride=2, scope='L1', padding='VALID') - l2 = slim.conv2d(l1, 1, [3, 1], stride=2, scope='L2', padding='VALID') - # ReLU. - nn.relu(l2, name='output') - return g - - -def create_test_network_6(): - """Aligned network with dropout for test. - - The graph is similar to create_test_network_1(), except that the right branch - has dropout normalization. - - Returns: - g: Tensorflow graph object (Graph proto). - """ - g = ops.Graph() - with g.as_default(): - # An input test image with unknown spatial resolution. - x = array_ops.placeholder( - dtypes.float32, (None, None, None, 1), name='input_image') - # Left branch. - l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID') - # Right branch. - l2_pad = array_ops.pad(x, [[0, 0], [1, 0], [1, 0], [0, 0]]) - l2 = slim.conv2d(l2_pad, 1, [3, 3], stride=2, scope='L2', padding='VALID') - l3 = slim.conv2d(l2, 1, [1, 1], stride=2, scope='L3', padding='VALID') - dropout = slim.dropout(l3) - # Addition. - nn.relu(l1 + dropout, name='output') - return g - - -def create_test_network_7(): - """Aligned network for test, with a control dependency. - - The graph is similar to create_test_network_1(), except that it includes an - assert operation on the left branch. - - Returns: - g: Tensorflow graph object (Graph proto). - """ - g = ops.Graph() - with g.as_default(): - # An 8x8 test image. - x = array_ops.placeholder(dtypes.float32, (1, 8, 8, 1), name='input_image') - # Left branch. - l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID') - l1_shape = array_ops.shape(l1) - assert_op = control_flow_ops.Assert( - gen_math_ops.equal(l1_shape[1], 2), [l1_shape], summarize=4) - # Right branch. - l2_pad = array_ops.pad(x, [[0, 0], [1, 0], [1, 0], [0, 0]]) - l2 = slim.conv2d(l2_pad, 1, [3, 3], stride=2, scope='L2', padding='VALID') - l3 = slim.conv2d(l2, 1, [1, 1], stride=2, scope='L3', padding='VALID') - # Addition. - with ops.control_dependencies([assert_op]): - nn.relu(l1 + l3, name='output') - return g - - -def create_test_network_8(): - """Aligned network for test, including an intermediate addition. - - The graph is similar to create_test_network_1(), except that it includes a few - more layers on top. The added layers compose two different branches whose - receptive fields are different. This makes this test case more challenging; in - particular, this test fails if a naive DFS-like algorithm is used for RF - computation. - - Returns: - g: Tensorflow graph object (Graph proto). - """ - g = ops.Graph() - with g.as_default(): - # An input test image with unknown spatial resolution. - x = array_ops.placeholder( - dtypes.float32, (None, None, None, 1), name='input_image') - # Left branch before first addition. - l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID') - # Right branch before first addition. - l2_pad = array_ops.pad(x, [[0, 0], [1, 0], [1, 0], [0, 0]]) - l2 = slim.conv2d(l2_pad, 1, [3, 3], stride=2, scope='L2', padding='VALID') - l3 = slim.conv2d(l2, 1, [1, 1], stride=2, scope='L3', padding='VALID') - # First addition. - l4 = nn.relu(l1 + l3) - # Left branch after first addition. - l5 = slim.conv2d(l4, 1, [1, 1], stride=2, scope='L5', padding='VALID') - # Right branch after first addition. - l6_pad = array_ops.pad(l4, [[0, 0], [1, 0], [1, 0], [0, 0]]) - l6 = slim.conv2d(l6_pad, 1, [3, 3], stride=2, scope='L6', padding='VALID') - # Final addition. - nn.relu(l5 + l6, name='output') - - return g - - -def create_test_network_9(): - """Aligned network for test, including an intermediate addition. - - The graph is the same as create_test_network_8(), except that VALID padding is - changed to SAME. - - Returns: - g: Tensorflow graph object (Graph proto). - """ - g = ops.Graph() - with g.as_default(): - # An input test image with unknown spatial resolution. - x = array_ops.placeholder( - dtypes.float32, (None, None, None, 1), name='input_image') - # Left branch before first addition. - l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='SAME') - # Right branch before first addition. - l2 = slim.conv2d(x, 1, [3, 3], stride=2, scope='L2', padding='SAME') - l3 = slim.conv2d(l2, 1, [1, 1], stride=2, scope='L3', padding='SAME') - # First addition. - l4 = nn.relu(l1 + l3) - # Left branch after first addition. - l5 = slim.conv2d(l4, 1, [1, 1], stride=2, scope='L5', padding='SAME') - # Right branch after first addition. - l6 = slim.conv2d(l4, 1, [3, 3], stride=2, scope='L6', padding='SAME') - # Final addition. - nn.relu(l5 + l6, name='output') - - return g - - -class ReceptiveFieldTest(test.TestCase): - - def testComputeRFFromGraphDefAligned(self): - graph_def = create_test_network_1().as_graph_def() - input_node = 'input_image' - output_node = 'output' - (receptive_field_x, receptive_field_y, effective_stride_x, - effective_stride_y, effective_padding_x, effective_padding_y) = ( - receptive_field.compute_receptive_field_from_graph_def( - graph_def, input_node, output_node)) - self.assertEqual(receptive_field_x, 3) - self.assertEqual(receptive_field_y, 3) - self.assertEqual(effective_stride_x, 4) - self.assertEqual(effective_stride_y, 4) - self.assertEqual(effective_padding_x, 1) - self.assertEqual(effective_padding_y, 1) - - def testComputeRFFromGraphDefAligned2(self): - graph_def = create_test_network_2().as_graph_def() - input_node = 'input_image' - output_node = 'output' - (receptive_field_x, receptive_field_y, effective_stride_x, - effective_stride_y, effective_padding_x, effective_padding_y) = ( - receptive_field.compute_receptive_field_from_graph_def( - graph_def, input_node, output_node)) - self.assertEqual(receptive_field_x, 3) - self.assertEqual(receptive_field_y, 3) - self.assertEqual(effective_stride_x, 4) - self.assertEqual(effective_stride_y, 4) - self.assertEqual(effective_padding_x, 1) - self.assertEqual(effective_padding_y, 1) - - def testComputeRFFromGraphDefUnaligned(self): - graph_def = create_test_network_3().as_graph_def() - input_node = 'input_image' - output_node = 'output' - with self.assertRaises(ValueError): - receptive_field.compute_receptive_field_from_graph_def( - graph_def, input_node, output_node) - - def testComputeRFFromGraphDefUndefinedPadding(self): - graph_def = create_test_network_4().as_graph_def() - input_node = 'input_image' - output_node = 'output' - (receptive_field_x, receptive_field_y, effective_stride_x, - effective_stride_y, effective_padding_x, effective_padding_y) = ( - receptive_field.compute_receptive_field_from_graph_def( - graph_def, input_node, output_node)) - self.assertEqual(receptive_field_x, 3) - self.assertEqual(receptive_field_y, 3) - self.assertEqual(effective_stride_x, 4) - self.assertEqual(effective_stride_y, 4) - self.assertEqual(effective_padding_x, None) - self.assertEqual(effective_padding_y, None) - - def testComputeRFFromGraphDefFixedInputDim(self): - graph_def = create_test_network_4().as_graph_def() - input_node = 'input_image' - output_node = 'output' - (receptive_field_x, receptive_field_y, effective_stride_x, - effective_stride_y, effective_padding_x, effective_padding_y) = ( - receptive_field.compute_receptive_field_from_graph_def( - graph_def, input_node, output_node, input_resolution=[9, 9])) - self.assertEqual(receptive_field_x, 3) - self.assertEqual(receptive_field_y, 3) - self.assertEqual(effective_stride_x, 4) - self.assertEqual(effective_stride_y, 4) - self.assertEqual(effective_padding_x, 1) - self.assertEqual(effective_padding_y, 1) - - def testComputeRFFromGraphDefUnalignedFixedInputDim(self): - graph_def = create_test_network_4().as_graph_def() - input_node = 'input_image' - output_node = 'output' - with self.assertRaises(ValueError): - receptive_field.compute_receptive_field_from_graph_def( - graph_def, input_node, output_node, input_resolution=[8, 8]) - - def testComputeRFFromGraphDefNonSquareRF(self): - graph_def = create_test_network_5().as_graph_def() - input_node = 'input_image' - output_node = 'output' - (receptive_field_x, receptive_field_y, effective_stride_x, - effective_stride_y, effective_padding_x, effective_padding_y) = ( - receptive_field.compute_receptive_field_from_graph_def( - graph_def, input_node, output_node)) - self.assertEqual(receptive_field_x, 5) - self.assertEqual(receptive_field_y, 7) - self.assertEqual(effective_stride_x, 4) - self.assertEqual(effective_stride_y, 4) - self.assertEqual(effective_padding_x, 0) - self.assertEqual(effective_padding_y, 0) - - def testComputeRFFromGraphDefStopPropagation(self): - graph_def = create_test_network_6().as_graph_def() - input_node = 'input_image' - output_node = 'output' - # Compute the receptive field but stop the propagation for the random - # uniform variable of the dropout. - (receptive_field_x, receptive_field_y, effective_stride_x, - effective_stride_y, effective_padding_x, effective_padding_y) = ( - receptive_field.compute_receptive_field_from_graph_def( - graph_def, input_node, output_node, - ['Dropout/dropout_1/random_uniform'])) - self.assertEqual(receptive_field_x, 3) - self.assertEqual(receptive_field_y, 3) - self.assertEqual(effective_stride_x, 4) - self.assertEqual(effective_stride_y, 4) - self.assertEqual(effective_padding_x, 1) - self.assertEqual(effective_padding_y, 1) - - def testComputeCoordinatesRoundtrip(self): - graph_def = create_test_network_1() - input_node = 'input_image' - output_node = 'output' - rf = receptive_field.compute_receptive_field_from_graph_def( - graph_def, input_node, output_node) - - x = np.random.randint(0, 100, (50, 2)) - y = rf.compute_feature_coordinates(x) - x2 = rf.compute_input_center_coordinates(y) - - self.assertAllEqual(x, x2) - - def testComputeRFFromGraphDefAlignedWithControlDependencies(self): - graph_def = create_test_network_7().as_graph_def() - input_node = 'input_image' - output_node = 'output' - (receptive_field_x, receptive_field_y, effective_stride_x, - effective_stride_y, effective_padding_x, effective_padding_y) = ( - receptive_field.compute_receptive_field_from_graph_def( - graph_def, input_node, output_node)) - self.assertEqual(receptive_field_x, 3) - self.assertEqual(receptive_field_y, 3) - self.assertEqual(effective_stride_x, 4) - self.assertEqual(effective_stride_y, 4) - self.assertEqual(effective_padding_x, 1) - self.assertEqual(effective_padding_y, 1) - - def testComputeRFFromGraphDefWithIntermediateAddNode(self): - graph_def = create_test_network_8().as_graph_def() - input_node = 'input_image' - output_node = 'output' - (receptive_field_x, receptive_field_y, effective_stride_x, - effective_stride_y, effective_padding_x, effective_padding_y) = ( - receptive_field.compute_receptive_field_from_graph_def( - graph_def, input_node, output_node)) - self.assertEqual(receptive_field_x, 11) - self.assertEqual(receptive_field_y, 11) - self.assertEqual(effective_stride_x, 8) - self.assertEqual(effective_stride_y, 8) - self.assertEqual(effective_padding_x, 5) - self.assertEqual(effective_padding_y, 5) - - def testComputeRFFromGraphDefWithIntermediateAddNodeSamePaddingFixedInputDim( - self): - graph_def = create_test_network_9().as_graph_def() - input_node = 'input_image' - output_node = 'output' - (receptive_field_x, receptive_field_y, effective_stride_x, - effective_stride_y, effective_padding_x, effective_padding_y) = ( - receptive_field.compute_receptive_field_from_graph_def( - graph_def, input_node, output_node, input_resolution=[17, 17])) - self.assertEqual(receptive_field_x, 11) - self.assertEqual(receptive_field_y, 11) - self.assertEqual(effective_stride_x, 8) - self.assertEqual(effective_stride_y, 8) - self.assertEqual(effective_padding_x, 5) - self.assertEqual(effective_padding_y, 5) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/receptive_field/receptive_field_api.py b/tensorflow/contrib/receptive_field/receptive_field_api.py deleted file mode 100644 index 4d81b4292df..00000000000 --- a/tensorflow/contrib/receptive_field/receptive_field_api.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Module that declares the functions in tf.contrib.receptive_field's API.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.contrib.receptive_field.python.util.graph_compute_order import get_compute_order -from tensorflow.contrib.receptive_field.python.util.receptive_field import compute_receptive_field_from_graph_def -# pylint: enable=unused-import - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/recurrent/BUILD b/tensorflow/contrib/recurrent/BUILD deleted file mode 100644 index 598fea5459e..00000000000 --- a/tensorflow/contrib/recurrent/BUILD +++ /dev/null @@ -1,110 +0,0 @@ -# Recurrent library. - -load("//tensorflow:tensorflow.bzl", "cuda_py_tests") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "recurrent_py", - srcs = ["python/recurrent_api.py"], - srcs_version = "PY2AND3", - deps = [ - ":functional_rnn_ops_py", - ":recurrent_ops_py", - ], -) - -py_library( - name = "recurrent_ops_py", - srcs = ["python/ops/recurrent.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:function", - "//tensorflow/python:functional_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - ], -) - -py_library( - name = "functional_rnn_ops_py", - srcs = ["python/ops/functional_rnn.py"], - srcs_version = "PY2AND3", - deps = [ - ":recurrent_ops_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:function", - "//tensorflow/python:gradients", - "//tensorflow/python:math_ops", - "//tensorflow/python:standard_ops", - ], -) - -cuda_py_tests( - name = "recurrent_ops_test", - size = "small", - srcs = ["python/kernel_tests/recurrent_test.py"], - additional_deps = [ - ":recurrent_ops_py", - "//third_party/py/numpy", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:script_ops", - "//tensorflow/python:variables", - ], - tags = ["nopip"], -) - -cuda_py_tests( - name = "functional_rnn_ops_test", - size = "small", - srcs = ["python/kernel_tests/functional_rnn_test.py"], - additional_deps = [ - ":functional_rnn_ops_py", - "//third_party/py/numpy", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/tpu:tpu", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:init_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:rnn", - "//tensorflow/python:rnn_cell", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], - tags = [ - "nopip", - "optonly", - ], -) diff --git a/tensorflow/contrib/recurrent/README.md b/tensorflow/contrib/recurrent/README.md deleted file mode 100644 index 86e10eee517..00000000000 --- a/tensorflow/contrib/recurrent/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Recurrent computation library - -The recurrent computation library contains code to perform recurrent -computations. - -Its chief application is to implement recurrent neural networks (RNNs, LSTMs, -etc), which is implemented in `functional_rnn.py`. Similar techniques may be -used to implement deep networks. - -The computation saves the activations in the forward pass, and computes the -gradients in the backward pass using a single accumulator. - -The `functional_rnn` interface is compatible with the `dynamic_rnn` API. diff --git a/tensorflow/contrib/recurrent/python/kernel_tests/functional_rnn_test.py b/tensorflow/contrib/recurrent/python/kernel_tests/functional_rnn_test.py deleted file mode 100644 index 1800edc05ae..00000000000 --- a/tensorflow/contrib/recurrent/python/kernel_tests/functional_rnn_test.py +++ /dev/null @@ -1,289 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Functional RNN.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - - -from tensorflow.contrib.recurrent.python.ops import functional_rnn -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import rnn as rnn_lib -from tensorflow.python.ops import rnn_cell_impl -from tensorflow.python.ops import variables -import tensorflow.python.ops.nn_grad # pylint: disable=unused-import -import tensorflow.python.ops.tensor_array_grad # pylint: disable=unused-import -from tensorflow.python.platform import test as test_lib -from tensorflow.python.platform import tf_logging as logging - - -def _CreateStackedLstmCell(*cell_sizes): - subcells = [rnn_cell_impl.LSTMCell(cell_size) for cell_size in cell_sizes] - return rnn_cell_impl.MultiRNNCell(subcells) - - -class FunctionalRnnTest(test_util.TensorFlowTestCase): - - _BATCH_SIZE = 3 - _TOTAL_TIME = 5 - _INPUT_SIZE = 11 - _NUM_UNITS = 7 - - # Set this to some output if you want to use it. - _LSTM_GRAPH_DEF_FILEPATH = None - - _CELLDEFS = { - 'gru': (rnn_cell_impl.GRUCell, [_NUM_UNITS]), - 'lstm': (rnn_cell_impl.LSTMCell, [_NUM_UNITS]), - 'stacked_lstm': (_CreateStackedLstmCell, [_NUM_UNITS] * 3) - } - - def _CreateCell(self, celldef_name): - func, args = self._CELLDEFS[celldef_name] - return func(*args) - - def _CreateInputs(self, time_major=False): - if time_major: - inputs = np.random.random([ - FunctionalRnnTest._TOTAL_TIME, FunctionalRnnTest._BATCH_SIZE, - FunctionalRnnTest._INPUT_SIZE - ]) - else: - inputs = np.random.random([ - FunctionalRnnTest._BATCH_SIZE, FunctionalRnnTest._TOTAL_TIME, - FunctionalRnnTest._INPUT_SIZE - ]) - # Always leave one time slot empty, to check max_length behavior. - sequence_length = np.random.randint( - 0, high=FunctionalRnnTest._TOTAL_TIME - 1, - size=FunctionalRnnTest._BATCH_SIZE, - dtype=np.int) - return (inputs, sequence_length) - - def _CreateSymmetricInputs(self): - # total time = batch size - inputs = np.zeros( - (FunctionalRnnTest._BATCH_SIZE, FunctionalRnnTest._BATCH_SIZE, - FunctionalRnnTest._INPUT_SIZE)) - for i in range(FunctionalRnnTest._BATCH_SIZE): - for j in range(i, FunctionalRnnTest._BATCH_SIZE): - inputs[i][j] = np.random.random([FunctionalRnnTest._INPUT_SIZE]) - inputs[j][i] = inputs[i][j] - - # Always leave one time slot empty, to check max_length behavior. - sequence_length = np.random.randint( - 0, - high=FunctionalRnnTest._BATCH_SIZE - 1, - size=FunctionalRnnTest._BATCH_SIZE, - dtype=np.int) - return (inputs, sequence_length) - - def _CreateRnnGraph(self, - create_rnn_computation_func, - cell, - tf_inputs, - tf_sequence_length, - is_bidirectional, - initial_state=None, - time_major=None, - scope=None): - if is_bidirectional: - tf_result = create_rnn_computation_func( - cell_fw=cell, - cell_bw=cell, - inputs=tf_inputs, - sequence_length=tf_sequence_length, - dtype=dtypes.float32, - time_major=time_major, - scope=scope) - else: - tf_result = create_rnn_computation_func( - cell=cell, - inputs=tf_inputs, - sequence_length=tf_sequence_length, - initial_state=initial_state, - dtype=dtypes.float32, - time_major=time_major, - scope=scope) - grad = gradients_impl.gradients(tf_result, variables.trainable_variables()) - return {'inference': tf_result, 'grad': grad} - - def _MaybeResetVariables(self, variable_cache, sess, var_list): - """Possibly resets the variables to a previously seen value.""" - reset_ops = [] - fetches = [] - for var in var_list: - if var.name in variable_cache: - reset_ops += [var.assign(variable_cache[var.name])] - else: - fetches += [(var.name, var)] - if reset_ops: - sess.run(reset_ops) - if fetches: - val = sess.run(dict(fetches)) - for n, v in val.items(): - assert n not in variable_cache - variable_cache[n] = v - - def _RunRnn(self, numpy_inputs, numpy_slen, cell_name, variable_cache, - is_dynamic, time_major=None, is_bidirectional=False): - with ops.Graph().as_default() as graph: - tf_inputs = array_ops.placeholder( - dtypes.float32, shape=numpy_inputs.shape) - tf_slen = array_ops.placeholder(dtypes.int32) - feeds = {tf_inputs: numpy_inputs, tf_slen: numpy_slen} - cell = self._CreateCell(cell_name) - if is_dynamic: - if is_bidirectional: - fn = rnn_lib.bidirectional_dynamic_rnn - else: - fn = rnn_lib.dynamic_rnn - else: - if is_bidirectional: - fn = functional_rnn.bidirectional_functional_rnn - else: - fn = functional_rnn.functional_rnn - - fetches = self._CreateRnnGraph( - fn, cell, tf_inputs, tf_slen, is_bidirectional, time_major=time_major) - with self.session(graph=graph) as sess: - sess.run(variables.global_variables_initializer()) - # Note that cell.trainable_variables it not always set. - self._MaybeResetVariables(variable_cache, sess, - variables.trainable_variables()) - val = sess.run(fetches, feed_dict=feeds) - graph_def = graph.as_graph_def() - return graph_def, val - - def testRunLstm(self): - """Runs a simple LSTM. Does not check output.""" - np_inputs, np_slen = self._CreateInputs() - var_cache = {} - graphdef, _ = self._RunRnn(np_inputs, np_slen, 'lstm', var_cache, False) - logging.info('graphdef: %s', graphdef) - if self._LSTM_GRAPH_DEF_FILEPATH: - with open(self._LSTM_GRAPH_DEF_FILEPATH, 'w') as f: - f.write(str(graphdef)) - - def testLstm(self): - """Checks an LSTM against the reference implementation.""" - np_inputs, np_slen = self._CreateInputs() - var_cache = {} - _, func_rnn = self._RunRnn(np_inputs, np_slen, 'lstm', var_cache, False) - _, dyn_rnn = self._RunRnn(np_inputs, np_slen, 'lstm', var_cache, True) - self.assertAllClose(dyn_rnn['inference'], func_rnn['inference']) - self.assertAllClose(dyn_rnn['grad'], func_rnn['grad']) - - def testGru(self): - """Checks a GRU cell against the reference implementation.""" - np_inputs, np_slen = self._CreateInputs() - var_cache = {} - _, func_rnn = self._RunRnn(np_inputs, np_slen, 'gru', var_cache, False) - _, dyn_rnn = self._RunRnn(np_inputs, np_slen, 'gru', var_cache, True) - self.assertAllClose(dyn_rnn['inference'], func_rnn['inference']) - self.assertAllClose(dyn_rnn['grad'], func_rnn['grad']) - - def testStackedLstm(self): - """Checks a stacked LSTM cell against the reference implementation.""" - np_inputs, np_slen = self._CreateInputs() - var_cache = {} - args = [np_inputs, np_slen, 'stacked_lstm', var_cache] - _, func_rnn = self._RunRnn(*(args + [False])) - _, dyn_rnn = self._RunRnn(*(args + [True])) - self.assertAllClose(dyn_rnn['inference'], func_rnn['inference']) - self.assertAllClose(dyn_rnn['grad'], func_rnn['grad']) - - def testLstmWithTimeMajorInputs(self): - """Checks an LSTM against the reference implementation, with time_major.""" - time_major = True - np_inputs, np_slen = self._CreateInputs(time_major=True) - var_cache = {} - args = [np_inputs, np_slen, 'lstm', var_cache] - _, func_rnn = self._RunRnn(*(args + [False]), time_major=time_major) - _, dyn_rnn = self._RunRnn(*(args + [True]), time_major=time_major) - self.assertAllClose(dyn_rnn['inference'], func_rnn['inference']) - self.assertAllClose(dyn_rnn['grad'], func_rnn['grad']) - - def testBidirectionalLstmWithTimeMajorInputs(self): - """Checks a bi-directional LSTM with time-major inputs.""" - time_major = True - np_inputs, np_slen = self._CreateInputs(time_major) - var_cache = {} - args = [np_inputs, np_slen, 'lstm', var_cache] - _, func_rnn = self._RunRnn( - *(args + [False]), time_major=time_major, is_bidirectional=True) - _, dyn_rnn = self._RunRnn( - *(args + [True]), time_major=time_major, is_bidirectional=True) - self.assertAllClose(dyn_rnn['inference'], func_rnn['inference']) - # TODO(b/112170761): comment out this line after the bug is fixed. - # self.assertAllClose(dyn_rnn['grad'], func_rnn['grad']) - - def testBidirectionalLstm(self): - """Checks time-major and batch-major rnn produce consistent results.""" - time_major_inputs, np_slen = self._CreateInputs(True) - batch_major_inputs = np.transpose(time_major_inputs, [1, 0, 2]) - var_cache = {} - args = [np_slen, 'lstm', var_cache, False] - _, time_major_rnn = self._RunRnn( - *([time_major_inputs] + args), time_major=True, is_bidirectional=True) - _, batch_major_rnn = self._RunRnn( - *([batch_major_inputs]+ args), time_major=False, is_bidirectional=True) - # Convert the batch-major outputs to be time-major before the comparasion. - outputs, state = batch_major_rnn['inference'] - outputs = [np.transpose(x, [1, 0, 2]) for x in outputs] - batch_major_rnn['inference'] = [outputs, state] - self.assertAllClose(time_major_rnn['inference'], - batch_major_rnn['inference']) - self.assertAllClose(time_major_rnn['grad'], batch_major_rnn['grad']) - - def testBidirectionalLstmWithSymmetricInputs(self): - """Checks a bi-directional LSTM with symmetric inputs. - - time-major and batch-major rnn produce the same result with symmetric - inputs. - """ - np_inputs, np_slen = self._CreateSymmetricInputs() - var_cache = {} - args = [np_inputs, np_slen, 'lstm', var_cache] - _, time_major_func_rnn = self._RunRnn( - *(args + [False]), time_major=True, is_bidirectional=True) - _, batch_major_func_rnn = self._RunRnn( - *(args + [False]), time_major=False, is_bidirectional=True) - _, time_major_dyn_rnn = self._RunRnn( - *(args + [True]), time_major=True, is_bidirectional=True) - _, batch_major_dyn_rnn = self._RunRnn( - *(args + [True]), time_major=False, is_bidirectional=True) - self.assertAllClose(time_major_func_rnn['inference'], - batch_major_func_rnn['inference']) - self.assertAllClose(time_major_func_rnn['grad'], - batch_major_func_rnn['grad']) - self.assertAllClose(time_major_dyn_rnn['inference'], - batch_major_dyn_rnn['inference']) - self.assertAllClose(time_major_dyn_rnn['grad'], batch_major_dyn_rnn['grad']) - self.assertAllClose(time_major_func_rnn['inference'], - batch_major_dyn_rnn['inference']) - self.assertAllClose(time_major_func_rnn['grad'], - batch_major_dyn_rnn['grad']) - - -if __name__ == '__main__': - test_lib.main() diff --git a/tensorflow/contrib/recurrent/python/kernel_tests/recurrent_test.py b/tensorflow/contrib/recurrent/python/kernel_tests/recurrent_test.py deleted file mode 100644 index aea80a5256e..00000000000 --- a/tensorflow/contrib/recurrent/python/kernel_tests/recurrent_test.py +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Recurrent ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib.recurrent.python.ops import recurrent -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import function -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import test as test_lib -from tensorflow.python.platform import tf_logging as logging - - -_ElmanState = collections.namedtuple('ElmanState', ('h')) -_ElmanTheta = collections.namedtuple('ElmanTheta', ('w', 'b')) -_ElmanInputs = collections.namedtuple('ElmanInputs', ('x')) - - -# TODO(drpng): add test for max length computation. -class RecurrentTest(test_util.TensorFlowTestCase): - - def testBasic(self): - # pylint:disable=invalid-name - _PolyState = collections.namedtuple('PolyState', ('value', 'x_power')) - _PolyTheta = collections.namedtuple('PolyTheta', ('x')) - _PolyInputs = collections.namedtuple('PolyInputs', ('coeff')) - # pylint:enable=invalid-name - - def Poly(theta, state, inputs): - next_state = _PolyState( - value=state.value + inputs.coeff * state.x_power, - x_power=state.x_power * theta.x) - return next_state, [] - - with self.cached_session() as sess: - theta = _PolyTheta(x=array_ops.constant(2.0)) - state = _PolyState( - value=array_ops.constant(0.0), - x_power=array_ops.constant(1.0)) - inputs = _PolyInputs(coeff=array_ops.constant([1., 2., 3.])) - - # x = 2 - # 1 + 2*x + 3*x^2 - ret = recurrent.Recurrent(theta, state, inputs, Poly) - - acc, state = sess.run(ret) - self.assertAllClose(acc.value, [1., 5., 17.]) - self.assertAllClose(acc.x_power, [2., 4., 8.]) - self.assertAllClose(state.value, 17.) - self.assertAllClose(state.x_power, 8.) - - y = ret[1].value - dx, d_coeff = gradients_impl.gradients(ys=[y], xs=[theta.x, inputs.coeff]) - dx_val, d_coeff_val = sess.run([dx, d_coeff]) - - # 2 + 6*x - self.assertAllClose(dx_val, 14.) - self.assertAllClose(d_coeff_val, [1., 2., 4.]) - - # acc = [1, 1+2x, 1+2x+3x^2] - # sum(acc) = 3 + 4x + 3x^2 - acc = ret[0].value - dx, d_coeff = gradients_impl.gradients( - ys=[math_ops.reduce_sum(acc)], xs=[theta.x, inputs.coeff]) - dx_val, d_coeff_val = sess.run([dx, d_coeff]) - # 4 + 6*x - self.assertAllClose(dx_val, 16.) - self.assertAllClose(d_coeff_val, [3., 4., 4.]) - - @staticmethod - def Rand(shape): - return random_ops.random_uniform( - shape, minval=-0.2, maxval=0.2, dtype=dtypes.float64) - - @staticmethod - def Elman(theta, state0, inputs): - h0, w, b, x = state0.h, theta.w, theta.b, inputs.x - xw = math_ops.matmul(array_ops.concat([x, h0], axis=1), w) - h1 = math_ops.sigmoid(xw + b) - state1 = _ElmanState(h=h1) - return (state1, state1) - - @staticmethod - def ElmanGrad(theta, state0, inputs, extras, dstate1): - - @function.Defun() - def Grad(h0, w, b, x, h1, dh1): - del b - # We hand-roll the gradient for the 2nd half of the cell as a demo. - dxwb = (dh1 * (1 - h1) * h1) - dxw, db = dxwb, math_ops.reduce_sum(dxwb, axis=0) - - # Uses tf.gradient for the 1nd half of the cell as a demo. - xw = math_ops.matmul(array_ops.concat([x, h0], axis=1), w) - dh0, dx, dw = gradients_impl.gradients( - ys=[xw], xs=[h0, x, w], grad_ys=[dxw]) - - return dh0, dx, dw, db - - dh0, dx, dw, db = Grad(state0.h, theta.w, theta.b, inputs.x, - extras.h, dstate1.h) - dstate0 = _ElmanState(h=dh0) - dinputs = _ElmanInputs(x=dx) - return (_ElmanTheta(w=dw, b=db), dstate0, dinputs) - - @staticmethod - def ElmanOut(state1): - return _ElmanState(x=state1.h) - - @staticmethod - def ElmanOutGrad(dout): - return _ElmanState(h=dout.x) - - def testElman(self): - for seqlen, use_grad in [(1, False), (1, True), (7, False), (7, True)]: - logging.info('== Elman: seqlen=%s, use_grad=%s', seqlen, use_grad) - self._ParameterizedTestElman(seqlen, use_grad) - - def _ParameterizedTestElman(self, seqlen, use_grad): - - with self.cached_session() as sess: - random_seed.set_random_seed(342462) - - batch = 3 - dims = 4 - theta = _ElmanTheta(w=RecurrentTest.Rand([2 * dims, dims]), - b=RecurrentTest.Rand([dims])) - state0 = _ElmanState(h=RecurrentTest.Rand([batch, dims])) - inputs = _ElmanInputs(x=RecurrentTest.Rand([seqlen, batch, dims])) - - # Statically unrolled. - s = state0 - out = [] - for i in xrange(seqlen): - inp = _ElmanInputs(x=inputs.x[i, :]) - s, _ = RecurrentTest.Elman(theta, s, inp) - out += [s.h] - acc0, final0 = array_ops.stack(out), s.h - loss0 = math_ops.reduce_sum(acc0) + math_ops.reduce_sum(final0) - (dw0, db0, dh0, di0) = gradients_impl.gradients( - loss0, [theta.w, theta.b, state0.h, inputs.x]) - - acc1, final1 = recurrent.Recurrent( - theta=theta, - state0=state0, - inputs=inputs, - cell_fn=RecurrentTest.Elman, - cell_grad=RecurrentTest.ElmanGrad if use_grad else None) - assert isinstance(acc1, _ElmanState) - assert isinstance(final1, _ElmanState) - acc1, final1 = acc1.h, final1.h - loss1 = math_ops.reduce_sum(acc1) + math_ops.reduce_sum(final1) - (dw1, db1, dh1, di1) = gradients_impl.gradients( - loss1, [theta.w, theta.b, state0.h, inputs.x]) - - # Fetches a few values and compare them. - (acc0, acc1, final0, final1, dw0, dw1, db0, db1, dh0, dh1, di0, - di1) = sess.run( - [acc0, acc1, final0, final1, dw0, dw1, db0, db1, dh0, dh1, di0, di1]) - self.assertAllClose(acc0, acc1) - self.assertAllClose(final0, final1) - self.assertAllClose(dw0, dw1) - self.assertAllClose(db0, db1) - self.assertAllClose(dh0, dh1) - self.assertAllClose(di0, di1) - -if __name__ == '__main__': - test_lib.main() diff --git a/tensorflow/contrib/recurrent/python/ops/functional_rnn.py b/tensorflow/contrib/recurrent/python/ops/functional_rnn.py deleted file mode 100644 index 6b4a5fbe8bc..00000000000 --- a/tensorflow/contrib/recurrent/python/ops/functional_rnn.py +++ /dev/null @@ -1,451 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A tf.compat.v1.nn.dynamic_rnn variant, built on the Recurrent class.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import copy - -from tensorflow.contrib.recurrent.python.ops import recurrent -from tensorflow.python.framework import function -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.util import nest - - -def _GetDTypesFromStructure(struct): - dtypes_list = [] - for x in nest.flatten(struct): - x = ops.convert_to_tensor(x) - dtypes_list.append(x.dtype) - return dtypes_list - - -def _SetShapeFromTemplate(struct, struct_template): - as_list = nest.flatten(struct) - template_as_list = nest.flatten(struct_template) - for element, template in zip(as_list, template_as_list): - element.set_shape(template.shape) - - -class _FunctionalRnnCell(object): - """Wrapper around RNNCell which separates state from computation. - - This class accomplishes the following: - * Turn the cell's `__call__` function into a pure function. The global - side effects are separated as `theta`. They are the variables created - for the weights of the computation. - * Unless the output is aliased as part of the state, extend the state to - contain the output so that we store the history in `Recurrent`. - * Set static shapes as required. - """ - - def __init__(self, rnn_cell, seq_inputs, initial_state): - assert initial_state is not None - - # TODO(drpng): Dtype needs to be configurable. - input_dtypes = [seq_inputs.dtype] + _GetDTypesFromStructure(initial_state) - # See _index. - like_inputs_t = nest.map_structure( - lambda x: array_ops.stop_gradient(array_ops.gather(x, 0)), seq_inputs) - input_structure = (like_inputs_t, initial_state) - - @function.Defun(*input_dtypes) - def FlatCellStep(*flat_inputs): - """The flattened version of `rnn_cell`.""" - inputs_t, state0 = nest.pack_sequence_as(input_structure, flat_inputs) - _SetShapeFromTemplate(state0, initial_state) - _SetShapeFromTemplate(inputs_t, like_inputs_t) - outputs_t, state1 = rnn_cell(inputs_t, state0) - state_list = nest.flatten(state1) - self._output_shape = outputs_t.shape - - if outputs_t in state_list: - output_index_in_state = state_list.index(outputs_t) - else: - output_index_in_state = None - - if output_index_in_state is None: - self._prepend_output = True - self._output_state_idx = 0 - return [outputs_t] + state_list - else: - self._output_state_idx = output_index_in_state - self._prepend_output = False - # To save memory, we don't store return the output separately - # from the state list, since we know it's the same. - return state_list - - def _ToPureFunction(func): - # NOTE: This forces the creating of the function. - if func.captured_inputs: - pure_func = copy.copy(func) - # pylint: disable=protected-access - pure_func._extra_inputs = [] - return pure_func - return func - - pure_flat_cell_step = _ToPureFunction(FlatCellStep) - - def CellStep(theta, extended_state0, inputs_t): - """Performs one time steps on structured inputs. - - The purpose of this function is to turn the parameters into flattened - versions, and to resolve the parameter order difference between - `Recurrent` and `RNNCell`. - - In the event the cell returns a transformed output that is not aliased - within its state, the `extended_state0` also contains the output as its - first element. - - Args: - theta: Weights required for the computation. A structure of tensors. - extended_state0: the state0, and possibly the output at the previous - time step. A structure of tensors. - inputs_t: the inputs at time t. - - Returns: - A pair of the next state (inclusive of the output), and an empty list - (unused `extras`). - The next state is congruent to state0. - """ - extended_state0_flat = nest.flatten(extended_state0) - state0_flat = self.MaybeRemoveOutputFromState(extended_state0_flat) - full_inputs = [inputs_t] + state0_flat + theta - # Note that the thetas are additional inputs appeneded as extra - # parameters. - cell_out = pure_flat_cell_step(*full_inputs) - return cell_out, [] - - self._cell_step = CellStep - self._theta = FlatCellStep.captured_inputs - self._zero_state = rnn_cell.zero_state - self._state_template = initial_state - self._output_size = rnn_cell.output_size - - @property - def extended_initial_state(self): - if self._prepend_output: - return [ - array_ops.zeros( - self._output_shape, - dtype=_GetDTypesFromStructure(self._state_template)[0]), - self._state_template - ] - else: - # The base case, where the output is just the hidden state. - return self._state_template - - @property - def cell_step(self): - return self._cell_step - - @property - def theta(self): - return self._theta - - @property - def state_template(self): - return self._state_template - - @property - def output_shape(self): - return self._output_shape - - def GetOutputFromState(self, state): - return nest.flatten(state)[self._output_state_idx] - - def MaybeRemoveOutputFromState(self, flat_state): - if self._prepend_output: - return flat_state[1:] - return flat_state - - -def _ApplyLengthsToBatch(sequence_lengths, tf_output): - # TODO(drpng): just use Update so that we don't carry over the gradients? - """Sets the output to be zero at the end of the sequence.""" - # output is batch major. - shape = array_ops.shape(tf_output) - batch_size, max_time, vector_size = shape[0], shape[1], shape[2] - output_time = array_ops.tile(math_ops.range(0, max_time), [batch_size]) - output_time = array_ops.reshape(output_time, [batch_size, max_time]) - lengths = array_ops.tile( - array_ops.reshape(sequence_lengths, [-1, 1]), [1, max_time]) - is_less = math_ops.cast( - math_ops.less(output_time, lengths), dtype=tf_output.dtype) - keep_mask = array_ops.tile( - array_ops.expand_dims(is_less, -1), [1, 1, vector_size]) - final_output = keep_mask * tf_output - return final_output - - -def _PickFinalStateFromHistory(acc_state, sequence_length): - """Implements acc_state[sequence_length - 1].""" - # This will work on all platforms, unlike the regular slice. - last_value = [] - for state_var in nest.flatten(acc_state): - # We compute the following with matrix operations: - # last_var = state_var[sequence_length - 1] - shape = array_ops.shape(state_var) - max_time, batch_size = shape[0], shape[1] - output_time = array_ops.tile(math_ops.range(0, max_time), [batch_size]) - output_time = array_ops.reshape(output_time, [batch_size, max_time]) - lengths = array_ops.tile( - array_ops.reshape(sequence_length, [-1, 1]), [1, max_time]) - last_idx = math_ops.cast( - math_ops.equal(output_time, lengths - 1), dtype=state_var.dtype) - last_idx = array_ops.transpose(last_idx) - last_idx_for_bcast = array_ops.expand_dims(last_idx, -1) - sliced = math_ops.multiply(last_idx_for_bcast, state_var) - last_var = math_ops.reduce_sum(sliced, 0) - last_value += [last_var] - return nest.pack_sequence_as(acc_state, last_value) - - -def _PostProcessOutput(extended_acc_state, extended_final_state, func_cell, - total_time, inputs_lengths, is_reversed): - """Post-process output of recurrent. - - This function takes the accumulated extended state and extracts the requested - state and output. - - When `inputs_lengths` has been set, it extracts the output from the - accumulated state. It also sets outputs past. - - When `is_reversed` is true, the output will be reversed in this function. - - It also sets the static shape information. - - Args: - extended_acc_state: A structure containing the accumulated state at each - time. It may contain the output at each time as well. - extended_final_state: A structure containing the final state. It may contain - the output at the final time. - func_cell: The functional wrapper around the cell. - total_time: A scalar integer tensor. - inputs_lengths: An integer tensor with one entry per input. - is_reversed: A boolean to indicate if the sequence is reversed. - - Returns: - A tuple with the outputs at each time, and the final state. - """ - if inputs_lengths is None or is_reversed: - flat_final_state = func_cell.MaybeRemoveOutputFromState( - nest.flatten(extended_final_state)) - tf_state = nest.pack_sequence_as(func_cell.state_template, flat_final_state) - else: - # The accumulated state is over the entire sequence, so we pick it - # out from the acc_state sequence. - flat_acc_state = func_cell.MaybeRemoveOutputFromState( - nest.flatten(extended_acc_state)) - acc_state = nest.pack_sequence_as(func_cell.state_template, flat_acc_state) - tf_state = _PickFinalStateFromHistory(acc_state, inputs_lengths) - - output_from_state = func_cell.GetOutputFromState(extended_acc_state) - if is_reversed: - output_from_state = array_ops.reverse(output_from_state, [0]) - tf_output = array_ops.transpose(output_from_state, [1, 0, 2]) - tf_output.set_shape( - [func_cell.output_shape[0], total_time, func_cell.output_shape[1]]) - if inputs_lengths is not None: - # Need set the outputs to zero. - tf_output = _ApplyLengthsToBatch(inputs_lengths, tf_output) - _SetShapeFromTemplate(tf_state, func_cell.state_template) - return tf_output, tf_state - - -# pylint: disable=invalid-name -def functional_rnn(cell, - inputs, - sequence_length=None, - initial_state=None, - dtype=None, - time_major=False, - scope=None, - use_tpu=False, - reverse=False): - """Same interface as `tf.compat.v1.nn.dynamic_rnn`.""" - with variable_scope.variable_scope(scope or 'rnn'): - if not time_major: - inputs = nest.map_structure(lambda t: array_ops.transpose(t, [1, 0, 2]), - inputs) - inputs_flat = nest.flatten(inputs) - batch_size = array_ops.shape(inputs_flat[0])[1] - if initial_state is None: - initial_state = cell.zero_state(batch_size, dtype) - func_cell = _FunctionalRnnCell(cell, inputs, initial_state) - if sequence_length is not None: - max_length = math_ops.reduce_max(sequence_length) - else: - max_length = None - if reverse: - inputs = array_ops.reverse(inputs, [0]) - extended_acc_state, extended_final_state = recurrent.Recurrent( - theta=func_cell.theta, - state0=func_cell.extended_initial_state, - inputs=inputs, - cell_fn=func_cell.cell_step, - max_input_length=max_length, - use_tpu=use_tpu, - aligned_end=reverse) - - tf_output, tf_state = _PostProcessOutput( - extended_acc_state, - extended_final_state, - func_cell, - inputs_flat[0].shape[0], - sequence_length, - is_reversed=reverse) - - if time_major: - tf_output = array_ops.transpose(tf_output, [1, 0, 2]) - return tf_output, tf_state - - -def bidirectional_functional_rnn(cell_fw, - cell_bw, - inputs, - initial_state_fw=None, - initial_state_bw=None, - dtype=None, - sequence_length=None, - time_major=False, - use_tpu=False, - fast_reverse=False, - scope=None): - """Creates a bidirectional recurrent neural network. - - Performs fully dynamic unrolling of inputs in both directions. Built to be API - compatible with `tf.compat.v1.nn.bidirectional_dynamic_rnn`, but implemented - with - functional control flow for TPU compatibility. - - Args: - cell_fw: An instance of `tf.compat.v1.nn.rnn_cell.RNNCell`. - cell_bw: An instance of `tf.compat.v1.nn.rnn_cell.RNNCell`. - inputs: The RNN inputs. If time_major == False (default), this must be a - Tensor (or hierarchical structure of Tensors) of shape [batch_size, - max_time, ...]. If time_major == True, this must be a Tensor - (or hierarchical structure of Tensors) of shape: [max_time, batch_size, - ...]. The first two dimensions must match across all the inputs, but - otherwise the ranks and other shape components may differ. - initial_state_fw: An optional initial state for `cell_fw`. Should match - `cell_fw.zero_state` in structure and type. - initial_state_bw: An optional initial state for `cell_bw`. Should match - `cell_bw.zero_state` in structure and type. - dtype: (optional) The data type for the initial state and expected output. - Required if initial_states are not provided or RNN state has a - heterogeneous dtype. - sequence_length: An optional int32/int64 vector sized [batch_size]. Used to - copy-through state and zero-out outputs when past a batch element's - sequence length. So it's more for correctness than performance. - time_major: Whether the `inputs` tensor is in "time major" format. - use_tpu: Whether to enable TPU-compatible operation. If True, does not truly - reverse `inputs` in the backwards RNN. Once b/69305369 is fixed, we can - remove this flag. - fast_reverse: Whether to use fast tf.reverse to replace tf.reverse_sequence. - This is only possible when either all sequence lengths are the same inside - the batch, or when the cell function does not change the state on padded - input. - scope: An optional scope name for the dynamic RNN. - - Returns: - outputs: A tuple of `(output_fw, output_bw)`. The output of the forward and - backward RNN. If time_major == False (default), these will - be Tensors shaped: [batch_size, max_time, cell.output_size]. If - time_major == True, these will be Tensors shaped: - [max_time, batch_size, cell.output_size]. Note, if cell.output_size is a - (possibly nested) tuple of integers or TensorShape objects, then the - output for that direction will be a tuple having the same structure as - cell.output_size, containing Tensors having shapes corresponding to the - shape data in cell.output_size. - final_states: A tuple of `(final_state_fw, final_state_bw)`. A Tensor or - hierarchical structure of Tensors indicating the final cell state in each - direction. Must have the same structure and shape as cell.zero_state. - - Raises: - ValueError: If `initial_state_fw` is None or `initial_state_bw` is None and - `dtype` is not provided. - """ - # Keep this code in sync with tf.compat.v1.nn.dynamic_rnn for compatibility. - with variable_scope.variable_scope(scope or 'bidirectional_rnn'): - # Forward direction - with variable_scope.variable_scope('fw') as fw_scope: - output_fw, output_state_fw = functional_rnn( - cell=cell_fw, - inputs=inputs, - sequence_length=sequence_length, - initial_state=initial_state_fw, - dtype=dtype, - time_major=time_major, - scope=fw_scope, - use_tpu=use_tpu) - # Backward direction - if not time_major: - time_dim = 1 - batch_dim = 0 - else: - time_dim = 0 - batch_dim = 1 - - def _reverse(input_, seq_lengths, seq_dim, batch_dim): - if seq_lengths is not None: - return array_ops.reverse_sequence( - input=input_, - seq_lengths=seq_lengths, - seq_dim=seq_dim, - batch_dim=batch_dim) - else: - # See b/69305369. - assert not use_tpu, ( - 'Bidirectional with variable sequence lengths unsupported on TPU') - return array_ops.reverse(input_, axis=[seq_dim]) - - with variable_scope.variable_scope('bw') as bw_scope: - if not fast_reverse: - inputs = _reverse( - inputs, - seq_lengths=sequence_length, - seq_dim=time_dim, - batch_dim=batch_dim) - output_bw, output_state_bw = functional_rnn( - cell=cell_bw, - inputs=inputs, - sequence_length=sequence_length, - initial_state=initial_state_bw, - dtype=dtype, - time_major=time_major, - scope=bw_scope, - use_tpu=use_tpu, - reverse=fast_reverse) - - if not fast_reverse: - output_bw = _reverse( - output_bw, - seq_lengths=sequence_length, - seq_dim=time_dim, - batch_dim=batch_dim) - - outputs = (output_fw, output_bw) - output_states = (output_state_fw, output_state_bw) - - return (outputs, output_states) - - -# pylint: enable=invalid-name diff --git a/tensorflow/contrib/recurrent/python/ops/recurrent.py b/tensorflow/contrib/recurrent/python/ops/recurrent.py deleted file mode 100644 index fbe11a11589..00000000000 --- a/tensorflow/contrib/recurrent/python/ops/recurrent.py +++ /dev/null @@ -1,742 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Recurrent computation. - -The main interface of this module is Recurrent(). -A recurrent computation describes an auto-regressive process, where outputs -of one time step are fed to the output of the next time step. - -This module uses: - theta: the "weights" each RNN uses. - state0: the initial state of each RNN. - cell_fn: A python function describing RNN cell. It must has the following - signature: - cell_fn: (theta, state0, inputs) -> (state1, extras) - state1 is the next RNN state, extras are computed by cell_fn - and the library forwards extras to cell_fn's gradient function. - cell_grad: A python function describing the backprop gradient function - for the RNN cell. It must has the following signature: - cell_grad: (theta, state0, inputs, extras, dstate1) -> ( - dtheta, dstate0, dinputs) - dstate1 is what the backprop algorithm provides representing - gradients of state1 w.r.t. the final loss. - -In this module, we handle structures of tensors for theta, state0, inputs, -and extras. The structure is an arbitrarily nested python structure, such -as a dictionary of named tuples. - -Because the computation is a left-to-right chain, a single in-place accumulator -can be used rather than a stack. Thus a special gradient was written to reduce -unnecessary memory usage. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import function -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import functional_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import inplace_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops.inplace_ops import alias_inplace_update -from tensorflow.python.util import nest - - -def _AssertIsCompatible(a, b): - """Checks that `a` and `b` are nested structures of the same type.""" - # TODO(drpng): implement. - del a - del b - - -def _Index(struct, index): - """Returns a structure with `x[index]` for each tensor `x` in the structure. - - Args: - struct: A structure of tensors. - index: A scalar integer tensor. Performance is better if `index` is - on the host memory. - - Returns: - A structure of tensors congruent to `struct`. - For each key in `ret`, `rets[key] = struct[key][index]`. - """ - index = ops.convert_to_tensor(index) - index.get_shape().assert_has_rank(0) - return nest.map_structure(lambda x: array_ops.gather(x, index), struct) - - -def _Update(struct_acc, struct_x, t): - """Updates t-th row in accumulators. - - Args: - struct_acc: The accumulators. A structure of tensors. - struct_x: The new values. A structure of tensors congruent to `struct_acc`. - t: A scalar integer. Performance is better if `t` is on the device - memory. - - Returns: - A structure of tensors. Say, ret is a returned dictionary. Then, for - each key, we have: - ret[key] = struct_acc[key]; - ret[key][t, :] = struct_x[key] - """ - to_skip_update = set() - acc_lst = nest.flatten(struct_acc) - x_lst = nest.flatten(struct_x) - t = math_ops.cast( - [t], dtypes.int32) # tf.compat.v1.to_int32 casts on-device tensors. - lst = [] - for acc, x in zip(acc_lst, x_lst): - if acc in to_skip_update: - # Until b/62105730 is fixed, we need to avoid inplace update for tensors - # of rank 1. could reshape to handle it, but we don't really need the - # values applied to these, so just skip their modification. - lst += [acc] - else: - lst += [alias_inplace_update(acc, t, array_ops.expand_dims(x, 0))] - return nest.pack_sequence_as(struct_acc, lst) - - -def _SeqLenDim(struct): - """Returns the 0-th dim size of tensors in a structure of tensors. - - This is the max sequence length according to the shape of the inputs. - - Args: - struct: A structure of tensors. Every tensor's 0-th dim has the same size. - - Returns: - A scalar tensor which is the size of 0-th dim of every tensors in struct. - """ - xs = nest.flatten(struct) - assert xs - dim0 = array_ops.shape(xs[0])[0] - return dim0 - - -def _Flatten(struct): - """Flattens a structure.""" - return nest.flatten(struct) - - -def _Pack(elements, struct_template): - """Packs the list of tensors according to the structure. - - In the event that `elements` should be a scalar, `struct_template` must - contain exactly one non-trivial element (for instance, `[[], {'x':elt}]`). - - Args: - elements: Elements to be packed. A list of tensor, or a single tensor. - struct_template: The container structure in which to pack them. - Returns: - A python structure of the same type as `struct_template`, containing - `elements` as its contained elements. - """ - if not nest.is_sequence(elements): - return nest.pack_sequence_as(struct_template, [elements]) - return nest.pack_sequence_as(struct_template, elements) - - -def _EmptyAcc(slen, struct_template): - """Creates a set of accumulators for tensors in structure. - - Args: - slen: The sequence length. A scalar tensor. - struct_template: A structure of tensors. - - Returns: - A structure congruent to `struct_template`. Say ret is a returned - dictionary. Then, `ret.key`, a tensor, has the same dtype as - `struct_template.key`. The tensor's shape has 1 more dimension - than the tensor `struct_template.key`. The extra 0-th dimension is of size - `slen`. E.g., if `slen=10` and `struct_template.key`'s shape is `[3, 5]`, - then, `ret.key`'s shape is `[10, 3, 5]`. - """ - - def _EmptyAccForTensor(tensor): - return inplace_ops.empty( - array_ops.concat([[slen], array_ops.shape(tensor)], axis=0), - tensor.dtype, - init=True) - - return nest.map_structure(_EmptyAccForTensor, struct_template) - - -def _EmptyLike(struct): - """Creates a set of empty initialized tensors. - - Args: - struct: A structure of tensors. - - Returns: - A struct of tensors. Each tensor has the same shape and dtype as - its corresponding tensor in `struct`. And each tensor is initialized. - """ - return nest.map_structure( - lambda x: inplace_ops.empty_like(x, init=True), struct) - - -def _Add(struct_x, struct_y): - """Adds tensors in `struct_x` with respective tensors in `struct_y`. - - Args: - struct_x: A struct of tensors. - struct_y: A struct of tensors congruent to `struct_x`. - - Returns: - A struct of tensors. Each element of the returned value - equals `x + y`, with corresponding values in `struct_x` and `struct_y`. - """ - list_x = nest.flatten(struct_x) - list_y = nest.flatten(struct_y) - z = [] - for x, y in zip(list_x, list_y): - z += [math_ops.add(x, y)] - return nest.pack_sequence_as(struct_x, z) - - -def _Dtypes(struct): - """Returns all tensors' data types in a list.""" - return [x.dtype for x in nest.flatten(struct)] - - -def _ConvertNoneGradientToZeros(xs, dxs): - """Sanitize dxs so that None becomes zeros appropriately. - - Args: - xs: A list of tensors. - dxs: A list of tensors. dxs[i] corresponds to xs[i]'s gradient. - - Returns: - A structure same as `dxs` with `None` replaced by a zero tensor. - """ - list_xs = nest.flatten(xs) - list_dxs = nest.flatten(dxs) - - # If x does not get any backprop-ed gradient, propagate zeros. - rets = [] - for (x, dx) in zip(list_xs, list_dxs): - if dx is None: - rets.append(array_ops.zeros_like(x)) - else: - rets.append(dx) - - return nest.pack_sequence_as(dxs, rets) - - -# All structures are flattened for use internally. This is for simplicity -# and also to use the Defun construct. -# In the forward pass (inference), the computation is structured as follows. -# Forward: [gradient = _Recurrent.Grad] -# Flatten structures, create accumulators. -# for t = 0..max_input_length: -# Defun ForwardLoopBody: -# Defun Fwd: flatten/pack around cell_fn -# state1 = Fwd(inputs[t], state0) -# acc_state += [state1] -# Pack structures. -# During the backward pass (backpropping the gradient from the last time -# step to the first, through the structure), the computation is structured -# as follows. -# Grad: -# Flatten structures. -# Defun Backward: -# Create create accumulated derivatives: d_theta, d_inputs, d_acc_state. -# Regarding the note at the top of the file, there is only one accumulator -# for d_theta accumulated over the whole sequence. -# for t = max_input_length -1..0: -# Defun BackwardLoopBody: -# Retrieve acc_state[t] computed in the forward pass. -# Defun Bak: flatten/back around cell_fn_grad. -# d_state1 is d_state0 from previous step (ie next time). -# d_acc_state[dev_t] += d_state1 -# d_theta_t, d_state0, d_inputs_t, = Bak() -# d_inputs[dev_t] += d_inputs -# d_theta += d_theta_t -# d_acc_state[t] += d_state1 -# Pack structures and return. -class _Recurrent(object): - """A helper class to construct a recurrent neural net.""" - - def __init__(self, - cell_fn, - cell_grad, - theta, - state0, - inputs, - max_input_length, - extras, - use_tpu, - aligned_end=False): - """RNN helper class. - - Args: - cell_fn: A python function, which computes: - state1, extras = cell_fn(theta, state0, inputs[t, :]) - cell_grad: A python function which computes: - dtheta, dstate0, dinputs[t, :] = cell_grad( - theta, state0, inputs[t, :], extras, dstate1) - theta: weights. A structure of tensors. - state0: initial state. A structure of tensors. - inputs: inputs. A structure of tensors. - max_input_length: None, or the maximum effective length of the input over - all batches. A scalar tensor. - extras: A structure of tensors. The 2nd return value of every - invocation of cell_fn is a structure of tensors with matching keys - and shapes of this `extras`. - use_tpu: A boolean indicating whether the computation is mean to - run on a TPU. - aligned_end: A boolean indicating whether the sequence is aligned at - the end. - """ - self._theta = theta - self._state = state0 - self._inputs = inputs - self._max_input_length = self._MaybeComputeMaxInputLength( - inputs, max_input_length) - self._cell_fn = cell_fn - self._cell_grad = cell_grad - self._extras = extras - self._aligned_end = aligned_end - - # pylint: disable=unbalanced-tuple-unpacking - - # NOTE: TF Function (Fwd, Bak, ForwardLoopBody, BackwardLoopBody, - # Forward and Backward defined below) simply takes a list of - # Tensors and returns a list of Tensors. When we pass in a - # structure (a list of structures of Tensors), we use _Flatten to - # convert the structure into a list of tensor. Conversely, the - # following code often uses _Pack to formulate a structure from a - # list of tensors based on a "template". - - # Wraps cell_fn in a TF Function: - # state1 = cell_fn(theta, state0, inputs) - fwd_sig = [self._theta, self._state, self._inputs] - - compiled = use_tpu - noinline = not compiled - dev_t_type = dtypes.int32 if use_tpu else dtypes.int64 - - @function.Defun(*_Dtypes(fwd_sig)) - def Fwd(*args): - (theta, state0, inputs) = _Pack(args, fwd_sig) - state1, extras = self._cell_fn(theta, state0, inputs) - assert not function.get_extra_args(), ( - 'cell_fn is not pure with extra args: %s.' % - (function.get_extra_args())) - _AssertIsCompatible(state1, self._state) - _AssertIsCompatible(extras, self._extras) - return _Flatten([state1, extras]) - - # Wraps cell_fn in a TF Function as a for-loop's body. - # - # The loop state is composed of: - # t: The loop variable. Timestep id. - # dev_t: The loop variable mirrored on the device. - # theta: the recurrent net's weights. - # state0: the previous recurrent state. - # inputs: inputs to the recurrent net. inputs[t, :] are for the timestep t. - # acc_state: Each timestep's computed new state is also stashed into - # acc_state. - # acc_extras: Each timestep's computed extras is stashed into acc_extras - fwdloop_sig = [ - self._theta, self._state, self._inputs, self._state, self._extras - ] - - @function.Defun(dtypes.int32, dev_t_type, *_Dtypes(fwdloop_sig)) - def ForwardLoopBody(*args): - """The body of forward loop.""" - t, dev_t = args[0], args[1] - (theta, state0, inputs, acc_state, acc_extras) = _Pack( - args[2:], fwdloop_sig) - inputs_t = _Index(inputs, t) # external input at time step t. - fwd = Fwd(*_Flatten([theta, state0, inputs_t])) - state1, extras = _Pack(fwd, [self._state, self._extras]) - # Saves state1 and extras in their accumulators. - acc_state = _Update(acc_state, state1, dev_t) - acc_extras = _Update(acc_extras, extras, dev_t) - - return [math_ops.add(dev_t, 1)] + _Flatten( - [theta, state1, inputs, acc_state, acc_extras]) - - def Grad(op, *args): - """The python grad function for the Forward function.""" - - # NOTE: tf.gradient backprops None for int32/int64 while zeros - # for float32/float64. For consistency, we always backprop - # zeros. - args = list(args) - for i, dy in enumerate(args): - if dy is None: - args[i] = array_ops.zeros_like(op.outputs[i]) - # TODO(drpng): getting the extra state here? - op_inputs = [x for x in op.inputs] - op_struct = [ - self._theta, self._state, self._inputs, self._max_input_length, - self._extras - ] - (theta, state0, inputs, max_input_length, _) = _Pack(op_inputs, op_struct) - # acc_state and acc_extras are computed by the Forward pass and - # needed by the Backward pass. - acc_state, _, acc_extras = _Pack([x for x in op.outputs], - [self._state, self._state, self._extras]) - - # Forward computes acc_state, the final state and - # acc_extras. tf.gradients gives us their gradients w.r.t. the - # final loss. Because acc_extras are not exposed by Compute(), - # it has no gradients w.r.t. the final loss (i.e., by - # construction, it must be zeros). - d_acc_state, d_state1, _ = _Pack(args, - [self._state, self._state, self._extras]) - return Backward(*_Flatten([ - theta, state0, inputs, max_input_length, acc_state, acc_extras, - d_acc_state, d_state1 - ])) - - # Forward calls ForwardLoopBody n times. Each time computes one - # time step of the recurrent net. - forward_sig = [ - self._theta, self._state, self._inputs, self._max_input_length, - self._extras - ] - - @function.Defun( - *_Dtypes(forward_sig), python_grad_func=Grad, noinline=noinline) - def Forward(*args): - """Forward pass of the recurrent net.""" - theta, state0, inputs, max_input_length, extras = _Pack(args, forward_sig) - - slen_dim = _SeqLenDim(inputs) - - # Creates accumulators for state0 and extras. - acc_state = _EmptyAcc(slen_dim, state0) - acc_extras = _EmptyAcc(slen_dim, extras) - - t = slen_dim - max_input_length if self._aligned_end else 0 - dev_t = math_ops.cast(t, dtypes.int32) if use_tpu else math_ops.cast( - t, dtypes.int64) - run = functional_ops.For( - start=t, - limit=slen_dim if self._aligned_end else max_input_length, - delta=1, - inputs=[dev_t] + _Flatten( - [theta, state0, inputs, acc_state, acc_extras]), - body=ForwardLoopBody, - rewrite_with_while=compiled) - _, state1, _, acc_state, acc_extras = _Pack( - run[1:], - [self._theta, self._state, self._inputs, self._state, self._extras]) - - return _Flatten([acc_state, state1, acc_extras]) - - # The per-step backward computes: - # d_theta, d_state0, d_inputs = cell_grad( - # theta, state0, inputs, extras, d_state1) - # where d_state1 is the backprop-ed gradient for state1, and - # extras is the computed by the forward step to facilitate the - # backward step. - bak_sig = [ - self._theta, self._state, self._inputs, self._extras, self._state - ] - - @function.Defun(*_Dtypes(bak_sig)) - def Bak(*args): - """Backward step.""" - (theta, state0, inputs, extras, d_state1) = _Pack(args, bak_sig) - (dtheta, dstate0, dinputs) = self._cell_grad(theta, state0, inputs, - extras, d_state1) - assert not function.get_extra_args(), ( - 'cell_grad is not pure with extra args: %s.' % - (function.get_extra_args())) - _AssertIsCompatible(dtheta, self._theta) - _AssertIsCompatible(dstate0, self._state) - _AssertIsCompatible(dinputs, self._inputs) - return _Flatten( - _ConvertNoneGradientToZeros([theta, state0, inputs], - [dtheta, dstate0, dinputs])) - - # Define defuns used by a functional_ops.If in BackwardLoopBody. - state_if_sig = [self._state, self._state] - - @function.Defun(*_Dtypes(state_if_sig)) - def ReturnOrigState0(*args): - """Returns original state0 from inputs.""" - (_, orig_state0) = _Pack(args, state_if_sig) - return nest.flatten(orig_state0) - - @function.Defun(*_Dtypes(state_if_sig)) - def ReturnAccState(*args): - """Returns acc_state[t-1] from inputs.""" - (acc_state, _) = _Pack(args, state_if_sig) - return nest.flatten(acc_state) - - # Wraps cell_grad gradient function in a TF Function as a - # for-loop's body for the Backward pass. - # - # The loop state is composed of: - # t: The loop variable. Timestep id. - # state0: the initial state for the entire backward loop. - # dev_t: The loop variable mirrored on the device. - # theta: the recurrent net's weights. - # inputs: inputs to the recurrent net. inputs[t, :] are for the timestep t. - # acc_state: Each timestep's computed new state was stashed into - # acc_state by the Forward pass. - # acc_extras: Each timestep's computed extras was stashed into - # acc_extras by the Forward pass. - # d_theta: All timestep's gradient for theta is accumulated (added) into - # d_theta. - # d_state1: The backprop-ed gradient for the new stated computed by - # timestep t. - # d_inputs: d_inputs[t, :] is populated by the backward time step t. - # d_acc_state: The backprop-ed gradient for acc_state. - bakloop_sig = [ - self._theta, self._state, self._inputs, self._state, self._extras, - self._theta, self._state, self._inputs, self._state - ] - - @function.Defun(dtypes.int32, dev_t_type, *_Dtypes(bakloop_sig)) - def BackwardLoopBody(*args): - """Backward loop body function.""" - t, dev_t = args[0], args[1] - (theta, orig_state0, inputs, acc_state, acc_extras, d_theta, d_state1, - d_inputs, d_acc_state) = _Pack(args[2:], bakloop_sig) - - # The input recurrent state for time step t is previous time step's - # output, or the original state0 when on time step 0. - state_from_acc = _Index(acc_state, math_ops.maximum(0, t - 1)) - state0 = functional_ops.If( - math_ops.equal(t, array_ops.constant(0, dtypes.int32)), - _Flatten([state_from_acc, orig_state0]), ReturnOrigState0, - ReturnAccState) - state0 = nest.pack_sequence_as(orig_state0, state0) - - # The external inputs for time step t. - inputs_t = _Index(inputs, t) - # The extras for time step t. - extras_t = _Index(acc_extras, t) - - d_state1 = _Add(_Index(d_acc_state, t), d_state1) - (d_theta_t, d_state0, d_inputs_t) = _Pack( - Bak(*_Flatten([theta, state0, inputs_t, extras_t, d_state1])), - [self._theta, self._state, self._inputs]) - d_theta = _Add(d_theta, d_theta_t) - d_inputs = _Update(d_inputs, d_inputs_t, dev_t) - return [math_ops.subtract(dev_t, 1)] + _Flatten([ - theta, orig_state0, inputs, acc_state, acc_extras, d_theta, d_state0, - d_inputs, d_acc_state - ]) - - # Backward calls BackwardLoopBody n times. Each time computes the backprop - # for one time step of the recurrent net. - backward_sig = [ - self._theta, self._state, self._inputs, self._max_input_length, - self._state, self._extras, self._state, self._state - ] - - @function.Defun(*_Dtypes(backward_sig), noinline=noinline) - def Backward(*args): - """Backward pass for the recurrent net.""" - # theta, state0, inputs are Forward's inputs. - # acc_state is the accumulated 1st output of Forward. - # acc_extras is the accumulated 2nd output of Forward. - # d_acc_state is the gradient for acc_state. - # d_state1 is the gradient for the final state computed by Forward. - (theta, state0, inputs, max_input_length, acc_state, acc_extras, - d_acc_state, d_state1) = _Pack(args, backward_sig) - - # Accumulators for gradients. - d_theta = _EmptyLike(theta) - d_inputs = _EmptyLike(inputs) - - slen_dim = _SeqLenDim(inputs) - - # Loop backwards. Note the loop's limit is open-ended, so goes through - # t=0. - t = slen_dim - 1 if self._aligned_end else max_input_length - 1 - dev_t = math_ops.cast(t, dtypes.int32) if use_tpu else math_ops.cast( - t, dtypes.int64) - limit = slen_dim - max_input_length - 1 if self._aligned_end else -1 - run = functional_ops.For( - start=t, - limit=limit, - delta=-1, - inputs=[dev_t] + _Flatten([ - theta, state0, inputs, acc_state, acc_extras, d_theta, d_state1, - d_inputs, d_acc_state - ]), - body=BackwardLoopBody, - rewrite_with_while=compiled) - - (theta, state0, inputs, acc_state, acc_extras, d_theta, d_state0, - d_inputs, d_acc_state) = _Pack(run[1:], bakloop_sig) - - d_max_input_length = array_ops.constant(0, dtype=max_input_length.dtype) - return _Flatten( - [d_theta, d_state0, d_inputs, d_max_input_length, acc_extras]) - - self._forward = Forward - - def _MaybeComputeMaxInputLength(self, inputs, max_input_length): - if max_input_length is not None: - return max_input_length - return math_ops.reduce_max(array_ops.shape(nest.flatten(inputs)[0])[0]) - - def Compute(self): - return _Pack( - self._forward(*_Flatten([ - self._theta, self._state, self._inputs, self._max_input_length, - self._extras - ])), [self._state, self._state, self._extras])[:2] - - -def _GetCellGrad(cell_fn, cell_grad): - """Returns the gradient function for cell_fn. - - Args: - cell_fn: The recurrent neural net's cell function. - cell_grad: If not None, cell_fn's gradient function. - - Returns: - Returns cell_grad if not None. Otherwise, assume cell_fn is a python - function representing the recurrent neural net's cell function, i.e., - cell_fn: (theta, state0, inputs) -> (state1, extra) - returns its default gradient python function, i.e., - cell_grad: (theta, state0, inputs, extras, dstate1) -> ( - dtheta, dstate0, dinputs) - """ - - if cell_grad: - return cell_grad - - def CellGrad(theta, state0, inputs, extras, dstate1): - """Default gradient function for cell_fn.""" - # NOTE: The default grad function recomputes the forward - # function and does not take advantage of 'extras' returned by - # the forward function. - del extras - state1, extras = cell_fn(theta, state0, inputs) - ys = _Flatten([state1]) - xs = _Flatten([theta, state0, inputs]) - grad_ys = _Flatten([dstate1]) - grads = gradients_impl.gradients(ys=ys, xs=xs, grad_ys=grad_ys) - return _ConvertNoneGradientToZeros([theta, state0, inputs], - _Pack(grads, [theta, state0, inputs])) - - return CellGrad - - -def _IsSingleTimeStep(inputs, max_input_length): - """Returns True only if the time dimension of inputs is 1.""" - if not isinstance(max_input_length, ops.Tensor): - return max_input_length == 1 - for x in nest.flatten(inputs): - if x.shape.dims is None or x.shape[0].value != 1: - return False - return True - - -def Recurrent(theta, - state0, - inputs, - cell_fn, - cell_grad=None, - extras=None, - max_input_length=None, - use_tpu=False, - aligned_end=False): - """Compute a recurrent neural net. - - Roughly, Recurrent() computes the following: - state = state0 - for t in inputs' sequence length: - state = cell_fn(theta, state, inputs[t, :]) - accumulate_state[t, :] = state - return accumulate_state, state - - theta, state, inputs are all structures of tensors. - - inputs[t, :] means taking a slice out from every tensor in the inputs. - - accumulate_state[t, :] = state means that we stash every tensor in - 'state' into a slice of the corresponding tensor in - accumulate_state. - - cell_fn is a python callable computing (building up a TensorFlow - graph) the recurrent neural network's one forward step. Two calls of - cell_fn must describe two identical computations. - - By construction, Recurrent()'s backward computation does not access - any intermediate values computed by cell_fn during forward - computation. We may extend Recurrent() to support that by taking a - customized backward function of cell_fn. - - Args: - theta: weights. A structure of tensors. - state0: initial state. A structure of tensors. - inputs: inputs. A structure of tensors. - cell_fn: A python function, which computes: - state1, extras = cell_fn(theta, state0, inputs[t, :]) - cell_grad: A python function which computes: - dtheta, dstate0, dinputs[t, :] = cell_grad( - theta, state0, inputs[t, :], extras, dstate1) - extras: A structure of tensors. The 2nd return value of every - invocation of cell_fn is a structure of tensors with matching keys - and shapes of this `extras`. - max_input_length: maximum length of effective input. This is used to - truncate the computation if the inputs have been allocated to a - larger size. A scalar tensor. - use_tpu: whether or not we are on TPU. - aligned_end: A boolean indicating whether the sequence is aligned at - the end. - - Returns: - accumulate_state and the final state. - """ - if cell_grad is None and _IsSingleTimeStep(inputs, max_input_length): - # The seqlen length is staticly known as 1. Hence, we just need to - # call cell_fn once without putting it into a loop. - inputs = nest.map_structure(lambda x: array_ops.squeeze(x, axis=0), inputs) - state1, _ = cell_fn(theta, state0, inputs) - acc_state = nest.map_structure(lambda x: array_ops.expand_dims(x, axis=0), - state1) - return acc_state, state1 - - # If cell_grad is not given, derives the gradient function from - # cell_fn. - cell_grad = _GetCellGrad(cell_fn, cell_grad) - - if extras is None: - # Derives 'extras' so that we can allocate extras' accumulator. - _, extras = cell_fn(theta, state0, _Index(inputs, 0)) - extras = nest.map_structure(array_ops.zeros_like, extras) - else: - _, actual = cell_fn(theta, state0, _Index(inputs, 0)) - _AssertIsCompatible(extras, actual) - - return _Recurrent( - cell_fn=cell_fn, - cell_grad=cell_grad, - theta=theta, - state0=state0, - inputs=inputs, - max_input_length=max_input_length, - extras=extras, - use_tpu=use_tpu, - aligned_end=aligned_end).Compute() diff --git a/tensorflow/contrib/recurrent/python/recurrent_api.py b/tensorflow/contrib/recurrent/python/recurrent_api.py deleted file mode 100644 index f1c97927dfe..00000000000 --- a/tensorflow/contrib/recurrent/python/recurrent_api.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Recurrent computations library.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.contrib.recurrent.python.ops.functional_rnn import bidirectional_functional_rnn -from tensorflow.contrib.recurrent.python.ops.functional_rnn import functional_rnn -from tensorflow.contrib.recurrent.python.ops.recurrent import Recurrent -# pylint: enable=unused-import - -del absolute_import -del division -del print_function diff --git a/tensorflow/contrib/reduce_slice_ops/BUILD b/tensorflow/contrib/reduce_slice_ops/BUILD deleted file mode 100644 index aeb2c67317e..00000000000 --- a/tensorflow/contrib/reduce_slice_ops/BUILD +++ /dev/null @@ -1,101 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "tf_cc_test", "tf_custom_op_library", "tf_gen_op_libs", "tf_gen_op_wrapper_py", "tf_kernel_library") -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") -load("//tensorflow/core/platform:default/build_config.bzl", "tf_kernel_tests_linkstatic") - -package( - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_custom_op_library( - name = "python/ops/_reduce_slice_ops.so", - srcs = [ - "kernels/reduce_slice_ops.cc", - "kernels/reduce_slice_ops.h", - "ops/reduce_slice_ops.cc", - ], - gpu_srcs = [ - "kernels/reduce_slice_ops.h", - "kernels/reduce_slice_ops_gpu.cu.cc", - ], -) - -tf_kernel_library( - name = "reduce_slice_ops_kernels", - srcs = [ - "kernels/reduce_slice_ops.cc", - ], - hdrs = [ - "kernels/reduce_slice_ops.h", - ], - gpu_srcs = [ - "kernels/reduce_slice_ops.h", - "kernels/reduce_slice_ops_gpu.cu.cc", - ], - deps = [ - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//third_party/eigen3", - ], -) - -tf_gen_op_libs( - op_lib_names = ["reduce_slice_ops"], -) - -tf_gen_op_wrapper_py( - name = "reduce_slice_ops", - deps = [":reduce_slice_ops_op_lib"], -) - -tf_custom_op_py_library( - name = "reduce_slice_ops_py", - srcs = [ - "__init__.py", - "python/ops/reduce_slice_ops.py", - ], - dso = [ - ":python/ops/_reduce_slice_ops.so", - ], - kernels = [ - ":reduce_slice_ops_kernels", - ":reduce_slice_ops_op_lib", - ], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":reduce_slice_ops", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:framework", - "//tensorflow/python:platform", - ], -) - -cuda_py_test( - name = "reduce_slice_ops_test", - size = "small", - srcs = ["python/kernel_tests/reduce_slice_ops_test.py"], - additional_deps = [ - ":reduce_slice_ops_py", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//third_party/py/numpy", - ], -) - -tf_cc_test( - name = "reduce_slice_ops_test_cc", - size = "small", - srcs = [ - "ops/reduce_slice_ops_test.cc", - ], - linkstatic = tf_kernel_tests_linkstatic(), - deps = [ - ":reduce_slice_ops_op_lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) diff --git a/tensorflow/contrib/reduce_slice_ops/__init__.py b/tensorflow/contrib/reduce_slice_ops/__init__.py deleted file mode 100644 index d0364587b58..00000000000 --- a/tensorflow/contrib/reduce_slice_ops/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""reduce by slice - -@@reduce_slice_sum -@@reduce_slice_prod -@@reduce_slice_min -@@reduce_slice_max -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.reduce_slice_ops.python.ops import * diff --git a/tensorflow/contrib/reduce_slice_ops/kernels/reduce_slice_ops.cc b/tensorflow/contrib/reduce_slice_ops/kernels/reduce_slice_ops.cc deleted file mode 100644 index edcef3adeaa..00000000000 --- a/tensorflow/contrib/reduce_slice_ops/kernels/reduce_slice_ops.cc +++ /dev/null @@ -1,249 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#define EIGEN_USE_THREADS - -#include "tensorflow/contrib/reduce_slice_ops/kernels/reduce_slice_ops.h" -#include -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/lib/core/threadpool.h" - -namespace tensorflow { - -using GPUDevice = Eigen::GpuDevice; -using CPUDevice = Eigen::ThreadPoolDevice; -using thread::ThreadPool; - -namespace functor { - -#define Sum(a, b) ((a) + (b)) -#define Prod(a, b) ((a) * (b)) -#define Max(a, b) ((a) > (b) ? (a) : (b)) -#define Min(a, b) ((a) < (b) ? (a) : (b)) - -#define CPUReduceSliceFunctorReduceop(reduceop, beginning) \ - template \ - struct ReduceSliceFunctor##reduceop { \ - private: \ - struct XYZ { \ - Index x, y, z; \ - XYZ() = default; \ - XYZ(Index x, Index y, Index z) : x(x), y(y), z(z) {} \ - }; \ - inline static XYZ global_index_to_xyz(Index global, XYZ size) { \ - XYZ ret; \ - ret.x = global / (size.y * size.z); \ - ret.y = global % (size.y * size.z) / size.z; \ - ret.z = global % size.z; \ - return ret; \ - } \ - \ - public: \ - virtual ~ReduceSliceFunctor##reduceop() {} \ - virtual void operator()(OpKernelContext* ctx, const CPUDevice& d, \ - Index indices_width, \ - typename TTypes::ConstTensor indices, \ - typename TTypes::ConstTensor data, \ - typename TTypes::Tensor output) { \ - Index bound = data.dimension(1); \ - Index dim1 = output.dimension(0); \ - Index dim2 = output.dimension(1); \ - Index dim3 = output.dimension(2); \ - Index size = dim1 * dim2 * dim3; \ - if (size == 0) { \ - return; \ - } \ - T zero = beginning(); \ - ThreadPool* thread_pool = \ - ctx->device()->tensorflow_cpu_worker_threads()->workers; \ - /* shard the work */ \ - auto work = [&](Index start, Index end) { \ - for (Index global = start; global < end; ++global) { \ - XYZ xyz = global_index_to_xyz(global, XYZ(dim1, dim2, dim3)); \ - Index x = xyz.x; \ - Index y = xyz.y; \ - Index z = xyz.z; \ - output(x, y, z) = zero; \ - Index slice_head = indices(y * indices_width); \ - Index slice_end = std::min(indices(y * indices_width + 1), bound); \ - for (Index i = slice_head; i < slice_end; ++i) { \ - output(x, y, z) = reduceop(output(x, y, z), data(x, i, z)); \ - } \ - } \ - }; \ - /* Here assumes the number of average CPU cycles for each slice equals \ - * the average length of each slice */ \ - thread_pool->ParallelFor(size, std::max(bound / dim2, (Index)1), work); \ - } \ - }; - -CALL_ALL_REDUCEOPS(CPUReduceSliceFunctorReduceop) -#undef CPUReduceSliceFunctorReduceop - -#define DEFINE_CPU_SUMPROD_SPECS_INDEX(T, Index) \ - template struct ReduceSliceFunctorSum; \ - template struct ReduceSliceFunctorProd; - -#define DEFINE_CPU_MINMAX_SPECS_INDEX(T, Index) \ - template struct ReduceSliceFunctorMax; \ - template struct ReduceSliceFunctorMin; - -#define DEFINE_CPU_SUMPROD_SPECS(T) \ - DEFINE_CPU_SUMPROD_SPECS_INDEX(T, int32); \ - DEFINE_CPU_SUMPROD_SPECS_INDEX(T, int64); - -#define DEFINE_CPU_MINMAX_SPECS(T) \ - DEFINE_CPU_MINMAX_SPECS_INDEX(T, int32); \ - DEFINE_CPU_MINMAX_SPECS_INDEX(T, int64); - -TF_CALL_NUMBER_TYPES(DEFINE_CPU_SUMPROD_SPECS) -TF_CALL_REAL_NUMBER_TYPES(DEFINE_CPU_MINMAX_SPECS) - -#undef DEFINE_CPU_SUMPROD_SPECS_INDEX -#undef DEFINE_CPU_MINMAX_SPECS_INDEX -#undef DEFINE_CPU_SUMPROD_SPECS -#undef DEFINE_CPU_MINMAX_SPECS - -} // namespace functor - -template - class Functor> -class ReduceSliceKernel : public OpKernel { - public: - explicit ReduceSliceKernel(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - const Tensor& data = context->input(0); - const Tensor& indices = context->input(1); - const Tensor& _axis = context->input(2); - int64 axis = _axis.scalar()(); - - int indices_width = 2; - int out_axis_dim_size = indices.shape().dim_size(0); - if (indices.dims() == 1 || indices.shape().dim_size(1) == 1) { - indices_width = 1; - if (out_axis_dim_size > 0) { - out_axis_dim_size--; - } - } - - TensorShape output_shape = data.shape(); - output_shape.set_dim(axis, out_axis_dim_size); - Tensor* output = nullptr; - OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, &output)); - auto functor = Functor(); - functor(context, context->eigen_device(), indices_width, - indices.flat(), data.flat_inner_outer_dims(axis - 1), - output->flat_inner_outer_dims(axis - 1)); - } -}; - -#define REGISTER_CPU_SUMPROD_REDUCE_SLICE_KERNELS(type, index_type) \ - REGISTER_KERNEL_BUILDER(Name("ReduceSliceSum") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tindices"), \ - ReduceSliceKernel); \ - REGISTER_KERNEL_BUILDER(Name("ReduceSliceProd") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tindices"), \ - ReduceSliceKernel); - -#define REGISTER_CPU_MINMAX_REDUCE_SLICE_KERNELS(type, index_type) \ - REGISTER_KERNEL_BUILDER(Name("ReduceSliceMax") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tindices"), \ - ReduceSliceKernel); \ - REGISTER_KERNEL_BUILDER(Name("ReduceSliceMin") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tindices"), \ - ReduceSliceKernel); - -#define REGISTER_CPU_SUMPROD_REDUCE_SLICE_KERNELS_ALL(type) \ - REGISTER_CPU_SUMPROD_REDUCE_SLICE_KERNELS(type, int32); \ - REGISTER_CPU_SUMPROD_REDUCE_SLICE_KERNELS(type, int64); - -#define REGISTER_CPU_MINMAX_REDUCE_SLICE_KERNELS_ALL(type) \ - REGISTER_CPU_MINMAX_REDUCE_SLICE_KERNELS(type, int32); \ - REGISTER_CPU_MINMAX_REDUCE_SLICE_KERNELS(type, int64); - -TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_MINMAX_REDUCE_SLICE_KERNELS_ALL) -TF_CALL_NUMBER_TYPES(REGISTER_CPU_SUMPROD_REDUCE_SLICE_KERNELS_ALL) - -#undef REGISTER_CPU_SUMPROD_REDUCE_SLICE_KERNELS -#undef REGISTER_CPU_MINMAX_REDUCE_SLICE_KERNELS -#undef REGISTER_CPU_SUMPROD_REDUCE_SLICE_KERNELS_ALL -#undef REGISTER_CPU_MINMAX_REDUCE_SLICE_KERNELS_ALL - -#if GOOGLE_CUDA - -#define REGISTER_GPU_REDUCE_SLICE_KERNELS(type, index_type) \ - REGISTER_KERNEL_BUILDER(Name("ReduceSliceSum") \ - .Device(DEVICE_GPU) \ - .HostMemory("axis") \ - .TypeConstraint("T") \ - .TypeConstraint("Tindices"), \ - ReduceSliceKernel); \ - REGISTER_KERNEL_BUILDER(Name("ReduceSliceProd") \ - .Device(DEVICE_GPU) \ - .HostMemory("axis") \ - .TypeConstraint("T") \ - .TypeConstraint("Tindices"), \ - ReduceSliceKernel); \ - REGISTER_KERNEL_BUILDER(Name("ReduceSliceMax") \ - .Device(DEVICE_GPU) \ - .HostMemory("axis") \ - .TypeConstraint("T") \ - .TypeConstraint("Tindices"), \ - ReduceSliceKernel); \ - REGISTER_KERNEL_BUILDER(Name("ReduceSliceMin") \ - .Device(DEVICE_GPU) \ - .HostMemory("axis") \ - .TypeConstraint("T") \ - .TypeConstraint("Tindices"), \ - ReduceSliceKernel); - -#define REGISTER_GPU_REDUCE_SLICE_KERNELS_ALL(type) \ - REGISTER_GPU_REDUCE_SLICE_KERNELS(type, int32); \ - REGISTER_GPU_REDUCE_SLICE_KERNELS(type, int64); - -TF_CALL_REAL_NUMBER_TYPES(REGISTER_GPU_REDUCE_SLICE_KERNELS_ALL); - -#undef REGISTER_GPU_REDUCE_SLICE_KERNELS -#undef REGISTER_GPU_REDUCE_SLICE_KERNELS_ALL - -#undef Sum -#undef Prod -#undef Min -#undef Max - -#endif // GOOGLE_CUDA - -} // namespace tensorflow diff --git a/tensorflow/contrib/reduce_slice_ops/kernels/reduce_slice_ops.h b/tensorflow/contrib/reduce_slice_ops/kernels/reduce_slice_ops.h deleted file mode 100644 index 12bff1e9161..00000000000 --- a/tensorflow/contrib/reduce_slice_ops/kernels/reduce_slice_ops.h +++ /dev/null @@ -1,79 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_REDUCE_SLICE_OPS_KERNELS_REDUCE_SLICE_OPS_H_ -#define TENSORFLOW_CONTRIB_REDUCE_SLICE_OPS_KERNELS_REDUCE_SLICE_OPS_H_ - -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_types.h" - -namespace tensorflow { - -class OpKernelContext; - -namespace functor { - -namespace reduce_functions { - -template -inline T zero() { - return T(0); -} - -template -inline T one() { - return T(1); -} - -template -inline T infinity() { - return std::max(std::numeric_limits::max(), - std::numeric_limits::infinity()); -} - -template -inline T negative_infinity() { - return std::min(-std::numeric_limits::infinity(), - std::numeric_limits::min()); -} - -} // namespace reduce_functions - -#define CALL_ALL_REDUCEOPS(func, ...) \ - func(Sum, functor::reduce_functions::zero, ##__VA_ARGS__) \ - func(Prod, functor::reduce_functions::one, ##__VA_ARGS__) func( \ - Max, functor::reduce_functions::negative_infinity, ##__VA_ARGS__) \ - func(Min, functor::reduce_functions::infinity, ##__VA_ARGS__) - -#define ReduceSliceFunctorReduceop(reduceop, dummy) \ - template \ - struct ReduceSliceFunctor##reduceop { \ - virtual ~ReduceSliceFunctor##reduceop() {} \ - virtual void operator()(OpKernelContext* ctx, const Device& d, \ - Index indices_width, \ - typename TTypes::ConstTensor indices, \ - typename TTypes::ConstTensor data, \ - typename TTypes::Tensor output); \ - }; - -CALL_ALL_REDUCEOPS(ReduceSliceFunctorReduceop) -#undef ReduceSliceFunctorReduceop - -} // namespace functor -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_REDUCE_SLICE_OPS_KERNELS_REDUCE_SLICE_OPS_H_ diff --git a/tensorflow/contrib/reduce_slice_ops/kernels/reduce_slice_ops_gpu.cu.cc b/tensorflow/contrib/reduce_slice_ops/kernels/reduce_slice_ops_gpu.cu.cc deleted file mode 100644 index 4ca7e835735..00000000000 --- a/tensorflow/contrib/reduce_slice_ops/kernels/reduce_slice_ops_gpu.cu.cc +++ /dev/null @@ -1,110 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#if GOOGLE_CUDA - -#define EIGEN_USE_GPU - -#include "tensorflow/contrib/reduce_slice_ops/kernels/reduce_slice_ops.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/util/gpu_kernel_helper.h" - -namespace tensorflow { - -using GPUDevice = Eigen::GpuDevice; - -namespace functor { - -#define Sum(a, b) ((a) + (b)) -#define Prod(a, b) ((a) * (b)) -#define Max(a, b) ((a) > (b) ? (a) : (b)) -#define Min(a, b) ((a) < (b) ? (a) : (b)) - -#define GPUReduceSliceFunctorReduceop(reduceop, beginning) \ - template \ - __global__ void ReduceSliceDeviceKernel##reduceop( \ - Gpu3DLaunchConfig config, Index indices_width, Index bound, \ - const T begin, const Index *indices, const T *input, T *out) { \ - CUDA_AXIS_KERNEL_LOOP(x, config.virtual_thread_count.x, X) { \ - CUDA_AXIS_KERNEL_LOOP(y, config.virtual_thread_count.y, Y) { \ - CUDA_AXIS_KERNEL_LOOP(z, config.virtual_thread_count.z, Z) { \ - Index outidx = x * config.virtual_thread_count.y * \ - config.virtual_thread_count.z + \ - y * config.virtual_thread_count.z + z; \ - out[outidx] = begin; \ - Index start = indices[y * indices_width]; \ - Index end = Min(bound, indices[y * indices_width + 1]); \ - for (Index yin = start; yin < end; yin++) { \ - Index inidx = x * bound * config.virtual_thread_count.z + \ - yin * config.virtual_thread_count.z + z; \ - out[outidx] = reduceop(out[outidx], input[inidx]); \ - } \ - } \ - } \ - } \ - } \ - \ - template \ - struct ReduceSliceFunctor##reduceop { \ - virtual ~ReduceSliceFunctor##reduceop() {} \ - virtual void operator()(OpKernelContext *ctx, const GPUDevice &d, \ - Index indices_width, \ - typename TTypes::ConstTensor indices, \ - typename TTypes::ConstTensor data, \ - typename TTypes::Tensor output) { \ - Index bound = data.dimension(1); \ - int sizex = output.dimension(0); \ - int sizey = output.dimension(1); \ - int sizez = output.dimension(2); \ - if (sizex * sizey * sizez == 0) { \ - return; \ - } \ - Gpu3DLaunchConfig config = GetGpu3DLaunchConfig( \ - sizex, sizey, sizez, d, ReduceSliceDeviceKernel##reduceop, \ - 0, 0); \ - \ - TF_CHECK_OK(GpuLaunchKernel( \ - ReduceSliceDeviceKernel##reduceop, config.block_count, \ - config.thread_per_block, 0, d.stream(), config, indices_width, \ - bound, beginning(), indices.data(), data.data(), output.data())); \ - } \ - }; - -CALL_ALL_REDUCEOPS(GPUReduceSliceFunctorReduceop) -#undef GPUReduceSliceFunctorReduceop - -#define DEFINE_GPU_REDUCEOP_SPECS_INDEX(reduceop, dummy, T) \ - template struct ReduceSliceFunctor##reduceop; \ - template struct ReduceSliceFunctor##reduceop; - -#define DEFINE_GPU_SPECS(T) \ - CALL_ALL_REDUCEOPS(DEFINE_GPU_REDUCEOP_SPECS_INDEX, T) - -TF_CALL_REAL_NUMBER_TYPES(DEFINE_GPU_SPECS) - -#undef DEFINE_GPU_REDUCEOP_SPECS_INDEX -#undef DEFINE_GPU_SPECS - -#undef Sum -#undef Prod -#undef Min -#undef Max - -} // namespace functor -} // namespace tensorflow - -#endif diff --git a/tensorflow/contrib/reduce_slice_ops/ops/reduce_slice_ops.cc b/tensorflow/contrib/reduce_slice_ops/ops/reduce_slice_ops.cc deleted file mode 100644 index 92879ab5356..00000000000 --- a/tensorflow/contrib/reduce_slice_ops/ops/reduce_slice_ops.cc +++ /dev/null @@ -1,282 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { - -using shape_inference::DimensionHandle; -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -namespace { - -Status ReduceSliceShapeFn(InferenceContext* c) { - ShapeHandle handle; - DimensionHandle dimhandle; - DimensionHandle dim_axis = c->UnknownDim(); - // "axis" must be a scala - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &handle)); - // "data" must have rank at least 1 - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &handle)); - // "indices" must have have rank 1 or rank 2 with the number of columns must - // be 2 - if (c->RankKnown(c->input(1))) { - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &handle)); - TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(1), 2, &handle)); - if (c->Rank(c->input(1)) == 1) { - // if "indices" is a vector of 0 elements, then the axis dimension of - // output tensor should be of dimension 0. - DimensionHandle raw_dim_axis; - TF_RETURN_IF_ERROR(c->Max(c->Dim(c->input(1), 0), 1, &raw_dim_axis)); - TF_RETURN_IF_ERROR(c->Subtract(raw_dim_axis, 1, &dim_axis)); - } else { // c->Rank(c->input(1)) == 2 - TF_RETURN_IF_ERROR( - c->Merge(c->Dim(c->input(1), 1), c->MakeDim(2), &dimhandle)); - dim_axis = c->Dim(c->input(1), 0); - } - } - // shape of output tensor - const Tensor* _axis = c->input_tensor(2); - if (nullptr == _axis) { - c->set_output(0, c->UnknownShapeOfRank(c->Rank(c->input(0)))); - } else { - int64 axis = _axis->scalar()(); - TF_RETURN_IF_ERROR(c->ReplaceDim(handle, axis, dim_axis, &handle)); - c->set_output(0, handle); - } - return Status::OK(); -} - -} // namespace - -REGISTER_OP("ReduceSliceSum") - .Input("data: T") - .Input("indices: Tindices") - .Input("axis: int64") - .Output("output: T") - .Attr("T: numbertype") - .Attr("Tindices: {int32,int64}") - .SetShapeFn(ReduceSliceShapeFn) - .Doc(R"doc( -Dynamically sum over the first dimension of a tensor according to start and end -indices specified at 'index'. - -For example: - -```prettyprint -# if 'data' is [[ 1, 2, 3] - [ 40, 50, 60] - [ 700, 800, 900] - [1000,2000,3000]], - -and 'indices' is [[0,1] - [1,1] - [0,2]], - -the output will be [[ 1, 2, 3] - [ 0, 0, 0] - [41,52,63]]. -``` - -The data must be at least rank 1. The indices must be of shape (?,2) where the -first column is start indices and the second column is end indices. The end indices -are not included in the reduce operation, which means, if you want to do a reduce -over indices 0,1,2, then you should have start index 0 and end index 3. If end -index is smaller than or equal to start, the result will be zero. If end index is -out of bounds, then the reduce operation will automatically stop at the bound, so -feel free to put a large number as your end of your index if you want to do the -reduction until the bound. - -data: The source of data where the computation will be taken from. -indices: start, end indices that controls which part to be included. -T: the type of data. -Tindices: the type of indices, must be int32 or int64. -output: the computed sum values. -)doc"); - -REGISTER_OP("ReduceSliceProd") - .Input("data: T") - .Input("indices: Tindices") - .Input("axis: int64") - .Output("output: T") - .Attr("T: numbertype") - .Attr("Tindices: {int32,int64}") - .SetShapeFn(ReduceSliceShapeFn) - .Doc(R"doc( -Dynamically compute the product over the first dimension of a tensor according -to start and end indices specified at 'indices'. - -For example: - -```prettyprint -# if 'data' is [[ 1, 2, 3] - [ 40, 50, 60] - [ 700, 800, 900] - [1000,2000,3000]], - -and 'indices' is [[0,1] - [1,1] - [0,2]], - -the output will be [[ 1, 2, 3] - [ 1, 1, 1] - [40,100,180]]. -``` - -The data must be at least rank 1. The indices can be of shape (?,2) where the -first column is start indices and the second column is end indices. The end indices -are not included in the reduce operation, which means, if you want to do a reduce -over indices 0,1,2, then you should have start index 0 and end index 3. If end -index is smaller than or equal to start, the result will be 1. If end index is -out of bounds, then the reduce operation will automatically stop at the bound, so -feel free to put a large number as your end of your index if you want to do the -reduction until the bound. The indices can also be of shape (?), in this case, the -start index of i will be the element at i, then end index of i will be the element -at i+1. That is: - -```prettyprint -indices = [0,5,11,115] - -is equivalent to - -indices = [ [0,5], - [5,11], - [11,115]] -``` - -data: The source of data where the computation will be taken from. -indices: start, end indices that controls which part to be included. -T: the type of data. -Tindices: the type of indices, must be int32 or int64. -output: the computed product values. -)doc"); - -REGISTER_OP("ReduceSliceMax") - .Input("data: T") - .Input("indices: Tindices") - .Input("axis: int64") - .Output("output: T") - .Attr("T: numbertype") - .Attr("Tindices: {int32,int64}") - .SetShapeFn(ReduceSliceShapeFn) - .Doc(R"doc( -Dynamically compute the maximum over the first dimension of a tensor according -to start and end indices specified at "indices". - -For example: - -```prettyprint -# if 'data' is [[ 1, 20, 3] - [ 400, 5, 60] - [ 70, 8, 900] - [1000,2000,3000]], - -and 'indices' is [[0,1] - [1,1] - [0,2]], - -the output will be [[ 1, 20, 3] - [ -BIG_VALUE, -BIG_VALUE, -BIG_VALUE] - [ 400, 20, 60]]. -``` - -The data must be at least rank 1. The indices can be of shape (?,2) where the -first column is start indices and the second column is end indices. The end indices -are not included in the reduce operation, which means, if you want to do a reduce -over indices 0,1,2, then you should have start index 0 and end index 3. If end -index is smaller than or equal to start, the result will be 1. If end index is -out of bounds, then the reduce operation will automatically stop at the bound, so -feel free to put a large number as your end of your index if you want to do the -reduction until the bound. The indices can also be of shape (?), in this case, the -start index of i will be the element at i, then end index of i will be the element -at i+1. That is: - -```prettyprint -indices = [0,5,11,115] - -is equivalent to - -indices = [ [0,5], - [5,11], - [11,115]] -``` - -data: The source of data where the computation will be taken from. -indices: start, end indices that controls which part to be included. -T: the type of data. -Tindices: the type of indices, must be int32 or int64. -output: the computed product values. -)doc"); - -REGISTER_OP("ReduceSliceMin") - .Input("data: T") - .Input("indices: Tindices") - .Input("axis: int64") - .Output("output: T") - .Attr("T: numbertype") - .Attr("Tindices: {int32,int64}") - .SetShapeFn(ReduceSliceShapeFn) - .Doc(R"doc( -Dynamically compute the minimum over the first dimension of a tensor according -to start and end indices specified at 'indices'. - -For example: - -```prettyprint -# if 'data' is [[ 1, 20, 3] - [ 400, 5, 60] - [ 70, 8, 900] - [1000,2000,3000]], - -and 'indices' is [[0,1] - [1,1] - [0,2]], - -the output will be [[ 1, 20, 3] - [ +BIG_VALUE, +BIG_VALUE, +BIG_VALUE] - [ 1, 5, 3]]. -``` - -The data must be at least rank 1. The indices can be of shape (?,2) where the -first column is start indices and the second column is end indices. The end indices -are not included in the reduce operation, which means, if you want to do a reduce -over indices 0,1,2, then you should have start index 0 and end index 3. If end -index is smaller than or equal to start, the result will be 1. If end index is -out of bounds, then the reduce operation will automatically stop at the bound, so -feel free to put a large number as your end of your index if you want to do the -reduction until the bound. The indices can also be of shape (?), in this case, the -start index of i will be the element at i, then end index of i will be the element -at i+1. That is: - -```prettyprint -indices = [0,5,11,115] - -is equivalent to - -indices = [ [0,5], - [5,11], - [11,115]] -``` - -data: The source of data where the computation will be taken from. -indices: start, end indices that controls which part to be included. -T: the type of data. -Tindices: the type of indices, must be int32 or int64. -output: the computed product values. -)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/reduce_slice_ops/ops/reduce_slice_ops_test.cc b/tensorflow/contrib/reduce_slice_ops/ops/reduce_slice_ops_test.cc deleted file mode 100644 index 777ad9bf150..00000000000 --- a/tensorflow/contrib/reduce_slice_ops/ops/reduce_slice_ops_test.cc +++ /dev/null @@ -1,41 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); - -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/shape_inference_testutil.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { - -TEST(ReduceSliceOpsTest, ReduceSliceSum_ShapeFn) { - ShapeInferenceTestOp op("ReduceSliceSum"); - INFER_OK(op, "?;?;?", "?"); - INFER_OK(op, "[10,20];[100,2];[]", "[?,?]"); - INFER_OK(op, "[10,20];[?,2];[]", "[?,?]"); - INFER_OK(op, "[10,20];[0];[]", "[?,?]"); - INFER_OK(op, "[10,20];[1];[]", "[?,?]"); - INFER_OK(op, "[10,20];[?];[]", "[?,?]"); - INFER_OK(op, "[?,?];[?,2];[]", "[?,?]"); - INFER_OK(op, "[?,?];[25,2];[]", "[?,?]"); - INFER_OK(op, "[?];[123,2];[]", "[?]"); - INFER_OK(op, "[1,2,3,4];[100,2];[]", "[?,?,?,?]"); - - INFER_ERROR("must be rank 0", op, "?;[?,2];[?]"); - INFER_ERROR("must be at least rank 1", op, "?;[];[]"); - INFER_ERROR("must be at most rank 2", op, "?;[1,2,3];[]"); - INFER_ERROR("must be equal, but are 1 and 2", op, "?;[?,1];[]"); - INFER_ERROR("must be at least rank 1", op, "[];?;[]"); -} - -} // end namespace tensorflow diff --git a/tensorflow/contrib/reduce_slice_ops/python/kernel_tests/reduce_slice_ops_test.py b/tensorflow/contrib/reduce_slice_ops/python/kernel_tests/reduce_slice_ops_test.py deleted file mode 100644 index 468886da200..00000000000 --- a/tensorflow/contrib/reduce_slice_ops/python/kernel_tests/reduce_slice_ops_test.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.contrib.reduce_slice_ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.reduce_slice_ops.python.ops import reduce_slice_ops -from tensorflow.python.framework.test_util import TensorFlowTestCase -from tensorflow.python.platform import googletest - - -class ReduceSliceTest(TensorFlowTestCase): - - def testReduceSliceSum1D(self): - x = np.array([1, 40, 700], dtype=np.int32) - indices = np.array([[0, 1], [0, 3], [1, 2], [1, 3], [0, 2]], dtype=np.int32) - result = np.array([1, 741, 40, 740, 41], dtype=np.int32) - with self.test_session(use_gpu=True): - y_tf = reduce_slice_ops.reduce_slice_sum(x, indices, 0).eval() - self.assertAllEqual(y_tf, result) - - def testReduceSliceSum2D(self): - x = np.array([[1, 2, 3], [40, 50, 60], [700, 800, 900]], dtype=np.int32) - indices = np.array([[0, 1], [0, 3], [1, 2], [1, 3], [0, 2]], dtype=np.int32) - result = np.array([[1, 2, 3], [741, 852, 963], [40, 50, 60], - [740, 850, 960], [41, 52, 63]], dtype=np.int32) - with self.test_session(use_gpu=True): - y_tf = reduce_slice_ops.reduce_slice_sum(x, indices, 0).eval() - self.assertAllEqual(y_tf, result) - - def testReduceSliceSum3D(self): - x = np.array([[[1, 2], [3, 4]], [[50, 60], [70, 80]], - [[600, 700], [800, 900]]], dtype=np.int32) - indices = np.array([[0, 1], [0, 3], [1, 2], [1, 3], [0, 2]], dtype=np.int32) - result = np.array([[[1, 2], [3, 4]], - [[651, 762], [873, 984]], - [[50, 60], [70, 80]], - [[650, 760], [870, 980]], - [[51, 62], [73, 84]]], dtype=np.int32) - with self.test_session(use_gpu=True): - y_tf = reduce_slice_ops.reduce_slice_sum(x, indices, 0).eval() - self.assertAllEqual(y_tf, result) - - def testReduceSliceSumAxis1(self): - x = np.transpose(np.array([[1, 2, 3], [40, 50, 60], - [700, 800, 900]], dtype=np.int32)) - indices = np.array([[0, 1], [0, 3], [1, 2], [1, 3], [0, 2]], dtype=np.int32) - result = np.transpose(np.array([[1, 2, 3], - [741, 852, 963], - [40, 50, 60], - [740, 850, 960], - [41, 52, 63]], dtype=np.int32)) - with self.test_session(use_gpu=True): - y_tf = reduce_slice_ops.reduce_slice_sum(x, indices, 1).eval() - self.assertAllEqual(y_tf, result) - - def testReduceSliceSum1DIndices(self): - x = np.array([[1, 2, 3], [40, 50, 60], [700, 800, 900], - [1000, 2000, 3000], [40000, 50000, 60000]], dtype=np.int32) - indices = np.array([0, 0, 2, 5], dtype=np.int32) - result = np.array([[0, 0, 0], [41, 52, 63], - [41700, 52800, 63900]], dtype=np.int32) - with self.test_session(use_gpu=True): - y_tf = reduce_slice_ops.reduce_slice_sum(x, indices, 0).eval() - self.assertAllEqual(y_tf, result) - - def testReduceSliceProd(self): - x = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.int32) - indices = np.array([[0, 1], [0, 3], [1, 2], [1, 3], [0, 2]], dtype=np.int32) - result = np.array([[1, 2, 3], [28, 80, 162], [4, 5, 6], - [28, 40, 54], [4, 10, 18]], dtype=np.int32) - with self.test_session(use_gpu=True): - y_tf = reduce_slice_ops.reduce_slice_prod(x, indices, 0).eval() - self.assertAllEqual(y_tf, result) - - def testReduceSliceMax(self): - x = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.int32) - indices = np.array([[0, 1], [0, 3], [1, 2], [1, 3], [0, 2]], dtype=np.int32) - result = np.array([[1, 2, 3], [7, 8, 9], [4, 5, 6], - [7, 8, 9], [4, 5, 6]], dtype=np.int32) - with self.test_session(use_gpu=True): - y_tf = reduce_slice_ops.reduce_slice_max(x, indices, 0).eval() - self.assertAllEqual(y_tf, result) - - def testReduceSliceMin(self): - x = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.int32) - indices = np.array([[0, 1], [0, 3], [1, 2], [1, 3], [0, 2]], dtype=np.int32) - result = np.array([[1, 2, 3], [1, 2, 3], [4, 5, 6], - [4, 5, 6], [1, 2, 3]], dtype=np.int32) - with self.test_session(use_gpu=True): - y_tf = reduce_slice_ops.reduce_slice_min(x, indices, 0).eval() - self.assertAllEqual(y_tf, result) - - def testReduceSliceEmptyDataRows(self): - x = np.empty((0, 1, 2, 3, 4, 5, 6), dtype=np.int32) - indices = np.array([[0, 1], [0, 3], [1, 2], [1, 3], [0, 2]], dtype=np.int32) - result = np.zeros((5, 1, 2, 3, 4, 5, 6), dtype=np.int32) - with self.test_session(use_gpu=True): - y_tf = reduce_slice_ops.reduce_slice_sum(x, indices, 0).eval() - self.assertAllEqual(y_tf, result) - - def testReduceSliceEmptyDataCols(self): - x = np.empty((100, 0, 2, 3, 4, 5, 6), dtype=np.int32) - indices = np.array([[0, 1], [0, 3], [1, 2], [1, 3], [0, 2]], dtype=np.int32) - result = np.empty((5, 0, 2, 3, 4, 5, 6), dtype=np.int32) - with self.test_session(use_gpu=True): - y_tf = reduce_slice_ops.reduce_slice_sum(x, indices, 0).eval() - self.assertAllEqual(y_tf, result) - - def testReduceSliceEmptyIndicesRows(self): - x = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.int32) - indices = np.empty((0, 2), dtype=np.int32) - result = np.empty((0, 3), dtype=np.int32) - with self.test_session(use_gpu=True): - y_tf = reduce_slice_ops.reduce_slice_sum(x, indices, 0).eval() - self.assertAllEqual(y_tf, result) - - def testReduceSliceEmpty0Indices1D(self): - x = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.int32) - indices = np.empty((0,), dtype=np.int32) - result = np.empty((0, 3), dtype=np.int32) - with self.test_session(use_gpu=True): - y_tf = reduce_slice_ops.reduce_slice_sum(x, indices, 0).eval() - self.assertAllEqual(y_tf, result) - - def testReduceSliceEmpty1Indices1D(self): - x = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.int32) - indices = np.array([0], dtype=np.int32) - result = np.empty((0, 3), dtype=np.int32) - with self.test_session(use_gpu=True): - y_tf = reduce_slice_ops.reduce_slice_sum(x, indices, 0).eval() - self.assertAllEqual(y_tf, result) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/reduce_slice_ops/python/ops/reduce_slice_ops.py b/tensorflow/contrib/reduce_slice_ops/python/ops/reduce_slice_ops.py deleted file mode 100644 index 12cbde31073..00000000000 --- a/tensorflow/contrib/reduce_slice_ops/python/ops/reduce_slice_ops.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python wrapper for the reduce slice operators.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.reduce_slice_ops.ops import gen_reduce_slice_ops -from tensorflow.contrib.util import loader -from tensorflow.python.platform import resource_loader - - -_reduce_slice_ops = loader.load_op_library( - resource_loader.get_path_to_datafile("_reduce_slice_ops.so")) - - -reduce_slice_sum = gen_reduce_slice_ops.reduce_slice_sum -reduce_slice_prod = gen_reduce_slice_ops.reduce_slice_prod -reduce_slice_max = gen_reduce_slice_ops.reduce_slice_max -reduce_slice_min = gen_reduce_slice_ops.reduce_slice_min diff --git a/tensorflow/contrib/remote_fused_graph/README.md b/tensorflow/contrib/remote_fused_graph/README.md deleted file mode 100644 index 267cfa10192..00000000000 --- a/tensorflow/contrib/remote_fused_graph/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Remote Fused Graph - -## Description - -This module contains libraries for remote fused graph utilities - -Maintainers: -- Satoshi Kataoka (satok@google.com, github.com/satok16) diff --git a/tensorflow/contrib/remote_fused_graph/pylib/BUILD b/tensorflow/contrib/remote_fused_graph/pylib/BUILD deleted file mode 100644 index 809adbd5a7f..00000000000 --- a/tensorflow/contrib/remote_fused_graph/pylib/BUILD +++ /dev/null @@ -1,51 +0,0 @@ -# Description: -# Contains ops for remote fused graph - -load("//tensorflow:tensorflow.bzl", "py_test") -load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_gen_op_wrapper_py( - name = "gen_remote_fused_graph_ops", - out = "python/ops/gen_remote_fused_graph_ops.py", - deps = [ - "//tensorflow/core:remote_fused_graph_ops_op_lib", - ], -) - -py_library( - name = "remote_fused_graph_ops_py", - srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - srcs_version = "PY2AND3", - deps = [ - ":gen_remote_fused_graph_ops", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:util", - "//third_party/py/numpy", - ], -) - -py_test( - name = "remote_fused_graph_ops_test", - size = "small", - srcs = ["python/ops/remote_fused_graph_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":remote_fused_graph_ops_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/remote_fused_graph/pylib/__init__.py b/tensorflow/contrib/remote_fused_graph/pylib/__init__.py deleted file mode 100644 index 4d23c38932e..00000000000 --- a/tensorflow/contrib/remote_fused_graph/pylib/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Remote fused graph ops python library. - -## This package provides classes for remote fused graph ops. - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import, line-too-long -from tensorflow.contrib.remote_fused_graph.pylib.python.ops.remote_fused_graph_ops import * -# pylint: enable=unused-import,wildcard-import,line-too-long - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = ['remote_fused_graph_execute'] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/remote_fused_graph/pylib/python/__init__.py b/tensorflow/contrib/remote_fused_graph/pylib/python/__init__.py deleted file mode 100644 index b66091f8759..00000000000 --- a/tensorflow/contrib/remote_fused_graph/pylib/python/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Remote fused graph ops python library.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/remote_fused_graph/pylib/python/ops/__init__.py b/tensorflow/contrib/remote_fused_graph/pylib/python/ops/__init__.py deleted file mode 100644 index b66091f8759..00000000000 --- a/tensorflow/contrib/remote_fused_graph/pylib/python/ops/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Remote fused graph ops python library.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/remote_fused_graph/pylib/python/ops/remote_fused_graph_ops.py b/tensorflow/contrib/remote_fused_graph/pylib/python/ops/remote_fused_graph_ops.py deleted file mode 100644 index 7e79785d286..00000000000 --- a/tensorflow/contrib/remote_fused_graph/pylib/python/ops/remote_fused_graph_ops.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Operations to execute a subgraph on a remote processor.""" - -# pylint: disable=g-bad-name -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import, line-too-long -from tensorflow.contrib.remote_fused_graph.pylib.python.ops import gen_remote_fused_graph_ops -from tensorflow.core.framework import remote_fused_graph_execute_info_pb2 as info_pb2 -# pylint: enable=unused-import,wildcard-import,line-too-long - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops - -# RemoteFusedGraphExecute is not differenciable op. -ops.NotDifferentiable("RemoteFusedGraphExecute") - - -def remote_fused_graph_execute(inputs, - output_types, - graph_def, - graph_input_node_names, - graph_output_node_names, - executor_name, - serialized_executor_parameters, - default_graph_input_tensor_type_shapes=None, - default_graph_output_tensor_type_shapes=None): - """A wrapper for remote_fused_graph_execute.""" - info_proto = info_pb2.RemoteFusedGraphExecuteInfo() - info_proto.remote_graph.CopyFrom(graph_def) - info_proto.graph_input_node_name.extend(graph_input_node_names) - info_proto.graph_output_node_name.extend(graph_output_node_names) - info_proto.executor_name = executor_name - info_proto.serialized_executor_parameters = serialized_executor_parameters - if default_graph_input_tensor_type_shapes: - for type_shape in default_graph_input_tensor_type_shapes: - type_shape_proto = info_proto.default_graph_input_tensor_shape.add() - type_shape_proto.dtype = dtypes.as_dtype(type_shape[0]).as_datatype_enum - for dim in type_shape[1]: - type_shape_proto.shape.dim.add().size = dim - if default_graph_output_tensor_type_shapes: - for type_shape in default_graph_output_tensor_type_shapes: - type_shape_proto = info_proto.default_graph_output_tensor_shape.add() - type_shape_proto.dtype = dtypes.as_dtype(type_shape[0]).as_datatype_enum - for dim in type_shape[1]: - type_shape_proto.shape.dim.add().size = dim - - serialized_info = info_proto.SerializeToString() - - return gen_remote_fused_graph_ops.remote_fused_graph_execute( - inputs, output_types, serialized_info) diff --git a/tensorflow/contrib/remote_fused_graph/pylib/python/ops/remote_fused_graph_ops_test.py b/tensorflow/contrib/remote_fused_graph/pylib/python/ops/remote_fused_graph_ops_test.py deleted file mode 100644 index 45df9091482..00000000000 --- a/tensorflow/contrib/remote_fused_graph/pylib/python/ops/remote_fused_graph_ops_test.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.ops.remote_fused_graph_ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -# pylint: disable=unused-import,wildcard-import, line-too-long -from tensorflow.contrib.remote_fused_graph.pylib.python.ops import remote_fused_graph_ops -# pylint: enable=unused-import,wildcard-import,line-too-long - -from tensorflow.core.framework import graph_pb2 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.platform import googletest - - -class RemoteFusedGraphExecuteTest(test_util.TensorFlowTestCase): - """Tests for RemoteFusedGraphExecute op.""" - - def testBuild(self): - graph = graph_pb2.GraphDef() - node = graph.node.add() - node.name = "a" - node.op = "op0" - node = graph.node.add() - node.name = "b" - node.op = "op1" - inputs = [ops.convert_n_to_tensor([1], dtypes.int64)] - output_types = [np.int64, np.int64] - graph_input_node_names = ["a"] - graph_output_node_names = ["a", "b"] - executor_name = "" - serialized_executor_parameters = b"" - default_graph_input_tensor_type_shapes = [[dtypes.int64, [1]]] - default_graph_output_tensor_type_shapes = [[dtypes.int64, [1]], - [dtypes.int64, [1]]] - - output_nodes = remote_fused_graph_ops.remote_fused_graph_execute( - inputs, output_types, graph, graph_input_node_names, - graph_output_node_names, executor_name, serialized_executor_parameters, - default_graph_input_tensor_type_shapes, - default_graph_output_tensor_type_shapes) - self.assertEqual(2, len(output_nodes)) - for output_node in output_nodes: - with self.test_session(use_gpu=False): - output_node.eval() - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/resampler/BUILD b/tensorflow/contrib/resampler/BUILD deleted file mode 100644 index 2f5bb21452c..00000000000 --- a/tensorflow/contrib/resampler/BUILD +++ /dev/null @@ -1,125 +0,0 @@ -load( - "//tensorflow:tensorflow.bzl", - "tf_custom_op_library", - "tf_gen_op_libs", - "tf_gen_op_wrapper_py", - "tf_kernel_library", -) -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") -load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 License -) - -exports_files(["LICENSE"]) - -tf_custom_op_py_library( - name = "resampler_py", - srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - dso = [":python/ops/_resampler_ops.so"], - kernels = [ - ":resampler_ops_kernels", - ], - visibility = ["//visibility:public"], - deps = [ - ":resampler_ops", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", - "//tensorflow/python:platform", - "//tensorflow/python:util", - "//third_party/py/numpy", - ], -) - -tf_kernel_library( - name = "resampler_ops_kernels", - srcs = [ - "kernels/resampler_ops.cc", - "kernels/resampler_ops.h", - ], - gpu_srcs = [ - "kernels/resampler_ops_gpu.cu.cc", - "kernels/resampler_ops.h", - ], - prefix = "resampler_ops", - deps = [ - ":resampler_ops_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - ] + select({ - "//tensorflow:with_xla_support": [ - "//tensorflow/compiler/tf2xla/kernels:resampler_ops", - ], - "//conditions:default": [], - }), - alwayslink = 1, -) - -tf_custom_op_library( - name = "python/ops/_resampler_ops.so", - srcs = [ - "kernels/resampler_ops.cc", - "kernels/resampler_ops.h", - "ops/resampler_ops.cc", - ], - gpu_srcs = [ - "kernels/resampler_ops_gpu.cu.cc", - "kernels/resampler_ops.h", - ], -) - -tf_gen_op_libs( - op_lib_names = [ - "resampler_ops", - ], -) - -tf_gen_op_wrapper_py( - name = "resampler_ops", - deps = [":resampler_ops_op_lib"], -) - -cuda_py_test( - name = "resampler_ops_test", - size = "small", - srcs = ["python/ops/resampler_ops_test.py"], - additional_deps = [ - ":resampler_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:array_ops", - ], -) - -tf_xla_py_test( - name = "resampler_ops_xla_test", - size = "small", - srcs = ["xla/resampler_ops_xla_test.py"], - disabled_backends = [ - # TODO(b/74459949) Support BatchDot in CPU backend. - "cpu", - "cpu_ondemand", - ], - # TODO(b/112295522): the OSS build will not likely work in the short to medium term, currently it is blocked by the fact that bazel does not allow py_library to depend on cc_library: https://github.com/bazelbuild/bazel/issues/701 which may not be resolvable. - tags = ["no_oss"], - deps = [ - "//tensorflow/compiler/tests:xla_test", - "//tensorflow/compiler/tf2xla/kernels:resampler_ops", - "//tensorflow/contrib/resampler:resampler_ops", - "//tensorflow/contrib/resampler:resampler_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:platform_test", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/resampler/__init__.py b/tensorflow/contrib/resampler/__init__.py deleted file mode 100644 index 3e04e5762da..00000000000 --- a/tensorflow/contrib/resampler/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops and modules related to resampler.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -# pylint: disable=wildcard-import -from tensorflow.contrib.resampler.python.ops.resampler_ops import * -from tensorflow.python.util.all_util import remove_undocumented - -remove_undocumented(__name__, ["resampler"]) diff --git a/tensorflow/contrib/resampler/kernels/resampler_ops.cc b/tensorflow/contrib/resampler/kernels/resampler_ops.cc deleted file mode 100644 index 63c72836d79..00000000000 --- a/tensorflow/contrib/resampler/kernels/resampler_ops.cc +++ /dev/null @@ -1,416 +0,0 @@ -// Copyright 2017 The Sonnet Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#define EIGEN_USE_THREADS - -#include "tensorflow/contrib/resampler/kernels/resampler_ops.h" - -#include -#include -#include - -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -using CPUDevice = Eigen::ThreadPoolDevice; -using GPUDevice = Eigen::GpuDevice; - -namespace functor { - -template -struct Resampler2DFunctor { - void operator()(::tensorflow::OpKernelContext* ctx, const CPUDevice& d, - const T* __restrict__ data, const T* __restrict__ warp, - T* __restrict__ output, const int batch_size, - const int data_height, const int data_width, - const int data_channels, const int num_sampling_points) { - const int warp_batch_stride = num_sampling_points * 2; - const int data_batch_stride = data_height * data_width * data_channels; - const int output_batch_stride = num_sampling_points * data_channels; - const T zero = static_cast(0.0); - const T one = static_cast(1.0); - - auto resample_batches = [&](const int start, const int limit) { - for (int batch_id = start; batch_id < limit; ++batch_id) { - // Utility lambda to access data point and set output values. - // The functions take care of performing the relevant pointer - // arithmetics abstracting away the low level details in the - // main loop over samples. Note that data is stored in NHWC format. - auto set_output = [&](const int sample_id, const int channel, - const T value) { - output[batch_id * output_batch_stride + sample_id * data_channels + - channel] = value; - }; - - auto get_data_point = [&](const int x, const int y, const int chan) { - const bool point_is_in_range = - (x >= 0 && y >= 0 && x <= data_width - 1 && y <= data_height - 1); - return point_is_in_range - ? data[batch_id * data_batch_stride + - data_channels * (y * data_width + x) + chan] - : zero; - }; - - for (int sample_id = 0; sample_id < num_sampling_points; ++sample_id) { - const T x = warp[batch_id * warp_batch_stride + sample_id * 2]; - const T y = warp[batch_id * warp_batch_stride + sample_id * 2 + 1]; - // The interpolation function: - // a) implicitly pads the input data with 0s (hence the unusual checks - // with {x,y} > -1) - // b) returns 0 when sampling outside the (padded) image. - // The effect is that the sampled signal smoothly goes to 0 outside - // the original input domain, rather than presenting a jump - // discontinuity at the image boundaries. - if (x > static_cast(-1.0) && y > static_cast(-1.0) && - x < static_cast(data_width) && - y < static_cast(data_height)) { - // Precompute floor (f) and ceil (c) values for x and y. - const int fx = std::floor(static_cast(x)); - const int fy = std::floor(static_cast(y)); - const int cx = fx + 1; - const int cy = fy + 1; - const T dx = static_cast(cx) - x; - const T dy = static_cast(cy) - y; - - for (int chan = 0; chan < data_channels; ++chan) { - const T img_fxfy = dx * dy * get_data_point(fx, fy, chan); - const T img_cxcy = - (one - dx) * (one - dy) * get_data_point(cx, cy, chan); - const T img_fxcy = dx * (one - dy) * get_data_point(fx, cy, chan); - const T img_cxfy = (one - dx) * dy * get_data_point(cx, fy, chan); - set_output(sample_id, chan, - img_fxfy + img_cxcy + img_fxcy + img_cxfy); - } - } else { - for (int chan = 0; chan < data_channels; ++chan) { - set_output(sample_id, chan, zero); - } - } - } - } - }; - // Rough estimate of work for each batch entry. - // From third_party/tensorflow/core/util/work_sharder.cc we gather that an - // estimate of the cost of each work unit is needed to correctly shard the - // workload. Shard assumes each cost unit is 1ns, minimum cost per shard - // being 10us. - const int64 cost = - static_cast(num_sampling_points) * data_channels * 1000; - auto worker_threads = *(ctx->device()->tensorflow_cpu_worker_threads()); - ::tensorflow::Shard(worker_threads.num_threads, worker_threads.workers, - batch_size, cost, resample_batches); - } -}; - -} // namespace functor - -template -class ResamplerOp : public ::tensorflow::OpKernel { - public: - explicit ResamplerOp(::tensorflow::OpKernelConstruction* context) - : ::tensorflow::OpKernel(context) {} - - void Compute(::tensorflow::OpKernelContext* ctx) override { - const ::tensorflow::Tensor& data = ctx->input(0); - const ::tensorflow::Tensor& warp = ctx->input(1); - - const ::tensorflow::TensorShape& data_shape = data.shape(); - OP_REQUIRES(ctx, data_shape.dims() == 4, - ::tensorflow::errors::Unimplemented( - "Only bilinear interpolation is currently supported. The " - "input data shape must be [batch_size, data_height, " - "data_width, data_channels], but is: ", - data_shape.DebugString())); - const ::tensorflow::TensorShape& warp_shape = warp.shape(); - OP_REQUIRES(ctx, - ::tensorflow::TensorShapeUtils::IsMatrixOrHigher(warp_shape), - ::tensorflow::errors::InvalidArgument( - "warp should be at least a matrix, got shape ", - warp_shape.DebugString())); - OP_REQUIRES(ctx, warp_shape.dim_size(warp_shape.dims() - 1) == 2, - ::tensorflow::errors::Unimplemented( - "Only bilinear interpolation is supported, warping " - "coordinates must be 2D; warp shape last entry should be " - "2, but shape vector is: ", - warp_shape.DebugString())); - OP_REQUIRES(ctx, data_shape.dim_size(0) == warp_shape.dim_size(0), - ::tensorflow::errors::InvalidArgument( - "Batch size of data and warp tensor must be the same, but " - "input shapes are: ", - data_shape.DebugString(), ", ", warp_shape.DebugString())); - const int batch_size = data_shape.dim_size(0); - const int data_height = data_shape.dim_size(1); - const int data_width = data_shape.dim_size(2); - const int data_channels = data_shape.dim_size(3); - ::tensorflow::TensorShape output_shape = warp.shape(); - output_shape.set_dim(output_shape.dims() - 1, data_channels); - const int num_sampling_points = warp.NumElements() / batch_size / 2; - ::tensorflow::Tensor* output = nullptr; - OP_REQUIRES_OK(ctx, ctx->allocate_output(0, output_shape, &output)); - - // Execute kernel only for nonempty output; otherwise Eigen crashes on GPU. - if (num_sampling_points > 0) { - functor::Resampler2DFunctor()( - ctx, ctx->eigen_device(), data.flat().data(), - warp.flat().data(), output->flat().data(), batch_size, - data_height, data_width, data_channels, num_sampling_points); - } - } - - private: - TF_DISALLOW_COPY_AND_ASSIGN(ResamplerOp); -}; - -#define REGISTER(TYPE) \ - REGISTER_KERNEL_BUILDER( \ - Name("Resampler").Device(DEVICE_CPU).TypeConstraint("T"), \ - ResamplerOp); - -TF_CALL_half(REGISTER); -TF_CALL_float(REGISTER); -TF_CALL_double(REGISTER); -#undef REGISTER - -#if GOOGLE_CUDA -#define REGISTER(TYPE) \ - REGISTER_KERNEL_BUILDER( \ - Name("Resampler").Device(DEVICE_GPU).TypeConstraint("T"), \ - ResamplerOp) -TF_CALL_float(REGISTER); -TF_CALL_double(REGISTER); -#undef REGISTER -#endif // GOOGLE_CUDA - -namespace functor { - -template -struct ResamplerGrad2DFunctor { - void operator()(::tensorflow::OpKernelContext* ctx, const CPUDevice& d, - const T* __restrict__ data, const T* __restrict__ warp, - const T* __restrict__ grad_output, T* __restrict__ grad_data, - T* __restrict__ grad_warp, const int batch_size, - const int data_height, const int data_width, - const int data_channels, const int num_sampling_points) { - // Set gradients to 0, because the kernel incrementally updates the - // tensor entries by adding partial contributions. - const int resampler_output_size = - batch_size * num_sampling_points * data_channels; - const int grad_warp_size = resampler_output_size / data_channels * 2; - const int grad_data_size = - data_height * data_width * data_channels * batch_size; - memset(grad_data, 0, sizeof(T) * grad_data_size); - memset(grad_warp, 0, sizeof(T) * grad_warp_size); - - const auto&& data_batch_stride = data_height * data_width * data_channels; - const auto&& warp_batch_stride = num_sampling_points * 2; - const int output_batch_stride = num_sampling_points * data_channels; - const T zero = static_cast(0.0); - const T one = static_cast(1.0); - - auto update_grads_for_batches = [&](const int start, const int limit) { - for (int batch_id = start; batch_id < limit; ++batch_id) { - // Utility lambdas to access data and update gradient tensors. - // The functions take care of performing the relevant pointer - // arithmetics abstracting away the low level details in the - // main loop over samples. Note that data is stored in NHWC format. - auto get_data_point = [&](const int x, const int y, const int chan) { - const bool point_is_in_range = - (x >= 0 && y >= 0 && x <= data_width - 1 && y <= data_height - 1); - return point_is_in_range - ? data[batch_id * data_batch_stride + - data_channels * (y * data_width + x) + chan] - : zero; - }; - - auto update_grad_data = [&](const int x, const int y, const int chan, - const T value) { - const bool point_is_in_range = - (x >= 0 && y >= 0 && x <= data_width - 1 && y <= data_height - 1); - if (point_is_in_range) { - grad_data[batch_id * data_batch_stride + - data_channels * (y * data_width + x) + chan] += value; - } - }; - - auto update_grad_warp = [&](const int sample_id, const int channel, - const T value) { - grad_warp[batch_id * warp_batch_stride + sample_id * 2 + channel] += - value; - }; - - for (int sample_id = 0; sample_id < num_sampling_points; ++sample_id) { - const T x = warp[batch_id * warp_batch_stride + sample_id * 2]; - const T y = warp[batch_id * warp_batch_stride + sample_id * 2 + 1]; - // The interpolation function whose gradient this function implements: - // a) implicitly pads the input data with 0s (hence the unusual checks - // with {x,y} > -1) - // b) returns 0 when sampling outside the (padded) image. - // The effect is that the sampled signal smoothly goes to 0 outside - // the original input domain, rather than presenting a jump - // discontinuity at the image boundaries. - if (x > static_cast(-1.0) && y > static_cast(-1.0) && - x < static_cast(data_width) && - y < static_cast(data_height)) { - // Precompute floor (f) and ceil (c) values for x and y. - const int fx = std::floor(static_cast(x)); - const int fy = std::floor(static_cast(y)); - const int cx = fx + 1; - const int cy = fy + 1; - const T dx = static_cast(cx) - x; - const T dy = static_cast(cy) - y; - - for (int chan = 0; chan < data_channels; ++chan) { - const T grad_output_value = - grad_output[batch_id * output_batch_stride + - sample_id * data_channels + chan]; - const T img_fxfy = get_data_point(fx, fy, chan); - const T img_cxcy = get_data_point(cx, cy, chan); - const T img_fxcy = get_data_point(fx, cy, chan); - const T img_cxfy = get_data_point(cx, fy, chan); - - // Update partial gradients wrt relevant warp field entries - update_grad_warp( - sample_id, 0, - grad_output_value * ((one - dy) * (img_cxcy - img_fxcy) + - dy * (img_cxfy - img_fxfy))); - - update_grad_warp( - sample_id, 1, - grad_output_value * ((one - dx) * (img_cxcy - img_cxfy) + - dx * (img_fxcy - img_fxfy))); - - // Update partial gradients wrt sampled data - update_grad_data(fx, fy, chan, grad_output_value * dx * dy); - update_grad_data(cx, cy, chan, - grad_output_value * (one - dx) * (one - dy)); - update_grad_data(fx, cy, chan, - grad_output_value * dx * (one - dy)); - update_grad_data(cx, fy, chan, - grad_output_value * (one - dx) * dy); - } - } - } - } - }; - // Rough estimate of work for each batch entry. - // From third_party/tensorflow/core/util/work_sharder.cc we gather that an - // estimate of the cost of each work unit is needed to correctly shard the - // workload. Shard assumes each cost unit is 1ns, minimum cost per shard - // being 10us. - // TODO(fviola): Check out if there is a better way of doing this. - auto worker_threads = *(ctx->device()->tensorflow_cpu_worker_threads()); - const int64 cost = - static_cast(num_sampling_points) * data_channels * 1000; - ::tensorflow::Shard(worker_threads.num_threads, worker_threads.workers, - batch_size, cost, update_grads_for_batches); - } -}; - -} // namespace functor - -template -class ResamplerGradOp : public ::tensorflow::OpKernel { - public: - explicit ResamplerGradOp(::tensorflow::OpKernelConstruction* context) - : ::tensorflow::OpKernel(context) {} - - void Compute(::tensorflow::OpKernelContext* ctx) override { - const ::tensorflow::Tensor& data = ctx->input(0); - const ::tensorflow::Tensor& warp = ctx->input(1); - const ::tensorflow::Tensor& grad_output = ctx->input(2); - - const ::tensorflow::TensorShape& data_shape = data.shape(); - OP_REQUIRES(ctx, data_shape.dims() == 4, - ::tensorflow::errors::Unimplemented( - "Only bilinear interpolation is supported, the input data " - "tensor must be a batch of 2d data; data shape should have " - "4 entries corresponding to [batch_size, data_height, " - "data_width, data_channels], but is: ", - data_shape.DebugString())); - const int batch_size = data_shape.dim_size(0); - const int data_height = data_shape.dim_size(1); - const int data_width = data_shape.dim_size(2); - const int data_channels = data_shape.dim_size(3); - const ::tensorflow::TensorShape& warp_shape = warp.shape(); - OP_REQUIRES(ctx, - ::tensorflow::TensorShapeUtils::IsMatrixOrHigher(warp_shape), - ::tensorflow::errors::InvalidArgument( - "warp should be at least a matrix, got shape ", - warp_shape.DebugString())); - OP_REQUIRES(ctx, warp_shape.dim_size(warp_shape.dims() - 1) == 2, - ::tensorflow::errors::Unimplemented( - "Only bilinear interpolation is supported, warping " - "coordinates must be 2D; warp shape last entry should be " - "2, but shape vector is: ", - warp_shape.DebugString())); - const ::tensorflow::TensorShape& grad_output_shape = grad_output.shape(); - ::tensorflow::TensorShape resampler_output_shape = warp.shape(); - resampler_output_shape.set_dim(resampler_output_shape.dims() - 1, - data_channels); - OP_REQUIRES(ctx, grad_output_shape == resampler_output_shape, - ::tensorflow::errors::InvalidArgument( - "grad_output shape is not consistent with data and warp " - "shapes; it should be ", - resampler_output_shape.DebugString(), " but is ", - grad_output_shape.DebugString())); - const int num_sampling_points = warp.NumElements() / batch_size / 2; - ::tensorflow::Tensor* grad_data = nullptr; - ::tensorflow::Tensor* grad_warp = nullptr; - OP_REQUIRES_OK(ctx, ctx->allocate_output(0, data.shape(), &grad_data)); - OP_REQUIRES_OK(ctx, ctx->allocate_output(1, warp.shape(), &grad_warp)); - // Execute kernel only for nonempty output; otherwise Eigen crashes on GPU. - if (num_sampling_points > 0) { - functor::ResamplerGrad2DFunctor()( - ctx, ctx->eigen_device(), data.flat().data(), - warp.flat().data(), grad_output.flat().data(), - grad_data->flat().data(), grad_warp->flat().data(), batch_size, - data_height, data_width, data_channels, num_sampling_points); - } - } - - private: - TF_DISALLOW_COPY_AND_ASSIGN(ResamplerGradOp); -}; - -#define REGISTER(TYPE) \ - REGISTER_KERNEL_BUILDER( \ - Name("ResamplerGrad").Device(DEVICE_CPU).TypeConstraint("T"), \ - ResamplerGradOp); - -TF_CALL_half(REGISTER); -TF_CALL_float(REGISTER); -TF_CALL_double(REGISTER); -#undef REGISTER - -#if GOOGLE_CUDA -#define REGISTER(TYPE) \ - REGISTER_KERNEL_BUILDER( \ - Name("ResamplerGrad").Device(DEVICE_GPU).TypeConstraint("T"), \ - ResamplerGradOp) -// Disable half and double precision since atomicAdds are not supported -// TF_CALL_half(REGISTER); -// TF_CALL_double(REGISTER); -TF_CALL_float(REGISTER); - -#undef REGISTER -#endif // GOOGLE_CUDA - -} // namespace tensorflow diff --git a/tensorflow/contrib/resampler/kernels/resampler_ops.h b/tensorflow/contrib/resampler/kernels/resampler_ops.h deleted file mode 100644 index 7fe3b9c0df7..00000000000 --- a/tensorflow/contrib/resampler/kernels/resampler_ops.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 The Sonnet Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#ifndef TENSORFLOW_CONTRIB_RESAMPLER_KERNELS_RESAMPLER_OPS_H_ -#define TENSORFLOW_CONTRIB_RESAMPLER_KERNELS_RESAMPLER_OPS_H_ - -#if PLATFORM_WINDOWS -#define __restrict__ __restrict -#endif - -namespace tensorflow { -class OpKernelContext; -} - -namespace tensorflow { -namespace functor { - -// Helper functor for the Resampler Op in 2D -template -struct Resampler2DFunctor { - void operator()(::tensorflow::OpKernelContext* ctx, const Device& d, - const T* __restrict__ data, const T* __restrict__ warp, - T* __restrict__ output, const int batch_size, - const int data_height, const int data_width, - const int data_channels, const int num_sampling_points); -}; - -// Helper functor for the Resampler Gradient Op in 2D -template -struct ResamplerGrad2DFunctor { - void operator()(::tensorflow::OpKernelContext* ctx, const Device& d, - const T* __restrict__ data, const T* __restrict__ warp, - const T* __restrict__ grad_output, T* __restrict__ grad_data, - T* __restrict__ grad_warp, const int batch_size, - const int data_height, const int data_width, - const int data_channels, const int num_sampling_points); -}; - -} // namespace functor -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_RESAMPLER_KERNELS_RESAMPLER_OPS_H_ diff --git a/tensorflow/contrib/resampler/kernels/resampler_ops_gpu.cu.cc b/tensorflow/contrib/resampler/kernels/resampler_ops_gpu.cu.cc deleted file mode 100644 index cec2d4c21c9..00000000000 --- a/tensorflow/contrib/resampler/kernels/resampler_ops_gpu.cu.cc +++ /dev/null @@ -1,283 +0,0 @@ -// Copyright 2016 The Sonnet Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#if GOOGLE_CUDA - -#define EIGEN_USE_GPU - -#include - -#include - -#include "tensorflow/contrib/resampler/kernels/resampler_ops.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/util/gpu_kernel_helper.h" - -namespace tensorflow { - -using GPUDevice = Eigen::GpuDevice; - -namespace { - -#define GET_DATA_POINT(x, y) \ - data[batch_id * data_batch_stride + data_channels * (y * data_width + x) + \ - chan] - -template -__global__ void Resampler2DKernel(const T* __restrict__ data, - const T* __restrict__ warp, - T* __restrict__ output, const int batch_size, - const int data_height, const int data_width, - const int data_channels, - const int num_sampling_points) { - const int output_data_size = batch_size * num_sampling_points * data_channels; - CUDA_1D_KERNEL_LOOP(index, output_data_size) { - const int out_index = index; - - // Get (idxSample, channel, point) from the index. - // Use this formula - // index = batch_id * num_sampling_points * num_chans + - // sample_id * num_chans + chan_id, - // with sample_id = [0, ... ,num_sampling_points) - const int data_batch_stride = data_height * data_width * data_channels; - const int warp_batch_stride = num_sampling_points * 2; - const int output_batch_stride = num_sampling_points * data_channels; - - const int batch_id = index / output_batch_stride; - const int index_in_batch = index % output_batch_stride; - const int chan = index_in_batch % data_channels; - const int sample_id = index_in_batch / data_channels; - - // Get coords of 2D point where data will be resampled - const T x = warp[batch_id * warp_batch_stride + sample_id * 2]; - const T y = warp[batch_id * warp_batch_stride + sample_id * 2 + 1]; - const T zero = static_cast(0.0); - const T one = static_cast(1.0); - // The interpolation function: - // a) implicitly pads the input data with 0s (hence the unusual checks - // with {x,y} > -1) - // b) returns 0 when sampling outside the (padded) image. - // The effect is that the sampled signal smoothly goes to 0 outside - // the original input domain, rather than presenting a jump - // discontinuity at the image boundaries. - if (x > static_cast(-1.0) && y > static_cast(-1.0) && - x < static_cast(data_width) && y < static_cast(data_height)) { - // Precompute floor (f) and ceil (c) values for x and y. - const int fx = std::floor(static_cast(x)); - const int fy = std::floor(static_cast(y)); - const int cx = fx + 1; - const int cy = fy + 1; - const T dx = static_cast(cx) - x; - const T dy = static_cast(cy) - y; - - const T img_fxfy = - (fx >= 0 && fy >= 0) ? dx * dy * GET_DATA_POINT(fx, fy) : zero; - - const T img_cxcy = (cx <= data_width - 1 && cy <= data_height - 1) - ? (one - dx) * (one - dy) * GET_DATA_POINT(cx, cy) - : zero; - - const T img_fxcy = (fx >= 0 && cy <= data_height - 1) - ? dx * (one - dy) * GET_DATA_POINT(fx, cy) - : zero; - - const T img_cxfy = (cx <= data_width - 1 && fy >= 0) - ? (one - dx) * dy * GET_DATA_POINT(cx, fy) - : zero; - - output[out_index] = img_fxfy + img_cxcy + img_fxcy + img_cxfy; - } else { - output[out_index] = zero; - } - } -} - -} // namespace - -namespace functor { - -template -struct Resampler2DFunctor { - void operator()(::tensorflow::OpKernelContext* ctx, const GPUDevice& d, - const T* __restrict__ data, const T* __restrict__ warp, - T* __restrict__ output, const int batch_size, - const int data_height, const int data_width, - const int data_channels, const int num_sampling_points) { - const int output_data_size = - batch_size * num_sampling_points * data_channels; - ::tensorflow::GpuLaunchConfig config = - ::tensorflow::GetGpuLaunchConfig(output_data_size, d); - TF_CHECK_OK(GpuLaunchKernel( - Resampler2DKernel, config.block_count, config.thread_per_block, 0, - d.stream(), data, warp, output, batch_size, data_height, data_width, - data_channels, num_sampling_points)); - } -}; - -// TODO(fviola): gcudacc fails at compile time with Eigen::half. -// template struct Resampler2DFunctor; -template struct Resampler2DFunctor; -template struct Resampler2DFunctor; - -} // namespace functor - -namespace { - -#define UPDATE_GRAD_DATA_POINT(x, y, v) \ - atomicAdd(grad_data + (batch_id * data_batch_stride + \ - data_channels * (y * data_width + x) + chan), \ - v) - -template -__global__ void ResamplerGrad2DKernel( - const T* __restrict__ data, const T* __restrict__ warp, - const T* __restrict__ grad_output, T* __restrict__ grad_data, - T* __restrict__ grad_warp, const int batch_size, const int data_height, - const int data_width, const int data_channels, - const int num_sampling_points) { - const int resampler_output_size = - batch_size * num_sampling_points * data_channels; - CUDA_1D_KERNEL_LOOP(index, resampler_output_size) { - const int out_index = index; - - // Get (idxSample, channel, point) from the index. - // Use this formula - // index = batch_id * num_sampling_points * num_chans + - // sample_id * num_chans + chan_id, - // with sample_id = [0, ... ,num_sampling_points) - const int data_batch_stride = data_height * data_width * data_channels; - const int warp_batch_stride = num_sampling_points * 2; - const int output_batch_stride = num_sampling_points * data_channels; - - const int batch_id = index / output_batch_stride; - const int index_in_batch = index % output_batch_stride; - const int chan = index_in_batch % data_channels; - const int sample_id = index_in_batch / data_channels; - - // Get coords of 2D point where data will be resampled - const int warp_id_x = batch_id * warp_batch_stride + sample_id * 2; - const int warp_id_y = warp_id_x + 1; - const T x = warp[warp_id_x]; - const T y = warp[warp_id_y]; - const T zero = static_cast(0.0); - const T one = static_cast(1.0); - - // Get grad output - const T grad_output_value = grad_output[out_index]; - // The interpolation function whose gradient this kernel implements: - // a) implicitly pads the input data with 0s (hence the unusual checks - // with {x,y} > -1) - // b) returns 0 when sampling outside the (padded) image. - // The effect is that the sampled signal smoothly goes to 0 outside - // the original input domain, rather than presenting a jump - // discontinuity at the image boundaries. - if (x > static_cast(-1.0) && y > static_cast(-1.0) && - x < static_cast(data_width) && y < static_cast(data_height)) { - // Precompute floor (f) and ceil (c) values for x and y. - const int fx = std::floor(static_cast(x)); - const int fy = std::floor(static_cast(y)); - const int cx = fx + 1; - const int cy = fy + 1; - const T dx = static_cast(cx) - x; - const T dy = static_cast(cy) - y; - - const T img_fxfy = (fx >= 0 && fy >= 0) ? GET_DATA_POINT(fx, fy) : zero; - - const T img_cxcy = (cx <= data_width - 1 && cy <= data_height - 1) - ? GET_DATA_POINT(cx, cy) - : zero; - - const T img_fxcy = - (fx >= 0 && cy <= data_height - 1) ? GET_DATA_POINT(fx, cy) : zero; - - const T img_cxfy = - (cx <= data_width - 1 && fy >= 0) ? GET_DATA_POINT(cx, fy) : zero; - - // Update partial gradients wrt relevant warp field entries - atomicAdd(grad_warp + warp_id_x, - grad_output_value * ((one - dy) * (img_cxcy - img_fxcy) + - dy * (img_cxfy - img_fxfy))); - atomicAdd(grad_warp + warp_id_y, - grad_output_value * ((one - dx) * (img_cxcy - img_cxfy) + - dx * (img_fxcy - img_fxfy))); - - // Update partial gradients wrt sampled data - if (fx >= 0 && fy >= 0) { - UPDATE_GRAD_DATA_POINT(fx, fy, grad_output_value * dx * dy); - } - if (cx <= data_width - 1 && cy <= data_height - 1) { - UPDATE_GRAD_DATA_POINT(cx, cy, - grad_output_value * (one - dx) * (one - dy)); - } - if (fx >= 0 && cy <= data_height - 1) { - UPDATE_GRAD_DATA_POINT(fx, cy, grad_output_value * dx * (one - dy)); - } - if (cx <= data_width - 1 && fy >= 0) { - UPDATE_GRAD_DATA_POINT(cx, fy, grad_output_value * (one - dx) * dy); - } - } - } -} - -#undef GET_DATA_POINT -#undef UPDATE_GRAD_DATA_POINT - -} // namespace - -namespace functor { - -template -struct ResamplerGrad2DFunctor { - void operator()(::tensorflow::OpKernelContext* ctx, const GPUDevice& d, - const T* __restrict__ data, const T* __restrict__ warp, - const T* __restrict__ grad_output, T* __restrict__ grad_data, - T* __restrict__ grad_warp, const int batch_size, - const int data_height, const int data_width, - const int data_channels, const int num_sampling_points) { - // Set gradients to 0, because the kernel incrementally updates the - // tensor entries by adding partial contributions. - const int grad_warp_size = batch_size * num_sampling_points * 2; - const int grad_data_size = - batch_size * data_height * data_width * data_channels; - - ::tensorflow::GpuLaunchConfig config = - ::tensorflow::GetGpuLaunchConfig(grad_warp_size, d); - TF_CHECK_OK(::tensorflow::GpuLaunchKernel( - SetZero, config.block_count, config.thread_per_block, 0, d.stream(), - grad_warp_size, grad_warp)); - - config = ::tensorflow::GetGpuLaunchConfig(grad_data_size, d); - TF_CHECK_OK(::tensorflow::GpuLaunchKernel( - SetZero, config.block_count, config.thread_per_block, 0, d.stream(), - grad_data_size, grad_data)); - - const int resampler_output_size = - batch_size * num_sampling_points * data_channels; - config = ::tensorflow::GetGpuLaunchConfig(resampler_output_size, d); - TF_CHECK_OK(GpuLaunchKernel(ResamplerGrad2DKernel, config.block_count, - config.thread_per_block, 0, d.stream(), data, - warp, grad_output, grad_data, grad_warp, - batch_size, data_height, data_width, - data_channels, num_sampling_points)); - } -}; - -template struct ResamplerGrad2DFunctor; - -} // namespace functor - -} // namespace tensorflow - -#endif // GOOGLE_CUDA diff --git a/tensorflow/contrib/resampler/ops/resampler_ops.cc b/tensorflow/contrib/resampler/ops/resampler_ops.cc deleted file mode 100644 index f785d4ee5fc..00000000000 --- a/tensorflow/contrib/resampler/ops/resampler_ops.cc +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2017 The Sonnet Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { - -using ::tensorflow::shape_inference::InferenceContext; -using ::tensorflow::shape_inference::ShapeHandle; - -REGISTER_OP("Resampler") - .Input("data: T") - .Input("warp: T") - .Output("output: T") - .Attr("T: {half, bfloat16, float, double}") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle data; - ShapeHandle warp; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &data)); - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &warp)); - - ShapeHandle output; // will be warp[:-1] + [data[-1]] - TF_RETURN_IF_ERROR(c->Subshape(warp, 0, -1, &output)); - TF_RETURN_IF_ERROR( - c->Concatenate(output, c->Vector(c->Dim(data, -1)), &output)); - - c->set_output(0, output); - return ::tensorflow::Status::OK(); - }) - .Doc(R"doc(Resampler op.)doc"); - -REGISTER_OP("ResamplerGrad") - .Input("data: T") - .Input("warp: T") - .Input("grad_output: T") - .Output("grad_data: T") - .Output("grad_warp: T") - .Attr("T: {half, bfloat16, float, double}") - .SetShapeFn([](InferenceContext* c) { - c->set_output(0, c->input(0)); - c->set_output(1, c->input(1)); - return ::tensorflow::Status::OK(); - }) - .Doc(R"doc(Resampler Grad op.)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/resampler/python/__init__.py b/tensorflow/contrib/resampler/python/__init__.py deleted file mode 100644 index c5ca3a623fb..00000000000 --- a/tensorflow/contrib/resampler/python/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ops module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/resampler/python/ops/resampler_ops.py b/tensorflow/contrib/resampler/python/ops/resampler_ops.py deleted file mode 100644 index 0ee224a4782..00000000000 --- a/tensorflow/contrib/resampler/python/ops/resampler_ops.py +++ /dev/null @@ -1,71 +0,0 @@ -# pylint: disable=g-bad-file-header -# Copyright 2017 The Sonnet Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -"""Tensorflow op performing differentiable resampling.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.resampler.ops import gen_resampler_ops -from tensorflow.contrib.util import loader -from tensorflow.python.framework import ops -from tensorflow.python.platform import resource_loader - -_resampler_so = loader.load_op_library( - resource_loader.get_path_to_datafile("_resampler_ops.so")) - - -def resampler(data, warp, name="resampler"): - """Resamples input data at user defined coordinates. - - The resampler currently only supports bilinear interpolation of 2D data. - - Args: - data: Tensor of shape `[batch_size, data_height, data_width, - data_num_channels]` containing 2D data that will be resampled. - warp: Tensor of minimum rank 2 containing the coordinates at which - resampling will be performed. Since only bilinear interpolation is - currently supported, the last dimension of the `warp` tensor must be 2, - representing the (x, y) coordinate where x is the index for width and y is - the index for height. - name: Optional name of the op. - - Returns: - Tensor of resampled values from `data`. The output tensor shape is - determined by the shape of the warp tensor. For example, if `data` is of - shape `[batch_size, data_height, data_width, data_num_channels]` and warp of - shape `[batch_size, dim_0, ... , dim_n, 2]` the output will be of shape - `[batch_size, dim_0, ... , dim_n, data_num_channels]`. - - Raises: - ImportError: if the wrapper generated during compilation is not present when - the function is called. - """ - with ops.name_scope(name, "resampler", [data, warp]): - data_tensor = ops.convert_to_tensor(data, name="data") - warp_tensor = ops.convert_to_tensor(warp, name="warp") - return gen_resampler_ops.resampler(data_tensor, warp_tensor) - - -@ops.RegisterGradient("Resampler") -def _resampler_grad(op, grad_output): - data, warp = op.inputs - grad_output_tensor = ops.convert_to_tensor(grad_output, name="grad_output") - return gen_resampler_ops.resampler_grad(data, warp, grad_output_tensor) - - -ops.NotDifferentiable("ResamplerGrad") diff --git a/tensorflow/contrib/resampler/python/ops/resampler_ops_test.py b/tensorflow/contrib/resampler/python/ops/resampler_ops_test.py deleted file mode 100644 index e30e7255fac..00000000000 --- a/tensorflow/contrib/resampler/python/ops/resampler_ops_test.py +++ /dev/null @@ -1,270 +0,0 @@ -# pylint: disable=g-bad-file-header -# Copyright 2017 The Sonnet Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -"""Tests for contrib.resampler.python.ops.resampler_ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib import resampler -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -def _bilinearly_interpolate(data, x, y): - """Performs bilinenar interpolation of grid data at user defined coordinates. - - This interpolation function: - a) implicitly pads the input data with 0s. - b) returns 0 when sampling outside the (padded) image. - The effect is that the sampled signal smoothly goes to 0 outside the original - input domain, rather than producing a jump discontinuity at the image - boundaries. - - Args: - data: numpy array of shape `[data_height, data_width]` containing data - samples assumed to be defined at the corresponding pixel coordinates. - x: numpy array of shape `[warp_height, warp_width]` containing x coordinates - at which interpolation will be performed. - y: numpy array of shape `[warp_height, warp_width]` containing y coordinates - at which interpolation will be performed. - - Returns: - Numpy array of shape `[warp_height, warp_width]` containing interpolated - values. - """ - shape = x.shape - x = np.asarray(x) + 1 - y = np.asarray(y) + 1 - data = np.lib.pad(data, 1, "constant", constant_values=0) - - x_0 = np.floor(x).astype(int) - x_1 = x_0 + 1 - y_0 = np.floor(y).astype(int) - y_1 = y_0 + 1 - - x_0 = np.clip(x_0, 0, data.shape[1] - 1) - x_1 = np.clip(x_1, 0, data.shape[1] - 1) - y_0 = np.clip(y_0, 0, data.shape[0] - 1) - y_1 = np.clip(y_1, 0, data.shape[0] - 1) - - i_a = data[y_0, x_0] - i_b = data[y_1, x_0] - i_c = data[y_0, x_1] - i_d = data[y_1, x_1] - - w_a = (x_1 - x) * (y_1 - y) - w_b = (x_1 - x) * (y - y_0) - w_c = (x - x_0) * (y_1 - y) - w_d = (x - x_0) * (y - y_0) - - samples = (w_a * i_a + w_b * i_b + w_c * i_c + w_d * i_d) - samples.reshape(shape) - - return samples - - -def _make_warp(batch_size, warp_height, warp_width, dtype): - """Creates batch of warping coordinates.""" - x, y = np.meshgrid(np.linspace(0, warp_width - 1, warp_width), - np.linspace(0, warp_height - 1, warp_height)) - warp = np.concatenate((x.reshape([warp_height, warp_width, 1]), - y.reshape([warp_height, warp_width, 1])), 2) - warp = np.tile(warp.reshape([1, warp_height, warp_width, 2]), - [batch_size, 1, 1, 1]) - warp += np.random.randn(*warp.shape) - return warp.astype(dtype) - - -class ResamplerTest(test.TestCase): - - def test_op_forward_pass_gpu_float32(self): - self._test_op_forward_pass(True, dtypes.float32, 1e-4) - - def test_op_forward_pass_gpu_float64(self): - self._test_op_forward_pass(True, dtypes.float64, 1e-5) - - def test_op_forward_pass_cpu_float16(self): - self._test_op_forward_pass(False, dtypes.float16, 1e-2) - - def test_op_forward_pass_cpu_float32(self): - self._test_op_forward_pass(False, dtypes.float32, 1e-4) - - def test_op_forward_pass_cpu_float64(self): - self._test_op_forward_pass(False, dtypes.float64, 1e-5) - - def test_op_backward_pass_gpu_float32(self): - self._test_op_backward_pass(True, dtypes.float32, 1e-3) - - def test_op_backward_pass_cpu_float16(self): - self._test_op_backward_pass(False, dtypes.float16, 1e-3) - - def test_op_backward_pass_cpu_float32(self): - self._test_op_backward_pass(False, dtypes.float32, 1e-4) - - def test_op_backward_pass_cpu_float64(self): - self._test_op_backward_pass(False, dtypes.float64, 1e-6) - - def _test_op_forward_pass(self, on_gpu, dtype, tol): - np.random.seed(0) - data_width = 7 - data_height = 9 - data_channels = 5 - warp_width = 4 - warp_height = 8 - batch_size = 10 - - warp = _make_warp(batch_size, warp_height, warp_width, dtype.as_numpy_dtype) - data_shape = (batch_size, data_height, data_width, data_channels) - data = np.random.rand(*data_shape).astype(dtype.as_numpy_dtype) - - with self.test_session(use_gpu=on_gpu, force_gpu=False) as sess: - data_ph = array_ops.placeholder(dtype, shape=(None,) + data.shape[1:]) - warp_ph = array_ops.placeholder(dtype, shape=(None,) + warp.shape[1:]) - outputs = resampler.resampler(data=data_ph, warp=warp_ph) - self.assertEqual(outputs.get_shape().as_list(), - [None, warp_height, warp_width, data_channels]) - out = sess.run(outputs, feed_dict={data_ph: data, warp_ph: warp}) - - # Generate reference output via bilinear interpolation in numpy - reference_output = np.zeros_like(out) - for batch in xrange(batch_size): - for c in xrange(data_channels): - reference_output[batch, :, :, c] = _bilinearly_interpolate( - data[batch, :, :, c], - warp[batch, :, :, 0], - warp[batch, :, :, 1]) - - self.assertAllClose(out, reference_output, rtol=tol, atol=tol) - - def _test_op_backward_pass(self, on_gpu, dtype, tol): - np.random.seed(13) - data_width = 5 - data_height = 4 - data_channels = 3 - warp_width = 2 - warp_height = 6 - batch_size = 3 - - warp = _make_warp(batch_size, warp_height, warp_width, dtype.as_numpy_dtype) - data_shape = (batch_size, data_height, data_width, data_channels) - data = np.random.rand(*data_shape).astype(dtype.as_numpy_dtype) - - with self.test_session(use_gpu=on_gpu, force_gpu=False): - data_tensor = constant_op.constant(data) - warp_tensor = constant_op.constant(warp) - output_tensor = resampler.resampler(data=data_tensor, warp=warp_tensor) - - grads = test.compute_gradient([data_tensor, warp_tensor], [ - data_tensor.get_shape().as_list(), - warp_tensor.get_shape().as_list() - ], output_tensor, output_tensor.get_shape().as_list(), [data, warp]) - - if not on_gpu: - # On CPU we perform numerical differentiation at the best available - # precision, and compare against that. This is necessary for test to - # pass for float16. - data_tensor_64 = constant_op.constant(data, dtype=dtypes.float64) - warp_tensor_64 = constant_op.constant(warp, dtype=dtypes.float64) - output_tensor_64 = resampler.resampler(data=data_tensor_64, - warp=warp_tensor_64) - grads_64 = test.compute_gradient([data_tensor_64, warp_tensor_64], [ - data_tensor.get_shape().as_list(), - warp_tensor.get_shape().as_list() - ], output_tensor_64, output_tensor.get_shape().as_list(), [data, warp]) - - for g, g_64 in zip(grads, grads_64): - self.assertLess(np.fabs(g[0] - g_64[1]).max(), tol) - - else: - for g in grads: - self.assertLess(np.fabs(g[0] - g[1]).max(), tol) - - def test_op_errors(self): - data_width = 7 - data_height = 9 - data_depth = 3 - data_channels = 5 - warp_width = 4 - warp_height = 8 - batch_size = 10 - - # Input data shape is not defined over a 2D grid, i.e. its shape is not like - # (batch_size, data_height, data_width, data_channels). - with self.cached_session() as sess: - data_shape = (batch_size, data_height, data_width, data_depth, - data_channels) - data = np.zeros(data_shape) - warp_shape = (batch_size, warp_height, warp_width, 2) - warp = np.zeros(warp_shape) - outputs = resampler.resampler(constant_op.constant(data), - constant_op.constant(warp)) - - with self.assertRaisesRegexp(errors_impl.UnimplementedError, - "Only bilinear interpolation is currently " - "supported."): - sess.run(outputs) - - # Warp tensor must be at least a matrix, with shape [batch_size, 2]. - with self.cached_session() as sess: - data_shape = (batch_size, data_height, data_width, data_channels) - data = np.zeros(data_shape) - warp_shape = (batch_size,) - warp = np.zeros(warp_shape) - outputs = resampler.resampler(constant_op.constant(data), - constant_op.constant(warp)) - - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - "warp should be at least a matrix"): - sess.run(outputs) - - # The batch size of the data and warp tensors must be the same. - with self.cached_session() as sess: - data_shape = (batch_size, data_height, data_width, data_channels) - data = np.zeros(data_shape) - warp_shape = (batch_size+1, warp_height, warp_width, 2) - warp = np.zeros(warp_shape) - outputs = resampler.resampler(constant_op.constant(data), - constant_op.constant(warp)) - - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - "Batch size of data and warp tensor"): - sess.run(outputs) - - # The warp tensor must contain 2D coordinates, i.e. its shape last dimension - # must be 2. - with self.cached_session() as sess: - data_shape = (batch_size, data_height, data_width, data_channels) - data = np.zeros(data_shape) - warp_shape = (batch_size, warp_height, warp_width, 3) - warp = np.zeros(warp_shape) - outputs = resampler.resampler(constant_op.constant(data), - constant_op.constant(warp)) - - with self.assertRaisesRegexp(errors_impl.UnimplementedError, - "Only bilinear interpolation is supported, " - "warping"): - sess.run(outputs) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/resampler/xla/resampler_ops_xla_test.py b/tensorflow/contrib/resampler/xla/resampler_ops_xla_test.py deleted file mode 100644 index 558cb9015ad..00000000000 --- a/tensorflow/contrib/resampler/xla/resampler_ops_xla_test.py +++ /dev/null @@ -1,241 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for resampler ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.compiler.tests import xla_test -from tensorflow.contrib import resampler -from tensorflow.contrib.resampler.ops import gen_resampler_ops -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class ResamplerOpsTest(xla_test.XLATestCase): - - def _assertForwardOpMatchesExpected(self, image_np, warp_np, expected): - with self.session() as sess, self.test_scope(): - input_image = array_ops.placeholder(image_np.dtype) - warp = array_ops.placeholder(warp_np.dtype) - resampled = resampler.resampler(input_image, warp, name='resampler') - out = sess.run(resampled, {input_image: image_np, warp: warp_np}) - - self.assertAllCloseAccordingToType( - expected, out, rtol=5e-3, half_rtol=1e-2, bfloat16_rtol=3e-2) - - def _assertBackwardOpMatchesExpected(self, input_np, warp_np, grad_output_np, - expected_grad_data, expected_grad_warp): - with self.session() as sess, self.test_scope(): - input_image = array_ops.placeholder(input_np.dtype) - warp = array_ops.placeholder(warp_np.dtype) - grad_output = array_ops.placeholder(grad_output_np.dtype) - - grad_data, grad_warp = gen_resampler_ops.resampler_grad( - input_image, warp, grad_output) - - grad_data_tf, grad_warp_tf = sess.run([grad_data, grad_warp], { - input_image: input_np, - warp: warp_np, - grad_output: grad_output_np - }) - - self.assertAllCloseAccordingToType( - expected_grad_warp, grad_warp_tf, half_rtol=1e-2, bfloat16_rtol=3e-2) - self.assertAllCloseAccordingToType( - expected_grad_data, grad_data_tf, half_rtol=1e-2, bfloat16_rtol=3e-2) - - def testSimple(self): - for dtype in self.float_types: - input_shape = [1, 2, 2, 1] - input_data = [0, 5, 13, 54] - input_np = np.array(input_data, dtype=dtype).reshape(input_shape) - - warp_shape = [1, 2] - warp_data = [0.7, 0.6] - warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape) - expected = [[26.42]] - self._assertForwardOpMatchesExpected(input_np, warp_np, expected) - - grad_output = np.ones([1, 1], dtype=dtype) - - expected_grad_data = [[[[0.12], [0.27999997]], [[0.18000001], - [0.42000002]]]] - - expected_grad_warp = [[26.60000038, 38.20000076]] - - self._assertBackwardOpMatchesExpected(input_np, warp_np, grad_output, - expected_grad_data, - expected_grad_warp) - - def testMultiChannel(self): - for dtype in self.float_types: - input_shape = [1, 2, 2, 3] - input_rgb_data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1] - input_np = np.array(input_rgb_data, dtype=dtype).reshape(input_shape) - - warp_shape = [1, 2] - warp_data = [0.7, 0.6] - warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape) - expected = [[59.58000183, 146.94000244, 107.37999725]] - self._assertForwardOpMatchesExpected(input_np, warp_np, expected) - - grad_output = np.ones([1, 3], dtype=dtype) - - expected_grad_data = [[[[0.12, 0.12, 0.12], - [0.27999997, 0.27999997, 0.27999997]], - [[0.18000001, 0.18000001, 0.18000001], - [0.42000002, 0.42000002, 0.42000002]]]] - - expected_grad_warp = [[199, 30]] - - self._assertBackwardOpMatchesExpected(input_np, warp_np, grad_output, - expected_grad_data, - expected_grad_warp) - - def testBatch2Height3byWidth3RGB(self): - for dtype in self.float_types: - input_shape = [2, 3, 3, 3] - input_rgb_data = [ - 0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1, 30, 105, 2, 40, 115, - 3, 50, 125, 4, 60, 135, 5, 70, 145, 6, 0, 5, 13, 54, 135, 226, 37, 8, - 234, 90, 255, 1, 30, 105, 2, 40, 115, 3, 50, 125, 4, 60, 135, 5, 70, - 145, 6 - ] - input_np = np.array(input_rgb_data, dtype=dtype).reshape(input_shape) - - # 2 batches and 2 samples for each batch. - warp_shape = [2, 2, 2] - warp_data = [0.7, 0.6, 1, 0.7, 0.9, 1.2, 1.3, 1.6] - warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape) - - expected_forward = [[[43.92, 128.4, 65.86], [37.2, 114., 69.2]], - [[40.6, 122.8, 2.5], [51., 126, 4.1]]] - - self._assertForwardOpMatchesExpected(input_np, warp_np, expected_forward) - - expected_grad_data = [[[[0.12, 0.12, 0.12], - [0.57999998, 0.57999998, 0.57999998], - [0., 0., 0.]], - [[0.18000001, 0.18000001, 0.18000001], - [1.12, 1.12, 1.12], [0., 0., 0.]], - [[0., 0., 0.], [0., 0., 0.], [0., 0., 0.]]], - [[[0., 0., 0.], [0., 0., 0.], [0., 0., 0.]], - [[0.08000001, 0.08000001, 0.08000001], - [0.99999988, 0.99999988, 0.99999988], - [0.11999997, 0.11999997, 0.11999997]], - [[0.02000001, 0.02000001, 0.02000001], - [0.60000008, 0.60000008, 0.60000008], - [0.17999998, 0.17999998, 0.17999998]]]] - expected_grad_warp = [[[33.39999008, -96.20000458], [-26.10000229, - -278.]], - [[-162.99998474, 39.99999619], [21., 63.]]] - - grad_output = np.ones([2, 2, 3], dtype=dtype) - self._assertBackwardOpMatchesExpected(input_np, warp_np, grad_output, - expected_grad_data, - expected_grad_warp) - - def testOutOfBoundWarps(self): - # (x, y) are both less than 0. - for dtype in self.float_types: - input_shape = [1, 2, 2, 1] - input_data = [10, 5, 13, 54] - input_np = np.array(input_data, dtype=dtype).reshape(input_shape) - - warp_shape = [1, 2, 2] - warp_data = [-1, -1, 0.7, 0.6] - warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape) - expected = [[[0.0], [27.62]]] - self._assertForwardOpMatchesExpected(input_np, warp_np, expected) - - expected_grad_data = [[[[0.12], [0.27999997]], [[0.18000001], - [0.42000002]]]] - expected_grad_warp = [[[0., 0.], [22.60000038, 35.20000076]]] - - grad_output = np.ones([1, 2, 1], dtype=dtype) - self._assertBackwardOpMatchesExpected(input_np, warp_np, grad_output, - expected_grad_data, - expected_grad_warp) - - # One of (x, y) is less than 0. - for dtype in self.float_types: - input_shape = [1, 2, 2, 1] - input_data = [10, 5, 13, 54] - input_np = np.array(input_data, dtype=dtype).reshape(input_shape) - - warp_shape = [1, 2, 2] - # -1 is out of bound for grad_warp. - warp_data = [-1, 0.1, 0.7, 0.6] - warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape) - expected = [[[0.0], [27.62]]] - self._assertForwardOpMatchesExpected(input_np, warp_np, expected) - - expected_grad_data = [[[[0.12], [0.27999997]], [[0.18000001], - [0.42000002]]]] - expected_grad_warp = [[[0., 0.], [22.60000038, 35.20000076]]] - - grad_output = np.ones([1, 2, 1], dtype=dtype) - self._assertBackwardOpMatchesExpected(input_np, warp_np, grad_output, - expected_grad_data, - expected_grad_warp) - - # Both of (x, y) are greater than image size. - for dtype in self.float_types: - input_shape = [1, 2, 2, 1] - input_data = [10, 5, 13, 54] - input_np = np.array(input_data, dtype=dtype).reshape(input_shape) - - warp_shape = [1, 2, 2] - # -0.1 is *inbound* for grad_warp and grad_data, 2.1 is out of bound. - warp_data = [-0.1, 0.1, 1.2, 2.1] - warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape) - expected = [[[0.0], [0.0]]] - self._assertForwardOpMatchesExpected(input_np, warp_np, expected) - - expected_grad_data = [[[[0.81], [0.0]], [[0.09], [0.0]]]] - expected_grad_warp = [[[10.30, 2.7], [0.0, 0.0]]] - - grad_output = np.ones([1, 2, 1], dtype=dtype) - self._assertBackwardOpMatchesExpected(input_np, warp_np, grad_output, - expected_grad_data, - expected_grad_warp) - - # One of (x, y) is greater than image size. - for dtype in self.float_types: - input_shape = [1, 2, 2, 1] - input_data = [10, 5, 13, 54] - input_np = np.array(input_data, dtype=dtype).reshape(input_shape) - - warp_shape = [1, 2, 2] - warp_data = [0.1, -0.1, 1.2, 0.1] - warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape) - expected = [[[0.0], [0.0]]] - self._assertForwardOpMatchesExpected(input_np, warp_np, expected) - - expected_grad_data = [[[[0.81], [0.81]], [[0.0], [0.08]]]] - expected_grad_warp = [[[-4.5, 9.5], [-9.9, 39.20]]] - - grad_output = np.ones([1, 2, 1], dtype=dtype) - self._assertBackwardOpMatchesExpected(input_np, warp_np, grad_output, - expected_grad_data, - expected_grad_warp) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD deleted file mode 100644 index a8c4212ab8e..00000000000 --- a/tensorflow/contrib/rnn/BUILD +++ /dev/null @@ -1,268 +0,0 @@ -# Description: -# Contains ops to train linear models on top of TensorFlow. -# APIs here are meant to evolve over time. - -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load( - "//tensorflow:tensorflow.bzl", - "tf_kernel_library", - "tf_py_test", -) - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "rnn_py", - srcs = ["__init__.py"] + glob(["python/ops/*.py"]) + [ - "python/tools/checkpoint_convert.py", - ], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":benchmarking", - "//tensorflow/contrib/compiler:compiler_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:clip_ops", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform", - "//tensorflow/python:pywrap_tensorflow", - "//tensorflow/python:random_ops", - "//tensorflow/python:rnn", - "//tensorflow/python:rnn_cell", - "//tensorflow/python:rnn_ops_gen", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -cuda_py_test( - name = "rnn_cell_test", - size = "medium", - srcs = ["python/kernel_tests/rnn_cell_test.py"], - additional_deps = [ - ":rnn_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:random_ops", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], - tags = ["optonly"], - xla_enabled = True, -) - -cuda_py_test( - name = "rnn_test", - size = "medium", - srcs = ["python/kernel_tests/rnn_test.py"], - additional_deps = [ - ":rnn_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], - tags = [ - "optonly", - ], -) - -tf_py_test( - name = "fused_rnn_cell_test", - size = "medium", - srcs = ["python/kernel_tests/fused_rnn_cell_test.py"], - additional_deps = [ - ":rnn_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:init_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:rnn", - "//tensorflow/python:rnn_cell", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -cuda_py_test( - name = "lstm_ops_test", - size = "medium", - srcs = ["python/kernel_tests/lstm_ops_test.py"], - additional_deps = [ - ":rnn_py", - "@absl_py//absl/testing:parameterized", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:init_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:rnn", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], - tags = ["noasan"], -) - -cuda_py_test( - name = "gru_ops_test", - size = "small", - srcs = ["python/kernel_tests/gru_ops_test.py"], - additional_deps = [ - ":rnn_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:rnn", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], - tags = [ - "no_oss", - "noasan", - ], -) - -py_binary( - name = "checkpoint_convert", - srcs = ["python/tools/checkpoint_convert.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [":checkpoint_convert_lib"], -) - -py_library( - name = "checkpoint_convert_lib", - srcs = ["python/tools/checkpoint_convert.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/core:protos_all_py", - "//tensorflow/python:framework_ops", - "//tensorflow/python:platform", - "//tensorflow/python:pywrap_tensorflow", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "checkpoint_convert_test", - size = "small", - srcs = ["python/tools/checkpoint_convert_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [ - ":checkpoint_convert_lib", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_ops", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], -) - -py_library( - name = "benchmarking", - srcs = ["python/kernel_tests/benchmarking.py"], - srcs_version = "PY2AND3", - deps = ["//tensorflow/python:framework_ops"], -) - -# These are compatibility re-exports. Remove once there are -# no usages left. - -USE_CORE_TF = "contrib/rnn kernels and ops are now part of core TensorFlow" - -cc_library( - name = "all_ops", - deprecation = USE_CORE_TF, - deps = [ - "//tensorflow/core:rnn_ops_op_lib", - ], -) - -cc_library( - name = "all_kernels", - deprecation = USE_CORE_TF, - deps = [ - ":gru_ops_kernels", - ":lstm_ops_kernels", - ], -) - -cc_library( - name = "gru_ops_op_lib", - deprecation = USE_CORE_TF, - deps = [ - "//tensorflow/core:rnn_ops_op_lib", - ], -) - -cc_library( - name = "lstm_ops_op_lib", - deprecation = USE_CORE_TF, - deps = [ - "//tensorflow/core:rnn_ops_op_lib", - ], -) - -tf_kernel_library( - name = "gru_ops_kernels", - deprecation = USE_CORE_TF, - deps = [ - "//tensorflow/core/kernels/rnn:gru_ops", - ], -) - -tf_kernel_library( - name = "lstm_ops_kernels", - deprecation = USE_CORE_TF, - deps = [ - "//tensorflow/core/kernels/rnn:lstm_ops", - ], -) diff --git a/tensorflow/contrib/rnn/__init__.py b/tensorflow/contrib/rnn/__init__.py deleted file mode 100644 index cbc8af53502..00000000000 --- a/tensorflow/contrib/rnn/__init__.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""RNN Cells and additional RNN operations. - - -@@RNNCell -@@LayerRNNCell -@@BasicRNNCell -@@BasicLSTMCell -@@GRUCell -@@LSTMCell -@@LSTMStateTuple -@@DropoutWrapper -@@MultiRNNCell -@@DeviceWrapper -@@ResidualWrapper - - -@@EmbeddingWrapper -@@InputProjectionWrapper -@@OutputProjectionWrapper - - -@@LayerNormBasicLSTMCell -@@LSTMBlockWrapper -@@LSTMBlockCell -@@GRUBlockCell -@@GRUBlockCellV2 -@@FusedRNNCell -@@FusedRNNCellAdaptor -@@TimeReversedFusedRNN -@@LSTMBlockFusedCell -@@CoupledInputForgetGateLSTMCell -@@TimeFreqLSTMCell -@@GridLSTMCell -@@BidirectionalGridLSTMCell -@@NASCell -@@UGRNNCell -@@IntersectionRNNCell -@@PhasedLSTMCell -@@ConvLSTMCell -@@Conv1DLSTMCell -@@Conv2DLSTMCell -@@Conv3DLSTMCell -@@HighwayWrapper -@@GLSTMCell -@@SRUCell -@@IndRNNCell -@@IndyGRUCell -@@IndyLSTMCell - - -@@AttentionCellWrapper -@@CompiledWrapper - - -@@static_rnn -@@static_state_saving_rnn -@@static_bidirectional_rnn -@@stack_bidirectional_dynamic_rnn -@@stack_bidirectional_rnn - - -@@transpose_batch_time -@@best_effort_input_batch_size -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import,line-too-long -from tensorflow.contrib.rnn.python.ops.core_rnn_cell import EmbeddingWrapper -from tensorflow.contrib.rnn.python.ops.core_rnn_cell import InputProjectionWrapper -from tensorflow.contrib.rnn.python.ops.core_rnn_cell import OutputProjectionWrapper - -from tensorflow.contrib.rnn.python.ops.fused_rnn_cell import * -from tensorflow.contrib.rnn.python.ops.gru_ops import * -from tensorflow.contrib.rnn.python.ops.lstm_ops import * -from tensorflow.contrib.rnn.python.ops.rnn import * -from tensorflow.contrib.rnn.python.ops.rnn_cell import * - -from tensorflow.python.ops.rnn import _best_effort_input_batch_size as best_effort_input_batch_size -from tensorflow.python.ops.rnn import _transpose_batch_time as transpose_batch_time -from tensorflow.python.ops.rnn import static_bidirectional_rnn -from tensorflow.python.ops.rnn import static_rnn -from tensorflow.python.ops.rnn import static_state_saving_rnn - -from tensorflow.python.ops.rnn_cell import * -# pylint: enable=unused-import,wildcard-import,line-too-long - -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/rnn/python/__init__.py b/tensorflow/contrib/rnn/python/__init__.py deleted file mode 100644 index c5ca3a623fb..00000000000 --- a/tensorflow/contrib/rnn/python/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ops module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/rnn/python/kernel_tests/benchmarking.py b/tensorflow/contrib/rnn/python/kernel_tests/benchmarking.py deleted file mode 100644 index a48cd58706e..00000000000 --- a/tensorflow/contrib/rnn/python/kernel_tests/benchmarking.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Library for benchmarking OpKernels.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools -import time - -from tensorflow.python.framework import ops - - -def device(use_gpu=False): - """TensorFlow device to assign ops to.""" - if use_gpu: - return ops.device("/gpu:0") - return ops.device("/cpu:0") - - -def seconds_per_run(op, sess, num_runs=50): - """Number of seconds taken to execute 'op' once on average.""" - for _ in range(2): - sess.run(op) - - start_time = time.time() - for _ in range(num_runs): - sess.run(op) - - end_time = time.time() - time_taken = (end_time - start_time) / num_runs - return time_taken - - -def dict_product(dicts): - """Constructs iterator over outer product of entries in a dict-of-lists. - - Example: - >>> dict_products({"a": [1,2], "b": [3, 4]}) - >>> [{"a": 1, "b": 3}, - {"a": 1, "b": 4}, - {"a": 2, "b": 3}, - {"a": 2, "b": 4}] - - Args: - dicts: dictionary with string keys and list values. - - Yields: - Individual dicts from outer product. - """ - keys, values = zip(*dicts.items()) - for config_values in itertools.product(*values): - yield dict(zip(keys, config_values)) diff --git a/tensorflow/contrib/rnn/python/kernel_tests/fused_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/fused_rnn_cell_test.py deleted file mode 100644 index 8d34b9e8525..00000000000 --- a/tensorflow/contrib/rnn/python/kernel_tests/fused_rnn_cell_test.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.contrib.rnn.python.ops.fused_rnn_cell.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.rnn.python.ops import fused_rnn_cell -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import rnn -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class FusedRnnCellTest(test.TestCase): - - def testBasicRNNFusedWrapper(self): - """This test checks that using a wrapper for BasicRNN works as expected.""" - - with self.cached_session() as sess: - initializer = init_ops.random_uniform_initializer( - -0.01, 0.01, seed=19890212) - cell = rnn_cell.BasicRNNCell(10) - batch_size = 5 - input_size = 20 - timelen = 15 - inputs = constant_op.constant( - np.random.randn(timelen, batch_size, input_size)) - with variable_scope.variable_scope("basic", initializer=initializer): - unpacked_inputs = array_ops.unstack(inputs) - outputs, state = rnn.static_rnn( - cell, unpacked_inputs, dtype=dtypes.float64) - packed_outputs = array_ops.stack(outputs) - basic_vars = [ - v for v in variables.trainable_variables() - if v.name.startswith("basic/") - ] - sess.run([variables.global_variables_initializer()]) - basic_outputs, basic_state = sess.run([packed_outputs, state]) - basic_grads = sess.run(gradients_impl.gradients(packed_outputs, inputs)) - basic_wgrads = sess.run( - gradients_impl.gradients(packed_outputs, basic_vars)) - - with variable_scope.variable_scope( - "fused_static", initializer=initializer): - fused_cell = fused_rnn_cell.FusedRNNCellAdaptor( - rnn_cell.BasicRNNCell(10)) - outputs, state = fused_cell(inputs, dtype=dtypes.float64) - fused_static_vars = [ - v for v in variables.trainable_variables() - if v.name.startswith("fused_static/") - ] - sess.run([variables.global_variables_initializer()]) - fused_static_outputs, fused_static_state = sess.run([outputs, state]) - fused_static_grads = sess.run(gradients_impl.gradients(outputs, inputs)) - fused_static_wgrads = sess.run( - gradients_impl.gradients(outputs, fused_static_vars)) - - self.assertAllClose(basic_outputs, fused_static_outputs) - self.assertAllClose(basic_state, fused_static_state) - self.assertAllClose(basic_grads, fused_static_grads) - for basic, fused in zip(basic_wgrads, fused_static_wgrads): - self.assertAllClose(basic, fused, rtol=1e-2, atol=1e-2) - - with variable_scope.variable_scope( - "fused_dynamic", initializer=initializer): - fused_cell = fused_rnn_cell.FusedRNNCellAdaptor( - rnn_cell.BasicRNNCell(10), use_dynamic_rnn=True) - outputs, state = fused_cell(inputs, dtype=dtypes.float64) - fused_dynamic_vars = [ - v for v in variables.trainable_variables() - if v.name.startswith("fused_dynamic/") - ] - sess.run([variables.global_variables_initializer()]) - fused_dynamic_outputs, fused_dynamic_state = sess.run([outputs, state]) - fused_dynamic_grads = sess.run( - gradients_impl.gradients(outputs, inputs)) - fused_dynamic_wgrads = sess.run( - gradients_impl.gradients(outputs, fused_dynamic_vars)) - - self.assertAllClose(basic_outputs, fused_dynamic_outputs) - self.assertAllClose(basic_state, fused_dynamic_state) - self.assertAllClose(basic_grads, fused_dynamic_grads) - for basic, fused in zip(basic_wgrads, fused_dynamic_wgrads): - self.assertAllClose(basic, fused, rtol=1e-2, atol=1e-2) - - def testTimeReversedFusedRNN(self): - with self.cached_session() as sess: - initializer = init_ops.random_uniform_initializer( - -0.01, 0.01, seed=19890213) - fw_cell = rnn_cell.BasicRNNCell(10) - bw_cell = rnn_cell.BasicRNNCell(10) - batch_size = 5 - input_size = 20 - timelen = 15 - inputs = constant_op.constant( - np.random.randn(timelen, batch_size, input_size)) - - # test bi-directional rnn - with variable_scope.variable_scope("basic", initializer=initializer): - unpacked_inputs = array_ops.unstack(inputs) - outputs, fw_state, bw_state = rnn.static_bidirectional_rnn( - fw_cell, bw_cell, unpacked_inputs, dtype=dtypes.float64) - packed_outputs = array_ops.stack(outputs) - basic_vars = [ - v for v in variables.trainable_variables() - if v.name.startswith("basic/") - ] - sess.run([variables.global_variables_initializer()]) - basic_outputs, basic_fw_state, basic_bw_state = sess.run( - [packed_outputs, fw_state, bw_state]) - basic_grads = sess.run(gradients_impl.gradients(packed_outputs, inputs)) - basic_wgrads = sess.run( - gradients_impl.gradients(packed_outputs, basic_vars)) - - with variable_scope.variable_scope("fused", initializer=initializer): - fused_cell = fused_rnn_cell.FusedRNNCellAdaptor( - rnn_cell.BasicRNNCell(10)) - fused_bw_cell = fused_rnn_cell.TimeReversedFusedRNN( - fused_rnn_cell.FusedRNNCellAdaptor(rnn_cell.BasicRNNCell(10))) - fw_outputs, fw_state = fused_cell( - inputs, dtype=dtypes.float64, scope="fw") - bw_outputs, bw_state = fused_bw_cell( - inputs, dtype=dtypes.float64, scope="bw") - outputs = array_ops.concat([fw_outputs, bw_outputs], 2) - fused_vars = [ - v for v in variables.trainable_variables() - if v.name.startswith("fused/") - ] - sess.run([variables.global_variables_initializer()]) - fused_outputs, fused_fw_state, fused_bw_state = sess.run( - [outputs, fw_state, bw_state]) - fused_grads = sess.run(gradients_impl.gradients(outputs, inputs)) - fused_wgrads = sess.run(gradients_impl.gradients(outputs, fused_vars)) - - self.assertAllClose(basic_outputs, fused_outputs) - self.assertAllClose(basic_fw_state, fused_fw_state) - self.assertAllClose(basic_bw_state, fused_bw_state) - self.assertAllClose(basic_grads, fused_grads) - for basic, fused in zip(basic_wgrads, fused_wgrads): - self.assertAllClose(basic, fused, rtol=1e-2, atol=1e-2) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/rnn/python/kernel_tests/gru_ops_test.py b/tensorflow/contrib/rnn/python/kernel_tests/gru_ops_test.py deleted file mode 100644 index 50d0da6eaf2..00000000000 --- a/tensorflow/contrib/rnn/python/kernel_tests/gru_ops_test.py +++ /dev/null @@ -1,606 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Block GRU module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.rnn.python.kernel_tests import benchmarking -from tensorflow.contrib.rnn.python.ops import gru_ops -from tensorflow.python.client import session -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradient_checker -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import rnn -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import gradient_descent - - -class GRUBlockCellTest(test.TestCase): - - def testNoneDimsWithDynamicRNN(self): - with self.session(use_gpu=True, graph=ops.Graph()) as sess: - batch_size = 4 - cell_size = 5 - input_size = 6 - num_steps = 7 - - cell = gru_ops.GRUBlockCell(cell_size) - - x = array_ops.placeholder(dtypes.float32, shape=(None, None, input_size)) - _, output = rnn.dynamic_rnn( - cell, x, time_major=True, dtype=dtypes.float32) - sess.run(variables.global_variables_initializer()) - feed = {} - feed[x] = np.random.randn(num_steps, batch_size, input_size) - sess.run(output, feed) - - def testBlockGRUToGRUCellSingleStep(self): - with self.session(use_gpu=True, graph=ops.Graph()) as sess: - batch_size = 4 - cell_size = 5 - input_size = 6 - - seed = 1994 - initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=seed) - - # Inputs - x = array_ops.zeros([batch_size, input_size]) - h = array_ops.zeros([batch_size, cell_size]) - - # Values for the inputs. - x_value = np.random.rand(batch_size, input_size) - h_value = np.random.rand(batch_size, cell_size) - - # Output from the basic GRU cell implementation. - with vs.variable_scope("basic", initializer=initializer): - output = rnn_cell.GRUCell(cell_size)(x, h) - sess.run([variables.global_variables_initializer()]) - basic_res = sess.run([output], {x: x_value, h: h_value}) - - # Output from the block GRU cell implementation. - with vs.variable_scope("block", initializer=initializer): - output = gru_ops.GRUBlockCell(cell_size)(x, h) - sess.run([variables.global_variables_initializer()]) - block_res = sess.run([output], {x: x_value, h: h_value}) - - self.assertEqual(len(block_res), len(basic_res)) - for block, basic in zip(block_res, basic_res): - self.assertAllClose(block, basic) - - def testBlockGRUToGRUCellMultiStep(self): - with self.session(use_gpu=True, graph=ops.Graph()) as sess: - batch_size = 2 - cell_size = 3 - input_size = 3 - time_steps = 4 - - # Random initializers. - seed = 1994 - initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=seed) - np.random.seed(seed) - - # Inputs - concat_x = array_ops.placeholder( - dtypes.float32, shape=(time_steps, batch_size, input_size)) - h = array_ops.zeros([batch_size, cell_size]) - - # Values for the inputs. - x_values = np.random.rand(time_steps, batch_size, input_size) - h_value = np.random.rand(batch_size, cell_size) - - # Output from the block GRU cell implementation. - with vs.variable_scope("block", initializer=initializer): - cell = gru_ops.GRUBlockCell(cell_size) - outputs_dynamic, state_dynamic = rnn.dynamic_rnn( - cell, - inputs=concat_x, - initial_state=h, - time_major=True, - dtype=dtypes.float32) - feeds = {concat_x: x_values, h: h_value} - sess.run([variables.global_variables_initializer()]) - block_res = sess.run([outputs_dynamic, state_dynamic], feeds) - - # Output from the basic GRU cell implementation. - with vs.variable_scope("basic", initializer=initializer): - cell = rnn_cell.GRUCell(cell_size) - outputs_dynamic, state_dynamic = rnn.dynamic_rnn( - cell, - inputs=concat_x, - initial_state=h, - time_major=True, - dtype=dtypes.float32) - feeds = {concat_x: x_values, h: h_value} - sess.run([variables.global_variables_initializer()]) - basic_res = sess.run([outputs_dynamic, state_dynamic], feeds) - - # Check the lengths of the outputs_dynamic, and states. - self.assertEqual(len(block_res), len(basic_res)) - self.assertEqual(len(block_res[0]), len(basic_res[0])) - self.assertEqual(len(block_res[1]), len(basic_res[1])) - - # Check the outputs_dynamic values. - for block_output, basic_output in zip(block_res[0], basic_res[0]): - self.assertAllClose(block_output, basic_output) - - # Check the state_dynamic value. - self.assertAllClose(block_res[1], block_res[1]) - - def testDerivativeOfBlockGRUToGRUCellSingleStep(self): - with self.session(use_gpu=True, graph=ops.Graph()) as sess: - batch_size = 2 - cell_size = 3 - input_size = 4 - - seed = 1994 - initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=seed) - np.random.seed(seed) - - # Inputs - x = array_ops.zeros([batch_size, input_size]) - h = array_ops.zeros([batch_size, cell_size]) - - # Values for the inputs. - x_value = np.random.rand(batch_size, input_size) - h_value = np.random.rand(batch_size, cell_size) - - # Gradients from the block GRU cell implementation. - with vs.variable_scope("block", initializer=initializer): - output = gru_ops.GRUBlockCell(cell_size)(x, h) - sess.run([variables.global_variables_initializer()]) - - all_variables = variables.global_variables()[0:4] - [w_ru, b_ru, w_c, b_c] = all_variables - - d_new_h_wrt_x = gradients_impl.gradients([output], x) - d_new_h_wrt_h = gradients_impl.gradients([output], h) - d_new_h_wrt_w_ru = gradients_impl.gradients([output], w_ru) - d_new_h_wrt_w_c = gradients_impl.gradients([output], w_c) - d_new_h_wrt_b_ru = gradients_impl.gradients([output], b_ru) - d_new_h_wrt_b_c = gradients_impl.gradients([output], b_c) - - d_block_res = sess.run([ - d_new_h_wrt_x, d_new_h_wrt_h, d_new_h_wrt_w_ru, d_new_h_wrt_w_c, - d_new_h_wrt_b_ru, d_new_h_wrt_b_c - ], {x: x_value, - h: h_value}) - - # Gradients from the basic GRU cell implementation. - with vs.variable_scope("basic", initializer=initializer): - output = rnn_cell.GRUCell(cell_size)(x, h) - sess.run([variables.global_variables_initializer()]) - - all_variables = variables.global_variables()[4:8] - [w_ru, b_ru, w_c, b_c] = all_variables - - d_new_h_wrt_x = gradients_impl.gradients([output], x) - d_new_h_wrt_h = gradients_impl.gradients([output], h) - d_new_h_wrt_w_ru = gradients_impl.gradients([output], w_ru) - d_new_h_wrt_w_c = gradients_impl.gradients([output], w_c) - d_new_h_wrt_b_ru = gradients_impl.gradients([output], b_ru) - d_new_h_wrt_b_c = gradients_impl.gradients([output], b_c) - - d_basic_res = sess.run([ - d_new_h_wrt_x, d_new_h_wrt_h, d_new_h_wrt_w_ru, d_new_h_wrt_w_c, - d_new_h_wrt_b_ru, d_new_h_wrt_b_c - ], {x: x_value, - h: h_value}) - - # Check lengths of derivative results. - self.assertEqual(len(d_block_res), len(d_basic_res)) - # Check the value of every derivative result. - for block, basic in zip(d_block_res, d_basic_res): - self.assertAllClose(block, basic) - - def testDerivativeOfBlockGRUToGRUCellMultiSteps(self): - batch_size = 2 - cell_size = 3 - input_size = 4 - time_steps = 2 - with self.session(use_gpu=True, graph=ops.Graph()) as sess: - # Random initializers. - seed = 1994 - initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=seed) - np.random.seed(seed) - - # Inputs - concat_x = array_ops.placeholder( - dtypes.float32, shape=(time_steps, batch_size, input_size)) - h = array_ops.zeros([batch_size, cell_size]) - - # Values for the inputs. - x_values = np.random.rand(time_steps, batch_size, input_size) - h_value = np.random.rand(batch_size, cell_size) - feeds = {concat_x: x_values, h: h_value} - - # Gradients from the block GRU cell implementation. - with vs.variable_scope("block", initializer=initializer): - cell = gru_ops.GRUBlockCell(cell_size) - - outputs_dynamic, _ = rnn.dynamic_rnn( - cell, - inputs=concat_x, - initial_state=h, - time_major=True, - dtype=dtypes.float32) - grad_output_wrt_x = gradients_impl.gradients([outputs_dynamic[0]], - concat_x) - grad_output_wrt_h = gradients_impl.gradients([outputs_dynamic[0]], h) - - sess.run([variables.global_variables_initializer()]) - block_grad_res_x, block_grad_res_h = sess.run( - [grad_output_wrt_x, grad_output_wrt_h], feeds) - - # Gradients from the basic GRU cell implementation. - with vs.variable_scope("basic", initializer=initializer): - cell = rnn_cell.GRUCell(cell_size) - - outputs_dynamic, _ = rnn.dynamic_rnn( - cell, - inputs=concat_x, - initial_state=h, - time_major=True, - dtype=dtypes.float32) - grad_output_wrt_x = gradients_impl.gradients([outputs_dynamic[0]], - concat_x) - grad_output_wrt_h = gradients_impl.gradients([outputs_dynamic[0]], h) - - sess.run([variables.global_variables_initializer()]) - basic_grad_res_x, basic_grad_res_h = sess.run( - [grad_output_wrt_x, grad_output_wrt_h], feeds) - - # Check derivatives values of the outputs wrt to x. - self.assertEqual(len(block_grad_res_x), len(basic_grad_res_x)) - - # Check derivatives values of the outputs wrt to h. - for block, basic in zip(block_grad_res_x, basic_grad_res_x): - self.assertAllClose(block, basic) - - # Check derivatives values of the outputs wrt to x. - self.assertEqual(len(block_grad_res_h), len(basic_grad_res_h)) - - # Check derivatives values of the outputs wrt to h. - for block, basic in zip(block_grad_res_h, basic_grad_res_h): - self.assertAllClose(block, basic) - - def testGradient(self): - with self.session(use_gpu=True, graph=ops.Graph()) as sess: - batch_size = 1 - cell_size = 3 - input_size = 2 - - # Inputs - x = array_ops.zeros([batch_size, input_size]) - h = array_ops.zeros([batch_size, cell_size]) - output = gru_ops.GRUBlockCell(cell_size)(x, h) - - sess.run([variables.global_variables_initializer()]) - - all_variables = variables.global_variables() - - [w_ru, b_ru, w_c, b_c] = all_variables[:4] - - error_x = gradient_checker.compute_gradient_error( - x, (batch_size, input_size), output[0], (batch_size, cell_size)) - error_h = gradient_checker.compute_gradient_error(h, - (batch_size, cell_size), - output[0], - (batch_size, cell_size)) - error_w_ru = gradient_checker.compute_gradient_error( - w_ru, (input_size + cell_size, 2 * cell_size), output[0], - (batch_size, cell_size)) - error_w_c = gradient_checker.compute_gradient_error( - w_c, (input_size + cell_size, cell_size), output[0], - (batch_size, cell_size)) - error_b_ru = gradient_checker.compute_gradient_error( - b_ru, (2 * cell_size,), output[0], (batch_size, cell_size)) - error_b_c = gradient_checker.compute_gradient_error( - b_c, (cell_size,), output[0], (batch_size, cell_size)) - - eps = 1e-4 - self.assertLess(error_x, eps) - self.assertLess(error_h, eps) - self.assertLess(error_w_ru, eps) - self.assertLess(error_w_c, eps) - self.assertLess(error_b_ru, eps) - self.assertLess(error_b_c, eps) - - -#### Benchmarking GRUBlockCell vs GRUCell. - - -def training_gru_block_vs_gru_cell(batch_size, - cell_size, - input_size, - time_steps, - use_gpu=False, - iters=30): - """Benchmark training speed between GRUBlockCell vs GRUCell.""" - ops.reset_default_graph() - with session.Session(graph=ops.Graph()) as sess: - # Specify the device which is been used. - with benchmarking.device(use_gpu): - - # Random initializers. - seed = 1994 - initializer = init_ops.random_uniform_initializer(-1, 1, seed=seed) - np.random.seed(seed) - - # Inputs - concat_x = vs.get_variable("concat_x", - [time_steps, batch_size, input_size]) - h = vs.get_variable("h", [batch_size, cell_size]) - y = vs.get_variable("y", [time_steps, batch_size, cell_size]) - - # Output from the basic GRU cell implementation. - with vs.variable_scope("basic", initializer=initializer): - cell = rnn_cell.GRUCell(cell_size) - - outputs_dynamic, _ = rnn.dynamic_rnn( - cell, - inputs=concat_x, - initial_state=h, - time_major=True, - dtype=dtypes.float32) - sess.run([variables.global_variables_initializer()]) - cost = math_ops.reduce_mean(math_ops.square(outputs_dynamic - y)) - learning_rate = 0.01 - optimizer = gradient_descent.GradientDescentOptimizer( - learning_rate).minimize(cost) - - # time for a training step. - basic_time_training = benchmarking.seconds_per_run( - optimizer, sess, iters) - - # Output from the basic GRU cell implementation. - with vs.variable_scope("block", initializer=initializer): - cell = gru_ops.GRUBlockCell(cell_size) - - outputs_dynamic, _ = rnn.dynamic_rnn( - cell, - inputs=concat_x, - initial_state=h, - time_major=True, - dtype=dtypes.float32) - sess.run([variables.global_variables_initializer()]) - cost = math_ops.reduce_mean(math_ops.square(outputs_dynamic - y)) - learning_rate = 0.01 - optimizer = gradient_descent.GradientDescentOptimizer( - learning_rate).minimize(cost) - - # time for a training step. - block_time_training = benchmarking.seconds_per_run( - optimizer, sess, iters) - - performance_training = ( - basic_time_training - block_time_training) * 100 / basic_time_training - - print(",".join([ - str(batch_size), str(cell_size), str(input_size), str(time_steps), str( - use_gpu), str(basic_time_training), str(block_time_training), str( - performance_training) - ])) - - return basic_time_training, block_time_training - - -def inference_gru_block_vs_gru_cell(batch_size, - cell_size, - input_size, - time_steps, - use_gpu=False, - iters=30): - """Benchmark inference speed between GRUBlockCell vs GRUCell.""" - ops.reset_default_graph() - with session.Session(graph=ops.Graph()) as sess: - with benchmarking.device(use_gpu): - - # Random initializers. - seed = 1994 - initializer = init_ops.random_uniform_initializer(-1, 1, seed=seed) - np.random.seed(seed) - - # Inputs - concat_x = vs.get_variable("concat_x", - [time_steps, batch_size, input_size]) - h = vs.get_variable("h", [batch_size, cell_size]) - - # Output from the basic GRU cell implementation. - with vs.variable_scope("basic", initializer=initializer): - cell = rnn_cell.GRUCell(cell_size) - outputs_dynamic, _ = rnn.dynamic_rnn( - cell, - inputs=concat_x, - initial_state=h, - time_major=True, - dtype=dtypes.float32) - sess.run([variables.global_variables_initializer()]) - basic_time_inference = benchmarking.seconds_per_run( - outputs_dynamic, sess, iters) - - # Output from the block GRU cell implementation. - with vs.variable_scope("block", initializer=initializer): - cell = gru_ops.GRUBlockCell(cell_size) - outputs_dynamic, _ = rnn.dynamic_rnn( - cell, - inputs=concat_x, - initial_state=h, - time_major=True, - dtype=dtypes.float32) - sess.run([variables.global_variables_initializer()]) - block_time_inference = benchmarking.seconds_per_run( - outputs_dynamic, sess, iters) - - performance_inference = (basic_time_inference - block_time_inference - ) * 100 / basic_time_inference - print(",".join([ - str(batch_size), str(cell_size), str(input_size), str(time_steps), str( - use_gpu), str(basic_time_inference), str(block_time_inference), str( - performance_inference) - ])) - - return basic_time_inference, block_time_inference - - -def single_bprop_step_gru_block_vs_gru_cell(batch_size, - cell_size, - input_size, - use_gpu=False, - iters=30): - """Benchmark single bprop step speed between GRUBlockCell vs GRUCell.""" - ops.reset_default_graph() - with session.Session(graph=ops.Graph()) as sess: - with benchmarking.device(use_gpu): - initializer = init_ops.random_uniform_initializer(-1, 1, seed=1989) - # Inputs - x = vs.get_variable("x", [batch_size, input_size]) - h = vs.get_variable("h", [batch_size, cell_size]) - - # Output from the basic GRU cell implementation. - with vs.variable_scope("basic", initializer=initializer): - output = rnn_cell.GRUCell(cell_size)(array_ops.identity(x), - array_ops.identity(h)) - sess.run([variables.global_variables_initializer()]) - grad_output_wrt_input = gradients_impl.gradients([output], h) - basic_time_bprop = benchmarking.seconds_per_run(grad_output_wrt_input, - sess, iters) - - # Output from the block GRU cell implementation. - with vs.variable_scope("block", initializer=initializer): - output = gru_ops.GRUBlockCell(cell_size)(array_ops.identity(x), - array_ops.identity(h)) - sess.run([variables.global_variables_initializer()]) - grad_output_wrt_input = gradients_impl.gradients([output], h) - block_time_bprop = benchmarking.seconds_per_run(grad_output_wrt_input, - sess, iters) - - performance_inference = ( - basic_time_bprop - block_time_bprop) * 100 / basic_time_bprop - - print(",".join([ - str(batch_size), str(cell_size), str(input_size), str(use_gpu), str( - basic_time_bprop), str(block_time_bprop), str(performance_inference) - ])) - - return basic_time_bprop, block_time_bprop - - -class BenchmarkGRUBlock(test.Benchmark): - - def benchmarkTrainingBlockGRUVsGRUCell(self): - print("Comparison GRUBlockCell vs GRUCell") - print("--------------------------------------------------------------") - print("Training speed GRUBlockCell vs GRUCell") - print("batch_size, cell_size, input_size, time_steps, GPU, " - "basic_time_training, block_time_training, performance_training[%]") - iters = 10 - - for config in benchmarking.dict_product({ - "use_gpu": [True, False], - "batch_size": [1, 32, 128], - "cell_size": [128, 512], - "input_size": [128, 512], - "time_steps": [50] - }): - basic_time, block_time = training_gru_block_vs_gru_cell( - config["batch_size"], config["cell_size"], config["input_size"], - config["time_steps"], config["use_gpu"], iters) - self.report_benchmark( - name="GRUCell_training_time_BS%i_CS%i_IS%i_TS%i_gpu_%s" % - (config["batch_size"], config["cell_size"], config["input_size"], - config["time_steps"], config["use_gpu"]), - iters=iters, - wall_time=basic_time) - self.report_benchmark( - name="GRUBlockCell_training_time_BS%i_CS%i_IS%i_TS%i_gpu_%s" % - (config["batch_size"], config["cell_size"], config["input_size"], - config["time_steps"], config["use_gpu"]), - iters=iters, - wall_time=block_time) - - def benchmarkInferenceBlockGRUVsGRUCell(self): - print("--------------------------------------------------------------") - print("Inference speed GRUBlockCell vs GRUCell") - print( - "batch_size, cell_size, input_size, time_steps, GPU, " - "basic_time_inference, block_time_inference, performance_inference[%]") - iters = 10 - for config in benchmarking.dict_product({ - "use_gpu": [True, False], - "batch_size": [1, 32, 128], - "cell_size": [128, 512], - "input_size": [128, 512], - "time_steps": [50] - }): - basic_time, block_time = inference_gru_block_vs_gru_cell( - config["batch_size"], config["cell_size"], config["input_size"], - config["time_steps"], config["use_gpu"], iters) - self.report_benchmark( - name="GRUCell_inference_time_BS%i_CS%i_IS%i_TS%i_gpu_%s" % - (config["batch_size"], config["cell_size"], config["input_size"], - config["time_steps"], config["use_gpu"]), - iters=iters, - wall_time=basic_time) - self.report_benchmark( - name="GRUBlockCell_inference_time_BS%i_CS%i_IS%i_TS%i_gpu_%s" % - (config["batch_size"], config["cell_size"], config["input_size"], - config["time_steps"], config["use_gpu"]), - iters=iters, - wall_time=block_time) - - def benchmarkSingleBpropStepBlockGRUVsGRUCell(self): - print("--------------------------------------------------------------") - print("Single bprop step speed GRUBlockCell vs GRUCell") - print("batch_size, cell_size, input_size, GPU, basic_time, " - "block_time, performance_inference[%]") - iters = 10 - for config in benchmarking.dict_product({ - "use_gpu": [True, False], - "batch_size": [1, 32, 128], - "cell_size": [128, 512], - "input_size": [128, 512] - }): - basic_time, block_time = single_bprop_step_gru_block_vs_gru_cell( - config["batch_size"], config["cell_size"], config["input_size"], - config["use_gpu"], iters) - self.report_benchmark( - name="GRUCell_Bprop_single_step_time_BS%i_CS%i_IS%i_gpu_%s" % - (config["batch_size"], config["cell_size"], config["input_size"], - config["use_gpu"]), - iters=iters, - wall_time=basic_time) - self.report_benchmark( - name="GRUBlockCell_Bprop_single_step_time_BS%i_CS%i_IS%i_gpu_%s" % - (config["batch_size"], config["cell_size"], config["input_size"], - config["use_gpu"]), - iters=iters, - wall_time=block_time) - - print("--------------------------------------------------------------") - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py deleted file mode 100644 index 1a9e7053c55..00000000000 --- a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py +++ /dev/null @@ -1,641 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""LSTM Block Cell ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy as np - -from tensorflow.contrib.rnn.python.kernel_tests import benchmarking -from tensorflow.contrib.rnn.python.ops import lstm_ops -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_bitwise_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import rnn -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - -block_lstm = lstm_ops._block_lstm # pylint: disable=protected-access - - -class _MaskedRandomUniformInitializer(init_ops.RandomUniform): - """Initializer for uniform dist tensors with trailing bits zeroed-out. - - Allow returning tensors with last few mantissa bits set to 0. This potentially - helps avoid getting into precision issues when testing low precision (float16) - computation. - """ - - def __init__(self, - minval=0, - maxval=None, - seed=None, - dtype=dtypes.float16, - num_valid_mantissa_bits=4): - """Constructor. - - Args: - minval: A python scalar or a scalar tensor. Lower bound of the range of - random values to generate. - maxval: A python scalar or a scalar tensor. Upper bound of the range of - random values to generate. Defaults to 1 for float types. - seed: A Python integer. Used to create random seeds. See - `tf.compat.v1.set_random_seed` for behavior. - dtype: The data type. Only supports tf.float16 for now. - num_valid_mantissa_bits: number of non-zero mantissa bits, default to 4. - - Raises: - ValueError: An error if `dtype` is not tf.float16. - """ - if dtype not in (dtypes.float16,): - raise ValueError("dtype: %s not supported" % dtype.name) - - super(_MaskedRandomUniformInitializer, self).__init__( - minval=minval, maxval=maxval, seed=seed, dtype=dtype) - self._num_mantissa_bits = 10 - self._num_valid_mantissa_bits = num_valid_mantissa_bits - - def __call__(self, shape, dtype=dtypes.float16, partition_info=None): - if dtype and dtype != dtypes.float16: - raise ValueError("dtype: %s not supported" % dtype.name) - res = super(_MaskedRandomUniformInitializer, self).__call__( - shape, dtype, partition_info) - # get uint16 view of the underlying buffer. - res = gen_array_ops.bitcast(res, dtypes.uint16) - - # mask the last `shift` mantissa bits. - shift = self._num_mantissa_bits - self._num_valid_mantissa_bits - mask = (0xffff >> shift) << shift - res = gen_bitwise_ops.bitwise_and(res, mask) - - # restore float16 view. - return gen_array_ops.bitcast(res, dtype) - - -def _get_initializer(init_bound, dtype, seed): - if dtype == dtypes.float16: - return _MaskedRandomUniformInitializer( - -init_bound, init_bound, dtype=dtype, seed=seed) - else: - return init_ops.random_uniform_initializer( - -init_bound, init_bound, dtype=dtype, seed=seed) - - -def blocks_match(sess, use_peephole, dtype=dtypes.float32, cell_clip=None): - batch_size = 2 - input_size = 3 - cell_size = 4 - sequence_length = 4 - - inputs = [] - for _ in range(sequence_length): - inp = ops.convert_to_tensor( - np.random.randn(batch_size, input_size), dtype=dtype) - inputs.append(inp) - stacked_inputs = array_ops.stack(inputs) - - init_bound = 1e-1 if dtype == dtypes.float16 else 1e-2 - initializer = _get_initializer(init_bound, dtype=dtype, seed=19890212) - - with variable_scope.variable_scope("test", initializer=initializer): - # magic naming so that the cells pick up these variables and reuse them - if use_peephole: - wci = variable_scope.get_variable( - "rnn/lstm_cell/w_i_diag", shape=[cell_size], dtype=dtype) - wcf = variable_scope.get_variable( - "rnn/lstm_cell/w_f_diag", shape=[cell_size], dtype=dtype) - wco = variable_scope.get_variable( - "rnn/lstm_cell/w_o_diag", shape=[cell_size], dtype=dtype) - - w = variable_scope.get_variable( - "rnn/lstm_cell/kernel", - shape=[input_size + cell_size, cell_size * 4], - dtype=dtype) - b = variable_scope.get_variable( - "rnn/lstm_cell/bias", - shape=[cell_size * 4], - dtype=dtype, - initializer=init_ops.zeros_initializer()) - - basic_cell = rnn_cell.LSTMCell( - cell_size, - use_peepholes=use_peephole, - cell_clip=cell_clip, - dtype=dtype, - state_is_tuple=True, - reuse=True) - basic_outputs_op, basic_state_op = rnn.static_rnn( - basic_cell, inputs, dtype=dtype) - - if use_peephole: - _, _, _, _, _, _, block_outputs_op = block_lstm( - ops.convert_to_tensor(sequence_length, dtype=dtypes.int64), - inputs, - w, - b, - wci=wci, - wcf=wcf, - wco=wco, - cell_clip=cell_clip, - use_peephole=True) - else: - _, _, _, _, _, _, block_outputs_op = block_lstm( - ops.convert_to_tensor(sequence_length, dtype=dtypes.int64), - inputs, - w, - b, - cell_clip=cell_clip) - - fused_cell = lstm_ops.LSTMBlockFusedCell( - cell_size, - cell_clip=cell_clip, - use_peephole=use_peephole, - reuse=True, - name="rnn/lstm_cell") - fused_outputs_op, fused_state_op = fused_cell(stacked_inputs, dtype=dtype) - - sess.run([variables.global_variables_initializer()]) - basic_outputs, basic_state = sess.run([basic_outputs_op, basic_state_op[0]]) - basic_grads = sess.run(gradients_impl.gradients(basic_outputs_op, inputs)) - xs = [w, b] - if use_peephole: - xs += [wci, wcf, wco] - basic_wgrads = sess.run(gradients_impl.gradients(basic_outputs_op, xs)) - - block_outputs = sess.run(block_outputs_op) - block_grads = sess.run(gradients_impl.gradients(block_outputs_op, inputs)) - block_wgrads = sess.run(gradients_impl.gradients(block_outputs_op, xs)) - - xs = [w, b] - if use_peephole: - xs += [wci, wcf, wco] - fused_outputs, fused_state = sess.run([fused_outputs_op, fused_state_op[0]]) - fused_grads = sess.run(gradients_impl.gradients(fused_outputs_op, inputs)) - fused_wgrads = sess.run(gradients_impl.gradients(fused_outputs_op, xs)) - - return (basic_state, fused_state, basic_outputs, block_outputs, - fused_outputs, basic_grads, block_grads, fused_grads, basic_wgrads, - block_wgrads, fused_wgrads) - - -class LSTMBlockCellTest(test.TestCase, parameterized.TestCase): - - TEST_CASES = ({ - "testcase_name": "Fp32", - "dtype": dtypes.float32, - "rtol": 1e-6, - "atol": 1e-6 - }, { - "testcase_name": "Fp16", - "dtype": dtypes.float16, - "rtol": 8e-3, - "atol": 8e-4 - }) - - def testNoneDimsWithDynamicRNN(self): - with self.session(use_gpu=True, graph=ops.Graph()) as sess: - batch_size = 4 - num_steps = 5 - input_dim = 6 - cell_size = 7 - - cell = lstm_ops.LSTMBlockCell(cell_size) - x = array_ops.placeholder(dtypes.float32, shape=(None, None, input_dim)) - - output, _ = rnn.dynamic_rnn( - cell, x, time_major=True, dtype=dtypes.float32) - sess.run(variables.global_variables_initializer()) - feed = {} - feed[x] = np.random.randn(num_steps, batch_size, input_dim) - sess.run(output, feed) - - def testLSTMBlockCell(self): - with self.session(use_gpu=True, graph=ops.Graph()) as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 2]) - m0 = array_ops.zeros([1, 2]) - m1 = array_ops.zeros([1, 2]) - m2 = array_ops.zeros([1, 2]) - m3 = array_ops.zeros([1, 2]) - g, ((out_m0, out_m1), (out_m2, out_m3)) = rnn_cell.MultiRNNCell( - [lstm_ops.LSTMBlockCell(2) - for _ in range(2)], state_is_tuple=True)(x, ((m0, m1), (m2, m3))) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g, out_m0, out_m1, out_m2, out_m3], { - x.name: np.array([[1., 1.]]), - m0.name: 0.1 * np.ones([1, 2]), - m1.name: 0.1 * np.ones([1, 2]), - m2.name: 0.1 * np.ones([1, 2]), - m3.name: 0.1 * np.ones([1, 2]) - }) - self.assertEqual(len(res), 5) - self.assertAllClose(res[0], [[0.24024698, 0.24024698]]) - # These numbers are from testBasicLSTMCell and only test c/h. - self.assertAllClose(res[1], [[0.68967271, 0.68967271]]) - self.assertAllClose(res[2], [[0.44848421, 0.44848421]]) - self.assertAllClose(res[3], [[0.39897051, 0.39897051]]) - self.assertAllClose(res[4], [[0.24024698, 0.24024698]]) - - def testCompatibleNames(self): - with self.session(use_gpu=True, graph=ops.Graph()): - cell = rnn_cell.LSTMCell(10) - pcell = rnn_cell.LSTMCell(10, use_peepholes=True) - inputs = [array_ops.zeros([4, 5])] * 6 - rnn.static_rnn(cell, inputs, dtype=dtypes.float32, scope="basic") - rnn.static_rnn(pcell, inputs, dtype=dtypes.float32, scope="peephole") - basic_names = { - v.name: v.get_shape() - for v in variables.trainable_variables() - } - - with self.session(use_gpu=True, graph=ops.Graph()): - cell = lstm_ops.LSTMBlockCell(10) - pcell = lstm_ops.LSTMBlockCell(10, use_peephole=True) - inputs = [array_ops.zeros([4, 5])] * 6 - rnn.static_rnn(cell, inputs, dtype=dtypes.float32, scope="basic") - rnn.static_rnn(pcell, inputs, dtype=dtypes.float32, scope="peephole") - block_names = { - v.name: v.get_shape() - for v in variables.trainable_variables() - } - - with self.session(use_gpu=True, graph=ops.Graph()): - cell = lstm_ops.LSTMBlockFusedCell(10) - pcell = lstm_ops.LSTMBlockFusedCell(10, use_peephole=True) - inputs = array_ops.stack([array_ops.zeros([4, 5])] * 6) - cell(inputs, dtype=dtypes.float32, scope="basic/lstm_cell") - pcell(inputs, dtype=dtypes.float32, scope="peephole/lstm_cell") - fused_names = { - v.name: v.get_shape() - for v in variables.trainable_variables() - } - - self.assertEqual(basic_names, block_names) - self.assertEqual(basic_names, fused_names) - - def testLSTMBasicToBlockCell(self): - with self.session(use_gpu=True) as sess: - x = array_ops.zeros([1, 2]) - x_values = np.random.randn(1, 2) - - m0_val = 0.1 * np.ones([1, 2]) - m1_val = -0.1 * np.ones([1, 2]) - m2_val = -0.2 * np.ones([1, 2]) - m3_val = 0.2 * np.ones([1, 2]) - - initializer = init_ops.random_uniform_initializer( - -0.01, 0.01, seed=19890212) - with variable_scope.variable_scope("basic", initializer=initializer): - m0 = array_ops.zeros([1, 2]) - m1 = array_ops.zeros([1, 2]) - m2 = array_ops.zeros([1, 2]) - m3 = array_ops.zeros([1, 2]) - g, ((out_m0, out_m1), (out_m2, out_m3)) = rnn_cell.MultiRNNCell( - [rnn_cell.BasicLSTMCell(2, state_is_tuple=True) for _ in range(2)], - state_is_tuple=True)(x, ((m0, m1), (m2, m3))) - sess.run([variables.global_variables_initializer()]) - basic_res = sess.run([g, out_m0, out_m1, out_m2, out_m3], { - x.name: x_values, - m0.name: m0_val, - m1.name: m1_val, - m2.name: m2_val, - m3.name: m3_val - }) - - with variable_scope.variable_scope("block", initializer=initializer): - m0 = array_ops.zeros([1, 2]) - m1 = array_ops.zeros([1, 2]) - m2 = array_ops.zeros([1, 2]) - m3 = array_ops.zeros([1, 2]) - g, ((out_m0, out_m1), (out_m2, out_m3)) = rnn_cell.MultiRNNCell( - [lstm_ops.LSTMBlockCell(2) - for _ in range(2)], state_is_tuple=True)(x, ((m0, m1), (m2, m3))) - sess.run([variables.global_variables_initializer()]) - block_res = sess.run([g, out_m0, out_m1, out_m2, out_m3], { - x.name: x_values, - m0.name: m0_val, - m1.name: m1_val, - m2.name: m2_val, - m3.name: m3_val - }) - - self.assertEqual(len(basic_res), len(block_res)) - for basic, block in zip(basic_res, block_res): - self.assertAllClose(basic, block) - - def testLSTMBasicToBlockCellPeeping(self): - with self.session(use_gpu=True) as sess: - x = array_ops.zeros([1, 2]) - x_values = np.random.randn(1, 2) - - m0_val = 0.1 * np.ones([1, 2]) - m1_val = -0.1 * np.ones([1, 2]) - m2_val = -0.2 * np.ones([1, 2]) - m3_val = 0.2 * np.ones([1, 2]) - - initializer = init_ops.random_uniform_initializer( - -0.01, 0.01, seed=19890212) - with variable_scope.variable_scope("basic", initializer=initializer): - m0 = array_ops.zeros([1, 2]) - m1 = array_ops.zeros([1, 2]) - m2 = array_ops.zeros([1, 2]) - m3 = array_ops.zeros([1, 2]) - g, ((out_m0, out_m1), (out_m2, out_m3)) = rnn_cell.MultiRNNCell( - [ - rnn_cell.LSTMCell(2, use_peepholes=True, state_is_tuple=True) - for _ in range(2) - ], - state_is_tuple=True)(x, ((m0, m1), (m2, m3))) - sess.run([variables.global_variables_initializer()]) - basic_res = sess.run([g, out_m0, out_m1, out_m2, out_m3], { - x.name: x_values, - m0.name: m0_val, - m1.name: m1_val, - m2.name: m2_val, - m3.name: m3_val - }) - - with variable_scope.variable_scope("block", initializer=initializer): - m0 = array_ops.zeros([1, 2]) - m1 = array_ops.zeros([1, 2]) - m2 = array_ops.zeros([1, 2]) - m3 = array_ops.zeros([1, 2]) - g, ((out_m0, out_m1), (out_m2, out_m3)) = rnn_cell.MultiRNNCell( - [lstm_ops.LSTMBlockCell(2, use_peephole=True) for _ in range(2)], - state_is_tuple=True)(x, ((m0, m1), (m2, m3))) - sess.run([variables.global_variables_initializer()]) - block_res = sess.run([g, out_m0, out_m1, out_m2, out_m3], { - x.name: x_values, - m0.name: m0_val, - m1.name: m1_val, - m2.name: m2_val, - m3.name: m3_val - }) - - self.assertEqual(len(basic_res), len(block_res)) - for basic, block in zip(basic_res, block_res): - self.assertAllClose(basic, block) - - def LSTMBasicToBlockTestHelper(self, - dtype=dtypes.float32, - use_peephole=False, - cell_clip=None, - rtol=1e-6, - atol=1e-6): - with self.session(use_gpu=True, graph=ops.Graph()) as sess: - (basic_state, fused_state, basic_outputs, block_outputs, fused_outputs, - basic_grads, block_grads, fused_grads, basic_wgrads, block_wgrads, - fused_wgrads) = blocks_match( - sess, use_peephole=use_peephole, dtype=dtype, cell_clip=cell_clip) - - self.assertAllClose(basic_outputs, block_outputs, rtol=rtol, atol=atol) - self.assertAllClose(basic_grads, block_grads, rtol=rtol, atol=atol) - for basic, block in zip(basic_wgrads, block_wgrads): - self.assertAllClose(basic, block, rtol=rtol, atol=atol) - - self.assertAllClose(basic_outputs, fused_outputs, rtol=rtol, atol=atol) - self.assertAllClose(basic_state, fused_state, rtol=rtol, atol=atol) - self.assertAllClose(basic_grads, fused_grads, rtol=rtol, atol=atol) - for basic, fused in zip(basic_wgrads, fused_wgrads): - self.assertAllClose(basic, fused, rtol=rtol, atol=atol) - - @parameterized.named_parameters(*TEST_CASES) - def testLSTMBasicToBlock(self, dtype, rtol, atol): - self.LSTMBasicToBlockTestHelper( - dtype, use_peephole=False, rtol=rtol, atol=atol) - - @parameterized.named_parameters(*TEST_CASES) - def testLSTMBasicToBlockPeeping(self, dtype, rtol, atol): - self.LSTMBasicToBlockTestHelper( - dtype, use_peephole=True, rtol=rtol, atol=atol) - - @parameterized.named_parameters(*TEST_CASES) - def testLSTMBasicToBlockCellClip(self, dtype, rtol, atol): - self.LSTMBasicToBlockTestHelper( - dtype, use_peephole=True, cell_clip=0.5, rtol=rtol, atol=atol) - - def testLSTMFusedSequenceLengths(self): - """Verify proper support for sequence lengths in LSTMBlockFusedCell.""" - with self.session(use_gpu=True) as sess: - batch_size = 3 - input_size = 4 - cell_size = 5 - max_sequence_length = 6 - - inputs = [] - for _ in range(max_sequence_length): - inp = ops.convert_to_tensor( - np.random.randn(batch_size, input_size), dtype=dtypes.float32) - inputs.append(inp) - seq_lengths = constant_op.constant([3, 4, 5]) - cell_inputs = array_ops.stack(inputs) - - initializer = init_ops.random_uniform_initializer( - -0.01, 0.01, seed=19890213) - - with variable_scope.variable_scope("lstm_cell", initializer=initializer): - # magic naming so that the cells pick up these variables and reuse them - variable_scope.get_variable( - "kernel", - shape=[input_size + cell_size, cell_size * 4], - dtype=dtypes.float32) - - variable_scope.get_variable( - "bias", - shape=[cell_size * 4], - dtype=dtypes.float32, - initializer=init_ops.zeros_initializer()) - - cell = lstm_ops.LSTMBlockFusedCell( - cell_size, cell_clip=0, use_peephole=False, reuse=True, - name="lstm_cell") - - fused_outputs_op, fused_state_op = cell( - cell_inputs, dtype=dtypes.float32, sequence_length=seq_lengths) - - cell_vars = [ - v for v in variables.trainable_variables() - if v.name.endswith("kernel") or v.name.endswith("bias") - ] - - # Verify that state propagation works if we turn our sequence into - # tiny (single-time) subsequences, i.e. unfuse the cell - unfused_outputs_op = [] - state = None - with variable_scope.variable_scope( - variable_scope.get_variable_scope(), reuse=True): - for i, inp in enumerate(inputs): - lengths = [int(i < l) for l in seq_lengths.eval()] - output, state = cell( - array_ops.expand_dims(inp, 0), - initial_state=state, - dtype=dtypes.float32, - sequence_length=lengths) - unfused_outputs_op.append(output[0]) - unfused_outputs_op = array_ops.stack(unfused_outputs_op) - - sess.run([variables.global_variables_initializer()]) - unfused_outputs, unfused_state = sess.run([unfused_outputs_op, state[0]]) - unfused_grads = sess.run( - gradients_impl.gradients(unfused_outputs_op, inputs)) - unfused_wgrads = sess.run( - gradients_impl.gradients(unfused_outputs_op, cell_vars)) - - fused_outputs, fused_state = sess.run( - [fused_outputs_op, fused_state_op[0]]) - fused_grads = sess.run(gradients_impl.gradients(fused_outputs_op, inputs)) - fused_wgrads = sess.run( - gradients_impl.gradients(fused_outputs_op, cell_vars)) - - self.assertAllClose(fused_outputs, unfused_outputs) - self.assertAllClose(fused_state, unfused_state) - self.assertAllClose(fused_grads, unfused_grads) - for fused, unfused in zip(fused_wgrads, unfused_wgrads): - self.assertAllClose(fused, unfused, rtol=1e-6, atol=1e-6) - -#### Benchmarking. - - -class BenchmarkLSTMBlock(test.Benchmark): - - def benchmarkLSTMBlockCellFpropWithDynamicRNN(self): - print("BlockLSTMCell forward propagation via dynamic_rnn().") - print("--------------------------------------------------------------") - print("LSTMBlockCell Seconds per inference.") - print("batch_size,cell_size,input_size,time_steps,use_gpu,wall_time") - iters = 10 - for config in benchmarking.dict_product({ - "batch_size": [1, 8, 13, 32, 67, 128], - "cell_size": [128, 250, 512, 650, 1024, 1350], - "time_steps": [40], - "use_gpu": [True, False], - "dtype": ["float32", "float16"], - }): - dtype = dtypes.float32 if config["dtype"] == "float32" else dtypes.float16 - with ops.Graph().as_default(): - with benchmarking.device(use_gpu=config["use_gpu"]): - inputs = variable_scope.get_variable( - "x", - dtype=dtype, - shape=[ - config["time_steps"], config["batch_size"], - config["cell_size"] - ]) - cell = lstm_ops.LSTMBlockCell(config["cell_size"], dtype=dtype) - outputs = rnn.dynamic_rnn(cell, inputs, time_major=True, dtype=dtype) - init_op = variables.global_variables_initializer() - - with session.Session() as sess: - sess.run(init_op) - wall_time = benchmarking.seconds_per_run(outputs, sess, iters) - - # Print to stdout. If the TEST_REPORT_FILE_PREFIX environment variable - # is set, this will produce a copy-paste-able CSV file. - print(",".join( - map(str, [ - config["dtype"], config["batch_size"], config["cell_size"], - config["cell_size"], config["time_steps"], config["use_gpu"], - wall_time - ]))) - benchmark_name_template = "_".join([ - "LSTMBlockCell_fprop", "DT_%(dtype)s", "BS%(batch_size)i", - "CS%(cell_size)i", "IS%(cell_size)i", "TS%(time_steps)i", - "gpu_%(use_gpu)s" - ]) - - self.report_benchmark( - name=benchmark_name_template % config, - iters=iters, - wall_time=wall_time, - extras=config) - - def benchmarkLSTMBlockCellBpropWithDynamicRNN(self): - print("BlockLSTMCell backward propagation via dynamic_rnn().") - print("--------------------------------------------------------------") - print("LSTMBlockCell Seconds per inference.") - print("batch_size,cell_size,input_size,time_steps,use_gpu,wall_time") - iters = 10 - for config in benchmarking.dict_product({ - "batch_size": [1, 8, 13, 32, 67, 128], - "cell_size": [128, 250, 512, 650, 1024, 1350], - "time_steps": [40], - "use_gpu": [True, False], - "dtype": ["float32", "float16"], - }): - dtype = dtypes.float32 if config["dtype"] == "float32" else dtypes.float16 - with ops.Graph().as_default(): - with benchmarking.device(use_gpu=config["use_gpu"]): - time_steps = config["time_steps"] - batch_size = config["batch_size"] - cell_size = input_size = config["cell_size"] - inputs = variable_scope.get_variable( - "x", [time_steps, batch_size, cell_size], - trainable=False, - dtype=dtype) - with variable_scope.variable_scope( - "rnn", reuse=variable_scope.AUTO_REUSE): - w = variable_scope.get_variable( - "rnn/lstm_cell/kernel", - shape=[input_size + cell_size, cell_size * 4], - dtype=dtype) - b = variable_scope.get_variable( - "rnn/lstm_cell/bias", - shape=[cell_size * 4], - dtype=dtype, - initializer=init_ops.zeros_initializer()) - cell = lstm_ops.LSTMBlockCell(cell_size, dtype=dtype) - outputs = rnn.dynamic_rnn( - cell, inputs, time_major=True, dtype=dtype) - grads = gradients_impl.gradients(outputs, [inputs, w, b]) - init_op = variables.global_variables_initializer() - - with session.Session() as sess: - sess.run(init_op) - wall_time = benchmarking.seconds_per_run(grads, sess, iters) - - # Print to stdout. If the TEST_REPORT_FILE_PREFIX environment variable - # is set, this will produce a copy-paste-able CSV file. - print(",".join( - map(str, [ - config["dtype"], batch_size, cell_size, cell_size, time_steps, - config["use_gpu"], wall_time - ]))) - benchmark_name_template = "_".join([ - "LSTMBlockCell_bprop", "DT_%(dtype)s", "BS%(batch_size)i", - "CS%(cell_size)i", "IS%(cell_size)i", "TS%(time_steps)i", - "gpu_%(use_gpu)s" - ]) - - self.report_benchmark( - name=benchmark_name_template % config, - iters=iters, - wall_time=wall_time, - extras=config) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py deleted file mode 100644 index 921b4baae43..00000000000 --- a/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py +++ /dev/null @@ -1,2190 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for RNN cells.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools - -import numpy as np - -from tensorflow.contrib.rnn.python.ops import core_rnn_cell as legacy_rnn_cell -from tensorflow.contrib.rnn.python.ops import rnn_cell as contrib_rnn_cell -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import test_util -from tensorflow.python.keras import initializers -from tensorflow.python.keras import layers as keras_layers -from tensorflow.python.keras import testing_utils -from tensorflow.python.keras import utils -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import rnn -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import rnn_cell_impl -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import test -from tensorflow.python.training import training -from tensorflow.python.util import nest - - -class RNNCellTest(test.TestCase): - - def _assert_cell_builds(self, cell_class, dtype, batch_size, in_size, - out_size): - cell = cell_class(out_size, dtype=dtype) - in_shape = tensor_shape.TensorShape((batch_size, in_size)) - cell.build(in_shape) - state_output = cell.get_initial_state( - inputs=None, batch_size=batch_size, dtype=dtype) - cell_output, _ = cell(array_ops.zeros(in_shape, dtype), state_output) - self.assertAllEqual([batch_size, out_size], cell_output.shape.as_list()) - - def testCellsBuild(self): - f32 = dtypes.float32 - f64 = dtypes.float64 - self._assert_cell_builds(contrib_rnn_cell.IndRNNCell, f32, 5, 7, 3) - self._assert_cell_builds(contrib_rnn_cell.IndRNNCell, f64, 5, 7, 3) - self._assert_cell_builds(contrib_rnn_cell.IndyGRUCell, f32, 5, 7, 3) - self._assert_cell_builds(contrib_rnn_cell.IndyGRUCell, f64, 5, 7, 3) - self._assert_cell_builds(contrib_rnn_cell.IndyLSTMCell, f32, 5, 7, 3) - self._assert_cell_builds(contrib_rnn_cell.IndyLSTMCell, f64, 5, 7, 3) - - def testIndRNNCell(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 2]) - m = array_ops.zeros([1, 2]) - cell = contrib_rnn_cell.IndRNNCell(2) - g, _ = cell(x, m) - self.assertEqual([ - "root/ind_rnn_cell/%s_w:0" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - "root/ind_rnn_cell/%s_u:0" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - "root/ind_rnn_cell/%s:0" % rnn_cell_impl._BIAS_VARIABLE_NAME - ], [v.name for v in cell.trainable_variables]) - self.assertFalse(cell.non_trainable_variables) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g], { - x.name: np.array([[1., 1.]]), - m.name: np.array([[0.1, 0.1]]) - }) - self.assertEqual(res[0].shape, (1, 2)) - - def testIndyGRUCell(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 2]) - m = array_ops.zeros([1, 2]) - g, _ = contrib_rnn_cell.IndyGRUCell(2)(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g], { - x.name: np.array([[1., 1.]]), - m.name: np.array([[0.1, 0.1]]) - }) - # Smoke test - self.assertAllClose(res[0], [[0.185265, 0.17704]]) - with variable_scope.variable_scope( - "other", initializer=init_ops.constant_initializer(0.5)): - # Test IndyGRUCell with input_size != num_units. - x = array_ops.zeros([1, 3]) - m = array_ops.zeros([1, 2]) - g, _ = contrib_rnn_cell.IndyGRUCell(2)(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g], { - x.name: np.array([[1., 1., 1.]]), - m.name: np.array([[0.1, 0.1]]) - }) - # Smoke test - self.assertAllClose(res[0], [[0.155127, 0.157328]]) - - def testIndyLSTMCell(self): - for dtype in [dtypes.float16, dtypes.float32]: - np_dtype = dtype.as_numpy_dtype - with self.session(graph=ops.Graph()) as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 2], dtype=dtype) - state_0 = (array_ops.zeros([1, 2], dtype=dtype),) * 2 - state_1 = (array_ops.zeros([1, 2], dtype=dtype),) * 2 - cell = rnn_cell_impl.MultiRNNCell( - [contrib_rnn_cell.IndyLSTMCell(2) for _ in range(2)]) - self.assertEqual(cell.dtype, None) - self.assertEqual("cell-0", cell._checkpoint_dependencies[0].name) - self.assertEqual("cell-1", cell._checkpoint_dependencies[1].name) - cell.get_config() # Should not throw an error - g, (out_state_0, out_state_1) = cell(x, (state_0, state_1)) - # Layer infers the input type. - self.assertEqual(cell.dtype, dtype.name) - expected_variable_names = [ - "root/multi_rnn_cell/cell_0/indy_lstm_cell/%s_w:0" % - rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - "root/multi_rnn_cell/cell_0/indy_lstm_cell/%s_u:0" % - rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - "root/multi_rnn_cell/cell_0/indy_lstm_cell/%s:0" % - rnn_cell_impl._BIAS_VARIABLE_NAME, - "root/multi_rnn_cell/cell_1/indy_lstm_cell/%s_w:0" % - rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - "root/multi_rnn_cell/cell_1/indy_lstm_cell/%s_u:0" % - rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - "root/multi_rnn_cell/cell_1/indy_lstm_cell/%s:0" % - rnn_cell_impl._BIAS_VARIABLE_NAME - ] - self.assertEqual(expected_variable_names, - [v.name for v in cell.trainable_variables]) - self.assertFalse(cell.non_trainable_variables) - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [g, out_state_0, out_state_1], { - x.name: np.array([[1., 1.]]), - state_0[0].name: 0.1 * np.ones([1, 2]), - state_0[1].name: 0.1 * np.ones([1, 2]), - state_1[0].name: 0.1 * np.ones([1, 2]), - state_1[1].name: 0.1 * np.ones([1, 2]), - }) - self.assertEqual(len(res), 3) - global_variables = variables.global_variables() - self.assertEqual(expected_variable_names, - [v.name for v in global_variables]) - # Only check the range of outputs as this is just a smoke test. - self.assertAllInRange(res[0], -1.0, 1.0) - self.assertAllInRange(res[1], -1.0, 1.0) - self.assertAllInRange(res[2], -1.0, 1.0) - with variable_scope.variable_scope( - "other", initializer=init_ops.constant_initializer(0.5)): - # Test IndyLSTMCell with input_size != num_units. - x = array_ops.zeros([1, 3], dtype=dtype) - state = (array_ops.zeros([1, 2], dtype=dtype),) * 2 - g, out_state = contrib_rnn_cell.IndyLSTMCell(2)(x, state) - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [g, out_state], { - x.name: np.array([[1., 1., 1.]], dtype=np_dtype), - state[0].name: 0.1 * np.ones([1, 2], dtype=np_dtype), - state[1].name: 0.1 * np.ones([1, 2], dtype=np_dtype), - }) - self.assertEqual(len(res), 2) - - def testLSTMCellLayerNorm(self): - with self.cached_session() as sess: - num_units = 2 - num_proj = 3 - batch_size = 1 - input_size = 4 - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([batch_size, input_size]) - c = array_ops.zeros([batch_size, num_units]) - h = array_ops.zeros([batch_size, num_proj]) - state = rnn_cell_impl.LSTMStateTuple(c, h) - cell = contrib_rnn_cell.LayerNormLSTMCell( - num_units=num_units, - num_proj=num_proj, - forget_bias=1.0, - layer_norm=True, - norm_gain=1.0, - norm_shift=0.0) - g, out_m = cell(x, state) - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [g, out_m], { - x.name: np.ones((batch_size, input_size)), - c.name: 0.1 * np.ones((batch_size, num_units)), - h.name: 0.1 * np.ones((batch_size, num_proj)) - }) - self.assertEqual(len(res), 2) - # The numbers in results were not calculated, this is mostly just a - # smoke test. - self.assertEqual(res[0].shape, (batch_size, num_proj)) - self.assertEqual(res[1][0].shape, (batch_size, num_units)) - self.assertEqual(res[1][1].shape, (batch_size, num_proj)) - # Different inputs so different outputs and states - for i in range(1, batch_size): - self.assertTrue( - float(np.linalg.norm((res[0][0, :] - res[0][i, :]))) < 1e-6) - self.assertTrue( - float(np.linalg.norm((res[1][0, :] - res[1][i, :]))) < 1e-6) - - def testOutputProjectionWrapper(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 3]) - m = array_ops.zeros([1, 3]) - cell = legacy_rnn_cell.OutputProjectionWrapper( - rnn_cell_impl.GRUCell(3), 2) - g, new_m = cell(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g, new_m], { - x.name: np.array([[1., 1., 1.]]), - m.name: np.array([[0.1, 0.1, 0.1]]) - }) - self.assertEqual(res[1].shape, (1, 3)) - # The numbers in results were not calculated, this is just a smoke test. - self.assertAllClose(res[0], [[0.231907, 0.231907]]) - - def testInputProjectionWrapper(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 2]) - m = array_ops.zeros([1, 3]) - cell = legacy_rnn_cell.InputProjectionWrapper( - rnn_cell_impl.GRUCell(3), num_proj=3) - g, new_m = cell(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g, new_m], { - x.name: np.array([[1., 1.]]), - m.name: np.array([[0.1, 0.1, 0.1]]) - }) - self.assertEqual(res[1].shape, (1, 3)) - # The numbers in results were not calculated, this is just a smoke test. - self.assertAllClose(res[0], [[0.154605, 0.154605, 0.154605]]) - - def testEmbeddingWrapper(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 1], dtype=dtypes.int32) - m = array_ops.zeros([1, 2]) - embedding_cell = legacy_rnn_cell.EmbeddingWrapper( - rnn_cell_impl.GRUCell(2), embedding_classes=3, embedding_size=2) - self.assertEqual(embedding_cell.output_size, 2) - g, new_m = embedding_cell(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g, new_m], { - x.name: np.array([[1]]), - m.name: np.array([[0.1, 0.1]]) - }) - self.assertEqual(res[1].shape, (1, 2)) - # The numbers in results were not calculated, this is just a smoke test. - self.assertAllClose(res[0], [[0.17139, 0.17139]]) - - def testEmbeddingWrapperWithDynamicRnn(self): - with self.cached_session() as sess: - with variable_scope.variable_scope("root"): - inputs = ops.convert_to_tensor([[[0], [0]]], dtype=dtypes.int64) - input_lengths = ops.convert_to_tensor([2], dtype=dtypes.int64) - embedding_cell = legacy_rnn_cell.EmbeddingWrapper( - rnn_cell_impl.BasicLSTMCell(1, state_is_tuple=True), - embedding_classes=1, - embedding_size=2) - outputs, _ = rnn.dynamic_rnn( - cell=embedding_cell, - inputs=inputs, - sequence_length=input_lengths, - dtype=dtypes.float32) - sess.run([variables.global_variables_initializer()]) - # This will fail if output's dtype is inferred from input's. - sess.run(outputs) - - def testSRUCell(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 2]) - m = array_ops.zeros([1, 2]) - g, _ = contrib_rnn_cell.SRUCell(2)(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g], { - x.name: np.array([[1., 1.]]), - m.name: np.array([[0.1, 0.1]]) - }) - # Smoke test - self.assertAllClose(res[0], [[0.509682, 0.509682]]) - - def testSRUCellKerasRNN(self): - """Tests that SRUCell works with keras RNN layer.""" - cell = contrib_rnn_cell.SRUCell(10) - seq_input = ops.convert_to_tensor( - np.random.rand(2, 3, 5), name="seq_input", dtype=dtypes.float32) - rnn_layer = keras_layers.RNN(cell=cell) - rnn_outputs_keras = rnn_layer(seq_input) - with self.cached_session() as sess: - sess.run([variables.global_variables_initializer()]) - self.assertEqual(sess.run(rnn_outputs_keras).shape, (2, 10)) - - def testSRUCellBiasType(self): - """Tests that the bias' dtype is properly set.""" - cell = contrib_rnn_cell.SRUCell(10) - cell.build((2, 3, 5)) - self.assertEqual(cell._bias.dtype, dtypes.float32_ref) - - cell = contrib_rnn_cell.SRUCell(10, dtype=dtypes.int32) - cell.build((2, 3, 5)) - self.assertEqual(cell._bias.dtype, dtypes.int32_ref) - - cell_input = ops.convert_to_tensor( - np.random.rand(2, 5), name="cell_input", dtype=dtypes.float16) - cell_state = ops.convert_to_tensor( - np.random.rand(2, 10), name="cell_state", dtype=dtypes.float16) - cell = contrib_rnn_cell.SRUCell(10) - cell(cell_input, [cell_state]) - self.assertEqual(cell._bias.dtype, dtypes.float16_ref) - - def testSRUCellWithDiffSize(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 3]) - m = array_ops.zeros([1, 2]) - g, _ = contrib_rnn_cell.SRUCell(2)(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g], { - x.name: np.array([[1., 1., 1.]]), - m.name: np.array([[0.1, 0.1]]) - }) - # Smoke test - self.assertAllClose(res[0], [[0.55255556, 0.55255556]]) - - def testCoupledInputForgetGateLSTMCell(self): - with self.cached_session() as sess: - num_units = 2 - state_size = num_units * 2 - batch_size = 3 - input_size = 4 - expected_output = np.array( - [[0.121753, 0.121753], [0.103349, 0.103349], [0.100178, 0.100178]], - dtype=np.float32) - expected_state = np.array( - [[0.137523, 0.137523, 0.121753, 0.121753], [ - 0.105450, 0.105450, 0.103349, 0.103349 - ], [0.100742, 0.100742, 0.100178, 0.100178]], - dtype=np.float32) - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([batch_size, input_size]) - m = array_ops.zeros([batch_size, state_size]) - output, state = contrib_rnn_cell.CoupledInputForgetGateLSTMCell( - num_units=num_units, forget_bias=1.0, state_is_tuple=False)(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [output, state], { - x.name: - np.array([[1., 1., 1., 1.], [2., 2., 2., 2.], - [3., 3., 3., 3.]]), - m.name: - 0.1 * np.ones((batch_size, state_size)) - }) - # This is a smoke test: Only making sure expected values didn't change. - self.assertEqual(len(res), 2) - self.assertAllClose(res[0], expected_output) - self.assertAllClose(res[1], expected_state) - - def testTimeFreqLSTMCell(self): - with self.cached_session() as sess: - num_units = 8 - state_size = num_units * 2 - batch_size = 3 - input_size = 4 - feature_size = 2 - frequency_skip = 1 - num_shifts = (input_size - feature_size) // frequency_skip + 1 - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([batch_size, input_size]) - m = array_ops.zeros([batch_size, state_size * num_shifts]) - output, state = contrib_rnn_cell.TimeFreqLSTMCell( - num_units=num_units, - feature_size=feature_size, - frequency_skip=frequency_skip, - forget_bias=1.0)(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [output, state], { - x.name: - np.array([[1., 1., 1., 1.], [2., 2., 2., 2.], - [3., 3., 3., 3.]]), - m.name: - 0.1 * np.ones((batch_size, int(state_size * (num_shifts)))) - }) - self.assertEqual(len(res), 2) - # The numbers in results were not calculated, this is mostly just a - # smoke test. - self.assertEqual(res[0].shape, (batch_size, num_units * num_shifts)) - self.assertEqual(res[1].shape, (batch_size, state_size * num_shifts)) - # Different inputs so different outputs and states - for i in range(1, batch_size): - self.assertTrue( - float(np.linalg.norm((res[0][0, :] - res[0][i, :]))) > 1e-6) - self.assertTrue( - float(np.linalg.norm((res[1][0, :] - res[1][i, :]))) > 1e-6) - - def testGridLSTMCell(self): - with self.cached_session() as sess: - num_units = 8 - batch_size = 3 - input_size = 4 - feature_size = 2 - frequency_skip = 1 - num_shifts = int((input_size - feature_size) / frequency_skip + 1) - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - cell = contrib_rnn_cell.GridLSTMCell( - num_units=num_units, - feature_size=feature_size, - frequency_skip=frequency_skip, - forget_bias=1.0, - num_frequency_blocks=[num_shifts], - couple_input_forget_gates=True, - state_is_tuple=True) - inputs = constant_op.constant( - np.array( - [[1., 1., 1., 1.], [2., 2., 2., 2.], [3., 3., 3., 3.]], - dtype=np.float32), - dtype=dtypes.float32) - state_value = constant_op.constant( - 0.1 * np.ones((batch_size, num_units), dtype=np.float32), - dtype=dtypes.float32) - init_state = cell.state_tuple_type(*( - [state_value, state_value] * num_shifts)) - output, state = cell(inputs, init_state) - sess.run([variables.global_variables_initializer()]) - res = sess.run([output, state]) - self.assertEqual(len(res), 2) - # The numbers in results were not calculated, this is mostly just a - # smoke test. - self.assertEqual(res[0].shape, (batch_size, num_units * num_shifts * 2)) - for ss in res[1]: - self.assertEqual(ss.shape, (batch_size, num_units)) - # Different inputs so different outputs and states - for i in range(1, batch_size): - self.assertTrue( - float(np.linalg.norm((res[0][0, :] - res[0][i, :]))) > 1e-6) - self.assertTrue( - float( - np.linalg.norm((res[1].state_f00_b00_c[0, :] - res[1] - .state_f00_b00_c[i, :]))) > 1e-6) - - def testGridLSTMCellWithFrequencyBlocks(self): - with self.cached_session() as sess: - num_units = 8 - batch_size = 3 - feature_size = 2 - frequency_skip = 1 - num_frequency_blocks = [1, 1] - total_blocks = num_frequency_blocks[0] + num_frequency_blocks[1] - start_freqindex_list = [0, 2] - end_freqindex_list = [2, 4] - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - cell = contrib_rnn_cell.GridLSTMCell( - num_units=num_units, - feature_size=feature_size, - frequency_skip=frequency_skip, - forget_bias=1.0, - num_frequency_blocks=num_frequency_blocks, - start_freqindex_list=start_freqindex_list, - end_freqindex_list=end_freqindex_list, - couple_input_forget_gates=True, - state_is_tuple=True) - inputs = constant_op.constant( - np.array( - [[1., 1., 1., 1.], [2., 2., 2., 2.], [3., 3., 3., 3.]], - dtype=np.float32), - dtype=dtypes.float32) - state_value = constant_op.constant( - 0.1 * np.ones((batch_size, num_units), dtype=np.float32), - dtype=dtypes.float32) - init_state = cell.state_tuple_type(*( - [state_value, state_value] * total_blocks)) - output, state = cell(inputs, init_state) - sess.run([variables.global_variables_initializer()]) - res = sess.run([output, state]) - self.assertEqual(len(res), 2) - # The numbers in results were not calculated, this is mostly just a - # smoke test. - self.assertEqual(res[0].shape, - (batch_size, num_units * total_blocks * 2)) - for ss in res[1]: - self.assertEqual(ss.shape, (batch_size, num_units)) - # Different inputs so different outputs and states - for i in range(1, batch_size): - self.assertTrue( - float(np.linalg.norm((res[0][0, :] - res[0][i, :]))) > 1e-6) - self.assertTrue( - float( - np.linalg.norm((res[1].state_f00_b00_c[0, :] - res[1] - .state_f00_b00_c[i, :]))) > 1e-6) - - def testGridLstmCellWithCoupledInputForgetGates(self): - num_units = 2 - batch_size = 3 - input_size = 4 - feature_size = 2 - frequency_skip = 1 - num_shifts = int((input_size - feature_size) / frequency_skip + 1) - expected_output = np.array( - [[ - 0.416383, 0.416383, 0.403238, 0.403238, 0.524020, 0.524020, - 0.565425, 0.565425, 0.557865, 0.557865, 0.609699, 0.609699 - ], [ - 0.627331, 0.627331, 0.622393, 0.622393, 0.688342, 0.688342, - 0.708078, 0.708078, 0.694245, 0.694245, 0.715171, 0.715171 - ], [ - 0.711050, 0.711050, 0.709197, 0.709197, 0.736533, 0.736533, - 0.744264, 0.744264, 0.737390, 0.737390, 0.745250, 0.745250 - ]], - dtype=np.float32) - expected_state = np.array( - [[ - 0.625556, 0.625556, 0.416383, 0.416383, 0.759134, 0.759134, - 0.524020, 0.524020, 0.798795, 0.798795, 0.557865, 0.557865 - ], [ - 0.875488, 0.875488, 0.627331, 0.627331, 0.936432, 0.936432, - 0.688342, 0.688342, 0.941961, 0.941961, 0.694245, 0.694245 - ], [ - 0.957327, 0.957327, 0.711050, 0.711050, 0.979522, 0.979522, - 0.736533, 0.736533, 0.980245, 0.980245, 0.737390, 0.737390 - ]], - dtype=np.float32) - for state_is_tuple in [False, True]: - with self.cached_session() as sess: - with variable_scope.variable_scope( - "state_is_tuple" + str(state_is_tuple), - initializer=init_ops.constant_initializer(0.5)): - cell = contrib_rnn_cell.GridLSTMCell( - num_units=num_units, - feature_size=feature_size, - frequency_skip=frequency_skip, - forget_bias=1.0, - num_frequency_blocks=[num_shifts], - couple_input_forget_gates=True, - state_is_tuple=state_is_tuple) - inputs = constant_op.constant( - np.array( - [[1., 1., 1., 1.], [2., 2., 2., 2.], [3., 3., 3., 3.]], - dtype=np.float32), - dtype=dtypes.float32) - if state_is_tuple: - state_value = constant_op.constant( - 0.1 * np.ones((batch_size, num_units), dtype=np.float32), - dtype=dtypes.float32) - init_state = cell.state_tuple_type(*( - [state_value, state_value] * num_shifts)) - else: - init_state = constant_op.constant( - 0.1 * np.ones( - (batch_size, num_units * num_shifts * 2), dtype=np.float32), - dtype=dtypes.float32) - output, state = cell(inputs, init_state) - sess.run([variables.global_variables_initializer()]) - res = sess.run([output, state]) - # This is a smoke test: Only making sure expected values not change. - self.assertEqual(len(res), 2) - self.assertAllClose(res[0], expected_output) - if not state_is_tuple: - self.assertAllClose(res[1], expected_state) - else: - # There should be num_shifts * 2 states in the tuple. - self.assertEqual(len(res[1]), num_shifts * 2) - # Checking the shape of each state to be batch_size * num_units - for ss in res[1]: - self.assertEqual(ss.shape[0], batch_size) - self.assertEqual(ss.shape[1], num_units) - self.assertAllClose(np.concatenate(res[1], axis=1), expected_state) - - def testBidirectionGridLSTMCell(self): - with self.cached_session() as sess: - num_units = 2 - batch_size = 3 - input_size = 4 - feature_size = 2 - frequency_skip = 1 - num_shifts = int((input_size - feature_size) / frequency_skip + 1) - expected_output = np.array( - [[ - 0.464130, 0.464130, 0.419165, 0.419165, 0.593283, 0.593283, - 0.738350, 0.738350, 0.661638, 0.661638, 0.866774, 0.866774, - 0.520789, 0.520789, 0.476968, 0.476968, 0.604341, 0.604341, - 0.760207, 0.760207, 0.635773, 0.635773, 0.850218, 0.850218 - ], [ - 0.669636, 0.669636, 0.628966, 0.628966, 0.736057, 0.736057, - 0.895927, 0.895927, 0.755559, 0.755559, 0.954359, 0.954359, - 0.692621, 0.692621, 0.652363, 0.652363, 0.737517, 0.737517, - 0.899558, 0.899558, 0.745984, 0.745984, 0.946840, 0.946840 - ], [ - 0.751109, 0.751109, 0.711716, 0.711716, 0.778357, 0.778357, - 0.940779, 0.940779, 0.784530, 0.784530, 0.980604, 0.980604, - 0.759940, 0.759940, 0.720652, 0.720652, 0.778552, 0.778552, - 0.941606, 0.941606, 0.781035, 0.781035, 0.977731, 0.977731 - ]], - dtype=np.float32) - expected_state = np.array( - [[ - 0.710660, 0.710660, 0.464130, 0.464130, 0.877293, 0.877293, - 0.593283, 0.593283, 0.958505, 0.958505, 0.661638, 0.661638, - 0.785405, 0.785405, 0.520789, 0.520789, 0.890836, 0.890836, - 0.604341, 0.604341, 0.928512, 0.928512, 0.635773, 0.635773 - ], [ - 0.967579, 0.967579, 0.669636, 0.669636, 1.038811, 1.038811, - 0.736057, 0.736057, 1.058201, 1.058201, 0.755559, 0.755559, - 0.993088, 0.993088, 0.692621, 0.692621, 1.040288, 1.040288, - 0.737517, 0.737517, 1.048773, 1.048773, 0.745984, 0.745984 - ], [ - 1.053842, 1.053842, 0.751109, 0.751109, 1.079919, 1.079919, - 0.778357, 0.778357, 1.085620, 1.085620, 0.784530, 0.784530, - 1.062455, 1.062455, 0.759940, 0.759940, 1.080101, 1.080101, - 0.778552, 0.778552, 1.082402, 1.082402, 0.781035, 0.781035 - ]], - dtype=np.float32) - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - cell = contrib_rnn_cell.BidirectionalGridLSTMCell( - num_units=num_units, - feature_size=feature_size, - share_time_frequency_weights=True, - frequency_skip=frequency_skip, - forget_bias=1.0, - num_frequency_blocks=[num_shifts]) - inputs = constant_op.constant( - np.array( - [[1.0, 1.1, 1.2, 1.3], [2.0, 2.1, 2.2, 2.3], - [3.0, 3.1, 3.2, 3.3]], - dtype=np.float32), - dtype=dtypes.float32) - state_value = constant_op.constant( - 0.1 * np.ones((batch_size, num_units), dtype=np.float32), - dtype=dtypes.float32) - init_state = cell.state_tuple_type(*( - [state_value, state_value] * num_shifts * 2)) - output, state = cell(inputs, init_state) - sess.run([variables.global_variables_initializer()]) - res = sess.run([output, state]) - self.assertEqual(len(res), 2) - # The numbers in results were not calculated, this is mostly just a - # smoke test. - self.assertEqual(res[0].shape, (batch_size, num_units * num_shifts * 4)) - self.assertAllClose(res[0], expected_output) - # There should be num_shifts * 4 states in the tuple. - self.assertEqual(len(res[1]), num_shifts * 4) - # Checking the shape of each state to be batch_size * num_units - for ss in res[1]: - self.assertEqual(ss.shape[0], batch_size) - self.assertEqual(ss.shape[1], num_units) - self.assertAllClose(np.concatenate(res[1], axis=1), expected_state) - - def testBidirectionGridLSTMCellWithSliceOffset(self): - with self.cached_session() as sess: - num_units = 2 - batch_size = 3 - input_size = 4 - feature_size = 2 - frequency_skip = 1 - num_shifts = int((input_size - feature_size) / frequency_skip + 1) - expected_output = np.array( - [[ - 0.464130, 0.464130, 0.419165, 0.419165, 0.593283, 0.593283, - 0.738350, 0.738350, 0.661638, 0.661638, 0.866774, 0.866774, - 0.322645, 0.322645, 0.276068, 0.276068, 0.584654, 0.584654, - 0.690292, 0.690292, 0.640446, 0.640446, 0.840071, 0.840071 - ], [ - 0.669636, 0.669636, 0.628966, 0.628966, 0.736057, 0.736057, - 0.895927, 0.895927, 0.755559, 0.755559, 0.954359, 0.954359, - 0.493625, 0.493625, 0.449236, 0.449236, 0.730828, 0.730828, - 0.865996, 0.865996, 0.749429, 0.749429, 0.944958, 0.944958 - ], [ - 0.751109, 0.751109, 0.711716, 0.711716, 0.778357, 0.778357, - 0.940779, 0.940779, 0.784530, 0.784530, 0.980604, 0.980604, - 0.608587, 0.608587, 0.566683, 0.566683, 0.777345, 0.777345, - 0.925820, 0.925820, 0.782597, 0.782597, 0.976858, 0.976858 - ]], - dtype=np.float32) - expected_state = np.array( - [[ - 0.710660, 0.710660, 0.464130, 0.464130, 0.877293, 0.877293, - 0.593283, 0.593283, 0.958505, 0.958505, 0.661638, 0.661638, - 0.516575, 0.516575, 0.322645, 0.322645, 0.866628, 0.866628, - 0.584654, 0.584654, 0.934002, 0.934002, 0.640446, 0.640446 - ], [ - 0.967579, 0.967579, 0.669636, 0.669636, 1.038811, 1.038811, - 0.736057, 0.736057, 1.058201, 1.058201, 0.755559, 0.755559, - 0.749836, 0.749836, 0.493625, 0.493625, 1.033488, 1.033488, - 0.730828, 0.730828, 1.052186, 1.052186, 0.749429, 0.749429 - ], [ - 1.053842, 1.053842, 0.751109, 0.751109, 1.079919, 1.079919, - 0.778357, 0.778357, 1.085620, 1.085620, 0.784530, 0.784530, - 0.895999, 0.895999, 0.608587, 0.608587, 1.078978, 1.078978, - 0.777345, 0.777345, 1.083843, 1.083843, 0.782597, 0.782597 - ]], - dtype=np.float32) - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - cell = contrib_rnn_cell.BidirectionalGridLSTMCell( - num_units=num_units, - feature_size=feature_size, - share_time_frequency_weights=True, - frequency_skip=frequency_skip, - forget_bias=1.0, - num_frequency_blocks=[num_shifts], - backward_slice_offset=1) - inputs = constant_op.constant( - np.array( - [[1.0, 1.1, 1.2, 1.3], [2.0, 2.1, 2.2, 2.3], - [3.0, 3.1, 3.2, 3.3]], - dtype=np.float32), - dtype=dtypes.float32) - state_value = constant_op.constant( - 0.1 * np.ones((batch_size, num_units), dtype=np.float32), - dtype=dtypes.float32) - init_state = cell.state_tuple_type(*( - [state_value, state_value] * num_shifts * 2)) - output, state = cell(inputs, init_state) - sess.run([variables.global_variables_initializer()]) - res = sess.run([output, state]) - self.assertEqual(len(res), 2) - # The numbers in results were not calculated, this is mostly just a - # smoke test. - self.assertEqual(res[0].shape, (batch_size, num_units * num_shifts * 4)) - self.assertAllClose(res[0], expected_output) - # There should be num_shifts * 4 states in the tuple. - self.assertEqual(len(res[1]), num_shifts * 4) - # Checking the shape of each state to be batch_size * num_units - for ss in res[1]: - self.assertEqual(ss.shape[0], batch_size) - self.assertEqual(ss.shape[1], num_units) - self.assertAllClose(np.concatenate(res[1], axis=1), expected_state) - - def testAttentionCellWrapperFailures(self): - with self.assertRaisesRegexp( - TypeError, rnn_cell_impl.ASSERT_LIKE_RNNCELL_ERROR_REGEXP): - contrib_rnn_cell.AttentionCellWrapper(None, 0) - - num_units = 8 - for state_is_tuple in [False, True]: - with ops.Graph().as_default(): - lstm_cell = rnn_cell.BasicLSTMCell( - num_units, state_is_tuple=state_is_tuple) - with self.assertRaisesRegexp( - ValueError, "attn_length should be greater than zero, got 0"): - contrib_rnn_cell.AttentionCellWrapper( - lstm_cell, 0, state_is_tuple=state_is_tuple) - with self.assertRaisesRegexp( - ValueError, "attn_length should be greater than zero, got -1"): - contrib_rnn_cell.AttentionCellWrapper( - lstm_cell, -1, state_is_tuple=state_is_tuple) - with ops.Graph().as_default(): - lstm_cell = rnn_cell.BasicLSTMCell(num_units, state_is_tuple=True) - with self.assertRaisesRegexp( - ValueError, "Cell returns tuple of states, but the flag " - "state_is_tuple is not set. State size is: *"): - contrib_rnn_cell.AttentionCellWrapper( - lstm_cell, 4, state_is_tuple=False) - - def testAttentionCellWrapperZeros(self): - num_units = 8 - attn_length = 16 - batch_size = 3 - input_size = 4 - for state_is_tuple in [False, True]: - with ops.Graph().as_default(): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "state_is_tuple_" + str(state_is_tuple)): - lstm_cell = rnn_cell.BasicLSTMCell( - num_units, state_is_tuple=state_is_tuple) - cell = contrib_rnn_cell.AttentionCellWrapper( - lstm_cell, attn_length, state_is_tuple=state_is_tuple) - if state_is_tuple: - zeros = array_ops.zeros([batch_size, num_units], dtype=np.float32) - attn_state_zeros = array_ops.zeros( - [batch_size, attn_length * num_units], dtype=np.float32) - zero_state = ((zeros, zeros), zeros, attn_state_zeros) - else: - zero_state = array_ops.zeros( - [ - batch_size, - num_units * 2 + attn_length * num_units + num_units - ], - dtype=np.float32) - inputs = array_ops.zeros( - [batch_size, input_size], dtype=dtypes.float32) - output, state = cell(inputs, zero_state) - self.assertEquals(output.get_shape(), [batch_size, num_units]) - if state_is_tuple: - self.assertEquals(len(state), 3) - self.assertEquals(len(state[0]), 2) - self.assertEquals(state[0][0].get_shape(), - [batch_size, num_units]) - self.assertEquals(state[0][1].get_shape(), - [batch_size, num_units]) - self.assertEquals(state[1].get_shape(), [batch_size, num_units]) - self.assertEquals(state[2].get_shape(), - [batch_size, attn_length * num_units]) - tensors = [output] + list(state) - else: - self.assertEquals(state.get_shape(), [ - batch_size, - num_units * 2 + num_units + attn_length * num_units - ]) - tensors = [output, state] - zero_result = sum( - [math_ops.reduce_sum(math_ops.abs(x)) for x in tensors]) - sess.run(variables.global_variables_initializer()) - self.assertTrue(sess.run(zero_result) < 1e-6) - - def testAttentionCellWrapperValues(self): - num_units = 8 - attn_length = 16 - batch_size = 3 - for state_is_tuple in [False, True]: - with ops.Graph().as_default(): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "state_is_tuple_" + str(state_is_tuple)): - lstm_cell = rnn_cell.BasicLSTMCell( - num_units, state_is_tuple=state_is_tuple) - cell = contrib_rnn_cell.AttentionCellWrapper( - lstm_cell, attn_length, state_is_tuple=state_is_tuple) - if state_is_tuple: - zeros = constant_op.constant( - 0.1 * np.ones([batch_size, num_units], dtype=np.float32), - dtype=dtypes.float32) - attn_state_zeros = constant_op.constant( - 0.1 * np.ones( - [batch_size, attn_length * num_units], dtype=np.float32), - dtype=dtypes.float32) - zero_state = ((zeros, zeros), zeros, attn_state_zeros) - else: - zero_state = constant_op.constant( - 0.1 * np.ones( - [ - batch_size, - num_units * 2 + num_units + attn_length * num_units - ], - dtype=np.float32), - dtype=dtypes.float32) - inputs = constant_op.constant( - np.array( - [[1., 1., 1., 1.], [2., 2., 2., 2.], [3., 3., 3., 3.]], - dtype=np.float32), - dtype=dtypes.float32) - output, state = cell(inputs, zero_state) - if state_is_tuple: - concat_state = array_ops.concat( - [state[0][0], state[0][1], state[1], state[2]], 1) - else: - concat_state = state - sess.run(variables.global_variables_initializer()) - output, state = sess.run([output, concat_state]) - # Different inputs so different outputs and states - for i in range(1, batch_size): - self.assertTrue( - float(np.linalg.norm((output[0, :] - output[i, :]))) > 1e-6) - self.assertTrue( - float(np.linalg.norm((state[0, :] - state[i, :]))) > 1e-6) - - def _testAttentionCellWrapperCorrectResult(self): - num_units = 4 - attn_length = 6 - batch_size = 2 - expected_output = np.array( - [[1.068372, 0.45496, -0.678277, 0.340538], - [1.018088, 0.378983, -0.572179, 0.268591]], - dtype=np.float32) - expected_state = np.array( - [[ - 0.74946702, 0.34681597, 0.26474735, 1.06485605, 0.38465962, - 0.11420801, 0.10272158, 0.30925757, 0.63899988, 0.7181077, - 0.47534478, 0.33715725, 0.58086717, 0.49446869, 0.7641536, - 0.12814975, 0.92231739, 0.89857256, 0.21889746, 0.38442063, - 0.53481543, 0.8876909, 0.45823169, 0.5905602, 0.78038228, - 0.56501579, 0.03971386, 0.09870267, 0.8074435, 0.66821432, - 0.99211812, 0.12295902, 1.14606023, 0.34370938, -0.79251152, - 0.51843399 - ], [ - 0.5179342, 0.48682183, -0.25426468, 0.96810579, 0.28809637, - 0.13607743, -0.11446252, 0.26792109, 0.78047138, 0.63460857, - 0.49122369, 0.52007174, 0.73000264, 0.66986895, 0.73576689, - 0.86301267, 0.87887371, 0.35185754, 0.93417215, 0.64732957, - 0.63173044, 0.66627824, 0.53644657, 0.20477486, 0.98458421, - 0.38277245, 0.03746676, 0.92510188, 0.57714164, 0.84932971, - 0.36127412, 0.12125921, 1.1362772, 0.34361625, -0.78150457, - 0.70582712 - ]], - dtype=np.float32) - seed = 12345 - random_seed.set_random_seed(seed) - rnn_scope = None - for state_is_tuple in [False, True]: - with session.Session() as sess: - with variable_scope.variable_scope( - "state_is_tuple", - reuse=state_is_tuple, - initializer=init_ops.glorot_uniform_initializer()): - lstm_cell = rnn_cell.BasicLSTMCell( - num_units, state_is_tuple=state_is_tuple) - cell = contrib_rnn_cell.AttentionCellWrapper( - lstm_cell, attn_length, state_is_tuple=state_is_tuple) - # This is legacy behavior to preserve the test. Weight - # sharing no longer works by creating a new RNNCell in the - # same variable scope; so here we restore the scope of the - # RNNCells after the first use below. - if rnn_scope is not None: - (cell._scope, lstm_cell._scope) = rnn_scope # pylint: disable=protected-access,unpacking-non-sequence - zeros1 = random_ops.random_uniform( - (batch_size, num_units), 0.0, 1.0, seed=seed + 1) - zeros2 = random_ops.random_uniform( - (batch_size, num_units), 0.0, 1.0, seed=seed + 2) - zeros3 = random_ops.random_uniform( - (batch_size, num_units), 0.0, 1.0, seed=seed + 3) - attn_state_zeros = random_ops.random_uniform( - (batch_size, attn_length * num_units), 0.0, 1.0, seed=seed + 4) - zero_state = ((zeros1, zeros2), zeros3, attn_state_zeros) - if not state_is_tuple: - zero_state = array_ops.concat([ - zero_state[0][0], zero_state[0][1], zero_state[1], zero_state[2] - ], 1) - inputs = random_ops.random_uniform( - (batch_size, num_units), 0.0, 1.0, seed=seed + 5) - output, state = cell(inputs, zero_state) - # This is legacy behavior to preserve the test. Weight - # sharing no longer works by creating a new RNNCell in the - # same variable scope; so here we store the scope of the - # first RNNCell for reuse above. - if rnn_scope is None: - rnn_scope = (cell._scope, lstm_cell._scope) # pylint: disable=protected-access - if state_is_tuple: - state = array_ops.concat( - [state[0][0], state[0][1], state[1], state[2]], 1) - sess.run(variables.global_variables_initializer()) - self.assertAllClose(sess.run(output), expected_output) - self.assertAllClose(sess.run(state), expected_state) - - def testNASCell(self): - num_units = 6 - batch_size = 3 - expected_output = np.array( - [[0.576751, 0.576751, 0.576751, 0.576751, 0.576751, 0.576751], - [0.618936, 0.618936, 0.618936, 0.618936, 0.618936, 0.618936], - [0.627393, 0.627393, 0.627393, 0.627393, 0.627393, 0.627393]]) - expected_state = np.array([[ - 0.71579772, 0.71579772, 0.71579772, 0.71579772, 0.71579772, 0.71579772, - 0.57675087, 0.57675087, 0.57675087, 0.57675087, 0.57675087, 0.57675087 - ], [ - 0.78041625, 0.78041625, 0.78041625, 0.78041625, 0.78041625, 0.78041625, - 0.6189357, 0.6189357, 0.61893570, 0.6189357, 0.6189357, 0.6189357 - ], [ - 0.79457647, 0.79457647, 0.79457647, 0.79457647, 0.79457653, 0.79457653, - 0.62739348, 0.62739348, 0.62739348, 0.62739348, 0.62739348, 0.62739348 - ]]) - with self.cached_session() as sess: - with variable_scope.variable_scope( - "nas_test", initializer=init_ops.constant_initializer(0.5)): - cell = contrib_rnn_cell.NASCell(num_units=num_units) - inputs = constant_op.constant( - np.array( - [[1., 1., 1., 1.], [2., 2., 2., 2.], [3., 3., 3., 3.]], - dtype=np.float32), - dtype=dtypes.float32) - state_value = constant_op.constant( - 0.1 * np.ones((batch_size, num_units), dtype=np.float32), - dtype=dtypes.float32) - init_state = rnn_cell.LSTMStateTuple(state_value, state_value) - output, state = cell(inputs, init_state) - sess.run([variables.global_variables_initializer()]) - res = sess.run([output, state]) - - # This is a smoke test: Only making sure expected values not change. - self.assertEqual(len(res), 2) - self.assertAllClose(res[0], expected_output) - # There should be 2 states in the tuple. - self.assertEqual(len(res[1]), 2) - # Checking the shape of each state to be batch_size * num_units - new_c, new_h = res[1] - self.assertEqual(new_c.shape[0], batch_size) - self.assertEqual(new_c.shape[1], num_units) - self.assertEqual(new_h.shape[0], batch_size) - self.assertEqual(new_h.shape[1], num_units) - self.assertAllClose(np.concatenate(res[1], axis=1), expected_state) - - def testNASCellProj(self): - num_units = 6 - batch_size = 3 - num_proj = 5 - expected_output = np.array( - [[1.697418, 1.697418, 1.697418, 1.697418, - 1.697418], [1.840037, 1.840037, 1.840037, 1.840037, 1.840037], - [1.873985, 1.873985, 1.873985, 1.873985, 1.873985]]) - expected_state = np.array([[ - 0.69855207, 0.69855207, 0.69855207, 0.69855207, 0.69855207, 0.69855207, - 1.69741797, 1.69741797, 1.69741797, 1.69741797, 1.69741797 - ], [ - 0.77073824, 0.77073824, 0.77073824, 0.77073824, 0.77073824, 0.77073824, - 1.84003687, 1.84003687, 1.84003687, 1.84003687, 1.84003687 - ], [ - 0.78973997, 0.78973997, 0.78973997, 0.78973997, 0.78973997, 0.78973997, - 1.87398517, 1.87398517, 1.87398517, 1.87398517, 1.87398517 - ]]) - with self.cached_session() as sess: - with variable_scope.variable_scope( - "nas_proj_test", initializer=init_ops.constant_initializer(0.5)): - cell = contrib_rnn_cell.NASCell(num_units=num_units, num_proj=num_proj) - inputs = constant_op.constant( - np.array( - [[1., 1., 1., 1.], [2., 2., 2., 2.], [3., 3., 3., 3.]], - dtype=np.float32), - dtype=dtypes.float32) - state_value_c = constant_op.constant( - 0.1 * np.ones((batch_size, num_units), dtype=np.float32), - dtype=dtypes.float32) - state_value_h = constant_op.constant( - 0.1 * np.ones((batch_size, num_proj), dtype=np.float32), - dtype=dtypes.float32) - init_state = rnn_cell.LSTMStateTuple(state_value_c, state_value_h) - output, state = cell(inputs, init_state) - sess.run([variables.global_variables_initializer()]) - res = sess.run([output, state]) - - # This is a smoke test: Only making sure expected values not change. - self.assertEqual(len(res), 2) - self.assertAllClose(res[0], expected_output) - # There should be 2 states in the tuple. - self.assertEqual(len(res[1]), 2) - # Checking the shape of each state to be batch_size * num_units - new_c, new_h = res[1] - self.assertEqual(new_c.shape[0], batch_size) - self.assertEqual(new_c.shape[1], num_units) - self.assertEqual(new_h.shape[0], batch_size) - self.assertEqual(new_h.shape[1], num_proj) - self.assertAllClose(np.concatenate(res[1], axis=1), expected_state) - - @test_util.run_in_graph_and_eager_modes - def testNASCellKerasRNN(self): - """Tests that NASCell works with keras RNN layer.""" - cell = contrib_rnn_cell.NASCell(10) - seq_input = ops.convert_to_tensor( - np.random.rand(2, 3, 5), name="seq_input", dtype=dtypes.float32) - rnn_layer = keras_layers.RNN(cell=cell) - rnn_outputs = rnn_layer(seq_input) - self.evaluate([variables.global_variables_initializer()]) - self.assertEqual(self.evaluate(rnn_outputs).shape, (2, 10)) - - def testUGRNNCell(self): - num_units = 2 - batch_size = 3 - expected_state_and_output = np.array( - [[0.13752282, 0.13752282], [0.10545051, 0.10545051], - [0.10074195, 0.10074195]], - dtype=np.float32) - with self.cached_session() as sess: - with variable_scope.variable_scope( - "ugrnn_cell_test", initializer=init_ops.constant_initializer(0.5)): - cell = contrib_rnn_cell.UGRNNCell(num_units=num_units) - inputs = constant_op.constant( - np.array( - [[1., 1., 1., 1.], [2., 2., 2., 2.], [3., 3., 3., 3.]], - dtype=np.float32), - dtype=dtypes.float32) - init_state = constant_op.constant( - 0.1 * np.ones((batch_size, num_units), dtype=np.float32), - dtype=dtypes.float32) - output, state = cell(inputs, init_state) - sess.run([variables.global_variables_initializer()]) - res = sess.run([output, state]) - # This is a smoke test: Only making sure expected values didn't change. - self.assertEqual(len(res), 2) - self.assertAllClose(res[0], expected_state_and_output) - self.assertAllClose(res[1], expected_state_and_output) - - def testIntersectionRNNCell(self): - num_units = 2 - batch_size = 3 - expected_state = np.array( - [[0.13752282, 0.13752282], [0.10545051, 0.10545051], - [0.10074195, 0.10074195]], - dtype=np.float32) - expected_output = np.array( - [[2.00431061, 2.00431061], [4.00060606, 4.00060606], - [6.00008249, 6.00008249]], - dtype=np.float32) - with self.cached_session() as sess: - with variable_scope.variable_scope( - "intersection_rnn_cell_test", - initializer=init_ops.constant_initializer(0.5)): - cell = contrib_rnn_cell.IntersectionRNNCell( - num_units=num_units, num_in_proj=num_units) - inputs = constant_op.constant( - np.array( - [[1., 1., 1., 1.], [2., 2., 2., 2.], [3., 3., 3., 3.]], - dtype=np.float32), - dtype=dtypes.float32) - init_state = constant_op.constant( - 0.1 * np.ones((batch_size, num_units), dtype=np.float32), - dtype=dtypes.float32) - output, state = cell(inputs, init_state) - sess.run([variables.global_variables_initializer()]) - res = sess.run([output, state]) - # This is a smoke test: Only making sure expected values didn't change. - self.assertEqual(len(res), 2) - self.assertAllClose(res[0], expected_output) - self.assertAllClose(res[1], expected_state) - - def testIntersectionRNNCellFailure(self): - num_units = 2 - batch_size = 3 - cell = contrib_rnn_cell.IntersectionRNNCell(num_units=num_units) - inputs = constant_op.constant( - np.array( - [[1., 1., 1., 1.], [2., 2., 2., 2.], [3., 3., 3., 3.]], - dtype=np.float32), - dtype=dtypes.float32) - init_state = constant_op.constant( - 0.1 * np.ones((batch_size, num_units), dtype=np.float32), - dtype=dtypes.float32) - with self.assertRaisesRegexp(ValueError, - "Must have input size == output size for " - "Intersection RNN. To fix, num_in_proj should " - "be set to num_units at cell init."): - cell(inputs, init_state) - - def testPhasedLSTMCell(self): - with self.cached_session() as sess: - num_units = 2 - batch_size = 3 - input_size = 4 - expected_state_c = np.array( - [[0.00072015, 0.00036633], [0.00083481, 0.00047266], - [0.00085111, 0.00053054]], - dtype=np.float32) - expected_state_h = np.array( - [[0.0005159, 0.00026243], [0.00062958, 0.00035646], - [0.00064732, 0.00040351]], - dtype=np.float32) - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - t = array_ops.zeros([batch_size, 1], dtype=dtypes.float64) - x = array_ops.zeros([batch_size, input_size]) - c0 = array_ops.zeros([batch_size, 2]) - h0 = array_ops.zeros([batch_size, 2]) - state0 = rnn_cell.LSTMStateTuple(c0, h0) - output, state = contrib_rnn_cell.PhasedLSTMCell(num_units=num_units)( - (t, x), state0) - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [output, state], { - t.name: - np.array([[1.], [2.], [3.]]), - x.name: - np.array([[1., 1., 1., 1.], [2., 2., 2., 2.], - [3., 3., 3., 3.]]), - }) - # This is a smoke test, making sure expected values are unchanged. - self.assertEqual(len(res), 2) - self.assertAllClose(res[0], res[1].h) - self.assertAllClose(res[1].c, expected_state_c) - self.assertAllClose(res[1].h, expected_state_h) - - def testConv1DLSTMCell(self): - with self.cached_session() as sess: - shape = [2, 1] - filter_size = [3] - num_features = 1 - expected_state_c = np.array( - [[[1.4375670191], [1.4375670191]], [[2.7542609292], [2.7542609292]]], - dtype=np.float32) - expected_state_h = np.array( - [[[0.6529865603], [0.6529865603]], [[0.8736877431], [0.8736877431]]], - dtype=np.float32) - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(1.0 / 2.0)): - x = array_ops.placeholder(dtypes.float32, [None, None, 1]) - cell = contrib_rnn_cell.Conv1DLSTMCell( - input_shape=shape, - kernel_shape=filter_size, - output_channels=num_features) - hidden = cell.zero_state(array_ops.shape(x)[0], dtypes.float32) - output, state = cell(x, hidden) - - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [output, state], { - hidden[0].name: np.array([[[1.], [1.]], [[2.], [2.]]]), - x.name: np.array([[[1.], [1.]], [[2.], [2.]]]), - }) - # This is a smoke test, making sure expected values are unchanged. - self.assertEqual(len(res), 2) - self.assertAllClose(res[0], res[1].h) - self.assertAllClose(res[1].c, expected_state_c) - self.assertAllClose(res[1].h, expected_state_h) - - def testConv2DLSTMCell(self): - with self.cached_session() as sess: - shape = [2, 2, 1] - filter_size = [3, 3] - num_features = 1 - expected_state_c = np.array( - [[[[1.4375670191], [1.4375670191]], [[1.4375670191], [1.4375670191]]], - [[[2.7542609292], [2.7542609292]], [[2.7542609292], [2.7542609292]] - ]], - dtype=np.float32) - expected_state_h = np.array( - [[[[0.6529865603], [0.6529865603]], [[0.6529865603], [0.6529865603]]], - [[[0.8736877431], [0.8736877431]], [[0.8736877431], [0.8736877431]] - ]], - dtype=np.float32) - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(1.0 / 4.0)): - x = array_ops.placeholder(dtypes.float32, [None, None, None, 1]) - cell = contrib_rnn_cell.Conv2DLSTMCell( - input_shape=shape, - kernel_shape=filter_size, - output_channels=num_features) - hidden = cell.zero_state(array_ops.shape(x)[0], dtypes.float32) - output, state = cell(x, hidden) - - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [output, state], { - hidden[0].name: - np.array([[[[1.], [1.]], [[1.], [1.]]], [[[2.], [2.]], - [[2.], [2.]]]]), - x.name: - np.array([[[[1.], [1.]], [[1.], [1.]]], [[[2.], [2.]], - [[2.], [2.]]]]), - }) - # This is a smoke test, making sure expected values are unchanged. - self.assertEqual(len(res), 2) - self.assertAllClose(res[0], res[1].h) - self.assertAllClose(res[1].c, expected_state_c) - self.assertAllClose(res[1].h, expected_state_h) - - def testConv3DLSTMCell(self): - with self.cached_session() as sess: - shape = [2, 2, 2, 1] - filter_size = [3, 3, 3] - num_features = 1 - expected_state_c = np.array( - [[[[[1.4375670191], [1.4375670191]], [[1.4375670191], [1.4375670191]] - ], [[[1.4375670191], [1.4375670191]], [[1.4375670191], - [1.4375670191]]]], - [[[[2.7542609292], [2.7542609292]], [[2.7542609292], [2.7542609292]] - ], [[[2.7542609292], [2.7542609292]], [[2.7542609292], - [2.7542609292]]]]], - dtype=np.float32) - expected_state_h = np.array( - [[[[[0.6529865603], [0.6529865603]], [[0.6529865603], [0.6529865603]] - ], [[[0.6529865603], [0.6529865603]], [[0.6529865603], - [0.6529865603]]]], - [[[[0.8736877431], [0.8736877431]], [[0.8736877431], [0.8736877431]] - ], [[[0.8736877431], [0.8736877431]], [[0.8736877431], - [0.8736877431]]]]], - dtype=np.float32) - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(1.0 / 8.0)): - x = array_ops.placeholder(dtypes.float32, [None, None, None, None, 1]) - cell = contrib_rnn_cell.Conv3DLSTMCell( - input_shape=shape, - kernel_shape=filter_size, - output_channels=num_features) - hidden = cell.zero_state(array_ops.shape(x)[0], dtypes.float32) - output, state = cell(x, hidden) - - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [output, state], { - hidden[0].name: - np.array([[[[[1.], [1.]], [[1.], [1.]]], [[[1.], [1.]], [[ - 1. - ], [1.]]]], [[[[2.], [2.]], [[2.], [2.]]], - [[[2.], [2.]], [[2.], [2.]]]]]), - x.name: - np.array([[[[[1.], [1.]], [[1.], [1.]]], [[[1.], [1.]], [[ - 1. - ], [1.]]]], [[[[2.], [2.]], [[2.], [2.]]], [[[2.], [2.]], - [[2.], [2.]]]]]) - }) - # This is a smoke test, making sure expected values are unchanged. - self.assertEqual(len(res), 2) - self.assertAllClose(res[0], res[1].h) - self.assertAllClose(res[1].c, expected_state_c) - self.assertAllClose(res[1].h, expected_state_h) - - def testHighwayWrapper(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "base_cell", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 3]) - m = array_ops.zeros([1, 3]) - base_cell = rnn_cell.GRUCell(3) - g, m_new = base_cell(x, m) - with variable_scope.variable_scope( - "hw_cell", initializer=init_ops.constant_initializer(0.5)): - hw_cell = contrib_rnn_cell.HighwayWrapper( - rnn_cell.GRUCell(3), carry_bias_init=-100.0) - g_res, m_new_res = hw_cell(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g, g_res, m_new, m_new_res], { - x: np.array([[1., 1., 1.]]), - m: np.array([[0.1, 0.1, 0.1]]) - }) - # As carry_bias_init is very negative, the carry gate is 'open' and the - # transform gate is 'closed'. This means the output equals the input. - self.assertAllClose(res[1], res[0]) - # States are left untouched - self.assertAllClose(res[2], res[3]) - - def testGLSTMCell(self): - # Ensure that G-LSTM matches LSTM when number_of_groups = 1 - batch_size = 2 - num_units = 4 - number_of_groups = 1 - - # Try with input dimension equal to num_units or not. - for num_inputs in [num_units, num_units + number_of_groups]: - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root1_%d" % num_inputs, - initializer=init_ops.constant_initializer(0.5)): - x = array_ops.ones([batch_size, num_inputs]) - # When number_of_groups = 1, G-LSTM is equivalent to regular LSTM - gcell = contrib_rnn_cell.GLSTMCell( - num_units=num_units, number_of_groups=number_of_groups) - cell = rnn_cell.LSTMCell(num_units=num_units) - self.assertTrue(isinstance(gcell.state_size, tuple)) - zero_state = gcell.zero_state( - batch_size=batch_size, dtype=dtypes.float32) - gh, gs = gcell(x, zero_state) - h, g = cell(x, zero_state) - - sess.run([variables.global_variables_initializer()]) - glstm_result = sess.run([gh, gs]) - lstm_result = sess.run([h, g]) - - self.assertAllClose(glstm_result[0], lstm_result[0], 1e-5) - self.assertAllClose(glstm_result[1], lstm_result[1], 1e-5) - - # Test that G-LSTM subgroup act like corresponding sub-LSTMs - batch_size = 2 - num_units = 4 - number_of_groups = 2 - - # Try with num_inputs equal to or not equal to num_units. - for num_inputs in [num_units, num_units + number_of_groups]: - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root2_%d" % num_inputs, - initializer=init_ops.constant_initializer(0.5)): - # input for G-LSTM with 2 groups - glstm_input = array_ops.ones([batch_size, num_inputs]) - gcell = contrib_rnn_cell.GLSTMCell( - num_units=num_units, number_of_groups=number_of_groups) - gcell_zero_state = gcell.zero_state( - batch_size=batch_size, dtype=dtypes.float32) - gh, gs = gcell(glstm_input, gcell_zero_state) - - # input for LSTM cell simulating single G-LSTM group - lstm_input = array_ops.ones( - [batch_size, num_inputs / number_of_groups]) - # note division by number_of_groups. This cell one simulates G-LSTM - # group - cell = rnn_cell.LSTMCell(num_units=int(num_units / number_of_groups)) - cell_zero_state = cell.zero_state( - batch_size=batch_size, dtype=dtypes.float32) - h, g = cell(lstm_input, cell_zero_state) - - sess.run([variables.global_variables_initializer()]) - [gh_res, h_res] = sess.run([gh, h]) - self.assertAllClose(gh_res[:, 0:int(num_units / number_of_groups)], - h_res, 1e-5) - self.assertAllClose(gh_res[:, int(num_units / number_of_groups):], - h_res, 1e-5) - - def testGLSTMCellFailure(self): - batch_size = 2 - num_units = 4 - number_of_groups = 2 - with self.cached_session(): - with variable_scope.variable_scope( - "glstm_failure", initializer=init_ops.constant_initializer(0.5)): - gcell = contrib_rnn_cell.GLSTMCell( - num_units=num_units, number_of_groups=number_of_groups) - gcell_zero_state = gcell.zero_state( - batch_size=batch_size, dtype=dtypes.float32) - - # Try an input with statically-unknown innermost dimension. - glstm_input = array_ops.placeholder( - dtypes.float32, shape=[batch_size, None]) - with self.assertRaisesRegexp(ValueError, - "input size must be statically known"): - gcell(glstm_input, gcell_zero_state) - - # Try an input whose innermost dimension isn't divisible into groups. - glstm_input = array_ops.placeholder( - dtypes.float32, shape=[batch_size, 3]) - with self.assertRaisesRegexp( - ValueError, - r"input size \(3\) must be divisible by number_of_groups \(2\)"): - gcell(glstm_input, gcell_zero_state) - - def testCFNCell(self): - with self.cached_session() as sess: - with variable_scope.variable_scope("root"): - x = array_ops.zeros([1, 2]) - m = array_ops.zeros([1, 2]) - cell = contrib_rnn_cell.CFNCell( - units=2, - kernel_initializer=initializers.Constant(0.5)) - g, _ = cell(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g], { - x.name: np.array([[1., 1.]]), - m.name: np.array([[0.1, 0.1]]) - }) - # Smoke test - self.assertAllClose(res[0], [[0.17188203, 0.17188203]]) - with variable_scope.variable_scope("other"): - # Test CFN with input_size != num_units. - x = array_ops.zeros([1, 3]) - m = array_ops.zeros([1, 2]) - cell = contrib_rnn_cell.CFNCell( - units=2, - kernel_initializer=initializers.Constant(0.5)) - g, _ = cell(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g], { - x.name: np.array([[1., 1., 1.]]), - m.name: np.array([[0.1, 0.1]]) - }) - # Smoke test - self.assertAllClose(res[0], [[0.15535763, 0.15535763]]) - - def testCFNCellEndToEnd(self): - with self.cached_session() as sess: - input_shape = 10 - output_shape = 5 - timestep = 4 - batch = 100 - (x_train, y_train), _ = testing_utils.get_test_data( - train_samples=batch, - test_samples=0, - input_shape=(timestep, input_shape), - num_classes=output_shape) - y_train = utils.to_categorical(y_train) - cell = contrib_rnn_cell.CFNCell(output_shape) - - inputs = array_ops.placeholder( - dtypes.float32, shape=(None, timestep, input_shape)) - predict = array_ops.placeholder( - dtypes.float32, shape=(None, output_shape)) - - outputs, state = rnn.dynamic_rnn( - cell, inputs, dtype=dtypes.float32) - self.assertEqual(outputs.shape.as_list(), [None, timestep, output_shape]) - self.assertEqual(state.shape.as_list(), [None, output_shape]) - loss = losses.softmax_cross_entropy(predict, state) - train_op = training.GradientDescentOptimizer(0.001).minimize(loss) - - sess.run([variables.global_variables_initializer()]) - _, outputs, state = sess.run( - [train_op, outputs, state], {inputs: x_train, predict: y_train}) - - self.assertEqual(len(outputs), batch) - self.assertEqual(len(state), batch) - - def testMinimalRNNCell(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root"): - x = array_ops.zeros([1, 2]) - m = array_ops.zeros([1, 2]) - cell = contrib_rnn_cell.MinimalRNNCell( - units=2, - kernel_initializer=initializers.Constant(0.5)) - g, _ = cell(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g], { - x.name: np.array([[1., 1.]]), - m.name: np.array([[0.1, 0.1]]) - }) - # Smoke test - self.assertAllClose(res[0], [[0.18899589, 0.18899589]]) - with variable_scope.variable_scope( - "other"): - # Test MinimalRNN with input_size != num_units. - x = array_ops.zeros([1, 3]) - m = array_ops.zeros([1, 2]) - cell = contrib_rnn_cell.MinimalRNNCell( - units=2, - kernel_initializer=initializers.Constant(0.5)) - g, _ = cell(x, m) - sess.run([variables.global_variables_initializer()]) - res = sess.run([g], { - x.name: np.array([[1., 1., 1.]]), - m.name: np.array([[0.1, 0.1]]) - }) - # Smoke test - self.assertAllClose(res[0], [[0.19554167, 0.19554167]]) - - def testMinimalRNNCellEndToEnd(self): - with self.cached_session() as sess: - input_shape = 10 - output_shape = 5 - timestep = 4 - batch = 100 - (x_train, y_train), _ = testing_utils.get_test_data( - train_samples=batch, - test_samples=0, - input_shape=(timestep, input_shape), - num_classes=output_shape) - y_train = utils.to_categorical(y_train) - cell = contrib_rnn_cell.MinimalRNNCell(output_shape) - - inputs = array_ops.placeholder( - dtypes.float32, shape=(None, timestep, input_shape)) - predict = array_ops.placeholder( - dtypes.float32, shape=(None, output_shape)) - - outputs, state = rnn.dynamic_rnn( - cell, inputs, dtype=dtypes.float32) - self.assertEqual(outputs.shape.as_list(), [None, timestep, output_shape]) - self.assertEqual(state.shape.as_list(), [None, output_shape]) - loss = losses.softmax_cross_entropy(predict, state) - train_op = training.GradientDescentOptimizer(0.001).minimize(loss) - - sess.run([variables.global_variables_initializer()]) - _, outputs, state = sess.run( - [train_op, outputs, state], {inputs: x_train, predict: y_train}) - - self.assertEqual(len(outputs), batch) - self.assertEqual(len(state), batch) - - def testNTMCell(self): - expected_output = np.array( - [[-0.04973561, -0.00020032, -0.09586009, -0.05049511], - [-0.02199885, 0.02302885, -0.05558189, -0.02051288], - [-0.01399924, 0.02543444, -0.06975862, -0.03782758], - [-0.02238393, 0.0135776, -0.09102941, -0.05594013]], - dtype=np.float32) - expected_read_vector_list = np.array( - [[1e-6, 1e-6, 1e-6, 1e-6], [1e-6, 1e-6, 1e-6, 1e-6], - [1e-6, 1e-6, 1e-6, 1e-6], [1e-6, 1e-6, 1e-6, 1e-6]], - dtype=np.float32) - expected_w_list = np.array( - [[[0.15837428, 0.21354634, 0.22115856, 0.21117255, 0.19574821], - [0.15826838, 0.2150458, 0.2228198, 0.20747298, 0.19639312], - [0.15750293, 0.21550071, 0.22280747, 0.20737495, 0.19681393], - [0.15763053, 0.21473582, 0.22187267, 0.20920397, 0.19655706]], - [[0.21703579, 0.19425659, 0.22143759, 0.18024713, 0.18702294], - [0.2164267, 0.19451937, 0.22112325, 0.18051708, 0.18741359], - [0.21567065, 0.1947548, 0.22107735, 0.18058982, 0.18790732], - [0.2163743, 0.194361, 0.22131558, 0.18042919, 0.1875199]]], - dtype=np.float32) - expected_M_0 = np.array( - [[-0.00553495, -0.01089884, 0.00683121, -0.00273276], - [-0.00495392, -0.00975483, 0.00611433, -0.00244583], - [-0.00564722, -0.0111199, 0.00696973, -0.0027882], - [-0.00459658, -0.00905126, 0.00567345, -0.00226937], - [-0.00476941, -0.00939155, 0.00588669, -0.00235472]], - dtype=np.float32) - - with session.Session() as sess: - with variable_scope.variable_scope("root"): - seed = 1234 - random_seed.set_random_seed(seed) - batch_size = 4 - inputs = random_ops.random_uniform((batch_size, 4), - 0.0, - 1.0, - seed=seed + 1) - cell = contrib_rnn_cell.NTMCell( - controller=rnn_cell_impl.LSTMCell(num_units=4), - memory_size=5, - memory_vector_dim=4, - read_head_num=1, - write_head_num=1) - output, state = cell(inputs, cell.zero_state(batch_size, - dtypes.float32)) - sess.run([variables.global_variables_initializer()]) - res, read_vector_list, w_list, M = sess.run( - [output, state.read_vector_list, state.w_list, state.M]) - # Smoke test - self.assertAllClose(res, expected_output) - self.assertAllClose(read_vector_list[0], expected_read_vector_list) - self.assertAllClose(w_list, expected_w_list) - self.assertAllClose(M[0], expected_M_0) - - -class LayerNormBasicLSTMCellTest(test.TestCase): - - # NOTE: all the values in the current test case have been calculated. - - def testBasicLSTMCell(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 2]) - c0 = array_ops.zeros([1, 2]) - h0 = array_ops.zeros([1, 2]) - state0 = rnn_cell.LSTMStateTuple(c0, h0) - c1 = array_ops.zeros([1, 2]) - h1 = array_ops.zeros([1, 2]) - state1 = rnn_cell.LSTMStateTuple(c1, h1) - state = (state0, state1) - single_cell = lambda: contrib_rnn_cell.LayerNormBasicLSTMCell(2) - cell = rnn_cell.MultiRNNCell([single_cell() for _ in range(2)]) - g, out_m = cell(x, state) - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [g, out_m], { - x.name: np.array([[1., 1.]]), - c0.name: 0.1 * np.asarray([[0, 1]]), - h0.name: 0.1 * np.asarray([[2, 3]]), - c1.name: 0.1 * np.asarray([[4, 5]]), - h1.name: 0.1 * np.asarray([[6, 7]]), - }) - - expected_h = np.array([[-0.38079708, 0.38079708]]) - expected_state0_c = np.array([[-1.0, 1.0]]) - expected_state0_h = np.array([[-0.38079708, 0.38079708]]) - expected_state1_c = np.array([[-1.0, 1.0]]) - expected_state1_h = np.array([[-0.38079708, 0.38079708]]) - - actual_h = res[0] - actual_state0_c = res[1][0].c - actual_state0_h = res[1][0].h - actual_state1_c = res[1][1].c - actual_state1_h = res[1][1].h - - self.assertAllClose(actual_h, expected_h, 1e-5) - self.assertAllClose(expected_state0_c, actual_state0_c, 1e-5) - self.assertAllClose(expected_state0_h, actual_state0_h, 1e-5) - self.assertAllClose(expected_state1_c, actual_state1_c, 1e-5) - self.assertAllClose(expected_state1_h, actual_state1_h, 1e-5) - - with variable_scope.variable_scope( - "other", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros( - [1, 3]) # Test BasicLSTMCell with input_size != num_units. - c = array_ops.zeros([1, 2]) - h = array_ops.zeros([1, 2]) - state = rnn_cell.LSTMStateTuple(c, h) - cell = contrib_rnn_cell.LayerNormBasicLSTMCell(2) - g, out_m = cell(x, state) - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [g, out_m], { - x.name: np.array([[1., 1., 1.]]), - c.name: 0.1 * np.asarray([[0, 1]]), - h.name: 0.1 * np.asarray([[2, 3]]), - }) - - expected_h = np.array([[-0.38079708, 0.38079708]]) - expected_c = np.array([[-1.0, 1.0]]) - self.assertEqual(len(res), 2) - self.assertAllClose(res[0], expected_h, 1e-5) - self.assertAllClose(res[1].c, expected_c, 1e-5) - self.assertAllClose(res[1].h, expected_h, 1e-5) - - def testBasicLSTMCellWithoutNorm(self): - """Tests that BasicLSTMCell with layer_norm=False.""" - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 2]) - c0 = array_ops.zeros([1, 2]) - h0 = array_ops.zeros([1, 2]) - state0 = rnn_cell.LSTMStateTuple(c0, h0) - c1 = array_ops.zeros([1, 2]) - h1 = array_ops.zeros([1, 2]) - state1 = rnn_cell.LSTMStateTuple(c1, h1) - state = (state0, state1) - single_cell = lambda: contrib_rnn_cell.LayerNormBasicLSTMCell(2, layer_norm=False) # pylint: disable=line-too-long - cell = rnn_cell.MultiRNNCell([single_cell() for _ in range(2)]) - g, out_m = cell(x, state) - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [g, out_m], { - x.name: np.array([[1., 1.]]), - c0.name: 0.1 * np.asarray([[0, 1]]), - h0.name: 0.1 * np.asarray([[2, 3]]), - c1.name: 0.1 * np.asarray([[4, 5]]), - h1.name: 0.1 * np.asarray([[6, 7]]), - }) - - expected_h = np.array([[0.70230919, 0.72581059]]) - expected_state0_c = np.array([[0.8020075, 0.89599884]]) - expected_state0_h = np.array([[0.56668288, 0.60858738]]) - expected_state1_c = np.array([[1.17500675, 1.26892781]]) - expected_state1_h = np.array([[0.70230919, 0.72581059]]) - - actual_h = res[0] - actual_state0_c = res[1][0].c - actual_state0_h = res[1][0].h - actual_state1_c = res[1][1].c - actual_state1_h = res[1][1].h - - self.assertAllClose(actual_h, expected_h, 1e-5) - self.assertAllClose(expected_state0_c, actual_state0_c, 1e-5) - self.assertAllClose(expected_state0_h, actual_state0_h, 1e-5) - self.assertAllClose(expected_state1_c, actual_state1_c, 1e-5) - self.assertAllClose(expected_state1_h, actual_state1_h, 1e-5) - - with variable_scope.variable_scope( - "other", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros( - [1, 3]) # Test BasicLSTMCell with input_size != num_units. - c = array_ops.zeros([1, 2]) - h = array_ops.zeros([1, 2]) - state = rnn_cell.LSTMStateTuple(c, h) - cell = contrib_rnn_cell.LayerNormBasicLSTMCell(2, layer_norm=False) - g, out_m = cell(x, state) - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [g, out_m], { - x.name: np.array([[1., 1., 1.]]), - c.name: 0.1 * np.asarray([[0, 1]]), - h.name: 0.1 * np.asarray([[2, 3]]), - }) - - expected_h = np.array([[0.64121795, 0.68166804]]) - expected_c = np.array([[0.88477188, 0.98103917]]) - self.assertEqual(len(res), 2) - self.assertAllClose(res[0], expected_h, 1e-5) - self.assertAllClose(res[1].c, expected_c, 1e-5) - self.assertAllClose(res[1].h, expected_h, 1e-5) - - def testBasicLSTMCellWithStateTuple(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 2]) - c0 = array_ops.zeros([1, 2]) - h0 = array_ops.zeros([1, 2]) - state0 = rnn_cell.LSTMStateTuple(c0, h0) - c1 = array_ops.zeros([1, 2]) - h1 = array_ops.zeros([1, 2]) - state1 = rnn_cell.LSTMStateTuple(c1, h1) - cell = rnn_cell.MultiRNNCell( - [contrib_rnn_cell.LayerNormBasicLSTMCell(2) for _ in range(2)]) - h, (s0, s1) = cell(x, (state0, state1)) - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [h, s0, s1], { - x.name: np.array([[1., 1.]]), - c0.name: 0.1 * np.asarray([[0, 1]]), - h0.name: 0.1 * np.asarray([[2, 3]]), - c1.name: 0.1 * np.asarray([[4, 5]]), - h1.name: 0.1 * np.asarray([[6, 7]]), - }) - - expected_h = np.array([[-0.38079708, 0.38079708]]) - expected_h0 = np.array([[-0.38079708, 0.38079708]]) - expected_c0 = np.array([[-1.0, 1.0]]) - expected_h1 = np.array([[-0.38079708, 0.38079708]]) - expected_c1 = np.array([[-1.0, 1.0]]) - - self.assertEqual(len(res), 3) - self.assertAllClose(res[0], expected_h, 1e-5) - self.assertAllClose(res[1].c, expected_c0, 1e-5) - self.assertAllClose(res[1].h, expected_h0, 1e-5) - self.assertAllClose(res[2].c, expected_c1, 1e-5) - self.assertAllClose(res[2].h, expected_h1, 1e-5) - - def testBasicLSTMCellWithStateTupleLayerNorm(self): - """The results of LSTMCell and LayerNormBasicLSTMCell should be the same.""" - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 2]) - c0 = array_ops.zeros([1, 2]) - h0 = array_ops.zeros([1, 2]) - state0 = rnn_cell_impl.LSTMStateTuple(c0, h0) - c1 = array_ops.zeros([1, 2]) - h1 = array_ops.zeros([1, 2]) - state1 = rnn_cell_impl.LSTMStateTuple(c1, h1) - cell = rnn_cell_impl.MultiRNNCell([ - contrib_rnn_cell.LayerNormLSTMCell( - 2, layer_norm=True, norm_gain=1.0, norm_shift=0.0) - for _ in range(2) - ]) - h, (s0, s1) = cell(x, (state0, state1)) - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [h, s0, s1], { - x.name: np.array([[1., 1.]]), - c0.name: 0.1 * np.asarray([[0, 1]]), - h0.name: 0.1 * np.asarray([[2, 3]]), - c1.name: 0.1 * np.asarray([[4, 5]]), - h1.name: 0.1 * np.asarray([[6, 7]]), - }) - - expected_h = np.array([[-0.38079708, 0.38079708]]) - expected_h0 = np.array([[-0.38079708, 0.38079708]]) - expected_c0 = np.array([[-1.0, 1.0]]) - expected_h1 = np.array([[-0.38079708, 0.38079708]]) - expected_c1 = np.array([[-1.0, 1.0]]) - - self.assertEqual(len(res), 3) - self.assertAllClose(res[0], expected_h, 1e-5) - self.assertAllClose(res[1].c, expected_c0, 1e-5) - self.assertAllClose(res[1].h, expected_h0, 1e-5) - self.assertAllClose(res[2].c, expected_c1, 1e-5) - self.assertAllClose(res[2].h, expected_h1, 1e-5) - - def testBasicLSTMCellWithDropout(self): - - def _is_close(x, y, digits=4): - delta = x - y - return delta < 10**(-digits) - - def _is_close_in(x, items, digits=4): - for i in items: - if _is_close(x, i, digits): - return True - return False - - keep_prob = 0.5 - c_high = 2.9998924946 - c_low = 0.999983298578 - h_low = 0.761552567265 - h_high = 0.995008519604 - num_units = 5 - allowed_low = [1, 2, 3] - - with self.cached_session() as sess: - with variable_scope.variable_scope( - "other", initializer=init_ops.constant_initializer(1)): - x = array_ops.zeros([1, 5]) - c = array_ops.zeros([1, 5]) - h = array_ops.zeros([1, 5]) - state = rnn_cell.LSTMStateTuple(c, h) - cell = contrib_rnn_cell.LayerNormBasicLSTMCell( - num_units, layer_norm=False, dropout_keep_prob=keep_prob) - - g, s = cell(x, state) - sess.run([variables.global_variables_initializer()]) - res = sess.run( - [g, s], { - x.name: np.ones([1, 5]), - c.name: np.ones([1, 5]), - h.name: np.ones([1, 5]), - }) - - # Since the returned tensors are of size [1,n] - # get the first component right now. - actual_h = res[0][0] - actual_state_c = res[1].c[0] - actual_state_h = res[1].h[0] - - # For each item in `c` (the cell inner state) check that - # it is equal to one of the allowed values `c_high` (not - # dropped out) or `c_low` (dropped out) and verify that the - # corresponding item in `h` (the cell activation) is coherent. - # Count the dropped activations and check that their number is - # coherent with the dropout probability. - dropped_count = 0 - self.assertTrue((actual_h == actual_state_h).all()) - for citem, hitem in zip(actual_state_c, actual_state_h): - self.assertTrue(_is_close_in(citem, [c_low, c_high])) - if _is_close(citem, c_low): - self.assertTrue(_is_close(hitem, h_low)) - dropped_count += 1 - elif _is_close(citem, c_high): - self.assertTrue(_is_close(hitem, h_high)) - self.assertIn(dropped_count, allowed_low) - - -def _create_multi_lstm_cell_ops(batch_size, num_units, input_depth, num_layers, - max_time, compiled): - with variable_scope.variable_scope( - "root", - initializer=init_ops.random_uniform_initializer(-0.1, 0.1, seed=2)): - inputs = variable_scope.get_variable( - "inputs", - initializer=random_ops.random_uniform( - (max_time, batch_size, input_depth), seed=1)) - maybe_xla = lambda c: contrib_rnn_cell.CompiledWrapper(c) if compiled else c - cell = rnn_cell.MultiRNNCell( - [maybe_xla(rnn_cell.LSTMCell(num_units)) for _ in range(num_layers)]) - initial_state = cell.zero_state(batch_size=batch_size, dtype=dtypes.float32) - outputs, final_state = rnn.dynamic_rnn( - cell=cell, inputs=inputs, initial_state=initial_state, time_major=True) - flat_final_state = nest.flatten(final_state) - trainable_variables = variables.trainable_variables() - outputs_grad = gradients_impl.gradients( - [outputs], trainable_variables + [inputs] + nest.flatten(initial_state)) - final_state_grad = gradients_impl.gradients( - flat_final_state, - trainable_variables + [inputs] + nest.flatten(initial_state)) - - return { - "outputs": outputs, - "final_state": flat_final_state, - "outputs_grad": outputs_grad, - "final_state_grad": final_state_grad - } - - -class CompiledWrapperTest(test.TestCase): - - def testMultiRNNCellWithLSTMCellAndXLA(self): - # TODO(b/34735319): Don't run this test if XLA is not available. - batch_size = 16 - num_units = 32 - input_depth = 12 - num_layers = 2 - max_time = 20 - - atol = 1e-5 - - random_seed.set_random_seed(1234) - with self.session(graph=ops.Graph()) as sess: - xla_ops = _create_multi_lstm_cell_ops( - batch_size=batch_size, - num_units=num_units, - input_depth=input_depth, - num_layers=num_layers, - max_time=max_time, - compiled=True) - sess.run([variables.global_variables_initializer()]) - xla_results = sess.run(xla_ops) - - random_seed.set_random_seed(1234) - with self.session(graph=ops.Graph()) as sess: - non_xla_ops = _create_multi_lstm_cell_ops( - batch_size=batch_size, - num_units=num_units, - input_depth=input_depth, - num_layers=num_layers, - max_time=max_time, - compiled=False) - sess.run([variables.global_variables_initializer()]) - non_xla_results = sess.run(non_xla_ops) - - self.assertAllClose( - non_xla_results["outputs"], xla_results["outputs"], atol=atol) - - for xla_value, non_xla_value in zip(xla_results["final_state"], - non_xla_results["final_state"]): - self.assertAllClose(xla_value, non_xla_value, atol=atol) - - for xla_g, non_xla_g in zip(xla_results["outputs_grad"], - non_xla_results["outputs_grad"]): - self.assertAllClose(xla_g, non_xla_g, atol=atol) - - for xla_g, non_xla_g in zip(xla_results["final_state_grad"], - non_xla_results["final_state_grad"]): - self.assertAllClose(xla_g, non_xla_g, atol=atol) - - def testMultiRNNCellWithStateTuple(self): - with self.cached_session() as sess: - with variable_scope.variable_scope( - "root", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros([1, 2]) - m_bad = array_ops.zeros([1, 4]) - m_good = (array_ops.zeros([1, 2]), array_ops.zeros([1, 2])) - - # Test incorrectness of state - with self.assertRaisesRegexp(ValueError, "Expected state .* a tuple"): - rnn_cell.MultiRNNCell( - [rnn_cell.GRUCell(2) for _ in range(2)], - state_is_tuple=True)(x, m_bad) - - _, ml = rnn_cell.MultiRNNCell( - [rnn_cell.GRUCell(2) for _ in range(2)], - state_is_tuple=True)(x, m_good) - - sess.run([variables.global_variables_initializer()]) - res = sess.run( - ml, { - x.name: np.array([[1., 1.]]), - m_good[0].name: np.array([[0.1, 0.1]]), - m_good[1].name: np.array([[0.1, 0.1]]) - }) - - # The numbers in results were not calculated, this is just a - # smoke test. However, these numbers should match those of - # the test testMultiRNNCell. - self.assertAllClose(res[0], [[0.175991, 0.175991]]) - self.assertAllClose(res[1], [[0.13248, 0.13248]]) - - -class BenchmarkLSTMCellXLA(test.Benchmark): - - def benchmarkDynamicRNNWithMultiLSTMCell(self): - num_layers = 3 - max_time = 50 - print("benchmarkDynamicRNNWithMultiLSTMCell") - print("\t" + "\t".join([ - "inter_th", "intra_th", "batch_size", "num_units", "input_depth", - "device", "compiled", "wall_time" - ])) - - warmup_run = True - for (threads, device, num_units, batch_size, input_depth, - compiled) in itertools.product([{ - "inter": 0, - "intra": 0 - }, { - "inter": 1, - "intra": 4 - }], ["cpu", "gpu"], [32, 512], [1, 32, 256], [32, 512], [False, True]): - if threads["inter"] != 0: - # We only care about testing inter/intra op limitations on - # CPU with small batch size, to mimic embedded devices. - if device != "cpu" or batch_size != 1: - continue - if device == "cpu" and batch_size > 32: - continue - random_seed.set_random_seed(1234) - config = config_pb2.ConfigProto( - inter_op_parallelism_threads=threads["inter"], - intra_op_parallelism_threads=threads["intra"], - allow_soft_placement=False) - with session.Session(config=config, graph=ops.Graph()) as sess: - with ops.device("/%s:0" % device): - ops_dict = _create_multi_lstm_cell_ops( - batch_size=batch_size, - num_units=num_units, - input_depth=input_depth, - num_layers=num_layers, - max_time=max_time, - compiled=compiled) - sess.run([variables.global_variables_initializer()]) - all_ops = nest.flatten(ops_dict.values()) - all_ops_group = control_flow_ops.group(*all_ops) - name_suffix = ("inter_th_%d_intra_th_%d_bs_%d_units_%d_inputdepth_%d" - "_device_%s_xla_%s" % - (threads["inter"], threads["intra"], batch_size, - num_units, input_depth, device, compiled)) - if warmup_run: - self.run_op_benchmark( - sess, all_ops_group, min_iters=30, name="ignore_warmup") - warmup_run = False - benchmark_results = self.run_op_benchmark( - sess, - all_ops_group, - min_iters=50, - name="benchmarkDynamicRNNWithMultiLSTMCell_%s" % name_suffix) - print("\t" + "\t".join([ - "%s" % x - for x in [ - threads["inter"], threads["intra"], batch_size, num_units, - input_depth, device, compiled, benchmark_results["wall_time"] - ] - ])) - - -class WeightNormLSTMCellTest(test.TestCase): - """Compared cell output with pre-calculated values.""" - - def _cell_output(self, cell): - """Calculates cell output.""" - - with self.cached_session() as sess: - init = init_ops.constant_initializer(0.5) - with variable_scope.variable_scope("root", - initializer=init): - x = array_ops.zeros([1, 2]) - c0 = array_ops.zeros([1, 2]) - h0 = array_ops.zeros([1, 2]) - - state0 = rnn_cell.LSTMStateTuple(c0, h0) - - xout, sout = cell()(x, state0) - - sess.run([variables.global_variables_initializer()]) - res = sess.run([xout, sout], { - x.name: np.array([[1., 1.]]), - c0.name: 0.1 * np.asarray([[0, 1]]), - h0.name: 0.1 * np.asarray([[2, 3]]), - }) - - actual_state_c = res[1].c - actual_state_h = res[1].h - - return actual_state_c, actual_state_h - - def testBasicCell(self): - """Tests cell w/o peepholes and w/o normalisation.""" - - def cell(): - return contrib_rnn_cell.WeightNormLSTMCell(2, - norm=False, - use_peepholes=False) - - actual_c, actual_h = self._cell_output(cell) - - expected_c = np.array([[0.65937078, 0.74983585]]) - expected_h = np.array([[0.44923624, 0.49362513]]) - - self.assertAllClose(expected_c, actual_c, 1e-5) - self.assertAllClose(expected_h, actual_h, 1e-5) - - def testNonbasicCell(self): - """Tests cell with peepholes and w/o normalisation.""" - - def cell(): - return contrib_rnn_cell.WeightNormLSTMCell(2, - norm=False, - use_peepholes=True) - - actual_c, actual_h = self._cell_output(cell) - - expected_c = np.array([[0.65937084, 0.7574988]]) - expected_h = np.array([[0.4792085, 0.53470564]]) - - self.assertAllClose(expected_c, actual_c, 1e-5) - self.assertAllClose(expected_h, actual_h, 1e-5) - - def testBasicCellWithNorm(self): - """Tests cell w/o peepholes and with normalisation.""" - - def cell(): - return contrib_rnn_cell.WeightNormLSTMCell(2, - norm=True, - use_peepholes=False) - - actual_c, actual_h = self._cell_output(cell) - - expected_c = np.array([[0.50125383, 0.58805949]]) - expected_h = np.array([[0.32770363, 0.37397948]]) - - self.assertAllClose(expected_c, actual_c, 1e-5) - self.assertAllClose(expected_h, actual_h, 1e-5) - - def testNonBasicCellWithNorm(self): - """Tests cell with peepholes and with normalisation.""" - - def cell(): - return contrib_rnn_cell.WeightNormLSTMCell(2, - norm=True, - use_peepholes=True) - - actual_c, actual_h = self._cell_output(cell) - - expected_c = np.array([[0.50125383, 0.59587258]]) - expected_h = np.array([[0.35041603, 0.40873795]]) - - self.assertAllClose(expected_c, actual_c, 1e-5) - self.assertAllClose(expected_h, actual_h, 1e-5) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/rnn/python/kernel_tests/rnn_test.py b/tensorflow/contrib/rnn/python/kernel_tests/rnn_test.py deleted file mode 100644 index 32df1db964a..00000000000 --- a/tensorflow/contrib/rnn/python/kernel_tests/rnn_test.py +++ /dev/null @@ -1,457 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for rnn module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools - -import numpy as np - -from tensorflow.contrib.rnn.python.ops import rnn as contrib_rnn -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging - - -class StackBidirectionalRNNTest(test.TestCase): - - def setUp(self): - self._seed = 23489 - np.random.seed(self._seed) - - def _createStackBidirectionalRNN(self, - use_gpu, - use_shape, - use_sequence_length, - initial_states_fw=None, - initial_states_bw=None, - scope=None): - self.layers = [2, 3] - input_size = 5 - batch_size = 2 - max_length = 8 - - initializer = init_ops.random_uniform_initializer( - -0.01, 0.01, seed=self._seed) - sequence_length = array_ops.placeholder( - dtypes.int64) if use_sequence_length else None - - self.cells_fw = [ - rnn_cell.LSTMCell( - num_units, - input_size, - initializer=initializer, - state_is_tuple=False) for num_units in self.layers - ] - self.cells_bw = [ - rnn_cell.LSTMCell( - num_units, - input_size, - initializer=initializer, - state_is_tuple=False) for num_units in self.layers - ] - - inputs = max_length * [ - array_ops.placeholder( - dtypes.float32, - shape=(batch_size, input_size) if use_shape else (None, input_size)) - ] - outputs, state_fw, state_bw = contrib_rnn.stack_bidirectional_rnn( - self.cells_fw, - self.cells_bw, - inputs, - initial_states_fw, - initial_states_bw, - dtype=dtypes.float32, - sequence_length=sequence_length, - scope=scope) - - self.assertEqual(len(outputs), len(inputs)) - for out in outputs: - self.assertAlmostEqual( - out.get_shape().as_list(), - [batch_size if use_shape else None, 2 * self.layers[-1]]) - - input_value = np.random.randn(batch_size, input_size) - outputs = array_ops.stack(outputs) - - return input_value, inputs, outputs, state_fw, state_bw, sequence_length - - def _testStackBidirectionalRNN(self, use_gpu, use_shape): - with self.session(use_gpu=use_gpu, graph=ops.Graph()) as sess: - input_value, inputs, outputs, state_fw, state_bw, sequence_length = ( - self._createStackBidirectionalRNN(use_gpu, use_shape, True)) - variables.global_variables_initializer().run() - # Run with pre-specified sequence lengths of 2, 3. - out, s_fw, s_bw = sess.run( - [outputs, state_fw, state_bw], - feed_dict={inputs[0]: input_value, - sequence_length: [2, 3]}) - - # Since the forward and backward LSTM cells were initialized with the - # same parameters, the forward and backward states of the first layer - # must be the same. - # For the next layers, since the input is a concat of forward and backward - # outputs of the previous layers the symmetry is broken and the following - # states and outputs differ. - # We cannot access the intermediate values between layers but we can - # check that the forward and backward states of the first layer match. - - self.assertAllClose(s_fw[0], s_bw[0]) - - # If outputs are not concat between layers the output of the forward - # and backward would be the same but symmetric. - # Check that it is not the case. - # Due to depth concatenation (as num_units=3 for both RNNs): - # - forward output: out[][][depth] for 0 <= depth < 3 - # - backward output: out[][][depth] for 4 <= depth < 6 - # First sequence in batch is length=2 - # Check that the time=0 forward output is not equal to time=1 backward. - self.assertNotEqual(out[0][0][0], out[1][0][3]) - self.assertNotEqual(out[0][0][1], out[1][0][4]) - self.assertNotEqual(out[0][0][2], out[1][0][5]) - # Check that the time=1 forward output is not equal to time=0 backward. - self.assertNotEqual(out[1][0][0], out[0][0][3]) - self.assertNotEqual(out[1][0][1], out[0][0][4]) - self.assertNotEqual(out[1][0][2], out[0][0][5]) - - # Second sequence in batch is length=3 - # Check that the time=0 forward output is not equal to time=2 backward. - self.assertNotEqual(out[0][1][0], out[2][1][3]) - self.assertNotEqual(out[0][1][1], out[2][1][4]) - self.assertNotEqual(out[0][1][2], out[2][1][5]) - # Check that the time=1 forward output is not equal to time=1 backward. - self.assertNotEqual(out[1][1][0], out[1][1][3]) - self.assertNotEqual(out[1][1][1], out[1][1][4]) - self.assertNotEqual(out[1][1][2], out[1][1][5]) - # Check that the time=2 forward output is not equal to time=0 backward. - self.assertNotEqual(out[2][1][0], out[0][1][3]) - self.assertNotEqual(out[2][1][1], out[0][1][4]) - self.assertNotEqual(out[2][1][2], out[0][1][5]) - - def _testStackBidirectionalRNNStates(self, use_gpu): - # Check that the states are correctly initialized. - # - Create a net and iterate for 3 states. Keep the state (state_3). - # - Reset states, and iterate for 5 steps. Last state is state_5. - # - Reset the sets to state_3 and iterate for 2 more steps, - # last state will be state_5'. - # - Check that the state_5 and state_5' (forward and backward) are the - # same for the first layer (it does not apply for the second layer since - # it has forward-backward dependencies). - with self.session(use_gpu=use_gpu, graph=ops.Graph()) as sess: - batch_size = 2 - # Create states placeholders. - initial_states_fw = [ - array_ops.placeholder( - dtypes.float32, shape=(batch_size, layer * 2)) - for layer in self.layers - ] - initial_states_bw = [ - array_ops.placeholder( - dtypes.float32, shape=(batch_size, layer * 2)) - for layer in self.layers - ] - # Create the net - input_value, inputs, outputs, state_fw, state_bw, sequence_length = ( - self._createStackBidirectionalRNN(use_gpu, True, True, - initial_states_fw, - initial_states_bw)) - variables.global_variables_initializer().run() - - # Run 3 steps. - feed_dict = {inputs[0]: input_value, sequence_length: [3, 2]} - # Initialize to empty state. - for i, layer in enumerate(self.layers): - feed_dict[initial_states_fw[i]] = np.zeros( - (batch_size, layer * 2), dtype=np.float32) - feed_dict[initial_states_bw[i]] = np.zeros( - (batch_size, layer * 2), dtype=np.float32) - _, st_3_fw, st_3_bw = sess.run([outputs, state_fw, state_bw], - feed_dict=feed_dict) - - # Reset the net and run 5 steps. - feed_dict = {inputs[0]: input_value, sequence_length: [5, 3]} - for i, layer in enumerate(self.layers): - feed_dict[initial_states_fw[i]] = np.zeros( - (batch_size, layer * 2), dtype=np.float32) - feed_dict[initial_states_bw[i]] = np.zeros( - (batch_size, layer * 2), dtype=np.float32) - _, st_5_fw, st_5_bw = sess.run([outputs, state_fw, state_bw], - feed_dict=feed_dict) - - # Reset the net to state_3 and run 2 more steps. - feed_dict = {inputs[0]: input_value, sequence_length: [2, 1]} - for i, _ in enumerate(self.layers): - feed_dict[initial_states_fw[i]] = st_3_fw[i] - feed_dict[initial_states_bw[i]] = st_3_bw[i] - out_5p, st_5p_fw, st_5p_bw = sess.run([outputs, state_fw, state_bw], - feed_dict=feed_dict) - - # Check that the 3+2 and 5 first layer states. - self.assertAllEqual(st_5_fw[0], st_5p_fw[0]) - self.assertAllEqual(st_5_bw[0], st_5p_bw[0]) - - def testStackBidirectionalRNN(self): - self._testStackBidirectionalRNN(use_gpu=False, use_shape=False) - self._testStackBidirectionalRNN(use_gpu=True, use_shape=False) - self._testStackBidirectionalRNN(use_gpu=False, use_shape=True) - self._testStackBidirectionalRNN(use_gpu=True, use_shape=True) - self._testStackBidirectionalRNNStates(use_gpu=False) - self._testStackBidirectionalRNNStates(use_gpu=True) - - def _createStackBidirectionalDynamicRNN(self, - use_gpu, - use_shape, - use_state_tuple, - initial_states_fw=None, - initial_states_bw=None, - scope=None): - self.layers = [2, 3] - input_size = 5 - batch_size = 2 - max_length = 8 - - initializer = init_ops.random_uniform_initializer( - -0.01, 0.01, seed=self._seed) - sequence_length = array_ops.placeholder(dtypes.int64) - - self.cells_fw = [ - rnn_cell.LSTMCell( - num_units, - input_size, - initializer=initializer, - state_is_tuple=False) for num_units in self.layers - ] - self.cells_bw = [ - rnn_cell.LSTMCell( - num_units, - input_size, - initializer=initializer, - state_is_tuple=False) for num_units in self.layers - ] - - inputs = max_length * [ - array_ops.placeholder( - dtypes.float32, - shape=(batch_size, input_size) if use_shape else (None, input_size)) - ] - inputs_c = array_ops.stack(inputs) - inputs_c = array_ops.transpose(inputs_c, [1, 0, 2]) - outputs, st_fw, st_bw = contrib_rnn.stack_bidirectional_dynamic_rnn( - self.cells_fw, - self.cells_bw, - inputs_c, - initial_states_fw=initial_states_fw, - initial_states_bw=initial_states_bw, - dtype=dtypes.float32, - sequence_length=sequence_length, - scope=scope) - - # Outputs has shape (batch_size, max_length, 2* layer[-1]. - output_shape = [None, max_length, 2 * self.layers[-1]] - if use_shape: - output_shape[0] = batch_size - - self.assertAllEqual(outputs.get_shape().as_list(), output_shape) - - input_value = np.random.randn(batch_size, input_size) - - return input_value, inputs, outputs, st_fw, st_bw, sequence_length - - def _testStackBidirectionalDynamicRNN(self, use_gpu, use_shape, - use_state_tuple): - with self.session(use_gpu=use_gpu, graph=ops.Graph()) as sess: - input_value, inputs, outputs, state_fw, state_bw, sequence_length = ( - self._createStackBidirectionalDynamicRNN(use_gpu, use_shape, - use_state_tuple)) - variables.global_variables_initializer().run() - # Run with pre-specified sequence length of 2, 3 - out, s_fw, s_bw = sess.run( - [outputs, state_fw, state_bw], - feed_dict={inputs[0]: input_value, - sequence_length: [2, 3]}) - - # Since the forward and backward LSTM cells were initialized with the - # same parameters, the forward and backward states of the first layer has - # to be the same. - # For the next layers, since the input is a concat of forward and backward - # outputs of the previous layers the symmetry is broken and the following - # states and outputs differ. - # We cannot access the intermediate values between layers but we can - # check that the forward and backward states of the first layer match. - - self.assertAllClose(s_fw[0], s_bw[0]) - out = np.swapaxes(out, 0, 1) - # If outputs are not concat between layers the output of the forward - # and backward would be the same but symmetric. - # Check that is not the case. - # Due to depth concatenation (as num_units=3 for both RNNs): - # - forward output: out[][][depth] for 0 <= depth < 3 - # - backward output: out[][][depth] for 4 <= depth < 6 - # First sequence in batch is length=2 - # Check that the time=0 forward output is not equal to time=1 backward. - self.assertNotEqual(out[0][0][0], out[1][0][3]) - self.assertNotEqual(out[0][0][1], out[1][0][4]) - self.assertNotEqual(out[0][0][2], out[1][0][5]) - # Check that the time=1 forward output is not equal to time=0 backward. - self.assertNotEqual(out[1][0][0], out[0][0][3]) - self.assertNotEqual(out[1][0][1], out[0][0][4]) - self.assertNotEqual(out[1][0][2], out[0][0][5]) - - # Second sequence in batch is length=3 - # Check that the time=0 forward output is not equal to time=2 backward. - self.assertNotEqual(out[0][1][0], out[2][1][3]) - self.assertNotEqual(out[0][1][1], out[2][1][4]) - self.assertNotEqual(out[0][1][2], out[2][1][5]) - # Check that the time=1 forward output is not equal to time=1 backward. - self.assertNotEqual(out[1][1][0], out[1][1][3]) - self.assertNotEqual(out[1][1][1], out[1][1][4]) - self.assertNotEqual(out[1][1][2], out[1][1][5]) - # Check that the time=2 forward output is not equal to time=0 backward. - self.assertNotEqual(out[2][1][0], out[0][1][3]) - self.assertNotEqual(out[2][1][1], out[0][1][4]) - self.assertNotEqual(out[2][1][2], out[0][1][5]) - - def _testStackBidirectionalDynamicRNNStates(self, use_gpu): - - # Check that the states are correctly initialized. - # - Create a net and iterate for 3 states. Keep the state (state_3). - # - Reset states, and iterate for 5 steps. Last state is state_5. - # - Reset the sets to state_3 and iterate for 2 more steps, - # last state will be state_5'. - # - Check that the state_5 and state_5' (forward and backward) are the - # same for the first layer (it does not apply for the second layer since - # it has forward-backward dependencies). - with self.session(use_gpu=use_gpu, graph=ops.Graph()) as sess: - batch_size = 2 - # Create states placeholders. - initial_states_fw = [ - array_ops.placeholder( - dtypes.float32, shape=(batch_size, layer * 2)) - for layer in self.layers - ] - initial_states_bw = [ - array_ops.placeholder( - dtypes.float32, shape=(batch_size, layer * 2)) - for layer in self.layers - ] - # Create the net - input_value, inputs, outputs, state_fw, state_bw, sequence_length = ( - self._createStackBidirectionalDynamicRNN( - use_gpu, - use_shape=True, - use_state_tuple=False, - initial_states_fw=initial_states_fw, - initial_states_bw=initial_states_bw)) - variables.global_variables_initializer().run() - - # Run 3 steps. - feed_dict = {inputs[0]: input_value, sequence_length: [3, 2]} - # Initialize to empty state. - for i, layer in enumerate(self.layers): - feed_dict[initial_states_fw[i]] = np.zeros( - (batch_size, layer * 2), dtype=np.float32) - feed_dict[initial_states_bw[i]] = np.zeros( - (batch_size, layer * 2), dtype=np.float32) - _, st_3_fw, st_3_bw = sess.run([outputs, state_fw, state_bw], - feed_dict=feed_dict) - - # Reset the net and run 5 steps. - feed_dict = {inputs[0]: input_value, sequence_length: [5, 3]} - for i, layer in enumerate(self.layers): - feed_dict[initial_states_fw[i]] = np.zeros( - (batch_size, layer * 2), dtype=np.float32) - feed_dict[initial_states_bw[i]] = np.zeros( - (batch_size, layer * 2), dtype=np.float32) - _, st_5_fw, st_5_bw = sess.run([outputs, state_fw, state_bw], - feed_dict=feed_dict) - - # Reset the net to state_3 and run 2 more steps. - feed_dict = {inputs[0]: input_value, sequence_length: [2, 1]} - for i, _ in enumerate(self.layers): - feed_dict[initial_states_fw[i]] = st_3_fw[i] - feed_dict[initial_states_bw[i]] = st_3_bw[i] - out_5p, st_5p_fw, st_5p_bw = sess.run([outputs, state_fw, state_bw], - feed_dict=feed_dict) - - # Check that the 3+2 and 5 first layer states. - self.assertAllEqual(st_5_fw[0], st_5p_fw[0]) - self.assertAllEqual(st_5_bw[0], st_5p_bw[0]) - - def testBidirectionalRNN(self): - # Generate 2^3 option values - # from [True, True, True] to [False, False, False] - options = itertools.product([True, False], repeat=3) - for option in options: - self._testStackBidirectionalDynamicRNN( - use_gpu=option[0], use_shape=option[1], use_state_tuple=option[2]) - # Check States. - self._testStackBidirectionalDynamicRNNStates(use_gpu=False) - self._testStackBidirectionalDynamicRNNStates(use_gpu=True) - - def _testScope(self, factory, prefix="prefix", use_outer_scope=True): - # REMARKS: factory(scope) is a function accepting a scope - # as an argument, such scope can be None, a string - # or a VariableScope instance. - with self.session(use_gpu=True, graph=ops.Graph()): - if use_outer_scope: - with variable_scope.variable_scope(prefix) as scope: - factory(scope) - else: - factory(prefix) - - # check that all the variables names starts with the proper scope. - variables.global_variables_initializer() - all_vars = variables.global_variables() - prefix = prefix or "stack_bidirectional_rnn" - scope_vars = [v for v in all_vars if v.name.startswith(prefix + "/")] - tf_logging.info("StackRNN with scope: %s (%s)" % - (prefix, "scope" if use_outer_scope else "str")) - for v in scope_vars: - tf_logging.info(v.name) - self.assertEqual(len(scope_vars), len(all_vars)) - - def testStackBidirectionalRNNScope(self): - - def factory(scope): - return self._createStackBidirectionalRNN( - use_gpu=True, use_shape=True, use_sequence_length=True, scope=scope) - - self._testScope(factory, use_outer_scope=True) - self._testScope(factory, use_outer_scope=False) - self._testScope(factory, prefix=None, use_outer_scope=False) - - def testBidirectionalDynamicRNNScope(self): - - def factory(scope): - return self._createStackBidirectionalDynamicRNN( - use_gpu=True, use_shape=True, use_state_tuple=True, scope=scope) - - self._testScope(factory, use_outer_scope=True) - self._testScope(factory, use_outer_scope=False) - self._testScope(factory, prefix=None, use_outer_scope=False) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/rnn/python/ops/core_rnn_cell.py b/tensorflow/contrib/rnn/python/ops/core_rnn_cell.py deleted file mode 100644 index 6da58747e89..00000000000 --- a/tensorflow/contrib/rnn/python/ops/core_rnn_cell.py +++ /dev/null @@ -1,384 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Module implementing RNN Cells that used to be in core. - -@@EmbeddingWrapper -@@InputProjectionWrapper -@@OutputProjectionWrapper -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import rnn_cell_impl -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util import nest - - -# pylint: disable=protected-access,invalid-name -RNNCell = rnn_cell_impl.RNNCell -_WEIGHTS_VARIABLE_NAME = rnn_cell_impl._WEIGHTS_VARIABLE_NAME -_BIAS_VARIABLE_NAME = rnn_cell_impl._BIAS_VARIABLE_NAME -# pylint: enable=protected-access,invalid-name - - -class _Linear(object): - """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable. - - Args: - args: a 2D Tensor or a list of 2D, batch, n, Tensors. - output_size: int, second dimension of weight variable. - dtype: data type for variables. - build_bias: boolean, whether to build a bias variable. - bias_initializer: starting value to initialize the bias - (default is all zeros). - kernel_initializer: starting value to initialize the weight. - - Raises: - ValueError: if inputs_shape is wrong. - """ - - def __init__(self, - args, - output_size, - build_bias, - bias_initializer=None, - kernel_initializer=None): - self._build_bias = build_bias - - if args is None or (nest.is_sequence(args) and not args): - raise ValueError("`args` must be specified") - if not nest.is_sequence(args): - args = [args] - self._is_sequence = False - else: - self._is_sequence = True - - # Calculate the total size of arguments on dimension 1. - total_arg_size = 0 - shapes = [a.get_shape() for a in args] - for shape in shapes: - if shape.ndims != 2: - raise ValueError("linear is expecting 2D arguments: %s" % shapes) - if shape.dims[1].value is None: - raise ValueError("linear expects shape[1] to be provided for shape %s, " - "but saw %s" % (shape, shape[1])) - else: - total_arg_size += shape.dims[1].value - - dtype = [a.dtype for a in args][0] - - scope = vs.get_variable_scope() - with vs.variable_scope(scope) as outer_scope: - self._weights = vs.get_variable( - _WEIGHTS_VARIABLE_NAME, [total_arg_size, output_size], - dtype=dtype, - initializer=kernel_initializer) - if build_bias: - with vs.variable_scope(outer_scope) as inner_scope: - inner_scope.set_partitioner(None) - if bias_initializer is None: - bias_initializer = init_ops.constant_initializer(0.0, dtype=dtype) - self._biases = vs.get_variable( - _BIAS_VARIABLE_NAME, [output_size], - dtype=dtype, - initializer=bias_initializer) - - def __call__(self, args): - if not self._is_sequence: - args = [args] - - if len(args) == 1: - res = math_ops.matmul(args[0], self._weights) - else: - # Explicitly creating a one for a minor performance improvement. - one = constant_op.constant(1, dtype=dtypes.int32) - res = math_ops.matmul(array_ops.concat(args, one), self._weights) - if self._build_bias: - res = nn_ops.bias_add(res, self._biases) - return res - - -# TODO(xpan): Remove this function in a follow up. -def _linear(args, - output_size, - bias, - bias_initializer=None, - kernel_initializer=None): - """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable. - - Args: - args: a 2D Tensor or a list of 2D, batch, n, Tensors. - output_size: int, second dimension of W[i]. - bias: boolean, whether to add a bias term or not. - bias_initializer: starting value to initialize the bias - (default is all zeros). - kernel_initializer: starting value to initialize the weight. - - Returns: - A 2D Tensor with shape `[batch, output_size]` equal to - sum_i(args[i] * W[i]), where W[i]s are newly created matrices. - - Raises: - ValueError: if some of the arguments has unspecified or wrong shape. - """ - if args is None or (nest.is_sequence(args) and not args): - raise ValueError("`args` must be specified") - if not nest.is_sequence(args): - args = [args] - - # Calculate the total size of arguments on dimension 1. - total_arg_size = 0 - shapes = [a.get_shape() for a in args] - for shape in shapes: - if shape.ndims != 2: - raise ValueError("linear is expecting 2D arguments: %s" % shapes) - if shape.dims[1].value is None: - raise ValueError("linear expects shape[1] to be provided for shape %s, " - "but saw %s" % (shape, shape[1])) - else: - total_arg_size += shape.dims[1].value - - dtype = [a.dtype for a in args][0] - - # Now the computation. - scope = vs.get_variable_scope() - with vs.variable_scope(scope) as outer_scope: - weights = vs.get_variable( - _WEIGHTS_VARIABLE_NAME, [total_arg_size, output_size], - dtype=dtype, - initializer=kernel_initializer) - if len(args) == 1: - res = math_ops.matmul(args[0], weights) - else: - res = math_ops.matmul(array_ops.concat(args, 1), weights) - if not bias: - return res - with vs.variable_scope(outer_scope) as inner_scope: - inner_scope.set_partitioner(None) - if bias_initializer is None: - bias_initializer = init_ops.constant_initializer(0.0, dtype=dtype) - biases = vs.get_variable( - _BIAS_VARIABLE_NAME, [output_size], - dtype=dtype, - initializer=bias_initializer) - return nn_ops.bias_add(res, biases) - - -class EmbeddingWrapper(RNNCell): - """Operator adding input embedding to the given cell. - - Note: in many cases it may be more efficient to not use this wrapper, - but instead concatenate the whole sequence of your inputs in time, - do the embedding on this batch-concatenated sequence, then split it and - feed into your RNN. - """ - - def __init__(self, - cell, - embedding_classes, - embedding_size, - initializer=None, - reuse=None): - """Create a cell with an added input embedding. - - Args: - cell: an RNNCell, an embedding will be put before its inputs. - embedding_classes: integer, how many symbols will be embedded. - embedding_size: integer, the size of the vectors we embed into. - initializer: an initializer to use when creating the embedding; - if None, the initializer from variable scope or a default one is used. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - - Raises: - TypeError: if cell is not an RNNCell. - ValueError: if embedding_classes is not positive. - """ - super(EmbeddingWrapper, self).__init__(_reuse=reuse) - rnn_cell_impl.assert_like_rnncell("cell", cell) - if embedding_classes <= 0 or embedding_size <= 0: - raise ValueError("Both embedding_classes and embedding_size must be > 0: " - "%d, %d." % (embedding_classes, embedding_size)) - self._cell = cell - self._embedding_classes = embedding_classes - self._embedding_size = embedding_size - self._initializer = initializer - - @property - def state_size(self): - return self._cell.state_size - - @property - def output_size(self): - return self._cell.output_size - - def zero_state(self, batch_size, dtype): - with ops.name_scope(type(self).__name__ + "ZeroState", values=[batch_size]): - return self._cell.zero_state(batch_size, dtype) - - def call(self, inputs, state): - """Run the cell on embedded inputs.""" - with ops.device("/cpu:0"): - if self._initializer: - initializer = self._initializer - elif vs.get_variable_scope().initializer: - initializer = vs.get_variable_scope().initializer - else: - # Default initializer for embeddings should have variance=1. - sqrt3 = math.sqrt(3) # Uniform(-sqrt(3), sqrt(3)) has variance=1. - initializer = init_ops.random_uniform_initializer(-sqrt3, sqrt3) - - if isinstance(state, tuple): - data_type = state[0].dtype - else: - data_type = state.dtype - - embedding = vs.get_variable( - "embedding", [self._embedding_classes, self._embedding_size], - initializer=initializer, - dtype=data_type) - embedded = embedding_ops.embedding_lookup(embedding, - array_ops.reshape(inputs, [-1])) - - return self._cell(embedded, state) - - -class InputProjectionWrapper(RNNCell): - """Operator adding an input projection to the given cell. - - Note: in many cases it may be more efficient to not use this wrapper, - but instead concatenate the whole sequence of your inputs in time, - do the projection on this batch-concatenated sequence, then split it. - """ - - def __init__(self, - cell, - num_proj, - activation=None, - input_size=None, - reuse=None): - """Create a cell with input projection. - - Args: - cell: an RNNCell, a projection of inputs is added before it. - num_proj: Python integer. The dimension to project to. - activation: (optional) an optional activation function. - input_size: Deprecated and unused. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - - Raises: - TypeError: if cell is not an RNNCell. - """ - super(InputProjectionWrapper, self).__init__(_reuse=reuse) - if input_size is not None: - logging.warn("%s: The input_size parameter is deprecated.", self) - rnn_cell_impl.assert_like_rnncell("cell", cell) - self._cell = cell - self._num_proj = num_proj - self._activation = activation - self._linear = None - - @property - def state_size(self): - return self._cell.state_size - - @property - def output_size(self): - return self._cell.output_size - - def zero_state(self, batch_size, dtype): - with ops.name_scope(type(self).__name__ + "ZeroState", values=[batch_size]): - return self._cell.zero_state(batch_size, dtype) - - def call(self, inputs, state): - """Run the input projection and then the cell.""" - # Default scope: "InputProjectionWrapper" - if self._linear is None: - self._linear = _Linear(inputs, self._num_proj, True) - projected = self._linear(inputs) - if self._activation: - projected = self._activation(projected) - return self._cell(projected, state) - - -class OutputProjectionWrapper(RNNCell): - """Operator adding an output projection to the given cell. - - Note: in many cases it may be more efficient to not use this wrapper, - but instead concatenate the whole sequence of your outputs in time, - do the projection on this batch-concatenated sequence, then split it - if needed or directly feed into a softmax. - """ - - def __init__(self, cell, output_size, activation=None, reuse=None): - """Create a cell with output projection. - - Args: - cell: an RNNCell, a projection to output_size is added to it. - output_size: integer, the size of the output after projection. - activation: (optional) an optional activation function. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - - Raises: - TypeError: if cell is not an RNNCell. - ValueError: if output_size is not positive. - """ - super(OutputProjectionWrapper, self).__init__(_reuse=reuse) - rnn_cell_impl.assert_like_rnncell("cell", cell) - if output_size < 1: - raise ValueError("Parameter output_size must be > 0: %d." % output_size) - self._cell = cell - self._output_size = output_size - self._activation = activation - self._linear = None - - @property - def state_size(self): - return self._cell.state_size - - @property - def output_size(self): - return self._output_size - - def zero_state(self, batch_size, dtype): - with ops.name_scope(type(self).__name__ + "ZeroState", values=[batch_size]): - return self._cell.zero_state(batch_size, dtype) - - def call(self, inputs, state): - """Run the cell and output projection on inputs, starting from state.""" - output, res_state = self._cell(inputs, state) - if self._linear is None: - self._linear = _Linear(output, self._output_size, True) - projected = self._linear(output) - if self._activation: - projected = self._activation(projected) - return projected, res_state diff --git a/tensorflow/contrib/rnn/python/ops/fused_rnn_cell.py b/tensorflow/contrib/rnn/python/ops/fused_rnn_cell.py deleted file mode 100644 index 74b422a3845..00000000000 --- a/tensorflow/contrib/rnn/python/ops/fused_rnn_cell.py +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Module for constructing fused RNN cells.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import rnn - - -@six.add_metaclass(abc.ABCMeta) -class FusedRNNCell(object): - """Abstract object representing a fused RNN cell. - - A fused RNN cell represents the entire RNN expanded over the time - dimension. In effect, this represents an entire recurrent network. - - Unlike RNN cells which are subclasses of `rnn_cell.RNNCell`, a `FusedRNNCell` - operates on the entire time sequence at once, by putting the loop over time - inside the cell. This usually leads to much more efficient, but more complex - and less flexible implementations. - - Every `FusedRNNCell` must implement `__call__` with the following signature. - """ - - @abc.abstractmethod - def __call__(self, - inputs, - initial_state=None, - dtype=None, - sequence_length=None, - scope=None): - """Run this fused RNN on inputs, starting from the given state. - - Args: - inputs: `3-D` tensor with shape `[time_len x batch_size x input_size]` - or a list of `time_len` tensors of shape `[batch_size x input_size]`. - initial_state: either a tensor with shape `[batch_size x state_size]` - or a tuple with shapes `[batch_size x s] for s in state_size`, if the - cell takes tuples. If this is not provided, the cell is expected to - create a zero initial state of type `dtype`. - dtype: The data type for the initial state and expected output. Required - if `initial_state` is not provided or RNN state has a heterogeneous - dtype. - sequence_length: Specifies the length of each sequence in inputs. An - `int32` or `int64` vector (tensor) size `[batch_size]`, values in `[0, - time_len)`. - Defaults to `time_len` for each element. - scope: `VariableScope` or `string` for the created subgraph; defaults to - class name. - - Returns: - A pair containing: - - - Output: A `3-D` tensor of shape `[time_len x batch_size x output_size]` - or a list of `time_len` tensors of shape `[batch_size x output_size]`, - to match the type of the `inputs`. - - Final state: Either a single `2-D` tensor, or a tuple of tensors - matching the arity and shapes of `initial_state`. - """ - pass - - -class FusedRNNCellAdaptor(FusedRNNCell): - """This is an adaptor for RNNCell classes to be used with `FusedRNNCell`.""" - - def __init__(self, cell, use_dynamic_rnn=False): - """Initialize the adaptor. - - Args: - cell: an instance of a subclass of a `rnn_cell.RNNCell`. - use_dynamic_rnn: whether to use dynamic (or static) RNN. - """ - self._cell = cell - self._use_dynamic_rnn = use_dynamic_rnn - - def __call__(self, - inputs, - initial_state=None, - dtype=None, - sequence_length=None, - scope=None): - is_list = isinstance(inputs, list) - if self._use_dynamic_rnn: - if is_list: - inputs = array_ops.stack(inputs) - outputs, state = rnn.dynamic_rnn( - self._cell, - inputs, - sequence_length=sequence_length, - initial_state=initial_state, - dtype=dtype, - time_major=True, - scope=scope) - if is_list: - # Convert outputs back to list - outputs = array_ops.unstack(outputs) - else: # non-dynamic rnn - if not is_list: - inputs = array_ops.unstack(inputs) - outputs, state = rnn.static_rnn( - self._cell, - inputs, - initial_state=initial_state, - dtype=dtype, - sequence_length=sequence_length, - scope=scope) - if not is_list: - # Convert outputs back to tensor - outputs = array_ops.stack(outputs) - - return outputs, state - - -class TimeReversedFusedRNN(FusedRNNCell): - """This is an adaptor to time-reverse a FusedRNNCell. - - For example, - - ```python - cell = tf.compat.v1.nn.rnn_cell.BasicRNNCell(10) - fw_lstm = tf.contrib.rnn.FusedRNNCellAdaptor(cell, use_dynamic_rnn=True) - bw_lstm = tf.contrib.rnn.TimeReversedFusedRNN(fw_lstm) - fw_out, fw_state = fw_lstm(inputs) - bw_out, bw_state = bw_lstm(inputs) - ``` - """ - - def __init__(self, cell): - self._cell = cell - - def _reverse(self, t, lengths): - """Time reverse the provided tensor or list of tensors. - - Assumes the top dimension is the time dimension. - - Args: - t: 3D tensor or list of 2D tensors to be reversed - lengths: 1D tensor of lengths, or `None` - - Returns: - A reversed tensor or list of tensors - """ - if isinstance(t, list): - return list(reversed(t)) - else: - if lengths is None: - return array_ops.reverse_v2(t, [0]) - else: - return array_ops.reverse_sequence(t, lengths, 0, 1) - - def __call__(self, - inputs, - initial_state=None, - dtype=None, - sequence_length=None, - scope=None): - inputs = self._reverse(inputs, sequence_length) - outputs, state = self._cell( - inputs, - initial_state=initial_state, - dtype=dtype, - sequence_length=sequence_length, - scope=scope) - outputs = self._reverse(outputs, sequence_length) - return outputs, state diff --git a/tensorflow/contrib/rnn/python/ops/gru_ops.py b/tensorflow/contrib/rnn/python/ops/gru_ops.py deleted file mode 100644 index db986f58f09..00000000000 --- a/tensorflow/contrib/rnn/python/ops/gru_ops.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python wrapper for the Block GRU Op.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.keras.engine import input_spec -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_rnn_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import rnn_cell_impl -from tensorflow.python.util.deprecation import deprecated_args - - -LayerRNNCell = rnn_cell_impl.LayerRNNCell # pylint: disable=invalid-name - - -@ops.RegisterGradient("GRUBlockCell") -def _GRUBlockCellGrad(op, *grad): - r"""Gradient for GRUBlockCell. - - Args: - op: Op for which the gradient is defined. - *grad: Gradients of the optimization function wrt output - for the Op. - - Returns: - d_x: Gradients wrt to x - d_h: Gradients wrt to h - d_w_ru: Gradients wrt to w_ru - d_w_c: Gradients wrt to w_c - d_b_ru: Gradients wrt to b_ru - d_b_c: Gradients wrt to b_c - - Mathematics behind the Gradients below: - ``` - d_c_bar = d_h \circ (1-u) \circ (1-c \circ c) - d_u_bar = d_h \circ (h-c) \circ u \circ (1-u) - - d_r_bar_u_bar = [d_r_bar d_u_bar] - - [d_x_component_1 d_h_prev_component_1] = d_r_bar_u_bar * w_ru^T - - [d_x_component_2 d_h_prevr] = d_c_bar * w_c^T - - d_x = d_x_component_1 + d_x_component_2 - - d_h_prev = d_h_prev_component_1 + d_h_prevr \circ r + u - ``` - Below calculation is performed in the python wrapper for the Gradients - (not in the gradient kernel.) - ``` - d_w_ru = x_h_prevr^T * d_c_bar - - d_w_c = x_h_prev^T * d_r_bar_u_bar - - d_b_ru = sum of d_r_bar_u_bar along axis = 0 - - d_b_c = sum of d_c_bar along axis = 0 - ``` - """ - x, h_prev, w_ru, w_c, b_ru, b_c = op.inputs - r, u, c, _ = op.outputs - _, _, _, d_h = grad - - d_x, d_h_prev, d_c_bar, d_r_bar_u_bar = gen_rnn_ops.gru_block_cell_grad( - x, h_prev, w_ru, w_c, b_ru, b_c, r, u, c, d_h) - - x_h_prev = array_ops.concat([x, h_prev], 1) - d_w_ru = math_ops.matmul(x_h_prev, d_r_bar_u_bar, transpose_a=True) - d_b_ru = nn_ops.bias_add_grad(d_r_bar_u_bar) - - x_h_prevr = array_ops.concat([x, h_prev * r], 1) - d_w_c = math_ops.matmul(x_h_prevr, d_c_bar, transpose_a=True) - d_b_c = nn_ops.bias_add_grad(d_c_bar) - - return d_x, d_h_prev, d_w_ru, d_w_c, d_b_ru, d_b_c - - -class GRUBlockCell(LayerRNNCell): - r"""Block GRU cell implementation. - - Deprecated: use GRUBlockCellV2 instead. - - The implementation is based on: http://arxiv.org/abs/1406.1078 - Computes the GRU cell forward propagation for 1 time step. - - This kernel op implements the following mathematical equations: - - Biases are initialized with: - - * `b_ru` - constant_initializer(1.0) - * `b_c` - constant_initializer(0.0) - - ``` - x_h_prev = [x, h_prev] - - [r_bar u_bar] = x_h_prev * w_ru + b_ru - - r = sigmoid(r_bar) - u = sigmoid(u_bar) - - h_prevr = h_prev \circ r - - x_h_prevr = [x h_prevr] - - c_bar = x_h_prevr * w_c + b_c - c = tanh(c_bar) - - h = (1-u) \circ c + u \circ h_prev - ``` - - """ - - @deprecated_args(None, "cell_size is deprecated, use num_units instead", - "cell_size") - def __init__(self, - num_units=None, - cell_size=None, - reuse=None, - name="gru_cell"): - """Initialize the Block GRU cell. - - Args: - num_units: int, The number of units in the GRU cell. - cell_size: int, The old (deprecated) name for `num_units`. - reuse: (optional) boolean describing whether to reuse variables in an - existing scope. If not `True`, and the existing scope already has the - given variables, an error is raised. - name: String, the name of the layer. Layers with the same name will - share weights, but to avoid mistakes we require reuse=True in such - cases. By default this is "lstm_cell", for variable-name compatibility - with `tf.compat.v1.nn.rnn_cell.GRUCell`. - - Raises: - ValueError: if both cell_size and num_units are not None; - or both are None. - """ - super(GRUBlockCell, self).__init__(_reuse=reuse, name=name) - if (cell_size is None) == (num_units is None): - raise ValueError( - "Exactly one of num_units or cell_size must be provided.") - if num_units is None: - num_units = cell_size - self._cell_size = num_units - # Inputs must be 2-dimensional. - self.input_spec = input_spec.InputSpec(ndim=2) - - @property - def state_size(self): - return self._cell_size - - @property - def output_size(self): - return self._cell_size - - def build(self, input_shape): - # Check if the input size exist. - input_size = tensor_shape.dimension_value(input_shape[1]) - if input_size is None: - raise ValueError("Expecting input_size to be set.") - - self._gate_kernel = self.add_variable( - "w_ru", [input_size + self._cell_size, self._cell_size * 2]) - self._gate_bias = self.add_variable( - "b_ru", [self._cell_size * 2], - initializer=init_ops.constant_initializer(1.0)) - self._candidate_kernel = self.add_variable( - "w_c", [input_size + self._cell_size, self._cell_size]) - self._candidate_bias = self.add_variable( - "b_c", [self._cell_size], - initializer=init_ops.constant_initializer(0.0)) - - self.built = True - - def call(self, inputs, h_prev): - """GRU cell.""" - # Check cell_size == state_size from h_prev. - cell_size = h_prev.get_shape().with_rank(2)[1] - if cell_size != self._cell_size: - raise ValueError("Shape of h_prev[1] incorrect: cell_size %i vs %s" % - (self._cell_size, cell_size)) - - _gru_block_cell = gen_rnn_ops.gru_block_cell # pylint: disable=invalid-name - _, _, _, new_h = _gru_block_cell( - x=inputs, - h_prev=h_prev, - w_ru=self._gate_kernel, - w_c=self._candidate_kernel, - b_ru=self._gate_bias, - b_c=self._candidate_bias) - - return new_h, new_h - - -class GRUBlockCellV2(GRUBlockCell): - """Temporary GRUBlockCell impl with a different variable naming scheme. - - Only differs from GRUBlockCell by variable names. - """ - - def build(self, input_shape): - """GRU cell.""" - input_size = tensor_shape.dimension_value(input_shape[1]) - if input_size is None: - raise ValueError("Expecting input_size to be set.") - - self._gate_kernel = self.add_variable( - "gates/kernel", [input_size + self._cell_size, self._cell_size * 2]) - self._gate_bias = self.add_variable( - "gates/bias", [self._cell_size * 2], - initializer=init_ops.constant_initializer(1.0)) - self._candidate_kernel = self.add_variable( - "candidate/kernel", [input_size + self._cell_size, self._cell_size]) - self._candidate_bias = self.add_variable( - "candidate/bias", [self._cell_size], - initializer=init_ops.constant_initializer(0.0)) - - self.built = True diff --git a/tensorflow/contrib/rnn/python/ops/lstm_ops.py b/tensorflow/contrib/rnn/python/ops/lstm_ops.py deleted file mode 100644 index 78ea6374220..00000000000 --- a/tensorflow/contrib/rnn/python/ops/lstm_ops.py +++ /dev/null @@ -1,667 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""LSTM Block Cell ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.keras.engine import input_spec -from tensorflow.python.layers import base as base_layer -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_rnn_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import rnn_cell_impl - -LayerRNNCell = rnn_cell_impl.LayerRNNCell # pylint: disable=invalid-name - - -# pylint: disable=invalid-name -def _lstm_block_cell(x, - cs_prev, - h_prev, - w, - b, - wci=None, - wcf=None, - wco=None, - forget_bias=None, - cell_clip=None, - use_peephole=None, - name=None): - r"""Computes the LSTM cell forward propagation for 1 time step. - - This implementation uses 1 weight matrix and 1 bias vector, and there's an - optional peephole connection. - - This kernel op implements the following mathematical equations: - - ```python - xh = [x, h_prev] - [i, ci, f, o] = xh * w + b - f = f + forget_bias - - if not use_peephole: - wci = wcf = wco = 0 - - i = sigmoid(cs_prev * wci + i) - f = sigmoid(cs_prev * wcf + f) - ci = tanh(ci) - - cs = ci .* i + cs_prev .* f - cs = clip(cs, cell_clip) - - o = sigmoid(cs * wco + o) - co = tanh(cs) - h = co .* o - ``` - - Args: - x: A `Tensor`. Must be one of the following types: `float32`. - The input to the LSTM cell, shape (batch_size, num_inputs). - cs_prev: A `Tensor`. Must have the same type as `x`. - Value of the cell state at previous time step. - h_prev: A `Tensor`. Must have the same type as `x`. - Output of the previous cell at previous time step. - w: A `Tensor`. Must have the same type as `x`. The weight matrix. - b: A `Tensor`. Must have the same type as `x`. The bias vector. - wci: A `Tensor`. Must have the same type as `x`. - The weight matrix for input gate peephole connection. - wcf: A `Tensor`. Must have the same type as `x`. - The weight matrix for forget gate peephole connection. - wco: A `Tensor`. Must have the same type as `x`. - The weight matrix for output gate peephole connection. - forget_bias: An optional `float`. Defaults to `1`. The forget gate bias. - cell_clip: An optional `float`. Defaults to `-1` (no clipping). - Value to clip the 'cs' value to. Disable by setting to negative value. - use_peephole: An optional `bool`. Defaults to `False`. - Whether to use peephole weights. - name: A name for the operation (optional). - - Returns: - A tuple of `Tensor` objects (i, cs, f, o, ci, co, h). - i: A `Tensor`. Has the same type as `x`. The input gate. - cs: A `Tensor`. Has the same type as `x`. The cell state before the tanh. - f: A `Tensor`. Has the same type as `x`. The forget gate. - o: A `Tensor`. Has the same type as `x`. The output gate. - ci: A `Tensor`. Has the same type as `x`. The cell input. - co: A `Tensor`. Has the same type as `x`. The cell after the tanh. - h: A `Tensor`. Has the same type as `x`. The output h vector. - - Raises: - ValueError: If cell_size is None. - """ - if wci is None: - cell_size = cs_prev.get_shape().with_rank(2).dims[1].value - if cell_size is None: - raise ValueError("cell_size from `cs_prev` should not be None.") - wci = array_ops.constant(0, dtype=dtypes.float32, shape=[cell_size]) - wcf = wci - wco = wci - - # pylint: disable=protected-access - return gen_rnn_ops.lstm_block_cell( - x=x, - cs_prev=cs_prev, - h_prev=h_prev, - w=w, - wci=wci, - wcf=wcf, - wco=wco, - b=b, - forget_bias=forget_bias, - cell_clip=cell_clip if cell_clip is not None else -1, - use_peephole=use_peephole, - name=name) - # pylint: enable=protected-access - - -def _block_lstm(seq_len_max, - x, - w, - b, - cs_prev=None, - h_prev=None, - wci=None, - wcf=None, - wco=None, - forget_bias=None, - cell_clip=None, - use_peephole=None, - name=None): - r"""TODO(williamchan): add doc. - - Args: - seq_len_max: A `Tensor` of type `int64`. - x: A list of at least 1 `Tensor` objects of the same type. - w: A `Tensor`. Must have the same type as `x`. - b: A `Tensor`. Must have the same type as `x`. - cs_prev: A `Tensor`. Must have the same type as `x`. - h_prev: A `Tensor`. Must have the same type as `x`. - wci: A `Tensor`. Must have the same type as `x`. - wcf: A `Tensor`. Must have the same type as `x`. - wco: A `Tensor`. Must have the same type as `x`. - forget_bias: An optional `float`. Defaults to `1`. - cell_clip: An optional `float`. Defaults to `-1` (no clipping). - use_peephole: An optional `bool`. Defaults to `False`. - name: A name for the operation (optional). - - Returns: - A tuple of `Tensor` objects (i, cs, f, o, ci, co, h). - i: A list with the same number of `Tensor` objects as `x` of `Tensor` - objects of the same type as x. - cs: A list with the same number of `Tensor` objects as `x` of `Tensor` - objects of the same type as x. - f: A list with the same number of `Tensor` objects as `x` of `Tensor` - objects of the same type as x. - o: A list with the same number of `Tensor` objects as `x` of `Tensor` - objects of the same type as x. - ci: A list with the same number of `Tensor` objects as `x` of `Tensor` - objects of the same type as x. - co: A list with the same number of `Tensor` objects as `x` of `Tensor` - objects of the same type as x. - h: A list with the same number of `Tensor` objects as `x` of `Tensor` - objects of the same type as x. - - Raises: - ValueError: If `b` does not have a valid shape. - """ - dtype = x[0].dtype - batch_size = x[0].get_shape().with_rank(2).dims[0].value - cell_size4 = b.get_shape().with_rank(1).dims[0].value - if cell_size4 is None: - raise ValueError("`b` shape must not be None.") - cell_size = cell_size4 / 4 - zero_state = None - if cs_prev is None or h_prev is None: - zero_state = array_ops.constant( - 0, dtype=dtype, shape=[batch_size, cell_size]) - if cs_prev is None: - cs_prev = zero_state - if h_prev is None: - h_prev = zero_state - if wci is None: - wci = array_ops.constant(0, dtype=dtype, shape=[cell_size]) - wcf = wci - wco = wci - - # pylint: disable=protected-access - i, cs, f, o, ci, co, h = gen_rnn_ops.block_lstm( - seq_len_max=seq_len_max, - x=array_ops.stack(x), - cs_prev=cs_prev, - h_prev=h_prev, - w=w, - wci=wci, - wcf=wcf, - wco=wco, - b=b, - forget_bias=forget_bias, - cell_clip=cell_clip if cell_clip is not None else -1, - name=name, - use_peephole=use_peephole) - - return array_ops.unstack(i), array_ops.unstack(cs), array_ops.unstack( - f), array_ops.unstack(o), array_ops.unstack(ci), array_ops.unstack( - co), array_ops.unstack(h) - # pylint: enable=protected-access - # pylint: enable=invalid-name - - -@ops.RegisterGradient("LSTMBlockCell") -def _LSTMBlockCellGrad(op, *grad): - """Gradient for LSTMBlockCell.""" - (x, cs_prev, h_prev, w, wci, wcf, wco, b) = op.inputs - (i, cs, f, o, ci, co, _) = op.outputs - (_, cs_grad, _, _, _, _, h_grad) = grad - - batch_size = x.get_shape().with_rank(2).dims[0].value - if batch_size is None: - batch_size = -1 - input_size = x.get_shape().with_rank(2).dims[1].value - if input_size is None: - raise ValueError("input_size from `x` should not be None.") - cell_size = cs_prev.get_shape().with_rank(2).dims[1].value - if cell_size is None: - raise ValueError("cell_size from `cs_prev` should not be None.") - - (cs_prev_grad, dgates, wci_grad, wcf_grad, - wco_grad) = gen_rnn_ops.lstm_block_cell_grad( - x=x, - cs_prev=cs_prev, - h_prev=h_prev, - w=w, - wci=wci, - wcf=wcf, - wco=wco, - b=b, - i=i, - cs=cs, - f=f, - o=o, - ci=ci, - co=co, - cs_grad=cs_grad, - h_grad=h_grad, - use_peephole=op.get_attr("use_peephole")) - - # Backprop from dgates to xh. - xh_grad = math_ops.matmul(dgates, w, transpose_b=True) - - x_grad = array_ops.slice(xh_grad, (0, 0), (batch_size, input_size)) - x_grad.get_shape().merge_with(x.get_shape()) - - h_prev_grad = array_ops.slice(xh_grad, (0, input_size), - (batch_size, cell_size)) - h_prev_grad.get_shape().merge_with(h_prev.get_shape()) - - # Backprop from dgates to w. - xh = array_ops.concat([x, h_prev], 1) - w_grad = math_ops.matmul(xh, dgates, transpose_a=True) - w_grad.get_shape().merge_with(w.get_shape()) - - # Backprop from dgates to b. - b_grad = nn_ops.bias_add_grad(dgates) - b_grad.get_shape().merge_with(b.get_shape()) - - return (x_grad, cs_prev_grad, h_prev_grad, w_grad, wci_grad, wcf_grad, - wco_grad, b_grad) - - -class LSTMBlockCell(LayerRNNCell): - """Basic LSTM recurrent network cell. - - The implementation is based on: http://arxiv.org/abs/1409.2329. - - We add `forget_bias` (default: 1) to the biases of the forget gate in order to - reduce the scale of forgetting in the beginning of the training. - - Unlike `rnn_cell_impl.LSTMCell`, this is a monolithic op and should be much - faster. The weight and bias matrices should be compatible as long as the - variable scope matches. - """ - - def __init__(self, - num_units, - forget_bias=1.0, - cell_clip=None, - use_peephole=False, - dtype=None, - reuse=None, - name="lstm_cell"): - """Initialize the basic LSTM cell. - - Args: - num_units: int, The number of units in the LSTM cell. - forget_bias: float, The bias added to forget gates (see above). - cell_clip: An optional `float`. Defaults to `-1` (no clipping). - use_peephole: Whether to use peephole connections or not. - dtype: the variable dtype of this layer. Default to tf.float32. - reuse: (optional) boolean describing whether to reuse variables in an - existing scope. If not `True`, and the existing scope already has the - given variables, an error is raised. - name: String, the name of the layer. Layers with the same name will - share weights, but to avoid mistakes we require reuse=True in such - cases. By default this is "lstm_cell", for variable-name compatibility - with `tf.compat.v1.nn.rnn_cell.LSTMCell`. - - When restoring from CudnnLSTM-trained checkpoints, must use - CudnnCompatibleLSTMBlockCell instead. - """ - super(LSTMBlockCell, self).__init__(_reuse=reuse, dtype=dtype, name=name) - self._num_units = num_units - self._forget_bias = forget_bias - self._use_peephole = use_peephole - self._cell_clip = cell_clip if cell_clip is not None else -1 - self._names = { - "W": "kernel", - "b": "bias", - "wci": "w_i_diag", - "wcf": "w_f_diag", - "wco": "w_o_diag", - "scope": "lstm_cell" - } - # Inputs must be 2-dimensional. - self.input_spec = input_spec.InputSpec(ndim=2) - - @property - def state_size(self): - return rnn_cell_impl.LSTMStateTuple(self._num_units, self._num_units) - - @property - def output_size(self): - return self._num_units - - def build(self, inputs_shape): - if not inputs_shape.dims[1].value: - raise ValueError( - "Expecting inputs_shape[1] to be set: %s" % str(inputs_shape)) - input_size = inputs_shape.dims[1].value - self._kernel = self.add_variable( - self._names["W"], [input_size + self._num_units, self._num_units * 4]) - self._bias = self.add_variable( - self._names["b"], [self._num_units * 4], - initializer=init_ops.constant_initializer(0.0)) - if self._use_peephole: - self._w_i_diag = self.add_variable(self._names["wci"], [self._num_units]) - self._w_f_diag = self.add_variable(self._names["wcf"], [self._num_units]) - self._w_o_diag = self.add_variable(self._names["wco"], [self._num_units]) - - self.built = True - - def call(self, inputs, state): - """Long short-term memory cell (LSTM).""" - if len(state) != 2: - raise ValueError("Expecting state to be a tuple with length 2.") - - if self._use_peephole: - wci = self._w_i_diag - wcf = self._w_f_diag - wco = self._w_o_diag - else: - wci = wcf = wco = array_ops.zeros([self._num_units], dtype=self.dtype) - - (cs_prev, h_prev) = state - (_, cs, _, _, _, _, h) = _lstm_block_cell( - inputs, - cs_prev, - h_prev, - self._kernel, - self._bias, - wci=wci, - wcf=wcf, - wco=wco, - forget_bias=self._forget_bias, - cell_clip=self._cell_clip, - use_peephole=self._use_peephole) - - new_state = rnn_cell_impl.LSTMStateTuple(cs, h) - return h, new_state - - -@six.add_metaclass(abc.ABCMeta) -class LSTMBlockWrapper(base_layer.Layer): - """This is a helper class that provides housekeeping for LSTM cells. - - This may be useful for alternative LSTM and similar type of cells. - The subclasses must implement `_call_cell` method and `num_units` property. - """ - - @abc.abstractproperty - def num_units(self): - """Number of units in this cell (output dimension).""" - pass - - @abc.abstractmethod - def _call_cell(self, inputs, initial_cell_state, initial_output, dtype, - sequence_length): - """Run this LSTM on inputs, starting from the given state. - - This method must be implemented by subclasses and does the actual work - of calling the cell. - - Args: - inputs: `3-D` tensor with shape `[time_len, batch_size, input_size]` - initial_cell_state: initial value for cell state, shape `[batch_size, - self._num_units]` - initial_output: initial value of cell output, shape `[batch_size, - self._num_units]` - dtype: The data type for the initial state and expected output. - sequence_length: Specifies the length of each sequence in inputs. An int32 - or int64 vector (tensor) size [batch_size], values in [0, time_len) or - None. - - Returns: - A pair containing: - - - State: A `3-D` tensor of shape `[time_len, batch_size, output_size]` - - Output: A `3-D` tensor of shape `[time_len, batch_size, output_size]` - """ - pass - - def call(self, inputs, initial_state=None, dtype=None, sequence_length=None): - """Run this LSTM on inputs, starting from the given state. - - Args: - inputs: `3-D` tensor with shape `[time_len, batch_size, input_size]`. - initial_state: a tuple `(initial_cell_state, initial_output)` with tensors - of shape `[batch_size, self._num_units]`. If this is not provided, the - cell is expected to create a zero initial state of type `dtype`. - dtype: The data type for the initial state and expected output. Required - if `initial_state` is not provided or RNN state has a heterogeneous - dtype. - sequence_length: Specifies the length of each sequence in inputs. An - `int32` or `int64` vector (tensor) size `[batch_size]`, values in `[0, - time_len).` - Defaults to `time_len` for each element. - - Returns: - A pair containing: - - - Output: A `3-D` tensor of shape `[time_len, batch_size, output_size]` - or a list of time_len tensors of shape `[batch_size, output_size]`, - to match the type of the `inputs`. - - Final state: a tuple `(cell_state, output)` matching `initial_state`. - - Raises: - ValueError: in case of shape mismatches - """ - is_list = isinstance(inputs, list) - if is_list: - inputs = array_ops.stack(inputs) - inputs_shape = inputs.get_shape().with_rank(3) - if not inputs_shape[2]: - raise ValueError("Expecting inputs_shape[2] to be set: %s" % inputs_shape) - batch_size = inputs_shape.dims[1].value - if batch_size is None: - batch_size = array_ops.shape(inputs)[1] - time_len = inputs_shape.dims[0].value - if time_len is None: - time_len = array_ops.shape(inputs)[0] - - # Provide default values for initial_state and dtype - if initial_state is None: - if dtype is None: - raise ValueError("Either initial_state or dtype needs to be specified") - z = array_ops.zeros( - array_ops.stack([batch_size, self.num_units]), dtype=dtype) - initial_state = z, z - else: - if len(initial_state) != 2: - raise ValueError( - "Expecting initial_state to be a tuple with length 2 or None") - if dtype is None: - dtype = initial_state[0].dtype - - # create the actual cell - if sequence_length is not None: - sequence_length = ops.convert_to_tensor(sequence_length) - initial_cell_state, initial_output = initial_state # pylint: disable=unpacking-non-sequence - cell_states, outputs = self._call_cell( - inputs, initial_cell_state, initial_output, dtype, sequence_length) - - if sequence_length is not None: - # Mask out the part beyond sequence_length - mask = array_ops.transpose( - array_ops.sequence_mask(sequence_length, time_len, dtype=dtype), - [1, 0]) - mask = array_ops.tile( - array_ops.expand_dims(mask, [-1]), [1, 1, self.num_units]) - outputs *= mask - # Prepend initial states to cell_states and outputs for indexing to work - # correctly,since we want to access the last valid state at - # sequence_length - 1, which can even be -1, corresponding to the - # initial state. - mod_cell_states = array_ops.concat( - [array_ops.expand_dims(initial_cell_state, [0]), cell_states], 0) - mod_outputs = array_ops.concat( - [array_ops.expand_dims(initial_output, [0]), outputs], 0) - final_cell_state = self._gather_states(mod_cell_states, sequence_length, - batch_size) - final_output = self._gather_states(mod_outputs, sequence_length, - batch_size) - else: - # No sequence_lengths used: final state is the last state - final_cell_state = cell_states[-1] - final_output = outputs[-1] - - if is_list: - # Input was a list, so return a list - outputs = array_ops.unstack(outputs) - - final_state = rnn_cell_impl.LSTMStateTuple(final_cell_state, final_output) - return outputs, final_state - - def _gather_states(self, data, indices, batch_size): - """Produce `out`, s.t. out(i, j) = data(indices(i), i, j).""" - return array_ops.gather_nd( - data, array_ops.stack([indices, math_ops.range(batch_size)], axis=1)) - - -class LSTMBlockFusedCell(LSTMBlockWrapper): - """FusedRNNCell implementation of LSTM. - - This is an extremely efficient LSTM implementation, that uses a single TF op - for the entire LSTM. It should be both faster and more memory-efficient than - LSTMBlockCell defined above. - - The implementation is based on: http://arxiv.org/abs/1409.2329. - - We add forget_bias (default: 1) to the biases of the forget gate in order to - reduce the scale of forgetting in the beginning of the training. - - The variable naming is consistent with `rnn_cell_impl.LSTMCell`. - """ - - def __init__(self, - num_units, - forget_bias=1.0, - cell_clip=None, - use_peephole=False, - reuse=None, - dtype=None, - name="lstm_fused_cell"): - """Initialize the LSTM cell. - - Args: - num_units: int, The number of units in the LSTM cell. - forget_bias: float, The bias added to forget gates (see above). - cell_clip: clip the cell to this value. Defaults is no cell clipping. - use_peephole: Whether to use peephole connections or not. - reuse: (optional) boolean describing whether to reuse variables in an - existing scope. If not `True`, and the existing scope already has the - given variables, an error is raised. - dtype: the dtype of variables of this layer. - name: String, the name of the layer. Layers with the same name will - share weights, but to avoid mistakes we require reuse=True in such - cases. By default this is "lstm_cell", for variable-name compatibility - with `tf.compat.v1.nn.rnn_cell.LSTMCell`. - """ - super(LSTMBlockFusedCell, self).__init__( - _reuse=reuse, name=name, dtype=dtype) - self._num_units = num_units - self._forget_bias = forget_bias - self._cell_clip = cell_clip if cell_clip is not None else -1 - self._use_peephole = use_peephole - - # Inputs must be 3-dimensional. - self.input_spec = input_spec.InputSpec(ndim=3) - - @property - def num_units(self): - """Number of units in this cell (output dimension).""" - return self._num_units - - def build(self, input_shape): - input_size = input_shape.dims[2].value - self._kernel = self.add_variable( - "kernel", [input_size + self._num_units, self._num_units * 4]) - self._bias = self.add_variable( - "bias", [self._num_units * 4], - initializer=init_ops.constant_initializer(0.0)) - if self._use_peephole: - self._w_i_diag = self.add_variable("w_i_diag", [self._num_units]) - self._w_f_diag = self.add_variable("w_f_diag", [self._num_units]) - self._w_o_diag = self.add_variable("w_o_diag", [self._num_units]) - - self.built = True - - def _call_cell(self, - inputs, - initial_cell_state=None, - initial_output=None, - dtype=None, - sequence_length=None): - """Run this LSTM on inputs, starting from the given state. - - Args: - inputs: `3-D` tensor with shape `[time_len, batch_size, input_size]` - initial_cell_state: initial value for cell state, shape `[batch_size, - self._num_units]` - initial_output: initial value of cell output, shape `[batch_size, - self._num_units]` - dtype: The data type for the initial state and expected output. - sequence_length: Specifies the length of each sequence in inputs. An - `int32` or `int64` vector (tensor) size `[batch_size]`, values in `[0, - time_len)` or None. - - Returns: - A pair containing: - - - Cell state (cs): A `3-D` tensor of shape `[time_len, batch_size, - output_size]` - - Output (h): A `3-D` tensor of shape `[time_len, batch_size, - output_size]` - """ - - inputs_shape = inputs.get_shape().with_rank(3) - time_len = inputs_shape.dims[0].value - if time_len is None: - time_len = array_ops.shape(inputs)[0] - - if self._use_peephole: - wci = self._w_i_diag - wco = self._w_o_diag - wcf = self._w_f_diag - else: - wci = wcf = wco = array_ops.zeros([self._num_units], dtype=dtype) - - if sequence_length is None: - max_seq_len = math_ops.cast(time_len, dtypes.int64) - else: - max_seq_len = math_ops.cast(math_ops.reduce_max(sequence_length), - dtypes.int64) - - _, cs, _, _, _, _, h = gen_rnn_ops.block_lstm( - seq_len_max=max_seq_len, - x=inputs, - cs_prev=initial_cell_state, - h_prev=initial_output, - w=self._kernel, - wci=wci, - wcf=wcf, - wco=wco, - b=self._bias, - forget_bias=self._forget_bias, - cell_clip=self._cell_clip, - use_peephole=self._use_peephole) - return cs, h diff --git a/tensorflow/contrib/rnn/python/ops/rnn.py b/tensorflow/contrib/rnn/python/ops/rnn.py deleted file mode 100644 index 41b1698321e..00000000000 --- a/tensorflow/contrib/rnn/python/ops/rnn.py +++ /dev/null @@ -1,245 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""RNN helpers for TensorFlow models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import rnn -from tensorflow.python.ops import variable_scope as vs - - -def stack_bidirectional_rnn(cells_fw, - cells_bw, - inputs, - initial_states_fw=None, - initial_states_bw=None, - dtype=None, - sequence_length=None, - scope=None): - """Creates a bidirectional recurrent neural network. - - Stacks several bidirectional rnn layers. The combined forward and backward - layer outputs are used as input of the next layer. tf.bidirectional_rnn - does not allow to share forward and backward information between layers. - The input_size of the first forward and backward cells must match. - The initial state for both directions is zero and no intermediate states - are returned. - - As described in https://arxiv.org/abs/1303.5778 - - Args: - cells_fw: List of instances of RNNCell, one per layer, - to be used for forward direction. - cells_bw: List of instances of RNNCell, one per layer, - to be used for backward direction. - inputs: A length T list of inputs, each a tensor of shape - [batch_size, input_size], or a nested tuple of such elements. - initial_states_fw: (optional) A list of the initial states (one per layer) - for the forward RNN. - Each tensor must has an appropriate type and shape - `[batch_size, cell_fw.state_size]`. - initial_states_bw: (optional) Same as for `initial_states_fw`, but using - the corresponding properties of `cells_bw`. - dtype: (optional) The data type for the initial state. Required if - either of the initial states are not provided. - sequence_length: (optional) An int32/int64 vector, size `[batch_size]`, - containing the actual lengths for each of the sequences. - scope: VariableScope for the created subgraph; defaults to None. - - Returns: - A tuple (outputs, output_state_fw, output_state_bw) where: - outputs is a length `T` list of outputs (one for each input), which - are depth-concatenated forward and backward outputs. - output_states_fw is the final states, one tensor per layer, - of the forward rnn. - output_states_bw is the final states, one tensor per layer, - of the backward rnn. - - Raises: - TypeError: If `cell_fw` or `cell_bw` is not an instance of `RNNCell`. - ValueError: If inputs is None, not a list or an empty list. - """ - if not cells_fw: - raise ValueError("Must specify at least one fw cell for BidirectionalRNN.") - if not cells_bw: - raise ValueError("Must specify at least one bw cell for BidirectionalRNN.") - if not isinstance(cells_fw, list): - raise ValueError("cells_fw must be a list of RNNCells (one per layer).") - if not isinstance(cells_bw, list): - raise ValueError("cells_bw must be a list of RNNCells (one per layer).") - if len(cells_fw) != len(cells_bw): - raise ValueError("Forward and Backward cells must have the same depth.") - if (initial_states_fw is not None and - (not isinstance(initial_states_fw, list) or - len(initial_states_fw) != len(cells_fw))): - raise ValueError( - "initial_states_fw must be a list of state tensors (one per layer).") - if (initial_states_bw is not None and - (not isinstance(initial_states_bw, list) or - len(initial_states_bw) != len(cells_bw))): - raise ValueError( - "initial_states_bw must be a list of state tensors (one per layer).") - states_fw = [] - states_bw = [] - prev_layer = inputs - - with vs.variable_scope(scope or "stack_bidirectional_rnn"): - for i, (cell_fw, cell_bw) in enumerate(zip(cells_fw, cells_bw)): - initial_state_fw = None - initial_state_bw = None - if initial_states_fw: - initial_state_fw = initial_states_fw[i] - if initial_states_bw: - initial_state_bw = initial_states_bw[i] - - with vs.variable_scope("cell_%d" % i) as cell_scope: - prev_layer, state_fw, state_bw = rnn.static_bidirectional_rnn( - cell_fw, - cell_bw, - prev_layer, - initial_state_fw=initial_state_fw, - initial_state_bw=initial_state_bw, - sequence_length=sequence_length, - dtype=dtype, - scope=cell_scope) - states_fw.append(state_fw) - states_bw.append(state_bw) - - return prev_layer, tuple(states_fw), tuple(states_bw) - - -def stack_bidirectional_dynamic_rnn(cells_fw, - cells_bw, - inputs, - initial_states_fw=None, - initial_states_bw=None, - dtype=None, - sequence_length=None, - parallel_iterations=None, - time_major=False, - scope=None, - swap_memory=False): - """Creates a dynamic bidirectional recurrent neural network. - - Stacks several bidirectional rnn layers. The combined forward and backward - layer outputs are used as input of the next layer. tf.bidirectional_rnn - does not allow to share forward and backward information between layers. - The input_size of the first forward and backward cells must match. - The initial state for both directions is zero and no intermediate states - are returned. - - Args: - cells_fw: List of instances of RNNCell, one per layer, - to be used for forward direction. - cells_bw: List of instances of RNNCell, one per layer, - to be used for backward direction. - inputs: The RNN inputs. this must be a tensor of shape: - `[batch_size, max_time, ...]`, or a nested tuple of such elements. - initial_states_fw: (optional) A list of the initial states (one per layer) - for the forward RNN. - Each tensor must has an appropriate type and shape - `[batch_size, cell_fw.state_size]`. - initial_states_bw: (optional) Same as for `initial_states_fw`, but using - the corresponding properties of `cells_bw`. - dtype: (optional) The data type for the initial state. Required if - either of the initial states are not provided. - sequence_length: (optional) An int32/int64 vector, size `[batch_size]`, - containing the actual lengths for each of the sequences. - parallel_iterations: (Default: 32). The number of iterations to run in - parallel. Those operations which do not have any temporal dependency - and can be run in parallel, will be. This parameter trades off - time for space. Values >> 1 use more memory but take less time, - while smaller values use less memory but computations take longer. - time_major: The shape format of the inputs and outputs Tensors. If true, - these Tensors must be shaped [max_time, batch_size, depth]. If false, - these Tensors must be shaped [batch_size, max_time, depth]. Using - time_major = True is a bit more efficient because it avoids transposes at - the beginning and end of the RNN calculation. However, most TensorFlow - data is batch-major, so by default this function accepts input and emits - output in batch-major form. - scope: VariableScope for the created subgraph; defaults to None. - swap_memory: Transparently swap the tensors produced in forward inference - but needed for back prop from GPU to CPU. This allows training RNNs - which would typically not fit on a single GPU, with very minimal (or no) - performance penalty. - - Returns: - A tuple (outputs, output_state_fw, output_state_bw) where: - outputs: Output `Tensor` shaped: - `[batch_size, max_time, layers_output]`. Where layers_output - are depth-concatenated forward and backward outputs. - output_states_fw is the final states, one tensor per layer, - of the forward rnn. - output_states_bw is the final states, one tensor per layer, - of the backward rnn. - - Raises: - TypeError: If `cell_fw` or `cell_bw` is not an instance of `RNNCell`. - ValueError: If inputs is `None`. - """ - if not cells_fw: - raise ValueError("Must specify at least one fw cell for BidirectionalRNN.") - if not cells_bw: - raise ValueError("Must specify at least one bw cell for BidirectionalRNN.") - if not isinstance(cells_fw, list): - raise ValueError("cells_fw must be a list of RNNCells (one per layer).") - if not isinstance(cells_bw, list): - raise ValueError("cells_bw must be a list of RNNCells (one per layer).") - if len(cells_fw) != len(cells_bw): - raise ValueError("Forward and Backward cells must have the same depth.") - if (initial_states_fw is not None and - (not isinstance(initial_states_fw, list) or - len(initial_states_fw) != len(cells_fw))): - raise ValueError( - "initial_states_fw must be a list of state tensors (one per layer).") - if (initial_states_bw is not None and - (not isinstance(initial_states_bw, list) or - len(initial_states_bw) != len(cells_bw))): - raise ValueError( - "initial_states_bw must be a list of state tensors (one per layer).") - - states_fw = [] - states_bw = [] - prev_layer = inputs - - with vs.variable_scope(scope or "stack_bidirectional_rnn"): - for i, (cell_fw, cell_bw) in enumerate(zip(cells_fw, cells_bw)): - initial_state_fw = None - initial_state_bw = None - if initial_states_fw: - initial_state_fw = initial_states_fw[i] - if initial_states_bw: - initial_state_bw = initial_states_bw[i] - - with vs.variable_scope("cell_%d" % i): - outputs, (state_fw, state_bw) = rnn.bidirectional_dynamic_rnn( - cell_fw, - cell_bw, - prev_layer, - initial_state_fw=initial_state_fw, - initial_state_bw=initial_state_bw, - sequence_length=sequence_length, - parallel_iterations=parallel_iterations, - dtype=dtype, - swap_memory=swap_memory, - time_major=time_major) - # Concat the outputs to create the new input. - prev_layer = array_ops.concat(outputs, 2) - states_fw.append(state_fw) - states_bw.append(state_bw) - - return prev_layer, tuple(states_fw), tuple(states_bw) diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py deleted file mode 100644 index c0939c84c44..00000000000 --- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py +++ /dev/null @@ -1,4006 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Module for constructing RNN Cells.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import math - -from tensorflow.contrib.compiler import jit -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.rnn.python.ops import core_rnn_cell -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import op_def_registry -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.keras import activations -from tensorflow.python.keras import initializers -from tensorflow.python.keras.engine import input_spec -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_impl # pylint: disable=unused-import -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import partitioned_variables # pylint: disable=unused-import -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import rnn_cell_impl -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util import nest - - -def _get_concat_variable(name, shape, dtype, num_shards): - """Get a sharded variable concatenated into one tensor.""" - sharded_variable = _get_sharded_variable(name, shape, dtype, num_shards) - if len(sharded_variable) == 1: - return sharded_variable[0] - - concat_name = name + "/concat" - concat_full_name = vs.get_variable_scope().name + "/" + concat_name + ":0" - for value in ops.get_collection(ops.GraphKeys.CONCATENATED_VARIABLES): - if value.name == concat_full_name: - return value - - concat_variable = array_ops.concat(sharded_variable, 0, name=concat_name) - ops.add_to_collection(ops.GraphKeys.CONCATENATED_VARIABLES, concat_variable) - return concat_variable - - -def _get_sharded_variable(name, shape, dtype, num_shards): - """Get a list of sharded variables with the given dtype.""" - if num_shards > shape[0]: - raise ValueError("Too many shards: shape=%s, num_shards=%d" % (shape, - num_shards)) - unit_shard_size = int(math.floor(shape[0] / num_shards)) - remaining_rows = shape[0] - unit_shard_size * num_shards - - shards = [] - for i in range(num_shards): - current_size = unit_shard_size - if i < remaining_rows: - current_size += 1 - shards.append( - vs.get_variable( - name + "_%d" % i, [current_size] + shape[1:], dtype=dtype)) - return shards - - -def _norm(g, b, inp, scope): - shape = inp.get_shape()[-1:] - gamma_init = init_ops.constant_initializer(g) - beta_init = init_ops.constant_initializer(b) - with vs.variable_scope(scope): - # Initialize beta and gamma for use by layer_norm. - vs.get_variable("gamma", shape=shape, initializer=gamma_init) - vs.get_variable("beta", shape=shape, initializer=beta_init) - normalized = layers.layer_norm(inp, reuse=True, scope=scope) - return normalized - - -class CoupledInputForgetGateLSTMCell(rnn_cell_impl.RNNCell): - """Long short-term memory unit (LSTM) recurrent network cell. - - The default non-peephole implementation is based on: - - https://pdfs.semanticscholar.org/1154/0131eae85b2e11d53df7f1360eeb6476e7f4.pdf - - Felix Gers, Jurgen Schmidhuber, and Fred Cummins. - "Learning to forget: Continual prediction with LSTM." IET, 850-855, 1999. - - The peephole implementation is based on: - - https://research.google.com/pubs/archive/43905.pdf - - Hasim Sak, Andrew Senior, and Francoise Beaufays. - "Long short-term memory recurrent neural network architectures for - large scale acoustic modeling." INTERSPEECH, 2014. - - The coupling of input and forget gate is based on: - - http://arxiv.org/pdf/1503.04069.pdf - - Greff et al. "LSTM: A Search Space Odyssey" - - The class uses optional peep-hole connections, and an optional projection - layer. - Layer normalization implementation is based on: - - https://arxiv.org/abs/1607.06450. - - "Layer Normalization" - Jimmy Lei Ba, Jamie Ryan Kiros, Geoffrey E. Hinton - - and is applied before the internal nonlinearities. - - """ - - def __init__(self, - num_units, - use_peepholes=False, - initializer=None, - num_proj=None, - proj_clip=None, - num_unit_shards=1, - num_proj_shards=1, - forget_bias=1.0, - state_is_tuple=True, - activation=math_ops.tanh, - reuse=None, - layer_norm=False, - norm_gain=1.0, - norm_shift=0.0): - """Initialize the parameters for an LSTM cell. - - Args: - num_units: int, The number of units in the LSTM cell - use_peepholes: bool, set True to enable diagonal/peephole connections. - initializer: (optional) The initializer to use for the weight and - projection matrices. - num_proj: (optional) int, The output dimensionality for the projection - matrices. If None, no projection is performed. - proj_clip: (optional) A float value. If `num_proj > 0` and `proj_clip` is - provided, then the projected values are clipped elementwise to within - `[-proj_clip, proj_clip]`. - num_unit_shards: How to split the weight matrix. If >1, the weight - matrix is stored across num_unit_shards. - num_proj_shards: How to split the projection matrix. If >1, the - projection matrix is stored across num_proj_shards. - forget_bias: Biases of the forget gate are initialized by default to 1 - in order to reduce the scale of forgetting at the beginning of - the training. - state_is_tuple: If True, accepted and returned states are 2-tuples of - the `c_state` and `m_state`. By default (False), they are concatenated - along the column axis. This default behavior will soon be deprecated. - activation: Activation function of the inner states. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - layer_norm: If `True`, layer normalization will be applied. - norm_gain: float, The layer normalization gain initial value. If - `layer_norm` has been set to `False`, this argument will be ignored. - norm_shift: float, The layer normalization shift initial value. If - `layer_norm` has been set to `False`, this argument will be ignored. - """ - super(CoupledInputForgetGateLSTMCell, self).__init__(_reuse=reuse) - if not state_is_tuple: - logging.warn("%s: Using a concatenated state is slower and will soon be " - "deprecated. Use state_is_tuple=True.", self) - self._num_units = num_units - self._use_peepholes = use_peepholes - self._initializer = initializer - self._num_proj = num_proj - self._proj_clip = proj_clip - self._num_unit_shards = num_unit_shards - self._num_proj_shards = num_proj_shards - self._forget_bias = forget_bias - self._state_is_tuple = state_is_tuple - self._activation = activation - self._reuse = reuse - self._layer_norm = layer_norm - self._norm_gain = norm_gain - self._norm_shift = norm_shift - - if num_proj: - self._state_size = ( - rnn_cell_impl.LSTMStateTuple(num_units, num_proj) - if state_is_tuple else num_units + num_proj) - self._output_size = num_proj - else: - self._state_size = ( - rnn_cell_impl.LSTMStateTuple(num_units, num_units) - if state_is_tuple else 2 * num_units) - self._output_size = num_units - - @property - def state_size(self): - return self._state_size - - @property - def output_size(self): - return self._output_size - - def call(self, inputs, state): - """Run one step of LSTM. - - Args: - inputs: input Tensor, 2D, batch x num_units. - state: if `state_is_tuple` is False, this must be a state Tensor, - `2-D, batch x state_size`. If `state_is_tuple` is True, this must be a - tuple of state Tensors, both `2-D`, with column sizes `c_state` and - `m_state`. - - Returns: - A tuple containing: - - A `2-D, [batch x output_dim]`, Tensor representing the output of the - LSTM after reading `inputs` when previous state was `state`. - Here output_dim is: - num_proj if num_proj was set, - num_units otherwise. - - Tensor(s) representing the new state of LSTM after reading `inputs` when - the previous state was `state`. Same type and shape(s) as `state`. - - Raises: - ValueError: If input size cannot be inferred from inputs via - static shape inference. - """ - sigmoid = math_ops.sigmoid - - num_proj = self._num_units if self._num_proj is None else self._num_proj - - if self._state_is_tuple: - (c_prev, m_prev) = state - else: - c_prev = array_ops.slice(state, [0, 0], [-1, self._num_units]) - m_prev = array_ops.slice(state, [0, self._num_units], [-1, num_proj]) - - dtype = inputs.dtype - input_size = inputs.get_shape().with_rank(2).dims[1] - if input_size.value is None: - raise ValueError("Could not infer input size from inputs.get_shape()[-1]") - concat_w = _get_concat_variable( - "W", - [input_size.value + num_proj, 3 * self._num_units], - dtype, - self._num_unit_shards) - - b = vs.get_variable( - "B", - shape=[3 * self._num_units], - initializer=init_ops.zeros_initializer(), - dtype=dtype) - - # j = new_input, f = forget_gate, o = output_gate - cell_inputs = array_ops.concat([inputs, m_prev], 1) - lstm_matrix = math_ops.matmul(cell_inputs, concat_w) - - # If layer nomalization is applied, do not add bias - if not self._layer_norm: - lstm_matrix = nn_ops.bias_add(lstm_matrix, b) - - j, f, o = array_ops.split(value=lstm_matrix, num_or_size_splits=3, axis=1) - - # Apply layer normalization - if self._layer_norm: - j = _norm(self._norm_gain, self._norm_shift, j, "transform") - f = _norm(self._norm_gain, self._norm_shift, f, "forget") - o = _norm(self._norm_gain, self._norm_shift, o, "output") - - # Diagonal connections - if self._use_peepholes: - w_f_diag = vs.get_variable( - "W_F_diag", shape=[self._num_units], dtype=dtype) - w_o_diag = vs.get_variable( - "W_O_diag", shape=[self._num_units], dtype=dtype) - - if self._use_peepholes: - f_act = sigmoid(f + self._forget_bias + w_f_diag * c_prev) - else: - f_act = sigmoid(f + self._forget_bias) - c = (f_act * c_prev + (1 - f_act) * self._activation(j)) - - # Apply layer normalization - if self._layer_norm: - c = _norm(self._norm_gain, self._norm_shift, c, "state") - - if self._use_peepholes: - m = sigmoid(o + w_o_diag * c) * self._activation(c) - else: - m = sigmoid(o) * self._activation(c) - - if self._num_proj is not None: - concat_w_proj = _get_concat_variable("W_P", - [self._num_units, self._num_proj], - dtype, self._num_proj_shards) - - m = math_ops.matmul(m, concat_w_proj) - if self._proj_clip is not None: - # pylint: disable=invalid-unary-operand-type - m = clip_ops.clip_by_value(m, -self._proj_clip, self._proj_clip) - # pylint: enable=invalid-unary-operand-type - - new_state = ( - rnn_cell_impl.LSTMStateTuple(c, m) - if self._state_is_tuple else array_ops.concat([c, m], 1)) - return m, new_state - - -class TimeFreqLSTMCell(rnn_cell_impl.RNNCell): - """Time-Frequency Long short-term memory unit (LSTM) recurrent network cell. - - This implementation is based on: - - Tara N. Sainath and Bo Li - "Modeling Time-Frequency Patterns with LSTM vs. Convolutional Architectures - for LVCSR Tasks." submitted to INTERSPEECH, 2016. - - It uses peep-hole connections and optional cell clipping. - """ - - def __init__(self, - num_units, - use_peepholes=False, - cell_clip=None, - initializer=None, - num_unit_shards=1, - forget_bias=1.0, - feature_size=None, - frequency_skip=1, - reuse=None): - """Initialize the parameters for an LSTM cell. - - Args: - num_units: int, The number of units in the LSTM cell - use_peepholes: bool, set True to enable diagonal/peephole connections. - cell_clip: (optional) A float value, if provided the cell state is clipped - by this value prior to the cell output activation. - initializer: (optional) The initializer to use for the weight and - projection matrices. - num_unit_shards: int, How to split the weight matrix. If >1, the weight - matrix is stored across num_unit_shards. - forget_bias: float, Biases of the forget gate are initialized by default - to 1 in order to reduce the scale of forgetting at the beginning - of the training. - feature_size: int, The size of the input feature the LSTM spans over. - frequency_skip: int, The amount the LSTM filter is shifted by in - frequency. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - """ - super(TimeFreqLSTMCell, self).__init__(_reuse=reuse) - self._num_units = num_units - self._use_peepholes = use_peepholes - self._cell_clip = cell_clip - self._initializer = initializer - self._num_unit_shards = num_unit_shards - self._forget_bias = forget_bias - self._feature_size = feature_size - self._frequency_skip = frequency_skip - self._state_size = 2 * num_units - self._output_size = num_units - self._reuse = reuse - - @property - def output_size(self): - return self._output_size - - @property - def state_size(self): - return self._state_size - - def call(self, inputs, state): - """Run one step of LSTM. - - Args: - inputs: input Tensor, 2D, batch x num_units. - state: state Tensor, 2D, batch x state_size. - - Returns: - A tuple containing: - - A 2D, batch x output_dim, Tensor representing the output of the LSTM - after reading "inputs" when previous state was "state". - Here output_dim is num_units. - - A 2D, batch x state_size, Tensor representing the new state of LSTM - after reading "inputs" when previous state was "state". - Raises: - ValueError: if an input_size was specified and the provided inputs have - a different dimension. - """ - sigmoid = math_ops.sigmoid - tanh = math_ops.tanh - - freq_inputs = self._make_tf_features(inputs) - dtype = inputs.dtype - actual_input_size = freq_inputs[0].get_shape().as_list()[1] - - concat_w = _get_concat_variable( - "W", [actual_input_size + 2 * self._num_units, 4 * self._num_units], - dtype, self._num_unit_shards) - - b = vs.get_variable( - "B", - shape=[4 * self._num_units], - initializer=init_ops.zeros_initializer(), - dtype=dtype) - - # Diagonal connections - if self._use_peepholes: - w_f_diag = vs.get_variable( - "W_F_diag", shape=[self._num_units], dtype=dtype) - w_i_diag = vs.get_variable( - "W_I_diag", shape=[self._num_units], dtype=dtype) - w_o_diag = vs.get_variable( - "W_O_diag", shape=[self._num_units], dtype=dtype) - - # initialize the first freq state to be zero - m_prev_freq = array_ops.zeros( - [inputs.shape.dims[0].value or inputs.get_shape()[0], self._num_units], - dtype) - for fq in range(len(freq_inputs)): - c_prev = array_ops.slice(state, [0, 2 * fq * self._num_units], - [-1, self._num_units]) - m_prev = array_ops.slice(state, [0, (2 * fq + 1) * self._num_units], - [-1, self._num_units]) - # i = input_gate, j = new_input, f = forget_gate, o = output_gate - cell_inputs = array_ops.concat([freq_inputs[fq], m_prev, m_prev_freq], 1) - lstm_matrix = nn_ops.bias_add(math_ops.matmul(cell_inputs, concat_w), b) - i, j, f, o = array_ops.split( - value=lstm_matrix, num_or_size_splits=4, axis=1) - - if self._use_peepholes: - c = ( - sigmoid(f + self._forget_bias + w_f_diag * c_prev) * c_prev + - sigmoid(i + w_i_diag * c_prev) * tanh(j)) - else: - c = (sigmoid(f + self._forget_bias) * c_prev + sigmoid(i) * tanh(j)) - - if self._cell_clip is not None: - # pylint: disable=invalid-unary-operand-type - c = clip_ops.clip_by_value(c, -self._cell_clip, self._cell_clip) - # pylint: enable=invalid-unary-operand-type - - if self._use_peepholes: - m = sigmoid(o + w_o_diag * c) * tanh(c) - else: - m = sigmoid(o) * tanh(c) - m_prev_freq = m - if fq == 0: - state_out = array_ops.concat([c, m], 1) - m_out = m - else: - state_out = array_ops.concat([state_out, c, m], 1) - m_out = array_ops.concat([m_out, m], 1) - return m_out, state_out - - def _make_tf_features(self, input_feat): - """Make the frequency features. - - Args: - input_feat: input Tensor, 2D, batch x num_units. - - Returns: - A list of frequency features, with each element containing: - - A 2D, batch x output_dim, Tensor representing the time-frequency feature - for that frequency index. Here output_dim is feature_size. - Raises: - ValueError: if input_size cannot be inferred from static shape inference. - """ - input_size = input_feat.get_shape().with_rank(2).dims[-1].value - if input_size is None: - raise ValueError("Cannot infer input_size from static shape inference.") - num_feats = int( - (input_size - self._feature_size) / (self._frequency_skip)) + 1 - freq_inputs = [] - for f in range(num_feats): - cur_input = array_ops.slice(input_feat, [0, f * self._frequency_skip], - [-1, self._feature_size]) - freq_inputs.append(cur_input) - return freq_inputs - - -class GridLSTMCell(rnn_cell_impl.RNNCell): - """Grid Long short-term memory unit (LSTM) recurrent network cell. - - The default is based on: - Nal Kalchbrenner, Ivo Danihelka and Alex Graves - "Grid Long Short-Term Memory," Proc. ICLR 2016. - http://arxiv.org/abs/1507.01526 - - When peephole connections are used, the implementation is based on: - Tara N. Sainath and Bo Li - "Modeling Time-Frequency Patterns with LSTM vs. Convolutional Architectures - for LVCSR Tasks." submitted to INTERSPEECH, 2016. - - The code uses optional peephole connections, shared_weights and cell clipping. - """ - - def __init__(self, - num_units, - use_peepholes=False, - share_time_frequency_weights=False, - cell_clip=None, - initializer=None, - num_unit_shards=1, - forget_bias=1.0, - feature_size=None, - frequency_skip=None, - num_frequency_blocks=None, - start_freqindex_list=None, - end_freqindex_list=None, - couple_input_forget_gates=False, - state_is_tuple=True, - reuse=None): - """Initialize the parameters for an LSTM cell. - - Args: - num_units: int, The number of units in the LSTM cell - use_peepholes: (optional) bool, default False. Set True to enable - diagonal/peephole connections. - share_time_frequency_weights: (optional) bool, default False. Set True to - enable shared cell weights between time and frequency LSTMs. - cell_clip: (optional) A float value, default None, if provided the cell - state is clipped by this value prior to the cell output activation. - initializer: (optional) The initializer to use for the weight and - projection matrices, default None. - num_unit_shards: (optional) int, default 1, How to split the weight - matrix. If > 1, the weight matrix is stored across num_unit_shards. - forget_bias: (optional) float, default 1.0, The initial bias of the - forget gates, used to reduce the scale of forgetting at the beginning - of the training. - feature_size: (optional) int, default None, The size of the input feature - the LSTM spans over. - frequency_skip: (optional) int, default None, The amount the LSTM filter - is shifted by in frequency. - num_frequency_blocks: [required] A list of frequency blocks needed to - cover the whole input feature splitting defined by start_freqindex_list - and end_freqindex_list. - start_freqindex_list: [optional], list of ints, default None, The - starting frequency index for each frequency block. - end_freqindex_list: [optional], list of ints, default None. The ending - frequency index for each frequency block. - couple_input_forget_gates: (optional) bool, default False, Whether to - couple the input and forget gates, i.e. f_gate = 1.0 - i_gate, to reduce - model parameters and computation cost. - state_is_tuple: If True, accepted and returned states are 2-tuples of - the `c_state` and `m_state`. By default (False), they are concatenated - along the column axis. This default behavior will soon be deprecated. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - Raises: - ValueError: if the num_frequency_blocks list is not specified - """ - super(GridLSTMCell, self).__init__(_reuse=reuse) - if not state_is_tuple: - logging.warn("%s: Using a concatenated state is slower and will soon be " - "deprecated. Use state_is_tuple=True.", self) - self._num_units = num_units - self._use_peepholes = use_peepholes - self._share_time_frequency_weights = share_time_frequency_weights - self._couple_input_forget_gates = couple_input_forget_gates - self._state_is_tuple = state_is_tuple - self._cell_clip = cell_clip - self._initializer = initializer - self._num_unit_shards = num_unit_shards - self._forget_bias = forget_bias - self._feature_size = feature_size - self._frequency_skip = frequency_skip - self._start_freqindex_list = start_freqindex_list - self._end_freqindex_list = end_freqindex_list - self._num_frequency_blocks = num_frequency_blocks - self._total_blocks = 0 - self._reuse = reuse - if self._num_frequency_blocks is None: - raise ValueError("Must specify num_frequency_blocks") - - for block_index in range(len(self._num_frequency_blocks)): - self._total_blocks += int(self._num_frequency_blocks[block_index]) - if state_is_tuple: - state_names = "" - for block_index in range(len(self._num_frequency_blocks)): - for freq_index in range(self._num_frequency_blocks[block_index]): - name_prefix = "state_f%02d_b%02d" % (freq_index, block_index) - state_names += ("%s_c, %s_m," % (name_prefix, name_prefix)) - self._state_tuple_type = collections.namedtuple("GridLSTMStateTuple", - state_names.strip(",")) - self._state_size = self._state_tuple_type(*( - [num_units, num_units] * self._total_blocks)) - else: - self._state_tuple_type = None - self._state_size = num_units * self._total_blocks * 2 - self._output_size = num_units * self._total_blocks * 2 - - @property - def output_size(self): - return self._output_size - - @property - def state_size(self): - return self._state_size - - @property - def state_tuple_type(self): - return self._state_tuple_type - - def call(self, inputs, state): - """Run one step of LSTM. - - Args: - inputs: input Tensor, 2D, [batch, feature_size]. - state: Tensor or tuple of Tensors, 2D, [batch, state_size], depends on the - flag self._state_is_tuple. - - Returns: - A tuple containing: - - A 2D, [batch, output_dim], Tensor representing the output of the LSTM - after reading "inputs" when previous state was "state". - Here output_dim is num_units. - - A 2D, [batch, state_size], Tensor representing the new state of LSTM - after reading "inputs" when previous state was "state". - Raises: - ValueError: if an input_size was specified and the provided inputs have - a different dimension. - """ - batch_size = tensor_shape.dimension_value( - inputs.shape[0]) or array_ops.shape(inputs)[0] - freq_inputs = self._make_tf_features(inputs) - m_out_lst = [] - state_out_lst = [] - for block in range(len(freq_inputs)): - m_out_lst_current, state_out_lst_current = self._compute( - freq_inputs[block], - block, - state, - batch_size, - state_is_tuple=self._state_is_tuple) - m_out_lst.extend(m_out_lst_current) - state_out_lst.extend(state_out_lst_current) - if self._state_is_tuple: - state_out = self._state_tuple_type(*state_out_lst) - else: - state_out = array_ops.concat(state_out_lst, 1) - m_out = array_ops.concat(m_out_lst, 1) - return m_out, state_out - - def _compute(self, - freq_inputs, - block, - state, - batch_size, - state_prefix="state", - state_is_tuple=True): - """Run the actual computation of one step LSTM. - - Args: - freq_inputs: list of Tensors, 2D, [batch, feature_size]. - block: int, current frequency block index to process. - state: Tensor or tuple of Tensors, 2D, [batch, state_size], it depends on - the flag state_is_tuple. - batch_size: int32, batch size. - state_prefix: (optional) string, name prefix for states, defaults to - "state". - state_is_tuple: boolean, indicates whether the state is a tuple or Tensor. - - Returns: - A tuple, containing: - - A list of [batch, output_dim] Tensors, representing the output of the - LSTM given the inputs and state. - - A list of [batch, state_size] Tensors, representing the LSTM state - values given the inputs and previous state. - """ - sigmoid = math_ops.sigmoid - tanh = math_ops.tanh - num_gates = 3 if self._couple_input_forget_gates else 4 - dtype = freq_inputs[0].dtype - actual_input_size = freq_inputs[0].get_shape().as_list()[1] - - concat_w_f = _get_concat_variable( - "W_f_%d" % block, - [actual_input_size + 2 * self._num_units, num_gates * self._num_units], - dtype, self._num_unit_shards) - b_f = vs.get_variable( - "B_f_%d" % block, - shape=[num_gates * self._num_units], - initializer=init_ops.zeros_initializer(), - dtype=dtype) - if not self._share_time_frequency_weights: - concat_w_t = _get_concat_variable("W_t_%d" % block, [ - actual_input_size + 2 * self._num_units, num_gates * self._num_units - ], dtype, self._num_unit_shards) - b_t = vs.get_variable( - "B_t_%d" % block, - shape=[num_gates * self._num_units], - initializer=init_ops.zeros_initializer(), - dtype=dtype) - - if self._use_peepholes: - # Diagonal connections - if not self._couple_input_forget_gates: - w_f_diag_freqf = vs.get_variable( - "W_F_diag_freqf_%d" % block, shape=[self._num_units], dtype=dtype) - w_f_diag_freqt = vs.get_variable( - "W_F_diag_freqt_%d" % block, shape=[self._num_units], dtype=dtype) - w_i_diag_freqf = vs.get_variable( - "W_I_diag_freqf_%d" % block, shape=[self._num_units], dtype=dtype) - w_i_diag_freqt = vs.get_variable( - "W_I_diag_freqt_%d" % block, shape=[self._num_units], dtype=dtype) - w_o_diag_freqf = vs.get_variable( - "W_O_diag_freqf_%d" % block, shape=[self._num_units], dtype=dtype) - w_o_diag_freqt = vs.get_variable( - "W_O_diag_freqt_%d" % block, shape=[self._num_units], dtype=dtype) - if not self._share_time_frequency_weights: - if not self._couple_input_forget_gates: - w_f_diag_timef = vs.get_variable( - "W_F_diag_timef_%d" % block, shape=[self._num_units], dtype=dtype) - w_f_diag_timet = vs.get_variable( - "W_F_diag_timet_%d" % block, shape=[self._num_units], dtype=dtype) - w_i_diag_timef = vs.get_variable( - "W_I_diag_timef_%d" % block, shape=[self._num_units], dtype=dtype) - w_i_diag_timet = vs.get_variable( - "W_I_diag_timet_%d" % block, shape=[self._num_units], dtype=dtype) - w_o_diag_timef = vs.get_variable( - "W_O_diag_timef_%d" % block, shape=[self._num_units], dtype=dtype) - w_o_diag_timet = vs.get_variable( - "W_O_diag_timet_%d" % block, shape=[self._num_units], dtype=dtype) - - # initialize the first freq state to be zero - m_prev_freq = array_ops.zeros([batch_size, self._num_units], dtype) - c_prev_freq = array_ops.zeros([batch_size, self._num_units], dtype) - for freq_index in range(len(freq_inputs)): - if state_is_tuple: - name_prefix = "%s_f%02d_b%02d" % (state_prefix, freq_index, block) - c_prev_time = getattr(state, name_prefix + "_c") - m_prev_time = getattr(state, name_prefix + "_m") - else: - c_prev_time = array_ops.slice( - state, [0, 2 * freq_index * self._num_units], [-1, self._num_units]) - m_prev_time = array_ops.slice( - state, [0, (2 * freq_index + 1) * self._num_units], - [-1, self._num_units]) - - # i = input_gate, j = new_input, f = forget_gate, o = output_gate - cell_inputs = array_ops.concat( - [freq_inputs[freq_index], m_prev_time, m_prev_freq], 1) - - # F-LSTM - lstm_matrix_freq = nn_ops.bias_add( - math_ops.matmul(cell_inputs, concat_w_f), b_f) - if self._couple_input_forget_gates: - i_freq, j_freq, o_freq = array_ops.split( - value=lstm_matrix_freq, num_or_size_splits=num_gates, axis=1) - f_freq = None - else: - i_freq, j_freq, f_freq, o_freq = array_ops.split( - value=lstm_matrix_freq, num_or_size_splits=num_gates, axis=1) - # T-LSTM - if self._share_time_frequency_weights: - i_time = i_freq - j_time = j_freq - f_time = f_freq - o_time = o_freq - else: - lstm_matrix_time = nn_ops.bias_add( - math_ops.matmul(cell_inputs, concat_w_t), b_t) - if self._couple_input_forget_gates: - i_time, j_time, o_time = array_ops.split( - value=lstm_matrix_time, num_or_size_splits=num_gates, axis=1) - f_time = None - else: - i_time, j_time, f_time, o_time = array_ops.split( - value=lstm_matrix_time, num_or_size_splits=num_gates, axis=1) - - # F-LSTM c_freq - # input gate activations - if self._use_peepholes: - i_freq_g = sigmoid(i_freq + w_i_diag_freqf * c_prev_freq + - w_i_diag_freqt * c_prev_time) - else: - i_freq_g = sigmoid(i_freq) - # forget gate activations - if self._couple_input_forget_gates: - f_freq_g = 1.0 - i_freq_g - else: - if self._use_peepholes: - f_freq_g = sigmoid(f_freq + self._forget_bias + w_f_diag_freqf * - c_prev_freq + w_f_diag_freqt * c_prev_time) - else: - f_freq_g = sigmoid(f_freq + self._forget_bias) - # cell state - c_freq = f_freq_g * c_prev_freq + i_freq_g * tanh(j_freq) - if self._cell_clip is not None: - # pylint: disable=invalid-unary-operand-type - c_freq = clip_ops.clip_by_value(c_freq, -self._cell_clip, - self._cell_clip) - # pylint: enable=invalid-unary-operand-type - - # T-LSTM c_freq - # input gate activations - if self._use_peepholes: - if self._share_time_frequency_weights: - i_time_g = sigmoid(i_time + w_i_diag_freqf * c_prev_freq + - w_i_diag_freqt * c_prev_time) - else: - i_time_g = sigmoid(i_time + w_i_diag_timef * c_prev_freq + - w_i_diag_timet * c_prev_time) - else: - i_time_g = sigmoid(i_time) - # forget gate activations - if self._couple_input_forget_gates: - f_time_g = 1.0 - i_time_g - else: - if self._use_peepholes: - if self._share_time_frequency_weights: - f_time_g = sigmoid(f_time + self._forget_bias + w_f_diag_freqf * - c_prev_freq + w_f_diag_freqt * c_prev_time) - else: - f_time_g = sigmoid(f_time + self._forget_bias + w_f_diag_timef * - c_prev_freq + w_f_diag_timet * c_prev_time) - else: - f_time_g = sigmoid(f_time + self._forget_bias) - # cell state - c_time = f_time_g * c_prev_time + i_time_g * tanh(j_time) - if self._cell_clip is not None: - # pylint: disable=invalid-unary-operand-type - c_time = clip_ops.clip_by_value(c_time, -self._cell_clip, - self._cell_clip) - # pylint: enable=invalid-unary-operand-type - - # F-LSTM m_freq - if self._use_peepholes: - m_freq = sigmoid(o_freq + w_o_diag_freqf * c_freq + - w_o_diag_freqt * c_time) * tanh(c_freq) - else: - m_freq = sigmoid(o_freq) * tanh(c_freq) - - # T-LSTM m_time - if self._use_peepholes: - if self._share_time_frequency_weights: - m_time = sigmoid(o_time + w_o_diag_freqf * c_freq + - w_o_diag_freqt * c_time) * tanh(c_time) - else: - m_time = sigmoid(o_time + w_o_diag_timef * c_freq + - w_o_diag_timet * c_time) * tanh(c_time) - else: - m_time = sigmoid(o_time) * tanh(c_time) - - m_prev_freq = m_freq - c_prev_freq = c_freq - # Concatenate the outputs for T-LSTM and F-LSTM for each shift - if freq_index == 0: - state_out_lst = [c_time, m_time] - m_out_lst = [m_time, m_freq] - else: - state_out_lst.extend([c_time, m_time]) - m_out_lst.extend([m_time, m_freq]) - - return m_out_lst, state_out_lst - - def _make_tf_features(self, input_feat, slice_offset=0): - """Make the frequency features. - - Args: - input_feat: input Tensor, 2D, [batch, num_units]. - slice_offset: (optional) Python int, default 0, the slicing offset is only - used for the backward processing in the BidirectionalGridLSTMCell. It - specifies a different starting point instead of always 0 to enable the - forward and backward processing look at different frequency blocks. - - Returns: - A list of frequency features, with each element containing: - - A 2D, [batch, output_dim], Tensor representing the time-frequency - feature for that frequency index. Here output_dim is feature_size. - Raises: - ValueError: if input_size cannot be inferred from static shape inference. - """ - input_size = input_feat.get_shape().with_rank(2).dims[-1].value - if input_size is None: - raise ValueError("Cannot infer input_size from static shape inference.") - if slice_offset > 0: - # Padding to the end - inputs = array_ops.pad(input_feat, - array_ops.constant( - [0, 0, 0, slice_offset], - shape=[2, 2], - dtype=dtypes.int32), "CONSTANT") - elif slice_offset < 0: - # Padding to the front - inputs = array_ops.pad(input_feat, - array_ops.constant( - [0, 0, -slice_offset, 0], - shape=[2, 2], - dtype=dtypes.int32), "CONSTANT") - slice_offset = 0 - else: - inputs = input_feat - freq_inputs = [] - if not self._start_freqindex_list: - if len(self._num_frequency_blocks) != 1: - raise ValueError("Length of num_frequency_blocks" - " is not 1, but instead is %d" % - len(self._num_frequency_blocks)) - num_feats = int( - (input_size - self._feature_size) / (self._frequency_skip)) + 1 - if num_feats != self._num_frequency_blocks[0]: - raise ValueError( - "Invalid num_frequency_blocks, requires %d but gets %d, please" - " check the input size and filter config are correct." % - (self._num_frequency_blocks[0], num_feats)) - block_inputs = [] - for f in range(num_feats): - cur_input = array_ops.slice( - inputs, [0, slice_offset + f * self._frequency_skip], - [-1, self._feature_size]) - block_inputs.append(cur_input) - freq_inputs.append(block_inputs) - else: - if len(self._start_freqindex_list) != len(self._end_freqindex_list): - raise ValueError("Length of start and end freqindex_list" - " does not match %d %d", - len(self._start_freqindex_list), - len(self._end_freqindex_list)) - if len(self._num_frequency_blocks) != len(self._start_freqindex_list): - raise ValueError("Length of num_frequency_blocks" - " is not equal to start_freqindex_list %d %d", - len(self._num_frequency_blocks), - len(self._start_freqindex_list)) - for b in range(len(self._start_freqindex_list)): - start_index = self._start_freqindex_list[b] - end_index = self._end_freqindex_list[b] - cur_size = end_index - start_index - block_feats = int( - (cur_size - self._feature_size) / (self._frequency_skip)) + 1 - if block_feats != self._num_frequency_blocks[b]: - raise ValueError( - "Invalid num_frequency_blocks, requires %d but gets %d, please" - " check the input size and filter config are correct." % - (self._num_frequency_blocks[b], block_feats)) - block_inputs = [] - for f in range(block_feats): - cur_input = array_ops.slice( - inputs, - [0, start_index + slice_offset + f * self._frequency_skip], - [-1, self._feature_size]) - block_inputs.append(cur_input) - freq_inputs.append(block_inputs) - return freq_inputs - - -class BidirectionalGridLSTMCell(GridLSTMCell): - """Bidirectional GridLstm cell. - - The bidirection connection is only used in the frequency direction, which - hence doesn't affect the time direction's real-time processing that is - required for online recognition systems. - The current implementation uses different weights for the two directions. - """ - - def __init__(self, - num_units, - use_peepholes=False, - share_time_frequency_weights=False, - cell_clip=None, - initializer=None, - num_unit_shards=1, - forget_bias=1.0, - feature_size=None, - frequency_skip=None, - num_frequency_blocks=None, - start_freqindex_list=None, - end_freqindex_list=None, - couple_input_forget_gates=False, - backward_slice_offset=0, - reuse=None): - """Initialize the parameters for an LSTM cell. - - Args: - num_units: int, The number of units in the LSTM cell - use_peepholes: (optional) bool, default False. Set True to enable - diagonal/peephole connections. - share_time_frequency_weights: (optional) bool, default False. Set True to - enable shared cell weights between time and frequency LSTMs. - cell_clip: (optional) A float value, default None, if provided the cell - state is clipped by this value prior to the cell output activation. - initializer: (optional) The initializer to use for the weight and - projection matrices, default None. - num_unit_shards: (optional) int, default 1, How to split the weight - matrix. If > 1, the weight matrix is stored across num_unit_shards. - forget_bias: (optional) float, default 1.0, The initial bias of the - forget gates, used to reduce the scale of forgetting at the beginning - of the training. - feature_size: (optional) int, default None, The size of the input feature - the LSTM spans over. - frequency_skip: (optional) int, default None, The amount the LSTM filter - is shifted by in frequency. - num_frequency_blocks: [required] A list of frequency blocks needed to - cover the whole input feature splitting defined by start_freqindex_list - and end_freqindex_list. - start_freqindex_list: [optional], list of ints, default None, The - starting frequency index for each frequency block. - end_freqindex_list: [optional], list of ints, default None. The ending - frequency index for each frequency block. - couple_input_forget_gates: (optional) bool, default False, Whether to - couple the input and forget gates, i.e. f_gate = 1.0 - i_gate, to reduce - model parameters and computation cost. - backward_slice_offset: (optional) int32, default 0, the starting offset to - slice the feature for backward processing. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - """ - super(BidirectionalGridLSTMCell, self).__init__( - num_units, use_peepholes, share_time_frequency_weights, cell_clip, - initializer, num_unit_shards, forget_bias, feature_size, frequency_skip, - num_frequency_blocks, start_freqindex_list, end_freqindex_list, - couple_input_forget_gates, True, reuse) - self._backward_slice_offset = int(backward_slice_offset) - state_names = "" - for direction in ["fwd", "bwd"]: - for block_index in range(len(self._num_frequency_blocks)): - for freq_index in range(self._num_frequency_blocks[block_index]): - name_prefix = "%s_state_f%02d_b%02d" % (direction, freq_index, - block_index) - state_names += ("%s_c, %s_m," % (name_prefix, name_prefix)) - self._state_tuple_type = collections.namedtuple( - "BidirectionalGridLSTMStateTuple", state_names.strip(",")) - self._state_size = self._state_tuple_type(*( - [num_units, num_units] * self._total_blocks * 2)) - self._output_size = 2 * num_units * self._total_blocks * 2 - - def call(self, inputs, state): - """Run one step of LSTM. - - Args: - inputs: input Tensor, 2D, [batch, num_units]. - state: tuple of Tensors, 2D, [batch, state_size]. - - Returns: - A tuple containing: - - A 2D, [batch, output_dim], Tensor representing the output of the LSTM - after reading "inputs" when previous state was "state". - Here output_dim is num_units. - - A 2D, [batch, state_size], Tensor representing the new state of LSTM - after reading "inputs" when previous state was "state". - Raises: - ValueError: if an input_size was specified and the provided inputs have - a different dimension. - """ - batch_size = tensor_shape.dimension_value( - inputs.shape[0]) or array_ops.shape(inputs)[0] - fwd_inputs = self._make_tf_features(inputs) - if self._backward_slice_offset: - bwd_inputs = self._make_tf_features(inputs, self._backward_slice_offset) - else: - bwd_inputs = fwd_inputs - - # Forward processing - with vs.variable_scope("fwd"): - fwd_m_out_lst = [] - fwd_state_out_lst = [] - for block in range(len(fwd_inputs)): - fwd_m_out_lst_current, fwd_state_out_lst_current = self._compute( - fwd_inputs[block], - block, - state, - batch_size, - state_prefix="fwd_state", - state_is_tuple=True) - fwd_m_out_lst.extend(fwd_m_out_lst_current) - fwd_state_out_lst.extend(fwd_state_out_lst_current) - # Backward processing - bwd_m_out_lst = [] - bwd_state_out_lst = [] - with vs.variable_scope("bwd"): - for block in range(len(bwd_inputs)): - # Reverse the blocks - bwd_inputs_reverse = bwd_inputs[block][::-1] - bwd_m_out_lst_current, bwd_state_out_lst_current = self._compute( - bwd_inputs_reverse, - block, - state, - batch_size, - state_prefix="bwd_state", - state_is_tuple=True) - bwd_m_out_lst.extend(bwd_m_out_lst_current) - bwd_state_out_lst.extend(bwd_state_out_lst_current) - state_out = self._state_tuple_type(*(fwd_state_out_lst + bwd_state_out_lst)) - # Outputs are always concated as it is never used separately. - m_out = array_ops.concat(fwd_m_out_lst + bwd_m_out_lst, 1) - return m_out, state_out - - -# pylint: disable=protected-access -_Linear = core_rnn_cell._Linear # pylint: disable=invalid-name - -# pylint: enable=protected-access - - -class AttentionCellWrapper(rnn_cell_impl.RNNCell): - """Basic attention cell wrapper. - - Implementation based on https://arxiv.org/abs/1601.06733. - """ - - def __init__(self, - cell, - attn_length, - attn_size=None, - attn_vec_size=None, - input_size=None, - state_is_tuple=True, - reuse=None): - """Create a cell with attention. - - Args: - cell: an RNNCell, an attention is added to it. - attn_length: integer, the size of an attention window. - attn_size: integer, the size of an attention vector. Equal to - cell.output_size by default. - attn_vec_size: integer, the number of convolutional features calculated - on attention state and a size of the hidden layer built from - base cell state. Equal attn_size to by default. - input_size: integer, the size of a hidden linear layer, - built from inputs and attention. Derived from the input tensor - by default. - state_is_tuple: If True, accepted and returned states are n-tuples, where - `n = len(cells)`. By default (False), the states are all - concatenated along the column axis. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - - Raises: - TypeError: if cell is not an RNNCell. - ValueError: if cell returns a state tuple but the flag - `state_is_tuple` is `False` or if attn_length is zero or less. - """ - super(AttentionCellWrapper, self).__init__(_reuse=reuse) - rnn_cell_impl.assert_like_rnncell("cell", cell) - if nest.is_sequence(cell.state_size) and not state_is_tuple: - raise ValueError( - "Cell returns tuple of states, but the flag " - "state_is_tuple is not set. State size is: %s" % str(cell.state_size)) - if attn_length <= 0: - raise ValueError( - "attn_length should be greater than zero, got %s" % str(attn_length)) - if not state_is_tuple: - logging.warn("%s: Using a concatenated state is slower and will soon be " - "deprecated. Use state_is_tuple=True.", self) - if attn_size is None: - attn_size = cell.output_size - if attn_vec_size is None: - attn_vec_size = attn_size - self._state_is_tuple = state_is_tuple - self._cell = cell - self._attn_vec_size = attn_vec_size - self._input_size = input_size - self._attn_size = attn_size - self._attn_length = attn_length - self._reuse = reuse - self._linear1 = None - self._linear2 = None - self._linear3 = None - - @property - def state_size(self): - size = (self._cell.state_size, self._attn_size, - self._attn_size * self._attn_length) - if self._state_is_tuple: - return size - else: - return sum(list(size)) - - @property - def output_size(self): - return self._attn_size - - def call(self, inputs, state): - """Long short-term memory cell with attention (LSTMA).""" - if self._state_is_tuple: - state, attns, attn_states = state - else: - states = state - state = array_ops.slice(states, [0, 0], [-1, self._cell.state_size]) - attns = array_ops.slice(states, [0, self._cell.state_size], - [-1, self._attn_size]) - attn_states = array_ops.slice( - states, [0, self._cell.state_size + self._attn_size], - [-1, self._attn_size * self._attn_length]) - attn_states = array_ops.reshape(attn_states, - [-1, self._attn_length, self._attn_size]) - input_size = self._input_size - if input_size is None: - input_size = inputs.get_shape().as_list()[1] - if self._linear1 is None: - self._linear1 = _Linear([inputs, attns], input_size, True) - inputs = self._linear1([inputs, attns]) - cell_output, new_state = self._cell(inputs, state) - if self._state_is_tuple: - new_state_cat = array_ops.concat(nest.flatten(new_state), 1) - else: - new_state_cat = new_state - new_attns, new_attn_states = self._attention(new_state_cat, attn_states) - with vs.variable_scope("attn_output_projection"): - if self._linear2 is None: - self._linear2 = _Linear([cell_output, new_attns], self._attn_size, True) - output = self._linear2([cell_output, new_attns]) - new_attn_states = array_ops.concat( - [new_attn_states, array_ops.expand_dims(output, 1)], 1) - new_attn_states = array_ops.reshape( - new_attn_states, [-1, self._attn_length * self._attn_size]) - new_state = (new_state, new_attns, new_attn_states) - if not self._state_is_tuple: - new_state = array_ops.concat(list(new_state), 1) - return output, new_state - - def _attention(self, query, attn_states): - conv2d = nn_ops.conv2d - reduce_sum = math_ops.reduce_sum - softmax = nn_ops.softmax - tanh = math_ops.tanh - - with vs.variable_scope("attention"): - k = vs.get_variable("attn_w", - [1, 1, self._attn_size, self._attn_vec_size]) - v = vs.get_variable("attn_v", [self._attn_vec_size]) - hidden = array_ops.reshape(attn_states, - [-1, self._attn_length, 1, self._attn_size]) - hidden_features = conv2d(hidden, k, [1, 1, 1, 1], "SAME") - if self._linear3 is None: - self._linear3 = _Linear(query, self._attn_vec_size, True) - y = self._linear3(query) - y = array_ops.reshape(y, [-1, 1, 1, self._attn_vec_size]) - s = reduce_sum(v * tanh(hidden_features + y), [2, 3]) - a = softmax(s) - d = reduce_sum( - array_ops.reshape(a, [-1, self._attn_length, 1, 1]) * hidden, [1, 2]) - new_attns = array_ops.reshape(d, [-1, self._attn_size]) - new_attn_states = array_ops.slice(attn_states, [0, 1, 0], [-1, -1, -1]) - return new_attns, new_attn_states - - -class HighwayWrapper(rnn_cell_impl.RNNCell): - """RNNCell wrapper that adds highway connection on cell input and output. - - Based on: - R. K. Srivastava, K. Greff, and J. Schmidhuber, "Highway networks", - arXiv preprint arXiv:1505.00387, 2015. - https://arxiv.org/abs/1505.00387 - """ - - def __init__(self, - cell, - couple_carry_transform_gates=True, - carry_bias_init=1.0): - """Constructs a `HighwayWrapper` for `cell`. - - Args: - cell: An instance of `RNNCell`. - couple_carry_transform_gates: boolean, should the Carry and Transform gate - be coupled. - carry_bias_init: float, carry gates bias initialization. - """ - self._cell = cell - self._couple_carry_transform_gates = couple_carry_transform_gates - self._carry_bias_init = carry_bias_init - - @property - def state_size(self): - return self._cell.state_size - - @property - def output_size(self): - return self._cell.output_size - - def zero_state(self, batch_size, dtype): - with ops.name_scope(type(self).__name__ + "ZeroState", values=[batch_size]): - return self._cell.zero_state(batch_size, dtype) - - def _highway(self, inp, out): - input_size = inp.get_shape().with_rank(2).dims[1].value - carry_weight = vs.get_variable("carry_w", [input_size, input_size]) - carry_bias = vs.get_variable( - "carry_b", [input_size], - initializer=init_ops.constant_initializer(self._carry_bias_init)) - carry = math_ops.sigmoid(nn_ops.xw_plus_b(inp, carry_weight, carry_bias)) - if self._couple_carry_transform_gates: - transform = 1 - carry - else: - transform_weight = vs.get_variable("transform_w", - [input_size, input_size]) - transform_bias = vs.get_variable( - "transform_b", [input_size], - initializer=init_ops.constant_initializer(-self._carry_bias_init)) - transform = math_ops.sigmoid( - nn_ops.xw_plus_b(inp, transform_weight, transform_bias)) - return inp * carry + out * transform - - def __call__(self, inputs, state, scope=None): - """Run the cell and add its inputs to its outputs. - - Args: - inputs: cell inputs. - state: cell state. - scope: optional cell scope. - - Returns: - Tuple of cell outputs and new state. - - Raises: - TypeError: If cell inputs and outputs have different structure (type). - ValueError: If cell inputs and outputs have different structure (value). - """ - outputs, new_state = self._cell(inputs, state, scope=scope) - nest.assert_same_structure(inputs, outputs) - - # Ensure shapes match - def assert_shape_match(inp, out): - inp.get_shape().assert_is_compatible_with(out.get_shape()) - - nest.map_structure(assert_shape_match, inputs, outputs) - res_outputs = nest.map_structure(self._highway, inputs, outputs) - return (res_outputs, new_state) - - -class LayerNormBasicLSTMCell(rnn_cell_impl.RNNCell): - """LSTM unit with layer normalization and recurrent dropout. - - This class adds layer normalization and recurrent dropout to a - basic LSTM unit. Layer normalization implementation is based on: - - https://arxiv.org/abs/1607.06450. - - "Layer Normalization" - Jimmy Lei Ba, Jamie Ryan Kiros, Geoffrey E. Hinton - - and is applied before the internal nonlinearities. - Recurrent dropout is base on: - - https://arxiv.org/abs/1603.05118 - - "Recurrent Dropout without Memory Loss" - Stanislau Semeniuta, Aliaksei Severyn, Erhardt Barth. - """ - - def __init__(self, - num_units, - forget_bias=1.0, - input_size=None, - activation=math_ops.tanh, - layer_norm=True, - norm_gain=1.0, - norm_shift=0.0, - dropout_keep_prob=1.0, - dropout_prob_seed=None, - reuse=None): - """Initializes the basic LSTM cell. - - Args: - num_units: int, The number of units in the LSTM cell. - forget_bias: float, The bias added to forget gates (see above). - input_size: Deprecated and unused. - activation: Activation function of the inner states. - layer_norm: If `True`, layer normalization will be applied. - norm_gain: float, The layer normalization gain initial value. If - `layer_norm` has been set to `False`, this argument will be ignored. - norm_shift: float, The layer normalization shift initial value. If - `layer_norm` has been set to `False`, this argument will be ignored. - dropout_keep_prob: unit Tensor or float between 0 and 1 representing the - recurrent dropout probability value. If float and 1.0, no dropout will - be applied. - dropout_prob_seed: (optional) integer, the randomness seed. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - """ - super(LayerNormBasicLSTMCell, self).__init__(_reuse=reuse) - - if input_size is not None: - logging.warn("%s: The input_size parameter is deprecated.", self) - - self._num_units = num_units - self._activation = activation - self._forget_bias = forget_bias - self._keep_prob = dropout_keep_prob - self._seed = dropout_prob_seed - self._layer_norm = layer_norm - self._norm_gain = norm_gain - self._norm_shift = norm_shift - self._reuse = reuse - - @property - def state_size(self): - return rnn_cell_impl.LSTMStateTuple(self._num_units, self._num_units) - - @property - def output_size(self): - return self._num_units - - def _norm(self, inp, scope, dtype=dtypes.float32): - shape = inp.get_shape()[-1:] - gamma_init = init_ops.constant_initializer(self._norm_gain) - beta_init = init_ops.constant_initializer(self._norm_shift) - with vs.variable_scope(scope): - # Initialize beta and gamma for use by layer_norm. - vs.get_variable("gamma", shape=shape, initializer=gamma_init, dtype=dtype) - vs.get_variable("beta", shape=shape, initializer=beta_init, dtype=dtype) - normalized = layers.layer_norm(inp, reuse=True, scope=scope) - return normalized - - def _linear(self, args): - out_size = 4 * self._num_units - proj_size = args.get_shape()[-1] - dtype = args.dtype - weights = vs.get_variable("kernel", [proj_size, out_size], dtype=dtype) - out = math_ops.matmul(args, weights) - if not self._layer_norm: - bias = vs.get_variable("bias", [out_size], dtype=dtype) - out = nn_ops.bias_add(out, bias) - return out - - def call(self, inputs, state): - """LSTM cell with layer normalization and recurrent dropout.""" - c, h = state - args = array_ops.concat([inputs, h], 1) - concat = self._linear(args) - dtype = args.dtype - - i, j, f, o = array_ops.split(value=concat, num_or_size_splits=4, axis=1) - if self._layer_norm: - i = self._norm(i, "input", dtype=dtype) - j = self._norm(j, "transform", dtype=dtype) - f = self._norm(f, "forget", dtype=dtype) - o = self._norm(o, "output", dtype=dtype) - - g = self._activation(j) - if (not isinstance(self._keep_prob, float)) or self._keep_prob < 1: - g = nn_ops.dropout(g, self._keep_prob, seed=self._seed) - - new_c = ( - c * math_ops.sigmoid(f + self._forget_bias) + math_ops.sigmoid(i) * g) - if self._layer_norm: - new_c = self._norm(new_c, "state", dtype=dtype) - new_h = self._activation(new_c) * math_ops.sigmoid(o) - - new_state = rnn_cell_impl.LSTMStateTuple(new_c, new_h) - return new_h, new_state - - -class NASCell(rnn_cell_impl.LayerRNNCell): - """Neural Architecture Search (NAS) recurrent network cell. - - This implements the recurrent cell from the paper: - - https://arxiv.org/abs/1611.01578 - - Barret Zoph and Quoc V. Le. - "Neural Architecture Search with Reinforcement Learning" Proc. ICLR 2017. - - The class uses an optional projection layer. - """ - - # NAS cell's architecture base. - _NAS_BASE = 8 - - def __init__(self, num_units, num_proj=None, use_bias=False, reuse=None, - **kwargs): - """Initialize the parameters for a NAS cell. - - Args: - num_units: int, The number of units in the NAS cell. - num_proj: (optional) int, The output dimensionality for the projection - matrices. If None, no projection is performed. - use_bias: (optional) bool, If True then use biases within the cell. This - is False by default. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - **kwargs: Additional keyword arguments. - """ - super(NASCell, self).__init__(_reuse=reuse, **kwargs) - self._num_units = num_units - self._num_proj = num_proj - self._use_bias = use_bias - self._reuse = reuse - - if num_proj is not None: - self._state_size = rnn_cell_impl.LSTMStateTuple(num_units, num_proj) - self._output_size = num_proj - else: - self._state_size = rnn_cell_impl.LSTMStateTuple(num_units, num_units) - self._output_size = num_units - - @property - def state_size(self): - return self._state_size - - @property - def output_size(self): - return self._output_size - - def build(self, inputs_shape): - input_size = tensor_shape.dimension_value( - tensor_shape.TensorShape(inputs_shape).with_rank(2)[1]) - if input_size is None: - raise ValueError("Could not infer input size from inputs.get_shape()[-1]") - - num_proj = self._num_units if self._num_proj is None else self._num_proj - - # Variables for the NAS cell. `recurrent_kernel` is all matrices multiplying - # the hiddenstate and `kernel` is all matrices multiplying the inputs. - self.recurrent_kernel = self.add_variable( - "recurrent_kernel", [num_proj, self._NAS_BASE * self._num_units]) - self.kernel = self.add_variable( - "kernel", [input_size, self._NAS_BASE * self._num_units]) - - if self._use_bias: - self.bias = self.add_variable("bias", - shape=[self._NAS_BASE * self._num_units], - initializer=init_ops.zeros_initializer) - - # Projection layer if specified - if self._num_proj is not None: - self.projection_weights = self.add_variable( - "projection_weights", [self._num_units, self._num_proj]) - - self.built = True - - def call(self, inputs, state): - """Run one step of NAS Cell. - - Args: - inputs: input Tensor, 2D, batch x num_units. - state: This must be a tuple of state Tensors, both `2-D`, with column - sizes `c_state` and `m_state`. - - Returns: - A tuple containing: - - A `2-D, [batch x output_dim]`, Tensor representing the output of the - NAS Cell after reading `inputs` when previous state was `state`. - Here output_dim is: - num_proj if num_proj was set, - num_units otherwise. - - Tensor(s) representing the new state of NAS Cell after reading `inputs` - when the previous state was `state`. Same type and shape(s) as `state`. - - Raises: - ValueError: If input size cannot be inferred from inputs via - static shape inference. - """ - sigmoid = math_ops.sigmoid - tanh = math_ops.tanh - relu = nn_ops.relu - - (c_prev, m_prev) = state - - m_matrix = math_ops.matmul(m_prev, self.recurrent_kernel) - inputs_matrix = math_ops.matmul(inputs, self.kernel) - - if self._use_bias: - m_matrix = nn_ops.bias_add(m_matrix, self.bias) - - # The NAS cell branches into 8 different splits for both the hiddenstate - # and the input - m_matrix_splits = array_ops.split( - axis=1, num_or_size_splits=self._NAS_BASE, value=m_matrix) - inputs_matrix_splits = array_ops.split( - axis=1, num_or_size_splits=self._NAS_BASE, value=inputs_matrix) - - # First layer - layer1_0 = sigmoid(inputs_matrix_splits[0] + m_matrix_splits[0]) - layer1_1 = relu(inputs_matrix_splits[1] + m_matrix_splits[1]) - layer1_2 = sigmoid(inputs_matrix_splits[2] + m_matrix_splits[2]) - layer1_3 = relu(inputs_matrix_splits[3] * m_matrix_splits[3]) - layer1_4 = tanh(inputs_matrix_splits[4] + m_matrix_splits[4]) - layer1_5 = sigmoid(inputs_matrix_splits[5] + m_matrix_splits[5]) - layer1_6 = tanh(inputs_matrix_splits[6] + m_matrix_splits[6]) - layer1_7 = sigmoid(inputs_matrix_splits[7] + m_matrix_splits[7]) - - # Second layer - l2_0 = tanh(layer1_0 * layer1_1) - l2_1 = tanh(layer1_2 + layer1_3) - l2_2 = tanh(layer1_4 * layer1_5) - l2_3 = sigmoid(layer1_6 + layer1_7) - - # Inject the cell - l2_0 = tanh(l2_0 + c_prev) - - # Third layer - l3_0_pre = l2_0 * l2_1 - new_c = l3_0_pre # create new cell - l3_0 = l3_0_pre - l3_1 = tanh(l2_2 + l2_3) - - # Final layer - new_m = tanh(l3_0 * l3_1) - - # Projection layer if specified - if self._num_proj is not None: - new_m = math_ops.matmul(new_m, self.projection_weights) - - new_state = rnn_cell_impl.LSTMStateTuple(new_c, new_m) - return new_m, new_state - - -class UGRNNCell(rnn_cell_impl.RNNCell): - """Update Gate Recurrent Neural Network (UGRNN) cell. - - Compromise between a LSTM/GRU and a vanilla RNN. There is only one - gate, and that is to determine whether the unit should be - integrating or computing instantaneously. This is the recurrent - idea of the feedforward Highway Network. - - This implements the recurrent cell from the paper: - - https://arxiv.org/abs/1611.09913 - - Jasmine Collins, Jascha Sohl-Dickstein, and David Sussillo. - "Capacity and Trainability in Recurrent Neural Networks" Proc. ICLR 2017. - """ - - def __init__(self, - num_units, - initializer=None, - forget_bias=1.0, - activation=math_ops.tanh, - reuse=None): - """Initialize the parameters for an UGRNN cell. - - Args: - num_units: int, The number of units in the UGRNN cell - initializer: (optional) The initializer to use for the weight matrices. - forget_bias: (optional) float, default 1.0, The initial bias of the - forget gate, used to reduce the scale of forgetting at the beginning - of the training. - activation: (optional) Activation function of the inner states. - Default is `tf.tanh`. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - """ - super(UGRNNCell, self).__init__(_reuse=reuse) - self._num_units = num_units - self._initializer = initializer - self._forget_bias = forget_bias - self._activation = activation - self._reuse = reuse - self._linear = None - - @property - def state_size(self): - return self._num_units - - @property - def output_size(self): - return self._num_units - - def call(self, inputs, state): - """Run one step of UGRNN. - - Args: - inputs: input Tensor, 2D, batch x input size. - state: state Tensor, 2D, batch x num units. - - Returns: - new_output: batch x num units, Tensor representing the output of the UGRNN - after reading `inputs` when previous state was `state`. Identical to - `new_state`. - new_state: batch x num units, Tensor representing the state of the UGRNN - after reading `inputs` when previous state was `state`. - - Raises: - ValueError: If input size cannot be inferred from inputs via - static shape inference. - """ - sigmoid = math_ops.sigmoid - - input_size = inputs.get_shape().with_rank(2).dims[1] - if input_size.value is None: - raise ValueError("Could not infer input size from inputs.get_shape()[-1]") - - with vs.variable_scope( - vs.get_variable_scope(), initializer=self._initializer): - cell_inputs = array_ops.concat([inputs, state], 1) - if self._linear is None: - self._linear = _Linear(cell_inputs, 2 * self._num_units, True) - rnn_matrix = self._linear(cell_inputs) - - [g_act, c_act] = array_ops.split( - axis=1, num_or_size_splits=2, value=rnn_matrix) - - c = self._activation(c_act) - g = sigmoid(g_act + self._forget_bias) - new_state = g * state + (1.0 - g) * c - new_output = new_state - - return new_output, new_state - - -class IntersectionRNNCell(rnn_cell_impl.RNNCell): - """Intersection Recurrent Neural Network (+RNN) cell. - - Architecture with coupled recurrent gate as well as coupled depth - gate, designed to improve information flow through stacked RNNs. As the - architecture uses depth gating, the dimensionality of the depth - output (y) also should not change through depth (input size == output size). - To achieve this, the first layer of a stacked Intersection RNN projects - the inputs to N (num units) dimensions. Therefore when initializing an - IntersectionRNNCell, one should set `num_in_proj = N` for the first layer - and use default settings for subsequent layers. - - This implements the recurrent cell from the paper: - - https://arxiv.org/abs/1611.09913 - - Jasmine Collins, Jascha Sohl-Dickstein, and David Sussillo. - "Capacity and Trainability in Recurrent Neural Networks" Proc. ICLR 2017. - - The Intersection RNN is built for use in deeply stacked - RNNs so it may not achieve best performance with depth 1. - """ - - def __init__(self, - num_units, - num_in_proj=None, - initializer=None, - forget_bias=1.0, - y_activation=nn_ops.relu, - reuse=None): - """Initialize the parameters for an +RNN cell. - - Args: - num_units: int, The number of units in the +RNN cell - num_in_proj: (optional) int, The input dimensionality for the RNN. - If creating the first layer of an +RNN, this should be set to - `num_units`. Otherwise, this should be set to `None` (default). - If `None`, dimensionality of `inputs` should be equal to `num_units`, - otherwise ValueError is thrown. - initializer: (optional) The initializer to use for the weight matrices. - forget_bias: (optional) float, default 1.0, The initial bias of the - forget gates, used to reduce the scale of forgetting at the beginning - of the training. - y_activation: (optional) Activation function of the states passed - through depth. Default is 'tf.nn.relu`. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - """ - super(IntersectionRNNCell, self).__init__(_reuse=reuse) - self._num_units = num_units - self._initializer = initializer - self._forget_bias = forget_bias - self._num_input_proj = num_in_proj - self._y_activation = y_activation - self._reuse = reuse - self._linear1 = None - self._linear2 = None - - @property - def state_size(self): - return self._num_units - - @property - def output_size(self): - return self._num_units - - def call(self, inputs, state): - """Run one step of the Intersection RNN. - - Args: - inputs: input Tensor, 2D, batch x input size. - state: state Tensor, 2D, batch x num units. - - Returns: - new_y: batch x num units, Tensor representing the output of the +RNN - after reading `inputs` when previous state was `state`. - new_state: batch x num units, Tensor representing the state of the +RNN - after reading `inputs` when previous state was `state`. - - Raises: - ValueError: If input size cannot be inferred from `inputs` via - static shape inference. - ValueError: If input size != output size (these must be equal when - using the Intersection RNN). - """ - sigmoid = math_ops.sigmoid - tanh = math_ops.tanh - - input_size = inputs.get_shape().with_rank(2).dims[1] - if input_size.value is None: - raise ValueError("Could not infer input size from inputs.get_shape()[-1]") - - with vs.variable_scope( - vs.get_variable_scope(), initializer=self._initializer): - # read-in projections (should be used for first layer in deep +RNN - # to transform size of inputs from I --> N) - if input_size.value != self._num_units: - if self._num_input_proj: - with vs.variable_scope("in_projection"): - if self._linear1 is None: - self._linear1 = _Linear(inputs, self._num_units, True) - inputs = self._linear1(inputs) - else: - raise ValueError("Must have input size == output size for " - "Intersection RNN. To fix, num_in_proj should " - "be set to num_units at cell init.") - - n_dim = i_dim = self._num_units - cell_inputs = array_ops.concat([inputs, state], 1) - if self._linear2 is None: - self._linear2 = _Linear(cell_inputs, 2 * n_dim + 2 * i_dim, True) - rnn_matrix = self._linear2(cell_inputs) - - gh_act = rnn_matrix[:, :n_dim] # b x n - h_act = rnn_matrix[:, n_dim:2 * n_dim] # b x n - gy_act = rnn_matrix[:, 2 * n_dim:2 * n_dim + i_dim] # b x i - y_act = rnn_matrix[:, 2 * n_dim + i_dim:2 * n_dim + 2 * i_dim] # b x i - - h = tanh(h_act) - y = self._y_activation(y_act) - gh = sigmoid(gh_act + self._forget_bias) - gy = sigmoid(gy_act + self._forget_bias) - - new_state = gh * state + (1.0 - gh) * h # passed thru time - new_y = gy * inputs + (1.0 - gy) * y # passed thru depth - - return new_y, new_state - - -_REGISTERED_OPS = None - - -class CompiledWrapper(rnn_cell_impl.RNNCell): - """Wraps step execution in an XLA JIT scope.""" - - def __init__(self, cell, compile_stateful=False): - """Create CompiledWrapper cell. - - Args: - cell: Instance of `RNNCell`. - compile_stateful: Whether to compile stateful ops like initializers - and random number generators (default: False). - """ - self._cell = cell - self._compile_stateful = compile_stateful - - @property - def state_size(self): - return self._cell.state_size - - @property - def output_size(self): - return self._cell.output_size - - def zero_state(self, batch_size, dtype): - with ops.name_scope(type(self).__name__ + "ZeroState", values=[batch_size]): - return self._cell.zero_state(batch_size, dtype) - - def __call__(self, inputs, state, scope=None): - if self._compile_stateful: - compile_ops = True - else: - - def compile_ops(node_def): - global _REGISTERED_OPS - if _REGISTERED_OPS is None: - _REGISTERED_OPS = op_def_registry.get_registered_ops() - return not _REGISTERED_OPS[node_def.op].is_stateful - - with jit.experimental_jit_scope(compile_ops=compile_ops): - return self._cell(inputs, state, scope=scope) - - -def _random_exp_initializer(minval, maxval, seed=None, dtype=dtypes.float32): - """Returns an exponential distribution initializer. - - Args: - minval: float or a scalar float Tensor. With value > 0. Lower bound of the - range of random values to generate. - maxval: float or a scalar float Tensor. With value > minval. Upper bound of - the range of random values to generate. - seed: An integer. Used to create random seeds. - dtype: The data type. - - Returns: - An initializer that generates tensors with an exponential distribution. - """ - - def _initializer(shape, dtype=dtype, partition_info=None): - del partition_info # Unused. - return math_ops.exp( - random_ops.random_uniform( - shape, math_ops.log(minval), math_ops.log(maxval), dtype, - seed=seed)) - - return _initializer - - -class PhasedLSTMCell(rnn_cell_impl.RNNCell): - """Phased LSTM recurrent network cell. - - https://arxiv.org/pdf/1610.09513v1.pdf - """ - - def __init__(self, - num_units, - use_peepholes=False, - leak=0.001, - ratio_on=0.1, - trainable_ratio_on=True, - period_init_min=1.0, - period_init_max=1000.0, - reuse=None): - """Initialize the Phased LSTM cell. - - Args: - num_units: int, The number of units in the Phased LSTM cell. - use_peepholes: bool, set True to enable peephole connections. - leak: float or scalar float Tensor with value in [0, 1]. Leak applied - during training. - ratio_on: float or scalar float Tensor with value in [0, 1]. Ratio of the - period during which the gates are open. - trainable_ratio_on: bool, weather ratio_on is trainable. - period_init_min: float or scalar float Tensor. With value > 0. - Minimum value of the initialized period. - The period values are initialized by drawing from the distribution: - e^U(log(period_init_min), log(period_init_max)) - Where U(.,.) is the uniform distribution. - period_init_max: float or scalar float Tensor. - With value > period_init_min. Maximum value of the initialized period. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - """ - # We pass autocast=False because this layer can accept inputs of different - # dtypes, so we do not want to automatically cast them to the same dtype. - super(PhasedLSTMCell, self).__init__(_reuse=reuse, autocast=False) - self._num_units = num_units - self._use_peepholes = use_peepholes - self._leak = leak - self._ratio_on = ratio_on - self._trainable_ratio_on = trainable_ratio_on - self._period_init_min = period_init_min - self._period_init_max = period_init_max - self._reuse = reuse - self._linear1 = None - self._linear2 = None - self._linear3 = None - - @property - def state_size(self): - return rnn_cell_impl.LSTMStateTuple(self._num_units, self._num_units) - - @property - def output_size(self): - return self._num_units - - def _mod(self, x, y): - """Modulo function that propagates x gradients.""" - return array_ops.stop_gradient(math_ops.mod(x, y) - x) + x - - def _get_cycle_ratio(self, time, phase, period): - """Compute the cycle ratio in the dtype of the time.""" - phase_casted = math_ops.cast(phase, dtype=time.dtype) - period_casted = math_ops.cast(period, dtype=time.dtype) - shifted_time = time - phase_casted - cycle_ratio = self._mod(shifted_time, period_casted) / period_casted - return math_ops.cast(cycle_ratio, dtype=dtypes.float32) - - def call(self, inputs, state): - """Phased LSTM Cell. - - Args: - inputs: A tuple of 2 Tensor. - The first Tensor has shape [batch, 1], and type float32 or float64. - It stores the time. - The second Tensor has shape [batch, features_size], and type float32. - It stores the features. - state: rnn_cell_impl.LSTMStateTuple, state from previous timestep. - - Returns: - A tuple containing: - - A Tensor of float32, and shape [batch_size, num_units], representing the - output of the cell. - - A rnn_cell_impl.LSTMStateTuple, containing 2 Tensors of float32, shape - [batch_size, num_units], representing the new state and the output. - """ - (c_prev, h_prev) = state - (time, x) = inputs - - in_mask_gates = [x, h_prev] - if self._use_peepholes: - in_mask_gates.append(c_prev) - - with vs.variable_scope("mask_gates"): - if self._linear1 is None: - self._linear1 = _Linear(in_mask_gates, 2 * self._num_units, True) - - mask_gates = math_ops.sigmoid(self._linear1(in_mask_gates)) - [input_gate, forget_gate] = array_ops.split( - axis=1, num_or_size_splits=2, value=mask_gates) - - with vs.variable_scope("new_input"): - if self._linear2 is None: - self._linear2 = _Linear([x, h_prev], self._num_units, True) - new_input = math_ops.tanh(self._linear2([x, h_prev])) - - new_c = (c_prev * forget_gate + input_gate * new_input) - - in_out_gate = [x, h_prev] - if self._use_peepholes: - in_out_gate.append(new_c) - - with vs.variable_scope("output_gate"): - if self._linear3 is None: - self._linear3 = _Linear(in_out_gate, self._num_units, True) - output_gate = math_ops.sigmoid(self._linear3(in_out_gate)) - - new_h = math_ops.tanh(new_c) * output_gate - - period = vs.get_variable( - "period", [self._num_units], - initializer=_random_exp_initializer(self._period_init_min, - self._period_init_max)) - phase = vs.get_variable( - "phase", [self._num_units], - initializer=init_ops.random_uniform_initializer(0., - period.initial_value)) - ratio_on = vs.get_variable( - "ratio_on", [self._num_units], - initializer=init_ops.constant_initializer(self._ratio_on), - trainable=self._trainable_ratio_on) - - cycle_ratio = self._get_cycle_ratio(time, phase, period) - - k_up = 2 * cycle_ratio / ratio_on - k_down = 2 - k_up - k_closed = self._leak * cycle_ratio - - k = array_ops.where(cycle_ratio < ratio_on, k_down, k_closed) - k = array_ops.where(cycle_ratio < 0.5 * ratio_on, k_up, k) - - new_c = k * new_c + (1 - k) * c_prev - new_h = k * new_h + (1 - k) * h_prev - - new_state = rnn_cell_impl.LSTMStateTuple(new_c, new_h) - - return new_h, new_state - - -class ConvLSTMCell(rnn_cell_impl.RNNCell): - """Convolutional LSTM recurrent network cell. - - https://arxiv.org/pdf/1506.04214v1.pdf - """ - - def __init__(self, - conv_ndims, - input_shape, - output_channels, - kernel_shape, - use_bias=True, - skip_connection=False, - forget_bias=1.0, - initializers=None, - name="conv_lstm_cell"): - """Construct ConvLSTMCell. - - Args: - conv_ndims: Convolution dimensionality (1, 2 or 3). - input_shape: Shape of the input as int tuple, excluding the batch size. - output_channels: int, number of output channels of the conv LSTM. - kernel_shape: Shape of kernel as an int tuple (of size 1, 2 or 3). - use_bias: (bool) Use bias in convolutions. - skip_connection: If set to `True`, concatenate the input to the - output of the conv LSTM. Default: `False`. - forget_bias: Forget bias. - initializers: Unused. - name: Name of the module. - - Raises: - ValueError: If `skip_connection` is `True` and stride is different from 1 - or if `input_shape` is incompatible with `conv_ndims`. - """ - super(ConvLSTMCell, self).__init__(name=name) - - if conv_ndims != len(input_shape) - 1: - raise ValueError("Invalid input_shape {} for conv_ndims={}.".format( - input_shape, conv_ndims)) - - self._conv_ndims = conv_ndims - self._input_shape = input_shape - self._output_channels = output_channels - self._kernel_shape = list(kernel_shape) - self._use_bias = use_bias - self._forget_bias = forget_bias - self._skip_connection = skip_connection - - self._total_output_channels = output_channels - if self._skip_connection: - self._total_output_channels += self._input_shape[-1] - - state_size = tensor_shape.TensorShape( - self._input_shape[:-1] + [self._output_channels]) - self._state_size = rnn_cell_impl.LSTMStateTuple(state_size, state_size) - self._output_size = tensor_shape.TensorShape( - self._input_shape[:-1] + [self._total_output_channels]) - - @property - def output_size(self): - return self._output_size - - @property - def state_size(self): - return self._state_size - - def call(self, inputs, state, scope=None): - cell, hidden = state - new_hidden = _conv([inputs, hidden], self._kernel_shape, - 4 * self._output_channels, self._use_bias) - gates = array_ops.split( - value=new_hidden, num_or_size_splits=4, axis=self._conv_ndims + 1) - - input_gate, new_input, forget_gate, output_gate = gates - new_cell = math_ops.sigmoid(forget_gate + self._forget_bias) * cell - new_cell += math_ops.sigmoid(input_gate) * math_ops.tanh(new_input) - output = math_ops.tanh(new_cell) * math_ops.sigmoid(output_gate) - - if self._skip_connection: - output = array_ops.concat([output, inputs], axis=-1) - new_state = rnn_cell_impl.LSTMStateTuple(new_cell, output) - return output, new_state - - -class Conv1DLSTMCell(ConvLSTMCell): - """1D Convolutional LSTM recurrent network cell. - - https://arxiv.org/pdf/1506.04214v1.pdf - """ - - def __init__(self, name="conv_1d_lstm_cell", **kwargs): - """Construct Conv1DLSTM. See `ConvLSTMCell` for more details.""" - super(Conv1DLSTMCell, self).__init__(conv_ndims=1, name=name, **kwargs) - - -class Conv2DLSTMCell(ConvLSTMCell): - """2D Convolutional LSTM recurrent network cell. - - https://arxiv.org/pdf/1506.04214v1.pdf - """ - - def __init__(self, name="conv_2d_lstm_cell", **kwargs): - """Construct Conv2DLSTM. See `ConvLSTMCell` for more details.""" - super(Conv2DLSTMCell, self).__init__(conv_ndims=2, name=name, **kwargs) - - -class Conv3DLSTMCell(ConvLSTMCell): - """3D Convolutional LSTM recurrent network cell. - - https://arxiv.org/pdf/1506.04214v1.pdf - """ - - def __init__(self, name="conv_3d_lstm_cell", **kwargs): - """Construct Conv3DLSTM. See `ConvLSTMCell` for more details.""" - super(Conv3DLSTMCell, self).__init__(conv_ndims=3, name=name, **kwargs) - - -def _conv(args, filter_size, num_features, bias, bias_start=0.0): - """Convolution. - - Args: - args: a Tensor or a list of Tensors of dimension 3D, 4D or 5D, - batch x n, Tensors. - filter_size: int tuple of filter shape (of size 1, 2 or 3). - num_features: int, number of features. - bias: Whether to use biases in the convolution layer. - bias_start: starting value to initialize the bias; 0 by default. - - Returns: - A 3D, 4D, or 5D Tensor with shape [batch ... num_features] - - Raises: - ValueError: if some of the arguments has unspecified or wrong shape. - """ - - # Calculate the total size of arguments on dimension 1. - total_arg_size_depth = 0 - shapes = [a.get_shape().as_list() for a in args] - shape_length = len(shapes[0]) - for shape in shapes: - if len(shape) not in [3, 4, 5]: - raise ValueError("Conv Linear expects 3D, 4D " - "or 5D arguments: %s" % str(shapes)) - if len(shape) != len(shapes[0]): - raise ValueError("Conv Linear expects all args " - "to be of same Dimension: %s" % str(shapes)) - else: - total_arg_size_depth += shape[-1] - dtype = [a.dtype for a in args][0] - - # determine correct conv operation - if shape_length == 3: - conv_op = nn_ops.conv1d - strides = 1 - elif shape_length == 4: - conv_op = nn_ops.conv2d - strides = shape_length * [1] - elif shape_length == 5: - conv_op = nn_ops.conv3d - strides = shape_length * [1] - - # Now the computation. - kernel = vs.get_variable( - "kernel", filter_size + [total_arg_size_depth, num_features], dtype=dtype) - if len(args) == 1: - res = conv_op(args[0], kernel, strides, padding="SAME") - else: - res = conv_op( - array_ops.concat(axis=shape_length - 1, values=args), - kernel, - strides, - padding="SAME") - if not bias: - return res - bias_term = vs.get_variable( - "biases", [num_features], - dtype=dtype, - initializer=init_ops.constant_initializer(bias_start, dtype=dtype)) - return res + bias_term - - -class GLSTMCell(rnn_cell_impl.RNNCell): - """Group LSTM cell (G-LSTM). - - The implementation is based on: - - https://arxiv.org/abs/1703.10722 - - O. Kuchaiev and B. Ginsburg - "Factorization Tricks for LSTM Networks", ICLR 2017 workshop. - - In brief, a G-LSTM cell consists of one LSTM sub-cell per group, where each - sub-cell operates on an evenly-sized sub-vector of the input and produces an - evenly-sized sub-vector of the output. For example, a G-LSTM cell with 128 - units and 4 groups consists of 4 LSTMs sub-cells with 32 units each. If that - G-LSTM cell is fed a 200-dim input, then each sub-cell receives a 50-dim part - of the input and produces a 32-dim part of the output. - """ - - def __init__(self, - num_units, - initializer=None, - num_proj=None, - number_of_groups=1, - forget_bias=1.0, - activation=math_ops.tanh, - reuse=None): - """Initialize the parameters of G-LSTM cell. - - Args: - num_units: int, The number of units in the G-LSTM cell - initializer: (optional) The initializer to use for the weight and - projection matrices. - num_proj: (optional) int, The output dimensionality for the projection - matrices. If None, no projection is performed. - number_of_groups: (optional) int, number of groups to use. - If `number_of_groups` is 1, then it should be equivalent to LSTM cell - forget_bias: Biases of the forget gate are initialized by default to 1 - in order to reduce the scale of forgetting at the beginning of - the training. - activation: Activation function of the inner states. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already - has the given variables, an error is raised. - - Raises: - ValueError: If `num_units` or `num_proj` is not divisible by - `number_of_groups`. - """ - super(GLSTMCell, self).__init__(_reuse=reuse) - self._num_units = num_units - self._initializer = initializer - self._num_proj = num_proj - self._forget_bias = forget_bias - self._activation = activation - self._number_of_groups = number_of_groups - - if self._num_units % self._number_of_groups != 0: - raise ValueError("num_units must be divisible by number_of_groups") - if self._num_proj: - if self._num_proj % self._number_of_groups != 0: - raise ValueError("num_proj must be divisible by number_of_groups") - self._group_shape = [ - int(self._num_proj / self._number_of_groups), - int(self._num_units / self._number_of_groups) - ] - else: - self._group_shape = [ - int(self._num_units / self._number_of_groups), - int(self._num_units / self._number_of_groups) - ] - - if num_proj: - self._state_size = rnn_cell_impl.LSTMStateTuple(num_units, num_proj) - self._output_size = num_proj - else: - self._state_size = rnn_cell_impl.LSTMStateTuple(num_units, num_units) - self._output_size = num_units - self._linear1 = [None] * number_of_groups - self._linear2 = None - - @property - def state_size(self): - return self._state_size - - @property - def output_size(self): - return self._output_size - - def _get_input_for_group(self, inputs, group_id, group_size): - """Slices inputs into groups to prepare for processing by cell's groups. - - Args: - inputs: cell input or it's previous state, - a Tensor, 2D, [batch x num_units] - group_id: group id, a Scalar, for which to prepare input - group_size: size of the group - - Returns: - subset of inputs corresponding to group "group_id", - a Tensor, 2D, [batch x num_units/number_of_groups] - """ - return array_ops.slice( - input_=inputs, - begin=[0, group_id * group_size], - size=[self._batch_size, group_size], - name=("GLSTM_group%d_input_generation" % group_id)) - - def call(self, inputs, state): - """Run one step of G-LSTM. - - Args: - inputs: input Tensor, 2D, [batch x num_inputs]. num_inputs must be - statically-known and evenly divisible into groups. The innermost - vectors of the inputs are split into evenly-sized sub-vectors and fed - into the per-group LSTM sub-cells. - state: this must be a tuple of state Tensors, both `2-D`, with column - sizes `c_state` and `m_state`. - - Returns: - A tuple containing: - - - A `2-D, [batch x output_dim]`, Tensor representing the output of the - G-LSTM after reading `inputs` when previous state was `state`. - Here output_dim is: - num_proj if num_proj was set, - num_units otherwise. - - LSTMStateTuple representing the new state of G-LSTM cell - after reading `inputs` when the previous state was `state`. - - Raises: - ValueError: If input size cannot be inferred from inputs via - static shape inference, or if the input shape is incompatible - with the number of groups. - """ - (c_prev, m_prev) = state - - self._batch_size = tensor_shape.dimension_value( - inputs.shape[0]) or array_ops.shape(inputs)[0] - - # If the input size is statically-known, calculate and validate its group - # size. Otherwise, use the output group size. - input_size = tensor_shape.dimension_value(inputs.shape[1]) - if input_size is None: - raise ValueError("input size must be statically known") - if input_size % self._number_of_groups != 0: - raise ValueError( - "input size (%d) must be divisible by number_of_groups (%d)" % - (input_size, self._number_of_groups)) - input_group_size = int(input_size / self._number_of_groups) - - dtype = inputs.dtype - scope = vs.get_variable_scope() - with vs.variable_scope(scope, initializer=self._initializer): - i_parts = [] - j_parts = [] - f_parts = [] - o_parts = [] - - for group_id in range(self._number_of_groups): - with vs.variable_scope("group%d" % group_id): - x_g_id = array_ops.concat( - [ - self._get_input_for_group(inputs, group_id, input_group_size), - self._get_input_for_group(m_prev, group_id, - self._group_shape[0]) - ], - axis=1) - linear = self._linear1[group_id] - if linear is None: - linear = _Linear(x_g_id, 4 * self._group_shape[1], False) - self._linear1[group_id] = linear - R_k = linear(x_g_id) # pylint: disable=invalid-name - i_k, j_k, f_k, o_k = array_ops.split(R_k, 4, 1) - - i_parts.append(i_k) - j_parts.append(j_k) - f_parts.append(f_k) - o_parts.append(o_k) - - bi = vs.get_variable( - name="bias_i", - shape=[self._num_units], - dtype=dtype, - initializer=init_ops.constant_initializer(0.0, dtype=dtype)) - bj = vs.get_variable( - name="bias_j", - shape=[self._num_units], - dtype=dtype, - initializer=init_ops.constant_initializer(0.0, dtype=dtype)) - bf = vs.get_variable( - name="bias_f", - shape=[self._num_units], - dtype=dtype, - initializer=init_ops.constant_initializer(0.0, dtype=dtype)) - bo = vs.get_variable( - name="bias_o", - shape=[self._num_units], - dtype=dtype, - initializer=init_ops.constant_initializer(0.0, dtype=dtype)) - - i = nn_ops.bias_add(array_ops.concat(i_parts, axis=1), bi) - j = nn_ops.bias_add(array_ops.concat(j_parts, axis=1), bj) - f = nn_ops.bias_add(array_ops.concat(f_parts, axis=1), bf) - o = nn_ops.bias_add(array_ops.concat(o_parts, axis=1), bo) - - c = ( - math_ops.sigmoid(f + self._forget_bias) * c_prev + - math_ops.sigmoid(i) * math_ops.tanh(j)) - m = math_ops.sigmoid(o) * self._activation(c) - - if self._num_proj is not None: - with vs.variable_scope("projection"): - if self._linear2 is None: - self._linear2 = _Linear(m, self._num_proj, False) - m = self._linear2(m) - - new_state = rnn_cell_impl.LSTMStateTuple(c, m) - return m, new_state - - -class LayerNormLSTMCell(rnn_cell_impl.RNNCell): - """Long short-term memory unit (LSTM) recurrent network cell. - - The default non-peephole implementation is based on: - - https://pdfs.semanticscholar.org/1154/0131eae85b2e11d53df7f1360eeb6476e7f4.pdf - - Felix Gers, Jurgen Schmidhuber, and Fred Cummins. - "Learning to forget: Continual prediction with LSTM." IET, 850-855, 1999. - - The peephole implementation is based on: - - https://research.google.com/pubs/archive/43905.pdf - - Hasim Sak, Andrew Senior, and Francoise Beaufays. - "Long short-term memory recurrent neural network architectures for - large scale acoustic modeling." INTERSPEECH, 2014. - - The class uses optional peep-hole connections, optional cell clipping, and - an optional projection layer. - - Layer normalization implementation is based on: - - https://arxiv.org/abs/1607.06450. - - "Layer Normalization" - Jimmy Lei Ba, Jamie Ryan Kiros, Geoffrey E. Hinton - - and is applied before the internal nonlinearities. - - """ - - def __init__(self, - num_units, - use_peepholes=False, - cell_clip=None, - initializer=None, - num_proj=None, - proj_clip=None, - forget_bias=1.0, - activation=None, - layer_norm=False, - norm_gain=1.0, - norm_shift=0.0, - reuse=None): - """Initialize the parameters for an LSTM cell. - - Args: - num_units: int, The number of units in the LSTM cell - use_peepholes: bool, set True to enable diagonal/peephole connections. - cell_clip: (optional) A float value, if provided the cell state is clipped - by this value prior to the cell output activation. - initializer: (optional) The initializer to use for the weight and - projection matrices. - num_proj: (optional) int, The output dimensionality for the projection - matrices. If None, no projection is performed. - proj_clip: (optional) A float value. If `num_proj > 0` and `proj_clip` is - provided, then the projected values are clipped elementwise to within - `[-proj_clip, proj_clip]`. - forget_bias: Biases of the forget gate are initialized by default to 1 - in order to reduce the scale of forgetting at the beginning of - the training. Must set it manually to `0.0` when restoring from - CudnnLSTM trained checkpoints. - activation: Activation function of the inner states. Default: `tanh`. - layer_norm: If `True`, layer normalization will be applied. - norm_gain: float, The layer normalization gain initial value. If - `layer_norm` has been set to `False`, this argument will be ignored. - norm_shift: float, The layer normalization shift initial value. If - `layer_norm` has been set to `False`, this argument will be ignored. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - - When restoring from CudnnLSTM-trained checkpoints, must use - CudnnCompatibleLSTMCell instead. - """ - super(LayerNormLSTMCell, self).__init__(_reuse=reuse) - - self._num_units = num_units - self._use_peepholes = use_peepholes - self._cell_clip = cell_clip - self._initializer = initializer - self._num_proj = num_proj - self._proj_clip = proj_clip - self._forget_bias = forget_bias - self._activation = activation or math_ops.tanh - self._layer_norm = layer_norm - self._norm_gain = norm_gain - self._norm_shift = norm_shift - - if num_proj: - self._state_size = (rnn_cell_impl.LSTMStateTuple(num_units, num_proj)) - self._output_size = num_proj - else: - self._state_size = (rnn_cell_impl.LSTMStateTuple(num_units, num_units)) - self._output_size = num_units - - @property - def state_size(self): - return self._state_size - - @property - def output_size(self): - return self._output_size - - def _linear(self, - args, - output_size, - bias, - bias_initializer=None, - kernel_initializer=None, - layer_norm=False): - """Linear map: sum_i(args[i] * W[i]), where W[i] is a Variable. - - Args: - args: a 2D Tensor or a list of 2D, batch x n, Tensors. - output_size: int, second dimension of W[i]. - bias: boolean, whether to add a bias term or not. - bias_initializer: starting value to initialize the bias - (default is all zeros). - kernel_initializer: starting value to initialize the weight. - layer_norm: boolean, whether to apply layer normalization. - - - Returns: - A 2D Tensor with shape [batch x output_size] taking value - sum_i(args[i] * W[i]), where each W[i] is a newly created Variable. - - Raises: - ValueError: if some of the arguments has unspecified or wrong shape. - """ - if args is None or (nest.is_sequence(args) and not args): - raise ValueError("`args` must be specified") - if not nest.is_sequence(args): - args = [args] - - # Calculate the total size of arguments on dimension 1. - total_arg_size = 0 - shapes = [a.get_shape() for a in args] - for shape in shapes: - if shape.ndims != 2: - raise ValueError("linear is expecting 2D arguments: %s" % shapes) - if tensor_shape.dimension_value(shape[1]) is None: - raise ValueError("linear expects shape[1] to be provided for shape %s, " - "but saw %s" % (shape, shape[1])) - else: - total_arg_size += tensor_shape.dimension_value(shape[1]) - - dtype = [a.dtype for a in args][0] - - # Now the computation. - scope = vs.get_variable_scope() - with vs.variable_scope(scope) as outer_scope: - weights = vs.get_variable( - "kernel", [total_arg_size, output_size], - dtype=dtype, - initializer=kernel_initializer) - if len(args) == 1: - res = math_ops.matmul(args[0], weights) - else: - res = math_ops.matmul(array_ops.concat(args, 1), weights) - if not bias: - return res - with vs.variable_scope(outer_scope) as inner_scope: - inner_scope.set_partitioner(None) - if bias_initializer is None: - bias_initializer = init_ops.constant_initializer(0.0, dtype=dtype) - biases = vs.get_variable( - "bias", [output_size], dtype=dtype, initializer=bias_initializer) - - if not layer_norm: - res = nn_ops.bias_add(res, biases) - - return res - - def call(self, inputs, state): - """Run one step of LSTM. - - Args: - inputs: input Tensor, 2D, batch x num_units. - state: this must be a tuple of state Tensors, - both `2-D`, with column sizes `c_state` and - `m_state`. - - Returns: - A tuple containing: - - - A `2-D, [batch x output_dim]`, Tensor representing the output of the - LSTM after reading `inputs` when previous state was `state`. - Here output_dim is: - num_proj if num_proj was set, - num_units otherwise. - - Tensor(s) representing the new state of LSTM after reading `inputs` when - the previous state was `state`. Same type and shape(s) as `state`. - - Raises: - ValueError: If input size cannot be inferred from inputs via - static shape inference. - """ - sigmoid = math_ops.sigmoid - - (c_prev, m_prev) = state - - dtype = inputs.dtype - input_size = inputs.get_shape().with_rank(2).dims[1] - if input_size.value is None: - raise ValueError("Could not infer input size from inputs.get_shape()[-1]") - scope = vs.get_variable_scope() - with vs.variable_scope(scope, initializer=self._initializer) as unit_scope: - - # i = input_gate, j = new_input, f = forget_gate, o = output_gate - lstm_matrix = self._linear( - [inputs, m_prev], - 4 * self._num_units, - bias=True, - bias_initializer=None, - layer_norm=self._layer_norm) - i, j, f, o = array_ops.split( - value=lstm_matrix, num_or_size_splits=4, axis=1) - - if self._layer_norm: - i = _norm(self._norm_gain, self._norm_shift, i, "input") - j = _norm(self._norm_gain, self._norm_shift, j, "transform") - f = _norm(self._norm_gain, self._norm_shift, f, "forget") - o = _norm(self._norm_gain, self._norm_shift, o, "output") - - # Diagonal connections - if self._use_peepholes: - with vs.variable_scope(unit_scope): - w_f_diag = vs.get_variable( - "w_f_diag", shape=[self._num_units], dtype=dtype) - w_i_diag = vs.get_variable( - "w_i_diag", shape=[self._num_units], dtype=dtype) - w_o_diag = vs.get_variable( - "w_o_diag", shape=[self._num_units], dtype=dtype) - - if self._use_peepholes: - c = ( - sigmoid(f + self._forget_bias + w_f_diag * c_prev) * c_prev + - sigmoid(i + w_i_diag * c_prev) * self._activation(j)) - else: - c = ( - sigmoid(f + self._forget_bias) * c_prev + - sigmoid(i) * self._activation(j)) - - if self._layer_norm: - c = _norm(self._norm_gain, self._norm_shift, c, "state") - - if self._cell_clip is not None: - # pylint: disable=invalid-unary-operand-type - c = clip_ops.clip_by_value(c, -self._cell_clip, self._cell_clip) - # pylint: enable=invalid-unary-operand-type - if self._use_peepholes: - m = sigmoid(o + w_o_diag * c) * self._activation(c) - else: - m = sigmoid(o) * self._activation(c) - - if self._num_proj is not None: - with vs.variable_scope("projection"): - m = self._linear(m, self._num_proj, bias=False) - - if self._proj_clip is not None: - # pylint: disable=invalid-unary-operand-type - m = clip_ops.clip_by_value(m, -self._proj_clip, self._proj_clip) - # pylint: enable=invalid-unary-operand-type - - new_state = (rnn_cell_impl.LSTMStateTuple(c, m)) - return m, new_state - - -class SRUCell(rnn_cell_impl.LayerRNNCell): - """SRU, Simple Recurrent Unit. - - Implementation based on - Training RNNs as Fast as CNNs (cf. https://arxiv.org/abs/1709.02755). - - This variation of RNN cell is characterized by the simplified data - dependence - between hidden states of two consecutive time steps. Traditionally, hidden - states from a cell at time step t-1 needs to be multiplied with a matrix - W_hh before being fed into the ensuing cell at time step t. - This flavor of RNN replaces the matrix multiplication between h_{t-1} - and W_hh with a pointwise multiplication, resulting in performance - gain. - - Args: - num_units: int, The number of units in the SRU cell. - activation: Nonlinearity to use. Default: `tanh`. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - name: (optional) String, the name of the layer. Layers with the same name - will share weights, but to avoid mistakes we require reuse=True in such - cases. - **kwargs: Additional keyword arguments. - """ - - def __init__(self, num_units, activation=None, reuse=None, name=None, - **kwargs): - super(SRUCell, self).__init__(_reuse=reuse, name=name, **kwargs) - self._num_units = num_units - self._activation = activation or math_ops.tanh - - # Restrict inputs to be 2-dimensional matrices - self.input_spec = input_spec.InputSpec(ndim=2) - - @property - def state_size(self): - return self._num_units - - @property - def output_size(self): - return self._num_units - - def build(self, inputs_shape): - if tensor_shape.dimension_value(inputs_shape[1]) is None: - raise ValueError( - "Expected inputs.shape[-1] to be known, saw shape: %s" % inputs_shape) - - input_depth = tensor_shape.dimension_value(inputs_shape[1]) - - # pylint: disable=protected-access - self._kernel = self.add_variable( - rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - shape=[input_depth, 4 * self._num_units]) - # pylint: enable=protected-access - self._bias = self.add_variable( - rnn_cell_impl._BIAS_VARIABLE_NAME, # pylint: disable=protected-access - shape=[2 * self._num_units], - initializer=init_ops.zeros_initializer) - - self._built = True - - def call(self, inputs, state): - """Simple recurrent unit (SRU) with num_units cells.""" - - U = math_ops.matmul(inputs, self._kernel) # pylint: disable=invalid-name - x_bar, f_intermediate, r_intermediate, x_tx = array_ops.split( - value=U, num_or_size_splits=4, axis=1) - - f_r = math_ops.sigmoid( - nn_ops.bias_add( - array_ops.concat([f_intermediate, r_intermediate], 1), self._bias)) - f, r = array_ops.split(value=f_r, num_or_size_splits=2, axis=1) - - c = f * state + (1.0 - f) * x_bar - h = r * self._activation(c) + (1.0 - r) * x_tx - - return h, c - - -class WeightNormLSTMCell(rnn_cell_impl.RNNCell): - """Weight normalized LSTM Cell. Adapted from `rnn_cell_impl.LSTMCell`. - - The weight-norm implementation is based on: - https://arxiv.org/abs/1602.07868 - Tim Salimans, Diederik P. Kingma. - Weight Normalization: A Simple Reparameterization to Accelerate - Training of Deep Neural Networks - - The default LSTM implementation based on: - - https://pdfs.semanticscholar.org/1154/0131eae85b2e11d53df7f1360eeb6476e7f4.pdf - - Felix Gers, Jurgen Schmidhuber, and Fred Cummins. - "Learning to forget: Continual prediction with LSTM." IET, 850-855, 1999. - - The class uses optional peephole connections, optional cell clipping - and an optional projection layer. - - The optional peephole implementation is based on: - https://research.google.com/pubs/archive/43905.pdf - Hasim Sak, Andrew Senior, and Francoise Beaufays. - "Long short-term memory recurrent neural network architectures for - large scale acoustic modeling." INTERSPEECH, 2014. - """ - - def __init__(self, - num_units, - norm=True, - use_peepholes=False, - cell_clip=None, - initializer=None, - num_proj=None, - proj_clip=None, - forget_bias=1, - activation=None, - reuse=None): - """Initialize the parameters of a weight-normalized LSTM cell. - - Args: - num_units: int, The number of units in the LSTM cell - norm: If `True`, apply normalization to the weight matrices. If False, - the result is identical to that obtained from `rnn_cell_impl.LSTMCell` - use_peepholes: bool, set `True` to enable diagonal/peephole connections. - cell_clip: (optional) A float value, if provided the cell state is clipped - by this value prior to the cell output activation. - initializer: (optional) The initializer to use for the weight matrices. - num_proj: (optional) int, The output dimensionality for the projection - matrices. If None, no projection is performed. - proj_clip: (optional) A float value. If `num_proj > 0` and `proj_clip` is - provided, then the projected values are clipped elementwise to within - `[-proj_clip, proj_clip]`. - forget_bias: Biases of the forget gate are initialized by default to 1 - in order to reduce the scale of forgetting at the beginning of - the training. - activation: Activation function of the inner states. Default: `tanh`. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - """ - super(WeightNormLSTMCell, self).__init__(_reuse=reuse) - - self._scope = "wn_lstm_cell" - self._num_units = num_units - self._norm = norm - self._initializer = initializer - self._use_peepholes = use_peepholes - self._cell_clip = cell_clip - self._num_proj = num_proj - self._proj_clip = proj_clip - self._activation = activation or math_ops.tanh - self._forget_bias = forget_bias - - self._weights_variable_name = "kernel" - self._bias_variable_name = "bias" - - if num_proj: - self._state_size = rnn_cell_impl.LSTMStateTuple(num_units, num_proj) - self._output_size = num_proj - else: - self._state_size = rnn_cell_impl.LSTMStateTuple(num_units, num_units) - self._output_size = num_units - - @property - def state_size(self): - return self._state_size - - @property - def output_size(self): - return self._output_size - - def _normalize(self, weight, name): - """Apply weight normalization. - - Args: - weight: a 2D tensor with known number of columns. - name: string, variable name for the normalizer. - Returns: - A tensor with the same shape as `weight`. - """ - - output_size = weight.get_shape().as_list()[1] - g = vs.get_variable(name, [output_size], dtype=weight.dtype) - return nn_impl.l2_normalize(weight, axis=0) * g - - def _linear(self, - args, - output_size, - norm, - bias, - bias_initializer=None, - kernel_initializer=None): - """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable. - - Args: - args: a 2D Tensor or a list of 2D, batch x n, Tensors. - output_size: int, second dimension of W[i]. - norm: bool, whether to normalize the weights. - bias: boolean, whether to add a bias term or not. - bias_initializer: starting value to initialize the bias - (default is all zeros). - kernel_initializer: starting value to initialize the weight. - - Returns: - A 2D Tensor with shape [batch x output_size] equal to - sum_i(args[i] * W[i]), where W[i]s are newly created matrices. - - Raises: - ValueError: if some of the arguments has unspecified or wrong shape. - """ - if args is None or (nest.is_sequence(args) and not args): - raise ValueError("`args` must be specified") - if not nest.is_sequence(args): - args = [args] - - # Calculate the total size of arguments on dimension 1. - total_arg_size = 0 - shapes = [a.get_shape() for a in args] - for shape in shapes: - if shape.ndims != 2: - raise ValueError("linear is expecting 2D arguments: %s" % shapes) - if tensor_shape.dimension_value(shape[1]) is None: - raise ValueError("linear expects shape[1] to be provided for shape %s, " - "but saw %s" % (shape, shape[1])) - else: - total_arg_size += tensor_shape.dimension_value(shape[1]) - - dtype = [a.dtype for a in args][0] - - # Now the computation. - scope = vs.get_variable_scope() - with vs.variable_scope(scope) as outer_scope: - weights = vs.get_variable( - self._weights_variable_name, [total_arg_size, output_size], - dtype=dtype, - initializer=kernel_initializer) - if norm: - wn = [] - st = 0 - with ops.control_dependencies(None): - for i in range(len(args)): - en = st + tensor_shape.dimension_value(shapes[i][1]) - wn.append( - self._normalize(weights[st:en, :], name="norm_{}".format(i))) - st = en - - weights = array_ops.concat(wn, axis=0) - - if len(args) == 1: - res = math_ops.matmul(args[0], weights) - else: - res = math_ops.matmul(array_ops.concat(args, 1), weights) - if not bias: - return res - - with vs.variable_scope(outer_scope) as inner_scope: - inner_scope.set_partitioner(None) - if bias_initializer is None: - bias_initializer = init_ops.constant_initializer(0.0, dtype=dtype) - - biases = vs.get_variable( - self._bias_variable_name, [output_size], - dtype=dtype, - initializer=bias_initializer) - - return nn_ops.bias_add(res, biases) - - def call(self, inputs, state): - """Run one step of LSTM. - - Args: - inputs: input Tensor, 2D, batch x num_units. - state: A tuple of state Tensors, both `2-D`, with column sizes - `c_state` and `m_state`. - - Returns: - A tuple containing: - - - A `2-D, [batch x output_dim]`, Tensor representing the output of the - LSTM after reading `inputs` when previous state was `state`. - Here output_dim is: - num_proj if num_proj was set, - num_units otherwise. - - Tensor(s) representing the new state of LSTM after reading `inputs` when - the previous state was `state`. Same type and shape(s) as `state`. - - Raises: - ValueError: If input size cannot be inferred from inputs via - static shape inference. - """ - dtype = inputs.dtype - num_units = self._num_units - sigmoid = math_ops.sigmoid - c, h = state - - input_size = inputs.get_shape().with_rank(2).dims[1] - if input_size.value is None: - raise ValueError("Could not infer input size from inputs.get_shape()[-1]") - - with vs.variable_scope(self._scope, initializer=self._initializer): - - concat = self._linear( - [inputs, h], 4 * num_units, norm=self._norm, bias=True) - - # i = input_gate, j = new_input, f = forget_gate, o = output_gate - i, j, f, o = array_ops.split(value=concat, num_or_size_splits=4, axis=1) - - if self._use_peepholes: - w_f_diag = vs.get_variable("w_f_diag", shape=[num_units], dtype=dtype) - w_i_diag = vs.get_variable("w_i_diag", shape=[num_units], dtype=dtype) - w_o_diag = vs.get_variable("w_o_diag", shape=[num_units], dtype=dtype) - - new_c = ( - c * sigmoid(f + self._forget_bias + w_f_diag * c) + - sigmoid(i + w_i_diag * c) * self._activation(j)) - else: - new_c = ( - c * sigmoid(f + self._forget_bias) + - sigmoid(i) * self._activation(j)) - - if self._cell_clip is not None: - # pylint: disable=invalid-unary-operand-type - new_c = clip_ops.clip_by_value(new_c, -self._cell_clip, self._cell_clip) - # pylint: enable=invalid-unary-operand-type - if self._use_peepholes: - new_h = sigmoid(o + w_o_diag * new_c) * self._activation(new_c) - else: - new_h = sigmoid(o) * self._activation(new_c) - - if self._num_proj is not None: - with vs.variable_scope("projection"): - new_h = self._linear( - new_h, self._num_proj, norm=self._norm, bias=False) - - if self._proj_clip is not None: - # pylint: disable=invalid-unary-operand-type - new_h = clip_ops.clip_by_value(new_h, -self._proj_clip, - self._proj_clip) - # pylint: enable=invalid-unary-operand-type - - new_state = rnn_cell_impl.LSTMStateTuple(new_c, new_h) - return new_h, new_state - - -class IndRNNCell(rnn_cell_impl.LayerRNNCell): - """Independently Recurrent Neural Network (IndRNN) cell - (cf. https://arxiv.org/abs/1803.04831). - - Args: - num_units: int, The number of units in the RNN cell. - activation: Nonlinearity to use. Default: `tanh`. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - name: String, the name of the layer. Layers with the same name will - share weights, but to avoid mistakes we require reuse=True in such - cases. - dtype: Default dtype of the layer (default of `None` means use the type - of the first input). Required when `build` is called before `call`. - """ - - def __init__(self, - num_units, - activation=None, - reuse=None, - name=None, - dtype=None): - super(IndRNNCell, self).__init__(_reuse=reuse, name=name, dtype=dtype) - - # Inputs must be 2-dimensional. - self.input_spec = input_spec.InputSpec(ndim=2) - - self._num_units = num_units - self._activation = activation or math_ops.tanh - - @property - def state_size(self): - return self._num_units - - @property - def output_size(self): - return self._num_units - - def build(self, inputs_shape): - if tensor_shape.dimension_value(inputs_shape[1]) is None: - raise ValueError( - "Expected inputs.shape[-1] to be known, saw shape: %s" % inputs_shape) - - input_depth = tensor_shape.dimension_value(inputs_shape[1]) - # pylint: disable=protected-access - self._kernel_w = self.add_variable( - "%s_w" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - shape=[input_depth, self._num_units]) - self._kernel_u = self.add_variable( - "%s_u" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - shape=[1, self._num_units], - initializer=init_ops.random_uniform_initializer( - minval=-1, maxval=1, dtype=self.dtype)) - self._bias = self.add_variable( - rnn_cell_impl._BIAS_VARIABLE_NAME, - shape=[self._num_units], - initializer=init_ops.zeros_initializer(dtype=self.dtype)) - # pylint: enable=protected-access - - self.built = True - - def call(self, inputs, state): - """IndRNN: output = new_state = act(W * input + u * state + B).""" - - gate_inputs = math_ops.matmul(inputs, self._kernel_w) + ( - state * self._kernel_u) - gate_inputs = nn_ops.bias_add(gate_inputs, self._bias) - output = self._activation(gate_inputs) - return output, output - - -class IndyGRUCell(rnn_cell_impl.LayerRNNCell): - r"""Independently Gated Recurrent Unit cell. - - Based on IndRNNs (https://arxiv.org/abs/1803.04831) and similar to GRUCell, - yet with the \\(U_r\\), \\(U_z\\), and \\(U\\) matrices in equations 5, 6, and - 8 of http://arxiv.org/abs/1406.1078 respectively replaced by diagonal - matrices, i.e. a Hadamard product with a single vector: - - $$r_j = \sigma\left([\mathbf W_r\mathbf x]_j + - [\mathbf u_r\circ \mathbf h_{(t-1)}]_j\right)$$ - $$z_j = \sigma\left([\mathbf W_z\mathbf x]_j + - [\mathbf u_z\circ \mathbf h_{(t-1)}]_j\right)$$ - $$\tilde{h}^{(t)}_j = \phi\left([\mathbf W \mathbf x]_j + - [\mathbf u \circ \mathbf r \circ \mathbf h_{(t-1)}]_j\right)$$ - - where \\(\circ\\) denotes the Hadamard operator. This means that each IndyGRU - node sees only its own state, as opposed to seeing all states in the same - layer. - - Args: - num_units: int, The number of units in the GRU cell. - activation: Nonlinearity to use. Default: `tanh`. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - kernel_initializer: (optional) The initializer to use for the weight - matrices applied to the input. - bias_initializer: (optional) The initializer to use for the bias. - name: String, the name of the layer. Layers with the same name will - share weights, but to avoid mistakes we require reuse=True in such - cases. - dtype: Default dtype of the layer (default of `None` means use the type - of the first input). Required when `build` is called before `call`. - """ - - def __init__(self, - num_units, - activation=None, - reuse=None, - kernel_initializer=None, - bias_initializer=None, - name=None, - dtype=None): - super(IndyGRUCell, self).__init__(_reuse=reuse, name=name, dtype=dtype) - - # Inputs must be 2-dimensional. - self.input_spec = input_spec.InputSpec(ndim=2) - - self._num_units = num_units - self._activation = activation or math_ops.tanh - self._kernel_initializer = kernel_initializer - self._bias_initializer = bias_initializer - - @property - def state_size(self): - return self._num_units - - @property - def output_size(self): - return self._num_units - - def build(self, inputs_shape): - if tensor_shape.dimension_value(inputs_shape[1]) is None: - raise ValueError( - "Expected inputs.shape[-1] to be known, saw shape: %s" % inputs_shape) - - input_depth = tensor_shape.dimension_value(inputs_shape[1]) - # pylint: disable=protected-access - self._gate_kernel_w = self.add_variable( - "gates/%s_w" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - shape=[input_depth, 2 * self._num_units], - initializer=self._kernel_initializer) - self._gate_kernel_u = self.add_variable( - "gates/%s_u" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - shape=[1, 2 * self._num_units], - initializer=init_ops.random_uniform_initializer( - minval=-1, maxval=1, dtype=self.dtype)) - self._gate_bias = self.add_variable( - "gates/%s" % rnn_cell_impl._BIAS_VARIABLE_NAME, - shape=[2 * self._num_units], - initializer=(self._bias_initializer - if self._bias_initializer is not None else - init_ops.constant_initializer(1.0, dtype=self.dtype))) - self._candidate_kernel_w = self.add_variable( - "candidate/%s" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - shape=[input_depth, self._num_units], - initializer=self._kernel_initializer) - self._candidate_kernel_u = self.add_variable( - "candidate/%s_u" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - shape=[1, self._num_units], - initializer=init_ops.random_uniform_initializer( - minval=-1, maxval=1, dtype=self.dtype)) - self._candidate_bias = self.add_variable( - "candidate/%s" % rnn_cell_impl._BIAS_VARIABLE_NAME, - shape=[self._num_units], - initializer=(self._bias_initializer - if self._bias_initializer is not None else - init_ops.zeros_initializer(dtype=self.dtype))) - # pylint: enable=protected-access - - self.built = True - - def call(self, inputs, state): - """Recurrently independent Gated Recurrent Unit (GRU) with nunits cells.""" - - gate_inputs = math_ops.matmul(inputs, self._gate_kernel_w) + ( - gen_array_ops.tile(state, [1, 2]) * self._gate_kernel_u) - gate_inputs = nn_ops.bias_add(gate_inputs, self._gate_bias) - - value = math_ops.sigmoid(gate_inputs) - r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1) - - r_state = r * state - - candidate = math_ops.matmul(inputs, self._candidate_kernel_w) + ( - r_state * self._candidate_kernel_u) - candidate = nn_ops.bias_add(candidate, self._candidate_bias) - - c = self._activation(candidate) - new_h = u * state + (1 - u) * c - return new_h, new_h - - -class IndyLSTMCell(rnn_cell_impl.LayerRNNCell): - r"""Basic IndyLSTM recurrent network cell. - - Based on IndRNNs (https://arxiv.org/abs/1803.04831) and similar to - BasicLSTMCell, yet with the \\(U_f\\), \\(U_i\\), \\(U_o\\) and \\(U_c\\) - matrices in the regular LSTM equations replaced by diagonal matrices, i.e. a - Hadamard product with a single vector: - - $$f_t = \sigma_g\left(W_f x_t + u_f \circ h_{t-1} + b_f\right)$$ - $$i_t = \sigma_g\left(W_i x_t + u_i \circ h_{t-1} + b_i\right)$$ - $$o_t = \sigma_g\left(W_o x_t + u_o \circ h_{t-1} + b_o\right)$$ - $$c_t = f_t \circ c_{t-1} + - i_t \circ \sigma_c\left(W_c x_t + u_c \circ h_{t-1} + b_c\right)$$ - - where \\(\circ\\) denotes the Hadamard operator. This means that each IndyLSTM - node sees only its own state \\(h\\) and \\(c\\), as opposed to seeing all - states in the same layer. - - We add forget_bias (default: 1) to the biases of the forget gate in order to - reduce the scale of forgetting in the beginning of the training. - - It does not allow cell clipping, a projection layer, and does not - use peep-hole connections: it is the basic baseline. - - For a detailed analysis of IndyLSTMs, see https://arxiv.org/abs/1903.08023. - """ - - def __init__(self, - num_units, - forget_bias=1.0, - activation=None, - reuse=None, - kernel_initializer=None, - bias_initializer=None, - name=None, - dtype=None): - """Initialize the IndyLSTM cell. - - Args: - num_units: int, The number of units in the LSTM cell. - forget_bias: float, The bias added to forget gates (see above). - Must set to `0.0` manually when restoring from CudnnLSTM-trained - checkpoints. - activation: Activation function of the inner states. Default: `tanh`. - reuse: (optional) Python boolean describing whether to reuse variables - in an existing scope. If not `True`, and the existing scope already has - the given variables, an error is raised. - kernel_initializer: (optional) The initializer to use for the weight - matrix applied to the inputs. - bias_initializer: (optional) The initializer to use for the bias. - name: String, the name of the layer. Layers with the same name will - share weights, but to avoid mistakes we require reuse=True in such - cases. - dtype: Default dtype of the layer (default of `None` means use the type - of the first input). Required when `build` is called before `call`. - """ - super(IndyLSTMCell, self).__init__(_reuse=reuse, name=name, dtype=dtype) - - # Inputs must be 2-dimensional. - self.input_spec = input_spec.InputSpec(ndim=2) - - self._num_units = num_units - self._forget_bias = forget_bias - self._activation = activation or math_ops.tanh - self._kernel_initializer = kernel_initializer - self._bias_initializer = bias_initializer - - @property - def state_size(self): - return rnn_cell_impl.LSTMStateTuple(self._num_units, self._num_units) - - @property - def output_size(self): - return self._num_units - - def build(self, inputs_shape): - if tensor_shape.dimension_value(inputs_shape[1]) is None: - raise ValueError( - "Expected inputs.shape[-1] to be known, saw shape: %s" % inputs_shape) - - input_depth = tensor_shape.dimension_value(inputs_shape[1]) - # pylint: disable=protected-access - self._kernel_w = self.add_variable( - "%s_w" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - shape=[input_depth, 4 * self._num_units], - initializer=self._kernel_initializer) - self._kernel_u = self.add_variable( - "%s_u" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - shape=[1, 4 * self._num_units], - initializer=init_ops.random_uniform_initializer( - minval=-1, maxval=1, dtype=self.dtype)) - self._bias = self.add_variable( - rnn_cell_impl._BIAS_VARIABLE_NAME, - shape=[4 * self._num_units], - initializer=(self._bias_initializer - if self._bias_initializer is not None else - init_ops.zeros_initializer(dtype=self.dtype))) - # pylint: enable=protected-access - - self.built = True - - def call(self, inputs, state): - """Independent Long short-term memory cell (IndyLSTM). - - Args: - inputs: `2-D` tensor with shape `[batch_size, input_size]`. - state: An `LSTMStateTuple` of state tensors, each shaped - `[batch_size, num_units]`. - - Returns: - A pair containing the new hidden state, and the new state (a - `LSTMStateTuple`). - """ - sigmoid = math_ops.sigmoid - one = constant_op.constant(1, dtype=dtypes.int32) - c, h = state - - gate_inputs = math_ops.matmul(inputs, self._kernel_w) - gate_inputs += gen_array_ops.tile(h, [1, 4]) * self._kernel_u - gate_inputs = nn_ops.bias_add(gate_inputs, self._bias) - - # i = input_gate, j = new_input, f = forget_gate, o = output_gate - i, j, f, o = array_ops.split( - value=gate_inputs, num_or_size_splits=4, axis=one) - - forget_bias_tensor = constant_op.constant(self._forget_bias, dtype=f.dtype) - # Note that using `add` and `multiply` instead of `+` and `*` gives a - # performance improvement. So using those at the cost of readability. - add = math_ops.add - multiply = math_ops.multiply - new_c = add( - multiply(c, sigmoid(add(f, forget_bias_tensor))), - multiply(sigmoid(i), self._activation(j))) - new_h = multiply(self._activation(new_c), sigmoid(o)) - - new_state = rnn_cell_impl.LSTMStateTuple(new_c, new_h) - return new_h, new_state - - -NTMControllerState = collections.namedtuple( - "NTMControllerState", - ("controller_state", "read_vector_list", "w_list", "M", "time")) - - -class NTMCell(rnn_cell_impl.LayerRNNCell): - """Neural Turing Machine Cell with RNN controller. - - Implementation based on: - https://arxiv.org/abs/1807.08518 - Mark Collier, Joeran Beel - - which is in turn based on the source code of: - https://github.com/snowkylin/ntm - - and of course the original NTM paper: - Neural Turing Machines - https://arxiv.org/abs/1410.5401 - A Graves, G Wayne, I Danihelka - """ - - def __init__(self, - controller, - memory_size, - memory_vector_dim, - read_head_num, - write_head_num, - shift_range=1, - output_dim=None, - clip_value=20, - dtype=dtypes.float32, - name=None): - """Initialize the NTM Cell. - - Args: - controller: an RNNCell, the RNN controller. - memory_size: int, The number of memory locations in the NTM memory - matrix - memory_vector_dim: int, The dimensionality of each location in the NTM - memory matrix - read_head_num: int, The number of read heads from the controller into - memory - write_head_num: int, The number of write heads from the controller into - memory - shift_range: int, The number of places to the left/right it is possible - to iterate the previous address to in a single step - output_dim: int, The number of dimensions to make a linear projection of - the NTM controller outputs to. If None, no linear projection is - applied - clip_value: float, The maximum absolute value the controller parameters - are clipped to - dtype: Default dtype of the layer (default of `None` means use the type - of the first input). Required when `build` is called before `call`. - name: String, the name of the layer. Layers with the same name will - share weights, but to avoid mistakes we require reuse=True in such - cases. - """ - super(NTMCell, self).__init__(dtype=dtype, name=name) - - rnn_cell_impl.assert_like_rnncell("NTM RNN controller cell", controller) - - self.controller = controller - self.memory_size = memory_size - self.memory_vector_dim = memory_vector_dim - self.read_head_num = read_head_num - self.write_head_num = write_head_num - self.clip_value = clip_value - - self.output_dim = output_dim - self.shift_range = shift_range - - self.num_parameters_per_head = ( - self.memory_vector_dim + 2 * self.shift_range + 4) - self.num_heads = self.read_head_num + self.write_head_num - self.total_parameter_num = ( - self.num_parameters_per_head * self.num_heads + - self.memory_vector_dim * 2 * self.write_head_num) - - @property - def state_size(self): - return NTMControllerState( - controller_state=self.controller.state_size, - read_vector_list=[ - self.memory_vector_dim for _ in range(self.read_head_num) - ], - w_list=[ - self.memory_size - for _ in range(self.read_head_num + self.write_head_num) - ], - M=tensor_shape.TensorShape([self.memory_size * self.memory_vector_dim]), - time=tensor_shape.TensorShape([])) - - @property - def output_size(self): - return self.output_dim - - def build(self, inputs_shape): - if self.output_dim is None: - if inputs_shape[1].value is None: - raise ValueError( - "Expected inputs.shape[-1] to be known, saw shape: %s" % - inputs_shape) - else: - self.output_dim = inputs_shape[1].value - - def _create_linear_initializer(input_size, dtype=dtypes.float32): - stddev = 1.0 / math.sqrt(input_size) - return init_ops.truncated_normal_initializer(stddev=stddev, dtype=dtype) - - self._params_kernel = self.add_variable( - "parameters_kernel", - shape=[self.controller.output_size, self.total_parameter_num], - initializer=_create_linear_initializer(self.controller.output_size)) - - self._params_bias = self.add_variable( - "parameters_bias", - shape=[self.total_parameter_num], - initializer=init_ops.constant_initializer(0.0, dtype=self.dtype)) - - self._output_kernel = self.add_variable( - "output_kernel", - shape=[ - self.controller.output_size + - self.memory_vector_dim * self.read_head_num, self.output_dim - ], - initializer=_create_linear_initializer(self.controller.output_size + - self.memory_vector_dim * - self.read_head_num)) - - self._output_bias = self.add_variable( - "output_bias", - shape=[self.output_dim], - initializer=init_ops.constant_initializer(0.0, dtype=self.dtype)) - - self._init_read_vectors = [ - self.add_variable( - "initial_read_vector_%d" % i, - shape=[1, self.memory_vector_dim], - initializer=initializers.glorot_uniform()) - for i in range(self.read_head_num) - ] - - self._init_address_weights = [ - self.add_variable( - "initial_address_weights_%d" % i, - shape=[1, self.memory_size], - initializer=initializers.glorot_uniform()) - for i in range(self.read_head_num + self.write_head_num) - ] - - self._M = self.add_variable( - "memory", - shape=[self.memory_size, self.memory_vector_dim], - initializer=init_ops.constant_initializer(1e-6, dtype=self.dtype)) - - self.built = True - - def call(self, x, prev_state): - # Addressing Mechanisms (Sec 3.3) - - def _prev_read_vector_list_initial_value(): - return [ - self._expand( - math_ops.tanh( - array_ops.squeeze( - math_ops.matmul( - array_ops.ones([1, 1]), self._init_read_vectors[i]))), - dim=0, - N=x.shape[0].value or array_ops.shape(x)[0]) - for i in range(self.read_head_num) - ] - - prev_read_vector_list = control_flow_ops.cond( - math_ops.equal(prev_state.time, - 0), _prev_read_vector_list_initial_value, lambda: - prev_state.read_vector_list) - if self.read_head_num == 1: - prev_read_vector_list = [prev_read_vector_list] - - controller_input = array_ops.concat([x] + prev_read_vector_list, axis=1) - controller_output, controller_state = self.controller( - controller_input, prev_state.controller_state) - - parameters = math_ops.matmul(controller_output, self._params_kernel) - parameters = nn_ops.bias_add(parameters, self._params_bias) - parameters = clip_ops.clip_by_value(parameters, -self.clip_value, - self.clip_value) - head_parameter_list = array_ops.split( - parameters[:, :self.num_parameters_per_head * self.num_heads], - self.num_heads, - axis=1) - erase_add_list = array_ops.split( - parameters[:, self.num_parameters_per_head * self.num_heads:], - 2 * self.write_head_num, - axis=1) - - def _prev_w_list_initial_value(): - return [ - self._expand( - nn_ops.softmax( - array_ops.squeeze( - math_ops.matmul( - array_ops.ones([1, 1]), - self._init_address_weights[i]))), - dim=0, - N=x.shape[0].value or array_ops.shape(x)[0]) - for i in range(self.read_head_num + self.write_head_num) - ] - - prev_w_list = control_flow_ops.cond( - math_ops.equal(prev_state.time, 0), - _prev_w_list_initial_value, lambda: prev_state.w_list) - if (self.read_head_num + self.write_head_num) == 1: - prev_w_list = [prev_w_list] - - prev_M = control_flow_ops.cond( - math_ops.equal(prev_state.time, 0), lambda: self._expand( - self._M, dim=0, N=x.shape[0].value or array_ops.shape(x)[0]), - lambda: prev_state.M) - - w_list = [] - for i, head_parameter in enumerate(head_parameter_list): - k = math_ops.tanh(head_parameter[:, 0:self.memory_vector_dim]) - beta = nn_ops.softplus(head_parameter[:, self.memory_vector_dim]) - g = math_ops.sigmoid(head_parameter[:, self.memory_vector_dim + 1]) - s = nn_ops.softmax(head_parameter[:, self.memory_vector_dim + - 2:(self.memory_vector_dim + 2 + - (self.shift_range * 2 + 1))]) - gamma = nn_ops.softplus(head_parameter[:, -1]) + 1 - w = self._addressing(k, beta, g, s, gamma, prev_M, prev_w_list[i]) - w_list.append(w) - - # Reading (Sec 3.1) - - read_w_list = w_list[:self.read_head_num] - read_vector_list = [] - for i in range(self.read_head_num): - read_vector = math_ops.reduce_sum( - array_ops.expand_dims(read_w_list[i], dim=2) * prev_M, axis=1) - read_vector_list.append(read_vector) - - # Writing (Sec 3.2) - - write_w_list = w_list[self.read_head_num:] - M = prev_M - for i in range(self.write_head_num): - w = array_ops.expand_dims(write_w_list[i], axis=2) - erase_vector = array_ops.expand_dims( - math_ops.sigmoid(erase_add_list[i * 2]), axis=1) - add_vector = array_ops.expand_dims( - math_ops.tanh(erase_add_list[i * 2 + 1]), axis=1) - erase_M = array_ops.ones_like(M) - math_ops.matmul(w, erase_vector) - M = M * erase_M + math_ops.matmul(w, add_vector) - - output = math_ops.matmul( - array_ops.concat([controller_output] + read_vector_list, axis=1), - self._output_kernel) - output = nn_ops.bias_add(output, self._output_bias) - output = clip_ops.clip_by_value(output, -self.clip_value, self.clip_value) - - return output, NTMControllerState( - controller_state=controller_state, - read_vector_list=read_vector_list, - w_list=w_list, - M=M, - time=prev_state.time + 1) - - def _expand(self, x, dim, N): - return array_ops.concat([array_ops.expand_dims(x, dim) for _ in range(N)], - axis=dim) - - def _addressing(self, k, beta, g, s, gamma, prev_M, prev_w): - # Sec 3.3.1 Focusing by Content - - k = array_ops.expand_dims(k, axis=2) - inner_product = math_ops.matmul(prev_M, k) - k_norm = math_ops.sqrt( - math_ops.reduce_sum(math_ops.square(k), axis=1, keepdims=True)) - M_norm = math_ops.sqrt( - math_ops.reduce_sum(math_ops.square(prev_M), axis=2, keepdims=True)) - norm_product = M_norm * k_norm - - # eq (6) - K = array_ops.squeeze(inner_product / (norm_product + 1e-8)) - - K_amplified = math_ops.exp(array_ops.expand_dims(beta, axis=1) * K) - - # eq (5) - w_c = K_amplified / math_ops.reduce_sum(K_amplified, axis=1, keepdims=True) - - # Sec 3.3.2 Focusing by Location - - g = array_ops.expand_dims(g, axis=1) - - # eq (7) - w_g = g * w_c + (1 - g) * prev_w - - s = array_ops.concat([ - s[:, :self.shift_range + 1], - array_ops.zeros([ - s.shape[0].value or array_ops.shape(s)[0], self.memory_size - - (self.shift_range * 2 + 1) - ]), s[:, -self.shift_range:] - ], - axis=1) - t = array_ops.concat( - [array_ops.reverse(s, axis=[1]), - array_ops.reverse(s, axis=[1])], - axis=1) - s_matrix = array_ops.stack([ - t[:, self.memory_size - i - 1:self.memory_size * 2 - i - 1] - for i in range(self.memory_size) - ], - axis=1) - - # eq (8) - w_ = math_ops.reduce_sum( - array_ops.expand_dims(w_g, axis=1) * s_matrix, axis=2) - w_sharpen = math_ops.pow(w_, array_ops.expand_dims(gamma, axis=1)) - - # eq (9) - w = w_sharpen / math_ops.reduce_sum(w_sharpen, axis=1, keepdims=True) - - return w - - def zero_state(self, batch_size, dtype): - read_vector_list = [ - array_ops.zeros([batch_size, self.memory_vector_dim]) - for _ in range(self.read_head_num) - ] - - w_list = [ - array_ops.zeros([batch_size, self.memory_size]) - for _ in range(self.read_head_num + self.write_head_num) - ] - - controller_init_state = self.controller.zero_state(batch_size, dtype) - - M = array_ops.zeros([batch_size, self.memory_size, self.memory_vector_dim]) - - return NTMControllerState( - controller_state=controller_init_state, - read_vector_list=read_vector_list, - w_list=w_list, - M=M, - time=0) - - -class MinimalRNNCell(rnn_cell_impl.LayerRNNCell): - """MinimalRNN cell. - - The implementation is based on: - - https://arxiv.org/pdf/1806.05394v2.pdf - - Minmin Chen, Jeffrey Pennington, Samuel S. Schoenholz. - "Dynamical Isometry and a Mean Field Theory of RNNs: Gating Enables Signal - Propagation in Recurrent Neural Networks." ICML, 2018. - - A MinimalRNN cell first projects the input to the hidden space. The new - hidden state is then calculated as a weighted sum of the projected input and - the previous hidden state, using a single update gate. - """ - - def __init__(self, - units, - activation="tanh", - kernel_initializer="glorot_uniform", - bias_initializer="ones", - name=None, - dtype=None, - **kwargs): - """Initialize the parameters for a MinimalRNN cell. - - Args: - units: int, The number of units in the MinimalRNN cell. - activation: Nonlinearity to use in the feedforward network. Default: - `tanh`. - kernel_initializer: The initializer to use for the weight in the update - gate and feedforward network. Default: `glorot_uniform`. - bias_initializer: The initializer to use for the bias in the update - gate. Default: `ones`. - name: String, the name of the cell. - dtype: Default dtype of the cell. - **kwargs: Dict, keyword named properties for common cell attributes. - """ - super(MinimalRNNCell, self).__init__(name=name, dtype=dtype, **kwargs) - - # Inputs must be 2-dimensional. - self.input_spec = input_spec.InputSpec(ndim=2) - - self.units = units - self.activation = activations.get(activation) - self.kernel_initializer = initializers.get(kernel_initializer) - self.bias_initializer = initializers.get(bias_initializer) - - @property - def state_size(self): - return self.units - - @property - def output_size(self): - return self.units - - def build(self, inputs_shape): - if inputs_shape[-1] is None: - raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" - % str(inputs_shape)) - - input_size = inputs_shape[-1] - # pylint: disable=protected-access - # self._kernel contains W_x, W, V - self.kernel = self.add_weight( - name=rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - shape=[input_size + 2 * self.units, self.units], - initializer=self.kernel_initializer) - self.bias = self.add_weight( - name=rnn_cell_impl._BIAS_VARIABLE_NAME, - shape=[self.units], - initializer=self.bias_initializer) - # pylint: enable=protected-access - - self.built = True - - def call(self, inputs, state): - """Run one step of MinimalRNN. - - Args: - inputs: input Tensor, must be 2-D, `[batch, input_size]`. - state: state Tensor, must be 2-D, `[batch, state_size]`. - - Returns: - A tuple containing: - - - Output: A `2-D` tensor with shape `[batch_size, state_size]`. - - New state: A `2-D` tensor with shape `[batch_size, state_size]`. - - Raises: - ValueError: If input size cannot be inferred from inputs via - static shape inference. - """ - input_size = inputs.get_shape()[1] - if tensor_shape.dimension_value(input_size) is None: - raise ValueError("Could not infer input size from inputs.get_shape()[-1]") - - feedforward_weight, gate_weight = array_ops.split( - value=self.kernel, - num_or_size_splits=[tensor_shape.dimension_value(input_size), - 2 * self.units], - axis=0) - - feedforward = math_ops.matmul(inputs, feedforward_weight) - feedforward = self.activation(feedforward) - - gate_inputs = math_ops.matmul( - array_ops.concat([feedforward, state], 1), gate_weight) - gate_inputs = nn_ops.bias_add(gate_inputs, self.bias) - u = math_ops.sigmoid(gate_inputs) - - new_h = u * state + (1 - u) * feedforward - return new_h, new_h - - -class CFNCell(rnn_cell_impl.LayerRNNCell): - """Chaos Free Network cell. - - The implementation is based on: - - https://openreview.net/pdf?id=S1dIzvclg - - Thomas Laurent, James von Brecht. - "A recurrent neural network without chaos." ICLR, 2017. - - A CFN cell first projects the input to the hidden space. The hidden state - goes through a contractive mapping. The new hidden state is then calculated - as a linear combination of the projected input and the contracted previous - hidden state, using decoupled input and forget gates. - """ - - def __init__(self, - units, - activation="tanh", - kernel_initializer="glorot_uniform", - bias_initializer="ones", - name=None, - dtype=None, - **kwargs): - """Initialize the parameters for a CFN cell. - - Args: - units: int, The number of units in the CFN cell. - activation: Nonlinearity to use. Default: `tanh`. - kernel_initializer: Initializer for the `kernel` weights - matrix. Default: `glorot_uniform`. - bias_initializer: The initializer to use for the bias in the - gates. Default: `ones`. - name: String, the name of the cell. - dtype: Default dtype of the cell. - **kwargs: Dict, keyword named properties for common cell attributes. - """ - super(CFNCell, self).__init__(name=name, dtype=dtype, **kwargs) - - # Inputs must be 2-dimensional. - self.input_spec = input_spec.InputSpec(ndim=2) - - self.units = units - self.activation = activations.get(activation) - self.kernel_initializer = initializers.get(kernel_initializer) - self.bias_initializer = initializers.get(bias_initializer) - - @property - def state_size(self): - return self.units - - @property - def output_size(self): - return self.units - - def build(self, inputs_shape): - if inputs_shape[-1] is None: - raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" - % str(inputs_shape)) - - input_size = inputs_shape[-1] - # pylint: disable=protected-access - # `self.kernel` contains V_{\theta}, V_{\eta}, W. - # `self.recurrent_kernel` contains U_{\theta}, U_{\eta}. - # `self.bias` contains b_{\theta}, b_{\eta}. - self.kernel = self.add_weight( - shape=[input_size, 3 * self.units], - name=rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - initializer=self.kernel_initializer) - self.recurrent_kernel = self.add_weight( - shape=[self.units, 2 * self.units], - name="recurrent_%s" % rnn_cell_impl._WEIGHTS_VARIABLE_NAME, - initializer=self.kernel_initializer) - self.bias = self.add_weight( - shape=[2 * self.units], - name=rnn_cell_impl._BIAS_VARIABLE_NAME, - initializer=self.bias_initializer) - # pylint: enable=protected-access - - self.built = True - - def call(self, inputs, state): - """Run one step of CFN. - - Args: - inputs: input Tensor, must be 2-D, `[batch, input_size]`. - state: state Tensor, must be 2-D, `[batch, state_size]`. - - Returns: - A tuple containing: - - - Output: A `2-D` tensor with shape `[batch_size, state_size]`. - - New state: A `2-D` tensor with shape `[batch_size, state_size]`. - - Raises: - ValueError: If input size cannot be inferred from inputs via - static shape inference. - """ - input_size = inputs.get_shape()[-1] - if tensor_shape.dimension_value(input_size) is None: - raise ValueError("Could not infer input size from inputs.get_shape()[-1]") - - # The variable names u, v, w, b are consistent with the notations in the - # original paper. - v, w = array_ops.split( - value=self.kernel, - num_or_size_splits=[2 * self.units, self.units], - axis=1) - u = self.recurrent_kernel - b = self.bias - - gates = math_ops.matmul(state, u) + math_ops.matmul(inputs, v) - gates = nn_ops.bias_add(gates, b) - gates = math_ops.sigmoid(gates) - theta, eta = array_ops.split(value=gates, - num_or_size_splits=2, - axis=1) - - proj_input = math_ops.matmul(inputs, w) - - # The input gate is (1 - eta), which is different from the original paper. - # This is for the propose of initialization. With the default - # bias_initializer `ones`, the input gate is initialized to a small number. - new_h = theta * self.activation(state) + (1 - eta) * self.activation( - proj_input) - - return new_h, new_h diff --git a/tensorflow/contrib/rnn/python/tools/checkpoint_convert.py b/tensorflow/contrib/rnn/python/tools/checkpoint_convert.py deleted file mode 100644 index 460e172a6d9..00000000000 --- a/tensorflow/contrib/rnn/python/tools/checkpoint_convert.py +++ /dev/null @@ -1,274 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -r"""Convert checkpoints using RNNCells to new name convention. - -Usage: - - python checkpoint_convert.py [--write_v1_checkpoint] \ - '/path/to/checkpoint' '/path/to/new_checkpoint' - -For example, if there is a V2 checkpoint to be converted and the files include: - /tmp/my_checkpoint/model.ckpt.data-00000-of-00001 - /tmp/my_checkpoint/model.ckpt.index - /tmp/my_checkpoint/model.ckpt.meta - -use the following command: - mkdir /tmp/my_converted_checkpoint && - python checkpoint_convert.py \ - /tmp/my_checkpoint/model.ckpt /tmp/my_converted_checkpoint/model.ckpt - -This will generate three converted checkpoint files corresponding to the three -old ones in the new directory: - /tmp/my_converted_checkpoint/model.ckpt.data-00000-of-00001 - /tmp/my_converted_checkpoint/model.ckpt.index - /tmp/my_converted_checkpoint/model.ckpt.meta -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import collections -import re -import sys - -from tensorflow.core.protobuf import saver_pb2 -from tensorflow.python import pywrap_tensorflow -from tensorflow.python.client import session -from tensorflow.python.framework import ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import app -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import saver as saver_lib - -# Mapping between old <=> new names. Externalized so that user scripts that -# may need to consume multiple checkpoint formats can use this metadata. -RNN_NAME_REPLACEMENTS = collections.OrderedDict([ - ############################################################################ - # contrib/rnn/python/ops/core_rnn_cell_impl.py - # BasicRNNCell - ('basic_rnn_cell/weights', 'basic_rnn_cell/kernel'), - ('basic_rnn_cell/biases', 'basic_rnn_cell/bias'), - # GRUCell - ('gru_cell/weights', 'gru_cell/kernel'), - ('gru_cell/biases', 'gru_cell/bias'), - ('gru_cell/gates/weights', 'gru_cell/gates/kernel'), - ('gru_cell/gates/biases', 'gru_cell/gates/bias'), - ('gru_cell/candidate/weights', 'gru_cell/candidate/kernel'), - ('gru_cell/candidate/biases', 'gru_cell/candidate/bias'), - # BasicLSTMCell - ('basic_lstm_cell/weights', 'basic_lstm_cell/kernel'), - ('basic_lstm_cell/biases', 'basic_lstm_cell/bias'), - # LSTMCell - ('lstm_cell/weights', 'lstm_cell/kernel'), - ('lstm_cell/biases', 'lstm_cell/bias'), - ('lstm_cell/projection/weights', 'lstm_cell/projection/kernel'), - ('lstm_cell/projection/biases', 'lstm_cell/projection/bias'), - # OutputProjectionWrapper - ('output_projection_wrapper/weights', 'output_projection_wrapper/kernel'), - ('output_projection_wrapper/biases', 'output_projection_wrapper/bias'), - # InputProjectionWrapper - ('input_projection_wrapper/weights', 'input_projection_wrapper/kernel'), - ('input_projection_wrapper/biases', 'input_projection_wrapper/bias'), - ############################################################################ - # contrib/rnn/python/ops/lstm_ops.py - # LSTMBlockFusedCell ?? - ('lstm_block_wrapper/weights', 'lstm_block_wrapper/kernel'), - ('lstm_block_wrapper/biases', 'lstm_block_wrapper/bias'), - ############################################################################ - # contrib/rnn/python/ops/rnn_cell.py - # LayerNormBasicLSTMCell - ('layer_norm_basic_lstm_cell/weights', 'layer_norm_basic_lstm_cell/kernel'), - ('layer_norm_basic_lstm_cell/biases', 'layer_norm_basic_lstm_cell/bias'), - # UGRNNCell, not found in g3, but still need it? - ('ugrnn_cell/weights', 'ugrnn_cell/kernel'), - ('ugrnn_cell/biases', 'ugrnn_cell/bias'), - # NASCell - ('nas_rnn/weights', 'nas_rnn/kernel'), - ('nas_rnn/recurrent_weights', 'nas_rnn/recurrent_kernel'), - # IntersectionRNNCell - ('intersection_rnn_cell/weights', 'intersection_rnn_cell/kernel'), - ('intersection_rnn_cell/biases', 'intersection_rnn_cell/bias'), - ('intersection_rnn_cell/in_projection/weights', - 'intersection_rnn_cell/in_projection/kernel'), - ('intersection_rnn_cell/in_projection/biases', - 'intersection_rnn_cell/in_projection/bias'), - # PhasedLSTMCell - ('phased_lstm_cell/mask_gates/weights', - 'phased_lstm_cell/mask_gates/kernel'), - ('phased_lstm_cell/mask_gates/biases', 'phased_lstm_cell/mask_gates/bias'), - ('phased_lstm_cell/new_input/weights', 'phased_lstm_cell/new_input/kernel'), - ('phased_lstm_cell/new_input/biases', 'phased_lstm_cell/new_input/bias'), - ('phased_lstm_cell/output_gate/weights', - 'phased_lstm_cell/output_gate/kernel'), - ('phased_lstm_cell/output_gate/biases', - 'phased_lstm_cell/output_gate/bias'), - # AttentionCellWrapper - ('attention_cell_wrapper/weights', 'attention_cell_wrapper/kernel'), - ('attention_cell_wrapper/biases', 'attention_cell_wrapper/bias'), - ('attention_cell_wrapper/attn_output_projection/weights', - 'attention_cell_wrapper/attn_output_projection/kernel'), - ('attention_cell_wrapper/attn_output_projection/biases', - 'attention_cell_wrapper/attn_output_projection/bias'), - ('attention_cell_wrapper/attention/weights', - 'attention_cell_wrapper/attention/kernel'), - ('attention_cell_wrapper/attention/biases', - 'attention_cell_wrapper/attention/bias'), - ############################################################################ - # contrib/legacy_seq2seq/python/ops/seq2seq.py - ('attention_decoder/weights', 'attention_decoder/kernel'), - ('attention_decoder/biases', 'attention_decoder/bias'), - ('attention_decoder/Attention_0/weights', - 'attention_decoder/Attention_0/kernel'), - ('attention_decoder/Attention_0/biases', - 'attention_decoder/Attention_0/bias'), - ('attention_decoder/AttnOutputProjection/weights', - 'attention_decoder/AttnOutputProjection/kernel'), - ('attention_decoder/AttnOutputProjection/biases', - 'attention_decoder/AttnOutputProjection/bias'), - # contrib/legacy_seq2seq/python/ops/seq2seq.py before cl/140060366 - ('attention_decoder/Attention_0/Linear/Bias', - 'attention_decoder/Attention_0/bias'), - ('attention_decoder/Attention_0/Linear/Matrix', - 'attention_decoder/Attention_0/kernel'), - ('attention_decoder/AttnOutputProjection/Linear/Bias', - 'attention_decoder/AttnOutputProjection/bias'), - ('attention_decoder/AttnOutputProjection/Linear/Matrix', - 'attention_decoder/AttnOutputProjection/kernel'), - ('attention_decoder/LSTMCell/B', 'attention_decoder/lstm_cell/bias'), - ('attention_decoder/LSTMCell/W_0', 'attention_decoder/lstm_cell/kernel'), - ('attention_decoder/Linear/Bias', 'attention_decoder/bias'), - ('attention_decoder/Linear/Matrix', 'attention_decoder/kernel') -]) - -_RNN_SHARDED_NAME_REPLACEMENTS = collections.OrderedDict([ - ('LSTMCell/W_', 'lstm_cell/weights/part_'), - ('BasicLSTMCell/Linear/Matrix_', 'basic_lstm_cell/weights/part_'), - ('GRUCell/W_', 'gru_cell/weights/part_'), - ('MultiRNNCell/Cell', 'multi_rnn_cell/cell_'), -]) - - -def _rnn_name_replacement(var_name): - for pattern in RNN_NAME_REPLACEMENTS: - if pattern in var_name: - old_var_name = var_name - var_name = var_name.replace(pattern, RNN_NAME_REPLACEMENTS[pattern]) - logging.info('Converted: %s --> %s' % (old_var_name, var_name)) - break - return var_name - - -def _rnn_name_replacement_sharded(var_name): - for pattern in _RNN_SHARDED_NAME_REPLACEMENTS: - if pattern in var_name: - old_var_name = var_name - var_name = var_name.replace(pattern, - _RNN_SHARDED_NAME_REPLACEMENTS[pattern]) - logging.info('Converted: %s --> %s' % (old_var_name, var_name)) - return var_name - - -def _split_sharded_vars(name_shape_map): - """Split shareded variables. - - Args: - name_shape_map: A dict from variable name to variable shape. - - Returns: - not_sharded: Names of the non-sharded variables. - sharded: Names of the sharded variables. - """ - sharded = [] - not_sharded = [] - for name in name_shape_map: - if re.match(name, '_[0-9]+$'): - if re.sub('_[0-9]+$', '_1', name) in name_shape_map: - sharded.append(name) - else: - not_sharded.append(name) - else: - not_sharded.append(name) - return not_sharded, sharded - - -def convert_names(checkpoint_from_path, - checkpoint_to_path, - write_v1_checkpoint=False): - """Migrates the names of variables within a checkpoint. - - Args: - checkpoint_from_path: Path to source checkpoint to be read in. - checkpoint_to_path: Path to checkpoint to be written out. - write_v1_checkpoint: Whether the output checkpoint will be in V1 format. - - Returns: - A dictionary that maps the new variable names to the Variable objects. - A dictionary that maps the old variable names to the new variable names. - """ - with ops.Graph().as_default(): - logging.info('Reading checkpoint_from_path %s' % checkpoint_from_path) - reader = pywrap_tensorflow.NewCheckpointReader(checkpoint_from_path) - name_shape_map = reader.get_variable_to_shape_map() - not_sharded, sharded = _split_sharded_vars(name_shape_map) - new_variable_map = {} - conversion_map = {} - for var_name in not_sharded: - new_var_name = _rnn_name_replacement(var_name) - tensor = reader.get_tensor(var_name) - var = variables.Variable(tensor, name=var_name) - new_variable_map[new_var_name] = var - if new_var_name != var_name: - conversion_map[var_name] = new_var_name - for var_name in sharded: - new_var_name = _rnn_name_replacement_sharded(var_name) - var = variables.Variable(tensor, name=var_name) - new_variable_map[new_var_name] = var - if new_var_name != var_name: - conversion_map[var_name] = new_var_name - - write_version = (saver_pb2.SaverDef.V1 - if write_v1_checkpoint else saver_pb2.SaverDef.V2) - saver = saver_lib.Saver(new_variable_map, write_version=write_version) - - with session.Session() as sess: - sess.run(variables.global_variables_initializer()) - logging.info('Writing checkpoint_to_path %s' % checkpoint_to_path) - saver.save(sess, checkpoint_to_path) - - logging.info('Summary:') - logging.info(' Converted %d variable name(s).' % len(new_variable_map)) - return new_variable_map, conversion_map - - -def main(_): - convert_names( - FLAGS.checkpoint_from_path, - FLAGS.checkpoint_to_path, - write_v1_checkpoint=FLAGS.write_v1_checkpoint) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.register('type', 'bool', lambda v: v.lower() == 'true') - parser.add_argument('checkpoint_from_path', type=str, - help='Path to source checkpoint to be read in.') - parser.add_argument('checkpoint_to_path', type=str, - help='Path to checkpoint to be written out.') - parser.add_argument('--write_v1_checkpoint', action='store_true', - help='Write v1 checkpoint') - FLAGS, unparsed = parser.parse_known_args() - - app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/rnn/python/tools/checkpoint_convert_test.py b/tensorflow/contrib/rnn/python/tools/checkpoint_convert_test.py deleted file mode 100644 index b4785ee395a..00000000000 --- a/tensorflow/contrib/rnn/python/tools/checkpoint_convert_test.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Unit tests for checkpoint converter.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import glob -import os -import tempfile - -from tensorflow.contrib.rnn.python.tools import checkpoint_convert -from tensorflow.python.client import session -from tensorflow.python.framework import ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import saver as saver_lib - - -class CheckpointConvertTest(test.TestCase): - - def setUp(self): - self._old_ckpt_path = tempfile.mktemp() - self._new_ckpt_path = tempfile.mktemp() - ops.reset_default_graph() - - def tearDown(self): - for file_name in glob.glob(self._old_ckpt_path + "*"): - os.remove(file_name) - for file_name in glob.glob(self._new_ckpt_path + "*"): - os.remove(file_name) - - def testReplacementDictsContainUniqueAndNonEmptyVariableNames(self): - for old_name in checkpoint_convert.RNN_NAME_REPLACEMENTS: - new_name = checkpoint_convert.RNN_NAME_REPLACEMENTS[old_name] - self.assertTrue(old_name) - self.assertTrue(new_name) - self.assertNotEqual(old_name, new_name) - for old_name in checkpoint_convert._RNN_SHARDED_NAME_REPLACEMENTS: - new_name = checkpoint_convert._RNN_SHARDED_NAME_REPLACEMENTS[old_name] - self.assertTrue(old_name) - self.assertTrue(new_name) - self.assertNotEqual(old_name, new_name) - - def testConversionFromV2WithConvertedVariableNamesSucceeds(self): - variables.Variable(10.0, name="a") - for old_name in checkpoint_convert.RNN_NAME_REPLACEMENTS: - variables.Variable(20.0, name=old_name) - with session.Session() as sess: - saver = saver_lib.Saver() - sess.run(variables.global_variables_initializer()) - saver.save(sess, self._old_ckpt_path) - - new_var_map, conversion_map = checkpoint_convert.convert_names( - self._old_ckpt_path, self._new_ckpt_path) - self.assertTrue(glob.glob(self._new_ckpt_path + "*")) - self.assertItemsEqual( - set(checkpoint_convert.RNN_NAME_REPLACEMENTS.values()).union(["a"]), - new_var_map.keys()) - self.assertEqual(checkpoint_convert.RNN_NAME_REPLACEMENTS, conversion_map) - - def testConversionFromV2WithoutConvertedVariableNamesSucceeds(self): - variables.Variable(10.0, name="a") - with session.Session() as sess: - saver = saver_lib.Saver() - sess.run(variables.global_variables_initializer()) - saver.save(sess, self._old_ckpt_path) - - new_var_map, conversion_map = checkpoint_convert.convert_names( - self._old_ckpt_path, self._new_ckpt_path) - self.assertItemsEqual(["a"], new_var_map.keys()) - self.assertFalse(conversion_map) - - def testConversionToV1Succeeds(self): - variables.Variable(10.0, name="a") - variables.Variable( - 20.0, name=list(checkpoint_convert.RNN_NAME_REPLACEMENTS.keys())[-1]) - - with session.Session() as sess: - saver = saver_lib.Saver() - sess.run(variables.global_variables_initializer()) - saver.save(sess, self._old_ckpt_path) - - new_var_map, conversion_map = checkpoint_convert.convert_names( - self._old_ckpt_path, self._new_ckpt_path, write_v1_checkpoint=True) - self.assertItemsEqual( - ["a", list(checkpoint_convert.RNN_NAME_REPLACEMENTS.values())[-1]], - new_var_map.keys()) - self.assertEqual( - {list(checkpoint_convert.RNN_NAME_REPLACEMENTS.keys())[-1]: - list(checkpoint_convert.RNN_NAME_REPLACEMENTS.values())[-1]}, - conversion_map) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/rpc/BUILD b/tensorflow/contrib/rpc/BUILD deleted file mode 100644 index f092af17a90..00000000000 --- a/tensorflow/contrib/rpc/BUILD +++ /dev/null @@ -1,30 +0,0 @@ -load("//tensorflow/core/platform:default/build_config_root.bzl", "if_static") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "rpc", - srcs = [ - "__init__.py", - ], - deps = ["//tensorflow/contrib/rpc/python/ops:rpc_op_py"], -) - -py_library( - name = "rpc_pip", - data = if_static( - [], - otherwise = ["//tensorflow/contrib/rpc/python/kernel_tests:libtestexample.so"], - ), - deps = [ - ":rpc", - "//tensorflow/contrib/rpc/python/kernel_tests:py_test_deps", - "//tensorflow/contrib/rpc/python/kernel_tests:rpc_op_test_base", - "//tensorflow/contrib/rpc/python/kernel_tests:rpc_op_test_servicer", - ], -) diff --git a/tensorflow/contrib/rpc/__init__.py b/tensorflow/contrib/rpc/__init__.py deleted file mode 100644 index c65c1a05def..00000000000 --- a/tensorflow/contrib/rpc/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops and modules related to RPC. - -@@rpc -@@try_rpc -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.rpc.python.ops.rpc_op import rpc -from tensorflow.contrib.rpc.python.ops.rpc_op import try_rpc - -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/rpc/python/kernel_tests/BUILD b/tensorflow/contrib/rpc/python/kernel_tests/BUILD deleted file mode 100644 index db197d10cd8..00000000000 --- a/tensorflow/contrib/rpc/python/kernel_tests/BUILD +++ /dev/null @@ -1,78 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "tf_py_test") -load("//tensorflow:tensorflow.bzl", "tf_cc_shared_object") -load("//tensorflow/core/platform:default/build_config_root.bzl", "if_static") -load("//tensorflow/core/platform:default/build_config.bzl", "tf_proto_library") -# Placeholder for loading internal BUILD rule. - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_proto_library( - name = "test_example_proto", - srcs = ["test_example.proto"], - has_services = 1, - cc_api_version = 2, -) - -py_library( - name = "py_test_deps", - deps = [":test_example_proto_py"], -) - -py_library( - name = "rpc_op_test_base", - srcs = ["rpc_op_test_base.py"], - deps = [ - ":test_example_proto_py", - "//tensorflow/contrib/proto", - "//tensorflow/contrib/rpc", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//third_party/py/numpy", - ], -) - -py_library( - name = "rpc_op_test_servicer", - srcs = ["rpc_op_test_servicer.py"], - deps = [ - ":py_test_deps", - ":rpc_op_test_base", - "//tensorflow/core:protos_all_py", - "//third_party/py/numpy", - ], -) - -tf_cc_shared_object( - name = "libtestexample.so", - linkstatic = 1, - deps = [ - ":test_example_proto_cc", - ], -) - -tf_py_test( - name = "rpc_op_test", - size = "small", - srcs = ["rpc_op_test.py"], - additional_deps = [ - ":py_test_deps", - ":rpc_op_test_base", - ":rpc_op_test_servicer", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client_testlib", - ], - data = if_static( - [], - otherwise = [":libtestexample.so"], - ), - tags = [ - "no_pip", # TODO(b/78026780) - "no_windows", # TODO(b/78028010) - ], -) diff --git a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py deleted file mode 100644 index 110025030b7..00000000000 --- a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Tests for RpcOp.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import ctypes as ct -import os - -import grpc -from grpc.framework.foundation import logging_pool -import portpicker - -from tensorflow.contrib.rpc.python.kernel_tests import rpc_op_test_base -from tensorflow.contrib.rpc.python.kernel_tests import rpc_op_test_servicer -from tensorflow.contrib.rpc.python.kernel_tests import test_example_pb2_grpc -from tensorflow.python.platform import test - - -class RpcOpTest(test.TestCase, rpc_op_test_base.RpcOpTestBase): - _protocol = 'grpc' - - invalid_method_string = 'Method not found' - - def __init__(self, methodName='runTest'): # pylint: disable=invalid-name - super(RpcOpTest, self).__init__(methodName) - lib = os.path.join(os.path.dirname(__file__), 'libtestexample.so') - if os.path.isfile(lib): - ct.cdll.LoadLibrary(lib) - - def get_method_name(self, suffix): - return '/tensorflow.contrib.rpc.TestCaseService/%s' % suffix - - def setUp(self): - super(RpcOpTest, self).setUp() - - service_port = portpicker.pick_unused_port() - - server = grpc.server(logging_pool.pool(max_workers=25)) - servicer = rpc_op_test_servicer.RpcOpTestServicer() - test_example_pb2_grpc.add_TestCaseServiceServicer_to_server( - servicer, server) - self._address = 'localhost:%d' % service_port - server.add_insecure_port(self._address) - server.start() - self._server = server - - def tearDown(self): - self._server.stop(grace=None) - super(RpcOpTest, self).tearDown() - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py deleted file mode 100644 index ec073011f75..00000000000 --- a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py +++ /dev/null @@ -1,345 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Base class for RpcOp tests.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools - -import numpy as np - -from tensorflow.contrib.rpc.python.kernel_tests import test_example_pb2 -from tensorflow.contrib.rpc.python.ops import rpc_op -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.ops import proto_ops - -__all__ = ['I_WARNED_YOU', 'RpcOpTestBase'] - -I_WARNED_YOU = 'I warned you!' - - -class RpcOpTestBase(object): - # pylint: disable=missing-docstring,invalid-name - """Base class for RpcOp tests.""" - - def get_method_name(self, suffix): - raise NotImplementedError - - def rpc(self, *args, **kwargs): - return rpc_op.rpc(*args, protocol=self._protocol, **kwargs) - - def try_rpc(self, *args, **kwargs): - return rpc_op.try_rpc(*args, protocol=self._protocol, **kwargs) - - def testScalarHostPortRpc(self): - with self.cached_session() as sess: - request_tensors = ( - test_example_pb2.TestCase(values=[1, 2, 3]).SerializeToString()) - response_tensors = self.rpc( - method=self.get_method_name('Increment'), - address=self._address, - request=request_tensors) - self.assertEqual(response_tensors.shape, ()) - response_values = sess.run(response_tensors) - response_message = test_example_pb2.TestCase() - self.assertTrue(response_message.ParseFromString(response_values)) - self.assertAllEqual([2, 3, 4], response_message.values) - - def testScalarHostPortTryRpc(self): - with self.cached_session() as sess: - request_tensors = ( - test_example_pb2.TestCase(values=[1, 2, 3]).SerializeToString()) - response_tensors, status_code, status_message = self.try_rpc( - method=self.get_method_name('Increment'), - address=self._address, - request=request_tensors) - self.assertEqual(status_code.shape, ()) - self.assertEqual(status_message.shape, ()) - self.assertEqual(response_tensors.shape, ()) - response_values, status_code_values, status_message_values = ( - sess.run((response_tensors, status_code, status_message))) - response_message = test_example_pb2.TestCase() - self.assertTrue(response_message.ParseFromString(response_values)) - self.assertAllEqual([2, 3, 4], response_message.values) - # For the base Rpc op, don't expect to get error status back. - self.assertEqual(errors.OK, status_code_values) - self.assertEqual(b'', status_message_values) - - def testEmptyHostPortRpc(self): - with self.cached_session() as sess: - request_tensors = [] - response_tensors = self.rpc( - method=self.get_method_name('Increment'), - address=self._address, - request=request_tensors) - self.assertAllEqual(response_tensors.shape, [0]) - response_values = sess.run(response_tensors) - self.assertAllEqual(response_values.shape, [0]) - - def testInvalidMethod(self): - for method in [ - '/InvalidService.Increment', - self.get_method_name('InvalidMethodName') - ]: - with self.cached_session() as sess: - with self.assertRaisesOpError(self.invalid_method_string): - sess.run(self.rpc(method=method, address=self._address, request='')) - - _, status_code_value, status_message_value = sess.run( - self.try_rpc(method=method, address=self._address, request='')) - self.assertEqual(errors.UNIMPLEMENTED, status_code_value) - self.assertTrue( - self.invalid_method_string in status_message_value.decode('ascii')) - - def testInvalidAddress(self): - # This covers the case of address='' and address='localhost:293874293874' - address = 'unix:/tmp/this_unix_socket_doesnt_exist_97820348!!@' - with self.cached_session() as sess: - with self.assertRaises(errors.UnavailableError): - sess.run( - self.rpc( - method=self.get_method_name('Increment'), - address=address, - request='')) - _, status_code_value, status_message_value = sess.run( - self.try_rpc( - method=self.get_method_name('Increment'), - address=address, - request='')) - self.assertEqual(errors.UNAVAILABLE, status_code_value) - - def testAlwaysFailingMethod(self): - with self.cached_session() as sess: - response_tensors = self.rpc( - method=self.get_method_name('AlwaysFailWithInvalidArgument'), - address=self._address, - request='') - self.assertEqual(response_tensors.shape, ()) - with self.assertRaisesOpError(I_WARNED_YOU): - sess.run(response_tensors) - - response_tensors, status_code, status_message = self.try_rpc( - method=self.get_method_name('AlwaysFailWithInvalidArgument'), - address=self._address, - request='') - self.assertEqual(response_tensors.shape, ()) - self.assertEqual(status_code.shape, ()) - self.assertEqual(status_message.shape, ()) - status_code_value, status_message_value = sess.run((status_code, - status_message)) - self.assertEqual(errors.INVALID_ARGUMENT, status_code_value) - self.assertTrue(I_WARNED_YOU in status_message_value.decode('ascii')) - - def testSometimesFailingMethodWithManyRequests(self): - with self.cached_session() as sess: - # Fail hard by default. - response_tensors = self.rpc( - method=self.get_method_name('SometimesFailWithInvalidArgument'), - address=self._address, - request=[''] * 20) - self.assertEqual(response_tensors.shape, (20,)) - with self.assertRaisesOpError(I_WARNED_YOU): - sess.run(response_tensors) - - # Don't fail hard, use TryRpc - return the failing status instead. - response_tensors, status_code, status_message = self.try_rpc( - method=self.get_method_name('SometimesFailWithInvalidArgument'), - address=self._address, - request=[''] * 20) - self.assertEqual(response_tensors.shape, (20,)) - self.assertEqual(status_code.shape, (20,)) - self.assertEqual(status_message.shape, (20,)) - status_code_values, status_message_values = sess.run((status_code, - status_message)) - self.assertTrue([ - x in (errors.OK, errors.INVALID_ARGUMENT) for x in status_code_values - ]) - expected_message_values = np.where( - status_code_values == errors.INVALID_ARGUMENT, - I_WARNED_YOU.encode('ascii'), b'') - for msg, expected in zip(status_message_values, expected_message_values): - self.assertTrue(expected in msg, - '"%s" did not contain "%s"' % (msg, expected)) - - def testVecHostPortRpc(self): - with self.cached_session() as sess: - request_tensors = [ - test_example_pb2.TestCase( - values=[i, i + 1, i + 2]).SerializeToString() for i in range(20) - ] - response_tensors = self.rpc( - method=self.get_method_name('Increment'), - address=self._address, - request=request_tensors) - self.assertEqual(response_tensors.shape, (20,)) - response_values = sess.run(response_tensors) - self.assertEqual(response_values.shape, (20,)) - for i in range(20): - response_message = test_example_pb2.TestCase() - self.assertTrue(response_message.ParseFromString(response_values[i])) - self.assertAllEqual([i + 1, i + 2, i + 3], response_message.values) - - def testVecHostPortManyParallelRpcs(self): - with self.cached_session() as sess: - request_tensors = [ - test_example_pb2.TestCase( - values=[i, i + 1, i + 2]).SerializeToString() for i in range(20) - ] - many_response_tensors = [ - self.rpc( - method=self.get_method_name('Increment'), - address=self._address, - request=request_tensors) for _ in range(10) - ] - # Launch parallel 10 calls to the RpcOp, each containing 20 rpc requests. - many_response_values = sess.run(many_response_tensors) - self.assertEqual(10, len(many_response_values)) - for response_values in many_response_values: - self.assertEqual(response_values.shape, (20,)) - for i in range(20): - response_message = test_example_pb2.TestCase() - self.assertTrue(response_message.ParseFromString(response_values[i])) - self.assertAllEqual([i + 1, i + 2, i + 3], response_message.values) - - def testVecHostPortRpcUsingEncodeAndDecodeProto(self): - with self.cached_session() as sess: - request_tensors = proto_ops.encode_proto( - message_type='tensorflow.contrib.rpc.TestCase', - field_names=['values'], - sizes=[[3]] * 20, - values=[ - [[i, i + 1, i + 2] for i in range(20)], - ]) - response_tensor_strings = self.rpc( - method=self.get_method_name('Increment'), - address=self._address, - request=request_tensors) - _, (response_shape,) = proto_ops.decode_proto( - bytes=response_tensor_strings, - message_type='tensorflow.contrib.rpc.TestCase', - field_names=['values'], - output_types=[dtypes.int32]) - response_shape_values = sess.run(response_shape) - self.assertAllEqual([[i + 1, i + 2, i + 3] - for i in range(20)], response_shape_values) - - def testVecHostPortRpcCancelsUponSessionTimeOutWhenSleepingForever(self): - with self.cached_session() as sess: - request_tensors = [''] * 25 # This will launch 25 RPC requests. - response_tensors = self.rpc( - method=self.get_method_name('SleepForever'), - address=self._address, - request=request_tensors) - for timeout_ms in [1, 500, 1000]: - options = config_pb2.RunOptions(timeout_in_ms=timeout_ms) - with self.assertRaises((errors.UnavailableError, - errors.DeadlineExceededError)): - sess.run(response_tensors, options=options) - - def testVecHostPortRpcCancelsUponConfiguredTimeOutWhenSleepingForever(self): - with self.cached_session() as sess: - request_tensors = [''] * 25 # This will launch 25 RPC requests. - response_tensors = self.rpc( - method=self.get_method_name('SleepForever'), - address=self._address, - timeout_in_ms=1000, - request=request_tensors) - with self.assertRaises(errors.DeadlineExceededError): - sess.run(response_tensors) - - def testTryRpcPropagatesDeadlineErrorWithSometimesTimingOutRequests(self): - with self.cached_session() as sess: - response_tensors, status_code, status_message = self.try_rpc( - method=self.get_method_name('SometimesSleepForever'), - timeout_in_ms=1000, - address=self._address, - request=[''] * 20) - self.assertEqual(response_tensors.shape, (20,)) - self.assertEqual(status_code.shape, (20,)) - self.assertEqual(status_message.shape, (20,)) - status_code_values = sess.run(status_code) - self.assertTrue([ - x in (errors.OK, errors.DEADLINE_EXCEEDED) for x in status_code_values - ]) - - def testTryRpcWithMultipleAddressesSingleRequest(self): - flatten = lambda x: list(itertools.chain.from_iterable(x)) - with self.cached_session() as sess: - addresses = flatten([[ - self._address, 'unix:/tmp/this_unix_socket_doesnt_exist_97820348!!@' - ] for _ in range(10)]) - request = test_example_pb2.TestCase(values=[0, 1, 2]).SerializeToString() - response_tensors, status_code, _ = self.try_rpc( - method=self.get_method_name('Increment'), - address=addresses, - request=request) - response_tensors_values, status_code_values = sess.run((response_tensors, - status_code)) - self.assertAllEqual( - flatten([errors.OK, errors.UNAVAILABLE] for _ in range(10)), - status_code_values) - for i in range(10): - self.assertTrue(response_tensors_values[2 * i]) - self.assertFalse(response_tensors_values[2 * i + 1]) - - def testTryRpcWithMultipleMethodsSingleRequest(self): - flatten = lambda x: list(itertools.chain.from_iterable(x)) - with self.cached_session() as sess: - methods = flatten( - [[self.get_method_name('Increment'), 'InvalidMethodName'] - for _ in range(10)]) - request = test_example_pb2.TestCase(values=[0, 1, 2]).SerializeToString() - response_tensors, status_code, _ = self.try_rpc( - method=methods, address=self._address, request=request) - response_tensors_values, status_code_values = sess.run((response_tensors, - status_code)) - self.assertAllEqual( - flatten([errors.OK, errors.UNIMPLEMENTED] for _ in range(10)), - status_code_values) - for i in range(10): - self.assertTrue(response_tensors_values[2 * i]) - self.assertFalse(response_tensors_values[2 * i + 1]) - - def testTryRpcWithMultipleAddressesAndRequests(self): - flatten = lambda x: list(itertools.chain.from_iterable(x)) - with self.cached_session() as sess: - addresses = flatten([[ - self._address, 'unix:/tmp/this_unix_socket_doesnt_exist_97820348!!@' - ] for _ in range(10)]) - requests = [ - test_example_pb2.TestCase( - values=[i, i + 1, i + 2]).SerializeToString() for i in range(20) - ] - response_tensors, status_code, _ = self.try_rpc( - method=self.get_method_name('Increment'), - address=addresses, - request=requests) - response_tensors_values, status_code_values = sess.run((response_tensors, - status_code)) - self.assertAllEqual( - flatten([errors.OK, errors.UNAVAILABLE] for _ in range(10)), - status_code_values) - for i in range(20): - if i % 2 == 1: - self.assertFalse(response_tensors_values[i]) - else: - response_message = test_example_pb2.TestCase() - self.assertTrue( - response_message.ParseFromString(response_tensors_values[i])) - self.assertAllEqual([i + 1, i + 2, i + 3], response_message.values) diff --git a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py deleted file mode 100644 index 265254aa51c..00000000000 --- a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Test servicer for RpcOp tests.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random -import time - -import grpc - -from tensorflow.contrib.rpc.python.kernel_tests import rpc_op_test_base -from tensorflow.contrib.rpc.python.kernel_tests import test_example_pb2_grpc - - -class RpcOpTestServicer(test_example_pb2_grpc.TestCaseServiceServicer): - """Test servicer for RpcOp tests.""" - - def Increment(self, request, context): - """Increment the entries in the `values` attribute of request. - - Args: - request: input TestCase. - context: the rpc context. - - Returns: - output TestCase. - """ - for i in range(len(request.values)): - request.values[i] += 1 - return request - - def AlwaysFailWithInvalidArgument(self, request, context): - """Always fails with an InvalidArgument status. - - Args: - request: input TestCase. - context: the rpc context. - - Returns: - output TestCase. - """ - del request - context.set_code(grpc.StatusCode.INVALID_ARGUMENT) - context.set_details(rpc_op_test_base.I_WARNED_YOU) - - def SometimesFailWithInvalidArgument(self, request, context): - """Sometimes fails with an InvalidArgument status. - - Args: - request: input TestCase. - context: the rpc context. - - Returns: - output TestCase. - """ - if random.randint(0, 1) == 1: - context.set_code(grpc.StatusCode.INVALID_ARGUMENT) - context.set_details(rpc_op_test_base.I_WARNED_YOU) - return request - - def SleepForever(self, request, context): - """Sleeps forever. - - Args: - request: input TestCase. - context: the rpc context. - - Returns: - output TestCase. - """ - # TODO(ebrevdo): Make this async wait like the stubby version. - time.sleep(5) - - def SometimesSleepForever(self, request, context): - """Sometimes sleeps forever. - - Args: - request: input TestCase. - context: the rpc context. - - Returns: - output TestCase. - """ - if random.randint(0, 1) == 1: - time.sleep(5) - return request diff --git a/tensorflow/contrib/rpc/python/kernel_tests/test_example.proto b/tensorflow/contrib/rpc/python/kernel_tests/test_example.proto deleted file mode 100644 index 8141466349a..00000000000 --- a/tensorflow/contrib/rpc/python/kernel_tests/test_example.proto +++ /dev/null @@ -1,32 +0,0 @@ -// Test description and protos to work with it. - -syntax = "proto2"; - -package tensorflow.contrib.rpc; - -// A TestCase holds a sequence of values. -message TestCase { - repeated int32 values = 1; -}; - -service TestCaseService { - // Copy input, and increment each entry in 'values' by 1. - rpc Increment(TestCase) returns (TestCase) { - } - - // Sleep forever. - rpc SleepForever(TestCase) returns (TestCase) { - } - - // Sleep forever 50% of the time, return immediately the other 50%. - rpc SometimesSleepForever(TestCase) returns (TestCase) { - } - - // Always fails with InvalidArgument. - rpc AlwaysFailWithInvalidArgument(TestCase) returns (TestCase) { - } - - // Fails with InvalidArgument 50% of the time. - rpc SometimesFailWithInvalidArgument(TestCase) returns (TestCase) { - } -}; diff --git a/tensorflow/contrib/rpc/python/ops/BUILD b/tensorflow/contrib/rpc/python/ops/BUILD deleted file mode 100644 index 4330928bd46..00000000000 --- a/tensorflow/contrib/rpc/python/ops/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "rpc_op_py", - srcs = ["rpc_op.py"], - deps = [ - ":gen_rpc_op_py", - "//tensorflow/python:framework_ops", - ], -) - -tf_gen_op_wrapper_py( - name = "gen_rpc_op_py", - out = "gen_rpc_op.py", - deps = [ - "//tensorflow/core:rpc_ops_op_lib", - ], -) diff --git a/tensorflow/contrib/rpc/python/ops/rpc_op.py b/tensorflow/contrib/rpc/python/ops/rpc_op.py deleted file mode 100644 index e1b6c411378..00000000000 --- a/tensorflow/contrib/rpc/python/ops/rpc_op.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -# pylint: disable=wildcard-import,unused-import -"""RPC communication.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.rpc.python.ops.gen_rpc_op import rpc -from tensorflow.contrib.rpc.python.ops.gen_rpc_op import try_rpc -from tensorflow.python.framework import ops -ops.NotDifferentiable("Rpc") -ops.NotDifferentiable("TryRpc") diff --git a/tensorflow/contrib/saved_model/BUILD b/tensorflow/contrib/saved_model/BUILD deleted file mode 100644 index 3b10de6b06e..00000000000 --- a/tensorflow/contrib/saved_model/BUILD +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# Description: -# SavedModel contrib libraries. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "saved_model_py", - srcs = [ - "__init__.py", - "python/__init__.py", - ] + glob( - ["python/saved_model/*.py"], - exclude = ["python/saved_model/*_test.py"], - ), - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":keras_saved_model", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:framework_ops", - "//tensorflow/python:lib", - "//tensorflow/python:util", - "//tensorflow/python/saved_model:builder", - "//tensorflow/python/saved_model:constants", - "//tensorflow/python/saved_model:signature_constants", - "//tensorflow/python/saved_model:signature_def_utils", - "//tensorflow/python/saved_model:tag_constants", - ], -) - -py_library( - name = "reader", - srcs = ["python/saved_model/reader.py"], - srcs_version = "PY2AND3", - tags = ["no_windows"], # TODO: needs investigation on Windows - visibility = ["//visibility:public"], - deps = [ - ":saved_model_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:lib", - "//tensorflow/python:variables", - "//tensorflow/python/saved_model:builder", - "//tensorflow/python/saved_model:tag_constants", - ], -) - -py_test( - name = "reader_test", - size = "small", - srcs = ["python/saved_model/reader_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_windows"], # TODO: needs investigation on Windows - visibility = ["//visibility:private"], - deps = [ - ":reader", - "//tensorflow/python:client_testlib", - ], -) - -py_library( - name = "keras_saved_model", - srcs = ["python/saved_model/keras_saved_model.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/python/keras", - ], -) diff --git a/tensorflow/contrib/saved_model/__init__.py b/tensorflow/contrib/saved_model/__init__.py deleted file mode 100644 index ac95e380114..00000000000 --- a/tensorflow/contrib/saved_model/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""SavedModel contrib support. - -SavedModel provides a language-neutral format to save machine-learned models -that is recoverable and hermetic. It enables higher-level systems and tools to -produce, consume and transform TensorFlow models. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import,line-too-long -from tensorflow.contrib.saved_model.python.saved_model.keras_saved_model import * -# pylint: enable=unused-import,wildcard-import,line-too-long - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = [ - "load_keras_model", - "save_keras_model"] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/saved_model/cc/saved_model/BUILD b/tensorflow/contrib/saved_model/cc/saved_model/BUILD deleted file mode 100644 index 20e80849cf5..00000000000 --- a/tensorflow/contrib/saved_model/cc/saved_model/BUILD +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# Description: -# SavedModel contrib libraries for C++. - -load("//tensorflow:tensorflow.bzl", "tf_cc_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -cc_library( - name = "signature_def_utils", - srcs = ["signature_def_utils.cc"], - hdrs = ["signature_def_utils.h"], - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/cc/saved_model:signature_constants", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_proto_parsing", - "//tensorflow/core:protos_all_cc", - ], -) - -tf_cc_test( - name = "signature_def_utils_test", - srcs = ["signature_def_utils_test.cc"], - deps = [ - ":signature_def_utils", - "//tensorflow/cc/saved_model:signature_constants", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_proto_parsing", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) diff --git a/tensorflow/contrib/saved_model/cc/saved_model/signature_def_utils.cc b/tensorflow/contrib/saved_model/cc/saved_model/signature_def_utils.cc deleted file mode 100644 index e87e497e5ff..00000000000 --- a/tensorflow/contrib/saved_model/cc/saved_model/signature_def_utils.cc +++ /dev/null @@ -1,158 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/saved_model/cc/saved_model/signature_def_utils.h" - -#include "tensorflow/cc/saved_model/signature_constants.h" -#include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/platform/protobuf.h" - -namespace tensorflow { - -namespace { -template -Status FindInProtobufMap(StringPiece description, - const protobuf::Map& map, const string& key, - const T** value) { - const auto it = map.find(key); - if (it == map.end()) { - return errors::NotFound("Could not find ", description, " for key: ", key); - } - *value = &it->second; - return Status::OK(); -} - -// Looks up the TensorInfo for the given key in the given map and verifies that -// its datatype matches the given correct datatype. -bool VerifyTensorInfoForKeyInMap(const protobuf::Map& map, - const string& key, DataType correct_dtype) { - const TensorInfo* tensor_info; - const Status& status = FindInProtobufMap("", map, key, &tensor_info); - if (!status.ok()) { - return false; - } - if (tensor_info->dtype() != correct_dtype) { - return false; - } - return true; -} - -bool IsValidPredictSignature(const SignatureDef& signature_def) { - if (signature_def.method_name() != kPredictMethodName) { - return false; - } - if (signature_def.inputs().empty()) { - return false; - } - if (signature_def.outputs().empty()) { - return false; - } - return true; -} - -bool IsValidRegressionSignature(const SignatureDef& signature_def) { - if (signature_def.method_name() != kRegressMethodName) { - return false; - } - if (!VerifyTensorInfoForKeyInMap(signature_def.inputs(), kRegressInputs, - DT_STRING)) { - return false; - } - if (!VerifyTensorInfoForKeyInMap(signature_def.outputs(), kRegressOutputs, - DT_FLOAT)) { - return false; - } - return true; -} - -bool IsValidClassificationSignature(const SignatureDef& signature_def) { - if (signature_def.method_name() != kClassifyMethodName) { - return false; - } - if (!VerifyTensorInfoForKeyInMap(signature_def.inputs(), kClassifyInputs, - DT_STRING)) { - return false; - } - if (signature_def.outputs().empty()) { - return false; - } - for (auto const& output : signature_def.outputs()) { - const string& key = output.first; - const TensorInfo& tensor_info = output.second; - if (key == kClassifyOutputClasses) { - if (tensor_info.dtype() != DT_STRING) { - return false; - } - } else if (key == kClassifyOutputScores) { - if (tensor_info.dtype() != DT_FLOAT) { - return false; - } - } else { - return false; - } - } - return true; -} - -} // namespace - -Status FindSignatureDefByKey(const MetaGraphDef& meta_graph_def, - const string& signature_def_key, - const SignatureDef** signature_def) { - return FindInProtobufMap("SignatureDef", meta_graph_def.signature_def(), - signature_def_key, signature_def); -} - -Status FindInputTensorInfoByKey(const SignatureDef& signature_def, - const string& tensor_info_key, - const TensorInfo** tensor_info) { - return FindInProtobufMap("input TensorInfo", signature_def.inputs(), - tensor_info_key, tensor_info); -} - -Status FindOutputTensorInfoByKey(const SignatureDef& signature_def, - const string& tensor_info_key, - const TensorInfo** tensor_info) { - return FindInProtobufMap("output TensorInfo", signature_def.outputs(), - tensor_info_key, tensor_info); -} - -Status FindInputTensorNameByKey(const SignatureDef& signature_def, - const string& tensor_info_key, string* name) { - const TensorInfo* tensor_info; - TF_RETURN_IF_ERROR( - FindInputTensorInfoByKey(signature_def, tensor_info_key, &tensor_info)); - *name = tensor_info->name(); - return Status::OK(); -} - -Status FindOutputTensorNameByKey(const SignatureDef& signature_def, - const string& tensor_info_key, string* name) { - const TensorInfo* tensor_info; - TF_RETURN_IF_ERROR( - FindOutputTensorInfoByKey(signature_def, tensor_info_key, &tensor_info)); - *name = tensor_info->name(); - return Status::OK(); -} - -bool IsValidSignature(const SignatureDef& signature_def) { - return IsValidClassificationSignature(signature_def) || - IsValidRegressionSignature(signature_def) || - IsValidPredictSignature(signature_def); -} - -} // namespace tensorflow diff --git a/tensorflow/contrib/saved_model/cc/saved_model/signature_def_utils.h b/tensorflow/contrib/saved_model/cc/saved_model/signature_def_utils.h deleted file mode 100644 index bb24faa989b..00000000000 --- a/tensorflow/contrib/saved_model/cc/saved_model/signature_def_utils.h +++ /dev/null @@ -1,72 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Helpers for working with the SignatureDefs of TensorFlow SavedModels. - -#ifndef TENSORFLOW_CONTRIB_SAVED_MODEL_CC_SAVED_MODEL_SIGNATURE_DEF_UTILS_H_ -#define TENSORFLOW_CONTRIB_SAVED_MODEL_CC_SAVED_MODEL_SIGNATURE_DEF_UTILS_H_ - -#include -#include -#include - -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/protobuf/meta_graph.pb.h" - -namespace tensorflow { - -// Finds the entry in meta_graph_def.signature_def with the given key, or -// returns NotFound and leaves *signature_def unchanged. NOTE: The output -// SignatureDef* points into meta_graph_def and may be invalidated by changes -// to that protocol buffer, as usual. -Status FindSignatureDefByKey(const MetaGraphDef& meta_graph_def, - const string& signature_def_key, - const SignatureDef** signature_def); - -// Finds the entry in signature_def.inputs with the given key, or returns -// NotFound and leaves *tensor_info unchanged. NOTE: The output TensorInfo* -// points into signature_def and may be invalidated by changes to that protocol -// buffer, as usual. -Status FindInputTensorInfoByKey(const SignatureDef& signature_def, - const string& tensor_info_key, - const TensorInfo** tensor_info); - -// Finds the entry in signature_def.outputs with the given key, or returns -// NotFound and leaves *tensor_info unchanged. NOTE: The output TensorInfo* -// points into signature_def and may be invalidated by changes to that protocol -// buffer, as usual. -Status FindOutputTensorInfoByKey(const SignatureDef& signature_def, - const string& tensor_info_key, - const TensorInfo** tensor_info); - -// Finds the entry in signature_def.inputs with the given key and copies out -// the name of this Tensor in the graph, or returns NotFound and leaves *name -// unchanged. -Status FindInputTensorNameByKey(const SignatureDef& signature_def, - const string& tensor_info_key, string* name); - -// Finds the entry in signature_def.outputs with the given key and copies out -// the name of this Tensor in the graph, or returns NotFound and leaves *name -// unchanged. -Status FindOutputTensorNameByKey(const SignatureDef& signature_def, - const string& tensor_info_key, string* name); - -// Determine whether a SignatureDef can be served by TensorFlow Serving. -bool IsValidSignature(const SignatureDef& signature_def); - -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_SAVED_MODEL_CC_SAVED_MODEL_SIGNATURE_DEF_UTILS_H_ diff --git a/tensorflow/contrib/saved_model/cc/saved_model/signature_def_utils_test.cc b/tensorflow/contrib/saved_model/cc/saved_model/signature_def_utils_test.cc deleted file mode 100644 index c743112ce06..00000000000 --- a/tensorflow/contrib/saved_model/cc/saved_model/signature_def_utils_test.cc +++ /dev/null @@ -1,199 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/saved_model/cc/saved_model/signature_def_utils.h" - -#include "tensorflow/cc/saved_model/signature_constants.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { - -class FindByKeyTest : public ::testing::Test { - protected: - MetaGraphDef MakeSampleMetaGraphDef() { - MetaGraphDef result; - (*result.mutable_signature_def())["blah"].set_method_name("foo"); - (*result.mutable_signature_def())[kSignatureKey] = MakeSampleSignatureDef(); - (*result.mutable_signature_def())["gnarl"].set_method_name("blah"); - return result; - } - - void SetInputNameForKey(const string& key, const string& name, - SignatureDef* signature_def) { - (*signature_def->mutable_inputs())[key].set_name(name); - } - - void SetOutputNameForKey(const string& key, const string& name, - SignatureDef* signature_def) { - (*signature_def->mutable_outputs())[key].set_name(name); - } - - SignatureDef MakeSampleSignatureDef() { - SignatureDef result; - result.set_method_name(kMethodName); - SetInputNameForKey(kInput1Key, kInput1Name, &result); - SetInputNameForKey(kInput2Key, kInput2Name, &result); - SetOutputNameForKey(kOutput1Key, kOutput1Name, &result); - SetOutputNameForKey(kOutput2Key, kOutput2Name, &result); - return result; - } - - const string kSignatureKey = "my_signature"; - const string kMethodName = "my_method"; - const string kInput1Key = "input_one_key"; - const string kInput1Name = "input_one"; - const string kInput2Key = "input_two_key"; - const string kInput2Name = "input_two"; - const string kOutput1Key = "output_one_key"; - const string kOutput1Name = "output_one"; - const string kOutput2Key = "output_two_key"; - const string kOutput2Name = "output_two"; -}; - -TEST_F(FindByKeyTest, FindSignatureDefByKey) { - const MetaGraphDef meta_graph_def = MakeSampleMetaGraphDef(); - const SignatureDef* signature_def; - // Succeeds for an existing signature. - TF_ASSERT_OK( - FindSignatureDefByKey(meta_graph_def, kSignatureKey, &signature_def)); - EXPECT_EQ(kMethodName, signature_def->method_name()); - // Fails for a missing signature. - EXPECT_FALSE( - FindSignatureDefByKey(meta_graph_def, "nonexistent", &signature_def) - .ok()); -} - -TEST_F(FindByKeyTest, FindInputTensorNameByKey) { - const SignatureDef signature_def = MakeSampleSignatureDef(); - string name; - // Succeeds for an existing input. - TF_ASSERT_OK(FindInputTensorNameByKey(signature_def, kInput2Key, &name)); - EXPECT_EQ(kInput2Name, name); - // Fails for a missing input. - EXPECT_FALSE( - FindInputTensorNameByKey(signature_def, "nonexistent", &name).ok()); -} - -TEST_F(FindByKeyTest, FindOutputTensorNameByKey) { - const SignatureDef signature_def = MakeSampleSignatureDef(); - string name; - // Succeeds for an existing output. - TF_ASSERT_OK(FindOutputTensorNameByKey(signature_def, kOutput2Key, &name)); - EXPECT_EQ(kOutput2Name, name); - // Fails for a missing output. - EXPECT_FALSE( - FindOutputTensorNameByKey(signature_def, "nonexistent", &name).ok()); -} - -class IsValidSignatureTest : public ::testing::Test { - protected: - void SetInputDataTypeForKey(const string& key, DataType dtype) { - (*signature_def_.mutable_inputs())[key].set_dtype(dtype); - } - - void SetOutputDataTypeForKey(const string& key, DataType dtype) { - (*signature_def_.mutable_outputs())[key].set_dtype(dtype); - } - - void EraseOutputKey(const string& key) { - (*signature_def_.mutable_outputs()).erase(key); - } - - void ExpectInvalidSignature() { - EXPECT_FALSE(IsValidSignature(signature_def_)); - } - - void ExpectValidSignature() { EXPECT_TRUE(IsValidSignature(signature_def_)); } - - SignatureDef signature_def_; -}; - -TEST_F(IsValidSignatureTest, IsValidPredictSignature) { - signature_def_.set_method_name("not_kPredictMethodName"); - // Incorrect method name - ExpectInvalidSignature(); - - signature_def_.set_method_name(kPredictMethodName); - // No inputs - ExpectInvalidSignature(); - - SetInputDataTypeForKey(kPredictInputs, DT_STRING); - // No outputs - ExpectInvalidSignature(); - - SetOutputDataTypeForKey(kPredictOutputs, DT_STRING); - ExpectValidSignature(); -} - -TEST_F(IsValidSignatureTest, IsValidRegressionSignature) { - signature_def_.set_method_name("not_kRegressMethodName"); - // Incorrect method name - ExpectInvalidSignature(); - - signature_def_.set_method_name(kRegressMethodName); - // No inputs - ExpectInvalidSignature(); - - SetInputDataTypeForKey(kRegressInputs, DT_STRING); - // No outputs - ExpectInvalidSignature(); - - SetOutputDataTypeForKey(kRegressOutputs, DT_STRING); - // Incorrect data type - ExpectInvalidSignature(); - - SetOutputDataTypeForKey(kRegressOutputs, DT_FLOAT); - ExpectValidSignature(); -} - -TEST_F(IsValidSignatureTest, IsValidClassificationSignature) { - signature_def_.set_method_name("not_kClassifyMethodName"); - // Incorrect method name - ExpectInvalidSignature(); - - signature_def_.set_method_name(kClassifyMethodName); - // No inputs - ExpectInvalidSignature(); - - SetInputDataTypeForKey(kClassifyInputs, DT_STRING); - // No outputs - ExpectInvalidSignature(); - - SetOutputDataTypeForKey("invalidKey", DT_FLOAT); - // Invalid key - ExpectInvalidSignature(); - - EraseOutputKey("invalidKey"); - SetOutputDataTypeForKey(kClassifyOutputClasses, DT_FLOAT); - // Invalid dtype for classes - ExpectInvalidSignature(); - - SetOutputDataTypeForKey(kClassifyOutputClasses, DT_STRING); - // Valid without scores - ExpectValidSignature(); - - SetOutputDataTypeForKey(kClassifyOutputScores, DT_STRING); - // Invalid dtype for scores - ExpectInvalidSignature(); - - SetOutputDataTypeForKey(kClassifyOutputScores, DT_FLOAT); - // Valid with both classes and scores - ExpectValidSignature(); -} - -} // namespace tensorflow diff --git a/tensorflow/contrib/saved_model/python/__init__.py b/tensorflow/contrib/saved_model/python/__init__.py deleted file mode 100644 index f186c520c55..00000000000 --- a/tensorflow/contrib/saved_model/python/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""SavedModel contrib support. - -SavedModel provides a language-neutral format to save machine-learned models -that is recoverable and hermetic. It enables higher-level systems and tools to -produce, consume and transform TensorFlow models. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.saved_model.python.saved_model import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/saved_model/python/saved_model/__init__.py b/tensorflow/contrib/saved_model/python/saved_model/__init__.py deleted file mode 100644 index fd3dc1d7aa2..00000000000 --- a/tensorflow/contrib/saved_model/python/saved_model/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""SavedModel contrib support. - -SavedModel provides a language-neutral format to save machine-learned models -that is recoverable and hermetic. It enables higher-level systems and tools to -produce, consume and transform TensorFlow models. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.saved_model.python.saved_model import keras_saved_model -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py deleted file mode 100644 index 4ee679051fb..00000000000 --- a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# pylint: disable=protected-access -"""Utility functions to save/load keras Model to/from SavedModel.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.saving import saved_model_experimental - - -# TODO(kathywu): Remove all contrib callers, switch to tf.keras. -save_keras_model = saved_model_experimental.export_saved_model -load_keras_model = saved_model_experimental.load_from_saved_model diff --git a/tensorflow/contrib/saved_model/python/saved_model/reader.py b/tensorflow/contrib/saved_model/python/saved_model/reader.py deleted file mode 100644 index b9e5319181d..00000000000 --- a/tensorflow/contrib/saved_model/python/saved_model/reader.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""SavedModel functionality to read a SavedModel from disk.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from google.protobuf import message -from google.protobuf import text_format -from tensorflow.core.protobuf import saved_model_pb2 -from tensorflow.python.lib.io import file_io -from tensorflow.python.saved_model import constants -from tensorflow.python.util import compat - - -def read_saved_model(saved_model_dir): - """Reads the savedmodel.pb or savedmodel.pbtxt file containing `SavedModel`. - - Args: - saved_model_dir: Directory containing the SavedModel file. - - Returns: - A `SavedModel` protocol buffer. - - Raises: - IOError: If the file does not exist, or cannot be successfully parsed. - """ - # Build the path to the SavedModel in pbtxt format. - path_to_pbtxt = os.path.join( - compat.as_bytes(saved_model_dir), - compat.as_bytes(constants.SAVED_MODEL_FILENAME_PBTXT)) - # Build the path to the SavedModel in pb format. - path_to_pb = os.path.join( - compat.as_bytes(saved_model_dir), - compat.as_bytes(constants.SAVED_MODEL_FILENAME_PB)) - - # Ensure that the SavedModel exists at either path. - if not file_io.file_exists(path_to_pbtxt) and not file_io.file_exists( - path_to_pb): - raise IOError("SavedModel file does not exist at: %s" % saved_model_dir) - - # Parse the SavedModel protocol buffer. - saved_model = saved_model_pb2.SavedModel() - if file_io.file_exists(path_to_pb): - try: - file_content = file_io.FileIO(path_to_pb, "rb").read() - saved_model.ParseFromString(file_content) - return saved_model - except message.DecodeError as e: - raise IOError("Cannot parse file %s: %s." % (path_to_pb, str(e))) - elif file_io.file_exists(path_to_pbtxt): - try: - file_content = file_io.FileIO(path_to_pbtxt, "rb").read() - text_format.Merge(file_content.decode("utf-8"), saved_model) - return saved_model - except text_format.ParseError as e: - raise IOError("Cannot parse file %s: %s." % (path_to_pbtxt, str(e))) - else: - raise IOError("SavedModel file does not exist at: %s/{%s|%s}" % - (saved_model_dir, constants.SAVED_MODEL_FILENAME_PBTXT, - constants.SAVED_MODEL_FILENAME_PB)) - - -def get_saved_model_tag_sets(saved_model_dir): - """Retrieves all the tag-sets available in the SavedModel. - - Args: - saved_model_dir: Directory containing the SavedModel. - - Returns: - String representation of all tag-sets in the SavedModel. - """ - saved_model = read_saved_model(saved_model_dir) - all_tags = [] - for meta_graph_def in saved_model.meta_graphs: - all_tags.append(list(meta_graph_def.meta_info_def.tags)) - return all_tags diff --git a/tensorflow/contrib/saved_model/python/saved_model/reader_test.py b/tensorflow/contrib/saved_model/python/saved_model/reader_test.py deleted file mode 100644 index 3e6ff65c330..00000000000 --- a/tensorflow/contrib/saved_model/python/saved_model/reader_test.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for SavedModel Reader.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from tensorflow.contrib.saved_model.python.saved_model import reader -from tensorflow.python.framework import ops -from tensorflow.python.lib.io import file_io -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.saved_model import builder as saved_model_builder -from tensorflow.python.saved_model import tag_constants - - -def tearDownModule(): - file_io.delete_recursively(test.get_temp_dir()) - - -class ReaderTest(test.TestCase): - - def _init_and_validate_variable(self, sess, variable_name, variable_value): - v = variables.Variable(variable_value, name=variable_name) - sess.run(variables.global_variables_initializer()) - self.assertEqual(variable_value, v.eval()) - - def testReadSavedModelValid(self): - saved_model_dir = os.path.join(test.get_temp_dir(), "valid_saved_model") - builder = saved_model_builder.SavedModelBuilder(saved_model_dir) - with self.session(graph=ops.Graph()) as sess: - self._init_and_validate_variable(sess, "v", 42) - builder.add_meta_graph_and_variables(sess, [tag_constants.TRAINING]) - builder.save() - - actual_saved_model_pb = reader.read_saved_model(saved_model_dir) - self.assertEqual(len(actual_saved_model_pb.meta_graphs), 1) - self.assertEqual( - len(actual_saved_model_pb.meta_graphs[0].meta_info_def.tags), 1) - self.assertEqual(actual_saved_model_pb.meta_graphs[0].meta_info_def.tags[0], - tag_constants.TRAINING) - - def testReadSavedModelInvalid(self): - saved_model_dir = os.path.join(test.get_temp_dir(), "invalid_saved_model") - with self.assertRaisesRegexp( - IOError, "SavedModel file does not exist at: %s" % saved_model_dir): - reader.read_saved_model(saved_model_dir) - - def testGetSavedModelTagSets(self): - saved_model_dir = os.path.join(test.get_temp_dir(), "test_tags") - builder = saved_model_builder.SavedModelBuilder(saved_model_dir) - - # Graph with a single variable. SavedModel invoked to: - # - add with weights. - # - a single tag (from predefined constants). - with self.session(graph=ops.Graph()) as sess: - self._init_and_validate_variable(sess, "v", 42) - builder.add_meta_graph_and_variables(sess, [tag_constants.TRAINING]) - - # Graph that updates the single variable. SavedModel invoked to: - # - simply add the model (weights are not updated). - # - a single tag (from predefined constants). - with self.session(graph=ops.Graph()) as sess: - self._init_and_validate_variable(sess, "v", 43) - builder.add_meta_graph([tag_constants.SERVING]) - - # Graph that updates the single variable. SavedModel is invoked: - # - to add the model (weights are not updated). - # - multiple predefined tags. - with self.session(graph=ops.Graph()) as sess: - self._init_and_validate_variable(sess, "v", 44) - builder.add_meta_graph([tag_constants.SERVING, tag_constants.GPU]) - - # Graph that updates the single variable. SavedModel is invoked: - # - to add the model (weights are not updated). - # - multiple predefined tags for serving on TPU. - with self.session(graph=ops.Graph()) as sess: - self._init_and_validate_variable(sess, "v", 44) - builder.add_meta_graph([tag_constants.SERVING, tag_constants.TPU]) - - # Graph that updates the single variable. SavedModel is invoked: - # - to add the model (weights are not updated). - # - multiple custom tags. - with self.session(graph=ops.Graph()) as sess: - self._init_and_validate_variable(sess, "v", 45) - builder.add_meta_graph(["foo", "bar"]) - - # Save the SavedModel to disk. - builder.save() - - actual_tags = reader.get_saved_model_tag_sets(saved_model_dir) - expected_tags = [["train"], ["serve"], ["serve", "gpu"], ["serve", "tpu"], - ["foo", "bar"]] - self.assertEqual(expected_tags, actual_tags) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/seq2seq/BUILD b/tensorflow/contrib/seq2seq/BUILD deleted file mode 100644 index 3f9400a6748..00000000000 --- a/tensorflow/contrib/seq2seq/BUILD +++ /dev/null @@ -1,271 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -load("//tensorflow:tensorflow.bzl", "cuda_py_test") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") -load( - "//tensorflow:tensorflow.bzl", - "tf_custom_op_library", - "tf_gen_op_libs", - "tf_gen_op_wrapper_py", - "tf_kernel_library", -) - -package( - default_visibility = [ - "//learning/brain/google/xla/tests:__subpackages__", - "//tensorflow:__subpackages__", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_custom_op_py_library( - name = "seq2seq_py", - srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - dso = [ - ":python/ops/_beam_search_ops.so", - ], - kernels = [ - ":beam_search_ops_kernels", - ":beam_search_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":beam_search_ops", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:clip_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:control_flow_util", - "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:functional_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:layers", - "//tensorflow/python:layers_base", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:rnn", - "//tensorflow/python:rnn_cell", - "//tensorflow/python:script_ops", - "//tensorflow/python:tensor_array_ops", - "//tensorflow/python:tensor_util", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -tf_custom_op_library( - name = "python/ops/_beam_search_ops.so", - srcs = [ - "kernels/beam_search_ops.cc", - "kernels/beam_search_ops.h", - "ops/beam_search_ops.cc", - ], - gpu_srcs = [ - "kernels/beam_search_ops_gpu.cu.cc", - "kernels/beam_search_ops.h", - ], -) - -tf_gen_op_wrapper_py( - name = "beam_search_ops", - deps = [":beam_search_ops_op_lib"], -) - -tf_gen_op_libs( - op_lib_names = [ - "beam_search_ops", - ], -) - -tf_kernel_library( - name = "beam_search_ops_kernels", - prefix = "kernels/beam_search_ops", - deps = [ - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//third_party/eigen3", - ], -) - -cuda_py_test( - name = "loss_test", - size = "medium", - srcs = ["python/kernel_tests/loss_test.py"], - additional_deps = [ - ":seq2seq_py", - "//third_party/py/numpy", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:variable_scope", - ], -) - -cuda_py_test( - name = "basic_decoder_test", - size = "medium", - srcs = ["python/kernel_tests/basic_decoder_test.py"], - additional_deps = [ - ":seq2seq_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:rnn", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -cuda_py_test( - name = "basic_decoder_v2_test", - size = "medium", - srcs = ["python/kernel_tests/basic_decoder_v2_test.py"], - additional_deps = [ - ":seq2seq_py", - "@absl_py//absl/testing:parameterized", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:rnn", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -cuda_py_test( - name = "beam_search_ops_test", - size = "medium", - srcs = ["python/kernel_tests/beam_search_ops_test.py"], - additional_deps = [ - ":seq2seq_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -cuda_py_test( - name = "decoder_test", - size = "medium", - srcs = ["python/kernel_tests/decoder_test.py"], - additional_deps = [ - ":seq2seq_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:rnn", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -cuda_py_test( - name = "decoder_v2_test", - size = "medium", - srcs = ["python/kernel_tests/decoder_v2_test.py"], - additional_deps = [ - ":seq2seq_py", - "@absl_py//absl/testing:parameterized", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:rnn", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -cuda_py_test( - name = "beam_search_decoder_test", - size = "medium", - srcs = ["python/kernel_tests/beam_search_decoder_test.py"], - additional_deps = [ - ":seq2seq_py", - "//third_party/py/numpy", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:rnn", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -cuda_py_test( - name = "attention_wrapper_test", - size = "medium", - srcs = ["python/kernel_tests/attention_wrapper_test.py"], - additional_deps = [ - ":seq2seq_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:rnn", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], - xla_enable_strict_auto_jit = False, -) - -cuda_py_test( - name = "attention_wrapper_v2_test", - size = "medium", - srcs = ["python/kernel_tests/attention_wrapper_v2_test.py"], - additional_deps = [ - ":seq2seq_py", - "@absl_py//absl/testing:parameterized", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:platform_test", - "//tensorflow/python:variables", - ], - shard_count = 4, -) diff --git a/tensorflow/contrib/seq2seq/__init__.py b/tensorflow/contrib/seq2seq/__init__.py deleted file mode 100644 index 674f7cdb224..00000000000 --- a/tensorflow/contrib/seq2seq/__init__.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Ops for building neural network seq2seq decoders and losses. - -See the -[Contrib Seq2seq](https://tensorflow.org/api_guides/python/contrib.seq2seq) -guide. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import,line-too-long -from tensorflow.contrib.seq2seq.python.ops.attention_wrapper import * -from tensorflow.contrib.seq2seq.python.ops.basic_decoder import * -from tensorflow.contrib.seq2seq.python.ops.beam_search_decoder import * -from tensorflow.contrib.seq2seq.python.ops.beam_search_ops import * -from tensorflow.contrib.seq2seq.python.ops.decoder import * -from tensorflow.contrib.seq2seq.python.ops.helper import * -from tensorflow.contrib.seq2seq.python.ops.loss import * -from tensorflow.python.util.all_util import remove_undocumented -# pylint: enable=unused-import,widcard-import,line-too-long - -_allowed_symbols = [ - "sequence_loss", - "Decoder", - "dynamic_decode", - "BasicDecoder", - "BasicDecoderOutput", - "BeamSearchDecoder", - "BeamSearchDecoderOutput", - "BeamSearchDecoderState", - "Helper", - "CustomHelper", - "FinalBeamSearchDecoderOutput", - "gather_tree", - "GreedyEmbeddingHelper", - "InferenceHelper", - "SampleEmbeddingHelper", - "ScheduledEmbeddingTrainingHelper", - "ScheduledOutputTrainingHelper", - "TrainingHelper", - "BahdanauAttention", - "LuongAttention", - "hardmax", - "AttentionWrapperState", - "AttentionWrapper", - "AttentionMechanism", - "tile_batch", - "safe_cumprod", - "monotonic_attention", - "monotonic_probability_fn", - "BahdanauMonotonicAttention", - "LuongMonotonicAttention", -] - - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc deleted file mode 100644 index a9a32b7b25d..00000000000 --- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc +++ /dev/null @@ -1,195 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#define EIGEN_USE_THREADS - -#if GOOGLE_CUDA -#define EIGEN_USE_GPU -#endif // GOOGLE_CUDA - -#include "tensorflow/contrib/seq2seq/kernels/beam_search_ops.h" - -#include -#include - -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -typedef Eigen::ThreadPoolDevice CPUDevice; -typedef Eigen::GpuDevice GPUDevice; - -template -class GatherTreeOp : public OpKernel { - public: - explicit GatherTreeOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} - - void Compute(OpKernelContext* ctx) override { - const Device& device = ctx->eigen_device(); - const Tensor& step_ids = ctx->input(0); - const Tensor& parent_ids = ctx->input(1); - const Tensor& max_sequence_lengths = ctx->input(2); - const Tensor& end_token = ctx->input(3); - const TensorShape& step_ids_shape = step_ids.shape(); - OP_REQUIRES( - ctx, step_ids_shape.dims() == 3, - errors::InvalidArgument("step_ids must be a 3-tensor, saw shape: ", - step_ids_shape.DebugString())); - OP_REQUIRES(ctx, TensorShapeUtils::IsVector(max_sequence_lengths.shape()), - errors::InvalidArgument( - "max_sequence_lengths must be a vector, saw shape: ", - max_sequence_lengths.shape().DebugString())); - OP_REQUIRES( - ctx, TensorShapeUtils::IsScalar(end_token.shape()), - errors::InvalidArgument("end_token must be a scalar, saw shape: ", - end_token.shape().DebugString())); - OP_REQUIRES( - ctx, step_ids_shape == parent_ids.shape(), - errors::InvalidArgument( - "step_ids.shape must match parent_ids.shape. but shapes are: ", - step_ids_shape.DebugString(), " and ", - parent_ids.shape().DebugString())); - OP_REQUIRES( - ctx, - step_ids_shape.dim_size(1) == max_sequence_lengths.shape().dim_size(0), - errors::InvalidArgument("batch size dimensions step_ids.shape[1] and " - "max_sequence_lengths.shape[0] must match. " - "but shapes are: ", - step_ids_shape.DebugString(), " and ", - max_sequence_lengths.shape().DebugString())); - Tensor* beams; - OP_REQUIRES_OK(ctx, ctx->allocate_output(0, step_ids_shape, &beams)); - typename TTypes::ConstTensor step_ids_t(step_ids.tensor()); - typename TTypes::ConstTensor parent_ids_t(parent_ids.tensor()); - typename TTypes::ConstVec max_seq_lens_t = - max_sequence_lengths.vec(); - typename TTypes::ConstScalar end_token_t(end_token.scalar()); - typename TTypes::Tensor beams_t(beams->tensor()); - const T end_token_value = end_token_t(); - functor::GatherTree()(ctx, device, step_ids_t, parent_ids_t, - max_seq_lens_t, end_token_value, beams_t); - } -}; - -#define REGISTER_KERNEL(T) \ - REGISTER_KERNEL_BUILDER( \ - Name("GatherTree").Device(DEVICE_CPU).TypeConstraint("T"), \ - GatherTreeOp); -REGISTER_KERNEL(int32); -#undef REGISTER_KERNEL - -namespace functor { - -// CPU specialization -template <> -struct GatherTree { - void operator()(OpKernelContext* ctx, const CPUDevice& d, - TTypes::ConstTensor step_ids, - TTypes::ConstTensor parent_ids, - TTypes::ConstVec max_sequence_lengths, - const int32 end_token, TTypes::Tensor beams) { - const int32 max_time = parent_ids.dimension(0); - const int32 batch_size = parent_ids.dimension(1); - const int32 beam_width = parent_ids.dimension(2); - beams.setConstant(end_token); - - auto DoWork = [&, ctx, end_token](int start_batch_beam, - int limit_batch_beam) { - for (int32 i = start_batch_beam; i < limit_batch_beam; ++i) { - const int32 batch = i / beam_width; - const int32 beam = i % beam_width; - const int32 max_seq_len_b = - Eigen::numext::mini(max_time, max_sequence_lengths(batch)); - if (max_seq_len_b <= 0) { - continue; - } - beams(max_seq_len_b - 1, batch, beam) = - step_ids(max_seq_len_b - 1, batch, beam); - int32 parent = parent_ids(max_seq_len_b - 1, batch, beam); - for (int32 level = max_seq_len_b - 2; level >= 0; --level) { - if (parent < 0 || parent > beam_width) { - ctx->SetStatus( - errors::InvalidArgument("Saw invalid parent id ", parent, - " at (batch, time, beam) == (", batch, - ", ", level, ", ", beam, ")")); - return; - } - beams(level, batch, beam) = step_ids(level, batch, parent); - parent = parent_ids(level, batch, parent); - } - // Not necessary when using a BeamSearchDecoder, but necessary - // when a user feeds in possibly broken trajectory (i.e., non-eos - // entries in a beam following eos entries). - bool finished = false; - for (int32 time = 0; time < max_seq_len_b; ++time) { - if (finished) { - beams(time, batch, beam) = end_token; - } else if (beams(time, batch, beam) == end_token) { - finished = true; - } - } - } - }; - // Guesstimate of cost; ~5 lookup/store/compare per inner beam - // traversal time step. - const int64 batch_beam_cost = - Eigen::TensorOpCost::DivCost() + - 6 * Eigen::TensorOpCost::AddCost() + - 2 * max_time * (5 * Eigen::TensorOpCost::AddCost()); - auto worker_threads = *(ctx->device()->tensorflow_cpu_worker_threads()); - Shard(worker_threads.num_threads, worker_threads.workers, - batch_size * beam_width, batch_beam_cost, DoWork); - } -}; - -} // namespace functor - -#if GOOGLE_CUDA -namespace functor { -#define DECLARE_GPU_SPEC(T) \ - template <> \ - void GatherTree::operator()( \ - OpKernelContext* ctx, const GPUDevice& d, \ - typename TTypes::ConstTensor step_ids, \ - typename TTypes::ConstTensor parent_ids, \ - TTypes::ConstVec max_sequence_lengths, const T end_token, \ - typename TTypes::Tensor beams); \ - extern template struct GatherTree; - -DECLARE_GPU_SPEC(int32); -#undef DECLARE_GPU_SPEC -} // end namespace functor - -#define REGISTER_GPU_KERNEL(T) \ - REGISTER_KERNEL_BUILDER(Name("GatherTree") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .HostMemory("end_token"), \ - GatherTreeOp); - -REGISTER_GPU_KERNEL(int32); -#undef REGISTER_GPU_KERNEL -#endif // GOOGLE_CUDA - -} // end namespace tensorflow diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h deleted file mode 100644 index c0b3091fb8d..00000000000 --- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h +++ /dev/null @@ -1,40 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_SEQ2SEQ_KERNELS_BEAM_SEARCH_OPS_H_ -#define TENSORFLOW_CONTRIB_SEQ2SEQ_KERNELS_BEAM_SEARCH_OPS_H_ - -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -class OpKernelContext; - -namespace functor { - -template -struct GatherTree { - void operator()(OpKernelContext* ctx, const Device& d, - typename TTypes::ConstTensor step_ids, - typename TTypes::ConstTensor parent_ids, - TTypes::ConstVec max_sequence_lengths, - const T end_token, typename TTypes::Tensor beams); -}; - -} // namespace functor -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_SEQ2SEQ_KERNELS_BEAM_SEARCH_OPS_H_ diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc deleted file mode 100644 index 013c9faa059..00000000000 --- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc +++ /dev/null @@ -1,109 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#if GOOGLE_CUDA - -#define EIGEN_USE_GPU - -#include "tensorflow/contrib/seq2seq/kernels/beam_search_ops.h" -#include "tensorflow/core/util/gpu_kernel_helper.h" - -namespace tensorflow { -namespace functor { - -typedef Eigen::GpuDevice GPUDevice; - -template -__global__ void GatherTreeOpKernel(const int32 batch_size, const int32 max_time, - const int32 beam_width, const T* step_ids, - const T* parent_ids, - const int32* max_sequence_lengths, - const T end_token, T* beams) { - CUDA_1D_KERNEL_LOOP(i, batch_size * beam_width) { - const int32 batch = i / beam_width; - const int32 beam = i % beam_width; - - const int32 max_seq_len_b = - Eigen::numext::mini(max_time, ldg(max_sequence_lengths + batch)); - if (max_seq_len_b <= 0) { - continue; - } - -#define GET_IX(time_ix, beam_ix) \ - (batch_size * beam_width * (time_ix) + beam_width * batch + (beam_ix)) - const int32 initial_beam_ix = GET_IX(max_seq_len_b - 1, beam); - beams[initial_beam_ix] = ldg(step_ids + initial_beam_ix); - int32 parent = ldg(parent_ids + initial_beam_ix); - bool found_bad = false; - for (int32 level = max_seq_len_b - 2; level >= 0; --level) { - const int32 level_beam_ix = GET_IX(level, beam); - const int32 level_parent_ix = GET_IX(level, parent); - if (parent < 0 || parent > beam_width) { - beams[level_beam_ix] = -1; - parent = -1; - found_bad = true; - } else { - beams[level_beam_ix] = ldg(step_ids + level_parent_ix); - parent = ldg(parent_ids + level_parent_ix); - } - } - // Not necessary when using a BeamSearchDecoder, but necessary - // when a user feeds in possibly broken trajectory (i.e., non-eos - // entries in a beam following eos entries). - if (!found_bad) { - bool finished = false; - for (int32 time = 0; time < max_seq_len_b; ++time) { - const int32 level_beam_ix = GET_IX(time, beam); - if (finished) { - beams[level_beam_ix] = end_token; - } else if (beams[level_beam_ix] == end_token) { - finished = true; - } - } - } -#undef GET_IX - } -} - -template -struct GatherTree { - void operator()(OpKernelContext* ctx, const GPUDevice& d, - typename TTypes::ConstTensor step_ids, - typename TTypes::ConstTensor parent_ids, - TTypes::ConstVec max_sequence_length, - const T end_token, typename TTypes::Tensor beams) { - const int32 max_time = parent_ids.dimension(0); - const int32 batch_size = parent_ids.dimension(1); - const int32 beam_width = parent_ids.dimension(2); - // First kernel launch to "zero" things out - beams.device(d) = beams.constant(end_token); - - GpuLaunchConfig config = GetGpuLaunchConfig(batch_size * beam_width, d); - TF_CHECK_OK(GpuLaunchKernel( - GatherTreeOpKernel, config.block_count, config.thread_per_block, 0, - d.stream(), batch_size, max_time, beam_width, step_ids.data(), - parent_ids.data(), max_sequence_length.data(), end_token, - beams.data())); - } -}; - -#define DEFINE_GPU_SPECS(T) template struct GatherTree; - -DEFINE_GPU_SPECS(int32); -#undef DEFINE_GPU_SPECS - -} // end namespace functor -} // end namespace tensorflow -#endif // GOOGLE_CUDA diff --git a/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc b/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc deleted file mode 100644 index 71539b6f592..00000000000 --- a/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc +++ /dev/null @@ -1,72 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { - -using shape_inference::DimensionHandle; -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -REGISTER_OP("GatherTree") - .Input("step_ids: T") - .Input("parent_ids: T") - .Input("max_sequence_lengths: int32") - .Input("end_token: T") - .Output("beams: T") - .Attr("T: {int32}") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle step_ids, parent_ids, max_sequence_lengths, end_token; - - // step_ids, parent_ids, and output are all shaped: - // [max_time, batch_size, beam_width]. - // max_sequence_length is shaped [batch_size] and end_token is a scalar. - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 3, &step_ids)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 3, &parent_ids)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &max_sequence_lengths)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &end_token)); - TF_RETURN_IF_ERROR(c->Merge(step_ids, parent_ids, &step_ids)); - DimensionHandle batch_size = c->Dim(step_ids, 1); - TF_RETURN_IF_ERROR( - c->Merge(batch_size, c->Dim(max_sequence_lengths, 0), &batch_size)); - ShapeHandle step_ids_prefix = c->Matrix(c->Dim(step_ids, 0), batch_size); - TF_RETURN_IF_ERROR(c->MergePrefix(step_ids, step_ids_prefix, &step_ids, - &step_ids_prefix)); - - c->set_output(0, step_ids); - return tensorflow::Status::OK(); - }) - .Doc(R"doc( -Calculates the full beams from the per-step ids and parent beam ids. - -On CPU, if an out of bound parent id is found, an error is returned. -On GPU, if an out of bound parent id is found, a -1 is stored in the -corresponding output value and the execution for that beam returns early. - -For a given beam, past the time step containing the first decoded `end_token` -all values are filled in with `end_token`. - -TODO(ebrevdo): fill in the remainder of this docstring. - -step_ids: `[max_time, batch_size, beam_width]`. -parent_ids: `[max_time, batch_size, beam_width]`. -max_sequence_lengths: `[batch_size]`. -end_token: `[]`. -beams: `[max_time, batch_size, beam_width]`. -)doc"); - -} // end namespace tensorflow diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/__init__.py b/tensorflow/contrib/seq2seq/python/kernel_tests/__init__.py deleted file mode 100644 index 52e83069cb0..00000000000 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py deleted file mode 100644 index 634bc4ea21e..00000000000 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py +++ /dev/null @@ -1,1059 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for contrib.seq2seq.python.ops.attention_wrapper.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import functools - -import numpy as np - -from tensorflow.contrib.seq2seq.python.ops import decoder -from tensorflow.contrib.seq2seq.python.ops import attention_wrapper as wrapper -from tensorflow.contrib.seq2seq.python.ops import helper as helper_py -from tensorflow.contrib.seq2seq.python.ops import basic_decoder -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import test_util -from tensorflow.python.layers import core as layers_core -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import rnn -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import variables -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.platform import test -from tensorflow.python.util import nest - -# pylint: enable=g-import-not-at-top - - -# for testing -AttentionWrapperState = wrapper.AttentionWrapperState # pylint: disable=invalid-name -LSTMStateTuple = rnn_cell.LSTMStateTuple # pylint: disable=invalid-name -BasicDecoderOutput = basic_decoder.BasicDecoderOutput # pylint: disable=invalid-name -float32 = np.float32 -int32 = np.int32 -array = np.array -dtype = np.dtype - - -class ResultSummary( - collections.namedtuple('ResultSummary', ('shape', 'dtype', 'mean'))): - pass - - -def get_result_summary(x): - if isinstance(x, np.ndarray): - return ResultSummary(x.shape, x.dtype, x.mean()) - return x - - -@test_util.run_v1_only('contrib code not supported in TF2.0') -class AttentionWrapperTest(test.TestCase): - - def assertAllCloseOrEqual(self, x, y, **kwargs): - if isinstance(x, np.ndarray) or isinstance(x, float): - return super(AttentionWrapperTest, self).assertAllClose( - x, y, atol=1e-3, **kwargs) - else: - self.assertAllEqual(x, y, **kwargs) - - def testAttentionWrapperState(self): - num_fields = len(wrapper.AttentionWrapperState._fields) # pylint: disable=protected-access - state = wrapper.AttentionWrapperState(*([None] * num_fields)) - new_state = state.clone(time=1) - self.assertEqual(state.time, None) - self.assertEqual(new_state.time, 1) - - def testAttentionWrapperStateShapePropgation(self): - batch_size = 5 - max_time = 5 - num_units = 5 - - memory = random_ops.random_uniform( - [batch_size, max_time, num_units], seed=1) - mechanism = wrapper.LuongAttention(num_units, memory) - cell = wrapper.AttentionWrapper(rnn_cell.LSTMCell(num_units), mechanism) - - # Create zero state with static batch size. - static_state = cell.zero_state(batch_size, dtypes.float32) - # Create zero state without static batch size. - state = cell.zero_state(array_ops.shape(memory)[0], dtypes.float32) - - state = static_state.clone( - cell_state=state.cell_state, attention=state.attention) - - self.assertEqual(state.cell_state.c.shape, static_state.cell_state.c.shape) - self.assertEqual(state.cell_state.h.shape, static_state.cell_state.h.shape) - self.assertEqual(state.attention.shape, static_state.attention.shape) - - def _testWithAttention(self, - create_attention_mechanism, - expected_final_output, - expected_final_state, - attention_mechanism_depth=3, - alignment_history=False, - expected_final_alignment_history=None, - attention_layer_size=6, - attention_layer=None, - name=''): - attention_layer_sizes = ( - [attention_layer_size] if attention_layer_size is not None else None) - attention_layers = ( - [attention_layer] if attention_layer is not None else None) - self._testWithMaybeMultiAttention( - is_multi=False, - create_attention_mechanisms=[create_attention_mechanism], - expected_final_output=expected_final_output, - expected_final_state=expected_final_state, - attention_mechanism_depths=[attention_mechanism_depth], - alignment_history=alignment_history, - expected_final_alignment_history=expected_final_alignment_history, - attention_layer_sizes=attention_layer_sizes, - attention_layers=attention_layers, - name=name) - - def _testWithMaybeMultiAttention(self, - is_multi, - create_attention_mechanisms, - expected_final_output, - expected_final_state, - attention_mechanism_depths, - alignment_history=False, - expected_final_alignment_history=None, - attention_layer_sizes=None, - attention_layers=None, - name=''): - # Allow is_multi to be True with a single mechanism to enable test for - # passing in a single mechanism in a list. - assert len(create_attention_mechanisms) == 1 or is_multi - encoder_sequence_length = [3, 2, 3, 1, 1] - decoder_sequence_length = [2, 0, 1, 2, 3] - batch_size = 5 - encoder_max_time = 8 - decoder_max_time = 4 - input_depth = 7 - encoder_output_depth = 10 - cell_depth = 9 - - if attention_layer_sizes is not None: - # Compute sum of attention_layer_sizes. Use encoder_output_depth if None. - attention_depth = sum(attention_layer_size or encoder_output_depth - for attention_layer_size in attention_layer_sizes) - elif attention_layers is not None: - # Compute sum of attention_layers output depth. - attention_depth = sum( - attention_layer.compute_output_shape( - [batch_size, cell_depth + encoder_output_depth]).dims[-1].value - for attention_layer in attention_layers) - else: - attention_depth = encoder_output_depth * len(create_attention_mechanisms) - - decoder_inputs = array_ops.placeholder_with_default( - np.random.randn(batch_size, decoder_max_time, - input_depth).astype(np.float32), - shape=(None, None, input_depth)) - encoder_outputs = array_ops.placeholder_with_default( - np.random.randn(batch_size, encoder_max_time, - encoder_output_depth).astype(np.float32), - shape=(None, None, encoder_output_depth)) - - attention_mechanisms = [ - creator(num_units=depth, - memory=encoder_outputs, - memory_sequence_length=encoder_sequence_length) - for creator, depth in zip(create_attention_mechanisms, - attention_mechanism_depths)] - - with self.session(use_gpu=True) as sess: - with vs.variable_scope( - 'root', - initializer=init_ops.random_normal_initializer(stddev=0.01, seed=3)): - attention_layer_size = attention_layer_sizes - attention_layer = attention_layers - if not is_multi: - if attention_layer_size is not None: - attention_layer_size = attention_layer_size[0] - if attention_layer is not None: - attention_layer = attention_layer[0] - cell = rnn_cell.LSTMCell(cell_depth) - cell = wrapper.AttentionWrapper( - cell, - attention_mechanisms if is_multi else attention_mechanisms[0], - attention_layer_size=attention_layer_size, - alignment_history=alignment_history, - attention_layer=attention_layer) - helper = helper_py.TrainingHelper(decoder_inputs, - decoder_sequence_length) - my_decoder = basic_decoder.BasicDecoder( - cell=cell, - helper=helper, - initial_state=cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size)) - - final_outputs, final_state, _ = decoder.dynamic_decode(my_decoder) - - self.assertTrue( - isinstance(final_outputs, basic_decoder.BasicDecoderOutput)) - self.assertTrue( - isinstance(final_state, wrapper.AttentionWrapperState)) - self.assertTrue( - isinstance(final_state.cell_state, rnn_cell.LSTMStateTuple)) - - self.assertEqual((batch_size, None, attention_depth), - tuple(final_outputs.rnn_output.get_shape().as_list())) - self.assertEqual((batch_size, None), - tuple(final_outputs.sample_id.get_shape().as_list())) - - self.assertEqual((batch_size, attention_depth), - tuple(final_state.attention.get_shape().as_list())) - self.assertEqual((batch_size, cell_depth), - tuple(final_state.cell_state.c.get_shape().as_list())) - self.assertEqual((batch_size, cell_depth), - tuple(final_state.cell_state.h.get_shape().as_list())) - - if alignment_history: - if is_multi: - state_alignment_history = [] - for history_array in final_state.alignment_history: - history = history_array.stack() - self.assertEqual( - (None, batch_size, None), - tuple(history.get_shape().as_list())) - state_alignment_history.append(history) - state_alignment_history = tuple(state_alignment_history) - else: - state_alignment_history = final_state.alignment_history.stack() - self.assertEqual( - (None, batch_size, None), - tuple(state_alignment_history.get_shape().as_list())) - nest.assert_same_structure( - cell.state_size, - cell.zero_state(batch_size, dtypes.float32)) - # Remove the history from final_state for purposes of the - # remainder of the tests. - final_state = final_state._replace(alignment_history=()) # pylint: disable=protected-access - else: - state_alignment_history = () - - sess.run(variables.global_variables_initializer()) - sess_results = sess.run({ - 'final_outputs': final_outputs, - 'final_state': final_state, - 'state_alignment_history': state_alignment_history, - }) - - final_output_info = nest.map_structure(get_result_summary, - sess_results['final_outputs']) - final_state_info = nest.map_structure(get_result_summary, - sess_results['final_state']) - print(name) - print('Copy/paste:\nexpected_final_output = %s' % str(final_output_info)) - print('expected_final_state = %s' % str(final_state_info)) - nest.map_structure(self.assertAllCloseOrEqual, expected_final_output, - final_output_info) - nest.map_structure(self.assertAllCloseOrEqual, expected_final_state, - final_state_info) - if alignment_history: # by default, the wrapper emits attention as output - final_alignment_history_info = nest.map_structure( - get_result_summary, sess_results['state_alignment_history']) - print('expected_final_alignment_history = %s' % - str(final_alignment_history_info)) - nest.map_structure( - self.assertAllCloseOrEqual, - # outputs are batch major but the stacked TensorArray is time major - expected_final_alignment_history, - final_alignment_history_info) - - def testBahdanauNormalizedDType(self): - for dtype in [np.float16, np.float32, np.float64]: - num_units = 128 - encoder_outputs = array_ops.placeholder(dtype, shape=[64, None, 256]) - encoder_sequence_length = array_ops.placeholder(dtypes.int32, shape=[64]) - decoder_inputs = array_ops.placeholder(dtype, shape=[64, None, 128]) - decoder_sequence_length = array_ops.placeholder(dtypes.int32, shape=[64]) - batch_size = 64 - attention_mechanism = wrapper.BahdanauAttention( - num_units=num_units, - memory=encoder_outputs, - memory_sequence_length=encoder_sequence_length, - normalize=True, - dtype=dtype, - ) - cell = rnn_cell.LSTMCell(num_units) - cell = wrapper.AttentionWrapper(cell, attention_mechanism) - - helper = helper_py.TrainingHelper(decoder_inputs, - decoder_sequence_length) - my_decoder = basic_decoder.BasicDecoder( - cell=cell, - helper=helper, - initial_state=cell.zero_state( - dtype=dtype, batch_size=batch_size)) - - final_outputs, final_state, _ = decoder.dynamic_decode(my_decoder) - self.assertTrue( - isinstance(final_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual(final_outputs.rnn_output.dtype, dtype) - self.assertTrue( - isinstance(final_state, wrapper.AttentionWrapperState)) - self.assertTrue( - isinstance(final_state.cell_state, rnn_cell.LSTMStateTuple)) - - def testBahdanauNotNormalized(self): - create_attention_mechanism = wrapper.BahdanauAttention - - expected_final_output = BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0052250605), - sample_id=ResultSummary( - shape=(5, 3), dtype=dtype('int32'), mean=1.4)) - expected_final_state = AttentionWrapperState( - cell_state=LSTMStateTuple( - c=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0040092287), - h=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0020015112)), - attention=ResultSummary( - shape=(5, 6), dtype=dtype('float32'), mean=-0.0052052638), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125), - attention_state=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125), - alignment_history=()) - expected_final_alignment_history = ResultSummary( - shape=(3, 5, 8), dtype=dtype('float32'), mean=0.12500001) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - alignment_history=True, - expected_final_alignment_history=expected_final_alignment_history, - name='testBahdanauNotNormalized') - - def testBahdanauNormalized(self): - create_attention_mechanism = functools.partial( - wrapper.BahdanauAttention, normalize=True) - - expected_final_output = BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.00597103), - sample_id=ResultSummary( - shape=(5, 3), dtype=dtype('int32'), mean=1.4)) - expected_final_state = AttentionWrapperState( - cell_state=LSTMStateTuple( - c=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0040052128), - h=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0019996136)), - attention=ResultSummary( - shape=(5, 6), dtype=dtype('float32'), mean=-0.00595117), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125), - attention_state=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125), - alignment_history=()) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - name='testBahdanauNormalized') - - def testLuongNotNormalized(self): - create_attention_mechanism = wrapper.LuongAttention - - expected_final_output = BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0052615386), - sample_id=ResultSummary( - shape=(5, 3), dtype=dtype('int32'), mean=1.4)) - expected_final_state = AttentionWrapperState( - cell_state=LSTMStateTuple( - c=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.004009536), - h=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0020016613)), - attention=ResultSummary( - shape=(5, 6), dtype=dtype('float32'), mean=-0.0051812846), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125), - attention_state=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125), - alignment_history=()) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - attention_mechanism_depth=9, - name='testLuongNotNormalized') - - def testLuongScaledDType(self): - # Test case for GitHub issue 18099 - for dt in [np.float16, np.float32, np.float64]: - num_units = 128 - encoder_outputs = array_ops.placeholder(dt, shape=[64, None, 256]) - encoder_sequence_length = array_ops.placeholder(dtypes.int32, shape=[64]) - decoder_inputs = array_ops.placeholder(dt, shape=[64, None, 128]) - decoder_sequence_length = array_ops.placeholder(dtypes.int32, shape=[64]) - batch_size = 64 - attention_mechanism = wrapper.LuongAttention( - num_units=num_units, - memory=encoder_outputs, - memory_sequence_length=encoder_sequence_length, - scale=True, - dtype=dt, - ) - cell = rnn_cell.LSTMCell(num_units) - cell = wrapper.AttentionWrapper(cell, attention_mechanism) - - helper = helper_py.TrainingHelper(decoder_inputs, - decoder_sequence_length) - my_decoder = basic_decoder.BasicDecoder( - cell=cell, - helper=helper, - initial_state=cell.zero_state( - dtype=dt, batch_size=batch_size)) - - final_outputs, final_state, _ = decoder.dynamic_decode(my_decoder) - self.assertTrue( - isinstance(final_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual(final_outputs.rnn_output.dtype, dt) - self.assertTrue( - isinstance(final_state, wrapper.AttentionWrapperState)) - self.assertTrue( - isinstance(final_state.cell_state, rnn_cell.LSTMStateTuple)) - - def testLuongScaled(self): - create_attention_mechanism = functools.partial( - wrapper.LuongAttention, scale=True) - - expected_final_output = BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0052615386), - sample_id=ResultSummary( - shape=(5, 3), dtype=dtype('int32'), mean=1.4)) - expected_final_state = AttentionWrapperState( - cell_state=LSTMStateTuple( - c=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.004009536), - h=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0020016613)), - attention=ResultSummary( - shape=(5, 6), dtype=dtype('float32'), mean=-0.0051812846), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125), - attention_state=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125), - alignment_history=()) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - attention_mechanism_depth=9, - name='testLuongScaled') - - def testNotUseAttentionLayer(self): - create_attention_mechanism = wrapper.BahdanauAttention - - expected_final_output = BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 10), dtype=dtype('float32'), mean=0.117389656), - sample_id=ResultSummary( - shape=(5, 3), dtype=dtype('int32'), mean=4.5999999999999996)) - expected_final_state = AttentionWrapperState( - cell_state=LSTMStateTuple( - c=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0063607907), - h=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.00323448)), - attention=ResultSummary( - shape=(5, 10), dtype=dtype('float32'), mean=0.117389656,), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125), - attention_state=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125), - alignment_history=()) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - attention_layer_size=None, - name='testNotUseAttentionLayer') - - def test_safe_cumprod(self): - # Create some random test input - test_input = np.random.uniform(size=(10, 20)) - - for axis in [0, 1]: - for exclusive in [True, False]: - with self.cached_session(): - # Compute cumprod with regular tf.math.cumprod - cumprod_output = math_ops.cumprod( - test_input, axis=axis, exclusive=exclusive).eval() - # Compute cumprod with safe_cumprod - safe_cumprod_output = wrapper.safe_cumprod( - test_input, axis=axis, exclusive=exclusive).eval() - for x, y in zip(cumprod_output.shape, safe_cumprod_output.shape): - self.assertEqual(x, y) - for x, y in zip(cumprod_output.flatten(), - safe_cumprod_output.flatten()): - # Use assertAlmostEqual for the actual values due to floating point - self.assertAlmostEqual(x, y, places=5) - - def test_monotonic_attention(self): - def monotonic_attention_explicit(p_choose_i, previous_attention): - """Explicitly compute monotonic attention distribution using numpy.""" - # Base case for recurrence relation - out = [previous_attention[0]] - # Explicitly follow the recurrence relation - for j in range(1, p_choose_i.shape[0]): - out.append((1 - p_choose_i[j - 1])*out[j - 1] + previous_attention[j]) - return p_choose_i*np.array(out) - - # Generate a random batch of choosing probabilities for seq. len. 20 - p_choose_i = np.random.uniform(size=(10, 20)).astype(np.float32) - # Generate random previous attention distributions - previous_attention = np.random.uniform(size=(10, 20)).astype(np.float32) - previous_attention /= previous_attention.sum(axis=1).reshape((-1, 1)) - - # Create the output to test against - explicit_output = np.array([ - monotonic_attention_explicit(p, a) - for p, a in zip(p_choose_i, previous_attention)]) - - # Compute output with TensorFlow function, for both calculation types - with self.cached_session(): - recursive_output = wrapper.monotonic_attention( - p_choose_i, previous_attention, 'recursive').eval() - - self.assertEqual(recursive_output.ndim, explicit_output.ndim) - for x, y in zip(recursive_output.shape, explicit_output.shape): - self.assertEqual(x, y) - for x, y in zip(recursive_output.flatten(), explicit_output.flatten()): - # Use assertAlmostEqual for the actual values due to floating point - self.assertAlmostEqual(x, y, places=5) - - # Generate new p_choose_i for parallel, which is unstable when p_choose_i[n] - # is close to 1 - p_choose_i = np.random.uniform(0, 0.9, size=(10, 20)).astype(np.float32) - - # Create new output to test against - explicit_output = np.array([ - monotonic_attention_explicit(p, a) - for p, a in zip(p_choose_i, previous_attention)]) - - # Compute output with TensorFlow function, for both calculation types - with self.cached_session(): - parallel_output = wrapper.monotonic_attention( - p_choose_i, previous_attention, 'parallel').eval() - - self.assertEqual(parallel_output.ndim, explicit_output.ndim) - for x, y in zip(parallel_output.shape, explicit_output.shape): - self.assertEqual(x, y) - for x, y in zip(parallel_output.flatten(), explicit_output.flatten()): - # Use assertAlmostEqual for the actual values due to floating point - self.assertAlmostEqual(x, y, places=5) - - # Now, test hard mode, where probabilities must be 0 or 1 - p_choose_i = np.random.choice(np.array([0, 1], np.float32), (10, 20)) - previous_attention = np.zeros((10, 20), np.float32) - # Randomly choose input sequence indices at each timestep - random_idx = np.random.randint(0, previous_attention.shape[1], - previous_attention.shape[0]) - previous_attention[np.arange(previous_attention.shape[0]), random_idx] = 1 - - # Create the output to test against - explicit_output = np.array([ - monotonic_attention_explicit(p, a) - for p, a in zip(p_choose_i, previous_attention)]) - - # Compute output with TensorFlow function, for both calculation types - with self.cached_session(): - hard_output = wrapper.monotonic_attention( - # TensorFlow is unhappy when these are not wrapped as tf.constant - constant_op.constant(p_choose_i), - constant_op.constant(previous_attention), - 'hard').eval() - - self.assertEqual(hard_output.ndim, explicit_output.ndim) - for x, y in zip(hard_output.shape, explicit_output.shape): - self.assertEqual(x, y) - for x, y in zip(hard_output.flatten(), explicit_output.flatten()): - # Use assertAlmostEqual for the actual values due to floating point - self.assertAlmostEqual(x, y, places=5) - - # Now, test recursively computing attention distributions vs. sampling - def sample(p_choose_i): - """Generate a sequence of emit-ingest decisions from p_choose_i.""" - output = np.zeros(p_choose_i.shape) - t_im1 = 0 - for i in range(p_choose_i.shape[0]): - for j in range(t_im1, p_choose_i.shape[1]): - if np.random.uniform() <= p_choose_i[i, j]: - output[i, j] = 1 - t_im1 = j - break - else: - t_im1 = p_choose_i.shape[1] - return output - - # Now, the first axis is output timestep and second is input timestep - p_choose_i = np.random.uniform(size=(4, 5)).astype(np.float32) - # Generate the average of a bunch of samples - n_samples = 100000 - sampled_output = np.mean( - [sample(p_choose_i) for _ in range(n_samples)], axis=0) - - # Create initial previous_attention base case - recursive_output = [np.array([1] + [0]*(p_choose_i.shape[1] - 1), - np.float32)] - # Compute output with TensorFlow function, for both calculation types - with self.cached_session(): - for j in range(p_choose_i.shape[0]): - # Compute attention distribution for this output time step - recursive_output.append(wrapper.monotonic_attention( - # newaxis is for adding the expected batch dimension - p_choose_i[j][np.newaxis], - recursive_output[-1][np.newaxis], 'recursive').eval()[0]) - # Stack together distributions; remove basecase - recursive_output = np.array(recursive_output[1:]) - - self.assertEqual(recursive_output.ndim, sampled_output.ndim) - for x, y in zip(recursive_output.shape, sampled_output.shape): - self.assertEqual(x, y) - for x, y in zip(recursive_output.flatten(), sampled_output.flatten()): - # Use a very forgiving threshold since we are sampling - self.assertAlmostEqual(x, y, places=2) - - def testBahdanauMonotonicNotNormalized(self): - create_attention_mechanism = functools.partial( - wrapper.BahdanauMonotonicAttention, sigmoid_noise=1.0, - sigmoid_noise_seed=3) - - expected_final_output = BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.002122893), - sample_id=ResultSummary( - shape=(5, 3), dtype=dtype('int32'), mean=1.7333333333333334)) - expected_final_state = AttentionWrapperState( - cell_state=LSTMStateTuple( - c=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0040002423), - h=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0019968653)), - attention=ResultSummary( - shape=(5, 6), dtype=dtype('float32'), mean=-5.9313523e-05), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.032228071), - attention_state=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.032228071), - alignment_history=()) - expected_final_alignment_history = ResultSummary( - shape=(3, 5, 8), dtype=dtype('float32'), mean=0.050430927) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - alignment_history=True, - expected_final_alignment_history=expected_final_alignment_history, - name='testBahdanauMonotonicNotNormalized') - - def testBahdanauMonotonicNormalized(self): - create_attention_mechanism = functools.partial( - wrapper.BahdanauMonotonicAttention, normalize=True, - sigmoid_noise=1.0, sigmoid_noise_seed=3) - - expected_final_output = BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0025896581), - sample_id=ResultSummary( - shape=(5, 3), dtype=dtype('int32'), mean=1.73333333)) - expected_final_state = AttentionWrapperState( - cell_state=LSTMStateTuple( - c=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0040013152), - h=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0019973689)), - attention=ResultSummary( - shape=(5, 6), dtype=dtype('float32'), mean=-0.00069823361), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.029914695), - attention_state=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.029914695), - alignment_history=()) - expected_final_alignment_history = ResultSummary( - shape=(3, 5, 8), dtype=dtype('float32'), mean=0.0465225502849) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - alignment_history=True, - expected_final_alignment_history=expected_final_alignment_history, - name='testBahdanauMonotonicNormalized') - - def testBahdanauMonotonicHard(self): - # Run attention mechanism with mode='hard', make sure probabilities are hard - b, t, u, d = 10, 20, 30, 40 - with self.session(use_gpu=True) as sess: - a = wrapper.BahdanauMonotonicAttention( - d, - random_ops.random_normal((b, t, u)), - mode='hard') - # Just feed previous attention as [1, 0, 0, ...] - attn, unused_state = a( - random_ops.random_normal((b, d)), array_ops.one_hot([0]*b, t)) - sess.run(variables.global_variables_initializer()) - attn_out = attn.eval() - # All values should be 0 or 1 - self.assertTrue(np.all(np.logical_or(attn_out == 0, attn_out == 1))) - # Sum of distributions should be 0 or 1 (0 when all p_choose_i are 0) - self.assertTrue(np.all(np.logical_or(attn_out.sum(axis=1) == 1, - attn_out.sum(axis=1) == 0))) - - def testLuongMonotonicNotNormalized(self): - create_attention_mechanism = functools.partial( - wrapper.LuongMonotonicAttention, sigmoid_noise=1.0, - sigmoid_noise_seed=3) - - expected_final_output = BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0021257224), - sample_id=ResultSummary( - shape=(5, 3), dtype=dtype('int32'), mean=1.7333333333333334)) - expected_final_state = AttentionWrapperState( - cell_state=LSTMStateTuple( - c=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0040003359), - h=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.001996913)), - attention=ResultSummary( - shape=(5, 6), dtype=dtype('float32'), mean=-5.2024145e-05), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.032198936), - attention_state=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.032198936), - alignment_history=()) - expected_final_alignment_history = ResultSummary( - shape=(3, 5, 8), dtype=dtype('float32'), mean=0.050387777) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - attention_mechanism_depth=9, - alignment_history=True, - expected_final_alignment_history=expected_final_alignment_history, - name='testLuongMonotonicNotNormalized') - - def testLuongMonotonicScaled(self): - create_attention_mechanism = functools.partial( - wrapper.LuongMonotonicAttention, scale=True, sigmoid_noise=1.0, - sigmoid_noise_seed=3) - - expected_final_output = BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=dtype('float32'), mean=-0.0021257224), - sample_id=ResultSummary( - shape=(5, 3), dtype=dtype('int32'), mean=1.7333333333333334)) - expected_final_state = AttentionWrapperState( - cell_state=LSTMStateTuple( - c=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0040003359), - h=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.001996913)), - attention=ResultSummary( - shape=(5, 6), dtype=dtype('float32'), mean=-5.2024145e-05), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.032198936), - attention_state=ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.032198936), - alignment_history=()) - expected_final_alignment_history = ResultSummary( - shape=(3, 5, 8), dtype=dtype('float32'), mean=0.050387777) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - attention_mechanism_depth=9, - alignment_history=True, - expected_final_alignment_history=expected_final_alignment_history, - name='testLuongMonotonicScaled') - - def testMultiAttention(self): - create_attention_mechanisms = ( - wrapper.BahdanauAttention, wrapper.LuongAttention) - - expected_final_output = BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 7), dtype=dtype('float32'), mean=0.0011709079), - sample_id=ResultSummary( - shape=(5, 3), dtype=dtype('int32'), mean=3.2000000000000002)) - expected_final_state = AttentionWrapperState( - cell_state=LSTMStateTuple( - c=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0038725811), - h=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0019329828)), - attention=ResultSummary( - shape=(5, 7), dtype=dtype('float32'), mean=0.001174294), - time=3, - alignments=( - ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125), - ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125)), - attention_state=( - ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125), - ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125)), - alignment_history=()) - - expected_final_alignment_history = ( - ResultSummary(shape=(3, 5, 8), dtype=dtype('float32'), mean=0.125), - ResultSummary(shape=(3, 5, 8), dtype=dtype('float32'), mean=0.125)) - - self._testWithMaybeMultiAttention( - True, - create_attention_mechanisms, - expected_final_output, - expected_final_state, - attention_mechanism_depths=[9, 9], - attention_layer_sizes=[3, 4], - alignment_history=True, - expected_final_alignment_history=expected_final_alignment_history, - name='testMultiAttention') - - def testMultiAttentionWithLayerInstances(self): - create_attention_mechanisms = ( - wrapper.BahdanauAttention, wrapper.LuongAttention) - - expected_final_output = BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 7), dtype=dtype('float32'), mean=0.0011709079), - sample_id=ResultSummary( - shape=(5, 3), dtype=dtype('int32'), mean=3.2000000000000002)) - expected_final_state = AttentionWrapperState( - cell_state=LSTMStateTuple( - c=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0038725811), - h=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0019329828)), - attention=ResultSummary( - shape=(5, 7), dtype=dtype('float32'), mean=0.001174294), - time=3, - alignments=( - ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125), - ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125)), - attention_state=( - ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125), - ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125)), - alignment_history=()) - - expected_final_alignment_history = ( - ResultSummary(shape=(3, 5, 8), dtype=dtype('float32'), mean=0.125), - ResultSummary(shape=(3, 5, 8), dtype=dtype('float32'), mean=0.125)) - - self._testWithMaybeMultiAttention( - True, - create_attention_mechanisms, - expected_final_output, - expected_final_state, - attention_mechanism_depths=[9, 9], - attention_layers=[layers_core.Dense(3, use_bias=False), - layers_core.Dense(4, use_bias=False)], - alignment_history=True, - expected_final_alignment_history=expected_final_alignment_history, - name='testMultiAttention') - - def testLuongMonotonicHard(self): - # Run attention mechanism with mode='hard', make sure probabilities are hard - b, t, u, d = 10, 20, 30, 40 - with self.session(use_gpu=True) as sess: - a = wrapper.LuongMonotonicAttention( - d, - random_ops.random_normal((b, t, u)), - mode='hard') - # Just feed previous attention as [1, 0, 0, ...] - attn, unused_state = a( - random_ops.random_normal((b, d)), array_ops.one_hot([0]*b, t)) - sess.run(variables.global_variables_initializer()) - attn_out = attn.eval() - # All values should be 0 or 1 - self.assertTrue(np.all(np.logical_or(attn_out == 0, attn_out == 1))) - # Sum of distributions should be 0 or 1 (0 when all p_choose_i are 0) - self.assertTrue(np.all(np.logical_or(attn_out.sum(axis=1) == 1, - attn_out.sum(axis=1) == 0))) - - def testMultiAttentionNoAttentionLayer(self): - create_attention_mechanisms = ( - wrapper.BahdanauAttention, wrapper.LuongAttention) - - expected_final_output = BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 20), dtype=dtype('float32'), mean=0.115853324533), - sample_id=ResultSummary( - shape=(5, 3), dtype=dtype('int32'), mean=8.6)) - expected_final_state = AttentionWrapperState( - cell_state=LSTMStateTuple( - c=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.003545674), - h=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0018327223)), - attention=ResultSummary( - shape=(5, 20), dtype=dtype('float32'), mean=0.11462739855), - time=3, - alignments=(ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125), - ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125)), - alignment_history=(), - attention_state=(ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125), - ResultSummary( - shape=(5, 8), dtype=dtype('float32'), mean=0.125))) - expected_final_alignment_history = ( - ResultSummary(shape=(3, 5, 8), dtype=dtype('float32'), mean=0.125), - ResultSummary(shape=(3, 5, 8), dtype=dtype('float32'), mean=0.125)) - - self._testWithMaybeMultiAttention( - is_multi=True, - create_attention_mechanisms=create_attention_mechanisms, - expected_final_output=expected_final_output, - expected_final_state=expected_final_state, - attention_mechanism_depths=[9, 9], - alignment_history=True, - expected_final_alignment_history=expected_final_alignment_history, - name='testMultiAttention') - - def testSingleAttentionAsList(self): - create_attention_mechanisms = [wrapper.BahdanauAttention] - - expected_final_output = BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 3), dtype=dtype('float32'), mean=-0.0098485695), - sample_id=ResultSummary( - shape=(5, 3), dtype=dtype('int32'), mean=1.8)) - expected_final_state = AttentionWrapperState( - cell_state=LSTMStateTuple( - c=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0040023471), - h=ResultSummary( - shape=(5, 9), dtype=dtype('float32'), mean=-0.0019979973)), - attention=ResultSummary( - shape=(5, 3), dtype=dtype('float32'), mean=-0.0098808752), - time=3, - alignments=( - ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125),), - attention_state=( - ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125),), - alignment_history=()) - - expected_final_alignment_history = ( - ResultSummary(shape=(3, 5, 8), dtype=dtype('float32'), mean=0.125),) - - self._testWithMaybeMultiAttention( - is_multi=True, # pass the AttentionMechanism wrapped in a list - create_attention_mechanisms=create_attention_mechanisms, - expected_final_output=expected_final_output, - expected_final_state=expected_final_state, - attention_mechanism_depths=[9], - attention_layer_sizes=[3], - alignment_history=True, - expected_final_alignment_history=expected_final_alignment_history, - name='testMultiAttention') - - def testCustomizedAttention(self): - batch_size = 2 - max_time = 3 - num_units = 2 - memory = constant_op.constant([[[1., 1.], [2., 2.], [3., 3.]], - [[4., 4.], [5., 5.], [6., 6.]]]) - memory_sequence_length = constant_op.constant([3, 2]) - attention_mechanism = wrapper.BahdanauAttention(num_units, memory, - memory_sequence_length) - - # Sets all returned values to be all ones. - def _customized_attention(unused_attention_mechanism, unused_cell_output, - unused_attention_state, unused_attention_layer): - """Customized attention. - - Returns: - attention: `Tensor` of shape [batch_size, num_units], attention output. - alignments: `Tensor` of shape [batch_size, max_time], sigma value for - each input memory (prob. function of input keys). - next_attention_state: A `Tensor` representing the next state for the - attention. - """ - attention = array_ops.ones([batch_size, num_units]) - alignments = array_ops.ones([batch_size, max_time]) - next_attention_state = alignments - return attention, alignments, next_attention_state - - attention_cell = wrapper.AttentionWrapper( - rnn_cell.LSTMCell(2), - attention_mechanism, - attention_layer_size=None, # don't use attention layer. - output_attention=False, - alignment_history=(), - attention_fn=_customized_attention, - name='attention') - self.assertEqual(num_units, attention_cell.output_size) - - initial_state = attention_cell.zero_state( - batch_size=2, dtype=dtypes.float32) - source_input_emb = array_ops.ones([2, 3, 2]) - source_input_length = constant_op.constant([3, 2]) - - # 'state' is a tuple of - # (cell_state, h, attention, alignments, alignment_history, attention_state) - output, state = rnn.dynamic_rnn( - attention_cell, - inputs=source_input_emb, - sequence_length=source_input_length, - initial_state=initial_state, - dtype=dtypes.float32) - - with self.session() as sess: - sess.run(variables.global_variables_initializer()) - output_value, state_value = sess.run([output, state], feed_dict={}) - self.assertAllEqual(np.array([2, 3, 2]), output_value.shape) - self.assertAllClose(np.array([[1., 1.], [1., 1.]]), state_value.attention) - self.assertAllClose( - np.array([[1., 1., 1.], [1., 1., 1.]]), state_value.alignments) - self.assertAllClose( - np.array([[1., 1., 1.], [1., 1., 1.]]), state_value.attention_state) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py deleted file mode 100644 index 824c8dad43d..00000000000 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py +++ /dev/null @@ -1,753 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for contrib.seq2seq.python.ops.attention_wrapper.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from absl.testing import parameterized -import numpy as np - -from tensorflow.contrib.seq2seq.python.ops import attention_wrapper as wrapper -from tensorflow.contrib.seq2seq.python.ops import basic_decoder -from tensorflow.contrib.seq2seq.python.ops import sampler as sampler_py -from tensorflow.python import keras -from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import test_util -from tensorflow.python.keras import initializers -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.util import nest - - -@test_util.run_all_in_graph_and_eager_modes -class AttentionMechanismTest(test.TestCase, parameterized.TestCase): - - def setUp(self): - super(AttentionMechanismTest, self).setUp() - self.batch = 10 - self.timestep = 5 - self.memory_size = 6 - self.units = 8 - - self.memory = np.random.randn(self.batch, self.timestep, - self.memory_size).astype(np.float32) - self.query = np.random.randn(self.batch, self.units).astype(np.float32) - self.state = np.random.randn(self.batch, self.timestep).astype(np.float32) - - @parameterized.named_parameters( - ("luong", wrapper.LuongAttentionV2), - ("luong_monotonic", wrapper.LuongMonotonicAttentionV2), - ("bahdanau", wrapper.BahdanauAttentionV2), - ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2), - ) - def test_attention_shape_inference(self, attention_cls): - attention = attention_cls(self.units, self.memory) - attention_score = attention([self.query, self.state]) - self.assertLen(attention_score, 2) - self.assertEqual(attention_score[0].shape, (self.batch, self.timestep)) - self.assertEqual(attention_score[1].shape, (self.batch, self.timestep)) - - @parameterized.named_parameters( - ("luong", wrapper.LuongAttentionV2), - ("luong_monotonic", wrapper.LuongMonotonicAttentionV2), - ("bahdanau", wrapper.BahdanauAttentionV2), - ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2), - ) - def test_get_config(self, attention_cls): - attention = attention_cls(self.units, self.memory) - config = attention.get_config() - - attention_from_config = attention_cls.from_config(config) - config_from_clone = attention_from_config.get_config() - - self.assertDictEqual(config, config_from_clone) - - @parameterized.named_parameters( - ("luong", wrapper.LuongAttentionV2), - ("luong_monotonic", wrapper.LuongMonotonicAttentionV2), - ("bahdanau", wrapper.BahdanauAttentionV2), - ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2), - ) - def test_layer_output(self, attention_cls): - attention = attention_cls(self.units, self.memory) - score = attention([self.query, self.state]) - self.evaluate(variables.variables_initializer(attention.variables)) - - score_val = self.evaluate(score) - self.assertLen(score_val, 2) - self.assertEqual(score_val[0].shape, (self.batch, self.timestep)) - self.assertEqual(score_val[1].shape, (self.batch, self.timestep)) - - @parameterized.named_parameters( - ("luong", wrapper.LuongAttentionV2), - ("luong_monotonic", wrapper.LuongMonotonicAttentionV2), - ("bahdanau", wrapper.BahdanauAttentionV2), - ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2), - ) - def test_passing_memory_from_call(self, attention_cls): - attention = attention_cls(self.units, self.memory) - weights_before_query = attention.get_weights() - ref_score = attention([self.query, self.state]) - - self.evaluate(variables.global_variables_initializer()) - ref_score_val = self.evaluate(ref_score) - - all_weights = attention.get_weights() - config = attention.get_config() - # Simulate the twice invocation of calls here. - attention_from_config = attention_cls.from_config(config) - attention_from_config.build(self.memory.shape) - attention_from_config.set_weights(weights_before_query) - attention_from_config(self.memory, setup_memory=True) - attention_from_config.build([self.query.shape, self.state.shape]) - attention_from_config.set_weights(all_weights) - score = attention_from_config([self.query, self.state]) - - score_val = self.evaluate(score) - self.assertAllClose(ref_score_val, score_val) - - @parameterized.named_parameters( - ("luong", wrapper.LuongAttentionV2), - ("luong_monotonic", wrapper.LuongMonotonicAttentionV2), - ("bahdanau", wrapper.BahdanauAttentionV2), - ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2), - ) - def test_save_load_layer(self, attention_cls): - vocab = 20 - embedding_dim = 6 - inputs = keras.layers.Input(shape=[self.timestep]) - encoder_input = keras.layers.Embedding( - vocab, embedding_dim, mask_zero=True)( - inputs) - encoder_output = keras.layers.LSTM( - self.memory_size, return_sequences=True)( - encoder_input) - - attention = attention_cls(self.units, encoder_output) - query = keras.layers.Input(shape=[self.units]) - state = keras.layers.Input(shape=[self.timestep]) - - score = attention([query, state]) - - x = np.random.randint(vocab, size=(self.batch, self.timestep)) - x_test = np.random.randint(vocab, size=(self.batch, self.timestep)) - y = np.random.randn(self.batch, self.timestep) - model = keras.models.Model([inputs, query, state], score) - # TODO(b/138592586): Run with single-execution-path - model.compile("rmsprop", "mse", experimental_run_tf_function=False) - model.fit([x, self.query, self.state], (y, y)) - y_ref = model.predict_on_batch([x_test, self.query, self.state]) - - config = model.get_config() - weights = model.get_weights() - loaded_model = keras.models.Model.from_config( - config, custom_objects={attention_cls.__name__: attention_cls}) - loaded_model.set_weights(weights) - - # TODO(b/138592586): Run with single-execution-path - loaded_model.compile("rmsprop", "mse", experimental_run_tf_function=False) - - y = loaded_model.predict_on_batch([x_test, self.query, self.state]) - - self.assertAllClose(y_ref, y) - - # TODO(scottzhu): Add tests for model.compile(run_eagerly=True) - - -class ResultSummary( - collections.namedtuple("ResultSummary", ("shape", "dtype", "mean"))): - pass - - -def get_result_summary(x): - if isinstance(x, np.ndarray): - return ResultSummary(x.shape, x.dtype, x.mean()) - return x - - -@test_util.run_all_in_graph_and_eager_modes -class AttentionWrapperV2Test(test.TestCase, parameterized.TestCase): - - def assertAllCloseOrEqual(self, x, y, **kwargs): - if isinstance(x, np.ndarray) or isinstance(x, float): - return super(AttentionWrapperV2Test, self).assertAllClose( - x, y, atol=1e-3, **kwargs) - else: - self.assertAllEqual(x, y, **kwargs) - - def setUp(self): - super(AttentionWrapperV2Test, self).setUp() - self.batch = 64 - self.units = 128 - self.encoder_timestep = 10 - self.encoder_dim = 256 - self.decoder_timestep = 12 - self.encoder_outputs = np.random.randn(self.batch, self.encoder_timestep, - self.encoder_dim) - self.encoder_sequence_length = np.random.randint( - self.encoder_timestep, size=(self.batch,)).astype(np.int32) - self.decoder_inputs = np.random.randn(self.batch, self.decoder_timestep, - self.units) - self.decoder_sequence_length = np.random.randint( - self.decoder_timestep, size=(self.batch,)).astype(np.int32) - - def _testWithAttention(self, - create_attention_mechanism, - expected_final_output, - expected_final_state, - attention_mechanism_depth=3, - alignment_history=False, - expected_final_alignment_history=None, - attention_layer_size=6, - attention_layer=None, - create_query_layer=False, - create_memory_layer=True, - create_attention_kwargs=None): - attention_layer_sizes = ([attention_layer_size] - if attention_layer_size is not None else None) - attention_layers = ([attention_layer] - if attention_layer is not None else None) - self._testWithMaybeMultiAttention( - is_multi=False, - create_attention_mechanisms=[create_attention_mechanism], - expected_final_output=expected_final_output, - expected_final_state=expected_final_state, - attention_mechanism_depths=[attention_mechanism_depth], - alignment_history=alignment_history, - expected_final_alignment_history=expected_final_alignment_history, - attention_layer_sizes=attention_layer_sizes, - attention_layers=attention_layers, - create_query_layer=create_query_layer, - create_memory_layer=create_memory_layer, - create_attention_kwargs=create_attention_kwargs) - - def _testWithMaybeMultiAttention(self, - is_multi, - create_attention_mechanisms, - expected_final_output, - expected_final_state, - attention_mechanism_depths, - alignment_history=False, - expected_final_alignment_history=None, - attention_layer_sizes=None, - attention_layers=None, - create_query_layer=False, - create_memory_layer=True, - create_attention_kwargs=None): - # Allow is_multi to be True with a single mechanism to enable test for - # passing in a single mechanism in a list. - assert len(create_attention_mechanisms) == 1 or is_multi - encoder_sequence_length = [3, 2, 3, 1, 1] - decoder_sequence_length = [2, 0, 1, 2, 3] - batch_size = 5 - encoder_max_time = 8 - decoder_max_time = 4 - input_depth = 7 - encoder_output_depth = 10 - cell_depth = 9 - create_attention_kwargs = create_attention_kwargs or {} - - if attention_layer_sizes is not None: - # Compute sum of attention_layer_sizes. Use encoder_output_depth if None. - attention_depth = sum(attention_layer_size or encoder_output_depth - for attention_layer_size in attention_layer_sizes) - elif attention_layers is not None: - # Compute sum of attention_layers output depth. - attention_depth = sum( - attention_layer.compute_output_shape( - [batch_size, cell_depth + encoder_output_depth]).dims[-1].value - for attention_layer in attention_layers) - else: - attention_depth = encoder_output_depth * len(create_attention_mechanisms) - - decoder_inputs = np.random.randn(batch_size, decoder_max_time, - input_depth).astype(np.float32) - encoder_outputs = np.random.randn(batch_size, encoder_max_time, - encoder_output_depth).astype(np.float32) - - attention_mechanisms = [] - for creator, depth in zip(create_attention_mechanisms, - attention_mechanism_depths): - # Create a memory layer with deterministic initializer to avoid randomness - # in the test between graph and eager. - if create_query_layer: - create_attention_kwargs["query_layer"] = keras.layers.Dense( - depth, kernel_initializer="ones", use_bias=False) - if create_memory_layer: - create_attention_kwargs["memory_layer"] = keras.layers.Dense( - depth, kernel_initializer="ones", use_bias=False) - - attention_mechanisms.append( - creator( - units=depth, - memory=encoder_outputs, - memory_sequence_length=encoder_sequence_length, - **create_attention_kwargs)) - - with self.cached_session(use_gpu=True): - attention_layer_size = attention_layer_sizes - attention_layer = attention_layers - if not is_multi: - if attention_layer_size is not None: - attention_layer_size = attention_layer_size[0] - if attention_layer is not None: - attention_layer = attention_layer[0] - cell = keras.layers.LSTMCell(cell_depth, - recurrent_activation="sigmoid", - kernel_initializer="ones", - recurrent_initializer="ones") - cell = wrapper.AttentionWrapper( - cell, - attention_mechanisms if is_multi else attention_mechanisms[0], - attention_layer_size=attention_layer_size, - alignment_history=alignment_history, - attention_layer=attention_layer) - if cell._attention_layers is not None: - for layer in cell._attention_layers: - if getattr(layer, "kernel_initializer") is None: - layer.kernel_initializer = initializers.glorot_uniform(seed=1337) - - sampler = sampler_py.TrainingSampler() - my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler) - initial_state = cell.get_initial_state( - dtype=dtypes.float32, batch_size=batch_size) - final_outputs, final_state, _ = my_decoder( - decoder_inputs, - initial_state=initial_state, - sequence_length=decoder_sequence_length) - - self.assertIsInstance(final_outputs, basic_decoder.BasicDecoderOutput) - self.assertIsInstance(final_state, wrapper.AttentionWrapperState) - - expected_time = ( - expected_final_state.time if context.executing_eagerly() else None) - self.assertEqual((batch_size, expected_time, attention_depth), - tuple(final_outputs.rnn_output.get_shape().as_list())) - self.assertEqual((batch_size, expected_time), - tuple(final_outputs.sample_id.get_shape().as_list())) - - self.assertEqual((batch_size, attention_depth), - tuple(final_state.attention.get_shape().as_list())) - self.assertEqual((batch_size, cell_depth), - tuple(final_state.cell_state[0].get_shape().as_list())) - self.assertEqual((batch_size, cell_depth), - tuple(final_state.cell_state[1].get_shape().as_list())) - - if alignment_history: - if is_multi: - state_alignment_history = [] - for history_array in final_state.alignment_history: - history = history_array.stack() - self.assertEqual((expected_time, batch_size, encoder_max_time), - tuple(history.get_shape().as_list())) - state_alignment_history.append(history) - state_alignment_history = tuple(state_alignment_history) - else: - state_alignment_history = final_state.alignment_history.stack() - self.assertEqual((expected_time, batch_size, encoder_max_time), - tuple(state_alignment_history.get_shape().as_list())) - nest.assert_same_structure(cell.state_size, - cell.zero_state(batch_size, dtypes.float32)) - # Remove the history from final_state for purposes of the - # remainder of the tests. - final_state = final_state._replace(alignment_history=()) # pylint: disable=protected-access - else: - state_alignment_history = () - - self.evaluate(variables.global_variables_initializer()) - eval_result = self.evaluate({ - "final_outputs": final_outputs, - "final_state": final_state, - "state_alignment_history": state_alignment_history, - }) - - final_output_info = nest.map_structure(get_result_summary, - eval_result["final_outputs"]) - final_state_info = nest.map_structure(get_result_summary, - eval_result["final_state"]) - print("final_output_info: ", final_output_info) - print("final_state_info: ", final_state_info) - - nest.map_structure(self.assertAllCloseOrEqual, expected_final_output, - final_output_info) - nest.map_structure(self.assertAllCloseOrEqual, expected_final_state, - final_state_info) - if alignment_history: # by default, the wrapper emits attention as output - final_alignment_history_info = nest.map_structure( - get_result_summary, eval_result["state_alignment_history"]) - print("final_alignment_history_info: ", final_alignment_history_info) - nest.map_structure( - self.assertAllCloseOrEqual, - # outputs are batch major but the stacked TensorArray is time major - expected_final_alignment_history, - final_alignment_history_info) - - # TODO(b/126893309): reenable np.float16 once the bug is fixed. - @parameterized.parameters([np.float32, np.float64]) - def testBahdanauNormalizedDType(self, dtype): - encoder_outputs = self.encoder_outputs.astype(dtype) - decoder_inputs = self.decoder_inputs.astype(dtype) - attention_mechanism = wrapper.BahdanauAttentionV2( - units=self.units, - memory=encoder_outputs, - memory_sequence_length=self.encoder_sequence_length, - normalize=True, - dtype=dtype) - cell = keras.layers.LSTMCell(self.units, recurrent_activation="sigmoid", - dtype=dtype) - cell = wrapper.AttentionWrapper(cell, attention_mechanism, dtype=dtype) - - sampler = sampler_py.TrainingSampler() - my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler, - dtype=dtype) - - final_outputs, final_state, _ = my_decoder( - decoder_inputs, - initial_state=cell.zero_state(dtype=dtype, batch_size=self.batch), - sequence_length=self.decoder_sequence_length) - self.assertIsInstance(final_outputs, basic_decoder.BasicDecoderOutput) - self.assertEqual(final_outputs.rnn_output.dtype, dtype) - self.assertIsInstance(final_state, wrapper.AttentionWrapperState) - - # TODO(b/126893309): reenable np.float16 once the bug is fixed. - @parameterized.parameters([np.float32, np.float64]) - def testLuongScaledDType(self, dtype): - # Test case for GitHub issue 18099 - encoder_outputs = self.encoder_outputs.astype(dtype) - decoder_inputs = self.decoder_inputs.astype(dtype) - attention_mechanism = wrapper.LuongAttentionV2( - units=self.units, - memory=encoder_outputs, - memory_sequence_length=self.encoder_sequence_length, - scale=True, - dtype=dtype, - ) - cell = keras.layers.LSTMCell(self.units, recurrent_activation="sigmoid", - dtype=dtype) - cell = wrapper.AttentionWrapper(cell, attention_mechanism, dtype=dtype) - - sampler = sampler_py.TrainingSampler() - my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler, - dtype=dtype) - - final_outputs, final_state, _ = my_decoder( - decoder_inputs, - initial_state=cell.zero_state(dtype=dtype, batch_size=self.batch), - sequence_length=self.decoder_sequence_length) - self.assertIsInstance(final_outputs, basic_decoder.BasicDecoderOutput) - self.assertEqual(final_outputs.rnn_output.dtype, dtype) - self.assertIsInstance(final_state, wrapper.AttentionWrapperState) - - def testBahdanauNotNormalized(self): - create_attention_mechanism = wrapper.BahdanauAttentionV2 - create_attention_kwargs = {"kernel_initializer": "ones"} - expected_final_output = basic_decoder.BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=np.dtype(np.float32), mean=0.051747426), - sample_id=ResultSummary( - shape=(5, 3), dtype=np.dtype(np.int32), mean=3.33333333)) - expected_final_state = wrapper.AttentionWrapperState( - cell_state=[ - ResultSummary( - shape=(5, 9), dtype=np.dtype(np.float32), mean=0.44189346), - ResultSummary( - shape=(5, 9), dtype=np.dtype(np.float32), mean=0.65429491)], - attention=ResultSummary( - shape=(5, 6), dtype=np.dtype(np.float32), mean=0.073610783), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=np.dtype(np.float32), mean=0.125), - attention_state=ResultSummary( - shape=(5, 8), dtype=np.dtype(np.float32), mean=0.125), - alignment_history=()) - expected_final_alignment_history = ResultSummary( - shape=(3, 5, 8), dtype=np.dtype(np.float32), mean=0.125) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - alignment_history=True, - create_query_layer=True, - expected_final_alignment_history=expected_final_alignment_history, - create_attention_kwargs=create_attention_kwargs) - - def testBahdanauNormalized(self): - create_attention_mechanism = wrapper.BahdanauAttentionV2 - create_attention_kwargs = {"kernel_initializer": "ones", "normalize": True} - - expected_final_output = basic_decoder.BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=np.dtype("float32"), mean=0.047594748), - sample_id=ResultSummary( - shape=(5, 3), dtype=np.dtype("int32"), mean=3.6)) - expected_final_state = wrapper.AttentionWrapperState( - cell_state=[ - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.41311637), - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.61683208)], - attention=ResultSummary( - shape=(5, 6), dtype=np.dtype("float32"), mean=0.090581432), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.125), - attention_state=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.125), - alignment_history=()) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - create_query_layer=True, - create_attention_kwargs=create_attention_kwargs) - - def testLuongNotNormalized(self): - create_attention_mechanism = wrapper.LuongAttentionV2 - - expected_final_output = basic_decoder.BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=np.dtype("float32"), mean=0.05481226), - sample_id=ResultSummary( - shape=(5, 3), dtype=np.dtype("int32"), mean=3.13333333)) - expected_final_state = wrapper.AttentionWrapperState( - cell_state=[ - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.38453412), - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.5785929)], - attention=ResultSummary( - shape=(5, 6), dtype=np.dtype("float32"), mean=0.16311775), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.125), - attention_state=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.125), - alignment_history=()) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - attention_mechanism_depth=9) - - def testLuongScaled(self): - create_attention_mechanism = wrapper.LuongAttentionV2 - create_attention_kwargs = {"scale": True} - - expected_final_output = basic_decoder.BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=np.dtype("float32"), mean=0.05481226), - sample_id=ResultSummary( - shape=(5, 3), dtype=np.dtype("int32"), mean=3.13333333)) - expected_final_state = wrapper.AttentionWrapperState( - cell_state=[ - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.38453412), - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.5785929)], - attention=ResultSummary( - shape=(5, 6), dtype=np.dtype("float32"), mean=0.16311775), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.125), - attention_state=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.125), - alignment_history=()) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - attention_mechanism_depth=9, - create_attention_kwargs=create_attention_kwargs) - - def testNotUseAttentionLayer(self): - create_attention_mechanism = wrapper.BahdanauAttentionV2 - create_attention_kwargs = {"kernel_initializer": "ones"} - - expected_final_output = basic_decoder.BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 10), dtype=np.dtype("float32"), mean=0.072406612), - sample_id=ResultSummary( - shape=(5, 3), dtype=np.dtype("int32"), mean=3.86666666)) - expected_final_state = wrapper.AttentionWrapperState( - cell_state=[ - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.61177742), - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=1.032002)], - attention=ResultSummary( - shape=(5, 10), dtype=np.dtype("float32"), mean=0.011346335), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.125), - attention_state=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.125), - alignment_history=()) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - attention_layer_size=None, - create_query_layer=True, - create_attention_kwargs=create_attention_kwargs) - - def testBahdanauMonotonicNotNormalized(self): - create_attention_mechanism = wrapper.BahdanauMonotonicAttentionV2 - create_attention_kwargs = {"kernel_initializer": "ones"} - - expected_final_output = basic_decoder.BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=np.dtype("float32"), mean=0.041342419), - sample_id=ResultSummary( - shape=(5, 3), dtype=np.dtype("int32"), mean=3.53333333)) - expected_final_state = wrapper.AttentionWrapperState( - cell_state=[ - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.33866978), - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.46913195)], - attention=ResultSummary( - shape=(5, 6), dtype=np.dtype("float32"), mean=0.092498459), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.12079944), - attention_state=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.12079944), - alignment_history=()) - expected_final_alignment_history = ResultSummary( - shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.121448785067) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - alignment_history=True, - expected_final_alignment_history=expected_final_alignment_history, - create_query_layer=True, - create_attention_kwargs=create_attention_kwargs) - - def testBahdanauMonotonicNormalized(self): - create_attention_mechanism = wrapper.BahdanauMonotonicAttentionV2 - create_attention_kwargs = {"kernel_initializer": "ones", - "normalize": True} - expected_final_output = basic_decoder.BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=np.dtype("float32"), mean=0.043294173), - sample_id=ResultSummary( - shape=(5, 3), dtype=np.dtype("int32"), mean=3.53333333)) - expected_final_state = wrapper.AttentionWrapperState( - cell_state=[ - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.40034312), - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.5925445)], - attention=ResultSummary( - shape=(5, 6), dtype=np.dtype("float32"), mean=0.096119694), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.1211452), - attention_state=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.1211452), - alignment_history=()) - expected_final_alignment_history = ResultSummary( - shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.12258384) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - alignment_history=True, - expected_final_alignment_history=expected_final_alignment_history, - create_query_layer=True, - create_attention_kwargs=create_attention_kwargs) - - def testLuongMonotonicNotNormalized(self): - create_attention_mechanism = wrapper.LuongMonotonicAttentionV2 - - expected_final_output = basic_decoder.BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=np.dtype("float32"), mean=0.027387079), - sample_id=ResultSummary( - shape=(5, 3), dtype=np.dtype("int32"), mean=3.133333333)) - expected_final_state = wrapper.AttentionWrapperState( - cell_state=[ - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.32660431), - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.52464348)], - attention=ResultSummary( - shape=(5, 6), dtype=np.dtype("float32"), mean=0.089345723), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.11831035), - attention_state=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.11831035), - alignment_history=()) - expected_final_alignment_history = ResultSummary( - shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.12194442004) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - attention_mechanism_depth=9, - alignment_history=True, - expected_final_alignment_history=expected_final_alignment_history) - - def testLuongMonotonicScaled(self): - create_attention_mechanism = wrapper.LuongMonotonicAttentionV2 - create_attention_kwargs = {"scale": True} - - expected_final_output = basic_decoder.BasicDecoderOutput( - rnn_output=ResultSummary( - shape=(5, 3, 6), dtype=np.dtype("float32"), mean=0.027387079), - sample_id=ResultSummary( - shape=(5, 3), dtype=np.dtype("int32"), mean=3.13333333)) - expected_final_state = wrapper.AttentionWrapperState( - cell_state=[ - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.32660431), - ResultSummary( - shape=(5, 9), dtype=np.dtype("float32"), mean=0.52464348)], - attention=ResultSummary( - shape=(5, 6), dtype=np.dtype("float32"), mean=0.089345723), - time=3, - alignments=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.11831035), - attention_state=ResultSummary( - shape=(5, 8), dtype=np.dtype("float32"), mean=0.11831035), - alignment_history=()) - expected_final_alignment_history = ResultSummary( - shape=(3, 5, 8), dtype=np.dtype("float32"), mean=0.12194442004) - - self._testWithAttention( - create_attention_mechanism, - expected_final_output, - expected_final_state, - attention_mechanism_depth=9, - alignment_history=True, - expected_final_alignment_history=expected_final_alignment_history, - create_attention_kwargs=create_attention_kwargs) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_test.py deleted file mode 100644 index 2183761bf11..00000000000 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_test.py +++ /dev/null @@ -1,667 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for contrib.seq2seq.python.seq2seq.basic_decoder.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.seq2seq.python.ops import basic_decoder -from tensorflow.contrib.seq2seq.python.ops import helper as helper_py - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import test_util -from tensorflow.python.layers import core as layers_core -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -@test_util.run_v1_only("contrib code not supported in TF2.0") -class BasicDecoderTest(test.TestCase): - - def _testStepWithTrainingHelper(self, use_output_layer): - sequence_length = [3, 4, 3, 1, 0] - batch_size = 5 - max_time = 8 - input_depth = 7 - cell_depth = 10 - output_layer_depth = 3 - - with self.session(use_gpu=True) as sess: - inputs = np.random.randn(batch_size, max_time, - input_depth).astype(np.float32) - cell = rnn_cell.LSTMCell(cell_depth) - helper = helper_py.TrainingHelper( - inputs, sequence_length, time_major=False) - if use_output_layer: - output_layer = layers_core.Dense(output_layer_depth, use_bias=False) - expected_output_depth = output_layer_depth - else: - output_layer = None - expected_output_depth = cell_depth - my_decoder = basic_decoder.BasicDecoder( - cell=cell, - helper=helper, - initial_state=cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size), - output_layer=output_layer) - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(expected_output_depth, - tensor_shape.TensorShape([])), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), - output_dtype) - - (first_finished, first_inputs, first_state) = my_decoder.initialize() - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, expected_output_depth), - step_outputs[0].get_shape()) - self.assertEqual((batch_size,), step_outputs[1].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) - - if use_output_layer: - # The output layer was accessed - self.assertEqual(len(output_layer.variables), 1) - - sess.run(variables.global_variables_initializer()) - sess_results = sess.run({ - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - }) - - self.assertAllEqual([False, False, False, False, True], - sess_results["first_finished"]) - self.assertAllEqual([False, False, False, True, True], - sess_results["step_finished"]) - self.assertEqual(output_dtype.sample_id, - sess_results["step_outputs"].sample_id.dtype) - self.assertAllEqual( - np.argmax(sess_results["step_outputs"].rnn_output, -1), - sess_results["step_outputs"].sample_id) - - def testStepWithTrainingHelperNoOutputLayer(self): - self._testStepWithTrainingHelper(use_output_layer=False) - - def testStepWithTrainingHelperWithOutputLayer(self): - self._testStepWithTrainingHelper(use_output_layer=True) - - def testStepWithGreedyEmbeddingHelper(self): - batch_size = 5 - vocabulary_size = 7 - cell_depth = vocabulary_size # cell's logits must match vocabulary size - input_depth = 10 - start_tokens = np.random.randint(0, vocabulary_size, size=batch_size) - end_token = 1 - - with self.session(use_gpu=True) as sess: - embeddings = np.random.randn(vocabulary_size, - input_depth).astype(np.float32) - cell = rnn_cell.LSTMCell(vocabulary_size) - helper = helper_py.GreedyEmbeddingHelper(embeddings, start_tokens, - end_token) - my_decoder = basic_decoder.BasicDecoder( - cell=cell, - helper=helper, - initial_state=cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size)) - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(cell_depth, - tensor_shape.TensorShape([])), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), - output_dtype) - - (first_finished, first_inputs, first_state) = my_decoder.initialize() - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape()) - self.assertEqual((batch_size,), step_outputs[1].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) - - sess.run(variables.global_variables_initializer()) - sess_results = sess.run({ - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - }) - - expected_sample_ids = np.argmax( - sess_results["step_outputs"].rnn_output, -1) - expected_step_finished = (expected_sample_ids == end_token) - expected_step_next_inputs = embeddings[expected_sample_ids] - self.assertAllEqual([False, False, False, False, False], - sess_results["first_finished"]) - self.assertAllEqual(expected_step_finished, sess_results["step_finished"]) - self.assertEqual(output_dtype.sample_id, - sess_results["step_outputs"].sample_id.dtype) - self.assertAllEqual(expected_sample_ids, - sess_results["step_outputs"].sample_id) - self.assertAllEqual(expected_step_next_inputs, - sess_results["step_next_inputs"]) - - def testStepWithSampleEmbeddingHelper(self): - batch_size = 5 - vocabulary_size = 7 - cell_depth = vocabulary_size # cell's logits must match vocabulary size - input_depth = 10 - np.random.seed(0) - start_tokens = np.random.randint(0, vocabulary_size, size=batch_size) - end_token = 1 - - with self.session(use_gpu=True) as sess: - with variable_scope.variable_scope( - "testStepWithSampleEmbeddingHelper", - initializer=init_ops.constant_initializer(0.01)): - embeddings = np.random.randn(vocabulary_size, - input_depth).astype(np.float32) - cell = rnn_cell.LSTMCell(vocabulary_size) - helper = helper_py.SampleEmbeddingHelper(embeddings, start_tokens, - end_token, seed=0) - my_decoder = basic_decoder.BasicDecoder( - cell=cell, - helper=helper, - initial_state=cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size)) - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(cell_depth, - tensor_shape.TensorShape([])), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), - output_dtype) - - (first_finished, first_inputs, first_state) = my_decoder.initialize() - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape()) - self.assertEqual((batch_size,), step_outputs[1].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) - - sess.run(variables.global_variables_initializer()) - sess_results = sess.run({ - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - }) - - sample_ids = sess_results["step_outputs"].sample_id - self.assertEqual(output_dtype.sample_id, sample_ids.dtype) - expected_step_finished = (sample_ids == end_token) - expected_step_next_inputs = embeddings[sample_ids] - self.assertAllEqual(expected_step_finished, - sess_results["step_finished"]) - self.assertAllEqual(expected_step_next_inputs, - sess_results["step_next_inputs"]) - - def testStepWithScheduledEmbeddingTrainingHelper(self): - sequence_length = [3, 4, 3, 1, 0] - batch_size = 5 - max_time = 8 - input_depth = 7 - vocabulary_size = 10 - - with self.session(use_gpu=True) as sess: - inputs = np.random.randn( - batch_size, max_time, input_depth).astype(np.float32) - embeddings = np.random.randn( - vocabulary_size, input_depth).astype(np.float32) - half = constant_op.constant(0.5) - cell = rnn_cell.LSTMCell(vocabulary_size) - helper = helper_py.ScheduledEmbeddingTrainingHelper( - inputs=inputs, - sequence_length=sequence_length, - embedding=embeddings, - sampling_probability=half, - time_major=False) - my_decoder = basic_decoder.BasicDecoder( - cell=cell, - helper=helper, - initial_state=cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size)) - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(vocabulary_size, - tensor_shape.TensorShape([])), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), - output_dtype) - - (first_finished, first_inputs, first_state) = my_decoder.initialize() - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, vocabulary_size), - step_outputs[0].get_shape()) - self.assertEqual((batch_size,), step_outputs[1].get_shape()) - self.assertEqual((batch_size, vocabulary_size), - first_state[0].get_shape()) - self.assertEqual((batch_size, vocabulary_size), - first_state[1].get_shape()) - self.assertEqual((batch_size, vocabulary_size), - step_state[0].get_shape()) - self.assertEqual((batch_size, vocabulary_size), - step_state[1].get_shape()) - self.assertEqual((batch_size, input_depth), - step_next_inputs.get_shape()) - - sess.run(variables.global_variables_initializer()) - sess_results = sess.run({ - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - }) - - self.assertAllEqual([False, False, False, False, True], - sess_results["first_finished"]) - self.assertAllEqual([False, False, False, True, True], - sess_results["step_finished"]) - sample_ids = sess_results["step_outputs"].sample_id - self.assertEqual(output_dtype.sample_id, sample_ids.dtype) - batch_where_not_sampling = np.where(sample_ids == -1) - batch_where_sampling = np.where(sample_ids > -1) - self.assertAllClose( - sess_results["step_next_inputs"][batch_where_sampling], - embeddings[sample_ids[batch_where_sampling]]) - self.assertAllClose( - sess_results["step_next_inputs"][batch_where_not_sampling], - np.squeeze(inputs[batch_where_not_sampling, 1])) - - def _testStepWithScheduledOutputTrainingHelper( - self, sampling_probability, use_next_inputs_fn, use_auxiliary_inputs): - sequence_length = [3, 4, 3, 1, 0] - batch_size = 5 - max_time = 8 - input_depth = 7 - cell_depth = input_depth - if use_auxiliary_inputs: - auxiliary_input_depth = 4 - auxiliary_inputs = np.random.randn( - batch_size, max_time, auxiliary_input_depth).astype(np.float32) - else: - auxiliary_inputs = None - - with self.session(use_gpu=True) as sess: - inputs = np.random.randn(batch_size, max_time, - input_depth).astype(np.float32) - cell = rnn_cell.LSTMCell(cell_depth) - sampling_probability = constant_op.constant(sampling_probability) - - if use_next_inputs_fn: - def next_inputs_fn(outputs): - # Use deterministic function for test. - samples = math_ops.argmax(outputs, axis=1) - return array_ops.one_hot(samples, cell_depth, dtype=dtypes.float32) - else: - next_inputs_fn = None - - helper = helper_py.ScheduledOutputTrainingHelper( - inputs=inputs, - sequence_length=sequence_length, - sampling_probability=sampling_probability, - time_major=False, - next_inputs_fn=next_inputs_fn, - auxiliary_inputs=auxiliary_inputs) - - my_decoder = basic_decoder.BasicDecoder( - cell=cell, - helper=helper, - initial_state=cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size)) - - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(cell_depth, - tensor_shape.TensorShape([])), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), - output_dtype) - - (first_finished, first_inputs, first_state) = my_decoder.initialize() - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - - if use_next_inputs_fn: - output_after_next_inputs_fn = next_inputs_fn(step_outputs.rnn_output) - - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape()) - self.assertEqual((batch_size,), step_outputs[1].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) - - sess.run(variables.global_variables_initializer()) - - fetches = { - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - } - if use_next_inputs_fn: - fetches["output_after_next_inputs_fn"] = output_after_next_inputs_fn - - sess_results = sess.run(fetches) - - self.assertAllEqual([False, False, False, False, True], - sess_results["first_finished"]) - self.assertAllEqual([False, False, False, True, True], - sess_results["step_finished"]) - - sample_ids = sess_results["step_outputs"].sample_id - self.assertEqual(output_dtype.sample_id, sample_ids.dtype) - batch_where_not_sampling = np.where(np.logical_not(sample_ids)) - batch_where_sampling = np.where(sample_ids) - - auxiliary_inputs_to_concat = ( - auxiliary_inputs[:, 1] if use_auxiliary_inputs else - np.array([]).reshape(batch_size, 0).astype(np.float32)) - - expected_next_sampling_inputs = np.concatenate( - (sess_results["output_after_next_inputs_fn"][batch_where_sampling] - if use_next_inputs_fn else - sess_results["step_outputs"].rnn_output[batch_where_sampling], - auxiliary_inputs_to_concat[batch_where_sampling]), - axis=-1) - self.assertAllClose( - sess_results["step_next_inputs"][batch_where_sampling], - expected_next_sampling_inputs) - - self.assertAllClose( - sess_results["step_next_inputs"][batch_where_not_sampling], - np.concatenate( - (np.squeeze(inputs[batch_where_not_sampling, 1], axis=0), - auxiliary_inputs_to_concat[batch_where_not_sampling]), - axis=-1)) - - def testStepWithScheduledOutputTrainingHelperWithoutNextInputsFnOrAuxInputs( - self): - self._testStepWithScheduledOutputTrainingHelper( - sampling_probability=0.5, use_next_inputs_fn=False, - use_auxiliary_inputs=False) - - def testStepWithScheduledOutputTrainingHelperWithNextInputsFn(self): - self._testStepWithScheduledOutputTrainingHelper( - sampling_probability=0.5, use_next_inputs_fn=True, - use_auxiliary_inputs=False) - - def testStepWithScheduledOutputTrainingHelperWithAuxiliaryInputs(self): - self._testStepWithScheduledOutputTrainingHelper( - sampling_probability=0.5, use_next_inputs_fn=False, - use_auxiliary_inputs=True) - - def testStepWithScheduledOutputTrainingHelperWithNextInputsFnAndAuxInputs( - self): - self._testStepWithScheduledOutputTrainingHelper( - sampling_probability=0.5, use_next_inputs_fn=True, - use_auxiliary_inputs=True) - - def testStepWithScheduledOutputTrainingHelperWithNoSampling(self): - self._testStepWithScheduledOutputTrainingHelper( - sampling_probability=0.0, use_next_inputs_fn=True, - use_auxiliary_inputs=True) - - def testStepWithInferenceHelperCategorical(self): - batch_size = 5 - vocabulary_size = 7 - cell_depth = vocabulary_size - start_token = 0 - end_token = 6 - - start_inputs = array_ops.one_hot( - np.ones(batch_size) * start_token, - vocabulary_size) - - # The sample function samples categorically from the logits. - sample_fn = lambda x: helper_py.categorical_sample(logits=x) - # The next inputs are a one-hot encoding of the sampled labels. - next_inputs_fn = ( - lambda x: array_ops.one_hot(x, vocabulary_size, dtype=dtypes.float32)) - end_fn = lambda sample_ids: math_ops.equal(sample_ids, end_token) - - with self.session(use_gpu=True) as sess: - with variable_scope.variable_scope( - "testStepWithInferenceHelper", - initializer=init_ops.constant_initializer(0.01)): - cell = rnn_cell.LSTMCell(vocabulary_size) - helper = helper_py.InferenceHelper( - sample_fn, sample_shape=(), sample_dtype=dtypes.int32, - start_inputs=start_inputs, end_fn=end_fn, - next_inputs_fn=next_inputs_fn) - my_decoder = basic_decoder.BasicDecoder( - cell=cell, - helper=helper, - initial_state=cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size)) - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(cell_depth, - tensor_shape.TensorShape([])), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), - output_dtype) - - (first_finished, first_inputs, first_state) = my_decoder.initialize() - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape()) - self.assertEqual((batch_size,), step_outputs[1].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) - - sess.run(variables.global_variables_initializer()) - sess_results = sess.run({ - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - }) - - sample_ids = sess_results["step_outputs"].sample_id - self.assertEqual(output_dtype.sample_id, sample_ids.dtype) - expected_step_finished = (sample_ids == end_token) - expected_step_next_inputs = np.zeros((batch_size, vocabulary_size)) - expected_step_next_inputs[np.arange(batch_size), sample_ids] = 1.0 - self.assertAllEqual(expected_step_finished, - sess_results["step_finished"]) - self.assertAllEqual(expected_step_next_inputs, - sess_results["step_next_inputs"]) - - def testStepWithInferenceHelperMultilabel(self): - batch_size = 5 - vocabulary_size = 7 - cell_depth = vocabulary_size - start_token = 0 - end_token = 6 - - start_inputs = array_ops.one_hot( - np.ones(batch_size) * start_token, - vocabulary_size) - - # The sample function samples independent bernoullis from the logits. - sample_fn = ( - lambda x: helper_py.bernoulli_sample(logits=x, dtype=dtypes.bool)) - # The next inputs are a one-hot encoding of the sampled labels. - next_inputs_fn = math_ops.to_float - end_fn = lambda sample_ids: sample_ids[:, end_token] - - with self.session(use_gpu=True) as sess: - with variable_scope.variable_scope( - "testStepWithInferenceHelper", - initializer=init_ops.constant_initializer(0.01)): - cell = rnn_cell.LSTMCell(vocabulary_size) - helper = helper_py.InferenceHelper( - sample_fn, sample_shape=[cell_depth], sample_dtype=dtypes.bool, - start_inputs=start_inputs, end_fn=end_fn, - next_inputs_fn=next_inputs_fn) - my_decoder = basic_decoder.BasicDecoder( - cell=cell, - helper=helper, - initial_state=cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size)) - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(cell_depth, cell_depth), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.bool), - output_dtype) - - (first_finished, first_inputs, first_state) = my_decoder.initialize() - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_outputs[1].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) - - sess.run(variables.global_variables_initializer()) - sess_results = sess.run({ - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - }) - - sample_ids = sess_results["step_outputs"].sample_id - self.assertEqual(output_dtype.sample_id, sample_ids.dtype) - expected_step_finished = sample_ids[:, end_token] - expected_step_next_inputs = sample_ids.astype(np.float32) - self.assertAllEqual(expected_step_finished, - sess_results["step_finished"]) - self.assertAllEqual(expected_step_next_inputs, - sess_results["step_next_inputs"]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_v2_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_v2_test.py deleted file mode 100644 index 2341ebb77ab..00000000000 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_v2_test.py +++ /dev/null @@ -1,670 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for contrib.seq2seq.python.seq2seq.basic_decoder_v2.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy as np - -from tensorflow.contrib.seq2seq.python.ops import basic_decoder -from tensorflow.contrib.seq2seq.python.ops import sampler as sampler_py - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape -from tensorflow.python.keras import keras_parameterized -from tensorflow.python.layers import core as layers_core -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -@keras_parameterized.run_all_keras_modes -class BasicDecoderTest(keras_parameterized.TestCase): - """Unit test for basic_decoder.BasicDecoderV2.""" - - @parameterized.named_parameters( - ("use_output_layer", True), - ("without_output_layer", False)) - def testStepWithTrainingHelperOutputLayer(self, use_output_layer): - sequence_length = [3, 4, 3, 1, 0] - batch_size = 5 - max_time = 8 - input_depth = 7 - cell_depth = 10 - output_layer_depth = 3 - - with self.cached_session(use_gpu=True): - inputs = np.random.randn(batch_size, max_time, - input_depth).astype(np.float32) - input_t = constant_op.constant(inputs) - cell = rnn_cell.LSTMCell(cell_depth) - sampler = sampler_py.TrainingSampler(time_major=False) - if use_output_layer: - output_layer = layers_core.Dense(output_layer_depth, use_bias=False) - expected_output_depth = output_layer_depth - else: - output_layer = None - expected_output_depth = cell_depth - initial_state = cell.zero_state(dtype=dtypes.float32, - batch_size=batch_size) - my_decoder = basic_decoder.BasicDecoderV2( - cell=cell, - sampler=sampler, - output_layer=output_layer) - - (first_finished, - first_inputs, - first_state) = my_decoder.initialize(input_t, - initial_state=initial_state, - sequence_length=sequence_length) - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(expected_output_depth, - tensor_shape.TensorShape([])), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), - output_dtype) - - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, expected_output_depth), - step_outputs[0].get_shape()) - self.assertEqual((batch_size,), step_outputs[1].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) - - if use_output_layer: - # The output layer was accessed - self.assertEqual(len(output_layer.variables), 1) - - self.evaluate(variables.global_variables_initializer()) - eval_result = self.evaluate({ - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - }) - - self.assertAllEqual([False, False, False, False, True], - eval_result["first_finished"]) - self.assertAllEqual([False, False, False, True, True], - eval_result["step_finished"]) - self.assertEqual(output_dtype.sample_id, - eval_result["step_outputs"].sample_id.dtype) - self.assertAllEqual( - np.argmax(eval_result["step_outputs"].rnn_output, -1), - eval_result["step_outputs"].sample_id) - - def DISABLED_testStepWithGreedyEmbeddingHelper(self): - batch_size = 5 - vocabulary_size = 7 - cell_depth = vocabulary_size # cell's logits must match vocabulary size - input_depth = 10 - start_tokens = np.random.randint(0, vocabulary_size, size=batch_size) - end_token = 1 - - with self.cached_session(use_gpu=True): - embeddings = np.random.randn(vocabulary_size, - input_depth).astype(np.float32) - embeddings_t = constant_op.constant(embeddings) - cell = rnn_cell.LSTMCell(vocabulary_size) - sampler = sampler_py.GreedyEmbeddingSampler() - initial_state = cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size) - my_decoder = basic_decoder.BasicDecoderV2( - cell=cell, - sampler=sampler) - (first_finished, first_inputs, first_state) = my_decoder.initialize( - embeddings_t, - start_tokens=start_tokens, - end_token=end_token, - initial_state=initial_state) - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(cell_depth, - tensor_shape.TensorShape([])), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), - output_dtype) - - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape()) - self.assertEqual((batch_size,), step_outputs[1].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) - - self.evaluate(variables.global_variables_initializer()) - eval_result = self.evaluate({ - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - }) - - expected_sample_ids = np.argmax( - eval_result["step_outputs"].rnn_output, -1) - expected_step_finished = (expected_sample_ids == end_token) - expected_step_next_inputs = embeddings[expected_sample_ids] - self.assertAllEqual([False, False, False, False, False], - eval_result["first_finished"]) - self.assertAllEqual(expected_step_finished, eval_result["step_finished"]) - self.assertEqual(output_dtype.sample_id, - eval_result["step_outputs"].sample_id.dtype) - self.assertAllEqual(expected_sample_ids, - eval_result["step_outputs"].sample_id) - self.assertAllEqual(expected_step_next_inputs, - eval_result["step_next_inputs"]) - - def testStepWithSampleEmbeddingHelper(self): - batch_size = 5 - vocabulary_size = 7 - cell_depth = vocabulary_size # cell's logits must match vocabulary size - input_depth = 10 - np.random.seed(0) - start_tokens = np.random.randint(0, vocabulary_size, size=batch_size) - end_token = 1 - - with self.cached_session(use_gpu=True): - embeddings = np.random.randn(vocabulary_size, - input_depth).astype(np.float32) - embeddings_t = constant_op.constant(embeddings) - cell = rnn_cell.LSTMCell(vocabulary_size) - sampler = sampler_py.SampleEmbeddingSampler(seed=0) - initial_state = cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size) - my_decoder = basic_decoder.BasicDecoderV2(cell=cell, sampler=sampler) - (first_finished, - first_inputs, - first_state) = my_decoder.initialize(embeddings_t, - start_tokens=start_tokens, - end_token=end_token, - initial_state=initial_state) - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(cell_depth, - tensor_shape.TensorShape([])), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), - output_dtype) - - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape()) - self.assertEqual((batch_size,), step_outputs[1].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) - - self.evaluate(variables.global_variables_initializer()) - eval_result = self.evaluate({ - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - }) - - sample_ids = eval_result["step_outputs"].sample_id - self.assertEqual(output_dtype.sample_id, sample_ids.dtype) - expected_step_finished = (sample_ids == end_token) - expected_step_next_inputs = embeddings[sample_ids] - self.assertAllEqual(expected_step_finished, - eval_result["step_finished"]) - self.assertAllEqual(expected_step_next_inputs, - eval_result["step_next_inputs"]) - - def testStepWithScheduledEmbeddingTrainingHelper(self): - sequence_length = [3, 4, 3, 1, 0] - batch_size = 5 - max_time = 8 - input_depth = 7 - vocabulary_size = 10 - - with self.cached_session(use_gpu=True): - inputs = np.random.randn( - batch_size, max_time, input_depth).astype(np.float32) - input_t = constant_op.constant(inputs) - embeddings = np.random.randn( - vocabulary_size, input_depth).astype(np.float32) - half = constant_op.constant(0.5) - cell = rnn_cell.LSTMCell(vocabulary_size) - sampler = sampler_py.ScheduledEmbeddingTrainingSampler( - sampling_probability=half, - time_major=False) - initial_state = cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size) - my_decoder = basic_decoder.BasicDecoderV2( - cell=cell, - sampler=sampler) - (first_finished, first_inputs, first_state) = my_decoder.initialize( - input_t, sequence_length=sequence_length, embedding=embeddings, - initial_state=initial_state) - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(vocabulary_size, - tensor_shape.TensorShape([])), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), - output_dtype) - - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, vocabulary_size), - step_outputs[0].get_shape()) - self.assertEqual((batch_size,), step_outputs[1].get_shape()) - self.assertEqual((batch_size, vocabulary_size), - first_state[0].get_shape()) - self.assertEqual((batch_size, vocabulary_size), - first_state[1].get_shape()) - self.assertEqual((batch_size, vocabulary_size), - step_state[0].get_shape()) - self.assertEqual((batch_size, vocabulary_size), - step_state[1].get_shape()) - self.assertEqual((batch_size, input_depth), - step_next_inputs.get_shape()) - - self.evaluate(variables.global_variables_initializer()) - eval_result = self.evaluate({ - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - }) - - self.assertAllEqual([False, False, False, False, True], - eval_result["first_finished"]) - self.assertAllEqual([False, False, False, True, True], - eval_result["step_finished"]) - sample_ids = eval_result["step_outputs"].sample_id - self.assertEqual(output_dtype.sample_id, sample_ids.dtype) - batch_where_not_sampling = np.where(sample_ids == -1) - batch_where_sampling = np.where(sample_ids > -1) - self.assertAllClose( - eval_result["step_next_inputs"][batch_where_sampling], - embeddings[sample_ids[batch_where_sampling]]) - self.assertAllClose( - eval_result["step_next_inputs"][batch_where_not_sampling], - np.squeeze(inputs[batch_where_not_sampling, 1], axis=0)) - - def _testStepWithScheduledOutputTrainingHelper( - self, sampling_probability, use_next_inputs_fn, use_auxiliary_inputs): - sequence_length = [3, 4, 3, 1, 0] - batch_size = 5 - max_time = 8 - input_depth = 7 - cell_depth = input_depth - if use_auxiliary_inputs: - auxiliary_input_depth = 4 - auxiliary_inputs = np.random.randn( - batch_size, max_time, auxiliary_input_depth).astype(np.float32) - else: - auxiliary_inputs = None - - with self.cached_session(use_gpu=True): - inputs = np.random.randn(batch_size, max_time, - input_depth).astype(np.float32) - input_t = constant_op.constant(inputs) - cell = rnn_cell.LSTMCell(cell_depth) - sampling_probability = constant_op.constant(sampling_probability) - - if use_next_inputs_fn: - def next_inputs_fn(outputs): - # Use deterministic function for test. - samples = math_ops.argmax(outputs, axis=1) - return array_ops.one_hot(samples, cell_depth, dtype=dtypes.float32) - else: - next_inputs_fn = None - - sampler = sampler_py.ScheduledOutputTrainingSampler( - sampling_probability=sampling_probability, - time_major=False, - next_inputs_fn=next_inputs_fn) - initial_state = cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size) - - my_decoder = basic_decoder.BasicDecoderV2( - cell=cell, - sampler=sampler) - - (first_finished, - first_inputs, - first_state) = my_decoder.initialize(input_t, - sequence_length=sequence_length, - initial_state=initial_state, - auxiliary_inputs=auxiliary_inputs) - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(cell_depth, - tensor_shape.TensorShape([])), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), - output_dtype) - - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - - if use_next_inputs_fn: - output_after_next_inputs_fn = next_inputs_fn(step_outputs.rnn_output) - - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape()) - self.assertEqual((batch_size,), step_outputs[1].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) - - self.evaluate(variables.global_variables_initializer()) - - fetches = { - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - } - if use_next_inputs_fn: - fetches["output_after_next_inputs_fn"] = output_after_next_inputs_fn - - eval_result = self.evaluate(fetches) - - self.assertAllEqual([False, False, False, False, True], - eval_result["first_finished"]) - self.assertAllEqual([False, False, False, True, True], - eval_result["step_finished"]) - - sample_ids = eval_result["step_outputs"].sample_id - self.assertEqual(output_dtype.sample_id, sample_ids.dtype) - batch_where_not_sampling = np.where(np.logical_not(sample_ids)) - batch_where_sampling = np.where(sample_ids) - - auxiliary_inputs_to_concat = ( - auxiliary_inputs[:, 1] if use_auxiliary_inputs else - np.array([]).reshape(batch_size, 0).astype(np.float32)) - - expected_next_sampling_inputs = np.concatenate( - (eval_result["output_after_next_inputs_fn"][batch_where_sampling] - if use_next_inputs_fn else - eval_result["step_outputs"].rnn_output[batch_where_sampling], - auxiliary_inputs_to_concat[batch_where_sampling]), - axis=-1) - self.assertAllClose( - eval_result["step_next_inputs"][batch_where_sampling], - expected_next_sampling_inputs) - - self.assertAllClose( - eval_result["step_next_inputs"][batch_where_not_sampling], - np.concatenate( - (np.squeeze(inputs[batch_where_not_sampling, 1], axis=0), - auxiliary_inputs_to_concat[batch_where_not_sampling]), - axis=-1)) - - def testStepWithScheduledOutputTrainingHelperWithoutNextInputsFnOrAuxInputs( - self): - self._testStepWithScheduledOutputTrainingHelper( - sampling_probability=0.5, use_next_inputs_fn=False, - use_auxiliary_inputs=False) - - def testStepWithScheduledOutputTrainingHelperWithNextInputsFn(self): - self._testStepWithScheduledOutputTrainingHelper( - sampling_probability=0.5, use_next_inputs_fn=True, - use_auxiliary_inputs=False) - - def testStepWithScheduledOutputTrainingHelperWithAuxiliaryInputs(self): - self._testStepWithScheduledOutputTrainingHelper( - sampling_probability=0.5, use_next_inputs_fn=False, - use_auxiliary_inputs=True) - - def testStepWithScheduledOutputTrainingHelperWithNextInputsFnAndAuxInputs( - self): - self._testStepWithScheduledOutputTrainingHelper( - sampling_probability=0.5, use_next_inputs_fn=True, - use_auxiliary_inputs=True) - - def testStepWithScheduledOutputTrainingHelperWithNoSampling(self): - self._testStepWithScheduledOutputTrainingHelper( - sampling_probability=0.0, use_next_inputs_fn=True, - use_auxiliary_inputs=True) - - def testStepWithInferenceHelperCategorical(self): - batch_size = 5 - vocabulary_size = 7 - cell_depth = vocabulary_size - start_token = 0 - end_token = 6 - - start_inputs = array_ops.one_hot( - np.ones(batch_size, dtype=np.int32) * start_token, - vocabulary_size) - - # The sample function samples categorically from the logits. - sample_fn = lambda x: sampler_py.categorical_sample(logits=x) - # The next inputs are a one-hot encoding of the sampled labels. - next_inputs_fn = ( - lambda x: array_ops.one_hot(x, vocabulary_size, dtype=dtypes.float32)) - end_fn = lambda sample_ids: math_ops.equal(sample_ids, end_token) - - with self.cached_session(use_gpu=True): - cell = rnn_cell.LSTMCell(vocabulary_size) - sampler = sampler_py.InferenceSampler( - sample_fn, sample_shape=(), sample_dtype=dtypes.int32, end_fn=end_fn, - next_inputs_fn=next_inputs_fn) - initial_state = cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size) - my_decoder = basic_decoder.BasicDecoderV2( - cell=cell, - sampler=sampler) - (first_finished, first_inputs, first_state) = my_decoder.initialize( - start_inputs, initial_state=initial_state) - - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(cell_depth, - tensor_shape.TensorShape([])), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), - output_dtype) - - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape()) - self.assertEqual((batch_size,), step_outputs[1].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) - - self.evaluate(variables.global_variables_initializer()) - eval_result = self.evaluate({ - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - }) - - sample_ids = eval_result["step_outputs"].sample_id - self.assertEqual(output_dtype.sample_id, sample_ids.dtype) - expected_step_finished = (sample_ids == end_token) - expected_step_next_inputs = np.zeros((batch_size, vocabulary_size)) - expected_step_next_inputs[np.arange(batch_size), sample_ids] = 1.0 - self.assertAllEqual(expected_step_finished, - eval_result["step_finished"]) - self.assertAllEqual(expected_step_next_inputs, - eval_result["step_next_inputs"]) - - def testStepWithInferenceHelperMultilabel(self): - batch_size = 5 - vocabulary_size = 7 - cell_depth = vocabulary_size - start_token = 0 - end_token = 6 - - start_inputs = array_ops.one_hot( - np.ones(batch_size, dtype=np.int32) * start_token, - vocabulary_size) - - # The sample function samples independent bernoullis from the logits. - sample_fn = ( - lambda x: sampler_py.bernoulli_sample(logits=x, dtype=dtypes.bool)) - # The next inputs are a one-hot encoding of the sampled labels. - next_inputs_fn = math_ops.to_float - end_fn = lambda sample_ids: sample_ids[:, end_token] - - with self.cached_session(use_gpu=True): - cell = rnn_cell.LSTMCell(vocabulary_size) - sampler = sampler_py.InferenceSampler( - sample_fn, sample_shape=[cell_depth], sample_dtype=dtypes.bool, - end_fn=end_fn, next_inputs_fn=next_inputs_fn) - initial_state = cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size) - my_decoder = basic_decoder.BasicDecoderV2( - cell=cell, - sampler=sampler) - (first_finished, first_inputs, first_state) = my_decoder.initialize( - start_inputs, initial_state=initial_state) - output_size = my_decoder.output_size - output_dtype = my_decoder.output_dtype - self.assertEqual( - basic_decoder.BasicDecoderOutput(cell_depth, cell_depth), - output_size) - self.assertEqual( - basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.bool), - output_dtype) - - (step_outputs, step_state, step_next_inputs, - step_finished) = my_decoder.step( - constant_op.constant(0), first_inputs, first_state) - batch_size_t = my_decoder.batch_size - - self.assertTrue(isinstance(first_state, rnn_cell.LSTMStateTuple)) - self.assertTrue(isinstance(step_state, rnn_cell.LSTMStateTuple)) - self.assertTrue( - isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) - self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_outputs[1].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) - self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) - - self.evaluate(variables.global_variables_initializer()) - eval_result = self.evaluate({ - "batch_size": batch_size_t, - "first_finished": first_finished, - "first_inputs": first_inputs, - "first_state": first_state, - "step_outputs": step_outputs, - "step_state": step_state, - "step_next_inputs": step_next_inputs, - "step_finished": step_finished - }) - - sample_ids = eval_result["step_outputs"].sample_id - self.assertEqual(output_dtype.sample_id, sample_ids.dtype) - expected_step_finished = sample_ids[:, end_token] - expected_step_next_inputs = sample_ids.astype(np.float32) - self.assertAllEqual(expected_step_finished, - eval_result["step_finished"]) - self.assertAllEqual(expected_step_next_inputs, - eval_result["step_next_inputs"]) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py deleted file mode 100644 index 343e5f4be69..00000000000 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py +++ /dev/null @@ -1,704 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for contrib.seq2seq.python.seq2seq.beam_search_decoder.""" -# pylint: disable=unused-import,g-bad-import-order -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -# pylint: enable=unused-import - -import numpy as np - -from tensorflow.contrib.seq2seq.python.ops import attention_wrapper -from tensorflow.contrib.seq2seq.python.ops import beam_search_decoder -from tensorflow.contrib.seq2seq.python.ops import beam_search_ops -from tensorflow.contrib.seq2seq.python.ops import decoder -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.keras import layers -from tensorflow.python.layers import core as layers_core -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - -# pylint: enable=g-import-not-at-top - - -class TestGatherTree(test.TestCase): - """Tests the gather_tree function.""" - - def test_gather_tree(self): - # (max_time = 3, batch_size = 2, beam_width = 3) - - # create (batch_size, max_time, beam_width) matrix and transpose it - predicted_ids = np.array( - [[[1, 2, 3], [4, 5, 6], [7, 8, 9]], [[2, 3, 4], [5, 6, 7], [8, 9, 10]]], - dtype=np.int32).transpose([1, 0, 2]) - parent_ids = np.array( - [[[0, 0, 0], [0, 1, 1], [2, 1, 2]], [[0, 0, 0], [1, 2, 0], [2, 1, 1]]], - dtype=np.int32).transpose([1, 0, 2]) - - # sequence_lengths is shaped (batch_size = 3) - max_sequence_lengths = [3, 3] - - expected_result = np.array([[[2, 2, 2], [6, 5, 6], [7, 8, 9]], - [[2, 4, 4], [7, 6, 6], - [8, 9, 10]]]).transpose([1, 0, 2]) - - res = beam_search_ops.gather_tree( - predicted_ids, - parent_ids, - max_sequence_lengths=max_sequence_lengths, - end_token=11) - - with self.cached_session() as sess: - res_ = sess.run(res) - - self.assertAllEqual(expected_result, res_) - - def _test_gather_tree_from_array(self, - depth_ndims=0, - merged_batch_beam=False): - array = np.array( - [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [0, 0, 0]], - [[2, 3, 4], [5, 6, 7], [8, 9, 10], [11, 12, 0]]]).transpose([1, 0, 2]) - parent_ids = np.array( - [[[0, 0, 0], [0, 1, 1], [2, 1, 2], [-1, -1, -1]], - [[0, 0, 0], [1, 1, 0], [2, 0, 1], [0, 1, 0]]]).transpose([1, 0, 2]) - expected_array = np.array( - [[[2, 2, 2], [6, 5, 6], [7, 8, 9], [0, 0, 0]], - [[2, 3, 2], [7, 5, 7], [8, 9, 8], [11, 12, 0]]]).transpose([1, 0, 2]) - sequence_length = [[3, 3, 3], [4, 4, 3]] - - array = ops.convert_to_tensor( - array, dtype=dtypes.float32) - parent_ids = ops.convert_to_tensor( - parent_ids, dtype=dtypes.int32) - expected_array = ops.convert_to_tensor( - expected_array, dtype=dtypes.float32) - - max_time = array_ops.shape(array)[0] - batch_size = array_ops.shape(array)[1] - beam_width = array_ops.shape(array)[2] - - def _tile_in_depth(tensor): - # Generate higher rank tensors by concatenating tensor and tensor + 1. - for _ in range(depth_ndims): - tensor = array_ops.stack([tensor, tensor + 1], -1) - return tensor - - if merged_batch_beam: - array = array_ops.reshape( - array, [max_time, batch_size * beam_width]) - expected_array = array_ops.reshape( - expected_array, [max_time, batch_size * beam_width]) - - if depth_ndims > 0: - array = _tile_in_depth(array) - expected_array = _tile_in_depth(expected_array) - - sorted_array = beam_search_decoder.gather_tree_from_array( - array, parent_ids, sequence_length) - - with self.cached_session() as sess: - sorted_array = sess.run(sorted_array) - expected_array = sess.run(expected_array) - self.assertAllEqual(expected_array, sorted_array) - - def test_gather_tree_from_array_scalar(self): - self._test_gather_tree_from_array() - - def test_gather_tree_from_array_1d(self): - self._test_gather_tree_from_array(depth_ndims=1) - - def test_gather_tree_from_array_1d_with_merged_batch_beam(self): - self._test_gather_tree_from_array(depth_ndims=1, merged_batch_beam=True) - - def test_gather_tree_from_array_2d(self): - self._test_gather_tree_from_array(depth_ndims=2) - - def test_gather_tree_from_array_complex_trajectory(self): - # Max. time = 7, batch = 1, beam = 5. - array = np.expand_dims(np.array( - [[[25, 12, 114, 89, 97]], - [[9, 91, 64, 11, 162]], - [[34, 34, 34, 34, 34]], - [[2, 4, 2, 2, 4]], - [[2, 3, 6, 2, 2]], - [[2, 2, 2, 3, 2]], - [[2, 2, 2, 2, 2]]]), -1) - parent_ids = np.array( - [[[0, 0, 0, 0, 0]], - [[0, 0, 0, 0, 0]], - [[0, 1, 2, 3, 4]], - [[0, 0, 1, 2, 1]], - [[0, 1, 1, 2, 3]], - [[0, 1, 3, 1, 2]], - [[0, 1, 2, 3, 4]]]) - expected_array = np.expand_dims(np.array( - [[[25, 25, 25, 25, 25]], - [[9, 9, 91, 9, 9]], - [[34, 34, 34, 34, 34]], - [[2, 4, 2, 4, 4]], - [[2, 3, 6, 3, 6]], - [[2, 2, 2, 3, 2]], - [[2, 2, 2, 2, 2]]]), -1) - sequence_length = [[4, 6, 4, 7, 6]] - - array = ops.convert_to_tensor( - array, dtype=dtypes.float32) - parent_ids = ops.convert_to_tensor( - parent_ids, dtype=dtypes.int32) - expected_array = ops.convert_to_tensor( - expected_array, dtype=dtypes.float32) - - sorted_array = beam_search_decoder.gather_tree_from_array( - array, parent_ids, sequence_length) - - with self.cached_session() as sess: - sorted_array, expected_array = sess.run([sorted_array, expected_array]) - self.assertAllEqual(expected_array, sorted_array) - - -class TestArrayShapeChecks(test.TestCase): - - def _test_array_shape_dynamic_checks(self, static_shape, dynamic_shape, - batch_size, beam_width, is_valid=True): - t = array_ops.placeholder_with_default( - np.random.randn(*static_shape).astype(np.float32), - shape=dynamic_shape) - - batch_size = array_ops.constant(batch_size) - - def _test_body(): - # pylint: disable=protected-access - if context.executing_eagerly(): - beam_search_decoder._check_batch_beam(t, batch_size, beam_width) - else: - with self.cached_session(): - check_op = beam_search_decoder._check_batch_beam( - t, batch_size, beam_width) - self.evaluate(check_op) - # pylint: enable=protected-access - - if is_valid: - _test_body() - else: - with self.assertRaises(errors.InvalidArgumentError): - _test_body() - - def test_array_shape_dynamic_checks(self): - self._test_array_shape_dynamic_checks( - (8, 4, 5, 10), (None, None, 5, 10), 4, 5, is_valid=True) - self._test_array_shape_dynamic_checks( - (8, 20, 10), (None, None, 10), 4, 5, is_valid=True) - self._test_array_shape_dynamic_checks( - (8, 21, 10), (None, None, 10), 4, 5, is_valid=False) - self._test_array_shape_dynamic_checks( - (8, 4, 6, 10), (None, None, None, 10), 4, 5, is_valid=False) - self._test_array_shape_dynamic_checks( - (8, 4), (None, None), 4, 5, is_valid=False) - - -class TestEosMasking(test.TestCase): - """Tests EOS masking used in beam search.""" - - def test_eos_masking(self): - probs = constant_op.constant([ - [[-.2, -.2, -.2, -.2, -.2], [-.3, -.3, -.3, 3, 0], [5, 6, 0, 0, 0]], - [[-.2, -.2, -.2, -.2, 0], [-.3, -.3, -.1, 3, 0], [5, 6, 3, 0, 0]], - ]) - - eos_token = 0 - previously_finished = np.array([[0, 1, 0], [0, 1, 1]], dtype=bool) - masked = beam_search_decoder._mask_probs(probs, eos_token, - previously_finished) - - with self.cached_session() as sess: - probs = sess.run(probs) - masked = sess.run(masked) - - self.assertAllEqual(probs[0][0], masked[0][0]) - self.assertAllEqual(probs[0][2], masked[0][2]) - self.assertAllEqual(probs[1][0], masked[1][0]) - - self.assertEqual(masked[0][1][0], 0) - self.assertEqual(masked[1][1][0], 0) - self.assertEqual(masked[1][2][0], 0) - - for i in range(1, 5): - self.assertAllClose(masked[0][1][i], np.finfo('float32').min) - self.assertAllClose(masked[1][1][i], np.finfo('float32').min) - self.assertAllClose(masked[1][2][i], np.finfo('float32').min) - - -class TestBeamStep(test.TestCase): - """Tests a single step of beam search.""" - - def setUp(self): - super(TestBeamStep, self).setUp() - self.batch_size = 2 - self.beam_width = 3 - self.vocab_size = 5 - self.end_token = 0 - self.length_penalty_weight = 0.6 - self.coverage_penalty_weight = 0.0 - - def test_step(self): - dummy_cell_state = array_ops.zeros([self.batch_size, self.beam_width]) - beam_state = beam_search_decoder.BeamSearchDecoderState( - cell_state=dummy_cell_state, - log_probs=nn_ops.log_softmax( - array_ops.ones([self.batch_size, self.beam_width])), - lengths=constant_op.constant( - 2, shape=[self.batch_size, self.beam_width], dtype=dtypes.int64), - finished=array_ops.zeros( - [self.batch_size, self.beam_width], dtype=dtypes.bool), - accumulated_attention_probs=()) - - logits_ = np.full([self.batch_size, self.beam_width, self.vocab_size], - 0.0001) - logits_[0, 0, 2] = 1.9 - logits_[0, 0, 3] = 2.1 - logits_[0, 1, 3] = 3.1 - logits_[0, 1, 4] = 0.9 - logits_[1, 0, 1] = 0.5 - logits_[1, 1, 2] = 2.7 - logits_[1, 2, 2] = 10.0 - logits_[1, 2, 3] = 0.2 - logits = ops.convert_to_tensor(logits_, dtype=dtypes.float32) - log_probs = nn_ops.log_softmax(logits) - - outputs, next_beam_state = beam_search_decoder._beam_search_step( - time=2, - logits=logits, - next_cell_state=dummy_cell_state, - beam_state=beam_state, - batch_size=ops.convert_to_tensor(self.batch_size), - beam_width=self.beam_width, - end_token=self.end_token, - length_penalty_weight=self.length_penalty_weight, - coverage_penalty_weight=self.coverage_penalty_weight) - - with self.cached_session() as sess: - outputs_, next_state_, state_, log_probs_ = sess.run( - [outputs, next_beam_state, beam_state, log_probs]) - - self.assertAllEqual(outputs_.predicted_ids, [[3, 3, 2], [2, 2, 1]]) - self.assertAllEqual(outputs_.parent_ids, [[1, 0, 0], [2, 1, 0]]) - self.assertAllEqual(next_state_.lengths, [[3, 3, 3], [3, 3, 3]]) - self.assertAllEqual(next_state_.finished, - [[False, False, False], [False, False, False]]) - - expected_log_probs = [] - expected_log_probs.append(state_.log_probs[0][[1, 0, 0]]) - expected_log_probs.append(state_.log_probs[1][[2, 1, 0]]) # 0 --> 1 - expected_log_probs[0][0] += log_probs_[0, 1, 3] - expected_log_probs[0][1] += log_probs_[0, 0, 3] - expected_log_probs[0][2] += log_probs_[0, 0, 2] - expected_log_probs[1][0] += log_probs_[1, 2, 2] - expected_log_probs[1][1] += log_probs_[1, 1, 2] - expected_log_probs[1][2] += log_probs_[1, 0, 1] - self.assertAllEqual(next_state_.log_probs, expected_log_probs) - - def test_step_with_eos(self): - dummy_cell_state = array_ops.zeros([self.batch_size, self.beam_width]) - beam_state = beam_search_decoder.BeamSearchDecoderState( - cell_state=dummy_cell_state, - log_probs=nn_ops.log_softmax( - array_ops.ones([self.batch_size, self.beam_width])), - lengths=ops.convert_to_tensor( - [[2, 1, 2], [2, 2, 1]], dtype=dtypes.int64), - finished=ops.convert_to_tensor( - [[False, True, False], [False, False, True]], dtype=dtypes.bool), - accumulated_attention_probs=()) - - logits_ = np.full([self.batch_size, self.beam_width, self.vocab_size], - 0.0001) - logits_[0, 0, 2] = 1.9 - logits_[0, 0, 3] = 2.1 - logits_[0, 1, 3] = 3.1 - logits_[0, 1, 4] = 0.9 - logits_[1, 0, 1] = 0.5 - logits_[1, 1, 2] = 5.7 # why does this not work when it's 2.7? - logits_[1, 2, 2] = 1.0 - logits_[1, 2, 3] = 0.2 - logits = ops.convert_to_tensor(logits_, dtype=dtypes.float32) - log_probs = nn_ops.log_softmax(logits) - - outputs, next_beam_state = beam_search_decoder._beam_search_step( - time=2, - logits=logits, - next_cell_state=dummy_cell_state, - beam_state=beam_state, - batch_size=ops.convert_to_tensor(self.batch_size), - beam_width=self.beam_width, - end_token=self.end_token, - length_penalty_weight=self.length_penalty_weight, - coverage_penalty_weight=self.coverage_penalty_weight) - - with self.cached_session() as sess: - outputs_, next_state_, state_, log_probs_ = sess.run( - [outputs, next_beam_state, beam_state, log_probs]) - - self.assertAllEqual(outputs_.parent_ids, [[1, 0, 0], [1, 2, 0]]) - self.assertAllEqual(outputs_.predicted_ids, [[0, 3, 2], [2, 0, 1]]) - self.assertAllEqual(next_state_.lengths, [[1, 3, 3], [3, 1, 3]]) - self.assertAllEqual(next_state_.finished, - [[True, False, False], [False, True, False]]) - - expected_log_probs = [] - expected_log_probs.append(state_.log_probs[0][[1, 0, 0]]) - expected_log_probs.append(state_.log_probs[1][[1, 2, 0]]) - expected_log_probs[0][1] += log_probs_[0, 0, 3] - expected_log_probs[0][2] += log_probs_[0, 0, 2] - expected_log_probs[1][0] += log_probs_[1, 1, 2] - expected_log_probs[1][2] += log_probs_[1, 0, 1] - self.assertAllEqual(next_state_.log_probs, expected_log_probs) - - -class TestLargeBeamStep(test.TestCase): - """Tests large beam step. - - Tests a single step of beam search in such case that beam size is larger than - vocabulary size. - """ - - def setUp(self): - super(TestLargeBeamStep, self).setUp() - self.batch_size = 2 - self.beam_width = 8 - self.vocab_size = 5 - self.end_token = 0 - self.length_penalty_weight = 0.6 - self.coverage_penalty_weight = 0.0 - - def test_step(self): - - def get_probs(): - """this simulates the initialize method in BeamSearchDecoder.""" - log_prob_mask = array_ops.one_hot( - array_ops.zeros([self.batch_size], dtype=dtypes.int32), - depth=self.beam_width, - on_value=True, - off_value=False, - dtype=dtypes.bool) - - log_prob_zeros = array_ops.zeros( - [self.batch_size, self.beam_width], dtype=dtypes.float32) - log_prob_neg_inf = array_ops.ones( - [self.batch_size, self.beam_width], dtype=dtypes.float32) * -np.Inf - - log_probs = array_ops.where_v2(log_prob_mask, log_prob_zeros, - log_prob_neg_inf) - return log_probs - - log_probs = get_probs() - dummy_cell_state = array_ops.zeros([self.batch_size, self.beam_width]) - - # pylint: disable=invalid-name - _finished = array_ops.one_hot( - array_ops.zeros([self.batch_size], dtype=dtypes.int32), - depth=self.beam_width, - on_value=False, - off_value=True, - dtype=dtypes.bool) - _lengths = np.zeros([self.batch_size, self.beam_width], dtype=np.int64) - _lengths[:, 0] = 2 - _lengths = constant_op.constant(_lengths, dtype=dtypes.int64) - - beam_state = beam_search_decoder.BeamSearchDecoderState( - cell_state=dummy_cell_state, - log_probs=log_probs, - lengths=_lengths, - finished=_finished, - accumulated_attention_probs=()) - - logits_ = np.full([self.batch_size, self.beam_width, self.vocab_size], - 0.0001) - logits_[0, 0, 2] = 1.9 - logits_[0, 0, 3] = 2.1 - logits_[0, 1, 3] = 3.1 - logits_[0, 1, 4] = 0.9 - logits_[1, 0, 1] = 0.5 - logits_[1, 1, 2] = 2.7 - logits_[1, 2, 2] = 10.0 - logits_[1, 2, 3] = 0.2 - logits = constant_op.constant(logits_, dtype=dtypes.float32) - log_probs = nn_ops.log_softmax(logits) - - outputs, next_beam_state = beam_search_decoder._beam_search_step( - time=2, - logits=logits, - next_cell_state=dummy_cell_state, - beam_state=beam_state, - batch_size=ops.convert_to_tensor(self.batch_size), - beam_width=self.beam_width, - end_token=self.end_token, - length_penalty_weight=self.length_penalty_weight, - coverage_penalty_weight=self.coverage_penalty_weight) - - with self.cached_session() as sess: - outputs_, next_state_, _, _ = sess.run( - [outputs, next_beam_state, beam_state, log_probs]) - - self.assertEqual(outputs_.predicted_ids[0, 0], 3) - self.assertEqual(outputs_.predicted_ids[0, 1], 2) - self.assertEqual(outputs_.predicted_ids[1, 0], 1) - neg_inf = -np.Inf - self.assertAllEqual( - next_state_.log_probs[:, -3:], - [[neg_inf, neg_inf, neg_inf], [neg_inf, neg_inf, neg_inf]]) - self.assertEqual((next_state_.log_probs[:, :-3] > neg_inf).all(), True) - self.assertEqual((next_state_.lengths[:, :-3] > 0).all(), True) - self.assertAllEqual(next_state_.lengths[:, -3:], [[0, 0, 0], [0, 0, 0]]) - - -@test_util.run_v1_only('contrib code not supported in TF2.0') -class BeamSearchDecoderTest(test.TestCase): - - def _testDynamicDecodeRNN(self, time_major, has_attention, - with_alignment_history=False): - encoder_sequence_length = np.array([3, 2, 3, 1, 1]) - decoder_sequence_length = np.array([2, 0, 1, 2, 3]) - batch_size = 5 - decoder_max_time = 4 - input_depth = 7 - cell_depth = 9 - attention_depth = 6 - vocab_size = 20 - end_token = vocab_size - 1 - start_token = 0 - embedding_dim = 50 - max_out = max(decoder_sequence_length) - output_layer = layers_core.Dense(vocab_size, use_bias=True, activation=None) - beam_width = 3 - - with self.cached_session() as sess: - batch_size_tensor = constant_op.constant(batch_size) - embedding = np.random.randn(vocab_size, embedding_dim).astype(np.float32) - cell = rnn_cell.LSTMCell(cell_depth) - initial_state = cell.zero_state(batch_size, dtypes.float32) - coverage_penalty_weight = 0.0 - if has_attention: - coverage_penalty_weight = 0.2 - inputs = array_ops.placeholder_with_default( - np.random.randn(batch_size, decoder_max_time, input_depth).astype( - np.float32), - shape=(None, None, input_depth)) - tiled_inputs = beam_search_decoder.tile_batch( - inputs, multiplier=beam_width) - tiled_sequence_length = beam_search_decoder.tile_batch( - encoder_sequence_length, multiplier=beam_width) - attention_mechanism = attention_wrapper.BahdanauAttention( - num_units=attention_depth, - memory=tiled_inputs, - memory_sequence_length=tiled_sequence_length) - initial_state = beam_search_decoder.tile_batch( - initial_state, multiplier=beam_width) - cell = attention_wrapper.AttentionWrapper( - cell=cell, - attention_mechanism=attention_mechanism, - attention_layer_size=attention_depth, - alignment_history=with_alignment_history) - cell_state = cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size_tensor * beam_width) - if has_attention: - cell_state = cell_state.clone(cell_state=initial_state) - bsd = beam_search_decoder.BeamSearchDecoder( - cell=cell, - embedding=embedding, - start_tokens=array_ops.fill([batch_size_tensor], start_token), - end_token=end_token, - initial_state=cell_state, - beam_width=beam_width, - output_layer=output_layer, - length_penalty_weight=0.0, - coverage_penalty_weight=coverage_penalty_weight) - - final_outputs, final_state, final_sequence_lengths = ( - decoder.dynamic_decode( - bsd, output_time_major=time_major, maximum_iterations=max_out)) - - def _t(shape): - if time_major: - return (shape[1], shape[0]) + shape[2:] - return shape - - self.assertIsInstance( - final_outputs, beam_search_decoder.FinalBeamSearchDecoderOutput) - self.assertIsInstance( - final_state, beam_search_decoder.BeamSearchDecoderState) - - beam_search_decoder_output = final_outputs.beam_search_decoder_output - self.assertEqual( - _t((batch_size, None, beam_width)), - tuple(beam_search_decoder_output.scores.get_shape().as_list())) - self.assertEqual( - _t((batch_size, None, beam_width)), - tuple(final_outputs.predicted_ids.get_shape().as_list())) - - sess.run(variables.global_variables_initializer()) - sess_results = sess.run({ - 'final_outputs': final_outputs, - 'final_state': final_state, - 'final_sequence_lengths': final_sequence_lengths - }) - - max_sequence_length = np.max(sess_results['final_sequence_lengths']) - - # A smoke test - self.assertEqual( - _t((batch_size, max_sequence_length, beam_width)), - sess_results['final_outputs'].beam_search_decoder_output.scores.shape) - self.assertEqual( - _t((batch_size, max_sequence_length, beam_width)), sess_results[ - 'final_outputs'].beam_search_decoder_output.predicted_ids.shape) - - def testDynamicDecodeRNNBatchMajorNoAttention(self): - self._testDynamicDecodeRNN(time_major=False, has_attention=False) - - def testDynamicDecodeRNNBatchMajorYesAttention(self): - self._testDynamicDecodeRNN(time_major=False, has_attention=True) - - def testDynamicDecodeRNNBatchMajorYesAttentionWithAlignmentHistory(self): - self._testDynamicDecodeRNN( - time_major=False, - has_attention=True, - with_alignment_history=True) - - -@test_util.run_all_in_graph_and_eager_modes -class BeamSearchDecoderV2Test(test.TestCase): - - def _testDynamicDecodeRNN(self, time_major, has_attention, - with_alignment_history=False): - encoder_sequence_length = np.array([3, 2, 3, 1, 1]) - decoder_sequence_length = np.array([2, 0, 1, 2, 3]) - batch_size = 5 - decoder_max_time = 4 - input_depth = 7 - cell_depth = 9 - attention_depth = 6 - vocab_size = 20 - end_token = vocab_size - 1 - start_token = 0 - embedding_dim = 50 - max_out = max(decoder_sequence_length) - output_layer = layers.Dense(vocab_size, use_bias=True, activation=None) - beam_width = 3 - - with self.cached_session(): - batch_size_tensor = constant_op.constant(batch_size) - embedding = np.random.randn(vocab_size, embedding_dim).astype(np.float32) - cell = rnn_cell.LSTMCell(cell_depth) - initial_state = cell.zero_state(batch_size, dtypes.float32) - coverage_penalty_weight = 0.0 - if has_attention: - coverage_penalty_weight = 0.2 - inputs = array_ops.placeholder_with_default( - np.random.randn(batch_size, decoder_max_time, input_depth).astype( - np.float32), - shape=(None, None, input_depth)) - tiled_inputs = beam_search_decoder.tile_batch( - inputs, multiplier=beam_width) - tiled_sequence_length = beam_search_decoder.tile_batch( - encoder_sequence_length, multiplier=beam_width) - attention_mechanism = attention_wrapper.BahdanauAttention( - num_units=attention_depth, - memory=tiled_inputs, - memory_sequence_length=tiled_sequence_length) - initial_state = beam_search_decoder.tile_batch( - initial_state, multiplier=beam_width) - cell = attention_wrapper.AttentionWrapper( - cell=cell, - attention_mechanism=attention_mechanism, - attention_layer_size=attention_depth, - alignment_history=with_alignment_history) - cell_state = cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size_tensor * beam_width) - if has_attention: - cell_state = cell_state.clone(cell_state=initial_state) - bsd = beam_search_decoder.BeamSearchDecoderV2( - cell=cell, - beam_width=beam_width, - output_layer=output_layer, - length_penalty_weight=0.0, - coverage_penalty_weight=coverage_penalty_weight, - output_time_major=time_major, - maximum_iterations=max_out) - - final_outputs, final_state, final_sequence_lengths = bsd( - embedding, - start_tokens=array_ops.fill([batch_size_tensor], start_token), - end_token=end_token, - initial_state=cell_state) - - def _t(shape): - if time_major: - return (shape[1], shape[0]) + shape[2:] - return shape - - self.assertIsInstance( - final_outputs, beam_search_decoder.FinalBeamSearchDecoderOutput) - self.assertIsInstance( - final_state, beam_search_decoder.BeamSearchDecoderState) - - beam_search_decoder_output = final_outputs.beam_search_decoder_output - expected_seq_length = 3 if context.executing_eagerly() else None - self.assertEqual( - _t((batch_size, expected_seq_length, beam_width)), - tuple(beam_search_decoder_output.scores.get_shape().as_list())) - self.assertEqual( - _t((batch_size, expected_seq_length, beam_width)), - tuple(final_outputs.predicted_ids.get_shape().as_list())) - - self.evaluate(variables.global_variables_initializer()) - eval_results = self.evaluate({ - 'final_outputs': final_outputs, - 'final_sequence_lengths': final_sequence_lengths - }) - - max_sequence_length = np.max(eval_results['final_sequence_lengths']) - - # A smoke test - self.assertEqual( - _t((batch_size, max_sequence_length, beam_width)), - eval_results['final_outputs'].beam_search_decoder_output.scores.shape) - self.assertEqual( - _t((batch_size, max_sequence_length, beam_width)), eval_results[ - 'final_outputs'].beam_search_decoder_output.predicted_ids.shape) - - def testDynamicDecodeRNNBatchMajorNoAttention(self): - self._testDynamicDecodeRNN(time_major=False, has_attention=False) - - def testDynamicDecodeRNNBatchMajorYesAttention(self): - self._testDynamicDecodeRNN(time_major=False, has_attention=True) - - def testDynamicDecodeRNNBatchMajorYesAttentionWithAlignmentHistory(self): - self._testDynamicDecodeRNN( - time_major=False, - has_attention=True, - with_alignment_history=True) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py deleted file mode 100644 index 5506aa8b8ee..00000000000 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for contrib.seq2seq.python.seq2seq.beam_search_ops.""" -# pylint: disable=unused-import,g-bad-import-order -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -# pylint: enable=unused-import - -import itertools - -import numpy as np - -from tensorflow.contrib.seq2seq.python.ops import beam_search_ops -from tensorflow.python.framework import ops -from tensorflow.python.platform import test - - -def _transpose_batch_time(x): - return np.transpose(x, [1, 0, 2]).astype(np.int32) - - -class GatherTreeTest(test.TestCase): - - def testGatherTreeOne(self): - # (max_time = 4, batch_size = 1, beams = 3) - end_token = 10 - step_ids = _transpose_batch_time( - [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]]) - parent_ids = _transpose_batch_time( - [[[0, 0, 0], [0, 1, 1], [2, 1, 2], [-1, -1, -1]]]) - max_sequence_lengths = [3] - expected_result = _transpose_batch_time([[[2, 2, 2], [6, 5, 6], [7, 8, 9], - [10, 10, 10]]]) - beams = beam_search_ops.gather_tree( - step_ids=step_ids, - parent_ids=parent_ids, - max_sequence_lengths=max_sequence_lengths, - end_token=end_token) - with self.cached_session(use_gpu=True): - self.assertAllEqual(expected_result, self.evaluate(beams)) - - def testBadParentValuesOnCPU(self): - # (batch_size = 1, max_time = 4, beams = 3) - # bad parent in beam 1 time 1 - end_token = 10 - step_ids = _transpose_batch_time( - [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]]) - parent_ids = _transpose_batch_time( - [[[0, 0, 0], [0, -1, 1], [2, 1, 2], [-1, -1, -1]]]) - max_sequence_lengths = [3] - with ops.device("/cpu:0"): - with self.assertRaisesOpError( - r"parent id -1 at \(batch, time, beam\) == \(0, 0, 1\)"): - beams = beam_search_ops.gather_tree( - step_ids=step_ids, - parent_ids=parent_ids, - max_sequence_lengths=max_sequence_lengths, - end_token=end_token) - self.evaluate(beams) - - def testBadParentValuesOnGPU(self): - # Only want to run this test on CUDA devices, as gather_tree is not - # registered for SYCL devices. - if not test.is_gpu_available(cuda_only=True): - return - # (max_time = 4, batch_size = 1, beams = 3) - # bad parent in beam 1 time 1; appears as a negative index at time 0 - end_token = 10 - step_ids = _transpose_batch_time( - [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]]) - parent_ids = _transpose_batch_time( - [[[0, 0, 0], [0, -1, 1], [2, 1, 2], [-1, -1, -1]]]) - max_sequence_lengths = [3] - expected_result = _transpose_batch_time([[[2, -1, 2], [6, 5, 6], [7, 8, 9], - [10, 10, 10]]]) - with ops.device("/device:GPU:0"): - beams = beam_search_ops.gather_tree( - step_ids=step_ids, - parent_ids=parent_ids, - max_sequence_lengths=max_sequence_lengths, - end_token=end_token) - self.assertAllEqual(expected_result, self.evaluate(beams)) - - def testGatherTreeBatch(self): - batch_size = 10 - beam_width = 15 - max_time = 8 - max_sequence_lengths = [0, 1, 2, 4, 7, 8, 9, 10, 11, 0] - end_token = 5 - - with self.cached_session(use_gpu=True): - step_ids = np.random.randint( - 0, high=end_token + 1, size=(max_time, batch_size, beam_width)) - parent_ids = np.random.randint( - 0, high=beam_width - 1, size=(max_time, batch_size, beam_width)) - - beams = beam_search_ops.gather_tree( - step_ids=step_ids.astype(np.int32), - parent_ids=parent_ids.astype(np.int32), - max_sequence_lengths=max_sequence_lengths, - end_token=end_token) - - self.assertEqual((max_time, batch_size, beam_width), beams.shape) - beams_value = self.evaluate(beams) - for b in range(batch_size): - # Past max_sequence_lengths[b], we emit all end tokens. - b_value = beams_value[max_sequence_lengths[b]:, b, :] - self.assertAllClose(b_value, end_token * np.ones_like(b_value)) - for batch, beam in itertools.product( - range(batch_size), range(beam_width)): - v = np.squeeze(beams_value[:, batch, beam]) - if end_token in v: - found_bad = np.where(v == -1)[0] - self.assertEqual(0, len(found_bad)) - found = np.where(v == end_token)[0] - found = found[0] # First occurrence of end_token. - # If an end_token is found, everything before it should be a - # valid id and everything after it should be -1. - if found > 0: - self.assertAllEqual( - v[:found - 1] >= 0, np.ones_like(v[:found - 1], dtype=bool)) - self.assertAllClose(v[found + 1:], - end_token * np.ones_like(v[found + 1:])) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/decoder_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/decoder_test.py deleted file mode 100644 index 3661daf3a26..00000000000 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/decoder_test.py +++ /dev/null @@ -1,181 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for contrib.seq2seq.python.seq2seq.decoder.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.seq2seq.python.ops import basic_decoder -from tensorflow.contrib.seq2seq.python.ops import decoder -from tensorflow.contrib.seq2seq.python.ops import helper as helper_py -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import test_util -from tensorflow.python.ops import rnn -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import variables -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.platform import test - - -@test_util.run_v1_only("contrib code not supported in TF2.0") -class DynamicDecodeRNNTest(test.TestCase): - - def _testDynamicDecodeRNN(self, time_major, maximum_iterations=None): - - sequence_length = [3, 4, 3, 1, 0] - batch_size = 5 - max_time = 8 - input_depth = 7 - cell_depth = 10 - max_out = max(sequence_length) - - with self.session(use_gpu=True) as sess: - if time_major: - inputs = np.random.randn(max_time, batch_size, - input_depth).astype(np.float32) - else: - inputs = np.random.randn(batch_size, max_time, - input_depth).astype(np.float32) - cell = rnn_cell.LSTMCell(cell_depth) - helper = helper_py.TrainingHelper( - inputs, sequence_length, time_major=time_major) - my_decoder = basic_decoder.BasicDecoder( - cell=cell, - helper=helper, - initial_state=cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size)) - - final_outputs, final_state, final_sequence_length = ( - decoder.dynamic_decode(my_decoder, output_time_major=time_major, - maximum_iterations=maximum_iterations)) - - def _t(shape): - if time_major: - return (shape[1], shape[0]) + shape[2:] - return shape - - self.assertTrue( - isinstance(final_outputs, basic_decoder.BasicDecoderOutput)) - self.assertTrue(isinstance(final_state, rnn_cell.LSTMStateTuple)) - - self.assertEqual( - (batch_size,), - tuple(final_sequence_length.get_shape().as_list())) - self.assertEqual( - _t((batch_size, None, cell_depth)), - tuple(final_outputs.rnn_output.get_shape().as_list())) - self.assertEqual( - _t((batch_size, None)), - tuple(final_outputs.sample_id.get_shape().as_list())) - - sess.run(variables.global_variables_initializer()) - sess_results = sess.run({ - "final_outputs": final_outputs, - "final_state": final_state, - "final_sequence_length": final_sequence_length, - }) - - # Mostly a smoke test - time_steps = max_out - expected_length = sequence_length - if maximum_iterations is not None: - time_steps = min(max_out, maximum_iterations) - expected_length = [min(x, maximum_iterations) for x in expected_length] - self.assertEqual( - _t((batch_size, time_steps, cell_depth)), - sess_results["final_outputs"].rnn_output.shape) - self.assertEqual( - _t((batch_size, time_steps)), - sess_results["final_outputs"].sample_id.shape) - self.assertItemsEqual(expected_length, - sess_results["final_sequence_length"]) - - def testDynamicDecodeRNNBatchMajor(self): - self._testDynamicDecodeRNN(time_major=False) - - def testDynamicDecodeRNNTimeMajor(self): - self._testDynamicDecodeRNN(time_major=True) - - def testDynamicDecodeRNNZeroMaxIters(self): - self._testDynamicDecodeRNN(time_major=True, maximum_iterations=0) - - def testDynamicDecodeRNNOneMaxIter(self): - self._testDynamicDecodeRNN(time_major=True, maximum_iterations=1) - - def _testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNN( - self, use_sequence_length): - sequence_length = [3, 4, 3, 1, 0] - batch_size = 5 - max_time = 8 - input_depth = 7 - cell_depth = 10 - max_out = max(sequence_length) - - with self.session(use_gpu=True) as sess: - inputs = np.random.randn(batch_size, max_time, - input_depth).astype(np.float32) - - cell = rnn_cell.LSTMCell(cell_depth) - zero_state = cell.zero_state(dtype=dtypes.float32, batch_size=batch_size) - helper = helper_py.TrainingHelper(inputs, sequence_length) - my_decoder = basic_decoder.BasicDecoder( - cell=cell, helper=helper, initial_state=zero_state) - - # Match the variable scope of dynamic_rnn below so we end up - # using the same variables - with vs.variable_scope("root") as scope: - final_decoder_outputs, final_decoder_state, _ = decoder.dynamic_decode( - my_decoder, - # impute_finished=True ensures outputs and final state - # match those of dynamic_rnn called with sequence_length not None - impute_finished=use_sequence_length, - scope=scope) - - with vs.variable_scope(scope, reuse=True) as scope: - final_rnn_outputs, final_rnn_state = rnn.dynamic_rnn( - cell, - inputs, - sequence_length=sequence_length if use_sequence_length else None, - initial_state=zero_state, - scope=scope) - - sess.run(variables.global_variables_initializer()) - sess_results = sess.run({ - "final_decoder_outputs": final_decoder_outputs, - "final_decoder_state": final_decoder_state, - "final_rnn_outputs": final_rnn_outputs, - "final_rnn_state": final_rnn_state - }) - - # Decoder only runs out to max_out; ensure values are identical - # to dynamic_rnn, which also zeros out outputs and passes along state. - self.assertAllClose(sess_results["final_decoder_outputs"].rnn_output, - sess_results["final_rnn_outputs"][:, 0:max_out, :]) - if use_sequence_length: - self.assertAllClose(sess_results["final_decoder_state"], - sess_results["final_rnn_state"]) - - def testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNNWithSeqLen(self): - self._testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNN( - use_sequence_length=True) - - def testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNNNoSeqLen(self): - self._testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNN( - use_sequence_length=False) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/decoder_v2_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/decoder_v2_test.py deleted file mode 100644 index b5bba2b32e9..00000000000 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/decoder_v2_test.py +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for contrib.seq2seq.python.seq2seq.decoder.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.seq2seq.python.ops import basic_decoder -from tensorflow.contrib.seq2seq.python.ops import sampler as sampler_py -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.keras import keras_parameterized -from tensorflow.python.ops import rnn -from tensorflow.python.ops import rnn_cell -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -@keras_parameterized.run_all_keras_modes -class DecodeV2RNNTest(keras_parameterized.TestCase, test.TestCase): - """Tests for DecoderV2.""" - - def _testDecodeRNN(self, time_major, maximum_iterations=None): - - sequence_length = [3, 4, 3, 1, 0] - batch_size = 5 - max_time = 8 - input_depth = 7 - cell_depth = 10 - max_out = max(sequence_length) - - with self.cached_session(use_gpu=True): - if time_major: - inputs = np.random.randn(max_time, batch_size, - input_depth).astype(np.float32) - else: - inputs = np.random.randn(batch_size, max_time, - input_depth).astype(np.float32) - input_t = constant_op.constant(inputs) - cell = rnn_cell.LSTMCell(cell_depth) - sampler = sampler_py.TrainingSampler(time_major=time_major) - my_decoder = basic_decoder.BasicDecoderV2( - cell=cell, - sampler=sampler, - output_time_major=time_major, - maximum_iterations=maximum_iterations) - - initial_state = cell.zero_state( - dtype=dtypes.float32, batch_size=batch_size) - (final_outputs, unused_final_state, final_sequence_length) = my_decoder( - input_t, initial_state=initial_state, sequence_length=sequence_length) - - def _t(shape): - if time_major: - return (shape[1], shape[0]) + shape[2:] - return shape - - if not context.executing_eagerly(): - self.assertEqual((batch_size,), - tuple(final_sequence_length.get_shape().as_list())) - self.assertEqual( - _t((batch_size, None, cell_depth)), - tuple(final_outputs.rnn_output.get_shape().as_list())) - self.assertEqual( - _t((batch_size, None)), - tuple(final_outputs.sample_id.get_shape().as_list())) - - self.evaluate(variables.global_variables_initializer()) - final_outputs = self.evaluate(final_outputs) - final_sequence_length = self.evaluate(final_sequence_length) - - # Mostly a smoke test - time_steps = max_out - expected_length = sequence_length - if maximum_iterations is not None: - time_steps = min(max_out, maximum_iterations) - expected_length = [min(x, maximum_iterations) for x in expected_length] - if context.executing_eagerly() and maximum_iterations != 0: - # Only check the shape of output when maximum_iterations > 0, see - # b/123431432 for more details. - self.assertEqual( - _t((batch_size, time_steps, cell_depth)), - final_outputs.rnn_output.shape) - self.assertEqual( - _t((batch_size, time_steps)), final_outputs.sample_id.shape) - self.assertItemsEqual(expected_length, final_sequence_length) - - def testDynamicDecodeRNNBatchMajor(self): - self._testDecodeRNN(time_major=False) - - def testDynamicDecodeRNNTimeMajor(self): - self._testDecodeRNN(time_major=True) - - def testDynamicDecodeRNNZeroMaxIters(self): - self._testDecodeRNN(time_major=True, maximum_iterations=0) - - def testDynamicDecodeRNNOneMaxIter(self): - self._testDecodeRNN(time_major=True, maximum_iterations=1) - - def _testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNN( - self, use_sequence_length): - sequence_length = [3, 4, 3, 1, 0] - batch_size = 5 - max_time = 8 - input_depth = 7 - cell_depth = 10 - max_out = max(sequence_length) - - with self.cached_session(use_gpu=True): - inputs = np.random.randn(batch_size, max_time, - input_depth).astype(np.float32) - inputs = constant_op.constant(inputs) - - cell = rnn_cell.LSTMCell(cell_depth) - zero_state = cell.zero_state(dtype=dtypes.float32, batch_size=batch_size) - sampler = sampler_py.TrainingSampler() - my_decoder = basic_decoder.BasicDecoderV2( - cell=cell, sampler=sampler, impute_finished=use_sequence_length) - - final_decoder_outputs, final_decoder_state, _ = my_decoder( - inputs, initial_state=zero_state, sequence_length=sequence_length) - - final_rnn_outputs, final_rnn_state = rnn.dynamic_rnn( - cell, - inputs, - sequence_length=sequence_length if use_sequence_length else None, - initial_state=zero_state) - - self.evaluate(variables.global_variables_initializer()) - eval_result = self.evaluate({ - "final_decoder_outputs": final_decoder_outputs, - "final_decoder_state": final_decoder_state, - "final_rnn_outputs": final_rnn_outputs, - "final_rnn_state": final_rnn_state - }) - - # Decoder only runs out to max_out; ensure values are identical - # to dynamic_rnn, which also zeros out outputs and passes along state. - self.assertAllClose(eval_result["final_decoder_outputs"].rnn_output, - eval_result["final_rnn_outputs"][:, 0:max_out, :]) - if use_sequence_length: - self.assertAllClose(eval_result["final_decoder_state"], - eval_result["final_rnn_state"]) - - def testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNNWithSeqLen(self): - self._testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNN( - use_sequence_length=True) - - def testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNNNoSeqLen(self): - self._testDynamicDecodeRNNWithTrainingHelperMatchesDynamicRNN( - use_sequence_length=False) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/loss_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/loss_test.py deleted file mode 100644 index 7eb544a921c..00000000000 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/loss_test.py +++ /dev/null @@ -1,272 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for contrib.seq2seq.python.seq2seq.loss_ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.seq2seq.python.ops import loss -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -@test_util.run_all_in_graph_and_eager_modes -class LossTest(test.TestCase): - - def config_default_values(self): - self.batch_size = 2 - self.sequence_length = 3 - self.number_of_classes = 5 - logits = [ - constant_op.constant(i + 0.5, shape=[self.batch_size, - self.number_of_classes]) - for i in range(self.sequence_length) - ] - self.logits = array_ops.stack(logits, axis=1) - targets = [ - constant_op.constant(i, dtypes.int32, shape=[self.batch_size]) - for i in range(self.sequence_length) - ] - self.targets = array_ops.stack(targets, axis=1) - weights = [ - constant_op.constant(1.0, shape=[self.batch_size]) - for _ in range(self.sequence_length) - ] - self.weights = array_ops.stack(weights, axis=1) - # expected_loss = sparse_softmax_cross_entropy_with_logits(targets, logits) - # where targets = [0, 1, 2], and logits = [[0.5] * 5, [1.5] * 5, [2.5] * 5] - self.expected_loss = 1.60944 - - def testSequenceLoss(self): - self.config_default_values() - with self.cached_session(use_gpu=True): - average_loss_per_example = loss.sequence_loss( - self.logits, self.targets, self.weights, - average_across_timesteps=True, - average_across_batch=True) - res = self.evaluate(average_loss_per_example) - self.assertAllClose(self.expected_loss, res) - - average_loss_per_sequence = loss.sequence_loss( - self.logits, self.targets, self.weights, - average_across_timesteps=False, - average_across_batch=True) - res = self.evaluate(average_loss_per_sequence) - compare_per_sequence = np.full((self.sequence_length), self.expected_loss) - self.assertAllClose(compare_per_sequence, res) - - average_loss_per_batch = loss.sequence_loss( - self.logits, self.targets, self.weights, - average_across_timesteps=True, - average_across_batch=False) - res = self.evaluate(average_loss_per_batch) - compare_per_batch = np.full((self.batch_size), self.expected_loss) - self.assertAllClose(compare_per_batch, res) - - total_loss = loss.sequence_loss( - self.logits, self.targets, self.weights, - average_across_timesteps=False, - average_across_batch=False) - res = self.evaluate(total_loss) - compare_total = np.full((self.batch_size, self.sequence_length), - self.expected_loss) - self.assertAllClose(compare_total, res) - - def testSequenceLossClass(self): - self.config_default_values() - with self.cached_session(use_gpu=True): - seq_loss = loss.SequenceLoss(average_across_timesteps=True, - average_across_batch=True, - sum_over_timesteps=False, - sum_over_batch=False) - average_loss_per_example = seq_loss( - self.targets, self.logits, self.weights) - res = self.evaluate(average_loss_per_example) - self.assertAllClose(self.expected_loss, res) - - seq_loss = loss.SequenceLoss(average_across_timesteps=False, - average_across_batch=True, - sum_over_timesteps=False, - sum_over_batch=False) - average_loss_per_sequence = seq_loss( - self.targets, self.logits, self.weights) - res = self.evaluate(average_loss_per_sequence) - compare_per_sequence = np.full((self.sequence_length), self.expected_loss) - self.assertAllClose(compare_per_sequence, res) - - seq_loss = loss.SequenceLoss(average_across_timesteps=True, - average_across_batch=False, - sum_over_timesteps=False, - sum_over_batch=False) - average_loss_per_batch = seq_loss( - self.targets, self.logits, self.weights) - res = self.evaluate(average_loss_per_batch) - compare_per_batch = np.full((self.batch_size), self.expected_loss) - self.assertAllClose(compare_per_batch, res) - - seq_loss = loss.SequenceLoss(average_across_timesteps=False, - average_across_batch=False, - sum_over_timesteps=False, - sum_over_batch=False) - total_loss = seq_loss( - self.targets, self.logits, self.weights) - res = self.evaluate(total_loss) - compare_total = np.full((self.batch_size, self.sequence_length), - self.expected_loss) - self.assertAllClose(compare_total, res) - - def testSumReduction(self): - self.config_default_values() - with self.cached_session(use_gpu=True): - seq_loss = loss.SequenceLoss(average_across_timesteps=False, - average_across_batch=False, - sum_over_timesteps=True, - sum_over_batch=True) - average_loss_per_example = seq_loss( - self.targets, self.logits, self.weights) - res = self.evaluate(average_loss_per_example) - self.assertAllClose(self.expected_loss, res) - - seq_loss = loss.SequenceLoss(average_across_timesteps=False, - average_across_batch=False, - sum_over_timesteps=False, - sum_over_batch=True) - average_loss_per_sequence = seq_loss( - self.targets, self.logits, self.weights) - res = self.evaluate(average_loss_per_sequence) - compare_per_sequence = np.full((self.sequence_length), self.expected_loss) - self.assertAllClose(compare_per_sequence, res) - - seq_loss = loss.SequenceLoss(average_across_timesteps=False, - average_across_batch=False, - sum_over_timesteps=True, - sum_over_batch=False) - average_loss_per_batch = seq_loss( - self.targets, self.logits, self.weights) - res = self.evaluate(average_loss_per_batch) - compare_per_batch = np.full((self.batch_size), self.expected_loss) - self.assertAllClose(compare_per_batch, res) - - seq_loss = loss.SequenceLoss(average_across_timesteps=False, - average_across_batch=False, - sum_over_timesteps=False, - sum_over_batch=False) - total_loss = seq_loss( - self.targets, self.logits, self.weights) - res = self.evaluate(total_loss) - compare_total = np.full((self.batch_size, self.sequence_length), - self.expected_loss) - self.assertAllClose(compare_total, res) - - def testWeightedSumReduction(self): - self.config_default_values() - weights = [ - constant_op.constant(1.0, shape=[self.batch_size]) - for _ in range(self.sequence_length) - ] - # Make the last element in the sequence to have zero weights. - weights[-1] = constant_op.constant(0.0, shape=[self.batch_size]) - self.weights = array_ops.stack(weights, axis=1) - with self.cached_session(use_gpu=True): - seq_loss = loss.SequenceLoss(average_across_timesteps=False, - average_across_batch=False, - sum_over_timesteps=True, - sum_over_batch=True) - average_loss_per_example = seq_loss( - self.targets, self.logits, self.weights) - res = self.evaluate(average_loss_per_example) - self.assertAllClose(self.expected_loss, res) - - seq_loss = loss.SequenceLoss(average_across_timesteps=False, - average_across_batch=False, - sum_over_timesteps=False, - sum_over_batch=True) - average_loss_per_sequence = seq_loss( - self.targets, self.logits, self.weights) - res = self.evaluate(average_loss_per_sequence) - compare_per_sequence = np.full((self.sequence_length), self.expected_loss) - # The last element in every sequence are zeros, which will be filtered. - compare_per_sequence[-1] = 0. - self.assertAllClose(compare_per_sequence, res) - - seq_loss = loss.SequenceLoss(average_across_timesteps=False, - average_across_batch=False, - sum_over_timesteps=True, - sum_over_batch=False) - average_loss_per_batch = seq_loss(self.targets, self.logits, self.weights) - res = self.evaluate(average_loss_per_batch) - compare_per_batch = np.full((self.batch_size), self.expected_loss) - self.assertAllClose(compare_per_batch, res) - - seq_loss = loss.SequenceLoss(average_across_timesteps=False, - average_across_batch=False, - sum_over_timesteps=False, - sum_over_batch=False) - total_loss = seq_loss(self.targets, self.logits, self.weights) - res = self.evaluate(total_loss) - compare_total = np.full((self.batch_size, self.sequence_length), - self.expected_loss) - # The last element in every sequence are zeros, which will be filtered. - compare_total[:, -1] = 0 - self.assertAllClose(compare_total, res) - - def testZeroWeights(self): - self.config_default_values() - weights = [ - constant_op.constant(0.0, shape=[self.batch_size]) - for _ in range(self.sequence_length) - ] - weights = array_ops.stack(weights, axis=1) - with self.cached_session(use_gpu=True): - average_loss_per_example = loss.sequence_loss( - self.logits, self.targets, weights, - average_across_timesteps=True, - average_across_batch=True) - res = self.evaluate(average_loss_per_example) - self.assertAllClose(0.0, res) - - average_loss_per_sequence = loss.sequence_loss( - self.logits, self.targets, weights, - average_across_timesteps=False, - average_across_batch=True) - res = self.evaluate(average_loss_per_sequence) - compare_per_sequence = np.zeros((self.sequence_length)) - self.assertAllClose(compare_per_sequence, res) - - average_loss_per_batch = loss.sequence_loss( - self.logits, self.targets, weights, - average_across_timesteps=True, - average_across_batch=False) - res = self.evaluate(average_loss_per_batch) - compare_per_batch = np.zeros((self.batch_size)) - self.assertAllClose(compare_per_batch, res) - - total_loss = loss.sequence_loss( - self.logits, self.targets, weights, - average_across_timesteps=False, - average_across_batch=False) - res = self.evaluate(total_loss) - compare_total = np.zeros((self.batch_size, self.sequence_length)) - self.assertAllClose(compare_total, res) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/seq2seq/python/ops/__init__.py b/tensorflow/contrib/seq2seq/python/ops/__init__.py deleted file mode 100644 index 52e83069cb0..00000000000 --- a/tensorflow/contrib/seq2seq/python/ops/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py deleted file mode 100644 index 0e19d1e3205..00000000000 --- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py +++ /dev/null @@ -1,2538 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A powerful dynamic attention wrapper object.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import functools -import math - -import numpy as np - -from tensorflow.contrib.framework.python.framework import tensor_util -from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.keras import initializers -from tensorflow.python.keras import layers -from tensorflow.python.keras.engine import base_layer_utils -from tensorflow.python.layers import base as layers_base -from tensorflow.python.layers import core as layers_core -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import functional_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import rnn_cell_impl -from tensorflow.python.ops import tensor_array_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.util import nest - -__all__ = [ - "AttentionMechanism", - "AttentionWrapper", - "AttentionWrapperState", - "LuongAttention", - "BahdanauAttention", - "hardmax", - "safe_cumprod", - "monotonic_attention", - "BahdanauMonotonicAttention", - "LuongMonotonicAttention", -] - -_zero_state_tensors = rnn_cell_impl._zero_state_tensors # pylint: disable=protected-access - - -class AttentionMechanism(object): - - @property - def alignments_size(self): - raise NotImplementedError - - @property - def state_size(self): - raise NotImplementedError - - -class _BaseAttentionMechanism(AttentionMechanism): - """A base AttentionMechanism class providing common functionality. - - Common functionality includes: - 1. Storing the query and memory layers. - 2. Preprocessing and storing the memory. - """ - - def __init__(self, - query_layer, - memory, - probability_fn, - memory_sequence_length=None, - memory_layer=None, - check_inner_dims_defined=True, - score_mask_value=None, - custom_key_value_fn=None, - name=None): - """Construct base AttentionMechanism class. - - Args: - query_layer: Callable. Instance of `tf.compat.v1.layers.Layer`. The - layer's depth must match the depth of `memory_layer`. If `query_layer` - is not provided, the shape of `query` must match that of `memory_layer`. - memory: The memory to query; usually the output of an RNN encoder. This - tensor should be shaped `[batch_size, max_time, ...]`. - probability_fn: A `callable`. Converts the score and previous alignments - to probabilities. Its signature should be: `probabilities = - probability_fn(score, state)`. - memory_sequence_length (optional): Sequence lengths for the batch entries - in memory. If provided, the memory tensor rows are masked with zeros - for values past the respective sequence lengths. - memory_layer: Instance of `tf.compat.v1.layers.Layer` (may be None). The - layer's depth must match the depth of `query_layer`. If `memory_layer` - is not provided, the shape of `memory` must match that of `query_layer`. - check_inner_dims_defined: Python boolean. If `True`, the `memory` - argument's shape is checked to ensure all but the two outermost - dimensions are fully defined. - score_mask_value: (optional): The mask value for score before passing into - `probability_fn`. The default is -inf. Only used if - `memory_sequence_length` is not None. - custom_key_value_fn: (optional): The custom function for - computing keys and values. - name: Name to use when creating ops. - """ - if (query_layer is not None and - not isinstance(query_layer, layers_base.Layer)): - raise TypeError("query_layer is not a Layer: %s" % - type(query_layer).__name__) - if (memory_layer is not None and - not isinstance(memory_layer, layers_base.Layer)): - raise TypeError("memory_layer is not a Layer: %s" % - type(memory_layer).__name__) - self._query_layer = query_layer - self._memory_layer = memory_layer - self.dtype = memory_layer.dtype - if not callable(probability_fn): - raise TypeError("probability_fn must be callable, saw type: %s" % - type(probability_fn).__name__) - if score_mask_value is None: - score_mask_value = dtypes.as_dtype( - self._memory_layer.dtype).as_numpy_dtype(-np.inf) - self._probability_fn = lambda score, prev: ( # pylint:disable=g-long-lambda - probability_fn( - _maybe_mask_score( - score, - memory_sequence_length=memory_sequence_length, - score_mask_value=score_mask_value), prev)) - with ops.name_scope(name, "BaseAttentionMechanismInit", - nest.flatten(memory)): - self._values = _prepare_memory( - memory, - memory_sequence_length=memory_sequence_length, - check_inner_dims_defined=check_inner_dims_defined) - self._keys = ( - self.memory_layer(self._values) if self.memory_layer # pylint: disable=not-callable - else self._values) - if custom_key_value_fn is not None: - self._keys, self._values = custom_key_value_fn(self._keys, self._values) - self._batch_size = ( - tensor_shape.dimension_value(self._keys.shape[0]) or - array_ops.shape(self._keys)[0]) - self._alignments_size = ( - tensor_shape.dimension_value(self._keys.shape[1]) or - array_ops.shape(self._keys)[1]) - - @property - def memory_layer(self): - return self._memory_layer - - @property - def query_layer(self): - return self._query_layer - - @property - def values(self): - return self._values - - @property - def keys(self): - return self._keys - - @property - def batch_size(self): - return self._batch_size - - @property - def alignments_size(self): - return self._alignments_size - - @property - def state_size(self): - return self._alignments_size - - def initial_alignments(self, batch_size, dtype): - """Creates the initial alignment values for the `AttentionWrapper` class. - - This is important for AttentionMechanisms that use the previous alignment - to calculate the alignment at the next time step (e.g. monotonic attention). - - The default behavior is to return a tensor of all zeros. - - Args: - batch_size: `int32` scalar, the batch_size. - dtype: The `dtype`. - - Returns: - A `dtype` tensor shaped `[batch_size, alignments_size]` - (`alignments_size` is the values' `max_time`). - """ - max_time = self._alignments_size - return _zero_state_tensors(max_time, batch_size, dtype) - - def initial_state(self, batch_size, dtype): - """Creates the initial state values for the `AttentionWrapper` class. - - This is important for AttentionMechanisms that use the previous alignment - to calculate the alignment at the next time step (e.g. monotonic attention). - - The default behavior is to return the same output as initial_alignments. - - Args: - batch_size: `int32` scalar, the batch_size. - dtype: The `dtype`. - - Returns: - A structure of all-zero tensors with shapes as described by `state_size`. - """ - return self.initial_alignments(batch_size, dtype) - - -class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer): - """A base AttentionMechanism class providing common functionality. - - Common functionality includes: - 1. Storing the query and memory layers. - 2. Preprocessing and storing the memory. - - Note that this layer takes memory as its init parameter, which is an - anti-pattern of Keras API, we have to keep the memory as init parameter for - performance and dependency reason. Under the hood, during `__init__()`, it - will invoke `base_layer.__call__(memory, setup_memory=True)`. This will let - keras to keep track of the memory tensor as the input of this layer. Once - the `__init__()` is done, then user can query the attention by - `score = att_obj([query, state])`, and use it as a normal keras layer. - - Special attention is needed when adding using this class as the base layer for - new attention: - 1. Build() could be invoked at least twice. So please make sure weights are - not duplicated. - 2. Layer.get_weights() might return different set of weights if the instance - has `query_layer`. The query_layer weights is not initialized until the - memory is configured. - - Also note that this layer does not work with Keras model when - `model.compile(run_eagerly=True)` due to the fact that this layer is stateful. - The support for that will be added in a future version. - """ - - def __init__(self, - memory, - probability_fn, - query_layer=None, - memory_layer=None, - memory_sequence_length=None, - **kwargs): - """Construct base AttentionMechanism class. - - Args: - memory: The memory to query; usually the output of an RNN encoder. This - tensor should be shaped `[batch_size, max_time, ...]`. - probability_fn: A `callable`. Converts the score and previous alignments - to probabilities. Its signature should be: `probabilities = - probability_fn(score, state)`. - query_layer: (optional): Instance of `tf.keras.Layer`. The layer's depth - must match the depth of `memory_layer`. If `query_layer` is not - provided, the shape of `query` must match that of `memory_layer`. - memory_layer: (optional): Instance of `tf.keras.Layer`. The layer's depth - must match the depth of `query_layer`. If `memory_layer` is not - provided, the shape of `memory` must match that of `query_layer`. - memory_sequence_length (optional): Sequence lengths for the batch entries - in memory. If provided, the memory tensor rows are masked with zeros for - values past the respective sequence lengths. - **kwargs: Dictionary that contains other common arguments for layer - creation. - """ - if (query_layer is not None and not isinstance(query_layer, layers.Layer)): - raise TypeError("query_layer is not a Layer: %s" % - type(query_layer).__name__) - if (memory_layer is not None and - not isinstance(memory_layer, layers.Layer)): - raise TypeError("memory_layer is not a Layer: %s" % - type(memory_layer).__name__) - self.query_layer = query_layer - self.memory_layer = memory_layer - if self.memory_layer is not None and "dtype" not in kwargs: - kwargs["dtype"] = self.memory_layer.dtype - super(_BaseAttentionMechanismV2, self).__init__(**kwargs) - if not callable(probability_fn): - raise TypeError("probability_fn must be callable, saw type: %s" % - type(probability_fn).__name__) - self.probability_fn = probability_fn - - self.keys = None - self.values = None - self.batch_size = None - self._memory_initialized = False - self._check_inner_dims_defined = True - self.supports_masking = True - self.score_mask_value = dtypes.as_dtype(self.dtype).as_numpy_dtype(-np.inf) - - if memory is not None: - # Setup the memory by self.__call__() with memory and memory_seq_length. - # This will make the attention follow the keras convention which takes - # all the tensor inputs via __call__(). - if memory_sequence_length is None: - inputs = memory - else: - inputs = [memory, memory_sequence_length] - - self.values = super(_BaseAttentionMechanismV2, self).__call__( - inputs, setup_memory=True) - - def build(self, input_shape): - if not self._memory_initialized: - # This is for setting up the memory, which contains memory and optional - # memory_sequence_length. Build the memory_layer with memory shape. - if self.memory_layer is not None and not self.memory_layer.built: - if isinstance(input_shape, list): - self.memory_layer.build(input_shape[0]) - else: - self.memory_layer.build(input_shape) - else: - # The input_shape should be query.shape and state.shape. Use the query - # to init the query layer. - if self.query_layer is not None and not self.query_layer.built: - self.query_layer.build(input_shape[0]) - - def __call__(self, inputs, **kwargs): - """Preprocess the inputs before calling `base_layer.__call__()`. - - Note that there are situation here, one for setup memory, and one with - actual query and state. - 1. When the memory has not been configured, we just pass all the param to - base_layer.__call__(), which will then invoke self.call() with proper - inputs, which allows this class to setup memory. - 2. When the memory has already been setup, the input should contain query - and state, and optionally processed memory. If the processed memory is - not included in the input, we will have to append it to the inputs and - give it to the base_layer.__call__(). The processed memory is the output - of first invocation of self.__call__(). If we don't add it here, then from - keras perspective, the graph is disconnected since the output from - previous call is never used. - - Args: - inputs: the inputs tensors. - **kwargs: dict, other keyeword arguments for the `__call__()` - """ - if self._memory_initialized: - if len(inputs) not in (2, 3): - raise ValueError("Expect the inputs to have 2 or 3 tensors, got %d" % - len(inputs)) - if len(inputs) == 2: - # We append the calculated memory here so that the graph will be - # connected. - inputs.append(self.values) - return super(_BaseAttentionMechanismV2, self).__call__(inputs, **kwargs) - - def call(self, inputs, mask=None, setup_memory=False, **kwargs): - """Setup the memory or query the attention. - - There are two case here, one for setup memory, and the second is query the - attention score. `setup_memory` is the flag to indicate which mode it is. - The input list will be treated differently based on that flag. - - Args: - inputs: a list of tensor that could either be `query` and `state`, or - `memory` and `memory_sequence_length`. `query` is the tensor of dtype - matching `memory` and shape `[batch_size, query_depth]`. `state` is the - tensor of dtype matching `memory` and shape `[batch_size, - alignments_size]`. (`alignments_size` is memory's `max_time`). `memory` - is the memory to query; usually the output of an RNN encoder. The tensor - should be shaped `[batch_size, max_time, ...]`. `memory_sequence_length` - (optional) is the sequence lengths for the batch entries in memory. If - provided, the memory tensor rows are masked with zeros for values past - the respective sequence lengths. - mask: optional bool tensor with shape `[batch, max_time]` for the mask of - memory. If it is not None, the corresponding item of the memory should - be filtered out during calculation. - setup_memory: boolean, whether the input is for setting up memory, or - query attention. - **kwargs: Dict, other keyword arguments for the call method. - - Returns: - Either processed memory or attention score, based on `setup_memory`. - """ - if setup_memory: - if isinstance(inputs, list): - if len(inputs) not in (1, 2): - raise ValueError("Expect inputs to have 1 or 2 tensors, got %d" % - len(inputs)) - memory = inputs[0] - memory_sequence_length = inputs[1] if len(inputs) == 2 else None - memory_mask = mask - else: - memory, memory_sequence_length = inputs, None - memory_mask = mask - self._setup_memory(memory, memory_sequence_length, memory_mask) - # We force the self.built to false here since only memory is initialized, - # but the real query/state has not been call() yet. The layer should be - # build and call again. - self.built = False - # Return the processed memory in order to create the Keras connectivity - # data for it. - return self.values - else: - if not self._memory_initialized: - raise ValueError("Cannot query the attention before the setup of " - "memory") - if len(inputs) not in (2, 3): - raise ValueError("Expect the inputs to have query, state, and optional " - "processed memory, got %d items" % len(inputs)) - # Ignore the rest of the inputs and only care about the query and state - query, state = inputs[0], inputs[1] - return self._calculate_attention(query, state) - - def _setup_memory(self, memory, memory_sequence_length=None, - memory_mask=None): - """Pre-process the memory before actually query the memory. - - This should only be called once at the first invocation of call(). - - Args: - memory: The memory to query; usually the output of an RNN encoder. This - tensor should be shaped `[batch_size, max_time, ...]`. - memory_sequence_length (optional): Sequence lengths for the batch entries - in memory. If provided, the memory tensor rows are masked with zeros for - values past the respective sequence lengths. - memory_mask: (Optional) The boolean tensor with shape `[batch_size, - max_time]`. For any value equal to False, the corresponding value in - memory should be ignored. - """ - if self._memory_initialized: - raise ValueError("The memory for the attention has already been setup.") - if memory_sequence_length is not None and memory_mask is not None: - raise ValueError("memory_sequence_length and memory_mask cannot be " - "used at same time for attention.") - with ops.name_scope(self.name, "BaseAttentionMechanismInit", - nest.flatten(memory)): - self.values = _prepare_memory( - memory, - memory_sequence_length=memory_sequence_length, - memory_mask=memory_mask, - check_inner_dims_defined=self._check_inner_dims_defined) - # Mark the value as check since the memory and memory mask might not - # passed from __call__(), which does not have proper keras metadata. - # TODO(omalleyt): Remove this hack once the mask the has proper keras - # history. - base_layer_utils.mark_checked(self.values) - if self.memory_layer is not None: - self.keys = self.memory_layer(self.values) - else: - self.keys = self.values - self.batch_size = ( - tensor_shape.dimension_value(self.keys.shape[0]) or - array_ops.shape(self.keys)[0]) - self._alignments_size = ( - tensor_shape.dimension_value(self.keys.shape[1]) or - array_ops.shape(self.keys)[1]) - if memory_mask is not None: - unwrapped_probability_fn = self.probability_fn - - def _mask_probability_fn(score, prev): - return unwrapped_probability_fn( - _maybe_mask_score( - score, - memory_mask=memory_mask, - memory_sequence_length=memory_sequence_length, - score_mask_value=self.score_mask_value), prev) - - self.probability_fn = _mask_probability_fn - self._memory_initialized = True - - def _calculate_attention(self, query, state): - raise NotImplementedError( - "_calculate_attention need to be implemented by subclasses.") - - def compute_mask(self, inputs, mask=None): - # There real input of the attention is query and state, and the memory layer - # mask shouldn't be pass down. Returning None for all output mask here. - return None, None - - def get_config(self): - config = {} - # Since the probability_fn is likely to be a wrapped function, the child - # class should preserve the original function and how its wrapped. - - if self.query_layer is not None: - config["query_layer"] = { - "class_name": self.query_layer.__class__.__name__, - "config": self.query_layer.get_config(), - } - if self.memory_layer is not None: - config["memory_layer"] = { - "class_name": self.memory_layer.__class__.__name__, - "config": self.memory_layer.get_config(), - } - # memory is a required init parameter and its a tensor. It cannot be - # serialized to config, so we put a placeholder for it. - config["memory"] = None - base_config = super(_BaseAttentionMechanismV2, self).get_config() - return dict(list(base_config.items()) + list(config.items())) - - def _process_probability_fn(self, func_name): - """Helper method to retrieve the probably function by string input.""" - valid_probability_fns = { - "softmax": nn_ops.softmax, - "hardmax": hardmax, - } - if func_name not in valid_probability_fns.keys(): - raise ValueError("Invalid probability function: %s, options are %s" % - (func_name, valid_probability_fns.keys())) - return valid_probability_fns[func_name] - - @classmethod - def deserialize_inner_layer_from_config(cls, config, custom_objects): - """Helper method that reconstruct the query and memory from the config. - - In the get_config() method, the query and memory layer configs are - serialized into dict for persistence, this method perform the reverse action - to reconstruct the layer from the config. - - Args: - config: dict, the configs that will be used to reconstruct the object. - custom_objects: dict mapping class names (or function names) of custom - (non-Keras) objects to class/functions. - - Returns: - config: dict, the config with layer instance created, which is ready to be - used as init parameters. - """ - # Reconstruct the query and memory layer for parent class. - from tensorflow.python.keras.layers import deserialize as deserialize_layer # pylint: disable=g-import-not-at-top - # Instead of updating the input, create a copy and use that. - config = config.copy() - query_layer_config = config.pop("query_layer", None) - if query_layer_config: - query_layer = deserialize_layer( - query_layer_config, custom_objects=custom_objects) - config["query_layer"] = query_layer - memory_layer_config = config.pop("memory_layer", None) - if memory_layer_config: - memory_layer = deserialize_layer( - memory_layer_config, custom_objects=custom_objects) - config["memory_layer"] = memory_layer - return config - - @property - def alignments_size(self): - return self._alignments_size - - @property - def state_size(self): - return self._alignments_size - - def initial_alignments(self, batch_size, dtype): - """Creates the initial alignment values for the `AttentionWrapper` class. - - This is important for AttentionMechanisms that use the previous alignment - to calculate the alignment at the next time step (e.g. monotonic attention). - - The default behavior is to return a tensor of all zeros. - - Args: - batch_size: `int32` scalar, the batch_size. - dtype: The `dtype`. - - Returns: - A `dtype` tensor shaped `[batch_size, alignments_size]` - (`alignments_size` is the values' `max_time`). - """ - max_time = self._alignments_size - return _zero_state_tensors(max_time, batch_size, dtype) - - def initial_state(self, batch_size, dtype): - """Creates the initial state values for the `AttentionWrapper` class. - - This is important for AttentionMechanisms that use the previous alignment - to calculate the alignment at the next time step (e.g. monotonic attention). - - The default behavior is to return the same output as initial_alignments. - - Args: - batch_size: `int32` scalar, the batch_size. - dtype: The `dtype`. - - Returns: - A structure of all-zero tensors with shapes as described by `state_size`. - """ - return self.initial_alignments(batch_size, dtype) - - -def _luong_score(query, keys, scale): - """Implements Luong-style (multiplicative) scoring function. - - This attention has two forms. The first is standard Luong attention, - as described in: - - Minh-Thang Luong, Hieu Pham, Christopher D. Manning. - "Effective Approaches to Attention-based Neural Machine Translation." - EMNLP 2015. https://arxiv.org/abs/1508.04025 - - The second is the scaled form inspired partly by the normalized form of - Bahdanau attention. - - To enable the second form, call this function with `scale=True`. - - Args: - query: Tensor, shape `[batch_size, num_units]` to compare to keys. - keys: Processed memory, shape `[batch_size, max_time, num_units]`. - scale: the optional tensor to scale the attention score. - - Returns: - A `[batch_size, max_time]` tensor of unnormalized score values. - - Raises: - ValueError: If `key` and `query` depths do not match. - """ - depth = query.get_shape()[-1] - key_units = keys.get_shape()[-1] - if depth != key_units: - raise ValueError( - "Incompatible or unknown inner dimensions between query and keys. " - "Query (%s) has units: %s. Keys (%s) have units: %s. " - "Perhaps you need to set num_units to the keys' dimension (%s)?" % - (query, depth, keys, key_units, key_units)) - - # Reshape from [batch_size, depth] to [batch_size, 1, depth] - # for matmul. - query = array_ops.expand_dims(query, 1) - - # Inner product along the query units dimension. - # matmul shapes: query is [batch_size, 1, depth] and - # keys is [batch_size, max_time, depth]. - # the inner product is asked to **transpose keys' inner shape** to get a - # batched matmul on: - # [batch_size, 1, depth] . [batch_size, depth, max_time] - # resulting in an output shape of: - # [batch_size, 1, max_time]. - # we then squeeze out the center singleton dimension. - score = math_ops.matmul(query, keys, transpose_b=True) - score = array_ops.squeeze(score, [1]) - - if scale is not None: - score = scale * score - return score - - -class LuongAttention(_BaseAttentionMechanism): - """Implements Luong-style (multiplicative) attention scoring. - - This attention has two forms. The first is standard Luong attention, - as described in: - - Minh-Thang Luong, Hieu Pham, Christopher D. Manning. - [Effective Approaches to Attention-based Neural Machine Translation. - EMNLP 2015.](https://arxiv.org/abs/1508.04025) - - The second is the scaled form inspired partly by the normalized form of - Bahdanau attention. - - To enable the second form, construct the object with parameter - `scale=True`. - """ - - def __init__(self, - num_units, - memory, - memory_sequence_length=None, - scale=False, - probability_fn=None, - score_mask_value=None, - dtype=None, - custom_key_value_fn=None, - name="LuongAttention"): - """Construct the AttentionMechanism mechanism. - - Args: - num_units: The depth of the attention mechanism. - memory: The memory to query; usually the output of an RNN encoder. This - tensor should be shaped `[batch_size, max_time, ...]`. - memory_sequence_length: (optional) Sequence lengths for the batch entries - in memory. If provided, the memory tensor rows are masked with zeros - for values past the respective sequence lengths. - scale: Python boolean. Whether to scale the energy term. - probability_fn: (optional) A `callable`. Converts the score to - probabilities. The default is `tf.nn.softmax`. Other options include - `tf.contrib.seq2seq.hardmax` and `tf.contrib.sparsemax.sparsemax`. - Its signature should be: `probabilities = probability_fn(score)`. - score_mask_value: (optional) The mask value for score before passing into - `probability_fn`. The default is -inf. Only used if - `memory_sequence_length` is not None. - dtype: The data type for the memory layer of the attention mechanism. - custom_key_value_fn: (optional): The custom function for - computing keys and values. - name: Name to use when creating ops. - """ - # For LuongAttention, we only transform the memory layer; thus - # num_units **must** match expected the query depth. - if probability_fn is None: - probability_fn = nn_ops.softmax - if dtype is None: - dtype = dtypes.float32 - wrapped_probability_fn = lambda score, _: probability_fn(score) - super(LuongAttention, self).__init__( - query_layer=None, - memory_layer=layers_core.Dense( - num_units, name="memory_layer", use_bias=False, dtype=dtype), - memory=memory, - probability_fn=wrapped_probability_fn, - memory_sequence_length=memory_sequence_length, - score_mask_value=score_mask_value, - custom_key_value_fn=custom_key_value_fn, - name=name) - self._num_units = num_units - self._scale = scale - self._name = name - - def __call__(self, query, state): - """Score the query based on the keys and values. - - Args: - query: Tensor of dtype matching `self.values` and shape `[batch_size, - query_depth]`. - state: Tensor of dtype matching `self.values` and shape `[batch_size, - alignments_size]` (`alignments_size` is memory's `max_time`). - - Returns: - alignments: Tensor of dtype matching `self.values` and shape - `[batch_size, alignments_size]` (`alignments_size` is memory's - `max_time`). - """ - with variable_scope.variable_scope(None, "luong_attention", [query]): - attention_g = None - if self._scale: - attention_g = variable_scope.get_variable( - "attention_g", - dtype=query.dtype, - initializer=init_ops.ones_initializer, - shape=()) - score = _luong_score(query, self._keys, attention_g) - alignments = self._probability_fn(score, state) - next_state = alignments - return alignments, next_state - - -class LuongAttentionV2(_BaseAttentionMechanismV2): - """Implements Luong-style (multiplicative) attention scoring. - - This attention has two forms. The first is standard Luong attention, - as described in: - - Minh-Thang Luong, Hieu Pham, Christopher D. Manning. - [Effective Approaches to Attention-based Neural Machine Translation. - EMNLP 2015.](https://arxiv.org/abs/1508.04025) - - The second is the scaled form inspired partly by the normalized form of - Bahdanau attention. - - To enable the second form, construct the object with parameter - `scale=True`. - """ - - def __init__(self, - units, - memory, - memory_sequence_length=None, - scale=False, - probability_fn="softmax", - dtype=None, - name="LuongAttention", - **kwargs): - """Construct the AttentionMechanism mechanism. - - Args: - units: The depth of the attention mechanism. - memory: The memory to query; usually the output of an RNN encoder. This - tensor should be shaped `[batch_size, max_time, ...]`. - memory_sequence_length: (optional): Sequence lengths for the batch entries - in memory. If provided, the memory tensor rows are masked with zeros - for values past the respective sequence lengths. - scale: Python boolean. Whether to scale the energy term. - probability_fn: (optional) string, the name of function to convert the - attention score to probabilities. The default is `softmax` which is - `tf.nn.softmax`. Other options is `hardmax`, which is hardmax() within - this module. Any other value will result intovalidation error. Default - to use `softmax`. - dtype: The data type for the memory layer of the attention mechanism. - name: Name to use when creating ops. - **kwargs: Dictionary that contains other common arguments for layer - creation. - """ - # For LuongAttention, we only transform the memory layer; thus - # num_units **must** match expected the query depth. - self.probability_fn_name = probability_fn - probability_fn = self._process_probability_fn(self.probability_fn_name) - wrapped_probability_fn = lambda score, _: probability_fn(score) - if dtype is None: - dtype = dtypes.float32 - memory_layer = kwargs.pop("memory_layer", None) - if not memory_layer: - memory_layer = layers.Dense( - units, name="memory_layer", use_bias=False, dtype=dtype) - self.units = units - self.scale = scale - self.scale_weight = None - super(LuongAttentionV2, self).__init__( - memory=memory, - memory_sequence_length=memory_sequence_length, - query_layer=None, - memory_layer=memory_layer, - probability_fn=wrapped_probability_fn, - name=name, - dtype=dtype, - **kwargs) - - def build(self, input_shape): - super(LuongAttentionV2, self).build(input_shape) - if self.scale and self.scale_weight is None: - self.scale_weight = self.add_weight( - "attention_g", initializer=init_ops.ones_initializer, shape=()) - self.built = True - - def _calculate_attention(self, query, state): - """Score the query based on the keys and values. - - Args: - query: Tensor of dtype matching `self.values` and shape `[batch_size, - query_depth]`. - state: Tensor of dtype matching `self.values` and shape `[batch_size, - alignments_size]` (`alignments_size` is memory's `max_time`). - - Returns: - alignments: Tensor of dtype matching `self.values` and shape - `[batch_size, alignments_size]` (`alignments_size` is memory's - `max_time`). - next_state: Same as the alignments. - """ - score = _luong_score(query, self.keys, self.scale_weight) - alignments = self.probability_fn(score, state) - next_state = alignments - return alignments, next_state - - def get_config(self): - config = { - "units": self.units, - "scale": self.scale, - "probability_fn": self.probability_fn_name, - } - base_config = super(LuongAttentionV2, self).get_config() - return dict(list(base_config.items()) + list(config.items())) - - @classmethod - def from_config(cls, config, custom_objects=None): - config = _BaseAttentionMechanismV2.deserialize_inner_layer_from_config( - config, custom_objects=custom_objects) - return cls(**config) - - -def _bahdanau_score(processed_query, - keys, - attention_v, - attention_g=None, - attention_b=None): - """Implements Bahdanau-style (additive) scoring function. - - This attention has two forms. The first is Bhandanau attention, - as described in: - - Dzmitry Bahdanau, Kyunghyun Cho, Yoshua Bengio. - "Neural Machine Translation by Jointly Learning to Align and Translate." - ICLR 2015. https://arxiv.org/abs/1409.0473 - - The second is the normalized form. This form is inspired by the - weight normalization article: - - Tim Salimans, Diederik P. Kingma. - "Weight Normalization: A Simple Reparameterization to Accelerate - Training of Deep Neural Networks." - https://arxiv.org/abs/1602.07868 - - To enable the second form, set please pass in attention_g and attention_b. - - Args: - processed_query: Tensor, shape `[batch_size, num_units]` to compare to keys. - keys: Processed memory, shape `[batch_size, max_time, num_units]`. - attention_v: Tensor, shape `[num_units]`. - attention_g: Optional scalar tensor for normalization. - attention_b: Optional tensor with shape `[num_units]` for normalization. - - Returns: - A `[batch_size, max_time]` tensor of unnormalized score values. - """ - # Reshape from [batch_size, ...] to [batch_size, 1, ...] for broadcasting. - processed_query = array_ops.expand_dims(processed_query, 1) - if attention_g is not None and attention_b is not None: - normed_v = attention_g * attention_v * math_ops.rsqrt( - math_ops.reduce_sum(math_ops.square(attention_v))) - return math_ops.reduce_sum( - normed_v * math_ops.tanh(keys + processed_query + attention_b), [2]) - else: - return math_ops.reduce_sum( - attention_v * math_ops.tanh(keys + processed_query), [2]) - - -class BahdanauAttention(_BaseAttentionMechanism): - """Implements Bahdanau-style (additive) attention. - - This attention has two forms. The first is Bahdanau attention, - as described in: - - Dzmitry Bahdanau, Kyunghyun Cho, Yoshua Bengio. - "Neural Machine Translation by Jointly Learning to Align and Translate." - ICLR 2015. https://arxiv.org/abs/1409.0473 - - The second is the normalized form. This form is inspired by the - weight normalization article: - - Tim Salimans, Diederik P. Kingma. - "Weight Normalization: A Simple Reparameterization to Accelerate - Training of Deep Neural Networks." - https://arxiv.org/abs/1602.07868 - - To enable the second form, construct the object with parameter - `normalize=True`. - """ - - def __init__(self, - num_units, - memory, - memory_sequence_length=None, - normalize=False, - probability_fn=None, - score_mask_value=None, - dtype=None, - custom_key_value_fn=None, - name="BahdanauAttention"): - """Construct the Attention mechanism. - - Args: - num_units: The depth of the query mechanism. - memory: The memory to query; usually the output of an RNN encoder. This - tensor should be shaped `[batch_size, max_time, ...]`. - memory_sequence_length: (optional) Sequence lengths for the batch entries - in memory. If provided, the memory tensor rows are masked with zeros - for values past the respective sequence lengths. - normalize: Python boolean. Whether to normalize the energy term. - probability_fn: (optional) A `callable`. Converts the score to - probabilities. The default is `tf.nn.softmax`. Other options include - `tf.contrib.seq2seq.hardmax` and `tf.contrib.sparsemax.sparsemax`. - Its signature should be: `probabilities = probability_fn(score)`. - score_mask_value: (optional): The mask value for score before passing into - `probability_fn`. The default is -inf. Only used if - `memory_sequence_length` is not None. - dtype: The data type for the query and memory layers of the attention - mechanism. - custom_key_value_fn: (optional): The custom function for - computing keys and values. - name: Name to use when creating ops. - """ - if probability_fn is None: - probability_fn = nn_ops.softmax - if dtype is None: - dtype = dtypes.float32 - wrapped_probability_fn = lambda score, _: probability_fn(score) - super(BahdanauAttention, self).__init__( - query_layer=layers_core.Dense( - num_units, name="query_layer", use_bias=False, dtype=dtype), - memory_layer=layers_core.Dense( - num_units, name="memory_layer", use_bias=False, dtype=dtype), - memory=memory, - probability_fn=wrapped_probability_fn, - custom_key_value_fn=custom_key_value_fn, - memory_sequence_length=memory_sequence_length, - score_mask_value=score_mask_value, - name=name) - self._num_units = num_units - self._normalize = normalize - self._name = name - - def __call__(self, query, state): - """Score the query based on the keys and values. - - Args: - query: Tensor of dtype matching `self.values` and shape `[batch_size, - query_depth]`. - state: Tensor of dtype matching `self.values` and shape `[batch_size, - alignments_size]` (`alignments_size` is memory's `max_time`). - - Returns: - alignments: Tensor of dtype matching `self.values` and shape - `[batch_size, alignments_size]` (`alignments_size` is memory's - `max_time`). - """ - with variable_scope.variable_scope(None, "bahdanau_attention", [query]): - processed_query = self.query_layer(query) if self.query_layer else query - attention_v = variable_scope.get_variable( - "attention_v", [self._num_units], dtype=query.dtype) - if not self._normalize: - attention_g = None - attention_b = None - else: - attention_g = variable_scope.get_variable( - "attention_g", - dtype=query.dtype, - initializer=init_ops.constant_initializer( - math.sqrt((1. / self._num_units))), - shape=()) - attention_b = variable_scope.get_variable( - "attention_b", [self._num_units], - dtype=query.dtype, - initializer=init_ops.zeros_initializer()) - - score = _bahdanau_score( - processed_query, - self._keys, - attention_v, - attention_g=attention_g, - attention_b=attention_b) - alignments = self._probability_fn(score, state) - next_state = alignments - return alignments, next_state - - -class BahdanauAttentionV2(_BaseAttentionMechanismV2): - """Implements Bahdanau-style (additive) attention. - - This attention has two forms. The first is Bahdanau attention, - as described in: - - Dzmitry Bahdanau, Kyunghyun Cho, Yoshua Bengio. - "Neural Machine Translation by Jointly Learning to Align and Translate." - ICLR 2015. https://arxiv.org/abs/1409.0473 - - The second is the normalized form. This form is inspired by the - weight normalization article: - - Tim Salimans, Diederik P. Kingma. - "Weight Normalization: A Simple Reparameterization to Accelerate - Training of Deep Neural Networks." - https://arxiv.org/abs/1602.07868 - - To enable the second form, construct the object with parameter - `normalize=True`. - """ - - def __init__(self, - units, - memory, - memory_sequence_length=None, - normalize=False, - probability_fn="softmax", - kernel_initializer="glorot_uniform", - dtype=None, - name="BahdanauAttention", - **kwargs): - """Construct the Attention mechanism. - - Args: - units: The depth of the query mechanism. - memory: The memory to query; usually the output of an RNN encoder. This - tensor should be shaped `[batch_size, max_time, ...]`. - memory_sequence_length: (optional): Sequence lengths for the batch entries - in memory. If provided, the memory tensor rows are masked with zeros - for values past the respective sequence lengths. - normalize: Python boolean. Whether to normalize the energy term. - probability_fn: (optional) string, the name of function to convert the - attention score to probabilities. The default is `softmax` which is - `tf.nn.softmax`. Other options is `hardmax`, which is hardmax() within - this module. Any other value will result into validation error. Default - to use `softmax`. - kernel_initializer: (optional), the name of the initializer for the - attention kernel. - dtype: The data type for the query and memory layers of the attention - mechanism. - name: Name to use when creating ops. - **kwargs: Dictionary that contains other common arguments for layer - creation. - """ - self.probability_fn_name = probability_fn - probability_fn = self._process_probability_fn(self.probability_fn_name) - wrapped_probability_fn = lambda score, _: probability_fn(score) - if dtype is None: - dtype = dtypes.float32 - query_layer = kwargs.pop("query_layer", None) - if not query_layer: - query_layer = layers.Dense( - units, name="query_layer", use_bias=False, dtype=dtype) - memory_layer = kwargs.pop("memory_layer", None) - if not memory_layer: - memory_layer = layers.Dense( - units, name="memory_layer", use_bias=False, dtype=dtype) - self.units = units - self.normalize = normalize - self.kernel_initializer = initializers.get(kernel_initializer) - self.attention_v = None - self.attention_g = None - self.attention_b = None - super(BahdanauAttentionV2, self).__init__( - memory=memory, - memory_sequence_length=memory_sequence_length, - query_layer=query_layer, - memory_layer=memory_layer, - probability_fn=wrapped_probability_fn, - name=name, - dtype=dtype, - **kwargs) - - def build(self, input_shape): - super(BahdanauAttentionV2, self).build(input_shape) - if self.attention_v is None: - self.attention_v = self.add_weight( - "attention_v", [self.units], - dtype=self.dtype, - initializer=self.kernel_initializer) - if self.normalize and self.attention_g is None and self.attention_b is None: - self.attention_g = self.add_weight( - "attention_g", - initializer=init_ops.constant_initializer( - math.sqrt((1. / self.units))), - shape=()) - self.attention_b = self.add_weight( - "attention_b", - shape=[self.units], - initializer=init_ops.zeros_initializer()) - self.built = True - - def _calculate_attention(self, query, state): - """Score the query based on the keys and values. - - Args: - query: Tensor of dtype matching `self.values` and shape `[batch_size, - query_depth]`. - state: Tensor of dtype matching `self.values` and shape `[batch_size, - alignments_size]` (`alignments_size` is memory's `max_time`). - - Returns: - alignments: Tensor of dtype matching `self.values` and shape - `[batch_size, alignments_size]` (`alignments_size` is memory's - `max_time`). - next_state: same as alignments. - """ - processed_query = self.query_layer(query) if self.query_layer else query - score = _bahdanau_score( - processed_query, - self.keys, - self.attention_v, - attention_g=self.attention_g, - attention_b=self.attention_b) - alignments = self.probability_fn(score, state) - next_state = alignments - return alignments, next_state - - def get_config(self): - config = { - "units": self.units, - "normalize": self.normalize, - "probability_fn": self.probability_fn_name, - "kernel_initializer": initializers.serialize(self.kernel_initializer) - } - base_config = super(BahdanauAttentionV2, self).get_config() - return dict(list(base_config.items()) + list(config.items())) - - @classmethod - def from_config(cls, config, custom_objects=None): - config = _BaseAttentionMechanismV2.deserialize_inner_layer_from_config( - config, custom_objects=custom_objects) - return cls(**config) - - -def safe_cumprod(x, *args, **kwargs): - """Computes cumprod of x in logspace using cumsum to avoid underflow. - - The cumprod function and its gradient can result in numerical instabilities - when its argument has very small and/or zero values. As long as the argument - is all positive, we can instead compute the cumulative product as - exp(cumsum(log(x))). This function can be called identically to tf.cumprod. - - Args: - x: Tensor to take the cumulative product of. - *args: Passed on to cumsum; these are identical to those in cumprod. - **kwargs: Passed on to cumsum; these are identical to those in cumprod. - - Returns: - Cumulative product of x. - """ - with ops.name_scope(None, "SafeCumprod", [x]): - x = ops.convert_to_tensor(x, name="x") - tiny = np.finfo(x.dtype.as_numpy_dtype).tiny - return math_ops.exp( - math_ops.cumsum( - math_ops.log(clip_ops.clip_by_value(x, tiny, 1)), *args, **kwargs)) - - -def monotonic_attention(p_choose_i, previous_attention, mode): - """Compute monotonic attention distribution from choosing probabilities. - - Monotonic attention implies that the input sequence is processed in an - explicitly left-to-right manner when generating the output sequence. In - addition, once an input sequence element is attended to at a given output - timestep, elements occurring before it cannot be attended to at subsequent - output timesteps. This function generates attention distributions according - to these assumptions. For more information, see `Online and Linear-Time - Attention by Enforcing Monotonic Alignments`. - - Args: - p_choose_i: Probability of choosing input sequence/memory element i. Should - be of shape (batch_size, input_sequence_length), and should all be in the - range [0, 1]. - previous_attention: The attention distribution from the previous output - timestep. Should be of shape (batch_size, input_sequence_length). For - the first output timestep, preevious_attention[n] should be [1, 0, 0, ..., - 0] for all n in [0, ... batch_size - 1]. - mode: How to compute the attention distribution. Must be one of - 'recursive', 'parallel', or 'hard'. * 'recursive' uses tf.scan to - recursively compute the distribution. This is slowest but is exact, - general, and does not suffer from numerical instabilities. * 'parallel' - uses parallelized cumulative-sum and cumulative-product operations to - compute a closed-form solution to the recurrence relation defining the - attention distribution. This makes it more efficient than 'recursive', - but it requires numerical checks which make the distribution non-exact. - This can be a problem in particular when input_sequence_length is long - and/or p_choose_i has entries very close to 0 or 1. * 'hard' requires that - the probabilities in p_choose_i are all either 0 or 1, and subsequently - uses a more efficient and exact solution. - - Returns: - A tensor of shape (batch_size, input_sequence_length) representing the - attention distributions for each sequence in the batch. - - Raises: - ValueError: mode is not one of 'recursive', 'parallel', 'hard'. - """ - # Force things to be tensors - p_choose_i = ops.convert_to_tensor(p_choose_i, name="p_choose_i") - previous_attention = ops.convert_to_tensor( - previous_attention, name="previous_attention") - if mode == "recursive": - # Use .shape[0] when it's not None, or fall back on symbolic shape - batch_size = tensor_shape.dimension_value( - p_choose_i.shape[0]) or array_ops.shape(p_choose_i)[0] - # Compute [1, 1 - p_choose_i[0], 1 - p_choose_i[1], ..., 1 - p_choose_i[-2]] - shifted_1mp_choose_i = array_ops.concat( - [array_ops.ones((batch_size, 1)), 1 - p_choose_i[:, :-1]], 1) - # Compute attention distribution recursively as - # q[i] = (1 - p_choose_i[i - 1])*q[i - 1] + previous_attention[i] - # attention[i] = p_choose_i[i]*q[i] - attention = p_choose_i * array_ops.transpose( - functional_ops.scan( - # Need to use reshape to remind TF of the shape between loop iterations - lambda x, yz: array_ops.reshape(yz[0] * x + yz[1], (batch_size,)), - # Loop variables yz[0] and yz[1] - [ - array_ops.transpose(shifted_1mp_choose_i), - array_ops.transpose(previous_attention) - ], - # Initial value of x is just zeros - array_ops.zeros((batch_size,)))) - elif mode == "parallel": - # safe_cumprod computes cumprod in logspace with numeric checks - cumprod_1mp_choose_i = safe_cumprod(1 - p_choose_i, axis=1, exclusive=True) - # Compute recurrence relation solution - attention = p_choose_i * cumprod_1mp_choose_i * math_ops.cumsum( - previous_attention / - # Clip cumprod_1mp to avoid divide-by-zero - clip_ops.clip_by_value(cumprod_1mp_choose_i, 1e-10, 1.), - axis=1) - elif mode == "hard": - # Remove any probabilities before the index chosen last time step - p_choose_i *= math_ops.cumsum(previous_attention, axis=1) - # Now, use exclusive cumprod to remove probabilities after the first - # chosen index, like so: - # p_choose_i = [0, 0, 0, 1, 1, 0, 1, 1] - # cumprod(1 - p_choose_i, exclusive=True) = [1, 1, 1, 1, 0, 0, 0, 0] - # Product of above: [0, 0, 0, 1, 0, 0, 0, 0] - attention = p_choose_i * math_ops.cumprod( - 1 - p_choose_i, axis=1, exclusive=True) - else: - raise ValueError("mode must be 'recursive', 'parallel', or 'hard'.") - return attention - - -def _monotonic_probability_fn(score, - previous_alignments, - sigmoid_noise, - mode, - seed=None): - """Attention probability function for monotonic attention. - - Takes in unnormalized attention scores, adds pre-sigmoid noise to encourage - the model to make discrete attention decisions, passes them through a sigmoid - to obtain "choosing" probabilities, and then calls monotonic_attention to - obtain the attention distribution. For more information, see - - Colin Raffel, Minh-Thang Luong, Peter J. Liu, Ron J. Weiss, Douglas Eck, - "Online and Linear-Time Attention by Enforcing Monotonic Alignments." - ICML 2017. https://arxiv.org/abs/1704.00784 - - Args: - score: Unnormalized attention scores, shape `[batch_size, alignments_size]` - previous_alignments: Previous attention distribution, shape `[batch_size, - alignments_size]` - sigmoid_noise: Standard deviation of pre-sigmoid noise. Setting this larger - than 0 will encourage the model to produce large attention scores, - effectively making the choosing probabilities discrete and the resulting - attention distribution one-hot. It should be set to 0 at test-time, and - when hard attention is not desired. - mode: How to compute the attention distribution. Must be one of - 'recursive', 'parallel', or 'hard'. See the docstring for - `tf.contrib.seq2seq.monotonic_attention` for more information. - seed: (optional) Random seed for pre-sigmoid noise. - - Returns: - A `[batch_size, alignments_size]`-shape tensor corresponding to the - resulting attention distribution. - """ - # Optionally add pre-sigmoid noise to the scores - if sigmoid_noise > 0: - noise = random_ops.random_normal( - array_ops.shape(score), dtype=score.dtype, seed=seed) - score += sigmoid_noise * noise - # Compute "choosing" probabilities from the attention scores - if mode == "hard": - # When mode is hard, use a hard sigmoid - p_choose_i = math_ops.cast(score > 0, score.dtype) - else: - p_choose_i = math_ops.sigmoid(score) - # Convert from choosing probabilities to attention distribution - return monotonic_attention(p_choose_i, previous_alignments, mode) - - -class _BaseMonotonicAttentionMechanism(_BaseAttentionMechanism): - """Base attention mechanism for monotonic attention. - - Simply overrides the initial_alignments function to provide a dirac - distribution, which is needed in order for the monotonic attention - distributions to have the correct behavior. - """ - - def initial_alignments(self, batch_size, dtype): - """Creates the initial alignment values for the monotonic attentions. - - Initializes to dirac distributions, i.e. [1, 0, 0, ...memory length..., 0] - for all entries in the batch. - - Args: - batch_size: `int32` scalar, the batch_size. - dtype: The `dtype`. - - Returns: - A `dtype` tensor shaped `[batch_size, alignments_size]` - (`alignments_size` is the values' `max_time`). - """ - max_time = self._alignments_size - return array_ops.one_hot( - array_ops.zeros((batch_size,), dtype=dtypes.int32), - max_time, - dtype=dtype) - - -class _BaseMonotonicAttentionMechanismV2(_BaseAttentionMechanismV2): - """Base attention mechanism for monotonic attention. - - Simply overrides the initial_alignments function to provide a dirac - distribution, which is needed in order for the monotonic attention - distributions to have the correct behavior. - """ - - def initial_alignments(self, batch_size, dtype): - """Creates the initial alignment values for the monotonic attentions. - - Initializes to dirac distributions, i.e. [1, 0, 0, ...memory length..., 0] - for all entries in the batch. - - Args: - batch_size: `int32` scalar, the batch_size. - dtype: The `dtype`. - - Returns: - A `dtype` tensor shaped `[batch_size, alignments_size]` - (`alignments_size` is the values' `max_time`). - """ - max_time = self._alignments_size - return array_ops.one_hot( - array_ops.zeros((batch_size,), dtype=dtypes.int32), - max_time, - dtype=dtype) - - -class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism): - """Monotonic attention mechanism with Bahadanau-style energy function. - - This type of attention enforces a monotonic constraint on the attention - distributions; that is once the model attends to a given point in the memory - it can't attend to any prior points at subsequence output timesteps. It - achieves this by using the _monotonic_probability_fn instead of softmax to - construct its attention distributions. Since the attention scores are passed - through a sigmoid, a learnable scalar bias parameter is applied after the - score function and before the sigmoid. Otherwise, it is equivalent to - BahdanauAttention. This approach is proposed in - - Colin Raffel, Minh-Thang Luong, Peter J. Liu, Ron J. Weiss, Douglas Eck, - "Online and Linear-Time Attention by Enforcing Monotonic Alignments." - ICML 2017. https://arxiv.org/abs/1704.00784 - """ - - def __init__(self, - num_units, - memory, - memory_sequence_length=None, - normalize=False, - score_mask_value=None, - sigmoid_noise=0., - sigmoid_noise_seed=None, - score_bias_init=0., - mode="parallel", - dtype=None, - name="BahdanauMonotonicAttention"): - """Construct the Attention mechanism. - - Args: - num_units: The depth of the query mechanism. - memory: The memory to query; usually the output of an RNN encoder. This - tensor should be shaped `[batch_size, max_time, ...]`. - memory_sequence_length (optional): Sequence lengths for the batch entries - in memory. If provided, the memory tensor rows are masked with zeros - for values past the respective sequence lengths. - normalize: Python boolean. Whether to normalize the energy term. - score_mask_value: (optional): The mask value for score before passing into - `probability_fn`. The default is -inf. Only used if - `memory_sequence_length` is not None. - sigmoid_noise: Standard deviation of pre-sigmoid noise. See the docstring - for `_monotonic_probability_fn` for more information. - sigmoid_noise_seed: (optional) Random seed for pre-sigmoid noise. - score_bias_init: Initial value for score bias scalar. It's recommended to - initialize this to a negative value when the length of the memory is - large. - mode: How to compute the attention distribution. Must be one of - 'recursive', 'parallel', or 'hard'. See the docstring for - `tf.contrib.seq2seq.monotonic_attention` for more information. - dtype: The data type for the query and memory layers of the attention - mechanism. - name: Name to use when creating ops. - """ - # Set up the monotonic probability fn with supplied parameters - if dtype is None: - dtype = dtypes.float32 - wrapped_probability_fn = functools.partial( - _monotonic_probability_fn, - sigmoid_noise=sigmoid_noise, - mode=mode, - seed=sigmoid_noise_seed) - super(BahdanauMonotonicAttention, self).__init__( - query_layer=layers_core.Dense( - num_units, name="query_layer", use_bias=False, dtype=dtype), - memory_layer=layers_core.Dense( - num_units, name="memory_layer", use_bias=False, dtype=dtype), - memory=memory, - probability_fn=wrapped_probability_fn, - memory_sequence_length=memory_sequence_length, - score_mask_value=score_mask_value, - name=name) - self._num_units = num_units - self._normalize = normalize - self._name = name - self._score_bias_init = score_bias_init - - def __call__(self, query, state): - """Score the query based on the keys and values. - - Args: - query: Tensor of dtype matching `self.values` and shape `[batch_size, - query_depth]`. - state: Tensor of dtype matching `self.values` and shape `[batch_size, - alignments_size]` (`alignments_size` is memory's `max_time`). - - Returns: - alignments: Tensor of dtype matching `self.values` and shape - `[batch_size, alignments_size]` (`alignments_size` is memory's - `max_time`). - """ - with variable_scope.variable_scope(None, "bahdanau_monotonic_attention", - [query]): - processed_query = self.query_layer(query) if self.query_layer else query - attention_v = variable_scope.get_variable( - "attention_v", [self._num_units], dtype=query.dtype) - if not self._normalize: - attention_g = None - attention_b = None - else: - attention_g = variable_scope.get_variable( - "attention_g", - dtype=query.dtype, - initializer=init_ops.constant_initializer( - math.sqrt((1. / self._num_units))), - shape=()) - attention_b = variable_scope.get_variable( - "attention_b", [self._num_units], - dtype=query.dtype, - initializer=init_ops.zeros_initializer()) - score = _bahdanau_score( - processed_query, - self._keys, - attention_v, - attention_g=attention_g, - attention_b=attention_b) - score_bias = variable_scope.get_variable( - "attention_score_bias", - dtype=processed_query.dtype, - initializer=self._score_bias_init) - score += score_bias - alignments = self._probability_fn(score, state) - next_state = alignments - return alignments, next_state - - -class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2): - """Monotonic attention mechanism with Bahadanau-style energy function. - - This type of attention enforces a monotonic constraint on the attention - distributions; that is once the model attends to a given point in the memory - it can't attend to any prior points at subsequence output timesteps. It - achieves this by using the _monotonic_probability_fn instead of softmax to - construct its attention distributions. Since the attention scores are passed - through a sigmoid, a learnable scalar bias parameter is applied after the - score function and before the sigmoid. Otherwise, it is equivalent to - BahdanauAttention. This approach is proposed in - - Colin Raffel, Minh-Thang Luong, Peter J. Liu, Ron J. Weiss, Douglas Eck, - "Online and Linear-Time Attention by Enforcing Monotonic Alignments." - ICML 2017. https://arxiv.org/abs/1704.00784 - """ - - def __init__(self, - units, - memory, - memory_sequence_length=None, - normalize=False, - sigmoid_noise=0., - sigmoid_noise_seed=None, - score_bias_init=0., - mode="parallel", - kernel_initializer="glorot_uniform", - dtype=None, - name="BahdanauMonotonicAttention", - **kwargs): - """Construct the Attention mechanism. - - Args: - units: The depth of the query mechanism. - memory: The memory to query; usually the output of an RNN encoder. This - tensor should be shaped `[batch_size, max_time, ...]`. - memory_sequence_length: (optional): Sequence lengths for the batch entries - in memory. If provided, the memory tensor rows are masked with zeros - for values past the respective sequence lengths. - normalize: Python boolean. Whether to normalize the energy term. - sigmoid_noise: Standard deviation of pre-sigmoid noise. See the docstring - for `_monotonic_probability_fn` for more information. - sigmoid_noise_seed: (optional) Random seed for pre-sigmoid noise. - score_bias_init: Initial value for score bias scalar. It's recommended to - initialize this to a negative value when the length of the memory is - large. - mode: How to compute the attention distribution. Must be one of - 'recursive', 'parallel', or 'hard'. See the docstring for - `tf.contrib.seq2seq.monotonic_attention` for more information. - kernel_initializer: (optional), the name of the initializer for the - attention kernel. - dtype: The data type for the query and memory layers of the attention - mechanism. - name: Name to use when creating ops. - **kwargs: Dictionary that contains other common arguments for layer - creation. - """ - # Set up the monotonic probability fn with supplied parameters - if dtype is None: - dtype = dtypes.float32 - wrapped_probability_fn = functools.partial( - _monotonic_probability_fn, - sigmoid_noise=sigmoid_noise, - mode=mode, - seed=sigmoid_noise_seed) - query_layer = kwargs.pop("query_layer", None) - if not query_layer: - query_layer = layers.Dense( - units, name="query_layer", use_bias=False, dtype=dtype) - memory_layer = kwargs.pop("memory_layer", None) - if not memory_layer: - memory_layer = layers.Dense( - units, name="memory_layer", use_bias=False, dtype=dtype) - self.units = units - self.normalize = normalize - self.sigmoid_noise = sigmoid_noise - self.sigmoid_noise_seed = sigmoid_noise_seed - self.score_bias_init = score_bias_init - self.mode = mode - self.kernel_initializer = initializers.get(kernel_initializer) - self.attention_v = None - self.attention_score_bias = None - self.attention_g = None - self.attention_b = None - super(BahdanauMonotonicAttentionV2, self).__init__( - memory=memory, - memory_sequence_length=memory_sequence_length, - query_layer=query_layer, - memory_layer=memory_layer, - probability_fn=wrapped_probability_fn, - name=name, - dtype=dtype, - **kwargs) - - def build(self, input_shape): - super(BahdanauMonotonicAttentionV2, self).build(input_shape) - if self.attention_v is None: - self.attention_v = self.add_weight( - "attention_v", [self.units], - dtype=self.dtype, - initializer=self.kernel_initializer) - if self.attention_score_bias is None: - self.attention_score_bias = self.add_weight( - "attention_score_bias", - shape=(), - dtype=self.dtype, - initializer=init_ops.constant_initializer( - self.score_bias_init, dtype=self.dtype)) - if self.normalize and self.attention_g is None and self.attention_b is None: - self.attention_g = self.add_weight( - "attention_g", - dtype=self.dtype, - initializer=init_ops.constant_initializer( - math.sqrt((1. / self.units))), - shape=()) - self.attention_b = self.add_weight( - "attention_b", [self.units], - dtype=self.dtype, - initializer=init_ops.zeros_initializer()) - self.built = True - - def _calculate_attention(self, query, state): - """Score the query based on the keys and values. - - Args: - query: Tensor of dtype matching `self.values` and shape `[batch_size, - query_depth]`. - state: Tensor of dtype matching `self.values` and shape `[batch_size, - alignments_size]` (`alignments_size` is memory's `max_time`). - - Returns: - alignments: Tensor of dtype matching `self.values` and shape - `[batch_size, alignments_size]` (`alignments_size` is memory's - `max_time`). - """ - processed_query = self.query_layer(query) if self.query_layer else query - score = _bahdanau_score( - processed_query, - self.keys, - self.attention_v, - attention_g=self.attention_g, - attention_b=self.attention_b) - score += self.attention_score_bias - alignments = self.probability_fn(score, state) - next_state = alignments - return alignments, next_state - - def get_config(self): - config = { - "units": self.units, - "normalize": self.normalize, - "sigmoid_noise": self.sigmoid_noise, - "sigmoid_noise_seed": self.sigmoid_noise_seed, - "score_bias_init": self.score_bias_init, - "mode": self.mode, - "kernel_initializer": initializers.serialize(self.kernel_initializer), - } - base_config = super(BahdanauMonotonicAttentionV2, self).get_config() - return dict(list(base_config.items()) + list(config.items())) - - @classmethod - def from_config(cls, config, custom_objects=None): - config = _BaseAttentionMechanismV2.deserialize_inner_layer_from_config( - config, custom_objects=custom_objects) - return cls(**config) - - -class LuongMonotonicAttention(_BaseMonotonicAttentionMechanism): - """Monotonic attention mechanism with Luong-style energy function. - - This type of attention enforces a monotonic constraint on the attention - distributions; that is once the model attends to a given point in the memory - it can't attend to any prior points at subsequence output timesteps. It - achieves this by using the _monotonic_probability_fn instead of softmax to - construct its attention distributions. Otherwise, it is equivalent to - LuongAttention. This approach is proposed in - - Colin Raffel, Minh-Thang Luong, Peter J. Liu, Ron J. Weiss, Douglas Eck, - "Online and Linear-Time Attention by Enforcing Monotonic Alignments." - ICML 2017. https://arxiv.org/abs/1704.00784 - """ - - def __init__(self, - num_units, - memory, - memory_sequence_length=None, - scale=False, - score_mask_value=None, - sigmoid_noise=0., - sigmoid_noise_seed=None, - score_bias_init=0., - mode="parallel", - dtype=None, - name="LuongMonotonicAttention"): - """Construct the Attention mechanism. - - Args: - num_units: The depth of the query mechanism. - memory: The memory to query; usually the output of an RNN encoder. This - tensor should be shaped `[batch_size, max_time, ...]`. - memory_sequence_length (optional): Sequence lengths for the batch entries - in memory. If provided, the memory tensor rows are masked with zeros - for values past the respective sequence lengths. - scale: Python boolean. Whether to scale the energy term. - score_mask_value: (optional): The mask value for score before passing into - `probability_fn`. The default is -inf. Only used if - `memory_sequence_length` is not None. - sigmoid_noise: Standard deviation of pre-sigmoid noise. See the docstring - for `_monotonic_probability_fn` for more information. - sigmoid_noise_seed: (optional) Random seed for pre-sigmoid noise. - score_bias_init: Initial value for score bias scalar. It's recommended to - initialize this to a negative value when the length of the memory is - large. - mode: How to compute the attention distribution. Must be one of - 'recursive', 'parallel', or 'hard'. See the docstring for - `tf.contrib.seq2seq.monotonic_attention` for more information. - dtype: The data type for the query and memory layers of the attention - mechanism. - name: Name to use when creating ops. - """ - # Set up the monotonic probability fn with supplied parameters - if dtype is None: - dtype = dtypes.float32 - wrapped_probability_fn = functools.partial( - _monotonic_probability_fn, - sigmoid_noise=sigmoid_noise, - mode=mode, - seed=sigmoid_noise_seed) - super(LuongMonotonicAttention, self).__init__( - query_layer=None, - memory_layer=layers_core.Dense( - num_units, name="memory_layer", use_bias=False, dtype=dtype), - memory=memory, - probability_fn=wrapped_probability_fn, - memory_sequence_length=memory_sequence_length, - score_mask_value=score_mask_value, - name=name) - self._num_units = num_units - self._scale = scale - self._score_bias_init = score_bias_init - self._name = name - - def __call__(self, query, state): - """Score the query based on the keys and values. - - Args: - query: Tensor of dtype matching `self.values` and shape `[batch_size, - query_depth]`. - state: Tensor of dtype matching `self.values` and shape `[batch_size, - alignments_size]` (`alignments_size` is memory's `max_time`). - - Returns: - alignments: Tensor of dtype matching `self.values` and shape - `[batch_size, alignments_size]` (`alignments_size` is memory's - `max_time`). - """ - with variable_scope.variable_scope(None, "luong_monotonic_attention", - [query]): - attention_g = None - if self._scale: - attention_g = variable_scope.get_variable( - "attention_g", - dtype=query.dtype, - initializer=init_ops.ones_initializer, - shape=()) - score = _luong_score(query, self._keys, attention_g) - score_bias = variable_scope.get_variable( - "attention_score_bias", - dtype=query.dtype, - initializer=self._score_bias_init) - score += score_bias - alignments = self._probability_fn(score, state) - next_state = alignments - return alignments, next_state - - -class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2): - """Monotonic attention mechanism with Luong-style energy function. - - This type of attention enforces a monotonic constraint on the attention - distributions; that is once the model attends to a given point in the memory - it can't attend to any prior points at subsequence output timesteps. It - achieves this by using the _monotonic_probability_fn instead of softmax to - construct its attention distributions. Otherwise, it is equivalent to - LuongAttention. This approach is proposed in - - [Colin Raffel, Minh-Thang Luong, Peter J. Liu, Ron J. Weiss, Douglas Eck, - "Online and Linear-Time Attention by Enforcing Monotonic Alignments." - ICML 2017.](https://arxiv.org/abs/1704.00784) - """ - - def __init__(self, - units, - memory, - memory_sequence_length=None, - scale=False, - sigmoid_noise=0., - sigmoid_noise_seed=None, - score_bias_init=0., - mode="parallel", - dtype=None, - name="LuongMonotonicAttention", - **kwargs): - """Construct the Attention mechanism. - - Args: - units: The depth of the query mechanism. - memory: The memory to query; usually the output of an RNN encoder. This - tensor should be shaped `[batch_size, max_time, ...]`. - memory_sequence_length: (optional): Sequence lengths for the batch entries - in memory. If provided, the memory tensor rows are masked with zeros - for values past the respective sequence lengths. - scale: Python boolean. Whether to scale the energy term. - sigmoid_noise: Standard deviation of pre-sigmoid noise. See the docstring - for `_monotonic_probability_fn` for more information. - sigmoid_noise_seed: (optional) Random seed for pre-sigmoid noise. - score_bias_init: Initial value for score bias scalar. It's recommended to - initialize this to a negative value when the length of the memory is - large. - mode: How to compute the attention distribution. Must be one of - 'recursive', 'parallel', or 'hard'. See the docstring for - `tf.contrib.seq2seq.monotonic_attention` for more information. - dtype: The data type for the query and memory layers of the attention - mechanism. - name: Name to use when creating ops. - **kwargs: Dictionary that contains other common arguments for layer - creation. - """ - # Set up the monotonic probability fn with supplied parameters - if dtype is None: - dtype = dtypes.float32 - wrapped_probability_fn = functools.partial( - _monotonic_probability_fn, - sigmoid_noise=sigmoid_noise, - mode=mode, - seed=sigmoid_noise_seed) - memory_layer = kwargs.pop("memory_layer", None) - if not memory_layer: - memory_layer = layers.Dense( - units, name="memory_layer", use_bias=False, dtype=dtype) - self.units = units - self.scale = scale - self.sigmoid_noise = sigmoid_noise - self.sigmoid_noise_seed = sigmoid_noise_seed - self.score_bias_init = score_bias_init - self.mode = mode - self.attention_g = None - self.attention_score_bias = None - super(LuongMonotonicAttentionV2, self).__init__( - memory=memory, - memory_sequence_length=memory_sequence_length, - query_layer=None, - memory_layer=memory_layer, - probability_fn=wrapped_probability_fn, - name=name, - dtype=dtype, - **kwargs) - - def build(self, input_shape): - super(LuongMonotonicAttentionV2, self).build(input_shape) - if self.scale and self.attention_g is None: - self.attention_g = self.add_weight( - "attention_g", initializer=init_ops.ones_initializer, shape=()) - if self.attention_score_bias is None: - self.attention_score_bias = self.add_weight( - "attention_score_bias", - shape=(), - initializer=init_ops.constant_initializer( - self.score_bias_init, dtype=self.dtype)) - self.built = True - - def _calculate_attention(self, query, state): - """Score the query based on the keys and values. - - Args: - query: Tensor of dtype matching `self.values` and shape `[batch_size, - query_depth]`. - state: Tensor of dtype matching `self.values` and shape `[batch_size, - alignments_size]` (`alignments_size` is memory's `max_time`). - - Returns: - alignments: Tensor of dtype matching `self.values` and shape - `[batch_size, alignments_size]` (`alignments_size` is memory's - `max_time`). - next_state: Same as alignments - """ - score = _luong_score(query, self.keys, self.attention_g) - score += self.attention_score_bias - alignments = self.probability_fn(score, state) - next_state = alignments - return alignments, next_state - - def get_config(self): - config = { - "units": self.units, - "scale": self.scale, - "sigmoid_noise": self.sigmoid_noise, - "sigmoid_noise_seed": self.sigmoid_noise_seed, - "score_bias_init": self.score_bias_init, - "mode": self.mode, - } - base_config = super(LuongMonotonicAttentionV2, self).get_config() - return dict(list(base_config.items()) + list(config.items())) - - @classmethod - def from_config(cls, config, custom_objects=None): - config = _BaseAttentionMechanismV2.deserialize_inner_layer_from_config( - config, custom_objects=custom_objects) - return cls(**config) - - -class AttentionWrapperState( - collections.namedtuple("AttentionWrapperState", - ("cell_state", "attention", "time", "alignments", - "alignment_history", "attention_state"))): - """`namedtuple` storing the state of a `AttentionWrapper`. - - Contains: - - - `cell_state`: The state of the wrapped `RNNCell` at the previous time - step. - - `attention`: The attention emitted at the previous time step. - - `time`: int32 scalar containing the current time step. - - `alignments`: A single or tuple of `Tensor`(s) containing the alignments - emitted at the previous time step for each attention mechanism. - - `alignment_history`: (if enabled) a single or tuple of `TensorArray`(s) - containing alignment matrices from all time steps for each attention - mechanism. Call `stack()` on each to convert to a `Tensor`. - - `attention_state`: A single or tuple of nested objects - containing attention mechanism state for each attention mechanism. - The objects may contain Tensors or TensorArrays. - """ - - def clone(self, **kwargs): - """Clone this object, overriding components provided by kwargs. - - The new state fields' shape must match original state fields' shape. This - will be validated, and original fields' shape will be propagated to new - fields. - - Example: - - ```python - initial_state = attention_wrapper.zero_state(dtype=..., batch_size=...) - initial_state = initial_state.clone(cell_state=encoder_state) - ``` - - Args: - **kwargs: Any properties of the state object to replace in the returned - `AttentionWrapperState`. - - Returns: - A new `AttentionWrapperState` whose properties are the same as - this one, except any overridden properties as provided in `kwargs`. - """ - - def with_same_shape(old, new): - """Check and set new tensor's shape.""" - if isinstance(old, ops.Tensor) and isinstance(new, ops.Tensor): - if not context.executing_eagerly(): - return tensor_util.with_same_shape(old, new) - else: - if old.shape.as_list() != new.shape.as_list(): - raise ValueError("The shape of the AttentionWrapperState is " - "expected to be same as the one to clone. " - "self.shape: %s, input.shape: %s" % - (old.shape, new.shape)) - return new - return new - - return nest.map_structure( - with_same_shape, self, - super(AttentionWrapperState, self)._replace(**kwargs)) - - -def _prepare_memory(memory, - memory_sequence_length=None, - memory_mask=None, - check_inner_dims_defined=True): - """Convert to tensor and possibly mask `memory`. - - Args: - memory: `Tensor`, shaped `[batch_size, max_time, ...]`. - memory_sequence_length: `int32` `Tensor`, shaped `[batch_size]`. - memory_mask: `boolean` tensor with shape [batch_size, max_time]. The memory - should be skipped when the corresponding mask is False. - check_inner_dims_defined: Python boolean. If `True`, the `memory` - argument's shape is checked to ensure all but the two outermost dimensions - are fully defined. - - Returns: - A (possibly masked), checked, new `memory`. - - Raises: - ValueError: If `check_inner_dims_defined` is `True` and not - `memory.shape[2:].is_fully_defined()`. - """ - memory = nest.map_structure(lambda m: ops.convert_to_tensor(m, name="memory"), - memory) - if memory_sequence_length is not None and memory_mask is not None: - raise ValueError("memory_sequence_length and memory_mask can't be provided " - "at same time.") - if memory_sequence_length is not None: - memory_sequence_length = ops.convert_to_tensor( - memory_sequence_length, name="memory_sequence_length") - if check_inner_dims_defined: - - def _check_dims(m): - if not m.get_shape()[2:].is_fully_defined(): - raise ValueError("Expected memory %s to have fully defined inner dims, " - "but saw shape: %s" % (m.name, m.get_shape())) - - nest.map_structure(_check_dims, memory) - if memory_sequence_length is None and memory_mask is None: - return memory - elif memory_sequence_length is not None: - seq_len_mask = array_ops.sequence_mask( - memory_sequence_length, - maxlen=array_ops.shape(nest.flatten(memory)[0])[1], - dtype=nest.flatten(memory)[0].dtype) - else: - # For memory_mask is not None - seq_len_mask = math_ops.cast( - memory_mask, dtype=nest.flatten(memory)[0].dtype) - - def _maybe_mask(m, seq_len_mask): - """Mask the memory based on the memory mask.""" - rank = m.get_shape().ndims - rank = rank if rank is not None else array_ops.rank(m) - extra_ones = array_ops.ones(rank - 2, dtype=dtypes.int32) - seq_len_mask = array_ops.reshape( - seq_len_mask, - array_ops.concat((array_ops.shape(seq_len_mask), extra_ones), 0)) - return m * seq_len_mask - - return nest.map_structure(lambda m: _maybe_mask(m, seq_len_mask), memory) - - -def _maybe_mask_score(score, - memory_sequence_length=None, - memory_mask=None, - score_mask_value=None): - """Mask the attention score based on the masks.""" - if memory_sequence_length is None and memory_mask is None: - return score - if memory_sequence_length is not None and memory_mask is not None: - raise ValueError("memory_sequence_length and memory_mask can't be provided " - "at same time.") - if memory_sequence_length is not None: - message = "All values in memory_sequence_length must be greater than zero." - with ops.control_dependencies( - [check_ops.assert_positive(memory_sequence_length, message=message)]): - memory_mask = array_ops.sequence_mask( - memory_sequence_length, maxlen=array_ops.shape(score)[1]) - score_mask_values = score_mask_value * array_ops.ones_like(score) - return array_ops.where(memory_mask, score, score_mask_values) - - -def hardmax(logits, name=None): - """Returns batched one-hot vectors. - - The depth index containing the `1` is that of the maximum logit value. - - Args: - logits: A batch tensor of logit values. - name: Name to use when creating ops. - - Returns: - A batched one-hot tensor. - """ - with ops.name_scope(name, "Hardmax", [logits]): - logits = ops.convert_to_tensor(logits, name="logits") - if tensor_shape.dimension_value(logits.get_shape()[-1]) is not None: - depth = tensor_shape.dimension_value(logits.get_shape()[-1]) - else: - depth = array_ops.shape(logits)[-1] - return array_ops.one_hot( - math_ops.argmax(logits, -1), depth, dtype=logits.dtype) - - -def _compute_attention(attention_mechanism, cell_output, attention_state, - attention_layer): - """Computes the attention and alignments for a given attention_mechanism.""" - if isinstance(attention_mechanism, _BaseAttentionMechanismV2): - alignments, next_attention_state = attention_mechanism( - [cell_output, attention_state]) - else: - # For other class, assume they are following _BaseAttentionMechanism, which - # takes query and state as separate parameter. - alignments, next_attention_state = attention_mechanism( - cell_output, state=attention_state) - - # Reshape from [batch_size, memory_time] to [batch_size, 1, memory_time] - expanded_alignments = array_ops.expand_dims(alignments, 1) - # Context is the inner product of alignments and values along the - # memory time dimension. - # alignments shape is - # [batch_size, 1, memory_time] - # attention_mechanism.values shape is - # [batch_size, memory_time, memory_size] - # the batched matmul is over memory_time, so the output shape is - # [batch_size, 1, memory_size]. - # we then squeeze out the singleton dim. - context_ = math_ops.matmul(expanded_alignments, attention_mechanism.values) - context_ = array_ops.squeeze(context_, [1]) - - if attention_layer is not None: - attention = attention_layer(array_ops.concat([cell_output, context_], 1)) - else: - attention = context_ - - return attention, alignments, next_attention_state - - -class AttentionWrapper(rnn_cell_impl.RNNCell): - """Wraps another `RNNCell` with attention.""" - - def __init__(self, - cell, - attention_mechanism, - attention_layer_size=None, - alignment_history=False, - cell_input_fn=None, - output_attention=True, - initial_cell_state=None, - name=None, - attention_layer=None, - attention_fn=None, - dtype=None): - """Construct the `AttentionWrapper`. - - **NOTE** If you are using the `BeamSearchDecoder` with a cell wrapped in - `AttentionWrapper`, then you must ensure that: - - - The encoder output has been tiled to `beam_width` via - `tf.contrib.seq2seq.tile_batch` (NOT `tf.tile`). - - The `batch_size` argument passed to the `zero_state` method of this - wrapper is equal to `true_batch_size * beam_width`. - - The initial state created with `zero_state` above contains a - `cell_state` value containing properly tiled final state from the - encoder. - - An example: - - ``` - tiled_encoder_outputs = tf.contrib.seq2seq.tile_batch( - encoder_outputs, multiplier=beam_width) - tiled_encoder_final_state = tf.conrib.seq2seq.tile_batch( - encoder_final_state, multiplier=beam_width) - tiled_sequence_length = tf.contrib.seq2seq.tile_batch( - sequence_length, multiplier=beam_width) - attention_mechanism = MyFavoriteAttentionMechanism( - num_units=attention_depth, - memory=tiled_inputs, - memory_sequence_length=tiled_sequence_length) - attention_cell = AttentionWrapper(cell, attention_mechanism, ...) - decoder_initial_state = attention_cell.zero_state( - dtype, batch_size=true_batch_size * beam_width) - decoder_initial_state = decoder_initial_state.clone( - cell_state=tiled_encoder_final_state) - ``` - - Args: - cell: An instance of `RNNCell`. - attention_mechanism: A list of `AttentionMechanism` instances or a single - instance. - attention_layer_size: A list of Python integers or a single Python - integer, the depth of the attention (output) layer(s). If None - (default), use the context as attention at each time step. Otherwise, - feed the context and cell output into the attention layer to generate - attention at each time step. If attention_mechanism is a list, - attention_layer_size must be a list of the same length. If - attention_layer is set, this must be None. If attention_fn is set, it - must guaranteed that the outputs of attention_fn also meet the above - requirements. - alignment_history: Python boolean, whether to store alignment history from - all time steps in the final output state (currently stored as a time - major `TensorArray` on which you must call `stack()`). - cell_input_fn: (optional) A `callable`. The default is: - `lambda inputs, attention: array_ops.concat([inputs, attention], -1)`. - output_attention: Python bool. If `True` (default), the output at each - time step is the attention value. This is the behavior of Luong-style - attention mechanisms. If `False`, the output at each time step is the - output of `cell`. This is the behavior of Bhadanau-style attention - mechanisms. In both cases, the `attention` tensor is propagated to the - next time step via the state and is used there. This flag only controls - whether the attention mechanism is propagated up to the next cell in an - RNN stack or to the top RNN output. - initial_cell_state: The initial state value to use for the cell when the - user calls `zero_state()`. Note that if this value is provided now, and - the user uses a `batch_size` argument of `zero_state` which does not - match the batch size of `initial_cell_state`, proper behavior is not - guaranteed. - name: Name to use when creating ops. - attention_layer: A list of `tf.compat.v1.layers.Layer` instances or a - single `tf.compat.v1.layers.Layer` instance taking the context and cell - output as inputs to generate attention at each time step. If None - (default), use the context as attention at each time step. If - attention_mechanism is a list, attention_layer must be a list of the - same length. If attention_layers_size is set, this must be None. - attention_fn: An optional callable function that allows users to provide - their own customized attention function, which takes input - (attention_mechanism, cell_output, attention_state, attention_layer) and - outputs (attention, alignments, next_attention_state). If provided, the - attention_layer_size should be the size of the outputs of attention_fn. - dtype: The cell dtype - - Raises: - TypeError: `attention_layer_size` is not None and (`attention_mechanism` - is a list but `attention_layer_size` is not; or vice versa). - ValueError: if `attention_layer_size` is not None, `attention_mechanism` - is a list, and its length does not match that of `attention_layer_size`; - if `attention_layer_size` and `attention_layer` are set simultaneously. - """ - super(AttentionWrapper, self).__init__(name=name, dtype=dtype) - rnn_cell_impl.assert_like_rnncell("cell", cell) - if isinstance(attention_mechanism, (list, tuple)): - self._is_multi = True - attention_mechanisms = attention_mechanism - for attention_mechanism in attention_mechanisms: - if not isinstance(attention_mechanism, AttentionMechanism): - raise TypeError("attention_mechanism must contain only instances of " - "AttentionMechanism, saw type: %s" % - type(attention_mechanism).__name__) - else: - self._is_multi = False - if not isinstance(attention_mechanism, AttentionMechanism): - raise TypeError( - "attention_mechanism must be an AttentionMechanism or list of " - "multiple AttentionMechanism instances, saw type: %s" % - type(attention_mechanism).__name__) - attention_mechanisms = (attention_mechanism,) - - if cell_input_fn is None: - cell_input_fn = ( - lambda inputs, attention: array_ops.concat([inputs, attention], -1)) - else: - if not callable(cell_input_fn): - raise TypeError("cell_input_fn must be callable, saw type: %s" % - type(cell_input_fn).__name__) - - if attention_layer_size is not None and attention_layer is not None: - raise ValueError("Only one of attention_layer_size and attention_layer " - "should be set") - - if attention_layer_size is not None: - attention_layer_sizes = tuple( - attention_layer_size if isinstance(attention_layer_size, ( - list, tuple)) else (attention_layer_size,)) - if len(attention_layer_sizes) != len(attention_mechanisms): - raise ValueError( - "If provided, attention_layer_size must contain exactly one " - "integer per attention_mechanism, saw: %d vs %d" % - (len(attention_layer_sizes), len(attention_mechanisms))) - self._attention_layers = tuple( - layers_core.Dense( - attention_layer_size, - name="attention_layer", - use_bias=False, - dtype=attention_mechanisms[i].dtype) - for i, attention_layer_size in enumerate(attention_layer_sizes)) - self._attention_layer_size = sum(attention_layer_sizes) - elif attention_layer is not None: - self._attention_layers = tuple( - attention_layer if isinstance(attention_layer, (list, tuple)) else ( - attention_layer,)) - if len(self._attention_layers) != len(attention_mechanisms): - raise ValueError( - "If provided, attention_layer must contain exactly one " - "layer per attention_mechanism, saw: %d vs %d" % - (len(self._attention_layers), len(attention_mechanisms))) - self._attention_layer_size = sum( - tensor_shape.dimension_value( - layer.compute_output_shape([ - None, cell.output_size + - tensor_shape.dimension_value(mechanism.values.shape[-1]) - ])[-1]) for layer, mechanism in zip(self._attention_layers, - attention_mechanisms)) - else: - self._attention_layers = None - self._attention_layer_size = sum( - tensor_shape.dimension_value(attention_mechanism.values.shape[-1]) - for attention_mechanism in attention_mechanisms) - - if attention_fn is None: - attention_fn = _compute_attention - self._attention_fn = attention_fn - - self._cell = cell - self._attention_mechanisms = attention_mechanisms - self._cell_input_fn = cell_input_fn - self._output_attention = output_attention - self._alignment_history = alignment_history - with ops.name_scope(name, "AttentionWrapperInit"): - if initial_cell_state is None: - self._initial_cell_state = None - else: - final_state_tensor = nest.flatten(initial_cell_state)[-1] - state_batch_size = ( - tensor_shape.dimension_value(final_state_tensor.shape[0]) or - array_ops.shape(final_state_tensor)[0]) - error_message = ( - "When constructing AttentionWrapper %s: " % self._base_name + - "Non-matching batch sizes between the memory " - "(encoder output) and initial_cell_state. Are you using " - "the BeamSearchDecoder? You may need to tile your initial state " - "via the tf.contrib.seq2seq.tile_batch function with argument " - "multiple=beam_width.") - with ops.control_dependencies( - self._batch_size_checks(state_batch_size, error_message)): - self._initial_cell_state = nest.map_structure( - lambda s: array_ops.identity(s, name="check_initial_cell_state"), - initial_cell_state) - - def _batch_size_checks(self, batch_size, error_message): - return [ - check_ops.assert_equal( - batch_size, attention_mechanism.batch_size, message=error_message) - for attention_mechanism in self._attention_mechanisms - ] - - def _item_or_tuple(self, seq): - """Returns `seq` as tuple or the singular element. - - Which is returned is determined by how the AttentionMechanism(s) were passed - to the constructor. - - Args: - seq: A non-empty sequence of items or generator. - - Returns: - Either the values in the sequence as a tuple if AttentionMechanism(s) - were passed to the constructor as a sequence or the singular element. - """ - t = tuple(seq) - if self._is_multi: - return t - else: - return t[0] - - @property - def output_size(self): - if self._output_attention: - return self._attention_layer_size - else: - return self._cell.output_size - - @property - def state_size(self): - """The `state_size` property of `AttentionWrapper`. - - Returns: - An `AttentionWrapperState` tuple containing shapes used by this object. - """ - return AttentionWrapperState( - cell_state=self._cell.state_size, - time=tensor_shape.TensorShape([]), - attention=self._attention_layer_size, - alignments=self._item_or_tuple( - a.alignments_size for a in self._attention_mechanisms), - attention_state=self._item_or_tuple( - a.state_size for a in self._attention_mechanisms), - alignment_history=self._item_or_tuple( - a.alignments_size if self._alignment_history else () - for a in self._attention_mechanisms)) # sometimes a TensorArray - - def zero_state(self, batch_size, dtype): - """Return an initial (zero) state tuple for this `AttentionWrapper`. - - **NOTE** Please see the initializer documentation for details of how - to call `zero_state` if using an `AttentionWrapper` with a - `BeamSearchDecoder`. - - Args: - batch_size: `0D` integer tensor: the batch size. - dtype: The internal state data type. - - Returns: - An `AttentionWrapperState` tuple containing zeroed out tensors and, - possibly, empty `TensorArray` objects. - - Raises: - ValueError: (or, possibly at runtime, InvalidArgument), if - `batch_size` does not match the output size of the encoder passed - to the wrapper object at initialization time. - """ - with ops.name_scope(type(self).__name__ + "ZeroState", values=[batch_size]): - if self._initial_cell_state is not None: - cell_state = self._initial_cell_state - else: - cell_state = self._cell.get_initial_state( - batch_size=batch_size, dtype=dtype) - error_message = ( - "When calling zero_state of AttentionWrapper %s: " % self._base_name + - "Non-matching batch sizes between the memory " - "(encoder output) and the requested batch size. Are you using " - "the BeamSearchDecoder? If so, make sure your encoder output has " - "been tiled to beam_width via tf.contrib.seq2seq.tile_batch, and " - "the batch_size= argument passed to zero_state is " - "batch_size * beam_width.") - with ops.control_dependencies( - self._batch_size_checks(batch_size, error_message)): - cell_state = nest.map_structure( - lambda s: array_ops.identity(s, name="checked_cell_state"), - cell_state) - initial_alignments = [ - attention_mechanism.initial_alignments(batch_size, dtype) - for attention_mechanism in self._attention_mechanisms - ] - return AttentionWrapperState( - cell_state=cell_state, - time=array_ops.zeros([], dtype=dtypes.int32), - attention=_zero_state_tensors(self._attention_layer_size, batch_size, - dtype), - alignments=self._item_or_tuple(initial_alignments), - attention_state=self._item_or_tuple( - attention_mechanism.initial_state(batch_size, dtype) - for attention_mechanism in self._attention_mechanisms), - alignment_history=self._item_or_tuple( - tensor_array_ops.TensorArray( - dtype, - size=0, - dynamic_size=True, - element_shape=alignment.shape) if self._alignment_history else - () for alignment in initial_alignments)) - - def call(self, inputs, state): - """Perform a step of attention-wrapped RNN. - - - Step 1: Mix the `inputs` and previous step's `attention` output via - `cell_input_fn`. - - Step 2: Call the wrapped `cell` with this input and its previous state. - - Step 3: Score the cell's output with `attention_mechanism`. - - Step 4: Calculate the alignments by passing the score through the - `normalizer`. - - Step 5: Calculate the context vector as the inner product between the - alignments and the attention_mechanism's values (memory). - - Step 6: Calculate the attention output by concatenating the cell output - and context through the attention layer (a linear layer with - `attention_layer_size` outputs). - - Args: - inputs: (Possibly nested tuple of) Tensor, the input at this time step. - state: An instance of `AttentionWrapperState` containing tensors from the - previous time step. - - Returns: - A tuple `(attention_or_cell_output, next_state)`, where: - - - `attention_or_cell_output` depending on `output_attention`. - - `next_state` is an instance of `AttentionWrapperState` - containing the state calculated at this time step. - - Raises: - TypeError: If `state` is not an instance of `AttentionWrapperState`. - """ - if not isinstance(state, AttentionWrapperState): - raise TypeError("Expected state to be instance of AttentionWrapperState. " - "Received type %s instead." % type(state)) - - # Step 1: Calculate the true inputs to the cell based on the - # previous attention value. - cell_inputs = self._cell_input_fn(inputs, state.attention) - cell_state = state.cell_state - cell_output, next_cell_state = self._cell(cell_inputs, cell_state) - - cell_batch_size = ( - tensor_shape.dimension_value(cell_output.shape[0]) or - array_ops.shape(cell_output)[0]) - error_message = ( - "When applying AttentionWrapper %s: " % self.name + - "Non-matching batch sizes between the memory " - "(encoder output) and the query (decoder output). Are you using " - "the BeamSearchDecoder? You may need to tile your memory input via " - "the tf.contrib.seq2seq.tile_batch function with argument " - "multiple=beam_width.") - with ops.control_dependencies( - self._batch_size_checks(cell_batch_size, error_message)): - cell_output = array_ops.identity(cell_output, name="checked_cell_output") - - if self._is_multi: - previous_attention_state = state.attention_state - previous_alignment_history = state.alignment_history - else: - previous_attention_state = [state.attention_state] - previous_alignment_history = [state.alignment_history] - - all_alignments = [] - all_attentions = [] - all_attention_states = [] - maybe_all_histories = [] - for i, attention_mechanism in enumerate(self._attention_mechanisms): - attention, alignments, next_attention_state = self._attention_fn( - attention_mechanism, cell_output, previous_attention_state[i], - self._attention_layers[i] if self._attention_layers else None) - alignment_history = previous_alignment_history[i].write( - state.time, alignments) if self._alignment_history else () - - all_attention_states.append(next_attention_state) - all_alignments.append(alignments) - all_attentions.append(attention) - maybe_all_histories.append(alignment_history) - - attention = array_ops.concat(all_attentions, 1) - next_state = AttentionWrapperState( - time=state.time + 1, - cell_state=next_cell_state, - attention=attention, - attention_state=self._item_or_tuple(all_attention_states), - alignments=self._item_or_tuple(all_alignments), - alignment_history=self._item_or_tuple(maybe_all_histories)) - - if self._output_attention: - return attention, next_state - else: - return cell_output, next_state diff --git a/tensorflow/contrib/seq2seq/python/ops/basic_decoder.py b/tensorflow/contrib/seq2seq/python/ops/basic_decoder.py deleted file mode 100644 index d4a9c211214..00000000000 --- a/tensorflow/contrib/seq2seq/python/ops/basic_decoder.py +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A class of Decoders that may sample to generate the next input.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from tensorflow.contrib.seq2seq.python.ops import decoder -from tensorflow.contrib.seq2seq.python.ops import helper as helper_py -from tensorflow.contrib.seq2seq.python.ops import sampler as sampler_py -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.keras import layers -from tensorflow.python.layers import base as layers_base -from tensorflow.python.ops import rnn_cell_impl -from tensorflow.python.util import nest - -__all__ = [ - "BasicDecoderOutput", - "BasicDecoder", -] - - -class BasicDecoderOutput( - collections.namedtuple("BasicDecoderOutput", ("rnn_output", "sample_id"))): - pass - - -class BasicDecoder(decoder.Decoder): - """Basic sampling decoder.""" - - def __init__(self, cell, helper, initial_state, output_layer=None): - """Initialize BasicDecoder. - - Args: - cell: An `RNNCell` instance. - helper: A `Helper` instance. - initial_state: A (possibly nested tuple of...) tensors and TensorArrays. - The initial state of the RNNCell. - output_layer: (Optional) An instance of `tf.compat.v1.layers.Layer`, i.e., - `tf.compat.v1.layers.Dense`. Optional layer to apply to the RNN output - prior to storing the result or sampling. - - Raises: - TypeError: if `cell`, `helper` or `output_layer` have an incorrect type. - """ - rnn_cell_impl.assert_like_rnncell("cell", cell) - if not isinstance(helper, helper_py.Helper): - raise TypeError("helper must be a Helper, received: %s" % type(helper)) - if (output_layer is not None and - not isinstance(output_layer, layers_base.Layer)): - raise TypeError("output_layer must be a Layer, received: %s" % - type(output_layer)) - self._cell = cell - self._helper = helper - self._initial_state = initial_state - self._output_layer = output_layer - - @property - def batch_size(self): - return self._helper.batch_size - - def _rnn_output_size(self): - size = self._cell.output_size - if self._output_layer is None: - return size - else: - # To use layer's compute_output_shape, we need to convert the - # RNNCell's output_size entries into shapes with an unknown - # batch size. We then pass this through the layer's - # compute_output_shape and read off all but the first (batch) - # dimensions to get the output size of the rnn with the layer - # applied to the top. - output_shape_with_unknown_batch = nest.map_structure( - lambda s: tensor_shape.TensorShape([None]).concatenate(s), size) - layer_output_shape = self._output_layer.compute_output_shape( - output_shape_with_unknown_batch) - return nest.map_structure(lambda s: s[1:], layer_output_shape) - - @property - def output_size(self): - # Return the cell output and the id - return BasicDecoderOutput( - rnn_output=self._rnn_output_size(), - sample_id=self._helper.sample_ids_shape) - - @property - def output_dtype(self): - # Assume the dtype of the cell is the output_size structure - # containing the input_state's first component's dtype. - # Return that structure and the sample_ids_dtype from the helper. - dtype = nest.flatten(self._initial_state)[0].dtype - return BasicDecoderOutput( - nest.map_structure(lambda _: dtype, self._rnn_output_size()), - self._helper.sample_ids_dtype) - - def initialize(self, name=None): - """Initialize the decoder. - - Args: - name: Name scope for any created operations. - - Returns: - `(finished, first_inputs, initial_state)`. - """ - return self._helper.initialize() + (self._initial_state,) - - def step(self, time, inputs, state, name=None): - """Perform a decoding step. - - Args: - time: scalar `int32` tensor. - inputs: A (structure of) input tensors. - state: A (structure of) state tensors and TensorArrays. - name: Name scope for any created operations. - - Returns: - `(outputs, next_state, next_inputs, finished)`. - """ - with ops.name_scope(name, "BasicDecoderStep", (time, inputs, state)): - cell_outputs, cell_state = self._cell(inputs, state) - if self._output_layer is not None: - cell_outputs = self._output_layer(cell_outputs) - sample_ids = self._helper.sample( - time=time, outputs=cell_outputs, state=cell_state) - (finished, next_inputs, next_state) = self._helper.next_inputs( - time=time, - outputs=cell_outputs, - state=cell_state, - sample_ids=sample_ids) - outputs = BasicDecoderOutput(cell_outputs, sample_ids) - return (outputs, next_state, next_inputs, finished) - - -class BasicDecoderV2(decoder.BaseDecoder): - """Basic sampling decoder.""" - - def __init__(self, cell, sampler, output_layer=None, **kwargs): - """Initialize BasicDecoder. - - Args: - cell: An `RNNCell` instance. - sampler: A `Sampler` instance. - output_layer: (Optional) An instance of `tf.compat.v1.layers.Layer`, i.e., - `tf.compat.v1.layers.Dense`. Optional layer to apply to the RNN output - prior to storing the result or sampling. - **kwargs: Other keyward arguments for layer creation. - - Raises: - TypeError: if `cell`, `helper` or `output_layer` have an incorrect type. - """ - rnn_cell_impl.assert_like_rnncell("cell", cell) - if not isinstance(sampler, sampler_py.Sampler): - raise TypeError("sampler must be a Sampler, received: %s" % (sampler,)) - if (output_layer is not None and - not isinstance(output_layer, layers.Layer)): - raise TypeError("output_layer must be a Layer, received: %s" % - (output_layer,)) - self.cell = cell - self.sampler = sampler - self.output_layer = output_layer - super(BasicDecoderV2, self).__init__(**kwargs) - - def initialize(self, inputs, initial_state=None, **kwargs): - """Initialize the decoder.""" - # Assume the dtype of the cell is the output_size structure - # containing the input_state's first component's dtype. - self._cell_dtype = nest.flatten(initial_state)[0].dtype - return self.sampler.initialize(inputs, **kwargs) + (initial_state,) - - @property - def batch_size(self): - return self.sampler.batch_size - - def _rnn_output_size(self): - size = tensor_shape.TensorShape(self.cell.output_size) - if self.output_layer is None: - return size - else: - # To use layer's compute_output_shape, we need to convert the - # RNNCell's output_size entries into shapes with an unknown - # batch size. We then pass this through the layer's - # compute_output_shape and read off all but the first (batch) - # dimensions to get the output size of the rnn with the layer - # applied to the top. - output_shape_with_unknown_batch = nest.map_structure( - lambda s: tensor_shape.TensorShape([None]).concatenate(s), size) - layer_output_shape = self.output_layer.compute_output_shape( - output_shape_with_unknown_batch) - return nest.map_structure(lambda s: s[1:], layer_output_shape) - - @property - def output_size(self): - # Return the cell output and the id - return BasicDecoderOutput( - rnn_output=self._rnn_output_size(), - sample_id=self.sampler.sample_ids_shape) - - @property - def output_dtype(self): - # Assume the dtype of the cell is the output_size structure - # containing the input_state's first component's dtype. - # Return that structure and the sample_ids_dtype from the helper. - dtype = self._cell_dtype - return BasicDecoderOutput( - nest.map_structure(lambda _: dtype, self._rnn_output_size()), - self.sampler.sample_ids_dtype) - - def step(self, time, inputs, state): - """Perform a decoding step. - - Args: - time: scalar `int32` tensor. - inputs: A (structure of) input tensors. - state: A (structure of) state tensors and TensorArrays. - - Returns: - `(outputs, next_state, next_inputs, finished)`. - """ - cell_outputs, cell_state = self.cell(inputs, state) - if self.output_layer is not None: - cell_outputs = self.output_layer(cell_outputs) - sample_ids = self.sampler.sample( - time=time, outputs=cell_outputs, state=cell_state) - (finished, next_inputs, next_state) = self.sampler.next_inputs( - time=time, - outputs=cell_outputs, - state=cell_state, - sample_ids=sample_ids) - outputs = BasicDecoderOutput(cell_outputs, sample_ids) - return (outputs, next_state, next_inputs, finished) diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py deleted file mode 100644 index 80bdcb88539..00000000000 --- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py +++ /dev/null @@ -1,1370 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A decoder that performs beam search.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import numpy as np - -from tensorflow.contrib.seq2seq.python.ops import attention_wrapper -from tensorflow.contrib.seq2seq.python.ops import beam_search_ops -from tensorflow.contrib.seq2seq.python.ops import decoder -from tensorflow.python.eager import context -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.keras import layers -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import rnn_cell_impl -from tensorflow.python.ops import tensor_array_ops -from tensorflow.python.platform import tf_logging -from tensorflow.python.util import nest - -__all__ = [ - "BeamSearchDecoderOutput", - "BeamSearchDecoderState", - "BeamSearchDecoder", - "FinalBeamSearchDecoderOutput", - "tile_batch", -] - - -class BeamSearchDecoderState( - collections.namedtuple("BeamSearchDecoderState", - ("cell_state", "log_probs", "finished", "lengths", - "accumulated_attention_probs"))): - pass - - -class BeamSearchDecoderOutput( - collections.namedtuple("BeamSearchDecoderOutput", - ("scores", "predicted_ids", "parent_ids"))): - pass - - -class FinalBeamSearchDecoderOutput( - collections.namedtuple("FinalBeamDecoderOutput", - ["predicted_ids", "beam_search_decoder_output"])): - """Final outputs returned by the beam search after all decoding is finished. - - Args: - predicted_ids: The final prediction. A tensor of shape - `[batch_size, T, beam_width]` (or `[T, batch_size, beam_width]` if - `output_time_major` is True). Beams are ordered from best to worst. - beam_search_decoder_output: An instance of `BeamSearchDecoderOutput` that - describes the state of the beam search. - """ - pass - - -def _tile_batch(t, multiplier): - """Core single-tensor implementation of tile_batch.""" - t = ops.convert_to_tensor(t, name="t") - shape_t = array_ops.shape(t) - if t.shape.ndims is None or t.shape.ndims < 1: - raise ValueError("t must have statically known rank") - tiling = [1] * (t.shape.ndims + 1) - tiling[1] = multiplier - tiled_static_batch_size = ( - t.shape.dims[0].value * multiplier - if t.shape.dims[0].value is not None else None) - tiled = array_ops.tile(array_ops.expand_dims(t, 1), tiling) - tiled = array_ops.reshape(tiled, - array_ops.concat( - ([shape_t[0] * multiplier], shape_t[1:]), 0)) - tiled.set_shape( - tensor_shape.TensorShape([tiled_static_batch_size]).concatenate( - t.shape[1:])) - return tiled - - -def tile_batch(t, multiplier, name=None): - """Tile the batch dimension of a (possibly nested structure of) tensor(s) t. - - For each tensor t in a (possibly nested structure) of tensors, - this function takes a tensor t shaped `[batch_size, s0, s1, ...]` composed of - minibatch entries `t[0], ..., t[batch_size - 1]` and tiles it to have a shape - `[batch_size * multiplier, s0, s1, ...]` composed of minibatch entries - `t[0], t[0], ..., t[1], t[1], ...` where each minibatch entry is repeated - `multiplier` times. - - Args: - t: `Tensor` shaped `[batch_size, ...]`. - multiplier: Python int. - name: Name scope for any created operations. - - Returns: - A (possibly nested structure of) `Tensor` shaped - `[batch_size * multiplier, ...]`. - - Raises: - ValueError: if tensor(s) `t` do not have a statically known rank or - the rank is < 1. - """ - flat_t = nest.flatten(t) - with ops.name_scope(name, "tile_batch", flat_t + [multiplier]): - return nest.map_structure(lambda t_: _tile_batch(t_, multiplier), t) - - -def gather_tree_from_array(t, parent_ids, sequence_length): - """Calculates the full beams for `TensorArray`s. - - Args: - t: A stacked `TensorArray` of size `max_time` that contains `Tensor`s of - shape `[batch_size, beam_width, s]` or `[batch_size * beam_width, s]` - where `s` is the depth shape. - parent_ids: The parent ids of shape `[max_time, batch_size, beam_width]`. - sequence_length: The sequence length of shape `[batch_size, beam_width]`. - - Returns: - A `Tensor` which is a stacked `TensorArray` of the same size and type as - `t` and where beams are sorted in each `Tensor` according to `parent_ids`. - """ - max_time = parent_ids.shape.dims[0].value or array_ops.shape(parent_ids)[0] - batch_size = parent_ids.shape.dims[1].value or array_ops.shape(parent_ids)[1] - beam_width = parent_ids.shape.dims[2].value or array_ops.shape(parent_ids)[2] - - # Generate beam ids that will be reordered by gather_tree. - beam_ids = array_ops.expand_dims( - array_ops.expand_dims(math_ops.range(beam_width), 0), 0) - beam_ids = array_ops.tile(beam_ids, [max_time, batch_size, 1]) - - max_sequence_lengths = math_ops.cast( - math_ops.reduce_max(sequence_length, axis=1), dtypes.int32) - sorted_beam_ids = beam_search_ops.gather_tree( - step_ids=beam_ids, - parent_ids=parent_ids, - max_sequence_lengths=max_sequence_lengths, - end_token=beam_width + 1) - - # For out of range steps, simply copy the same beam. - in_bound_steps = array_ops.transpose( - array_ops.sequence_mask(sequence_length, maxlen=max_time), - perm=[2, 0, 1]) - sorted_beam_ids = array_ops.where( - in_bound_steps, x=sorted_beam_ids, y=beam_ids) - - # Generate indices for gather_nd. - time_ind = array_ops.tile(array_ops.reshape( - math_ops.range(max_time), [-1, 1, 1]), [1, batch_size, beam_width]) - batch_ind = array_ops.tile(array_ops.reshape( - math_ops.range(batch_size), [-1, 1, 1]), [1, max_time, beam_width]) - batch_ind = array_ops.transpose(batch_ind, perm=[1, 0, 2]) - indices = array_ops.stack([time_ind, batch_ind, sorted_beam_ids], -1) - - # Gather from a tensor with collapsed additional dimensions. - gather_from = t - final_shape = array_ops.shape(gather_from) - gather_from = array_ops.reshape( - gather_from, [max_time, batch_size, beam_width, -1]) - ordered = array_ops.gather_nd(gather_from, indices) - ordered = array_ops.reshape(ordered, final_shape) - - return ordered - - -def _check_ndims(t): - if t.shape.ndims is None: - raise ValueError( - "Expected tensor (%s) to have known rank, but ndims == None." % t) - - -def _check_static_batch_beam_maybe(shape, batch_size, beam_width): - """Raises an exception if dimensions are known statically and can not be - reshaped to [batch_size, beam_size, -1]. - """ - reshaped_shape = tensor_shape.TensorShape([batch_size, beam_width, None]) - if (batch_size is not None and shape.dims[0].value is not None - and (shape[0] != batch_size * beam_width - or (shape.ndims >= 2 and shape.dims[1].value is not None - and (shape[0] != batch_size or shape[1] != beam_width)))): - tf_logging.warn("TensorArray reordering expects elements to be " - "reshapable to %s which is incompatible with the " - "current shape %s. Consider setting " - "reorder_tensor_arrays to False to disable TensorArray " - "reordering during the beam search." - % (reshaped_shape, shape)) - return False - return True - - -def _check_batch_beam(t, batch_size, beam_width): - """Returns an Assert operation checking that the elements of the stacked - TensorArray can be reshaped to [batch_size, beam_size, -1]. At this point, - the TensorArray elements have a known rank of at least 1. - """ - error_message = ("TensorArray reordering expects elements to be " - "reshapable to [batch_size, beam_size, -1] which is " - "incompatible with the dynamic shape of %s elements. " - "Consider setting reorder_tensor_arrays to False to disable " - "TensorArray reordering during the beam search." - % (t if context.executing_eagerly() else t.name)) - rank = t.shape.ndims - shape = array_ops.shape(t) - if rank == 2: - condition = math_ops.equal(shape[1], batch_size * beam_width) - else: - condition = math_ops.logical_or( - math_ops.equal(shape[1], batch_size * beam_width), - math_ops.logical_and( - math_ops.equal(shape[1], batch_size), - math_ops.equal(shape[2], beam_width))) - return control_flow_ops.Assert(condition, [error_message]) - - -class BeamSearchDecoderMixin(object): - """BeamSearchDecoderMixin contains the common methods for BeamSearchDecoder. - - It is expected to be used a base class for concrete BeamSearchDecoder. Since - this is a mixin class, it is expected to be used together with other class as - base. - """ - - def __init__(self, - cell, - beam_width, - output_layer=None, - length_penalty_weight=0.0, - coverage_penalty_weight=0.0, - reorder_tensor_arrays=True, - **kwargs): - """Initialize the BeamSearchDecoderMixin. - - Args: - cell: An `RNNCell` instance. - beam_width: Python integer, the number of beams. - output_layer: (Optional) An instance of `tf.keras.layers.Layer`, i.e., - `tf.keras.layers.Dense`. Optional layer to apply to the RNN output - prior to storing the result or sampling. - length_penalty_weight: Float weight to penalize length. Disabled with 0.0. - coverage_penalty_weight: Float weight to penalize the coverage of source - sentence. Disabled with 0.0. - reorder_tensor_arrays: If `True`, `TensorArray`s' elements within the cell - state will be reordered according to the beam search path. If the - `TensorArray` can be reordered, the stacked form will be returned. - Otherwise, the `TensorArray` will be returned as is. Set this flag to - `False` if the cell state contains `TensorArray`s that are not amenable - to reordering. - **kwargs: Dict, other keyword arguments for parent class. - - Raises: - TypeError: if `cell` is not an instance of `RNNCell`, - or `output_layer` is not an instance of `tf.keras.layers.Layer`. - """ - rnn_cell_impl.assert_like_rnncell("cell", cell) # pylint: disable=protected-access - if (output_layer is not None and - not isinstance(output_layer, layers.Layer)): - raise TypeError( - "output_layer must be a Layer, received: %s" % type(output_layer)) - self._cell = cell - self._output_layer = output_layer - self._reorder_tensor_arrays = reorder_tensor_arrays - - self._start_tokens = None - self._end_token = None - self._batch_size = None - self._beam_width = beam_width - self._length_penalty_weight = length_penalty_weight - self._coverage_penalty_weight = coverage_penalty_weight - super(BeamSearchDecoderMixin, self).__init__(**kwargs) - - @property - def batch_size(self): - return self._batch_size - - def _rnn_output_size(self): - """Get the output shape from the RNN layer.""" - size = self._cell.output_size - if self._output_layer is None: - return size - else: - # To use layer's compute_output_shape, we need to convert the - # RNNCell's output_size entries into shapes with an unknown - # batch size. We then pass this through the layer's - # compute_output_shape and read off all but the first (batch) - # dimensions to get the output size of the rnn with the layer - # applied to the top. - output_shape_with_unknown_batch = nest.map_structure( - lambda s: tensor_shape.TensorShape([None]).concatenate(s), size) - layer_output_shape = self._output_layer.compute_output_shape( - output_shape_with_unknown_batch) - return nest.map_structure(lambda s: s[1:], layer_output_shape) - - @property - def tracks_own_finished(self): - """The BeamSearchDecoder shuffles its beams and their finished state. - - For this reason, it conflicts with the `dynamic_decode` function's - tracking of finished states. Setting this property to true avoids - early stopping of decoding due to mismanagement of the finished state - in `dynamic_decode`. - - Returns: - `True`. - """ - return True - - @property - def output_size(self): - # Return the cell output and the id - return BeamSearchDecoderOutput( - scores=tensor_shape.TensorShape([self._beam_width]), - predicted_ids=tensor_shape.TensorShape([self._beam_width]), - parent_ids=tensor_shape.TensorShape([self._beam_width])) - - def finalize(self, outputs, final_state, sequence_lengths): - """Finalize and return the predicted_ids. - - Args: - outputs: An instance of BeamSearchDecoderOutput. - final_state: An instance of BeamSearchDecoderState. Passed through to the - output. - sequence_lengths: An `int64` tensor shaped `[batch_size, beam_width]`. - The sequence lengths determined for each beam during decode. - **NOTE** These are ignored; the updated sequence lengths are stored in - `final_state.lengths`. - - Returns: - outputs: An instance of `FinalBeamSearchDecoderOutput` where the - predicted_ids are the result of calling _gather_tree. - final_state: The same input instance of `BeamSearchDecoderState`. - """ - del sequence_lengths - # Get max_sequence_length across all beams for each batch. - max_sequence_lengths = math_ops.cast( - math_ops.reduce_max(final_state.lengths, axis=1), dtypes.int32) - predicted_ids = beam_search_ops.gather_tree( - outputs.predicted_ids, - outputs.parent_ids, - max_sequence_lengths=max_sequence_lengths, - end_token=self._end_token) - if self._reorder_tensor_arrays: - final_state = final_state._replace(cell_state=nest.map_structure( - lambda t: self._maybe_sort_array_beams( - t, outputs.parent_ids, final_state.lengths), - final_state.cell_state)) - outputs = FinalBeamSearchDecoderOutput( - beam_search_decoder_output=outputs, predicted_ids=predicted_ids) - return outputs, final_state - - def _merge_batch_beams(self, t, s=None): - """Merges the tensor from a batch of beams into a batch by beams. - - More exactly, t is a tensor of dimension [batch_size, beam_width, s]. We - reshape this into [batch_size*beam_width, s] - - Args: - t: Tensor of dimension [batch_size, beam_width, s] - s: (Possibly known) depth shape. - - Returns: - A reshaped version of t with dimension [batch_size * beam_width, s]. - """ - if isinstance(s, ops.Tensor): - s = tensor_shape.as_shape(tensor_util.constant_value(s)) - else: - s = tensor_shape.TensorShape(s) - t_shape = array_ops.shape(t) - static_batch_size = tensor_util.constant_value(self._batch_size) - batch_size_beam_width = ( - None - if static_batch_size is None else static_batch_size * self._beam_width) - reshaped_t = array_ops.reshape( - t, - array_ops.concat(([self._batch_size * self._beam_width], t_shape[2:]), - 0)) - reshaped_t.set_shape( - (tensor_shape.TensorShape([batch_size_beam_width]).concatenate(s))) - return reshaped_t - - def _split_batch_beams(self, t, s=None): - """Splits the tensor from a batch by beams into a batch of beams. - - More exactly, t is a tensor of dimension [batch_size*beam_width, s]. We - reshape this into [batch_size, beam_width, s] - - Args: - t: Tensor of dimension [batch_size*beam_width, s]. - s: (Possibly known) depth shape. - - Returns: - A reshaped version of t with dimension [batch_size, beam_width, s]. - - Raises: - ValueError: If, after reshaping, the new tensor is not shaped - `[batch_size, beam_width, s]` (assuming batch_size and beam_width - are known statically). - """ - if isinstance(s, ops.Tensor): - s = tensor_shape.TensorShape(tensor_util.constant_value(s)) - else: - s = tensor_shape.TensorShape(s) - t_shape = array_ops.shape(t) - reshaped_t = array_ops.reshape( - t, - array_ops.concat(([self._batch_size, self._beam_width], t_shape[1:]), - 0)) - static_batch_size = tensor_util.constant_value(self._batch_size) - expected_reshaped_shape = tensor_shape.TensorShape( - [static_batch_size, self._beam_width]).concatenate(s) - if not reshaped_t.shape.is_compatible_with(expected_reshaped_shape): - raise ValueError("Unexpected behavior when reshaping between beam width " - "and batch size. The reshaped tensor has shape: %s. " - "We expected it to have shape " - "(batch_size, beam_width, depth) == %s. Perhaps you " - "forgot to create a zero_state with " - "batch_size=encoder_batch_size * beam_width?" % - (reshaped_t.shape, expected_reshaped_shape)) - reshaped_t.set_shape(expected_reshaped_shape) - return reshaped_t - - def _maybe_split_batch_beams(self, t, s): - """Maybe splits the tensor from a batch by beams into a batch of beams. - - We do this so that we can use nest and not run into problems with shapes. - - Args: - t: `Tensor`, either scalar or shaped `[batch_size * beam_width] + s`. - s: `Tensor`, Python int, or `TensorShape`. - - Returns: - If `t` is a matrix or higher order tensor, then the return value is - `t` reshaped to `[batch_size, beam_width] + s`. Otherwise `t` is - returned unchanged. - - Raises: - ValueError: If the rank of `t` is not statically known. - """ - if isinstance(t, tensor_array_ops.TensorArray): - return t - _check_ndims(t) - if t.shape.ndims >= 1: - return self._split_batch_beams(t, s) - else: - return t - - def _maybe_merge_batch_beams(self, t, s): - """Splits the tensor from a batch by beams into a batch of beams. - - More exactly, `t` is a tensor of dimension `[batch_size * beam_width] + s`, - then we reshape it to `[batch_size, beam_width] + s`. - - Args: - t: `Tensor` of dimension `[batch_size * beam_width] + s`. - s: `Tensor`, Python int, or `TensorShape`. - - Returns: - A reshaped version of t with shape `[batch_size, beam_width] + s`. - - Raises: - ValueError: If the rank of `t` is not statically known. - """ - if isinstance(t, tensor_array_ops.TensorArray): - return t - _check_ndims(t) - if t.shape.ndims >= 2: - return self._merge_batch_beams(t, s) - else: - return t - - def _maybe_sort_array_beams(self, t, parent_ids, sequence_length): - """Maybe sorts beams within a `TensorArray`. - - Args: - t: A `TensorArray` of size `max_time` that contains `Tensor`s of shape - `[batch_size, beam_width, s]` or `[batch_size * beam_width, s]` where - `s` is the depth shape. - parent_ids: The parent ids of shape `[max_time, batch_size, beam_width]`. - sequence_length: The sequence length of shape `[batch_size, beam_width]`. - - Returns: - A `TensorArray` where beams are sorted in each `Tensor` or `t` itself if - it is not a `TensorArray` or does not meet shape requirements. - """ - if not isinstance(t, tensor_array_ops.TensorArray): - return t - if t.element_shape.ndims is None or t.element_shape.ndims < 1: - tf_logging.warn("The TensorArray %s in the cell state is not amenable to " - "sorting based on the beam search result. For a " - "TensorArray to be sorted, its elements shape must be " - "defined and have at least a rank of 1, but saw shape: %s" - % (t.handle.name, t.element_shape)) - return t - if not _check_static_batch_beam_maybe( - t.element_shape, tensor_util.constant_value(self._batch_size), - self._beam_width): - return t - t = t.stack() - with ops.control_dependencies( - [_check_batch_beam(t, self._batch_size, self._beam_width)]): - return gather_tree_from_array(t, parent_ids, sequence_length) - - def step(self, time, inputs, state, name=None): - """Perform a decoding step. - - Args: - time: scalar `int32` tensor. - inputs: A (structure of) input tensors. - state: A (structure of) state tensors and TensorArrays. - name: Name scope for any created operations. - - Returns: - `(outputs, next_state, next_inputs, finished)`. - """ - batch_size = self._batch_size - beam_width = self._beam_width - end_token = self._end_token - length_penalty_weight = self._length_penalty_weight - coverage_penalty_weight = self._coverage_penalty_weight - - with ops.name_scope(name, "BeamSearchDecoderStep", (time, inputs, state)): - cell_state = state.cell_state - inputs = nest.map_structure( - lambda inp: self._merge_batch_beams(inp, s=inp.shape[2:]), inputs) - cell_state = nest.map_structure(self._maybe_merge_batch_beams, cell_state, - self._cell.state_size) - cell_outputs, next_cell_state = self._cell(inputs, cell_state) - cell_outputs = nest.map_structure( - lambda out: self._split_batch_beams(out, out.shape[1:]), cell_outputs) - next_cell_state = nest.map_structure( - self._maybe_split_batch_beams, next_cell_state, self._cell.state_size) - - if self._output_layer is not None: - cell_outputs = self._output_layer(cell_outputs) - - beam_search_output, beam_search_state = _beam_search_step( - time=time, - logits=cell_outputs, - next_cell_state=next_cell_state, - beam_state=state, - batch_size=batch_size, - beam_width=beam_width, - end_token=end_token, - length_penalty_weight=length_penalty_weight, - coverage_penalty_weight=coverage_penalty_weight) - - finished = beam_search_state.finished - sample_ids = beam_search_output.predicted_ids - next_inputs = control_flow_ops.cond( - math_ops.reduce_all(finished), lambda: self._start_inputs, - lambda: self._embedding_fn(sample_ids)) - - return (beam_search_output, beam_search_state, next_inputs, finished) - - -class BeamSearchDecoder(BeamSearchDecoderMixin, decoder.Decoder): - # Note that the inheritance hierarchy is important here. The Mixin has to be - # the first parent class since we will use super().__init__(), and Mixin which - # is a object will properly invoke the __init__ method of other parent class. - """BeamSearch sampling decoder. - - **NOTE** If you are using the `BeamSearchDecoder` with a cell wrapped in - `AttentionWrapper`, then you must ensure that: - - - The encoder output has been tiled to `beam_width` via - `tf.contrib.seq2seq.tile_batch` (NOT `tf.tile`). - - The `batch_size` argument passed to the `zero_state` method of this - wrapper is equal to `true_batch_size * beam_width`. - - The initial state created with `zero_state` above contains a - `cell_state` value containing properly tiled final state from the - encoder. - - An example: - - ``` - tiled_encoder_outputs = tf.contrib.seq2seq.tile_batch( - encoder_outputs, multiplier=beam_width) - tiled_encoder_final_state = tf.contrib.seq2seq.tile_batch( - encoder_final_state, multiplier=beam_width) - tiled_sequence_length = tf.contrib.seq2seq.tile_batch( - sequence_length, multiplier=beam_width) - attention_mechanism = MyFavoriteAttentionMechanism( - num_units=attention_depth, - memory=tiled_inputs, - memory_sequence_length=tiled_sequence_length) - attention_cell = AttentionWrapper(cell, attention_mechanism, ...) - decoder_initial_state = attention_cell.zero_state( - dtype, batch_size=true_batch_size * beam_width) - decoder_initial_state = decoder_initial_state.clone( - cell_state=tiled_encoder_final_state) - ``` - - Meanwhile, with `AttentionWrapper`, coverage penalty is suggested to use - when computing scores (https://arxiv.org/pdf/1609.08144.pdf). It encourages - the decoder to cover all inputs. - """ - - def __init__(self, - cell, - embedding, - start_tokens, - end_token, - initial_state, - beam_width, - output_layer=None, - length_penalty_weight=0.0, - coverage_penalty_weight=0.0, - reorder_tensor_arrays=True): - """Initialize the BeamSearchDecoder. - - Args: - cell: An `RNNCell` instance. - embedding: A callable that takes a vector tensor of `ids` (argmax ids), - or the `params` argument for `embedding_lookup`. - start_tokens: `int32` vector shaped `[batch_size]`, the start tokens. - end_token: `int32` scalar, the token that marks end of decoding. - initial_state: A (possibly nested tuple of...) tensors and TensorArrays. - beam_width: Python integer, the number of beams. - output_layer: (Optional) An instance of `tf.keras.layers.Layer`, i.e., - `tf.keras.layers.Dense`. Optional layer to apply to the RNN output - prior to storing the result or sampling. - length_penalty_weight: Float weight to penalize length. Disabled with 0.0. - coverage_penalty_weight: Float weight to penalize the coverage of source - sentence. Disabled with 0.0. - reorder_tensor_arrays: If `True`, `TensorArray`s' elements within the cell - state will be reordered according to the beam search path. If the - `TensorArray` can be reordered, the stacked form will be returned. - Otherwise, the `TensorArray` will be returned as is. Set this flag to - `False` if the cell state contains `TensorArray`s that are not amenable - to reordering. - - Raises: - TypeError: if `cell` is not an instance of `RNNCell`, - or `output_layer` is not an instance of `tf.keras.layers.Layer`. - ValueError: If `start_tokens` is not a vector or - `end_token` is not a scalar. - """ - super(BeamSearchDecoder, self).__init__( - cell, - beam_width, - output_layer=output_layer, - length_penalty_weight=length_penalty_weight, - coverage_penalty_weight=coverage_penalty_weight, - reorder_tensor_arrays=reorder_tensor_arrays) - - if callable(embedding): - self._embedding_fn = embedding - else: - self._embedding_fn = ( - lambda ids: embedding_ops.embedding_lookup(embedding, ids)) - - self._start_tokens = ops.convert_to_tensor( - start_tokens, dtype=dtypes.int32, name="start_tokens") - if self._start_tokens.get_shape().ndims != 1: - raise ValueError("start_tokens must be a vector") - self._end_token = ops.convert_to_tensor( - end_token, dtype=dtypes.int32, name="end_token") - if self._end_token.get_shape().ndims != 0: - raise ValueError("end_token must be a scalar") - - self._batch_size = array_ops.size(start_tokens) - self._initial_cell_state = nest.map_structure( - self._maybe_split_batch_beams, initial_state, self._cell.state_size) - self._start_tokens = array_ops.tile( - array_ops.expand_dims(self._start_tokens, 1), [1, self._beam_width]) - self._start_inputs = self._embedding_fn(self._start_tokens) - - self._finished = array_ops.one_hot( - array_ops.zeros([self._batch_size], dtype=dtypes.int32), - depth=self._beam_width, - on_value=False, - off_value=True, - dtype=dtypes.bool) - - def initialize(self, name=None): - """Initialize the decoder. - - Args: - name: Name scope for any created operations. - - Returns: - `(finished, start_inputs, initial_state)`. - """ - finished, start_inputs = self._finished, self._start_inputs - - dtype = nest.flatten(self._initial_cell_state)[0].dtype - log_probs = array_ops.one_hot( # shape(batch_sz, beam_sz) - array_ops.zeros([self._batch_size], dtype=dtypes.int32), - depth=self._beam_width, - on_value=ops.convert_to_tensor(0.0, dtype=dtype), - off_value=ops.convert_to_tensor(-np.Inf, dtype=dtype), - dtype=dtype) - init_attention_probs = get_attention_probs( - self._initial_cell_state, self._coverage_penalty_weight) - if init_attention_probs is None: - init_attention_probs = () - - initial_state = BeamSearchDecoderState( - cell_state=self._initial_cell_state, - log_probs=log_probs, - finished=finished, - lengths=array_ops.zeros( - [self._batch_size, self._beam_width], dtype=dtypes.int64), - accumulated_attention_probs=init_attention_probs) - - return (finished, start_inputs, initial_state) - - @property - def output_dtype(self): - # Assume the dtype of the cell is the output_size structure - # containing the input_state's first component's dtype. - # Return that structure and int32 (the id) - dtype = nest.flatten(self._initial_cell_state)[0].dtype - return BeamSearchDecoderOutput( - scores=nest.map_structure(lambda _: dtype, self._rnn_output_size()), - predicted_ids=dtypes.int32, - parent_ids=dtypes.int32) - - -class BeamSearchDecoderV2(BeamSearchDecoderMixin, decoder.BaseDecoder): - # Note that the inheritance hierarchy is important here. The Mixin has to be - # the first parent class since we will use super().__init__(), and Mixin which - # is a object will properly invoke the __init__ method of other parent class. - """BeamSearch sampling decoder. - - **NOTE** If you are using the `BeamSearchDecoder` with a cell wrapped in - `AttentionWrapper`, then you must ensure that: - - - The encoder output has been tiled to `beam_width` via - `tf.contrib.seq2seq.tile_batch` (NOT `tf.tile`). - - The `batch_size` argument passed to the `zero_state` method of this - wrapper is equal to `true_batch_size * beam_width`. - - The initial state created with `zero_state` above contains a - `cell_state` value containing properly tiled final state from the - encoder. - - An example: - - ``` - tiled_encoder_outputs = tf.contrib.seq2seq.tile_batch( - encoder_outputs, multiplier=beam_width) - tiled_encoder_final_state = tf.contrib.seq2seq.tile_batch( - encoder_final_state, multiplier=beam_width) - tiled_sequence_length = tf.contrib.seq2seq.tile_batch( - sequence_length, multiplier=beam_width) - attention_mechanism = MyFavoriteAttentionMechanism( - num_units=attention_depth, - memory=tiled_inputs, - memory_sequence_length=tiled_sequence_length) - attention_cell = AttentionWrapper(cell, attention_mechanism, ...) - decoder_initial_state = attention_cell.zero_state( - dtype, batch_size=true_batch_size * beam_width) - decoder_initial_state = decoder_initial_state.clone( - cell_state=tiled_encoder_final_state) - ``` - - Meanwhile, with `AttentionWrapper`, coverage penalty is suggested to use - when computing scores (https://arxiv.org/pdf/1609.08144.pdf). It encourages - the decoding to cover all inputs. - """ - - def __init__(self, - cell, - beam_width, - embedding_fn=None, - output_layer=None, - length_penalty_weight=0.0, - coverage_penalty_weight=0.0, - reorder_tensor_arrays=True, - **kwargs): - """Initialize the BeamSearchDecoderV2. - - Args: - cell: An `RNNCell` instance. - beam_width: Python integer, the number of beams. - embedding_fn: A callable that takes a vector tensor of `ids` (argmax ids). - output_layer: (Optional) An instance of `tf.keras.layers.Layer`, i.e., - `tf.keras.layers.Dense`. Optional layer to apply to the RNN output - prior to storing the result or sampling. - length_penalty_weight: Float weight to penalize length. Disabled with 0.0. - coverage_penalty_weight: Float weight to penalize the coverage of source - sentence. Disabled with 0.0. - reorder_tensor_arrays: If `True`, `TensorArray`s' elements within the cell - state will be reordered according to the beam search path. If the - `TensorArray` can be reordered, the stacked form will be returned. - Otherwise, the `TensorArray` will be returned as is. Set this flag to - `False` if the cell state contains `TensorArray`s that are not amenable - to reordering. - **kwargs: Dict, other keyword arguments for initialization. - - Raises: - TypeError: if `cell` is not an instance of `RNNCell`, - or `output_layer` is not an instance of `tf.keras.layers.Layer`. - """ - super(BeamSearchDecoderV2, self).__init__( - cell, - beam_width, - output_layer=output_layer, - length_penalty_weight=length_penalty_weight, - coverage_penalty_weight=coverage_penalty_weight, - reorder_tensor_arrays=reorder_tensor_arrays, - **kwargs) - - if embedding_fn is None or callable(embedding_fn): - self._embedding_fn = embedding_fn - else: - raise ValueError("embedding_fn is expected to be a callable, got %s" % - type(embedding_fn)) - - def initialize(self, - embedding, - start_tokens, - end_token, - initial_state): - """Initialize the decoder. - - Args: - embedding: A tensor from the embedding layer output, which is the - `params` argument for `embedding_lookup`. - start_tokens: `int32` vector shaped `[batch_size]`, the start tokens. - end_token: `int32` scalar, the token that marks end of decoding. - initial_state: A (possibly nested tuple of...) tensors and TensorArrays. - Returns: - `(finished, start_inputs, initial_state)`. - Raises: - ValueError: If `start_tokens` is not a vector or `end_token` is not a - scalar. - """ - if embedding is not None and self._embedding_fn is not None: - raise ValueError( - "embedding and embedding_fn cannot be provided at same time") - elif embedding is not None: - self._embedding_fn = ( - lambda ids: embedding_ops.embedding_lookup(embedding, ids)) - - self._start_tokens = ops.convert_to_tensor( - start_tokens, dtype=dtypes.int32, name="start_tokens") - if self._start_tokens.get_shape().ndims != 1: - raise ValueError("start_tokens must be a vector") - self._end_token = ops.convert_to_tensor( - end_token, dtype=dtypes.int32, name="end_token") - if self._end_token.get_shape().ndims != 0: - raise ValueError("end_token must be a scalar") - - self._batch_size = array_ops.size(start_tokens) - self._initial_cell_state = nest.map_structure( - self._maybe_split_batch_beams, initial_state, self._cell.state_size) - self._start_tokens = array_ops.tile( - array_ops.expand_dims(self._start_tokens, 1), [1, self._beam_width]) - self._start_inputs = self._embedding_fn(self._start_tokens) - - self._finished = array_ops.one_hot( - array_ops.zeros([self._batch_size], dtype=dtypes.int32), - depth=self._beam_width, - on_value=False, - off_value=True, - dtype=dtypes.bool) - - finished, start_inputs = self._finished, self._start_inputs - - dtype = nest.flatten(self._initial_cell_state)[0].dtype - log_probs = array_ops.one_hot( # shape(batch_sz, beam_sz) - array_ops.zeros([self._batch_size], dtype=dtypes.int32), - depth=self._beam_width, - on_value=ops.convert_to_tensor(0.0, dtype=dtype), - off_value=ops.convert_to_tensor(-np.Inf, dtype=dtype), - dtype=dtype) - init_attention_probs = get_attention_probs( - self._initial_cell_state, self._coverage_penalty_weight) - if init_attention_probs is None: - init_attention_probs = () - - initial_state = BeamSearchDecoderState( - cell_state=self._initial_cell_state, - log_probs=log_probs, - finished=finished, - lengths=array_ops.zeros( - [self._batch_size, self._beam_width], dtype=dtypes.int64), - accumulated_attention_probs=init_attention_probs) - - return (finished, start_inputs, initial_state) - - @property - def output_dtype(self): - # Assume the dtype of the cell is the output_size structure - # containing the input_state's first component's dtype. - # Return that structure and int32 (the id) - dtype = nest.flatten(self._initial_cell_state)[0].dtype - return BeamSearchDecoderOutput( - scores=nest.map_structure(lambda _: dtype, self._rnn_output_size()), - predicted_ids=dtypes.int32, - parent_ids=dtypes.int32) - - def call(self, embeddning, start_tokens, end_token, initial_state, **kwargs): - init_kwargs = kwargs - init_kwargs["start_tokens"] = start_tokens - init_kwargs["end_token"] = end_token - init_kwargs["initial_state"] = initial_state - return decoder.dynamic_decode(self, - output_time_major=self.output_time_major, - impute_finished=self.impute_finished, - maximum_iterations=self.maximum_iterations, - parallel_iterations=self.parallel_iterations, - swap_memory=self.swap_memory, - decoder_init_input=embeddning, - decoder_init_kwargs=init_kwargs) - - -def _beam_search_step(time, logits, next_cell_state, beam_state, batch_size, - beam_width, end_token, length_penalty_weight, - coverage_penalty_weight): - """Performs a single step of Beam Search Decoding. - - Args: - time: Beam search time step, should start at 0. At time 0 we assume - that all beams are equal and consider only the first beam for - continuations. - logits: Logits at the current time step. A tensor of shape - `[batch_size, beam_width, vocab_size]` - next_cell_state: The next state from the cell, e.g. an instance of - AttentionWrapperState if the cell is attentional. - beam_state: Current state of the beam search. - An instance of `BeamSearchDecoderState`. - batch_size: The batch size for this input. - beam_width: Python int. The size of the beams. - end_token: The int32 end token. - length_penalty_weight: Float weight to penalize length. Disabled with 0.0. - coverage_penalty_weight: Float weight to penalize the coverage of source - sentence. Disabled with 0.0. - - Returns: - A new beam state. - """ - static_batch_size = tensor_util.constant_value(batch_size) - - # Calculate the current lengths of the predictions - prediction_lengths = beam_state.lengths - previously_finished = beam_state.finished - not_finished = math_ops.logical_not(previously_finished) - - # Calculate the total log probs for the new hypotheses - # Final Shape: [batch_size, beam_width, vocab_size] - step_log_probs = nn_ops.log_softmax(logits) - step_log_probs = _mask_probs(step_log_probs, end_token, previously_finished) - total_probs = array_ops.expand_dims(beam_state.log_probs, 2) + step_log_probs - - # Calculate the continuation lengths by adding to all continuing beams. - vocab_size = logits.shape.dims[-1].value or array_ops.shape(logits)[-1] - lengths_to_add = array_ops.one_hot( - indices=array_ops.fill([batch_size, beam_width], end_token), - depth=vocab_size, - on_value=math_ops.to_int64(0), - off_value=math_ops.to_int64(1), - dtype=dtypes.int64) - add_mask = math_ops.cast(not_finished, dtypes.int64) - lengths_to_add *= array_ops.expand_dims(add_mask, 2) - new_prediction_lengths = ( - lengths_to_add + array_ops.expand_dims(prediction_lengths, 2)) - - # Calculate the accumulated attention probabilities if coverage penalty is - # enabled. - accumulated_attention_probs = None - attention_probs = get_attention_probs( - next_cell_state, coverage_penalty_weight) - if attention_probs is not None: - attention_probs *= array_ops.expand_dims( - math_ops.cast(not_finished, dtypes.float32), 2) - accumulated_attention_probs = ( - beam_state.accumulated_attention_probs + attention_probs) - - # Calculate the scores for each beam - scores = _get_scores( - log_probs=total_probs, - sequence_lengths=new_prediction_lengths, - length_penalty_weight=length_penalty_weight, - coverage_penalty_weight=coverage_penalty_weight, - finished=previously_finished, - accumulated_attention_probs=accumulated_attention_probs) - - time = ops.convert_to_tensor(time, name="time") - # During the first time step we only consider the initial beam - scores_flat = array_ops.reshape(scores, [batch_size, -1]) - - # Pick the next beams according to the specified successors function - next_beam_size = ops.convert_to_tensor( - beam_width, dtype=dtypes.int32, name="beam_width") - next_beam_scores, word_indices = nn_ops.top_k(scores_flat, k=next_beam_size) - - next_beam_scores.set_shape([static_batch_size, beam_width]) - word_indices.set_shape([static_batch_size, beam_width]) - - # Pick out the probs, beam_ids, and states according to the chosen predictions - next_beam_probs = _tensor_gather_helper( - gather_indices=word_indices, - gather_from=total_probs, - batch_size=batch_size, - range_size=beam_width * vocab_size, - gather_shape=[-1], - name="next_beam_probs") - # Note: just doing the following - # math_ops.cast( - # word_indices % vocab_size, - # dtypes.int32, - # name="next_beam_word_ids") - # would be a lot cleaner but for reasons unclear, that hides the results of - # the op which prevents capturing it with tfdbg debug ops. - raw_next_word_ids = math_ops.mod( - word_indices, vocab_size, name="next_beam_word_ids") - next_word_ids = math_ops.cast(raw_next_word_ids, dtypes.int32) - next_beam_ids = math_ops.cast( - word_indices / vocab_size, dtypes.int32, name="next_beam_parent_ids") - - # Append new ids to current predictions - previously_finished = _tensor_gather_helper( - gather_indices=next_beam_ids, - gather_from=previously_finished, - batch_size=batch_size, - range_size=beam_width, - gather_shape=[-1]) - next_finished = math_ops.logical_or( - previously_finished, - math_ops.equal(next_word_ids, end_token), - name="next_beam_finished") - - # Calculate the length of the next predictions. - # 1. Finished beams remain unchanged. - # 2. Beams that are now finished (EOS predicted) have their length - # increased by 1. - # 3. Beams that are not yet finished have their length increased by 1. - lengths_to_add = math_ops.cast( - math_ops.logical_not(previously_finished), dtypes.int64) - next_prediction_len = _tensor_gather_helper( - gather_indices=next_beam_ids, - gather_from=beam_state.lengths, - batch_size=batch_size, - range_size=beam_width, - gather_shape=[-1]) - next_prediction_len += lengths_to_add - next_accumulated_attention_probs = () - if accumulated_attention_probs is not None: - next_accumulated_attention_probs = _tensor_gather_helper( - gather_indices=next_beam_ids, - gather_from=accumulated_attention_probs, - batch_size=batch_size, - range_size=beam_width, - gather_shape=[batch_size * beam_width, -1], - name="next_accumulated_attention_probs") - - # Pick out the cell_states according to the next_beam_ids. We use a - # different gather_shape here because the cell_state tensors, i.e. - # the tensors that would be gathered from, all have dimension - # greater than two and we need to preserve those dimensions. - # pylint: disable=g-long-lambda - next_cell_state = nest.map_structure( - lambda gather_from: _maybe_tensor_gather_helper( - gather_indices=next_beam_ids, - gather_from=gather_from, - batch_size=batch_size, - range_size=beam_width, - gather_shape=[batch_size * beam_width, -1]), - next_cell_state) - # pylint: enable=g-long-lambda - - next_state = BeamSearchDecoderState( - cell_state=next_cell_state, - log_probs=next_beam_probs, - lengths=next_prediction_len, - finished=next_finished, - accumulated_attention_probs=next_accumulated_attention_probs) - - output = BeamSearchDecoderOutput( - scores=next_beam_scores, - predicted_ids=next_word_ids, - parent_ids=next_beam_ids) - - return output, next_state - - -def get_attention_probs(next_cell_state, coverage_penalty_weight): - """Get attention probabilities from the cell state. - - Args: - next_cell_state: The next state from the cell, e.g. an instance of - AttentionWrapperState if the cell is attentional. - coverage_penalty_weight: Float weight to penalize the coverage of source - sentence. Disabled with 0.0. - - Returns: - The attention probabilities with shape `[batch_size, beam_width, max_time]` - if coverage penalty is enabled. Otherwise, returns None. - - Raises: - ValueError: If no cell is attentional but coverage penalty is enabled. - """ - if coverage_penalty_weight == 0.0: - return None - - # Attention probabilities of each attention layer. Each with shape - # `[batch_size, beam_width, max_time]`. - probs_per_attn_layer = [] - if isinstance(next_cell_state, attention_wrapper.AttentionWrapperState): - probs_per_attn_layer = [attention_probs_from_attn_state(next_cell_state)] - elif isinstance(next_cell_state, tuple): - for state in next_cell_state: - if isinstance(state, attention_wrapper.AttentionWrapperState): - probs_per_attn_layer.append(attention_probs_from_attn_state(state)) - - if not probs_per_attn_layer: - raise ValueError( - "coverage_penalty_weight must be 0.0 if no cell is attentional.") - - if len(probs_per_attn_layer) == 1: - attention_probs = probs_per_attn_layer[0] - else: - # Calculate the average attention probabilities from all attention layers. - attention_probs = [ - array_ops.expand_dims(prob, -1) for prob in probs_per_attn_layer] - attention_probs = array_ops.concat(attention_probs, -1) - attention_probs = math_ops.reduce_mean(attention_probs, -1) - - return attention_probs - - -def _get_scores(log_probs, sequence_lengths, length_penalty_weight, - coverage_penalty_weight, finished, accumulated_attention_probs): - """Calculates scores for beam search hypotheses. - - Args: - log_probs: The log probabilities with shape - `[batch_size, beam_width, vocab_size]`. - sequence_lengths: The array of sequence lengths. - length_penalty_weight: Float weight to penalize length. Disabled with 0.0. - coverage_penalty_weight: Float weight to penalize the coverage of source - sentence. Disabled with 0.0. - finished: A boolean tensor of shape `[batch_size, beam_width]` that - specifies which elements in the beam are finished already. - accumulated_attention_probs: Accumulated attention probabilities up to the - current time step, with shape `[batch_size, beam_width, max_time]` if - coverage_penalty_weight is not 0.0. - - Returns: - The scores normalized by the length_penalty and coverage_penalty. - - Raises: - ValueError: accumulated_attention_probs is None when coverage penalty is - enabled. - """ - length_penalty_ = _length_penalty( - sequence_lengths=sequence_lengths, penalty_factor=length_penalty_weight) - length_penalty_ = math_ops.cast(length_penalty_, dtype=log_probs.dtype) - scores = log_probs / length_penalty_ - - coverage_penalty_weight = ops.convert_to_tensor( - coverage_penalty_weight, name="coverage_penalty_weight") - if coverage_penalty_weight.shape.ndims != 0: - raise ValueError("coverage_penalty_weight should be a scalar, " - "but saw shape: %s" % coverage_penalty_weight.shape) - - if tensor_util.constant_value(coverage_penalty_weight) == 0.0: - return scores - - if accumulated_attention_probs is None: - raise ValueError( - "accumulated_attention_probs can be None only if coverage penalty is " - "disabled.") - - # Add source sequence length mask before computing coverage penalty. - accumulated_attention_probs = array_ops.where( - math_ops.equal(accumulated_attention_probs, 0.0), - array_ops.ones_like(accumulated_attention_probs), - accumulated_attention_probs) - - # coverage penalty = - # sum over `max_time` {log(min(accumulated_attention_probs, 1.0))} - coverage_penalty = math_ops.reduce_sum( - math_ops.log(math_ops.minimum(accumulated_attention_probs, 1.0)), 2) - # Apply coverage penalty to finished predictions. - coverage_penalty *= math_ops.cast(finished, dtypes.float32) - weighted_coverage_penalty = coverage_penalty * coverage_penalty_weight - # Reshape from [batch_size, beam_width] to [batch_size, beam_width, 1] - weighted_coverage_penalty = array_ops.expand_dims( - weighted_coverage_penalty, 2) - return scores + weighted_coverage_penalty - - -def attention_probs_from_attn_state(attention_state): - """Calculates the average attention probabilities. - - Args: - attention_state: An instance of `AttentionWrapperState`. - - Returns: - The attention probabilities in the given AttentionWrapperState. - If there're multiple attention mechanisms, return the average value from - all attention mechanisms. - """ - # Attention probabilities over time steps, with shape - # `[batch_size, beam_width, max_time]`. - attention_probs = attention_state.alignments - if isinstance(attention_probs, tuple): - attention_probs = [ - array_ops.expand_dims(prob, -1) for prob in attention_probs] - attention_probs = array_ops.concat(attention_probs, -1) - attention_probs = math_ops.reduce_mean(attention_probs, -1) - return attention_probs - - -def _length_penalty(sequence_lengths, penalty_factor): - """Calculates the length penalty. See https://arxiv.org/abs/1609.08144. - - Returns the length penalty tensor: - ``` - [(5+sequence_lengths)/6]**penalty_factor - ``` - where all operations are performed element-wise. - - Args: - sequence_lengths: `Tensor`, the sequence lengths of each hypotheses. - penalty_factor: A scalar that weights the length penalty. - - Returns: - If the penalty is `0`, returns the scalar `1.0`. Otherwise returns - the length penalty factor, a tensor with the same shape as - `sequence_lengths`. - """ - penalty_factor = ops.convert_to_tensor(penalty_factor, name="penalty_factor") - penalty_factor.set_shape(()) # penalty should be a scalar. - static_penalty = tensor_util.constant_value(penalty_factor) - if static_penalty is not None and static_penalty == 0: - return 1.0 - return math_ops.div( - (5. + math_ops.cast(sequence_lengths, dtypes.float32))**penalty_factor, - (5. + 1.)**penalty_factor) - - -def _mask_probs(probs, eos_token, finished): - """Masks log probabilities. - - The result is that finished beams allocate all probability mass to eos and - unfinished beams remain unchanged. - - Args: - probs: Log probabilities of shape `[batch_size, beam_width, vocab_size]` - eos_token: An int32 id corresponding to the EOS token to allocate - probability to. - finished: A boolean tensor of shape `[batch_size, beam_width]` that - specifies which elements in the beam are finished already. - - Returns: - A tensor of shape `[batch_size, beam_width, vocab_size]`, where unfinished - beams stay unchanged and finished beams are replaced with a tensor with all - probability on the EOS token. - """ - vocab_size = array_ops.shape(probs)[2] - # All finished examples are replaced with a vector that has all - # probability on EOS - finished_row = array_ops.one_hot( - eos_token, - vocab_size, - dtype=probs.dtype, - on_value=ops.convert_to_tensor(0., dtype=probs.dtype), - off_value=probs.dtype.min) - finished_probs = array_ops.tile( - array_ops.reshape(finished_row, [1, 1, -1]), - array_ops.concat([array_ops.shape(finished), [1]], 0)) - finished_mask = array_ops.tile( - array_ops.expand_dims(finished, 2), [1, 1, vocab_size]) - - return array_ops.where(finished_mask, finished_probs, probs) - - -def _maybe_tensor_gather_helper(gather_indices, gather_from, batch_size, - range_size, gather_shape): - """Maybe applies _tensor_gather_helper. - - This applies _tensor_gather_helper when the gather_from dims is at least as - big as the length of gather_shape. This is used in conjunction with nest so - that we don't apply _tensor_gather_helper to inapplicable values like scalars. - - Args: - gather_indices: The tensor indices that we use to gather. - gather_from: The tensor that we are gathering from. - batch_size: The batch size. - range_size: The number of values in each range. Likely equal to beam_width. - gather_shape: What we should reshape gather_from to in order to preserve the - correct values. An example is when gather_from is the attention from an - AttentionWrapperState with shape [batch_size, beam_width, attention_size]. - There, we want to preserve the attention_size elements, so gather_shape is - [batch_size * beam_width, -1]. Then, upon reshape, we still have the - attention_size as desired. - - Returns: - output: Gathered tensor of shape tf.shape(gather_from)[:1+len(gather_shape)] - or the original tensor if its dimensions are too small. - """ - if isinstance(gather_from, tensor_array_ops.TensorArray): - return gather_from - _check_ndims(gather_from) - if gather_from.shape.ndims >= len(gather_shape): - return _tensor_gather_helper( - gather_indices=gather_indices, - gather_from=gather_from, - batch_size=batch_size, - range_size=range_size, - gather_shape=gather_shape) - else: - return gather_from - - -def _tensor_gather_helper(gather_indices, - gather_from, - batch_size, - range_size, - gather_shape, - name=None): - """Helper for gathering the right indices from the tensor. - - This works by reshaping gather_from to gather_shape (e.g. [-1]) and then - gathering from that according to the gather_indices, which are offset by - the right amounts in order to preserve the batch order. - - Args: - gather_indices: The tensor indices that we use to gather. - gather_from: The tensor that we are gathering from. - batch_size: The input batch size. - range_size: The number of values in each range. Likely equal to beam_width. - gather_shape: What we should reshape gather_from to in order to preserve the - correct values. An example is when gather_from is the attention from an - AttentionWrapperState with shape [batch_size, beam_width, attention_size]. - There, we want to preserve the attention_size elements, so gather_shape is - [batch_size * beam_width, -1]. Then, upon reshape, we still have the - attention_size as desired. - name: The tensor name for set of operations. By default this is - 'tensor_gather_helper'. The final output is named 'output'. - - Returns: - output: Gathered tensor of shape tf.shape(gather_from)[:1+len(gather_shape)] - """ - with ops.name_scope(name, "tensor_gather_helper"): - range_ = array_ops.expand_dims(math_ops.range(batch_size) * range_size, 1) - gather_indices = array_ops.reshape(gather_indices + range_, [-1]) - output = array_ops.gather( - array_ops.reshape(gather_from, gather_shape), gather_indices) - final_shape = array_ops.shape(gather_from)[:1 + len(gather_shape)] - static_batch_size = tensor_util.constant_value(batch_size) - final_static_shape = ( - tensor_shape.TensorShape([static_batch_size]).concatenate( - gather_from.shape[1:1 + len(gather_shape)])) - output = array_ops.reshape(output, final_shape, name="output") - output.set_shape(final_static_shape) - return output diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_ops.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_ops.py deleted file mode 100644 index 7d9fcc0c90a..00000000000 --- a/tensorflow/contrib/seq2seq/python/ops/beam_search_ops.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Beam Search helper ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.seq2seq.ops import gen_beam_search_ops -from tensorflow.contrib.util import loader -from tensorflow.python.platform import resource_loader - -_beam_search_ops_so = loader.load_op_library( - resource_loader.get_path_to_datafile("_beam_search_ops.so")) - -gather_tree = gen_beam_search_ops.gather_tree diff --git a/tensorflow/contrib/seq2seq/python/ops/decoder.py b/tensorflow/contrib/seq2seq/python/ops/decoder.py deleted file mode 100644 index 6543f09e40d..00000000000 --- a/tensorflow/contrib/seq2seq/python/ops/decoder.py +++ /dev/null @@ -1,499 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Seq2seq layer operations for use in neural networks.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc -import six - -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.keras import layers -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import control_flow_util -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import rnn -from tensorflow.python.ops import rnn_cell_impl -from tensorflow.python.ops import tensor_array_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.util import nest - - -__all__ = ["Decoder", "dynamic_decode"] - - -_transpose_batch_time = rnn._transpose_batch_time # pylint: disable=protected-access -_zero_state_tensors = rnn_cell_impl._zero_state_tensors # pylint: disable=protected-access - - -@six.add_metaclass(abc.ABCMeta) -class Decoder(object): - """An RNN Decoder abstract interface object. - - Concepts used by this interface: - - `inputs`: (structure of) tensors and TensorArrays that is passed as input to - the RNNCell composing the decoder, at each time step. - - `state`: (structure of) tensors and TensorArrays that is passed to the - RNNCell instance as the state. - - `finished`: boolean tensor telling whether each sequence in the batch is - finished. - - `outputs`: Instance of BasicDecoderOutput. Result of the decoding, at each - time step. - """ - - @property - def batch_size(self): - """The batch size of input values.""" - raise NotImplementedError - - @property - def output_size(self): - """A (possibly nested tuple of...) integer[s] or `TensorShape` object[s].""" - raise NotImplementedError - - @property - def output_dtype(self): - """A (possibly nested tuple of...) dtype[s].""" - raise NotImplementedError - - @abc.abstractmethod - def initialize(self, name=None): - """Called before any decoding iterations. - - This methods must compute initial input values and initial state. - - Args: - name: Name scope for any created operations. - - Returns: - `(finished, initial_inputs, initial_state)`: initial values of - 'finished' flags, inputs and state. - """ - raise NotImplementedError - - @abc.abstractmethod - def step(self, time, inputs, state, name=None): - """Called per step of decoding (but only once for dynamic decoding). - - Args: - time: Scalar `int32` tensor. Current step number. - inputs: RNNCell input (possibly nested tuple of) tensor[s] for this time - step. - state: RNNCell state (possibly nested tuple of) tensor[s] from previous - time step. - name: Name scope for any created operations. - - Returns: - `(outputs, next_state, next_inputs, finished)`: `outputs` is an object - containing the decoder output, `next_state` is a (structure of) state - tensors and TensorArrays, `next_inputs` is the tensor that should be used - as input for the next step, `finished` is a boolean tensor telling whether - the sequence is complete, for each sequence in the batch. - """ - raise NotImplementedError - - def finalize(self, outputs, final_state, sequence_lengths): - """Called after decoding iterations complete. - - Args: - outputs: RNNCell outputs (possibly nested tuple of) tensor[s] for all time - steps. - final_state: RNNCell final state (possibly nested tuple of) tensor[s] for - last time step. - sequence_lengths: 1-D `int32` tensor containing lengths of each sequence. - - Returns: - `(final_outputs, final_state)`: `final_outputs` is an object containing - the final decoder output, `final_state` is a (structure of) state tensors - and TensorArrays. - """ - raise NotImplementedError - - @property - def tracks_own_finished(self): - """Describes whether the Decoder keeps track of finished states. - - Most decoders will emit a true/false `finished` value independently - at each time step. In this case, the `dynamic_decode` function keeps track - of which batch entries are already finished, and performs a logical OR to - insert new batches to the finished set. - - Some decoders, however, shuffle batches / beams between time steps and - `dynamic_decode` will mix up the finished state across these entries because - it does not track the reshuffle across time steps. In this case, it is - up to the decoder to declare that it will keep track of its own finished - state by setting this property to `True`. - - Returns: - Python bool. - """ - return False - - -class BaseDecoder(layers.Layer): - """An RNN Decoder that is based on a Keras layer. - - Concepts used by this interface: - - `inputs`: (structure of) tensors and TensorArrays that is passed as input to - the RNNCell composing the decoder, at each time step. - - `state`: (structure of) tensors and TensorArrays that is passed to the - RNNCell instance as the state. - - `memory`: (sturecute of) tensors that is usually the full output of the - encoder, which will be used for the attention wrapper for the RNNCell. - - `finished`: boolean tensor telling whether each sequence in the batch is - finished. - - `outputs`: Instance of BasicDecoderOutput. Result of the decoding, at each - time step. - """ - - def __init__(self, - output_time_major=False, - impute_finished=False, - maximum_iterations=None, - parallel_iterations=32, - swap_memory=False, - **kwargs): - self.output_time_major = output_time_major - self.impute_finished = impute_finished - self.maximum_iterations = maximum_iterations - self.parallel_iterations = parallel_iterations - self.swap_memory = swap_memory - super(BaseDecoder, self).__init__(**kwargs) - - def call(self, inputs, initial_state=None, **kwargs): - init_kwargs = kwargs - init_kwargs["initial_state"] = initial_state - return dynamic_decode(self, - output_time_major=self.output_time_major, - impute_finished=self.impute_finished, - maximum_iterations=self.maximum_iterations, - parallel_iterations=self.parallel_iterations, - swap_memory=self.swap_memory, - decoder_init_input=inputs, - decoder_init_kwargs=init_kwargs) - - @property - def batch_size(self): - """The batch size of input values.""" - raise NotImplementedError - - @property - def output_size(self): - """A (possibly nested tuple of...) integer[s] or `TensorShape` object[s].""" - raise NotImplementedError - - @property - def output_dtype(self): - """A (possibly nested tuple of...) dtype[s].""" - raise NotImplementedError - - def initialize(self, inputs, initial_state=None, **kwargs): - """Called before any decoding iterations. - - This methods must compute initial input values and initial state. - - Args: - inputs: (structure of) tensors that contains the input for the decoder. In - the normal case, its a tensor with shape [batch, timestep, embedding]. - initial_state: (structure of) tensors that contains the initial state for - the RNNCell. - **kwargs: Other arguments that are passed in from layer.call() method. It - could contains item like input sequence_length, or masking for input. - - Returns: - `(finished, initial_inputs, initial_state)`: initial values of - 'finished' flags, inputs and state. - """ - raise NotImplementedError - - def step(self, time, inputs, state): - """Called per step of decoding (but only once for dynamic decoding). - - Args: - time: Scalar `int32` tensor. Current step number. - inputs: RNNCell input (possibly nested tuple of) tensor[s] for this time - step. - state: RNNCell state (possibly nested tuple of) tensor[s] from previous - time step. - - Returns: - `(outputs, next_state, next_inputs, finished)`: `outputs` is an object - containing the decoder output, `next_state` is a (structure of) state - tensors and TensorArrays, `next_inputs` is the tensor that should be used - as input for the next step, `finished` is a boolean tensor telling whether - the sequence is complete, for each sequence in the batch. - """ - raise NotImplementedError - - def finalize(self, outputs, final_state, sequence_lengths): - raise NotImplementedError - - @property - def tracks_own_finished(self): - """Describes whether the Decoder keeps track of finished states. - - Most decoders will emit a true/false `finished` value independently - at each time step. In this case, the `dynamic_decode` function keeps track - of which batch entries are already finished, and performs a logical OR to - insert new batches to the finished set. - - Some decoders, however, shuffle batches / beams between time steps and - `dynamic_decode` will mix up the finished state across these entries because - it does not track the reshuffle across time steps. In this case, it is - up to the decoder to declare that it will keep track of its own finished - state by setting this property to `True`. - - Returns: - Python bool. - """ - return False - - # TODO(scottzhu): Add build/get_config/from_config and other layer methods. - - -def _create_zero_outputs(size, dtype, batch_size): - """Create a zero outputs Tensor structure.""" - def _create(s, d): - return _zero_state_tensors(s, batch_size, d) - - return nest.map_structure(_create, size, dtype) - - -def dynamic_decode(decoder, - output_time_major=False, - impute_finished=False, - maximum_iterations=None, - parallel_iterations=32, - swap_memory=False, - scope=None, - **kwargs): - """Perform dynamic decoding with `decoder`. - - Calls initialize() once and step() repeatedly on the Decoder object. - - Args: - decoder: A `Decoder` instance. - output_time_major: Python boolean. Default: `False` (batch major). If - `True`, outputs are returned as time major tensors (this mode is faster). - Otherwise, outputs are returned as batch major tensors (this adds extra - time to the computation). - impute_finished: Python boolean. If `True`, then states for batch - entries which are marked as finished get copied through and the - corresponding outputs get zeroed out. This causes some slowdown at - each time step, but ensures that the final state and outputs have - the correct values and that backprop ignores time steps that were - marked as finished. - maximum_iterations: `int32` scalar, maximum allowed number of decoding - steps. Default is `None` (decode until the decoder is fully done). - parallel_iterations: Argument passed to `tf.while_loop`. - swap_memory: Argument passed to `tf.while_loop`. - scope: Optional variable scope to use. - **kwargs: dict, other keyword arguments for dynamic_decode. It might contain - arguments for `BaseDecoder` to initialize, which takes all tensor inputs - during call(). - - Returns: - `(final_outputs, final_state, final_sequence_lengths)`. - - Raises: - TypeError: if `decoder` is not an instance of `Decoder`. - ValueError: if `maximum_iterations` is provided but is not a scalar. - """ - if not isinstance(decoder, (Decoder, BaseDecoder)): - raise TypeError("Expected decoder to be type Decoder, but saw: %s" % - type(decoder)) - - with variable_scope.variable_scope(scope, "decoder") as varscope: - # Determine context types. - ctxt = ops.get_default_graph()._get_control_flow_context() # pylint: disable=protected-access - is_xla = control_flow_util.GetContainingXLAContext(ctxt) is not None - in_while_loop = ( - control_flow_util.GetContainingWhileContext(ctxt) is not None) - # Properly cache variable values inside the while_loop. - # Don't set a caching device when running in a loop, since it is possible - # that train steps could be wrapped in a tf.while_loop. In that scenario - # caching prevents forward computations in loop iterations from re-reading - # the updated weights. - if not context.executing_eagerly() and not in_while_loop: - if varscope.caching_device is None: - varscope.set_caching_device(lambda op: op.device) - - if maximum_iterations is not None: - maximum_iterations = ops.convert_to_tensor( - maximum_iterations, dtype=dtypes.int32, name="maximum_iterations") - if maximum_iterations.get_shape().ndims != 0: - raise ValueError("maximum_iterations must be a scalar") - - if isinstance(decoder, Decoder): - initial_finished, initial_inputs, initial_state = decoder.initialize() - else: - # For BaseDecoder that takes tensor inputs during call. - decoder_init_input = kwargs.pop("decoder_init_input", None) - decoder_init_kwargs = kwargs.pop("decoder_init_kwargs", {}) - initial_finished, initial_inputs, initial_state = decoder.initialize( - decoder_init_input, **decoder_init_kwargs) - - zero_outputs = _create_zero_outputs(decoder.output_size, - decoder.output_dtype, - decoder.batch_size) - - if is_xla and maximum_iterations is None: - raise ValueError("maximum_iterations is required for XLA compilation.") - if maximum_iterations is not None: - initial_finished = math_ops.logical_or( - initial_finished, 0 >= maximum_iterations) - initial_sequence_lengths = array_ops.zeros_like( - initial_finished, dtype=dtypes.int32) - initial_time = constant_op.constant(0, dtype=dtypes.int32) - - def _shape(batch_size, from_shape): - if (not isinstance(from_shape, tensor_shape.TensorShape) or - from_shape.ndims == 0): - return None - else: - batch_size = tensor_util.constant_value( - ops.convert_to_tensor( - batch_size, name="batch_size")) - return tensor_shape.TensorShape([batch_size]).concatenate(from_shape) - - dynamic_size = maximum_iterations is None or not is_xla - - def _create_ta(s, d): - return tensor_array_ops.TensorArray( - dtype=d, - size=0 if dynamic_size else maximum_iterations, - dynamic_size=dynamic_size, - element_shape=_shape(decoder.batch_size, s)) - - initial_outputs_ta = nest.map_structure(_create_ta, decoder.output_size, - decoder.output_dtype) - - def condition(unused_time, unused_outputs_ta, unused_state, unused_inputs, - finished, unused_sequence_lengths): - return math_ops.logical_not(math_ops.reduce_all(finished)) - - def body(time, outputs_ta, state, inputs, finished, sequence_lengths): - """Internal while_loop body. - - Args: - time: scalar int32 tensor. - outputs_ta: structure of TensorArray. - state: (structure of) state tensors and TensorArrays. - inputs: (structure of) input tensors. - finished: bool tensor (keeping track of what's finished). - sequence_lengths: int32 tensor (keeping track of time of finish). - - Returns: - `(time + 1, outputs_ta, next_state, next_inputs, next_finished, - next_sequence_lengths)`. - ``` - """ - (next_outputs, decoder_state, next_inputs, - decoder_finished) = decoder.step(time, inputs, state) - decoder_state_sequence_lengths = False - if decoder.tracks_own_finished: - next_finished = decoder_finished - lengths = getattr(decoder_state, "lengths", None) - if lengths is not None: - # sequence lengths are provided by decoder_state.lengths; overwrite - # our sequence lengths. - decoder_state_sequence_lengths = True - sequence_lengths = math_ops.cast(lengths, dtypes.int32) - else: - next_finished = math_ops.logical_or(decoder_finished, finished) - - if decoder_state_sequence_lengths: - # Just pass something through the loop; at the next iteration we'll pull - # the sequence lengths from the decoder_state again. - next_sequence_lengths = sequence_lengths - else: - next_sequence_lengths = array_ops.where( - math_ops.logical_not(finished), - array_ops.fill(array_ops.shape(sequence_lengths), time + 1), - sequence_lengths) - - nest.assert_same_structure(state, decoder_state) - nest.assert_same_structure(outputs_ta, next_outputs) - nest.assert_same_structure(inputs, next_inputs) - - # Zero out output values past finish - if impute_finished: - emit = nest.map_structure( - lambda out, zero: array_ops.where(finished, zero, out), - next_outputs, - zero_outputs) - else: - emit = next_outputs - - # Copy through states past finish - def _maybe_copy_state(new, cur): - # TensorArrays and scalar states get passed through. - if isinstance(cur, tensor_array_ops.TensorArray): - pass_through = True - else: - new.set_shape(cur.shape) - pass_through = (new.shape.ndims == 0) - return new if pass_through else array_ops.where(finished, cur, new) - - if impute_finished: - next_state = nest.map_structure( - _maybe_copy_state, decoder_state, state) - else: - next_state = decoder_state - - outputs_ta = nest.map_structure(lambda ta, out: ta.write(time, out), - outputs_ta, emit) - return (time + 1, outputs_ta, next_state, next_inputs, next_finished, - next_sequence_lengths) - - res = control_flow_ops.while_loop( - condition, - body, - loop_vars=( - initial_time, - initial_outputs_ta, - initial_state, - initial_inputs, - initial_finished, - initial_sequence_lengths, - ), - parallel_iterations=parallel_iterations, - maximum_iterations=maximum_iterations, - swap_memory=swap_memory) - - final_outputs_ta = res[1] - final_state = res[2] - final_sequence_lengths = res[5] - - final_outputs = nest.map_structure(lambda ta: ta.stack(), final_outputs_ta) - - try: - final_outputs, final_state = decoder.finalize( - final_outputs, final_state, final_sequence_lengths) - except NotImplementedError: - pass - - if not output_time_major: - final_outputs = nest.map_structure(_transpose_batch_time, final_outputs) - - return final_outputs, final_state, final_sequence_lengths diff --git a/tensorflow/contrib/seq2seq/python/ops/helper.py b/tensorflow/contrib/seq2seq/python/ops/helper.py deleted file mode 100644 index 033c2eb0801..00000000000 --- a/tensorflow/contrib/seq2seq/python/ops/helper.py +++ /dev/null @@ -1,736 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A library of helpers for use with SamplingDecoders. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - -from tensorflow.contrib.seq2seq.python.ops import decoder -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import tensor_array_ops -from tensorflow.python.util import nest - -__all__ = [ - "Helper", - "TrainingHelper", - "GreedyEmbeddingHelper", - "SampleEmbeddingHelper", - "CustomHelper", - "ScheduledEmbeddingTrainingHelper", - "ScheduledOutputTrainingHelper", - "InferenceHelper", -] - -_transpose_batch_time = decoder._transpose_batch_time # pylint: disable=protected-access - - -# The following sample functions (_call_sampler, bernoulli_sample, -# categorical_sample) mimic TensorFlow Probability distribution semantics. - - -def _call_sampler(sample_n_fn, sample_shape, name=None): - """Reshapes vector of samples.""" - with ops.name_scope(name, "call_sampler", values=[sample_shape]): - sample_shape = ops.convert_to_tensor( - sample_shape, dtype=dtypes.int32, name="sample_shape") - # Ensure sample_shape is a vector (vs just a scalar). - pad = math_ops.cast(math_ops.equal(array_ops.rank(sample_shape), 0), - dtypes.int32) - sample_shape = array_ops.reshape( - sample_shape, - array_ops.pad(array_ops.shape(sample_shape), - paddings=[[pad, 0]], - constant_values=1)) - samples = sample_n_fn(math_ops.reduce_prod(sample_shape)) - batch_event_shape = array_ops.shape(samples)[1:] - final_shape = array_ops.concat([sample_shape, batch_event_shape], 0) - return array_ops.reshape(samples, final_shape) - - -def bernoulli_sample(probs=None, logits=None, dtype=dtypes.int32, - sample_shape=(), seed=None): - """Samples from Bernoulli distribution.""" - if probs is None: - probs = math_ops.sigmoid(logits, name="probs") - else: - probs = ops.convert_to_tensor(probs, name="probs") - batch_shape_tensor = array_ops.shape(probs) - def _sample_n(n): - """Sample vector of Bernoullis.""" - new_shape = array_ops.concat([[n], batch_shape_tensor], 0) - uniform = random_ops.random_uniform( - new_shape, seed=seed, dtype=probs.dtype) - return math_ops.cast(math_ops.less(uniform, probs), dtype) - return _call_sampler(_sample_n, sample_shape) - - -def categorical_sample(logits, dtype=dtypes.int32, - sample_shape=(), seed=None): - """Samples from categorical distribution.""" - logits = ops.convert_to_tensor(logits, name="logits") - event_size = array_ops.shape(logits)[-1] - batch_shape_tensor = array_ops.shape(logits)[:-1] - def _sample_n(n): - """Sample vector of categoricals.""" - if logits.shape.ndims == 2: - logits_2d = logits - else: - logits_2d = array_ops.reshape(logits, [-1, event_size]) - sample_dtype = dtypes.int64 if logits.dtype.size > 4 else dtypes.int32 - draws = random_ops.multinomial( - logits_2d, n, seed=seed, output_dtype=sample_dtype) - draws = array_ops.reshape( - array_ops.transpose(draws), - array_ops.concat([[n], batch_shape_tensor], 0)) - return math_ops.cast(draws, dtype) - return _call_sampler(_sample_n, sample_shape) - - -def _unstack_ta(inp): - return tensor_array_ops.TensorArray( - dtype=inp.dtype, size=array_ops.shape(inp)[0], - element_shape=inp.get_shape()[1:]).unstack(inp) - - -@six.add_metaclass(abc.ABCMeta) -class Helper(object): - """Interface for implementing sampling in seq2seq decoders. - - Helper instances are used by `BasicDecoder`. - """ - - @abc.abstractproperty - def batch_size(self): - """Batch size of tensor returned by `sample`. - - Returns a scalar int32 tensor. - """ - raise NotImplementedError("batch_size has not been implemented") - - @abc.abstractproperty - def sample_ids_shape(self): - """Shape of tensor returned by `sample`, excluding the batch dimension. - - Returns a `TensorShape`. - """ - raise NotImplementedError("sample_ids_shape has not been implemented") - - @abc.abstractproperty - def sample_ids_dtype(self): - """DType of tensor returned by `sample`. - - Returns a DType. - """ - raise NotImplementedError("sample_ids_dtype has not been implemented") - - @abc.abstractmethod - def initialize(self, name=None): - """Returns `(initial_finished, initial_inputs)`.""" - pass - - @abc.abstractmethod - def sample(self, time, outputs, state, name=None): - """Returns `sample_ids`.""" - pass - - @abc.abstractmethod - def next_inputs(self, time, outputs, state, sample_ids, name=None): - """Returns `(finished, next_inputs, next_state)`.""" - pass - - -class CustomHelper(Helper): - """Base abstract class that allows the user to customize sampling.""" - - def __init__(self, initialize_fn, sample_fn, next_inputs_fn, - sample_ids_shape=None, sample_ids_dtype=None): - """Initializer. - - Args: - initialize_fn: callable that returns `(finished, next_inputs)` - for the first iteration. - sample_fn: callable that takes `(time, outputs, state)` - and emits tensor `sample_ids`. - next_inputs_fn: callable that takes `(time, outputs, state, sample_ids)` - and emits `(finished, next_inputs, next_state)`. - sample_ids_shape: Either a list of integers, or a 1-D Tensor of type - `int32`, the shape of each value in the `sample_ids` batch. Defaults to - a scalar. - sample_ids_dtype: The dtype of the `sample_ids` tensor. Defaults to int32. - """ - self._initialize_fn = initialize_fn - self._sample_fn = sample_fn - self._next_inputs_fn = next_inputs_fn - self._batch_size = None - self._sample_ids_shape = tensor_shape.TensorShape(sample_ids_shape or []) - self._sample_ids_dtype = sample_ids_dtype or dtypes.int32 - - @property - def batch_size(self): - if self._batch_size is None: - raise ValueError("batch_size accessed before initialize was called") - return self._batch_size - - @property - def sample_ids_shape(self): - return self._sample_ids_shape - - @property - def sample_ids_dtype(self): - return self._sample_ids_dtype - - def initialize(self, name=None): - with ops.name_scope(name, "%sInitialize" % type(self).__name__): - (finished, next_inputs) = self._initialize_fn() - if self._batch_size is None: - self._batch_size = array_ops.size(finished) - return (finished, next_inputs) - - def sample(self, time, outputs, state, name=None): - with ops.name_scope( - name, "%sSample" % type(self).__name__, (time, outputs, state)): - return self._sample_fn(time=time, outputs=outputs, state=state) - - def next_inputs(self, time, outputs, state, sample_ids, name=None): - with ops.name_scope( - name, "%sNextInputs" % type(self).__name__, (time, outputs, state)): - return self._next_inputs_fn( - time=time, outputs=outputs, state=state, sample_ids=sample_ids) - - -class TrainingHelper(Helper): - """A helper for use during training. Only reads inputs. - - Returned sample_ids are the argmax of the RNN output logits. - """ - - def __init__(self, inputs, sequence_length, time_major=False, name=None): - """Initializer. - - Args: - inputs: A (structure of) input tensors. - sequence_length: An int32 vector tensor. - time_major: Python bool. Whether the tensors in `inputs` are time major. - If `False` (default), they are assumed to be batch major. - name: Name scope for any created operations. - - Raises: - ValueError: if `sequence_length` is not a 1D tensor. - """ - with ops.name_scope(name, "TrainingHelper", [inputs, sequence_length]): - inputs = ops.convert_to_tensor(inputs, name="inputs") - self._inputs = inputs - if not time_major: - inputs = nest.map_structure(_transpose_batch_time, inputs) - - self._input_tas = nest.map_structure(_unstack_ta, inputs) - self._sequence_length = ops.convert_to_tensor( - sequence_length, name="sequence_length") - if self._sequence_length.get_shape().ndims != 1: - raise ValueError( - "Expected sequence_length to be a vector, but received shape: %s" % - self._sequence_length.get_shape()) - - self._zero_inputs = nest.map_structure( - lambda inp: array_ops.zeros_like(inp[0, :]), inputs) - - self._batch_size = array_ops.size(sequence_length) - - @property - def inputs(self): - return self._inputs - - @property - def sequence_length(self): - return self._sequence_length - - @property - def batch_size(self): - return self._batch_size - - @property - def sample_ids_shape(self): - return tensor_shape.TensorShape([]) - - @property - def sample_ids_dtype(self): - return dtypes.int32 - - def initialize(self, name=None): - with ops.name_scope(name, "TrainingHelperInitialize"): - finished = math_ops.equal(0, self._sequence_length) - all_finished = math_ops.reduce_all(finished) - next_inputs = control_flow_ops.cond( - all_finished, lambda: self._zero_inputs, - lambda: nest.map_structure(lambda inp: inp.read(0), self._input_tas)) - return (finished, next_inputs) - - def sample(self, time, outputs, name=None, **unused_kwargs): - with ops.name_scope(name, "TrainingHelperSample", [time, outputs]): - sample_ids = math_ops.cast( - math_ops.argmax(outputs, axis=-1), dtypes.int32) - return sample_ids - - def next_inputs(self, time, outputs, state, name=None, **unused_kwargs): - """next_inputs_fn for TrainingHelper.""" - with ops.name_scope(name, "TrainingHelperNextInputs", - [time, outputs, state]): - next_time = time + 1 - finished = (next_time >= self._sequence_length) - all_finished = math_ops.reduce_all(finished) - def read_from_ta(inp): - return inp.read(next_time) - next_inputs = control_flow_ops.cond( - all_finished, lambda: self._zero_inputs, - lambda: nest.map_structure(read_from_ta, self._input_tas)) - return (finished, next_inputs, state) - - -class ScheduledEmbeddingTrainingHelper(TrainingHelper): - """A training helper that adds scheduled sampling. - - Returns -1s for sample_ids where no sampling took place; valid sample id - values elsewhere. - """ - - def __init__(self, inputs, sequence_length, embedding, sampling_probability, - time_major=False, seed=None, scheduling_seed=None, name=None): - """Initializer. - - Args: - inputs: A (structure of) input tensors. - sequence_length: An int32 vector tensor. - embedding: A callable that takes a vector tensor of `ids` (argmax ids), - or the `params` argument for `embedding_lookup`. - sampling_probability: A 0D `float32` tensor: the probability of sampling - categorically from the output ids instead of reading directly from the - inputs. - time_major: Python bool. Whether the tensors in `inputs` are time major. - If `False` (default), they are assumed to be batch major. - seed: The sampling seed. - scheduling_seed: The schedule decision rule sampling seed. - name: Name scope for any created operations. - - Raises: - ValueError: if `sampling_probability` is not a scalar or vector. - """ - with ops.name_scope(name, "ScheduledEmbeddingSamplingWrapper", - [embedding, sampling_probability]): - if callable(embedding): - self._embedding_fn = embedding - else: - self._embedding_fn = ( - lambda ids: embedding_ops.embedding_lookup(embedding, ids)) - self._sampling_probability = ops.convert_to_tensor( - sampling_probability, name="sampling_probability") - if self._sampling_probability.get_shape().ndims not in (0, 1): - raise ValueError( - "sampling_probability must be either a scalar or a vector. " - "saw shape: %s" % (self._sampling_probability.get_shape())) - self._seed = seed - self._scheduling_seed = scheduling_seed - super(ScheduledEmbeddingTrainingHelper, self).__init__( - inputs=inputs, - sequence_length=sequence_length, - time_major=time_major, - name=name) - - def initialize(self, name=None): - return super(ScheduledEmbeddingTrainingHelper, self).initialize(name=name) - - def sample(self, time, outputs, state, name=None): - with ops.name_scope(name, "ScheduledEmbeddingTrainingHelperSample", - [time, outputs, state]): - # Return -1s where we did not sample, and sample_ids elsewhere - select_sample = bernoulli_sample( - probs=self._sampling_probability, - dtype=dtypes.bool, - sample_shape=self.batch_size, - seed=self._scheduling_seed) - return array_ops.where( - select_sample, - categorical_sample(logits=outputs, seed=self._seed), - gen_array_ops.fill([self.batch_size], -1)) - - def next_inputs(self, time, outputs, state, sample_ids, name=None): - with ops.name_scope(name, "ScheduledEmbeddingTrainingHelperNextInputs", - [time, outputs, state, sample_ids]): - (finished, base_next_inputs, state) = ( - super(ScheduledEmbeddingTrainingHelper, self).next_inputs( - time=time, - outputs=outputs, - state=state, - sample_ids=sample_ids, - name=name)) - - def maybe_sample(): - """Perform scheduled sampling.""" - where_sampling = math_ops.cast( - array_ops.where(sample_ids > -1), dtypes.int32) - where_not_sampling = math_ops.cast( - array_ops.where(sample_ids <= -1), dtypes.int32) - sample_ids_sampling = array_ops.gather_nd(sample_ids, where_sampling) - inputs_not_sampling = array_ops.gather_nd( - base_next_inputs, where_not_sampling) - sampled_next_inputs = self._embedding_fn(sample_ids_sampling) - base_shape = array_ops.shape(base_next_inputs) - return (array_ops.scatter_nd(indices=where_sampling, - updates=sampled_next_inputs, - shape=base_shape) - + array_ops.scatter_nd(indices=where_not_sampling, - updates=inputs_not_sampling, - shape=base_shape)) - - all_finished = math_ops.reduce_all(finished) - next_inputs = control_flow_ops.cond( - all_finished, lambda: base_next_inputs, maybe_sample) - return (finished, next_inputs, state) - - -class ScheduledOutputTrainingHelper(TrainingHelper): - """A training helper that adds scheduled sampling directly to outputs. - - Returns False for sample_ids where no sampling took place; True elsewhere. - """ - - def __init__(self, inputs, sequence_length, sampling_probability, - time_major=False, seed=None, next_inputs_fn=None, - auxiliary_inputs=None, name=None): - """Initializer. - - Args: - inputs: A (structure) of input tensors. - sequence_length: An int32 vector tensor. - sampling_probability: A 0D `float32` tensor: the probability of sampling - from the outputs instead of reading directly from the inputs. - time_major: Python bool. Whether the tensors in `inputs` are time major. - If `False` (default), they are assumed to be batch major. - seed: The sampling seed. - next_inputs_fn: (Optional) callable to apply to the RNN outputs to create - the next input when sampling. If `None` (default), the RNN outputs will - be used as the next inputs. - auxiliary_inputs: An optional (structure of) auxiliary input tensors with - a shape that matches `inputs` in all but (potentially) the final - dimension. These tensors will be concatenated to the sampled output or - the `inputs` when not sampling for use as the next input. - name: Name scope for any created operations. - - Raises: - ValueError: if `sampling_probability` is not a scalar or vector. - """ - with ops.name_scope(name, "ScheduledOutputTrainingHelper", - [inputs, auxiliary_inputs, sampling_probability]): - self._sampling_probability = ops.convert_to_tensor( - sampling_probability, name="sampling_probability") - if self._sampling_probability.get_shape().ndims not in (0, 1): - raise ValueError( - "sampling_probability must be either a scalar or a vector. " - "saw shape: %s" % (self._sampling_probability.get_shape())) - - if auxiliary_inputs is None: - maybe_concatenated_inputs = inputs - else: - inputs = ops.convert_to_tensor(inputs, name="inputs") - auxiliary_inputs = ops.convert_to_tensor( - auxiliary_inputs, name="auxiliary_inputs") - maybe_concatenated_inputs = nest.map_structure( - lambda x, y: array_ops.concat((x, y), -1), - inputs, auxiliary_inputs) - if not time_major: - auxiliary_inputs = nest.map_structure( - _transpose_batch_time, auxiliary_inputs) - - self._auxiliary_input_tas = ( - nest.map_structure(_unstack_ta, auxiliary_inputs) - if auxiliary_inputs is not None else None) - - self._seed = seed - - self._next_inputs_fn = next_inputs_fn - - super(ScheduledOutputTrainingHelper, self).__init__( - inputs=maybe_concatenated_inputs, - sequence_length=sequence_length, - time_major=time_major, - name=name) - - def initialize(self, name=None): - return super(ScheduledOutputTrainingHelper, self).initialize(name=name) - - def sample(self, time, outputs, state, name=None): - with ops.name_scope(name, "ScheduledOutputTrainingHelperSample", - [time, outputs, state]): - return bernoulli_sample( - probs=self._sampling_probability, - sample_shape=self.batch_size, - seed=self._seed) - - def next_inputs(self, time, outputs, state, sample_ids, name=None): - with ops.name_scope(name, "ScheduledOutputTrainingHelperNextInputs", - [time, outputs, state, sample_ids]): - (finished, base_next_inputs, state) = ( - super(ScheduledOutputTrainingHelper, self).next_inputs( - time=time, - outputs=outputs, - state=state, - sample_ids=sample_ids, - name=name)) - sample_ids = math_ops.cast(sample_ids, dtypes.bool) - - def maybe_sample(): - """Perform scheduled sampling.""" - - def maybe_concatenate_auxiliary_inputs(outputs_, indices=None): - """Concatenate outputs with auxiliary inputs, if they exist.""" - if self._auxiliary_input_tas is None: - return outputs_ - - next_time = time + 1 - auxiliary_inputs = nest.map_structure( - lambda ta: ta.read(next_time), self._auxiliary_input_tas) - if indices is not None: - auxiliary_inputs = array_ops.gather_nd(auxiliary_inputs, indices) - return nest.map_structure( - lambda x, y: array_ops.concat((x, y), -1), - outputs_, auxiliary_inputs) - - if self._next_inputs_fn is None: - return array_ops.where( - sample_ids, maybe_concatenate_auxiliary_inputs(outputs), - base_next_inputs) - - where_sampling = math_ops.cast( - array_ops.where(sample_ids), dtypes.int32) - where_not_sampling = math_ops.cast( - array_ops.where(math_ops.logical_not(sample_ids)), dtypes.int32) - outputs_sampling = array_ops.gather_nd(outputs, where_sampling) - inputs_not_sampling = array_ops.gather_nd(base_next_inputs, - where_not_sampling) - sampled_next_inputs = maybe_concatenate_auxiliary_inputs( - self._next_inputs_fn(outputs_sampling), where_sampling) - - base_shape = array_ops.shape(base_next_inputs) - return (array_ops.scatter_nd(indices=where_sampling, - updates=sampled_next_inputs, - shape=base_shape) - + array_ops.scatter_nd(indices=where_not_sampling, - updates=inputs_not_sampling, - shape=base_shape)) - - all_finished = math_ops.reduce_all(finished) - no_samples = math_ops.logical_not(math_ops.reduce_any(sample_ids)) - next_inputs = control_flow_ops.cond( - math_ops.logical_or(all_finished, no_samples), - lambda: base_next_inputs, maybe_sample) - return (finished, next_inputs, state) - - -class GreedyEmbeddingHelper(Helper): - """A helper for use during inference. - - Uses the argmax of the output (treated as logits) and passes the - result through an embedding layer to get the next input. - """ - - def __init__(self, embedding, start_tokens, end_token): - """Initializer. - - Args: - embedding: A callable that takes a vector tensor of `ids` (argmax ids), - or the `params` argument for `embedding_lookup`. The returned tensor - will be passed to the decoder input. - start_tokens: `int32` vector shaped `[batch_size]`, the start tokens. - end_token: `int32` scalar, the token that marks end of decoding. - - Raises: - ValueError: if `start_tokens` is not a 1D tensor or `end_token` is not a - scalar. - """ - if callable(embedding): - self._embedding_fn = embedding - else: - self._embedding_fn = ( - lambda ids: embedding_ops.embedding_lookup(embedding, ids)) - - self._start_tokens = ops.convert_to_tensor( - start_tokens, dtype=dtypes.int32, name="start_tokens") - self._end_token = ops.convert_to_tensor( - end_token, dtype=dtypes.int32, name="end_token") - if self._start_tokens.get_shape().ndims != 1: - raise ValueError("start_tokens must be a vector") - self._batch_size = array_ops.size(start_tokens) - if self._end_token.get_shape().ndims != 0: - raise ValueError("end_token must be a scalar") - self._start_inputs = self._embedding_fn(self._start_tokens) - - @property - def batch_size(self): - return self._batch_size - - @property - def sample_ids_shape(self): - return tensor_shape.TensorShape([]) - - @property - def sample_ids_dtype(self): - return dtypes.int32 - - def initialize(self, name=None): - finished = array_ops.tile([False], [self._batch_size]) - return (finished, self._start_inputs) - - def sample(self, time, outputs, state, name=None): - """sample for GreedyEmbeddingHelper.""" - del time, state # unused by sample_fn - # Outputs are logits, use argmax to get the most probable id - if not isinstance(outputs, ops.Tensor): - raise TypeError("Expected outputs to be a single Tensor, got: %s" % - type(outputs)) - sample_ids = math_ops.argmax(outputs, axis=-1, output_type=dtypes.int32) - return sample_ids - - def next_inputs(self, time, outputs, state, sample_ids, name=None): - """next_inputs_fn for GreedyEmbeddingHelper.""" - del time, outputs # unused by next_inputs_fn - finished = math_ops.equal(sample_ids, self._end_token) - all_finished = math_ops.reduce_all(finished) - next_inputs = control_flow_ops.cond( - all_finished, - # If we're finished, the next_inputs value doesn't matter - lambda: self._start_inputs, - lambda: self._embedding_fn(sample_ids)) - return (finished, next_inputs, state) - - -class SampleEmbeddingHelper(GreedyEmbeddingHelper): - """A helper for use during inference. - - Uses sampling (from a distribution) instead of argmax and passes the - result through an embedding layer to get the next input. - """ - - def __init__(self, embedding, start_tokens, end_token, - softmax_temperature=None, seed=None): - """Initializer. - - Args: - embedding: A callable that takes a vector tensor of `ids` (argmax ids), - or the `params` argument for `embedding_lookup`. The returned tensor - will be passed to the decoder input. - start_tokens: `int32` vector shaped `[batch_size]`, the start tokens. - end_token: `int32` scalar, the token that marks end of decoding. - softmax_temperature: (Optional) `float32` scalar, value to divide the - logits by before computing the softmax. Larger values (above 1.0) result - in more random samples, while smaller values push the sampling - distribution towards the argmax. Must be strictly greater than 0. - Defaults to 1.0. - seed: (Optional) The sampling seed. - - Raises: - ValueError: if `start_tokens` is not a 1D tensor or `end_token` is not a - scalar. - """ - super(SampleEmbeddingHelper, self).__init__( - embedding, start_tokens, end_token) - self._softmax_temperature = softmax_temperature - self._seed = seed - - def sample(self, time, outputs, state, name=None): - """sample for SampleEmbeddingHelper.""" - del time, state # unused by sample_fn - # Outputs are logits, we sample instead of argmax (greedy). - if not isinstance(outputs, ops.Tensor): - raise TypeError("Expected outputs to be a single Tensor, got: %s" % - type(outputs)) - if self._softmax_temperature is None: - logits = outputs - else: - logits = outputs / self._softmax_temperature - - sample_ids = categorical_sample(logits=logits, seed=self._seed) - - return sample_ids - - -class InferenceHelper(Helper): - """A helper to use during inference with a custom sampling function.""" - - def __init__(self, sample_fn, sample_shape, sample_dtype, - start_inputs, end_fn, next_inputs_fn=None): - """Initializer. - - Args: - sample_fn: A callable that takes `outputs` and emits tensor `sample_ids`. - sample_shape: Either a list of integers, or a 1-D Tensor of type `int32`, - the shape of the each sample in the batch returned by `sample_fn`. - sample_dtype: the dtype of the sample returned by `sample_fn`. - start_inputs: The initial batch of inputs. - end_fn: A callable that takes `sample_ids` and emits a `bool` vector - shaped `[batch_size]` indicating whether each sample is an end token. - next_inputs_fn: (Optional) A callable that takes `sample_ids` and returns - the next batch of inputs. If not provided, `sample_ids` is used as the - next batch of inputs. - """ - self._sample_fn = sample_fn - self._end_fn = end_fn - self._sample_shape = tensor_shape.TensorShape(sample_shape) - self._sample_dtype = sample_dtype - self._next_inputs_fn = next_inputs_fn - self._batch_size = array_ops.shape(start_inputs)[0] - self._start_inputs = ops.convert_to_tensor( - start_inputs, name="start_inputs") - - @property - def batch_size(self): - return self._batch_size - - @property - def sample_ids_shape(self): - return self._sample_shape - - @property - def sample_ids_dtype(self): - return self._sample_dtype - - def initialize(self, name=None): - finished = array_ops.tile([False], [self._batch_size]) - return (finished, self._start_inputs) - - def sample(self, time, outputs, state, name=None): - del time, state # unused by sample - return self._sample_fn(outputs) - - def next_inputs(self, time, outputs, state, sample_ids, name=None): - del time, outputs # unused by next_inputs - if self._next_inputs_fn is None: - next_inputs = sample_ids - else: - next_inputs = self._next_inputs_fn(sample_ids) - finished = self._end_fn(sample_ids) - return (finished, next_inputs, state) diff --git a/tensorflow/contrib/seq2seq/python/ops/loss.py b/tensorflow/contrib/seq2seq/python/ops/loss.py deleted file mode 100644 index 0fbfd618703..00000000000 --- a/tensorflow/contrib/seq2seq/python/ops/loss.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Seq2seq loss operations for use in sequence models. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.keras.losses import Loss -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops - -__all__ = ["sequence_loss", "SequenceLoss"] - - -def sequence_loss(logits, - targets, - weights, - average_across_timesteps=True, - average_across_batch=True, - sum_over_timesteps=False, - sum_over_batch=False, - softmax_loss_function=None, - name=None): - """Weighted cross-entropy loss for a sequence of logits. - - Depending on the values of `average_across_timesteps` / `sum_over_timesteps` - and `average_across_batch` / `sum_over_batch`, the return Tensor will have - rank 0, 1, or 2 as these arguments reduce the cross-entropy at each target, - which has shape `[batch_size, sequence_length]`, over their respective - dimensions. For example, if `average_across_timesteps` is `True` and - `average_across_batch` is `False`, then the return Tensor will have shape - `[batch_size]`. - - Note that `average_across_timesteps` and `sum_over_timesteps` cannot be True - at same time. Same for `average_across_batch` and `sum_over_batch`. - - The recommended loss reduction in tf 2.0 has been changed to sum_over, instead - of weighted average. User are recommend to use `sum_over_timesteps` and - `sum_over_batch` for reduction. - - Args: - logits: A Tensor of shape - `[batch_size, sequence_length, num_decoder_symbols]` and dtype float. - The logits correspond to the prediction across all classes at each - timestep. - targets: A Tensor of shape `[batch_size, sequence_length]` and dtype - int. The target represents the true class at each timestep. - weights: A Tensor of shape `[batch_size, sequence_length]` and dtype - float. `weights` constitutes the weighting of each prediction in the - sequence. When using `weights` as masking, set all valid timesteps to 1 - and all padded timesteps to 0, e.g. a mask returned by `tf.sequence_mask`. - average_across_timesteps: If set, sum the cost across the sequence - dimension and divide the cost by the total label weight across timesteps. - average_across_batch: If set, sum the cost across the batch dimension and - divide the returned cost by the batch size. - sum_over_timesteps: If set, sum the cost across the sequence dimension and - divide the size of the sequence. Note that any element with 0 weights will - be excluded from size calculation. - sum_over_batch: if set, sum the cost across the batch dimension and divide - the total cost by the batch size. Not that any element with 0 weights will - be excluded from size calculation. - softmax_loss_function: Function (labels, logits) -> loss-batch - to be used instead of the standard softmax (the default if this is None). - **Note that to avoid confusion, it is required for the function to accept - named arguments.** - name: Optional name for this operation, defaults to "sequence_loss". - - Returns: - A float Tensor of rank 0, 1, or 2 depending on the - `average_across_timesteps` and `average_across_batch` arguments. By default, - it has rank 0 (scalar) and is the weighted average cross-entropy - (log-perplexity) per symbol. - - Raises: - ValueError: logits does not have 3 dimensions or targets does not have 2 - dimensions or weights does not have 2 dimensions. - """ - if len(logits.get_shape()) != 3: - raise ValueError("Logits must be a " - "[batch_size x sequence_length x logits] tensor") - if len(targets.get_shape()) != 2: - raise ValueError("Targets must be a [batch_size x sequence_length] tensor") - if len(weights.get_shape()) != 2: - raise ValueError("Weights must be a [batch_size x sequence_length] tensor") - if average_across_timesteps and sum_over_timesteps: - raise ValueError("average_across_timesteps and sum_over_timesteps cannot " - "be set to True at same time.") - if average_across_batch and sum_over_batch: - raise ValueError("average_across_batch and sum_over_batch cannot be set " - "to True at same time.") - with ops.name_scope(name, "sequence_loss", [logits, targets, weights]): - num_classes = array_ops.shape(logits)[2] - logits_flat = array_ops.reshape(logits, [-1, num_classes]) - targets = array_ops.reshape(targets, [-1]) - if softmax_loss_function is None: - crossent = nn_ops.sparse_softmax_cross_entropy_with_logits( - labels=targets, logits=logits_flat) - else: - crossent = softmax_loss_function(labels=targets, logits=logits_flat) - crossent *= array_ops.reshape(weights, [-1]) - if average_across_timesteps and average_across_batch: - crossent = math_ops.reduce_sum(crossent) - total_size = math_ops.reduce_sum(weights) - crossent = math_ops.div_no_nan(crossent, total_size) - elif sum_over_timesteps and sum_over_batch: - crossent = math_ops.reduce_sum(crossent) - total_count = math_ops.cast(math_ops.count_nonzero(weights), - crossent.dtype) - crossent = math_ops.div_no_nan(crossent, total_count) - else: - crossent = array_ops.reshape(crossent, array_ops.shape(logits)[0:2]) - if average_across_timesteps or average_across_batch: - reduce_axis = [0] if average_across_batch else [1] - crossent = math_ops.reduce_sum(crossent, axis=reduce_axis) - total_size = math_ops.reduce_sum(weights, axis=reduce_axis) - crossent = math_ops.div_no_nan(crossent, total_size) - elif sum_over_timesteps or sum_over_batch: - reduce_axis = [0] if sum_over_batch else [1] - crossent = math_ops.reduce_sum(crossent, axis=reduce_axis) - total_count = math_ops.cast( - math_ops.count_nonzero(weights, axis=reduce_axis), - dtype=crossent.dtype) - crossent = math_ops.div_no_nan(crossent, total_count) - return crossent - - -class SequenceLoss(Loss): - """Weighted cross-entropy loss for a sequence of logits.""" - - def __init__(self, - average_across_timesteps=False, - average_across_batch=False, - sum_over_timesteps=True, - sum_over_batch=True, - softmax_loss_function=None, - name=None): - super(SequenceLoss, self).__init__(name=name) - self.average_across_timesteps = average_across_timesteps - self.average_across_batch = average_across_batch - self.sum_over_timesteps = sum_over_timesteps - self.sum_over_batch = sum_over_batch - self.softmax_loss_function = softmax_loss_function - - def __call__(self, y_true, y_pred, sample_weight=None): - """Override the parent __call__ to have a customized reduce behavior.""" - return sequence_loss(y_pred, y_true, sample_weight, - average_across_timesteps=self.average_across_timesteps, - average_across_batch=self.average_across_batch, - sum_over_timesteps=self.sum_over_timesteps, - sum_over_batch=self.sum_over_batch, - softmax_loss_function=self.softmax_loss_function, - name=self.name) - - def call(self, y_true, y_pred): - # Skip this method since the __call__ contains real implementation. - pass diff --git a/tensorflow/contrib/seq2seq/python/ops/sampler.py b/tensorflow/contrib/seq2seq/python/ops/sampler.py deleted file mode 100644 index 9e3e48b3bc6..00000000000 --- a/tensorflow/contrib/seq2seq/python/ops/sampler.py +++ /dev/null @@ -1,765 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A library of sampler for use with SamplingDecoders.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - -from tensorflow.contrib.seq2seq.python.ops import decoder -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import tensor_array_ops -from tensorflow.python.util import nest - -__all__ = [ - "Sampler", - "TrainingSampler", - "GreedyEmbeddingSampler", - "SampleEmbeddingSampler", - "CustomSampler", - "ScheduledEmbeddingTrainingSampler", - "ScheduledOutputTrainingSampler", - "InferenceSampler", -] - -_transpose_batch_time = decoder._transpose_batch_time # pylint: disable=protected-access - - -@six.add_metaclass(abc.ABCMeta) -class Sampler(object): - """Interface for implementing sampling in seq2seq decoders. - - Sampler instances are used by `BasicDecoder`. The normal usage of a sampler is - like below: - sampler = Sampler(init_args) - (initial_finished, initial_inputs) = sampler.initialize(input_tensors) - for time_step in range(time): - cell_output, cell_state = cell.call(cell_input, previous_state) - sample_ids = sampler.sample(time_step, cell_output, cell_state) - (finished, next_inputs, next_state) = sampler.next_inputs( - time_step,cell_output, cell_state) - - Note that all the tensor input should not be feed to Sampler as __init__() - parameters, instead, they should be feed by decoders via initialize(). - """ - - @abc.abstractmethod - def initialize(self, inputs, **kwargs): - """initialize the sampler with the input tensors. - - This method suppose to be only invoke once before the calling other methods - of the Sampler. - - Args: - inputs: A (structure of) input tensors, it could be a nested tuple or a - single tensor. - **kwargs: Other kwargs for initialization. It could contain tensors like - mask for inputs, or non tensor parameter. - - Returns: - `(initial_finished, initial_inputs)`. - """ - pass - - @abc.abstractmethod - def sample(self, time, outputs, state): - """Returns `sample_ids`.""" - pass - - @abc.abstractmethod - def next_inputs(self, time, outputs, state, sample_ids): - """Returns `(finished, next_inputs, next_state)`.""" - pass - - @abc.abstractproperty - def batch_size(self): - """Batch size of tensor returned by `sample`. - - Returns a scalar int32 tensor. The return value might not available before - the invocation of initialize(), in this case, ValueError is raised. - """ - raise NotImplementedError("batch_size has not been implemented") - - @abc.abstractproperty - def sample_ids_shape(self): - """Shape of tensor returned by `sample`, excluding the batch dimension. - - Returns a `TensorShape`. The return value might not available before the - invocation of initialize(). - """ - raise NotImplementedError("sample_ids_shape has not been implemented") - - @abc.abstractproperty - def sample_ids_dtype(self): - """DType of tensor returned by `sample`. - - Returns a DType. The return value might not available before the - invocation of initialize(). - """ - raise NotImplementedError("sample_ids_dtype has not been implemented") - - -class CustomSampler(Sampler): - """Base abstract class that allows the user to customize sampling.""" - - def __init__(self, - initialize_fn, - sample_fn, - next_inputs_fn, - sample_ids_shape=None, - sample_ids_dtype=None): - """Initializer. - - Args: - initialize_fn: callable that returns `(finished, next_inputs)` for the - first iteration. - sample_fn: callable that takes `(time, outputs, state)` and emits tensor - `sample_ids`. - next_inputs_fn: callable that takes `(time, outputs, state, sample_ids)` - and emits `(finished, next_inputs, next_state)`. - sample_ids_shape: Either a list of integers, or a 1-D Tensor of type - `int32`, the shape of each value in the `sample_ids` batch. Defaults to - a scalar. - sample_ids_dtype: The dtype of the `sample_ids` tensor. Defaults to int32. - """ - self._initialize_fn = initialize_fn - self._sample_fn = sample_fn - self._next_inputs_fn = next_inputs_fn - self._batch_size = None - self._sample_ids_shape = tensor_shape.TensorShape(sample_ids_shape or []) - self._sample_ids_dtype = sample_ids_dtype or dtypes.int32 - - @property - def batch_size(self): - if self._batch_size is None: - raise ValueError("batch_size accessed before initialize was called") - return self._batch_size - - @property - def sample_ids_shape(self): - return self._sample_ids_shape - - @property - def sample_ids_dtype(self): - return self._sample_ids_dtype - - def initialize(self, inputs, **kwargs): - (finished, next_inputs) = self._initialize_fn(inputs, **kwargs) - if self._batch_size is None: - self._batch_size = array_ops.size(finished) - return (finished, next_inputs) - - def sample(self, time, outputs, state): - return self._sample_fn(time=time, outputs=outputs, state=state) - - def next_inputs(self, time, outputs, state, sample_ids): - return self._next_inputs_fn( - time=time, outputs=outputs, state=state, sample_ids=sample_ids) - - -class TrainingSampler(Sampler): - """A Sampler for use during training. - - Only reads inputs. - - Returned sample_ids are the argmax of the RNN output logits. - """ - - def __init__(self, time_major=False): - """Initializer. - - Args: - time_major: Python bool. Whether the tensors in `inputs` are time major. - If `False` (default), they are assumed to be batch major. - - Raises: - ValueError: if `sequence_length` is not a 1D tensor. - """ - self.time_major = time_major - self._batch_size = None - - @property - def batch_size(self): - if self._batch_size is None: - raise ValueError("batch_size accessed before initialize was called") - return self._batch_size - - @property - def sample_ids_shape(self): - return tensor_shape.TensorShape([]) - - @property - def sample_ids_dtype(self): - return dtypes.int32 - - def initialize(self, inputs, sequence_length=None): - """Initialize the TrainSampler. - - Args: - inputs: A (structure of) input tensors. - sequence_length: An int32 vector tensor. - - Returns: - (finished, next_inputs), a tuple of two items. The first item is a boolean - vector to indicate whether the item in the batch has finished. The - second item is the first slide of input data based on the timestep - dimension (usually the second dim of the input). - """ - self.inputs = ops.convert_to_tensor(inputs, name="inputs") - if not self.time_major: - inputs = nest.map_structure(_transpose_batch_time, inputs) - - self.input_tas = nest.map_structure(_unstack_ta, inputs) - if sequence_length is None: - raise ValueError("sequence_length is required for TrainingSampler") - self.sequence_length = ops.convert_to_tensor( - sequence_length, name="sequence_length") - if self.sequence_length.get_shape().ndims != 1: - raise ValueError( - "Expected sequence_length to be a vector, but received shape: %s" % - self._sequence_length.get_shape()) - - self.zero_inputs = nest.map_structure( - lambda inp: array_ops.zeros_like(inp[0, :]), inputs) - - self._batch_size = array_ops.size(self.sequence_length) - - finished = math_ops.equal(0, self.sequence_length) - all_finished = math_ops.reduce_all(finished) - next_inputs = control_flow_ops.cond( - all_finished, - lambda: self.zero_inputs, - lambda: nest.map_structure(lambda inp: inp.read(0), self.input_tas)) - return (finished, next_inputs) - - def sample(self, time, outputs, state): - del state - sample_ids = math_ops.cast(math_ops.argmax(outputs, axis=-1), dtypes.int32) - return sample_ids - - def next_inputs(self, time, outputs, state, sample_ids): - del sample_ids - next_time = time + 1 - finished = (next_time >= self.sequence_length) - all_finished = math_ops.reduce_all(finished) - - def read_from_ta(inp): - return inp.read(next_time) - - next_inputs = control_flow_ops.cond( - all_finished, - lambda: self.zero_inputs, - lambda: nest.map_structure(read_from_ta, self.input_tas)) - return (finished, next_inputs, state) - - -class ScheduledEmbeddingTrainingSampler(TrainingSampler): - """A training sampler that adds scheduled sampling. - - Returns -1s for sample_ids where no sampling took place; valid sample id - values elsewhere. - """ - - def __init__(self, - sampling_probability, - embedding_fn=None, - time_major=False, - seed=None, - scheduling_seed=None): - """Initializer. - - Args: - sampling_probability: A `float32` 0-D or 1-D tensor: the probability of - sampling categorically from the output ids instead of reading directly - from the inputs. - embedding_fn: A callable that takes a vector tensor of `ids` (argmax ids), - or the `params` argument for `embedding_lookup`. - time_major: Python bool. Whether the tensors in `inputs` are time major. - If `False` (default), they are assumed to be batch major. - seed: The sampling seed. - scheduling_seed: The schedule decision rule sampling seed. - - Raises: - ValueError: if `sampling_probability` is not a scalar or vector. - """ - if callable(embedding_fn) or embedding_fn is None: - self.embedding_fn = embedding_fn - else: - raise ValueError("embedding_fn is expected to be callable, got %s" - % type(embedding_fn)) - self.sampling_probability = ops.convert_to_tensor( - sampling_probability, name="sampling_probability") - if self.sampling_probability.get_shape().ndims not in (0, 1): - raise ValueError( - "sampling_probability must be either a scalar or a vector. " - "saw shape: %s" % (self.sampling_probability.get_shape())) - self.seed = seed - self.scheduling_seed = scheduling_seed - super(ScheduledEmbeddingTrainingSampler, - self).__init__(time_major=time_major) - - def initialize(self, inputs, sequence_length=None, embedding=None): - if self.embedding_fn is None: - if embedding is None: - raise ValueError("embedding is required as a keyword argument for " - "ScheduledEmbeddingTrainingSampler") - self.embedding_fn = ( - lambda ids: embedding_ops.embedding_lookup(embedding, ids)) - return super(ScheduledEmbeddingTrainingSampler, self).initialize( - inputs, sequence_length=sequence_length) - - def sample(self, time, outputs, state): - del state - # Return -1s where we did not sample, and sample_ids elsewhere - select_sample = bernoulli_sample( - probs=self.sampling_probability, - dtype=dtypes.bool, - sample_shape=self.batch_size, - seed=self.scheduling_seed) - return array_ops.where(select_sample, - categorical_sample(logits=outputs, seed=self.seed), - gen_array_ops.fill([self.batch_size], -1)) - - def next_inputs(self, time, outputs, state, sample_ids): - (finished, base_next_inputs, state) = ( - super(ScheduledEmbeddingTrainingSampler, self).next_inputs( - time=time, outputs=outputs, state=state, sample_ids=sample_ids)) - - def maybe_sample(): - """Perform scheduled sampling.""" - where_sampling = math_ops.cast( - array_ops.where(sample_ids > -1), dtypes.int32) - where_not_sampling = math_ops.cast( - array_ops.where(sample_ids <= -1), dtypes.int32) - sample_ids_sampling = array_ops.gather_nd(sample_ids, where_sampling) - inputs_not_sampling = array_ops.gather_nd(base_next_inputs, - where_not_sampling) - sampled_next_inputs = self.embedding_fn(sample_ids_sampling) - base_shape = array_ops.shape(base_next_inputs) - return (array_ops.scatter_nd( - indices=where_sampling, updates=sampled_next_inputs, shape=base_shape) - + array_ops.scatter_nd( - indices=where_not_sampling, - updates=inputs_not_sampling, - shape=base_shape)) - - all_finished = math_ops.reduce_all(finished) - next_inputs = control_flow_ops.cond(all_finished, lambda: base_next_inputs, - maybe_sample) - return (finished, next_inputs, state) - - -class ScheduledOutputTrainingSampler(TrainingSampler): - """A training sampler that adds scheduled sampling directly to outputs. - - Returns False for sample_ids where no sampling took place; True elsewhere. - """ - - def __init__(self, - sampling_probability, - time_major=False, - seed=None, - next_inputs_fn=None): - """Initializer. - - Args: - sampling_probability: A `float32` scalar tensor: the probability of - sampling from the outputs instead of reading directly from the inputs. - time_major: Python bool. Whether the tensors in `inputs` are time major. - If `False` (default), they are assumed to be batch major. - seed: The sampling seed. - next_inputs_fn: (Optional) callable to apply to the RNN outputs to create - the next input when sampling. If `None` (default), the RNN outputs will - be used as the next inputs. - - Raises: - ValueError: if `sampling_probability` is not a scalar or vector. - """ - self.sampling_probability = ops.convert_to_tensor( - sampling_probability, name="sampling_probability") - if self.sampling_probability.get_shape().ndims not in (0, 1): - raise ValueError( - "sampling_probability must be either a scalar or a vector. " - "saw shape: %s" % (self._sampling_probability.get_shape())) - - self.seed = seed - self.next_inputs_fn = next_inputs_fn - - super(ScheduledOutputTrainingSampler, self).__init__(time_major=time_major) - - def initialize(self, inputs, sequence_length=None, auxiliary_inputs=None): - if auxiliary_inputs is None: - maybe_concatenated_inputs = inputs - else: - inputs = ops.convert_to_tensor(inputs) - auxiliary_inputs = ops.convert_to_tensor(auxiliary_inputs) - maybe_concatenated_inputs = nest.map_structure( - lambda x, y: array_ops.concat((x, y), -1), inputs, auxiliary_inputs) - if not self.time_major: - auxiliary_inputs = nest.map_structure(_transpose_batch_time, - auxiliary_inputs) - if auxiliary_inputs is not None: - self._auxiliary_input_tas = nest.map_structure(_unstack_ta, - auxiliary_inputs) - else: - self._auxiliary_input_tas = None - - return super(ScheduledOutputTrainingSampler, self).initialize( - maybe_concatenated_inputs, sequence_length=sequence_length) - - def sample(self, time, outputs, state): - del state - return bernoulli_sample( - probs=self.sampling_probability, - sample_shape=self.batch_size, - seed=self.seed) - - def next_inputs(self, time, outputs, state, sample_ids): - (finished, base_next_inputs, state) = ( - super(ScheduledOutputTrainingSampler, self).next_inputs( - time=time, outputs=outputs, state=state, sample_ids=sample_ids)) - sample_ids = math_ops.cast(sample_ids, dtypes.bool) - - def maybe_sample(): - """Perform scheduled sampling.""" - - def maybe_concatenate_auxiliary_inputs(outputs_, indices=None): - """Concatenate outputs with auxiliary inputs, if they exist.""" - if self._auxiliary_input_tas is None: - return outputs_ - - next_time = time + 1 - auxiliary_inputs = nest.map_structure(lambda ta: ta.read(next_time), - self._auxiliary_input_tas) - if indices is not None: - auxiliary_inputs = array_ops.gather_nd(auxiliary_inputs, indices) - return nest.map_structure(lambda x, y: array_ops.concat((x, y), -1), - outputs_, auxiliary_inputs) - - if self.next_inputs_fn is None: - return array_ops.where(sample_ids, - maybe_concatenate_auxiliary_inputs(outputs), - base_next_inputs) - - where_sampling = math_ops.cast(array_ops.where(sample_ids), dtypes.int32) - where_not_sampling = math_ops.cast( - array_ops.where(math_ops.logical_not(sample_ids)), dtypes.int32) - outputs_sampling = array_ops.gather_nd(outputs, where_sampling) - inputs_not_sampling = array_ops.gather_nd(base_next_inputs, - where_not_sampling) - sampled_next_inputs = maybe_concatenate_auxiliary_inputs( - self.next_inputs_fn(outputs_sampling), where_sampling) - - base_shape = array_ops.shape(base_next_inputs) - return (array_ops.scatter_nd( - indices=where_sampling, updates=sampled_next_inputs, shape=base_shape) - + array_ops.scatter_nd( - indices=where_not_sampling, - updates=inputs_not_sampling, - shape=base_shape)) - - all_finished = math_ops.reduce_all(finished) - no_samples = math_ops.logical_not(math_ops.reduce_any(sample_ids)) - next_inputs = control_flow_ops.cond( - math_ops.logical_or(all_finished, no_samples), lambda: base_next_inputs, - maybe_sample) - return (finished, next_inputs, state) - - -class GreedyEmbeddingSampler(Sampler): - """A sampler for use during inference. - - Uses the argmax of the output (treated as logits) and passes the - result through an embedding layer to get the next input. - """ - - def __init__(self, embedding_fn=None): - """Initializer. - - Args: - embedding_fn: A optional callable that takes a vector tensor of `ids` - (argmax ids), or the `params` argument for `embedding_lookup`. The - returned tensor will be passed to the decoder input. Default to use - `embedding_ops.embedding_lookup`. - """ - if embedding_fn is None or callable(embedding_fn): - self.embedding_fn = embedding_fn - else: - raise ValueError("embedding_fn is expected to be a callable, got %s" % - type(embedding_fn)) - self._batch_size = None - - @property - def batch_size(self): - if self._batch_size is None: - raise ValueError("batch_size accessed before initialize was called") - return self._batch_size - - @property - def sample_ids_shape(self): - return tensor_shape.TensorShape([]) - - @property - def sample_ids_dtype(self): - return dtypes.int32 - - def initialize(self, embedding, start_tokens=None, end_token=None): - """Initialize the GreedyEmbeddingSampler. - - Args: - embedding: tensor that contains embedding states matrix. It will be used - to generate generate outputs with start_tokens and end_tokens. The - embedding will be ignored if the embedding_fn has been provided at - __init__(). - start_tokens: `int32` vector shaped `[batch_size]`, the start tokens. - end_token: `int32` scalar, the token that marks end of decoding. - - Returns: - Tuple of two items: `(finished, self.start_inputs)`. - Raises: - ValueError: if `start_tokens` is not a 1D tensor or `end_token` is not a - scalar. - """ - if self.embedding_fn is None: - self.embedding_fn = ( - lambda ids: embedding_ops.embedding_lookup(embedding, ids)) - - self.start_tokens = ops.convert_to_tensor( - start_tokens, dtype=dtypes.int32, name="start_tokens") - self.end_token = ops.convert_to_tensor( - end_token, dtype=dtypes.int32, name="end_token") - if self.start_tokens.get_shape().ndims != 1: - raise ValueError("start_tokens must be a vector") - self._batch_size = array_ops.size(start_tokens) - if self.end_token.get_shape().ndims != 0: - raise ValueError("end_token must be a scalar") - self.start_inputs = self.embedding_fn(self.start_tokens) - - finished = array_ops.tile([False], [self._batch_size]) - return (finished, self.start_inputs) - - def sample(self, time, outputs, state): - """sample for GreedyEmbeddingHelper.""" - del time, state # unused by sample_fn - # Outputs are logits, use argmax to get the most probable id - if not isinstance(outputs, ops.Tensor): - raise TypeError( - "Expected outputs to be a single Tensor, got: %s" % type(outputs)) - sample_ids = math_ops.argmax(outputs, axis=-1, output_type=dtypes.int32) - return sample_ids - - def next_inputs(self, time, outputs, state, sample_ids): - """next_inputs_fn for GreedyEmbeddingHelper.""" - del time, outputs # unused by next_inputs_fn - finished = math_ops.equal(sample_ids, self.end_token) - all_finished = math_ops.reduce_all(finished) - next_inputs = control_flow_ops.cond( - all_finished, - # If we're finished, the next_inputs value doesn't matter - lambda: self.start_inputs, - lambda: self.embedding_fn(sample_ids)) - return (finished, next_inputs, state) - - -class SampleEmbeddingSampler(GreedyEmbeddingSampler): - """A sampler for use during inference. - - Uses sampling (from a distribution) instead of argmax and passes the - result through an embedding layer to get the next input. - """ - - def __init__(self, embedding_fn=None, softmax_temperature=None, seed=None): - """Initializer. - - Args: - embedding_fn: (Optional) A callable that takes a vector tensor of `ids` - (argmax ids), or the `params` argument for `embedding_lookup`. The - returned tensor will be passed to the decoder input. - softmax_temperature: (Optional) `float32` scalar, value to divide the - logits by before computing the softmax. Larger values (above 1.0) result - in more random samples, while smaller values push the sampling - distribution towards the argmax. Must be strictly greater than 0. - Defaults to 1.0. - seed: (Optional) The sampling seed. - - Raises: - ValueError: if `start_tokens` is not a 1D tensor or `end_token` is not a - scalar. - """ - super(SampleEmbeddingSampler, self).__init__(embedding_fn) - self.softmax_temperature = softmax_temperature - self.seed = seed - - def sample(self, time, outputs, state): - """sample for SampleEmbeddingHelper.""" - del time, state # unused by sample_fn - # Outputs are logits, we sample instead of argmax (greedy). - if not isinstance(outputs, ops.Tensor): - raise TypeError( - "Expected outputs to be a single Tensor, got: %s" % type(outputs)) - if self.softmax_temperature is None: - logits = outputs - else: - logits = outputs / self.softmax_temperature - - return categorical_sample(logits=logits, seed=self.seed) - - -class InferenceSampler(Sampler): - """A helper to use during inference with a custom sampling function.""" - - def __init__(self, - sample_fn, - sample_shape, - sample_dtype, - end_fn, - next_inputs_fn=None): - """Initializer. - - Args: - sample_fn: A callable that takes `outputs` and emits tensor `sample_ids`. - sample_shape: Either a list of integers, or a 1-D Tensor of type `int32`, - the shape of the each sample in the batch returned by `sample_fn`. - sample_dtype: the dtype of the sample returned by `sample_fn`. - end_fn: A callable that takes `sample_ids` and emits a `bool` vector - shaped `[batch_size]` indicating whether each sample is an end token. - next_inputs_fn: (Optional) A callable that takes `sample_ids` and returns - the next batch of inputs. If not provided, `sample_ids` is used as the - next batch of inputs. - """ - self.sample_fn = sample_fn - self.sample_shape = tensor_shape.TensorShape(sample_shape) - self.sample_dtype = sample_dtype - self.end_fn = end_fn - self.next_inputs_fn = next_inputs_fn - self._batch_size = None - - @property - def batch_size(self): - if self._batch_size is None: - raise ValueError("batch_size accessed before initialize was called") - return self._batch_size - - @property - def sample_ids_shape(self): - return self.sample_shape - - @property - def sample_ids_dtype(self): - return self.sample_dtype - - def initialize(self, start_inputs): - self.start_inputs = ops.convert_to_tensor(start_inputs, name="start_inputs") - self._batch_size = array_ops.shape(start_inputs)[0] - finished = array_ops.tile([False], [self._batch_size]) - return (finished, self.start_inputs) - - def sample(self, time, outputs, state): - del time, state # unused by sample - return self.sample_fn(outputs) - - def next_inputs(self, time, outputs, state, sample_ids): - del time, outputs # unused by next_inputs - if self.next_inputs_fn is None: - next_inputs = sample_ids - else: - next_inputs = self.next_inputs_fn(sample_ids) - finished = self.end_fn(sample_ids) - return (finished, next_inputs, state) - - -# The following sample functions (_call_sampler, bernoulli_sample, -# categorical_sample) mimic TensorFlow Probability distribution semantics. -def _call_sampler(sample_n_fn, sample_shape, name=None): - """Reshapes vector of samples.""" - with ops.name_scope(name, "call_sampler", values=[sample_shape]): - sample_shape = ops.convert_to_tensor( - sample_shape, dtype=dtypes.int32, name="sample_shape") - # Ensure sample_shape is a vector (vs just a scalar). - pad = math_ops.cast( - math_ops.equal(array_ops.rank(sample_shape), 0), dtypes.int32) - sample_shape = array_ops.reshape( - sample_shape, - array_ops.pad( - array_ops.shape(sample_shape), - paddings=[[pad, 0]], - constant_values=1)) - samples = sample_n_fn(math_ops.reduce_prod(sample_shape)) - batch_event_shape = array_ops.shape(samples)[1:] - final_shape = array_ops.concat([sample_shape, batch_event_shape], 0) - return array_ops.reshape(samples, final_shape) - - -def bernoulli_sample(probs=None, - logits=None, - dtype=dtypes.int32, - sample_shape=(), - seed=None): - """Samples from Bernoulli distribution.""" - if probs is None: - probs = math_ops.sigmoid(logits, name="probs") - else: - probs = ops.convert_to_tensor(probs, name="probs") - batch_shape_tensor = array_ops.shape(probs) - - def _sample_n(n): - """Sample vector of Bernoullis.""" - new_shape = array_ops.concat([[n], batch_shape_tensor], 0) - uniform = random_ops.random_uniform(new_shape, seed=seed, dtype=probs.dtype) - return math_ops.cast(math_ops.less(uniform, probs), dtype) - - return _call_sampler(_sample_n, sample_shape) - - -def categorical_sample(logits, dtype=dtypes.int32, sample_shape=(), seed=None): - """Samples from categorical distribution.""" - logits = ops.convert_to_tensor(logits, name="logits") - event_size = array_ops.shape(logits)[-1] - batch_shape_tensor = array_ops.shape(logits)[:-1] - - def _sample_n(n): - """Sample vector of categoricals.""" - if logits.shape.ndims == 2: - logits_2d = logits - else: - logits_2d = array_ops.reshape(logits, [-1, event_size]) - sample_dtype = dtypes.int64 if logits.dtype.size > 4 else dtypes.int32 - draws = random_ops.multinomial( - logits_2d, n, seed=seed, output_dtype=sample_dtype) - draws = array_ops.reshape( - array_ops.transpose(draws), - array_ops.concat([[n], batch_shape_tensor], 0)) - return math_ops.cast(draws, dtype) - - return _call_sampler(_sample_n, sample_shape) - - -def _unstack_ta(inp): - return tensor_array_ops.TensorArray( - dtype=inp.dtype, - size=array_ops.shape(inp)[0], - element_shape=inp.get_shape()[1:]).unstack(inp) diff --git a/tensorflow/contrib/session_bundle/BUILD b/tensorflow/contrib/session_bundle/BUILD deleted file mode 100644 index edfc6639b0f..00000000000 --- a/tensorflow/contrib/session_bundle/BUILD +++ /dev/null @@ -1,418 +0,0 @@ -# Description: -# TensorFlow Serving session bundle. - -load( - "//tensorflow:tensorflow.bzl", - "if_android", - "if_ios", - "if_mobile", - "if_not_mobile", - "py_test", - "tf_cc_test", -) -load("//tensorflow/core/platform:default/build_config.bzl", "tf_proto_library") -# Placeholder: load("//tensorflow:tensorflow.bzl", "tf_portable_proto_lib") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -# TODO(b/32673259): add a test to continuously validate these files. -filegroup( - name = "session_bundle_half_plus_two", - srcs = glob([ - "testdata/half_plus_two/**", - "testdata/half_plus_two_ckpt_v2/**", - ]), -) - -# Transitive dependencies of this target will be included in the pip package. -py_library( - name = "session_bundle_pip", - deps = [ - ":bundle_shim_py", - ":exporter", - ":gc", - ], -) - -py_library( - name = "bundle_shim_py", - srcs = ["bundle_shim.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":constants", - ":manifest_proto_py", - ":session_bundle_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client", - "//tensorflow/python:framework", - "//tensorflow/python:platform", - "//tensorflow/python:session", - "//tensorflow/python/saved_model:constants", - "//tensorflow/python/saved_model:loader", - "//tensorflow/python/saved_model:signature_constants", - ], -) - -py_test( - name = "bundle_shim_py_test", - size = "small", - srcs = ["bundle_shim_test.py"], - data = [ - ":session_bundle_half_plus_two", - "//tensorflow/cc/saved_model:saved_model_half_plus_two", - ], - main = "bundle_shim_test.py", - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [ - ":bundle_shim_py", - ":constants", - ":manifest_proto_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:util", - "//tensorflow/python/saved_model:signature_constants", - "//tensorflow/python/saved_model:tag_constants", - ], -) - -# DEPRECATED: No longer supported. Switch to SavedModel immediately. -py_library( - name = "constants", - srcs = ["constants.py"], - deprecation = "No longer supported. Switch to SavedModel immediately.", - srcs_version = "PY2AND3", -) - -# DEPRECATED: No longer supported. Switch to SavedModel immediately. -py_library( - name = "exporter", - srcs = ["exporter.py"], - deprecation = "No longer supported. Switch to SavedModel immediately.", - srcs_version = "PY2AND3", - deps = [ - ":constants", - ":gc", - ":manifest_proto_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:training", - "//tensorflow/python:util", - "@six_archive//:six", - ], -) - -py_test( - name = "exporter_test", - size = "small", - srcs = ["exporter_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - visibility = ["//visibility:private"], - deps = [ - ":constants", - ":exporter", - ":gc", - ":manifest_proto_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:session", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], -) - -# DEPRECATED: No longer supported. Switch to SavedModel immediately. -py_library( - name = "gc", - srcs = ["gc.py"], - deprecation = "No longer supported. Switch to SavedModel immediately.", - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:framework", - "//tensorflow/python:platform", - "//tensorflow/python:util", - ], -) - -py_test( - name = "gc_test", - srcs = ["gc_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_windows"], # TODO: needs investigation on Windows - visibility = ["//visibility:private"], - deps = [ - ":gc", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform", - ], -) - -filegroup( - name = "session_bundle_srcs", - srcs = [ - "session_bundle.cc", - "session_bundle.h", - ], -) - -filegroup( - name = "signature_srcs", - srcs = [ - "signature.cc", - "signature.h", - ], -) - -# DEPRECATED: No longer supported. Switch to SavedModel immediately. -cc_library( - name = "session_bundle", - hdrs = ["session_bundle.h"], - deprecation = "No longer supported. Switch to SavedModel immediately.", - visibility = ["//visibility:public"], - deps = [ - ":manifest_proto_cc", - ":session_bundle_lite", - ":signature", - "//tensorflow/core:core_cpu", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - ], -) - -# DEPRECATED: No longer supported. Switch to SavedModel immediately. -# This is a lite version of the session_bundle target that does not link in any -# Tensorflow ops in order to minimize its size. Clients using this should link -# any required ops manually. -cc_library( - name = "session_bundle_lite", - srcs = ["session_bundle.cc"], - hdrs = ["session_bundle.h"], - copts = if_ios(["-DGOOGLE_LOGGING"]), - deprecation = "No longer supported. Switch to SavedModel immediately.", - visibility = ["//visibility:public"], - deps = [ - ":signature_lite", - "//tensorflow/core:lib_internal", - ] + if_not_mobile([ - ":manifest_proto_cc", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core/util/tensor_bundle:naming", - # mobile not supported yet - ]), -) - -tf_cc_test( - name = "session_bundle_test", - size = "medium", - srcs = ["session_bundle_test.cc"], - data = [":session_bundle_half_plus_two"], - # Link in all registered kernels. - linkstatic = 1, - visibility = ["//visibility:private"], - deps = [ - ":session_bundle", - ":signature", - ":test_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -# DEPRECATED: No longer supported. Switch to SavedModel immediately. -py_library( - name = "session_bundle_py", - srcs = ["session_bundle.py"], - deprecation = "No longer supported. Switch to SavedModel immediately.", - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":constants", - ":manifest_proto_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:lib", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:util", - ], -) - -py_test( - name = "session_bundle_py_test", - size = "small", - srcs = ["session_bundle_test.py"], - data = [":session_bundle_half_plus_two"], - main = "session_bundle_test.py", - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [ - ":constants", - ":manifest_proto_py", - ":session_bundle_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:graph_util", - "//tensorflow/python:math_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -# DEPRECATED: No longer supported. Switch to SavedModel immediately. -# This is a lite version of the signature target that does not link in any -# Tensorflow ops in order to minimize its size. Clients using this should -# link any required ops manually. -cc_library( - name = "signature_lite", - srcs = ["signature.cc"], - hdrs = ["signature.h"], - deprecation = "No longer supported. Switch to SavedModel immediately.", - visibility = ["//visibility:public"], - deps = if_not_mobile([ - ":manifest_proto_cc", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//tensorflow/core:protos_all_cc", - # mobile not supported yet - ]), -) - -# DEPRECATED: No longer supported. Switch to SavedModel immediately. -cc_library( - name = "signature", - hdrs = ["signature.h"], - deprecation = "No longer supported. Switch to SavedModel immediately.", - visibility = ["//visibility:public"], - deps = [ - ":manifest_proto_cc", - ":signature_lite", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - ] + if_not_mobile([ - "//tensorflow/core:tensorflow_opensource", - ]), -) - -tf_cc_test( - name = "signature_test", - size = "small", - srcs = ["signature_test.cc"], - visibility = ["//visibility:private"], - deps = [ - ":manifest_proto_cc", - ":signature", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:tensorflow_opensource", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -# DEPRECATED: No longer supported. Switch to SavedModel immediately. -cc_library( - name = "test_util", - testonly = 1, - srcs = ["test_util.cc"], - hdrs = ["test_util.h"], - deprecation = "No longer supported. Switch to SavedModel immediately.", - visibility = ["//visibility:private"], - deps = [ - "//tensorflow/core:lib", - "//tensorflow/core:test", - ], -) - -cc_library( - name = "bundle_shim", - srcs = ["bundle_shim.cc"], - hdrs = ["bundle_shim.h"], - copts = if_ios(["-DGOOGLE_LOGGING"]), - visibility = ["//visibility:public"], - deps = [ - ":session_bundle", - ":signature", - "//tensorflow/cc/saved_model:loader", - "//tensorflow/cc/saved_model:signature_constants", - ] + if_not_mobile([ - ":manifest_proto_cc", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - # mobile not supported yet - ]), -) - -tf_cc_test( - name = "bundle_shim_test", - size = "small", - srcs = ["bundle_shim_test.cc"], - data = [ - ":session_bundle_half_plus_two", - "//tensorflow/cc/saved_model:saved_model_half_plus_two", - ], - # Link in all registered kernels. - linkstatic = 1, - deps = [ - ":bundle_shim", - ":test_util", - "//tensorflow/cc/saved_model:signature_constants", - "//tensorflow/cc/saved_model:tag_constants", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -# DEPRECATED: No longer supported. Switch to SavedModel immediately. -tf_proto_library( - name = "manifest_proto", - srcs = ["manifest.proto"], - cc_api_version = 2, - visibility = ["//visibility:public"], -) diff --git a/tensorflow/contrib/session_bundle/README.md b/tensorflow/contrib/session_bundle/README.md deleted file mode 100644 index e3a87a0eb90..00000000000 --- a/tensorflow/contrib/session_bundle/README.md +++ /dev/null @@ -1,273 +0,0 @@ -# TensorFlow Inference Model Format - -WARNING: SessionBundle has been deprecated and is no longer supported. Switch to -[SavedModel](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md) -immediately. - -[TOC] - -## Overview - -This document describes the data formats and layouts for exporting -[TensorFlow](https://www.tensorflow.org/) models for inference. - -These exports have the following properties: - -* Recoverable - * given an export the graph can easily be initialized and run -* Hermetic - * an export directory is self-contained to facilitate distribution - -The TensorFlow `Saver` writes **checkpoints** (graph variables) while training -so it can recover if it crashes. A TensorFlow Serving **export** contains a -checkpoint with the current state of the graph variables along with a MetaGraph -definition that's needed for serving. - -## Directory Structure - -~~~ -# Directory overview -00000000/ - assets/ - export.meta - export-?????-of-????? -~~~ - -* `00000000` -- Export version - * Format `%08d` -* `assets` -- Asset file directory - * Holds auxiliary files for the graph (e.g., vocabularies) -* `export.meta` -- MetaGraph Definition - * Binary [`tensorflow::MetaGraphDef`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/core/protobuf/meta_graph.proto) -* `export-?????-of-?????` - * A checkpoint of the Graph Variables - * Outputs from Python [`Saver`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python/training/saver.py) - with `sharded=True`. - -## Exporting (Python code) - -The [`Exporter`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/session_bundle/exporter.py) -class can be used to export a model in the above format from a TensorFlow Python -binary. - -### Exporting TF.learn models - -TF.learn uses an -[Exporter wrapper](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/learn/python/learn/utils/export.py) -that can be used for building signatures. Use the `BaseEstimator.export` -function to export your Estimator with a signature. - -## Importing (C++ code) - -The [`LoadSessionBundleFromPath`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/session_bundle/session_bundle.h) -function can be used to create a `tensorflow::Session` and initialize it from an -export. This function takes session options and the path to the export as -arguments and returns a bundle of export data including a `tensorflow::Session` -which can be run. - -## Signatures - -Graphs used for inference tasks typically have set of inputs and outputs used at -inference time. We call this a 'Signature'. - -### Standard Signatures (standard usage) - -Graphs used for standard inference tasks have standard sets of inputs and -outputs. For example, a graph used for a regression task has an input tensor for -the data and an output tensor for the regression values. The signature mechanism -makes it easy to identify the relevant input and output tensors for common graph -applications. - -The [`Manifest`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/session_bundle/manifest.proto) -contains a `Signature` message which contains the task specific inputs and -outputs. - -~~~proto -// A Signature specifies the inputs and outputs of commonly used graphs. -message Signature { - oneof type { - RegressionSignature regression_signature = 1; - ClassificationSignature classification_signature = 2; - GenericSignature generic_signature = 3; - } -}; -~~~ - -A Standard Signature can be set at export time using the `Exporter` API. - -~~~python -# Run an export. -signature = exporter.classification_signature(input_tensor=input, - classes_tensor=output) -export = exporter.Exporter(saver) -export.init(sess.graph.as_graph_def(), default_graph_signature=signature) -export.export(export_path, global_step_tensor, sess) -~~~ - -#### TF.learn signatures - -TF.learn models can use the `BaseEstimator.export` function directly to export. -To specify a Signature, use the [Exporter wrapper](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/learn/python/learn/utils/export.py) -helpers (e.g. `classification_signature_fn`). - -~~~python -estimator = tf.contrib.learn.Estimator(...) -... -# Other possible parameters omitted for the sake of brevity. -estimator.export( - export_path, - signature_fn=tf.contrib.learn.utils.export.classification_signature_fn) -~~~ - -#### Recovering signatures - -These can be recovered at serving time using utilities in [`signature.h`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/session_bundle/signature.h) - -~~~c++ -// Get the classification signature. -ClassificationSignature signature; -TF_CHECK_OK(GetClassificationSignature(bundle->meta_graph_def, &signature)); - -// Run the graph. -Tensor input_tensor = GetInputTensor(); -Tensor classes_tensor; -Tensor scores_tensor; -TF_CHECK_OK(RunClassification(signature, input_tensor, session, &classes_tensor, - &scores_tensor)); -~~~ - -### Generic Signatures (custom or advanced usage) - -Generic Signatures enable fully custom usage of the `tensorflow::Session` API. -They are recommended for when the Standard Signatures do not satisfy a -particular use-case. A general example of when to use these is for a model -taking a single input and generating multiple outputs performing different -inferences. - -~~~proto -// GenericSignature specifies a map from logical name to Tensor name. -// Typical application of GenericSignature is to use a single GenericSignature -// that includes all of the Tensor nodes and target names that may be useful at -// serving, analysis or debugging time. The recommended name for this signature -// is "generic_bindings". -message GenericSignature { - map map = 1; -}; -~~~ - -Generic Signatures can be used to compliment a Standard Signature, for example -to support debugging. Here is an example that includes the Standard regression -Signature and a Generic Signature. - -~~~python -named_tensor_bindings = {"logical_input_A": v0, - "logical_input_B": v1} -signatures = { - "regression": exporter.regression_signature(input_tensor=v0, - output_tensor=v1), - "generic": exporter.generic_signature(named_tensor_bindings)} -export = exporter.Exporter(saver) -export.init(sess.graph.as_graph_def(), named_graph_signatures=signatures) -export.export(export_path, global_step_tensor, sess) -~~~ - -Generic Signature does not differentiate between input and output tensors. It -provides full flexibility to specify the input and output tensors you need. -The benefit is preserving a mapping between names that you specify at export -time (we call these the logical names), and the actual graph node names that may -be less stable and/or auto-generated by TensorFlow. - -In [`signature.h`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/session_bundle/signature.h) -note that the generic signature methods `BindGenericInputs` and -`BindGenericNames` are doing simple string to string mapping as a convenience. -These methods map from the names used at training time to actual names in the -graph. - -The bound results from those methods can be used as inputs to -`tensorflow::Session->Run()`. Specifically, the bound result -`vector>` from `BindGenericInputs` can be supplied as the -first parameter `inputs` to `tensorflow::Session->Run()`. Similarly, the bound -result `vector` from `BindGenericNames`, can be mapped to -`output_tensor_names` in the `tensorflow::Session->Run()` arguments. The next -parameter, `target_node_names` is typically null at inference time. The last -parameter `outputs` is for the results, which share the same order as the -supplied `output_tensor_names`. - -## Custom Initialization - -Some graphs many require custom initialization after the variables have been -restored. Such initialization, done through an arbitrary Op, can be added using -the `Exporter` API. If set, `LoadSessionBundleFromPath` will automatically run -the Op when restoring a `Session` following the loading of variables. - -## Assets - -In many cases we have Ops which depend on external files for initialization -(such as vocabularies). These "assets" are not stored in the graph and are -needed for both training and inference. - -In order to create hermetic exports these asset files need to be: - -1. copied to each export directory, and -2. read when recovering a session from an export base directory. - -Copying assets to the export directory is handled with a callback mechanism. -The callback function receives two parameters: - -1. the dictionary of source files to desired basename, and -2. the export directory. -The default callback uses `gfile.Copy` to perform the copy. - -The tensor that contains the filepath to be copied is specified by passing the -collection of asset filepath tensor, which is usually extracted from the graph -by `tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS)`. - -~~~python -# Run an export. -export = exporter.Exporter(save) -export.init( - sess.graph.as_graph_def(), - asset_collection=tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS)) -export.export(export_path, global_step_tensor, sess) -~~~ - -Users can use their own callbacks as shown in the following example, with the -requirement to keep the basename of the original files: - -~~~python -def my_custom_copy_callback(files_to_copy, export_dir_path): - # Copy all source files (keys) in files_to_copy to export_dir_path using the - # corresponding basename (value). - ... - -# Run an export. -export = exporter.Exporter(save) -export.init( - sess.graph.as_graph_def(), - asset_collection=tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS), - asset_callback=my_custom_copy_callback) -export.export(export_path, global_step_tensor, sess) -~~~ - -`AssetFile` binds the name of a tensor in the graph to the name of a file -within the assets directory. `LoadSessionBundleFromPath` will handle the base -path and asset directory swap/concatenation such that the tensor is set with -the fully qualified filename upon return. - -# Exporter Usage - -The typical workflow of model exporting is: - -1. Build model graph G. -2. Train variables or load trained variables from checkpoint in session S. -3. [Optional] Build inference graph I. -4. Export G. - -The Exporter should be used as follows: - -1. The Saver used in Exporter(saver) should be created within the context of G. -2. Exporter.init() should be called within the context of G. -3. Exporter.export() should be called using session S. -4. If I is provided for Exporter.init(), an exact same Saver should be created - under I as the saver under G -- in the way that exact same Save/Restore ops - are created in both G and S. diff --git a/tensorflow/contrib/session_bundle/bundle_shim.cc b/tensorflow/contrib/session_bundle/bundle_shim.cc deleted file mode 100644 index c669ced9977..00000000000 --- a/tensorflow/contrib/session_bundle/bundle_shim.cc +++ /dev/null @@ -1,393 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/session_bundle/bundle_shim.h" - -#include "tensorflow/cc/saved_model/loader.h" -#include "tensorflow/cc/saved_model/signature_constants.h" -#include "tensorflow/contrib/session_bundle/manifest.pb.h" -#include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/contrib/session_bundle/signature.h" -#include "tensorflow/core/graph/graph_constructor.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/protobuf/meta_graph.pb.h" -#include "tensorflow/core/public/session.h" -#include "tensorflow/core/public/session_options.h" - -namespace tensorflow { -namespace serving { -namespace { -/////////////////////////////////////////////////////////////////////////////// -// Helper functions to check Signature type. - -bool IsClassificationSignature(const Signature& signature) { - return signature.type_case() == Signature::kClassificationSignature; -} - -bool IsRegressionSignature(const Signature& signature) { - return signature.type_case() == Signature::kRegressionSignature; -} - -/////////////////////////////////////////////////////////////////////////////// -// Helper functions to build `Classification`, `Regression` and `Predict` -// SignatureDefs. - -SignatureDef BuildRegressionSignatureDef( - const RegressionSignature& regression_signature, - const std::unordered_map& tensor_name_to_dtype) { - SignatureDef signature_def; - signature_def.set_method_name(kRegressMethodName); - internal::AddInputToSignatureDef(regression_signature.input().tensor_name(), - tensor_name_to_dtype, kRegressInputs, - &signature_def); - internal::AddOutputToSignatureDef(regression_signature.output().tensor_name(), - tensor_name_to_dtype, kRegressOutputs, - &signature_def); - return signature_def; -} - -SignatureDef BuildClassificationSignatureDef( - const ClassificationSignature& classification_signature, - const std::unordered_map& tensor_name_to_dtype) { - SignatureDef signature_def; - signature_def.set_method_name(kClassifyMethodName); - internal::AddInputToSignatureDef( - classification_signature.input().tensor_name(), tensor_name_to_dtype, - kClassifyInputs, &signature_def); - internal::AddOutputToSignatureDef( - classification_signature.classes().tensor_name(), tensor_name_to_dtype, - kClassifyOutputClasses, &signature_def); - internal::AddOutputToSignatureDef( - classification_signature.scores().tensor_name(), tensor_name_to_dtype, - kClassifyOutputScores, &signature_def); - return signature_def; -} - -Status MaybeBuildPredictSignatureDef( - const std::unordered_map& tensor_name_to_dtype, - MetaGraphDef* meta_graph_def) { - Signature input_signature, output_signature; - // Ensure that named signatures corresponding to `inputs` and `outputs` keys - // exist. - if (!GetNamedSignature(kPredictInputs, *meta_graph_def, &input_signature) - .ok() || - !GetNamedSignature(kPredictOutputs, *meta_graph_def, &output_signature) - .ok()) { - return Status(error::Code::INVALID_ARGUMENT, - "Named signatures can only be up-converted if entries " - "corresponding to both `inputs` and `outputs` exist."); - } - // Ensure the `inputs` and `outputs` named signatures are generic signatures. - if (input_signature.type_case() != Signature::TypeCase::kGenericSignature || - output_signature.type_case() != Signature::TypeCase::kGenericSignature) { - return Status(error::Code::INVALID_ARGUMENT, - "Named signatures corresponding to `inputs` and `outputs` " - "can only be up-converted if they are GenericSignatures."); - } - SignatureDef signature_def; - signature_def.set_method_name(kPredictMethodName); - // Add map entries from the `inputs` generic signature to the input map in the - // signature def. - for (const auto& map_entry : input_signature.generic_signature().map()) { - internal::AddInputToSignatureDef(map_entry.second.tensor_name(), - tensor_name_to_dtype, map_entry.first, - &signature_def); - } - // Add map entries from the `outputs` generic signature to the output map in - // the signature def. - for (const auto& map_entry : output_signature.generic_signature().map()) { - internal::AddOutputToSignatureDef(map_entry.second.tensor_name(), - tensor_name_to_dtype, map_entry.first, - &signature_def); - } - // Add the constructed signature def to the signature def map of the meta - // graph def. Use the default key if it isn't already in use. - const bool already_has_default_signature = - meta_graph_def->signature_def().find(kDefaultServingSignatureDefKey) != - meta_graph_def->signature_def().end(); - const string signature_def_key = - already_has_default_signature - ? strings::StrCat(kDefaultServingSignatureDefKey, "_from_named") - : kDefaultServingSignatureDefKey; - (*meta_graph_def->mutable_signature_def())[signature_def_key] = signature_def; - return Status::OK(); -} - -Status LoadSavedModelFromLegacySessionBundlePath( - const SessionOptions& session_options, const RunOptions& run_options, - const StringPiece session_bundle_export_dir, - SavedModelBundle* saved_model_bundle) { - if (session_bundle_export_dir.empty()) { - return Status(error::Code::NOT_FOUND, "Export directory path is empty."); - } - if (!IsPossibleExportDirectory(session_bundle_export_dir)) { - return Status( - error::Code::NOT_FOUND, - "Export directory does not contain a valid SessionBundle export."); - } - - // Build the session-bundle. - SessionBundle session_bundle; - TF_RETURN_IF_ERROR(LoadSessionBundleFromPathUsingRunOptions( - session_options, run_options, session_bundle_export_dir, - &session_bundle)); - - // Convert the session-bundle to a saved-model-bundle. - return internal::ConvertSessionBundleToSavedModelBundle(session_bundle, - saved_model_bundle); -} - -/////////////////////////////////////////////////////////////////////////////// -// Helper functions to convert `Default` and `Named` signatures to -// SignatureDefs. - -// Up-conversion of default signatures is supported for classification and -// regression. -Status ConvertDefaultSignatureToSignatureDef( - const Signatures& signatures, - const std::unordered_map& tensor_name_to_dtype, - MetaGraphDef* meta_graph_def) { - if (!signatures.has_default_signature()) { - return Status::OK(); - } - const bool already_has_default_signature = - meta_graph_def->signature_def().find(kDefaultServingSignatureDefKey) != - meta_graph_def->signature_def().end(); - if (already_has_default_signature) { - return Status(error::Code::ALREADY_EXISTS, - strings::StrCat( - "Default signature cannot be up-converted since ", - kDefaultServingSignatureDefKey, " key already exists.")); - } - const Signature& signature = signatures.default_signature(); - if (IsRegressionSignature(signature)) { - (*meta_graph_def->mutable_signature_def())[kDefaultServingSignatureDefKey] = - BuildRegressionSignatureDef(signature.regression_signature(), - tensor_name_to_dtype); - } else if (IsClassificationSignature(signature)) { - (*meta_graph_def->mutable_signature_def())[kDefaultServingSignatureDefKey] = - BuildClassificationSignatureDef(signature.classification_signature(), - tensor_name_to_dtype); - } else { - LOG(WARNING) << "Default signature up-conversion to SignatureDef is only " - "supported for `Classification` and `Regression`. Could " - "not up-convert signature: " - << signature.DebugString() - << ". (If using SessionRun with the SessionBundle export " - "format please ignore this warning.)"; - } - return Status::OK(); -} - -Status ConvertNamedSignaturesToSignatureDef( - const Signatures& signatures, - const std::unordered_map& tensor_name_to_dtype, - MetaGraphDef* meta_graph_def) { - if (signatures.named_signatures().empty()) { - return Status::OK(); - } - // Check for a Predict signature for up-conversion. - Status predict_signature_def_status = - MaybeBuildPredictSignatureDef(tensor_name_to_dtype, meta_graph_def); - for (const auto& it_named_signature : signatures.named_signatures()) { - const string key = it_named_signature.first; - // If a Predict SignatureDef was successfully constructed, skip the entries - // corresponding to `inputs` and `outputs`. - if (predict_signature_def_status.ok()) { - if (key == kPredictInputs || key == kPredictOutputs) { - continue; - } - } - const Signature signature = it_named_signature.second; - if (IsRegressionSignature(signature)) { - (*meta_graph_def->mutable_signature_def())[key] = - BuildRegressionSignatureDef(signature.regression_signature(), - tensor_name_to_dtype); - } else if (IsClassificationSignature(signature)) { - (*meta_graph_def->mutable_signature_def())[key] = - BuildClassificationSignatureDef(signature.classification_signature(), - tensor_name_to_dtype); - } else { - LOG(WARNING) - << "Named signature up-conversion to SignatureDef is only supported " - "for `Classification`, `Regression` or if two `GenericSignatures` " - "signatures called `inputs` and `outputs` exist, corresponding " - "to the `Prediction` API. Could not up-convert signature: " - << signature.DebugString(); - } - } - return Status::OK(); -} - -} // namespace - -namespace internal { -/////////////////////////////////////////////////////////////////////////////// -// Helper functions to populate SignatureDef fields. - -// Adds an entry to the `inputs` map of the supplied SignatureDef. -void AddInputToSignatureDef( - const string& tensor_name, - const std::unordered_map& tensor_name_to_dtype, - const string& input_key, SignatureDef* signature_def) { - if (tensor_name.empty()) { - LOG(WARNING) << "Tensor name not provided. Cannot add TensorInfo to " - "SignatureDef inputs."; - return; - } - // Extract the tensor-name in case the supplied string is a tensor-reference. - // Example: Extract "x" from "x:0". - std::size_t pos = tensor_name.find(":"); - const string key = - (pos != std::string::npos) ? tensor_name.substr(0, pos) : tensor_name; - const auto it_tensor_info = tensor_name_to_dtype.find(key); - TensorInfo tensor_info; - tensor_info.set_name(tensor_name); - if (it_tensor_info != tensor_name_to_dtype.end()) { - tensor_info.set_dtype(it_tensor_info->second); - } else { - LOG(WARNING) - << "No dtype found for tensor with name: " << tensor_name << ". " - << "Building TensorInfo with only name for SignatureDef inputs. " - << "Downstream functionality including validation may be " - << "impacted."; - } - (*signature_def->mutable_inputs())[input_key] = tensor_info; -} - -// Adds an entry to the `outputs` map of the supplied SignatureDef. -void AddOutputToSignatureDef( - const string& tensor_name, - const std::unordered_map& tensor_name_to_dtype, - const string& output_key, SignatureDef* signature_def) { - if (tensor_name.empty()) { - LOG(WARNING) << "Tensor name not provided. Cannot add TensorInfo to " - "SignatureDef outputs."; - return; - } - // Extract the tensor-name in case the supplied string is a tensor-reference. - // Example: Extract "x" from "x:0". - std::size_t pos = tensor_name.find(":"); - const string key = - (pos != std::string::npos) ? tensor_name.substr(0, pos) : tensor_name; - const auto it_tensor_info = tensor_name_to_dtype.find(key); - TensorInfo tensor_info; - tensor_info.set_name(tensor_name); - if (it_tensor_info != tensor_name_to_dtype.end()) { - tensor_info.set_dtype(it_tensor_info->second); - } else { - LOG(WARNING) - << "No dtype found for tensor with name: " << tensor_name << ". " - << "Building TensorInfo with only name for SignatureDef outputs." - << " Downstream functionality including validation may be " - << "impacted."; - } - (*signature_def->mutable_outputs())[output_key] = tensor_info; -} - -// Builds a map from tensor name to the corresponding datatype, by parsing the -// MetaGraphDef. -Status BuildTensorNameToDtypeMap( - const MetaGraphDef& meta_graph_def, - std::unordered_map* tensor_name_to_dtype) { - GraphConstructorOptions opts; - Graph graph(OpRegistry::Global()); - TF_RETURN_IF_ERROR( - ConvertGraphDefToGraph(opts, meta_graph_def.graph_def(), &graph)); - for (Node* node : graph.nodes()) { - for (auto dt : node->output_types()) { - tensor_name_to_dtype->insert(std::make_pair(node->name(), dt)); - } - } - return Status::OK(); -} - -// Converts SessionBundle signatures to SavedModel signature-defs. -Status ConvertSignaturesToSignatureDefs(MetaGraphDef* meta_graph_def) { - Signatures signatures; - GetSignatures(*meta_graph_def, &signatures).IgnoreError(); - - // Build a map of tensor-names to the corresponding tensor-info with `name` - // and `dtype` fields. - std::unordered_map tensor_name_to_dtype; - TF_RETURN_IF_ERROR( - BuildTensorNameToDtypeMap(*meta_graph_def, &tensor_name_to_dtype)); - - TF_RETURN_IF_ERROR(ConvertDefaultSignatureToSignatureDef( - signatures, tensor_name_to_dtype, meta_graph_def)); - TF_RETURN_IF_ERROR(ConvertNamedSignaturesToSignatureDef( - signatures, tensor_name_to_dtype, meta_graph_def)); - return Status::OK(); -} - -// Converts a SessionBundle to a SavedModelBundle. -Status ConvertSessionBundleToSavedModelBundle( - SessionBundle& session_bundle, SavedModelBundle* saved_model_bundle) { - // Transfer ownership of the session from old to new. - saved_model_bundle->session = std::move(session_bundle.session); - - // Copy the meta graph def from the SessionBundle to the SavedModelBundle. - saved_model_bundle->meta_graph_def = session_bundle.meta_graph_def; - - // Convert signatures from session-bundle to signature-defs in - // saved-model-bundle. - return internal::ConvertSignaturesToSignatureDefs( - &saved_model_bundle->meta_graph_def); -} - -} // namespace internal - -Status LoadSessionBundleOrSavedModelBundle( - const SessionOptions& session_options, const RunOptions& run_options, - const string& export_dir, - const std::unordered_set& saved_model_tags, - SavedModelBundle* saved_model_bundle, bool* is_session_bundle) { - if (is_session_bundle != nullptr) { - *is_session_bundle = false; - } - if (MaybeSavedModelDirectory(export_dir)) { - LOG(INFO) - << "Attempting to load native SavedModelBundle in bundle-shim from: " - << export_dir; - - return LoadSavedModel(session_options, run_options, export_dir, - saved_model_tags, saved_model_bundle); - } else if (IsPossibleExportDirectory(export_dir)) { - LOG(ERROR) << "Found possible SessionBundle in export directory. " - "SessionBundle is deprecated. Use SavedModel instead."; - LOG(INFO) << "Attempting to up-convert SessionBundle to SavedModelBundle " - "in bundle-shim from: " - << export_dir; - if (is_session_bundle != nullptr) { - *is_session_bundle = true; - } - return LoadSavedModelFromLegacySessionBundlePath( - session_options, run_options, export_dir, saved_model_bundle); - } - return Status( - error::Code::NOT_FOUND, - strings::StrCat( - "Specified file path does not appear to contain a:\n" - "- Session bundle (should have a file called `export.meta`)\n" - "- or, SavedModel bundle (should have a file called " - "`saved_model.pb`)\n" - "Specified file path: ", - export_dir)); -} - -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow/contrib/session_bundle/bundle_shim.h b/tensorflow/contrib/session_bundle/bundle_shim.h deleted file mode 100644 index 7f0f9958d7d..00000000000 --- a/tensorflow/contrib/session_bundle/bundle_shim.h +++ /dev/null @@ -1,72 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Shim for systems that need to load both SessionBundle and -// SavedModelBundle interchangeably during migration to SavedModel. -#ifndef TENSORFLOW_CONTRIB_SESSION_BUNDLE_BUNDLE_SHIM_H_ -#define TENSORFLOW_CONTRIB_SESSION_BUNDLE_BUNDLE_SHIM_H_ - -#include - -#include "tensorflow/cc/saved_model/loader.h" -#include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/protobuf/meta_graph.pb.h" -#include "tensorflow/core/public/session.h" -#include "tensorflow/core/public/session_options.h" - -namespace tensorflow { -namespace serving { -namespace internal { - -// Adds an entry (key and value) to the input map of the signature def. Builds -// TensorInfos for the SignatureDefs by using the name and dtype information -// from the supplied map. -void AddInputToSignatureDef( - const string& tensor_name, - const std::unordered_map& tensor_name_to_dtype, - const string& input_map_key, SignatureDef* signature_def); - -// Adds an entry (key and value) to the output map of the signature def. Builds -// TensorInfos for the SignatureDefs by using the name and dtype information -// from the supplied map. -void AddOutputToSignatureDef( - const string& tensor_name, - const std::unordered_map& tensor_name_to_dtype, - const string& output_map_key, SignatureDef* signature_def); - -// Converts signatures in the MetaGraphDef into a SignatureDefs in the -// MetaGraphDef. -Status ConvertSignaturesToSignatureDefs(MetaGraphDef* meta_graph_def); - -// Converts a SessionBundle to a SavedModelBundle. -Status ConvertSessionBundleToSavedModelBundle( - SessionBundle& session_bundle, SavedModelBundle* saved_model_bundle); - -} // namespace internal - -// Loads a SavedModel from either a session-bundle path or a SavedModel bundle -// path. If `is_session_bundle` is not a nullptr, sets it to `true` iff -// SavedModel was up-converted and loaded from a SessionBundle. -// `is_session_bundle` value should not be used if error is returned. -Status LoadSessionBundleOrSavedModelBundle( - const SessionOptions& session_options, const RunOptions& run_options, - const string& export_dir, const std::unordered_set& tags, - SavedModelBundle* bundle, bool* is_session_bundle = nullptr); - -} // namespace serving -} // namespace tensorflow -#endif // TENSORFLOW_CONTRIB_SESSION_BUNDLE_BUNDLE_SHIM_H_ diff --git a/tensorflow/contrib/session_bundle/bundle_shim.py b/tensorflow/contrib/session_bundle/bundle_shim.py deleted file mode 100644 index f05cd5dab68..00000000000 --- a/tensorflow/contrib/session_bundle/bundle_shim.py +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Shim for systems that need to load both SessionBundle and SavedModel. - -This is intended to be used during migration to SavedModel. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from tensorflow.contrib.session_bundle import constants as legacy_constants -from tensorflow.contrib.session_bundle import manifest_pb2 -from tensorflow.contrib.session_bundle import session_bundle -from tensorflow.core.protobuf import meta_graph_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import meta_graph -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.saved_model import loader -from tensorflow.python.saved_model import signature_constants - - -def _add_input_to_signature_def(tensor_name, map_key, signature_def): - """Add input tensor to signature_def. - - Args: - tensor_name: string name of tensor to add to signature_def inputs - map_key: string key to key into signature_def inputs map - signature_def: object of type meta_graph_pb2.SignatureDef() - Sideffect: adds a TensorInfo with tensor_name to signature_def inputs map - keyed with map_key - """ - tensor_info = meta_graph_pb2.TensorInfo(name=tensor_name) - signature_def.inputs[map_key].CopyFrom(tensor_info) - - -def _add_output_to_signature_def(tensor_name, map_key, signature_def): - """Add output tensor to signature_def. - - Args: - tensor_name: string name of tensor to add to signature_def outputs - map_key: string key to key into signature_def outputs map - signature_def: object of type meta_graph_pb2.SignatureDef() - Sideffect: adds a TensorInfo with tensor_name to signature_def outputs map - keyed with map_key - """ - - tensor_info = meta_graph_pb2.TensorInfo(name=tensor_name) - signature_def.outputs[map_key].CopyFrom(tensor_info) - - -def _convert_default_signature_to_signature_def(signatures): - """Convert default signature to object of type SignatureDef. - - Args: - signatures: object of type manifest_pb2.Signatures() - - Returns: - object of type SignatureDef which contains a converted version of default - signature from input signatures object - - Returns None if signature is of generic type because it cannot be converted - to SignatureDef. - - """ - default_signature = signatures.default_signature - signature_def = meta_graph_pb2.SignatureDef() - if (default_signature.WhichOneof("type") == - legacy_constants.REGRESSION_SIGNATURE): - regression_signature = default_signature.regression_signature - signature_def.method_name = signature_constants.REGRESS_METHOD_NAME - _add_input_to_signature_def(regression_signature.input.tensor_name, - signature_constants.REGRESS_INPUTS, - signature_def) - _add_output_to_signature_def(regression_signature.output.tensor_name, - signature_constants.REGRESS_OUTPUTS, - signature_def) - elif (default_signature.WhichOneof("type") == - legacy_constants.CLASSIFICATION_SIGNATURE): - classification_signature = default_signature.classification_signature - signature_def.method_name = signature_constants.CLASSIFY_METHOD_NAME - _add_input_to_signature_def(classification_signature.input.tensor_name, - signature_constants.CLASSIFY_INPUTS, - signature_def) - _add_output_to_signature_def(classification_signature.classes.tensor_name, - signature_constants.CLASSIFY_OUTPUT_CLASSES, - signature_def) - _add_output_to_signature_def(classification_signature.scores.tensor_name, - signature_constants.CLASSIFY_OUTPUT_SCORES, - signature_def) - else: - logging.error( - "Only classification and regression default signatures " - "are supported for up-conversion. %s is not " - "supported", default_signature.WhichOneof("type")) - return None - return signature_def - - -def _convert_named_signatures_to_signature_def(signatures): - """Convert named signatures to object of type SignatureDef. - - Args: - signatures: object of type manifest_pb2.Signatures() - - Returns: - object of type SignatureDef which contains a converted version of named - signatures from input signatures object - - Raises: - RuntimeError: if input and output named signatures are not of type - GenericSignature - """ - signature_def = meta_graph_pb2.SignatureDef() - input_signature = signatures.named_signatures[ - signature_constants.PREDICT_INPUTS] - output_signature = signatures.named_signatures[ - signature_constants.PREDICT_OUTPUTS] - # TODO(pdudnik): what if there are other signatures? Mimic cr/140900781 once - # it is submitted. - if (input_signature.WhichOneof("type") != legacy_constants.GENERIC_SIGNATURE - or output_signature.WhichOneof("type") != - legacy_constants.GENERIC_SIGNATURE): - raise RuntimeError("Named input and output signatures can only be " - "up-converted if they are generic signature. " - "Input signature type is %s, output signature type is " - "%s" % (input_signature.WhichOneof("type"), - output_signature.WhichOneof("type"))) - - signature_def.method_name = signature_constants.PREDICT_METHOD_NAME - for key, val in input_signature.generic_signature.map.items(): - _add_input_to_signature_def(val.tensor_name, key, signature_def) - for key, val in output_signature.generic_signature.map.items(): - _add_output_to_signature_def(val.tensor_name, key, signature_def) - return signature_def - - -def _convert_signatures_to_signature_defs(metagraph_def): - """Produce default and named upconverted SignatureDef objects from Signatures. - - Args: - metagraph_def: object of type meta_graph_pb2.MetaGraphDef containing legacy - format Session Bundle signatures - - Returns: - default_signature_def: object of type SignatureDef which contains an - upconverted version of default signatures in metagraph_def - named_signature_def: object of type SignatureDef which contains an - upconverted version of named signatures in metagraph_def - """ - - collection_def = metagraph_def.collection_def - signatures_proto = manifest_pb2.Signatures() - signatures = collection_def[legacy_constants.SIGNATURES_KEY].any_list.value[0] - signatures.Unpack(signatures_proto) - - default_signature_def = None - named_signature_def = None - if signatures_proto.HasField("default_signature"): - default_signature_def = _convert_default_signature_to_signature_def( - signatures_proto) - if len(signatures_proto.named_signatures) > 1: - named_signature_def = _convert_named_signatures_to_signature_def( - signatures_proto) - return default_signature_def, named_signature_def - - -def _load_saved_model_from_session_bundle_path(export_dir, target, config): - """Load legacy TF Exporter/SessionBundle checkpoint. - - Args: - export_dir: the directory that contains files exported by exporter. - target: The execution engine to connect to. See target in - tf.compat.v1.Session() - config: A ConfigProto proto with configuration options. See config in - tf.compat.v1.Session() - - Returns: - session: a tensorflow session created from the variable files. - metagraph_def: The `MetaGraphDef` protocol buffer loaded in the provided - session. This can be used to further extract signature-defs, - collection-defs, etc. - This model is up-converted to SavedModel format. Specifically, metagraph_def - SignatureDef field is populated with Signatures converted from legacy - signatures contained within CollectionDef - - Raises: - RuntimeError: If metagraph already contains signature_def and cannot be - up-converted. - """ - - meta_graph_filename = os.path.join(export_dir, - legacy_constants.META_GRAPH_DEF_FILENAME) - - metagraph_def = meta_graph.read_meta_graph_file(meta_graph_filename) - if metagraph_def.signature_def: - raise RuntimeError("Legacy graph contains signature def, unable to " - "up-convert.") - - # Add SignatureDef to metagraph. - default_signature_def, named_signature_def = ( - _convert_signatures_to_signature_defs(metagraph_def)) - if default_signature_def: - metagraph_def.signature_def[ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY].CopyFrom( - default_signature_def) - if named_signature_def: - signature_def_key = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY - if default_signature_def: - signature_def_key += "_from_named" - metagraph_def.signature_def[signature_def_key].CopyFrom(named_signature_def) - - # We cannot just output session we loaded with older metagraph_def and - # up-converted metagraph definition because Session has an internal object of - # type Graph which is populated from meta_graph_def. If we do not create - # session with our new meta_graph_def, then Graph will be out of sync with - # meta_graph_def. - sess, metagraph_def = session_bundle.load_session_bundle_from_path( - export_dir, target, config, meta_graph_def=metagraph_def) - return sess, metagraph_def - - -def load_session_bundle_or_saved_model_bundle_from_path(export_dir, - tags=None, - target="", - config=None): - """Load session bundle from the given path. - - The function reads input from the export_dir, constructs the graph data to the - default graph and restores the parameters for the session created. - - Args: - export_dir: the directory that contains files exported by exporter. - tags: Set of string tags to identify the required MetaGraphDef when model is - saved as SavedModel. These should correspond to the tags used when saving - the variables using the SavedModel `save()` API. - target: The execution engine to connect to. See target in - tf.compat.v1.Session() - config: A ConfigProto proto with configuration options. See config in - tf.compat.v1.Session() - - Returns: - session: a tensorflow session created from the variable files. - meta_graph: a meta graph proto saved in the exporter directory. - - Raises: - RuntimeError: if the required files are missing or contain unrecognizable - fields, i.e. the exported model is invalid. - """ - metagraph_def = None - sess = None - if loader.maybe_saved_model_directory(export_dir): - sess = session.Session(target, graph=None, config=config) - metagraph_def = loader.load(sess, tags, export_dir) - elif session_bundle.maybe_session_bundle_dir(export_dir): - sess, metagraph_def = _load_saved_model_from_session_bundle_path( - export_dir, target, config) - else: - raise RuntimeError("SessionBundle or SavedModelBundle not found at " - "specified export location: %s" % export_dir) - - return sess, metagraph_def diff --git a/tensorflow/contrib/session_bundle/bundle_shim_test.cc b/tensorflow/contrib/session_bundle/bundle_shim_test.cc deleted file mode 100644 index 121fc2239dd..00000000000 --- a/tensorflow/contrib/session_bundle/bundle_shim_test.cc +++ /dev/null @@ -1,601 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/session_bundle/bundle_shim.h" - -#include "google/protobuf/any.pb.h" -#include "tensorflow/cc/saved_model/signature_constants.h" -#include "tensorflow/cc/saved_model/tag_constants.h" -#include "tensorflow/contrib/session_bundle/test_util.h" -#include "tensorflow/core/example/example.pb.h" -#include "tensorflow/core/example/feature.pb.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/protobuf/meta_graph.pb.h" - -namespace tensorflow { -namespace serving { -namespace internal { -namespace { - -constexpr char kSessionBundlePath[] = - "session_bundle/testdata/half_plus_two/00000123"; -constexpr char kSavedModelBundlePath[] = - "cc/saved_model/testdata/half_plus_two/00000123"; - -string MakeSerializedExample(float x) { - tensorflow::Example example; - auto* feature_map = example.mutable_features()->mutable_feature(); - (*feature_map)["x"].mutable_float_list()->add_value(x); - return example.SerializeAsString(); -} - -void ValidateHalfPlusTwo(const SavedModelBundle& saved_model_bundle, - const string& input_tensor_name, - const string& output_tensor_name) { - // Validate the half plus two behavior. - std::vector serialized_examples; - for (float x : {0, 1, 2, 3}) { - serialized_examples.push_back(MakeSerializedExample(x)); - } - Tensor input = test::AsTensor(serialized_examples, TensorShape({4})); - - std::vector outputs; - TF_ASSERT_OK(saved_model_bundle.session->Run( - {{input_tensor_name, input}}, {output_tensor_name}, {}, &outputs)); - ASSERT_EQ(outputs.size(), 1); - test::ExpectTensorEqual( - outputs[0], test::AsTensor({2, 2.5, 3, 3.5}, TensorShape({4, 1}))); -} - -void LoadAndValidateSavedModelBundle(const string& export_dir, - const std::unordered_set& tags, - const string& signature_def_key, - bool expect_session_bundle) { - SessionOptions session_options; - RunOptions run_options; - SavedModelBundle saved_model_bundle; - bool is_session_bundle = false; - TF_ASSERT_OK(LoadSessionBundleOrSavedModelBundle( - session_options, run_options, export_dir, tags, &saved_model_bundle, - &is_session_bundle)); - EXPECT_EQ(expect_session_bundle, is_session_bundle); - const MetaGraphDef meta_graph_def = saved_model_bundle.meta_graph_def; - const auto& signature_def_map = meta_graph_def.signature_def(); - - const auto& regression_entry = signature_def_map.find(signature_def_key); - ASSERT_FALSE(regression_entry == signature_def_map.end()); - SignatureDef regression_signature_def = regression_entry->second; - - EXPECT_EQ(1, regression_signature_def.inputs_size()); - ASSERT_FALSE(regression_signature_def.inputs().find(kRegressInputs) == - regression_signature_def.inputs().end()); - TensorInfo input_tensor_info = - regression_signature_def.inputs().find(kRegressInputs)->second; - EXPECT_EQ(1, regression_signature_def.outputs_size()); - // Ensure the TensorInfo has dtype populated. - EXPECT_EQ(DT_STRING, input_tensor_info.dtype()); - - ASSERT_FALSE(regression_signature_def.outputs().find(kRegressOutputs) == - regression_signature_def.outputs().end()); - TensorInfo output_tensor_info = - regression_signature_def.outputs().find(kRegressOutputs)->second; - // Ensure the TensorInfo has dtype populated. - EXPECT_EQ(DT_FLOAT, output_tensor_info.dtype()); - ValidateHalfPlusTwo(saved_model_bundle, input_tensor_info.name(), - output_tensor_info.name()); -} - -// Helper function to validate that the SignatureDef found in the MetaGraphDef -// with the provided key has the expected string representation. -void ValidateSignatureDef(const MetaGraphDef& meta_graph_def, const string& key, - const string& expected_string_signature_def) { - tensorflow::SignatureDef expected_signature; - CHECK(protobuf::TextFormat::ParseFromString(expected_string_signature_def, - &expected_signature)); - auto iter = meta_graph_def.signature_def().find(key); - ASSERT_TRUE(iter != meta_graph_def.signature_def().end()); - EXPECT_EQ(expected_signature.DebugString(), iter->second.DebugString()); -} - -// Checks that the input map in a signature def is populated correctly. -TEST(BundleShimTest, AddInputToSignatureDef) { - SignatureDef signature_def; - const string tensor_name = "foo_tensor"; - const string map_key = "foo_key"; - - // Build a map of tensor-name to dtype, for the unit-test. - std::unordered_map tensor_name_to_dtype; - tensor_name_to_dtype[tensor_name] = tensorflow::DT_STRING; - - AddInputToSignatureDef(tensor_name, tensor_name_to_dtype, map_key, - &signature_def); - EXPECT_EQ(1, signature_def.inputs_size()); - EXPECT_EQ(tensor_name, signature_def.inputs().find(map_key)->second.name()); -} - -// Checks that the output map in a signature def is populated correctly. -TEST(BundleShimTest, AddOutputToSignatureDef) { - SignatureDef signature_def; - const string tensor_name = "foo_tensor"; - const string map_key = "foo_key"; - - // Build a map of tensor-name to dtype, for the unit-test. - std::unordered_map tensor_name_to_dtype; - tensor_name_to_dtype[tensor_name] = tensorflow::DT_STRING; - - AddOutputToSignatureDef(tensor_name, tensor_name_to_dtype, map_key, - &signature_def); - EXPECT_EQ(1, signature_def.outputs_size()); - EXPECT_EQ(tensor_name, signature_def.outputs().find(map_key)->second.name()); -} - -// Checks that no signature defs are added if the default signature is missing. -TEST(BundleShimTest, DefaultSignatureMissing) { - MetaGraphDef meta_graph_def; - // Signatures signatures; - TF_EXPECT_OK(ConvertSignaturesToSignatureDefs(&meta_graph_def)); - EXPECT_EQ(0, meta_graph_def.signature_def_size()); -} - -// Checks that no signature defs are added if the default signature is empty. -TEST(BundleShimTest, DefaultSignatureEmpty) { - Signatures signatures; - signatures.mutable_default_signature(); - - MetaGraphDef meta_graph_def; - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - TF_EXPECT_OK(ConvertSignaturesToSignatureDefs(&meta_graph_def)); - EXPECT_EQ(0, meta_graph_def.signature_def_size()); -} - -// Checks the conversion to signature def for a regression default signature. -TEST(BundleShimTest, DefaultSignatureRegression) { - Signatures signatures; - RegressionSignature* regression_signature = - signatures.mutable_default_signature()->mutable_regression_signature(); - regression_signature->mutable_input()->set_tensor_name("foo-input"); - regression_signature->mutable_output()->set_tensor_name("foo-output"); - MetaGraphDef meta_graph_def; - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - TF_EXPECT_OK(ConvertSignaturesToSignatureDefs(&meta_graph_def)); - EXPECT_EQ(1, meta_graph_def.signature_def_size()); - const auto actual_signature_def = - meta_graph_def.signature_def().find(kDefaultServingSignatureDefKey); - EXPECT_EQ("foo-input", actual_signature_def->second.inputs() - .find(kRegressInputs) - ->second.name()); - EXPECT_EQ("foo-output", actual_signature_def->second.outputs() - .find(kRegressOutputs) - ->second.name()); - EXPECT_EQ(kRegressMethodName, actual_signature_def->second.method_name()); -} - -// Checks the conversion to signature def for a classification default -// signature. -TEST(BundleShimTest, DefaultSignatureClassification) { - Signatures signatures; - ClassificationSignature* classification_signature = - signatures.mutable_default_signature() - ->mutable_classification_signature(); - classification_signature->mutable_input()->set_tensor_name("foo-input"); - classification_signature->mutable_classes()->set_tensor_name("foo-classes"); - classification_signature->mutable_scores()->set_tensor_name("foo-scores"); - MetaGraphDef meta_graph_def; - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - TF_EXPECT_OK(ConvertSignaturesToSignatureDefs(&meta_graph_def)); - EXPECT_EQ(1, meta_graph_def.signature_def_size()); - const auto actual_signature_def = - meta_graph_def.signature_def().find(kDefaultServingSignatureDefKey); - EXPECT_EQ("foo-input", actual_signature_def->second.inputs() - .find(kClassifyInputs) - ->second.name()); - EXPECT_EQ("foo-classes", actual_signature_def->second.outputs() - .find(kClassifyOutputClasses) - ->second.name()); - EXPECT_EQ("foo-scores", actual_signature_def->second.outputs() - .find(kClassifyOutputScores) - ->second.name()); - EXPECT_EQ(kClassifyMethodName, actual_signature_def->second.method_name()); -} - -// Checks that generic default signatures are not up converted. -TEST(BundleShimTest, DefaultSignatureGeneric) { - TensorBinding input_binding; - input_binding.set_tensor_name("foo-input"); - - TensorBinding output_binding; - output_binding.set_tensor_name("foo-output"); - - Signatures signatures; - GenericSignature* generic_signature = - signatures.mutable_default_signature()->mutable_generic_signature(); - generic_signature->mutable_map()->insert({kPredictInputs, input_binding}); - generic_signature->mutable_map()->insert({kPredictOutputs, output_binding}); - - MetaGraphDef meta_graph_def; - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - TF_EXPECT_OK(ConvertSignaturesToSignatureDefs(&meta_graph_def)); - EXPECT_EQ(0, meta_graph_def.signature_def_size()); -} - -TEST(BundleShimTest, NamedRegressionSignatures) { - Signatures signatures; - - RegressionSignature* foo_regression_signature = - (*signatures.mutable_named_signatures())["foo"] - .mutable_regression_signature(); - foo_regression_signature->mutable_input()->set_tensor_name("foo-input"); - foo_regression_signature->mutable_output()->set_tensor_name("foo-output"); - - RegressionSignature* bar_regression_signature = - (*signatures.mutable_named_signatures())["bar"] - .mutable_regression_signature(); - bar_regression_signature->mutable_input()->set_tensor_name("bar-input"); - bar_regression_signature->mutable_output()->set_tensor_name("bar-output"); - - MetaGraphDef meta_graph_def; - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - TF_EXPECT_OK(ConvertSignaturesToSignatureDefs(&meta_graph_def)); - ASSERT_EQ(2, meta_graph_def.signature_def_size()); - - ValidateSignatureDef(meta_graph_def, "foo", - "inputs { " - " key: \"inputs\" " - " value { " - "name: \"foo-input\" " - " } " - "} " - "outputs { " - " key: \"outputs\" " - " value { " - " name: \"foo-output\" " - " } " - "} " - "method_name: \"tensorflow/serving/regress\" "); - ValidateSignatureDef(meta_graph_def, "bar", - "inputs { " - " key: \"inputs\" " - " value { " - "name: \"bar-input\" " - " } " - "} " - "outputs { " - " key: \"outputs\" " - " value { " - " name: \"bar-output\" " - " } " - "} " - "method_name: \"tensorflow/serving/regress\" "); -} - -TEST(BundleShimTest, NamedClassificationSignatures) { - Signatures signatures; - - ClassificationSignature* foo_classification_signature = - (*signatures.mutable_named_signatures())["foo"] - .mutable_classification_signature(); - foo_classification_signature->mutable_input()->set_tensor_name("foo-input"); - foo_classification_signature->mutable_classes()->set_tensor_name( - "foo-classes"); - - ClassificationSignature* bar_classification_signature = - (*signatures.mutable_named_signatures())["bar"] - .mutable_classification_signature(); - bar_classification_signature->mutable_input()->set_tensor_name("bar-input"); - bar_classification_signature->mutable_scores()->set_tensor_name("bar-scores"); - - MetaGraphDef meta_graph_def; - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - TF_EXPECT_OK(ConvertSignaturesToSignatureDefs(&meta_graph_def)); - ASSERT_EQ(2, meta_graph_def.signature_def_size()); - - ValidateSignatureDef(meta_graph_def, "foo", - "inputs { " - " key: \"inputs\" " - " value { " - "name: \"foo-input\" " - " } " - "} " - "outputs { " - " key: \"classes\" " - " value { " - " name: \"foo-classes\" " - " } " - "} " - "method_name: \"tensorflow/serving/classify\" "); - ValidateSignatureDef(meta_graph_def, "bar", - "inputs { " - " key: \"inputs\" " - " value { " - "name: \"bar-input\" " - " } " - "} " - "outputs { " - " key: \"scores\" " - " value { " - " name: \"bar-scores\" " - " } " - "} " - "method_name: \"tensorflow/serving/classify\" "); -} - -// Checks the Predict SignatureDef created when the named signatures have -// `inputs` and `outputs`. -TEST(BundleShimTest, NamedSignatureGenericInputsAndOutputs) { - TensorBinding input_binding; - input_binding.set_tensor_name("foo-input"); - - TensorBinding output_binding; - output_binding.set_tensor_name("foo-output"); - - Signatures signatures; - GenericSignature* input_generic_signature = - (*signatures.mutable_named_signatures())[kPredictInputs] - .mutable_generic_signature(); - input_generic_signature->mutable_map()->insert({"foo-input", input_binding}); - - GenericSignature* output_generic_signature = - (*signatures.mutable_named_signatures())[kPredictOutputs] - .mutable_generic_signature(); - output_generic_signature->mutable_map()->insert( - {"foo-output", output_binding}); - - MetaGraphDef meta_graph_def; - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - TF_EXPECT_OK(ConvertSignaturesToSignatureDefs(&meta_graph_def)); - EXPECT_EQ(1, meta_graph_def.signature_def_size()); - const auto actual_signature_def = - meta_graph_def.signature_def().find(kDefaultServingSignatureDefKey); - ASSERT_FALSE(actual_signature_def == meta_graph_def.signature_def().end()); - ASSERT_FALSE(actual_signature_def->second.inputs().find("foo-input") == - actual_signature_def->second.inputs().end()); - EXPECT_EQ( - "foo-input", - actual_signature_def->second.inputs().find("foo-input")->second.name()); - ASSERT_FALSE(actual_signature_def->second.outputs().find("foo-output") == - actual_signature_def->second.outputs().end()); - EXPECT_EQ( - "foo-output", - actual_signature_def->second.outputs().find("foo-output")->second.name()); - EXPECT_EQ(kPredictMethodName, actual_signature_def->second.method_name()); -} - -// Checks that a signature def is not added if the named signatures is generic -// but does not have `inputs` and `outputs`. -TEST(BundleShimTest, NamedSignatureGenericNoInputsOrOutputs) { - TensorBinding input_binding; - input_binding.set_tensor_name("foo-input"); - - TensorBinding output_binding; - output_binding.set_tensor_name("foo-output"); - - Signatures signatures; - GenericSignature* generic_signature = - (*signatures.mutable_named_signatures())["unknown"] - .mutable_generic_signature(); - generic_signature->mutable_map()->insert({kPredictInputs, input_binding}); - generic_signature->mutable_map()->insert({kPredictOutputs, output_binding}); - - MetaGraphDef meta_graph_def; - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - TF_EXPECT_OK(ConvertSignaturesToSignatureDefs(&meta_graph_def)); - EXPECT_EQ(0, meta_graph_def.signature_def_size()); -} - -// Checks that a signature def is not added when the named signatures have only -// one of `inputs` and `outputs`. -TEST(BundleShimTest, NamedSignatureGenericOnlyInput) { - TensorBinding input_binding; - input_binding.set_tensor_name("foo-input"); - - Signatures signatures; - GenericSignature* input_generic_signature = - (*signatures.mutable_named_signatures())[kPredictInputs] - .mutable_generic_signature(); - input_generic_signature->mutable_map()->insert({"foo-input", input_binding}); - - MetaGraphDef meta_graph_def; - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - TF_EXPECT_OK(ConvertSignaturesToSignatureDefs(&meta_graph_def)); - EXPECT_EQ(0, meta_graph_def.signature_def_size()); -} - -// Tests up-conversion of Signatures to SignatureDefs when both `default` and -// `named` signatures are present. -TEST(BundleShimTest, DefaultAndNamedSignatureWithPredict) { - Signatures signatures; - - // Build a generic signature corresponding to `inputs` and add it to the - // Signatures to up-convert. - TensorBinding input_binding; - input_binding.set_tensor_name("foo-input"); - GenericSignature* input_generic_signature = - (*signatures.mutable_named_signatures())[kPredictInputs] - .mutable_generic_signature(); - input_generic_signature->mutable_map()->insert({"foo-input", input_binding}); - - // Build a generic signature corresponding to `outputs` and add it to the - // Signatures to up-convert. - TensorBinding output_binding; - output_binding.set_tensor_name("foo-output"); - GenericSignature* output_generic_signature = - (*signatures.mutable_named_signatures())[kPredictOutputs] - .mutable_generic_signature(); - output_generic_signature->mutable_map()->insert( - {"foo-output", output_binding}); - - // Build a regression signature and set it as the default signature. - RegressionSignature* inputs_regression_signature = - (*signatures.mutable_default_signature()).mutable_regression_signature(); - inputs_regression_signature->mutable_input()->set_tensor_name("bar-input"); - - // Up-convert the available signatures to SignatureDefs. - MetaGraphDef meta_graph_def; - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - TF_EXPECT_OK(ConvertSignaturesToSignatureDefs(&meta_graph_def)); - EXPECT_EQ(2, meta_graph_def.signature_def_size()); - - // Verify that the default regression signature is converted to a - // SignatureDef that corresponds to the kDefaultServingSignatureDefKey. - const auto actual_signature_def_regress = - meta_graph_def.signature_def().find(kDefaultServingSignatureDefKey); - ASSERT_FALSE(actual_signature_def_regress == - meta_graph_def.signature_def().end()); - ASSERT_FALSE( - actual_signature_def_regress->second.inputs().find(kRegressInputs) == - actual_signature_def_regress->second.inputs().end()); - - // Verify that the `Predict` SignatureDef is created under a different key. - const auto actual_signature_def_predict = meta_graph_def.signature_def().find( - strings::StrCat(kDefaultServingSignatureDefKey, "_from_named")); - ASSERT_FALSE(actual_signature_def_predict == - meta_graph_def.signature_def().end()); - ASSERT_FALSE( - actual_signature_def_predict->second.inputs().find("foo-input") == - actual_signature_def_predict->second.inputs().end()); - EXPECT_EQ("foo-input", actual_signature_def_predict->second.inputs() - .find("foo-input") - ->second.name()); - ASSERT_FALSE( - actual_signature_def_predict->second.outputs().find("foo-output") == - actual_signature_def_predict->second.outputs().end()); - EXPECT_EQ("foo-output", actual_signature_def_predict->second.outputs() - .find("foo-output") - ->second.name()); - EXPECT_EQ(kPredictMethodName, - actual_signature_def_predict->second.method_name()); -} - -// Checks a basic up conversion for half plus two for SessionBundle. -TEST(BundleShimTest, BasicExportSessionBundle) { - const std::unordered_set tags = {"tag"}; - const string session_bundle_export_dir = - test_util::TestSrcDirPath(kSessionBundlePath); - LoadAndValidateSavedModelBundle(session_bundle_export_dir, tags, - kDefaultServingSignatureDefKey, - /*expect_session_bundle=*/true); - - // Verify that the named signature is also present. - SessionOptions session_options; - RunOptions run_options; - SavedModelBundle saved_model_bundle; - TF_ASSERT_OK(LoadSessionBundleOrSavedModelBundle(session_options, run_options, - session_bundle_export_dir, - tags, &saved_model_bundle)); - const MetaGraphDef meta_graph_def = saved_model_bundle.meta_graph_def; - const auto& signature_def_map = meta_graph_def.signature_def(); - bool found_named_signature = false; - for (const auto& entry : signature_def_map) { - const string& key = entry.first; - const SignatureDef& signature_def = entry.second; - - // We're looking for the key that is *not* kDefaultServingSignatureDefKey. - if (key == kDefaultServingSignatureDefKey) { - continue; - } - found_named_signature = true; - - EXPECT_EQ(1, signature_def.inputs_size()); - const auto it_inputs_x = signature_def.inputs().find("x"); - EXPECT_FALSE(it_inputs_x == signature_def.inputs().end()); - // Ensure the TensorInfo has name and dtype populated. - const TensorInfo& tensor_info_x = it_inputs_x->second; - EXPECT_EQ("x:0", tensor_info_x.name()); - EXPECT_EQ(DT_FLOAT, tensor_info_x.dtype()); - - EXPECT_EQ(1, signature_def.outputs_size()); - const auto it_outputs_y = signature_def.outputs().find("y"); - EXPECT_FALSE(it_outputs_y == signature_def.outputs().end()); - // Ensure the TensorInfo has name and dtype populated. - const TensorInfo& tensor_info_y = it_outputs_y->second; - EXPECT_EQ("y:0", tensor_info_y.name()); - EXPECT_EQ(DT_FLOAT, tensor_info_y.dtype()); - } - EXPECT_TRUE(found_named_signature); -} - -// Checks a basic load for half plus two for SavedModelBundle. -TEST(BundleShimTest, BasicExportSavedModel) { - const string saved_model_bundle_export_dir = - io::JoinPath(testing::TensorFlowSrcRoot(), kSavedModelBundlePath); - LoadAndValidateSavedModelBundle(saved_model_bundle_export_dir, - {kSavedModelTagServe}, "regress_x_to_y", - /*expect_session_bundle=*/false); -} - -// Checks a basic load fails with an invalid export path. -TEST(BundleShimTest, InvalidPath) { - const string invalid_export_dir = testing::TensorFlowSrcRoot(); - SessionOptions session_options; - RunOptions run_options; - SavedModelBundle saved_model_bundle; - Status status = LoadSessionBundleOrSavedModelBundle( - session_options, run_options, invalid_export_dir, {kSavedModelTagServe}, - &saved_model_bundle); - EXPECT_EQ(error::Code::NOT_FOUND, status.code()); -} - -// Checks that if loading a session bundle fails, the error is propagated to -// LoadSessionBundleOrSavedModelBundle(). -TEST(BundleShimTest, LoadSessionBundleError) { - const string session_bundle_export_dir = - test_util::TestSrcDirPath(kSessionBundlePath); - SessionOptions session_options; - RunOptions run_options; - // Invalid threadpool index to use for session-run calls. - run_options.set_inter_op_thread_pool(100); - SavedModelBundle saved_model_bundle; - EXPECT_FALSE(LoadSessionBundleOrSavedModelBundle(session_options, run_options, - session_bundle_export_dir, - {"tag"}, &saved_model_bundle) - .ok()); -} - -} // namespace -} // namespace internal -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow/contrib/session_bundle/bundle_shim_test.py b/tensorflow/contrib/session_bundle/bundle_shim_test.py deleted file mode 100644 index 86704432344..00000000000 --- a/tensorflow/contrib/session_bundle/bundle_shim_test.py +++ /dev/null @@ -1,362 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for bundle_shim.py.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os.path - -from tensorflow.contrib.session_bundle import bundle_shim -from tensorflow.contrib.session_bundle import constants -from tensorflow.contrib.session_bundle import manifest_pb2 -from tensorflow.core.protobuf import config_pb2 -from tensorflow.core.protobuf import meta_graph_pb2 -from tensorflow.python.framework import meta_graph -from tensorflow.python.framework import ops -import tensorflow.python.ops.parsing_ops # pylint: disable=unused-import -from tensorflow.python.platform import test -from tensorflow.python.saved_model import signature_constants -from tensorflow.python.saved_model import tag_constants -from tensorflow.python.util import compat - -SAVED_MODEL_PATH = ("cc/saved_model/testdata/half_plus_two/00000123") -SESSION_BUNDLE_PATH = "contrib/session_bundle/testdata/half_plus_two/00000123" - - -class BundleShimTest(test.TestCase): - - def testBadPath(self): - base_path = test.test_src_dir_path("/no/such/a/dir") - ops.reset_default_graph() - with self.assertRaises(RuntimeError): - _, _ = bundle_shim.load_session_bundle_or_saved_model_bundle_from_path( - base_path) - - def testAddInputToSignatureDef(self): - signature_def = meta_graph_pb2.SignatureDef() - signature_def_compare = meta_graph_pb2.SignatureDef() - - # Add input to signature-def corresponding to `foo_key`. - bundle_shim._add_input_to_signature_def("foo-name", "foo-key", - signature_def) - self.assertEqual(len(signature_def.inputs), 1) - self.assertEqual(len(signature_def.outputs), 0) - self.assertProtoEquals( - signature_def.inputs["foo-key"], - meta_graph_pb2.TensorInfo(name="foo-name")) - - # Attempt to add another input to the signature-def with the same tensor - # name and key. - bundle_shim._add_input_to_signature_def("foo-name", "foo-key", - signature_def) - self.assertEqual(len(signature_def.inputs), 1) - self.assertEqual(len(signature_def.outputs), 0) - self.assertProtoEquals( - signature_def.inputs["foo-key"], - meta_graph_pb2.TensorInfo(name="foo-name")) - - # Add another input to the signature-def corresponding to `bar-key`. - bundle_shim._add_input_to_signature_def("bar-name", "bar-key", - signature_def) - self.assertEqual(len(signature_def.inputs), 2) - self.assertEqual(len(signature_def.outputs), 0) - self.assertProtoEquals( - signature_def.inputs["bar-key"], - meta_graph_pb2.TensorInfo(name="bar-name")) - - # Add an input to the signature-def corresponding to `foo-key` with an - # updated tensor name. - bundle_shim._add_input_to_signature_def("bar-name", "foo-key", - signature_def) - self.assertEqual(len(signature_def.inputs), 2) - self.assertEqual(len(signature_def.outputs), 0) - self.assertProtoEquals( - signature_def.inputs["foo-key"], - meta_graph_pb2.TensorInfo(name="bar-name")) - - # Test that there are no other side-effects. - del signature_def.inputs["foo-key"] - del signature_def.inputs["bar-key"] - self.assertProtoEquals(signature_def, signature_def_compare) - - def testAddOutputToSignatureDef(self): - signature_def = meta_graph_pb2.SignatureDef() - signature_def_compare = meta_graph_pb2.SignatureDef() - - # Add output to signature-def corresponding to `foo_key`. - bundle_shim._add_output_to_signature_def("foo-name", "foo-key", - signature_def) - self.assertEqual(len(signature_def.outputs), 1) - self.assertEqual(len(signature_def.inputs), 0) - self.assertProtoEquals( - signature_def.outputs["foo-key"], - meta_graph_pb2.TensorInfo(name="foo-name")) - - # Attempt to add another output to the signature-def with the same tensor - # name and key. - bundle_shim._add_output_to_signature_def("foo-name", "foo-key", - signature_def) - self.assertEqual(len(signature_def.outputs), 1) - self.assertEqual(len(signature_def.inputs), 0) - self.assertProtoEquals( - signature_def.outputs["foo-key"], - meta_graph_pb2.TensorInfo(name="foo-name")) - - # Add another output to the signature-def corresponding to `bar-key`. - bundle_shim._add_output_to_signature_def("bar-name", "bar-key", - signature_def) - self.assertEqual(len(signature_def.outputs), 2) - self.assertEqual(len(signature_def.inputs), 0) - self.assertProtoEquals( - signature_def.outputs["bar-key"], - meta_graph_pb2.TensorInfo(name="bar-name")) - - # Add an output to the signature-def corresponding to `foo-key` with an - # updated tensor name. - bundle_shim._add_output_to_signature_def("bar-name", "foo-key", - signature_def) - self.assertEqual(len(signature_def.outputs), 2) - self.assertEqual(len(signature_def.inputs), 0) - self.assertProtoEquals( - signature_def.outputs["foo-key"], - meta_graph_pb2.TensorInfo(name="bar-name")) - - # Test that there are no other sideeffects. - del signature_def.outputs["foo-key"] - del signature_def.outputs["bar-key"] - self.assertProtoEquals(signature_def, signature_def_compare) - - def testConvertDefaultSignatureGenericToSignatureDef(self): - signatures_proto = manifest_pb2.Signatures() - generic_signature = manifest_pb2.GenericSignature() - signatures_proto.default_signature.generic_signature.CopyFrom( - generic_signature) - signature_def = bundle_shim._convert_default_signature_to_signature_def( - signatures_proto) - self.assertEquals(signature_def, None) - - def testConvertDefaultSignatureRegressionToSignatureDef(self): - signatures_proto = manifest_pb2.Signatures() - regression_signature = manifest_pb2.RegressionSignature() - regression_signature.input.CopyFrom( - manifest_pb2.TensorBinding( - tensor_name=signature_constants.REGRESS_INPUTS)) - regression_signature.output.CopyFrom( - manifest_pb2.TensorBinding( - tensor_name=signature_constants.REGRESS_OUTPUTS)) - signatures_proto.default_signature.regression_signature.CopyFrom( - regression_signature) - signature_def = bundle_shim._convert_default_signature_to_signature_def( - signatures_proto) - - # Validate regression signature correctly copied over. - self.assertEqual(signature_def.method_name, - signature_constants.REGRESS_METHOD_NAME) - self.assertEqual(len(signature_def.inputs), 1) - self.assertEqual(len(signature_def.outputs), 1) - self.assertProtoEquals( - signature_def.inputs[signature_constants.REGRESS_INPUTS], - meta_graph_pb2.TensorInfo(name=signature_constants.REGRESS_INPUTS)) - self.assertProtoEquals( - signature_def.outputs[signature_constants.REGRESS_OUTPUTS], - meta_graph_pb2.TensorInfo(name=signature_constants.REGRESS_OUTPUTS)) - - def testConvertDefaultSignatureClassificationToSignatureDef(self): - signatures_proto = manifest_pb2.Signatures() - classification_signature = manifest_pb2.ClassificationSignature() - classification_signature.input.CopyFrom( - manifest_pb2.TensorBinding( - tensor_name=signature_constants.CLASSIFY_INPUTS)) - classification_signature.classes.CopyFrom( - manifest_pb2.TensorBinding( - tensor_name=signature_constants.CLASSIFY_OUTPUT_CLASSES)) - classification_signature.scores.CopyFrom( - manifest_pb2.TensorBinding( - tensor_name=signature_constants.CLASSIFY_OUTPUT_SCORES)) - signatures_proto.default_signature.classification_signature.CopyFrom( - classification_signature) - - signatures_proto.default_signature.classification_signature.CopyFrom( - classification_signature) - signature_def = bundle_shim._convert_default_signature_to_signature_def( - signatures_proto) - - # Validate classification signature correctly copied over. - self.assertEqual(signature_def.method_name, - signature_constants.CLASSIFY_METHOD_NAME) - self.assertEqual(len(signature_def.inputs), 1) - self.assertEqual(len(signature_def.outputs), 2) - self.assertProtoEquals( - signature_def.inputs[signature_constants.CLASSIFY_INPUTS], - meta_graph_pb2.TensorInfo(name=signature_constants.CLASSIFY_INPUTS)) - self.assertProtoEquals( - signature_def.outputs[signature_constants.CLASSIFY_OUTPUT_SCORES], - meta_graph_pb2.TensorInfo( - name=signature_constants.CLASSIFY_OUTPUT_SCORES)) - self.assertProtoEquals( - signature_def.outputs[signature_constants.CLASSIFY_OUTPUT_CLASSES], - meta_graph_pb2.TensorInfo( - name=signature_constants.CLASSIFY_OUTPUT_CLASSES)) - - def testConvertNamedSignatureNonGenericToSignatureDef(self): - signatures_proto = manifest_pb2.Signatures() - regression_signature = manifest_pb2.RegressionSignature() - signatures_proto.named_signatures[ - signature_constants.PREDICT_INPUTS].regression_signature.CopyFrom( - regression_signature) - with self.assertRaises(RuntimeError): - _ = bundle_shim._convert_named_signatures_to_signature_def( - signatures_proto) - signatures_proto = manifest_pb2.Signatures() - classification_signature = manifest_pb2.ClassificationSignature() - signatures_proto.named_signatures[ - signature_constants.PREDICT_INPUTS].classification_signature.CopyFrom( - classification_signature) - with self.assertRaises(RuntimeError): - _ = bundle_shim._convert_named_signatures_to_signature_def( - signatures_proto) - - def testConvertNamedSignatureToSignatureDef(self): - signatures_proto = manifest_pb2.Signatures() - generic_signature = manifest_pb2.GenericSignature() - generic_signature.map["input_key"].CopyFrom( - manifest_pb2.TensorBinding(tensor_name="input")) - signatures_proto.named_signatures[ - signature_constants.PREDICT_INPUTS].generic_signature.CopyFrom( - generic_signature) - - generic_signature = manifest_pb2.GenericSignature() - generic_signature.map["output_key"].CopyFrom( - manifest_pb2.TensorBinding(tensor_name="output")) - signatures_proto.named_signatures[ - signature_constants.PREDICT_OUTPUTS].generic_signature.CopyFrom( - generic_signature) - signature_def = bundle_shim._convert_named_signatures_to_signature_def( - signatures_proto) - self.assertEqual(signature_def.method_name, - signature_constants.PREDICT_METHOD_NAME) - self.assertEqual(len(signature_def.inputs), 1) - self.assertEqual(len(signature_def.outputs), 1) - self.assertProtoEquals( - signature_def.inputs["input_key"], - meta_graph_pb2.TensorInfo(name="input")) - self.assertProtoEquals( - signature_def.outputs["output_key"], - meta_graph_pb2.TensorInfo(name="output")) - - def testConvertSignaturesToSignatureDefs(self): - base_path = test.test_src_dir_path(SESSION_BUNDLE_PATH) - meta_graph_filename = os.path.join(base_path, - constants.META_GRAPH_DEF_FILENAME) - metagraph_def = meta_graph.read_meta_graph_file(meta_graph_filename) - default_signature_def, named_signature_def = ( - bundle_shim._convert_signatures_to_signature_defs(metagraph_def)) - self.assertEqual(default_signature_def.method_name, - signature_constants.REGRESS_METHOD_NAME) - self.assertEqual(len(default_signature_def.inputs), 1) - self.assertEqual(len(default_signature_def.outputs), 1) - self.assertProtoEquals( - default_signature_def.inputs[signature_constants.REGRESS_INPUTS], - meta_graph_pb2.TensorInfo(name="tf_example:0")) - self.assertProtoEquals( - default_signature_def.outputs[signature_constants.REGRESS_OUTPUTS], - meta_graph_pb2.TensorInfo(name="Identity:0")) - self.assertEqual(named_signature_def.method_name, - signature_constants.PREDICT_METHOD_NAME) - self.assertEqual(len(named_signature_def.inputs), 1) - self.assertEqual(len(named_signature_def.outputs), 1) - self.assertProtoEquals( - named_signature_def.inputs["x"], meta_graph_pb2.TensorInfo(name="x:0")) - self.assertProtoEquals( - named_signature_def.outputs["y"], meta_graph_pb2.TensorInfo(name="y:0")) - - # Now try default signature only - collection_def = metagraph_def.collection_def - signatures_proto = manifest_pb2.Signatures() - signatures = collection_def[constants.SIGNATURES_KEY].any_list.value[0] - signatures.Unpack(signatures_proto) - named_only_signatures_proto = manifest_pb2.Signatures() - named_only_signatures_proto.CopyFrom(signatures_proto) - - default_only_signatures_proto = manifest_pb2.Signatures() - default_only_signatures_proto.CopyFrom(signatures_proto) - default_only_signatures_proto.named_signatures.clear() - default_only_signatures_proto.ClearField("named_signatures") - metagraph_def.collection_def[constants.SIGNATURES_KEY].any_list.value[ - 0].Pack(default_only_signatures_proto) - default_signature_def, named_signature_def = ( - bundle_shim._convert_signatures_to_signature_defs(metagraph_def)) - self.assertEqual(default_signature_def.method_name, - signature_constants.REGRESS_METHOD_NAME) - self.assertEqual(named_signature_def, None) - - named_only_signatures_proto.ClearField("default_signature") - metagraph_def.collection_def[constants.SIGNATURES_KEY].any_list.value[ - 0].Pack(named_only_signatures_proto) - default_signature_def, named_signature_def = ( - bundle_shim._convert_signatures_to_signature_defs(metagraph_def)) - self.assertEqual(named_signature_def.method_name, - signature_constants.PREDICT_METHOD_NAME) - self.assertEqual(default_signature_def, None) - - def testLegacyBasic(self): - base_path = test.test_src_dir_path(SESSION_BUNDLE_PATH) - ops.reset_default_graph() - sess, meta_graph_def = ( - bundle_shim.load_session_bundle_or_saved_model_bundle_from_path( - base_path, - tags=[""], - target="", - config=config_pb2.ConfigProto(device_count={"CPU": 2}))) - - self.assertTrue(sess) - asset_path = os.path.join(base_path, constants.ASSETS_DIRECTORY) - with sess.as_default(): - path1, path2 = sess.run(["filename1:0", "filename2:0"]) - self.assertEqual( - compat.as_bytes(os.path.join(asset_path, "hello1.txt")), path1) - self.assertEqual( - compat.as_bytes(os.path.join(asset_path, "hello2.txt")), path2) - - collection_def = meta_graph_def.collection_def - - signatures_any = collection_def[constants.SIGNATURES_KEY].any_list.value - self.assertEqual(len(signatures_any), 1) - - def testSavedModelBasic(self): - base_path = test.test_src_dir_path(SAVED_MODEL_PATH) - ops.reset_default_graph() - sess, meta_graph_def = ( - bundle_shim.load_session_bundle_or_saved_model_bundle_from_path( - base_path, - tags=[tag_constants.SERVING], - target="", - config=config_pb2.ConfigProto(device_count={"CPU": 2}))) - - self.assertTrue(sess) - - # Check basic signature def property. - signature_def = meta_graph_def.signature_def - self.assertEqual(signature_def["regress_x_to_y"].method_name, - signature_constants.REGRESS_METHOD_NAME) - with sess.as_default(): - output1 = sess.run(["filename_tensor:0"]) - self.assertEqual([compat.as_bytes("foo.txt")], output1) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/session_bundle/constants.py b/tensorflow/contrib/session_bundle/constants.py deleted file mode 100644 index e833baee791..00000000000 --- a/tensorflow/contrib/session_bundle/constants.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Constants for export/import.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -VERSION_FORMAT_SPECIFIER = "%08d" -ASSETS_DIRECTORY = "assets" -EXPORT_BASE_NAME = "export" -EXPORT_SUFFIX_NAME = "meta" -META_GRAPH_DEF_FILENAME = "export.meta" -VARIABLES_FILENAME = "export" -VARIABLES_FILENAME_PATTERN = "export-?????-of-?????" -VARIABLES_FILENAME_PATTERN_V2 = "export.data-?????-of-?????" -VARIABLES_INDEX_FILENAME_V2 = "export.index" -INIT_OP_KEY = "serving_init_op" -SIGNATURES_KEY = "serving_signatures" -ASSETS_KEY = "serving_assets" -GRAPH_KEY = "serving_graph" -REGRESSION_SIGNATURE = "regression_signature" -CLASSIFICATION_SIGNATURE = "classification_signature" -GENERIC_SIGNATURE = "generic_signature" diff --git a/tensorflow/contrib/session_bundle/example/BUILD b/tensorflow/contrib/session_bundle/example/BUILD deleted file mode 100644 index 37c8656616f..00000000000 --- a/tensorflow/contrib/session_bundle/example/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -# Description: TensorFlow Serving session_bundle example. - -package( - default_visibility = ["//tensorflow/contrib/session_bundle:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -# vardef("PYTHON_BIN_PATH", "/usr/bin/python") - -py_binary( - name = "export_half_plus_two", - srcs = [ - "export_half_plus_two.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/session_bundle:exporter", - ], -) diff --git a/tensorflow/contrib/session_bundle/example/export_half_plus_two.py b/tensorflow/contrib/session_bundle/example/export_half_plus_two.py deleted file mode 100644 index 83e91d390fe..00000000000 --- a/tensorflow/contrib/session_bundle/example/export_half_plus_two.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Exports a toy linear regression inference graph. - -Exports a TensorFlow graph to /tmp/half_plus_two/ based on the Exporter -format. - -This graph calculates, - y = a*x + b -where a and b are variables with a=0.5 and b=2. - -Output from this program is typically used to exercise Session -loading and execution code. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import sys - -import tensorflow as tf - -from tensorflow.contrib.session_bundle import exporter - -FLAGS = None - - -def Export(export_dir, use_checkpoint_v2): - with tf.Session() as sess: - # Make model parameters a&b variables instead of constants to - # exercise the variable reloading mechanisms. - a = tf.Variable(0.5, name="a") - b = tf.Variable(2.0, name="b") - - # Create a placeholder for serialized tensorflow.Example messages to be fed. - serialized_tf_example = tf.placeholder(tf.string, name="tf_example") - - # Parse the tensorflow.Example looking for a feature named "x" with a single - # floating point value. - feature_configs = {"x": tf.FixedLenFeature([1], dtype=tf.float32),} - tf_example = tf.parse_example(serialized_tf_example, feature_configs) - # Use tf.identity() to assign name - x = tf.identity(tf_example["x"], name="x") - - # Calculate, y = a*x + b - y = tf.add(tf.multiply(a, x), b, name="y") - - # Setup a standard Saver for our variables. - save = tf.train.Saver( - { - "a": a, - "b": b - }, - sharded=True, - write_version=tf.train.SaverDef.V2 if use_checkpoint_v2 else - tf.train.SaverDef.V1) - - # asset_path contains the base directory of assets used in training (e.g. - # vocabulary files). - original_asset_path = tf.constant("/tmp/original/export/assets") - # Ops reading asset files should reference the asset_path tensor - # which stores the original asset path at training time and the - # overridden assets directory at restore time. - asset_path = tf.Variable(original_asset_path, - name="asset_path", - trainable=False, - collections=[]) - assign_asset_path = asset_path.assign(original_asset_path) - - # Use a fixed global step number. - global_step_tensor = tf.Variable(123, name="global_step") - - # Create a RegressionSignature for our input and output. - regression_signature = exporter.regression_signature( - input_tensor=serialized_tf_example, - # Use tf.identity here because we export two signatures here. - # Otherwise only graph for one of the signatures will be loaded - # (whichever is created first) during serving. - output_tensor=tf.identity(y)) - named_graph_signature = { - "inputs": exporter.generic_signature({"x": x}), - "outputs": exporter.generic_signature({"y": y}) - } - - # Create two filename assets and corresponding tensors. - # TODO(b/26254158) Consider adding validation of file existence as well as - # hashes (e.g. sha1) for consistency. - original_filename1 = tf.constant("hello1.txt") - tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, original_filename1) - filename1 = tf.Variable(original_filename1, - name="filename1", - trainable=False, - collections=[]) - assign_filename1 = filename1.assign(original_filename1) - original_filename2 = tf.constant("hello2.txt") - tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, original_filename2) - filename2 = tf.Variable(original_filename2, - name="filename2", - trainable=False, - collections=[]) - assign_filename2 = filename2.assign(original_filename2) - - # Init op contains a group of all variables that we assign. - init_op = tf.group(assign_asset_path, assign_filename1, assign_filename2) - - # CopyAssets is used as a callback during export to copy files to the - # given export directory. - def CopyAssets(filepaths, export_path): - print("copying asset files to: %s" % export_path) - for filepath in filepaths: - print("copying asset file: %s" % filepath) - - # Run an export. - tf.global_variables_initializer().run() - export = exporter.Exporter(save) - export.init( - sess.graph.as_graph_def(), - init_op=init_op, - default_graph_signature=regression_signature, - named_graph_signatures=named_graph_signature, - assets_collection=tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS), - assets_callback=CopyAssets) - export.export(export_dir, global_step_tensor, sess) - - -def main(_): - Export(FLAGS.export_dir, FLAGS.use_checkpoint_v2) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.register("type", "bool", lambda v: v.lower() == "true") - parser.add_argument( - "--export_dir", - type=str, - default="/tmp/half_plus_two", - help="Directory where to export inference model." - ) - parser.add_argument( - "--use_checkpoint_v2", - type="bool", - nargs="?", - const=True, - default=False, - help="If true, write v2 checkpoint files.") - FLAGS, unparsed = parser.parse_known_args() - tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/session_bundle/exporter.py b/tensorflow/contrib/session_bundle/exporter.py deleted file mode 100644 index a78985b2e8f..00000000000 --- a/tensorflow/contrib/session_bundle/exporter.py +++ /dev/null @@ -1,332 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Export a TensorFlow model. - -See: go/tf-exporter -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import re -import six - -from google.protobuf.any_pb2 import Any - -from tensorflow.contrib.session_bundle import constants -from tensorflow.contrib.session_bundle import gc -from tensorflow.contrib.session_bundle import manifest_pb2 -from tensorflow.core.framework import graph_pb2 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.platform import gfile -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import saver as tf_saver -from tensorflow.python.training import training_util -from tensorflow.python.util import compat -from tensorflow.python.util.deprecation import deprecated - - -@deprecated("2017-06-30", - "No longer supported. Switch to SavedModel immediately.") -def gfile_copy_callback(files_to_copy, export_dir_path): - """Callback to copy files using `gfile.copy` to an export directory. - - This method is used as the default `assets_callback` in `Exporter.init` to - copy assets from the `assets_collection`. It can also be invoked directly to - copy additional supplementary files into the export directory (in which case - it is not a callback). - - Args: - files_to_copy: A dictionary that maps original file paths to desired - basename in the export directory. - export_dir_path: Directory to copy the files to. - """ - logging.info("Write assets into: %s using gfile_copy.", export_dir_path) - gfile.MakeDirs(export_dir_path) - for source_filepath, basename in files_to_copy.items(): - new_path = os.path.join( - compat.as_bytes(export_dir_path), compat.as_bytes(basename)) - logging.info("Copying asset %s to path %s.", source_filepath, new_path) - - if gfile.Exists(new_path): - # Guard against being restarted while copying assets, and the file - # existing and being in an unknown state. - # TODO(b/28676216): Do some file checks before deleting. - logging.info("Removing file %s.", new_path) - gfile.Remove(new_path) - gfile.Copy(source_filepath, new_path) - - -@deprecated("2017-06-30", - "No longer supported. Switch to SavedModel immediately.") -def regression_signature(input_tensor, output_tensor): - """Creates a regression signature. - - Args: - input_tensor: Tensor specifying the input to a graph. - output_tensor: Tensor specifying the output of a graph. - - Returns: - A Signature message. - """ - signature = manifest_pb2.Signature() - signature.regression_signature.input.tensor_name = input_tensor.name - signature.regression_signature.output.tensor_name = output_tensor.name - return signature - - -@deprecated("2017-06-30", - "No longer supported. Switch to SavedModel immediately.") -def classification_signature(input_tensor, - classes_tensor=None, - scores_tensor=None): - """Creates a classification signature. - - Args: - input_tensor: Tensor specifying the input to a graph. - classes_tensor: Tensor specifying the output classes of a graph. - scores_tensor: Tensor specifying the scores of the output classes. - - Returns: - A Signature message. - """ - signature = manifest_pb2.Signature() - signature.classification_signature.input.tensor_name = input_tensor.name - if classes_tensor is not None: - signature.classification_signature.classes.tensor_name = classes_tensor.name - if scores_tensor is not None: - signature.classification_signature.scores.tensor_name = scores_tensor.name - return signature - - -@deprecated("2017-06-30", - "No longer supported. Switch to SavedModel immediately.") -def generic_signature(name_tensor_map): - """Creates a generic signature of name to Tensor name. - - Args: - name_tensor_map: Map from logical name to Tensor. - - Returns: - A Signature message. - """ - signature = manifest_pb2.Signature() - for name, tensor in six.iteritems(name_tensor_map): - signature.generic_signature.map[name].tensor_name = tensor.name - return signature - - -class Exporter(object): - """Exporter helps package a TensorFlow model for serving. - - Args: - saver: Saver object. - """ - - def __init__(self, saver): - # Makes a copy of the saver-def and disables garbage-collection, since the - # exporter enforces garbage-collection independently. Specifically, since - # the exporter performs atomic copies of the saver output, it is required - # that garbage-collection via the underlying saver be disabled. - saver_def = saver.as_saver_def() - saver_def.ClearField("max_to_keep") - self._saver = tf_saver.Saver(saver_def=saver_def) - self._has_init = False - self._assets_to_copy = {} - - @deprecated("2017-06-30", - "No longer supported. Switch to SavedModel immediately.") - def init(self, - graph_def=None, - init_op=None, - clear_devices=False, - default_graph_signature=None, - named_graph_signatures=None, - assets_collection=None, - assets_callback=gfile_copy_callback): - """Initialization. - - Args: - graph_def: A GraphDef message of the graph to be used in inference. - GraphDef of default graph is used when None. - init_op: Op to be used in initialization. - clear_devices: If device info of the graph should be cleared upon export. - default_graph_signature: Default signature of the graph. - named_graph_signatures: Map of named input/output signatures of the graph. - assets_collection: A collection of constant asset filepath tensors. If set - the assets will be exported into the asset directory. - assets_callback: callback with two argument called during export with the - list of files to copy and the asset path. - Raises: - RuntimeError: if init is called more than once. - TypeError: if init_op is not an Operation or None. - ValueError: if asset file path tensors are not non-empty constant string - scalar tensors. - """ - # Avoid Dangerous default value [] - if named_graph_signatures is None: - named_graph_signatures = {} - assets = [] - if assets_collection: - for asset_tensor in assets_collection: - asset_filepath = self._file_path_value(asset_tensor) - if not asset_filepath: - raise ValueError("invalid asset filepath tensor %s" % asset_tensor) - basename = os.path.basename(asset_filepath) - assets.append((basename, asset_tensor)) - self._assets_to_copy[asset_filepath] = basename - - if self._has_init: - raise RuntimeError("init should be called only once") - self._has_init = True - - if graph_def or clear_devices: - copy = graph_pb2.GraphDef() - if graph_def: - copy.CopyFrom(graph_def) - else: - copy.CopyFrom(ops.get_default_graph().as_graph_def()) - if clear_devices: - for node in copy.node: - node.device = "" - graph_any_buf = Any() - graph_any_buf.Pack(copy) - ops.add_to_collection(constants.GRAPH_KEY, graph_any_buf) - - if init_op: - if not isinstance(init_op, ops.Operation): - raise TypeError("init_op needs to be an Operation: %s" % init_op) - ops.add_to_collection(constants.INIT_OP_KEY, init_op) - - signatures_proto = manifest_pb2.Signatures() - if default_graph_signature: - signatures_proto.default_signature.CopyFrom(default_graph_signature) - for signature_name, signature in six.iteritems(named_graph_signatures): - signatures_proto.named_signatures[signature_name].CopyFrom(signature) - signatures_any_buf = Any() - signatures_any_buf.Pack(signatures_proto) - ops.add_to_collection(constants.SIGNATURES_KEY, signatures_any_buf) - - for filename, tensor in assets: - asset = manifest_pb2.AssetFile() - asset.filename = filename - asset.tensor_binding.tensor_name = tensor.name - asset_any_buf = Any() - asset_any_buf.Pack(asset) - ops.add_to_collection(constants.ASSETS_KEY, asset_any_buf) - - self._assets_callback = assets_callback - - @deprecated("2017-06-30", - "No longer supported. Switch to SavedModel immediately.") - def export(self, - export_dir_base, - global_step_tensor, - sess=None, - exports_to_keep=None): - """Exports the model. - - Args: - export_dir_base: A string path to the base export dir. - global_step_tensor: An Tensor or tensor name providing the - global step counter to append to the export directory path and set - in the manifest version. - sess: A Session to use to save the parameters. - exports_to_keep: a gc.Path filter function used to determine the set of - exports to keep. If set to None, all versions will be kept. - - Returns: - The string path to the exported directory. - - Raises: - RuntimeError: if init is not called. - RuntimeError: if the export would overwrite an existing directory. - """ - if not self._has_init: - raise RuntimeError("init must be called first") - - # Export dir must not end with / or it will break exports to keep. Strip /. - if export_dir_base.endswith("/"): - export_dir_base = export_dir_base[:-1] - - global_step = training_util.global_step(sess, global_step_tensor) - export_dir = os.path.join( - compat.as_bytes(export_dir_base), - compat.as_bytes(constants.VERSION_FORMAT_SPECIFIER % global_step)) - - # Prevent overwriting on existing exports which could lead to bad/corrupt - # storage and loading of models. This is an important check that must be - # done before any output files or directories are created. - if gfile.Exists(export_dir): - raise RuntimeError("Overwriting exports can cause corruption and are " - "not allowed. Duplicate export dir: %s" % export_dir) - - # Output to a temporary directory which is atomically renamed to the final - # directory when complete. - tmp_export_dir = compat.as_text(export_dir) + "-tmp" - gfile.MakeDirs(tmp_export_dir) - - self._saver.save( - sess, - os.path.join( - compat.as_text(tmp_export_dir), - compat.as_text(constants.EXPORT_BASE_NAME)), - meta_graph_suffix=constants.EXPORT_SUFFIX_NAME) - - # Run the asset callback. - if self._assets_callback and self._assets_to_copy: - assets_dir = os.path.join( - compat.as_bytes(tmp_export_dir), - compat.as_bytes(constants.ASSETS_DIRECTORY)) - gfile.MakeDirs(assets_dir) - self._assets_callback(self._assets_to_copy, assets_dir) - - # TODO(b/27794910): Delete *checkpoint* file before rename. - gfile.Rename(tmp_export_dir, export_dir) - - if exports_to_keep: - # create a simple parser that pulls the export_version from the directory. - def parser(path): - if os.name == "nt": - match = re.match( - r"^" + export_dir_base.replace("\\", "/") + r"/(\d{8})$", - path.path.replace("\\", "/")) - else: - match = re.match(r"^" + export_dir_base + r"/(\d{8})$", path.path) - if not match: - return None - return path._replace(export_version=int(match.group(1))) - - paths_to_delete = gc.negation(exports_to_keep) - for p in paths_to_delete(gc.get_paths(export_dir_base, parser=parser)): - gfile.DeleteRecursively(p.path) - - return export_dir - - def _file_path_value(self, path_tensor): - """Returns the filepath value stored in constant `path_tensor`.""" - if not isinstance(path_tensor, ops.Tensor): - raise TypeError("tensor is not a Tensor") - if path_tensor.op.type != "Const": - raise TypeError("Only constants tensor are supported") - if path_tensor.dtype != dtypes.string: - raise TypeError("File paths should be string") - str_value = path_tensor.op.get_attr("value").string_val - if len(str_value) != 1: - raise TypeError("Only scalar tensors are supported") - return str_value[0] diff --git a/tensorflow/contrib/session_bundle/exporter_test.py b/tensorflow/contrib/session_bundle/exporter_test.py deleted file mode 100644 index 33f10a47c59..00000000000 --- a/tensorflow/contrib/session_bundle/exporter_test.py +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for exporter.py.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os.path - -from tensorflow.contrib.session_bundle import constants -from tensorflow.contrib.session_bundle import exporter -from tensorflow.contrib.session_bundle import gc -from tensorflow.contrib.session_bundle import manifest_pb2 -from tensorflow.core.framework import graph_pb2 -from tensorflow.core.protobuf import config_pb2 -from tensorflow.core.protobuf import saver_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import flags -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test -from tensorflow.python.training import saver - -FLAGS = flags.FLAGS - -GLOBAL_STEP = 222 - - -def tearDownModule(): - gfile.DeleteRecursively(test.get_temp_dir()) - - -class SaveRestoreShardedTest(test.TestCase): - - def doBasicsOneExportPath(self, - export_path, - clear_devices=False, - global_step=GLOBAL_STEP, - sharded=True, - export_count=1): - # Build a graph with 2 parameter nodes on different devices. - ops.reset_default_graph() - with session.Session( - target="", - config=config_pb2.ConfigProto(device_count={"CPU": 2})) as sess: - # v2 is an unsaved variable derived from v0 and v1. It is used to - # exercise the ability to run an init op when restoring a graph. - with sess.graph.device("/cpu:0"): - v0 = variables.VariableV1(10, name="v0") - with sess.graph.device("/cpu:1"): - v1 = variables.VariableV1(20, name="v1") - v2 = variables.VariableV1(1, name="v2", trainable=False, collections=[]) - assign_v2 = state_ops.assign(v2, math_ops.add(v0, v1)) - init_op = control_flow_ops.group(assign_v2, name="init_op") - - ops.add_to_collection("v", v0) - ops.add_to_collection("v", v1) - ops.add_to_collection("v", v2) - - named_tensor_bindings = {"logical_input_A": v0, "logical_input_B": v1} - signatures = { - "foo": - exporter.regression_signature( - input_tensor=v0, output_tensor=v1), - "generic": - exporter.generic_signature(named_tensor_bindings) - } - - asset_filepath_orig = os.path.join(test.get_temp_dir(), "hello42.txt") - asset_file = constant_op.constant(asset_filepath_orig, name="filename42") - ops.add_to_collection(ops.GraphKeys.ASSET_FILEPATHS, asset_file) - - with gfile.GFile(asset_filepath_orig, "w") as f: - f.write("your data here") - assets_collection = ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS) - - ignored_asset = os.path.join(test.get_temp_dir(), "ignored.txt") - with gfile.GFile(ignored_asset, "w") as f: - f.write("additional data here") - - variables.global_variables_initializer().run() - - # Run an export. - save = saver.Saver( - { - "v0": v0, - "v1": v1 - }, - restore_sequentially=True, - sharded=sharded, - write_version=saver_pb2.SaverDef.V1) - export = exporter.Exporter(save) - compare_def = ops.get_default_graph().as_graph_def() - export.init( - compare_def, - init_op=init_op, - clear_devices=clear_devices, - default_graph_signature=exporter.classification_signature( - input_tensor=v0), - named_graph_signatures=signatures, - assets_collection=assets_collection) - - for x in range(export_count): - export.export( - export_path, - constant_op.constant(global_step + x), - sess, - exports_to_keep=gc.largest_export_versions(2)) - # Set global_step to the last exported version, as the rest of the test - # uses it to construct model export path, loads model from it, and does - # verifications. We want to make sure to always use the last exported - # version, as old ones may have be garbage-collected. - global_step += export_count - 1 - - # Restore graph. - ops.reset_default_graph() - with session.Session( - target="", - config=config_pb2.ConfigProto(device_count={"CPU": 2})) as sess: - save = saver.import_meta_graph( - os.path.join(export_path, constants.VERSION_FORMAT_SPECIFIER % - global_step, constants.META_GRAPH_DEF_FILENAME)) - self.assertIsNotNone(save) - meta_graph_def = save.export_meta_graph() - collection_def = meta_graph_def.collection_def - - # Validate custom graph_def. - graph_def_any = collection_def[constants.GRAPH_KEY].any_list.value - self.assertEquals(len(graph_def_any), 1) - graph_def = graph_pb2.GraphDef() - graph_def_any[0].Unpack(graph_def) - if clear_devices: - for node in compare_def.node: - node.device = "" - self.assertProtoEquals(compare_def, graph_def) - - # Validate init_op. - init_ops = collection_def[constants.INIT_OP_KEY].node_list.value - self.assertEquals(len(init_ops), 1) - self.assertEquals(init_ops[0], "init_op") - - # Validate signatures. - signatures_any = collection_def[constants.SIGNATURES_KEY].any_list.value - self.assertEquals(len(signatures_any), 1) - signatures = manifest_pb2.Signatures() - signatures_any[0].Unpack(signatures) - default_signature = signatures.default_signature - self.assertEqual( - default_signature.classification_signature.input.tensor_name, "v0:0") - bindings = signatures.named_signatures["generic"].generic_signature.map - self.assertEquals(bindings["logical_input_A"].tensor_name, "v0:0") - self.assertEquals(bindings["logical_input_B"].tensor_name, "v1:0") - read_foo_signature = ( - signatures.named_signatures["foo"].regression_signature) - self.assertEquals(read_foo_signature.input.tensor_name, "v0:0") - self.assertEquals(read_foo_signature.output.tensor_name, "v1:0") - - # Validate the assets. - assets_any = collection_def[constants.ASSETS_KEY].any_list.value - self.assertEquals(len(assets_any), 1) - asset = manifest_pb2.AssetFile() - assets_any[0].Unpack(asset) - assets_path = os.path.join(export_path, - constants.VERSION_FORMAT_SPECIFIER % - global_step, constants.ASSETS_DIRECTORY, - "hello42.txt") - asset_contents = gfile.GFile(assets_path).read() - self.assertEqual(asset_contents, "your data here") - self.assertEquals("hello42.txt", asset.filename) - self.assertEquals("filename42:0", asset.tensor_binding.tensor_name) - ignored_asset_path = os.path.join(export_path, - constants.VERSION_FORMAT_SPECIFIER % - global_step, constants.ASSETS_DIRECTORY, - "ignored.txt") - self.assertFalse(gfile.Exists(ignored_asset_path)) - - # Validate graph restoration. - if sharded: - save.restore(sess, - os.path.join(export_path, - constants.VERSION_FORMAT_SPECIFIER % - global_step, - constants.VARIABLES_FILENAME_PATTERN)) - else: - save.restore(sess, - os.path.join(export_path, - constants.VERSION_FORMAT_SPECIFIER % - global_step, constants.VARIABLES_FILENAME)) - self.assertEqual(10, ops.get_collection("v")[0].eval()) - self.assertEqual(20, ops.get_collection("v")[1].eval()) - ops.get_collection(constants.INIT_OP_KEY)[0].run() - self.assertEqual(30, ops.get_collection("v")[2].eval()) - - def testDuplicateExportRaisesError(self): - export_path = os.path.join(test.get_temp_dir(), "export_duplicates") - self.doBasicsOneExportPath(export_path) - self.assertRaises(RuntimeError, self.doBasicsOneExportPath, export_path) - - def testBasics(self): - export_path = os.path.join(test.get_temp_dir(), "export") - self.doBasicsOneExportPath(export_path) - - def testBasicsNoShard(self): - export_path = os.path.join(test.get_temp_dir(), "export_no_shard") - self.doBasicsOneExportPath(export_path, sharded=False) - - def testClearDevice(self): - export_path = os.path.join(test.get_temp_dir(), "export_clear_device") - self.doBasicsOneExportPath(export_path, clear_devices=True) - - def testGC(self): - export_path = os.path.join(test.get_temp_dir(), "gc") - self.doBasicsOneExportPath(export_path, global_step=100) - self.assertEquals(gfile.ListDirectory(export_path), ["00000100"]) - self.doBasicsOneExportPath(export_path, global_step=101) - self.assertEquals( - sorted(gfile.ListDirectory(export_path)), ["00000100", "00000101"]) - self.doBasicsOneExportPath(export_path, global_step=102) - self.assertEquals( - sorted(gfile.ListDirectory(export_path)), ["00000101", "00000102"]) - - def testExportMultipleTimes(self): - export_path = os.path.join(test.get_temp_dir(), "export_multiple_times") - self.doBasicsOneExportPath(export_path, export_count=10) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/session_bundle/gc.py b/tensorflow/contrib/session_bundle/gc.py deleted file mode 100644 index 4a9adcf4838..00000000000 --- a/tensorflow/contrib/session_bundle/gc.py +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -r"""System for specifying garbage collection (GC) of path based data. - -This framework allows for GC of data specified by path names, for example files -on disk. gc.Path objects each represent a single item stored at a path and may -be a base directory, - /tmp/exports/0/... - /tmp/exports/1/... - ... -or a fully qualified file, - /tmp/train-1.ckpt - /tmp/train-2.ckpt - ... - -A gc filter function takes and returns a list of gc.Path items. Filter -functions are responsible for selecting Path items for preservation or deletion. -Note that functions should always return a sorted list. - -For example, - base_dir = "/tmp" - # create the directories - for e in xrange(10): - os.mkdir("%s/%d" % (base_dir, e), 0o755) - - # create a simple parser that pulls the export_version from the directory - def parser(path): - match = re.match("^" + base_dir + "/(\\d+)$", path.path) - if not match: - return None - return path._replace(export_version=int(match.group(1))) - - path_list = gc.get_paths("/tmp", parser) # contains all ten Paths - - every_fifth = gc.mod_export_version(5) - print(every_fifth(path_list)) # shows ["/tmp/0", "/tmp/5"] - - largest_three = gc.largest_export_versions(3) - print(largest_three(all_paths)) # shows ["/tmp/7", "/tmp/8", "/tmp/9"] - - both = gc.union(every_fifth, largest_three) - print(both(all_paths)) # shows ["/tmp/0", "/tmp/5", - # "/tmp/7", "/tmp/8", "/tmp/9"] - # delete everything not in 'both' - to_delete = gc.negation(both) - for p in to_delete(all_paths): - gfile.rmtree(p.path) # deletes: "/tmp/1", "/tmp/2", - # "/tmp/3", "/tmp/4", "/tmp/6", -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import heapq -import math -import os - -from tensorflow.python.platform import gfile -from tensorflow.python.util.deprecation import deprecated - -Path = collections.namedtuple('Path', 'path export_version') - - -@deprecated('2017-06-30', - 'No longer supported. Switch to SavedModel immediately.') -def largest_export_versions(n): - """Creates a filter that keeps the largest n export versions. - - Args: - n: number of versions to keep. - - Returns: - A filter function that keeps the n largest paths. - """ - def keep(paths): - heap = [] - for idx, path in enumerate(paths): - if path.export_version is not None: - heapq.heappush(heap, (path.export_version, idx)) - keepers = [paths[i] for _, i in heapq.nlargest(n, heap)] - return sorted(keepers) - - return keep - - -@deprecated('2017-06-30', - 'No longer supported. Switch to SavedModel immediately.') -def one_of_every_n_export_versions(n): - r"""Creates a filter that keeps one of every n export versions. - - Args: - n: interval size. - - Returns: - A filter function that keeps exactly one path from each interval - [0, n], (n, 2n], (2n, 3n], etc... If more than one path exists in an - interval the largest is kept. - """ - def keep(paths): - keeper_map = {} # map from interval to largest path seen in that interval - for p in paths: - if p.export_version is None: - # Skip missing export_versions. - continue - # Find the interval (with a special case to map export_version = 0 to - # interval 0. - interval = math.floor( - (p.export_version - 1) / n) if p.export_version else 0 - existing = keeper_map.get(interval, None) - if (not existing) or (existing.export_version < p.export_version): - keeper_map[interval] = p - return sorted(keeper_map.values()) - - return keep - - -@deprecated('2017-06-30', - 'No longer supported. Switch to SavedModel immediately.') -def mod_export_version(n): - """Creates a filter that keeps every export that is a multiple of n. - - Args: - n: step size. - - Returns: - A filter function that keeps paths where export_version % n == 0. - """ - def keep(paths): - keepers = [] - for p in paths: - if p.export_version % n == 0: - keepers.append(p) - return sorted(keepers) - return keep - - -@deprecated('2017-06-30', - 'No longer supported. Switch to SavedModel immediately.') -def union(lf, rf): - """Creates a filter that keeps the union of two filters. - - Args: - lf: first filter - rf: second filter - - Returns: - A filter function that keeps the n largest paths. - """ - def keep(paths): - l = set(lf(paths)) - r = set(rf(paths)) - return sorted(list(l|r)) - return keep - - -@deprecated('2017-06-30', - 'No longer supported. Switch to SavedModel immediately.') -def negation(f): - """Negate a filter. - - Args: - f: filter function to invert - - Returns: - A filter function that returns the negation of f. - """ - def keep(paths): - l = set(paths) - r = set(f(paths)) - return sorted(list(l-r)) - return keep - - -@deprecated('2017-06-30', - 'No longer supported. Switch to SavedModel immediately.') -def get_paths(base_dir, parser): - """Gets a list of Paths in a given directory. - - Args: - base_dir: directory. - parser: a function which gets the raw Path and can augment it with - information such as the export_version, or ignore the path by returning - None. An example parser may extract the export version from a path - such as "/tmp/exports/100" an another may extract from a full file - name such as "/tmp/checkpoint-99.out". - - Returns: - A list of Paths contained in the base directory with the parsing function - applied. - By default the following fields are populated, - - Path.path - The parsing function is responsible for populating, - - Path.export_version - """ - raw_paths = gfile.ListDirectory(base_dir) - paths = [] - for r in raw_paths: - p = parser(Path(os.path.join(base_dir, r), None)) - if p: - paths.append(p) - return sorted(paths) diff --git a/tensorflow/contrib/session_bundle/gc_test.py b/tensorflow/contrib/session_bundle/gc_test.py deleted file mode 100644 index 02725bb1cbb..00000000000 --- a/tensorflow/contrib/session_bundle/gc_test.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for session_bundle.gc.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import re - -from six.moves import xrange # pylint: disable=redefined-builtin - -from tensorflow.contrib.session_bundle import gc -from tensorflow.python.framework import test_util -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test - - -class GcTest(test_util.TensorFlowTestCase): - - def testLargestExportVersions(self): - paths = [gc.Path("/foo", 8), gc.Path("/foo", 9), gc.Path("/foo", 10)] - newest = gc.largest_export_versions(2) - n = newest(paths) - self.assertEquals(n, [gc.Path("/foo", 9), gc.Path("/foo", 10)]) - - def testLargestExportVersionsDoesNotDeleteZeroFolder(self): - paths = [gc.Path("/foo", 0), gc.Path("/foo", 3)] - newest = gc.largest_export_versions(2) - n = newest(paths) - self.assertEquals(n, [gc.Path("/foo", 0), gc.Path("/foo", 3)]) - - def testModExportVersion(self): - paths = [ - gc.Path("/foo", 4), gc.Path("/foo", 5), gc.Path("/foo", 6), - gc.Path("/foo", 9) - ] - mod = gc.mod_export_version(2) - self.assertEquals(mod(paths), [gc.Path("/foo", 4), gc.Path("/foo", 6)]) - mod = gc.mod_export_version(3) - self.assertEquals(mod(paths), [gc.Path("/foo", 6), gc.Path("/foo", 9)]) - - def testOneOfEveryNExportVersions(self): - paths = [ - gc.Path("/foo", 0), gc.Path("/foo", 1), gc.Path("/foo", 3), - gc.Path("/foo", 5), gc.Path("/foo", 6), gc.Path("/foo", 7), - gc.Path("/foo", 8), gc.Path("/foo", 33) - ] - one_of = gc.one_of_every_n_export_versions(3) - self.assertEquals( - one_of(paths), [ - gc.Path("/foo", 3), gc.Path("/foo", 6), gc.Path("/foo", 8), - gc.Path("/foo", 33) - ]) - - def testOneOfEveryNExportVersionsZero(self): - # Zero is a special case since it gets rolled into the first interval. - # Test that here. - paths = [gc.Path("/foo", 0), gc.Path("/foo", 4), gc.Path("/foo", 5)] - one_of = gc.one_of_every_n_export_versions(3) - self.assertEquals(one_of(paths), [gc.Path("/foo", 0), gc.Path("/foo", 5)]) - - def testUnion(self): - paths = [] - for i in xrange(10): - paths.append(gc.Path("/foo", i)) - f = gc.union(gc.largest_export_versions(3), gc.mod_export_version(3)) - self.assertEquals( - f(paths), [ - gc.Path("/foo", 0), gc.Path("/foo", 3), gc.Path("/foo", 6), - gc.Path("/foo", 7), gc.Path("/foo", 8), gc.Path("/foo", 9) - ]) - - def testNegation(self): - paths = [ - gc.Path("/foo", 4), gc.Path("/foo", 5), gc.Path("/foo", 6), - gc.Path("/foo", 9) - ] - mod = gc.negation(gc.mod_export_version(2)) - self.assertEquals(mod(paths), [gc.Path("/foo", 5), gc.Path("/foo", 9)]) - mod = gc.negation(gc.mod_export_version(3)) - self.assertEquals(mod(paths), [gc.Path("/foo", 4), gc.Path("/foo", 5)]) - - def testPathsWithParse(self): - base_dir = os.path.join(test.get_temp_dir(), "paths_parse") - self.assertFalse(gfile.Exists(base_dir)) - for p in xrange(3): - gfile.MakeDirs(os.path.join(base_dir, "%d" % p)) - # add a base_directory to ignore - gfile.MakeDirs(os.path.join(base_dir, "ignore")) - - # create a simple parser that pulls the export_version from the directory. - def parser(path): - match = re.match(r"^" + base_dir + r"/(\d+)$", path.path) - if not match: - return None - return path._replace(export_version=int(match.group(1))) - - self.assertEquals( - gc.get_paths( - base_dir, parser=parser), [ - gc.Path(os.path.join(base_dir, "0"), 0), - gc.Path(os.path.join(base_dir, "1"), 1), - gc.Path(os.path.join(base_dir, "2"), 2) - ]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/session_bundle/manifest.proto b/tensorflow/contrib/session_bundle/manifest.proto deleted file mode 100644 index 482ed372dc1..00000000000 --- a/tensorflow/contrib/session_bundle/manifest.proto +++ /dev/null @@ -1,70 +0,0 @@ -syntax = "proto3"; - -package tensorflow.serving; - -// Signatures of model export. -message Signatures { - // Default signature of the graph. - // WARNING(break-tutorial-inline-code): The following code snippet is - // in-lined in tutorials, please update tutorial documents accordingly - // whenever code changes. - Signature default_signature = 1; - - // Named signatures of the graph. - map named_signatures = 2; -}; - -// A binding to a tensor including the name and, possibly in the future, type -// or other metadata. For example, this may specify whether a tensor supports -// batch vs single inference. -message TensorBinding { - // The name of the tensor to bind to. - string tensor_name = 1; -}; - -// An asset file or set of sharded files with the same name that will be bound -// to a tensor at init / session_bundle load time. -message AssetFile { - // The tensor to bind the asset filename to. - TensorBinding tensor_binding = 1; - // The filename within the assets directory. Note: does not include the base - // path or asset directory prefix. Base paths can and will change when models - // are deployed for serving. - string filename = 2; -} - -// A Signature specifies the inputs and outputs of commonly used graphs. -message Signature { - oneof type { - RegressionSignature regression_signature = 1; - ClassificationSignature classification_signature = 2; - GenericSignature generic_signature = 3; - } -}; - -// RegressionSignature specifies a graph that takes an input and returns an -// output. -message RegressionSignature { - TensorBinding input = 1; - TensorBinding output = 2; -}; - -// ClassificationSignature specifies a graph that takes an input and returns -// classes and their scores. -// WARNING(break-tutorial-inline-code): The following code snippet is -// in-lined in tutorials, please update tutorial documents accordingly -// whenever code changes. -message ClassificationSignature { - TensorBinding input = 1; - TensorBinding classes = 2; - TensorBinding scores = 3; -}; - -// GenericSignature specifies a map from logical name to Tensor name. -// Typical application of GenericSignature is to use a single GenericSignature -// that includes all of the Tensor nodes and target names that may be useful at -// serving, analysis or debugging time. The recommended name for this signature -// in the ModelManifest is "generic_bindings". -message GenericSignature { - map map = 1; -}; diff --git a/tensorflow/contrib/session_bundle/session_bundle.cc b/tensorflow/contrib/session_bundle/session_bundle.cc deleted file mode 100644 index 996e4ce0b80..00000000000 --- a/tensorflow/contrib/session_bundle/session_bundle.cc +++ /dev/null @@ -1,273 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/session_bundle/session_bundle.h" - -#include -#include -#include - -#include "google/protobuf/any.pb.h" -#include "tensorflow/contrib/session_bundle/manifest.pb.h" -#include "tensorflow/core/framework/graph.pb.h" -#include "tensorflow/core/framework/graph_def_util.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/lib/monitoring/counter.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/protobuf_internal.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/protobuf/meta_graph.pb.h" -#include "tensorflow/core/protobuf/saver.pb.h" -#include "tensorflow/core/public/session_options.h" -#include "tensorflow/core/util/tensor_bundle/naming.h" - -namespace tensorflow { -namespace serving { -namespace { - -auto* load_attempt_count = monitoring::Counter<2>::New( - "/tensorflow/contrib/session_bundle/load_attempt_count", - "The number of times a SessionBundle was requested to be loaded.", - "model_path", "status"); -auto* load_latency = monitoring::Counter<1>::New( - "/tensorflow/contrib/session_bundle/load_latency", - "Latency in microseconds for SessionBundles that were successfully loaded.", - "model_path"); -constexpr char kLoadAttemptFail[] = "fail"; -constexpr char kLoadAttemptSuccess[] = "success"; - -// Create a session using the given options and load the graph. -Status CreateSessionFromGraphDef(const SessionOptions& options, - const GraphDef& graph, - std::unique_ptr* session) { - session->reset(NewSession(options)); - return (*session)->Create(graph); -} - -Status GetMetaGraphDefFromExport(const StringPiece export_dir, - MetaGraphDef* meta_graph_def) { - const string meta_graph_def_path = - io::JoinPath(export_dir, kMetaGraphDefFilename); - return ReadBinaryProto(Env::Default(), meta_graph_def_path, meta_graph_def); -} - -// Creates a string tensor. -Tensor CreateStringTensor(const string& value) { - Tensor tensor(DT_STRING, TensorShape({})); - tensor.scalar()() = value; - return tensor; -} - -// Adds Assets related tensors (assets_dir and asset files) to the inputs. -void AddAssetsTensorsToInputs(const StringPiece export_dir, - const std::vector& asset_files, - std::vector>* inputs) { - if (asset_files.empty()) { - return; - } - for (auto& asset : asset_files) { - Tensor assets_file_tensor = CreateStringTensor( - io::JoinPath(export_dir, kAssetsDirectory, asset.filename())); - inputs->push_back( - {asset.tensor_binding().tensor_name(), assets_file_tensor}); - } -} - -// Historically, model exporter(exporter.py) takes only saver with sharded=True, -// and therefore always exports checkpoint in pattern file names. In practice, -// instead of training from scratch and export directly, we usually want to -// restore from existing checkpoints and then export directly. To support such -// case, model exporter now supports reusing saver object restored from existing -// checkpoint, that may have sharded=False - it will then export checkpoint file -// in plain file name. This method is to support models exported by both types -// of saver object. The change is backward-compatible, therefore no changes are -// needed for existing model exports. -// -// Checkpoint v2 support: Variables exported using tf-exporter in the checkpoint -// v2 format will have export.index and export.data-?????-of-????? files as -// opposed to just an export or export-?????-of-????? file. The V2 save/restore -// code accepts a filename prefix and assumes both prefix.index and -// prefix.data-* are present in the filesystem. So if we see export.index -// present in the export_dir, we know the export is in V2 format and we return -// /export as this prefix. -string GetVariablesFilename(const StringPiece export_dir) { - const char kVariablesFilename[] = "export"; - const string kVariablesIndexFilename = MetaFilename("export"); // V2 ckpts - const char kVariablesFilenamePattern[] = "export-\?\?\?\?\?-of-\?\?\?\?\?"; - if (Env::Default() - ->FileExists(io::JoinPath(export_dir, kVariablesFilename)) - .ok() || - // This works for the case of V2 because the variables filename is taken - // as a prefix in the save/restore abstraction, and the index and actual - // variables are meant to be present as prefix.index and - // prefix.data-?????-of-?????. - Env::Default() - ->FileExists(io::JoinPath(export_dir, kVariablesIndexFilename)) - .ok()) { - return io::JoinPath(export_dir, kVariablesFilename); - } else { - return io::JoinPath(export_dir, kVariablesFilenamePattern); - } -} - -Status RunRestoreOp(const RunOptions& run_options, const StringPiece export_dir, - const std::vector& asset_files, - const StringPiece restore_op_name, - const StringPiece variables_filename_const_op_name, - Session* session) { - LOG(INFO) << "Running restore op for SessionBundle: " << restore_op_name - << ", " << variables_filename_const_op_name; - Tensor variables_tensor = - CreateStringTensor(GetVariablesFilename(export_dir)); - std::vector> inputs = { - {string(variables_filename_const_op_name), variables_tensor}}; - AddAssetsTensorsToInputs(export_dir, asset_files, &inputs); - RunMetadata run_metadata; - return session->Run(run_options, inputs, {}, {string(restore_op_name)}, - nullptr /* outputs */, &run_metadata); -} - -Status RunInitOp(const RunOptions& run_options, const StringPiece export_dir, - const std::vector& asset_files, - const StringPiece init_op_name, Session* session) { - LOG(INFO) << "Running init op for SessionBundle"; - std::vector> inputs; - AddAssetsTensorsToInputs(export_dir, asset_files, &inputs); - RunMetadata run_metadata; - return session->Run(run_options, inputs, {}, {string(init_op_name)}, - nullptr /* outputs */, &run_metadata); -} - -Status LoadSessionBundleFromPathUsingRunOptionsInternal( - const SessionOptions& options, const RunOptions& run_options, - const StringPiece export_dir, SessionBundle* const bundle) { - LOG(INFO) << "Attempting to load a SessionBundle from: " << export_dir; - LOG(INFO) << "Using RunOptions: " << DebugStringIfAvailable(run_options); - TF_RETURN_IF_ERROR( - GetMetaGraphDefFromExport(export_dir, &(bundle->meta_graph_def))); - - // Deprecated SessionBundle models may fail to load because newly added - // attributes are not added to the Graph in the default Session initialization - // flow. Add an explicit call here when first loading the graph from disk. - TF_RETURN_IF_ERROR( - AddDefaultAttrsToGraphDef(bundle->meta_graph_def.mutable_graph_def(), - *OpRegistry::Global(), 0 /* node_offset */)); - - const auto& collection_def_map = bundle->meta_graph_def.collection_def(); - const auto graph_it = bundle->meta_graph_def.collection_def().find(kGraphKey); - if (graph_it != collection_def_map.end()) { - const CollectionDef& graph_collection_def = graph_it->second; - // Use serving graph_def in MetaGraphDef collection_def. - if (graph_collection_def.any_list().value_size() != 1) { - return errors::FailedPrecondition( - "Expected exactly one serving GraphDef in : ", export_dir); - } - const auto& any = graph_collection_def.any_list().value(0); - GraphDef graph_def; - TF_RETURN_IF_ERROR(ParseAny(any, &graph_def, "tensorflow.GraphDef")); - TF_RETURN_IF_ERROR( - CreateSessionFromGraphDef(options, graph_def, &bundle->session)); - } else { - // Fallback to use the graph_def in the MetaGraphDef. - const GraphDef& graph_def = bundle->meta_graph_def.graph_def(); - TF_RETURN_IF_ERROR( - CreateSessionFromGraphDef(options, graph_def, &bundle->session)); - } - - std::vector asset_files; - const auto assets_it = collection_def_map.find(kAssetsKey); - if (assets_it != collection_def_map.end()) { - const auto& any_assets = assets_it->second.any_list().value(); - for (const auto& any_asset : any_assets) { - AssetFile asset_file; - TF_RETURN_IF_ERROR( - ParseAny(any_asset, &asset_file, "tensorflow.serving.AssetFile")); - asset_files.push_back(asset_file); - } - } - - TF_RETURN_IF_ERROR( - RunRestoreOp(run_options, export_dir, asset_files, - bundle->meta_graph_def.saver_def().restore_op_name(), - bundle->meta_graph_def.saver_def().filename_tensor_name(), - bundle->session.get())); - - const auto init_op_it = collection_def_map.find(kInitOpKey); - if (init_op_it != collection_def_map.end()) { - if (init_op_it->second.node_list().value_size() != 1) { - return errors::FailedPrecondition(strings::StrCat( - "Expected exactly one serving init op in : ", export_dir)); - } - TF_RETURN_IF_ERROR(RunInitOp(run_options, export_dir, asset_files, - init_op_it->second.node_list().value(0), - bundle->session.get())); - } - - return Status::OK(); -} - -} // namespace - -Status LoadSessionBundleFromPath(const SessionOptions& options, - const StringPiece export_dir, - SessionBundle* const bundle) { - TF_RETURN_IF_ERROR(LoadSessionBundleFromPathUsingRunOptions( - options, RunOptions(), export_dir, bundle)); - return Status::OK(); -} - -Status LoadSessionBundleFromPathUsingRunOptions(const SessionOptions& options, - const RunOptions& run_options, - const StringPiece export_dir, - SessionBundle* const bundle) { - const uint64 start_microseconds = Env::Default()->NowMicros(); - const Status status = LoadSessionBundleFromPathUsingRunOptionsInternal( - options, run_options, export_dir, bundle); - - const uint64 load_latency_microsecs = [&]() -> uint64 { - const uint64 end_microseconds = Env::Default()->NowMicros(); - // Avoid clock skew. - if (end_microseconds < start_microseconds) return 0; - return end_microseconds - start_microseconds; - }(); - auto log_and_count = [&](const string& status_str) { - LOG(INFO) << "Loading SessionBundle: " << status_str << ". Took " - << load_latency_microsecs << " microseconds."; - load_attempt_count->GetCell(string(export_dir), status_str)->IncrementBy(1); - }; - if (status.ok()) { - log_and_count(kLoadAttemptSuccess); - } else { - log_and_count(kLoadAttemptFail); - } - load_latency->GetCell(string(export_dir)) - ->IncrementBy(load_latency_microsecs); - return status; -} - -bool IsPossibleExportDirectory(const StringPiece directory) { - const string meta_graph_def_path = - io::JoinPath(directory, kMetaGraphDefFilename); - return Env::Default()->FileExists(meta_graph_def_path).ok(); -} - -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow/contrib/session_bundle/session_bundle.h b/tensorflow/contrib/session_bundle/session_bundle.h deleted file mode 100644 index b2be46efa6d..00000000000 --- a/tensorflow/contrib/session_bundle/session_bundle.h +++ /dev/null @@ -1,85 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Low-level functionality for setting up a inference Session. - -#ifndef TENSORFLOW_CONTRIB_SESSION_BUNDLE_SESSION_BUNDLE_H_ -#define TENSORFLOW_CONTRIB_SESSION_BUNDLE_SESSION_BUNDLE_H_ - -#include - -#include "tensorflow/contrib/session_bundle/manifest.pb.h" -#include "tensorflow/contrib/session_bundle/signature.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/protobuf/meta_graph.pb.h" -#include "tensorflow/core/protobuf/saver.pb.h" -#include "tensorflow/core/public/session.h" -#include "tensorflow/core/public/session_options.h" - -namespace tensorflow { -namespace serving { - -const char kMetaGraphDefFilename[] = "export.meta"; -const char kAssetsDirectory[] = "assets"; -const char kInitOpKey[] = "serving_init_op"; -const char kAssetsKey[] = "serving_assets"; -const char kGraphKey[] = "serving_graph"; - -// Data and objects loaded from a python Exporter export. -// WARNING(break-tutorial-inline-code): The following code snippet is -// in-lined in tutorials, please update tutorial documents accordingly -// whenever code changes. -struct SessionBundle { - std::unique_ptr session; - MetaGraphDef meta_graph_def; - - // A TensorFlow Session does not Close itself on destruction. To avoid - // resource leaks, we explicitly call Close on Sessions that we create. - ~SessionBundle() { - if (session) { - session->Close().IgnoreError(); - } - } - - SessionBundle(SessionBundle&&) = default; - SessionBundle() = default; -}; - -// Loads a manifest and initialized session using the output of an Exporter. -Status LoadSessionBundleFromPath(const SessionOptions& options, - const StringPiece export_dir, - SessionBundle* bundle); - -// Similar to the LoadSessionBundleFromPath(), but also allows the session run -// invocations for the restore and init ops to be configured with -// tensorflow::RunOptions. -// -// This method is EXPERIMENTAL and may change or be removed. -Status LoadSessionBundleFromPathUsingRunOptions( - const SessionOptions& session_options, const RunOptions& run_options, - const StringPiece export_dir, SessionBundle* bundle); - -// Sanity checks whether the directory looks like an export directory. Note that -// we don't try to load any data in this method. -// -// If the method returns false this is definitely not an export directory, if it -// returns true, it is no guarantee that the model will load. -bool IsPossibleExportDirectory(const StringPiece export_dir); - -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_SESSION_BUNDLE_SESSION_BUNDLE_H_ diff --git a/tensorflow/contrib/session_bundle/session_bundle.py b/tensorflow/contrib/session_bundle/session_bundle.py deleted file mode 100644 index 0911432823b..00000000000 --- a/tensorflow/contrib/session_bundle/session_bundle.py +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Importer for an exported TensorFlow model. - -This module provides a function to create a SessionBundle containing both the -Session and MetaGraph. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from tensorflow.contrib.session_bundle import constants -from tensorflow.contrib.session_bundle import manifest_pb2 -from tensorflow.core.framework import graph_pb2 -from tensorflow.core.protobuf import meta_graph_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import ops -from tensorflow.python.lib.io import file_io -from tensorflow.python.training import saver as saver_lib -from tensorflow.python.util.deprecation import deprecated - - -@deprecated("2017-06-30", - "No longer supported. Switch to SavedModel immediately.") -def maybe_session_bundle_dir(export_dir): - """Checks if the model path contains session bundle model. - - Args: - export_dir: string path to model checkpoint, for example 'model/00000123' - - Returns: - true if path contains session bundle model files, ie META_GRAPH_DEF_FILENAME - """ - - meta_graph_filename = os.path.join(export_dir, - constants.META_GRAPH_DEF_FILENAME) - return file_io.file_exists(meta_graph_filename) - - -@deprecated("2017-06-30", - "No longer supported. Switch to SavedModel immediately.") -def load_session_bundle_from_path(export_dir, - target="", - config=None, - meta_graph_def=None): - """Load session bundle from the given path. - - The function reads input from the export_dir, constructs the graph data to the - default graph and restores the parameters for the session created. - - Args: - export_dir: the directory that contains files exported by exporter. - target: The execution engine to connect to. See target in - tf.compat.v1.Session() - config: A ConfigProto proto with configuration options. See config in - tf.compat.v1.Session() - meta_graph_def: optional object of type MetaGraphDef. If this object is - present, then it is used instead of parsing MetaGraphDef from export_dir. - - Returns: - session: a tensorflow session created from the variable files. - meta_graph: a meta graph proto saved in the exporter directory. - - Raises: - RuntimeError: if the required files are missing or contain unrecognizable - fields, i.e. the exported model is invalid. - """ - if not meta_graph_def: - meta_graph_filename = os.path.join(export_dir, - constants.META_GRAPH_DEF_FILENAME) - if not file_io.file_exists(meta_graph_filename): - raise RuntimeError("Expected meta graph file missing %s" % - meta_graph_filename) - # Reads meta graph file. - meta_graph_def = meta_graph_pb2.MetaGraphDef() - meta_graph_def.ParseFromString( - file_io.read_file_to_string(meta_graph_filename, binary_mode=True)) - - variables_filename = "" - variables_filename_list = [] - checkpoint_sharded = False - - variables_index_filename = os.path.join(export_dir, - constants.VARIABLES_INDEX_FILENAME_V2) - checkpoint_v2 = file_io.file_exists(variables_index_filename) - - # Find matching checkpoint files. - if checkpoint_v2: - # The checkpoint is in v2 format. - variables_filename_pattern = os.path.join( - export_dir, constants.VARIABLES_FILENAME_PATTERN_V2) - variables_filename_list = file_io.get_matching_files( - variables_filename_pattern) - checkpoint_sharded = True - else: - variables_filename = os.path.join(export_dir, constants.VARIABLES_FILENAME) - if file_io.file_exists(variables_filename): - variables_filename_list = [variables_filename] - else: - variables_filename = os.path.join(export_dir, - constants.VARIABLES_FILENAME_PATTERN) - variables_filename_list = file_io.get_matching_files(variables_filename) - checkpoint_sharded = True - - # Prepare the files to restore a session. - if not variables_filename_list: - restore_files = "" - elif checkpoint_v2 or not checkpoint_sharded: - # For checkpoint v2 or v1 with non-sharded files, use "export" to restore - # the session. - restore_files = constants.VARIABLES_FILENAME - else: - restore_files = constants.VARIABLES_FILENAME_PATTERN - - assets_dir = os.path.join(export_dir, constants.ASSETS_DIRECTORY) - - collection_def = meta_graph_def.collection_def - graph_def = graph_pb2.GraphDef() - if constants.GRAPH_KEY in collection_def: - # Use serving graph_def in MetaGraphDef collection_def if exists - graph_def_any = collection_def[constants.GRAPH_KEY].any_list.value - if len(graph_def_any) != 1: - raise RuntimeError("Expected exactly one serving GraphDef in : %s" % - meta_graph_def) - else: - graph_def_any[0].Unpack(graph_def) - # Replace the graph def in meta graph proto. - meta_graph_def.graph_def.CopyFrom(graph_def) - - ops.reset_default_graph() - sess = session.Session(target, graph=None, config=config) - # Import the graph. - saver = saver_lib.import_meta_graph(meta_graph_def) - # Restore the session. - if restore_files: - saver.restore(sess, os.path.join(export_dir, restore_files)) - - init_op_tensor = None - if constants.INIT_OP_KEY in collection_def: - init_ops = collection_def[constants.INIT_OP_KEY].node_list.value - if len(init_ops) != 1: - raise RuntimeError("Expected exactly one serving init op in : %s" % - meta_graph_def) - init_op_tensor = ops.get_collection(constants.INIT_OP_KEY)[0] - - # Create asset input tensor list. - asset_tensor_dict = {} - if constants.ASSETS_KEY in collection_def: - assets_any = collection_def[constants.ASSETS_KEY].any_list.value - for asset in assets_any: - asset_pb = manifest_pb2.AssetFile() - asset.Unpack(asset_pb) - asset_tensor_dict[asset_pb.tensor_binding.tensor_name] = os.path.join( - assets_dir, asset_pb.filename) - - if init_op_tensor: - # Run the init op. - sess.run(fetches=[init_op_tensor], feed_dict=asset_tensor_dict) - - return sess, meta_graph_def diff --git a/tensorflow/contrib/session_bundle/session_bundle_test.cc b/tensorflow/contrib/session_bundle/session_bundle_test.cc deleted file mode 100644 index 108806e3328..00000000000 --- a/tensorflow/contrib/session_bundle/session_bundle_test.cc +++ /dev/null @@ -1,415 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/session_bundle/session_bundle.h" - -#include -#include -#include - -#include "google/protobuf/any.pb.h" -#include "tensorflow/contrib/session_bundle/signature.h" -#include "tensorflow/contrib/session_bundle/test_util.h" -#include "tensorflow/core/example/example.pb.h" -#include "tensorflow/core/example/feature.pb.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/public/session.h" -#include "tensorflow/core/public/session_options.h" - -namespace tensorflow { -namespace serving { -namespace { - -// Constants for the export path and file-names. -const char kExportPath[] = "session_bundle/testdata/half_plus_two/00000123"; -const char kExportCheckpointV2Path[] = - "session_bundle/testdata/half_plus_two_ckpt_v2/00000123"; -const char kMetaGraphDefFilename[] = "export.meta"; -const char kVariablesFilename[] = "export-00000-of-00001"; - -// Function used to rewrite a MetaGraphDef. -using MetaGraphDefTwiddler = std::function; - -// Copy the base half_plus_two to `export_path`. -// Outputs the files using the passed names (typically the constants above). -// The Twiddler can be used to update the MetaGraphDef before output. -Status CopyExport(const string& export_path, const string& variables_filename, - const string& meta_graph_def_filename, - const MetaGraphDefTwiddler& twiddler) { - TF_RETURN_IF_ERROR(Env::Default()->CreateDir(export_path)); - const string orig_path = test_util::TestSrcDirPath(kExportPath); - { - const string source = io::JoinPath(orig_path, kVariablesFilename); - const string sink = io::JoinPath(export_path, variables_filename); - - string data; - TF_RETURN_IF_ERROR(ReadFileToString(Env::Default(), source, &data)); - TF_RETURN_IF_ERROR(WriteStringToFile(Env::Default(), sink, data)); - } - { - const string source = io::JoinPath(orig_path, kMetaGraphDefFilename); - const string sink = io::JoinPath(export_path, meta_graph_def_filename); - - MetaGraphDef graph_def; - TF_RETURN_IF_ERROR(ReadBinaryProto(Env::Default(), source, &graph_def)); - twiddler(&graph_def); - TF_RETURN_IF_ERROR( - WriteStringToFile(Env::Default(), sink, graph_def.SerializeAsString())); - } - return Status::OK(); -} - -string MakeSerializedExample(float x) { - tensorflow::Example example; - auto* feature_map = example.mutable_features()->mutable_feature(); - (*feature_map)["x"].mutable_float_list()->add_value(x); - return example.SerializeAsString(); -} - -void CheckRegressionSignature(const Signatures& signatures, - const SessionBundle& bundle) { - // Recover the Tensor names of our inputs and outputs. - ASSERT_TRUE(signatures.default_signature().has_regression_signature()); - const RegressionSignature regression_signature = - signatures.default_signature().regression_signature(); - - const string input_name = regression_signature.input().tensor_name(); - const string output_name = regression_signature.output().tensor_name(); - - // Validate the half plus two behavior. - std::vector serialized_examples; - for (float x : {0, 1, 2, 3}) { - serialized_examples.push_back(MakeSerializedExample(x)); - } - Tensor input = test::AsTensor(serialized_examples, TensorShape({4})); - std::vector outputs; - TF_ASSERT_OK( - bundle.session->Run({{input_name, input}}, {output_name}, {}, &outputs)); - ASSERT_EQ(outputs.size(), 1); - test::ExpectTensorEqual( - outputs[0], test::AsTensor({2, 2.5, 3, 3.5}, TensorShape({4, 1}))); -} - -void CheckNamedSignatures(const Signatures& signatures, - const SessionBundle& bundle) { - // Recover the Tensor names of our inputs and outputs. - const string input_name = signatures.named_signatures() - .at("inputs") - .generic_signature() - .map() - .at("x") - .tensor_name(); - const string output_name = signatures.named_signatures() - .at("outputs") - .generic_signature() - .map() - .at("y") - .tensor_name(); - - // Validate the half plus two behavior. - Tensor input = test::AsTensor({0, 1, 2, 3}, TensorShape({4, 1})); - std::vector outputs; - TF_ASSERT_OK( - bundle.session->Run({{input_name, input}}, {output_name}, {}, &outputs)); - ASSERT_EQ(outputs.size(), 1); - test::ExpectTensorEqual( - outputs[0], test::AsTensor({2, 2.5, 3, 3.5}, TensorShape({4, 1}))); -} - -void CheckSessionBundle(const string& export_path, - const SessionBundle& bundle) { - const string asset_path = io::JoinPath(export_path, kAssetsDirectory); - // Validate the assets behavior. - std::vector path_outputs; - TF_ASSERT_OK(bundle.session->Run({}, {"filename1:0", "filename2:0"}, {}, - &path_outputs)); - ASSERT_EQ(2, path_outputs.size()); - // Validate the two asset file tensors are set by the init_op and include the - // base_path and asset directory. - test::ExpectTensorEqual( - test::AsTensor({io::JoinPath(asset_path, "hello1.txt")}, - TensorShape({})), - path_outputs[0]); - test::ExpectTensorEqual( - test::AsTensor({io::JoinPath(asset_path, "hello2.txt")}, - TensorShape({})), - path_outputs[1]); - - Signatures signatures; - TF_ASSERT_OK(GetSignatures(bundle.meta_graph_def, &signatures)); - CheckRegressionSignature(signatures, bundle); - CheckNamedSignatures(signatures, bundle); -} - -void BasicTest(const string& export_path) { - SessionOptions options; - SessionBundle bundle; - TF_ASSERT_OK(LoadSessionBundleFromPath(options, export_path, &bundle)); - CheckSessionBundle(export_path, bundle); -} - -// Test for resource leaks when loading and unloading large numbers of -// SessionBundles. Concurrent with adding this test, we had a leak where the -// TensorFlow Session was not being closed, which leaked memory. -// TODO(b/31711147): Increase the SessionBundle ResourceLeakTest iterations and -// move outside of the test suite; decrease test size back to small at the same -// time. -TEST(LoadSessionBundleFromPath, ResourceLeakTest) { - const string export_path = test_util::TestSrcDirPath(kExportPath); - for (int i = 0; i < 100; i++) { - BasicTest(export_path); - } -} - -TEST(LoadSessionBundleFromPath, BasicTensorFlowContrib) { - const string export_path = test_util::TestSrcDirPath(kExportPath); - BasicTest(export_path); -} - -TEST(LoadSessionBundleFromPath, BasicTestRunOptions) { - const string export_path = test_util::TestSrcDirPath(kExportPath); - - // Use default session-options. - SessionOptions session_options; - - // Setup run-options with full-traces. - RunOptions run_options; - run_options.set_trace_level(RunOptions::FULL_TRACE); - - SessionBundle bundle; - TF_ASSERT_OK(LoadSessionBundleFromPathUsingRunOptions( - session_options, run_options, export_path, &bundle)); - CheckSessionBundle(export_path, bundle); -} - -TEST(LoadSessionBundleFromPath, BasicTestRunOptionsThreadPool) { - const string export_path = test_util::TestSrcDirPath(kExportPath); - const int32 threadpool_index = 1; - - // Setup session-options with separate thread-pools. - SessionOptions session_options; - session_options.config.add_session_inter_op_thread_pool(); - session_options.config.add_session_inter_op_thread_pool()->set_num_threads(2); - - // Setup run-options with the threadpool index to use. - RunOptions run_options; - run_options.set_inter_op_thread_pool(threadpool_index); - - SessionBundle bundle; - TF_ASSERT_OK(LoadSessionBundleFromPathUsingRunOptions( - session_options, run_options, export_path, &bundle)); - CheckSessionBundle(export_path, bundle); -} - -TEST(LoadSessionBundleFromPath, BasicTestRunOptionsThreadPoolInvalid) { - const string export_path = test_util::TestSrcDirPath(kExportPath); - const int32 invalid_threadpool_index = 2; - - // Setup session-options with separate thread-pools. - SessionOptions session_options; - session_options.config.add_session_inter_op_thread_pool(); - session_options.config.add_session_inter_op_thread_pool()->set_num_threads(2); - - // Setup run-options with an invalid threadpool index. - RunOptions run_options; - run_options.set_inter_op_thread_pool(invalid_threadpool_index); - - SessionBundle bundle; - Status status = LoadSessionBundleFromPathUsingRunOptions( - session_options, run_options, export_path, &bundle); - - // Expect failed session run calls with invalid run-options. - EXPECT_FALSE(status.ok()); - EXPECT_TRUE(absl::StrContains(status.error_message(), - "Invalid inter_op_thread_pool: 2")) - << status.error_message(); -} - -TEST(LoadSessionBundleFromPath, BadExportPath) { - const string export_path = test_util::TestSrcDirPath("/tmp/bigfoot"); - SessionOptions options; - options.target = "local"; - SessionBundle bundle; - const auto status = LoadSessionBundleFromPath(options, export_path, &bundle); - ASSERT_FALSE(status.ok()); - const string msg = status.ToString(); - EXPECT_TRUE(msg.find("Not found") != std::string::npos) << msg; -} - -TEST(CheckpointV2Test, LoadSessionBundleFromPath) { - const string export_path = test_util::TestSrcDirPath(kExportCheckpointV2Path); - BasicTest(export_path); -} - -TEST(CheckpointV2Test, IsPossibleExportDirectory) { - const string export_path = test_util::TestSrcDirPath(kExportCheckpointV2Path); - EXPECT_TRUE(IsPossibleExportDirectory(export_path)); -} - -class SessionBundleTest : public ::testing::Test { - protected: - // Copy the half_plus_two graph and apply the twiddler to rewrite the - // MetaGraphDef. - // Returns the path of the export. - // ** Should only be called once per test ** - string SetupExport(const MetaGraphDefTwiddler& twiddler) { - return SetupExport(twiddler, kVariablesFilename, kMetaGraphDefFilename); - } - // SetupExport that allows for the variables and meta_graph_def filenames - // to be overridden. - string SetupExport(const MetaGraphDefTwiddler& twiddler, - const string& variables_filename, - const string& meta_graph_def_filename) { - // Construct a unique path name based on the test name. - const ::testing::TestInfo* const test_info = - ::testing::UnitTest::GetInstance()->current_test_info(); - const string export_path = io::JoinPath( - testing::TmpDir(), - strings::StrCat(test_info->test_case_name(), test_info->name())); - TF_CHECK_OK(CopyExport(export_path, variables_filename, - meta_graph_def_filename, twiddler)); - return export_path; - } - - SessionOptions options_; - SessionBundle bundle_; - Status status_; -}; - -TEST_F(SessionBundleTest, Basic) { - const string export_path = SetupExport([](MetaGraphDef*) {}); - BasicTest(export_path); -} - -TEST_F(SessionBundleTest, UnshardedVariableFile) { - // Test that we can properly read the variables when exported - // without sharding. - const string export_path = - SetupExport([](MetaGraphDef*) {}, "export", kMetaGraphDefFilename); - BasicTest(export_path); -} - -TEST_F(SessionBundleTest, ServingGraphEmpty) { - const string path = SetupExport([](MetaGraphDef* def) { - (*def->mutable_collection_def())[kGraphKey].clear_any_list(); - }); - status_ = LoadSessionBundleFromPath(options_, path, &bundle_); - EXPECT_FALSE(status_.ok()); - EXPECT_TRUE(absl::StrContains(status_.error_message(), - "Expected exactly one serving GraphDef")) - << status_.error_message(); -} - -TEST_F(SessionBundleTest, ServingGraphAnyIncorrectType) { - const string path = SetupExport([](MetaGraphDef* def) { - // Pack an unexpected type in the GraphDef Any. - (*def->mutable_collection_def())[kGraphKey].clear_any_list(); - auto* any = (*def->mutable_collection_def())[kGraphKey] - .mutable_any_list() - ->add_value(); - any->PackFrom(AssetFile()); - }); - status_ = LoadSessionBundleFromPath(options_, path, &bundle_); - EXPECT_FALSE(status_.ok()); - EXPECT_TRUE( - absl::StrContains(status_.error_message(), - "Expected Any type_url for: tensorflow.GraphDef")) - << status_.error_message(); -} - -TEST_F(SessionBundleTest, ServingGraphAnyValueCorrupted) { - const string path = SetupExport([](MetaGraphDef* def) { - // Pack an unexpected type in the GraphDef Any. - (*def->mutable_collection_def())[kGraphKey].clear_any_list(); - auto* any = (*def->mutable_collection_def())[kGraphKey] - .mutable_any_list() - ->add_value(); - any->PackFrom(GraphDef()); - any->set_value("junk junk"); - }); - status_ = LoadSessionBundleFromPath(options_, path, &bundle_); - EXPECT_FALSE(status_.ok()); - EXPECT_TRUE(absl::StrContains(status_.error_message(), "Failed to unpack")) - << status_.error_message(); -} - -TEST_F(SessionBundleTest, AssetFileAnyIncorrectType) { - const string path = SetupExport([](MetaGraphDef* def) { - // Pack an unexpected type in the AssetFile Any. - (*def->mutable_collection_def())[kAssetsKey].clear_any_list(); - auto* any = (*def->mutable_collection_def())[kAssetsKey] - .mutable_any_list() - ->add_value(); - any->PackFrom(GraphDef()); - }); - status_ = LoadSessionBundleFromPath(options_, path, &bundle_); - EXPECT_FALSE(status_.ok()); - EXPECT_TRUE(absl::StrContains( - status_.error_message(), - "Expected Any type_url for: tensorflow.serving.AssetFile")) - << status_.error_message(); -} - -TEST_F(SessionBundleTest, AssetFileAnyValueCorrupted) { - const string path = SetupExport([](MetaGraphDef* def) { - // Pack an unexpected type in the AssetFile Any. - (*def->mutable_collection_def())[kAssetsKey].clear_any_list(); - auto* any = (*def->mutable_collection_def())[kAssetsKey] - .mutable_any_list() - ->add_value(); - any->PackFrom(AssetFile()); - any->set_value("junk junk"); - }); - status_ = LoadSessionBundleFromPath(options_, path, &bundle_); - EXPECT_FALSE(status_.ok()); - EXPECT_TRUE(absl::StrContains(status_.error_message(), "Failed to unpack")) - << status_.error_message(); -} - -TEST_F(SessionBundleTest, InitOpTooManyValues) { - const string path = SetupExport([](MetaGraphDef* def) { - // Pack multiple init ops in to the collection. - (*def->mutable_collection_def())[kInitOpKey].clear_node_list(); - auto* node_list = - (*def->mutable_collection_def())[kInitOpKey].mutable_node_list(); - node_list->add_value("foo"); - node_list->add_value("bar"); - }); - status_ = LoadSessionBundleFromPath(options_, path, &bundle_); - EXPECT_FALSE(status_.ok()); - EXPECT_TRUE(absl::StrContains(status_.error_message(), - "Expected exactly one serving init op")) - << status_.error_message(); -} - -TEST_F(SessionBundleTest, PossibleExportDirectory) { - const string export_path = SetupExport([](MetaGraphDef*) {}); - EXPECT_TRUE(IsPossibleExportDirectory(export_path)); - - EXPECT_FALSE( - IsPossibleExportDirectory(io::JoinPath(export_path, kAssetsDirectory))); -} - -} // namespace -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow/contrib/session_bundle/session_bundle_test.py b/tensorflow/contrib/session_bundle/session_bundle_test.py deleted file mode 100644 index 3c06ec048d6..00000000000 --- a/tensorflow/contrib/session_bundle/session_bundle_test.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for session_bundle.py.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os.path -import shutil - -import numpy as np - -from tensorflow.contrib.session_bundle import constants -from tensorflow.contrib.session_bundle import manifest_pb2 -from tensorflow.contrib.session_bundle import session_bundle -from tensorflow.core.example.example_pb2 import Example -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import graph_util -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables -import tensorflow.python.ops.parsing_ops # pylint: disable=unused-import -from tensorflow.python.platform import test -from tensorflow.python.training import saver -from tensorflow.python.util import compat - -SAVED_MODEL_PATH = ( - "python/saved_model/example/saved_model_half_plus_two/00000123") -SESSION_BUNDLE_PATH = "contrib/session_bundle/testdata/half_plus_two/00000123" - - -def _make_serialized_example(x): - example = Example() - example.features.feature["x"].float_list.value.append(x) - return example.SerializeToString() - - -class SessionBundleLoadTest(test.TestCase): - - def _checkRegressionSignature(self, signatures, sess): - default_signature = signatures.default_signature - input_name = default_signature.regression_signature.input.tensor_name - output_name = default_signature.regression_signature.output.tensor_name - tf_example = [_make_serialized_example(x) for x in [0, 1, 2, 3]] - y = sess.run([output_name], {input_name: tf_example}) - # The operation is y = 0.5 * x + 2 - self.assertEqual(y[0][0], 2) - self.assertEqual(y[0][1], 2.5) - self.assertEqual(y[0][2], 3) - self.assertEqual(y[0][3], 3.5) - - def _checkNamedSignatures(self, signatures, sess): - named_signatures = signatures.named_signatures - input_name = (named_signatures["inputs"].generic_signature.map["x"] - .tensor_name) - output_name = (named_signatures["outputs"].generic_signature.map["y"] - .tensor_name) - y = sess.run([output_name], {input_name: np.array([[0], [1], [2], [3]])}) - # The operation is y = 0.5 * x + 2 - self.assertEqual(y[0][0], 2) - self.assertEqual(y[0][1], 2.5) - self.assertEqual(y[0][2], 3) - self.assertEqual(y[0][3], 3.5) - - def testMaybeSessionBundleDir(self): - base_path = test.test_src_dir_path(SESSION_BUNDLE_PATH) - self.assertTrue(session_bundle.maybe_session_bundle_dir(base_path)) - base_path = test.test_src_dir_path(SAVED_MODEL_PATH) - self.assertFalse(session_bundle.maybe_session_bundle_dir(base_path)) - base_path = "complete_garbage" - self.assertFalse(session_bundle.maybe_session_bundle_dir(base_path)) - - def testBasic(self): - base_path = test.test_src_dir_path(SESSION_BUNDLE_PATH) - ops.reset_default_graph() - sess, meta_graph_def = session_bundle.load_session_bundle_from_path( - base_path, - target="", - config=config_pb2.ConfigProto(device_count={"CPU": 2})) - - self.assertTrue(sess) - asset_path = os.path.join(base_path, constants.ASSETS_DIRECTORY) - with sess.as_default(): - path1, path2 = sess.run(["filename1:0", "filename2:0"]) - self.assertEqual( - compat.as_bytes(os.path.join(asset_path, "hello1.txt")), path1) - self.assertEqual( - compat.as_bytes(os.path.join(asset_path, "hello2.txt")), path2) - - collection_def = meta_graph_def.collection_def - - signatures_any = collection_def[constants.SIGNATURES_KEY].any_list.value - self.assertEquals(len(signatures_any), 1) - - signatures = manifest_pb2.Signatures() - signatures_any[0].Unpack(signatures) - self._checkRegressionSignature(signatures, sess) - self._checkNamedSignatures(signatures, sess) - - def testBadPath(self): - base_path = test.test_src_dir_path("/no/such/a/dir") - ops.reset_default_graph() - with self.assertRaises(RuntimeError) as cm: - _, _ = session_bundle.load_session_bundle_from_path( - base_path, - target="local", - config=config_pb2.ConfigProto(device_count={"CPU": 2})) - self.assertTrue("Expected meta graph file missing" in str(cm.exception)) - - def testVarCheckpointV2(self): - base_path = test.test_src_dir_path( - "contrib/session_bundle/testdata/half_plus_two_ckpt_v2/00000123") - ops.reset_default_graph() - sess, meta_graph_def = session_bundle.load_session_bundle_from_path( - base_path, - target="", - config=config_pb2.ConfigProto(device_count={"CPU": 2})) - - self.assertTrue(sess) - asset_path = os.path.join(base_path, constants.ASSETS_DIRECTORY) - with sess.as_default(): - path1, path2 = sess.run(["filename1:0", "filename2:0"]) - self.assertEqual( - compat.as_bytes(os.path.join(asset_path, "hello1.txt")), path1) - self.assertEqual( - compat.as_bytes(os.path.join(asset_path, "hello2.txt")), path2) - - collection_def = meta_graph_def.collection_def - - signatures_any = collection_def[constants.SIGNATURES_KEY].any_list.value - self.assertEquals(len(signatures_any), 1) - - signatures = manifest_pb2.Signatures() - signatures_any[0].Unpack(signatures) - self._checkRegressionSignature(signatures, sess) - self._checkNamedSignatures(signatures, sess) - - -class SessionBundleLoadNoVarsTest(test.TestCase): - """Test the case where there are no variables in the graph.""" - - def setUp(self): - self.base_path = os.path.join(test.get_temp_dir(), "no_vars") - if not os.path.exists(self.base_path): - os.mkdir(self.base_path) - - # Create a simple graph with a variable, then convert variables to - # constants and export the graph. - with ops.Graph().as_default() as g: - x = array_ops.placeholder(dtypes.float32, name="x") - w = variables.Variable(3.0) - y = math_ops.subtract(w * x, 7.0, name="y") # pylint: disable=unused-variable - ops.add_to_collection("meta", "this is meta") - - with self.session(graph=g) as session: - variables.global_variables_initializer().run() - new_graph_def = graph_util.convert_variables_to_constants( - session, g.as_graph_def(), ["y"]) - - filename = os.path.join(self.base_path, constants.META_GRAPH_DEF_FILENAME) - saver.export_meta_graph( - filename, graph_def=new_graph_def, collection_list=["meta"]) - - def tearDown(self): - shutil.rmtree(self.base_path) - - def testGraphWithoutVarsLoadsCorrectly(self): - session, _ = session_bundle.load_session_bundle_from_path(self.base_path) - got = session.run(["y:0"], {"x:0": 5.0})[0] - self.assertEquals(got, 5.0 * 3.0 - 7.0) - self.assertEquals(ops.get_collection("meta"), [b"this is meta"]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/session_bundle/signature.cc b/tensorflow/contrib/session_bundle/signature.cc deleted file mode 100644 index ed70a5b91b2..00000000000 --- a/tensorflow/contrib/session_bundle/signature.cc +++ /dev/null @@ -1,277 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/session_bundle/signature.h" - -#include -#include -#include - -#include "google/protobuf/any.pb.h" -#include "tensorflow/contrib/session_bundle/manifest.pb.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/protobuf_internal.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/protobuf/meta_graph.pb.h" -#include "tensorflow/core/public/session.h" - -namespace tensorflow { -namespace serving { -namespace { - -// Returns OK if the input and output batch sizes match. -Status BatchSizesMatch(const Tensor& input, const Tensor& output) { - // Ensure the number of outputs match the number of inputs. - if (input.dim_size(0) != output.dim_size(0)) { - return errors::Internal(strings::StrCat( - "Input batch size did not match output batch size: ", input.dim_size(0), - " vs. ", output.dim_size(0))); - } - return Status::OK(); -} -} // namespace - -Status GetSignatures(const tensorflow::MetaGraphDef& meta_graph_def, - Signatures* signatures) { - const auto& collection_def = meta_graph_def.collection_def(); - const auto it = collection_def.find(kSignaturesKey); - if (it == collection_def.end() || it->second.any_list().value_size() != 1) { - return errors::FailedPrecondition( - strings::StrCat("Expected exactly one signatures proto in : ", - DebugStringIfAvailable(meta_graph_def))); - } - const auto& any = it->second.any_list().value(0); - return ParseAny(any, signatures, "tensorflow.serving.Signatures"); -} - -Status SetSignatures(const Signatures& signatures, - tensorflow::MetaGraphDef* meta_graph_def) { - auto& collection_def = *(meta_graph_def->mutable_collection_def()); - auto* any_list = collection_def[kSignaturesKey].mutable_any_list(); - any_list->mutable_value()->Clear(); -#ifdef TENSORFLOW_LITE_PROTOS - signatures.SerializeToString( - any_list->mutable_value()->Add()->mutable_value()); -#else - any_list->mutable_value()->Add()->PackFrom(signatures); -#endif - return Status::OK(); -} - -Status GetClassificationSignature( - const tensorflow::MetaGraphDef& meta_graph_def, - ClassificationSignature* signature) { - Signatures signatures; - TF_RETURN_IF_ERROR(GetSignatures(meta_graph_def, &signatures)); - if (!signatures.has_default_signature()) { - return errors::FailedPrecondition( - strings::StrCat("Expected a default signature in: ", - DebugStringIfAvailable(signatures))); - } - if (!signatures.default_signature().has_classification_signature()) { - return errors::FailedPrecondition(strings::StrCat( - "Expected a classification signature in: ", - DebugStringIfAvailable(signatures.default_signature()))); - } - *signature = signatures.default_signature().classification_signature(); - return Status::OK(); -} - -Status GetNamedClassificationSignature( - const string& name, const tensorflow::MetaGraphDef& meta_graph_def, - ClassificationSignature* signature) { - Signatures signatures; - TF_RETURN_IF_ERROR(GetSignatures(meta_graph_def, &signatures)); - const auto& it = signatures.named_signatures().find(name); - if (it == signatures.named_signatures().end()) { - return errors::NotFound( - strings::StrCat("Missing signature named \"", name, - "\" in: ", DebugStringIfAvailable(signatures))); - } - if (!it->second.has_classification_signature()) { - return errors::FailedPrecondition( - strings::StrCat("Expected a classification signature for name \"", name, - "\" in: ", DebugStringIfAvailable(it->second))); - } - *signature = it->second.classification_signature(); - return Status::OK(); -} - -Status RunClassification(const ClassificationSignature& signature, - const Tensor& input, Session* session, Tensor* classes, - Tensor* scores) { - std::vector output_tensor_names; - if (classes) { - output_tensor_names.push_back(signature.classes().tensor_name()); - } - if (scores) { - output_tensor_names.push_back(signature.scores().tensor_name()); - } - // Run the graph with our inputs and outputs. - std::vector outputs; - const Status run_status = - session->Run({{signature.input().tensor_name(), input}}, - output_tensor_names, {}, &outputs); - if (!run_status.ok()) { - return run_status; - } - // Ensure the output is shaped how we expect. - // There should be one string Tensor of shape, - // [batch_size, num_recommendations]. - if (outputs.size() != output_tensor_names.size()) { - return errors::Internal( - strings::StrCat("Expected ", output_tensor_names.size(), - " output tensor(s). Got: ", outputs.size())); - } - if (classes) { - *classes = outputs[0]; - TF_RETURN_IF_ERROR(BatchSizesMatch(input, *classes)); - } - if (scores) { - *scores = outputs[classes ? 1 : 0]; - TF_RETURN_IF_ERROR(BatchSizesMatch(input, *scores)); - } - return Status::OK(); -} - -Status GetRegressionSignature(const tensorflow::MetaGraphDef& meta_graph_def, - RegressionSignature* signature) { - Signatures signatures; - TF_RETURN_IF_ERROR(GetSignatures(meta_graph_def, &signatures)); - if (!signatures.has_default_signature()) { - return errors::FailedPrecondition( - strings::StrCat("Expected a default signature in: ", - DebugStringIfAvailable(signatures))); - } - if (!signatures.default_signature().has_regression_signature()) { - return errors::FailedPrecondition(strings::StrCat( - "Expected a regression signature in: ", - DebugStringIfAvailable(signatures.default_signature()))); - } - *signature = signatures.default_signature().regression_signature(); - return Status::OK(); -} - -Status RunRegression(const RegressionSignature& signature, - const Tensor& regression_input, Session* session, - Tensor* regression_output) { - std::vector output_tensor_names; - if (regression_output) { - output_tensor_names.push_back(signature.output().tensor_name()); - } - // Run the graph with our inputs and outputs. - std::vector outputs; - const Status run_status = - session->Run({{signature.input().tensor_name(), regression_input}}, - output_tensor_names, {}, &outputs); - if (!run_status.ok()) { - return run_status; - } - // Ensure the regression score output is shaped how we expect. - // There should be one float Tensor of shape, - // [batch_size, num_recommendations]. - if (outputs.size() != output_tensor_names.size()) { - return errors::Internal( - strings::StrCat("Expected ", output_tensor_names.size(), - " output tensor(s). Got: ", outputs.size())); - } - if (regression_output) { - *regression_output = outputs[0]; - TF_RETURN_IF_ERROR(BatchSizesMatch(regression_input, *regression_output)); - } - return Status::OK(); -} - -Status GetGenericSignature(const string& name, - const tensorflow::MetaGraphDef& meta_graph_def, - GenericSignature* signature) { - Signatures signatures; - TF_RETURN_IF_ERROR(GetSignatures(meta_graph_def, &signatures)); - const auto& it = signatures.named_signatures().find(name); - if (it == signatures.named_signatures().end()) { - return errors::InvalidArgument( - strings::StrCat("Missing generic signature named \"", name, "\" in ", - DebugStringIfAvailable(signatures))); - } - if (!it->second.has_generic_signature()) { - return errors::InvalidArgument(strings::StrCat( - "Expected a generic signature: ", DebugStringIfAvailable(it->second))); - } - *signature = it->second.generic_signature(); - return Status::OK(); -} - -Status GetDefaultSignature(const tensorflow::MetaGraphDef& meta_graph_def, - Signature* default_signature) { - Signatures signatures; - TF_RETURN_IF_ERROR(GetSignatures(meta_graph_def, &signatures)); - *default_signature = signatures.default_signature(); - return Status::OK(); -} - -Status GetNamedSignature(const string& name, - const tensorflow::MetaGraphDef& meta_graph_def, - Signature* signature) { - Signatures signatures; - TF_RETURN_IF_ERROR(GetSignatures(meta_graph_def, &signatures)); - const auto& it = signatures.named_signatures().find(name); - if (it == signatures.named_signatures().end()) { - return errors::NotFound( - strings::StrCat("Missing signature named \"", name, - "\" in: ", DebugStringIfAvailable(signatures))); - } - *signature = it->second; - return Status::OK(); -} - -Status BindGenericInputs(const GenericSignature& signature, - const std::vector>& inputs, - std::vector>* bound_inputs) { - const protobuf::Map& bindings = - signature.map(); - - for (const auto& entry : inputs) { - const auto mapped = bindings.find(entry.first); - if (mapped == bindings.end()) { - return errors::NotFound( - strings::StrCat("Could not find generic binding for: ", entry.first)); - } - bound_inputs->push_back({mapped->second.tensor_name(), entry.second}); - } - return Status::OK(); -} - -Status BindGenericNames(const GenericSignature& signature, - const std::vector& input_names, - std::vector* bound_names) { - const protobuf::Map& bindings = - signature.map(); - - for (const string& entry : input_names) { - const auto mapped = bindings.find(entry); - if (mapped == bindings.end()) { - return errors::NotFound( - strings::StrCat("Could not find generic binding for: ", entry)); - } - bound_names->push_back(mapped->second.tensor_name()); - } - return Status::OK(); -} - -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow/contrib/session_bundle/signature.h b/tensorflow/contrib/session_bundle/signature.h deleted file mode 100644 index 4ef1277cec4..00000000000 --- a/tensorflow/contrib/session_bundle/signature.h +++ /dev/null @@ -1,124 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Helpers for working with TensorFlow exports and their signatures. - -#ifndef TENSORFLOW_CONTRIB_SESSION_BUNDLE_SIGNATURE_H_ -#define TENSORFLOW_CONTRIB_SESSION_BUNDLE_SIGNATURE_H_ - -#include -#include -#include - -#include "tensorflow/contrib/session_bundle/manifest.pb.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/protobuf/meta_graph.pb.h" -#include "tensorflow/core/protobuf/saver.pb.h" -#include "tensorflow/core/public/session.h" - -namespace tensorflow { -namespace serving { - -const char kSignaturesKey[] = "serving_signatures"; - -// Get Signatures from a MetaGraphDef. -Status GetSignatures(const tensorflow::MetaGraphDef& meta_graph_def, - Signatures* signatures); - -// (Re)set Signatures in a MetaGraphDef. -Status SetSignatures(const Signatures& signatures, - tensorflow::MetaGraphDef* meta_graph_def); - -// Gets a ClassificationSignature from a MetaGraphDef's default signature. -// Returns an error if the default signature is not a ClassificationSignature, -// or does not exist. -Status GetClassificationSignature( - const tensorflow::MetaGraphDef& meta_graph_def, - ClassificationSignature* signature); - -// Gets a named ClassificationSignature from a MetaGraphDef. -// Returns an error if a ClassificationSignature with the given name does -// not exist. -Status GetNamedClassificationSignature( - const string& name, const tensorflow::MetaGraphDef& meta_graph_def, - ClassificationSignature* signature); - -// Gets a RegressionSignature from a MetaGraphDef's default signature. -// Returns an error if the default signature is not a RegressionSignature, -// or does not exist. -Status GetRegressionSignature(const tensorflow::MetaGraphDef& meta_graph_def, - RegressionSignature* signature); - -// Runs a classification using the provided signature and initialized Session. -// input: input batch of items to classify -// classes: output batch of classes; may be null if not needed -// scores: output batch of scores; may be null if not needed -// Validates sizes of the inputs and outputs are consistent (e.g., input -// batch size equals output batch sizes). -// Does not do any type validation. -Status RunClassification(const ClassificationSignature& signature, - const Tensor& input, Session* session, Tensor* classes, - Tensor* scores); - -// Runs regression using the provided signature and initialized Session. -// input: input batch of items to run the regression model against -// output: output targets -// Validates sizes of the inputs and outputs are consistent (e.g., input -// batch size equals output batch sizes). -// Does not do any type validation. -Status RunRegression(const RegressionSignature& signature, const Tensor& input, - Session* session, Tensor* output); - -// Gets the named GenericSignature from a MetaGraphDef. -// Returns an error if a GenericSignature with the given name does not exist. -Status GetGenericSignature(const string& name, - const tensorflow::MetaGraphDef& meta_graph_def, - GenericSignature* signature); - -// Gets the default signature from a MetaGraphDef. -Status GetDefaultSignature(const tensorflow::MetaGraphDef& meta_graph_def, - Signature* default_signature); - -// Gets a named Signature from a MetaGraphDef. -// Returns an error if a Signature with the given name does not exist. -Status GetNamedSignature(const string& name, - const tensorflow::MetaGraphDef& meta_graph_def, - Signature* default_signature); - -// Binds TensorFlow inputs specified by the caller using the logical names -// specified at Graph export time, to the actual Graph names. -// Returns an error if any of the inputs do not have a binding in the export's -// MetaGraphDef. -Status BindGenericInputs(const GenericSignature& signature, - const std::vector>& inputs, - std::vector>* bound_inputs); - -// Binds the input names specified by the caller using the logical names -// specified at Graph export time, to the actual Graph names. This is useful -// for binding names of both the TensorFlow output tensors and target nodes, -// with the latter (target nodes) being optional and rarely used (if ever) at -// serving time. -// Returns an error if any of the input names do not have a binding in the -// export's MetaGraphDef. -Status BindGenericNames(const GenericSignature& signature, - const std::vector& input_names, - std::vector* bound_names); - -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_SESSION_BUNDLE_SIGNATURE_H_ diff --git a/tensorflow/contrib/session_bundle/signature_test.cc b/tensorflow/contrib/session_bundle/signature_test.cc deleted file mode 100644 index 99b55e3c3be..00000000000 --- a/tensorflow/contrib/session_bundle/signature_test.cc +++ /dev/null @@ -1,696 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/session_bundle/signature.h" - -#include - -#include "google/protobuf/any.pb.h" -#include "tensorflow/contrib/session_bundle/manifest.pb.h" -#include "tensorflow/core/framework/graph.pb.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/public/session.h" - -namespace tensorflow { -namespace serving { -namespace { - -static bool HasSubstr(StringPiece base, StringPiece substr) { - bool ok = absl::StrContains(base, substr); - EXPECT_TRUE(ok) << base << ", expected substring " << substr; - return ok; -} - -TEST(GetClassificationSignature, Basic) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - ClassificationSignature* input_signature = - signatures.mutable_default_signature() - ->mutable_classification_signature(); - input_signature->mutable_input()->set_tensor_name("flow"); - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - ClassificationSignature signature; - const Status status = GetClassificationSignature(meta_graph_def, &signature); - TF_ASSERT_OK(status); - EXPECT_EQ(signature.input().tensor_name(), "flow"); -} - -TEST(GetClassificationSignature, MissingSignature) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - signatures.mutable_default_signature(); - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - ClassificationSignature signature; - const Status status = GetClassificationSignature(meta_graph_def, &signature); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(absl::StrContains(status.error_message(), - "Expected a classification signature")) - << status.error_message(); -} - -TEST(GetClassificationSignature, WrongSignatureType) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - signatures.mutable_default_signature()->mutable_regression_signature(); - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - ClassificationSignature signature; - const Status status = GetClassificationSignature(meta_graph_def, &signature); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(absl::StrContains(status.error_message(), - "Expected a classification signature")) - << status.error_message(); -} - -TEST(GetNamedClassificationSignature, Basic) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - ClassificationSignature* input_signature = - (*signatures.mutable_named_signatures())["foo"] - .mutable_classification_signature(); - input_signature->mutable_input()->set_tensor_name("flow"); - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - ClassificationSignature signature; - const Status status = - GetNamedClassificationSignature("foo", meta_graph_def, &signature); - TF_ASSERT_OK(status); - EXPECT_EQ(signature.input().tensor_name(), "flow"); -} - -TEST(GetNamedClassificationSignature, MissingSignature) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - ClassificationSignature signature; - const Status status = - GetNamedClassificationSignature("foo", meta_graph_def, &signature); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(absl::StrContains(status.error_message(), - "Missing signature named \"foo\"")) - << status.error_message(); -} - -TEST(GetNamedClassificationSignature, WrongSignatureType) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - (*signatures.mutable_named_signatures())["foo"] - .mutable_regression_signature(); - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - ClassificationSignature signature; - const Status status = - GetNamedClassificationSignature("foo", meta_graph_def, &signature); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE( - absl::StrContains(status.error_message(), - "Expected a classification signature for name \"foo\"")) - << status.error_message(); -} - -TEST(GetRegressionSignature, Basic) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - RegressionSignature* input_signature = - signatures.mutable_default_signature()->mutable_regression_signature(); - input_signature->mutable_input()->set_tensor_name("flow"); - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - RegressionSignature signature; - const Status status = GetRegressionSignature(meta_graph_def, &signature); - TF_ASSERT_OK(status); - EXPECT_EQ(signature.input().tensor_name(), "flow"); -} - -TEST(GetRegressionSignature, MissingSignature) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - signatures.mutable_default_signature(); - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - RegressionSignature signature; - const Status status = GetRegressionSignature(meta_graph_def, &signature); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(absl::StrContains(status.error_message(), - "Expected a regression signature")) - << status.error_message(); -} - -TEST(GetRegressionSignature, WrongSignatureType) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - signatures.mutable_default_signature()->mutable_classification_signature(); - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - RegressionSignature signature; - const Status status = GetRegressionSignature(meta_graph_def, &signature); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(absl::StrContains(status.error_message(), - "Expected a regression signature")) - << status.error_message(); -} - -TEST(GetNamedSignature, Basic) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - ClassificationSignature* input_signature = - (*signatures.mutable_named_signatures())["foo"] - .mutable_classification_signature(); - input_signature->mutable_input()->set_tensor_name("flow"); - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - Signature signature; - const Status status = GetNamedSignature("foo", meta_graph_def, &signature); - TF_ASSERT_OK(status); - EXPECT_EQ(signature.classification_signature().input().tensor_name(), "flow"); -} - -TEST(GetNamedSignature, MissingSignature) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - Signature signature; - const Status status = GetNamedSignature("foo", meta_graph_def, &signature); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(absl::StrContains(status.error_message(), - "Missing signature named \"foo\"")) - << status.error_message(); -} - -// MockSession used to test input and output interactions with a -// tensorflow::Session. -struct MockSession : public tensorflow::Session { - ~MockSession() override = default; - - Status Create(const GraphDef& graph) override { - return errors::Unimplemented("Not implemented for mock."); - } - - Status Extend(const GraphDef& graph) override { - return errors::Unimplemented("Not implemented for mock."); - } - - // Sets the input and output arguments. - Status Run(const std::vector>& inputs_arg, - const std::vector& output_tensor_names_arg, - const std::vector& target_node_names_arg, - std::vector* outputs_arg) override { - inputs = inputs_arg; - output_tensor_names = output_tensor_names_arg; - target_node_names = target_node_names_arg; - *outputs_arg = outputs; - return status; - } - - Status Close() override { - return errors::Unimplemented("Not implemented for mock."); - } - - Status ListDevices(std::vector* response) override { - return errors::Unimplemented("Not implemented for mock."); - } - - // Arguments stored on a Run call. - std::vector> inputs; - std::vector output_tensor_names; - std::vector target_node_names; - - // Output argument set by Run; should be set before calling. - std::vector outputs; - - // Return value for Run; should be set before calling. - Status status; -}; - -constexpr char kInputName[] = "in:0"; -constexpr char kClassesName[] = "classes:0"; -constexpr char kScoresName[] = "scores:0"; - -class RunClassificationTest : public ::testing::Test { - public: - void SetUp() override { - signature_.mutable_input()->set_tensor_name(kInputName); - signature_.mutable_classes()->set_tensor_name(kClassesName); - signature_.mutable_scores()->set_tensor_name(kScoresName); - } - - protected: - ClassificationSignature signature_; - Tensor input_tensor_; - Tensor classes_tensor_; - Tensor scores_tensor_; - MockSession session_; -}; - -TEST_F(RunClassificationTest, Basic) { - input_tensor_ = test::AsTensor({99}); - session_.outputs = {test::AsTensor({3}), test::AsTensor({2})}; - const Status status = RunClassification(signature_, input_tensor_, &session_, - &classes_tensor_, &scores_tensor_); - - // Validate outputs. - TF_ASSERT_OK(status); - test::ExpectTensorEqual(test::AsTensor({3}), classes_tensor_); - test::ExpectTensorEqual(test::AsTensor({2}), scores_tensor_); - - // Validate inputs. - ASSERT_EQ(1, session_.inputs.size()); - EXPECT_EQ(kInputName, session_.inputs[0].first); - test::ExpectTensorEqual(test::AsTensor({99}), - session_.inputs[0].second); - - ASSERT_EQ(2, session_.output_tensor_names.size()); - EXPECT_EQ(kClassesName, session_.output_tensor_names[0]); - EXPECT_EQ(kScoresName, session_.output_tensor_names[1]); -} - -TEST_F(RunClassificationTest, ClassesOnly) { - input_tensor_ = test::AsTensor({99}); - session_.outputs = {test::AsTensor({3})}; - const Status status = RunClassification(signature_, input_tensor_, &session_, - &classes_tensor_, nullptr); - - // Validate outputs. - TF_ASSERT_OK(status); - test::ExpectTensorEqual(test::AsTensor({3}), classes_tensor_); - - // Validate inputs. - ASSERT_EQ(1, session_.inputs.size()); - EXPECT_EQ(kInputName, session_.inputs[0].first); - test::ExpectTensorEqual(test::AsTensor({99}), - session_.inputs[0].second); - - ASSERT_EQ(1, session_.output_tensor_names.size()); - EXPECT_EQ(kClassesName, session_.output_tensor_names[0]); -} - -TEST_F(RunClassificationTest, ScoresOnly) { - input_tensor_ = test::AsTensor({99}); - session_.outputs = {test::AsTensor({2})}; - const Status status = RunClassification(signature_, input_tensor_, &session_, - nullptr, &scores_tensor_); - - // Validate outputs. - TF_ASSERT_OK(status); - test::ExpectTensorEqual(test::AsTensor({2}), scores_tensor_); - - // Validate inputs. - ASSERT_EQ(1, session_.inputs.size()); - EXPECT_EQ(kInputName, session_.inputs[0].first); - test::ExpectTensorEqual(test::AsTensor({99}), - session_.inputs[0].second); - - ASSERT_EQ(1, session_.output_tensor_names.size()); - EXPECT_EQ(kScoresName, session_.output_tensor_names[0]); -} - -TEST(RunClassification, RunNotOk) { - ClassificationSignature signature; - signature.mutable_input()->set_tensor_name("in:0"); - signature.mutable_classes()->set_tensor_name("classes:0"); - Tensor input_tensor = test::AsTensor({99}); - MockSession session; - session.status = errors::DataLoss("Data is gone"); - Tensor classes_tensor; - const Status status = RunClassification(signature, input_tensor, &session, - &classes_tensor, nullptr); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(absl::StrContains(status.error_message(), "Data is gone")) - << status.error_message(); -} - -TEST(RunClassification, TooManyOutputs) { - ClassificationSignature signature; - signature.mutable_input()->set_tensor_name("in:0"); - signature.mutable_classes()->set_tensor_name("classes:0"); - Tensor input_tensor = test::AsTensor({99}); - MockSession session; - session.outputs = {test::AsTensor({3}), test::AsTensor({4})}; - - Tensor classes_tensor; - const Status status = RunClassification(signature, input_tensor, &session, - &classes_tensor, nullptr); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(absl::StrContains(status.error_message(), "Expected 1 output")) - << status.error_message(); -} - -TEST(RunClassification, WrongBatchOutputs) { - ClassificationSignature signature; - signature.mutable_input()->set_tensor_name("in:0"); - signature.mutable_classes()->set_tensor_name("classes:0"); - Tensor input_tensor = test::AsTensor({99, 100}); - MockSession session; - session.outputs = {test::AsTensor({3})}; - - Tensor classes_tensor; - const Status status = RunClassification(signature, input_tensor, &session, - &classes_tensor, nullptr); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE( - absl::StrContains(status.error_message(), - "Input batch size did not match output batch size")) - << status.error_message(); -} - -constexpr char kRegressionsName[] = "regressions:0"; - -class RunRegressionTest : public ::testing::Test { - public: - void SetUp() override { - signature_.mutable_input()->set_tensor_name(kInputName); - signature_.mutable_output()->set_tensor_name(kRegressionsName); - } - - protected: - RegressionSignature signature_; - Tensor input_tensor_; - Tensor output_tensor_; - MockSession session_; -}; - -TEST_F(RunRegressionTest, Basic) { - input_tensor_ = test::AsTensor({99, 100}); - session_.outputs = {test::AsTensor({1, 2})}; - const Status status = - RunRegression(signature_, input_tensor_, &session_, &output_tensor_); - - // Validate outputs. - TF_ASSERT_OK(status); - test::ExpectTensorEqual(test::AsTensor({1, 2}), output_tensor_); - - // Validate inputs. - ASSERT_EQ(1, session_.inputs.size()); - EXPECT_EQ(kInputName, session_.inputs[0].first); - test::ExpectTensorEqual(test::AsTensor({99, 100}), - session_.inputs[0].second); - - ASSERT_EQ(1, session_.output_tensor_names.size()); - EXPECT_EQ(kRegressionsName, session_.output_tensor_names[0]); -} - -TEST_F(RunRegressionTest, RunNotOk) { - input_tensor_ = test::AsTensor({99}); - session_.status = errors::DataLoss("Data is gone"); - const Status status = - RunRegression(signature_, input_tensor_, &session_, &output_tensor_); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(absl::StrContains(status.error_message(), "Data is gone")) - << status.error_message(); -} - -TEST_F(RunRegressionTest, MismatchedSizeForBatchInputAndOutput) { - input_tensor_ = test::AsTensor({99, 100}); - session_.outputs = {test::AsTensor({3})}; - - const Status status = - RunRegression(signature_, input_tensor_, &session_, &output_tensor_); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE( - absl::StrContains(status.error_message(), - "Input batch size did not match output batch size")) - << status.error_message(); -} - -TEST(SetAndGetSignatures, RoundTrip) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - signatures.mutable_default_signature() - ->mutable_classification_signature() - ->mutable_input() - ->set_tensor_name("in:0"); - TF_ASSERT_OK(SetSignatures(signatures, &meta_graph_def)); - Signatures read_signatures; - TF_ASSERT_OK(GetSignatures(meta_graph_def, &read_signatures)); - - EXPECT_EQ("in:0", read_signatures.default_signature() - .classification_signature() - .input() - .tensor_name()); -} - -TEST(GetSignatures, MissingSignature) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures read_signatures; - const auto status = GetSignatures(meta_graph_def, &read_signatures); - EXPECT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code()); - EXPECT_TRUE(absl::StrContains(status.error_message(), "Expected exactly one")) - << status.error_message(); -} - -TEST(GetSignatures, WrongProtoInAny) { - tensorflow::MetaGraphDef meta_graph_def; - auto& collection_def = *(meta_graph_def.mutable_collection_def()); - auto* any = - collection_def[kSignaturesKey].mutable_any_list()->mutable_value()->Add(); - // Put an unexpected type into the Signatures Any. - any->PackFrom(TensorBinding()); - Signatures read_signatures; - const auto status = GetSignatures(meta_graph_def, &read_signatures); - EXPECT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code()); - EXPECT_TRUE(absl::StrContains(status.error_message(), - "Expected Any type_url for: " - "tensorflow.serving.Signatures")) - << status.error_message(); -} - -TEST(GetSignatures, JunkInAny) { - tensorflow::MetaGraphDef meta_graph_def; - auto& collection_def = *(meta_graph_def.mutable_collection_def()); - auto* any = - collection_def[kSignaturesKey].mutable_any_list()->mutable_value()->Add(); - // Create a valid Any then corrupt it. - any->PackFrom(Signatures()); - any->set_value("junk junk"); - Signatures read_signatures; - const auto status = GetSignatures(meta_graph_def, &read_signatures); - EXPECT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code()); - EXPECT_TRUE(absl::StrContains(status.error_message(), "Failed to unpack")) - << status.error_message(); -} - -TEST(GetSignatures, DefaultAndNamedTogetherOK) { - tensorflow::MetaGraphDef meta_graph_def; - auto& collection_def = *(meta_graph_def.mutable_collection_def()); - auto* any = - collection_def[kSignaturesKey].mutable_any_list()->mutable_value()->Add(); - Signatures signatures; - signatures.mutable_default_signature() - ->mutable_classification_signature() - ->mutable_input() - ->set_tensor_name("in:0"); - ClassificationSignature* input_signature = - (*signatures.mutable_named_signatures())["foo"] - .mutable_classification_signature(); - input_signature->mutable_input()->set_tensor_name("flow"); - - any->PackFrom(signatures); - Signatures read_signatures; - const auto status = GetSignatures(meta_graph_def, &read_signatures); - - EXPECT_TRUE(status.ok()); -} - -// Check that we only have one 'Signatures' entry in the collection_def map. -// Note that each such object can have multiple named_signatures inside of it. -TEST(GetSignatures, MultipleSignaturesNotOK) { - tensorflow::MetaGraphDef meta_graph_def; - auto& collection_def = *(meta_graph_def.mutable_collection_def()); - auto* any = - collection_def[kSignaturesKey].mutable_any_list()->mutable_value()->Add(); - Signatures signatures; - signatures.mutable_default_signature() - ->mutable_classification_signature() - ->mutable_input() - ->set_tensor_name("in:0"); - any->PackFrom(signatures); - - // Add another signatures object. - any = - collection_def[kSignaturesKey].mutable_any_list()->mutable_value()->Add(); - any->PackFrom(signatures); - Signatures read_signatures; - const auto status = GetSignatures(meta_graph_def, &read_signatures); - EXPECT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code()); - EXPECT_TRUE(absl::StrContains(status.error_message(), "Expected exactly one")) - << status.error_message(); -} - -// GenericSignature test fixture that contains a signature initialized with two -// bound Tensors. -class GenericSignatureTest : public ::testing::Test { - protected: - GenericSignatureTest() { - TensorBinding binding; - binding.set_tensor_name("graph_A"); - signature_.mutable_map()->insert({"logical_A", binding}); - - binding.set_tensor_name("graph_B"); - signature_.mutable_map()->insert({"logical_B", binding}); - } - - // GenericSignature that contains two bound Tensors. - GenericSignature signature_; -}; - -// GenericSignature tests. - -TEST_F(GenericSignatureTest, GetGenericSignatureBasic) { - Signature expected_signature; - expected_signature.mutable_generic_signature()->MergeFrom(signature_); - - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - signatures.mutable_named_signatures()->insert( - {"generic_bindings", expected_signature}); - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - GenericSignature actual_signature; - TF_ASSERT_OK(GetGenericSignature("generic_bindings", meta_graph_def, - &actual_signature)); - ASSERT_EQ("graph_A", actual_signature.map().at("logical_A").tensor_name()); - ASSERT_EQ("graph_B", actual_signature.map().at("logical_B").tensor_name()); -} - -TEST(GetGenericSignature, MissingSignature) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - GenericSignature signature; - const Status status = - GetGenericSignature("generic_bindings", meta_graph_def, &signature); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(HasSubstr(status.error_message(), - "Missing generic signature named \"generic_bindings\"")) - << status.error_message(); -} - -TEST(GetGenericSignature, WrongSignatureType) { - tensorflow::MetaGraphDef meta_graph_def; - Signatures signatures; - (*signatures.mutable_named_signatures())["generic_bindings"] - .mutable_regression_signature(); - (*meta_graph_def.mutable_collection_def())[kSignaturesKey] - .mutable_any_list() - ->add_value() - ->PackFrom(signatures); - - GenericSignature signature; - const Status status = - GetGenericSignature("generic_bindings", meta_graph_def, &signature); - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(absl::StrContains(status.error_message(), - "Expected a generic signature:")) - << status.error_message(); -} - -// BindGeneric Tests. - -TEST_F(GenericSignatureTest, BindGenericInputsBasic) { - const std::vector> inputs = { - {"logical_A", test::AsTensor({-1.0})}, - {"logical_B", test::AsTensor({-2.0})}}; - - std::vector> bound_inputs; - TF_ASSERT_OK(BindGenericInputs(signature_, inputs, &bound_inputs)); - - EXPECT_EQ("graph_A", bound_inputs[0].first); - EXPECT_EQ("graph_B", bound_inputs[1].first); - test::ExpectTensorEqual(test::AsTensor({-1.0}), - bound_inputs[0].second); - test::ExpectTensorEqual(test::AsTensor({-2.0}), - bound_inputs[1].second); -} - -TEST_F(GenericSignatureTest, BindGenericInputsMissingBinding) { - const std::vector> inputs = { - {"logical_A", test::AsTensor({-42.0})}, - {"logical_MISSING", test::AsTensor({-43.0})}}; - - std::vector> bound_inputs; - const Status status = BindGenericInputs(signature_, inputs, &bound_inputs); - ASSERT_FALSE(status.ok()); -} - -TEST_F(GenericSignatureTest, BindGenericNamesBasic) { - const std::vector input_names = {"logical_B", "logical_A"}; - std::vector bound_names; - TF_ASSERT_OK(BindGenericNames(signature_, input_names, &bound_names)); - - EXPECT_EQ("graph_B", bound_names[0]); - EXPECT_EQ("graph_A", bound_names[1]); -} - -TEST_F(GenericSignatureTest, BindGenericNamesMissingBinding) { - const std::vector input_names = {"logical_B", "logical_MISSING"}; - std::vector bound_names; - const Status status = BindGenericNames(signature_, input_names, &bound_names); - ASSERT_FALSE(status.ok()); -} - -} // namespace -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow/contrib/session_bundle/test_util.cc b/tensorflow/contrib/session_bundle/test_util.cc deleted file mode 100644 index 348a2979c3a..00000000000 --- a/tensorflow/contrib/session_bundle/test_util.cc +++ /dev/null @@ -1,35 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/session_bundle/test_util.h" - -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace serving { -namespace test_util { - -string TestSrcDirPath(const string& relative_path) { - const string base_path = tensorflow::testing::TensorFlowSrcRoot(); - const string contrib_path = tensorflow::io::JoinPath( - tensorflow::testing::TensorFlowSrcRoot(), "/contrib"); - return tensorflow::io::JoinPath(contrib_path, relative_path); -} - -} // namespace test_util -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow/contrib/session_bundle/test_util.h b/tensorflow/contrib/session_bundle/test_util.h deleted file mode 100644 index f0d41ce5a4b..00000000000 --- a/tensorflow/contrib/session_bundle/test_util.h +++ /dev/null @@ -1,38 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_SESSION_BUNDLE_TEST_UTIL_H_ -#define TENSORFLOW_CONTRIB_SESSION_BUNDLE_TEST_UTIL_H_ - -#include - -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/protobuf.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace serving { -namespace test_util { - -// Creates an absolute test srcdir path to the linked in runfiles given a path -// relative to third_party/tensorflow/contrib/. -// e.g. relative path = "session_bundle/example". -string TestSrcDirPath(const string& relative_path); - -} // namespace test_util -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_SESSION_BUNDLE_TEST_UTIL_H_ diff --git a/tensorflow/contrib/session_bundle/testdata/half_plus_two/00000123/export-00000-of-00001 b/tensorflow/contrib/session_bundle/testdata/half_plus_two/00000123/export-00000-of-00001 deleted file mode 100755 index e1ac9e900e8..00000000000 Binary files a/tensorflow/contrib/session_bundle/testdata/half_plus_two/00000123/export-00000-of-00001 and /dev/null differ diff --git a/tensorflow/contrib/session_bundle/testdata/half_plus_two/00000123/export.meta b/tensorflow/contrib/session_bundle/testdata/half_plus_two/00000123/export.meta deleted file mode 100755 index 41e1353cf4c..00000000000 Binary files a/tensorflow/contrib/session_bundle/testdata/half_plus_two/00000123/export.meta and /dev/null differ diff --git a/tensorflow/contrib/session_bundle/testdata/half_plus_two_ckpt_v2/00000123/export.data-00000-of-00001 b/tensorflow/contrib/session_bundle/testdata/half_plus_two_ckpt_v2/00000123/export.data-00000-of-00001 deleted file mode 100755 index 20bc7d454dd..00000000000 Binary files a/tensorflow/contrib/session_bundle/testdata/half_plus_two_ckpt_v2/00000123/export.data-00000-of-00001 and /dev/null differ diff --git a/tensorflow/contrib/session_bundle/testdata/half_plus_two_ckpt_v2/00000123/export.index b/tensorflow/contrib/session_bundle/testdata/half_plus_two_ckpt_v2/00000123/export.index deleted file mode 100755 index e7df518f5b5..00000000000 Binary files a/tensorflow/contrib/session_bundle/testdata/half_plus_two_ckpt_v2/00000123/export.index and /dev/null differ diff --git a/tensorflow/contrib/session_bundle/testdata/half_plus_two_ckpt_v2/00000123/export.meta b/tensorflow/contrib/session_bundle/testdata/half_plus_two_ckpt_v2/00000123/export.meta deleted file mode 100755 index 9d383f5224d..00000000000 Binary files a/tensorflow/contrib/session_bundle/testdata/half_plus_two_ckpt_v2/00000123/export.meta and /dev/null differ diff --git a/tensorflow/contrib/signal/BUILD b/tensorflow/contrib/signal/BUILD deleted file mode 100644 index 61798014da2..00000000000 --- a/tensorflow/contrib/signal/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "signal_py", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/ops/signal", - ], -) diff --git a/tensorflow/contrib/signal/__init__.py b/tensorflow/contrib/signal/__init__.py deleted file mode 100644 index d01f5ccf51c..00000000000 --- a/tensorflow/contrib/signal/__init__.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Signal processing operations. - -`tf.contrib.signal` has been renamed to `tf.signal`. `tf.contrib.signal` will be -removed in TensorFlow 2.0. - -See the -[Contrib Signal](https://tensorflow.org/api_guides/python/contrib.signal) -guide. - -@@frame -@@hamming_window -@@hann_window -@@inverse_stft -@@inverse_stft_window_fn -@@mfccs_from_log_mel_spectrograms -@@linear_to_mel_weight_matrix -@@overlap_and_add -@@stft - -[hamming]: https://en.wikipedia.org/wiki/Window_function#Hamming_window -[hann]: https://en.wikipedia.org/wiki/Window_function#Hann_window -[mel]: https://en.wikipedia.org/wiki/Mel_scale -[mfcc]: https://en.wikipedia.org/wiki/Mel-frequency_cepstrum -[stft]: https://en.wikipedia.org/wiki/Short-time_Fourier_transform -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops.signal.mel_ops import linear_to_mel_weight_matrix -from tensorflow.python.ops.signal.mfcc_ops import mfccs_from_log_mel_spectrograms -from tensorflow.python.ops.signal.reconstruction_ops import overlap_and_add -from tensorflow.python.ops.signal.shape_ops import frame -from tensorflow.python.ops.signal.spectral_ops import inverse_stft -from tensorflow.python.ops.signal.spectral_ops import inverse_stft_window_fn -from tensorflow.python.ops.signal.spectral_ops import stft -from tensorflow.python.ops.signal.window_ops import hamming_window -from tensorflow.python.ops.signal.window_ops import hann_window - -from tensorflow.python.util.all_util import remove_undocumented - -# `frame` used to be named `frames`, which is a noun and not a verb. -# Keep an alias to `frames` for backwards compatibility. -frames = frame - -remove_undocumented(__name__) diff --git a/tensorflow/contrib/signal/python/ops/__init__.py b/tensorflow/contrib/signal/python/ops/__init__.py deleted file mode 100644 index e672d1146c5..00000000000 --- a/tensorflow/contrib/signal/python/ops/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Signal ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/slim/BUILD b/tensorflow/contrib/slim/BUILD deleted file mode 100644 index 9ca79edbeb8..00000000000 --- a/tensorflow/contrib/slim/BUILD +++ /dev/null @@ -1,184 +0,0 @@ -# Description: -# Contains the Slim library, including common neural networks and examples. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "evaluation", - srcs = ["python/slim/evaluation.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/training:training_py", - "//tensorflow/python:summary", - "//tensorflow/python:training", - ], -) - -py_test( - name = "evaluation_test", - srcs = ["python/slim/evaluation_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":evaluation", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/metrics:metrics_py", - "//tensorflow/contrib/training:training_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:summary", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//tensorflow/python/debug:debug_data", - "//tensorflow/python/debug:hooks", - "//third_party/py/numpy", - ], -) - -py_library( - name = "learning", - srcs = ["python/slim/learning.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/training:training_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client", - "//tensorflow/python:clip_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:lib", - "//tensorflow/python:lookup_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:summary", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "learning_test", - srcs = ["python/slim/learning_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["manual"], - deps = [ - ":learning", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/losses:losses_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:session", - "//tensorflow/python:summary", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//tensorflow/python/debug:debug_data", - "//tensorflow/python/debug:dumping_wrapper", - "//third_party/py/numpy", - ], -) - -py_library( - name = "model_analyzer", - srcs = ["python/slim/model_analyzer.py"], - srcs_version = "PY2AND3", -) - -py_library( - name = "nets", - srcs = ["nets.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/slim/python/slim/nets:alexnet", - "//tensorflow/contrib/slim/python/slim/nets:inception", - "//tensorflow/contrib/slim/python/slim/nets:overfeat", - "//tensorflow/contrib/slim/python/slim/nets:resnet_utils", - "//tensorflow/contrib/slim/python/slim/nets:resnet_v1", - "//tensorflow/contrib/slim/python/slim/nets:resnet_v2", - "//tensorflow/contrib/slim/python/slim/nets:vgg", - "//tensorflow/python:util", - ], -) - -py_library( - name = "queues", - srcs = ["python/slim/queues.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:training", - ], -) - -py_library( - name = "slim", - srcs = [ - "__init__.py", - ], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:__subpackages__"], - deps = [ - ":evaluation", - ":learning", - ":model_analyzer", - ":queues", - ":summaries", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/losses:losses_py", - "//tensorflow/contrib/metrics:metrics_py", - "//tensorflow/contrib/slim/python/slim/data:data_decoder", - "//tensorflow/contrib/slim/python/slim/data:data_provider", - "//tensorflow/contrib/slim/python/slim/data:dataset", - "//tensorflow/contrib/slim/python/slim/data:dataset_data_provider", - "//tensorflow/contrib/slim/python/slim/data:parallel_reader", - "//tensorflow/contrib/slim/python/slim/data:prefetch_queue", - "//tensorflow/contrib/slim/python/slim/data:tfexample_decoder", - "//tensorflow/python:util", - ], -) - -py_library( - name = "summaries", - srcs = ["python/slim/summaries.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:framework_ops", - "//tensorflow/python:logging_ops", - "//tensorflow/python:nn", - "//tensorflow/python:summary", - ], -) - -py_test( - name = "summaries_test", - srcs = ["python/slim/summaries_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":summaries", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_ops", - "//tensorflow/python:platform", - "//tensorflow/python:summary", - ], -) diff --git a/tensorflow/contrib/slim/README.md b/tensorflow/contrib/slim/README.md deleted file mode 100644 index 2f6b006da77..00000000000 --- a/tensorflow/contrib/slim/README.md +++ /dev/null @@ -1,915 +0,0 @@ -# TensorFlow-Slim - -TF-Slim is a lightweight library for defining, training and evaluating complex -models in TensorFlow. Components of tf-slim can be freely mixed with native -tensorflow, as well as other frameworks, such as tf.contrib.learn. - -## Usage -```python -import tensorflow.contrib.slim as slim -``` - -## Why TF-Slim? - -TF-Slim is a library that makes defining, training and evaluating neural -networks simple: - -* Allows the user to define models much more compactly by eliminating -boilerplate code. This is accomplished through the use of -[argument scoping](https://www.tensorflow.org/code/tensorflow/contrib/framework/python/ops/arg_scope.py) -and numerous high level -[layers](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -and -[variables](https://www.tensorflow.org/code/tensorflow/contrib/framework/python/ops/variables.py). -These tools increase readability and maintainability, reduce the likelihood -of an error from copy-and-pasting hyperparameter values and simplifies -hyperparameter tuning. -* Makes developing models simple by providing commonly used -[regularizers](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/regularizers.py). -* Several widely used computer vision models (e.g., VGG, AlexNet) have been -developed in slim, and are -[available](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/nets/) -to users. These can either be used as black boxes, or can be extended in various -ways, e.g., by adding "multiple heads" to different internal layers. -* Slim makes it easy to extend complex models, and to warm start training -algorithms by using pieces of pre-existing model checkpoints. - -## What are the various components of TF-Slim? - -TF-Slim is composed of several parts which were design to exist independently. -These include the following main pieces (explained in detail below). - -* [arg_scope](https://www.tensorflow.org/code/tensorflow/contrib/framework/python/ops/arg_scope.py): -provides a new scope named `arg_scope` that allows a user to define default -arguments for specific operations within that scope. -* [data](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/data/): -contains TF-slim's -[dataset](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/data/dataset.py) -definition, -[data providers](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/data/data_provider.py), -[parallel_reader](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/data/parallel_reader.py), -and -[decoding](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/data/data_decoder.py) -utilities. -* [evaluation](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/evaluation.py): -contains routines for evaluating models. -* [layers](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py): -contains high level layers for building models using tensorflow. -* [learning](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/learning.py): -contains routines for training models. -* [losses](https://www.tensorflow.org/code/tensorflow/contrib/losses/python/losses/loss_ops.py): -contains commonly used loss functions. -* [metrics](https://www.tensorflow.org/code/tensorflow/contrib/metrics/python/ops/metric_ops.py): -contains popular evaluation metrics. -* [nets](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/nets/): -contains popular network definitions such as -[VGG](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/nets/vgg.py) -and -[AlexNet](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/nets/alexnet.py) -models. -* [queues](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/queues.py): -provides a context manager for easily and safely starting and closing -QueueRunners. -* [regularizers](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/regularizers.py): -contains weight regularizers. -* [variables](https://www.tensorflow.org/code/tensorflow/contrib/framework/python/ops/variables.py): -provides convenience wrappers for variable creation and manipulation. - -## Defining Models - -Models can be succinctly defined using TF-Slim by combining its variables, -layers and scopes. Each of these elements is defined below. - -### Variables - -Creating -[`Variables`](https://www.tensorflow.org/how_tos/variables/index.html) -in native tensorflow requires either a predefined value or an initialization -mechanism (e.g. randomly sampled from a Gaussian). Furthermore, if a variable -needs to be created -on a specific device, such as a GPU, the specification must be -[made explicit](https://www.tensorflow.org/how_tos/using_gpu/index.html). -To alleviate the code required for variable creation, TF-Slim provides a set -of thin wrapper functions in -[variables.py](https://www.tensorflow.org/code/tensorflow/contrib/framework/python/ops/variables.py) -which allow callers to easily define variables. - -For example, to create a `weights` variable, initialize it using a truncated -normal distribution, regularize it with an `l2_loss` and place it on the `CPU`, -one need only declare the following: - -```python -weights = slim.variable('weights', - shape=[10, 10, 3 , 3], - initializer=tf.truncated_normal_initializer(stddev=0.1), - regularizer=slim.l2_regularizer(0.05), - device='/CPU:0') -``` - -Note that in native TensorFlow, there are two types of variables: regular -variables and local (transient) variables. The vast majority of variables are -regular variables: once created, they can be saved to disk using a -[saver](https://www.tensorflow.org/api_docs/python/tf/train/Saver). -Local variables are those variables that only exist for the duration of a -session and are not saved to disk. - -TF-Slim further differentiates variables by defining *model variables*, which -are variables that represent parameters of a model. Model variables are -trained or fine-tuned during learning and are loaded -from a checkpoint during evaluation or inference. Examples include the variables -created by a `slim.fully_connected` or `slim.conv2d` layer. Non-model variables -are all other variables that are used during learning or evaluation but are not -required for actually performing inference. For example, the `global_step` is -a variable using during learning and evaluation but it is not actually part of -the model. Similarly, moving average variables might mirror model variables, -but the moving averages are not themselves model variables. - -Both model variables and regular variables can be easily created and retrieved -via TF-Slim: - -```python -# Model Variables -weights = slim.model_variable('weights', - shape=[10, 10, 3 , 3], - initializer=tf.truncated_normal_initializer(stddev=0.1), - regularizer=slim.l2_regularizer(0.05), - device='/CPU:0') -model_variables = slim.get_model_variables() - -# Regular variables -my_var = slim.variable('my_var', - shape=[20, 1], - initializer=tf.zeros_initializer()) -regular_variables_and_model_variables = slim.get_variables() -``` - -How does this work? When you create a model variable via TF-Slim's layers or -directly via the `slim.model_variable` function, TF-Slim adds the variable to -the `tf.GraphKeys.MODEL_VARIABLES` collection. What if you have your own -custom layers or variable creation routine but still want TF-Slim to manage or -be aware of your model variables? TF-Slim provides a convenience function for -adding the model variable to its collection: - -```python -my_model_variable = CreateViaCustomCode() - -# Letting TF-Slim know about the additional variable. -slim.add_model_variable(my_model_variable) -``` - - -### Layers - -While the set of TensorFlow operations is quite extensive, developers of neural -networks typically think of models in terms of higher level concepts like -"layers", "losses", "metrics", and "networks". A layer, such as a Convolutional -Layer, a Fully Connected Layer or a BatchNorm Layer is more abstract than a -single TensorFlow operation and typically involve several operations. -Furthermore, a layer usually (but not always) has variables (tunable parameters) -associated with it, unlike more primitive operations. For example, a -Convolutional Layer in a neural network is composed of several low level -operations: - -1. Creating the weight and bias variables -2. Convolving the weights with the input from the previous layer -3. Adding the biases to the result of the convolution. -4. Applying an activation function. - -Using only plain TensorFlow code, this can be rather laborious: - -```python -input = ... -with tf.name_scope('conv1_1') as scope: - kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 128], dtype=tf.float32, - stddev=1e-1), name='weights') - conv = tf.nn.conv2d(input, kernel, [1, 1, 1, 1], padding='SAME') - biases = tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32), - trainable=True, name='biases') - bias = tf.nn.bias_add(conv, biases) - conv1 = tf.nn.relu(bias, name=scope) -``` - -To alleviate the need to duplicate this code repeatedly, TF-Slim provides a -number of convenient operations defined at the more abstract level of -neural network layers. For example, compare the code above to an invocation -of the corresponding TF-Slim code: - -```python -input = ... -net = slim.conv2d(input, 128, [3, 3], scope='conv1_1') -``` - -TF-Slim provides standard implementations for numerous components for building -neural networks. These include: - -Layer | TF-Slim -------- | -------- -BiasAdd | [slim.bias_add](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -BatchNorm | [slim.batch_norm](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -Conv2d | [slim.conv2d](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -Conv2dInPlane | [slim.conv2d_in_plane](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -Conv2dTranspose (Deconv) | [slim.conv2d_transpose](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -FullyConnected | [slim.fully_connected](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -AvgPool2D | [slim.avg_pool2d](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -Dropout| [slim.dropout](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -Flatten | [slim.flatten](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -MaxPool2D | [slim.max_pool2d](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -OneHotEncoding | [slim.one_hot_encoding](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -SeparableConv2 | [slim.separable_conv2d](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -UnitNorm | [slim.unit_norm](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) - -TF-Slim also provides two meta-operations called `repeat` and `stack` that -allow users to repeatedly perform the same operation. For example, consider the -following snippet from the -[VGG](https://www.robots.ox.ac.uk/~vgg/research/very_deep/) network whose layers -perform several convolutions in a row between pooling layers: - -```python -net = ... -net = slim.conv2d(net, 256, [3, 3], scope='conv3_1') -net = slim.conv2d(net, 256, [3, 3], scope='conv3_2') -net = slim.conv2d(net, 256, [3, 3], scope='conv3_3') -net = slim.max_pool2d(net, [2, 2], scope='pool2') -``` - -One way to reduce this code duplication would be via a `for` loop: - -```python -net = ... -for i in range(3): - net = slim.conv2d(net, 256, [3, 3], scope='conv3_%d' % (i+1)) -net = slim.max_pool2d(net, [2, 2], scope='pool2') -``` - -This can be made even cleaner by using TF-Slim's `repeat` operation: - -```python -net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], scope='conv3') -net = slim.max_pool2d(net, [2, 2], scope='pool2') -``` - -Notice that the `slim.repeat` not only applies the same argument in-line, it -also is smart enough to unroll the scopes such that the scopes assigned to each -subsequent call of `slim.conv2d` are appended with an underscore and iteration -number. More concretely, the scopes in the example above would be named -'conv3/conv3_1', 'conv3/conv3_2' and 'conv3/conv3_3'. - -Furthermore, TF-Slim's `slim.stack` operator allows a caller to repeatedly apply -the same operation with different arguments to create a *stack* or tower of -layers. `slim.stack` also creates a new `tf.variable_scope` for each -operation created. For example, a simple way to create a Multi-Layer Perceptron -(MLP): - -```python -# Verbose way: -x = slim.fully_connected(x, 32, scope='fc/fc_1') -x = slim.fully_connected(x, 64, scope='fc/fc_2') -x = slim.fully_connected(x, 128, scope='fc/fc_3') - -# Equivalent, TF-Slim way using slim.stack: -slim.stack(x, slim.fully_connected, [32, 64, 128], scope='fc') -``` - -In this example, `slim.stack` calls `slim.fully_connected` three times passing -the output of one invocation of the function to the next. However, the number of -hidden units in each invocation changes from 32 to 64 to 128. Similarly, one -can use stack to simplify a tower of multiple convolutions: - -```python -# Verbose way: -x = slim.conv2d(x, 32, [3, 3], scope='core/core_1') -x = slim.conv2d(x, 32, [1, 1], scope='core/core_2') -x = slim.conv2d(x, 64, [3, 3], scope='core/core_3') -x = slim.conv2d(x, 64, [1, 1], scope='core/core_4') - -# Using stack: -slim.stack(x, slim.conv2d, [(32, [3, 3]), (32, [1, 1]), (64, [3, 3]), (64, [1, 1])], scope='core') -``` - -### Scopes - -In addition to the types of scope mechanisms in TensorFlow -([name_scope](https://www.tensorflow.org/api_docs/python/tf/name_scope), -[variable_scope](https://www.tensorflow.org/api_docs/python/tf/variable_scope)), -TF-Slim adds a new scoping mechanism called -[arg_scope](https://www.tensorflow.org/api_docs/python/tf/contrib/framework/arg_scope). -This new scope allows a user to specify one or more operations and a set of -arguments which will be passed to each of the operations defined in the -`arg_scope`. This functionality is best illustrated by example. Consider the -following code snippet: - - -```python -net = slim.conv2d(inputs, 64, [11, 11], 4, padding='SAME', - weights_initializer=tf.truncated_normal_initializer(stddev=0.01), - weights_regularizer=slim.l2_regularizer(0.0005), scope='conv1') -net = slim.conv2d(net, 128, [11, 11], padding='VALID', - weights_initializer=tf.truncated_normal_initializer(stddev=0.01), - weights_regularizer=slim.l2_regularizer(0.0005), scope='conv2') -net = slim.conv2d(net, 256, [11, 11], padding='SAME', - weights_initializer=tf.truncated_normal_initializer(stddev=0.01), - weights_regularizer=slim.l2_regularizer(0.0005), scope='conv3') -``` - -It should be clear that these three convolution layers share many of the same -hyperparameters. Two have the same padding, all three have the same -weights_initializer and weight_regularizer. This code is hard to read and -contains a lot of repeated values that should be factored out. One solution -would be to specify default values using variables: - -```python -padding = 'SAME' -initializer = tf.truncated_normal_initializer(stddev=0.01) -regularizer = slim.l2_regularizer(0.0005) -net = slim.conv2d(inputs, 64, [11, 11], 4, - padding=padding, - weights_initializer=initializer, - weights_regularizer=regularizer, - scope='conv1') -net = slim.conv2d(net, 128, [11, 11], - padding='VALID', - weights_initializer=initializer, - weights_regularizer=regularizer, - scope='conv2') -net = slim.conv2d(net, 256, [11, 11], - padding=padding, - weights_initializer=initializer, - weights_regularizer=regularizer, - scope='conv3') -``` - -This solution ensures that all three convolutions share the exact same parameter -values but doesn't reduce completely the code clutter. By using an `arg_scope`, -we can both ensure that each layer uses the same values and simplify the code: - -```python - with slim.arg_scope([slim.conv2d], padding='SAME', - weights_initializer=tf.truncated_normal_initializer(stddev=0.01) - weights_regularizer=slim.l2_regularizer(0.0005)): - net = slim.conv2d(inputs, 64, [11, 11], 4, scope='conv1') - net = slim.conv2d(net, 128, [11, 11], padding='VALID', scope='conv2') - net = slim.conv2d(net, 256, [11, 11], scope='conv3') -``` - -As the example illustrates, the use of arg_scope makes the code cleaner, -simpler and easier to maintain. Notice that while argument values are specified -in the arg_scope, they can be overwritten locally. In particular, while -the padding argument has been set to 'SAME', the second convolution overrides -it with the value of 'VALID'. - -One can also nest `arg_scopes` and use multiple operations in the same scope. -For example: - -```python -with slim.arg_scope([slim.conv2d, slim.fully_connected], - activation_fn=tf.nn.relu, - weights_initializer=tf.truncated_normal_initializer(stddev=0.01), - weights_regularizer=slim.l2_regularizer(0.0005)): - with slim.arg_scope([slim.conv2d], stride=1, padding='SAME'): - net = slim.conv2d(inputs, 64, [11, 11], 4, padding='VALID', scope='conv1') - net = slim.conv2d(net, 256, [5, 5], - weights_initializer=tf.truncated_normal_initializer(stddev=0.03), - scope='conv2') - net = slim.fully_connected(net, 1000, activation_fn=None, scope='fc') -``` - -In this example, the first `arg_scope` applies the same `weights_initializer` -and `weights_regularizer` arguments to the `conv2d` and `fully_connected` layers -in its scope. In the second `arg_scope`, additional default arguments to -`conv2d` only are specified. - -### Working Example: Specifying the VGG16 Layers - -By combining TF-Slim Variables, Operations and scopes, we can write a normally -very complex network with very few lines of code. For example, the entire -[VGG](https://www.robots.ox.ac.uk/~vgg/research/very_deep/) architecture can be -defined with just the following snippet: - -```python -def vgg16(inputs): - with slim.arg_scope([slim.conv2d, slim.fully_connected], - activation_fn=tf.nn.relu, - weights_initializer=tf.truncated_normal_initializer(0.0, 0.01), - weights_regularizer=slim.l2_regularizer(0.0005)): - net = slim.repeat(inputs, 2, slim.conv2d, 64, [3, 3], scope='conv1') - net = slim.max_pool2d(net, [2, 2], scope='pool1') - net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2') - net = slim.max_pool2d(net, [2, 2], scope='pool2') - net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], scope='conv3') - net = slim.max_pool2d(net, [2, 2], scope='pool3') - net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv4') - net = slim.max_pool2d(net, [2, 2], scope='pool4') - net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv5') - net = slim.max_pool2d(net, [2, 2], scope='pool5') - net = slim.fully_connected(net, 4096, scope='fc6') - net = slim.dropout(net, 0.5, scope='dropout6') - net = slim.fully_connected(net, 4096, scope='fc7') - net = slim.dropout(net, 0.5, scope='dropout7') - net = slim.fully_connected(net, 1000, activation_fn=None, scope='fc8') - return net -``` - -## Training Models - -Training Tensorflow models requires a model, a loss function, the gradient -computation and a training routine that iteratively computes the gradients -of the model weights relative to the loss and updates the weights accordingly. -TF-Slim provides both common loss functions and a set of helper functions -that run the training and evaluation routines. - -### Losses - -The loss function defines a quantity that we want to minimize. For -classification problems, this is typically the cross entropy between the true -distribution and the predicted probability distribution across -classes. For regression problems, this is often the sum-of-squares differences -between the predicted and true values. - -Certain models, such as multi-task -learning models, require the use of multiple loss functions simultaneously. In -other words, the loss function ultimately being minimized is the sum of various -other loss functions. For example, consider a model that predicts both -the type of scene in an image as well as the depth from the -camera of each pixel. This model's loss function would be the sum of the -classification loss and depth prediction loss. - -TF-Slim provides an easy-to-use mechanism for defining and keeping track of -loss functions via the -[losses](https://www.tensorflow.org/code/tensorflow/contrib/losses/python/losses/loss_ops.py) -module. Consider the simple case where we want to train the VGG network: - - -```python -import tensorflow as tf -import tensorflow.contrib.slim.nets as nets -vgg = nets.vgg - -# Load the images and labels. -images, labels = ... - -# Create the model. -predictions, _ = vgg.vgg_16(images) - -# Define the loss functions and get the total loss. -loss = slim.losses.softmax_cross_entropy(predictions, labels) -``` - -In this example, we start by creating the model (using TF-Slim's VGG -implementation), and add the standard classification loss. Now, let's turn to -the case where we have a multi-task model that produces multiple outputs: - -```python -# Load the images and labels. -images, scene_labels, depth_labels = ... - -# Create the model. -scene_predictions, depth_predictions = CreateMultiTaskModel(images) - -# Define the loss functions and get the total loss. -classification_loss = slim.losses.softmax_cross_entropy(scene_predictions, scene_labels) -sum_of_squares_loss = slim.losses.sum_of_squares(depth_predictions, depth_labels) - -# The following two lines have the same effect: -total_loss = classification_loss + sum_of_squares_loss -total_loss = slim.losses.get_total_loss(add_regularization_losses=False) -``` - -In this example, we have two losses which we add by calling -`slim.losses.softmax_cross_entropy` and `slim.losses.sum_of_squares`. We can -obtain the total loss by adding them together (`total_loss`) or by calling -`slim.losses.get_total_loss()`. How did this work? -When you create a loss function via TF-Slim, TF-Slim adds the loss to a -special TensorFlow collection of loss functions. This enables you to either -manage the total loss manually, or allow TF-Slim to manage them for you. - -What if you want to let TF-Slim manage the losses for you but have a custom loss -function? -[loss_ops.py](https://www.tensorflow.org/code/tensorflow/contrib/losses/python/losses/loss_ops.py) -also has a function that adds this loss to TF-Slims collection. For example: - -```python -# Load the images and labels. -images, scene_labels, depth_labels, pose_labels = ... - -# Create the model. -scene_predictions, depth_predictions, pose_predictions = CreateMultiTaskModel(images) - -# Define the loss functions and get the total loss. -classification_loss = slim.losses.softmax_cross_entropy(scene_predictions, scene_labels) -sum_of_squares_loss = slim.losses.sum_of_squares(depth_predictions, depth_labels) -pose_loss = MyCustomLossFunction(pose_predictions, pose_labels) -slim.losses.add_loss(pose_loss) # Letting TF-Slim know about the additional loss. - -# The following two ways to compute the total loss are equivalent: -regularization_loss = tf.add_n(slim.losses.get_regularization_losses()) -total_loss1 = classification_loss + sum_of_squares_loss + pose_loss + regularization_loss - -# (Regularization Loss is included in the total loss by default). -total_loss2 = slim.losses.get_total_loss() -``` -In this example, we can again either produce the total loss function manually -or let TF-Slim know about the additional loss and let TF-Slim handle the losses. - -### Training Loop - -TF-Slim provides a simple but powerful set of tools for training models -found in -[learning.py](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/learning.py). -These include a Train function that repeatedly measures the loss, computes -gradients and saves the model to disk, as well as several convenience functions -for manipulating gradients. For example, once we've -specified the model, the loss function and the optimization scheme, we can -call `slim.learning.create_train_op` and `slim.learning.train` to perform the -optimization: - -```python -g = tf.Graph() - -# Create the model and specify the losses... -... - -total_loss = slim.losses.get_total_loss() -optimizer = tf.train.GradientDescentOptimizer(learning_rate) - -# create_train_op ensures that each time we ask for the loss, the update_ops -# are run and the gradients being computed are applied too. -train_op = slim.learning.create_train_op(total_loss, optimizer) -logdir = ... # Where checkpoints are stored. - -slim.learning.train( - train_op, - logdir, - number_of_steps=1000, - save_summaries_secs=300, - save_interval_secs=600): -``` - -In this example, `slim.learning.train` is provided with the `train_op` which is -used to (a) compute the loss and (b) apply the gradient step. `logdir` specifies -the directory where the checkpoints and event files are stored. We can limit the -number of gradient steps taken to any number. In this case, we've asked for -`1000` steps to be taken. Finally, `save_summaries_secs=300` indicates that -we'll compute summaries every 5 minutes and `save_interval_secs=600` indicates -that we'll save a model checkpoint every 10 minutes. - -### Working Example: Training the VGG16 Model - -To illustrate this, let's examine the following sample of training the VGG -network: - -```python -import tensorflow as tf -import tensorflow.contrib.slim.nets as nets - -slim = tf.contrib.slim -vgg = nets.vgg - -... - -train_log_dir = ... -if not tf.gfile.Exists(train_log_dir): - tf.gfile.MakeDirs(train_log_dir) - -with tf.Graph().as_default(): - # Set up the data loading: - images, labels = ... - - # Define the model: - predictions = vgg.vgg_16(images, is_training=True) - - # Specify the loss function: - slim.losses.softmax_cross_entropy(predictions, labels) - - total_loss = slim.losses.get_total_loss() - tf.summary.scalar('losses/total_loss', total_loss) - - # Specify the optimization scheme: - optimizer = tf.train.GradientDescentOptimizer(learning_rate=.001) - - # create_train_op that ensures that when we evaluate it to get the loss, - # the update_ops are done and the gradient updates are computed. - train_tensor = slim.learning.create_train_op(total_loss, optimizer) - - # Actually runs training. - slim.learning.train(train_tensor, train_log_dir) -``` - -## Fine-Tuning Existing Models - -### Brief Recap on Restoring Variables from a Checkpoint - -After a model has been trained, it can be restored using `tf.train.Saver()` -which restores `Variables` from a given checkpoint. For many cases, -`tf.train.Saver()` provides a simple mechanism to restore all or just a -few variables. - -```python -# Create some variables. -v1 = tf.Variable(..., name="v1") -v2 = tf.Variable(..., name="v2") -... -# Add ops to restore all the variables. -restorer = tf.train.Saver() - -# Add ops to restore some variables. -restorer = tf.train.Saver([v1, v2]) - -# Later, launch the model, use the saver to restore variables from disk, and -# do some work with the model. -with tf.Session() as sess: - # Restore variables from disk. - restorer.restore(sess, "/tmp/model.ckpt") - print("Model restored.") - # Do some work with the model - ... -``` - -See [Restoring Variables](https://www.tensorflow.org/how_tos/variables/index.html#restoring-variables) -and -[Choosing which Variables to Save and Restore](https://www.tensorflow.org/how_tos/variables/index.html#choosing-which-variables-to-save-and-restore) -sections of the [Variables](https://www.tensorflow.org/how_tos/variables/index.html) -page for more details. - -### Partially Restoring Models - -It is often desirable to fine-tune a pre-trained model on an entirely new -dataset or even a new task. In these situations, one can use TF-Slim's -helper functions to select a subset of variables to restore: - -```python -# Create some variables. -v1 = slim.variable(name="v1", ...) -v2 = slim.variable(name="nested/v2", ...) -... - -# Get list of variables to restore (which contains only 'v2'). These are all -# equivalent methods: -variables_to_restore = slim.get_variables_by_name("v2") -# or -variables_to_restore = slim.get_variables_by_suffix("2") -# or -variables_to_restore = slim.get_variables(scope="nested") -# or -variables_to_restore = slim.get_variables_to_restore(include=["nested"]) -# or -variables_to_restore = slim.get_variables_to_restore(exclude=["v1"]) - -# Create the saver which will be used to restore the variables. -restorer = tf.train.Saver(variables_to_restore) - -with tf.Session() as sess: - # Restore variables from disk. - restorer.restore(sess, "/tmp/model.ckpt") - print("Model restored.") - # Do some work with the model - ... -``` - -### Restoring models with different variable names - -When restoring variables from a checkpoint, the `Saver` -locates the variable names in a checkpoint file and maps them to variables in -the current graph. Above, we created a saver by passing to it a list of -variables. In this case, the names of the variables to locate in the checkpoint -file were implicitly obtained from each provided variable's `var.op.name`. - -This works well when the variable names in the checkpoint file match those in -the graph. However, sometimes, we want to restore a model from a checkpoint -whose variables have different names to those in the current graph. In this case, -we must provide the `Saver` a dictionary that maps from each checkpoint variable -name to each graph variable. Consider the following example where the checkpoint -variables names are obtained via a simple function: - -```python -# Assuming that 'conv1/weights' should be restored from 'vgg16/conv1/weights' -def name_in_checkpoint(var): - return 'vgg16/' + var.op.name - -# Assuming that 'conv1/weights' and 'conv1/bias' should be restored from 'conv1/params1' and 'conv1/params2' -def name_in_checkpoint(var): - if "weights" in var.op.name: - return var.op.name.replace("weights", "params1") - if "bias" in var.op.name: - return var.op.name.replace("bias", "params2") - -variables_to_restore = slim.get_model_variables() -variables_to_restore = {name_in_checkpoint(var):var for var in variables_to_restore} -restorer = tf.train.Saver(variables_to_restore) - -with tf.Session() as sess: - # Restore variables from disk. - restorer.restore(sess, "/tmp/model.ckpt") -``` - -### Fine-Tuning a Model on a different task - -Consider the case where we have a pre-trained VGG16 model. The model was -trained on the ImageNet dataset, which has 1000 classes. However, we would -like to apply it to the Pascal VOC dataset which has only 20 classes. To -do so, we can initialize our new model using the values of the pre-trained -model excluding the final layer: - -```python -# Load the Pascal VOC data -image, label = MyPascalVocDataLoader(...) -images, labels = tf.train.batch([image, label], batch_size=32) - -# Create the model -predictions = vgg.vgg_16(images) - -train_op = slim.learning.create_train_op(...) - -# Specify where the Model, trained on ImageNet, was saved. -model_path = '/path/to/pre_trained_on_imagenet.checkpoint' - -# Specify where the new model will live: -log_dir = '/path/to/my_pascal_model_dir/' - -# Restore only the convolutional layers: -variables_to_restore = slim.get_variables_to_restore(exclude=['fc6', 'fc7', 'fc8']) -init_fn = assign_from_checkpoint_fn(model_path, variables_to_restore) - -# Start training. -slim.learning.train(train_op, log_dir, init_fn=init_fn) -``` - -## Evaluating Models. - -Once we've trained a model (or even while the model is busy training) we'd like -to see how well the model performs in practice. This is accomplished by picking -a set of evaluation metrics, which will grade the model's performance, and the -evaluation code which actually loads the data, performs inference, compares the -results to the ground truth and records the evaluation scores. This step may be -performed once or repeated periodically. - -### Metrics - -We define a metric to be a performance measure that is not a loss function -(losses are directly optimized during training), but which we are still -interested in for the purpose of evaluating our model. -For example, we might want to minimize log loss, but our metrics of interest -might be F1 score (test accuracy), or Intersection Over Union score (which are not -differentiable, and therefore cannot be used as losses). - -TF-Slim provides a set of metric operations that makes evaluating models -easy. Abstractly, computing the value of a metric can be divided into three -parts: - -1. Initialization: initialize the variables used to compute the metrics. -2. Aggregation: perform operations (sums, etc) used to compute the metrics. -3. Finalization: (optionally) perform any final operation to compute metric -values. For example, computing means, mins, maxes, etc. - -For example, to compute `mean_absolute_error`, two variables (`count` and -`total`) are *initialized* to zero. During *aggregation*, we observed -some set of predictions and labels, compute their absolute differences and add -the total to `total`. Each time we observe another value, -`count` is incremented. Finally, during *finalization*, `total` is divided -by `count` to obtain the mean. - -The following example demonstrates the API for declaring metrics. Because -metrics are often evaluated on a test set which is different from the training -set (upon which the loss is computed), we'll assume we're using test data: - -```python -images, labels = LoadTestData(...) -predictions = MyModel(images) - -mae_value_op, mae_update_op = slim.metrics.streaming_mean_absolute_error(predictions, labels) -mre_value_op, mre_update_op = slim.metrics.streaming_mean_relative_error(predictions, labels) -pl_value_op, pl_update_op = slim.metrics.percentage_less(mean_relative_errors, 0.3) -``` - -As the example illustrates, the creation of a metric returns two values: -a *value_op* and an *update_op*. The value_op is an idempotent operation that -returns the current value of the metric. The update_op is an operation that -performs the *aggregation* step mentioned above as well as returning the value -of the metric. - -Keeping track of each `value_op` and `update_op` can be laborious. To deal with -this, TF-Slim provides two convenience functions: - -```python - -# Aggregates the value and update ops in two lists: -value_ops, update_ops = slim.metrics.aggregate_metrics( - slim.metrics.streaming_mean_absolute_error(predictions, labels), - slim.metrics.streaming_mean_squared_error(predictions, labels)) - -# Aggregates the value and update ops in two dictionaries: -names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({ - "eval/mean_absolute_error": slim.metrics.streaming_mean_absolute_error(predictions, labels), - "eval/mean_squared_error": slim.metrics.streaming_mean_squared_error(predictions, labels), -}) - -``` - -### Working example: Tracking Multiple Metrics - -Putting it all together: - -```python -import tensorflow as tf -import tensorflow.contrib.slim.nets as nets - -slim = tf.contrib.slim -vgg = nets.vgg - - -# Load the data -images, labels = load_data(...) - -# Define the network -predictions = vgg.vgg_16(images) - -# Choose the metrics to compute: -names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({ - "eval/mean_absolute_error": slim.metrics.streaming_mean_absolute_error(predictions, labels), - "eval/mean_squared_error": slim.metrics.streaming_mean_squared_error(predictions, labels), -}) - -# Evaluate the model using 1000 batches of data: -num_batches = 1000 - -with tf.Session() as sess: - sess.run(tf.global_variables_initializer()) - sess.run(tf.local_variables_initializer()) - - for batch_id in range(num_batches): - sess.run(names_to_updates.values()) - - metric_values = sess.run(names_to_values.values()) - for metric, value in zip(names_to_values.keys(), metric_values): - print('Metric %s has value: %f' % (metric, value)) -``` - -Note that -[metric_ops.py](https://www.tensorflow.org/code/tensorflow/contrib/metrics/python/ops/metric_ops.py) -can be used in isolation without using either -[layers.py](https://www.tensorflow.org/code/tensorflow/contrib/layers/python/layers/layers.py) -or -[loss_ops.py](https://www.tensorflow.org/code/tensorflow/contrib/losses/python/losses/loss_ops.py) - -### Evaluation Loop - -TF-Slim provides an evaluation module -([evaluation.py](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/evaluation.py)), -which contains helper functions for writing model evaluation scripts using -metrics from -the [metric_ops.py](https://www.tensorflow.org/code/tensorflow/contrib/metrics/python/ops/metric_ops.py) -module. These include a function for periodically running evaluations, -evaluating -metrics over batches of data and printing and summarizing metric results. For -example: - -```python -import tensorflow as tf - -slim = tf.contrib.slim - -# Load the data -images, labels = load_data(...) - -# Define the network -predictions = MyModel(images) - -# Choose the metrics to compute: -names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({ - 'accuracy': slim.metrics.accuracy(predictions, labels), - 'precision': slim.metrics.precision(predictions, labels), - 'recall': slim.metrics.recall(mean_relative_errors, 0.3), -}) - -# Create the summary ops such that they also print out to std output: -summary_ops = [] -for metric_name, metric_value in names_to_values.iteritems(): - op = tf.summary.scalar(metric_name, metric_value) - op = tf.Print(op, [metric_value], metric_name) - summary_ops.append(op) - -num_examples = 10000 -batch_size = 32 -num_batches = math.ceil(num_examples / float(batch_size)) - -# Setup the global step. -slim.get_or_create_global_step() - -output_dir = ... # Where the summaries are stored. -eval_interval_secs = ... # How often to run the evaluation. -slim.evaluation.evaluation_loop( - 'local', - checkpoint_dir, - log_dir, - num_evals=num_batches, - eval_op=names_to_updates.values(), - summary_op=tf.summary.merge(summary_ops), - eval_interval_secs=eval_interval_secs) -``` - -## Authors -Sergio Guadarrama and Nathan Silberman - -## Citation -"TensorFlow-Slim: a lightweight library for defining, training and evaluating complex models in TensorFlow" -S. Guadarrama, N. Silberman, 2016. -https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/slim diff --git a/tensorflow/contrib/slim/__init__.py b/tensorflow/contrib/slim/__init__.py deleted file mode 100644 index 22998b11265..00000000000 --- a/tensorflow/contrib/slim/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Slim is an interface to contrib functions, examples and models. - -TODO(nsilberman): flesh out documentation. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,line-too-long,g-importing-member,wildcard-import -# TODO(jart): Delete non-slim imports -from tensorflow.contrib import losses -from tensorflow.contrib import metrics -from tensorflow.contrib.framework.python.ops.arg_scope import * -from tensorflow.contrib.framework.python.ops.variables import * -from tensorflow.contrib.layers.python.layers import * -from tensorflow.contrib.layers.python.layers.initializers import * -from tensorflow.contrib.layers.python.layers.regularizers import * -from tensorflow.contrib.slim.python.slim import evaluation -from tensorflow.contrib.slim.python.slim import learning -from tensorflow.contrib.slim.python.slim import model_analyzer -from tensorflow.contrib.slim.python.slim import queues -from tensorflow.contrib.slim.python.slim import summaries -from tensorflow.contrib.slim.python.slim.data import data_decoder -from tensorflow.contrib.slim.python.slim.data import data_provider -from tensorflow.contrib.slim.python.slim.data import dataset -from tensorflow.contrib.slim.python.slim.data import dataset_data_provider -from tensorflow.contrib.slim.python.slim.data import parallel_reader -from tensorflow.contrib.slim.python.slim.data import prefetch_queue -from tensorflow.contrib.slim.python.slim.data import tfexample_decoder -from tensorflow.python.util.all_util import make_all -# pylint: enable=unused-import,line-too-long,g-importing-member,wildcard-import - -__all__ = make_all(__name__) diff --git a/tensorflow/contrib/slim/nets.py b/tensorflow/contrib/slim/nets.py deleted file mode 100644 index 3b329a880ab..00000000000 --- a/tensorflow/contrib/slim/nets.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""TF-Slim Nets. - -## Standard Networks. -@@alexnet_v2 -@@inception_v1 -@@inception_v1_base -@@inception_v2 -@@inception_v2_base -@@inception_v3 -@@inception_v3_base -@@overfeat -@@vgg_a -@@vgg_16 -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import, -# Collapse nets into a single namespace. -from tensorflow.contrib.slim.python.slim.nets import alexnet -from tensorflow.contrib.slim.python.slim.nets import inception -from tensorflow.contrib.slim.python.slim.nets import overfeat -from tensorflow.contrib.slim.python.slim.nets import resnet_utils -from tensorflow.contrib.slim.python.slim.nets import resnet_v1 -from tensorflow.contrib.slim.python.slim.nets import resnet_v2 -from tensorflow.contrib.slim.python.slim.nets import vgg -from tensorflow.python.util.all_util import make_all -# pylint: enable=unused-import - -__all__ = make_all(__name__) diff --git a/tensorflow/contrib/slim/python/slim/data/BUILD b/tensorflow/contrib/slim/python/slim/data/BUILD deleted file mode 100644 index ebcf82ac586..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/BUILD +++ /dev/null @@ -1,201 +0,0 @@ -# Description: -# Contains packages used for creating and loading datasets. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "data", - deps = [ - ":dataset", - ":dataset_data_provider", - ":parallel_reader", - ":prefetch_queue", - ":tfexample_decoder", - ], -) - -# Transitive dependencies of this target will be included in the pip package. -py_library( - name = "data_pip", - deps = [ - ":data", - ":test_utils", - ], -) - -py_library( - name = "data_decoder", - srcs = ["data_decoder.py"], - srcs_version = "PY2AND3", -) - -py_library( - name = "data_provider", - srcs = ["data_provider.py"], - srcs_version = "PY2AND3", -) - -py_library( - name = "dataset", - srcs = ["dataset.py"], - srcs_version = "PY2AND3", -) - -py_library( - name = "dataset_data_provider", - srcs = ["dataset_data_provider.py"], - srcs_version = "PY2AND3", - deps = [ - ":data_provider", - ":parallel_reader", - ], -) - -py_test( - name = "dataset_data_provider_test", - srcs = ["dataset_data_provider_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_windows"], # TODO: needs investigation on Windows - deps = [ - ":dataset", - ":dataset_data_provider", - ":test_utils", - ":tfexample_decoder", - "//tensorflow/contrib/slim:queues", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:image_ops", - "//tensorflow/python:io_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:platform", - "//tensorflow/python:session", - ], -) - -py_library( - name = "parallel_reader", - srcs = ["parallel_reader.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:io_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:summary", - "//tensorflow/python:training", - ], -) - -py_test( - name = "parallel_reader_test", - size = "small", - srcs = ["parallel_reader_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":parallel_reader", - ":test_utils", - "//tensorflow/contrib/slim:queues", - "//tensorflow/python:client_testlib", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:io_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], -) - -py_library( - name = "prefetch_queue", - srcs = ["prefetch_queue.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:summary", - "//tensorflow/python:training", - ], -) - -py_test( - name = "prefetch_queue_test", - size = "small", - srcs = ["prefetch_queue_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":prefetch_queue", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:random_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_library( - name = "test_utils", - srcs = ["test_utils.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/core:protos_all_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:image_ops", - "//tensorflow/python:lib", - "//third_party/py/numpy", - ], -) - -py_library( - name = "tfexample_decoder", - srcs = ["tfexample_decoder.py"], - srcs_version = "PY2AND3", - deps = [ - ":data_decoder", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:functional_ops", - "//tensorflow/python:image_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:sparse_tensor", - ], -) - -py_test( - name = "tfexample_decoder_test", - srcs = ["tfexample_decoder_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":tfexample_decoder", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:image_ops", - "//tensorflow/python:lookup_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:parsing_ops", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/slim/python/slim/data/README.md b/tensorflow/contrib/slim/python/slim/data/README.md deleted file mode 100644 index fe15a10b99d..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/README.md +++ /dev/null @@ -1,153 +0,0 @@ -# TensorFlow-Slim Data - -TF-Slim provides a data loading library for facilitating the reading of data -from various formats. TF-Slim's data modules are composed of several layers of -abstraction to make it flexible enough to support multiple file storage types, -such as TFRecords or Text files, data encoding and features naming schemes. - -# Overview - -The task of loading data has two main components: (1) specification of how -a dataset is represented so it can be read and interpreted and (2) instruction -for providing the data to consumers of the dataset. - -Secondly, one must specify instructions for how -the data is actually provided and housed in memory. For example, if the data is -sharded over many sources, should it be read in parallel from these sources? -Should it be read serially? Should the data be shuffled in memory? - -# Dataset Specification - -TF-Slim defines a dataset to be a set of files (that may or may not be encoded) -representing a finite set of samples, and which can be read to provide a -predefined set of entities or `items`. For example, a dataset might be stored -over thousands of files or a single file. The files might store the data in -clear text or some advanced encoding scheme. It might provide a single `item`, -like an image, or several `items`, like an image, a class label and a scene -label. - -More concretely, TF-Slim's -[dataset](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/data/dataset.py) -is a tuple that encapsulates the following elements of a dataset specification: - -* `data_sources`: A list of file paths that together make up the dataset -* `reader`: A TensorFlow -[Reader](https://www.tensorflow.org/api_docs/python/io_ops.html#ReaderBase) -appropriate for the file type in `data_sources`. -* `decoder`: A TF-Slim -[data_decoder](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/data/data_decoder.py) -class which is used to decode the content of the read dataset files. -* `num_samples`: The number of samples in the dataset. -* `items_to_descriptions`: A map from the items provided by the dataset to -descriptions of each. - -In a nutshell, a dataset is read by (a) opening the files specified by -`data_sources` using the given `reader` class (b) decoding the files using -the given `decoder` and (c) allowing the user to request a list of `items` to -be returned as `Tensors`. - -## Data Decoders - -A -[data_decoder](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/data/data_decoder.py) -is a class which is given some (possibly serialized/encoded) data and returns a -list of `Tensors`. In particular, a given data decoder is able to decode a -predefined list of `items` and can return a subset or all of them, when -requested: - -```python -# Load the data -my_encoded_data = ... -data_decoder = MyDataDecoder() - -# Decode the inputs and labels: -decoded_input, decoded_labels = data_decoder.Decode(data, ['input', 'labels']) - -# Decode just the inputs: -decoded_input = data_decoder.Decode(data, ['input']) - -# Check which items a data decoder knows how to decode: -for item in data_decoder.list_items(): - print(item) -``` - -## Example: TFExampleDecoder - -The -[tfexample_decoder.py](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py) -is a data decoder which decodes serialized `TFExample` protocol buffers. A -`TFExample` protocol buffer is a map from keys (strings) to either a -`tf.FixedLenFeature` or `tf.VarLenFeature`. Consequently, to decode a -`TFExample`, one must provide a mapping from one or more `TFExample` fields -to each of the `items` that the `tfexample_decoder` can provide. For -example, a dataset of `TFExamples` might store images in various formats and -each `TFExample` might contain an `encoding` key and a `format` key which can -be used to decode the image using the appropriate decoder (jpg, png, etc). - -To make this possible, the `tfexample_decoder` is constructed by specifying -the a map of `TFExample` keys to either `tf.FixedLenFeature` or -`tf.VarLenFeature` as well as a set of `ItemHandlers`. An `ItemHandler` -provides a mapping from `TFExample` keys to the item being provided. Because a -`tfexample_decoder` might return multiple `items`, one often constructs a -`tfexample_decoder` using multiple `ItemHandlers`. - -`tfexample_decoder` provides some predefined `ItemHandlers` which take care -of the common cases of mapping `TFExamples` to images, `Tensors` and -`SparseTensors`. For example, the following specification might be -used to decode a dataset of images: - -```python -keys_to_features = { - 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), - 'image/format': tf.FixedLenFeature((), tf.string, default_value='raw'), - 'image/class/label': tf.FixedLenFeature( - [1], tf.int64, default_value=tf.zeros([1], dtype=tf.int64)), -} - -items_to_handlers = { - 'image': tfexample_decoder.Image( - image_key = 'image/encoded', - format_key = 'image/format', - shape=[28, 28], - channels=1), - 'label': tfexample_decoder.Tensor('image/class/label'), -} - -decoder = tfexample_decoder.TFExampleDecoder( - keys_to_features, items_to_handlers) -``` - -Notice that the TFExample is parsed using three keys: `image/encoded`, -`image/format` and `image/class/label`. Additionally, the first two keys are -mapped to a single `item` named 'image'. As defined, this `data_decoder` -provides two `items` named 'image' and 'label'. - -# Data Provision - -A -[data_provider](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/data/data_provider.py) -is a class which provides `Tensors` for each item requested: - -```python -my_data_provider = ... -image, class_label, bounding_box = my_data_provider.get( - ['image', 'label', 'bb']) -``` - -The -[dataset_data_provider](https://www.tensorflow.org/code/tensorflow/contrib/slim/python/slim/data/dataset_data_provider.py) -is a `data_provider` that provides data from a given `dataset` specification: - -```python -dataset = GetDataset(...) -data_provider = dataset_data_provider.DatasetDataProvider( - dataset, common_queue_capacity=32, common_queue_min=8) -``` - -The `dataset_data_provider` enables control over several elements of data -provision: - -* How many concurrent readers are used. -* Whether the data is shuffled as its loaded into its queue -* Whether to take a single pass over the data or read data indefinitely. - diff --git a/tensorflow/contrib/slim/python/slim/data/data_decoder.py b/tensorflow/contrib/slim/python/slim/data/data_decoder.py deleted file mode 100644 index 46d33597e42..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/data_decoder.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains helper functions and classes necessary for decoding data. - -While data providers read data from disk, sstables or other formats, data -decoders decode the data (if necessary). A data decoder is provided with a -serialized or encoded piece of data as well as a list of items and -returns a set of tensors, each of which correspond to the requested list of -items extracted from the data: - - def Decode(self, data, items): - ... - -For example, if data is a compressed map, the implementation might be: - - def Decode(self, data, items): - decompressed_map = _Decompress(data) - outputs = [] - for item in items: - outputs.append(decompressed_map[item]) - return outputs. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - - -@six.add_metaclass(abc.ABCMeta) -class DataDecoder(object): - """An abstract class which is used to decode data for a provider.""" - - @abc.abstractmethod - def decode(self, data, items): - """Decodes the data to returns the tensors specified by the list of items. - - Args: - data: A possibly encoded data format. - items: A list of strings, each of which indicate a particular data type. - - Returns: - A list of `Tensors`, whose length matches the length of `items`, where - each `Tensor` corresponds to each item. - - Raises: - ValueError: If any of the items cannot be satisfied. - """ - pass - - @abc.abstractmethod - def list_items(self): - """Lists the names of the items that the decoder can decode. - - Returns: - A list of string names. - """ - pass diff --git a/tensorflow/contrib/slim/python/slim/data/data_provider.py b/tensorflow/contrib/slim/python/slim/data/data_provider.py deleted file mode 100644 index 3252b4fe847..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/data_provider.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains code for the DataProvider. - -A DataProvider is a class which provides some predefined data types from some -source (TFRecord, etc). The most basic function of a -data provider is the `Get` operation where one requests one or more types of -data, or 'items': - - provider.get(items=['image', 'sentence', 'class']) - -More concretely, a data provider (a subclass of BaseDataProvider) returns a -single tensor for each requested item (data type): - - provider = MyDataProvider(...) - image, sentence, clazz = provider.get(['image', 'sentence', 'class']) - -In this example, the provider `MyDataProvider` must know how to load each item. -A data provider may be written in a way that the logic necessary to map from -each item to tensor is completely encapsulated within the data_provider itself. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - - -@six.add_metaclass(abc.ABCMeta) -class DataProvider(object): - """Maps a list of requested data items to tensors from a data source. - - All data providers must inherit from DataProvider and implement the Get - method which returns arbitrary types of data. No assumption is made about the - source of the data nor the mechanism for providing it. - """ - - def __init__(self, items_to_tensors, num_samples): - """Constructs the Data Provider. - - Args: - items_to_tensors: a dictionary of names to tensors. - num_samples: the number of samples in the dataset being provided. - """ - self._items_to_tensors = items_to_tensors - self._num_samples = num_samples - - def get(self, items): - """Returns a list of tensors specified by the given list of items. - - The list of items is arbitrary different data providers satisfy different - lists of items. For example the Pascal VOC might accept items 'image' and - 'semantics', whereas the NYUDepthV2 data provider might accept items - 'image', 'depths' and 'normals'. - - Args: - items: a list of strings, each of which indicate a particular data type. - - Returns: - a list of tensors, whose length matches the length of `items`, where each - tensor corresponds to each item. - - Raises: - ValueError: if any of the items cannot be satisfied. - """ - self._validate_items(items) - return [self._items_to_tensors[item] for item in items] - - def list_items(self): - """Returns the list of item names that can be provided by the data provider. - - Returns: - a list of item names that can be passed to Get([items]). - """ - return self._items_to_tensors.keys() - - def num_samples(self): - """Returns the number of data samples in the dataset. - - Returns: - a positive whole number. - """ - return self._num_samples - - def _validate_items(self, items): - """Verifies that each given item is a member of the list from ListItems(). - - Args: - items: a list or tuple of strings. - - Raises: - ValueError: if `items` is not a tuple or list or if any of the elements of - `items` is not found in the list provided by self.ListItems(). - """ - if not isinstance(items, (list, tuple)): - raise ValueError('items must be a list or tuple') - - valid_items = self.list_items() - for item in items: - if item not in valid_items: - raise ValueError('Item [%s] is invalid. Valid entries include: %s' % - (item, valid_items)) diff --git a/tensorflow/contrib/slim/python/slim/data/dataset.py b/tensorflow/contrib/slim/python/slim/data/dataset.py deleted file mode 100644 index 110a74d4f2b..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/dataset.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains the definition of a Dataset. - -A Dataset is a collection of several components: (1) a list of data sources -(2) a Reader class that can read those sources and returns possibly encoded -samples of data (3) a decoder that decodes each sample of data provided by the -reader (4) the total number of samples and (5) an optional dictionary mapping -the list of items returns to a description of those items. - -Data can be loaded from a dataset specification using a dataset_data_provider: - - dataset = CreateMyDataset(...) - provider = dataset_data_provider.DatasetDataProvider( - dataset, shuffle=False) - image, label = provider.get(['image', 'label']) - -See slim.data.dataset_data_provider for additional examples. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -class Dataset(object): - """Represents a Dataset specification.""" - - def __init__(self, data_sources, reader, decoder, num_samples, - items_to_descriptions, **kwargs): - """Initializes the dataset. - - Args: - data_sources: A list of files that make up the dataset. - reader: The reader class, a subclass of BaseReader such as TextLineReader - or TFRecordReader. - decoder: An instance of a data_decoder. - num_samples: The number of samples in the dataset. - items_to_descriptions: A map from the items that the dataset provides to - the descriptions of those items. - **kwargs: Any remaining dataset-specific fields. - """ - kwargs['data_sources'] = data_sources - kwargs['reader'] = reader - kwargs['decoder'] = decoder - kwargs['num_samples'] = num_samples - kwargs['items_to_descriptions'] = items_to_descriptions - self.__dict__.update(kwargs) diff --git a/tensorflow/contrib/slim/python/slim/data/dataset_data_provider.py b/tensorflow/contrib/slim/python/slim/data/dataset_data_provider.py deleted file mode 100644 index c42c7b3391d..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/dataset_data_provider.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A DataProvider that provides data from a Dataset. - -DatasetDataProviders provide data from datasets. The provide can be configured -to use multiple readers simultaneously or read via a single reader. -Additionally, the data being read can be optionally shuffled. - -For example, to read data using a single thread without shuffling: - - pascal_voc_data_provider = DatasetDataProvider( - slim.datasets.pascal_voc.get_split('train'), - shuffle=False) - images, labels = pascal_voc_data_provider.get(['images', 'labels']) - -To read data using multiple readers simultaneous with shuffling: - - pascal_voc_data_provider = DatasetDataProvider( - slim.datasets.pascal_voc.Dataset(), - num_readers=10, - shuffle=True) - images, labels = pascal_voc_data_provider.get(['images', 'labels']) - -Equivalently, one may request different fields of the same sample separately: - - [images] = pascal_voc_data_provider.get(['images']) - [labels] = pascal_voc_data_provider.get(['labels']) - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.slim.python.slim.data import data_provider -from tensorflow.contrib.slim.python.slim.data import parallel_reader - - -class DatasetDataProvider(data_provider.DataProvider): - - def __init__(self, - dataset, - num_readers=1, - reader_kwargs=None, - shuffle=True, - num_epochs=None, - common_queue_capacity=256, - common_queue_min=128, - record_key='record_key', - seed=None, - scope=None): - """Creates a DatasetDataProvider. - Note: if `num_epochs` is not `None`, local counter `epochs` will be created - by relevant function. Use `local_variables_initializer()` to initialize - local variables. - Args: - dataset: An instance of the Dataset class. - num_readers: The number of parallel readers to use. - reader_kwargs: An optional dict of kwargs for the reader. - shuffle: Whether to shuffle the data sources and common queue when - reading. - num_epochs: The number of times each data source is read. If left as None, - the data will be cycled through indefinitely. - common_queue_capacity: The capacity of the common queue. - common_queue_min: The minimum number of elements in the common queue after - a dequeue. - record_key: The item name to use for the dataset record keys in the - provided tensors. - seed: The seed to use if shuffling. - scope: Optional name scope for the ops. - Raises: - ValueError: If `record_key` matches one of the items in the dataset. - """ - key, data = parallel_reader.parallel_read( - dataset.data_sources, - reader_class=dataset.reader, - num_epochs=num_epochs, - num_readers=num_readers, - reader_kwargs=reader_kwargs, - shuffle=shuffle, - capacity=common_queue_capacity, - min_after_dequeue=common_queue_min, - seed=seed, - scope=scope) - - items = dataset.decoder.list_items() - tensors = dataset.decoder.decode(data, items) - - items_to_tensors = dict(zip(items, tensors)) - if record_key in items_to_tensors: - raise ValueError('The item name used for `record_key` cannot also be ' - 'used for a dataset item: %s', record_key) - items_to_tensors[record_key] = key - - super(DatasetDataProvider, self).__init__( - items_to_tensors=items_to_tensors, - num_samples=dataset.num_samples) diff --git a/tensorflow/contrib/slim/python/slim/data/dataset_data_provider_test.py b/tensorflow/contrib/slim/python/slim/data/dataset_data_provider_test.py deleted file mode 100644 index 795de6a4088..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/dataset_data_provider_test.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.data.dataset_data_provider.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import tempfile - -from tensorflow.contrib.slim.python.slim import queues -from tensorflow.contrib.slim.python.slim.data import dataset -from tensorflow.contrib.slim.python.slim.data import dataset_data_provider -from tensorflow.contrib.slim.python.slim.data import test_utils -from tensorflow.contrib.slim.python.slim.data import tfexample_decoder -from tensorflow.python.client import session -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import image_ops -from tensorflow.python.ops import io_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test - - -def _resize_image(image, height, width): - image = array_ops.expand_dims(image, 0) - image = image_ops.resize_bilinear(image, [height, width]) - return array_ops.squeeze(image, [0]) - - -def _create_tfrecord_dataset(tmpdir): - if not gfile.Exists(tmpdir): - gfile.MakeDirs(tmpdir) - - data_sources = test_utils.create_tfrecord_files(tmpdir, num_files=1) - - keys_to_features = { - 'image/encoded': - parsing_ops.FixedLenFeature( - shape=(), dtype=dtypes.string, default_value=''), - 'image/format': - parsing_ops.FixedLenFeature( - shape=(), dtype=dtypes.string, default_value='jpeg'), - 'image/class/label': - parsing_ops.FixedLenFeature( - shape=[1], - dtype=dtypes.int64, - default_value=array_ops.zeros( - [1], dtype=dtypes.int64)) - } - - items_to_handlers = { - 'image': tfexample_decoder.Image(), - 'label': tfexample_decoder.Tensor('image/class/label'), - } - - decoder = tfexample_decoder.TFExampleDecoder(keys_to_features, - items_to_handlers) - - return dataset.Dataset( - data_sources=data_sources, - reader=io_ops.TFRecordReader, - decoder=decoder, - num_samples=100, - items_to_descriptions=None) - - -class DatasetDataProviderTest(test.TestCase): - - def testTFRecordDataset(self): - dataset_dir = tempfile.mkdtemp(prefix=os.path.join(self.get_temp_dir(), - 'tfrecord_dataset')) - - height = 300 - width = 280 - - with self.cached_session(): - test_dataset = _create_tfrecord_dataset(dataset_dir) - provider = dataset_data_provider.DatasetDataProvider(test_dataset) - key, image, label = provider.get(['record_key', 'image', 'label']) - image = _resize_image(image, height, width) - - with session.Session('') as sess: - with queues.QueueRunners(sess): - key, image, label = sess.run([key, image, label]) - split_key = key.decode('utf-8').split(':') - self.assertEqual(2, len(split_key)) - self.assertEqual(test_dataset.data_sources[0], split_key[0]) - self.assertTrue(split_key[1].isdigit()) - self.assertListEqual([height, width, 3], list(image.shape)) - self.assertListEqual([1], list(label.shape)) - - def testTFRecordSeparateGetDataset(self): - dataset_dir = tempfile.mkdtemp(prefix=os.path.join(self.get_temp_dir(), - 'tfrecord_separate_get')) - - height = 300 - width = 280 - - with self.cached_session(): - provider = dataset_data_provider.DatasetDataProvider( - _create_tfrecord_dataset(dataset_dir)) - [image] = provider.get(['image']) - [label] = provider.get(['label']) - image = _resize_image(image, height, width) - - with session.Session('') as sess: - with queues.QueueRunners(sess): - image, label = sess.run([image, label]) - self.assertListEqual([height, width, 3], list(image.shape)) - self.assertListEqual([1], list(label.shape)) - - def testConflictingRecordKeyItem(self): - dataset_dir = tempfile.mkdtemp(prefix=os.path.join(self.get_temp_dir(), - 'tfrecord_dataset')) - - with self.cached_session(): - with self.assertRaises(ValueError): - dataset_data_provider.DatasetDataProvider( - _create_tfrecord_dataset(dataset_dir), record_key='image') - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/data/parallel_reader.py b/tensorflow/contrib/slim/python/slim/data/parallel_reader.py deleted file mode 100644 index f3dc03aaa2b..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/parallel_reader.py +++ /dev/null @@ -1,317 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implements a parallel data reader with queues and optional shuffling.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes as tf_dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import io_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import gfile -from tensorflow.python.summary import summary -from tensorflow.python.training import input as tf_input -from tensorflow.python.training import queue_runner - - -class ParallelReader(io_ops.ReaderBase): - """Reader class that uses multiple readers in parallel to improve speed. - - See ReaderBase for supported methods. - """ - - def __init__(self, - reader_class, - common_queue, - num_readers=4, - reader_kwargs=None): - """ParallelReader creates num_readers instances of the reader_class. - - Each instance is created by calling the `reader_class` function passing - the arguments specified in `reader_kwargs` as in: - reader_class(**read_kwargs) - - When you read from a ParallelReader, with its `read()` method, - you just dequeue examples from the `common_queue`. - - The readers will read different files in parallel, asynchronously enqueueing - their output into `common_queue`. The `common_queue.dtypes` must be - [tf.string, tf.string] - - Because each reader can read from a different file, the examples in the - `common_queue` could be from different files. Due to the asynchronous - reading there is no guarantee that all the readers will read the same - number of examples. - - If the `common_queue` is a shuffling queue, then the examples are shuffled. - - Usage: - common_queue = tf.queue.RandomShuffleQueue( - capacity=256, - min_after_dequeue=128, - dtypes=[tf.string, tf.string]) - p_reader = ParallelReader(tf.compat.v1.TFRecordReader, common_queue) - - common_queue = tf.queue.FIFOQueue( - capacity=256, - dtypes=[tf.string, tf.string]) - p_reader = ParallelReader(readers, common_queue, num_readers=2) - - - Args: - reader_class: one of the io_ops.ReaderBase subclasses ex: TFRecordReader - common_queue: a Queue to hold (key, value pairs) with `dtypes` equal to - [tf.string, tf.string]. Must be one of the data_flow_ops.Queues - instances, ex. `tf.queue.FIFOQueue()`, `tf.queue.RandomShuffleQueue()`, - ... - num_readers: a integer, number of instances of reader_class to create. - reader_kwargs: an optional dict of kwargs to create the readers. - - Raises: - TypeError: if `common_queue.dtypes` is not [tf.string, tf.string]. - """ - if len(common_queue.dtypes) != 2: - raise TypeError('common_queue.dtypes must be [tf.string, tf.string]') - for dtype in common_queue.dtypes: - if not dtype.is_compatible_with(tf_dtypes.string): - raise TypeError('common_queue.dtypes must be [tf.string, tf.string]') - - reader_kwargs = reader_kwargs or {} - self._readers = [reader_class(**reader_kwargs) for _ in range(num_readers)] - self._common_queue = common_queue - - @property - def num_readers(self): - return len(self._readers) - - @property - def common_queue(self): - return self._common_queue - - def read(self, queue, name=None): - """Returns the next record (key, value pair) produced by the reader. - - The multiple reader instances are all configured to `read()` from the - filenames listed in `queue` and enqueue their output into the `common_queue` - passed to the constructor, and this method returns the next record dequeued - from that `common_queue`. - - - Readers dequeue a work unit from `queue` if necessary (e.g. when a - reader needs to start reading from a new file since it has finished with - the previous file). - - A queue runner for enqueuing in the `common_queue` is automatically added - to the TF QueueRunners collection. - - Args: - queue: A Queue or a mutable string Tensor representing a handle to a - Queue, with string work items. - name: A name for the operation (optional). - - Returns: - The next record (i.e. (key, value pair)) from the common_queue. - """ - - self._configure_readers_by(queue) - return self._common_queue.dequeue(name=name) - - def read_up_to(self, queue, num_records, name=None): - """Returns up to num_records (key, value pairs) produced by a reader. - - Will dequeue a work unit from queue if necessary (e.g., when the - Reader needs to start reading from a new file since it has - finished with the previous file). - It may return less than num_records even before the last batch. - - **Note** This operation is not supported by all types of `common_queue`s. - If a `common_queue` does not support `dequeue_up_to()`, then a - `tf.errors.UnimplementedError` is raised. - - Args: - queue: A Queue or a mutable string Tensor representing a handle to a - Queue, with string work items. - num_records: Number of records to read. - name: A name for the operation (optional). - - Returns: - A tuple of Tensors (keys, values) from common_queue. - keys: A 1-D string Tensor. - values: A 1-D string Tensor. - """ - self._configure_readers_by(queue) - return self._common_queue.dequeue_up_to(num_records, name) - - def _configure_readers_by(self, queue): - enqueue_ops = [] - for reader in self._readers: - enqueue_ops.append(self._common_queue.enqueue(reader.read(queue))) - - queue_runner.add_queue_runner( - queue_runner.QueueRunner(self._common_queue, enqueue_ops)) - - def num_records_produced(self, name=None): - """Returns the number of records this reader has produced. - - Args: - name: A name for the operation (optional). - - Returns: - An int64 Tensor. - - """ - num_records = [r.num_records_produced() for r in self._readers] - return math_ops.add_n(num_records, name=name) - - def num_work_units_completed(self, name=None): - """Returns the number of work units this reader has finished processing. - - Args: - name: A name for the operation (optional). - - Returns: - An int64 Tensor. - """ - num_work_units = [r.num_work_units_completed() for r in self._readers] - return math_ops.add_n(num_work_units, name=name) - - -def parallel_read(data_sources, - reader_class, - num_epochs=None, - num_readers=4, - reader_kwargs=None, - shuffle=True, - dtypes=None, - capacity=256, - min_after_dequeue=128, - seed=None, - scope=None): - """Reads multiple records in parallel from data_sources using n readers. - - It uses a ParallelReader to read from multiple files in parallel using - multiple readers created using `reader_class` with `reader_kwargs'. - - If shuffle is True the common_queue would be a RandomShuffleQueue otherwise - it would be a FIFOQueue. - - Usage: - data_sources = ['path_to/train*'] - key, value = parallel_read(data_sources, tf.CSVReader, num_readers=4) - - Args: - data_sources: a list/tuple of files or the location of the data, i.e. - /path/to/train@128, /path/to/train* or /tmp/.../train* - reader_class: one of the io_ops.ReaderBase subclasses ex: TFRecordReader - num_epochs: The number of times each data source is read. If left as None, - the data will be cycled through indefinitely. - num_readers: a integer, number of Readers to create. - reader_kwargs: an optional dict, of kwargs for the reader. - shuffle: boolean, whether should shuffle the files and the records by using - RandomShuffleQueue as common_queue. - dtypes: A list of types. The length of dtypes must equal the number of - elements in each record. If it is None it will default to [tf.string, - tf.string] for (key, value). - capacity: integer, capacity of the common_queue. - min_after_dequeue: integer, minimum number of records in the common_queue - after dequeue. Needed for a good shuffle. - seed: A seed for RandomShuffleQueue. - scope: Optional name scope for the ops. - - Returns: - key, value: a tuple of keys and values from the data_source. - """ - data_files = get_data_files(data_sources) - with ops.name_scope(scope, 'parallel_read'): - filename_queue = tf_input.string_input_producer( - data_files, - num_epochs=num_epochs, - shuffle=shuffle, - seed=seed, - name='filenames') - dtypes = dtypes or [tf_dtypes.string, tf_dtypes.string] - if shuffle: - common_queue = data_flow_ops.RandomShuffleQueue( - capacity=capacity, - min_after_dequeue=min_after_dequeue, - dtypes=dtypes, - seed=seed, - name='common_queue') - else: - common_queue = data_flow_ops.FIFOQueue( - capacity=capacity, dtypes=dtypes, name='common_queue') - - summary.scalar( - 'fraction_of_%d_full' % capacity, - math_ops.cast(common_queue.size(), tf_dtypes.float32) * (1. / capacity)) - - return ParallelReader( - reader_class, - common_queue, - num_readers=num_readers, - reader_kwargs=reader_kwargs).read(filename_queue) - - -def single_pass_read(data_sources, reader_class, reader_kwargs=None, - scope=None): - """Reads sequentially the data_sources using the reader, doing a single pass. - - Args: - data_sources: a list/tuple of files or the location of the data, i.e. - /path/to/train@128, /path/to/train* or /tmp/.../train* - reader_class: one of the io_ops.ReaderBase subclasses ex: TFRecordReader. - reader_kwargs: an optional dict, of kwargs for the reader. - scope: Optional name scope for the ops. - - Returns: - key, value: a tuple of keys and values from the data_source. - """ - data_files = get_data_files(data_sources) - with ops.name_scope(scope, 'single_pass_read'): - filename_queue = tf_input.string_input_producer( - data_files, num_epochs=1, shuffle=False, capacity=1, name='filenames') - reader_kwargs = reader_kwargs or {} - return reader_class(**reader_kwargs).read(filename_queue) - - -def get_data_files(data_sources): - """Get data_files from data_sources. - - Args: - data_sources: a list/tuple of files or the location of the data, i.e. - /path/to/train@128, /path/to/train* or /tmp/.../train* - - Returns: - a list of data_files. - - Raises: - ValueError: if data files are not found - - """ - if isinstance(data_sources, (list, tuple)): - data_files = [] - for source in data_sources: - data_files += get_data_files(source) - else: - if '*' in data_sources or '?' in data_sources or '[' in data_sources: - data_files = gfile.Glob(data_sources) - else: - data_files = [data_sources] - if not data_files: - raise ValueError('No data files found in %s' % (data_sources,)) - return data_files diff --git a/tensorflow/contrib/slim/python/slim/data/parallel_reader_test.py b/tensorflow/contrib/slim/python/slim/data/parallel_reader_test.py deleted file mode 100644 index dec5cbc6d22..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/parallel_reader_test.py +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.data.parallel_reader.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.slim.python.slim import queues -from tensorflow.contrib.slim.python.slim.data import parallel_reader -from tensorflow.contrib.slim.python.slim.data import test_utils -from tensorflow.python.framework import dtypes as dtypes_lib -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import io_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import input as input_lib -from tensorflow.python.training import supervisor - - -class ParallelReaderTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def _verify_all_data_sources_read(self, shared_queue): - with self.cached_session(): - tfrecord_paths = test_utils.create_tfrecord_files( - self.get_temp_dir(), num_files=3) - - num_readers = len(tfrecord_paths) - p_reader = parallel_reader.ParallelReader( - io_ops.TFRecordReader, shared_queue, num_readers=num_readers) - - data_files = parallel_reader.get_data_files(tfrecord_paths) - filename_queue = input_lib.string_input_producer(data_files) - key, value = p_reader.read(filename_queue) - - count0 = 0 - count1 = 0 - count2 = 0 - - num_reads = 50 - - sv = supervisor.Supervisor(logdir=self.get_temp_dir()) - with sv.prepare_or_wait_for_session() as sess: - sv.start_queue_runners(sess) - - for _ in range(num_reads): - current_key, _ = sess.run([key, value]) - if '0-of-3' in str(current_key): - count0 += 1 - if '1-of-3' in str(current_key): - count1 += 1 - if '2-of-3' in str(current_key): - count2 += 1 - - self.assertGreater(count0, 0) - self.assertGreater(count1, 0) - self.assertGreater(count2, 0) - self.assertEquals(count0 + count1 + count2, num_reads) - - def _verify_read_up_to_out(self, shared_queue): - with self.cached_session(): - num_files = 3 - num_records_per_file = 7 - tfrecord_paths = test_utils.create_tfrecord_files( - self.get_temp_dir(), - num_files=num_files, - num_records_per_file=num_records_per_file) - - p_reader = parallel_reader.ParallelReader( - io_ops.TFRecordReader, shared_queue, num_readers=5) - - data_files = parallel_reader.get_data_files(tfrecord_paths) - filename_queue = input_lib.string_input_producer(data_files, num_epochs=1) - key, value = p_reader.read_up_to(filename_queue, 4) - - count0 = 0 - count1 = 0 - count2 = 0 - all_keys_count = 0 - all_values_count = 0 - - sv = supervisor.Supervisor(logdir=self.get_temp_dir()) - with sv.prepare_or_wait_for_session() as sess: - sv.start_queue_runners(sess) - while True: - try: - current_keys, current_values = sess.run([key, value]) - self.assertEquals(len(current_keys), len(current_values)) - all_keys_count += len(current_keys) - all_values_count += len(current_values) - for current_key in current_keys: - if '0-of-3' in str(current_key): - count0 += 1 - if '1-of-3' in str(current_key): - count1 += 1 - if '2-of-3' in str(current_key): - count2 += 1 - except errors_impl.OutOfRangeError: - break - - self.assertEquals(count0, num_records_per_file) - self.assertEquals(count1, num_records_per_file) - self.assertEquals(count2, num_records_per_file) - self.assertEquals( - all_keys_count, - num_files * num_records_per_file) - self.assertEquals(all_values_count, all_keys_count) - self.assertEquals( - count0 + count1 + count2, - all_keys_count) - - def testRandomShuffleQueue(self): - shared_queue = data_flow_ops.RandomShuffleQueue( - capacity=256, - min_after_dequeue=128, - dtypes=[dtypes_lib.string, dtypes_lib.string]) - self._verify_all_data_sources_read(shared_queue) - - def testFIFOSharedQueue(self): - shared_queue = data_flow_ops.FIFOQueue( - capacity=256, dtypes=[dtypes_lib.string, dtypes_lib.string]) - self._verify_all_data_sources_read(shared_queue) - - def testReadUpToFromRandomShuffleQueue(self): - shared_queue = data_flow_ops.RandomShuffleQueue( - capacity=55, - min_after_dequeue=28, - dtypes=[dtypes_lib.string, dtypes_lib.string], - shapes=[tensor_shape.TensorShape([]), - tensor_shape.TensorShape([])]) - self._verify_read_up_to_out(shared_queue) - - def testReadUpToFromFIFOQueue(self): - shared_queue = data_flow_ops.FIFOQueue( - capacity=99, - dtypes=[dtypes_lib.string, dtypes_lib.string], - shapes=[tensor_shape.TensorShape([]), - tensor_shape.TensorShape([])]) - self._verify_read_up_to_out(shared_queue) - - -class ParallelReadTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testTFRecordReader(self): - with self.cached_session(): - self._tfrecord_paths = test_utils.create_tfrecord_files( - self.get_temp_dir(), num_files=3) - - key, value = parallel_reader.parallel_read( - self._tfrecord_paths, reader_class=io_ops.TFRecordReader, num_readers=3) - - sv = supervisor.Supervisor(logdir=self.get_temp_dir()) - with sv.prepare_or_wait_for_session() as sess: - sv.start_queue_runners(sess) - - flowers = 0 - num_reads = 100 - for _ in range(num_reads): - current_key, _ = sess.run([key, value]) - if 'flowers' in str(current_key): - flowers += 1 - self.assertGreater(flowers, 0) - self.assertEquals(flowers, num_reads) - - -class SinglePassReadTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - def testOutOfRangeError(self): - with self.cached_session(): - [tfrecord_path] = test_utils.create_tfrecord_files( - self.get_temp_dir(), num_files=1) - - key, value = parallel_reader.single_pass_read( - tfrecord_path, reader_class=io_ops.TFRecordReader) - init_op = variables.local_variables_initializer() - - with self.cached_session() as sess: - sess.run(init_op) - with queues.QueueRunners(sess): - num_reads = 11 - with self.assertRaises(errors_impl.OutOfRangeError): - for _ in range(num_reads): - sess.run([key, value]) - - def testTFRecordReader(self): - with self.cached_session(): - [tfrecord_path] = test_utils.create_tfrecord_files( - self.get_temp_dir(), num_files=1) - - key, value = parallel_reader.single_pass_read( - tfrecord_path, reader_class=io_ops.TFRecordReader) - init_op = variables.local_variables_initializer() - - with self.cached_session() as sess: - sess.run(init_op) - with queues.QueueRunners(sess): - flowers = 0 - num_reads = 9 - for _ in range(num_reads): - current_key, _ = sess.run([key, value]) - if 'flowers' in str(current_key): - flowers += 1 - self.assertGreater(flowers, 0) - self.assertEquals(flowers, num_reads) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py b/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py deleted file mode 100644 index 7895e809f82..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implements a simple prefetch_queue.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes as _dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.summary import summary -from tensorflow.python.training import queue_runner - - -def _which_queue(dynamic_pad): - return (data_flow_ops.PaddingFIFOQueue - if dynamic_pad else data_flow_ops.FIFOQueue) - - -def prefetch_queue(tensors, - capacity=8, - num_threads=1, - dynamic_pad=False, - shared_name=None, - name=None): - """Creates a queue to prefetch tensors from `tensors`. - - A queue runner for enqueuing tensors into the prefetch_queue is automatically - added to the TF QueueRunners collection. - - Example: - This is for example useful to pre-assemble input batches read with - `tf.compat.v1.train.batch()` and enqueue the pre-assembled batches. Ops that - dequeue - from the pre-assembled queue will not pay the cost of assembling the batch. - - images, labels = tf.compat.v1.train.batch([image, label], batch_size=32, - num_threads=4) - batch_queue = prefetch_queue([images, labels]) - images, labels = batch_queue.dequeue() - logits = Net(images) - loss = Loss(logits, labels) - - Args: - tensors: A list or dictionary of `Tensors` to enqueue in the buffer. - capacity: An integer. The maximum number of elements in the queue. - num_threads: An integer. Number of threads running the enqueue op. - dynamic_pad: Boolean. Whether to allow variable dimensions in input shapes. - shared_name: (optional). If set, this queue will be shared under the given - name across multiple sessions. - name: (Optional) A name for the operations. - - Returns: - A queue from which you can dequeue tensors with the same type and shape - as `tensors`. - """ - if isinstance(tensors, dict): - # Need to wrap the keys and values in list() since Python3 returns views. - # We sort the keys so the order is consistent across runs. - names = list(sorted(tensors.keys())) - tensor_list = list([tensors[n] for n in names]) - else: - names = None - tensor_list = tensors - - with ops.name_scope(name, "prefetch_queue", tensor_list) as name: - dtypes = [t.dtype for t in tensor_list] - shapes = [t.get_shape() for t in tensor_list] - queue = _which_queue(dynamic_pad)( - capacity=capacity, - dtypes=dtypes, - shapes=shapes, - names=names, - shared_name=shared_name) - enqueue_op = queue.enqueue(tensors) - queue_runner.add_queue_runner( - queue_runner.QueueRunner(queue, [enqueue_op] * num_threads)) - summary.scalar( - "fraction_of_%d_full" % capacity, - math_ops.cast(queue.size(), _dtypes.float32) * (1. / capacity)) - return queue diff --git a/tensorflow/contrib/slim/python/slim/data/prefetch_queue_test.py b/tensorflow/contrib/slim/python/slim/data/prefetch_queue_test.py deleted file mode 100644 index 7caa42dcb9f..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/prefetch_queue_test.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.data.prefetch_queue.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.slim.python.slim.data import prefetch_queue -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import input as input_lib -from tensorflow.python.training import queue_runner_impl - - -class PrefetchQueueTest(test.TestCase): - - def testOneThread(self): - with self.cached_session() as sess: - batch_size = 10 - image_size = 32 - num_batches = 5 - - zero64 = constant_op.constant(0, dtype=dtypes.int64) - - examples = variables.Variable(zero64) - counter = examples.count_up_to(num_batches * batch_size) - image = random_ops.random_normal( - [image_size, image_size, 3], dtype=dtypes.float32, name='images') - label = random_ops.random_uniform( - [1], 0, 10, dtype=dtypes.int32, name='labels') - - batches = input_lib.batch( - [counter, image, label], batch_size=batch_size, num_threads=1) - - batches = prefetch_queue.prefetch_queue(batches).dequeue() - - variables.global_variables_initializer().run() - threads = queue_runner_impl.start_queue_runners() - - for i in range(num_batches): - results = sess.run(batches) - self.assertAllEqual(results[0], - np.arange(i * batch_size, (i + 1) * batch_size)) - self.assertEquals(results[1].shape, - (batch_size, image_size, image_size, 3)) - self.assertEquals(results[2].shape, (batch_size, 1)) - - # Reached the limit. - with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batches) - for thread in threads: - thread.join() - - def testMultiThread(self): - with self.cached_session() as sess: - batch_size = 10 - image_size = 32 - num_batches = 5 - - zero64 = constant_op.constant(0, dtype=dtypes.int64) - - examples = variables.Variable(zero64) - counter = examples.count_up_to(num_batches * batch_size) - image = random_ops.random_normal( - [image_size, image_size, 3], dtype=dtypes.float32, name='images') - label = random_ops.random_uniform( - [1], 0, 10, dtype=dtypes.int32, name='labels') - - batches = input_lib.batch( - [counter, image, label], batch_size=batch_size, num_threads=4) - - batches = prefetch_queue.prefetch_queue(batches).dequeue() - - variables.global_variables_initializer().run() - threads = queue_runner_impl.start_queue_runners() - - value_counter = [] - for _ in range(num_batches): - results = sess.run(batches) - value_counter.append(results[0]) - self.assertEqual(results[1].shape, - (batch_size, image_size, image_size, 3)) - self.assertEqual(results[2].shape, (batch_size, 1)) - - self.assertAllEqual( - np.sort(np.concatenate(value_counter)), - np.arange(0, num_batches * batch_size)) - # Reached the limit. - with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batches) - for thread in threads: - thread.join() - - def testMultipleDequeue(self): - with self.cached_session() as sess: - batch_size = 10 - image_size = 32 - num_batches = 4 - - zero64 = constant_op.constant(0, dtype=dtypes.int64) - - examples = variables.Variable(zero64) - counter = examples.count_up_to(num_batches * batch_size) - image = random_ops.random_normal( - [image_size, image_size, 3], dtype=dtypes.float32, name='images') - label = random_ops.random_uniform( - [1], 0, 10, dtype=dtypes.int32, name='labels') - - batches = input_lib.batch( - [counter, image, label], batch_size=batch_size, num_threads=4) - - batcher = prefetch_queue.prefetch_queue(batches) - batches_list = [batcher.dequeue() for _ in range(2)] - - variables.global_variables_initializer().run() - threads = queue_runner_impl.start_queue_runners() - - value_counter = [] - for _ in range(int(num_batches / 2)): - for batches in batches_list: - results = sess.run(batches) - value_counter.append(results[0]) - self.assertEquals(results[1].shape, - (batch_size, image_size, image_size, 3)) - self.assertEquals(results[2].shape, (batch_size, 1)) - - self.assertAllEqual( - np.sort(np.concatenate(value_counter)), - np.arange(0, num_batches * batch_size)) - # Reached the limit. - with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batches) - for thread in threads: - thread.join() - - def testDynamicPad_failure(self): - with ops.Graph().as_default(): - variable_tensor = array_ops.placeholder(dtypes.int32, shape=[None, 3]) - with self.assertRaisesRegexp(ValueError, 'shapes must be fully defined'): - prefetch_queue.prefetch_queue([variable_tensor]) - - def testDynamicPad(self): - with self.cached_session() as sess: - # Create 3 tensors of variable but compatible shapes. - var_shape = [None, 2] - p1 = constant_op.constant([[1, 2], [3, 4]]) - p1.set_shape(var_shape) - p2 = constant_op.constant([[5, 6], [7, 8], [9, 10]]) - p2.set_shape(var_shape) - p3 = constant_op.constant([[11, 12]]) - p3.set_shape(var_shape) - batch = [p1, p2, p3] - batch_size = len(batch) - - zero64 = constant_op.constant(0, dtype=dtypes.int64) - examples = variables.Variable(zero64) - counter = examples.count_up_to(batch_size) - - # Create a PaddingFIFOQueue to enqueue these tensors. - q = data_flow_ops.PaddingFIFOQueue( - capacity=10, dtypes=[dtypes.int32], shapes=[var_shape]) - for tensor in [p1, p2, p3]: - q.enqueue([tensor]).run() - - # Dequeue from the queue and batch them using batch(). - batches = input_lib.batch([q.dequeue(), counter], batch_size=batch_size, - num_threads=1, dynamic_pad=True) - self.assertEqual([batch_size, None, 2], batches[0].shape.as_list()) - - # Finally, assemble them into prefetch_queue with dynamic_pad. - batcher = prefetch_queue.prefetch_queue(batches, dynamic_pad=True) - batches = batcher.dequeue() - self.assertEqual([batch_size, None, 2], batches[0].shape.as_list()) - - variables.global_variables_initializer().run() - threads = queue_runner_impl.start_queue_runners() - - values, _ = sess.run(batches) - # We enqueued 3 tensors of [None, 2] shapes, so using dynamic_pad - # they should be padded to the fixed size [3, 3, 2], where 3 - # is the maximum length of the batch. - self.assertTrue(np.array_equal( - np.array([[[1, 2], [3, 4], [0, 0]], - [[5, 6], [7, 8], [9, 10]], - [[11, 12], [0, 0], [0, 0]]]), - values)) - - with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batches) - for thread in threads: - thread.join() - - def testDictConstruction(self): - with ops.Graph().as_default(): - batches = { - 'first': constant_op.constant([1]), - 'second': constant_op.constant([2.0, 2.1]) - } - prefetcher = prefetch_queue.prefetch_queue(batches) - dequeued = prefetcher.dequeue() - self.assertTrue(isinstance(dequeued, dict)) - self.assertEqual(2, len(dequeued)) - self.assertEqual(dtypes.int32, dequeued['first'].dtype) - self.assertEqual(dtypes.float32, dequeued['second'].dtype) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/data/test_utils.py b/tensorflow/contrib/slim/python/slim/data/test_utils.py deleted file mode 100644 index 4a6983cbd78..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/test_utils.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains test utilities.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -import numpy as np - -from tensorflow.core.example import example_pb2 -from tensorflow.core.example import feature_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.lib.io import tf_record -from tensorflow.python.ops import image_ops - - -def _encoded_int64_feature(ndarray): - return feature_pb2.Feature(int64_list=feature_pb2.Int64List( - value=ndarray.flatten().tolist())) - - -def _encoded_bytes_feature(tf_encoded): - encoded = tf_encoded.eval() - - def string_to_bytes(value): - return feature_pb2.BytesList(value=[value]) - - return feature_pb2.Feature(bytes_list=string_to_bytes(encoded)) - - -def _string_feature(value): - value = value.encode('utf-8') - return feature_pb2.Feature(bytes_list=feature_pb2.BytesList(value=[value])) - - -def _encoder(image, image_format): - assert image_format in ['jpeg', 'png'] - if image_format == 'jpeg': - tf_image = constant_op.constant(image, dtype=dtypes.uint8) - return image_ops.encode_jpeg(tf_image) - if image_format == 'png': - tf_image = constant_op.constant(image, dtype=dtypes.uint8) - return image_ops.encode_png(tf_image) - - -def generate_image(image_shape, image_format='jpeg', label=0): - """Generates an image and an example containing the encoded image. - - GenerateImage must be called within an active session. - - Args: - image_shape: the shape of the image to generate. - image_format: the encoding format of the image. - label: the int64 labels for the image. - - Returns: - image: the generated image. - example: a TF-example with a feature key 'image/encoded' set to the - serialized image and a feature key 'image/format' set to the image - encoding format ['jpeg', 'png']. - """ - image = np.random.random_integers(0, 255, size=image_shape) - tf_encoded = _encoder(image, image_format) - example = example_pb2.Example(features=feature_pb2.Features(feature={ - 'image/encoded': _encoded_bytes_feature(tf_encoded), - 'image/format': _string_feature(image_format), - 'image/class/label': _encoded_int64_feature(np.array(label)), - })) - - return image, example.SerializeToString() - - -def create_tfrecord_files(output_dir, num_files=3, num_records_per_file=10): - """Creates TFRecords files. - - The method must be called within an active session. - - Args: - output_dir: The directory where the files are stored. - num_files: The number of files to create. - num_records_per_file: The number of records per file. - - Returns: - A list of the paths to the TFRecord files. - """ - tfrecord_paths = [] - for i in range(num_files): - path = os.path.join(output_dir, - 'flowers.tfrecord-%d-of-%s' % (i, num_files)) - tfrecord_paths.append(path) - - writer = tf_record.TFRecordWriter(path) - for _ in range(num_records_per_file): - _, example = generate_image(image_shape=(10, 10, 3)) - writer.write(example) - writer.close() - - return tfrecord_paths diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py deleted file mode 100644 index 381d5941e5a..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py +++ /dev/null @@ -1,525 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains the TFExampleDecoder its associated helper classes. - -The TFExampleDecode is a DataDecoder used to decode TensorFlow Example protos. -In order to do so each requested item must be paired with one or more Example -features that are parsed to produce the Tensor-based manifestation of the item. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - -from tensorflow.contrib.slim.python.slim.data import data_decoder -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import map_fn -from tensorflow.python.ops import image_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import sparse_ops - - -@six.add_metaclass(abc.ABCMeta) -class ItemHandler(object): - """Specifies the item-to-Features mapping for tf.parse_example. - - An ItemHandler both specifies a list of Features used for parsing an Example - proto as well as a function that post-processes the results of Example - parsing. - """ - - def __init__(self, keys): - """Constructs the handler with the name of the tf.Feature keys to use. - - See third_party/tensorflow/core/example/feature.proto - - Args: - keys: the name of the TensorFlow Example Feature. - """ - if not isinstance(keys, (tuple, list)): - keys = [keys] - self._keys = keys - - @property - def keys(self): - return self._keys - - @abc.abstractmethod - def tensors_to_item(self, keys_to_tensors): - """Maps the given dictionary of tensors to the requested item. - - Args: - keys_to_tensors: a mapping of TF-Example keys to parsed tensors. - - Returns: - the final tensor representing the item being handled. - """ - pass - - -class ItemHandlerCallback(ItemHandler): - """An ItemHandler that converts the parsed tensors via a given function. - - Unlike other ItemHandlers, the ItemHandlerCallback resolves its item via - a callback function rather than using prespecified behavior. - """ - - def __init__(self, keys, func): - """Initializes the ItemHandler. - - Args: - keys: a list of TF-Example keys. - func: a function that takes as an argument a dictionary from `keys` to - parsed Tensors. - """ - super(ItemHandlerCallback, self).__init__(keys) - self._func = func - - def tensors_to_item(self, keys_to_tensors): - return self._func(keys_to_tensors) - - -class BoundingBox(ItemHandler): - """An ItemHandler that concatenates a set of parsed Tensors to Bounding Boxes. - """ - - def __init__(self, keys=None, prefix=''): - """Initialize the bounding box handler. - - Args: - keys: A list of four key names representing the ymin, xmin, ymax, mmax - prefix: An optional prefix for each of the bounding box keys. - If provided, `prefix` is appended to each key in `keys`. - - Raises: - ValueError: if keys is not `None` and also not a list of exactly 4 keys - """ - if keys is None: - keys = ['ymin', 'xmin', 'ymax', 'xmax'] - elif len(keys) != 4: - raise ValueError('BoundingBox expects 4 keys but got {}'.format( - len(keys))) - self._prefix = prefix - self._keys = keys - self._full_keys = [prefix + k for k in keys] - super(BoundingBox, self).__init__(self._full_keys) - - def tensors_to_item(self, keys_to_tensors): - """Maps the given dictionary of tensors to a concatenated list of bboxes. - - Args: - keys_to_tensors: a mapping of TF-Example keys to parsed tensors. - - Returns: - [num_boxes, 4] tensor of bounding box coordinates, - i.e. 1 bounding box per row, in order [y_min, x_min, y_max, x_max]. - """ - sides = [] - for key in self._full_keys: - side = keys_to_tensors[key] - if isinstance(side, sparse_tensor.SparseTensor): - side = side.values - side = array_ops.expand_dims(side, 0) - sides.append(side) - - bounding_box = array_ops.concat(sides, 0) - return array_ops.transpose(bounding_box) - - -class Tensor(ItemHandler): - """An ItemHandler that returns a parsed Tensor.""" - - def __init__(self, tensor_key, shape_keys=None, shape=None, default_value=0): - """Initializes the Tensor handler. - - Tensors are, by default, returned without any reshaping. However, there are - two mechanisms which allow reshaping to occur at load time. If `shape_keys` - is provided, both the `Tensor` corresponding to `tensor_key` and - `shape_keys` is loaded and the former `Tensor` is reshaped with the values - of the latter. Alternatively, if a fixed `shape` is provided, the `Tensor` - corresponding to `tensor_key` is loaded and reshape appropriately. - If neither `shape_keys` nor `shape` are provided, the `Tensor` will be - returned without any reshaping. - - Args: - tensor_key: the name of the `TFExample` feature to read the tensor from. - shape_keys: Optional name or list of names of the TF-Example feature in - which the tensor shape is stored. If a list, then each corresponds to - one dimension of the shape. - shape: Optional output shape of the `Tensor`. If provided, the `Tensor` is - reshaped accordingly. - default_value: The value used when the `tensor_key` is not found in a - particular `TFExample`. - - Raises: - ValueError: if both `shape_keys` and `shape` are specified. - """ - if shape_keys and shape is not None: - raise ValueError('Cannot specify both shape_keys and shape parameters.') - if shape_keys and not isinstance(shape_keys, list): - shape_keys = [shape_keys] - self._tensor_key = tensor_key - self._shape_keys = shape_keys - self._shape = shape - self._default_value = default_value - keys = [tensor_key] - if shape_keys: - keys.extend(shape_keys) - super(Tensor, self).__init__(keys) - - def tensors_to_item(self, keys_to_tensors): - tensor = keys_to_tensors[self._tensor_key] - shape = self._shape - if self._shape_keys: - shape_dims = [] - for k in self._shape_keys: - shape_dim = keys_to_tensors[k] - if isinstance(shape_dim, sparse_tensor.SparseTensor): - shape_dim = sparse_ops.sparse_tensor_to_dense(shape_dim) - shape_dims.append(shape_dim) - shape = array_ops.reshape(array_ops.stack(shape_dims), [-1]) - if isinstance(tensor, sparse_tensor.SparseTensor): - if shape is not None: - tensor = sparse_ops.sparse_reshape(tensor, shape) - tensor = sparse_ops.sparse_tensor_to_dense(tensor, self._default_value) - else: - if shape is not None: - tensor = array_ops.reshape(tensor, shape) - return tensor - - -class LookupTensor(Tensor): - """An ItemHandler that returns a parsed Tensor, the result of a lookup.""" - - def __init__(self, - tensor_key, - table, - shape_keys=None, - shape=None, - default_value=''): - """Initializes the LookupTensor handler. - - See Tensor. Simply calls a vocabulary (most often, a label mapping) lookup. - - Args: - tensor_key: the name of the `TFExample` feature to read the tensor from. - table: A tf.lookup table. - shape_keys: Optional name or list of names of the TF-Example feature in - which the tensor shape is stored. If a list, then each corresponds to - one dimension of the shape. - shape: Optional output shape of the `Tensor`. If provided, the `Tensor` is - reshaped accordingly. - default_value: The value used when the `tensor_key` is not found in a - particular `TFExample`. - - Raises: - ValueError: if both `shape_keys` and `shape` are specified. - """ - self._table = table - super(LookupTensor, self).__init__(tensor_key, shape_keys, shape, - default_value) - - def tensors_to_item(self, keys_to_tensors): - unmapped_tensor = super(LookupTensor, self).tensors_to_item(keys_to_tensors) - return self._table.lookup(unmapped_tensor) - - -class BackupHandler(ItemHandler): - """An ItemHandler that tries two ItemHandlers in order.""" - - def __init__(self, handler, backup): - """Initializes the BackupHandler handler. - - If the first Handler's tensors_to_item returns a Tensor with no elements, - the second Handler is used. - - Args: - handler: The primary ItemHandler. - backup: The backup ItemHandler. - - Raises: - ValueError: if either is not an ItemHandler. - """ - if not isinstance(handler, ItemHandler): - raise ValueError('Primary handler is of type %s instead of ItemHandler' - % type(handler)) - if not isinstance(backup, ItemHandler): - raise ValueError('Backup handler is of type %s instead of ItemHandler' - % type(backup)) - self._handler = handler - self._backup = backup - super(BackupHandler, self).__init__(handler.keys + backup.keys) - - def tensors_to_item(self, keys_to_tensors): - item = self._handler.tensors_to_item(keys_to_tensors) - return control_flow_ops.cond( - pred=math_ops.equal(math_ops.reduce_prod(array_ops.shape(item)), 0), - true_fn=lambda: self._backup.tensors_to_item(keys_to_tensors), - false_fn=lambda: item) - - -class SparseTensor(ItemHandler): - """An ItemHandler for SparseTensors.""" - - def __init__(self, - indices_key=None, - values_key=None, - shape_key=None, - shape=None, - densify=False, - default_value=0): - """Initializes the Tensor handler. - - Args: - indices_key: the name of the TF-Example feature that contains the ids. - Defaults to 'indices'. - values_key: the name of the TF-Example feature that contains the values. - Defaults to 'values'. - shape_key: the name of the TF-Example feature that contains the shape. - If provided it would be used. - shape: the output shape of the SparseTensor. If `shape_key` is not - provided this `shape` would be used. - densify: whether to convert the SparseTensor into a dense Tensor. - default_value: Scalar value to set when making dense for indices not - specified in the `SparseTensor`. - """ - indices_key = indices_key or 'indices' - values_key = values_key or 'values' - self._indices_key = indices_key - self._values_key = values_key - self._shape_key = shape_key - self._shape = shape - self._densify = densify - self._default_value = default_value - keys = [indices_key, values_key] - if shape_key: - keys.append(shape_key) - super(SparseTensor, self).__init__(keys) - - def tensors_to_item(self, keys_to_tensors): - indices = keys_to_tensors[self._indices_key] - values = keys_to_tensors[self._values_key] - if self._shape_key: - shape = keys_to_tensors[self._shape_key] - if isinstance(shape, sparse_tensor.SparseTensor): - shape = sparse_ops.sparse_tensor_to_dense(shape) - elif self._shape: - shape = self._shape - else: - shape = indices.dense_shape - indices_shape = array_ops.shape(indices.indices) - rank = indices_shape[1] - ids = math_ops.cast(indices.values, dtypes.int64) - indices_columns_to_preserve = array_ops.slice( - indices.indices, [0, 0], array_ops.stack([-1, rank - 1])) - new_indices = array_ops.concat( - [indices_columns_to_preserve, array_ops.reshape(ids, [-1, 1])], 1) - - tensor = sparse_tensor.SparseTensor(new_indices, values.values, shape) - if self._densify: - tensor = sparse_ops.sparse_tensor_to_dense(tensor, self._default_value) - return tensor - - -class Image(ItemHandler): - """An ItemHandler that decodes a parsed Tensor as an image.""" - - def __init__(self, - image_key=None, - format_key=None, - shape=None, - channels=3, - dtype=dtypes.uint8, - repeated=False, - dct_method=''): - """Initializes the image. - - Args: - image_key: the name of the TF-Example feature in which the encoded image - is stored. - format_key: the name of the TF-Example feature in which the image format - is stored. - shape: the output shape of the image as 1-D `Tensor` - [height, width, channels]. If provided, the image is reshaped - accordingly. If left as None, no reshaping is done. A shape should - be supplied only if all the stored images have the same shape. - channels: the number of channels in the image. - dtype: images will be decoded at this bit depth. Different formats - support different bit depths. - See tf.image.decode_image, - tf.io.decode_raw, - repeated: if False, decodes a single image. If True, decodes a - variable number of image strings from a 1D tensor of strings. - dct_method: An optional string. Defaults to empty string. It only takes - effect when image format is jpeg, used to specify a hint about the - algorithm used for jpeg decompression. Currently valid values - are ['INTEGER_FAST', 'INTEGER_ACCURATE']. The hint may be ignored, for - example, the jpeg library does not have that specific option. - """ - if not image_key: - image_key = 'image/encoded' - if not format_key: - format_key = 'image/format' - - super(Image, self).__init__([image_key, format_key]) - self._image_key = image_key - self._format_key = format_key - self._shape = shape - self._channels = channels - self._dtype = dtype - self._repeated = repeated - self._dct_method = dct_method - - def tensors_to_item(self, keys_to_tensors): - """See base class.""" - image_buffer = keys_to_tensors[self._image_key] - image_format = keys_to_tensors[self._format_key] - - if self._repeated: - return map_fn.map_fn(lambda x: self._decode(x, image_format), - image_buffer, dtype=self._dtype) - else: - return self._decode(image_buffer, image_format) - - def _decode(self, image_buffer, image_format): - """Decodes the image buffer. - - Args: - image_buffer: The tensor representing the encoded image tensor. - image_format: The image format for the image in `image_buffer`. If image - format is `raw`, all images are expected to be in this format, otherwise - this op can decode a mix of `jpg` and `png` formats. - - Returns: - A tensor that represents decoded image of self._shape, or - (?, ?, self._channels) if self._shape is not specified. - """ - - def decode_image(): - """Decodes a image based on the headers.""" - return math_ops.cast( - image_ops.decode_image(image_buffer, channels=self._channels), - self._dtype) - - def decode_jpeg(): - """Decodes a jpeg image with specified '_dct_method'.""" - return math_ops.cast( - image_ops.decode_jpeg( - image_buffer, - channels=self._channels, - dct_method=self._dct_method), self._dtype) - - def check_jpeg(): - """Checks if an image is jpeg.""" - # For jpeg, we directly use image_ops.decode_jpeg rather than decode_image - # in order to feed the jpeg specify parameter 'dct_method'. - return control_flow_ops.cond( - image_ops.is_jpeg(image_buffer), - decode_jpeg, - decode_image, - name='cond_jpeg') - - def decode_raw(): - """Decodes a raw image.""" - return parsing_ops.decode_raw(image_buffer, out_type=self._dtype) - - pred_fn_pairs = [(math_ops.logical_or( - math_ops.equal(image_format, 'raw'), - math_ops.equal(image_format, 'RAW')), decode_raw)] - image = control_flow_ops.case( - pred_fn_pairs, default=check_jpeg, exclusive=True) - - image.set_shape([None, None, self._channels]) - if self._shape is not None: - image = array_ops.reshape(image, self._shape) - - return image - - -class TFExampleDecoder(data_decoder.DataDecoder): - """A decoder for TensorFlow Examples. - - Decoding Example proto buffers is comprised of two stages: (1) Example parsing - and (2) tensor manipulation. - - In the first stage, the tf.io.parse_example function is called with a list of - FixedLenFeatures and SparseLenFeatures. These instances tell TF how to parse - the example. The output of this stage is a set of tensors. - - In the second stage, the resulting tensors are manipulated to provide the - requested 'item' tensors. - - To perform this decoding operation, an ExampleDecoder is given a list of - ItemHandlers. Each ItemHandler indicates the set of features for stage 1 and - contains the instructions for post_processing its tensors for stage 2. - """ - - def __init__(self, keys_to_features, items_to_handlers): - """Constructs the decoder. - - Args: - keys_to_features: a dictionary from TF-Example keys to either - tf.io.VarLenFeature or tf.io.FixedLenFeature instances. See tensorflow's - parsing_ops.py. - items_to_handlers: a dictionary from items (strings) to ItemHandler - instances. Note that the ItemHandler's are provided the keys that they - use to return the final item Tensors. - """ - self._keys_to_features = keys_to_features - self._items_to_handlers = items_to_handlers - - def list_items(self): - """See base class.""" - return list(self._items_to_handlers.keys()) - - def decode(self, serialized_example, items=None): - """Decodes the given serialized TF-example. - - Args: - serialized_example: a serialized TF-example tensor. - items: the list of items to decode. These must be a subset of the item - keys in self._items_to_handlers. If `items` is left as None, then all - of the items in self._items_to_handlers are decoded. - - Returns: - the decoded items, a list of tensor. - """ - example = parsing_ops.parse_single_example(serialized_example, - self._keys_to_features) - - # Reshape non-sparse elements just once, adding the reshape ops in - # deterministic order. - for k in sorted(self._keys_to_features): - v = self._keys_to_features[k] - if isinstance(v, parsing_ops.FixedLenFeature): - example[k] = array_ops.reshape(example[k], v.shape) - - if not items: - items = self._items_to_handlers.keys() - - outputs = [] - for item in items: - handler = self._items_to_handlers[item] - keys_to_tensors = {key: example[key] for key in handler.keys} - outputs.append(handler.tensors_to_item(keys_to_tensors)) - return outputs diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py deleted file mode 100644 index 109d8bbaf86..00000000000 --- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py +++ /dev/null @@ -1,965 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.data.tfexample_decoder.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import sys - -from tensorflow.contrib.slim.python.slim.data import tfexample_decoder -from tensorflow.core.example import example_pb2 -from tensorflow.core.example import feature_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import image_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.platform import test - - -class TFExampleDecoderTest(test.TestCase): - - def _EncodedFloatFeature(self, ndarray): - return feature_pb2.Feature( - float_list=feature_pb2.FloatList(value=ndarray.flatten().tolist())) - - def _EncodedInt64Feature(self, ndarray): - return feature_pb2.Feature( - int64_list=feature_pb2.Int64List(value=ndarray.flatten().tolist())) - - def _EncodedBytesFeature(self, tf_encoded): - with self.cached_session(): - encoded = tf_encoded.eval() - - def BytesList(value): - return feature_pb2.BytesList(value=[value]) - - return feature_pb2.Feature(bytes_list=BytesList(encoded)) - - def _BytesFeature(self, ndarray): - values = ndarray.flatten().tolist() - for i in range(len(values)): - values[i] = values[i].encode('utf-8') - return feature_pb2.Feature(bytes_list=feature_pb2.BytesList(value=values)) - - def _StringFeature(self, value): - value = value.encode('utf-8') - return feature_pb2.Feature(bytes_list=feature_pb2.BytesList(value=[value])) - - def _Encoder(self, image, image_format): - assert image_format in ['jpeg', 'JPEG', 'png', 'PNG', 'raw', 'RAW'] - if image_format in ['jpeg', 'JPEG']: - tf_image = constant_op.constant(image, dtype=dtypes.uint8) - return image_ops.encode_jpeg(tf_image) - if image_format in ['png', 'PNG']: - tf_image = constant_op.constant(image, dtype=dtypes.uint8) - return image_ops.encode_png(tf_image) - if image_format in ['raw', 'RAW']: - # If machine is big endian, change the byte ordering in case of dtype float32 - # so that it should be interpreted correctly. - if image.dtype == np.float32 and sys.byteorder == 'big': - image = image.astype(' 0, 'guinea pig' -> 1, 'cat' -> 2 - table = lookup_ops.index_table_from_tensor( - constant_op.constant(['dog', 'guinea pig', 'cat'])) - - with self.cached_session() as sess: - sess.run(lookup_ops.tables_initializer()) - - serialized_example = array_ops.reshape(serialized_example, shape=[]) - - keys_to_features = { - 'image/object/class/text': parsing_ops.VarLenFeature(dtypes.string), - } - - items_to_handlers = { - 'labels': - tfexample_decoder.LookupTensor('image/object/class/text', table), - } - - decoder = tfexample_decoder.TFExampleDecoder(keys_to_features, - items_to_handlers) - obtained_class_ids = decoder.decode(serialized_example)[0].eval() - - self.assertAllClose([2, 0, 1], obtained_class_ids) - - def testDecodeExampleWithBackupHandlerLookup(self): - - example1 = example_pb2.Example( - features=feature_pb2.Features( - feature={ - 'image/object/class/text': - self._BytesFeature(np.array(['cat', 'dog', 'guinea pig'])), - 'image/object/class/label': - self._EncodedInt64Feature(np.array([42, 10, 900])) - })) - example2 = example_pb2.Example( - features=feature_pb2.Features( - feature={ - 'image/object/class/text': - self._BytesFeature(np.array(['cat', 'dog', 'guinea pig'])), - })) - example3 = example_pb2.Example( - features=feature_pb2.Features( - feature={ - 'image/object/class/label': - self._EncodedInt64Feature(np.array([42, 10, 901])) - })) - # 'dog' -> 0, 'guinea pig' -> 1, 'cat' -> 2 - table = lookup_ops.index_table_from_tensor( - constant_op.constant(['dog', 'guinea pig', 'cat'])) - keys_to_features = { - 'image/object/class/text': parsing_ops.VarLenFeature(dtypes.string), - 'image/object/class/label': parsing_ops.VarLenFeature(dtypes.int64), - } - backup_handler = tfexample_decoder.BackupHandler( - handler=tfexample_decoder.Tensor('image/object/class/label'), - backup=tfexample_decoder.LookupTensor('image/object/class/text', table)) - items_to_handlers = { - 'labels': backup_handler, - } - decoder = tfexample_decoder.TFExampleDecoder(keys_to_features, - items_to_handlers) - obtained_class_ids_each_example = [] - with self.cached_session() as sess: - sess.run(lookup_ops.tables_initializer()) - for example in [example1, example2, example3]: - serialized_example = array_ops.reshape( - example.SerializeToString(), shape=[]) - obtained_class_ids_each_example.append( - decoder.decode(serialized_example)[0].eval()) - - self.assertAllClose([42, 10, 900], obtained_class_ids_each_example[0]) - self.assertAllClose([2, 0, 1], obtained_class_ids_each_example[1]) - self.assertAllClose([42, 10, 901], obtained_class_ids_each_example[2]) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/evaluation.py b/tensorflow/contrib/slim/python/slim/evaluation.py deleted file mode 100644 index b272e2543e2..00000000000 --- a/tensorflow/contrib/slim/python/slim/evaluation.py +++ /dev/null @@ -1,323 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains functions for evaluation and summarization of metrics. - -The evaluation.py module contains helper functions for evaluating TensorFlow -modules using a variety of metrics and summarizing the results. - -********************** -* Evaluating Metrics * -********************** - -In the simplest use case, we use a model to create the predictions, then specify -the metrics and choose one model checkpoint, finally call the`evaluation_once` -method: - - # Create model and obtain the predictions: - images, labels = LoadData(...) - predictions = MyModel(images) - - # Choose the metrics to compute: - names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({ - "accuracy": slim.metrics.accuracy(predictions, labels), - "mse": slim.metrics.mean_squared_error(predictions, labels), - }) - - checkpoint_path = '/tmp/my_model_dir/my_checkpoint' - log_dir = '/tmp/my_model_eval/' - - initial_op = tf.group( - tf.compat.v1.global_variables_initializer(), - tf.compat.v1.local_variables_initializer()) - - metric_values = slim.evaluate_once( - master='', - checkpoint_path=checkpoint_path, - log_dir=log_dir, - num_evals=1, - initial_op=initial_op, - eval_op=names_to_updates.values(), - final_op=name_to_values.values()) - - for metric, value in zip(names_to_values.keys(), metric_values): - logging.info('Metric %s has value: %f', metric, value) - -************************************************ -* Evaluating a Checkpointed Model with Metrics * -************************************************ - -Often, one wants to evaluate a model checkpoint saved on disk. This can be -performed once or repeatedly on a set schedule. - -To evaluate a particular model, users define zero or more metrics and zero or -more summaries and call the evaluation_loop method: - - # Create model and obtain the predictions: - images, labels = LoadData(...) - predictions = MyModel(images) - - # Choose the metrics to compute: - names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({ - "accuracy": slim.metrics.accuracy(predictions, labels), - "mse": slim.metrics.mean_squared_error(predictions, labels), - }) - - # Define the summaries to write: - for metric_name, metric_value in metrics_to_values.iteritems(): - tf.compat.v1.summary.scalar(metric_name, metric_value) - - checkpoint_dir = '/tmp/my_model_dir/' - log_dir = '/tmp/my_model_eval/' - - # We'll evaluate 1000 batches: - num_evals = 1000 - - # Evaluate every 10 minutes: - slim.evaluation_loop( - '', - checkpoint_dir, - logdir, - num_evals=num_evals, - eval_op=names_to_updates.values(), - summary_op=tf.contrib.deprecated.merge_summary(summary_ops), - eval_interval_secs=600) - -************************************************** -* Evaluating a Checkpointed Model with Summaries * -************************************************** - -At times, an evaluation can be performed without metrics at all but rather -with only summaries. The user need only leave out the 'eval_op' argument: - - # Create model and obtain the predictions: - images, labels = LoadData(...) - predictions = MyModel(images) - - # Define the summaries to write: - tf.compat.v1.summary.scalar(...) - tf.compat.v1.summary.histogram(...) - - checkpoint_dir = '/tmp/my_model_dir/' - log_dir = '/tmp/my_model_eval/' - - # Evaluate once every 10 minutes. - slim.evaluation_loop( - master='', - checkpoint_dir, - logdir, - num_evals=1, - summary_op=tf.contrib.deprecated.merge_summary(summary_ops), - eval_interval_secs=600) - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.training.python.training import evaluation -from tensorflow.python.summary import summary -from tensorflow.python.training import monitored_session -from tensorflow.python.training import saver as tf_saver - -__all__ = [ - 'evaluate_once', - 'evaluation_loop', - 'wait_for_new_checkpoint', - 'checkpoints_iterator', -] - -wait_for_new_checkpoint = evaluation.wait_for_new_checkpoint -checkpoints_iterator = evaluation.checkpoints_iterator - -_USE_DEFAULT = 0 - - -def evaluate_once(master, - checkpoint_path, - logdir, - num_evals=1, - initial_op=None, - initial_op_feed_dict=None, - eval_op=None, - eval_op_feed_dict=None, - final_op=None, - final_op_feed_dict=None, - summary_op=_USE_DEFAULT, - summary_op_feed_dict=None, - variables_to_restore=None, - session_config=None, - hooks=None): - """Evaluates the model at the given checkpoint path. - - Args: - master: The BNS address of the TensorFlow master. - checkpoint_path: The path to a checkpoint to use for evaluation. - logdir: The directory where the TensorFlow summaries are written to. - num_evals: The number of times to run `eval_op`. - initial_op: An operation run at the beginning of evaluation. - initial_op_feed_dict: A feed dictionary to use when executing `initial_op`. - eval_op: A operation run `num_evals` times. - eval_op_feed_dict: The feed dictionary to use when executing the `eval_op`. - final_op: An operation to execute after all of the `eval_op` executions. The - value of `final_op` is returned. - final_op_feed_dict: A feed dictionary to use when executing `final_op`. - summary_op: The summary_op to evaluate after running TF-Slims metric ops. By - default the summary_op is set to tf.compat.v1.summary.merge_all(). - summary_op_feed_dict: An optional feed dictionary to use when running the - `summary_op`. - variables_to_restore: A list of TensorFlow variables to restore during - evaluation. If the argument is left as `None` then - slim.variables.GetVariablesToRestore() is used. - session_config: An instance of `tf.compat.v1.ConfigProto` that will be used - to configure the `Session`. If left as `None`, the default will be used. - hooks: A list of additional `SessionRunHook` objects to pass during the - evaluation. - - Returns: - The value of `final_op` or `None` if `final_op` is `None`. - """ - if summary_op == _USE_DEFAULT: - summary_op = summary.merge_all() - - all_hooks = [ - evaluation.StopAfterNEvalsHook(num_evals), - ] - - if summary_op is not None: - all_hooks.append( - evaluation.SummaryAtEndHook( - log_dir=logdir, - summary_op=summary_op, - feed_dict=summary_op_feed_dict)) - if hooks is not None: - all_hooks.extend(hooks) - - saver = None - if variables_to_restore is not None: - saver = tf_saver.Saver(variables_to_restore) - - return evaluation.evaluate_once( - checkpoint_path, - master=master, - scaffold=monitored_session.Scaffold( - init_op=initial_op, init_feed_dict=initial_op_feed_dict, saver=saver), - eval_ops=eval_op, - feed_dict=eval_op_feed_dict, - final_ops=final_op, - final_ops_feed_dict=final_op_feed_dict, - hooks=all_hooks, - config=session_config) - - -def evaluation_loop(master, - checkpoint_dir, - logdir, - num_evals=1, - initial_op=None, - initial_op_feed_dict=None, - init_fn=None, - eval_op=None, - eval_op_feed_dict=None, - final_op=None, - final_op_feed_dict=None, - summary_op=_USE_DEFAULT, - summary_op_feed_dict=None, - variables_to_restore=None, - eval_interval_secs=60, - max_number_of_evaluations=None, - session_config=None, - timeout=None, - timeout_fn=None, - hooks=None): - """Runs TF-Slim's Evaluation Loop. - - Args: - master: The BNS address of the TensorFlow master. - checkpoint_dir: The directory where checkpoints are stored. - logdir: The directory where the TensorFlow summaries are written to. - num_evals: The number of times to run `eval_op`. - initial_op: An operation run at the beginning of evaluation. - initial_op_feed_dict: A feed dictionary to use when executing `initial_op`. - init_fn: An optional callable to be executed after `init_op` is called. The - callable must accept one argument, the session being initialized. - eval_op: A operation run `num_evals` times. - eval_op_feed_dict: The feed dictionary to use when executing the `eval_op`. - final_op: An operation to execute after all of the `eval_op` executions. The - value of `final_op` is returned. - final_op_feed_dict: A feed dictionary to use when executing `final_op`. - summary_op: The summary_op to evaluate after running TF-Slims metric ops. By - default the summary_op is set to tf.compat.v1.summary.merge_all(). - summary_op_feed_dict: An optional feed dictionary to use when running the - `summary_op`. - variables_to_restore: A list of TensorFlow variables to restore during - evaluation. If the argument is left as `None` then - slim.variables.GetVariablesToRestore() is used. - eval_interval_secs: The minimum number of seconds between evaluations. - max_number_of_evaluations: the max number of iterations of the evaluation. - If the value is left as 'None', the evaluation continues indefinitely. - session_config: An instance of `tf.compat.v1.ConfigProto` that will be used - to configure the `Session`. If left as `None`, the default will be used. - timeout: The maximum amount of time to wait between checkpoints. If left as - `None`, then the process will wait indefinitely. - timeout_fn: Optional function to call after a timeout. If the function - returns True, then it means that no new checkpoints will be generated and - the iterator will exit. The function is called with no arguments. - hooks: A list of additional `SessionRunHook` objects to pass during repeated - evaluations. - - Returns: - The value of `final_op` or `None` if `final_op` is `None`. - """ - if summary_op == _USE_DEFAULT: - summary_op = summary.merge_all() - - all_hooks = [ - evaluation.StopAfterNEvalsHook(num_evals), - ] - - if summary_op is not None: - all_hooks.append( - evaluation.SummaryAtEndHook( - log_dir=logdir, - summary_op=summary_op, - feed_dict=summary_op_feed_dict)) - - if hooks is not None: - # Add custom hooks if provided. - all_hooks.extend(hooks) - - saver = None - if variables_to_restore is not None: - saver = tf_saver.Saver(variables_to_restore) - - return evaluation.evaluate_repeatedly( - checkpoint_dir, - master=master, - scaffold=monitored_session.Scaffold( - init_op=initial_op, - init_feed_dict=initial_op_feed_dict, - init_fn=init_fn, - saver=saver), - eval_ops=eval_op, - feed_dict=eval_op_feed_dict, - final_ops=final_op, - final_ops_feed_dict=final_op_feed_dict, - eval_interval_secs=eval_interval_secs, - hooks=all_hooks, - config=session_config, - max_number_of_evaluations=max_number_of_evaluations, - timeout=timeout, - timeout_fn=timeout_fn) diff --git a/tensorflow/contrib/slim/python/slim/evaluation_test.py b/tensorflow/contrib/slim/python/slim/evaluation_test.py deleted file mode 100644 index cbfdaeb45d7..00000000000 --- a/tensorflow/contrib/slim/python/slim/evaluation_test.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.evaluation.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import glob -import os -import shutil -import time - -import numpy as np - -from tensorflow.contrib.framework.python.ops import variables as variables_lib -from tensorflow.contrib.slim.python.slim import evaluation -from tensorflow.contrib.training.python.training import evaluation as evaluation_lib -from tensorflow.core.protobuf import saver_pb2 -from tensorflow.python.debug.lib import debug_data -from tensorflow.python.debug.wrappers import hooks -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import metrics -from tensorflow.python.ops import variables -from tensorflow.python.platform import flags -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test -from tensorflow.python.summary import summary_iterator -from tensorflow.python.training import input # pylint: disable=redefined-builtin -from tensorflow.python.training import saver as saver_lib -from tensorflow.python.training import session_run_hook - - -FLAGS = flags.FLAGS - - -def GenerateTestData(num_classes, batch_size): - inputs = np.random.rand(batch_size, num_classes) - - np.random.seed(0) - labels = np.random.randint(low=0, high=num_classes, size=batch_size) - labels = labels.reshape((batch_size,)) - return inputs, labels - - -def TestModel(inputs): - scale = variables.Variable(1.0, trainable=False) - - # Scaling the outputs wont change the result... - outputs = math_ops.multiply(inputs, scale) - return math_ops.argmax(outputs, 1), scale - - -def GroundTruthAccuracy(inputs, labels, batch_size): - predictions = np.argmax(inputs, 1) - num_correct = np.sum(predictions == labels) - return float(num_correct) / batch_size - - -class EvaluationTest(test.TestCase): - - def setUp(self): - super(EvaluationTest, self).setUp() - - num_classes = 8 - batch_size = 16 - inputs, labels = GenerateTestData(num_classes, batch_size) - self._expected_accuracy = GroundTruthAccuracy(inputs, labels, batch_size) - - self._global_step = variables_lib.get_or_create_global_step() - self._inputs = constant_op.constant(inputs, dtype=dtypes.float32) - self._labels = constant_op.constant(labels, dtype=dtypes.int64) - self._predictions, self._scale = TestModel(self._inputs) - - def testFinalOpsOnEvaluationLoop(self): - value_op, update_op = metrics.accuracy( - labels=self._labels, predictions=self._predictions) - init_op = control_flow_ops.group(variables.global_variables_initializer(), - variables.local_variables_initializer()) - # Create checkpoint and log directories: - chkpt_dir = os.path.join(self.get_temp_dir(), 'tmp_logs/') - gfile.MakeDirs(chkpt_dir) - logdir = os.path.join(self.get_temp_dir(), 'tmp_logs2/') - gfile.MakeDirs(logdir) - - # Save initialized variables to a checkpoint directory: - saver = saver_lib.Saver() - with self.cached_session() as sess: - init_op.run() - saver.save(sess, os.path.join(chkpt_dir, 'chkpt')) - - class Object(object): - - def __init__(self): - self.hook_was_run = False - - obj = Object() - - # Create a custom session run hook. - class CustomHook(session_run_hook.SessionRunHook): - - def __init__(self, obj): - self.obj = obj - - def end(self, session): - self.obj.hook_was_run = True - - # Now, run the evaluation loop: - accuracy_value = evaluation.evaluation_loop( - '', - chkpt_dir, - logdir, - eval_op=update_op, - final_op=value_op, - hooks=[CustomHook(obj)], - max_number_of_evaluations=1) - self.assertAlmostEqual(accuracy_value, self._expected_accuracy) - - # Validate that custom hook ran. - self.assertTrue(obj.hook_was_run) - - def _create_names_to_metrics(self, predictions, labels): - accuracy0, update_op0 = metrics.accuracy( - labels=labels, predictions=predictions) - accuracy1, update_op1 = metrics.accuracy( - labels=labels, predictions=predictions + 1) - - names_to_values = {'Accuracy': accuracy0, 'Another_accuracy': accuracy1} - names_to_updates = {'Accuracy': update_op0, 'Another_accuracy': update_op1} - return names_to_values, names_to_updates - - def _verify_summaries(self, output_dir, names_to_values): - """Verifies that the given `names_to_values` are found in the summaries. - - Args: - output_dir: An existing directory where summaries are found. - names_to_values: A dictionary of strings to values. - """ - # Check that the results were saved. The events file may have additional - # entries, e.g. the event version stamp, so have to parse things a bit. - output_filepath = glob.glob(os.path.join(output_dir, '*')) - self.assertEqual(len(output_filepath), 1) - - events = summary_iterator.summary_iterator(output_filepath[0]) - summaries = [e.summary for e in events if e.summary.value] - values = [] - for summary in summaries: - for value in summary.value: - values.append(value) - saved_results = {v.tag: v.simple_value for v in values} - for name in names_to_values: - self.assertAlmostEqual(names_to_values[name], saved_results[name]) - - def testLatestCheckpointReturnsNoneAfterTimeout(self): - start = time.time() - ret = evaluation_lib.wait_for_new_checkpoint( - '/non-existent-dir', 'foo', timeout=1.0, seconds_to_sleep=0.5) - end = time.time() - self.assertIsNone(ret) - # We've waited one time. - self.assertGreater(end, start + 0.5) - # The timeout kicked in. - self.assertLess(end, start + 1.1) - - def testTimeoutFnOnEvaluationLoop(self): - # We require a mutable object (e.g. list but not an int) to maintain state - # across calls of a nested function. - timeout_fn_calls = [0] - def _TimeoutFn(): - timeout_fn_calls[0] += 1 - return timeout_fn_calls[0] >= 3 - # Need not do any evaluation, but should just call timeout_fn repeatedly. - evaluation.evaluation_loop('', '', '', timeout=0, timeout_fn=_TimeoutFn) - self.assertEqual(timeout_fn_calls[0], 3) - - def testMonitorCheckpointsLoopTimeout(self): - ret = list( - evaluation_lib.checkpoints_iterator( - '/non-existent-dir', timeout=0)) - self.assertEqual(ret, []) - - def testWithEpochLimit(self): - predictions_limited = input.limit_epochs(self._predictions, num_epochs=1) - labels_limited = input.limit_epochs(self._labels, num_epochs=1) - - value_op, update_op = metrics.accuracy( - labels=labels_limited, predictions=predictions_limited) - - init_op = control_flow_ops.group(variables.global_variables_initializer(), - variables.local_variables_initializer()) - # Create checkpoint and log directories: - chkpt_dir = os.path.join(self.get_temp_dir(), 'tmp_logs/') - gfile.MakeDirs(chkpt_dir) - logdir = os.path.join(self.get_temp_dir(), 'tmp_logs2/') - gfile.MakeDirs(logdir) - - # Save initialized variables to a checkpoint directory: - saver = saver_lib.Saver() - with self.cached_session() as sess: - init_op.run() - saver.save(sess, os.path.join(chkpt_dir, 'chkpt')) - - # Now, run the evaluation loop: - accuracy_value = evaluation.evaluation_loop( - '', chkpt_dir, logdir, eval_op=update_op, final_op=value_op, - max_number_of_evaluations=1, num_evals=10000) - self.assertAlmostEqual(accuracy_value, self._expected_accuracy) - - -class SingleEvaluationTest(test.TestCase): - - def setUp(self): - super(SingleEvaluationTest, self).setUp() - - num_classes = 8 - batch_size = 16 - inputs, labels = GenerateTestData(num_classes, batch_size) - self._expected_accuracy = GroundTruthAccuracy(inputs, labels, batch_size) - - self._global_step = variables_lib.get_or_create_global_step() - self._inputs = constant_op.constant(inputs, dtype=dtypes.float32) - self._labels = constant_op.constant(labels, dtype=dtypes.int64) - self._predictions, self._scale = TestModel(self._inputs) - - def testErrorRaisedIfCheckpointDoesntExist(self): - checkpoint_path = os.path.join(self.get_temp_dir(), - 'this_file_doesnt_exist') - log_dir = os.path.join(self.get_temp_dir(), 'error_raised') - with self.assertRaises(ValueError): - evaluation.evaluate_once('', checkpoint_path, log_dir) - - def _prepareCheckpoint(self, checkpoint_path): - init_op = control_flow_ops.group(variables.global_variables_initializer(), - variables.local_variables_initializer()) - saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V1) - with self.cached_session() as sess: - sess.run(init_op) - saver.save(sess, checkpoint_path) - - def testRestoredModelPerformance(self): - checkpoint_path = os.path.join(self.get_temp_dir(), 'model.ckpt') - log_dir = os.path.join(self.get_temp_dir(), 'log_dir1/') - - # First, save out the current model to a checkpoint: - self._prepareCheckpoint(checkpoint_path) - - # Next, determine the metric to evaluate: - value_op, update_op = metrics.accuracy( - labels=self._labels, predictions=self._predictions) - - # Run the evaluation and verify the results: - accuracy_value = evaluation.evaluate_once( - '', checkpoint_path, log_dir, eval_op=update_op, final_op=value_op) - self.assertAlmostEqual(accuracy_value, self._expected_accuracy) - - def testAdditionalHooks(self): - checkpoint_path = os.path.join(self.get_temp_dir(), 'model.ckpt') - log_dir = os.path.join(self.get_temp_dir(), 'log_dir1/') - - # First, save out the current model to a checkpoint: - self._prepareCheckpoint(checkpoint_path) - - # Next, determine the metric to evaluate: - value_op, update_op = metrics.accuracy( - labels=self._labels, predictions=self._predictions) - - dumping_root = os.path.join(self.get_temp_dir(), 'tfdbg_dump_dir') - dumping_hook = hooks.DumpingDebugHook(dumping_root, log_usage=False) - try: - # Run the evaluation and verify the results: - accuracy_value = evaluation.evaluate_once( - '', checkpoint_path, log_dir, eval_op=update_op, final_op=value_op, - hooks=[dumping_hook]) - self.assertAlmostEqual(accuracy_value, self._expected_accuracy) - - dump = debug_data.DebugDumpDir( - glob.glob(os.path.join(dumping_root, 'run_*'))[0]) - # Here we simply assert that the dumped data has been loaded and is - # non-empty. We do not care about the detailed model-internal tensors or - # their values. - self.assertTrue(dump.dumped_tensor_data) - finally: - if os.path.isdir(dumping_root): - shutil.rmtree(dumping_root) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/learning.py b/tensorflow/contrib/slim/python/slim/learning.py deleted file mode 100644 index 605191b654f..00000000000 --- a/tensorflow/contrib/slim/python/slim/learning.py +++ /dev/null @@ -1,798 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains TF-Slim code for training models. - -This script contains various functions for training models. These include -manipulating gradients, creating a `train_op` (an operation that computes the -loss and applies the gradients) and a training loop function. The training loop -allows the user to pass in the `train_op` and runs the optimization according -to user-specified arguments. Note that the training loop uses the -tf.compat.v1.train.Supervisor and its managed_session in its implementation to -ensure the -ability of worker processes to recover from failures. - -************************************ -* A simple working training script * -************************************ - - # Load data and create the model: - images, labels = LoadData(...) - predictions = MyModel(images) - - # Define the loss: - slim.losses.log_loss(predictions, labels) - total_loss = slim.losses.get_total_loss() - - # Define the optimizer: - optimizer = tf.compat.v1.train.MomentumOptimizer(FLAGS.learning_rate, - FLAGS.momentum) - - # Create the train_op - train_op = slim.learning.create_train_op(total_loss, optimizer) - - # Run training. - slim.learning.train(train_op, my_log_dir) - -************************* -* Creating the train_op * -************************* - -In order to train, TF-Slim's train loop needs a train_op: an `Operation` that -(a) computes the loss, (b) applies the gradients to update the weights and -(c) returns the value of the loss. slim.learning.create_train_op creates -such an `Operation`. This function also provides the ability to manipulate -the gradients using a few arguments: - - # Create the train_op and clip the gradient norms: - train_op = slim.learning.create_train_op( - total_loss, - optimizer, - clip_gradient_norm=4) - - # Create the train_op and scale the gradients by providing a map from variable - # name (or variable) to a scaling coefficient: - gradient_multipliers = { - 'conv0/weights': 1.2, - 'fc8/weights': 3.4, - } - train_op = slim.learning.create_train_op( - total_loss, - optimizer, - gradient_multipliers=gradient_multipliers) - -**************************************************************** -* Performing additional (non-gradient) updates during training * -**************************************************************** - -Many networks utilize modules, like BatchNorm, that require performing a series -of non-gradient updates during training. slim.learning.create_train_op allows -a user to pass in a list of update_ops to call along with the gradient updates. - - train_op = slim.learning.create_train_op(total_loss, optimizer, update_ops) - -By default, slim.learning.create_train_op includes all update ops that are -part of the `tf.GraphKeys.UPDATE_OPS` collection. Additionally, TF-Slim's -slim.batch_norm function adds the moving mean and moving variance updates to -this collection. Consequently, users who want to use slim.batch_norm will not -need to take any additional steps in order to have the moving mean and moving -variance updates be computed. - -However, users with additional, specialized updates can either override the -default update ops or simply add additional update ops to the -`tf.GraphKeys.UPDATE_OPS` collection: - - # Force TF-Slim NOT to use ANY update_ops: - train_op = slim.learning.create_train_op( - total_loss, - optimizer, - update_ops=[]) - - # Use an alternative set of update ops: - train_op = slim.learning.create_train_op( - total_loss, - optimizer, - update_ops=my_other_update_ops) - - # Use an alternative set of update ops in addition to the default updates: - tf.compat.v1.add_to_collection(tf.GraphKeys.UPDATE_OPS, my_update0) - tf.compat.v1.add_to_collection(tf.GraphKeys.UPDATE_OPS, my_update1) - - train_op = slim.learning.create_train_op( - total_loss, - optimizer) - - # Which is the same as: - train_op = slim.learning.create_train_op( - total_loss, - optimizer, - update_ops=tf.compat.v1.get_collection(tf.GraphKeys.UPDATE_OPS)) - -****************************************** -* Initializing a model from a checkpoint * -****************************************** - -It is common to want to 'warm-start' a model from a pre-trained checkpoint. -TF-Slim provides a convenient mechanism for doing so: - - ... - - # Create the train_op - train_op = slim.learning.create_train_op(total_loss, optimizer) - - # Create the initial assignment op - checkpoint_path = '/path/to/old_model_checkpoint' - variables_to_restore = slim.get_model_variables() - init_assign_op, init_feed_dict = slim.assign_from_checkpoint( - checkpoint_path, variables_to_restore) - - # Create an initial assignment function. - def InitAssignFn(sess): - sess.run(init_assign_op, init_feed_dict) - - # Run training. - slim.learning.train(train_op, my_log_dir, init_fn=InitAssignFn) - -*************************************************************************** -* Initializing a model from a checkpoint whose variable names don't match * -*************************************************************************** - -At times, a user may want to initialize a new model with values from a -checkpoint whose variable names do not match those of the current model. In this -case, one needs to create a mapping from the checkpoint variable names to the -current model variables. This requires only a small modification of the code -above: - ... - # Creates a model with two variables, var0 and var1 - predictions = MyModel(images) - ... - - # Create the train_op - train_op = slim.learning.create_train_op(total_loss, optimizer) - - checkpoint_path = '/path/to/old_model_checkpoint' - - # Create the mapping: - variables_to_restore = { - 'name_var_0_in_checkpoint': slim.get_unique_variable('var0'), - 'name_var_1_in_checkpoint': slim.get_unique_variable('var1') - } - init_assign_op, init_feed_dict = slim.assign_from_checkpoint( - checkpoint_path, variables_to_restore) - - # Create an initial assignment function. - def InitAssignFn(sess): - sess.run(init_assign_op, init_feed_dict) - - # Run training. - slim.learning.train(train_op, my_log_dir, init_fn=InitAssignFn) - - -************************************************* -* Fine-Tuning Part of a model from a checkpoint * -************************************************* - -Rather than initializing all of the weights of a given model, we sometimes -only want to restore some of the weights from a checkpoint. To do this, one -need only filter those variables to initialize as follows: - - ... - - # Create the train_op - train_op = slim.learning.create_train_op(total_loss, optimizer) - - checkpoint_path = '/path/to/old_model_checkpoint' - - # Specify the variables to restore via a list of inclusion or exclusion - # patterns: - variables_to_restore = slim.get_variables_to_restore( - include=["conv"], exclude=["fc8", "fc9]) - # or - variables_to_restore = slim.get_variables_to_restore(exclude=["conv"]) - - init_assign_op, init_feed_dict = slim.assign_from_checkpoint( - checkpoint_path, variables_to_restore) - - # Create an initial assignment function. - def InitAssignFn(sess): - sess.run(init_assign_op, init_feed_dict) - - # Run training. - slim.learning.train(train_op, my_log_dir, init_fn=InitAssignFn) - -****************************************************** -* Initializing model variables from values in memory * -****************************************************** - -One may want to initialize the weights of a model from values from an arbitrary -source (a text document, matlab file, etc). While this is technically feasible -using plain TensorFlow, it also results in the values of your weights being -stored in the graph. For large models, this becomes prohibitively large. TF-Slim -allows you to perform this initial assignment without having to store the values -of the initial model in the graph itself by using placeholders and a feed -dictionary: - - ... - - # Create the train_op - train_op = slim.learning.create_train_op(total_loss, optimizer) - - # Create the mapping from variable names to values: - var0_initial_value = ReadFromDisk(...) - var1_initial_value = ReadFromDisk(...) - - var_names_to_values = { - 'var0': var0_initial_value, - 'var1': var1_initial_value, - } - init_assign_op, init_feed_dict = slim.assign_from_values(var_names_to_values) - - # Create an initial assignment function. - def InitAssignFn(sess): - sess.run(init_assign_op, init_feed_dict) - - # Run training. - slim.learning.train(train_op, my_log_dir, init_fn=InitAssignFn) -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import sys -import time - -from tensorflow.contrib.training.python.training import training -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import timeline -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.lib.io import file_io -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.summary import summary -from tensorflow.python.training import optimizer as tf_optimizer -from tensorflow.python.training import saver as tf_saver -from tensorflow.python.training import supervisor -from tensorflow.python.training import sync_replicas_optimizer -from tensorflow.python.training import training_util - -__all__ = [ - 'add_gradients_summaries', 'clip_gradient_norms', 'multiply_gradients', - 'create_train_op', 'train_step', 'train' -] - - -def clip_gradient_norms(gradients_to_variables, max_norm): - """Clips the gradients by the given value. - - Args: - gradients_to_variables: A list of gradient to variable pairs (tuples). - max_norm: the maximum norm value. - - Returns: - A list of clipped gradient to variable pairs. - """ - clipped_grads_and_vars = [] - for grad, var in gradients_to_variables: - if grad is not None: - if isinstance(grad, ops.IndexedSlices): - tmp = clip_ops.clip_by_norm(grad.values, max_norm) - grad = ops.IndexedSlices(tmp, grad.indices, grad.dense_shape) - else: - grad = clip_ops.clip_by_norm(grad, max_norm) - clipped_grads_and_vars.append((grad, var)) - return clipped_grads_and_vars - - -def multiply_gradients(grads_and_vars, gradient_multipliers): - """Multiply specified gradients. - - Args: - grads_and_vars: A list of gradient to variable pairs (tuples). - gradient_multipliers: A map from either `Variables` or `Variable` op names - to the coefficient by which the associated gradient should be scaled. - - Returns: - The updated list of gradient to variable pairs. - - Raises: - ValueError: If `grads_and_vars` is not a list or if `gradient_multipliers` - is empty or None or if `gradient_multipliers` is not a dictionary. - """ - if not isinstance(grads_and_vars, list): - raise ValueError('`grads_and_vars` must be a list.') - if not gradient_multipliers: - raise ValueError('`gradient_multipliers` is empty.') - if not isinstance(gradient_multipliers, dict): - raise ValueError('`gradient_multipliers` must be a dict.') - - multiplied_grads_and_vars = [] - for grad, var in grads_and_vars: - if var in gradient_multipliers or var.op.name in gradient_multipliers: - key = var if var in gradient_multipliers else var.op.name - if grad is None: - raise ValueError('Requested multiple of `None` gradient.') - - multiplier = gradient_multipliers[key] - if not isinstance(multiplier, ops.Tensor): - multiplier = constant_op.constant(multiplier, dtype=grad.dtype) - - if isinstance(grad, ops.IndexedSlices): - tmp = grad.values * multiplier - grad = ops.IndexedSlices(tmp, grad.indices, grad.dense_shape) - else: - grad *= multiplier - multiplied_grads_and_vars.append((grad, var)) - return multiplied_grads_and_vars - - -def add_gradients_summaries(grads_and_vars): - """Add summaries to gradients. - - Args: - grads_and_vars: A list of gradient to variable pairs (tuples). - - Returns: - The list of created summaries. - """ - summaries = [] - for grad, var in grads_and_vars: - if grad is not None: - if isinstance(grad, ops.IndexedSlices): - grad_values = grad.values - else: - grad_values = grad - summaries.append( - summary.histogram(var.op.name + '/gradient', grad_values)) - summaries.append( - summary.scalar(var.op.name + '/gradient_norm', - clip_ops.global_norm([grad_values]))) - else: - logging.info('Var %s has no gradient', var.op.name) - - return summaries - - -_USE_GLOBAL_STEP = 0 - - -def create_train_op(total_loss, - optimizer, - global_step=_USE_GLOBAL_STEP, - update_ops=None, - variables_to_train=None, - clip_gradient_norm=0, - summarize_gradients=False, - gate_gradients=tf_optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - gradient_multipliers=None, - check_numerics=True): - """Creates an `Operation` that evaluates the gradients and returns the loss. - - Args: - total_loss: A `Tensor` representing the total loss. - optimizer: A tf.Optimizer to use for computing the gradients. - global_step: A `Tensor` representing the global step variable. If left as - `_USE_GLOBAL_STEP`, then tf.contrib.framework.global_step() is used. - update_ops: An optional list of updates to execute. If `update_ops` is - `None`, then the update ops are set to the contents of the - `tf.GraphKeys.UPDATE_OPS` collection. If `update_ops` is not `None`, but - it doesn't contain all of the update ops in `tf.GraphKeys.UPDATE_OPS`, a - warning will be displayed. - variables_to_train: an optional list of variables to train. If None, it will - default to all tf.compat.v1.trainable_variables(). - clip_gradient_norm: If greater than 0 then the gradients would be clipped by - it. - summarize_gradients: Whether or not add summaries for each gradient. - gate_gradients: How to gate the computation of gradients. See tf.Optimizer. - aggregation_method: Specifies the method used to combine gradient terms. - Valid values are defined in the class `AggregationMethod`. - colocate_gradients_with_ops: Whether or not to try colocating the gradients - with the ops that generated them. - gradient_multipliers: A dictionary of either `Variables` or `Variable` op - names to the coefficient by which the associated gradient should be - scaled. - check_numerics: Whether or not we apply check_numerics. - - Returns: - A `Tensor` that when evaluated, computes the gradients and returns the total - loss value. - """ - - def transform_grads_fn(grads): - if gradient_multipliers: - with ops.name_scope('multiply_grads'): - grads = multiply_gradients(grads, gradient_multipliers) - - # Clip gradients. - if clip_gradient_norm > 0: - with ops.name_scope('clip_grads'): - grads = clip_gradient_norms(grads, clip_gradient_norm) - return grads - - return training.create_train_op( - total_loss=total_loss, - optimizer=optimizer, - global_step=global_step, - update_ops=update_ops, - variables_to_train=variables_to_train, - transform_grads_fn=transform_grads_fn, - summarize_gradients=summarize_gradients, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, - check_numerics=check_numerics) - - -def _wait_for_step(sess, global_step, step): - """Wait till the global step has reached at least 'step'. - - Args: - sess: A session. - global_step: A Tensor. - step: Int. The global step to reach. - """ - while True: - if training_util.global_step(sess, global_step) >= step: - break - time.sleep(1.0) - - -def train_step(sess, train_op, global_step, train_step_kwargs): - """Function that takes a gradient step and specifies whether to stop. - - Args: - sess: The current session. - train_op: An `Operation` that evaluates the gradients and returns the total - loss. - global_step: A `Tensor` representing the global training step. - train_step_kwargs: A dictionary of keyword arguments. - - Returns: - The total loss and a boolean indicating whether or not to stop training. - - Raises: - ValueError: if 'should_trace' is in `train_step_kwargs` but `logdir` is not. - """ - start_time = time.time() - - trace_run_options = None - run_metadata = None - if 'should_trace' in train_step_kwargs: - if 'logdir' not in train_step_kwargs: - raise ValueError('logdir must be present in train_step_kwargs when ' - 'should_trace is present') - if sess.run(train_step_kwargs['should_trace']): - trace_run_options = config_pb2.RunOptions( - trace_level=config_pb2.RunOptions.FULL_TRACE) - run_metadata = config_pb2.RunMetadata() - - total_loss, np_global_step = sess.run([train_op, global_step], - options=trace_run_options, - run_metadata=run_metadata) - time_elapsed = time.time() - start_time - - if run_metadata is not None: - tl = timeline.Timeline(run_metadata.step_stats) - trace = tl.generate_chrome_trace_format() - trace_filename = os.path.join(train_step_kwargs['logdir'], - 'tf_trace-%d.json' % np_global_step) - logging.info('Writing trace to %s', trace_filename) - file_io.write_string_to_file(trace_filename, trace) - if 'summary_writer' in train_step_kwargs: - train_step_kwargs['summary_writer'].add_run_metadata( - run_metadata, 'run_metadata-%d' % np_global_step) - - if 'should_log' in train_step_kwargs: - if sess.run(train_step_kwargs['should_log']): - logging.info('global step %d: loss = %.4f (%.3f sec/step)', - np_global_step, total_loss, time_elapsed) - - # TODO(nsilberman): figure out why we can't put this into sess.run. The - # issue right now is that the stop check depends on the global step. The - # increment of global step often happens via the train op, which used - # created using optimizer.apply_gradients. - # - # Since running `train_op` causes the global step to be incremented, one - # would expected that using a control dependency would allow the - # should_stop check to be run in the same session.run call: - # - # with ops.control_dependencies([train_op]): - # should_stop_op = ... - # - # However, this actually seems not to work on certain platforms. - if 'should_stop' in train_step_kwargs: - should_stop = sess.run(train_step_kwargs['should_stop']) - else: - should_stop = False - - return total_loss, should_stop - - -_USE_DEFAULT = 0 - - -def train(train_op, - logdir, - train_step_fn=train_step, - train_step_kwargs=_USE_DEFAULT, - log_every_n_steps=1, - graph=None, - master='', - is_chief=True, - global_step=None, - number_of_steps=None, - init_op=_USE_DEFAULT, - init_feed_dict=None, - local_init_op=_USE_DEFAULT, - init_fn=None, - ready_op=_USE_DEFAULT, - summary_op=_USE_DEFAULT, - save_summaries_secs=600, - summary_writer=_USE_DEFAULT, - startup_delay_steps=0, - saver=None, - save_interval_secs=600, - sync_optimizer=None, - session_config=None, - session_wrapper=None, - trace_every_n_steps=None, - ignore_live_threads=False): - """Runs a training loop using a TensorFlow supervisor. - - When the sync_optimizer is supplied, gradient updates are applied - synchronously. Otherwise, gradient updates are applied asynchronous. - - Args: - train_op: A `Tensor` that, when executed, will apply the gradients and - return the loss value. - logdir: The directory where training logs are written to. If None, model - checkpoints and summaries will not be written. - train_step_fn: The function to call in order to execute a single gradient - step. The function must have take exactly four arguments: the current - session, the `train_op` `Tensor`, a global step `Tensor` and a - dictionary. - train_step_kwargs: A dictionary which is passed to the `train_step_fn`. By - default, two `Boolean`, scalar ops called "should_stop" and "should_log" - are provided. - log_every_n_steps: The frequency, in terms of global steps, that the loss - and global step are logged. - graph: The graph to pass to the supervisor. If no graph is supplied the - default graph is used. - master: The address of the tensorflow master. - is_chief: Specifies whether or not the training is being run by the primary - replica during replica training. - global_step: The `Tensor` representing the global step. If left as `None`, - then training_util.get_or_create_global_step(), that is, - tf.contrib.framework.global_step() is used. - number_of_steps: The max number of gradient steps to take during training, - as measured by 'global_step': training will stop if global_step is greater - than 'number_of_steps'. If the value is left as None, training proceeds - indefinitely. - init_op: The initialization operation. If left to its default value, then - the session is initialized by calling - `tf.compat.v1.global_variables_initializer()`. - init_feed_dict: A feed dictionary to use when executing the `init_op`. - local_init_op: The local initialization operation. If left to its default - value, then the session is initialized by calling - `tf.compat.v1.local_variables_initializer()` and - `tf.compat.v1.tables_initializer()`. - init_fn: An optional callable to be executed after `init_op` is called. The - callable must accept one argument, the session being initialized. - ready_op: Operation to check if the model is ready to use. If left to its - default value, then the session checks for readiness by calling - `tf.compat.v1.report_uninitialized_variables()`. - summary_op: The summary operation. - save_summaries_secs: How often, in seconds, to save summaries. - summary_writer: `SummaryWriter` to use. Can be `None` to indicate that no - summaries should be written. If unset, we create a SummaryWriter. - startup_delay_steps: The number of steps to wait for before beginning. Note - that this must be 0 if a sync_optimizer is supplied. - saver: Saver to save checkpoints. If None, a default one will be created and - used. - save_interval_secs: How often, in seconds, to save the model to `logdir`. - sync_optimizer: an instance of tf.compat.v1.train.SyncReplicasOptimizer, or - a list of them. If the argument is supplied, gradient updates will be - synchronous. If left as `None`, gradient updates will be asynchronous. - session_config: An instance of `tf.compat.v1.ConfigProto` that will be used - to configure the `Session`. If left as `None`, the default will be used. - session_wrapper: A function that takes a `tf.compat.v1.Session` object as - the only argument and returns a wrapped session object that has the same - methods that the original object has, or `None`. Iff not `None`, the - wrapped object will be used for training. - trace_every_n_steps: produce and save a `Timeline` in Chrome trace format - and add it to the summaries every `trace_every_n_steps`. If None, no trace - information will be produced or saved. - ignore_live_threads: If `True` ignores threads that remain running after a - grace period when stopping the supervisor, instead of raising a - RuntimeError. - - Returns: - the value of the loss function after training. - - Raises: - ValueError: if `train_op` is empty or if `startup_delay_steps` is - non-zero when `sync_optimizer` is supplied, if `number_of_steps` is - negative, or if `trace_every_n_steps` is not `None` and no `logdir` is - provided. - """ - if train_op is None: - raise ValueError('train_op cannot be None.') - - if logdir is None: - if summary_op != _USE_DEFAULT: - raise ValueError('Cannot provide summary_op because logdir=None') - if saver is not None: - raise ValueError('Cannot provide saver because logdir=None') - if trace_every_n_steps is not None: - raise ValueError('Cannot provide trace_every_n_steps because ' - 'logdir=None') - - if isinstance(sync_optimizer, sync_replicas_optimizer.SyncReplicasOptimizer): - sync_optimizer = [sync_optimizer] - if sync_optimizer is not None and startup_delay_steps > 0: - raise ValueError( - 'startup_delay_steps must be zero when sync_optimizer is supplied.') - - if number_of_steps is not None and number_of_steps <= 0: - raise ValueError( - '`number_of_steps` must be either None or a positive number.') - - graph = graph or ops.get_default_graph() - with graph.as_default(): - if global_step is None: - global_step = training_util.get_or_create_global_step() - saver = saver or tf_saver.Saver() - - if sync_optimizer is not None: - for opt in sync_optimizer: - if not isinstance(opt, sync_replicas_optimizer.SyncReplicasOptimizer): - raise ValueError( - '`sync_optimizer` must be a tf.train.SyncReplicasOptimizer.') - - with ops.name_scope('init_ops'): - if init_op == _USE_DEFAULT: - init_op = variables.global_variables_initializer() - - if ready_op == _USE_DEFAULT: - ready_op = variables.report_uninitialized_variables() - - if local_init_op == _USE_DEFAULT: - local_init_op = control_flow_ops.group( - variables.local_variables_initializer(), - lookup_ops.tables_initializer()) - - if sync_optimizer is not None and isinstance(sync_optimizer, list): - with ops.control_dependencies( - [local_init_op] if local_init_op is not None else []): - if is_chief: - local_init_op = control_flow_ops.group( - *[opt.chief_init_op for opt in sync_optimizer]) - else: - local_init_op = control_flow_ops.group( - *[opt.local_step_init_op for opt in sync_optimizer]) - ready_for_local_init_op = control_flow_ops.group( - *[opt.ready_for_local_init_op for opt in sync_optimizer]) - else: - ready_for_local_init_op = None - - if summary_op == _USE_DEFAULT: - summary_op = summary.merge_all() - - if summary_writer == _USE_DEFAULT: - summary_writer = supervisor.Supervisor.USE_DEFAULT - - if is_chief and sync_optimizer is not None: - # Need to create these BEFORE the supervisor finalizes the graph: - init_tokens_op = [opt.get_init_tokens_op() for opt in sync_optimizer] - chief_queue_runner = [ - opt.get_chief_queue_runner() for opt in sync_optimizer - ] - - if train_step_kwargs == _USE_DEFAULT: - with ops.name_scope('train_step'): - train_step_kwargs = {} - - if number_of_steps: - should_stop_op = math_ops.greater_equal(global_step, number_of_steps) - else: - should_stop_op = constant_op.constant(False) - train_step_kwargs['should_stop'] = should_stop_op - if log_every_n_steps > 0: - train_step_kwargs['should_log'] = math_ops.equal( - math_ops.mod(global_step, log_every_n_steps), 0) - if is_chief and trace_every_n_steps is not None: - train_step_kwargs['should_trace'] = math_ops.equal( - math_ops.mod(global_step, trace_every_n_steps), 0) - train_step_kwargs['logdir'] = logdir - - sv = supervisor.Supervisor( - graph=graph, - is_chief=is_chief, - logdir=logdir, - init_op=init_op, - init_feed_dict=init_feed_dict, - local_init_op=local_init_op, - ready_for_local_init_op=ready_for_local_init_op, - ready_op=ready_op, - summary_op=summary_op, - summary_writer=summary_writer, - global_step=global_step, - saver=saver, - save_summaries_secs=save_summaries_secs, - save_model_secs=save_interval_secs, - init_fn=init_fn) - - if summary_writer is not None: - train_step_kwargs['summary_writer'] = sv.summary_writer - - total_loss = None - should_retry = True - while should_retry: - try: - should_retry = False - with sv.managed_session( - master, start_standard_services=False, config=session_config) as sess: - logging.info('Starting Session.') - if session_wrapper is not None: - logging.info('Wrapping session with wrapper function: %s', - session_wrapper) - sess = session_wrapper(sess) - if is_chief: - if logdir: - sv.start_standard_services(sess) - elif startup_delay_steps > 0: - # (use sys.maxsize because sys.maxint doesn't exist in Python 3) - _wait_for_step( - sess, global_step, - min(startup_delay_steps, number_of_steps or sys.maxsize)) - threads = sv.start_queue_runners(sess) - logging.info('Starting Queues.') - if is_chief and sync_optimizer is not None: - sv.start_queue_runners(sess, chief_queue_runner) - sess.run(init_tokens_op) - try: - while not sv.should_stop(): - total_loss, should_stop = train_step_fn(sess, train_op, global_step, - train_step_kwargs) - if should_stop: - logging.info('Stopping Training.') - sv.request_stop() - break - except errors.OutOfRangeError as e: - # OutOfRangeError is thrown when epoch limit per - # tf.compat.v1.train.limit_epochs is reached. - logging.info('Caught OutOfRangeError. Stopping Training. %s', e) - if logdir and sv.is_chief: - logging.info('Finished training! Saving model to disk.') - sv.saver.save(sess, sv.save_path, global_step=sv.global_step) - sv.stop( - threads, - close_summary_writer=True, - ignore_live_threads=ignore_live_threads) - - except errors.AbortedError: - # Always re-run on AbortedError as it indicates a restart of one of the - # distributed tensorflow servers. - logging.info('Retrying training!') - should_retry = True - - return total_loss diff --git a/tensorflow/contrib/slim/python/slim/learning_test.py b/tensorflow/contrib/slim/python/slim/learning_test.py deleted file mode 100644 index aefc07696b9..00000000000 --- a/tensorflow/contrib/slim/python/slim/learning_test.py +++ /dev/null @@ -1,985 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.learning.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import glob -import os -import tempfile - -import numpy as np -from numpy import testing as np_testing - -from tensorflow.contrib.framework.python.ops import variables as variables_lib2 -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.losses.python.losses import loss_ops -from tensorflow.contrib.slim.python.slim import learning -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session -from tensorflow.python.debug.lib import debug_data -from tensorflow.python.debug.wrappers import dumping_wrapper as dumping_wrapper_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.platform import test -from tensorflow.python.summary import summary -from tensorflow.python.training import gradient_descent -from tensorflow.python.training import input as input_lib -from tensorflow.python.training import saver as saver_lib - - -class ClipGradientNormsTest(test.TestCase): - - def clip_values(self, arr): - norm = np.sqrt(np.sum(arr**2)) - if norm > self._max_norm: - return self._max_norm * arr / np.sqrt(np.sum(arr**2)) - return arr - - def setUp(self): - np.random.seed(0) - - self._max_norm = 1.0 - self._grad_vec = np.array([1., 2., 3.]) - self._clipped_grad_vec = self.clip_values(self._grad_vec) - self._zero_vec = np.zeros(self._grad_vec.size) - - def testOrdinaryGradIsClippedCorrectly(self): - gradient = constant_op.constant(self._grad_vec, dtype=dtypes.float32) - variable = variables_lib.Variable(self._zero_vec, dtype=dtypes.float32) - gradients_to_variables = (gradient, variable) - [gradients_to_variables - ] = learning.clip_gradient_norms([gradients_to_variables], self._max_norm) - - # Ensure the variable passed through. - self.assertEqual(gradients_to_variables[1], variable) - - with self.cached_session() as sess: - actual_gradient = sess.run(gradients_to_variables[0]) - np_testing.assert_almost_equal(actual_gradient, self._clipped_grad_vec) - - def testNoneGradPassesThroughCorrectly(self): - gradient = None - variable = variables_lib.Variable(self._zero_vec, dtype=dtypes.float32) - - gradients_to_variables = (gradient, variable) - [gradients_to_variables - ] = learning.clip_gradient_norms([gradients_to_variables], self._max_norm) - - self.assertEqual(gradients_to_variables[0], None) - self.assertEqual(gradients_to_variables[1], variable) - - def testIndexedSlicesGradIsClippedCorrectly(self): - sparse_grad_indices = np.array([0, 1, 4]) - sparse_grad_dense_shape = [self._grad_vec.size] - - values = constant_op.constant(self._grad_vec, dtype=dtypes.float32) - indices = constant_op.constant(sparse_grad_indices, dtype=dtypes.int32) - dense_shape = constant_op.constant( - sparse_grad_dense_shape, dtype=dtypes.int32) - - gradient = ops.IndexedSlices(values, indices, dense_shape) - variable = variables_lib.Variable(self._zero_vec, dtype=dtypes.float32) - - gradients_to_variables = (gradient, variable) - gradients_to_variables = learning.clip_gradient_norms( - [gradients_to_variables], self._max_norm)[0] - - # Ensure the built IndexedSlice has the right form. - self.assertEqual(gradients_to_variables[1], variable) - self.assertEqual(gradients_to_variables[0].indices, indices) - self.assertEqual(gradients_to_variables[0].dense_shape, dense_shape) - - with session.Session() as sess: - actual_gradient = sess.run(gradients_to_variables[0].values) - np_testing.assert_almost_equal(actual_gradient, self._clipped_grad_vec) - - -class MultiplyGradientsTest(test.TestCase): - - def setUp(self): - np.random.seed(0) - self._multiplier = 3.7 - self._grad_vec = np.array([1., 2., 3.]) - self._multiplied_grad_vec = np.multiply(self._grad_vec, self._multiplier) - - def testNonListGradsRaisesError(self): - gradient = constant_op.constant(self._grad_vec, dtype=dtypes.float32) - variable = variables_lib.Variable(array_ops.zeros_like(gradient)) - grad_to_var = (gradient, variable) - gradient_multipliers = {variable: self._multiplier} - with self.assertRaises(ValueError): - learning.multiply_gradients(grad_to_var, gradient_multipliers) - - def testEmptyMultiplesRaisesError(self): - gradient = constant_op.constant(self._grad_vec, dtype=dtypes.float32) - variable = variables_lib.Variable(array_ops.zeros_like(gradient)) - grad_to_var = (gradient, variable) - with self.assertRaises(ValueError): - learning.multiply_gradients([grad_to_var], {}) - - def testNonDictMultiplierRaisesError(self): - gradient = constant_op.constant(self._grad_vec, dtype=dtypes.float32) - variable = variables_lib.Variable(array_ops.zeros_like(gradient)) - grad_to_var = (gradient, variable) - with self.assertRaises(ValueError): - learning.multiply_gradients([grad_to_var], 3) - - def testMultipleOfNoneGradRaisesError(self): - gradient = constant_op.constant(self._grad_vec, dtype=dtypes.float32) - variable = variables_lib.Variable(array_ops.zeros_like(gradient)) - grad_to_var = (None, variable) - gradient_multipliers = {variable: self._multiplier} - with self.assertRaises(ValueError): - learning.multiply_gradients(grad_to_var, gradient_multipliers) - - def testMultipleGradientsWithVariables(self): - gradient = constant_op.constant(self._grad_vec, dtype=dtypes.float32) - variable = variables_lib.Variable(array_ops.zeros_like(gradient)) - grad_to_var = (gradient, variable) - gradient_multipliers = {variable: self._multiplier} - - [grad_to_var] = learning.multiply_gradients([grad_to_var], - gradient_multipliers) - - # Ensure the variable passed through. - self.assertEqual(grad_to_var[1], variable) - - with self.cached_session() as sess: - actual_gradient = sess.run(grad_to_var[0]) - np_testing.assert_almost_equal(actual_gradient, self._multiplied_grad_vec, - 5) - - def testIndexedSlicesGradIsMultiplied(self): - values = constant_op.constant(self._grad_vec, dtype=dtypes.float32) - indices = constant_op.constant([0, 1, 2], dtype=dtypes.int32) - dense_shape = constant_op.constant([self._grad_vec.size], - dtype=dtypes.int32) - - gradient = ops.IndexedSlices(values, indices, dense_shape) - variable = variables_lib.Variable(array_ops.zeros((1, 3))) - grad_to_var = (gradient, variable) - gradient_multipliers = {variable: self._multiplier} - - [grad_to_var] = learning.multiply_gradients([grad_to_var], - gradient_multipliers) - - # Ensure the built IndexedSlice has the right form. - self.assertEqual(grad_to_var[1], variable) - self.assertEqual(grad_to_var[0].indices, indices) - self.assertEqual(grad_to_var[0].dense_shape, dense_shape) - - with self.cached_session() as sess: - actual_gradient = sess.run(grad_to_var[0].values) - np_testing.assert_almost_equal(actual_gradient, self._multiplied_grad_vec, - 5) - - def testTensorMultiplierOfGradient(self): - gradient = constant_op.constant(self._grad_vec, dtype=dtypes.float32) - variable = variables_lib.Variable(array_ops.zeros_like(gradient)) - multiplier_flag = variables_lib.Variable(True) - tensor_multiplier = array_ops.where_v2(multiplier_flag, self._multiplier, - 1.0) - grad_to_var = (gradient, variable) - gradient_multipliers = {variable: tensor_multiplier} - - [grad_to_var] = learning.multiply_gradients([grad_to_var], - gradient_multipliers) - - with self.cached_session() as sess: - sess.run(variables_lib.global_variables_initializer()) - gradient_true_flag = sess.run(grad_to_var[0]) - sess.run(multiplier_flag.assign(False)) - gradient_false_flag = sess.run(grad_to_var[0]) - np_testing.assert_almost_equal(gradient_true_flag, - self._multiplied_grad_vec, 5) - np_testing.assert_almost_equal(gradient_false_flag, self._grad_vec, 5) - - -def LogisticClassifier(inputs): - return layers.fully_connected(inputs, 1, activation_fn=math_ops.sigmoid) - - -def BatchNormClassifier(inputs): - inputs = layers.batch_norm(inputs, decay=0.1, fused=True) - return layers.fully_connected(inputs, 1, activation_fn=math_ops.sigmoid) - - -class TrainBNClassifierTest(test.TestCase): - - def setUp(self): - # Create an easy training set: - np.random.seed(0) - - self._inputs = np.zeros((16, 4)) - self._labels = np.random.randint(0, 2, size=(16, 1)).astype(np.float32) - - for i in range(16): - j = int(2 * self._labels[i] + np.random.randint(0, 2)) - self._inputs[i, j] = 1 - - def testTrainWithNoInitAssignCanAchieveZeroLoss(self): - logdir = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') - g = ops.Graph() - with g.as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = BatchNormClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - loss = learning.train( - train_op, logdir, number_of_steps=300, log_every_n_steps=10) - self.assertLess(loss, .1) - - -class CreateTrainOpTest(test.TestCase): - - def setUp(self): - # Create an easy training set: - np.random.seed(0) - self._inputs = np.random.rand(16, 4).astype(np.float32) - self._labels = np.random.randint(0, 2, size=(16, 1)).astype(np.float32) - - def _addBesselsCorrection(self, sample_size, expected_var): - correction_factor = sample_size / (sample_size - 1) - expected_var *= correction_factor - return expected_var - - def testUseUpdateOps(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - expected_mean = np.mean(self._inputs, axis=(0)) - expected_var = np.var(self._inputs, axis=(0)) - expected_var = self._addBesselsCorrection(16, expected_var) - - tf_predictions = BatchNormClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - moving_mean = variables_lib2.get_variables_by_name('moving_mean')[0] - moving_variance = variables_lib2.get_variables_by_name( - 'moving_variance')[0] - - with session.Session() as sess: - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - mean, variance = sess.run([moving_mean, moving_variance]) - # After initialization moving_mean == 0 and moving_variance == 1. - self.assertAllClose(mean, [0] * 4) - self.assertAllClose(variance, [1] * 4) - - for _ in range(10): - sess.run([train_op]) - mean = moving_mean.eval() - variance = moving_variance.eval() - # After 10 updates with decay 0.1 moving_mean == expected_mean and - # moving_variance == expected_var. - self.assertAllClose(mean, expected_mean) - self.assertAllClose(variance, expected_var) - - def testEmptyUpdateOps(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = BatchNormClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer, update_ops=[]) - - moving_mean = variables_lib2.get_variables_by_name('moving_mean')[0] - moving_variance = variables_lib2.get_variables_by_name( - 'moving_variance')[0] - - with session.Session() as sess: - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - mean, variance = sess.run([moving_mean, moving_variance]) - # After initialization moving_mean == 0 and moving_variance == 1. - self.assertAllClose(mean, [0] * 4) - self.assertAllClose(variance, [1] * 4) - - for _ in range(10): - sess.run([train_op]) - mean = moving_mean.eval() - variance = moving_variance.eval() - # Since we skip update_ops the moving_vars are not updated. - self.assertAllClose(mean, [0] * 4) - self.assertAllClose(variance, [1] * 4) - - def testUseGlobalStep(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = BatchNormClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - global_step = variables_lib2.get_or_create_global_step() - - with session.Session() as sess: - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - - for _ in range(10): - sess.run([train_op]) - global_step = global_step.eval() - # After 10 updates global_step should be 10. - self.assertAllClose(global_step, 10) - - def testNoneGlobalStep(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = BatchNormClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op( - total_loss, optimizer, global_step=None) - - global_step = variables_lib2.get_or_create_global_step() - - with session.Session() as sess: - # Initialize all variables - sess.run(variables_lib.global_variables_initializer()) - - for _ in range(10): - sess.run([train_op]) - global_step = global_step.eval() - # Since train_op don't use global_step it shouldn't change. - self.assertAllClose(global_step, 0) - - def testRecordTrainOpInCollection(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - train_op = learning.create_train_op(total_loss, optimizer) - - # Make sure the training op was recorded in the proper collection - self.assertTrue(train_op in ops.get_collection(ops.GraphKeys.TRAIN_OP)) - - -class TrainTest(test.TestCase): - - def setUp(self): - # Create an easy training set: - np.random.seed(0) - - self._inputs = np.zeros((16, 4)) - self._labels = np.random.randint(0, 2, size=(16, 1)).astype(np.float32) - - for i in range(16): - j = int(2 * self._labels[i] + np.random.randint(0, 2)) - self._inputs[i, j] = 1 - - def testTrainWithNonDefaultGraph(self): - logdir = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') - g = ops.Graph() - with g.as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - loss = learning.train( - train_op, logdir, number_of_steps=300, log_every_n_steps=10, graph=g) - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - - def testTrainWithNoneAsLogdir(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - loss = learning.train( - train_op, None, number_of_steps=300, log_every_n_steps=10) - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - - def testTrainWithSessionConfig(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - session_config = config_pb2.ConfigProto(allow_soft_placement=True) - loss = learning.train( - train_op, - None, - number_of_steps=300, - log_every_n_steps=10, - session_config=session_config) - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - - def testTrainWithSessionWrapper(self): - """Test that slim.learning.train can take `session_wrapper` args. - - One of the applications of `session_wrapper` is the wrappers of TensorFlow - Debugger (tfdbg), which intercept methods calls to `tf.compat.v1.Session` - (e.g., run) - to achieve debugging. `DumpingDebugWrapperSession` is used here for testing - purpose. - """ - dump_root = tempfile.mkdtemp() - - def dumping_wrapper(sess): # pylint: disable=invalid-name - return dumping_wrapper_lib.DumpingDebugWrapperSession(sess, dump_root) - - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - loss = learning.train( - train_op, None, number_of_steps=1, session_wrapper=dumping_wrapper) - self.assertIsNotNone(loss) - - run_root = glob.glob(os.path.join(dump_root, 'run_*'))[-1] - dump = debug_data.DebugDumpDir(run_root) - self.assertAllEqual(0, - dump.get_tensors('global_step', 0, 'DebugIdentity')[0]) - - def testTrainWithTrace(self): - logdir = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - summary.scalar('total_loss', total_loss) - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - loss = learning.train( - train_op, - logdir, - number_of_steps=300, - log_every_n_steps=10, - trace_every_n_steps=100) - self.assertIsNotNone(loss) - for trace_step in [1, 101, 201]: - trace_filename = 'tf_trace-%d.json' % trace_step - self.assertTrue(os.path.isfile(os.path.join(logdir, trace_filename))) - - def testTrainWithNoneAsLogdirWhenUsingSummariesRaisesError(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - summary.scalar('total_loss', total_loss) - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - summary_op = summary.merge_all() - - with self.assertRaises(ValueError): - learning.train( - train_op, None, number_of_steps=300, summary_op=summary_op) - - def testTrainWithNoneAsLogdirWhenUsingTraceRaisesError(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - with self.assertRaises(ValueError): - learning.train( - train_op, None, number_of_steps=300, trace_every_n_steps=10) - - def testTrainWithNoneAsLogdirWhenUsingSaverRaisesError(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - saver = saver_lib.Saver() - - with self.assertRaises(ValueError): - learning.train( - train_op, None, init_op=None, number_of_steps=300, saver=saver) - - def testTrainWithNoneAsInitWhenUsingVarsRaisesError(self): - logdir = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - with self.assertRaises(RuntimeError): - learning.train(train_op, logdir, init_op=None, number_of_steps=300) - - def testTrainWithNoInitAssignCanAchieveZeroLoss(self): - logdir = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - loss = learning.train( - train_op, logdir, number_of_steps=300, log_every_n_steps=10) - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - - def testTrainWithLocalVariable(self): - logdir = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - local_multiplier = variables_lib2.local_variable(1.0) - - tf_predictions = LogisticClassifier(tf_inputs) * local_multiplier - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - loss = learning.train( - train_op, logdir, number_of_steps=300, log_every_n_steps=10) - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - - def testResumeTrainAchievesRoughlyTheSameLoss(self): - logdir = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') - number_of_steps = [300, 301, 305] - - for i in range(len(number_of_steps)): - with ops.Graph().as_default(): - random_seed.set_random_seed(i) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - loss = learning.train( - train_op, - logdir, - number_of_steps=number_of_steps[i], - log_every_n_steps=10) - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - - def create_train_op(self, learning_rate=1.0, gradient_multiplier=1.0): - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer( - learning_rate=learning_rate) - - if gradient_multiplier != 1.0: - variables = variables_lib.trainable_variables() - gradient_multipliers = {var: gradient_multiplier for var in variables} - else: - gradient_multipliers = None - - return learning.create_train_op( - total_loss, optimizer, gradient_multipliers=gradient_multipliers) - - def testTrainWithInitFromCheckpoint(self): - logdir1 = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs1') - logdir2 = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs2') - - # First, train the model one step (make sure the error is high). - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - train_op = self.create_train_op() - loss = learning.train(train_op, logdir1, number_of_steps=1) - self.assertGreater(loss, .5) - - # Next, train the model to convergence. - with ops.Graph().as_default(): - random_seed.set_random_seed(1) - train_op = self.create_train_op() - loss = learning.train( - train_op, logdir1, number_of_steps=300, log_every_n_steps=10) - self.assertIsNotNone(loss) - self.assertLess(loss, .02) - - # Finally, advance the model a single step and validate that the loss is - # still low. - with ops.Graph().as_default(): - random_seed.set_random_seed(2) - train_op = self.create_train_op() - - model_variables = variables_lib.global_variables() - model_path = os.path.join(logdir1, 'model.ckpt-300') - - init_op = variables_lib.global_variables_initializer() - op, init_feed_dict = variables_lib2.assign_from_checkpoint( - model_path, model_variables) - - def InitAssignFn(sess): - sess.run(op, init_feed_dict) - - loss = learning.train( - train_op, - logdir2, - number_of_steps=1, - init_op=init_op, - init_fn=InitAssignFn) - - self.assertIsNotNone(loss) - self.assertLess(loss, .02) - - def testTrainWithInitFromFn(self): - logdir1 = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs1') - logdir2 = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs2') - - # First, train the model one step (make sure the error is high). - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - train_op = self.create_train_op() - loss = learning.train(train_op, logdir1, number_of_steps=1) - self.assertGreater(loss, .5) - - # Next, train the model to convergence. - with ops.Graph().as_default(): - random_seed.set_random_seed(1) - train_op = self.create_train_op() - loss = learning.train( - train_op, logdir1, number_of_steps=300, log_every_n_steps=10) - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - - # Finally, advance the model a single step and validate that the loss is - # still low. - with ops.Graph().as_default(): - random_seed.set_random_seed(2) - train_op = self.create_train_op() - - model_variables = variables_lib.global_variables() - model_path = os.path.join(logdir1, 'model.ckpt-300') - saver = saver_lib.Saver(model_variables) - - def RestoreFn(sess): - saver.restore(sess, model_path) - - loss = learning.train( - train_op, logdir2, number_of_steps=1, init_fn=RestoreFn) - - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - - def ModelLoss(self): - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = LogisticClassifier(tf_inputs) - loss_ops.log_loss(tf_predictions, tf_labels) - return loss_ops.get_total_loss() - - def testTrainAllVarsHasLowerLossThanTrainSubsetOfVars(self): - logdir1 = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs1') - - # First, train only the weights of the model. - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - total_loss = self.ModelLoss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - weights = variables_lib2.get_variables_by_name('weights') - - train_op = learning.create_train_op( - total_loss, optimizer, variables_to_train=weights) - - loss = learning.train( - train_op, logdir1, number_of_steps=200, log_every_n_steps=10) - self.assertGreater(loss, .015) - self.assertLess(loss, .05) - - # Next, train the biases of the model. - with ops.Graph().as_default(): - random_seed.set_random_seed(1) - total_loss = self.ModelLoss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - biases = variables_lib2.get_variables_by_name('biases') - - train_op = learning.create_train_op( - total_loss, optimizer, variables_to_train=biases) - - loss = learning.train( - train_op, logdir1, number_of_steps=300, log_every_n_steps=10) - self.assertGreater(loss, .015) - self.assertLess(loss, .05) - - # Finally, train both weights and bias to get lower loss. - with ops.Graph().as_default(): - random_seed.set_random_seed(2) - total_loss = self.ModelLoss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - loss = learning.train( - train_op, logdir1, number_of_steps=400, log_every_n_steps=10) - - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - - def testTrainingSubsetsOfVariablesOnlyUpdatesThoseVariables(self): - # First, train only the weights of the model. - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - total_loss = self.ModelLoss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - weights, biases = variables_lib2.get_variables() - - train_op = learning.create_train_op(total_loss, optimizer) - train_weights = learning.create_train_op( - total_loss, optimizer, variables_to_train=[weights]) - train_biases = learning.create_train_op( - total_loss, optimizer, variables_to_train=[biases]) - - with session.Session() as sess: - # Initialize the variables. - sess.run(variables_lib.global_variables_initializer()) - - # Get the initial weights and biases values. - weights_values, biases_values = sess.run([weights, biases]) - self.assertGreater(np.linalg.norm(weights_values), 0) - self.assertAlmostEqual(np.linalg.norm(biases_values), 0) - - # Update weights and biases. - loss = sess.run(train_op) - self.assertGreater(loss, .5) - new_weights, new_biases = sess.run([weights, biases]) - - # Check that the weights and biases have been updated. - self.assertGreater(np.linalg.norm(weights_values - new_weights), 0) - self.assertGreater(np.linalg.norm(biases_values - new_biases), 0) - - weights_values, biases_values = new_weights, new_biases - - # Update only weights. - loss = sess.run(train_weights) - self.assertGreater(loss, .5) - new_weights, new_biases = sess.run([weights, biases]) - - # Check that the weights have been updated, but biases have not. - self.assertGreater(np.linalg.norm(weights_values - new_weights), 0) - self.assertAlmostEqual(np.linalg.norm(biases_values - new_biases), 0) - weights_values = new_weights - - # Update only biases. - loss = sess.run(train_biases) - self.assertGreater(loss, .5) - new_weights, new_biases = sess.run([weights, biases]) - - # Check that the biases have been updated, but weights have not. - self.assertAlmostEqual(np.linalg.norm(weights_values - new_weights), 0) - self.assertGreater(np.linalg.norm(biases_values - new_biases), 0) - - def testTrainWithAlteredGradients(self): - # Use the same learning rate but different gradient multipliers - # to train two models. Model with equivalently larger learning - # rate (i.e., learning_rate * gradient_multiplier) has smaller - # training loss. - logdir1 = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs1') - logdir2 = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs2') - - multipliers = [1., 1000.] - number_of_steps = 10 - losses = [] - learning_rate = 0.001 - - # First, train the model with equivalently smaller learning rate. - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - train_op = self.create_train_op( - learning_rate=learning_rate, gradient_multiplier=multipliers[0]) - loss = learning.train(train_op, logdir1, number_of_steps=number_of_steps) - losses.append(loss) - self.assertGreater(loss, .5) - - # Second, train the model with equivalently larger learning rate. - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - train_op = self.create_train_op( - learning_rate=learning_rate, gradient_multiplier=multipliers[1]) - loss = learning.train(train_op, logdir2, number_of_steps=number_of_steps) - losses.append(loss) - self.assertIsNotNone(loss) - self.assertLess(loss, .5) - - # The loss of the model trained with larger learning rate should - # be smaller. - self.assertGreater(losses[0], losses[1]) - - def testTrainWithEpochLimit(self): - logdir = os.path.join( - tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - tf_inputs_limited = input_lib.limit_epochs(tf_inputs, num_epochs=300) - tf_labels_limited = input_lib.limit_epochs(tf_labels, num_epochs=300) - - tf_predictions = LogisticClassifier(tf_inputs_limited) - loss_ops.log_loss(tf_predictions, tf_labels_limited) - total_loss = loss_ops.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = learning.create_train_op(total_loss, optimizer) - - loss = learning.train(train_op, logdir, log_every_n_steps=10) - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - self.assertTrue(os.path.isfile('{}/model.ckpt-300.index'.format(logdir))) - self.assertTrue( - os.path.isfile('{}/model.ckpt-300.data-00000-of-00001'.format(logdir))) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/model_analyzer.py b/tensorflow/contrib/slim/python/slim/model_analyzer.py deleted file mode 100644 index aad968997ea..00000000000 --- a/tensorflow/contrib/slim/python/slim/model_analyzer.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tools for analyzing the operations and variables in a TensorFlow graph. - -To analyze the operations in a graph: - - images, labels = LoadData(...) - predictions = MyModel(images) - - slim.model_analyzer.analyze_ops(tf.compat.v1.get_default_graph(), - print_info=True) - -To analyze the model variables in a graph: - - variables = tf.compat.v1.model_variables() - slim.model_analyzer.analyze_vars(variables, print_info=False) -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -def tensor_description(var): - """Returns a compact and informative string about a tensor. - - Args: - var: A tensor variable. - - Returns: - a string with type and size, e.g.: (float32 1x8x8x1024). - """ - description = '(' + str(var.dtype.name) + ' ' - sizes = var.get_shape() - for i, size in enumerate(sizes): - description += str(size) - if i < len(sizes) - 1: - description += 'x' - description += ')' - return description - - -def analyze_ops(graph, print_info=False): - """Compute the estimated size of the ops.outputs in the graph. - - Args: - graph: the graph containing the operations. - print_info: Optional, if true print ops and their outputs. - - Returns: - total size of the ops.outputs - """ - if print_info: - print('---------') - print('Operations: name -> (type shapes) [size]') - print('---------') - total_size = 0 - for op in graph.get_operations(): - op_size = 0 - shapes = [] - for output in op.outputs: - # if output.num_elements() is None or [] assume size 0. - output_size = output.get_shape().num_elements() or 0 - if output.get_shape(): - shapes.append(tensor_description(output)) - op_size += output_size - if print_info: - print(op.name, '\t->', ', '.join(shapes), '[' + str(op_size) + ']') - total_size += op_size - return total_size - - -def analyze_vars(variables, print_info=False): - """Prints the names and shapes of the variables. - - Args: - variables: list of variables, for example tf.compat.v1.global_variables(). - print_info: Optional, if true print variables and their shape. - - Returns: - (total size of the variables, total bytes of the variables) - """ - if print_info: - print('---------') - print('Variables: name (type shape) [size]') - print('---------') - total_size = 0 - total_bytes = 0 - for var in variables: - # if var.num_elements() is None or [] assume size 0. - var_size = var.get_shape().num_elements() or 0 - var_bytes = var_size * var.dtype.size - total_size += var_size - total_bytes += var_bytes - if print_info: - print(var.name, tensor_description(var), - '[%d, bytes: %d]' % (var_size, var_bytes)) - if print_info: - print('Total size of variables: %d' % total_size) - print('Total bytes of variables: %d' % total_bytes) - return total_size, total_bytes diff --git a/tensorflow/contrib/slim/python/slim/nets/BUILD b/tensorflow/contrib/slim/python/slim/nets/BUILD deleted file mode 100644 index 5d3929efd43..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/BUILD +++ /dev/null @@ -1,328 +0,0 @@ -# Description: -# Contains typical networks definitions. - -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = [ - "//tensorflow:__subpackages__", - "//tensorflow_models:__subpackages__", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -# Transitive dependencies of this target will be included in the pip package. -py_library( - name = "nets_pip", - deps = [ - ":alexnet", - ":inception", - ":overfeat", - ":resnet_v1", - ":resnet_v2", - ":vgg", - ], -) - -py_library( - name = "alexnet", - srcs = ["alexnet.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:variable_scope", - ], -) - -py_test( - name = "alexnet_test", - size = "medium", - srcs = ["alexnet_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":alexnet", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -py_library( - name = "inception", - srcs = ["inception.py"], - srcs_version = "PY2AND3", - deps = [ - ":inception_v1", - ":inception_v2", - ":inception_v3", - ], -) - -py_library( - name = "inception_v1", - srcs = ["inception_v1.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:variable_scope", - ], -) - -py_library( - name = "inception_v2", - srcs = ["inception_v2.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:variable_scope", - ], -) - -py_library( - name = "inception_v3", - srcs = ["inception_v3.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:variable_scope", - ], -) - -py_test( - name = "inception_v1_test", - size = "medium", - srcs = ["inception_v1_test.py"], - python_version = "PY2", - shard_count = 8, - srcs_version = "PY2AND3", - deps = [ - ":inception_v1", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/slim:model_analyzer", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "inception_v2_test", - size = "medium", - srcs = ["inception_v2_test.py"], - python_version = "PY2", - shard_count = 8, - srcs_version = "PY2AND3", - deps = [ - ":inception_v2", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/slim:model_analyzer", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "inception_v3_test", - size = "medium", - srcs = ["inception_v3_test.py"], - python_version = "PY2", - shard_count = 8, - srcs_version = "PY2AND3", - deps = [ - ":inception_v3", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/slim:model_analyzer", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_library( - name = "overfeat", - srcs = ["overfeat.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:variable_scope", - ], -) - -py_test( - name = "overfeat_test", - size = "medium", - srcs = ["overfeat_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":overfeat", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) - -py_library( - name = "resnet_utils", - srcs = ["resnet_utils.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:nn_ops", - "//tensorflow/python:variable_scope", - ], -) - -py_library( - name = "resnet_v1", - srcs = ["resnet_v1.py"], - srcs_version = "PY2AND3", - deps = [ - ":resnet_utils", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:variable_scope", - ], -) - -py_test( - name = "resnet_v1_test", - size = "medium", - srcs = ["resnet_v1_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - deps = [ - ":resnet_utils", - ":resnet_v1", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_library( - name = "resnet_v2", - srcs = ["resnet_v2.py"], - srcs_version = "PY2AND3", - deps = [ - ":resnet_utils", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:variable_scope", - ], -) - -py_test( - name = "resnet_v2_test", - size = "medium", - srcs = ["resnet_v2_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - deps = [ - ":resnet_utils", - ":resnet_v2", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_library( - name = "vgg", - srcs = ["vgg.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:variable_scope", - ], -) - -py_test( - name = "vgg_test", - size = "medium", - srcs = ["vgg_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":vgg", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - ], -) diff --git a/tensorflow/contrib/slim/python/slim/nets/alexnet.py b/tensorflow/contrib/slim/python/slim/nets/alexnet.py deleted file mode 100644 index 51451803b19..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/alexnet.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains a model definition for AlexNet. - -This work was first described in: - ImageNet Classification with Deep Convolutional Neural Networks - Alex Krizhevsky, Ilya Sutskever and Geoffrey E. Hinton - -and later refined in: - One weird trick for parallelizing convolutional neural networks - Alex Krizhevsky, 2014 - -Here we provide the implementation proposed in "One weird trick" and not -"ImageNet Classification", as per the paper, the LRN layers have been removed. - -Usage: - with slim.arg_scope(alexnet.alexnet_v2_arg_scope()): - outputs, end_points = alexnet.alexnet_v2(inputs) - -@@alexnet_v2 -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.layers.python.layers import layers as layers_lib -from tensorflow.contrib.layers.python.layers import regularizers -from tensorflow.contrib.layers.python.layers import utils -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope - -trunc_normal = lambda stddev: init_ops.truncated_normal_initializer(0.0, stddev) - - -def alexnet_v2_arg_scope(weight_decay=0.0005): - with arg_scope( - [layers.conv2d, layers_lib.fully_connected], - activation_fn=nn_ops.relu, - biases_initializer=init_ops.constant_initializer(0.1), - weights_regularizer=regularizers.l2_regularizer(weight_decay)): - with arg_scope([layers.conv2d], padding='SAME'): - with arg_scope([layers_lib.max_pool2d], padding='VALID') as arg_sc: - return arg_sc - - -def alexnet_v2(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.5, - spatial_squeeze=True, - scope='alexnet_v2'): - """AlexNet version 2. - - Described in: http://arxiv.org/pdf/1404.5997v2.pdf - Parameters from: - github.com/akrizhevsky/cuda-convnet2/blob/master/layers/ - layers-imagenet-1gpu.cfg - - Note: All the fully_connected layers have been transformed to conv2d layers. - To use in classification mode, resize input to 224x224. To use in fully - convolutional mode, set spatial_squeeze to false. - The LRN layers have been removed and change the initializers from - random_normal_initializer to xavier_initializer. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether or not the model is being trained. - dropout_keep_prob: the probability that activations are kept in the dropout - layers during training. - spatial_squeeze: whether or not should squeeze the spatial dimensions of the - outputs. Useful to remove unnecessary dimensions for classification. - scope: Optional scope for the variables. - - Returns: - the last op containing the log predictions and end_points dict. - """ - with variable_scope.variable_scope(scope, 'alexnet_v2', [inputs]) as sc: - end_points_collection = sc.original_name_scope + '_end_points' - # Collect outputs for conv2d, fully_connected and max_pool2d. - with arg_scope( - [layers.conv2d, layers_lib.fully_connected, layers_lib.max_pool2d], - outputs_collections=[end_points_collection]): - net = layers.conv2d( - inputs, 64, [11, 11], 4, padding='VALID', scope='conv1') - net = layers_lib.max_pool2d(net, [3, 3], 2, scope='pool1') - net = layers.conv2d(net, 192, [5, 5], scope='conv2') - net = layers_lib.max_pool2d(net, [3, 3], 2, scope='pool2') - net = layers.conv2d(net, 384, [3, 3], scope='conv3') - net = layers.conv2d(net, 384, [3, 3], scope='conv4') - net = layers.conv2d(net, 256, [3, 3], scope='conv5') - net = layers_lib.max_pool2d(net, [3, 3], 2, scope='pool5') - - # Use conv2d instead of fully_connected layers. - with arg_scope( - [layers.conv2d], - weights_initializer=trunc_normal(0.005), - biases_initializer=init_ops.constant_initializer(0.1)): - net = layers.conv2d(net, 4096, [5, 5], padding='VALID', scope='fc6') - net = layers_lib.dropout( - net, dropout_keep_prob, is_training=is_training, scope='dropout6') - net = layers.conv2d(net, 4096, [1, 1], scope='fc7') - net = layers_lib.dropout( - net, dropout_keep_prob, is_training=is_training, scope='dropout7') - net = layers.conv2d( - net, - num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - biases_initializer=init_ops.zeros_initializer(), - scope='fc8') - - # Convert end_points_collection into a end_point dict. - end_points = utils.convert_collection_to_dict(end_points_collection) - if spatial_squeeze: - net = array_ops.squeeze(net, [1, 2], name='fc8/squeezed') - end_points[sc.name + '/fc8'] = net - return net, end_points - - -alexnet_v2.default_image_size = 224 diff --git a/tensorflow/contrib/slim/python/slim/nets/alexnet_test.py b/tensorflow/contrib/slim/python/slim/nets/alexnet_test.py deleted file mode 100644 index b6d1afd27d4..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/alexnet_test.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.nets.alexnet.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework.python.ops import variables as variables_lib -from tensorflow.contrib.slim.python.slim.nets import alexnet -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class AlexnetV2Test(test.TestCase): - - def testBuild(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = alexnet.alexnet_v2(inputs, num_classes) - self.assertEquals(logits.op.name, 'alexnet_v2/fc8/squeezed') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testFullyConvolutional(self): - batch_size = 1 - height, width = 300, 400 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = alexnet.alexnet_v2(inputs, num_classes, spatial_squeeze=False) - self.assertEquals(logits.op.name, 'alexnet_v2/fc8/BiasAdd') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, 4, 7, num_classes]) - - def testEndPoints(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - _, end_points = alexnet.alexnet_v2(inputs, num_classes) - expected_names = [ - 'alexnet_v2/conv1', 'alexnet_v2/pool1', 'alexnet_v2/conv2', - 'alexnet_v2/pool2', 'alexnet_v2/conv3', 'alexnet_v2/conv4', - 'alexnet_v2/conv5', 'alexnet_v2/pool5', 'alexnet_v2/fc6', - 'alexnet_v2/fc7', 'alexnet_v2/fc8' - ] - self.assertSetEqual(set(end_points.keys()), set(expected_names)) - - def testModelVariables(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - alexnet.alexnet_v2(inputs, num_classes) - expected_names = [ - 'alexnet_v2/conv1/weights', - 'alexnet_v2/conv1/biases', - 'alexnet_v2/conv2/weights', - 'alexnet_v2/conv2/biases', - 'alexnet_v2/conv3/weights', - 'alexnet_v2/conv3/biases', - 'alexnet_v2/conv4/weights', - 'alexnet_v2/conv4/biases', - 'alexnet_v2/conv5/weights', - 'alexnet_v2/conv5/biases', - 'alexnet_v2/fc6/weights', - 'alexnet_v2/fc6/biases', - 'alexnet_v2/fc7/weights', - 'alexnet_v2/fc7/biases', - 'alexnet_v2/fc8/weights', - 'alexnet_v2/fc8/biases', - ] - model_variables = [v.op.name for v in variables_lib.get_model_variables()] - self.assertSetEqual(set(model_variables), set(expected_names)) - - def testEvaluation(self): - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - with self.cached_session(): - eval_inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = alexnet.alexnet_v2(eval_inputs, is_training=False) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - predictions = math_ops.argmax(logits, 1) - self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) - - def testTrainEvalWithReuse(self): - train_batch_size = 2 - eval_batch_size = 1 - train_height, train_width = 224, 224 - eval_height, eval_width = 300, 400 - num_classes = 1000 - with self.cached_session(): - train_inputs = random_ops.random_uniform( - (train_batch_size, train_height, train_width, 3)) - logits, _ = alexnet.alexnet_v2(train_inputs) - self.assertListEqual(logits.get_shape().as_list(), - [train_batch_size, num_classes]) - variable_scope.get_variable_scope().reuse_variables() - eval_inputs = random_ops.random_uniform( - (eval_batch_size, eval_height, eval_width, 3)) - logits, _ = alexnet.alexnet_v2( - eval_inputs, is_training=False, spatial_squeeze=False) - self.assertListEqual(logits.get_shape().as_list(), - [eval_batch_size, 4, 7, num_classes]) - logits = math_ops.reduce_mean(logits, [1, 2]) - predictions = math_ops.argmax(logits, 1) - self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) - - def testForward(self): - batch_size = 1 - height, width = 224, 224 - with self.cached_session() as sess: - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = alexnet.alexnet_v2(inputs) - sess.run(variables.global_variables_initializer()) - output = sess.run(logits) - self.assertTrue(output.any()) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/nets/inception.py b/tensorflow/contrib/slim/python/slim/nets/inception.py deleted file mode 100644 index b92ebf1981b..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/inception.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Brings inception_v1, inception_v2 and inception_v3 under one namespace.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.contrib.slim.python.slim.nets.inception_v1 import inception_v1 -from tensorflow.contrib.slim.python.slim.nets.inception_v1 import inception_v1_arg_scope -from tensorflow.contrib.slim.python.slim.nets.inception_v1 import inception_v1_base -from tensorflow.contrib.slim.python.slim.nets.inception_v2 import inception_v2 -from tensorflow.contrib.slim.python.slim.nets.inception_v2 import inception_v2_arg_scope -from tensorflow.contrib.slim.python.slim.nets.inception_v2 import inception_v2_base -from tensorflow.contrib.slim.python.slim.nets.inception_v3 import inception_v3 -from tensorflow.contrib.slim.python.slim.nets.inception_v3 import inception_v3_arg_scope -from tensorflow.contrib.slim.python.slim.nets.inception_v3 import inception_v3_base -# pylint: enable=unused-import diff --git a/tensorflow/contrib/slim/python/slim/nets/inception_v1.py b/tensorflow/contrib/slim/python/slim/nets/inception_v1.py deleted file mode 100644 index ef9a4f6f685..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/inception_v1.py +++ /dev/null @@ -1,415 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains the definition for inception v1 classification network.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.layers.python.layers import initializers -from tensorflow.contrib.layers.python.layers import layers as layers_lib -from tensorflow.contrib.layers.python.layers import regularizers -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope - -trunc_normal = lambda stddev: init_ops.truncated_normal_initializer(0.0, stddev) - - -def inception_v1_base(inputs, final_endpoint='Mixed_5c', scope='InceptionV1'): - """Defines the Inception V1 base architecture. - - This architecture is defined in: - Going deeper with convolutions - Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, - Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich. - http://arxiv.org/pdf/1409.4842v1.pdf. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - final_endpoint: specifies the endpoint to construct the network up to. It - can be one of ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', - 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', - 'MaxPool_4a_3x3', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', - 'Mixed_4f', 'MaxPool_5a_2x2', 'Mixed_5b', 'Mixed_5c'] - scope: Optional variable_scope. - - Returns: - A dictionary from components of the network to the corresponding activation. - - Raises: - ValueError: if final_endpoint is not set to one of the predefined values. - """ - end_points = {} - with variable_scope.variable_scope(scope, 'InceptionV1', [inputs]): - with arg_scope( - [layers.conv2d, layers_lib.fully_connected], - weights_initializer=trunc_normal(0.01)): - with arg_scope( - [layers.conv2d, layers_lib.max_pool2d], stride=1, padding='SAME'): - end_point = 'Conv2d_1a_7x7' - net = layers.conv2d(inputs, 64, [7, 7], stride=2, scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - end_point = 'MaxPool_2a_3x3' - net = layers_lib.max_pool2d(net, [3, 3], stride=2, scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - end_point = 'Conv2d_2b_1x1' - net = layers.conv2d(net, 64, [1, 1], scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - end_point = 'Conv2d_2c_3x3' - net = layers.conv2d(net, 192, [3, 3], scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - end_point = 'MaxPool_3a_3x3' - net = layers_lib.max_pool2d(net, [3, 3], stride=2, scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - - end_point = 'Mixed_3b' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d(net, 64, [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d(net, 96, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, 128, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d(net, 16, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, 32, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.max_pool2d( - net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, 32, [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - - end_point = 'Mixed_3c' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d(net, 128, [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d(net, 128, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, 192, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d(net, 32, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, 96, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.max_pool2d( - net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - - end_point = 'MaxPool_4a_3x3' - net = layers_lib.max_pool2d(net, [3, 3], stride=2, scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - - end_point = 'Mixed_4b' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d(net, 192, [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d(net, 96, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, 208, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d(net, 16, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, 48, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.max_pool2d( - net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - - end_point = 'Mixed_4c' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d(net, 160, [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d(net, 112, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, 224, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d(net, 24, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, 64, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.max_pool2d( - net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - - end_point = 'Mixed_4d' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d(net, 128, [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d(net, 128, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, 256, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d(net, 24, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, 64, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.max_pool2d( - net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - - end_point = 'Mixed_4e' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d(net, 112, [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d(net, 144, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, 288, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d(net, 32, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, 64, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.max_pool2d( - net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, 64, [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - - end_point = 'Mixed_4f' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d(net, 256, [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d(net, 160, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, 320, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d(net, 32, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, 128, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.max_pool2d( - net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, 128, [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - - end_point = 'MaxPool_5a_2x2' - net = layers_lib.max_pool2d(net, [2, 2], stride=2, scope=end_point) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - - end_point = 'Mixed_5b' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d(net, 256, [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d(net, 160, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, 320, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d(net, 32, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, 128, [3, 3], scope='Conv2d_0a_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.max_pool2d( - net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, 128, [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - - end_point = 'Mixed_5c' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d(net, 384, [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d(net, 192, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, 384, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d(net, 48, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, 128, [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.max_pool2d( - net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, 128, [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if final_endpoint == end_point: - return net, end_points - raise ValueError('Unknown final endpoint %s' % final_endpoint) - - -def inception_v1(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.8, - prediction_fn=layers_lib.softmax, - spatial_squeeze=True, - reuse=None, - scope='InceptionV1'): - """Defines the Inception V1 architecture. - - This architecture is defined in: - - Going deeper with convolutions - Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, - Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich. - http://arxiv.org/pdf/1409.4842v1.pdf. - - The default image size used to train this network is 224x224. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether is training or not. - dropout_keep_prob: the percentage of activation values that are retained. - prediction_fn: a function to get predictions out of logits. - spatial_squeeze: if True, logits is of shape is [B, C], if false logits is - of shape [B, 1, 1, C], where B is batch_size and C is number of classes. - reuse: whether or not the network and its variables should be reused. To be - able to reuse 'scope' must be given. - scope: Optional variable_scope. - - Returns: - logits: the pre-softmax activations, a tensor of size - [batch_size, num_classes] - end_points: a dictionary from components of the network to the corresponding - activation. - """ - # Final pooling and prediction - with variable_scope.variable_scope( - scope, 'InceptionV1', [inputs, num_classes], reuse=reuse) as scope: - with arg_scope( - [layers_lib.batch_norm, layers_lib.dropout], is_training=is_training): - net, end_points = inception_v1_base(inputs, scope=scope) - with variable_scope.variable_scope('Logits'): - net = layers_lib.avg_pool2d( - net, [7, 7], stride=1, scope='MaxPool_0a_7x7') - net = layers_lib.dropout(net, dropout_keep_prob, scope='Dropout_0b') - logits = layers.conv2d( - net, - num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - scope='Conv2d_0c_1x1') - if spatial_squeeze: - logits = array_ops.squeeze(logits, [1, 2], name='SpatialSqueeze') - - end_points['Logits'] = logits - end_points['Predictions'] = prediction_fn(logits, scope='Predictions') - return logits, end_points - - -inception_v1.default_image_size = 224 - - -def inception_v1_arg_scope(weight_decay=0.00004, - use_batch_norm=True, - batch_norm_var_collection='moving_vars'): - """Defines the default InceptionV1 arg scope. - - Note: Althougth the original paper didn't use batch_norm we found it useful. - - Args: - weight_decay: The weight decay to use for regularizing the model. - use_batch_norm: "If `True`, batch_norm is applied after each convolution. - batch_norm_var_collection: The name of the collection for the batch norm - variables. - - Returns: - An `arg_scope` to use for the inception v3 model. - """ - batch_norm_params = { - # Decay for the moving averages. - 'decay': 0.9997, - # epsilon to prevent 0s in variance. - 'epsilon': 0.001, - # collection containing update_ops. - 'updates_collections': ops.GraphKeys.UPDATE_OPS, - # collection containing the moving mean and moving variance. - 'variables_collections': { - 'beta': None, - 'gamma': None, - 'moving_mean': [batch_norm_var_collection], - 'moving_variance': [batch_norm_var_collection], - } - } - if use_batch_norm: - normalizer_fn = layers_lib.batch_norm - normalizer_params = batch_norm_params - else: - normalizer_fn = None - normalizer_params = {} - # Set weight_decay for weights in Conv and FC layers. - with arg_scope( - [layers.conv2d, layers_lib.fully_connected], - weights_regularizer=regularizers.l2_regularizer(weight_decay)): - with arg_scope( - [layers.conv2d], - weights_initializer=initializers.variance_scaling_initializer(), - activation_fn=nn_ops.relu, - normalizer_fn=normalizer_fn, - normalizer_params=normalizer_params) as sc: - return sc diff --git a/tensorflow/contrib/slim/python/slim/nets/inception_v1_test.py b/tensorflow/contrib/slim/python/slim/nets/inception_v1_test.py deleted file mode 100644 index 34f12d75915..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/inception_v1_test.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for nets.inception_v1.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.framework.python.ops import variables as variables_lib -from tensorflow.contrib.slim.python.slim import model_analyzer -from tensorflow.contrib.slim.python.slim.nets import inception_v1 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class InceptionV1Test(test.TestCase): - - def testBuildClassificationNetwork(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, end_points = inception_v1.inception_v1(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV1/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertTrue('Predictions' in end_points) - self.assertListEqual(end_points['Predictions'].get_shape().as_list(), - [batch_size, num_classes]) - - def testBuildBaseNetwork(self): - batch_size = 5 - height, width = 224, 224 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - mixed_6c, end_points = inception_v1.inception_v1_base(inputs) - self.assertTrue(mixed_6c.op.name.startswith('InceptionV1/Mixed_5c')) - self.assertListEqual(mixed_6c.get_shape().as_list(), - [batch_size, 7, 7, 1024]) - expected_endpoints = [ - 'Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', 'Conv2d_2c_3x3', - 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', 'MaxPool_4a_3x3', 'Mixed_4b', - 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', 'Mixed_4f', 'MaxPool_5a_2x2', - 'Mixed_5b', 'Mixed_5c' - ] - self.assertItemsEqual(end_points.keys(), expected_endpoints) - - def testBuildOnlyUptoFinalEndpoint(self): - batch_size = 5 - height, width = 224, 224 - endpoints = [ - 'Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', 'Conv2d_2c_3x3', - 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', 'MaxPool_4a_3x3', 'Mixed_4b', - 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', 'Mixed_4f', 'MaxPool_5a_2x2', - 'Mixed_5b', 'Mixed_5c' - ] - for index, endpoint in enumerate(endpoints): - with ops.Graph().as_default(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - out_tensor, end_points = inception_v1.inception_v1_base( - inputs, final_endpoint=endpoint) - self.assertTrue( - out_tensor.op.name.startswith('InceptionV1/' + endpoint)) - self.assertItemsEqual(endpoints[:index + 1], end_points) - - def testBuildAndCheckAllEndPointsUptoMixed5c(self): - batch_size = 5 - height, width = 224, 224 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - _, end_points = inception_v1.inception_v1_base( - inputs, final_endpoint='Mixed_5c') - endpoints_shapes = { - 'Conv2d_1a_7x7': [5, 112, 112, 64], - 'MaxPool_2a_3x3': [5, 56, 56, 64], - 'Conv2d_2b_1x1': [5, 56, 56, 64], - 'Conv2d_2c_3x3': [5, 56, 56, 192], - 'MaxPool_3a_3x3': [5, 28, 28, 192], - 'Mixed_3b': [5, 28, 28, 256], - 'Mixed_3c': [5, 28, 28, 480], - 'MaxPool_4a_3x3': [5, 14, 14, 480], - 'Mixed_4b': [5, 14, 14, 512], - 'Mixed_4c': [5, 14, 14, 512], - 'Mixed_4d': [5, 14, 14, 512], - 'Mixed_4e': [5, 14, 14, 528], - 'Mixed_4f': [5, 14, 14, 832], - 'MaxPool_5a_2x2': [5, 7, 7, 832], - 'Mixed_5b': [5, 7, 7, 832], - 'Mixed_5c': [5, 7, 7, 1024] - } - - self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) - for endpoint_name in endpoints_shapes: - expected_shape = endpoints_shapes[endpoint_name] - self.assertTrue(endpoint_name in end_points) - self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), - expected_shape) - - def testModelHasExpectedNumberOfParameters(self): - batch_size = 5 - height, width = 224, 224 - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - with arg_scope(inception_v1.inception_v1_arg_scope()): - inception_v1.inception_v1_base(inputs) - total_params, _ = model_analyzer.analyze_vars( - variables_lib.get_model_variables()) - self.assertAlmostEqual(5607184, total_params) - - def testHalfSizeImages(self): - batch_size = 5 - height, width = 112, 112 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - mixed_5c, _ = inception_v1.inception_v1_base(inputs) - self.assertTrue(mixed_5c.op.name.startswith('InceptionV1/Mixed_5c')) - self.assertListEqual(mixed_5c.get_shape().as_list(), - [batch_size, 4, 4, 1024]) - - def testUnknownImageShape(self): - ops.reset_default_graph() - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) - with self.cached_session() as sess: - inputs = array_ops.placeholder( - dtypes.float32, shape=(batch_size, None, None, 3)) - logits, end_points = inception_v1.inception_v1(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV1/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - pre_pool = end_points['Mixed_5c'] - feed_dict = {inputs: input_np} - variables.global_variables_initializer().run() - pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) - self.assertListEqual(list(pre_pool_out.shape), [batch_size, 7, 7, 1024]) - - def testUnknownBatchSize(self): - batch_size = 1 - height, width = 224, 224 - num_classes = 1000 - - inputs = array_ops.placeholder(dtypes.float32, (None, height, width, 3)) - logits, _ = inception_v1.inception_v1(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV1/Logits')) - self.assertListEqual(logits.get_shape().as_list(), [None, num_classes]) - images = random_ops.random_uniform((batch_size, height, width, 3)) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(logits, {inputs: images.eval()}) - self.assertEquals(output.shape, (batch_size, num_classes)) - - def testEvaluation(self): - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - - eval_inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = inception_v1.inception_v1( - eval_inputs, num_classes, is_training=False) - predictions = math_ops.argmax(logits, 1) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(predictions) - self.assertEquals(output.shape, (batch_size,)) - - def testTrainEvalWithReuse(self): - train_batch_size = 5 - eval_batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - - train_inputs = random_ops.random_uniform( - (train_batch_size, height, width, 3)) - inception_v1.inception_v1(train_inputs, num_classes) - eval_inputs = random_ops.random_uniform((eval_batch_size, height, width, 3)) - logits, _ = inception_v1.inception_v1(eval_inputs, num_classes, reuse=True) - predictions = math_ops.argmax(logits, 1) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(predictions) - self.assertEquals(output.shape, (eval_batch_size,)) - - def testLogitsNotSqueezed(self): - num_classes = 25 - images = random_ops.random_uniform([1, 224, 224, 3]) - logits, _ = inception_v1.inception_v1( - images, num_classes=num_classes, spatial_squeeze=False) - - with self.cached_session() as sess: - variables.global_variables_initializer().run() - logits_out = sess.run(logits) - self.assertListEqual(list(logits_out.shape), [1, 1, 1, num_classes]) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/nets/inception_v2.py b/tensorflow/contrib/slim/python/slim/nets/inception_v2.py deleted file mode 100644 index e44cb770ba9..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/inception_v2.py +++ /dev/null @@ -1,643 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains the definition for inception v2 classification network.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.layers.python.layers import initializers -from tensorflow.contrib.layers.python.layers import layers as layers_lib -from tensorflow.contrib.layers.python.layers import regularizers -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope - -trunc_normal = lambda stddev: init_ops.truncated_normal_initializer(0.0, stddev) - - -def inception_v2_base(inputs, - final_endpoint='Mixed_5c', - min_depth=16, - depth_multiplier=1.0, - scope=None): - """Inception v2 (6a2). - - Constructs an Inception v2 network from inputs to the given final endpoint. - This method can construct the network up to the layer inception(5b) as - described in http://arxiv.org/abs/1502.03167. - - Args: - inputs: a tensor of shape [batch_size, height, width, channels]. - final_endpoint: specifies the endpoint to construct the network up to. It - can be one of ['Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', - 'Conv2d_2c_3x3', 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', 'Mixed_4a', - 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', 'Mixed_5a', 'Mixed_5b', - 'Mixed_5c']. - min_depth: Minimum depth value (number of channels) for all convolution ops. - Enforced when depth_multiplier < 1, and not an active constraint when - depth_multiplier >= 1. - depth_multiplier: Float multiplier for the depth (number of channels) - for all convolution ops. The value must be greater than zero. Typical - usage will be to set this value in (0, 1) to reduce the number of - parameters or computation cost of the model. - scope: Optional variable_scope. - - Returns: - tensor_out: output tensor corresponding to the final_endpoint. - end_points: a set of activations for external use, for example summaries or - losses. - - Raises: - ValueError: if final_endpoint is not set to one of the predefined values, - or depth_multiplier <= 0 - """ - - # end_points will collect relevant activations for external use, for example - # summaries or losses. - end_points = {} - - # Used to find thinned depths for each layer. - if depth_multiplier <= 0: - raise ValueError('depth_multiplier is not greater than zero.') - depth = lambda d: max(int(d * depth_multiplier), min_depth) - - with variable_scope.variable_scope(scope, 'InceptionV2', [inputs]): - with arg_scope( - [ - layers.conv2d, layers_lib.max_pool2d, layers_lib.avg_pool2d, - layers.separable_conv2d - ], - stride=1, - padding='SAME'): - - # Note that sizes in the comments below assume an input spatial size of - # 224x224, however, the inputs can be of any size greater 32x32. - - # 224 x 224 x 3 - end_point = 'Conv2d_1a_7x7' - # depthwise_multiplier here is different from depth_multiplier. - # depthwise_multiplier determines the output channels of the initial - # depthwise conv (see docs for tf.nn.separable_conv2d), while - # depth_multiplier controls the # channels of the subsequent 1x1 - # convolution. Must have - # in_channels * depthwise_multipler <= out_channels - # so that the separable convolution is not overparameterized. - depthwise_multiplier = min(int(depth(64) / 3), 8) - net = layers.separable_conv2d( - inputs, - depth(64), [7, 7], - depth_multiplier=depthwise_multiplier, - stride=2, - weights_initializer=trunc_normal(1.0), - scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 112 x 112 x 64 - end_point = 'MaxPool_2a_3x3' - net = layers_lib.max_pool2d(net, [3, 3], scope=end_point, stride=2) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 56 x 56 x 64 - end_point = 'Conv2d_2b_1x1' - net = layers.conv2d( - net, - depth(64), [1, 1], - scope=end_point, - weights_initializer=trunc_normal(0.1)) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 56 x 56 x 64 - end_point = 'Conv2d_2c_3x3' - net = layers.conv2d(net, depth(192), [3, 3], scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 56 x 56 x 192 - end_point = 'MaxPool_3a_3x3' - net = layers_lib.max_pool2d(net, [3, 3], scope=end_point, stride=2) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 28 x 28 x 192 - # Inception module. - end_point = 'Mixed_3b' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, - depth(64), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(64), [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, - depth(64), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(96), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = layers.conv2d( - branch_2, depth(96), [3, 3], scope='Conv2d_0c_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, - depth(32), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 28 x 28 x 256 - end_point = 'Mixed_3c' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, - depth(64), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(96), [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, - depth(64), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(96), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = layers.conv2d( - branch_2, depth(96), [3, 3], scope='Conv2d_0c_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, - depth(64), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 28 x 28 x 320 - end_point = 'Mixed_4a' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, - depth(128), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_0 = layers.conv2d( - branch_0, depth(160), [3, 3], stride=2, scope='Conv2d_1a_3x3') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, - depth(64), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(96), [3, 3], scope='Conv2d_0b_3x3') - branch_1 = layers.conv2d( - branch_1, depth(96), [3, 3], stride=2, scope='Conv2d_1a_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers_lib.max_pool2d( - net, [3, 3], stride=2, scope='MaxPool_1a_3x3') - net = array_ops.concat([branch_0, branch_1, branch_2], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 14 x 14 x 576 - end_point = 'Mixed_4b' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(224), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, - depth(64), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(96), [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, - depth(96), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(128), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = layers.conv2d( - branch_2, depth(128), [3, 3], scope='Conv2d_0c_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, - depth(128), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 14 x 14 x 576 - end_point = 'Mixed_4c' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, - depth(96), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(128), [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, - depth(96), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(128), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = layers.conv2d( - branch_2, depth(128), [3, 3], scope='Conv2d_0c_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, - depth(128), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 14 x 14 x 576 - end_point = 'Mixed_4d' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(160), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, - depth(128), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(160), [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, - depth(128), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(160), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = layers.conv2d( - branch_2, depth(160), [3, 3], scope='Conv2d_0c_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, - depth(96), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - - # 14 x 14 x 576 - end_point = 'Mixed_4e' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(96), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, - depth(128), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(192), [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, - depth(160), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(192), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = layers.conv2d( - branch_2, depth(192), [3, 3], scope='Conv2d_0c_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, - depth(96), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 14 x 14 x 576 - end_point = 'Mixed_5a' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, - depth(128), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_0 = layers.conv2d( - branch_0, depth(192), [3, 3], stride=2, scope='Conv2d_1a_3x3') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, - depth(192), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(256), [3, 3], scope='Conv2d_0b_3x3') - branch_1 = layers.conv2d( - branch_1, depth(256), [3, 3], stride=2, scope='Conv2d_1a_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers_lib.max_pool2d( - net, [3, 3], stride=2, scope='MaxPool_1a_3x3') - net = array_ops.concat([branch_0, branch_1, branch_2], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 7 x 7 x 1024 - end_point = 'Mixed_5b' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(352), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, - depth(192), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(320), [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, - depth(160), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(224), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = layers.conv2d( - branch_2, depth(224), [3, 3], scope='Conv2d_0c_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, - depth(128), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - - # 7 x 7 x 1024 - end_point = 'Mixed_5c' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(352), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, - depth(192), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(320), [3, 3], scope='Conv2d_0b_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, - depth(192), [1, 1], - weights_initializer=trunc_normal(0.09), - scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(224), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = layers.conv2d( - branch_2, depth(224), [3, 3], scope='Conv2d_0c_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.max_pool2d(net, [3, 3], scope='MaxPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, - depth(128), [1, 1], - weights_initializer=trunc_normal(0.1), - scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - raise ValueError('Unknown final endpoint %s' % final_endpoint) - - -def inception_v2(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.8, - min_depth=16, - depth_multiplier=1.0, - prediction_fn=layers_lib.softmax, - spatial_squeeze=True, - reuse=None, - scope='InceptionV2'): - """Inception v2 model for classification. - - Constructs an Inception v2 network for classification as described in - http://arxiv.org/abs/1502.03167. - - The recommended image size used to train this network is 224x224. For image - sizes that differ substantially, it is recommended to use inception_v2_base() - and connect custom final layers to the output. - - Args: - inputs: a tensor of shape [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether is training or not. - dropout_keep_prob: the percentage of activation values that are retained. - min_depth: Minimum depth value (number of channels) for all convolution ops. - Enforced when depth_multiplier < 1, and not an active constraint when - depth_multiplier >= 1. - depth_multiplier: Float multiplier for the depth (number of channels) - for all convolution ops. The value must be greater than zero. Typical - usage will be to set this value in (0, 1) to reduce the number of - parameters or computation cost of the model. - prediction_fn: a function to get predictions out of logits. - spatial_squeeze: if True, logits is of shape [B, C], if false logits is - of shape [B, 1, 1, C], where B is batch_size and C is number of classes. - Note that input image sizes other than 224x224 might lead to different - spatial dimensions, and hence cannot be squeezed. In this event, - it is best to set spatial_squeeze as False, and perform a reduce_mean - over the resulting spatial dimensions with sizes exceeding 1. - reuse: whether or not the network and its variables should be reused. To be - able to reuse 'scope' must be given. - scope: Optional variable_scope. - - Returns: - logits: the pre-softmax activations, a tensor of size - [batch_size, num_classes] - end_points: a dictionary from components of the network to the corresponding - activation. - - Raises: - ValueError: if depth_multiplier <= 0. - """ - if depth_multiplier <= 0: - raise ValueError('depth_multiplier is not greater than zero.') - - # Final pooling and prediction - with variable_scope.variable_scope( - scope, 'InceptionV2', [inputs, num_classes], reuse=reuse) as scope: - with arg_scope( - [layers_lib.batch_norm, layers_lib.dropout], is_training=is_training): - net, end_points = inception_v2_base( - inputs, - scope=scope, - min_depth=min_depth, - depth_multiplier=depth_multiplier) - with variable_scope.variable_scope('Logits'): - kernel_size = _reduced_kernel_size_for_small_input(net, [7, 7]) - net = layers_lib.avg_pool2d( - net, - kernel_size, - padding='VALID', - scope='AvgPool_1a_{}x{}'.format(*kernel_size)) - # 1 x 1 x 1024 - net = layers_lib.dropout( - net, keep_prob=dropout_keep_prob, scope='Dropout_1b') - logits = layers.conv2d( - net, - num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - scope='Conv2d_1c_1x1') - if spatial_squeeze: - logits = array_ops.squeeze(logits, [1, 2], name='SpatialSqueeze') - end_points['Logits'] = logits - end_points['Predictions'] = prediction_fn(logits, scope='Predictions') - return logits, end_points - - -inception_v2.default_image_size = 224 - - -def _reduced_kernel_size_for_small_input(input_tensor, kernel_size): - """Define kernel size which is automatically reduced for small input. - - If the shape of the input images is unknown at graph construction time this - function assumes that the input images are is large enough. - - Args: - input_tensor: input tensor of size [batch_size, height, width, channels]. - kernel_size: desired kernel size of length 2: [kernel_height, kernel_width] - - Returns: - a tensor with the kernel size. - - TODO(jrru): Make this function work with unknown shapes. Theoretically, this - can be done with the code below. Problems are two-fold: (1) If the shape was - known, it will be lost. (2) inception.slim.ops._two_element_tuple cannot - handle tensors that define the kernel size. - shape = tf.shape(input_tensor) - return = tf.stack([tf.minimum(shape[1], kernel_size[0]), - tf.minimum(shape[2], kernel_size[1])]) - - """ - shape = input_tensor.get_shape().as_list() - if shape[1] is None or shape[2] is None: - kernel_size_out = kernel_size - else: - kernel_size_out = [ - min(shape[1], kernel_size[0]), min(shape[2], kernel_size[1]) - ] - return kernel_size_out - - -def inception_v2_arg_scope(weight_decay=0.00004, - batch_norm_var_collection='moving_vars'): - """Defines the default InceptionV2 arg scope. - - Args: - weight_decay: The weight decay to use for regularizing the model. - batch_norm_var_collection: The name of the collection for the batch norm - variables. - - Returns: - An `arg_scope` to use for the inception v3 model. - """ - batch_norm_params = { - # Decay for the moving averages. - 'decay': 0.9997, - # epsilon to prevent 0s in variance. - 'epsilon': 0.001, - # collection containing update_ops. - 'updates_collections': ops.GraphKeys.UPDATE_OPS, - # collection containing the moving mean and moving variance. - 'variables_collections': { - 'beta': None, - 'gamma': None, - 'moving_mean': [batch_norm_var_collection], - 'moving_variance': [batch_norm_var_collection], - } - } - - # Set weight_decay for weights in Conv and FC layers. - with arg_scope( - [layers.conv2d, layers_lib.fully_connected], - weights_regularizer=regularizers.l2_regularizer(weight_decay)): - with arg_scope( - [layers.conv2d], - weights_initializer=initializers.variance_scaling_initializer(), - activation_fn=nn_ops.relu, - normalizer_fn=layers_lib.batch_norm, - normalizer_params=batch_norm_params) as sc: - return sc diff --git a/tensorflow/contrib/slim/python/slim/nets/inception_v2_test.py b/tensorflow/contrib/slim/python/slim/nets/inception_v2_test.py deleted file mode 100644 index 66effba9444..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/inception_v2_test.py +++ /dev/null @@ -1,274 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for nets.inception_v2.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.framework.python.ops import variables as variables_lib -from tensorflow.contrib.slim.python.slim import model_analyzer -from tensorflow.contrib.slim.python.slim.nets import inception_v2 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class InceptionV2Test(test.TestCase): - - def testBuildClassificationNetwork(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, end_points = inception_v2.inception_v2(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV2/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertTrue('Predictions' in end_points) - self.assertListEqual(end_points['Predictions'].get_shape().as_list(), - [batch_size, num_classes]) - - def testBuildBaseNetwork(self): - batch_size = 5 - height, width = 224, 224 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - mixed_5c, end_points = inception_v2.inception_v2_base(inputs) - self.assertTrue(mixed_5c.op.name.startswith('InceptionV2/Mixed_5c')) - self.assertListEqual(mixed_5c.get_shape().as_list(), - [batch_size, 7, 7, 1024]) - expected_endpoints = [ - 'Mixed_3b', 'Mixed_3c', 'Mixed_4a', 'Mixed_4b', 'Mixed_4c', 'Mixed_4d', - 'Mixed_4e', 'Mixed_5a', 'Mixed_5b', 'Mixed_5c', 'Conv2d_1a_7x7', - 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', 'Conv2d_2c_3x3', 'MaxPool_3a_3x3' - ] - self.assertItemsEqual(end_points.keys(), expected_endpoints) - - def testBuildOnlyUptoFinalEndpoint(self): - batch_size = 5 - height, width = 224, 224 - endpoints = [ - 'Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', 'Conv2d_2c_3x3', - 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', 'Mixed_4a', 'Mixed_4b', - 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', 'Mixed_5a', 'Mixed_5b', 'Mixed_5c' - ] - for index, endpoint in enumerate(endpoints): - with ops.Graph().as_default(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - out_tensor, end_points = inception_v2.inception_v2_base( - inputs, final_endpoint=endpoint) - self.assertTrue( - out_tensor.op.name.startswith('InceptionV2/' + endpoint)) - self.assertItemsEqual(endpoints[:index + 1], end_points) - - def testBuildAndCheckAllEndPointsUptoMixed5c(self): - batch_size = 5 - height, width = 224, 224 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - _, end_points = inception_v2.inception_v2_base( - inputs, final_endpoint='Mixed_5c') - endpoints_shapes = { - 'Mixed_3b': [batch_size, 28, 28, 256], - 'Mixed_3c': [batch_size, 28, 28, 320], - 'Mixed_4a': [batch_size, 14, 14, 576], - 'Mixed_4b': [batch_size, 14, 14, 576], - 'Mixed_4c': [batch_size, 14, 14, 576], - 'Mixed_4d': [batch_size, 14, 14, 576], - 'Mixed_4e': [batch_size, 14, 14, 576], - 'Mixed_5a': [batch_size, 7, 7, 1024], - 'Mixed_5b': [batch_size, 7, 7, 1024], - 'Mixed_5c': [batch_size, 7, 7, 1024], - 'Conv2d_1a_7x7': [batch_size, 112, 112, 64], - 'MaxPool_2a_3x3': [batch_size, 56, 56, 64], - 'Conv2d_2b_1x1': [batch_size, 56, 56, 64], - 'Conv2d_2c_3x3': [batch_size, 56, 56, 192], - 'MaxPool_3a_3x3': [batch_size, 28, 28, 192] - } - self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) - for endpoint_name in endpoints_shapes: - expected_shape = endpoints_shapes[endpoint_name] - self.assertTrue(endpoint_name in end_points) - self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), - expected_shape) - - def testModelHasExpectedNumberOfParameters(self): - batch_size = 5 - height, width = 224, 224 - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - with arg_scope(inception_v2.inception_v2_arg_scope()): - inception_v2.inception_v2_base(inputs) - total_params, _ = model_analyzer.analyze_vars( - variables_lib.get_model_variables()) - self.assertAlmostEqual(10173112, total_params) - - def testBuildEndPointsWithDepthMultiplierLessThanOne(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - _, end_points = inception_v2.inception_v2(inputs, num_classes) - - endpoint_keys = [ - key for key in end_points.keys() - if key.startswith('Mixed') or key.startswith('Conv') - ] - - _, end_points_with_multiplier = inception_v2.inception_v2( - inputs, num_classes, scope='depth_multiplied_net', depth_multiplier=0.5) - - for key in endpoint_keys: - original_depth = end_points[key].get_shape().as_list()[3] - new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] - self.assertEqual(0.5 * original_depth, new_depth) - - def testBuildEndPointsWithDepthMultiplierGreaterThanOne(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - _, end_points = inception_v2.inception_v2(inputs, num_classes) - - endpoint_keys = [ - key for key in end_points.keys() - if key.startswith('Mixed') or key.startswith('Conv') - ] - - _, end_points_with_multiplier = inception_v2.inception_v2( - inputs, num_classes, scope='depth_multiplied_net', depth_multiplier=2.0) - - for key in endpoint_keys: - original_depth = end_points[key].get_shape().as_list()[3] - new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] - self.assertEqual(2.0 * original_depth, new_depth) - - def testRaiseValueErrorWithInvalidDepthMultiplier(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - with self.assertRaises(ValueError): - _ = inception_v2.inception_v2(inputs, num_classes, depth_multiplier=-0.1) - with self.assertRaises(ValueError): - _ = inception_v2.inception_v2(inputs, num_classes, depth_multiplier=0.0) - - def testHalfSizeImages(self): - batch_size = 5 - height, width = 112, 112 - num_classes = 1000 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, end_points = inception_v2.inception_v2(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV2/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - pre_pool = end_points['Mixed_5c'] - self.assertListEqual(pre_pool.get_shape().as_list(), - [batch_size, 4, 4, 1024]) - - def testUnknownImageShape(self): - ops.reset_default_graph() - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) - with self.cached_session() as sess: - inputs = array_ops.placeholder( - dtypes.float32, shape=(batch_size, None, None, 3)) - logits, end_points = inception_v2.inception_v2(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV2/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - pre_pool = end_points['Mixed_5c'] - feed_dict = {inputs: input_np} - variables.global_variables_initializer().run() - pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) - self.assertListEqual(list(pre_pool_out.shape), [batch_size, 7, 7, 1024]) - - def testUnknownBatchSize(self): - batch_size = 1 - height, width = 224, 224 - num_classes = 1000 - - inputs = array_ops.placeholder(dtypes.float32, (None, height, width, 3)) - logits, _ = inception_v2.inception_v2(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV2/Logits')) - self.assertListEqual(logits.get_shape().as_list(), [None, num_classes]) - images = random_ops.random_uniform((batch_size, height, width, 3)) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(logits, {inputs: images.eval()}) - self.assertEquals(output.shape, (batch_size, num_classes)) - - def testEvaluation(self): - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - - eval_inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = inception_v2.inception_v2( - eval_inputs, num_classes, is_training=False) - predictions = math_ops.argmax(logits, 1) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(predictions) - self.assertEquals(output.shape, (batch_size,)) - - def testTrainEvalWithReuse(self): - train_batch_size = 5 - eval_batch_size = 2 - height, width = 150, 150 - num_classes = 1000 - - train_inputs = random_ops.random_uniform( - (train_batch_size, height, width, 3)) - inception_v2.inception_v2(train_inputs, num_classes) - eval_inputs = random_ops.random_uniform((eval_batch_size, height, width, 3)) - logits, _ = inception_v2.inception_v2(eval_inputs, num_classes, reuse=True) - predictions = math_ops.argmax(logits, 1) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(predictions) - self.assertEquals(output.shape, (eval_batch_size,)) - - def testLogitsNotSqueezed(self): - num_classes = 25 - images = random_ops.random_uniform([1, 224, 224, 3]) - logits, _ = inception_v2.inception_v2( - images, num_classes=num_classes, spatial_squeeze=False) - - with self.cached_session() as sess: - variables.global_variables_initializer().run() - logits_out = sess.run(logits) - self.assertListEqual(list(logits_out.shape), [1, 1, 1, num_classes]) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/nets/inception_v3.py b/tensorflow/contrib/slim/python/slim/nets/inception_v3.py deleted file mode 100644 index afe261e43a9..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/inception_v3.py +++ /dev/null @@ -1,729 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains the definition for inception v3 classification network.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.layers.python.layers import initializers -from tensorflow.contrib.layers.python.layers import layers as layers_lib -from tensorflow.contrib.layers.python.layers import regularizers -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope - -trunc_normal = lambda stddev: init_ops.truncated_normal_initializer(0.0, stddev) - - -def inception_v3_base(inputs, - final_endpoint='Mixed_7c', - min_depth=16, - depth_multiplier=1.0, - scope=None): - """Inception model from http://arxiv.org/abs/1512.00567. - - Constructs an Inception v3 network from inputs to the given final endpoint. - This method can construct the network up to the final inception block - Mixed_7c. - - Note that the names of the layers in the paper do not correspond to the names - of the endpoints registered by this function although they build the same - network. - - Here is a mapping from the old_names to the new names: - Old name | New name - ======================================= - conv0 | Conv2d_1a_3x3 - conv1 | Conv2d_2a_3x3 - conv2 | Conv2d_2b_3x3 - pool1 | MaxPool_3a_3x3 - conv3 | Conv2d_3b_1x1 - conv4 | Conv2d_4a_3x3 - pool2 | MaxPool_5a_3x3 - mixed_35x35x256a | Mixed_5b - mixed_35x35x288a | Mixed_5c - mixed_35x35x288b | Mixed_5d - mixed_17x17x768a | Mixed_6a - mixed_17x17x768b | Mixed_6b - mixed_17x17x768c | Mixed_6c - mixed_17x17x768d | Mixed_6d - mixed_17x17x768e | Mixed_6e - mixed_8x8x1280a | Mixed_7a - mixed_8x8x2048a | Mixed_7b - mixed_8x8x2048b | Mixed_7c - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - final_endpoint: specifies the endpoint to construct the network up to. It - can be one of ['Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', - 'MaxPool_3a_3x3', 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', 'MaxPool_5a_3x3', - 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', - 'Mixed_6d', 'Mixed_6e', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c']. - min_depth: Minimum depth value (number of channels) for all convolution ops. - Enforced when depth_multiplier < 1, and not an active constraint when - depth_multiplier >= 1. - depth_multiplier: Float multiplier for the depth (number of channels) - for all convolution ops. The value must be greater than zero. Typical - usage will be to set this value in (0, 1) to reduce the number of - parameters or computation cost of the model. - scope: Optional variable_scope. - - Returns: - tensor_out: output tensor corresponding to the final_endpoint. - end_points: a set of activations for external use, for example summaries or - losses. - - Raises: - ValueError: if final_endpoint is not set to one of the predefined values, - or depth_multiplier <= 0 - """ - # end_points will collect relevant activations for external use, for example - # summaries or losses. - end_points = {} - - if depth_multiplier <= 0: - raise ValueError('depth_multiplier is not greater than zero.') - depth = lambda d: max(int(d * depth_multiplier), min_depth) - - with variable_scope.variable_scope(scope, 'InceptionV3', [inputs]): - with arg_scope( - [layers.conv2d, layers_lib.max_pool2d, layers_lib.avg_pool2d], - stride=1, - padding='VALID'): - # 299 x 299 x 3 - end_point = 'Conv2d_1a_3x3' - net = layers.conv2d(inputs, depth(32), [3, 3], stride=2, scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 149 x 149 x 32 - end_point = 'Conv2d_2a_3x3' - net = layers.conv2d(net, depth(32), [3, 3], scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 147 x 147 x 32 - end_point = 'Conv2d_2b_3x3' - net = layers.conv2d( - net, depth(64), [3, 3], padding='SAME', scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 147 x 147 x 64 - end_point = 'MaxPool_3a_3x3' - net = layers_lib.max_pool2d(net, [3, 3], stride=2, scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 73 x 73 x 64 - end_point = 'Conv2d_3b_1x1' - net = layers.conv2d(net, depth(80), [1, 1], scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 73 x 73 x 80. - end_point = 'Conv2d_4a_3x3' - net = layers.conv2d(net, depth(192), [3, 3], scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 71 x 71 x 192. - end_point = 'MaxPool_5a_3x3' - net = layers_lib.max_pool2d(net, [3, 3], stride=2, scope=end_point) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # 35 x 35 x 192. - - # Inception blocks - with arg_scope( - [layers.conv2d, layers_lib.max_pool2d, layers_lib.avg_pool2d], - stride=1, - padding='SAME'): - # mixed: 35 x 35 x 256. - end_point = 'Mixed_5b' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, depth(48), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(64), [5, 5], scope='Conv2d_0b_5x5') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(96), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = layers.conv2d( - branch_2, depth(96), [3, 3], scope='Conv2d_0c_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, depth(32), [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - - # mixed_1: 35 x 35 x 288. - end_point = 'Mixed_5c' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, depth(48), [1, 1], scope='Conv2d_0b_1x1') - branch_1 = layers.conv2d( - branch_1, depth(64), [5, 5], scope='Conv_1_0c_5x5') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(96), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = layers.conv2d( - branch_2, depth(96), [3, 3], scope='Conv2d_0c_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, depth(64), [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - - # mixed_2: 35 x 35 x 288. - end_point = 'Mixed_5d' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, depth(48), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(64), [5, 5], scope='Conv2d_0b_5x5') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(96), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = layers.conv2d( - branch_2, depth(96), [3, 3], scope='Conv2d_0c_3x3') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, depth(64), [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - - # mixed_3: 17 x 17 x 768. - end_point = 'Mixed_6a' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, - depth(384), [3, 3], - stride=2, - padding='VALID', - scope='Conv2d_1a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, depth(64), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(96), [3, 3], scope='Conv2d_0b_3x3') - branch_1 = layers.conv2d( - branch_1, - depth(96), [3, 3], - stride=2, - padding='VALID', - scope='Conv2d_1a_1x1') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers_lib.max_pool2d( - net, [3, 3], stride=2, padding='VALID', scope='MaxPool_1a_3x3') - net = array_ops.concat([branch_0, branch_1, branch_2], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - - # mixed4: 17 x 17 x 768. - end_point = 'Mixed_6b' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, depth(128), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(128), [1, 7], scope='Conv2d_0b_1x7') - branch_1 = layers.conv2d( - branch_1, depth(192), [7, 1], scope='Conv2d_0c_7x1') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, depth(128), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(128), [7, 1], scope='Conv2d_0b_7x1') - branch_2 = layers.conv2d( - branch_2, depth(128), [1, 7], scope='Conv2d_0c_1x7') - branch_2 = layers.conv2d( - branch_2, depth(128), [7, 1], scope='Conv2d_0d_7x1') - branch_2 = layers.conv2d( - branch_2, depth(192), [1, 7], scope='Conv2d_0e_1x7') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, depth(192), [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - - # mixed_5: 17 x 17 x 768. - end_point = 'Mixed_6c' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, depth(160), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(160), [1, 7], scope='Conv2d_0b_1x7') - branch_1 = layers.conv2d( - branch_1, depth(192), [7, 1], scope='Conv2d_0c_7x1') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, depth(160), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(160), [7, 1], scope='Conv2d_0b_7x1') - branch_2 = layers.conv2d( - branch_2, depth(160), [1, 7], scope='Conv2d_0c_1x7') - branch_2 = layers.conv2d( - branch_2, depth(160), [7, 1], scope='Conv2d_0d_7x1') - branch_2 = layers.conv2d( - branch_2, depth(192), [1, 7], scope='Conv2d_0e_1x7') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, depth(192), [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # mixed_6: 17 x 17 x 768. - end_point = 'Mixed_6d' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, depth(160), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(160), [1, 7], scope='Conv2d_0b_1x7') - branch_1 = layers.conv2d( - branch_1, depth(192), [7, 1], scope='Conv2d_0c_7x1') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, depth(160), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(160), [7, 1], scope='Conv2d_0b_7x1') - branch_2 = layers.conv2d( - branch_2, depth(160), [1, 7], scope='Conv2d_0c_1x7') - branch_2 = layers.conv2d( - branch_2, depth(160), [7, 1], scope='Conv2d_0d_7x1') - branch_2 = layers.conv2d( - branch_2, depth(192), [1, 7], scope='Conv2d_0e_1x7') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, depth(192), [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - - # mixed_7: 17 x 17 x 768. - end_point = 'Mixed_6e' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(192), [1, 7], scope='Conv2d_0b_1x7') - branch_1 = layers.conv2d( - branch_1, depth(192), [7, 1], scope='Conv2d_0c_7x1') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(192), [7, 1], scope='Conv2d_0b_7x1') - branch_2 = layers.conv2d( - branch_2, depth(192), [1, 7], scope='Conv2d_0c_1x7') - branch_2 = layers.conv2d( - branch_2, depth(192), [7, 1], scope='Conv2d_0d_7x1') - branch_2 = layers.conv2d( - branch_2, depth(192), [1, 7], scope='Conv2d_0e_1x7') - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, depth(192), [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - - # mixed_8: 8 x 8 x 1280. - end_point = 'Mixed_7a' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - branch_0 = layers.conv2d( - branch_0, - depth(320), [3, 3], - stride=2, - padding='VALID', - scope='Conv2d_1a_3x3') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, depth(192), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = layers.conv2d( - branch_1, depth(192), [1, 7], scope='Conv2d_0b_1x7') - branch_1 = layers.conv2d( - branch_1, depth(192), [7, 1], scope='Conv2d_0c_7x1') - branch_1 = layers.conv2d( - branch_1, - depth(192), [3, 3], - stride=2, - padding='VALID', - scope='Conv2d_1a_3x3') - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers_lib.max_pool2d( - net, [3, 3], stride=2, padding='VALID', scope='MaxPool_1a_3x3') - net = array_ops.concat([branch_0, branch_1, branch_2], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - # mixed_9: 8 x 8 x 2048. - end_point = 'Mixed_7b' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(320), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, depth(384), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = array_ops.concat( - [ - layers.conv2d( - branch_1, depth(384), [1, 3], scope='Conv2d_0b_1x3'), - layers.conv2d( - branch_1, depth(384), [3, 1], scope='Conv2d_0b_3x1') - ], - 3) - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, depth(448), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(384), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = array_ops.concat( - [ - layers.conv2d( - branch_2, depth(384), [1, 3], scope='Conv2d_0c_1x3'), - layers.conv2d( - branch_2, depth(384), [3, 1], scope='Conv2d_0d_3x1') - ], - 3) - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, depth(192), [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - - # mixed_10: 8 x 8 x 2048. - end_point = 'Mixed_7c' - with variable_scope.variable_scope(end_point): - with variable_scope.variable_scope('Branch_0'): - branch_0 = layers.conv2d( - net, depth(320), [1, 1], scope='Conv2d_0a_1x1') - with variable_scope.variable_scope('Branch_1'): - branch_1 = layers.conv2d( - net, depth(384), [1, 1], scope='Conv2d_0a_1x1') - branch_1 = array_ops.concat( - [ - layers.conv2d( - branch_1, depth(384), [1, 3], scope='Conv2d_0b_1x3'), - layers.conv2d( - branch_1, depth(384), [3, 1], scope='Conv2d_0c_3x1') - ], - 3) - with variable_scope.variable_scope('Branch_2'): - branch_2 = layers.conv2d( - net, depth(448), [1, 1], scope='Conv2d_0a_1x1') - branch_2 = layers.conv2d( - branch_2, depth(384), [3, 3], scope='Conv2d_0b_3x3') - branch_2 = array_ops.concat( - [ - layers.conv2d( - branch_2, depth(384), [1, 3], scope='Conv2d_0c_1x3'), - layers.conv2d( - branch_2, depth(384), [3, 1], scope='Conv2d_0d_3x1') - ], - 3) - with variable_scope.variable_scope('Branch_3'): - branch_3 = layers_lib.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = layers.conv2d( - branch_3, depth(192), [1, 1], scope='Conv2d_0b_1x1') - net = array_ops.concat([branch_0, branch_1, branch_2, branch_3], 3) - end_points[end_point] = net - if end_point == final_endpoint: - return net, end_points - raise ValueError('Unknown final endpoint %s' % final_endpoint) - - -def inception_v3(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.8, - min_depth=16, - depth_multiplier=1.0, - prediction_fn=layers_lib.softmax, - spatial_squeeze=True, - reuse=None, - scope='InceptionV3'): - """Inception model from http://arxiv.org/abs/1512.00567. - - "Rethinking the Inception Architecture for Computer Vision" - - Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, - Zbigniew Wojna. - - With the default arguments this method constructs the exact model defined in - the paper. However, one can experiment with variations of the inception_v3 - network by changing arguments dropout_keep_prob, min_depth and - depth_multiplier. - - The default image size used to train this network is 299x299. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether is training or not. - dropout_keep_prob: the percentage of activation values that are retained. - min_depth: Minimum depth value (number of channels) for all convolution ops. - Enforced when depth_multiplier < 1, and not an active constraint when - depth_multiplier >= 1. - depth_multiplier: Float multiplier for the depth (number of channels) - for all convolution ops. The value must be greater than zero. Typical - usage will be to set this value in (0, 1) to reduce the number of - parameters or computation cost of the model. - prediction_fn: a function to get predictions out of logits. - spatial_squeeze: if True, logits is of shape is [B, C], if false logits is - of shape [B, 1, 1, C], where B is batch_size and C is number of classes. - To use this parameter, the input images must be smaller - than 300x300 pixels, in which case the output logit layer - does not contain spatial information and can be removed. - reuse: whether or not the network and its variables should be reused. To be - able to reuse 'scope' must be given. - scope: Optional variable_scope. - - Returns: - logits: the pre-softmax activations, a tensor of size - [batch_size, num_classes] - end_points: a dictionary from components of the network to the corresponding - activation. - - Raises: - ValueError: if 'depth_multiplier' is less than or equal to zero. - """ - if depth_multiplier <= 0: - raise ValueError('depth_multiplier is not greater than zero.') - depth = lambda d: max(int(d * depth_multiplier), min_depth) - - with variable_scope.variable_scope( - scope, 'InceptionV3', [inputs, num_classes], reuse=reuse) as scope: - with arg_scope( - [layers_lib.batch_norm, layers_lib.dropout], is_training=is_training): - net, end_points = inception_v3_base( - inputs, - scope=scope, - min_depth=min_depth, - depth_multiplier=depth_multiplier) - - # Auxiliary Head logits - with arg_scope( - [layers.conv2d, layers_lib.max_pool2d, layers_lib.avg_pool2d], - stride=1, - padding='SAME'): - aux_logits = end_points['Mixed_6e'] - with variable_scope.variable_scope('AuxLogits'): - aux_logits = layers_lib.avg_pool2d( - aux_logits, [5, 5], - stride=3, - padding='VALID', - scope='AvgPool_1a_5x5') - aux_logits = layers.conv2d( - aux_logits, depth(128), [1, 1], scope='Conv2d_1b_1x1') - - # Shape of feature map before the final layer. - kernel_size = _reduced_kernel_size_for_small_input(aux_logits, [5, 5]) - aux_logits = layers.conv2d( - aux_logits, - depth(768), - kernel_size, - weights_initializer=trunc_normal(0.01), - padding='VALID', - scope='Conv2d_2a_{}x{}'.format(*kernel_size)) - aux_logits = layers.conv2d( - aux_logits, - num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - weights_initializer=trunc_normal(0.001), - scope='Conv2d_2b_1x1') - if spatial_squeeze: - aux_logits = array_ops.squeeze( - aux_logits, [1, 2], name='SpatialSqueeze') - end_points['AuxLogits'] = aux_logits - - # Final pooling and prediction - with variable_scope.variable_scope('Logits'): - kernel_size = _reduced_kernel_size_for_small_input(net, [8, 8]) - net = layers_lib.avg_pool2d( - net, - kernel_size, - padding='VALID', - scope='AvgPool_1a_{}x{}'.format(*kernel_size)) - # 1 x 1 x 2048 - net = layers_lib.dropout( - net, keep_prob=dropout_keep_prob, scope='Dropout_1b') - end_points['PreLogits'] = net - # 2048 - logits = layers.conv2d( - net, - num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - scope='Conv2d_1c_1x1') - if spatial_squeeze: - logits = array_ops.squeeze(logits, [1, 2], name='SpatialSqueeze') - # 1000 - end_points['Logits'] = logits - end_points['Predictions'] = prediction_fn(logits, scope='Predictions') - return logits, end_points - - -inception_v3.default_image_size = 299 - - -def _reduced_kernel_size_for_small_input(input_tensor, kernel_size): - """Define kernel size which is automatically reduced for small input. - - If the shape of the input images is unknown at graph construction time this - function assumes that the input images are is large enough. - - Args: - input_tensor: input tensor of size [batch_size, height, width, channels]. - kernel_size: desired kernel size of length 2: [kernel_height, kernel_width] - - Returns: - a tensor with the kernel size. - - TODO(jrru): Make this function work with unknown shapes. Theoretically, this - can be done with the code below. Problems are two-fold: (1) If the shape was - known, it will be lost. (2) inception.tf.contrib.slim.ops._two_element_tuple - cannot - handle tensors that define the kernel size. - shape = tf.shape(input_tensor) - return = tf.stack([tf.minimum(shape[1], kernel_size[0]), - tf.minimum(shape[2], kernel_size[1])]) - - """ - shape = input_tensor.get_shape().as_list() - if shape[1] is None or shape[2] is None: - kernel_size_out = kernel_size - else: - kernel_size_out = [ - min(shape[1], kernel_size[0]), min(shape[2], kernel_size[1]) - ] - return kernel_size_out - - -def inception_v3_arg_scope(weight_decay=0.00004, - batch_norm_var_collection='moving_vars', - batch_norm_decay=0.9997, - batch_norm_epsilon=0.001, - updates_collections=ops.GraphKeys.UPDATE_OPS, - use_fused_batchnorm=True): - """Defines the default InceptionV3 arg scope. - - Args: - weight_decay: The weight decay to use for regularizing the model. - batch_norm_var_collection: The name of the collection for the batch norm - variables. - batch_norm_decay: Decay for batch norm moving average - batch_norm_epsilon: Small float added to variance to avoid division by zero - updates_collections: Collections for the update ops of the layer - use_fused_batchnorm: Enable fused batchnorm. - - Returns: - An `arg_scope` to use for the inception v3 model. - """ - batch_norm_params = { - # Decay for the moving averages. - 'decay': batch_norm_decay, - # epsilon to prevent 0s in variance. - 'epsilon': batch_norm_epsilon, - # collection containing update_ops. - 'updates_collections': updates_collections, - # Use fused batch norm if possible. - 'fused': use_fused_batchnorm, - # collection containing the moving mean and moving variance. - 'variables_collections': { - 'beta': None, - 'gamma': None, - 'moving_mean': [batch_norm_var_collection], - 'moving_variance': [batch_norm_var_collection], - } - } - - # Set weight_decay for weights in Conv and FC layers. - with arg_scope( - [layers.conv2d, layers_lib.fully_connected], - weights_regularizer=regularizers.l2_regularizer(weight_decay)): - with arg_scope( - [layers.conv2d], - weights_initializer=initializers.variance_scaling_initializer(), - activation_fn=nn_ops.relu, - normalizer_fn=layers_lib.batch_norm, - normalizer_params=batch_norm_params) as sc: - return sc diff --git a/tensorflow/contrib/slim/python/slim/nets/inception_v3_test.py b/tensorflow/contrib/slim/python/slim/nets/inception_v3_test.py deleted file mode 100644 index 0f9cca7bbd9..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/inception_v3_test.py +++ /dev/null @@ -1,304 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for nets.inception_v3.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.framework.python.ops import variables as variables_lib -from tensorflow.contrib.slim.python.slim import model_analyzer -from tensorflow.contrib.slim.python.slim.nets import inception_v3 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class InceptionV3Test(test.TestCase): - - def testBuildClassificationNetwork(self): - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, end_points = inception_v3.inception_v3(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV3/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertTrue('Predictions' in end_points) - self.assertListEqual(end_points['Predictions'].get_shape().as_list(), - [batch_size, num_classes]) - - def testBuildBaseNetwork(self): - batch_size = 5 - height, width = 299, 299 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - final_endpoint, end_points = inception_v3.inception_v3_base(inputs) - self.assertTrue(final_endpoint.op.name.startswith('InceptionV3/Mixed_7c')) - self.assertListEqual(final_endpoint.get_shape().as_list(), - [batch_size, 8, 8, 2048]) - expected_endpoints = [ - 'Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', 'MaxPool_3a_3x3', - 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', 'MaxPool_5a_3x3', 'Mixed_5b', - 'Mixed_5c', 'Mixed_5d', 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', - 'Mixed_6e', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c' - ] - self.assertItemsEqual(end_points.keys(), expected_endpoints) - - def testBuildOnlyUptoFinalEndpoint(self): - batch_size = 5 - height, width = 299, 299 - endpoints = [ - 'Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', 'MaxPool_3a_3x3', - 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', 'MaxPool_5a_3x3', 'Mixed_5b', - 'Mixed_5c', 'Mixed_5d', 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', - 'Mixed_6e', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c' - ] - - for index, endpoint in enumerate(endpoints): - with ops.Graph().as_default(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - out_tensor, end_points = inception_v3.inception_v3_base( - inputs, final_endpoint=endpoint) - self.assertTrue( - out_tensor.op.name.startswith('InceptionV3/' + endpoint)) - self.assertItemsEqual(endpoints[:index + 1], end_points) - - def testBuildAndCheckAllEndPointsUptoMixed7c(self): - batch_size = 5 - height, width = 299, 299 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - _, end_points = inception_v3.inception_v3_base( - inputs, final_endpoint='Mixed_7c') - endpoints_shapes = { - 'Conv2d_1a_3x3': [batch_size, 149, 149, 32], - 'Conv2d_2a_3x3': [batch_size, 147, 147, 32], - 'Conv2d_2b_3x3': [batch_size, 147, 147, 64], - 'MaxPool_3a_3x3': [batch_size, 73, 73, 64], - 'Conv2d_3b_1x1': [batch_size, 73, 73, 80], - 'Conv2d_4a_3x3': [batch_size, 71, 71, 192], - 'MaxPool_5a_3x3': [batch_size, 35, 35, 192], - 'Mixed_5b': [batch_size, 35, 35, 256], - 'Mixed_5c': [batch_size, 35, 35, 288], - 'Mixed_5d': [batch_size, 35, 35, 288], - 'Mixed_6a': [batch_size, 17, 17, 768], - 'Mixed_6b': [batch_size, 17, 17, 768], - 'Mixed_6c': [batch_size, 17, 17, 768], - 'Mixed_6d': [batch_size, 17, 17, 768], - 'Mixed_6e': [batch_size, 17, 17, 768], - 'Mixed_7a': [batch_size, 8, 8, 1280], - 'Mixed_7b': [batch_size, 8, 8, 2048], - 'Mixed_7c': [batch_size, 8, 8, 2048] - } - self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys()) - for endpoint_name in endpoints_shapes: - expected_shape = endpoints_shapes[endpoint_name] - self.assertTrue(endpoint_name in end_points) - self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), - expected_shape) - - def testModelHasExpectedNumberOfParameters(self): - batch_size = 5 - height, width = 299, 299 - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - with arg_scope(inception_v3.inception_v3_arg_scope()): - inception_v3.inception_v3_base(inputs) - total_params, _ = model_analyzer.analyze_vars( - variables_lib.get_model_variables()) - self.assertAlmostEqual(21802784, total_params) - - def testBuildEndPoints(self): - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - _, end_points = inception_v3.inception_v3(inputs, num_classes) - self.assertTrue('Logits' in end_points) - logits = end_points['Logits'] - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertTrue('AuxLogits' in end_points) - aux_logits = end_points['AuxLogits'] - self.assertListEqual(aux_logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertTrue('Mixed_7c' in end_points) - pre_pool = end_points['Mixed_7c'] - self.assertListEqual(pre_pool.get_shape().as_list(), - [batch_size, 8, 8, 2048]) - self.assertTrue('PreLogits' in end_points) - pre_logits = end_points['PreLogits'] - self.assertListEqual(pre_logits.get_shape().as_list(), - [batch_size, 1, 1, 2048]) - - def testBuildEndPointsWithDepthMultiplierLessThanOne(self): - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - _, end_points = inception_v3.inception_v3(inputs, num_classes) - - endpoint_keys = [ - key for key in end_points.keys() - if key.startswith('Mixed') or key.startswith('Conv') - ] - - _, end_points_with_multiplier = inception_v3.inception_v3( - inputs, num_classes, scope='depth_multiplied_net', depth_multiplier=0.5) - - for key in endpoint_keys: - original_depth = end_points[key].get_shape().as_list()[3] - new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] - self.assertEqual(0.5 * original_depth, new_depth) - - def testBuildEndPointsWithDepthMultiplierGreaterThanOne(self): - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - _, end_points = inception_v3.inception_v3(inputs, num_classes) - - endpoint_keys = [ - key for key in end_points.keys() - if key.startswith('Mixed') or key.startswith('Conv') - ] - - _, end_points_with_multiplier = inception_v3.inception_v3( - inputs, num_classes, scope='depth_multiplied_net', depth_multiplier=2.0) - - for key in endpoint_keys: - original_depth = end_points[key].get_shape().as_list()[3] - new_depth = end_points_with_multiplier[key].get_shape().as_list()[3] - self.assertEqual(2.0 * original_depth, new_depth) - - def testRaiseValueErrorWithInvalidDepthMultiplier(self): - batch_size = 5 - height, width = 299, 299 - num_classes = 1000 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - with self.assertRaises(ValueError): - _ = inception_v3.inception_v3(inputs, num_classes, depth_multiplier=-0.1) - with self.assertRaises(ValueError): - _ = inception_v3.inception_v3(inputs, num_classes, depth_multiplier=0.0) - - def testHalfSizeImages(self): - batch_size = 5 - height, width = 150, 150 - num_classes = 1000 - - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, end_points = inception_v3.inception_v3(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV3/Logits')) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - pre_pool = end_points['Mixed_7c'] - self.assertListEqual(pre_pool.get_shape().as_list(), - [batch_size, 3, 3, 2048]) - - def testUnknownImageShape(self): - ops.reset_default_graph() - batch_size = 2 - height, width = 299, 299 - num_classes = 1000 - input_np = np.random.uniform(0, 1, (batch_size, height, width, 3)) - with self.cached_session() as sess: - inputs = array_ops.placeholder( - dtypes.float32, shape=(batch_size, None, None, 3)) - logits, end_points = inception_v3.inception_v3(inputs, num_classes) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - pre_pool = end_points['Mixed_7c'] - feed_dict = {inputs: input_np} - variables.global_variables_initializer().run() - pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict) - self.assertListEqual(list(pre_pool_out.shape), [batch_size, 8, 8, 2048]) - - def testUnknownBatchSize(self): - batch_size = 1 - height, width = 299, 299 - num_classes = 1000 - - inputs = array_ops.placeholder(dtypes.float32, (None, height, width, 3)) - logits, _ = inception_v3.inception_v3(inputs, num_classes) - self.assertTrue(logits.op.name.startswith('InceptionV3/Logits')) - self.assertListEqual(logits.get_shape().as_list(), [None, num_classes]) - images = random_ops.random_uniform((batch_size, height, width, 3)) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(logits, {inputs: images.eval()}) - self.assertEquals(output.shape, (batch_size, num_classes)) - - def testEvaluation(self): - batch_size = 2 - height, width = 299, 299 - num_classes = 1000 - - eval_inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = inception_v3.inception_v3( - eval_inputs, num_classes, is_training=False) - predictions = math_ops.argmax(logits, 1) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(predictions) - self.assertEquals(output.shape, (batch_size,)) - - def testTrainEvalWithReuse(self): - train_batch_size = 5 - eval_batch_size = 2 - height, width = 150, 150 - num_classes = 1000 - - train_inputs = random_ops.random_uniform( - (train_batch_size, height, width, 3)) - inception_v3.inception_v3(train_inputs, num_classes) - eval_inputs = random_ops.random_uniform((eval_batch_size, height, width, 3)) - logits, _ = inception_v3.inception_v3( - eval_inputs, num_classes, is_training=False, reuse=True) - predictions = math_ops.argmax(logits, 1) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(predictions) - self.assertEquals(output.shape, (eval_batch_size,)) - - def testLogitsNotSqueezed(self): - num_classes = 25 - images = random_ops.random_uniform([1, 299, 299, 3]) - logits, _ = inception_v3.inception_v3( - images, num_classes=num_classes, spatial_squeeze=False) - - with self.cached_session() as sess: - variables.global_variables_initializer().run() - logits_out = sess.run(logits) - self.assertListEqual(list(logits_out.shape), [1, 1, 1, num_classes]) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/nets/overfeat.py b/tensorflow/contrib/slim/python/slim/nets/overfeat.py deleted file mode 100644 index fb00d37f1e7..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/overfeat.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains the model definition for the OverFeat network. - -The definition for the network was obtained from: - OverFeat: Integrated Recognition, Localization and Detection using - Convolutional Networks - Pierre Sermanet, David Eigen, Xiang Zhang, Michael Mathieu, Rob Fergus and - Yann LeCun, 2014 - http://arxiv.org/abs/1312.6229 - -Usage: - with slim.arg_scope(overfeat.overfeat_arg_scope()): - outputs, end_points = overfeat.overfeat(inputs) - -@@overfeat -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.layers.python.layers import layers as layers_lib -from tensorflow.contrib.layers.python.layers import regularizers -from tensorflow.contrib.layers.python.layers import utils -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope - -trunc_normal = lambda stddev: init_ops.truncated_normal_initializer(0.0, stddev) - - -def overfeat_arg_scope(weight_decay=0.0005): - with arg_scope( - [layers.conv2d, layers_lib.fully_connected], - activation_fn=nn_ops.relu, - weights_regularizer=regularizers.l2_regularizer(weight_decay), - biases_initializer=init_ops.zeros_initializer()): - with arg_scope([layers.conv2d], padding='SAME'): - with arg_scope([layers_lib.max_pool2d], padding='VALID') as arg_sc: - return arg_sc - - -def overfeat(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.5, - spatial_squeeze=True, - scope='overfeat'): - """Contains the model definition for the OverFeat network. - - The definition for the network was obtained from: - OverFeat: Integrated Recognition, Localization and Detection using - Convolutional Networks - Pierre Sermanet, David Eigen, Xiang Zhang, Michael Mathieu, Rob Fergus and - Yann LeCun, 2014 - http://arxiv.org/abs/1312.6229 - - Note: All the fully_connected layers have been transformed to conv2d layers. - To use in classification mode, resize input to 231x231. To use in fully - convolutional mode, set spatial_squeeze to false. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether or not the model is being trained. - dropout_keep_prob: the probability that activations are kept in the dropout - layers during training. - spatial_squeeze: whether or not should squeeze the spatial dimensions of the - outputs. Useful to remove unnecessary dimensions for classification. - scope: Optional scope for the variables. - - Returns: - the last op containing the log predictions and end_points dict. - - """ - with variable_scope.variable_scope(scope, 'overfeat', [inputs]) as sc: - end_points_collection = sc.name + '_end_points' - # Collect outputs for conv2d, fully_connected and max_pool2d - with arg_scope( - [layers.conv2d, layers_lib.fully_connected, layers_lib.max_pool2d], - outputs_collections=end_points_collection): - net = layers.conv2d( - inputs, 64, [11, 11], 4, padding='VALID', scope='conv1') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool1') - net = layers.conv2d(net, 256, [5, 5], padding='VALID', scope='conv2') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool2') - net = layers.conv2d(net, 512, [3, 3], scope='conv3') - net = layers.conv2d(net, 1024, [3, 3], scope='conv4') - net = layers.conv2d(net, 1024, [3, 3], scope='conv5') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool5') - with arg_scope( - [layers.conv2d], - weights_initializer=trunc_normal(0.005), - biases_initializer=init_ops.constant_initializer(0.1)): - # Use conv2d instead of fully_connected layers. - net = layers.conv2d(net, 3072, [6, 6], padding='VALID', scope='fc6') - net = layers_lib.dropout( - net, dropout_keep_prob, is_training=is_training, scope='dropout6') - net = layers.conv2d(net, 4096, [1, 1], scope='fc7') - net = layers_lib.dropout( - net, dropout_keep_prob, is_training=is_training, scope='dropout7') - net = layers.conv2d( - net, - num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - biases_initializer=init_ops.zeros_initializer(), - scope='fc8') - # Convert end_points_collection into a end_point dict. - end_points = utils.convert_collection_to_dict(end_points_collection) - if spatial_squeeze: - net = array_ops.squeeze(net, [1, 2], name='fc8/squeezed') - end_points[sc.name + '/fc8'] = net - return net, end_points diff --git a/tensorflow/contrib/slim/python/slim/nets/overfeat_test.py b/tensorflow/contrib/slim/python/slim/nets/overfeat_test.py deleted file mode 100644 index 44fa35ad14b..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/overfeat_test.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.nets.overfeat.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework.python.ops import variables as variables_lib -from tensorflow.contrib.slim.python.slim.nets import overfeat -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class OverFeatTest(test.TestCase): - - def testBuild(self): - batch_size = 5 - height, width = 231, 231 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = overfeat.overfeat(inputs, num_classes) - self.assertEquals(logits.op.name, 'overfeat/fc8/squeezed') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testFullyConvolutional(self): - batch_size = 1 - height, width = 281, 281 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = overfeat.overfeat(inputs, num_classes, spatial_squeeze=False) - self.assertEquals(logits.op.name, 'overfeat/fc8/BiasAdd') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, 2, 2, num_classes]) - - def testEndPoints(self): - batch_size = 5 - height, width = 231, 231 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - _, end_points = overfeat.overfeat(inputs, num_classes) - expected_names = [ - 'overfeat/conv1', 'overfeat/pool1', 'overfeat/conv2', - 'overfeat/pool2', 'overfeat/conv3', 'overfeat/conv4', - 'overfeat/conv5', 'overfeat/pool5', 'overfeat/fc6', 'overfeat/fc7', - 'overfeat/fc8' - ] - self.assertSetEqual(set(end_points.keys()), set(expected_names)) - - def testModelVariables(self): - batch_size = 5 - height, width = 231, 231 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - overfeat.overfeat(inputs, num_classes) - expected_names = [ - 'overfeat/conv1/weights', - 'overfeat/conv1/biases', - 'overfeat/conv2/weights', - 'overfeat/conv2/biases', - 'overfeat/conv3/weights', - 'overfeat/conv3/biases', - 'overfeat/conv4/weights', - 'overfeat/conv4/biases', - 'overfeat/conv5/weights', - 'overfeat/conv5/biases', - 'overfeat/fc6/weights', - 'overfeat/fc6/biases', - 'overfeat/fc7/weights', - 'overfeat/fc7/biases', - 'overfeat/fc8/weights', - 'overfeat/fc8/biases', - ] - model_variables = [v.op.name for v in variables_lib.get_model_variables()] - self.assertSetEqual(set(model_variables), set(expected_names)) - - def testEvaluation(self): - batch_size = 2 - height, width = 231, 231 - num_classes = 1000 - with self.cached_session(): - eval_inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = overfeat.overfeat(eval_inputs, is_training=False) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - predictions = math_ops.argmax(logits, 1) - self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) - - def testTrainEvalWithReuse(self): - train_batch_size = 2 - eval_batch_size = 1 - train_height, train_width = 231, 231 - eval_height, eval_width = 281, 281 - num_classes = 1000 - with self.cached_session(): - train_inputs = random_ops.random_uniform( - (train_batch_size, train_height, train_width, 3)) - logits, _ = overfeat.overfeat(train_inputs) - self.assertListEqual(logits.get_shape().as_list(), - [train_batch_size, num_classes]) - variable_scope.get_variable_scope().reuse_variables() - eval_inputs = random_ops.random_uniform( - (eval_batch_size, eval_height, eval_width, 3)) - logits, _ = overfeat.overfeat( - eval_inputs, is_training=False, spatial_squeeze=False) - self.assertListEqual(logits.get_shape().as_list(), - [eval_batch_size, 2, 2, num_classes]) - logits = math_ops.reduce_mean(logits, [1, 2]) - predictions = math_ops.argmax(logits, 1) - self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) - - def testForward(self): - batch_size = 1 - height, width = 231, 231 - with self.cached_session() as sess: - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = overfeat.overfeat(inputs) - sess.run(variables.global_variables_initializer()) - output = sess.run(logits) - self.assertTrue(output.any()) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/nets/resnet_utils.py b/tensorflow/contrib/slim/python/slim/nets/resnet_utils.py deleted file mode 100644 index cfafee5d8c7..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/resnet_utils.py +++ /dev/null @@ -1,270 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains building blocks for various versions of Residual Networks. - -Residual networks (ResNets) were proposed in: - Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Deep Residual Learning for Image Recognition. arXiv:1512.03385, 2015 - -More variants were introduced in: - Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Identity Mappings in Deep Residual Networks. arXiv: 1603.05027, 2016 - -We can obtain different ResNet variants by changing the network depth, width, -and form of residual unit. This module implements the infrastructure for -building them. Concrete ResNet units and full ResNet networks are implemented in -the accompanying resnet_v1.py and resnet_v2.py modules. - -Compared to https://github.com/KaimingHe/deep-residual-networks, in the current -implementation we subsample the output activations in the last residual unit of -each block, instead of subsampling the input activations in the first residual -unit of each block. The two implementations give identical results but our -implementation is more memory efficient. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from tensorflow.contrib import layers as layers_lib -from tensorflow.contrib.framework.python.ops import add_arg_scope -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.layers.python.layers import initializers -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.layers.python.layers import regularizers -from tensorflow.contrib.layers.python.layers import utils -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope - - -class Block(collections.namedtuple('Block', ['scope', 'unit_fn', 'args'])): - """A named tuple describing a ResNet block. - - Its parts are: - scope: The scope of the `Block`. - unit_fn: The ResNet unit function which takes as input a `Tensor` and - returns another `Tensor` with the output of the ResNet unit. - args: A list of length equal to the number of units in the `Block`. The list - contains one (depth, depth_bottleneck, stride) tuple for each unit in the - block to serve as argument to unit_fn. - """ - - -def subsample(inputs, factor, scope=None): - """Subsamples the input along the spatial dimensions. - - Args: - inputs: A `Tensor` of size [batch, height_in, width_in, channels]. - factor: The subsampling factor. - scope: Optional variable_scope. - - Returns: - output: A `Tensor` of size [batch, height_out, width_out, channels] with the - input, either intact (if factor == 1) or subsampled (if factor > 1). - """ - if factor == 1: - return inputs - else: - return layers.max_pool2d(inputs, [1, 1], stride=factor, scope=scope) - - -def conv2d_same(inputs, num_outputs, kernel_size, stride, rate=1, scope=None): - """Strided 2-D convolution with 'SAME' padding. - - When stride > 1, then we do explicit zero-padding, followed by conv2d with - 'VALID' padding. - - Note that - - net = conv2d_same(inputs, num_outputs, 3, stride=stride) - - is equivalent to - - net = tf.contrib.layers.conv2d(inputs, num_outputs, 3, stride=1, - padding='SAME') - net = subsample(net, factor=stride) - - whereas - - net = tf.contrib.layers.conv2d(inputs, num_outputs, 3, stride=stride, - padding='SAME') - - is different when the input's height or width is even, which is why we add the - current function. For more details, see ResnetUtilsTest.testConv2DSameEven(). - - Args: - inputs: A 4-D tensor of size [batch, height_in, width_in, channels]. - num_outputs: An integer, the number of output filters. - kernel_size: An int with the kernel_size of the filters. - stride: An integer, the output stride. - rate: An integer, rate for atrous convolution. - scope: Scope. - - Returns: - output: A 4-D tensor of size [batch, height_out, width_out, channels] with - the convolution output. - """ - if stride == 1: - return layers_lib.conv2d( - inputs, - num_outputs, - kernel_size, - stride=1, - rate=rate, - padding='SAME', - scope=scope) - else: - kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1) - pad_total = kernel_size_effective - 1 - pad_beg = pad_total // 2 - pad_end = pad_total - pad_beg - inputs = array_ops.pad( - inputs, [[0, 0], [pad_beg, pad_end], [pad_beg, pad_end], [0, 0]]) - return layers_lib.conv2d( - inputs, - num_outputs, - kernel_size, - stride=stride, - rate=rate, - padding='VALID', - scope=scope) - - -@add_arg_scope -def stack_blocks_dense(net, - blocks, - output_stride=None, - outputs_collections=None): - """Stacks ResNet `Blocks` and controls output feature density. - - First, this function creates scopes for the ResNet in the form of - 'block_name/unit_1', 'block_name/unit_2', etc. - - Second, this function allows the user to explicitly control the ResNet - output_stride, which is the ratio of the input to output spatial resolution. - This is useful for dense prediction tasks such as semantic segmentation or - object detection. - - Most ResNets consist of 4 ResNet blocks and subsample the activations by a - factor of 2 when transitioning between consecutive ResNet blocks. This results - to a nominal ResNet output_stride equal to 8. If we set the output_stride to - half the nominal network stride (e.g., output_stride=4), then we compute - responses twice. - - Control of the output feature density is implemented by atrous convolution. - - Args: - net: A `Tensor` of size [batch, height, width, channels]. - blocks: A list of length equal to the number of ResNet `Blocks`. Each - element is a ResNet `Block` object describing the units in the `Block`. - output_stride: If `None`, then the output will be computed at the nominal - network stride. If output_stride is not `None`, it specifies the requested - ratio of input to output spatial resolution, which needs to be equal to - the product of unit strides from the start up to some level of the ResNet. - For example, if the ResNet employs units with strides 1, 2, 1, 3, 4, 1, - then valid values for the output_stride are 1, 2, 6, 24 or None (which - is equivalent to output_stride=24). - outputs_collections: Collection to add the ResNet block outputs. - - Returns: - net: Output tensor with stride equal to the specified output_stride. - - Raises: - ValueError: If the target output_stride is not valid. - """ - # The current_stride variable keeps track of the effective stride of the - # activations. This allows us to invoke atrous convolution whenever applying - # the next residual unit would result in the activations having stride larger - # than the target output_stride. - current_stride = 1 - - # The atrous convolution rate parameter. - rate = 1 - - for block in blocks: - with variable_scope.variable_scope(block.scope, 'block', [net]) as sc: - for i, unit in enumerate(block.args): - if output_stride is not None and current_stride > output_stride: - raise ValueError('The target output_stride cannot be reached.') - - with variable_scope.variable_scope('unit_%d' % (i + 1), values=[net]): - # If we have reached the target output_stride, then we need to employ - # atrous convolution with stride=1 and multiply the atrous rate by the - # current unit's stride for use in subsequent layers. - if output_stride is not None and current_stride == output_stride: - net = block.unit_fn(net, rate=rate, **dict(unit, stride=1)) - rate *= unit.get('stride', 1) - - else: - net = block.unit_fn(net, rate=1, **unit) - current_stride *= unit.get('stride', 1) - net = utils.collect_named_outputs(outputs_collections, sc.name, net) - - if output_stride is not None and current_stride != output_stride: - raise ValueError('The target output_stride cannot be reached.') - - return net - - -def resnet_arg_scope(weight_decay=0.0001, - batch_norm_decay=0.997, - batch_norm_epsilon=1e-5, - batch_norm_scale=True): - """Defines the default ResNet arg scope. - - TODO(gpapan): The batch-normalization related default values above are - appropriate for use in conjunction with the reference ResNet models - released at https://github.com/KaimingHe/deep-residual-networks. When - training ResNets from scratch, they might need to be tuned. - - Args: - weight_decay: The weight decay to use for regularizing the model. - batch_norm_decay: The moving average decay when estimating layer activation - statistics in batch normalization. - batch_norm_epsilon: Small constant to prevent division by zero when - normalizing activations by their variance in batch normalization. - batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the - activations in the batch normalization layer. - - Returns: - An `arg_scope` to use for the resnet models. - """ - batch_norm_params = { - 'decay': batch_norm_decay, - 'epsilon': batch_norm_epsilon, - 'scale': batch_norm_scale, - 'updates_collections': ops.GraphKeys.UPDATE_OPS, - } - - with arg_scope( - [layers_lib.conv2d], - weights_regularizer=regularizers.l2_regularizer(weight_decay), - weights_initializer=initializers.variance_scaling_initializer(), - activation_fn=nn_ops.relu, - normalizer_fn=layers.batch_norm, - normalizer_params=batch_norm_params): - with arg_scope([layers.batch_norm], **batch_norm_params): - # The following implies padding='SAME' for pool1, which makes feature - # alignment easier for dense prediction tasks. This is also used in - # https://github.com/facebook/fb.resnet.torch. However the accompanying - # code of 'Deep Residual Learning for Image Recognition' uses - # padding='VALID' for pool1. You can switch to that choice by setting - # tf.contrib.framework.arg_scope([tf.contrib.layers.max_pool2d], padding='VALID'). - with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc: - return arg_sc diff --git a/tensorflow/contrib/slim/python/slim/nets/resnet_v1.py b/tensorflow/contrib/slim/python/slim/nets/resnet_v1.py deleted file mode 100644 index 11c4214176a..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/resnet_v1.py +++ /dev/null @@ -1,352 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains definitions for the original form of Residual Networks. - -The 'v1' residual networks (ResNets) implemented in this module were proposed -by: -[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Deep Residual Learning for Image Recognition. arXiv:1512.03385 - -Other variants were introduced in: -[2] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Identity Mappings in Deep Residual Networks. arXiv: 1603.05027 - -The networks defined in this module utilize the bottleneck building block of -[1] with projection shortcuts only for increasing depths. They employ batch -normalization *after* every weight layer. This is the architecture used by -MSRA in the Imagenet and MSCOCO 2016 competition models ResNet-101 and -ResNet-152. See [2; Fig. 1a] for a comparison between the current 'v1' -architecture and the alternative 'v2' architecture of [2] which uses batch -normalization *before* every weight layer in the so-called full pre-activation -units. - -Typical use: - - from tensorflow.contrib.slim.python.slim.nets import - resnet_v1 - -ResNet-101 for image classification into 1000 classes: - - # inputs has shape [batch, 224, 224, 3] - with slim.arg_scope(resnet_v1.resnet_arg_scope()): - net, end_points = resnet_v1.resnet_v1_101(inputs, 1000, is_training=False) - -ResNet-101 for semantic segmentation into 21 classes: - - # inputs has shape [batch, 513, 513, 3] - with slim.arg_scope(resnet_v1.resnet_arg_scope()): - net, end_points = resnet_v1.resnet_v1_101(inputs, - 21, - is_training=False, - global_pool=False, - output_stride=16) -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.framework.python.ops import add_arg_scope -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.layers.python.layers import layers as layers_lib -from tensorflow.contrib.layers.python.layers import utils -from tensorflow.contrib.slim.python.slim.nets import resnet_utils -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope - -resnet_arg_scope = resnet_utils.resnet_arg_scope - - -@add_arg_scope -def bottleneck(inputs, - depth, - depth_bottleneck, - stride, - rate=1, - outputs_collections=None, - scope=None): - """Bottleneck residual unit variant with BN after convolutions. - - This is the original residual unit proposed in [1]. See Fig. 1(a) of [2] for - its definition. Note that we use here the bottleneck variant which has an - extra bottleneck layer. - - When putting together two consecutive ResNet blocks that use this unit, one - should use stride = 2 in the last unit of the first block. - - Args: - inputs: A tensor of size [batch, height, width, channels]. - depth: The depth of the ResNet unit output. - depth_bottleneck: The depth of the bottleneck layers. - stride: The ResNet unit's stride. Determines the amount of downsampling of - the units output compared to its input. - rate: An integer, rate for atrous convolution. - outputs_collections: Collection to add the ResNet unit output. - scope: Optional variable_scope. - - Returns: - The ResNet unit's output. - """ - with variable_scope.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc: - depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) - if depth == depth_in: - shortcut = resnet_utils.subsample(inputs, stride, 'shortcut') - else: - shortcut = layers.conv2d( - inputs, - depth, [1, 1], - stride=stride, - activation_fn=None, - scope='shortcut') - - residual = layers.conv2d( - inputs, depth_bottleneck, [1, 1], stride=1, scope='conv1') - residual = resnet_utils.conv2d_same( - residual, depth_bottleneck, 3, stride, rate=rate, scope='conv2') - residual = layers.conv2d( - residual, depth, [1, 1], stride=1, activation_fn=None, scope='conv3') - - output = nn_ops.relu(shortcut + residual) - - return utils.collect_named_outputs(outputs_collections, sc.name, output) - - -def resnet_v1(inputs, - blocks, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - include_root_block=True, - reuse=None, - scope=None): - """Generator for v1 ResNet models. - - This function generates a family of ResNet v1 models. See the resnet_v1_*() - methods for specific model instantiations, obtained by selecting different - block instantiations that produce ResNets of various depths. - - Training for image classification on Imagenet is usually done with [224, 224] - inputs, resulting in [7, 7] feature maps at the output of the last ResNet - block for the ResNets defined in [1] that have nominal stride equal to 32. - However, for dense prediction tasks we advise that one uses inputs with - spatial dimensions that are multiples of 32 plus 1, e.g., [321, 321]. In - this case the feature maps at the ResNet output will have spatial shape - [(height - 1) / output_stride + 1, (width - 1) / output_stride + 1] - and corners exactly aligned with the input image corners, which greatly - facilitates alignment of the features to the image. Using as input [225, 225] - images results in [8, 8] feature maps at the output of the last ResNet block. - - For dense prediction tasks, the ResNet needs to run in fully-convolutional - (FCN) mode and global_pool needs to be set to False. The ResNets in [1, 2] all - have nominal stride equal to 32 and a good choice in FCN mode is to use - output_stride=16 in order to increase the density of the computed features at - small computational and memory overhead, cf. http://arxiv.org/abs/1606.00915. - - Args: - inputs: A tensor of size [batch, height_in, width_in, channels]. - blocks: A list of length equal to the number of ResNet blocks. Each element - is a resnet_utils.Block object describing the units in the block. - num_classes: Number of predicted classes for classification tasks. If None - we return the features before the logit layer. - is_training: whether batch_norm layers are in training mode. - global_pool: If True, we perform global average pooling before computing the - logits. Set to True for image classification, False for dense prediction. - output_stride: If None, then the output will be computed at the nominal - network stride. If output_stride is not None, it specifies the requested - ratio of input to output spatial resolution. - include_root_block: If True, include the initial convolution followed by - max-pooling, if False excludes it. - reuse: whether or not the network and its variables should be reused. To be - able to reuse 'scope' must be given. - scope: Optional variable_scope. - - Returns: - net: A rank-4 tensor of size [batch, height_out, width_out, channels_out]. - If global_pool is False, then height_out and width_out are reduced by a - factor of output_stride compared to the respective height_in and width_in, - else both height_out and width_out equal one. If num_classes is None, then - net is the output of the last ResNet block, potentially after global - average pooling. If num_classes is not None, net contains the pre-softmax - activations. - end_points: A dictionary from components of the network to the corresponding - activation. - - Raises: - ValueError: If the target output_stride is not valid. - """ - with variable_scope.variable_scope( - scope, 'resnet_v1', [inputs], reuse=reuse) as sc: - end_points_collection = sc.original_name_scope + '_end_points' - with arg_scope( - [layers.conv2d, bottleneck, resnet_utils.stack_blocks_dense], - outputs_collections=end_points_collection): - with arg_scope([layers.batch_norm], is_training=is_training): - net = inputs - if include_root_block: - if output_stride is not None: - if output_stride % 4 != 0: - raise ValueError('The output_stride needs to be a multiple of 4.') - output_stride /= 4 - net = resnet_utils.conv2d_same(net, 64, 7, stride=2, scope='conv1') - net = layers_lib.max_pool2d(net, [3, 3], stride=2, scope='pool1') - net = resnet_utils.stack_blocks_dense(net, blocks, output_stride) - if global_pool: - # Global average pooling. - net = math_ops.reduce_mean(net, [1, 2], name='pool5', keepdims=True) - if num_classes is not None: - net = layers.conv2d( - net, - num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - scope='logits') - # Convert end_points_collection into a dictionary of end_points. - end_points = utils.convert_collection_to_dict(end_points_collection) - if num_classes is not None: - end_points['predictions'] = layers_lib.softmax( - net, scope='predictions') - return net, end_points -resnet_v1.default_image_size = 224 - - -def resnet_v1_block(scope, base_depth, num_units, stride): - """Helper function for creating a resnet_v1 bottleneck block. - - Args: - scope: The scope of the block. - base_depth: The depth of the bottleneck layer for each unit. - num_units: The number of units in the block. - stride: The stride of the block, implemented as a stride in the last unit. - All other units have stride=1. - - Returns: - A resnet_v1 bottleneck block. - """ - return resnet_utils.Block(scope, bottleneck, [{ - 'depth': base_depth * 4, - 'depth_bottleneck': base_depth, - 'stride': 1 - }] * (num_units - 1) + [{ - 'depth': base_depth * 4, - 'depth_bottleneck': base_depth, - 'stride': stride - }]) - - -def resnet_v1_50(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v1_50'): - """ResNet-50 model of [1]. See resnet_v1() for arg and return description.""" - blocks = [ - resnet_v1_block('block1', base_depth=64, num_units=3, stride=2), - resnet_v1_block('block2', base_depth=128, num_units=4, stride=2), - resnet_v1_block('block3', base_depth=256, num_units=6, stride=2), - resnet_v1_block('block4', base_depth=512, num_units=3, stride=1), - ] - return resnet_v1( - inputs, - blocks, - num_classes, - is_training, - global_pool, - output_stride, - include_root_block=True, - reuse=reuse, - scope=scope) - - -def resnet_v1_101(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v1_101'): - """ResNet-101 model of [1]. See resnet_v1() for arg and return description.""" - blocks = [ - resnet_v1_block('block1', base_depth=64, num_units=3, stride=2), - resnet_v1_block('block2', base_depth=128, num_units=4, stride=2), - resnet_v1_block('block3', base_depth=256, num_units=23, stride=2), - resnet_v1_block('block4', base_depth=512, num_units=3, stride=1), - ] - return resnet_v1( - inputs, - blocks, - num_classes, - is_training, - global_pool, - output_stride, - include_root_block=True, - reuse=reuse, - scope=scope) - - -def resnet_v1_152(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v1_152'): - """ResNet-152 model of [1]. See resnet_v1() for arg and return description.""" - blocks = [ - resnet_v1_block('block1', base_depth=64, num_units=3, stride=2), - resnet_v1_block('block2', base_depth=128, num_units=8, stride=2), - resnet_v1_block('block3', base_depth=256, num_units=36, stride=2), - resnet_v1_block('block4', base_depth=512, num_units=3, stride=1), - ] - return resnet_v1( - inputs, - blocks, - num_classes, - is_training, - global_pool, - output_stride, - include_root_block=True, - reuse=reuse, - scope=scope) - - -def resnet_v1_200(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v1_200'): - """ResNet-200 model of [2]. See resnet_v1() for arg and return description.""" - blocks = [ - resnet_v1_block('block1', base_depth=64, num_units=3, stride=2), - resnet_v1_block('block2', base_depth=128, num_units=24, stride=2), - resnet_v1_block('block3', base_depth=256, num_units=36, stride=2), - resnet_v1_block('block4', base_depth=512, num_units=3, stride=1), - ] - return resnet_v1( - inputs, - blocks, - num_classes, - is_training, - global_pool, - output_stride, - include_root_block=True, - reuse=reuse, - scope=scope) diff --git a/tensorflow/contrib/slim/python/slim/nets/resnet_v1_test.py b/tensorflow/contrib/slim/python/slim/nets/resnet_v1_test.py deleted file mode 100644 index 1cc54b15514..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/resnet_v1_test.py +++ /dev/null @@ -1,449 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.nets.resnet_v1.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib import layers -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.layers.python.layers import utils -from tensorflow.contrib.slim.python.slim.nets import resnet_utils -from tensorflow.contrib.slim.python.slim.nets import resnet_v1 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def create_test_input(batch_size, height, width, channels): - """Create test input tensor. - - Args: - batch_size: The number of images per batch or `None` if unknown. - height: The height of each image or `None` if unknown. - width: The width of each image or `None` if unknown. - channels: The number of channels per image or `None` if unknown. - - Returns: - Either a placeholder `Tensor` of dimension - [batch_size, height, width, channels] if any of the inputs are `None` or a - constant `Tensor` with the mesh grid values along the spatial dimensions. - """ - if None in [batch_size, height, width, channels]: - return array_ops.placeholder(dtypes.float32, - (batch_size, height, width, channels)) - else: - return math_ops.cast( - np.tile( - np.reshape( - np.reshape(np.arange(height), [height, 1]) + np.reshape( - np.arange(width), [1, width]), [1, height, width, 1]), - [batch_size, 1, 1, channels]), dtypes.float32) - - -class ResnetUtilsTest(test.TestCase): - - def testSubsampleThreeByThree(self): - x = array_ops.reshape(math_ops.cast(math_ops.range(9), dtypes.float32), - [1, 3, 3, 1]) - x = resnet_utils.subsample(x, 2) - expected = array_ops.reshape( - constant_op.constant([0, 2, 6, 8]), [1, 2, 2, 1]) - with self.cached_session(): - self.assertAllClose(x.eval(), expected.eval()) - - def testSubsampleFourByFour(self): - x = array_ops.reshape(math_ops.cast(math_ops.range(16), dtypes.float32), - [1, 4, 4, 1]) - x = resnet_utils.subsample(x, 2) - expected = array_ops.reshape( - constant_op.constant([0, 2, 8, 10]), [1, 2, 2, 1]) - with self.cached_session(): - self.assertAllClose(x.eval(), expected.eval()) - - def testConv2DSameEven(self): - n, n2 = 4, 2 - - # Input image. - x = create_test_input(1, n, n, 1) - - # Convolution kernel. - w = create_test_input(1, 3, 3, 1) - w = array_ops.reshape(w, [3, 3, 1, 1]) - - variable_scope.get_variable('Conv/weights', initializer=w) - variable_scope.get_variable('Conv/biases', initializer=array_ops.zeros([1])) - variable_scope.get_variable_scope().reuse_variables() - - y1 = layers.conv2d(x, 1, [3, 3], stride=1, scope='Conv') - y1_expected = math_ops.cast([[14, 28, 43, 26], [28, 48, 66, 37], - [43, 66, 84, 46], [26, 37, 46, 22]], - dtypes.float32) - y1_expected = array_ops.reshape(y1_expected, [1, n, n, 1]) - - y2 = resnet_utils.subsample(y1, 2) - y2_expected = math_ops.cast([[14, 43], [43, 84]], dtypes.float32) - y2_expected = array_ops.reshape(y2_expected, [1, n2, n2, 1]) - - y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv') - y3_expected = y2_expected - - y4 = layers.conv2d(x, 1, [3, 3], stride=2, scope='Conv') - y4_expected = math_ops.cast([[48, 37], [37, 22]], dtypes.float32) - y4_expected = array_ops.reshape(y4_expected, [1, n2, n2, 1]) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertAllClose(y1.eval(), y1_expected.eval()) - self.assertAllClose(y2.eval(), y2_expected.eval()) - self.assertAllClose(y3.eval(), y3_expected.eval()) - self.assertAllClose(y4.eval(), y4_expected.eval()) - - def testConv2DSameOdd(self): - n, n2 = 5, 3 - - # Input image. - x = create_test_input(1, n, n, 1) - - # Convolution kernel. - w = create_test_input(1, 3, 3, 1) - w = array_ops.reshape(w, [3, 3, 1, 1]) - - variable_scope.get_variable('Conv/weights', initializer=w) - variable_scope.get_variable('Conv/biases', initializer=array_ops.zeros([1])) - variable_scope.get_variable_scope().reuse_variables() - - y1 = layers.conv2d(x, 1, [3, 3], stride=1, scope='Conv') - y1_expected = math_ops.cast([[14, 28, 43, 58, 34], - [28, 48, 66, 84, 46], - [43, 66, 84, 102, 55], - [58, 84, 102, 120, 64], - [34, 46, 55, 64, 30]], - dtypes.float32) - y1_expected = array_ops.reshape(y1_expected, [1, n, n, 1]) - - y2 = resnet_utils.subsample(y1, 2) - y2_expected = math_ops.cast([[14, 43, 34], - [43, 84, 55], - [34, 55, 30]], - dtypes.float32) - y2_expected = array_ops.reshape(y2_expected, [1, n2, n2, 1]) - - y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv') - y3_expected = y2_expected - - y4 = layers.conv2d(x, 1, [3, 3], stride=2, scope='Conv') - y4_expected = y2_expected - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertAllClose(y1.eval(), y1_expected.eval()) - self.assertAllClose(y2.eval(), y2_expected.eval()) - self.assertAllClose(y3.eval(), y3_expected.eval()) - self.assertAllClose(y4.eval(), y4_expected.eval()) - - def _resnet_plain(self, inputs, blocks, output_stride=None, scope=None): - """A plain ResNet without extra layers before or after the ResNet blocks.""" - with variable_scope.variable_scope(scope, values=[inputs]): - with arg_scope([layers.conv2d], outputs_collections='end_points'): - net = resnet_utils.stack_blocks_dense(inputs, blocks, output_stride) - end_points = utils.convert_collection_to_dict('end_points') - return net, end_points - - def testEndPointsV1(self): - """Test the end points of a tiny v1 bottleneck network.""" - blocks = [ - resnet_v1.resnet_v1_block( - 'block1', base_depth=1, num_units=2, stride=2), - resnet_v1.resnet_v1_block( - 'block2', base_depth=2, num_units=2, stride=1), - ] - inputs = create_test_input(2, 32, 16, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_plain(inputs, blocks, scope='tiny') - expected = [ - 'tiny/block1/unit_1/bottleneck_v1/shortcut', - 'tiny/block1/unit_1/bottleneck_v1/conv1', - 'tiny/block1/unit_1/bottleneck_v1/conv2', - 'tiny/block1/unit_1/bottleneck_v1/conv3', - 'tiny/block1/unit_2/bottleneck_v1/conv1', - 'tiny/block1/unit_2/bottleneck_v1/conv2', - 'tiny/block1/unit_2/bottleneck_v1/conv3', - 'tiny/block2/unit_1/bottleneck_v1/shortcut', - 'tiny/block2/unit_1/bottleneck_v1/conv1', - 'tiny/block2/unit_1/bottleneck_v1/conv2', - 'tiny/block2/unit_1/bottleneck_v1/conv3', - 'tiny/block2/unit_2/bottleneck_v1/conv1', - 'tiny/block2/unit_2/bottleneck_v1/conv2', - 'tiny/block2/unit_2/bottleneck_v1/conv3'] - self.assertItemsEqual(expected, end_points) - - def _stack_blocks_nondense(self, net, blocks): - """A simplified ResNet Block stacker without output stride control.""" - for block in blocks: - with variable_scope.variable_scope(block.scope, 'block', [net]): - for i, unit in enumerate(block.args): - with variable_scope.variable_scope('unit_%d' % (i + 1), values=[net]): - net = block.unit_fn(net, rate=1, **unit) - return net - - def testAtrousValuesBottleneck(self): - """Verify the values of dense feature extraction by atrous convolution. - - Make sure that dense feature extraction by stack_blocks_dense() followed by - subsampling gives identical results to feature extraction at the nominal - network output stride using the simple self._stack_blocks_nondense() above. - """ - block = resnet_v1.resnet_v1_block - blocks = [ - block('block1', base_depth=1, num_units=2, stride=2), - block('block2', base_depth=2, num_units=2, stride=2), - block('block3', base_depth=4, num_units=2, stride=2), - block('block4', base_depth=8, num_units=2, stride=1), - ] - nominal_stride = 8 - - # Test both odd and even input dimensions. - height = 30 - width = 31 - with arg_scope(resnet_utils.resnet_arg_scope()): - with arg_scope([layers.batch_norm], is_training=False): - for output_stride in [1, 2, 4, 8, None]: - with ops.Graph().as_default(): - with self.cached_session() as sess: - random_seed.set_random_seed(0) - inputs = create_test_input(1, height, width, 3) - # Dense feature extraction followed by subsampling. - output = resnet_utils.stack_blocks_dense(inputs, blocks, - output_stride) - if output_stride is None: - factor = 1 - else: - factor = nominal_stride // output_stride - - output = resnet_utils.subsample(output, factor) - # Make the two networks use the same weights. - variable_scope.get_variable_scope().reuse_variables() - # Feature extraction at the nominal network rate. - expected = self._stack_blocks_nondense(inputs, blocks) - sess.run(variables.global_variables_initializer()) - output, expected = sess.run([output, expected]) - self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4) - - -class ResnetCompleteNetworkTest(test.TestCase): - """Tests with complete small ResNet v1 networks.""" - - def _resnet_small(self, - inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - include_root_block=True, - reuse=None, - scope='resnet_v1_small'): - """A shallow and thin ResNet v1 for faster tests.""" - block = resnet_v1.resnet_v1_block - blocks = [ - block('block1', base_depth=1, num_units=3, stride=2), - block('block2', base_depth=2, num_units=3, stride=2), - block('block3', base_depth=4, num_units=3, stride=2), - block('block4', base_depth=8, num_units=2, stride=1), - ] - return resnet_v1.resnet_v1(inputs, blocks, num_classes, is_training, - global_pool, output_stride, include_root_block, - reuse, scope) - - def testClassificationEndPoints(self): - global_pool = True - num_classes = 10 - inputs = create_test_input(2, 224, 224, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - logits, end_points = self._resnet_small( - inputs, num_classes, global_pool=global_pool, scope='resnet') - self.assertTrue(logits.op.name.startswith('resnet/logits')) - self.assertListEqual(logits.get_shape().as_list(), [2, 1, 1, num_classes]) - self.assertTrue('predictions' in end_points) - self.assertListEqual(end_points['predictions'].get_shape().as_list(), - [2, 1, 1, num_classes]) - - def testClassificationShapes(self): - global_pool = True - num_classes = 10 - inputs = create_test_input(2, 224, 224, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small( - inputs, num_classes, global_pool=global_pool, scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 28, 28, 4], - 'resnet/block2': [2, 14, 14, 8], - 'resnet/block3': [2, 7, 7, 16], - 'resnet/block4': [2, 7, 7, 32] - } - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testFullyConvolutionalEndpointShapes(self): - global_pool = False - num_classes = 10 - inputs = create_test_input(2, 321, 321, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small( - inputs, num_classes, global_pool=global_pool, scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 41, 41, 4], - 'resnet/block2': [2, 21, 21, 8], - 'resnet/block3': [2, 11, 11, 16], - 'resnet/block4': [2, 11, 11, 32] - } - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testRootlessFullyConvolutionalEndpointShapes(self): - global_pool = False - num_classes = 10 - inputs = create_test_input(2, 128, 128, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small( - inputs, - num_classes, - global_pool=global_pool, - include_root_block=False, - scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 64, 64, 4], - 'resnet/block2': [2, 32, 32, 8], - 'resnet/block3': [2, 16, 16, 16], - 'resnet/block4': [2, 16, 16, 32] - } - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testAtrousFullyConvolutionalEndpointShapes(self): - global_pool = False - num_classes = 10 - output_stride = 8 - inputs = create_test_input(2, 321, 321, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small( - inputs, - num_classes, - global_pool=global_pool, - output_stride=output_stride, - scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 41, 41, 4], - 'resnet/block2': [2, 41, 41, 8], - 'resnet/block3': [2, 41, 41, 16], - 'resnet/block4': [2, 41, 41, 32] - } - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testAtrousFullyConvolutionalValues(self): - """Verify dense feature extraction with atrous convolution.""" - nominal_stride = 32 - for output_stride in [4, 8, 16, 32, None]: - with arg_scope(resnet_utils.resnet_arg_scope()): - with ops.Graph().as_default(): - with self.cached_session() as sess: - random_seed.set_random_seed(0) - inputs = create_test_input(2, 81, 81, 3) - # Dense feature extraction followed by subsampling. - output, _ = self._resnet_small( - inputs, - None, - is_training=False, - global_pool=False, - output_stride=output_stride) - if output_stride is None: - factor = 1 - else: - factor = nominal_stride // output_stride - output = resnet_utils.subsample(output, factor) - # Make the two networks use the same weights. - variable_scope.get_variable_scope().reuse_variables() - # Feature extraction at the nominal network rate. - expected, _ = self._resnet_small( - inputs, None, is_training=False, global_pool=False) - sess.run(variables.global_variables_initializer()) - self.assertAllClose( - output.eval(), expected.eval(), atol=2e-4, rtol=1e-4) - - def testUnknownBatchSize(self): - batch = 2 - height, width = 65, 65 - global_pool = True - num_classes = 10 - inputs = create_test_input(None, height, width, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - logits, _ = self._resnet_small( - inputs, num_classes, global_pool=global_pool, scope='resnet') - self.assertTrue(logits.op.name.startswith('resnet/logits')) - self.assertListEqual(logits.get_shape().as_list(), - [None, 1, 1, num_classes]) - images = create_test_input(batch, height, width, 3) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(logits, {inputs: images.eval()}) - self.assertEqual(output.shape, (batch, 1, 1, num_classes)) - - def testFullyConvolutionalUnknownHeightWidth(self): - batch = 2 - height, width = 65, 65 - global_pool = False - inputs = create_test_input(batch, None, None, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - output, _ = self._resnet_small(inputs, None, global_pool=global_pool) - self.assertListEqual(output.get_shape().as_list(), [batch, None, None, 32]) - images = create_test_input(batch, height, width, 3) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(output, {inputs: images.eval()}) - self.assertEqual(output.shape, (batch, 3, 3, 32)) - - def testAtrousFullyConvolutionalUnknownHeightWidth(self): - batch = 2 - height, width = 65, 65 - global_pool = False - output_stride = 8 - inputs = create_test_input(batch, None, None, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - output, _ = self._resnet_small( - inputs, None, global_pool=global_pool, output_stride=output_stride) - self.assertListEqual(output.get_shape().as_list(), [batch, None, None, 32]) - images = create_test_input(batch, height, width, 3) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(output, {inputs: images.eval()}) - self.assertEqual(output.shape, (batch, 9, 9, 32)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/nets/resnet_v2.py b/tensorflow/contrib/slim/python/slim/nets/resnet_v2.py deleted file mode 100644 index 19e0538dd1e..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/resnet_v2.py +++ /dev/null @@ -1,365 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains definitions for the preactivation form of Residual Networks. - -Residual networks (ResNets) were originally proposed in: -[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Deep Residual Learning for Image Recognition. arXiv:1512.03385 - -The full preactivation 'v2' ResNet variant implemented in this module was -introduced by: -[2] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Identity Mappings in Deep Residual Networks. arXiv: 1603.05027 - -The key difference of the full preactivation 'v2' variant compared to the -'v1' variant in [1] is the use of batch normalization before every weight layer. - -Typical use: - - from tensorflow.contrib.slim.python.slim.nets import - resnet_v2 - -ResNet-101 for image classification into 1000 classes: - - # inputs has shape [batch, 224, 224, 3] - with slim.arg_scope(resnet_v2.resnet_arg_scope()): - net, end_points = resnet_v2.resnet_v2_101(inputs, 1000, is_training=False) - -ResNet-101 for semantic segmentation into 21 classes: - - # inputs has shape [batch, 513, 513, 3] - with slim.arg_scope(resnet_v2.resnet_arg_scope()): - net, end_points = resnet_v2.resnet_v2_101(inputs, - 21, - is_training=False, - global_pool=False, - output_stride=16) -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers as layers_lib -from tensorflow.contrib.framework.python.ops import add_arg_scope -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.layers.python.layers import utils -from tensorflow.contrib.slim.python.slim.nets import resnet_utils -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope - -resnet_arg_scope = resnet_utils.resnet_arg_scope - - -@add_arg_scope -def bottleneck(inputs, - depth, - depth_bottleneck, - stride, - rate=1, - outputs_collections=None, - scope=None): - """Bottleneck residual unit variant with BN before convolutions. - - This is the full preactivation residual unit variant proposed in [2]. See - Fig. 1(b) of [2] for its definition. Note that we use here the bottleneck - variant which has an extra bottleneck layer. - - When putting together two consecutive ResNet blocks that use this unit, one - should use stride = 2 in the last unit of the first block. - - Args: - inputs: A tensor of size [batch, height, width, channels]. - depth: The depth of the ResNet unit output. - depth_bottleneck: The depth of the bottleneck layers. - stride: The ResNet unit's stride. Determines the amount of downsampling of - the units output compared to its input. - rate: An integer, rate for atrous convolution. - outputs_collections: Collection to add the ResNet unit output. - scope: Optional variable_scope. - - Returns: - The ResNet unit's output. - """ - with variable_scope.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc: - depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) - preact = layers.batch_norm( - inputs, activation_fn=nn_ops.relu, scope='preact') - if depth == depth_in: - shortcut = resnet_utils.subsample(inputs, stride, 'shortcut') - else: - shortcut = layers_lib.conv2d( - preact, - depth, [1, 1], - stride=stride, - normalizer_fn=None, - activation_fn=None, - scope='shortcut') - - residual = layers_lib.conv2d( - preact, depth_bottleneck, [1, 1], stride=1, scope='conv1') - residual = resnet_utils.conv2d_same( - residual, depth_bottleneck, 3, stride, rate=rate, scope='conv2') - residual = layers_lib.conv2d( - residual, - depth, [1, 1], - stride=1, - normalizer_fn=None, - activation_fn=None, - scope='conv3') - - output = shortcut + residual - - return utils.collect_named_outputs(outputs_collections, sc.name, output) - - -def resnet_v2(inputs, - blocks, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - include_root_block=True, - reuse=None, - scope=None): - """Generator for v2 (preactivation) ResNet models. - - This function generates a family of ResNet v2 models. See the resnet_v2_*() - methods for specific model instantiations, obtained by selecting different - block instantiations that produce ResNets of various depths. - - Training for image classification on Imagenet is usually done with [224, 224] - inputs, resulting in [7, 7] feature maps at the output of the last ResNet - block for the ResNets defined in [1] that have nominal stride equal to 32. - However, for dense prediction tasks we advise that one uses inputs with - spatial dimensions that are multiples of 32 plus 1, e.g., [321, 321]. In - this case the feature maps at the ResNet output will have spatial shape - [(height - 1) / output_stride + 1, (width - 1) / output_stride + 1] - and corners exactly aligned with the input image corners, which greatly - facilitates alignment of the features to the image. Using as input [225, 225] - images results in [8, 8] feature maps at the output of the last ResNet block. - - For dense prediction tasks, the ResNet needs to run in fully-convolutional - (FCN) mode and global_pool needs to be set to False. The ResNets in [1, 2] all - have nominal stride equal to 32 and a good choice in FCN mode is to use - output_stride=16 in order to increase the density of the computed features at - small computational and memory overhead, cf. http://arxiv.org/abs/1606.00915. - - Args: - inputs: A tensor of size [batch, height_in, width_in, channels]. - blocks: A list of length equal to the number of ResNet blocks. Each element - is a resnet_utils.Block object describing the units in the block. - num_classes: Number of predicted classes for classification tasks. If None - we return the features before the logit layer. - is_training: whether batch_norm layers are in training mode. - global_pool: If True, we perform global average pooling before computing the - logits. Set to True for image classification, False for dense prediction. - output_stride: If None, then the output will be computed at the nominal - network stride. If output_stride is not None, it specifies the requested - ratio of input to output spatial resolution. - include_root_block: If True, include the initial convolution followed by - max-pooling, if False excludes it. If excluded, `inputs` should be the - results of an activation-less convolution. - reuse: whether or not the network and its variables should be reused. To be - able to reuse 'scope' must be given. - scope: Optional variable_scope. - - - Returns: - net: A rank-4 tensor of size [batch, height_out, width_out, channels_out]. - If global_pool is False, then height_out and width_out are reduced by a - factor of output_stride compared to the respective height_in and width_in, - else both height_out and width_out equal one. If num_classes is None, then - net is the output of the last ResNet block, potentially after global - average pooling. If num_classes is not None, net contains the pre-softmax - activations. - end_points: A dictionary from components of the network to the corresponding - activation. - - Raises: - ValueError: If the target output_stride is not valid. - """ - with variable_scope.variable_scope( - scope, 'resnet_v2', [inputs], reuse=reuse) as sc: - end_points_collection = sc.original_name_scope + '_end_points' - with arg_scope( - [layers_lib.conv2d, bottleneck, resnet_utils.stack_blocks_dense], - outputs_collections=end_points_collection): - with arg_scope([layers.batch_norm], is_training=is_training): - net = inputs - if include_root_block: - if output_stride is not None: - if output_stride % 4 != 0: - raise ValueError('The output_stride needs to be a multiple of 4.') - output_stride /= 4 - # We do not include batch normalization or activation functions in - # conv1 because the first ResNet unit will perform these. Cf. - # Appendix of [2]. - with arg_scope( - [layers_lib.conv2d], activation_fn=None, normalizer_fn=None): - net = resnet_utils.conv2d_same(net, 64, 7, stride=2, scope='conv1') - net = layers.max_pool2d(net, [3, 3], stride=2, scope='pool1') - net = resnet_utils.stack_blocks_dense(net, blocks, output_stride) - # This is needed because the pre-activation variant does not have batch - # normalization or activation functions in the residual unit output. See - # Appendix of [2]. - net = layers.batch_norm( - net, activation_fn=nn_ops.relu, scope='postnorm') - if global_pool: - # Global average pooling. - net = math_ops.reduce_mean(net, [1, 2], name='pool5', keepdims=True) - if num_classes is not None: - net = layers_lib.conv2d( - net, - num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - scope='logits') - # Convert end_points_collection into a dictionary of end_points. - end_points = utils.convert_collection_to_dict(end_points_collection) - if num_classes is not None: - end_points['predictions'] = layers.softmax(net, scope='predictions') - return net, end_points -resnet_v2.default_image_size = 224 - - -def resnet_v2_block(scope, base_depth, num_units, stride): - """Helper function for creating a resnet_v2 bottleneck block. - - Args: - scope: The scope of the block. - base_depth: The depth of the bottleneck layer for each unit. - num_units: The number of units in the block. - stride: The stride of the block, implemented as a stride in the last unit. - All other units have stride=1. - - Returns: - A resnet_v2 bottleneck block. - """ - return resnet_utils.Block(scope, bottleneck, [{ - 'depth': base_depth * 4, - 'depth_bottleneck': base_depth, - 'stride': 1 - }] * (num_units - 1) + [{ - 'depth': base_depth * 4, - 'depth_bottleneck': base_depth, - 'stride': stride - }]) - - -def resnet_v2_50(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v2_50'): - """ResNet-50 model of [1]. See resnet_v2() for arg and return description.""" - blocks = [ - resnet_v2_block('block1', base_depth=64, num_units=3, stride=2), - resnet_v2_block('block2', base_depth=128, num_units=4, stride=2), - resnet_v2_block('block3', base_depth=256, num_units=6, stride=2), - resnet_v2_block('block4', base_depth=512, num_units=3, stride=1), - ] - return resnet_v2( - inputs, - blocks, - num_classes, - is_training, - global_pool, - output_stride, - include_root_block=True, - reuse=reuse, - scope=scope) - - -def resnet_v2_101(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v2_101'): - """ResNet-101 model of [1]. See resnet_v2() for arg and return description.""" - blocks = [ - resnet_v2_block('block1', base_depth=64, num_units=3, stride=2), - resnet_v2_block('block2', base_depth=128, num_units=4, stride=2), - resnet_v2_block('block3', base_depth=256, num_units=23, stride=2), - resnet_v2_block('block4', base_depth=512, num_units=3, stride=1), - ] - return resnet_v2( - inputs, - blocks, - num_classes, - is_training, - global_pool, - output_stride, - include_root_block=True, - reuse=reuse, - scope=scope) - - -def resnet_v2_152(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v2_152'): - """ResNet-152 model of [1]. See resnet_v2() for arg and return description.""" - blocks = [ - resnet_v2_block('block1', base_depth=64, num_units=3, stride=2), - resnet_v2_block('block2', base_depth=128, num_units=8, stride=2), - resnet_v2_block('block3', base_depth=256, num_units=36, stride=2), - resnet_v2_block('block4', base_depth=512, num_units=3, stride=1), - ] - return resnet_v2( - inputs, - blocks, - num_classes, - is_training, - global_pool, - output_stride, - include_root_block=True, - reuse=reuse, - scope=scope) - - -def resnet_v2_200(inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - reuse=None, - scope='resnet_v2_200'): - """ResNet-200 model of [2]. See resnet_v2() for arg and return description.""" - blocks = [ - resnet_v2_block('block1', base_depth=64, num_units=3, stride=2), - resnet_v2_block('block2', base_depth=128, num_units=24, stride=2), - resnet_v2_block('block3', base_depth=256, num_units=36, stride=2), - resnet_v2_block('block4', base_depth=512, num_units=3, stride=1), - ] - return resnet_v2( - inputs, - blocks, - num_classes, - is_training, - global_pool, - output_stride, - include_root_block=True, - reuse=reuse, - scope=scope) diff --git a/tensorflow/contrib/slim/python/slim/nets/resnet_v2_test.py b/tensorflow/contrib/slim/python/slim/nets/resnet_v2_test.py deleted file mode 100644 index 31bdea9fbcd..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/resnet_v2_test.py +++ /dev/null @@ -1,453 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.nets.resnet_v2.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib import layers -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.layers.python.layers import utils -from tensorflow.contrib.slim.python.slim.nets import resnet_utils -from tensorflow.contrib.slim.python.slim.nets import resnet_v2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def create_test_input(batch_size, height, width, channels): - """Create test input tensor. - - Args: - batch_size: The number of images per batch or `None` if unknown. - height: The height of each image or `None` if unknown. - width: The width of each image or `None` if unknown. - channels: The number of channels per image or `None` if unknown. - - Returns: - Either a placeholder `Tensor` of dimension - [batch_size, height, width, channels] if any of the inputs are `None` or a - constant `Tensor` with the mesh grid values along the spatial dimensions. - """ - if None in [batch_size, height, width, channels]: - return array_ops.placeholder(dtypes.float32, - (batch_size, height, width, channels)) - else: - return math_ops.cast( - np.tile( - np.reshape( - np.reshape(np.arange(height), [height, 1]) + np.reshape( - np.arange(width), [1, width]), [1, height, width, 1]), - [batch_size, 1, 1, channels]), - dtypes.float32) - - -class ResnetUtilsTest(test.TestCase): - - def testSubsampleThreeByThree(self): - x = array_ops.reshape(math_ops.cast(math_ops.range(9), dtypes.float32), - [1, 3, 3, 1]) - x = resnet_utils.subsample(x, 2) - expected = array_ops.reshape( - constant_op.constant([0, 2, 6, 8]), [1, 2, 2, 1]) - with self.cached_session(): - self.assertAllClose(x.eval(), expected.eval()) - - def testSubsampleFourByFour(self): - x = array_ops.reshape(math_ops.cast(math_ops.range(16), dtypes.float32), - [1, 4, 4, 1]) - x = resnet_utils.subsample(x, 2) - expected = array_ops.reshape( - constant_op.constant([0, 2, 8, 10]), [1, 2, 2, 1]) - with self.cached_session(): - self.assertAllClose(x.eval(), expected.eval()) - - def testConv2DSameEven(self): - n, n2 = 4, 2 - - # Input image. - x = create_test_input(1, n, n, 1) - - # Convolution kernel. - w = create_test_input(1, 3, 3, 1) - w = array_ops.reshape(w, [3, 3, 1, 1]) - - variable_scope.get_variable('Conv/weights', initializer=w) - variable_scope.get_variable('Conv/biases', initializer=array_ops.zeros([1])) - variable_scope.get_variable_scope().reuse_variables() - - y1 = layers.conv2d(x, 1, [3, 3], stride=1, scope='Conv') - y1_expected = math_ops.cast([[14, 28, 43, 26], - [28, 48, 66, 37], - [43, 66, 84, 46], - [26, 37, 46, 22]], - dtypes.float32) - y1_expected = array_ops.reshape(y1_expected, [1, n, n, 1]) - - y2 = resnet_utils.subsample(y1, 2) - y2_expected = math_ops.cast([[14, 43], [43, 84]], dtypes.float32) - y2_expected = array_ops.reshape(y2_expected, [1, n2, n2, 1]) - - y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv') - y3_expected = y2_expected - - y4 = layers.conv2d(x, 1, [3, 3], stride=2, scope='Conv') - y4_expected = math_ops.cast([[48, 37], [37, 22]], dtypes.float32) - y4_expected = array_ops.reshape(y4_expected, [1, n2, n2, 1]) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertAllClose(y1.eval(), y1_expected.eval()) - self.assertAllClose(y2.eval(), y2_expected.eval()) - self.assertAllClose(y3.eval(), y3_expected.eval()) - self.assertAllClose(y4.eval(), y4_expected.eval()) - - def testConv2DSameOdd(self): - n, n2 = 5, 3 - - # Input image. - x = create_test_input(1, n, n, 1) - - # Convolution kernel. - w = create_test_input(1, 3, 3, 1) - w = array_ops.reshape(w, [3, 3, 1, 1]) - - variable_scope.get_variable('Conv/weights', initializer=w) - variable_scope.get_variable('Conv/biases', initializer=array_ops.zeros([1])) - variable_scope.get_variable_scope().reuse_variables() - - y1 = layers.conv2d(x, 1, [3, 3], stride=1, scope='Conv') - y1_expected = math_ops.cast([[14, 28, 43, 58, 34], - [28, 48, 66, 84, 46], - [43, 66, 84, 102, 55], - [58, 84, 102, 120, 64], - [34, 46, 55, 64, 30]], - dtypes.float32) - y1_expected = array_ops.reshape(y1_expected, [1, n, n, 1]) - - y2 = resnet_utils.subsample(y1, 2) - y2_expected = math_ops.cast([[14, 43, 34], - [43, 84, 55], - [34, 55, 30]], - dtypes.float32) - y2_expected = array_ops.reshape(y2_expected, [1, n2, n2, 1]) - - y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv') - y3_expected = y2_expected - - y4 = layers.conv2d(x, 1, [3, 3], stride=2, scope='Conv') - y4_expected = y2_expected - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertAllClose(y1.eval(), y1_expected.eval()) - self.assertAllClose(y2.eval(), y2_expected.eval()) - self.assertAllClose(y3.eval(), y3_expected.eval()) - self.assertAllClose(y4.eval(), y4_expected.eval()) - - def _resnet_plain(self, inputs, blocks, output_stride=None, scope=None): - """A plain ResNet without extra layers before or after the ResNet blocks.""" - with variable_scope.variable_scope(scope, values=[inputs]): - with arg_scope([layers.conv2d], outputs_collections='end_points'): - net = resnet_utils.stack_blocks_dense(inputs, blocks, output_stride) - end_points = utils.convert_collection_to_dict('end_points') - return net, end_points - - def testEndPointsV2(self): - """Test the end points of a tiny v2 bottleneck network.""" - blocks = [ - resnet_v2.resnet_v2_block( - 'block1', base_depth=1, num_units=2, stride=2), - resnet_v2.resnet_v2_block( - 'block2', base_depth=2, num_units=2, stride=1), - ] - inputs = create_test_input(2, 32, 16, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_plain(inputs, blocks, scope='tiny') - expected = [ - 'tiny/block1/unit_1/bottleneck_v2/shortcut', - 'tiny/block1/unit_1/bottleneck_v2/conv1', - 'tiny/block1/unit_1/bottleneck_v2/conv2', - 'tiny/block1/unit_1/bottleneck_v2/conv3', - 'tiny/block1/unit_2/bottleneck_v2/conv1', - 'tiny/block1/unit_2/bottleneck_v2/conv2', - 'tiny/block1/unit_2/bottleneck_v2/conv3', - 'tiny/block2/unit_1/bottleneck_v2/shortcut', - 'tiny/block2/unit_1/bottleneck_v2/conv1', - 'tiny/block2/unit_1/bottleneck_v2/conv2', - 'tiny/block2/unit_1/bottleneck_v2/conv3', - 'tiny/block2/unit_2/bottleneck_v2/conv1', - 'tiny/block2/unit_2/bottleneck_v2/conv2', - 'tiny/block2/unit_2/bottleneck_v2/conv3' - ] - self.assertItemsEqual(expected, end_points) - - def _stack_blocks_nondense(self, net, blocks): - """A simplified ResNet Block stacker without output stride control.""" - for block in blocks: - with variable_scope.variable_scope(block.scope, 'block', [net]): - for i, unit in enumerate(block.args): - with variable_scope.variable_scope('unit_%d' % (i + 1), values=[net]): - net = block.unit_fn(net, rate=1, **unit) - return net - - def testAtrousValuesBottleneck(self): - """Verify the values of dense feature extraction by atrous convolution. - - Make sure that dense feature extraction by stack_blocks_dense() followed by - subsampling gives identical results to feature extraction at the nominal - network output stride using the simple self._stack_blocks_nondense() above. - """ - block = resnet_v2.resnet_v2_block - blocks = [ - block('block1', base_depth=1, num_units=2, stride=2), - block('block2', base_depth=2, num_units=2, stride=2), - block('block3', base_depth=4, num_units=2, stride=2), - block('block4', base_depth=8, num_units=2, stride=1), - ] - nominal_stride = 8 - - # Test both odd and even input dimensions. - height = 30 - width = 31 - with arg_scope(resnet_utils.resnet_arg_scope()): - with arg_scope([layers.batch_norm], is_training=False): - for output_stride in [1, 2, 4, 8, None]: - with ops.Graph().as_default(): - with self.cached_session() as sess: - random_seed.set_random_seed(0) - inputs = create_test_input(1, height, width, 3) - # Dense feature extraction followed by subsampling. - output = resnet_utils.stack_blocks_dense(inputs, blocks, - output_stride) - if output_stride is None: - factor = 1 - else: - factor = nominal_stride // output_stride - - output = resnet_utils.subsample(output, factor) - # Make the two networks use the same weights. - variable_scope.get_variable_scope().reuse_variables() - # Feature extraction at the nominal network rate. - expected = self._stack_blocks_nondense(inputs, blocks) - sess.run(variables.global_variables_initializer()) - output, expected = sess.run([output, expected]) - self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4) - - -class ResnetCompleteNetworkTest(test.TestCase): - """Tests with complete small ResNet v2 networks.""" - - def _resnet_small(self, - inputs, - num_classes=None, - is_training=True, - global_pool=True, - output_stride=None, - include_root_block=True, - reuse=None, - scope='resnet_v2_small'): - """A shallow and thin ResNet v2 for faster tests.""" - block = resnet_v2.resnet_v2_block - blocks = [ - block('block1', base_depth=1, num_units=3, stride=2), - block('block2', base_depth=2, num_units=3, stride=2), - block('block3', base_depth=4, num_units=3, stride=2), - block('block4', base_depth=8, num_units=2, stride=1), - ] - return resnet_v2.resnet_v2(inputs, blocks, num_classes, is_training, - global_pool, output_stride, include_root_block, - reuse, scope) - - def testClassificationEndPoints(self): - global_pool = True - num_classes = 10 - inputs = create_test_input(2, 224, 224, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - logits, end_points = self._resnet_small( - inputs, num_classes, global_pool=global_pool, scope='resnet') - self.assertTrue(logits.op.name.startswith('resnet/logits')) - self.assertListEqual(logits.get_shape().as_list(), [2, 1, 1, num_classes]) - self.assertTrue('predictions' in end_points) - self.assertListEqual(end_points['predictions'].get_shape().as_list(), - [2, 1, 1, num_classes]) - - def testClassificationShapes(self): - global_pool = True - num_classes = 10 - inputs = create_test_input(2, 224, 224, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small( - inputs, num_classes, global_pool=global_pool, scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 28, 28, 4], - 'resnet/block2': [2, 14, 14, 8], - 'resnet/block3': [2, 7, 7, 16], - 'resnet/block4': [2, 7, 7, 32] - } - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testFullyConvolutionalEndpointShapes(self): - global_pool = False - num_classes = 10 - inputs = create_test_input(2, 321, 321, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small( - inputs, num_classes, global_pool=global_pool, scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 41, 41, 4], - 'resnet/block2': [2, 21, 21, 8], - 'resnet/block3': [2, 11, 11, 16], - 'resnet/block4': [2, 11, 11, 32] - } - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testRootlessFullyConvolutionalEndpointShapes(self): - global_pool = False - num_classes = 10 - inputs = create_test_input(2, 128, 128, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small( - inputs, - num_classes, - global_pool=global_pool, - include_root_block=False, - scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 64, 64, 4], - 'resnet/block2': [2, 32, 32, 8], - 'resnet/block3': [2, 16, 16, 16], - 'resnet/block4': [2, 16, 16, 32] - } - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testAtrousFullyConvolutionalEndpointShapes(self): - global_pool = False - num_classes = 10 - output_stride = 8 - inputs = create_test_input(2, 321, 321, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - _, end_points = self._resnet_small( - inputs, - num_classes, - global_pool=global_pool, - output_stride=output_stride, - scope='resnet') - endpoint_to_shape = { - 'resnet/block1': [2, 41, 41, 4], - 'resnet/block2': [2, 41, 41, 8], - 'resnet/block3': [2, 41, 41, 16], - 'resnet/block4': [2, 41, 41, 32] - } - for endpoint in endpoint_to_shape: - shape = endpoint_to_shape[endpoint] - self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape) - - def testAtrousFullyConvolutionalValues(self): - """Verify dense feature extraction with atrous convolution.""" - nominal_stride = 32 - for output_stride in [4, 8, 16, 32, None]: - with arg_scope(resnet_utils.resnet_arg_scope()): - with ops.Graph().as_default(): - with self.cached_session() as sess: - random_seed.set_random_seed(0) - inputs = create_test_input(2, 81, 81, 3) - # Dense feature extraction followed by subsampling. - output, _ = self._resnet_small( - inputs, - None, - is_training=False, - global_pool=False, - output_stride=output_stride) - if output_stride is None: - factor = 1 - else: - factor = nominal_stride // output_stride - output = resnet_utils.subsample(output, factor) - # Make the two networks use the same weights. - variable_scope.get_variable_scope().reuse_variables() - # Feature extraction at the nominal network rate. - expected, _ = self._resnet_small( - inputs, None, is_training=False, global_pool=False) - sess.run(variables.global_variables_initializer()) - self.assertAllClose( - output.eval(), expected.eval(), atol=1e-4, rtol=1e-4) - - def testUnknownBatchSize(self): - batch = 2 - height, width = 65, 65 - global_pool = True - num_classes = 10 - inputs = create_test_input(None, height, width, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - logits, _ = self._resnet_small( - inputs, num_classes, global_pool=global_pool, scope='resnet') - self.assertTrue(logits.op.name.startswith('resnet/logits')) - self.assertListEqual(logits.get_shape().as_list(), - [None, 1, 1, num_classes]) - images = create_test_input(batch, height, width, 3) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(logits, {inputs: images.eval()}) - self.assertEqual(output.shape, (batch, 1, 1, num_classes)) - - def testFullyConvolutionalUnknownHeightWidth(self): - batch = 2 - height, width = 65, 65 - global_pool = False - inputs = create_test_input(batch, None, None, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - output, _ = self._resnet_small(inputs, None, global_pool=global_pool) - self.assertListEqual(output.get_shape().as_list(), [batch, None, None, 32]) - images = create_test_input(batch, height, width, 3) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(output, {inputs: images.eval()}) - self.assertEqual(output.shape, (batch, 3, 3, 32)) - - def testAtrousFullyConvolutionalUnknownHeightWidth(self): - batch = 2 - height, width = 65, 65 - global_pool = False - output_stride = 8 - inputs = create_test_input(batch, None, None, 3) - with arg_scope(resnet_utils.resnet_arg_scope()): - output, _ = self._resnet_small( - inputs, None, global_pool=global_pool, output_stride=output_stride) - self.assertListEqual(output.get_shape().as_list(), [batch, None, None, 32]) - images = create_test_input(batch, height, width, 3) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(output, {inputs: images.eval()}) - self.assertEqual(output.shape, (batch, 9, 9, 32)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/nets/vgg.py b/tensorflow/contrib/slim/python/slim/nets/vgg.py deleted file mode 100644 index d4eb43cbb2e..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/vgg.py +++ /dev/null @@ -1,270 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains model definitions for versions of the Oxford VGG network. - -These model definitions were introduced in the following technical report: - - Very Deep Convolutional Networks For Large-Scale Image Recognition - Karen Simonyan and Andrew Zisserman - arXiv technical report, 2015 - PDF: http://arxiv.org/pdf/1409.1556.pdf - ILSVRC 2014 Slides: http://www.robots.ox.ac.uk/~karen/pdf/ILSVRC_2014.pdf - CC-BY-4.0 - -More information can be obtained from the VGG website: -www.robots.ox.ac.uk/~vgg/research/very_deep/ - -Usage: - with slim.arg_scope(vgg.vgg_arg_scope()): - outputs, end_points = vgg.vgg_a(inputs) - - with slim.arg_scope(vgg.vgg_arg_scope()): - outputs, end_points = vgg.vgg_16(inputs) - -@@vgg_a -@@vgg_16 -@@vgg_19 -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.framework.python.ops import arg_scope -from tensorflow.contrib.layers.python.layers import layers as layers_lib -from tensorflow.contrib.layers.python.layers import regularizers -from tensorflow.contrib.layers.python.layers import utils -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope - - -def vgg_arg_scope(weight_decay=0.0005): - """Defines the VGG arg scope. - - Args: - weight_decay: The l2 regularization coefficient. - - Returns: - An arg_scope. - """ - with arg_scope( - [layers.conv2d, layers_lib.fully_connected], - activation_fn=nn_ops.relu, - weights_regularizer=regularizers.l2_regularizer(weight_decay), - biases_initializer=init_ops.zeros_initializer()): - with arg_scope([layers.conv2d], padding='SAME') as arg_sc: - return arg_sc - - -def vgg_a(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.5, - spatial_squeeze=True, - scope='vgg_a'): - """Oxford Net VGG 11-Layers version A Example. - - Note: All the fully_connected layers have been transformed to conv2d layers. - To use in classification mode, resize input to 224x224. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether or not the model is being trained. - dropout_keep_prob: the probability that activations are kept in the dropout - layers during training. - spatial_squeeze: whether or not should squeeze the spatial dimensions of the - outputs. Useful to remove unnecessary dimensions for classification. - scope: Optional scope for the variables. - - Returns: - the last op containing the log predictions and end_points dict. - """ - with variable_scope.variable_scope(scope, 'vgg_a', [inputs]) as sc: - end_points_collection = sc.original_name_scope + '_end_points' - # Collect outputs for conv2d, fully_connected and max_pool2d. - with arg_scope( - [layers.conv2d, layers_lib.max_pool2d], - outputs_collections=end_points_collection): - net = layers_lib.repeat( - inputs, 1, layers.conv2d, 64, [3, 3], scope='conv1') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool1') - net = layers_lib.repeat(net, 1, layers.conv2d, 128, [3, 3], scope='conv2') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool2') - net = layers_lib.repeat(net, 2, layers.conv2d, 256, [3, 3], scope='conv3') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool3') - net = layers_lib.repeat(net, 2, layers.conv2d, 512, [3, 3], scope='conv4') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool4') - net = layers_lib.repeat(net, 2, layers.conv2d, 512, [3, 3], scope='conv5') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool5') - # Use conv2d instead of fully_connected layers. - net = layers.conv2d(net, 4096, [7, 7], padding='VALID', scope='fc6') - net = layers_lib.dropout( - net, dropout_keep_prob, is_training=is_training, scope='dropout6') - net = layers.conv2d(net, 4096, [1, 1], scope='fc7') - net = layers_lib.dropout( - net, dropout_keep_prob, is_training=is_training, scope='dropout7') - net = layers.conv2d( - net, - num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - scope='fc8') - # Convert end_points_collection into a end_point dict. - end_points = utils.convert_collection_to_dict(end_points_collection) - if spatial_squeeze: - net = array_ops.squeeze(net, [1, 2], name='fc8/squeezed') - end_points[sc.name + '/fc8'] = net - return net, end_points - - -vgg_a.default_image_size = 224 - - -def vgg_16(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.5, - spatial_squeeze=True, - scope='vgg_16'): - """Oxford Net VGG 16-Layers version D Example. - - Note: All the fully_connected layers have been transformed to conv2d layers. - To use in classification mode, resize input to 224x224. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether or not the model is being trained. - dropout_keep_prob: the probability that activations are kept in the dropout - layers during training. - spatial_squeeze: whether or not should squeeze the spatial dimensions of the - outputs. Useful to remove unnecessary dimensions for classification. - scope: Optional scope for the variables. - - Returns: - the last op containing the log predictions and end_points dict. - """ - with variable_scope.variable_scope(scope, 'vgg_16', [inputs]) as sc: - end_points_collection = sc.original_name_scope + '_end_points' - # Collect outputs for conv2d, fully_connected and max_pool2d. - with arg_scope( - [layers.conv2d, layers_lib.fully_connected, layers_lib.max_pool2d], - outputs_collections=end_points_collection): - net = layers_lib.repeat( - inputs, 2, layers.conv2d, 64, [3, 3], scope='conv1') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool1') - net = layers_lib.repeat(net, 2, layers.conv2d, 128, [3, 3], scope='conv2') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool2') - net = layers_lib.repeat(net, 3, layers.conv2d, 256, [3, 3], scope='conv3') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool3') - net = layers_lib.repeat(net, 3, layers.conv2d, 512, [3, 3], scope='conv4') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool4') - net = layers_lib.repeat(net, 3, layers.conv2d, 512, [3, 3], scope='conv5') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool5') - # Use conv2d instead of fully_connected layers. - net = layers.conv2d(net, 4096, [7, 7], padding='VALID', scope='fc6') - net = layers_lib.dropout( - net, dropout_keep_prob, is_training=is_training, scope='dropout6') - net = layers.conv2d(net, 4096, [1, 1], scope='fc7') - net = layers_lib.dropout( - net, dropout_keep_prob, is_training=is_training, scope='dropout7') - net = layers.conv2d( - net, - num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - scope='fc8') - # Convert end_points_collection into a end_point dict. - end_points = utils.convert_collection_to_dict(end_points_collection) - if spatial_squeeze: - net = array_ops.squeeze(net, [1, 2], name='fc8/squeezed') - end_points[sc.name + '/fc8'] = net - return net, end_points - - -vgg_16.default_image_size = 224 - - -def vgg_19(inputs, - num_classes=1000, - is_training=True, - dropout_keep_prob=0.5, - spatial_squeeze=True, - scope='vgg_19'): - """Oxford Net VGG 19-Layers version E Example. - - Note: All the fully_connected layers have been transformed to conv2d layers. - To use in classification mode, resize input to 224x224. - - Args: - inputs: a tensor of size [batch_size, height, width, channels]. - num_classes: number of predicted classes. - is_training: whether or not the model is being trained. - dropout_keep_prob: the probability that activations are kept in the dropout - layers during training. - spatial_squeeze: whether or not should squeeze the spatial dimensions of the - outputs. Useful to remove unnecessary dimensions for classification. - scope: Optional scope for the variables. - - Returns: - the last op containing the log predictions and end_points dict. - """ - with variable_scope.variable_scope(scope, 'vgg_19', [inputs]) as sc: - end_points_collection = sc.name + '_end_points' - # Collect outputs for conv2d, fully_connected and max_pool2d. - with arg_scope( - [layers.conv2d, layers_lib.fully_connected, layers_lib.max_pool2d], - outputs_collections=end_points_collection): - net = layers_lib.repeat( - inputs, 2, layers.conv2d, 64, [3, 3], scope='conv1') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool1') - net = layers_lib.repeat(net, 2, layers.conv2d, 128, [3, 3], scope='conv2') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool2') - net = layers_lib.repeat(net, 4, layers.conv2d, 256, [3, 3], scope='conv3') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool3') - net = layers_lib.repeat(net, 4, layers.conv2d, 512, [3, 3], scope='conv4') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool4') - net = layers_lib.repeat(net, 4, layers.conv2d, 512, [3, 3], scope='conv5') - net = layers_lib.max_pool2d(net, [2, 2], scope='pool5') - # Use conv2d instead of fully_connected layers. - net = layers.conv2d(net, 4096, [7, 7], padding='VALID', scope='fc6') - net = layers_lib.dropout( - net, dropout_keep_prob, is_training=is_training, scope='dropout6') - net = layers.conv2d(net, 4096, [1, 1], scope='fc7') - net = layers_lib.dropout( - net, dropout_keep_prob, is_training=is_training, scope='dropout7') - net = layers.conv2d( - net, - num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - scope='fc8') - # Convert end_points_collection into a end_point dict. - end_points = utils.convert_collection_to_dict(end_points_collection) - if spatial_squeeze: - net = array_ops.squeeze(net, [1, 2], name='fc8/squeezed') - end_points[sc.name + '/fc8'] = net - return net, end_points - - -vgg_19.default_image_size = 224 - -# Alias -vgg_d = vgg_16 -vgg_e = vgg_19 diff --git a/tensorflow/contrib/slim/python/slim/nets/vgg_test.py b/tensorflow/contrib/slim/python/slim/nets/vgg_test.py deleted file mode 100644 index 71ce4b89cd5..00000000000 --- a/tensorflow/contrib/slim/python/slim/nets/vgg_test.py +++ /dev/null @@ -1,428 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for slim.nets.vgg.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework.python.ops import variables as variables_lib -from tensorflow.contrib.slim.python.slim.nets import vgg -from tensorflow.python.framework import ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class VGGATest(test.TestCase): - - def testBuild(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_a(inputs, num_classes) - self.assertEquals(logits.op.name, 'vgg_a/fc8/squeezed') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testFullyConvolutional(self): - batch_size = 1 - height, width = 256, 256 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_a(inputs, num_classes, spatial_squeeze=False) - self.assertEquals(logits.op.name, 'vgg_a/fc8/BiasAdd') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, 2, 2, num_classes]) - - def testEndPoints(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - for is_training in [True, False]: - with ops.Graph().as_default(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - _, end_points = vgg.vgg_a(inputs, num_classes, is_training=is_training) - expected_names = [ - 'vgg_a/conv1/conv1_1', 'vgg_a/pool1', 'vgg_a/conv2/conv2_1', - 'vgg_a/pool2', 'vgg_a/conv3/conv3_1', 'vgg_a/conv3/conv3_2', - 'vgg_a/pool3', 'vgg_a/conv4/conv4_1', 'vgg_a/conv4/conv4_2', - 'vgg_a/pool4', 'vgg_a/conv5/conv5_1', 'vgg_a/conv5/conv5_2', - 'vgg_a/pool5', 'vgg_a/fc6', 'vgg_a/fc7', 'vgg_a/fc8' - ] - self.assertSetEqual(set(end_points.keys()), set(expected_names)) - - def testModelVariables(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - vgg.vgg_a(inputs, num_classes) - expected_names = [ - 'vgg_a/conv1/conv1_1/weights', - 'vgg_a/conv1/conv1_1/biases', - 'vgg_a/conv2/conv2_1/weights', - 'vgg_a/conv2/conv2_1/biases', - 'vgg_a/conv3/conv3_1/weights', - 'vgg_a/conv3/conv3_1/biases', - 'vgg_a/conv3/conv3_2/weights', - 'vgg_a/conv3/conv3_2/biases', - 'vgg_a/conv4/conv4_1/weights', - 'vgg_a/conv4/conv4_1/biases', - 'vgg_a/conv4/conv4_2/weights', - 'vgg_a/conv4/conv4_2/biases', - 'vgg_a/conv5/conv5_1/weights', - 'vgg_a/conv5/conv5_1/biases', - 'vgg_a/conv5/conv5_2/weights', - 'vgg_a/conv5/conv5_2/biases', - 'vgg_a/fc6/weights', - 'vgg_a/fc6/biases', - 'vgg_a/fc7/weights', - 'vgg_a/fc7/biases', - 'vgg_a/fc8/weights', - 'vgg_a/fc8/biases', - ] - model_variables = [v.op.name for v in variables_lib.get_model_variables()] - self.assertSetEqual(set(model_variables), set(expected_names)) - - def testEvaluation(self): - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - with self.cached_session(): - eval_inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_a(eval_inputs, is_training=False) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - predictions = math_ops.argmax(logits, 1) - self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) - - def testTrainEvalWithReuse(self): - train_batch_size = 2 - eval_batch_size = 1 - train_height, train_width = 224, 224 - eval_height, eval_width = 256, 256 - num_classes = 1000 - with self.cached_session(): - train_inputs = random_ops.random_uniform( - (train_batch_size, train_height, train_width, 3)) - logits, _ = vgg.vgg_a(train_inputs) - self.assertListEqual(logits.get_shape().as_list(), - [train_batch_size, num_classes]) - variable_scope.get_variable_scope().reuse_variables() - eval_inputs = random_ops.random_uniform( - (eval_batch_size, eval_height, eval_width, 3)) - logits, _ = vgg.vgg_a( - eval_inputs, is_training=False, spatial_squeeze=False) - self.assertListEqual(logits.get_shape().as_list(), - [eval_batch_size, 2, 2, num_classes]) - logits = math_ops.reduce_mean(logits, [1, 2]) - predictions = math_ops.argmax(logits, 1) - self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) - - def testForward(self): - batch_size = 1 - height, width = 224, 224 - with self.cached_session() as sess: - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_a(inputs) - sess.run(variables.global_variables_initializer()) - output = sess.run(logits) - self.assertTrue(output.any()) - - -class VGG16Test(test.TestCase): - - def testBuild(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_16(inputs, num_classes) - self.assertEquals(logits.op.name, 'vgg_16/fc8/squeezed') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testFullyConvolutional(self): - batch_size = 1 - height, width = 256, 256 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_16(inputs, num_classes, spatial_squeeze=False) - self.assertEquals(logits.op.name, 'vgg_16/fc8/BiasAdd') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, 2, 2, num_classes]) - - def testEndPoints(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - for is_training in [True, False]: - with ops.Graph().as_default(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - _, end_points = vgg.vgg_16(inputs, num_classes, is_training=is_training) - expected_names = [ - 'vgg_16/conv1/conv1_1', 'vgg_16/conv1/conv1_2', 'vgg_16/pool1', - 'vgg_16/conv2/conv2_1', 'vgg_16/conv2/conv2_2', 'vgg_16/pool2', - 'vgg_16/conv3/conv3_1', 'vgg_16/conv3/conv3_2', - 'vgg_16/conv3/conv3_3', 'vgg_16/pool3', 'vgg_16/conv4/conv4_1', - 'vgg_16/conv4/conv4_2', 'vgg_16/conv4/conv4_3', 'vgg_16/pool4', - 'vgg_16/conv5/conv5_1', 'vgg_16/conv5/conv5_2', - 'vgg_16/conv5/conv5_3', 'vgg_16/pool5', 'vgg_16/fc6', 'vgg_16/fc7', - 'vgg_16/fc8' - ] - self.assertSetEqual(set(end_points.keys()), set(expected_names)) - - def testModelVariables(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - vgg.vgg_16(inputs, num_classes) - expected_names = [ - 'vgg_16/conv1/conv1_1/weights', - 'vgg_16/conv1/conv1_1/biases', - 'vgg_16/conv1/conv1_2/weights', - 'vgg_16/conv1/conv1_2/biases', - 'vgg_16/conv2/conv2_1/weights', - 'vgg_16/conv2/conv2_1/biases', - 'vgg_16/conv2/conv2_2/weights', - 'vgg_16/conv2/conv2_2/biases', - 'vgg_16/conv3/conv3_1/weights', - 'vgg_16/conv3/conv3_1/biases', - 'vgg_16/conv3/conv3_2/weights', - 'vgg_16/conv3/conv3_2/biases', - 'vgg_16/conv3/conv3_3/weights', - 'vgg_16/conv3/conv3_3/biases', - 'vgg_16/conv4/conv4_1/weights', - 'vgg_16/conv4/conv4_1/biases', - 'vgg_16/conv4/conv4_2/weights', - 'vgg_16/conv4/conv4_2/biases', - 'vgg_16/conv4/conv4_3/weights', - 'vgg_16/conv4/conv4_3/biases', - 'vgg_16/conv5/conv5_1/weights', - 'vgg_16/conv5/conv5_1/biases', - 'vgg_16/conv5/conv5_2/weights', - 'vgg_16/conv5/conv5_2/biases', - 'vgg_16/conv5/conv5_3/weights', - 'vgg_16/conv5/conv5_3/biases', - 'vgg_16/fc6/weights', - 'vgg_16/fc6/biases', - 'vgg_16/fc7/weights', - 'vgg_16/fc7/biases', - 'vgg_16/fc8/weights', - 'vgg_16/fc8/biases', - ] - model_variables = [v.op.name for v in variables_lib.get_model_variables()] - self.assertSetEqual(set(model_variables), set(expected_names)) - - def testEvaluation(self): - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - with self.cached_session(): - eval_inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_16(eval_inputs, is_training=False) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - predictions = math_ops.argmax(logits, 1) - self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) - - def testTrainEvalWithReuse(self): - train_batch_size = 2 - eval_batch_size = 1 - train_height, train_width = 224, 224 - eval_height, eval_width = 256, 256 - num_classes = 1000 - with self.cached_session(): - train_inputs = random_ops.random_uniform( - (train_batch_size, train_height, train_width, 3)) - logits, _ = vgg.vgg_16(train_inputs) - self.assertListEqual(logits.get_shape().as_list(), - [train_batch_size, num_classes]) - variable_scope.get_variable_scope().reuse_variables() - eval_inputs = random_ops.random_uniform( - (eval_batch_size, eval_height, eval_width, 3)) - logits, _ = vgg.vgg_16( - eval_inputs, is_training=False, spatial_squeeze=False) - self.assertListEqual(logits.get_shape().as_list(), - [eval_batch_size, 2, 2, num_classes]) - logits = math_ops.reduce_mean(logits, [1, 2]) - predictions = math_ops.argmax(logits, 1) - self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) - - def testForward(self): - batch_size = 1 - height, width = 224, 224 - with self.cached_session() as sess: - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_16(inputs) - sess.run(variables.global_variables_initializer()) - output = sess.run(logits) - self.assertTrue(output.any()) - - -class VGG19Test(test.TestCase): - - def testBuild(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_19(inputs, num_classes) - self.assertEquals(logits.op.name, 'vgg_19/fc8/squeezed') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testFullyConvolutional(self): - batch_size = 1 - height, width = 256, 256 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_19(inputs, num_classes, spatial_squeeze=False) - self.assertEquals(logits.op.name, 'vgg_19/fc8/BiasAdd') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, 2, 2, num_classes]) - - def testEndPoints(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - for is_training in [True, False]: - with ops.Graph().as_default(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - _, end_points = vgg.vgg_19(inputs, num_classes, is_training=is_training) - expected_names = [ - 'vgg_19/conv1/conv1_1', 'vgg_19/conv1/conv1_2', 'vgg_19/pool1', - 'vgg_19/conv2/conv2_1', 'vgg_19/conv2/conv2_2', 'vgg_19/pool2', - 'vgg_19/conv3/conv3_1', 'vgg_19/conv3/conv3_2', - 'vgg_19/conv3/conv3_3', 'vgg_19/conv3/conv3_4', 'vgg_19/pool3', - 'vgg_19/conv4/conv4_1', 'vgg_19/conv4/conv4_2', - 'vgg_19/conv4/conv4_3', 'vgg_19/conv4/conv4_4', 'vgg_19/pool4', - 'vgg_19/conv5/conv5_1', 'vgg_19/conv5/conv5_2', - 'vgg_19/conv5/conv5_3', 'vgg_19/conv5/conv5_4', 'vgg_19/pool5', - 'vgg_19/fc6', 'vgg_19/fc7', 'vgg_19/fc8' - ] - self.assertSetEqual(set(end_points.keys()), set(expected_names)) - - def testModelVariables(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - with self.cached_session(): - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - vgg.vgg_19(inputs, num_classes) - expected_names = [ - 'vgg_19/conv1/conv1_1/weights', - 'vgg_19/conv1/conv1_1/biases', - 'vgg_19/conv1/conv1_2/weights', - 'vgg_19/conv1/conv1_2/biases', - 'vgg_19/conv2/conv2_1/weights', - 'vgg_19/conv2/conv2_1/biases', - 'vgg_19/conv2/conv2_2/weights', - 'vgg_19/conv2/conv2_2/biases', - 'vgg_19/conv3/conv3_1/weights', - 'vgg_19/conv3/conv3_1/biases', - 'vgg_19/conv3/conv3_2/weights', - 'vgg_19/conv3/conv3_2/biases', - 'vgg_19/conv3/conv3_3/weights', - 'vgg_19/conv3/conv3_3/biases', - 'vgg_19/conv3/conv3_4/weights', - 'vgg_19/conv3/conv3_4/biases', - 'vgg_19/conv4/conv4_1/weights', - 'vgg_19/conv4/conv4_1/biases', - 'vgg_19/conv4/conv4_2/weights', - 'vgg_19/conv4/conv4_2/biases', - 'vgg_19/conv4/conv4_3/weights', - 'vgg_19/conv4/conv4_3/biases', - 'vgg_19/conv4/conv4_4/weights', - 'vgg_19/conv4/conv4_4/biases', - 'vgg_19/conv5/conv5_1/weights', - 'vgg_19/conv5/conv5_1/biases', - 'vgg_19/conv5/conv5_2/weights', - 'vgg_19/conv5/conv5_2/biases', - 'vgg_19/conv5/conv5_3/weights', - 'vgg_19/conv5/conv5_3/biases', - 'vgg_19/conv5/conv5_4/weights', - 'vgg_19/conv5/conv5_4/biases', - 'vgg_19/fc6/weights', - 'vgg_19/fc6/biases', - 'vgg_19/fc7/weights', - 'vgg_19/fc7/biases', - 'vgg_19/fc8/weights', - 'vgg_19/fc8/biases', - ] - model_variables = [v.op.name for v in variables_lib.get_model_variables()] - self.assertSetEqual(set(model_variables), set(expected_names)) - - def testEvaluation(self): - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - with self.cached_session(): - eval_inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_19(eval_inputs, is_training=False) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - predictions = math_ops.argmax(logits, 1) - self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) - - def testTrainEvalWithReuse(self): - train_batch_size = 2 - eval_batch_size = 1 - train_height, train_width = 224, 224 - eval_height, eval_width = 256, 256 - num_classes = 1000 - with self.cached_session(): - train_inputs = random_ops.random_uniform( - (train_batch_size, train_height, train_width, 3)) - logits, _ = vgg.vgg_19(train_inputs) - self.assertListEqual(logits.get_shape().as_list(), - [train_batch_size, num_classes]) - variable_scope.get_variable_scope().reuse_variables() - eval_inputs = random_ops.random_uniform( - (eval_batch_size, eval_height, eval_width, 3)) - logits, _ = vgg.vgg_19( - eval_inputs, is_training=False, spatial_squeeze=False) - self.assertListEqual(logits.get_shape().as_list(), - [eval_batch_size, 2, 2, num_classes]) - logits = math_ops.reduce_mean(logits, [1, 2]) - predictions = math_ops.argmax(logits, 1) - self.assertEquals(predictions.get_shape().as_list(), [eval_batch_size]) - - def testForward(self): - batch_size = 1 - height, width = 224, 224 - with self.cached_session() as sess: - inputs = random_ops.random_uniform((batch_size, height, width, 3)) - logits, _ = vgg.vgg_19(inputs) - sess.run(variables.global_variables_initializer()) - output = sess.run(logits) - self.assertTrue(output.any()) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/slim/python/slim/queues.py b/tensorflow/contrib/slim/python/slim/queues.py deleted file mode 100644 index 1bc8d2c89b5..00000000000 --- a/tensorflow/contrib/slim/python/slim/queues.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains a helper context for running queue runners. - -@@NestedQueueRunnerError -@@QueueRunners -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from contextlib import contextmanager -import threading - -from tensorflow.python.framework import ops -from tensorflow.python.training import coordinator - -__all__ = [ - 'NestedQueueRunnerError', - 'QueueRunners', -] - -_queue_runner_lock = threading.Lock() - - -class NestedQueueRunnerError(Exception): - pass - - -@contextmanager -def QueueRunners(session): - """Creates a context manager that handles starting and stopping queue runners. - - Args: - session: the currently running session. - - Yields: - a context in which queues are run. - - Raises: - NestedQueueRunnerError: if a QueueRunners context is nested within another. - """ - if not _queue_runner_lock.acquire(False): - raise NestedQueueRunnerError('QueueRunners cannot be nested') - - coord = coordinator.Coordinator() - threads = [] - for qr in ops.get_collection(ops.GraphKeys.QUEUE_RUNNERS): - threads.extend( - qr.create_threads( - session, coord=coord, daemon=True, start=True)) - try: - yield - finally: - coord.request_stop() - try: - coord.join(threads, stop_grace_period_secs=120) - except RuntimeError: - session.close() - - _queue_runner_lock.release() diff --git a/tensorflow/contrib/slim/python/slim/summaries.py b/tensorflow/contrib/slim/python/slim/summaries.py deleted file mode 100644 index a7dc3f6723a..00000000000 --- a/tensorflow/contrib/slim/python/slim/summaries.py +++ /dev/null @@ -1,220 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains helper functions for creating summaries. - -This module contains various helper functions for quickly and easily adding -tensorflow summaries. These allow users to print summary values -automatically as they are computed and add prefixes to collections of summaries. - -Example usage: - - import tensorflow as tf - slim = tf.contrib.slim - - slim.summaries.add_histogram_summaries(slim.variables.get_model_variables()) - slim.summaries.add_scalar_summary(total_loss, 'Total Loss') - slim.summaries.add_scalar_summary(learning_rate, 'Learning Rate') - slim.summaries.add_histogram_summaries(my_tensors) - slim.summaries.add_zero_fraction_summaries(my_tensors) -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import nn_impl as nn -from tensorflow.python.summary import summary - - -def _get_summary_name(tensor, name=None, prefix=None, postfix=None): - """Produces the summary name given. - - Args: - tensor: A variable or op `Tensor`. - name: The optional name for the summary. - prefix: An optional prefix for the summary name. - postfix: An optional postfix for the summary name. - - Returns: - a summary name. - """ - if not name: - name = tensor.op.name - if prefix: - name = prefix + '/' + name - if postfix: - name = name + '/' + postfix - return name - - -def add_histogram_summary(tensor, name=None, prefix=None): - """Adds a histogram summary for the given tensor. - - Args: - tensor: A variable or op tensor. - name: The optional name for the summary. - prefix: An optional prefix for the summary names. - - Returns: - A scalar `Tensor` of type `string` whose contents are the serialized - `Summary` protocol buffer. - """ - return summary.histogram( - _get_summary_name(tensor, name, prefix), tensor) - - -def add_image_summary(tensor, name=None, prefix=None, print_summary=False): - """Adds an image summary for the given tensor. - - Args: - tensor: a variable or op tensor with shape [batch,height,width,channels] - name: the optional name for the summary. - prefix: An optional prefix for the summary names. - print_summary: If `True`, the summary is printed to stdout when the summary - is computed. - - Returns: - An image `Tensor` of type `string` whose contents are the serialized - `Summary` protocol buffer. - """ - summary_name = _get_summary_name(tensor, name, prefix) - # If print_summary, then we need to make sure that this call doesn't add the - # non-printing op to the collection. We'll add it to the collection later. - collections = [] if print_summary else None - op = summary.image( - name=summary_name, tensor=tensor, collections=collections) - if print_summary: - op = logging_ops.Print(op, [tensor], summary_name) - ops.add_to_collection(ops.GraphKeys.SUMMARIES, op) - return op - - -def add_scalar_summary(tensor, name=None, prefix=None, print_summary=False): - """Adds a scalar summary for the given tensor. - - Args: - tensor: a variable or op tensor. - name: the optional name for the summary. - prefix: An optional prefix for the summary names. - print_summary: If `True`, the summary is printed to stdout when the summary - is computed. - - Returns: - A scalar `Tensor` of type `string` whose contents are the serialized - `Summary` protocol buffer. - """ - collections = [] if print_summary else None - summary_name = _get_summary_name(tensor, name, prefix) - - # If print_summary, then we need to make sure that this call doesn't add the - # non-printing op to the collection. We'll add it to the collection later. - op = summary.scalar( - name=summary_name, tensor=tensor, collections=collections) - if print_summary: - op = logging_ops.Print(op, [tensor], summary_name) - ops.add_to_collection(ops.GraphKeys.SUMMARIES, op) - return op - - -def add_zero_fraction_summary(tensor, name=None, prefix=None, - print_summary=False): - """Adds a summary for the percentage of zero values in the given tensor. - - Args: - tensor: a variable or op tensor. - name: the optional name for the summary. - prefix: An optional prefix for the summary names. - print_summary: If `True`, the summary is printed to stdout when the summary - is computed. - - Returns: - A scalar `Tensor` of type `string` whose contents are the serialized - `Summary` protocol buffer. - """ - name = _get_summary_name(tensor, name, prefix, 'Fraction_of_Zero_Values') - tensor = nn.zero_fraction(tensor) - return add_scalar_summary(tensor, name, print_summary=print_summary) - - -def add_histogram_summaries(tensors, prefix=None): - """Adds a histogram summary for each of the given tensors. - - Args: - tensors: A list of variable or op tensors. - prefix: An optional prefix for the summary names. - - Returns: - A list of scalar `Tensors` of type `string` whose contents are the - serialized `Summary` protocol buffer. - """ - summary_ops = [] - for tensor in tensors: - summary_ops.append(add_histogram_summary(tensor, prefix=prefix)) - return summary_ops - - -def add_image_summaries(tensors, prefix=None): - """Adds an image summary for each of the given tensors. - - Args: - tensors: A list of variable or op tensors. - prefix: An optional prefix for the summary names. - - Returns: - A list of scalar `Tensors` of type `string` whose contents are the - serialized `Summary` protocol buffer. - """ - summary_ops = [] - for tensor in tensors: - summary_ops.append(add_image_summary(tensor, prefix=prefix)) - return summary_ops - - -def add_scalar_summaries(tensors, prefix=None, print_summary=False): - """Adds a scalar summary for each of the given tensors. - - Args: - tensors: a list of variable or op tensors. - prefix: An optional prefix for the summary names. - print_summary: If `True`, the summary is printed to stdout when the summary - is computed. - - Returns: - A list of scalar `Tensors` of type `string` whose contents are the - serialized `Summary` protocol buffer. - """ - summary_ops = [] - for tensor in tensors: - summary_ops.append(add_scalar_summary(tensor, prefix=prefix, - print_summary=print_summary)) - return summary_ops - - -def add_zero_fraction_summaries(tensors, prefix=None): - """Adds a scalar zero-fraction summary for each of the given tensors. - - Args: - tensors: a list of variable or op tensors. - prefix: An optional prefix for the summary names. - - Returns: - A list of scalar `Tensors` of type `string` whose contents are the - serialized `Summary` protocol buffer. - """ - summary_ops = [] - for tensor in tensors: - summary_ops.append(add_zero_fraction_summary(tensor, prefix=prefix)) - return summary_ops diff --git a/tensorflow/contrib/slim/python/slim/summaries_test.py b/tensorflow/contrib/slim/python/slim/summaries_test.py deleted file mode 100644 index c6017f073ed..00000000000 --- a/tensorflow/contrib/slim/python/slim/summaries_test.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.contrib.slim.summaries.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import glob -import os - - -from tensorflow.contrib.slim.python.slim import summaries -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test -from tensorflow.python.summary import summary -from tensorflow.python.summary import summary_iterator - - -class SummariesTest(test.TestCase): - - def safe_create(self, output_dir): - if gfile.Exists(output_dir): - gfile.DeleteRecursively(output_dir) - gfile.MakeDirs(output_dir) - - def assert_scalar_summary(self, output_dir, names_to_values): - """Asserts that the given output directory contains written summaries. - - Args: - output_dir: The output directory in which to look for even tfiles. - names_to_values: A dictionary of summary names to values. - """ - # The events file may have additional entries, e.g. the event version - # stamp, so have to parse things a bit. - output_filepath = glob.glob(os.path.join(output_dir, '*')) - self.assertEqual(len(output_filepath), 1) - - events = summary_iterator.summary_iterator(output_filepath[0]) - summaries_list = [e.summary for e in events if e.summary.value] - values = [] - for item in summaries_list: - for value in item.value: - values.append(value) - saved_results = {v.tag: v.simple_value for v in values} - for name in names_to_values: - self.assertAlmostEqual(names_to_values[name], saved_results[name]) - - def testScalarSummaryIsPartOfCollectionWithNoPrint(self): - tensor = array_ops.ones([]) * 3 - name = 'my_score' - prefix = 'eval' - op = summaries.add_scalar_summary(tensor, name, prefix, print_summary=False) - self.assertTrue(op in ops.get_collection(ops.GraphKeys.SUMMARIES)) - - def testScalarSummaryIsPartOfCollectionWithPrint(self): - tensor = array_ops.ones([]) * 3 - name = 'my_score' - prefix = 'eval' - op = summaries.add_scalar_summary(tensor, name, prefix, print_summary=True) - self.assertTrue(op in ops.get_collection(ops.GraphKeys.SUMMARIES)) - - def verify_scalar_summary_is_written(self, print_summary): - value = 3 - tensor = array_ops.ones([]) * value - name = 'my_score' - prefix = 'eval' - summaries.add_scalar_summary(tensor, name, prefix, print_summary) - - output_dir = os.path.join(self.get_temp_dir(), - 'scalar_summary_no_print_test') - self.safe_create(output_dir) - - summary_op = summary.merge_all() - - summary_writer = summary.FileWriter(output_dir) - with self.cached_session() as sess: - new_summary = sess.run(summary_op) - summary_writer.add_summary(new_summary, 1) - summary_writer.flush() - - self.assert_scalar_summary(output_dir, { - '%s/%s' % (prefix, name): value - }) - - def testScalarSummaryIsWrittenWithNoPrint(self): - self.verify_scalar_summary_is_written(print_summary=False) - - def testScalarSummaryIsWrittenWithPrint(self): - self.verify_scalar_summary_is_written(print_summary=True) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/solvers/BUILD b/tensorflow/contrib/solvers/BUILD deleted file mode 100644 index 7831fc3d7a6..00000000000 --- a/tensorflow/contrib/solvers/BUILD +++ /dev/null @@ -1,96 +0,0 @@ -# Description: -# Contains ops for iterative solvers for linear systems, linear least-squares -# problems, singular value decomposition and eigenvalue decomposition. - -load("//tensorflow:tensorflow.bzl", "cuda_py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "solvers_py", - srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:tensor_array_ops", - ], -) - -# Ops tests -cuda_py_test( - name = "lanczos_test", - srcs = [ - "python/kernel_tests/lanczos_test.py", - ], - additional_deps = [ - ":solvers_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], - shard_count = 4, -) - -cuda_py_test( - name = "least_squares_test", - srcs = [ - "python/kernel_tests/least_squares_test.py", - ], - additional_deps = [ - ":solvers_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], - shard_count = 4, -) - -cuda_py_test( - name = "linear_equations_test", - srcs = [ - "python/kernel_tests/linear_equations_test.py", - ], - additional_deps = [ - ":solvers_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], - shard_count = 4, -) - -cuda_py_test( - name = "util_test", - srcs = [ - "python/kernel_tests/util_test.py", - ], - additional_deps = [ - ":solvers_py", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) diff --git a/tensorflow/contrib/solvers/__init__.py b/tensorflow/contrib/solvers/__init__.py deleted file mode 100644 index 13d857c5983..00000000000 --- a/tensorflow/contrib/solvers/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for representing Bayesian computation. - -## This package provides classes for Bayesian computation with TensorFlow. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import,line-too-long -from tensorflow.contrib.solvers.python.ops import lanczos -from tensorflow.contrib.solvers.python.ops import least_squares -from tensorflow.contrib.solvers.python.ops import linear_equations -from tensorflow.contrib.solvers.python.ops import util diff --git a/tensorflow/contrib/solvers/python/__init__.py b/tensorflow/contrib/solvers/python/__init__.py deleted file mode 100644 index c5ca3a623fb..00000000000 --- a/tensorflow/contrib/solvers/python/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ops module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/solvers/python/kernel_tests/lanczos_test.py b/tensorflow/contrib/solvers/python/kernel_tests/lanczos_test.py deleted file mode 100644 index f31bdbd399c..00000000000 --- a/tensorflow/contrib/solvers/python/kernel_tests/lanczos_test.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python import tf2 -from tensorflow.contrib.solvers.python.ops import lanczos -from tensorflow.contrib.solvers.python.ops import util -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test as test_lib - - -def _add_test(test, test_name, fn): - test_name = "_".join(["test", test_name]) - if hasattr(test, test_name): - raise RuntimeError("Test %s defined more than once" % test_name) - setattr(test, test_name, fn) - - -class LanczosBidiagTest(test_lib.TestCase): - pass # Filled in below. - - -def _get_lanczos_tests(dtype_, use_static_shape_, shape_, orthogonalize_, - steps_): - - def test_lanczos_bidiag(self): - np.random.seed(1) - a_np = np.random.uniform( - low=-1.0, high=1.0, size=np.prod(shape_)).reshape(shape_).astype(dtype_) - tol = 1e-12 if dtype_ == np.float64 else 1e-5 - - with self.cached_session() as sess: - if use_static_shape_: - a = constant_op.constant(a_np) - else: - a = array_ops.placeholder(dtype_) - operator = util.create_operator(a) - lbd = lanczos.lanczos_bidiag( - operator, steps_, orthogonalize=orthogonalize_) - - # The computed factorization should satisfy the equations - # A * V = U * B - # A' * U[:, :-1] = V * B[:-1, :]' - av = math_ops.matmul(a, lbd.v) - ub = lanczos.bidiag_matmul(lbd.u, lbd.alpha, lbd.beta, adjoint_b=False) - atu = math_ops.matmul(a, lbd.u[:, :-1], adjoint_a=True) - vbt = lanczos.bidiag_matmul(lbd.v, lbd.alpha, lbd.beta, adjoint_b=True) - - if use_static_shape_: - av_val, ub_val, atu_val, vbt_val = sess.run([av, ub, atu, vbt]) - else: - av_val, ub_val, atu_val, vbt_val = sess.run([av, ub, atu, vbt], - feed_dict={a: a_np}) - self.assertAllClose(av_val, ub_val, atol=tol, rtol=tol) - self.assertAllClose(atu_val, vbt_val, atol=tol, rtol=tol) - - return [test_lanczos_bidiag] - - -if __name__ == "__main__": - for dtype in np.float32, np.float64: - for shape in [[4, 4], [7, 4], [5, 8]]: - for orthogonalize in True, False: - for steps in range(1, min(shape) + 1): - # TF2 does not support placeholders so we skip it - for use_static_shape in set([True, tf2.enabled()]): - arg_string = "%s_%s_%s_%s_staticshape_%s" % ( - dtype.__name__, "_".join(map(str, shape)), orthogonalize, steps, - use_static_shape) - for test_fn in _get_lanczos_tests(dtype, use_static_shape, shape, - orthogonalize, steps): - name = "_".join(["Lanczos", test_fn.__name__, arg_string]) - _add_test(LanczosBidiagTest, name, test_fn) - - test_lib.main() diff --git a/tensorflow/contrib/solvers/python/kernel_tests/least_squares_test.py b/tensorflow/contrib/solvers/python/kernel_tests/least_squares_test.py deleted file mode 100644 index 841a41a2339..00000000000 --- a/tensorflow/contrib/solvers/python/kernel_tests/least_squares_test.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python import tf2 -from tensorflow.contrib.solvers.python.ops import least_squares -from tensorflow.contrib.solvers.python.ops import util -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test as test_lib - - -def _add_test(test, test_name, fn): - test_name = "_".join(["test", test_name]) - if hasattr(test, test_name): - raise RuntimeError("Test %s defined more than once" % test_name) - setattr(test, test_name, fn) - - -class LeastSquaresTest(test_lib.TestCase): - pass # Filled in below. - - -def _get_least_squares_tests(dtype_, use_static_shape_, shape_): - - def test_cgls(self): - np.random.seed(1) - a_np = np.random.uniform( - low=-1.0, high=1.0, size=np.prod(shape_)).reshape(shape_).astype(dtype_) - rhs_np = np.random.uniform( - low=-1.0, high=1.0, size=shape_[0]).astype(dtype_) - tol = 1e-12 if dtype_ == np.float64 else 1e-6 - max_iter = 20 - with self.cached_session() as sess: - if use_static_shape_: - a = constant_op.constant(a_np) - rhs = constant_op.constant(rhs_np) - else: - a = array_ops.placeholder(dtype_) - rhs = array_ops.placeholder(dtype_) - operator = util.create_operator(a) - cgls_graph = least_squares.cgls(operator, rhs, tol=tol, max_iter=max_iter) - if use_static_shape_: - cgls_val = sess.run(cgls_graph) - else: - cgls_val = sess.run(cgls_graph, feed_dict={a: a_np, rhs: rhs_np}) - # Below we use s = A^* (rhs - A x), s0 = A^* rhs - norm_s0 = np.linalg.norm(np.dot(a_np.T, rhs_np)) - norm_s = np.sqrt(cgls_val.gamma) - self.assertLessEqual(norm_s, tol * norm_s0) - # Validate that we get an equally small residual norm with numpy - # using the computed solution. - r_np = rhs_np - np.dot(a_np, cgls_val.x) - norm_s_np = np.linalg.norm(np.dot(a_np.T, r_np)) - self.assertLessEqual(norm_s_np, tol * norm_s0) - - return [test_cgls] - - -if __name__ == "__main__": - for dtype in np.float32, np.float64: - for shape in [[4, 4], [8, 5], [3, 7]]: - # TF2 does not support placeholders under eager so we skip it - for use_static_shape in set([True, tf2.enabled()]): - arg_string = "%s_%s_staticshape_%s" % (dtype.__name__, - "_".join(map(str, shape)), - use_static_shape) - for test_fn in _get_least_squares_tests(dtype, use_static_shape, shape): - name = "_".join(["LeastSquares", test_fn.__name__, arg_string]) - _add_test(LeastSquaresTest, name, test_fn) - - test_lib.main() diff --git a/tensorflow/contrib/solvers/python/kernel_tests/linear_equations_test.py b/tensorflow/contrib/solvers/python/kernel_tests/linear_equations_test.py deleted file mode 100644 index 10807f7a806..00000000000 --- a/tensorflow/contrib/solvers/python/kernel_tests/linear_equations_test.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python import tf2 -from tensorflow.contrib.solvers.python.ops import linear_equations -from tensorflow.contrib.solvers.python.ops import util -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test as test_lib - - -def _add_test(test, test_name, fn): - test_name = "_".join(["test", test_name]) - if hasattr(test, test_name): - raise RuntimeError("Test %s defined more than once" % test_name) - setattr(test, test_name, fn) - - -class LinearEquationsTest(test_lib.TestCase): - pass # Filled in below. - - -def _get_linear_equations_tests(dtype_, use_static_shape_, shape_): - - def test_conjugate_gradient(self): - np.random.seed(1) - a_np = np.random.uniform( - low=-1.0, high=1.0, size=np.prod(shape_)).reshape(shape_).astype(dtype_) - # Make a selfadjoint, positive definite. - a_np = np.dot(a_np.T, a_np) - # jacobi preconditioner - jacobi_np = np.zeros_like(a_np) - jacobi_np[range(a_np.shape[0]), range(a_np.shape[1])] = ( - 1.0 / a_np.diagonal()) - rhs_np = np.random.uniform( - low=-1.0, high=1.0, size=shape_[0]).astype(dtype_) - x_np = np.zeros_like(rhs_np) - tol = 1e-6 if dtype_ == np.float64 else 1e-3 - max_iter = 20 - with self.cached_session() as sess: - if use_static_shape_: - a = constant_op.constant(a_np) - rhs = constant_op.constant(rhs_np) - x = constant_op.constant(x_np) - jacobi = constant_op.constant(jacobi_np) - else: - a = array_ops.placeholder(dtype_) - rhs = array_ops.placeholder(dtype_) - x = array_ops.placeholder(dtype_) - jacobi = array_ops.placeholder(dtype_) - operator = util.create_operator(a) - preconditioners = [ - None, util.identity_operator(a), - util.create_operator(jacobi) - ] - cg_results = [] - for preconditioner in preconditioners: - cg_graph = linear_equations.conjugate_gradient( - operator, - rhs, - preconditioner=preconditioner, - x=x, - tol=tol, - max_iter=max_iter) - if use_static_shape_: - cg_val = sess.run(cg_graph) - else: - cg_val = sess.run( - cg_graph, - feed_dict={ - a: a_np, - rhs: rhs_np, - x: x_np, - jacobi: jacobi_np - }) - norm_r0 = np.linalg.norm(rhs_np) - norm_r = np.linalg.norm(cg_val.r) - self.assertLessEqual(norm_r, tol * norm_r0) - # Validate that we get an equally small residual norm with numpy - # using the computed solution. - r_np = rhs_np - np.dot(a_np, cg_val.x) - norm_r_np = np.linalg.norm(r_np) - self.assertLessEqual(norm_r_np, tol * norm_r0) - cg_results.append(cg_val) - # Validate that we get same results using identity_preconditioner - # and None - self.assertEqual(cg_results[0].i, cg_results[1].i) - self.assertAlmostEqual(cg_results[0].gamma, cg_results[1].gamma) - self.assertAllClose(cg_results[0].r, cg_results[1].r, rtol=tol) - self.assertAllClose(cg_results[0].x, cg_results[1].x, rtol=tol) - self.assertAllClose(cg_results[0].p, cg_results[1].p, rtol=tol) - - return [test_conjugate_gradient] - - -if __name__ == "__main__": - for dtype in np.float32, np.float64: - for size in 1, 4, 10: - # TF2 does not support placeholders under eager so we skip it - for use_static_shape in set([True, tf2.enabled()]): - shape = [size, size] - arg_string = "%s_%s_staticshape_%s" % (dtype.__name__, size, - use_static_shape) - for test_fn in _get_linear_equations_tests(dtype, use_static_shape, - shape): - name = "_".join(["LinearEquations", test_fn.__name__, arg_string]) - _add_test(LinearEquationsTest, name, test_fn) - - test_lib.main() diff --git a/tensorflow/contrib/solvers/python/kernel_tests/util_test.py b/tensorflow/contrib/solvers/python/kernel_tests/util_test.py deleted file mode 100644 index 57b49966891..00000000000 --- a/tensorflow/contrib/solvers/python/kernel_tests/util_test.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.solvers.python.ops import util -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class UtilTest(test.TestCase): - - def _testCreateOperator(self, use_static_shape_): - for dtype in np.float32, np.float64: - a_np = np.array([[1., 2.], [3., 4.], [5., 6.]], dtype=dtype) - x_np = np.array([[2.], [-3.]], dtype=dtype) - y_np = np.array([[2], [-3.], [5.]], dtype=dtype) - with self.cached_session() as sess: - if use_static_shape_: - a = constant_op.constant(a_np, dtype=dtype) - x = constant_op.constant(x_np, dtype=dtype) - y = constant_op.constant(y_np, dtype=dtype) - else: - a = array_ops.placeholder(dtype) - x = array_ops.placeholder(dtype) - y = array_ops.placeholder(dtype) - op = util.create_operator(a) - ax = op.apply(x) - aty = op.apply_adjoint(y) - op_shape = ops.convert_to_tensor(op.shape) - if use_static_shape_: - op_shape_val, ax_val, aty_val = sess.run([op_shape, ax, aty]) - else: - op_shape_val, ax_val, aty_val = sess.run( - [op_shape, ax, aty], feed_dict={a: a_np, - x: x_np, - y: y_np}) - self.assertAllEqual(op_shape_val, [3, 2]) - self.assertAllClose(ax_val, np.dot(a_np, x_np)) - self.assertAllClose(aty_val, np.dot(a_np.T, y_np)) - - def testCreateOperator(self): - self._testCreateOperator(True) - - def testCreateOperatorUnknownShape(self): - self._testCreateOperator(False) - - def _testIdentityOperator(self, use_static_shape_): - for dtype in np.float32, np.float64: - a_np = np.array([[1., 2.], [3., 4.], [5., 6.]], dtype=dtype) - x_np = np.array([[2.], [-3.]], dtype=dtype) - y_np = np.array([[2], [-3.], [5.]], dtype=dtype) - with self.cached_session() as sess: - if use_static_shape_: - a = constant_op.constant(a_np, dtype=dtype) - x = constant_op.constant(x_np, dtype=dtype) - y = constant_op.constant(y_np, dtype=dtype) - else: - a = array_ops.placeholder(dtype) - x = array_ops.placeholder(dtype) - y = array_ops.placeholder(dtype) - id_op = util.identity_operator(a) - ax = id_op.apply(x) - aty = id_op.apply_adjoint(y) - op_shape = ops.convert_to_tensor(id_op.shape) - if use_static_shape_: - op_shape_val, ax_val, aty_val = sess.run([op_shape, ax, aty]) - else: - op_shape_val, ax_val, aty_val = sess.run( - [op_shape, ax, aty], feed_dict={ - a: a_np, - x: x_np, - y: y_np - }) - self.assertAllEqual(op_shape_val, [3, 2]) - self.assertAllClose(ax_val, x_np) - self.assertAllClose(aty_val, y_np) - - def testIdentityOperator(self): - self._testIdentityOperator(True) - - def testIdentityOperatorUnknownShape(self): - self._testIdentityOperator(False) - - def testL2Norm(self): - with self.cached_session(): - x_np = np.array([[2], [-3.], [5.]]) - x_norm_np = np.linalg.norm(x_np) - x_normalized_np = x_np / x_norm_np - x = constant_op.constant(x_np) - l2norm = util.l2norm(x) - l2norm_squared = util.l2norm_squared(x) - x_normalized, x_norm = util.l2normalize(x) - self.assertAllClose(l2norm.eval(), x_norm_np) - self.assertAllClose(l2norm_squared.eval(), np.square(x_norm_np)) - self.assertAllClose(x_norm.eval(), x_norm_np) - self.assertAllClose(x_normalized.eval(), x_normalized_np) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/solvers/python/ops/lanczos.py b/tensorflow/contrib/solvers/python/ops/lanczos.py deleted file mode 100644 index af1b2937436..00000000000 --- a/tensorflow/contrib/solvers/python/ops/lanczos.py +++ /dev/null @@ -1,237 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Lanczos algorithms.""" - -# TODO(rmlarsen): Add implementation of symmetric Lanczos algorithm. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from tensorflow.contrib.solvers.python.ops import util -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import tensor_array_ops - - -def lanczos_bidiag(operator, - k, - orthogonalize=True, - starting_vector=None, - name="lanczos_bidiag"): - """Computes a Lanczos bidiagonalization for a linear operator. - - Computes matrices `U` of shape `[m, k+1]`, `V` of shape `[n, k]` and lower - bidiagonal matrix `B` of shape `[k+1, k]`, that satisfy the equations - `A * V = U * B` and `A' * U[:, :-1] = V * B[:-1, :]'`. - - The columns of `U` are orthonormal and form a basis for the Krylov subspace - `K(A*A', U[:,0])`. - - The columns of `V` are orthonormal and form a basis for the Krylov subspace - `K(A'*A, A' U[:,0])`. - - Args: - operator: An object representing a linear operator with attributes: - - shape: Either a list of integers or a 1-D `Tensor` of type `int32` of - length 2. `shape[0]` is the dimension on the domain of the operator, - `shape[1]` is the dimension of the co-domain of the operator. On other - words, if operator represents an M x N matrix A, `shape` must contain - `[M, N]`. - - dtype: The datatype of input to and output from `apply` and - `apply_adjoint`. - - apply: Callable object taking a vector `x` as input and returning a - vector with the result of applying the operator to `x`, i.e. if - `operator` represents matrix `A`, `apply` should return `A * x`. - - apply_adjoint: Callable object taking a vector `x` as input and - returning a vector with the result of applying the adjoint operator - to `x`, i.e. if `operator` represents matrix `A`, `apply_adjoint` should - return `conj(transpose(A)) * x`. - k: An integer or a scalar Tensor of type `int32`. Determines the maximum - number of steps to run. If an invariant subspace is found, the algorithm - may terminate before `k` steps have been run. - orthogonalize: If `True`, perform full orthogonalization. If `False` no - orthogonalization is performed. - starting_vector: If not null, must be a `Tensor` of shape `[n]`. - name: A name scope for the operation. - - Returns: - output: A namedtuple representing a Lanczos bidiagonalization of - `operator` with attributes: - u: A rank-2 `Tensor` of type `operator.dtype` and shape - `[operator.shape[0], k_actual+1]`, where `k_actual` is the number of - steps run. - v: A rank-2 `Tensor` of type `operator.dtype` and shape - `[operator.shape[1], k_actual]`, where `k_actual` is the number of steps - run. - alpha: A rank-1 `Tensor` of type `operator.dtype` and shape `[k]`. - beta: A rank-1 `Tensor` of type `operator.dtype` and shape `[k]`. - """ - - def tarray(size, dtype, name): - return tensor_array_ops.TensorArray( - dtype=dtype, size=size, tensor_array_name=name, clear_after_read=False) - - # Reads a row-vector at location i in tarray and returns it as a - # column-vector. - def read_colvec(tarray, i): - return array_ops.expand_dims(tarray.read(i), -1) - - # Writes an column-vector as a row-vecor at location i in tarray. - def write_colvec(tarray, colvec, i): - return tarray.write(i, array_ops.squeeze(colvec)) - - # Ephemeral class holding Lanczos bidiagonalization state: - # u = left Lanczos vectors - # v = right Lanczos vectors - # alpha = diagonal of B_k. - # beta = subdiagonal of B_k. - # Notice that we store the left and right Lanczos vectors as the _rows_ - # of u and v. This is done because tensors are stored row-major and - # TensorArray only supports packing along dimension 0. - lanzcos_bidiag_state = collections.namedtuple("LanczosBidiagState", - ["u", "v", "alpha", "beta"]) - - def update_state(old, i, u, v, alpha, beta): - return lanzcos_bidiag_state( - write_colvec(old.u, u, i + 1), - write_colvec(old.v, v, i), - old.alpha.write(i, alpha), old.beta.write(i, beta)) - - def gram_schmidt_step(j, basis, v): - """Makes v orthogonal to the j'th vector in basis.""" - v_shape = v.get_shape() - basis_vec = read_colvec(basis, j) - v -= math_ops.matmul(basis_vec, v, adjoint_a=True) * basis_vec - v.set_shape(v_shape) - return j + 1, basis, v - - def orthogonalize_once(i, basis, v): - j = constant_op.constant(0, dtype=dtypes.int32) - _, _, v = control_flow_ops.while_loop(lambda j, basis, v: j < i, - gram_schmidt_step, [j, basis, v]) - return util.l2normalize(v) - - # Iterated modified Gram-Schmidt orthogonalization adapted from PROPACK. - # TODO(rmlarsen): This is possibly the slowest implementation of - # iterated Gram-Schmidt orthogonalization since the abacus. Move to C++. - def orthogonalize_(i, basis, v): - v_norm = util.l2norm(v) - v_new, v_new_norm = orthogonalize_once(i, basis, v) - # If the norm decreases more than 1/sqrt(2), run a second - # round of MGS. See proof in: - # B. N. Parlett, ``The Symmetric Eigenvalue Problem'', - # Prentice-Hall, Englewood Cliffs, NJ, 1980. pp. 105-109 - return control_flow_ops.cond(v_new_norm < 0.7071 * v_norm, - lambda: orthogonalize_once(i, basis, v), - lambda: (v_new, v_new_norm)) - - def stopping_criterion(i, _): - # TODO(rmlarsen): Stop if an invariant subspace is detected. - return i < k - - def lanczos_bidiag_step(i, ls): - """Extends the Lanczos bidiagonalization ls by one step.""" - u = read_colvec(ls.u, i) - r = operator.apply_adjoint(u) - # The shape inference doesn't work across cond, save and reapply the shape. - r_shape = r.get_shape() - r = control_flow_ops.cond( - i > 0, lambda: r - ls.beta.read(i - 1) * read_colvec(ls.v, i - 1), - lambda: r) - r.set_shape(r_shape) - if orthogonalize: - v, alpha = orthogonalize_(i - 1, ls.v, r) - else: - v, alpha = util.l2normalize(r) - p = operator.apply(v) - alpha * u - if orthogonalize: - u, beta = orthogonalize_(i, ls.u, p) - else: - u, beta = util.l2normalize(p) - - return i + 1, update_state(ls, i, u, v, alpha, beta) - - with ops.name_scope(name): - dtype = operator.dtype - if starting_vector is None: - starting_vector = random_ops.random_uniform( - operator.shape[:1], -1, 1, dtype=dtype) - u0, _ = util.l2normalize(starting_vector) - ls = lanzcos_bidiag_state( - u=write_colvec(tarray(k + 1, dtype, "u"), u0, 0), - v=tarray(k, dtype, "v"), - alpha=tarray(k, dtype, "alpha"), - beta=tarray(k, dtype, "beta")) - i = constant_op.constant(0, dtype=dtypes.int32) - _, ls = control_flow_ops.while_loop(stopping_criterion, lanczos_bidiag_step, - [i, ls]) - return lanzcos_bidiag_state( - array_ops.matrix_transpose(ls.u.stack()), - array_ops.matrix_transpose(ls.v.stack()), - ls.alpha.stack(), ls.beta.stack()) - - -# TODO(rmlarsen): Implement C++ ops for handling bidiagonal matrices -# efficiently. Such a module should provide -# - multiplication, -# - linear system solution by back-substitution, -# - QR factorization, -# - SVD. -def bidiag_matmul(matrix, alpha, beta, adjoint_b=False, name="bidiag_matmul"): - """Multiplies a matrix by a bidiagonal matrix. - - alpha and beta are length k vectors representing the diagonal and first lower - subdiagonal of (K+1) x K matrix B. - If adjoint_b is False, computes A * B as follows: - - A * B = A[:, :-1] * diag(alpha) + A[:, 1:] * diag(beta) - - If adjoint_b is True, computes A * B[:-1, :]' as follows - - A * B[:-1, :]' = - A * diag(alpha) + [zeros(m,1), A[:, :-1] * diag(beta[:-1])] - - Args: - matrix: A rank-2 `Tensor` representing matrix A. - alpha: A rank-1 `Tensor` representing the diagonal of B. - beta: A rank-1 `Tensor` representing the lower subdiagonal diagonal of B. - adjoint_b: `bool` determining what to compute. - name: A name scope for the operation. - - Returns: - If `adjoint_b` is False the `A * B` is returned. - If `adjoint_b` is True the `A * B'` is returned. - """ - with ops.name_scope(name): - alpha = array_ops.expand_dims(alpha, 0) - if adjoint_b is False: - beta = array_ops.expand_dims(beta, 0) - return matrix[:, :-1] * alpha + matrix[:, 1:] * beta - else: - beta = array_ops.expand_dims(beta[:-1], 0) - shape = array_ops.shape(matrix) - zero_column = array_ops.expand_dims( - array_ops.zeros( - shape[:1], dtype=matrix.dtype), 1) - return matrix * alpha + array_ops.concat( - [zero_column, matrix[:, :-1] * beta], 1) diff --git a/tensorflow/contrib/solvers/python/ops/least_squares.py b/tensorflow/contrib/solvers/python/ops/least_squares.py deleted file mode 100644 index 6e164f53420..00000000000 --- a/tensorflow/contrib/solvers/python/ops/least_squares.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Solvers for linear least-squares.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from tensorflow.contrib.solvers.python.ops import util -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops - - -def cgls(operator, rhs, tol=1e-6, max_iter=20, name="cgls"): - r"""Conjugate gradient least squares solver. - - Solves a linear least squares problem \\(||A x - rhs||_2\\) for a single - right-hand side, using an iterative, matrix-free algorithm where the action of - the matrix A is represented by `operator`. The CGLS algorithm implicitly - applies the symmetric conjugate gradient algorithm to the normal equations - \\(A^* A x = A^* rhs\\). The iteration terminates when either - the number of iterations exceeds `max_iter` or when the norm of the conjugate - residual (residual of the normal equations) have been reduced to `tol` times - its initial initial value, i.e. - \\(||A^* (rhs - A x_k)|| <= tol ||A^* rhs||\\). - - Args: - operator: An object representing a linear operator with attributes: - - shape: Either a list of integers or a 1-D `Tensor` of type `int32` of - length 2. `shape[0]` is the dimension on the domain of the operator, - `shape[1]` is the dimension of the co-domain of the operator. On other - words, if operator represents an M x N matrix A, `shape` must contain - `[M, N]`. - - dtype: The datatype of input to and output from `apply` and - `apply_adjoint`. - - apply: Callable object taking a vector `x` as input and returning a - vector with the result of applying the operator to `x`, i.e. if - `operator` represents matrix `A`, `apply` should return `A * x`. - - apply_adjoint: Callable object taking a vector `x` as input and - returning a vector with the result of applying the adjoint operator - to `x`, i.e. if `operator` represents matrix `A`, `apply_adjoint` should - return `conj(transpose(A)) * x`. - - rhs: A rank-1 `Tensor` of shape `[M]` containing the right-hand size vector. - tol: A float scalar convergence tolerance. - max_iter: An integer giving the maximum number of iterations. - name: A name scope for the operation. - - - Returns: - output: A namedtuple representing the final state with fields: - - i: A scalar `int32` `Tensor`. Number of iterations executed. - - x: A rank-1 `Tensor` of shape `[N]` containing the computed solution. - - r: A rank-1 `Tensor` of shape `[M]` containing the residual vector. - - p: A rank-1 `Tensor` of shape `[N]`. The next descent direction. - - gamma: \\(||A^* r||_2^2\\) - """ - # ephemeral class holding CGLS state. - cgls_state = collections.namedtuple("CGLSState", - ["i", "x", "r", "p", "gamma"]) - - def stopping_criterion(i, state): - return math_ops.logical_and(i < max_iter, state.gamma > tol) - - # TODO(rmlarsen): add preconditioning - def cgls_step(i, state): - q = operator.apply(state.p) - alpha = state.gamma / util.l2norm_squared(q) - x = state.x + alpha * state.p - r = state.r - alpha * q - s = operator.apply_adjoint(r) - gamma = util.l2norm_squared(s) - beta = gamma / state.gamma - p = s + beta * state.p - return i + 1, cgls_state(i + 1, x, r, p, gamma) - - with ops.name_scope(name): - n = operator.shape[1:] - rhs = array_ops.expand_dims(rhs, -1) - s0 = operator.apply_adjoint(rhs) - gamma0 = util.l2norm_squared(s0) - tol = tol * tol * gamma0 - x = array_ops.expand_dims( - array_ops.zeros( - n, dtype=rhs.dtype.base_dtype), -1) - i = constant_op.constant(0, dtype=dtypes.int32) - state = cgls_state(i=i, x=x, r=rhs, p=s0, gamma=gamma0) - _, state = control_flow_ops.while_loop(stopping_criterion, cgls_step, - [i, state]) - return cgls_state( - state.i, - x=array_ops.squeeze(state.x), - r=array_ops.squeeze(state.r), - p=array_ops.squeeze(state.p), - gamma=state.gamma) diff --git a/tensorflow/contrib/solvers/python/ops/linear_equations.py b/tensorflow/contrib/solvers/python/ops/linear_equations.py deleted file mode 100644 index 85918bf8506..00000000000 --- a/tensorflow/contrib/solvers/python/ops/linear_equations.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Solvers for linear equations.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from tensorflow.contrib.solvers.python.ops import util -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops - - -def conjugate_gradient(operator, - rhs, - preconditioner=None, - x=None, - tol=1e-4, - max_iter=20, - name="conjugate_gradient"): - r"""Conjugate gradient solver. - - Solves a linear system of equations `A*x = rhs` for selfadjoint, positive - definite matrix `A` and right-hand side vector `rhs`, using an iterative, - matrix-free algorithm where the action of the matrix A is represented by - `operator`. The iteration terminates when either the number of iterations - exceeds `max_iter` or when the residual norm has been reduced to `tol` - times its initial value, i.e. \\(||rhs - A x_k|| <= tol ||rhs||\\). - - Args: - operator: An object representing a linear operator with attributes: - - shape: Either a list of integers or a 1-D `Tensor` of type `int32` of - length 2. `shape[0]` is the dimension on the domain of the operator, - `shape[1]` is the dimension of the co-domain of the operator. On other - words, if operator represents an N x N matrix A, `shape` must contain - `[N, N]`. - - dtype: The datatype of input to and output from `apply`. - - apply: Callable object taking a vector `x` as input and returning a - vector with the result of applying the operator to `x`, i.e. if - `operator` represents matrix `A`, `apply` should return `A * x`. - rhs: A rank-1 `Tensor` of shape `[N]` containing the right-hand size vector. - preconditioner: An object representing a linear operator, see `operator` - for detail. The preconditioner should approximate the inverse of `A`. - An efficient preconditioner could dramatically improve the rate of - convergence. If `preconditioner` represents matrix `M`(`M` approximates - `A^{-1}`), the algorithm uses `preconditioner.apply(x)` to estimate - `A^{-1}x`. For this to be useful, the cost of applying `M` should be - much lower than computing `A^{-1}` directly. - x: A rank-1 `Tensor` of shape `[N]` containing the initial guess for the - solution. - tol: A float scalar convergence tolerance. - max_iter: An integer giving the maximum number of iterations. - name: A name scope for the operation. - - Returns: - output: A namedtuple representing the final state with fields: - - i: A scalar `int32` `Tensor`. Number of iterations executed. - - x: A rank-1 `Tensor` of shape `[N]` containing the computed solution. - - r: A rank-1 `Tensor` of shape `[M]` containing the residual vector. - - p: A rank-1 `Tensor` of shape `[N]`. `A`-conjugate basis vector. - - gamma: \\(r \dot M \dot r\\), equivalent to \\(||r||_2^2\\) when - `preconditioner=None`. - """ - # ephemeral class holding CG state. - cg_state = collections.namedtuple("CGState", ["i", "x", "r", "p", "gamma"]) - - def stopping_criterion(i, state): - return math_ops.logical_and(i < max_iter, linalg_ops.norm(state.r) > tol) - - def cg_step(i, state): # pylint: disable=missing-docstring - z = operator.apply(state.p) - alpha = state.gamma / util.dot(state.p, z) - x = state.x + alpha * state.p - r = state.r - alpha * z - if preconditioner is None: - gamma = util.dot(r, r) - beta = gamma / state.gamma - p = r + beta * state.p - else: - q = preconditioner.apply(r) - gamma = util.dot(r, q) - beta = gamma / state.gamma - p = q + beta * state.p - return i + 1, cg_state(i + 1, x, r, p, gamma) - - with ops.name_scope(name): - n = operator.shape[1:] - rhs = array_ops.expand_dims(rhs, -1) - if x is None: - x = array_ops.expand_dims( - array_ops.zeros(n, dtype=rhs.dtype.base_dtype), -1) - r0 = rhs - else: - x = array_ops.expand_dims(x, -1) - r0 = rhs - operator.apply(x) - if preconditioner is None: - p0 = r0 - else: - p0 = preconditioner.apply(r0) - gamma0 = util.dot(r0, p0) - tol *= linalg_ops.norm(r0) - i = constant_op.constant(0, dtype=dtypes.int32) - state = cg_state(i=i, x=x, r=r0, p=p0, gamma=gamma0) - _, state = control_flow_ops.while_loop(stopping_criterion, cg_step, - [i, state]) - return cg_state( - state.i, - x=array_ops.squeeze(state.x), - r=array_ops.squeeze(state.r), - p=array_ops.squeeze(state.p), - gamma=state.gamma) diff --git a/tensorflow/contrib/solvers/python/ops/util.py b/tensorflow/contrib/solvers/python/ops/util.py deleted file mode 100644 index 96947e8eea1..00000000000 --- a/tensorflow/contrib/solvers/python/ops/util.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utility functions for solvers.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops - - -def create_operator(matrix): - """Creates a linear operator from a rank-2 tensor.""" - - linear_operator = collections.namedtuple( - "LinearOperator", ["shape", "dtype", "apply", "apply_adjoint"]) - - # TODO(rmlarsen): Handle SparseTensor. - shape = matrix.get_shape() - if shape.is_fully_defined(): - shape = shape.as_list() - else: - shape = array_ops.shape(matrix) - return linear_operator( - shape=shape, - dtype=matrix.dtype, - apply=lambda v: math_ops.matmul(matrix, v, adjoint_a=False), - apply_adjoint=lambda v: math_ops.matmul(matrix, v, adjoint_a=True)) - - -def identity_operator(matrix): - """Creates a linear operator from a rank-2 identity tensor.""" - - linear_operator = collections.namedtuple( - "LinearOperator", ["shape", "dtype", "apply", "apply_adjoint"]) - shape = matrix.get_shape() - if shape.is_fully_defined(): - shape = shape.as_list() - else: - shape = array_ops.shape(matrix) - return linear_operator( - shape=shape, - dtype=matrix.dtype, - apply=lambda v: v, - apply_adjoint=lambda v: v) - - -# TODO(rmlarsen): Measure if we should just call matmul. -def dot(x, y): - return math_ops.reduce_sum(math_ops.conj(x) * y) - - -# TODO(rmlarsen): Implement matrix/vector norm op in C++ in core. -# We need 1-norm, inf-norm, and Frobenius norm. -def l2norm_squared(v): - return constant_op.constant(2, dtype=v.dtype.base_dtype) * nn_ops.l2_loss(v) - - -def l2norm(v): - return math_ops.sqrt(l2norm_squared(v)) - - -def l2normalize(v): - norm = l2norm(v) - return v / norm, norm diff --git a/tensorflow/contrib/sparsemax/BUILD b/tensorflow/contrib/sparsemax/BUILD deleted file mode 100644 index 7bb73f5a415..00000000000 --- a/tensorflow/contrib/sparsemax/BUILD +++ /dev/null @@ -1,74 +0,0 @@ -# Description: -# Contains ops to train linear models on top of TensorFlow. -# APIs here are meant to evolve over time. - -load("//tensorflow:tensorflow.bzl", "cuda_py_tests") -load( - "//tensorflow:tensorflow.bzl", - "tf_custom_op_library", - "tf_py_test", -) -load( - "//tensorflow/core/platform:default/build_config.bzl", - "tf_kernel_tests_linkstatic", -) - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "sparsemax_py", - srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:platform", - "//tensorflow/python:util", - ], -) - -cuda_py_tests( - name = "sparsemax_test", - size = "medium", - srcs = ["python/kernel_tests/sparsemax_test.py"], - additional_deps = [ - ":sparsemax_py", - "//tensorflow:tensorflow_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], - tags = [ - "oss_serial", - ], -) - -cuda_py_tests( - name = "sparsemax_loss_test", - size = "medium", - srcs = ["python/kernel_tests/sparsemax_loss_test.py"], - additional_deps = [ - ":sparsemax_py", - "//tensorflow:tensorflow_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:gradients", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform_test", - ], - tags = [ - "oss_serial", - ], -) diff --git a/tensorflow/contrib/sparsemax/__init__.py b/tensorflow/contrib/sparsemax/__init__.py deleted file mode 100644 index 7bc726f4a84..00000000000 --- a/tensorflow/contrib/sparsemax/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Module that implements sparsemax and sparsemax loss, see [1]. - -[1]: https://arxiv.org/abs/1602.02068 - -## Sparsemax - -@@sparsemax -@@sparsemax_loss -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.sparsemax.python.ops.sparsemax import sparsemax -from tensorflow.contrib.sparsemax.python.ops.sparsemax_loss \ - import sparsemax_loss -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = ['sparsemax', 'sparsemax_loss'] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_loss_test.py b/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_loss_test.py deleted file mode 100644 index 7743f5b4a7f..00000000000 --- a/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_loss_test.py +++ /dev/null @@ -1,262 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for SparsemaxLossOp.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.sparsemax import sparsemax, sparsemax_loss -from tensorflow.python.ops import gradient_checker -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.framework import constant_op -from tensorflow.python.platform import test - -test_obs = 10 - - -class SparsemaxLossTest(test.TestCase): - - def _np_sparsemax(self, z): - z = z - np.mean(z, axis=1)[:, np.newaxis] - - # sort z - z_sorted = np.sort(z, axis=1)[:, ::-1] - - # calculate k(z) - z_cumsum = np.cumsum(z_sorted, axis=1) - k = np.arange(1, z.shape[1] + 1) - z_check = 1 + k * z_sorted > z_cumsum - # use argmax to get the index by row as .nonzero() doesn't - # take an axis argument. np.argmax return the first index, but the last - # index is required here, use np.flip to get the last index and - # `z.shape[axis]` to compensate for np.flip afterwards. - k_z = z.shape[1] - np.argmax(z_check[:, ::-1], axis=1) - - # calculate tau(z) - tau_sum = z_cumsum[np.arange(0, z.shape[0]), k_z - 1] - tau_z = ((tau_sum - 1) / k_z).reshape(-1, 1) - - # calculate p - return np.maximum(0, z - tau_z) - - def _np_sparsemax_loss(self, z, q): - z = z - np.mean(z, axis=1)[:, np.newaxis] - - # Calculate q^T * z - z_k = np.sum(q * z, axis=1) - - # calculate sum over S(z) - p = self._np_sparsemax(z) - s = p > 0 - # z_i^2 - tau(z)^2 = p_i (2 * z_i - p_i) for i \in S(z) - S_sum = np.sum(s * p * (2 * z - p), axis=1) - - # because q is binary, sum([q_1^2, q_2^2, ...]) is just sum(q) - q_norm = np.sum(q, axis=1) - - return -z_k + 0.5 * S_sum + 0.5 * q_norm - - def _np_sparsemax_loss_grad(self, z, q): - # chain rule - grad = 1 - - return grad * (-q + self._np_sparsemax(z)) - - def _tf_sparsemax(self, z, dtype, use_gpu): - with self.test_session(use_gpu=use_gpu): - tf_sparsemax_op = sparsemax(z.astype(dtype)) - tf_sparsemax_out = tf_sparsemax_op.eval() - - return tf_sparsemax_op, tf_sparsemax_out - - def _tf_sparsemax_loss(self, z, q, dtype, use_gpu): - z = z.astype(dtype) - q = q.astype(dtype) - - with self.test_session(use_gpu=use_gpu): - tf_sparsemax_op = sparsemax(z) - tf_loss_op = sparsemax_loss(z, tf_sparsemax_op, q) - tf_loss_out = tf_loss_op.eval() - - return tf_loss_op, tf_loss_out - - def _test_sparsemax_loss_against_numpy(self, dtype, random, use_gpu): - """check sparsemax-loss kernel against numpy""" - z = random.uniform(low=-3, high=3, size=(test_obs, 10)) - q = np.zeros((test_obs, 10)) - q[np.arange(0, test_obs), random.randint(0, 10, size=test_obs)] = 1 - - tf_loss_op, tf_loss_out = self._tf_sparsemax_loss(z, q, dtype, use_gpu) - np_loss = self._np_sparsemax_loss(z, q).astype(dtype) - - self.assertAllCloseAccordingToType( - np_loss, tf_loss_out, half_atol=1e-2, half_rtol=5e-3) - self.assertShapeEqual(np_loss, tf_loss_op) - - def _test_sparsemax_loss_of_nan(self, dtype, random, use_gpu): - """check sparsemax-loss transfers nan""" - q = np.asarray([[0, 0, 1], [0, 0, 1], [0, 0, 1]]) - z_nan = np.asarray([[0, np.nan, 0], [0, np.nan, np.nan], - [np.nan, np.nan, np.nan]]).astype(dtype) - - _, tf_loss_nan = self._tf_sparsemax_loss(z_nan, q, dtype, use_gpu) - self.assertAllCloseAccordingToType([np.nan, np.nan, np.nan], tf_loss_nan) - - def _test_sparsemax_loss_of_inf(self, dtype, random, use_gpu): - """check sparsemax-loss is infinity safe""" - q = np.asarray([[0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1]]) - z_neg = np.asarray([ - [0, -np.inf, 0], - [0, -np.inf, -np.inf], - [-np.inf, -np.inf, 0], - [-np.inf, -np.inf, -np.inf], - ]).astype(dtype) - z_pos = np.asarray([[0, np.inf, 0], [0, np.inf, - np.inf], [np.inf, np.inf, 0], - [np.inf, np.inf, np.inf]]).astype(dtype) - z_mix = np.asarray([[0, np.inf, 0], [0, np.inf, -np.inf], - [-np.inf, np.inf, 0], [-np.inf, np.inf, - -np.inf]]).astype(dtype) - - _, tf_loss_neg = self._tf_sparsemax_loss(z_neg, q, dtype, use_gpu) - self.assertAllCloseAccordingToType([0.25, np.inf, 0, np.nan], tf_loss_neg) - - _, tf_loss_pos = self._tf_sparsemax_loss(z_pos, q, dtype, use_gpu) - self.assertAllCloseAccordingToType([np.nan, np.nan, np.nan, np.nan], - tf_loss_pos) - - _, tf_loss_mix = self._tf_sparsemax_loss(z_mix, q, dtype, use_gpu) - self.assertAllCloseAccordingToType([np.nan, np.nan, np.nan, np.nan], - tf_loss_mix) - - def _test_constant_add(self, dtype, random, use_gpu): - """check sparsemax-loss proposition 3""" - z = random.uniform(low=-3, high=3, size=(test_obs, 10)) - c = random.uniform(low=-3, high=3, size=(test_obs, 1)) - q = np.zeros((test_obs, 10)) - q[np.arange(0, test_obs), np.random.randint(0, 10, size=test_obs)] = 1 - - _, tf_loss_zpc = self._tf_sparsemax_loss(z + c, q, dtype, use_gpu) - - _, tf_loss_z = self._tf_sparsemax_loss(z, q, dtype, use_gpu) - - self.assertAllCloseAccordingToType( - tf_loss_zpc, - tf_loss_z, - float_atol=5e-6, - float_rtol=5e-6, - half_atol=1e-2, - half_rtol=1e-2) - - def _test_sparsemax_loss_positive(self, dtype, random, use_gpu): - """check sparsemax-loss proposition 4""" - z = random.uniform(low=-3, high=3, size=(test_obs, 10)) - q = np.zeros((test_obs, 10)) - q[np.arange(0, test_obs), random.randint(0, 10, size=test_obs)] = 1 - - tf_loss_op, tf_loss_out = self._tf_sparsemax_loss(z, q, dtype, use_gpu) - - self.assertAllCloseAccordingToType(np.abs(tf_loss_out), tf_loss_out) - self.assertShapeEqual(np.zeros(test_obs), tf_loss_op) - - def _test_sparsemax_loss_zero(self, dtype, random, use_gpu): - """check sparsemax-loss proposition 5""" - # construct z and q, such that z_k >= 1 + max_{j!=k} z_k holds for - # delta_0 = 1. - z = random.uniform(low=-3, high=3, size=(test_obs, 10)) - z[:, 0] = np.max(z, axis=1) + 1.05 - - q = np.zeros((test_obs, 10)) - q[:, 0] = 1 - - tf_loss_op, tf_loss_out = self._tf_sparsemax_loss(z, q, dtype, use_gpu) - tf_sparsemax_op, tf_sparsemax_out = self._tf_sparsemax(z, dtype, use_gpu) - - self.assertAllCloseAccordingToType(np.zeros(test_obs), tf_loss_out) - self.assertShapeEqual(np.zeros(test_obs), tf_loss_op) - - self.assertAllCloseAccordingToType(q, tf_sparsemax_out) - self.assertShapeEqual(q, tf_sparsemax_op) - - def _test_gradient_against_estimate(self, dtype, random, use_gpu): - """check sparsemax-loss Rop, against estimated-loss Rop""" - z = random.uniform(low=-3, high=3, size=(test_obs, 10)).astype(dtype) - q = np.zeros((test_obs, 10)).astype(dtype) - q[np.arange(0, test_obs), np.random.randint(0, 10, size=test_obs)] = 1 - - logits = array_ops.placeholder(dtype, name='z') - sparsemax_op = sparsemax(logits) - loss_op = sparsemax_loss(logits, sparsemax_op, q) - - with self.test_session(use_gpu=use_gpu): - err = gradient_checker.compute_gradient_error( - logits, z.shape, loss_op, (test_obs,), x_init_value=z, delta=1e-9) - - self.assertLess(err, 1e-4) - - def _test_gradient_against_numpy(self, dtype, random, use_gpu): - """check sparsemax-loss Rop, against numpy Rop""" - z = random.uniform(low=-3, high=3, size=(test_obs, 10)) - q = np.zeros((test_obs, 10)) - q[np.arange(0, test_obs), np.random.randint(0, 10, size=test_obs)] = 1 - - logits = constant_op.constant(z.astype(dtype), name='z') - sparsemax_op = sparsemax(logits) - loss_op = sparsemax_loss(logits, sparsemax_op, q.astype(dtype)) - loss_grad_op = gradients_impl.gradients(loss_op, [logits])[0] - - with self.test_session(use_gpu=use_gpu): - tf_grad = loss_grad_op.eval() - np_grad = self._np_sparsemax_loss_grad(z, q).astype(dtype) - - self.assertAllCloseAccordingToType( - np_grad, tf_grad, half_atol=1e-2, half_rtol=5e-3) - self.assertShapeEqual(np_grad, loss_grad_op) - - def _test_dtype(self, dtype): - random = np.random.RandomState(1) - - self._test_sparsemax_loss_against_numpy(dtype, random, use_gpu=False) - - self._test_sparsemax_loss_of_nan(dtype, random, use_gpu=False) - - self._test_sparsemax_loss_of_inf(dtype, random, use_gpu=False) - - self._test_constant_add(dtype, random, use_gpu=False) - - self._test_sparsemax_loss_positive(dtype, random, use_gpu=False) - - self._test_sparsemax_loss_zero(dtype, random, use_gpu=False) - - # sparsemax is not a smooth function so gradient estimation is only - # possibol for float64. - if dtype == 'float64': - self._test_gradient_against_estimate(dtype, random, use_gpu=False) - - self._test_gradient_against_numpy(dtype, random, use_gpu=False) - - def testFloat(self): - self._test_dtype('float32') - - def testDouble(self): - self._test_dtype('float64') - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_test.py b/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_test.py deleted file mode 100644 index c95b9da1e4c..00000000000 --- a/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_test.py +++ /dev/null @@ -1,284 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for SparsemaxOp.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.sparsemax import sparsemax -from tensorflow.python.ops import gradient_checker -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients_impl -from tensorflow.python.framework import constant_op -from tensorflow.python.platform import test - -test_obs = 10 - - -class SparsemaxTest(test.TestCase): - - def _np_sparsemax(self, z): - z = z - np.mean(z, axis=1)[:, np.newaxis] - - # sort z - z_sorted = np.sort(z, axis=1)[:, ::-1] - - # calculate k(z) - z_cumsum = np.cumsum(z_sorted, axis=1) - k = np.arange(1, z.shape[1] + 1) - z_check = 1 + k * z_sorted > z_cumsum - # use argmax to get the index by row as .nonzero() doesn't - # take an axis argument. np.argmax return the first index, but the last - # index is required here, use np.flip to get the last index and - # `z.shape[axis]` to compensate for np.flip afterwards. - k_z = z.shape[1] - np.argmax(z_check[:, ::-1], axis=1) - - # calculate tau(z) - tau_sum = z_cumsum[np.arange(0, z.shape[0]), k_z - 1] - tau_z = ((tau_sum - 1) / k_z).reshape(-1, 1) - - # calculate p - return np.maximum(0, z - tau_z) - - def _np_sparsemax_grad(self, z): - # chain rule - grad = np.ones_like(z) - - # Construct S(z) - probability = self._np_sparsemax(z) - support = probability > 0 - - # Calculate \hat{v}, which will be a vector (scalar for each z) - v_hat = np.sum(grad * support, axis=1) / np.sum(support, axis=1) - - # Calculates J(z) * v - return support * (grad - v_hat[:, np.newaxis]) - - def _tf_sparsemax(self, z, dtype, use_gpu): - with self.test_session(use_gpu=use_gpu): - tf_sparsemax_op = sparsemax(z.astype(dtype)) - tf_sparsemax_out = tf_sparsemax_op.eval() - - return tf_sparsemax_op, tf_sparsemax_out - - def _test_sparsemax_against_numpy(self, dtype, random, use_gpu): - """check sparsemax kernel against numpy""" - z = random.uniform(low=-3, high=3, size=(test_obs, 10)) - - tf_sparsemax_op, tf_sparsemax_out = self._tf_sparsemax(z, dtype, use_gpu) - p_sparemax = self._np_sparsemax(z).astype(dtype) - - self.assertAllCloseAccordingToType( - p_sparemax, tf_sparsemax_out, half_atol=5e-3) - self.assertShapeEqual(p_sparemax, tf_sparsemax_op) - - def _test_sparsemax_of_nan(self, dtype, random, use_gpu): - """check sparsemax transfers nan""" - z_nan = np.asarray([ - [0, np.nan, 0], - [0, np.nan, np.nan], - [np.nan, np.nan, np.nan], - ]).astype(dtype) - - _, tf_sparsemax_nan = self._tf_sparsemax(z_nan, dtype, use_gpu) - self.assertAllCloseAccordingToType( - [[np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan], - [np.nan, np.nan, np.nan]], tf_sparsemax_nan) - - def _test_sparsemax_of_inf(self, dtype, random, use_gpu): - """check sparsemax is infinity safe""" - z_neg = np.asarray([ - [0, -np.inf, 0], - [0, -np.inf, -np.inf], - [-np.inf, -np.inf, -np.inf], - ]).astype(dtype) - z_pos = np.asarray([[0, np.inf, 0], [0, np.inf, np.inf], - [np.inf, np.inf, np.inf]]).astype(dtype) - z_mix = np.asarray([[0, np.inf, 0], [0, np.inf, -np.inf], - [-np.inf, np.inf, -np.inf]]).astype(dtype) - - _, tf_sparsemax_neg = self._tf_sparsemax(z_neg, dtype, use_gpu) - self.assertAllCloseAccordingToType( - [[0.5, 0, 0.5], [1, 0, 0], [np.nan, np.nan, np.nan]], tf_sparsemax_neg) - - _, tf_sparsemax_pos = self._tf_sparsemax(z_pos, dtype, use_gpu) - self.assertAllCloseAccordingToType( - [[np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan], - [np.nan, np.nan, np.nan]], tf_sparsemax_pos) - - _, tf_sparsemax_mix = self._tf_sparsemax(z_mix, dtype, use_gpu) - self.assertAllCloseAccordingToType( - [[np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan], - [np.nan, np.nan, np.nan]], tf_sparsemax_mix) - - - def _test_sparsemax_of_zero(self, dtype, random, use_gpu): - """check sparsemax proposition 1, part 1""" - z = np.zeros((1, 10)) - - tf_sparsemax_op, tf_sparsemax_out = self._tf_sparsemax(z, dtype, use_gpu) - p_sparemax = np.ones_like(z, dtype=dtype) / z.size - - self.assertAllCloseAccordingToType(p_sparemax, tf_sparsemax_out) - self.assertShapeEqual(p_sparemax, tf_sparsemax_op) - - def _test_sparsemax_of_to_inf(self, dtype, random, use_gpu): - """check sparsemax proposition 1, part 2""" - z = random.uniform(low=-3, high=3, size=(test_obs, 10)) - - # assume |A(z)| = 1, as z is continues random - z_sort_arg = np.argsort(z, axis=1)[:, ::-1] - z_sort = np.sort(z, axis=-1)[:, ::-1] - gamma_z = z_sort[:, 0] - z_sort[:, 1] - epsilon = (0.99 * gamma_z * 1).reshape(-1, 1) - - # construct the expected 1_A(z) array - p_expected = np.zeros((test_obs, 10), dtype=dtype) - p_expected[np.arange(0, test_obs), z_sort_arg[:, 0]] = 1 - - tf_sparsemax_op, tf_sparsemax_out = self._tf_sparsemax((1 / epsilon) * z, - dtype, use_gpu) - - self.assertAllCloseAccordingToType(p_expected, tf_sparsemax_out) - self.assertShapeEqual(p_expected, tf_sparsemax_op) - - def _test_constant_add(self, dtype, random, use_gpu): - """check sparsemax proposition 2""" - z = random.uniform(low=-3, high=3, size=(test_obs, 10)).astype(dtype) - c = random.uniform(low=-3, high=3, size=(test_obs, 1)).astype(dtype) - - _, tf_sparsemax_zpc = self._tf_sparsemax(z + c, dtype, use_gpu) - - _, tf_sparsemax_z = self._tf_sparsemax(z, dtype, use_gpu) - - self.assertAllCloseAccordingToType( - tf_sparsemax_zpc, tf_sparsemax_z, half_atol=5e-3) - - def _test_permutation(self, dtype, random, use_gpu): - """check sparsemax proposition 3""" - z = random.uniform(low=-3, high=3, size=(test_obs, 10)) - _, p = self._tf_sparsemax(z, dtype, use_gpu) - - for i in range(test_obs): - per = random.permutation(10) - - tf_sparsemax_op, tf_sparsemax_out = self._tf_sparsemax( - z[i, per].reshape(1, -1), dtype, use_gpu) - p_expected = p[i, per].reshape(1, -1) - - self.assertAllCloseAccordingToType( - p_expected, tf_sparsemax_out, half_atol=5e-3) - self.assertShapeEqual(p_expected, tf_sparsemax_op) - - def _test_diffrence(self, dtype, random, use_gpu): - """check sparsemax proposition 4""" - z = random.uniform(low=-3, high=3, size=(test_obs, 10)) - _, p = self._tf_sparsemax(z, dtype, use_gpu) - - etol = {'float16': 1e-2, 'float32': 1e-6, 'float64': 1e-9}[dtype] - - for val in range(0, test_obs): - for i in range(0, 10): - for j in range(0, 10): - # check condition, the obesite pair will be checked anyway - if z[val, i] > z[val, j]: - continue - - self.assertTrue( - 0 <= p[val, j] - p[val, i] <= z[val, j] - z[val, i] + etol, - '0 <= %.10f <= %.10f' % (p[val, j] - p[val, i], - z[val, j] - z[val, i] + etol)) - - def _test_two_dimentional(self, dtype, random, use_gpu): - """check two dimentation sparsemax case""" - t = np.linspace(-2, 2, test_obs, dtype=dtype) - z = np.vstack([t, np.zeros(test_obs, dtype=dtype)]).T - - tf_sparsemax_op, tf_sparsemax_out = self._tf_sparsemax(z, dtype, use_gpu) - - p0_expected = np.select([t < -1, t <= 1, t > 1], [0, (t + 1) / 2, 1]) - - self.assertAllCloseAccordingToType(p0_expected, tf_sparsemax_out[:, 0]) - self.assertAllCloseAccordingToType(1 - p0_expected, tf_sparsemax_out[:, 1]) - self.assertShapeEqual(z, tf_sparsemax_op) - - def _test_gradient_against_estimate(self, dtype, random, use_gpu): - """check sparsemax Rop, against estimated Rop""" - z = random.uniform(low=-3, high=3, size=(test_obs, 10)).astype(dtype) - - logits = array_ops.placeholder(dtype, name='z') - sparsemax_op = sparsemax(logits) - - with self.test_session(use_gpu=use_gpu): - err = gradient_checker.compute_gradient_error( - logits, z.shape, sparsemax_op, z.shape, x_init_value=z, delta=1e-9) - - self.assertLess(err, 1e-4) - - def _test_gradient_against_numpy(self, dtype, random, use_gpu): - """check sparsemax Rop, against numpy Rop""" - z = random.uniform(low=-3, high=3, size=(test_obs, 10)).astype(dtype) - - logits = constant_op.constant(z, name='z') - sparsemax_op = sparsemax(logits) - sparsemax_grad_op = gradients_impl.gradients(sparsemax_op, [logits])[0] - - with self.test_session(use_gpu=use_gpu): - tf_grad = sparsemax_grad_op.eval() - np_grad = self._np_sparsemax_grad(z) - - self.assertAllCloseAccordingToType(np_grad, tf_grad) - self.assertShapeEqual(np_grad, sparsemax_grad_op) - - def _test_dtype(self, dtype): - random = np.random.RandomState(1) - - self._test_sparsemax_against_numpy(dtype, random, use_gpu=False) - - self._test_sparsemax_of_nan(dtype, random, use_gpu=False) - - self._test_sparsemax_of_inf(dtype, random, use_gpu=False) - - self._test_sparsemax_of_zero(dtype, random, use_gpu=False) - - self._test_sparsemax_of_to_inf(dtype, random, use_gpu=False) - - self._test_constant_add(dtype, random, use_gpu=False) - - self._test_permutation(dtype, random, use_gpu=False) - - self._test_diffrence(dtype, random, use_gpu=False) - - self._test_two_dimentional(dtype, random, use_gpu=False) - - # sparsemax is not a smooth function so gradient estimation is only - # possibol for float64. - if dtype == 'float64': - self._test_gradient_against_estimate(dtype, random, use_gpu=False) - - self._test_gradient_against_numpy(dtype, random, use_gpu=False) - - def testFloat(self): - self._test_dtype('float32') - - def testDouble(self): - self._test_dtype('float64') - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/sparsemax/python/ops/sparsemax.py b/tensorflow/contrib/sparsemax/python/ops/sparsemax.py deleted file mode 100644 index f79c93f3475..00000000000 --- a/tensorflow/contrib/sparsemax/python/ops/sparsemax.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Sparsemax op.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn - -__all__ = ["sparsemax"] - - -def sparsemax(logits, name=None): - """Computes sparsemax activations [1]. - - For each batch `i` and class `j` we have - $$sparsemax[i, j] = max(logits[i, j] - tau(logits[i, :]), 0)$$ - - [1]: https://arxiv.org/abs/1602.02068 - - Args: - logits: A `Tensor`. Must be one of the following types: `half`, `float32`, - `float64`. - name: A name for the operation (optional). - - Returns: - A `Tensor`. Has the same type as `logits`. - """ - - with ops.name_scope(name, "sparsemax", [logits]) as name: - logits = ops.convert_to_tensor(logits, name="logits") - obs = array_ops.shape(logits)[0] - dims = array_ops.shape(logits)[1] - - # In the paper, they call the logits z. - # The mean(logits) can be substracted from logits to make the algorithm - # more numerically stable. the instability in this algorithm comes mostly - # from the z_cumsum. Substacting the mean will cause z_cumsum to be close - # to zero. However, in practise the numerical instability issues are very - # minor and substacting the mean causes extra issues with inf and nan - # input. - z = logits - - # sort z - z_sorted, _ = nn.top_k(z, k=dims) - - # calculate k(z) - z_cumsum = math_ops.cumsum(z_sorted, axis=1) - k = math_ops.range( - 1, math_ops.cast(dims, logits.dtype) + 1, dtype=logits.dtype) - z_check = 1 + k * z_sorted > z_cumsum - # because the z_check vector is always [1,1,...1,0,0,...0] finding the - # (index + 1) of the last `1` is the same as just summing the number of 1. - k_z = math_ops.reduce_sum(math_ops.cast(z_check, dtypes.int32), axis=1) - - # calculate tau(z) - # If there are inf values or all values are -inf, the k_z will be zero, - # this is mathematically invalid and will also cause the gather_nd to fail. - # Prevent this issue for now by setting k_z = 1 if k_z = 0, this is then - # fixed later (see p_safe) by returning p = nan. This results in the same - # behavior as softmax. - k_z_safe = math_ops.maximum(k_z, 1) - indices = array_ops.stack([math_ops.range(0, obs), k_z_safe - 1], axis=1) - tau_sum = array_ops.gather_nd(z_cumsum, indices) - tau_z = (tau_sum - 1) / math_ops.cast(k_z, logits.dtype) - - # calculate p - p = math_ops.maximum( - math_ops.cast(0, logits.dtype), z - tau_z[:, array_ops.newaxis]) - # If k_z = 0 or if z = nan, then the input is invalid - p_safe = array_ops.where( - math_ops.logical_or( - math_ops.equal(k_z, 0), math_ops.is_nan(z_cumsum[:, -1])), - array_ops.fill([obs, dims], math_ops.cast(float("nan"), logits.dtype)), - p) - - return p_safe diff --git a/tensorflow/contrib/sparsemax/python/ops/sparsemax_loss.py b/tensorflow/contrib/sparsemax/python/ops/sparsemax_loss.py deleted file mode 100644 index c0438f16bc8..00000000000 --- a/tensorflow/contrib/sparsemax/python/ops/sparsemax_loss.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Sparsemax Loss op.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops - -__all__ = ["sparsemax_loss"] - - -def sparsemax_loss(logits, sparsemax, labels, name=None): - """Computes sparsemax loss function [1]. - - [1]: https://arxiv.org/abs/1602.02068 - - Args: - logits: A `Tensor`. Must be one of the following types: `half`, `float32`, - `float64`. - sparsemax: A `Tensor`. Must have the same type as `logits`. - labels: A `Tensor`. Must have the same type as `logits`. - name: A name for the operation (optional). - - Returns: - A `Tensor`. Has the same type as `logits`. - """ - - with ops.name_scope(name, "sparsemax_loss", - [logits, sparsemax, labels]) as name: - logits = ops.convert_to_tensor(logits, name="logits") - sparsemax = ops.convert_to_tensor(sparsemax, name="sparsemax") - labels = ops.convert_to_tensor(labels, name="labels") - - # In the paper, they call the logits z. - # A constant can be substracted from logits to make the algorithm - # more numerically stable in theory. However, there are really no major - # source numerical instability in this algorithm. - z = logits - - # sum over support - # Use a conditional where instead of a multiplication to support z = -inf. - # If z = -inf, and there is no support (sparsemax = 0), a multiplication - # would cause 0 * -inf = nan, which is not correct in this case. - sum_s = array_ops.where( - math_ops.logical_or(sparsemax > 0, math_ops.is_nan(sparsemax)), - sparsemax * (z - 0.5 * sparsemax), array_ops.zeros_like(sparsemax)) - - # - z_k + ||q||^2 - q_part = labels * (0.5 * labels - z) - # Fix the case where labels = 0 and z = -inf, where q_part would - # otherwise be 0 * -inf = nan. But since the lables = 0, no cost for - # z = -inf should be consideredself. - # The code below also coveres the case where z = inf. Howeverm in this - # caose the sparsemax will be nan, which means the sum_s will also be nan, - # therefor this case doesn't need addtional special treatment. - q_part_safe = array_ops.where( - math_ops.logical_and(math_ops.equal(labels, 0), math_ops.is_inf(z)), - array_ops.zeros_like(z), q_part) - - return math_ops.reduce_sum(sum_s + q_part_safe, axis=1) diff --git a/tensorflow/contrib/specs/BUILD b/tensorflow/contrib/specs/BUILD deleted file mode 100644 index a2d5a44edf2..00000000000 --- a/tensorflow/contrib/specs/BUILD +++ /dev/null @@ -1,63 +0,0 @@ -# Description: -# A small domain-specific language (DSL) for defining deep learning networks. - -load("//tensorflow:tensorflow.bzl", "tf_py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "specs", - srcs = [ - "__init__.py", - "python/__init__.py", - "python/params_ops.py", - "python/specs.py", - "python/specs_lib.py", - "python/specs_ops.py", - "python/summaries.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:logging_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:nn_ops", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "@six_archive//:six", - ], -) - -tf_py_test( - name = "specs_test", - srcs = ["python/specs_test.py"], - additional_deps = [ - ":specs", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:variables", - ], -) - -tf_py_test( - name = "summaries_test", - srcs = ["python/summaries_test.py"], - additional_deps = [ - ":specs", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:variables", - ], -) diff --git a/tensorflow/contrib/specs/README.md b/tensorflow/contrib/specs/README.md deleted file mode 100644 index bcf34e601f1..00000000000 --- a/tensorflow/contrib/specs/README.md +++ /dev/null @@ -1,233 +0,0 @@ -# specs -- simple specifications for TensorFlow networks - -This library implements a simple domain-specific language for specifying -deep neural networks in TensorFlow. - -From a high level, there are a set of standard operators and ways of -combining them: - - - operator `|` takes the output from one layer and "pipes" it into the next - - operator `**` repeats a layer multiple times - -Naming conventions: - - - single character names are reserved to users - - built-in layers are capitalized, not CamelCase (Relu, Fs, etc.) - - built-in layers that are common are usually two letters (Cr, Fs, etc.) - - less common operations are longer (Relu, Conc, etc.) - - temporary names should end in _ - -Common layers: - -Common layers are defined by short, capitalized abbreviations. For layers -that take an activation function (fully_connected, conv2d), the acronym -is a conjunction of a base layer and the activation. For example, `Fs` -represents a fully connected layer followed by a sigmoid, whereas `Ft` -represents a fully connected layer followed by a Tanh. - - - `Fx` = tf.contrib.layers.fully_connected; x = activation function, one of s/t/r/l/m - - `Cx` = tf.contrib.layers.conv2d; x = activation function, one of s/t/r/l/m - - `Mp` = tf.contrib.layers.max_pool2d - - `Ap` = tf.contrib.layers.avg_pool2d - - `Bn` = tf.contrib.layers.batch_norm - -Nonlinearities (suffixes for C/F, so Cs = convolutional layer + sigmoid): - - - `s` = sigmoid - - `t` = tanh - - `r` = relu - - `l` = linear (i.e., None) - - `m` = softmax - -Positional and keyword arguments are the same as for the underlying -slim and TensorFlow functions. Therefore, common usage patterns are: - - Cr(64, [5, 5]) # conv2d with a 5x5 footprint and 64 outputs - Mp([2, 2]) # max pooling using [2, 2] steps - -Explicit nonlinearities: - - - `Relu` = tf.nn.relu - - `Sig` = tf.nn.sigmoid - - `Tanh` = tf.nn.tanh - - `Smax` = tf.nn.softmax - -Reshaping: - - - `Flat` = slim.flatten - - `Reshape` = tf.reshape - - `Squeeze` = tf.squeeze - - `Expand` = tf.expand_dims - -Other: - - - `Id` = identity - - `Do` = tf.contrib.layers.dropout - - `Lrn` = tf.nn.local_response_normalization - - `Unit` = tf.contrib.layers.unit_norm - - `Conc` is roughly tf.nn.concat - -Binding external functions: - - - `External` - import an external function using module path - - `Import` - import an external function using statements - -A network specification is a sequence of `name = expression` Python statements, -with the `net` variable holding the network that is being defined. That is, -your specification must have a statement of the form `net = ...` as its -last statement. - -So, a simple MNIST network might look like: - - net = Cr(64, [5, 5]) | Fs(10) - -More complicated: - - net = (Cr(64, [5, 5]) | Mp([2, 2])) ** 3 | Fs(10) - -With temporary names: - - cmp_ = Cr(64, [5, 5]) | Mp([2, 2]) - net = cmp_ ** 3 | Fs(10) - -(You can also separate statements with `;` instead of `\n`) - -General model structure: - - - Models are sequences of `name = expression` statements - in Python syntax. - - Other kinds of statements are not allowed (with a few - exceptions, like calling `debug()`) - - Names should be assigned only once. - -These constraints are only partially enforced by the library right -now, but may be strictly enforced in the future. - -# More Details - -The spec language is intended for rapid experimentation with common -layer types; it's not a replacement for the standard TensorFlow or -slim APIs. If you have some complex layer type or construct that's -difficult to represent in `spec`, you can implement it directly in -Python and then easily make it available as a `spec` operator. - -Since partial application with positional arguments can be a little -confusing, you can also specify positional arguments with keywords like -`_1`: - - cr5_ = Cr(_1=[5, 5]); net = cr5_(64) ** 3 | Fs(10) - -You can enable debugging by putting `debug()` at the beginning of your network -definition: - - debug(); net = Cr(64, [5, 5]) | Fs(10) - -The module is a "combinator library". To make the syntax work nicely -with Python, the `__call__` operator is overloaded to perform partial -application. - -To create a network from Python, you just call the following: - - inputs = tf.placeholder(...) - spec = "net = (Cr(64, [5, 5]) | Mp([2, 2])) ** 3 | Fs(10)" - outputs = specs.create_net(spec, inputs) - -You can pass variable bindings into `create_net`: - - inputs = tf.placeholder(...) - spec = "net = (Cr(64, [5, 5]) | Mp([2, 2])) ** depth | Fs(10)" - outputs = specs.create_net(spec, inputs, dict(depth=3)) - -# Using `specs` in Code - -The specs operators are defined in the module `specs_ops`. To facilitate -using the `specs` DSL in your code without namespace pollution, you can -use the `specs.ops` context manager, which will temporarily make the -`specs` operators available in your code: - - import tensorflow as tf - import numpy.random as npr - specs = tf.contrib.specs.python - - with specs.ops: - net = (Cr(64, [2, 2]) | Mp([2, 2])) ** 3 | Flat | Fs(10) - inputs = tf.placeholder(tf.float32, [None, 28, 28, 1]) - outputs = net.funcall(inputs) - - sess = tf.InteractiveSession() - tf.global_variables_initializer().run() - sess.run([outputs], feed_dict={inputs: npr.uniform(size=(17, 28, 28, 1))}) - -# Sharing and Variables - -You can share variables among subnets by wrapping them with `Shared`: - - f = Shared(Fr(100)) - g = f | f | f | f - -This will stack four fully connected ReLU layers, sharing the same -weights and biases. - -You can also create variables explicitly: - - v = Var("v") - -You can use this to write expressions like this: - - net = Cl(100, 3) + Var("b", shape=[128, 128, 100])) - -Note that, under the covers, both the `Cl` operator and the `Var` operator -generate functions that are eventually applied via `funcall` to an input -tensor; the function generated by the `Var` operator ignores its argument -and calls `tf.get_variable` with the supplied arguments. - -# Pulling in New Primitives - -If you need some special function in your spec language, you can make -it available using `External` or `Import`. The following two statements -are equivalent: - - Sig = External("some_module", "some_op") - Sig = Import("import tensorflow as tf; f = tf.nn.sigmoid") - -You probably will want to use `Import` because TensorFlow contains a -number of imports that look like they are in modules, but they are -actually just values placed in the namespace somehow. The `Import` -function takes an arbitrary Python statement that eventually needs to -assign a value to the variable `f` that is then wrapped up as a function. - -# Summaries - -There are a number of functions that give you information about the structure -of specs (and other, similarly structured, TensorFlow graphs); the first -number is the number of parameters, followed by the op, and the shape. - - >>> summaries.tf_spec_summary("net = Cr(100, [3,3]) | Flat | Fs(10)", - input_shape=(17, 28, 28, 1)) - 0 Placeholder [17, 28, 28, 1] - 1000 Conv [17, 28, 28, 100] - 0 Flatten [17, 78400] - 784010 fully_connected [17, 10] - >>> - -# ToDo - -More documentation, comments. - -The following features are intended to be added soon (names subject to change): - - - add sequence processing layers - - add named point cuts - - Seq(a, b, c).add(name=layer).add(name=layer) for explicit seq. structures - - S2d, D2s (space/depth operators) - - `Shared(...)` -- variable sharing - - `Mix(...)` -- weighted convex combination of layer outputs - - `Lincom(...)` -- weighted linear combination of layer outputs - - `SameDepth(A)` -- makes output depth same as input - - summary ops - - slim's `arg_scope` - - automatic wrapping of long-name slim layers - - depth-to-space, etc. - -Eventually, there may be a similar spec language for -input layers and pipelines. diff --git a/tensorflow/contrib/specs/__init__.py b/tensorflow/contrib/specs/__init__.py deleted file mode 100644 index 52e83069cb0..00000000000 --- a/tensorflow/contrib/specs/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/specs/python/__init__.py b/tensorflow/contrib/specs/python/__init__.py deleted file mode 100644 index b6cc7540238..00000000000 --- a/tensorflow/contrib/specs/python/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Init file, giving convenient access to all specs ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,g-importing-member,redefined-builtin -from tensorflow.contrib.specs.python.params_ops import * -from tensorflow.contrib.specs.python.specs import * -from tensorflow.contrib.specs.python.specs_lib import * -from tensorflow.contrib.specs.python.specs_ops import * -from tensorflow.contrib.specs.python.summaries import * -# pylint: enable=wildcard-import,redefined-builtin diff --git a/tensorflow/contrib/specs/python/params_ops.py b/tensorflow/contrib/specs/python/params_ops.py deleted file mode 100644 index 7cabdfd9b85..00000000000 --- a/tensorflow/contrib/specs/python/params_ops.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Operators for concise TensorFlow parameter specifications. - -This module is used as an environment for evaluating expressions -in the "params" DSL. - -Specifications are intended to assign simple numerical -values. Examples: - - --params "n=64; d=5" --spec "(Cr(n) | Mp([2, 2])) ** d | Fm" - -The random parameter primitives are useful for running large numbers -of experiments with randomly distributed parameters: - - --params "n=Li(5,500); d=Ui(1,5)" --spec "(Cr(n) | Mp([2, 2])) ** d | Fm" - -Internally, this might be implemented as follows: - - params = specs.create_params(FLAGS.params, {}) - logging.info(repr(params)) - net = specs.create_net(FLAGS.spec, inputs, params) - -Note that separating the specifications into parameters and network -creation allows us to log the random parameter values easily. - -The implementation of this will change soon in order to support -hyperparameter tuning with steering. Instead of returning a number, -the primitives below will return a class instance that is then -used to generate a random number by the framework. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# Lint disabled because these are operators in the DSL, not regular -# Python functions. -# pylint: disable=invalid-name -# pylint: disable=wildcard-import,unused-wildcard-import,redefining-builtin -# pylint: disable=redefined-builtin,g-importing-member,no-member -# make available all math expressions -import math -from math import * -import random -# pylint: enable=wildcard-import,unused-wildcard-import,redefining-builtin -# pylint: enable=redefined-builtin,g-importing-member,no-member - - -def Uf(lo=0.0, hi=1.0): - """Uniformly distributed floating number.""" - return random.uniform(lo, hi) - - -def Ui(lo, hi): - """Uniformly distributed integer, inclusive limits.""" - return random.randint(lo, hi) - - -def Lf(lo, hi): - """Log-uniform distributed floatint point number.""" - return math.exp(random.uniform(math.log(lo), math.log(hi))) - - -def Li(lo, hi): - """Log-uniform distributed integer, inclusive limits.""" - return int(math.floor(math.exp(random.uniform(math.log(lo), - math.log(hi+1-1e-5))))) - - -def Nt(mu, sigma, limit=3.0): - """Normally distributed floating point number with truncation.""" - return min(max(random.gauss(mu, sigma), mu-limit*sigma), mu+limit*sigma) - - -# pylint: enable=invalid-name diff --git a/tensorflow/contrib/specs/python/specs.py b/tensorflow/contrib/specs/python/specs.py deleted file mode 100644 index d5223b9b551..00000000000 --- a/tensorflow/contrib/specs/python/specs.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Builder for TensorFlow models specified using specs_ops. - -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from six import exec_ -from tensorflow.contrib.specs.python import params_ops -from tensorflow.contrib.specs.python import specs_lib -from tensorflow.contrib.specs.python import specs_ops -from tensorflow.python.util import tf_inspect - - -def eval_params(params, environment=None): - """Evaluates a parameter specification and returns the environment. - - Args: - params: parameter assignments as a string - environment: a dictionary of input bindings - - Returns: - Environment with additional bindings created by - executing `params` - - Raises: - Exception: other exceptions raised during execution of `params` - """ - specs_lib.check_keywords(params) - bindings = {} - if environment: - bindings.update(environment) - exec_(params, vars(params_ops), bindings) # pylint: disable=exec-used - return bindings - - -def eval_spec(spec, environment=None): - """Evaluates a spec and returns the environment. - - This function allows you to use a spec to obtain multiple bindings - in an environment. That is useful if you use the spec language to - specify multiple components of a larger network, for example: "left - = Cr(64, [5,5]); right = Fc(64)" Usually, you will want to use - `create_net` or `create_net_fun` below. - - Args: - spec: specification as a string - environment: a dictionary of input bindings - - Returns: - Environment with additional bindings created by spec. - - Raises: - Exception: other exceptions raised during execution of `spec` - - """ - specs_lib.check_keywords(spec) - bindings = {} - if environment: - bindings.update(environment) - exec_(spec, vars(specs_ops), bindings) # pylint: disable=exec-used - return bindings - - -def create_net_fun(spec, environment=None): - """Evaluates a spec and returns the binding of `net`. - - Specs are written in a DSL based on function composition. A spec - like `net = Cr(64, [3, 3])` assigns an object that represents a - single argument function capable of creating a network to - the variable `net`. - - Args: - spec: specification as a string, ending with a `net = ...` statement - environment: a dictionary of input bindings - - Returns: - A callable that instantiates the `net` binding. - - Raises: - ValueError: spec failed to create a `net` - Exception: other exceptions raised during execution of `spec` - - """ - bindings = eval_spec(spec, environment) - net = bindings.get("net", None) - if net is None: - raise ValueError("spec failed to create 'net': %s" % (spec,)) - return net.funcall - - -def create_net(spec, inputs, environment=None): - """Evaluates a spec and creates a network instance given the inputs. - - Args: - spec: specification as a string, ending with a `net = ...` statement - inputs: input that `net` is applied to - environment: a dictionary of input bindings - - Returns: - A callable that instantiates the `net` binding. - - Raises: - ValueError: spec failed to create a `net` - Exception: other exceptions raised during execution of `spec` - """ - return create_net_fun(spec, environment)(inputs) - - -class LocalImport(object): - """A class that allows us to temporarily import something. - - Attributes: - frame: the frame in which the context manager was invocked - names: a dictionary containing the new bindings - old: variable bindings that have been shadowed by the import - """ - - def __init__(self, names): - """Create a context manager that binds the names in values. - - Args: - names: A dictionary or module containing the bindings. - """ - if not isinstance(names, dict): - names = vars(names) - self.names = names - - def __enter__(self): - self.frame = tf_inspect.currentframe() - bindings = self.frame.f_back.f_globals - self.old = {k: bindings.get(k, None) for k in self.names.keys()} - bindings.update(self.names) - - def __exit__(self, some_type, value, traceback): - del some_type, value, traceback - bindings = self.frame.f_back.f_globals - bindings.update(self.old) - for k, v in self.old.items(): - if v is None: - del bindings[k] - del self.frame - - -ops = LocalImport(specs_ops) diff --git a/tensorflow/contrib/specs/python/specs_lib.py b/tensorflow/contrib/specs/python/specs_lib.py deleted file mode 100644 index 12663c27882..00000000000 --- a/tensorflow/contrib/specs/python/specs_lib.py +++ /dev/null @@ -1,238 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implement the "specs" DSL for describing deep networks.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -import importlib -import operator -import re - -from six import exec_ - -QUOTED = re.compile(r""" -"([^"\\]|\\.)*" | -'([^'\\]|\\.)*' -""", re.VERBOSE) -KEYWORDS = re.compile(r"""\b(import|while|def|exec)\b""") - - -debug_ = False - - -def check_keywords(spec): - """Check for common Python keywords in spec. - - This function discourages the use of complex constructs - in TensorFlow specs; it doesn't completely prohibit them - (if necessary, we could check the AST). - - Args: - spec: spec string - - Raises: - ValueError: raised if spec contains a prohibited keyword. - """ - spec = re.sub(QUOTED, "", spec) - match = re.search(KEYWORDS, spec) - if match: - raise ValueError("keyword '%s' found in spec" % match.group(1)) - - -def get_positional(args, kw, kw_overrides=False): - """Interpolates keyword arguments into argument lists. - - If `kw` contains keywords of the form "_0", "_1", etc., these - are positionally interpolated into the argument list. - - Args: - args: argument list - kw: keyword dictionary - kw_overrides: key/value pairs that override kw - - Returns: - (new_args, new_kw), new argument lists and keyword dictionaries - with values interpolated. - """ - new_kw = {k: v for k, v in kw.items() if k[0] != "_"} - if len(new_kw) == len(kw): - return args, kw - new_args = list(args) - for key, value in kw.items(): - if key[0] != "_": continue - index = int(key[1:]) - while len(new_args) <= index: - new_args += [None] - if kw_overrides or new_args[index] is None: - new_args[index] = value - return new_args, new_kw - - -class Composable(object): - """A composable function. - - This defines the operators common to all composable objects. - Currently defines copmosition (via "|") and repeated application - (via "**"), and maps addition ("+") and multiplication ("*") - as "(f + g)(x) = f(x) + g(x)". - """ - - def __or__(self, f): - return Composition(self, f) - - def __add__(self, g): - return Operator(operator.add, self, g) - - def __mul__(self, g): - return Operator(operator.mul, self, g) - - def __pow__(self, n): - assert n >= 0 - if n == 0: - return Function(lambda x, *args, **kw: x) - result = self - for _ in range(n-1): - result = Composition(result, self) - return result - - -class Callable(Composable): - """A composable function that simply defers to a callable function. - """ - - def __init__(self, f): - self.f = f - - def funcall(self, x): - return self.f(x) - - -class Operator(Composable): - """A wrapper for an operator. - - This takes an operator and an argument list and returns - the result of applying the operator to the results of applying - the functions in the argument list. - """ - - def __init__(self, op, *args): - self.op = op - self.funs = args - - def funcall(self, x): - outputs = [f.funcall(x) for f in self.funs] - return self.op(*outputs) - - -class Function(Composable): - """A composable function wrapper for a regular Python function. - - This overloads the regular __call__ operator for currying, i.e., - arguments passed to __call__ are remembered for the eventual - function application. - - The final function application happens via the `of` method. - """ - - def __init__(self, f, *args, **kw): - if not callable(f): - raise ValueError("%s: is not callable" % f) - self.f = f - self.args = list(args) - self.kw = kw - - def __call__(self, *args, **kw): - new_args = list(args) + self.args - new_kw = self.kw.copy() - new_kw.update(kw) - return Function(self.f, *new_args, **new_kw) - - # TODO(tmb) The `of` method may be renamed to `function`. - def funcall(self, x): - args, kw = get_positional(self.args, self.kw) - if debug_: - print("DEBUG:", self.f, x, args, kw) - return self.f(x, *args, **kw) - - -class Composition(Composable): - """A function composition. - - This simply composes its two argument functions when - applied to a final argument via `of`. - """ - - def __init__(self, f, g): - self.f = f - self.g = g - - def funcall(self, x): - return self.g.funcall(self.f.funcall(x)) - - -# These are DSL names, not Python names -# pylint: disable=invalid-name, exec-used -def External(module_name, function_name): - """Import a function from an external module. - - Note that the `module_name` must be a module name - that works with the usual import mechanisms. Shorthands - like "tf.nn" will not work. - - Args: - module_name: name of the module - function_name: name of the function within the module - - Returns: - Function-wrapped value of symbol. - """ - module = importlib.import_module(module_name) - return Function(vars(module)[function_name]) - - -def Import(statements): - """Import a function by exec. - - Args: - statements: Python statements - - Returns: - Function-wrapped value of `f`. - - Raises: - ValueError: the statements didn't define a value for "f" - """ - environ = {} - exec_(statements, environ) - if "f" not in environ: - raise ValueError("failed to define \"f\": %s", statements) - f = environ["f"] - return Function(f) - - -# pylint: enable=invalid-name, exec-used -def debug(mode=True): - """Turn on/off debugging mode. - - Debugging mode prints more information about the construction - of a network. - - Args: - mode: True if turned on, False otherwise - """ - global debug_ - debug_ = mode diff --git a/tensorflow/contrib/specs/python/specs_ops.py b/tensorflow/contrib/specs/python/specs_ops.py deleted file mode 100644 index 49b989b8d0f..00000000000 --- a/tensorflow/contrib/specs/python/specs_ops.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Operators for concise TensorFlow network models. - -This module is used as an environment for evaluating expressions -in the "specs" DSL. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.specs.python import specs_lib -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope - -# The following assignments don't appear to follow Google naming -# conventions, but that's because these are functions defined by -# higher-order function application, not "constants" and because they -# are the commands of the DSL. -# pylint: disable=invalid-name - - -class Idx(specs_lib.Composable): - """Implements the identity function in network specifications.""" - - def funcall(self, x): - return x - - -class Conc(specs_lib.Composable): - """Implements tensor concatenation in network specifications.""" - - def __init__(self, dim, *args): - """Concatenates tensors along the given dimension. - - Args: - dim: dimension along which concatenation takes place - *args: argument tensor functions to be concatenated - """ - self.dim = dim - self.funs = args - - def funcall(self, x): - outputs = [f.funcall(x) for f in self.funs] - return array_ops.concat(outputs, self.dim) - - -External = specs_lib.External -Import = specs_lib.Import -Fun = specs_lib.Function -debug = specs_lib.debug -Print = Fun(logging_ops.Print) -Id = Fun(array_ops.identity) - -# TODO(tmb) add Assert - -# Two letter names for the most common layers. - -# 2D Convolutional layers with nonlinearities (s/t/r/m/l) -# TODO(tmb) add Cbs, Fbs etc. for batch norms - -Cx = Fun(layers.conv2d) -Cs = Fun(layers.conv2d, activation_fn=math_ops.sigmoid) -Ct = Fun(layers.conv2d, activation_fn=math_ops.tanh) -Cr = Fun(layers.conv2d, activation_fn=nn_ops.relu) -Cm = Fun(layers.conv2d, activation_fn=nn_ops.softmax) -Cl = Fun(layers.conv2d, activation_fn=None) - -# Fully connected slim with nonlinearities (s/t/r/m/l) - -Fx = Fun(layers.fully_connected) -Fs = Fun(layers.fully_connected, activation_fn=math_ops.sigmoid) -Ft = Fun(layers.fully_connected, activation_fn=math_ops.tanh) -Fr = Fun(layers.fully_connected, activation_fn=nn_ops.relu) -Fm = Fun(layers.fully_connected, activation_fn=nn_ops.softmax) -Fl = Fun(layers.fully_connected, activation_fn=None) - -# Pooling - -Mp = Fun(layers.max_pool2d) -Ap = Fun(layers.avg_pool2d) - -# Batch manipulations - -Do = Fun(layers.dropout) -Bn = Fun(layers.batch_norm) -Lrn = Fun(nn.local_response_normalization) -Unit = Fun(layers.unit_norm) - -# Shape changes - -Flat = Fun(layers.flatten) -Reshape = Fun(array_ops.reshape) -Transpose = Fun(array_ops.transpose) -Squeeze = Fun(array_ops.squeeze) -Expand = Fun(array_ops.expand_dims) - -# Nonlinearities (rarely needed on their own) - -Relu = Fun(nn_ops.relu) -Sig = Fun(math_ops.sigmoid) -Tanh = Fun(math_ops.tanh) -Smax = Fun(nn_ops.softmax) - - -def Dws(n): - """Depth-wise convolution + sigmoid (used after LSTM).""" - return Cs(n, [1, 1]) - - -def Dwm(n): - """Depth-wise convolution + softmax (used after LSTM).""" - return Cm(n, [1, 1]) - -# Sharing of Variables - - -def Var(name, *args, **kw): - """Implements an operator that generates a variable. - - This function is still experimental. Use it only - for generating a single variable instance for - each name. - - Args: - name: Name of the variable. - *args: Other arguments to get_variable. - **kw: Other keywords for get_variable. - - Returns: - A specs object for generating a variable. - """ - - def var(_): - return variable_scope.get_variable(name, *args, **kw) - - return specs_lib.Callable(var) - - -class Shared(specs_lib.Composable): - """Wraps a scope with variable reuse around the subnetwork. - - This function is still experimental. - - Attributes: - f: The shared subnetwork. - name: A name for the shared scope. - used: A flag indicating whether the scope has already been used. - """ - - shared_number = 1 - - def __init__(self, subnet, name=None, scope=None): - """Create the Shared operator. - - Use this as: - - f = Shared(Cr(100, 3)) - g = f | f | f - - Ordinarily, you do not need to provide either a name or a scope. - Providing a name is useful if you want a well-defined namespace - for the variables (e.g., for saving a subnet). - - Args: - subnet: Definition of the shared network. - name: Optional name for the shared context. - scope: Optional shared scope (must be a Scope, not a string). - - Raises: - ValueError: Scope is not of type tf.Scope, name is not - of type string, or both scope and name are given together. - """ - if scope is not None and not isinstance(scope, - variable_scope.VariableScope): - raise ValueError("scope must be None or a VariableScope") - if name is not None and not isinstance(scope, str): - raise ValueError("name must be None or a string") - if scope is not None and name is not None: - raise ValueError("cannot provide both a name and a scope") - if name is None: - name = "Shared_%d" % Shared.shared_number - Shared.shared_number += 1 - self.subnet = subnet - self.name = name - self.scope = scope - - def funcall(self, x): - """Apply the shared operator to an input. - - This wraps a variable scope around the creation of the subnet. - - Args: - x: The input argument on which the subnet is invoked. - - Returns: - The output tensor from invoking the subnet constructor. - """ - if self.scope is None: - with variable_scope.variable_scope(self.name, values=[x]) as scope: - self.scope = scope - return self.subnet.funcall(x) - else: - with variable_scope.variable_scope(self.scope, values=[x], reuse=True): - return self.subnet.funcall(x) diff --git a/tensorflow/contrib/specs/python/specs_test.py b/tensorflow/contrib/specs/python/specs_test.py deleted file mode 100644 index 9ff038673d9..00000000000 --- a/tensorflow/contrib/specs/python/specs_test.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Testing specs specifications.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.specs import python -from tensorflow.contrib.specs.python import summaries -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import variables -import tensorflow.python.ops.math_ops # pylint: disable=unused-import -from tensorflow.python.platform import test - -specs = python - - -def _rand(*size): - return np.random.uniform(size=size).astype("f") - - -class SpecsTest(test.TestCase): - - def testSimpleConv(self): - with self.cached_session(): - inputs = constant_op.constant(_rand(1, 18, 19, 5)) - spec = "net = Cr(64, [5, 5])" - outputs = specs.create_net(spec, inputs) - self.assertEqual(outputs.get_shape().as_list(), [1, 18, 19, 64]) - variables.global_variables_initializer().run() - result = outputs.eval() - self.assertEqual(tuple(result.shape), (1, 18, 19, 64)) - self.assertEqual( - summaries.tf_spec_structure(spec, inputs), - "_ variablev2 conv variablev2 biasadd relu") - - def testUnary(self): - # This is just a quick and dirty check that these ops exist - # and work as unary ops. - with self.cached_session(): - inputs = constant_op.constant(_rand(17, 55)) - spec = "net = Do(0.5) | Bn | Unit(1) | Relu | Sig | Tanh | Smax" - outputs = specs.create_net(spec, inputs) - self.assertEqual(outputs.get_shape().as_list(), [17, 55]) - variables.global_variables_initializer().run() - result = outputs.eval() - self.assertEqual(tuple(result.shape), (17, 55)) - - def testAdd(self): - with self.cached_session(): - inputs = constant_op.constant(_rand(17, 55)) - spec = "net = Fs(10) + Fr(10)" - outputs = specs.create_net(spec, inputs) - self.assertEqual(outputs.get_shape().as_list(), [17, 10]) - variables.global_variables_initializer().run() - result = outputs.eval() - self.assertEqual(tuple(result.shape), (17, 10)) - self.assertRegexpMatches( - summaries.tf_spec_structure(spec, inputs), - "_ variablev2 dot variablev2 biasadd sig " - "<> variablev2 dot variablev2 biasadd relu add(v2)?") - - def testMpPower(self): - with self.cached_session(): - inputs = constant_op.constant(_rand(1, 64, 64, 5)) - spec = "M2 = Mp([2, 2]); net = M2**3" - outputs = specs.create_net(spec, inputs) - self.assertEqual(outputs.get_shape().as_list(), [1, 8, 8, 5]) - variables.global_variables_initializer().run() - result = outputs.eval() - self.assertEqual(tuple(result.shape), (1, 8, 8, 5)) - self.assertEqual( - summaries.tf_spec_structure(spec, inputs), - "_ maxpool maxpool maxpool") - - def testAbbrevPower(self): - with self.cached_session(): - inputs = constant_op.constant(_rand(1, 64, 64, 5)) - spec = "C3 = Cr([3, 3]); M2 = Mp([2, 2]); net = (C3(5) | M2)**3" - outputs = specs.create_net(spec, inputs) - self.assertEqual(outputs.get_shape().as_list(), [1, 8, 8, 5]) - variables.global_variables_initializer().run() - result = outputs.eval() - self.assertEqual(tuple(result.shape), (1, 8, 8, 5)) - self.assertEqual( - summaries.tf_spec_structure(spec, inputs), - "_ variablev2 conv variablev2 biasadd relu maxpool" - " variablev2 conv variablev2" - " biasadd relu maxpool variablev2 conv variablev2" - " biasadd relu maxpool") - - def testAbbrevPower2(self): - with self.cached_session(): - inputs = constant_op.constant(_rand(1, 64, 64, 5)) - spec = "C3 = Cr(_1=[3, 3]); M2 = Mp([2, 2]);" - spec += "net = (C3(_0=5) | M2)**3" - outputs = specs.create_net(spec, inputs) - self.assertEqual(outputs.get_shape().as_list(), [1, 8, 8, 5]) - variables.global_variables_initializer().run() - result = outputs.eval() - self.assertEqual(tuple(result.shape), (1, 8, 8, 5)) - self.assertEqual( - summaries.tf_spec_structure(spec, inputs), - "_ variablev2 conv variablev2 biasadd relu maxpool" - " variablev2 conv variablev2 biasadd relu" - " maxpool variablev2 conv variablev2 biasadd relu" - " maxpool") - - def testConc(self): - with self.cached_session(): - inputs = constant_op.constant(_rand(10, 20)) - spec = "net = Conc(1, Fs(20), Fs(10))" - outputs = specs.create_net(spec, inputs) - self.assertEqual(outputs.get_shape().as_list(), [10, 30]) - variables.global_variables_initializer().run() - result = outputs.eval() - self.assertEqual(tuple(result.shape), (10, 30)) - self.assertEqual( - summaries.tf_spec_structure(spec, inputs), - "_ variablev2 dot variablev2 biasadd sig " - "<> variablev2 dot variablev2 biasadd sig _ concatv2") - - def testImport(self): - with self.cached_session(): - inputs = constant_op.constant(_rand(10, 20)) - spec = ("S = Import('from tensorflow.python.ops" + - " import math_ops; f = math_ops.sigmoid')") - spec += "; net = S | S" - outputs = specs.create_net(spec, inputs) - self.assertEqual(outputs.get_shape().as_list(), [10, 20]) - variables.global_variables_initializer().run() - result = outputs.eval() - self.assertEqual(tuple(result.shape), (10, 20)) - self.assertEqual(summaries.tf_spec_structure(spec, inputs), "_ sig sig") - - def testKeywordRestriction(self): - with self.cached_session(): - inputs = constant_op.constant(_rand(10, 20)) - spec = "import re; net = Conc(1, Fs(20), Fs(10))" - self.assertRaises(ValueError, lambda: specs.create_net(spec, inputs)) - - def testParams(self): - params = "x = 3; y = Ui(-10, 10); z = Lf(1, 100); q = Nt(0.0, 1.0)" - bindings = specs.eval_params(params, {}) - self.assertTrue("x" in bindings) - self.assertEqual(bindings["x"], 3) - self.assertTrue("y" in bindings) - self.assertTrue("z" in bindings) - self.assertTrue("q" in bindings) - - # XXX: the cleverness of this code is over 9000 - # TODO: original author please fix - def DISABLED_testSpecsOps(self): - # pylint: disable=undefined-variable - with self.assertRaises(NameError): - _ = Cr - with specs.ops: - self.assertIsNotNone(Cr) - self.assertTrue(callable(Cr(64, [3, 3]))) - with self.assertRaises(NameError): - _ = Cr - - # XXX: the cleverness of this code is over 9000 - # TODO: original author please fix - def DISABLED_testVar(self): - with self.cached_session() as sess: - with specs.ops: - # pylint: disable=undefined-variable - v = Var("test_var", - shape=[2, 2], - initializer=init_ops.constant_initializer(42.0)) - inputs = constant_op.constant(_rand(10, 100)) - outputs = v.funcall(inputs) - self.assertEqual(len(variables.global_variables()), 1) - sess.run([outputs.initializer]) - outputs_value = outputs.eval() - self.assertEqual(outputs_value.shape, (2, 2)) - self.assertEqual(outputs_value[1, 1], 42.0) - - # XXX: the cleverness of this code is over 9000 - # TODO: original author please fix - def DISABLED_testShared(self): - with self.cached_session(): - with specs.ops: - # pylint: disable=undefined-variable - f = Shared(Fr(100)) - g = f | f | f | f - inputs = constant_op.constant(_rand(10, 100)) - _ = g.funcall(inputs) - self.assertEqual(len(variables.global_variables()), 2) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/specs/python/summaries.py b/tensorflow/contrib/specs/python/summaries.py deleted file mode 100644 index cd730d57e71..00000000000 --- a/tensorflow/contrib/specs/python/summaries.py +++ /dev/null @@ -1,313 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Functions for summarizing and describing TensorFlow graphs. - -This contains functions that generate string descriptions from -TensorFlow graphs, for debugging, testing, and model size -estimation. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re -from tensorflow.contrib.specs.python import specs -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops - -# These are short abbreviations for common TensorFlow operations used -# in test cases with tf_structure to verify that specs_lib generates a -# graph structure with the right operations. Operations outside the -# scope of specs (e.g., Const and Placeholder) are just assigned "_" -# since they are not relevant to testing. - -SHORT_NAMES_SRC = """ -BiasAdd biasadd -Const _ -Conv2D conv -MatMul dot -Placeholder _ -Sigmoid sig -Variable var -""".split() - -SHORT_NAMES = { - x: y - for x, y in zip(SHORT_NAMES_SRC[::2], SHORT_NAMES_SRC[1::2]) -} - - -def _truncate_structure(x): - """A helper function that disables recursion in tf_structure. - - Some constructs (e.g., HorizontalLstm) are complex unrolled - structures and don't need to be represented in the output - of tf_structure or tf_print. This helper function defines - which tree branches should be pruned. This is a very imperfect - way of dealing with unrolled LSTM's (since it truncates - useful information as well), but it's not worth doing something - better until the new fused and unrolled ops are ready. - - Args: - x: a Tensor or Op - - Returns: - A bool indicating whether the subtree should be pruned. - """ - if "/HorizontalLstm/" in x.name: - return True - return False - - -def tf_structure(x, include_shapes=False, finished=None): - """A postfix expression summarizing the TF graph. - - This is intended to be used as part of test cases to - check for gross differences in the structure of the graph. - The resulting string is not invertible or unabiguous - and cannot be used to reconstruct the graph accurately. - - Args: - x: a tf.Tensor or tf.Operation - include_shapes: include shapes in the output string - finished: a set of ops that have already been output - - Returns: - A string representing the structure as a string of - postfix operations. - """ - if finished is None: - finished = set() - if isinstance(x, ops.Tensor): - shape = x.get_shape().as_list() - x = x.op - else: - shape = [] - if x in finished: - return " <>" - finished |= {x} - result = "" - if not _truncate_structure(x): - for y in x.inputs: - result += tf_structure(y, include_shapes, finished) - if include_shapes: - result += " %s" % (shape,) - if x.type != "Identity": - name = SHORT_NAMES.get(x.type, x.type.lower()) - result += " " + name - return result - - -def tf_print(x, depth=0, finished=None, printer=print): - """A simple print function for a TensorFlow graph. - - Args: - x: a tf.Tensor or tf.Operation - depth: current printing depth - finished: set of nodes already output - printer: print function to use - - Returns: - Total number of parameters found in the - subtree. - """ - - if finished is None: - finished = set() - if isinstance(x, ops.Tensor): - shape = x.get_shape().as_list() - x = x.op - else: - shape = "" - if x.type == "Identity": - x = x.inputs[0].op - if x in finished: - printer("%s<%s> %s %s" % (" " * depth, x.name, x.type, shape)) - return - finished |= {x} - printer("%s%s %s %s" % (" " * depth, x.name, x.type, shape)) - if not _truncate_structure(x): - for y in x.inputs: - tf_print(y, depth + 1, finished, printer=printer) - - -def tf_num_params(x): - """Number of parameters in a TensorFlow subgraph. - - Args: - x: root of the subgraph (Tensor, Operation) - - Returns: - Total number of elements found in all Variables - in the subgraph. - """ - - if isinstance(x, ops.Tensor): - shape = x.get_shape() - x = x.op - if x.type in ["Variable", "VariableV2"]: - return shape.num_elements() - totals = [tf_num_params(y) for y in x.inputs] - return sum(totals) - - -def tf_left_split(op): - """Split the parameters of op for left recursion. - - Args: - op: tf.Operation - - Returns: - A tuple of the leftmost input tensor and a list of the - remaining arguments. - """ - - if len(op.inputs) < 1: - return None, [] - if op.type == "Concat": - return op.inputs[1], op.inputs[2:] - return op.inputs[0], op.inputs[1:] - - -def tf_parameter_iter(x): - """Iterate over the left branches of a graph and yield sizes. - - Args: - x: root of the subgraph (Tensor, Operation) - - Yields: - A triple of name, number of params, and shape. - """ - - while 1: - if isinstance(x, ops.Tensor): - shape = x.get_shape().as_list() - x = x.op - else: - shape = "" - left, right = tf_left_split(x) - totals = [tf_num_params(y) for y in right] - total = sum(totals) - yield x.name, total, shape - if left is None: - break - x = left - - -def _combine_filter(x): - """A filter for combining successive layers with similar names.""" - last_name = None - last_total = 0 - last_shape = None - for name, total, shape in x: - name = re.sub("/.*", "", name) - if name == last_name: - last_total += total - continue - if last_name is not None: - yield last_name, last_total, last_shape - last_name = name - last_total = total - last_shape = shape - if last_name is not None: - yield last_name, last_total, last_shape - - -def tf_parameter_summary(x, printer=print, combine=True): - """Summarize parameters by depth. - - Args: - x: root of the subgraph (Tensor, Operation) - printer: print function for output - combine: combine layers by top-level scope - """ - seq = tf_parameter_iter(x) - if combine: - seq = _combine_filter(seq) - seq = reversed(list(seq)) - for name, total, shape in seq: - printer("%10d %-20s %s" % (total, name, shape)) - - -def tf_spec_structure(spec, - inputs=None, - input_shape=None, - input_type=dtypes.float32): - """Return a postfix representation of the specification. - - This is intended to be used as part of test cases to - check for gross differences in the structure of the graph. - The resulting string is not invertible or unabiguous - and cannot be used to reconstruct the graph accurately. - - Args: - spec: specification - inputs: input to the spec construction (usually a Tensor) - input_shape: tensor shape (in lieu of inputs) - input_type: type of the input tensor - - Returns: - A string with a postfix representation of the - specification. - """ - - if inputs is None: - inputs = array_ops.placeholder(input_type, input_shape) - outputs = specs.create_net(spec, inputs) - return str(tf_structure(outputs).strip()) - - -def tf_spec_summary(spec, - inputs=None, - input_shape=None, - input_type=dtypes.float32): - """Output a summary of the specification. - - This prints a list of left-most tensor operations and summarized the - variables found in the right branches. This kind of representation - is particularly useful for networks that are generally structured - like pipelines. - - Args: - spec: specification - inputs: input to the spec construction (usually a Tensor) - input_shape: optional shape of input - input_type: type of the input tensor - """ - - if inputs is None: - inputs = array_ops.placeholder(input_type, input_shape) - outputs = specs.create_net(spec, inputs) - tf_parameter_summary(outputs) - - -def tf_spec_print(spec, - inputs=None, - input_shape=None, - input_type=dtypes.float32): - """Print a tree representing the spec. - - Args: - spec: specification - inputs: input to the spec construction (usually a Tensor) - input_shape: optional shape of input - input_type: type of the input tensor - """ - - if inputs is None: - inputs = array_ops.placeholder(input_type, input_shape) - outputs = specs.create_net(spec, inputs) - tf_print(outputs) diff --git a/tensorflow/contrib/specs/python/summaries_test.py b/tensorflow/contrib/specs/python/summaries_test.py deleted file mode 100644 index b82ba06d3f5..00000000000 --- a/tensorflow/contrib/specs/python/summaries_test.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for specs-related summarization functions.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.specs.python import specs -from tensorflow.contrib.specs.python import summaries -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -def _rand(*size): - return np.random.uniform(size=size).astype("f") - - -class SummariesTest(test.TestCase): - - def testStructure(self): - with self.cached_session(): - inputs_shape = (1, 18, 19, 5) - inputs = constant_op.constant(_rand(*inputs_shape)) - spec = "net = Cr(64, [5, 5])" - outputs = specs.create_net(spec, inputs) - variables.global_variables_initializer().run() - result = outputs.eval() - self.assertEqual(tuple(result.shape), (1, 18, 19, 64)) - self.assertEqual( - summaries.tf_spec_structure( - spec, input_shape=inputs_shape), - "_ variablev2 conv variablev2 biasadd relu") - - def testStructureFromTensor(self): - with self.cached_session(): - inputs = constant_op.constant(_rand(1, 18, 19, 5)) - spec = "net = Cr(64, [5, 5])" - outputs = specs.create_net(spec, inputs) - variables.global_variables_initializer().run() - result = outputs.eval() - self.assertEqual(tuple(result.shape), (1, 18, 19, 64)) - self.assertEqual( - summaries.tf_spec_structure(spec, inputs), - "_ variablev2 conv variablev2 biasadd relu") - - def testPrint(self): - with self.cached_session(): - inputs = constant_op.constant(_rand(1, 18, 19, 5)) - spec = "net = Cr(64, [5, 5])" - outputs = specs.create_net(spec, inputs) - variables.global_variables_initializer().run() - result = outputs.eval() - self.assertEqual(tuple(result.shape), (1, 18, 19, 64)) - summaries.tf_spec_print(spec, inputs) - - def testSummary(self): - with self.cached_session(): - inputs = constant_op.constant(_rand(1, 18, 19, 5)) - spec = "net = Cr(64, [5, 5])" - outputs = specs.create_net(spec, inputs) - variables.global_variables_initializer().run() - result = outputs.eval() - self.assertEqual(tuple(result.shape), (1, 18, 19, 64)) - summaries.tf_spec_summary(spec, inputs) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/staging/BUILD b/tensorflow/contrib/staging/BUILD deleted file mode 100644 index 96f7066646c..00000000000 --- a/tensorflow/contrib/staging/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -package( - default_visibility = [ - "//visibility:public", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "staging", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - deps = ["//tensorflow/python:data_flow_ops"], -) diff --git a/tensorflow/contrib/staging/__init__.py b/tensorflow/contrib/staging/__init__.py deleted file mode 100644 index e58ac31918a..00000000000 --- a/tensorflow/contrib/staging/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""contrib module containing StagingArea.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops.data_flow_ops import StagingArea diff --git a/tensorflow/contrib/stat_summarizer/BUILD b/tensorflow/contrib/stat_summarizer/BUILD deleted file mode 100644 index dd9e39933fe..00000000000 --- a/tensorflow/contrib/stat_summarizer/BUILD +++ /dev/null @@ -1,36 +0,0 @@ -# Description: -# Contains a Python wrapper for the StatSummarizer C++ class. - -load("//tensorflow:tensorflow.bzl", "tf_py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "stat_summarizer_py", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:_pywrap_stat_summarizer", - "//tensorflow/python:util", - ], -) - -tf_py_test( - name = "stat_summarizer_test", - size = "small", - srcs = ["python/stat_summarizer_test.py"], - additional_deps = [ - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:_pywrap_stat_summarizer", - "//tensorflow/python:math_ops", - "//tensorflow/python:variables", - ], - tags = ["notap"], # TODO(b/80546574): test is flaky -) diff --git a/tensorflow/contrib/stat_summarizer/__init__.py b/tensorflow/contrib/stat_summarizer/__init__.py deleted file mode 100644 index a1012f542ea..00000000000 --- a/tensorflow/contrib/stat_summarizer/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Exposes the Python wrapper for StatSummarizer utility class. - -The wrapper implementation is in tensorflow/python/util/stat_summarizer.i for -technical reasons, but it should be accessed via tf.contrib.stat_summarizer. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import, line-too-long -from tensorflow.python._pywrap_stat_summarizer import StatSummarizer - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = ['StatSummarizer'] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/stat_summarizer/python/stat_summarizer_test.py b/tensorflow/contrib/stat_summarizer/python/stat_summarizer_test.py deleted file mode 100644 index 542189f3f23..00000000000 --- a/tensorflow/contrib/stat_summarizer/python/stat_summarizer_test.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for StatSummarizer Python wrapper.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python import _pywrap_stat_summarizer -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class StatSummarizerTest(test.TestCase): - - def testStatSummarizer(self): - with ops.Graph().as_default() as graph: - matrix1 = constant_op.constant([[3., 3.]], name=r"m1") - matrix2 = constant_op.constant([[2.], [2.]], name=r"m2") - product = math_ops.matmul(matrix1, matrix2, name=r"product") - - graph_def = graph.as_graph_def() - ss = _pywrap_stat_summarizer.StatSummarizer( - graph_def.SerializeToString()) - - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - - for _ in range(20): - run_metadata = config_pb2.RunMetadata() - run_options = config_pb2.RunOptions( - trace_level=config_pb2.RunOptions.FULL_TRACE) - sess.run(product, options=run_options, run_metadata=run_metadata) - - ss.ProcessStepStatsStr(run_metadata.step_stats.SerializeToString()) - - output_string = ss.GetOutputString() - - print(output_string) - - # Test it recorded running the expected number of times. - self.assertRegexpMatches(output_string, r"count=20") - - # Test that a header line got printed. - self.assertRegexpMatches(output_string, r"====== .* ======") - - # Test that the nodes we added were analyzed. - # The line for the op should contain both the op type (MatMul) - # and the name of the node (product) - self.assertRegexpMatches(output_string, r"MatMul.*product") - self.assertRegexpMatches(output_string, r"Const.*m1") - self.assertRegexpMatches(output_string, r"Const.*m2") - - # Test that a CDF summed to 100% - self.assertRegexpMatches(output_string, r"100\.") - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/stateless/BUILD b/tensorflow/contrib/stateless/BUILD deleted file mode 100644 index e4c2bca12ed..00000000000 --- a/tensorflow/contrib/stateless/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -# Stateless random ops - -load("//tensorflow:tensorflow.bzl", "cuda_py_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "stateless", - srcs = [ - "__init__.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:framework_ops", - "//tensorflow/python:stateless_random_ops", - "//tensorflow/python:util", - ], -) - -cuda_py_test( - name = "stateless_random_ops_test", - srcs = ["python/kernel_tests/stateless_random_ops_test.py"], - additional_deps = [ - ":stateless", - "//tensorflow/python:client_testlib", - ], -) diff --git a/tensorflow/contrib/stateless/__init__.py b/tensorflow/contrib/stateless/__init__.py deleted file mode 100644 index 1a3a78bb5be..00000000000 --- a/tensorflow/contrib/stateless/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stateless random ops which take seed as a tensor input. - -DEPRECATED: Use `tf.random.stateless_uniform` rather than -`tf.contrib.stateless.stateless_random_uniform`, and similarly for the other -routines. - -Instead of taking `seed` as an attr which initializes a mutable state within -the op, these random ops take `seed` as an input, and the random numbers are -a deterministic function of `shape` and `seed`. - -WARNING: These ops are in contrib, and are not stable. They should be -consistent across multiple runs on the same hardware, but only for the same -version of the code. - -@@stateless_multinomial -@@stateless_random_uniform -@@stateless_random_normal -@@stateless_truncated_normal -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops.stateless_random_ops import stateless_random_uniform -from tensorflow.python.ops.stateless_random_ops import stateless_random_normal -from tensorflow.python.ops.stateless_random_ops import stateless_truncated_normal -from tensorflow.python.ops.stateless_random_ops import stateless_multinomial - -from tensorflow.python.util.all_util import remove_undocumented - -remove_undocumented(__name__) diff --git a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py b/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py deleted file mode 100644 index 8373cf62dcb..00000000000 --- a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.contrib.stateless API. - -The real tests are in python/kernel_tests/random/stateless_random_ops_test.py. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import stateless -from tensorflow.python.ops import stateless_random_ops -from tensorflow.python.platform import test - - -class StatelessOpsTest(test.TestCase): - - def testAPI(self): - self.assertIs(stateless.stateless_random_uniform, - stateless_random_ops.stateless_random_uniform) - self.assertIs(stateless.stateless_random_normal, - stateless_random_ops.stateless_random_normal) - self.assertIs(stateless.stateless_truncated_normal, - stateless_random_ops.stateless_truncated_normal) - self.assertIs(stateless.stateless_multinomial, - stateless_random_ops.stateless_multinomial) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/summary/BUILD b/tensorflow/contrib/summary/BUILD deleted file mode 100644 index acfccd4a75c..00000000000 --- a/tensorflow/contrib/summary/BUILD +++ /dev/null @@ -1,78 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - licenses = ["notice"], # Apache 2.0 -) - -exports_files([ - "LICENSE", -]) - -py_test( - name = "summary_ops_test", - srcs = ["summary_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":summary", - ":summary_test_util", - "//tensorflow/python:array_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python/eager:function", - "//tensorflow/python/eager:test", - "@six_archive//:six", - ], -) - -py_test( - name = "summary_ops_graph_test", - srcs = ["summary_ops_graph_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":summary", - ":summary_test_util", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "@six_archive//:six", - ], -) - -py_library( - name = "summary", - srcs = ["summary.py"], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python:summary_ops_v2", - ], -) - -# NOTE: target cannot be testonly because it needs to be in the pip -# package. Sigh. -py_library( - name = "summary_test_util", - srcs = ["summary_test_util.py"], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/core:protos_all_py", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:lib", - "//tensorflow/python:platform", - "//tensorflow/python:summary_ops_v2", - "@org_sqlite//:python", - ], -) diff --git a/tensorflow/contrib/summary/summary.py b/tensorflow/contrib/summary/summary.py deleted file mode 100644 index e0159a8e8e4..00000000000 --- a/tensorflow/contrib/summary/summary.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""TensorFlow Summary API v2. - -The operations in this package are safe to use with eager execution turned on or -off. It has a more flexible API that allows summaries to be written directly -from ops to places other than event log files, rather than propagating protos -from `tf.summary.merge_all` to `tf.summary.FileWriter`. - -To use with eager execution enabled, write your code as follows: - -```python -global_step = tf.train.get_or_create_global_step() -summary_writer = tf.contrib.summary.create_file_writer( - train_dir, flush_millis=10000) -with summary_writer.as_default(), tf.contrib.summary.always_record_summaries(): - # model code goes here - # and in it call - tf.contrib.summary.scalar("loss", my_loss) - # In this case every call to tf.contrib.summary.scalar will generate a record - # ... -``` - -To use it with graph execution, write your code as follows: - -```python -global_step = tf.train.get_or_create_global_step() -summary_writer = tf.contrib.summary.create_file_writer( - train_dir, flush_millis=10000) -with summary_writer.as_default(), tf.contrib.summary.always_record_summaries(): - # model definition code goes here - # and in it call - tf.contrib.summary.scalar("loss", my_loss) - # In this case every call to tf.contrib.summary.scalar will generate an op, - # note the need to run tf.contrib.summary.all_summary_ops() to make sure these - # ops get executed. - # ... - train_op = .... - -with tf.Session(...) as sess: - tf.global_variables_initializer().run() - tf.contrib.summary.initialize(graph=tf.get_default_graph()) - # ... - while not_done_training: - sess.run([train_op, tf.contrib.summary.all_summary_ops()]) - # ... -``` -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.python.ops.summary_ops_v2 import all_v2_summary_ops as all_summary_ops -from tensorflow.python.ops.summary_ops_v2 import always_record_summaries -from tensorflow.python.ops.summary_ops_v2 import audio -from tensorflow.python.ops.summary_ops_v2 import create_db_writer -from tensorflow.python.ops.summary_ops_v2 import create_file_writer -from tensorflow.python.ops.summary_ops_v2 import create_summary_file_writer -from tensorflow.python.ops.summary_ops_v2 import eval_dir -from tensorflow.python.ops.summary_ops_v2 import flush -from tensorflow.python.ops.summary_ops_v2 import generic -from tensorflow.python.ops.summary_ops_v2 import graph -from tensorflow.python.ops.summary_ops_v2 import histogram -from tensorflow.python.ops.summary_ops_v2 import image -from tensorflow.python.ops.summary_ops_v2 import import_event -from tensorflow.python.ops.summary_ops_v2 import initialize -from tensorflow.python.ops.summary_ops_v2 import never_record_summaries -from tensorflow.python.ops.summary_ops_v2 import record_summaries_every_n_global_steps -from tensorflow.python.ops.summary_ops_v2 import scalar -from tensorflow.python.ops.summary_ops_v2 import should_record_summaries -from tensorflow.python.ops.summary_ops_v2 import summary_writer_initializer_op -from tensorflow.python.ops.summary_ops_v2 import SummaryWriter diff --git a/tensorflow/contrib/summary/summary_ops_graph_test.py b/tensorflow/contrib/summary/summary_ops_graph_test.py deleted file mode 100644 index bc00250f002..00000000000 --- a/tensorflow/contrib/summary/summary_ops_graph_test.py +++ /dev/null @@ -1,361 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import tempfile -import time - -import unittest -import six - -from tensorflow.contrib.summary import summary as summary_ops -from tensorflow.contrib.summary import summary_test_util -from tensorflow.core.framework import graph_pb2 -from tensorflow.core.framework import node_def_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test -from tensorflow.python.training import training_util - -get_all = summary_test_util.get_all - - -class GraphFileTest(test_util.TensorFlowTestCase): - - def testSummaryOps(self): - logdir = self.get_temp_dir() - writer = summary_ops.create_file_writer(logdir, max_queue=0) - with writer.as_default(), summary_ops.always_record_summaries(): - summary_ops.generic('tensor', 1, step=1) - summary_ops.scalar('scalar', 2.0, step=1) - summary_ops.histogram('histogram', [1.0], step=1) - summary_ops.image('image', [[[[1.0]]]], step=1) - summary_ops.audio('audio', [[1.0]], 1.0, 1, step=1) - with self.cached_session() as sess: - sess.run(summary_ops.summary_writer_initializer_op()) - sess.run(summary_ops.all_summary_ops()) - # The working condition of the ops is tested in the C++ test so we just - # test here that we're calling them correctly. - self.assertTrue(gfile.Exists(logdir)) - - def testSummaryName(self): - logdir = self.get_temp_dir() - writer = summary_ops.create_file_writer(logdir, max_queue=0) - with writer.as_default(), summary_ops.always_record_summaries(): - summary_ops.scalar('scalar', 2.0, step=1) - with self.cached_session() as sess: - sess.run(summary_ops.summary_writer_initializer_op()) - sess.run(summary_ops.all_summary_ops()) - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(2, len(events)) - self.assertEqual('scalar', events[1].summary.value[0].tag) - - def testSummaryNameScope(self): - logdir = self.get_temp_dir() - writer = summary_ops.create_file_writer(logdir, max_queue=0) - with writer.as_default(), summary_ops.always_record_summaries(): - with ops.name_scope('scope'): - summary_ops.scalar('scalar', 2.0, step=1) - with self.cached_session() as sess: - sess.run(summary_ops.summary_writer_initializer_op()) - sess.run(summary_ops.all_summary_ops()) - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(2, len(events)) - self.assertEqual('scope/scalar', events[1].summary.value[0].tag) - - def testSummaryGlobalStep(self): - training_util.get_or_create_global_step() - logdir = self.get_temp_dir() - writer = summary_ops.create_file_writer(logdir, max_queue=0) - with writer.as_default(), summary_ops.always_record_summaries(): - summary_ops.scalar('scalar', 2.0) - with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - sess.run(summary_ops.summary_writer_initializer_op()) - step, _ = sess.run( - [training_util.get_global_step(), summary_ops.all_summary_ops()]) - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(2, len(events)) - self.assertEqual(step, events[1].step) - - def testMaxQueue(self): - logdir = self.get_temp_dir() - writer = summary_ops.create_file_writer( - logdir, max_queue=1, flush_millis=999999) - with writer.as_default(), summary_ops.always_record_summaries(): - summary_ops.scalar('scalar', 2.0, step=1) - with self.cached_session() as sess: - sess.run(summary_ops.summary_writer_initializer_op()) - get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) - # Note: First tf.compat.v1.Event is always file_version. - self.assertEqual(1, get_total()) - sess.run(summary_ops.all_summary_ops()) - self.assertEqual(1, get_total()) - # Should flush after second summary since max_queue = 1 - sess.run(summary_ops.all_summary_ops()) - self.assertEqual(3, get_total()) - - def testFlushFunction(self): - logdir = self.get_temp_dir() - writer = summary_ops.create_file_writer( - logdir, max_queue=999999, flush_millis=999999) - with writer.as_default(), summary_ops.always_record_summaries(): - summary_ops.scalar('scalar', 2.0, step=1) - flush_op = summary_ops.flush() - with self.cached_session() as sess: - sess.run(summary_ops.summary_writer_initializer_op()) - get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) - # Note: First tf.compat.v1.Event is always file_version. - self.assertEqual(1, get_total()) - sess.run(summary_ops.all_summary_ops()) - self.assertEqual(1, get_total()) - sess.run(flush_op) - self.assertEqual(2, get_total()) - # Test "writer" parameter - sess.run(summary_ops.all_summary_ops()) - sess.run(summary_ops.flush(writer=writer)) - self.assertEqual(3, get_total()) - sess.run(summary_ops.all_summary_ops()) - sess.run(summary_ops.flush(writer=writer._resource)) # pylint:disable=protected-access - self.assertEqual(4, get_total()) - - def testSharedName(self): - logdir = self.get_temp_dir() - with summary_ops.always_record_summaries(): - # Create with default shared name (should match logdir) - writer1 = summary_ops.create_file_writer(logdir) - with writer1.as_default(): - summary_ops.scalar('one', 1.0, step=1) - # Create with explicit logdir shared name (should be same resource/file) - shared_name = 'logdir:' + logdir - writer2 = summary_ops.create_file_writer(logdir, name=shared_name) - with writer2.as_default(): - summary_ops.scalar('two', 2.0, step=2) - # Create with different shared name (should be separate resource/file) - writer3 = summary_ops.create_file_writer(logdir, name='other') - with writer3.as_default(): - summary_ops.scalar('three', 3.0, step=3) - - with self.cached_session() as sess: - # Run init ops across writers sequentially to avoid race condition. - # TODO(nickfelt): fix race condition in resource manager lookup or create - sess.run(writer1.init()) - sess.run(writer2.init()) - time.sleep(1.1) # Ensure filename has a different timestamp - sess.run(writer3.init()) - sess.run(summary_ops.all_summary_ops()) - sess.run([writer1.flush(), writer2.flush(), writer3.flush()]) - - event_files = iter(sorted(gfile.Glob(os.path.join(logdir, '*tfevents*')))) - - # First file has tags "one" and "two" - events = summary_test_util.events_from_file(next(event_files)) - self.assertEqual('brain.Event:2', events[0].file_version) - tags = [e.summary.value[0].tag for e in events[1:]] - self.assertItemsEqual(['one', 'two'], tags) - - # Second file has tag "three" - events = summary_test_util.events_from_file(next(event_files)) - self.assertEqual('brain.Event:2', events[0].file_version) - tags = [e.summary.value[0].tag for e in events[1:]] - self.assertItemsEqual(['three'], tags) - - # No more files - self.assertRaises(StopIteration, lambda: next(event_files)) - - def testWriterInitAndClose(self): - logdir = self.get_temp_dir() - with summary_ops.always_record_summaries(): - writer = summary_ops.create_file_writer( - logdir, max_queue=100, flush_millis=1000000) - with writer.as_default(): - summary_ops.scalar('one', 1.0, step=1) - with self.cached_session() as sess: - sess.run(summary_ops.summary_writer_initializer_op()) - get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) - self.assertEqual(1, get_total()) # file_version Event - # Running init() again while writer is open has no effect - sess.run(writer.init()) - self.assertEqual(1, get_total()) - sess.run(summary_ops.all_summary_ops()) - self.assertEqual(1, get_total()) - # Running close() should do an implicit flush - sess.run(writer.close()) - self.assertEqual(2, get_total()) - # Running init() on a closed writer should start a new file - time.sleep(1.1) # Ensure filename has a different timestamp - sess.run(writer.init()) - sess.run(summary_ops.all_summary_ops()) - sess.run(writer.close()) - files = sorted(gfile.Glob(os.path.join(logdir, '*tfevents*'))) - self.assertEqual(2, len(files)) - self.assertEqual(2, len(summary_test_util.events_from_file(files[1]))) - - def testWriterFlush(self): - logdir = self.get_temp_dir() - with summary_ops.always_record_summaries(): - writer = summary_ops.create_file_writer( - logdir, max_queue=100, flush_millis=1000000) - with writer.as_default(): - summary_ops.scalar('one', 1.0, step=1) - with self.cached_session() as sess: - sess.run(summary_ops.summary_writer_initializer_op()) - get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) - self.assertEqual(1, get_total()) # file_version Event - sess.run(summary_ops.all_summary_ops()) - self.assertEqual(1, get_total()) - sess.run(writer.flush()) - self.assertEqual(2, get_total()) - - -class GraphDbTest(summary_test_util.SummaryDbTest): - - def testGraphPassedToGraph_isForbiddenForThineOwnSafety(self): - with self.assertRaises(TypeError): - summary_ops.graph(ops.Graph()) - with self.assertRaises(TypeError): - summary_ops.graph('') - - # TODO(b/133791853) Re-enable these tests. - @unittest.skip('Skipping because of b/133791853.') - def testGraphSummary(self): - training_util.get_or_create_global_step() - name = 'hi' - graph = graph_pb2.GraphDef(node=(node_def_pb2.NodeDef(name=name),)) - with self.cached_session(): - with self.create_db_writer().as_default(): - summary_ops.initialize(graph=graph) - six.assertCountEqual(self, [name], - get_all(self.db, 'SELECT node_name FROM Nodes')) - - def testScalarSummary(self): - """Test record_summaries_every_n_global_steps and all_summaries().""" - with ops.Graph().as_default(), self.cached_session() as sess: - global_step = training_util.get_or_create_global_step() - global_step.initializer.run() - with ops.device('/cpu:0'): - step_increment = state_ops.assign_add(global_step, 1) - sess.run(step_increment) # Increment global step from 0 to 1 - - logdir = tempfile.mkdtemp() - with summary_ops.create_file_writer(logdir, max_queue=0, - name='t2').as_default(): - with summary_ops.record_summaries_every_n_global_steps(2): - summary_ops.initialize() - summary_op = summary_ops.scalar('my_scalar', 2.0) - - # Neither of these should produce a summary because - # global_step is 1 and "1 % 2 != 0" - sess.run(summary_ops.all_summary_ops()) - sess.run(summary_op) - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 1) - - # Increment global step from 1 to 2 and check that the summary - # is now written - sess.run(step_increment) - sess.run(summary_ops.all_summary_ops()) - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 2) - self.assertEqual(events[1].summary.value[0].tag, 'my_scalar') - - def testScalarSummaryNameScope(self): - """Test record_summaries_every_n_global_steps and all_summaries().""" - with ops.Graph().as_default(), self.cached_session() as sess: - global_step = training_util.get_or_create_global_step() - global_step.initializer.run() - with ops.device('/cpu:0'): - step_increment = state_ops.assign_add(global_step, 1) - sess.run(step_increment) # Increment global step from 0 to 1 - - logdir = tempfile.mkdtemp() - with summary_ops.create_file_writer(logdir, max_queue=0, - name='t2').as_default(): - with summary_ops.record_summaries_every_n_global_steps(2): - summary_ops.initialize() - with ops.name_scope('scope'): - summary_op = summary_ops.scalar('my_scalar', 2.0) - - # Neither of these should produce a summary because - # global_step is 1 and "1 % 2 != 0" - sess.run(summary_ops.all_summary_ops()) - sess.run(summary_op) - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 1) - - # Increment global step from 1 to 2 and check that the summary - # is now written - sess.run(step_increment) - sess.run(summary_ops.all_summary_ops()) - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 2) - self.assertEqual(events[1].summary.value[0].tag, 'scope/my_scalar') - - def testSummaryGraphModeCond(self): - with ops.Graph().as_default(), self.cached_session(): - training_util.get_or_create_global_step() - logdir = tempfile.mkdtemp() - with summary_ops.create_file_writer( - logdir, max_queue=0, - name='t2').as_default(), summary_ops.always_record_summaries(): - summary_ops.initialize() - training_util.get_or_create_global_step().initializer.run() - def f(): - summary_ops.scalar('scalar', 2.0) - return constant_op.constant(True) - pred = array_ops.placeholder(dtypes.bool) - x = control_flow_ops.cond(pred, f, - lambda: constant_op.constant(False)) - x.eval(feed_dict={pred: True}) - - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 2) - self.assertEqual(events[1].summary.value[0].tag, 'cond/scalar') - - def testSummaryGraphModeWhile(self): - with ops.Graph().as_default(), self.cached_session(): - training_util.get_or_create_global_step() - logdir = tempfile.mkdtemp() - with summary_ops.create_file_writer( - logdir, max_queue=0, - name='t2').as_default(), summary_ops.always_record_summaries(): - summary_ops.initialize() - training_util.get_or_create_global_step().initializer.run() - def body(unused_pred): - summary_ops.scalar('scalar', 2.0) - return constant_op.constant(False) - def cond(pred): - return pred - pred = array_ops.placeholder(dtypes.bool) - x = control_flow_ops.while_loop(cond, body, [pred]) - x.eval(feed_dict={pred: True}) - - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 2) - self.assertEqual(events[1].summary.value[0].tag, 'while/scalar') - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py deleted file mode 100644 index b098161829a..00000000000 --- a/tensorflow/contrib/summary/summary_ops_test.py +++ /dev/null @@ -1,413 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import tempfile -import time - -import unittest -import sqlite3 - -import numpy as np -import six - -from tensorflow.contrib.summary import summary as summary_ops -from tensorflow.contrib.summary import summary_test_util -from tensorflow.core.framework import graph_pb2 -from tensorflow.core.framework import node_def_pb2 -from tensorflow.core.framework import types_pb2 -from tensorflow.python.eager import function -from tensorflow.python.eager import test -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.platform import gfile -from tensorflow.python.training import training_util - -get_all = summary_test_util.get_all -get_one = summary_test_util.get_one - -_NUMPY_NUMERIC_TYPES = { - types_pb2.DT_HALF: np.float16, - types_pb2.DT_FLOAT: np.float32, - types_pb2.DT_DOUBLE: np.float64, - types_pb2.DT_INT8: np.int8, - types_pb2.DT_INT16: np.int16, - types_pb2.DT_INT32: np.int32, - types_pb2.DT_INT64: np.int64, - types_pb2.DT_UINT8: np.uint8, - types_pb2.DT_UINT16: np.uint16, - types_pb2.DT_UINT32: np.uint32, - types_pb2.DT_UINT64: np.uint64, - types_pb2.DT_COMPLEX64: np.complex64, - types_pb2.DT_COMPLEX128: np.complex128, - types_pb2.DT_BOOL: np.bool_, -} - - -class EagerFileTest(test_util.TensorFlowTestCase): - - def testShouldRecordSummary(self): - self.assertFalse(summary_ops.should_record_summaries()) - with summary_ops.always_record_summaries(): - self.assertTrue(summary_ops.should_record_summaries()) - - def testSummaryOps(self): - training_util.get_or_create_global_step() - logdir = tempfile.mkdtemp() - with summary_ops.create_file_writer( - logdir, max_queue=0, - name='t0').as_default(), summary_ops.always_record_summaries(): - summary_ops.generic('tensor', 1, '') - summary_ops.scalar('scalar', 2.0) - summary_ops.histogram('histogram', [1.0]) - summary_ops.image('image', [[[[1.0]]]]) - summary_ops.audio('audio', [[1.0]], 1.0, 1) - # The working condition of the ops is tested in the C++ test so we just - # test here that we're calling them correctly. - self.assertTrue(gfile.Exists(logdir)) - - @test_util.assert_no_new_pyobjects_executing_eagerly - def testEagerMemory(self): - training_util.get_or_create_global_step() - logdir = self.get_temp_dir() - with summary_ops.create_file_writer( - logdir, max_queue=0, - name='t0').as_default(), summary_ops.always_record_summaries(): - summary_ops.generic('tensor', 1, '') - summary_ops.scalar('scalar', 2.0) - summary_ops.histogram('histogram', [1.0]) - summary_ops.image('image', [[[[1.0]]]]) - summary_ops.audio('audio', [[1.0]], 1.0, 1) - - def testDefunSummarys(self): - training_util.get_or_create_global_step() - logdir = tempfile.mkdtemp() - with summary_ops.create_file_writer( - logdir, max_queue=0, - name='t1').as_default(), summary_ops.always_record_summaries(): - - @function.defun - def write(): - summary_ops.scalar('scalar', 2.0) - - write() - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 2) - self.assertEqual(events[1].summary.value[0].simple_value, 2.0) - - def testSummaryName(self): - training_util.get_or_create_global_step() - logdir = tempfile.mkdtemp() - with summary_ops.create_file_writer( - logdir, max_queue=0, - name='t2').as_default(), summary_ops.always_record_summaries(): - - summary_ops.scalar('scalar', 2.0) - - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 2) - self.assertEqual(events[1].summary.value[0].tag, 'scalar') - - def testSummaryNameScope(self): - training_util.get_or_create_global_step() - logdir = tempfile.mkdtemp() - with summary_ops.create_file_writer( - logdir, max_queue=0, - name='t2').as_default(), summary_ops.always_record_summaries(): - - with ops.name_scope('scope'): - summary_ops.scalar('scalar', 2.0) - - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 2) - self.assertEqual(events[1].summary.value[0].tag, 'scope/scalar') - - def testSummaryGlobalStep(self): - step = training_util.get_or_create_global_step() - logdir = tempfile.mkdtemp() - with summary_ops.create_file_writer( - logdir, max_queue=0, - name='t2').as_default(), summary_ops.always_record_summaries(): - - summary_ops.scalar('scalar', 2.0, step=step) - - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 2) - self.assertEqual(events[1].summary.value[0].tag, 'scalar') - - def testRecordEveryNGlobalSteps(self): - step = training_util.get_or_create_global_step() - logdir = tempfile.mkdtemp() - - def run_step(): - summary_ops.scalar('scalar', i, step=step) - step.assign_add(1) - - with summary_ops.create_file_writer( - logdir).as_default(), summary_ops.record_summaries_every_n_global_steps( - 2, step): - for i in range(10): - run_step() - # And another 10 steps as a graph function. - run_step_fn = function.defun(run_step) - for i in range(10): - run_step_fn() - - events = summary_test_util.events_from_logdir(logdir) - self.assertEqual(len(events), 11) - - def testMaxQueue(self): - logs = tempfile.mkdtemp() - with summary_ops.create_file_writer( - logs, max_queue=1, flush_millis=999999, - name='lol').as_default(), summary_ops.always_record_summaries(): - get_total = lambda: len(summary_test_util.events_from_logdir(logs)) - # Note: First tf.compat.v1.Event is always file_version. - self.assertEqual(1, get_total()) - summary_ops.scalar('scalar', 2.0, step=1) - self.assertEqual(1, get_total()) - # Should flush after second summary since max_queue = 1 - summary_ops.scalar('scalar', 2.0, step=2) - self.assertEqual(3, get_total()) - - def testFlushFunction(self): - logs = tempfile.mkdtemp() - writer = summary_ops.create_file_writer( - logs, max_queue=999999, flush_millis=999999, name='lol') - with writer.as_default(), summary_ops.always_record_summaries(): - get_total = lambda: len(summary_test_util.events_from_logdir(logs)) - # Note: First tf.compat.v1.Event is always file_version. - self.assertEqual(1, get_total()) - summary_ops.scalar('scalar', 2.0, step=1) - summary_ops.scalar('scalar', 2.0, step=2) - self.assertEqual(1, get_total()) - summary_ops.flush() - self.assertEqual(3, get_total()) - # Test "writer" parameter - summary_ops.scalar('scalar', 2.0, step=3) - summary_ops.flush(writer=writer) - self.assertEqual(4, get_total()) - summary_ops.scalar('scalar', 2.0, step=4) - summary_ops.flush(writer=writer._resource) # pylint:disable=protected-access - self.assertEqual(5, get_total()) - - def testSharedName(self): - logdir = self.get_temp_dir() - with summary_ops.always_record_summaries(): - # Create with default shared name (should match logdir) - writer1 = summary_ops.create_file_writer(logdir) - with writer1.as_default(): - summary_ops.scalar('one', 1.0, step=1) - summary_ops.flush() - # Create with explicit logdir shared name (should be same resource/file) - shared_name = 'logdir:' + logdir - writer2 = summary_ops.create_file_writer(logdir, name=shared_name) - with writer2.as_default(): - summary_ops.scalar('two', 2.0, step=2) - summary_ops.flush() - # Create with different shared name (should be separate resource/file) - time.sleep(1.1) # Ensure filename has a different timestamp - writer3 = summary_ops.create_file_writer(logdir, name='other') - with writer3.as_default(): - summary_ops.scalar('three', 3.0, step=3) - summary_ops.flush() - - event_files = iter(sorted(gfile.Glob(os.path.join(logdir, '*tfevents*')))) - - # First file has tags "one" and "two" - events = iter(summary_test_util.events_from_file(next(event_files))) - self.assertEqual('brain.Event:2', next(events).file_version) - self.assertEqual('one', next(events).summary.value[0].tag) - self.assertEqual('two', next(events).summary.value[0].tag) - self.assertRaises(StopIteration, lambda: next(events)) - - # Second file has tag "three" - events = iter(summary_test_util.events_from_file(next(event_files))) - self.assertEqual('brain.Event:2', next(events).file_version) - self.assertEqual('three', next(events).summary.value[0].tag) - self.assertRaises(StopIteration, lambda: next(events)) - - # No more files - self.assertRaises(StopIteration, lambda: next(event_files)) - - def testWriterInitAndClose(self): - logdir = self.get_temp_dir() - get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) - with summary_ops.always_record_summaries(): - writer = summary_ops.create_file_writer( - logdir, max_queue=100, flush_millis=1000000) - self.assertEqual(1, get_total()) # file_version Event - # Calling init() again while writer is open has no effect - writer.init() - self.assertEqual(1, get_total()) - try: - # Not using .as_default() to avoid implicit flush when exiting - writer.set_as_default() - summary_ops.scalar('one', 1.0, step=1) - self.assertEqual(1, get_total()) - # Calling .close() should do an implicit flush - writer.close() - self.assertEqual(2, get_total()) - # Calling init() on a closed writer should start a new file - time.sleep(1.1) # Ensure filename has a different timestamp - writer.init() - files = sorted(gfile.Glob(os.path.join(logdir, '*tfevents*'))) - self.assertEqual(2, len(files)) - get_total = lambda: len(summary_test_util.events_from_file(files[1])) - self.assertEqual(1, get_total()) # file_version Event - summary_ops.scalar('two', 2.0, step=2) - writer.close() - self.assertEqual(2, get_total()) - finally: - # Clean up by resetting default writer - summary_ops.create_file_writer(None).set_as_default() - - def testWriterFlush(self): - logdir = self.get_temp_dir() - get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) - with summary_ops.always_record_summaries(): - writer = summary_ops.create_file_writer( - logdir, max_queue=100, flush_millis=1000000) - self.assertEqual(1, get_total()) # file_version Event - with writer.as_default(): - summary_ops.scalar('one', 1.0, step=1) - self.assertEqual(1, get_total()) - writer.flush() - self.assertEqual(2, get_total()) - summary_ops.scalar('two', 2.0, step=2) - # Exiting the "as_default()" should do an implicit flush of the "two" tag - self.assertEqual(3, get_total()) - - -class EagerDbTest(summary_test_util.SummaryDbTest): - - # TODO(b/133791853) Re-enable these tests. - @unittest.skip('Skipping because of b/133791853.') - def testDbURIOpen(self): - tmpdb_path = os.path.join(self.get_temp_dir(), 'tmpDbURITest.sqlite') - tmpdb_uri = six.moves.urllib_parse.urljoin('file:', tmpdb_path) - tmpdb_writer = summary_ops.create_db_writer(tmpdb_uri, 'experimentA', - 'run1', 'user1') - with summary_ops.always_record_summaries(): - with tmpdb_writer.as_default(): - summary_ops.scalar('t1', 2.0) - tmpdb = sqlite3.connect(tmpdb_path) - num = get_one(tmpdb, 'SELECT count(*) FROM Tags WHERE tag_name = "t1"') - self.assertEqual(num, 1) - tmpdb.close() - - # TODO(b/133791853) Re-enable these tests. - @unittest.skip('Skipping because of b/133791853.') - def testIntegerSummaries(self): - step = training_util.create_global_step() - writer = self.create_db_writer() - - def adder(x, y): - state_ops.assign_add(step, 1) - summary_ops.generic('x', x) - summary_ops.generic('y', y) - sum_ = x + y - summary_ops.generic('sum', sum_) - return sum_ - - with summary_ops.always_record_summaries(): - with writer.as_default(): - self.assertEqual(5, adder(int64(2), int64(3)).numpy()) - - six.assertCountEqual( - self, [1, 1, 1], - get_all(self.db, 'SELECT step FROM Tensors WHERE dtype IS NOT NULL')) - six.assertCountEqual(self, ['x', 'y', 'sum'], - get_all(self.db, 'SELECT tag_name FROM Tags')) - x_id = get_one(self.db, 'SELECT tag_id FROM Tags WHERE tag_name = "x"') - y_id = get_one(self.db, 'SELECT tag_id FROM Tags WHERE tag_name = "y"') - sum_id = get_one(self.db, 'SELECT tag_id FROM Tags WHERE tag_name = "sum"') - - with summary_ops.always_record_summaries(): - with writer.as_default(): - self.assertEqual(9, adder(int64(4), int64(5)).numpy()) - - six.assertCountEqual( - self, [1, 1, 1, 2, 2, 2], - get_all(self.db, 'SELECT step FROM Tensors WHERE dtype IS NOT NULL')) - six.assertCountEqual(self, [x_id, y_id, sum_id], - get_all(self.db, 'SELECT tag_id FROM Tags')) - self.assertEqual(2, get_tensor(self.db, x_id, 1)) - self.assertEqual(3, get_tensor(self.db, y_id, 1)) - self.assertEqual(5, get_tensor(self.db, sum_id, 1)) - self.assertEqual(4, get_tensor(self.db, x_id, 2)) - self.assertEqual(5, get_tensor(self.db, y_id, 2)) - self.assertEqual(9, get_tensor(self.db, sum_id, 2)) - six.assertCountEqual( - self, ['experiment'], - get_all(self.db, 'SELECT experiment_name FROM Experiments')) - six.assertCountEqual(self, ['run'], - get_all(self.db, 'SELECT run_name FROM Runs')) - six.assertCountEqual(self, ['user'], - get_all(self.db, 'SELECT user_name FROM Users')) - - def testBadExperimentName(self): - with self.assertRaises(ValueError): - self.create_db_writer(experiment_name='\0') - - def testBadRunName(self): - with self.assertRaises(ValueError): - self.create_db_writer(run_name='\0') - - def testBadUserName(self): - with self.assertRaises(ValueError): - self.create_db_writer(user_name='-hi') - with self.assertRaises(ValueError): - self.create_db_writer(user_name='hi-') - with self.assertRaises(ValueError): - self.create_db_writer(user_name='@') - - # TODO(b/133791853) Re-enable these tests. - @unittest.skip('Skipping because of b/133791853.') - def testGraphSummary(self): - training_util.get_or_create_global_step() - name = 'hi' - graph = graph_pb2.GraphDef(node=(node_def_pb2.NodeDef(name=name),)) - with summary_ops.always_record_summaries(): - with self.create_db_writer().as_default(): - summary_ops.graph(graph) - six.assertCountEqual(self, [name], - get_all(self.db, 'SELECT node_name FROM Nodes')) - - -def get_tensor(db, tag_id, step): - cursor = db.execute( - 'SELECT dtype, shape, data FROM Tensors WHERE series = ? AND step = ?', - (tag_id, step)) - dtype, shape, data = cursor.fetchone() - assert dtype in _NUMPY_NUMERIC_TYPES - buf = np.frombuffer(data, dtype=_NUMPY_NUMERIC_TYPES[dtype]) - if not shape: - return buf[0] - return buf.reshape([int(i) for i in shape.split(',')]) - - -def int64(x): - return array_ops.constant(x, dtypes.int64) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/summary/summary_test_util.py b/tensorflow/contrib/summary/summary_test_util.py deleted file mode 100644 index f15b7aebcc9..00000000000 --- a/tensorflow/contrib/summary/summary_test_util.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Utilities to test summaries.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import os - -import sqlite3 - -from tensorflow.core.util import event_pb2 -from tensorflow.python.framework import test_util -from tensorflow.python.lib.io import tf_record -from tensorflow.python.ops import summary_ops_v2 as summary_ops -from tensorflow.python.platform import gfile - - -class SummaryDbTest(test_util.TensorFlowTestCase): - """Helper for summary database testing.""" - - def setUp(self): - super(SummaryDbTest, self).setUp() - self.db_path = os.path.join(self.get_temp_dir(), 'DbTest.sqlite') - if os.path.exists(self.db_path): - os.unlink(self.db_path) - self.db = sqlite3.connect(self.db_path) - self.create_db_writer = functools.partial( - summary_ops.create_db_writer, - db_uri=self.db_path, - experiment_name='experiment', - run_name='run', - user_name='user') - - def tearDown(self): - self.db.close() - super(SummaryDbTest, self).tearDown() - - -def events_from_file(filepath): - """Returns all events in a single event file. - - Args: - filepath: Path to the event file. - - Returns: - A list of all tf.compat.v1.Event protos in the event file. - """ - records = list(tf_record.tf_record_iterator(filepath)) - result = [] - for r in records: - event = event_pb2.Event() - event.ParseFromString(r) - result.append(event) - return result - - -def events_from_logdir(logdir): - """Returns all events in the single eventfile in logdir. - - Args: - logdir: The directory in which the single event file is sought. - - Returns: - A list of all tf.compat.v1.Event protos from the single event file. - - Raises: - AssertionError: If logdir does not contain exactly one file. - """ - assert gfile.Exists(logdir) - files = gfile.ListDirectory(logdir) - assert len(files) == 1, 'Found not exactly one file in logdir: %s' % files - return events_from_file(os.path.join(logdir, files[0])) - - -def get_one(db, q, *p): - return db.execute(q, p).fetchone()[0] - - -def get_all(db, q, *p): - return unroll(db.execute(q, p).fetchall()) - - -def unroll(list_of_tuples): - return sum(list_of_tuples, ()) diff --git a/tensorflow/contrib/tensor_forest/BUILD b/tensorflow/contrib/tensor_forest/BUILD deleted file mode 100644 index ca246f912be..00000000000 --- a/tensorflow/contrib/tensor_forest/BUILD +++ /dev/null @@ -1,561 +0,0 @@ -# TensorFlow code for training random forests. - -load("//tensorflow:tensorflow.bzl", "py_test") -load("//tensorflow:tensorflow.bzl", "tf_gen_op_libs") -load("//tensorflow:tensorflow.bzl", "tf_cc_shared_object") -load("//tensorflow:tensorflow.bzl", "tf_cc_test") -load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_library") -load("//tensorflow:tensorflow.bzl", "tf_kernel_library") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") -load("//tensorflow/core/platform:default/build_config_root.bzl", "if_static") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -# ---------------------------------- V2 ops ------------------------------------------# -filegroup( - name = "v2_op_sources", - srcs = [ - "kernels/reinterpret_string_to_float_op.cc", - "kernels/scatter_add_ndim_op.cc", - ], -) - -filegroup( - name = "v2_op_defs", - srcs = [ - "ops/tensor_forest_ops.cc", - ], -) - -cc_library( - name = "v2_ops", - srcs = [ - ":v2_op_defs", - ":v2_op_sources", - ], - deps = [ - ":tree_utils", - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - "@com_google_protobuf//:protobuf_headers", - ], - alwayslink = 1, -) - -py_library( - name = "data_ops_py", - srcs = ["python/ops/data_ops.py"], - srcs_version = "PY2AND3", - deps = [ - ":tensor_forest_ops_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:sparse_tensor", - ], -) - -tf_gen_op_libs( - op_lib_names = ["tensor_forest_ops"], -) - -tf_gen_op_wrapper_py( - name = "gen_tensor_forest_ops", - out = "python/ops/gen_tensor_forest_ops.py", - deps = [":tensor_forest_ops_op_lib"], -) - -tf_custom_op_library( - name = "python/ops/_tensor_forest_ops.so", - srcs = [ - ":v2_op_defs", - ":v2_op_sources", - ] + if_static( - extra_deps = [], - otherwise = [ - ":libforestprotos.so", - ], - ), - deps = [ - ":tree_utils", - ], -) - -py_library( - name = "init_py", - srcs = [ - "__init__.py", - "client/__init__.py", - "python/__init__.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":data_ops_py", - ":eval_metrics", - ":model_ops_py", - ":random_forest", - ":stats_ops_py", - ":tensor_forest_ops_py", - ":tensor_forest_py", - ], -) - -tf_kernel_library( - name = "tensor_forest_kernels", - srcs = [":v2_op_sources"], - deps = [ - ":tree_utils", - "//tensorflow/core:framework_headers_lib", - "//tensorflow/core/kernels:bounds_check", - ], -) - -tf_custom_op_py_library( - name = "tensor_forest_ops_py", - srcs = ["python/ops/tensor_forest_ops.py"], - dso = ["python/ops/_tensor_forest_ops.so"], - kernels = [ - ":tensor_forest_kernels", - ":tensor_forest_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":gen_tensor_forest_ops", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:sparse_ops", - ], -) - -tf_cc_test( - name = "tensor_forest_ops_test", - size = "small", - srcs = [ - "kernels/tensor_forest_ops_test.cc", - ":v2_op_defs", - ":v2_op_sources", - ], - deps = [ - ":tree_utils", - "//tensorflow/core", - "//tensorflow/core:framework_headers_lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - "//third_party/eigen3", - ], -) - -# -------------------------------------- V4 ops ------------------------------- # -cc_library( - name = "tensor_forest_v4_kernels", - deps = [ - ":model_ops_kernels", - ":stats_ops_kernels", - ], -) - -cc_library( - name = "tensor_forest_v4_ops_op_lib", - deps = [ - ":model_ops_op_lib", - ":stats_ops_op_lib", - ], -) - -py_library( - name = "tensor_forest_v4_ops_py", - srcs_version = "PY2AND3", - deps = [ - ":model_ops_py", - ":stats_ops_py", - ], -) - -# Model Ops. -cc_library( - name = "model_ops_lib", - srcs = ["kernels/model_ops.cc"], - deps = [ - "//tensorflow/contrib/tensor_forest:tree_utils", - "//tensorflow/contrib/tensor_forest/kernels/v4:decision-tree-resource", - "//tensorflow/contrib/tensor_forest/kernels/v4:input_data", - "//tensorflow/core:framework_headers_lib", - ] + if_static( - extra_deps = [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_cc", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc", - ], - otherwise = [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc_headers_only", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc_headers_only", - ], - ), - alwayslink = 1, -) - -tf_gen_op_libs( - op_lib_names = ["model_ops"], -) - -tf_gen_op_wrapper_py( - name = "gen_model_ops_py", - out = "python/ops/gen_model_ops.py", - deps = [":model_ops_op_lib"], -) - -tf_kernel_library( - name = "model_ops_kernels", - deps = [ - ":model_ops_lib", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - ], - alwayslink = 1, -) - -tf_custom_op_library( - name = "python/ops/_model_ops.so", - srcs = [ - "ops/model_ops.cc", - ] + if_static( - extra_deps = [], - otherwise = [ - ":libforestprotos.so", - ], - ), - deps = [":model_ops_lib"], -) - -tf_custom_op_py_library( - name = "model_ops_py", - srcs = ["python/ops/model_ops.py"], - dso = ["python/ops/_model_ops.so"], - kernels = [ - ":model_ops_kernels", - ":model_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":gen_model_ops_py", - ":stats_ops_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:resources", - "//tensorflow/python:training", - ], -) - -tf_cc_test( - name = "model_ops_test", - size = "small", - srcs = [ - "kernels/model_ops_test.cc", - "ops/model_ops.cc", - ], - deps = [ - ":forest_proto_impl", - ":model_ops_lib", - "//tensorflow/contrib/tensor_forest/kernels/v4:decision-tree-resource_impl", - "//tensorflow/core:framework_headers_lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -# Stats Ops. -cc_library( - name = "stats_ops_lib", - srcs = ["kernels/stats_ops.cc"], - deps = [ - "//third_party/eigen3", - "//tensorflow/contrib/tensor_forest:tree_utils", - "//tensorflow/contrib/tensor_forest/kernels/v4:decision-tree-resource", - "//tensorflow/contrib/tensor_forest/kernels/v4:fertile-stats-resource", - "//tensorflow/contrib/tensor_forest/kernels/v4:input_data", - "//tensorflow/contrib/tensor_forest/kernels/v4:input_target", - "//tensorflow/contrib/tensor_forest/kernels/v4:params", - "//tensorflow/core:framework_headers_lib", - ] + if_static( - extra_deps = [ - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc", - ], - otherwise = [ - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc_headers_only", - ], - ), - alwayslink = 1, -) - -tf_gen_op_libs( - op_lib_names = ["stats_ops"], -) - -tf_gen_op_wrapper_py( - name = "gen_stats_ops_py", - out = "python/ops/gen_stats_ops.py", - deps = [":stats_ops_op_lib"], -) - -tf_kernel_library( - name = "stats_ops_kernels", - deps = [ - ":stats_ops_lib", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - ], - alwayslink = 1, -) - -tf_custom_op_library( - name = "python/ops/_stats_ops.so", - srcs = [ - "ops/stats_ops.cc", - ] + if_static( - extra_deps = [], - otherwise = [ - ":libforestprotos.so", - ], - ), - deps = [":stats_ops_lib"], -) - -tf_custom_op_py_library( - name = "stats_ops_py", - srcs = ["python/ops/stats_ops.py"], - dso = ["python/ops/_stats_ops.so"], - kernels = [ - ":stats_ops_kernels", - ":stats_ops_op_lib", - ], - srcs_version = "PY2AND3", - deps = [ - ":gen_stats_ops_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:resources", - "//tensorflow/python:training", - ], -) - -tf_cc_test( - name = "stats_ops_test", - size = "small", - srcs = [ - "kernels/stats_ops_test.cc", - "ops/stats_ops.cc", - ], - deps = [ - ":forest_proto_impl", - ":stats_ops_lib", - "//tensorflow/contrib/tensor_forest/kernels/v4:decision-tree-resource_impl", - "//tensorflow/core", - "//tensorflow/core:framework_headers_lib", - "//tensorflow/core:lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - "//third_party/eigen3", - ], -) - -# ---------------------------------- Common libs ------------------------ # -cc_library( - name = "tree_utils", - srcs = ["kernels/tree_utils.cc"], - hdrs = [ - "kernels/data_spec.h", - "kernels/tree_utils.h", - ], - deps = [ - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - "@com_google_protobuf//:protobuf_headers", - ], -) - -cc_library( - name = "forest_proto_impl", - deps = [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_cc", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc", - ], -) - -# Protocol buffer dependencies shared between multiple op shared objects. This -# avoids attempting to register the same protocol buffer multiple times. -tf_cc_shared_object( - name = "libforestprotos.so", - # This object does not depend on TensorFlow. - framework_so = [], - linkstatic = 1, - deps = [ - ":forest_proto_impl", - "//tensorflow/contrib/tensor_forest/kernels/v4:decision-tree-resource_impl", - "@com_google_protobuf//:protobuf", - ], -) - -# --------------------------------- Python -------------------------------- # - -py_library( - name = "eval_metrics", - srcs = ["client/eval_metrics.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/learn:estimator_constants_py", - "//tensorflow/contrib/losses:losses_py", - "//tensorflow/contrib/metrics:metrics_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//third_party/py/numpy", - ], -) - -py_test( - name = "eval_metrics_test", - size = "small", - srcs = ["client/eval_metrics_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":eval_metrics", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:variables", - ], -) - -py_library( - name = "client_lib", - srcs_version = "PY2AND3", - deps = [ - ":eval_metrics", - ":tensor_forest_ops_py", - ":tensor_forest_py", - ":tensor_forest_v4_ops_py", - ], -) - -py_test( - name = "scatter_add_ndim_op_test", - size = "small", - srcs = ["python/kernel_tests/scatter_add_ndim_op_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_gpu", - "no_pip_gpu", - ], - deps = [ - ":tensor_forest_ops_py", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:variables", - ], -) - -py_library( - name = "tensor_forest_py", - srcs = ["python/tensor_forest.py"], - srcs_version = "PY2AND3", - deps = [ - ":data_ops_py", - ":tensor_forest_ops_py", - ":tensor_forest_v4_ops_py", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_py", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "@six_archive//:six", - ], -) - -py_test( - name = "tensor_forest_test", - size = "small", - srcs = ["python/tensor_forest_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":tensor_forest_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:sparse_tensor", - ], -) - -py_library( - name = "random_forest", - srcs = ["client/random_forest.py"], - srcs_version = "PY2AND3", - deps = [ - ":client_lib", - "//tensorflow/contrib/estimator:estimator_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/learn", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:state_ops", - "//tensorflow/python:summary", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - ], -) - -py_test( - name = "random_forest_test", - size = "medium", - srcs = ["client/random_forest_test.py"], - python_version = "PY2", - shard_count = 6, - srcs_version = "PY2AND3", - tags = [ - "noasan", - "nomac", # b/63258195 - "notsan", - ], - deps = [ - ":random_forest", - ":tensor_forest_py", - "//tensorflow/contrib/learn/python/learn/datasets", - "//tensorflow/python:client_testlib", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/tensor_forest/README.md b/tensorflow/contrib/tensor_forest/README.md deleted file mode 100644 index 9e1491ea666..00000000000 --- a/tensorflow/contrib/tensor_forest/README.md +++ /dev/null @@ -1,149 +0,0 @@ -# TensorForest - -TensorForest is an implementation of random forests in TensorFlow using an -online, [extremely randomized trees]( -https://en.wikipedia.org/wiki/Random_forest#ExtraTrees) -training algorithm. It supports both -classification (binary and multiclass) and regression (scalar and vector). - -## Usage - -TensorForest is a tf.learn Estimator: - -```import tensorflow as tf - -params = tf.contrib.tensor_forest.python.tensor_forest.ForestHParams( - num_classes=2, num_features=10, regression=False, - num_trees=50, max_nodes=1000) - -classifier = -tf.contrib.tensor_forest.client.random_forest.TensorForestEstimator(params) - -classifier.fit(x=x_train, y=y_train) - -y_out = classifier.predict(x=x_test) -``` - -TensorForest users are implored to properly shuffle their training data, -as our training algorithm strongly assumes it is in random order. - -## Algorithm - -Each tree in the forest is trained independently in parallel. For each -tree, we maintain the following data: - -* The tree structure, giving the two children of each non-leaf node and -the *split* used to route data between them. Each split looks at a single -input feature and compares it to a threshold value. - -* Leaf statistics. Each leaf needs to gather statistics, and those -statistics have the property that at the end of training, they can be -turned into predictions. For classification problems, the statistics are -class counts, and for regression problems they are the vector sum of the -values seen at the leaf, along with a count of those values. - -* Growing statistics. Each leaf needs to gather data that will potentially -allow it to grow into a non-leaf parent node. That data usually consists -of a list of potential splits, along with statistics for each of those splits. -Split statistics in turn consist of leaf statistics for their left and -right branches, along with some other information that allows us to assess -the quality of the split. For classification problems, that's usually -the [gini -impurity](https://en.wikipedia.org/wiki/Decision_tree_learning#Gini_impurity) -of the split, while for regression problems it's the mean-squared error. - -At the start of training, the tree structure is initialized to a root node, -and the leaf and growing statistics for it are both empty. Then, for -each batch `{(x_i, y_i)}` of training data, the following steps are performed: - -1. Given the current tree structure, each `x_i` is used to find the leaf -assignment `l_i`. - -2. `y_i` is used to update the leaf statistics of leaf `l_i`. - -3. If the growing statistics for the leaf `l_i` do not yet contain -`num_splits_to_consider` splits, `x_i` is used to generate another split. -Specifically, a random feature value is chosen, and `x_i`'s value at that -feature is used for the split's threshold. - -4. Otherwise, `(x_i, y_i)` is used to update the statistics of every -split in the growing statistics of leaf `l_i`. If leaf `l_i` has now seen -`split_after_samples` data points since creating all of its potential splits, -the split with the best score is chosen, and the tree structure is grown. - -## Parameters - -The following ForestHParams parameters are required: - -* `num_classes`. The number of classes in a classification problem, or -the number of dimensions in the output of a regression problem. - -* `num_features`. The number of input features. - -The following ForestHParams parameters are important but not required: - -* `regression`. True for regression problems, False for classification tasks. - Defaults to False (classification). -For regression problems, TensorForests's output are the predicted regression -values. For classification, the outputs are the per-class probabilities. - -* `num_trees`. The number of trees to create. Defaults to 100. There -usually isn't any accuracy gain from using higher values. - -* `max_nodes`. Defaults to 10,000. No tree is allowed to grow beyond -`max_nodes` nodes, and training stops when all trees in the forest are this -large. - -The remaining ForestHParams parameters don't usually require being set by the -user: - -* `num_splits_to_consider`. Defaults to `sqrt(num_features)` capped to be -between 10 and 1000. In the extremely randomized tree training algorithm, -only this many potential splits are evaluated for each tree node. - -* `split_after_samples`. Defaults to 250. In our online version of -extremely randomized tree training, we pick a split for a node after it has -accumulated this many training samples. - -* `bagging_fraction`. If less than 1.0, -then each tree sees only a different, random sampled (without replacement), -`bagging_fraction` sized subset of -the training data. Defaults to 1.0 (no bagging) because it fails to give -any accuracy improvement our experiments so far. - -* `feature_bagging_fraction`. If less than 1.0, then each tree sees only -a different `feature_bagging_fraction * num_features` sized subset of the -input features. Defaults to 1.0 (no feature bagging). - -* `base_random_seed`. By default (`base_random_seed = 0`), the random number -generator for each tree is seeded by a 64-bit random value when -each tree is first created. Using a non-zero value causes tree training to -be deterministic, in that the i-th tree's random number generator is seeded -with the value `base_random_seed + i`. - -## Implementation - -The python code in `python/tensor_forest.py` assigns default values to the -parameters, handles both instance and feature bagging, and creates the -TensorFlow graphs for training and inference. The graphs themselves are -quite simple, as most of the work is done in custom ops. There is a single -op (`model_ops.tree_predictions_v4`) that does inference for a single tree, -and four custom ops that do training on a single tree over a single batch, -with each op roughly corresponding to one of the four steps from the -algorithm section above. - -The training data itself is stored in TensorFlow _resources_, which provide -a means of non-tensor based persistence storage. (See -`core/framework/resource_mgr.h` for more information about resources.) -The tree -structure is stored in the `DecisionTreeResource` defined in -`kernels/v4/decision-tree-resource.h` and the leaf and growing statistics -are stored in the `FertileStatsResource` defined in -`kernels/v4/fertile-stats-resource.h`. - -## More information - -* [Kaggle kernel demonstrating TensorForest on Iris - dataset](https://www.kaggle.com/thomascolthurst/tensorforest-on-iris/notebook) -* [TensorForest - paper from NIPS 2016 Workshop](https://docs.google.com/viewer?a=v&pid=sites&srcid=ZGVmYXVsdGRvbWFpbnxtbHN5c25pcHMyMDE2fGd4OjFlNTRiOWU2OGM2YzA4MjE) diff --git a/tensorflow/contrib/tensor_forest/__init__.py b/tensorflow/contrib/tensor_forest/__init__.py deleted file mode 100644 index b9aea7f4b9e..00000000000 --- a/tensorflow/contrib/tensor_forest/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Random forest implementation in tensorflow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.tensor_forest.client import * -from tensorflow.contrib.tensor_forest.python import * -# pylint: enable=unused-import,wildcard-import diff --git a/tensorflow/contrib/tensor_forest/client/__init__.py b/tensorflow/contrib/tensor_forest/client/__init__.py deleted file mode 100644 index 335c1e4c43c..00000000000 --- a/tensorflow/contrib/tensor_forest/client/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Random forest implementation in tensorflow.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.contrib.tensor_forest.client import eval_metrics -from tensorflow.contrib.tensor_forest.client import random_forest -# pylint: enable=unused-import diff --git a/tensorflow/contrib/tensor_forest/client/eval_metrics.py b/tensorflow/contrib/tensor_forest/client/eval_metrics.py deleted file mode 100644 index 0b4125f00f9..00000000000 --- a/tensorflow/contrib/tensor_forest/client/eval_metrics.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A collection of functions to be used as evaluation metrics.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib import losses -from tensorflow.contrib.learn.python.learn.estimators import prediction_key - -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import metrics -from tensorflow.python.ops import nn - -INFERENCE_PROB_NAME = prediction_key.PredictionKey.PROBABILITIES -INFERENCE_PRED_NAME = prediction_key.PredictionKey.CLASSES - -FEATURE_IMPORTANCE_NAME = 'global_feature_importance' - - -def _top_k_generator(k): - def _top_k(probabilities, targets): - targets = math_ops.cast(targets, dtypes.int32) - if targets.get_shape().ndims > 1: - targets = array_ops.squeeze(targets, axis=[1]) - return metrics.mean(nn.in_top_k(probabilities, targets, k)) - return _top_k - - -def _accuracy(predictions, targets, weights=None): - return metrics.accuracy( - labels=targets, predictions=predictions, weights=weights) - - -def _r2(probabilities, targets, weights=None): - targets = math_ops.cast(targets, dtypes.float32) - y_mean = math_ops.reduce_mean(targets, 0) - squares_total = math_ops.reduce_sum( - math_ops.squared_difference(targets, y_mean), 0) - squares_residuals = math_ops.reduce_sum( - math_ops.squared_difference(targets, probabilities), 0) - score = 1 - math_ops.reduce_sum(squares_residuals / squares_total) - return metrics.mean(score, weights=weights) - - -def _squeeze_and_onehot(targets, depth): - targets = array_ops.squeeze(targets, axis=[1]) - return array_ops.one_hot(math_ops.cast(targets, dtypes.int32), depth) - - -def _sigmoid_entropy(probabilities, targets, weights=None): - return metrics.mean( - losses.sigmoid_cross_entropy(probabilities, - _squeeze_and_onehot( - targets, - array_ops.shape(probabilities)[1])), - weights=weights) - - -def _softmax_entropy(probabilities, targets, weights=None): - return metrics.mean( - losses.sparse_softmax_cross_entropy(probabilities, - math_ops.cast(targets, dtypes.int32)), - weights=weights) - - -def _predictions(predictions, unused_targets, **unused_kwargs): - return predictions - - -def _class_log_loss(probabilities, targets, weights=None): - return metrics.mean( - losses.log_loss(probabilities, - _squeeze_and_onehot(targets, - array_ops.shape(probabilities)[1])), - weights=weights) - - -def _precision(predictions, targets, weights=None): - return metrics.precision( - labels=targets, predictions=predictions, weights=weights) - - -def _precision_at_thresholds(predictions, targets, weights=None): - return metrics.precision_at_thresholds( - labels=targets, - predictions=array_ops.slice(predictions, [0, 1], [-1, 1]), - thresholds=np.arange(0, 1, 0.01, dtype=np.float32), - weights=weights) - - -def _recall(predictions, targets, weights=None): - return metrics.recall( - labels=targets, predictions=predictions, weights=weights) - - -def _recall_at_thresholds(predictions, targets, weights=None): - return metrics.recall_at_thresholds( - labels=targets, - predictions=array_ops.slice(predictions, [0, 1], [-1, 1]), - thresholds=np.arange(0, 1, 0.01, dtype=np.float32), - weights=weights) - - -def _auc(probs, targets, weights=None): - return metrics.auc( - labels=targets, - predictions=array_ops.slice(probs, [0, 1], [-1, 1]), - weights=weights) - - -_EVAL_METRICS = { - 'auc': _auc, - 'sigmoid_entropy': _sigmoid_entropy, - 'softmax_entropy': _softmax_entropy, - 'accuracy': _accuracy, - 'r2': _r2, - 'predictions': _predictions, - 'classification_log_loss': _class_log_loss, - 'precision': _precision, - 'precision_at_thresholds': _precision_at_thresholds, - 'recall': _recall, - 'recall_at_thresholds': _recall_at_thresholds, - 'top_5': _top_k_generator(5) -} - -_PREDICTION_KEYS = { - 'auc': INFERENCE_PROB_NAME, - 'sigmoid_entropy': INFERENCE_PROB_NAME, - 'softmax_entropy': INFERENCE_PROB_NAME, - 'accuracy': INFERENCE_PRED_NAME, - 'r2': prediction_key.PredictionKey.SCORES, - 'predictions': INFERENCE_PRED_NAME, - 'classification_log_loss': INFERENCE_PROB_NAME, - 'precision': INFERENCE_PRED_NAME, - 'precision_at_thresholds': INFERENCE_PROB_NAME, - 'recall': INFERENCE_PRED_NAME, - 'recall_at_thresholds': INFERENCE_PROB_NAME, - 'top_5': INFERENCE_PROB_NAME -} - - -def get_metric(metric_name): - """Given a metric name, return the corresponding metric function.""" - return _EVAL_METRICS[metric_name] - - -def get_prediction_key(metric_name): - return _PREDICTION_KEYS[metric_name] diff --git a/tensorflow/contrib/tensor_forest/client/eval_metrics_test.py b/tensorflow/contrib/tensor_forest/client/eval_metrics_test.py deleted file mode 100644 index d49928e3f1e..00000000000 --- a/tensorflow/contrib/tensor_forest/client/eval_metrics_test.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.contrib.tensor_forest.client.eval_metrics.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.client import eval_metrics -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import test_util -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest - - -class EvalMetricsTest(test_util.TensorFlowTestCase): - - def testTop2(self): - top_2_fn = eval_metrics._top_k_generator(2) - probabilities = constant_op.constant([[0.1, 0.2, 0.3], [0.4, 0.7, 0.5], - [0.9, 0.8, 0.2], [0.6, 0.4, 0.8]]) - targets = constant_op.constant([[0], [2], [1], [1]]) - in_top_2_op, update_op = top_2_fn(probabilities, targets) - with self.cached_session(): - # initializes internal accuracy vars - variables.local_variables_initializer().run() - # need to call in order to run the in_top_2_op internal operations because - # it is a streaming function - update_op.eval() - self.assertNear(0.5, in_top_2_op.eval(), 0.0001) - - def testTop3(self): - top_3_fn = eval_metrics._top_k_generator(3) - probabilities = constant_op.constant([[0.1, 0.2, 0.6, 0.3, 0.5, 0.5], - [0.1, 0.4, 0.7, 0.3, 0.5, 0.2], - [0.1, 0.3, 0.8, 0.7, 0.4, 0.9], - [0.9, 0.8, 0.1, 0.8, 0.2, 0.7], - [0.3, 0.6, 0.9, 0.4, 0.8, 0.6]]) - targets = constant_op.constant([3, 0, 2, 5, 1]) - in_top_3_op, update_op = top_3_fn(probabilities, targets) - with self.cached_session(): - # initializes internal accuracy vars - variables.local_variables_initializer().run() - # need to call in order to run the in_top_3_op internal operations because - # it is a streaming function - update_op.eval() - self.assertNear(0.4, in_top_3_op.eval(), 0.0001) - - def testAccuracy(self): - predictions = constant_op.constant([0, 1, 3, 6, 5, 2, 7, 6, 4, 9]) - targets = constant_op.constant([0, 1, 4, 6, 5, 1, 7, 5, 4, 8]) - accuracy_op, update_op = eval_metrics._accuracy(predictions, targets) - with self.cached_session(): - variables.local_variables_initializer().run() - # need to call in order to run the accuracy_op internal operations because - # it is a streaming function - update_op.eval() - self.assertNear(0.6, accuracy_op.eval(), 0.0001) - - def testR2(self): - scores = constant_op.constant( - [1.2, 3.9, 2.1, 0.9, 2.2, 0.1, 6.0, 4.0, 0.9]) - targets = constant_op.constant( - [1.0, 4.3, 2.6, 0.5, 1.1, 0.7, 5.1, 3.4, 1.8]) - r2_op, update_op = eval_metrics._r2(scores, targets) - with self.cached_session(): - # initializes internal accuracy vars - variables.local_variables_initializer().run() - # need to call in order to run the r2_op internal operations because - # it is a streaming function - update_op.eval() - self.assertNear(0.813583, r2_op.eval(), 0.0001) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/tensor_forest/client/random_forest.py b/tensorflow/contrib/tensor_forest/client/random_forest.py deleted file mode 100644 index 6e3bfbb9bd7..00000000000 --- a/tensorflow/contrib/tensor_forest/client/random_forest.py +++ /dev/null @@ -1,744 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A tf.learn implementation of online extremely random forests.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.estimator.python.estimator import head as core_head_lib -from tensorflow.contrib.learn.python.learn.estimators import constants -from tensorflow.contrib.learn.python.learn.estimators import estimator -from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib -from tensorflow.contrib.tensor_forest.client import eval_metrics -from tensorflow.contrib.tensor_forest.python import tensor_forest -from tensorflow.python.estimator import estimator as core_estimator -from tensorflow.python.estimator.export.export_output import PredictOutput -from tensorflow.python.feature_column import feature_column as fc_core -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.summary import summary -from tensorflow.python.training import session_run_hook -from tensorflow.python.training import training_util - -KEYS_NAME = 'keys' -LOSS_NAME = 'rf_training_loss' -TREE_PATHS_PREDICTION_KEY = 'tree_paths' -VARIANCE_PREDICTION_KEY = 'prediction_variance' -ALL_SERVING_KEY = 'tensorforest_all' -EPSILON = 0.000001 - - -class ModelBuilderOutputType(object): - MODEL_FN_OPS = 0 - ESTIMATOR_SPEC = 1 - - -class TensorForestRunOpAtEndHook(session_run_hook.SessionRunHook): - - def __init__(self, op_dict): - """Ops is a dict of {name: op} to run before the session is destroyed.""" - self._ops = op_dict - - def end(self, session): - for name in sorted(self._ops.keys()): - logging.info('{0}: {1}'.format(name, session.run(self._ops[name]))) - - -class TensorForestLossHook(session_run_hook.SessionRunHook): - """Monitor to request stop when loss stops decreasing.""" - - def __init__(self, - early_stopping_rounds, - early_stopping_loss_threshold=None, - loss_op=None): - self.early_stopping_rounds = early_stopping_rounds - self.early_stopping_loss_threshold = early_stopping_loss_threshold - self.loss_op = loss_op - self.min_loss = None - self.last_step = -1 - # self.steps records the number of steps for which the loss has been - # non-decreasing - self.steps = 0 - - def before_run(self, run_context): - loss = (self.loss_op if self.loss_op is not None else - run_context.session.graph.get_operation_by_name( - LOSS_NAME).outputs[0]) - return session_run_hook.SessionRunArgs( - {'global_step': training_util.get_global_step(), - 'current_loss': loss}) - - def after_run(self, run_context, run_values): - current_loss = run_values.results['current_loss'] - current_step = run_values.results['global_step'] - self.steps += 1 - # Guard against the global step going backwards, which might happen - # if we recover from something. - if self.last_step == -1 or self.last_step > current_step: - logging.info('TensorForestLossHook resetting last_step.') - self.last_step = current_step - self.steps = 0 - self.min_loss = None - return - - self.last_step = current_step - if (self.min_loss is None or current_loss < - (self.min_loss - self.min_loss * self.early_stopping_loss_threshold)): - self.min_loss = current_loss - self.steps = 0 - if self.steps > self.early_stopping_rounds: - logging.info('TensorForestLossHook requesting stop.') - run_context.request_stop() - - -def _get_default_head(params, weights_name, output_type, name=None): - """Creates a default head based on a type of a problem.""" - if output_type == ModelBuilderOutputType.MODEL_FN_OPS: - if params.regression: - return head_lib.regression_head( - weight_column_name=weights_name, - label_dimension=params.num_outputs, - enable_centered_bias=False, - head_name=name) - else: - return head_lib.multi_class_head( - params.num_classes, - weight_column_name=weights_name, - enable_centered_bias=False, - head_name=name) - else: - if params.regression: - return core_head_lib.regression_head( - weight_column=weights_name, - label_dimension=params.num_outputs, - name=name, - loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE) - else: - if params.num_classes == 2: - return core_head_lib.binary_classification_head( - weight_column=weights_name, - name=name, - loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE) - else: - return core_head_lib.multi_class_head( - n_classes=params.num_classes, - weight_column=weights_name, - name=name, - loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE) - -def get_model_fn(params, - graph_builder_class, - device_assigner, - feature_columns=None, - weights_name=None, - model_head=None, - keys_name=None, - early_stopping_rounds=100, - early_stopping_loss_threshold=0.001, - num_trainers=1, - trainer_id=0, - report_feature_importances=False, - local_eval=False, - head_scope=None, - include_all_in_serving=False, - output_type=ModelBuilderOutputType.MODEL_FN_OPS): - """Return a model function given a way to construct a graph builder.""" - if model_head is None: - model_head = _get_default_head(params, weights_name, output_type) - - def _model_fn(features, labels, mode): - """Function that returns predictions, training loss, and training op.""" - - if (isinstance(features, ops.Tensor) or - isinstance(features, sparse_tensor.SparseTensor)): - features = {'features': features} - if feature_columns: - features = features.copy() - - if output_type == ModelBuilderOutputType.MODEL_FN_OPS: - features.update(layers.transform_features(features, feature_columns)) - else: - for fc in feature_columns: - tensor = fc_core._transform_features(features, [fc])[fc] # pylint: disable=protected-access - features[fc.name] = tensor - - weights = None - if weights_name and weights_name in features: - weights = features.pop(weights_name) - - keys = None - if keys_name and keys_name in features: - keys = features.pop(keys_name) - - # If we're doing eval, optionally ignore device_assigner. - # Also ignore device assigner if we're exporting (mode == INFER) - dev_assn = device_assigner - if (mode == model_fn_lib.ModeKeys.INFER or - (local_eval and mode == model_fn_lib.ModeKeys.EVAL)): - dev_assn = None - - graph_builder = graph_builder_class(params, - device_assigner=dev_assn) - - logits, tree_paths, regression_variance = graph_builder.inference_graph( - features) - - summary.scalar('average_tree_size', graph_builder.average_size()) - # For binary classification problems, convert probabilities to logits. - # Includes hack to get around the fact that a probability might be 0 or 1. - if not params.regression and params.num_classes == 2: - class_1_probs = array_ops.slice(logits, [0, 1], [-1, 1]) - logits = math_ops.log( - math_ops.maximum(class_1_probs / math_ops.maximum( - 1.0 - class_1_probs, EPSILON), EPSILON)) - - # labels might be None if we're doing prediction (which brings up the - # question of why we force everything to adhere to a single model_fn). - training_graph = None - training_hooks = [] - if labels is not None and mode == model_fn_lib.ModeKeys.TRAIN: - with ops.control_dependencies([logits.op]): - training_graph = control_flow_ops.group( - graph_builder.training_graph( - features, labels, input_weights=weights, - num_trainers=num_trainers, - trainer_id=trainer_id), - state_ops.assign_add(training_util.get_global_step(), 1)) - - # Put weights back in - if weights is not None: - features[weights_name] = weights - - # TensorForest's training graph isn't calculated directly from the loss - # like many other models. - def _train_fn(unused_loss): - return training_graph - - - # Ops are run in lexigraphical order of their keys. Run the resource - # clean-up op last. - all_handles = graph_builder.get_all_resource_handles() - ops_at_end = { - '9: clean up resources': - control_flow_ops.group(*[ - resource_variable_ops.destroy_resource_op(handle) - for handle in all_handles - ]) - } - - if report_feature_importances: - ops_at_end['1: feature_importances'] = ( - graph_builder.feature_importances()) - - training_hooks = [TensorForestRunOpAtEndHook(ops_at_end)] - - if output_type == ModelBuilderOutputType.MODEL_FN_OPS: - model_ops = model_head.create_model_fn_ops( - features=features, - labels=labels, - mode=mode, - train_op_fn=_train_fn, - logits=logits, - scope=head_scope) - - if early_stopping_rounds: - training_hooks.append( - TensorForestLossHook( - early_stopping_rounds, - early_stopping_loss_threshold=early_stopping_loss_threshold, - loss_op=model_ops.loss)) - - model_ops.training_hooks.extend(training_hooks) - - if keys is not None: - model_ops.predictions[keys_name] = keys - - if params.inference_tree_paths: - model_ops.predictions[TREE_PATHS_PREDICTION_KEY] = tree_paths - - model_ops.predictions[VARIANCE_PREDICTION_KEY] = regression_variance - - if include_all_in_serving: - # In order to serve the variance we need to add the prediction dict - # to output_alternatives dict. - if not model_ops.output_alternatives: - model_ops.output_alternatives = {} - model_ops.output_alternatives[ALL_SERVING_KEY] = ( - constants.ProblemType.UNSPECIFIED, model_ops.predictions) - - return model_ops - - else: - # Estimator spec - estimator_spec = model_head.create_estimator_spec( - features=features, - mode=mode, - labels=labels, - train_op_fn=_train_fn, - logits=logits) - - if early_stopping_rounds: - training_hooks.append( - TensorForestLossHook( - early_stopping_rounds, - early_stopping_loss_threshold=early_stopping_loss_threshold, - loss_op=estimator_spec.loss)) - - estimator_spec = estimator_spec._replace( - training_hooks=training_hooks + list(estimator_spec.training_hooks)) - if keys is not None: - estimator_spec.predictions[keys_name] = keys - if params.inference_tree_paths: - estimator_spec.predictions[TREE_PATHS_PREDICTION_KEY] = tree_paths - estimator_spec.predictions[VARIANCE_PREDICTION_KEY] = regression_variance - - if include_all_in_serving: - outputs = estimator_spec.export_outputs - if not outputs: - outputs = {} - outputs = {ALL_SERVING_KEY: PredictOutput(estimator_spec.predictions)} - print(estimator_spec.export_outputs) - # In order to serve the variance we need to add the prediction dict - # to output_alternatives dict. - estimator_spec = estimator_spec._replace(export_outputs=outputs) - - return estimator_spec - - return _model_fn - - -class TensorForestEstimator(estimator.Estimator): - """An estimator that can train and evaluate a random forest. - - Example: - - ```python - params = tf.contrib.tensor_forest.python.tensor_forest.ForestHParams( - num_classes=2, num_features=40, num_trees=10, max_nodes=1000) - - # Estimator using the default graph builder. - estimator = TensorForestEstimator(params, model_dir=model_dir) - - # Or estimator using TrainingLossForest as the graph builder. - estimator = TensorForestEstimator( - params, graph_builder_class=tensor_forest.TrainingLossForest, - model_dir=model_dir) - - # Input builders - def input_fn_train: # returns x, y - ... - def input_fn_eval: # returns x, y - ... - estimator.fit(input_fn=input_fn_train) - estimator.evaluate(input_fn=input_fn_eval) - - # Predict returns an iterable of dicts. - results = list(estimator.predict(x=x)) - prob0 = results[0][eval_metrics.INFERENCE_PROB_NAME] - prediction0 = results[0][eval_metrics.INFERENCE_PRED_NAME] - ``` - """ - - def __init__(self, - params, - device_assigner=None, - model_dir=None, - feature_columns=None, - graph_builder_class=tensor_forest.RandomForestGraphs, - config=None, - weight_column=None, - keys_column=None, - feature_engineering_fn=None, - early_stopping_rounds=100, - early_stopping_loss_threshold=0.001, - num_trainers=1, - trainer_id=0, - report_feature_importances=False, - local_eval=False, - version=None, - head=None, - include_all_in_serving=False): - """Initializes a TensorForestEstimator instance. - - Args: - params: ForestHParams object that holds random forest hyperparameters. - These parameters will be passed into `model_fn`. - device_assigner: An `object` instance that controls how trees get - assigned to devices. If `None`, will use - `tensor_forest.RandomForestDeviceAssigner`. - model_dir: Directory to save model parameters, graph, etc. To continue - training a previously saved model, load checkpoints saved to this - directory into an estimator. - feature_columns: An iterable containing all the feature columns used by - the model. All items in the set should be instances of classes derived - from `_FeatureColumn`. - graph_builder_class: An `object` instance that defines how TF graphs for - random forest training and inference are built. By default will use - `tensor_forest.RandomForestGraphs`. Can be overridden by version - kwarg. - config: `RunConfig` object to configure the runtime settings. - weight_column: A string defining feature column name representing - weights. Will be multiplied by the loss of the example. Used to - downweight or boost examples during training. - keys_column: A string naming one of the features to strip out and - pass through into the inference/eval results dict. Useful for - associating specific examples with their prediction. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - early_stopping_rounds: Allows training to terminate early if the forest is - no longer growing. 100 by default. Set to a Falsy value to disable - the default training hook. - early_stopping_loss_threshold: Percentage (as fraction) that loss must - improve by within early_stopping_rounds steps, otherwise training will - terminate. - num_trainers: Number of training jobs, which will partition trees - among them. - trainer_id: Which trainer this instance is. - report_feature_importances: If True, print out feature importances - during evaluation. - local_eval: If True, don't use a device assigner for eval. This is to - support some common setups where eval is done on a single machine, even - though training might be distributed. - version: Unused. - head: A heads_lib.Head object that calculates losses and such. If None, - one will be automatically created based on params. - include_all_in_serving: if True, allow preparation of the complete - prediction dict including the variance to be exported for serving with - the Servo lib; and it also requires calling export_savedmodel with - default_output_alternative_key=ALL_SERVING_KEY, i.e. - estimator.export_savedmodel(export_dir_base=your_export_dir, - serving_input_fn=your_export_input_fn, - default_output_alternative_key=ALL_SERVING_KEY) - if False, resort to default behavior, i.e. export scores and - probabilities but no variances. In this case - default_output_alternative_key should be None while calling - export_savedmodel(). - Note, that due to backward compatibility we cannot always set - include_all_in_serving to True because in this case calling - export_saved_model() without - default_output_alternative_key=ALL_SERVING_KEY (legacy behavior) the - saved_model_export_utils.get_output_alternatives() would raise - ValueError. - - Returns: - A `TensorForestEstimator` instance. - """ - # Override default number of trainers if config is provided. - if num_trainers == 1 and config is not None: - num_trainers = max(1, config.num_worker_replicas) - - super(TensorForestEstimator, self).__init__( - model_fn=get_model_fn( - params.fill(), - graph_builder_class, - device_assigner, - feature_columns=feature_columns, - model_head=head, - weights_name=weight_column, - keys_name=keys_column, - early_stopping_rounds=early_stopping_rounds, - early_stopping_loss_threshold=early_stopping_loss_threshold, - num_trainers=num_trainers, - trainer_id=trainer_id, - report_feature_importances=report_feature_importances, - local_eval=local_eval, - include_all_in_serving=include_all_in_serving, - ), - model_dir=model_dir, - config=config, - feature_engineering_fn=feature_engineering_fn) - - -def get_combined_model_fn(model_fns): - """Get a combined model function given a list of other model fns. - - The model function returned will call the individual model functions and - combine them appropriately. For: - - training ops: tf.group them. - loss: average them. - predictions: concat probabilities such that predictions[*][0-C1] are the - probabilities for output 1 (where C1 is the number of classes in output 1), - predictions[*][C1-(C1+C2)] are the probabilities for output 2 (where C2 - is the number of classes in output 2), etc. Also stack predictions such - that predictions[i][j] is the class prediction for example i and output j. - - This assumes that labels are 2-dimensional, with labels[i][j] being the - label for example i and output j, where forest j is trained using only - output j. - - Args: - model_fns: A list of model functions obtained from get_model_fn. - - Returns: - A ModelFnOps instance. - """ - def _model_fn(features, labels, mode): - """Function that returns predictions, training loss, and training op.""" - model_fn_ops = [] - for i in range(len(model_fns)): - with variable_scope.variable_scope('label_{0}'.format(i)): - sliced_labels = array_ops.slice(labels, [0, i], [-1, 1]) - model_fn_ops.append( - model_fns[i](features, sliced_labels, mode)) - training_hooks = [] - for mops in model_fn_ops: - training_hooks += mops.training_hooks - predictions = {} - if (mode == model_fn_lib.ModeKeys.EVAL or - mode == model_fn_lib.ModeKeys.INFER): - # Flatten the probabilities into one dimension. - predictions[eval_metrics.INFERENCE_PROB_NAME] = array_ops.concat( - [mops.predictions[eval_metrics.INFERENCE_PROB_NAME] - for mops in model_fn_ops], axis=1) - predictions[eval_metrics.INFERENCE_PRED_NAME] = array_ops.stack( - [mops.predictions[eval_metrics.INFERENCE_PRED_NAME] - for mops in model_fn_ops], axis=1) - loss = None - if (mode == model_fn_lib.ModeKeys.EVAL or - mode == model_fn_lib.ModeKeys.TRAIN): - loss = math_ops.reduce_sum( - array_ops.stack( - [mops.loss for mops in model_fn_ops])) / len(model_fn_ops) - - train_op = None - if mode == model_fn_lib.ModeKeys.TRAIN: - train_op = control_flow_ops.group( - *[mops.train_op for mops in model_fn_ops]) - return model_fn_lib.ModelFnOps( - mode=mode, - predictions=predictions, - loss=loss, - train_op=train_op, - training_hooks=training_hooks, - scaffold=None, - output_alternatives=None) - - return _model_fn - - -class MultiForestMultiHeadEstimator(estimator.Estimator): - """An estimator that can train a forest for a multi-headed problems. - - This class essentially trains separate forests (each with their own - ForestHParams) for each output. - - For multi-headed regression, a single-headed TensorForestEstimator can - be used to train a single model that predicts all outputs. This class can - be used to train separate forests for each output. - """ - - def __init__(self, - params_list, - device_assigner=None, - model_dir=None, - feature_columns=None, - graph_builder_class=tensor_forest.RandomForestGraphs, - config=None, - weight_column=None, - keys_column=None, - feature_engineering_fn=None, - early_stopping_rounds=100, - num_trainers=1, - trainer_id=0, - report_feature_importances=False, - local_eval=False): - """See TensorForestEstimator.__init__.""" - model_fns = [] - # Override default number of trainers if config is provided. - if num_trainers == 1 and config is not None: - num_trainers = max(1, config.num_worker_replicas) - - for i in range(len(params_list)): - params = params_list[i].fill() - model_fns.append( - get_model_fn( - params, - graph_builder_class, - device_assigner, - model_head=_get_default_head( - params, - weight_column, - name='head{0}'.format(i), - output_type=ModelBuilderOutputType.MODEL_FN_OPS), - weights_name=weight_column, - keys_name=keys_column, - early_stopping_rounds=early_stopping_rounds, - num_trainers=num_trainers, - trainer_id=trainer_id, - report_feature_importances=report_feature_importances, - local_eval=local_eval, - head_scope='output{0}'.format(i))) - - super(MultiForestMultiHeadEstimator, self).__init__( - model_fn=get_combined_model_fn(model_fns), - model_dir=model_dir, - config=config, - feature_engineering_fn=feature_engineering_fn) - - -class CoreTensorForestEstimator(core_estimator.Estimator): - """A CORE estimator that can train and evaluate a random forest. - - Example: - - ```python - params = tf.contrib.tensor_forest.python.tensor_forest.ForestHParams( - num_classes=2, num_features=40, num_trees=10, max_nodes=1000) - - # Estimator using the default graph builder. - estimator = CoreTensorForestEstimator(params, model_dir=model_dir) - - # Or estimator using TrainingLossForest as the graph builder. - estimator = CoreTensorForestEstimator( - params, graph_builder_class=tensor_forest.TrainingLossForest, - model_dir=model_dir) - - # Input builders - def input_fn_train: # returns x, y - ... - def input_fn_eval: # returns x, y - ... - estimator.train(input_fn=input_fn_train) - estimator.evaluate(input_fn=input_fn_eval) - - # Predict returns an iterable of dicts. - results = list(estimator.predict(x=x)) - prob0 = results[0][eval_metrics.INFERENCE_PROB_NAME] - prediction0 = results[0][eval_metrics.INFERENCE_PRED_NAME] - ``` - """ - - def __init__(self, - params, - device_assigner=None, - model_dir=None, - feature_columns=None, - graph_builder_class=tensor_forest.RandomForestGraphs, - config=None, - weight_column=None, - keys_column=None, - feature_engineering_fn=None, - early_stopping_rounds=100, - early_stopping_loss_threshold=0.001, - num_trainers=1, - trainer_id=0, - report_feature_importances=False, - local_eval=False, - version=None, - head=None, - include_all_in_serving=False): - """Initializes a TensorForestEstimator instance. - - Args: - params: ForestHParams object that holds random forest hyperparameters. - These parameters will be passed into `model_fn`. - device_assigner: An `object` instance that controls how trees get - assigned to devices. If `None`, will use - `tensor_forest.RandomForestDeviceAssigner`. - model_dir: Directory to save model parameters, graph, etc. To continue - training a previously saved model, load checkpoints saved to this - directory into an estimator. - feature_columns: An iterable containing all the feature columns used by - the model. All items in the set should be instances of classes derived - from `_FeatureColumn`. - graph_builder_class: An `object` instance that defines how TF graphs for - random forest training and inference are built. By default will use - `tensor_forest.RandomForestGraphs`. Can be overridden by version - kwarg. - config: `RunConfig` object to configure the runtime settings. - weight_column: A string defining feature column name representing - weights. Will be multiplied by the loss of the example. Used to - downweight or boost examples during training. - keys_column: A string naming one of the features to strip out and - pass through into the inference/eval results dict. Useful for - associating specific examples with their prediction. - feature_engineering_fn: Feature engineering function. Takes features and - labels which are the output of `input_fn` and returns features and - labels which will be fed into the model. - early_stopping_rounds: Allows training to terminate early if the forest is - no longer growing. 100 by default. Set to a Falsy value to disable - the default training hook. - early_stopping_loss_threshold: Percentage (as fraction) that loss must - improve by within early_stopping_rounds steps, otherwise training will - terminate. - num_trainers: Number of training jobs, which will partition trees - among them. - trainer_id: Which trainer this instance is. - report_feature_importances: If True, print out feature importances - during evaluation. - local_eval: If True, don't use a device assigner for eval. This is to - support some common setups where eval is done on a single machine, even - though training might be distributed. - version: Unused. - head: A heads_lib.Head object that calculates losses and such. If None, - one will be automatically created based on params. - include_all_in_serving: if True, allow preparation of the complete - prediction dict including the variance to be exported for serving with - the Servo lib; and it also requires calling export_savedmodel with - default_output_alternative_key=ALL_SERVING_KEY, i.e. - estimator.export_savedmodel(export_dir_base=your_export_dir, - serving_input_fn=your_export_input_fn, - default_output_alternative_key=ALL_SERVING_KEY) - if False, resort to default behavior, i.e. export scores and - probabilities but no variances. In this case - default_output_alternative_key should be None while calling - export_savedmodel(). - Note, that due to backward compatibility we cannot always set - include_all_in_serving to True because in this case calling - export_saved_model() without - default_output_alternative_key=ALL_SERVING_KEY (legacy behavior) the - saved_model_export_utils.get_output_alternatives() would raise - ValueError. - - Returns: - A `TensorForestEstimator` instance. - """ - # Override default number of trainers if config is provided. - if num_trainers == 1 and config is not None: - num_trainers = max(1, config.num_worker_replicas) - if trainer_id == 0 and config is not None: - trainer_id = config.global_id_in_cluster - - super(CoreTensorForestEstimator, self).__init__( - model_fn=get_model_fn( - params.fill(), - graph_builder_class, - device_assigner, - feature_columns=feature_columns, - model_head=head, - weights_name=weight_column, - keys_name=keys_column, - early_stopping_rounds=early_stopping_rounds, - early_stopping_loss_threshold=early_stopping_loss_threshold, - num_trainers=num_trainers, - trainer_id=trainer_id, - report_feature_importances=report_feature_importances, - local_eval=local_eval, - include_all_in_serving=include_all_in_serving, - output_type=ModelBuilderOutputType.ESTIMATOR_SPEC), - model_dir=model_dir, - config=config) diff --git a/tensorflow/contrib/tensor_forest/client/random_forest_test.py b/tensorflow/contrib/tensor_forest/client/random_forest_test.py deleted file mode 100644 index aa0016b7408..00000000000 --- a/tensorflow/contrib/tensor_forest/client/random_forest_test.py +++ /dev/null @@ -1,358 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for TensorForestTrainer.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.learn.python.learn.datasets import base -from tensorflow.contrib.tensor_forest.client import random_forest -from tensorflow.contrib.tensor_forest.python import tensor_forest -from tensorflow.python.estimator.canned import head as head_lib -from tensorflow.python.estimator.inputs import numpy_io -from tensorflow.python.feature_column import feature_column_lib as core_feature_column -from tensorflow.python.framework import ops -from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import test -from tensorflow.python.training import checkpoint_utils - - -def _get_classification_input_fns(): - iris = base.load_iris() - data = iris.data.astype(np.float32) - labels = iris.target.astype(np.int32) - - train_input_fn = numpy_io.numpy_input_fn( - x=data, y=labels, batch_size=150, num_epochs=None, shuffle=False) - - predict_input_fn = numpy_io.numpy_input_fn( - x=data[:1,], y=None, batch_size=1, num_epochs=1, shuffle=False) - return train_input_fn, predict_input_fn - - -def _get_regression_input_fns(): - boston = base.load_boston() - data = boston.data.astype(np.float32) - labels = boston.target.astype(np.int32) - - train_input_fn = numpy_io.numpy_input_fn( - x=data, y=labels, batch_size=506, num_epochs=None, shuffle=False) - - predict_input_fn = numpy_io.numpy_input_fn( - x=data[:1,], y=None, batch_size=1, num_epochs=1, shuffle=False) - return train_input_fn, predict_input_fn - - -class TensorForestTrainerTests(test.TestCase): - - def testClassification(self): - """Tests multi-class classification using matrix data as input.""" - hparams = tensor_forest.ForestHParams( - num_trees=3, - max_nodes=1000, - num_classes=3, - num_features=4, - split_after_samples=20, - inference_tree_paths=True) - classifier = random_forest.TensorForestEstimator(hparams.fill()) - - input_fn, predict_input_fn = _get_classification_input_fns() - classifier.fit(input_fn=input_fn, steps=100) - res = classifier.evaluate(input_fn=input_fn, steps=10) - - self.assertEqual(1.0, res['accuracy']) - self.assertAllClose(0.55144483, res['loss']) - - predictions = list(classifier.predict(input_fn=predict_input_fn)) - self.assertAllClose([[0.576117, 0.211942, 0.211942]], - [pred['probabilities'] for pred in predictions]) - - def testRegression(self): - """Tests regression using matrix data as input.""" - - hparams = tensor_forest.ForestHParams( - num_trees=5, - max_nodes=1000, - num_classes=1, - num_features=13, - regression=True, - split_after_samples=20) - - regressor = random_forest.TensorForestEstimator(hparams.fill()) - - input_fn, predict_input_fn = _get_regression_input_fns() - - regressor.fit(input_fn=input_fn, steps=100) - res = regressor.evaluate(input_fn=input_fn, steps=10) - self.assertGreaterEqual(0.1, res['loss']) - - predictions = list(regressor.predict(input_fn=predict_input_fn)) - self.assertAllClose([24.], [pred['scores'] for pred in predictions], atol=1) - - def testAdditionalOutputs(self): - """Tests multi-class classification using matrix data as input.""" - hparams = tensor_forest.ForestHParams( - num_trees=1, - max_nodes=100, - num_classes=3, - num_features=4, - split_after_samples=20, - inference_tree_paths=True) - classifier = random_forest.TensorForestEstimator( - hparams.fill(), keys_column='keys', include_all_in_serving=True) - - iris = base.load_iris() - data = iris.data.astype(np.float32) - labels = iris.target.astype(np.int32) - - input_fn = numpy_io.numpy_input_fn( - x={ - 'x': data, - 'keys': np.arange(len(iris.data)).reshape(150, 1) - }, - y=labels, - batch_size=10, - num_epochs=1, - shuffle=False) - - classifier.fit(input_fn=input_fn, steps=100) - predictions = list(classifier.predict(input_fn=input_fn)) - # Check that there is a key column, tree paths and var. - for pred in predictions: - self.assertTrue('keys' in pred) - self.assertTrue('tree_paths' in pred) - self.assertTrue('prediction_variance' in pred) - - def _assert_checkpoint(self, model_dir, global_step): - reader = checkpoint_utils.load_checkpoint(model_dir) - self.assertLessEqual( - reader.get_tensor(ops.GraphKeys.GLOBAL_STEP), global_step) - - def testEarlyStopping(self): - """Tests multi-class classification using matrix data as input.""" - hparams = tensor_forest.ForestHParams( - num_trees=100, - max_nodes=10000, - num_classes=3, - num_features=4, - split_after_samples=20, - inference_tree_paths=True) - classifier = random_forest.TensorForestEstimator( - hparams.fill(), - # Set a crazy threshold - 30% loss change. - early_stopping_loss_threshold=0.3, - early_stopping_rounds=2) - - input_fn, _ = _get_classification_input_fns() - classifier.fit(input_fn=input_fn, steps=100) - - # We stopped early. - self._assert_checkpoint(classifier.model_dir, global_step=5) - - -class CoreTensorForestTests(test.TestCase): - - def testTrainEvaluateInferDoesNotThrowErrorForClassifier(self): - head_fn = head_lib._multi_class_head_with_softmax_cross_entropy_loss( - n_classes=3, loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS) - - hparams = tensor_forest.ForestHParams( - num_trees=3, - max_nodes=1000, - num_classes=3, - num_features=4, - split_after_samples=20, - inference_tree_paths=True) - - est = random_forest.CoreTensorForestEstimator(hparams.fill(), head=head_fn) - - input_fn, predict_input_fn = _get_classification_input_fns() - - est.train(input_fn=input_fn, steps=100) - res = est.evaluate(input_fn=input_fn, steps=1) - - self.assertEqual(1.0, res['accuracy']) - self.assertAllClose(0.55144483, res['loss']) - - predictions = list(est.predict(input_fn=predict_input_fn)) - self.assertAllClose([[0.576117, 0.211942, 0.211942]], - [pred['probabilities'] for pred in predictions]) - - def testRegression(self): - """Tests regression using matrix data as input.""" - head_fn = head_lib._regression_head( - label_dimension=1, - loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS) - - hparams = tensor_forest.ForestHParams( - num_trees=5, - max_nodes=1000, - num_classes=1, - num_features=13, - regression=True, - split_after_samples=20) - - regressor = random_forest.CoreTensorForestEstimator( - hparams.fill(), head=head_fn) - - input_fn, predict_input_fn = _get_regression_input_fns() - - regressor.train(input_fn=input_fn, steps=100) - res = regressor.evaluate(input_fn=input_fn, steps=10) - self.assertGreaterEqual(0.1, res['loss']) - - predictions = list(regressor.predict(input_fn=predict_input_fn)) - self.assertAllClose( - [[24.]], [pred['predictions'] for pred in predictions], atol=1) - - def testWithFeatureColumns(self): - head_fn = head_lib._multi_class_head_with_softmax_cross_entropy_loss( - n_classes=3, loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS) - - hparams = tensor_forest.ForestHParams( - num_trees=3, - max_nodes=1000, - num_classes=3, - num_features=4, - split_after_samples=20, - inference_tree_paths=True) - - est = random_forest.CoreTensorForestEstimator( - hparams.fill(), - head=head_fn, - feature_columns=[core_feature_column.numeric_column('x')]) - - iris = base.load_iris() - data = {'x': iris.data.astype(np.float32)} - labels = iris.target.astype(np.int32) - - input_fn = numpy_io.numpy_input_fn( - x=data, y=labels, batch_size=150, num_epochs=None, shuffle=False) - - est.train(input_fn=input_fn, steps=100) - res = est.evaluate(input_fn=input_fn, steps=1) - - self.assertEqual(1.0, res['accuracy']) - self.assertAllClose(0.55144483, res['loss']) - - def testAutofillsClassificationHead(self): - hparams = tensor_forest.ForestHParams( - num_trees=3, - max_nodes=1000, - num_classes=3, - num_features=4, - split_after_samples=20, - inference_tree_paths=True) - - est = random_forest.CoreTensorForestEstimator(hparams.fill()) - - input_fn, _ = _get_classification_input_fns() - - est.train(input_fn=input_fn, steps=100) - res = est.evaluate(input_fn=input_fn, steps=1) - - self.assertEqual(1.0, res['accuracy']) - self.assertAllClose(0.55144483, res['loss']) - - def testAutofillsRegressionHead(self): - hparams = tensor_forest.ForestHParams( - num_trees=5, - max_nodes=1000, - num_classes=1, - num_features=13, - regression=True, - split_after_samples=20) - - regressor = random_forest.CoreTensorForestEstimator(hparams.fill()) - - input_fn, predict_input_fn = _get_regression_input_fns() - - regressor.train(input_fn=input_fn, steps=100) - res = regressor.evaluate(input_fn=input_fn, steps=10) - self.assertGreaterEqual(0.1, res['loss']) - - predictions = list(regressor.predict(input_fn=predict_input_fn)) - self.assertAllClose( - [[24.]], [pred['predictions'] for pred in predictions], atol=1) - - def testAdditionalOutputs(self): - """Tests multi-class classification using matrix data as input.""" - hparams = tensor_forest.ForestHParams( - num_trees=1, - max_nodes=100, - num_classes=3, - num_features=4, - split_after_samples=20, - inference_tree_paths=True) - classifier = random_forest.CoreTensorForestEstimator( - hparams.fill(), keys_column='keys', include_all_in_serving=True) - - iris = base.load_iris() - data = iris.data.astype(np.float32) - labels = iris.target.astype(np.int32) - - input_fn = numpy_io.numpy_input_fn( - x={ - 'x': data, - 'keys': np.arange(len(iris.data)).reshape(150, 1) - }, - y=labels, - batch_size=10, - num_epochs=1, - shuffle=False) - - classifier.train(input_fn=input_fn, steps=100) - predictions = list(classifier.predict(input_fn=input_fn)) - # Check that there is a key column, tree paths and var. - for pred in predictions: - self.assertTrue('keys' in pred) - self.assertTrue('tree_paths' in pred) - self.assertTrue('prediction_variance' in pred) - - def _assert_checkpoint(self, model_dir, global_step): - reader = checkpoint_utils.load_checkpoint(model_dir) - self.assertLessEqual( - reader.get_tensor(ops.GraphKeys.GLOBAL_STEP), global_step) - - def testEarlyStopping(self): - head_fn = head_lib._multi_class_head_with_softmax_cross_entropy_loss( - n_classes=3, loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS) - - hparams = tensor_forest.ForestHParams( - num_trees=3, - max_nodes=1000, - num_classes=3, - num_features=4, - split_after_samples=20, - inference_tree_paths=True) - - est = random_forest.CoreTensorForestEstimator( - hparams.fill(), - head=head_fn, - # Set a crazy threshold - 30% loss change. - early_stopping_loss_threshold=0.3, - early_stopping_rounds=2) - - input_fn, _ = _get_classification_input_fns() - est.train(input_fn=input_fn, steps=100) - # We stopped early. - self._assert_checkpoint(est.model_dir, global_step=8) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/tensor_forest/hybrid/BUILD b/tensorflow/contrib/tensor_forest/hybrid/BUILD deleted file mode 100644 index f21a0f695e8..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/BUILD +++ /dev/null @@ -1,371 +0,0 @@ -# TensorFlow code for training hybrid neural network / decision tree models. - -load("//tensorflow:tensorflow.bzl", "py_test") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_library") -load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py") -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -filegroup( - name = "custom_op_sources", - srcs = glob( - ["core/ops/*.cc"], - exclude = ["core/ops/*_test.cc"], - ), -) - -filegroup( - name = "custom_op_headers", - srcs = glob(["core/ops/*.h"]), -) - -cc_library( - name = "all_kernels", - srcs = [ - "core/ops/hard_routing_function_op.cc", - "core/ops/k_feature_gradient_op.cc", - "core/ops/k_feature_routing_function_op.cc", - "core/ops/routing_function_op.cc", - "core/ops/routing_gradient_op.cc", - "core/ops/stochastic_hard_routing_function_op.cc", - "core/ops/stochastic_hard_routing_gradient_op.cc", - "core/ops/unpack_path_op.cc", - ], - deps = [ - ":utils", - "//tensorflow/contrib/tensor_forest:tree_utils", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - ], - alwayslink = 1, -) - -tf_gen_op_wrapper_py( - name = "training_ops", - deps = [":all_kernels"], -) - -tf_custom_op_library( - name = "python/ops/_training_ops.so", - srcs = [ - "core/ops/hard_routing_function_op.cc", - "core/ops/k_feature_gradient_op.cc", - "core/ops/k_feature_routing_function_op.cc", - "core/ops/routing_function_op.cc", - "core/ops/routing_gradient_op.cc", - "core/ops/stochastic_hard_routing_function_op.cc", - "core/ops/stochastic_hard_routing_gradient_op.cc", - "core/ops/unpack_path_op.cc", - ], - deps = [ - ":utils", - "//tensorflow/contrib/tensor_forest:tree_utils", - ], -) - -cc_library( - name = "utils", - srcs = ["core/ops/utils.cc"], - hdrs = ["core/ops/utils.h"], - deps = [ - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - "@com_google_protobuf//:protobuf_headers", - ], -) - -tf_custom_op_py_library( - name = "ops_lib", - srcs = [ - "__init__.py", - "python/ops/training_ops.py", - ], - dso = ["python/ops/_training_ops.so"], - kernels = [ - ":all_kernels", - ], - srcs_version = "PY2AND3", - deps = [ - ":training_ops", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - ], -) - -py_library( - name = "all_layers", - deps = [ - ":decisions_to_data_layer", - ":fully_connected_layer", - ], -) - -py_library( - name = "hybrid_layer", - srcs = [ - "python/hybrid_layer.py", - ], - srcs_version = "PY2AND3", - deps = ["//tensorflow/contrib/framework:framework_py"], -) - -py_test( - name = "hybrid_layer_test", - size = "small", - srcs = ["python/hybrid_layer_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":fully_connected_layer", - ":hybrid_model", - "//tensorflow/contrib/tensor_forest:tensor_forest_py", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -py_library( - name = "hybrid_model", - srcs = ["python/hybrid_model.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], -) - -py_library( - name = "fully_connected_layer", - srcs = ["python/layers/fully_connected.py"], - srcs_version = "PY2AND3", - deps = [ - ":hybrid_layer", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - ], -) - -py_test( - name = "routing_function_op_test", - size = "small", - srcs = ["python/kernel_tests/routing_function_op_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["manual"], - deps = [ - ":ops_lib", - ":training_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -py_test( - name = "k_feature_routing_function_op_test", - size = "small", - srcs = ["python/kernel_tests/k_feature_routing_function_op_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["manual"], - deps = [ - ":ops_lib", - ":training_ops", - "//tensorflow/contrib/tensor_forest:tensor_forest_py", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - ], -) - -py_library( - name = "decisions_to_data_layer", - srcs = ["python/layers/decisions_to_data.py"], - srcs_version = "PY2AND3", - deps = [ - ":hybrid_layer", - ":ops_lib", - ":training_ops", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", - "//tensorflow/python:variable_scope", - ], -) - -py_test( - name = "decisions_to_data_test", - srcs = ["python/layers/decisions_to_data_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":decisions_to_data_layer", - "//tensorflow/contrib/tensor_forest:tensor_forest_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:variable_scope", - ], -) - -py_library( - name = "decisions_to_data_then_nn", - srcs = ["python/models/decisions_to_data_then_nn.py"], - srcs_version = "PY2AND3", - deps = [ - ":decisions_to_data_layer", - ":fully_connected_layer", - ":hybrid_model", - "//tensorflow/python:training", - ], -) - -py_library( - name = "hard_decisions_to_data_then_nn", - srcs = ["python/models/hard_decisions_to_data_then_nn.py"], - srcs_version = "PY2AND3", - deps = [ - ":decisions_to_data_layer", - ":fully_connected_layer", - ":hybrid_model", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:nn_ops", - "//tensorflow/python:training", - ], -) - -py_test( - name = "decisions_to_data_then_nn_test", - size = "small", - srcs = ["python/models/decisions_to_data_then_nn_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":decisions_to_data_then_nn", - "//tensorflow/contrib/tensor_forest:tensor_forest_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:variable_scope", - ], -) - -py_library( - name = "k_feature_decisions_to_data_then_nn", - srcs = ["python/models/k_feature_decisions_to_data_then_nn.py"], - srcs_version = "PY2AND3", - deps = [ - ":decisions_to_data_layer", - ":fully_connected_layer", - ":hybrid_model", - "//tensorflow/python:training", - ], -) - -py_test( - name = "k_feature_decisions_to_data_then_nn_test", - size = "small", - srcs = ["python/models/k_feature_decisions_to_data_then_nn_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":k_feature_decisions_to_data_then_nn", - "//tensorflow/contrib/tensor_forest:tensor_forest_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:variable_scope", - ], -) - -py_library( - name = "forest_to_data_then_nn", - srcs = ["python/models/forest_to_data_then_nn.py"], - srcs_version = "PY2AND3", - deps = [ - ":decisions_to_data_layer", - ":fully_connected_layer", - ":hybrid_model", - "//tensorflow/python:training", - ], -) - -py_test( - name = "forest_to_data_then_nn_test", - size = "small", - srcs = ["python/models/forest_to_data_then_nn_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":forest_to_data_then_nn", - "//tensorflow/contrib/tensor_forest:tensor_forest_py", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:variable_scope", - ], -) - -py_library( - name = "nn", - srcs = ["python/models/nn.py"], - srcs_version = "PY2AND3", - deps = [ - ":fully_connected_layer", - ":hybrid_model", - "//tensorflow/python:training", - ], -) - -py_library( - name = "stochastic_hard_decisions_to_data_then_nn", - srcs = ["python/models/stochastic_hard_decisions_to_data_then_nn.py"], - srcs_version = "PY2AND3", - deps = [ - ":decisions_to_data_layer", - ":fully_connected_layer", - ":hard_decisions_to_data_then_nn", - "//tensorflow/python:training", - ], -) - -py_library( - name = "stochastic_soft_decisions_to_data_then_nn", - srcs = ["python/models/stochastic_soft_decisions_to_data_then_nn.py"], - srcs_version = "PY2AND3", - deps = [ - ":decisions_to_data_layer", - ":fully_connected_layer", - ":hard_decisions_to_data_then_nn", - "//tensorflow/python:training", - ], -) - -# Transitive dependencies of this target will be included in the pip package. -py_library( - name = "hybrid_pip", - deps = [ - ":all_layers", - ":decisions_to_data_then_nn", - ":forest_to_data_then_nn", - ":hard_decisions_to_data_then_nn", - ":k_feature_decisions_to_data_then_nn", - ":nn", - ":stochastic_hard_decisions_to_data_then_nn", - ":stochastic_soft_decisions_to_data_then_nn", - ], -) diff --git a/tensorflow/contrib/tensor_forest/hybrid/__init__.py b/tensorflow/contrib/tensor_forest/hybrid/__init__.py deleted file mode 100644 index 2a65ab2f138..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Initialize tensor_forest/hybrid.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.tensor_forest.hybrid.python import * -# pylint: enable=unused-import,wildcard-import diff --git a/tensorflow/contrib/tensor_forest/hybrid/core/ops/hard_routing_function_op.cc b/tensorflow/contrib/tensor_forest/hybrid/core/ops/hard_routing_function_op.cc deleted file mode 100644 index 06bfe871fdf..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/core/ops/hard_routing_function_op.cc +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -// RoutingFunction returns the probability of reaching each leaf node -// in a soft decision tree. - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.h" -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/gtl/top_n.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -using tensorforest::CheckTensorBounds; -using tensorforest::LeftProbability; - -// The term 'routing function' is synonymous with 'the probability -// that an instance is routed to each leaf node.' It is defined in -// 'Deep Neural Decision Forests' by Kontschieder et al. -REGISTER_OP("HardRoutingFunction") - .Attr("max_nodes: int") - .Attr("tree_depth: int") - .Input("input_data: float") - .Input("tree_parameters: float") - .Input("tree_biases: float") - .Output("path_probability: float") - .Output("path: int32") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle input; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &input)); - int64 tree_depth; - TF_RETURN_IF_ERROR(c->GetAttr("tree_depth", &tree_depth)); - - auto out = c->Matrix(c->Dim(input, 0), tree_depth); - c->set_output(0, out); - c->set_output(1, out); - return Status::OK(); - }) - .Doc(R"doc( - Chooses a single path for each instance in `input_data` and returns the leaf - the probability of the path and the path taken. - - tree_depth: The depth of the decision tree. - - input_data: The training batch's features as a 2-d tensor; `input_data[i][j]` - gives the j-th feature of the i-th input. - tree_parameters: `tree_parameters[i]` gives the weight of - the logistic regression model that translates from node features to - probabilities. - tree_biases: `tree_biases[i]` gives the bias of the logistic - regression model that translates from node features to - probabilities. - - path_probability: `path_probability[i]` gives the probability of reaching each - node in `path[i]`. - path: `path[i][j]` gives the jth node in the path taken by the ith data - instance. -)doc"); - -class HardRoutingFunction : public OpKernel { - public: - explicit HardRoutingFunction(OpKernelConstruction* context) - : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr("tree_depth", &tree_depth_)); - } - - void Compute(OpKernelContext* context) override { - const Tensor& input_data = context->input(0); - const Tensor& tree_parameters_tensor = context->input(1); - const Tensor& tree_biases_tensor = context->input(2); - - if (input_data.shape().dim_size(0) > 0) { - OP_REQUIRES( - context, input_data.shape().dims() == 2, - errors::InvalidArgument("input_data should be two-dimensional")); - } - - // Check tensor bounds. - if (!CheckTensorBounds(context, input_data)) return; - - const int32 num_data = static_cast(input_data.shape().dim_size(0)); - const int32 num_features = - static_cast(input_data.shape().dim_size(1)); - - Tensor* output_probability = nullptr; - TensorShape output_probability_shape; - output_probability_shape.AddDim(num_data); - output_probability_shape.AddDim(tree_depth_); - - Tensor* output_path = nullptr; - TensorShape output_path_shape; - output_path_shape.AddDim(num_data); - output_path_shape.AddDim(tree_depth_); - - OP_REQUIRES_OK(context, - context->allocate_output(0, output_probability_shape, - &output_probability)); - OP_REQUIRES_OK( - context, context->allocate_output(1, output_path_shape, &output_path)); - - auto out_probability = output_probability->tensor(); - auto out_path = output_path->tensor(); - - const auto data = input_data.tensor(); - const auto tree_parameters = tree_parameters_tensor.tensor(); - const auto tree_biases = tree_biases_tensor.tensor(); - - // Deterministically traverse the tree to a leaf. - for (int i = 0; i < num_data; i++) { - const Tensor point = input_data.Slice(i, i + 1); - int32 node = 0; - - out_probability(i, 0) = 1.0; - out_path(i, 0) = 0; - for (int j = 0; j < tree_depth_ - 1; j++) { - float left_prob = - LeftProbability(point, tree_parameters_tensor.Slice(j, j + 1), - tree_biases(j), num_features); - - int32 left_child = 2 * node + 1; - int32 right_child = left_child + 1; - - float dot_product = 0.0; - for (int k = 0; k < num_features; k++) { - dot_product += tree_parameters(j, k) * data(i, k); - } - if (dot_product < tree_biases(j)) { - out_probability(i, j + 1) = left_prob * out_probability(i, j); - out_path(i, j + 1) = left_child; - node = left_child; - } else { - out_probability(i, j + 1) = (1.0 - left_prob) * out_probability(i, j); - out_path(i, j + 1) = right_child; - node = right_child; - } - } - } - } - - private: - int32 tree_depth_; -}; - -REGISTER_KERNEL_BUILDER(Name("HardRoutingFunction").Device(DEVICE_CPU), - HardRoutingFunction); -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/hybrid/core/ops/k_feature_gradient_op.cc b/tensorflow/contrib/tensor_forest/hybrid/core/ops/k_feature_gradient_op.cc deleted file mode 100644 index f64155fa55a..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/core/ops/k_feature_gradient_op.cc +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.h" -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/gtl/top_n.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -using tensorforest::LeftProbabilityK; - -REGISTER_OP("KFeatureGradient") - .Attr("layer_num: int") - .Attr("random_seed: int") - .Input("input_data: float") - .Input("tree_parameters: float") - .Input("tree_biases: float") - .Input("routes: float") - .Output("routing_gradient: float") - .Output("data_gradient: float") - .Output("weight_gradient: float") - .Doc(R"doc( - Computes the derivative of the routing loss with respect to each decision - node. Each decision node is constrained to make a decision based on only - k features. - - layer_num: The layer number of this tree. - random_seed: The base random seed. - - input_data: The training batch's features as a 2-d tensor; - `input_data[i][j]` gives the j-th feature of the i-th input. - tree_parameters: `tree_parameters[i]` gives the weight of - the logistic regression model that translates from node features to - probabilities. - tree_biases: `tree_biases[i]` gives the bias of the logistic - regression model that translates from node features to - probabilities. - routes: The routes computed by routing_function_op. - - routing_gradient: `routing_gradient` provides du / df, where u is the - routing function and f is the (vector of) decision functions. A decision - function f_i computes the routing decision at node i. - - data_gradient: `data_gradient` provides df / dx, where f is the (vector - of) decision functions and x is a batch of data. - - weights_gradient: `weights_gradient` provides df / dw, where f is the - (vector of) decision functions and w is the matrix of parameters that - determine how instances are routed through a tree. - - f_i, the decision function at node i, is parameterized by t_i (parameters) - and b_i (bias) and takes data x as input. This op is called in - training_ops.py to compute du / df, and we use that to compute - - du / dx = du / df * df / dx, - du / dt = du / df * df / dt, and - du / db = du / df * df / db. -)doc"); - -class KFeatureGradient : public OpKernel { - public: - explicit KFeatureGradient(OpKernelConstruction* context) : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr("layer_num", &layer_num_)); - OP_REQUIRES_OK(context, context->GetAttr("random_seed", &random_seed_)); - } - - void Compute(OpKernelContext* context) override { - // Gather input. - const Tensor& input_data_tensor = context->input(0); - const Tensor& tree_parameters_tensor = context->input(1); - const Tensor& tree_biases_tensor = context->input(2); - const Tensor& routing_tensor = context->input(3); - - // Extract dimensions from input tensors. - const int32 num_data = - static_cast(input_data_tensor.shape().dim_size(0)); - const int32 num_features = - static_cast(input_data_tensor.shape().dim_size(1)); - const int32 num_nodes = - static_cast(tree_parameters_tensor.shape().dim_size(0)); - const int32 num_features_per_node = - static_cast(tree_parameters_tensor.shape().dim_size(1)); - - // Construct output tensors. - Tensor* out_routes = nullptr; - TensorShape out_routes_shape; - out_routes_shape.AddDim(num_data); - out_routes_shape.AddDim(num_nodes); - - Tensor* out_data = nullptr; - TensorShape out_data_shape; - out_data_shape.AddDim(num_nodes); - out_data_shape.AddDim(num_features); - - Tensor* out_weights = nullptr; - TensorShape out_weights_shape; - out_weights_shape.AddDim(num_data); - out_weights_shape.AddDim(num_nodes); - out_weights_shape.AddDim(num_features_per_node); - - OP_REQUIRES_OK(context, - context->allocate_output(0, out_routes_shape, &out_routes)); - OP_REQUIRES_OK(context, - context->allocate_output(1, out_data_shape, &out_data)); - OP_REQUIRES_OK( - context, context->allocate_output(2, out_weights_shape, &out_weights)); - - tensorforest::Initialize(*out_data, 0.0f); - - // Compute output. - const auto input_data = input_data_tensor.tensor(); - const auto tree_parameters = tree_parameters_tensor.tensor(); - const auto tree_biases = tree_biases_tensor.tensor(); - const auto routes = routing_tensor.tensor(); - - auto routes_grad = out_routes->tensor(); - auto data_grad = out_data->tensor(); - auto weights_grad = out_weights->tensor(); - - std::vector feature_set; - for (int i = 0; i < num_data; i++) { - const Tensor point = input_data_tensor.Slice(i, i + 1); - feature_set.clear(); - - // Traverse the tree from the bottom up. - for (int j = num_nodes - 1; j >= 0; j--) { - tensorforest::GetFeatureSet(layer_num_, j, random_seed_, num_features, - num_features_per_node, &feature_set); - - // Compute routing gradient. - // j is a leaf node. - if (j >= num_nodes / 2) { - routes_grad(i, j) = routes(i, j); - } else { // j is not a leaf node - int32 left_child = 2 * j + 1; - int32 right_child = left_child + 1; - - float left_prob = LeftProbabilityK( - point, feature_set, tree_parameters_tensor.Slice(j, j + 1), - tree_biases(j), num_features, num_features_per_node); - - float right_prob = 1.0f - left_prob; - - routes_grad(i, j) = (right_prob * routes(i, left_child) + - left_prob * routes(i, right_child)); - } - // Compute data and weight gradient. - for (int k = 0; k < num_features_per_node; k++) { - CHECK_LT(feature_set[k], num_features); - data_grad(j, feature_set[k]) = tree_parameters(j, k); - weights_grad(i, j, k) = input_data(i, feature_set[k]); - } - } - } - } - - private: - int32 layer_num_; - int32 random_seed_; -}; - -REGISTER_KERNEL_BUILDER(Name("KFeatureGradient").Device(DEVICE_CPU), - KFeatureGradient); -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/hybrid/core/ops/k_feature_routing_function_op.cc b/tensorflow/contrib/tensor_forest/hybrid/core/ops/k_feature_routing_function_op.cc deleted file mode 100644 index e7cafb144da..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/core/ops/k_feature_routing_function_op.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -// RoutingFunction returns the probability of reaching each leaf node -// in a soft decision tree. - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.h" -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/gtl/top_n.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -using tensorforest::CheckTensorBounds; -using tensorforest::LeftProbabilityK; - -// The term 'routing function' is synonymous with 'the probability -// that an instance is routed to each leaf node.' It is defined in -// 'Deep Neural Decision Forests' by Kontschieder et al. -REGISTER_OP("KFeatureRoutingFunction") - .Attr("layer_num: int") - .Attr("max_nodes: int") - .Attr("num_features_per_node: int") - .Attr("random_seed: int") - .Input("input_data: float") - .Input("tree_parameters: float") - .Input("tree_biases: float") - .Output("probabilities: float") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle input, params; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &input)); - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, ¶ms)); - - c->set_output(0, c->Matrix(c->Dim(input, 0), c->Dim(params, 0))); - return Status::OK(); - }) - .Doc(R"doc( - - Returns the probability that each input will reach each leaf node. Each - decision is made based on k features. - - layer_num: The layer number of this tree. - max_nodes: The number of nodes in the tree. - num_features_per_node: The number of features each node can use to make a - decision. - random_seed: The base random seed. - - input_data: The training batch's features as a 2-d tensor; `input_data[i][j]` - gives the j-th feature of the i-th input. - tree_parameters: `tree_parameters[i]` gives the weight of - the logistic regression model that translates from node features to - probabilities. - tree_biases: `tree_biases[i]` gives the bias of the logistic - regression model that translates from node features to - probabilities. - tree_features: `tree_features[i]` gives the decision feature for node i. - - probabilities: `probabilities[i][j]` is the probability that input i - will reach node j. -)doc"); - -class KFeatureRoutingFunction : public OpKernel { - public: - explicit KFeatureRoutingFunction(OpKernelConstruction* context) - : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr("max_nodes", &max_nodes_)); - OP_REQUIRES_OK(context, context->GetAttr("num_features_per_node", - &num_features_per_node_)); - OP_REQUIRES_OK(context, context->GetAttr("layer_num", &layer_num_)); - OP_REQUIRES_OK(context, context->GetAttr("random_seed", &random_seed_)); - } - - void Compute(OpKernelContext* context) override { - const Tensor& input_data = context->input(0); - const Tensor& tree_parameters_tensor = context->input(1); - const Tensor& tree_biases_tensor = context->input(2); - - if (input_data.shape().dim_size(0) > 0) { - OP_REQUIRES( - context, input_data.shape().dims() == 2, - errors::InvalidArgument("input_data should be two-dimensional")); - } - - // Check tensor bounds. - if (!CheckTensorBounds(context, input_data)) return; - - const int32 num_data = static_cast(input_data.shape().dim_size(0)); - const int32 num_features = - static_cast(input_data.shape().dim_size(1)); - - Tensor* output_probabilities = nullptr; - TensorShape output_shape; - output_shape.AddDim(num_data); - output_shape.AddDim(max_nodes_); - - OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, - &output_probabilities)); - - auto out_probs = output_probabilities->tensor(); - const auto tree_biases = tree_biases_tensor.tensor(); - - // Iteratively compute the probability of reaching each leaf. - std::vector feature_set; - for (int i = 0; i < num_data; i++) { - const Tensor point = input_data.Slice(i, i + 1); - - out_probs(i, 0) = 1.0f; - - for (int j = 0; j < max_nodes_ / 2; j++) { - feature_set.clear(); - tensorforest::GetFeatureSet(layer_num_, i, random_seed_, num_features, - num_features_per_node_, &feature_set); - - int32 left_child = 2 * j + 1; - int32 right_child = left_child + 1; - - float prob = out_probs(i, j); - float left_prob = LeftProbabilityK( - point, feature_set, tree_parameters_tensor.Slice(j, j + 1), - tree_biases(j), num_features, num_features_per_node_); - - out_probs(i, left_child) = prob * left_prob; - out_probs(i, right_child) = prob * (1.0f - left_prob); - } - } - } - - private: - int32 layer_num_; - int32 max_nodes_; - int32 num_features_per_node_; - int32 random_seed_; -}; - -REGISTER_KERNEL_BUILDER(Name("KFeatureRoutingFunction").Device(DEVICE_CPU), - KFeatureRoutingFunction); -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/hybrid/core/ops/routing_function_op.cc b/tensorflow/contrib/tensor_forest/hybrid/core/ops/routing_function_op.cc deleted file mode 100644 index 0c2eaabe8f3..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/core/ops/routing_function_op.cc +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -// RoutingFunction returns the probability of reaching each leaf node -// in a soft decision tree. - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.h" -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/gtl/top_n.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -using tensorforest::CheckTensorBounds; -using tensorforest::LeftProbability; - -// The term 'routing function' is synonymous with 'the probability -// that an instance is routed to each leaf node.' It is defined in -// 'Deep Neural Decision Forests' by Kontschieder et al. -REGISTER_OP("RoutingFunction") - .Attr("max_nodes: int") - .Input("input_data: float") - .Input("tree_parameters: float") - .Input("tree_biases: float") - .Output("probabilities: float") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle input, params; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &input)); - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, ¶ms)); - - c->set_output(0, c->Matrix(c->Dim(input, 0), c->Dim(params, 0))); - return Status::OK(); - }) - .Doc(R"doc( - Returns the probability that each input will reach each leaf node. - - max_nodes: The number of nodes in the tree. - - input_data: The training batch's features as a 2-d tensor; `input_data[i][j]` - gives the j-th feature of the i-th input. - tree_parameters: `tree_parameters[i]` gives the weight of - the logistic regression model that translates from node features to - probabilities. - tree_biases: `tree_biases[i]` gives the bias of the logistic - regression model that translates from node features to - probabilities. - - probabilities: `probabilities[i][j]` is the probability that input i - will reach node j. -)doc"); - -class RoutingFunction : public OpKernel { - public: - explicit RoutingFunction(OpKernelConstruction* context) : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr("max_nodes", &max_nodes_)); - } - - void Compute(OpKernelContext* context) override { - const Tensor& input_data = context->input(0); - const Tensor& tree_parameters_tensor = context->input(1); - const Tensor& tree_biases_tensor = context->input(2); - - if (input_data.shape().dim_size(0) > 0) { - OP_REQUIRES( - context, input_data.shape().dims() == 2, - errors::InvalidArgument("input_data should be two-dimensional")); - } - - // Check tensor bounds. - if (!CheckTensorBounds(context, input_data)) return; - - const int32 num_data = static_cast(input_data.shape().dim_size(0)); - const int32 num_features = - static_cast(input_data.shape().dim_size(1)); - - Tensor* output_probabilities = nullptr; - TensorShape output_shape; - output_shape.AddDim(num_data); - output_shape.AddDim(max_nodes_); - - OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, - &output_probabilities)); - - auto out_probs = output_probabilities->tensor(); - const auto tree_biases = tree_biases_tensor.tensor(); - - // Iteratively compute the probability of reaching each leaf. - for (int i = 0; i < num_data; i++) { - const Tensor point = input_data.Slice(i, i + 1); - - out_probs(i, 0) = 1.0; - - for (int j = 0; j < max_nodes_ / 2; j++) { - int32 left_child = 2 * j + 1; - int32 right_child = left_child + 1; - - float prob = out_probs(i, j); - float left_prob = - LeftProbability(point, tree_parameters_tensor.Slice(j, j + 1), - tree_biases(j), num_features); - - out_probs(i, left_child) = prob * left_prob; - out_probs(i, right_child) = prob * (1.0 - left_prob); - } - } - } - - private: - int32 max_nodes_; -}; - -REGISTER_KERNEL_BUILDER(Name("RoutingFunction").Device(DEVICE_CPU), - RoutingFunction); -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/hybrid/core/ops/routing_gradient_op.cc b/tensorflow/contrib/tensor_forest/hybrid/core/ops/routing_gradient_op.cc deleted file mode 100644 index 5aca54d131c..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/core/ops/routing_gradient_op.cc +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.h" -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/gtl/top_n.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -using tensorforest::LeftProbability; - -// This op computes the derivative of the routing loss with respect to each -// decision node. -REGISTER_OP("RoutingGradient") - .Attr("max_nodes: int") - .Input("input_data: float") - .Input("tree_parameters: float") - .Input("tree_biases: float") - .Input("routes: float") - .Output("routing_gradient: float") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle input, params; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &input)); - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, ¶ms)); - - c->set_output(0, c->Matrix(c->Dim(input, 0), c->Dim(params, 0))); - return Status::OK(); - }) - .Doc(R"doc( - Computes the derivative of the routing loss with respect to each decision - node. - - max_nodes: The number of nodes in the tree. - - tree_parameters: `tree_parameters[i]` gives the weight of - the logistic regression model that translates from node features to - probabilities. - tree_biases: `tree_biases[i]` gives the bias of the logistic - regression model that translates from node features to - probabilities. - routes: The routes computed by routing_function_op. - - routing_gradient: `routing_gradient` provides du / df, where u is the routing - function and f is the (vector of) decision functions. A decision function - f_i computes the routing decision at node i. - - f_i is parameterized by t_i (parameters) and b_i (bias) and takes data x as - input. This op is called in training_ops.py to compute du / df, and we use - that to compute - - du / dx = du / df * df / dx, - du / dt = du / df * df / dt, and - du / db = du / df * df / db. -)doc"); - -class RoutingGradient : public OpKernel { - public: - explicit RoutingGradient(OpKernelConstruction* context) : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr("max_nodes", &max_nodes_)); - } - - void Compute(OpKernelContext* context) override { - const Tensor& input_data = context->input(0); - const Tensor& tree_parameters_tensor = context->input(1); - const Tensor& tree_biases_tensor = context->input(2); - const Tensor& routing_tensor = context->input(3); - - // TODO(atwoodj): Add dimension checks. - - const int32 num_data = static_cast(input_data.shape().dim_size(0)); - const int32 num_features = - static_cast(input_data.shape().dim_size(1)); - - Tensor* output = nullptr; - TensorShape output_shape; - output_shape.AddDim(num_data); - output_shape.AddDim(max_nodes_); - - OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, &output)); - - auto out = output->tensor(); - const auto tree_biases = tree_biases_tensor.tensor(); - const auto routes = routing_tensor.tensor(); - - // A derivation of the gradient can be found at go/routingderivation. - for (int i = 0; i < num_data; i++) { - const Tensor point = input_data.Slice(i, i + 1); - - // Traverses the tree from the bottom up. - for (int j = max_nodes_ - 1; j >= 0; j--) { - // j is a leaf node - if (j >= max_nodes_ / 2) { - out(i, j) = routes(i, j); - } else { // j is not a leaf node - int32 left_child = 2 * j + 1; - int32 right_child = left_child + 1; - float left_prob = - LeftProbability(point, tree_parameters_tensor.Slice(j, j + 1), - tree_biases(j), num_features); - - float right_prob = 1 - left_prob; - - out(i, j) = (right_prob * routes(i, left_child) + - left_prob * routes(i, right_child)); - } - } - } - } - - private: - int32 max_nodes_; -}; - -REGISTER_KERNEL_BUILDER(Name("RoutingGradient").Device(DEVICE_CPU), - RoutingGradient); -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_function_op.cc b/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_function_op.cc deleted file mode 100644 index 1a055756c08..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_function_op.cc +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -// RoutingFunction returns the probability of reaching each leaf node -// in a soft decision tree. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.h" -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" - -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/gtl/top_n.h" -#include "tensorflow/core/lib/random/philox_random.h" -#include "tensorflow/core/lib/random/simple_philox.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -using tensorforest::CheckTensorBounds; -using tensorforest::LeftProbability; - -// The term 'routing function' is synonymous with 'the probability -// that an instance is routed to each leaf node.' It is defined in -// 'Deep Neural Decision Forests' by Kontschieder et al. -REGISTER_OP("StochasticHardRoutingFunction") - .Attr("tree_depth: int") - .Attr("random_seed: int") - .Input("input_data: float") - .Input("tree_parameters: float") - .Input("tree_biases: float") - .Output("path_probability: float") - .Output("path: int32") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle input; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &input)); - int64 tree_depth; - TF_RETURN_IF_ERROR(c->GetAttr("tree_depth", &tree_depth)); - - auto out = c->Matrix(c->Dim(input, 0), tree_depth); - c->set_output(0, out); - c->set_output(1, out); - return Status::OK(); - }) - .Doc(R"doc( - Samples a path for each instance in `input_data` and returns the - probability of the path and the path taken. - - tree_depth: The depth of the decision tree. - random_seed: The base random seed. - - input_data: The training batch's features as a 2-d tensor; `input_data[i][j]` - gives the j-th feature of the i-th input. - tree_parameters: `tree_parameters[i]` gives the weight of - the logistic regression model that translates from node features to - probabilities. - tree_biases: `tree_biases[i]` gives the bias of the logistic - regression model that translates from node features to - probabilities. - - path_probability: `path_probability[i]` gives the probability of reaching each - node in `path[i]`. - path: `path[i][j]` gives the jth node in the path taken by the ith data - instance. -)doc"); - -class StochasticHardRoutingFunction : public OpKernel { - public: - explicit StochasticHardRoutingFunction(OpKernelConstruction* context) - : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr("tree_depth", &tree_depth_)); - OP_REQUIRES_OK(context, context->GetAttr("random_seed", &random_seed_)); - single_rand_ = std::unique_ptr( - new random::PhiloxRandom(random_seed_)); - rng_ = std::unique_ptr( - new random::SimplePhilox(single_rand_.get())); - } - - void Compute(OpKernelContext* context) override { - VLOG(1) << "stochastic routing start"; - const Tensor& input_data = context->input(0); - const Tensor& tree_parameters_tensor = context->input(1); - const Tensor& tree_biases_tensor = context->input(2); - - if (input_data.shape().dim_size(0) > 0) { - OP_REQUIRES( - context, input_data.shape().dims() == 2, - errors::InvalidArgument("input_data should be two-dimensional")); - } - - // Check tensor bounds. - if (!CheckTensorBounds(context, input_data)) return; - - const int32 num_data = static_cast(input_data.shape().dim_size(0)); - const int32 num_features = - static_cast(input_data.shape().dim_size(1)); - const int32 num_nodes = - static_cast(tree_parameters_tensor.shape().dim_size(0)); - - Tensor* output_probability = nullptr; - TensorShape output_probability_shape; - output_probability_shape.AddDim(num_data); - output_probability_shape.AddDim(tree_depth_); - - Tensor* output_path = nullptr; - TensorShape output_path_shape; - output_path_shape.AddDim(num_data); - output_path_shape.AddDim(tree_depth_); - - OP_REQUIRES_OK(context, - context->allocate_output(0, output_probability_shape, - &output_probability)); - OP_REQUIRES_OK( - context, context->allocate_output(1, output_path_shape, &output_path)); - - auto out_probability = output_probability->tensor(); - auto out_path = output_path->tensor(); - const auto tree_biases = tree_biases_tensor.tensor(); - - // Stochastically traverse the tree to a leaf. - - for (int i = 0; i < num_data; i++) { - const Tensor point = input_data.Slice(i, i + 1); - - int32 node = 0; - out_probability(i, 0) = 1.0; - out_path(i, 0) = 0; - for (int j = 0; j < tree_depth_ - 1; j++) { - int32 left_child = 2 * node + 1; - int32 right_child = left_child + 1; - - float left_prob = - LeftProbability(point, tree_parameters_tensor.Slice(j, j + 1), - tree_biases(j), num_features); - - if (left_prob < rng_->RandFloat()) { - CHECK_LT(i, num_data); - CHECK_LT(j + 1, tree_depth_); - CHECK_LT(left_child, num_nodes); - - out_probability(i, j + 1) = left_prob * out_probability(i, j); - out_path(i, j + 1) = left_child; - node = left_child; - } else { - CHECK_LT(i, num_data); - CHECK_LT(j + 1, tree_depth_); - CHECK_LT(right_child, num_nodes); - - out_probability(i, j + 1) = (1.0 - left_prob) * out_probability(i, j); - out_path(i, j + 1) = right_child; - node = right_child; - } - } - } - VLOG(1) << "stochastic routing end"; - } - - private: - std::unique_ptr single_rand_; - std::unique_ptr rng_; - int32 tree_depth_; - int32 random_seed_; -}; - -REGISTER_KERNEL_BUILDER( - Name("StochasticHardRoutingFunction").Device(DEVICE_CPU), - StochasticHardRoutingFunction); -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_gradient_op.cc b/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_gradient_op.cc deleted file mode 100644 index 7d092bbc24d..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_gradient_op.cc +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.h" -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/gtl/top_n.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -using tensorforest::LeftProbability; - -// This op computes the derivative of the routing loss with respect to each -// decision node. -REGISTER_OP("StochasticHardRoutingGradient") - .Attr("tree_depth: int") - .Input("input_data: float") - .Input("tree_parameters: float") - .Input("tree_biases: float") - .Input("path_probability: float") - .Input("path: int32") - .Output("routing_gradient: float") - .Output("data_gradient: float") - .Output("parameter_gradient: float") - .Output("bias_gradient: float") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle input, params; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 2, &input)); - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, ¶ms)); - - auto num_points = c->Dim(input, 0); - auto num_features = c->Dim(input, 1); - auto num_nodes = c->Dim(params, 0); - - c->set_output(0, c->Matrix(num_points, num_nodes)); - c->set_output(1, c->Matrix(num_nodes, num_features)); - c->set_output(2, c->MakeShape({num_points, num_nodes, num_features})); - c->set_output(3, c->Vector(num_nodes)); - return Status::OK(); - }) - .Doc(R"doc( - Computes the derivative of the routing loss with respect to each decision - node. - - tree_depth: The depth of the decision tree. - - input_data: The training batch's features as a 2-d tensor; `input_data[i][j]` - gives the j-th feature of the i-th input - tree_parameters: `tree_parameters[i]` gives the weight of - the logistic regression model that translates from node features to - probabilities. - tree_biases: `tree_biases[i]` gives the bias of the logistic - regression model that translates from node features to - probabilities. - path_probability: `path_probability[i]` gives the probability of reaching each - node in `path[i]`. - path: `path[i][j]` gives the jth node in the path taken by the ith data - instance. - - routing_gradient: `routing_gradient` provides du / df, where u is the routing - function and f is the (vector of) decision functions. A decision function - f_i computes the routing decision at node i. - data_gradient: `data_gradient` provides df / dx, where f is the (vector - of) decision functions and x is a batch of data. - parameter_gradient: `parameter_gradient` provides df / dw, where f is the - (vector of) decision functions and w is the matrix of parameters that - determine how instances are routed through a tree. - bias_gradient: `bias_gradient` provides df / db, where f is the - (vector of) decision functions and b is the vector of bias parameters that - determine how instances are routed through a tree. - - f_i is parameterized by t_i (parameters) and b_i (bias) and takes data x as - input. This op is called in training_ops.py to compute du / df, and we use - that to compute - - du / dx = du / df * df / dx, - du / dt = du / df * df / dt, and - du / db = du / df * df / db. -)doc"); - -class StochasticHardRoutingGradient : public OpKernel { - public: - explicit StochasticHardRoutingGradient(OpKernelConstruction* context) - : OpKernel(context) { - OP_REQUIRES_OK(context, context->GetAttr("tree_depth", &tree_depth_)); - } - - void Compute(OpKernelContext* context) override { - VLOG(1) << "stochastic gradient start"; - const Tensor& input_data = context->input(0); - const Tensor& tree_parameters_tensor = context->input(1); - const Tensor& tree_biases_tensor = context->input(2); - - const Tensor& path_probability_tensor = context->input(3); - const Tensor& path_tensor = context->input(4); - - const int32 num_data = static_cast(input_data.shape().dim_size(0)); - const int32 num_features = - static_cast(input_data.shape().dim_size(1)); - const int32 num_nodes = - static_cast(tree_parameters_tensor.shape().dim_size(0)); - - Tensor* output_routing = nullptr; - TensorShape output_routing_shape; - output_routing_shape.AddDim(num_data); - output_routing_shape.AddDim(num_nodes); - - Tensor* output_data = nullptr; - TensorShape output_data_shape; - output_data_shape.AddDim(num_nodes); - output_data_shape.AddDim(num_features); - - Tensor* output_parameters = nullptr; - TensorShape output_parameters_shape; - output_parameters_shape.AddDim(num_data); - output_parameters_shape.AddDim(num_nodes); - output_parameters_shape.AddDim(num_features); - - Tensor* output_bias = nullptr; - TensorShape output_bias_shape; - output_bias_shape.AddDim(num_data); - - OP_REQUIRES_OK(context, context->allocate_output(0, output_routing_shape, - &output_routing)); - OP_REQUIRES_OK( - context, context->allocate_output(1, output_data_shape, &output_data)); - OP_REQUIRES_OK(context, context->allocate_output(2, output_parameters_shape, - &output_parameters)); - OP_REQUIRES_OK( - context, context->allocate_output(3, output_bias_shape, &output_bias)); - - tensorforest::Initialize(*output_routing, 0.0); - tensorforest::Initialize(*output_data, 0.0); - tensorforest::Initialize(*output_parameters, 0.0); - tensorforest::Initialize(*output_bias, 0.0); - - auto out_routing = output_routing->tensor(); - auto out_data = output_data->tensor(); - auto out_parameters = output_parameters->tensor(); - auto out_bias = output_bias->tensor(); - - const auto data = input_data.tensor(); - const auto tree_parameters = tree_parameters_tensor.tensor(); - const auto tree_biases = tree_biases_tensor.tensor(); - const auto path_probability = path_probability_tensor.tensor(); - const auto path = path_tensor.tensor(); - - for (int i = 0; i < num_data; i++) { - const Tensor point = input_data.Slice(i, i + 1); - - // Traverses the tree from the bottom up. - for (int j = tree_depth_ - 1; j > -1; j--) { - int32 node = path(i, j); - - CHECK_LT(node, num_nodes); - CHECK_GT(node, -1); - - // Compute data, parameter, and bias gradients. - // TODO(atwoodj): Should these be normalized? Loss looks pretty large. - for (int k = 0; k < num_features; k++) { - out_data(node, k) = tree_parameters(node, k); - out_parameters(i, node, k) = out_parameters(i, node, k) + data(i, k); - } - out_bias(node) = out_bias(node) + 1.0; - - // Compute decision gradient. - // node is a leaf - if (node >= num_nodes / 2) { - CHECK_LT(node, num_nodes); - out_routing(i, node) = path_probability(i, j); - } else { // node is not a leaf - int32 left_child = 2 * j + 1; - - float left_prob = - LeftProbability(point, tree_parameters_tensor.Slice(j, j + 1), - tree_biases(j), num_features); - - float right_prob = 1 - left_prob; - - CHECK_GT(j - 1, -1); - if (path(i, j - 1) == left_child) { - CHECK_LT(node, num_nodes); - out_routing(i, node) = right_prob * path_probability(i, j - 1); - } else { - CHECK_LT(node, num_nodes); - out_routing(i, node) = left_prob * path_probability(i, j - 1); - } - } - } - } - VLOG(1) << "stochastic gradient end"; - } - - private: - int32 tree_depth_; -}; - -REGISTER_KERNEL_BUILDER( - Name("StochasticHardRoutingGradient").Device(DEVICE_CPU), - StochasticHardRoutingGradient); -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/hybrid/core/ops/unpack_path_op.cc b/tensorflow/contrib/tensor_forest/hybrid/core/ops/unpack_path_op.cc deleted file mode 100644 index 25825a78a14..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/core/ops/unpack_path_op.cc +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#include "tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.h" -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/lib/gtl/top_n.h" -#include "tensorflow/core/lib/math/math_util.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -using shape_inference::InferenceContext; -using shape_inference::ShapeHandle; - -REGISTER_OP("UnpackPath") - .Input("path: int32") - .Input("path_values: float") - .Output("unpacked_path: float") - .SetShapeFn([](InferenceContext* c) { - ShapeHandle input, params; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &input)); - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 2, ¶ms)); - - auto num_points = c->Dim(input, 0); - - auto tree_depth = c->Dim(params, 1); - int64 num_nodes = InferenceContext::kUnknownDim; - if (c->ValueKnown(tree_depth)) { - num_nodes = (static_cast(1) << c->Value(tree_depth)) - 1; - } - - c->set_output(0, c->Matrix(num_points, num_nodes)); - return Status::OK(); - }) - .Doc(R"doc( - Takes a batch of paths through a tree and a batch of values along those paths - and returns a batch_size by num_nodes encoding of the path values. - - path: `path[i][j]` gives the jth node in the path taken by the ith data - instance. - path_values: `path_values[i][j]` gives the value associated with node j in the - path defined by the ith instance - - unpacked_paths: `unpacked_paths[i][path[i][k]]` is path_values[i][k] for k in - [0, tree_depth). All other elements of unpacked_paths are zero. -)doc"); - -class UnpackPath : public OpKernel { - public: - explicit UnpackPath(OpKernelConstruction* context) : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - VLOG(1) << "unpack start"; - const Tensor& path_tensor = context->input(0); - const Tensor& path_values_tensor = context->input(1); - - const int32 num_data = static_cast(path_tensor.shape().dim_size(0)); - const int32 tree_depth = - static_cast(path_tensor.shape().dim_size(1)); - - const int32 num_nodes = MathUtil::IPow(2, tree_depth) - 1; - - VLOG(1) << "num_data: " << num_data; - VLOG(1) << "tree_depth: " << tree_depth; - VLOG(1) << "num_nodes: " << num_nodes; - - Tensor* output = nullptr; - TensorShape output_shape; - output_shape.AddDim(num_data); - output_shape.AddDim(num_nodes); - - OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, &output)); - VLOG(1) << "unpack before init"; - tensorforest::Initialize(*output, 0.0f); - VLOG(1) << "unpack after init"; - - auto out = output->tensor(); - - const auto path = path_tensor.tensor(); - const auto path_values = path_values_tensor.tensor(); - - for (int i = 0; i < num_data; i++) { - for (int j = 0; j < tree_depth; j++) { - CHECK_LT(path(i, j), num_nodes); - out(i, path(i, j)) = path_values(i, j); - } - } - VLOG(1) << "unpack end"; - } -}; - -REGISTER_KERNEL_BUILDER(Name("UnpackPath").Device(DEVICE_CPU), UnpackPath); - -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.cc b/tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.cc deleted file mode 100644 index c7c6a85bf6a..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.cc +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.h" - -#include -#include -#include - -#include "tensorflow/core/lib/random/philox_random.h" -#include "tensorflow/core/lib/random/simple_philox.h" - -namespace tensorflow { -namespace tensorforest { - -using tensorflow::Tensor; - -float LeftProbability(const Tensor& point, const Tensor& weight, float bias, - int num_features) { - const auto p = point.unaligned_flat(); - const auto w = weight.unaligned_flat(); - float dot_product = 0.0; - for (int i = 0; i < num_features; i++) { - dot_product += w(i) * p(i); - } - - // TODO(thomaswc): At some point we should consider - // //learning/logistic/logodds-to-prob.h - return 1.0 / (1.0 + std::exp(-dot_product + bias)); -} - -float LeftProbabilityK(const Tensor& point, std::vector feature_set, - const Tensor& weight, float bias, int num_features, - int k) { - const auto p = point.unaligned_flat(); - const auto w = weight.unaligned_flat(); - - float dot_product = 0.0; - - for (int32 i = 0; i < k; i++) { - CHECK_LT(feature_set[i], num_features); - dot_product += p(feature_set[i]) * w(i); - } - - // TODO(thomaswc): At some point we should consider - // //learning/logistic/logodds-to-prob.h - return 1.0 / (1.0 + std::exp(-dot_product + bias)); -} - -void GetFeatureSet(int32 tree_num, int32 node_num, int32 random_seed, - int32 num_features, int32 num_features_to_pick, - std::vector* features) { - features->clear(); - uint64 seed = node_num ^ (tree_num << 16) ^ random_seed; - random::PhiloxRandom rng(seed); - for (int i = 0; i < num_features_to_pick; ++i) { - // PhiloxRandom returns an array of int32's - const random::PhiloxRandom::ResultType rand = rng(); - const int32 feature = (rand[0] + rand[1]) % num_features; - features->push_back(feature); - } -} - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.h b/tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.h deleted file mode 100644 index 1ed3d8ca2e1..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/core/ops/utils.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_HYBRID_CORE_OPS_UTILS_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_HYBRID_CORE_OPS_UTILS_H_ -#include - -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace tensorforest { - -// Returns the probability that the point falls to the left. -float LeftProbability(const Tensor& point, const Tensor& weight, float bias, - int num_features); - -float LeftProbabilityK(const Tensor& point, std::vector feature_set, - const Tensor& weight, float bias, int num_features, - int k); - -// Returns a random set of num_features_to_pick features in the -// range [0, num_features). Must return the same set of -// features for subsequent calls with the same tree_num, node_num, and -// random_seed. This allows us to calculate feature sets between calls to ops -// without having to store their values. -void GetFeatureSet(int32 tree_num, int32 node_num, int32 random_seed, - int32 num_features, int32 num_features_to_pick, - std::vector* features); - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_HYBRID_CORE_OPS_UTILS_H_ diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/__init__.py b/tensorflow/contrib/tensor_forest/hybrid/python/__init__.py deleted file mode 100644 index 2bb96a67d30..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Initialize tensor_forest/hybrid/python.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.hybrid.python import layers -from tensorflow.contrib.tensor_forest.hybrid.python import models -from tensorflow.contrib.tensor_forest.hybrid.python.ops import training_ops diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/hybrid_layer.py b/tensorflow/contrib/tensor_forest/hybrid/python/hybrid_layer.py deleted file mode 100644 index 7527b12c0e7..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/hybrid_layer.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Defines the layer abstraction for hybrid models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.framework.python.ops import variables as framework_variables - - -class HybridLayer(object): - """Layers are building blocks for hybrid models.""" - - def _define_vars(self, - params, - **kwargs): - """Override to define the TensorFlow variables for the layer.""" - raise NotImplementedError - - # pylint: disable=unused-argument - def __init__(self, params, layer_num, device_assigner, *args, **kwargs): - self.layer_num = layer_num - self.device_assigner = ( - device_assigner or framework_variables.VariableDeviceChooser()) - self.params = params - self._define_vars(params, **kwargs) - - def inference_graph(self, data, data_spec=None): - raise NotImplementedError diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/hybrid_layer_test.py b/tensorflow/contrib/tensor_forest/hybrid/python/hybrid_layer_test.py deleted file mode 100644 index 26670a9041d..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/hybrid_layer_test.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the hybrid tensor forest model.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import - -from tensorflow.contrib.tensor_forest.hybrid.python import hybrid_model -from tensorflow.contrib.tensor_forest.hybrid.python.layers import fully_connected -from tensorflow.contrib.tensor_forest.python import tensor_forest -from tensorflow.python.framework import test_util -from tensorflow.python.platform import googletest - - -class HybridLayerTest(test_util.TensorFlowTestCase): - - def setUp(self): - self.params = tensor_forest.ForestHParams( - num_classes=3, - num_features=7, - layer_size=11, - num_layers=13, - num_trees=17, - connection_probability=0.1, - hybrid_tree_depth=4, - regularization_strength=0.01, - regularization="", - weight_init_mean=0.0, - weight_init_std=0.1) - self.params.num_nodes = 2**self.params.hybrid_tree_depth - 1 - self.params.num_leaves = 2**(self.params.hybrid_tree_depth - 1) - - def testLayerNums(self): - l1 = fully_connected.FullyConnectedLayer(self.params, 0, None) - self.assertEquals(l1.layer_num, 0) - - l2 = fully_connected.FullyConnectedLayer(self.params, 1, None) - self.assertEquals(l2.layer_num, 1) - - l3 = fully_connected.FullyConnectedLayer(self.params, 2, None) - self.assertEquals(l3.layer_num, 2) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/hybrid_model.py b/tensorflow/contrib/tensor_forest/hybrid/python/hybrid_model.py deleted file mode 100644 index a8a5b574691..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/hybrid_model.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Defines the model abstraction for hybrid models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.framework.python.ops import variables as framework_variables - -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variables - -from tensorflow.python.training import adagrad -from tensorflow.python.util.compat import collections_abc - - -class HybridModel(object): - """Defines a hybrid model. - - Models chain together the results of inference layers and provide training - capabilities. - """ - - # pylint: disable=unused-argument - def __init__(self, - params, - device_assigner=None, - optimizer_class=adagrad.AdagradOptimizer, - **kwargs): - - self.device_assigner = ( - device_assigner or framework_variables.VariableDeviceChooser()) - - self.params = params - - self.optimizer = optimizer_class(self.params.learning_rate) - - self.is_regression = params.regression - - self.regularizer = None - if params.regularization == "l1": - self.regularizer = layers.l1_regularizer( - self.params.regularization_strength) - elif params.regularization == "l2": - self.regularizer = layers.l2_regularizer( - self.params.regularization_strength) - - def _do_layer_inference(self, layer, data): - - # If this is a collection of layers, return the mean of their inference - # results. - if isinstance(layer, collections_abc.Iterable): - return math_ops.reduce_mean( - array_ops.stack([l.inference_graph(data) for l in layer]), 0) - # If this is a single layer, return its inference result. - else: - return layer.inference_graph(data) - - def _base_inference(self, data, data_spec=None): - """Returns an op that performs inference without a softmax.""" - inference_result = self._do_layer_inference(self.layers[0], data) - - for layer in self.layers[1:]: - inference_result = self._do_layer_inference(layer, inference_result) - - output_size = 1 if self.is_regression else self.params.num_classes - output = layers.fully_connected( - inference_result, output_size, activation_fn=array_ops.identity) - - return output - - def inference_graph(self, data, data_spec=None): - """Returns the op that performs inference on a batch of data.""" - - return nn_ops.softmax(self._base_inference(data, data_spec=data_spec)) - - def training_inference_graph(self, data, data_spec=None): - """Returns an inference-without-softmax op for training purposes.""" - - return self._base_inference(data, data_spec=data_spec) - - def predict_proba(self, data, data_spec=None): - inference_result = self.inference_graph(data, data_spec=data_spec) - - probabilities = nn_ops.softmax(inference_result, name="probabilities") - - return probabilities - - def training_graph(self, data, labels, data_spec=None, epoch=None): - """Returns the op that trains the hybrid model.""" - return self.optimizer.minimize(self.training_loss(data, labels)) - - def loss(self, data, labels): - """The loss to minimize while training.""" - - if self.is_regression: - diff = self.training_inference_graph(data) - math_ops.cast( - labels, dtypes.float32) - mean_squared_error = math_ops.reduce_mean(diff * diff) - root_mean_squared_error = math_ops.sqrt(mean_squared_error, name="loss") - loss = root_mean_squared_error - else: - loss = math_ops.reduce_mean( - nn_ops.sparse_softmax_cross_entropy_with_logits( - labels=array_ops.squeeze(math_ops.cast(labels, dtypes.int32)), - logits=self.training_inference_graph(data)), - name="loss") - if self.regularizer: - loss += layers.apply_regularization(self.regularizer, - variables.trainable_variables()) - return loss - - def training_loss(self, data, labels): - return self.loss(data, labels) - - def validation_loss(self, data, labels): - return self.loss(data, labels) diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/kernel_tests/k_feature_routing_function_op_test.py b/tensorflow/contrib/tensor_forest/hybrid/python/kernel_tests/k_feature_routing_function_op_test.py deleted file mode 100644 index cc053f3b94d..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/kernel_tests/k_feature_routing_function_op_test.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the routing function op.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.hybrid.ops import gen_training_ops -from tensorflow.contrib.tensor_forest.hybrid.python.ops import training_ops -from tensorflow.contrib.tensor_forest.python import tensor_forest - -from tensorflow.python.framework import test_util -from tensorflow.python.platform import googletest - - -class KFeatureRoutingFunctionTest(test_util.TensorFlowTestCase): - - def setUp(self): - self.input_data = [[-1., 0.], [-1., 2.], - [1., 0.], [1., -2.]] - self.input_labels = [0., 1., 2., 3.] - self.tree = [[1, 0], [-1, 0], [-1, 0]] - self.tree_weights = [[1.0, 0.0], [1.0, 0.0], [1.0, 0.0]] - self.tree_thresholds = [0., 0., 0.] - - self.ops = training_ops.Load() - - self.params = tensor_forest.ForestHParams( - num_features=2, - hybrid_tree_depth=2, - base_random_seed=10, - feature_bagging_fraction=1.0, - regularization_strength=0.01, - regularization="", - weight_init_mean=0.0, - weight_init_std=0.1) - self.params.num_nodes = 2**self.params.hybrid_tree_depth - 1 - self.params.num_leaves = 2**(self.params.hybrid_tree_depth - 1) - self.params.num_features_per_node = ( - self.params.feature_bagging_fraction * self.params.num_features) - self.params.regression = False - - def testParams(self): - self.assertEquals(self.params.num_nodes, 3) - self.assertEquals(self.params.num_features, 2) - self.assertEquals(self.params.num_features_per_node, 2) - - def testRoutingFunction(self): - with self.cached_session(): - route_tensor = gen_training_ops.k_feature_routing_function( - self.input_data, - self.tree_weights, - self.tree_thresholds, - max_nodes=self.params.num_nodes, - num_features_per_node=self.params.num_features_per_node, - layer_num=0, - random_seed=self.params.base_random_seed) - - route_tensor_shape = route_tensor.get_shape() - self.assertEquals(len(route_tensor_shape), 2) - self.assertEquals(route_tensor_shape[0], 4) - self.assertEquals(route_tensor_shape[1], 3) - - routes = route_tensor.eval() - print(routes) - - # Point 1 - # Node 1 is a decision node => probability = 1.0 - self.assertAlmostEquals(1.0, routes[0, 0]) - # Probability left output = 1.0 / (1.0 + exp(1.0)) = 0.26894142 - self.assertAlmostEquals(0.26894142, routes[0, 1]) - # Probability right = 1 - 0.2689414 = 0.73105858 - self.assertAlmostEquals(0.73105858, routes[0, 2]) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/kernel_tests/routing_function_op_test.py b/tensorflow/contrib/tensor_forest/hybrid/python/kernel_tests/routing_function_op_test.py deleted file mode 100644 index 554f7b0d7a9..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/kernel_tests/routing_function_op_test.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the routing function op.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.hybrid.ops import gen_training_ops -from tensorflow.contrib.tensor_forest.hybrid.python.ops import training_ops - -from tensorflow.python.framework import test_util -from tensorflow.python.platform import googletest - - -class RoutingFunctionTest(test_util.TensorFlowTestCase): - - def setUp(self): - self.input_data = [[-1., 0.], [-1., 2.], - [1., 0.], [1., -2.]] - self.input_labels = [0., 1., 2., 3.] - self.tree = [[1, 0], [-1, 0], [-1, 0]] - self.tree_weights = [[1.0, 0.0], [1.0, 0.0], [1.0, 0.0]] - self.tree_thresholds = [0., 0., 0.] - self.ops = training_ops.Load() - - def testRoutingFunction(self): - with self.cached_session(): - route_tensor = gen_training_ops.routing_function( - self.input_data, self.tree_weights, self.tree_thresholds, max_nodes=3) - - route_tensor_shape = route_tensor.get_shape() - self.assertEquals(len(route_tensor_shape), 2) - self.assertEquals(route_tensor_shape[0], 4) - self.assertEquals(route_tensor_shape[1], 3) - - routes = route_tensor.eval() - - # Point 1 - # Node 1 is a decision node => probability = 1.0 - self.assertAlmostEquals(1.0, routes[0, 0]) - # Probability left output = 1.0 / (1.0 + exp(1.0)) = 0.26894142 - self.assertAlmostEquals(0.26894142, routes[0, 1]) - # Probability right = 1 - 0.2689414 = 0.73105858 - self.assertAlmostEquals(0.73105858, routes[0, 2]) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/layers/__init__.py b/tensorflow/contrib/tensor_forest/hybrid/python/layers/__init__.py deleted file mode 100644 index db6776fc61f..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/layers/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Initialize tensor_forest/hybrid/python/layers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/layers/decisions_to_data.py b/tensorflow/contrib/tensor_forest/hybrid/python/layers/decisions_to_data.py deleted file mode 100644 index b57b1434751..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/layers/decisions_to_data.py +++ /dev/null @@ -1,240 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Treats a decision tree as a representation transformation layer. - -A decision tree transformer takes features as input and returns the probability -of reaching each leaf as output. The routing throughout the tree is learnable -via backpropagation. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.hybrid.ops import gen_training_ops -from tensorflow.contrib.tensor_forest.hybrid.python import hybrid_layer -from tensorflow.contrib.tensor_forest.hybrid.python.ops import training_ops -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import variable_scope - - -class DecisionsToDataLayer(hybrid_layer.HybridLayer): - """A layer that treats soft decisions as data.""" - - def _define_vars(self, params, **kwargs): - with ops.device(self.device_assigner): - - self.tree_parameters = variable_scope.get_variable( - name='tree_parameters_%d' % self.layer_num, - shape=[params.num_nodes, params.num_features], - initializer=init_ops.truncated_normal_initializer( - mean=params.weight_init_mean, stddev=params.weight_init_std)) - - self.tree_thresholds = variable_scope.get_variable( - name='tree_thresholds_%d' % self.layer_num, - shape=[params.num_nodes], - initializer=init_ops.truncated_normal_initializer( - mean=params.weight_init_mean, stddev=params.weight_init_std)) - - def __init__(self, params, layer_num, device_assigner, - *args, **kwargs): - super(DecisionsToDataLayer, self).__init__( - params, layer_num, device_assigner, *args, **kwargs) - - self._training_ops = training_ops.Load() - - def inference_graph(self, data): - with ops.device(self.device_assigner): - routing_probabilities = gen_training_ops.routing_function( - data, - self.tree_parameters, - self.tree_thresholds, - max_nodes=self.params.num_nodes) - - output = array_ops.slice( - routing_probabilities, - [0, self.params.num_nodes - self.params.num_leaves - 1], - [-1, self.params.num_leaves]) - - return output - - -class KFeatureDecisionsToDataLayer(hybrid_layer.HybridLayer): - """A layer that treats soft decisions made on single features as data.""" - - def _define_vars(self, params, **kwargs): - with ops.device(self.device_assigner): - - self.tree_parameters = variable_scope.get_variable( - name='tree_parameters_%d' % self.layer_num, - shape=[params.num_nodes, params.num_features_per_node], - initializer=init_ops.truncated_normal_initializer( - mean=params.weight_init_mean, stddev=params.weight_init_std)) - - self.tree_thresholds = variable_scope.get_variable( - name='tree_thresholds_%d' % self.layer_num, - shape=[params.num_nodes], - initializer=init_ops.truncated_normal_initializer( - mean=params.weight_init_mean, stddev=params.weight_init_std)) - - def __init__(self, params, layer_num, device_assigner, - *args, **kwargs): - super(KFeatureDecisionsToDataLayer, self).__init__( - params, layer_num, device_assigner, *args, **kwargs) - - self._training_ops = training_ops.Load() - - # pylint: disable=unused-argument - def inference_graph(self, data): - with ops.device(self.device_assigner): - routing_probabilities = gen_training_ops.k_feature_routing_function( - data, - self.tree_parameters, - self.tree_thresholds, - max_nodes=self.params.num_nodes, - num_features_per_node=self.params.num_features_per_node, - layer_num=0, - random_seed=self.params.base_random_seed) - - output = array_ops.slice( - routing_probabilities, - [0, self.params.num_nodes - self.params.num_leaves - 1], - [-1, self.params.num_leaves]) - - return output - - -class HardDecisionsToDataLayer(DecisionsToDataLayer): - """A layer that learns a soft decision tree but treats it as hard at test.""" - - def _define_vars(self, params, **kwargs): - with ops.device(self.device_assigner): - - self.tree_parameters = variable_scope.get_variable( - name='hard_tree_parameters_%d' % self.layer_num, - shape=[params.num_nodes, params.num_features], - initializer=variable_scope.truncated_normal_initializer( - mean=params.weight_init_mean, stddev=params.weight_init_std)) - - self.tree_thresholds = variable_scope.get_variable( - name='hard_tree_thresholds_%d' % self.layer_num, - shape=[params.num_nodes], - initializer=variable_scope.truncated_normal_initializer( - mean=params.weight_init_mean, stddev=params.weight_init_std)) - - def soft_inference_graph(self, data): - return super(HardDecisionsToDataLayer, self).inference_graph(data) - - def inference_graph(self, data): - with ops.device(self.device_assigner): - path_probability, path = gen_training_ops.hard_routing_function( - data, - self.tree_parameters, - self.tree_thresholds, - max_nodes=self.params.num_nodes, - tree_depth=self.params.hybrid_tree_depth) - - output = array_ops.slice( - gen_training_ops.unpack_path(path, path_probability), - [0, self.params.num_nodes - self.params.num_leaves - 1], - [-1, self.params.num_leaves]) - - return output - - -class StochasticHardDecisionsToDataLayer(HardDecisionsToDataLayer): - """A layer that learns a soft decision tree by sampling paths.""" - - def _define_vars(self, params, **kwargs): - with ops.device(self.device_assigner): - - self.tree_parameters = variable_scope.get_variable( - name='stochastic_hard_tree_parameters_%d' % self.layer_num, - shape=[params.num_nodes, params.num_features], - initializer=init_ops.truncated_normal_initializer( - mean=params.weight_init_mean, stddev=params.weight_init_std)) - - self.tree_thresholds = variable_scope.get_variable( - name='stochastic_hard_tree_thresholds_%d' % self.layer_num, - shape=[params.num_nodes], - initializer=init_ops.truncated_normal_initializer( - mean=params.weight_init_mean, stddev=params.weight_init_std)) - - def soft_inference_graph(self, data): - with ops.device(self.device_assigner): - path_probability, path = ( - gen_training_ops.stochastic_hard_routing_function( - data, - self.tree_parameters, - self.tree_thresholds, - tree_depth=self.params.hybrid_tree_depth, - random_seed=self.params.base_random_seed)) - - output = array_ops.slice( - gen_training_ops.unpack_path(path, path_probability), - [0, self.params.num_nodes - self.params.num_leaves - 1], - [-1, self.params.num_leaves]) - - return output - - def inference_graph(self, data): - with ops.device(self.device_assigner): - path_probability, path = gen_training_ops.hard_routing_function( - data, - self.tree_parameters, - self.tree_thresholds, - max_nodes=self.params.num_nodes, - tree_depth=self.params.hybrid_tree_depth) - - output = array_ops.slice( - gen_training_ops.unpack_path(path, path_probability), - [0, self.params.num_nodes - self.params.num_leaves - 1], - [-1, self.params.num_leaves]) - - return output - - -class StochasticSoftDecisionsToDataLayer(StochasticHardDecisionsToDataLayer): - """A layer that learns a soft decision tree by sampling paths.""" - - def _define_vars(self, params, **kwargs): - with ops.device(self.device_assigner): - - self.tree_parameters = variable_scope.get_variable( - name='stochastic_soft_tree_parameters_%d' % self.layer_num, - shape=[params.num_nodes, params.num_features], - initializer=init_ops.truncated_normal_initializer( - mean=params.weight_init_mean, stddev=params.weight_init_std)) - - self.tree_thresholds = variable_scope.get_variable( - name='stochastic_soft_tree_thresholds_%d' % self.layer_num, - shape=[params.num_nodes], - initializer=init_ops.truncated_normal_initializer( - mean=params.weight_init_mean, stddev=params.weight_init_std)) - - def inference_graph(self, data): - with ops.device(self.device_assigner): - routes = gen_training_ops.routing_function( - data, - self.tree_parameters, - self.tree_thresholds, - max_nodes=self.params.num_nodes) - - leaf_routes = array_ops.slice( - routes, [0, self.params.num_nodes - self.params.num_leaves - 1], - [-1, self.params.num_leaves]) - - return leaf_routes diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/layers/decisions_to_data_test.py b/tensorflow/contrib/tensor_forest/hybrid/python/layers/decisions_to_data_test.py deleted file mode 100644 index 1ed8a5c8081..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/layers/decisions_to_data_test.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random - -# pylint: disable=unused-import - -from tensorflow.contrib.tensor_forest.hybrid.python.layers import decisions_to_data -from tensorflow.contrib.tensor_forest.python import tensor_forest -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import test_util -from tensorflow.python.framework.ops import Operation -from tensorflow.python.framework.ops import Tensor -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import googletest - - -class DecisionsToDataTest(test_util.TensorFlowTestCase): - - def setUp(self): - self.params = tensor_forest.ForestHParams( - num_classes=2, - num_features=31, - layer_size=11, - num_layers=13, - num_trees=17, - connection_probability=0.1, - hybrid_tree_depth=4, - regularization_strength=0.01, - regularization="", - learning_rate=0.01, - weight_init_mean=0.0, - weight_init_std=0.1) - self.params.regression = False - self.params.num_nodes = 2**self.params.hybrid_tree_depth - 1 - self.params.num_leaves = 2**(self.params.hybrid_tree_depth - 1) - - # pylint: disable=W0612 - self.input_data = constant_op.constant( - [[random.uniform(-1, 1) for i in range(self.params.num_features)] - for _ in range(100)]) - - def testInferenceConstruction(self): - with variable_scope.variable_scope( - "DecisionsToDataTest_testInferenceContruction"): - graph_builder = decisions_to_data.DecisionsToDataLayer(self.params, 0, - None) - unused_graph = graph_builder.inference_graph(self.input_data) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/layers/fully_connected.py b/tensorflow/contrib/tensor_forest/hybrid/python/layers/fully_connected.py deleted file mode 100644 index 745a5b1caf2..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/layers/fully_connected.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Neural network components for hybrid models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.tensor_forest.hybrid.python import hybrid_layer - -from tensorflow.python.framework import ops - -from tensorflow.python.ops import array_ops - - -class FullyConnectedLayer(hybrid_layer.HybridLayer): - """A stacked, fully-connected feed-forward neural network layer.""" - - def _define_vars(self, params): - pass - - def inference_graph(self, data): - with ops.device(self.device_assigner): - # Compute activations for the neural network. - nn_activations = layers.fully_connected(data, self.params.layer_size) - - for _ in range(1, self.params.num_layers): - # pylint: disable=W0106 - nn_activations = layers.fully_connected(nn_activations, - self.params.layer_size) - return nn_activations - - -class ManyToOneLayer(hybrid_layer.HybridLayer): - - def _define_vars(self, params): - pass - - def inference_graph(self, data): - with ops.device(self.device_assigner): - # Compute activations for the neural network. - nn_activations = layers.fully_connected(data, 1) - - # There is always one activation per instance by definition, so squeeze - # away the extra dimension. - return array_ops.squeeze(nn_activations, axis=[1]) - - -class FlattenedFullyConnectedLayer(hybrid_layer.HybridLayer): - """A stacked, fully-connected flattened feed-forward neural network layer.""" - - def _define_vars(self, params): - pass - - def inference_graph(self, data): - with ops.device(self.device_assigner): - # Compute activations for the neural network. - nn_activations = [layers.fully_connected(data, self.params.layer_size)] - - for _ in range(1, self.params.num_layers): - # pylint: disable=W0106 - nn_activations.append( - layers.fully_connected( - nn_activations[-1], - self.params.layer_size)) - - nn_activations_tensor = array_ops.concat( - nn_activations, 1, name="flattened_nn_activations") - - return nn_activations_tensor diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/models/__init__.py b/tensorflow/contrib/tensor_forest/hybrid/python/models/__init__.py deleted file mode 100644 index 6ad14006fbf..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/models/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Initialize tensor_forest/hybrid/python/models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/models/decisions_to_data_then_nn.py b/tensorflow/contrib/tensor_forest/hybrid/python/models/decisions_to_data_then_nn.py deleted file mode 100644 index fa054cf2ffa..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/models/decisions_to_data_then_nn.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A model that places a decision tree embedding before a neural net.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.hybrid.python import hybrid_model -from tensorflow.contrib.tensor_forest.hybrid.python.layers import decisions_to_data -from tensorflow.contrib.tensor_forest.hybrid.python.layers import fully_connected -from tensorflow.python.training import adagrad - - -class DecisionsToDataThenNN(hybrid_model.HybridModel): - """A model that places a decision tree embedding before a neural net.""" - - def __init__(self, - params, - device_assigner=None, - optimizer_class=adagrad.AdagradOptimizer, - **kwargs): - super(DecisionsToDataThenNN, self).__init__( - params, - device_assigner=device_assigner, - optimizer_class=optimizer_class, - **kwargs) - - self.layers = [decisions_to_data.DecisionsToDataLayer(params, - 0, device_assigner), - fully_connected.FullyConnectedLayer( - params, 1, device_assigner=device_assigner)] diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/models/decisions_to_data_then_nn_test.py b/tensorflow/contrib/tensor_forest/hybrid/python/models/decisions_to_data_then_nn_test.py deleted file mode 100644 index a56beeeb2c1..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/models/decisions_to_data_then_nn_test.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the hybrid tensor forest model.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random - -# pylint: disable=unused-import - -from tensorflow.contrib.tensor_forest.hybrid.python.models import decisions_to_data_then_nn -from tensorflow.contrib.tensor_forest.python import tensor_forest -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import test_util -from tensorflow.python.framework.ops import Operation -from tensorflow.python.framework.ops import Tensor -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import googletest - - -class DecisionsToDataThenNNTest(test_util.TensorFlowTestCase): - - def setUp(self): - self.params = tensor_forest.ForestHParams( - num_classes=2, - num_features=31, - layer_size=11, - num_layers=13, - num_trees=17, - connection_probability=0.1, - hybrid_tree_depth=4, - regularization_strength=0.01, - learning_rate=0.01, - regularization="", - weight_init_mean=0.0, - weight_init_std=0.1) - self.params.regression = False - self.params.num_nodes = 2**self.params.hybrid_tree_depth - 1 - self.params.num_leaves = 2**(self.params.hybrid_tree_depth - 1) - - def testHParams(self): - self.assertEquals(self.params.num_classes, 2) - self.assertEquals(self.params.num_features, 31) - self.assertEquals(self.params.layer_size, 11) - self.assertEquals(self.params.num_layers, 13) - self.assertEquals(self.params.num_trees, 17) - self.assertEquals(self.params.hybrid_tree_depth, 4) - self.assertEquals(self.params.connection_probability, 0.1) - - # Building the graphs modifies the params. - with variable_scope.variable_scope("DecisionsToDataThenNNTest_testHParams"): - # pylint: disable=W0612 - graph_builder = decisions_to_data_then_nn.DecisionsToDataThenNN( - self.params) - - # Tree with depth 4 should have 2**0 + 2**1 + 2**2 + 2**3 = 15 nodes. - self.assertEquals(self.params.num_nodes, 15) - - def testConstructionPollution(self): - """Ensure that graph building doesn't modify the params in a bad way.""" - # pylint: disable=W0612 - data = [[random.uniform(-1, 1) for i in range(self.params.num_features)] - for _ in range(100)] - - self.assertTrue(isinstance(self.params, tensor_forest.ForestHParams)) - self.assertFalse( - isinstance(self.params.num_trees, tensor_forest.ForestHParams)) - - with variable_scope.variable_scope( - "DecisionsToDataThenNNTest_testConstructionPollution"): - graph_builder = decisions_to_data_then_nn.DecisionsToDataThenNN( - self.params) - - self.assertTrue(isinstance(self.params, tensor_forest.ForestHParams)) - self.assertFalse( - isinstance(self.params.num_trees, tensor_forest.ForestHParams)) - - def testInferenceConstruction(self): - # pylint: disable=W0612 - data = constant_op.constant( - [[random.uniform(-1, 1) for i in range(self.params.num_features)] - for _ in range(100)]) - - with variable_scope.variable_scope( - "DecisionsToDataThenNNTest_testInferenceConstruction"): - graph_builder = decisions_to_data_then_nn.DecisionsToDataThenNN( - self.params) - graph = graph_builder.inference_graph(data, None) - - self.assertTrue(isinstance(graph, Tensor)) - - def testTrainingConstruction(self): - # pylint: disable=W0612 - data = constant_op.constant( - [[random.uniform(-1, 1) for i in range(self.params.num_features)] - for _ in range(100)]) - - labels = [1 for _ in range(100)] - - with variable_scope.variable_scope( - "DecisionsToDataThenNNTest_testTrainingConstruction"): - graph_builder = decisions_to_data_then_nn.DecisionsToDataThenNN( - self.params) - graph = graph_builder.training_graph(data, labels, None) - - self.assertTrue(isinstance(graph, Operation)) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/models/forest_to_data_then_nn.py b/tensorflow/contrib/tensor_forest/hybrid/python/models/forest_to_data_then_nn.py deleted file mode 100644 index 8e0483a592b..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/models/forest_to_data_then_nn.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A model that combines a decision forest embedding with a neural net.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.hybrid.python import hybrid_model -from tensorflow.contrib.tensor_forest.hybrid.python.layers import decisions_to_data -from tensorflow.contrib.tensor_forest.hybrid.python.layers import fully_connected -from tensorflow.python.training import adagrad - - -class ForestToDataThenNN(hybrid_model.HybridModel): - """A model that combines a decision forest embedding with a neural net.""" - - def __init__(self, - params, - device_assigner=None, - optimizer_class=adagrad.AdagradOptimizer, - **kwargs): - super(ForestToDataThenNN, self).__init__( - params, - device_assigner=device_assigner, - optimizer_class=optimizer_class, - **kwargs) - - self.layers = [[decisions_to_data.KFeatureDecisionsToDataLayer( - params, i, device_assigner) - for i in range(self.params.num_trees)], - fully_connected.FullyConnectedLayer( - params, - self.params.num_trees, - device_assigner=device_assigner)] diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/models/forest_to_data_then_nn_test.py b/tensorflow/contrib/tensor_forest/hybrid/python/models/forest_to_data_then_nn_test.py deleted file mode 100644 index e0b7109f996..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/models/forest_to_data_then_nn_test.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the hybrid tensor forest model.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random - -# pylint: disable=unused-import - -from tensorflow.contrib.tensor_forest.hybrid.python.models import forest_to_data_then_nn -from tensorflow.contrib.tensor_forest.python import tensor_forest -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import test_util -from tensorflow.python.framework.ops import Operation -from tensorflow.python.framework.ops import Tensor -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import googletest - - -class ForestToDataThenNNTest(test_util.TensorFlowTestCase): - - def setUp(self): - self.params = tensor_forest.ForestHParams( - num_classes=2, - num_features=31, - layer_size=11, - num_layers=13, - num_trees=3, - connection_probability=0.1, - hybrid_tree_depth=4, - regularization_strength=0.01, - regularization="", - base_random_seed=10, - feature_bagging_fraction=1.0, - learning_rate=0.01, - weight_init_mean=0.0, - weight_init_std=0.1) - self.params.regression = False - self.params.num_nodes = 2**self.params.hybrid_tree_depth - 1 - self.params.num_leaves = 2**(self.params.hybrid_tree_depth - 1) - - self.params.num_features_per_node = (self.params.feature_bagging_fraction * - self.params.num_features) - - def testInferenceConstruction(self): - # pylint: disable=W0612 - data = constant_op.constant( - [[random.uniform(-1, 1) for i in range(self.params.num_features)] - for _ in range(100)]) - - with variable_scope.variable_scope( - "ForestToDataThenNNTest_testInferenceContruction"): - graph_builder = forest_to_data_then_nn.ForestToDataThenNN(self.params) - graph = graph_builder.inference_graph(data, None) - - self.assertTrue(isinstance(graph, Tensor)) - - def testTrainingConstruction(self): - # pylint: disable=W0612 - data = constant_op.constant( - [[random.uniform(-1, 1) for i in range(self.params.num_features)] - for _ in range(100)]) - - labels = [1 for _ in range(100)] - - with variable_scope.variable_scope( - "ForestToDataThenNNTest.testTrainingContruction"): - graph_builder = forest_to_data_then_nn.ForestToDataThenNN(self.params) - graph = graph_builder.training_graph(data, labels, None) - - self.assertTrue(isinstance(graph, Operation)) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/models/hard_decisions_to_data_then_nn.py b/tensorflow/contrib/tensor_forest/hybrid/python/models/hard_decisions_to_data_then_nn.py deleted file mode 100644 index 3ea9aff85a1..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/models/hard_decisions_to_data_then_nn.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A model that places a hard decision tree embedding before a neural net.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import layers -from tensorflow.contrib.tensor_forest.hybrid.python import hybrid_model -from tensorflow.contrib.tensor_forest.hybrid.python.layers import decisions_to_data -from tensorflow.contrib.tensor_forest.hybrid.python.layers import fully_connected -from tensorflow.python.ops import nn_ops -from tensorflow.python.training import adagrad - - -class HardDecisionsToDataThenNN(hybrid_model.HybridModel): - """A model that treats tree inference as hard at test.""" - - def __init__(self, - params, - device_assigner=None, - optimizer_class=adagrad.AdagradOptimizer, - **kwargs): - - super(HardDecisionsToDataThenNN, self).__init__( - params, - device_assigner=device_assigner, - optimizer_class=optimizer_class, - **kwargs) - - self.layers = [decisions_to_data.HardDecisionsToDataLayer( - params, 0, device_assigner), - fully_connected.FullyConnectedLayer( - params, 1, device_assigner=device_assigner)] - - def _base_inference(self, data, data_spec=None, soft=False): - if soft: - inference_result = self.layers[0].soft_inference_graph(data) - else: - inference_result = self._do_layer_inference(self.layers[0], data) - - for layer in self.layers[1:]: - inference_result = self._do_layer_inference(layer, inference_result) - - output_size = 1 if self.is_regression else self.params.num_classes - output = layers.fully_connected( - inference_result, output_size, activation_fn=nn_ops.softmax) - return output - - def inference_graph(self, data, data_spec=None): - """Returns the op that performs inference on a batch of data.""" - - return nn_ops.softmax( - self._base_inference( - data, data_spec=data_spec, soft=True)) - - # pylint: disable=unused-argument - def training_inference_graph(self, data, data_spec=None): - return self._base_inference(data, data_spec=data_spec, soft=False) diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/models/k_feature_decisions_to_data_then_nn.py b/tensorflow/contrib/tensor_forest/hybrid/python/models/k_feature_decisions_to_data_then_nn.py deleted file mode 100644 index af3cdd7ab82..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/models/k_feature_decisions_to_data_then_nn.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A model that places a soft decision tree embedding before a neural net.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.hybrid.python import hybrid_model -from tensorflow.contrib.tensor_forest.hybrid.python.layers import decisions_to_data -from tensorflow.contrib.tensor_forest.hybrid.python.layers import fully_connected -from tensorflow.python.training import adagrad - - -class KFeatureDecisionsToDataThenNN(hybrid_model.HybridModel): - """A model that places a soft decision tree embedding before a neural net.""" - - def __init__(self, - params, - device_assigner=None, - optimizer_class=adagrad.AdagradOptimizer, - **kwargs): - super(KFeatureDecisionsToDataThenNN, self).__init__( - params, - device_assigner=device_assigner, - optimizer_class=optimizer_class, - **kwargs) - - self.layers = [decisions_to_data.KFeatureDecisionsToDataLayer( - params, 0, device_assigner), - fully_connected.FullyConnectedLayer( - params, 1, device_assigner=device_assigner)] diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/models/k_feature_decisions_to_data_then_nn_test.py b/tensorflow/contrib/tensor_forest/hybrid/python/models/k_feature_decisions_to_data_then_nn_test.py deleted file mode 100644 index 60dae6d4467..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/models/k_feature_decisions_to_data_then_nn_test.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the hybrid tensor forest model.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random - -# pylint: disable=unused-import - -from tensorflow.contrib.tensor_forest.hybrid.python.models import k_feature_decisions_to_data_then_nn -from tensorflow.contrib.tensor_forest.python import tensor_forest -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import test_util -from tensorflow.python.framework.ops import Operation -from tensorflow.python.framework.ops import Tensor -from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import googletest - - -class KFeatureDecisionsToDataThenNNTest(test_util.TensorFlowTestCase): - - def setUp(self): - self.params = tensor_forest.ForestHParams( - num_classes=2, - num_features=31, - layer_size=11, - num_layers=13, - num_trees=17, - connection_probability=0.1, - hybrid_tree_depth=4, - regularization_strength=0.01, - regularization="", - base_random_seed=10, - hybrid_feature_bagging_fraction=1.0, - learning_rate=0.01, - weight_init_mean=0.0, - weight_init_std=0.1) - self.params.regression = False - self.params.num_nodes = 2**self.params.hybrid_tree_depth - 1 - self.params.num_leaves = 2**(self.params.hybrid_tree_depth - 1) - self.params.num_features_per_node = (self.params.feature_bagging_fraction * - self.params.num_features) - - def testKFeatureInferenceConstruction(self): - # pylint: disable=W0612 - data = constant_op.constant( - [[random.uniform(-1, 1) for i in range(self.params.num_features)] - for _ in range(100)]) - - with variable_scope.variable_scope( - "KFeatureDecisionsToDataThenNNTest.testKFeatureInferenceContruction"): - graph_builder = ( - k_feature_decisions_to_data_then_nn.KFeatureDecisionsToDataThenNN( - self.params)) - graph = graph_builder.inference_graph(data, None) - - self.assertTrue(isinstance(graph, Tensor)) - - def testKFeatureTrainingConstruction(self): - # pylint: disable=W0612 - data = constant_op.constant( - [[random.uniform(-1, 1) for i in range(self.params.num_features)] - for _ in range(100)]) - - labels = [1 for _ in range(100)] - - with variable_scope.variable_scope( - "KFeatureDecisionsToDataThenNNTest.testKFeatureTrainingContruction"): - graph_builder = ( - k_feature_decisions_to_data_then_nn.KFeatureDecisionsToDataThenNN( - self.params)) - graph = graph_builder.training_graph(data, labels, None) - - self.assertTrue(isinstance(graph, Operation)) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/models/nn.py b/tensorflow/contrib/tensor_forest/hybrid/python/models/nn.py deleted file mode 100644 index bc90261334a..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/models/nn.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A simple baseline feed-forward neural network.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.hybrid.python import hybrid_model -from tensorflow.contrib.tensor_forest.hybrid.python.layers import fully_connected -from tensorflow.python.training import adagrad - - -class NN(hybrid_model.HybridModel): - """A simple baseline feed-forward neural network.""" - - def __init__(self, - params, - device_assigner=None, - optimizer_class=adagrad.AdagradOptimizer, - **kwargs): - super(NN, self).__init__( - params, - device_assigner=device_assigner, - optimizer_class=optimizer_class, - **kwargs) - - self.layers = [fully_connected.FullyConnectedLayer( - params, 0, device_assigner=device_assigner)] diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/models/stochastic_hard_decisions_to_data_then_nn.py b/tensorflow/contrib/tensor_forest/hybrid/python/models/stochastic_hard_decisions_to_data_then_nn.py deleted file mode 100644 index 1cac0a1e615..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/models/stochastic_hard_decisions_to_data_then_nn.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A hybrid model that samples paths when training.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.hybrid.python.layers import decisions_to_data -from tensorflow.contrib.tensor_forest.hybrid.python.layers import fully_connected -from tensorflow.contrib.tensor_forest.hybrid.python.models import hard_decisions_to_data_then_nn -from tensorflow.python.training import adagrad - - -class StochasticHardDecisionsToDataThenNN( - hard_decisions_to_data_then_nn.HardDecisionsToDataThenNN): - """A hybrid model that samples paths when training.""" - - def __init__(self, - params, - device_assigner=None, - optimizer_class=adagrad.AdagradOptimizer, - **kwargs): - - super(StochasticHardDecisionsToDataThenNN, self).__init__( - params, - device_assigner=device_assigner, - optimizer_class=optimizer_class, - **kwargs) - - self.layers = [decisions_to_data.StochasticHardDecisionsToDataLayer( - params, 0, device_assigner), - fully_connected.FullyConnectedLayer( - params, 1, device_assigner=device_assigner)] diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/models/stochastic_soft_decisions_to_data_then_nn.py b/tensorflow/contrib/tensor_forest/hybrid/python/models/stochastic_soft_decisions_to_data_then_nn.py deleted file mode 100644 index c2f3f603415..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/models/stochastic_soft_decisions_to_data_then_nn.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A hybrid model that samples paths when training.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.hybrid.python.layers import decisions_to_data -from tensorflow.contrib.tensor_forest.hybrid.python.layers import fully_connected -from tensorflow.contrib.tensor_forest.hybrid.python.models import hard_decisions_to_data_then_nn -from tensorflow.python.training import adagrad - - -class StochasticSoftDecisionsToDataThenNN( - hard_decisions_to_data_then_nn.HardDecisionsToDataThenNN): - """A hybrid model that samples paths when training.""" - - def __init__(self, - params, - device_assigner=None, - optimizer_class=adagrad.AdagradOptimizer, - **kwargs): - - super(StochasticSoftDecisionsToDataThenNN, self).__init__( - params, - device_assigner=device_assigner, - optimizer_class=optimizer_class, - **kwargs) - - self.layers = [decisions_to_data.StochasticSoftDecisionsToDataLayer( - params, 0, device_assigner), - fully_connected.FullyConnectedLayer( - params, 1, device_assigner=device_assigner)] diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/ops/training_ops.py b/tensorflow/contrib/tensor_forest/hybrid/python/ops/training_ops.py deleted file mode 100644 index 7f403ec825b..00000000000 --- a/tensorflow/contrib/tensor_forest/hybrid/python/ops/training_ops.py +++ /dev/null @@ -1,294 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for hybrid model training.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import threading - -from tensorflow.contrib.tensor_forest.hybrid.ops import gen_training_ops -from tensorflow.contrib.util import loader -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import resource_loader -from tensorflow.python.platform import tf_logging as logging - -TRAINING_OPS_FILE = '_training_ops.so' - -_training_ops = None -_ops_lock = threading.Lock() - -# TODO(b/31222613): Some of these ops are probably differentiable, and -# there may be latent bugs here. -ops.NotDifferentiable('HardRoutingFunction') -ops.NotDifferentiable('RoutingGradient') -ops.NotDifferentiable('KFeatureDataGradient') -ops.NotDifferentiable('KFeatureRoutingGradient') -ops.NotDifferentiable('KFeatureWeightGradient') -ops.NotDifferentiable('UnpackPath') - - -@ops.RegisterGradient('RoutingFunction') -def _RoutingFunctionGradient(op, grad): - """The gradient of RoutingFunction. - - Args: - op: The RoutingFunction op. - grad: Gradient with respect to the output of the RoutingFunction op. - - Returns: - Gradients with respect to the input of the RoutingFunction op. - """ - routing_gradient = gen_training_ops.routing_gradient - - input_data_tensor = op.inputs[0] - tree_weights_tensor = op.inputs[1] - tree_thresholds_tensor = op.inputs[2] - - routing_function_tensor = op.outputs[0] - - # The derivatives below are each defined over one or two of three dimensions: - # (batch_size, num_nodes, num_features). We explicitly expand each derivative - # to three dimensions to ensure that they're broadcasted correctly. - - # dl / du is the derivative of the loss with respect to the output of the - # routing function, which is provided by tensorflow. - # - # dl / du has dimension (batch_size, num_nodes), which we expand to - # (batch_size, num_nodes, 1). - dl_du = array_ops.expand_dims(grad, 2) - - # du / df is the derivative of the output of the routing function with respect - # to the decision function at each node. It is computed by - # routing_gradient_op.cc. - # - # du / df has dimension (batch_size, num_nodes), which we expand to - # (batch_size, num_nodes, 1). - du_df = array_ops.expand_dims( - routing_gradient( - input_data_tensor, - tree_weights_tensor, - tree_thresholds_tensor, - routing_function_tensor, - max_nodes=op.get_attr('max_nodes')), - 2) - - # df / dx is the derivative of the decision function with respect to the input - # data. f_i(x) = (-t_i * x + b_i), so df_i / dx = -t_i. - # - # df / dx has dimension (num_nodes, num_features), which we expand to - # (1, num_nodes, num_features). - df_dx = -array_ops.expand_dims(tree_weights_tensor, 0) - - # df / dt is the derivative of the decision function with respect to its - # parameters. f_i(x) = (-t_i * x + b_i), so df_i / d t_i = -x. - # - # df / dt has dimension (batch_size, num_features), which we expand to - # (batch_size, 1, num_features). - df_dt = -array_ops.expand_dims(input_data_tensor, 1) - # df / dt is the derivative of the decision function with respect to its - # bias parameter. f_i(x) = (-t_i * x + b_i), so df_i / d t_i = 1. - # - # df / db has dimension (num_nodes), which we expand to - # (1, num_nodes, 1). - df_db = array_ops.expand_dims( - array_ops.expand_dims(array_ops.ones_like(tree_thresholds_tensor), 0), 2) - - # Compute the derivatives of the loss with respect to the inputs using the - # chain rule (backpropagation). - dl_dx = math_ops.reduce_mean(dl_du * du_df * df_dx, 1) - dl_dt = math_ops.reduce_mean(dl_du * du_df * df_dt, 0) - dl_db = math_ops.reduce_mean(array_ops.squeeze(dl_du * du_df * df_db, [2]), 0) - - input_gradients = [dl_dx, dl_dt, dl_db] - - return input_gradients - - -@ops.RegisterGradient('StochasticHardRoutingFunction') -def _StochasticHardRoutingFunctionGradient(op, routing_grad, unused_path_grad): - """The gradient of RoutingFunction. - - Args: - op: The RoutingFunction op. - routing_grad: Gradient with respect to the output of the RoutingFunction op. - - Returns: - Gradients with respect to the input of the RoutingFunction op. - """ - gradient_op = gen_training_ops.stochastic_hard_routing_gradient - unpack_path_op = gen_training_ops.unpack_path - - input_data_tensor = op.inputs[0] - tree_weights_tensor = op.inputs[1] - tree_thresholds_tensor = op.inputs[2] - - path_probability_tensor = op.outputs[0] - path_tensor = op.outputs[1] - - # The derivatives below are each defined over one or two of three dimensions: - # (batch_size, num_nodes, num_features). We explicitly expand each derivative - # to three dimensions to ensure that they're broadcasted correctly. - du_df_raw, df_dx_raw, df_dt_raw, df_db_raw = gradient_op( - input_data_tensor, - tree_weights_tensor, - tree_thresholds_tensor, - path_probability_tensor, - path_tensor, - tree_depth=op.get_attr('tree_depth')) - - # dl / du is the derivative of the loss with respect to the output of the - # routing function, which is provided by tensorflow. - # - # dl / du has dimension (batch_size, num_nodes), which we expand to - # (batch_size, num_nodes, 1). - dl_du = array_ops.expand_dims(unpack_path_op(path_tensor, routing_grad), 2) - - # du / df is the derivative of the output of the routing function with respect - # to the decision function at each node. It is computed by - # single_feature_routing_gradient_op.cc. - # - # du / df has dimension (batch_size, num_nodes), which we expand to - # (batch_size, num_nodes, 1). - du_df = array_ops.expand_dims(du_df_raw, 2) - - # df / dx is the derivative of the decision function with respect to the input - # data. f(x) = (-t * x + b), so df / dx = -t for the selected features and - # zero elsewhere. - # - # df / dx has dimension (num_nodes, num_features), which we expand to - # (1, num_nodes, num_features). - df_dx = array_ops.expand_dims(df_dx_raw, 0) - - # df / dt is the derivative of the decision function with respect to its - # parameters. f(x) = (-t * x + b), so df / dt = -x[feature]. - # - # df / dt has dimension (batch_size, num_nodes, num_features). - df_dt = -df_dt_raw - - # df / dt is the derivative of the decision function with respect to its - # bias parameter. f(x) = (-t * x + b), so df / dt = 1. - # - # df / db has dimension (num_nodes), which we expand to - # (1, num_nodes, 1). - df_db = array_ops.expand_dims(array_ops.expand_dims(df_db_raw, 0), 2) - - # Compute the derivatives of the loss with respect to the inputs using the - # chain rule (backpropagation). - dl_dx = math_ops.reduce_mean(dl_du * du_df * df_dx, 1) - dl_dt = math_ops.reduce_mean(dl_du * du_df * df_dt, 0) - dl_db = math_ops.reduce_mean(array_ops.squeeze(dl_du * du_df * df_db, [2]), 0) - - input_gradients = [dl_dx, dl_dt, dl_db] - - return input_gradients - - -@ops.RegisterGradient('KFeatureRoutingFunction') -def _KFeatureRoutingFunctionGradient(op, grad): - """The gradient of RoutingFunction. - - Args: - op: The RoutingFunction op. - grad: Gradient with respect to the output of the RoutingFunction op. - - Returns: - Gradients with respect to the input of the RoutingFunction op. - """ - gradient_op = gen_training_ops.k_feature_gradient - - input_data_tensor = op.inputs[0] - tree_weights_tensor = op.inputs[1] - tree_thresholds_tensor = op.inputs[2] - - routing_function_tensor = op.outputs[0] - - # The derivatives below are each defined over one or two of three dimensions: - # (batch_size, num_nodes, num_features). We explicitly expand each derivative - # to three dimensions to ensure that they're broadcasted correctly. - du_df_raw, df_dx_raw, df_dt_raw = gradient_op( - input_data_tensor, - tree_weights_tensor, - tree_thresholds_tensor, - routing_function_tensor, - layer_num=op.get_attr('layer_num'), - random_seed=op.get_attr('random_seed')) - - # dl / du is the derivative of the loss with respect to the output of the - # routing function, which is provided by tensorflow. - # - # dl / du has dimension (batch_size, num_nodes), which we expand to - # (batch_size, num_nodes, 1). - dl_du = array_ops.expand_dims(grad, 2) - - # du / df is the derivative of the output of the routing function with respect - # to the decision function at each node. It is computed by - # single_feature_routing_gradient_op.cc. - # - # du / df has dimension (batch_size, num_nodes), which we expand to - # (batch_size, num_nodes, 1). - du_df = array_ops.expand_dims(du_df_raw, 2) - - # df / dx is the derivative of the decision function with respect to the input - # data. f(x) = (-t * x + b), so df / dx = -t for the selected features and - # zero elsewhere. - # - # df / dx has dimension (num_nodes, num_features), which we expand to - # (1, num_nodes, num_features). - df_dx = array_ops.expand_dims(df_dx_raw, 0) - - # df / dt is the derivative of the decision function with respect to its - # parameters. f(x) = (-t * x + b), so df / dt = -x[feature]. - # - # df / dt has dimension (batch_size, num_nodes, num_features). - df_dt = -df_dt_raw - - # df / dt is the derivative of the decision function with respect to its - # bias parameter. f(x) = (-t * x + b), so df / dt = 1. - # - # df / db has dimension (num_nodes), which we expand to - # (1, num_nodes, 1). - df_db = array_ops.expand_dims( - array_ops.expand_dims(array_ops.ones_like(tree_thresholds_tensor), 0), 2) - - # Compute the derivatives of the loss with respect to the inputs using the - # chain rule (backpropagation). - dl_dx = math_ops.reduce_mean(dl_du * du_df * df_dx, 1) - dl_dt = math_ops.reduce_mean(dl_du * du_df * df_dt, 0) - dl_db = math_ops.reduce_mean(array_ops.squeeze(dl_du * du_df * df_db, [2]), 0) - - input_gradients = [dl_dx, dl_dt, dl_db] - - return input_gradients - - -# Workaround for the fact that importing tensorflow imports contrib -# (even if a user isn't using this or any other contrib op), but -# there's not yet any guarantee that the shared object exists. -# In which case, "import tensorflow" will always crash, even for users that -# never use contrib. -def Load(): - """Load training ops library and return the loaded module.""" - with _ops_lock: - global _training_ops - if not _training_ops: - ops_path = resource_loader.get_path_to_datafile(TRAINING_OPS_FILE) - logging.info('data path: %s', ops_path) - _training_ops = loader.load_op_library(ops_path) - - assert _training_ops, 'Could not load _training_ops.so' - return _training_ops diff --git a/tensorflow/contrib/tensor_forest/kernels/data_spec.h b/tensorflow/contrib/tensor_forest/kernels/data_spec.h deleted file mode 100644 index 336a7a32398..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/data_spec.h +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -// This is a surrogate for using a proto, since it doesn't seem to be possible -// to use protos in a dynamically-loaded/shared-linkage library, which is -// what is used for custom ops in tensorflow/contrib. -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_DATA_SPEC_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_DATA_SPEC_H_ -#include - -#include "tensorflow/core/lib/strings/numbers.h" -#include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/platform/logging.h" - -namespace tensorflow { -namespace tensorforest { - -using tensorflow::strings::safe_strto32; - -// DataColumn holds information about one feature of the original data. -// A feature could be dense or sparse, and be of any size. -class DataColumn { - public: - DataColumn() {} - - // Parses a serialized DataColumn produced from the SerializeToString() - // function of a python data_ops.DataColumn object. - // It should look like a proto ASCII format, i.e. - // name: original_type: size: - void ParseFromString(const string& serialized) { - std::vector tokens = tensorflow::str_util::Split(serialized, ' '); - CHECK_EQ(tokens.size(), 6); - name_ = tokens[1]; - safe_strto32(tokens[3], &original_type_); - safe_strto32(tokens[5], &size_); - } - - const string& name() const { return name_; } - - int original_type() const { return original_type_; } - - int size() const { return size_; } - - void set_name(const string& n) { name_ = n; } - - void set_original_type(int o) { original_type_ = o; } - - void set_size(int s) { size_ = s; } - - private: - string name_; - int original_type_; - int size_; -}; - -// TensorForestDataSpec holds information about the original features of the -// data set, which were flattened to a single dense float tensor and/or a -// single sparse float tensor. -class TensorForestDataSpec { - public: - TensorForestDataSpec() {} - - // Parses a serialized DataColumn produced from the SerializeToString() - // function of a python data_ops.TensorForestDataSpec object. - // It should look something like: - // dense_features_size: dense: [{}{}] sparse: [{}] - void ParseFromString(const string& serialized) { - std::vector tokens = tensorflow::str_util::Split(serialized, "[]"); - std::vector first_part = - tensorflow::str_util::Split(tokens[0], ' '); - safe_strto32(first_part[1], &dense_features_size_); - ParseColumns(tokens[1], &dense_); - ParseColumns(tokens[3], &sparse_); - - int total = 0; - for (const DataColumn& col : dense_) { - for (int i = 0; i < col.size(); ++i) { - feature_to_type_.push_back(col.original_type()); - ++total; - } - } - } - - const DataColumn& dense(int i) const { return dense_.at(i); } - - const DataColumn& sparse(int i) const { return sparse_.at(i); } - - DataColumn* mutable_sparse(int i) { return &sparse_[i]; } - - int dense_size() const { return dense_.size(); } - - int sparse_size() const { return sparse_.size(); } - - int dense_features_size() const { return dense_features_size_; } - - void set_dense_features_size(int s) { dense_features_size_ = s; } - - DataColumn* add_dense() { - dense_.push_back(DataColumn()); - return &dense_[dense_.size() - 1]; - } - - int GetDenseFeatureType(int feature) const { - return feature_to_type_[feature]; - } - - private: - void ParseColumns(const string& cols, std::vector* vec) { - std::vector tokens = tensorflow::str_util::Split(cols, "{}"); - for (const string& tok : tokens) { - if (!tok.empty()) { - DataColumn col; - col.ParseFromString(tok); - vec->push_back(col); - } - } - } - - std::vector dense_; - std::vector sparse_; - int dense_features_size_; - - // This map tracks features in the total dense feature space to their - // original type for fast lookup. - std::vector feature_to_type_; -}; - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_DATA_SPEC_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/model_ops.cc b/tensorflow/contrib/tensor_forest/kernels/model_ops.cc deleted file mode 100644 index 5f997c2fba0..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/model_ops.cc +++ /dev/null @@ -1,466 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include - -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model_extensions.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/data_spec.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_data.h" -#include "tensorflow/contrib/tensor_forest/proto/tensor_forest_params.pb.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/lib/core/refcount.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/thread_annotations.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { -namespace tensorforest { - -// Creates a tree variable. -class CreateTreeVariableOp : public OpKernel { - public: - explicit CreateTreeVariableOp(OpKernelConstruction* context) - : OpKernel(context) { - string serialized_params; - OP_REQUIRES_OK(context, context->GetAttr("params", &serialized_params)); - ParseProtoUnlimited(¶m_proto_, serialized_params); - } - - void Compute(OpKernelContext* context) override { - const Tensor* tree_config_t; - OP_REQUIRES_OK(context, context->input("tree_config", &tree_config_t)); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(tree_config_t->shape()), - errors::InvalidArgument("Tree config must be a scalar.")); - - auto* result = new DecisionTreeResource(param_proto_); - if (!ParseProtoUnlimited(result->mutable_decision_tree(), - tree_config_t->scalar()())) { - result->Unref(); - OP_REQUIRES(context, false, - errors::InvalidArgument("Unable to parse tree config.")); - } - - result->MaybeInitialize(); - - // Only create one, if one does not exist already. Report status for all - // other exceptions. - auto status = CreateResource(context, HandleFromInput(context, 0), result); - if (!status.ok() && status.code() != tensorflow::error::ALREADY_EXISTS) { - OP_REQUIRES(context, false, status); - } - } - - private: - TensorForestParams param_proto_; -}; - -// Op for serializing a model. -class TreeSerializeOp : public OpKernel { - public: - explicit TreeSerializeOp(OpKernelConstruction* context) : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr decision_tree_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &decision_tree_resource)); - mutex_lock l(*decision_tree_resource->get_mutex()); - Tensor* output_config_t = nullptr; - OP_REQUIRES_OK( - context, context->allocate_output(0, TensorShape(), &output_config_t)); - output_config_t->scalar()() = - decision_tree_resource->decision_tree().SerializeAsString(); - } -}; - -// Op for deserializing a tree variable from a checkpoint. -class TreeDeserializeOp : public OpKernel { - public: - explicit TreeDeserializeOp(OpKernelConstruction* context) - : OpKernel(context) { - string serialized_params; - OP_REQUIRES_OK(context, context->GetAttr("params", &serialized_params)); - ParseProtoUnlimited(¶m_proto_, serialized_params); - } - - void Compute(OpKernelContext* context) override { - core::RefCountPtr decision_tree_resource; - auto handle = HandleFromInput(context, 0); - OP_REQUIRES_OK(context, - LookupResource(context, handle, &decision_tree_resource)); - mutex_lock l(*decision_tree_resource->get_mutex()); - - const Tensor* tree_config_t; - OP_REQUIRES_OK(context, context->input("tree_config", &tree_config_t)); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(tree_config_t->shape()), - errors::InvalidArgument("Tree config must be a scalar.")); - // Deallocate all the previous objects on the resource. - decision_tree_resource->Reset(); - decision_trees::Model* config = - decision_tree_resource->mutable_decision_tree(); - OP_REQUIRES(context, - ParseProtoUnlimited(config, tree_config_t->scalar()()), - errors::InvalidArgument("Unable to parse tree config.")); - decision_tree_resource->MaybeInitialize(); - } - - private: - TensorForestParams param_proto_; -}; - -// Op for getting tree size. -class TreeSizeOp : public OpKernel { - public: - explicit TreeSizeOp(OpKernelConstruction* context) : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - core::RefCountPtr decision_tree_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &decision_tree_resource)); - mutex_lock l(*decision_tree_resource->get_mutex()); - Tensor* output_t = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(0, TensorShape(), &output_t)); - output_t->scalar()() = - decision_tree_resource->decision_tree().decision_tree().nodes_size(); - } -}; - -void TraverseTree(DecisionTreeResource* tree_resource, - const std::unique_ptr& data, int32 start, - int32 end, - const std::function& set_leaf_id, - std::vector* tree_paths) { - for (int i = start; i < end; ++i) { - const int32 id = tree_resource->TraverseTree( - data, i, nullptr, - (tree_paths == nullptr) ? nullptr : &(*tree_paths)[i]); - set_leaf_id(i, id); - } -} - -// Op for tree inference. -class TreePredictionsV4Op : public OpKernel { - public: - explicit TreePredictionsV4Op(OpKernelConstruction* context) - : OpKernel(context) { - string serialized_params; - OP_REQUIRES_OK(context, context->GetAttr("params", &serialized_params)); - ParseProtoUnlimited(¶m_proto_, serialized_params); - - string serialized_proto; - OP_REQUIRES_OK(context, context->GetAttr("input_spec", &serialized_proto)); - input_spec_.ParseFromString(serialized_proto); - model_op_ = LeafModelOperatorFactory::CreateLeafModelOperator(param_proto_); - } - - void Compute(OpKernelContext* context) override { - const Tensor& input_data = context->input(1); - const Tensor& sparse_input_indices = context->input(2); - const Tensor& sparse_input_values = context->input(3); - const Tensor& sparse_input_shape = context->input(4); - - std::unique_ptr data_set(new TensorDataSet(input_spec_, 0)); - data_set->set_input_tensors(input_data, sparse_input_indices, - sparse_input_values, sparse_input_shape); - - core::RefCountPtr resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &resource)); - mutex_lock l(*resource->get_mutex()); - - const int num_data = data_set->NumItems(); - const int32 num_outputs = param_proto_.num_outputs(); - - Tensor* output_predictions = nullptr; - TensorShape output_shape; - output_shape.AddDim(num_data); - output_shape.AddDim(num_outputs); - OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, - &output_predictions)); - TTypes::Tensor out = output_predictions->tensor(); - - std::vector tree_paths( - param_proto_.inference_tree_paths() ? num_data : 0); - - auto worker_threads = context->device()->tensorflow_cpu_worker_threads(); - int num_threads = worker_threads->num_threads; - const int64 costPerTraverse = 500; - auto traverse = [this, &out, &data_set, &resource, num_data, &tree_paths]( - int64 start, int64 end) { - CHECK(start <= end); - CHECK(end <= num_data); - - TraverseTree(resource.get(), data_set, static_cast(start), - static_cast(end), - std::bind(&TreePredictionsV4Op::set_output_value, this, - std::placeholders::_1, std::placeholders::_2, - resource.get(), &out), - param_proto_.inference_tree_paths() ? &tree_paths : nullptr); - }; - Shard(num_threads, worker_threads->workers, num_data, costPerTraverse, - traverse); - - Tensor* output_tree_paths = nullptr; - TensorShape output_paths_shape; - output_paths_shape.AddDim(param_proto_.inference_tree_paths() ? num_data - : 0); - OP_REQUIRES_OK(context, context->allocate_output(1, output_paths_shape, - &output_tree_paths)); - auto out_paths = output_tree_paths->unaligned_flat(); - - // TODO(gilberth): If this slows down inference too much, consider having - // a filter that only serializes paths for the predicted label that we're - // interested in. - for (int i = 0; i < tree_paths.size(); ++i) { - out_paths(i) = tree_paths[i].SerializeAsString(); - } - } - - void set_output_value(int32 i, int32 id, - DecisionTreeResource* decision_tree_resource, - TTypes::Tensor* out) { - const decision_trees::Leaf& leaf = decision_tree_resource->get_leaf(id); - - float sum = 0; - for (int j = 0; j < param_proto_.num_outputs(); ++j) { - const float count = model_op_->GetOutputValue(leaf, j); - (*out)(i, j) = count; - sum += count; - } - - if (!param_proto_.is_regression() && sum > 0 && sum != 1) { - for (int j = 0; j < param_proto_.num_outputs(); ++j) { - (*out)(i, j) /= sum; - } - } - } - - private: - tensorforest::TensorForestDataSpec input_spec_; - std::unique_ptr model_op_; - TensorForestParams param_proto_; -}; - -// Outputs leaf ids for the given examples. -class TraverseTreeV4Op : public OpKernel { - public: - explicit TraverseTreeV4Op(OpKernelConstruction* context) : OpKernel(context) { - string serialized_params; - OP_REQUIRES_OK(context, context->GetAttr("params", &serialized_params)); - ParseProtoUnlimited(¶m_proto_, serialized_params); - - string serialized_proto; - OP_REQUIRES_OK(context, context->GetAttr("input_spec", &serialized_proto)); - input_spec_.ParseFromString(serialized_proto); - } - - void Compute(OpKernelContext* context) override { - const Tensor& input_data = context->input(1); - const Tensor& sparse_input_indices = context->input(2); - const Tensor& sparse_input_values = context->input(3); - const Tensor& sparse_input_shape = context->input(4); - - std::unique_ptr data_set(new TensorDataSet(input_spec_, 0)); - data_set->set_input_tensors(input_data, sparse_input_indices, - sparse_input_values, sparse_input_shape); - - core::RefCountPtr resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &resource)); - mutex_lock l(*resource->get_mutex()); - - const int num_data = data_set->NumItems(); - - Tensor* output_predictions = nullptr; - TensorShape output_shape; - output_shape.AddDim(num_data); - OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, - &output_predictions)); - - auto leaf_ids = output_predictions->tensor(); - - auto set_leaf_ids = [&leaf_ids](int32 i, int32 id) { leaf_ids(i) = id; }; - - auto worker_threads = context->device()->tensorflow_cpu_worker_threads(); - int num_threads = worker_threads->num_threads; - const int64 costPerTraverse = 500; - auto traverse = [&set_leaf_ids, &data_set, &resource, num_data](int64 start, - int64 end) { - CHECK(start <= end); - CHECK(end <= num_data); - TraverseTree(resource.get(), data_set, static_cast(start), - static_cast(end), set_leaf_ids, nullptr); - }; - Shard(num_threads, worker_threads->workers, num_data, costPerTraverse, - traverse); - } - - private: - tensorforest::TensorForestDataSpec input_spec_; - TensorForestParams param_proto_; -}; - -// Update the given leaf models using the batch of labels. -class UpdateModelV4Op : public OpKernel { - public: - explicit UpdateModelV4Op(OpKernelConstruction* context) : OpKernel(context) { - string serialized_params; - OP_REQUIRES_OK(context, context->GetAttr("params", &serialized_params)); - ParseProtoUnlimited(¶m_proto_, serialized_params); - - model_op_ = LeafModelOperatorFactory::CreateLeafModelOperator(param_proto_); - } - - void Compute(OpKernelContext* context) override { - const Tensor& leaf_ids = context->input(1); - const Tensor& input_labels = context->input(2); - const Tensor& input_weights = context->input(3); - - core::RefCountPtr decision_tree_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &decision_tree_resource)); - mutex_lock l(*decision_tree_resource->get_mutex()); - - const int num_data = input_labels.shape().dim_size(0); - const int32 label_dim = - input_labels.shape().dims() <= 1 - ? 0 - : static_cast(input_labels.shape().dim_size(1)); - const int32 num_targets = - param_proto_.is_regression() ? (std::max(1, label_dim)) : 1; - - TensorInputTarget target(input_labels, input_weights, num_targets); - - // TODO(gilberth): Make this thread safe and multi-thread. - UpdateModel(leaf_ids, target, 0, num_data, decision_tree_resource); - } - - void UpdateModel( - const Tensor& leaf_ids, const TensorInputTarget& target, int32 start, - int32 end, - const core::RefCountPtr& decision_tree_resource) { - const auto leaves = leaf_ids.unaligned_flat(); - for (int i = start; i < end; ++i) { - model_op_->UpdateModel( - decision_tree_resource->get_mutable_tree_node(leaves(i)) - ->mutable_leaf(), - &target, i); - } - } - - private: - std::unique_ptr model_op_; - TensorForestParams param_proto_; -}; - -// Op for getting feature usage counts. -class FeatureUsageCountsOp : public OpKernel { - public: - explicit FeatureUsageCountsOp(OpKernelConstruction* context) - : OpKernel(context) { - string serialized_params; - OP_REQUIRES_OK(context, context->GetAttr("params", &serialized_params)); - ParseProtoUnlimited(¶m_proto_, serialized_params); - } - - void Compute(OpKernelContext* context) override { - core::RefCountPtr decision_tree_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &decision_tree_resource)); - mutex_lock l(*decision_tree_resource->get_mutex()); - - const auto& tree = decision_tree_resource->decision_tree(); - - Tensor* output_counts = nullptr; - TensorShape output_shape; - output_shape.AddDim(param_proto_.num_features()); - OP_REQUIRES_OK(context, - context->allocate_output(0, output_shape, &output_counts)); - - auto counts = output_counts->unaligned_flat(); - counts.setZero(); - - for (const auto& node : tree.decision_tree().nodes()) { - if (node.has_custom_node_type()) { - LOG(WARNING) << "Can't count feature usage for custom nodes."; - } else if (node.has_binary_node()) { - const auto& bnode = node.binary_node(); - if (bnode.has_custom_left_child_test()) { - decision_trees::MatchingValuesTest test; - if (!bnode.custom_left_child_test().UnpackTo(&test)) { - LOG(WARNING) << "Unknown custom child test"; - continue; - } - int32 feat; - safe_strto32(test.feature_id().id().value(), &feat); - ++counts(feat); - } else { - const auto& test = bnode.inequality_left_child_test(); - if (test.has_feature_id()) { - int32 feat; - safe_strto32(test.feature_id().id().value(), &feat); - ++counts(feat); - } else if (test.has_oblique()) { - for (const auto& featid : test.oblique().features()) { - int32 feat; - safe_strto32(featid.id().value(), &feat); - ++counts(feat); - } - } - } - } - } - } - - private: - TensorForestParams param_proto_; -}; - -REGISTER_RESOURCE_HANDLE_KERNEL(DecisionTreeResource); - -REGISTER_KERNEL_BUILDER(Name("TreeIsInitializedOp").Device(DEVICE_CPU), - IsResourceInitialized); - -REGISTER_KERNEL_BUILDER(Name("CreateTreeVariable").Device(DEVICE_CPU), - CreateTreeVariableOp); - -REGISTER_KERNEL_BUILDER(Name("TreeSerialize").Device(DEVICE_CPU), - TreeSerializeOp); - -REGISTER_KERNEL_BUILDER(Name("TreeDeserialize").Device(DEVICE_CPU), - TreeDeserializeOp); - -REGISTER_KERNEL_BUILDER(Name("TreeSize").Device(DEVICE_CPU), TreeSizeOp); - -REGISTER_KERNEL_BUILDER(Name("TreePredictionsV4").Device(DEVICE_CPU), - TreePredictionsV4Op); - -REGISTER_KERNEL_BUILDER(Name("TraverseTreeV4").Device(DEVICE_CPU), - TraverseTreeV4Op); - -REGISTER_KERNEL_BUILDER(Name("FeatureUsageCounts").Device(DEVICE_CPU), - FeatureUsageCountsOp); - -REGISTER_KERNEL_BUILDER(Name("UpdateModelV4").Device(DEVICE_CPU), - UpdateModelV4Op); - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/model_ops_test.cc b/tensorflow/contrib/tensor_forest/kernels/model_ops_test.cc deleted file mode 100644 index 1ac1d27761d..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/model_ops_test.cc +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/core/framework/node_def_builder.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference_testutil.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { - -TEST(ModelOpsTest, CreateTreeVariable_ShapeFn) { - ShapeInferenceTestOp op("CreateTreeVariable"); - INFER_OK(op, "[1];[1]", ""); -} - -TEST(ModelOpsTest, TreeSerialize_ShapeFn) { - ShapeInferenceTestOp op("TreeSerialize"); - INFER_OK(op, "[1]", "[]"); -} - -TEST(ModelOpsTest, TreeDeserialize_ShapeFn) { - ShapeInferenceTestOp op("TreeDeserialize"); - INFER_OK(op, "[1];[1]", ""); -} - -TEST(ModelOpsTest, TreeSize_ShapeFn) { - ShapeInferenceTestOp op("TreeSize"); - INFER_OK(op, "[1]", "[]"); -} - -TEST(ModelOpsTest, TreePredictionsV4_ShapeFn) { - ShapeInferenceTestOp op("TreePredictionsV4"); - TF_ASSERT_OK(NodeDefBuilder("test", "TreePredictionsV4") - .Input("a", 0, DT_RESOURCE) - .Input("b", 1, DT_FLOAT) - .Input("c", 2, DT_INT64) - .Input("d", 3, DT_FLOAT) - .Input("e", 5, DT_INT64) - .Attr("input_spec", "") - .Attr("params", "") - .Finalize(&op.node_def)); - - // num_points = 2, sparse shape not known - INFER_OK(op, "?;[2,3];?;?;?", "[d1_0,?];[?]"); - - // num_points = 2, sparse and dense shape rank known and > 1 - INFER_OK(op, "?;[2,3];?;?;[10,11]", "[d1_0,?];[?]"); - - // num_points = 2, sparse shape rank known and > 1 - INFER_OK(op, "?;?;?;?;[10,11]", "[?,?];[?]"); -} - -TEST(ModelOpsTest, TraverseTreeV4_ShapeFn) { - ShapeInferenceTestOp op("TraverseTreeV4"); - TF_ASSERT_OK(NodeDefBuilder("test", "TraverseTreeV4") - .Input("a", 0, DT_RESOURCE) - .Input("b", 1, DT_FLOAT) - .Input("c", 2, DT_INT64) - .Input("d", 3, DT_FLOAT) - .Input("e", 5, DT_INT64) - .Attr("input_spec", "") - .Attr("params", "") - .Finalize(&op.node_def)); - - // num_points = 2, sparse shape not known - INFER_OK(op, "?;[2,3];?;?;?", "[d1_0]"); - - // num_points = 2, sparse and dense shape rank known and > 1 - INFER_OK(op, "?;[2,3];?;?;[10,11]", "[d1_0]"); - - // num_points = 2, sparse shape rank known and > 1 - INFER_OK(op, "?;?;?;?;[10,11]", "[?]"); -} - -TEST(ModelOpsTest, UpdateModelV4_ShapeFn) { - ShapeInferenceTestOp op("UpdateModelV4"); - INFER_OK(op, "[1];?;?;?", ""); -} - -TEST(ModelOpsTest, FeatureUsageCounts_ShapeFn) { - ShapeInferenceTestOp op("FeatureUsageCounts"); - INFER_OK(op, "[1]", "[?]"); -} - -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/reinterpret_string_to_float_op.cc b/tensorflow/contrib/tensor_forest/kernels/reinterpret_string_to_float_op.cc deleted file mode 100644 index fcea240dee9..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/reinterpret_string_to_float_op.cc +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -// Converts strings of arbitrary length to float values by -// hashing and cramming bits. -#include - -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" - -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/lib/strings/numbers.h" -#include "tensorflow/core/lib/strings/strcat.h" - -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -using tensorforest::CheckTensorBounds; - -float Convert(const string& in) { - const std::size_t intval = std::hash()(in); - return static_cast(intval); -} - -void Evaluate(const Tensor& input_data, Tensor output_data, int32 start, - int32 end) { - auto out_data = output_data.unaligned_flat(); - const auto in_data = input_data.unaligned_flat(); - - for (int32 i = start; i < end; ++i) { - out_data(i) = Convert(in_data(i)); - } -} - -class ReinterpretStringToFloat : public OpKernel { - public: - explicit ReinterpretStringToFloat(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - const Tensor& input_data = context->input(0); - - // Check tensor bounds. - if (!CheckTensorBounds(context, input_data)) return; - - Tensor* output_data = nullptr; - OP_REQUIRES_OK( - context, context->allocate_output(0, input_data.shape(), &output_data)); - - // Evaluate input data in parallel. - const int32 num_data = static_cast(input_data.NumElements()); - auto worker_threads = context->device()->tensorflow_cpu_worker_threads(); - int num_threads = worker_threads->num_threads; - if (num_threads <= 1) { - Evaluate(input_data, *output_data, 0, num_data); - } else { - auto work = [&input_data, output_data, num_data](int64 start, int64 end) { - CHECK(start <= end); - CHECK(end <= num_data); - Evaluate(input_data, *output_data, static_cast(start), - static_cast(end)); - }; - Shard(num_threads, worker_threads->workers, num_data, 100, work); - } - } -}; - -REGISTER_KERNEL_BUILDER(Name("ReinterpretStringToFloat").Device(DEVICE_CPU), - ReinterpretStringToFloat); - -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/scatter_add_ndim_op.cc b/tensorflow/contrib/tensor_forest/kernels/scatter_add_ndim_op.cc deleted file mode 100644 index 60740c2be37..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/scatter_add_ndim_op.cc +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -// ScatterAddNdim implements a scatter_add that can operate on sparse -// updates without being limited to the first dimension for indices. - -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" - -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/platform/logging.h" - -namespace tensorflow { - -using tensorforest::CheckTensorBounds; - -class ScatterAddNdim : public OpKernel { - public: - explicit ScatterAddNdim(OpKernelConstruction* context) : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - Tensor input_tensor = context->mutable_input(0, false); - const Tensor& indices_tensor = context->input(1); - const Tensor& deltas_tensor = context->input(2); - - if (indices_tensor.shape().dim_size(0) > 0) { - OP_REQUIRES(context, indices_tensor.shape().dims() == 2, - errors::InvalidArgument("indices should be two-dimensional")); - const int32 delta_dims = deltas_tensor.shape().dims(); - OP_REQUIRES( - context, - indices_tensor.shape().dim_size(1) + delta_dims == - input_tensor.shape().dims() + 1, - errors::InvalidArgument( - "Number of indices dimensions should be the same as input " - "rank.")); - OP_REQUIRES( - context, - indices_tensor.shape().dim_size(0) == - deltas_tensor.shape().dim_size(0), - errors::InvalidArgument( - "Number of updates should be same as number of indices.")); - } else { - return; - } - - // Check tensor bounds. - if (!CheckTensorBounds(context, input_tensor)) return; - if (!CheckTensorBounds(context, indices_tensor)) return; - if (!CheckTensorBounds(context, deltas_tensor)) return; - - auto input = input_tensor.flat(); - - const auto indices = indices_tensor.tensor(); - const auto deltas = deltas_tensor.unaligned_flat(); - - const int32 num_dims = - static_cast(indices_tensor.shape().dim_size(1)); - - // Figure out if indices don't specify a complete position in the - // input tensor. - int32 num_data_per_index = 1; - for (int32 i = 0; i < input_tensor.shape().dims() - num_dims; ++i) { - num_data_per_index *= input_tensor.shape().dim_size(num_dims + i); - } - - // Calculate index multipliers. - std::vector multipliers; - OP_REQUIRES(context, input.size() < std::numeric_limits::max(), - errors::InvalidArgument( - "Input must contain less than 2^31 total elements")); - int32 last_size = static_cast(input.size()); - - for (int32 j = 0; j < num_dims; j++) { - const int32 m = last_size / input_tensor.shape().dim_size(j); - multipliers.push_back(m); - last_size = m; - } - - // Perform updates. - for (int32 i = 0; i < indices_tensor.shape().dim_size(0); i++) { - int32 start_index = 0; - for (int32 j = 0; j < num_dims; j++) { - start_index += indices(i, j) * multipliers[j]; - } - for (int32 offset = 0; offset < num_data_per_index; ++offset) { - const int32 input_index = start_index + offset; - const int32 delta_index = i * num_data_per_index + offset; - CHECK(input_index < input.size()); - CHECK(delta_index < deltas.size()); - input(input_index) += deltas(delta_index); - } - } - } -}; - -REGISTER_KERNEL_BUILDER(Name("ScatterAddNdim").Device(DEVICE_CPU), - ScatterAddNdim); -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/stats_ops.cc b/tensorflow/contrib/tensor_forest/kernels/stats_ops.cc deleted file mode 100644 index e4693cf68dc..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/stats_ops.cc +++ /dev/null @@ -1,512 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include - -#include "tensorflow/contrib/tensor_forest/kernels/data_spec.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_data.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_target.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/params.h" -#include "tensorflow/contrib/tensor_forest/proto/fertile_stats.pb.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/lib/core/refcount.h" -#include "tensorflow/core/lib/gtl/map_util.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/thread_annotations.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { -namespace tensorforest { - -using gtl::FindOrNull; - -// Creates a stats variable. -class CreateFertileStatsVariableOp : public OpKernel { - public: - explicit CreateFertileStatsVariableOp(OpKernelConstruction* context) - : OpKernel(context) { - string serialized_params; - OP_REQUIRES_OK(context, context->GetAttr("params", &serialized_params)); - ParseProtoUnlimited(¶m_proto_, serialized_params); - } - - void Compute(OpKernelContext* context) override { - const Tensor* stats_config_t; - OP_REQUIRES_OK(context, context->input("stats_config", &stats_config_t)); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(stats_config_t->shape()), - errors::InvalidArgument("Stats config must be a scalar.")); - auto* result = new FertileStatsResource(param_proto_); - FertileStats stats; - if (!ParseProtoUnlimited(&stats, stats_config_t->scalar()())) { - result->Unref(); - OP_REQUIRES(context, false, - errors::InvalidArgument("Unable to parse stats config.")); - } - - result->ExtractFromProto(stats); - result->MaybeInitialize(); - - // Only create one, if one does not exist already. Report status for all - // other exceptions. - auto status = CreateResource(context, HandleFromInput(context, 0), result); - if (!status.ok() && status.code() != tensorflow::error::ALREADY_EXISTS) { - OP_REQUIRES(context, false, status); - } - } - - private: - TensorForestParams param_proto_; -}; - -// Op for serializing a model. -class FertileStatsSerializeOp : public OpKernel { - public: - explicit FertileStatsSerializeOp(OpKernelConstruction* context) - : OpKernel(context) { - string serialized_params; - OP_REQUIRES_OK(context, context->GetAttr("params", &serialized_params)); - ParseProtoUnlimited(¶m_proto_, serialized_params); - } - - void Compute(OpKernelContext* context) override { - core::RefCountPtr fertile_stats_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &fertile_stats_resource)); - mutex_lock l(*fertile_stats_resource->get_mutex()); - Tensor* output_config_t = nullptr; - OP_REQUIRES_OK( - context, context->allocate_output(0, TensorShape(), &output_config_t)); - - FertileStats stats; - fertile_stats_resource->PackToProto(&stats); - output_config_t->scalar()() = stats.SerializeAsString(); - } - - private: - TensorForestParams param_proto_; -}; - -// Op for deserializing a stats variable from a checkpoint. -class FertileStatsDeserializeOp : public OpKernel { - public: - explicit FertileStatsDeserializeOp(OpKernelConstruction* context) - : OpKernel(context) { - string serialized_params; - OP_REQUIRES_OK(context, context->GetAttr("params", &serialized_params)); - ParseProtoUnlimited(¶m_proto_, serialized_params); - } - - void Compute(OpKernelContext* context) override { - core::RefCountPtr fertile_stats_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &fertile_stats_resource)); - mutex_lock l(*fertile_stats_resource->get_mutex()); - - const Tensor* stats_config_t; - OP_REQUIRES_OK(context, context->input("stats_config", &stats_config_t)); - OP_REQUIRES(context, TensorShapeUtils::IsScalar(stats_config_t->shape()), - errors::InvalidArgument("Stats config must be a scalar.")); - // Deallocate all the previous objects on the resource. - fertile_stats_resource->Reset(); - FertileStats stats; - OP_REQUIRES( - context, - ParseProtoUnlimited(&stats, stats_config_t->scalar()()), - errors::InvalidArgument("Unable to parse stats config.")); - - fertile_stats_resource->ExtractFromProto(stats); - fertile_stats_resource->MaybeInitialize(); - } - - private: - TensorForestParams param_proto_; -}; - -// Try to update a leaf's stats by acquiring its lock. If it can't be -// acquired, put it in a waiting queue to come back to later and try the next -// one. Once all leaf_ids have been visited, cycle through the waiting ids -// until they're gone. -void UpdateStats(const core::RefCountPtr& resource, - const std::unique_ptr& data, - const TensorInputTarget& target, int num_targets, - const Tensor& leaf_ids_tensor, - std::unordered_map>* locks, - mutex* set_lock, int32 start, int32 end, - std::unordered_set* ready_to_split) { - const auto leaf_ids = leaf_ids_tensor.unaligned_flat(); - - // Stores leaf_id, leaf_depth, example_id for examples that are waiting - // on another to finish. - std::queue> waiting; - - int32 i = start; - while (i < end || !waiting.empty()) { - int32 leaf_id; - int32 example_id; - bool was_waiting = false; - if (i >= end) { - std::tie(leaf_id, example_id) = waiting.front(); - waiting.pop(); - was_waiting = true; - } else { - leaf_id = leaf_ids(i); - example_id = i; - ++i; - } - const std::unique_ptr& leaf_lock = (*locks)[leaf_id]; - if (was_waiting) { - leaf_lock->lock(); - } else { - if (!leaf_lock->try_lock()) { - waiting.emplace(leaf_id, example_id); - continue; - } - } - - bool is_finished; - resource->AddExampleToStatsAndInitialize(data, &target, {example_id}, - leaf_id, &is_finished); - leaf_lock->unlock(); - if (is_finished) { - set_lock->lock(); - ready_to_split->insert(leaf_id); - set_lock->unlock(); - } - } -} - -// Update leaves from start through end in the leaf_examples iterator. -void UpdateStatsCollated( - const core::RefCountPtr& fertile_stats_resource, - const core::RefCountPtr& tree_resource, - const std::unique_ptr& data, const TensorInputTarget& target, - int num_targets, - const std::unordered_map>& leaf_examples, - mutex* set_lock, int32 start, int32 end, - std::unordered_set* ready_to_split) { - auto it = leaf_examples.begin(); - std::advance(it, start); - auto end_it = leaf_examples.begin(); - std::advance(end_it, end); - while (it != end_it) { - int32 leaf_id = it->first; - bool is_finished; - fertile_stats_resource->AddExampleToStatsAndInitialize( - data, &target, it->second, leaf_id, &is_finished); - if (is_finished) { - set_lock->lock(); - ready_to_split->insert(leaf_id); - set_lock->unlock(); - } - ++it; - } -} - -// Op for traversing the tree with each example, accumulating statistics, and -// outputting node ids that are ready to split. -class ProcessInputOp : public OpKernel { - public: - explicit ProcessInputOp(OpKernelConstruction* context) : OpKernel(context) { - string serialized_params; - OP_REQUIRES_OK(context, context->GetAttr("params", &serialized_params)); - ParseProtoUnlimited(¶m_proto_, serialized_params); - - OP_REQUIRES_OK(context, context->GetAttr("random_seed", &random_seed_)); - - string serialized_proto; - OP_REQUIRES_OK(context, context->GetAttr("input_spec", &serialized_proto)); - input_spec_.ParseFromString(serialized_proto); - } - - void Compute(OpKernelContext* context) override { - const Tensor& input_data = context->input(2); - const Tensor& sparse_input_indices = context->input(3); - const Tensor& sparse_input_values = context->input(4); - const Tensor& sparse_input_shape = context->input(5); - const Tensor& input_labels = context->input(6); - const Tensor& input_weights = context->input(7); - const Tensor& leaf_ids_tensor = context->input(8); - - std::unique_ptr data_set( - new TensorDataSet(input_spec_, random_seed_)); - data_set->set_input_tensors(input_data, sparse_input_indices, - sparse_input_values, sparse_input_shape); - - core::RefCountPtr fertile_stats_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 1), - &fertile_stats_resource)); - core::RefCountPtr tree_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &tree_resource)); - mutex_lock l1(*fertile_stats_resource->get_mutex()); - mutex_lock l2(*tree_resource->get_mutex()); - - const int32 num_data = data_set->NumItems(); - auto worker_threads = context->device()->tensorflow_cpu_worker_threads(); - int num_threads = worker_threads->num_threads; - - const auto leaf_ids = leaf_ids_tensor.unaligned_flat(); - - // Create one mutex per leaf. We need to protect access to leaf pointers, - // so instead of grouping examples by leaf, we spread examples out among - // threads to provide uniform work for each of them and protect access - // with mutexes. - std::unordered_map> locks; - std::unordered_map> leaf_examples; - if (param_proto_.collate_examples()) { - for (int i = 0; i < num_data; ++i) { - leaf_examples[leaf_ids(i)].push_back(i); - } - } else { - for (int i = 0; i < num_data; ++i) { - const int32 id = leaf_ids(i); - if (FindOrNull(locks, id) == nullptr) { - // TODO(gilberth): Consider using a memory pool for these. - locks[id] = std::unique_ptr(new mutex); - } - } - } - - const int32 num_leaves = leaf_examples.size(); - const int32 label_dim = - input_labels.shape().dims() <= 1 - ? 0 - : static_cast(input_labels.shape().dim_size(1)); - const int32 num_targets = - param_proto_.is_regression() ? (std::max(1, label_dim)) : 1; - - // Ids of leaves that can split. - std::unordered_set ready_to_split; - mutex set_lock; - - TensorInputTarget target(input_labels, input_weights, num_targets); - - // TODO(gilberth): This is a rough approximation based on measurements - // from a digits run on local desktop. Heuristics might be necessary - // if it really matters that much. - const int64 costPerUpdate = 1000; - auto update = [&target, &leaf_ids_tensor, &num_targets, &data_set, - &fertile_stats_resource, &locks, &set_lock, &ready_to_split, - num_data](int64 start, int64 end) { - CHECK(start <= end); - CHECK(end <= num_data); - UpdateStats(fertile_stats_resource, data_set, target, num_targets, - leaf_ids_tensor, &locks, &set_lock, static_cast(start), - static_cast(end), &ready_to_split); - }; - - auto update_collated = [&target, &num_targets, &fertile_stats_resource, - &tree_resource, &leaf_examples, &set_lock, - &ready_to_split, &data_set, - num_leaves](int64 start, int64 end) { - CHECK(start <= end); - CHECK(end <= num_leaves); - UpdateStatsCollated(fertile_stats_resource, tree_resource, data_set, - target, num_targets, leaf_examples, &set_lock, - static_cast(start), static_cast(end), - &ready_to_split); - }; - - if (param_proto_.collate_examples()) { - Shard(num_threads, worker_threads->workers, num_leaves, costPerUpdate, - update_collated); - } else { - Shard(num_threads, worker_threads->workers, num_data, costPerUpdate, - update); - } - - Tensor* output_finished_t = nullptr; - TensorShape output_shape; - output_shape.AddDim(ready_to_split.size()); - OP_REQUIRES_OK( - context, context->allocate_output(0, output_shape, &output_finished_t)); - auto output = output_finished_t->unaligned_flat(); - std::copy(ready_to_split.begin(), ready_to_split.end(), output.data()); - } - - private: - int32 random_seed_; - tensorforest::TensorForestDataSpec input_spec_; - TensorForestParams param_proto_; -}; - -// Op for growing finished nodes. -class GrowTreeOp : public OpKernel { - public: - explicit GrowTreeOp(OpKernelConstruction* context) : OpKernel(context) { - string serialized_params; - OP_REQUIRES_OK(context, context->GetAttr("params", &serialized_params)); - ParseProtoUnlimited(¶m_proto_, serialized_params); - } - - void Compute(OpKernelContext* context) override { - core::RefCountPtr fertile_stats_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 1), - &fertile_stats_resource)); - core::RefCountPtr tree_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &tree_resource)); - mutex_lock l1(*fertile_stats_resource->get_mutex()); - mutex_lock l2(*tree_resource->get_mutex()); - - const Tensor& finished_nodes = context->input(2); - - const auto finished = finished_nodes.unaligned_flat(); - - const int32 num_nodes = - static_cast(finished_nodes.shape().dim_size(0)); - - // This op takes so little of the time for one batch that it isn't worth - // threading this. - for (int i = 0; - i < num_nodes && - tree_resource->decision_tree().decision_tree().nodes_size() < - param_proto_.max_nodes(); - ++i) { - const int32 node = finished(i); - std::unique_ptr best(new SplitCandidate); - int32 parent_depth; - // TODO(gilberth): Pushing these to an output would allow the complete - // decoupling of tree from resource. - bool found = - fertile_stats_resource->BestSplit(node, best.get(), &parent_depth); - if (found) { - std::vector new_children; - tree_resource->SplitNode(node, best.get(), &new_children); - fertile_stats_resource->Allocate(parent_depth, new_children); - // We are done with best, so it is now safe to clear node. - fertile_stats_resource->Clear(node); - CHECK(tree_resource->get_mutable_tree_node(node)->has_leaf() == false); - } else { // reset - fertile_stats_resource->ResetSplitStats(node, parent_depth); - } - } - } - - private: - tensorforest::TensorForestDataSpec input_spec_; - TensorForestParams param_proto_; -}; - -void FinalizeLeaf(bool is_regression, bool drop_final_class, - const std::unique_ptr& leaf_op, - decision_trees::Leaf* leaf) { - // regression models are already stored in leaf in normalized form. - if (is_regression) { - return; - } - - // TODO(gilberth): Calculate the leaf's sum. - float sum = 0; - LOG(FATAL) << "FinalizeTreeOp is disabled for now."; - if (sum <= 0.0) { - LOG(WARNING) << "Leaf with sum " << sum << " has stats " - << leaf->ShortDebugString(); - return; - } - - if (leaf->has_vector()) { - for (int i = 0; i < leaf->vector().value_size(); i++) { - auto* v = leaf->mutable_vector()->mutable_value(i); - v->set_float_value(v->float_value() / sum); - } - if (drop_final_class) { - leaf->mutable_vector()->mutable_value()->RemoveLast(); - } - return; - } - - if (leaf->has_sparse_vector()) { - for (auto& it : *leaf->mutable_sparse_vector()->mutable_sparse_value()) { - it.second.set_float_value(it.second.float_value() / sum); - } - return; - } - - LOG(FATAL) << "Unknown leaf type in " << leaf->DebugString(); -} - -// Op for finalizing a tree at the end of training. -class FinalizeTreeOp : public OpKernel { - public: - explicit FinalizeTreeOp(OpKernelConstruction* context) : OpKernel(context) { - string serialized_params; - OP_REQUIRES_OK(context, context->GetAttr("params", &serialized_params)); - ParseProtoUnlimited(¶m_proto_, serialized_params); - - model_op_ = LeafModelOperatorFactory::CreateLeafModelOperator(param_proto_); - } - - void Compute(OpKernelContext* context) override { - core::RefCountPtr tree_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), - &tree_resource)); - core::RefCountPtr fertile_stats_resource; - OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 1), - &fertile_stats_resource)); - - mutex_lock l1(*fertile_stats_resource->get_mutex()); - mutex_lock l2(*tree_resource->get_mutex()); - - // TODO(thomaswc): Add threads - int num_nodes = tree_resource->decision_tree().decision_tree().nodes_size(); - for (int i = 0; i < num_nodes; i++) { - auto* node = tree_resource->mutable_decision_tree() - ->mutable_decision_tree() - ->mutable_nodes(i); - if (node->has_leaf()) { - FinalizeLeaf(param_proto_.is_regression(), - param_proto_.drop_final_class(), model_op_, - node->mutable_leaf()); - } - } - } - - private: - std::unique_ptr model_op_; - TensorForestParams param_proto_; -}; - -REGISTER_RESOURCE_HANDLE_KERNEL(FertileStatsResource); - -REGISTER_KERNEL_BUILDER(Name("FertileStatsIsInitializedOp").Device(DEVICE_CPU), - IsResourceInitialized); - -REGISTER_KERNEL_BUILDER(Name("CreateFertileStatsVariable").Device(DEVICE_CPU), - CreateFertileStatsVariableOp); - -REGISTER_KERNEL_BUILDER(Name("FertileStatsSerialize").Device(DEVICE_CPU), - FertileStatsSerializeOp); - -REGISTER_KERNEL_BUILDER(Name("FertileStatsDeserialize").Device(DEVICE_CPU), - FertileStatsDeserializeOp); - -REGISTER_KERNEL_BUILDER(Name("ProcessInputV4").Device(DEVICE_CPU), - ProcessInputOp); - -REGISTER_KERNEL_BUILDER(Name("GrowTreeV4").Device(DEVICE_CPU), GrowTreeOp); - -REGISTER_KERNEL_BUILDER(Name("FinalizeTree").Device(DEVICE_CPU), - FinalizeTreeOp); - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/stats_ops_test.cc b/tensorflow/contrib/tensor_forest/kernels/stats_ops_test.cc deleted file mode 100644 index b3aa3a96f43..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/stats_ops_test.cc +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/core/framework/node_def_builder.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference_testutil.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { - -TEST(StatsOpsTest, CreateFertileStatsVariable_ShapeFn) { - ShapeInferenceTestOp op("CreateFertileStatsVariable"); - INFER_OK(op, "[1];[1]", ""); -} - -TEST(StatsOpsTest, FertileStatsSerialize_ShapeFn) { - ShapeInferenceTestOp op("FertileStatsSerialize"); - INFER_OK(op, "[1]", "[]"); -} - -TEST(StatsOpsTest, FertileStatsDeserialize_ShapeFn) { - ShapeInferenceTestOp op("FertileStatsDeserialize"); - INFER_OK(op, "[1];[1]", ""); -} - -TEST(StatsOpsTest, GrowTreeV4_ShapeFn) { - ShapeInferenceTestOp op("GrowTreeV4"); - INFER_OK(op, "[1];[1];?", ""); -} - -TEST(StatsOpsTest, ProcessInputV4_ShapeFn) { - ShapeInferenceTestOp op("ProcessInputV4"); - INFER_OK(op, "[1];[1];?;?;?;?;?;?;?", "[?]"); -} - -TEST(StatsOpsTest, FinalizeTree_ShapeFn) { - ShapeInferenceTestOp op("FinalizeTree"); - INFER_OK(op, "[1];[1]", ""); -} - -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/tensor_forest_ops_test.cc b/tensorflow/contrib/tensor_forest/kernels/tensor_forest_ops_test.cc deleted file mode 100644 index 8b33e0d8191..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/tensor_forest_ops_test.cc +++ /dev/null @@ -1,32 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); - -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// TODO(cwhipkey): iwyu -#include "tensorflow/core/framework/node_def_builder.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference_testutil.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { - -TEST(TrainingOpsTest, ScatterAddNdim_ShapeFn) { - ShapeInferenceTestOp op("ScatterAddNdim"); - INFER_OK(op, "?;?;?", ""); -} - -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/tree_utils.cc b/tensorflow/contrib/tensor_forest/kernels/tree_utils.cc deleted file mode 100644 index dd5d0283141..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/tree_utils.cc +++ /dev/null @@ -1,628 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" -#include -#include -#include "tensorflow/core/lib/random/philox_random.h" -#include "tensorflow/core/platform/logging.h" - -namespace tensorflow { -namespace tensorforest { - -using tensorflow::Tensor; - -DataColumnTypes FindDenseFeatureSpec( - int32 input_feature, const tensorforest::TensorForestDataSpec& spec) { - return static_cast(spec.GetDenseFeatureType(input_feature)); -} - -DataColumnTypes FindSparseFeatureSpec( - int32 input_feature, const tensorforest::TensorForestDataSpec& spec) { - // TODO(thomaswc): Binary search here, especially when we start using more - // than one sparse column - int32 size_sum = spec.sparse(0).size(); - int32 column_num = 0; - while (input_feature >= size_sum && column_num < spec.sparse_size()) { - ++column_num; - size_sum += spec.sparse(column_num).size(); - } - - return static_cast(spec.sparse(column_num).original_type()); -} - -void GetTwoBest(int max, const std::function& score_fn, - float* best_score, int* best_index, float* second_best_score, - int* second_best_index) { - *best_index = -1; - *second_best_index = -1; - *best_score = FLT_MAX; - *second_best_score = FLT_MAX; - for (int i = 0; i < max; i++) { - float score = score_fn(i); - if (score < *best_score) { - *second_best_score = *best_score; - *second_best_index = *best_index; - *best_score = score; - *best_index = i; - } else if (score < *second_best_score) { - *second_best_score = score; - *second_best_index = i; - } - } -} - -float ClassificationSplitScore( - const Eigen::Tensor& splits, - const Eigen::Tensor& rights, int32 num_classes, - int i) { - Eigen::array offsets; - // Class counts are stored with the total in [0], so the length of each - // count vector is num_classes + 1. - offsets[0] = i * (num_classes + 1) + 1; - Eigen::array extents; - extents[0] = num_classes; - return WeightedGiniImpurity(splits.slice(offsets, extents)) + - WeightedGiniImpurity(rights.slice(offsets, extents)); -} - -void GetTwoBestClassification(const Tensor& total_counts, - const Tensor& split_counts, int32 accumulator, - float* best_score, int* best_index, - float* second_best_score, - int* second_best_index) { - const int32 num_splits = static_cast(split_counts.shape().dim_size(1)); - const int32 num_classes = - static_cast(split_counts.shape().dim_size(2)) - 1; - - // Ideally, Eigen::Tensor::chip would be best to use here but it results - // in seg faults, so we have to go with flat views of these tensors. However, - // it is still pretty efficient because we put off evaluation until the - // score is actually returned. - const auto tc = - total_counts.Slice(accumulator, accumulator + 1).unaligned_flat(); - - // TODO(gilberth): See if we can delay evaluation here by templating the - // arguments to ClassificationSplitScore. - const Eigen::Tensor splits = - split_counts.Slice(accumulator, accumulator + 1).unaligned_flat(); - Eigen::array bcast; - bcast[0] = num_splits; - const Eigen::Tensor rights = - tc.broadcast(bcast) - splits; - - std::function score_fn = - std::bind(ClassificationSplitScore, splits, rights, num_classes, - std::placeholders::_1); - - GetTwoBest(num_splits, score_fn, best_score, best_index, second_best_score, - second_best_index); -} - -int32 BestFeatureClassification(const Tensor& total_counts, - const Tensor& split_counts, int32 accumulator) { - float best_score; - float second_best_score; - int best_feature_index; - int second_best_index; - GetTwoBestClassification(total_counts, split_counts, accumulator, &best_score, - &best_feature_index, &second_best_score, - &second_best_index); - return best_feature_index; -} - -float RegressionSplitScore( - const Eigen::Tensor& splits_count_accessor, - const Eigen::Tensor& totals_count_accessor, - const Eigen::Tensor& splits_sum, - const Eigen::Tensor& splits_square, - const Eigen::Tensor& right_sums, - const Eigen::Tensor& right_squares, - int32 accumulator, int32 num_regression_dims, int i) { - Eigen::array offsets = {i * num_regression_dims + 1}; - Eigen::array extents = {num_regression_dims - 1}; - float left_count = splits_count_accessor(accumulator, i, 0); - float right_count = totals_count_accessor(accumulator, 0) - left_count; - - float score = 0; - - // Guard against divide-by-zero. - if (left_count > 0) { - score += - WeightedVariance(splits_sum.slice(offsets, extents), - splits_square.slice(offsets, extents), left_count); - } - - if (right_count > 0) { - score += - WeightedVariance(right_sums.slice(offsets, extents), - right_squares.slice(offsets, extents), right_count); - } - return score; -} - -void GetTwoBestRegression(const Tensor& total_sums, const Tensor& total_squares, - const Tensor& split_sums, const Tensor& split_squares, - int32 accumulator, float* best_score, int* best_index, - float* second_best_score, int* second_best_index) { - const int32 num_splits = static_cast(split_sums.shape().dim_size(1)); - const int32 num_regression_dims = - static_cast(split_sums.shape().dim_size(2)); - // Ideally, Eigen::Tensor::chip would be best to use here but it results - // in seg faults, so we have to go with flat views of these tensors. However, - // it is still pretty efficient because we put off evaluation until the - // score is actually returned. - const auto tc_sum = - total_sums.Slice(accumulator, accumulator + 1).unaligned_flat(); - const auto tc_square = - total_squares.Slice(accumulator, accumulator + 1).unaligned_flat(); - const auto splits_sum = - split_sums.Slice(accumulator, accumulator + 1).unaligned_flat(); - const auto splits_square = - split_squares.Slice(accumulator, accumulator + 1).unaligned_flat(); - // Eigen is infuriating to work with, usually resulting in all kinds of - // unhelpful compiler errors when trying something that seems sane. This - // helps us do a simple thing like access the first element (the counts) - // of these tensors so we can calculate expected value in Variance(). - const auto splits_count_accessor = split_sums.tensor(); - const auto totals_count_accessor = total_sums.tensor(); - - Eigen::array bcast; - bcast[0] = num_splits; - const auto right_sums = tc_sum.broadcast(bcast) - splits_sum; - const auto right_squares = tc_square.broadcast(bcast) - splits_square; - - GetTwoBest(num_splits, - std::bind(RegressionSplitScore, splits_count_accessor, - totals_count_accessor, splits_sum, splits_square, - right_sums, right_squares, accumulator, - num_regression_dims, std::placeholders::_1), - best_score, best_index, second_best_score, second_best_index); -} - -int32 BestFeatureRegression(const Tensor& total_sums, - const Tensor& total_squares, - const Tensor& split_sums, - const Tensor& split_squares, int32 accumulator) { - float best_score; - float second_best_score; - int best_feature_index; - int second_best_index; - GetTwoBestRegression(total_sums, total_squares, split_sums, split_squares, - accumulator, &best_score, &best_feature_index, - &second_best_score, &second_best_index); - return best_feature_index; -} - -bool BestSplitDominatesRegression(const Tensor& total_sums, - const Tensor& total_squares, - const Tensor& split_sums, - const Tensor& split_squares, - int32 accumulator) { - // TODO(thomaswc): Implement this, probably as part of v3. - return false; -} - -int BootstrapGini(int n, int s, const random::DistributionSampler& ds, - random::SimplePhilox* rand) { - std::vector counts(s, 0); - for (int i = 0; i < n; i++) { - int j = ds.Sample(rand); - counts[j] += 1; - } - int g = 0; - for (int j = 0; j < s; j++) { - g += counts[j] * counts[j]; - } - // The true gini is 1 + (-g) / n^2 - return -g; -} - -// Populate *weights with the smoothed per-class frequencies needed to -// initialize a DistributionSampler. Returns the total number of samples -// seen by this accumulator. -int MakeBootstrapWeights(const Tensor& total_counts, const Tensor& split_counts, - int32 accumulator, int index, - std::vector* weights) { - const int32 num_classes = - static_cast(split_counts.shape().dim_size(2)) - 1; - - auto tc = total_counts.tensor(); - auto lc = split_counts.tensor(); - - int n = tc(accumulator, 0); - - float denom = static_cast(n) + static_cast(num_classes); - - weights->resize(num_classes * 2); - for (int i = 0; i < num_classes; i++) { - // Use the Laplace smoothed per-class probabilities when generating the - // bootstrap samples. - float left_count = lc(accumulator, index, i + 1); - (*weights)[i] = (left_count + 1.0) / denom; - float right_count = tc(accumulator, i + 1) - left_count; - (*weights)[num_classes + i] = (right_count + 1.0) / denom; - } - - return n; -} - -bool BestSplitDominatesClassificationBootstrap(const Tensor& total_counts, - const Tensor& split_counts, - int32 accumulator, - float dominate_fraction, - random::SimplePhilox* rand) { - float best_score; - float second_best_score; - int best_feature_index; - int second_best_index; - GetTwoBestClassification(total_counts, split_counts, accumulator, &best_score, - &best_feature_index, &second_best_score, - &second_best_index); - - std::vector weights1; - int n1 = MakeBootstrapWeights(total_counts, split_counts, accumulator, - best_feature_index, &weights1); - random::DistributionSampler ds1(weights1); - - std::vector weights2; - int n2 = MakeBootstrapWeights(total_counts, split_counts, accumulator, - second_best_index, &weights2); - random::DistributionSampler ds2(weights2); - - const int32 num_classes = - static_cast(split_counts.shape().dim_size(2)) - 1; - - float p = 1.0 - dominate_fraction; - if (p <= 0 || p > 1.0) { - LOG(FATAL) << "Invalid dominate fraction " << dominate_fraction; - } - - int bootstrap_samples = 1; - while (p < 1.0) { - bootstrap_samples += 1; - p = p * 2; - } - - int worst_g1 = 0; - for (int i = 0; i < bootstrap_samples; i++) { - int g1 = BootstrapGini(n1, 2 * num_classes, ds1, rand); - worst_g1 = std::max(worst_g1, g1); - } - - int best_g2 = 99; - for (int i = 0; i < bootstrap_samples; i++) { - int g2 = BootstrapGini(n2, 2 * num_classes, ds2, rand); - best_g2 = std::min(best_g2, g2); - } - - return worst_g1 < best_g2; -} - -bool BestSplitDominatesClassificationHoeffding(const Tensor& total_counts, - const Tensor& split_counts, - int32 accumulator, - float dominate_fraction) { - float best_score; - float second_best_score; - int best_feature_index; - int second_best_index; - VLOG(1) << "BSDC for accumulator " << accumulator; - GetTwoBestClassification(total_counts, split_counts, accumulator, &best_score, - &best_feature_index, &second_best_score, - &second_best_index); - VLOG(1) << "Best score = " << best_score; - VLOG(1) << "2nd best score = " << second_best_score; - - const int32 num_classes = - static_cast(split_counts.shape().dim_size(2)) - 1; - const float n = total_counts.Slice(accumulator, accumulator + 1) - .unaligned_flat()(0); - - // Each term in the Gini impurity can range from 0 to 0.5 * 0.5. - float range = 0.25 * static_cast(num_classes) * n; - - float hoeffding_bound = - range * sqrt(log(1.0 / (1.0 - dominate_fraction)) / (2.0 * n)); - - VLOG(1) << "num_classes = " << num_classes; - VLOG(1) << "n = " << n; - VLOG(1) << "range = " << range; - VLOG(1) << "hoeffding_bound = " << hoeffding_bound; - return (second_best_score - best_score) > hoeffding_bound; -} - -double DirichletCovarianceTrace(const Tensor& total_counts, - const Tensor& split_counts, int32 accumulator, - int index) { - const int32 num_classes = - static_cast(split_counts.shape().dim_size(2)) - 1; - - auto tc = total_counts.tensor(); - auto lc = split_counts.tensor(); - - double leftc = 0.0; - double leftc2 = 0.0; - double rightc = 0.0; - double rightc2 = 0.0; - for (int i = 1; i <= num_classes; i++) { - double l = lc(accumulator, index, i) + 1.0; - leftc += l; - leftc2 += l * l; - - double r = tc(accumulator, i) - lc(accumulator, index, i) + 1.0; - rightc += r; - rightc2 += r * r; - } - - double left_trace = (1.0 - leftc2 / (leftc * leftc)) / (leftc + 1.0); - double right_trace = (1.0 - rightc2 / (rightc * rightc)) / (rightc + 1.0); - return left_trace + right_trace; -} - -void getDirichletMean(const Tensor& total_counts, const Tensor& split_counts, - int32 accumulator, int index, std::vector* mu) { - const int32 num_classes = - static_cast(split_counts.shape().dim_size(2)) - 1; - - mu->resize(num_classes * 2); - auto tc = total_counts.tensor(); - auto lc = split_counts.tensor(); - - double total = tc(accumulator, 0); - - for (int i = 0; i < num_classes; i++) { - double l = lc(accumulator, index, i + 1); - mu->at(i) = (l + 1.0) / (total + num_classes); - - double r = tc(accumulator, i) - l; - mu->at(i + num_classes) = (r + 1.) / (total + num_classes); - } -} - -// Given lambda3, returns the distance from (mu1, mu2) to the surface. -double getDistanceFromLambda3(double lambda3, const std::vector& mu1, - const std::vector& mu2) { - if (fabs(lambda3) == 1.0) { - return 0.0; - } - - int n = mu1.size(); - double lambda1 = -2.0 * lambda3 / n; - double lambda2 = 2.0 * lambda3 / n; - // From below, - // x = (lambda_1 1 + 2 mu1) / (2 - 2 lambda_3) - // y = (lambda_2 1 + 2 mu2) / (2 + 2 lambda_3) - double dist = 0.0; - for (size_t i = 0; i < mu1.size(); i++) { - double diff = (lambda1 + 2.0 * mu1[i]) / (2.0 - 2.0 * lambda3) - mu1[i]; - dist += diff * diff; - diff = (lambda2 + 2.0 * mu2[i]) / (2.0 + 2.0 * lambda3) - mu2[i]; - dist += diff * diff; - } - return dist; -} - -// Returns the distance between (mu1, mu2) and (x, y), where (x, y) is the -// nearest point that lies on the surface defined by -// {x dot 1 = 1, y dot 1 = 1, x dot x - y dot y = 0}. -double getChebyshevEpsilon(const std::vector& mu1, - const std::vector& mu2) { - // Math time!! - // We are trying to minimize d = |mu1 - x|^2 + |mu2 - y|^2 over the surface. - // Using Lagrange multipliers, we get - // partial d / partial x = -2 mu1 + 2 x = lambda_1 1 + 2 lambda_3 x - // partial d / partial y = -2 mu2 + 2 y = lambda_2 1 - 2 lambda_3 y - // or - // x = (lambda_1 1 + 2 mu1) / (2 - 2 lambda_3) - // y = (lambda_2 1 + 2 mu2) / (2 + 2 lambda_3) - // which implies - // 2 - 2 lambda_3 = lambda_1 1 dot 1 + 2 mu1 dot 1 - // 2 + 2 lambda_3 = lambda_2 1 dot 1 + 2 mu2 dot 1 - // |lambda_1 1 + 2 mu1|^2 (2 + 2 lambda_3)^2 = - // |lambda_2 1 + 2 mu2|^2 (2 - 2 lambda_3)^2 - // So solving for the lambda's and using the fact that - // mu1 dot 1 = 1 and mu2 dot 1 = 1, - // lambda_1 = -2 lambda_3 / (1 dot 1) - // lambda_2 = 2 lambda_3 / (1 dot 1) - // and (letting n = 1 dot 1) - // | - lambda_3 1 + n mu1 |^2 (1 + lambda_3)^2 = - // | lambda_3 1 + n mu2 |^2 (1 - lambda_3)^2 - // or - // (lambda_3^2 n - 2 n lambda_3 + n^2 mu1 dot mu1)(1 + lambda_3)^2 = - // (lambda_3^2 n + 2 n lambda_3 + n^2 mu2 dot mu2)(1 - lambda_3)^2 - // or - // (lambda_3^2 - 2 lambda_3 + n mu1 dot mu1)(1 + 2 lambda_3 + lambda_3^2) = - // (lambda_3^2 + 2 lambda_3 + n mu2 dot mu2)(1 - 2 lambda_3 + lambda_3^2) - // or - // lambda_3^2 - 2 lambda_3 + n mu1 dot mu1 - // + 2 lambda_3^3 - 2 lambda_3^2 + 2n lambda_3 mu1 dot mu1 - // + lambda_3^4 - 2 lambda_3^3 + n lambda_3^2 mu1 dot mu1 - // = - // lambda_3^2 + 2 lambda_3 + n mu2 dot mu2 - // - 2 lambda_3^3 -4 lambda_3^2 - 2n lambda_3 mu2 dot mu2 - // + lambda_3^4 + 2 lambda_3^3 + n lambda_3^2 mu2 dot mu2 - // or - // - 2 lambda_3 + n mu1 dot mu1 - // - 2 lambda_3^2 + 2n lambda_3 mu1 dot mu1 - // + n lambda_3^2 mu1 dot mu1 - // = - // + 2 lambda_3 + n mu2 dot mu2 - // -4 lambda_3^2 - 2n lambda_3 mu2 dot mu2 - // + n lambda_3^2 mu2 dot mu2 - // or - // lambda_3^2 (2 + n mu1 dot mu1 + n mu2 dot mu2) - // + lambda_3 (2n mu1 dot mu1 + 2n mu2 dot mu2 - 4) - // + n mu1 dot mu1 - n mu2 dot mu2 = 0 - // which can be solved using the quadratic formula. - int n = mu1.size(); - double len1 = 0.0; - for (float m : mu1) { - len1 += m * m; - } - double len2 = 0.0; - for (float m : mu2) { - len2 += m * m; - } - double a = 2 + n * (len1 + len2); - double b = 2 * n * (len1 + len2) - 4; - double c = n * (len1 - len2); - double discrim = b * b - 4 * a * c; - if (discrim < 0.0) { - LOG(WARNING) << "Negative discriminant " << discrim; - return 0.0; - } - - double sdiscrim = sqrt(discrim); - // TODO(thomaswc): Analyze whatever one of these is always closer. - double v1 = (-b + sdiscrim) / (2 * a); - double v2 = (-b - sdiscrim) / (2 * a); - double dist1 = getDistanceFromLambda3(v1, mu1, mu2); - double dist2 = getDistanceFromLambda3(v2, mu1, mu2); - return std::min(dist1, dist2); -} - -bool BestSplitDominatesClassificationChebyshev(const Tensor& total_counts, - const Tensor& split_counts, - int32 accumulator, - float dominate_fraction) { - float best_score; - float second_best_score; - int best_feature_index; - int second_best_index; - VLOG(1) << "BSDC for accumulator " << accumulator; - GetTwoBestClassification(total_counts, split_counts, accumulator, &best_score, - &best_feature_index, &second_best_score, - &second_best_index); - VLOG(1) << "Best score = " << best_score; - VLOG(1) << "2nd best score = " << second_best_score; - - const int32 num_classes = - static_cast(split_counts.shape().dim_size(2)) - 1; - const float n = total_counts.Slice(accumulator, accumulator + 1) - .unaligned_flat()(0); - - VLOG(1) << "num_classes = " << num_classes; - VLOG(1) << "n = " << n; - double trace = DirichletCovarianceTrace(total_counts, split_counts, - accumulator, best_feature_index) + - DirichletCovarianceTrace(total_counts, split_counts, - accumulator, second_best_index); - - std::vector mu1; - getDirichletMean(total_counts, split_counts, accumulator, best_feature_index, - &mu1); - std::vector mu2; - getDirichletMean(total_counts, split_counts, accumulator, second_best_index, - &mu2); - double epsilon = getChebyshevEpsilon(mu1, mu2); - - if (epsilon == 0.0) { - return false; - } - - double dirichlet_bound = 1.0 - trace / (epsilon * epsilon); - return dirichlet_bound > dominate_fraction; -} - -GetFeatureFnType GetDenseFunctor(const Tensor& dense) { - if (dense.shape().dims() == 2) { - const auto dense_features = dense.matrix(); - // Here we capture by value, which shouldn't incur a copy of the data - // because of the underlying use of Eigen::TensorMap. - return [dense_features](int32 i, int32 feature) { - return dense_features(i, feature); - }; - } else { - return [](int32 i, int32 feature) { - LOG(ERROR) << "trying to access nonexistent dense features."; - return 0; - }; - } -} - -GetFeatureFnType GetSparseFunctor(const Tensor& sparse_indices, - const Tensor& sparse_values) { - if (sparse_indices.shape().dims() == 2) { - const auto indices = sparse_indices.matrix(); - const auto values = sparse_values.vec(); - // Here we capture by value, which shouldn't incur a copy of the data - // because of the underlying use of Eigen::TensorMap. - return [indices, values](int32 i, int32 feature) { - return tensorforest::FindSparseValue(indices, values, i, feature); - }; - } else { - return [](int32 i, int32 feature) { - LOG(ERROR) << "trying to access nonexistent sparse features."; - return 0; - }; - } -} - -bool DecideNode(const GetFeatureFnType& get_dense, - const GetFeatureFnType& get_sparse, int32 i, int32 feature, - float bias, const tensorforest::TensorForestDataSpec& spec) { - if (feature < spec.dense_features_size()) { - return Decide(get_dense(i, feature), bias, - FindDenseFeatureSpec(feature, spec)); - } else { - const int32 sparse_feature = feature - spec.dense_features_size(); - return Decide(get_sparse(i, sparse_feature), bias, - FindSparseFeatureSpec(sparse_feature, spec)); - } -} - -bool Decide(float value, float bias, DataColumnTypes type) { - switch (type) { - case kDataFloat: - return value >= bias; - - case kDataCategorical: - // We arbitrarily define categorical equality as going left. - return value != bias; - - default: - LOG(ERROR) << "Got unknown column type: " << type; - return false; - } -} - -void GetParentWeightedMean(float leaf_sum, const float* leaf_data, - float parent_sum, const float* parent_data, - float valid_leaf_threshold, int num_outputs, - std::vector* mean) { - float parent_weight = 0.0; - if (leaf_sum < valid_leaf_threshold && parent_sum >= 0) { - VLOG(1) << "not enough samples at leaf, including parent counts." - << "child sum = " << leaf_sum; - // Weight the parent's counts just enough so that the new sum is - // valid_leaf_threshold_, but never give any counts a weight of - // more than 1. - parent_weight = - std::min(1.0f, (valid_leaf_threshold - leaf_sum) / parent_sum); - leaf_sum += parent_weight * parent_sum; - VLOG(1) << "Sum w/ parent included = " << leaf_sum; - } - - for (int c = 0; c < num_outputs; c++) { - float w = leaf_data[c]; - if (parent_weight > 0.0) { - w += parent_weight * parent_data[c]; - } - (*mean)[c] = w / leaf_sum; - } -} - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/tree_utils.h b/tensorflow/contrib/tensor_forest/kernels/tree_utils.h deleted file mode 100644 index 774da472f15..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/tree_utils.h +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_TREE_UTILS_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_TREE_UTILS_H_ - -#include - -#include "tensorflow/contrib/tensor_forest/kernels/data_spec.h" -#include "tensorflow/core/framework/bounds_check.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/lib/random/distribution_sampler.h" -#include "tensorflow/core/lib/random/simple_philox.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace tensorforest { - -// We hide Eigen's hideous types behind a function that returns the (i, j)-th -// entry of a two dimensional tensor; this is that function's type. -using GetFeatureFnType = std::function; - -// TODO(gilberth): Put these in protos so they can be shared by C++ and python. -// Indexes in the tree representation's 2nd dimension for children and features. -const int32 CHILDREN_INDEX = 0; -const int32 FEATURE_INDEX = 1; - -// Used in the tree's children sub-tensor to indicate leaf and free nodes. -const int32 LEAF_NODE = -1; -const int32 FREE_NODE = -2; - -// Used to indicate column types, e.g. categorical vs. float -enum DataColumnTypes { kDataFloat = 0, kDataCategorical = 1 }; - -// Calculates the sum of a tensor. -template -T Sum(Tensor counts) { - Eigen::Tensor count_sum = - counts.unaligned_flat().sum(); - return count_sum(0); -} - -// Get the two best scores and their indices among max splits. -void GetTwoBest(int max, const std::function& score_fn, - float* best_score, int* best_index, float* second_best_score, - int* second_best_index); - -// If the Gini Impurity is g, this returns n^2 (g - 1). This is an int -// rather than a float, and so can be more easily checked for ties. -int BootstrapGini(int n, int s, const random::DistributionSampler& ds, - random::SimplePhilox* rand); - -// Get the DataColumnTypes number for the given feature. -DataColumnTypes FindDenseFeatureSpec( - int32 input_feature, const tensorforest::TensorForestDataSpec& spec); -DataColumnTypes FindSparseFeatureSpec( - int32 input_feature, const tensorforest::TensorForestDataSpec& spec); - -// Given an Eigen::Tensor type, calculate the Gini impurity. -template -float RawWeightedGiniImpurity(const T& counts) { - // Our split score is the Gini impurity times the number of examples - // seen by the leaf. If c(i) denotes the i-th class count and c = sum_i c(i) - // then - // score = c * (1 - sum_i ( c(i) / c )^2 ) - // = c - sum_i c(i)^2 / c - const auto sum = counts.sum(); - const auto sum2 = counts.square().sum(); - Eigen::Tensor ret = sum - (sum2 / sum); - return ret(0); -} - -// Given an Eigen::Tensor type, calculate the smoothed Gini impurity, which we -// use to determine the best split (lowest) and which nodes to allocate first -// (highest). -template -float WeightedGiniImpurity(const T& counts) { - const auto smoothed = counts + counts.constant(1.0f); - return RawWeightedGiniImpurity(smoothed); -} - -template -float WeightedVariance(const T1& sums, const T2& squares, float count) { - const auto e_x = sums / count; - const auto e_x2 = squares / count; - Eigen::Tensor ret = (e_x2 - e_x.square()).sum(); - return count * ret(0); -} - -// Returns the best split to use based on the (lowest) Gini impurity. -// Takes in the whole total and per-split count tensors because using -// Tensor::Slice returns a tensor of the same dimensionality, which makes -// things a little awkward. -int32 BestFeatureClassification(const Tensor& total_counts, - const Tensor& split_counts, int32 accumulator); - -// Returns the best split to use based on the (lowest) variance. -int32 BestFeatureRegression(const Tensor& total_sums, - const Tensor& total_squares, - const Tensor& split_sums, - const Tensor& split_squares, int32 accumulator); - -// Returns true if the best split's variance is sufficiently smaller than -// that of the next best split. -bool BestSplitDominatesRegression(const Tensor& total_sums, - const Tensor& total_squares, - const Tensor& split_sums, - const Tensor& split_squares, - int32 accumulator); - -// Performs bootstrap_samples bootstrap samples of the best split's class -// counts and the second best splits's class counts, and returns true if at -// least dominate_fraction of the time, the former has a better (lower) -// Gini impurity. Does not take over ownership of *rand. -bool BestSplitDominatesClassificationBootstrap( - const Tensor& total_counts, const Tensor& split_counts, int32 accumulator, - float dominate_fraction, tensorflow::random::SimplePhilox* rand); - -// Returns true if the best split's Gini impurity is sufficiently smaller than -// that of the next best split, as measured by the Hoeffding Tree bound. -bool BestSplitDominatesClassificationHoeffding(const Tensor& total_counts, - const Tensor& split_counts, - int32 accumulator, - float dominate_fraction); - -// Returns true if the best split's Gini impurity is sufficiently smaller than -// that of the next best split, as measured by a Chebyshev bound. -bool BestSplitDominatesClassificationChebyshev(const Tensor& total_counts, - const Tensor& split_counts, - int32 accumulator, - float dominate_fraction); - -// Initializes everything in the given tensor to the given value. -template -void Initialize(Tensor counts, T val = 0) { - auto flat = counts.unaligned_flat(); - std::fill(flat.data(), flat.data() + flat.size(), val); -} - -// Returns a function that accesses the (i,j)-th element of the specified two -// dimensional tensor. -GetFeatureFnType GetDenseFunctor(const Tensor& dense); - -// Returns a function that looks for the j-th feature of the i-th example -// in the sparse data, or the default value if not found. See FindSparseValue. -GetFeatureFnType GetSparseFunctor(const Tensor& sparse_indices, - const Tensor& sparse_values); - -// Returns true if the point falls to the right (i.e., the selected feature -// of the input point is greater than the bias threshold), and false if it -// falls to the left. -// Even though our input data is forced into float Tensors, it could have -// originally been something else (e.g. categorical string data) which -// we treat differently. -bool DecideNode(const GetFeatureFnType& get_dense, - const GetFeatureFnType& get_sparse, int32 i, int32 feature, - float bias, const tensorforest::TensorForestDataSpec& spec); - -// If T is a sparse float matrix represented by sparse_input_indices and -// sparse_input_values, FindSparseValue returns T(i,j), or 0.0 if (i,j) -// isn't present in sparse_input_indices. sparse_input_indices is assumed -// to be sorted. -template -float FindSparseValue(const T1& sparse_input_indices, - const T2& sparse_input_values, int32 i, int32 j) { - int32 low = 0; - int32 high = sparse_input_values.dimension(0); - while (low < high) { - int32 mid = (low + high) / 2; - int64 midi = internal::SubtleMustCopy(sparse_input_indices(mid, 0)); - int64 midj = internal::SubtleMustCopy(sparse_input_indices(mid, 1)); - if (midi == i) { - if (midj == j) { - return sparse_input_values(mid); - } - if (midj < j) { - low = mid + 1; - } else { - high = mid; - } - continue; - } - if (midi < i) { - low = mid + 1; - } else { - high = mid; - } - } - return 0.0; -} - -// Returns the number of sparse values for example input_index. -// Also returns the index where those features start in sparse_input_start -// if any were found. -// Assumes that the first column in indices is ordered. -template -int32 GetNumSparseFeatures(const T1& indices, int32 input_index, - int64* sparse_input_start) { - // Binary search for input_index. - // TODO(gilberth): Consider using std::lower_bound, std::upper_bound - // for a simpler but possibly slower solution, or searching for - // input_start and input_end simultaneously. - const int64 num_total = indices.dimension(0); - int64 index; - int64 low = 0; - int64 high = num_total; - *sparse_input_start = -1; // Easy error checking. - - while (true) { - if (low == high) { - return 0; - } - index = low + (high - low) / 2; - const int64 feature_index = indices(index, 0); - if (feature_index == input_index) { - // found it. - break; - } else if (feature_index < input_index) { - // Correct for the implicit floor in the index assignment. - if (low == index) { - return 0; - } - low = index; - } else { - high = index; - } - } - - // Scan for the start and end of the input_index range. - int64 input_start = index; - int64 val = indices(input_start, 0); - while (val == input_index) { - --input_start; - if (input_start < 0) { - break; - } - val = indices(input_start, 0); - } - *sparse_input_start = input_start + 1; - int32 input_end = index; - val = indices(input_end, 0); - while (val == input_index) { - ++input_end; - if (input_end >= num_total) { - break; - } - val = indices(input_end, 0); - } - return input_end - input_start - 1; -} - -// Returns left/right decision between the input value and the threshold bias. -// For floating point types, the decision is value > bias, but for -// categorical data, it is value != bias. -bool Decide(float value, float bias, DataColumnTypes type = kDataFloat); - -// Returns true if all the splits are initialized. Since they get initialized -// in order, we can simply infer this from the last split. -// This should only be called for a single allocator's candidate features -// (i.e. candidate_split_features.Slice(accumulator, accumulator + 1) ). -template -bool IsAllInitialized(const EigenType& features, int32 accumulator, - int32 num_splits) { - return features(accumulator, num_splits - 1) >= 0; -} - -// Tensorforest currently only allows tensors up to 2^31 elements. Return false -// if any dimension is greater than that, true otherwise. -inline bool CheckTensorBounds(OpKernelContext* context, const Tensor& tensor) { - for (int i = 0; i < (tensor).dims(); ++i) { - if (!TF_PREDICT_TRUE(tensor.shape().dim_size(i) < - std::numeric_limits::max())) { - context->CtxFailure((errors::InvalidArgument( - strings::StrCat("Tensor has a dimension that is greater than 2^31: ", - tensor.DebugString())))); - return false; - } - } - return true; -} - -void GetParentWeightedMean(float leaf_sum, const float* leaf_data, - float parent_sum, const float* parent_data, - float valid_leaf_threshold, int num_outputs, - std::vector* mean); - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_TREE_UTILS_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/tree_utils_test.cc b/tensorflow/contrib/tensor_forest/kernels/tree_utils_test.cc deleted file mode 100644 index 08553545502..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/tree_utils_test.cc +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" -#include "testing/base/public/gunit.h" -#include "tensorflow/core/framework/tensor_testutil.h" - -namespace tensorflow { -namespace tensorforest { - -TEST(TestSum, Basic) { - Tensor two_dim = test::AsTensor({1, 3, 5, 7, 2, 4, 6, 8}, {2, 4}); - EXPECT_EQ(Sum(two_dim), 36.0); - - Tensor multi_dim_single_value = test::AsTensor({42}, {1, 1, 1, 1}); - EXPECT_EQ(Sum(multi_dim_single_value), 42); - - Tensor nothing = test::AsTensor({}, {0}); - EXPECT_EQ(Sum(nothing), 0.0); -} - -TEST(TestWeightedGiniImpurity, Basic) { - Tensor vals = test::AsTensor({2, 2, 2}, {3}); - EXPECT_EQ(WeightedGiniImpurity(vals.unaligned_flat()), 6); - - Tensor zero = test::AsTensor({0}, {1}); - EXPECT_EQ(WeightedGiniImpurity(zero.unaligned_flat()), 0); -} - -TEST(TestWeightedVariance, Basic) { - // Lets say values were: (2, 2), (3, 2), (4, 2) - Tensor sums = test::AsTensor({9, 6}, {2}); - Tensor squares = test::AsTensor({29, 12}, {2}); - - EXPECT_FLOAT_EQ(WeightedVariance(sums.unaligned_flat(), - squares.unaligned_flat(), 3), - 2.0); - - Tensor zero = test::AsTensor({0}, {1}); - EXPECT_FLOAT_EQ(WeightedVariance(zero.unaligned_flat(), - zero.unaligned_flat(), 1), - 0); -} - -TEST(TestInitialize, Basic) { - Tensor t = test::AsTensor({0, 0, 0, 0, 0, 0, 0, 0}, {4, 2}); - - Initialize(t, 42.0); - - const auto vals = t.tensor(); - EXPECT_FLOAT_EQ(vals(0, 0), 42); - EXPECT_FLOAT_EQ(vals(1, 1), 42); - EXPECT_FLOAT_EQ(vals(3, 0), 42); - - Tensor nothing = test::AsTensor({}, {0}); - - // Just make sure this runs. - Initialize(nothing, -1.0); -} - -TEST(DecideNode, Basic) { - // Even though we only want a 1-D point, make sure to test a 2-D tensor - // because that's usually what comes off a Slice(). - Tensor point = test::AsTensor({10, 10, 10, 10}, {1, 4}); - - EXPECT_TRUE(DecideNode(point, 0, 9)); - EXPECT_TRUE(DecideNode(point, 1, 0)); - EXPECT_TRUE(DecideNode(point, 2, -3)); - EXPECT_FALSE(DecideNode(point, 3, 11)); -} - -TEST(IsAllInitialized, Basic) { - // Even though we only want a 1-D feature set, make sure to test a 2-D tensor - // because that's usually what comes off a Slice(). - Tensor features = test::AsTensor({10, 2, -1}, {1, 3}); - - EXPECT_FALSE(IsAllInitialized(features)); - - features = test::AsTensor({10, 2, 0}, {1, 3}); - - EXPECT_TRUE(IsAllInitialized(features)); -} - -TEST(BestFeatureClassification, Basic) { - const int32 num_accumulators = 4; - const int32 num_splits = 3; - const int32 num_classes = 4; - Tensor totals = test::AsTensor( - {1, 5, 6, 7, 0, 0, 0, 0, 30, 10, 10, 10, // this one - -1, -1, -1, -1}, - {num_accumulators, num_classes}); - Tensor splits = - test::AsTensor({1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 10, - 10, 10, 10, 0, 0, 10, 19, 5, 6, 8, // this one - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {num_accumulators, num_splits, num_classes}); - - EXPECT_EQ(BestFeatureClassification(totals, splits, 2), 1); -} - -TEST(BestFeatureClassification, NoWinner) { - const int32 num_accumulators = 4; - const int32 num_splits = 3; - const int32 num_classes = 4; - // When counts are all the same, the most reasonable thing to do is pick 0. - Tensor totals = - test::AsTensor({1, 5, 6, 7, 0, 0, 0, 0, 18, 6, 6, 6, // this one - -1, -1, -1, -1}, - {num_accumulators, num_classes}); - Tensor splits = - test::AsTensor({1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, - 3, 3, 9, 3, 3, 3, 9, 3, 3, 3, // this one - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {num_accumulators, num_splits, num_classes}); - - EXPECT_EQ(BestFeatureClassification(totals, splits, 2), 0); -} - -TEST(BestFeatureRegression, Basic) { - const int32 num_accumulators = 4; - const int32 num_splits = 3; - const int32 num_classes = 4; - Tensor total_sums = - test::AsTensor({1, 5, 6, 7, 0, 0, 0, 0, 10, 8, 6, 9, // this one - -1, -1, -1, -1}, - {num_accumulators, num_classes}); - Tensor total_squares = test::AsTensor( - {1, 5, 6, 7, 0, 0, 0, 0, 100, 50, 40, 45, // this one - -1, -1, -1, -1}, - {num_accumulators, num_classes}); - - Tensor split_sums = - test::AsTensor({1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8, - 6, 9, 9, 8, 5, 9, 0, 0, 0, 0, // this one - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {num_accumulators, num_splits, num_classes}); - - // lower the variance by lowering one of the squares just a little. - Tensor split_squares = - test::AsTensor( - {1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 100, 50, 40, 45, 100, 50, 40, 43, 0, 0, 0, 0, // this one - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {num_accumulators, num_splits, num_classes}); - - EXPECT_EQ(BestFeatureRegression(total_sums, total_squares, split_sums, - split_squares, 2), - 1); -} - -TEST(BestFeatureRegression, NoWinner) { - const int32 num_accumulators = 4; - const int32 num_splits = 3; - const int32 num_classes = 4; - // when counts are all the same, the most reasonable thing to do is pick 0. - Tensor total_sums = - test::AsTensor({1, 5, 6, 7, 0, 0, 0, 0, 10, 8, 6, 9, // this one - -1, -1, -1, -1}, - {num_accumulators, num_classes}); - Tensor total_squares = test::AsTensor( - {1, 5, 6, 7, 0, 0, 0, 0, 100, 50, 40, 45, // this one - -1, -1, -1, -1}, - {num_accumulators, num_classes}); - - Tensor split_sums = - test::AsTensor({1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8, - 6, 9, 10, 8, 6, 9, 10, 8, 6, 9, // this one - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {num_accumulators, num_splits, num_classes}); - - Tensor split_squares = test::AsTensor( - {1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 100, 50, 40, 45, 100, 50, 40, 45, 100, 50, 40, 45, // this one - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {num_accumulators, num_splits, num_classes}); - - EXPECT_EQ(BestFeatureRegression(total_sums, total_squares, split_sums, - split_squares, 2), - 0); -} - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/BUILD b/tensorflow/contrib/tensor_forest/kernels/v4/BUILD deleted file mode 100644 index 71bfa5bbb8c..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/BUILD +++ /dev/null @@ -1,343 +0,0 @@ -# TensorFlow code for training random forests. - -load("//tensorflow:tensorflow.bzl", "tf_cc_test") -load("//tensorflow/core/platform:default/build_config_root.bzl", "if_static") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -DECISION_TREE_RESOURCE_DEPS = [ - ":decision_node_evaluator", - ":input_data", - ":leaf_model_operators", - "//tensorflow/core:framework_headers_lib", -] + if_static( - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc", - ], - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc_headers_only", - ], -) - -cc_library( - name = "decision-tree-resource", - hdrs = ["decision-tree-resource.h"], - deps = DECISION_TREE_RESOURCE_DEPS + if_static([":decision-tree-resource_impl"]), -) - -cc_library( - name = "decision-tree-resource_impl", - srcs = ["decision-tree-resource.cc"], - hdrs = ["decision-tree-resource.h"], - deps = DECISION_TREE_RESOURCE_DEPS, -) - -cc_library( - name = "fertile-stats-resource", - srcs = ["fertile-stats-resource.cc"], - hdrs = ["fertile-stats-resource.h"], - deps = [ - ":decision_node_evaluator", - ":input_data", - ":input_target", - ":leaf_model_operators", - ":split_collection_operators", - "//tensorflow/core:framework_headers_lib", - ] + if_static( - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc", - ], - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc_headers_only", - ], - ), -) - -cc_library( - name = "input_data", - srcs = ["input_data.cc"], - hdrs = ["input_data.h"], - deps = [ - "//tensorflow/contrib/tensor_forest:tree_utils", - "//tensorflow/core:framework_headers_lib", - ] + if_static( - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_cc", - ], - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc_headers_only", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_cc_headers_only", - ], - ), -) - -cc_library( - name = "input_target", - hdrs = ["input_target.h"], - deps = [ - "//tensorflow/core:framework_headers_lib", - "//third_party/eigen3", - ], -) - -cc_library( - name = "leaf_model_operators", - srcs = ["leaf_model_operators.cc"], - hdrs = ["leaf_model_operators.h"], - deps = [ - ":input_target", - ":params", - ] + if_static( - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc", - ], - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc_headers_only", - ], - ), -) - -tf_cc_test( - name = "leaf_model_operators_test", - srcs = ["leaf_model_operators_test.cc"], - deps = [ - ":leaf_model_operators", - ":test_utils", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_cc", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc", - "//tensorflow/core", - "//tensorflow/core:lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -cc_library( - name = "grow_stats", - srcs = ["grow_stats.cc"], - hdrs = ["grow_stats.h"], - deps = [ - ":decision_node_evaluator", - ":input_data", - ":input_target", - ":params", - ":stat_utils", - "//tensorflow/contrib/tensor_forest:tree_utils", - "//tensorflow/core:framework_headers_lib", - ] + if_static( - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc", - ], - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc_headers_only", - ], - ), -) - -tf_cc_test( - name = "grow_stats_test", - srcs = ["grow_stats_test.cc"], - deps = [ - ":grow_stats", - ":test_utils", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_cc", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc", - "//tensorflow/core", - "//tensorflow/core:lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -cc_library( - name = "candidate_graph_runner", - srcs = ["candidate_graph_runner.cc"], - hdrs = ["candidate_graph_runner.h"], - deps = [ - ":input_data", - ":input_target", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - ] + if_static( - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc", - ], - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc_headers_only", - ], - ), -) - -cc_library( - name = "decision_node_evaluator", - srcs = ["decision_node_evaluator.cc"], - hdrs = ["decision_node_evaluator.h"], - deps = [ - ":input_data", - "//tensorflow/core:framework_headers_lib", - ] + if_static( - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_cc", - ], - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc_headers_only", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_cc_headers_only", - ], - ), -) - -tf_cc_test( - name = "decision_node_evaluator_test", - srcs = ["decision_node_evaluator_test.cc"], - deps = [ - ":decision_node_evaluator", - ":test_utils", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_cc", - "//tensorflow/core", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -cc_library( - name = "split_collection_operators", - srcs = ["split_collection_operators.cc"], - hdrs = ["split_collection_operators.h"], - deps = [ - ":grow_stats", - ":input_data", - ":input_target", - ":leaf_model_operators", - ":params", - ":stat_utils", - "//tensorflow/contrib/tensor_forest:tree_utils", - ] + if_static( - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_cc", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc", - ], - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc_headers_only", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc_headers_only", - ], - ), -) - -cc_library( - name = "graph_collection_operator", - srcs = ["graph_collection_operator.cc"], - hdrs = ["graph_collection_operator.h"], - deps = [ - ":candidate_graph_runner", - ":grow_stats", - ":input_data", - ":input_target", - ":leaf_model_operators", - ":params", - ":split_collection_operators", - "//tensorflow/contrib/tensor_forest:tree_utils", - ] + if_static( - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc", - ], - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc_headers_only", - ], - ), -) - -cc_library( - name = "stat_utils", - srcs = ["stat_utils.cc"], - hdrs = ["stat_utils.h"], - deps = [ - "//tensorflow/core:framework_headers_lib", - ] + if_static( - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc", - ], - [ - "//third_party/eigen3", - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:fertile_stats_proto_cc_headers_only", - ], - ), -) - -cc_library( - name = "test_utils", - hdrs = ["test_utils.h"], - deps = [ - ":input_data", - ":input_target", - ], -) - -cc_library( - name = "params", - srcs = ["params.cc"], - hdrs = ["params.h"], - deps = [ - "//third_party/eigen3", - "//tensorflow/core:framework_headers_lib", - ] + if_static( - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc", - ], - [ - "//tensorflow/contrib/decision_trees/proto:generic_tree_model_cc_headers_only", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc_headers_only", - ], - ), -) - -tf_cc_test( - name = "params_test", - srcs = ["params_test.cc"], - deps = [ - ":params", - "//tensorflow/contrib/tensor_forest/proto:tensor_forest_params_proto_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/candidate_graph_runner.cc b/tensorflow/contrib/tensor_forest/kernels/v4/candidate_graph_runner.cc deleted file mode 100644 index 417cb6f7420..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/candidate_graph_runner.cc +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/candidate_graph_runner.h" - -#include "tensorflow/core/framework/graph.pb.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/platform/env.h" - -namespace tensorflow { -namespace tensorforest { - -// Names of ops in the graph to run. -constexpr char kInitializeOp[] = "init"; -constexpr char kAddExampleOp[] = "add_example"; -constexpr char kSplitScoreName[] = "split_score"; -constexpr char kGetSplitName[] = "get_split"; -constexpr char kGetLeftStatsName[] = "get_left_stats"; -constexpr char kGetRightStatsName[] = "get_right_stats"; - -// Names of files written by python graph builder. -constexpr char kGraphFilename[] = "graph"; -constexpr char kSaverDefFilename[] = "saver"; -constexpr char kMetaDefFilename[] = "meta"; - -// Names of Tensor inputs. -constexpr char kFeaturesName[] = "features"; -constexpr char kInputDataName[] = "input_data"; -constexpr char kTargetsName[] = "targets"; -constexpr char kExamplesName[] = "examples"; - -constexpr char kNoOp[] = "none"; - -CandidateGraphRunner::CandidateGraphRunner( - const string& graph_dir, const decision_trees::BinaryNode& split) - : split_(split) { - // read graph from file. - GraphDef graph_def; - TF_CHECK_OK(ReadBinaryProto( - Env::Default(), io::JoinPath(graph_dir, kGraphFilename), &graph_def)) - << "Could not read graph def."; - - // create session. - session_.reset(::tensorflow::NewSession(SessionOptions())); - TF_CHECK_OK(session_->Create(graph_def)) << "Failed to create session"; - - // Features don't change, store them in a tensor. - const auto& oblique = split.inequality_left_child_test().oblique(); - const int32 feat_size = oblique.features_size(); - features_.reset(new Tensor(tensorflow::DT_INT32, TensorShape({feat_size}))); - auto feat = features_->flat(); - int i = 0; - for (const auto& id : oblique.features()) { - safe_strto32(id.id().value(), &feat(i++)); - } -} - -void CandidateGraphRunner::RunOp(const string& name, - const TensorNameValueList& inputs, - const std::vector& output_tensor_names, - std::vector* outputs) { - std::vector op_name; - if (name != kNoOp) { - op_name.push_back(name); - } - TF_CHECK_OK(session_->Run(inputs, output_tensor_names, op_name, outputs)) - << "Failed to run: " << name; -} - -void CandidateGraphRunner::Init() { - RunOp(kInitializeOp, TensorNameValueList(), std::vector(), nullptr); -} - -void CandidateGraphRunner::AddExample(const Tensor& input_data, - const Tensor& target, - const Tensor& examples) { - TensorNameValueList inputs; - inputs.emplace_back(kFeaturesName, *features_); - inputs.emplace_back(kExamplesName, examples); - inputs.emplace_back(kInputDataName, input_data); - inputs.emplace_back(kTargetsName, target); - - RunOp(kAddExampleOp, inputs, std::vector(), nullptr); -} - -float CandidateGraphRunner::SplitScore() { - std::vector outputs; - RunOp(kNoOp, TensorNameValueList(), {kSplitScoreName}, &outputs); - return outputs[0].unaligned_flat()(0); -} - -void CandidateGraphRunner::GetSplit(decision_trees::BinaryNode* node) { - std::vector outputs; - RunOp(kNoOp, TensorNameValueList(), {kGetSplitName}, &outputs); - ParseProtoUnlimited(node, outputs[0].unaligned_flat()(0)); - const auto& oblique = split_.inequality_left_child_test().oblique(); - auto* new_split = - node->mutable_inequality_left_child_test()->mutable_oblique(); - for (const auto& id : oblique.features()) { - *new_split->add_features() = id; - } -} - -void CandidateGraphRunner::GetLeftStats(LeafStat* stats) { - std::vector outputs; - RunOp(kNoOp, TensorNameValueList(), {kGetLeftStatsName}, &outputs); - const auto& counts = outputs[0].unaligned_flat(); - auto* dense = stats->mutable_classification()->mutable_dense_counts(); - for (int i = 0; i < counts.size(); ++i) { - dense->add_value()->set_float_value(counts(i)); - } -} - -void CandidateGraphRunner::GetRightStats(LeafStat* stats) { - std::vector outputs; - RunOp(kNoOp, TensorNameValueList(), {kGetRightStatsName}, &outputs); - const auto& counts = outputs[0].unaligned_flat(); - auto* dense = stats->mutable_classification()->mutable_dense_counts(); - for (int i = 0; i < counts.size(); ++i) { - dense->add_value()->set_float_value(counts(i)); - } -} - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/candidate_graph_runner.h b/tensorflow/contrib/tensor_forest/kernels/v4/candidate_graph_runner.h deleted file mode 100644 index 2e7368dc12c..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/candidate_graph_runner.h +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_CANDIDATE_GRAPH_RUNNER_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_CANDIDATE_GRAPH_RUNNER_H_ -#include -#include - -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_data.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_target.h" -#include "tensorflow/contrib/tensor_forest/proto/fertile_stats.pb.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/public/session.h" - -namespace tensorflow { -namespace tensorforest { - -typedef std::vector> - TensorNameValueList; - -// Class that represents one split candidate, and can perform operations -// on a session created from a graph. -class CandidateGraphRunner { - public: - // split should contain the features that are being used. - CandidateGraphRunner(const string& graph_dir, - const decision_trees::BinaryNode& split); - - // Input the given data and target Tensors to the add_example op. - void AddExample(const Tensor& input_data, const Tensor& target, - const Tensor& examples); - - // Get the candidates' split score with the split_score op. - float SplitScore(); - - // Fills in the split in node with weights and threshold. - void GetSplit(decision_trees::BinaryNode* node); - - // Fills in the stats for the left-branch taken. - void GetLeftStats(LeafStat* stats); - - // Fills in the stats for the right-branch taken. - void GetRightStats(LeafStat* stats); - - // Initializes variables, must be run before other ops. - void Init(); - - protected: - void RunOp(const string& name, const TensorNameValueList& inputs, - const std::vector& output_tensor_names, - std::vector* outputs); - - std::unique_ptr session_; - decision_trees::BinaryNode split_; - std::unique_ptr features_; -}; - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_CANDIDATE_GRAPH_RUNNER_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.cc b/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.cc deleted file mode 100644 index f5a090eaa77..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.cc +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h" - -namespace tensorflow { -namespace tensorforest { - -using decision_trees::DecisionTree; -using decision_trees::Leaf; -using decision_trees::TreeNode; - -DecisionTreeResource::DecisionTreeResource(const TensorForestParams& params) - : params_(params), decision_tree_(new decision_trees::Model()) { - model_op_ = LeafModelOperatorFactory::CreateLeafModelOperator(params_); -} - -int32 DecisionTreeResource::TraverseTree( - const std::unique_ptr& input_data, int example, - int32* leaf_depth, TreePath* path) const { - const DecisionTree& tree = decision_tree_->decision_tree(); - int32 current_id = 0; - int32 depth = 0; - while (true) { - const TreeNode& current = tree.nodes(current_id); - if (path != nullptr) { - *path->add_nodes_visited() = current; - } - if (current.has_leaf()) { - if (leaf_depth != nullptr) { - *leaf_depth = depth; - } - return current_id; - } - ++depth; - const int32 next_id = - node_evaluators_[current_id]->Decide(input_data, example); - current_id = tree.nodes(next_id).node_id().value(); - } -} - -void DecisionTreeResource::SplitNode(int32 node_id, SplitCandidate* best, - std::vector* new_children) { - DecisionTree* tree = decision_tree_->mutable_decision_tree(); - TreeNode* node = tree->mutable_nodes(node_id); - int32 newid = tree->nodes_size(); - - // left - new_children->push_back(newid); - TreeNode* new_left = tree->add_nodes(); - new_left->mutable_node_id()->set_value(newid++); - Leaf* left_leaf = new_left->mutable_leaf(); - model_op_->ExportModel(best->left_stats(), left_leaf); - - // right - new_children->push_back(newid); - TreeNode* new_right = tree->add_nodes(); - new_right->mutable_node_id()->set_value(newid); - Leaf* right_leaf = new_right->mutable_leaf(); - model_op_->ExportModel(best->right_stats(), right_leaf); - - node->clear_leaf(); - node->mutable_binary_node()->Swap(best->mutable_split()); - node->mutable_binary_node()->mutable_left_child_id()->set_value(newid - 1); - node->mutable_binary_node()->mutable_right_child_id()->set_value(newid); - while (node_evaluators_.size() <= node_id) { - node_evaluators_.emplace_back(nullptr); - } - node_evaluators_[node_id] = CreateDecisionNodeEvaluator(*node); -} - -void DecisionTreeResource::MaybeInitialize() { - DecisionTree* tree = decision_tree_->mutable_decision_tree(); - if (tree->nodes_size() == 0) { - model_op_->InitModel(tree->add_nodes()->mutable_leaf()); - } else if (node_evaluators_.empty()) { // reconstruct evaluators - for (const auto& node : tree->nodes()) { - if (node.has_leaf()) { - node_evaluators_.emplace_back(nullptr); - } else { - node_evaluators_.push_back(CreateDecisionNodeEvaluator(node)); - } - } - } -} - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h b/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h deleted file mode 100644 index 3100a5a0e5d..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_DECISION_TREE_RESOURCE_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_DECISION_TREE_RESOURCE_H_ - -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_data.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators.h" -#include "tensorflow/contrib/tensor_forest/proto/fertile_stats.pb.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/platform/mutex.h" - -namespace tensorflow { -namespace tensorforest { - -// Keep a tree ensemble in memory for efficient evaluation and mutation. -class DecisionTreeResource : public ResourceBase { - public: - // Constructor. - explicit DecisionTreeResource(const TensorForestParams& params); - - string DebugString() const override { - return strings::StrCat("DecisionTree[size=", - decision_tree_->decision_tree().nodes_size(), "]"); - } - - void MaybeInitialize(); - - const decision_trees::Model& decision_tree() const { return *decision_tree_; } - - decision_trees::Model* mutable_decision_tree() { - return decision_tree_.get(); - } - - const decision_trees::Leaf& get_leaf(int32 id) const { - return decision_tree_->decision_tree().nodes(id).leaf(); - } - - decision_trees::TreeNode* get_mutable_tree_node(int32 id) { - return decision_tree_->mutable_decision_tree()->mutable_nodes(id); - } - - // Resets the resource and frees the proto. - // Caller needs to hold the mutex lock while calling this. - void Reset() { decision_tree_.reset(new decision_trees::Model()); } - - mutex* get_mutex() { return &mu_; } - - // Return the TreeNode for the leaf that the example ends up at according - // to decision_tree_. Also fill in that leaf's depth if it isn't nullptr. - int32 TraverseTree(const std::unique_ptr& input_data, - int example, int32* depth, TreePath* path) const; - - // Split the given node_id, turning it from a Leaf to a BinaryNode and - // setting it's split to the given best. Add new children ids to - // new_children. - void SplitNode(int32 node_id, SplitCandidate* best, - std::vector* new_children); - - private: - mutex mu_; - const TensorForestParams params_; - std::unique_ptr decision_tree_; - std::shared_ptr model_op_; - std::vector> node_evaluators_; -}; - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_DECISION_TREE_RESOURCE_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.cc b/tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.cc deleted file mode 100644 index 7716536ba48..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.cc +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.h" -#include "tensorflow/core/lib/strings/numbers.h" - -namespace tensorflow { -namespace tensorforest { - -std::unique_ptr CreateDecisionNodeEvaluator( - const decision_trees::TreeNode& node) { - const decision_trees::BinaryNode& bnode = node.binary_node(); - return CreateBinaryDecisionNodeEvaluator(bnode, bnode.left_child_id().value(), - bnode.right_child_id().value()); -} - -std::unique_ptr CreateBinaryDecisionNodeEvaluator( - const decision_trees::BinaryNode& bnode, int32 left, int32 right) { - if (bnode.has_inequality_left_child_test()) { - const auto& test = bnode.inequality_left_child_test(); - if (test.has_oblique()) { - return std::unique_ptr( - new ObliqueInequalityDecisionNodeEvaluator(test, left, right)); - } else { - return std::unique_ptr( - new InequalityDecisionNodeEvaluator(test, left, right)); - } - } else { - decision_trees::MatchingValuesTest test; - if (bnode.custom_left_child_test().UnpackTo(&test)) { - return std::unique_ptr( - new MatchingValuesDecisionNodeEvaluator(test, left, right)); - } else { - LOG(ERROR) << "Unknown split test: " << bnode.DebugString(); - return nullptr; - } - } -} - -InequalityDecisionNodeEvaluator::InequalityDecisionNodeEvaluator( - const decision_trees::InequalityTest& test, int32 left, int32 right) - : BinaryDecisionNodeEvaluator(left, right) { - CHECK(safe_strto32(test.feature_id().id().value(), &feature_num_)) - << "Invalid feature ID: [" << test.feature_id().id().value() << "]"; - threshold_ = test.threshold().float_value(); - _test_type = test.type(); -} - -int32 InequalityDecisionNodeEvaluator::Decide( - const std::unique_ptr& dataset, int example) const { - const float val = dataset->GetExampleValue(example, feature_num_); - switch (_test_type) { - case decision_trees::InequalityTest::LESS_OR_EQUAL: - return val <= threshold_ ? left_child_id_ : right_child_id_; - case decision_trees::InequalityTest::LESS_THAN: - return val < threshold_ ? left_child_id_ : right_child_id_; - case decision_trees::InequalityTest::GREATER_OR_EQUAL: - return val >= threshold_ ? left_child_id_ : right_child_id_; - case decision_trees::InequalityTest::GREATER_THAN: - return val > threshold_ ? left_child_id_ : right_child_id_; - default: - LOG(ERROR) << "Unknown split test type: " << _test_type; - return -1; - } -} - -ObliqueInequalityDecisionNodeEvaluator::ObliqueInequalityDecisionNodeEvaluator( - const decision_trees::InequalityTest& test, int32 left, int32 right) - : BinaryDecisionNodeEvaluator(left, right) { - for (int i = 0; i < test.oblique().features_size(); ++i) { - int32 val; - CHECK(safe_strto32(test.oblique().features(i).id().value(), &val)) - << "Invalid feature ID: [" << test.oblique().features(i).id().value() - << "]"; - feature_num_.push_back(val); - feature_weights_.push_back(test.oblique().weights(i)); - } - threshold_ = test.threshold().float_value(); -} - -int32 ObliqueInequalityDecisionNodeEvaluator::Decide( - const std::unique_ptr& dataset, int example) const { - float val = 0; - for (int i = 0; i < feature_num_.size(); ++i) { - val += feature_weights_[i] * - dataset->GetExampleValue(example, feature_num_[i]); - } - - if (val <= threshold_) { - return left_child_id_; - } else { - return right_child_id_; - } -} - -MatchingValuesDecisionNodeEvaluator::MatchingValuesDecisionNodeEvaluator( - const decision_trees::MatchingValuesTest& test, int32 left, int32 right) - : BinaryDecisionNodeEvaluator(left, right) { - CHECK(safe_strto32(test.feature_id().id().value(), &feature_num_)) - << "Invalid feature ID: [" << test.feature_id().id().value() << "]"; - for (const auto& val : test.value()) { - values_.push_back(val.float_value()); - } - inverse_ = test.inverse(); -} - -int32 MatchingValuesDecisionNodeEvaluator::Decide( - const std::unique_ptr& dataset, int example) const { - const float val = dataset->GetExampleValue(example, feature_num_); - for (float testval : values_) { - if (val == testval) { - return inverse_ ? right_child_id_ : left_child_id_; - } - } - - return inverse_ ? left_child_id_ : right_child_id_; -} - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.h b/tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.h deleted file mode 100644 index 6497787f848..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.h +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_DECISION_NODE_EVALUATOR_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_DECISION_NODE_EVALUATOR_H_ - -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model_extensions.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_data.h" - -namespace tensorflow { -namespace tensorforest { - -// Base class for evaluators of decision nodes that effectively copy proto -// contents into C++ structures for faster execution. -class DecisionNodeEvaluator { - public: - virtual ~DecisionNodeEvaluator() {} - - // Returns the index of the child node. - virtual int32 Decide(const std::unique_ptr& dataset, - int example) const = 0; -}; - -// An evaluator for binary decisions with left and right children. -class BinaryDecisionNodeEvaluator : public DecisionNodeEvaluator { - protected: - BinaryDecisionNodeEvaluator(int32 left, int32 right) - : left_child_id_(left), right_child_id_(right) {} - - int32 left_child_id_; - int32 right_child_id_; -}; - -// Evaluator for basic inequality decisions (f[x] <= T). -class InequalityDecisionNodeEvaluator : public BinaryDecisionNodeEvaluator { - public: - InequalityDecisionNodeEvaluator(const decision_trees::InequalityTest& test, - int32 left, int32 right); - - int32 Decide(const std::unique_ptr& dataset, - int example) const override; - - protected: - int32 feature_num_; - float threshold_; - ::tensorflow::decision_trees::InequalityTest_Type _test_type; -}; - -// Evaluator for splits with multiple weighted features. -class ObliqueInequalityDecisionNodeEvaluator - : public BinaryDecisionNodeEvaluator { - public: - ObliqueInequalityDecisionNodeEvaluator( - const decision_trees::InequalityTest& test, int32 left, int32 right); - - int32 Decide(const std::unique_ptr& dataset, - int example) const override; - - protected: - std::vector feature_num_; - std::vector feature_weights_; - float threshold_; -}; - -// Evaluator for contains-in-set decisions. Also supports inverse (not-in-set). -class MatchingValuesDecisionNodeEvaluator : public BinaryDecisionNodeEvaluator { - public: - MatchingValuesDecisionNodeEvaluator( - const decision_trees::MatchingValuesTest& test, int32 left, int32 right); - - int32 Decide(const std::unique_ptr& dataset, - int example) const override; - - protected: - int32 feature_num_; - std::vector values_; - bool inverse_; -}; - -std::unique_ptr CreateDecisionNodeEvaluator( - const decision_trees::TreeNode& node); -std::unique_ptr CreateBinaryDecisionNodeEvaluator( - const decision_trees::BinaryNode& node, int32 left, int32 right); - -struct CandidateEvalatorCollection { - std::vector> splits; -}; - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_DECISION_NODE_EVALUATOR_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator_test.cc b/tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator_test.cc deleted file mode 100644 index 3db13355637..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator_test.cc +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.h" -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/test_utils.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace { - -using tensorflow::decision_trees::InequalityTest; -using tensorflow::decision_trees::MatchingValuesTest; -using tensorflow::tensorforest::InequalityDecisionNodeEvaluator; -using tensorflow::tensorforest::MatchingValuesDecisionNodeEvaluator; -using tensorflow::tensorforest::ObliqueInequalityDecisionNodeEvaluator; - -TEST(InequalityDecisionNodeEvaluatorTest, TestLessOrEqual) { - InequalityTest test; - test.mutable_feature_id()->mutable_id()->set_value("0"); - test.mutable_threshold()->set_float_value(3.0); - test.set_type(InequalityTest::LESS_OR_EQUAL); - std::unique_ptr eval( - new InequalityDecisionNodeEvaluator(test, 0, 1)); - - std::unique_ptr dataset( - new tensorflow::tensorforest::TestableDataSet( - {0.0, 1.0, 2.0, 3.0, 4.0, 5.0}, 1)); - - ASSERT_EQ(eval->Decide(dataset, 2), 0); - ASSERT_EQ(eval->Decide(dataset, 3), 0); - ASSERT_EQ(eval->Decide(dataset, 4), 1); -} - -TEST(InequalityDecisionNodeEvaluatorTest, TestStrictlyLess) { - InequalityTest test; - test.mutable_feature_id()->mutable_id()->set_value("0"); - test.mutable_threshold()->set_float_value(3.0); - test.set_type(InequalityTest::LESS_THAN); - std::unique_ptr eval( - new InequalityDecisionNodeEvaluator(test, 0, 1)); - - std::unique_ptr dataset( - new tensorflow::tensorforest::TestableDataSet( - {0.0, 1.0, 2.0, 3.0, 4.0, 5.0}, 1)); - - ASSERT_EQ(eval->Decide(dataset, 2), 0); - ASSERT_EQ(eval->Decide(dataset, 3), 1); - ASSERT_EQ(eval->Decide(dataset, 4), 1); -} - -TEST(InequalityDecisionNodeEvaluatorTest, TestGreaterOrEqual) { - InequalityTest test; - test.mutable_feature_id()->mutable_id()->set_value("0"); - test.mutable_threshold()->set_float_value(3.0); - test.set_type(InequalityTest::GREATER_OR_EQUAL); - std::unique_ptr eval( - new InequalityDecisionNodeEvaluator(test, 0, 1)); - - std::unique_ptr dataset( - new tensorflow::tensorforest::TestableDataSet( - {0.0, 1.0, 2.0, 3.0, 4.0, 5.0}, 1)); - - ASSERT_EQ(eval->Decide(dataset, 2), 1); - ASSERT_EQ(eval->Decide(dataset, 3), 0); - ASSERT_EQ(eval->Decide(dataset, 4), 0); -} - -TEST(InequalityDecisionNodeEvaluatorTest, TestStrictlyGreater) { - InequalityTest test; - test.mutable_feature_id()->mutable_id()->set_value("0"); - test.mutable_threshold()->set_float_value(3.0); - test.set_type(InequalityTest::GREATER_THAN); - std::unique_ptr eval( - new InequalityDecisionNodeEvaluator(test, 0, 1)); - - std::unique_ptr dataset( - new tensorflow::tensorforest::TestableDataSet( - {0.0, 1.0, 2.0, 3.0, 4.0, 5.0}, 1)); - - ASSERT_EQ(eval->Decide(dataset, 2), 1); - ASSERT_EQ(eval->Decide(dataset, 3), 1); - ASSERT_EQ(eval->Decide(dataset, 4), 0); -} - -TEST(MatchingDecisionNodeEvaluatorTest, Basic) { - MatchingValuesTest test; - test.mutable_feature_id()->mutable_id()->set_value("0"); - test.add_value()->set_float_value(3.0); - test.add_value()->set_float_value(5.0); - - std::unique_ptr eval( - new MatchingValuesDecisionNodeEvaluator(test, 0, 1)); - - std::unique_ptr dataset( - new tensorflow::tensorforest::TestableDataSet( - {0.0, 1.0, 2.0, 3.0, 4.0, 5.0}, 1)); - - ASSERT_EQ(eval->Decide(dataset, 2), 1); - ASSERT_EQ(eval->Decide(dataset, 3), 0); - ASSERT_EQ(eval->Decide(dataset, 4), 1); - ASSERT_EQ(eval->Decide(dataset, 5), 0); -} - -TEST(MatchingDecisionNodeEvaluatorTest, Inverse) { - MatchingValuesTest test; - test.mutable_feature_id()->mutable_id()->set_value("0"); - test.add_value()->set_float_value(3.0); - test.add_value()->set_float_value(5.0); - test.set_inverse(true); - - std::unique_ptr eval( - new MatchingValuesDecisionNodeEvaluator(test, 0, 1)); - - std::unique_ptr dataset( - new tensorflow::tensorforest::TestableDataSet( - {0.0, 1.0, 2.0, 3.0, 4.0, 5.0}, 1)); - - ASSERT_EQ(eval->Decide(dataset, 2), 0); - ASSERT_EQ(eval->Decide(dataset, 3), 1); - ASSERT_EQ(eval->Decide(dataset, 4), 0); - ASSERT_EQ(eval->Decide(dataset, 5), 1); -} - -TEST(ObliqueDecisionNodeEvaluatorTest, Basic) { - InequalityTest test; - auto* feat1 = test.mutable_oblique()->add_features(); - feat1->mutable_id()->set_value("0"); - test.mutable_oblique()->add_weights(1.0); - auto* feat2 = test.mutable_oblique()->add_features(); - feat2->mutable_id()->set_value("1"); - test.mutable_oblique()->add_weights(1.0); - - test.mutable_threshold()->set_float_value(3.0); - test.set_type(InequalityTest::LESS_OR_EQUAL); - - std::unique_ptr eval( - new ObliqueInequalityDecisionNodeEvaluator(test, 0, 1)); - - std::unique_ptr dataset( - new tensorflow::tensorforest::TestableDataSet( - {0.0, 1.0, 2.0, 3.0, 4.0, 5.0}, 2)); - - ASSERT_EQ(eval->Decide(dataset, 0), 0); - ASSERT_EQ(eval->Decide(dataset, 1), 1); -} - -} // namespace -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.cc b/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.cc deleted file mode 100644 index 7f914aac319..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.cc +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.h" - -#include - -namespace tensorflow { -namespace tensorforest { - -void FertileStatsResource::AddExampleToStatsAndInitialize( - const std::unique_ptr& input_data, const InputTarget* target, - const std::vector& examples, int32 node_id, bool* is_finished) { - // Update stats or initialize if needed. - if (collection_op_->IsInitialized(node_id)) { - collection_op_->AddExample(input_data, target, examples, node_id); - } else { - // This throws away any extra examples, which is more inefficient towards - // the top but gradually becomes less of an issue as the tree grows. - for (int example : examples) { - collection_op_->CreateAndInitializeCandidateWithExample( - input_data, target, example, node_id); - if (collection_op_->IsInitialized(node_id)) { - break; - } - } - } - - *is_finished = collection_op_->IsFinished(node_id); -} - -void FertileStatsResource::AllocateNode(int32 node_id, int32 depth) { - collection_op_->InitializeSlot(node_id, depth); -} - -void FertileStatsResource::Allocate(int32 parent_depth, - const std::vector& new_children) { - const int32 children_depth = parent_depth + 1; - for (const int32 child : new_children) { - AllocateNode(child, children_depth); - } -} - -void FertileStatsResource::Clear(int32 node) { - collection_op_->ClearSlot(node); -} - -bool FertileStatsResource::BestSplit(int32 node_id, SplitCandidate* best, - int32* depth) { - return collection_op_->BestSplit(node_id, best, depth); -} - -void FertileStatsResource::MaybeInitialize() { - collection_op_->MaybeInitialize(); -} - -void FertileStatsResource::ExtractFromProto(const FertileStats& stats) { - collection_op_ = - SplitCollectionOperatorFactory::CreateSplitCollectionOperator(params_); - collection_op_->ExtractFromProto(stats); -} - -void FertileStatsResource::PackToProto(FertileStats* stats) const { - collection_op_->PackToProto(stats); -} -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.h b/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.h deleted file mode 100644 index 44f2b3f473b..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.h +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_FERTILE_STATS_RESOURCE_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_FERTILE_STATS_RESOURCE_H_ - -#include - -#include "tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_data.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_target.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/split_collection_operators.h" -#include "tensorflow/contrib/tensor_forest/proto/fertile_stats.pb.h" -#include "tensorflow/contrib/tensor_forest/proto/tensor_forest_params.pb.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/platform/mutex.h" - -namespace tensorflow { -namespace tensorforest { - -// Stores a FertileStats proto and implements operations on it. -class FertileStatsResource : public ResourceBase { - public: - // Constructor. - explicit FertileStatsResource(const TensorForestParams& params) - : params_(params) { - model_op_ = LeafModelOperatorFactory::CreateLeafModelOperator(params_); - } - - string DebugString() const override { return "FertileStats"; } - - void ExtractFromProto(const FertileStats& stats); - - void PackToProto(FertileStats* stats) const; - - // Resets the resource and frees the proto. - // Caller needs to hold the mutex lock while calling this. - void Reset() {} - - // Reset the stats for a node, but leave the leaf_stats intact. - void ResetSplitStats(int32 node_id, int32 depth) { - collection_op_->ClearSlot(node_id); - collection_op_->InitializeSlot(node_id, depth); - } - - mutex* get_mutex() { return &mu_; } - - void MaybeInitialize(); - - // Applies the example to the given leaf's statistics. Also applies it to the - // node's fertile slot's statistics if or initializes a split candidate, - // where applicable. Returns if the node is finished or if it's ready to - // allocate to a fertile slot. - void AddExampleToStatsAndInitialize( - const std::unique_ptr& input_data, - const InputTarget* target, const std::vector& examples, - int32 node_id, bool* is_finished); - - // Allocate a fertile slot for each ready node, then new children up to - // max_fertile_nodes_. - void Allocate(int32 parent_depth, const std::vector& new_children); - - // Remove a node's fertile slot. Should only be called when the node is - // no longer a leaf. - void Clear(int32 node); - - // Return the best SplitCandidate for a node, or NULL if no suitable split - // was found. - bool BestSplit(int32 node_id, SplitCandidate* best, int32* depth); - - private: - mutex mu_; - std::shared_ptr model_op_; - std::unique_ptr collection_op_; - const TensorForestParams params_; - - void AllocateNode(int32 node_id, int32 depth); -}; - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_FERTILE_STATS_RESOURCE_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/graph_collection_operator.cc b/tensorflow/contrib/tensor_forest/kernels/v4/graph_collection_operator.cc deleted file mode 100644 index c7faea0aef1..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/graph_collection_operator.cc +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/graph_collection_operator.h" - -#include - -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" - -namespace tensorflow { -namespace tensorforest { - -REGISTER_SPLIT_COLLECTION(GRAPH_RUNNER_COLLECTION, - GraphRunnerSplitCollectionOperator); - -std::unique_ptr GraphRunnerSplitCollectionOperator::CreateGrowStats( - int32 node_id, int32 depth) const { - return std::unique_ptr(new SimpleStats(params_, depth)); -} - -int64 GraphRunnerSplitCollectionOperator::UniqueId(int32 node_id, - int32 split_id) const { - return node_id * num_splits_to_consider_ + split_id; -} - -bool GraphRunnerSplitCollectionOperator::BestSplit(int32 node_id, - SplitCandidate* best, - int32* depth) const { - float min_score = FLT_MAX; - int best_index = -1; - auto* slot = stats_.at(node_id).get(); - *depth = slot->depth(); - for (int i = 0; i < slot->num_splits(); ++i) { - // TODO(gilberth): Support uselessness. - auto& runner = runners_[UniqueId(node_id, i)]; - const float split_score = runner->SplitScore(); - if (split_score < min_score) { - min_score = split_score; - best_index = i; - } - } - - // This could happen if all the splits are useless. - if (best_index < 0) { - return false; - } - - // Fill in split info and left/right stats to initialize models with. - *best = SplitCandidate(); - auto& runner = runners_[UniqueId(node_id, best_index)]; - runner->GetLeftStats(best->mutable_left_stats()); - runner->GetRightStats(best->mutable_right_stats()); - runner->GetSplit(best->mutable_split()); - return true; -} - -void GraphRunnerSplitCollectionOperator::AddExample( - const std::unique_ptr& input_data, const InputTarget* target, - const std::vector& examples, int32 node_id) const { - // Build input Tensors. - int size = examples.size(); - Tensor examples_t(tensorflow::DT_INT32, TensorShape({size})); - auto ex_data = examples_t.flat(); - std::copy(examples.begin(), examples.end(), ex_data.data()); - - const TensorInputTarget* tensor_target = - dynamic_cast(target); - CHECK_NOTNULL(tensor_target); - - const Tensor& data_t = input_data->original_tensor(); - const Tensor& target_t = tensor_target->original_tensor(); - - // Add to candidates. - auto* slot = stats_.at(node_id).get(); - for (int i = 0; i < slot->num_splits(); ++i) { - auto& runner = runners_[UniqueId(node_id, i)]; - runner->AddExample(data_t, target_t, examples_t); - } - - // Update simple weight sums so we know when we're done. - for (int example : examples) { - slot->AddExample(input_data, target, example); - } -} - -void GraphRunnerSplitCollectionOperator:: - CreateAndInitializeCandidateWithExample( - const std::unique_ptr& input_data, - const InputTarget* target, int example, int32 node_id) const { - auto* slot = stats_.at(node_id).get(); - int cand_num = slot->num_splits(); - const int64 unique_id = UniqueId(node_id, cand_num); - - decision_trees::BinaryNode split; - - decision_trees::InequalityTest* test = - split.mutable_inequality_left_child_test(); - auto* oblique = test->mutable_oblique(); - for (int i = 0; i < features_per_node_; ++i) { - float bias; - int type; - // This is really just a way to select a list of random features. - // Also a way to warn the user that categoricals don't make sense here. - input_data->RandomSample(example, oblique->add_features(), &bias, &type); - - if (type == kDataFloat) { - test->set_type(decision_trees::InequalityTest::LESS_OR_EQUAL); - - // The comparison bias is assumed to be zero. - test->mutable_threshold()->set_float_value(0); - } else { - LOG(ERROR) << "Categorical features not supported with this system."; - return; - } - } - - slot->AddSplit(split, input_data, target, example); - - runners_[unique_id].reset(new CandidateGraphRunner(graph_dir_, split)); - runners_[unique_id]->Init(); -} - -void GraphRunnerSplitCollectionOperator::ClearSlot(int32 node_id) { - SplitCollectionOperator::ClearSlot(node_id); - for (int i = 0; i < num_splits_to_consider_; ++i) { - runners_.erase(UniqueId(node_id, i)); - } -} - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/graph_collection_operator.h b/tensorflow/contrib/tensor_forest/kernels/v4/graph_collection_operator.h deleted file mode 100644 index 4ae48179afc..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/graph_collection_operator.h +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_GRAPH_COLLECTION_OPERATOR_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_GRAPH_COLLECTION_OPERATOR_H_ - -#include -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/candidate_graph_runner.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_data.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_target.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/params.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/split_collection_operators.h" -#include "tensorflow/contrib/tensor_forest/proto/fertile_stats.pb.h" -#include "tensorflow/contrib/tensor_forest/proto/tensor_forest_params.pb.h" - -namespace tensorflow { -namespace tensorforest { - -// Holds split candidates that are trained by running any TF graph. -class GraphRunnerSplitCollectionOperator : public SplitCollectionOperator { - public: - explicit GraphRunnerSplitCollectionOperator(const TensorForestParams& params) - : SplitCollectionOperator(params) { - if (params.num_splits_to_consider().ParamType_case() == - DepthDependentParam::PARAMTYPE_NOT_SET) { - LOG(FATAL) << "GRAPH_RUNNER_COLLECTION must specify a constant value for " - << " num_splits_to_consider"; - } else { - num_splits_to_consider_ = - params.num_splits_to_consider().constant_value(); - } - } - - std::unique_ptr CreateGrowStats(int32 node_id, - int32 depth) const override; - - // Updates the slot's candidates with the new example. - // Assumes slot has been initialized. - void AddExample(const std::unique_ptr& input_data, - const InputTarget* target, const std::vector& examples, - int32 node_id) const override; - - // Create a new candidate and initialize it with the given example. - void CreateAndInitializeCandidateWithExample( - const std::unique_ptr& input_data, - const InputTarget* target, int example, int32 node_id) const override; - - bool BestSplit(int32 node_id, SplitCandidate* best, - int32* depth) const override; - - void ClearSlot(int32 node_id) override; - - protected: - int64 UniqueId(int32 node_id, int32 split_id) const; - - mutable std::unordered_map> - runners_; - int features_per_node_; - string graph_dir_; - // Must have a constant value because of how we make unique ids right now. - int32 num_splits_to_consider_; -}; - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_GRAPH_COLLECTION_OPERATOR_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.cc b/tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.cc deleted file mode 100644 index e8b494fc166..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.cc +++ /dev/null @@ -1,957 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.h" - -#include -#include -#include -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/stat_utils.h" -#include "tensorflow/core/lib/random/distribution_sampler.h" -#include "tensorflow/core/lib/random/random.h" - -namespace tensorflow { -namespace tensorforest { - -// When creating evaluators for the split candidates, use these -// for the left and right return values. -static const int32 LEFT_INDEX = 0; -static const int32 RIGHT_INDEX = 1; - -GrowStats::GrowStats(const TensorForestParams& params, int32 depth) - : weight_sum_(0), - depth_(depth), - params_(params), - split_after_samples_(ResolveParam(params.split_after_samples(), depth)), - num_splits_to_consider_( - ResolveParam(params.num_splits_to_consider(), depth)), - num_outputs_(params.num_outputs()) {} - -void GrowStats::AddSplit(const decision_trees::BinaryNode& split, - const std::unique_ptr& input_data, - const InputTarget* target, int example) { - // It's possible that the split collection calls AddSplit, but we actually - // have all the splits we need and are just waiting for them to be fully - // initialized. - if (splits_.size() < num_splits_to_consider_) { - splits_.push_back(split); - evaluators_.emplace_back( - CreateBinaryDecisionNodeEvaluator(split, LEFT_INDEX, RIGHT_INDEX)); - AddSplitStats(target, example); - } - - if (input_data != nullptr && target != nullptr && - params_.initialize_average_splits()) { - AdditionalInitializationExample(input_data, target, example); - } -} - -void GrowStats::RemoveSplit(int split_num) { - splits_.erase(splits_.begin() + split_num); - evaluators_.erase(evaluators_.begin() + split_num); - RemoveSplitStats(split_num); -} - -// ------------------------ Classification --------------------------- // - -ClassificationStats::ClassificationStats(const TensorForestParams& params, - int32 depth) - : GrowStats(params, depth), finish_early_(false) { - // Early splitting params. - if (params.finish_type().type() == SPLIT_FINISH_BASIC) { - min_split_samples_ = split_after_samples_; - finish_sample_epoch_ = 1; - finish_check_every_ = split_after_samples_ * 2; - } else { - if (!params.has_dominate_fraction() || !params.has_min_split_samples()) { - LOG(FATAL) << "dominate_fraction and min_split_samples " - << "required for early-finish strategy."; - } else { - min_split_samples_ = ResolveParam(params.min_split_samples(), depth); - finish_check_every_ = - ResolveParam(params.finish_type().check_every_steps(), depth); - finish_sample_epoch_ = min_split_samples_ / finish_check_every_; - - dominate_fraction_ = ResolveParam(params.dominate_fraction(), depth_); - if (dominate_fraction_ <= 0 || dominate_fraction_ > 1.0) { - LOG(FATAL) << "Invalid dominate fraction " << dominate_fraction_; - } - } - } - - // Pruning params. - if (params.pruning_type().type() != SPLIT_PRUNE_NONE) { - prune_check_every_ = - ResolveParam(params.pruning_type().prune_every_samples(), depth); - prune_sample_epoch_ = 1; - prune_fraction_ = 0.0; - switch (params_.pruning_type().type()) { - case SPLIT_PRUNE_HALF: - prune_fraction_ = 0.5; - break; - case SPLIT_PRUNE_QUARTER: - prune_fraction_ = 0.25; - break; - case SPLIT_PRUNE_10_PERCENT: - prune_fraction_ = 0.10; - break; - case SPLIT_PRUNE_HOEFFDING: - dominate_fraction_ = ResolveParam(params.dominate_fraction(), depth_); - half_ln_dominate_frac_ = 0.5 * log(1.0 / (1.0 - dominate_fraction_)); - break; - default: - LOG(WARNING) << "Unknown pruning type"; - } - } else { - prune_check_every_ = split_after_samples_ * 2; - prune_sample_epoch_ = 1; - } - - if (params.use_running_stats_method()) { - left_gini_.reset(new RunningGiniScores()); - right_gini_.reset(new RunningGiniScores()); - } - - single_rand_ = std::unique_ptr( - new random::PhiloxRandom(random::New64())); - rng_ = std::unique_ptr( - new random::SimplePhilox(single_rand_.get())); -} - -void ClassificationStats::AdditionalInitializationExample( - const std::unique_ptr& input_data, const InputTarget* target, - int example) { - const int32 new_target = target->GetTargetAsClassIndex(example, 0); - std::unordered_set to_erase; - for (auto it = half_initialized_splits_.begin(); - it != half_initialized_splits_.end(); ++it) { - if (it->second != new_target) { - auto& split = splits_[it->first]; - if (split.has_inequality_left_child_test()) { - auto& test = split.inequality_left_child_test(); - auto* thresh = - split.mutable_inequality_left_child_test()->mutable_threshold(); - if (test.has_feature_id()) { - const float val = - input_data->GetExampleValue(example, test.feature_id()); - thresh->set_float_value((thresh->float_value() + val) / 2); - } - } - to_erase.insert(it->first); - } - } - - for (const int split_id : to_erase) { - half_initialized_splits_.erase(split_id); - } -} - -bool ClassificationStats::IsFinished() const { - bool basic = (weight_sum_ >= split_after_samples_) && !is_pure(); - return basic || finish_early_; -} - -float ClassificationStats::MaybeCachedGiniScore(int split, float* left_sum, - float* right_sum) const { - if (left_gini_ == nullptr) { - return GiniScore(split, left_sum, right_sum); - } else { - *left_sum = left_gini_->sum(split); - const float left = WeightedSmoothedGini( - *left_sum, left_gini_->square(split), num_outputs_); - - *right_sum = right_gini_->sum(split); - const float right = WeightedSmoothedGini( - *right_sum, right_gini_->square(split), num_outputs_); - - return left + right; - } -} - -void ClassificationStats::AddExample( - const std::unique_ptr& input_data, const InputTarget* target, - int example) { - const int64 int_label = target->GetTargetAsClassIndex(example, 0); - const float weight = target->GetTargetWeight(example); - - for (int i = 0; i < num_splits(); ++i) { - auto& eval = evaluators_[i]; - if (eval->Decide(input_data, example) == LEFT_INDEX) { - if (left_gini_ != nullptr) { - left_gini_->update(i, left_count(i, int_label), weight); - } - ClassificationAddLeftExample(i, int_label, weight); - } else { - if (right_gini_ != nullptr) { - right_gini_->update(i, right_count(i, int_label), weight); - } - ClassificationAddRightExample(i, int_label, weight); - } - } - - ClassificationAddTotalExample(int_label, weight); - - weight_sum_ += weight; - - CheckFinishEarly(); - CheckPrune(); -} - -void ClassificationStats::CheckPrune() { - if (params_.pruning_type().type() == SPLIT_PRUNE_NONE || IsFinished() || - weight_sum_ < prune_sample_epoch_ * prune_check_every_) { - return; - } - ++prune_sample_epoch_; - - if (params_.pruning_type().type() == SPLIT_PRUNE_HOEFFDING) { - CheckPruneHoeffding(); - return; - } - - const int to_remove = num_splits() * prune_fraction_; - if (to_remove <= 0) { - return; - } - - // pair ordering is first-then-second by default, no need for custom - // comparison. Use std::greater to make it a min-heap. - std::priority_queue, std::vector>, - std::greater>> - worst; - - // Track indices that are in the heap so we can iterate over them - // by largest-first later. - std::set indices; - - for (int i = 0; i < num_splits(); ++i) { - float left, right; - const float split_score = MaybeCachedGiniScore(i, &left, &right); - if (worst.size() < to_remove) { - worst.push(std::pair(split_score, i)); - indices.insert(i); - } else if (worst.top().first < split_score) { - indices.erase(worst.top().second); - worst.pop(); - worst.push(std::pair(split_score, i)); - indices.insert(i); - } - } - - // traverse indices from the back so that they are removed correctly. - for (auto it = indices.rbegin(); it != indices.rend(); ++it) { - RemoveSplit(*it); - } -} - -void ClassificationStats::CheckPruneHoeffding() { - std::vector split_scores(num_splits()); - // Find best split score - float best_split_score = FLT_MAX; - for (int i = 0; i < num_splits(); ++i) { - float left, right; - split_scores[i] = MaybeCachedGiniScore(i, &left, &right); - if (split_scores[i] < best_split_score) { - best_split_score = split_scores[i]; - } - } - - // We apply the Hoeffding bound to the difference between the best split - // score and the i-th split score. - // Raw Gini ranges from 0 to 1 - (1/n), but our gini score is weighted. - const float num_classes = params_.num_outputs(); - const float gini_diff_range = weight_sum_ * (1.0 - 1.0 / num_classes); - float epsilon = - gini_diff_range * std::sqrt(half_ln_dominate_frac_ / weight_sum_); - for (int i = num_splits() - 1; i >= 0; i--) { - if (split_scores[i] - best_split_score > epsilon) { - RemoveSplit(i); - } - } -} - -void ClassificationStats::CheckFinishEarly() { - if (weight_sum_ < min_split_samples_ || - weight_sum_ < finish_sample_epoch_ * finish_check_every_) { - return; - } - ++finish_sample_epoch_; - - if (params_.finish_type().type() == SPLIT_FINISH_DOMINATE_HOEFFDING) { - CheckFinishEarlyHoeffding(); - } else if (params_.finish_type().type() == SPLIT_FINISH_DOMINATE_BOOTSTRAP) { - CheckFinishEarlyBootstrap(); - } -} - -void ClassificationStats::CheckFinishEarlyHoeffding() { - // Each term in the Gini impurity can range from 0 to 0.5 * 0.5. - float range = 0.25 * static_cast(params_.num_outputs()) * weight_sum_; - - float hoeffding_bound = - range * sqrt(log(1.0 / (1.0 - dominate_fraction_)) / (2.0 * weight_sum_)); - - float unused_left_sum, unused_right_sum; - std::function score_fn = - std::bind(&ClassificationStats::MaybeCachedGiniScore, this, - std::placeholders::_1, &unused_left_sum, &unused_right_sum); - - float best_score; - int32 best_index; - float second_best_score; - int32 second_best_index; - GetTwoBest(num_splits(), score_fn, &best_score, &best_index, - &second_best_score, &second_best_index); - - finish_early_ = (second_best_score - best_score) > hoeffding_bound; -} - -void ClassificationStats::MakeBootstrapWeights(int index, - std::vector* weights) { - int n = weight_sum_; - float denom = static_cast(n) + static_cast(num_outputs_); - for (int i = 0; i < num_outputs_; ++i) { - // Use the Laplace smoothed per-class probabilities when generating the - // bootstrap samples. - (*weights)[i] = (left_count(index, i) + 1.0) / denom; - (*weights)[num_outputs_ + i] = (right_count(index, i) + 1.0) / denom; - } -} - -int ClassificationStats::NumBootstrapSamples() const { - float p = 1.0 - dominate_fraction_; - int bootstrap_samples = 1; - while (p < 1.0) { - ++bootstrap_samples; - p = p * 2; - } - return bootstrap_samples; -} - -void ClassificationStats::CheckFinishEarlyBootstrap() { - float unused_left_sum, unused_right_sum; - std::function score_fn = - std::bind(&ClassificationStats::MaybeCachedGiniScore, this, - std::placeholders::_1, &unused_left_sum, &unused_right_sum); - - float best_score; - int32 best_index; - float second_best_score; - int32 second_best_index; - GetTwoBest(num_splits(), score_fn, &best_score, &best_index, - &second_best_score, &second_best_index); - - std::vector weights1(num_outputs_ * 2); - MakeBootstrapWeights(best_index, &weights1); - random::DistributionSampler ds1(weights1); - - std::vector weights2(num_outputs_ * 2); - MakeBootstrapWeights(second_best_index, &weights2); - random::DistributionSampler ds2(weights2); - - const int bootstrap_samples = NumBootstrapSamples(); - - int worst_g1 = 0; - for (int i = 0; i < bootstrap_samples; i++) { - int g1 = BootstrapGini(weight_sum_, 2 * num_outputs_, ds1, rng_.get()); - worst_g1 = std::max(worst_g1, g1); - } - - int best_g2 = 99; - for (int i = 0; i < bootstrap_samples; i++) { - int g2 = BootstrapGini(weight_sum_, 2 * num_outputs_, ds2, rng_.get()); - best_g2 = std::min(best_g2, g2); - } - - finish_early_ = worst_g1 < best_g2; -} - -bool ClassificationStats::BestSplit(SplitCandidate* best) const { - float min_score = FLT_MAX; - int best_index = -1; - float best_left_sum, best_right_sum; - - // Calculate sums. - for (int i = 0; i < num_splits(); ++i) { - float left_sum, right_sum; - const float split_score = MaybeCachedGiniScore(i, &left_sum, &right_sum); - // Find the lowest gini. - if (left_sum > 0 && right_sum > 0 && - split_score < min_score) { // useless check - min_score = split_score; - best_index = i; - best_left_sum = left_sum; - best_right_sum = right_sum; - } - } - - // This could happen if all the splits are useless. - if (best_index < 0) { - return false; - } - - // Fill in stats to be used for leaf model. - *best->mutable_split() = splits_[best_index]; - auto* left = best->mutable_left_stats(); - left->set_weight_sum(best_left_sum); - auto* right = best->mutable_right_stats(); - right->set_weight_sum(best_right_sum); - InitLeafClassStats(best_index, left, right); - - return true; -} - -// ------------------------ Dense Classification --------------------------- // -void DenseClassificationGrowStats::ExtractFromProto(const FertileSlot& slot) { - Initialize(); - if (!slot.has_post_init_leaf_stats()) { - return; - } - const int32 num_classes = params_.num_outputs(); - weight_sum_ = slot.post_init_leaf_stats().weight_sum(); - const auto& class_stats = - slot.post_init_leaf_stats().classification().dense_counts(); - - // Total counts. - for (int i = 0; i < num_classes; ++i) { - total_counts_[i] = class_stats.value(i).float_value(); - num_outputs_seen_ += total_counts_[i] != 0; - } - - // Candidate counts and splits. - int split_num = 0; - for (const auto& cand : slot.candidates()) { - AddSplit(cand.split(), nullptr, nullptr, -1); - const auto& left_stats = cand.left_stats().classification().dense_counts(); - for (int i = 0; i < num_classes; ++i) { - const float val = left_stats.value(i).float_value(); - mutable_left_count(split_num, i) = val; - MaybeInitializeRunningCount(split_num, val); - } - ++split_num; - } -} - -void DenseClassificationGrowStats::PackToProto(FertileSlot* slot) const { - auto* slot_stats = slot->mutable_post_init_leaf_stats(); - slot_stats->set_weight_sum(weight_sum_); - - auto* class_stats = slot->mutable_post_init_leaf_stats() - ->mutable_classification() - ->mutable_dense_counts(); - for (int i = 0; i < num_outputs_; ++i) { - class_stats->add_value()->set_float_value(total_counts_[i]); - } - - for (int split_num = 0; split_num < num_splits(); ++split_num) { - auto* cand = slot->add_candidates(); - *cand->mutable_split() = splits_[split_num]; - auto* left_stats = cand->mutable_left_stats() - ->mutable_classification() - ->mutable_dense_counts(); - for (int i = 0; i < num_outputs_; ++i) { - left_stats->add_value()->set_float_value(left_count(split_num, i)); - } - } -} - -float DenseClassificationGrowStats::GiniScore(int split, float* left_sum, - float* right_sum) const { - float left_square = 0, right_square = 0; - *left_sum = 0; - *right_sum = 0; - for (int j = 0; j < num_outputs_; ++j) { - const float left = left_count(split, j); - *left_sum += left; - left_square += left * left; - const float right = right_count(split, j); - *right_sum += right; - right_square += right * right; - } - - const float left_score = - WeightedSmoothedGini(*left_sum, left_square, num_outputs_); - const float right_score = - WeightedSmoothedGini(*right_sum, right_square, num_outputs_); - return left_score + right_score; -} - -void DenseClassificationGrowStats::InitLeafClassStats( - int best_split_index, LeafStat* left_stats, LeafStat* right_stats) const { - auto* left_class_stats = left_stats->mutable_classification(); - auto* left_counts = left_class_stats->mutable_dense_counts(); - for (int i = 0; i < params_.num_outputs(); ++i) { - left_counts->add_value()->set_float_value(left_count(best_split_index, i)); - } - - auto* right_class_stats = right_stats->mutable_classification(); - auto* right_counts = right_class_stats->mutable_dense_counts(); - for (int i = 0; i < params_.num_outputs(); ++i) { - right_counts->add_value()->set_float_value(total_counts_[i] - - left_count(best_split_index, i)); - } -} - -// ------------------------ Sparse Classification --------------------------- // -void SparseClassificationGrowStats::ExtractFromProto(const FertileSlot& slot) { - Initialize(); - if (!slot.has_post_init_leaf_stats()) { - return; - } - weight_sum_ = slot.post_init_leaf_stats().weight_sum(); - const auto& class_stats = - slot.post_init_leaf_stats().classification().sparse_counts(); - - // Total counts. - for (auto const& entry : class_stats.sparse_value()) { - total_counts_[entry.first] = entry.second.float_value(); - } - - // Candidate counts and splits. - int split_num = 0; - for (const auto& cand : slot.candidates()) { - AddSplit(cand.split(), nullptr, nullptr, -1); - const auto& left_stats = cand.left_stats().classification().sparse_counts(); - for (auto const& entry : left_stats.sparse_value()) { - const float val = entry.second.float_value(); - left_counts_[split_num][entry.first] = val; - MaybeInitializeRunningCount(split_num, val); - } - ++split_num; - } -} - -void SparseClassificationGrowStats::PackToProto(FertileSlot* slot) const { - auto* slot_stats = slot->mutable_post_init_leaf_stats(); - slot_stats->set_weight_sum(weight_sum_); - - auto* class_stats = slot->mutable_post_init_leaf_stats() - ->mutable_classification() - ->mutable_sparse_counts() - ->mutable_sparse_value(); - for (const auto& entry : total_counts_) { - decision_trees::Value val; - val.set_float_value(entry.second); - (*class_stats)[entry.first] = val; - } - - for (int split_num = 0; split_num < num_splits(); ++split_num) { - auto* cand = slot->add_candidates(); - *cand->mutable_split() = splits_[split_num]; - auto* left_stats = cand->mutable_left_stats() - ->mutable_classification() - ->mutable_sparse_counts() - ->mutable_sparse_value(); - for (const auto& entry : left_counts_[split_num]) { - decision_trees::Value val; - val.set_float_value(entry.second); - (*left_stats)[entry.first] = val; - } - } -} - -float SparseClassificationGrowStats::GiniScore(int split, float* left_sum, - float* right_sum) const { - float left_square = 0, right_square = 0; - *left_sum = 0; - *right_sum = 0; - for (const auto& entry : total_counts_) { - const int label = entry.first; - float left = 0; - float right = 0; - auto it = left_counts_[split].find(label); - if (it == left_counts_[split].end()) { - right = entry.second; - } else { - left = it->second; - right = entry.second - it->second; - } - *left_sum += left; - left_square += left * left; - *right_sum += right; - right_square += right * right; - } - const int32 num_classes = params_.num_outputs(); - const float left_score = - WeightedSmoothedGini(*left_sum, left_square, num_classes); - const float right_score = - WeightedSmoothedGini(*right_sum, right_square, num_classes); - return left_score + right_score; -} - -void SparseClassificationGrowStats::InitLeafClassStats( - int best_split_index, LeafStat* left_stats, LeafStat* right_stats) const { - auto* left_class_stats = left_stats->mutable_classification(); - auto* left_counts = - left_class_stats->mutable_sparse_counts()->mutable_sparse_value(); - auto* right_class_stats = right_stats->mutable_classification(); - auto* right_counts = - right_class_stats->mutable_sparse_counts()->mutable_sparse_value(); - - for (const auto& entry : total_counts_) { - auto it = left_counts_[best_split_index].find(entry.first); - if (it == left_counts_[best_split_index].end()) { - (*right_counts)[entry.first].set_float_value(entry.second); - } else { - const float left = it->second; - const float right = entry.second - it->second; - (*left_counts)[entry.first].set_float_value(left); - if (right > 0) { - (*right_counts)[entry.first].set_float_value(right); - } - } - } -} - -// -------------------- FixedSizeClassStats --------------------------------- // - -// FixedSizeClassStats implements the "SpaceSaving" algorithm by -// Ahmed Metwally, Divyakant Agrawal and Amr El Abbadi. See for example -// https://pdfs.semanticscholar.org/72f1/5aba2e67b1cc9cd1fb12c99e101c4c1aae4b.pdf - -int argmin(const std::unordered_map& m) { - int c = -1; - float f = FLT_MAX; - for (const auto it : m) { - if (it.second < f) { - f = it.second; - c = it.first; - } - } - return c; -} - -void FixedSizeClassStats::accumulate(int c, float w) { - auto it = class_weights_.find(c); - if (it != class_weights_.end()) { - it->second += w; - if (c == smallest_weight_class_) { - smallest_weight_class_ = argmin(class_weights_); - } - return; - } - - if (class_weights_.size() < n_) { - class_weights_.insert(it, std::pair(c, w)); - if (class_weights_.size() == n_) { - // Can't assume last added has the smallest weight, because the - // w's might be all different. - smallest_weight_class_ = argmin(class_weights_); - } - return; - } - - // This is the slightly unintuitive heart of the SpaceSaving algorithm: - // if the map is full and we see a new class, we find the entry with the - // smallest weight and "take it over": we add our weight to its weight, - // and assign it all to the new seen class. - it = class_weights_.find(smallest_weight_class_); - float new_weight = it->second + w; - class_weights_.erase(it); - class_weights_[c] = new_weight; - smallest_weight_class_ = argmin(class_weights_); -} - -float FixedSizeClassStats::get_weight(int c) const { - // Every entry in class_weights_ might be overstated by as much as the - // smallest_weight. We therefore assume that each has been overstated - // by smallest_weight / 2.0, and we re-distribute that mass over all - // num_classes_ classes. - float smallest_weight = 0.0; - auto it = class_weights_.find(smallest_weight_class_); - if (it != class_weights_.end()) { - smallest_weight = it->second; - } - float w = (smallest_weight / 2.0) * n_ / static_cast(num_classes_); - it = class_weights_.find(c); - if (it != class_weights_.end()) { - w += it->second - smallest_weight / 2.0; - } - return w; -} - -void FixedSizeClassStats::set_sum_and_square(float* sum, float* square) const { - *sum = 0.0; - *square = 0.0; - - float smallest_weight = 0.0; - auto it = class_weights_.find(smallest_weight_class_); - if (it != class_weights_.end()) { - smallest_weight = it->second; - } - - float w; - for (const auto it : class_weights_) { - *sum += it.second; - w = get_weight(it.first); - *square += w * w; - } - - w = (smallest_weight / 2.0) * n_ / static_cast(num_classes_); - *square += (num_classes_ - n_) * w * w; -} - -void FixedSizeClassStats::ExtractFromProto( - const decision_trees::SparseVector& sparse_vector) { - for (const auto& it : sparse_vector.sparse_value()) { - class_weights_[it.first] = it.second.float_value(); - } - if (class_weights_.size() == n_) { - smallest_weight_class_ = argmin(class_weights_); - } -} - -void FixedSizeClassStats::PackToProto( - decision_trees::SparseVector* sparse_vector) const { - for (const auto it : class_weights_) { - (*sparse_vector->mutable_sparse_value())[it.first].set_float_value( - it.second); - } -} - -// --------------------- FixedSizeSparseClassificationGrowStats ------------- // - -void FixedSizeSparseClassificationGrowStats::ExtractFromProto( - const FertileSlot& slot) { - Initialize(); - if (!slot.has_post_init_leaf_stats()) { - return; - } - weight_sum_ = slot.post_init_leaf_stats().weight_sum(); - - // Candidate counts and splits. - int split_num = 0; - left_counts_.clear(); - right_counts_.clear(); - for (const auto& cand : slot.candidates()) { - AddSplit(cand.split(), nullptr, nullptr, -1); - const auto& left_stats = cand.left_stats().classification().sparse_counts(); - left_counts_.emplace_back(params_.num_classes_to_track(), - params_.num_outputs()); - left_counts_[split_num].ExtractFromProto(left_stats); - const auto& right_stats = - cand.right_stats().classification().sparse_counts(); - right_counts_.emplace_back(params_.num_classes_to_track(), - params_.num_outputs()); - right_counts_[split_num].ExtractFromProto(right_stats); - ++split_num; - } -} - -void FixedSizeSparseClassificationGrowStats::PackToProto( - FertileSlot* slot) const { - auto* slot_stats = slot->mutable_post_init_leaf_stats(); - slot_stats->set_weight_sum(weight_sum_); - - for (int split_num = 0; split_num < num_splits(); ++split_num) { - auto* cand = slot->add_candidates(); - *cand->mutable_split() = splits_[split_num]; - auto* left_stats = cand->mutable_left_stats() - ->mutable_classification() - ->mutable_sparse_counts(); - left_counts_[split_num].PackToProto(left_stats); - auto* right_stats = cand->mutable_right_stats() - ->mutable_classification() - ->mutable_sparse_counts(); - right_counts_[split_num].PackToProto(right_stats); - } -} - -float FixedSizeSparseClassificationGrowStats::GiniScore( - int split, float* left_sum, float* right_sum) const { - float left_square, right_square; - left_counts_[split].set_sum_and_square(left_sum, &left_square); - right_counts_[split].set_sum_and_square(right_sum, &right_square); - const int32 num_classes = params_.num_outputs(); - const float left_score = - WeightedSmoothedGini(*left_sum, left_square, num_classes); - const float right_score = - WeightedSmoothedGini(*right_sum, right_square, num_classes); - return left_score + right_score; -} - -void FixedSizeSparseClassificationGrowStats::InitLeafClassStats( - int best_split_index, LeafStat* left_stats, LeafStat* right_stats) const { - auto* left_class_stats = left_stats->mutable_classification(); - auto* left_counts = left_class_stats->mutable_sparse_counts(); - left_counts_[best_split_index].PackToProto(left_counts); - - auto* right_class_stats = right_stats->mutable_classification(); - auto* right_counts = right_class_stats->mutable_sparse_counts(); - right_counts_[best_split_index].PackToProto(right_counts); -} - -// --------------------- Least Squares Regression --------------------------- // -void LeastSquaresRegressionGrowStats::ExtractFromProto( - const FertileSlot& slot) { - const int32 num_outputs = params_.num_outputs(); - Initialize(); - if (!slot.has_post_init_leaf_stats()) { - return; - } - weight_sum_ = slot.post_init_leaf_stats().weight_sum(); - const auto& total_sums = - slot.post_init_leaf_stats().regression().mean_output(); - const auto& total_squares = - slot.post_init_leaf_stats().regression().mean_output_squares(); - - // Total counts. - for (int i = 0; i < num_outputs; ++i) { - total_sum_[i] = total_sums.value(i).float_value(); - total_sum_squares_[i] = total_squares.value(i).float_value(); - } - - // Candidate counts and splits. - int split_num = 0; - for (const auto& cand : slot.candidates()) { - AddSplit(cand.split(), nullptr, nullptr, -1); - const auto& sums = cand.left_stats().regression().mean_output(); - const auto& squares = cand.left_stats().regression().mean_output_squares(); - for (int i = 0; i < num_outputs; ++i) { - left_sum(split_num, i) = sums.value(i).float_value(); - left_square(split_num, i) = squares.value(i).float_value(); - } - left_counts_[split_num] = cand.left_stats().weight_sum(); - ++split_num; - } -} - -void LeastSquaresRegressionGrowStats::PackToProto(FertileSlot* slot) const { - const int32 num_outputs = params_.num_outputs(); - auto* slot_stats = slot->mutable_post_init_leaf_stats(); - slot_stats->set_weight_sum(weight_sum_); - - auto* total_sums = slot->mutable_post_init_leaf_stats() - ->mutable_regression() - ->mutable_mean_output(); - auto* total_squares = slot->mutable_post_init_leaf_stats() - ->mutable_regression() - ->mutable_mean_output_squares(); - - for (int i = 0; i < total_sum_.size(); ++i) { - total_sums->add_value()->set_float_value(total_sum_[i]); - total_squares->add_value()->set_float_value(total_sum_squares_[i]); - } - - for (int split_num = 0; split_num < num_splits(); ++split_num) { - auto* cand = slot->add_candidates(); - *cand->mutable_split() = splits_[split_num]; - auto* sums = - cand->mutable_left_stats()->mutable_regression()->mutable_mean_output(); - auto* squares = cand->mutable_left_stats() - ->mutable_regression() - ->mutable_mean_output_squares(); - for (int i = 0; i < num_outputs; ++i) { - sums->add_value()->set_float_value(left_sum(split_num, i)); - squares->add_value()->set_float_value(left_square(split_num, i)); - } - cand->mutable_left_stats()->set_weight_sum(left_counts_[split_num]); - } -} - -void LeastSquaresRegressionGrowStats::AddExample( - const std::unique_ptr& input_data, const InputTarget* target, - int example) { - const int32 num_outputs = params_.num_outputs(); - // Update splits. - for (int i = 0; i < num_splits(); ++i) { - auto& eval = evaluators_[i]; - if (eval->Decide(input_data, example) == LEFT_INDEX) { - for (int j = 0; j < num_outputs; ++j) { - const float output = target->GetTargetAsContinuous(example, j); - left_sum(i, j) += output; - left_square(i, j) += output * output; - } - ++left_counts_[i]; - } - } - - // Update totals. - for (int i = 0; i < num_outputs; ++i) { - const float output = target->GetTargetAsContinuous(example, i); - total_sum_[i] += output; - total_sum_squares_[i] += output * output; - } - weight_sum_ += 1.0; -} - -float LeastSquaresRegressionGrowStats::SplitVariance(int split) const { - float total_variance = 0; - for (int i = 0; i < params_.num_outputs(); ++i) { - // Left side - const float le_x = left_sum(split, i) / left_counts_[split]; - - const float le_x2 = left_square(split, i) / left_counts_[split]; - total_variance += le_x2 - le_x * le_x; - - // Right side - const float re_x = (total_sum_[i] - left_sum(split, i)) / - (weight_sum_ - left_counts_[split]); - - const float re_x2 = (total_sum_squares_[i] - left_square(split, i)) / - (weight_sum_ - left_counts_[split]); - total_variance += re_x2 - re_x * re_x; - } - return total_variance; -} - -bool LeastSquaresRegressionGrowStats::BestSplit(SplitCandidate* best) const { - float min_score = FLT_MAX; - int best_index = -1; - const int32 num_outputs = params_.num_outputs(); - for (int i = 0; i < num_splits(); ++i) { - if (left_counts_[i] > 0 && weight_sum_ - left_counts_[i] > 0) { - const float split_score = SplitVariance(i); - if (split_score < min_score) { - min_score = split_score; - best_index = i; - } - } - } - - // This could happen if all the splits are useless. - if (best_index < 0) { - return false; - } - - // Fill in right stats to be used for leaf model. - *best->mutable_split() = splits_[best_index]; - // Left - auto* left = best->mutable_left_stats(); - auto* left_reg_stats = left->mutable_regression(); - left->set_weight_sum(left_counts_[best_index]); - auto* left_output_sum = left_reg_stats->mutable_mean_output(); - for (int i = 0; i < num_outputs; ++i) { - left_output_sum->add_value()->set_float_value(left_sum(best_index, i)); - } - - // Right - auto* right = best->mutable_right_stats(); - auto* right_reg_stats = right->mutable_regression(); - right->set_weight_sum(weight_sum_ - left_counts_[best_index]); - auto* right_output_sum = right_reg_stats->mutable_mean_output(); - for (int i = 0; i < num_outputs; ++i) { - right_output_sum->add_value()->set_float_value(total_sum_[i] - - left_sum(best_index, i)); - } - return true; -} - -bool LeastSquaresRegressionGrowStats::IsFinished() const { - return weight_sum_ >= split_after_samples_; -} - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.h b/tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.h deleted file mode 100644 index dc3e9fe79d3..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.h +++ /dev/null @@ -1,603 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_GROW_STATS_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_GROW_STATS_H_ -#include -#include - -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_data.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_target.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/params.h" -#include "tensorflow/contrib/tensor_forest/proto/fertile_stats.pb.h" -#include "tensorflow/contrib/tensor_forest/proto/tensor_forest_params.pb.h" -#include "tensorflow/core/lib/random/philox_random.h" -#include "tensorflow/core/lib/random/simple_philox.h" - -namespace tensorflow { -namespace tensorforest { - -// Base class for tracking stats necessary to split a leaf. -// Holds and tracks stats for every candidate split. -class GrowStats { - public: - virtual ~GrowStats() {} - // Perform any initialization. - virtual void Initialize() = 0; - - // Add an example to any stats being collected. - virtual void AddExample(const std::unique_ptr& input_data, - const InputTarget* target, int example) = 0; - - // Fill in the best split, return false if none were valid. - virtual bool BestSplit(SplitCandidate* best) const = 0; - - // Return true if this leaf is finished splitting. - virtual bool IsFinished() const = 0; - - // Get the split_num BinaryNode. - const decision_trees::BinaryNode& Split(int split_num) const { - return splits_[split_num]; - } - - // Clear all state. - virtual void Clear() { - weight_sum_ = 0; - splits_.clear(); - evaluators_.clear(); - ClearInternal(); - } - - virtual void ExtractFromProto(const FertileSlot& slot) = 0; - virtual void PackToProto(FertileSlot* slot) const = 0; - - // Add split to the list of candidate splits. - void AddSplit(const decision_trees::BinaryNode& split, - const std::unique_ptr& input_data, - const InputTarget* target, int example); - virtual void AdditionalInitializationExample( - const std::unique_ptr& input_data, - const InputTarget* target, int example) {} - void RemoveSplit(int split_num); - - int num_splits() const { return splits_.size(); } - - float weight_sum() const { return weight_sum_; } - - virtual bool IsInitialized() const { - return weight_sum_ > 0 || splits_.size() == num_splits_to_consider_; - } - - int32 depth() const { return depth_; } - - protected: - GrowStats(const TensorForestParams& params, int32 depth); - - // Function called by AddSplit for subclasses to initialize stats for a split. - virtual void AddSplitStats(const InputTarget* target, int example) = 0; - - virtual void RemoveSplitStats(int split_num) = 0; - - // Function called by Clear for subclasses to clear their state. - virtual void ClearInternal() = 0; - - std::vector splits_; - std::vector> evaluators_; - - float weight_sum_; - - const int32 depth_; - - const TensorForestParams& params_; - - // We cache these because they're used often. - const int split_after_samples_; - const int num_splits_to_consider_; - - const int32 num_outputs_; -}; - -// Don't track anything, useful for systems that want to track split -// candidates but train the model in some other way. -class SimpleStats : public GrowStats { - public: - SimpleStats(const TensorForestParams& params, int32 depth) - : GrowStats(params, depth) {} - void Initialize() override {} - - void ExtractFromProto(const FertileSlot& slot) override {} - void PackToProto(FertileSlot* slot) const override {} - - void AddExample(const std::unique_ptr& input_data, - const InputTarget* target, int example) override { - weight_sum_ += target->GetTargetWeight(example); - } - - bool BestSplit(SplitCandidate* best) const override { return false; } - - bool IsFinished() const override { - return weight_sum_ >= split_after_samples_; - } - - protected: - void AddSplitStats(const InputTarget* target, int example) override {} - void RemoveSplitStats(int split_num) override {} - void ClearInternal() override {} -}; - -// Tracks the sum and square of one side of a split for each Gini calculation. -class RunningGiniScores { - public: - float sum(int split) const { return sum_[split]; } - float square(int split) const { return square_[split]; } - - void update(int split, float old_val, float weight) { - sum_[split] += weight; - const float new_val = old_val + weight; - square_[split] = square_[split] - old_val * old_val + new_val * new_val; - } - - void add_split() { - sum_.push_back(0); - square_.push_back(0); - } - - void remove_split(int i) { - sum_.erase(sum_.begin() + i); - square_.erase(square_.begin() + i); - } - - private: - std::vector sum_; - std::vector square_; -}; - -class ClassificationStats : public GrowStats { - public: - ClassificationStats(const TensorForestParams& params, int32 depth); - - bool IsFinished() const override; - - void AddExample(const std::unique_ptr& input_data, - const InputTarget* target, int example) override; - - void AdditionalInitializationExample( - const std::unique_ptr& input_data, - const InputTarget* target, int example) override; - - bool IsInitialized() const override { - return weight_sum_ > 0 || (splits_.size() == num_splits_to_consider_ && - half_initialized_splits_.empty()); - } - - bool BestSplit(SplitCandidate* best) const override; - // When best_split_index has been chosen as the best split, - // InitLeafClassStats is used to initialize the LeafStat's of the two - // new leaves. - virtual void InitLeafClassStats(int best_split_index, LeafStat* left_stats, - LeafStat* right_stats) const = 0; - - protected: - virtual float GiniScore(int split, float* left_sum, - float* right_sum) const = 0; - - // is_pure should return true if at most one class label has been seen - // at the node, and false if two or more have been seen. - virtual bool is_pure() const = 0; - virtual float left_count(int split, int class_num) const = 0; - virtual float right_count(int split, int class_num) const = 0; - - virtual void ClassificationAddLeftExample(int split, int64 int_label, - float weight) = 0; - virtual void ClassificationAddRightExample(int split, int64 int_label, - float weight) { - // Does nothing by default, but sub-classes can override. - } - virtual void ClassificationAddTotalExample(int64 int_label, float weight) = 0; - - virtual void ClassificationAddSplitStats() = 0; - virtual void ClassificationRemoveSplitStats(int split) = 0; - - void AddSplitStats(const InputTarget* target, int example) override { - if (left_gini_ != nullptr) { - left_gini_->add_split(); - right_gini_->add_split(); - } - if (params_.initialize_average_splits()) { - if (splits_[splits_.size() - 1].has_inequality_left_child_test()) { - half_initialized_splits_[splits_.size() - 1] = - target->GetTargetAsClassIndex(example, 0); - } - } - ClassificationAddSplitStats(); - } - void RemoveSplitStats(int split) override { - if (left_gini_ != nullptr) { - left_gini_->remove_split(split); - right_gini_->remove_split(split); - } - ClassificationRemoveSplitStats(split); - } - - // Virtual so we can override these to test. - virtual void CheckFinishEarly(); - virtual void CheckFinishEarlyHoeffding(); - virtual void CheckFinishEarlyBootstrap(); - - virtual void CheckPrune(); - - // Implement SplitPruningStrategyType::SPLIT_PRUNE_HOEFFDING. - void CheckPruneHoeffding(); - - // Return the gini score, possibly being calculated from sums and squares - // saved in left_gini_ and right_gini_, otherwise calculated from raw counts. - float MaybeCachedGiniScore(int split, float* left_sum, - float* right_sum) const; - - // Initialize the sum and squares of left_gini_ and right_gini_ for given - // split and value (being extracted from a proto), if left_gini_ isn't null. - void MaybeInitializeRunningCount(int split, float val) { - if (left_gini_ != nullptr) { - left_gini_->update(split, 0, val); - right_gini_->update(split, 0, val); - } - } - - int NumBootstrapSamples() const; - - // Populate *weights with the smoothed per-class frequencies needed to - // initialize a DistributionSampler. - void MakeBootstrapWeights(int index, std::vector* weights); - - // Accessors for RunningGiniScores objects, for testing. - virtual const std::unique_ptr& get_left_gini() const { - return left_gini_; - } - virtual const std::unique_ptr& get_right_gini() const { - return right_gini_; - } - - private: - // Tracks how many check_every_samples epochs we've seen go by in weight_sum. - int32 finish_sample_epoch_; - int32 finish_check_every_; - int32 prune_sample_epoch_; - int32 prune_check_every_; - bool finish_early_; - int32 min_split_samples_; - float dominate_fraction_; - float prune_fraction_; - - // When using SPLIT_PRUNE_HOEFFDING, we precompute and store - // 0.5 * ln(1 / (1.0 - dominate_fraction_)). - float half_ln_dominate_frac_; - - std::unique_ptr single_rand_; - std::unique_ptr rng_; - - std::unique_ptr left_gini_; - std::unique_ptr right_gini_; - - // Stores split number -> class that was first seen. - std::unordered_map half_initialized_splits_; -}; - -// Tracks classification stats by storing class counts densely. -class DenseClassificationGrowStats : public ClassificationStats { - public: - DenseClassificationGrowStats(const TensorForestParams& params, int32 depth) - : ClassificationStats(params, depth) {} - - void Initialize() override { - Clear(); - total_counts_.resize(num_outputs_); - } - - void ExtractFromProto(const FertileSlot& slot) override; - void PackToProto(FertileSlot* slot) const override; - - void InitLeafClassStats(int best_split_index, LeafStat* left_stats, - LeafStat* right_stats) const override; - - protected: - void ClassificationAddSplitStats() override { - left_counts_.resize(num_outputs_ * num_splits()); - } - void ClassificationRemoveSplitStats(int split_num) override { - left_counts_.erase(left_counts_.begin() + num_outputs_ * split_num, - left_counts_.begin() + num_outputs_ * (split_num + 1)); - } - void ClearInternal() override { - total_counts_.clear(); - left_counts_.clear(); - num_outputs_seen_ = 0; - } - - bool is_pure() const override { return num_outputs_seen_ <= 1; } - - void ClassificationAddLeftExample(int split, int64 int_label, - float weight) override { - mutable_left_count(split, int_label) += weight; - } - void ClassificationAddTotalExample(int64 int_label, float weight) override { - num_outputs_seen_ += total_counts_[int_label] == 0 && weight > 0; - total_counts_[int_label] += weight; - } - - float GiniScore(int split, float* left_sum, float* right_sum) const override; - - float left_count(int split, int class_num) const override { - return left_counts_[split * num_outputs_ + class_num]; - } - float right_count(int split, int class_num) const override { - return total_counts_[class_num] - - left_counts_[split * num_outputs_ + class_num]; - } - - private: - inline float& mutable_left_count(int split, int class_num) { - return left_counts_[split * num_outputs_ + class_num]; - } - // Total class counts seen at this leaf - std::vector total_counts_; - - // Also track the number of classes seen for not splitting pure leaves. - int num_outputs_seen_; - - // Left-branch taken class counts at this leaf for each split. - // This is a flat vector for memory-performance reasons. - // left_counts_[i * num_outputs_ + j] has the j-th class count for split i. - std::vector left_counts_; -}; - -// Tracks classification stats by storing class counts sparsely. -class SparseClassificationGrowStats : public ClassificationStats { - public: - SparseClassificationGrowStats(const TensorForestParams& params, int32 depth) - : ClassificationStats(params, depth) {} - - void Initialize() override { Clear(); } - - void ExtractFromProto(const FertileSlot& slot) override; - void PackToProto(FertileSlot* slot) const override; - - void InitLeafClassStats(int best_split_index, LeafStat* left_stats, - LeafStat* right_stats) const override; - - protected: - void ClassificationAddSplitStats() override { - left_counts_.resize(num_splits()); - } - void ClassificationRemoveSplitStats(int split_num) override { - left_counts_.erase(left_counts_.begin() + split_num, - left_counts_.begin() + (split_num + 1)); - } - void ClearInternal() override { - total_counts_.clear(); - left_counts_.clear(); - } - - bool is_pure() const override { return total_counts_.size() <= 1; } - - void ClassificationAddLeftExample(int split, int64 int_label, - float weight) override { - left_counts_[split][int_label] += weight; - } - void ClassificationAddTotalExample(int64 int_label, float weight) override { - total_counts_[int_label] += weight; - } - - float GiniScore(int split, float* left_sum, float* right_sum) const override; - - float left_count(int split, int class_num) const override { - return left_counts_[split].at(class_num); - } - float right_count(int split, int class_num) const override { - return total_counts_.at(class_num) - left_counts_[split].at(class_num); - } - - private: - // Total class counts seen at this leaf - std::unordered_map total_counts_; - - // Left-branch taken class counts at this leaf for each split. - // left_counts_[i][j] has the j-th class count for split i. - std::vector> left_counts_; -}; - -// Accumulates weights for the most popular classes while only using a -// fixed amount of space. -class FixedSizeClassStats { - public: - // n specifies how many classes are tracked. - FixedSizeClassStats(int n, int num_classes) - : n_(n), num_classes_(num_classes), smallest_weight_class_(-1) {} - - // Add weight w to the class c. - void accumulate(int c, float w); - - // Return the approximate accumulated weight for class c. If c isn't one - // of the n-most popular classes, this can be 0 even if c has accumulated - // some weight. - float get_weight(int c) const; - - // Put the sum of all weights seen into *sum, and - // \sum_c get_weight(c)^2 - // into *square. *sum will be exact, but *square will be approximate. - void set_sum_and_square(float* sum, float* square) const; - - void ExtractFromProto(const decision_trees::SparseVector& sparse_vector); - void PackToProto(decision_trees::SparseVector* sparse_vector) const; - - private: - // For our typical use cases, n_ is between 10 and 100, so there's no - // need to track the smallest weight with a min_heap or the like. - int n_; - int num_classes_; - - // This tracks the class of the smallest weight, but isn't set until - // class_weights_.size() == n_. - int smallest_weight_class_; - - std::unordered_map class_weights_; -}; - -// Tracks classification stats sparsely in a fixed amount of space. -class FixedSizeSparseClassificationGrowStats : public ClassificationStats { - public: - FixedSizeSparseClassificationGrowStats(const TensorForestParams& params, - int32 depth) - : ClassificationStats(params, depth) {} - - void Initialize() override { Clear(); } - - void ExtractFromProto(const FertileSlot& slot) override; - void PackToProto(FertileSlot* slot) const override; - - void InitLeafClassStats(int best_split_index, LeafStat* left_stats, - LeafStat* right_stats) const override; - - protected: - void ClassificationAddSplitStats() override { - FixedSizeClassStats stats(params_.num_classes_to_track(), - params_.num_outputs()); - left_counts_.resize(num_splits(), stats); - right_counts_.resize(num_splits(), stats); - } - void ClassificationRemoveSplitStats(int split_num) override { - left_counts_.erase(left_counts_.begin() + split_num, - left_counts_.begin() + (split_num + 1)); - right_counts_.erase(right_counts_.begin() + split_num, - right_counts_.begin() + (split_num + 1)); - } - void ClearInternal() override { - left_counts_.clear(); - right_counts_.clear(); - } - - bool is_pure() const override { return first_two_classes_seen_.size() <= 1; } - - void ClassificationAddLeftExample(int split, int64 int_label, - float weight) override { - left_counts_[split].accumulate(int_label, weight); - } - void ClassificationAddRightExample(int split, int64 int_label, - float weight) override { - right_counts_[split].accumulate(int_label, weight); - } - void ClassificationAddTotalExample(int64 int_label, float weight) override { - if (is_pure()) { - first_two_classes_seen_.insert(int_label); - } - } - - float GiniScore(int split, float* left_sum, float* right_sum) const override; - - float left_count(int split, int class_num) const override { - return left_counts_[split].get_weight(class_num); - } - - float right_count(int split, int class_num) const override { - return right_counts_[split].get_weight(class_num); - } - - private: - std::vector left_counts_; - std::vector right_counts_; - - // We keep track of the first two class labels seen, so we can tell if - // the node is pure (= all of one class) or not. - std::set first_two_classes_seen_; -}; - -// Tracks regression stats using least-squares minimization. -class LeastSquaresRegressionGrowStats : public GrowStats { - public: - LeastSquaresRegressionGrowStats(const TensorForestParams& params, int32 depth) - : GrowStats(params, depth) {} - - void Initialize() override { - Clear(); - total_sum_.resize(num_outputs_); - total_sum_squares_.resize(num_outputs_); - } - - void ExtractFromProto(const FertileSlot& slot) override; - void PackToProto(FertileSlot* slot) const override; - - void AddExample(const std::unique_ptr& input_data, - const InputTarget* target, int example) override; - bool BestSplit(SplitCandidate* best) const override; - bool IsFinished() const override; - - protected: - // Returns the variance of split. - float SplitVariance(int split) const; - - void AddSplitStats(const InputTarget* target, int example) override { - left_sums_.resize(num_outputs_ * num_splits()); - left_squares_.resize(num_outputs_ * num_splits()); - left_counts_.push_back(0); - } - void RemoveSplitStats(int split_num) override { - left_sums_.erase(left_sums_.begin() + num_outputs_ * split_num, - left_sums_.begin() + num_outputs_ * (split_num + 1)); - left_squares_.erase(left_squares_.begin() + num_outputs_ * split_num, - left_squares_.begin() + num_outputs_ * (split_num + 1)); - left_counts_.erase(left_counts_.begin() + split_num, - left_counts_.begin() + (split_num + 1)); - } - - void ClearInternal() override { - total_sum_.clear(); - total_sum_squares_.clear(); - left_sums_.clear(); - left_squares_.clear(); - } - - private: - // Convenience methods for accessing the flat count vectors. - inline const float& left_sum(int split, int output_num) const { - return left_sums_[split * num_outputs_ + output_num]; - } - inline float& left_sum(int split, int output_num) { - return left_sums_[split * num_outputs_ + output_num]; - } - inline const float& left_square(int split, int output_num) const { - return left_squares_[split * num_outputs_ + output_num]; - } - inline float& left_square(int split, int output_num) { - return left_squares_[split * num_outputs_ + output_num]; - } - - // Total sums and squares seen at this leaf. - // sum[i] is the sum of the i-th output. - std::vector total_sum_; - std::vector total_sum_squares_; - - // Per-split sums and squares, stored flat for performance. - // left_sums_[i * num_outputs_ + j] has the j-th sum for split i. - std::vector left_sums_; - std::vector left_squares_; - - // The number of example seen at each split. - std::vector left_counts_; -}; - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_GROW_STATS_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/grow_stats_test.cc b/tensorflow/contrib/tensor_forest/kernels/v4/grow_stats_test.cc deleted file mode 100644 index 26e989928e0..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/grow_stats_test.cc +++ /dev/null @@ -1,423 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.h" - -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/test_utils.h" -#include "tensorflow/contrib/tensor_forest/proto/tensor_forest_params.pb.h" -#include "tensorflow/core/platform/protobuf.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace { - -using tensorflow::decision_trees::BinaryNode; -using tensorflow::decision_trees::FeatureId; -using tensorflow::decision_trees::InequalityTest; -using tensorflow::tensorforest::DenseClassificationGrowStats; -using tensorflow::tensorforest::FertileSlot; -using tensorflow::tensorforest::FixedSizeClassStats; -using tensorflow::tensorforest::FixedSizeSparseClassificationGrowStats; -using tensorflow::tensorforest::GrowStats; -using tensorflow::tensorforest::LeastSquaresRegressionGrowStats; -using tensorflow::tensorforest::SparseClassificationGrowStats; -using tensorflow::tensorforest::SPLIT_FINISH_BASIC; -using tensorflow::tensorforest::SPLIT_FINISH_DOMINATE_HOEFFDING; -using tensorflow::tensorforest::SPLIT_PRUNE_HOEFFDING; -using tensorflow::tensorforest::TensorForestParams; -using tensorflow::tensorforest::TestableInputTarget; - -BinaryNode MakeSplit(const string& feat, float val) { - BinaryNode split; - InequalityTest* test = split.mutable_inequality_left_child_test(); - FeatureId feature_id; - feature_id.mutable_id()->set_value(feat); - *test->mutable_feature_id() = feature_id; - test->mutable_threshold()->set_float_value(val); - test->set_type(InequalityTest::LESS_OR_EQUAL); - - return split; -} - -void RunBatch(GrowStats* stats, const TestableInputTarget* target) { - std::unique_ptr dataset( - new tensorflow::tensorforest::TestableDataSet( - {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}, 2)); - - stats->AddSplit(MakeSplit("0", 10.0), dataset, target, 0); - stats->AddSplit(MakeSplit("1", 4.0), dataset, target, 0); - - for (int i = 0; i < target->NumItems(); ++i) { - stats->AddExample(dataset, target, i); - } -} - -TEST(GrowStatsDenseClassificationTest, Basic) { - TensorForestParams params; - params.set_num_outputs(2); - params.mutable_split_after_samples()->set_constant_value(2); - params.mutable_num_splits_to_consider()->set_constant_value(2); - std::unique_ptr stat( - new DenseClassificationGrowStats(params, 1)); - stat->Initialize(); - - std::vector labels = {1, 0, 1}; - std::vector weights = {2.3, 20.3, 1.1}; - std::unique_ptr target( - new TestableInputTarget(labels, weights, 1)); - - RunBatch(stat.get(), target.get()); - CHECK(stat->IsFinished()); - - FertileSlot slot; - stat->PackToProto(&slot); - - string serialized = slot.DebugString(); - - std::unique_ptr new_stat( - new DenseClassificationGrowStats(params, 1)); - new_stat->ExtractFromProto(slot); - FertileSlot second_one; - new_stat->PackToProto(&second_one); - string serialized_again = second_one.DebugString(); - ASSERT_EQ(serialized_again, serialized); -} - -class TestableRunningStats : public DenseClassificationGrowStats { - public: - TestableRunningStats(const TensorForestParams& params, int32 depth) - : DenseClassificationGrowStats(params, depth) {} - - float test_left_sum(int split) { return get_left_gini()->sum(split); } - float test_left_square(int split) { return get_left_gini()->square(split); } - float test_right_sum(int split) { return get_right_gini()->sum(split); } - float test_right_square(int split) { return get_right_gini()->square(split); } -}; - -TEST(GrowStatsDenseClassificationTest, BasicRunningStats) { - TensorForestParams params; - params.set_num_outputs(2); - params.mutable_split_after_samples()->set_constant_value(2); - params.mutable_num_splits_to_consider()->set_constant_value(2); - params.set_use_running_stats_method(true); - std::unique_ptr stat( - new TestableRunningStats(params, 1)); - stat->Initialize(); - - std::vector labels = {1, 0, 1}; - std::vector weights = {2.3, 20.3, 1.1}; - std::unique_ptr target( - new TestableInputTarget(labels, weights, 1)); - - RunBatch(stat.get(), target.get()); - CHECK(stat->IsFinished()); - - ASSERT_FLOAT_EQ(stat->test_left_sum(0), 2.3 + 20.3 + 1.1); - ASSERT_FLOAT_EQ(stat->test_left_square(0), 3.4 * 3.4 + 20.3 * 20.3); - ASSERT_FLOAT_EQ(stat->test_right_sum(0), 0.0); - ASSERT_FLOAT_EQ(stat->test_right_square(0), 0.0); - - ASSERT_FLOAT_EQ(stat->test_left_sum(1), 2.3 + 20.3); - ASSERT_FLOAT_EQ(stat->test_left_square(1), 2.3 * 2.3 + 20.3 * 20.3); - ASSERT_FLOAT_EQ(stat->test_right_sum(1), 1.1); - ASSERT_FLOAT_EQ(stat->test_right_square(1), 1.1 * 1.1); - - FertileSlot slot; - stat->PackToProto(&slot); - - string serialized = slot.DebugString(); - - std::unique_ptr new_stat( - new DenseClassificationGrowStats(params, 1)); - new_stat->ExtractFromProto(slot); - FertileSlot second_one; - new_stat->PackToProto(&second_one); - string serialized_again = second_one.DebugString(); - ASSERT_EQ(serialized_again, serialized); -} - -class TestableFinishEarly : public DenseClassificationGrowStats { - public: - TestableFinishEarly(const TensorForestParams& params, int32 depth) - : DenseClassificationGrowStats(params, depth), num_times_called_(0) {} - - int num_times_called_; - - protected: - void CheckFinishEarlyHoeffding() override { ++num_times_called_; } -}; - -TEST(GrowStatsDenseClassificationTest, TestFinishEarly) { - TensorForestParams params; - params.set_num_outputs(2); - params.mutable_split_after_samples()->set_constant_value(2); - params.mutable_num_splits_to_consider()->set_constant_value(2); - params.mutable_min_split_samples()->set_constant_value(15); - params.mutable_dominate_fraction()->set_constant_value(0.99); - auto* finish = params.mutable_finish_type(); - finish->set_type(SPLIT_FINISH_DOMINATE_HOEFFDING); - finish->mutable_check_every_steps()->set_constant_value(5); - std::unique_ptr stat(new TestableFinishEarly(params, 1)); - stat->Initialize(); - - std::vector labels = {1, 0, 1}; - std::vector weights = {1, 1, 1}; - std::unique_ptr target( - new TestableInputTarget(labels, weights, 1)); - std::unique_ptr dataset( - new tensorflow::tensorforest::TestableDataSet( - {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}, 2)); - - // Run through the 3 examples - RunBatch(stat.get(), target.get()); - - ASSERT_EQ(stat->num_times_called_, 0); - - // Go over min_split_samples. - for (int i = 0; i < 13; ++i) { - stat->AddExample(dataset, target.get(), 0); - } - - ASSERT_EQ(stat->num_times_called_, 1); - - // More examples up to 55. - for (int i = 0; i < 39; ++i) { - stat->AddExample(dataset, target.get(), 0); - } - - ASSERT_EQ(stat->num_times_called_, 9); -} - -TEST(GrowStatsDenseClassificationTest, TestCheckPruneHoeffding) { - TensorForestParams params; - params.set_num_outputs(2); - params.mutable_split_after_samples()->set_constant_value(2000); - params.mutable_num_splits_to_consider()->set_constant_value(2); - params.mutable_min_split_samples()->set_constant_value(15); - params.mutable_dominate_fraction()->set_constant_value(0.99); - auto* finish = params.mutable_finish_type(); - finish->set_type(SPLIT_FINISH_BASIC); - finish->mutable_check_every_steps()->set_constant_value(100); - params.mutable_pruning_type()->set_type(SPLIT_PRUNE_HOEFFDING); - params.mutable_pruning_type() - ->mutable_prune_every_samples() - ->set_constant_value(1); - - // On each iteration, we add two examples, one of class 0 and one - // of class 1. Split #0 classifies them perfectly, while split #1 - // sends them both to the left. - std::vector labels = {0, 1}; - std::vector weights = {1, 1}; - TestableInputTarget target(labels, weights, 1); - std::unique_ptr dataset( - new tensorflow::tensorforest::TestableDataSet({-1.0, -1.0, 1.0, -1.0}, - 2)); - - DenseClassificationGrowStats stats(params, 1); - stats.Initialize(); - stats.AddSplit(MakeSplit("0", 0.0), dataset, &target, 0); - stats.AddSplit(MakeSplit("1", 0.0), dataset, &target, 0); - - // Math time! - // After 2n samples, - // split 0 has smoothed counts (n+1,1);(1,n+1) and - // split 1 has smoothed counts (n+1,n+1);(1,1) - // split 0 smoothed ginis are both 1 - (n+1)^2/(n+2)^2 - 1/(n+2)^2 and - // split 1 smoothed ginis are 1 - 2 (n+1)^2 / (2n+2)^2 and 1 - 2 (1/4) = 1/2 - // split 0 weighted smoothed ginis are both n (1 - (n^2 + 2n + 2) / (n+2)^2) - // split 1 weighted smoothed ginis are 0 and 2n (1 - 2(n+1)^2 / (2n+2)^2) - // split 0 split score = 2n (1 - (n^2 + 2n + 2) / (n+2)^2) - // split 1 spilt score = 2n (1 - 2(n+1)^2 / (2n+2)^2) - // split 1 score - split 0 score = - // 2n ( (n^2 + 2n + 2) / (n+2)^2 - 2(n+1)^2 / (2n+2)^2 ) - // = 2n ( (n^2 + 2n + 2) (2n+2)^2 - 2(n+1)^2 (n+2)^2 ) / ((n+2)^2 (2n+2)^2 ) - // = 2n ((n^2+2n+2)(4n^2+8n+4) - 2(n^2+2n+1)(n^2+4n+4)) / ((n+2)^2 (2n+2)^2) - // = 2n (4n^4+8n^3+4n^2+8n^3+16n^2+8n+8n^2+16n+8 - // - (2n^4+8n^3+8n^2+4n^3+16n^2+16n+2n^2+8n+8)) / ((n+2)^2 (2n+2)^2) - // = 2n (2n^4 + 4n^3 + 2n^2) / ((n+2)^2 (2n+2)^2) - // = 4n^3 (n^2 + 2n + 1) / ((n+2)^2 (2n+2)^2) - // = n^3 / (n+2)^2 - // Meanwhile, after 2n samples, - // epsilon = 2n (1 - 1/2) sqrt(0.5 ln(1/0.01) / 2n) - // = n sqrt( ln(10) / 2n) - // Graphical comparison says that epsilon is greater between 0 and 4.5, - // and then the split score difference is greater for n >= 5. - // n = 1 - stats.AddExample(dataset, &target, 0); - stats.AddExample(dataset, &target, 1); - ASSERT_EQ(stats.num_splits(), 2); - - // n = 2 - stats.AddExample(dataset, &target, 0); - stats.AddExample(dataset, &target, 1); - ASSERT_EQ(stats.num_splits(), 2); - - // n = 3 - stats.AddExample(dataset, &target, 0); - stats.AddExample(dataset, &target, 1); - ASSERT_EQ(stats.num_splits(), 2); - - // n = 4 - stats.AddExample(dataset, &target, 0); - stats.AddExample(dataset, &target, 1); - ASSERT_EQ(stats.num_splits(), 2); - - // n = 5 - stats.AddExample(dataset, &target, 0); - stats.AddExample(dataset, &target, 1); - ASSERT_EQ(stats.num_splits(), 1); - - // n = 6 - stats.AddExample(dataset, &target, 0); - stats.AddExample(dataset, &target, 1); - ASSERT_EQ(stats.num_splits(), 1); -} - -TEST(GrowStatsLeastSquaresRegressionTest, Basic) { - TensorForestParams params; - params.set_num_outputs(1); - params.mutable_split_after_samples()->set_constant_value(2); - params.mutable_num_splits_to_consider()->set_constant_value(2); - std::unique_ptr stat( - new LeastSquaresRegressionGrowStats(params, 1)); - stat->Initialize(); - - std::vector labels = {2.3, 5.6, 1.1}; - std::unique_ptr target( - new TestableInputTarget(labels, {}, 1)); - std::vector branches = {1, 0, 1, 1, 0, 0}; - - RunBatch(stat.get(), target.get()); - CHECK(stat->IsFinished()); - - FertileSlot slot; - stat->PackToProto(&slot); - - string serialized = slot.DebugString(); - - std::unique_ptr new_stat( - new LeastSquaresRegressionGrowStats(params, 1)); - new_stat->ExtractFromProto(slot); - FertileSlot second_one; - new_stat->PackToProto(&second_one); - string serialized_again = second_one.DebugString(); - - ASSERT_EQ(serialized_again, serialized); -} - -TEST(GrowStatsSparseClassificationTest, Basic) { - TensorForestParams params; - params.set_num_outputs(2); - params.mutable_split_after_samples()->set_constant_value(2); - params.mutable_num_splits_to_consider()->set_constant_value(2); - std::unique_ptr stat( - new SparseClassificationGrowStats(params, 1)); - stat->Initialize(); - - std::vector labels = {100, 1000, 1}; - std::vector weights = {2.3, 20.3, 1.1}; - std::unique_ptr target( - new TestableInputTarget(labels, weights, 1)); - std::vector branches = {1, 0, 1, 1, 0, 0}; - - RunBatch(stat.get(), target.get()); - CHECK(stat->IsFinished()); - - FertileSlot slot; - stat->PackToProto(&slot); - - string serialized = slot.DebugString(); - - std::unique_ptr new_stat( - new SparseClassificationGrowStats(params, 1)); - new_stat->ExtractFromProto(slot); - FertileSlot second_one; - new_stat->PackToProto(&second_one); - string serialized_again = second_one.DebugString(); - ASSERT_EQ(serialized_again, serialized); -} - -TEST(FixedSizeClassStats, Exact) { - FixedSizeClassStats stats(10, 100); - - stats.accumulate(1, 1.0); - stats.accumulate(2, 2.0); - stats.accumulate(3, 3.0); - - EXPECT_EQ(stats.get_weight(1), 1.0); - EXPECT_EQ(stats.get_weight(2), 2.0); - EXPECT_EQ(stats.get_weight(3), 3.0); - - float sum; - float square; - stats.set_sum_and_square(&sum, &square); - - EXPECT_EQ(sum, 6.0); - EXPECT_EQ(square, 14.0); -} - -TEST(FixedSizeClassStats, Approximate) { - FixedSizeClassStats stats(5, 10); - - for (int i = 1; i <= 10; i++) { - stats.accumulate(i, i * 1.0); - } - - // We should be off by no more than *half* of the least weight - // in the class_weights_, which is 7. - float tolerance = 3.5; - for (int i = 1; i <= 10; i++) { - float diff = stats.get_weight(i) - i * 1.0; - EXPECT_LE(diff, tolerance); - EXPECT_GE(diff, -tolerance); - } -} - -TEST(GrowStatsFixedSizeSparseClassificationTest, Basic) { - TensorForestParams params; - params.set_num_outputs(2); - params.set_num_classes_to_track(5); - params.mutable_split_after_samples()->set_constant_value(2); - params.mutable_num_splits_to_consider()->set_constant_value(2); - std::unique_ptr stat( - new FixedSizeSparseClassificationGrowStats(params, 1)); - stat->Initialize(); - - std::vector labels = {100, 1000, 1}; - std::vector weights = {2.3, 20.3, 1.1}; - std::unique_ptr target( - new TestableInputTarget(labels, weights, 1)); - std::vector branches = {1, 0, 1, 1, 0, 0}; - - RunBatch(stat.get(), target.get()); - CHECK(stat->IsFinished()); - - FertileSlot slot; - stat->PackToProto(&slot); - - string serialized = slot.DebugString(); - - std::unique_ptr new_stat( - new FixedSizeSparseClassificationGrowStats(params, 1)); - new_stat->ExtractFromProto(slot); - FertileSlot second_one; - new_stat->PackToProto(&second_one); - string serialized_again = second_one.DebugString(); - ASSERT_EQ(serialized_again, serialized); -} - -} // namespace -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/input_data.cc b/tensorflow/contrib/tensor_forest/kernels/v4/input_data.cc deleted file mode 100644 index 99c58003912..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/input_data.cc +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_data.h" -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model_extensions.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" -#include "tensorflow/core/lib/strings/numbers.h" - -namespace tensorflow { -namespace tensorforest { -namespace { - -bool DecideInequalityTest(const decision_trees::InequalityTest& test, - float value) { - float bias = test.threshold().float_value(); - switch (test.type()) { - case decision_trees::InequalityTest::LESS_OR_EQUAL: - return value <= bias; - - case decision_trees::InequalityTest::LESS_THAN: - return value < bias; - - case decision_trees::InequalityTest::GREATER_OR_EQUAL: - return value >= bias; - - case decision_trees::InequalityTest::GREATER_THAN: - return value > bias; - - default: - return false; - } -} - -bool DecideMatchingValuesTest(const decision_trees::MatchingValuesTest& test, - float value) { - for (const decision_trees::Value& test_value : test.value()) { - if (test_value.float_value() == value) { - return true; - } - } - return false; -} - -} // namespace - -bool TensorDataSet::Decide(const decision_trees::BinaryNode& node, - int example) const { - // TODO(gilberth): Support missing values. - float val = 0; - const auto& test = node.inequality_left_child_test(); - - if (test.has_oblique()) { - for (int i = 0; i < test.oblique().features_size(); ++i) { - val += test.oblique().weights(i) * - GetExampleValue(example, test.oblique().features(i)); - } - } else { - val = GetExampleValue(example, test.feature_id()); - } - - if (node.has_inequality_left_child_test()) { - return DecideInequalityTest(node.inequality_left_child_test(), val); - } else { - decision_trees::MatchingValuesTest test; - if (node.custom_left_child_test().UnpackTo(&test)) { - return DecideMatchingValuesTest(test, val); - } else { - return false; - } - } -} - -float TensorDataSet::GetExampleValue( - int example, const decision_trees::FeatureId& feature_id) const { - int32 feature; - safe_strto32(feature_id.id().value(), &feature); - if (feature >= input_spec_.dense_features_size()) { - return FindSparseValue(*sparse_indices_, *sparse_values_, example, feature); - } else { - return (*dense_data_)(example, feature); - } -} - -float TensorDataSet::GetExampleValue(int example, int32 feature_id) const { - if (feature_id >= input_spec_.dense_features_size()) { - return FindSparseValue(*sparse_indices_, *sparse_values_, example, - feature_id); - } else { - return (*dense_data_)(example, feature_id); - } -} - -void TensorDataSet::set_input_tensors(const Tensor& dense, - const Tensor& sparse_indices, - const Tensor& sparse_values, - const Tensor& sparse_shape) { - if (dense.shape().dims() == 2) { - dense_data_.reset(new DenseStorageType(dense.tensor())); - } - if (sparse_indices.shape().dims() == 2) { - sparse_indices_.reset( - new SparseIndicesStorageType(sparse_indices.tensor())); - sparse_values_.reset( - new SparseValuesStorageType(sparse_values.tensor())); - sparse_batch_size_ = sparse_shape.tensor()(0); - } - original_dense_tensor_ = dense; -} - -void TensorDataSet::RandomSample(int example, - decision_trees::FeatureId* feature_id, - float* bias, int* type) const { - int32 num_total_features = input_spec_.dense_features_size(); - int64 sparse_input_start; - if (sparse_indices_ != nullptr) { - const int32 num_sparse = tensorforest::GetNumSparseFeatures( - *sparse_indices_, example, &sparse_input_start); - if (sparse_input_start >= 0) { - num_total_features += num_sparse; - } - } - int rand_feature = 0; - { - mutex_lock lock(mu_); - rand_feature = rng_->Uniform(num_total_features); - } - if (rand_feature < available_features_.size()) { // it's dense. - *feature_id = available_features_[rand_feature]; - *type = input_spec_.GetDenseFeatureType(rand_feature); - } else { - const int32 sparse_index = - sparse_input_start + rand_feature - input_spec_.dense_features_size(); - const int32 saved_index = - (*sparse_indices_)(sparse_index, 1) + input_spec_.dense_features_size(); - *feature_id = decision_trees::FeatureId(); - feature_id->mutable_id()->set_value(strings::StrCat(saved_index)); - - // TODO(gilberth): Remove this shortcut when different sparse types are - // allowed. - *type = input_spec_.sparse(0).original_type(); - } - - *bias = GetExampleValue(example, *feature_id); -} - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/input_data.h b/tensorflow/contrib/tensor_forest/kernels/v4/input_data.h deleted file mode 100644 index 4945b53007e..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/input_data.h +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_INPUT_DATA_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_INPUT_DATA_H_ -#include -#include -#include "google/protobuf/any.pb.h" -#include "google/protobuf/wrappers.pb.h" -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/data_spec.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/lib/random/philox_random.h" -#include "tensorflow/core/lib/random/random.h" -#include "tensorflow/core/lib/random/simple_philox.h" -#include "tensorflow/core/platform/mutex.h" - -namespace tensorflow { -namespace tensorforest { - -typedef TTypes::ConstTensor DenseStorageType; -typedef TTypes::ConstTensor SparseIndicesStorageType; -typedef TTypes::ConstTensor SparseValuesStorageType; - -class TensorDataSet { - public: - TensorDataSet(const tensorforest::TensorForestDataSpec& input_spec, - int32 seed) - : dense_data_(nullptr), - sparse_indices_(nullptr), - sparse_values_(nullptr), - input_spec_(input_spec), - split_sampling_random_seed_(seed) { - int column_count = 0; - for (int i = 0; i < input_spec_.dense_size(); ++i) { - for (int j = 0; j < input_spec_.dense(i).size(); ++j) { - ++column_count; - } - } - available_features_.reserve(column_count); - decision_trees::FeatureId id; - for (int i = 0; i < column_count; i++) { - id.mutable_id()->set_value(strings::StrCat(i)); - available_features_.emplace_back(id); - } - - // Set up the random number generator. - if (split_sampling_random_seed_ == 0) { - single_rand_ = std::unique_ptr( - new random::PhiloxRandom(random::New64())); - } else { - single_rand_ = std::unique_ptr( - new random::PhiloxRandom(split_sampling_random_seed_)); - } - - rng_ = std::unique_ptr( - new random::SimplePhilox(single_rand_.get())); - } - virtual ~TensorDataSet() {} - - void set_input_tensors(const Tensor& dense, const Tensor& sparse_indices, - const Tensor& sparse_values, - const Tensor& sparse_shape); - - float get_input_value(int offset, int col) { - return (*dense_data_)(offset, col); - } - - int NumItems() const { - if (dense_data_ != nullptr) { - return dense_data_->dimensions()[0]; - } else if (sparse_indices_ != nullptr) { - return sparse_batch_size_; - } else { - return 0; - } - } - - // This looks up a value by example and int32_id, which is much faster than - // GetFeature. - float GetExampleValue(int example, - const decision_trees::FeatureId& feature_id) const; - - // Same as overload with FeatureId, but if you already have the feature as - // an int32 you can avoid the atoi32. - virtual float GetExampleValue(int example, int32 feature_id) const; - - int num_features() { return available_features_.size(); } - - const Tensor& original_tensor() const { return original_dense_tensor_; } - - bool Decide(const decision_trees::BinaryNode& node, int example) const; - - // Randomly samples a feature from example, returns its id in feature_name, - // the value in bias, and it's type from input_spec in type. - void RandomSample(int example, decision_trees::FeatureId* feature_name, - float* bias, int* type) const; - - private: - std::unique_ptr dense_data_; - std::unique_ptr sparse_indices_; - std::unique_ptr sparse_values_; - int sparse_batch_size_; - - Tensor original_dense_tensor_; - const tensorforest::TensorForestDataSpec input_spec_; - std::vector available_features_; - - int32 split_sampling_random_seed_; - std::unique_ptr single_rand_; - std::unique_ptr rng_; - // Mutex for using random number generator. - mutable mutex mu_; -}; -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_INPUT_DATA_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/input_target.h b/tensorflow/contrib/tensor_forest/kernels/v4/input_target.h deleted file mode 100644 index d4402b6055a..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/input_target.h +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_INPUT_TARGET_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_INPUT_TARGET_H_ -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_types.h" - -namespace tensorflow { -namespace tensorforest { - -typedef TTypes::UnalignedConstTensor SingleDimStorageType; - -// Base class for classes that hold labels and weights. Mostly for testing -// purposes, because it's inconvenient to construct nasty Eigen::things. -class InputTarget { - public: - virtual ~InputTarget() {} - virtual int32 GetTargetAsClassIndex(int example_index, - int target_index) const = 0; - - virtual float GetTargetWeight(int example_index) const = 0; - - virtual float GetTargetAsContinuous(int example_index, - int target_index) const = 0; -}; - -template -class StoredInputTarget : public InputTarget { - protected: - // Takes ownership of t and w with a std::unique_ptr. - StoredInputTarget(const T* t, const T* w, int num_targets) - : target_(t), weight_(w), num_targets_(num_targets) {} - - const std::unique_ptr target_; - const std::unique_ptr weight_; - int num_targets_; -}; - -// Holds labels/targets and weights. Assumes that tensors are passed as -// t.unaligned_flat(). For multi-output, specifying the number of -// outputs will correctly index the flattened data. -class TensorInputTarget : public StoredInputTarget { - public: - TensorInputTarget(const Tensor& target, const Tensor& weight, int num_targets) - : StoredInputTarget( - new SingleDimStorageType(target.unaligned_flat()), - new SingleDimStorageType(weight.unaligned_flat()), - num_targets), - original_tensor_(target) {} - - int32 GetTargetAsClassIndex(int example_index, - int target_index) const override { - return static_cast( - GetTargetAsContinuous(example_index, target_index)); - } - - float GetTargetWeight(int example_index) const override { - const size_t num_weights = weight_->size(); - return num_weights > 0 && example_index < num_weights - ? (*weight_)(example_index) - : 1.0; - } - - float GetTargetAsContinuous(int example_index, - int target_index) const override { - QCHECK_LT(target_index, num_targets_); - return (*target_)(example_index * num_targets_ + target_index); - } - - const Tensor& original_tensor() const { return original_tensor_; } - - protected: - Tensor original_tensor_; -}; -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_INPUT_TARGET_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators.cc b/tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators.cc deleted file mode 100644 index 83614a25314..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators.cc +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators.h" - -namespace tensorflow { -namespace tensorforest { - -using decision_trees::Leaf; - -std::unique_ptr -LeafModelOperatorFactory::CreateLeafModelOperator( - const TensorForestParams& params) { - switch (params.leaf_type()) { - case MODEL_DENSE_CLASSIFICATION: - return std::unique_ptr( - new DenseClassificationLeafModelOperator(params)); - - case MODEL_SPARSE_CLASSIFICATION: - return std::unique_ptr( - new SparseClassificationLeafModelOperator(params)); - - case MODEL_SPARSE_OR_DENSE_CLASSIFICATION: - return std::unique_ptr( - new SparseOrDenseClassificationLeafModelOperator(params)); - - case MODEL_REGRESSION: - return std::unique_ptr( - new RegressionLeafModelOperator(params)); - - default: - LOG(ERROR) << "Unknown model operator: " << params.leaf_type(); - return nullptr; - } -} - -// ------------------------ Dense ----------------------------- // -float DenseClassificationLeafModelOperator::GetOutputValue( - const decision_trees::Leaf& leaf, int32 o) const { - return leaf.vector().value(o).float_value(); -} - -void DenseClassificationLeafModelOperator::UpdateModel( - Leaf* leaf, const InputTarget* target, int example) const { - const int32 int_label = target->GetTargetAsClassIndex(example, 0); - QCHECK_LT(int_label, params_.num_outputs()) - << "Got label greater than indicated number of classes. Is " - "params.num_classes set correctly?"; - QCHECK_GE(int_label, 0); - auto* val = leaf->mutable_vector()->mutable_value(int_label); - - float weight = target->GetTargetWeight(example); - val->set_float_value(val->float_value() + weight); -} - -void DenseClassificationLeafModelOperator::InitModel(Leaf* leaf) const { - for (int i = 0; i < params_.num_outputs(); ++i) { - leaf->mutable_vector()->add_value(); - } -} - -void DenseClassificationLeafModelOperator::ExportModel( - const LeafStat& stat, decision_trees::Leaf* leaf) const { - *leaf->mutable_vector() = stat.classification().dense_counts(); -} - -// ------------------------- Sparse -------------------------- // -float SparseClassificationLeafModelOperator::GetOutputValue( - const decision_trees::Leaf& leaf, int32 o) const { - const auto it = leaf.sparse_vector().sparse_value().find(o); - if (it == leaf.sparse_vector().sparse_value().end()) { - return 0; // default value - } else { - return it->second.float_value(); - } -} - -void SparseClassificationLeafModelOperator::UpdateModel( - Leaf* leaf, const InputTarget* target, int example) const { - const int32 int_label = target->GetTargetAsClassIndex(example, 0); - QCHECK_LT(int_label, params_.num_outputs()) - << "Got label greater than indicated number of classes. Is " - "params.num_classes set correctly?"; - QCHECK_GE(int_label, 0); - const float weight = target->GetTargetWeight(example); - - auto value_map = leaf->mutable_sparse_vector()->mutable_sparse_value(); - auto it = value_map->find(int_label); - if (it == value_map->end()) { - (*value_map)[int_label].set_float_value(weight); - } else { - it->second.set_float_value(it->second.float_value() + weight); - } -} - -void SparseClassificationLeafModelOperator::ExportModel( - const LeafStat& stat, decision_trees::Leaf* leaf) const { - *leaf->mutable_sparse_vector() = stat.classification().sparse_counts(); -} - -// ------------------------- SparseOrDense -------------------------- // -float SparseOrDenseClassificationLeafModelOperator::GetOutputValue( - const decision_trees::Leaf& leaf, int32 o) const { - if (leaf.has_vector()) { - return dense_->GetOutputValue(leaf, o); - } else { - return sparse_->GetOutputValue(leaf, o); - } -} - -void SparseOrDenseClassificationLeafModelOperator::UpdateModel( - Leaf* leaf, const InputTarget* target, int example) const { - if (leaf->has_vector()) { - return dense_->UpdateModel(leaf, target, example); - } else { - return sparse_->UpdateModel(leaf, target, example); - } -} - -void SparseOrDenseClassificationLeafModelOperator::ExportModel( - const LeafStat& stat, decision_trees::Leaf* leaf) const { - if (stat.classification().has_dense_counts()) { - return dense_->ExportModel(stat, leaf); - } else { - return sparse_->ExportModel(stat, leaf); - } -} - -// ------------------------ Regression ----------------------------- // -float RegressionLeafModelOperator::GetOutputValue( - const decision_trees::Leaf& leaf, int32 o) const { - return leaf.vector().value(o).float_value(); -} - -void RegressionLeafModelOperator::InitModel(Leaf* leaf) const { - for (int i = 0; i < params_.num_outputs(); ++i) { - leaf->mutable_vector()->add_value(); - } -} - -void RegressionLeafModelOperator::ExportModel( - const LeafStat& stat, decision_trees::Leaf* leaf) const { - leaf->clear_vector(); - for (int i = 0; i < params_.num_outputs(); ++i) { - const float new_val = - stat.regression().mean_output().value(i).float_value() / - stat.weight_sum(); - leaf->mutable_vector()->add_value()->set_float_value(new_val); - } -} - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators.h b/tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators.h deleted file mode 100644 index cc4ec8dc9e3..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators.h +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_LEAF_MODEL_OPERATORS_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_LEAF_MODEL_OPERATORS_H_ - -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_target.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/params.h" -#include "tensorflow/contrib/tensor_forest/proto/fertile_stats.pb.h" -#include "tensorflow/contrib/tensor_forest/proto/tensor_forest_params.pb.h" - -namespace tensorflow { -namespace tensorforest { - -// Abstract base class for classes that can initialize, get, and update leaf -// models. -class LeafModelOperator { - public: - // Number of outputs is interpreted differently for classification and - // regression. For classification, it's the number of possible classes. - // For regression, it's the target dimensions. - explicit LeafModelOperator(const TensorForestParams& params) - : params_(params) {} - virtual ~LeafModelOperator() {} - - // Returns the value of the requested output, which should be - // in [0, num_outputs_). For classification, it's the class count (weighted - // number of instances seen). For regression, it's e.g. the average value. - virtual float GetOutputValue(const decision_trees::Leaf& leaf, - int32 o) const = 0; - - // Update the given Leaf's model with the given example. - virtual void UpdateModel(decision_trees::Leaf* leaf, - const InputTarget* target, int example) const = 0; - - // Initialize an empty Leaf model. - virtual void InitModel(decision_trees::Leaf* leaf) const = 0; - - virtual void ExportModel(const LeafStat& stat, - decision_trees::Leaf* leaf) const = 0; - - protected: - const TensorForestParams& params_; -}; - -// LeafModelOperator that stores class counts in a dense vector. -class DenseClassificationLeafModelOperator : public LeafModelOperator { - public: - explicit DenseClassificationLeafModelOperator( - const TensorForestParams& params) - : LeafModelOperator(params) {} - float GetOutputValue(const decision_trees::Leaf& leaf, - int32 o) const override; - - void UpdateModel(decision_trees::Leaf* leaf, const InputTarget* target, - int example) const override; - - void InitModel(decision_trees::Leaf* leaf) const override; - - void ExportModel(const LeafStat& stat, - decision_trees::Leaf* leaf) const override; -}; - -// LeafModelOperator that stores class counts sparsely in a map. Assumes default -// value for yet-unseen classes is 0. -class SparseClassificationLeafModelOperator : public LeafModelOperator { - public: - explicit SparseClassificationLeafModelOperator( - const TensorForestParams& params) - : LeafModelOperator(params) {} - float GetOutputValue(const decision_trees::Leaf& leaf, - int32 o) const override; - - void UpdateModel(decision_trees::Leaf* leaf, const InputTarget* target, - int example) const override; - - void InitModel(decision_trees::Leaf* leaf) const override {} - - void ExportModel(const LeafStat& stat, - decision_trees::Leaf* leaf) const override; -}; - -class SparseOrDenseClassificationLeafModelOperator : public LeafModelOperator { - public: - explicit SparseOrDenseClassificationLeafModelOperator( - const TensorForestParams& params) - : LeafModelOperator(params), - dense_(new DenseClassificationLeafModelOperator(params)), - sparse_(new SparseClassificationLeafModelOperator(params)) {} - float GetOutputValue(const decision_trees::Leaf& leaf, - int32 o) const override; - - void UpdateModel(decision_trees::Leaf* leaf, const InputTarget* target, - int example) const override; - - void InitModel(decision_trees::Leaf* leaf) const override {} - - void ExportModel(const LeafStat& stat, - decision_trees::Leaf* leaf) const override; - - protected: - std::unique_ptr dense_; - std::unique_ptr sparse_; -}; - -// LeafModelOperator that stores regression leaf models with constant-value -// prediction. -class RegressionLeafModelOperator : public LeafModelOperator { - public: - explicit RegressionLeafModelOperator(const TensorForestParams& params) - : LeafModelOperator(params) {} - float GetOutputValue(const decision_trees::Leaf& leaf, - int32 o) const override; - - // TODO(gilberth): Quick experimentation suggests it's not even worth - // updating model and just using the seeded values. Can add this in - // with additional_data, though protobuf::Any is slow. Maybe make it - // optional. Maybe make any update optional. - void UpdateModel(decision_trees::Leaf* leaf, const InputTarget* target, - int example) const override {} - - void InitModel(decision_trees::Leaf* leaf) const override; - - void ExportModel(const LeafStat& stat, - decision_trees::Leaf* leaf) const override; -}; - -class LeafModelOperatorFactory { - public: - static std::unique_ptr CreateLeafModelOperator( - const TensorForestParams& params); -}; - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_LEAF_MODEL_OPERATORS_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators_test.cc b/tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators_test.cc deleted file mode 100644 index ab4191809b6..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators_test.cc +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators.h" -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/test_utils.h" -#include "tensorflow/contrib/tensor_forest/proto/tensor_forest_params.pb.h" -#include "tensorflow/core/platform/protobuf.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace { - -using tensorflow::decision_trees::Leaf; -using tensorflow::tensorforest::DenseClassificationLeafModelOperator; -using tensorflow::tensorforest::LeafModelOperator; -using tensorflow::tensorforest::LeafStat; -using tensorflow::tensorforest::RegressionLeafModelOperator; -using tensorflow::tensorforest::SparseClassificationLeafModelOperator; -using tensorflow::tensorforest::SparseOrDenseClassificationLeafModelOperator; -using tensorflow::tensorforest::TensorForestParams; -using tensorflow::tensorforest::TestableInputTarget; - -const int32 kNumClasses = 3; - -constexpr char kRegressionStatProto[] = - "weight_sum: 3 " - "regression { " - "mean_output { " - "value { " - " float_value: 27 " - "} " - "value { " - " float_value: 282 " - "} " - "value { " - " float_value: 10 " - "} " - "} " - "mean_output_squares { " - "value {" - " float_value: 245" - "}" - "value {" - " float_value: 26564" - "}" - "value {" - " float_value: 46" - "}" - "}" - "}"; - -void TestClassificationNormalUse(const std::unique_ptr& op) { - Leaf l; - op->InitModel(&l); - // Make sure it was initialized correctly. - for (int i = 0; i < kNumClasses; ++i) { - EXPECT_EQ(op->GetOutputValue(l, i), 0); - } - - std::vector labels = {1, 0, 1}; - std::vector weights = {2.3, 20.3, 1.1}; - std::unique_ptr target( - new TestableInputTarget(labels, weights, 1)); - - // Update and check value. - op->UpdateModel(&l, target.get(), 0); - op->UpdateModel(&l, target.get(), 1); - op->UpdateModel(&l, target.get(), 2); - - EXPECT_FLOAT_EQ(op->GetOutputValue(l, 1), 3.4); -} - -TEST(DenseLeafModelOperatorsTest, NormalUse) { - TensorForestParams params; - params.set_num_outputs(kNumClasses); - std::unique_ptr op( - new DenseClassificationLeafModelOperator(params)); - TestClassificationNormalUse(op); -} - -TEST(SparseLeafModelOperatorsTest, NormalUse) { - TensorForestParams params; - params.set_num_outputs(kNumClasses); - std::unique_ptr op( - new SparseClassificationLeafModelOperator(params)); - TestClassificationNormalUse(op); -} - -TEST(DenseLeafModelOperatorsTest, InitWithExisting) { - TensorForestParams params; - params.set_num_outputs(kNumClasses); - std::unique_ptr op( - new DenseClassificationLeafModelOperator(params)); - - std::unique_ptr stat(new LeafStat); - stat->mutable_classification() - ->mutable_dense_counts() - ->add_value() - ->set_float_value(1.1); - stat->mutable_classification() - ->mutable_dense_counts() - ->add_value() - ->set_float_value(2.2); - stat->mutable_classification() - ->mutable_dense_counts() - ->add_value() - ->set_float_value(3.3); - - std::unique_ptr leaf(new Leaf); - - op->ExportModel(*stat, leaf.get()); - - // Make sure it was initialized correctly. - EXPECT_EQ(leaf->vector().value_size(), kNumClasses); - EXPECT_FLOAT_EQ(op->GetOutputValue(*leaf, 0), 1.1); - EXPECT_FLOAT_EQ(op->GetOutputValue(*leaf, 1), 2.2); - EXPECT_FLOAT_EQ(op->GetOutputValue(*leaf, 2), 3.3); -} - -TEST(SparseOrDenseClassificationLeafModelOperator, InitWithExisting) { - TensorForestParams params; - params.set_num_outputs(kNumClasses); - std::unique_ptr op( - new SparseOrDenseClassificationLeafModelOperator(params)); - - std::unique_ptr stat(new LeafStat); - (*stat->mutable_classification() - ->mutable_sparse_counts() - ->mutable_sparse_value())[0] - .set_float_value(1.1); - (*stat->mutable_classification() - ->mutable_sparse_counts() - ->mutable_sparse_value())[1] - .set_float_value(2.2); - (*stat->mutable_classification() - ->mutable_sparse_counts() - ->mutable_sparse_value())[2] - .set_float_value(3.3); - - std::unique_ptr leaf(new Leaf); - - op->ExportModel(*stat, leaf.get()); - - // Make sure it was initialized correctly. - EXPECT_FLOAT_EQ(op->GetOutputValue(*leaf, 0), 1.1); - EXPECT_FLOAT_EQ(op->GetOutputValue(*leaf, 1), 2.2); - EXPECT_FLOAT_EQ(op->GetOutputValue(*leaf, 2), 3.3); -} - -TEST(SparseLeafModelOperatorsTest, InitWithExisting) { - TensorForestParams params; - params.set_num_outputs(kNumClasses); - std::unique_ptr op( - new SparseClassificationLeafModelOperator(params)); - std::unique_ptr stat(new LeafStat); - (*stat->mutable_classification() - ->mutable_sparse_counts() - ->mutable_sparse_value())[0] - .set_float_value(1.1); - (*stat->mutable_classification() - ->mutable_sparse_counts() - ->mutable_sparse_value())[1] - .set_float_value(2.2); - (*stat->mutable_classification() - ->mutable_sparse_counts() - ->mutable_sparse_value())[2] - .set_float_value(3.3); - - std::unique_ptr leaf(new Leaf); - - op->ExportModel(*stat, leaf.get()); - - // Make sure it was initialized correctly. - EXPECT_FLOAT_EQ(op->GetOutputValue(*leaf, 0), 1.1); - EXPECT_FLOAT_EQ(op->GetOutputValue(*leaf, 1), 2.2); - EXPECT_FLOAT_EQ(op->GetOutputValue(*leaf, 2), 3.3); - - // check default value. - EXPECT_FLOAT_EQ(op->GetOutputValue(*leaf, 100), 0); - EXPECT_EQ(leaf->sparse_vector().sparse_value().size(), kNumClasses); -} - -TEST(RegressionLeafModelOperatorsTest, NormalUse) { - TensorForestParams params; - params.set_num_outputs(kNumClasses); - std::unique_ptr op( - new RegressionLeafModelOperator(params)); - - std::unique_ptr stat(new LeafStat()); - const string contents(kRegressionStatProto); - ::tensorflow::protobuf::TextFormat::ParseFromString(contents, stat.get()); - - std::unique_ptr leaf(new Leaf); - op->ExportModel(*stat, leaf.get()); - - // Make sure it was initialized correctly. - EXPECT_FLOAT_EQ(op->GetOutputValue(*leaf, 0), 9); - EXPECT_FLOAT_EQ(op->GetOutputValue(*leaf, 1), 94); - EXPECT_FLOAT_EQ(op->GetOutputValue(*leaf, 2), 3.3333333); -} - -} // namespace -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/params.cc b/tensorflow/contrib/tensor_forest/kernels/v4/params.cc deleted file mode 100644 index 6387e7c2bc7..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/params.cc +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/params.h" -#include -#include -#include -#include "tensorflow/core/platform/logging.h" - -namespace tensorflow { -namespace tensorforest { - -float ResolveParam(const DepthDependentParam& param, int32 depth) { - float val; - switch (param.ParamType_case()) { - case DepthDependentParam::kConstantValue: - return param.constant_value(); - - case DepthDependentParam::kLinear: - val = depth * param.linear().slope() + param.linear().y_intercept(); - return std::min(std::max(val, param.linear().min_val()), - param.linear().max_val()); - - case DepthDependentParam::kExponential: - return param.exponential().bias() + - param.exponential().multiplier() * - static_cast( - std::pow(param.exponential().base(), - param.exponential().depth_multiplier() * depth)); - - case DepthDependentParam::kThreshold: - if (depth >= param.threshold().threshold()) { - return param.threshold().on_value(); - } else { - return param.threshold().off_value(); - } - - default: - LOG(FATAL) << "unknown parameter type"; - } -} - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/params.h b/tensorflow/contrib/tensor_forest/kernels/v4/params.h deleted file mode 100644 index 7583e3d0402..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/params.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_PARAMS_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_PARAMS_H_ - -#include "tensorflow/contrib/tensor_forest/proto/tensor_forest_params.pb.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace tensorforest { - -// Return the value of the given depth-dependent parameter given a leaf's depth. -float ResolveParam(const DepthDependentParam& param, int32 depth); - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_PARAMS_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/params_test.cc b/tensorflow/contrib/tensor_forest/kernels/v4/params_test.cc deleted file mode 100644 index 4010a71006d..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/params_test.cc +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/params.h" -#include "tensorflow/contrib/tensor_forest/proto/tensor_forest_params.pb.h" -#include "tensorflow/core/platform/test.h" - -namespace { - -using tensorflow::tensorforest::DepthDependentParam; -using tensorflow::tensorforest::ResolveParam; - -TEST(ParamsTest, TestConstant) { - DepthDependentParam param; - param.set_constant_value(10.0); - - ASSERT_EQ(ResolveParam(param, 0), 10.0); - ASSERT_EQ(ResolveParam(param, 100), 10.0); -} - -TEST(ParamsTest, TestLinear) { - DepthDependentParam param; - auto* linear = param.mutable_linear(); - linear->set_y_intercept(100.0); - linear->set_slope(-10.0); - linear->set_min_val(23.0); - linear->set_max_val(90.0); - - ASSERT_EQ(ResolveParam(param, 0), 90); - ASSERT_EQ(ResolveParam(param, 1), 90); - ASSERT_EQ(ResolveParam(param, 2), 80); - - ASSERT_EQ(ResolveParam(param, 30), 23); -} - -TEST(ParamsTest, TestExponential) { - DepthDependentParam param; - auto* expo = param.mutable_exponential(); - expo->set_bias(100.0); - expo->set_base(10.0); - expo->set_multiplier(-1.0); - expo->set_depth_multiplier(1.0); - - ASSERT_EQ(ResolveParam(param, 0), 99); - ASSERT_EQ(ResolveParam(param, 1), 90); - ASSERT_EQ(ResolveParam(param, 2), 0); -} - -TEST(ParamsTest, TestThreshold) { - DepthDependentParam param; - auto* threshold = param.mutable_threshold(); - threshold->set_on_value(100.0); - threshold->set_off_value(10.0); - threshold->set_threshold(5.0); - - ASSERT_EQ(ResolveParam(param, 0), 10); - ASSERT_EQ(ResolveParam(param, 4), 10); - ASSERT_EQ(ResolveParam(param, 5), 100); - ASSERT_EQ(ResolveParam(param, 6), 100); -} - -} // namespace diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/split_collection_operators.cc b/tensorflow/contrib/tensor_forest/kernels/v4/split_collection_operators.cc deleted file mode 100644 index b7b60d0ab8c..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/split_collection_operators.cc +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/split_collection_operators.h" - -#include - -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model_extensions.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/stat_utils.h" - -namespace tensorflow { -namespace tensorforest { - -std::unordered_map - SplitCollectionOperatorFactory::factories_; // NOLINT -REGISTER_SPLIT_COLLECTION(COLLECTION_BASIC, SplitCollectionOperator); - -std::unique_ptr -SplitCollectionOperatorFactory::CreateSplitCollectionOperator( - const TensorForestParams& params) { - auto it = factories_.find(params.collection_type()); - if (it == factories_.end()) { - LOG(ERROR) << "Unknown split collection operator: " - << params.collection_type(); - return nullptr; - } else { - return it->second->Create(params); - } -} - -std::unique_ptr SplitCollectionOperator::CreateGrowStats( - int32 node_id, int32 depth) const { - switch (params_.stats_type()) { - case STATS_DENSE_GINI: - return std::unique_ptr( - new DenseClassificationGrowStats(params_, depth)); - - case STATS_SPARSE_GINI: - return std::unique_ptr( - new SparseClassificationGrowStats(params_, depth)); - - case STATS_LEAST_SQUARES_REGRESSION: - return std::unique_ptr( - new LeastSquaresRegressionGrowStats(params_, depth)); - - case STATS_FIXED_SIZE_SPARSE_GINI: - return std::unique_ptr( - new FixedSizeSparseClassificationGrowStats(params_, depth)); - - default: - LOG(ERROR) << "Unknown grow stats type: " << params_.stats_type(); - return nullptr; - } -} - -void SplitCollectionOperator::ExtractFromProto( - const FertileStats& stats_proto) { - for (int i = 0; i < stats_proto.node_to_slot_size(); ++i) { - const auto& slot = stats_proto.node_to_slot(i); - stats_[slot.node_id()] = CreateGrowStats(slot.node_id(), slot.depth()); - stats_[slot.node_id()]->ExtractFromProto(slot); - } -} - -void SplitCollectionOperator::PackToProto(FertileStats* stats_proto) const { - for (const auto& pair : stats_) { - auto* new_slot = stats_proto->add_node_to_slot(); - new_slot->set_node_id(pair.first); - if (params_.checkpoint_stats()) { - pair.second->PackToProto(new_slot); - } - new_slot->set_depth(pair.second->depth()); - } -} - -void SplitCollectionOperator::InitializeSlot(int32 node_id, int32 depth) { - stats_[node_id] = std::unique_ptr(CreateGrowStats(node_id, depth)); - stats_[node_id]->Initialize(); -} - -void SplitCollectionOperator::AddExample( - const std::unique_ptr& input_data, const InputTarget* target, - const std::vector& examples, int32 node_id) const { - auto* slot = stats_.at(node_id).get(); - for (int example : examples) { - slot->AddExample(input_data, target, example); - } -} - -bool SplitCollectionOperator::IsInitialized(int32 node_id) const { - auto it = stats_.find(node_id); - if (it == stats_.end()) { - LOG(WARNING) << "IsInitialized called with unknown node_id = " << node_id; - return false; - } - return it->second->IsInitialized(); -} - -void SplitCollectionOperator::CreateAndInitializeCandidateWithExample( - const std::unique_ptr& input_data, const InputTarget* target, - int example, int32 node_id) const { - // Assumes split_initializations_per_input == 1. - decision_trees::BinaryNode split; - float bias; - int type; - decision_trees::FeatureId feature_id; - input_data->RandomSample(example, &feature_id, &bias, &type); - - if (type == kDataFloat) { - decision_trees::InequalityTest* test = - split.mutable_inequality_left_child_test(); - *test->mutable_feature_id() = feature_id; - test->mutable_threshold()->set_float_value(bias); - test->set_type(params_.inequality_test_type()); - } else if (type == kDataCategorical) { - decision_trees::MatchingValuesTest test; - *test.mutable_feature_id() = feature_id; - test.add_value()->set_float_value(bias); - split.mutable_custom_left_child_test()->PackFrom(test); - } else { - LOG(ERROR) << "Unknown feature type " << type << ", not sure which " - << "node type to use."; - } - stats_.at(node_id)->AddSplit(split, input_data, target, example); -} - -bool SplitCollectionOperator::BestSplit(int32 node_id, SplitCandidate* best, - int32* depth) const { - auto* slot = stats_.at(node_id).get(); - *depth = slot->depth(); - return slot->BestSplit(best); -} -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/split_collection_operators.h b/tensorflow/contrib/tensor_forest/kernels/v4/split_collection_operators.h deleted file mode 100644 index c606ff98c67..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/split_collection_operators.h +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_SPLIT_COLLECTION_OPERATORS_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_SPLIT_COLLECTION_OPERATORS_H_ - -#include -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_data.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_target.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/leaf_model_operators.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/params.h" -#include "tensorflow/contrib/tensor_forest/proto/fertile_stats.pb.h" -#include "tensorflow/contrib/tensor_forest/proto/tensor_forest_params.pb.h" - -namespace tensorflow { -namespace tensorforest { - -// Class that can initialize and update split collections, and -// report if one is finished and ready to split. Designed to be inherited -// from to implement techniques such as pruning and early/delayed finishing. -class SplitCollectionOperator { - public: - explicit SplitCollectionOperator(const TensorForestParams& params) - : params_(params) {} - virtual ~SplitCollectionOperator() {} - - // Return a new GrowStats object according to stats_type_; - virtual std::unique_ptr CreateGrowStats(int32 node_id, - int32 depth) const; - - // Initialize from a previously serialized proto. - virtual void ExtractFromProto(const FertileStats& stats); - - // Serialize contents to the given proto. - virtual void PackToProto(FertileStats* stats) const; - - // Updates the slot's candidates with the new example. - // Assumes slot has been initialized. - virtual void AddExample(const std::unique_ptr& input_data, - const InputTarget* target, - const std::vector& examples, - int32 node_id) const; - - // Create a new candidate and initialize it with the given example. - virtual void CreateAndInitializeCandidateWithExample( - const std::unique_ptr& input_data, - const InputTarget* target, int example, int32 node_id) const; - - // Create a new GrowStats for the given node id and initialize it. - virtual void InitializeSlot(int32 node_id, int32 depth); - - // Called when the resource is deserialized, possibly needing an - // initialization. - virtual void MaybeInitialize() { - if (stats_.empty()) { - InitializeSlot(0, 0); - } - } - - // Perform any necessary cleanup for any tracked state for the slot. - virtual void ClearSlot(int32 node_id) { stats_.erase(node_id); } - - // Return true if slot is fully initialized. - virtual bool IsInitialized(int32 node_id) const; - - // Return true if slot is finished. - virtual bool IsFinished(int32 node_id) const { - return stats_.at(node_id)->IsFinished(); - } - - // Fill in best with the best split that node_id has, return true if this - // was successful, false if no good split was found. - virtual bool BestSplit(int32 node_id, SplitCandidate* best, - int32* depth) const; - - protected: - const TensorForestParams& params_; - std::unordered_map> stats_; -}; - -class CollectionCreator { - public: - virtual std::unique_ptr Create( - const TensorForestParams& params) = 0; - virtual ~CollectionCreator() {} -}; - -class SplitCollectionOperatorFactory { - public: - static std::unique_ptr CreateSplitCollectionOperator( - const TensorForestParams& params); - - static std::unordered_map factories_; -}; - -template -class AnyCollectionCreator : public CollectionCreator { - public: - AnyCollectionCreator(SplitCollectionType type) { - SplitCollectionOperatorFactory::factories_[type] = this; - } - virtual std::unique_ptr Create( - const TensorForestParams& params) { - return std::unique_ptr(new T(params)); - } -}; - -#define REGISTER_SPLIT_COLLECTION(name, cls) \ - namespace { \ - AnyCollectionCreator creator(name); \ - } - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_SPLIT_COLLECTION_OPERATORS_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/stat_utils.cc b/tensorflow/contrib/tensor_forest/kernels/v4/stat_utils.cc deleted file mode 100644 index c749fbe69e1..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/stat_utils.cc +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/contrib/tensor_forest/kernels/v4/stat_utils.h" -#include - -#include "tensorflow/contrib/decision_trees/proto/generic_tree_model.pb.h" - -namespace tensorflow { -namespace tensorforest { - -// When using smoothing but only tracking sum and squares, and we're adding -// num_classes for smoothing each class, then Gini looks more like this: -// Gini = 1 - \sum_i (c_i + 1)^2 / C^2 -// = 1 - (1 / C^2) ( (\sum_i c_i)^2 + 2 (\sum_i c_i) + (\sum_i 1)) -// = 1 - (1 / C^2) ( stats.square() + 2 stats.sum() + #_classes) -// = 1 - ( stats.square() + 2 stats.sum() + #_classes) / (smoothed_sum * -// smoothed_sum) -// -// where -// smoothed_sum = stats.sum() + #_classes -float GiniImpurity(const LeafStat& stats, int32 num_classes) { - const float smoothed_sum = num_classes + stats.weight_sum(); - return 1.0 - ((stats.classification().gini().square() + - 2 * stats.weight_sum() + num_classes) / - (smoothed_sum * smoothed_sum)); -} - -float WeightedGiniImpurity(const LeafStat& stats, int32 num_classes) { - return stats.weight_sum() * GiniImpurity(stats, num_classes); -} - -void UpdateGini(LeafStat* stats, float old_val, float weight) { - stats->set_weight_sum(stats->weight_sum() + weight); - // Equivalent to stats->square() - old_val * old_val + new_val * new_val, - // (for new_val = old_val + weight), but more numerically stable. - stats->mutable_classification()->mutable_gini()->set_square( - stats->classification().gini().square() + weight * weight + - 2 * old_val * weight); -} - -float Variance(const LeafStat& stats, int output) { - if (stats.weight_sum() == 0) { - return 0; - } - const float e_x = - stats.regression().mean_output().value(output).float_value() / - stats.weight_sum(); - const auto e_x2 = - stats.regression().mean_output_squares().value(output).float_value() / - stats.weight_sum(); - return e_x2 - e_x * e_x; -} - -float TotalVariance(const LeafStat& stats) { - float sum = 0; - for (int i = 0; i < stats.regression().mean_output().value_size(); ++i) { - sum += Variance(stats, i); - } - return sum; -} - -float SmoothedGini(float sum, float square, int num_classes) { - // See comments for GiniImpurity above. - const float smoothed_sum = num_classes + sum; - return 1.0 - (square + 2 * sum + num_classes) / (smoothed_sum * smoothed_sum); -} - -float WeightedSmoothedGini(float sum, float square, int num_classes) { - return sum * SmoothedGini(sum, square, num_classes); -} - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/stat_utils.h b/tensorflow/contrib/tensor_forest/kernels/v4/stat_utils.h deleted file mode 100644 index e6140065bbf..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/stat_utils.h +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_STAT_UTILS_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_STAT_UTILS_H_ -#include "tensorflow/contrib/tensor_forest/proto/fertile_stats.pb.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace tensorforest { - -// Returns the smoothed, unweighted Gini impurity. -float GiniImpurity(const LeafStat& stats, int32 num_classes); - -// Returns the smoothed, weighted Gini impurity -float WeightedGiniImpurity(const LeafStat& stats, int32 num_classes); - -// Updates the GiniStats given the old and new values of a class count that -// was updated. -void UpdateGini(LeafStat* stats, float old_val, float weight); - -// Returns the variance in stats for the given output. -float Variance(const LeafStat& stats, int output); - -// Returns the variance sum for all outputs. -float TotalVariance(const LeafStat& stats); - -// ------- functions used by C++ stats classes -------- // -// Returns the smoothed gini score given the sum and sum of the squares of the -// class counts. -float SmoothedGini(float sum, float square, int num_classes); - -// Returns the smoothed gini score weighted by the sum. -float WeightedSmoothedGini(float sum, float square, int num_classes); - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_STAT_UTILS_H_ diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/test_utils.h b/tensorflow/contrib/tensor_forest/kernels/v4/test_utils.h deleted file mode 100644 index 38deb3e3cd8..00000000000 --- a/tensorflow/contrib/tensor_forest/kernels/v4/test_utils.h +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#ifndef TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_TEST_UTILS_H_ -#define TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_TEST_UTILS_H_ -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_data.h" -#include "tensorflow/contrib/tensor_forest/kernels/v4/input_target.h" - -namespace tensorflow { -namespace tensorforest { - -class TestableInputTarget : public StoredInputTarget> { - public: - TestableInputTarget(const std::vector& t, const std::vector& w, - int num_t) - : StoredInputTarget(new std::vector(t), new std::vector(w), - num_t) {} - - int NumItems() const { return target_->size(); } - - int32 GetTargetAsClassIndex(int example_index, - int target_index) const override { - return static_cast( - GetTargetAsContinuous(example_index, target_index)); - } - - float GetTargetWeight(int example_index) const override { - const size_t num_weights = weight_->size(); - return num_weights > 0 && example_index < num_weights - ? (*weight_)[example_index] - : 1.0; - } - - float GetTargetAsContinuous(int example_index, - int target_index) const override { - QCHECK_LT(target_index, num_targets_); - return (*target_)[example_index * num_targets_ + target_index]; - } -}; - -class TestableDataSet : public TensorDataSet { - public: - TestableDataSet(const std::vector& data, int num_features) - : TensorDataSet(TensorForestDataSpec(), 11), - num_features_(num_features), - data_(data) {} - - float GetExampleValue(int example, int32 feature_id) const override { - return data_[example * num_features_ + feature_id]; - } - - protected: - int num_features_; - std::vector data_; -}; - -} // namespace tensorforest -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_TENSOR_FOREST_KERNELS_V4_TEST_UTILS_H_ diff --git a/tensorflow/contrib/tensor_forest/ops/model_ops.cc b/tensorflow/contrib/tensor_forest/ops/model_ops.cc deleted file mode 100644 index 98124d519c7..00000000000 --- a/tensorflow/contrib/tensor_forest/ops/model_ops.cc +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { -using shape_inference::DimensionHandle; -using shape_inference::InferenceContext; - -namespace tensorforest { - -REGISTER_RESOURCE_HANDLE_OP(DecisionTreeResource); - -REGISTER_OP("TreeIsInitializedOp") - .Input("tree_handle: resource") - .Output("is_initialized: bool") - .SetShapeFn(tensorflow::shape_inference::ScalarShape) - .Doc(R"doc( -Checks whether a tree has been initialized. -)doc"); - -REGISTER_OP("CreateTreeVariable") - .Attr("params: string") - .Input("tree_handle: resource") - .Input("tree_config: string") - .SetShapeFn(tensorflow::shape_inference::NoOutputs) - .Doc(R"doc( -Creates a tree model and returns a handle to it. - -params: A serialized TensorForestParams proto. -tree_handle: handle to the tree resource to be created. -tree_config: Serialized proto of the tree. -)doc"); - -REGISTER_OP("TreeSerialize") - .Input("tree_handle: resource") - .Output("tree_config: string") - .SetShapeFn(tensorflow::shape_inference::ScalarShape) - .Doc(R"doc( -Serializes the tree to a proto. - -tree_handle: The handle to the tree. -tree_config: Serialized proto of the tree. -)doc"); - -REGISTER_OP("TreeDeserialize") - .Attr("params: string") - .Input("tree_handle: resource") - .Input("tree_config: string") - .SetShapeFn(tensorflow::shape_inference::NoOutputs) - .Doc(R"doc( -Deserializes a serialized tree config and replaces current tree. - -params: A serialized TensorForestParams proto. -tree_handle: The handle to the tree . -tree_config: Serialized proto of the . -)doc"); - -REGISTER_OP("TreeSize") - .Input("tree_handle: resource") - .Output("tree_size: int32") - .SetShapeFn(tensorflow::shape_inference::ScalarShape) - .Doc(R"doc( -Outputs the size of the tree, including leaves. - -tree_handle: The handle to the tree. -tree_size: Size scalar. -)doc"); - -REGISTER_OP("TreePredictionsV4") - .Attr("input_spec: string") - .Attr("params: string") - .Input("tree_handle: resource") - .Input("input_data: float") - .Input("sparse_input_indices: int64") - .Input("sparse_input_values: float") - .Input("sparse_input_shape: int64") - .Output("predictions: float") - .Output("tree_paths: string") - .SetShapeFn([](InferenceContext* c) { - DimensionHandle num_points = c->UnknownDim(); - - if (c->RankKnown(c->input(1)) && c->Rank(c->input(1)) > 0 && - c->Value(c->Dim(c->input(1), 0)) > 0) { - num_points = c->Dim(c->input(1), 0); - } - - c->set_output(0, c->Matrix(num_points, c->UnknownDim())); - c->set_output(1, c->Vector(c->UnknownDim())); - return Status::OK(); - }) - .Doc(R"doc( -Outputs the predictions for the given input data. - -params: A serialized TensorForestParams proto. -tree_handle: The handle to the tree. -input_data: The training batch's features as a 2-d tensor; `input_data[i][j]` - gives the j-th feature of the i-th input. -sparse_input_indices: The indices tensor from the SparseTensor input. -sparse_input_values: The values tensor from the SparseTensor input. -sparse_input_shape: The shape tensor from the SparseTensor input. -predictions: `predictions[i][j]` is the probability that input i is class j. -tree_paths: `tree_paths[i]` is a serialized TreePath proto for example i. -)doc"); - -REGISTER_OP("TraverseTreeV4") - .Attr("input_spec: string") - .Attr("params: string") - .Input("tree_handle: resource") - .Input("input_data: float") - .Input("sparse_input_indices: int64") - .Input("sparse_input_values: float") - .Input("sparse_input_shape: int64") - .Output("leaf_ids: int32") - .SetShapeFn([](InferenceContext* c) { - DimensionHandle num_points = c->UnknownDim(); - - if (c->RankKnown(c->input(1)) && c->Rank(c->input(1)) > 0 && - c->Value(c->Dim(c->input(1), 0)) > 0) { - num_points = c->Dim(c->input(1), 0); - } - - c->set_output(0, c->Vector(num_points)); - return Status::OK(); - }) - .Doc(R"doc( -Outputs the leaf ids for the given input data. - -params: A serialized TensorForestParams proto. -tree_handle: The handle to the tree. -input_data: The training batch's features as a 2-d tensor; `input_data[i][j]` - gives the j-th feature of the i-th input. -sparse_input_indices: The indices tensor from the SparseTensor input. -sparse_input_values: The values tensor from the SparseTensor input. -sparse_input_shape: The shape tensor from the SparseTensor input. -leaf_ids: `leaf_ids[i]` is the leaf id for input i. -)doc"); - -REGISTER_OP("UpdateModelV4") - .Attr("params: string") - .Input("tree_handle: resource") - .Input("leaf_ids: int32") - .Input("input_labels: float") - .Input("input_weights: float") - .SetShapeFn(tensorflow::shape_inference::NoOutputs) - .Doc(R"doc( -Updates the given leaves for each example with the new labels. - -params: A serialized TensorForestParams proto. -tree_handle: The handle to the tree. -leaf_ids: `leaf_ids[i]` is the leaf id for input i. -input_labels: The training batch's labels as a 1 or 2-d tensor. - 'input_labels[i][j]' gives the j-th label/target for the i-th input. -input_weights: The training batch's weights as a 1-d tensor. - 'input_weights[i]' gives the weight for the i-th input. -)doc"); - -REGISTER_OP("FeatureUsageCounts") - .Attr("params: string") - .Input("tree_handle: resource") - .Output("feature_counts: int32") - .SetShapeFn([](InferenceContext* c) { - c->set_output(0, c->Vector(c->UnknownDim())); - return Status::OK(); - }) - .Doc(R"doc( -Outputs the number of times each feature was used in a split. - -params: A serialized TensorForestParams proto. -tree_handle: The handle to the tree. -feature_counts: `feature_counts[i]` is the number of times feature i was used - in a split. -)doc"); - -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/ops/stats_ops.cc b/tensorflow/contrib/tensor_forest/ops/stats_ops.cc deleted file mode 100644 index 5be581aaec4..00000000000 --- a/tensorflow/contrib/tensor_forest/ops/stats_ops.cc +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2017 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { -using shape_inference::InferenceContext; - -namespace tensorforest { - -REGISTER_RESOURCE_HANDLE_OP(FertileStatsResource); - -REGISTER_OP("FertileStatsIsInitializedOp") - .Input("stats_handle: resource") - .Output("is_initialized: bool") - .SetShapeFn(tensorflow::shape_inference::ScalarShape) - .Doc(R"doc( -Checks whether a stats has been initialized. -)doc"); - -REGISTER_OP("CreateFertileStatsVariable") - .Attr("params: string") - .Input("stats_handle: resource") - .Input("stats_config: string") - .SetShapeFn(tensorflow::shape_inference::NoOutputs) - .Doc(R"doc( -Creates a stats model and returns a handle to it. - -params: A serialized TensorForestParams proto. -stats_handle: handle to the stats resource to be created. -stats_config: Serialized proto of the stats. -)doc"); - -REGISTER_OP("FertileStatsSerialize") - .Attr("params: string") - .Input("stats_handle: resource") - .Output("stats_config: string") - .SetShapeFn(tensorflow::shape_inference::ScalarShape) - .Doc(R"doc( -Serializes the stats to a proto. - -params: A serialized TensorForestParams proto. -stats_handle: The handle to the stats. -stats_config: Serialized proto of the stats. -)doc"); - -REGISTER_OP("FertileStatsDeserialize") - .Attr("params: string") - .Input("stats_handle: resource") - .Input("stats_config: string") - .SetShapeFn(tensorflow::shape_inference::NoOutputs) - .Doc(R"doc( -Deserializes a serialized stats config and replaces current stats. - -params: A serialized TensorForestParams proto. -stats_handle: The handle to the stats. -stats_config: Serialized proto of the stats. -)doc"); - -REGISTER_OP("GrowTreeV4") - .Attr("params: string") - .Input("tree_handle: resource") - .Input("stats_handle: resource") - .Input("finished_nodes: int32") - .SetShapeFn(tensorflow::shape_inference::NoOutputs) - .Doc(R"doc( -Grows the tree for finished nodes and allocates waiting nodes. - -params: A serialized TensorForestParams proto. -tree_handle: The handle to the tree. -stats_handle: The handle to the stats. -finished_nodes: A 1-d Tensor of finished node ids from ProcessInput. -)doc"); - -REGISTER_OP("ProcessInputV4") - .Attr("random_seed: int") - .Attr("input_spec: string") - .Attr("params: string") - .Input("tree_handle: resource") - .Input("stats_handle: resource") - .Input("input_data: float") - .Input("sparse_input_indices: int64") - .Input("sparse_input_values: float") - .Input("sparse_input_shape: int64") - .Input("input_labels: float") - .Input("input_weights: float") - .Input("leaf_ids: int32") - .Output("finished_nodes: int32") - .SetShapeFn([](InferenceContext* c) { - c->set_output(0, c->Vector(c->UnknownDim())); - return Status::OK(); - }) - .Doc(R"doc( -Add labels to stats after traversing the tree for each example. - -Outputs node ids that are finished. - -params: A serialized TensorForestParams proto. -tree_handle: The handle to the tree. -stats_handle: The handle to the stats. -input_data: The training batch's features as a 2-d tensor; `input_data[i][j]` - gives the j-th feature of the i-th input. -sparse_input_indices: The indices tensor from the SparseTensor input. -sparse_input_values: The values tensor from the SparseTensor input. -sparse_input_shape: The shape tensor from the SparseTensor input. -input_labels: The training batch's labels as a 1 or 2-d tensor. - 'input_labels[i][j]' gives the j-th label/target for the i-th input. -input_weights: The training batch's weights as a 1-d tensor. - 'input_weights[i]' gives the weight for the i-th input. -finished_nodes: A 1-d tensor of node ids that have finished and are ready to - grow. -leaf_ids: `leaf_ids[i]` is the leaf id for input i. -)doc"); - -REGISTER_OP("FinalizeTree") - .Attr("params: string") - .Input("tree_handle: resource") - .Input("stats_handle: resource") - .SetShapeFn([](InferenceContext* c) { return Status::OK(); }) - .Doc(R"doc( -Puts the Leaf models inside the tree into their final form. - -If drop_final_class is true, the per-class probability prediction of the -last class is not stored in the leaf models. - -params: A serialized TensorForestParams proto. -tree_handle: The handle to the tree. -stats_handle: The handle to the stats. -)doc"); -} // namespace tensorforest -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/ops/tensor_forest_ops.cc b/tensorflow/contrib/tensor_forest/ops/tensor_forest_ops.cc deleted file mode 100644 index 7974e21ee63..00000000000 --- a/tensorflow/contrib/tensor_forest/ops/tensor_forest_ops.cc +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { - -using shape_inference::InferenceContext; - -REGISTER_OP("ScatterAddNdim") - .Input("input: Ref(float)") - .Input("indices: int32") - .Input("deltas: float") - .SetShapeFn([](InferenceContext* c) { return Status::OK(); }) - .Doc(R"doc( - Add elements in deltas to mutable input according to indices. - - input: A N-dimensional float tensor to mutate. - indices:= A 2-D int32 tensor. The size of dimension 0 is the number of - deltas, the size of dimension 1 is the rank of the input. `indices[i]` - gives the coordinates of input that `deltas[i]` should add to. If - `indices[i]` does not fully specify a location (it has less indices than - there are dimensions in `input`), it is assumed that they are start - indices and that deltas contains enough values to fill in the remaining - input dimensions. - deltas: `deltas[i]` is the value to add to input at index indices[i][:] -)doc"); - -REGISTER_OP("ReinterpretStringToFloat") - .Input("input_data: string") - .Output("output_data: float") - .SetShapeFn(shape_inference::UnchangedShape) - .Doc(R"doc( - Converts byte arrays represented by strings to 32-bit - floating point numbers. The output numbers themselves are meaningless, and - should only be used in == comparisons. - - input_data: A batch of string features as a 2-d tensor; `input_data[i][j]` - gives the j-th feature of the i-th input. - output_data: A tensor of the same shape as input_data but the values are - float32. - -)doc"); - -} // namespace tensorflow diff --git a/tensorflow/contrib/tensor_forest/proto/BUILD b/tensorflow/contrib/tensor_forest/proto/BUILD deleted file mode 100644 index 702dbed7fc0..00000000000 --- a/tensorflow/contrib/tensor_forest/proto/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -load("//tensorflow/core/platform:default/build_config.bzl", "tf_proto_library") - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_proto_library( - name = "fertile_stats_proto", - srcs = ["fertile_stats.proto"], - cc_api_version = 2, - protodeps = ["//tensorflow/contrib/decision_trees/proto:generic_tree_model"], - visibility = ["//visibility:public"], -) - -tf_proto_library( - name = "tensor_forest_params_proto", - srcs = ["tensor_forest_params.proto"], - cc_api_version = 2, - protodeps = ["//tensorflow/contrib/decision_trees/proto:generic_tree_model"], - visibility = ["//visibility:public"], -) diff --git a/tensorflow/contrib/tensor_forest/proto/fertile_stats.proto b/tensorflow/contrib/tensor_forest/proto/fertile_stats.proto deleted file mode 100644 index d568fa3081c..00000000000 --- a/tensorflow/contrib/tensor_forest/proto/fertile_stats.proto +++ /dev/null @@ -1,99 +0,0 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package tensorflow.tensorforest; - -import "tensorflow/contrib/decision_trees/proto/generic_tree_model.proto"; - - -message FertileStats { - // Tracks stats for each node. node_to_slot[i] is the FertileSlot for node i. - // This may be sized to max_nodes initially, or grow dynamically as needed. - repeated FertileSlot node_to_slot = 1; -} - - -message GiniStats { - // This allows us to quickly track and calculate impurity (classification) - // by storing the sum of input weights and the sum of the squares of the - // input weights. Weighted gini is then: 1 - (square / sum * sum). - // Updates to these numbers are: - // old_i = leaf->value(label) - // new_i = old_i + incoming_weight - // sum -> sum + incoming_weight - // square -> square - (old_i ^ 2) + (new_i ^ 2) - // total_left_sum -> total_left_sum - old_left_i * old_total_i + - // new_left_i * new_total_i - float square = 2; -} - -message LeafStat { - // The sum of the weights of the training examples that we have seen. - // This is here, outside of the leaf_stat oneof, because almost all - // types will want it. - float weight_sum = 3; - - // TODO(thomaswc): Move the GiniStats out of LeafStats and into something - // that only tracks them for splits. - message GiniImpurityClassificationStats { - oneof counts { - decision_trees.Vector dense_counts = 1; - decision_trees.SparseVector sparse_counts = 2; - } - GiniStats gini = 3; - } - - // This is the info needed for calculating variance for regression. - // Variance will still have to be summed over every output, but the - // number of outputs in regression problems is almost always 1. - message LeastSquaresRegressionStats { - decision_trees.Vector mean_output = 1; - decision_trees.Vector mean_output_squares = 2; - } - - oneof leaf_stat { - GiniImpurityClassificationStats classification = 1; - LeastSquaresRegressionStats regression = 2; - // TODO(thomaswc): Add in v5's SparseClassStats. - } -} - -message FertileSlot { - // The statistics for *all* the examples seen at this leaf. - LeafStat leaf_stats = 4; - - repeated SplitCandidate candidates = 1; - - // The statistics for the examples seen at this leaf after all the - // splits have been initialized. If post_init_leaf_stats.weight_sum - // is > 0, then all candidates have been initialized. We need to track - // both leaf_stats and post_init_leaf_stats because the first is used - // to create the decision_tree::Leaf and the second is used to infer - // the statistics for the right side of a split (given the leaf side - // stats). - LeafStat post_init_leaf_stats = 6; - - int32 node_id = 5; - int32 depth = 7; -} - -message SplitCandidate { - // proto representing the potential node. - decision_trees.BinaryNode split = 1; - - // Right counts are inferred from FertileSlot.leaf_stats and left. - LeafStat left_stats = 4; - - // Right stats (not full counts) are kept here. - LeafStat right_stats = 5; - - // Fields used when training with a graph runner. - string unique_id = 6; -} - -// Proto used for tracking tree paths during inference time. -message TreePath { - // Nodes are listed in order that they were traversed. i.e. nodes_visited[0] - // is the tree's root node. - repeated decision_trees.TreeNode nodes_visited = 1; -} diff --git a/tensorflow/contrib/tensor_forest/proto/tensor_forest_params.proto b/tensorflow/contrib/tensor_forest/proto/tensor_forest_params.proto deleted file mode 100644 index 4545a8a6755..00000000000 --- a/tensorflow/contrib/tensor_forest/proto/tensor_forest_params.proto +++ /dev/null @@ -1,154 +0,0 @@ -syntax = "proto3"; - -package tensorflow.tensorforest; - -import "tensorflow/contrib/decision_trees/proto/generic_tree_model.proto"; - -// Leaf models specify what is returned at inference time, and how it is -// stored in the decision_trees.Leaf protos. -enum LeafModelType { - MODEL_DENSE_CLASSIFICATION = 0; - MODEL_SPARSE_CLASSIFICATION = 1; - MODEL_REGRESSION = 2; - MODEL_SPARSE_OR_DENSE_CLASSIFICATION = 3; -} - -// Stats models generally specify information that is collected which is -// necessary to choose a split at a node. Specifically, they operate on -// a SplitCandidate::LeafStat proto. -enum StatsModelType { - STATS_DENSE_GINI = 0; - STATS_SPARSE_GINI = 1; - STATS_LEAST_SQUARES_REGRESSION = 2; - // STATS_SPARSE_THEN_DENSE_GINI is deprecated and no longer supported. - STATS_SPARSE_THEN_DENSE_GINI = 3; - STATS_FIXED_SIZE_SPARSE_GINI = 4; -} - -// Allows selection of operations on the collection of split candidates. -// Basic infers right split stats from the leaf stats and each candidate's -// left stats. -enum SplitCollectionType { - COLLECTION_BASIC = 0; - GRAPH_RUNNER_COLLECTION = 1; -} - -// Pruning strategies define how candidates are pruned over time. -// SPLIT_PRUNE_HALF prunes the worst half of splits every prune_ever_samples, -// etc. Note that prune_every_samples plays against the depth-dependent -// split_after_samples, so they should be set together. -enum SplitPruningStrategyType { - SPLIT_PRUNE_NONE = 0; - SPLIT_PRUNE_HALF = 1; - SPLIT_PRUNE_QUARTER = 2; - SPLIT_PRUNE_10_PERCENT = 3; - // SPLIT_PRUNE_HOEFFDING prunes splits whose Gini impurity is worst than - // the best split's by more than the Hoeffding bound. - SPLIT_PRUNE_HOEFFDING = 4; -} - -message SplitPruningConfig { - DepthDependentParam prune_every_samples = 1; - SplitPruningStrategyType type = 2; -} - -// Finish strategies define when slots are considered finished. -// Basic requires at least split_after_samples, and doesn't allow slots to -// finish until the leaf has received more than one class. Hoeffding splits -// early after min_split_samples if one split is dominating the rest according -// to hoeffding bounds. Bootstrap does the same but compares gini's calculated -// with sampled smoothed counts. -enum SplitFinishStrategyType { - SPLIT_FINISH_BASIC = 0; - SPLIT_FINISH_DOMINATE_HOEFFDING = 2; - SPLIT_FINISH_DOMINATE_BOOTSTRAP = 3; -} - -message SplitFinishConfig { - // Configure how often we check for finish, because some finish methods - // are expensive to perform. - DepthDependentParam check_every_steps = 1; - SplitFinishStrategyType type = 2; -} - -// A parameter that changes linearly with depth, with upper and lower bounds. -message LinearParam { - float slope = 1; - float y_intercept = 2; - float min_val = 3; - float max_val = 4; -} - -// A parameter that changes expoentially with the form -// f = c + mb^(k*d) -// where: -// c: constant bias -// b: base -// m: multiplier -// k: depth multiplier -// d: depth -message ExponentialParam { - float bias = 1; - float base = 2; - float multiplier = 3; - float depth_multiplier = 4; -} - -// A parameter that is 'off' until depth >= a threshold, then is 'on'. -message ThresholdParam { - float on_value = 1; - float off_value = 2; - float threshold = 3; -} - -// A parameter that may change with node depth. -message DepthDependentParam { - oneof ParamType { - float constant_value = 1; - LinearParam linear = 2; - ExponentialParam exponential = 3; - ThresholdParam threshold = 4; - } -} - -message TensorForestParams { - // ------------ Types that control training subsystems ------ // - LeafModelType leaf_type = 1; - StatsModelType stats_type = 2; - SplitCollectionType collection_type = 3; - SplitPruningConfig pruning_type = 4; - SplitFinishConfig finish_type = 5; - - // --------- Parameters that can't change by definition --------------- // - int32 num_trees = 6; - int32 max_nodes = 7; - int32 num_features = 21; - - decision_trees.InequalityTest.Type inequality_test_type = 19; - - // Some booleans controlling execution - bool is_regression = 8; - bool drop_final_class = 9; - bool collate_examples = 10; - bool checkpoint_stats = 11; - bool use_running_stats_method = 20; - bool initialize_average_splits = 22; - bool inference_tree_paths = 23; - - // Number of classes (classification) or targets (regression) - int32 num_outputs = 12; - - // --------- Parameters that could be depth-dependent --------------- // - DepthDependentParam num_splits_to_consider = 13; - DepthDependentParam split_after_samples = 14; - DepthDependentParam dominate_fraction = 15; - DepthDependentParam min_split_samples = 18; - - // --------- Parameters for experimental features ---------------------- // - string graph_dir = 16; - int32 num_select_features = 17; - - // When using a FixedSizeSparseClassificationGrowStats, keep track of - // this many classes. - int32 num_classes_to_track = 24; -} diff --git a/tensorflow/contrib/tensor_forest/python/__init__.py b/tensorflow/contrib/tensor_forest/python/__init__.py deleted file mode 100644 index 0688f7c8168..00000000000 --- a/tensorflow/contrib/tensor_forest/python/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Random forest implementation in tensorflow.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.python import tensor_forest -from tensorflow.contrib.tensor_forest.python.ops import data_ops -from tensorflow.contrib.tensor_forest.python.ops import model_ops -from tensorflow.contrib.tensor_forest.python.ops import stats_ops -from tensorflow.contrib.tensor_forest.python.ops import tensor_forest_ops diff --git a/tensorflow/contrib/tensor_forest/python/kernel_tests/scatter_add_ndim_op_test.py b/tensorflow/contrib/tensor_forest/python/kernel_tests/scatter_add_ndim_op_test.py deleted file mode 100644 index 0b02bdcb50c..00000000000 --- a/tensorflow/contrib/tensor_forest/python/kernel_tests/scatter_add_ndim_op_test.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.contrib.tensor_forest.ops.scatter_add_ndim_op.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.python.ops import tensor_forest_ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest - - -class ScatterAddNdimTest(test_util.TensorFlowTestCase): - - def test1dim(self): - input_data = variables.VariableV1( - [1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12.]) - indices = [[1], [10]] - updates = [100., 200.] - - with self.cached_session(): - variables.global_variables_initializer().run() - tensor_forest_ops.scatter_add_ndim(input_data, indices, updates).run() - self.assertAllEqual( - [1., 102., 3., 4., 5., 6., 7., 8., 9., 10., 211., 12.], - input_data.eval()) - - def test3dim(self): - input_data = variables.VariableV1([[[1., 2., 3.], [4., 5., 6.]], - [[7., 8., 9.], [10., 11., 12.]]]) - indices = [[0, 0, 1], [1, 1, 2]] - updates = [100., 200.] - - with self.cached_session(): - variables.global_variables_initializer().run() - tensor_forest_ops.scatter_add_ndim(input_data, indices, updates).run() - self.assertAllEqual([[[1., 102., 3.], [4., 5., 6.]], - [[7., 8., 9.], [10., 11., 212.]]], input_data.eval()) - - def testNoUpdates(self): - init_val = [[[1., 2., 3.], [4., 5., 6.]], [[7., 8., 9.], [10., 11., 12.]]] - input_data = variables.VariableV1(init_val) - indices = [] - updates = [] - - with self.cached_session(): - variables.global_variables_initializer().run() - tensor_forest_ops.scatter_add_ndim(input_data, indices, updates).run() - self.assertAllEqual(init_val, input_data.eval()) - - def testBadInput(self): - init_val = [[[1., 2., 3.], [4., 5., 6.]], [[7., 8., 9.], [10., 11., 12.]]] - input_data = variables.VariableV1(init_val) - indices = [[0, 0, 1], [1, 1, 2]] - updates = [100.] - with self.cached_session(): - variables.global_variables_initializer().run() - with self.assertRaisesOpError( - 'Number of updates should be same as number of indices.'): - tensor_forest_ops.scatter_add_ndim(input_data, indices, updates).run() - self.assertAllEqual(init_val, input_data.eval()) - - def testIncompleteIndices(self): - input_data = variables.VariableV1([[[1., 2., 3.], [4., 5., 6.]], - [[7., 8., 9.], [10., 11., 12.]]]) - indices = [[0, 0], [1, 1]] - updates = [[100., 200., 300.], [400., 500., 600.]] - - with self.cached_session(): - variables.global_variables_initializer().run() - tensor_forest_ops.scatter_add_ndim(input_data, indices, updates).run() - self.assertAllEqual([[[101., 202., 303.], [4., 5., 6.]], - [[7., 8., 9.], [410., 511., 612.]]], - input_data.eval()) - - -if __name__ == '__main__': - googletest.main() diff --git a/tensorflow/contrib/tensor_forest/python/ops/data_ops.py b/tensorflow/contrib/tensor_forest/python/ops/data_ops.py deleted file mode 100644 index 5c1fe23981d..00000000000 --- a/tensorflow/contrib/tensor_forest/python/ops/data_ops.py +++ /dev/null @@ -1,214 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Ops for preprocessing data.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensor_forest.python.ops import tensor_forest_ops - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.platform import tf_logging as logging - -# Data column types for indicating categorical or other non-float values. -DATA_FLOAT = 0 -DATA_CATEGORICAL = 1 - -DTYPE_TO_FTYPE = { - dtypes.string: DATA_CATEGORICAL, - dtypes.int32: DATA_CATEGORICAL, - dtypes.int64: DATA_CATEGORICAL, - dtypes.float32: DATA_FLOAT, - dtypes.float64: DATA_FLOAT -} - - -def CastToFloat(tensor): - if tensor.dtype == dtypes.string: - return tensor_forest_ops.reinterpret_string_to_float(tensor) - elif tensor.dtype.is_integer: - return math_ops.cast(tensor, dtypes.float32) - else: - return tensor - - -# TODO(gilberth): If protos are ever allowed in dynamically loaded custom -# op libraries, convert this to a proto like a sane person. -class TensorForestDataSpec(object): - - def __init__(self): - self.sparse = DataColumnCollection() - self.dense = DataColumnCollection() - self.dense_features_size = 0 - - def SerializeToString(self): - return 'dense_features_size: %d dense: [%s] sparse: [%s]' % ( - self.dense_features_size, self.dense.SerializeToString(), - self.sparse.SerializeToString()) - - -class DataColumnCollection(object): - """Collection of DataColumns, meant to mimic a proto repeated field.""" - - def __init__(self): - self.cols = [] - - def add(self): # pylint: disable=invalid-name - self.cols.append(DataColumn()) - return self.cols[-1] - - def size(self): # pylint: disable=invalid-name - return len(self.cols) - - def SerializeToString(self): - ret = '' - for c in self.cols: - ret += '{%s}' % c.SerializeToString() - return ret - - -class DataColumn(object): - - def __init__(self): - self.name = '' - self.original_type = '' - self.size = 0 - - def SerializeToString(self): - return 'name: {0} original_type: {1} size: {2}'.format(self.name, - self.original_type, - self.size) - - -def GetColumnName(column_key, col_num): - if isinstance(column_key, str): - return column_key - else: - return getattr(column_key, 'column_name', str(col_num)) - - -def ParseDataTensorOrDict(data): - """Return a tensor to use for input data. - - The incoming features can be a dict where keys are the string names of the - columns, which we turn into a single 2-D tensor. - - Args: - data: `Tensor` or `dict` of `Tensor` objects. - - Returns: - A 2-D tensor for input to tensor_forest, a keys tensor for the - tf.Examples if they exist, and a list of the type of each column - (e.g. continuous float, categorical). - """ - data_spec = TensorForestDataSpec() - if isinstance(data, dict): - dense_features_size = 0 - dense_features = [] - sparse_features = [] - for k in sorted(data.keys()): - is_sparse = isinstance(data[k], sparse_tensor.SparseTensor) - if is_sparse: - # TODO(gilberth): support sparse continuous. - if data[k].dtype == dtypes.float32: - logging.info('TensorForest does not support sparse continuous.') - continue - elif data_spec.sparse.size() == 0: - col_spec = data_spec.sparse.add() - col_spec.original_type = DATA_CATEGORICAL - col_spec.name = 'all_sparse' - col_spec.size = -1 - sparse_features.append( - sparse_tensor.SparseTensor(data[ - k].indices, CastToFloat(data[k].values), data[k].dense_shape)) - else: - col_spec = data_spec.dense.add() - - col_spec.original_type = DTYPE_TO_FTYPE[data[k].dtype] - col_spec.name = GetColumnName(k, len(dense_features)) - # the second dimension of get_shape should always be known. - shape = data[k].get_shape() - if len(shape) == 1: - col_spec.size = 1 - else: - col_spec.size = shape[1].value - - dense_features_size += col_spec.size - dense_features.append(CastToFloat(data[k])) - - processed_dense_features = None - processed_sparse_features = None - if dense_features: - processed_dense_features = array_ops.concat(dense_features, 1) - data_spec.dense_features_size = dense_features_size - if sparse_features: - processed_sparse_features = sparse_ops.sparse_concat(1, sparse_features) - logging.info(data_spec.SerializeToString()) - return processed_dense_features, processed_sparse_features, data_spec - elif isinstance(data, sparse_tensor.SparseTensor): - col_spec = data_spec.sparse.add() - col_spec.name = 'sparse_features' - col_spec.original_type = DTYPE_TO_FTYPE[data.dtype] - col_spec.size = -1 - data_spec.dense_features_size = 0 - return None, data, data_spec - else: - data = ops.convert_to_tensor(data) - col_spec = data_spec.dense.add() - col_spec.name = 'dense_features' - col_spec.original_type = DTYPE_TO_FTYPE[data.dtype] - col_spec.size = data.get_shape()[1] - data_spec.dense_features_size = col_spec.size - return data, None, data_spec - - -def ParseLabelTensorOrDict(labels): - """Return a tensor to use for input labels to tensor_forest. - - The incoming targets can be a dict where keys are the string names of the - columns, which we turn into a single 1-D tensor for classification or - 2-D tensor for regression. - - Converts sparse tensors to dense ones. - - Args: - labels: `Tensor` or `dict` of `Tensor` objects. - - Returns: - A 2-D tensor for labels/outputs. - """ - if isinstance(labels, dict): - return math_ops.cast( - array_ops.concat( - [ - sparse_ops.sparse_tensor_to_dense( - labels[k], default_value=-1) if isinstance( - labels, sparse_tensor.SparseTensor) else labels[k] - for k in sorted(labels.keys()) - ], - 1), - dtypes.float32) - else: - if isinstance(labels, sparse_tensor.SparseTensor): - return math_ops.cast( - sparse_ops.sparse_tensor_to_dense(labels, default_value=-1), - dtypes.float32) - else: - return math_ops.cast(labels, dtypes.float32) diff --git a/tensorflow/contrib/tensor_forest/python/ops/model_ops.py b/tensorflow/contrib/tensor_forest/python/ops/model_ops.py deleted file mode 100644 index d36d0eb0c46..00000000000 --- a/tensorflow/contrib/tensor_forest/python/ops/model_ops.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model ops python wrappers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -from tensorflow.contrib.tensor_forest.python.ops import gen_model_ops - -# pylint: disable=unused-import -from tensorflow.contrib.tensor_forest.python.ops.gen_model_ops import feature_usage_counts -from tensorflow.contrib.tensor_forest.python.ops.gen_model_ops import traverse_tree_v4 -from tensorflow.contrib.tensor_forest.python.ops.gen_model_ops import tree_predictions_v4 -from tensorflow.contrib.tensor_forest.python.ops.gen_model_ops import tree_size -from tensorflow.contrib.tensor_forest.python.ops.gen_model_ops import update_model_v4 -# pylint: enable=unused-import - -from tensorflow.contrib.util import loader -from tensorflow.python.eager import context -from tensorflow.python.framework import ops -from tensorflow.python.ops import resources -from tensorflow.python.platform import resource_loader -from tensorflow.python.training import saver -from tensorflow.python.training.tracking import tracking - - -_model_ops = loader.load_op_library( - resource_loader.get_path_to_datafile("_model_ops.so")) - - -ops.NotDifferentiable("TreeVariable") -ops.NotDifferentiable("TreeSerialize") -ops.NotDifferentiable("TreeDeserialize") -ops.NotDifferentiable("TreeSize") -ops.NotDifferentiable("TreePredictionsV4") -ops.NotDifferentiable("FeatureUsageCounts") - - -class TreeVariableSavable(saver.BaseSaverBuilder.SaveableObject): - """SaveableObject implementation for TreeVariable.""" - - def __init__(self, params, tree_handle, stats_handle, create_op, name): - """Creates a TreeVariableSavable object. - - Args: - params: A TensorForestParams object. - tree_handle: handle to the tree variable. - stats_handle: handle to the stats variable. - create_op: the op to initialize the variable. - name: the name to save the tree variable under. - """ - self.params = params - tensor = gen_model_ops.tree_serialize(tree_handle) - # slice_spec is useful for saving a slice from a variable. - # It's not meaningful the tree variable. So we just pass an empty value. - slice_spec = "" - specs = [saver.BaseSaverBuilder.SaveSpec(tensor, slice_spec, name),] - super(TreeVariableSavable, - self).__init__(tree_handle, specs, name) - self._tree_handle = tree_handle - self._create_op = create_op - - def restore(self, restored_tensors, unused_restored_shapes): - """Restores the associated tree from 'restored_tensors'. - - Args: - restored_tensors: the tensors that were loaded from a checkpoint. - unused_restored_shapes: the shapes this object should conform to after - restore. Not meaningful for trees. - - Returns: - The operation that restores the state of the tree variable. - """ - with ops.control_dependencies([self._create_op]): - return gen_model_ops.tree_deserialize( - self._tree_handle, - restored_tensors[0], - params=self.params.serialized_params_proto) - - -class TreeVariable(tracking.TrackableResource): - """A tree model.""" - - def __init__(self, params, tree_config, stats_handle, name, container=None): - self._params = params - self._tree_config = tree_config - self._stats_handle = stats_handle - self._name = name - self._container = container - self._init_op = None - super(TreeVariable, self).__init__() - self._resource_handle = self._create_resource() - - def _create_resource(self): - if context.executing_eagerly(): - # TODO(allenl): This will leak memory due to kernel caching by the - # shared_name attribute value (but is better than the alternative of - # sharing everything by default when executing eagerly; hopefully creating - # tables in a loop is uncommon). - shared_name = "tree_variable_%d" % (ops.uid(),) - else: - shared_name = self._name - return gen_model_ops.decision_tree_resource_handle_op( - self._container, shared_name=shared_name, name=self._name) - - def _initialize(self): - return gen_model_ops.create_tree_variable( - self.resource_handle, - self._tree_config, - params=self._params.serialized_params_proto) - - @property - def initializer(self): - if self._init_op is None: - self._init_op = self._initialize() - return self._init_op - - def is_initialized(self): - return gen_model_ops.tree_is_initialized_op(self.resource_handle) - - def _gather_saveables_for_checkpoint(self): - """For object-based checkpointing.""" - return { - "tree_variable": - functools.partial( - TreeVariableSavable, - params=self._params, - tree_handle=self.resource_handle, - stats_handle=self._stats_handle, - create_op=self._init_op) - } - - -def tree_variable(params, tree_config, stats_handle, name, container=None): - r"""Creates a tree model and returns a handle to it. - - Args: - params: A TensorForestParams object. - tree_config: A `Tensor` of type `string`. Serialized proto of the tree. - stats_handle: Resource handle to the stats object. - name: A name for the variable. - container: An optional `string`. Defaults to `""`. - - Returns: - A `Tensor` of type mutable `string`. The handle to the tree. - """ - with ops.name_scope(name, "TreeVariable") as name: - tree_var = TreeVariable(params, tree_config, stats_handle, name, container) - resource_handle = tree_var.resource_handle - create_op = tree_var.initializer - is_initialized_op = tree_var.is_initialized() - # Adds the variable to the savable list. - saveable = tree_var._gather_saveables_for_checkpoint()["tree_variable"]( # pylint: disable=protected-access - name=resource_handle.name) - ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) - resources.register_resource(resource_handle, create_op, is_initialized_op) - return resource_handle diff --git a/tensorflow/contrib/tensor_forest/python/ops/stats_ops.py b/tensorflow/contrib/tensor_forest/python/ops/stats_ops.py deleted file mode 100644 index 7ac68fed20c..00000000000 --- a/tensorflow/contrib/tensor_forest/python/ops/stats_ops.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stats ops python wrappers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -from tensorflow.contrib.tensor_forest.python.ops import gen_stats_ops -# pylint: disable=unused-import -from tensorflow.contrib.tensor_forest.python.ops.gen_stats_ops import finalize_tree -from tensorflow.contrib.tensor_forest.python.ops.gen_stats_ops import grow_tree_v4 -from tensorflow.contrib.tensor_forest.python.ops.gen_stats_ops import process_input_v4 -# pylint: enable=unused-import - -from tensorflow.contrib.util import loader -from tensorflow.python.eager import context -from tensorflow.python.framework import ops -from tensorflow.python.ops import resources -from tensorflow.python.platform import resource_loader -from tensorflow.python.training import saver -from tensorflow.python.training.tracking import tracking - - -_stats_ops = loader.load_op_library( - resource_loader.get_path_to_datafile("_stats_ops.so")) - - -ops.NotDifferentiable("FertileStatsVariable") -ops.NotDifferentiable("FertileStatsSerialize") -ops.NotDifferentiable("FertileStatsDeserialize") -ops.NotDifferentiable("GrowTreeV4") -ops.NotDifferentiable("ProcessInputV4") -ops.NotDifferentiable("FinalizeTree") - - -class FertileStatsVariableSavable(saver.BaseSaverBuilder.SaveableObject): - """SaveableObject implementation for FertileStatsVariable.""" - - def __init__(self, params, stats_handle, create_op, name): - """Creates a FertileStatsVariableSavable object. - - Args: - params: A TensorForestParams object. - stats_handle: handle to the tree variable. - create_op: the op to initialize the variable. - name: the name to save the tree variable under. - """ - self.params = params - tensor = gen_stats_ops.fertile_stats_serialize( - stats_handle, params=params.serialized_params_proto) - # slice_spec is useful for saving a slice from a variable. - # It's not meaningful the tree variable. So we just pass an empty value. - slice_spec = "" - specs = [saver.BaseSaverBuilder.SaveSpec(tensor, slice_spec, name),] - super(FertileStatsVariableSavable, - self).__init__(stats_handle, specs, name) - self._stats_handle = stats_handle - self._create_op = create_op - - def restore(self, restored_tensors, unused_restored_shapes): - """Restores the associated tree from 'restored_tensors'. - - Args: - restored_tensors: the tensors that were loaded from a checkpoint. - unused_restored_shapes: the shapes this object should conform to after - restore. Not meaningful for trees. - - Returns: - The operation that restores the state of the tree variable. - """ - with ops.control_dependencies([self._create_op]): - return gen_stats_ops.fertile_stats_deserialize( - self._stats_handle, restored_tensors[0], - params=self.params.serialized_params_proto) - - -class FertileStatsVariable(tracking.TrackableResource): - """A Fertile stats variable.""" - - def __init__(self, params, stats_config, name, container=None): - self._params = params - self._stats_config = stats_config - self._name = name - self._container = container - self._init_op = None - super(FertileStatsVariable, self).__init__() - self._resource_handle = self._create_resource() - - def _create_resource(self): - if context.executing_eagerly(): - # TODO(allenl): This will leak memory due to kernel caching by the - # shared_name attribute value (but is better than the alternative of - # sharing everything by default when executing eagerly; hopefully creating - # tables in a loop is uncommon). - shared_name = "fertile_stats_variable_%d" % (ops.uid(),) - else: - shared_name = self._name - return gen_stats_ops.fertile_stats_resource_handle_op( - self._container, shared_name=shared_name, name=self._name) - - def _initialize(self): - return gen_stats_ops.create_fertile_stats_variable( - self.resource_handle, - self._stats_config, - params=self._params.serialized_params_proto) - - @property - def initializer(self): - if self._init_op is None: - self._init_op = self._initialize() - return self._init_op - - def is_initialized(self): - return gen_stats_ops.fertile_stats_is_initialized_op(self.resource_handle) - - def _gather_saveables_for_checkpoint(self): - """For object-based checkpointing.""" - return { - "fertile_stats_variable": - functools.partial( - FertileStatsVariableSavable, - params=self._params, - stats_handle=self.resource_handle, - create_op=self.initializer) - } - - -def fertile_stats_variable(params, stats_config, name, container=None): - r"""Creates a stats object and returns a handle to it. - - Args: - params: A TensorForestParams object. - stats_config: A `Tensor` of type `string`. Serialized proto of the stats. - name: A name for the variable. - container: An optional `string`. Defaults to `""`. - - Returns: - A `Tensor` of type mutable `string`. The handle to the stats. - """ - with ops.name_scope(name, "FertileStatsVariable") as name: - fertile_stats_var = FertileStatsVariable(params, stats_config, name, - container) - resource_handle = fertile_stats_var.resource_handle - create_op = fertile_stats_var.initializer - is_initialized_op = fertile_stats_var.is_initialized() - # Adds the variable to the savable list. - saveable = ( - fertile_stats_var._gather_saveables_for_checkpoint()[ # pylint: disable=protected-access - "fertile_stats_variable"](name=resource_handle.name)) - ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) - resources.register_resource(resource_handle, create_op, is_initialized_op) - return resource_handle diff --git a/tensorflow/contrib/tensor_forest/python/ops/tensor_forest_ops.py b/tensorflow/contrib/tensor_forest/python/ops/tensor_forest_ops.py deleted file mode 100644 index 38166d0d824..00000000000 --- a/tensorflow/contrib/tensor_forest/python/ops/tensor_forest_ops.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Custom ops used by tensorforest.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# go/tf-wildcard-import -# pylint: disable=wildcard-import -from tensorflow.contrib.tensor_forest.python.ops.gen_tensor_forest_ops import * -# pylint: enable=wildcard-import -from tensorflow.contrib.util import loader -from tensorflow.python.platform import resource_loader - -_tensor_forest_ops = loader.load_op_library( - resource_loader.get_path_to_datafile('_tensor_forest_ops.so')) diff --git a/tensorflow/contrib/tensor_forest/python/tensor_forest.py b/tensorflow/contrib/tensor_forest/python/tensor_forest.py deleted file mode 100644 index 623e52ca0b6..00000000000 --- a/tensorflow/contrib/tensor_forest/python/tensor_forest.py +++ /dev/null @@ -1,708 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Extremely random forest graph builder. go/brain-tree.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import numbers -import random - -from google.protobuf import text_format - -from tensorflow.contrib.decision_trees.proto import generic_tree_model_pb2 as _tree_proto -from tensorflow.contrib.framework.python.ops import variables as framework_variables -from tensorflow.contrib.tensor_forest.proto import tensor_forest_params_pb2 as _params_proto -from tensorflow.contrib.tensor_forest.python.ops import data_ops -from tensorflow.contrib.tensor_forest.python.ops import model_ops -from tensorflow.contrib.tensor_forest.python.ops import stats_ops - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables as tf_variables -from tensorflow.python.platform import tf_logging as logging - -# Stores tuples of (leaf model type, stats model type) -CLASSIFICATION_LEAF_MODEL_TYPES = { - 'all_dense': (_params_proto.MODEL_DENSE_CLASSIFICATION, - _params_proto.STATS_DENSE_GINI), - 'all_sparse': (_params_proto.MODEL_SPARSE_CLASSIFICATION, - _params_proto.STATS_SPARSE_GINI), - 'sparse_then_dense': (_params_proto.MODEL_SPARSE_OR_DENSE_CLASSIFICATION, - _params_proto.STATS_SPARSE_THEN_DENSE_GINI), -} -REGRESSION_MODEL_TYPE = (_params_proto.MODEL_REGRESSION, - _params_proto.STATS_LEAST_SQUARES_REGRESSION, - _params_proto.COLLECTION_BASIC) - -FINISH_TYPES = { - 'basic': _params_proto.SPLIT_FINISH_BASIC, - 'hoeffding': _params_proto.SPLIT_FINISH_DOMINATE_HOEFFDING, - 'bootstrap': _params_proto.SPLIT_FINISH_DOMINATE_BOOTSTRAP -} -PRUNING_TYPES = { - 'none': _params_proto.SPLIT_PRUNE_NONE, - 'half': _params_proto.SPLIT_PRUNE_HALF, - 'quarter': _params_proto.SPLIT_PRUNE_QUARTER, - '10_percent': _params_proto.SPLIT_PRUNE_10_PERCENT, - 'hoeffding': _params_proto.SPLIT_PRUNE_HOEFFDING, -} -SPLIT_TYPES = { - 'less_or_equal': _tree_proto.InequalityTest.LESS_OR_EQUAL, - 'less': _tree_proto.InequalityTest.LESS_THAN -} - - -def parse_number_or_string_to_proto(proto, param): - if isinstance(param, numbers.Number): - proto.constant_value = param - else: # assume it's a string - if param.isdigit(): - proto.constant_value = int(param) - else: - text_format.Merge(param, proto) - - -def build_params_proto(params): - """Build a TensorForestParams proto out of the V4ForestHParams object.""" - proto = _params_proto.TensorForestParams() - proto.num_trees = params.num_trees - proto.max_nodes = params.max_nodes - proto.is_regression = params.regression - proto.num_outputs = params.num_classes - proto.num_features = params.num_features - - proto.leaf_type = params.leaf_model_type - proto.stats_type = params.stats_model_type - proto.collection_type = _params_proto.COLLECTION_BASIC - proto.pruning_type.type = params.pruning_type - proto.finish_type.type = params.finish_type - - proto.inequality_test_type = params.split_type - - proto.drop_final_class = False - proto.collate_examples = params.collate_examples - proto.checkpoint_stats = params.checkpoint_stats - proto.use_running_stats_method = params.use_running_stats_method - proto.initialize_average_splits = params.initialize_average_splits - proto.inference_tree_paths = params.inference_tree_paths - - parse_number_or_string_to_proto(proto.pruning_type.prune_every_samples, - params.prune_every_samples) - parse_number_or_string_to_proto(proto.finish_type.check_every_steps, - params.early_finish_check_every_samples) - parse_number_or_string_to_proto(proto.split_after_samples, - params.split_after_samples) - parse_number_or_string_to_proto(proto.num_splits_to_consider, - params.num_splits_to_consider) - - proto.dominate_fraction.constant_value = params.dominate_fraction - - if params.param_file: - with open(params.param_file) as f: - text_format.Merge(f.read(), proto) - - return proto - - -# A convenience class for holding random forest hyperparameters. -# -# To just get some good default parameters, use: -# hparams = ForestHParams(num_classes=2, num_features=40).fill() -# -# Note that num_classes can not be inferred and so must always be specified. -# Also, either num_splits_to_consider or num_features should be set. -# -# To override specific values, pass them to the constructor: -# hparams = ForestHParams(num_classes=5, num_trees=10, num_features=5).fill() -# -# TODO(thomaswc): Inherit from tf.HParams when that is publicly available. -class ForestHParams(object): - """A base class for holding hyperparameters and calculating good defaults.""" - - def __init__( - self, - num_trees=100, - max_nodes=10000, - bagging_fraction=1.0, - num_splits_to_consider=0, - feature_bagging_fraction=1.0, - max_fertile_nodes=0, # deprecated, unused. - split_after_samples=250, - valid_leaf_threshold=1, - dominate_method='bootstrap', - dominate_fraction=0.99, - model_name='all_dense', - split_finish_name='basic', - split_pruning_name='none', - prune_every_samples=0, - early_finish_check_every_samples=0, - collate_examples=False, - checkpoint_stats=False, - use_running_stats_method=False, - initialize_average_splits=False, - inference_tree_paths=False, - param_file=None, - split_name='less_or_equal', - **kwargs): - self.num_trees = num_trees - self.max_nodes = max_nodes - self.bagging_fraction = bagging_fraction - self.feature_bagging_fraction = feature_bagging_fraction - self.num_splits_to_consider = num_splits_to_consider - self.max_fertile_nodes = max_fertile_nodes - self.split_after_samples = split_after_samples - self.valid_leaf_threshold = valid_leaf_threshold - self.dominate_method = dominate_method - self.dominate_fraction = dominate_fraction - self.model_name = model_name - self.split_finish_name = split_finish_name - self.split_pruning_name = split_pruning_name - self.collate_examples = collate_examples - self.checkpoint_stats = checkpoint_stats - self.use_running_stats_method = use_running_stats_method - self.initialize_average_splits = initialize_average_splits - self.inference_tree_paths = inference_tree_paths - self.param_file = param_file - self.split_name = split_name - self.early_finish_check_every_samples = early_finish_check_every_samples - self.prune_every_samples = prune_every_samples - - for name, value in kwargs.items(): - setattr(self, name, value) - - def values(self): - return self.__dict__ - - def fill(self): - """Intelligently sets any non-specific parameters.""" - # Fail fast if num_classes or num_features isn't set. - _ = getattr(self, 'num_classes') - _ = getattr(self, 'num_features') - - self.bagged_num_features = int(self.feature_bagging_fraction * - self.num_features) - - self.bagged_features = None - if self.feature_bagging_fraction < 1.0: - self.bagged_features = [ - random.sample(range(self.num_features), self.bagged_num_features) - for _ in range(self.num_trees) - ] - - self.regression = getattr(self, 'regression', False) - - # Num_outputs is the actual number of outputs (a single prediction for - # classification, a N-dimensional point for regression). - self.num_outputs = self.num_classes if self.regression else 1 - - # Add an extra column to classes for storing counts, which is needed for - # regression and avoids having to recompute sums for classification. - self.num_output_columns = self.num_classes + 1 - - # Our experiments have found that num_splits_to_consider = num_features - # gives good accuracy. - self.num_splits_to_consider = self.num_splits_to_consider or min( - max(10, math.floor(math.sqrt(self.num_features))), 1000) - - # If base_random_seed is 0, the current time will be used to seed the - # random number generators for each tree. If non-zero, the i-th tree - # will be seeded with base_random_seed + i. - self.base_random_seed = getattr(self, 'base_random_seed', 0) - - # How to store leaf models. - self.leaf_model_type = ( - REGRESSION_MODEL_TYPE[0] if self.regression else - CLASSIFICATION_LEAF_MODEL_TYPES[self.model_name][0]) - - # How to store stats objects. - self.stats_model_type = ( - REGRESSION_MODEL_TYPE[1] if self.regression else - CLASSIFICATION_LEAF_MODEL_TYPES[self.model_name][1]) - - self.finish_type = ( - _params_proto.SPLIT_FINISH_BASIC - if self.regression else FINISH_TYPES[self.split_finish_name]) - - self.pruning_type = PRUNING_TYPES[self.split_pruning_name] - - if self.pruning_type == _params_proto.SPLIT_PRUNE_NONE: - self.prune_every_samples = 0 - else: - if (not self.prune_every_samples and - not (isinstance(numbers.Number) or - self.split_after_samples.isdigit())): - logging.error( - 'Must specify prune_every_samples if using a depth-dependent ' - 'split_after_samples') - # Pruning half-way through split_after_samples seems like a decent - # default, making it easy to select the number being pruned with - # pruning_type while not paying the cost of pruning too often. Note that - # this only holds if not using a depth-dependent split_after_samples. - self.prune_every_samples = ( - self.prune_every_samples or int(self.split_after_samples) / 2) - - if self.finish_type == _params_proto.SPLIT_FINISH_BASIC: - self.early_finish_check_every_samples = 0 - else: - if (not self.early_finish_check_every_samples and - not (isinstance(numbers.Number) or - self.split_after_samples.isdigit())): - logging.error( - 'Must specify prune_every_samples if using a depth-dependent ' - 'split_after_samples') - # Checking for early finish every quarter through split_after_samples - # seems like a decent default. We don't want to incur the checking cost - # too often, but (at least for hoeffding) it's lower than the cost of - # pruning so we can do it a little more frequently. - self.early_finish_check_every_samples = ( - self.early_finish_check_every_samples or - int(self.split_after_samples) / 4) - - self.split_type = SPLIT_TYPES[self.split_name] - - return self - - -def get_epoch_variable(): - """Returns the epoch variable, or [0] if not defined.""" - # Grab epoch variable defined in - # //third_party/tensorflow/python/training/input.py::limit_epochs - for v in tf_variables.local_variables(): - if 'limit_epochs/epoch' in v.op.name: - return array_ops.reshape(v, [1]) - # TODO(thomaswc): Access epoch from the data feeder. - return [0] - - -# A simple container to hold the training variables for a single tree. -class TreeVariables(object): - """Stores tf.Variables for training a single random tree. - - Uses tf.compat.v1.get_variable to get tree-specific names so that this can be - used - with a tf.learn-style implementation (one that trains a model, saves it, - then relies on restoring that model to evaluate). - """ - - def __init__(self, params, tree_num, training, tree_config='', tree_stat=''): - if (not hasattr(params, 'params_proto') or - not isinstance(params.params_proto, _params_proto.TensorForestParams)): - params.params_proto = build_params_proto(params) - - params.serialized_params_proto = params.params_proto.SerializeToString() - self.stats = None - if training: - # TODO(gilberth): Manually shard this to be able to fit it on - # multiple machines. - self.stats = stats_ops.fertile_stats_variable( - params, tree_stat, self.get_tree_name('stats', tree_num)) - self.tree = model_ops.tree_variable(params, tree_config, self.stats, - self.get_tree_name('tree', tree_num)) - - def get_tree_name(self, name, num): - return '{0}-{1}'.format(name, num) - - -class ForestVariables(object): - """A container for a forests training data, consisting of multiple trees. - - Instantiates a TreeVariables object for each tree. We override the - __getitem__ and __setitem__ function so that usage looks like this: - - forest_variables = ForestVariables(params) - - ... forest_variables.tree ... - """ - - def __init__(self, - params, - device_assigner, - training=True, - tree_variables_class=TreeVariables, - tree_configs=None, - tree_stats=None): - self.variables = [] - # Set up some scalar variables to run through the device assigner, then - # we can use those to colocate everything related to a tree. - self.device_dummies = [] - with ops.device(device_assigner): - for i in range(params.num_trees): - self.device_dummies.append( - variable_scope.get_variable(name='device_dummy_%d' % i, shape=0)) - - for i in range(params.num_trees): - with ops.device(self.device_dummies[i].device): - kwargs = {} - if tree_configs is not None: - kwargs.update(dict(tree_config=tree_configs[i])) - if tree_stats is not None: - kwargs.update(dict(tree_stat=tree_stats[i])) - self.variables.append( - tree_variables_class(params, i, training, **kwargs)) - - def __setitem__(self, t, val): - self.variables[t] = val - - def __getitem__(self, t): - return self.variables[t] - - -class RandomForestGraphs(object): - """Builds TF graphs for random forest training and inference.""" - - def __init__(self, - params, - tree_configs=None, - tree_stats=None, - device_assigner=None, - variables=None, - tree_variables_class=TreeVariables, - tree_graphs=None, - training=True): - self.params = params - self.device_assigner = ( - device_assigner or framework_variables.VariableDeviceChooser()) - logging.info('Constructing forest with params = ') - logging.info(self.params.__dict__) - self.variables = variables or ForestVariables( - self.params, - device_assigner=self.device_assigner, - training=training, - tree_variables_class=tree_variables_class, - tree_configs=tree_configs, - tree_stats=tree_stats) - tree_graph_class = tree_graphs or RandomTreeGraphs - self.trees = [ - tree_graph_class(self.variables[i], self.params, i) - for i in range(self.params.num_trees) - ] - - def _bag_features(self, tree_num, input_data): - split_data = array_ops.split( - value=input_data, num_or_size_splits=self.params.num_features, axis=1) - return array_ops.concat( - [split_data[ind] for ind in self.params.bagged_features[tree_num]], 1) - - def get_all_resource_handles(self): - return ([self.variables[i].tree for i in range(len(self.trees))] + - [self.variables[i].stats for i in range(len(self.trees))]) - - def training_graph(self, - input_data, - input_labels, - num_trainers=1, - trainer_id=0, - **tree_kwargs): - """Constructs a TF graph for training a random forest. - - Args: - input_data: A tensor or dict of string->Tensor for input data. - input_labels: A tensor or placeholder for labels associated with - input_data. - num_trainers: Number of parallel trainers to split trees among. - trainer_id: Which trainer this instance is. - **tree_kwargs: Keyword arguments passed to each tree's training_graph. - - Returns: - The last op in the random forest training graph. - - Raises: - NotImplementedError: If trying to use bagging with sparse features. - """ - processed_dense_features, processed_sparse_features, data_spec = ( - data_ops.ParseDataTensorOrDict(input_data)) - - if input_labels is not None: - labels = data_ops.ParseLabelTensorOrDict(input_labels) - - data_spec = data_spec or self.get_default_data_spec(input_data) - - tree_graphs = [] - trees_per_trainer = self.params.num_trees / num_trainers - tree_start = int(trainer_id * trees_per_trainer) - tree_end = int((trainer_id + 1) * trees_per_trainer) - for i in range(tree_start, tree_end): - with ops.device(self.variables.device_dummies[i].device): - seed = self.params.base_random_seed - if seed != 0: - seed += i - # If using bagging, randomly select some of the input. - tree_data = processed_dense_features - tree_labels = labels - if self.params.bagging_fraction < 1.0: - # TODO(gilberth): Support bagging for sparse features. - if processed_sparse_features is not None: - raise NotImplementedError( - 'Bagging not supported with sparse features.') - # TODO(thomaswc): This does sampling without replacement. Consider - # also allowing sampling with replacement as an option. - batch_size = array_ops.strided_slice( - array_ops.shape(processed_dense_features), [0], [1]) - r = random_ops.random_uniform(batch_size, seed=seed) - mask = math_ops.less( - r, - array_ops.ones_like(r) * self.params.bagging_fraction) - gather_indices = array_ops.squeeze(array_ops.where_v2(mask), axis=[1]) - # TODO(thomaswc): Calculate out-of-bag data and labels, and store - # them for use in calculating statistics later. - tree_data = array_ops.gather(processed_dense_features, gather_indices) - tree_labels = array_ops.gather(labels, gather_indices) - if self.params.bagged_features: - if processed_sparse_features is not None: - raise NotImplementedError( - 'Feature bagging not supported with sparse features.') - tree_data = self._bag_features(i, tree_data) - - tree_graphs.append(self.trees[i].training_graph( - tree_data, - tree_labels, - seed, - data_spec=data_spec, - sparse_features=processed_sparse_features, - **tree_kwargs)) - - return control_flow_ops.group(*tree_graphs, name='train') - - def inference_graph(self, input_data, **inference_args): - """Constructs a TF graph for evaluating a random forest. - - Args: - input_data: A tensor or dict of string->Tensor for the input data. This - input_data must generate the same spec as the - input_data used in training_graph: the dict must have the - same keys, for example, and all tensors must have the same - size in their first dimension. - **inference_args: Keyword arguments to pass through to each tree. - - Returns: - A tuple of (probabilities, tree_paths, variance). - - Raises: - NotImplementedError: If trying to use feature bagging with sparse - features. - """ - processed_dense_features, processed_sparse_features, data_spec = ( - data_ops.ParseDataTensorOrDict(input_data)) - - probabilities = [] - paths = [] - for i in range(self.params.num_trees): - with ops.device(self.variables.device_dummies[i].device): - tree_data = processed_dense_features - if self.params.bagged_features: - if processed_sparse_features is not None: - raise NotImplementedError( - 'Feature bagging not supported with sparse features.') - tree_data = self._bag_features(i, tree_data) - probs, path = self.trees[i].inference_graph( - tree_data, - data_spec, - sparse_features=processed_sparse_features, - **inference_args) - probabilities.append(probs) - paths.append(path) - with ops.device(self.variables.device_dummies[0].device): - # shape of all_predict should be [batch_size, num_trees, num_outputs] - all_predict = array_ops.stack(probabilities, axis=1) - average_values = math_ops.div( - math_ops.reduce_sum(all_predict, 1), - self.params.num_trees, - name='probabilities') - tree_paths = array_ops.stack(paths, axis=1) - - expected_squares = math_ops.div( - math_ops.reduce_sum(all_predict * all_predict, 1), - self.params.num_trees) - regression_variance = math_ops.maximum( - 0., expected_squares - average_values * average_values) - return average_values, tree_paths, regression_variance - - def average_size(self): - """Constructs a TF graph for evaluating the average size of a forest. - - Returns: - The average number of nodes over the trees. - """ - sizes = [] - for i in range(self.params.num_trees): - with ops.device(self.variables.device_dummies[i].device): - sizes.append(self.trees[i].size()) - return math_ops.reduce_mean( - math_ops.cast(array_ops.stack(sizes), dtypes.float32)) - - # pylint: disable=unused-argument - def training_loss(self, features, labels, name='training_loss'): - return math_ops.negative(self.average_size(), name=name) - - # pylint: disable=unused-argument - def validation_loss(self, features, labels): - return math_ops.negative(self.average_size()) - - def average_impurity(self): - """Constructs a TF graph for evaluating the leaf impurity of a forest. - - Returns: - The last op in the graph. - """ - impurities = [] - for i in range(self.params.num_trees): - with ops.device(self.variables.device_dummies[i].device): - impurities.append(self.trees[i].average_impurity()) - return math_ops.reduce_mean(array_ops.stack(impurities)) - - def feature_importances(self): - tree_counts = [ - self.trees[i].feature_usage_counts() - for i in range(self.params.num_trees) - ] - total_counts = math_ops.reduce_sum(array_ops.stack(tree_counts, 0), 0) - return total_counts / math_ops.reduce_sum(total_counts) - - -class RandomTreeGraphs(object): - """Builds TF graphs for random tree training and inference.""" - - def __init__(self, variables, params, tree_num): - self.variables = variables - self.params = params - self.tree_num = tree_num - - def training_graph(self, - input_data, - input_labels, - random_seed, - data_spec, - sparse_features=None, - input_weights=None): - """Constructs a TF graph for training a random tree. - - Args: - input_data: A tensor or placeholder for input data. - input_labels: A tensor or placeholder for labels associated with - input_data. - random_seed: The random number generator seed to use for this tree. 0 - means use the current time as the seed. - data_spec: A data_ops.TensorForestDataSpec object specifying the original - feature/columns of the data. - sparse_features: A tf.SparseTensor for sparse input data. - input_weights: A float tensor or placeholder holding per-input weights, or - None if all inputs are to be weighted equally. - - Returns: - The last op in the random tree training graph. - """ - # TODO(gilberth): Use this. - unused_epoch = math_ops.cast(get_epoch_variable(), dtypes.int32) - - if input_weights is None: - input_weights = [] - - sparse_indices = [] - sparse_values = [] - sparse_shape = [] - if sparse_features is not None: - sparse_indices = sparse_features.indices - sparse_values = sparse_features.values - sparse_shape = sparse_features.dense_shape - - if input_data is None: - input_data = [] - - leaf_ids = model_ops.traverse_tree_v4( - self.variables.tree, - input_data, - sparse_indices, - sparse_values, - sparse_shape, - input_spec=data_spec.SerializeToString(), - params=self.params.serialized_params_proto) - - update_model = model_ops.update_model_v4( - self.variables.tree, - leaf_ids, - input_labels, - input_weights, - params=self.params.serialized_params_proto) - - finished_nodes = stats_ops.process_input_v4( - self.variables.tree, - self.variables.stats, - input_data, - sparse_indices, - sparse_values, - sparse_shape, - input_labels, - input_weights, - leaf_ids, - input_spec=data_spec.SerializeToString(), - random_seed=random_seed, - params=self.params.serialized_params_proto) - - with ops.control_dependencies([update_model]): - return stats_ops.grow_tree_v4( - self.variables.tree, - self.variables.stats, - finished_nodes, - params=self.params.serialized_params_proto) - - def inference_graph(self, input_data, data_spec, sparse_features=None): - """Constructs a TF graph for evaluating a random tree. - - Args: - input_data: A tensor or placeholder for input data. - data_spec: A TensorForestDataSpec proto specifying the original input - columns. - sparse_features: A tf.SparseTensor for sparse input data. - - Returns: - A tuple of (probabilities, tree_paths). - """ - sparse_indices = [] - sparse_values = [] - sparse_shape = [] - if sparse_features is not None: - sparse_indices = sparse_features.indices - sparse_values = sparse_features.values - sparse_shape = sparse_features.dense_shape - if input_data is None: - input_data = [] - - return model_ops.tree_predictions_v4( - self.variables.tree, - input_data, - sparse_indices, - sparse_values, - sparse_shape, - input_spec=data_spec.SerializeToString(), - params=self.params.serialized_params_proto) - - def size(self): - """Constructs a TF graph for evaluating the current number of nodes. - - Returns: - The current number of nodes in the tree. - """ - return model_ops.tree_size(self.variables.tree) - - def feature_usage_counts(self): - return model_ops.feature_usage_counts( - self.variables.tree, params=self.params.serialized_params_proto) diff --git a/tensorflow/contrib/tensor_forest/python/tensor_forest_test.py b/tensorflow/contrib/tensor_forest/python/tensor_forest_test.py deleted file mode 100644 index e0f0c0d4ff6..00000000000 --- a/tensorflow/contrib/tensor_forest/python/tensor_forest_test.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.contrib.tensor_forest.ops.tensor_forest.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from google.protobuf.json_format import ParseDict -from tensorflow.contrib.decision_trees.proto import generic_tree_model_pb2 as _tree_proto -from tensorflow.contrib.tensor_forest.python import tensor_forest -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import test_util -from tensorflow.python.ops import resources -from tensorflow.python.ops import variables -from tensorflow.python.platform import googletest - - -class TensorForestTest(test_util.TensorFlowTestCase): - - def testForestHParams(self): - hparams = tensor_forest.ForestHParams( - num_classes=2, - num_trees=100, - max_nodes=1000, - split_after_samples=25, - num_features=60).fill() - self.assertEquals(2, hparams.num_classes) - self.assertEquals(3, hparams.num_output_columns) - self.assertEquals(10, hparams.num_splits_to_consider) - # Default value of valid_leaf_threshold - self.assertEquals(1, hparams.valid_leaf_threshold) - self.assertEquals(0, hparams.base_random_seed) - - def testForestHParamsBigTree(self): - hparams = tensor_forest.ForestHParams( - num_classes=2, - num_trees=100, - max_nodes=1000000, - split_after_samples=25, - num_features=1000).fill() - self.assertEquals(31, hparams.num_splits_to_consider) - - def testForestHParamsStringParams(self): - hparams = tensor_forest.ForestHParams( - num_classes=2, - num_trees=100, - max_nodes=1000000, - split_after_samples="25", - num_splits_to_consider="1000000", - num_features=1000).fill() - self.assertEquals("1000000", hparams.num_splits_to_consider) - - def testTrainingConstructionClassification(self): - input_data = [[-1., 0.], [-1., 2.], # node 1 - [1., 0.], [1., -2.]] # node 2 - input_labels = [0, 1, 2, 3] - - params = tensor_forest.ForestHParams( - num_classes=4, - num_features=2, - num_trees=10, - max_nodes=1000, - split_after_samples=25).fill() - - graph_builder = tensor_forest.RandomForestGraphs(params) - graph = graph_builder.training_graph(input_data, input_labels) - self.assertTrue(isinstance(graph, ops.Operation)) - - def testTrainingConstructionRegression(self): - input_data = [[-1., 0.], [-1., 2.], # node 1 - [1., 0.], [1., -2.]] # node 2 - input_labels = [0, 1, 2, 3] - - params = tensor_forest.ForestHParams( - num_classes=4, - num_features=2, - num_trees=10, - max_nodes=1000, - split_after_samples=25, - regression=True).fill() - - graph_builder = tensor_forest.RandomForestGraphs(params) - graph = graph_builder.training_graph(input_data, input_labels) - self.assertTrue(isinstance(graph, ops.Operation)) - - def testInferenceConstruction(self): - input_data = [[-1., 0.], [-1., 2.], # node 1 - [1., 0.], [1., -2.]] # node 2 - - params = tensor_forest.ForestHParams( - num_classes=4, - num_features=2, - num_trees=10, - max_nodes=1000, - split_after_samples=25).fill() - - graph_builder = tensor_forest.RandomForestGraphs(params) - probs, paths, var = graph_builder.inference_graph(input_data) - self.assertTrue(isinstance(probs, ops.Tensor)) - self.assertTrue(isinstance(paths, ops.Tensor)) - self.assertTrue(isinstance(var, ops.Tensor)) - - def testInfrenceFromRestoredModel(self): - input_data = [[-1., 0.], [-1., 2.], # node 1 - [1., 0.], [1., -2.]] # node 2 - expected_prediction = [[0.0, 1.0], [0.0, 1.0], - [0.0, 1.0], [0.0, 1.0]] - hparams = tensor_forest.ForestHParams( - num_classes=2, - num_features=2, - num_trees=1, - max_nodes=1000, - split_after_samples=25).fill() - tree_weight = {'decisionTree': - {'nodes': - [{'binaryNode': - {'rightChildId': 2, - 'leftChildId': 1, - 'inequalityLeftChildTest': - {'featureId': {'id': '0'}, - 'threshold': {'floatValue': 0}}}}, - {'leaf': {'vector': - {'value': [{'floatValue': 0.0}, - {'floatValue': 1.0}]}}, - 'nodeId': 1}, - {'leaf': {'vector': - {'value': [{'floatValue': 0.0}, - {'floatValue': 1.0}]}}, - 'nodeId': 2}]}} - restored_tree_param = ParseDict(tree_weight, - _tree_proto.Model()).SerializeToString() - graph_builder = tensor_forest.RandomForestGraphs(hparams, - [restored_tree_param]) - probs, paths, var = graph_builder.inference_graph(input_data) - self.assertTrue(isinstance(probs, ops.Tensor)) - self.assertTrue(isinstance(paths, ops.Tensor)) - self.assertTrue(isinstance(var, ops.Tensor)) - with self.cached_session(): - variables.global_variables_initializer().run() - resources.initialize_resources(resources.shared_resources()).run() - self.assertEquals(probs.eval().shape, (4, 2)) - self.assertEquals(probs.eval().tolist(), expected_prediction) - - def testTrainingConstructionClassificationSparse(self): - input_data = sparse_tensor.SparseTensor( - indices=[[0, 0], [0, 3], [1, 0], [1, 7], [2, 1], [3, 9]], - values=[-1.0, 0.0, -1., 2., 1., -2.0], - dense_shape=[4, 10]) - input_labels = [0, 1, 2, 3] - - params = tensor_forest.ForestHParams( - num_classes=4, - num_features=10, - num_trees=10, - max_nodes=1000, - split_after_samples=25).fill() - - graph_builder = tensor_forest.RandomForestGraphs(params) - graph = graph_builder.training_graph(input_data, input_labels) - self.assertTrue(isinstance(graph, ops.Operation)) - - def testInferenceConstructionSparse(self): - input_data = sparse_tensor.SparseTensor( - indices=[[0, 0], [0, 3], - [1, 0], [1, 7], - [2, 1], - [3, 9]], - values=[-1.0, 0.0, - -1., 2., - 1., - -2.0], - dense_shape=[4, 10]) - - params = tensor_forest.ForestHParams( - num_classes=4, - num_features=10, - num_trees=10, - max_nodes=1000, - regression=True, - split_after_samples=25).fill() - - graph_builder = tensor_forest.RandomForestGraphs(params) - probs, paths, var = graph_builder.inference_graph(input_data) - self.assertTrue(isinstance(probs, ops.Tensor)) - self.assertTrue(isinstance(paths, ops.Tensor)) - self.assertTrue(isinstance(var, ops.Tensor)) - - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/tensorboard/BUILD b/tensorflow/contrib/tensorboard/BUILD deleted file mode 100644 index 801fe67b069..00000000000 --- a/tensorflow/contrib/tensorboard/BUILD +++ /dev/null @@ -1,63 +0,0 @@ -# Description: -# TensorBoard module containing volatile or experimental code. - -# For platform specific build config -load("//tensorflow/core/platform:default/build_config.bzl", "tf_proto_library") -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_proto_library( - name = "protos_all", - srcs = glob(["**/*.proto"]), - visibility = ["//visibility:public"], -) - -# API methods in `tf.contrib.tensorboard` package. -py_library( - name = "tensorboard", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - deps = [":plugins"], -) - -# API methods in `tf.contrib.tensorboard.plugins` package. -py_library( - name = "plugins", - srcs = ["plugins/__init__.py"], - srcs_version = "PY2AND3", - deps = [ - ":projector", - ], -) - -# API methods and protos in `tf.contrib.tensorboard.plugins.projector` package. -py_library( - name = "projector", - srcs = ["plugins/projector/__init__.py"], - srcs_version = "PY2AND3", - deps = [ - ":protos_all_py", - "//tensorflow/python:lib", - ], -) - -py_test( - name = "projector_api_test", - size = "small", - srcs = ["plugins/projector/projector_api_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":projector", - ":protos_all_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:platform", - "//tensorflow/python:summary", - ], -) diff --git a/tensorflow/contrib/tensorboard/__init__.py b/tensorflow/contrib/tensorboard/__init__.py deleted file mode 100644 index 129f5feacf2..00000000000 --- a/tensorflow/contrib/tensorboard/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""tensorboard module containing volatile or experimental code.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# Add projects here, they will show up under tf.contrib.tensorboard. -from tensorflow.contrib.tensorboard import plugins diff --git a/tensorflow/contrib/tensorboard/plugins/__init__.py b/tensorflow/contrib/tensorboard/plugins/__init__.py deleted file mode 100644 index 4ba469eb523..00000000000 --- a/tensorflow/contrib/tensorboard/plugins/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""tensorboard plugins module containing volatile or experimental code.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# Add projects here, they will show up under tf.contrib.tensorboard.plugins -from tensorflow.contrib.tensorboard.plugins import projector - diff --git a/tensorflow/contrib/tensorboard/plugins/projector/__init__.py b/tensorflow/contrib/tensorboard/plugins/projector/__init__.py deleted file mode 100644 index 7b9be76757c..00000000000 --- a/tensorflow/contrib/tensorboard/plugins/projector/__init__.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Public API for the Embedding Projector. - -@@ProjectorPluginAsset -@@ProjectorConfig -@@EmbeddingInfo -@@EmbeddingMetadata -@@SpriteMetadata -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from google.protobuf import text_format -from tensorflow.contrib.tensorboard.plugins.projector import projector_config_pb2 -# pylint: disable=wildcard-import -from tensorflow.contrib.tensorboard.plugins.projector.projector_config_pb2 import * -# pylint: enable=wildcard-import -from tensorflow.python.lib.io import file_io - - -def visualize_embeddings(summary_writer, config): - """Stores a config file used by the embedding projector. - - Args: - summary_writer: The summary writer used for writing events. - config: `tf.contrib.tensorboard.plugins.projector.ProjectorConfig` - proto that holds the configuration for the projector such as paths to - checkpoint files and metadata files for the embeddings. If - `config.model_checkpoint_path` is none, it defaults to the - `logdir` used by the summary_writer. - - Raises: - ValueError: If the summary writer does not have a `logdir`. - """ - logdir = summary_writer.get_logdir() - - # Sanity checks. - if logdir is None: - raise ValueError('Summary writer must have a logdir') - - # Saving the config file in the logdir. - config_pbtxt = text_format.MessageToString(config) - # FYI - the 'projector_config.pbtxt' string is hardcoded in the projector - # plugin. - # TODO(dandelion): Restore this to a reference to the projector plugin - file_io.write_string_to_file( - os.path.join(logdir, 'projector_config.pbtxt'), config_pbtxt) diff --git a/tensorflow/contrib/tensorboard/plugins/projector/projector_api_test.py b/tensorflow/contrib/tensorboard/plugins/projector/projector_api_test.py deleted file mode 100644 index 9ad42bff47f..00000000000 --- a/tensorflow/contrib/tensorboard/plugins/projector/projector_api_test.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""API tests for the projector plugin in TensorBoard.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import shutil - -from google.protobuf import text_format - -from tensorflow.contrib.tensorboard.plugins import projector -from tensorflow.contrib.tensorboard.plugins.projector import projector_config_pb2 -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test -from tensorflow.python.summary.writer import writer as writer_lib - - -class ProjectorApiTest(test.TestCase): - - def testVisualizeEmbeddings(self): - # Create a dummy configuration. - config = projector_config_pb2.ProjectorConfig() - config.model_checkpoint_path = 'test' - emb1 = config.embeddings.add() - emb1.tensor_name = 'tensor1' - emb1.metadata_path = 'metadata1' - - # Call the API method to save the configuration to a temporary dir. - temp_dir = self.get_temp_dir() - self.addCleanup(shutil.rmtree, temp_dir) - writer = writer_lib.FileWriter(temp_dir) - projector.visualize_embeddings(writer, config) - - # Read the configurations from disk and make sure it matches the original. - with gfile.GFile(os.path.join(temp_dir, 'projector_config.pbtxt')) as f: - config2 = projector_config_pb2.ProjectorConfig() - text_format.Parse(f.read(), config2) - self.assertEqual(config, config2) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/tensorboard/plugins/projector/projector_config.proto b/tensorflow/contrib/tensorboard/plugins/projector/projector_config.proto deleted file mode 100644 index a9b18df9c1b..00000000000 --- a/tensorflow/contrib/tensorboard/plugins/projector/projector_config.proto +++ /dev/null @@ -1,46 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -syntax = "proto3"; - -package tensorflow; - -message SpriteMetadata { - string image_path = 1; - // [width, height] of a single image in the sprite. - repeated uint32 single_image_dim = 2; -} - -message EmbeddingInfo { - string tensor_name = 1; - string metadata_path = 2; - string bookmarks_path = 3; - // Shape of the 2D tensor [N x D]. If missing, it will be inferred from the - // model checkpoint. - repeated uint32 tensor_shape = 4; - SpriteMetadata sprite = 5; - // Path to the TSV file holding the tensor values. If missing, the tensor - // is assumed to be stored in the model checkpoint. - string tensor_path = 6; -} - -message ProjectorConfig { - // Path to the checkpoint file. Use either this or model_checkpoint_dir. - string model_checkpoint_path = 1; - repeated EmbeddingInfo embeddings = 2; - // Path to the checkpoint directory. The directory will be scanned for the - // latest checkpoint file. - string model_checkpoint_dir = 3; -} diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD deleted file mode 100644 index cc49cb86ac2..00000000000 --- a/tensorflow/contrib/tensorrt/BUILD +++ /dev/null @@ -1,65 +0,0 @@ -# Description: -# Wrap NVIDIA TensorRT (http://developer.nvidia.com/tensorrt) with tensorflow -# and provide TensorRT operators and converter package. -# APIs are meant to change over time. - -load( - "//tensorflow:tensorflow.bzl", - "tf_cuda_library", - "tf_custom_op_library_additional_deps", -) -load( - "@local_config_tensorrt//:build_defs.bzl", - "if_tensorrt", -) - -package( - default_visibility = ["//visibility:public"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -tf_cuda_library( - name = "trt_shape_function", - srcs = ["shape_fn/trt_shfn.cc"], - hdrs = ["shape_fn/trt_shfn.h"], - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/compiler/tf2tensorrt:trt_logging", - "//tensorflow/compiler/tf2tensorrt:trt_plugins", - ] + if_tensorrt([ - "@local_config_tensorrt//:tensorrt", - ]) + tf_custom_op_library_additional_deps(), -) - -py_library( - name = "init_py", - srcs = [ - "__init__.py", - "python/__init__.py", - "python/trt_convert.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/compiler/tensorrt:init_py", - ], -) - -# The following rules forward the libraries that were moved in order to not -# break other internal targets. - -alias( - name = "trt_conversion", - actual = "//tensorflow/compiler/tf2tensorrt:trt_conversion", -) - -alias( - name = "trt_op_kernels", - actual = "//tensorflow/compiler/tf2tensorrt:trt_op_kernels", -) - -alias( - name = "trt_engine_op_op_lib", - actual = "//tensorflow/compiler/tf2tensorrt:trt_engine_op_op_lib", -) diff --git a/tensorflow/contrib/tensorrt/__init__.py b/tensorflow/contrib/tensorrt/__init__.py deleted file mode 100644 index fd551d70b43..00000000000 --- a/tensorflow/contrib/tensorrt/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Exposes the python wrapper for TensorRT graph transforms.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.tensorrt.python import * -# pylint: enable=unused-import,wildcard-import diff --git a/tensorflow/contrib/tensorrt/python/__init__.py b/tensorflow/contrib/tensorrt/python/__init__.py deleted file mode 100644 index c29665b9a82..00000000000 --- a/tensorflow/contrib/tensorrt/python/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Exposes the python wrapper for TensorRT graph transforms.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,line-too-long -from tensorflow.contrib.tensorrt.python.trt_convert import create_inference_graph -# pylint: enable=unused-import,line-too-long diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py deleted file mode 100644 index eac1289a26f..00000000000 --- a/tensorflow/contrib/tensorrt/python/trt_convert.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Exposes the Python wrapper conversion to trt_graph.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.compiler.tensorrt import trt_convert - - -def create_inference_graph( - input_graph_def, - outputs, - max_batch_size=1, - max_workspace_size_bytes=trt_convert.DEFAULT_TRT_MAX_WORKSPACE_SIZE_BYTES, - precision_mode=trt_convert.TrtPrecisionMode.FP32, - minimum_segment_size=3, - is_dynamic_op=False, - maximum_cached_engines=1, - input_saved_model_dir=None, - input_saved_model_tags=None, - output_saved_model_dir=None, - session_config=None): - return trt_convert.create_inference_graph( - input_graph_def=input_graph_def, - outputs=outputs, - max_batch_size=max_batch_size, - max_workspace_size_bytes=max_workspace_size_bytes, - precision_mode=precision_mode, - minimum_segment_size=minimum_segment_size, - is_dynamic_op=is_dynamic_op, - maximum_cached_engines=maximum_cached_engines, - input_saved_model_dir=input_saved_model_dir, - input_saved_model_tags=input_saved_model_tags, - output_saved_model_dir=output_saved_model_dir, - session_config=session_config) diff --git a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc b/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc deleted file mode 100644 index c26f75ee3f2..00000000000 --- a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc +++ /dev/null @@ -1,78 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/tensorrt/shape_fn/trt_shfn.h" - -#include -#include - -#if GOOGLE_CUDA -#if GOOGLE_TENSORRT -#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h" -#include "tensorflow/core/lib/core/errors.h" -#include "third_party/tensorrt/NvInfer.h" - -namespace tensorflow { -namespace shape_inference { - -tensorflow::Status TRTEngineOpShapeInference(InferenceContext* c) { - for (int i = 0; i < c->num_outputs(); ++i) { - c->set_output(i, c->UnknownShape()); - } - - // Check the sanity of the input shapes. - std::vector input_shapes; - TF_RETURN_IF_ERROR(c->GetAttr("input_shapes", &input_shapes)); - if (input_shapes.size() != c->num_inputs()) { - return tensorflow::errors::InvalidArgument( - "The actual number of inputs doesn't match the number of input " - "shapes set in the attr: ", - c->num_inputs(), " vs ", input_shapes.size()); - } - bool input_match = true; - for (int i = 0; i < c->num_inputs(); ++i) { - ShapeHandle handle; - TF_RETURN_IF_ERROR( - c->MakeShapeFromTensorShape(input_shapes.at(i), &handle)); - ShapeHandle merged; - if (!c->Merge(c->input(i), handle, &merged).ok()) { - // Input shape doesn't match what was set in attr, fine. - input_match = false; - } - } - - // Check the sanity of the output shapes. - std::vector output_shapes; - TF_RETURN_IF_ERROR(c->GetAttr("output_shapes", &output_shapes)); - if (output_shapes.size() != c->num_outputs()) { - return tensorflow::errors::InvalidArgument( - "The actual number of outputs doesn't match the number of output " - "shapes set in the attr: ", - c->num_outputs(), " vs ", output_shapes.size()); - } - for (size_t i = 0; i < output_shapes.size(); ++i) { - ShapeHandle handle; - TF_RETURN_IF_ERROR( - c->MakeShapeFromTensorShape(output_shapes.at(i), &handle)); - if (input_match) c->set_output(i, handle); - } - return Status::OK(); -} - -} // namespace shape_inference -} // namespace tensorflow - -#endif // GOOGLE_TENSORRT -#endif // GOOGLE_CUDA diff --git a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.h b/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.h deleted file mode 100644 index 4b50f66699f..00000000000 --- a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.h +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_TENSORRT_SHAPE_FN_TRT_SHFN_H_ -#define TENSORFLOW_CONTRIB_TENSORRT_SHAPE_FN_TRT_SHFN_H_ - -#if GOOGLE_CUDA -#if GOOGLE_TENSORRT -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/lib/core/status.h" - -namespace tensorflow { -namespace shape_inference { -Status TRTEngineOpShapeInference(InferenceContext* c); -} // namespace shape_inference -} // namespace tensorflow - -#endif // GOOGLE_TENSORRT -#endif // GOOGLE_CUDA - -#endif // TENSORFLOW_CONTRIB_TENSORRT_SHAPE_FN_TRT_SHFN_H_ diff --git a/tensorflow/contrib/testing/BUILD b/tensorflow/contrib/testing/BUILD deleted file mode 100644 index 258026a6bdb..00000000000 --- a/tensorflow/contrib/testing/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "testing_py", - srcs = [ - "__init__.py", - "python/framework/fake_summary_writer.py", - "python/framework/util_test.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/core:protos_all_py", - "//tensorflow/python:summary", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/testing/__init__.py b/tensorflow/contrib/testing/__init__.py deleted file mode 100644 index a01e7c190a1..00000000000 --- a/tensorflow/contrib/testing/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Testing utilities.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.testing.python.framework.fake_summary_writer import * -from tensorflow.contrib.testing.python.framework.util_test import * -# pylint: enable=unused-import,wildcard-import diff --git a/tensorflow/contrib/testing/python/framework/fake_summary_writer.py b/tensorflow/contrib/testing/python/framework/fake_summary_writer.py deleted file mode 100644 index eac34afc4ad..00000000000 --- a/tensorflow/contrib/testing/python/framework/fake_summary_writer.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Fake summary writer for unit tests.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.core.framework import summary_pb2 -from tensorflow.python.framework import test_util -from tensorflow.python.summary.writer import writer -from tensorflow.python.summary.writer import writer_cache - - -# TODO(ptucker): Replace with mock framework. -class FakeSummaryWriter(object): - """Fake summary writer.""" - - _replaced_summary_writer = None - - @classmethod - def install(cls): - if cls._replaced_summary_writer: - raise ValueError('FakeSummaryWriter already installed.') - cls._replaced_summary_writer = writer.FileWriter - writer.FileWriter = FakeSummaryWriter - writer_cache.FileWriter = FakeSummaryWriter - - @classmethod - def uninstall(cls): - if not cls._replaced_summary_writer: - raise ValueError('FakeSummaryWriter not installed.') - writer.FileWriter = cls._replaced_summary_writer - writer_cache.FileWriter = cls._replaced_summary_writer - cls._replaced_summary_writer = None - - def __init__(self, logdir, graph=None): - self._logdir = logdir - self._graph = graph - self._summaries = {} - self._added_graphs = [] - self._added_meta_graphs = [] - self._added_session_logs = [] - self._added_run_metadata = {} - - @property - def summaries(self): - return self._summaries - - def assert_summaries(self, - test_case, - expected_logdir=None, - expected_graph=None, - expected_summaries=None, - expected_added_graphs=None, - expected_added_meta_graphs=None, - expected_session_logs=None): - """Assert expected items have been added to summary writer.""" - if expected_logdir is not None: - test_case.assertEqual(expected_logdir, self._logdir) - if expected_graph is not None: - test_case.assertTrue(expected_graph is self._graph) - expected_summaries = expected_summaries or {} - for step in expected_summaries: - test_case.assertTrue( - step in self._summaries, - msg='Missing step %s from %s.' % (step, self._summaries.keys())) - actual_simple_values = {} - for step_summary in self._summaries[step]: - for v in step_summary.value: - # Ignore global_step/sec since it's written by Supervisor in a - # separate thread, so it's non-deterministic how many get written. - if 'global_step/sec' != v.tag: - actual_simple_values[v.tag] = v.simple_value - test_case.assertEqual(expected_summaries[step], actual_simple_values) - if expected_added_graphs is not None: - test_case.assertEqual(expected_added_graphs, self._added_graphs) - if expected_added_meta_graphs is not None: - test_case.assertEqual(len(expected_added_meta_graphs), - len(self._added_meta_graphs)) - for expected, actual in zip(expected_added_meta_graphs, - self._added_meta_graphs): - test_util.assert_meta_graph_protos_equal(test_case, expected, actual) - if expected_session_logs is not None: - test_case.assertEqual(expected_session_logs, self._added_session_logs) - - def add_summary(self, summ, current_global_step): - """Add summary.""" - if isinstance(summ, bytes): - summary_proto = summary_pb2.Summary() - summary_proto.ParseFromString(summ) - summ = summary_proto - if current_global_step in self._summaries: - step_summaries = self._summaries[current_global_step] - else: - step_summaries = [] - self._summaries[current_global_step] = step_summaries - step_summaries.append(summ) - - # NOTE: Ignore global_step since its value is non-deterministic. - def add_graph(self, graph, global_step=None, graph_def=None): - """Add graph.""" - if (global_step is not None) and (global_step < 0): - raise ValueError('Invalid global_step %s.' % global_step) - if graph_def is not None: - raise ValueError('Unexpected graph_def %s.' % graph_def) - self._added_graphs.append(graph) - - def add_meta_graph(self, meta_graph_def, global_step=None): - """Add metagraph.""" - if (global_step is not None) and (global_step < 0): - raise ValueError('Invalid global_step %s.' % global_step) - self._added_meta_graphs.append(meta_graph_def) - - # NOTE: Ignore global_step since its value is non-deterministic. - def add_session_log(self, session_log, global_step=None): - # pylint: disable=unused-argument - self._added_session_logs.append(session_log) - - def add_run_metadata(self, run_metadata, tag, global_step=None): - if (global_step is not None) and (global_step < 0): - raise ValueError('Invalid global_step %s.' % global_step) - self._added_run_metadata[tag] = run_metadata - - def flush(self): - pass - - def reopen(self): - pass - - def close(self): - pass diff --git a/tensorflow/contrib/testing/python/framework/util_test.py b/tensorflow/contrib/testing/python/framework/util_test.py deleted file mode 100644 index 3ef48188d96..00000000000 --- a/tensorflow/contrib/testing/python/framework/util_test.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Test utilities.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import glob -import os -import numpy as np -from tensorflow.core.framework import summary_pb2 -from tensorflow.python.training import summary_io - - -def assert_summary(expected_tags, expected_simple_values, summary_proto): - """Asserts summary contains the specified tags and values. - - Args: - expected_tags: All tags in summary. - expected_simple_values: Simply values for some tags. - summary_proto: Summary to validate. - - Raises: - ValueError: if expectations are not met. - """ - actual_tags = set() - for value in summary_proto.value: - actual_tags.add(value.tag) - if value.tag in expected_simple_values: - expected = expected_simple_values[value.tag] - actual = value.simple_value - np.testing.assert_almost_equal( - actual, expected, decimal=2, err_msg=value.tag) - expected_tags = set(expected_tags) - if expected_tags != actual_tags: - raise ValueError('Expected tags %s, got %s.' % (expected_tags, actual_tags)) - - -def to_summary_proto(summary_str): - """Create summary based on latest stats. - - Args: - summary_str: Serialized summary. - Returns: - summary_pb2.Summary. - Raises: - ValueError: if tensor is not a valid summary tensor. - """ - summary = summary_pb2.Summary() - summary.ParseFromString(summary_str) - return summary - - -# TODO(ptucker): Move to a non-test package? -def latest_event_file(base_dir): - """Find latest event file in `base_dir`. - - Args: - base_dir: Base directory in which TF event flies are stored. - Returns: - File path, or `None` if none exists. - """ - file_paths = glob.glob(os.path.join(base_dir, 'events.*')) - return sorted(file_paths)[-1] if file_paths else None - - -def latest_events(base_dir): - """Parse events from latest event file in base_dir. - - Args: - base_dir: Base directory in which TF event flies are stored. - Returns: - Iterable of event protos. - Raises: - ValueError: if no event files exist under base_dir. - """ - file_path = latest_event_file(base_dir) - return summary_io.summary_iterator(file_path) if file_path else [] - - -def latest_summaries(base_dir): - """Parse summary events from latest event file in base_dir. - - Args: - base_dir: Base directory in which TF event flies are stored. - Returns: - List of event protos. - Raises: - ValueError: if no event files exist under base_dir. - """ - return [e for e in latest_events(base_dir) if e.HasField('summary')] - - -def simple_values_from_events(events, tags): - """Parse summaries from events with simple_value. - - Args: - events: List of tensorflow.Event protos. - tags: List of string event tags corresponding to simple_value summaries. - Returns: - dict of tag:value. - Raises: - ValueError: if a summary with a specified tag does not contain simple_value. - """ - step_by_tag = {} - value_by_tag = {} - for e in events: - if e.HasField('summary'): - for v in e.summary.value: - tag = v.tag - if tag in tags: - if not v.HasField('simple_value'): - raise ValueError('Summary for %s is not a simple_value.' % tag) - # The events are mostly sorted in step order, but we explicitly check - # just in case. - if tag not in step_by_tag or e.step > step_by_tag[tag]: - step_by_tag[tag] = e.step - value_by_tag[tag] = v.simple_value - return value_by_tag diff --git a/tensorflow/contrib/testing/testdata/mobilenet_224_gender_basic_fixed.mb b/tensorflow/contrib/testing/testdata/mobilenet_224_gender_basic_fixed.mb deleted file mode 100644 index fefe5f4d5f5..00000000000 Binary files a/tensorflow/contrib/testing/testdata/mobilenet_224_gender_basic_fixed.mb and /dev/null differ diff --git a/tensorflow/contrib/text/BUILD b/tensorflow/contrib/text/BUILD deleted file mode 100644 index d735ce8909d..00000000000 --- a/tensorflow/contrib/text/BUILD +++ /dev/null @@ -1,115 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which -# are not supported. - -load( - "//tensorflow:tensorflow.bzl", - "py_test", - "tf_custom_op_library", - "tf_gen_op_libs", - "tf_gen_op_wrapper_py", - "tf_kernel_library", -) -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = [ - "//learning/brain:__subpackages__", - "//tensorflow:__subpackages__", - ], - licenses = ["notice"], # Apache 2.0 -) - -tf_custom_op_py_library( - name = "text_py", - srcs = [ - "__init__.py", - "python/ops/__init__.py", - "python/ops/skip_gram_ops.py", - ], - dso = [ - ":python/ops/_skip_gram_ops.so", - ], - kernels = [ - ":all_kernels", - ":all_ops", - ], - srcs_version = "PY2AND3", - deps = [ - ":gen_skip_gram_ops", - "//tensorflow/contrib/lookup:lookup_py", - "//tensorflow/contrib/util:util_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:training", - "//tensorflow/python:util", - ], -) - -tf_kernel_library( - name = "skip_gram_kernels", - srcs = ["kernels/skip_gram_kernels.cc"], - deps = [ - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//third_party/eigen3", - ], - alwayslink = 1, -) - -cc_library( - name = "all_kernels", - deps = [":skip_gram_kernels"], -) - -tf_custom_op_library( - name = "python/ops/_skip_gram_ops.so", - srcs = [ - "kernels/skip_gram_kernels.cc", - "ops/skip_gram_ops.cc", - ], -) - -tf_gen_op_libs( - op_lib_names = ["skip_gram_ops"], -) - -cc_library( - name = "all_ops", - deps = [":skip_gram_ops_op_lib"], -) - -tf_gen_op_wrapper_py( - name = "gen_skip_gram_ops", - out = "python/ops/gen_skip_gram_ops.py", - deps = [":skip_gram_ops_op_lib"], -) - -py_test( - name = "skip_gram_ops_test", - size = "medium", - srcs = ["python/ops/skip_gram_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":text_py", - "//tensorflow/contrib/lookup:lookup_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:lookup_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:training", - ], -) diff --git a/tensorflow/contrib/text/__init__.py b/tensorflow/contrib/text/__init__.py deleted file mode 100644 index 35e66231890..00000000000 --- a/tensorflow/contrib/text/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Text-processing ops. - -@@skip_gram_sample -@@skip_gram_sample_with_text_vocab -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.text.python.ops import * -# pylint: enable=unused-import,wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented - -remove_undocumented(__name__) diff --git a/tensorflow/contrib/text/kernels/skip_gram_kernels.cc b/tensorflow/contrib/text/kernels/skip_gram_kernels.cc deleted file mode 100644 index 198388599e8..00000000000 --- a/tensorflow/contrib/text/kernels/skip_gram_kernels.cc +++ /dev/null @@ -1,139 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include -#include -#include - -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/random/philox_random.h" -#include "tensorflow/core/lib/random/simple_philox.h" -#include "tensorflow/core/util/guarded_philox_random.h" - -namespace tensorflow { - -template -class SkipGramGenerateCandidatesOp : public OpKernel { - public: - explicit SkipGramGenerateCandidatesOp(OpKernelConstruction* context) - : OpKernel(context) { - OP_REQUIRES_OK(context, generator_.Init(context)); - } - - void Compute(OpKernelContext* context) override { - const Tensor* input_tensor; - OP_REQUIRES_OK(context, context->input("input_tensor", &input_tensor)); - const auto input = input_tensor->flat(); - - const Tensor* min_skips_tensor; - OP_REQUIRES_OK(context, context->input("min_skips", &min_skips_tensor)); - const int min_skips = *(min_skips_tensor->scalar().data()); - const Tensor* max_skips_tensor; - OP_REQUIRES_OK(context, context->input("max_skips", &max_skips_tensor)); - const int max_skips = *(max_skips_tensor->scalar().data()); - - OP_REQUIRES( - context, min_skips >= 0 && max_skips >= 0, - errors::InvalidArgument("Both min_skips and max_skips must be >= 0.")); - OP_REQUIRES(context, min_skips <= max_skips, - errors::InvalidArgument("min_skips must be <= max_skips.")); - - const Tensor* start_tensor; - OP_REQUIRES_OK(context, context->input("start", &start_tensor)); - const int start = *(start_tensor->scalar().data()); - const Tensor* limit_tensor; - OP_REQUIRES_OK(context, context->input("limit", &limit_tensor)); - const int limit = *(limit_tensor->scalar().data()); - const int end = - limit < 0 ? input.size() - : std::min(start + limit, static_cast(input.size())); - - const Tensor* emit_self_tensor; - OP_REQUIRES_OK(context, - context->input("emit_self_as_target", &emit_self_tensor)); - const bool emit_self_as_target = *(emit_self_tensor->scalar().data()); - - std::vector tokens; - std::vector labels; - - // Reserve the number of random numbers we will use - we use one for each - // token between start and end. - random::PhiloxRandom local_gen = - generator_.ReserveSamples32(end - start + 1); - random::SimplePhilox rng(&local_gen); - - // For each token in the sentence, pick a random skip, then generates - // (token, label) pairs for all labels whose distances from the token are - // within the range [-skip, skip]. - for (int i = start; i < end; ++i) { - const int skips = min_skips + rng.Uniform(max_skips - min_skips + 1); - for (int j = -skips; j <= skips; ++j) { - if ((i + j < start) || (i + j >= end) || - (j == 0 && !emit_self_as_target)) { - continue; - } - tokens.push_back(input(i)); - labels.push_back(input(i + j)); - } - } - - Tensor* tokens_output = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output( - "tokens", TensorShape({static_cast(tokens.size())}), - &tokens_output)); - Tensor* labels_output = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output( - "labels", TensorShape({static_cast(labels.size())}), - &labels_output)); - OP_REQUIRES( - context, tokens_output->IsSameSize(*labels_output), - errors::Internal(strings::StrCat( - "Mismatch between tokens_output shape of ", - tokens_output->shape().DebugString(), - " and labels_output shape of ", - labels_output->shape().DebugString(), - ". This should never happen - contact ami-team@ if it does."))); - - // Copies results to output tensors. - for (int i = 0; i < tokens.size(); ++i) { - tokens_output->vec()(i) = tokens[i]; - labels_output->vec()(i) = labels[i]; - } - } - - private: - GuardedPhiloxRandom generator_; -}; - -#define REGISTER_KERNEL(type) \ - REGISTER_KERNEL_BUILDER(Name("SkipGramGenerateCandidates") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T"), \ - SkipGramGenerateCandidatesOp) - -REGISTER_KERNEL(tstring); -REGISTER_KERNEL(int64); -REGISTER_KERNEL(int32); -REGISTER_KERNEL(int16); -// TODO(weiho): Add other types if the need arises. - -#undef REGISTER_KERNEL - -} // namespace tensorflow diff --git a/tensorflow/contrib/text/ops/skip_gram_ops.cc b/tensorflow/contrib/text/ops/skip_gram_ops.cc deleted file mode 100644 index 9a7a20d81a9..00000000000 --- a/tensorflow/contrib/text/ops/skip_gram_ops.cc +++ /dev/null @@ -1,54 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/shape_inference.h" - -namespace tensorflow { -REGISTER_OP("SkipGramGenerateCandidates") - .Input("input_tensor: T") - .Input("min_skips: int32") - .Input("max_skips: int32") - .Input("start: int32") - .Input("limit: int32") - .Input("emit_self_as_target: bool") - .Output("tokens: T") - .Output("labels: T") - .Attr("T: type") - // The seed attributes are needed by GuardedPhiloxRandom - .Attr("seed: int = 0") - .Attr("seed2: int = 0") - .SetIsStateful() - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused; - // input_tensor must be of rank-1. - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &unused)); - // All other args must be scalar. - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused)); - - // Due to possible randomness in selecting skips, we only know that the - // outputs will be of rank-1, but not their sizes. - c->set_output(0, c->Vector(c->UnknownDim())); - c->set_output(1, c->Vector(c->UnknownDim())); - return Status::OK(); - }) - .Doc(R"doc( -Generates skip-gram token and label paired Tensors from the input tensor. -See docs for the public-facing skip_gram_sample() Python op for more details. -)doc"); -} // namespace tensorflow diff --git a/tensorflow/contrib/text/python/ops/__init__.py b/tensorflow/contrib/text/python/ops/__init__.py deleted file mode 100644 index bb47266dd2b..00000000000 --- a/tensorflow/contrib/text/python/ops/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Various contrib ops related to text-processing.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.text.python.ops.skip_gram_ops import skip_gram_sample -from tensorflow.contrib.text.python.ops.skip_gram_ops import skip_gram_sample_with_text_vocab diff --git a/tensorflow/contrib/text/python/ops/skip_gram_ops.py b/tensorflow/contrib/text/python/ops/skip_gram_ops.py deleted file mode 100644 index 7ed45031a3f..00000000000 --- a/tensorflow/contrib/text/python/ops/skip_gram_ops.py +++ /dev/null @@ -1,449 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Skip-gram sampling ops from https://arxiv.org/abs/1301.3781.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import csv - -from tensorflow.contrib import lookup -from tensorflow.contrib.text.python.ops import gen_skip_gram_ops -from tensorflow.contrib.util import loader -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.platform import gfile -from tensorflow.python.platform import resource_loader -from tensorflow.python.training import input as input_ops - -_checkpoint_ops_so = loader.load_op_library( - resource_loader.get_path_to_datafile("_skip_gram_ops.so")) - -ops.NotDifferentiable("SkipGramGenerateCandidates") - - -def skip_gram_sample(input_tensor, - min_skips=1, - max_skips=5, - start=0, - limit=-1, - emit_self_as_target=False, - vocab_freq_table=None, - vocab_min_count=None, - vocab_subsampling=None, - corpus_size=None, - batch_size=None, - batch_capacity=None, - seed=None, - name=None): - """Generates skip-gram token and label paired Tensors from the input tensor. - - Generates skip-gram `("token", "label")` pairs using each element in the - rank-1 `input_tensor` as a token. The window size used for each token will be - randomly selected from the range specified by `[min_skips, max_skips]`, - inclusive. See https://arxiv.org/abs/1301.3781 for more details about - skip-gram. - - For example, given `input_tensor = ["the", "quick", "brown", "fox", "jumps"]`, - `min_skips = 1`, `max_skips = 2`, `emit_self_as_target = False`, the output - `(tokens, labels)` pairs for the token "quick" will be randomly selected from - either `(tokens=["quick", "quick"], labels=["the", "brown"])` for 1 skip, or - `(tokens=["quick", "quick", "quick"], labels=["the", "brown", "fox"])` for 2 - skips. - - If `emit_self_as_target = True`, each token will also be emitted as a label - for itself. From the previous example, the output will be either - `(tokens=["quick", "quick", "quick"], labels=["the", "quick", "brown"])` for 1 - skip, or `(tokens=["quick", "quick", "quick", "quick"], labels=["the", - "quick", "brown", "fox"])` for 2 skips. - - The same process is repeated for each element of `input_tensor` and - concatenated together into the two output rank-1 `Tensors` (one for all the - tokens, another for all the labels). - - If `vocab_freq_table` is specified, tokens in `input_tensor` that are not - present in the vocabulary are discarded. Tokens whose frequency counts are - below `vocab_min_count` are also discarded. Tokens whose frequency proportions - in the corpus exceed `vocab_subsampling` may be randomly down-sampled. See - Eq. 5 in http://arxiv.org/abs/1310.4546 for more details about subsampling. - - Due to the random window sizes used for each token, the lengths of the outputs - are non-deterministic, unless `batch_size` is specified to batch the outputs - to always return `Tensors` of length `batch_size`. - - Args: - input_tensor: A rank-1 `Tensor` from which to generate skip-gram candidates. - min_skips: `int` or scalar `Tensor` specifying the minimum window size to - randomly use for each token. Must be >= 0 and <= `max_skips`. If - `min_skips` and `max_skips` are both 0, the only label outputted will be - the token itself when `emit_self_as_target = True` - or no output - otherwise. - max_skips: `int` or scalar `Tensor` specifying the maximum window size to - randomly use for each token. Must be >= 0. - start: `int` or scalar `Tensor` specifying the position in - `input_tensor` from which to start generating skip-gram candidates. - limit: `int` or scalar `Tensor` specifying the maximum number of - elements in `input_tensor` to use in generating skip-gram candidates. -1 - means to use the rest of the `Tensor` after `start`. - emit_self_as_target: `bool` or scalar `Tensor` specifying whether to emit - each token as a label for itself. - vocab_freq_table: (Optional) A lookup table (subclass of - `lookup.InitializableLookupTableBase`) that maps tokens to their raw - frequency counts. If specified, any token in `input_tensor` that is not - found in `vocab_freq_table` will be filtered out before generating - skip-gram candidates. While this will typically map to integer raw - frequency counts, it could also map to float frequency proportions. - `vocab_min_count` and `corpus_size` should be in the same units as this. - vocab_min_count: (Optional) `int`, `float`, or scalar `Tensor` specifying - minimum frequency threshold (from `vocab_freq_table`) for a token to be - kept in `input_tensor`. If this is specified, `vocab_freq_table` must also - be specified - and they should both be in the same units. - vocab_subsampling: (Optional) `float` specifying frequency proportion - threshold for tokens from `input_tensor`. Tokens that occur more - frequently (based on the ratio of the token's `vocab_freq_table` value to - the `corpus_size`) will be randomly down-sampled. Reasonable starting - values may be around 1e-3 or 1e-5. If this is specified, both - `vocab_freq_table` and `corpus_size` must also be specified. See Eq. 5 - in http://arxiv.org/abs/1310.4546 for more details. - corpus_size: (Optional) `int`, `float`, or scalar `Tensor` specifying the - total number of tokens in the corpus (e.g., sum of all the frequency - counts of `vocab_freq_table`). Used with `vocab_subsampling` for - down-sampling frequently occurring tokens. If this is specified, - `vocab_freq_table` and `vocab_subsampling` must also be specified. - batch_size: (Optional) `int` specifying batch size of returned `Tensors`. - batch_capacity: (Optional) `int` specifying batch capacity for the queue - used for batching returned `Tensors`. Only has an effect if - `batch_size` > 0. Defaults to 100 * `batch_size` if not specified. - seed: (Optional) `int` used to create a random seed for window size and - subsampling. See `set_random_seed` docs for behavior. - name: (Optional) A `string` name or a name scope for the operations. - - Returns: - A `tuple` containing (token, label) `Tensors`. Each output `Tensor` is of - rank-1 and has the same type as `input_tensor`. The `Tensors` will be of - length `batch_size`; if `batch_size` is not specified, they will be of - random length, though they will be in sync with each other as long as they - are evaluated together. - - Raises: - ValueError: If `vocab_freq_table` is not provided, but `vocab_min_count`, - `vocab_subsampling`, or `corpus_size` is specified. If `vocab_subsampling` - and `corpus_size` are not both present or both absent. - """ - - if vocab_freq_table is None and (vocab_min_count is not None or - vocab_subsampling is not None or - corpus_size is not None): - raise ValueError( - "vocab_freq_table is not provided, but vocab_min_count={}, " - "vocab_subsampling={}, or corpus_size={} is not None. These settings " - "are useless without a vocab_freq_table.".format( - vocab_min_count, vocab_subsampling, corpus_size)) - - if (vocab_subsampling is None) != (corpus_size is None): - raise ValueError( - "vocab_subsampling is {} while corpus_size is {} - both must be " - "provided in order for subsampling to work.".format( - vocab_subsampling, corpus_size)) - - with ops.name_scope( - name, - "skip_gram_sample", - values=[input_tensor, min_skips, max_skips, start, limit]): - - input_tensor = _filter_input( - input_tensor=input_tensor, - vocab_freq_table=vocab_freq_table, - vocab_min_count=vocab_min_count, - vocab_subsampling=vocab_subsampling, - corpus_size=corpus_size, - seed=seed) - - seed1, seed2 = random_seed.get_seed(seed) - tokens, labels = gen_skip_gram_ops.skip_gram_generate_candidates( - input_tensor=input_tensor, - min_skips=min_skips, - max_skips=max_skips, - start=start, - limit=limit, - emit_self_as_target=emit_self_as_target, - # Note that seed here should be seed1! This is due to - # GuardedPhiloxRandom's hard-coded attributes of "seed" and "seed2". - seed=seed1, - seed2=seed2) - - # TODO(weiho): If the need arises, add support for sparse input_tensor that - # figures out sentence boundaries, then calls - # skip_gram_generate_candidates() on each sentence. - - # Batches the (tokens, labels) outputs so that they will be of deterministic - # batch_size, to facilitate feeding them into the rest of the network. - if batch_size is not None and batch_size > 0: - batch_capacity = (batch_capacity - if (batch_capacity is not None and batch_capacity > 0) - else 100 * batch_size) - return input_ops.batch( - [tokens, labels], - batch_size, - capacity=batch_capacity, - enqueue_many=True) - - return tokens, labels - - -def skip_gram_sample_with_text_vocab(input_tensor, - vocab_freq_file, - vocab_token_index=0, - vocab_token_dtype=dtypes.string, - vocab_freq_index=1, - vocab_freq_dtype=dtypes.float64, - vocab_delimiter=",", - vocab_min_count=0, - vocab_subsampling=None, - corpus_size=None, - min_skips=1, - max_skips=5, - start=0, - limit=-1, - emit_self_as_target=False, - batch_size=None, - batch_capacity=None, - seed=None, - name=None): - """Skip-gram sampling with a text vocabulary file. - - Wrapper around `skip_gram_sample()` for use with a text vocabulary file. The - vocabulary file is expected to be a plain-text file, with lines of - `vocab_delimiter`-separated columns. The `vocab_token_index` column should - contain the vocabulary term, while the `vocab_freq_index` column should - contain the number of times that term occurs in the corpus. For example, with - a text vocabulary file of: - - ``` - bonjour,fr,42 - hello,en,777 - hola,es,99 - ``` - - You should set `vocab_delimiter=","`, `vocab_token_index=0`, and - `vocab_freq_index=2`. - - See `skip_gram_sample()` documentation for more details about the skip-gram - sampling process. - - Args: - input_tensor: A rank-1 `Tensor` from which to generate skip-gram candidates. - vocab_freq_file: `string` specifying full file path to the text vocab file. - vocab_token_index: `int` specifying which column in the text vocab file - contains the tokens. - vocab_token_dtype: `DType` specifying the format of the tokens in the text - vocab file. - vocab_freq_index: `int` specifying which column in the text vocab file - contains the frequency counts of the tokens. - vocab_freq_dtype: `DType` specifying the format of the frequency counts in - the text vocab file. - vocab_delimiter: `string` specifying the delimiter used in the text vocab - file. - vocab_min_count: `int`, `float`, or scalar `Tensor` specifying - minimum frequency threshold (from `vocab_freq_file`) for a token to be - kept in `input_tensor`. This should correspond with `vocab_freq_dtype`. - vocab_subsampling: (Optional) `float` specifying frequency proportion - threshold for tokens from `input_tensor`. Tokens that occur more - frequently will be randomly down-sampled. Reasonable starting values may - be around 1e-3 or 1e-5. See Eq. 5 in http://arxiv.org/abs/1310.4546 for - more details. - corpus_size: (Optional) `int`, `float`, or scalar `Tensor` specifying the - total number of tokens in the corpus (e.g., sum of all the frequency - counts of `vocab_freq_file`). Used with `vocab_subsampling` for - down-sampling frequently occurring tokens. If this is specified, - `vocab_freq_file` and `vocab_subsampling` must also be specified. - If `corpus_size` is needed but not supplied, then it will be calculated - from `vocab_freq_file`. You might want to supply your own value if you - have already eliminated infrequent tokens from your vocabulary files - (where frequency < vocab_min_count) to save memory in the internal token - lookup table. Otherwise, the unused tokens' variables will waste memory. - The user-supplied `corpus_size` value must be greater than or equal to the - sum of all the frequency counts of `vocab_freq_file`. - min_skips: `int` or scalar `Tensor` specifying the minimum window size to - randomly use for each token. Must be >= 0 and <= `max_skips`. If - `min_skips` and `max_skips` are both 0, the only label outputted will be - the token itself. - max_skips: `int` or scalar `Tensor` specifying the maximum window size to - randomly use for each token. Must be >= 0. - start: `int` or scalar `Tensor` specifying the position in `input_tensor` - from which to start generating skip-gram candidates. - limit: `int` or scalar `Tensor` specifying the maximum number of elements in - `input_tensor` to use in generating skip-gram candidates. -1 means to use - the rest of the `Tensor` after `start`. - emit_self_as_target: `bool` or scalar `Tensor` specifying whether to emit - each token as a label for itself. - batch_size: (Optional) `int` specifying batch size of returned `Tensors`. - batch_capacity: (Optional) `int` specifying batch capacity for the queue - used for batching returned `Tensors`. Only has an effect if - `batch_size` > 0. Defaults to 100 * `batch_size` if not specified. - seed: (Optional) `int` used to create a random seed for window size and - subsampling. See - [`set_random_seed`](../../g3doc/python/constant_op.md#set_random_seed) - for behavior. - name: (Optional) A `string` name or a name scope for the operations. - - Returns: - A `tuple` containing (token, label) `Tensors`. Each output `Tensor` is of - rank-1 and has the same type as `input_tensor`. The `Tensors` will be of - length `batch_size`; if `batch_size` is not specified, they will be of - random length, though they will be in sync with each other as long as they - are evaluated together. - - Raises: - ValueError: If `vocab_token_index` or `vocab_freq_index` is less than 0 or - exceeds the number of columns in `vocab_freq_file`. If `vocab_token_index` - and `vocab_freq_index` are both set to the same column. If any token in - `vocab_freq_file` has a negative frequency. - """ - - if vocab_token_index < 0 or vocab_freq_index < 0: - raise ValueError( - "vocab_token_index={} and vocab_freq_index={} must both be >= 0.". - format(vocab_token_index, vocab_freq_index)) - if vocab_token_index == vocab_freq_index: - raise ValueError( - "vocab_token_index and vocab_freq_index should be different, but are " - "both {}.".format(vocab_token_index)) - - # Iterates through the vocab file and calculates the number of vocab terms as - # well as the total corpus size (by summing the frequency counts of all the - # vocab terms). - calculated_corpus_size = 0.0 - vocab_size = 0 - with gfile.GFile(vocab_freq_file, mode="r") as f: - reader = csv.reader(f, delimiter=vocab_delimiter) - for row in reader: - if vocab_token_index >= len(row) or vocab_freq_index >= len(row): - raise ValueError( - "Row in vocab file only has {} columns, so vocab_token_index={} or " - "vocab_freq_index={} is out of bounds. Row content: {}".format( - len(row), vocab_token_index, vocab_freq_index, row)) - vocab_size += 1 - freq = vocab_freq_dtype.as_numpy_dtype(row[vocab_freq_index]) - if freq < 0: - raise ValueError( - "Row in vocab file has negative frequency of {}. Row content: {}". - format(freq, row)) - # Note: tokens whose frequencies are below vocab_min_count will still - # contribute to the total corpus size used for vocab subsampling. - calculated_corpus_size += freq - - if not corpus_size: - corpus_size = calculated_corpus_size - elif calculated_corpus_size - corpus_size > 1e-6: - raise ValueError( - "`corpus_size`={} must be greater than or equal to the sum of all the " - "frequency counts ({}) of `vocab_freq_file` ({}).".format( - corpus_size, calculated_corpus_size, vocab_freq_file)) - - vocab_freq_table = lookup.HashTable( - lookup.TextFileInitializer( - filename=vocab_freq_file, - key_dtype=vocab_token_dtype, - key_index=vocab_token_index, - value_dtype=vocab_freq_dtype, - value_index=vocab_freq_index, - vocab_size=vocab_size, - delimiter=vocab_delimiter), - # For vocab terms not in vocab file, use a default value of -1. - default_value=-1) - - return skip_gram_sample( - input_tensor, - min_skips=min_skips, - max_skips=max_skips, - start=start, - limit=limit, - emit_self_as_target=emit_self_as_target, - vocab_freq_table=vocab_freq_table, - vocab_min_count=vocab_min_count, - vocab_subsampling=vocab_subsampling, - # corpus_size is not used unless vocab_subsampling is specified. - corpus_size=None if vocab_subsampling is None else corpus_size, - batch_size=batch_size, - batch_capacity=batch_capacity, - seed=seed, - name=name) - - -def _filter_input(input_tensor, vocab_freq_table, vocab_min_count, - vocab_subsampling, corpus_size, seed): - """Filters input tensor based on vocab freq, threshold, and subsampling.""" - if vocab_freq_table is None: - return input_tensor - - if not isinstance(vocab_freq_table, lookup.InitializableLookupTableBase): - raise ValueError( - "vocab_freq_table must be a subclass of " - "InitializableLookupTableBase (such as HashTable) instead of type " - "{}.".format(type(vocab_freq_table))) - - with ops.name_scope( - "filter_vocab", values=[vocab_freq_table, input_tensor, vocab_min_count]): - freq = vocab_freq_table.lookup(input_tensor) - # Filters out elements in input_tensor that are not found in - # vocab_freq_table (table returns a default value of -1 specified above when - # an element is not found). - mask = math_ops.not_equal(freq, vocab_freq_table.default_value) - - # Filters out elements whose vocab frequencies are less than the threshold. - if vocab_min_count is not None: - cast_threshold = math_ops.cast(vocab_min_count, freq.dtype) - mask = math_ops.logical_and(mask, - math_ops.greater_equal(freq, cast_threshold)) - - input_tensor = array_ops.boolean_mask(input_tensor, mask) - freq = array_ops.boolean_mask(freq, mask) - - if not vocab_subsampling: - return input_tensor - - if vocab_subsampling < 0 or vocab_subsampling > 1: - raise ValueError( - "Invalid vocab_subsampling={} - it should be within range [0, 1].". - format(vocab_subsampling)) - - # Subsamples the input tokens based on vocabulary frequency and - # vocab_subsampling threshold (ie randomly discard commonly appearing - # tokens). - with ops.name_scope( - "subsample_vocab", values=[input_tensor, freq, vocab_subsampling]): - corpus_size = math_ops.cast(corpus_size, dtypes.float64) - freq = math_ops.cast(freq, dtypes.float64) - vocab_subsampling = math_ops.cast(vocab_subsampling, dtypes.float64) - - # From tensorflow_models/tutorials/embedding/word2vec_kernels.cc, which is - # suppose to correlate with Eq. 5 in http://arxiv.org/abs/1310.4546. - keep_prob = ((math_ops.sqrt(freq / - (vocab_subsampling * corpus_size)) + 1.0) * - (vocab_subsampling * corpus_size / freq)) - random_prob = random_ops.random_uniform( - array_ops.shape(freq), - minval=0, - maxval=1, - dtype=dtypes.float64, - seed=seed) - - mask = math_ops.less_equal(random_prob, keep_prob) - return array_ops.boolean_mask(input_tensor, mask) diff --git a/tensorflow/contrib/text/python/ops/skip_gram_ops_test.py b/tensorflow/contrib/text/python/ops/skip_gram_ops_test.py deleted file mode 100644 index 49260f272ee..00000000000 --- a/tensorflow/contrib/text/python/ops/skip_gram_ops_test.py +++ /dev/null @@ -1,606 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Skip-gram sampling ops tests.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import csv -import os - -from tensorflow.contrib import lookup -from tensorflow.contrib import text -from tensorflow.contrib.text.python.ops import skip_gram_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test -from tensorflow.python.training import coordinator -from tensorflow.python.training import queue_runner_impl - - -class SkipGramOpsTest(test.TestCase): - - def _split_tokens_labels(self, output): - tokens = [x[0] for x in output] - labels = [x[1] for x in output] - return tokens, labels - - def test_skip_gram_sample_skips_2(self): - """Tests skip-gram with min_skips = max_skips = 2.""" - input_tensor = constant_op.constant( - [b"the", b"quick", b"brown", b"fox", b"jumps"]) - tokens, labels = text.skip_gram_sample( - input_tensor, min_skips=2, max_skips=2) - expected_tokens, expected_labels = self._split_tokens_labels([ - (b"the", b"quick"), - (b"the", b"brown"), - (b"quick", b"the"), - (b"quick", b"brown"), - (b"quick", b"fox"), - (b"brown", b"the"), - (b"brown", b"quick"), - (b"brown", b"fox"), - (b"brown", b"jumps"), - (b"fox", b"quick"), - (b"fox", b"brown"), - (b"fox", b"jumps"), - (b"jumps", b"brown"), - (b"jumps", b"fox"), - ]) - with self.cached_session(): - self.assertAllEqual(expected_tokens, tokens.eval()) - self.assertAllEqual(expected_labels, labels.eval()) - - def test_skip_gram_sample_emit_self(self): - """Tests skip-gram with emit_self_as_target = True.""" - input_tensor = constant_op.constant( - [b"the", b"quick", b"brown", b"fox", b"jumps"]) - tokens, labels = text.skip_gram_sample( - input_tensor, min_skips=2, max_skips=2, emit_self_as_target=True) - expected_tokens, expected_labels = self._split_tokens_labels([ - (b"the", b"the"), - (b"the", b"quick"), - (b"the", b"brown"), - (b"quick", b"the"), - (b"quick", b"quick"), - (b"quick", b"brown"), - (b"quick", b"fox"), - (b"brown", b"the"), - (b"brown", b"quick"), - (b"brown", b"brown"), - (b"brown", b"fox"), - (b"brown", b"jumps"), - (b"fox", b"quick"), - (b"fox", b"brown"), - (b"fox", b"fox"), - (b"fox", b"jumps"), - (b"jumps", b"brown"), - (b"jumps", b"fox"), - (b"jumps", b"jumps"), - ]) - with self.cached_session(): - self.assertAllEqual(expected_tokens, tokens.eval()) - self.assertAllEqual(expected_labels, labels.eval()) - - def test_skip_gram_sample_skips_0(self): - """Tests skip-gram with min_skips = max_skips = 0.""" - input_tensor = constant_op.constant([b"the", b"quick", b"brown"]) - - # If emit_self_as_target is False (default), output will be empty. - tokens, labels = text.skip_gram_sample( - input_tensor, min_skips=0, max_skips=0, emit_self_as_target=False) - with self.cached_session(): - self.assertEqual(0, tokens.eval().size) - self.assertEqual(0, labels.eval().size) - - # If emit_self_as_target is True, each token will be its own label. - tokens, labels = text.skip_gram_sample( - input_tensor, min_skips=0, max_skips=0, emit_self_as_target=True) - expected_tokens, expected_labels = self._split_tokens_labels([ - (b"the", b"the"), - (b"quick", b"quick"), - (b"brown", b"brown"), - ]) - with self.cached_session(): - self.assertAllEqual(expected_tokens, tokens.eval()) - self.assertAllEqual(expected_labels, labels.eval()) - - def test_skip_gram_sample_skips_exceed_length(self): - """Tests skip-gram when min/max_skips exceed length of input.""" - input_tensor = constant_op.constant([b"the", b"quick", b"brown"]) - tokens, labels = text.skip_gram_sample( - input_tensor, min_skips=100, max_skips=100) - expected_tokens, expected_labels = self._split_tokens_labels([ - (b"the", b"quick"), - (b"the", b"brown"), - (b"quick", b"the"), - (b"quick", b"brown"), - (b"brown", b"the"), - (b"brown", b"quick"), - ]) - with self.cached_session(): - self.assertAllEqual(expected_tokens, tokens.eval()) - self.assertAllEqual(expected_labels, labels.eval()) - - def test_skip_gram_sample_start_limit(self): - """Tests skip-gram over a limited portion of the input.""" - input_tensor = constant_op.constant( - [b"foo", b"the", b"quick", b"brown", b"bar"]) - tokens, labels = text.skip_gram_sample( - input_tensor, min_skips=1, max_skips=1, start=1, limit=3) - expected_tokens, expected_labels = self._split_tokens_labels([ - (b"the", b"quick"), - (b"quick", b"the"), - (b"quick", b"brown"), - (b"brown", b"quick"), - ]) - with self.cached_session(): - self.assertAllEqual(expected_tokens, tokens.eval()) - self.assertAllEqual(expected_labels, labels.eval()) - - def test_skip_gram_sample_limit_exceeds(self): - """Tests skip-gram when limit exceeds the length of the input.""" - input_tensor = constant_op.constant([b"foo", b"the", b"quick", b"brown"]) - tokens, labels = text.skip_gram_sample( - input_tensor, min_skips=1, max_skips=1, start=1, limit=100) - expected_tokens, expected_labels = self._split_tokens_labels([ - (b"the", b"quick"), - (b"quick", b"the"), - (b"quick", b"brown"), - (b"brown", b"quick"), - ]) - with self.cached_session(): - self.assertAllEqual(expected_tokens, tokens.eval()) - self.assertAllEqual(expected_labels, labels.eval()) - - def test_skip_gram_sample_random_skips(self): - """Tests skip-gram with min_skips != max_skips, with random output.""" - # The number of outputs is non-deterministic in this case, so set random - # seed to help ensure the outputs remain constant for this test case. - random_seed.set_random_seed(42) - - input_tensor = constant_op.constant( - [b"the", b"quick", b"brown", b"fox", b"jumps", b"over"]) - tokens, labels = text.skip_gram_sample( - input_tensor, min_skips=1, max_skips=2, seed=9) - expected_tokens, expected_labels = self._split_tokens_labels([ - (b"the", b"quick"), - (b"the", b"brown"), - (b"quick", b"the"), - (b"quick", b"brown"), - (b"quick", b"fox"), - (b"brown", b"the"), - (b"brown", b"quick"), - (b"brown", b"fox"), - (b"brown", b"jumps"), - (b"fox", b"brown"), - (b"fox", b"jumps"), - (b"jumps", b"fox"), - (b"jumps", b"over"), - (b"over", b"fox"), - (b"over", b"jumps"), - ]) - with self.cached_session() as sess: - tokens_eval, labels_eval = sess.run([tokens, labels]) - self.assertAllEqual(expected_tokens, tokens_eval) - self.assertAllEqual(expected_labels, labels_eval) - - def test_skip_gram_sample_random_skips_default_seed(self): - """Tests outputs are still random when no op-level seed is specified.""" - # This is needed since tests set a graph-level seed by default. We want to - # explicitly avoid setting both graph-level seed and op-level seed, to - # simulate behavior under non-test settings when the user doesn't provide a - # seed to us. This results in random_seed.get_seed() returning None for both - # seeds, forcing the C++ kernel to execute its default seed logic. - random_seed.set_random_seed(None) - - # Uses an input tensor with 10 words, with possible skip ranges in [1, - # 5]. Thus, the probability that two random samplings would result in the - # same outputs is 1/5^10 ~ 1e-7 (aka the probability of this test being - # flaky). - input_tensor = constant_op.constant([str(x) for x in range(10)]) - - # Do not provide an op-level seed here! - tokens_1, labels_1 = text.skip_gram_sample( - input_tensor, min_skips=1, max_skips=5) - tokens_2, labels_2 = text.skip_gram_sample( - input_tensor, min_skips=1, max_skips=5) - - with self.cached_session() as sess: - tokens_1_eval, labels_1_eval, tokens_2_eval, labels_2_eval = sess.run( - [tokens_1, labels_1, tokens_2, labels_2]) - - if len(tokens_1_eval) == len(tokens_2_eval): - self.assertNotEqual(tokens_1_eval.tolist(), tokens_2_eval.tolist()) - if len(labels_1_eval) == len(labels_2_eval): - self.assertNotEqual(labels_1_eval.tolist(), labels_2_eval.tolist()) - - def test_skip_gram_sample_batch(self): - """Tests skip-gram with batching.""" - input_tensor = constant_op.constant([b"the", b"quick", b"brown", b"fox"]) - tokens, labels = text.skip_gram_sample( - input_tensor, min_skips=1, max_skips=1, batch_size=3) - expected_tokens, expected_labels = self._split_tokens_labels([ - (b"the", b"quick"), - (b"quick", b"the"), - (b"quick", b"brown"), - (b"brown", b"quick"), - (b"brown", b"fox"), - (b"fox", b"brown"), - ]) - with self.cached_session() as sess: - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(sess=sess, coord=coord) - - tokens_eval, labels_eval = sess.run([tokens, labels]) - self.assertAllEqual(expected_tokens[:3], tokens_eval) - self.assertAllEqual(expected_labels[:3], labels_eval) - tokens_eval, labels_eval = sess.run([tokens, labels]) - self.assertAllEqual(expected_tokens[3:6], tokens_eval) - self.assertAllEqual(expected_labels[3:6], labels_eval) - - coord.request_stop() - coord.join(threads) - - def test_skip_gram_sample_non_string_input(self): - """Tests skip-gram with non-string input.""" - input_tensor = constant_op.constant([1, 2, 3], dtype=dtypes.int16) - tokens, labels = text.skip_gram_sample( - input_tensor, min_skips=1, max_skips=1) - expected_tokens, expected_labels = self._split_tokens_labels([ - (1, 2), - (2, 1), - (2, 3), - (3, 2), - ]) - with self.cached_session(): - self.assertAllEqual(expected_tokens, tokens.eval()) - self.assertAllEqual(expected_labels, labels.eval()) - - def test_skip_gram_sample_errors(self): - """Tests various errors raised by skip_gram_sample().""" - input_tensor = constant_op.constant([b"the", b"quick", b"brown"]) - - invalid_skips = ( - # min_skips and max_skips must be >= 0. - (-1, 2), - (1, -2), - # min_skips must be <= max_skips. - (2, 1)) - for min_skips, max_skips in invalid_skips: - tokens, labels = text.skip_gram_sample( - input_tensor, min_skips=min_skips, max_skips=max_skips) - with self.cached_session() as sess, self.assertRaises( - errors.InvalidArgumentError): - sess.run([tokens, labels]) - - # input_tensor must be of rank 1. - with self.assertRaises(ValueError): - invalid_tensor = constant_op.constant([[b"the"], [b"quick"], [b"brown"]]) - text.skip_gram_sample(invalid_tensor) - - # vocab_freq_table must be provided if vocab_min_count, vocab_subsampling, - # or corpus_size is specified. - dummy_input = constant_op.constant([""]) - with self.assertRaises(ValueError): - text.skip_gram_sample( - dummy_input, vocab_freq_table=None, vocab_min_count=1) - with self.assertRaises(ValueError): - text.skip_gram_sample( - dummy_input, vocab_freq_table=None, vocab_subsampling=1e-5) - with self.assertRaises(ValueError): - text.skip_gram_sample(dummy_input, vocab_freq_table=None, corpus_size=100) - with self.assertRaises(ValueError): - text.skip_gram_sample( - dummy_input, - vocab_freq_table=None, - vocab_subsampling=1e-5, - corpus_size=100) - - # vocab_subsampling and corpus_size must both be present or absent. - dummy_table = lookup.HashTable( - lookup.KeyValueTensorInitializer([b"foo"], [10]), -1) - with self.assertRaises(ValueError): - text.skip_gram_sample( - dummy_input, - vocab_freq_table=dummy_table, - vocab_subsampling=None, - corpus_size=100) - with self.assertRaises(ValueError): - text.skip_gram_sample( - dummy_input, - vocab_freq_table=dummy_table, - vocab_subsampling=1e-5, - corpus_size=None) - - def test_filter_input_filter_vocab(self): - """Tests input filtering based on vocab frequency table and thresholds.""" - input_tensor = constant_op.constant( - [b"the", b"answer", b"to", b"life", b"and", b"universe"]) - keys = constant_op.constant([b"and", b"life", b"the", b"to", b"universe"]) - values = constant_op.constant([0, 1, 2, 3, 4], dtypes.int64) - vocab_freq_table = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), -1) - - with self.cached_session(): - vocab_freq_table.initializer.run() - - # No vocab_freq_table specified - output should be the same as input. - no_table_output = skip_gram_ops._filter_input( - input_tensor=input_tensor, - vocab_freq_table=None, - vocab_min_count=None, - vocab_subsampling=None, - corpus_size=None, - seed=None) - self.assertAllEqual(input_tensor.eval(), no_table_output.eval()) - - # vocab_freq_table specified, but no vocab_min_count - output should have - # filtered out tokens not in the table (b"answer"). - table_output = skip_gram_ops._filter_input( - input_tensor=input_tensor, - vocab_freq_table=vocab_freq_table, - vocab_min_count=None, - vocab_subsampling=None, - corpus_size=None, - seed=None) - self.assertAllEqual([b"the", b"to", b"life", b"and", b"universe"], - table_output.eval()) - - # vocab_freq_table and vocab_min_count specified - output should have - # filtered out tokens whose frequencies are below the threshold - # (b"and": 0, b"life": 1). - threshold_output = skip_gram_ops._filter_input( - input_tensor=input_tensor, - vocab_freq_table=vocab_freq_table, - vocab_min_count=2, - vocab_subsampling=None, - corpus_size=None, - seed=None) - self.assertAllEqual([b"the", b"to", b"universe"], threshold_output.eval()) - - def test_filter_input_subsample_vocab(self): - """Tests input filtering based on vocab subsampling.""" - # The outputs are non-deterministic, so set random seed to help ensure that - # the outputs remain constant for testing. - random_seed.set_random_seed(42) - - input_tensor = constant_op.constant([ - # keep_prob = (sqrt(30/(0.05*100)) + 1) * (0.05*100/30) = 0.57. - b"the", - b"answer", # Not in vocab. (Always discarded) - b"to", # keep_prob = 0.75. - b"life", # keep_prob > 1. (Always kept) - b"and", # keep_prob = 0.48. - b"universe" # Below vocab threshold of 3. (Always discarded) - ]) - keys = constant_op.constant([b"and", b"life", b"the", b"to", b"universe"]) - values = constant_op.constant([40, 8, 30, 20, 2], dtypes.int64) - vocab_freq_table = lookup.HashTable( - lookup.KeyValueTensorInitializer(keys, values), -1) - - with self.cached_session(): - vocab_freq_table.initializer.run() - output = skip_gram_ops._filter_input( - input_tensor=input_tensor, - vocab_freq_table=vocab_freq_table, - vocab_min_count=3, - vocab_subsampling=0.05, - corpus_size=math_ops.reduce_sum(values), - seed=9) - self.assertAllEqual([b"the", b"to", b"life", b"and"], output.eval()) - - def _make_text_vocab_freq_file(self): - filepath = os.path.join(test.get_temp_dir(), "vocab_freq.txt") - with open(filepath, "w") as f: - writer = csv.writer(f) - writer.writerows([ - ["and", 40], - ["life", 8], - ["the", 30], - ["to", 20], - ["universe", 2], - ]) - return filepath - - def _make_text_vocab_float_file(self): - filepath = os.path.join(test.get_temp_dir(), "vocab_freq_float.txt") - with open(filepath, "w") as f: - writer = csv.writer(f) - writer.writerows([ - ["and", 0.4], - ["life", 0.08], - ["the", 0.3], - ["to", 0.2], - ["universe", 0.02], - ]) - return filepath - - def test_skip_gram_sample_with_text_vocab_filter_vocab(self): - """Tests skip-gram sampling with text vocab and freq threshold filtering.""" - input_tensor = constant_op.constant([ - b"the", - b"answer", # Will be filtered before candidate generation. - b"to", - b"life", - b"and", - b"universe" # Will be filtered before candidate generation. - ]) - - # b"answer" is not in vocab file, and b"universe"'s frequency is below - # threshold of 3. - vocab_freq_file = self._make_text_vocab_freq_file() - - tokens, labels = text.skip_gram_sample_with_text_vocab( - input_tensor=input_tensor, - vocab_freq_file=vocab_freq_file, - vocab_token_index=0, - vocab_freq_index=1, - vocab_min_count=3, - min_skips=1, - max_skips=1) - - expected_tokens, expected_labels = self._split_tokens_labels([ - (b"the", b"to"), - (b"to", b"the"), - (b"to", b"life"), - (b"life", b"to"), - (b"life", b"and"), - (b"and", b"life"), - ]) - with self.cached_session(): - lookup_ops.tables_initializer().run() - self.assertAllEqual(expected_tokens, tokens.eval()) - self.assertAllEqual(expected_labels, labels.eval()) - - def _text_vocab_subsample_vocab_helper(self, vocab_freq_file, vocab_min_count, - vocab_freq_dtype, corpus_size=None): - # The outputs are non-deterministic, so set random seed to help ensure that - # the outputs remain constant for testing. - random_seed.set_random_seed(42) - - input_tensor = constant_op.constant([ - # keep_prob = (sqrt(30/(0.05*100)) + 1) * (0.05*100/30) = 0.57. - b"the", - b"answer", # Not in vocab. (Always discarded) - b"to", # keep_prob = 0.75. - b"life", # keep_prob > 1. (Always kept) - b"and", # keep_prob = 0.48. - b"universe" # Below vocab threshold of 3. (Always discarded) - ]) - # keep_prob calculated from vocab file with relative frequencies of: - # and: 40 - # life: 8 - # the: 30 - # to: 20 - # universe: 2 - - tokens, labels = text.skip_gram_sample_with_text_vocab( - input_tensor=input_tensor, - vocab_freq_file=vocab_freq_file, - vocab_token_index=0, - vocab_freq_index=1, - vocab_freq_dtype=vocab_freq_dtype, - vocab_min_count=vocab_min_count, - vocab_subsampling=0.05, - corpus_size=corpus_size, - min_skips=1, - max_skips=1, - seed=123) - - expected_tokens, expected_labels = self._split_tokens_labels([ - (b"the", b"to"), - (b"to", b"the"), - (b"to", b"life"), - (b"life", b"to"), - ]) - with self.cached_session() as sess: - lookup_ops.tables_initializer().run() - tokens_eval, labels_eval = sess.run([tokens, labels]) - self.assertAllEqual(expected_tokens, tokens_eval) - self.assertAllEqual(expected_labels, labels_eval) - - def test_skip_gram_sample_with_text_vocab_subsample_vocab(self): - """Tests skip-gram sampling with text vocab and vocab subsampling.""" - # Vocab file frequencies - # and: 40 - # life: 8 - # the: 30 - # to: 20 - # universe: 2 - # - # corpus_size for the above vocab is 40+8+30+20+2 = 100. - text_vocab_freq_file = self._make_text_vocab_freq_file() - self._text_vocab_subsample_vocab_helper( - vocab_freq_file=text_vocab_freq_file, - vocab_min_count=3, - vocab_freq_dtype=dtypes.int64) - self._text_vocab_subsample_vocab_helper( - vocab_freq_file=text_vocab_freq_file, - vocab_min_count=3, - vocab_freq_dtype=dtypes.int64, - corpus_size=100) - - # The user-supplied corpus_size should not be less than the sum of all - # the frequency counts of vocab_freq_file, which is 100. - with self.assertRaises(ValueError): - self._text_vocab_subsample_vocab_helper( - vocab_freq_file=text_vocab_freq_file, - vocab_min_count=3, - vocab_freq_dtype=dtypes.int64, - corpus_size=99) - - def test_skip_gram_sample_with_text_vocab_subsample_vocab_float(self): - """Tests skip-gram sampling with text vocab and subsampling with floats.""" - # Vocab file frequencies - # and: 0.4 - # life: 0.08 - # the: 0.3 - # to: 0.2 - # universe: 0.02 - # - # corpus_size for the above vocab is 0.4+0.08+0.3+0.2+0.02 = 1. - text_vocab_float_file = self._make_text_vocab_float_file() - self._text_vocab_subsample_vocab_helper( - vocab_freq_file=text_vocab_float_file, - vocab_min_count=0.03, - vocab_freq_dtype=dtypes.float32) - self._text_vocab_subsample_vocab_helper( - vocab_freq_file=text_vocab_float_file, - vocab_min_count=0.03, - vocab_freq_dtype=dtypes.float32, - corpus_size=1.0) - - # The user-supplied corpus_size should not be less than the sum of all - # the frequency counts of vocab_freq_file, which is 1. - with self.assertRaises(ValueError): - self._text_vocab_subsample_vocab_helper( - vocab_freq_file=text_vocab_float_file, - vocab_min_count=0.03, - vocab_freq_dtype=dtypes.float32, - corpus_size=0.99) - - def test_skip_gram_sample_with_text_vocab_errors(self): - """Tests various errors raised by skip_gram_sample_with_text_vocab().""" - dummy_input = constant_op.constant([""]) - vocab_freq_file = self._make_text_vocab_freq_file() - - invalid_indices = ( - # vocab_token_index can't be negative. - (-1, 0), - # vocab_freq_index can't be negative. - (0, -1), - # vocab_token_index can't be equal to vocab_freq_index. - (0, 0), - (1, 1), - # vocab_freq_file only has two columns. - (0, 2), - (2, 0)) - - for vocab_token_index, vocab_freq_index in invalid_indices: - with self.assertRaises(ValueError): - text.skip_gram_sample_with_text_vocab( - input_tensor=dummy_input, - vocab_freq_file=vocab_freq_file, - vocab_token_index=vocab_token_index, - vocab_freq_index=vocab_freq_index) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/tfprof/BUILD b/tensorflow/contrib/tfprof/BUILD deleted file mode 100644 index c8846391ccd..00000000000 --- a/tensorflow/contrib/tfprof/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "tfprof", - srcs = [ - "__init__.py", - "model_analyzer.py", - "tfprof_logger.py", - ], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:__subpackages__"], - deps = [ - "//tensorflow/python:util", - "//tensorflow/python/profiler:model_analyzer", - "//tensorflow/python/profiler:profile_context", - "//tensorflow/python/profiler:tfprof_logger", - ], -) diff --git a/tensorflow/contrib/tfprof/README.md b/tensorflow/contrib/tfprof/README.md deleted file mode 100644 index f40e76f554e..00000000000 --- a/tensorflow/contrib/tfprof/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# tfprof: TensorFlow Profiler and Beyond - -

Full Document in -tensorflow/core/profiler/README.md

- -### Features - -* Profile model architectures - * parameters, tensor shapes, float operations, device placement, etc. -* Profile model performance - * execution time, memory consumption - * Profile multiple steps. -* Auto profile and advise. - * accelerator utilization check - * expensive operation check - * operation configuration check - * distributed runtime check (Not OSS) - -### Interfaces - -* Python API -* Command Line -* Visualization diff --git a/tensorflow/contrib/tfprof/__init__.py b/tensorflow/contrib/tfprof/__init__.py deleted file mode 100644 index d5d3650e600..00000000000 --- a/tensorflow/contrib/tfprof/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""tfprof is a tool that profile various aspect of TensorFlow model. - -@@model_analyzer -@@tfprof_logger - -@@ProfileContext -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.contrib.tfprof import model_analyzer -from tensorflow.contrib.tfprof import tfprof_logger - -from tensorflow.contrib.tfprof.model_analyzer import ProfileContext - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = ['model_analyzer', 'tfprof_logger', 'ProfileContext'] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/tfprof/model_analyzer.py b/tensorflow/contrib/tfprof/model_analyzer.py deleted file mode 100644 index 00bd0b41104..00000000000 --- a/tensorflow/contrib/tfprof/model_analyzer.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model Analyzer. - -Analyze model, including shape, params, time, memory, structure, etc. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -# Import the names here for existing users. -# pylint: disable=unused-import -from tensorflow.python.profiler import tfprof_logger -from tensorflow.python.profiler.model_analyzer import advise as _advise -from tensorflow.python.profiler.model_analyzer import ALL_ADVICE -from tensorflow.python.profiler.model_analyzer import profile as _profile -from tensorflow.python.profiler.model_analyzer import Profiler -from tensorflow.python.profiler.profile_context import ProfileContext -from tensorflow.python.util.deprecation import deprecated - -_DEFAULT_PROFILE_OPTIONS = 0 -_DEFAULT_ADVISE_OPTIONS = 0 - -# pylint: disable=bad-whitespace -# pylint: disable=bad-continuation -# options examples for profiling API. -# -# Show the parameter statistics of trainable variables. -TRAINABLE_VARS_PARAMS_STAT_OPTIONS = { - 'max_depth': 10000, - 'min_bytes': 0, - 'min_micros': 0, - 'min_params': 0, - 'min_float_ops': 0, - 'order_by': 'name', - 'account_type_regexes': [tfprof_logger.TRAINABLE_VARIABLES], - 'start_name_regexes': ['.*'], - 'trim_name_regexes': [], - 'show_name_regexes': ['.*'], - 'hide_name_regexes': [], - 'account_displayed_op_only': True, - 'select': ['params'], - 'output': 'stdout', - 'dump_to_file': '' # Deprecated, use 'output': 'file:outfile=' -} - -# Show the number float operations. -FLOAT_OPS_OPTIONS = { - 'max_depth': 10000, - 'min_bytes': 0, - 'min_micros': 0, - 'min_params': 0, - 'min_float_ops': 1, - 'order_by': 'float_ops', - 'account_type_regexes': ['.*'], - 'start_name_regexes': ['.*'], - 'trim_name_regexes': [], - 'show_name_regexes': ['.*'], - 'hide_name_regexes': [], - 'account_displayed_op_only': True, - 'select': ['float_ops'], - 'output': 'stdout', - 'dump_to_file': '' # Deprecated, use 'output': 'file:outfile=' -} - - -# Show the timing stats and memory demands. -PRINT_ALL_TIMING_MEMORY = { - 'max_depth': 10000, - 'min_bytes': 1, # Only >=1 - 'min_micros': 1, # Only >=1 - 'min_params': 0, - 'min_float_ops': 0, - 'order_by': 'name', - 'account_type_regexes': ['.*'], - 'start_name_regexes': ['.*'], - 'trim_name_regexes': [], - 'show_name_regexes': ['.*'], - 'hide_name_regexes': [], - 'account_displayed_op_only': True, - 'select': ['micros', 'bytes'], - 'output': 'stdout', - 'dump_to_file': '' # Deprecated, use 'output': 'file:outfile=' -} - -# pylint: enable=bad-whitespace -# pylint: enable=bad-continuation - - -@deprecated('2018-01-01', - 'Use `tf.profiler.advise(graph, run_meta, options)`. See README.md') -def advise(graph, run_meta=None, tfprof_options=_DEFAULT_ADVISE_OPTIONS): - return _advise(graph, run_meta, tfprof_options) - - -@deprecated('2018-01-01', - 'Use `tf.profiler.profile(graph, run_meta, op_log, cmd, options)`. ' - 'Build `options` with `tf.profiler.ProfileOptionBuilder`. ' - 'See README.md for details') -def print_model_analysis(graph, - run_meta=None, - op_log=None, - tfprof_cmd='scope', - tfprof_options=_DEFAULT_PROFILE_OPTIONS): - return _profile(graph, run_meta, op_log, tfprof_cmd, tfprof_options) diff --git a/tensorflow/contrib/tfprof/tfprof_logger.py b/tensorflow/contrib/tfprof/tfprof_logger.py deleted file mode 100644 index 78eaca37a1f..00000000000 --- a/tensorflow/contrib/tfprof/tfprof_logger.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Logging tensorflow::tfprof::OpLogProto. - -OpLogProto is used to add extra model information for offline analysis. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.profiler.tfprof_logger import write_op_log as _write_op_log -from tensorflow.python.util.deprecation import deprecated - - -@deprecated("2018-01-01", "Use `tf.profiler.write_op_log. go/tfprof`") -def write_op_log(graph, log_dir, op_log=None, run_meta=None, add_trace=True): - _write_op_log(graph, log_dir, op_log, run_meta, add_trace) diff --git a/tensorflow/contrib/timeseries/BUILD b/tensorflow/contrib/timeseries/BUILD deleted file mode 100644 index 989085564a5..00000000000 --- a/tensorflow/contrib/timeseries/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -package( - default_visibility = [ - "//tensorflow:internal", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "timeseries", - srcs = [ - "__init__.py", - "python/__init__.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/timeseries/python/timeseries:py_init", - "//tensorflow/python:util", - ], -) - -py_library( - name = "timeseries_pip", - deps = [ - ":timeseries", - "//tensorflow/contrib/timeseries/examples:known_anomaly_main_lib", - "//tensorflow/contrib/timeseries/examples:lstm_main_lib", - "//tensorflow/contrib/timeseries/examples:multivariate_main_lib", - "//tensorflow/contrib/timeseries/examples:predict_main_lib", - "//tensorflow/contrib/timeseries/python/timeseries:test_utils", - "//tensorflow/contrib/timeseries/python/timeseries/state_space_models:test_utils", - ], -) diff --git a/tensorflow/contrib/timeseries/README.md b/tensorflow/contrib/timeseries/README.md deleted file mode 100644 index 0e15d162dd2..00000000000 --- a/tensorflow/contrib/timeseries/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# TensorFlow Time Series - -TensorFlow Time Series (TFTS) is a collection of ready-to-use classic models -(state space, autoregressive), and flexible infrastructure for building -high-performance time series models with custom architectures. It includes tools -for chunking and batching a series, and for saving model state across chunks, -making use of parallel computation even when training sequential models on long -series (using truncated backpropagation). - -To get started, take a look at the `examples/` directory, which includes: - - - Making probabilistic forecasts (`examples/predict.py`) - - Using exogenous features to train on data with known anomalies/changepoints (`examples/known_anomaly.py`) - - Learning correlations between series (multivariate forecasting/anomaly - detection; `examples/multivariate.py`) - - More advanced custom model building (`examples/lstm.py`) - -TFTS includes many other modeling tools, including non-linear autoregression -(see the `hidden_layer_sizes` argument to `ARRegressor` in `estimators.py`) and -a collection of components for linear state space modeling (level, trend, -period, vector autoregression, moving averages; see the -`StructuralEnsembleRegressor` in `estimators.py`). Both model classes support -heuristics for ignoring un-labeled anomalies in training data. Trained models -can be exported for inference/serving in -[SavedModel format](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md) -(see `examples/multivariate.py`). diff --git a/tensorflow/contrib/timeseries/__init__.py b/tensorflow/contrib/timeseries/__init__.py deleted file mode 100644 index 654a4db0987..00000000000 --- a/tensorflow/contrib/timeseries/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A time series library in TensorFlow (TFTS). - -@@StructuralEnsembleRegressor -@@ARRegressor - -@@ARModel - -@@CSVReader -@@NumpyReader -@@RandomWindowInputFn -@@WholeDatasetInputFn -@@predict_continuation_input_fn - -@@TrainEvalFeatures -@@FilteringResults - -@@TimeSeriesRegressor -@@OneShotPredictionHead -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.timeseries.python.timeseries import * -# pylint: enable=wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented - -remove_undocumented(module_name=__name__, - allowed_exception_list=['saved_model_utils']) diff --git a/tensorflow/contrib/timeseries/examples/BUILD b/tensorflow/contrib/timeseries/examples/BUILD deleted file mode 100644 index 03979932a96..00000000000 --- a/tensorflow/contrib/timeseries/examples/BUILD +++ /dev/null @@ -1,172 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "py_test") -load("//tensorflow:tensorflow.bzl", "py_binary") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -config_setting( - name = "empty_condition", - values = {"define": "UNUSED=unused"}, -) - -py_binary( - name = "predict", - srcs = ["predict.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [":predict_main_lib"], -) - -py_library( - name = "predict_main_lib", - srcs = ["predict.py"], - data = ["data/period_trend.csv"], - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = select({ - ":empty_condition": [], - "//conditions:default": [], - }) + [ - "//third_party/py/numpy", - "//tensorflow:tensorflow_py", - ], -) - -py_test( - name = "predict_test", - timeout = "long", # Moderate but for asan - srcs = ["predict_test.py"], - data = ["data/period_trend.csv"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_windows", # TODO: needs investigation on Windows - "notsan", # b/67513579 - ], - deps = [ - ":predict_main_lib", - "//tensorflow/python:client_testlib", - ], -) - -py_binary( - name = "known_anomaly", - srcs = ["known_anomaly.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [":known_anomaly_main_lib"], -) - -py_library( - name = "known_anomaly_main_lib", - srcs = ["known_anomaly.py"], - data = ["data/changepoints.csv"], - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = select({ - ":empty_condition": [], - "//conditions:default": [], - }) + [ - "//third_party/py/numpy", - "//tensorflow:tensorflow_py", - ], -) - -py_test( - name = "known_anomaly_test", - timeout = "long", # Moderate but for asan - srcs = ["known_anomaly_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":known_anomaly_main_lib", - "//tensorflow/python:client_testlib", - ], -) - -py_binary( - name = "multivariate", - srcs = ["multivariate.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [":multivariate_main_lib"], -) - -py_library( - name = "multivariate_main_lib", - srcs = ["multivariate.py"], - data = ["data/multivariate_level.csv"], - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = select({ - ":empty_condition": [], - "//conditions:default": [], - }) + [ - "//third_party/py/numpy", - "//tensorflow:tensorflow_py", - ], -) - -py_test( - name = "multivariate_test", - timeout = "long", # Moderate but for asan - srcs = [ - "multivariate_test.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":multivariate_main_lib", - "//tensorflow/python:client_testlib", - ], -) - -py_binary( - name = "lstm", - srcs = ["lstm.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["no_pip"], - visibility = ["//visibility:public"], - deps = [":lstm_main_lib"], -) - -py_library( - name = "lstm_main_lib", - srcs = ["lstm.py"], - data = ["data/multivariate_periods.csv"], - srcs_version = "PY2AND3", - tags = ["no_pip"], - visibility = ["//visibility:public"], - deps = select({ - ":empty_condition": [], - "//conditions:default": [], - }) + [ - "//third_party/py/numpy", - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/timeseries/python/timeseries:estimators", - "//tensorflow/contrib/timeseries/python/timeseries:model", - "//tensorflow/contrib/timeseries/python/timeseries:state_management", - ], -) - -py_test( - name = "lstm_test", - timeout = "long", # Moderate but for asan - srcs = ["lstm_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["notsan"], - deps = [ - ":lstm_main_lib", - "//tensorflow/python:client_testlib", - "//tensorflow/python/estimator:estimator_py", - ], -) diff --git a/tensorflow/contrib/timeseries/examples/__init__.py b/tensorflow/contrib/timeseries/examples/__init__.py deleted file mode 100644 index 52e83069cb0..00000000000 --- a/tensorflow/contrib/timeseries/examples/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/timeseries/examples/data/changepoints.csv b/tensorflow/contrib/timeseries/examples/data/changepoints.csv deleted file mode 100644 index 377091675f3..00000000000 --- a/tensorflow/contrib/timeseries/examples/data/changepoints.csv +++ /dev/null @@ -1,201 +0,0 @@ -time,value,is_changepoint -0,1.1214962292980595,no -1,0.9426651261919153,no -2,0.4031960302687991,no -3,-0.2934288551216625,no -4,-0.5743074870294929,no -5,-1.0849473811550254,no -6,-0.41937946806801163,no -7,0.018935886380438346,no -8,0.623316417711846,no -9,1.536149296834255,no -10,1.7409091603912816,no -11,2.051703703599627,no -12,0.6384001894023976,no -13,0.5486930106811457,no -14,-0.2125368668501388,no -15,0.013749126561190012,no -16,0.14880983597607,no -17,0.1635284853313118,no -18,1.897997315187439,no -19,1.6579340030214134,no -20,2.8447844385162195,no -21,1.6218988328670014,no -22,1.2696459559280746,no -23,0.9479057398771089,no -24,0.24823674890404254,no -25,0.27109416160211997,yes -26,0.6012714690807257,no -27,1.9638777275054928,no -28,2.4058943017150844,no -29,2.1603787636120795,no -30,2.653021551450762,no -31,2.6302422291515883,no -32,2.1672420831416983,no -33,1.900648370044954,no -34,0.6392456915214698,no -35,0.9261522810148497,no -36,1.7007734373862495,no -37,2.114997001994879,no -38,2.5608972952926123,no -39,3.544458039317542,no -40,3.012875548231654,no -41,3.73017069321913,no -42,2.6946843638167333,no -43,2.8311515823580367,no -44,1.5925217307525887,no -45,2.0309433269232073,no -46,1.9180019103182229,no -47,2.502600804744764,no -48,3.2807166427258965,no -49,3.2786059081935983,no -50,-0.03890687766970344,yes -51,0.11514469108191117,no -52,-0.26083055108544917,no -53,-1.6028085522978275,no -54,-1.4337505749859742,no -55,-1.5254043377879583,no -56,-1.8019508271507605,no -57,-0.938164718968729,no -58,-0.04584450628604822,no -59,0.21179188280568237,no -60,0.6070671683315334,no -61,0.6276767611311048,no -62,-0.03193958766467522,no -63,-0.5093408620803364,no -64,-0.8078327975830406,no -65,-0.7926626666688175,no -66,-1.263702632888315,no -67,-0.756316161320801,no -68,0.6539838846224881,no -69,1.039181766829142,no -70,0.9530783360822233,no -71,0.5846593087566678,no -72,0.3789183086882727,no -73,-0.005047297176443883,no -74,-0.3257930437614064,no -75,-12.073284032544285,yes -76,-10.985299021088675,no -77,-11.105443872355911,no -78,-10.159082851902342,no -79,-9.744065848145468,no -80,-9.73365351235421,no -81,-9.337054697472244,no -82,-9.562841115527014,no -83,-10.415757451905112,no -84,-11.342362490064435,no -85,-11.616152164963092,no -86,-11.151346186288702,no -87,-10.429303505950227,no -88,-9.387541028841701,no -89,-8.835597195885683,no -90,-9.042472454146061,no -91,-8.826062322444617,no -92,-9.58027111131134,no -93,-10.105297327026976,no -94,-10.87201188348659,no -95,-10.734063432244673,no -96,-10.564284857652627,no -97,-10.222152572358237,no -98,-10.034222253530091,no -99,-8.569926275463303,no -100,-20.94641864280508,yes -101,-20.328581083983565,no -102,-20.309010035007223,no -103,-21.903487576802988,no -104,-21.452703450083913,no -105,-22.456525013367447,no -106,-21.400574790162256,no -107,-21.31159417717666,no -108,-20.761111930463677,no -109,-20.03122903525928,no -110,-19.775288115644337,no -111,-20.204309173889285,no -112,-20.289200792476837,no -113,-20.996369123999006,no -114,-21.526187402690034,no -115,-21.954579978399675,no -116,-21.35551171030287,no -117,-20.81049230404345,no -118,-19.913346111752116,no -119,-19.69212526824288,no -120,-19.917131232233228,no -121,-20.06819925540892,no -122,-19.935797523800392,no -123,-20.450306566286283,no -124,-20.633236594032798,no -125,-13.032619653152857,yes -126,-13.78752822118153,no -127,-12.768298388725846,no -128,-12.592028243555017,no -129,-11.341855179200538,no -130,-11.409551272753419,no -131,-11.61577579285243,no -132,-12.100218074308154,no -133,-12.019249802111034,no -134,-13.11581463614673,no -135,-13.088783512286215,no -136,-13.070654237103765,no -137,-12.175249761266228,no -138,-11.093992412775233,no -139,-10.912276441852045,no -140,-11.084638857082574,no -141,-11.573963550454287,no -142,-11.831767648915175,no -143,-12.198248248363289,no -144,-12.647459048291175,no -145,-12.00526050990987,no -146,-12.86075732253657,no -147,-12.097659271522993,no -148,-11.304397011757692,no -149,-10.907765581608706,no -150,4.152294032629092,yes -151,3.846976311678118,no -152,3.7700912822157164,no -153,2.848094096503526,no -154,2.7852277792642517,no -155,2.083452358392344,no -156,3.10521897109034,no -157,2.6883597273679953,no -158,3.6853340905320797,no -159,4.657935435352187,no -160,4.205192591470813,no -161,4.181620027424676,no -162,3.7946083469988627,no -163,4.006143660715422,no -164,3.302226947074533,no -165,2.3998839690439953,no -166,2.359677958463434,no -167,3.9511382664496764,no -168,4.2544962146733365,no -169,4.776487678103945,no -170,5.524025961899998,no -171,4.609095497010403,no -172,4.420771370427846,no -173,4.5501449534143354,no -174,3.8649120165249387,no -175,7.974015818409917,yes -176,8.231372133173299,no -177,9.03995638047549,no -178,10.047862771863846,no -179,9.955465317225139,no -180,10.353973597173228,no -181,10.50998966501437,no -182,9.439675801327347,no -183,9.597509477726458,no -184,8.575204551834439,no -185,8.63550641368468,no -186,8.646333808129821,no -187,8.837543235768495,no -188,9.661532154505531,no -189,9.900003537805857,no -190,10.918387185224491,no -191,10.027537345249021,no -192,10.344965386383317,no -193,9.05542200420013,no -194,9.41967229823334,no -195,8.745145521533027,no -196,9.15826892409396,no -197,9.716409087979185,no -198,9.873840185125154,no -199,10.664518096235417,no diff --git a/tensorflow/contrib/timeseries/examples/data/multivariate_level.csv b/tensorflow/contrib/timeseries/examples/data/multivariate_level.csv deleted file mode 100644 index 17a9d4f879b..00000000000 --- a/tensorflow/contrib/timeseries/examples/data/multivariate_level.csv +++ /dev/null @@ -1,1000 +0,0 @@ -0,-0.0414646733535,-0.268848856679,-0.734895900031,-0.446004825141,-0.694399529702 -1,-0.152094899259,-0.322007819938,-0.332268116964,-0.105095114653,-0.178601838378 -2,0.250556262671,-0.751761834516,-0.770932451952,-0.455887554092,-0.831949277137 -3,-0.373766175967,0.738512142445,-0.199851965386,-0.113920326737,-0.746597969326 -4,-0.075847586994,2.49850954084,-0.195929673324,-0.072250117751,-0.849909928747 -5,-0.205779412656,2.36687030724,-0.0626565533539,-0.144351330587,-0.645795602996 -6,-0.846980328423,1.88954123472,0.268955172856,0.0955759146678,-0.491047551885 -7,-1.71690266703,1.59009549063,-0.819726283855,0.0704148937343,-1.36443168813 -8,-1.62116547912,1.12644651303,-0.548591331565,0.159809406514,-0.982531332323 -9,-2.30511521562,0.233327108262,-0.720377250338,0.185365674993,-0.581283365228 -10,-1.39509634301,0.727639387607,-0.9472391049,-0.24675750758,-1.24362211749 -11,-1.66199313961,-0.429411794076,-1.14309941456,-0.10410046279,-1.50770930313 -12,-1.94640731282,-0.273995766959,-0.716939919329,0.242478963841,-1.01869993398 -13,-1.73697738539,-0.297774741968,-1.80542901165,-0.595544065613,-1.88517320016 -14,-1.98436000953,0.835646566838,-1.48895743706,-0.64954533885,-2.07546616553 -15,-2.19661690257,0.135737360133,-2.04801419858,-0.83727343112,-2.24381736404 -16,-2.05515485655,0.524110912353,-1.07121349384,-0.235165807213,-1.28329038964 -17,-2.05194303098,1.12739698675,-1.19731273863,-0.718272345602,-1.8681304519 -18,-1.46342408153,1.60848661464,-0.982386371124,-0.241330484258,-1.53944424301 -19,-2.2011585973,0.862807915046,-0.995794140283,-0.277024972786,-1.47741597501 -20,-2.23578343098,0.211562996187,-1.80353693381,-0.750494902773,-2.28181147838 -21,-2.4248161381,1.12076760123,-1.59718553352,-0.425584020125,-2.04861425617 -22,-2.31880631104,1.12602215115,-1.91151504658,-0.75502569518,-2.63242617399 -23,-2.11892926843,-0.0776836392241,-1.56290301815,-0.415718204976,-2.30107410479 -24,-2.69745336988,1.1149476191,-1.43630550668,0.280056938851,-2.22883491018 -25,-2.96630018615,-0.806783462871,-0.841372722536,0.583627043401,-1.03872536926 -26,-3.05451403268,-0.41632426626,-1.58925935691,0.0555428856781,-2.1353650451 -27,-3.13467782667,-0.205721140141,-1.36817316767,0.415306030296,-1.8505395134 -28,-3.37583309658,-0.818083884025,-1.55652668932,0.145422266739,-1.66599545695 -29,-2.95706967161,-0.126708568148,-2.49609054237,-0.359693478269,-3.00630852278 -30,-2.38734689742,-0.350923993879,-3.11222966842,-0.984597775882,-3.62711470966 -31,-2.93362724797,-3.34981184909,-2.62299010845,0.0519325700903,-2.33361674027 -32,-2.45038317895,-2.14261343917,-2.11366789279,0.0743102043259,-2.11874446647 -33,-2.64495216711,-2.93675730003,-2.01659804851,0.220799850535,-1.87237671926 -34,-2.12812770812,-2.92126192546,-2.56372930688,0.132880158227,-2.29573778931 -35,-1.35216831892,-2.50483041175,-3.13901618308,-0.37336037936,-3.33252674805 -36,-1.04567351754,-1.83302495038,-3.08212228316,-0.545002988717,-3.6701174207 -37,-1.16496488915,-1.40938171638,-2.94209012758,-0.152560391217,-3.50296269877 -38,-0.84385282013,-0.648804927911,-2.75862460218,-0.279881663166,-3.63690027991 -39,-1.1037352916,-0.851582153896,-2.77523199927,-0.307039823577,-3.38853728212 -40,-0.957048835866,-0.716389490538,-2.5075186655,-0.368254150835,-3.12977069615 -41,-0.391472235219,0.226506834239,-2.52187570516,-0.61894949021,-3.47667477699 -42,-0.405242757907,0.418957308382,-2.62051759148,-0.9796272777,-3.42806374772 -43,-0.748240793265,0.0115709569987,-2.6684888437,-0.667482764946,-3.43428687409 -44,-0.216387397721,0.448920090064,-2.70192416248,-0.808861227389,-3.75630714954 -45,-1.12093785791,0.813988652808,-2.34432442826,-0.474816708663,-3.456833936 -46,-2.04823857595,-0.0815669196333,-1.86066257604,0.263513591785,-2.62013231073 -47,-2.23426540572,1.14990130874,-2.12369491495,-0.0668207664299,-3.32831571065 -48,-1.57555379584,1.82043610627,-1.9765782176,-0.102298636364,-3.43261409083 -49,-2.05662519541,0.591204633026,-2.25118904188,-0.104063054092,-3.20456376636 -50,-1.8285006832,0.104590609107,-2.44968651963,-0.484283165496,-3.4943191607 -51,-1.1158243274,1.51817970836,-2.92579532611,-0.95832702811,-4.14551045434 -52,-1.3300517478,1.4640581709,-3.08363690864,-0.908736611138,-4.69257252131 -53,-1.12868164589,1.57313046956,-3.16435479065,-0.957447102204,-4.56171750677 -54,-0.669145314902,1.75434515643,-2.81083675355,-0.774573791032,-4.59395663147 -55,-0.892340515591,0.173537296383,-2.49815052235,-0.375439267459,-3.52324421038 -56,-1.26929850789,0.649913804094,-2.57885164397,-0.46483791564,-3.59996099296 -57,-1.28158565705,-0.570153054645,-2.78819462608,-0.306397870867,-3.4258361014 -58,-1.28263745221,0.301364082853,-3.28815747643,-0.76434916506,-4.3591321818 -59,-1.64608482042,0.079268277506,-3.13957219344,-0.545170637087,-3.94856837712 -60,-1.77018149232,1.34239749475,-3.52189227312,-0.677099703264,-4.62920748509 -61,-1.75655632146,1.24086450834,-3.65478985863,-0.790231809835,-4.83729210022 -62,-1.4538974886,0.583118143939,-2.94897344935,-0.528868313288,-4.45102769399 -63,-1.79084029065,-0.182981048507,-2.30504593196,0.0348033949121,-3.11721052434 -64,-1.8073178267,0.277552929826,-2.50207638545,-0.0968222603598,-3.6080745155 -65,-2.46622391377,-0.669940539918,-2.51643284284,-0.163468699096,-3.19159436336 -66,-1.69971961367,-1.41118297672,-2.71054034025,-0.34638825907,-3.04528220656 -67,-1.96457404093,-1.67429576673,-2.54850971749,-0.067237849466,-2.90503073443 -68,-2.6973521408,-1.66907013816,-1.9623006077,0.479537600832,-2.30009440384 -69,-2.40620671435,-1.53765150204,-2.18583073806,0.446553641646,-2.63840332383 -70,-2.01757147472,-1.22761328,-2.55621892108,0.400640935149,-3.16530630377 -71,-1.94664421789,-0.129913185847,-2.71957281003,-0.147173342055,-3.58233183488 -72,-1.94706064604,1.15127195098,-2.61814243108,-0.304736837281,-3.9346910679 -73,-1.9350122257,1.07051301058,-2.20968576985,-0.149892580485,-3.74024140039 -74,-2.38619066594,1.68724247518,-2.41160829661,-0.00913356005116,-3.99782659316 -75,-2.24809078587,1.21157820875,-2.07388139686,0.407473663742,-3.21636196087 -76,-3.05601728047,-0.159092449639,-1.70662323792,0.549119297817,-2.37237670193 -77,-3.41423377011,0.383317925933,-2.1179809253,0.440933949104,-3.43621876374 -78,-3.06397628245,0.127724697663,-1.50730821646,0.792369512789,-2.51400429958 -79,-2.46206559193,1.39197941174,-2.07070430872,0.0723755941552,-3.42215049587 -80,-2.95861031286,0.492643378852,-2.21356929084,0.219154069898,-3.50736459718 -81,-3.6878555302,-1.16408933615,-1.90916720564,0.805245613936,-2.52416993459 -82,-3.62030177826,-1.61633663455,-1.91163502791,0.769301736417,-2.32917764535 -83,-4.01196983047,-2.48802319211,-1.64295948963,1.24479580458,-1.62606743949 -84,-4.47183588705,-1.86895788707,-1.71275388306,1.32487605749,-2.04746598078 -85,-5.4955157643,-2.66146586745,-1.3125133071,1.83584422007,-1.92358421676 -86,-4.76151142281,-4.0158514346,-1.20747992628,1.77597998686,-0.701607451242 -87,-5.31155771019,-3.15539819753,-0.921016402738,2.05873025447,-0.960023072679 -88,-4.16798254912,-3.00934457399,-0.588731448912,1.90691848431,-0.651736302671 -89,-3.72133637608,-2.63534770736,-0.751807733258,1.73266437472,-0.747133247155 -90,-4.89252968501,-2.11988988053,-0.909281004084,1.89134204562,-1.12181355673 -91,-4.76565397768,-2.45166833079,-0.345448694534,2.12063951416,-0.63677850405 -92,-4.63858695121,-3.55046524931,-0.56519223243,2.27714507148,-0.758053667144 -93,-4.48414155394,-3.7219215509,-0.619311750125,2.23433435538,-0.438222718006 -94,-4.30438313321,-3.95244620189,-0.359853685703,2.27004922971,0.170025525215 -95,-4.7937387984,-3.3691885737,0.178884395796,2.86298118154,0.232096604308 -96,-4.77291737506,-2.52854276077,-0.345707006275,2.50163637851,-0.456607039838 -97,-4.62298139493,-2.29649895418,-0.490691361517,2.46124256727,-0.82515464821 -98,-3.96843790784,-2.7210597373,-0.847691030288,1.82818727491,-0.965708260824 -99,-3.52732312968,-2.13122777966,-1.48257599212,1.58444779334,-1.88600127741 -100,-3.93577934279,-2.96124673527,-0.853709706956,1.9254416322,-0.904048307626 -101,-4.3401932336,-3.10020192226,-0.20986281874,2.57273546,-0.632369678503 -102,-3.87139651112,-2.49995721503,-0.302048022933,2.45265811426,-0.582067583909 -103,-3.68490099559,-1.86926403845,-1.07896542406,1.98128996318,-1.52227363904 -104,-3.73795132118,-1.2813093908,-1.02231707609,1.9052598382,-1.69077029177 -105,-4.11227647638,-0.97321018636,-1.24214456086,1.66677538807,-2.01422017997 -106,-4.31128617249,-0.859790642205,-1.20138722257,1.46077710527,-2.03451018811 -107,-3.09504395205,1.00438274912,-1.97264240173,0.732150386166,-3.32632736368 -108,-2.681407939,1.44318461325,-2.39764334203,0.411009469585,-4.08770189209 -109,-3.06128776345,0.164570601827,-2.83505639278,0.0309828158983,-3.93363349737 -110,-2.96002558525,0.568530678109,-3.06826048982,-0.120518420294,-4.27282226734 -111,-3.19185112296,-0.595185852518,-2.71221607519,0.166579412962,-3.44187869033 -112,-3.81846994181,-1.83388142802,-2.90816681647,0.235773907539,-3.20090724714 -113,-4.18583974049,-1.97129111676,-2.74503137596,0.699189049686,-3.18638906104 -114,-4.15164488936,-1.58883698504,-2.38619384026,0.896062753518,-2.97261262165 -115,-3.94254676388,-1.70839450301,-2.81258836056,0.491894248181,-3.43646144606 -116,-3.96761881212,-1.55620520351,-2.33133041981,0.626269020808,-3.01677540027 -117,-3.50646136496,-0.950732274753,-2.35560895864,0.256356938967,-2.79127289896 -118,-3.25934857603,-0.732566929111,-2.41419969521,0.264090641824,-3.03291722901 -119,-3.42002240184,-0.415338127265,-3.07048729009,-0.0231731719376,-4.02562825774 -120,-3.53921857173,0.0158398656482,-2.70532905559,0.0565707967172,-3.84427790925 -121,-3.15592574046,0.784247799282,-3.58692679922,-0.242085258667,-5.06601908662 -122,-3.76809400203,0.463131255016,-3.8035033346,-0.429273348415,-4.98980268082 -123,-2.96288215704,1.77966617942,-3.32459939883,-0.551051151437,-4.93441116438 -124,-2.4962953866,3.39328322058,-3.03997171564,-0.216002386353,-5.18521353952 -125,-2.82557238612,2.76978657257,-3.23390042871,-0.0004479395837,-5.2176856873 -126,-2.58945498797,2.52368015772,-3.15380668865,-0.0639940636338,-5.26689084107 -127,-2.18245541691,2.46233728874,-3.33317082971,-0.281707789522,-5.35318328452 -128,-2.62315865429,1.51428800961,-3.62498863249,-0.270500998569,-5.2760566531 -129,-1.67705168949,2.67883021797,-3.17315867722,-0.341422601729,-5.19145005014 -130,-1.13323473613,3.3659551068,-2.9678582214,-0.429528041988,-5.25094097547 -131,-1.04105880247,3.50972732877,-3.11167783436,-0.456888273574,-5.24428811476 -132,-1.04773214914,3.82875614139,-3.04149583424,-0.371839445182,-5.61930173874 -133,-1.82094528277,2.83823742395,-3.33646912963,-0.322370252244,-5.61791889275 -134,-2.91804499764,1.8780201922,-4.06997503816,-0.498867961513,-6.17519153519 -135,-2.46795673425,2.31056167769,-4.50715795956,-0.766293318265,-6.953363658 -136,-3.0183856337,1.30296095196,-3.7714443347,-0.182133829714,-5.68593518503 -137,-4.1459367179,1.35223688484,-3.76595632938,0.0559619112277,-5.76499263486 -138,-4.26014113282,0.840287216995,-4.24154634876,0.0170854664832,-5.97212592449 -139,-3.71553554501,1.76361822382,-4.04901573444,-0.184615894262,-6.13019444273 -140,-4.98134444903,0.643221203145,-3.63502050732,0.732739839835,-5.31102037188 -141,-4.93231227313,0.451617863662,-4.18338313485,0.358996447266,-5.87642542398 -142,-3.93259210355,0.149657165734,-3.83132770968,0.269488626765,-5.34545674445 -143,-4.11814307699,0.721393968739,-3.59007389208,0.370660172423,-5.42955757642 -144,-4.23367298838,1.05376963285,-3.53969923896,0.614112257164,-5.48630859728 -145,-3.95869636495,0.472136852797,-3.18310972263,0.689673233548,-4.80901429311 -146,-4.04538674379,0.0738523975768,-3.99667527523,0.330477257534,-5.41593638963 -147,-4.1468026697,0.415399188108,-3.91404753507,0.0872810690256,-5.67680456497 -148,-4.15460923758,0.0717509451784,-3.9588831606,0.147162867399,-5.76814370589 -149,-3.57523193245,0.195417056414,-4.80410000105,-0.473719267884,-6.38176324697 -150,-3.51025861874,0.856118375832,-5.40027963164,-0.624382376552,-7.1359928798 -151,-4.38237447519,1.60011445758,-4.7521606876,0.0518728621325,-7.00004915101 -152,-4.27623318984,1.47324864364,-4.30781497365,0.124827291806,-6.54377411992 -153,-4.32194428155,2.06587484172,-4.20631276842,0.607399497692,-6.33547724603 -154,-5.00493869459,2.22966866366,-3.9061149897,0.696656417204,-6.17196154778 -155,-5.17550776754,2.21188658685,-3.99954696629,0.727786103955,-6.21657810996 -156,-5.24496674952,1.83542511918,-3.88063039409,0.75800286432,-6.40628509389 -157,-5.54708551632,2.82198533139,-3.84132303481,0.85671393851,-6.64762596715 -158,-5.16618921679,4.2567793887,-4.16745677914,0.477109844238,-7.43645189108 -159,-4.81521278827,4.45683428742,-4.34746873006,0.519627524599,-7.68745975295 -160,-4.8275925528,3.88330106165,-5.18631184892,0.100119923085,-8.75285457556 -161,-5.2511588662,2.8143123273,-5.12844694126,0.182664198339,-7.88009814396 -162,-5.61992128109,2.26792622321,-5.18009673261,0.414975247919,-8.16165288241 -163,-5.8000776885,2.80532060518,-5.0908514781,0.427068246658,-8.20088294419 -164,-6.19575329831,3.46018564451,-4.95019927132,0.464146926353,-8.03609530647 -165,-5.98656441427,4.12718110229,-4.69880353388,0.66144547648,-8.01893703451 -166,-5.90243834383,3.40320096162,-4.73707576726,0.831951793066,-7.9500544624 -167,-5.52097545345,5.42275420718,-4.70631335864,0.625767110232,-8.55065884765 -168,-6.3208950302,4.69282424685,-3.95049725277,1.158893991,-7.72947279651 -169,-5.84293846832,5.68653008392,-3.95380995318,1.03375698147,-8.176175529 -170,-6.22912432085,6.26160115626,-3.48554924886,1.83394247508,-7.61795902223 -171,-6.79635647476,4.81763687682,-2.88341433446,2.40218881576,-6.7290747344 -172,-6.35740862152,4.86326806909,-3.90426205429,1.82254002966,-7.9740801346 -173,-6.41833669784,5.48670890573,-4.25844101784,1.56886914101,-8.73388254897 -174,-6.59970408703,6.96889783847,-5.12329365495,1.27291511887,-10.0637459177 -175,-6.89356714328,6.82125797655,-5.35183724204,1.28221224054,-10.299517114 -176,-6.53648411794,7.59971661375,-5.67104067257,0.937822866785,-10.6853124564 -177,-6.34795240544,7.43562146373,-5.59471665917,0.687679154495,-10.7924921375 -178,-6.74375937622,7.84481794642,-5.77388691142,0.671113346933,-10.8776476236 -179,-6.27766821353,7.77861696919,-5.76077074367,0.546799950703,-10.9823512841 -180,-6.49156694365,8.4168114601,-5.10671295801,0.767265383419,-10.2992665517 -181,-6.93745307283,7.99884405249,-4.49778243184,1.5053842749,-10.0219833424 -182,-7.62614402526,8.28362768788,-4.8817171443,1.58756869903,-10.0113326358 -183,-7.45408528272,9.21062050653,-4.50836413806,1.55357765269,-10.36611078 -184,-7.57686652686,8.34558837325,-5.11443691137,0.963096126747,-10.5108606964 -185,-6.92122064974,7.51524254836,-5.57250467259,1.01375152529,-10.7968229459 -186,-6.72655857299,8.04758597168,-6.21552896393,0.446764941624,-11.2492272194 -187,-5.85311297855,9.03487849651,-6.69493774758,-0.195398222154,-12.3710872296 -188,-6.00899190946,9.38922023344,-6.97730266135,-0.380856322772,-12.2637154276 -189,-6.49145245774,10.1647973339,-6.28129203131,0.115144503479,-12.3743463557 -190,-6.27834310837,9.99838303926,-5.88823594787,0.137306166694,-11.9359447297 -191,-6.18887468225,10.0129537645,-5.36641590697,0.556318460862,-11.2236130627 -192,-5.38824249705,8.39926750316,-5.62662536761,0.112733314113,-10.8342041582 -193,-5.94700531263,8.45621383638,-5.72646583607,0.104835375975,-11.0102721795 -194,-5.88731938235,7.87130356003,-6.04562962765,0.0582654951822,-11.0785293513 -195,-6.12296915533,7.67676320126,-6.08425425732,0.128442314493,-10.870182626 -196,-6.76745956648,6.90599173991,-5.86846152418,0.366938513283,-10.7343594584 -197,-6.61306016783,7.05872603181,-5.47259551971,0.977253158729,-10.3434841626 -198,-7.09239521343,7.14941614266,-4.80874910844,1.01243852786,-9.66069391146 -199,-7.38673297862,7.43814214737,-4.44623639066,1.5645798163,-9.15539407176 -200,-7.48246373967,8.55940983389,-4.33638555438,1.2240147956,-9.53626954869 -201,-7.21850098183,9.1950295545,-4.730392637,0.874271961768,-10.091394825 -202,-6.9392406751,10.2887634467,-5.41488091389,0.18265008598,-11.3076106182 -203,-7.26584899566,9.12976316461,-5.74460604835,0.187796976379,-11.0112649197 -204,-7.51134356753,9.05338864894,-6.56081603752,-0.0561586757562,-11.7831914994 -205,-7.97118026399,9.76330823758,-6.26730117464,0.376533822272,-12.2031607289 -206,-7.45868593516,10.2530837275,-6.07391131954,0.364262738555,-12.0736691866 -207,-8.12691253342,9.24459713855,-5.97428342628,0.635535612617,-11.2507334533 -208,-7.20091748623,11.1013098384,-6.83778235396,-0.138780408438,-12.9427713046 -209,-7.61964438298,10.327417776,-6.98359476483,-0.0664752350588,-13.1302270255 -210,-7.79613275942,10.837752242,-7.28147308701,-0.474615333034,-13.3508962504 -211,-7.78521335775,10.3730194336,-7.35618333602,-0.289191229756,-13.6843532603 -212,-8.04127663087,10.6233080294,-7.48542165156,-0.644399871806,-13.6645997419 -213,-8.01756661016,10.391940659,-7.98425489911,-0.962726403901,-13.7924918661 -214,-8.59317772223,11.1119446083,-7.40280873564,-0.497346500246,-13.7378251017 -215,-8.38305522644,12.1987008902,-7.45623668664,-0.582090762387,-14.3403441054 -216,-8.67060276519,11.5901567318,-7.25709578849,-0.515554082904,-13.7864233282 -217,-8.349348253,11.8027493028,-7.54164271484,-0.813689735705,-14.348414961 -218,-8.78380465277,11.3381534207,-7.5020327912,-0.355159369441,-14.0982531397 -219,-9.3214734511,11.1679500639,-7.67333636,-0.4250652702,-14.3222456079 -220,-9.15694333638,12.3560660117,-7.92809167696,-0.588298226567,-14.5592229741 -221,-8.73166048156,12.9604680058,-7.43962374061,-0.292518475383,-14.3984790658 -222,-8.97336044919,13.74188509,-7.35793426906,-0.159817176776,-14.756374594 -223,-8.75144955694,14.0589520743,-7.54994899643,-0.462314767605,-14.7541319882 -224,-8.63086320415,14.0005000344,-7.51702697384,-0.397726224712,-14.6442062488 -225,-8.70762188928,16.0671192568,-7.00784449217,-0.396231113707,-14.8201329203 -226,-8.61581099463,15.7018046731,-6.94624241624,-0.350128048379,-14.5712260419 -227,-7.85516546328,16.5305738539,-6.36847585564,-0.0971574185121,-14.2732253652 -228,-7.44856034418,16.6129622221,-6.95911528586,-0.687249360667,-14.7132252284 -229,-8.29375420796,16.16225917,-7.06728376394,-0.718870475558,-14.686513317 -230,-8.46427112341,16.0560304351,-6.8922471382,-0.677120915865,-14.7286081861 -231,-8.88992293868,14.8914382303,-6.90414048177,-0.666284849292,-14.1391128579 -232,-9.31120795733,14.0459663856,-6.75198833777,-0.154635341816,-13.7273488654 -233,-9.37635859426,13.7418797754,-6.6410488689,0.0613764927805,-13.4681990315 -234,-8.80448017418,13.6388580502,-6.99969731685,-0.0900762552979,-13.8724315274 -235,-9.01006306752,14.2852668918,-7.25491354287,-0.409362384944,-14.382897651 -236,-8.57792765048,15.7096818747,-7.00877065001,-0.595308753776,-15.0260436319 -237,-8.60184033492,14.3796851876,-7.19521914664,-0.550311620755,-14.265822895 -238,-8.94163840381,14.3183598613,-7.0359158869,-0.400976116498,-14.2532990052 -239,-9.3070203095,15.5652000641,-7.95128252992,-0.639667873342,-15.5896992352 -240,-8.82016405663,15.7330800842,-7.86869074318,-0.933230358533,-15.7089395104 -241,-9.4856501791,15.7912223759,-7.80735482929,-0.668568530685,-15.7856988097 -242,-8.9900527358,17.3718918969,-8.14598138032,-0.891178246628,-16.4867759227 -243,-8.88752095529,17.0948456317,-8.65523789694,-1.19384456759,-17.2502909815 -244,-9.57142544421,16.3720586957,-8.25086719699,-0.707862475354,-16.3197585858 -245,-9.48896482888,15.6989106851,-7.66201034479,-0.619600466391,-15.917874378 -246,-8.79593217567,15.5835619365,-8.24452537259,-1.12780083711,-16.3627709762 -247,-8.57322176601,15.8140158209,-8.52836593325,-1.08463712668,-16.5510991233 -248,-8.71874430208,15.3577624571,-8.62679442048,-1.37193794843,-16.5463190818 -249,-9.33830295829,13.0154673362,-8.49285848946,-1.00041668691,-15.6359758613 -250,-8.83607864155,13.574529239,-8.73003933687,-1.0069409092,-15.8121118717 -251,-8.73419046103,14.1413919698,-8.45361543575,-1.05530996662,-15.7008095844 -252,-8.51250301902,12.8454604082,-8.27395071761,-1.13899488147,-15.3212622241 -253,-8.42164535712,11.8453765781,-8.50516745095,-1.42570410787,-15.117382849 -254,-7.93476425082,12.2082524385,-8.46929344259,-1.46632372125,-15.0349582594 -255,-7.53426932033,11.3727331533,-8.77796978976,-1.76328714062,-14.9524826889 -256,-7.66194424498,10.6074570685,-8.02922988632,-1.42416786209,-13.9694181048 -257,-7.87020349787,9.28265577492,-7.82107325808,-1.31988163397,-13.7307340377 -258,-8.18580557787,9.65222218113,-8.53500645205,-1.30714240789,-14.2405722581 -259,-7.88996563906,10.2870025349,-8.4651922959,-1.4478220229,-14.5529501083 -260,-7.10369597376,10.1606032203,-9.39594661371,-2.33084317893,-15.1718326936 -261,-7.09484890298,10.809308086,-9.00822404135,-2.35340975254,-15.1491036286 -262,-6.97923701083,10.894171574,-9.34356851911,-2.3552075855,-15.3790138989 -263,-6.54815714427,10.5558119327,-9.60943596808,-2.83709923394,-15.6984261838 -264,-7.52466864813,10.6074764226,-8.81546833145,-2.12697832338,-14.8605232114 -265,-6.68068650388,11.4325142332,-8.6458933814,-2.0858974939,-14.7757513859 -266,-6.72542130087,10.7652185601,-8.92488578669,-2.55857898751,-14.8408148286 -267,-6.36853891919,10.0330817319,-8.06704438026,-1.97650693552,-13.7565890029 -268,-6.75972177878,9.46254218998,-8.17519398045,-2.0356655338,-13.7296226176 -269,-6.72830368775,9.95996574271,-8.09966107306,-1.88096455638,-13.4537590725 -270,-7.25281069894,9.00039802252,-8.43087794948,-1.59518807804,-13.8324318848 -271,-7.34676646312,9.2332963445,-8.87758902215,-1.91851145778,-14.2402770285 -272,-7.60555427514,10.3683663569,-8.62960432947,-1.8715362348,-14.4440048398 -273,-6.34609526555,9.56457806807,-8.82058757395,-2.32053449485,-14.2312523277 -274,-6.27500527223,9.7769411337,-8.50044891775,-2.04382988555,-13.8562530462 -275,-6.27120764234,10.1217570647,-8.18775047076,-2.10760833342,-13.8255438186 -276,-6.49987895418,9.1911629968,-8.45083088562,-2.08796801275,-13.9944685673 -277,-6.35046807084,10.2752194384,-8.69121785385,-2.11255263809,-14.3500695934 -278,-6.66043619212,10.0482845632,-8.07443586272,-1.68107376781,-13.7766137644 -279,-7.30234454298,9.44138356831,-8.48265665138,-1.99144464045,-13.9109921893 -280,-6.66700071077,9.75725682908,-8.50659322657,-2.55844250149,-14.0404929251 -281,-7.17284956409,9.3309388284,-9.02423273637,-2.65154399393,-14.4188418891 -282,-7.75673997054,8.86248796819,-8.93104147921,-2.18942615812,-14.2328347205 -283,-7.9307794564,8.30911121619,-8.82805191069,-2.18434065675,-13.9540073488 -284,-7.94515487849,7.16633108103,-8.93261674774,-2.05258990224,-13.921181043 -285,-6.89073732868,9.0669674291,-9.27322930982,-2.69984976871,-14.2888970314 -286,-7.68149430413,9.526019294,-9.37317120005,-2.50239215959,-14.8762473099 -287,-7.87506432698,9.55682993661,-8.73916120297,-1.98556331574,-14.4020632243 -288,-7.9308171581,9.27936808462,-8.96875334983,-1.99200813219,-14.4681893019 -289,-7.71933923761,9.52466498356,-8.57369359544,-1.97148092967,-13.9167103945 -290,-8.16797108102,9.0930328534,-8.36464379169,-1.63034152109,-13.6595520431 -291,-8.1816768367,8.32626244106,-8.61943531951,-1.78181836368,-13.9995756587 -292,-7.9645109582,7.06701926668,-8.44493703906,-1.74330898922,-13.4519493468 -293,-8.70877689778,6.488141635,-8.5036551661,-1.33518463487,-13.1214932344 -294,-8.21130712747,5.89761163474,-8.67487122629,-1.22358402498,-13.1215056619 -295,-8.11184176326,5.69659470364,-8.35843148429,-1.25345246377,-12.7367609041 -296,-7.59679352613,5.29345996205,-9.14700078738,-1.89139702176,-13.4180099999 -297,-8.14593559557,6.0823772954,-9.17188932674,-1.79373143414,-13.5899514482 -298,-8.25434689916,5.67908362977,-8.67798309869,-1.50536157367,-12.9812208909 -299,-8.31992334944,6.01770976425,-8.36447676215,-1.62051875941,-12.7740048704 -300,-8.55973573267,6.24266175088,-8.38454488839,-1.48289072379,-12.9518594323 -301,-8.07098868066,7.13097695654,-8.70841035451,-1.81933335558,-13.4692357278 -302,-8.00738027817,7.48517128583,-9.1175107329,-2.08638442837,-13.6289945429 -303,-7.67555797247,5.85535456848,-8.57835902147,-1.99672216847,-13.4519936916 -304,-7.17113560979,7.897123317,-8.58077837481,-1.9849486514,-13.5609516831 -305,-7.74501395333,8.23761870152,-8.02337810445,-1.80279122296,-13.1100399173 -306,-7.83553877289,8.08692810618,-7.79517677,-1.54128934086,-12.5675110288 -307,-7.59936150426,8.13782680576,-8.1637459926,-1.81708440891,-13.1358857094 -308,-6.8797520041,8.40385878651,-7.8875084341,-1.92904732502,-12.8213860036 -309,-7.33433976102,8.08088891939,-7.77382848857,-1.76305237547,-12.717040465 -310,-6.68713926075,7.72749856298,-8.21678039046,-1.90270285054,-13.2127751796 -311,-6.70434428608,6.61533749738,-8.9877239181,-2.35288063661,-13.7545458859 -312,-6.62325074124,5.36985664044,-8.53357076753,-2.07858885262,-12.6706567245 -313,-6.36793176198,4.67411807807,-8.93003824436,-2.48360228543,-12.9012562645 -314,-7.4251367394,3.76718041663,-8.75924948942,-1.98051655899,-12.3486642241 -315,-7.74323298007,3.9002170746,-8.70997761306,-2.02075466455,-12.4530152562 -316,-8.16195993167,3.05809953596,-8.37083266122,-1.46263058118,-11.8299483584 -317,-7.76294402749,3.33339938209,-9.0169500054,-2.01322423303,-12.7246180699 -318,-6.77300251583,4.1381755879,-9.96437118554,-2.64977253693,-13.9334170269 -319,-5.68919346979,4.15024496509,-10.1822536661,-3.02376284478,-13.8262785847 -320,-5.6660630642,3.32349919292,-9.56251399298,-2.66613135588,-13.2866909086 -321,-5.77263052353,3.54761446413,-9.59000948626,-2.77468082443,-13.340576542 -322,-5.59178974833,3.79160216461,-9.32578298592,-2.64936489619,-13.0391774098 -323,-5.9441302159,2.93665441759,-9.24012497249,-2.44269179206,-12.8489809931 -324,-6.22776210297,3.01137237874,-8.99107354733,-2.06959185216,-12.6158805468 -325,-6.04632024125,2.74512311451,-9.31973261204,-2.2294248457,-12.8887092527 -326,-6.39429216352,2.58110958534,-9.31349087742,-2.13489797997,-12.5581005801 -327,-6.35202191405,1.94057814029,-8.81292014758,-1.99170382564,-12.3683404001 -328,-7.30664497735,0.646753225813,-8.44332778639,-1.43134623832,-11.0274594572 -329,-7.03922913692,-0.172030452415,-8.08523119806,-1.33238525219,-10.4035148775 -330,-7.638767446,-0.215797324305,-7.58528456832,-0.535244386178,-9.94757304346 -331,-7.69345658182,1.0095577025,-7.42572493956,-0.764915839276,-10.0734793193 -332,-7.9740779231,1.24699877584,-7.43457837986,-0.695000777285,-10.3275779811 -333,-8.13450555948,1.0851730114,-8.26439951721,-1.18285132409,-11.1064610788 -334,-7.64604201635,1.17190534668,-8.32846475808,-1.26428583568,-11.1978145063 -335,-7.86227590763,1.23644329355,-7.98241891083,-1.01881893793,-11.1019154916 -336,-8.07392084404,1.29285054701,-7.75112983568,-0.887180853053,-10.3990134854 -337,-7.77074799392,1.33571271474,-8.17261149451,-1.2961960474,-10.9697625056 -338,-7.54518040047,1.35571580111,-8.36968178671,-1.63184603769,-11.3848167619 -339,-7.26396147425,1.25194427727,-8.2594101378,-1.84017201169,-11.0964468548 -340,-7.21692660812,1.17463895033,-8.24530098759,-1.72745172455,-10.9309613177 -341,-6.96431233538,1.39566590865,-8.21893296452,-1.99333090699,-11.0208185194 -342,-7.22518846005,1.78587044509,-7.89452043194,-1.54125303747,-10.5967924562 -343,-7.41977517323,2.09881464935,-8.15550645686,-1.4683919704,-11.0518180815 -344,-8.65233938538,1.83137629284,-8.95072994464,-1.91710308696,-11.8900397288 -345,-7.96531007734,2.84497555763,-8.41060200443,-1.74403434745,-11.5965730615 -346,-8.17512800379,2.99324435503,-7.6687530415,-1.33699922955,-10.8538569021 -347,-7.66730341149,3.81846474142,-7.27749787852,-1.13206673388,-10.581276293 -348,-7.99246315234,4.44538302363,-7.3661376177,-0.847999173145,-11.1595182428 -349,-8.09542213161,3.62834552547,-7.03258734543,-0.634758408362,-10.356529982 -350,-8.27277001696,2.83535037977,-6.86005974941,-0.403647888916,-10.2592523154 -351,-8.26342778767,1.59305662629,-6.74207121258,-0.0105493464392,-9.70449602712 -352,-8.96298112824,0.937331350565,-6.87160234577,-0.0947280487254,-9.39585530657 -353,-8.37289297775,2.23714548944,-6.50881832377,-0.313517794165,-9.52361761042 -354,-8.72558987342,2.15129451527,-6.63801943091,-0.30500141967,-9.7663517167 -355,-9.69325911088,2.26938504001,-6.58079800818,0.23540993923,-9.63629749511 -356,-9.32607150829,3.26416240131,-6.44532904681,0.0140546277237,-9.68575588712 -357,-9.45213458555,3.39206029738,-5.64506189906,0.580562413607,-9.06915234442 -358,-8.79778827057,4.43827280973,-5.62252020993,0.374243769749,-9.27801372635 -359,-8.30978473075,4.33244702297,-5.82748073003,0.00249565593622,-9.30666107791 -360,-8.40294793053,4.37059764263,-5.61862134091,0.425251368895,-8.95415809954 -361,-8.30590324107,5.03925469249,-5.13844936139,0.0120098731111,-8.98056487952 -362,-8.76442947313,2.88260536591,-5.47206586821,0.0107759697454,-8.49820042931 -363,-8.10484566069,3.4190123307,-5.61698767617,-0.115206338007,-8.85192466698 -364,-7.82034518727,3.31449030434,-6.16597727057,-0.77011725995,-9.27594400942 -365,-7.5371576461,2.53681231277,-5.54995506174,-0.486611898012,-8.25319686113 -366,-7.77279623646,2.16250984029,-5.88440952329,-0.481927501937,-8.51532364379 -367,-8.15782617037,1.33003334758,-5.37147430649,0.0658360700383,-7.53393410332 -368,-7.83253589206,0.437254273227,-5.60155383897,-0.208864608532,-7.57863904568 -369,-8.26990979582,0.824040921398,-5.76367892903,-0.33466404889,-8.14395492128 -370,-8.54100293687,0.934081863545,-6.06588862301,-0.318402102047,-8.06765646531 -371,-8.10309392539,2.04844276819,-5.88457436783,-0.203370433555,-8.24963880732 -372,-8.55713704274,2.66130315091,-5.60092150186,-0.247773585611,-8.51034862129 -373,-8.97491690133,2.63089653151,-6.19344763285,-0.372412848934,-9.01424694103 -374,-9.46440345525,3.27883707552,-6.14904204358,-0.130038700067,-9.09291725832 -375,-9.76852837167,2.22005774597,-6.64842922659,-0.638330195338,-9.40888660449 -376,-9.81343356342,2.54796033493,-6.54654622474,-0.466846945321,-9.37728410355 -377,-9.90595390825,3.11472798511,-6.08319228033,-0.276385566216,-8.89707105564 -378,-10.0606978494,2.92644277261,-5.90574021754,0.0367629974542,-9.14893104395 -379,-9.50652925057,3.28266883242,-4.9613208755,0.342209248718,-8.10537624928 -380,-9.58418665872,3.8766496612,-5.24830100967,0.359084238186,-8.19590693663 -381,-9.89401168579,2.06957552028,-5.63523655415,0.145310208416,-8.19634691258 -382,-9.65468123662,2.02909735743,-5.32637534899,0.423842420979,-8.06404805054 -383,-9.79396204752,1.54333684439,-5.19759372657,0.602031889142,-7.71507075773 -384,-9.88286541318,1.78013548446,-5.53743142504,0.282331702022,-7.92410392346 -385,-9.48530868847,1.92025383026,-5.46791292084,0.225389019431,-7.94717342937 -386,-8.55400180014,2.84556540415,-5.22055031165,0.00423736484871,-8.25373689445 -387,-8.40696358222,2.42245234412,-4.72874143522,0.315896235064,-7.36916602825 -388,-8.14322786372,1.30858876418,-4.82083008604,0.424401643227,-6.98000265843 -389,-8.16291600017,1.69186634645,-4.51155283419,0.291003204101,-6.70316047617 -390,-8.53091874205,1.27144221699,-4.50718964342,0.840421450226,-6.68204395158 -391,-8.66087628811,1.48490925703,-3.97132998311,0.875778086732,-6.30483045504 -392,-8.40002328914,1.38502525426,-3.75445926445,1.18526369322,-5.73899118538 -393,-8.27669129654,1.13692236299,-3.86856389371,0.850751134598,-6.09108617681 -394,-7.31317837859,1.76308330778,-5.1213071419,0.114118785201,-7.32826075836 -395,-7.25022422516,2.96161225375,-5.62420024952,-0.732659758151,-8.33834224117 -396,-7.86073669368,2.84663589523,-5.85914635384,-0.387925476411,-8.77189288818 -397,-8.14796647926,3.38900309609,-5.7991352622,-0.527063650768,-8.83085023799 -398,-8.56664316014,4.60772362136,-6.12900909676,-0.398177419983,-9.53720460939 -399,-8.61231338753,4.92462337169,-5.78979304588,-0.453294363648,-9.30874415042 -400,-7.70873173115,6.16721620762,-5.85656227757,-0.239444088884,-9.77125080503 -401,-7.26972385405,5.61093419903,-6.30580446895,-0.949776276452,-10.1532343556 -402,-8.18118410514,5.08660551332,-5.79117039661,-0.422191796007,-9.4646787436 -403,-8.39302835758,5.0186308153,-5.98055682464,-0.286398046377,-9.77303263751 -404,-8.45120564479,5.43622599565,-6.01100741435,-0.0924211653169,-9.66587896574 -405,-8.77717487724,4.57784356666,-5.42815805348,0.274359490904,-8.69207026548 -406,-7.99762789661,6.97906775455,-5.75459747037,-0.325703879872,-9.92401567847 -407,-7.47412209775,8.12110138125,-5.43940822872,-0.334143670966,-9.88979621842 -408,-7.5076625296,8.16014031735,-5.39546303451,-0.66865947211,-9.80992172332 -409,-7.56219875714,9.34156848616,-5.54616791243,-0.681005635408,-10.1546704081 -410,-7.63574817061,9.31637788592,-6.00730915613,-1.06318316396,-10.6843343988 -411,-7.65555736237,8.59215891383,-5.71515975211,-0.546468229129,-10.2026101588 -412,-7.66567019955,8.2000572421,-5.28553211424,-0.346009703564,-9.74519606652 -413,-8.59988941769,8.81778864354,-5.60867089774,-0.173846094319,-10.5147296075 -414,-8.28473746502,9.73446559574,-5.52025007461,-0.51324797308,-10.6856067909 -415,-8.0514735486,9.87640655675,-5.28097052559,-0.483061911563,-10.2841109492 -416,-7.50275965972,10.66586562,-5.40068269393,-0.721122223517,-10.5494663524 -417,-7.38409558973,9.84989909125,-5.51700728169,-0.551493987826,-10.6729821817 -418,-7.89270233241,9.4678135448,-4.80993870943,-0.0410824462506,-9.81241582448 -419,-7.73693337744,9.74385841501,-5.06706350035,-0.227931465969,-10.2169061939 -420,-7.25289280104,9.75097055911,-4.68360735941,0.0725458155731,-9.63680971649 -421,-7.85447518349,9.41684052024,-4.78934582494,0.0277483434469,-9.916599754 -422,-7.21944245027,8.15535815451,-4.33769370049,0.224186164904,-8.65592658869 -423,-7.40298953781,7.81105918663,-4.73301890265,0.106922482561,-8.94538459399 -424,-7.73029367255,7.60646949382,-4.68875474343,0.166924934716,-9.05172100717 -425,-8.05324202277,7.27356876639,-4.41979641869,0.130739269101,-8.59688325423 -426,-7.81084501956,7.7930148114,-4.29311237575,0.405559230994,-8.54447769864 -427,-7.53417549554,6.67778131612,-5.00513140275,-0.326244682892,-8.99173502529 -428,-7.7357111047,7.62045657247,-4.82306327693,-0.132391671396,-9.11775513748 -429,-7.7911062034,8.88230287703,-4.42487324004,-0.113841966333,-8.88818802495 -430,-7.83530916906,8.96587714813,-4.69677204484,0.0187803889705,-8.97989947253 -431,-7.42014546711,9.52102154038,-5.213329279,-0.666141912715,-10.2570045187 -432,-7.90113403405,8.94855601903,-5.43017559315,-0.416684186876,-10.2555224557 -433,-7.77056066986,9.67406943057,-5.27412712919,-0.330577152846,-10.225851485 -434,-7.53729176639,9.09677751245,-5.44013208696,-0.378973572394,-10.1139227671 -435,-7.90999682194,10.1914292853,-5.73555243988,-0.669245398375,-10.7891552325 -436,-7.07468896027,10.2961189,-5.83629197868,-1.20419418996,-11.087497576 -437,-7.76007059592,10.3100931082,-6.22272952033,-1.07571510279,-11.1875318833 -438,-7.96264221574,10.0917413626,-6.61961853455,-1.31866473913,-11.5703014082 -439,-7.70634352868,9.48925131874,-6.02747636691,-1.20374307985,-11.0233131669 -440,-7.92952012287,8.176289281,-6.25078541743,-1.213336004,-10.9163228899 -441,-7.61000038351,7.60002390721,-6.58005538523,-1.60599428606,-11.0160160331 -442,-7.71997695775,6.95657228664,-6.64504615318,-1.18628364509,-10.7089656534 -443,-8.45792022606,5.68756171623,-6.11406348849,-0.455851618868,-9.96455616534 -444,-8.63704101162,4.52356954888,-5.76194770581,-0.0735564578827,-9.32882534384 -445,-8.83852254797,4.59575340633,-6.26772767449,-0.304832549365,-9.89891221823 -446,-9.28388744959,6.76240785433,-5.62508526868,0.0406823127046,-10.1534140373 -447,-8.99402212503,6.05994562464,-5.1722868443,0.183630881805,-9.22305412258 -448,-9.46860166801,6.33427372021,-4.56946478971,0.81492218602,-8.3865172097 -449,-9.44904192017,7.10371649114,-3.84718258557,1.14522132528,-7.83567030285 -450,-9.30068905891,6.26780398737,-3.76264645987,1.05703889768,-7.60883361447 -451,-9.05363785907,7.08639425464,-3.5723518982,1.19752274819,-7.77917735641 -452,-9.45011447489,7.69958976553,-3.65613567541,1.26177311127,-7.86654932962 -453,-9.37828458035,7.90189506739,-3.18362808079,1.26932312211,-7.57080295941 -454,-9.00342903927,8.59705538568,-2.74452614018,1.41558921167,-7.40040203318 -455,-9.30866334623,8.92724332929,-2.82863765568,1.8799201478,-7.44004070415 -456,-8.80460535166,9.35788488726,-2.88301106748,1.46270161512,-7.76989572533 -457,-7.83491155484,9.05252174014,-2.61219108646,1.243057125,-7.23600233632 -458,-8.00626585156,8.99523619937,-3.24749002239,0.984196902256,-7.88996734183 -459,-8.28543558439,8.89861964252,-3.22518774513,1.45142473469,-7.76297147151 -460,-8.6252064253,8.86482028319,-4.1729426727,0.564955757709,-8.66786554543 -461,-8.86659274988,9.04428287641,-3.93088339641,0.760658009952,-8.65573753271 -462,-7.83673197153,9.67167516954,-4.45977710346,0.202469477781,-9.27234575235 -463,-7.61283804644,8.77427878023,-4.32300177412,0.211056749915,-9.39567839173 -464,-7.97006867205,9.2437852131,-3.91591221507,0.551959380889,-8.70345380127 -465,-8.12954182258,10.5250892462,-4.0265848175,0.620954610909,-9.2153873358 -466,-8.39415189099,8.79267278439,-4.24443524754,0.617886880166,-9.01116625927 -467,-7.75939747441,10.2206349381,-4.32542056965,0.0382391624687,-9.55060037024 -468,-7.3878455675,10.5931317384,-4.67966200294,-0.32270383705,-10.018580555 -469,-6.62747411302,11.3852625973,-4.11973958456,-0.138222869787,-9.61881878072 -470,-7.32967370223,11.436494309,-4.43570985115,-0.0384019039489,-9.76173887141 -471,-7.60119371513,11.7167191169,-4.29435287095,-0.138608189796,-9.94846811681 -472,-7.56984590221,10.6323592942,-4.83278406931,-0.554011667964,-10.0689564692 -473,-7.98024487248,10.3345816287,-5.21855708438,-0.345941788809,-10.2299248207 -474,-7.39348086618,11.0032546009,-5.59831945775,-0.989658264728,-10.9366152086 -475,-7.29383837744,10.8331985202,-5.69579470023,-0.914770698374,-10.9477323007 -476,-7.06461535502,11.1040234457,-5.77558791439,-1.25593234237,-11.1211327558 -477,-6.88896938016,11.5735224439,-6.39115832553,-1.7624374916,-11.9493852064 -478,-7.55671754218,13.4943718782,-6.46196393323,-1.65053404919,-12.4846864751 -479,-8.38609650853,13.1134194359,-6.76328291227,-1.73090050038,-12.8709323161 -480,-9.04284843042,14.5239363857,-6.74779448588,-1.73952140978,-13.0705720039 -481,-8.60716065284,14.8954501626,-7.1352385002,-1.98104858032,-13.5983782215 -482,-8.72574238064,13.7802052507,-6.98589932587,-1.97446305837,-13.338191356 -483,-8.73482675513,13.8210408929,-7.75921259593,-2.41182448906,-14.0620638562 -484,-9.06670465571,12.5822383107,-7.16145451299,-1.76309768158,-13.0663818785 -485,-8.33823360512,12.7726425667,-7.79413096054,-2.49821005812,-13.7482397933 -486,-7.80803101857,13.6969917396,-7.6772874059,-2.86938488621,-13.9728420688 -487,-7.90706317854,13.6001196399,-7.55718412173,-2.55031853651,-13.6750813602 -488,-7.70543957218,13.7961012533,-8.13150113839,-3.12691380322,-14.3613180308 -489,-7.46745148663,13.8870335928,-7.98469677293,-3.15082657539,-14.031984076 -490,-7.56894929039,13.529133025,-7.86581980201,-3.4904976915,-13.9537179422 -491,-7.90817321347,12.6224920923,-7.76928527731,-3.07432375361,-13.3906077063 -492,-7.74395775214,11.9421206595,-8.2383068654,-3.54607771194,-13.7826114429 -493,-6.89149021797,11.1459154364,-8.84651776388,-4.02705388465,-14.2798531336 -494,-6.83004452131,11.3949568352,-9.02745233278,-4.09550125277,-14.2779559998 -495,-8.20499265384,9.92810262799,-8.23624802267,-3.09030964092,-12.9438913323 -496,-7.90265306486,10.3879136248,-8.47991644205,-3.42574193544,-13.5207488054 -497,-6.99430831807,10.2745360103,-8.63073093963,-3.48171940609,-13.6908123216 -498,-6.5341483316,10.8149075985,-9.23225598149,-3.82382844706,-14.4066704898 -499,-7.22665817291,11.1019560664,-8.95168258846,-3.53128619196,-14.1223733425 -500,-7.86277256547,10.60752055,-8.45029118876,-3.0795014755,-13.8857299247 -501,-7.62848920276,11.4835661568,-8.6249884299,-3.37504848417,-14.4473529862 -502,-7.60851695553,11.4903822799,-8.42751680283,-3.17082651098,-14.151223807 -503,-7.41243008797,11.9379955165,-8.55682214019,-3.15903468408,-14.4977835601 -504,-7.64888210927,12.0776474255,-8.30432405903,-2.92603812562,-14.1611867869 -505,-8.6741774247,12.5280743316,-8.17741652592,-2.39848547493,-14.1288630903 -506,-8.41238889407,13.451442609,-8.03745344589,-2.83762630479,-14.5714832036 -507,-8.77066432459,13.8056270205,-8.52337183467,-2.92202821614,-14.8586924241 -508,-8.56343769158,12.9475210923,-9.14684844463,-3.26053368467,-15.6518225187 -509,-8.464455479,13.7823831968,-8.58680769101,-2.65905012676,-14.9408435091 -510,-8.85276625908,12.3419527514,-8.25968756831,-2.65926448761,-14.148178069 -511,-8.74103528242,12.492267677,-8.16660354159,-2.79576840443,-14.0153826665 -512,-8.6953577304,12.2037388265,-7.48737064047,-2.34766806188,-13.5043309345 -513,-8.31867578197,12.8511848476,-7.63850544137,-2.31687292089,-13.4887027518 -514,-8.72089212441,11.3608422123,-7.15361856699,-2.26748317198,-12.6046672194 -515,-8.97049831731,11.8138559199,-6.88358866448,-2.22921272844,-12.8922632333 -516,-9.03794706422,13.1022633982,-6.90344107633,-2.04962429964,-12.7236624824 -517,-8.76051670559,13.4409848492,-6.58883983792,-2.05127921863,-12.6074998148 -518,-8.91338039346,12.6519258915,-6.64030013965,-1.82264899838,-11.980924941 -519,-9.53039972035,11.9926068536,-5.73664271767,-1.19906734984,-11.3024098119 -520,-9.56933539459,12.4738179657,-5.36522119534,-1.27302916004,-11.3220620935 -521,-9.27093923233,12.1809713568,-6.05137153051,-1.56542168147,-11.6892131865 -522,-9.10470381783,12.8443664375,-6.17461429576,-1.70650623113,-12.2023759588 -523,-8.70481209955,12.8757796631,-5.62283784861,-1.02909920774,-11.3078745886 -524,-8.36832904481,14.0887254045,-4.54259677738,-0.756947847904,-10.7469425555 -525,-8.27192838772,13.5284975888,-4.98814929013,-0.982554080124,-10.9537711845 -526,-7.81605932555,12.3349289299,-5.04685757359,-1.063642532,-10.4764999707 -527,-7.62519968994,11.762425275,-4.95656489865,-0.900379743579,-10.276332642 -528,-8.1882585754,11.3375643458,-4.91191857095,-0.807628368749,-10.0569120581 -529,-8.45941601411,10.454784064,-4.74170003843,-0.542878703538,-9.73767739866 -530,-8.73117345399,11.4131380545,-4.28437756707,-0.327483941886,-9.51709313378 -531,-8.68143454688,12.1767437654,-4.44634362562,-0.355483985965,-9.99097182509 -532,-9.07184340599,11.2145931578,-4.61698978583,-0.283813726328,-9.90119512012 -533,-8.36517669787,10.90191099,-4.73905011775,-0.0679401761346,-10.0480795818 -534,-8.12854443861,12.7793891363,-4.60968203496,-0.684053053687,-10.4959523652 -535,-7.82630271147,13.2889108151,-4.03395603121,-0.0526246773507,-9.65707971333 -536,-7.13754030246,12.33305472,-4.62251351717,-0.513591809941,-10.0506041372 -537,-6.79153765699,11.5254685522,-5.28642436084,-0.796837909631,-10.7505435107 -538,-6.61249938734,10.5939083448,-5.29295891382,-0.916465477084,-10.3119753157 -539,-6.83068917195,10.4725693866,-4.36085444131,-0.368642493868,-9.382408942 -540,-5.6249721521,11.927704318,-4.24339737787,-0.619606917631,-9.81949407439 -541,-5.82758001032,12.6964738173,-4.03365702004,-0.416245692851,-9.38431980968 -542,-5.46903104807,13.2779186097,-4.12343285594,-0.682641438986,-9.87646142634 -543,-4.97778064306,13.4885100764,-4.2759284571,-0.793641438115,-9.72648765469 -544,-5.0560312196,14.051502874,-3.71367375886,-0.489199656744,-9.81689350133 -545,-4.55187916276,14.2140313756,-3.70650309796,-0.655470642907,-9.73140394667 -546,-5.04254752385,13.6516363661,-4.30195859918,-0.728160625021,-10.180321458 -547,-4.26246022655,13.9133421333,-3.84133734094,-0.583289916747,-9.97918976505 -548,-4.30214989462,13.1366364502,-3.81401849127,-0.608200897574,-9.74794696739 -549,-4.35823825962,14.4217154288,-4.46489684024,-1.00864119121,-10.7181584852 -550,-4.32602595593,14.5305788233,-3.98466464503,-1.03214650642,-10.1469047302 -551,-4.02879976477,14.7398679005,-4.06644601558,-0.721154120771,-10.2467158925 -552,-4.27308512946,14.6201722033,-4.55144033265,-0.952365352585,-10.747980141 -553,-4.28874816482,13.7473321846,-4.79460329121,-0.955454563345,-11.1410125472 -554,-3.74881459131,14.20530683,-5.36614021922,-1.34080791792,-11.8195624562 -555,-4.20199718521,13.5226440531,-5.81821952939,-1.38950062498,-11.9642409495 -556,-4.40355016096,14.3638955592,-5.87343422489,-1.50608877181,-12.4185984184 -557,-4.45454402232,15.042505325,-6.1696896542,-1.68202900454,-13.0148345617 -558,-4.31821523634,15.3293822395,-6.2316188467,-1.90132290723,-13.2911360142 -559,-4.04312675663,15.6577447607,-6.72475537859,-1.91372353818,-13.6463671282 -560,-3.42184078061,15.3309173613,-7.40708003453,-2.55459473853,-14.3931190219 -561,-3.46542523055,15.5265808332,-7.15205396266,-2.41805844221,-14.4278386867 -562,-3.78797354277,15.2357698666,-7.24297919198,-1.98435074217,-14.2522705939 -563,-4.20737944965,14.4814608454,-7.09199259694,-1.95163465121,-14.1030466685 -564,-4.54269522253,12.2715493338,-7.654516447,-2.1202464768,-14.0360563518 -565,-4.47303696956,12.2247565117,-8.17007775873,-2.19147781262,-14.8423105568 -566,-4.84128186341,11.8935839328,-8.38936181625,-2.18331007359,-14.448125846 -567,-5.12214988866,11.5443487689,-8.47707379112,-2.32353455782,-14.68071878 -568,-5.25781192718,11.1259589377,-8.27449247298,-1.89947257662,-14.1643465599 -569,-5.28658831291,10.7458012983,-7.88058968271,-1.50991744653,-13.7810715039 -570,-4.95171525155,10.2711007527,-7.59925773373,-1.55919448714,-13.5703836116 -571,-4.79287660952,9.96925879248,-7.58528757465,-1.68185852437,-13.4275989095 -572,-4.91077459307,9.35293272808,-7.34826410505,-0.994021443761,-12.632467089 -573,-5.02873222517,9.7990983064,-6.91338823615,-1.01585391571,-12.5090859382 -574,-4.62672655999,10.4287748512,-7.19565884292,-1.06561909499,-12.8337233374 -575,-3.9909839471,10.5360769285,-6.3867871409,-1.06884544163,-12.4760850685 -576,-4.12511021435,10.865201491,-6.63547505921,-0.909982534914,-12.5158359497 -577,-4.27063822465,10.7290321497,-6.91243856629,-1.30452901855,-12.9756509074 -578,-4.56000092545,12.508475048,-5.96571115089,-0.625251071604,-12.3235128486 -579,-5.20045551281,11.7188498044,-5.88311399809,-0.454483225893,-11.9946006971 -580,-5.94018271083,9.90268294036,-5.60932152216,-0.0347275761663,-11.3316152655 -581,-6.35766764757,9.20913413382,-5.2576362716,0.162587884754,-10.849867214 -582,-6.72627744267,8.18433300416,-5.29058658023,0.524518988078,-10.3179059859 -583,-6.51015900161,7.91043597658,-5.02336734175,0.475116906637,-9.8179849834 -584,-7.25652757603,7.80180711227,-5.14917539965,0.684433012121,-9.93995920855 -585,-7.44035353094,8.20685317741,-4.9611278416,0.810327631401,-9.84229115563 -586,-7.62766364006,7.92427710556,-5.1492800484,0.748908880878,-10.141460514 -587,-7.70178958959,7.63113921284,-5.43877867225,0.752647825505,-10.4246339377 -588,-7.49106609626,9.14308677063,-5.48085850864,0.635582264394,-10.7402738296 -589,-7.25616895029,8.8736707429,-5.76380250306,0.132945563895,-11.0433448485 -590,-7.33431616966,8.14642933723,-6.09803772091,-0.0205251631087,-10.8770554984 -591,-7.43448468892,9.45371300809,-6.27090816414,0.014245651137,-11.7834224741 -592,-7.19098227612,10.2584564544,-6.38153071625,-0.402953286303,-12.1013384295 -593,-7.54879122218,10.8772018605,-6.44407288881,-0.196619130316,-12.4926897215 -594,-7.43726130917,10.7988362824,-6.52533498918,-0.424125937965,-12.2598369654 -595,-7.51923947359,10.576279005,-6.23134715896,-0.447945723202,-12.0205301554 -596,-6.57639833166,11.5521989554,-5.8161016976,-0.660638769623,-12.1212685319 -597,-6.45425915778,11.1904992511,-5.40884639078,-0.57603862369,-11.0086974409 -598,-6.65475002972,10.507567486,-5.8045033429,-0.546150253913,-11.0720770569 -599,-6.83628199523,9.98916308058,-5.72309371328,-0.47068711995,-11.3171561967 -600,-6.40270013575,9.50088787203,-5.59674002321,-0.522702858737,-10.9974602158 -601,-6.31702264897,8.71390106346,-5.84838538666,-0.722974436048,-10.8802371565 -602,-5.85616493326,9.66760249247,-5.86858496783,-0.979889372383,-10.8289542507 -603,-4.99700649684,9.47359543822,-5.9264016263,-0.957137982385,-10.90037621 -604,-5.96165331384,9.29344332534,-5.21791642348,-0.329339871814,-10.0527965597 -605,-5.59958623603,9.55233799498,-5.25838326099,-0.666078081926,-10.2800391203 -606,-6.28423323392,10.5618365531,-5.33635198344,-0.338854421759,-10.7994108691 -607,-6.20609011005,10.6712030539,-5.6477416912,-0.567773066416,-10.8319140177 -608,-5.89383316309,10.6117497366,-5.06653971734,-0.595664771056,-10.2450349708 -609,-5.39098713941,10.2826363488,-5.42351347656,-0.93382506455,-10.6656641642 -610,-5.37558699323,10.5928691156,-5.69331708589,-0.913659120363,-11.2238003136 -611,-4.79321498006,10.9584722697,-5.79209207482,-1.12055325463,-11.1181507679 -612,-4.20012794958,11.3358361906,-6.29262536462,-1.7997411365,-12.0616244934 -613,-4.06827298003,11.2260338308,-6.5078844824,-1.73864594698,-11.9520265037 -614,-4.68824364941,9.53765424762,-6.39107707531,-1.64430306595,-11.4008937318 -615,-4.333568712,10.2322219592,-6.16100882455,-1.61071064434,-11.3360552224 -616,-5.28896089746,8.81440811911,-6.11083146356,-1.09024225699,-10.6386340873 -617,-5.61895478343,8.97582579379,-6.26562522823,-1.25483168214,-11.4151122526 -618,-5.39151720994,9.10308049435,-6.29574940775,-1.29122403525,-11.3080905271 -619,-5.67004035877,8.13514388254,-5.95060921903,-1.07624631024,-10.7249156376 -620,-5.9409193667,7.82636509969,-6.01479703818,-0.747904744361,-10.7382074755 -621,-6.08666094717,6.45166087746,-5.6912383329,-0.384432155041,-9.92811621033 -622,-5.96197903317,7.46389005128,-5.59272861253,-0.292797751004,-10.1161188471 -623,-6.35515903365,7.93597796374,-5.91736775562,-0.48114585324,-10.9649848953 -624,-7.32092249928,8.06970979388,-5.5646187706,0.148659138064,-10.5784669781 -625,-7.67352757707,8.58622024166,-5.90106242674,-0.107241677064,-10.9309742618 -626,-7.82316942682,8.06324909482,-6.57837839745,-0.337796398254,-11.3582637987 -627,-7.79055569042,8.35663991703,-6.03415399688,-0.136005286013,-11.0348750904 -628,-7.66418439941,8.15778365353,-6.77675085366,-0.782683321613,-11.8135527176 -629,-8.47793636094,6.43751021678,-6.22516237698,-0.410418798614,-10.3963509426 -630,-8.37018996314,7.12293519844,-6.41155372408,-0.170493319344,-11.2222577492 -631,-7.76247910431,8.04775925117,-6.15816802589,-0.375175553598,-11.2454400195 -632,-8.64277045125,6.63034588782,-5.99218096017,0.285219294969,-10.585924295 -633,-9.19485691557,6.63463311394,-6.01348867194,0.446238666716,-10.6987401232 -634,-8.78628767952,8.67650487412,-5.95220465628,0.257241434434,-11.2890938009 -635,-8.86689196065,9.07477663572,-6.46028028093,-0.0860344247153,-11.8903971948 -636,-8.42919690551,9.35005160334,-5.26054753395,0.266549698447,-10.4563193072 -637,-7.96179979842,9.36795539252,-6.15509700803,-0.319844350195,-11.4919166459 -638,-8.21500353198,10.1720461638,-5.80720898706,-0.0432955320264,-11.3745673095 -639,-8.34997922566,11.4328723533,-5.62650314648,0.0635914309315,-11.2690101799 -640,-8.72450254851,10.6212277058,-5.14718048962,0.392934800088,-10.8287851994 -641,-8.86936806254,10.3823506314,-5.25234903323,0.406129888894,-10.8226662622 -642,-7.55340396531,11.4108988611,-5.66510528278,-0.00188711481323,-11.4863768062 -643,-7.96629936617,9.91534871051,-5.88423383214,-0.0196619591553,-10.9315552741 -644,-7.38327037015,10.0729998009,-5.77174333374,-0.412355343943,-11.1572623847 -645,-7.39457357296,9.16923250992,-5.83076222376,-0.407360151486,-11.0243956425 -646,-8.00404327268,8.75338244604,-5.88121439097,-0.128696293641,-11.0727732758 -647,-7.52676260797,7.84656868254,-5.18995203862,0.410626342705,-10.0560473406 -648,-7.56769890592,7.13922319886,-4.90610149255,0.616152320526,-9.28084331636 -649,-7.36422240599,6.95918083598,-5.50779580176,0.0734546001841,-9.98589746207 -650,-7.31465254398,6.2066469425,-5.74855248183,-0.147086471988,-9.70986816079 -651,-7.36877709902,6.40480609309,-5.52745982897,-0.262173192346,-9.96687184193 -652,-7.22248648288,6.12935925218,-5.50562603459,-0.0288147824838,-9.71613313098 -653,-7.06059909056,6.96185396183,-5.12170245121,0.25200954223,-9.33429804032 -654,-7.33100628333,6.53773446505,-5.47195358789,0.07057154315,-9.475894278 -655,-6.9820374743,6.64369997084,-5.59125586929,-0.21023749067,-10.0974227438 -656,-7.42528052115,6.00550992939,-5.87736367459,-0.276582527915,-10.058745816 -657,-7.8951722821,5.89341262122,-5.0277623395,0.513264104677,-8.9419088894 -658,-7.82426425136,5.86610467231,-5.42973604633,0.579845870883,-9.23760800538 -659,-7.6612805635,5.54851677406,-5.35767785247,0.274720415231,-9.21986637073 -660,-7.31096488925,5.30055162713,-4.68068538588,0.456142560505,-8.43103327044 -661,-6.99940783035,5.53626344284,-4.8967008757,0.214130846864,-8.69255980792 -662,-6.74412698867,5.67978658306,-4.35881995758,0.426349839973,-8.18350599855 -663,-7.20716008137,5.71759636047,-4.76794301802,0.376503569826,-8.58264487246 -664,-7.4422827163,5.09575033732,-4.65277280088,0.540370731978,-8.44354929701 -665,-7.72407252154,3.8368952519,-5.10912362071,0.832151509446,-8.31656354701 -666,-7.6756765792,4.02904996763,-4.67220063132,0.923193983332,-7.87159862485 -667,-7.44766395806,3.86614359714,-4.7944621399,0.543414838244,-7.87279478449 -668,-7.25937962602,2.93869048913,-4.74002146987,0.710791482998,-7.92797864948 -669,-6.52761452884,2.78202577741,-5.51790285557,0.0561235542347,-8.66875779618 -670,-6.68665186158,1.94821419299,-4.84544453174,0.510960451429,-7.63541307249 -671,-6.92875602961,1.83058897211,-5.23800466058,0.505447681518,-7.85641627241 -672,-6.63536433283,1.20444406247,-5.71702347349,0.300327255742,-8.35310333729 -673,-6.3384086199,0.73017334055,-5.48238397369,0.375710125434,-7.75390922244 -674,-6.34357000752,1.55663058861,-5.03470808266,0.359971047376,-7.76053956747 -675,-5.94028553326,2.43199585766,-5.09621433527,0.229659270944,-8.07821189072 -676,-6.25897679782,2.69406242016,-5.17138954514,0.42042113816,-8.16018619034 -677,-6.69209433793,2.0324795398,-4.66977280949,0.68411650407,-7.44078895275 -678,-6.40468704147,1.61071567704,-4.23146923035,0.832000027117,-6.65512888455 -679,-6.66132500822,1.99604864852,-4.53336489546,0.639292369325,-7.22463873813 -680,-6.91208712765,2.40989957967,-4.61332209194,0.576387344841,-7.69925913415 -681,-6.73765670651,1.78720953306,-4.92693953402,0.422258171775,-7.89048876601 -682,-7.64061157539,2.37987586954,-5.20850149612,0.630708319893,-7.79528337721 -683,-7.54679349991,2.17942194278,-5.61934679035,0.218386160231,-8.39483144812 -684,-7.54125834141,0.835304170411,-4.86935961238,0.553335337336,-7.525139522 -685,-7.5086273713,1.36116757282,-4.90558279198,0.794107455743,-7.23086456321 -686,-7.7461923385,2.4193649484,-5.51033221406,0.00654594237631,-8.0356851685 -687,-7.76241899392,2.21803593906,-4.95928237875,0.212900845441,-8.04726441212 -688,-7.0433409924,3.24860727507,-5.57623238173,-0.461690807678,-8.6688356437 -689,-7.28969708048,2.80967718149,-6.10414906222,-0.427454179703,-9.1015527772 -690,-7.39644892549,2.87510414223,-6.1875061726,-0.449133740241,-9.21862992049 -691,-7.37646265439,2.61267170972,-6.20944729039,-0.421285115907,-8.96867571003 -692,-7.09646306357,3.49152325194,-5.39579635573,-0.365591886467,-8.30953814114 -693,-6.50513394267,5.15855802041,-5.66665072601,-0.640210396427,-9.28055135762 -694,-6.13051463364,6.41389810361,-5.34176976591,-0.823623519687,-9.20537449094 -695,-6.36723598408,5.54328985421,-5.74475914008,-0.975366290819,-9.43252755039 -696,-6.91214369815,5.44605892685,-6.23017574508,-0.961233988667,-9.94932866501 -697,-6.65001944482,4.71339054555,-6.79598385537,-1.18895185732,-10.3258154228 -698,-6.6469536433,4.77953859565,-6.73005042362,-1.19496905249,-10.2752635942 -699,-7.4023108563,4.48256127238,-6.4695661708,-1.2425981984,-9.77229943843 -700,-7.67969706944,4.90067978358,-6.51013757105,-0.699847921714,-10.0406919 -701,-7.62267589332,4.29231248937,-7.04878184319,-1.12180371462,-10.2244480923 -702,-8.03913354476,2.23863428466,-6.74472847184,-0.646262716956,-9.73640472641 -703,-8.50723674577,1.89745906234,-6.49109370708,-0.444589897447,-9.54682908844 -704,-7.75029099976,2.98203193151,-6.94346904671,-0.801790845952,-10.2251154176 -705,-7.28785811927,3.13113336047,-7.3116859698,-1.4377314492,-10.6418381098 -706,-6.98934980143,4.02979535667,-6.89126357266,-1.31623495347,-10.6754242063 -707,-7.48665821261,3.61452387991,-6.73606064589,-1.19715140442,-9.92213030459 -708,-6.83419122043,4.67947228607,-7.84574578667,-2.039294219,-11.575591842 -709,-7.47840517635,3.92011152907,-7.61895070354,-1.85481713606,-10.9481770032 -710,-7.55326784057,3.28603090769,-7.44202529127,-1.21637628248,-10.8060757203 -711,-7.79525152167,1.79222368955,-7.17323621434,-0.980525616591,-9.99120396889 -712,-7.61516673261,1.60884638757,-7.13883506973,-1.17278481839,-9.89665740678 -713,-7.47273256367,1.64096134297,-7.18350600842,-1.09082692103,-9.83737076811 -714,-8.08073073502,0.34036709366,-7.85247598452,-1.45719294076,-10.1103382971 -715,-7.28504311012,-0.131516143604,-8.2850712132,-2.04627026327,-10.1841090123 -716,-7.35496138612,-0.698449616329,-8.41319192417,-2.19707895325,-10.2932025533 -717,-7.55975613801,-1.38771383611,-8.49930382214,-2.06348952289,-10.1121907007 -718,-8.18194259285,-2.24582797484,-8.60276926588,-1.96278368365,-10.0567472033 -719,-7.87723984599,-2.40513159387,-8.90856781967,-2.16986749468,-10.4375983978 -720,-8.54871094765,-3.20447198015,-8.52984265654,-1.75093818422,-9.6195506035 -721,-8.30887049766,-3.11039927891,-8.1370700895,-1.88962388648,-9.37946117952 -722,-8.52478123779,-3.07006155417,-8.44093129431,-1.97028186819,-9.40056065167 -723,-8.52959757054,-2.93103223485,-8.35386134974,-1.8306290944,-9.5524735806 -724,-8.40016469044,-3.240631443,-7.7859386619,-1.50196984774,-8.68319548012 -725,-8.59832053918,-3.51219301753,-7.52539429167,-1.13003941015,-8.33059335705 -726,-8.53674267008,-2.76618311607,-7.7530733192,-1.69087066846,-8.96196019468 -727,-9.19387946828,-2.09173011762,-8.11641103356,-1.68864155233,-9.70160036684 -728,-9.71516949212,-2.62482117394,-7.90969632308,-1.26813949299,-8.96905926297 -729,-9.52794567184,-1.68622279228,-8.28791102338,-1.60448067804,-9.91633883276 -730,-9.57183342665,-1.81496469212,-8.17395895818,-1.43372380124,-10.1839728234 -731,-8.99515542033,-0.250160261099,-7.75076894272,-1.36923957925,-10.0688109461 -732,-8.83295039135,1.25519263025,-8.41144950163,-1.8664787253,-10.5848038246 -733,-8.74828754976,0.699157910387,-8.04868435757,-1.54160207748,-10.5024249452 -734,-9.30583193211,0.983113690689,-7.5205459678,-1.11311517965,-10.3627201353 -735,-8.20982783506,1.2065013237,-8.14327438861,-1.78182751349,-10.9668725757 -736,-7.87906463554,1.5188258917,-8.06185450647,-1.75657399227,-10.8879705059 -737,-7.87666466486,0.419814819599,-8.37849950898,-2.10739374139,-10.5729436234 -738,-8.13388013683,-0.316121636573,-8.54804736991,-2.09712190262,-10.8076758633 -739,-8.21410994533,-1.47890094587,-9.01606970706,-2.52170386093,-10.603555136 -740,-8.42632949015,-0.798181541948,-9.54681371394,-2.5773365312,-11.6090378118 -741,-7.49930210003,-0.84917442156,-9.45594393931,-2.73866916404,-11.2306569351 -742,-8.02482733789,-1.94745762639,-9.12481196805,-2.36548222365,-10.6961213331 -743,-8.17658714341,-1.67764505438,-9.72271321311,-2.5264638587,-11.4782089629 -744,-7.52625751472,-0.134031885533,-9.48827442084,-2.88193133131,-11.9667258197 -745,-6.76522036535,-1.2403510848,-9.4082814575,-2.84879748694,-11.0511462345 -746,-6.70032283363,-0.70871581707,-9.10485958634,-2.79375242472,-11.0742431844 -747,-6.39694751946,-2.67577505274,-8.92290004959,-2.4587626515,-10.0951297874 -748,-6.50803611445,-2.9409132331,-9.05649851838,-2.44089453865,-10.3345494614 -749,-6.71843096837,-4.09330699746,-9.13889571697,-2.14359951033,-10.0992804477 -750,-6.19027004409,-3.93193723844,-8.62311199117,-2.05344115029,-9.64988643498 -751,-5.52532828617,-3.25965524913,-8.21724737534,-2.02353190822,-9.35485880265 -752,-5.45782149369,-4.15164648266,-8.0418293307,-1.9214129807,-8.85922683562 -753,-5.43313572956,-5.11948686973,-8.1432013039,-2.00951314781,-8.57352596717 -754,-5.98022831689,-5.76742824891,-8.18231319605,-1.85589805443,-8.2537198511 -755,-7.37186032969,-7.60592929099,-8.13120689363,-1.43792835215,-7.61687051359 -756,-7.07220582922,-8.30202341168,-8.08044789156,-1.47175646366,-7.81358150205 -757,-6.63860809394,-7.12903781228,-9.23866261402,-2.06703243146,-8.96067559424 -758,-7.00161369044,-7.12676923747,-9.02934867312,-1.75960600043,-9.11285889003 -759,-7.16503633732,-7.27530599809,-8.03638123039,-1.08849865462,-8.07625355978 -760,-6.55683779099,-8.47383725447,-9.2929150117,-2.20782556809,-8.95891383008 -761,-5.85343931709,-7.09408525175,-9.17341410885,-2.33159568227,-9.21128318731 -762,-6.14974268131,-7.47507434432,-8.86395361181,-2.14024541583,-8.75968855905 -763,-5.85472174815,-7.41876132672,-8.92606746044,-2.39473914238,-8.86980147794 -764,-5.77682503944,-7.80458488217,-9.16613807779,-2.38145245119,-8.6081870417 -765,-5.52753885566,-8.2640123029,-9.40737630902,-2.29512943068,-9.08677762619 -766,-5.71482111437,-8.61702201056,-10.1491815553,-2.69672975536,-9.605407411 -767,-5.17286601838,-8.20744376087,-9.86396914758,-2.74099555848,-9.46591540033 -768,-6.24066735189,-9.57145025614,-10.4714986062,-2.78574171544,-9.63238368379 -769,-5.83243430426,-8.86447926464,-11.2251700642,-3.22312477123,-10.6399883773 -770,-6.50533240766,-10.0822555194,-10.8842257729,-3.01310870154,-10.1120901828 -771,-7.04331851901,-11.4224349418,-10.2445352237,-2.06945966606,-9.0720350199 -772,-7.20960196966,-11.7135406423,-9.69445401905,-1.48892315697,-8.50014413517 -773,-6.77879904408,-10.9098497644,-9.08724727013,-1.68474476946,-8.06826729631 -774,-6.67370506951,-11.1019363285,-8.8421376116,-1.48202359185,-7.84135210128 -775,-6.28924573554,-12.0271508956,-9.58191702864,-2.19805930201,-8.31285662507 -776,-5.24159091994,-11.9347932332,-9.77023874889,-2.63025686989,-8.14144923574 -777,-5.30571198974,-11.8734921481,-9.2612909609,-2.35795191309,-7.6826631425 -778,-4.82719999144,-9.13275377878,-9.43892590655,-2.74635106794,-8.81147257705 -779,-4.94339599756,-9.70879041703,-9.69485303762,-2.58426051003,-8.89948625305 -780,-4.69911816094,-8.18469113133,-9.85780441535,-2.99039908692,-9.53840201769 -781,-4.16937923924,-7.71569812857,-9.19258733366,-2.67089950399,-9.05103555919 -782,-3.74658846191,-7.80741210756,-8.97408173259,-2.53138216921,-8.60245317629 -783,-4.16636400254,-8.50191051501,-8.83798163077,-2.24051342002,-8.04678801391 -784,-4.22987739708,-8.18896719056,-8.73994530266,-2.20671322325,-8.18398369333 -785,-4.46237499869,-8.60546266644,-8.75484072625,-2.07291813141,-8.31526095617 -786,-4.32313638155,-8.17602589622,-8.57326357401,-2.0485540226,-8.03778459064 -787,-4.35119832634,-7.95918846227,-7.60268998452,-1.29836713334,-7.32421130626 -788,-3.4774808039,-6.88524451673,-8.31442790417,-2.23634977969,-8.2817847793 -789,-3.5367436959,-7.15856998427,-7.66291253354,-1.7689341912,-7.42919997291 -790,-3.73047905477,-7.02108521278,-7.64772686299,-1.78559302188,-7.3322780351 -791,-3.57092483255,-7.53859198786,-7.62325426896,-1.80310702834,-7.2811030197 -792,-4.07004258112,-7.89769765952,-7.93185777821,-2.09070689838,-7.24131372215 -793,-3.90598869552,-6.68611426923,-8.44676683037,-2.45596857582,-8.10824803235 -794,-3.62925950616,-7.05580766647,-8.75520760554,-2.54761341898,-8.66398761353 -795,-3.83539280671,-7.26472621244,-8.3838727897,-2.3083993919,-8.11786978314 -796,-3.39078237227,-6.77779215923,-7.54119347001,-2.05839956015,-7.20000181348 -797,-2.86095921973,-6.6460132865,-7.9846207344,-2.43156390162,-7.833959745 -798,-3.58339317213,-7.40658020612,-8.30747043936,-2.34127833832,-7.99400311332 -799,-4.05735150766,-7.6894007875,-8.0514269085,-1.71383242971,-7.60715122237 -800,-4.11334149223,-7.82041147573,-8.13460602521,-1.91266321281,-7.95127646808 -801,-4.19430614563,-7.15548397079,-8.81920405808,-2.00488109671,-8.85604026612 -802,-3.90560035133,-7.41564863107,-8.86559774402,-2.50477800344,-8.67311797548 -803,-4.04969939337,-5.75768881934,-8.20979519656,-2.20091628217,-8.81047399259 -804,-5.32580790109,-6.57969517922,-8.13859892574,-1.87340025205,-8.30150763772 -805,-5.51547171713,-7.3653493055,-7.94122203565,-1.62971818724,-7.66352114904 -806,-5.89615838955,-6.95628142084,-7.89982941063,-1.59282917714,-7.97140787565 -807,-5.44545940767,-7.41246619793,-8.45591381088,-1.75499785845,-8.32287480625 -808,-5.12696405191,-7.64332394915,-8.46022406057,-1.75660032298,-8.31078001037 -809,-5.10003621285,-7.69223206021,-9.45922248915,-2.65349194012,-9.15881194448 -810,-4.37879437467,-7.39837885934,-9.79537992918,-2.95324865162,-9.8285259825 -811,-5.28507183956,-8.53731789306,-9.6046984541,-2.68456242414,-9.22732569872 -812,-5.45159713421,-8.39419715392,-10.4423257643,-2.89190983186,-10.1082008821 -813,-5.91451554501,-9.10258415112,-10.5772502437,-2.58302722675,-10.3123414871 -814,-6.03426014442,-9.77741088846,-10.3617681731,-2.31586721078,-9.73571315453 -815,-6.9361788229,-10.6410389072,-10.4247564072,-2.1472551503,-9.68193300656 -816,-6.59124282241,-10.6686859648,-10.587027567,-2.09823590688,-9.73197770458 -817,-6.85724569273,-10.5963067931,-10.8504250501,-2.39356301649,-9.96640351421 -818,-6.67601557814,-10.2900261531,-10.5856453886,-2.04655961641,-10.2256689444 -819,-6.36975101338,-9.88548536245,-10.9941452176,-2.50293102934,-10.6884801676 -820,-5.7201019671,-9.62904701482,-11.0757644818,-2.81787443227,-10.811227566 -821,-5.88136828861,-10.0485305235,-11.0368333373,-2.74346407665,-10.4987071322 -822,-5.92244155638,-10.563683653,-11.3891633506,-3.27282179395,-10.5384888 -823,-5.56553525864,-10.1943597155,-11.1550286895,-3.091142523,-10.5200057918 -824,-5.1750861124,-9.58135637232,-10.9544165963,-3.10012581049,-10.4534999255 -825,-4.81068605986,-10.2512835849,-11.2523988611,-3.45149709098,-10.5455545113 -826,-4.77065553871,-9.27599529856,-10.5473517298,-3.5124067208,-10.0936271797 -827,-4.62187265825,-9.58385856423,-10.7835596315,-3.55065421529,-10.1495713732 -828,-4.99394186571,-9.73504199801,-10.4080283661,-3.19850469597,-9.86540060464 -829,-6.04301416832,-9.4905445057,-10.217630934,-2.7993198868,-9.39950645806 -830,-6.22607112393,-8.90197696891,-9.71195117055,-2.55266945055,-9.29596717094 -831,-6.47003509227,-9.84928292417,-9.64940994951,-2.30807130278,-8.86878170644 -832,-6.24414514767,-9.64658044598,-9.62867285951,-2.33264261079,-8.92542297282 -833,-5.70405163059,-9.87088180021,-9.66205052893,-2.48584625192,-8.77636805566 -834,-5.59710911671,-10.8667563765,-9.34316825892,-1.87013065519,-7.89508088269 -835,-5.86841467119,-9.31702725562,-9.22282437969,-2.38195919702,-8.77905185727 -836,-5.75751604756,-7.37561768028,-9.19797178729,-2.06448853087,-9.17194475488 -837,-5.66597217308,-7.702837823,-9.08959006793,-2.19657340287,-8.96951635093 -838,-5.5560004368,-8.15580988401,-9.12023651732,-1.95986394265,-8.95680066104 -839,-5.56789940234,-8.96612365277,-9.88672703435,-2.28632194079,-9.34330293209 -840,-5.90816704445,-8.65039384567,-9.28073482777,-1.80111495621,-9.09961445248 -841,-6.03571587002,-8.59828957237,-9.09556631715,-1.68073087689,-9.0948717191 -842,-6.23674476005,-8.5458085374,-8.99629920596,-1.72275668131,-8.89617133153 -843,-6.40904521546,-7.82437374598,-8.58441964198,-1.27414650706,-8.55570549555 -844,-6.31196036404,-8.04545898534,-8.10542652833,-1.26573880715,-8.02706433545 -845,-7.04057620737,-9.12438651353,-8.21153381541,-1.06291118812,-7.52584269483 -846,-7.43447207972,-9.91236672692,-7.79209857331,-0.665960786301,-7.00978620266 -847,-7.1077277343,-8.85393820368,-7.74386589419,-0.640450442391,-7.13684883516 -848,-6.73246185993,-8.46021072259,-8.63508648274,-1.37411002809,-8.12756512196 -849,-6.54166313699,-8.26414102661,-8.95422879427,-1.73344426103,-8.8797972288 -850,-6.49740218939,-7.8981763155,-9.34082421201,-1.98335386589,-9.46988943454 -851,-6.25836969256,-6.95659322533,-9.92701628524,-2.46135660166,-10.0835354762 -852,-5.63028082237,-5.12362477697,-9.94849221414,-2.72065732112,-10.8253574997 -853,-5.50004076646,-5.20771030873,-10.1054768228,-2.99905086854,-10.8757400348 -854,-5.7397058484,-5.44793639357,-10.8865135242,-3.19044704641,-11.4482661928 -855,-4.83515296569,-5.94793368511,-10.984490205,-3.56258441682,-11.5248475043 -856,-3.49352741632,-4.00243235228,-11.1489893181,-4.09441088131,-11.9491315213 -857,-3.5770969747,-5.06572530654,-11.1315834052,-4.01940012726,-11.7615704496 -858,-3.94842351769,-4.58122100883,-11.0071331814,-3.8025735928,-11.7854938583 -859,-3.4211792454,-4.97503654793,-10.8449773536,-3.93046499843,-11.8412663002 -860,-3.62690298857,-4.28207188284,-11.0050744151,-3.89103158697,-12.1670012829 -861,-3.71868534874,-2.90302446585,-10.8536688411,-4.04261559749,-12.279042204 -862,-3.73117059503,-2.85309613076,-11.5731838798,-4.39469820582,-13.0659652366 -863,-3.60836529353,-1.67020548391,-11.5401913026,-4.60672264662,-13.3690800682 -864,-3.37301631807,-0.873358903384,-11.4115331248,-4.32507657404,-13.5286194771 -865,-3.72844188778,-1.72656423838,-11.763358309,-4.53308391543,-13.6382324948 -866,-3.20795129173,-0.578835060185,-11.791821504,-4.64628962224,-14.0529916139 -867,-3.36840710688,-1.15767544124,-11.7286009163,-4.85919791973,-13.7629274302 -868,-3.99696038152,-0.844739867102,-11.4956065626,-4.59579700043,-13.7139545177 -869,-4.21651630045,-1.20666757418,-11.4409365306,-4.40324050056,-13.3790084561 -870,-4.30023850626,-0.105574700022,-11.0958188265,-4.24749649785,-13.8174479173 -871,-4.11564108077,1.58973067252,-11.1369488411,-4.13703426385,-14.1831915489 -872,-4.56786374208,1.36695328606,-11.3904547477,-4.28431290639,-14.4269468632 -873,-4.50056636709,2.25128049162,-10.8564830063,-4.06104934048,-14.2166133475 -874,-4.94600010137,2.32635390812,-10.1386450721,-3.42878455743,-13.3495463995 -875,-4.41764311496,3.13302363303,-10.9020615228,-3.93809970023,-14.5958420203 -876,-4.61049405127,2.36550122929,-10.2714575893,-3.44884237467,-13.7738492901 -877,-4.86266655361,2.90211961629,-10.2016289116,-3.32349381891,-13.8765687916 -878,-4.10224512836,4.28900758429,-10.3582535797,-3.66123383351,-14.2537660588 -879,-3.99572428667,4.92312524363,-9.92240178393,-3.15192212119,-14.0912440489 -880,-4.17482690469,2.82141903136,-10.3817372341,-3.38879952845,-14.0848619935 -881,-4.15388559346,2.83519451643,-10.2283454015,-3.11153755345,-13.7824999673 -882,-4.66708473879,2.27055533414,-10.068101875,-2.73121983697,-13.6321369875 -883,-4.72185523255,4.32901439084,-9.89926131515,-2.73739550829,-14.3959783089 -884,-4.62333271414,4.49791615508,-10.192144157,-2.7682136781,-14.765040152 -885,-4.42065022717,5.04638223181,-10.11667254,-2.76876495112,-14.8306344489 -886,-4.25287698714,4.46321940764,-10.6987002939,-3.08173196085,-15.1398990315 -887,-4.44527678316,4.80432837449,-10.9031982909,-3.03978127773,-15.5532613382 -888,-4.4642282456,4.63540402499,-10.0626863105,-2.70703732986,-14.7688870887 -889,-4.98732399964,4.87396862775,-10.0933839368,-2.3725316622,-14.925094003 -890,-5.50258285626,4.70974864013,-10.2136983639,-2.33421912597,-14.7739657493 -891,-6.0556292827,4.08759536053,-10.1966612527,-2.13033581665,-14.7915496701 -892,-6.20357855741,4.41719312508,-9.8421416554,-2.07446968619,-14.1713058575 -893,-6.84406671834,5.86978276688,-9.33795319327,-1.73984810104,-14.3196899621 -894,-6.80649620344,5.94516877123,-9.08785071528,-1.43538867005,-14.2552864661 -895,-6.7485085783,4.11530120378,-8.82862693553,-1.04215688796,-13.0594162959 -896,-6.51110684629,3.59145076114,-8.77190944029,-1.0336741571,-13.1947952633 -897,-6.68974487202,2.17969311498,-9.22802137982,-1.37409264821,-13.4683842394 -898,-6.55440942186,3.06243533357,-9.1698473504,-1.22021527104,-13.3344614816 -899,-6.63447988618,3.55724318482,-9.44164231674,-1.46933867665,-13.5789206573 -900,-6.10621651832,3.86726041575,-8.77940026076,-1.27191360892,-12.969591172 -901,-6.37728622241,4.69536630774,-8.83064573683,-1.48510895647,-13.2622278618 -902,-6.73732689789,4.59670094866,-9.30892122513,-1.85270026166,-13.7780337461 -903,-6.71798901303,4.45723210353,-9.1424686697,-1.52731047172,-13.6308255327 -904,-6.56073004994,4.50198204099,-9.24472131532,-2.11736724125,-13.732689125 -905,-6.76157327,3.93798583921,-9.29070539054,-1.7374687717,-13.6927290301 -906,-7.43721721675,3.5707456852,-9.26665110758,-1.51542861756,-13.7058162401 -907,-6.85342227753,3.82989187772,-8.98816923297,-1.33835204828,-13.1117000732 -908,-7.0993953667,3.17751065309,-9.06413387904,-1.27911918294,-13.3040051871 -909,-7.08694160784,3.03820583095,-9.10819228366,-1.36034995,-13.1626547474 -910,-8.18691005387,3.50249637009,-8.86726834627,-0.714119788469,-12.9565949834 -911,-9.07831367579,2.11456462943,-8.61472892476,-0.128253572059,-12.3898258754 -912,-8.3820305592,2.22008103714,-8.51299251791,-0.0986489157316,-12.6233941336 -913,-8.72000218177,3.08589880464,-8.66351295867,-0.411793736793,-12.7851524847 -914,-8.74651500043,2.85876888988,-8.4120536043,-0.420198109207,-12.5291559218 -915,-8.9968175645,2.9685564391,-8.02363684798,-0.0764392246616,-12.0284422995 -916,-8.74260823661,2.92566915507,-7.45471871959,0.257374447637,-11.518919906 -917,-8.93260604901,1.65576683465,-7.35321770881,0.393954098311,-11.0521551646 -918,-8.96243902825,2.45208538958,-7.69838778123,0.254203381415,-11.1617644952 -919,-9.17403416046,2.07207623048,-7.48620011154,0.278258872769,-11.0194718332 -920,-8.91691652509,2.7063931525,-7.33596000124,0.298617715114,-11.2794005695 -921,-8.3185258608,4.04538421473,-7.50710825059,0.187238944453,-11.4962398955 -922,-8.0880214816,4.86962512793,-6.79900357691,0.113003235773,-11.4543285503 -923,-7.7808560494,5.36315159783,-7.2641077495,-0.390436374957,-11.9166186106 -924,-8.30164135841,5.67266091001,-6.90854040016,0.091019078768,-11.6092027642 -925,-8.18716428083,5.90817103838,-6.53794712547,0.350449931756,-11.1997110809 -926,-7.97590318546,6.18876860301,-7.32446190805,-0.364186767392,-12.1064087473 -927,-7.36813973229,6.30080622833,-8.56444673032,-1.06607789153,-13.5327707721 -928,-7.47438341107,6.54564731749,-7.77778828611,-0.599102284886,-12.8316239236 -929,-8.43358868231,6.62729734146,-7.39925811474,0.0345529056643,-12.4093872413 -930,-8.52094198827,7.36948836644,-7.70961662968,-0.134881418035,-13.1902579702 -931,-8.60348660531,6.62975932886,-7.2778856103,0.0086373289791,-12.3167137895 -932,-7.75539739982,6.03802159915,-6.73220924154,0.0964482432497,-11.7150796308 -933,-8.07017826456,6.59761960451,-6.49024505985,0.570725237191,-11.4462016059 -934,-8.17032530401,7.15432012027,-6.13655820403,0.684695465667,-11.3025357581 -935,-8.82708352261,6.44731630625,-5.84459822382,1.14847078031,-10.4702312417 -936,-7.79227668835,6.85551389158,-5.64796318875,0.874638310579,-10.4993529134 -937,-8.42438939918,7.3398323113,-6.22577551108,0.648630454082,-11.173472379 -938,-7.79328671347,7.87646751961,-6.00995450033,0.65769025014,-11.0606936451 -939,-7.66422652233,8.42776912822,-5.04040669214,0.777026160172,-10.398857728 -940,-8.22094066572,8.88718803888,-4.54407938894,1.34296022466,-10.058501687 -941,-8.02010947554,8.58461439056,-4.81739676524,1.35158321272,-10.1755989433 -942,-8.29753691759,8.19549965909,-4.70111602242,1.6313734469,-9.91662167777 -943,-8.15323197518,7.33486798493,-4.77948295011,1.66233857286,-9.62911605955 -944,-7.34749352711,7.08587215854,-4.61738615971,1.40339212113,-9.73673844855 -945,-7.47898261999,6.14325567551,-5.18201817358,1.13230907003,-9.98463073065 -946,-8.13116166773,5.98978856631,-4.75412438167,1.7102099887,-9.35795393448 -947,-7.34994930371,6.19940931393,-5.73197537437,1.20326930426,-10.5180109564 -948,-7.17074949635,6.65816101079,-5.23026868798,1.28982433737,-10.0824753725 -949,-7.76597962386,6.95928381995,-5.02161019202,1.12113521372,-10.2769781877 -950,-7.63573676719,7.72866085033,-5.32728577955,1.04328250806,-10.8923822067 -951,-7.47901625134,7.41536748099,-5.59494941785,0.879389596137,-10.7638746251 -952,-7.42321529206,6.12958993416,-5.52542223264,0.891617493788,-10.5311623823 -953,-6.99206951689,7.91537991306,-5.55681707193,0.915533809983,-10.9806672293 -954,-6.37124585044,8.11731460297,-5.4829566968,1.06052146019,-10.7126439958 -955,-6.72935163183,9.0958054212,-5.13030598258,0.860570008679,-11.0866052948 -956,-6.65236463089,9.01413915524,-5.2841246082,0.811132797352,-10.8690269559 -957,-6.93006576998,8.22454116575,-4.68365081154,1.21933228307,-9.91900365174 -958,-6.67184011636,9.32827916986,-4.71485735297,1.01509710922,-10.1931641062 -959,-6.65769653531,9.28227772929,-5.02472824035,0.385566058008,-10.7554568399 -960,-7.30423354111,9.4812387472,-4.84852369843,0.526143113405,-10.5471712667 -961,-7.28934962714,9.86148338916,-5.13492549346,0.353130809676,-10.8341949572 -962,-7.00841309294,10.2045356708,-4.80972618532,0.699135917281,-10.7131394662 -963,-6.90972946885,10.4143403503,-4.7422674606,0.602850182543,-10.5041859753 -964,-7.65023596676,10.9693239141,-4.50852173423,0.971221611192,-10.3388038592 -965,-7.50126627756,11.4661764956,-3.236734438,1.79203786252,-9.31863397597 -966,-8.02045465062,11.8762395074,-2.77545963217,2.35293005294,-8.75664770575 -967,-8.43432012911,10.6208368746,-2.28478217692,2.74765803446,-7.8923953312 -968,-7.88589333975,12.5033881964,-2.53503668254,2.47466210079,-8.71515334392 -969,-7.53673170714,12.2116718174,-2.72984035849,2.16282647918,-8.93573710518 -970,-7.69872073681,11.4857215525,-2.39346209479,2.40525790079,-8.45390383993 -971,-7.00033300249,11.3998425081,-2.81561362755,1.81237426604,-8.63262360538 -972,-7.85319668953,10.7518573099,-1.78529137855,2.77836489073,-7.4217912996 -973,-7.99711165522,11.151184441,-2.08049690055,2.64514021668,-7.84814819036 -974,-7.29048555635,10.8819906589,-2.56843658326,2.17496851804,-8.2953431053 -975,-6.91382111079,10.8374233194,-2.72790803968,1.90892201825,-8.45117729851 -976,-7.25741583678,10.2184680232,-2.66528269525,2.16944805502,-8.11548455923 -977,-6.98910109847,10.2176840386,-2.8262837783,2.05835661081,-8.36467932703 -978,-6.3499524905,10.7241406883,-3.08748484407,1.67002593428,-8.75145354429 -979,-5.91619792118,9.95064553608,-3.78041771895,1.44452296998,-9.59888803693 -980,-6.13826994863,11.9834095094,-4.17279129411,1.10104563929,-10.3327667198 -981,-6.29500398514,12.3622287724,-4.76534180424,0.609719559297,-10.9902322729 -982,-6.85195847662,13.1848787975,-4.13520529452,0.885753334465,-10.7496293669 -983,-6.97057922544,12.8829187447,-4.04460919449,1.21189441308,-10.7778325643 -984,-7.06691359257,11.9004026628,-3.91885184965,1.58245193255,-10.5746611573 -985,-7.39212000577,11.9770072448,-3.85627942514,1.62518404806,-10.3266702518 -986,-6.52124402017,12.8881192594,-4.39810745782,0.889702194369,-11.2263151345 -987,-7.04968730844,13.3120566941,-4.32321839293,1.12292879779,-10.9405588007 -988,-6.56894496787,12.9905890346,-4.26867533523,1.05943977388,-11.0803884375 -989,-6.932860947,11.8288652796,-3.86875096472,1.37451711082,-10.2782928433 -990,-7.33048161823,10.4399185384,-3.31214022362,2.02390868227,-9.20001462799 -991,-6.78949234367,9.64456970707,-3.1401505871,2.24106970739,-8.53386008818 -992,-6.41293915988,10.6781564573,-2.93266980603,1.9617334364,-9.05850240844 -993,-6.50288168327,10.6140520766,-2.81098857002,1.8630736286,-8.95897104858 -994,-5.51236345328,9.61060953025,-3.12491960177,1.86000813216,-8.39402979191 -995,-6.30712424741,9.24251826438,-2.32572392628,2.2191243537,-7.82591351761 -996,-6.8317122362,9.27584638454,-2.01212302451,2.68442390382,-7.24995816723 -997,-7.01590099344,8.53905454516,-1.80943487815,2.88510983538,-6.82307711465 -998,-7.40812496934,7.88216643988,-1.37516747832,3.24706573563,-5.77880298028 -999,-7.02386098432,8.59456088374,-1.59027141406,2.79354013606,-6.32114172303 diff --git a/tensorflow/contrib/timeseries/examples/data/multivariate_periods.csv b/tensorflow/contrib/timeseries/examples/data/multivariate_periods.csv deleted file mode 100644 index 9b15b4f0b26..00000000000 --- a/tensorflow/contrib/timeseries/examples/data/multivariate_periods.csv +++ /dev/null @@ -1,100 +0,0 @@ -0,0.926906299771,1.99107237682,2.56546245685,3.07914768197,4.04839057867,1.,0.,strkeya -1,0.108010001864,1.41645361423,2.1686839775,2.94963962176,4.1263503303,1.,0.,strkeyb -2,-0.800567600028,1.0172132907,1.96434754116,2.99885333086,4.04300485864,1.,0.,strkey -3,0.0607042871898,0.719540073421,1.9765012584,2.89265588817,4.0951014426,1.,0.,strkey -4,0.933712200629,0.28052120776,1.41018552514,2.69232603996,4.06481164223,1.,0.,strkey -5,-0.171730652974,0.260054421028,1.48770816369,2.62199129293,4.44572807842,1.,0.,strkey -6,-1.00180162933,0.333045158863,1.50006392277,2.88888309683,4.24755865606,1.,0.,strkey -7,0.0580061875336,0.688929398826,1.56543458772,2.99840358953,4.52726873347,1.,0.,strkey -8,0.764139447412,1.24704875327,1.77649279698,3.13578593851,4.63238922951,1.,0.,strkey -9,-0.230331874785,1.47903998963,2.03547545751,3.20624030377,4.77980005228,1.,0.,strkey -10,-1.03846045211,2.01133000781,2.31977503972,3.67951536251,5.09716775897,1.,0.,strkeyc -11,0.188643592253,2.23285349038,2.68338482249,3.49817168611,5.24928239634,1.,0.,strkey -12,0.91207302309,2.24244446841,2.71362604985,3.96332587625,5.37802271594,1.,0.,strkey -13,-0.296588665881,2.02594634141,3.07733910479,3.99698324956,5.56365901394,1.,0.,strkey -14,-0.959961476551,1.45078629833,3.18996420137,4.3763059609,5.65356015609,1.,0.,strkey -15,0.46313530679,1.01141441548,3.4980215948,4.20224896882,5.88842247449,1.,0.,strkey -16,0.929354125798,0.626635305936,3.70508262244,4.51791573544,5.73945973251,1.,0.,strkey -17,-0.519110731957,0.269249223148,3.39866823332,4.46802003061,5.82768174382,1.,0.,strkey -18,-0.924330981367,0.349602834684,3.21762413294,4.72803587499,5.94918925767,1.,0.,strkey -19,0.253239387885,0.345158023497,3.11071425333,4.79311566935,5.9489259713,1.,0.,strkey -20,0.637408390225,0.698996675371,3.25232492145,4.73814732384,5.9612010251,1.,0.,strkey -21,-0.407396859412,1.17456342803,2.49526823723,4.59323415742,5.82501686811,1.,0.,strkey -22,-0.967485452118,1.66655933642,2.47284606244,4.58316034754,5.88721406681,1.,0.,strkey -23,0.474480867904,1.95018556323,2.0228950072,4.48651142819,5.8255943735,1.,0.,strkey -24,1.04309652155,2.23519892356,1.91924131572,4.19094661783,5.87457348436,1.,0.,strkey -25,-0.517861513772,2.12501967336,1.70266619979,4.05280882887,5.72160912899,1.,0.,strkey -26,-0.945301585146,1.65464653549,1.81567174251,3.92309850635,5.58270493814,1.,0.,strkey -27,0.501153868974,1.40600764889,1.53991387719,3.72853247942,5.60169001727,1.,0.,strkey -28,0.972859524418,1.00344321868,1.5175642828,3.64092376655,5.10567722582,1.,0.,strkey -29,-0.70553406135,0.465306263885,1.7038540803,3.33236870312,5.09182481555,1.,0.,strkey -30,-0.946093634916,0.294539309453,1.88052827037,2.93011492669,4.97354922696,1.,0.,strkey -31,0.47922123231,0.308465865031,2.03445883031,2.90772899045,4.86241793548,1.,0.,strkey -32,0.754030014252,0.549752241167,2.46115815089,2.95063349534,4.71834614627,1.,0.,strkey -33,-0.64875949826,0.894615488148,2.5922463381,2.81269864022,4.43480095104,1.,0.,strkey -34,-0.757829951086,1.39123914261,2.69258079904,2.61834837315,4.36580046156,1.,0.,strkey -35,0.565653301088,1.72360022693,2.97794913834,2.80403840334,4.27327248459,1.,0.,strkey -36,0.867440092372,2.21100730052,3.38648090792,2.84057515729,4.12210169576,1.,0.,strkey -37,-0.894567758095,2.17549105818,3.45532493329,2.90446025717,4.00251740584,1.,0.,strkeyd -38,-0.715442356893,2.15105389965,3.52041791902,3.03650393392,4.12809249577,1.,0.,strkey -39,0.80671703672,1.81504564517,3.60463324866,3.00747789871,3.98440762467,1.,0.,strkey -40,0.527014790142,1.31803513865,3.43842186337,3.3332594663,4.03232406566,1.,0.,strkey -41,-0.795936862129,0.847809114454,3.09875133548,3.52863155938,3.94883924909,1.,0.,strkey -42,-0.610245806946,0.425530441018,2.92581949152,3.77238736123,4.27287245021,1.,0.,strkey -43,0.611662279431,0.178432049837,2.48128214822,3.73212087883,4.17319013831,1.,0.,strkey -44,0.650866553108,0.220341648392,2.41694642022,4.2609098519,4.27271645905,1.,0.,strkey -45,-0.774156982023,0.632667602331,2.05474356052,4.32889204886,4.18029723271,1.,0.,strkey -46,-0.714058448409,0.924562377599,1.75706135146,4.52492718422,4.3972678094,1.,0.,strkey -47,0.889627293379,1.46207968841,1.78299357672,4.64466731095,4.56317887554,1.,0.,strkey -48,0.520140662861,1.8996333843,1.41377633823,4.48899091177,4.78805049769,1.,0.,strkey -49,-1.03816935616,2.08997002059,1.51218375351,4.84167764204,4.93026048606,1.,0.,strkey -50,-0.40772951362,2.30878972136,1.44144415128,4.76854460997,5.01538444629,1.,0.,strkey -51,0.792730684781,1.91367048509,1.58887384677,4.71739397335,5.25690012199,1.,0.,strkey -52,0.371311881576,1.67565079528,1.81688563053,4.60353107555,5.44265822961,1.,0.,strkey -53,-0.814398070371,1.13374634126,1.80328814859,4.72264252878,5.52674761122,1.,0.,strkey -54,-0.469017949323,0.601244136627,2.29690896736,4.49859178859,5.54126153454,1.,0.,strkey -55,0.871044371426,0.407597593794,2.7499112487,4.19060637761,5.57693767301,1.,0.,strkey -56,0.523764933017,0.247705192709,3.09002071379,4.02095509006,5.80510362182,1.,0.,strkey -57,-0.881326403531,0.31513103164,3.11358205718,3.96079100808,5.81000652365,1.,0.,strkey -58,-0.357928025339,0.486163915865,3.17884556771,3.72634990659,5.85693642011,1.,0.,strkey -59,0.853038779822,1.04218094475,3.45835384454,3.36703969978,5.9585988449,1.,0.,strkey -60,0.435311516013,1.59715085283,3.63313338588,3.11276729421,5.93643818229,1.,0.,strkey -61,-1.02703719138,1.92205832542,3.47606111735,3.06247155999,6.02106646259,1.,0.,strkey -62,-0.246661325557,2.14653802542,3.29446326567,2.89936259181,5.67531541272,1.,0.,strkey -63,1.02554736569,2.25943737733,3.07031591528,2.78176218013,5.78206328989,1.,0.,strkey -64,0.337814475969,2.07589147224,2.80356226089,2.55888206331,5.7094075496,1.,0.,strkey -65,-1.12023369929,1.25333011618,2.56497288445,2.77361359194,5.50799418376,1.,0.,strkey -66,-0.178980246554,1.11937139901,2.51598681313,2.91438309151,5.47469577206,1.,0.,strkey -67,0.97550951531,0.60553823137,2.11657741073,2.88081098981,5.37034999502,1.,0.,strkey -68,0.136653357206,0.365828836075,1.97386033165,3.13217903204,5.07254490219,1.,0.,strkey -69,-1.05607596951,0.153152115069,1.52110743825,3.01308794192,5.08902539125,1.,0.,strkey -70,-0.13095280331,0.337113974483,1.52703079853,3.16687131599,4.86649398514,1.,0.,strkey -71,1.07081057754,0.714247566736,1.53761382634,3.45151989484,4.75892309166,1.,0.,strkey -72,0.0153410376082,1.24631231847,1.61690939161,3.85481994498,4.35683752832,1.,0.,strkey -73,-0.912801257303,1.60791309476,1.8729264524,4.03037260012,4.36072588913,1.,0.,strkey -74,-0.0894895640338,2.02535207407,1.93484909619,4.09557485132,4.35327025188,1.,0.,strkey -75,0.978646999652,2.20085086625,2.09003440427,4.27542353033,4.1805058388,1.,0.,strkey -76,-0.113312642876,2.2444100761,2.50789248839,4.4151861502,4.03267168136,1.,0.,strkey -77,-1.00215099149,1.84305628445,2.61691237246,4.45425147595,3.81203553766,1.,0.,strkey -78,-0.0183234614205,1.49573923116,2.99308471214,4.71134960112,4.0273804959,1.,0.,strkey -79,1.0823738177,1.12211589848,3.27079386925,4.94288270502,4.01851068083,1.,0.,strkey -80,0.124370187893,0.616474412808,3.4284236674,4.76942168327,3.9749536483,1.,0.,strkey -81,-0.929423379352,0.290977090976,3.34131726136,4.78590392707,4.10190661656,1.,0.,strkey -82,0.23766302648,0.155302052254,3.49779513794,4.64605656795,4.15571321107,1.,0.,strkey -83,1.03531486192,0.359702776204,3.4880725919,4.48167586667,4.21134561991,1.,0.,strkey -84,-0.261234571382,0.713877760378,3.42756426614,4.426443869,4.25208300527,1.,0.,strkey -85,-1.03572442277,1.25001113691,2.96908341113,4.25500915322,4.25723010649,1.,0.,strkey -86,0.380034261243,1.70543355622,2.73605932518,4.16703432307,4.63700400788,1.,0.,strkey -87,1.03734873488,1.97544410562,2.55586572141,3.84976673263,4.55282864289,1.,0.,strkey -88,-0.177344253372,2.22614526325,2.09565864891,3.77378097953,4.82577400298,1.,0.,strkey -89,-0.976821526892,2.18385079177,1.78522284118,3.67768223554,5.06302440873,1.,0.,strkey -90,0.264820472091,1.86981946157,1.50048403865,3.43619796921,5.05651761669,1.,0.,strkey -91,1.05642344868,1.47568646076,1.51347671977,3.20898518885,5.50149047462,1.,0.,strkey -92,-0.311607433358,1.04226467636,1.52089650905,3.02291865417,5.4889046232,1.,0.,strkey -93,-0.724285777937,0.553052311957,1.48573560173,2.7365973598,5.72549174225,1.,0.,strkey -94,0.519859192905,0.226520626591,1.61543723167,2.84102086852,5.69330622288,1.,0.,strkey -95,1.0323195039,0.260873217055,1.81913034804,2.83951143848,5.90325028086,1.,0.,strkey -96,-0.53285682538,0.387695521405,1.70935609313,2.57977050631,5.79579213161,1.,0.,strkey -97,-0.975127997215,0.920948771589,2.51292643636,2.71004616612,5.87016469227,1.,0.,strkey -98,0.540246804099,1.36445470181,2.61949412896,2.98482553485,6.02447664937,1.,0.,strkey -99,0.987764008058,1.85581989607,2.84685706149,2.94760204892,6.0212151724,1.,0.,strkey diff --git a/tensorflow/contrib/timeseries/examples/data/period_trend.csv b/tensorflow/contrib/timeseries/examples/data/period_trend.csv deleted file mode 100644 index 39c1952339a..00000000000 --- a/tensorflow/contrib/timeseries/examples/data/period_trend.csv +++ /dev/null @@ -1,500 +0,0 @@ -1,-0.6656603714 -2,-0.1164380359 -3,0.7398626488 -4,0.7368633029 -5,0.2289480898 -6,2.257073255 -7,3.023457405 -8,2.481161007 -9,3.773638612 -10,5.059257738 -11,3.553186083 -12,4.554486452 -13,3.655475698 -14,3.419647598 -15,4.303376245 -16,4.830153934 -17,7.253057441 -18,5.064802335 -19,5.448082106 -20,6.251301517 -21,6.214335675 -22,3.07021164 -23,6.995487627 -24,7.180942656 -25,6.084876071 -26,6.95580607 -27,6.692312738 -28,6.339959049 -29,7.659013269 -30,6.157071564 -31,4.023661782 -32,7.380555018 -33,6.972155839 -34,6.655956847 -35,6.532594924 -36,6.780524726 -37,6.723407547 -38,7.616777776 -39,6.394157367 -40,5.046574011 -41,5.715326568 -42,6.536737479 -43,6.527307846 -44,5.671954159 -45,6.508512087 -46,4.740656344 -47,5.449062618 -48,5.796110609 -49,4.802213058 -50,4.627081034 -51,5.748934924 -52,4.05776044 -53,2.743057715 -54,3.590052501 -55,2.937786376 -56,5.333221794 -57,5.102383904 -58,5.097946146 -59,2.771776766 -60,3.75493571 -61,3.268329562 -62,3.127887555 -63,5.723894838 -64,2.365351066 -65,2.030890988 -66,5.74385257 -67,2.637874242 -68,2.851492945 -69,1.907194917 -70,2.568816256 -71,3.869259698 -72,3.989917724 -73,3.641515351 -74,2.812911768 -75,4.964828171 -76,3.050937945 -77,4.203046785 -78,4.269162745 -79,2.818643243 -80,3.334928424 -81,5.239741508 -82,4.972880771 -83,5.212782208 -84,6.056729012 -85,5.404247421 -86,4.733521027 -87,5.241044888 -88,6.844720502 -89,8.242617764 -90,6.686818708 -91,6.429035591 -92,7.45926043 -93,8.225717423 -94,7.661722793 -95,8.348721917 -96,8.029228135 -97,9.780942864 -98,9.755623978 -99,9.149489124 -100,8.947965351 -101,9.176768019 -102,8.768408716 -103,10.39624874 -104,10.39477408 -105,11.63126076 -106,11.8222078 -107,13.60107691 -108,14.54919169 -109,12.63475358 -110,13.77411599 -111,14.45808191 -112,13.27674112 -113,16.00004992 -114,13.04977221 -115,14.65730048 -116,14.76178039 -117,14.62716229 -118,16.20697047 -119,14.79470608 -120,16.70541749 -121,15.8638474 -122,15.63192699 -123,17.20433954 -124,16.29180965 -125,16.93688521 -126,16.07521662 -127,18.33942893 -128,15.62502668 -129,16.81519558 -130,16.86177911 -131,19.18323671 -132,16.68993279 -133,16.52735528 -134,15.22702085 -135,16.13574242 -136,16.08079964 -137,17.16828833 -138,16.09004409 -139,16.92712829 -140,15.54298161 -141,16.03893798 -142,15.38310389 -143,16.18064645 -144,16.22326501 -145,17.1657127 -146,14.87850136 -147,12.80968507 -148,16.25354113 -149,15.14082073 -150,15.79111348 -151,14.02005588 -152,14.32583767 -153,13.87437546 -154,14.47127314 -155,14.29661188 -156,14.68406313 -157,15.84514503 -158,13.89667867 -159,13.58135083 -160,14.26005818 -161,13.3826131 -162,12.85293827 -163,11.06745237 -164,14.08812275 -165,13.05949205 -166,12.18454971 -167,13.01005879 -168,12.45032762 -169,12.20445297 -170,14.39420173 -171,13.49261191 -172,14.91460871 -173,15.97672915 -174,13.96235436 -175,13.77840615 -176,14.39425289 -177,14.31499272 -178,14.37080989 -179,15.34130707 -180,13.42441434 -181,14.54726137 -182,12.51644144 -183,15.36040785 -184,14.52577002 -185,15.90562887 -186,15.12482026 -187,15.55534424 -188,12.22427756 -189,15.11554898 -190,14.23464612 -191,16.52156964 -192,18.14558077 -193,16.51932129 -194,16.88159194 -195,18.08337828 -196,18.70889734 -197,20.97040748 -198,18.98358689 -199,20.76308391 -200,19.81117586 -201,20.24139919 -202,20.78884634 -203,19.92458806 -204,21.60401889 -205,23.30040897 -206,22.2621713 -207,21.24305034 -208,22.07690632 -209,21.78022193 -210,22.94853418 -211,23.72076264 -212,24.12217213 -213,23.04498673 -214,23.8767225 -215,26.52157498 -216,26.24329682 -217,24.83932457 -218,25.66570111 -219,25.61834475 -220,24.41079934 -221,25.31871793 -222,26.7612452 -223,27.00663389 -224,27.86719501 -225,24.87319457 -226,27.85768696 -227,25.70405436 -228,26.11077958 -229,28.11250875 -230,27.6743468 -231,27.19705336 -232,28.08086799 -233,26.19946123 -234,27.32830376 -235,25.98334256 -236,26.71791978 -237,26.67921906 -238,26.25811051 -239,26.64228363 -240,26.20667398 -241,26.39816025 -242,24.83672957 -243,24.27745854 -244,26.10007483 -245,25.67761738 -246,25.91667268 -247,27.57057095 -248,25.68913621 -249,24.92375989 -250,25.5593706 -251,25.14638402 -252,26.46738639 -253,24.55740644 -254,23.5691458 -255,24.07138538 -256,24.94177528 -257,22.33546227 -258,22.32323763 -259,24.38075647 -260,22.40754744 -261,22.61183469 -262,23.28658677 -263,22.98637689 -264,25.46468191 -265,24.14497597 -266,22.97023633 -267,24.37831161 -268,24.86418705 -269,22.61185053 -270,21.70979546 -271,22.09389192 -272,23.25882086 -273,23.56494308 -274,24.13181731 -275,24.28160263 -276,24.43623736 -277,23.24956419 -278,21.76696726 -279,25.14997786 -280,24.67520728 -281,23.40400797 -282,26.24489282 -283,25.05952039 -284,24.53922399 -285,24.89917455 -286,25.13438134 -287,26.05220822 -288,26.94133112 -289,26.02788294 -290,26.65909349 -291,26.0832158 -292,27.39946496 -293,26.57973099 -294,27.49867838 -295,29.89834253 -296,27.78403709 -297,28.92405258 -298,26.58518509 -299,30.91291741 -300,31.73949474 -301,29.25173685 -302,30.3747463 -303,30.59695095 -304,31.50757627 -305,30.97036633 -306,31.27177079 -307,33.43369051 -308,33.9848363 -309,33.31775176 -310,31.69164009 -311,33.07897081 -312,33.10849644 -313,33.29428375 -314,35.60397723 -315,35.33614012 -316,33.95701506 -317,35.16914759 -318,35.92430987 -319,35.81820171 -320,37.36378976 -321,36.74459793 -322,35.27569759 -323,35.9767425 -324,36.17811539 -325,35.68567729 -326,35.54212562 -327,38.78114238 -328,36.46819618 -329,38.07352601 -330,36.56662256 -331,38.1938068 -332,37.42919226 -333,37.44666875 -334,37.16795054 -335,34.97440399 -336,35.6174255 -337,37.37634133 -338,37.26137677 -339,38.09726659 -340,36.04071363 -341,37.07494746 -342,34.4281316 -343,35.1959716 -344,35.26041345 -345,36.9398346 -346,33.58933988 -347,35.00075536 -348,35.97807689 -349,35.66631707 -350,35.44925794 -351,33.69565848 -352,35.38969147 -353,35.96432261 -354,33.6956667 -355,34.05230212 -356,32.70536873 -357,33.91009672 -358,34.45606416 -359,34.97972516 -360,32.36260234 -361,31.69621537 -362,33.02307596 -363,33.94445036 -364,32.2763097 -365,32.06228645 -366,34.25956906 -367,33.61620818 -368,35.00141908 -369,34.47493965 -370,34.31576327 -371,33.24772844 -372,32.95185358 -373,32.55224164 -374,33.06560689 -375,35.2082848 -376,34.50372086 -377,33.54922461 -378,35.46287805 -379,34.68829823 -380,35.04640557 -381,33.48711975 -382,34.03264662 -383,34.43296169 -384,35.7571391 -385,32.58466542 -386,34.44295272 -387,35.43369124 -388,37.7196386 -389,37.55863215 -390,35.11245844 -391,37.36667774 -392,36.41904568 -393,38.11951592 -394,39.351325 -395,38.87795167 -396,38.8144378 -397,38.96059714 -398,39.95536453 -399,39.78580611 -400,40.70319964 -401,41.32804151 -402,42.79937243 -403,38.43432481 -404,42.12051726 -405,42.50068551 -406,43.89812523 -407,42.18632495 -408,43.99716859 -409,43.67726129 -410,42.98072384 -411,43.59181621 -412,44.98283057 -413,42.17674627 -414,46.49541908 -415,45.58212027 -416,42.7202171 -417,45.66108535 -418,45.03844556 -419,44.96618253 -420,45.0371585 -421,46.12237848 -422,46.18891162 -423,46.82075672 -424,47.25058257 -425,45.91853936 -426,46.83241571 -427,47.77383153 -428,48.12984438 -429,46.74042025 -430,46.66834779 -431,47.41473153 -432,46.93101415 -433,48.24438209 -434,47.41007874 -435,46.92607209 -436,46.77346554 -437,47.80447575 -438,45.7000972 -439,46.60252512 -440,45.59290618 -441,47.37025588 -442,46.46333171 -443,46.19762396 -444,47.57763766 -445,46.92624737 -446,46.1536802 -447,45.94947611 -448,46.37457004 -449,44.22344538 -450,43.18937717 -451,44.3387774 -452,45.63204816 -453,43.87816917 -454,43.67301546 -455,42.11959709 -456,43.89387883 -457,44.40734798 -458,42.67367897 -459,43.76501429 -460,44.74698445 -461,43.14500236 -462,42.41214263 -463,44.1631715 -464,41.81378406 -465,43.00929934 -466,42.80360515 -467,44.30252713 -468,42.88123048 -469,43.47049118 -470,44.42168141 -471,42.43276664 -472,44.57582419 -473,43.56138481 -474,43.4549005 -475,43.06396235 -476,43.8737132 -477,42.1428636 -478,43.60856585 -479,44.16778079 -480,42.90474298 -481,44.99882414 -482,43.304605 -483,44.4468626 -484,45.49241923 -485,44.46713555 -486,46.27348465 -487,45.76034556 -488,45.37440079 -489,46.19246701 -490,48.28190231 -491,47.81719203 -492,47.23213374 -493,48.03313818 -494,46.73599653 -495,47.12327054 -496,48.58597108 -497,48.6738899 -498,48.52018743 -499,48.50385022 -500,50.17026668 diff --git a/tensorflow/contrib/timeseries/examples/known_anomaly.py b/tensorflow/contrib/timeseries/examples/known_anomaly.py deleted file mode 100644 index d575a2f1ba9..00000000000 --- a/tensorflow/contrib/timeseries/examples/known_anomaly.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Example of using an exogenous feature to ignore a known anomaly.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import csv -from os import path - -import numpy as np -import tensorflow as tf - - -try: - import matplotlib # pylint: disable=g-import-not-at-top - matplotlib.use("TkAgg") # Need Tk for interactive plots. - from matplotlib import pyplot # pylint: disable=g-import-not-at-top - HAS_MATPLOTLIB = True -except ImportError: - # Plotting requires matplotlib, but the unit test running this code may - # execute in an environment without it (i.e. matplotlib is not a build - # dependency). We'd still like to test the TensorFlow-dependent parts of this - # example, namely train_and_predict. - HAS_MATPLOTLIB = False - -_MODULE_PATH = path.dirname(__file__) -_DATA_FILE = path.join(_MODULE_PATH, "data/changepoints.csv") - - -def state_space_estimator(exogenous_feature_columns): - """Constructs a StructuralEnsembleRegressor.""" - - def _exogenous_update_condition(times, features): - del times # unused - # Make exogenous updates sparse by setting an update condition. This in - # effect allows missing exogenous features: if the condition evaluates to - # False, no update is performed. Otherwise we sometimes end up with "leaky" - # updates which add unnecessary uncertainty to the model even when there is - # no changepoint. - return tf.equal(tf.squeeze(features["is_changepoint"], axis=-1), "yes") - - return ( - tf.contrib.timeseries.StructuralEnsembleRegressor( - periodicities=12, - # Extract a smooth period by constraining the number of latent values - # being cycled between. - cycle_num_latent_values=3, - num_features=1, - exogenous_feature_columns=exogenous_feature_columns, - exogenous_update_condition=_exogenous_update_condition), - # Use truncated backpropagation with a window size of 64, batching - # together 4 of these windows (random offsets) per training step. Training - # with exogenous features often requires somewhat larger windows. - 4, 64) - - -def autoregressive_estimator(exogenous_feature_columns): - input_window_size = 8 - output_window_size = 2 - return ( - tf.contrib.timeseries.ARRegressor( - periodicities=12, - num_features=1, - input_window_size=input_window_size, - output_window_size=output_window_size, - exogenous_feature_columns=exogenous_feature_columns), - 64, input_window_size + output_window_size) - - -def train_and_evaluate_exogenous( - estimator_fn, csv_file_name=_DATA_FILE, train_steps=300): - """Training, evaluating, and predicting on a series with changepoints.""" - # Indicate the format of our exogenous feature, in this case a string - # representing a boolean value. - string_feature = tf.feature_column.categorical_column_with_vocabulary_list( - key="is_changepoint", vocabulary_list=["no", "yes"]) - # Specify the way this feature is presented to the model, here using a one-hot - # encoding. - one_hot_feature = tf.feature_column.indicator_column( - categorical_column=string_feature) - - estimator, batch_size, window_size = estimator_fn( - exogenous_feature_columns=[one_hot_feature]) - reader = tf.contrib.timeseries.CSVReader( - csv_file_name, - # Indicate the format of our CSV file. First we have two standard columns, - # one for times and one for values. The third column is a custom exogenous - # feature indicating whether each timestep is a changepoint. The - # changepoint feature name must match the string_feature column name - # above. - column_names=(tf.contrib.timeseries.TrainEvalFeatures.TIMES, - tf.contrib.timeseries.TrainEvalFeatures.VALUES, - "is_changepoint"), - # Indicate dtypes for our features. - column_dtypes=(tf.int64, tf.float32, tf.string), - # This CSV has a header line; here we just ignore it. - skip_header_lines=1) - train_input_fn = tf.contrib.timeseries.RandomWindowInputFn( - reader, batch_size=batch_size, window_size=window_size) - estimator.train(input_fn=train_input_fn, steps=train_steps) - evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader) - evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1) - # Create an input_fn for prediction, with a simulated changepoint. Since all - # of the anomalies in the training data are explained by the exogenous - # feature, we should get relatively confident predictions before the indicated - # changepoint (since we are telling the model that no changepoint exists at - # those times) and relatively uncertain predictions after. - (predictions,) = tuple(estimator.predict( - input_fn=tf.contrib.timeseries.predict_continuation_input_fn( - evaluation, steps=100, - exogenous_features={ - "is_changepoint": [["no"] * 49 + ["yes"] + ["no"] * 50]}))) - times = evaluation["times"][0] - observed = evaluation["observed"][0, :, 0] - mean = np.squeeze(np.concatenate( - [evaluation["mean"][0], predictions["mean"]], axis=0)) - variance = np.squeeze(np.concatenate( - [evaluation["covariance"][0], predictions["covariance"]], axis=0)) - all_times = np.concatenate([times, predictions["times"]], axis=0) - upper_limit = mean + np.sqrt(variance) - lower_limit = mean - np.sqrt(variance) - # Indicate the locations of the changepoints for plotting vertical lines. - anomaly_locations = [] - with open(csv_file_name, "r") as csv_file: - csv_reader = csv.DictReader(csv_file) - for row in csv_reader: - if row["is_changepoint"] == "yes": - anomaly_locations.append(int(row["time"])) - anomaly_locations.append(predictions["times"][49]) - return (times, observed, all_times, mean, upper_limit, lower_limit, - anomaly_locations) - - -def make_plot(name, training_times, observed, all_times, mean, - upper_limit, lower_limit, anomaly_locations): - """Plot the time series and anomalies in a new figure.""" - pyplot.figure() - pyplot.plot(training_times, observed, "b", label="training series") - pyplot.plot(all_times, mean, "r", label="forecast") - pyplot.axvline(anomaly_locations[0], linestyle="dotted", label="changepoints") - for anomaly_location in anomaly_locations[1:]: - pyplot.axvline(anomaly_location, linestyle="dotted") - pyplot.fill_between(all_times, lower_limit, upper_limit, color="grey", - alpha="0.2") - pyplot.axvline(training_times[-1], color="k", linestyle="--") - pyplot.xlabel("time") - pyplot.ylabel("observations") - pyplot.legend(loc=0) - pyplot.title(name) - - -def main(unused_argv): - if not HAS_MATPLOTLIB: - raise ImportError( - "Please install matplotlib to generate a plot from this example.") - make_plot("Ignoring a known anomaly (state space)", - *train_and_evaluate_exogenous( - estimator_fn=state_space_estimator)) - make_plot("Ignoring a known anomaly (autoregressive)", - *train_and_evaluate_exogenous( - estimator_fn=autoregressive_estimator, train_steps=3000)) - pyplot.show() - - -if __name__ == "__main__": - tf.compat.v1.app.run(main=main) diff --git a/tensorflow/contrib/timeseries/examples/known_anomaly_test.py b/tensorflow/contrib/timeseries/examples/known_anomaly_test.py deleted file mode 100644 index 57ccf8f260f..00000000000 --- a/tensorflow/contrib/timeseries/examples/known_anomaly_test.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests that the TensorFlow parts of the known anomaly example run.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.examples import known_anomaly - -from tensorflow.python.platform import test - - -class KnownAnomalyExampleTest(test.TestCase): - - def test_shapes_and_variance_structural_ar(self): - (times, observed, all_times, mean, upper_limit, lower_limit, - anomaly_locations) = known_anomaly.train_and_evaluate_exogenous( - train_steps=1, estimator_fn=known_anomaly.autoregressive_estimator) - self.assertAllEqual( - anomaly_locations, - [25, 50, 75, 100, 125, 150, 175, 249]) - self.assertAllEqual(all_times.shape, mean.shape) - self.assertAllEqual(all_times.shape, upper_limit.shape) - self.assertAllEqual(all_times.shape, lower_limit.shape) - self.assertAllEqual(times.shape, observed.shape) - - def test_shapes_and_variance_structural_ssm(self): - (times, observed, all_times, mean, upper_limit, lower_limit, - anomaly_locations) = known_anomaly.train_and_evaluate_exogenous( - train_steps=50, estimator_fn=known_anomaly.state_space_estimator) - self.assertAllEqual( - anomaly_locations, - [25, 50, 75, 100, 125, 150, 175, 249]) - self.assertAllEqual([200], times.shape) - self.assertAllEqual([200], observed.shape) - self.assertAllEqual([300], all_times.shape) - self.assertAllEqual([300], mean.shape) - self.assertAllEqual([300], upper_limit.shape) - self.assertAllEqual([300], lower_limit.shape) - # Check that initial predictions are relatively confident. - self.assertLess(upper_limit[210] - lower_limit[210], - 3.0 * (upper_limit[200] - lower_limit[200])) - # Check that post-changepoint predictions are less confident - self.assertGreater(upper_limit[290] - lower_limit[290], - 3.0 * (upper_limit[240] - lower_limit[240])) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/examples/lstm.py b/tensorflow/contrib/timeseries/examples/lstm.py deleted file mode 100644 index cc33c0fa73c..00000000000 --- a/tensorflow/contrib/timeseries/examples/lstm.py +++ /dev/null @@ -1,299 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A more advanced example, of building an RNN-based time series model.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -from os import path -import tempfile - -import numpy -import tensorflow as tf - -from tensorflow.contrib.timeseries.python.timeseries import estimators as ts_estimators -from tensorflow.contrib.timeseries.python.timeseries import model as ts_model -from tensorflow.contrib.timeseries.python.timeseries import state_management - -try: - import matplotlib # pylint: disable=g-import-not-at-top - matplotlib.use("TkAgg") # Need Tk for interactive plots. - from matplotlib import pyplot # pylint: disable=g-import-not-at-top - HAS_MATPLOTLIB = True -except ImportError: - # Plotting requires matplotlib, but the unit test running this code may - # execute in an environment without it (i.e. matplotlib is not a build - # dependency). We'd still like to test the TensorFlow-dependent parts of this - # example. - HAS_MATPLOTLIB = False - -_MODULE_PATH = path.dirname(__file__) -_DATA_FILE = path.join(_MODULE_PATH, "data/multivariate_periods.csv") - - -class _LSTMModel(ts_model.SequentialTimeSeriesModel): - """A time series model-building example using an RNNCell.""" - - def __init__(self, num_units, num_features, exogenous_feature_columns=None, - dtype=tf.float32): - """Initialize/configure the model object. - - Note that we do not start graph building here. Rather, this object is a - configurable factory for TensorFlow graphs which are run by an Estimator. - - Args: - num_units: The number of units in the model's LSTMCell. - num_features: The dimensionality of the time series (features per - timestep). - exogenous_feature_columns: A list of `tf.feature_column`s representing - features which are inputs to the model but are not predicted by - it. These must then be present for training, evaluation, and - prediction. - dtype: The floating point data type to use. - """ - super(_LSTMModel, self).__init__( - # Pre-register the metrics we'll be outputting (just a mean here). - train_output_names=["mean"], - predict_output_names=["mean"], - num_features=num_features, - exogenous_feature_columns=exogenous_feature_columns, - dtype=dtype) - self._num_units = num_units - # Filled in by initialize_graph() - self._lstm_cell = None - self._lstm_cell_run = None - self._predict_from_lstm_output = None - - def initialize_graph(self, input_statistics=None): - """Save templates for components, which can then be used repeatedly. - - This method is called every time a new graph is created. It's safe to start - adding ops to the current default graph here, but the graph should be - constructed from scratch. - - Args: - input_statistics: A math_utils.InputStatistics object. - """ - super(_LSTMModel, self).initialize_graph(input_statistics=input_statistics) - with tf.variable_scope("", use_resource=True): - # Use ResourceVariables to avoid race conditions. - self._lstm_cell = tf.nn.rnn_cell.LSTMCell(num_units=self._num_units) - # Create templates so we don't have to worry about variable reuse. - self._lstm_cell_run = tf.make_template( - name_="lstm_cell", - func_=self._lstm_cell, - create_scope_now_=True) - # Transforms LSTM output into mean predictions. - self._predict_from_lstm_output = tf.make_template( - name_="predict_from_lstm_output", - func_=functools.partial(tf.layers.dense, units=self.num_features), - create_scope_now_=True) - - def get_start_state(self): - """Return initial state for the time series model.""" - return ( - # Keeps track of the time associated with this state for error checking. - tf.zeros([], dtype=tf.int64), - # The previous observation or prediction. - tf.zeros([self.num_features], dtype=self.dtype), - # The most recently seen exogenous features. - tf.zeros(self._get_exogenous_embedding_shape(), dtype=self.dtype), - # The state of the RNNCell (batch dimension removed since this parent - # class will broadcast). - [tf.squeeze(state_element, axis=0) - for state_element - in self._lstm_cell.zero_state(batch_size=1, dtype=self.dtype)]) - - def _filtering_step(self, current_times, current_values, state, predictions): - """Update model state based on observations. - - Note that we don't do much here aside from computing a loss. In this case - it's easier to update the RNN state in _prediction_step, since that covers - running the RNN both on observations (from this method) and our own - predictions. This distinction can be important for probabilistic models, - where repeatedly predicting without filtering should lead to low-confidence - predictions. - - Args: - current_times: A [batch size] integer Tensor. - current_values: A [batch size, self.num_features] floating point Tensor - with new observations. - state: The model's state tuple. - predictions: The output of the previous `_prediction_step`. - Returns: - A tuple of new state and a predictions dictionary updated to include a - loss (note that we could also return other measures of goodness of fit, - although only "loss" will be optimized). - """ - state_from_time, prediction, exogenous, lstm_state = state - with tf.control_dependencies( - [tf.assert_equal(current_times, state_from_time)]): - # Subtract the mean and divide by the variance of the series. Slightly - # more efficient if done for a whole window (using the normalize_features - # argument to SequentialTimeSeriesModel). - transformed_values = self._scale_data(current_values) - # Use mean squared error across features for the loss. - predictions["loss"] = tf.reduce_mean( - (prediction - transformed_values) ** 2, axis=-1) - # Keep track of the new observation in model state. It won't be run - # through the LSTM until the next _imputation_step. - new_state_tuple = (current_times, transformed_values, - exogenous, lstm_state) - return (new_state_tuple, predictions) - - def _prediction_step(self, current_times, state): - """Advance the RNN state using a previous observation or prediction.""" - _, previous_observation_or_prediction, exogenous, lstm_state = state - # Update LSTM state based on the most recent exogenous and endogenous - # features. - inputs = tf.concat([previous_observation_or_prediction, exogenous], - axis=-1) - lstm_output, new_lstm_state = self._lstm_cell_run( - inputs=inputs, state=lstm_state) - next_prediction = self._predict_from_lstm_output(lstm_output) - new_state_tuple = (current_times, next_prediction, - exogenous, new_lstm_state) - return new_state_tuple, {"mean": self._scale_back_data(next_prediction)} - - def _imputation_step(self, current_times, state): - """Advance model state across a gap.""" - # Does not do anything special if we're jumping across a gap. More advanced - # models, especially probabilistic ones, would want a special case that - # depends on the gap size. - return state - - def _exogenous_input_step( - self, current_times, current_exogenous_regressors, state): - """Save exogenous regressors in model state for use in _prediction_step.""" - state_from_time, prediction, _, lstm_state = state - return (state_from_time, prediction, - current_exogenous_regressors, lstm_state) - - -def train_and_predict( - csv_file_name=_DATA_FILE, training_steps=200, estimator_config=None, - export_directory=None): - """Train and predict using a custom time series model.""" - # Construct an Estimator from our LSTM model. - categorical_column = tf.feature_column.categorical_column_with_hash_bucket( - key="categorical_exogenous_feature", hash_bucket_size=16) - exogenous_feature_columns = [ - # Exogenous features are not part of the loss, but can inform - # predictions. In this example the features have no extra information, but - # are included as an API example. - tf.feature_column.numeric_column( - "2d_exogenous_feature", shape=(2,)), - tf.feature_column.embedding_column( - categorical_column=categorical_column, dimension=10)] - estimator = ts_estimators.TimeSeriesRegressor( - model=_LSTMModel( - num_features=5, - num_units=128, - exogenous_feature_columns=exogenous_feature_columns), - optimizer=tf.compat.v1.train.AdamOptimizer(0.001), - config=estimator_config, - # Set state to be saved across windows. - state_manager=state_management.ChainingStateManager()) - reader = tf.contrib.timeseries.CSVReader( - csv_file_name, - column_names=((tf.contrib.timeseries.TrainEvalFeatures.TIMES,) - + (tf.contrib.timeseries.TrainEvalFeatures.VALUES,) * 5 - + ("2d_exogenous_feature",) * 2 - + ("categorical_exogenous_feature",)), - # Data types other than for `times` need to be specified if they aren't - # float32. In this case one of our exogenous features has string dtype. - column_dtypes=((tf.int64,) + (tf.float32,) * 7 + (tf.string,))) - train_input_fn = tf.contrib.timeseries.RandomWindowInputFn( - reader, batch_size=4, window_size=32) - estimator.train(input_fn=train_input_fn, steps=training_steps) - evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader) - evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1) - # Predict starting after the evaluation - predict_exogenous_features = { - "2d_exogenous_feature": numpy.concatenate( - [numpy.ones([1, 100, 1]), numpy.zeros([1, 100, 1])], - axis=-1), - "categorical_exogenous_feature": numpy.array( - ["strkey"] * 100)[None, :, None]} - (predictions,) = tuple(estimator.predict( - input_fn=tf.contrib.timeseries.predict_continuation_input_fn( - evaluation, steps=100, - exogenous_features=predict_exogenous_features))) - times = evaluation["times"][0] - observed = evaluation["observed"][0, :, :] - predicted_mean = numpy.squeeze(numpy.concatenate( - [evaluation["mean"][0], predictions["mean"]], axis=0)) - all_times = numpy.concatenate([times, predictions["times"]], axis=0) - - # Export the model in SavedModel format. We include a bit of extra boilerplate - # for "cold starting" as if we didn't have any state from the Estimator, which - # is the case when serving from a SavedModel. If Estimator output is - # available, the result of "Estimator.evaluate" can be passed directly to - # `tf.contrib.timeseries.saved_model_utils.predict_continuation` as the - # `continue_from` argument. - with tf.Graph().as_default(): - filter_feature_tensors, _ = evaluation_input_fn() - with tf.train.MonitoredSession() as session: - # Fetch the series to "warm up" our state, which will allow us to make - # predictions for its future values. This is just a dictionary of times, - # values, and exogenous features mapping to numpy arrays. The use of an - # input_fn is just a convenience for the example; they can also be - # specified manually. - filter_features = session.run(filter_feature_tensors) - if export_directory is None: - export_directory = tempfile.mkdtemp() - input_receiver_fn = estimator.build_raw_serving_input_receiver_fn() - export_location = estimator.export_saved_model(export_directory, - input_receiver_fn) - # Warm up and predict using the SavedModel - with tf.Graph().as_default(): - with tf.compat.v1.Session() as session: - signatures = tf.saved_model.loader.load( - session, [tf.saved_model.tag_constants.SERVING], export_location) - state = tf.contrib.timeseries.saved_model_utils.cold_start_filter( - signatures=signatures, session=session, features=filter_features) - saved_model_output = ( - tf.contrib.timeseries.saved_model_utils.predict_continuation( - continue_from=state, signatures=signatures, - session=session, steps=100, - exogenous_features=predict_exogenous_features)) - # The exported model gives the same results as the Estimator.predict() - # call above. - numpy.testing.assert_allclose( - predictions["mean"], - numpy.squeeze(saved_model_output["mean"], axis=0)) - return times, observed, all_times, predicted_mean - - -def main(unused_argv): - if not HAS_MATPLOTLIB: - raise ImportError( - "Please install matplotlib to generate a plot from this example.") - (observed_times, observations, - all_times, predictions) = train_and_predict() - pyplot.axvline(99, linestyle="dotted") - observed_lines = pyplot.plot( - observed_times, observations, label="Observed", color="k") - predicted_lines = pyplot.plot( - all_times, predictions, label="Predicted", color="b") - pyplot.legend(handles=[observed_lines[0], predicted_lines[0]], - loc="upper left") - pyplot.show() - - -if __name__ == "__main__": - tf.compat.v1.app.run(main=main) diff --git a/tensorflow/contrib/timeseries/examples/lstm_test.py b/tensorflow/contrib/timeseries/examples/lstm_test.py deleted file mode 100644 index c58e24e6d97..00000000000 --- a/tensorflow/contrib/timeseries/examples/lstm_test.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests that the TensorFlow parts of the LSTM example run.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.examples import lstm - -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.platform import test - - -class _SeedRunConfig(estimator_lib.RunConfig): - - @property - def tf_random_seed(self): - return 3 - - -class LSTMExampleTest(test.TestCase): - - def test_periodicity_learned(self): - (observed_times, observed_values, - all_times, predicted_values) = lstm.train_and_predict( - training_steps=2, estimator_config=_SeedRunConfig(), - export_directory=self.get_temp_dir()) - self.assertAllEqual([100], observed_times.shape) - self.assertAllEqual([100, 5], observed_values.shape) - self.assertAllEqual([200], all_times.shape) - self.assertAllEqual([200, 5], predicted_values.shape) - # TODO(allenl): Make the model deterministic so you can check something - # substantive. - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/examples/multivariate.py b/tensorflow/contrib/timeseries/examples/multivariate.py deleted file mode 100644 index 1ce7e4b6ddd..00000000000 --- a/tensorflow/contrib/timeseries/examples/multivariate.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A multivariate TFTS example. - -Fits a multivariate model, exports it, and visualizes the learned correlations -by iteratively predicting and sampling from the predictions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from os import path -import tempfile - -import numpy -import tensorflow as tf - -try: - import matplotlib # pylint: disable=g-import-not-at-top - matplotlib.use("TkAgg") # Need Tk for interactive plots. - from matplotlib import pyplot # pylint: disable=g-import-not-at-top - HAS_MATPLOTLIB = True -except ImportError: - # Plotting requires matplotlib, but the unit test running this code may - # execute in an environment without it (i.e. matplotlib is not a build - # dependency). We'd still like to test the TensorFlow-dependent parts of this - # example, namely train_and_predict. - HAS_MATPLOTLIB = False - -_MODULE_PATH = path.dirname(__file__) -_DATA_FILE = path.join(_MODULE_PATH, "data/multivariate_level.csv") - - -def multivariate_train_and_sample( - csv_file_name=_DATA_FILE, export_directory=None, training_steps=500): - """Trains, evaluates, and exports a multivariate model.""" - estimator = tf.contrib.timeseries.StructuralEnsembleRegressor( - periodicities=[], num_features=5) - reader = tf.contrib.timeseries.CSVReader( - csv_file_name, - column_names=((tf.contrib.timeseries.TrainEvalFeatures.TIMES,) - + (tf.contrib.timeseries.TrainEvalFeatures.VALUES,) * 5)) - train_input_fn = tf.contrib.timeseries.RandomWindowInputFn( - # Larger window sizes generally produce a better covariance matrix. - reader, batch_size=4, window_size=64) - estimator.train(input_fn=train_input_fn, steps=training_steps) - evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader) - current_state = estimator.evaluate(input_fn=evaluation_input_fn, steps=1) - values = [current_state["observed"]] - times = [current_state[tf.contrib.timeseries.FilteringResults.TIMES]] - # Export the model so we can do iterative prediction and filtering without - # reloading model checkpoints. - if export_directory is None: - export_directory = tempfile.mkdtemp() - input_receiver_fn = estimator.build_raw_serving_input_receiver_fn() - export_location = estimator.export_saved_model(export_directory, - input_receiver_fn) - with tf.Graph().as_default(): - numpy.random.seed(1) # Make the example a bit more deterministic - with tf.compat.v1.Session() as session: - signatures = tf.saved_model.loader.load( - session, [tf.saved_model.tag_constants.SERVING], export_location) - for _ in range(100): - current_prediction = ( - tf.contrib.timeseries.saved_model_utils.predict_continuation( - continue_from=current_state, signatures=signatures, - session=session, steps=1)) - next_sample = numpy.random.multivariate_normal( - # Squeeze out the batch and series length dimensions (both 1). - mean=numpy.squeeze(current_prediction["mean"], axis=(0, 1)), - cov=numpy.squeeze(current_prediction["covariance"], axis=(0, 1))) - # Update model state so that future predictions are conditional on the - # value we just sampled. - filtering_features = { - tf.contrib.timeseries.TrainEvalFeatures.TIMES: current_prediction[ - tf.contrib.timeseries.FilteringResults.TIMES], - tf.contrib.timeseries.TrainEvalFeatures.VALUES: next_sample[ - None, None, :]} - current_state = ( - tf.contrib.timeseries.saved_model_utils.filter_continuation( - continue_from=current_state, - session=session, - signatures=signatures, - features=filtering_features)) - values.append(next_sample[None, None, :]) - times.append(current_state["times"]) - all_observations = numpy.squeeze(numpy.concatenate(values, axis=1), axis=0) - all_times = numpy.squeeze(numpy.concatenate(times, axis=1), axis=0) - return all_times, all_observations - - -def main(unused_argv): - if not HAS_MATPLOTLIB: - raise ImportError( - "Please install matplotlib to generate a plot from this example.") - all_times, all_observations = multivariate_train_and_sample() - # Show where sampling starts on the plot - pyplot.axvline(1000, linestyle="dotted") - pyplot.plot(all_times, all_observations) - pyplot.show() - - -if __name__ == "__main__": - tf.compat.v1.app.run(main=main) diff --git a/tensorflow/contrib/timeseries/examples/multivariate_test.py b/tensorflow/contrib/timeseries/examples/multivariate_test.py deleted file mode 100644 index 3b8ee0606e4..00000000000 --- a/tensorflow/contrib/timeseries/examples/multivariate_test.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests that the TensorFlow parts of the multivariate example run.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.examples import multivariate - -from tensorflow.python.platform import test - - -class MultivariateExampleTest(test.TestCase): - - def test_shapes_structural(self): - times, values = multivariate.multivariate_train_and_sample( - export_directory=self.get_temp_dir(), training_steps=5) - self.assertAllEqual([1100], times.shape) - self.assertAllEqual([1100, 5], values.shape) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/examples/predict.py b/tensorflow/contrib/timeseries/examples/predict.py deleted file mode 100644 index 0e16e99a249..00000000000 --- a/tensorflow/contrib/timeseries/examples/predict.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""An example of training and predicting with a TFTS estimator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import os -import sys - -import numpy as np -import tensorflow as tf - - -try: - import matplotlib # pylint: disable=g-import-not-at-top - matplotlib.use("TkAgg") # Need Tk for interactive plots. - from matplotlib import pyplot # pylint: disable=g-import-not-at-top - HAS_MATPLOTLIB = True -except ImportError: - # Plotting requires matplotlib, but the unit test running this code may - # execute in an environment without it (i.e. matplotlib is not a build - # dependency). We'd still like to test the TensorFlow-dependent parts of this - # example, namely train_and_predict. - HAS_MATPLOTLIB = False - -FLAGS = None - - -_MODULE_PATH = os.path.dirname(__file__) -_DEFAULT_DATA_FILE = os.path.join(_MODULE_PATH, "data/period_trend.csv") - - -def structural_ensemble_train_and_predict(csv_file_name): - # Cycle between 5 latent values over a period of 100. This leads to a very - # smooth periodic component (and a small model), which is a good fit for our - # example data. Modeling high-frequency periodic variations will require a - # higher cycle_num_latent_values. - structural = tf.contrib.timeseries.StructuralEnsembleRegressor( - periodicities=100, num_features=1, cycle_num_latent_values=5) - return train_and_predict(structural, csv_file_name, training_steps=150) - - -def ar_train_and_predict(csv_file_name): - # An autoregressive model, with periodicity handled as a time-based - # regression. Note that this requires windows of size 16 (input_window_size + - # output_window_size) for training. - ar = tf.contrib.timeseries.ARRegressor( - periodicities=100, input_window_size=10, output_window_size=6, - num_features=1, - # Use the (default) normal likelihood loss to adaptively fit the - # variance. SQUARED_LOSS overestimates variance when there are trends in - # the series. - loss=tf.contrib.timeseries.ARModel.NORMAL_LIKELIHOOD_LOSS) - return train_and_predict(ar, csv_file_name, training_steps=600) - - -def train_and_predict(estimator, csv_file_name, training_steps): - """A simple example of training and predicting.""" - # Read data in the default "time,value" CSV format with no header - reader = tf.contrib.timeseries.CSVReader(csv_file_name) - # Set up windowing and batching for training - train_input_fn = tf.contrib.timeseries.RandomWindowInputFn( - reader, batch_size=16, window_size=16) - # Fit model parameters to data - estimator.train(input_fn=train_input_fn, steps=training_steps) - # Evaluate on the full dataset sequentially, collecting in-sample predictions - # for a qualitative evaluation. Note that this loads the whole dataset into - # memory. For quantitative evaluation, use RandomWindowChunker. - evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader) - evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1) - # Predict starting after the evaluation - (predictions,) = tuple(estimator.predict( - input_fn=tf.contrib.timeseries.predict_continuation_input_fn( - evaluation, steps=200))) - times = evaluation["times"][0] - observed = evaluation["observed"][0, :, 0] - mean = np.squeeze(np.concatenate( - [evaluation["mean"][0], predictions["mean"]], axis=0)) - variance = np.squeeze(np.concatenate( - [evaluation["covariance"][0], predictions["covariance"]], axis=0)) - all_times = np.concatenate([times, predictions["times"]], axis=0) - upper_limit = mean + np.sqrt(variance) - lower_limit = mean - np.sqrt(variance) - return times, observed, all_times, mean, upper_limit, lower_limit - - -def make_plot(name, training_times, observed, all_times, mean, - upper_limit, lower_limit): - """Plot a time series in a new figure.""" - pyplot.figure() - pyplot.plot(training_times, observed, "b", label="training series") - pyplot.plot(all_times, mean, "r", label="forecast") - pyplot.plot(all_times, upper_limit, "g", label="forecast upper bound") - pyplot.plot(all_times, lower_limit, "g", label="forecast lower bound") - pyplot.fill_between(all_times, lower_limit, upper_limit, color="grey", - alpha="0.2") - pyplot.axvline(training_times[-1], color="k", linestyle="--") - pyplot.xlabel("time") - pyplot.ylabel("observations") - pyplot.legend(loc=0) - pyplot.title(name) - - -def main(unused_argv): - if not HAS_MATPLOTLIB: - raise ImportError( - "Please install matplotlib to generate a plot from this example.") - input_filename = FLAGS.input_filename - if input_filename is None: - input_filename = _DEFAULT_DATA_FILE - make_plot("Structural ensemble", - *structural_ensemble_train_and_predict(input_filename)) - make_plot("AR", *ar_train_and_predict(input_filename)) - pyplot.show() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--input_filename", - type=str, - required=False, - help="Input csv file (omit to use the data/period_trend.csv).") - FLAGS, unparsed = parser.parse_known_args() - tf.compat.v1.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/timeseries/examples/predict_test.py b/tensorflow/contrib/timeseries/examples/predict_test.py deleted file mode 100644 index b353f85cb5d..00000000000 --- a/tensorflow/contrib/timeseries/examples/predict_test.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests that the TensorFlow parts of the prediction example run.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from os import path - -from tensorflow.contrib.timeseries.examples import predict - -from tensorflow.python.platform import test - - -_MODULE_PATH = path.dirname(__file__) -_DATA_FILE = path.join(_MODULE_PATH, "data/period_trend.csv") - - -class PeriodTrendExampleTest(test.TestCase): - - def test_shapes_and_variance_structural(self): - (times, observed, all_times, mean, upper_limit, lower_limit - ) = predict.structural_ensemble_train_and_predict(_DATA_FILE) - # Just check that plotting will probably be OK. We can't actually run the - # plotting code since we don't want to pull in matplotlib as a dependency - # for this test. - self.assertAllEqual([500], times.shape) - self.assertAllEqual([500], observed.shape) - self.assertAllEqual([700], all_times.shape) - self.assertAllEqual([700], mean.shape) - self.assertAllEqual([700], upper_limit.shape) - self.assertAllEqual([700], lower_limit.shape) - - def test_ar(self): - (times, observed, all_times, mean, - upper_limit, lower_limit) = predict.ar_train_and_predict(_DATA_FILE) - self.assertAllEqual(times.shape, observed.shape) - self.assertAllEqual(all_times.shape, mean.shape) - self.assertAllEqual(all_times.shape, upper_limit.shape) - self.assertAllEqual(all_times.shape, lower_limit.shape) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/__init__.py b/tensorflow/contrib/timeseries/python/__init__.py deleted file mode 100644 index 5072feab59d..00000000000 --- a/tensorflow/contrib/timeseries/python/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A time series library in TensorFlow (TFTS).""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import -from tensorflow.contrib.timeseries.python.timeseries import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD deleted file mode 100644 index 02e475367af..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/BUILD +++ /dev/null @@ -1,470 +0,0 @@ -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = [ - "//tensorflow:internal", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "py_init", - srcs = [ - "__init__.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":ar_model", - ":estimators", - ":feature_keys", - ":input_pipeline", - ":saved_model_utils", - ], -) - -py_library( - name = "feature_keys", - srcs = [ - "feature_keys.py", - ], - srcs_version = "PY2AND3", - deps = ["//tensorflow/python/saved_model:signature_constants"], -) - -py_library( - name = "saved_model_utils", - srcs = [ - "saved_model_utils.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":feature_keys", - ":head", - ":input_pipeline", - ":model_utils", - "//tensorflow/python:util", - ], -) - -py_library( - name = "model", - srcs = [ - "model.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":feature_keys", - ":math_utils", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:tensor_array_ops", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - ], -) - -py_library( - name = "estimators", - srcs = [ - "estimators.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":ar_model", - ":feature_keys", - ":head", - ":math_utils", - ":state_management", - "//tensorflow/contrib/timeseries/python/timeseries/state_space_models:filtering_postprocessor", - "//tensorflow/contrib/timeseries/python/timeseries/state_space_models:state_space_model", - "//tensorflow/contrib/timeseries/python/timeseries/state_space_models:structural_ensemble", - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:tensor_util", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python/estimator:estimator_py", - "//tensorflow/python/feature_column", - ], -) - -py_test( - name = "estimators_test", - timeout = "long", - srcs = [ - "estimators_test.py", - ], - python_version = "PY2", - shard_count = 3, - srcs_version = "PY2AND3", - tags = [ - "no_mac", - "no_pip_gpu", # b/63391119 - "nomsan", # Takes too long to run. - "notsan", # b/67865658 - "optonly", # Takes too long to run without optimization. - ], - deps = [ - ":ar_model", - ":estimators", - ":feature_keys", - ":input_pipeline", - ":saved_model_utils", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:session", - "//tensorflow/python/estimator:estimator_py", - "//tensorflow/python/saved_model:loader", - "//tensorflow/python/saved_model:tag_constants", - "//third_party/py/numpy", - ], -) - -py_library( - name = "head", - srcs = [ - "head.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":feature_keys", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:summary", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python/estimator:estimator_py", - ], -) - -py_test( - name = "head_test", - size = "medium", - srcs = [ - "head_test.py", - ], - python_version = "PY2", - shard_count = 10, - srcs_version = "PY2AND3", - tags = [ - "no_pip_gpu", # b/63391119 - "notap", # b/124520733 - ], - deps = [ - ":estimators", - ":feature_keys", - ":head", - ":input_pipeline", - ":model", - ":state_management", - "//tensorflow/contrib/timeseries/examples:lstm_main_lib", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:metrics", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//tensorflow/python/estimator:estimator_py", - "//tensorflow/python/feature_column", - "//tensorflow/python/saved_model:loader", - "//tensorflow/python/saved_model:tag_constants", - "//third_party/py/numpy", - "@absl_py//absl/testing:parameterized", - "@six_archive//:six", - ], -) - -py_library( - name = "model_utils", - srcs = [ - "model_utils.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":feature_keys", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:variable_scope", - "//third_party/py/numpy", - ], -) - -py_test( - name = "model_utils_test", - srcs = [ - "model_utils_test.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_pip_gpu", # b/63391119 - ], - deps = [ - ":model_utils", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - ], -) - -py_library( - name = "state_management", - srcs = [ - "state_management.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":feature_keys", - ":math_utils", - ":model", - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:util", - "//tensorflow/python/estimator:estimator_py", - ], -) - -py_test( - name = "state_management_test", - srcs = [ - "state_management_test.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_oss", - "no_pip", # b/64527635 - "no_pip_gpu", # b/63391119 - ], - deps = [ - ":feature_keys", - ":input_pipeline", - ":math_utils", - ":model", - ":state_management", - ":test_utils", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/estimator:estimator_py", - "//third_party/py/numpy", - ], -) - -py_library( - name = "input_pipeline", - srcs = [ - "input_pipeline.py", - ], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":feature_keys", - ":model_utils", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:io_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:tensor_array_ops", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python/estimator:estimator_py", - "//third_party/py/numpy", - ], -) - -py_test( - name = "input_pipeline_test", - srcs = [ - "input_pipeline_test.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_oss", # b/63709811 - "no_pip", # b/63709811 - "no_pip_gpu", # b/63391119 - ], - deps = [ - ":feature_keys", - ":input_pipeline", - ":test_utils", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_library( - name = "test_utils", - srcs = [ - "test_utils.py", - ], - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [ - ":estimators", - ":feature_keys", - ":input_pipeline", - ":state_management", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_seed", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variables", - "//tensorflow/python/estimator:estimator_py", - ], -) - -py_library( - name = "ar_model", - srcs = [ - "ar_model.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":feature_keys", - ":math_utils", - ":model", - ":model_utils", - "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", - "//tensorflow/python:tensor_array_ops", - "//tensorflow/python:variable_scope", - "//tensorflow/python/estimator:estimator_py", - ], -) - -py_test( - name = "ar_model_test", - timeout = "long", # Moderate but for asan - srcs = [ - "ar_model_test.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["notsan"], - deps = [ - ":ar_model", - ":estimators", - ":feature_keys", - ":input_pipeline", - ":test_utils", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_ops", - "//tensorflow/python:platform", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/estimator:estimator_py", - "//third_party/py/numpy", - ], -) - -py_library( - name = "math_utils", - srcs = [ - "math_utils.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":feature_keys", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/lookup:lookup_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:functional_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nn", - "//tensorflow/python:state_ops", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - ], -) - -py_test( - name = "math_utils_test", - srcs = [ - "math_utils_test.py", - ], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_pip_gpu", # b/63391119 - "no_windows", # TODO: needs investigation on Windows - ], - deps = [ - ":feature_keys", - ":input_pipeline", - ":math_utils", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/timeseries/python/timeseries/__init__.py b/tensorflow/contrib/timeseries/python/timeseries/__init__.py deleted file mode 100644 index 8462138339c..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A time series library in TensorFlow (TFTS).""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.python.timeseries import saved_model_utils - -# pylint: disable=wildcard-import -from tensorflow.contrib.timeseries.python.timeseries.ar_model import * -from tensorflow.contrib.timeseries.python.timeseries.estimators import * -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import * -from tensorflow.contrib.timeseries.python.timeseries.head import * -from tensorflow.contrib.timeseries.python.timeseries.input_pipeline import * -# pylint: enable=wildcard-import diff --git a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py deleted file mode 100644 index ac0753e9cfd..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py +++ /dev/null @@ -1,994 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Auto-Regressive models for time series data.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.rnn.python.ops import lstm_ops -from tensorflow.contrib.timeseries.python.timeseries import math_utils -from tensorflow.contrib.timeseries.python.timeseries import model -from tensorflow.contrib.timeseries.python.timeseries import model_utils -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import PredictionFeatures -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import TrainEvalFeatures - -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.keras.engine import sequential -from tensorflow.python.keras.engine import training -from tensorflow.python.keras.layers import core -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import tensor_array_ops -from tensorflow.python.ops import variable_scope - - -class FlatPredictionModel(training.Model): - """Flattens input and output windows and puts them through dense layers. - - This model does not operate on its own, but rather is a plugin to - `ARModel`. See `ARModel`'s constructor documentation - (`prediction_model_factory`) for a usage example. - """ - - def __init__(self, - num_features, - input_window_size, - output_window_size, - hidden_layer_sizes=None): - """Construct the flat prediction model. - - Args: - num_features: number of input features per time step. - input_window_size: Number of past time steps of data to look at when doing - the regression. - output_window_size: Number of future time steps to predict. Note that - setting it to > 1 empirically seems to give a better fit. - hidden_layer_sizes: list of sizes of hidden layers. - """ - super(FlatPredictionModel, self).__init__() - self._input_flatten = core.Flatten() - self._output_flatten = core.Flatten() - if hidden_layer_sizes: - self._hidden_layers = sequential.Sequential([ - core.Dense(layer_size, activation=nn_ops.relu) - for layer_size in hidden_layer_sizes]) - else: - self._hidden_layers = None - self._mean_transform = core.Dense(num_features * output_window_size, - name="predicted_mean") - self._covariance_transform = core.Dense(num_features * output_window_size, - name="log_sigma_square") - self._prediction_shape = [-1, output_window_size, num_features] - - def call(self, input_window_features, output_window_features): - """Compute predictions from input and output windows. - - Args: - input_window_features: A floating point Tensor with shape [batch size, - input window size, input features]. The batch dimension may not have - static shape information, but the window size and number of input - features are known at graph construction time and recorded in the static - shape information for the `input_window_features` `Tensor`. Note that - `input_window_size` may be zero. - output_window_features: A floating point Tensor with shape [batch size, - output window size, output features]. As with `input_window_features`, - the last two dimensions have static shape information. If there are no - output features, the size of the last dimension will be zero. - Returns: - A dictionary of predictions with keys "mean" and "covariance" (only - diagonal covariances are currently supported). Each has shape - [batch size, output window size, num_features], where num_features is the - same as the constructor argument. - """ - if input_window_features.shape.dims[1].value == 0: - # TODO(allenl): Make reshape()'s static shape information work on - # zero-size Tensors? Currently this special case is required because - # otherwise the Dense layers get unknown last dimensions. - activation = self._output_flatten(output_window_features) - elif output_window_features.shape.dims[2].value == 0: - activation = self._input_flatten(input_window_features) - else: - activation = array_ops.concat( - [self._input_flatten(input_window_features), - self._output_flatten(output_window_features)], - axis=1) - if self._hidden_layers: - activation = self._hidden_layers(activation) - predicted_mean = array_ops.reshape( - self._mean_transform(activation), - self._prediction_shape) - predicted_covariance = array_ops.reshape( - gen_math_ops.exp(self._covariance_transform(activation)), - self._prediction_shape) - return {"mean": predicted_mean, - "covariance": predicted_covariance} - - -class LSTMPredictionModel(training.Model): - """A simple encoder/decoder model using an LSTM. - - This model does not operate on its own, but rather is a plugin to - `ARModel`. See `ARModel`'s constructor documentation - (`prediction_model_factory`) for a usage example. - """ - - def __init__(self, - num_features, - input_window_size, - output_window_size, - num_units=128): - """Construct the LSTM prediction model. - - Args: - num_features: number of input features per time step. - input_window_size: Number of past time steps of data to look at when doing - the regression. - output_window_size: Number of future time steps to predict. Note that - setting it to > 1 empirically seems to give a better fit. - num_units: The number of units in the encoder and decoder LSTM cells. - """ - super(LSTMPredictionModel, self).__init__() - self._encoder = lstm_ops.LSTMBlockFusedCell( - num_units=num_units, name="encoder") - self._decoder = lstm_ops.LSTMBlockFusedCell( - num_units=num_units, name="decoder") - self._mean_transform = core.Dense(num_features, - name="mean_transform") - self._covariance_transform = core.Dense(num_features, - name="covariance_transform") - - def call(self, input_window_features, output_window_features): - """Compute predictions from input and output windows.""" - # Convert to time major - input_window_features = array_ops.transpose(input_window_features, - [1, 0, 2]) - output_window_features = array_ops.transpose(output_window_features, - [1, 0, 2]) - _, encoder_state = self._encoder( - input_window_features, dtype=self.dtype) - decoder_output, _ = self._decoder( - output_window_features, dtype=self.dtype, - initial_state=encoder_state) - - # Switch back to batch major - decoder_output = array_ops.transpose(decoder_output, [1, 0, 2]) - predicted_mean = self._mean_transform(decoder_output) - predicted_covariance = gen_math_ops.exp( - self._covariance_transform(decoder_output)) - return {"mean": predicted_mean, - "covariance": predicted_covariance} - - -class ARModel(model.TimeSeriesModel): - """Auto-regressive model, both linear and non-linear. - - Features to the model include time and values of input_window_size timesteps, - and times for output_window_size timesteps. These are passed through a - configurable prediction model, and then fed to a loss function (e.g. squared - loss). - - Note that this class can also be used to regress against time only by setting - the input_window_size to zero. - - Each periodicity in the `periodicities` arg is divided by the - `num_time_buckets` into time buckets that are represented as features added - to the model. - - A good heuristic for picking an appropriate periodicity for a given data set - would be the length of cycles in the data. For example, energy usage in a - home is typically cyclic each day. If the time feature in a home energy - usage dataset is in the unit of hours, then 24 would be an appropriate - periodicity. Similarly, a good heuristic for `num_time_buckets` is how often - the data is expected to change within the cycle. For the aforementioned home - energy usage dataset and periodicity of 24, then 48 would be a reasonable - value if usage is expected to change every half hour. - - Each feature's value for a given example with time t is the difference - between t and the start of the time bucket it falls under. If it doesn't fall - under a feature's associated time bucket, then that feature's value is zero. - - For example: if `periodicities` = (9, 12) and `num_time_buckets` = 3, then 6 - features would be added to the model, 3 for periodicity 9 and 3 for - periodicity 12. - - For an example data point where t = 17: - - It's in the 3rd time bucket for periodicity 9 (2nd period is 9-18 and 3rd - time bucket is 15-18) - - It's in the 2nd time bucket for periodicity 12 (2nd period is 12-24 and - 2nd time bucket is between 16-20). - - Therefore the 6 added features for this row with t = 17 would be: - - # Feature name (periodicity#_timebucket#), feature value - P9_T1, 0 # not in first time bucket - P9_T2, 0 # not in second time bucket - P9_T3, 2 # 17 - 15 since 15 is the start of the 3rd time bucket - P12_T1, 0 # not in first time bucket - P12_T2, 1 # 17 - 16 since 16 is the start of the 2nd time bucket - P12_T3, 0 # not in third time bucket - """ - SQUARED_LOSS = "squared_loss" - NORMAL_LIKELIHOOD_LOSS = "normal_likelihood_loss" - - def __init__(self, - periodicities, - input_window_size, - output_window_size, - num_features, - prediction_model_factory=FlatPredictionModel, - num_time_buckets=10, - loss=NORMAL_LIKELIHOOD_LOSS, - exogenous_feature_columns=None): - """Constructs an auto-regressive model. - - Args: - periodicities: periodicities of the input data, in the same units as the - time feature (for example 24 if feeding hourly data with a daily - periodicity, or 60 * 24 if feeding minute-level data with daily - periodicity). Note this can be a single value or a list of values for - multiple periodicities. - input_window_size: Number of past time steps of data to look at when doing - the regression. - output_window_size: Number of future time steps to predict. Note that - setting it to > 1 empirically seems to give a better fit. - num_features: number of input features per time step. - prediction_model_factory: A callable taking arguments `num_features`, - `input_window_size`, and `output_window_size` and returning a - `tf.keras.Model`. The `Model`'s `call()` takes two arguments: an input - window and an output window, and returns a dictionary of predictions. - See `FlatPredictionModel` for an example. Example usage: - - ```python model = ar_model.ARModel( periodicities=2, num_features=3, - prediction_model_factory=functools.partial( FlatPredictionModel, - hidden_layer_sizes=[10, 10])) ``` - - The default model computes predictions as a linear function of flattened - input and output windows. - num_time_buckets: Number of buckets into which to divide (time % - periodicity). This value multiplied by the number of periodicities is - the number of time features added to the model. - loss: Loss function to use for training. Currently supported values are - SQUARED_LOSS and NORMAL_LIKELIHOOD_LOSS. Note that for - NORMAL_LIKELIHOOD_LOSS, we train the covariance term as well. For - SQUARED_LOSS, the evaluation loss is reported based on un-scaled - observations and predictions, while the training loss is computed on - normalized data (if input statistics are available). - exogenous_feature_columns: A list of `tf.feature_column`s (for example - `tf.feature_column.embedding_column`) corresponding to - features which provide extra information to the model but are not part - of the series to be predicted. - """ - self._model_factory = prediction_model_factory - self.input_window_size = input_window_size - self.output_window_size = output_window_size - self.window_size = self.input_window_size + self.output_window_size - self.loss = loss - super(ARModel, self).__init__( - num_features=num_features, - exogenous_feature_columns=exogenous_feature_columns) - if exogenous_feature_columns is not None: - self.exogenous_size = self._get_exogenous_embedding_shape()[-1] - else: - self.exogenous_size = 0 - assert num_time_buckets > 0 - self._buckets = int(num_time_buckets) - if periodicities is None or not periodicities: - periodicities = [] - elif (not isinstance(periodicities, list) and - not isinstance(periodicities, tuple)): - periodicities = [periodicities] - self._periodicities = [int(p) for p in periodicities] - for p in self._periodicities: - assert p > 0 - assert len(self._periodicities) or self.input_window_size - assert output_window_size > 0 - - def initialize_graph(self, input_statistics=None): - super(ARModel, self).initialize_graph(input_statistics=input_statistics) - self._model_scope = variable_scope.variable_scope( - # The trailing slash means we strip all enclosing variable_scopes, which - # unfortunately is necessary because the model gets called inside and - # outside a "while" scope (for prediction and training respectively), - # and the variables names need to match. - "model/", use_resource=True) - self._model_instance = self._model_factory( - num_features=self.num_features, - input_window_size=self.input_window_size, - output_window_size=self.output_window_size) - - def get_start_state(self): - # State which matches the format we'll return later. Typically this will not - # be used by the model directly, but the shapes and dtypes should match so - # that the serving input_receiver_fn gets placeholder shapes correct. - return (array_ops.zeros([self.input_window_size], dtype=dtypes.int64), - array_ops.zeros( - [self.input_window_size, self.num_features], dtype=self.dtype), - array_ops.zeros( - [self.input_window_size, self.exogenous_size], - dtype=self.dtype)) - - # TODO(allenl,agarwal): Support sampling for AR. - def random_model_parameters(self, seed=None): - pass - - def generate(self, number_of_series, series_length, - model_parameters=None, seed=None): - pass - - def _predicted_covariance_op(self, activations, num_values): - activation, activation_size = activations[-1] - if self.loss == ARModel.NORMAL_LIKELIHOOD_LOSS: - log_sigma_square = model_utils.fully_connected( - activation, - activation_size, - self.output_window_size * num_values, - name="log_sigma_square", - activation=None) - predicted_covariance = gen_math_ops.exp(log_sigma_square) - predicted_covariance = array_ops.reshape( - predicted_covariance, [-1, self.output_window_size, num_values]) - else: - shape = array_ops.stack([ - array_ops.shape(activation)[0], - constant_op.constant(self.output_window_size), - constant_op.constant(num_values) - ]) - predicted_covariance = array_ops.ones(shape=shape, dtype=activation.dtype) - return predicted_covariance - - def _predicted_mean_op(self, activations): - activation, activation_size = activations[-1] - predicted_mean = model_utils.fully_connected( - activation, - activation_size, - self.output_window_size * self.num_features, - name="predicted_mean", - activation=None) - return array_ops.reshape(predicted_mean, - [-1, self.output_window_size, self.num_features]) - - def prediction_ops(self, times, values, exogenous_regressors): - """Compute model predictions given input data. - - Args: - times: A [batch size, self.window_size] integer Tensor, the first - self.input_window_size times in each part of the batch indicating - input features, and the last self.output_window_size times indicating - prediction times. - values: A [batch size, self.input_window_size, self.num_features] Tensor - with input features. - exogenous_regressors: A [batch size, self.window_size, - self.exogenous_size] Tensor with exogenous features. - Returns: - Tuple (predicted_mean, predicted_covariance), where each element is a - Tensor with shape [batch size, self.output_window_size, - self.num_features]. - """ - times.get_shape().assert_is_compatible_with([None, self.window_size]) - batch_size = array_ops.shape(times)[0] - if self.input_window_size: - values.get_shape().assert_is_compatible_with( - [None, self.input_window_size, self.num_features]) - if exogenous_regressors is not None: - exogenous_regressors.get_shape().assert_is_compatible_with( - [None, self.window_size, self.exogenous_size]) - # Create input features. - input_window_features = [] - input_feature_size = 0 - output_window_features = [] - output_feature_size = 0 - if self._periodicities: - _, time_features = self._compute_time_features(times) - num_time_features = self._buckets * len(self._periodicities) - time_features = array_ops.reshape( - time_features, - [batch_size, - self.window_size, - num_time_features]) - input_time_features, output_time_features = array_ops.split( - time_features, (self.input_window_size, self.output_window_size), - axis=1) - input_feature_size += num_time_features - output_feature_size += num_time_features - input_window_features.append(input_time_features) - output_window_features.append(output_time_features) - if self.input_window_size: - inp = array_ops.slice(values, [0, 0, 0], [-1, self.input_window_size, -1]) - input_window_features.append( - array_ops.reshape( - inp, - [batch_size, self.input_window_size, self.num_features])) - input_feature_size += self.num_features - if self.exogenous_size: - input_exogenous_features, output_exogenous_features = array_ops.split( - exogenous_regressors, - (self.input_window_size, self.output_window_size), - axis=1) - input_feature_size += self.exogenous_size - output_feature_size += self.exogenous_size - input_window_features.append(input_exogenous_features) - output_window_features.append(output_exogenous_features) - assert input_window_features - input_window_features = array_ops.concat(input_window_features, axis=2) - if output_window_features: - output_window_features = array_ops.concat(output_window_features, axis=2) - else: - output_window_features = array_ops.zeros( - [batch_size, self.output_window_size, 0], - dtype=self.dtype) - static_batch_size = times.get_shape().dims[0].value - input_window_features.set_shape( - [static_batch_size, self.input_window_size, input_feature_size]) - output_window_features.set_shape( - [static_batch_size, self.output_window_size, output_feature_size]) - return self._output_window_predictions(input_window_features, - output_window_features) - - def _output_window_predictions( - self, input_window_features, output_window_features): - with self._model_scope: - predictions = self._model_instance( - input_window_features, output_window_features) - result_shape = [None, self.output_window_size, self.num_features] - for v in predictions.values(): - v.set_shape(result_shape) - return predictions - - def loss_op(self, targets, prediction_ops): - """Create loss_op.""" - prediction = prediction_ops["mean"] - if self.loss == ARModel.NORMAL_LIKELIHOOD_LOSS: - covariance = prediction_ops["covariance"] - sigma = math_ops.sqrt(gen_math_ops.maximum(covariance, 1e-5)) - loss_op = -math_ops.reduce_sum( - math_utils.normal_log_prob(targets, sigma, prediction)) - else: - assert self.loss == ARModel.SQUARED_LOSS, self.loss - loss_op = math_ops.reduce_sum( - math_ops.squared_difference(prediction, targets)) - loss_op /= math_ops.cast( - math_ops.reduce_prod(array_ops.shape(targets)), loss_op.dtype) - return loss_op - - def _process_exogenous_features(self, times, features): - embedded = super(ARModel, self)._process_exogenous_features( - times=times, features=features) - if embedded is None: - assert self.exogenous_size == 0 - # No embeddings. Return a zero-size [batch, times, 0] array so we don't - # have to special case it downstream. - return array_ops.zeros( - array_ops.concat([array_ops.shape(times), constant_op.constant([0])], - axis=0)) - else: - return embedded - - # TODO(allenl, agarwal): Consider better ways of warm-starting predictions. - def predict(self, features): - """Computes predictions multiple steps into the future. - - Args: - features: A dictionary with the following key/value pairs: - PredictionFeatures.TIMES: A [batch size, predict window size] - integer Tensor of times, after the window of data indicated by - `STATE_TUPLE`, to make predictions for. - PredictionFeatures.STATE_TUPLE: A tuple of (times, values), times with - shape [batch size, self.input_window_size], values with shape [batch - size, self.input_window_size, self.num_features] representing a - segment of the time series before `TIMES`. This data is used - to start of the autoregressive computation. This should have data for - at least self.input_window_size timesteps. - And any exogenous features, with shapes prefixed by shape of `TIMES`. - Returns: - A dictionary with keys, "mean", "covariance". The - values are Tensors of shape [batch_size, predict window size, - num_features] and correspond to the values passed in `TIMES`. - """ - if not self._graph_initialized: - self.initialize_graph() - predict_times = math_ops.cast( - ops.convert_to_tensor(features[PredictionFeatures.TIMES]), dtypes.int32) - exogenous_regressors = self._process_exogenous_features( - times=predict_times, - features={key: value for key, value in features.items() - if key not in [TrainEvalFeatures.TIMES, - TrainEvalFeatures.VALUES, - PredictionFeatures.STATE_TUPLE]}) - with ops.control_dependencies( - [check_ops.assert_equal(array_ops.shape(predict_times)[1], - array_ops.shape(exogenous_regressors)[1])]): - exogenous_regressors = array_ops.identity(exogenous_regressors) - batch_size = array_ops.shape(predict_times)[0] - num_predict_values = array_ops.shape(predict_times)[1] - prediction_iterations = ((num_predict_values + self.output_window_size - 1) - // self.output_window_size) - # Pad predict_times and exogenous regressors so as to have exact multiple of - # self.output_window_size values per example. - padding_size = (prediction_iterations * self.output_window_size - - num_predict_values) - predict_times = array_ops.pad( - predict_times, [[0, 0], [0, padding_size]]) - exogenous_regressors = array_ops.pad( - exogenous_regressors, [[0, 0], [0, padding_size], [0, 0]]) - state = features[PredictionFeatures.STATE_TUPLE] - (state_times, state_values, state_exogenous_regressors) = state - state_times = math_ops.cast( - ops.convert_to_tensor(state_times), dtypes.int32) - state_values = ops.convert_to_tensor(state_values, dtype=self.dtype) - state_exogenous_regressors = ops.convert_to_tensor( - state_exogenous_regressors, dtype=self.dtype) - - initial_input_times = predict_times[:, :self.output_window_size] - initial_input_exogenous_regressors = ( - exogenous_regressors[:, :self.output_window_size, :]) - if self.input_window_size > 0: - initial_input_times = array_ops.concat( - [state_times[:, -self.input_window_size:], initial_input_times], 1) - values_size = array_ops.shape(state_values)[1] - times_size = array_ops.shape(state_times)[1] - with ops.control_dependencies([ - check_ops.assert_greater_equal(values_size, self.input_window_size), - check_ops.assert_equal(values_size, times_size) - ]): - initial_input_values = state_values[:, -self.input_window_size:, :] - initial_input_exogenous_regressors = array_ops.concat( - [state_exogenous_regressors[:, -self.input_window_size:, :], - initial_input_exogenous_regressors[ - :, :self.output_window_size, :]], - axis=1) - else: - initial_input_values = 0 - - # Iterate over the predict_times, predicting self.output_window_size values - # in each iteration. - def _while_condition(iteration_number, *unused_args): - return math_ops.less(iteration_number, prediction_iterations) - - def _while_body(iteration_number, input_times, input_values, - input_exogenous_regressors, mean_ta, covariance_ta): - """Predict self.output_window_size values.""" - prediction_ops = self.prediction_ops( - input_times, input_values, input_exogenous_regressors) - predicted_mean = prediction_ops["mean"] - predicted_covariance = prediction_ops["covariance"] - offset = self.output_window_size * gen_math_ops.minimum( - iteration_number + 1, prediction_iterations - 1) - if self.input_window_size > 0: - if self.output_window_size < self.input_window_size: - new_input_values = array_ops.concat( - [input_values[:, self.output_window_size:, :], predicted_mean], 1) - new_input_exogenous_regressors = array_ops.concat( - [input_exogenous_regressors[:, -self.input_window_size:, :], - exogenous_regressors[ - :, offset:offset + self.output_window_size, :]], - axis=1) - new_input_times = array_ops.concat([ - input_times[:, -self.input_window_size:], - predict_times[:, offset:offset + self.output_window_size] - ], 1) - else: - new_input_values = predicted_mean[:, -self.input_window_size:, :] - new_input_exogenous_regressors = exogenous_regressors[ - :, - offset - self.input_window_size:offset + self.output_window_size, - :] - new_input_times = predict_times[ - :, - offset - self.input_window_size:offset + self.output_window_size] - else: - new_input_values = input_values - new_input_exogenous_regressors = exogenous_regressors[ - :, offset:offset + self.output_window_size, :] - new_input_times = predict_times[:, - offset:offset + self.output_window_size] - new_input_times.set_shape(initial_input_times.get_shape()) - new_input_exogenous_regressors.set_shape( - initial_input_exogenous_regressors.get_shape()) - new_mean_ta = mean_ta.write(iteration_number, predicted_mean) - if isinstance(covariance_ta, tensor_array_ops.TensorArray): - new_covariance_ta = covariance_ta.write(iteration_number, - predicted_covariance) - else: - new_covariance_ta = covariance_ta - return (iteration_number + 1, - new_input_times, - new_input_values, - new_input_exogenous_regressors, - new_mean_ta, - new_covariance_ta) - - # Note that control_flow_ops.while_loop doesn't seem happy with None. Hence - # using 0 for cases where we don't want to predict covariance. - covariance_ta_init = (tensor_array_ops.TensorArray( - dtype=self.dtype, size=prediction_iterations) - if self.loss != ARModel.SQUARED_LOSS else 0.) - mean_ta_init = tensor_array_ops.TensorArray( - dtype=self.dtype, size=prediction_iterations) - _, _, _, _, mean_ta, covariance_ta = control_flow_ops.while_loop( - _while_condition, _while_body, [ - 0, - initial_input_times, - initial_input_values, - initial_input_exogenous_regressors, - mean_ta_init, - covariance_ta_init - ]) - - def _parse_ta(values_ta): - """Helper function to parse the returned TensorArrays.""" - - if not isinstance(values_ta, tensor_array_ops.TensorArray): - return None - predictions_length = prediction_iterations * self.output_window_size - # Shape [prediction_iterations, batch_size, self.output_window_size, - # self.num_features] - values_packed = values_ta.stack() - # Transpose to move batch dimension outside. - output_values = array_ops.reshape( - array_ops.transpose(values_packed, [1, 0, 2, 3]), - array_ops.stack([batch_size, predictions_length, -1])) - # Clip to desired size - return output_values[:, :num_predict_values, :] - - predicted_mean = _parse_ta(mean_ta) - predicted_covariance = _parse_ta(covariance_ta) - if predicted_covariance is None: - predicted_covariance = array_ops.ones_like(predicted_mean) - - # Transform and scale the mean and covariance appropriately. - predicted_mean = self._scale_back_data(predicted_mean) - predicted_covariance = self._scale_back_variance(predicted_covariance) - - return {"mean": predicted_mean, - "covariance": predicted_covariance} - - def _process_window(self, features, mode, exogenous_regressors): - """Compute model outputs on a single window of data.""" - times = math_ops.cast(features[TrainEvalFeatures.TIMES], dtypes.int64) - values = math_ops.cast(features[TrainEvalFeatures.VALUES], dtype=self.dtype) - exogenous_regressors = math_ops.cast(exogenous_regressors, dtype=self.dtype) - original_values = values - - # Extra shape checking for the window size (above that in - # `head.create_estimator_spec`). - expected_times_shape = [None, self.window_size] - if not times.get_shape().is_compatible_with(expected_times_shape): - raise ValueError( - ("ARModel with input_window_size={input_window_size} " - "and output_window_size={output_window_size} expects " - "feature '{times_feature}' to have shape (batch_size, " - "{window_size}) (for any batch_size), but got shape {times_shape}. " - "If you are using RandomWindowInputFn, set " - "window_size={window_size} or adjust the input_window_size and " - "output_window_size arguments to ARModel.").format( - input_window_size=self.input_window_size, - output_window_size=self.output_window_size, - times_feature=TrainEvalFeatures.TIMES, - window_size=self.window_size, - times_shape=times.get_shape())) - values = self._scale_data(values) - if self.input_window_size > 0: - input_values = values[:, :self.input_window_size, :] - else: - input_values = None - prediction_ops = self.prediction_ops( - times, input_values, exogenous_regressors) - prediction = prediction_ops["mean"] - covariance = prediction_ops["covariance"] - targets = array_ops.slice(values, [0, self.input_window_size, 0], - [-1, -1, -1]) - targets.get_shape().assert_is_compatible_with(prediction.get_shape()) - if (mode == estimator_lib.ModeKeys.EVAL - and self.loss == ARModel.SQUARED_LOSS): - # Report an evaluation loss which matches the expected - # (observed - predicted) ** 2. - # Note that this affects only evaluation; the training loss is unaffected. - loss = self.loss_op( - self._scale_back_data(targets), - {"mean": self._scale_back_data(prediction_ops["mean"])}) - else: - loss = self.loss_op(targets, prediction_ops) - - # Scale back the prediction. - prediction = self._scale_back_data(prediction) - covariance = self._scale_back_variance(covariance) - - return model.ModelOutputs( - loss=loss, - end_state=(times[:, -self.input_window_size:], - values[:, -self.input_window_size:, :], - exogenous_regressors[:, -self.input_window_size:, :]), - predictions={"mean": prediction, "covariance": covariance, - "observed": original_values[:, -self.output_window_size:]}, - prediction_times=times[:, -self.output_window_size:]) - - def get_batch_loss(self, features, mode, state): - """Computes predictions and a loss. - - Args: - features: A dictionary (such as is produced by a chunker) with the - following key/value pairs (shapes are given as required for training): - TrainEvalFeatures.TIMES: A [batch size, self.window_size] integer - Tensor with times for each observation. To train on longer - sequences, the data should first be chunked. - TrainEvalFeatures.VALUES: A [batch size, self.window_size, - self.num_features] Tensor with values for each observation. - When evaluating, `TIMES` and `VALUES` must have a window size of at - least self.window_size, but it may be longer, in which case the last - window_size - self.input_window_size times (or fewer if this is not - divisible by self.output_window_size) will be evaluated on with - non-overlapping output windows (and will have associated - predictions). This is primarily to support qualitative - evaluation/plotting, and is not a recommended way to compute evaluation - losses (since there is no overlap in the output windows, which for - window-based models is an undesirable bias). - mode: The tf.estimator.ModeKeys mode to use (TRAIN or EVAL). - state: Unused - Returns: - A model.ModelOutputs object. - Raises: - ValueError: If `mode` is not TRAIN or EVAL, or if static shape information - is incorrect. - """ - features = {feature_name: ops.convert_to_tensor(feature_value) - for feature_name, feature_value in features.items()} - times = features[TrainEvalFeatures.TIMES] - exogenous_regressors = self._process_exogenous_features( - times=times, - features={key: value for key, value in features.items() - if key not in [TrainEvalFeatures.TIMES, - TrainEvalFeatures.VALUES, - PredictionFeatures.STATE_TUPLE]}) - if mode == estimator_lib.ModeKeys.TRAIN: - # For training, we require the window size to be self.window_size as - # iterating sequentially on larger windows could introduce a bias. - return self._process_window( - features, mode=mode, exogenous_regressors=exogenous_regressors) - elif mode == estimator_lib.ModeKeys.EVAL: - # For evaluation, we allow the user to pass in a larger window, in which - # case we try to cover as much of the window as possible without - # overlap. Quantitative evaluation is more efficient/correct with fixed - # windows matching self.window_size (as with training), but this looping - # allows easy plotting of "in-sample" predictions. - times.get_shape().assert_has_rank(2) - static_window_size = times.get_shape().dims[1].value - if (static_window_size is not None - and static_window_size < self.window_size): - raise ValueError( - ("ARModel requires a window of at least input_window_size + " - "output_window_size to evaluate on (input_window_size={}, " - "output_window_size={}, and got shape {} for feature '{}' (batch " - "size, window size)).").format( - self.input_window_size, self.output_window_size, - times.get_shape(), TrainEvalFeatures.TIMES)) - num_iterations = ((array_ops.shape(times)[1] - self.input_window_size) - // self.output_window_size) - output_size = num_iterations * self.output_window_size - # Rather than dealing with overlapping windows of output, discard a bit at - # the beginning if output windows don't cover evenly. - crop_length = output_size + self.input_window_size - features = {feature_name: feature_value[:, -crop_length:] - for feature_name, feature_value in features.items()} - # Note that, unlike the ARModel's predict() while_loop and the - # SequentialTimeSeriesModel while_loop, each iteration here can run in - # parallel, since we are not feeding predictions or state from previous - # iterations. - def _while_condition(iteration_number, loss_ta, mean_ta, covariance_ta): - del loss_ta, mean_ta, covariance_ta # unused - return iteration_number < num_iterations - - def _while_body(iteration_number, loss_ta, mean_ta, covariance_ta): - """Perform a processing step on a single window of data.""" - base_offset = iteration_number * self.output_window_size - model_outputs = self._process_window( - features={ - feature_name: - feature_value[:, base_offset:base_offset + self.window_size] - for feature_name, feature_value in features.items()}, - mode=mode, - exogenous_regressors=exogenous_regressors[ - :, base_offset:base_offset + self.window_size]) - # This code needs to be updated if new predictions are added in - # self._process_window - assert len(model_outputs.predictions) == 3 - assert "mean" in model_outputs.predictions - assert "covariance" in model_outputs.predictions - assert "observed" in model_outputs.predictions - return (iteration_number + 1, - loss_ta.write( - iteration_number, model_outputs.loss), - mean_ta.write( - iteration_number, model_outputs.predictions["mean"]), - covariance_ta.write( - iteration_number, model_outputs.predictions["covariance"])) - _, loss_ta, mean_ta, covariance_ta = control_flow_ops.while_loop( - _while_condition, _while_body, - [0, - tensor_array_ops.TensorArray(dtype=self.dtype, size=num_iterations), - tensor_array_ops.TensorArray(dtype=self.dtype, size=num_iterations), - tensor_array_ops.TensorArray(dtype=self.dtype, size=num_iterations)]) - values = math_ops.cast(features[TrainEvalFeatures.VALUES], - dtype=self.dtype) - batch_size = array_ops.shape(times)[0] - prediction_shape = [batch_size, self.output_window_size * num_iterations, - self.num_features] - (previous_state_times, - previous_state_values, - previous_state_exogenous_regressors) = state - # Make sure returned state always has windows of self.input_window_size, - # even if we were passed fewer than self.input_window_size points this - # time. - if self.input_window_size > 0: - new_state_times = array_ops.concat( - [previous_state_times, - math_ops.cast(times, dtype=dtypes.int64)], - axis=1)[:, -self.input_window_size:] - new_state_times.set_shape((None, self.input_window_size)) - new_state_values = array_ops.concat( - [previous_state_values, - self._scale_data(values)], axis=1)[:, -self.input_window_size:, :] - new_state_values.set_shape((None, self.input_window_size, - self.num_features)) - new_exogenous_regressors = array_ops.concat( - [previous_state_exogenous_regressors, - exogenous_regressors], axis=1)[:, -self.input_window_size:, :] - new_exogenous_regressors.set_shape( - (None, - self.input_window_size, - self.exogenous_size)) - else: - # There is no state to keep, and the strided slices above do not handle - # input_window_size=0. - new_state_times = previous_state_times - new_state_values = previous_state_values - new_exogenous_regressors = previous_state_exogenous_regressors - return model.ModelOutputs( - loss=math_ops.reduce_mean(loss_ta.stack(), axis=0), - end_state=(new_state_times, - new_state_values, - new_exogenous_regressors), - predictions={ - "mean": array_ops.reshape( - array_ops.transpose(mean_ta.stack(), [1, 0, 2, 3]), - prediction_shape), - "covariance": array_ops.reshape( - array_ops.transpose(covariance_ta.stack(), [1, 0, 2, 3]), - prediction_shape), - "observed": values[:, -output_size:]}, - prediction_times=times[:, -output_size:]) - else: - raise ValueError( - "Unknown mode '{}' passed to get_batch_loss.".format(mode)) - - def _compute_time_features(self, time): - """Compute some features on the time value.""" - batch_size = array_ops.shape(time)[0] - num_periods = len(self._periodicities) - # Reshape to 3D. - periods = constant_op.constant( - self._periodicities, shape=[1, 1, num_periods, 1], dtype=time.dtype) - time = array_ops.reshape(time, [batch_size, -1, 1, 1]) - window_offset = time / self._periodicities - # Cast to appropriate type and scale to [0, 1) range - mod = (math_ops.cast(time % periods, self.dtype) * self._buckets / - math_ops.cast(periods, self.dtype)) - # Bucketize based on some fixed width intervals. For a value t and interval - # [a, b), we return (t - a) if a <= t < b, else 0. - intervals = array_ops.reshape( - math_ops.range(self._buckets, dtype=self.dtype), - [1, 1, 1, self._buckets]) - mod = nn_ops.relu(mod - intervals) - mod = array_ops.where_v2(mod < 1.0, mod, array_ops.zeros_like(mod)) - return window_offset, mod - - -class AnomalyMixtureARModel(ARModel): - """Model data as a mixture of normal and anomaly distributions. - - Note that this model works by changing the loss function to reduce the penalty - when predicting an anomalous target. However the predictions are still based - on anomalous input features, and this may affect the quality of fit. One - possible solution is to downweight/filter anomalous inputs, but that requires - more sequential processing instead of completely random windows. - """ - - GAUSSIAN_ANOMALY = "gaussian" - CAUCHY_ANOMALY = "cauchy" - - def __init__(self, - periodicities, - anomaly_prior_probability, - input_window_size, - output_window_size, - num_features, - prediction_model_factory=FlatPredictionModel, - anomaly_distribution=GAUSSIAN_ANOMALY, - num_time_buckets=10, - exogenous_feature_columns=None): - assert (anomaly_prior_probability < 1.0 and - anomaly_prior_probability > 0.0) - self._anomaly_prior_probability = anomaly_prior_probability - assert anomaly_distribution in [ - AnomalyMixtureARModel.GAUSSIAN_ANOMALY, - AnomalyMixtureARModel.CAUCHY_ANOMALY] - self._anomaly_distribution = anomaly_distribution - super(AnomalyMixtureARModel, self).__init__( - periodicities=periodicities, - num_features=num_features, - num_time_buckets=num_time_buckets, - input_window_size=input_window_size, - output_window_size=output_window_size, - loss=ARModel.NORMAL_LIKELIHOOD_LOSS, - prediction_model_factory=prediction_model_factory, - exogenous_feature_columns=exogenous_feature_columns) - - def _create_anomaly_ops(self, times, values, prediction_ops_dict): - anomaly_log_param = variable_scope.get_variable( - "anomaly_log_param", - shape=[], - dtype=self.dtype, - initializer=init_ops.zeros_initializer()) - # Anomaly param is the variance for Gaussian and scale for Cauchy - # distribution. - prediction_ops_dict["anomaly_params"] = gen_math_ops.exp(anomaly_log_param) - - def prediction_ops(self, times, values, exogenous_regressors): - prediction_ops_dict = super(AnomalyMixtureARModel, self).prediction_ops( - times, values, exogenous_regressors) - self._create_anomaly_ops(times, values, prediction_ops_dict) - return prediction_ops_dict - - def _anomaly_log_prob(self, targets, prediction_ops): - prediction = prediction_ops["mean"] - if self._anomaly_distribution == AnomalyMixtureARModel.GAUSSIAN_ANOMALY: - anomaly_variance = prediction_ops["anomaly_params"] - anomaly_sigma = math_ops.sqrt( - gen_math_ops.maximum(anomaly_variance, 1e-5)) - log_prob = math_utils.normal_log_prob(targets, anomaly_sigma, prediction) - else: - assert self._anomaly_distribution == AnomalyMixtureARModel.CAUCHY_ANOMALY - anomaly_scale = prediction_ops["anomaly_params"] - log_prob = math_utils.cauchy_log_prob(targets, anomaly_scale, prediction) - return log_prob - - def loss_op(self, targets, prediction_ops): - """Create loss_op.""" - prediction = prediction_ops["mean"] - covariance = prediction_ops["covariance"] - # Normal data log probability. - sigma = math_ops.sqrt(gen_math_ops.maximum(covariance, 1e-5)) - log_prob1 = math_utils.normal_log_prob(targets, sigma, prediction) - log_prob1 += math_ops.log(1 - self._anomaly_prior_probability) - # Anomaly log probability. - log_prob2 = self._anomaly_log_prob(targets, prediction_ops) - log_prob2 += math_ops.log(self._anomaly_prior_probability) - # We need to compute log(exp(log_prob1) + exp(log_prob2). For numerical - # stability, we rewrite the expression as below. - p1 = gen_math_ops.minimum(log_prob1, log_prob2) - p2 = gen_math_ops.maximum(log_prob1, log_prob2) - mixed_log_prob = p2 + math_ops.log(1 + gen_math_ops.exp(p1 - p2)) - loss_op = -math_ops.reduce_sum(mixed_log_prob) - loss_op /= math_ops.cast( - math_ops.reduce_prod(array_ops.shape(targets)), self.dtype) - return loss_op diff --git a/tensorflow/contrib/timeseries/python/timeseries/ar_model_test.py b/tensorflow/contrib/timeseries/python/timeseries/ar_model_test.py deleted file mode 100644 index de547f835d3..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/ar_model_test.py +++ /dev/null @@ -1,365 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for ar_model.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -import numpy as np - -from tensorflow.contrib.timeseries.python.timeseries import ar_model -from tensorflow.contrib.timeseries.python.timeseries import input_pipeline -from tensorflow.contrib.timeseries.python.timeseries import test_utils -from tensorflow.contrib.timeseries.python.timeseries.estimators import ARRegressor -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import PredictionFeatures -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import TrainEvalFeatures - -from tensorflow.python.client import session -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import coordinator as coordinator_lib -from tensorflow.python.training import queue_runner_impl -from tensorflow.python.training import training - - -class ARModelTest(test.TestCase): - - def create_data(self, - noise_stddev, - anomaly_prob, - multiple_periods=False, - anomaly_stddev_scale=20): - self.period = 25 - num_samples = 200 - time = 1 + 3 * np.arange(num_samples).astype(np.int64) - time_offset = (2 * np.pi * (time % self.period).astype(np.float) / - self.period).reshape([-1, 1]) - if multiple_periods: - period2 = 55 - self.period = [self.period, period2] - time_offset2 = ((time % period2).astype(np.float) / period2).reshape( - [-1, 1]) - data1 = np.sin(time_offset / 2.0) ** 2 * (1 + time_offset2) - else: - data1 = np.sin(2 * time_offset) + np.cos(3 * time_offset) - data1 += noise_stddev / 4. * np.random.randn(num_samples, 1) - data2 = (np.sin(3 * time_offset) + np.cos(5 * time_offset) + - noise_stddev / 3. * np.random.randn(num_samples, 1)) - # Add some anomalies to data1 - if anomaly_prob > 0.: - num_anomalies = int(anomaly_prob * num_samples) - anomaly_values = (anomaly_stddev_scale * noise_stddev / 4 * - np.random.randn(num_anomalies)) - indices = np.random.randint(0, num_samples, num_anomalies) - for index, val in zip(indices, anomaly_values): - data1[index] += val - - data = np.concatenate((4 * data1, 3 * data2), axis=1) - split = int(num_samples * 0.8) - train_data = {TrainEvalFeatures.TIMES: time[0:split], - TrainEvalFeatures.VALUES: data[0:split]} - test_data = {TrainEvalFeatures.TIMES: time[split:], - TrainEvalFeatures.VALUES: data[split:]} - return (train_data, test_data) - - # Note that most models will require many more steps to fully converge. We - # have used a small number of steps here to keep the running time small. - def train_helper(self, input_window_size, loss, - max_loss=None, train_steps=200, - anomaly_prob=0.01, - anomaly_distribution=None, - multiple_periods=False): - np.random.seed(3) - data_noise_stddev = 0.2 - if max_loss is None: - if loss == ar_model.ARModel.NORMAL_LIKELIHOOD_LOSS: - max_loss = 1.0 - else: - max_loss = 0.05 / (data_noise_stddev ** 2) - train_data, test_data = self.create_data( - noise_stddev=data_noise_stddev, - anomaly_prob=anomaly_prob, - multiple_periods=multiple_periods) - output_window_size = 10 - window_size = input_window_size + output_window_size - - class _RunConfig(estimator_lib.RunConfig): - - @property - def tf_random_seed(self): - return 3 - - estimator = ARRegressor( - periodicities=self.period, - anomaly_prior_probability=0.01 if anomaly_distribution else None, - anomaly_distribution=anomaly_distribution, - num_features=2, - output_window_size=output_window_size, - num_time_buckets=20, - input_window_size=input_window_size, - hidden_layer_sizes=[16], - loss=loss, - config=_RunConfig()) - train_input_fn = input_pipeline.RandomWindowInputFn( - time_series_reader=input_pipeline.NumpyReader(train_data), - window_size=window_size, - batch_size=64, - num_threads=1, - shuffle_seed=2) - test_input_fn = test_utils.AllWindowInputFn( - time_series_reader=input_pipeline.NumpyReader(test_data), - window_size=window_size) - - # Test training - estimator.train( - input_fn=train_input_fn, - steps=train_steps) - test_evaluation = estimator.evaluate(input_fn=test_input_fn, steps=1) - test_loss = test_evaluation["loss"] - logging.info("Final test loss: %f", test_loss) - self.assertLess(test_loss, max_loss) - if loss == ar_model.ARModel.SQUARED_LOSS: - # Test that the evaluation loss is reported without input scaling. - self.assertAllClose( - test_loss, - np.mean((test_evaluation["mean"] - test_evaluation["observed"]) ** 2)) - - # Test predict - train_data_times = train_data[TrainEvalFeatures.TIMES] - train_data_values = train_data[TrainEvalFeatures.VALUES] - test_data_times = test_data[TrainEvalFeatures.TIMES] - test_data_values = test_data[TrainEvalFeatures.VALUES] - predict_times = np.expand_dims(np.concatenate( - [train_data_times[input_window_size:], test_data_times]), 0) - predict_true_values = np.expand_dims(np.concatenate( - [train_data_values[input_window_size:], test_data_values]), 0) - state_times = np.expand_dims(train_data_times[:input_window_size], 0) - state_values = np.expand_dims( - train_data_values[:input_window_size, :], 0) - state_exogenous = state_times[:, :, None][:, :, :0] - - def prediction_input_fn(): - return ({ - PredictionFeatures.TIMES: training.limit_epochs( - predict_times, num_epochs=1), - PredictionFeatures.STATE_TUPLE: (state_times, - state_values, - state_exogenous) - }, {}) - (predictions,) = tuple(estimator.predict(input_fn=prediction_input_fn)) - predicted_mean = predictions["mean"][:, 0] - true_values = predict_true_values[0, :, 0] - - if loss == ar_model.ARModel.NORMAL_LIKELIHOOD_LOSS: - variances = predictions["covariance"][:, 0] - standard_deviations = np.sqrt(variances) - # Note that we may get tighter bounds with more training steps. - errors = np.abs(predicted_mean - true_values) > 4 * standard_deviations - fraction_errors = np.mean(errors) - logging.info("Fraction errors: %f", fraction_errors) - - def test_time_regression_squared(self): - self.train_helper(input_window_size=0, - train_steps=350, - loss=ar_model.ARModel.SQUARED_LOSS) - - def test_autoregression_squared(self): - self.train_helper(input_window_size=15, - loss=ar_model.ARModel.SQUARED_LOSS) - - def test_autoregression_short_input_window(self): - self.train_helper(input_window_size=8, - loss=ar_model.ARModel.SQUARED_LOSS) - - def test_autoregression_normal(self): - self.train_helper(input_window_size=10, - loss=ar_model.ARModel.NORMAL_LIKELIHOOD_LOSS, - train_steps=300, - max_loss=50., # Just make sure there are no exceptions. - anomaly_distribution=None) - - def test_autoregression_normal_multiple_periods(self): - self.train_helper(input_window_size=10, - loss=ar_model.ARModel.NORMAL_LIKELIHOOD_LOSS, - max_loss=2.0, - multiple_periods=True, - anomaly_distribution=None) - - def test_autoregression_normal_anomalies_normal(self): - self.train_helper( - input_window_size=10, - loss=ar_model.ARModel.NORMAL_LIKELIHOOD_LOSS, - anomaly_distribution=ar_model.AnomalyMixtureARModel.GAUSSIAN_ANOMALY) - - def test_autoregression_normal_anomalies_cauchy(self): - self.train_helper( - input_window_size=10, - max_loss=1.5, - loss=ar_model.ARModel.NORMAL_LIKELIHOOD_LOSS, - anomaly_distribution=ar_model.AnomalyMixtureARModel.CAUCHY_ANOMALY) - - def test_wrong_window_size(self): - estimator = ARRegressor( - periodicities=10, num_features=1, - input_window_size=10, output_window_size=6) - def _bad_window_size_input_fn(): - return ({TrainEvalFeatures.TIMES: [[1]], - TrainEvalFeatures.VALUES: [[[1.]]]}, - None) - def _good_data(): - return ({TrainEvalFeatures.TIMES: np.arange(16)[None, :], - TrainEvalFeatures.VALUES: array_ops.reshape( - np.arange(16), [1, 16, 1])}, - None) - with self.assertRaisesRegexp(ValueError, "set window_size=16"): - estimator.train(input_fn=_bad_window_size_input_fn, steps=1) - # Get a checkpoint for evaluation - estimator.train(input_fn=_good_data, steps=1) - with self.assertRaisesRegexp(ValueError, "requires a window of at least"): - estimator.evaluate(input_fn=_bad_window_size_input_fn, steps=1) - - def test_predictions_direct_flat(self): - g = ops.Graph() - with g.as_default(): - model = ar_model.ARModel(periodicities=2, - num_features=1, - num_time_buckets=10, - input_window_size=2, - output_window_size=2, - prediction_model_factory=functools.partial( - ar_model.FlatPredictionModel, - hidden_layer_sizes=[40, 10])) - with session.Session(): - predicted_values = model.predict({ - PredictionFeatures.TIMES: [[4, 6, 10]], - PredictionFeatures.STATE_TUPLE: ( - [[1, 2]], [[[1.], [2.]]], [[[], []]]) - }) - variables.global_variables_initializer().run() - self.assertAllEqual(predicted_values["mean"].eval().shape, - [1, 3, 1]) - - def test_predictions_direct_lstm(self): - g = ops.Graph() - with g.as_default(): - model = ar_model.ARModel(periodicities=2, - num_features=1, - num_time_buckets=10, - input_window_size=2, - output_window_size=2, - prediction_model_factory=functools.partial( - ar_model.LSTMPredictionModel, - num_units=16)) - with session.Session(): - predicted_values = model.predict({ - PredictionFeatures.TIMES: [[4, 6, 10]], - PredictionFeatures.STATE_TUPLE: ( - [[1, 2]], [[[1.], [2.]]], [[[], []]]) - }) - variables.global_variables_initializer().run() - self.assertAllEqual(predicted_values["mean"].eval().shape, - [1, 3, 1]) - - def test_long_eval(self): - g = ops.Graph() - with g.as_default(): - model = ar_model.ARModel(periodicities=2, - num_features=1, - num_time_buckets=10, - input_window_size=2, - output_window_size=1) - raw_features = { - TrainEvalFeatures.TIMES: [[1, 3, 5, 7, 11]], - TrainEvalFeatures.VALUES: [[[1.], [2.], [3.], [4.], [5.]]]} - chunked_features, _ = test_utils.AllWindowInputFn( - time_series_reader=input_pipeline.NumpyReader(raw_features), - window_size=3)() - model.initialize_graph() - with variable_scope.variable_scope("armodel") as scope: - raw_evaluation = model.define_loss( - raw_features, mode=estimator_lib.ModeKeys.EVAL) - with variable_scope.variable_scope(scope, reuse=True): - chunked_evaluation = model.define_loss( - chunked_features, mode=estimator_lib.ModeKeys.EVAL) - with session.Session() as sess: - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(sess, coord=coordinator) - variables.global_variables_initializer().run() - raw_evaluation_evaled, chunked_evaluation_evaled = sess.run( - [raw_evaluation, chunked_evaluation]) - self.assertAllClose(chunked_evaluation_evaled.loss, - raw_evaluation_evaled.loss) - last_chunk_evaluation_state = [ - state[-1, None] for state in - chunked_evaluation_evaled.end_state] - for last_chunk_state_member, raw_state_member in zip( - last_chunk_evaluation_state, raw_evaluation_evaled.end_state): - self.assertAllClose(last_chunk_state_member, raw_state_member) - self.assertAllEqual([[5, 7, 11]], - raw_evaluation_evaled.prediction_times) - for feature_name in raw_evaluation.predictions: - self.assertAllEqual( - [1, 3, 1], # batch, window, num_features. The window size has 2 - # cut off for the first input_window. - raw_evaluation_evaled.predictions[feature_name].shape) - self.assertAllClose( - np.reshape(chunked_evaluation_evaled.predictions[feature_name], - [-1]), - np.reshape(raw_evaluation_evaled.predictions[feature_name], - [-1])) - coordinator.request_stop() - coordinator.join() - - def test_long_eval_discard_indivisible(self): - g = ops.Graph() - with g.as_default(): - model = ar_model.ARModel(periodicities=2, - num_features=1, - num_time_buckets=10, - input_window_size=2, - output_window_size=2) - raw_features = { - TrainEvalFeatures.TIMES: [[1, 3, 5, 7, 11]], - TrainEvalFeatures.VALUES: [[[1.], [2.], [3.], [4.], [5.]]]} - model.initialize_graph() - raw_evaluation = model.define_loss( - raw_features, mode=estimator_lib.ModeKeys.EVAL) - with session.Session() as sess: - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(sess, coord=coordinator) - variables.global_variables_initializer().run() - raw_evaluation_evaled = sess.run(raw_evaluation) - self.assertAllEqual([[7, 11]], - raw_evaluation_evaled.prediction_times) - for feature_name in raw_evaluation.predictions: - self.assertAllEqual( - [1, 2, 1], # batch, window, num_features. The window has two cut - # off for the first input window and one discarded so - # that the remainder is divisible into output windows. - raw_evaluation_evaled.predictions[feature_name].shape) - coordinator.request_stop() - coordinator.join() - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/estimators.py b/tensorflow/contrib/timeseries/python/timeseries/estimators.py deleted file mode 100644 index ce50d3c9849..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/estimators.py +++ /dev/null @@ -1,719 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Estimators for time series models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -from tensorflow.contrib.timeseries.python.timeseries import ar_model -from tensorflow.contrib.timeseries.python.timeseries import feature_keys -from tensorflow.contrib.timeseries.python.timeseries import head as ts_head_lib -from tensorflow.contrib.timeseries.python.timeseries import math_utils -from tensorflow.contrib.timeseries.python.timeseries import state_management -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import state_space_model -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import structural_ensemble -from tensorflow.contrib.timeseries.python.timeseries.state_space_models.filtering_postprocessor import StateInterpolatingAnomalyDetector - -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.estimator.canned import optimizers -from tensorflow.python.estimator.export import export_lib -from tensorflow.python.feature_column import feature_column_lib as feature_column -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.training import training as train -from tensorflow.python.util import nest - - -class TimeSeriesRegressor(estimator_lib.Estimator): - """An Estimator to fit and evaluate a time series model.""" - - def __init__(self, - model, - state_manager=None, - optimizer=None, - model_dir=None, - config=None, - head_type=ts_head_lib.TimeSeriesRegressionHead): - """Initialize the Estimator. - - Args: - model: The time series model to wrap (inheriting from TimeSeriesModel). - state_manager: The state manager to use, or (by default) - PassthroughStateManager if none is needed. - optimizer: The optimization algorithm to use when training, inheriting - from tf.train.Optimizer. Defaults to Adam with step size 0.02. - model_dir: See `Estimator`. - config: See `Estimator`. - head_type: The kind of head to use for the model (inheriting from - `TimeSeriesRegressionHead`). - """ - input_statistics_generator = math_utils.InputStatisticsFromMiniBatch( - dtype=model.dtype, num_features=model.num_features) - if state_manager is None: - if isinstance(model, ar_model.ARModel): - state_manager = state_management.FilteringOnlyStateManager() - else: - state_manager = state_management.PassthroughStateManager() - if optimizer is None: - optimizer = train.AdamOptimizer(0.02) - self._model = model - ts_regression_head = head_type( - model=model, - state_manager=state_manager, - optimizer=optimizer, - input_statistics_generator=input_statistics_generator) - model_fn = ts_regression_head.create_estimator_spec - super(TimeSeriesRegressor, self).__init__( - model_fn=model_fn, model_dir=model_dir, config=config) - - def _model_start_state_placeholders(self, - batch_size_tensor, - static_batch_size=None): - """Creates placeholders with zeroed start state for the current model.""" - gathered_state = {} - # Models may not know the shape of their state without creating some - # variables/ops. Avoid polluting the default graph by making a new one. We - # use only static metadata from the returned Tensors. - with ops.Graph().as_default(): - self._model.initialize_graph() - - # Evaluate the initial state as same-dtype "zero" values. These zero - # constants aren't used, but are necessary for feeding to - # placeholder_with_default for the "cold start" case where state is not - # fed to the model. - def _zeros_like_constant(tensor): - return tensor_util.constant_value(array_ops.zeros_like(tensor)) - - start_state = nest.map_structure(_zeros_like_constant, - self._model.get_start_state()) - for prefixed_state_name, state in ts_head_lib.state_to_dictionary( - start_state).items(): - state_shape_with_batch = tensor_shape.TensorShape( - (static_batch_size,)).concatenate(state.shape) - default_state_broadcast = array_ops.tile( - state[None, ...], - multiples=array_ops.concat([ - batch_size_tensor[None], - array_ops.ones(len(state.shape), dtype=dtypes.int32) - ], - axis=0)) - gathered_state[prefixed_state_name] = array_ops.placeholder_with_default( - input=default_state_broadcast, - name=prefixed_state_name, - shape=state_shape_with_batch) - return gathered_state - - def build_one_shot_parsing_serving_input_receiver_fn(self, - filtering_length, - prediction_length, - default_batch_size=None, - values_input_dtype=None, - truncate_values=False): - """Build an input_receiver_fn for export_savedmodel accepting tf.Examples. - - Only compatible with `OneShotPredictionHead` (see `head`). - - Args: - filtering_length: The number of time steps used as input to the model, for - which values are provided. If more than `filtering_length` values are - provided (via `truncate_values`), only the first `filtering_length` - values are used. - prediction_length: The number of time steps requested as predictions from - the model. Times and all exogenous features must be provided for these - steps. - default_batch_size: If specified, must be a scalar integer. Sets the batch - size in the static shape information of all feature Tensors, which means - only this batch size will be accepted by the exported model. If None - (default), static shape information for batch sizes is omitted. - values_input_dtype: An optional dtype specification for values in the - tf.Example protos (either float32 or int64, since these are the numeric - types supported by tf.Example). After parsing, values are cast to the - model's dtype (float32 or float64). - truncate_values: If True, expects `filtering_length + prediction_length` - values to be provided, but only uses the first `filtering_length`. If - False (default), exactly `filtering_length` values must be provided. - - Returns: - An input_receiver_fn which may be passed to the Estimator's - export_savedmodel. - - Expects features contained in a vector of serialized tf.Examples with - shape [batch size] (dtype `tf.string`), each tf.Example containing - features with the following shapes: - times: [filtering_length + prediction_length] integer - values: [filtering_length, num features] floating point. If - `truncate_values` is True, expects `filtering_length + - prediction_length` values but only uses the first `filtering_length`. - all exogenous features: [filtering_length + prediction_length, ...] - (various dtypes) - """ - if values_input_dtype is None: - values_input_dtype = dtypes.float32 - if truncate_values: - values_proto_length = filtering_length + prediction_length - else: - values_proto_length = filtering_length - - def _serving_input_receiver_fn(): - """A receiver function to be passed to export_savedmodel.""" - times_column = feature_column.numeric_column( - key=feature_keys.TrainEvalFeatures.TIMES, dtype=dtypes.int64) - values_column = feature_column.numeric_column( - key=feature_keys.TrainEvalFeatures.VALUES, - dtype=values_input_dtype, - shape=(self._model.num_features,)) - parsed_features_no_sequence = ( - feature_column.make_parse_example_spec( - list(self._model.exogenous_feature_columns) + - [times_column, values_column])) - parsed_features = {} - for key, feature_spec in parsed_features_no_sequence.items(): - if isinstance(feature_spec, parsing_ops.FixedLenFeature): - if key == feature_keys.TrainEvalFeatures.VALUES: - parsed_features[key] = feature_spec._replace( - shape=((values_proto_length,) + feature_spec.shape)) - else: - parsed_features[key] = feature_spec._replace( - shape=((filtering_length + prediction_length,) + - feature_spec.shape)) - elif feature_spec.dtype == dtypes.string: - parsed_features[key] = parsing_ops.FixedLenFeature( - shape=(filtering_length + prediction_length,), - dtype=dtypes.string) - else: # VarLenFeature - raise ValueError("VarLenFeatures not supported, got %s for key %s" % - (feature_spec, key)) - tfexamples = array_ops.placeholder( - shape=[default_batch_size], dtype=dtypes.string, name="input") - features = parsing_ops.parse_example( - serialized=tfexamples, features=parsed_features) - features[feature_keys.TrainEvalFeatures.TIMES] = array_ops.squeeze( - features[feature_keys.TrainEvalFeatures.TIMES], axis=-1) - features[feature_keys.TrainEvalFeatures.VALUES] = math_ops.cast( - features[feature_keys.TrainEvalFeatures.VALUES], - dtype=self._model.dtype)[:, :filtering_length] - features.update( - self._model_start_state_placeholders( - batch_size_tensor=array_ops.shape( - features[feature_keys.TrainEvalFeatures.TIMES])[0], - static_batch_size=default_batch_size)) - return export_lib.ServingInputReceiver(features, {"examples": tfexamples}) - - return _serving_input_receiver_fn - - def build_raw_serving_input_receiver_fn(self, - default_batch_size=None, - default_series_length=None): - """Build an input_receiver_fn for export_savedmodel which accepts arrays. - - Automatically creates placeholders for exogenous `FeatureColumn`s passed to - the model. - - Args: - default_batch_size: If specified, must be a scalar integer. Sets the batch - size in the static shape information of all feature Tensors, which means - only this batch size will be accepted by the exported model. If None - (default), static shape information for batch sizes is omitted. - default_series_length: If specified, must be a scalar integer. Sets the - series length in the static shape information of all feature Tensors, - which means only this series length will be accepted by the exported - model. If None (default), static shape information for series length is - omitted. - - Returns: - An input_receiver_fn which may be passed to the Estimator's - export_savedmodel. - """ - - def _serving_input_receiver_fn(): - """A receiver function to be passed to export_savedmodel.""" - placeholders = {} - time_placeholder = array_ops.placeholder( - name=feature_keys.TrainEvalFeatures.TIMES, - dtype=dtypes.int64, - shape=[default_batch_size, default_series_length]) - placeholders[feature_keys.TrainEvalFeatures.TIMES] = time_placeholder - # Values are only necessary when filtering. For prediction the default - # value will be ignored. - placeholders[feature_keys.TrainEvalFeatures.VALUES] = ( - array_ops.placeholder_with_default( - name=feature_keys.TrainEvalFeatures.VALUES, - input=array_ops.zeros( - shape=[ - default_batch_size if default_batch_size else 0, - default_series_length if default_series_length else 0, - self._model.num_features - ], - dtype=self._model.dtype), - shape=(default_batch_size, default_series_length, - self._model.num_features))) - if self._model.exogenous_feature_columns: - with ops.Graph().as_default(): - # Default placeholders have only an unknown batch dimension. Make them - # in a separate graph, then splice in the series length to the shapes - # and re-create them in the outer graph. - parsed_features = ( - feature_column.make_parse_example_spec( - self._model.exogenous_feature_columns)) - placeholder_features = parsing_ops.parse_example( - serialized=array_ops.placeholder( - shape=[None], dtype=dtypes.string), - features=parsed_features) - exogenous_feature_shapes = { - key: (value.get_shape(), value.dtype) for key, value - in placeholder_features.items()} - for feature_key, (batch_only_feature_shape, - value_dtype) in (exogenous_feature_shapes.items()): - batch_only_feature_shape = ( - batch_only_feature_shape.with_rank_at_least(1).as_list()) - feature_shape = ([default_batch_size, default_series_length] + - batch_only_feature_shape[1:]) - placeholders[feature_key] = array_ops.placeholder( - dtype=value_dtype, name=feature_key, shape=feature_shape) - batch_size_tensor = array_ops.shape(time_placeholder)[0] - placeholders.update( - self._model_start_state_placeholders( - batch_size_tensor, static_batch_size=default_batch_size)) - return export_lib.ServingInputReceiver(placeholders, placeholders) - - return _serving_input_receiver_fn - - -class ARRegressor(TimeSeriesRegressor): - """An Estimator for an (optionally non-linear) autoregressive model. - - ARRegressor is a window-based model, inputting fixed windows of length - `input_window_size` and outputting fixed windows of length - `output_window_size`. These two parameters must add up to the window_size - passed to the `Chunker` used to create an `input_fn` for training or - evaluation. `RandomWindowInputFn` is suggested for both training and - evaluation, although it may be seeded for deterministic evaluation. - """ - - def __init__(self, - periodicities, - input_window_size, - output_window_size, - num_features, - exogenous_feature_columns=None, - num_time_buckets=10, - loss=ar_model.ARModel.NORMAL_LIKELIHOOD_LOSS, - hidden_layer_sizes=None, - anomaly_prior_probability=None, - anomaly_distribution=None, - optimizer=None, - model_dir=None, - config=None): - """Initialize the Estimator. - - Args: - periodicities: periodicities of the input data, in the same units as the - time feature. Note this can be a single value or a list of values for - multiple periodicities. - input_window_size: Number of past time steps of data to look at when doing - the regression. - output_window_size: Number of future time steps to predict. Note that - setting it to > 1 empirically seems to give a better fit. - num_features: The dimensionality of the time series (one for univariate, - more than one for multivariate). - exogenous_feature_columns: A list of `tf.feature_column`s (for example - `tf.feature_column.embedding_column`) corresponding to exogenous - features which provide extra information to the model but are not part - of the series to be predicted. Passed to - `tf.compat.v1.feature_column.input_layer`. - num_time_buckets: Number of buckets into which to divide (time % - periodicity) for generating time based features. - loss: Loss function to use for training. Currently supported values are - SQUARED_LOSS and NORMAL_LIKELIHOOD_LOSS. Note that for - NORMAL_LIKELIHOOD_LOSS, we train the covariance term as well. For - SQUARED_LOSS, the evaluation loss is reported based on un-scaled - observations and predictions, while the training loss is computed on - normalized data. - hidden_layer_sizes: list of sizes of hidden layers. - anomaly_prior_probability: If specified, constructs a mixture model under - which anomalies (modeled with `anomaly_distribution`) have this prior - probability. See `AnomalyMixtureARModel`. - anomaly_distribution: May not be specified unless - anomaly_prior_probability is specified and is not None. Controls the - distribution of anomalies under the mixture model. Currently either - `ar_model.AnomalyMixtureARModel.GAUSSIAN_ANOMALY` or - `ar_model.AnomalyMixtureARModel.CAUCHY_ANOMALY`. See - `AnomalyMixtureARModel`. Defaults to `GAUSSIAN_ANOMALY`. - optimizer: The optimization algorithm to use when training, inheriting - from tf.train.Optimizer. Defaults to Adagrad with step size 0.1. - model_dir: See `Estimator`. - config: See `Estimator`. - - Raises: - ValueError: For invalid combinations of arguments. - """ - if optimizer is None: - optimizer = train.AdagradOptimizer(0.1) - if anomaly_prior_probability is None and anomaly_distribution is not None: - raise ValueError("anomaly_prior_probability is required if " - "anomaly_distribution is specified.") - if anomaly_prior_probability is None: - if anomaly_distribution is None: - anomaly_distribution = ar_model.AnomalyMixtureARModel.GAUSSIAN_ANOMALY - model = ar_model.ARModel( - periodicities=periodicities, - num_features=num_features, - prediction_model_factory=functools.partial( - ar_model.FlatPredictionModel, - hidden_layer_sizes=hidden_layer_sizes), - exogenous_feature_columns=exogenous_feature_columns, - num_time_buckets=num_time_buckets, - input_window_size=input_window_size, - output_window_size=output_window_size, - loss=loss) - else: - if loss != ar_model.ARModel.NORMAL_LIKELIHOOD_LOSS: - raise ValueError( - "AnomalyMixtureARModel only supports " - "ar_model.ARModel.NORMAL_LIKELIHOOD_LOSS for its loss argument.") - model = ar_model.AnomalyMixtureARModel( - periodicities=periodicities, - input_window_size=input_window_size, - output_window_size=output_window_size, - num_features=num_features, - prediction_model_factory=functools.partial( - ar_model.FlatPredictionModel, - hidden_layer_sizes=hidden_layer_sizes), - exogenous_feature_columns=exogenous_feature_columns, - num_time_buckets=num_time_buckets, - anomaly_prior_probability=anomaly_prior_probability, - anomaly_distribution=anomaly_distribution) - state_manager = state_management.FilteringOnlyStateManager() - super(ARRegressor, self).__init__( - model=model, - state_manager=state_manager, - optimizer=optimizer, - model_dir=model_dir, - config=config) - - -# TODO(b/113684821): Add detailed documentation on what the input_fn should do. -# Add an example of making and returning a Dataset object. Determine if -# endogenous features can be passed in as FeatureColumns. Move ARModel's loss -# functions into a more general location. -class LSTMAutoRegressor(TimeSeriesRegressor): - """An Estimator for an LSTM autoregressive model. - - LSTMAutoRegressor is a window-based model, inputting fixed windows of length - `input_window_size` and outputting fixed windows of length - `output_window_size`. These two parameters must add up to the window_size - of data returned by the `input_fn`. - - Each periodicity in the `periodicities` arg is divided by the `num_timesteps` - into timesteps that are represented as time features added to the model. - - A good heuristic for picking an appropriate periodicity for a given data set - would be the length of cycles in the data. For example, energy usage in a - home is typically cyclic each day. If the time feature in a home energy - usage dataset is in the unit of hours, then 24 would be an appropriate - periodicity. Similarly, a good heuristic for `num_timesteps` is how often the - data is expected to change within the cycle. For the aforementioned home - energy usage dataset and periodicity of 24, then 48 would be a reasonable - value if usage is expected to change every half hour. - - Each feature's value for a given example with time t is the difference - between t and the start of the timestep it falls under. If it doesn't fall - under a feature's associated timestep, then that feature's value is zero. - - For example: if `periodicities` = (9, 12) and `num_timesteps` = 3, then 6 - features would be added to the model, 3 for periodicity 9 and 3 for - periodicity 12. - - For an example data point where t = 17: - - It's in the 3rd timestep for periodicity 9 (2nd period is 9-18 and 3rd - timestep is 15-18) - - It's in the 2nd timestep for periodicity 12 (2nd period is 12-24 and - 2nd timestep is between 16-20). - - Therefore the 6 added features for this row with t = 17 would be: - - # Feature name (periodicity#_timestep#), feature value - P9_T1, 0 # not in first timestep - P9_T2, 0 # not in second timestep - P9_T3, 2 # 17 - 15 since 15 is the start of the 3rd timestep - P12_T1, 0 # not in first timestep - P12_T2, 1 # 17 - 16 since 16 is the start of the 2nd timestep - P12_T3, 0 # not in third timestep - - Example Code: - - ```python - extra_feature_columns = ( - feature_column.numeric_column("exogenous_variable"), - ) - - estimator = LSTMAutoRegressor( - periodicities=10, - input_window_size=10, - output_window_size=5, - model_dir="/path/to/model/dir", - num_features=1, - extra_feature_columns=extra_feature_columns, - num_timesteps=50, - num_units=10, - optimizer=tf.compat.v1.train.ProximalAdagradOptimizer(...)) - - # Input builders - def input_fn_train(): - return { - "times": tf.range(15)[None, :], - "values": tf.random.normal(shape=[1, 15, 1]) - } - estimator.train(input_fn=input_fn_train, steps=100) - - def input_fn_eval(): - pass - metrics = estimator.evaluate(input_fn=input_fn_eval, steps=10) - - def input_fn_predict(): - pass - predictions = estimator.predict(input_fn=input_fn_predict) - ``` - """ - - def __init__(self, - periodicities, - input_window_size, - output_window_size, - model_dir=None, - num_features=1, - extra_feature_columns=None, - num_timesteps=10, - loss=ar_model.ARModel.NORMAL_LIKELIHOOD_LOSS, - num_units=128, - optimizer="Adam", - config=None): - """Initialize the Estimator. - - Args: - periodicities: periodicities of the input data, in the same units as the - time feature (for example 24 if feeding hourly data with a daily - periodicity, or 60 * 24 if feeding minute-level data with daily - periodicity). Note this can be a single value or a list of values for - multiple periodicities. - input_window_size: Number of past time steps of data to look at when doing - the regression. - output_window_size: Number of future time steps to predict. Note that - setting this value to > 1 empirically seems to give a better fit. - model_dir: Directory to save model parameters, graph and etc. This can - also be used to load checkpoints from the directory into a estimator to - continue training a previously saved model. - num_features: The dimensionality of the time series (default value is one - for univariate, more than one for multivariate). - extra_feature_columns: A list of `tf.feature_column`s (for example - `tf.feature_column.embedding_column`) corresponding to features which - provide extra information to the model but are not part of the series to - be predicted. - num_timesteps: Number of buckets into which to divide (time % - periodicity). This value multiplied by the number of periodicities is - the number of time features added to the model. - loss: Loss function to use for training. Currently supported values are - SQUARED_LOSS and NORMAL_LIKELIHOOD_LOSS. Note that for - NORMAL_LIKELIHOOD_LOSS, we train the covariance term as well. For - SQUARED_LOSS, the evaluation loss is reported based on un-scaled - observations and predictions, while the training loss is computed on - normalized data. - num_units: The size of the hidden state in the encoder and decoder LSTM - cells. - optimizer: string, `tf.compat.v1.train.Optimizer` object, or callable that - defines the optimizer algorithm to use for training. Defaults to the - Adam optimizer with a learning rate of 0.01. - config: Optional `estimator.RunConfig` object to configure the runtime - settings. - """ - optimizer = optimizers.get_optimizer_instance(optimizer, learning_rate=0.01) - model = ar_model.ARModel( - periodicities=periodicities, - input_window_size=input_window_size, - output_window_size=output_window_size, - num_features=num_features, - exogenous_feature_columns=extra_feature_columns, - num_time_buckets=num_timesteps, - loss=loss, - prediction_model_factory=functools.partial( - ar_model.LSTMPredictionModel, num_units=num_units)) - state_manager = state_management.FilteringOnlyStateManager() - super(LSTMAutoRegressor, self).__init__( - model=model, - state_manager=state_manager, - optimizer=optimizer, - model_dir=model_dir, - config=config, - head_type=ts_head_lib.OneShotPredictionHead) - - -class StateSpaceRegressor(TimeSeriesRegressor): - """An Estimator for general state space models.""" - - def __init__(self, - model, - state_manager=None, - optimizer=None, - model_dir=None, - config=None, - head_type=ts_head_lib.TimeSeriesRegressionHead): - """See TimeSeriesRegressor. Uses the ChainingStateManager by default.""" - if not isinstance(model, state_space_model.StateSpaceModel): - raise ValueError( - "StateSpaceRegressor only supports state space models (children of " - "StateSpaceModel) in its `model` argument, got {}.".format(model)) - if state_manager is None: - state_manager = state_management.ChainingStateManager() - super(StateSpaceRegressor, self).__init__( - model=model, - state_manager=state_manager, - optimizer=optimizer, - model_dir=model_dir, - config=config, - head_type=head_type) - - -class StructuralEnsembleRegressor(StateSpaceRegressor): - """An Estimator for structural time series models. - - "Structural" refers to the fact that this model explicitly accounts for - structure in the data, such as periodicity and trends. - - `StructuralEnsembleRegressor` is a state space model. It contains components - for modeling level, local linear trends, periodicity, and mean-reverting - transients via a moving average component. Multivariate series are fit with - full covariance matrices for observation and latent state transition noise, - each feature of the multivariate series having its own latent components. - - Note that unlike `ARRegressor`, `StructuralEnsembleRegressor` is sequential, - and so accepts variable window sizes with the same model. - - For training, `RandomWindowInputFn` is recommended as an `input_fn`. Model - state is managed through `ChainingStateManager`: since state space models are - inherently sequential, we save state from previous iterations to get - approximate/eventual consistency while achieving good performance through - batched computation. - - For evaluation, either pass a significant chunk of the series in a single - window (e.g. set `window_size` to the whole series with - `WholeDatasetInputFn`), or use enough random evaluation iterations to cover - several passes through the whole dataset. Either method will ensure that stale - saved state has been flushed. - """ - - def __init__(self, - periodicities, - num_features, - cycle_num_latent_values=11, - moving_average_order=4, - autoregressive_order=0, - exogenous_feature_columns=None, - exogenous_update_condition=None, - dtype=dtypes.float64, - anomaly_prior_probability=None, - optimizer=None, - model_dir=None, - config=None, - head_type=ts_head_lib.TimeSeriesRegressionHead): - """Initialize the Estimator. - - Args: - periodicities: The expected periodicity of the data (for example 24 if - feeding hourly data with a daily periodicity, or 60 * 24 if feeding - minute-level data with daily periodicity). Either a scalar or a list. - This parameter can be any real value, and does not control the size of - the model. However, increasing this without increasing - `num_values_per_cycle` will lead to smoother periodic behavior, as the - same number of distinct values will be cycled through over a longer - period of time. - num_features: The dimensionality of the time series (one for univariate, - more than one for multivariate). - cycle_num_latent_values: Along with `moving_average_order` and - `num_features`, controls the latent state size of the model. Square - matrices of size `num_features * (moving_average_order + - cycle_num_latent_values + 3)` are created and multiplied, so larger - values may be slow. The trade-off is with resolution: cycling between - a smaller number of latent values means that only smoother functions - can be modeled. - moving_average_order: Controls model size (along with - `cycle_num_latent_values` and `autoregressive_order`) and the number of - steps before transient deviations revert to the mean defined by the - period and level/trend components. - autoregressive_order: Each contribution from this component is a linear - combination of this many previous contributions. Also helps to determine - the model size. Learning autoregressive coefficients typically requires - more steps and a smaller step size than other components. - exogenous_feature_columns: A list of `tf.feature_column`s (for example - `tf.feature_column.embedding_column`) corresponding to exogenous - features which provide extra information to the model but are not part - of the series to be predicted. Passed to - `tf.compat.v1.feature_column.input_layer`. - exogenous_update_condition: A function taking two Tensor arguments, - `times` (shape [batch size]) and `features` (a dictionary mapping - exogenous feature keys to Tensors with shapes [batch size, ...]), and - returning a boolean Tensor with shape [batch size] indicating whether - state should be updated using exogenous features for each part of the - batch. Where it is False, no exogenous update is performed. If None - (default), exogenous updates are always performed. Useful for avoiding - "leaky" frequent exogenous updates when sparse updates are desired. - Called only during graph construction. See the "known anomaly" example - for example usage. - dtype: The floating point data type to compute with. float32 may be - faster, but can be problematic for larger models and longer time series. - anomaly_prior_probability: If not None, the model attempts to - automatically detect and ignore anomalies during training. This - parameter then controls the prior probability of an anomaly. Values - closer to 0 mean that points will be discarded less frequently. The - default value (None) means that anomalies are not discarded, which may - be slightly faster. - optimizer: The optimization algorithm to use when training, inheriting - from tf.train.Optimizer. Defaults to Adam with step size 0.02. - model_dir: See `Estimator`. - config: See `Estimator`. - head_type: The kind of head to use for the model (inheriting from - `TimeSeriesRegressionHead`). - """ - if anomaly_prior_probability is not None: - filtering_postprocessor = StateInterpolatingAnomalyDetector( - anomaly_prior_probability=anomaly_prior_probability) - else: - filtering_postprocessor = None - state_space_model_configuration = ( - state_space_model.StateSpaceModelConfiguration( - num_features=num_features, - dtype=dtype, - filtering_postprocessor=filtering_postprocessor, - exogenous_feature_columns=exogenous_feature_columns, - exogenous_update_condition=exogenous_update_condition)) - model = structural_ensemble.MultiResolutionStructuralEnsemble( - cycle_num_latent_values=cycle_num_latent_values, - moving_average_order=moving_average_order, - autoregressive_order=autoregressive_order, - periodicities=periodicities, - configuration=state_space_model_configuration) - super(StructuralEnsembleRegressor, self).__init__( - model=model, - optimizer=optimizer, - model_dir=model_dir, - config=config, - head_type=head_type) diff --git a/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py b/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py deleted file mode 100644 index 7d780559f97..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py +++ /dev/null @@ -1,265 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import tempfile - -import numpy -import six - -from tensorflow.contrib.timeseries.python.timeseries import ar_model -from tensorflow.contrib.timeseries.python.timeseries import estimators -from tensorflow.contrib.timeseries.python.timeseries import feature_keys -from tensorflow.contrib.timeseries.python.timeseries import input_pipeline -from tensorflow.contrib.timeseries.python.timeseries import saved_model_utils - -from tensorflow.python.client import session -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.feature_column import feature_column_lib as feature_column -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.platform import test -from tensorflow.python.saved_model import loader -from tensorflow.python.saved_model import tag_constants - - -class _SeedRunConfig(estimator_lib.RunConfig): - - @property - def tf_random_seed(self): - return 3 - - -class TimeSeriesRegressorTest(test.TestCase): - - def _fit_restore_fit_test_template(self, estimator_fn, dtype): - """Tests restoring previously fit models.""" - model_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) - exogenous_feature_columns = ( - feature_column.numeric_column("exogenous"), - ) - first_estimator = estimator_fn(model_dir, exogenous_feature_columns) - times = numpy.arange(20, dtype=numpy.int64) - values = numpy.arange(20, dtype=dtype.as_numpy_dtype) - exogenous = numpy.arange(20, dtype=dtype.as_numpy_dtype) - features = { - feature_keys.TrainEvalFeatures.TIMES: times, - feature_keys.TrainEvalFeatures.VALUES: values, - "exogenous": exogenous - } - train_input_fn = input_pipeline.RandomWindowInputFn( - input_pipeline.NumpyReader(features), shuffle_seed=2, num_threads=1, - batch_size=16, window_size=16) - eval_input_fn = input_pipeline.RandomWindowInputFn( - input_pipeline.NumpyReader(features), shuffle_seed=3, num_threads=1, - batch_size=16, window_size=16) - first_estimator.train(input_fn=train_input_fn, steps=1) - first_evaluation = first_estimator.evaluate( - input_fn=eval_input_fn, steps=1) - first_loss_before_fit = first_evaluation["loss"] - self.assertAllEqual(first_loss_before_fit, first_evaluation["average_loss"]) - self.assertAllEqual([], first_loss_before_fit.shape) - first_estimator.train(input_fn=train_input_fn, steps=1) - first_loss_after_fit = first_estimator.evaluate( - input_fn=eval_input_fn, steps=1)["loss"] - self.assertAllEqual([], first_loss_after_fit.shape) - second_estimator = estimator_fn(model_dir, exogenous_feature_columns) - second_estimator.train(input_fn=train_input_fn, steps=1) - whole_dataset_input_fn = input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader(features)) - whole_dataset_evaluation = second_estimator.evaluate( - input_fn=whole_dataset_input_fn, steps=1) - exogenous_values_ten_steps = { - "exogenous": numpy.arange( - 10, dtype=dtype.as_numpy_dtype)[None, :, None] - } - predict_input_fn = input_pipeline.predict_continuation_input_fn( - evaluation=whole_dataset_evaluation, - exogenous_features=exogenous_values_ten_steps, - steps=10) - # Also tests that limit_epochs in predict_continuation_input_fn prevents - # infinite iteration - (estimator_predictions, - ) = list(second_estimator.predict(input_fn=predict_input_fn)) - self.assertAllEqual([10, 1], estimator_predictions["mean"].shape) - input_receiver_fn = first_estimator.build_raw_serving_input_receiver_fn() - export_location = first_estimator.export_saved_model( - self.get_temp_dir(), input_receiver_fn) - with ops.Graph().as_default(): - with session.Session() as sess: - signatures = loader.load(sess, [tag_constants.SERVING], export_location) - # Test that prediction and filtering can continue from evaluation output - saved_prediction = saved_model_utils.predict_continuation( - continue_from=whole_dataset_evaluation, - steps=10, - exogenous_features=exogenous_values_ten_steps, - signatures=signatures, - session=sess) - # Saved model predictions should be the same as Estimator predictions - # starting from the same evaluation. - for prediction_key, prediction_value in estimator_predictions.items(): - self.assertAllClose(prediction_value, - numpy.squeeze( - saved_prediction[prediction_key], axis=0)) - first_filtering = saved_model_utils.filter_continuation( - continue_from=whole_dataset_evaluation, - features={ - feature_keys.FilteringFeatures.TIMES: times[None, -1] + 2, - feature_keys.FilteringFeatures.VALUES: values[None, -1] + 2., - "exogenous": values[None, -1, None] + 12. - }, - signatures=signatures, - session=sess) - # Test that prediction and filtering can continue from filtering output - second_saved_prediction = saved_model_utils.predict_continuation( - continue_from=first_filtering, - steps=1, - exogenous_features={ - "exogenous": numpy.arange( - 1, dtype=dtype.as_numpy_dtype)[None, :, None] - }, - signatures=signatures, - session=sess) - self.assertEqual( - times[-1] + 3, - numpy.squeeze( - second_saved_prediction[feature_keys.PredictionResults.TIMES])) - saved_model_utils.filter_continuation( - continue_from=first_filtering, - features={ - feature_keys.FilteringFeatures.TIMES: times[-1] + 3, - feature_keys.FilteringFeatures.VALUES: values[-1] + 3., - "exogenous": values[-1, None] + 13. - }, - signatures=signatures, - session=sess) - - # Test cold starting - six.assertCountEqual( - self, - [feature_keys.FilteringFeatures.TIMES, - feature_keys.FilteringFeatures.VALUES, - "exogenous"], - signatures.signature_def[ - feature_keys.SavedModelLabels.COLD_START_FILTER].inputs.keys()) - batch_numpy_times = numpy.tile( - numpy.arange(30, dtype=numpy.int64)[None, :], (10, 1)) - batch_numpy_values = numpy.ones([10, 30, 1]) - state = saved_model_utils.cold_start_filter( - signatures=signatures, - session=sess, - features={ - feature_keys.FilteringFeatures.TIMES: batch_numpy_times, - feature_keys.FilteringFeatures.VALUES: batch_numpy_values, - "exogenous": 10. + batch_numpy_values - } - ) - predict_times = numpy.tile( - numpy.arange(30, 45, dtype=numpy.int64)[None, :], (10, 1)) - predictions = saved_model_utils.predict_continuation( - continue_from=state, - times=predict_times, - exogenous_features={ - "exogenous": numpy.tile(numpy.arange( - 15, dtype=dtype.as_numpy_dtype), (10,))[None, :, None] - }, - signatures=signatures, - session=sess) - self.assertAllEqual([10, 15, 1], predictions["mean"].shape) - - def test_fit_restore_fit_ar_flat(self): - def _estimator_fn(model_dir, exogenous_feature_columns): - return estimators.ARRegressor( - periodicities=10, input_window_size=10, output_window_size=6, - num_features=1, model_dir=model_dir, config=_SeedRunConfig(), - # This test is flaky with normal likelihood loss (could add more - # training iterations instead). - loss=ar_model.ARModel.SQUARED_LOSS, - exogenous_feature_columns=exogenous_feature_columns) - self._fit_restore_fit_test_template(_estimator_fn, dtype=dtypes.float32) - - def test_fit_restore_fit_ar_lstm(self): - def _estimator_fn(model_dir, exogenous_feature_columns): - return estimators.TimeSeriesRegressor( - model=ar_model.ARModel( - periodicities=10, input_window_size=10, output_window_size=6, - num_features=1, - exogenous_feature_columns=exogenous_feature_columns, - prediction_model_factory=functools.partial( - ar_model.LSTMPredictionModel, - num_units=10)), - config=_SeedRunConfig(), - model_dir=model_dir) - self._fit_restore_fit_test_template(_estimator_fn, dtype=dtypes.float32) - - def test_fit_restore_fit_structural_ensemble_regressor(self): - dtype = dtypes.float32 - def _estimator_fn(model_dir, exogenous_feature_columns): - return estimators.StructuralEnsembleRegressor( - num_features=1, periodicities=10, model_dir=model_dir, dtype=dtype, - config=_SeedRunConfig(), - exogenous_feature_columns=exogenous_feature_columns) - self._fit_restore_fit_test_template(_estimator_fn, dtype=dtype) - - def test_structural_ensemble_numpy_input(self): - numpy_data = {"times": numpy.arange(50), - "values": numpy.random.normal(size=[50])} - estimators.StructuralEnsembleRegressor( - num_features=1, periodicities=[], model_dir=self.get_temp_dir(), - config=_SeedRunConfig()).train( - input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader(numpy_data)), - steps=1) - - def test_ar_lstm_regressor(self): - dtype = dtypes.float32 - model_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) - exogenous_feature_columns = ( - feature_column.numeric_column("exogenous"), - ) - estimator = estimators.LSTMAutoRegressor( - periodicities=10, - input_window_size=10, - output_window_size=6, - model_dir=model_dir, - num_features=1, - extra_feature_columns=exogenous_feature_columns, - num_units=10, - config=_SeedRunConfig()) - times = numpy.arange(20, dtype=numpy.int64) - values = numpy.arange(20, dtype=dtype.as_numpy_dtype) - exogenous = numpy.arange(20, dtype=dtype.as_numpy_dtype) - features = { - feature_keys.TrainEvalFeatures.TIMES: times, - feature_keys.TrainEvalFeatures.VALUES: values, - "exogenous": exogenous - } - train_input_fn = input_pipeline.RandomWindowInputFn( - input_pipeline.NumpyReader(features), shuffle_seed=2, num_threads=1, - batch_size=16, window_size=16) - eval_input_fn = input_pipeline.RandomWindowInputFn( - input_pipeline.NumpyReader(features), shuffle_seed=3, num_threads=1, - batch_size=16, window_size=16) - estimator.train(input_fn=train_input_fn, steps=1) - evaluation = estimator.evaluate( - input_fn=eval_input_fn, steps=1) - self.assertAllEqual(evaluation["loss"], evaluation["average_loss"]) - self.assertAllEqual([], evaluation["loss"].shape) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/feature_keys.py b/tensorflow/contrib/timeseries/python/timeseries/feature_keys.py deleted file mode 100644 index 56566ee2e32..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/feature_keys.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Commonly used special feature names for time series models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.saved_model import signature_constants - - -class State(object): - """Key formats for accepting/returning state.""" - # The model-dependent state to start from, as a single tuple. - STATE_TUPLE = "start_tuple" - # Same meaning as STATE_TUPLE, but prefixes keys representing flattened model - # state rather than mapping to a nested tuple containing model state, - # primarily for use with export_savedmodel. - STATE_PREFIX = "model_state" - - -class Times(object): - """Key formats for accepting/returning times.""" - # An increasing vector of integers. - TIMES = "times" - - -class Values(object): - """Key formats for accepting/returning values.""" - # Floating point, with one or more values corresponding to each time in TIMES. - VALUES = "values" - - -class TrainEvalFeatures(Times, Values): - """Feature names used during training and evaluation.""" - pass - - -class PredictionFeatures(Times, State): - """Feature names used during prediction.""" - pass - - -class FilteringFeatures(Times, Values, State): - """Special feature names for filtering.""" - pass - - -class PredictionResults(Times): - """Keys returned when predicting (not comprehensive).""" - pass - - -class FilteringResults(Times, State): - """Keys returned from evaluation/filtering.""" - pass - - -class SavedModelLabels(object): - """Names of signatures exported with export_savedmodel.""" - PREDICT = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY - FILTER = "filter" - COLD_START_FILTER = "cold_start_filter" diff --git a/tensorflow/contrib/timeseries/python/timeseries/head.py b/tensorflow/contrib/timeseries/python/timeseries/head.py deleted file mode 100644 index 1f9f9b7aa68..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/head.py +++ /dev/null @@ -1,482 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Timeseries head.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import re - -from tensorflow.contrib.timeseries.python.timeseries import feature_keys -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.estimator.canned import head as head_lib -from tensorflow.python.estimator.canned import metric_keys -from tensorflow.python.estimator.export import export_lib -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import metrics_impl -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.summary import summary -from tensorflow.python.training import training_util -from tensorflow.python.util import nest - - -class _NoStatePredictOutput(export_lib.PredictOutput): - - def as_signature_def(self, receiver_tensors): - no_state_receiver_tensors = { - key: value for key, value in receiver_tensors.items() - if not key.startswith(feature_keys.State.STATE_PREFIX)} - return super(_NoStatePredictOutput, self).as_signature_def( - receiver_tensors=no_state_receiver_tensors) - - -class TimeSeriesRegressionHead(head_lib._Head): # pylint:disable=protected-access - """Determines input and output signatures for a time series model.""" - - def __init__(self, - model, - state_manager, - optimizer, - input_statistics_generator=None, - name=None): - """Creates a `_Head` for time series regression. - - Args: - model: A model for time series regression. - state_manager: A state manager. - optimizer: An optimizer. - input_statistics_generator: A input statistics generator. - name: An optional name for the model. - """ - self.model = model - self.state_manager = state_manager - self.optimizer = optimizer - self.input_statistics_generator = input_statistics_generator - self._name = name - - @property - def name(self): - return self._name - - # TODO(terrytangyuan): consolidate `model_outputs` and `_Head.LossSpec` - # once `_Head.create_loss` becomes extendable - def create_loss(self, features, mode, logits=None, labels=None): - """See `_Head`.""" - model_outputs = self.state_manager.define_loss( - self.model, features, mode) - summary.scalar( - head_lib._summary_key(self._name, metric_keys.MetricKeys.LOSS), - model_outputs.loss) - return model_outputs - - @property - def logits_dimension(self): - """See `_Head`.""" - return 1 - - def _train_ops(self, features): - """Add training ops to the graph.""" - mode = estimator_lib.ModeKeys.TRAIN - with variable_scope.variable_scope( - "model", - # Use ResourceVariables to avoid race conditions. - use_resource=True): - model_outputs = self.create_loss(features, mode) - - train_op = self.optimizer.minimize( - model_outputs.loss, - global_step=training_util.get_global_step()) - return estimator_lib.EstimatorSpec( - loss=model_outputs.loss, - mode=mode, - train_op=train_op) - - def _evaluate_ops(self, features): - """Add ops for evaluation (aka filtering) to the graph.""" - mode = estimator_lib.ModeKeys.EVAL - with variable_scope.variable_scope("model", use_resource=True): - model_outputs = self.create_loss(features, mode) - metrics = {} - # Just output in-sample predictions for the last chunk seen - for prediction_key, prediction_value in model_outputs.predictions.items(): - metrics[prediction_key] = _identity_metric_single(prediction_key, - prediction_value) - metrics[feature_keys.FilteringResults.TIMES] = _identity_metric_single( - feature_keys.FilteringResults.TIMES, model_outputs.prediction_times) - metrics[feature_keys.FilteringResults.STATE_TUPLE] = ( - _identity_metric_nested(feature_keys.FilteringResults.STATE_TUPLE, - model_outputs.end_state)) - metrics[metric_keys.MetricKeys.LOSS_MEAN] = metrics_impl.mean( - model_outputs.loss, name="average_loss") - return estimator_lib.EstimatorSpec( - loss=model_outputs.loss, - mode=mode, - eval_metric_ops=metrics, - # needed for custom metrics. - predictions=model_outputs.predictions) - - def _predict_ops(self, features): - """Add ops for prediction to the graph.""" - with variable_scope.variable_scope("model", use_resource=True): - prediction = self.model.predict(features=features) - prediction[feature_keys.PredictionResults.TIMES] = features[ - feature_keys.PredictionFeatures.TIMES] - return estimator_lib.EstimatorSpec( - predictions=prediction, mode=estimator_lib.ModeKeys.PREDICT) - - def _serving_ops(self, features): - """Add ops for serving to the graph.""" - with variable_scope.variable_scope("model", use_resource=True): - prediction_outputs = self.model.predict(features=features) - with variable_scope.variable_scope("model", reuse=True): - filtering_outputs = self.create_loss( - features, estimator_lib.ModeKeys.EVAL) - with variable_scope.variable_scope("model", reuse=True): - no_state_features = { - k: v for k, v in features.items() - if not k.startswith(feature_keys.State.STATE_PREFIX)} - # Ignore any state management when cold-starting. The model's default - # start state is replicated across the batch. - cold_filtering_outputs = self.model.define_loss( - features=no_state_features, mode=estimator_lib.ModeKeys.EVAL) - return estimator_lib.EstimatorSpec( - mode=estimator_lib.ModeKeys.PREDICT, - export_outputs={ - feature_keys.SavedModelLabels.PREDICT: - export_lib.PredictOutput(prediction_outputs), - feature_keys.SavedModelLabels.FILTER: - export_lib.PredictOutput( - state_to_dictionary(filtering_outputs.end_state)), - feature_keys.SavedModelLabels.COLD_START_FILTER: - _NoStatePredictOutput( - state_to_dictionary(cold_filtering_outputs.end_state)) - }, - # Likely unused, but it is necessary to return `predictions` to satisfy - # the Estimator's error checking. - predictions={}) - - def _convert_feature_to_tensor(self, name, value): - """Casts features to the correct dtype based on their name.""" - if name in [ - feature_keys.TrainEvalFeatures.TIMES, - feature_keys.PredictionFeatures.TIMES - ]: - return math_ops.cast(value, dtypes.int64) - if name == feature_keys.TrainEvalFeatures.VALUES: - return math_ops.cast(value, self.model.dtype) - if name == feature_keys.PredictionFeatures.STATE_TUPLE: - return value # Correct dtypes are model-dependent - return sparse_tensor.convert_to_tensor_or_sparse_tensor(value) - - def _gather_state(self, features): - """Returns `features` with state packed, indicates if packing was done.""" - prefixed_state_re = re.compile(r"^" + feature_keys.State.STATE_PREFIX + - r"_(\d+)$") - numbered_state = [] - for key, tensor in features.items(): - search_result = prefixed_state_re.search(key) - if search_result: - numbered_state.append((int(search_result.group(1)), key, tensor)) - if not numbered_state: - return features, False - features = features.copy() - for _, key, _ in numbered_state: - del features[key] - numbered_state.sort(key=lambda number, *_: number) - features[feature_keys.State.STATE_TUPLE] = nest.pack_sequence_as( - structure=self.model.get_start_state(), - flat_sequence=[tensor for _, _, tensor in numbered_state]) - return features, True - - def _check_predict_features(self, features): - """Raises errors if features are not suitable for prediction.""" - if feature_keys.PredictionFeatures.TIMES not in features: - raise ValueError("Expected a '{}' feature for prediction.".format( - feature_keys.PredictionFeatures.TIMES)) - if feature_keys.PredictionFeatures.STATE_TUPLE not in features: - raise ValueError("Expected a '{}' feature for prediction.".format( - feature_keys.PredictionFeatures.STATE_TUPLE)) - times_feature = features[feature_keys.PredictionFeatures.TIMES] - if not times_feature.get_shape().is_compatible_with([None, None]): - raise ValueError( - ("Expected shape (batch dimension, window size) for feature '{}' " - "(got shape {})").format(feature_keys.PredictionFeatures.TIMES, - times_feature.get_shape())) - _check_feature_shapes_compatible_with( - features=features, - compatible_with_name=feature_keys.PredictionFeatures.TIMES, - compatible_with_value=times_feature, - ignore=set([ - # Model-dependent shapes - feature_keys.PredictionFeatures.STATE_TUPLE - ])) - - def create_estimator_spec(self, features, mode, labels=None): - """Performs basic error checking and returns an EstimatorSpec.""" - with ops.name_scope(self._name, "head"): - if labels is not None and labels != {}: # for better error messages. - raise ValueError( - "The model received a `labels`, which is not supported. " - "Pass '{}' and '{}' as features.".format( - feature_keys.TrainEvalFeatures.TIMES, - feature_keys.TrainEvalFeatures.VALUES)) - del labels - features = { - name: self._convert_feature_to_tensor(name=name, value=value) - for name, value in features.items() - } - if self.input_statistics_generator is not None: - input_statistics = self.input_statistics_generator.initialize_graph( - features, update_statistics=(mode == estimator_lib.ModeKeys.TRAIN)) - else: - input_statistics = None - self.model.initialize_graph(input_statistics=input_statistics) - - # _gather_state requires the model to have its graph initialized (so it - # has access to the structure of the model's state) - features, passed_flat_state = self._gather_state(features) - if (mode == estimator_lib.ModeKeys.TRAIN or - mode == estimator_lib.ModeKeys.EVAL): - _check_train_eval_features(features, self.model) - elif mode == estimator_lib.ModeKeys.PREDICT: - self._check_predict_features(features) - else: - raise ValueError("Unknown mode '{}' passed to model_fn.".format(mode)) - - self.state_manager.initialize_graph( - model=self.model, input_statistics=input_statistics) - - if mode == estimator_lib.ModeKeys.TRAIN: - return self._train_ops(features) - elif mode == estimator_lib.ModeKeys.EVAL: - return self._evaluate_ops(features) - elif mode == estimator_lib.ModeKeys.PREDICT and not passed_flat_state: - return self._predict_ops(features) - elif mode == estimator_lib.ModeKeys.PREDICT and passed_flat_state: - # The mode is PREDICT, but we're actually in export_savedmodel for - # serving. We want to return two graphs: one for filtering (state + data - # -> state) and one for predicting (state -> prediction). - return self._serving_ops(features) - - -class OneShotPredictionHead(TimeSeriesRegressionHead): - """A time series head which exports a single stateless serving signature. - - The serving default signature exported by this head expects `times`, `values`, - and any exogenous features, but no state. `values` has shape `[batch_size, - filter_length, num_features]` and `times` has shape `[batch_size, - total_length]`, where `total_length > filter_length`. Any exogenous features - must have their shapes prefixed by the shape of the `times` feature. - - When serving, first performs filtering on the series up to `filter_length` - starting from the default start state for the model, then computes predictions - on the remainder of the series, returning them. - - Model state is neither accepted nor returned, so filtering must be performed - each time predictions are requested when using this head. - """ - - def _check_predict_features(self, features): - """Raises errors if features are not suitable for one-shot prediction.""" - if feature_keys.PredictionFeatures.TIMES not in features: - raise ValueError("Expected a '{}' feature for prediction.".format( - feature_keys.PredictionFeatures.TIMES)) - if feature_keys.TrainEvalFeatures.VALUES not in features: - raise ValueError("Expected a '{}' feature for prediction.".format( - feature_keys.TrainEvalFeatures.VALUES)) - if feature_keys.PredictionFeatures.STATE_TUPLE not in features: - raise ValueError("Expected a '{}' feature for prediction.".format( - feature_keys.PredictionFeatures.STATE_TUPLE)) - times_feature = features[feature_keys.PredictionFeatures.TIMES] - if not times_feature.get_shape().is_compatible_with([None, None]): - raise ValueError( - ("Expected shape (batch dimension, window size) for feature '{}' " - "(got shape {})").format(feature_keys.PredictionFeatures.TIMES, - times_feature.get_shape())) - _check_feature_shapes_compatible_with( - features=features, - compatible_with_name=feature_keys.PredictionFeatures.TIMES, - compatible_with_value=times_feature, - ignore=set([ - # Model-dependent shapes - feature_keys.PredictionFeatures.STATE_TUPLE, - # One shot prediction head relies on values being shorter than - # times. Even though we're predicting eventually, we need values for - # the filtering phase. - feature_keys.TrainEvalFeatures.VALUES, - ])) - - def _evaluate_ops(self, features): - """Add ops for evaluation (aka filtering) to the graph.""" - spec = super(OneShotPredictionHead, self)._evaluate_ops(features) - # No state is fed to OneShotPredictionHead, so we don't return it; it being - # a tuple can cause issues for downstream infrastructure. - del spec.eval_metric_ops[feature_keys.State.STATE_TUPLE] - return spec - - def _serving_ops(self, features): - """Add ops for serving to the graph.""" - with variable_scope.variable_scope("model", use_resource=True): - filtering_features = {} - prediction_features = {} - values_length = array_ops.shape( - features[feature_keys.FilteringFeatures.VALUES])[1] - for key, value in features.items(): - if key == feature_keys.State.STATE_TUPLE: - # Ignore state input. The model's default start state is replicated - # across the batch. - continue - if key == feature_keys.FilteringFeatures.VALUES: - filtering_features[key] = value - else: - filtering_features[key] = value[:, :values_length] - prediction_features[key] = value[:, values_length:] - cold_filtering_outputs = self.model.define_loss( - features=filtering_features, mode=estimator_lib.ModeKeys.EVAL) - prediction_features[feature_keys.State.STATE_TUPLE] = ( - cold_filtering_outputs.end_state) - with variable_scope.variable_scope("model", reuse=True): - prediction_outputs = self.model.predict( - features=prediction_features) - return estimator_lib.EstimatorSpec( - mode=estimator_lib.ModeKeys.PREDICT, - export_outputs={ - feature_keys.SavedModelLabels.PREDICT: - _NoStatePredictOutput(prediction_outputs), - }, - # Likely unused, but it is necessary to return `predictions` to satisfy - # the Estimator's error checking. - predictions={}) - - -def _check_feature_shapes_compatible_with(features, - compatible_with_name, - compatible_with_value, - ignore=None): - """Checks all features are compatible with the given time-like feature.""" - if ignore is None: - ignore = set() - for name, value in features.items(): - if name in ignore: - continue - feature_shape = value.get_shape() - if feature_shape.ndims is None: - continue - if feature_shape.ndims < 2: - raise ValueError( - ("Features must have shape (batch dimension, window size, ...) " - "(got rank {} for feature '{}')").format(feature_shape.ndims, name)) - if not feature_shape[:2].is_compatible_with( - compatible_with_value.get_shape()): - raise ValueError( - ("Features must have shape (batch dimension, window size, ...) " - "where batch dimension and window size match the " - "'{times_feature}' feature (got shape {feature_shape} for " - "feature '{feature_name}' but shape {times_shape} for feature " - "'{times_feature}')").format( - times_feature=compatible_with_name, - feature_shape=feature_shape, - feature_name=name, - times_shape=compatible_with_value.get_shape())) - - -def _check_train_eval_features(features, model): - """Raise errors if features are not suitable for training/evaluation.""" - if feature_keys.TrainEvalFeatures.TIMES not in features: - raise ValueError("Expected a '{}' feature for training/evaluation.".format( - feature_keys.TrainEvalFeatures.TIMES)) - if feature_keys.TrainEvalFeatures.VALUES not in features: - raise ValueError("Expected a '{}' feature for training/evaluation.".format( - feature_keys.TrainEvalFeatures.VALUES)) - times_feature = features[feature_keys.TrainEvalFeatures.TIMES] - if not times_feature.get_shape().is_compatible_with([None, None]): - raise ValueError( - ("Expected shape (batch dimension, window size) for feature '{}' " - "(got shape {})").format(feature_keys.TrainEvalFeatures.TIMES, - times_feature.get_shape())) - values_feature = features[feature_keys.TrainEvalFeatures.VALUES] - if not values_feature.get_shape().is_compatible_with( - [None, None, model.num_features]): - raise ValueError( - ("Expected shape (batch dimension, window size, {num_features}) " - "for feature '{feature_name}', since the model was configured " - "with num_features={num_features} (got shape {got_shape})").format( - num_features=model.num_features, - feature_name=feature_keys.TrainEvalFeatures.VALUES, - got_shape=times_feature.get_shape())) - _check_feature_shapes_compatible_with( - features=features, - compatible_with_name=feature_keys.TrainEvalFeatures.TIMES, - compatible_with_value=times_feature, - ignore=set([ - feature_keys.State.STATE_TUPLE # Model-dependent shapes - ])) - - -def _identity_metric_single(name, input_tensor): - """A metric which takes on its last updated value. - - This keeps evaluation metrics in sync with one another, since update ops are - run separately from their result Tensors. Simply returning (input_tensor, - no_op) as a metric with a value but no update means that a metric will come - from a different batch of data than metrics which cache values in a Variable - (e.g. the default loss metric). - - Args: - name: A name for the metric. - input_tensor: Any Tensor. - Returns: - A tuple of (value, update_op). - """ - metric_variable = variable_scope.variable( - name="{}_identity_metric".format(name), - initial_value=array_ops.zeros([], dtype=input_tensor.dtype), - collections=[ops.GraphKeys.LOCAL_VARIABLES], - validate_shape=False) - update_op = state_ops.assign( - metric_variable, input_tensor, validate_shape=False) - # This shape will be correct once the first update runs (but may be - # incomplete, so is not helpful for initializing the variable). - metric_variable.set_shape(input_tensor.get_shape()) - return (metric_variable.value(), update_op) - - -def _identity_metric_nested(name, input_tensors): - """Create identity metrics for a nested tuple of Tensors.""" - update_ops = [] - value_tensors = [] - for tensor_number, tensor in enumerate(nest.flatten(input_tensors)): - value_tensor, update_op = _identity_metric_single( - name="{}_{}".format(name, tensor_number), input_tensor=tensor) - update_ops.append(update_op) - value_tensors.append(value_tensor) - return (nest.pack_sequence_as(input_tensors, value_tensors), - control_flow_ops.group(*update_ops)) - - -def state_to_dictionary(state_tuple): - """Flatten model state into a dictionary with string keys.""" - flattened = {} - for state_number, state_value in enumerate(nest.flatten(state_tuple)): - prefixed_state_name = "{}_{:02d}".format(feature_keys.State.STATE_PREFIX, - state_number) - flattened[prefixed_state_name] = state_value - return flattened diff --git a/tensorflow/contrib/timeseries/python/timeseries/head_test.py b/tensorflow/contrib/timeseries/python/timeseries/head_test.py deleted file mode 100644 index 8f692d94da4..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/head_test.py +++ /dev/null @@ -1,478 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for head.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import os - -from absl.testing import parameterized -import numpy -import six - -from tensorflow.contrib.estimator.python.estimator import extenders -from tensorflow.contrib.timeseries.examples import lstm as lstm_example -from tensorflow.contrib.timeseries.python.timeseries import ar_model -from tensorflow.contrib.timeseries.python.timeseries import estimators as ts_estimators -from tensorflow.contrib.timeseries.python.timeseries import feature_keys -from tensorflow.contrib.timeseries.python.timeseries import head as ts_head_lib -from tensorflow.contrib.timeseries.python.timeseries import input_pipeline -from tensorflow.contrib.timeseries.python.timeseries import model -from tensorflow.contrib.timeseries.python.timeseries import state_management -from tensorflow.core.example import example_pb2 - -from tensorflow.python.client import session as session_lib -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.feature_column import feature_column_lib as feature_column -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import metrics -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.saved_model import loader -from tensorflow.python.saved_model import tag_constants -from tensorflow.python.training import adam -from tensorflow.python.training import coordinator as coordinator_lib -from tensorflow.python.training import queue_runner_impl -from tensorflow.python.training import training as train - - -class HeadTest(test.TestCase): - - def test_labels_provided_error(self): - model_fn = _stub_model_fn() - for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL, - estimator_lib.ModeKeys.PREDICT]: - with self.assertRaisesRegexp(ValueError, "received a `labels`"): - model_fn(features={}, labels={"a": "b"}, mode=mode) - - with self.assertRaisesRegexp(ValueError, "received a `labels`"): - model_fn(features={}, labels=array_ops.zeros([]), mode=mode) - - def test_unknown_mode(self): - model_fn = _stub_model_fn() - with self.assertRaisesRegexp(ValueError, "Unknown mode 'Not a mode'"): - model_fn(features={}, labels={}, mode="Not a mode") - - -class _TickerModel(object): - num_features = 1 - dtype = dtypes.float32 - - def initialize_graph(self, input_statistics): - pass - - def define_loss(self, features, mode): - del mode # unused - return model.ModelOutputs( - loss=features["ticker"], - end_state=(features["ticker"], features["ticker"]), - prediction_times=array_ops.zeros(()), - predictions={"ticker": features["ticker"]}) - - -class EvaluationMetricsTests(test.TestCase): - - def test_metrics_consistent(self): - # Tests that the identity metrics used to report in-sample predictions match - # the behavior of standard metrics. - g = ops.Graph() - with g.as_default(): - features = { - feature_keys.TrainEvalFeatures.TIMES: - array_ops.zeros((1, 1)), - feature_keys.TrainEvalFeatures.VALUES: - array_ops.zeros((1, 1, 1)), - "ticker": - array_ops.reshape( - math_ops.cast( - variables.VariableV1( - name="ticker", - initial_value=0, - dtype=dtypes.int64, - collections=[ops.GraphKeys.LOCAL_VARIABLES]) - .count_up_to(10), - dtype=dtypes.float32), (1, 1, 1)) - } - model_fn = ts_head_lib.TimeSeriesRegressionHead( - model=_TickerModel(), - state_manager=state_management.PassthroughStateManager(), - optimizer=train.GradientDescentOptimizer(0.001)).create_estimator_spec - outputs = model_fn( - features=features, labels=None, mode=estimator_lib.ModeKeys.EVAL) - metric_update_ops = [ - metric[1] for metric in outputs.eval_metric_ops.values()] - loss_mean, loss_update = metrics.mean(outputs.loss) - metric_update_ops.append(loss_update) - with self.cached_session() as sess: - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(sess, coord=coordinator) - variables.local_variables_initializer().run() - sess.run(metric_update_ops) - loss_evaled, metric_evaled, nested_metric_evaled = sess.run( - (loss_mean, outputs.eval_metric_ops["ticker"][0], - outputs.eval_metric_ops[feature_keys.FilteringResults.STATE_TUPLE][ - 0][0])) - # The custom model_utils metrics for in-sample predictions should be in - # sync with the Estimator's mean metric for model loss. - self.assertAllClose(0., loss_evaled) - self.assertAllClose((((0.,),),), metric_evaled) - self.assertAllClose((((0.,),),), nested_metric_evaled) - coordinator.request_stop() - coordinator.join() - - def test_custom_metrics(self): - """Tests that the custom metrics can be applied to the estimator.""" - model_dir = self.get_temp_dir() - estimator = ts_estimators.TimeSeriesRegressor( - model=lstm_example._LSTMModel(num_features=1, num_units=4), - optimizer=adam.AdamOptimizer(0.001), - config=estimator_lib.RunConfig(tf_random_seed=4), - model_dir=model_dir) - - def input_fn(): - return { - feature_keys.TrainEvalFeatures.TIMES: [[1, 2, 3], [7, 8, 9]], - feature_keys.TrainEvalFeatures.VALUES: - numpy.array([[[0.], [1.], [0.]], [[2.], [3.], [2.]]]) - } - - def metrics_fn(predictions, features): - # checking that the inputs are properly passed. - predict = predictions["mean"] - target = features[feature_keys.TrainEvalFeatures.VALUES][:, -1, 0] - return { - "plain_boring_metric386": - (math_ops.reduce_mean(math_ops.abs(predict - target)), - control_flow_ops.no_op()), - "fun_metric101": (math_ops.reduce_sum(predict + target), - control_flow_ops.no_op()), - } - - # Evaluation without training is enough for testing custom metrics. - estimator = extenders.add_metrics(estimator, metrics_fn) - evaluation = estimator.evaluate(input_fn, steps=1) - self.assertIn("plain_boring_metric386", evaluation) - self.assertIn("fun_metric101", evaluation) - self.assertIn("average_loss", evaluation) - # The values are deterministic because of fixed tf_random_seed. - # However if they become flaky, remove such exacts comparisons. - self.assertAllClose(evaluation["plain_boring_metric386"], 1.130380) - self.assertAllClose(evaluation["fun_metric101"], 10.435442) - - -class _StubModel(object): - num_features = 3 - dtype = dtypes.float64 - - def initialize_graph(self, input_statistics): - del input_statistics # unused - - -def _stub_model_fn(): - return ts_head_lib.TimeSeriesRegressionHead( - model=_StubModel(), - state_manager=state_management.PassthroughStateManager(), - optimizer=train.AdamOptimizer(0.001)).create_estimator_spec - - -class TrainEvalFeatureCheckingTests(test.TestCase): - - def test_no_time_feature(self): - model_fn = _stub_model_fn() - for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]: - with self.assertRaisesRegexp(ValueError, "Expected a '{}' feature".format( - feature_keys.TrainEvalFeatures.TIMES)): - model_fn( - features={feature_keys.TrainEvalFeatures.VALUES: [[[1.]]]}, - labels=None, - mode=mode) - - def test_no_value_feature(self): - model_fn = _stub_model_fn() - for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]: - with self.assertRaisesRegexp(ValueError, "Expected a '{}' feature".format( - feature_keys.TrainEvalFeatures.VALUES)): - model_fn( - features={feature_keys.TrainEvalFeatures.TIMES: [[1]]}, - labels=None, - mode=mode) - - def test_bad_time_rank(self): - model_fn = _stub_model_fn() - for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]: - with self.assertRaisesRegexp(ValueError, - "Expected shape.*for feature '{}'".format( - feature_keys.TrainEvalFeatures.TIMES)): - model_fn( - features={ - feature_keys.TrainEvalFeatures.TIMES: [[[1]]], - feature_keys.TrainEvalFeatures.VALUES: [[[1.]]] - }, - labels=None, - mode=mode) - - def test_bad_value_rank(self): - model_fn = _stub_model_fn() - for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]: - with self.assertRaisesRegexp(ValueError, - "Expected shape.*for feature '{}'".format( - feature_keys.TrainEvalFeatures.VALUES)): - model_fn( - features={ - feature_keys.TrainEvalFeatures.TIMES: [[1]], - feature_keys.TrainEvalFeatures.VALUES: [[1.]] - }, - labels=None, - mode=mode) - - def test_bad_value_num_features(self): - model_fn = _stub_model_fn() - for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]: - with self.assertRaisesRegexp( - ValueError, "Expected shape.*, 3.*for feature '{}'".format( - feature_keys.TrainEvalFeatures.VALUES)): - model_fn( - features={ - feature_keys.TrainEvalFeatures.TIMES: [[1]], - feature_keys.TrainEvalFeatures.VALUES: [[[1.]]] - }, - labels=None, - mode=mode) - - def test_bad_exogenous_shape(self): - model_fn = _stub_model_fn() - for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]: - with self.assertRaisesRegexp( - ValueError, - "Features must have shape.*for feature 'exogenous'"): - model_fn( - features={ - feature_keys.TrainEvalFeatures.TIMES: [[1]], - feature_keys.TrainEvalFeatures.VALUES: [[[1., 2., 3.]]], - "exogenous": [[1], [2]] - }, - labels=None, - mode=mode) - - -class PredictFeatureCheckingTests(test.TestCase): - - def test_no_time_feature(self): - model_fn = _stub_model_fn() - with self.assertRaisesRegexp(ValueError, "Expected a '{}' feature".format( - feature_keys.PredictionFeatures.TIMES)): - model_fn( - features={ - feature_keys.PredictionFeatures.STATE_TUPLE: ([[[1.]]], 1.) - }, - labels=None, - mode=estimator_lib.ModeKeys.PREDICT) - - def test_no_start_state_feature(self): - model_fn = _stub_model_fn() - with self.assertRaisesRegexp(ValueError, "Expected a '{}' feature".format( - feature_keys.PredictionFeatures.STATE_TUPLE)): - model_fn( - features={feature_keys.PredictionFeatures.TIMES: [[1]]}, - labels=None, - mode=estimator_lib.ModeKeys.PREDICT) - - def test_bad_time_rank(self): - model_fn = _stub_model_fn() - with self.assertRaisesRegexp(ValueError, - "Expected shape.*for feature '{}'".format( - feature_keys.PredictionFeatures.TIMES)): - model_fn( - features={ - feature_keys.PredictionFeatures.TIMES: 1, - feature_keys.PredictionFeatures.STATE_TUPLE: (1, (2, 3.)) - }, - labels=None, - mode=estimator_lib.ModeKeys.PREDICT) - - def test_bad_exogenous_shape(self): - model_fn = _stub_model_fn() - with self.assertRaisesRegexp( - ValueError, - "Features must have shape.*for feature 'exogenous'"): - model_fn( - features={ - feature_keys.PredictionFeatures.TIMES: [[1]], - feature_keys.PredictionFeatures.STATE_TUPLE: (1, (2, 3.)), - "exogenous": 1. - }, - labels=None, - mode=estimator_lib.ModeKeys.PREDICT) - - -def _custom_time_series_regressor( - model_dir, head_type, exogenous_feature_columns): - return ts_estimators.TimeSeriesRegressor( - model=lstm_example._LSTMModel( - num_features=5, num_units=128, - exogenous_feature_columns=exogenous_feature_columns), - optimizer=adam.AdamOptimizer(0.001), - config=estimator_lib.RunConfig(tf_random_seed=4), - state_manager=state_management.ChainingStateManager(), - head_type=head_type, - model_dir=model_dir) - - -def _structural_ensemble_regressor( - model_dir, head_type, exogenous_feature_columns): - return ts_estimators.StructuralEnsembleRegressor( - periodicities=None, - num_features=5, - exogenous_feature_columns=exogenous_feature_columns, - head_type=head_type, - model_dir=model_dir) - - -def _ar_lstm_regressor( - model_dir, head_type, exogenous_feature_columns): - return ts_estimators.TimeSeriesRegressor( - model=ar_model.ARModel( - periodicities=10, input_window_size=10, output_window_size=6, - num_features=5, - exogenous_feature_columns=exogenous_feature_columns, - prediction_model_factory=functools.partial( - ar_model.LSTMPredictionModel, - num_units=10)), - head_type=head_type, - model_dir=model_dir) - - -class OneShotTests(parameterized.TestCase): - - @parameterized.named_parameters( - {"testcase_name": "ar_lstm_regressor", - "estimator_factory": _ar_lstm_regressor}, - {"testcase_name": "custom_time_series_regressor", - "estimator_factory": _custom_time_series_regressor}, - {"testcase_name": "structural_ensemble_regressor", - "estimator_factory": _structural_ensemble_regressor}) - def test_one_shot_prediction_head_export(self, estimator_factory): - def _new_temp_dir(): - return os.path.join(test.get_temp_dir(), str(ops.uid())) - model_dir = _new_temp_dir() - categorical_column = feature_column.categorical_column_with_hash_bucket( - key="categorical_exogenous_feature", hash_bucket_size=16) - exogenous_feature_columns = [ - feature_column.numeric_column( - "2d_exogenous_feature", shape=(2,)), - feature_column.embedding_column( - categorical_column=categorical_column, dimension=10)] - estimator = estimator_factory( - model_dir=model_dir, - exogenous_feature_columns=exogenous_feature_columns, - head_type=ts_head_lib.OneShotPredictionHead) - train_features = { - feature_keys.TrainEvalFeatures.TIMES: numpy.arange( - 20, dtype=numpy.int64), - feature_keys.TrainEvalFeatures.VALUES: numpy.tile(numpy.arange( - 20, dtype=numpy.float32)[:, None], [1, 5]), - "2d_exogenous_feature": numpy.ones([20, 2]), - "categorical_exogenous_feature": numpy.array( - ["strkey"] * 20)[:, None] - } - train_input_fn = input_pipeline.RandomWindowInputFn( - input_pipeline.NumpyReader(train_features), shuffle_seed=2, - num_threads=1, batch_size=16, window_size=16) - estimator.train(input_fn=train_input_fn, steps=5) - result = estimator.evaluate(input_fn=train_input_fn, steps=1) - self.assertIn("average_loss", result) - self.assertNotIn(feature_keys.State.STATE_TUPLE, result) - input_receiver_fn = estimator.build_raw_serving_input_receiver_fn() - export_location = estimator.export_saved_model(_new_temp_dir(), - input_receiver_fn) - graph = ops.Graph() - with graph.as_default(): - with session_lib.Session() as session: - signatures = loader.load( - session, [tag_constants.SERVING], export_location) - self.assertEqual([feature_keys.SavedModelLabels.PREDICT], - list(signatures.signature_def.keys())) - predict_signature = signatures.signature_def[ - feature_keys.SavedModelLabels.PREDICT] - six.assertCountEqual( - self, - [feature_keys.FilteringFeatures.TIMES, - feature_keys.FilteringFeatures.VALUES, - "2d_exogenous_feature", - "categorical_exogenous_feature"], - predict_signature.inputs.keys()) - features = { - feature_keys.TrainEvalFeatures.TIMES: numpy.tile( - numpy.arange(35, dtype=numpy.int64)[None, :], [2, 1]), - feature_keys.TrainEvalFeatures.VALUES: numpy.tile(numpy.arange( - 20, dtype=numpy.float32)[None, :, None], [2, 1, 5]), - "2d_exogenous_feature": numpy.ones([2, 35, 2]), - "categorical_exogenous_feature": numpy.tile(numpy.array( - ["strkey"] * 35)[None, :, None], [2, 1, 1]) - } - feeds = { - graph.as_graph_element(input_value.name): features[input_key] - for input_key, input_value in predict_signature.inputs.items()} - fetches = {output_key: graph.as_graph_element(output_value.name) - for output_key, output_value - in predict_signature.outputs.items()} - output = session.run(fetches, feed_dict=feeds) - self.assertEqual((2, 15, 5), output["mean"].shape) - # Build a parsing input function, then make a tf.Example for it to parse. - export_location = estimator.export_saved_model( - _new_temp_dir(), - estimator.build_one_shot_parsing_serving_input_receiver_fn( - filtering_length=20, prediction_length=15)) - graph = ops.Graph() - with graph.as_default(): - with session_lib.Session() as session: - example = example_pb2.Example() - times = example.features.feature[feature_keys.TrainEvalFeatures.TIMES] - values = example.features.feature[feature_keys.TrainEvalFeatures.VALUES] - times.int64_list.value.extend(range(35)) - for i in range(20): - values.float_list.value.extend( - [float(i) * 2. + feature_number - for feature_number in range(5)]) - real_feature = example.features.feature["2d_exogenous_feature"] - categortical_feature = example.features.feature[ - "categorical_exogenous_feature"] - for i in range(35): - real_feature.float_list.value.extend([1, 1]) - categortical_feature.bytes_list.value.append(b"strkey") - # Serialize the tf.Example for feeding to the Session - examples = [example.SerializeToString()] * 2 - signatures = loader.load( - session, [tag_constants.SERVING], export_location) - predict_signature = signatures.signature_def[ - feature_keys.SavedModelLabels.PREDICT] - ((_, input_value),) = predict_signature.inputs.items() - feeds = {graph.as_graph_element(input_value.name): examples} - fetches = {output_key: graph.as_graph_element(output_value.name) - for output_key, output_value - in predict_signature.outputs.items()} - output = session.run(fetches, feed_dict=feeds) - self.assertEqual((2, 15, 5), output["mean"].shape) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/input_pipeline.py b/tensorflow/contrib/timeseries/python/timeseries/input_pipeline.py deleted file mode 100644 index f9259a78393..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/input_pipeline.py +++ /dev/null @@ -1,888 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Defines ways of splicing and re-arranging time series. - -This file provides methods for reading, parsing, and re-arranging a time -series. The main departure from standard TensorFlow input pipelines is a focus -on "chunking" a time series, i.e. slicing it into small contiguous windows which -are then batched together for training, a form of truncated -backpropagation. This typically provides a significant speedup compared to -looping over the whole series sequentially, by exploiting data parallelism and -by reducing redundant contributions to gradients (due to redundant information -in the series itself). - -A series, consisting of times (an increasing vector of integers) and values (one -or more floating point values for each time) along with any exogenous features, -is stored either in memory or on disk in various formats (e.g. "one record per -timestep" on disk, or as a dictionary of Numpy arrays in memory). The location -and format is specified by configuring a `TimeSeriesReader` object -(e.g. `NumpyReader`, `CSVReader`), which reads the data into the TensorFlow -graph. A `TimeSeriesInputFn` object (typically `RandomWindowInputFn`) then -performs windowing and batching. - -Time series are passed through this pipeline as dictionaries mapping feature -names to their values. For training and evaluation, these require at minimum -`TrainEvalFeatures.TIMES` (scalar integers, one per timestep) and -`TrainEvalFeatures.VALUES` (may be either univariate or multivariate). Exogenous -features may have any shape, but are likewise associated with a timestep. Times -themselves need not be contiguous or regular (although smaller/fewer gaps are -generally better), but each timestep must have all `VALUES` and any exogenous -features (i.e. times may be missing, but given that a time is specified, every -other feature must also be specified for that step; some models may support -making exogenous updates conditional). - -The expected use case of a `TimeSeriesInputFn` is that it is first configured -(for example setting a batch or window size) and passed a reader (a -`TimeSeriesReader` object). The `TimeSeriesInputFn` can then be passed as the -input_fn of an Estimator. - -For example, `RandomWindowInputFn` is useful for creating batches of random -chunks of a series for training: - -``` - # Read data in the default "time,value" CSV format with no header - reader = input_pipeline.CSVReader(csv_file_name) - # Set up windowing and batching for training - train_input_fn = input_pipeline.RandomWindowInputFn( - reader, batch_size=16, window_size=16) - # Fit model parameters to data - estimator.train(input_fn=train_input_fn, steps=150) -``` - -`RandomWindowInputFn` is the primary tool for training and quantitative -evaluation of time series. `WholeDatasetInputFn`, which reads a whole series -into memory, is useful for qualitative evaluation and preparing to make -predictions with `predict_continuation_input_fn`. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import numpy - -from tensorflow.contrib.timeseries.python.timeseries import feature_keys -from tensorflow.contrib.timeseries.python.timeseries import model_utils - -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import io_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import tensor_array_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.training import input as input_lib -from tensorflow.python.training import training -from tensorflow.python.util import nest - - -def predict_continuation_input_fn(evaluation, - steps=None, - times=None, - exogenous_features=None): - """An Estimator input_fn for running predict() after evaluate(). - - If the call to evaluate() we are making predictions based on had a batch_size - greater than one, predictions will start after each of these windows - (i.e. will have the same batch dimension). - - Args: - evaluation: The dictionary returned by `Estimator.evaluate`, with keys - FilteringResults.STATE_TUPLE and FilteringResults.TIMES. - steps: The number of steps to predict (scalar), starting after the - evaluation. If `times` is specified, `steps` must not be; one is required. - times: A [batch_size x window_size] array of integers (not a Tensor) - indicating times to make predictions for. These times must be after the - corresponding evaluation. If `steps` is specified, `times` must not be; - one is required. If the batch dimension is omitted, it is assumed to be 1. - exogenous_features: Optional dictionary. If specified, indicates exogenous - features for the model to use while making the predictions. Values must - have shape [batch_size x window_size x ...], where `batch_size` matches - the batch dimension used when creating `evaluation`, and `window_size` is - either the `steps` argument or the `window_size` of the `times` argument - (depending on which was specified). - - Returns: - An `input_fn` suitable for passing to the `predict` function of a time - series `Estimator`. - Raises: - ValueError: If `times` or `steps` are misspecified. - """ - if exogenous_features is None: - exogenous_features = {} - predict_times = model_utils.canonicalize_times_or_steps_from_output( - times=times, steps=steps, previous_model_output=evaluation) - features = { - feature_keys.PredictionFeatures.STATE_TUPLE: - evaluation[feature_keys.FilteringResults.STATE_TUPLE], - feature_keys.PredictionFeatures.TIMES: - predict_times - } - features.update(exogenous_features) - - def _predict_input_fn(): - """An input_fn for predict().""" - # Prevents infinite iteration with a constant output in an Estimator's - # predict(). - limited_features = {} - for key, values in features.items(): - limited_values = nest.map_structure( - lambda value: training.limit_epochs(value, num_epochs=1), values) - limited_features[key] = limited_values - return (limited_features, None) - - return _predict_input_fn - - -class TimeSeriesReader(object): - """Reads from and parses a data source for a `TimeSeriesInputFn`. - - This class provides methods that read a few records (`read`) or the full data - set at once (`read_full`), and returns them as dictionaries mapping feature - names to feature Tensors. Please see note at the top of the file for the - structure of these dictionaries. The output is generally chunked by a - `TimeSeriesInputFn` before being passed to the model. - """ - - def check_dataset_size(self, minimum_dataset_size): - """When possible, raises an error if the dataset is too small. - - This method allows TimeSeriesReaders to raise informative error messages if - the user has selected a window size in their TimeSeriesInputFn which is - larger than the dataset size. However, many TimeSeriesReaders will not have - access to a dataset size, in which case they do not need to override this - method. - - Args: - minimum_dataset_size: The minimum number of records which should be - contained in the dataset. Readers should attempt to raise an error when - possible if an epoch of data contains fewer records. - """ - pass - - @abc.abstractmethod - def read(self): - """Parses one or more records into a feature dictionary. - - This method is expected to be called by a `TimeSeriesInputFn` object, and is - not for use with models directly. - - A `TimeSeriesReader` object reads multiple records at a single time for - efficiency; the size of these batches is an implementation detail internal - to the input pipeline. These records should generally be sequential, - although some out-of-order records due to file wraparounds are expected and - must be handled by callers. - - Returns: - A dictionary mapping feature names to `Tensor` values, each with an - arbitrary batch dimension (for efficiency) as their first dimension. - """ - pass - - @abc.abstractmethod - def read_full(self): - """Return the full dataset. - - Largely for interactive use/plotting (or evaluation on small - datasets). Generally not very efficient. Not recommended for training. - - Returns: - Same return type as `read`, but with the full dataset rather than an - arbitrary chunk of it. A dictionary mapping feature names to `Tensor` - values, where the size of the first dimension of each `Tensor` is the - number of samples in the entire dataset. These `Tensor`s should be - constant across graph invocations, assuming that the underlying data - remains constant. Current implementations re-read data on each graph - invocation, although this may change in the future. - """ - pass - - -class NumpyReader(TimeSeriesReader): - """A time series parser for feeding Numpy arrays to a `TimeSeriesInputFn`. - - Avoids embedding data in the graph as constants. - """ - - def __init__(self, data, read_num_records_hint=4096): - """Numpy array input for a `TimeSeriesInputFn`. - - Args: - data: A dictionary mapping feature names to Numpy arrays, with two - possible shapes (requires keys `TrainEvalFeatures.TIMES` and - `TrainEvalFeatures.VALUES`): Univariate; `TIMES` and `VALUES` are both - vectors of shape [series length] Multivariate; `TIMES` is a vector of - shape [series length], `VALUES` has shape [series length x number of - features]. In any case, `VALUES` and any exogenous features must have - their shapes prefixed by the shape of the value corresponding to the - `TIMES` key. - read_num_records_hint: The maximum number of samples to read at one time, - for efficiency. - """ - self._features = _canonicalize_numpy_data(data, require_single_batch=True) - self._read_num_records_hint = read_num_records_hint - - def check_dataset_size(self, minimum_dataset_size): - """Raise an error if the dataset is too small.""" - dataset_size = self._features[feature_keys.TrainEvalFeatures.TIMES].shape[1] - if dataset_size < minimum_dataset_size: - raise ValueError( - ("A TimeSeriesInputFn is configured to create windows of size {}, " - "but only {} records were available in the dataset. Either decrease " - "the window size or provide more records.").format( - minimum_dataset_size, dataset_size)) - - def read(self): - """Returns a large chunk of the Numpy arrays for later re-chunking.""" - # Remove the batch dimension from all features - features = { - key: numpy.squeeze(value, axis=0) - for key, value in self._features.items() - } - return estimator_lib.inputs.numpy_input_fn( - x=features, - # The first dimensions of features are the series length, since we have - # removed the batch dimension above. We now pull out - # self._read_num_records_hint steps of this single time series to pass - # to the TimeSeriesInputFn. - batch_size=self._read_num_records_hint, - num_epochs=None, - shuffle=False)() - - def read_full(self): - """Returns `Tensor` versions of the full Numpy arrays.""" - features = estimator_lib.inputs.numpy_input_fn( - x=self._features, - batch_size=1, - num_epochs=None, - queue_capacity=2, # Each queue element is a full copy of the dataset - shuffle=False)() - # TimeSeriesInputFn expect just a batch dimension - return { - feature_name: array_ops.squeeze(feature_value, axis=0) - for feature_name, feature_value in features.items() - } - - -class ReaderBaseTimeSeriesParser(TimeSeriesReader): - """Base for time series readers which wrap a `tf.compat.v1.ReaderBase`.""" - - def __init__(self, filenames, read_num_records_hint=4096): - """Configure the time series reader. - - Args: - filenames: A string or list of strings indicating files to read records - from. - read_num_records_hint: When not reading a full dataset, indicates the - number of records to transfer in a single chunk (for efficiency). The - actual number transferred at one time may vary. - """ - self._filenames = filenames - self._read_num_records_hint = read_num_records_hint - - @abc.abstractmethod - def _get_reader(self): - """Get an instance of the tf.compat.v1.ReaderBase associated with this class.""" - pass - - @abc.abstractmethod - def _process_records(self, lines): - """Given string items, return a processed dictionary of Tensors. - - Args: - lines: A 1-dimensional string Tensor, each representing a record to parse - (source dependent, e.g. a line of a file, or a serialized protocol - buffer). - - Returns: - A dictionary mapping feature names to their values. The batch dimensions - should match the length of `lines`. - """ - pass - - def _get_filename_queue(self, epoch_limit): - """Constructs a filename queue with an epoch limit. - - `epoch_limit` is intended as an error checking fallback to prevent a reader - from infinitely looping in its requests for more work items if none are - available in any file. It should be set high enough that it is never reached - assuming at least one record exists in some file. - - Args: - epoch_limit: The maximum number of times to read through the complete list - of files before throwing an OutOfRangeError. - - Returns: - A tuple of (filename_queue, epoch_limiter): - filename_queue: A FIFOQueue with filename work items. - epoch_limiter: The local variable used for epoch limitation. This should - be set to zero before a reader is passed `filename_queue` in order to - reset the epoch limiter's state. - """ - epoch_limiter = variable_scope.variable( - initial_value=constant_op.constant(0, dtype=dtypes.int64), - name="epoch_limiter", - trainable=False, - collections=[ops.GraphKeys.LOCAL_VARIABLES]) - filenames_tensor = array_ops.reshape( - ops.convert_to_tensor(self._filenames), [-1]) - # We can't rely on epoch_limiter being initialized, since queue runners are - # started before local variables are initialized. Instead, we ignore epoch - # limits before variable initialization. This means that prior to variable - # initialization, a QueueRunner may cause a reader to enter an un-checked - # infinite loop. However, as soon as local variables are initialized, we - # will start incrementing and checking epoch_limiter, which will interrupt - # any in-progress loops. - conditional_count_up_to = control_flow_ops.cond( - state_ops.is_variable_initialized( - epoch_limiter), lambda: epoch_limiter.count_up_to(epoch_limit), - lambda: constant_op.constant(0, dtype=dtypes.int64)) - with ops.control_dependencies([conditional_count_up_to]): - filenames_tensor = array_ops.identity(filenames_tensor) - filename_queue = input_lib.string_input_producer( - filenames_tensor, shuffle=False, capacity=1) - return filename_queue, epoch_limiter - - def read(self): - """Reads a chunk of data from the `tf.compat.v1.ReaderBase` for later re-chunking.""" - # Assuming there is at least one item to be read among all of the files in - # self._filenames, we will not need to go through more than - # self._read_num_records_hint epochs to get a batch of - # self._read_num_records_hint records. Setting this limit and resetting it - # before each reader.read_up_to call prevents infinite looping when there - # are no records available in any of the files. - filename_queue, epoch_limiter = self._get_filename_queue( - epoch_limit=self._read_num_records_hint) - reader = self._get_reader() - epoch_reset_op = state_ops.assign(epoch_limiter, 0) - with ops.control_dependencies([epoch_reset_op]): - _, records = reader.read_up_to(filename_queue, - self._read_num_records_hint) - return self._process_records(records) - - def read_full(self): - """Reads a full epoch of data into memory.""" - reader = self._get_reader() - # Set a hard limit of 2 epochs through self._filenames. If there are any - # records available, we should only end up reading the first record in the - # second epoch before exiting the while loop and subsequently resetting the - # epoch limit. If there are no records available in any of the files, this - # hard limit prevents the reader.read_up_to call from looping infinitely. - filename_queue, epoch_limiter = self._get_filename_queue(epoch_limit=2) - epoch_reset_op = state_ops.assign(epoch_limiter, 0) - with ops.control_dependencies([epoch_reset_op]): - first_key, first_value = reader.read_up_to(filename_queue, 1) - # Read until we get a duplicate key (one epoch) - def _while_condition(current_key, current_value, current_index, - collected_records): - del current_value, current_index, collected_records # unused - return math_ops.not_equal( - array_ops.squeeze(current_key, axis=0), - array_ops.squeeze(first_key, axis=0)) - - def _while_body(current_key, current_value, current_index, - collected_records): - del current_key # unused - new_key, new_value = reader.read_up_to(filename_queue, 1) - new_key.set_shape([1]) - new_value.set_shape([1]) - return (new_key, new_value, current_index + 1, - collected_records.write(current_index, current_value)) - - _, _, _, records_ta = control_flow_ops.while_loop( - _while_condition, - _while_body, - [ - constant_op.constant([""]), - first_value, - 0, # current_index starting value - tensor_array_ops.TensorArray( # collected_records - dtype=dtypes.string, - size=0, - dynamic_size=True) - ]) - records = records_ta.concat() - # Reset the reader when we're done so that subsequent requests for data get - # the dataset in the proper order. - with ops.control_dependencies([records]): - reader_reset_op = reader.reset() - with ops.control_dependencies([reader_reset_op]): - records = array_ops.identity(records) - return self._process_records(records) - - -class CSVReader(ReaderBaseTimeSeriesParser): - """Reads from a collection of CSV-formatted files.""" - - def __init__(self, - filenames, - column_names=(feature_keys.TrainEvalFeatures.TIMES, - feature_keys.TrainEvalFeatures.VALUES), - column_dtypes=None, - skip_header_lines=None, - read_num_records_hint=4096): - """CSV-parsing reader for a `TimeSeriesInputFn`. - - Args: - filenames: A filename or list of filenames to read the time series from. - Each line must have columns corresponding to `column_names`. - column_names: A list indicating names for each feature. - `TrainEvalFeatures.TIMES` and `TrainEvalFeatures.VALUES` are required; - `VALUES` may be repeated to indicate a multivariate series. - column_dtypes: If provided, must be a list with the same length as - `column_names`, indicating dtypes for each column. Defaults to - `tf.int64` for `TrainEvalFeatures.TIMES` and `tf.float32` for everything - else. - skip_header_lines: Passed on to `tf.compat.v1.TextLineReader`; skips this - number of lines at the beginning of each file. - read_num_records_hint: When not reading a full dataset, indicates the - number of records to parse/transfer in a single chunk (for efficiency). - The actual number transferred at one time may be more or less. - - Raises: - ValueError: If required column names are not specified, or if lengths do - not match. - """ - if feature_keys.TrainEvalFeatures.TIMES not in column_names: - raise ValueError("'{}' is a required column.".format( - feature_keys.TrainEvalFeatures.TIMES)) - if feature_keys.TrainEvalFeatures.VALUES not in column_names: - raise ValueError("'{}' is a required column.".format( - feature_keys.TrainEvalFeatures.VALUES)) - if column_dtypes is not None and len(column_dtypes) != len(column_names): - raise ValueError( - ("If specified, the length of column_dtypes must match the length of " - "column_names (got column_dtypes={} and column_names={}).").format( - column_dtypes, column_names)) - if sum(1 for column_name in column_names - if column_name == feature_keys.TrainEvalFeatures.TIMES) != 1: - raise ValueError("Got more than one times column ('{}'), but exactly " - "one is required.".format( - feature_keys.TrainEvalFeatures.TIMES)) - self._column_names = column_names - self._column_dtypes = column_dtypes - self._skip_header_lines = skip_header_lines - super(CSVReader, self).__init__( - filenames=filenames, read_num_records_hint=read_num_records_hint) - - def _get_reader(self): - return io_ops.TextLineReader(skip_header_lines=self._skip_header_lines) - - def _process_records(self, lines): - """Parse `lines` as CSV records.""" - if self._column_dtypes is None: - default_values = [(array_ops.zeros([], dtypes.int64),) if - column_name == feature_keys.TrainEvalFeatures.TIMES else - () for column_name in self._column_names] - else: - default_values = [ - (array_ops.zeros([], dtype),) for dtype in self._column_dtypes - ] - columns = parsing_ops.decode_csv(lines, default_values) - features_lists = {} - for column_name, value in zip(self._column_names, columns): - features_lists.setdefault(column_name, []).append(value) - features = {} - for column_name, values in features_lists.items(): - if column_name == feature_keys.TrainEvalFeatures.TIMES: - features[column_name] = values[0] - else: - features[column_name] = array_ops.stack(values, axis=1) - return features - - -class TFExampleReader(ReaderBaseTimeSeriesParser): - """Reads and parses `tf.Example`s from a TFRecords file.""" - - def __init__(self, filenames, features): - """Configure `tf.Example` parsing. - - Args: - filenames: A filename or list of filenames to read the time series from. - Each line must have columns corresponding to `column_names`. - features: A dictionary mapping from feature keys to - `tf.io.FixedLenFeature` objects. Must include `TrainEvalFeatures.TIMES` - (scalar integer) and `TrainEvalFeatures.VALUES` (floating point vector) - features. - - Raises: - ValueError: If required times/values features are not present. - """ - if feature_keys.TrainEvalFeatures.TIMES not in features: - raise ValueError("'{}' is a required column.".format( - feature_keys.TrainEvalFeatures.TIMES)) - if feature_keys.TrainEvalFeatures.VALUES not in features: - raise ValueError("'{}' is a required column.".format( - feature_keys.TrainEvalFeatures.VALUES)) - self._features = features - super(TFExampleReader, self).__init__(filenames=filenames) - - def _get_reader(self): - return io_ops.TFRecordReader() - - def _process_records(self, examples): - """Parse `tf.Example`s into `Tensors`.""" - return parsing_ops.parse_example( - serialized=examples, features=self._features) - - -class TimeSeriesInputFn(object): - """Base for classes which create batches of windows from a time series.""" - - @abc.abstractmethod - def create_batch(self): - """Creates chunked Tensors from times, values, and other features. - - Suitable for use as the input_fn argument of a tf.estimator.Estimator's - fit() or evaluate() method. - - Returns: - A tuple of (features, targets): - features: A dictionary with `TrainEvalFeatures.TIMES` and - `TrainEvalFeatures.VALUES` as keys, `TIMES` having an associated value - with shape [batch size x window length], `VALUES` with shape [batch - size x window length x number of features]. Any other features will - also have shapes prefixed with [batch size x window length]. - targets: Not used, but must have a value for compatibility with the - Estimator API. That value should be None. - """ - pass - - def __call__(self): - # Allow a TimeSeriesInputFn to be used as an input function directly - return self.create_batch() - - -class WholeDatasetInputFn(TimeSeriesInputFn): - """Supports passing a full time series to a model for evaluation/inference. - - Note that this `TimeSeriesInputFn` is not designed for high throughput, and - should not be used for training. It allows for sequential evaluation on a full - dataset (with sequential in-sample predictions), which then feeds naturally - into `predict_continuation_input_fn` for making out-of-sample - predictions. While this is useful for plotting and interactive use, - `RandomWindowInputFn` is better suited to training and quantitative - evaluation. - """ - - # TODO(allenl): A SequentialWindowInputFn for getting model end state without - # loading the whole dataset into memory (or for quantitative evaluation of - # sequential models). Note that an Estimator using such a TimeSeriesInputFn - # won't return in-sample predictions for the whole dataset, which means it - # won't be terribly useful for interactive use/plotting (unless the user - # passes in concat metrics). Also need to be careful about state saving for - # sequential models, particularly the gaps between chunks. - - def __init__(self, time_series_reader): - """Initialize the `TimeSeriesInputFn`. - - Args: - time_series_reader: A TimeSeriesReader object. - """ - self._reader = time_series_reader - super(WholeDatasetInputFn, self).__init__() - - def create_batch(self): - """A suitable `input_fn` for an `Estimator`'s `evaluate()`. - - Returns: - A dictionary mapping feature names to `Tensors`, each shape - prefixed by [1, data set size] (i.e. a batch size of 1). - """ - features = self._reader.read_full() - # Add a batch dimension of one to each feature. - return ({ - feature_name: feature_value[None, ...] - for feature_name, feature_value in features.items() - }, None) - - -class RandomWindowInputFn(TimeSeriesInputFn): - """Wraps a `TimeSeriesReader` to create random batches of windows. - - Tensors are first collected into sequential windows (in a windowing queue - created by `tf.compat.v1.train.batch`, based on the order returned from - `time_series_reader`), then these windows are randomly batched (in a - `RandomShuffleQueue`), the Tensors returned by `create_batch` having shapes - prefixed by [`batch_size`, `window_size`]. - - This `TimeSeriesInputFn` is useful for both training and quantitative - evaluation (but be sure to run several epochs for sequential models such as - `StructuralEnsembleRegressor` to completely flush stale state left over from - training). For qualitative evaluation or when preparing for predictions, use - `WholeDatasetInputFn`. - """ - - def __init__(self, - time_series_reader, - window_size, - batch_size, - queue_capacity_multiplier=1000, - shuffle_min_after_dequeue_multiplier=2, - discard_out_of_order=True, - discard_consecutive_batches_limit=1000, - jitter=True, - num_threads=2, - shuffle_seed=None): - """Configure the RandomWindowInputFn. - - Args: - time_series_reader: A TimeSeriesReader object. - window_size: The number of examples to keep together sequentially. This - controls the length of truncated backpropagation: smaller values mean - less sequential computation, which can lead to faster training, but - create a coarser approximation to the gradient (which would ideally be - computed by a forward pass over the entire sequence in order). - batch_size: The number of windows to place together in a batch. Larger - values will lead to more stable gradients during training. - queue_capacity_multiplier: The capacity for the queues used to create - batches, specified as a multiple of `batch_size` (for - RandomShuffleQueue) and `batch_size * window_size` (for the FIFOQueue). - Controls the maximum number of windows stored. Should be greater than - `shuffle_min_after_dequeue_multiplier`. - shuffle_min_after_dequeue_multiplier: The minimum number of windows in the - RandomShuffleQueue after a dequeue, which controls the amount of entropy - introduced during batching. Specified as a multiple of `batch_size`. - discard_out_of_order: If True, windows of data which have times which - decrease (a higher time followed by a lower time) are discarded. If - False, the window and associated features are instead sorted so that - times are non-decreasing. Discarding is typically faster, as models do - not have to deal with artificial gaps in the data. However, discarding - does create a bias where the beginnings and endings of files are - under-sampled. - discard_consecutive_batches_limit: Raise an OutOfRangeError if more than - this number of batches are discarded without a single non-discarded - window (prevents infinite looping when the dataset is too small). - jitter: If True, randomly discards examples between some windows in order - to avoid deterministic chunking patterns. This is important for models - like AR which may otherwise overfit a fixed chunking. - num_threads: Use this number of threads for queues. Setting a value of 1 - removes one source of non-determinism (and in combination with - shuffle_seed should provide deterministic windowing). - shuffle_seed: A seed for window shuffling. The default value of None - provides random behavior. With `shuffle_seed` set and `num_threads=1`, - provides deterministic behavior. - """ - self._reader = time_series_reader - self._window_size = window_size - self._reader.check_dataset_size(minimum_dataset_size=self._window_size) - self._batch_size = batch_size - self._queue_capacity_multiplier = queue_capacity_multiplier - self._shuffle_min_after_dequeue_multiplier = ( - shuffle_min_after_dequeue_multiplier) - self._discard_out_of_order = discard_out_of_order - self._discard_limit = discard_consecutive_batches_limit - self._jitter = jitter - if num_threads is None: - self._num_threads = self._batch_size - else: - self._num_threads = num_threads - self._shuffle_seed = shuffle_seed - super(RandomWindowInputFn, self).__init__() - - def create_batch(self): - """Create queues to window and batch time series data. - - Returns: - A dictionary of Tensors corresponding to the output of `self._reader` - (from the `time_series_reader` constructor argument), each with shapes - prefixed by [`batch_size`, `window_size`]. - """ - features = self._reader.read() - if self._jitter: - # TODO(agarwal, allenl): Figure out if more jitter is needed here. - jitter = random_ops.random_uniform(shape=[], maxval=2, dtype=dtypes.int32) - else: - jitter = 0 - # To keep things efficient, we pass from the windowing batcher to the - # batch-of-windows batcher in batches. This avoids the need for huge numbers - # of threads, but does mean that jitter is only applied occasionally. - # TODO(allenl): Experiment with different internal passing sizes. - internal_passing_size = self._batch_size - features_windowed = input_lib.batch( - features, - batch_size=self._window_size * internal_passing_size + jitter, - enqueue_many=True, - capacity=(self._queue_capacity_multiplier * internal_passing_size * - self._window_size), - num_threads=self._num_threads) - raw_features_windowed = features_windowed - if self._jitter: - features_windowed = { - key: value[jitter:] for key, value in features_windowed.items() - } - features_windowed = { - key: array_ops.reshape( - value, - array_ops.concat([[internal_passing_size, self._window_size], - array_ops.shape(value)[1:]], - axis=0)) - for key, value in features_windowed.items() - } - batch_and_window_shape = tensor_shape.TensorShape( - [internal_passing_size, self._window_size]) - for key in features_windowed.keys(): - features_windowed[key].set_shape( - batch_and_window_shape.concatenate( - raw_features_windowed[key].get_shape()[1:])) - # When switching files, we may end up with windows where the time is not - # decreasing, even if times within each file are sorted (and even if those - # files are visited in order, when looping back around to the beginning of - # the first file). This is hard for models to deal with, so we either - # discard such examples, creating a bias where the beginning and end of the - # series is under-sampled, or we sort the window, creating large gaps. - times = features_windowed[feature_keys.TrainEvalFeatures.TIMES] - if self._discard_out_of_order: - non_decreasing = math_ops.reduce_all( - times[:, 1:] >= times[:, :-1], axis=1) - # Ensure that no more than self._discard_limit complete batches are - # discarded contiguously (resetting the count when we find a single clean - # window). This prevents infinite looping when the dataset is smaller than - # the window size. - # TODO(allenl): Figure out a way to return informative errors from - # count_up_to. - discarded_windows_limiter = variable_scope.variable( - initial_value=constant_op.constant(0, dtype=dtypes.int64), - name="discarded_windows_limiter", - trainable=False, - collections=[ops.GraphKeys.LOCAL_VARIABLES]) - - def _initialized_limit_check(): - return control_flow_ops.cond( - math_ops.reduce_any(non_decreasing), - lambda: state_ops.assign(discarded_windows_limiter, 0), - lambda: discarded_windows_limiter.count_up_to(self._discard_limit)) - discard_limit_op = control_flow_ops.cond( - state_ops.is_variable_initialized(discarded_windows_limiter), - _initialized_limit_check, - lambda: constant_op.constant(0, dtype=dtypes.int64)) - with ops.control_dependencies([discard_limit_op]): - non_decreasing = array_ops.identity(non_decreasing) - else: - _, indices_descending = nn.top_k( - times, k=array_ops.shape(times)[-1], sorted=True) - indices = array_ops.reverse(indices_descending, axis=[0]) - features_windowed = { - key: array_ops.gather(params=value, indices=indices) - for key, value in features_windowed.items() - } - non_decreasing = True - features_batched = input_lib.maybe_shuffle_batch( - features_windowed, - num_threads=self._num_threads, - seed=self._shuffle_seed, - batch_size=self._batch_size, - capacity=self._queue_capacity_multiplier * self._batch_size, - min_after_dequeue=(self._shuffle_min_after_dequeue_multiplier * - self._batch_size), - keep_input=non_decreasing, - enqueue_many=True) - return (features_batched, None) - - -def _canonicalize_numpy_data(data, require_single_batch): - """Do basic checking and reshaping for Numpy data. - - Args: - data: A dictionary mapping keys to Numpy arrays, with several possible - shapes (requires keys `TrainEvalFeatures.TIMES` and - `TrainEvalFeatures.VALUES`): Single example; `TIMES` is a scalar and - `VALUES` is either a scalar or a vector of length [number of features]. - Sequence; `TIMES` is a vector of shape [series length], `VALUES` either - has shape [series length] (univariate) or [series length x number of - features] (multivariate). Batch of sequences; `TIMES` is a vector of - shape [batch size x series length], `VALUES` has shape [batch size x - series length] or [batch size x series length x number of features]. In - any case, `VALUES` and any exogenous features must have their shapes - prefixed by the shape of the value corresponding to the `TIMES` key. - require_single_batch: If True, raises an error if the provided data has a - batch dimension > 1. - - Returns: - A dictionary with features normalized to have shapes prefixed with [batch - size x series length]. The sizes of dimensions which were omitted in the - inputs are 1. - Raises: - ValueError: If dimensions are incorrect or do not match, or required - features are missing. - """ - features = {key: numpy.array(value) for key, value in data.items()} - if (feature_keys.TrainEvalFeatures.TIMES not in features or - feature_keys.TrainEvalFeatures.VALUES not in features): - raise ValueError("{} and {} are required features.".format( - feature_keys.TrainEvalFeatures.TIMES, - feature_keys.TrainEvalFeatures.VALUES)) - times = features[feature_keys.TrainEvalFeatures.TIMES] - for key, value in features.items(): - if value.shape[:len(times.shape)] != times.shape: - raise ValueError( - ("All features must have their shapes prefixed by the shape of the" - " times feature. Got shape {} for feature '{}', but shape {} for" - " '{}'").format(value.shape, key, times.shape, - feature_keys.TrainEvalFeatures.TIMES)) - if not times.shape: # a single example - if not features[feature_keys.TrainEvalFeatures.VALUES].shape: # univariate - # Add a feature dimension (with one feature) - features[feature_keys.TrainEvalFeatures.VALUES] = features[ - feature_keys.TrainEvalFeatures.VALUES][..., None] - elif len(features[feature_keys.TrainEvalFeatures.VALUES].shape) > 1: - raise ValueError( - ("Got an unexpected number of dimensions for the '{}' feature." - " Was expecting at most 1 dimension" - " ([number of features]) since '{}' does not " - "have a batch or time dimension, but got shape {}").format( - feature_keys.TrainEvalFeatures.VALUES, - feature_keys.TrainEvalFeatures.TIMES, - features[feature_keys.TrainEvalFeatures.VALUES].shape)) - # Add trivial batch and time dimensions for every feature - features = {key: value[None, None, ...] for key, value in features.items()} - if len(times.shape) == 1: # shape [series length] - if len(features[feature_keys.TrainEvalFeatures.VALUES].shape - ) == 1: # shape [series length] - # Add a feature dimension (with one feature) - features[feature_keys.TrainEvalFeatures.VALUES] = features[ - feature_keys.TrainEvalFeatures.VALUES][..., None] - elif len(features[feature_keys.TrainEvalFeatures.VALUES].shape) > 2: - raise ValueError( - ("Got an unexpected number of dimensions for the '{}' feature." - " Was expecting at most 2 dimensions" - " ([series length, number of features]) since '{}' does not " - "have a batch dimension, but got shape {}").format( - feature_keys.TrainEvalFeatures.VALUES, - feature_keys.TrainEvalFeatures.TIMES, - features[feature_keys.TrainEvalFeatures.VALUES].shape)) - # Add trivial batch dimensions for every feature - features = {key: value[None, ...] for key, value in features.items()} - elif len(features[feature_keys.TrainEvalFeatures.TIMES].shape - ) != 2: # shape [batch size, series length] - raise ValueError( - ("Got an unexpected number of dimensions for times. Was expecting at " - "most two ([batch size, series length]), but got shape {}.").format( - times.shape)) - if require_single_batch: - # We don't expect input to be already batched; batching is done later - if features[feature_keys.TrainEvalFeatures.TIMES].shape[0] != 1: - raise ValueError("Got batch input, was expecting unbatched input.") - return features diff --git a/tensorflow/contrib/timeseries/python/timeseries/input_pipeline_test.py b/tensorflow/contrib/timeseries/python/timeseries/input_pipeline_test.py deleted file mode 100644 index f92148b7880..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/input_pipeline_test.py +++ /dev/null @@ -1,379 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the time series input pipeline.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import csv -import tempfile - -import numpy - -from tensorflow.contrib.timeseries.python.timeseries import input_pipeline -from tensorflow.contrib.timeseries.python.timeseries import test_utils -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import TrainEvalFeatures - -from tensorflow.core.example import example_pb2 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.lib.io import tf_record -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import coordinator as coordinator_lib -from tensorflow.python.training import queue_runner_impl - - -def _make_csv_temp_file(to_write, test_tmpdir): - _, data_file = tempfile.mkstemp(dir=test_tmpdir) - with open(data_file, "w") as f: - csvwriter = csv.writer(f) - for record in to_write: - csvwriter.writerow(record) - return data_file - - -def _make_csv_time_series(num_features, num_samples, test_tmpdir): - filename = _make_csv_temp_file( - [[i] + [float(i) * 2. + feature_number - for feature_number in range(num_features)] - for i in range(num_samples)], - test_tmpdir=test_tmpdir) - return filename - - -def _make_tfexample_series(num_features, num_samples, test_tmpdir): - _, data_file = tempfile.mkstemp(dir=test_tmpdir) - with tf_record.TFRecordWriter(data_file) as writer: - for i in range(num_samples): - example = example_pb2.Example() - times = example.features.feature[TrainEvalFeatures.TIMES] - times.int64_list.value.append(i) - values = example.features.feature[TrainEvalFeatures.VALUES] - values.float_list.value.extend( - [float(i) * 2. + feature_number - for feature_number in range(num_features)]) - writer.write(example.SerializeToString()) - return data_file - - -def _make_numpy_time_series(num_features, num_samples): - times = numpy.arange(num_samples) - values = times[:, None] * 2. + numpy.arange(num_features)[None, :] - return {TrainEvalFeatures.TIMES: times, - TrainEvalFeatures.VALUES: values} - - -class RandomWindowInputFnTests(test.TestCase): - - def _random_window_input_fn_test_template( - self, time_series_reader, window_size, batch_size, num_features, - discard_out_of_order=False): - input_fn = input_pipeline.RandomWindowInputFn( - time_series_reader=time_series_reader, - window_size=window_size, batch_size=batch_size) - result, _ = input_fn() - init_op = variables.local_variables_initializer() - with self.cached_session() as session: - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(session, coord=coordinator) - session.run(init_op) - features = session.run(result) - coordinator.request_stop() - coordinator.join() - self.assertAllEqual([batch_size, window_size], - features[TrainEvalFeatures.TIMES].shape) - for window_position in range(window_size - 1): - for batch_position in range(batch_size): - # Checks that all times are contiguous - self.assertEqual( - features[TrainEvalFeatures.TIMES][batch_position, - window_position + 1], - features[TrainEvalFeatures.TIMES][batch_position, - window_position] + 1) - self.assertAllEqual([batch_size, window_size, num_features], - features[TrainEvalFeatures.VALUES].shape) - self.assertEqual("int64", features[TrainEvalFeatures.TIMES].dtype) - for feature_number in range(num_features): - self.assertAllEqual( - features[TrainEvalFeatures.TIMES] * 2. + feature_number, - features[TrainEvalFeatures.VALUES][:, :, feature_number]) - return features - - def _test_out_of_order(self, time_series_reader, discard_out_of_order): - self._random_window_input_fn_test_template( - time_series_reader=time_series_reader, - num_features=1, window_size=2, batch_size=5, - discard_out_of_order=discard_out_of_order) - - def test_csv_sort_out_of_order(self): - filename = _make_csv_time_series(num_features=1, num_samples=50, - test_tmpdir=self.get_temp_dir()) - time_series_reader = input_pipeline.CSVReader([filename]) - self._test_out_of_order(time_series_reader, discard_out_of_order=False) - - def test_tfexample_sort_out_of_order(self): - filename = _make_tfexample_series( - num_features=1, num_samples=50, - test_tmpdir=self.get_temp_dir()) - time_series_reader = input_pipeline.TFExampleReader( - [filename], - features={ - TrainEvalFeatures.TIMES: parsing_ops.FixedLenFeature( - shape=[], dtype=dtypes.int64), - TrainEvalFeatures.VALUES: parsing_ops.FixedLenFeature( - shape=[1], dtype=dtypes.float32)}) - self._test_out_of_order(time_series_reader, discard_out_of_order=False) - - def test_numpy_sort_out_of_order(self): - data = _make_numpy_time_series(num_features=1, num_samples=50) - time_series_reader = input_pipeline.NumpyReader(data) - self._test_out_of_order(time_series_reader, discard_out_of_order=False) - - def test_csv_discard_out_of_order(self): - filename = _make_csv_time_series(num_features=1, num_samples=50, - test_tmpdir=self.get_temp_dir()) - time_series_reader = input_pipeline.CSVReader([filename]) - self._test_out_of_order(time_series_reader, discard_out_of_order=True) - - def test_csv_discard_out_of_order_window_equal(self): - filename = _make_csv_time_series(num_features=1, num_samples=3, - test_tmpdir=self.get_temp_dir()) - time_series_reader = input_pipeline.CSVReader([filename]) - self._random_window_input_fn_test_template( - time_series_reader=time_series_reader, - num_features=1, window_size=3, batch_size=5, - discard_out_of_order=True) - - def test_csv_discard_out_of_order_window_too_large(self): - filename = _make_csv_time_series(num_features=1, num_samples=2, - test_tmpdir=self.get_temp_dir()) - time_series_reader = input_pipeline.CSVReader([filename]) - with self.assertRaises(errors.OutOfRangeError): - self._random_window_input_fn_test_template( - time_series_reader=time_series_reader, - num_features=1, window_size=3, batch_size=5, - discard_out_of_order=True) - - def test_csv_no_data(self): - filename = _make_csv_time_series(num_features=1, num_samples=0, - test_tmpdir=self.get_temp_dir()) - time_series_reader = input_pipeline.CSVReader([filename]) - with self.assertRaises(errors.OutOfRangeError): - self._test_out_of_order(time_series_reader, discard_out_of_order=True) - - def test_numpy_discard_out_of_order(self): - data = _make_numpy_time_series(num_features=1, num_samples=50) - time_series_reader = input_pipeline.NumpyReader(data) - self._test_out_of_order(time_series_reader, discard_out_of_order=True) - - def test_numpy_discard_out_of_order_window_equal(self): - data = _make_numpy_time_series(num_features=1, num_samples=3) - time_series_reader = input_pipeline.NumpyReader(data) - self._random_window_input_fn_test_template( - time_series_reader=time_series_reader, - num_features=1, window_size=3, batch_size=5, - discard_out_of_order=True) - - def test_numpy_discard_out_of_order_window_too_large(self): - data = _make_numpy_time_series(num_features=1, num_samples=2) - time_series_reader = input_pipeline.NumpyReader(data) - with self.assertRaisesRegexp(ValueError, "only 2 records were available"): - self._random_window_input_fn_test_template( - time_series_reader=time_series_reader, - num_features=1, window_size=3, batch_size=5, - discard_out_of_order=True) - - def _test_multivariate(self, time_series_reader, num_features): - self._random_window_input_fn_test_template( - time_series_reader=time_series_reader, - num_features=num_features, - window_size=2, - batch_size=5) - - def test_csv_multivariate(self): - filename = _make_csv_time_series(num_features=2, num_samples=50, - test_tmpdir=self.get_temp_dir()) - time_series_reader = input_pipeline.CSVReader( - [filename], - column_names=(TrainEvalFeatures.TIMES, TrainEvalFeatures.VALUES, - TrainEvalFeatures.VALUES)) - self._test_multivariate(time_series_reader=time_series_reader, - num_features=2) - - def test_tfexample_multivariate(self): - filename = _make_tfexample_series( - num_features=2, num_samples=50, - test_tmpdir=self.get_temp_dir()) - time_series_reader = input_pipeline.TFExampleReader( - [filename], - features={ - TrainEvalFeatures.TIMES: parsing_ops.FixedLenFeature( - shape=[], dtype=dtypes.int64), - TrainEvalFeatures.VALUES: parsing_ops.FixedLenFeature( - shape=[2], dtype=dtypes.float32)}) - self._test_multivariate(time_series_reader=time_series_reader, - num_features=2) - - def test_numpy_multivariate(self): - data = _make_numpy_time_series(num_features=3, num_samples=50) - time_series_reader = input_pipeline.NumpyReader(data) - self._test_multivariate(time_series_reader, num_features=3) - - def test_numpy_withbatch(self): - data_nobatch = _make_numpy_time_series(num_features=4, num_samples=100) - data = {feature_name: feature_value[None] - for feature_name, feature_value in data_nobatch.items()} - time_series_reader = input_pipeline.NumpyReader(data) - self._random_window_input_fn_test_template( - time_series_reader=time_series_reader, - num_features=4, - window_size=3, - batch_size=5) - - def test_numpy_nobatch_nofeatures(self): - data = _make_numpy_time_series(num_features=1, num_samples=100) - data[TrainEvalFeatures.VALUES] = data[TrainEvalFeatures.VALUES][:, 0] - time_series_reader = input_pipeline.NumpyReader(data) - self._random_window_input_fn_test_template( - time_series_reader=time_series_reader, - num_features=1, - window_size=16, - batch_size=16) - - -class WholeDatasetInputFnTests(test.TestCase): - - def _whole_dataset_input_fn_test_template( - self, time_series_reader, num_features, num_samples): - result, _ = input_pipeline.WholeDatasetInputFn(time_series_reader)() - with self.cached_session() as session: - session.run(variables.local_variables_initializer()) - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(session, coord=coordinator) - features = session.run(result) - coordinator.request_stop() - coordinator.join() - self.assertEqual("int64", features[TrainEvalFeatures.TIMES].dtype) - self.assertAllEqual(numpy.arange(num_samples, dtype=numpy.int64)[None, :], - features[TrainEvalFeatures.TIMES]) - for feature_number in range(num_features): - self.assertAllEqual( - features[TrainEvalFeatures.TIMES] * 2. + feature_number, - features[TrainEvalFeatures.VALUES][:, :, feature_number]) - - def test_csv(self): - filename = _make_csv_time_series(num_features=3, num_samples=50, - test_tmpdir=self.get_temp_dir()) - time_series_reader = input_pipeline.CSVReader( - [filename], - column_names=(TrainEvalFeatures.TIMES, TrainEvalFeatures.VALUES, - TrainEvalFeatures.VALUES, TrainEvalFeatures.VALUES)) - self._whole_dataset_input_fn_test_template( - time_series_reader=time_series_reader, num_features=3, num_samples=50) - - def test_csv_no_data(self): - filename = _make_csv_time_series(num_features=1, num_samples=0, - test_tmpdir=self.get_temp_dir()) - time_series_reader = input_pipeline.CSVReader([filename]) - with self.assertRaises(errors.OutOfRangeError): - self._whole_dataset_input_fn_test_template( - time_series_reader=time_series_reader, num_features=1, num_samples=50) - - def test_tfexample(self): - filename = _make_tfexample_series( - num_features=4, num_samples=100, - test_tmpdir=self.get_temp_dir()) - time_series_reader = input_pipeline.TFExampleReader( - [filename], - features={ - TrainEvalFeatures.TIMES: parsing_ops.FixedLenFeature( - shape=[], dtype=dtypes.int64), - TrainEvalFeatures.VALUES: parsing_ops.FixedLenFeature( - shape=[4], dtype=dtypes.float32)}) - self._whole_dataset_input_fn_test_template( - time_series_reader=time_series_reader, num_features=4, num_samples=100) - - def test_numpy(self): - data = _make_numpy_time_series(num_features=4, num_samples=100) - time_series_reader = input_pipeline.NumpyReader(data) - self._whole_dataset_input_fn_test_template( - time_series_reader=time_series_reader, num_features=4, num_samples=100) - - def test_numpy_withbatch(self): - data_nobatch = _make_numpy_time_series(num_features=4, num_samples=100) - data = {feature_name: feature_value[None] - for feature_name, feature_value in data_nobatch.items()} - time_series_reader = input_pipeline.NumpyReader(data) - self._whole_dataset_input_fn_test_template( - time_series_reader=time_series_reader, num_features=4, num_samples=100) - - def test_numpy_nobatch_nofeatures(self): - data = _make_numpy_time_series(num_features=1, num_samples=100) - data[TrainEvalFeatures.VALUES] = data[TrainEvalFeatures.VALUES][:, 0] - time_series_reader = input_pipeline.NumpyReader(data) - self._whole_dataset_input_fn_test_template( - time_series_reader=time_series_reader, num_features=1, num_samples=100) - - -class AllWindowInputFnTests(test.TestCase): - - def _all_window_input_fn_test_template( - self, time_series_reader, num_samples, window_size, - original_numpy_features=None): - input_fn = test_utils.AllWindowInputFn( - time_series_reader=time_series_reader, - window_size=window_size) - features, _ = input_fn() - init_op = variables.local_variables_initializer() - with self.cached_session() as session: - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(session, coord=coordinator) - session.run(init_op) - chunked_times, chunked_values = session.run( - [features[TrainEvalFeatures.TIMES], - features[TrainEvalFeatures.VALUES]]) - coordinator.request_stop() - coordinator.join() - self.assertAllEqual([num_samples - window_size + 1, window_size], - chunked_times.shape) - if original_numpy_features is not None: - original_times = original_numpy_features[TrainEvalFeatures.TIMES] - original_values = original_numpy_features[TrainEvalFeatures.VALUES] - self.assertAllEqual(original_times, numpy.unique(chunked_times)) - self.assertAllEqual(original_values[chunked_times], - chunked_values) - - def test_csv(self): - filename = _make_csv_time_series(num_features=1, num_samples=50, - test_tmpdir=self.get_temp_dir()) - time_series_reader = input_pipeline.CSVReader( - [filename], - column_names=(TrainEvalFeatures.TIMES, TrainEvalFeatures.VALUES)) - self._all_window_input_fn_test_template( - time_series_reader=time_series_reader, num_samples=50, window_size=10) - - def test_numpy(self): - data = _make_numpy_time_series(num_features=2, num_samples=31) - time_series_reader = input_pipeline.NumpyReader(data) - self._all_window_input_fn_test_template( - time_series_reader=time_series_reader, original_numpy_features=data, - num_samples=31, window_size=5) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py deleted file mode 100644 index b7375e5055e..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py +++ /dev/null @@ -1,984 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Miscellaneous utilities used by time series models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import math - -import numpy as np - -from tensorflow.contrib import lookup -from tensorflow.contrib.layers.python.layers import layers - -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import TrainEvalFeatures - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import functional_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.util import nest - - -def normal_log_prob(loc, scale, x): - """Computes the Normal log pdf.""" - z = (x - loc) / scale - return -0.5 * (math_ops.square(z) - + np.log(2. * np.pi) + math_ops.log(scale)) - - -def cauchy_log_prob(loc, scale, x): - """Computes the Cauchy log pdf.""" - z = (x - loc) / scale - return (-np.log(np.pi) - math_ops.log(scale) - - math_ops.log1p(math_ops.square(z))) - - -def mvn_tril_log_prob(loc, scale_tril, x): - """Computes the MVN log pdf under tril scale. Doesn't handle batches.""" - x0 = x - loc - z = linalg_ops.matrix_triangular_solve( - scale_tril, x0[..., array_ops.newaxis])[..., 0] - log_det_cov = 2. * math_ops.reduce_sum(math_ops.log( - array_ops.matrix_diag_part(scale_tril)), axis=-1) - d = math_ops.cast(array_ops.shape(scale_tril)[-1], log_det_cov.dtype) - return -0.5 * (math_ops.reduce_sum(math_ops.square(z), axis=-1) - + d * np.log(2. * np.pi) + log_det_cov) - - -def clip_covariance( - covariance_matrix, maximum_variance_ratio, minimum_variance): - """Enforce constraints on a covariance matrix to improve numerical stability. - - Args: - covariance_matrix: A [..., N, N] batch of covariance matrices. - maximum_variance_ratio: The maximum allowed ratio of two diagonal - entries. Any entries lower than the maximum entry divided by this ratio - will be set to that value. - minimum_variance: A floor for diagonal entries in the returned matrix. - Returns: - A new covariance matrix with the requested constraints enforced. If the - input was positive definite, the output will be too. - """ - # TODO(allenl): Smarter scaling here so that correlations are preserved when - # fiddling with diagonal elements. - diagonal = array_ops.matrix_diag_part(covariance_matrix) - maximum = math_ops.reduce_max(diagonal, axis=-1, keepdims=True) - new_diagonal = gen_math_ops.maximum( - diagonal, maximum / maximum_variance_ratio) - return array_ops.matrix_set_diag( - covariance_matrix, math_ops.maximum(new_diagonal, minimum_variance)) - - -def block_diagonal(matrices, dtype=dtypes.float32, name="block_diagonal"): - r"""Constructs block-diagonal matrices from a list of batched 2D tensors. - - Args: - matrices: A list of Tensors with shape [..., N_i, M_i] (i.e. a list of - matrices with the same batch dimension). - dtype: Data type to use. The Tensors in `matrices` must match this dtype. - name: A name for the returned op. - Returns: - A matrix with the input matrices stacked along its main diagonal, having - shape [..., \sum_i N_i, \sum_i M_i]. - """ - matrices = [ops.convert_to_tensor(matrix, dtype=dtype) for matrix in matrices] - blocked_rows = tensor_shape.Dimension(0) - blocked_cols = tensor_shape.Dimension(0) - batch_shape = tensor_shape.TensorShape(None) - for matrix in matrices: - full_matrix_shape = matrix.get_shape().with_rank_at_least(2) - batch_shape = batch_shape.merge_with(full_matrix_shape[:-2]) - blocked_rows += full_matrix_shape[-2] - blocked_cols += full_matrix_shape[-1] - ret_columns_list = [] - for matrix in matrices: - matrix_shape = array_ops.shape(matrix) - ret_columns_list.append(matrix_shape[-1]) - ret_columns = math_ops.add_n(ret_columns_list) - row_blocks = [] - current_column = 0 - for matrix in matrices: - matrix_shape = array_ops.shape(matrix) - row_before_length = current_column - current_column += matrix_shape[-1] - row_after_length = ret_columns - current_column - row_blocks.append( - array_ops.pad( - tensor=matrix, - paddings=array_ops.concat( - [ - array_ops.zeros( - [array_ops.rank(matrix) - 1, 2], dtype=dtypes.int32), [( - row_before_length, row_after_length)] - ], - axis=0))) - blocked = array_ops.concat(row_blocks, -2, name=name) - blocked.set_shape(batch_shape.concatenate((blocked_rows, blocked_cols))) - return blocked - - -def power_sums_tensor(array_size, power_matrix, multiplier): - r"""Computes \sum_{i=0}^{N-1} A^i B (A^i)^T for N=0..(array_size + 1). - - Args: - array_size: The number of non-trivial sums to pre-compute. - power_matrix: The "A" matrix above. - multiplier: The "B" matrix above - Returns: - A Tensor with S[N] = \sum_{i=0}^{N-1} A^i B (A^i)^T - S[0] is the zero matrix - S[1] is B - S[2] is A B A^T + B - ...and so on - """ - array_size = math_ops.cast(array_size, dtypes.int32) - power_matrix = ops.convert_to_tensor(power_matrix) - identity_like_power_matrix = linalg_ops.eye( - array_ops.shape(power_matrix)[0], dtype=power_matrix.dtype) - identity_like_power_matrix.set_shape( - ops.convert_to_tensor(power_matrix).get_shape()) - transition_powers = functional_ops.scan( - lambda previous_power, _: math_ops.matmul(previous_power, power_matrix), - math_ops.range(array_size - 1), - initializer=identity_like_power_matrix) - summed = math_ops.cumsum( - array_ops.concat([ - array_ops.expand_dims(multiplier, 0), math_ops.matmul( - batch_times_matrix(transition_powers, multiplier), - transition_powers, - adjoint_b=True) - ], 0)) - return array_ops.concat( - [array_ops.expand_dims(array_ops.zeros_like(multiplier), 0), summed], 0) - - -def matrix_to_powers(matrix, powers): - """Raise a single matrix to multiple powers.""" - matrix_tiled = array_ops.tile( - array_ops.expand_dims(matrix, 0), [array_ops.size(powers), 1, 1]) - return batch_matrix_pow(matrix_tiled, powers) - - -def batch_matrix_pow(matrices, powers): - """Compute powers of matrices, e.g. A^3 = matmul(matmul(A, A), A). - - Uses exponentiation by squaring, with O(log(p)) matrix multiplications to - compute A^p. - - Args: - matrices: [batch size x N x N] - powers: Which integer power to raise each matrix to [batch size] - Returns: - The matrices raised to their respective powers, same dimensions as the - "matrices" argument. - """ - - def terminate_when_all_zero(current_argument, residual_powers, accumulator): - del current_argument, accumulator # not used for condition - do_exit = math_ops.reduce_any( - math_ops.greater(residual_powers, array_ops.ones_like(residual_powers))) - return do_exit - - def do_iteration(current_argument, residual_powers, accumulator): - """Compute one step of iterative exponentiation by squaring. - - The recursive form is: - power(A, p) = { power(matmul(A, A), p / 2) for even p - { matmul(A, power(matmul(A, A), (p - 1) / 2)) for odd p - power(A, 0) = I - - The power(A, 0) = I case is handled by starting with accumulator set to the - identity matrix; matrices with zero residual powers are passed through - unchanged. - - Args: - current_argument: On this step, what is the first argument (A^2..^2) to - the (unrolled) recursive function? [batch size x N x N] - residual_powers: On this step, what is the second argument (residual p)? - [batch_size] - accumulator: Accumulates the exterior multiplications from the odd - powers (initially the identity matrix). [batch_size x N x N] - Returns: - Updated versions of each argument for one step of the unrolled - computation. Does not change parts of the batch which have a residual - power of zero. - """ - is_even = math_ops.equal(residual_powers % 2, - array_ops.zeros( - array_ops.shape(residual_powers), - dtype=dtypes.int32)) - new_accumulator = array_ops.where(is_even, accumulator, - math_ops.matmul(accumulator, - current_argument)) - new_argument = math_ops.matmul(current_argument, current_argument) - do_update = math_ops.greater(residual_powers, 1) - new_residual_powers = residual_powers - residual_powers % 2 - new_residual_powers //= 2 - # Stop updating if we've reached our base case; some batch elements may - # finish sooner than others - accumulator = array_ops.where(do_update, new_accumulator, accumulator) - current_argument = array_ops.where(do_update, new_argument, - current_argument) - residual_powers = array_ops.where(do_update, new_residual_powers, - residual_powers) - return (current_argument, residual_powers, accumulator) - - matrices = ops.convert_to_tensor(matrices) - powers = math_ops.cast(powers, dtype=dtypes.int32) - ident = array_ops.expand_dims( - array_ops.diag( - array_ops.ones([array_ops.shape(matrices)[1]], dtype=matrices.dtype)), - 0) - ident_tiled = array_ops.tile(ident, [array_ops.shape(matrices)[0], 1, 1]) - (final_argument, - final_residual_power, final_accumulator) = control_flow_ops.while_loop( - terminate_when_all_zero, do_iteration, [matrices, powers, ident_tiled]) - return array_ops.where( - math_ops.equal(final_residual_power, - array_ops.zeros_like( - final_residual_power, dtype=dtypes.int32)), - ident_tiled, math_ops.matmul(final_argument, final_accumulator)) - - -# TODO(allenl): would be useful if this was built into batch_matmul -def batch_times_matrix(batch, matrix, adj_x=False, adj_y=False): - """Multiply a batch of matrices by a single matrix. - - Functionally equivalent to: - tf.matmul(batch, array_ops.tile(gen_math_ops.expand_dims(matrix, 0), - [array_ops.shape(batch)[0], 1, 1]), - adjoint_a=adj_x, adjoint_b=adj_y) - - Args: - batch: [batch_size x N x M] after optional transpose - matrix: [M x P] after optional transpose - adj_x: If true, transpose the second two dimensions of "batch" before - multiplying. - adj_y: If true, transpose "matrix" before multiplying. - Returns: - [batch_size x N x P] - """ - batch = ops.convert_to_tensor(batch) - matrix = ops.convert_to_tensor(matrix) - assert batch.get_shape().ndims == 3 - assert matrix.get_shape().ndims == 2 - if adj_x: - batch = array_ops.transpose(batch, [0, 2, 1]) - batch_dimension = batch.get_shape().dims[0].value - first_dimension = batch.get_shape().dims[1].value - tensor_batch_shape = array_ops.shape(batch) - if batch_dimension is None: - batch_dimension = tensor_batch_shape[0] - if first_dimension is None: - first_dimension = tensor_batch_shape[1] - matrix_first_dimension, matrix_second_dimension = matrix.get_shape().as_list() - batch_reshaped = array_ops.reshape(batch, [-1, tensor_batch_shape[2]]) - if adj_y: - if matrix_first_dimension is None: - matrix_first_dimension = array_ops.shape(matrix)[0] - result_shape = [batch_dimension, first_dimension, matrix_first_dimension] - else: - if matrix_second_dimension is None: - matrix_second_dimension = array_ops.shape(matrix)[1] - result_shape = [batch_dimension, first_dimension, matrix_second_dimension] - return array_ops.reshape( - math_ops.matmul(batch_reshaped, matrix, adjoint_b=adj_y), result_shape) - - -def matrix_times_batch(matrix, batch, adj_x=False, adj_y=False): - """Like batch_times_matrix, but with the multiplication order swapped.""" - return array_ops.transpose( - batch_times_matrix( - batch=batch, matrix=matrix, adj_x=not adj_y, adj_y=not adj_x), - [0, 2, 1]) - - -def make_toeplitz_matrix(inputs, name=None): - """Make a symmetric Toeplitz matrix from input array of values. - - Args: - inputs: a 3-D tensor of shape [num_blocks, block_size, block_size]. - name: the name of the operation. - - Returns: - a symmetric Toeplitz matrix of shape - [num_blocks*block_size, num_blocks*block_size]. - """ - num_blocks = array_ops.shape(inputs)[0] - block_size = array_ops.shape(inputs)[1] - output_size = block_size * num_blocks - lags = array_ops.reshape(math_ops.range(num_blocks), shape=[1, -1]) - indices = math_ops.abs(lags - array_ops.transpose(lags)) - output = array_ops.gather(inputs, indices) - output = array_ops.reshape( - array_ops.transpose(output, [0, 2, 1, 3]), [output_size, output_size]) - return array_ops.identity(output, name=name) - - -# TODO(allenl): Investigate alternative parameterizations. -def sign_magnitude_positive_definite( - raw, off_diagonal_scale=0., overall_scale=0.): - """Constructs a positive definite matrix from an unconstrained input matrix. - - We want to keep the whole matrix on a log scale, but also allow off-diagonal - elements to be negative, so the sign of off-diagonal elements is modeled - separately from their magnitude (using the lower and upper triangles - respectively). Specifically: - - for i < j, we have: - output_cholesky[i, j] = raw[j, i] / (abs(raw[j, i]) + 1) * - exp((off_diagonal_scale + overall_scale + raw[i, j]) / 2) - - output_cholesky[i, i] = exp((raw[i, i] + overall_scale) / 2) - - output = output_cholesky^T * output_cholesky - - where raw, off_diagonal_scale, and overall_scale are - un-constrained real-valued variables. The resulting values are stable - around zero due to the exponential (and the softsign keeps the function - smooth). - - Args: - raw: A [..., M, M] Tensor. - off_diagonal_scale: A scalar or [...] shaped Tensor controlling the relative - scale of off-diagonal values in the output matrix. - overall_scale: A scalar or [...] shaped Tensor controlling the overall scale - of the output matrix. - Returns: - The `output` matrix described above, a [..., M, M] positive definite matrix. - - """ - raw = ops.convert_to_tensor(raw) - diagonal = array_ops.matrix_diag_part(raw) - def _right_pad_with_ones(tensor, target_rank): - # Allow broadcasting even if overall_scale and off_diagonal_scale have batch - # dimensions - tensor = ops.convert_to_tensor(tensor, dtype=raw.dtype.base_dtype) - return array_ops.reshape(tensor, - array_ops.concat( - [ - array_ops.shape(tensor), array_ops.ones( - [target_rank - array_ops.rank(tensor)], - dtype=target_rank.dtype) - ], - axis=0)) - # We divide the log values by 2 to compensate for the squaring that happens - # when transforming Cholesky factors into positive definite matrices. - sign_magnitude = (gen_math_ops.exp( - (raw + _right_pad_with_ones(off_diagonal_scale, array_ops.rank(raw)) + - _right_pad_with_ones(overall_scale, array_ops.rank(raw))) / 2.) * - nn.softsign(array_ops.matrix_transpose(raw))) - sign_magnitude.set_shape(raw.get_shape()) - cholesky_factor = array_ops.matrix_set_diag( - input=array_ops.matrix_band_part(sign_magnitude, 0, -1), - diagonal=gen_math_ops.exp((diagonal + _right_pad_with_ones( - overall_scale, array_ops.rank(diagonal))) / 2.)) - return math_ops.matmul(cholesky_factor, cholesky_factor, transpose_a=True) - - -def transform_to_covariance_matrices(input_vectors, matrix_size): - """Construct covariance matrices via transformations from input_vectors. - - Args: - input_vectors: A [batch size x input size] batch of vectors to transform. - matrix_size: An integer indicating one dimension of the (square) output - matrix. - Returns: - A [batch size x matrix_size x matrix_size] batch of covariance matrices. - """ - combined_values = layers.fully_connected( - input_vectors, matrix_size**2 + 2, activation_fn=None) - return sign_magnitude_positive_definite( - raw=array_ops.reshape(combined_values[..., :-2], - array_ops.concat([ - array_ops.shape(combined_values)[:-1], - [matrix_size, matrix_size] - ], 0)), - off_diagonal_scale=combined_values[..., -2], - overall_scale=combined_values[..., -1]) - - -def variable_covariance_matrix( - size, name, dtype, initial_diagonal_values=None, - initial_overall_scale_log=0.): - """Construct a Variable-parameterized positive definite matrix. - - Useful for parameterizing covariance matrices. - - Args: - size: The size of the main diagonal, the returned matrix having shape [size - x size]. - name: The name to use when defining variables and ops. - dtype: The floating point data type to use. - initial_diagonal_values: A Tensor with shape [size] with initial values for - the diagonal values of the returned matrix. Must be positive. - initial_overall_scale_log: Initial value of the bias term for every element - of the matrix in log space. - Returns: - A Variable-parameterized covariance matrix with shape [size x size]. - """ - raw_values = variable_scope.get_variable( - name + "_pre_transform", - dtype=dtype, - shape=[size, size], - initializer=init_ops.zeros_initializer()) - if initial_diagonal_values is not None: - raw_values += array_ops.matrix_diag(math_ops.log(initial_diagonal_values)) - return array_ops.identity( - sign_magnitude_positive_definite( - raw=raw_values, - off_diagonal_scale=variable_scope.get_variable( - name + "_off_diagonal_scale", - dtype=dtype, - initializer=constant_op.constant(-5., dtype=dtype)), - overall_scale=ops.convert_to_tensor( - initial_overall_scale_log, dtype=dtype) + - variable_scope.get_variable( - name + "_overall_scale", - dtype=dtype, - shape=[], - initializer=init_ops.zeros_initializer())), - name=name) - - -def batch_start_time(times): - return times[:, 0] - - -def batch_end_time(times): - return times[:, -1] - - -def log_noninformative_covariance_prior(covariance): - """Compute a relatively uninformative prior for noise parameters. - - Helpful for avoiding noise over-estimation, where noise otherwise decreases - very slowly during optimization. - - See: - Villegas, C. On the A Priori Distribution of the Covariance Matrix. - Ann. Math. Statist. 40 (1969), no. 3, 1098--1099. - - Args: - covariance: A covariance matrix. - Returns: - For a [p x p] matrix: - log(det(covariance)^(-(p + 1) / 2)) - """ - # Avoid zero/negative determinants due to numerical errors - covariance += array_ops.diag(1e-8 * array_ops.ones( - shape=[array_ops.shape(covariance)[0]], dtype=covariance.dtype)) - power = -(math_ops.cast(array_ops.shape(covariance)[0] + 1, - covariance.dtype) / 2.) - return power * math_ops.log(linalg_ops.matrix_determinant(covariance)) - - -def entropy_matched_cauchy_scale(covariance): - """Approximates a similar Cauchy distribution given a covariance matrix. - - Since Cauchy distributions do not have moments, entropy matching provides one - way to set a Cauchy's scale parameter in a way that provides a similar - distribution. The effect is dividing the standard deviation of an independent - Gaussian by a constant very near 3. - - To set the scale of the Cauchy distribution, we first select the diagonals of - `covariance`. Since this ignores cross terms, it overestimates the entropy of - the Gaussian. For each of these variances, we solve for the Cauchy scale - parameter which gives the same entropy as the Gaussian with that - variance. This means setting the (univariate) Gaussian entropy - 0.5 * ln(2 * variance * pi * e) - equal to the Cauchy entropy - ln(4 * pi * scale) - Solving, we get scale = sqrt(variance * (e / (8 pi))). - - Args: - covariance: A [batch size x N x N] batch of covariance matrices to produce - Cauchy scales for. - Returns: - A [batch size x N] set of Cauchy scale parameters for each part of the batch - and each dimension of the input Gaussians. - """ - return math_ops.sqrt(math.e / (8. * math.pi) * - array_ops.matrix_diag_part(covariance)) - - -class TensorValuedMutableDenseHashTable(lookup.MutableDenseHashTable): - """A version of MutableDenseHashTable which stores arbitrary Tensor shapes. - - Since MutableDenseHashTable only allows vectors right now, simply adds reshape - ops on both ends. - """ - - def __init__(self, key_dtype, value_dtype, default_value, *args, **kwargs): - self._non_vector_value_shape = array_ops.shape(default_value) - super(TensorValuedMutableDenseHashTable, self).__init__( - key_dtype=key_dtype, - value_dtype=value_dtype, - default_value=array_ops.reshape(default_value, [-1]), - *args, - **kwargs) - - def insert(self, keys, values, name=None): - keys = ops.convert_to_tensor(keys, dtype=self._key_dtype) - keys_flat = array_ops.reshape(keys, [-1]) - return super(TensorValuedMutableDenseHashTable, self).insert( - keys=keys_flat, - # Each key has one corresponding value, so the shape of the tensor of - # values for every key is key_shape + value_shape - values=array_ops.reshape(values, [array_ops.shape(keys_flat)[0], -1]), - name=name) - - def lookup(self, keys, name=None): - keys_flat = array_ops.reshape( - ops.convert_to_tensor(keys, dtype=self._key_dtype), [-1]) - return array_ops.reshape( - super(TensorValuedMutableDenseHashTable, self).lookup( - keys=keys_flat, name=name), - array_ops.concat([array_ops.shape(keys), self._non_vector_value_shape], - 0)) - - -class TupleOfTensorsLookup(lookup.LookupInterface): - """A LookupInterface with nested tuples of Tensors as values. - - Creates one MutableDenseHashTable per value Tensor, which has some unnecessary - overhead. - """ - - def __init__(self, - key_dtype, - default_values, - empty_key, - deleted_key, - name, - checkpoint=True): - default_values_flat = nest.flatten(default_values) - self._hash_tables = nest.pack_sequence_as(default_values, [ - TensorValuedMutableDenseHashTable( - key_dtype=key_dtype, - value_dtype=default_value.dtype.base_dtype, - default_value=default_value, - empty_key=empty_key, - deleted_key=deleted_key, - name=name + "_{}".format(table_number), - checkpoint=checkpoint) - for table_number, default_value in enumerate(default_values_flat) - ]) - self._name = name - - def lookup(self, keys): - return nest.pack_sequence_as( - self._hash_tables, - [hash_table.lookup(keys) - for hash_table in nest.flatten(self._hash_tables)]) - - def insert(self, keys, values): - nest.assert_same_structure(self._hash_tables, values) - # Avoid race conditions by requiring that all inputs are computed before any - # inserts happen (an issue if one key's update relies on another's value). - values_flat = [array_ops.identity(value) for value in nest.flatten(values)] - with ops.control_dependencies(values_flat): - insert_ops = [hash_table.insert(keys, value) - for hash_table, value - in zip(nest.flatten(self._hash_tables), - values_flat)] - return control_flow_ops.group(*insert_ops) - - def check_table_dtypes(self, key_dtype, value_dtype): - # dtype checking is done in the objects in self._hash_tables - pass - - -def replicate_state(start_state, batch_size): - """Create batch versions of state. - - Takes a list of Tensors, adds a batch dimension, and replicates - batch_size times across that batch dimension. Used to replicate the - non-batch state returned by get_start_state in define_loss. - - Args: - start_state: Model-defined state to replicate. - batch_size: Batch dimension for data. - Returns: - Replicated versions of the state. - """ - flattened_state = nest.flatten(start_state) - replicated_state = [ - array_ops.tile( - array_ops.expand_dims(state_nonbatch, 0), - array_ops.concat([[batch_size], array_ops.ones( - [array_ops.rank(state_nonbatch)], dtype=dtypes.int32)], 0)) - for state_nonbatch in flattened_state - ] - return nest.pack_sequence_as(start_state, replicated_state) - - -Moments = collections.namedtuple("Moments", ["mean", "variance"]) - - -# Currently all of these statistics are computed incrementally (i.e. are updated -# every time a new mini-batch of training data is presented) when this object is -# created in InputStatisticsFromMiniBatch. -InputStatistics = collections.namedtuple( - "InputStatistics", - ["series_start_moments", # The mean and variance of each feature in a chunk - # (with a size configured in the statistics - # object) at the start of the series. A tuple of - # (mean, variance), each with shape [number of - # features], floating point. One use is in state - # space models, to keep priors calibrated even as - # earlier parts of the series are presented. If - # this object was created by - # InputStatisticsFromMiniBatch, these moments are - # computed based on the earliest chunk of data - # presented so far. However, there is a race - # condition in the update, so these may reflect - # statistics later in the series, but should - # eventually reflect statistics in a chunk at the - # series start. - "overall_feature_moments", # The mean and variance of each feature over - # the entire series. A tuple of (mean, - # variance), each with shape [number of - # features]. If this object was created by - # InputStatisticsFromMiniBatch, these moments - # are estimates based on the data seen so far. - "start_time", # The first (lowest) time in the series, a scalar - # integer. If this object was created by - # InputStatisticsFromMiniBatch, this is the lowest time seen - # so far rather than the lowest time that will ever be seen - # (guaranteed to be at least as low as the lowest time - # presented in the current minibatch). - "total_observation_count", # Count of data points, a scalar integer. If - # this object was created by - # InputStatisticsFromMiniBatch, this is an - # estimate of the total number of observations - # in the whole dataset computed based on the - # density of the series and the minimum and - # maximum times seen. - ]) - - -# TODO(allenl): It would be nice to do something with full series statistics -# when the user provides that. -class InputStatisticsFromMiniBatch(object): - """Generate statistics from mini-batch input.""" - - def __init__(self, num_features, dtype, starting_variance_window_size=16): - """Configure the input statistics object. - - Args: - num_features: Number of features for the time series - dtype: The floating point data type to use. - starting_variance_window_size: The number of datapoints to use when - computing the mean and variance at the start of the series. - """ - self._starting_variance_window_size = starting_variance_window_size - self._num_features = num_features - self._dtype = dtype - - def initialize_graph(self, features, update_statistics=True): - """Create any ops needed to provide input statistics. - - Should be called before statistics are requested. - - Args: - features: A dictionary, the output of a `TimeSeriesInputFn` (with keys - TrainEvalFeatures.TIMES and TrainEvalFeatures.VALUES). - update_statistics: Whether `features` should be used to update adaptive - statistics. Typically True for training and false for evaluation. - Returns: - An InputStatistics object composed of Variables, which will be updated - based on mini-batches of data if requested. - """ - if (TrainEvalFeatures.TIMES in features - and TrainEvalFeatures.VALUES in features): - times = features[TrainEvalFeatures.TIMES] - values = features[TrainEvalFeatures.VALUES] - else: - # times and values may not be available, for example during prediction. We - # still need to retrieve our variables so that they can be read from, even - # if we're not going to update them. - times = None - values = None - # Create/retrieve variables representing input statistics, initialized - # without data to avoid deadlocking if variables are initialized before - # queue runners are started. - with variable_scope.variable_scope("input_statistics", use_resource=True): - statistics = self._create_variable_statistics_object() - with variable_scope.variable_scope( - "input_statistics_auxiliary", use_resource=True): - # Secondary statistics, necessary for the incremental computation of the - # primary statistics (e.g. counts and sums for computing a mean - # incrementally). - auxiliary_variables = self._AdaptiveInputAuxiliaryStatistics( - num_features=self._num_features, dtype=self._dtype) - if update_statistics and times is not None and values is not None: - # If we have times and values from mini-batch input, create update ops to - # take the new data into account. - assign_op = self._update_statistics_from_mini_batch( - statistics, auxiliary_variables, times, values) - with ops.control_dependencies([assign_op]): - stat_variables = nest.pack_sequence_as(statistics, [ - array_ops.identity(tensor) for tensor in nest.flatten(statistics) - ]) - # Since start time updates have a race condition, ensure that the - # reported start time is at least as low as the lowest time in this - # mini-batch. The start time should converge on the correct value - # eventually even with the race condition, but for example state space - # models have an assertion which could fail without this - # post-processing. - return stat_variables._replace(start_time=gen_math_ops.minimum( - stat_variables.start_time, math_ops.reduce_min(times))) - else: - return statistics - - class _AdaptiveInputAuxiliaryStatistics(collections.namedtuple( - "_AdaptiveInputAuxiliaryStatistics", - ["max_time_seen", # The maximum time seen (best effort if updated from - # multiple workers; see notes about race condition - # below). - "chunk_count", # The number of chunks seen. - "inter_observation_duration_sum", # The sum across chunks of their "time - # density" (number of times per - # example). - "example_count", # The number of examples seen (each example has a - # single time associated with it and one or more - # real-valued features). - "overall_feature_sum", # The sum of values for each feature. Shape - # [number of features]. - "overall_feature_sum_of_squares", # The sum of squared values for each - # feature. Shape [number of features] - ])): - """Extra statistics used to incrementally update InputStatistics.""" - - def __new__(cls, num_features, dtype): - return super( - InputStatisticsFromMiniBatch # pylint: disable=protected-access - ._AdaptiveInputAuxiliaryStatistics, - cls).__new__( - cls, - max_time_seen=variable_scope.get_variable( - name="max_time_seen", - initializer=dtypes.int64.min, - dtype=dtypes.int64, - trainable=False), - chunk_count=variable_scope.get_variable( - name="chunk_count", - initializer=init_ops.zeros_initializer(), - shape=[], - dtype=dtypes.int64, - trainable=False), - inter_observation_duration_sum=variable_scope.get_variable( - name="inter_observation_duration_sum", - initializer=init_ops.zeros_initializer(), - shape=[], - dtype=dtype, - trainable=False), - example_count=variable_scope.get_variable( - name="example_count", - shape=[], - dtype=dtypes.int64, - trainable=False), - overall_feature_sum=variable_scope.get_variable( - name="overall_feature_sum", - shape=[num_features], - dtype=dtype, - initializer=init_ops.zeros_initializer(), - trainable=False), - overall_feature_sum_of_squares=variable_scope.get_variable( - name="overall_feature_sum_of_squares", - shape=[num_features], - dtype=dtype, - initializer=init_ops.zeros_initializer(), - trainable=False)) - - def _update_statistics_from_mini_batch( - self, statistics, auxiliary_variables, times, values): - """Given mini-batch input, update `statistics` and `auxiliary_variables`.""" - values = math_ops.cast(values, self._dtype) - # The density (measured in times per observation) that we see in each part - # of the mini-batch. - batch_inter_observation_duration = (math_ops.cast( - math_ops.reduce_max(times, axis=1) - math_ops.reduce_min(times, axis=1), - self._dtype) / math_ops.cast( - array_ops.shape(times)[1] - 1, self._dtype)) - # Co-locate updates with their variables to minimize race conditions when - # updating statistics. - with ops.device(auxiliary_variables.max_time_seen.device): - # There is a race condition if this value is being updated from multiple - # workers. However, it should eventually reach the correct value if the - # last chunk is presented enough times. - max_time_seen_assign = state_ops.assign( - auxiliary_variables.max_time_seen, - gen_math_ops.maximum(auxiliary_variables.max_time_seen, - math_ops.reduce_max(times))) - with ops.device(auxiliary_variables.chunk_count.device): - chunk_count_assign = state_ops.assign_add(auxiliary_variables.chunk_count, - array_ops.shape( - times, - out_type=dtypes.int64)[0]) - with ops.device(auxiliary_variables.inter_observation_duration_sum.device): - inter_observation_duration_assign = state_ops.assign_add( - auxiliary_variables.inter_observation_duration_sum, - math_ops.reduce_sum(batch_inter_observation_duration)) - with ops.device(auxiliary_variables.example_count.device): - example_count_assign = state_ops.assign_add( - auxiliary_variables.example_count, - array_ops.size(times, out_type=dtypes.int64)) - # Note: These mean/variance updates assume that all points are equally - # likely, which is not true if _chunks_ are sampled uniformly from the space - # of all possible contiguous chunks, since points at the start and end of - # the series are then members of fewer chunks. For series which are much - # longer than the chunk size (the usual/expected case), this effect becomes - # irrelevant. - with ops.device(auxiliary_variables.overall_feature_sum.device): - overall_feature_sum_assign = state_ops.assign_add( - auxiliary_variables.overall_feature_sum, - math_ops.reduce_sum(values, axis=[0, 1])) - with ops.device(auxiliary_variables.overall_feature_sum_of_squares.device): - overall_feature_sum_of_squares_assign = state_ops.assign_add( - auxiliary_variables.overall_feature_sum_of_squares, - math_ops.reduce_sum(values**2, axis=[0, 1])) - per_chunk_aux_updates = control_flow_ops.group( - max_time_seen_assign, chunk_count_assign, - inter_observation_duration_assign, example_count_assign, - overall_feature_sum_assign, overall_feature_sum_of_squares_assign) - with ops.control_dependencies([per_chunk_aux_updates]): - example_count_float = math_ops.cast(auxiliary_variables.example_count, - self._dtype) - new_feature_mean = (auxiliary_variables.overall_feature_sum / - example_count_float) - overall_feature_mean_update = state_ops.assign( - statistics.overall_feature_moments.mean, new_feature_mean) - overall_feature_var_update = state_ops.assign( - statistics.overall_feature_moments.variance, - # De-biased n / (n - 1) variance correction - example_count_float / (example_count_float - 1.) * - (auxiliary_variables.overall_feature_sum_of_squares / - example_count_float - new_feature_mean**2)) - # TODO(b/35675805): Remove this cast - min_time_batch = math_ops.cast(math_ops.argmin(times[:, 0]), dtypes.int32) - def series_start_updates(): - # If this is the lowest-time chunk that we have seen so far, update - # series start moments to reflect that. Note that these statistics are - # "best effort", as there are race conditions in the update (however, - # they should eventually converge if the start of the series is - # presented enough times). - mean, variance = nn.moments( - values[min_time_batch, :self._starting_variance_window_size], - axes=[0]) - return control_flow_ops.group( - state_ops.assign(statistics.series_start_moments.mean, mean), - state_ops.assign(statistics.series_start_moments.variance, - variance)) - with ops.device(statistics.start_time.device): - series_start_update = control_flow_ops.cond( - # Update moments whenever we even match the lowest time seen so far, - # to ensure that series start statistics are eventually updated to - # their correct values, despite race conditions (i.e. eventually - # statistics.start_time will reflect the global lowest time, and - # given that we will eventually update the series start moments to - # their correct values). - math_ops.less_equal(times[min_time_batch, 0], - statistics.start_time), - series_start_updates, - control_flow_ops.no_op) - with ops.control_dependencies([series_start_update]): - # There is a race condition if this update is performed in parallel on - # multiple workers. Since models may be sensitive to being presented - # with times before the putative start time, the value of this - # variable is post-processed above to guarantee that each worker is - # presented with a start time which is at least as low as the lowest - # time in its current mini-batch. - start_time_update = state_ops.assign(statistics.start_time, - gen_math_ops.minimum( - statistics.start_time, - math_ops.reduce_min(times))) - inter_observation_duration_estimate = ( - auxiliary_variables.inter_observation_duration_sum / math_ops.cast( - auxiliary_variables.chunk_count, self._dtype)) - # Estimate the total number of observations as: - # (end time - start time + 1) * average intra-chunk time density - total_observation_count_update = state_ops.assign( - statistics.total_observation_count, - math_ops.cast( - gen_math_ops.round( - math_ops.cast(max_time_seen_assign - - start_time_update + 1, self._dtype) / - inter_observation_duration_estimate), dtypes.int64)) - per_chunk_stat_updates = control_flow_ops.group( - overall_feature_mean_update, overall_feature_var_update, - series_start_update, start_time_update, - total_observation_count_update) - return per_chunk_stat_updates - - def _create_variable_statistics_object(self): - """Creates non-trainable variables representing input statistics.""" - series_start_moments = Moments( - mean=variable_scope.get_variable( - name="series_start_mean", - shape=[self._num_features], - dtype=self._dtype, - initializer=init_ops.zeros_initializer(), - trainable=False), - variance=variable_scope.get_variable( - name="series_start_variance", - shape=[self._num_features], - dtype=self._dtype, - initializer=init_ops.ones_initializer(), - trainable=False)) - overall_feature_moments = Moments( - mean=variable_scope.get_variable( - name="overall_feature_mean", - shape=[self._num_features], - dtype=self._dtype, - initializer=init_ops.zeros_initializer(), - trainable=False), - variance=variable_scope.get_variable( - name="overall_feature_var", - shape=[self._num_features], - dtype=self._dtype, - initializer=init_ops.ones_initializer(), - trainable=False)) - start_time = variable_scope.get_variable( - name="start_time", - dtype=dtypes.int64, - initializer=dtypes.int64.max, - trainable=False) - total_observation_count = variable_scope.get_variable( - name="total_observation_count", - shape=[], - dtype=dtypes.int64, - initializer=init_ops.ones_initializer(), - trainable=False) - return InputStatistics( - series_start_moments=series_start_moments, - overall_feature_moments=overall_feature_moments, - start_time=start_time, - total_observation_count=total_observation_count) diff --git a/tensorflow/contrib/timeseries/python/timeseries/math_utils_test.py b/tensorflow/contrib/timeseries/python/timeseries/math_utils_test.py deleted file mode 100644 index 91265b9b2e6..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/math_utils_test.py +++ /dev/null @@ -1,347 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for math_utils.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy - -from tensorflow.contrib.timeseries.python.timeseries import input_pipeline -from tensorflow.contrib.timeseries.python.timeseries import math_utils -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import TrainEvalFeatures - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import coordinator as coordinator_lib -from tensorflow.python.training import queue_runner_impl - - -class MathUtilsTest(test.TestCase): - - def setUp(self): - numpy.random.seed(10) - - def test_power_sums_tensor(self): - transition = numpy.random.normal(size=[4, 4]).astype(numpy.float32) - addition = numpy.random.normal(size=[4, 4]).astype(numpy.float32) - array_size = 2 - result = [] - transition_power = numpy.identity(4) - running_sum = numpy.zeros([4, 4], dtype=numpy.float32) - for _ in range(array_size + 1): - result.append(running_sum) - current_contribution = numpy.dot(numpy.dot(transition_power, addition), - transition_power.T) - # pylint: disable=g-no-augmented-assignment - # += has different semantics here; want to make a copy - running_sum = running_sum + current_contribution - # pylint: enable=g-no-augmented-assignment - transition_power = numpy.dot(transition, transition_power) - with self.cached_session(): - self.assertAllClose(result, - math_utils.power_sums_tensor( - array_size, transition, addition).eval()) - - def test_matrix_to_powers(self): - matrix = numpy.random.normal(size=[4, 4]).astype(numpy.float32) - powers = numpy.random.randint(low=0, high=10, size=20) - result = [] - for i in range(powers.shape[0]): - result.append(numpy.linalg.matrix_power(matrix, powers[i])) - with self.cached_session(): - self.assertAllClose(result, - math_utils.matrix_to_powers(matrix, powers).eval(), - rtol=1e-5, - atol=1e-5) - - def test_batch_matrix_pow(self): - batch = numpy.random.normal(size=[15, 4, 4]).astype(numpy.float32) - powers = numpy.random.randint(low=0, high=10, size=batch.shape[0]) - result = [] - for i in range(batch.shape[0]): - result.append(numpy.linalg.matrix_power(batch[i], powers[i])) - with self.cached_session(): - # TODO(allenl): Numerical errors seem to be creeping in. Maybe it can be - # made slightly more stable? - self.assertAllClose(result, - math_utils.batch_matrix_pow(batch, powers).eval(), - rtol=1e-5, - atol=1e-5) - - def test_batch_times_matrix(self): - left = numpy.random.normal(size=[5, 3, 2]).astype(numpy.float32) - left_transpose = numpy.transpose(left, [0, 2, 1]) - right = numpy.random.normal(size=[2, 3]).astype(numpy.float32) - expected_result = numpy.dot(left, right) - with self.cached_session(): - self.assertAllClose(expected_result, - math_utils.batch_times_matrix( - left, right).eval()) - self.assertAllClose(expected_result, - math_utils.batch_times_matrix( - left_transpose, right, - adj_x=True).eval()) - self.assertAllClose(expected_result, - math_utils.batch_times_matrix( - left, right.T, - adj_y=True).eval()) - self.assertAllClose(expected_result, - math_utils.batch_times_matrix( - left_transpose, right.T, - adj_x=True, adj_y=True).eval()) - - def test_matrix_times_batch(self): - left = numpy.random.normal(size=[5, 7]).astype(numpy.float32) - right = numpy.random.normal(size=[3, 7, 9]).astype(numpy.float32) - right_transpose = numpy.transpose(right, [0, 2, 1]) - expected_result = numpy.transpose(numpy.dot(right_transpose, left.T), - [0, 2, 1]) - with self.cached_session(): - self.assertAllClose(expected_result, - math_utils.matrix_times_batch( - left, right).eval()) - self.assertAllClose(expected_result, - math_utils.matrix_times_batch( - left.T, right, - adj_x=True).eval()) - self.assertAllClose(expected_result, - math_utils.matrix_times_batch( - left, right_transpose, - adj_y=True).eval()) - self.assertAllClose(expected_result, - math_utils.matrix_times_batch( - left.T, right_transpose, - adj_x=True, adj_y=True).eval()) - - def test_make_diagonal_undefined_shapes(self): - with self.cached_session(): - completely_undefined = array_ops.placeholder(dtype=dtypes.float32) - partly_undefined = array_ops.placeholder( - shape=[None, None], dtype=dtypes.float32) - blocked = math_utils.block_diagonal([completely_undefined, - [[2.]], - partly_undefined]) - self.assertEqual([None, None], - blocked.get_shape().as_list()) - self.assertAllEqual( - [[1., 0., 0., 0.], - [0., 2., 0., 0.], - [0., 0., 3., 4.], - [0., 0., 5., 6.]], - blocked.eval(feed_dict={ - completely_undefined: [[1.]], - partly_undefined: [[3., 4.], - [5., 6.]]})) - - def test_make_diagonal_mostly_defined_shapes(self): - with self.cached_session(): - mostly_defined = array_ops.placeholder( - shape=[None, 2], dtype=dtypes.float32) - blocked = math_utils.block_diagonal([[[2.]], - mostly_defined, - [[7.]]]) - self.assertEqual([None, 4], - blocked.get_shape().as_list()) - self.assertAllEqual( - [[2., 0., 0., 0.], - [0., 3., 4., 0.], - [0., 5., 6., 0.], - [0., 0., 0., 7.]], - blocked.eval(feed_dict={ - mostly_defined: [[3., 4.], - [5., 6.]]})) - - -class TestMakeToeplitzMatrix(test.TestCase): - - def test_make_toeplitz_matrix_1(self): - inputs = numpy.array([[[1.]], [[2.]], [[3.]]]) - output_expected = numpy.array([[1., 2, 3], [2, 1, 2], [3, 2, 1]]) - self._test_make_toeplitz_matrix(inputs, output_expected) - - def test_make_toeplitz_matrix_2(self): - inputs = numpy.array( - [[[1, 2.], [3, 4]], [[5, 6], [7, 8]], [[8, 9], [10, 11]]]) - - output_expected = numpy.array( - [[1., 2., 5., 6, 8, 9], - [3, 4, 7, 8, 10, 11], - [5, 6, 1, 2, 5, 6], - [7, 8, 3, 4, 7, 8], - [8, 9, 5, 6, 1, 2], - [10, 11, 7, 8, 3, 4]]) - self._test_make_toeplitz_matrix(inputs, output_expected) - - def _test_make_toeplitz_matrix(self, inputs, output_expected): - output_tf = math_utils.make_toeplitz_matrix(inputs) - with self.cached_session() as sess: - output_tf_np = sess.run(output_tf) - self.assertAllClose(output_tf_np, output_expected) - - -class TestMakeCovarianceMatrix(test.TestCase): - - def test_zero_size_matrix(self): - raw = numpy.zeros([0, 0]) - with self.cached_session(): - constructed = math_utils.sign_magnitude_positive_definite(raw=raw).eval() - self.assertEqual((0, 0), constructed.shape) - - def test_sign_magnitude_positive_definite(self): - for dtype in [dtypes.float32, dtypes.float64]: - with self.cached_session(): - matrix_tensor = math_utils.sign_magnitude_positive_definite( - raw=constant_op.constant([[-1., -2.], [3., 4.]], dtype=dtype), - off_diagonal_scale=constant_op.constant(-1., dtype=dtype), - overall_scale=constant_op.constant(1., dtype=dtype)) - matrix_evaled = matrix_tensor.eval() - self.assertAllClose(matrix_evaled, matrix_evaled.T) - self.assertTrue(numpy.all(numpy.linalg.eigvals(matrix_evaled) > 0)) - - -class TestLookupTable(test.TestCase): - - def test_tuple_of_tensors_lookup(self): - hash_table = math_utils.TupleOfTensorsLookup( - key_dtype=dtypes.int64, - default_values=[[ - array_ops.ones([3, 2], dtype=dtypes.float32), - array_ops.zeros([5], dtype=dtypes.float64) - ], - array_ops.ones([7, 7], dtype=dtypes.int64)], - empty_key=-1, - deleted_key=-2, - name="test_lookup") - def stack_tensor(base_tensor): - return array_ops.stack([base_tensor + 1, base_tensor + 2]) - - with self.cached_session() as session: - ((float_output, double_output), int_output) = session.run( - hash_table.lookup([2, 1, 0])) - def expected_output_before_insert(base_tensor): - return [base_tensor, - base_tensor, - base_tensor] - self.assertAllClose( - expected_output_before_insert(numpy.ones([3, 2])), - float_output) - self.assertAllClose( - expected_output_before_insert(numpy.zeros([5])), - double_output) - self.assertAllEqual( - expected_output_before_insert(numpy.ones([7, 7], dtype=numpy.int64)), - int_output) - hash_table.insert( - keys=[1, 2], - values=[[ - stack_tensor(array_ops.ones([3, 2], dtype=dtypes.float32)), - stack_tensor(array_ops.zeros([5], dtype=dtypes.float64)) - ], stack_tensor(array_ops.ones([7, 7], dtype=dtypes.int64))]).run() - ((float_output, double_output), int_output) = session.run( - hash_table.lookup([2, 1, 0])) - def expected_output_after_insert(base_tensor): - return [base_tensor + 2, - base_tensor + 1, - base_tensor] - self.assertAllClose( - expected_output_after_insert(numpy.ones([3, 2])), - float_output) - self.assertAllClose( - expected_output_after_insert(numpy.zeros([5])), - double_output) - self.assertAllEqual( - expected_output_after_insert(numpy.ones([7, 7], dtype=numpy.int64)), - int_output) - - -class InputStatisticsTests(test.TestCase): - - def _input_statistics_test_template( - self, stat_object, num_features, dtype, give_full_data, - warmup_iterations=0, rtol=1e-6, data_length=500, chunk_size=4): - graph = ops.Graph() - with graph.as_default(): - numpy_dtype = dtype.as_numpy_dtype - values = ( - (numpy.arange(data_length, dtype=numpy_dtype)[..., None] - + numpy.arange(num_features, dtype=numpy_dtype)[None, ...])[None]) - times = 2 * (numpy.arange(data_length)[None]) - 3 - if give_full_data: - stat_object.set_data((times, values)) - features = {TrainEvalFeatures.TIMES: times, - TrainEvalFeatures.VALUES: values} - input_fn = input_pipeline.RandomWindowInputFn( - batch_size=16, window_size=chunk_size, - time_series_reader=input_pipeline.NumpyReader(features)) - statistics = stat_object.initialize_graph( - features=input_fn()[0]) - with self.session(graph=graph) as session: - variables.global_variables_initializer().run() - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(session, coord=coordinator) - for _ in range(warmup_iterations): - # A control dependency should ensure that, for queue-based statistics, - # a use of any statistic is preceded by an update of all adaptive - # statistics. - statistics.total_observation_count.eval() - self.assertAllClose( - range(num_features) + numpy.mean(numpy.arange(chunk_size))[None], - statistics.series_start_moments.mean.eval(), - rtol=rtol) - self.assertAllClose( - numpy.tile(numpy.var(numpy.arange(chunk_size))[None], - [num_features]), - statistics.series_start_moments.variance.eval(), - rtol=rtol) - self.assertAllClose( - numpy.mean(values[0], axis=0), - statistics.overall_feature_moments.mean.eval(), - rtol=rtol) - self.assertAllClose( - numpy.var(values[0], axis=0), - statistics.overall_feature_moments.variance.eval(), - rtol=rtol) - self.assertAllClose( - -3, - statistics.start_time.eval(), - rtol=rtol) - self.assertAllClose( - data_length, - statistics.total_observation_count.eval(), - rtol=rtol) - coordinator.request_stop() - coordinator.join() - - def test_queue(self): - for dtype in [dtypes.float32, dtypes.float64]: - for num_features in [1, 2, 3]: - self._input_statistics_test_template( - math_utils.InputStatisticsFromMiniBatch( - num_features=num_features, dtype=dtype), - num_features=num_features, - dtype=dtype, - give_full_data=False, - warmup_iterations=1000, - rtol=0.1) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/model.py b/tensorflow/contrib/timeseries/python/timeseries/model.py deleted file mode 100644 index 53995c15926..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/model.py +++ /dev/null @@ -1,826 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Base class for time series models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc -import collections - -import six - -from tensorflow.contrib.timeseries.python.timeseries import math_utils -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import PredictionFeatures -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import TrainEvalFeatures - -from tensorflow.python.feature_column import feature_column_lib as feature_column -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import tensor_array_ops -from tensorflow.python.ops import variable_scope - -from tensorflow.python.util import nest - - -ModelOutputs = collections.namedtuple( # pylint: disable=invalid-name - typename="ModelOutputs", - field_names=[ - "loss", # The scalar value to be minimized during training. - "end_state", # A nested tuple specifying the model's state after - # running on the specified data - "predictions", # A dictionary of predictions, each with shape prefixed - # by the shape of `prediction_times`. - "prediction_times" # A [batch size x window size] integer Tensor - # indicating times for which values in `predictions` - # were computed. - ]) - - -@six.add_metaclass(abc.ABCMeta) -class TimeSeriesModel(object): - """Base class for creating generative time series models.""" - - def __init__(self, - num_features, - exogenous_feature_columns=None, - dtype=dtypes.float32): - """Constructor for generative models. - - Args: - num_features: Number of features for the time series - exogenous_feature_columns: A list of `tf.feature_column`s (for example - `tf.feature_column.embedding_column`) corresponding to exogenous - features which provide extra information to the model but are not - part of the series to be predicted. Passed to - `tf.compat.v1.feature_column.input_layer`. - dtype: The floating point datatype to use. - """ - if exogenous_feature_columns: - self._exogenous_feature_columns = exogenous_feature_columns - else: - self._exogenous_feature_columns = [] - self.num_features = num_features - self.dtype = dtype - self._input_statistics = None - self._graph_initialized = False - self._stats_means = None - self._stats_sigmas = None - - @property - def exogenous_feature_columns(self): - """`tf.feature_colum`s for features which are not predicted.""" - return self._exogenous_feature_columns - - # TODO(allenl): Move more of the generic machinery for generating and - # predicting into TimeSeriesModel, and possibly share it between generate() - # and predict() - def generate(self, number_of_series, series_length, - model_parameters=None, seed=None): - """Sample synthetic data from model parameters, with optional substitutions. - - Returns `number_of_series` possible sequences of future values, sampled from - the generative model with each conditioned on the previous. Samples are - based on trained parameters, except for those parameters explicitly - overridden in `model_parameters`. - - For distributions over future observations, see predict(). - - Args: - number_of_series: Number of time series to create. - series_length: Length of each time series. - model_parameters: A dictionary mapping model parameters to values, which - replace trained parameters when generating data. - seed: If specified, return deterministic time series according to this - value. - Returns: - A dictionary with keys TrainEvalFeatures.TIMES (mapping to an array with - shape [number_of_series, series_length]) and TrainEvalFeatures.VALUES - (mapping to an array with shape [number_of_series, series_length, - num_features]). - """ - raise NotImplementedError("This model does not support generation.") - - def initialize_graph(self, input_statistics=None): - """Define ops for the model, not depending on any previously defined ops. - - Args: - input_statistics: A math_utils.InputStatistics object containing input - statistics. If None, data-independent defaults are used, which may - result in longer or unstable training. - """ - self._graph_initialized = True - self._input_statistics = input_statistics - if self._input_statistics: - self._stats_means, variances = ( - self._input_statistics.overall_feature_moments) - self._stats_sigmas = math_ops.sqrt(variances) - - def _scale_data(self, data): - """Scale data according to stats (input scale -> model scale).""" - if self._input_statistics is not None: - return (data - self._stats_means) / self._stats_sigmas - else: - return data - - def _scale_variance(self, variance): - """Scale variances according to stats (input scale -> model scale).""" - if self._input_statistics is not None: - return variance / self._input_statistics.overall_feature_moments.variance - else: - return variance - - def _scale_back_data(self, data): - """Scale back data according to stats (model scale -> input scale).""" - if self._input_statistics is not None: - return (data * self._stats_sigmas) + self._stats_means - else: - return data - - def _scale_back_variance(self, variance): - """Scale back variances according to stats (model scale -> input scale).""" - if self._input_statistics is not None: - return variance * self._input_statistics.overall_feature_moments.variance - else: - return variance - - def _check_graph_initialized(self): - if not self._graph_initialized: - raise ValueError( - "TimeSeriesModels require initialize_graph() to be called before " - "use. This defines variables and ops in the default graph, and " - "allows Tensor-valued input statistics to be specified.") - - def define_loss(self, features, mode): - """Default loss definition with state replicated across a batch. - - Time series passed to this model have a batch dimension, and each series in - a batch can be operated on in parallel. This loss definition assumes that - each element of the batch represents an independent sample conditioned on - the same initial state (i.e. it is simply replicated across the batch). A - batch size of one provides sequential operations on a single time series. - - More complex processing may operate instead on get_start_state() and - get_batch_loss() directly. - - Args: - features: A dictionary (such as is produced by a chunker) with at minimum - the following key/value pairs (others corresponding to the - `exogenous_feature_columns` argument to `__init__` may be included - representing exogenous regressors): - TrainEvalFeatures.TIMES: A [batch size x window size] integer Tensor - with times for each observation. If there is no artificial chunking, - the window size is simply the length of the time series. - TrainEvalFeatures.VALUES: A [batch size x window size x num features] - Tensor with values for each observation. - mode: The tf.estimator.ModeKeys mode to use (TRAIN, EVAL). For INFER, - see predict(). - Returns: - A ModelOutputs object. - """ - self._check_graph_initialized() - start_state = math_utils.replicate_state( - start_state=self.get_start_state(), - batch_size=array_ops.shape(features[TrainEvalFeatures.TIMES])[0]) - return self.get_batch_loss(features=features, mode=mode, state=start_state) - - # TODO(vitalyk,allenl): Better documentation surrounding options for chunking, - # references to papers, etc. - @abc.abstractmethod - def get_start_state(self): - """Returns a tuple of state for the start of the time series. - - For example, a mean and covariance. State should not have a batch - dimension, and will often be TensorFlow Variables to be learned along with - the rest of the model parameters. - """ - pass - - @abc.abstractmethod - def get_batch_loss(self, features, mode, state): - """Return predictions, losses, and end state for a time series. - - Args: - features: A dictionary with times, values, and (optionally) exogenous - regressors. See `define_loss`. - mode: The tf.estimator.ModeKeys mode to use (TRAIN, EVAL, INFER). - state: Model-dependent state, each with size [batch size x ...]. The - number and type will typically be fixed by the model (for example a - mean and variance). - Returns: - A ModelOutputs object. - """ - pass - - @abc.abstractmethod - def predict(self, features): - """Returns predictions of future observations given an initial state. - - Computes distributions for future observations. For sampled draws from the - model where each is conditioned on the previous, see generate(). - - Args: - features: A dictionary with at minimum the following key/value pairs - (others corresponding to the `exogenous_feature_columns` argument to - `__init__` may be included representing exogenous regressors): - PredictionFeatures.TIMES: A [batch size x window size] Tensor with - times to make predictions for. Times must be increasing within each - part of the batch, and must be greater than the last time `state` was - updated. - PredictionFeatures.STATE_TUPLE: Model-dependent state, each with size - [batch size x ...]. The number and type will typically be fixed by the - model (for example a mean and variance). Typically these will be the - end state returned by get_batch_loss, predicting beyond that data. - Returns: - A dictionary with model-dependent predictions corresponding to the - requested times. Keys indicate the type of prediction, and values have - shape [batch size x window size x ...]. For example state space models - return a "predicted_mean" and "predicted_covariance". - """ - pass - - def _get_exogenous_embedding_shape(self): - """Computes the shape of the vector returned by _process_exogenous_features. - - Returns: - The shape as a list. Does not include a batch dimension. - """ - if not self._exogenous_feature_columns: - return (0,) - with ops.Graph().as_default(): - parsed_features = ( - feature_column.make_parse_example_spec( - self._exogenous_feature_columns)) - placeholder_features = parsing_ops.parse_example( - serialized=array_ops.placeholder(shape=[None], dtype=dtypes.string), - features=parsed_features) - embedded = feature_column.input_layer( - features=placeholder_features, - feature_columns=self._exogenous_feature_columns) - return embedded.get_shape().as_list()[1:] - - def _process_exogenous_features(self, times, features): - """Create a single vector from exogenous features. - - Args: - times: A [batch size, window size] vector of times for this batch, - primarily used to check the shape information of exogenous features. - features: A dictionary of exogenous features corresponding to the columns - in self._exogenous_feature_columns. Each value should have a shape - prefixed by [batch size, window size]. - Returns: - A Tensor with shape [batch size, window size, exogenous dimension], where - the size of the exogenous dimension depends on the exogenous feature - columns passed to the model's constructor. - Raises: - ValueError: If an exogenous feature has an unknown rank. - """ - if self._exogenous_feature_columns: - exogenous_features_single_batch_dimension = {} - for name, tensor in features.items(): - if tensor.get_shape().ndims is None: - # input_from_feature_columns does not support completely unknown - # feature shapes, so we save on a bit of logic and provide a better - # error message by checking that here. - raise ValueError( - ("Features with unknown rank are not supported. Got shape {} for " - "feature {}.").format(tensor.get_shape(), name)) - tensor_shape_dynamic = array_ops.shape(tensor) - tensor = array_ops.reshape( - tensor, - array_ops.concat([[tensor_shape_dynamic[0] - * tensor_shape_dynamic[1]], - tensor_shape_dynamic[2:]], axis=0)) - # Avoid shape warnings when embedding "scalar" exogenous features (those - # with only batch and window dimensions); input_from_feature_columns - # expects input ranks to match the embedded rank. - if tensor.get_shape().ndims == 1 and tensor.dtype != dtypes.string: - exogenous_features_single_batch_dimension[name] = tensor[:, None] - else: - exogenous_features_single_batch_dimension[name] = tensor - embedded_exogenous_features_single_batch_dimension = ( - feature_column.input_layer( - features=exogenous_features_single_batch_dimension, - feature_columns=self._exogenous_feature_columns, - trainable=True)) - exogenous_regressors = array_ops.reshape( - embedded_exogenous_features_single_batch_dimension, - array_ops.concat( - [ - array_ops.shape(times), array_ops.shape( - embedded_exogenous_features_single_batch_dimension)[1:] - ], - axis=0)) - exogenous_regressors.set_shape(times.get_shape().concatenate( - embedded_exogenous_features_single_batch_dimension.get_shape()[1:])) - exogenous_regressors = math_ops.cast( - exogenous_regressors, dtype=self.dtype) - else: - # Not having any exogenous features is a special case so that models can - # avoid superfluous updates, which may not be free of side effects due to - # bias terms in transformations. - exogenous_regressors = None - return exogenous_regressors - - -# TODO(allenl): Add a superclass of SequentialTimeSeriesModel which fuses -# filtering/prediction/exogenous into one step, and move looping constructs to -# that class. -class SequentialTimeSeriesModel(TimeSeriesModel): - """Base class for recurrent generative models. - - Models implementing this interface have three main functions, corresponding to - abstract methods: - _filtering_step: Updates state based on observations and computes a loss. - _prediction_step: Predicts a batch of observations and new model state. - _imputation_step: Updates model state across a gap. - _exogenous_input_step: Updates state to account for exogenous regressors. - - Models may also specify a _window_initializer to prepare for a window of data. - - See StateSpaceModel for a concrete example of a model implementing this - interface. - - """ - - def __init__(self, - train_output_names, - predict_output_names, - num_features, - normalize_features=False, - dtype=dtypes.float32, - exogenous_feature_columns=None, - exogenous_update_condition=None, - static_unrolling_window_size_threshold=None): - """Initialize a SequentialTimeSeriesModel. - - Args: - train_output_names: A list of products/predictions returned from - _filtering_step. - predict_output_names: A list of products/predictions returned from - _prediction_step. - num_features: Number of features for the time series - normalize_features: Boolean. If True, `values` are passed normalized to - the model (via self._scale_data). Scaling is done for the whole window - as a batch, which is slightly more efficient than scaling inside the - window loop. The model must then define _scale_back_predictions, which - may use _scale_back_data or _scale_back_variance to return predictions - to the input scale. - dtype: The floating point datatype to use. - exogenous_feature_columns: A list of `tf.feature_column`s objects. See - `TimeSeriesModel`. - exogenous_update_condition: A function taking two Tensor arguments `times` - (shape [batch size]) and `features` (a dictionary mapping exogenous - feature keys to Tensors with shapes [batch size, ...]) and returning a - boolean Tensor with shape [batch size] indicating whether state should - be updated using exogenous features for each part of the batch. Where - it is False, no exogenous update is performed. If None (default), - exogenous updates are always performed. Useful for avoiding "leaky" - frequent exogenous updates when sparse updates are desired. Called - only during graph construction. - static_unrolling_window_size_threshold: Controls whether a `tf.while_loop` - is used when looping over a window of data. If - `static_unrolling_window_size_threshold` is None, a `tf.while_loop` is - always used. Otherwise it must be an integer, and the graph is - replicated for each step taken whenever the window size is less than - or equal to this value (if the window size is available in the static - shape information of the TrainEvalFeatures.TIMES feature). Static - unrolling generally decreases the per-step time for small window/batch - sizes, but increases graph construction time. - """ - super(SequentialTimeSeriesModel, self).__init__( - num_features=num_features, dtype=dtype, - exogenous_feature_columns=exogenous_feature_columns) - self._exogenous_update_condition = exogenous_update_condition - self._train_output_names = train_output_names - self._predict_output_names = predict_output_names - self._normalize_features = normalize_features - self._static_unrolling_window_size_threshold = ( - static_unrolling_window_size_threshold) - - def _scale_back_predictions(self, predictions): - """Return a window of predictions to input scale. - - Args: - predictions: A dictionary mapping from prediction names to Tensors. - Returns: - A dictionary with values corrected for input normalization (e.g. with - self._scale_back_mean and possibly self._scale_back_variance). May be a - mutated version of the argument. - """ - raise NotImplementedError( - "SequentialTimeSeriesModel normalized input data" - " (normalize_features=True), but no method was provided to transform " - "the predictions back to the input scale.") - - @abc.abstractmethod - def _filtering_step(self, current_times, current_values, state, predictions): - """Compute a single-step loss for a batch of data. - - Args: - current_times: A [batch size] Tensor of times for each observation. - current_values: A [batch size] Tensor of values for each observation. - state: Model state, updated to current_times. - predictions: The outputs of _prediction_step - Returns: - A tuple of (updated state, outputs): - updated state: Model state taking current_values into account. - outputs: A dictionary of Tensors with keys corresponding to - self._train_output_names, plus a special "loss" key. The value - corresponding to "loss" is minimized during training. Other outputs - may include one-step-ahead predictions, for example a predicted - location and scale. - """ - pass - - @abc.abstractmethod - def _prediction_step(self, current_times, state): - """Compute a batch of single-step predictions. - - Args: - current_times: A [batch size] Tensor of times for each observation. - state: Model state, imputed to one step before current_times. - Returns: - A tuple of (updated state, outputs): - updated state: Model state updated to current_times. - outputs: A dictionary of Tensors with keys corresponding to - self._predict_output_names. - """ - pass - - @abc.abstractmethod - def _imputation_step(self, current_times, state): - """Update model state across missing values. - - Called to prepare model state for _filtering_step and _prediction_step. - - Args: - current_times: A [batch size] Tensor; state will be imputed up to, but not - including, these timesteps. - state: The pre-imputation model state, Tensors with shape [batch size x - ...]. - Returns: - Updated/imputed model state, corresponding to `state`. - """ - pass - - @abc.abstractmethod - def _exogenous_input_step( - self, current_times, current_exogenous_regressors, state): - """Update state to account for exogenous regressors. - - Args: - current_times: A [batch size] Tensor of times for the exogenous values - being input. - current_exogenous_regressors: A [batch size x exogenous input dimension] - Tensor of exogenous values for each part of the batch. - state: Model state, a possibly nested list of Tensors, each with shape - [batch size x ...]. - Returns: - Updated model state, structure and shapes matching the `state` argument. - """ - pass - - # TODO(allenl): Move regularization to a separate object (optional and - # configurable) - def _loss_additions(self, times, values, mode): - """Additions to per-observation normalized loss, e.g. regularization. - - Args: - times: A [batch size x window size] Tensor with times for each - observation. - values: A [batch size x window size x num features] Tensor with values for - each observation. - mode: The tf.estimator.ModeKeys mode to use (TRAIN, EVAL, INFER). - Returns: - A scalar value to add to the per-observation normalized loss. - """ - del times, values, mode - return 0. - - def _window_initializer(self, times, state): - """Prepare for training or prediction on a window of data. - - Args: - times: A [batch size x window size] Tensor with times for each - observation. - state: Model-dependent state, each with size [batch size x ...]. The - number and type will typically be fixed by the model (for example a - mean and variance). - Returns: - Nothing - """ - pass - - def get_batch_loss(self, features, mode, state): - """Calls self._filtering_step. See TimeSeriesModel.get_batch_loss.""" - per_observation_loss, state, outputs = self.per_step_batch_loss( - features, mode, state) - # per_step_batch_loss returns [batch size, window size, ...] state, whereas - # get_batch_loss is expected to return [batch size, ...] state for the last - # element of a window - state = nest.pack_sequence_as( - state, - [state_element[:, -1] for state_element in nest.flatten(state)]) - outputs["observed"] = features[TrainEvalFeatures.VALUES] - return ModelOutputs( - loss=per_observation_loss, - end_state=state, - predictions=outputs, - prediction_times=features[TrainEvalFeatures.TIMES]) - - def _apply_exogenous_update( - self, current_times, step_number, state, raw_features, - embedded_exogenous_regressors): - """Performs a conditional state update based on exogenous features.""" - if embedded_exogenous_regressors is None: - return state - else: - current_exogenous_regressors = embedded_exogenous_regressors[ - :, step_number, :] - exogenous_updated_state = self._exogenous_input_step( - current_times=current_times, - current_exogenous_regressors=current_exogenous_regressors, - state=state) - if self._exogenous_update_condition is not None: - current_raw_exogenous_features = { - key: value[:, step_number] for key, value in raw_features.items() - if key not in [PredictionFeatures.STATE_TUPLE, - TrainEvalFeatures.TIMES, - TrainEvalFeatures.VALUES]} - conditionally_updated_state_flat = [] - for updated_state_element, original_state_element in zip( - nest.flatten(exogenous_updated_state), - nest.flatten(state)): - conditionally_updated_state_flat.append( - array_ops.where( - self._exogenous_update_condition( - times=current_times, - features=current_raw_exogenous_features), - updated_state_element, - original_state_element)) - return nest.pack_sequence_as(state, conditionally_updated_state_flat) - else: - return exogenous_updated_state - - def per_step_batch_loss(self, features, mode, state): - """Computes predictions, losses, and intermediate model states. - - Args: - features: A dictionary with times, values, and (optionally) exogenous - regressors. See `define_loss`. - mode: The tf.estimator.ModeKeys mode to use (TRAIN, EVAL, INFER). - state: Model-dependent state, each with size [batch size x ...]. The - number and type will typically be fixed by the model (for example a - mean and variance). - Returns: - A tuple of (loss, filtered_states, predictions) - loss: Average loss values across the batch. - filtered_states: For each Tensor in `state` with shape [batch size x - ...], `filtered_states` has a Tensor with shape [batch size x window - size x ...] with filtered state for each part of the batch and - window. - predictions: A dictionary with model-dependent one-step-ahead (or - at-least-one-step-ahead with missing values) predictions, with keys - indicating the type of prediction and values having shape [batch - size x window size x ...]. For example state space models provide - "mean", "covariance", and "log_likelihood". - - """ - self._check_graph_initialized() - times = math_ops.cast(features[TrainEvalFeatures.TIMES], dtype=dtypes.int64) - values = math_ops.cast(features[TrainEvalFeatures.VALUES], dtype=self.dtype) - if self._normalize_features: - values = self._scale_data(values) - exogenous_regressors = self._process_exogenous_features( - times=times, - features={key: value for key, value in features.items() - if key not in [TrainEvalFeatures.TIMES, - TrainEvalFeatures.VALUES]}) - def _batch_loss_filtering_step(step_number, current_times, state): - """Make a prediction and update it based on data.""" - current_values = values[:, step_number, :] - state = self._apply_exogenous_update( - step_number=step_number, current_times=current_times, state=state, - raw_features=features, - embedded_exogenous_regressors=exogenous_regressors) - predicted_state, predictions = self._prediction_step( - current_times=current_times, - state=state) - filtered_state, outputs = self._filtering_step( - current_times=current_times, - current_values=current_values, - state=predicted_state, - predictions=predictions) - return filtered_state, outputs - state, outputs = self._state_update_loop( - times=times, state=state, state_update_fn=_batch_loss_filtering_step, - outputs=["loss"] + self._train_output_names) - outputs["loss"].set_shape(times.get_shape()) - loss_sum = math_ops.reduce_sum(outputs["loss"]) - per_observation_loss = (loss_sum / math_ops.cast( - math_ops.reduce_prod(array_ops.shape(times)), dtype=self.dtype)) - per_observation_loss += self._loss_additions(times, values, mode) - # Since we have window-level additions to the loss, its per-step value is - # misleading, so we avoid returning it. - del outputs["loss"] - if self._normalize_features: - outputs = self._scale_back_predictions(outputs) - return per_observation_loss, state, outputs - - def predict(self, features): - """Calls self._prediction_step in a loop. See TimeSeriesModel.predict.""" - predict_times = ops.convert_to_tensor(features[PredictionFeatures.TIMES], - dtypes.int64) - start_state = features[PredictionFeatures.STATE_TUPLE] - exogenous_regressors = self._process_exogenous_features( - times=predict_times, - features={ - key: value - for key, value in features.items() - if key not in - [PredictionFeatures.TIMES, PredictionFeatures.STATE_TUPLE] - }) - def _call_prediction_step(step_number, current_times, state): - state = self._apply_exogenous_update( - step_number=step_number, current_times=current_times, state=state, - raw_features=features, - embedded_exogenous_regressors=exogenous_regressors) - state, outputs = self._prediction_step( - current_times=current_times, state=state) - return state, outputs - _, predictions = self._state_update_loop( - times=predict_times, state=start_state, - state_update_fn=_call_prediction_step, - outputs=self._predict_output_names) - if self._normalize_features: - predictions = self._scale_back_predictions(predictions) - return predictions - - class _FakeTensorArray(object): - """An interface for Python lists that is similar to TensorArray. - - Used for easy switching between static and dynamic looping. - """ - - def __init__(self): - self.values = [] - - def write(self, unused_position, value): - del unused_position - self.values.append(value) - return self - - def _state_update_loop(self, times, state, state_update_fn, outputs): - """Iterates over `times`, calling `state_update_fn` to collect outputs. - - Args: - times: A [batch size x window size] Tensor of integers to iterate over. - state: A list of model-specific state Tensors, each with shape [batch size - x ...]. - state_update_fn: A callback taking the following arguments - step_number; A scalar integer Tensor indicating the current position - in the window. - current_times; A [batch size] vector of Integers indicating times - for each part of the batch. - state; Current model state. - It returns a tuple of (updated state, output_values), output_values - being a dictionary of Tensors with keys corresponding to `outputs`. - outputs: A list of strings indicating values which will be saved while - iterating. Must match the keys of the dictionary returned by - state_update_fn. - Returns: - A tuple of (state, output_dict) - state: The final model state. - output_dict: A dictionary of outputs corresponding to those specified in - `outputs` and computed in state_update_fn. - """ - times = ops.convert_to_tensor(times, dtype=dtypes.int64) - window_static_shape = tensor_shape.dimension_value(times.shape[1]) - if self._static_unrolling_window_size_threshold is None: - static_unroll = False - else: - # The user has specified a threshold for static loop unrolling. - if window_static_shape is None: - # We don't have static shape information for the window size, so dynamic - # looping is our only option. - static_unroll = False - elif window_static_shape <= self._static_unrolling_window_size_threshold: - # The threshold is satisfied; unroll statically - static_unroll = True - else: - # A threshold was set but not satisfied - static_unroll = False - - self._window_initializer(times, state) - - def _run_condition(step_number, *unused): - del unused # not part of while loop run condition - return math_ops.less(step_number, window_size) - - def _state_update_step( - step_number, state, state_accumulators, output_accumulators, - reuse=False): - """Impute, then take one state_update_fn step, accumulating outputs.""" - with variable_scope.variable_scope("state_update_step", reuse=reuse): - current_times = times[:, step_number] - state = self._imputation_step(current_times=current_times, state=state) - output_accumulators_dict = { - accumulator_key: accumulator - for accumulator_key, accumulator - in zip(outputs, output_accumulators)} - step_state, output_values = state_update_fn( - step_number=step_number, - current_times=current_times, - state=state) - assert set(output_values.keys()) == set(outputs) - new_output_accumulators = [] - for output_key in outputs: - accumulator = output_accumulators_dict[output_key] - output_value = output_values[output_key] - new_output_accumulators.append( - accumulator.write(step_number, output_value)) - flat_step_state = nest.flatten(step_state) - assert len(state_accumulators) == len(flat_step_state) - new_state_accumulators = [] - new_state_flat = [] - for step_state_value, state_accumulator, original_state in zip( - flat_step_state, state_accumulators, nest.flatten(state)): - # Make sure the static shape information is complete so while_loop - # does not complain about shape information changing. - step_state_value.set_shape(original_state.get_shape()) - new_state_flat.append(step_state_value) - new_state_accumulators.append(state_accumulator.write( - step_number, step_state_value)) - step_state = nest.pack_sequence_as(state, new_state_flat) - return (step_number + 1, step_state, - new_state_accumulators, new_output_accumulators) - - window_size = array_ops.shape(times)[1] - - def _window_size_tensor_array(dtype): - if static_unroll: - return self._FakeTensorArray() - else: - return tensor_array_ops.TensorArray( - dtype=dtype, size=window_size, dynamic_size=False) - - initial_loop_arguments = [ - array_ops.zeros([], dtypes.int32), - state, - [_window_size_tensor_array(element.dtype) - for element in nest.flatten(state)], - [_window_size_tensor_array(self.dtype) for _ in outputs]] - if static_unroll: - arguments = initial_loop_arguments - for step_number in range(tensor_shape.dimension_value(times.shape[1])): - arguments = _state_update_step( - array_ops.constant(step_number, dtypes.int32), *arguments[1:], - reuse=(step_number > 0)) # Variable sharing between steps - else: - arguments = control_flow_ops.while_loop( - cond=_run_condition, - body=_state_update_step, - loop_vars=initial_loop_arguments) - (_, _, state_loop_result, outputs_loop_result) = arguments - - def _stack_and_transpose(tensor_array): - """Stack and re-order the dimensions of a TensorArray.""" - if static_unroll: - return array_ops.stack(tensor_array.values, axis=1) - else: - # TensorArrays from while_loop stack with window size as the first - # dimension, so this function swaps it and the batch dimension to - # maintain the [batch x window size x ...] convention used elsewhere. - stacked = tensor_array.stack() - return array_ops.transpose( - stacked, - perm=array_ops.concat([[1, 0], math_ops.range( - 2, array_ops.rank(stacked))], 0)) - - outputs_dict = {output_key: _stack_and_transpose(output) - for output_key, output - in zip(outputs, outputs_loop_result)} - full_state = nest.pack_sequence_as( - state, - [_stack_and_transpose(state_element) - for state_element in state_loop_result]) - return full_state, outputs_dict diff --git a/tensorflow/contrib/timeseries/python/timeseries/model_utils.py b/tensorflow/contrib/timeseries/python/timeseries/model_utils.py deleted file mode 100644 index b5d7cb376b6..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/model_utils.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Helper functions for training and constructing time series Models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy - -from tensorflow.contrib.timeseries.python.timeseries import feature_keys - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import variable_scope - - -# TODO(agarwal): Remove and replace with functionality from tf.slim -def fully_connected(inp, - inp_size, - layer_size, - name, - activation=nn_ops.relu, - dtype=dtypes.float32): - """Helper method to create a fully connected hidden layer.""" - wt = variable_scope.get_variable( - name="{}_weight".format(name), shape=[inp_size, layer_size], dtype=dtype) - bias = variable_scope.get_variable( - name="{}_bias".format(name), - shape=[layer_size], - initializer=init_ops.zeros_initializer()) - output = nn_ops.xw_plus_b(inp, wt, bias) - if activation is not None: - assert callable(activation) - output = activation(output) - return output - - -def parameter_switch(parameter_overrides): - """Create a function which chooses between overridden and model parameters. - - Args: - parameter_overrides: A dictionary with explicit overrides of model - parameters, mapping from Tensors to their overridden values. - Returns: - A function which takes a Tensor and returns the override if it is specified, - or otherwise the evaluated value (given current Variable values). - """ - def get_passed_or_trained_value(parameter): - return ops.convert_to_tensor( - parameter_overrides.get(parameter, parameter)).eval() - return get_passed_or_trained_value - - -def canonicalize_times_or_steps_from_output(times, steps, - previous_model_output): - """Canonicalizes either relative or absolute times, with error checking.""" - if steps is not None and times is not None: - raise ValueError("Only one of `steps` and `times` may be specified.") - if steps is None and times is None: - raise ValueError("One of `steps` and `times` must be specified.") - if times is not None: - times = numpy.array(times) - if len(times.shape) != 2: - times = times[None, ...] - if (previous_model_output[feature_keys.FilteringResults.TIMES].shape[0] != - times.shape[0]): - raise ValueError( - ("`times` must have a batch dimension matching" - " the previous model output (got a batch dimension of {} for `times`" - " and {} for the previous model output).").format( - times.shape[0], previous_model_output[ - feature_keys.FilteringResults.TIMES].shape[0])) - if not (previous_model_output[feature_keys.FilteringResults.TIMES][:, -1] < - times[:, 0]).all(): - raise ValueError("Prediction times must be after the corresponding " - "previous model output.") - if steps is not None: - predict_times = ( - previous_model_output[feature_keys.FilteringResults.TIMES][:, -1:] + 1 + - numpy.arange(steps)[None, ...]) - else: - predict_times = times - return predict_times diff --git a/tensorflow/contrib/timeseries/python/timeseries/model_utils_test.py b/tensorflow/contrib/timeseries/python/timeseries/model_utils_test.py deleted file mode 100644 index a049dbe773b..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/model_utils_test.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for model_utils.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.python.timeseries import model_utils - -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class ModelUtilsTest(test.TestCase): - - def test_parameter_switching(self): - parameter = array_ops.constant(5) - overridden_parameter = array_ops.constant(3) - with self.cached_session(): - getter = model_utils.parameter_switch({overridden_parameter: 4}) - self.assertEqual(5, getter(parameter)) - self.assertEqual(4, getter(overridden_parameter)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/saved_model_utils.py b/tensorflow/contrib/timeseries/python/timeseries/saved_model_utils.py deleted file mode 100644 index 27c8bfe653d..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/saved_model_utils.py +++ /dev/null @@ -1,220 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Convenience functions for working with time series saved_models. - -@@predict_continuation -@@cold_start_filter -@@filter_continuation -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.python.timeseries import feature_keys as _feature_keys -from tensorflow.contrib.timeseries.python.timeseries import head as _head -from tensorflow.contrib.timeseries.python.timeseries import input_pipeline as _input_pipeline -from tensorflow.contrib.timeseries.python.timeseries import model_utils as _model_utils - -from tensorflow.python.util.all_util import remove_undocumented - - -def _colate_features_to_feeds_and_fetches(signature, - features, - graph, - continue_from=None): - """Uses a saved model signature to construct feed and fetch dictionaries.""" - if continue_from is None: - state_values = {} - elif _feature_keys.FilteringResults.STATE_TUPLE in continue_from: - # We're continuing from an evaluation, so we need to unpack/flatten state. - state_values = _head.state_to_dictionary( - continue_from[_feature_keys.FilteringResults.STATE_TUPLE]) - else: - state_values = continue_from - input_feed_tensors_by_name = { - input_key: graph.as_graph_element(input_value.name) - for input_key, input_value in signature.inputs.items() - } - output_tensors_by_name = { - output_key: graph.as_graph_element(output_value.name) - for output_key, output_value in signature.outputs.items() - } - feed_dict = {} - for state_key, state_value in state_values.items(): - feed_dict[input_feed_tensors_by_name[state_key]] = state_value - for feature_key, feature_value in features.items(): - feed_dict[input_feed_tensors_by_name[feature_key]] = feature_value - return output_tensors_by_name, feed_dict - - -def predict_continuation(continue_from, - signatures, - session, - steps=None, - times=None, - exogenous_features=None): - """Perform prediction using an exported saved model. - - Analogous to _input_pipeline.predict_continuation_input_fn, but operates on a - saved model rather than feeding into Estimator's predict method. - - Args: - continue_from: A dictionary containing the results of either an Estimator's - evaluate method or filter_continuation. Used to determine the model state - to make predictions starting from. - signatures: The `MetaGraphDef` protocol buffer returned from - `tf.compat.v1.saved_model.loader.load`. Used to determine the names of - Tensors to feed and fetch. Must be from the same model as `continue_from`. - session: The session to use. The session's graph must be the one into which - `tf.compat.v1.saved_model.loader.load` loaded the model. - steps: The number of steps to predict (scalar), starting after the - evaluation or filtering. If `times` is specified, `steps` must not be; one - is required. - times: A [batch_size x window_size] array of integers (not a Tensor) - indicating times to make predictions for. These times must be after the - corresponding evaluation or filtering. If `steps` is specified, `times` - must not be; one is required. If the batch dimension is omitted, it is - assumed to be 1. - exogenous_features: Optional dictionary. If specified, indicates exogenous - features for the model to use while making the predictions. Values must - have shape [batch_size x window_size x ...], where `batch_size` matches - the batch dimension used when creating `continue_from`, and `window_size` - is either the `steps` argument or the `window_size` of the `times` - argument (depending on which was specified). - - Returns: - A dictionary with model-specific predictions (typically having keys "mean" - and "covariance") and a feature_keys.PredictionResults.TIMES key indicating - the times for which the predictions were computed. - Raises: - ValueError: If `times` or `steps` are misspecified. - """ - if exogenous_features is None: - exogenous_features = {} - predict_times = _model_utils.canonicalize_times_or_steps_from_output( - times=times, steps=steps, previous_model_output=continue_from) - features = {_feature_keys.PredictionFeatures.TIMES: predict_times} - features.update(exogenous_features) - predict_signature = signatures.signature_def[ - _feature_keys.SavedModelLabels.PREDICT] - output_tensors_by_name, feed_dict = _colate_features_to_feeds_and_fetches( - continue_from=continue_from, - signature=predict_signature, - features=features, - graph=session.graph) - output = session.run(output_tensors_by_name, feed_dict=feed_dict) - output[_feature_keys.PredictionResults.TIMES] = features[ - _feature_keys.PredictionFeatures.TIMES] - return output - - -def cold_start_filter(signatures, session, features): - """Perform filtering using an exported saved model. - - Filtering refers to updating model state based on new observations. - Predictions based on the returned model state will be conditioned on these - observations. - - Starts from the model's default/uninformed state. - - Args: - signatures: The `MetaGraphDef` protocol buffer returned from - `tf.compat.v1.saved_model.loader.load`. Used to determine the names of - Tensors to feed and fetch. Must be from the same model as `continue_from`. - session: The session to use. The session's graph must be the one into which - `tf.compat.v1.saved_model.loader.load` loaded the model. - features: A dictionary mapping keys to Numpy arrays, with several possible - shapes (requires keys `FilteringFeatures.TIMES` and - `FilteringFeatures.VALUES`): Single example; `TIMES` is a scalar and - `VALUES` is either a scalar or a vector of length [number of features]. - Sequence; `TIMES` is a vector of shape [series length], `VALUES` either - has shape [series length] (univariate) or [series length x number of - features] (multivariate). Batch of sequences; `TIMES` is a vector of - shape [batch size x series length], `VALUES` has shape [batch size x - series length] or [batch size x series length x number of features]. In - any case, `VALUES` and any exogenous features must have their shapes - prefixed by the shape of the value corresponding to the `TIMES` key. - - Returns: - A dictionary containing model state updated to account for the observations - in `features`. - """ - filter_signature = signatures.signature_def[ - _feature_keys.SavedModelLabels.COLD_START_FILTER] - features = _input_pipeline._canonicalize_numpy_data( # pylint: disable=protected-access - data=features, - require_single_batch=False) - output_tensors_by_name, feed_dict = _colate_features_to_feeds_and_fetches( - signature=filter_signature, features=features, graph=session.graph) - output = session.run(output_tensors_by_name, feed_dict=feed_dict) - # Make it easier to chain filter -> predict by keeping track of the current - # time. - output[_feature_keys.FilteringResults.TIMES] = features[ - _feature_keys.FilteringFeatures.TIMES] - return output - - -def filter_continuation(continue_from, signatures, session, features): - """Perform filtering using an exported saved model. - - Filtering refers to updating model state based on new observations. - Predictions based on the returned model state will be conditioned on these - observations. - - Args: - continue_from: A dictionary containing the results of either an Estimator's - evaluate method or a previous filter step (cold start or continuation). - Used to determine the model state to start filtering from. - signatures: The `MetaGraphDef` protocol buffer returned from - `tf.compat.v1.saved_model.loader.load`. Used to determine the names of - Tensors to feed and fetch. Must be from the same model as `continue_from`. - session: The session to use. The session's graph must be the one into which - `tf.compat.v1.saved_model.loader.load` loaded the model. - features: A dictionary mapping keys to Numpy arrays, with several possible - shapes (requires keys `FilteringFeatures.TIMES` and - `FilteringFeatures.VALUES`): Single example; `TIMES` is a scalar and - `VALUES` is either a scalar or a vector of length [number of features]. - Sequence; `TIMES` is a vector of shape [series length], `VALUES` either - has shape [series length] (univariate) or [series length x number of - features] (multivariate). Batch of sequences; `TIMES` is a vector of - shape [batch size x series length], `VALUES` has shape [batch size x - series length] or [batch size x series length x number of features]. In - any case, `VALUES` and any exogenous features must have their shapes - prefixed by the shape of the value corresponding to the `TIMES` key. - - Returns: - A dictionary containing model state updated to account for the observations - in `features`. - """ - filter_signature = signatures.signature_def[ - _feature_keys.SavedModelLabels.FILTER] - features = _input_pipeline._canonicalize_numpy_data( # pylint: disable=protected-access - data=features, - require_single_batch=False) - output_tensors_by_name, feed_dict = _colate_features_to_feeds_and_fetches( - continue_from=continue_from, - signature=filter_signature, - features=features, - graph=session.graph) - output = session.run(output_tensors_by_name, feed_dict=feed_dict) - # Make it easier to chain filter -> predict by keeping track of the current - # time. - output[_feature_keys.FilteringResults.TIMES] = features[ - _feature_keys.FilteringFeatures.TIMES] - return output - - -remove_undocumented(module_name=__name__) diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_management.py b/tensorflow/contrib/timeseries/python/timeseries/state_management.py deleted file mode 100644 index 138406c6168..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_management.py +++ /dev/null @@ -1,265 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Classes for wrapping a model to operate on different data shapes.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -from tensorflow.contrib.timeseries.python.timeseries import feature_keys -from tensorflow.contrib.timeseries.python.timeseries import math_utils -from tensorflow.contrib.timeseries.python.timeseries.model import ModelOutputs - -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.util import nest - - -class PassthroughStateManager(object): - """A minimal wrapper for models which do not need state management.""" - - def __init__(self): - self._input_statistics = None - self._graph_initialized = False - - def initialize_graph(self, model, input_statistics=None): - """Adds required operations to the graph.""" - del model # unused - self._graph_initialized = True - self._input_statistics = input_statistics - - def define_loss(self, model, features, mode): - """Wrap "model" with StateManager-specific operations. - - Args: - model: The model (inheriting from TimeSeriesModel) to manage state for. - features: A dictionary with the following key/value pairs: - feature_keys.TrainEvalFeatures.TIMES: A [batch size x window size] - Tensor with times for each observation. - feature_keys.TrainEvalFeatures.VALUES: A [batch size x window size x num - features] Tensor with values for each observation. - mode: The tf.estimator.ModeKeys mode to use (TRAIN or EVAL). - Returns: - A ModelOutputs object. - Raises: - ValueError: If start state was specified. - """ - if feature_keys.State.STATE_TUPLE in features: - raise ValueError( - "Overriding start state is not supported for this model.") - return model.define_loss(features, mode) - - -class _OverridableStateManager(PassthroughStateManager): - """Base class for state managers which support overriding model state.""" - - @abc.abstractmethod - def _define_loss_with_saved_state(self, model, features, mode): - pass - - def define_loss(self, model, features, mode): - """Switches between explicit start state and managed state.""" - if feature_keys.FilteringFeatures.STATE_TUPLE in features: - # Explicit start state has been provided, so we should use that. - if mode == estimator_lib.ModeKeys.TRAIN: - raise ValueError( - "Overriding saved state for training is not supported (but a value " - "for feature {} was specified).".format( - feature_keys.FilteringFeatures.STATE_TUPLE)) - start_state = features[feature_keys.FilteringFeatures.STATE_TUPLE] - del features[feature_keys.FilteringFeatures.STATE_TUPLE] - return model.get_batch_loss( - features=features, mode=mode, state=start_state) - else: - # No explicit start state; use managed state. - return self._define_loss_with_saved_state( - model=model, features=features, mode=mode) - - -class FilteringOnlyStateManager(_OverridableStateManager): - """State manager for models which use state only for filtering. - - Window-based models (ARModel) do not require state to be fed during training - (instead requiring a specific window size). Rather than requiring a minimum - window size for filtering, these models maintain this window in their state, - and so need state to be fed. - """ - - def _define_loss_with_saved_state(self, model, features, mode): - return model.define_loss(features, mode) - - -class ChainingStateManager(_OverridableStateManager): - """Maintains state across a batch for SequentialTimeSeriesModel subclasses. - - The batch dimension is treated as indexing sequential chunks of the same - timeseries. End state from each chunk is fed as start state to the next chunk - during the next timestep. This is an approximation to full-batch training for - sequential models, but is typically much faster while still accurately - recovering parameters. The speedup comes from reduced scheduling overhead of - TensorFlow ops, since each operation can do much more work. - """ - - def __init__(self, state_saving_interval=20, checkpoint_state=False): - """Initialize the state manager. - - Args: - state_saving_interval: This state manager saves intermediate model state - every `state_saving_interval` times. Larger values save memory, and - checkpoint size if `checkpoint_state` is enabled, but models - will need to impute across artificial gaps of up to this size - (i.e. gaps not appearing in the original data). This imputation may - affect training. Set state_saving_interval to 1 to avoid any - artificial imputation. - checkpoint_state: If True, saved intermediate model state will be - written to checkpoints. Checkpoints will then scale with dataset - size. If False, state will be freshly imputed from the beginning of a - series each time the model is restored, which means it may take a few - iterations for state to warm up. - """ - super(ChainingStateManager, self).__init__() - self._checkpoint_state = checkpoint_state - self._state_saving_interval = state_saving_interval - self._start_state = None - self._cached_states = None - - def initialize_graph(self, model, input_statistics=None): - """Adds required operations to the graph.""" - super(ChainingStateManager, self).initialize_graph( - model=model, input_statistics=input_statistics) - self._start_state = model.get_start_state() - self._cached_states = math_utils.TupleOfTensorsLookup( - key_dtype=dtypes.int64, - default_values=self._start_state, - empty_key=-1, - deleted_key=-2, - name="cached_states", - checkpoint=self._checkpoint_state) - - def _define_loss_with_saved_state(self, model, features, mode): - """Feeds end state from one training iteration into the next. - - Args: - model: The model to wrap. Compatible with children of TimeSeriesModel. - features: Dictionary with Tensor values defining the data to be - processed. The expected key/value pairs are at minimum: - feature_keys.TrainEvalFeatures.TIMES: A [number of chunks x window - size] Tensor with times for each observation, the result of chunking - a single longer time series. - feature_keys.TrainEvalFeatures.VALUES: A [number of chunks x window - size x num features] Tensor with values for each observation, - corresponding to times. - mode: The tf.estimator.ModeKeys mode to use. For EVAL and INFER, no - batching is performed, which may be slow. This is to avoid giving - cached and almost certainly stale values. - Returns: - A ModelOutputs object. - Raises: - ValueError: If initialize_graph has not been called. - """ - if not self._graph_initialized: - raise ValueError("ChainingStateManager requires initialize_graph() to be " - "called before use.") - (loss_op, end_state, batch_predictions) = self._update_cached_states( - model=model, - features=features, - mode=mode) - # Add a batch dimension so state can be used directly (e.g. for predictions) - # without the user manually reshaping it. - last_end_state_flat = [end_state_value[-1][None] - for end_state_value in nest.flatten(end_state)] - batch_predictions["observed"] = features[ - feature_keys.TrainEvalFeatures.VALUES] - return ModelOutputs( - loss=loss_op, - end_state=nest.pack_sequence_as(end_state, last_end_state_flat), - predictions=batch_predictions, - prediction_times=features[feature_keys.TrainEvalFeatures.TIMES]) - - def _get_chunk_number(self, time): - return time // self._state_saving_interval - - def _get_cached_states(self, times): - """Retrieve cached states for a batch of times.""" - read_chunk_numbers = self._get_chunk_number(times) - looked_up_state = list(self._cached_states.lookup( - math_ops.cast(read_chunk_numbers, dtypes.int64))) - looked_up_state = tuple(looked_up_state) - # We need to special-case the first chunk in a series to explicitly rely on - # the model's starting state so that gradients flow back to it. Otherwise it - # would affect only initialization, and would not be read from or updated - # during training. Not doing this also isolates that part of the graph, - # leading to errors on model reload if there are trainable variables - # affecting a model's start state. - if self._input_statistics is not None: - start_time = self._input_statistics.start_time - else: - start_time = 0 - set_to_start_state = math_ops.equal(read_chunk_numbers, - self._get_chunk_number(start_time)) - new_states = [] - for start_state_value, cache_variable in zip( - nest.flatten( - math_utils.replicate_state(self._start_state, - array_ops.shape(times)[0])), - nest.flatten(looked_up_state)): - - new_states.append( - array_ops.where(set_to_start_state, start_state_value, - cache_variable)) - looked_up_state = nest.pack_sequence_as(looked_up_state, new_states) - return looked_up_state - - def _update_cached_states(self, model, features, mode): - """Read, process, and write chunks to the cache.""" - times = features[feature_keys.TrainEvalFeatures.TIMES] - looked_up_state = self._get_cached_states(times[:, 0]) - (model_loss, intermediate_states, - batch_predictions) = model.per_step_batch_loss( - features=features, - mode=mode, - state=looked_up_state) - # We need to at least write to the bucket after the one we read from. - min_chunk_numbers = self._get_chunk_number(times) + 1 - # We write to the bucket that would have been read had the window started at - # the next sample (except for the last sample in the window, which gets - # written to the next bucket). This assumes fixed missing times (i.e. if we - # were presented with times [10, 50] we will never see times [30, 50]). - # - # TODO(allenl): Retrieve the highest time less than the current time rather - # than relying on fixed bucketing. - write_chunk_numbers = math_ops.maximum( - self._get_chunk_number(array_ops.concat( - [times[:, 1:], times[:, -1:] + 1], axis=1)), - min_chunk_numbers) - # Write once for every computed state; this may mean that we write multiple - # times to the same cell, but later writes will take precedence. - save_ops = [ - self._cached_states.insert( - keys=write_chunk_numbers, - values=intermediate_states)] - end_state = nest.pack_sequence_as( - intermediate_states, - [state_element[:, -1] - for state_element in nest.flatten(intermediate_states)]) - with ops.control_dependencies(save_ops): - # Make sure end states get saved at each iteration - loss_op = array_ops.identity(model_loss) - return loss_op, end_state, batch_predictions diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_management_test.py b/tensorflow/contrib/timeseries/python/timeseries/state_management_test.py deleted file mode 100644 index 42ba6e1c256..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_management_test.py +++ /dev/null @@ -1,313 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for state management.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy - -from tensorflow.contrib.timeseries.python.timeseries import feature_keys -from tensorflow.contrib.timeseries.python.timeseries import input_pipeline -from tensorflow.contrib.timeseries.python.timeseries import math_utils -from tensorflow.contrib.timeseries.python.timeseries import model -from tensorflow.contrib.timeseries.python.timeseries import state_management -from tensorflow.contrib.timeseries.python.timeseries import test_utils - -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import coordinator as coordinator_lib -from tensorflow.python.training import queue_runner_impl -from tensorflow.python.training import training as train -from tensorflow.python.util import nest - - -class StubTimeSeriesModel(model.TimeSeriesModel): - - def __init__(self, correct_offset=False): - self._correct_offset = correct_offset - super(StubTimeSeriesModel, self).__init__(1) - - def initialize_graph(self, input_statistics=None): - super(StubTimeSeriesModel, self).initialize_graph( - input_statistics=input_statistics) - self.prior_var = variable_scope.get_variable( - "prior", [], initializer=init_ops.constant_initializer(0.)) - - def generate(self, *args): - pass - - def predict(self, *args): - pass - - def get_start_state(self): - return (array_ops.zeros([], dtype=dtypes.int64), self.prior_var) - - def get_batch_loss(self, features, mode, state): - raise NotImplementedError("This stub only supports managed state.") - - def per_step_batch_loss(self, features, mode, state): - times = features[feature_keys.TrainEvalFeatures.TIMES] - values = features[feature_keys.TrainEvalFeatures.VALUES] - (priors_from_time, prior) = state - time_corrected_priors = prior + math_ops.cast( - math_utils.batch_start_time(times) - priors_from_time, dtypes.float32) - posterior = time_corrected_priors[:, None] + math_ops.cast( - times - math_utils.batch_start_time(times)[:, None], dtypes.float32) - batch_end_values = array_ops.squeeze( - array_ops.slice(values, [0, array_ops.shape(times)[1] - 1, 0], - [-1, 1, -1]), - axis=[1, 2]) - # A pretty odd but easy to think about loss: L1 loss on the batch end - # values. - loss = math_ops.reduce_sum( - math_ops.abs( - array_ops.reshape(posterior[:, -1], [-1]) - batch_end_values)) - if self._correct_offset: - posterior += batch_end_values[0] - posterior[0, -1] - posteriors = (times, posterior) - return loss, posteriors, {"dummy_predictions": array_ops.zeros_like(values)} - - -class ChainingStateManagerTest(test.TestCase): - - def _make_test_data(self, length, cut_start, cut_end, offset, step=1): - times_full = step * numpy.arange(length, dtype=numpy.int64) - values_full = offset + step * numpy.arange(length, dtype=numpy.float32) - if cut_start is not None: - times = numpy.concatenate((times_full[:cut_start], - times_full[cut_end:])) - values = numpy.concatenate((values_full[:cut_start], - values_full[cut_end:])) - else: - times = times_full - values = values_full - return { - feature_keys.TrainEvalFeatures.TIMES: times, - feature_keys.TrainEvalFeatures.VALUES: values - } - - def _test_initialization(self, warmup_iterations, batch_size): - stub_model = StubTimeSeriesModel() - data = self._make_test_data(length=20, cut_start=None, cut_end=None, - offset=0.) - if batch_size == -1: - input_fn = test_utils.AllWindowInputFn( - input_pipeline.NumpyReader(data), window_size=10) - else: - input_fn = input_pipeline.RandomWindowInputFn( - input_pipeline.NumpyReader(data), - window_size=10, - batch_size=batch_size) - chainer = state_management.ChainingStateManager( - state_saving_interval=1) - features, _ = input_fn() - stub_model.initialize_graph() - chainer.initialize_graph(model=stub_model) - model_outputs = chainer.define_loss( - model=stub_model, features=features, mode=estimator_lib.ModeKeys.TRAIN) - with self.cached_session() as session: - variables.global_variables_initializer().run() - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(session, coord=coordinator) - for _ in range(warmup_iterations): - # Warm up saved state - model_outputs.loss.eval() - outputs = model_outputs.loss.eval() - coordinator.request_stop() - coordinator.join() - return outputs - - def test_zero_initializations(self): - # Even with no initialization, we are imputing values up to each chunk, - # which in this case gives exact values. - self.assertEqual(0., self._test_initialization( - warmup_iterations=0, batch_size=-1)) - - def test_one_initializations(self): - # Further initialization should still be correct, if redundant - self.assertEqual(0., self._test_initialization( - warmup_iterations=1, batch_size=-1)) - - def test_stochastic_batch(self): - # It shouldn't matter whether we're using a full deterministic batch or a - # smaller stochastic batch. - self.assertEqual(0., self._test_initialization( - warmup_iterations=1, batch_size=5)) - - def _test_pass_to_next(self, read_offset, step, correct_offset): - stub_model = StubTimeSeriesModel(correct_offset=correct_offset) - data = self._make_test_data( - length=100 + read_offset, cut_start=None, cut_end=None, offset=100., - step=step) - init_input_fn = input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader( - {k: v[:-read_offset] for k, v in data.items()})) - result_input_fn = input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader( - {k: v[read_offset:] for k, v in data.items()})) - - chainer = state_management.ChainingStateManager( - state_saving_interval=1) - stub_model.initialize_graph() - chainer.initialize_graph(model=stub_model) - init_model_outputs = chainer.define_loss( - model=stub_model, features=init_input_fn()[0], - mode=estimator_lib.ModeKeys.TRAIN) - result_model_outputs = chainer.define_loss( - model=stub_model, features=result_input_fn()[0], - mode=estimator_lib.ModeKeys.TRAIN) - with self.cached_session() as session: - variables.global_variables_initializer().run() - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(session, coord=coordinator) - init_model_outputs.loss.eval() - returned_loss = result_model_outputs.loss.eval() - coordinator.request_stop() - coordinator.join() - return returned_loss - - def test_pass_to_next_step_one_no_correction(self): - self.assertEqual(100., self._test_pass_to_next( - read_offset=1, step=1, correct_offset=False)) - - def test_pass_to_next_step_one_with_correction(self): - self.assertEqual(0., self._test_pass_to_next( - read_offset=1, step=1, correct_offset=True)) - - def test_pass_to_next_step_three_with_correction(self): - self.assertEqual(0., self._test_pass_to_next( - read_offset=1, step=3, correct_offset=True)) - - def test_large_read_offset(self): - self.assertEqual(0., self._test_pass_to_next( - read_offset=50, step=20, correct_offset=True)) - - def test_past_init_offset(self): - self.assertEqual(100., self._test_pass_to_next( - read_offset=100, step=20, correct_offset=True)) - - def _test_missing_values(self, cut_start, cut_end, offset): - stub_model = StubTimeSeriesModel() - data = self._make_test_data( - length=100, cut_start=cut_start, cut_end=cut_end, offset=offset) - input_fn = test_utils.AllWindowInputFn( - input_pipeline.NumpyReader(data), window_size=10) - chainer = state_management.ChainingStateManager( - state_saving_interval=1) - features, _ = input_fn() - stub_model.initialize_graph() - chainer.initialize_graph(model=stub_model) - model_outputs = chainer.define_loss( - model=stub_model, features=features, mode=estimator_lib.ModeKeys.TRAIN) - with self.cached_session() as session: - variables.global_variables_initializer().run() - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(session, coord=coordinator) - for _ in range(10): - model_outputs.loss.eval() - returned_loss = model_outputs.loss.eval() - coordinator.request_stop() - coordinator.join() - return returned_loss - - def test_missing_values_ten(self): - # Each posterior should be off by 10 from the offset in the values. 90 - # values with a chunk size of 10 means 90 - 10 + 1 possible chunks. - self.assertEqual((90 - 10 + 1) * 10, self._test_missing_values( - cut_start=20, cut_end=30, offset=10.)) - - def test_missing_values_five(self): - self.assertEqual((95 - 10 + 1) * 10, self._test_missing_values( - cut_start=15, cut_end=20, offset=10.)) - - -class _StateOverrideModel(model.TimeSeriesModel): - - def __init__(self): - super(_StateOverrideModel, self).__init__(num_features=1) - - def generate(self, *args): - pass - - def predict(self, *args): - pass - - def get_start_state(self): - return (constant_op.constant([20, 30, 40], dtype=dtypes.int64), - (constant_op.constant(-10, dtype=dtypes.int64), - constant_op.constant([30., 50.], dtype=dtypes.float64))) - - def get_batch_loss(self, features, mode, state): - per_observation_loss, state, outputs = self.per_step_batch_loss( - features, mode, state) - state = nest.map_structure(lambda element: element[:, -1], state) - outputs["observed"] = features[feature_keys.TrainEvalFeatures.VALUES] - return model.ModelOutputs( - loss=per_observation_loss, - end_state=state, - predictions=outputs, - prediction_times=features[feature_keys.TrainEvalFeatures.TIMES]) - - def per_step_batch_loss(self, features, mode, state): - return ( - constant_op.constant(1.), - # Assumes only one step: this is the per-step loss. - nest.map_structure( - lambda element: ops.convert_to_tensor(element)[:, None], state), - { - "dummy_predictions": - array_ops.zeros_like( - features[feature_keys.TrainEvalFeatures.VALUES]) - }) - - -class _StateOverrideTest(test.TestCase): - - def test_state_override(self): - test_start_state = (numpy.array([[2, 3, 4]]), (numpy.array([2]), - numpy.array([[3., 5.]]))) - data = { - feature_keys.FilteringFeatures.TIMES: numpy.arange(5), - feature_keys.FilteringFeatures.VALUES: numpy.zeros(shape=[5, 3]) - } - features, _ = input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader(data))() - features[feature_keys.FilteringFeatures.STATE_TUPLE] = test_start_state - stub_model = _StateOverrideModel() - chainer = state_management.ChainingStateManager() - stub_model.initialize_graph() - chainer.initialize_graph(model=stub_model) - model_outputs = chainer.define_loss( - model=stub_model, features=features, mode=estimator_lib.ModeKeys.EVAL) - with train.MonitoredSession() as session: - end_state = session.run(model_outputs.end_state) - nest.assert_same_structure(test_start_state, end_state) - for expected, received in zip( - nest.flatten(test_start_state), nest.flatten(end_state)): - self.assertAllEqual(expected, received) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD deleted file mode 100644 index 38c3ac4dc4d..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD +++ /dev/null @@ -1,274 +0,0 @@ -# State space components and ensembles - -load("//tensorflow:tensorflow.bzl", "tf_py_test") -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = ["//tensorflow:internal"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "state_space_model", - srcs = ["state_space_model.py"], - srcs_version = "PY2AND3", - deps = [ - ":kalman_filter", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/timeseries/python/timeseries:feature_keys", - "//tensorflow/contrib/timeseries/python/timeseries:math_utils", - "//tensorflow/contrib/timeseries/python/timeseries:model", - "//tensorflow/contrib/timeseries/python/timeseries:model_utils", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:variable_scope", - "//tensorflow/python/estimator:estimator_py", - "//third_party/py/numpy", - ], -) - -py_test( - name = "state_space_model_test", - timeout = "long", # Moderate but for asan - srcs = ["state_space_model_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "no_mac", - "no_windows", # TODO: needs investigation on Windows - ], - deps = [ - ":state_space_model", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/timeseries/python/timeseries:estimators", - "//tensorflow/contrib/timeseries/python/timeseries:feature_keys", - "//tensorflow/contrib/timeseries/python/timeseries:input_pipeline", - "//tensorflow/contrib/timeseries/python/timeseries:math_utils", - "//tensorflow/contrib/timeseries/python/timeseries:saved_model_utils", - "//tensorflow/contrib/timeseries/python/timeseries:state_management", - "//tensorflow/contrib/timeseries/python/timeseries:test_utils", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/estimator:estimator_py", - "//tensorflow/python/saved_model:loader", - "//tensorflow/python/saved_model:tag_constants", - "//third_party/py/numpy", - ], -) - -py_library( - name = "kalman_filter", - srcs = ["kalman_filter.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/timeseries/python/timeseries:math_utils", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:numerics", - ], -) - -tf_py_test( - name = "kalman_filter_test", - srcs = ["kalman_filter_test.py"], - additional_deps = [ - ":kalman_filter", - "//third_party/py/numpy", - "//tensorflow/contrib/timeseries/python/timeseries:math_utils", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - ], -) - -py_library( - name = "level_trend", - srcs = ["level_trend.py"], - srcs_version = "PY2AND3", - deps = [ - ":state_space_model", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:variable_scope", - ], -) - -tf_py_test( - name = "level_trend_test", - srcs = ["level_trend_test.py"], - additional_deps = [ - ":level_trend", - ":test_utils", - ":state_space_model", - "//tensorflow/contrib/timeseries/python/timeseries:test_utils", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - ], -) - -py_library( - name = "periodic", - srcs = ["periodic.py"], - srcs_version = "PY2AND3", - deps = [ - ":state_space_model", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//third_party/py/numpy", - ], -) - -tf_py_test( - name = "periodic_test", - srcs = ["periodic_test.py"], - additional_deps = [ - ":periodic", - ":test_utils", - ":state_space_model", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - ], -) - -py_library( - name = "structural_ensemble", - srcs = ["structural_ensemble.py"], - srcs_version = "PY2AND3", - deps = [ - ":level_trend", - ":periodic", - ":state_space_model", - ":varma", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - ], -) - -py_test( - name = "structural_ensemble_test", - timeout = "long", # Moderate but for asan/tsan/msan timeouts - srcs = ["structural_ensemble_test.py"], - python_version = "PY2", - shard_count = 4, - srcs_version = "PY2AND3", - deps = [ - ":state_space_model", - ":structural_ensemble", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/timeseries/python/timeseries:estimators", - "//tensorflow/contrib/timeseries/python/timeseries:feature_keys", - "//tensorflow/contrib/timeseries/python/timeseries:input_pipeline", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python/estimator:estimator_py", - "//third_party/py/numpy", - ], -) - -py_library( - name = "varma", - srcs = ["varma.py"], - srcs_version = "PY2AND3", - deps = [ - ":state_space_model", - "//tensorflow/contrib/timeseries/python/timeseries:math_utils", - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:variable_scope", - ], -) - -tf_py_test( - name = "varma_test", - srcs = ["varma_test.py"], - additional_deps = [ - ":state_space_model", - ":varma", - "//tensorflow/contrib/timeseries/python/timeseries:feature_keys", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/estimator:estimator_py", - ], -) - -py_library( - name = "filtering_postprocessor", - srcs = ["filtering_postprocessor.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/timeseries/python/timeseries:math_utils", - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:util", - ], -) - -tf_py_test( - name = "filtering_postprocessor_test", - srcs = ["filtering_postprocessor_test.py"], - additional_deps = [ - ":filtering_postprocessor", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - ], -) - -py_library( - name = "test_utils", - srcs = ["test_utils.py"], - srcs_version = "PY2AND3", - tags = ["no_pip"], - deps = [ - "//tensorflow/contrib/timeseries/python/timeseries:math_utils", - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/__init__.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/__init__.py deleted file mode 100644 index 52e83069cb0..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor.py deleted file mode 100644 index 3fa2fbd9f77..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor.py +++ /dev/null @@ -1,262 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Filtering postprocessors for SequentialTimeSeriesModels.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - -from tensorflow.contrib.timeseries.python.timeseries import math_utils - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.util import nest - - -@six.add_metaclass(abc.ABCMeta) -class FilteringStepPostprocessor(object): - """Base class for processors that are applied after each filter step.""" - - @abc.abstractmethod - def process_filtering_step(self, current_times, current_values, - predicted_state, filtered_state, outputs): - """Extends/modifies a filtering step, altering state and loss. - - Args: - current_times: A [batch size] integer Tensor of times. - current_values: A [batch size x num features] Tensor of values filtering - is being performed on. - predicted_state: A (possibly nested) list of Tensors indicating model - state which does not take `current_times` and `current_values` into - account. - filtered_state: Same structure as predicted_state, but updated to take - `current_times` and `current_values` into account. - outputs: A dictionary of outputs produced by model filtering - (SequentialTimeSeriesModel._process_filtering_step). - Returns: A tuple of (new_state, updated_outputs); - new_state: Updated state with the same structure as `filtered_state` and - `predicted_state`. - updated_outputs: The `outputs` dictionary, updated with any new outputs - from this filtering postprocessor. - """ - pass - - @abc.abstractproperty - def output_names(self): - return [] - - -def cauchy_alternative_to_gaussian(current_times, current_values, outputs): - """A Cauchy anomaly distribution, centered at a Gaussian prediction. - - Performs an entropy-matching approximation of the scale parameters of - independent Cauchy distributions given the covariance matrix of a multivariate - Gaussian in outputs["covariance"], and centers the Cauchy distributions at - outputs["mean"]. This requires that the model that we are creating an - alternative/anomaly distribution for produces a mean and covariance. - - Args: - current_times: A [batch size] Tensor of times, unused. - current_values: A [batch size x num features] Tensor of values to evaluate - the anomaly distribution at. - outputs: A dictionary of Tensors with keys "mean" and "covariance" - describing the Gaussian to construct an anomaly distribution from. The - value corresponding to "mean" has shape [batch size x num features], and - the value corresponding to "covariance" has shape [batch size x num - features x num features]. - Returns: - A [batch size] Tensor of log likelihoods; the anomaly log PDF evaluated at - `current_values`. - """ - del current_times # unused - cauchy_scale = math_utils.entropy_matched_cauchy_scale(outputs["covariance"]) - individual_log_pdfs = math_utils.cauchy_log_prob( - loc=outputs["mean"], - scale=cauchy_scale, - x=current_values) - return math_ops.reduce_sum(individual_log_pdfs, axis=1) - - -def _interpolate_state_linear(first_state, second_state, first_responsibility): - """Interpolate between two model states linearly.""" - interpolated_state_flat = [] - for first_state_tensor, second_state_tensor in zip( - nest.flatten(first_state), nest.flatten(second_state)): - assert first_state_tensor.dtype == second_state_tensor.dtype - if first_state_tensor.dtype.is_floating: - # Pad the responsibility shape with ones up to the state's rank so that it - # broadcasts - first_responsibility_padded = array_ops.reshape( - tensor=first_responsibility, - shape=array_ops.concat([ - array_ops.shape(first_responsibility), array_ops.ones( - [array_ops.rank(first_state_tensor) - 1], dtype=dtypes.int32) - ], 0)) - interpolated_state = ( - first_responsibility_padded * first_state_tensor - + (1. - first_responsibility_padded) * second_state_tensor) - interpolated_state.set_shape(first_state_tensor.get_shape()) - interpolated_state_flat.append(interpolated_state) - else: - # Integer dtypes are probably representing times, and don't need - # interpolation. Make sure they're identical to be sure. - with ops.control_dependencies( - [check_ops.assert_equal(first_state_tensor, second_state_tensor)]): - interpolated_state_flat.append(array_ops.identity(first_state_tensor)) - return nest.pack_sequence_as(first_state, interpolated_state_flat) - - -class StateInterpolatingAnomalyDetector(FilteringStepPostprocessor): - """An anomaly detector which guards model state against outliers. - - Smoothly interpolates between a model's predicted and inferred states, based - on the posterior probability of an anomaly, p(anomaly | data). This is useful - if anomalies would otherwise lead to model state which is hard to recover - from (Gaussian state space models suffer from this, for example). - - Relies on (1) an alternative distribution, typically with heavier tails than - the model's normal predictions, and (2) a prior probability of an anomaly. The - prior probability acts as a penalty, discouraging the system from marking too - many points as anomalies. The alternative distribution indicates the - probability of a datapoint given that it is an anomaly, and is a heavy-tailed - distribution (Cauchy) centered around the model's predictions by default. - - Specifically, we have: - - p(anomaly | data) = p(data | anomaly) * anomaly_prior_probability - / (p(data | not anomaly) * (1 - anomaly_prior_probability) - + p(data | anomaly) * anomaly_prior_probability) - - This is simply Bayes' theorem, where p(data | anomaly) is the - alternative/anomaly distribution, p(data | not anomaly) is the model's - predicted distribution, and anomaly_prior_probability is the prior probability - of an anomaly occurring (user-specified, defaulting to 1%). - - Rather than computing p(anomaly | data) directly, we use the odds ratio: - - odds_ratio = p(data | anomaly) * anomaly_prior_probability - / (p(data | not anomaly) * (1 - anomaly_prior_probability)) - - This has the same information as p(anomaly | data): - - odds_ratio = p(anomaly | data) / p(not anomaly | data) - - A "responsibility" score is computed for the model based on the log odds - ratio, and state interpolated based on this responsibility: - - model_responsibility = 1 / (1 + exp(-responsibility_scaling - * ln(odds_ratio))) - model_state = filtered_model_state * model_responsibility - + predicted_model_state * (1 - model_responsibility) - loss = model_responsibility - * ln(p(data | not anomaly) * (1 - anomaly_prior_probability)) - + (1 - model_responsibility) - * ln(p(data | anomaly) * anomaly_prior_probability) - - """ - - output_names = ["anomaly_score"] - - def __init__(self, - anomaly_log_likelihood=cauchy_alternative_to_gaussian, - anomaly_prior_probability=0.01, - responsibility_scaling=1.0): - """Configure the anomaly detector. - - Args: - anomaly_log_likelihood: A function taking `current_times`, - `current_values`, and `outputs` (same as the corresponding arguments - to process_filtering_step) and returning a [batch size] Tensor of log - likelihoods under an anomaly distribution. - anomaly_prior_probability: A scalar value, between 0 and 1, indicating the - prior probability of a particular example being an anomaly. - responsibility_scaling: A positive scalar controlling how fast - interpolation transitions between not-anomaly and anomaly; lower - values (closer to 0) create a smoother/slower transition. - """ - self._anomaly_log_likelihood = anomaly_log_likelihood - self._responsibility_scaling = responsibility_scaling - self._anomaly_prior_probability = anomaly_prior_probability - - def process_filtering_step(self, current_times, current_values, - predicted_state, filtered_state, outputs): - """Fall back on `predicted_state` for anomalies. - - Args: - current_times: A [batch size] integer Tensor of times. - current_values: A [batch size x num features] Tensor of values filtering - is being performed on. - predicted_state: A (possibly nested) list of Tensors indicating model - state which does not take `current_times` and `current_values` into - account. - filtered_state: Same structure as predicted_state, but updated to take - `current_times` and `current_values` into account. - outputs: A dictionary of outputs produced by model filtering. Must - include `log_likelihood`, a [batch size] Tensor indicating the log - likelihood of the observations under the model's predictions. - Returns: - A tuple of (new_state, updated_outputs); - new_state: Updated state with the same structure as `filtered_state` and - `predicted_state`; predicted_state for anomalies and filtered_state - otherwise (per batch element). - updated_outputs: The `outputs` dictionary, updated with a new "loss" - (the interpolated negative log likelihoods under the model and - anomaly distributions) and "anomaly_score" (the log odds ratio of - each part of the batch being an anomaly). - """ - anomaly_log_likelihood = self._anomaly_log_likelihood( - current_times=current_times, - current_values=current_values, - outputs=outputs) - anomaly_prior_probability = ops.convert_to_tensor( - self._anomaly_prior_probability, dtype=current_values.dtype) - # p(data | anomaly) * p(anomaly) - data_and_anomaly_log_probability = ( - anomaly_log_likelihood + math_ops.log(anomaly_prior_probability)) - # p(data | no anomaly) * p(no anomaly) - data_and_no_anomaly_log_probability = ( - outputs["log_likelihood"] + math_ops.log(1. - anomaly_prior_probability) - ) - # A log odds ratio is slightly nicer here than computing p(anomaly | data), - # since it is centered around zero - anomaly_log_odds_ratio = ( - data_and_anomaly_log_probability - - data_and_no_anomaly_log_probability) - model_responsibility = math_ops.sigmoid(-self._responsibility_scaling * - anomaly_log_odds_ratio) - # Do a linear interpolation between predicted and inferred model state - # based on the model's "responsibility". If we knew for sure whether - # this was an anomaly or not (binary responsibility), this would be the - # correct thing to do, but given that we don't it's just a - # (differentiable) heuristic. - interpolated_state = _interpolate_state_linear( - first_state=filtered_state, - second_state=predicted_state, - first_responsibility=model_responsibility) - # TODO(allenl): Try different responsibility scalings and interpolation - # methods (e.g. average in probability space rather than log space). - interpolated_log_likelihood = ( - model_responsibility * data_and_no_anomaly_log_probability - + (1. - model_responsibility) * data_and_anomaly_log_probability) - outputs["loss"] = -interpolated_log_likelihood - outputs["anomaly_score"] = anomaly_log_odds_ratio - return (interpolated_state, outputs) diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor_test.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor_test.py deleted file mode 100644 index a77c507d9be..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor_test.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for filtering postprocessors.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import filtering_postprocessor - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.platform import test - - -class FilteringStepPostprocessorTest(test.TestCase): - - def test_gaussian_alternative(self): - for float_dtype in [dtypes.float32, dtypes.float64]: - detector = filtering_postprocessor.StateInterpolatingAnomalyDetector( - anomaly_log_likelihood=(filtering_postprocessor - .cauchy_alternative_to_gaussian), - responsibility_scaling=10.) - predicted_state = [ - constant_op.constant( - [[40.], [20.]], dtype=float_dtype), constant_op.constant( - [3., 6.], dtype=float_dtype), constant_op.constant([-1, -2]) - ] - filtered_state = [ - constant_op.constant( - [[80.], [180.]], dtype=float_dtype), constant_op.constant( - [1., 2.], dtype=float_dtype), constant_op.constant([-1, -2]) - ] - interpolated_state, updated_outputs = detector.process_filtering_step( - current_times=constant_op.constant([1, 2]), - current_values=constant_op.constant([[0.], [1.]], dtype=float_dtype), - predicted_state=predicted_state, - filtered_state=filtered_state, - outputs={ - "mean": - constant_op.constant([[0.1], [10.]], dtype=float_dtype), - "covariance": - constant_op.constant([[[1.0]], [[1.0]]], dtype=float_dtype), - "log_likelihood": - constant_op.constant([-1., -40.], dtype=float_dtype) - }) - # The first batch element is not anomalous, and so should use the inferred - # state. The second is anomalous, and should use the predicted state. - expected_state = [[[80.], [20.]], - [1., 6.], - [-1, -2]] - with self.cached_session(): - for interpolated, expected in zip(interpolated_state, expected_state): - self.assertAllClose(expected, interpolated.eval()) - self.assertGreater(0., updated_outputs["anomaly_score"][0].eval()) - self.assertLess(0., updated_outputs["anomaly_score"][1].eval()) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/g3doc/periodic_multires_derivation.md b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/g3doc/periodic_multires_derivation.md deleted file mode 100644 index 872474aee11..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/g3doc/periodic_multires_derivation.md +++ /dev/null @@ -1,279 +0,0 @@ -# Derivations for multi-resolution cycle transition matrix powers - -This document contains derivations for the special-cased matrix-to-powers and -power sums used in `state_space_models/periodic.py`'s `ResolutionCycleModel` as -part of TensorFlow Time Series (TFTS). - -## Setting and notation - -Let $$M$$ be the number of latent values being cycled through -(`num_latent_values` in the code). - -The `ResolutionCycleModel` transition matrix is based on roots of a matrix which -cycles through $$M$$ (odd) values and constrains their sum to be -zero (which when included in a state space model means that the expected sum -over a complete period is zero). Call this $$M - 1$$ x -$$M - 1$$ matrix $$C$$ (`cycle_matrix`): - -$$ {\boldsymbol C}_{i, j} = \begin{cases} -1 & i = 0\\ 1 & j = i - 1\\ 0 & -\text{otherwise}\end{cases} $$ - -`ResolutionCycleModel` takes roots of this matrix using the following -parameterization: - -$$ {\boldsymbol C}^p = \text{cycle_eigenvectors} * -\text{diag}(\text{cycle_eigenvalues})^{p} * \text{cycle_eigenvectors}^{-1} $$ - -Where: - -$$\text{cycle_eigenvectors}_{i, j} = w_{\lfloor j / 2 \rfloor + 1}^{i (-1)^{j + -1}} - w_{\lfloor j / 2 \rfloor + 1}^{(i + 1) (-1)^{j + 1}}$$ - -$$(\text{cycle_eigenvectors}^{-1})_{i, j} = \frac{1}{M} -\sum_{k=0}^j w_{\lfloor i / 2 \rfloor + 1}^{k (-1)^i}$$ - -$$\text{cycle_eigenvalues}_{j} = w_{\lfloor j / 2 \rfloor + 1}^{(-1)^j}$$ - -Where $$w_j$$ is a root of unity: - -$$w_j = e^{\frac{2 \pi j \sqrt{-1}}{M}}$$ - -In Sympy (useful for checking expressions when $$M$$ is small), -this looks like: - -```python -import sympy -def root_of_unity(nth, number, to_power=1): - return sympy.exp(2 * sympy.pi * number * sympy.I * to_power / nth) -matsize = 4 -def eigvec_mat_fn(i, j): - number = j // 2 + 1 - powersign = (j % 2) * 2 - 1 - return (root_of_unity(matsize + 1, number=number, - to_power=powersign * i) - - root_of_unity(matsize + 1, number=number, - to_power=powersign * (i + 1))) -def eigvec_inverse_mat_fn(row, column): - number = row // 2 + 1 - powersign = ((row + 1) % 2) * 2 - 1 - runningsum = 0 - for j in range(column + 1): - runningsum += root_of_unity( - matsize + 1, number, to_power=j * powersign) / (matsize + 1) - return runningsum -def make_eigval_mat_fn(to_power=1): - def eigval_mat_fn(i, j): - if i == j: - number = j // 2 + 1 - powersign = ((j + 1) % 2) * 2 - 1 - return root_of_unity(matsize + 1, number=number, - to_power=powersign*to_power) - else: - return 0 - return eigval_mat_fn -eigval_power = sympy.Rational(1, 1) -eigvecs = sympy.Matrix(matsize, matsize, eigvec_mat_fn) -eigvals = sympy.Matrix(matsize, matsize, make_eigval_mat_fn(eigval_power)) -eigvecs_inv = sympy.Matrix(matsize, matsize, eigvec_inverse_mat_fn) -print (eigvecs * eigvals * eigvecs_inv).evalf() -``` - -## Proof that these are eigenvectors/eigenvalues of `cycle_matrix` - -We want to show that: - -$${\boldsymbol C} * \text{cycle_eigenvectors}_{\bullet, j} = -\text{cycle_eigenvalues}_j * \text{cycle_eigenvectors}_{\bullet, j} $$ - -Where $$\text{cycle_eigenvectors}_{\bullet, j}$$ is a column vector containing -the $$j^\text{th}$$ eigenvector. - -We have telescoping sum in the first entry: - -$$({\boldsymbol C} * \text{cycle_eigenvectors}_{\bullet, j})_i = -\begin{cases} -\sum_{k=0}^{M - 2} -\text{cycle_eigenvectors}_{k, j} & i = 0\\ \text{cycle_eigenvectors}_{i - 1, j} -& \text{otherwise} \end{cases}$$ - -$$ = \begin{cases} w_{\lfloor j / 2 \rfloor + 1}^{(M - -1)(-1)^{j + 1}} - w_{\lfloor j / 2 \rfloor + 1}^{0(-1)^{j + 1}} & i = 0\\ -\text{cycle_eigenvectors}_{i - 1, j} & \text{otherwise} \end{cases}$$ - -$$ = \begin{cases} w_{\lfloor j / 2 \rfloor + 1}^{(-1)^{j}} \left (w_{\lfloor j -/ 2 \rfloor + 1}^{M(-1)^{j + 1}} - w_{\lfloor j / 2 -\rfloor + 1}^{(-1)^{j + 1}} \right ) & i = 0\\ \text{cycle_eigenvectors}_{i - 1, -j} & \text{otherwise} \end{cases}$$ - -$$ = \begin{cases} w_{\lfloor j / 2 \rfloor + 1}^{(-1)^{j}} \left (w_{\lfloor j -/ 2 \rfloor + 1}^{0(-1)^{j + 1}} - w_{\lfloor j / 2 \rfloor + 1}^{(-1)^{j + 1}} -\right ) & i = 0\\ \text{cycle_eigenvectors}_{i - 1, j} & \text{otherwise} -\end{cases}$$ - -$$ = \begin{cases} w_{\lfloor j / 2 \rfloor + 1}^{(-1)^{j}} -\text{cycle_eigenvectors}_{0, j} ) & i = 0\\ \text{cycle_eigenvectors}_{i - 1, -j} & \text{otherwise} \end{cases}$$ - -The remaining cases follow from the fact that: - -$$w_{\lfloor j / 2 \rfloor + 1}^{(-1)^{j}} \text{cycle_eigenvectors}_{i, j} = -\text{cycle_eigenvectors}_{i - 1, j}$$ - -$$w_{\lfloor j / 2 \rfloor + 1}^{(-1)^{j}} \left( w_{\lfloor j / 2 \rfloor + -1}^{i (-1)^{j + 1}} - w_{\lfloor j / 2 \rfloor + 1}^{(i + 1) (-1)^{j + 1}} -\right) = w_{\lfloor j / 2 \rfloor + 1}^{(i - 1) (-1)^{j + 1}} - w_{\lfloor j / -2 \rfloor + 1}^{i (-1)^{j + 1}}$$ - -## Proof of eigenvector inverse matrix - -We want to show that (for the expressions above): - -$$ I = \text{cycle_eigenvectors} * \text{cycle_eigenvectors}^{-1} $$ - -Multiplying it out, we have: - -$$(\text{cycle_eigenvectors} * \text{cycle_eigenvectors}^{-1})_{i, j} = -\sum_{k=0}^{M - 2} \text{cycle_eigenvectors}_{i, k} -(\text{cycle_eigenvectors}^{-1})_{k, j} $$ - -$$ = \frac{1}{M} \sum_{k=0}^{M - -2} \left[ \left( w_{\lfloor k / 2 \rfloor + 1}^{i (-1)^{k + 1}} - w_{\lfloor k / -2 \rfloor + 1}^{(i + 1) (-1)^{k + 1}} \right) \sum_{l=0}^j w_{\lfloor k / 2 -\rfloor + 1}^{l (-1)^{k}} \right]$$ - -$$ = \frac{1}{M} \sum_{k=0}^{M - -2} \sum_{l=0}^j \left[ \left( w_{\lfloor k / 2 \rfloor + 1}^{i (-1)^{k + 1}} - -w_{\lfloor k / 2 \rfloor + 1}^{(i + 1) (-1)^{k + 1}} \right) w_{\lfloor k / 2 -\rfloor + 1}^{l (-1)^{k}} \right]$$ - -$$ = \frac{1}{M} \sum_{k=0}^{M - -2} \sum_{l=0}^j \left[ w_{\lfloor k / 2 \rfloor + 1}^{(i - l) (-1)^{k + 1}} - -w_{\lfloor k / 2 \rfloor + 1}^{(i - l + 1) (-1)^{k + 1}} \right]$$ - -Using telescoping: - -$$ = \frac{1}{M} \sum_{k=0}^{M - -2} \left[ w_{\lfloor k / 2 \rfloor + 1}^{(i - j) (-1)^{k + 1}} - w_{\lfloor k / -2 \rfloor + 1}^{(i + 1) (-1)^{k + 1}} \right]$$ - -Since $$e^{-ix} = \text{conj}(e^{ix})$$, the imaginary components cancel out of -the sum: - -$$ = \frac{2}{M} \sum_{k=0}^{(M - -1) / 2 - 1} \text{Real}\left[ w_{k + 1}^{i - j} - w_{k + 1}^{i + 1} \right]$$ - -$$ = \frac{2}{M} \sum_{k=0}^{(M - -1) / 2 - 1} \left[ \text{cos}\left(\frac{2 \pi (i - j) (k + -1)}{M}\right) - \text{cos}\left(\frac{2 \pi (i + 1) (k + -1)}{M}\right) \right]$$ - -Using Lagrange's identity $$\sum_{n=1}^N \text{cos}(n \theta) = -\frac{1}{2} + -\frac{\text{sin}\left(\left(N + \frac{1}{2}\right) \theta\right)}{2 -\text{sin}\left(\frac{\theta}{2}\right)}$$: - -$$ = \frac{2}{M} \left( -\frac{\text{sin}\left(\left(\left(\frac{M - -1}{2}\right) + \frac{1}{2}\right) \frac{2 \pi (i - -j)}{M}\right)}{2 \text{sin}\left(\frac{\pi (i - -j)}{M}\right)} \\- -\frac{\text{sin}\left(\left(\left(\frac{M - -1}{2}\right) + \frac{1}{2}\right) \frac{2 \pi (i + -1)}{M}\right)}{2 \text{sin}\left(\frac{\pi (i + -1)}{M}\right)} \right)$$ - -$$ = \frac{1}{M} \left(\frac{\text{sin}(\pi (i - -j))}{\text{sin}\left(\frac{\pi (i - j)}{M}\right)} - -\frac{\text{sin}(\pi (i + 1))}{\text{sin}\left(\frac{\pi (i + -1)}{M}\right)} \right)$$ - -The second term will always be zero, since $$i + 1$$ is at most -$$M - 1$$ ($$i$$ ranges from $$0$$ to -$$M - 2$$). The first term is also zero unless $$i = j$$ -($$\text{sin}(x)$$ is zero at integer multiples of $$\pi$$). Taking a limit when -$$i = j$$, the expression evaluates to $$1$$ (L'Hospital's rule gives a ratio of -cosines, both one, along with a ratio of the arguments from the chain rule). - -## Simplification of expression for matrix to a power - -Having established that the eigenvalues, eigenvectors, and the inverse -eigenvector matrix are all correct, we can now use them to derive an expression -for the matrix raised to a power: - -$$ {\boldsymbol C}^p = \text{cycle_eigenvectors} * -\text{diag}(\text{cycle_eigenvalues})^p * \text{cycle_eigenvectors}^{-1} $$ - -$$({\boldsymbol C}^p)_{i, j} = \sum_{k=0}^{M - 2} -\text{cycle_eigenvectors}_{i, k} * \text{cycle_eigenvalues}_k^p * -(\text{cycle_eigenvectors}^{-1})_{k, j}$$ - -$$ = \frac{1}{M} \sum_{k=0}^{M - -2} \left[\left( w_{\lfloor k / 2 \rfloor + 1}^{(p - i) (-1)^{k + 1}} - -w_{\lfloor k / 2 \rfloor + 1}^{(p - i - 1) (-1)^{k + 1}} \right) \sum_{l=0}^j -w_{\lfloor k / 2 \rfloor + 1}^{l (-1)^{k}} \right]$$ - -Following the same logic as for the inverse matrix proof, this leads to -($$cos(x)$$ being an even function): - -$$ = \frac{2}{M} \sum_{k=0}^{(M - -1) / 2 - 1} \left[ \text{cos}\left(\frac{2 \pi (p - i + j) -k}{M}\right) - \text{cos}\left(\frac{2 \pi (p - i - 1) -k)}{M}\right) \right]$$ - -Applying Lagrange's identity and simplifying, we get: - -$$ = \frac{1}{M} \left(\frac{\text{sin}(\pi (p - i + -j))}{\text{sin}\left(\frac{\pi (p - i + j)}{M}\right)} - -\frac{\text{sin}(\pi (p - i - 1))}{\text{sin}\left(\frac{\pi (p - i - -1)}{M}\right)} \right)$$ - -As a special/limiting case, we get the integer powers of the original matrix -($$I(b)$$ is 1 if $$b$$ is true and zero otherwise): - -$$ ({\boldsymbol C}^p)_{i, j} = \frac{1}{M} \left( -M * I(p - i + j \equiv 0\pmod {M}) -\\- M * I(p - i - 1 \equiv 0\pmod -{M}) \right)$$ - -## Simplification of expression for pre- and post-multiplication of noise matrix to a power - -Raising the transition matrix to a power allows us to transform the inferred -mean state across time (imputing), but performing imputation on the covariance -of the state estimate requires adding the noise covariance at each step (and -then transforming by pre- and post-multiplying by the transition -matrix). However, this noise covariance has a very special form, having only one -non-zero element in the upper left hand corner. - -$$\text{noise_covariance}_{i, j} = \begin{cases} \text{noise_scalar} & i = j = -0\\ 0 & \text{otherwise}\end{cases} $$ - -This makes it easy to compute an expression for $${\boldsymbol C}^p * -\text{noise_covariance} * ({\boldsymbol C}^T)^p$$: - -$$\left ({\boldsymbol C}^p * \text{noise_covariance} \right)_{i, j} -= ({\boldsymbol C}^p)_{i, 0} * \text{noise_scalar} * I(j = 0)$$ - -$$\left ({\boldsymbol C}^p * \text{noise_covariance} * -({\boldsymbol C}^T \right)^p)_{i, j} = ({\boldsymbol C}^p)_{i, 0} * -\text{noise_scalar} * ({\boldsymbol C}^p)_{j, 0}$$ - -$$ = \frac{1}{M^2} \left(\frac{\text{sin}(\pi (p - -i))}{\text{sin}\left(\frac{\pi (p - i)}{M}\right)} - -\frac{\text{sin}(\pi (p - i - 1))}{\text{sin}\left(\frac{\pi (p - i - -1)}{M}\right)} \right) \\ \left(\frac{\text{sin}(\pi (p - -j))}{\text{sin}\left(\frac{\pi (p - j)}{M}\right)} - -\frac{\text{sin}(\pi (p - j - 1))}{\text{sin}\left(\frac{\pi (p - j - -1)}{M}\right)} \right)$$ - -This (and the transition-matrix-to-a-power expression above) can be simplified -slightly using the fact that $$\text{sin}((x + i)\pi) = (-1)^i \text{sin}(x -\pi)$$ for integers $$i$$. - -## Open questions - -- It would be nice to have an expression for the elements of $$f(\lambda, N) = - \sum_{k=0}^N {\boldsymbol C}^{\lambda k} * \text{noise_covariance} * - ({\boldsymbol C}^T)^{\lambda k}$$, especially for $$0 < \lambda < 1$$. -- It seems that when $$\lambda = - \frac{M}{\text{periodicity}}$$, then $$f(\lambda, - \text{periodicity}) = \frac{\text{periodicity}}{M} f(1, - M)$$, but I'm not exactly sure why (i.e. have not - proven it). diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py deleted file mode 100644 index c0ec797bc5b..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py +++ /dev/null @@ -1,340 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implements Kalman filtering for linear state space models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.python.timeseries import math_utils - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import numerics - - -# TODO(allenl): support for always-factored covariance matrices -class KalmanFilter(object): - """Inference on linear state models. - - The model for observations in a given state is: - observation(t) = observation_model * state(t) - + Gaussian(0, observation_noise_covariance) - - State updates take the following form: - state(t) = state_transition * state(t-1) - + state_noise_transform * Gaussian(0, state_transition_noise_covariance) - - This is a real-valued analog to hidden Markov models, with linear transitions - and a Gaussian noise model. Given initial conditions, noise, and state - transition, Kalman filtering recursively estimates states and observations, - along with their associated uncertainty. When fed observations, future state - and uncertainty estimates are conditioned on those observations (in a Bayesian - sense). - - Typically some "given"s mentioned above (noises) will be unknown, and so - optimizing the Kalman filter's probabilistic predictions with respect to these - parameters is a good approach. The state transition and observation models are - usually known a priori as a modeling decision. - - """ - - def __init__(self, dtype=dtypes.float32, - simplified_posterior_covariance_computation=False): - """Initialize the Kalman filter. - - Args: - dtype: The data type to use for floating point tensors. - simplified_posterior_covariance_computation: If True, uses an algebraic - simplification of the Kalman filtering posterior covariance update, - which is slightly faster at the cost of numerical stability. The - simplified update is often stable when using double precision on small - models or with fixed transition matrices. - """ - self._simplified_posterior_covariance_computation = ( - simplified_posterior_covariance_computation) - self.dtype = dtype - - def do_filter( - self, estimated_state, estimated_state_covariance, - predicted_observation, predicted_observation_covariance, - observation, observation_model, observation_noise): - """Convenience function for scoring predictions. - - Scores a prediction against an observation, and computes the updated - posterior over states. - - Shapes given below for arguments are for single-model Kalman filtering - (e.g. KalmanFilter). For ensembles, prior_state and prior_state_var are - same-length tuples of values corresponding to each model. - - Args: - estimated_state: A prior mean over states [batch size x state dimension] - estimated_state_covariance: Covariance of state prior [batch size x D x - D], with D depending on the Kalman filter implementation (typically - the state dimension). - predicted_observation: A prediction for the observed value, such as that - returned by observed_from_state. A [batch size x num features] Tensor. - predicted_observation_covariance: A covariance matrix corresponding to - `predicted_observation`, a [batch size x num features x num features] - Tensor. - observation: The observed value corresponding to the predictions - given [batch size x observation dimension] - observation_model: The [batch size x observation dimension x model state - dimension] Tensor indicating how a particular state is mapped to - (pre-noise) observations for each part of the batch. - observation_noise: A [batch size x observation dimension x observation - dimension] Tensor or [observation dimension x observation dimension] - Tensor with covariance matrices to use for each part of the batch (a - two-dimensional input will be broadcast). - Returns: - posterior_state, posterior_state_var: Posterior mean and - covariance, updated versions of prior_state and - prior_state_var. - log_prediction_prob: Log probability of the observations under - the priors, suitable for optimization (should be maximized). - - """ - symmetrized_observation_covariance = 0.5 * ( - predicted_observation_covariance + array_ops.matrix_transpose( - predicted_observation_covariance)) - instability_message = ( - "This may occur due to numerically unstable filtering when there is " - "a large difference in posterior variances, or when inferences are " - "near-deterministic. Considering tuning the " - "'filtering_maximum_posterior_variance_ratio' or " - "'filtering_minimum_posterior_variance' parameters in your " - "StateSpaceModelConfiguration, or tuning the transition matrix.") - symmetrized_observation_covariance = numerics.verify_tensor_all_finite( - symmetrized_observation_covariance, - "Predicted observation covariance was not finite. {}".format( - instability_message)) - diag = array_ops.matrix_diag_part(symmetrized_observation_covariance) - min_diag = math_ops.reduce_min(diag) - non_negative_assert = control_flow_ops.Assert( - min_diag >= 0., - [("The predicted observation covariance " - "has a negative diagonal entry. {}").format(instability_message), - min_diag]) - with ops.control_dependencies([non_negative_assert]): - observation_covariance_cholesky = linalg_ops.cholesky( - symmetrized_observation_covariance) - log_prediction_prob = math_utils.mvn_tril_log_prob( - loc=predicted_observation, - scale_tril=observation_covariance_cholesky, - x=observation) - (posterior_state, - posterior_state_var) = self.posterior_from_prior_state( - prior_state=estimated_state, - prior_state_var=estimated_state_covariance, - observation=observation, - observation_model=observation_model, - predicted_observations=(predicted_observation, - predicted_observation_covariance), - observation_noise=observation_noise) - return (posterior_state, posterior_state_var, log_prediction_prob) - - def predict_state_mean(self, prior_state, transition_matrices): - """Compute state transitions. - - Args: - prior_state: Current estimated state mean [batch_size x state_dimension] - transition_matrices: A [batch size, state dimension, state dimension] - batch of matrices (dtype matching the `dtype` argument to the - constructor) with the transition matrix raised to the power of the - number of steps to be taken (not element-wise; use - math_utils.matrix_to_powers if there is no efficient special case) if - more than one step is desired. - Returns: - State mean advanced based on `transition_matrices` (dimensions matching - first argument). - """ - advanced_state = array_ops.squeeze( - math_ops.matmul( - transition_matrices, - prior_state[..., None]), - axis=[-1]) - return advanced_state - - def predict_state_var( - self, prior_state_var, transition_matrices, transition_noise_sums): - r"""Compute variance for state transitions. - - Computes a noise estimate corresponding to the value returned by - predict_state_mean. - - Args: - prior_state_var: Covariance matrix specifying uncertainty of current state - estimate [batch size x state dimension x state dimension] - transition_matrices: A [batch size, state dimension, state dimension] - batch of matrices (dtype matching the `dtype` argument to the - constructor) with the transition matrix raised to the power of the - number of steps to be taken (not element-wise; use - math_utils.matrix_to_powers if there is no efficient special case). - transition_noise_sums: A [batch size, state dimension, state dimension] - Tensor (dtype matching the `dtype` argument to the constructor) with: - - \sum_{i=0}^{num_steps - 1} ( - state_transition_to_powers_fn(i) - * state_transition_noise_covariance - * state_transition_to_powers_fn(i)^T - ) - - for the number of steps to be taken in each part of the batch (this - should match `transition_matrices`). Use math_utils.power_sums_tensor - with `tf.gather` if there is no efficient special case. - Returns: - State variance advanced based on `transition_matrices` and - `transition_noise_sums` (dimensions matching first argument). - """ - prior_variance_transitioned = math_ops.matmul( - math_ops.matmul(transition_matrices, prior_state_var), - transition_matrices, - adjoint_b=True) - return prior_variance_transitioned + transition_noise_sums - - def posterior_from_prior_state(self, prior_state, prior_state_var, - observation, observation_model, - predicted_observations, - observation_noise): - """Compute a posterior over states given an observation. - - Args: - prior_state: Prior state mean [batch size x state dimension] - prior_state_var: Prior state covariance [batch size x state dimension x - state dimension] - observation: The observed value corresponding to the predictions given - [batch size x observation dimension] - observation_model: The [batch size x observation dimension x model state - dimension] Tensor indicating how a particular state is mapped to - (pre-noise) observations for each part of the batch. - predicted_observations: An (observation mean, observation variance) tuple - computed based on the current state, usually the output of - observed_from_state. - observation_noise: A [batch size x observation dimension x observation - dimension] or [observation dimension x observation dimension] Tensor - with covariance matrices to use for each part of the batch (a - two-dimensional input will be broadcast). - Returns: - Posterior mean and covariance (dimensions matching the first two - arguments). - - """ - observed_mean, observed_var = predicted_observations - residual = observation - observed_mean - # TODO(allenl): Can more of this be done using matrix_solve_ls? - kalman_solve_rhs = math_ops.matmul( - observation_model, prior_state_var, adjoint_b=True) - # This matrix_solve adjoint doesn't make a difference symbolically (since - # observed_var is a covariance matrix, and should be symmetric), but - # filtering on multivariate series is unstable without it. See - # test_multivariate_symmetric_covariance_float64 in kalman_filter_test.py - # for an example of the instability (fails with adjoint=False). - kalman_gain_transposed = linalg_ops.matrix_solve( - matrix=observed_var, rhs=kalman_solve_rhs, adjoint=True) - posterior_state = prior_state + array_ops.squeeze( - math_ops.matmul( - kalman_gain_transposed, - array_ops.expand_dims(residual, -1), - adjoint_a=True), - axis=[-1]) - gain_obs = math_ops.matmul( - kalman_gain_transposed, observation_model, adjoint_a=True) - identity_extradim = linalg_ops.eye( - array_ops.shape(gain_obs)[1], dtype=gain_obs.dtype)[None] - identity_minus_factor = identity_extradim - gain_obs - if self._simplified_posterior_covariance_computation: - # posterior covariance = - # (I - kalman_gain * observation_model) * prior_state_var - posterior_state_var = math_ops.matmul(identity_minus_factor, - prior_state_var) - else: - observation_noise = ops.convert_to_tensor(observation_noise) - # A Joseph form update, which provides better numeric stability than the - # simplified optimal Kalman gain update, at the cost of a few extra - # operations. Joseph form updates are valid for any gain (not just the - # optimal Kalman gain), and so are more forgiving of numerical errors in - # computing the optimal Kalman gain. - # - # posterior covariance = - # (I - kalman_gain * observation_model) * prior_state_var - # * (I - kalman_gain * observation_model)^T - # + kalman_gain * observation_noise * kalman_gain^T - left_multiplied_state_var = math_ops.matmul(identity_minus_factor, - prior_state_var) - multiplied_state_var = math_ops.matmul( - identity_minus_factor, left_multiplied_state_var, adjoint_b=True) - def _batch_observation_noise_update(): - return (multiplied_state_var + math_ops.matmul( - math_ops.matmul( - kalman_gain_transposed, observation_noise, adjoint_a=True), - kalman_gain_transposed)) - def _matrix_observation_noise_update(): - return (multiplied_state_var + math_ops.matmul( - math_utils.batch_times_matrix( - kalman_gain_transposed, observation_noise, adj_x=True), - kalman_gain_transposed)) - if observation_noise.get_shape().ndims is None: - posterior_state_var = control_flow_ops.cond( - math_ops.equal(array_ops.rank(observation_noise), 2), - _matrix_observation_noise_update, _batch_observation_noise_update) - else: - # If static shape information exists, it gets checked in each cond() - # branch, so we need a special case to avoid graph-build-time - # exceptions. - if observation_noise.get_shape().ndims == 2: - posterior_state_var = _matrix_observation_noise_update() - else: - posterior_state_var = _batch_observation_noise_update() - return posterior_state, posterior_state_var - - def observed_from_state(self, state_mean, state_var, observation_model, - observation_noise): - """Compute an observation distribution given a state distribution. - - Args: - state_mean: State mean vector [batch size x state dimension] - state_var: State covariance [batch size x state dimension x state - dimension] - observation_model: The [batch size x observation dimension x model state - dimension] Tensor indicating how a particular state is mapped to - (pre-noise) observations for each part of the batch. - observation_noise: A [batch size x observation dimension x observation - dimension] Tensor with covariance matrices to use for each part of the - batch. To remove observation noise, pass a Tensor of zeros (or simply - 0, which will broadcast). - Returns: - observed_mean: Observation mean vector [batch size x observation - dimension] - observed_var: Observation covariance [batch size x observation dimension x - observation dimension] - - """ - observed_mean = array_ops.squeeze( - math_ops.matmul( - array_ops.expand_dims(state_mean, 1), - observation_model, - adjoint_b=True), - axis=[1]) - observed_var = math_ops.matmul( - math_ops.matmul(observation_model, state_var), - observation_model, - adjoint_b=True) - observed_var += observation_noise - return observed_mean, observed_var diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter_test.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter_test.py deleted file mode 100644 index f636126a33c..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter_test.py +++ /dev/null @@ -1,425 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for Kalman filtering.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy - -from tensorflow.contrib.timeseries.python.timeseries import math_utils -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import kalman_filter - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -# Two-dimensional state model with "slope" and "level" components. -STATE_TRANSITION = [ - [1., 1.], # Add slope to level - [0., 1.] # Maintain slope -] -# Independent noise for each component -STATE_TRANSITION_NOISE = [[0.1, 0.0], [0.0, 0.2]] -OBSERVATION_MODEL = [[[0.5, 0.0], [0.0, 1.0]]] -OBSERVATION_NOISE = [[0.0001, 0.], [0., 0.0002]] -STATE_NOISE_TRANSFORM = [[1.0, 0.0], [0.0, 1.0]] - - -def _powers_and_sums_from_transition_matrix( - state_transition, state_transition_noise_covariance, - state_noise_transform, max_gap=1): - def _transition_matrix_powers(powers): - return math_utils.matrix_to_powers(state_transition, powers) - def _power_sums(num_steps): - power_sums_tensor = math_utils.power_sums_tensor( - max_gap + 1, state_transition, - math_ops.matmul(state_noise_transform, - math_ops.matmul( - state_transition_noise_covariance, - state_noise_transform, - adjoint_b=True))) - return array_ops.gather(power_sums_tensor, indices=num_steps) - return (_transition_matrix_powers, _power_sums) - - -class MultivariateTests(test.TestCase): - - def _multivariate_symmetric_covariance_test_template( - self, dtype, simplified_posterior_variance_computation): - """Check that errors aren't building up asymmetries in covariances.""" - kf = kalman_filter.KalmanFilter(dtype=dtype) - observation_noise_covariance = constant_op.constant( - [[1., 0.5], [0.5, 1.]], dtype=dtype) - observation_model = constant_op.constant( - [[[1., 0., 0., 0.], [0., 0., 1., 0.]]], dtype=dtype) - state = array_ops.placeholder(shape=[1, 4], dtype=dtype) - state_var = array_ops.placeholder(shape=[1, 4, 4], dtype=dtype) - observation = array_ops.placeholder(shape=[1, 2], dtype=dtype) - transition_fn, power_sum_fn = _powers_and_sums_from_transition_matrix( - state_transition=constant_op.constant( - [[1., 1., 0., 0.], [0., 1., 0., 0.], [0., 0., 1., 1.], - [0., 0., 0., 1.]], - dtype=dtype), - state_noise_transform=linalg_ops.eye(4, dtype=dtype), - state_transition_noise_covariance=constant_op.constant( - [[1., 0., 0.5, 0.], [0., 1., 0., 0.5], [0.5, 0., 1., 0.], - [0., 0.5, 0., 1.]], - dtype=dtype)) - pred_state = kf.predict_state_mean( - prior_state=state, transition_matrices=transition_fn([1])) - pred_state_var = kf.predict_state_var( - prior_state_var=state_var, transition_matrices=transition_fn([1]), - transition_noise_sums=power_sum_fn([1])) - observed_mean, observed_var = kf.observed_from_state( - state_mean=pred_state, state_var=pred_state_var, - observation_model=observation_model, - observation_noise=observation_noise_covariance) - post_state, post_state_var = kf.posterior_from_prior_state( - prior_state=pred_state, prior_state_var=pred_state_var, - observation=observation, - observation_model=observation_model, - predicted_observations=(observed_mean, observed_var), - observation_noise=observation_noise_covariance) - with self.cached_session() as session: - evaled_state = numpy.array([[1., 1., 1., 1.]]) - evaled_state_var = numpy.eye(4)[None] - for i in range(500): - evaled_state, evaled_state_var, evaled_observed_var = session.run( - [post_state, post_state_var, observed_var], - feed_dict={state: evaled_state, - state_var: evaled_state_var, - observation: [[float(i), float(i)]]}) - self.assertAllClose(evaled_observed_var[0], - evaled_observed_var[0].T) - self.assertAllClose(evaled_state_var[0], - evaled_state_var[0].T) - - def test_multivariate_symmetric_covariance_float32(self): - self._multivariate_symmetric_covariance_test_template( - dtypes.float32, simplified_posterior_variance_computation=False) - - def test_multivariate_symmetric_covariance_float64(self): - self._multivariate_symmetric_covariance_test_template( - dtypes.float64, simplified_posterior_variance_computation=True) - - -class KalmanFilterNonBatchTest(test.TestCase): - """Single-batch KalmanFilter tests.""" - - def setUp(self): - """The basic model defined above, with unit batches.""" - self.kalman_filter = kalman_filter.KalmanFilter() - self.transition_fn, self.power_sum_fn = ( - _powers_and_sums_from_transition_matrix( - state_transition=STATE_TRANSITION, - state_transition_noise_covariance=STATE_TRANSITION_NOISE, - state_noise_transform=STATE_NOISE_TRANSFORM, - max_gap=5)) - - def test_observed_from_state(self): - """Compare observation mean and noise to hand-computed values.""" - with self.cached_session(): - state = constant_op.constant([[2., 1.]]) - state_var = constant_op.constant([[[4., 0.], [0., 3.]]]) - observed_mean, observed_var = self.kalman_filter.observed_from_state( - state, state_var, - observation_model=OBSERVATION_MODEL, - observation_noise=OBSERVATION_NOISE) - observed_mean_override, observed_var_override = ( - self.kalman_filter.observed_from_state( - state, state_var, - observation_model=OBSERVATION_MODEL, - observation_noise=100 * constant_op.constant( - OBSERVATION_NOISE)[None])) - self.assertAllClose(numpy.array([[1., 1.]]), - observed_mean.eval()) - self.assertAllClose(numpy.array([[1., 1.]]), - observed_mean_override.eval()) - self.assertAllClose(numpy.array([[[1.0001, 0.], [0., 3.0002]]]), - observed_var.eval()) - self.assertAllClose(numpy.array([[[1.01, 0.], [0., 3.02]]]), - observed_var_override.eval()) - - def _posterior_from_prior_state_test_template( - self, state, state_var, observation, observation_model, observation_noise, - expected_state, expected_state_var): - """Test that repeated observations converge to the expected value.""" - predicted_observations = self.kalman_filter.observed_from_state( - state, state_var, observation_model, - observation_noise=observation_noise) - state_update, state_var_update = ( - self.kalman_filter.posterior_from_prior_state( - state, state_var, observation, - observation_model=observation_model, - predicted_observations=predicted_observations, - observation_noise=observation_noise)) - with self.cached_session() as session: - evaled_state, evaled_state_var = session.run([state, state_var]) - for _ in range(300): - evaled_state, evaled_state_var = session.run( - [state_update, state_var_update], - feed_dict={state: evaled_state, state_var: evaled_state_var}) - self.assertAllClose(expected_state, - evaled_state, - atol=1e-5) - self.assertAllClose( - expected_state_var, - evaled_state_var, - atol=1e-5) - - def test_posterior_from_prior_state_univariate(self): - self._posterior_from_prior_state_test_template( - state=constant_op.constant([[0.3]]), - state_var=constant_op.constant([[[1.]]]), - observation=constant_op.constant([[1.]]), - observation_model=[[[2.]]], - observation_noise=[[[0.01]]], - expected_state=numpy.array([[0.5]]), - expected_state_var=[[[0.]]]) - - def test_posterior_from_prior_state_univariate_unit_noise(self): - self._posterior_from_prior_state_test_template( - state=constant_op.constant([[0.3]]), - state_var=constant_op.constant([[[1e10]]]), - observation=constant_op.constant([[1.]]), - observation_model=[[[2.]]], - observation_noise=[[[1.0]]], - expected_state=numpy.array([[0.5]]), - expected_state_var=[[[1. / (300. * 2. ** 2)]]]) - - def test_posterior_from_prior_state_multivariate_2d(self): - self._posterior_from_prior_state_test_template( - state=constant_op.constant([[1.9, 1.]]), - state_var=constant_op.constant([[[1., 0.], [0., 2.]]]), - observation=constant_op.constant([[1., 1.]]), - observation_model=OBSERVATION_MODEL, - observation_noise=OBSERVATION_NOISE, - expected_state=numpy.array([[2., 1.]]), - expected_state_var=[[[0., 0.], [0., 0.]]]) - - def test_posterior_from_prior_state_multivariate_3d(self): - self._posterior_from_prior_state_test_template( - state=constant_op.constant([[1.9, 1., 5.]]), - state_var=constant_op.constant( - [[[200., 0., 1.], [0., 2000., 0.], [1., 0., 40000.]]]), - observation=constant_op.constant([[1., 1., 3.]]), - observation_model=constant_op.constant( - [[[0.5, 0., 0.], - [0., 10., 0.], - [0., 0., 100.]]]), - observation_noise=linalg_ops.eye(3) / 10000., - expected_state=numpy.array([[2., .1, .03]]), - expected_state_var=numpy.zeros([1, 3, 3])) - - def test_predict_state_mean(self): - """Compare state mean transitions with simple hand-computed values.""" - with self.cached_session(): - state = constant_op.constant([[4., 2.]]) - state = self.kalman_filter.predict_state_mean( - state, self.transition_fn([1])) - for _ in range(2): - state = self.kalman_filter.predict_state_mean( - state, self.transition_fn([1])) - self.assertAllClose( - numpy.array([[2. * 3. + 4., # Slope * time + base - 2.]]), - state.eval()) - - def test_predict_state_var(self): - """Compare a variance transition with simple hand-computed values.""" - with self.cached_session(): - state_var = constant_op.constant([[[1., 0.], [0., 2.]]]) - state_var = self.kalman_filter.predict_state_var( - state_var, self.transition_fn([1]), self.power_sum_fn([1])) - self.assertAllClose( - numpy.array([[[3.1, 2.0], [2.0, 2.2]]]), - state_var.eval()) - - def test_do_filter(self): - """Tests do_filter. - - Tests that correct values have high probability and incorrect values - have low probability when there is low uncertainty. - """ - with self.cached_session(): - state = constant_op.constant([[4., 2.]]) - state_var = constant_op.constant([[[0.0001, 0.], [0., 0.0001]]]) - observation = constant_op.constant([[ - .5 * ( - 4. # Base - + 2.), # State transition - 2. - ]]) - estimated_state = self.kalman_filter.predict_state_mean( - state, self.transition_fn([1])) - estimated_state_covariance = self.kalman_filter.predict_state_var( - state_var, self.transition_fn([1]), self.power_sum_fn([1])) - (predicted_observation, - predicted_observation_covariance) = ( - self.kalman_filter.observed_from_state( - estimated_state, estimated_state_covariance, - observation_model=OBSERVATION_MODEL, - observation_noise=OBSERVATION_NOISE)) - (_, _, first_log_prob) = self.kalman_filter.do_filter( - estimated_state=estimated_state, - estimated_state_covariance=estimated_state_covariance, - predicted_observation=predicted_observation, - predicted_observation_covariance=predicted_observation_covariance, - observation=observation, - observation_model=OBSERVATION_MODEL, - observation_noise=OBSERVATION_NOISE) - self.assertGreater(first_log_prob.eval()[0], numpy.log(0.99)) - - def test_predict_n_ahead_mean(self): - with self.cached_session(): - original_state = constant_op.constant([[4., 2.]]) - n = 5 - iterative_state = original_state - for i in range(n): - self.assertAllClose( - iterative_state.eval(), - self.kalman_filter.predict_state_mean( - original_state, - self.transition_fn([i])).eval()) - iterative_state = self.kalman_filter.predict_state_mean( - iterative_state, - self.transition_fn([1])) - - def test_predict_n_ahead_var(self): - with self.cached_session(): - original_var = constant_op.constant([[[2., 3.], [4., 5.]]]) - n = 5 - iterative_var = original_var - for i in range(n): - self.assertAllClose( - iterative_var.eval(), - self.kalman_filter.predict_state_var( - original_var, - self.transition_fn([i]), - self.power_sum_fn([i])).eval()) - iterative_var = self.kalman_filter.predict_state_var( - iterative_var, - self.transition_fn([1]), - self.power_sum_fn([1])) - - -class KalmanFilterBatchTest(test.TestCase): - """KalmanFilter tests with more than one element batches.""" - - def test_do_filter_batch(self): - """Tests do_filter, in batch mode. - - Tests that correct values have high probability and incorrect values - have low probability when there is low uncertainty. - """ - with self.cached_session(): - state = constant_op.constant([[4., 2.], [5., 3.], [6., 4.]]) - state_var = constant_op.constant(3 * [[[0.0001, 0.], [0., 0.0001]]]) - observation = constant_op.constant([ - [ - .5 * ( - 4. # Base - + 2.), # State transition - 2. - ], - [ - .5 * ( - 5. # Base - + 3.), # State transition - 3. - ], - [3.14, 2.71] - ]) # Low probability observation - kf = kalman_filter.KalmanFilter() - transition_fn, power_sum_fn = _powers_and_sums_from_transition_matrix( - state_transition=STATE_TRANSITION, - state_transition_noise_covariance=STATE_TRANSITION_NOISE, - state_noise_transform=STATE_NOISE_TRANSFORM, - max_gap=2) - estimated_state = kf.predict_state_mean(state, transition_fn(3*[1])) - estimated_state_covariance = kf.predict_state_var( - state_var, transition_fn(3*[1]), power_sum_fn(3*[1])) - observation_model = array_ops.tile(OBSERVATION_MODEL, [3, 1, 1]) - (predicted_observation, - predicted_observation_covariance) = ( - kf.observed_from_state( - estimated_state, estimated_state_covariance, - observation_model=observation_model, - observation_noise=OBSERVATION_NOISE)) - (state, state_var, log_prob) = kf.do_filter( - estimated_state=estimated_state, - estimated_state_covariance=estimated_state_covariance, - predicted_observation=predicted_observation, - predicted_observation_covariance=predicted_observation_covariance, - observation=observation, - observation_model=observation_model, - observation_noise=OBSERVATION_NOISE) - first_log_prob, second_log_prob, third_log_prob = log_prob.eval() - self.assertGreater(first_log_prob.sum(), numpy.log(0.99)) - self.assertGreater(second_log_prob.sum(), numpy.log(0.99)) - self.assertLess(third_log_prob.sum(), numpy.log(0.01)) - - def test_predict_n_ahead_mean(self): - with self.cached_session(): - kf = kalman_filter.KalmanFilter() - transition_fn, _ = _powers_and_sums_from_transition_matrix( - state_transition=STATE_TRANSITION, - state_transition_noise_covariance=STATE_TRANSITION_NOISE, - state_noise_transform=STATE_NOISE_TRANSFORM, - max_gap=2) - original_state = constant_op.constant([[4., 2.], [3., 1.], [6., 2.]]) - state0 = original_state - state1 = kf.predict_state_mean(state0, transition_fn(3 * [1])) - state2 = kf.predict_state_mean(state1, transition_fn(3 * [1])) - batch_eval = kf.predict_state_mean( - original_state, transition_fn([1, 0, 2])).eval() - self.assertAllClose(state0.eval()[1], batch_eval[1]) - self.assertAllClose(state1.eval()[0], batch_eval[0]) - self.assertAllClose(state2.eval()[2], batch_eval[2]) - - def test_predict_n_ahead_var(self): - with self.cached_session(): - kf = kalman_filter.KalmanFilter() - transition_fn, power_sum_fn = _powers_and_sums_from_transition_matrix( - state_transition=STATE_TRANSITION, - state_transition_noise_covariance=STATE_TRANSITION_NOISE, - state_noise_transform=STATE_NOISE_TRANSFORM, - max_gap=2) - base_var = 2.0 * numpy.identity(2) + numpy.ones([2, 2]) - original_var = constant_op.constant( - numpy.array( - [base_var, 2.0 * base_var, 3.0 * base_var], dtype=numpy.float32)) - var0 = original_var - var1 = kf.predict_state_var( - var0, transition_fn(3 * [1]), power_sum_fn(3 * [1])) - var2 = kf.predict_state_var( - var1, transition_fn(3 * [1]), power_sum_fn(3 * [1])) - batch_eval = kf.predict_state_var( - original_var, - transition_fn([1, 0, 2]), - power_sum_fn([1, 0, 2])).eval() - self.assertAllClose(var0.eval()[1], batch_eval[1]) - self.assertAllClose(var1.eval()[0], batch_eval[0]) - self.assertAllClose(var2.eval()[2], batch_eval[2]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/level_trend.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/level_trend.py deleted file mode 100644 index 56167c4f012..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/level_trend.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implements a state space model with level and local linear trends.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import state_space_model - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variable_scope - - -class AdderStateSpaceModel(state_space_model.StateSpaceModel): - """A state space model component with level and slope. - - At each timestep, level <- level + slope. Level is observed, slope is not. - """ - - def __init__( - self, - use_level_noise=True, - configuration=state_space_model.StateSpaceModelConfiguration()): - """Configure the model. - - Args: - use_level_noise: Whether to model the time series as having level noise. - configuration: A StateSpaceModelConfiguration object. - """ - self.use_level_noise = use_level_noise - super(AdderStateSpaceModel, self).__init__( - configuration=configuration) - - def get_prior_mean(self): - """If un-chunked data is available, set initial level to the first value.""" - with variable_scope.variable_scope(self._variable_scope): - if self._input_statistics is not None: - # TODO(allenl): Better support for multivariate series here. - initial_value = array_ops.stack([ - math_ops.reduce_mean( - self._scale_data( - self._input_statistics.series_start_moments.mean)), - 0. - ]) - return initial_value + variable_scope.get_variable( - name="prior_state_mean", - shape=initial_value.get_shape(), - initializer=init_ops.zeros_initializer(), - dtype=self.dtype, - trainable=self._configuration.trainable_start_state) - else: - return super(AdderStateSpaceModel, self).get_prior_mean() - - def transition_to_powers(self, powers): - """Computes powers of the adder transition matrix efficiently. - - Args: - powers: An integer Tensor, shape [...], with powers to raise the - transition matrix to. - Returns: - A floating point Tensor with shape [..., 2, 2] containing: - transition^power = [[1., power], - [0., 1.]] - """ - paddings = array_ops.concat( - [ - array_ops.zeros([array_ops.rank(powers), 2], dtype=dtypes.int32), - [(0, 1), (1, 0)] - ], - axis=0) - powers_padded = array_ops.pad(powers[..., None, None], paddings=paddings) - identity_matrices = linalg_ops.eye( - num_rows=2, batch_shape=array_ops.shape(powers), dtype=self.dtype) - return identity_matrices + math_ops.cast(powers_padded, self.dtype) - - def transition_power_noise_accumulator(self, num_steps): - """Computes power sums in closed form.""" - def _pack_and_reshape(*values): - return array_ops.reshape( - array_ops.stack(axis=1, values=values), - array_ops.concat(values=[array_ops.shape(num_steps), [2, 2]], axis=0)) - - num_steps = math_ops.cast(num_steps, self.dtype) - noise_transitions = num_steps - 1 - noise_transform = ops.convert_to_tensor(self.get_noise_transform(), - self.dtype) - noise_covariance_transformed = math_ops.matmul( - math_ops.matmul(noise_transform, - self.state_transition_noise_covariance), - noise_transform, - adjoint_b=True) - # Un-packing the transformed noise as: - # [[a b] - # [c d]] - a, b, c, d = array_ops.unstack( - array_ops.reshape(noise_covariance_transformed, [-1, 4]), axis=1) - sum_of_first_n = noise_transitions * (noise_transitions + 1) / 2 - sum_of_first_n_squares = sum_of_first_n * (2 * noise_transitions + 1) / 3 - return _pack_and_reshape( - num_steps * a + sum_of_first_n * (b + c) + sum_of_first_n_squares * d, - num_steps * b + sum_of_first_n * d, - num_steps * c + sum_of_first_n * d, - num_steps * d) - - def get_state_transition(self): - return [[1., 1.], # Add slope to level - [0., 1.]] # Maintain slope - - def get_noise_transform(self): - if self.use_level_noise: - return [[1., 0.], - [0., 1.]] - else: - return [[0.], - [1.]] - - def get_observation_model(self, times): - """Observe level but not slope. - - See StateSpaceModel.get_observation_model. - - Args: - times: Unused. See the parent class for details. - Returns: - A static, univariate observation model for later broadcasting. - """ - del times # Does not rely on times. Uses broadcasting from the parent. - return constant_op.constant([1., 0.], dtype=self.dtype) diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/level_trend_test.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/level_trend_test.py deleted file mode 100644 index 19110820860..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/level_trend_test.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for level and trend state space model components.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import level_trend -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import state_space_model -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import test_utils - -from tensorflow.python.framework import dtypes -from tensorflow.python.platform import test - - -class SpecialCaseTests(test.TestCase): - - def test_adder_transition_to_powers(self): - num_steps = 3 - dtype = dtypes.float64 - adder = level_trend.AdderStateSpaceModel( - configuration=state_space_model.StateSpaceModelConfiguration( - dtype=dtype)) - test_utils.transition_power_test_template( - test_case=self, model=adder, num_steps=num_steps) - - def test_adder_noise_accumulator(self): - num_steps = 3 - dtype = dtypes.float64 - use_level_noise = True - adder = level_trend.AdderStateSpaceModel( - use_level_noise=use_level_noise, - configuration=state_space_model.StateSpaceModelConfiguration( - dtype=dtype)) - test_utils.noise_accumulator_test_template( - test_case=self, model=adder, num_steps=num_steps) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/periodic.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/periodic.py deleted file mode 100644 index e70db93ea1e..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/periodic.py +++ /dev/null @@ -1,535 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""State space components for modeling seasonality.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy - -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import state_space_model - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops - - -class CycleStateSpaceModel(state_space_model.StateSpaceModel): - """A state space model component which cycles between values. - - Stores N values using N - 1 latent values, the Nth being the negative sum of - those explicitly stored. At any given timestep one of these values is - observed. Noise is assumed to affect only one of the transitions. - """ - - def __init__( - self, - periodicity, - configuration=state_space_model.StateSpaceModelConfiguration()): - self._periodicity = periodicity - super(CycleStateSpaceModel, self).__init__(configuration=configuration) - - def get_state_transition(self): - return self.transition_to_powers(array_ops.ones([], dtype=dtypes.int32)) - - def get_noise_transform(self): - # transition_power_noise_accumulator makes assumptions about this - # transformation. If the noise transform is modified or overridden, - # transition_power_noise_accumulator must be modified as well (or discarded, - # as it is simply an optimization). - return array_ops.pad( - array_ops.ones([1], dtype=self.dtype), - paddings=[(0, self._periodicity - 2)])[..., None] - - def transition_to_powers(self, powers): - """Computes powers of the cycle transition matrix efficiently. - - Args: - powers: An integer Tensor, shape [...], with powers to raise the - transition matrix to. - Returns: - A floating point Tensor with shape [..., self._periodicity - 1, - self._periodicity - 1] containing: - (transition^power)_{i, j} = { - 1 if (i - j) % self._periodicity == power % self._periodicity - -1 if (i + 1) % self._periodicity == power % self._periodicity - 0 otherwise} - """ - powers %= self._periodicity - range_shape_padded = array_ops.reshape( - math_ops.range(self._periodicity - 1, dtype=powers.dtype), - array_ops.concat( - [ - array_ops.ones([array_ops.rank(powers)], dtype=dtypes.int32), - [self._periodicity - 1] - ], - axis=0)) - is_row_negative = math_ops.equal(range_shape_padded + 1, powers[..., None]) - row_indicator_shape = array_ops.shape(is_row_negative) - negative_row_indicator = array_ops.where(is_row_negative, -array_ops.ones( - shape=row_indicator_shape, dtype=self.dtype), - array_ops.zeros( - row_indicator_shape, - dtype=self.dtype)) - coord_diff = (range_shape_padded[..., None] - - range_shape_padded[..., None, :]) - is_one = math_ops.equal(coord_diff % self._periodicity, - powers[..., None, None]) - positive_ones = array_ops.where(is_one, - array_ops.ones( - array_ops.shape(is_one), - dtype=self.dtype), - array_ops.zeros( - array_ops.shape(is_one), - dtype=self.dtype)) - return math_ops.cast(positive_ones + negative_row_indicator[..., None], - self.dtype) - - def transition_power_noise_accumulator( - self, num_steps, noise_addition_coefficient=1): - r"""Sum the transitioned covariance matrix over a number of steps. - - Assumes that state_transition_noise_covariance is a matrix with a single - non-zero value in the upper left. - - Args: - num_steps: A [...] shape integer Tensor with numbers of steps to compute - power sums for. - noise_addition_coefficient: A multiplier for the state transition noise - covariance (used in ResolutionCycleModel to compute multiples of full - period sums). - Returns: - The computed power sum, with shape [..., state dimension, state - dimension] containing: - - [\sum_{p=0}^{num_steps - 1} ( - state_transition^p - * state_transition_noise_covariance - * (state_transition^p)^T)]_{i, j} = { - -contribution_{j + 1} if j == i - 1 - contribution_{j + 1} + contribution{j} if j == i - -contribution_{j} if j == i + 1 - 0 otherwise - } - - contribution_k = noise_scalar - * ((num_steps + self._periodicity - 1 - (k % self._periodicity)) - // self._periodicity) - - Where contribution_k is the sum of noise_scalar additions to component k - of the periodicity. - """ - noise_addition_scalar = array_ops.squeeze( - self.state_transition_noise_covariance, axis=[-1, -2]) - period_range_reshaped = array_ops.reshape( - math_ops.range(self._periodicity, dtype=num_steps.dtype), - array_ops.concat( - [ - array_ops.ones([array_ops.rank(num_steps)], dtype=dtypes.int32), - [self._periodicity] - ], - axis=0)) - reversed_remaining_steps = ((period_range_reshaped - - (num_steps[..., None] - 1)) - % self._periodicity) - period_additions_reversed = (ops.convert_to_tensor( - noise_addition_coefficient, - self.dtype)[..., None] * noise_addition_scalar * math_ops.cast( - (num_steps[..., None] + reversed_remaining_steps) // - self._periodicity, - dtype=self.dtype)) - period_additions_diag = array_ops.matrix_diag(period_additions_reversed) - upper_band = array_ops.concat( - [ - array_ops.zeros_like(period_additions_diag[..., :-1, 0:1]), - -period_additions_diag[..., :-1, 0:-2] - ], - axis=-1) - lower_band = array_ops.concat( - [ - array_ops.zeros_like(period_additions_diag[..., 0:1, :-1]), - -period_additions_diag[..., 0:-2, :-1] - ], - axis=-2) - period_additions_rotated = array_ops.concat( - [ - period_additions_reversed[..., -1:], - period_additions_reversed[..., :-2] - ], - axis=-1) - diagonal = array_ops.matrix_diag(period_additions_reversed[..., :-1] + - period_additions_rotated) - return diagonal + lower_band + upper_band - - def get_observation_model(self, times): - """Observe only the first of the rotating latent values. - - See StateSpaceModel.get_observation_model. - Args: - times: Unused. See the parent class for details. - Returns: - A static, univariate observation model for later broadcasting. - """ - del times # Does not rely on times. Uses broadcasting from the parent. - return array_ops.concat( - values=[ - array_ops.ones([1], dtype=self.dtype), array_ops.zeros( - [self._periodicity - 2], dtype=self.dtype) - ], - axis=0) - - -class ResolutionCycleModel(CycleStateSpaceModel): - """A version of CycleStateSpaceModel with variable resolution. - - Cycles between "num_latent_values" latent values over a period of - "periodicity", smoothly interpolating. Simply raises the transition matrix - from CycleStateSpaceModel to the power (num_latent_values / periodicity). - - Specifically, ResolutionCycleModel uses the following eigendecomposition of - the CycleStateSpaceModel matrix (there are several parameterizations, others - leading to roots of the matrix with complex values): - - eigenvectors_{i, j} - = root_of_unity(floor(j / 2) + 1, i * (-1)^(j + 1)) - - root_of_unity(floor(j / 2) + 1, (i + 1) * (-1)^(j + 1)) - eigenvalues_j = root_of_unity(floor(j / 2) + 1, (-1)^j) - root_of_unity(root_number, to_power) - = exp(to_power * 2 * pi * sqrt(-1) * root_number - / num_latent_values) - - The transition matrix for ResolutionCycleModel is then: - - eigenvectors - * diag(eigenvalues^(num_latent_values / periodicity)) - * eigenvectors^-1 - - Since the eigenvalues are paired with their conjugates (conj(e^(sqrt(-1)*x)) = - e^(-sqrt(-1)*x)), the resulting matrix has real components (this is why only - odd numbers of latent values are supported, since the size of the matrix is - one less than the number of latent values and there must be an even number of - eigenvalues to pair them off). - - See ./g3doc/periodic_multires_derivation.md for details. - """ - - def __init__( - self, - num_latent_values, - periodicity, - near_integer_threshold=1e-8, - configuration=state_space_model.StateSpaceModelConfiguration()): - """Initialize the ResolutionCycleModel. - - Args: - num_latent_values: Controls the representational power and memory usage of - the model. The transition matrix has shape [num_latent_values - 1, - num_latent_values - 1]. Must be an odd integer (see class docstring for - why). - periodicity: The number of steps for cyclic behavior. May be a Tensor, and - need not be an integer (although integer values greater than - num_latent_values have more efficient special cases). - near_integer_threshold: When avoiding singularities, controls how close a - number should be to that singularity before the special case takes over. - configuration: A StateSpaceModelConfiguration object. - - Raises: - ValueError: If num_latent_values is not odd. - """ - if num_latent_values % 2 != 1: - raise ValueError("Only odd numbers of latent values are supported.") - self._num_latent_values = num_latent_values - self._true_periodicity = periodicity - self._near_integer_threshold = near_integer_threshold - super(ResolutionCycleModel, self).__init__( - periodicity=num_latent_values, - configuration=configuration) - - def _close_to_integer(self, value): - value = math_ops.cast(value, self.dtype) - return math_ops.less( - math_ops.abs(value - gen_math_ops.round(value)), - self._near_integer_threshold) - - def transition_to_powers(self, powers): - """Computes TransitionMatrix^power efficiently. - - For an n x n transition matrix we have: - - (TransitionMatrix**power)_{i, j) = (-1) ** i * sin(pi * power) / (n + 1) - * ((-1) ** j / sin(pi / (n + 1) * (power - i + j)) - + 1 / sin(pi / (n + 1) * (power - i - 1))) - - The sin(pi * power) term is zero whenever "power" is an integer. However, - the 1 / sin(x) terms (cosecants) occasionally (when their arguments are - multiples of pi) cancel out this value. The limit as the argument approaches - an integer value gives the "correct" result, but computing these separately - gives 0 * inf = NaN. Instead, there is a special case for near-integer - values. - - Args: - powers: A floating point Tensor of powers to raise the transition matrix - to. - Returns: - A [..., self._num_latent_values - 1, self._num_latent_values - 1] floating - point Tensor with the transition matrix raised to each power in - `powers`. - - """ - num_latent_values_float = math_ops.cast(self._num_latent_values, self.dtype) - latent_values_per_period = (num_latent_values_float / math_ops.cast( - self._true_periodicity, dtype=self.dtype)) - original_matrix_powers = (math_ops.cast(powers, self.dtype) * - latent_values_per_period) - global_coeff = (math_ops.sin(original_matrix_powers * numpy.pi) / - num_latent_values_float)[..., None, None] - matrix_dimension_range = array_ops.reshape( - math_ops.range(self._num_latent_values - 1), - array_ops.concat( - [ - array_ops.ones( - [array_ops.rank(original_matrix_powers)], - dtype=dtypes.int32), [self._num_latent_values - 1] - ], - axis=0)) - matrix_dimension_range_float = math_ops.cast(matrix_dimension_range, - self.dtype) - alternating = math_ops.cast(1 - 2 * (matrix_dimension_range % 2), - self.dtype) - row_addend = 1. / math_ops.sin(numpy.pi / num_latent_values_float * ( - original_matrix_powers[..., None] - matrix_dimension_range_float - 1)) - column_minus_row = (matrix_dimension_range_float[..., None, :] - - matrix_dimension_range_float[..., None]) - full_matrix_addend = (alternating[..., None, :] / math_ops.sin( - numpy.pi / num_latent_values_float * - (original_matrix_powers[..., None, None] + column_minus_row))) - continuous_construction = global_coeff * alternating[..., None] * ( - row_addend[..., None] + full_matrix_addend) - # For integer powers, the above formula is only correct in the limit, - # yielding NaNs as written. We defer to the super-class in such cases, which - # computes integer powers exactly. - return array_ops.where( - self._close_to_integer(original_matrix_powers), - super(ResolutionCycleModel, self).transition_to_powers( - math_ops.cast( - gen_math_ops.round(original_matrix_powers), dtypes.int64)), - continuous_construction) - - def transition_power_noise_accumulator(self, num_steps): - """Sum the transitioned covariance matrix over a number of steps. - - Args: - num_steps: An integer Tensor of any shape [...] indicating the number of - steps to compute for each part of the batch. - - Returns: - A [..., self._num_latent_values - 1, self._num_latent_values - 1] floating - point Tensor corresponding to each requested number of steps, containing: - - sum_{i=1}^{steps} transition^i * noise_covariance - * (transition^i)^T - """ - - def _whole_periods_folded(): - """A more efficient special casing for integer periods. - - We knock off full periods, leaving at most self._true_periodicity steps to - compute. - - Returns: - A tuple of (remaining_whole_steps, current_accumulation): - remaining_whole_steps: An integer Tensor with the same shape as the - `num_steps` argument to `transition_power_noise_accumulator`, - indicating the reduced number of steps which must be computed - sequentially and added to `current_accumulation`. - current_accumulation: A [..., self._num_latent_values - 1, - self._num_latent_values - 1] floating point Tensor corresponding to - the accumulations for steps which were computed in this function. - """ - original_transition_noise_addition_coefficient = (math_ops.cast( - self._true_periodicity, self.dtype) / math_ops.cast( - self._num_latent_values, self.dtype)) - full_period_accumulation = super( - ResolutionCycleModel, self).transition_power_noise_accumulator( - noise_addition_coefficient= - original_transition_noise_addition_coefficient, - num_steps=ops.convert_to_tensor( - self._num_latent_values, dtype=num_steps.dtype)) - periodicity_integer = math_ops.cast(self._true_periodicity, - num_steps.dtype) - full_periods = math_ops.cast(num_steps // periodicity_integer, self.dtype) - current_accumulation = full_periods[..., None, None] * array_ops.reshape( - full_period_accumulation, - array_ops.concat( - [ - array_ops.ones( - [array_ops.rank(full_periods)], dtype=dtypes.int32), - array_ops.shape(full_period_accumulation) - ], - axis=0)) - remaining_whole_steps = num_steps % periodicity_integer - return remaining_whole_steps, current_accumulation - def _no_whole_period_computation(): - """A less efficient special casing for real valued periods. - - This special casing is still preferable to computing using sequential - matrix multiplies (parallelizable, more numerically stable), but is linear - in the number of steps. - - Returns: - Same shapes and types as `_whole_periods_folded`, but no folding is done - in this function. - """ - current_accumulation = array_ops.zeros( - array_ops.concat( - [ - array_ops.shape(num_steps), - [self._num_latent_values - 1, self._num_latent_values - 1] - ], - axis=0), - dtype=self.dtype) - remaining_whole_steps = num_steps - return remaining_whole_steps, current_accumulation - # Decide whether it's feasible to compute whole periods in closed form, - # taking advantage of the fact that a sum over self._true_periodicity steps - # in our transition matrix is proportional to a sum over - # self._num_latent_values steps in the unmodified matrix (because each - # latent value gets the same treatment). This is possible for integer - # self._true_periodicity, since we stay aligned to integer steps. For real - # valued self._true_periodicity, or when the cyclic behavior is a higher - # resolution than 1 per step, taking whole periods leads to misalignment - # with integer steps, which would be difficult to recover from. - remaining_whole_steps, current_accumulation = control_flow_ops.cond( - self._whole_period_folding(), _whole_periods_folded, - _no_whole_period_computation) - steps_to_compute = math_ops.reduce_max(remaining_whole_steps) - remaining_step_noise_additions = self._power_sum_array(steps_to_compute) - noise_addition_scalar = array_ops.squeeze( - self.state_transition_noise_covariance, axis=[-1, -2]) - return current_accumulation + noise_addition_scalar * array_ops.gather( - remaining_step_noise_additions, indices=remaining_whole_steps) - - def _whole_period_folding(self): - """Decides whether computing a whole period maintains alignment.""" - return math_ops.logical_and( - self._close_to_integer(self._true_periodicity), - math_ops.greater_equal(self._true_periodicity, self._num_latent_values)) - - def _power_sum_array(self, max_remaining_steps): - r"""Computes \sum_{i=0}^{N-1} A^i B (A^i)^T for N=0..max_remaining_steps. - - A is the transition matrix and B is the noise covariance. - - This is more efficient in practice than math_utils.power_sums_tensor, since - each A^i B (A^i)^T term has a closed-form expression not depending on i - 1. - Thus vectorization can replace explicit looping. - - Uses a cumulative sum on the following expression: - - (transition^p * transition_covariance * (transition^p)^T)_{i, j} - = (-1)^(i + j) * sin^2(pi * p) / num_latent_values^2 - * (1/sin(pi / num_latent_values * (p - i)) - + 1/sin(pi / num_latent_values * (p - i - 1))) - * (1/sin(pi / num_latent_values * (p - j)) - + 1/sin(pi / num_latent_values * (p - j - 1))) - - The expression being derived from the eigenvectors and eigenvalues given in - the class docstring (and as with CycleStateSpaceModel taking advantage of - the sparsity of the transition covariance). - - Args: - max_remaining_steps: A scalar integer Tensor indicating the number of - non-trivial values to compute. - Returns: - A [max_remaining_steps + 1, self._num_latent_values - 1, - self._num_latent_values - 1] floating point Tensor S with cumulative power - sums. - - S[N] = \sum_{i=0}^{N-1} A^i B (A^i)^T - S[0] is the zero matrix - S[1] is B - S[2] is A B A^T + B - - """ - num_latent_values_float = math_ops.cast(self._num_latent_values, self.dtype) - latent_values_per_period = (num_latent_values_float / math_ops.cast( - self._true_periodicity, dtype=self.dtype)) - original_matrix_powers = (math_ops.cast( - math_ops.range(max_remaining_steps), - self.dtype) * latent_values_per_period) - matrix_dimension_range = math_ops.range( - self._num_latent_values - 1)[None, ...] - matrix_dimension_range_float = math_ops.cast(matrix_dimension_range, - self.dtype) - def _cosecant_with_freq(coefficient): - return 1. / math_ops.sin(numpy.pi / num_latent_values_float * coefficient) - power_minus_index = (original_matrix_powers[..., None] - - matrix_dimension_range_float) - mesh_values = (_cosecant_with_freq(power_minus_index) - + _cosecant_with_freq(power_minus_index - 1.)) - meshed = mesh_values[..., None, :] * mesh_values[..., None] - full_matrix_alternating = math_ops.cast(1 - 2 * ( - (matrix_dimension_range[..., None, :] + - matrix_dimension_range[..., None]) % 2), self.dtype) - def _sine_discontinuity(value): - """A special case for dealing with discontinuities. - - Decides whether `value` is close to an integer, and if so computes: - - lim x->n |sin(x * pi)| / sin(x * pi) = sign(sin(n * pi)) - = cos(n * pi) - - Args: - value: The floating point Tensor value which may lead to a - discontinuity. - Returns: - A tuple of (is_discontinuous, sign): - is_discontinuous: A boolean Tensor of the same shape as `value`, - indicating whether it is near an integer. - sign: A floating point Tensor indicating the sign of the discontinuity - (being near 1 or -1 when `is_discontinuous` is True), of the same - shape and type as `value`. - """ - normalized = value / num_latent_values_float - is_discontinuous = self._close_to_integer(normalized) - sign = math_ops.cos(normalized * numpy.pi) - return is_discontinuous, sign - index_discontinuous, index_sign = _sine_discontinuity( - original_matrix_powers[..., None] - - matrix_dimension_range_float) - index_minus_discontinuous, index_minus_sign = _sine_discontinuity( - original_matrix_powers[..., None] - - matrix_dimension_range_float - - 1) - ones_mask_vector = math_ops.logical_or(index_discontinuous, - index_minus_discontinuous) - ones_sign_vector = array_ops.where(index_discontinuous, index_sign, - index_minus_sign) - ones_mask = math_ops.logical_and(ones_mask_vector[..., None], - ones_mask_vector[..., None, :]) - zeros_mask = self._close_to_integer(original_matrix_powers) - zeroed = array_ops.where(zeros_mask, array_ops.zeros_like(meshed), meshed) - global_coefficient = (math_ops.sin(numpy.pi * original_matrix_powers) / - num_latent_values_float) - masked_meshed = array_ops.where( - ones_mask, ones_sign_vector[..., None] * ones_sign_vector[..., None, :], - zeroed * global_coefficient[..., None, None]**2) - powers_above_zero = full_matrix_alternating * masked_meshed - return array_ops.pad( - math_ops.cumsum(powers_above_zero), [(1, 0), (0, 0), (0, 0)]) diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/periodic_test.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/periodic_test.py deleted file mode 100644 index 9bd68e8ad2e..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/periodic_test.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for periodic state space model components.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import periodic -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import state_space_model -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import test_utils - -from tensorflow.python.framework import dtypes -from tensorflow.python.platform import test - - -class SpecialCaseTests(test.TestCase): - - def test_cycle_transition_to_powers(self): - num_steps = 3 - dtype = dtypes.float64 - periodicity = 3 - cycle = periodic.CycleStateSpaceModel( - periodicity=periodicity, - configuration=state_space_model.StateSpaceModelConfiguration( - dtype=dtype)) - test_utils.transition_power_test_template( - test_case=self, model=cycle, num_steps=num_steps) - - def test_resolution_cycle_transition_to_powers(self): - num_steps = 3 - dtype = dtypes.float64 - latent_values = 3 - periodicity = latent_values - 1 - cycle = periodic.ResolutionCycleModel( - num_latent_values=latent_values, - periodicity=periodicity, - configuration=state_space_model.StateSpaceModelConfiguration( - dtype=dtype)) - test_utils.transition_power_test_template( - test_case=self, model=cycle, num_steps=num_steps) - - def test_cycle_noise_accumulator(self): - num_steps = 3 - dtype = dtypes.float64 - periodicity = 3 - cycle = periodic.CycleStateSpaceModel( - periodicity=periodicity, - configuration=state_space_model.StateSpaceModelConfiguration( - dtype=dtype)) - test_utils.noise_accumulator_test_template( - test_case=self, model=cycle, num_steps=num_steps) - - def test_resolution_cycle_noise_accumulator(self): - num_steps = 3 - dtype = dtypes.float64 - latent_values = 3 - periodicity = latent_values + 0.1 - cycle = periodic.ResolutionCycleModel( - num_latent_values=latent_values, - periodicity=periodicity, - configuration=state_space_model.StateSpaceModelConfiguration( - dtype=dtype)) - test_utils.noise_accumulator_test_template( - test_case=self, model=cycle, num_steps=num_steps) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/state_space_model.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/state_space_model.py deleted file mode 100644 index d0f9a7df2f9..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/state_space_model.py +++ /dev/null @@ -1,1213 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Abstract base for state space models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc -import collections - -import numpy - -from tensorflow.contrib.layers.python.layers import layers - -from tensorflow.contrib.timeseries.python.timeseries import math_utils -from tensorflow.contrib.timeseries.python.timeseries import model -from tensorflow.contrib.timeseries.python.timeseries import model_utils -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import TrainEvalFeatures -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import kalman_filter - -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variable_scope - - -class StateSpaceModelConfiguration( - collections.namedtuple( - typename="StateSpaceModelConfiguration", - field_names=[ - "num_features", "use_observation_noise", "dtype", - "covariance_prior_fn", "bayesian_prior_weighting", - "filtering_postprocessor", "trainable_start_state", - "exogenous_noise_increases", "exogenous_noise_decreases", - "exogenous_feature_columns", "exogenous_update_condition", - "filtering_maximum_posterior_variance_ratio", - "filtering_minimum_posterior_variance", - "transition_covariance_initial_log_scale_bias", - "static_unrolling_window_size_threshold"])): - """Configuration options for StateSpaceModels.""" - - def __new__( - cls, - num_features=1, - use_observation_noise=True, - dtype=dtypes.float32, - covariance_prior_fn=math_utils.log_noninformative_covariance_prior, - bayesian_prior_weighting=True, - filtering_postprocessor=None, - trainable_start_state=False, - exogenous_noise_increases=True, - exogenous_noise_decreases=False, - exogenous_feature_columns=None, - exogenous_update_condition=None, - filtering_maximum_posterior_variance_ratio=1e6, - filtering_minimum_posterior_variance=0., - transition_covariance_initial_log_scale_bias=-5., - static_unrolling_window_size_threshold=None): - """Configuration options for StateSpaceModels. - - Args: - num_features: Output dimension for model - use_observation_noise: If true, observations are modeled as noisy - functions of the current state. If false, observations are a - deterministic function of the current state. Only applicable to the - top-level model in an ensemble. Consider also changing the - transition_covariance_initial_log_scale_bias when disabling observation - noise, as its default setting assumes that observation noise is part of - the model. - dtype: The float dtype to use when defining the model. - covariance_prior_fn: A function mapping from a covariance matrix to a - scalar value (e.g. log likelihood) which can be summed across - matrices. Defaults to an independent Jeffreys prior on the diagonal - elements (regularizing as log(1. / variance)). To use a flat prior - (i.e. no regularization), set to `lambda _: 0.`. Defaults to - relatively uninformative priors on state transition and observation - noise, which have the effect of encouraging low-noise solutions which - provide confident predictions when possible. Without regularization, - transition noise tends to remain high, and multi-step predictions are - under-confident. - bayesian_prior_weighting: If True, weights the prior (covariance_prior_fn) - based on an estimate of the full dataset size. If False, weights it - based on the mini-batch window size, which (while statistically - improper) can lead to more desirable low-noise solutions in cases - where the full dataset is large enough to overwhelm the prior. - filtering_postprocessor: A FilteringStepPostprocessor object to use, - useful for ignoring anomalies in training data. - trainable_start_state: If True, start state may depend on trainable - Variables. If False, it will not. - exogenous_noise_increases: If True, exogenous regressors can add to model - state, increasing uncertainty. If both this parameter and - exogenous_noise_decreases are False, exogenous regressors are ignored. - exogenous_noise_decreases: If True, exogenous regressors can "set" model - state, decreasing uncertainty. If both this parameter and - exogenous_noise_increases are False, exogenous regressors are ignored. - exogenous_feature_columns: A list of `tf.feature_column`s (for example - `tf.feature_column.embedding_column`) corresponding to exogenous - features which provide extra information to the model but are not part - of the series to be predicted. Passed to - `tf.compat.v1.feature_column.input_layer`. - exogenous_update_condition: A function taking two Tensor arguments `times` - (shape [batch size]) and `features` (a dictionary mapping exogenous - feature keys to Tensors with shapes [batch size, ...]) and returning a - boolean Tensor with shape [batch size] indicating whether state should - be updated using exogenous features for each part of the batch. Where - it is False, no exogenous update is performed. If None (default), - exogenous updates are always performed. Useful for avoiding "leaky" - frequent exogenous updates when sparse updates are desired. Called - only during graph construction. - filtering_maximum_posterior_variance_ratio: The maximum allowed ratio of - two diagonal entries in a state covariance matrix just prior to - filtering. Lower values mean that filtering will be more numerically - stable, at the cost of artificially increasing estimated uncertainty - in some cases. This parameter can be important when learning a - transition matrix. - filtering_minimum_posterior_variance: The minimum diagonal value in a - state covariance matrix just prior to filtering, preventing numerical - instability due to deterministic beliefs (sometimes an issue when - learning transition matrices). This value should be set several orders - of magnitude below any expected minimum state uncertainty. - transition_covariance_initial_log_scale_bias: Controls the initial - tradeoff between the transition noise covariance matrix and the - observation noise covariance matrix, on a log scale (the elements of - the transition noise covariance matrix are proportional to `e^{X + - transition_covariance_initial_log_scale_bias}` where `X` is learned - and may depend on input statistics, observation noise covariance is - proportional to `e^{Y - - transition_covariance_initial_log_scale_bias}`). For models *with* - observation noise, -5 is a reasonable value. Models which do not use - observation noise, and are not part of an ensemble which does use - observation noise, should have this set to 0 or more to avoid - numerical issues due to filtering with too little noise. - static_unrolling_window_size_threshold: Only relevant for the top-level - StateSpaceModel in an ensemble; enables switching between static and - dynamic looping (if not None, default, meaning that no static - unrolling is performed) based on the window size (windows with this - size and smaller will have their graphs unrolled statically). See the - SequentialTimeSeriesModel constructor for details. - Returns: - A StateSpaceModelConfiguration object. - """ - if exogenous_feature_columns is None: - exogenous_feature_columns = [] - return super(StateSpaceModelConfiguration, cls).__new__( - cls, num_features, use_observation_noise, dtype, - covariance_prior_fn, bayesian_prior_weighting, - filtering_postprocessor, trainable_start_state, - exogenous_noise_increases, exogenous_noise_decreases, - exogenous_feature_columns, exogenous_update_condition, - filtering_maximum_posterior_variance_ratio, - filtering_minimum_posterior_variance, - transition_covariance_initial_log_scale_bias, - static_unrolling_window_size_threshold) - - -class StateSpaceModel(model.SequentialTimeSeriesModel): - """Base class for linear state space models. - - Sub-classes can specify the model to be learned by overriding - get_state_transition, get_noise_transform, and get_observation_model. - - See kalman_filter.py for a detailed description of the class of models covered - by StateSpaceModel. - - Briefly, state space models are defined by a state transition equation: - - state[t] = StateTransition * state[t-1] + NoiseTransform * StateNoise[t] - + ExogenousNoiseIncreasing[t] - StateNoise[t] ~ Gaussian(0, StateNoiseCovariance) - ExogenousNoiseIncreasing[t] ~ Gaussian(ExogenousNoiseIncreasingMean[t], - ExogenousNoiseIncreasingCovariance[t]) - - And an observation model: - - observation[t] = ObservationModel * state[t] + ObservationNoise[t] - ObservationNoise[t] ~ Gaussian(0, ObservationNoiseCovariance) - - Additionally, exogenous regressors can act as observations, decreasing - uncertainty: - - ExogenousNoiseDecreasingObservation[t] ~ Gaussian( - ExogenousNoiseDecreasingMean[t], ExogenousNoiseDecreasingCovariance[t]) - - Attributes: - kalman_filter: If initialize_graph has been called, the initialized - KalmanFilter to use for inference. None otherwise. - prior_state_mean: If initialize_graph has been called, a - Variable-parameterized Tensor with shape [state dimension]; - the initial prior mean for one or more time series. None otherwise. - prior_state_var: If initialize_graph has been called, a - Variable-parameterized Tensor with shape [state dimension x state - dimension]; the initial prior covariance. None otherwise. - state_transition_noise_covariance: If initialize_graph has been called, a - Variable-parameterized Tensor with shape [state noise dimension x state - noise dimension] indicating the amount of noise added at each - transition. - """ - - def __init__(self, configuration): - """Initialize a state space model. - - Args: - configuration: A StateSpaceModelConfiguration object. - """ - self._configuration = configuration - if configuration.filtering_postprocessor is not None: - filtering_postprocessor_names = ( - configuration.filtering_postprocessor.output_names) - else: - filtering_postprocessor_names = [] - super(StateSpaceModel, self).__init__( - train_output_names=(["mean", "covariance", "log_likelihood"] - + filtering_postprocessor_names), - predict_output_names=["mean", "covariance"], - num_features=configuration.num_features, - normalize_features=True, - dtype=configuration.dtype, - exogenous_feature_columns=configuration.exogenous_feature_columns, - exogenous_update_condition=configuration.exogenous_update_condition, - static_unrolling_window_size_threshold= - configuration.static_unrolling_window_size_threshold) - self._kalman_filter = None - self.prior_state_mean = None - self.prior_state_var = None - self.state_transition_noise_covariance = None - self._total_observation_count = None - self._observation_noise_covariance = None - # Capture the current variable scope and use it to define all model - # variables. Especially useful for ensembles, where variables may be defined - # for every component model in one function call, which would otherwise - # prevent the user from separating variables from different models into - # different scopes. - self._variable_scope = variable_scope.get_variable_scope() - - def transition_power_noise_accumulator(self, num_steps): - r"""Sum a transitioned covariance matrix over a number of steps. - - Computes - - \sum_{i=0}^{num_steps - 1} ( - state_transition^i - * state_transition_noise_covariance - * (state_transition^i)^T) - - If special cases are available, overriding this function can lead to more - efficient inferences. - - Args: - num_steps: A [...] shape integer Tensor with numbers of steps to compute - power sums for. - Returns: - The computed power sum, with shape [..., state dimension, state - dimension]. - """ - # TODO(allenl): This general case should use cumsum if transition_to_powers - # can be computed in constant time (important for correlated ensembles, - # where transition_power_noise_accumulator special cases cannot be - # aggregated from member models). - noise_transform = ops.convert_to_tensor(self.get_noise_transform(), - self.dtype) - noise_transformed = math_ops.matmul( - math_ops.matmul(noise_transform, - self.state_transition_noise_covariance), - noise_transform, - transpose_b=True) - noise_additions = math_utils.power_sums_tensor( - math_ops.reduce_max(num_steps) + 1, - ops.convert_to_tensor(self.get_state_transition(), dtype=self.dtype), - noise_transformed) - return array_ops.gather(noise_additions, indices=num_steps) - - def transition_to_powers(self, powers): - """Raise the transition matrix to a batch of powers. - - Computes state_transition^powers. If special cases are available, overriding - this function can lead to more efficient inferences. - - Args: - powers: A [...] shape integer Tensor with powers to raise the transition - matrix to. - Returns: - The computed matrix powers, with shape [..., state dimension, state - dimension]. - """ - return math_utils.matrix_to_powers( - ops.convert_to_tensor(self.get_state_transition(), dtype=self.dtype), - powers) - - def _window_initializer(self, times, state): - """Prepare to impute across the gaps in a window.""" - _, _, priors_from_time = state - times = ops.convert_to_tensor(times) - priors_from_time = ops.convert_to_tensor(priors_from_time) - intra_batch_gaps = array_ops.reshape(times[:, 1:] - times[:, :-1], [-1]) - # Ignore negative starting gaps, since there will be transient start times - # as inputs statistics are computed. - starting_gaps = math_ops.maximum(times[:, 0] - priors_from_time, 0) - # Pre-define transition matrices raised to powers (and their sums) for every - # gap in this window. This avoids duplicate computation (for example many - # steps will use the transition matrix raised to the first power) and - # batches the computation rather than doing it inside the per-step loop. - unique_gaps, _ = array_ops.unique( - array_ops.concat([intra_batch_gaps, starting_gaps], axis=0)) - self._window_power_sums = self.transition_power_noise_accumulator( - unique_gaps) - self._window_transition_powers = self.transition_to_powers(unique_gaps) - self._window_gap_sizes = unique_gaps - - def _lookup_window_caches(self, caches, indices): - _, window_power_ids = array_ops.unique( - array_ops.concat( - [ - self._window_gap_sizes, math_ops.cast( - indices, self._window_gap_sizes.dtype) - ], - axis=0)) - all_gathered_indices = [] - for cache in caches: - gathered_indices = array_ops.gather( - cache, window_power_ids[-array_ops.shape(indices)[0]:]) - gathered_indices.set_shape(indices.get_shape().concatenate( - gathered_indices.get_shape()[-2:])) - all_gathered_indices.append(gathered_indices) - return all_gathered_indices - - def _cached_transition_powers_and_sums(self, num_steps): - return self._lookup_window_caches( - caches=[self._window_transition_powers, self._window_power_sums], - indices=num_steps) - - def _imputation_step(self, current_times, state): - """Add state transition noise to catch `state` up to `current_times`. - - State space models are inherently sequential, so we need to "predict - through" any missing time steps to catch up each element of the batch to its - next observation/prediction time. - - Args: - current_times: A [batch size] Tensor of times to impute up to, not - inclusive. - state: A tuple of (mean, covariance, previous_times) having shapes - mean; [batch size x state dimension] - covariance; [batch size x state dimension x state dimension] - previous_times; [batch size] - Returns: - Imputed model state corresponding to the `state` argument. - """ - estimated_state, estimated_state_var, previous_times = state - # Ignore negative imputation intervals due to transient start time - # estimates. - catchup_times = math_ops.maximum(current_times - previous_times, 0) - transition_matrices, transition_noise_sums = ( # pylint: disable=unbalanced-tuple-unpacking - self._cached_transition_powers_and_sums(catchup_times)) - estimated_state = self._kalman_filter.predict_state_mean( - estimated_state, transition_matrices) - estimated_state_var = self._kalman_filter.predict_state_var( - estimated_state_var, transition_matrices, transition_noise_sums) - return (estimated_state, estimated_state_var, - previous_times + catchup_times) - - def _filtering_step(self, current_times, current_values, state, predictions): - """Compute posteriors and accumulate one-step-ahead predictions. - - Args: - current_times: A [batch size] Tensor for times for each observation. - current_values: A [batch size] Tensor of values for each observation. - state: A tuple of (mean, covariance, previous_times) having shapes - mean; [batch size x state dimension] - covariance; [batch size x state dimension x state dimension] - previous_times; [batch size] - predictions: A dictionary containing mean and covariance Tensors, the - output of _prediction_step. - Returns: - A tuple of (posteriors, outputs): - posteriors: Model state updated to take `current_values` into account. - outputs: The `predictions` dictionary updated to include "loss" and - "log_likelihood" entries (loss simply being negative log - likelihood). - """ - estimated_state, estimated_state_covariance, previous_times = state - observation_model = self.get_broadcasted_observation_model(current_times) - imputed_to_current_step_assert = control_flow_ops.Assert( - math_ops.reduce_all(math_ops.equal(current_times, previous_times)), - ["Attempted to perform filtering without imputation/prediction"]) - with ops.control_dependencies([imputed_to_current_step_assert]): - estimated_state_covariance = math_utils.clip_covariance( - estimated_state_covariance, - self._configuration.filtering_maximum_posterior_variance_ratio, - self._configuration.filtering_minimum_posterior_variance) - (filtered_state, filtered_state_covariance, - log_prob) = self._kalman_filter.do_filter( - estimated_state=estimated_state, - estimated_state_covariance=estimated_state_covariance, - predicted_observation=predictions["mean"], - predicted_observation_covariance=predictions["covariance"], - observation=current_values, - observation_model=observation_model, - observation_noise=self._observation_noise_covariance) - filtered_state = (filtered_state, filtered_state_covariance, current_times) - log_prob.set_shape(current_times.get_shape()) - predictions["loss"] = -log_prob - predictions["log_likelihood"] = log_prob - if self._configuration.filtering_postprocessor is not None: - return self._configuration.filtering_postprocessor.process_filtering_step( - current_times=current_times, - current_values=current_values, - predicted_state=state, - filtered_state=filtered_state, - outputs=predictions) - return (filtered_state, predictions) - - def _scale_back_predictions(self, predictions): - """Return a window of predictions to input scale.""" - predictions["mean"] = self._scale_back_data(predictions["mean"]) - predictions["covariance"] = self._scale_back_variance( - predictions["covariance"]) - return predictions - - def _prediction_step(self, current_times, state): - """Make a prediction based on `state`. - - Computes predictions based on the current `state`, checking that it has - already been updated (in `_imputation_step`) to `current_times`. - - Args: - current_times: A [batch size] Tensor for times to make predictions for. - state: A tuple of (mean, covariance, previous_times) having shapes - mean; [batch size x state dimension] - covariance; [batch size x state dimension x state dimension] - previous_times; [batch size] - Returns: - A tuple of (updated state, predictions): - updated state: Model state with added transition noise. - predictions: A dictionary with "mean" and "covariance", having shapes - "mean": [batch size x num features] - "covariance: [batch size x num features x num features] - """ - estimated_state, estimated_state_var, previous_times = state - advanced_to_current_assert = control_flow_ops.Assert( - math_ops.reduce_all(math_ops.less_equal(current_times, previous_times)), - ["Attempted to predict without imputation"]) - with ops.control_dependencies([advanced_to_current_assert]): - observation_model = self.get_broadcasted_observation_model(current_times) - predicted_obs, predicted_obs_var = ( - self._kalman_filter.observed_from_state( - state_mean=estimated_state, - state_var=estimated_state_var, - observation_model=observation_model, - observation_noise=self._observation_noise_covariance)) - predicted_obs_var.set_shape( - ops.convert_to_tensor(current_times).get_shape() - .concatenate([self.num_features, self.num_features])) - predicted_obs.set_shape(current_times.get_shape().concatenate( - (self.num_features,))) - predicted_obs_var.set_shape(current_times.get_shape().concatenate( - (self.num_features, self.num_features))) - # Not scaled back to input-scale, since this also feeds into the - # loss. Instead, predictions are scaled back before being returned to the - # user in _scale_back_predictions. - predictions = { - "mean": predicted_obs, - "covariance": predicted_obs_var} - state = (estimated_state, estimated_state_var, current_times) - return (state, predictions) - - def _exogenous_noise_decreasing(self, current_times, exogenous_values, state): - """Update state with exogenous regressors, decreasing uncertainty. - - Constructs a mean and covariance based on transformations of - `exogenous_values`, then performs Bayesian inference on the constructed - observation. This has the effect of lowering uncertainty. - - This update refines or overrides previous inferences, useful for modeling - exogenous inputs which "set" state, e.g. we dumped boiling water on the - thermometer so we're pretty sure it's 100 degrees C. - - Args: - current_times: A [batch size] Tensor of times for the exogenous values - being input. - exogenous_values: A [batch size x exogenous input dimension] Tensor of - exogenous values for each part of the batch. - state: A tuple of (mean, covariance, previous_times) having shapes - mean; [batch size x state dimension] - covariance; [batch size x state dimension x state dimension] - previous_times; [batch size] - Returns: - Updated state taking the exogenous regressors into account (with lower - uncertainty than the input state). - - """ - estimated_state, estimated_state_covariance, previous_times = state - state_transition = ops.convert_to_tensor( - self.get_state_transition(), dtype=self.dtype) - state_dimension = tensor_shape.dimension_value(state_transition.shape[0]) - # Learning the observation model would be redundant since we transform - # `exogenous_values` to the state space via a linear transformation anyway. - observation_model = linalg_ops.eye( - state_dimension, - batch_shape=array_ops.shape(exogenous_values)[:-1], - dtype=self.dtype) - with variable_scope.variable_scope("exogenous_noise_decreasing_covariance"): - observation_noise = math_utils.transform_to_covariance_matrices( - exogenous_values, state_dimension) - with variable_scope.variable_scope( - "exogenous_noise_decreasing_observation"): - observation = layers.fully_connected( - exogenous_values, state_dimension, activation_fn=None) - # Pretend that we are making an observation with an observation model equal - # to the identity matrix (i.e. a direct observation of the latent state), - # with learned observation noise. - posterior_state, posterior_state_var = ( - self._kalman_filter.posterior_from_prior_state( - prior_state=estimated_state, - prior_state_var=estimated_state_covariance, - observation=observation, - observation_model=observation_model, - predicted_observations=( - estimated_state, - # The predicted noise covariance is noise due to current state - # uncertainty plus noise learned based on the exogenous - # observation (a somewhat trivial call to - # self._kalman_filter.observed_from_state has been omitted). - observation_noise + estimated_state_covariance), - observation_noise=observation_noise)) - return (posterior_state, posterior_state_var, previous_times) - - def _exogenous_noise_increasing(self, current_times, exogenous_values, state): - """Update state with exogenous regressors, increasing uncertainty. - - Adds to the state mean a linear transformation of `exogenous_values`, and - increases uncertainty by constructing a covariance matrix based on - `exogenous_values` and adding it to the state covariance. - - This update is useful for modeling changes relative to current state, - e.g. the furnace turned on so the temperature will be increasing at an - additional 1 degree per minute with some uncertainty, this uncertainty being - added to our current uncertainty in the per-minute change in temperature. - - Args: - current_times: A [batch size] Tensor of times for the exogenous values - being input. - exogenous_values: A [batch size x exogenous input dimension] Tensor of - exogenous values for each part of the batch. - state: A tuple of (mean, covariance, previous_times) having shapes - mean; [batch size x state dimension] - covariance; [batch size x state dimension x state dimension] - previous_times; [batch size] - Returns: - Updated state taking the exogenous regressors into account (with higher - uncertainty than the input state). - - """ - start_mean, start_covariance, previous_times = state - with variable_scope.variable_scope("exogenous_noise_increasing_mean"): - mean_addition = layers.fully_connected( - exogenous_values, - tensor_shape.dimension_value(start_mean.shape[1]), activation_fn=None) - state_dimension = tensor_shape.dimension_value(start_covariance.shape[1]) - with variable_scope.variable_scope("exogenous_noise_increasing_covariance"): - covariance_addition = ( - math_utils.transform_to_covariance_matrices( - exogenous_values, state_dimension)) - return (start_mean + mean_addition, - start_covariance + covariance_addition, - previous_times) - - def _exogenous_input_step( - self, current_times, current_exogenous_regressors, state): - """Update state with exogenous regressors. - - Allows both increases and decreases in uncertainty. - - Args: - current_times: A [batch size] Tensor of times for the exogenous values - being input. - current_exogenous_regressors: A [batch size x exogenous input dimension] - Tensor of exogenous values for each part of the batch. - state: A tuple of (mean, covariance, previous_times) having shapes - mean; [batch size x state dimension] - covariance; [batch size x state dimension x state dimension] - previous_times; [batch size] - Returns: - Updated state taking the exogenous regressors into account. - """ - if self._configuration.exogenous_noise_decreases: - state = self._exogenous_noise_decreasing( - current_times, current_exogenous_regressors, state) - if self._configuration.exogenous_noise_increases: - state = self._exogenous_noise_increasing( - current_times, current_exogenous_regressors, state) - return state - - def _loss_additions(self, times, values, mode): - """Add regularization during training.""" - if mode == estimator_lib.ModeKeys.TRAIN: - if (self._input_statistics is not None - and self._configuration.bayesian_prior_weighting): - normalization = 1. / math_ops.cast( - self._input_statistics.total_observation_count, self.dtype) - else: - # If there is no total observation count recorded, or if we are not - # doing a Bayesian prior weighting, assumes/pretends that the full - # dataset size is the window size. - normalization = 1. / math_ops.cast( - array_ops.shape(times)[1], self.dtype) - transition_contribution = ops.convert_to_tensor( - self._configuration.covariance_prior_fn( - self.state_transition_noise_covariance), - dtype=self.dtype) - if (self._configuration.use_observation_noise - and self._observation_noise_covariance is not None): - observation_contribution = ops.convert_to_tensor( - self._configuration.covariance_prior_fn( - self._observation_noise_covariance), - dtype=self.dtype) - regularization_sum = transition_contribution + observation_contribution - else: - regularization_sum = transition_contribution - return -normalization * regularization_sum - else: - return array_ops.zeros([], dtype=self.dtype) - - def _variable_observation_transition_tradeoff_log(self): - """Define a variable to trade off observation and transition noise.""" - return variable_scope.get_variable( - name="observation_transition_tradeoff_log_scale", - initializer=constant_op.constant( - -self._configuration.transition_covariance_initial_log_scale_bias, - dtype=self.dtype), - dtype=self.dtype) - - def _define_parameters(self, observation_transition_tradeoff_log=None): - """Define extra model-specific parameters. - - Models should wrap any variables defined here in the model's variable scope. - - Args: - observation_transition_tradeoff_log: An ensemble-global parameter - controlling the tradeoff between observation noise and transition - noise. If its value is not None, component transition noise should scale - with e^-observation_transition_tradeoff_log. - """ - with variable_scope.variable_scope(self._variable_scope): - # A scalar which allows the optimizer to quickly shift from observation - # noise to transition noise (this value is subtracted from log transition - # noise and added to log observation noise). - if observation_transition_tradeoff_log is None: - self._observation_transition_tradeoff_log_scale = ( - self._variable_observation_transition_tradeoff_log()) - else: - self._observation_transition_tradeoff_log_scale = ( - observation_transition_tradeoff_log) - self.state_transition_noise_covariance = ( - self.get_state_transition_noise_covariance()) - - def _set_input_statistics(self, input_statistics=None): - super(StateSpaceModel, self).initialize_graph( - input_statistics=input_statistics) - - def initialize_graph(self, input_statistics=None): - """Define variables and ops relevant to the top-level model in an ensemble. - - For generic model parameters, _define_parameters() is called recursively on - all members of an ensemble. - - Args: - input_statistics: A math_utils.InputStatistics object containing input - statistics. If None, data-independent defaults are used, which may - result in longer or unstable training. - """ - self._set_input_statistics(input_statistics=input_statistics) - self._define_parameters() - with variable_scope.variable_scope(self._variable_scope): - self._observation_noise_covariance = ops.convert_to_tensor( - self.get_observation_noise_covariance(), dtype=self.dtype) - self._kalman_filter = kalman_filter.KalmanFilter(dtype=self.dtype) - (self.prior_state_mean, - self.prior_state_var) = self._make_priors() - - def _make_priors(self): - """Creates and returns model priors.""" - prior_state_covariance = self.get_prior_covariance() - prior_state_mean = self.get_prior_mean() - return (prior_state_mean, prior_state_covariance) - - def get_prior_covariance(self): - """Constructs a variable prior covariance with data-based initialization. - - Models should wrap any variables defined here in the model's variable scope. - - Returns: - A two-dimensional [state dimension, state dimension] floating point Tensor - with a (positive definite) prior state covariance matrix. - """ - with variable_scope.variable_scope(self._variable_scope): - state_dimension = ops.convert_to_tensor( - self.get_state_transition()).get_shape().dims[0].value - if self._configuration.trainable_start_state: - base_covariance = math_utils.variable_covariance_matrix( - state_dimension, "prior_state_var", - dtype=self.dtype) - else: - return linalg_ops.eye(state_dimension, dtype=self.dtype) - if self._input_statistics is not None: - # Make sure initial latent value uncertainty is at least on the same - # scale as noise in the data. - covariance_multiplier = math_ops.reduce_max( - self._scale_variance( - self._input_statistics.series_start_moments.variance)) - return base_covariance * gen_math_ops.maximum( - covariance_multiplier, 1.0) - else: - return base_covariance - - def get_prior_mean(self): - """Constructs a Variable-parameterized prior mean. - - Models should wrap any variables defined here in the model's variable scope. - - Returns: - A one-dimensional floating point Tensor with shape [state dimension] - indicating the prior mean. - """ - with variable_scope.variable_scope(self._variable_scope): - state_transition = ops.convert_to_tensor( - self.get_state_transition(), dtype=self.dtype) - state_dimension = state_transition.get_shape().dims[0].value - return variable_scope.get_variable( - name="prior_state_mean", - shape=[state_dimension], - dtype=self.dtype, - trainable=self._configuration.trainable_start_state) - - # TODO(allenl): It would be nice if the generation were done with TensorFlow - # ops, and if the model parameters were somehow set instead of being passed - # around in a dictionary. Maybe unconditional generation should be through a - # special set of initializers? - def random_model_parameters(self, seed=None): - if self.num_features != 1: - raise NotImplementedError("Generation for multivariate state space models" - " is not currently implemented.") - if seed: - numpy.random.seed(seed) - state_dimension, noise_dimension = ops.convert_to_tensor( - self.get_noise_transform()).get_shape().as_list() - transition_var = 1.0 / numpy.random.gamma(shape=10., scale=10., - size=[noise_dimension]) - initial_state = numpy.random.normal(size=[state_dimension]) - params_dict = {} - if self.prior_state_mean is not None: - params_dict[self.prior_state_mean] = initial_state - if self.state_transition_noise_covariance is not None: - params_dict[self.state_transition_noise_covariance] = numpy.diag( - transition_var) - if self.prior_state_var is not None: - params_dict[self.prior_state_var] = numpy.zeros( - [state_dimension, state_dimension]) - if self._configuration.use_observation_noise: - observation_var = 1.0 / numpy.random.gamma(shape=4, scale=4) - params_dict[self._observation_noise_covariance] = [[observation_var]] - return params_dict - - def generate(self, number_of_series, series_length, - model_parameters=None, seed=None, add_observation_noise=None): - if seed is not None: - numpy.random.seed(seed) - if self.num_features != 1: - raise NotImplementedError("Generation for multivariate state space models" - " is not currently implemented.") - if add_observation_noise is None: - add_observation_noise = self._configuration.use_observation_noise - if model_parameters is None: - model_parameters = {} - transitions = ops.convert_to_tensor( - self.get_state_transition(), dtype=self.dtype).eval( - feed_dict=model_parameters) - noise_transform = ops.convert_to_tensor(self.get_noise_transform()).eval( - feed_dict=model_parameters) - - noise_dimension = noise_transform.shape[1] - get_passed_or_trained_value = model_utils.parameter_switch(model_parameters) - transition_var = numpy.diag(get_passed_or_trained_value( - self.state_transition_noise_covariance)) - transition_std = numpy.sqrt(transition_var) - if add_observation_noise: - observation_var = get_passed_or_trained_value( - self._observation_noise_covariance)[0][0] - observation_std = numpy.sqrt(observation_var) - initial_state = get_passed_or_trained_value(self.prior_state_mean) - current_state = numpy.tile(numpy.expand_dims(initial_state, 0), - [number_of_series, 1]) - observations = numpy.zeros([number_of_series, series_length]) - observation_models = self.get_broadcasted_observation_model( - times=math_ops.range(series_length)).eval(feed_dict=model_parameters) - for timestep, observation_model in enumerate(observation_models): - current_state = numpy.dot(current_state, transitions.T) - current_state += numpy.dot( - numpy.random.normal( - loc=numpy.zeros([number_of_series, noise_dimension]), - scale=numpy.tile(numpy.expand_dims(transition_std, 0), - [number_of_series, 1])), - noise_transform.T) - observation_mean = numpy.dot(current_state, observation_model[0].T) - if add_observation_noise: - observations[:, timestep] = numpy.random.normal(loc=observation_mean, - scale=observation_std) - else: - observations[:, timestep] = observation_mean - observations = numpy.expand_dims(observations, -1) - times = numpy.tile( - numpy.expand_dims(numpy.arange(observations.shape[1]), 0), - [observations.shape[0], 1]) - return {TrainEvalFeatures.TIMES: times, - TrainEvalFeatures.VALUES: observations} - - @abc.abstractmethod - def get_state_transition(self): - """Specifies the state transition model to use. - - Returns: - A [state dimension x state dimension] Tensor specifying how states - transition from one timestep to the next. - """ - pass - - @abc.abstractmethod - def get_noise_transform(self): - """Specifies the noise transition model to use. - - Returns: - A [state dimension x state noise dimension] Tensor specifying how noise - (generated with shape [state noise dimension]) affects the model's state. - """ - pass - - @abc.abstractmethod - def get_observation_model(self, times): - """Specifies the observation model to use. - - Args: - times: A [batch dimension] int32 Tensor with times for each part of the - batch, on which the observation model can depend. - Returns: - This function, when overridden, has three possible return values: - - A [state dimension] Tensor with a static, univariate observation - model. - - A [self.num_features x state dimension] static, multivariate model. - - A [batch dimension x self.num_features x state dimension] observation - model, which may depend on `times`. - See get_broadcasted_observation_model for details of the broadcasting. - """ - pass - - def get_broadcasted_observation_model(self, times): - """Broadcast this model's observation model if necessary. - - The model can define a univariate observation model which will be broadcast - over both self.num_features and the batch dimension of `times`. - - The model can define a multi-variate observation model which does not depend - on `times`, and it will be broadcast over the batch dimension of `times`. - - Finally, the model can define a multi-variate observation model with a batch - dimension, which will not be broadcast. - - Args: - times: A [batch dimension] int32 Tensor with times for each part of the - batch, on which the observation model can depend. - Returns: - A [batch dimension x self.num_features x state dimension] Tensor - specifying the observation model to use for each time in `times` and each - feature. - """ - unbroadcasted_model = ops.convert_to_tensor( - self.get_observation_model(times), dtype=self.dtype) - unbroadcasted_shape = (unbroadcasted_model.get_shape() - .with_rank_at_least(1).with_rank_at_most(3)) - if unbroadcasted_shape.ndims is None: - # Pass through fully undefined shapes, but make sure they're rank 3 at - # graph eval time - assert_op = control_flow_ops.Assert( - math_ops.equal(array_ops.rank(unbroadcasted_model), 3), - [array_ops.shape(unbroadcasted_model)]) - with ops.control_dependencies([assert_op]): - return array_ops.identity(unbroadcasted_model) - if unbroadcasted_shape.ndims == 1: - # Unbroadcasted shape [state dimension] - broadcasted_model = array_ops.tile( - array_ops.reshape(tensor=unbroadcasted_model, shape=[1, 1, -1]), - [array_ops.shape(times)[0], self.num_features, 1]) - elif unbroadcasted_shape.ndims == 2: - # Unbroadcasted shape [num features x state dimension] - broadcasted_model = array_ops.tile( - array_ops.expand_dims(unbroadcasted_model, axis=0), - [array_ops.shape(times)[0], 1, 1]) - elif unbroadcasted_shape.ndims == 3: - broadcasted_model = unbroadcasted_model - broadcasted_model.get_shape().assert_has_rank(3) - return broadcasted_model - - def get_state_transition_noise_covariance( - self, minimum_initial_variance=1e-5): - state_noise_transform = ops.convert_to_tensor( - self.get_noise_transform(), dtype=self.dtype) - state_noise_dimension = state_noise_transform.get_shape().dims[1].value - if self._input_statistics is not None: - feature_variance = self._scale_variance( - self._input_statistics.series_start_moments.variance) - initial_transition_noise_scale = math_ops.log( - gen_math_ops.maximum( - math_ops.reduce_mean(feature_variance) / math_ops.cast( - self._input_statistics.total_observation_count, self.dtype), - minimum_initial_variance)) - else: - initial_transition_noise_scale = 0. - # Generally high transition noise is undesirable; we want to set it quite - # low to start so that we don't need too much training to get to good - # solutions (i.e. with confident predictions into the future if possible), - # but not so low that training can't yield a high transition noise if the - # data demands it. - initial_transition_noise_scale -= ( - self._observation_transition_tradeoff_log_scale) - return math_utils.variable_covariance_matrix( - state_noise_dimension, "state_transition_noise", - dtype=self.dtype, - initial_overall_scale_log=initial_transition_noise_scale) - - def get_observation_noise_covariance(self, minimum_initial_variance=1e-5): - if self._configuration.use_observation_noise: - if self._input_statistics is not None: - # Get variance across the first few values in each batch for each - # feature, for an initial observation noise (over-)estimate. - feature_variance = self._scale_variance( - self._input_statistics.series_start_moments.variance) - else: - feature_variance = None - if feature_variance is not None: - feature_variance = gen_math_ops.maximum(feature_variance, - minimum_initial_variance) - return math_utils.variable_covariance_matrix( - size=self.num_features, - dtype=self.dtype, - name="observation_noise_covariance", - initial_diagonal_values=feature_variance, - initial_overall_scale_log=( - self._observation_transition_tradeoff_log_scale)) - else: - return array_ops.zeros( - shape=[self.num_features, self.num_features], - name="observation_noise_covariance", - dtype=self.dtype) - - def get_start_state(self): - """Defines and returns a non-batched prior state and covariance.""" - # TODO(allenl,vitalyk): Add an option for non-Gaussian priors once extended - # Kalman filtering is implemented (ideally any Distribution object). - if self._input_statistics is not None: - start_time = self._input_statistics.start_time - else: - start_time = array_ops.zeros([], dtype=dtypes.int64) - return (self.prior_state_mean, - self.prior_state_var, - start_time - 1) - - def get_features_for_timesteps(self, timesteps): - """Get features for a batch of timesteps. Default to no features.""" - return array_ops.zeros([array_ops.shape(timesteps)[0], 0], dtype=self.dtype) - - -class StateSpaceEnsemble(StateSpaceModel): - """Base class for combinations of state space models.""" - - def __init__(self, ensemble_members, configuration): - """Initialize the ensemble by specifying its members. - - Args: - ensemble_members: A list of StateSpaceModel objects which will be included - in this ensemble. - configuration: A StateSpaceModelConfiguration object. - """ - self._ensemble_members = ensemble_members - super(StateSpaceEnsemble, self).__init__(configuration=configuration) - - def _set_input_statistics(self, input_statistics): - super(StateSpaceEnsemble, self)._set_input_statistics(input_statistics) - for member in self._ensemble_members: - member._set_input_statistics(input_statistics) # pylint: disable=protected-access - - def _loss_additions(self, times, values, mode): - # Allow sub-models to regularize - return (super(StateSpaceEnsemble, self)._loss_additions( - times, values, mode) + math_ops.add_n([ - member._loss_additions(times, values, mode) # pylint: disable=protected-access - for member in self._ensemble_members - ])) - - def _compute_blocked(self, member_fn, name): - with variable_scope.variable_scope(self._variable_scope): - return math_utils.block_diagonal( - [member_fn(member) - for member in self._ensemble_members], - dtype=self.dtype, - name=name) - - def transition_to_powers(self, powers): - return self._compute_blocked( - member_fn=lambda member: member.transition_to_powers(powers), - name="ensemble_transition_to_powers") - - def _define_parameters(self, observation_transition_tradeoff_log=None): - with variable_scope.variable_scope(self._variable_scope): - if observation_transition_tradeoff_log is None: - # Define the tradeoff parameter between observation and transition noise - # once for the whole ensemble, and pass it down to members. - observation_transition_tradeoff_log = ( - self._variable_observation_transition_tradeoff_log()) - for member in self._ensemble_members: - member._define_parameters(observation_transition_tradeoff_log=( # pylint: disable=protected-access - observation_transition_tradeoff_log)) - super(StateSpaceEnsemble, self)._define_parameters( - observation_transition_tradeoff_log - =observation_transition_tradeoff_log) - - def random_model_parameters(self, seed=None): - param_union = {} - for i, member in enumerate(self._ensemble_members): - member_params = member.random_model_parameters( - seed=seed + i if seed else None) - param_union.update(member_params) - param_union.update( - super(StateSpaceEnsemble, self).random_model_parameters(seed=seed)) - return param_union - - def get_prior_mean(self): - return array_ops.concat( - values=[member.get_prior_mean() for member in self._ensemble_members], - axis=0, - name="ensemble_prior_state_mean") - - def get_state_transition(self): - return self._compute_blocked( - member_fn= - lambda member: member.get_state_transition(), - name="ensemble_state_transition") - - def get_noise_transform(self): - return self._compute_blocked( - member_fn= - lambda member: member.get_noise_transform(), - name="ensemble_noise_transform") - - def get_observation_model(self, times): - raise NotImplementedError("No un-broadcasted observation model defined for" - " ensembles.") - - def get_broadcasted_observation_model(self, times): - """Computes a combined observation model based on member models. - - The effect is that predicted observations from each model are summed. - - Args: - times: A [batch dimension] int32 Tensor with times for each part of the - batch, on which member observation models can depend. - Returns: - A [batch dimension x num features x combined state dimension] Tensor with - the combined observation model. - """ - member_observation_models = [ - ops.convert_to_tensor( - member.get_broadcasted_observation_model(times), dtype=self.dtype) - for member in self._ensemble_members - ] - return array_ops.concat(values=member_observation_models, axis=2) - - -class StateSpaceIndependentEnsemble(StateSpaceEnsemble): - """Implements ensembles of independent state space models. - - Useful for fitting multiple independent state space models together while - keeping their specifications decoupled. The "ensemble" is simply a state space - model with the observation models of its members concatenated, and the - transition matrices and noise transforms stacked in block-diagonal - matrices. This means that the dimensionality of the ensemble's state is the - sum of those of its components, which can lead to slow and memory-intensive - training and inference as the posterior (shape [state dimension x state - dimension]) gets large. - - Each individual model j's state at time t is defined by: - - state[t, j] = StateTransition[j] * state[t-1, j] - + NoiseTransform[j] * StateNoise[t, j] - StateNoise[t, j] ~ Gaussian(0, StateNoiseCovariance[j]) - - and the ensemble observation model is: - - observation[t] = Sum { ObservationModel[j] * state[t, j] } - + ObservationNoise[t] - ObservationNoise[t] ~ Gaussian(0, ObservationNoiseCovariance) - """ - - def transition_power_noise_accumulator(self, num_steps): - return self._compute_blocked( - member_fn=lambda m: m.transition_power_noise_accumulator(num_steps), - name="ensemble_power_noise_accumulator") - - def get_prior_covariance(self): - """Construct the ensemble prior covariance based on component models.""" - return self._compute_blocked( - member_fn= - lambda member: member.get_prior_covariance(), - name="ensemble_prior_state_covariance") - - def get_state_transition_noise_covariance(self): - """Construct the ensemble transition noise covariance from components.""" - return self._compute_blocked( - member_fn= - lambda member: member.state_transition_noise_covariance, - name="ensemble_state_transition_noise") - - -# TODO(allenl): It would be nice to have replicated feature models which are -# identical batched together to reduce the graph size. -# TODO(allenl): Support for sharing M independent models across N features, with -# N > M. -# TODO(allenl): Stack component prior covariances while allowing cross-model -# correlations to be learned (currently a full covariance prior is learned, but -# custom component model covariances are not used). -class StateSpaceCorrelatedFeaturesEnsemble(StateSpaceEnsemble): - """An correlated ensemble where each model represents a feature. - - Unlike `StateSpaceIndependentEnsemble`, a full state transition noise - covariance matrix is learned for this ensemble; the models are not assumed to - be independent. Rather than concatenating observation models (i.e. summing the - contributions of each model to each feature), - StateSpaceCorrelatedFeaturesEnsemble stacks observation models diagonally, - meaning that each model corresponds to one feature of the series. - - Behaves like (and is) a single state space model where: - - StateTransition = Diag(StateTransition[j] for models j) - ObservationModel = Diag(ObservationModel[j] for models j) - - Note that each ObservationModel[j] is a [1 x S_j] matrix (S_j being the state - dimension of model j), i.e. a univariate model. The combined model is - multivariate, the number of features of the series being equal to the number - of component models in the ensemble. - """ - - def __init__(self, ensemble_members, configuration): - """Specify the ensemble's configuration and component models. - - Args: - ensemble_members: A list of `StateSpaceModel` objects, with length equal - to `configuration.num_features`. Each of these models, which must be - univariate, corresponds to a single feature of the time series. - configuration: A StateSpaceModelConfiguration object. - Raises: - ValueError: If the length of `ensemble_members` does not equal the number - of features in the series, or any component is not univariate. - """ - if len(ensemble_members) != configuration.num_features: - raise ValueError( - "The number of members in a StateSpaceCorrelatedFeaturesEnsemble " - "must equal the number of features in the time series.") - for member in ensemble_members: - if member.num_features != 1: - raise ValueError( - "StateSpaceCorrelatedFeaturesEnsemble components must be " - "univariate.") - super(StateSpaceCorrelatedFeaturesEnsemble, self).__init__( - ensemble_members=ensemble_members, configuration=configuration) - - def transition_power_noise_accumulator(self, num_steps): - """Use a noise accumulator special case when possible.""" - if len(self._ensemble_members) == 1: - # If this is a univariate series, we should use the special casing built - # into the single component model. - return self._ensemble_members[0].transition_power_noise_accumulator( - num_steps=num_steps) - # If we have multiple features, and therefore multiple models, we have - # introduced correlations which make noise accumulation more - # complicated. Here we fall back to the general case, since we can't just - # aggregate member special cases. - return super(StateSpaceCorrelatedFeaturesEnsemble, - self).transition_power_noise_accumulator(num_steps=num_steps) - - def get_broadcasted_observation_model(self, times): - """Stack observation models diagonally.""" - def _member_observation_model(member): - return ops.convert_to_tensor( - member.get_broadcasted_observation_model(times), dtype=self.dtype) - return self._compute_blocked(member_fn=_member_observation_model, - name="feature_ensemble_observation_model") diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/state_space_model_test.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/state_space_model_test.py deleted file mode 100644 index 26857ba235e..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/state_space_model_test.py +++ /dev/null @@ -1,755 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for state space model infrastructure.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -import numpy - -from tensorflow.contrib import layers - -from tensorflow.contrib.timeseries.python.timeseries import estimators -from tensorflow.contrib.timeseries.python.timeseries import feature_keys -from tensorflow.contrib.timeseries.python.timeseries import input_pipeline -from tensorflow.contrib.timeseries.python.timeseries import math_utils -from tensorflow.contrib.timeseries.python.timeseries import saved_model_utils -from tensorflow.contrib.timeseries.python.timeseries import state_management -from tensorflow.contrib.timeseries.python.timeseries import test_utils -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import state_space_model - -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.saved_model import loader -from tensorflow.python.saved_model import tag_constants -from tensorflow.python.training import coordinator as coordinator_lib -from tensorflow.python.training import gradient_descent -from tensorflow.python.training import queue_runner_impl - - -class RandomStateSpaceModel(state_space_model.StateSpaceModel): - - def __init__(self, - state_dimension, - state_noise_dimension, - configuration=state_space_model.StateSpaceModelConfiguration()): - self.transition = numpy.random.normal( - size=[state_dimension, state_dimension]).astype( - configuration.dtype.as_numpy_dtype) - self.noise_transform = numpy.random.normal( - size=(state_dimension, state_noise_dimension)).astype( - configuration.dtype.as_numpy_dtype) - # Test batch broadcasting - self.observation_model = numpy.random.normal( - size=(configuration.num_features, state_dimension)).astype( - configuration.dtype.as_numpy_dtype) - super(RandomStateSpaceModel, self).__init__( - configuration=configuration._replace( - covariance_prior_fn=lambda _: 0.)) - - def get_state_transition(self): - return self.transition - - def get_noise_transform(self): - return self.noise_transform - - def get_observation_model(self, times): - return self.observation_model - - -class ConstructionTests(test.TestCase): - - def test_initialize_graph_error(self): - with self.assertRaisesRegexp(ValueError, "initialize_graph"): - model = RandomStateSpaceModel(2, 2) - outputs = model.define_loss( - features={ - feature_keys.TrainEvalFeatures.TIMES: - constant_op.constant([[1, 2]]), - feature_keys.TrainEvalFeatures.VALUES: - constant_op.constant([[[1.], [2.]]]) - }, - mode=estimator_lib.ModeKeys.TRAIN) - initializer = variables.global_variables_initializer() - with self.cached_session() as sess: - sess.run([initializer]) - outputs.loss.eval() - - def test_initialize_graph_state_manager_error(self): - with self.assertRaisesRegexp(ValueError, "initialize_graph"): - model = RandomStateSpaceModel(2, 2) - state_manager = state_management.ChainingStateManager() - outputs = state_manager.define_loss( - model=model, - features={ - feature_keys.TrainEvalFeatures.TIMES: - constant_op.constant([[1, 2]]), - feature_keys.TrainEvalFeatures.VALUES: - constant_op.constant([[[1.], [2.]]]) - }, - mode=estimator_lib.ModeKeys.TRAIN) - initializer = variables.global_variables_initializer() - with self.cached_session() as sess: - sess.run([initializer]) - outputs.loss.eval() - - -class GapTests(test.TestCase): - - def _gap_test_template(self, times, values): - random_model = RandomStateSpaceModel( - state_dimension=1, state_noise_dimension=1, - configuration=state_space_model.StateSpaceModelConfiguration( - num_features=1)) - random_model.initialize_graph() - input_fn = input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader({ - feature_keys.TrainEvalFeatures.TIMES: times, - feature_keys.TrainEvalFeatures.VALUES: values - })) - features, _ = input_fn() - times = features[feature_keys.TrainEvalFeatures.TIMES] - values = features[feature_keys.TrainEvalFeatures.VALUES] - model_outputs = random_model.get_batch_loss( - features={ - feature_keys.TrainEvalFeatures.TIMES: times, - feature_keys.TrainEvalFeatures.VALUES: values - }, - mode=None, - state=math_utils.replicate_state( - start_state=random_model.get_start_state(), - batch_size=array_ops.shape(times)[0])) - with self.cached_session() as session: - variables.global_variables_initializer().run() - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(session, coord=coordinator) - model_outputs.loss.eval() - coordinator.request_stop() - coordinator.join() - - def test_start_gap(self): - self._gap_test_template(times=[20, 21, 22], values=numpy.arange(3)) - - def test_mid_gap(self): - self._gap_test_template(times=[2, 60, 61], values=numpy.arange(3)) - - def test_end_gap(self): - self._gap_test_template(times=[2, 3, 73], values=numpy.arange(3)) - - def test_all_gaps(self): - self._gap_test_template(times=[2, 4, 8, 16, 32, 64, 128], - values=numpy.arange(7)) - - -class StateSpaceEquivalenceTests(test.TestCase): - - def test_savedmodel_state_override(self): - random_model = RandomStateSpaceModel( - state_dimension=5, - state_noise_dimension=4, - configuration=state_space_model.StateSpaceModelConfiguration( - exogenous_feature_columns=[layers.real_valued_column("exogenous")], - dtype=dtypes.float64, num_features=1)) - estimator = estimators.StateSpaceRegressor( - model=random_model, - optimizer=gradient_descent.GradientDescentOptimizer(0.1)) - combined_input_fn = input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader({ - feature_keys.FilteringFeatures.TIMES: [1, 2, 3, 4], - feature_keys.FilteringFeatures.VALUES: [1., 2., 3., 4.], - "exogenous": [-1., -2., -3., -4.] - })) - estimator.train(combined_input_fn, steps=1) - export_location = estimator.export_saved_model( - self.get_temp_dir(), estimator.build_raw_serving_input_receiver_fn()) - with ops.Graph().as_default() as graph: - random_model.initialize_graph() - with self.session(graph=graph) as session: - variables.global_variables_initializer().run() - evaled_start_state = session.run(random_model.get_start_state()) - evaled_start_state = [ - state_element[None, ...] for state_element in evaled_start_state] - with ops.Graph().as_default() as graph: - with self.session(graph=graph) as session: - signatures = loader.load( - session, [tag_constants.SERVING], export_location) - first_split_filtering = saved_model_utils.filter_continuation( - continue_from={ - feature_keys.FilteringResults.STATE_TUPLE: evaled_start_state}, - signatures=signatures, - session=session, - features={ - feature_keys.FilteringFeatures.TIMES: [1, 2], - feature_keys.FilteringFeatures.VALUES: [1., 2.], - "exogenous": [[-1.], [-2.]]}) - second_split_filtering = saved_model_utils.filter_continuation( - continue_from=first_split_filtering, - signatures=signatures, - session=session, - features={ - feature_keys.FilteringFeatures.TIMES: [3, 4], - feature_keys.FilteringFeatures.VALUES: [3., 4.], - "exogenous": [[-3.], [-4.]] - }) - combined_filtering = saved_model_utils.filter_continuation( - continue_from={ - feature_keys.FilteringResults.STATE_TUPLE: evaled_start_state}, - signatures=signatures, - session=session, - features={ - feature_keys.FilteringFeatures.TIMES: [1, 2, 3, 4], - feature_keys.FilteringFeatures.VALUES: [1., 2., 3., 4.], - "exogenous": [[-1.], [-2.], [-3.], [-4.]] - }) - split_predict = saved_model_utils.predict_continuation( - continue_from=second_split_filtering, - signatures=signatures, - session=session, - steps=1, - exogenous_features={ - "exogenous": [[[-5.]]]}) - combined_predict = saved_model_utils.predict_continuation( - continue_from=combined_filtering, - signatures=signatures, - session=session, - steps=1, - exogenous_features={ - "exogenous": [[[-5.]]]}) - for state_key, combined_state_value in combined_filtering.items(): - if state_key == feature_keys.FilteringResults.TIMES: - continue - self.assertAllClose( - combined_state_value, second_split_filtering[state_key]) - for prediction_key, combined_value in combined_predict.items(): - self.assertAllClose(combined_value, split_predict[prediction_key]) - - def _equivalent_to_single_model_test_template(self, model_generator): - with self.cached_session() as session: - random_model = RandomStateSpaceModel( - state_dimension=5, - state_noise_dimension=4, - configuration=state_space_model.StateSpaceModelConfiguration( - dtype=dtypes.float64, num_features=1)) - random_model.initialize_graph() - series_length = 10 - model_data = random_model.generate( - number_of_series=1, series_length=series_length, - model_parameters=random_model.random_model_parameters()) - input_fn = input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader(model_data)) - features, _ = input_fn() - model_outputs = random_model.get_batch_loss( - features=features, - mode=None, - state=math_utils.replicate_state( - start_state=random_model.get_start_state(), - batch_size=array_ops.shape( - features[feature_keys.TrainEvalFeatures.TIMES])[0])) - variables.global_variables_initializer().run() - compare_outputs_evaled_fn = model_generator( - random_model, model_data) - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(session, coord=coordinator) - compare_outputs_evaled = compare_outputs_evaled_fn(session) - model_outputs_evaled = session.run( - (model_outputs.end_state, model_outputs.predictions)) - coordinator.request_stop() - coordinator.join() - model_posteriors, model_predictions = model_outputs_evaled - (_, compare_posteriors, - compare_predictions) = compare_outputs_evaled - (model_posterior_mean, model_posterior_var, - model_from_time) = model_posteriors - (compare_posterior_mean, compare_posterior_var, - compare_from_time) = compare_posteriors - self.assertAllClose(model_posterior_mean, compare_posterior_mean[0]) - self.assertAllClose(model_posterior_var, compare_posterior_var[0]) - self.assertAllClose(model_from_time, compare_from_time) - self.assertEqual(sorted(model_predictions.keys()), - sorted(compare_predictions.keys())) - for prediction_name in model_predictions: - if prediction_name == "loss": - # Chunking means that losses will be different; skip testing them. - continue - # Compare the last chunk to their corresponding un-chunked model - # predictions - last_prediction_chunk = compare_predictions[prediction_name][-1] - comparison_values = last_prediction_chunk.shape[0] - model_prediction = ( - model_predictions[prediction_name][0, -comparison_values:]) - self.assertAllClose(model_prediction, - last_prediction_chunk) - - def _model_equivalent_to_chained_model_test_template(self, chunk_size): - def chained_model_outputs(original_model, data): - input_fn = test_utils.AllWindowInputFn( - input_pipeline.NumpyReader(data), window_size=chunk_size) - state_manager = state_management.ChainingStateManager( - state_saving_interval=1) - features, _ = input_fn() - state_manager.initialize_graph(original_model) - model_outputs = state_manager.define_loss( - model=original_model, - features=features, - mode=estimator_lib.ModeKeys.TRAIN) - def _eval_outputs(session): - for _ in range(50): - # Warm up saved state - model_outputs.loss.eval() - (posterior_mean, posterior_var, - priors_from_time) = model_outputs.end_state - posteriors = ((posterior_mean,), (posterior_var,), priors_from_time) - outputs = (model_outputs.loss, posteriors, - model_outputs.predictions) - chunked_outputs_evaled = session.run(outputs) - return chunked_outputs_evaled - return _eval_outputs - self._equivalent_to_single_model_test_template(chained_model_outputs) - - def test_model_equivalent_to_chained_model_chunk_size_one(self): - numpy.random.seed(2) - random_seed.set_random_seed(3) - self._model_equivalent_to_chained_model_test_template(1) - - def test_model_equivalent_to_chained_model_chunk_size_five(self): - numpy.random.seed(4) - random_seed.set_random_seed(5) - self._model_equivalent_to_chained_model_test_template(5) - - -class PredictionTests(test.TestCase): - - def _check_predictions( - self, predicted_mean, predicted_covariance, window_size): - self.assertAllEqual(predicted_covariance.shape, - [1, # batch - window_size, - 1, # num features - 1]) # num features - self.assertAllEqual(predicted_mean.shape, - [1, # batch - window_size, - 1]) # num features - for position in range(window_size - 2): - self.assertGreater(predicted_covariance[0, position + 2, 0, 0], - predicted_covariance[0, position, 0, 0]) - - def test_predictions_direct(self): - dtype = dtypes.float64 - with variable_scope.variable_scope(dtype.name): - random_model = RandomStateSpaceModel( - state_dimension=5, state_noise_dimension=4, - configuration=state_space_model.StateSpaceModelConfiguration( - dtype=dtype, num_features=1)) - random_model.initialize_graph() - prediction_dict = random_model.predict(features={ - feature_keys.PredictionFeatures.TIMES: [[1, 3, 5, 6]], - feature_keys.PredictionFeatures.STATE_TUPLE: - math_utils.replicate_state( - start_state=random_model.get_start_state(), batch_size=1) - }) - with self.cached_session(): - variables.global_variables_initializer().run() - predicted_mean = prediction_dict["mean"].eval() - predicted_covariance = prediction_dict["covariance"].eval() - self._check_predictions(predicted_mean, predicted_covariance, - window_size=4) - - def test_predictions_after_loss(self): - dtype = dtypes.float32 - with variable_scope.variable_scope(dtype.name): - random_model = RandomStateSpaceModel( - state_dimension=5, state_noise_dimension=4, - configuration=state_space_model.StateSpaceModelConfiguration( - dtype=dtype, num_features=1)) - features = { - feature_keys.TrainEvalFeatures.TIMES: [[1, 2, 3, 4]], - feature_keys.TrainEvalFeatures.VALUES: - array_ops.ones([1, 4, 1], dtype=dtype) - } - passthrough = state_management.PassthroughStateManager() - random_model.initialize_graph() - passthrough.initialize_graph(random_model) - model_outputs = passthrough.define_loss( - model=random_model, - features=features, - mode=estimator_lib.ModeKeys.EVAL) - predictions = random_model.predict({ - feature_keys.PredictionFeatures.TIMES: [[5, 7, 8]], - feature_keys.PredictionFeatures.STATE_TUPLE: model_outputs.end_state - }) - with self.cached_session(): - variables.global_variables_initializer().run() - predicted_mean = predictions["mean"].eval() - predicted_covariance = predictions["covariance"].eval() - self._check_predictions(predicted_mean, predicted_covariance, - window_size=3) - - -class ExogenousTests(test.TestCase): - - def test_noise_increasing(self): - for dtype in [dtypes.float32, dtypes.float64]: - with variable_scope.variable_scope(dtype.name): - random_model = RandomStateSpaceModel( - state_dimension=5, state_noise_dimension=4, - configuration=state_space_model.StateSpaceModelConfiguration( - dtype=dtype, num_features=1)) - original_covariance = array_ops.diag(array_ops.ones(shape=[5])) - _, new_covariance, _ = random_model._exogenous_noise_increasing( - current_times=[[1]], - exogenous_values=[[5.]], - state=[ - array_ops.ones(shape=[1, 5]), original_covariance[None], [0] - ]) - with self.cached_session() as session: - variables.global_variables_initializer().run() - evaled_new_covariance, evaled_original_covariance = session.run( - [new_covariance[0], original_covariance]) - new_variances = numpy.diag(evaled_new_covariance) - original_variances = numpy.diag(evaled_original_covariance) - for i in range(5): - self.assertGreater(new_variances[i], original_variances[i]) - - def test_noise_decreasing(self): - for dtype in [dtypes.float32, dtypes.float64]: - with variable_scope.variable_scope(dtype.name): - random_model = RandomStateSpaceModel( - state_dimension=5, state_noise_dimension=4, - configuration=state_space_model.StateSpaceModelConfiguration( - dtype=dtype, num_features=1)) - random_model.initialize_graph() - original_covariance = array_ops.diag( - array_ops.ones(shape=[5], dtype=dtype)) - _, new_covariance, _ = random_model._exogenous_noise_decreasing( - current_times=[[1]], - exogenous_values=constant_op.constant([[-2.]], dtype=dtype), - state=[ - -array_ops.ones(shape=[1, 5], dtype=dtype), - original_covariance[None], [0] - ]) - with self.cached_session() as session: - variables.global_variables_initializer().run() - evaled_new_covariance, evaled_original_covariance = session.run( - [new_covariance[0], original_covariance]) - new_variances = numpy.diag(evaled_new_covariance) - original_variances = numpy.diag(evaled_original_covariance) - for i in range(5): - self.assertLess(new_variances[i], original_variances[i]) - - -class StubStateSpaceModel(state_space_model.StateSpaceModel): - - def __init__(self, - transition, - state_noise_dimension, - configuration=state_space_model.StateSpaceModelConfiguration()): - self.transition = transition - self.noise_transform = numpy.random.normal( - size=(transition.shape[0], state_noise_dimension)).astype(numpy.float32) - # Test feature + batch broadcasting - self.observation_model = numpy.random.normal( - size=(transition.shape[0])).astype(numpy.float32) - super(StubStateSpaceModel, self).__init__( - configuration=configuration) - - def get_state_transition(self): - return self.transition - - def get_noise_transform(self): - return self.noise_transform - - def get_observation_model(self, times): - return self.observation_model - - -GeneratedModel = collections.namedtuple( - "GeneratedModel", ["model", "data", "true_parameters"]) - - -class PosteriorTests(test.TestCase): - - def _get_cycle_transition(self, period): - cycle_transition = numpy.zeros([period - 1, period - 1], - dtype=numpy.float32) - cycle_transition[0, :] = -1 - cycle_transition[1:, :-1] = numpy.identity(period - 2) - return cycle_transition - - _adder_transition = numpy.array([[1, 1], - [0, 1]], dtype=numpy.float32) - - def _get_single_model(self): - numpy.random.seed(8) - stub_model = StubStateSpaceModel( - transition=self._get_cycle_transition(5), state_noise_dimension=0) - series_length = 1000 - stub_model.initialize_graph() - true_params = stub_model.random_model_parameters() - data = stub_model.generate( - number_of_series=1, series_length=series_length, - model_parameters=true_params) - return GeneratedModel( - model=stub_model, data=data, true_parameters=true_params) - - def test_exact_posterior_recovery_no_transition_noise(self): - with self.cached_session() as session: - stub_model, data, true_params = self._get_single_model() - input_fn = input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader(data)) - features, _ = input_fn() - model_outputs = stub_model.get_batch_loss( - features=features, - mode=None, - state=math_utils.replicate_state( - start_state=stub_model.get_start_state(), - batch_size=array_ops.shape( - features[feature_keys.TrainEvalFeatures.TIMES])[0])) - variables.global_variables_initializer().run() - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(session, coord=coordinator) - posterior_mean, posterior_var, posterior_times = session.run( - # Feed the true model parameters so that this test doesn't depend on - # the generated parameters being close to the variable initializations - # (an alternative would be training steps to fit the noise values, - # which would be slow). - model_outputs.end_state, feed_dict=true_params) - coordinator.request_stop() - coordinator.join() - - self.assertAllClose(numpy.zeros([1, 4, 4]), posterior_var, - atol=1e-2) - self.assertAllClose( - numpy.dot( - numpy.linalg.matrix_power( - stub_model.transition, - data[feature_keys.TrainEvalFeatures.TIMES].shape[1]), - true_params[stub_model.prior_state_mean]), - posterior_mean[0], - rtol=1e-1) - self.assertAllClose( - math_utils.batch_end_time( - features[feature_keys.TrainEvalFeatures.TIMES]).eval(), - posterior_times) - - def test_chained_exact_posterior_recovery_no_transition_noise(self): - with self.cached_session() as session: - stub_model, data, true_params = self._get_single_model() - chunk_size = 10 - input_fn = test_utils.AllWindowInputFn( - input_pipeline.NumpyReader(data), window_size=chunk_size) - features, _ = input_fn() - state_manager = state_management.ChainingStateManager( - state_saving_interval=1) - state_manager.initialize_graph(stub_model) - model_outputs = state_manager.define_loss( - model=stub_model, - features=features, - mode=estimator_lib.ModeKeys.TRAIN) - variables.global_variables_initializer().run() - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(session, coord=coordinator) - for _ in range( - data[feature_keys.TrainEvalFeatures.TIMES].shape[1] // chunk_size): - model_outputs.loss.eval() - posterior_mean, posterior_var, posterior_times = session.run( - model_outputs.end_state, feed_dict=true_params) - coordinator.request_stop() - coordinator.join() - self.assertAllClose(numpy.zeros([1, 4, 4]), posterior_var, - atol=1e-2) - self.assertAllClose( - numpy.dot( - numpy.linalg.matrix_power( - stub_model.transition, - data[feature_keys.TrainEvalFeatures.TIMES].shape[1]), - true_params[stub_model.prior_state_mean]), - posterior_mean[0], - rtol=1e-1) - self.assertAllClose(data[feature_keys.TrainEvalFeatures.TIMES][:, -1], - posterior_times) - - -class TimeDependentStateSpaceModel(state_space_model.StateSpaceModel): - """A mostly trivial model which predicts values = times + 1.""" - - def __init__(self, static_unrolling_window_size_threshold=None): - super(TimeDependentStateSpaceModel, self).__init__( - configuration=state_space_model.StateSpaceModelConfiguration( - use_observation_noise=False, - transition_covariance_initial_log_scale_bias=5., - static_unrolling_window_size_threshold= - static_unrolling_window_size_threshold)) - - def get_state_transition(self): - return array_ops.ones(shape=[1, 1]) - - def get_noise_transform(self): - return array_ops.ones(shape=[1, 1]) - - def get_observation_model(self, times): - return array_ops.reshape( - tensor=math_ops.cast(times + 1, dtypes.float32), shape=[-1, 1, 1]) - - def make_priors(self): - return (ops.convert_to_tensor([1.]), ops.convert_to_tensor([[0.]])) - - -class UnknownShapeModel(TimeDependentStateSpaceModel): - - def get_observation_model(self, times): - parent_model = super(UnknownShapeModel, self).get_observation_model(times) - return array_ops.placeholder_with_default( - input=parent_model, shape=tensor_shape.unknown_shape()) - - -class TimeDependentTests(test.TestCase): - - def _time_dependency_test_template(self, model_type): - """Test that a time-dependent observation model influences predictions.""" - model = model_type() - estimator = estimators.StateSpaceRegressor( - model=model, optimizer=gradient_descent.GradientDescentOptimizer(0.1)) - values = numpy.reshape([1., 2., 3., 4.], - newshape=[1, 4, 1]) - input_fn = input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader({ - feature_keys.TrainEvalFeatures.TIMES: [[0, 1, 2, 3]], - feature_keys.TrainEvalFeatures.VALUES: values - })) - estimator.train(input_fn=input_fn, max_steps=1) - predicted_values = estimator.evaluate(input_fn=input_fn, steps=1)["mean"] - # Throw out the first value so we don't test the prior - self.assertAllEqual(values[1:], predicted_values[1:]) - - def test_undefined_shape_time_dependency(self): - self._time_dependency_test_template(UnknownShapeModel) - - def test_loop_unrolling(self): - """Tests running/restoring from a checkpoint with static unrolling.""" - model = TimeDependentStateSpaceModel( - # Unroll during training, but not evaluation - static_unrolling_window_size_threshold=2) - estimator = estimators.StateSpaceRegressor(model=model) - times = numpy.arange(100) - values = numpy.arange(100) - dataset = { - feature_keys.TrainEvalFeatures.TIMES: times, - feature_keys.TrainEvalFeatures.VALUES: values - } - train_input_fn = input_pipeline.RandomWindowInputFn( - input_pipeline.NumpyReader(dataset), batch_size=16, window_size=2) - eval_input_fn = input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader(dataset)) - estimator.train(input_fn=train_input_fn, max_steps=1) - estimator.evaluate(input_fn=eval_input_fn, steps=1) - - -class LevelOnlyModel(state_space_model.StateSpaceModel): - - def get_state_transition(self): - return linalg_ops.eye(1, dtype=self.dtype) - - def get_noise_transform(self): - return linalg_ops.eye(1, dtype=self.dtype) - - def get_observation_model(self, times): - return [1] - - -class MultivariateLevelModel( - state_space_model.StateSpaceCorrelatedFeaturesEnsemble): - - def __init__(self, configuration): - univariate_component_configuration = configuration._replace( - num_features=1) - components = [] - for feature in range(configuration.num_features): - with variable_scope.variable_scope("feature{}".format(feature)): - components.append( - LevelOnlyModel(configuration=univariate_component_configuration)) - super(MultivariateLevelModel, self).__init__( - ensemble_members=components, configuration=configuration) - - -class MultivariateTests(test.TestCase): - - def test_multivariate(self): - dtype = dtypes.float32 - num_features = 3 - covariance = numpy.eye(num_features) - # A single off-diagonal has a non-zero value in the true transition - # noise covariance. - covariance[-1, 0] = 1. - covariance[0, -1] = 1. - dataset_size = 100 - values = numpy.cumsum( - numpy.random.multivariate_normal( - mean=numpy.zeros(num_features), - cov=covariance, - size=dataset_size), - axis=0) - times = numpy.arange(dataset_size) - model = MultivariateLevelModel( - configuration=state_space_model.StateSpaceModelConfiguration( - num_features=num_features, - dtype=dtype, - use_observation_noise=False, - transition_covariance_initial_log_scale_bias=5.)) - estimator = estimators.StateSpaceRegressor( - model=model, optimizer=gradient_descent.GradientDescentOptimizer(0.1)) - data = { - feature_keys.TrainEvalFeatures.TIMES: times, - feature_keys.TrainEvalFeatures.VALUES: values - } - train_input_fn = input_pipeline.RandomWindowInputFn( - input_pipeline.NumpyReader(data), batch_size=16, window_size=16) - estimator.train(input_fn=train_input_fn, steps=1) - for component in model._ensemble_members: - # Check that input statistics propagated to component models - self.assertTrue(component._input_statistics) - - def test_ensemble_observation_noise(self): - model = MultivariateLevelModel( - configuration=state_space_model.StateSpaceModelConfiguration()) - model.initialize_graph() - outputs = model.define_loss( - features={ - feature_keys.TrainEvalFeatures.TIMES: - constant_op.constant([[1, 2]]), - feature_keys.TrainEvalFeatures.VALUES: - constant_op.constant([[[1.], [2.]]]) - }, - mode=estimator_lib.ModeKeys.TRAIN) - initializer = variables.global_variables_initializer() - with self.cached_session() as sess: - sess.run([initializer]) - outputs.loss.eval() - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/structural_ensemble.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/structural_ensemble.py deleted file mode 100644 index a7a80a8e3ef..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/structural_ensemble.py +++ /dev/null @@ -1,266 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Implements a time series model with seasonality, trends, and transients.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import level_trend -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import periodic -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import state_space_model -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import varma - -from tensorflow.python.ops import variable_scope -from tensorflow.python.util import nest - - -def _replicate_level_trend_models(multivariate_configuration, - univariate_configuration): - """Helper function to construct a multivariate level/trend component.""" - with variable_scope.variable_scope("adder"): - # Construct a level and trend model for each feature, with correlated - # transition noise. - adder_features = [] - for feature in range(multivariate_configuration.num_features): - with variable_scope.variable_scope("feature{}".format(feature)): - adder_features.append(level_trend.AdderStateSpaceModel( - configuration=univariate_configuration)) - adder_part = state_space_model.StateSpaceCorrelatedFeaturesEnsemble( - ensemble_members=adder_features, - configuration=multivariate_configuration) - return adder_part - - -class StructuralEnsemble(state_space_model.StateSpaceIndependentEnsemble): - r"""A structural state space time series model. - - In the spirit of: - - Scott, Steven L., and Hal R. Varian. "Predicting the present with bayesian - structural time series." International Journal of Mathematical Modelling and - Numerical Optimisation 5.1-2 (2014): 4-23. - - Without the spike-and-slab prior, and with point estimates of parameters - instead of sampling. - - The model includes level, trend, seasonality, and a transient moving average. - - An observation at time t is drawn according to: - observation_t = level_t + seasonality_t + moving_average_t - + observation_noise_t - level_t = level_{t-1} + trend_{t-1} + level_noise_t - trend_t = trend_{t-1} + trend_noise_t - seasonality_t = -\sum_{n=1}^{num_seasons-1} seasonality_{t-n} + - seasonality_noise_t - moving_average_t = transient_t - + \sum_{j=1}^{moving_average_order} ma_coefs_j * transient_{t - j} - - `observation_noise`, `level_noise`, `trend noise`, `seasonality_noise`, and - `transient` are (typically scalar) Gaussian random variables whose variance is - learned from data, and that variance is not time dependent in this - implementation. Level noise is optional due to its similarity with observation - noise in some cases. Seasonality is enforced by constraining a full cycle of - seasonal variables to have zero expectation, allowing seasonality to adapt - over time. The moving average coefficients `ma_coefs` are learned. - - When presented with a multivariate series (more than one "feature", here - referring to endogenous features of the series), the model is replicated - across these features (one copy per feature of each periodic component, and - one level/trend model per feature), and correlations in transition noise are - learned between these replicated components (see - StateSpaceCorrelatedFeaturesEnsemble). This is in addition to the learned - correlations in observation noise between features. While this is often the - most expressive thing to do with multiple features, it does mean that the - model grows quite quickly, creating and computing with square matrices with - each dimension equal to num_features * (sum(periodicities) + - moving_average_order + 3), meaning that some operations are approximately - cubic in this value. - """ - # TODO(allenl): Implement partial model replication/sharing for multivariate - # series (to save time/memory when the series presented can be modeled as a - # smaller number of underlying series). Likely just a modification of the - # observation model so that each feature of the series is a learned linear - # combination of the replicated models. - - def __init__(self, - periodicities, - moving_average_order, - autoregressive_order, - use_level_noise=True, - configuration=state_space_model.StateSpaceModelConfiguration()): - """Initialize the Basic Structural Time Series model. - - Args: - periodicities: Number of time steps for cyclic behavior. May be a list, in - which case one periodic component is created for each element. - moving_average_order: The number of moving average coefficients to use, - which also defines the number of steps after which transient - deviations revert to the mean defined by periodic and level/trend - components. - autoregressive_order: The number of steps back for autoregression. - use_level_noise: Whether to model the time series as having level - noise. See level_noise in the model description above. - configuration: A StateSpaceModelConfiguration object. - """ - component_model_configuration = configuration._replace( - use_observation_noise=False) - univariate_component_model_configuration = ( - component_model_configuration._replace( - num_features=1)) - - adder_part = _replicate_level_trend_models( - multivariate_configuration=component_model_configuration, - univariate_configuration=univariate_component_model_configuration) - with variable_scope.variable_scope("varma"): - varma_part = varma.VARMA( - autoregressive_order=autoregressive_order, - moving_average_order=moving_average_order, - configuration=component_model_configuration) - - cycle_parts = [] - periodicity_list = nest.flatten(periodicities) - for cycle_number, cycle_periodicity in enumerate(periodicity_list): - # For each specified periodicity, construct models for each feature with - # correlated noise. - with variable_scope.variable_scope("cycle{}".format(cycle_number)): - cycle_features = [] - for feature in range(configuration.num_features): - with variable_scope.variable_scope("feature{}".format(feature)): - cycle_features.append(periodic.CycleStateSpaceModel( - periodicity=cycle_periodicity, - configuration=univariate_component_model_configuration)) - cycle_parts.append( - state_space_model.StateSpaceCorrelatedFeaturesEnsemble( - ensemble_members=cycle_features, - configuration=component_model_configuration)) - - super(StructuralEnsemble, self).__init__( - ensemble_members=[adder_part, varma_part] + cycle_parts, - configuration=configuration) - - -# TODO(allenl): Implement a multi-resolution moving average component to -# decouple model size from the length of transient deviations. -class MultiResolutionStructuralEnsemble( - state_space_model.StateSpaceIndependentEnsemble): - """A structural ensemble modeling arbitrary periods with a fixed model size. - - See periodic.ResolutionCycleModel, which allows a fixed number of latent - values to cycle at multiple/variable resolutions, for more details on the - difference between MultiResolutionStructuralEnsemble and - StructuralEnsemble. With `cycle_num_latent_values` (controlling model size) - equal to `periodicities` (controlling the time over which these values - complete a full cycle), the models are - equivalent. MultiResolutionStructuralEnsemble allows `periodicities` to vary - while the model size remains fixed. Note that high `periodicities` without a - correspondingly high `cycle_num_latent_values` means that the modeled series - must have a relatively smooth periodic component. - - Multiple features are handled the same way as in StructuralEnsemble (one - replication per feature, with correlations learned between the replicated - models). This strategy produces a very flexible model, but means that series - with many features may be slow to train. - - Model size (the state dimension) is: - num_features * (sum(cycle_num_latent_values) - + max(moving_average_order + 1, autoregressive_order) + 2) - """ - - def __init__(self, - cycle_num_latent_values, - moving_average_order, - autoregressive_order, - periodicities, - use_level_noise=True, - configuration=state_space_model.StateSpaceModelConfiguration()): - """Initialize the multi-resolution structural ensemble. - - Args: - cycle_num_latent_values: Controls the model size and the number of latent - values cycled between (but not the periods over which they cycle). - Reducing this parameter can save significant amounts of memory, but - the tradeoff is with resolution: cycling between a smaller number of - latent values means that only smoother functions can be modeled. For - multivariate series, may either be a scalar integer (in which case it - is applied to all periodic components) or a list with length matching - `periodicities`. - moving_average_order: The number of moving average coefficients to use, - which also defines the number of steps after which transient - deviations revert to the mean defined by periodic and level/trend - components. Adds to model size. - autoregressive_order: The number of steps back for - autoregression. Learning autoregressive coefficients typically - requires more steps and a smaller step size than other components. - periodicities: Same meaning as for StructuralEnsemble: number of steps for - cyclic behavior. Floating point and Tensor values are supported. May - be a list of values, in which case one component is created for each - periodicity. If `periodicities` is a list while - `cycle_num_latent_values` is a scalar, its value is broadcast to each - periodic component. Otherwise they should be lists of the same length, - in which case they are paired. - use_level_noise: See StructuralEnsemble. - configuration: A StateSpaceModelConfiguration object. - Raises: - ValueError: If `cycle_num_latent_values` is neither a scalar nor agrees in - size with `periodicities`. - """ - component_model_configuration = configuration._replace( - use_observation_noise=False) - univariate_component_model_configuration = ( - component_model_configuration._replace( - num_features=1)) - - adder_part = _replicate_level_trend_models( - multivariate_configuration=component_model_configuration, - univariate_configuration=univariate_component_model_configuration) - with variable_scope.variable_scope("varma"): - varma_part = varma.VARMA( - autoregressive_order=autoregressive_order, - moving_average_order=moving_average_order, - configuration=component_model_configuration) - - cycle_parts = [] - if periodicities is None: - periodicities = [] - periodicity_list = nest.flatten(periodicities) - latent_values_list = nest.flatten(cycle_num_latent_values) - if len(periodicity_list) != len(latent_values_list): - if len(latent_values_list) != 1: - raise ValueError( - ("`cycle_num_latent_values` must either be a list with the same " - "size as `periodicity` or a scalar. Received length {} " - "`cycle_num_latent_values`, while `periodicities` has length {}.") - .format(len(latent_values_list), len(periodicity_list))) - latent_values_list *= len(periodicity_list) - for cycle_number, (cycle_periodicity, num_latent_values) in enumerate( - zip(periodicity_list, latent_values_list)): - with variable_scope.variable_scope("cycle{}".format(cycle_number)): - cycle_features = [] - for feature in range(configuration.num_features): - with variable_scope.variable_scope("feature{}".format(feature)): - cycle_features.append( - periodic.ResolutionCycleModel( - num_latent_values=num_latent_values, - periodicity=cycle_periodicity, - configuration=univariate_component_model_configuration)) - cycle_parts.append( - state_space_model.StateSpaceCorrelatedFeaturesEnsemble( - ensemble_members=cycle_features, - configuration=component_model_configuration)) - - super(MultiResolutionStructuralEnsemble, self).__init__( - ensemble_members=[adder_part, varma_part] + cycle_parts, - configuration=configuration) diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/structural_ensemble_test.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/structural_ensemble_test.py deleted file mode 100644 index a5f6a5341e8..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/structural_ensemble_test.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the structural state space ensembles.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy - -from tensorflow.contrib import layers -from tensorflow.contrib.layers.python.layers import feature_column - -from tensorflow.contrib.timeseries.python.timeseries import estimators -from tensorflow.contrib.timeseries.python.timeseries import input_pipeline -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import TrainEvalFeatures -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import state_space_model -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import structural_ensemble - -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.framework import dtypes -from tensorflow.python.platform import test - - -class StructuralEnsembleEstimatorTests(test.TestCase): - - def simple_data(self, sample_every, dtype, period, num_samples, num_features): - time = sample_every * numpy.arange(num_samples) - noise = numpy.random.normal( - scale=0.01, size=[num_samples, num_features]) - values = noise + numpy.sin( - numpy.arange(num_features)[None, ...] - + time[..., None] / float(period) * 2.0 * numpy.pi).astype( - dtype.as_numpy_dtype) - return {TrainEvalFeatures.TIMES: numpy.reshape(time, [1, -1]), - TrainEvalFeatures.VALUES: numpy.reshape( - values, [1, -1, num_features])} - - def dry_run_train_helper( - self, sample_every, period, num_samples, model_type, model_args, - num_features=1): - numpy.random.seed(1) - dtype = dtypes.float32 - features = self.simple_data( - sample_every, dtype=dtype, period=period, num_samples=num_samples, - num_features=num_features) - model = model_type( - configuration=( - state_space_model.StateSpaceModelConfiguration( - num_features=num_features, - dtype=dtype, - covariance_prior_fn=lambda _: 0.)), - **model_args) - - class _RunConfig(estimator_lib.RunConfig): - - @property - def tf_random_seed(self): - return 4 - - estimator = estimators.StateSpaceRegressor(model, config=_RunConfig()) - train_input_fn = input_pipeline.RandomWindowInputFn( - input_pipeline.NumpyReader(features), num_threads=1, shuffle_seed=1, - batch_size=16, window_size=16) - eval_input_fn = input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader(features)) - estimator.train(input_fn=train_input_fn, max_steps=1) - first_evaluation = estimator.evaluate(input_fn=eval_input_fn, steps=1) - estimator.train(input_fn=train_input_fn, max_steps=3) - second_evaluation = estimator.evaluate(input_fn=eval_input_fn, steps=1) - self.assertLess(second_evaluation["loss"], first_evaluation["loss"]) - - def test_structural_multivariate(self): - self.dry_run_train_helper( - sample_every=3, - period=5, - num_samples=100, - num_features=3, - model_type=structural_ensemble.StructuralEnsemble, - model_args={ - "periodicities": 2, - "moving_average_order": 2, - "autoregressive_order": 1 - }) - - def test_exogenous_input(self): - """Test that no errors are raised when using exogenous features.""" - dtype = dtypes.float64 - times = [1, 2, 3, 4, 5, 6] - values = [[0.01], [5.10], [5.21], [0.30], [5.41], [0.50]] - feature_a = [["off"], ["on"], ["on"], ["off"], ["on"], ["off"]] - sparse_column_a = feature_column.sparse_column_with_keys( - column_name="feature_a", keys=["on", "off"]) - one_hot_a = layers.one_hot_column(sparse_id_column=sparse_column_a) - regressor = estimators.StructuralEnsembleRegressor( - periodicities=[], - num_features=1, - moving_average_order=0, - exogenous_feature_columns=[one_hot_a], - dtype=dtype) - features = {TrainEvalFeatures.TIMES: times, - TrainEvalFeatures.VALUES: values, - "feature_a": feature_a} - train_input_fn = input_pipeline.RandomWindowInputFn( - input_pipeline.NumpyReader(features), - window_size=6, batch_size=1) - regressor.train(input_fn=train_input_fn, steps=1) - eval_input_fn = input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader(features)) - evaluation = regressor.evaluate(input_fn=eval_input_fn, steps=1) - predict_input_fn = input_pipeline.predict_continuation_input_fn( - evaluation, times=[[7, 8, 9]], - exogenous_features={"feature_a": [[["on"], ["off"], ["on"]]]}) - regressor.predict(input_fn=predict_input_fn) - - def test_no_periodicity(self): - """Test that no errors are raised when periodicites is None.""" - dtype = dtypes.float64 - times = [1, 2, 3, 4, 5, 6] - values = [[0.01], [5.10], [5.21], [0.30], [5.41], [0.50]] - regressor = estimators.StructuralEnsembleRegressor( - periodicities=None, - num_features=1, - moving_average_order=0, - dtype=dtype) - features = {TrainEvalFeatures.TIMES: times, - TrainEvalFeatures.VALUES: values} - train_input_fn = input_pipeline.RandomWindowInputFn( - input_pipeline.NumpyReader(features), - window_size=6, batch_size=1) - regressor.train(input_fn=train_input_fn, steps=1) - eval_input_fn = input_pipeline.WholeDatasetInputFn( - input_pipeline.NumpyReader(features)) - evaluation = regressor.evaluate(input_fn=eval_input_fn, steps=1) - predict_input_fn = input_pipeline.predict_continuation_input_fn( - evaluation, times=[[7, 8, 9]]) - regressor.predict(input_fn=predict_input_fn) - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/test_utils.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/test_utils.py deleted file mode 100644 index f7f5024b342..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/test_utils.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities for testing state space models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy - -from tensorflow.contrib.timeseries.python.timeseries import math_utils - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops - - -def transition_power_test_template(test_case, model, num_steps): - """Tests the transition_to_powers function of a state space model.""" - transition_matrix = ops.convert_to_tensor( - model.get_state_transition(), dtype=model.dtype) - step_number = array_ops.placeholder(shape=[], dtype=dtypes.int64) - state_dimension = tensor_shape.dimension_value(transition_matrix.shape[0]) - previous_matrix = array_ops.placeholder( - shape=[state_dimension, state_dimension], dtype=transition_matrix.dtype) - true_single_step_update = math_ops.matmul(previous_matrix, - transition_matrix) - model_output_tensor = model.transition_to_powers(powers=array_ops.stack( - [step_number, step_number])) - with test_case.test_session(): - starting_matrix = linalg_ops.eye( - state_dimension, batch_shape=array_ops.shape(num_steps)).eval() - evaled_current_matrix = starting_matrix - for iteration_number in range(num_steps): - model_output = model_output_tensor.eval( - feed_dict={step_number: iteration_number}) - test_case.assertAllClose( - evaled_current_matrix, - model_output[0], - rtol=1e-8 if evaled_current_matrix.dtype == numpy.float64 else 1e-4) - evaled_current_matrix = true_single_step_update.eval( - feed_dict={previous_matrix: evaled_current_matrix}) - - -def noise_accumulator_test_template(test_case, model, num_steps): - """Tests `model`'s transition_power_noise_accumulator.""" - transition_matrix = ops.convert_to_tensor( - model.get_state_transition(), dtype=model.dtype) - noise_transform = ops.convert_to_tensor( - model.get_noise_transform(), dtype=model.dtype) - state_dimension = tensor_shape.dimension_value(transition_matrix.shape[0]) - state_noise_dimension = tensor_shape.dimension_value(noise_transform.shape[1]) - gen_noise_addition = math_utils.sign_magnitude_positive_definite( - raw=random_ops.random_normal( - shape=[state_noise_dimension, state_noise_dimension], - dtype=model.dtype)) - gen_starting_noise = math_utils.sign_magnitude_positive_definite( - random_ops.random_normal( - shape=[state_dimension, state_dimension], dtype=model.dtype)) - starting_noise = array_ops.placeholder( - shape=[state_dimension, state_dimension], dtype=model.dtype) - step_number = array_ops.placeholder(shape=[], dtype=dtypes.int64) - starting_transitioned = math_ops.matmul( - math_ops.matmul(transition_matrix, starting_noise), - transition_matrix, - adjoint_b=True) - with test_case.test_session(): - evaled_starting_noise = gen_starting_noise.eval() - current_starting_noise_transitioned = evaled_starting_noise - current_noise = evaled_starting_noise - evaled_noise_addition = gen_noise_addition.eval() - evaled_noise_addition_transformed = math_ops.matmul( - math_ops.matmul(noise_transform, evaled_noise_addition), - noise_transform, - adjoint_b=True).eval() - model.state_transition_noise_covariance = evaled_noise_addition - model._window_initializer( # pylint: disable=protected-access - times=math_ops.range(num_steps + 1)[..., None], state=(None, None, 0)) - model_update = model.transition_power_noise_accumulator( - num_steps=step_number) - for iteration_number in range(num_steps): - model_new_noise = model_update.eval( - feed_dict={step_number: iteration_number}) - test_case.assertAllClose( - current_noise, - model_new_noise + current_starting_noise_transitioned, - rtol=1e-8 if current_noise.dtype == numpy.float64 else 1e-3) - current_starting_noise_transitioned = starting_transitioned.eval( - feed_dict={starting_noise: current_starting_noise_transitioned}) - current_noise = ( - starting_transitioned.eval( - feed_dict={starting_noise: current_noise}) - + evaled_noise_addition_transformed) diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py deleted file mode 100644 index ddee749b498..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -r"""Multivariate autoregressive model (vector autoregression). - -Implements the following model (num_blocks = max(ar_order, ma_order + 1)): - - y(t, 1) = \sum_{i=1}^{ar_order} ar_coefs[i] * y(t - 1, i) - y(t, i) = y(t - 1, i - 1) + ma_coefs[i - 1] * e(t) for 1 < i < num_blocks - y(t, num_blocks) = y(t - 1, num_blocks - 1) + e(t) - -Where e(t) are Gaussian with zero mean and learned covariance. - -Each element of ar_coefs and ma_coefs is a [num_features x num_features] -matrix. Each y(t, i) is a vector of length num_features. Indices in the above -equations are one-based. Initial conditions y(0, i) come from prior state (which -may either be learned or left as a constant with high prior covariance). - -If ar_order > ma_order, the observation model is: - y(t, 1) + observation_noise(t) - -If ma_order >= ar_order, it is (to observe the moving average component): - y(t, 1) + y(t, num_blocks) + observation_noise(t) - -Where observation_noise(t) are Gaussian with zero mean and learned covariance. - -This implementation uses a formulation which puts all of the autoregressive -coefficients in the transition equation for the observed component, which -enables learning using truncated backpropagation. Noise is not applied directly -to the observed component (with the exception of standard observation noise), -which further aids learning of the autoregressive coefficients when VARMA is in -an ensemble with other models (in which case having an observation noise term is -usually unavoidable). -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.python.timeseries import math_utils -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import state_space_model - -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variable_scope - - -class VARMA(state_space_model.StateSpaceModel): - """A VARMA model implementation as a special case of the state space model.""" - - def __init__(self, - autoregressive_order, - moving_average_order, - configuration=state_space_model.StateSpaceModelConfiguration()): - """Construct a VARMA model. - - The size of the latent state for this model is: - num_features * max(autoregressive_order, moving_average_order + 1) - Square matrices of this size are constructed and multiplied. - - Args: - autoregressive_order: The maximum autoregressive lag. - moving_average_order: The maximum moving average lag, after which - transient deviations are expected to return to their long-term mean. - configuration: A StateSpaceModelConfiguration object. - """ - self.ar_order = autoregressive_order - self.ma_order = moving_average_order - self.state_num_blocks = max(autoregressive_order, moving_average_order + 1) - super(VARMA, self).__init__(configuration=configuration) - self.state_dimension = self.state_num_blocks * self.num_features - - def _define_parameters(self, observation_transition_tradeoff_log=None): - with variable_scope.variable_scope(self._variable_scope): - # TODO(allenl): Evaluate parameter transformations for AR/MA coefficients - # which improve interpretability/stability. - self.ar_coefs = variable_scope.get_variable( - name="ar_coefs", - shape=[self.num_features, self.num_features, self.ar_order], - dtype=self.dtype, - initializer=init_ops.zeros_initializer()) - self.ma_coefs = variable_scope.get_variable( - name="ma_coefs", - initializer=array_ops.tile( - linalg_ops.eye(self.num_features, dtype=self.dtype)[None, :, :], - [self.ma_order, 1, 1]), - dtype=self.dtype) - super(VARMA, self)._define_parameters( - observation_transition_tradeoff_log=observation_transition_tradeoff_log) - - def get_state_transition(self): - """Construct state transition matrix from VARMA parameters. - - Returns: - the state transition matrix. It has shape - [self.state_dimension, self.state_dimension]. - """ - # Pad any unused AR blocks with zeros. The extra state is necessary if - # ma_order >= ar_order. - ar_coefs_padded = array_ops.reshape( - array_ops.pad(self.ar_coefs, - [[0, 0], [0, 0], - [0, self.state_num_blocks - self.ar_order]]), - [self.num_features, self.state_dimension]) - shift_matrix = array_ops.pad( - linalg_ops.eye( - (self.state_num_blocks - 1) * self.num_features, dtype=self.dtype), - [[0, 0], [0, self.num_features]]) - return array_ops.concat([ar_coefs_padded, shift_matrix], axis=0) - - def get_noise_transform(self): - """Construct state noise transform matrix from VARMA parameters. - - Returns: - the state noise transform matrix. It has shape - [self.state_dimension, self.num_features]. - """ - # Noise is broadcast, through the moving average coefficients, to - # un-observed parts of the latent state. - ma_coefs_padded = array_ops.reshape( - array_ops.pad(self.ma_coefs, - [[self.state_num_blocks - 1 - self.ma_order, 0], [0, 0], - [0, 0]]), - [(self.state_num_blocks - 1) * self.num_features, self.num_features], - name="noise_transform") - # Deterministically apply noise to the oldest component. - return array_ops.concat( - [ma_coefs_padded, - linalg_ops.eye(self.num_features, dtype=self.dtype)], - axis=0) - - def get_observation_model(self, times): - """Construct observation model matrix from VARMA parameters. - - Args: - times: A [batch size] vector indicating the times observation models are - requested for. Unused. - Returns: - the observation model matrix. It has shape - [self.num_features, self.state_dimension]. - """ - del times # StateSpaceModel will broadcast along the batch dimension - if self.ar_order > self.ma_order or self.state_num_blocks < 2: - return array_ops.pad( - linalg_ops.eye(self.num_features, dtype=self.dtype), - [[0, 0], [0, self.num_features * (self.state_num_blocks - 1)]], - name="observation_model") - else: - # Add a second observed component which "catches" the accumulated moving - # average errors as they reach the end of the state. If ar_order > - # ma_order, this is unnecessary, since accumulated errors cycle naturally. - return array_ops.concat( - [ - array_ops.pad( - linalg_ops.eye(self.num_features, dtype=self.dtype), - [[0, 0], [0, - self.num_features * (self.state_num_blocks - 2)]]), - linalg_ops.eye(self.num_features, dtype=self.dtype) - ], - axis=1, - name="observation_model") - - def get_state_transition_noise_covariance( - self, minimum_initial_variance=1e-5): - # Most state space models use only an explicit observation noise term to - # model deviations from expectations, and so a low initial transition noise - # parameter is helpful there. Since deviations from expectations are also - # modeled as transition noise in VARMA, we set its initial value based on a - # slight over-estimate empirical observation noise. - if self._input_statistics is not None: - feature_variance = self._scale_variance( - self._input_statistics.series_start_moments.variance) - initial_transition_noise_scale = math_ops.log( - math_ops.maximum( - math_ops.reduce_mean(feature_variance), minimum_initial_variance)) - else: - initial_transition_noise_scale = 0. - state_noise_transform = ops.convert_to_tensor( - self.get_noise_transform(), dtype=self.dtype) - state_noise_dimension = tensor_shape.dimension_value( - state_noise_transform.shape[1]) - return math_utils.variable_covariance_matrix( - state_noise_dimension, "state_transition_noise", - dtype=self.dtype, - initial_overall_scale_log=initial_transition_noise_scale) diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma_test.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma_test.py deleted file mode 100644 index e8875f4eb9f..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma_test.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for VARMA. - -Tests VARMA model building and utility functions. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import TrainEvalFeatures -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import state_space_model -from tensorflow.contrib.timeseries.python.timeseries.state_space_models import varma - -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class MakeModelTest(test.TestCase): - - def test_ar_smaller(self): - model = varma.VARMA( - autoregressive_order=0, - moving_average_order=3) - model.initialize_graph() - outputs = model.define_loss( - features={ - TrainEvalFeatures.TIMES: constant_op.constant([[1, 2]]), - TrainEvalFeatures.VALUES: constant_op.constant([[[1.], [2.]]]) - }, - mode=estimator_lib.ModeKeys.TRAIN) - initializer = variables.global_variables_initializer() - with self.cached_session() as sess: - sess.run([initializer]) - outputs.loss.eval() - - def test_ma_smaller(self): - model = varma.VARMA( - autoregressive_order=6, - moving_average_order=3, - configuration=state_space_model.StateSpaceModelConfiguration( - num_features=7)) - model.initialize_graph() - outputs = model.define_loss( - features={ - TrainEvalFeatures.TIMES: constant_op.constant([[1, 2]]), - TrainEvalFeatures.VALUES: constant_op.constant( - [[[1.] * 7, [2.] * 7]]) - }, - mode=estimator_lib.ModeKeys.TRAIN) - initializer = variables.global_variables_initializer() - with self.cached_session() as sess: - sess.run([initializer]) - outputs.loss.eval() - - def test_make_ensemble_no_errors(self): - with variable_scope.variable_scope("model_one"): - model_one = varma.VARMA(10, 5) - with variable_scope.variable_scope("model_two"): - model_two = varma.VARMA(0, 3) - configuration = state_space_model.StateSpaceModelConfiguration() - ensemble = state_space_model.StateSpaceIndependentEnsemble( - ensemble_members=[model_one, model_two], - configuration=configuration) - ensemble.initialize_graph() - outputs = ensemble.define_loss( - features={ - TrainEvalFeatures.TIMES: constant_op.constant([[1, 2]]), - TrainEvalFeatures.VALUES: constant_op.constant([[[1.], [2.]]])}, - mode=estimator_lib.ModeKeys.TRAIN) - initializer = variables.global_variables_initializer() - with self.cached_session() as sess: - sess.run([initializer]) - outputs.loss.eval() - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/timeseries/python/timeseries/test_utils.py b/tensorflow/contrib/timeseries/python/timeseries/test_utils.py deleted file mode 100644 index ed02960faef..00000000000 --- a/tensorflow/contrib/timeseries/python/timeseries/test_utils.py +++ /dev/null @@ -1,282 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities for testing time series models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.timeseries.python.timeseries import estimators -from tensorflow.contrib.timeseries.python.timeseries import input_pipeline -from tensorflow.contrib.timeseries.python.timeseries import state_management -from tensorflow.contrib.timeseries.python.timeseries.feature_keys import TrainEvalFeatures - -from tensorflow.python.client import session -from tensorflow.python.estimator import estimator_lib -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import adam -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.training import coordinator as coordinator_lib -from tensorflow.python.training import queue_runner_impl -from tensorflow.python.util import nest - - -class AllWindowInputFn(input_pipeline.TimeSeriesInputFn): - """Returns all contiguous windows of data from a full dataset. - - In contrast to WholeDatasetInputFn, which does basic shape checking but - maintains the flat sequencing of data, this `TimeSeriesInputFn` creates - batches of windows. However, unlike `RandomWindowInputFn` these windows are - deterministic, starting at every possible offset (i.e. batches of size - series_length - window_size + 1 are produced). - """ - - def __init__(self, time_series_reader, window_size): - """Initialize the input_pipeline. - - Args: - time_series_reader: A `input_pipeline.TimeSeriesReader` object. - window_size: The size of contiguous windows of data to produce. - """ - self._window_size = window_size - self._reader = time_series_reader - super(AllWindowInputFn, self).__init__() - - def create_batch(self): - features = self._reader.read_full() - times = features[TrainEvalFeatures.TIMES] - num_windows = array_ops.shape(times)[0] - self._window_size + 1 - indices = array_ops.reshape(math_ops.range(num_windows), [num_windows, 1]) - # indices contains the starting point for each window. We now extend these - # indices to include the elements inside the windows as well by doing a - # broadcast addition. - increments = array_ops.reshape(math_ops.range(self._window_size), [1, -1]) - all_indices = array_ops.reshape(indices + increments, [-1]) - # Select the appropriate elements in the batch and reshape the output to 3D. - features = { - key: array_ops.reshape( - array_ops.gather(value, all_indices), - array_ops.concat( - [[num_windows, self._window_size], array_ops.shape(value)[1:]], - axis=0)) - for key, value in features.items() - } - return (features, None) - - -class _SavingTensorHook(basic_session_run_hooks.LoggingTensorHook): - """A hook to save Tensors during training.""" - - def __init__(self, tensors, every_n_iter=None, every_n_secs=None): - self.tensor_values = {} - super(_SavingTensorHook, self).__init__( - tensors=tensors, every_n_iter=every_n_iter, - every_n_secs=every_n_secs) - - def after_run(self, run_context, run_values): - del run_context - if self._should_trigger: - for tag in self._current_tensors.keys(): - self.tensor_values[tag] = run_values.results[tag] - self._timer.update_last_triggered_step(self._iter_count) - self._iter_count += 1 - - -def _train_on_generated_data( - generate_fn, generative_model, train_iterations, seed, - learning_rate=0.1, ignore_params_fn=lambda _: (), - derived_param_test_fn=lambda _: (), - train_input_fn_type=input_pipeline.WholeDatasetInputFn, - train_state_manager=state_management.PassthroughStateManager()): - """The training portion of parameter recovery tests.""" - random_seed.set_random_seed(seed) - generate_graph = ops.Graph() - with generate_graph.as_default(): - with session.Session(graph=generate_graph): - generative_model.initialize_graph() - time_series_reader, true_parameters = generate_fn(generative_model) - true_parameters = { - tensor.name: value for tensor, value in true_parameters.items()} - eval_input_fn = input_pipeline.WholeDatasetInputFn(time_series_reader) - eval_state_manager = state_management.PassthroughStateManager() - true_parameter_eval_graph = ops.Graph() - with true_parameter_eval_graph.as_default(): - generative_model.initialize_graph() - ignore_params = ignore_params_fn(generative_model) - feature_dict, _ = eval_input_fn() - eval_state_manager.initialize_graph(generative_model) - feature_dict[TrainEvalFeatures.VALUES] = math_ops.cast( - feature_dict[TrainEvalFeatures.VALUES], generative_model.dtype) - model_outputs = eval_state_manager.define_loss( - model=generative_model, - features=feature_dict, - mode=estimator_lib.ModeKeys.EVAL) - with session.Session(graph=true_parameter_eval_graph) as sess: - variables.global_variables_initializer().run() - coordinator = coordinator_lib.Coordinator() - queue_runner_impl.start_queue_runners(sess, coord=coordinator) - true_param_loss = model_outputs.loss.eval(feed_dict=true_parameters) - true_transformed_params = { - param: param.eval(feed_dict=true_parameters) - for param in derived_param_test_fn(generative_model)} - coordinator.request_stop() - coordinator.join() - - saving_hook = _SavingTensorHook( - tensors=true_parameters.keys(), - every_n_iter=train_iterations - 1) - - class _RunConfig(estimator_lib.RunConfig): - - @property - def tf_random_seed(self): - return seed - - estimator = estimators.TimeSeriesRegressor( - model=generative_model, - config=_RunConfig(), - state_manager=train_state_manager, - optimizer=adam.AdamOptimizer(learning_rate)) - train_input_fn = train_input_fn_type(time_series_reader=time_series_reader) - trained_loss = (estimator.train( - input_fn=train_input_fn, - max_steps=train_iterations, - hooks=[saving_hook]).evaluate( - input_fn=eval_input_fn, steps=1))["loss"] - logging.info("Final trained loss: %f", trained_loss) - logging.info("True parameter loss: %f", true_param_loss) - return (ignore_params, true_parameters, true_transformed_params, - trained_loss, true_param_loss, saving_hook, - true_parameter_eval_graph) - - -def test_parameter_recovery( - generate_fn, generative_model, train_iterations, test_case, seed, - learning_rate=0.1, rtol=0.2, atol=0.1, train_loss_tolerance_coeff=0.99, - ignore_params_fn=lambda _: (), - derived_param_test_fn=lambda _: (), - train_input_fn_type=input_pipeline.WholeDatasetInputFn, - train_state_manager=state_management.PassthroughStateManager()): - """Test that a generative model fits generated data. - - Args: - generate_fn: A function taking a model and returning a `TimeSeriesReader` - object and dictionary mapping parameters to their - values. model.initialize_graph() will have been called on the model - before it is passed to this function. - generative_model: A timeseries.model.TimeSeriesModel instance to test. - train_iterations: Number of training steps. - test_case: A tf.test.TestCase to run assertions on. - seed: Same as for TimeSeriesModel.unconditional_generate(). - learning_rate: Step size for optimization. - rtol: Relative tolerance for tests. - atol: Absolute tolerance for tests. - train_loss_tolerance_coeff: Trained loss times this value must be less - than the loss evaluated using the generated parameters. - ignore_params_fn: Function mapping from a Model to a list of parameters - which are not tested for accurate recovery. - derived_param_test_fn: Function returning a list of derived parameters - (Tensors) which are checked for accurate recovery (comparing the value - evaluated with trained parameters to the value under the true - parameters). - - As an example, for VARMA, in addition to checking AR and MA parameters, - this function can be used to also check lagged covariance. See - varma_ssm.py for details. - train_input_fn_type: The `TimeSeriesInputFn` type to use when training - (likely `WholeDatasetInputFn` or `RandomWindowInputFn`). If None, use - `WholeDatasetInputFn`. - train_state_manager: The state manager to use when training (likely - `PassthroughStateManager` or `ChainingStateManager`). If None, use - `PassthroughStateManager`. - """ - (ignore_params, true_parameters, true_transformed_params, - trained_loss, true_param_loss, saving_hook, true_parameter_eval_graph - ) = _train_on_generated_data( - generate_fn=generate_fn, generative_model=generative_model, - train_iterations=train_iterations, seed=seed, learning_rate=learning_rate, - ignore_params_fn=ignore_params_fn, - derived_param_test_fn=derived_param_test_fn, - train_input_fn_type=train_input_fn_type, - train_state_manager=train_state_manager) - trained_parameter_substitutions = {} - for param in true_parameters.keys(): - evaled_value = saving_hook.tensor_values[param] - trained_parameter_substitutions[param] = evaled_value - true_value = true_parameters[param] - logging.info("True %s: %s, learned: %s", - param, true_value, evaled_value) - with session.Session(graph=true_parameter_eval_graph): - for transformed_param, true_value in true_transformed_params.items(): - trained_value = transformed_param.eval( - feed_dict=trained_parameter_substitutions) - logging.info("True %s [transformed parameter]: %s, learned: %s", - transformed_param, true_value, trained_value) - test_case.assertAllClose(true_value, trained_value, - rtol=rtol, atol=atol) - - if ignore_params is None: - ignore_params = [] - else: - ignore_params = nest.flatten(ignore_params) - ignore_params = [tensor.name for tensor in ignore_params] - if trained_loss > 0: - test_case.assertLess(trained_loss * train_loss_tolerance_coeff, - true_param_loss) - else: - test_case.assertLess(trained_loss / train_loss_tolerance_coeff, - true_param_loss) - for param in true_parameters.keys(): - if param in ignore_params: - continue - evaled_value = saving_hook.tensor_values[param] - true_value = true_parameters[param] - test_case.assertAllClose(true_value, evaled_value, - rtol=rtol, atol=atol) - - -def parameter_recovery_dry_run( - generate_fn, generative_model, seed, - learning_rate=0.1, - train_input_fn_type=input_pipeline.WholeDatasetInputFn, - train_state_manager=state_management.PassthroughStateManager()): - """Test that a generative model can train on generated data. - - Args: - generate_fn: A function taking a model and returning a - `input_pipeline.TimeSeriesReader` object and a dictionary mapping - parameters to their values. model.initialize_graph() will have been - called on the model before it is passed to this function. - generative_model: A timeseries.model.TimeSeriesModel instance to test. - seed: Same as for TimeSeriesModel.unconditional_generate(). - learning_rate: Step size for optimization. - train_input_fn_type: The type of `TimeSeriesInputFn` to use when training - (likely `WholeDatasetInputFn` or `RandomWindowInputFn`). If None, use - `WholeDatasetInputFn`. - train_state_manager: The state manager to use when training (likely - `PassthroughStateManager` or `ChainingStateManager`). If None, use - `PassthroughStateManager`. - """ - _train_on_generated_data( - generate_fn=generate_fn, generative_model=generative_model, - seed=seed, learning_rate=learning_rate, - train_input_fn_type=train_input_fn_type, - train_state_manager=train_state_manager, - train_iterations=2) diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD deleted file mode 100644 index 206aa7426ae..00000000000 --- a/tensorflow/contrib/tpu/BUILD +++ /dev/null @@ -1,199 +0,0 @@ -# Description: Operations defined for Cloud TPUs - -load( - "//tensorflow:tensorflow.bzl", - "tf_custom_op_library", - "tf_gen_op_libs", - "tf_gen_op_wrapper_py", - "tf_py_test", -) -load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") - -package( - default_visibility = [ - "//tensorflow:__subpackages__", - "//tensorflow_models:__subpackages__", - "//vr/perception:__subpackages__", - ], - licenses = ["notice"], # Apache 2.0 -) - -py_library( - name = "tpu_py", - srcs = ["python/ops/tpu_ops.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/tpu:tpu_py", - ], -) - -py_library( - name = "async_checkpoint", - srcs = ["python/tpu/async_checkpoint.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/tpu:async_checkpoint", - ], -) - -py_library( - name = "tpu_estimator", - srcs = [ - "python/tpu/_tpu_estimator_embedding.py", - "python/tpu/error_handling.py", - "python/tpu/tpu_config.py", - "python/tpu/tpu_context.py", - "python/tpu/tpu_estimator.py", - "python/tpu/util.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":async_checkpoint", - ":feature_column", - ":functional", - ":tpu_embedding", - ":tpu_lib", - "//tensorflow/contrib/training:training_py", - "//tensorflow/python/tpu:tpu_estimator", - ], -) - -py_library( - name = "functional", - srcs = ["python/tpu/functional.py"], - srcs_version = "PY2AND3", - visibility = [ - "//visibility:public", - ], - deps = [ - "//tensorflow/python/tpu:functional", - ], -) - -py_library( - name = "profiler", - srcs = ["python/profiler/__init__.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/tpu/profiler", - ], -) - -py_library( - name = "tpu", - srcs = [ - "__init__.py", - "python/tpu/__init__.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":feature_column", - ":keras_support", # split out to avoid cycle with tpu_strategy - ":tpu_embedding", - ":tpu_estimator", - ":tpu_lib", - "//tensorflow/python/tpu", - ], -) - -py_library( - name = "keras_support", - srcs = [ - "python/tpu/keras_support.py", - "python/tpu/keras_tpu_variables.py", - ], - srcs_version = "PY2AND3", - visibility = [ - "//cloud/vmm/testing/tests/tpu:__subpackages__", - "//learning/brain:__subpackages__", - "//tensorflow:__subpackages__", - "//third_party/cloud_tpu/models/keras_colab:__subpackages__", - "//third_party/cloud_tpu/models/resnet50_keras:__subpackages__", - ], - deps = [ - ":tpu_lib", - "//tensorflow/contrib/cluster_resolver:cluster_resolver_py", - "//tensorflow/contrib/distribute", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/core/protobuf/tpu:compilation_result_proto_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:session", - "//tensorflow/python:tensor_spec", - "//tensorflow/python:variable_scope", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/estimator:estimator_py", - "//tensorflow/python/keras:backend", - "//tensorflow/python/keras:engine", - "//tensorflow/python/keras:layers", - "//third_party/py/numpy", - ], -) - -py_library( - name = "tpu_lib", - srcs = [ - "python/tpu/__init__.py", - "python/tpu/bfloat16.py", - "python/tpu/device_assignment.py", - "python/tpu/session_support.py", - "python/tpu/tensor_tracer.py", - "python/tpu/topology.py", - "python/tpu/tpu.py", - "python/tpu/tpu_feed.py", - "python/tpu/tpu_function.py", - "python/tpu/tpu_optimizer.py", - "python/tpu/tpu_sharding.py", - "python/tpu/tpu_system_metadata.py", - "python/tpu/training_loop.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":datasets", - ":functional", - ":profiler", - ":tpu_py", - "//tensorflow/contrib/cluster_resolver:cluster_resolver_py", - "//tensorflow/contrib/compiler:xla", - "//tensorflow/python/tpu:tpu_lib", - ], -) - -py_library( - name = "datasets", - srcs = [ - "python/tpu/datasets.py", - ], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python/tpu:datasets", - ], -) - -py_library( - name = "tpu_embedding", - srcs = [ - "python/tpu/tpu_embedding.py", - "python/tpu/tpu_embedding_gradient.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":tpu_lib", - "//tensorflow/python/tpu:tpu_embedding", - ], -) - -py_library( - name = "feature_column", - srcs = ["python/tpu/feature_column.py"], - deps = [ - ":tpu_lib", - "//tensorflow/python/tpu:feature_column", - ], -) diff --git a/tensorflow/contrib/tpu/README.md b/tensorflow/contrib/tpu/README.md deleted file mode 100644 index 69fb11e344f..00000000000 --- a/tensorflow/contrib/tpu/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# TPU support for TensorFlow # - -This directory contains code required to re-target a TensorFlow model to run -on TPUs. - -## Example usage - TPU Estimator - -Below shows example usage of the TPU Estimator for a simple convolutional -network. - -```python -import tensorflow as tf - -from tensorflow.contrib.tpu.python.tpu import tpu_config -from tensorflow.contrib.tpu.python.tpu import tpu_estimator -from tensorflow.contrib.tpu.python.tpu import tpu_optimizer - -def model_fn(features, labels, mode, params): - # Define the model to construct the logits - logits = # ... - loss = tf.losses.softmax_cross_entropy(onehot_labels=labels, logits=logits) - optimizer = tpu_optimizer.CrossShardOptimizer( - tf.train.GradientDescentOptimizer(learning_rate=FLAGS.learning_rate)) - train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step()) - return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) - -def input_fn(params): - # ... - pass - -def main(): - run_config = tpu_config.RunConfig( - master=FLAGS.master, - # ... - ) - estimator = tpu_estimator.TpuEstimator( - model_fn=model_fn, - use_tpu=FLAGS.use_tpu, - config=run_config, - batch_size=FLAGS.batch_size) - estimator.train(input_fn=input_fn, max_steps=FLAGS.train_steps) -``` - -For the complete [executable] example, see our open source TPU models. diff --git a/tensorflow/contrib/tpu/__init__.py b/tensorflow/contrib/tpu/__init__.py deleted file mode 100644 index 6ce6b779a2d..00000000000 --- a/tensorflow/contrib/tpu/__init__.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Ops related to Tensor Processing Units. - -@@cross_replica_sum -@@infeed_dequeue -@@infeed_dequeue_tuple -@@infeed_enqueue -@@infeed_enqueue_tuple -@@outfeed_dequeue -@@outfeed_dequeue_tuple -@@outfeed_enqueue -@@outfeed_enqueue_tuple - -@@initialize_system -@@shutdown_system -@@device_assignment -@@core -@@replicate -@@shard -@@batch_parallel -@@rewrite -@@outside_compilation - -@@CrossShardOptimizer - -@@InfeedQueue - -@@DeviceAssignment -@@Topology - -@@while_loop -@@repeat - -@@TPUEstimator -@@TPUEstimatorSpec -@@export_estimator_savedmodel -@@RunConfig -@@InputPipelineConfig -@@TPUConfig -@@bfloat16_scope - -@@TPUDistributionStrategy -@@keras_to_tpu_model - -@@AsyncCheckpointSaverHook -@@TPUInMemoryEvalHook -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.contrib.tpu.python import profiler -from tensorflow.contrib.tpu.python.ops.tpu_ops import * -from tensorflow.contrib.tpu.python.tpu.async_checkpoint import * -from tensorflow.contrib.tpu.python.tpu.bfloat16 import * -from tensorflow.contrib.tpu.python.tpu.device_assignment import * -from tensorflow.contrib.tpu.python.tpu.keras_support import tpu_model as keras_to_tpu_model -from tensorflow.contrib.tpu.python.tpu.keras_support import TPUDistributionStrategy -from tensorflow.contrib.tpu.python.tpu.topology import * -from tensorflow.contrib.tpu.python.tpu.tpu import * -from tensorflow.contrib.tpu.python.tpu.tpu_config import * -from tensorflow.contrib.tpu.python.tpu.tpu_estimator import * -from tensorflow.contrib.tpu.python.tpu.tpu_feed import InfeedQueue -from tensorflow.contrib.tpu.python.tpu.tpu_optimizer import * -from tensorflow.contrib.tpu.python.tpu.training_loop import * -# pylint: enable=wildcard-import,unused-import - -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = ['profiler'] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/tpu/python/ops/tpu_ops.py b/tensorflow/contrib/tpu/python/ops/tpu_ops.py deleted file mode 100644 index 8605bae5c12..00000000000 --- a/tensorflow/contrib/tpu/python/ops/tpu_ops.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.ops.tpu_ops import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/ops/tpu_ordinal_selector_op.py b/tensorflow/contrib/tpu/python/ops/tpu_ordinal_selector_op.py deleted file mode 100644 index 788e1fe0568..00000000000 --- a/tensorflow/contrib/tpu/python/ops/tpu_ordinal_selector_op.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.ops.tpu_ordinal_selector_op import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/profiler/__init__.py b/tensorflow/contrib/tpu/python/profiler/__init__.py deleted file mode 100644 index aeb061dbe11..00000000000 --- a/tensorflow/contrib/tpu/python/profiler/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.profiler import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/__init__.py b/tensorflow/contrib/tpu/python/tpu/__init__.py deleted file mode 100644 index 82d4f68c022..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py b/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py deleted file mode 100644 index d85aae64871..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/_tpu_estimator_embedding.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow_estimator.python.estimator.tpu._tpu_estimator_embedding import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/async_checkpoint.py b/tensorflow/contrib/tpu/python/tpu/async_checkpoint.py deleted file mode 100644 index 5eb8034e474..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/async_checkpoint.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.async_checkpoint import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/bfloat16.py b/tensorflow/contrib/tpu/python/tpu/bfloat16.py deleted file mode 100644 index f3d392a8dae..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/bfloat16.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.bfloat16 import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/datasets.py b/tensorflow/contrib/tpu/python/tpu/datasets.py deleted file mode 100644 index c20aac7e36a..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/datasets.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.datasets import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/device_assignment.py b/tensorflow/contrib/tpu/python/tpu/device_assignment.py deleted file mode 100644 index 05dffef3a1e..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/device_assignment.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import,redefined-builtin -from tensorflow.python.tpu.device_assignment import * -# pylint: enable=wildcard-import,unused-import,redefined-builtin diff --git a/tensorflow/contrib/tpu/python/tpu/error_handling.py b/tensorflow/contrib/tpu/python/tpu/error_handling.py deleted file mode 100644 index 9cbb5084a54..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/error_handling.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow_estimator.python.estimator.tpu.error_handling import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/feature_column.py b/tensorflow/contrib/tpu/python/tpu/feature_column.py deleted file mode 100644 index ded75e975b1..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/feature_column.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.feature_column import * -# used by tests -from tensorflow.python.tpu.feature_column import _is_running_on_cpu -from tensorflow.python.tpu.feature_column import _record_variable_scope_and_name -from tensorflow.python.tpu.feature_column import _TPU_FC_TO_SCOPE -from tensorflow.python.tpu.feature_column import _TPUBaseEmbeddingColumn -from tensorflow.python.tpu.feature_column import _TPUEmbeddingColumn -from tensorflow.python.tpu.feature_column import _TPUSharedEmbeddingColumn -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/functional.py b/tensorflow/contrib/tpu/python/tpu/functional.py deleted file mode 100644 index 9a5759221ed..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/functional.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.functional import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/keras_support.py b/tensorflow/contrib/tpu/python/tpu/keras_support.py deleted file mode 100644 index 01b1b4af339..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/keras_support.py +++ /dev/null @@ -1,2259 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""*Experimental* support for running Keras models on the TPU. - -To use, wrap your model with the `keras_support.tpu_model` function. - -Example usage: - -``` -image = tf.keras.layers.Input(shape=(28, 28, 3), name='image') -c1 = tf.keras.layers.Conv2D(filters=16, kernel_size=(3, 3))( image) -flattened = tf.keras.layers.Flatten()(c1) -logits = tf.keras.layers.Dense(10, activation='softmax')(flattened) -model = tf.keras.Model(inputs=[image], outputs=[logits]) - -resolver = tf.contrib.cluster_resolver.TPUClusterResolver(tpu=tpu_name) -strategy = keras_support.TPUDistributionStrategy(resolver) -model = keras_support.tpu_model(model, strategy=strategy) - -# Only TF optimizers are currently supported. -model.compile(optimizer=tf.compat.v1.train.AdamOptimizer(), ...) - -# `images` and `labels` should be Numpy arrays. Support for tensor input -# (e.g. datasets) is planned. -model.fit(images, labels) -``` -""" - -# pylint: disable=protected-access - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc -import collections -import contextlib -import re -import sys -import time - -import numpy as np -import six - -from tensorflow.contrib.cluster_resolver.python.training import tpu_cluster_resolver as tpu_cluster_resolver_lib -from tensorflow.contrib.tpu.python.ops import tpu_ops -from tensorflow.contrib.tpu.python.tpu import keras_tpu_variables -from tensorflow.contrib.tpu.python.tpu import tpu -from tensorflow.contrib.tpu.python.tpu import tpu_function -from tensorflow.contrib.tpu.python.tpu import tpu_optimizer -from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib -from tensorflow.core.protobuf import config_pb2 -from tensorflow.core.protobuf.tpu import compilation_result_pb2 as tpu_compilation_result -from tensorflow.python import tf2 -from tensorflow.python.client import session as tf_session -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.ops import iterator_ops -from tensorflow.python.eager import context -from tensorflow.python.estimator import model_fn as model_fn_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_spec -from tensorflow.python.keras import backend as K -from tensorflow.python.keras import callbacks as cbks -from tensorflow.python.keras import metrics as metrics_module -from tensorflow.python.keras import models -from tensorflow.python.keras import optimizers as keras_optimizers -from tensorflow.python.keras.engine import base_layer -from tensorflow.python.keras.engine import base_layer_utils -from tensorflow.python.keras.engine import training_arrays -from tensorflow.python.keras.engine import training_utils -from tensorflow.python.keras.layers import embeddings -from tensorflow.python.keras.utils.generic_utils import make_batches -from tensorflow.python.keras.utils.generic_utils import slice_arrays -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util.deprecation import deprecated - - -# TODO(b/114775106): temporary shim to optionally initialize the TPU -# This increases the odds our session is initialized, but shouldn't be needed. -_TEST_REWRITE_OP = None - - -def _maybe_initialize_tpu(session): - """Initialize the TPU if it has not already been initialized.""" - global _TEST_REWRITE_OP - try: - # Try to use cached version to avoid another ground of graph optimization. - test_rewrite_op = _TEST_REWRITE_OP - if (test_rewrite_op is None or - test_rewrite_op[0].graph != ops.get_default_graph()): - - def test_op(): - return constant_op.constant(1) + constant_op.constant(1) - - test_rewrite_op = tpu.rewrite(test_op) - _TEST_REWRITE_OP = test_rewrite_op - - session.run(test_rewrite_op) - except errors.FailedPreconditionError as _: - session.run(tpu.initialize_system()) - - -@contextlib.contextmanager -def _tpu_session_context(): - """Initialize the TPU and cleans cache entries for bad sessions.""" - try: - _maybe_initialize_tpu(K.get_session()) - yield - except (errors.FailedPreconditionError, errors.AbortedError) as e: - K.clear_session() - raise Exception(""" -An error occurred connecting or initializing your TPU. - -The session has been reset. re-run keras_to_tpu_model to create a new session. -""" + str(e)) - - -def setup_tpu_session(cluster_resolver): - """Construct or return a `tf.Session` connected to the given cluster.""" - master = cluster_resolver.master() - - # Use the existing session if we're already connected to this TPU - # N.B K.get_session() is a non-trivial operation, and may fail if the remote - # session has been reset. - try: - default_session = K.get_session() - if (default_session._target == master and - getattr(default_session, '_tpu_initialized', None)): - return - except errors.AbortedError as _: - # We lost the remote session and need to re-initialize. - logging.warning('Lost remote session: creating a new session.') - - cluster_spec = cluster_resolver.cluster_spec() - config = config_pb2.ConfigProto(isolate_session_state=True) - if cluster_spec: - config.cluster_def.CopyFrom(cluster_spec.as_cluster_def()) - - tpu_session = tf_session.Session(target=master, config=config) - tpu_session.run(tpu.initialize_system()) - tpu_session._tpu_initialized = True - - # N.B. We have to call `K.set_session()` AND set our session as the - # TF default. `K.get_session()` surprisingly does not return the value - # supplied by K.set_session otherwise. - K.set_session(tpu_session) - - -try: - from scipy.sparse import issparse # pylint: disable=g-import-not-at-top -except ImportError: - issparse = None - - -def get_tpu_system_metadata(tpu_cluster_resolver): - """Retrieves TPU system metadata given a TPUClusterResolver.""" - master = tpu_cluster_resolver.master() - - # pylint: disable=protected-access - cluster_spec = tpu_cluster_resolver.cluster_spec() - cluster_def = cluster_spec.as_cluster_def() if cluster_spec else None - tpu_system_metadata = ( - tpu_system_metadata_lib._query_tpu_system_metadata( - master, cluster_def=cluster_def, query_topology=False)) - - return tpu_system_metadata - - -class TPUDistributionStrategy(object): - """The strategy to run Keras model on TPU.""" - - def __init__(self, tpu_cluster_resolver=None, using_single_core=False): - """Construct a TPUDistributionStrategy. - - Args: - tpu_cluster_resolver: Any instance of `TPUClusterResolver`. If None, will - create one with '' as master address. - using_single_core: Bool. This is the debugging option, which might be - removed in future once the model replication functionality is mature - enough. If `False` (default behavior), the system automatically finds - the best configuration, in terms of number of TPU cores, for the model - replication, typically using all available TPU cores. If overwrites as - `True`, force the model replication using single core, i.e., no - replication. - Raises: - Exception: No TPU Found on the given worker. - """ - if tf2.enabled(): - raise RuntimeError( - 'Keras support is now deprecated in support of TPU Strategy. ' - 'Please follow the distribution strategy guide on tensorflow.org ' - 'to migrate to the 2.0 supported version.') - else: - logging.warning( - 'Keras support is now deprecated in support of TPU Strategy. ' - 'Please follow the distribution strategy guide on tensorflow.org ' - 'to migrate to the 2.0 supported version.') - if tpu_cluster_resolver is None: - tpu_cluster_resolver = tpu_cluster_resolver_lib.TPUClusterResolver('') - - metadata = get_tpu_system_metadata(tpu_cluster_resolver) - self._tpu_metadata = metadata - self._tpu_cluster_resolver = tpu_cluster_resolver - self._num_cores = 1 if using_single_core else metadata.num_cores - - # Walk device list to identify TPU worker for enqueue/dequeue operations. - worker_re = re.compile('/job:([^/]+)') - for device in metadata.devices: - if 'TPU:0' in device.name: - self._worker_name = worker_re.search(device.name).group(1) - return - raise Exception('No TPU found on given worker.') - - def _make_assignment_for_model(self, cpu_model): - """Makes a `TPUAssignment` for the passed in `cpu_model`.""" - num_cores = self._num_cores - if num_cores > 1 and cpu_model.stateful: - logging.warning( - 'Model replication does not currently support stateful models. ' - 'Degrading to a single core.') - num_cores = 1 - - return TPUAssignment(worker_name=self._worker_name, num_cores=num_cores) - - -class TPUAssignment(object): - """This is object holding TPU resources assignment for the concrete model. - - `TPUDistributionStrategy` is responsible to create the instance of - `TPUAssignment`, so, it can dynamically adjust the `num_cores` to use based on - model and input batch sizes. - """ - - def __init__(self, worker_name, num_cores): - self._worker_name = worker_name - self._num_cores = num_cores - - @property - def worker_name(self): - return self._worker_name - - @property - def num_towers(self): - # TODO(xiejw): Support automatically assign num_cores based on inputs. - return self._num_cores - - -class TPUEmbedding(embeddings.Embedding): - """TPU compatible embedding layer. - - The default Keras layer is not TPU compatible. This layer is a drop-in - replacement: it has the same behavior and will work on CPU and GPU devices. - """ - - def build(self, input_shape): - if input_shape[0] is None: - raise ValueError( - 'TPUEmbeddings must have a fixed input_length or input shape.') - return super(TPUEmbedding, self).build(input_shape) - - def call(self, inputs): - if K.dtype(inputs) != 'int32': - inputs = math_ops.cast(inputs, 'int32') - - inputs = array_ops.one_hot(inputs, self.input_dim) - return math_ops.tensordot(inputs, self.embeddings, 1) - - -def _cross_replica_concat(tensor, core_id, num_cores, name): - """Concatenate `tensor` across cores. - - Args: - tensor: The tensor to be concatenated. Must be [int32 and float32]. - core_id: Tensor indicating the current TPU core. - num_cores: Python int. The total number of TPU cores in the system. - name: The string name to print for debugging. - - Returns: - The same concatenated Tensor on each core. - """ - - input_dtype = tensor.dtype - if input_dtype not in [dtypes.bfloat16, dtypes.float32, dtypes.int32]: - raise TypeError('For model replication, only (bfloat16, float32 and int32) ' - 'is supported for model outputs and targets. Got {} for ' - '{}.'.format(input_dtype, name)) - - batch_size = tensor.shape[0] - mask = math_ops.cast( - math_ops.equal(np.arange(num_cores, dtype=np.int32), core_id), - dtypes.float32) - mask = array_ops.reshape(mask, [num_cores] + [1] * tensor.shape.ndims) - result = mask * math_ops.cast(tensor, dtypes.float32) - local_tensor_with_holes = array_ops.reshape(result, - [-1] + result.shape.as_list()[2:]) - concat_tensor = tpu_ops.cross_replica_sum(local_tensor_with_holes) - concat_tensor.set_shape((num_cores * batch_size,) + tuple(tensor.shape[1:])) - - if concat_tensor != input_dtype: - concat_tensor = math_ops.cast(concat_tensor, input_dtype) - return concat_tensor - - -class KerasCrossShardOptimizer(keras_optimizers.Optimizer): - """An optimizer that averages gradients across TPU shards.""" - - def __init__(self, opt, name='KerasCrossShardOptimizer'): - """Construct a new cross-shard optimizer. - - Args: - opt: An existing `Optimizer` to encapsulate. - name: Optional name prefix for the operations created when applying - gradients. Defaults to "KerasCrossShardOptimizer". - - Raises: - ValueError: If reduction is not a valid cross-shard reduction. - """ - super(KerasCrossShardOptimizer, self).__init__() - self._name = name - self._opt = opt - logging.info('KerasCrossShard: %s %s', self._opt, self._opt.weights) - - def get_updates(self, loss, params): - self._opt.get_gradients = self.get_gradients - return self._opt.get_updates(loss, params) - - def get_gradients(self, loss, params): - num_shards = tpu_function.get_tpu_context().number_of_shards - grads = super(KerasCrossShardOptimizer, self).get_gradients(loss, params) - return [tpu_ops.cross_replica_sum(grad) / num_shards for grad in grads] - - def get_weights(self): - return self._opt.get_weights() - - def get_config(self): - return self._opt.get_config() - - # Defer remaining operations to the underlying optimizer - def __getattr__(self, key): - return getattr(self._opt, key) - - -class TPUModelOp( - collections.namedtuple('TPUModelOp', [ - 'compile_op', 'execute_op', 'infeed_tensors', 'infeed_op', 'outfeed_op' - ])): - pass - - -def _valid_name(tensor_name): - """Return a valid tensor name (strips '/', ':', etc).""" - return re.sub('[^a-zA-Z0-9_-]+', '', tensor_name) - - -def _replicated_optimizer(opt): - """Wrap the optimizer `opt` with CrossShardOptimizer if applicable.""" - # Always wrap `opt` with CrossShardOptimizer, even if we are running on a - # single core. This ensures Keras properly tracks and initializes optimizer - # variables. - if isinstance(opt, keras_optimizers.TFOptimizer): - return tpu_optimizer.CrossShardOptimizer(opt.optimizer) - else: - return KerasCrossShardOptimizer(opt) - - -def _clone_optimizer(optimizer, config=None, worker_name=None): - """Returns a cloned optimizer with the provided optimizer.config or config.""" - if not isinstance(optimizer, keras_optimizers.Optimizer): - # In the first call to tpu_model(model), Keras may not have wrapped the TF - # optimizer in the TFOptimizer helper, e.g., the given model isn't compiled - # or optimizer isn't set, and later generated tpu_model compiles with a TF - # optimizer. - return optimizer - - if isinstance(optimizer, keras_optimizers.TFOptimizer): - return keras_optimizers.TFOptimizer(optimizer.optimizer) - - if config is None: - config = optimizer.get_config() - logging.info('Cloning %s %s', optimizer.__class__.__name__, config) - with ops.device( - '%s/device:CPU:0' % ('/job:%s' % worker_name if worker_name else '')): - # Explicitly put optimizer parameter variables on TPU worker. - return optimizer.__class__.from_config(config) - - -class TPURewriteContext(object): - """Prepare the environment for a Keras model during `tpu.rewrite`. - - This overrides the default placeholder behaviour to instead refer to a preset - input mapping. Placeholders are unsupported in TPU compiled code, and must - be replaced with explicit inputs or values from the infeed queue. - - Instead of explicitly threading inputs all the way through the Keras codebase, - we override the behavior of the placeholder while compiling and inject the - Tensors from the infeed in place of the placeholder. - - Similarly, as we compile a new sub-graph for each unique shape and execution - mode, we need to override the behavior of an embedded `name_scope` call in - the base Keras layer code. This allows us to re-use the same weights across - many compiles and share a single session/graph. - """ - - def __init__(self, input_map): - self._input_map = input_map - self._default_placeholder = None - self._default_name_scope = None - - def __enter__(self): - - def _placeholder(dtype, shape=None, name=None): # pylint: disable=unused-argument - logging.info('Remapping placeholder for %s', name) - if name in self._input_map: - return self._input_map[name] - else: - logging.info('Default: %s', name) - return self._default_placeholder(dtype, shape, name) - - def _name_scope(name, default_name=None, values=None): - caller_frame = sys._getframe().f_back - caller_obj = caller_frame.f_locals.get('self') - if (caller_obj is not None and - isinstance(caller_obj, base_layer.Layer) and name is not None): - return variable_scope.variable_scope( - name, default_name, values, reuse=variable_scope.AUTO_REUSE) - - return self._default_name_scope(name, default_name, values) - - self._default_placeholder = array_ops.placeholder - self._default_name_scope = ops.name_scope - self._default_make_variable = base_layer_utils.make_variable - self._default_random_normal = random_ops.random_normal - self._default_qr = gen_linalg_ops.qr - - array_ops.placeholder = _placeholder - - # Replace random_ops.random_normal with a dummy function because - # `random_normal` isn't yet implemented on the TPU. Because these - # initialized values are overwritten by the CPU values, this is okay. - def random_normal(shape, - mean=0.0, - stddev=1.0, - dtype=dtypes.float32, - seed=None, - name=None): - del mean - del stddev - del seed - return array_ops.zeros(shape, dtype=dtype, name=name) - - random_ops.random_normal = random_normal - - # Replace gen_linalg_ops.qr because QR decomposition is not yet implemented. - # TODO(saeta): Remove qr override once we confirm the qr implementation is - # ok. - # pylint: disable=redefined-builtin - def qr(input, full_matrices=False, name=None): - """Dummy implementation of qr decomposition.""" - del full_matrices # TODO(saeta): Properly handle the full matrix case. - input_shape = input.shape - if len(input_shape) < 2: - raise ValueError('Invalid shape passed to qr: %s' % input_shape) - p = min(input_shape[-1], input_shape[-2]) - if len(input_shape) == 2: - q = array_ops.zeros((p, p), name=name) - r = array_ops.zeros(input_shape, name=name) - return (r, q) - elif len(input_shape) == 3: - n = input_shape[0] - q = array_ops.zeros((n, p, p), name=name) - r = array_ops.zeros(input_shape, name=name) - return (r, q) - else: - raise ValueError('Invalid shape passed to qr: %s' % input_shape) - - gen_linalg_ops.qr = qr - - ops.name_scope = _name_scope - base_layer_utils.make_variable = variable_scope.get_variable - logging.info('Overriding default placeholder.') - return - - def __exit__(self, exc_type, exc_val, exc_tb): - array_ops.placeholder = self._default_placeholder - ops.name_scope = self._default_name_scope - base_layer_utils.make_variable = self._default_make_variable - random_ops.random_normal = self._default_random_normal - gen_linalg_ops.qr = self._default_qr - - -class SizedInfeed( - collections.namedtuple('SizedInfeed', - ['sharded_infeed_tensors', 'infeed_ops'])): - """Represents an instantiation of the infeed ops for a concrete input shape. - - sharded_infeed_tensors: A data structure of Tensors used to represent the - placeholder tensors that must be fed when using feed_dicts. - - infeed_ops: the set of ops that will be run to drive infeed for a single step. - """ - pass - - -class TPUInfeedInstance(object): - """TPUInfeedInstance represents the logic to manage feeding in a single step. - - See the comments on the `TPUInfeedManager` for a description for how infeed - is managed. - """ - - @abc.abstractmethod - def make_input_specs(self, input_tensors): - """Constructs the infeed_specs for the given Infeed instance. - - Args: - input_tensors: The inputs to the model. - - Returns: - A list of - """ - pass - - def make_feed_dict(self, tpu_model_op): - """Constructs a feed_dict for this instance, given the tpu_model_op. - - Args: - tpu_model_op: A `TPUModelOp` representing the TPU Model for this - instance's input spec. - - Returns: - A dictionary to use as the feed_dict of a `session.run` call. - """ - pass - - -@six.add_metaclass(abc.ABCMeta) -class TPUInfeedManager(object): - """TPUInfeedManager manages the data infeeding of data to a TPU computation. - - Because there are multiple data sources (e.g. in-memory NumPy arrays, - `tf.data.Dataset`s), we abstract the different logic behind a single - interface: the `TPUInfeedManager`. - - (1) A `TPUFunction` is called with a set of inputs. Based on the inputs, - `TPUFunction` retrieves the corresponding `TPUInfeedManager` (or constructs a - new one if required). - - (2) The `TPUFunction` calls `make_infeed_instance` on the `TPUInfeedManager` - which returns a `TPUInfeedInstance`. - - (3) The `TPUFunction` checks in the shape cache for a pre-compiled instance of - the model based on the returned `input_specs` from `TPUInfeedInstance`. - - (4) [Optional.] If the model has not already been instantiated for the given - input spec, the `TPUFunction` compiles the model for the input spec (using the - `TPUInfeedManager`). - - (5) The `TPUInfeedInstance` constructs the session.run's feed_dict given the - compiled model instance corresponding to its shape. - """ - - @abc.abstractmethod - def make_infeed_instance(self, inputs): - """Given a single step's input, construct a `TPUInfeedInstance`. - - Args: - inputs: The inputs to a given step. - - Returns: - A subclass of `TPUInfeedInstance`. - """ - pass - - @abc.abstractmethod - def build_infeed_from_input_specs(self, input_specs, execution_mode): - """For a given input specification (size, type), construct the infeed ops. - - This is called only once for a given input specification and builds the - graph ops. It does not have a pointer to the actual infeed data. - - Args: - input_specs: TODO(saeta): Document me! - execution_mode: TODO(saeta): Document me! - - Returns: - A `SizedInfeed` instance. - """ - pass - - -class TPUNumpyInfeedManager(TPUInfeedManager): - """TPU Infeed manager for Numpy inputs.""" - - class NumpyInfeedInstance(TPUInfeedInstance): - """Infeed instance for Numpy inputs.""" - - def __init__(self, sharded_inputs): - self._sharded_inputs = sharded_inputs - - def make_input_specs(self, input_tensors): - # Compute an input specification (used to generate infeed enqueue and - # dequeue operations). We use the shape from our input array and the - # dtype from our model. A user may pass in a float64 for a float32 - # input: for model compatibility we still must generate a float32 infeed. - input_specs = [] - # We use the shape and dtype from the first shard to compute the input - # metadata (`input_specs`); all replicas have the same type and shape. - for tensor, ary in zip(input_tensors, self._sharded_inputs[0]): - input_specs.append( - tensor_spec.TensorSpec(ary.shape, tensor.dtype, - _valid_name(tensor.name))) - - return input_specs - - def make_feed_dict(self, tpu_model_op): - infeed_dict = {} - for infeed_tensors, inputs in zip(tpu_model_op.infeed_tensors, - self._sharded_inputs): - for tensor, value in zip(infeed_tensors, inputs): - infeed_dict[tensor] = value - return infeed_dict - - def __init__(self, tpu_assignment): - self._tpu_assignment = tpu_assignment - - def _split_tensors(self, inputs): - """Split input data across shards. - - Each input is sliced along the batch axis. - - Args: - inputs: List of Numpy arrays to run on the TPU. - - Returns: - List of lists containing the input to feed to each TPU shard. - """ - if self._tpu_assignment.num_towers == 1: - return [inputs] - - batch_size = inputs[0].shape[0] - assert batch_size % self._tpu_assignment.num_towers == 0, ( - 'batch_size must be divisible by the number of TPU cores in use (%s ' - 'vs %s)' % (batch_size, self._tpu_assignment.num_towers)) - shard_size = batch_size // self._tpu_assignment.num_towers - input_list = [] - for index in range(self._tpu_assignment.num_towers): - shard_inputs = [ - x[index * shard_size:(index + 1) * shard_size] for x in inputs - ] - input_list.append(shard_inputs) - return input_list - - def make_infeed_instance(self, inputs): - sharded_inputs = self._split_tensors(inputs) - return self.NumpyInfeedInstance(sharded_inputs) - - def build_infeed_from_input_specs(self, input_specs, execution_mode): - infeed_op = [] - shard_infeed_tensors = [] - - for shard_id in range(self._tpu_assignment.num_towers): - with ops.device( - '/job:%s/device:CPU:0' % self._tpu_assignment.worker_name): - infeed_tensors = [] - with ops.device('/device:TPU:%d' % shard_id): - for spec in input_specs: - # Construct placeholders for each of the inputs. - infeed_tensors.append( - array_ops.placeholder( - dtype=spec.dtype, - shape=spec.shape, - name='infeed-enqueue-%s-%d' % (spec.name, shard_id))) - shard_infeed_tensors.append(infeed_tensors) - - infeed_op.append( - tpu_ops.infeed_enqueue_tuple( - infeed_tensors, [spec.shape for spec in input_specs], - name='infeed-enqueue-%s-%d' % (execution_mode, shard_id), - device_ordinal=shard_id)) - return SizedInfeed( - infeed_ops=infeed_op, sharded_infeed_tensors=shard_infeed_tensors) - - -class TPUDatasetInfeedManager(TPUInfeedManager): - """Manages infeed for a `tf.data.Dataset` into a TPU computation. - - """ - - class DatasetInfeedInstance(TPUInfeedInstance): - """An instance of the TPU infeed.""" - - def __init__(self, input_specs): - self._input_specs = input_specs - - def make_input_specs(self, input_tensors): - # TODO(saeta): Do error checking here! - return self._input_specs - - def make_feed_dict(self, tpu_model_op): - # TODO(saeta): Verify tpu_model_op is as expected! - return {} - - # pylint: disable=redefined-outer-name - def __init__(self, dataset, tpu_assignment, mode): - """Constructs a TPUDatasetInfeedManager. - - Args: - dataset: A `tf.data.Dataset` to infeed. - tpu_assignment: The `TPUAssignment` used to configure the - Keras TPU model. - mode: ModeKeys enum. - """ - self._verify_dataset_shape(dataset) - - self._dataset = dataset - self._tpu_assignment = tpu_assignment - dataset_output_shapes = dataset_ops.get_legacy_output_shapes(dataset) - dummy_x_shape = dataset_output_shapes[0].as_list() - dummy_x_shape[0] *= tpu_assignment.num_towers - dummy_y_shape = dataset_output_shapes[1].as_list() - dummy_y_shape[0] *= tpu_assignment.num_towers - self._iterator = dataset_ops.make_initializable_iterator(dataset) - K.get_session().run(self._iterator.initializer) - - self._get_next_ops = [] - ctrl_deps = [] - for i in range(tpu_assignment.num_towers): - with ops.control_dependencies(ctrl_deps): # Ensure deterministic - # TODO(saeta): Ensure correct placement! - get_next_op = self._iterator.get_next() - self._get_next_ops.append(get_next_op) - ctrl_deps.extend(get_next_op) - - # Use dummy numpy inputs for the rest of Keras' shape checking. We - # intercept them when building the model. - dataset_output_types = dataset_ops.get_legacy_output_types(dataset) - self._dummy_x = np.zeros( - dummy_x_shape, dtype=dataset_output_types[0].as_numpy_dtype) - self._dummy_y = np.zeros( - dummy_y_shape, dtype=dataset_output_types[1].as_numpy_dtype) - - input_specs = [] - iterator_output_shapes = dataset_ops.get_legacy_output_shapes( - self._iterator) - iterator_output_types = dataset_ops.get_legacy_output_types(self._iterator) - if isinstance(iterator_output_shapes, tuple): - assert isinstance(iterator_output_types, tuple) - assert len(iterator_output_shapes) == len(iterator_output_types) - for i in range(len(iterator_output_shapes)): - spec = tensor_spec.TensorSpec(iterator_output_shapes[i], - iterator_output_types[i]) - input_specs.append(spec) - elif isinstance(iterator_output_shapes, tensor_shape.TensorShape): - spec = tensor_spec.TensorSpec(iterator_output_shapes, - iterator_output_types) - input_specs.append(spec) - - # Pre-process the inputs and get_next_ops before caching. - input_specs, self._get_next_ops = ( - _inject_tpu_inputs_for_dataset( - tpu_assignment, mode, input_specs, self._get_next_ops)) - self._infeed_instance = self.DatasetInfeedInstance(input_specs) - - def _verify_dataset_shape(self, dataset): - """Verifies a dataset is of an appropriate shape for TPUs.""" - dataset_output_shapes = dataset_ops.get_legacy_output_shapes(dataset) - dataset_output_classes = dataset_ops.get_legacy_output_classes(dataset) - if not isinstance(dataset, dataset_ops.DatasetV2): - raise ValueError('The function passed as the `x` parameter did not ' - 'return a `tf.data.Dataset`.') - if not isinstance(dataset_output_classes, tuple): - raise ValueError('The dataset must return a tuple of tf.Tensors, ' - 'instead it returns: %s' % dataset_output_classes) - if len(dataset_output_classes) != 2: - raise ValueError('The dataset must return a 2-element tuple, got ' - '%s output classes instead.' % (dataset_output_classes,)) - for i, cls in enumerate(dataset_output_classes): - if cls != ops.Tensor: - raise ValueError('The dataset returned a non-Tensor type (%s) at ' - 'index %d.' % (cls, i)) - for i, shape in enumerate(dataset_output_shapes): - if not shape: - raise ValueError('The dataset returns a scalar tensor in ' - 'tuple index %d. Did you forget to batch? ' - '(Output shapes: %s).' % (i, dataset_output_shapes)) - for j, dim in enumerate(shape): - if dim.value is None: - if j == 0: - hint = (' Hint: did you use `ds.batch(BATCH_SIZE, ' - 'drop_remainder=True)`?') - else: - hint = '' - raise ValueError( - 'The Keras-TPU integration for `tf.data` ' - 'currently requires static shapes. The provided ' - 'dataset only has a partially defined shape. ' - '(Dimension %d of output tensor %d is not statically known ' - 'for output shapes: %s.%s)' % (j, i, dataset_output_shapes, hint)) - - @property - def dummy_x(self): - return self._dummy_x - - @property - def dummy_y(self): - return self._dummy_y - - def make_infeed_instance(self, inputs): - # TODO(saeta): Verify inputs is as expected. - return self._infeed_instance - - def build_infeed_from_input_specs(self, input_specs, execution_mode): - shard_infeed_tensors = self._get_next_ops - assert len(shard_infeed_tensors) == self._tpu_assignment.num_towers - infeed_ops = [] - for shard_id in range(self._tpu_assignment.num_towers): - with ops.device( - '/job:%s/device:CPU:0' % self._tpu_assignment.worker_name): - infeed_ops.append( - tpu_ops.infeed_enqueue_tuple( - shard_infeed_tensors[shard_id], - [spec.shape for spec in input_specs], - name='infeed-enqueue-%s-%d' % (execution_mode, shard_id), - device_ordinal=shard_id)) - return SizedInfeed( - infeed_ops=infeed_ops, sharded_infeed_tensors=shard_infeed_tensors) - - -def _inject_tpu_inputs_for_dataset(tpu_assignment, mode, - input_specs, get_next_ops): - """Append core information to the set of dataset inputs.""" - # This is used during compilation to identify the current TPU core and enable - # concatenation operations across cores. - if mode not in [model_fn_lib.ModeKeys.TRAIN, model_fn_lib.ModeKeys.EVAL]: - return input_specs, get_next_ops - - # Dataset inputs operate on per core basis. - per_core_batch_size = input_specs[0].shape.as_list()[0] - - # Insert, at head, the tensor for core_id. - assert len(get_next_ops) == tpu_assignment.num_towers - for i in range(tpu_assignment.num_towers): - core_id_constant = constant_op.constant( - np.array([i] * per_core_batch_size).astype('int32'), - dtype=dtypes.int32, - name='cord_id_constant') - get_next_ops[i] = [core_id_constant] + list(get_next_ops[i]) - - # Insert the input spec at head also. - input_specs = [tensor_spec.TensorSpec([per_core_batch_size], dtypes.int32) - ] + input_specs - - return input_specs, get_next_ops - - -def _inject_tpu_inputs_for_infeed(tpu_assignment, mode, - core_id_place_holder, input_tensors, inputs): - """Append core information to the set of inputs.""" - # This is used during compilation to identify the current TPU core and enable - # concatenation operations across cores. - if mode not in [model_fn_lib.ModeKeys.TRAIN, model_fn_lib.ModeKeys.EVAL]: - return input_tensors, inputs - - # Puts a place holder in input spec. - input_tensors = [core_id_place_holder] + input_tensors - - # Now fill the core id. For `num_cores` = 2, `batch_size` = 8, we fill the - # core id inputs as [0, 0, 0, 0, 1, 1, 1, 1], so each core sees its core id - # (duplicated). - num_cores = tpu_assignment.num_towers - per_core_batch_size = inputs[0].shape[0] // num_cores - core_ids = np.arange(num_cores).repeat(per_core_batch_size) - inputs = [core_ids] + inputs - return input_tensors, inputs - - -def _read_tpu_coreid_from_infeed(mode, infeed_tensors): - """Popping out the core ids from infeed.""" - if mode not in [model_fn_lib.ModeKeys.TRAIN, model_fn_lib.ModeKeys.EVAL]: - return None, infeed_tensors - - if len(infeed_tensors) <= 1: - raise RuntimeError( - 'The infeed tensors on TPU core has only {} tensors. ' - 'This is not expected. Please report a bug.\nTensors: {}'.format( - len(infeed_tensors), infeed_tensors)) - - core_id = infeed_tensors[0][0] # Pop out the scalar version. - rest = infeed_tensors[1:] - return core_id, rest - - -class TPUFunction(object): - """K.function compatible interface for invoking a TPU compiled function. - - Recompilation is triggered on-demand for each set of new inputs shapes: the - results are cached for future execution. We expect most computations will - be dominated by a standard batch-size, followed by a straggler batch for - the end of training or evaluation. - - All `inputs` and `outputs` will be loaded via the infeed and outfeed queues - instead of being injected as `feed_dict` items or fetches. - """ - - def __init__(self, model, execution_mode, tpu_assignment): - self.model = model - self.execution_mode = execution_mode - self._tpu_assignment = tpu_assignment - self._compilation_cache = {} - self._cloned_model = None - self._cloned_optimizer = None - # Create a placeholder for the TPU core ID. Cache the placeholder to avoid - # modifying the graph for every batch. - self._core_id_place_holder = array_ops.placeholder( - dtype=dtypes.int32, shape=[1], name='core_id') - - def _specialize_model(self, input_specs, infeed_manager): - """Specialize `self.model` (a Keras model) for the given input shapes.""" - # Re-create our input and output layers inside our subgraph. They will be - # attached to the true computation when we clone our model in `tpu_fn`. - K.set_learning_phase(self.execution_mode == model_fn_lib.ModeKeys.TRAIN) - - # functools.partial and callable objects are not supported by tpu.rewrite - def _model_fn(): - """Compute fit/eval/predict for the TPU.""" - is_training = self.execution_mode == model_fn_lib.ModeKeys.TRAIN - is_test = self.execution_mode == model_fn_lib.ModeKeys.EVAL - is_predict = self.execution_mode == model_fn_lib.ModeKeys.PREDICT - - # During train/eval, we infeed our features as well as labels. - if is_training or is_test: - infeed_layers = self.model._input_layers + self.model._output_layers - else: - infeed_layers = self.model._input_layers - - # Generate our infeed operation to read features & labels. - infeed_tensors = tpu_ops.infeed_dequeue_tuple( - dtypes=[spec.dtype for spec in input_specs], - shapes=[spec.shape for spec in input_specs], - name='infeed-%s' % self.execution_mode) - - core_id, infeed_tensors = ( - _read_tpu_coreid_from_infeed( - mode=self.execution_mode, infeed_tensors=infeed_tensors)) - - assert len(infeed_tensors) == len(infeed_layers), ( - 'Infeed inputs did not match model: %s vs %s' % (infeed_layers, - infeed_tensors)) - - tpu_targets = [] - tpu_input_map = {} - - # Sort infeed outputs into inputs and labels for calling our Keras model. - for tensor, layer in zip(infeed_tensors, infeed_layers): - if layer in self.model._input_layers: - tpu_input_map[layer.name] = tensor - if layer in self.model._output_layers: - tpu_targets.append(tensor) - - # Clone our CPU model, running within the TPU device context. - # - # We use the id of the original model as a key to avoid weight collisions - # (if a user re-runs the same model multiple times, in e.g. Colab). - with TPURewriteContext(tpu_input_map): - with variable_scope.variable_scope('tpu_%s' % id(self.model)): - with keras_tpu_variables.replicated_scope( - self._tpu_assignment.num_towers): - if not self._cloned_optimizer: - self._cloned_optimizer = _clone_optimizer( - self.model.cpu_optimizer, - worker_name=self._tpu_assignment.worker_name) - - self._cloned_model = models.clone_model(self.model) - - # When running on more than one core, concatenate outputs at the end - # of processing. In backprop stage, the gradients will be - # calculated according to the local inputs as gradient of - # cross-replica-concat being zero for any outputs other than those - # from mlocal core so the loss calculation is identical. - num_towers = self.model._tpu_assignment.num_towers - if num_towers > 1 and (is_training or is_test): - new_outputs = [ - _cross_replica_concat( - o, core_id, num_towers, - name='model output ({})'.format(o.name)) - for o in self._cloned_model.outputs - ] - # Recast all low precision outputs back to float32 since we only - # casted the inputs to bfloat16 and not targets. This is done so - # that we can preserve precision when calculating the loss value. - if new_outputs and new_outputs[0].dtype == dtypes.bfloat16: - new_outputs = [ - math_ops.cast(o, dtypes.float32) for o in new_outputs] - self._cloned_model.outputs = new_outputs - tpu_targets = [ - _cross_replica_concat( - tensor, - core_id, - num_towers, - name='model target ({})'.format(tensor.name)) - for tensor in tpu_targets - ] - - if is_training or is_test: - with variable_scope.variable_scope( - 'metrics', reuse=variable_scope.AUTO_REUSE): - self._cloned_model.compile( - optimizer=_replicated_optimizer(self._cloned_optimizer), - loss=self.model.loss, - loss_weights=self.model.loss_weights, - metrics=metrics_module.clone_metrics( - self.model._compile_metrics), - weighted_metrics=metrics_module.clone_metrics( - self.model._compile_weighted_metrics), - target_tensors=tpu_targets, - ) - - # Compute our outfeed depending on the execution mode - if is_training: - if not isinstance(self._cloned_optimizer, keras_optimizers.TFOptimizer): - # For Keras optimizer, we try to place the variable weights on the TPU - # device. Keras creates optimizer variables (e.g. momentum values for - # the Momentum optimizer) when _make_train_function is invoked. - with keras_tpu_variables.replicated_variable_for_optimizer( - self._tpu_assignment.num_towers): - self._cloned_model._make_train_function() - else: - self._cloned_model._make_train_function() - - self._outfeed_spec = [ - tensor_spec.TensorSpec(tensor.shape, tensor.dtype, tensor.name) - for tensor in self._cloned_model.train_function.outputs - ] - return [ - self._cloned_model.train_function.updates_op, - tpu_ops.outfeed_enqueue_tuple( - self._cloned_model.train_function.outputs, - name='outfeed-enqueue-train') - ] - elif is_test: - self._cloned_model._make_test_function() - self._outfeed_spec = [ - tensor_spec.TensorSpec(tensor.shape, tensor.dtype, tensor.name) - for tensor in self._cloned_model.test_function.outputs - ] - return [ - tpu_ops.outfeed_enqueue_tuple( - self._cloned_model.test_function.outputs, - name='outfeed-enqueue-test') - ] - elif is_predict: - self._cloned_model._make_predict_function() - self._outfeed_spec = [ - tensor_spec.TensorSpec(tensor.shape, tensor.dtype, tensor.name) - for tensor in self._cloned_model.predict_function.outputs - ] - return [ - tpu_ops.outfeed_enqueue_tuple( - self._cloned_model.predict_function.outputs, - name='outfeed-enqueue-predict', - ) - ] - else: - assert False, 'Unexpected execution mode: %s' % self.execution_mode - - # Capture outfeed metadata computed during the rewrite. - self._outfeed_spec = None - - # Generate out TPU operations using `tpu.split_compile_and_replicate`. - # `compile_op` can be used to test the TPU model compiles before execution. - # `execute op` replicates `_model_fn` `num_replicas` times, with each shard - # running on a different logical core. - compile_op, execute_op = tpu.split_compile_and_replicate( - _model_fn, inputs=[[] for _ in range(self._tpu_assignment.num_towers)]) - - # Generate CPU side operations to enqueue features/labels and dequeue - # outputs from the model call. - sized_infeed = infeed_manager.build_infeed_from_input_specs( - input_specs, self.execution_mode) - # Build output ops. - outfeed_op = [] - for shard_id in range(self._tpu_assignment.num_towers): - with ops.device( - '/job:%s/device:CPU:0' % self._tpu_assignment.worker_name): - outfeed_op.extend( - tpu_ops.outfeed_dequeue_tuple( - dtypes=[spec.dtype for spec in self._outfeed_spec], - shapes=[spec.shape for spec in self._outfeed_spec], - name='outfeed-dequeue-%s-%d' % (self.execution_mode, shard_id), - device_ordinal=shard_id)) - - return TPUModelOp( - compile_op, - execute_op, - infeed_tensors=sized_infeed.sharded_infeed_tensors, - infeed_op=sized_infeed.infeed_ops, - outfeed_op=outfeed_op) - - def _test_model_compiles(self, tpu_model_ops): - """Verifies that the given TPUModelOp can be compiled via XLA.""" - logging.info('Started compiling') - start_time = time.time() - - result = K.get_session().run(tpu_model_ops.compile_op) - proto = tpu_compilation_result.CompilationResultProto() - proto.ParseFromString(result) - if proto.status_error_message: - raise RuntimeError('Compilation failed: {}'.format( - proto.status_error_message)) - - end_time = time.time() - logging.info('Finished compiling. Time elapsed: %s secs', - end_time - start_time) - - def _lookup_infeed_manager(self, inputs): - """Return an existing manager, or construct a new InfeedManager for inputs. - - _lookup_infeed_manager will return an existing InfeedManager if one has been - previously assigned for this model and input. If not, it will construct a - new TPUNumpyInfeedManager. - - Args: - inputs: A NumPy input to the model. - - Returns: - A `TPUInfeedManager` object to manage infeeds for this input. - """ - if inputs is None: - return None - - for x, mgr in self.model._numpy_to_infeed_manager_list: - if inputs[0] is x: - return mgr - - return TPUNumpyInfeedManager(self.model._tpu_assignment) - - def _tpu_model_ops_for_input_specs(self, input_specs, infeed_manager): - """Looks up the corresponding `TPUModelOp` for a given `input_specs`. - - It instantiates a new copy of the model for each unique input shape. - - Args: - input_specs: The specification of the inputs to train on. - infeed_manager: The infeed manager responsible for feeding in data. - - Returns: - A `TPUModelOp` instance that can be used to execute a step of the model. - """ - if input_specs is None or infeed_manager is None: - # Note: this condition is possible during the prologue or epilogue of the - # pipelined loop. - return None - - # XLA requires every operation in the graph has a fixed shape. To - # handle varying batch sizes we recompile a new sub-graph for each - # unique input shape. - shape_key = tuple([tuple(spec.shape.as_list()) for spec in input_specs]) - if shape_key not in self._compilation_cache: - logging.info( - 'New input shapes; (re-)compiling: mode=%s ' - '(# of cores %d), %s', self.execution_mode, - self._tpu_assignment.num_towers, input_specs) - new_tpu_model_ops = self._specialize_model(input_specs, - infeed_manager) - self._compilation_cache[shape_key] = new_tpu_model_ops - self._test_model_compiles(new_tpu_model_ops) - - return self._compilation_cache[shape_key] - - def _construct_input_tensors_and_inputs(self, inputs): - """Returns input tensors and numpy array inputs corresponding to `inputs`. - - Args: - inputs: NumPy inputs. - - Returns: - A tuple of `input_tensors`, and `inputs`. - """ - if inputs is None: - # Note: this condition is possible during the prologue or epilogue of the - # pipelined loop. - return None, None - - if isinstance(inputs[-1], int): - # Remove the learning_phase flag at the end. We currently hard code the - # learning_phase in TPUFunction. - inputs = inputs[:-1] - - if (self.execution_mode == model_fn_lib.ModeKeys.TRAIN or - self.execution_mode == model_fn_lib.ModeKeys.EVAL): - # Strip sample weight from inputs. - input_tensors = self.model._feed_inputs + self.model._feed_targets - else: - input_tensors = self.model._feed_inputs - - inputs = inputs[:len(input_tensors)] - input_tensors, inputs = ( - _inject_tpu_inputs_for_infeed( - self._tpu_assignment, self.execution_mode, - self._core_id_place_holder, input_tensors, inputs)) - return input_tensors, inputs - - def _process_outputs(self, outfeed_outputs): - """Processes the outputs of a model function execution. - - Args: - outfeed_outputs: The sharded outputs of the TPU computation. - - Returns: - The aggregated outputs of the TPU computation to be used in the rest of - the model execution. - """ - # TODO(xiejw): Decide how to reduce outputs, or discard all but first. - if self.execution_mode == model_fn_lib.ModeKeys.PREDICT: - outputs = [[] for _ in range(len(self._outfeed_spec))] - outputs_per_replica = len(self._outfeed_spec) - - for i in range(self._tpu_assignment.num_towers): - output_group = outfeed_outputs[i * outputs_per_replica:(i + 1) * - outputs_per_replica] - for j in range(outputs_per_replica): - outputs[j].append(output_group[j]) - - return [np.concatenate(group) for group in outputs] - else: - return outfeed_outputs[:len(outfeed_outputs) // - self._tpu_assignment.num_towers] - - def __call__(self, inputs): - """__call__ executes the function on the computational hardware. - - It handles executing infeed, and preprocessing in addition to executing the - model on the TPU hardware. - - Note: `__call__` has a sibling method `pipeline_run` which performs the same - operations, but with software pipelining. - - Args: - inputs: The inputs to use to train. - - Returns: - The output of the computation for the given mode it is executed in. - - Raises: - RuntimeError: If there is an inappropriate use of the function. - """ - assert isinstance(inputs, list) - - infeed_manager = self._lookup_infeed_manager(inputs) - input_tensors, inputs = self._construct_input_tensors_and_inputs(inputs) - infeed_instance = infeed_manager.make_infeed_instance(inputs) - del inputs # To avoid accident usage. - input_specs = infeed_instance.make_input_specs(input_tensors) - tpu_model_ops = self._tpu_model_ops_for_input_specs(input_specs, - infeed_manager) - infeed_dict = infeed_instance.make_feed_dict(tpu_model_ops) - - # Initialize our TPU weights on the first compile. - self.model._initialize_weights(self._cloned_model) - - _, _, outfeed_outputs = K.get_session().run([ - tpu_model_ops.infeed_op, tpu_model_ops.execute_op, - tpu_model_ops.outfeed_op - ], infeed_dict) - return self._process_outputs(outfeed_outputs) - - def pipeline_run(self, cur_step_inputs, next_step_inputs): - """pipeline_run executes the function on the computational hardware. - - pipeline_run performs the same computation as __call__, however it runs the - infeed in a software pipelined fashion compared to the on-device execution. - - Note: it is the responsibility of the caller to call `pipeline_run` in the - following sequence: - - Once with `cur_step_inputs=None` and `next_step_inputs=list(...)` - - `n` times with `cur_step_inputs` and `next_step_inputs` as `list`s - - Once with `cur_step_inputs=list(...)` and `next_step_inputs=None` - Additionally, it is the responsibility of the caller to pass - `next_step_inputs` as `cur_step_inputs` on the next invocation of - `pipeline_run`. - - Args: - cur_step_inputs: The current step's inputs. - next_step_inputs: The next step's inputs. - - Returns: - The output of the computation for the given mode it is executed in. - - Raises: - RuntimeError: If there is an inappropriate use of the function. - """ - # Software pipelined case. - next_step_infeed_manager = self._lookup_infeed_manager(next_step_inputs) - cur_step_infeed_manager = self._lookup_infeed_manager(cur_step_inputs) - - if (next_step_infeed_manager is not None and - cur_step_infeed_manager is not None): - assert type(next_step_infeed_manager) is type(cur_step_infeed_manager) - - next_input_tensors, next_step_inputs = ( - self._construct_input_tensors_and_inputs(next_step_inputs)) - cur_input_tensors, cur_step_inputs = ( - self._construct_input_tensors_and_inputs(cur_step_inputs)) - - cur_infeed_instance = None - if cur_step_infeed_manager: - cur_infeed_instance = cur_step_infeed_manager.make_infeed_instance( - cur_step_inputs) - next_infeed_instance = None - if next_step_infeed_manager: - next_infeed_instance = next_step_infeed_manager.make_infeed_instance( - next_step_inputs) - - del cur_step_inputs # Avoid accidental re-use. - del next_step_inputs # Avoid accidental re-use. - - cur_tpu_model_ops = None - next_tpu_model_ops = None - infeed_dict = None - - if cur_infeed_instance and cur_input_tensors and cur_step_infeed_manager: - cur_input_specs = cur_infeed_instance.make_input_specs(cur_input_tensors) - cur_tpu_model_ops = self._tpu_model_ops_for_input_specs( - cur_input_specs, cur_step_infeed_manager) - - if (next_infeed_instance and next_input_tensors and - next_step_infeed_manager): - next_input_specs = next_infeed_instance.make_input_specs( - next_input_tensors) - next_tpu_model_ops = self._tpu_model_ops_for_input_specs( - next_input_specs, next_step_infeed_manager) - infeed_dict = next_infeed_instance.make_feed_dict(next_tpu_model_ops) - - # Initialize our TPU weights on the first compile. - self.model._initialize_weights(self._cloned_model) - - if next_tpu_model_ops and cur_tpu_model_ops: - _, _, outfeed_outputs = K.get_session().run([ - next_tpu_model_ops.infeed_op, cur_tpu_model_ops.execute_op, - cur_tpu_model_ops.outfeed_op - ], infeed_dict) - return self._process_outputs(outfeed_outputs) - - if cur_tpu_model_ops: - _, outfeed_outputs = K.get_session().run( - [cur_tpu_model_ops.execute_op, cur_tpu_model_ops.outfeed_op]) - return self._process_outputs(outfeed_outputs) - - if next_tpu_model_ops: - K.get_session().run(next_tpu_model_ops.infeed_op, infeed_dict) - return None - raise RuntimeError('Internal error: both current & next tpu_model_ops ' - 'were None') - - -class KerasTPUModel(models.Model): - """TPU compatible Keras model wrapper.""" - - def __init__(self, cpu_model, strategy): - super(models.Model, self).__init__( # pylint: disable=bad-super-call - inputs=cpu_model.inputs, - outputs=cpu_model.outputs, - name=cpu_model.name, - ) - if tf2.enabled(): - raise RuntimeError( - 'Keras support is now deprecated in support of TPU Strategy. ' - 'Please follow the distribution strategy guide on tensorflow.org ' - 'to migrate to the 2.0 supported version.') - else: - logging.warning( - 'Keras support is now deprecated in support of TPU Strategy. ' - 'Please follow the distribution strategy guide on tensorflow.org ' - 'to migrate to the 2.0 supported version.') - # Create a mapping from numpy arrays to infeed managers. - # Note: uses a list of tuples instead of a map because numpy arrays are - # not hashable. - self._numpy_to_infeed_manager_list = [] - - # Add distribution specific arguments since we don't call the Model init. - self._distribution_strategy = None - self._compile_distribution = None - - self.predict_function = None - self.test_function = None - self.train_function = None - self._stateful_metric_functions = [] - - cluster_resolver = strategy._tpu_cluster_resolver - self._tpu_name_or_address = cluster_resolver.get_master() - self._cpu_model = cpu_model - self._tpu_assignment = strategy._make_assignment_for_model(cpu_model) - self._tpu_model = None - self._tpu_weights_initialized = False - - # If the input CPU model has already been compiled, compile our TPU model - # immediately. - if self._cpu_model.optimizer: - self.compile( - self._cpu_model.optimizer, - self._cpu_model.loss, - self._cpu_model._compile_metrics, - self._cpu_model.loss_weights, - self._cpu_model.sample_weight_mode, - self._cpu_model._compile_weighted_metrics, - self._cpu_model.target_tensors, - ) - - # This flag must be disabled upon model mutation, such as changing the model - # layers or recompiling the model to use a different optimizer. New function - # definitions are generated whenever this flag is disabled, ensuring that - # internal graph functions are always using the current model structure. - # - # Requires declaration here because this constructor skips the - # Model constructor. - self._built_graph_functions = False - - def get_config(self): - return { - 'cpu_model': self._cpu_model, - 'tpu_name_or_address': self._tpu_name_or_address, - 'tpu_assignment': self._tpu_assignment, - } - - def compile(self, - optimizer, - loss=None, - metrics=None, - loss_weights=None, - sample_weight_mode=None, - weighted_metrics=None, - target_tensors=None, - **kwargs): - if sample_weight_mode: - raise ValueError('sample_weight_mode not supported for TPU execution.') - if weighted_metrics: - raise ValueError('weighted_metrics not supported for TPU execution.') - if target_tensors: - raise ValueError('target_tensors is not supported for TPU execution.') - - self._cpu_model.compile( - _clone_optimizer(optimizer), loss, - metrics_module.clone_metrics(metrics), loss_weights, sample_weight_mode, - metrics_module.clone_metrics(weighted_metrics), target_tensors, - **kwargs) - - super(KerasTPUModel, self).compile(optimizer, loss, metrics, loss_weights, - sample_weight_mode, weighted_metrics, - target_tensors, **kwargs) - - def fit(self, - x=None, - y=None, - batch_size=None, - epochs=1, - verbose=1, - callbacks=None, - validation_split=0., - validation_data=None, - shuffle=True, - class_weight=None, - sample_weight=None, - initial_epoch=0, - steps_per_epoch=None, - validation_steps=None, - **kwargs): - if context.executing_eagerly(): - raise EnvironmentError('KerasTPUModel currently does not support eager ' - 'mode.') - - with _tpu_session_context(): - assert not self._numpy_to_infeed_manager_list # Ensure empty. - - infeed_managers = [] # Managers to clean up at the end of the fit call. - if isinstance(x, dataset_ops.DatasetV2): - # TODO(b/111413240): Support taking a tf.data.Dataset directly. - raise ValueError( - 'Taking a Dataset directly is not yet supported. Please ' - 'wrap your dataset construction code in a function and ' - 'pass that to fit instead. For examples, see: ' - 'https://github.com/tensorflow/tpu/tree/master/models/experimental' - '/keras') - if callable(x): - with ops.device( - '/job:%s/device:CPU:0' % self._tpu_assignment.worker_name): - dataset = x() - if steps_per_epoch is None: - raise ValueError('When using tf.data as input to a model, you ' - 'should specify the steps_per_epoch argument.') - if y is not None: - raise ValueError('When using tf.data as input to a model, y must ' - 'be None') - infeed_manager = TPUDatasetInfeedManager( - dataset, self._tpu_assignment, model_fn_lib.ModeKeys.TRAIN) - # Use dummy numpy inputs for the rest of Keras' shape checking. We - # intercept them when building the model. - x = infeed_manager.dummy_x - y = infeed_manager.dummy_y - infeed_managers.append((x, infeed_manager)) - - if isinstance(validation_data, dataset_ops.DatasetV2): - # TODO(b/111413240): Support taking a tf.data.Dataset directly. - raise ValueError( - 'Taking a Dataset directly is not yet supported. Please ' - 'wrap your dataset construction code in a function and ' - 'pass that to fit instead. For examples, see: ' - 'https://github.com/tensorflow/tpu/tree/master/models/experimental' - '/keras') - if callable(validation_data): - dataset = validation_data() - if validation_steps is None: - raise ValueError('When using tf.data as validation for a model, you ' - 'should specify the validation_steps argument.') - infeed_manager = TPUDatasetInfeedManager(dataset, self._tpu_assignment, - model_fn_lib.ModeKeys.EVAL) - # Use dummy numpy inputs for the rest of Keras' shape checking. We - # intercept them when building the model. - val_x = infeed_manager.dummy_x - val_y = infeed_manager.dummy_y - infeed_managers.append((val_x, infeed_manager)) - validation_data = (val_x, val_y) - - self._numpy_to_infeed_manager_list = infeed_managers - try: - pipeline = kwargs.get('_pipeline', True) - if '_pipeline' in kwargs: - kwargs.pop('_pipeline') - if not pipeline: - logging.info('Running non-pipelined training loop (`_pipeline=%s`).', - pipeline) - return super(KerasTPUModel, self).fit( - x, y, batch_size, epochs, verbose, callbacks, validation_split, - validation_data, shuffle, class_weight, sample_weight, - initial_epoch, steps_per_epoch, validation_steps, **kwargs) - return self._pipeline_fit(x, y, batch_size, epochs, verbose, callbacks, - validation_split, validation_data, shuffle, - class_weight, sample_weight, initial_epoch, - steps_per_epoch, validation_steps, **kwargs) - finally: - self._numpy_to_infeed_manager_list = [] - - def evaluate(self, - x=None, - y=None, - batch_size=None, - verbose=1, - sample_weight=None, - steps=None): - original_numpy_to_infeed_manager_list = [] - if self._numpy_to_infeed_manager_list: - # evaluate call may be executed as callbacks during the training. In this - # case, _numpy_to_infeed_manager_list is not empty, so save it for - # recovery at the end of evaluate call. - original_numpy_to_infeed_manager_list = self._numpy_to_infeed_manager_list - self._numpy_to_infeed_manager_list = [] - - with _tpu_session_context(): - # Managers to clean up at the end of the evaluate call. - infeed_managers = [] - if isinstance(x, dataset_ops.DatasetV2): - # TODO(b/111413240): Support taking a tf.data.Dataset directly. - raise ValueError( - 'Taking a Dataset directly is not yet supported. Please ' - 'wrap your dataset construction code in a function and ' - 'pass that to fit instead. For examples, see: ' - 'https://github.com/tensorflow/tpu/tree/master/models/experimental' - '/keras') - if callable(x): - dataset = x() - if steps is None: - raise ValueError('When using tf.data as input to a model, you ' - 'should specify the steps argument.') - if y is not None: - raise ValueError('When using tf.data as input to a model, y must be ' - 'None') - infeed_manager = TPUDatasetInfeedManager(dataset, self._tpu_assignment, - model_fn_lib.ModeKeys.EVAL) - # Use dummy numpy inputs for the rest of Keras' shape checking. We - # intercept them when building the model. - x = infeed_manager.dummy_x - y = infeed_manager.dummy_y - infeed_managers.append((x, infeed_manager)) - - self._numpy_to_infeed_manager_list = infeed_managers - try: - return super(KerasTPUModel, self).evaluate(x, y, batch_size, verbose, - sample_weight, steps) - finally: - self._numpy_to_infeed_manager_list = ( - original_numpy_to_infeed_manager_list) - - def _pipeline_fit(self, x, y, batch_size, epochs, verbose, callbacks, - validation_split, validation_data, shuffle, class_weight, - sample_weight, initial_epoch, steps_per_epoch, - validation_steps, **kwargs): - # Similar to super.fit(...), but modified to support software pipelining. - - # Backwards compatibility - if batch_size is None and steps_per_epoch is None: - batch_size = 32 - # Legacy support - if 'nb_epoch' in kwargs: - logging.warning('The `nb_epoch` argument in `fit` has been renamed ' - '`epochs`.') - epochs = kwargs.pop('nb_epoch') - if kwargs: - raise TypeError('Unrecognized keyword arguments: ' + str(kwargs)) - - # Validate and standardize user data - x, y, sample_weights = self._standardize_user_data( - x, - y, - sample_weight=sample_weight, - class_weight=class_weight, - batch_size=batch_size, - check_steps=True, - steps_name='steps_per_epoch', - steps=steps_per_epoch, - validation_split=validation_split) - - # Prepare validation data - x, y, val_x, val_y, val_sample_weights = self._prepare_validation_data( - validation_data, validation_split, validation_steps, x, y, - sample_weights, batch_size) - return self._pipeline_fit_loop( - x, - y, - sample_weights=sample_weights, - batch_size=batch_size, - epochs=epochs, - verbose=verbose, - callbacks=callbacks, - val_inputs=val_x, - val_targets=val_y, - val_sample_weights=val_sample_weights, - shuffle=shuffle, - initial_epoch=initial_epoch, - steps_per_epoch=steps_per_epoch, - validation_steps=validation_steps) - - def _pipeline_fit_loop(self, - inputs, - targets, - sample_weights, - batch_size, - epochs, - verbose, - callbacks, - val_inputs, - val_targets, - val_sample_weights, - shuffle, - initial_epoch, - steps_per_epoch, - validation_steps): - self._make_train_function() - sample_weights = sample_weights or [] - val_sample_weights = val_sample_weights or [] - if not isinstance(K.learning_phase(), int): - ins = inputs + targets + sample_weights + [1] - else: - ins = inputs + targets + sample_weights - - do_validation = False - if val_inputs: - do_validation = True - if (steps_per_epoch is None and verbose and inputs and - hasattr(inputs[0], 'shape') and hasattr(val_inputs[0], 'shape')): - print('Train on %d samples, validate on %d samples' % - (inputs[0].shape[0], val_inputs[0].shape[0])) - - if validation_steps: - do_validation = True - if steps_per_epoch is None: - raise ValueError('Can only use `validation_steps` when doing step-wise ' - 'training, i.e. `steps_per_epoch` must be set.') - - num_training_samples = training_utils.check_num_samples( - ins, batch_size, steps_per_epoch, 'steps_per_epoch') - count_mode = 'steps' if steps_per_epoch else 'samples' - callbacks = cbks.configure_callbacks( - callbacks, - self, - do_validation=do_validation, - batch_size=batch_size, - epochs=epochs, - steps_per_epoch=steps_per_epoch, - samples=num_training_samples, - verbose=verbose, - count_mode=count_mode) - - if num_training_samples is not None: - index_array = np.arange(num_training_samples) - - # To prevent a slowdown, we find beforehand the arrays that need conversion. - feed = self._feed_inputs + self._feed_targets + self._feed_sample_weights - indices_for_conversion_to_dense = [] - for i in range(len(feed)): - if issparse is not None and issparse(ins[i]) and not K.is_sparse(feed[i]): - indices_for_conversion_to_dense.append(i) - - callbacks.on_train_begin() - for epoch in range(initial_epoch, epochs): - # Reset stateful metrics - for m in self.metrics: - m.reset_states() - # Update callbacks - callbacks.on_epoch_begin(epoch) - epoch_logs = {} - if steps_per_epoch is not None: - # Step-wise fit loop. - self._pipeline_fit_loop_step_wise( - ins=ins, - callbacks=callbacks, - steps_per_epoch=steps_per_epoch, - epochs=epochs, - do_validation=do_validation, - val_inputs=val_inputs, - val_targets=val_targets, - val_sample_weights=val_sample_weights, - validation_steps=validation_steps, - epoch_logs=epoch_logs) - else: - # Sample-wise fit loop. - self._pipeline_fit_loop_sample_wise( - ins=ins, - callbacks=callbacks, - index_array=index_array, - shuffle=shuffle, - batch_size=batch_size, - num_training_samples=num_training_samples, - indices_for_conversion_to_dense=indices_for_conversion_to_dense, - do_validation=do_validation, - val_inputs=val_inputs, - val_targets=val_targets, - val_sample_weights=val_sample_weights, - validation_steps=validation_steps, - epoch_logs=epoch_logs) - - callbacks.on_epoch_end(epoch, epoch_logs) - if callbacks.model.stop_training: - break - callbacks.on_train_end() - return self.history - - def _pipeline_fit_loop_sample_wise(self, - ins, - callbacks, - index_array, - shuffle, - batch_size, - num_training_samples, - indices_for_conversion_to_dense, - do_validation, - val_inputs, - val_targets, - val_sample_weights, - validation_steps, - epoch_logs): - f = self.train_function - if shuffle == 'batch': - index_array = training_utils.batch_shuffle(index_array, batch_size) - elif shuffle: - np.random.shuffle(index_array) - batches = make_batches(num_training_samples, batch_size) - - ins_last_batch = None - last_batch_logs = None - batch_index = 0 - - for batch_index, (batch_start, batch_end) in enumerate(batches): - batch_ids = index_array[batch_start:batch_end] - try: - if isinstance(ins[-1], int): - # Do not slice the training phase flag. - ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] - else: - ins_batch = slice_arrays(ins, batch_ids) - except TypeError: - raise TypeError('TypeError while preparing batch. If using HDF5 ' - 'input data, pass shuffle="batch".') - - # Pipeline batch logs - next_batch_logs = {} - next_batch_logs['batch'] = batch_index - next_batch_logs['size'] = len(batch_ids) - if batch_index > 0: - # Callbacks operate one step behind in software pipeline. - callbacks.on_batch_begin(batch_index - 1, last_batch_logs) - for i in indices_for_conversion_to_dense: - ins_batch[i] = ins_batch[i].toarray() - - outs = f.pipeline_run( - cur_step_inputs=ins_last_batch, next_step_inputs=ins_batch) - ins_last_batch = ins_batch - - if batch_index == 0: - assert outs is None - else: - if not isinstance(outs, list): - outs = [outs] - for l, o in zip(self.metrics_names, outs): - last_batch_logs[l] = o # pylint: disable=unsupported-assignment-operation - callbacks.on_batch_end(batch_index - 1, last_batch_logs) - if callbacks.model.stop_training: - return - last_batch_logs = next_batch_logs - - # Final batch - callbacks.on_batch_begin(batch_index, last_batch_logs) - outs = f.pipeline_run(cur_step_inputs=ins_last_batch, next_step_inputs=None) - if not isinstance(outs, list): - outs = [outs] - for l, o in zip(self.metrics_names, outs): - last_batch_logs[l] = o - callbacks.on_batch_end(batch_index, last_batch_logs) - if callbacks.model.stop_training: - return - - if do_validation: - val_outs = training_arrays.test_loop( - self, - val_inputs, - val_targets, - sample_weights=val_sample_weights, - batch_size=batch_size, - steps=validation_steps, - verbose=0) - if not isinstance(val_outs, list): - val_outs = [val_outs] - # Same labels assumed. - for l, o in zip(self.metrics_names, val_outs): - epoch_logs['val_' + l] = o - - def _pipeline_fit_loop_step_wise(self, - ins, - callbacks, - steps_per_epoch, - epochs, - do_validation, - val_inputs, - val_targets, - val_sample_weights, - validation_steps, - epoch_logs): - f = self.train_function - - # Loop prologue - try: - outs = f.pipeline_run(cur_step_inputs=None, next_step_inputs=ins) - assert outs is None # Function shouldn't return anything! - except errors.OutOfRangeError: - logging.warning('Your dataset iterator ran out of data on the first step ' - 'of the epoch, preventing further training. Check to ' - 'make sure your paths are correct and you have ' - 'permissions to read the files. Skipping validation') - - for step_index in range(steps_per_epoch): - batch_logs = {'batch': step_index, 'size': 1} - callbacks.on_batch_begin(step_index, batch_logs) - try: - if step_index < steps_per_epoch - 1: - next_step_inputs = ins - else: - next_step_inputs = None - outs = f.pipeline_run( - cur_step_inputs=ins, next_step_inputs=next_step_inputs) - except errors.OutOfRangeError: - logging.warning('Your dataset iterator ran out of data; ' - 'interrupting training. Make sure that your ' - 'dataset can generate at least `steps_per_batch * ' - 'epochs` batches (in this case, %d batches). You ' - 'may need to use the repeat() function when ' - 'building your dataset.' % steps_per_epoch * epochs) - break - - if not isinstance(outs, list): - outs = [outs] - for l, o in zip(self.metrics_names, outs): - batch_logs[l] = o - - callbacks.on_batch_end(step_index, batch_logs) - if callbacks.model.stop_training: - break - - if do_validation: - val_outs = training_arrays.test_loop( - self, - val_inputs, - val_targets, - sample_weights=val_sample_weights, - steps=validation_steps, - verbose=0) - if not isinstance(val_outs, list): - val_outs = [val_outs] - # Same labels assumed. - for l, o in zip(self.metrics_names, val_outs): - epoch_logs['val_' + l] = o - - def _prepare_validation_data(self, validation_data, validation_split, - validation_steps, x, y, sample_weights, - batch_size): - """Prepares the validation dataset. - - Args: - validation_data: The validation data (if provided) - validation_split: The validation split (if provided) - validation_steps: The validation steps (if provided) - x: The main training data x (if provided) - y: The main training data y (if provided) - sample_weights: The sample weights (if provided) - batch_size: The training batch size (if provided) - - Returns: - A 5-tuple of (x, y, val_x, val_y, val_sample_weights). - - Raises: - ValueError: If the provided arguments are not compatible with - `KerasTPUModel`. - """ - # Note: this is similar to a section of $tf/python/keras/engine/training.py - # It differns in that tf.data objects are not allowed to be passed directly. - # Additionally, it handles validating shapes & types appropriately for use - # in TPUs. - if validation_data: - if (isinstance(validation_data, iterator_ops.Iterator) or - isinstance(validation_data, iterator_ops.IteratorV2) or - isinstance(validation_data, dataset_ops.DatasetV2)): - raise ValueError('KerasTPUModel cannot handle a Dataset or Iterator ' - 'for validation_data. Please instead pass a function ' - 'that returns a `tf.data.Dataset`.') - if len(validation_data) == 2: - val_x, val_y = validation_data # pylint: disable=unpacking-non-sequence - val_sample_weight = None - elif len(validation_data) == 3: - val_x, val_y, val_sample_weight = validation_data # pylint: disable=unpacking-non-sequence - else: - raise ValueError('When passing a `validation_data` argument, it must ' - 'contain either 2 items (x_val, y_val), or 3 items ' - '(x_val, y_val, val_sample_weights). However we ' - 'received `validation_data=%s`' % validation_data) - val_x, val_y, val_sample_weights = self._standardize_user_data( - val_x, - val_y, - sample_weight=val_sample_weight, - batch_size=batch_size, - steps=validation_steps) - elif validation_split and 0. < validation_split < 1.: - if training_utils.has_symbolic_tensors(x): - raise ValueError('If your data is in the form of symbolic tensors, you ' - 'cannot use `validation_split`.') - if hasattr(x[0], 'shape'): - split_at = int(x[0].shape[0] * (1. - validation_split)) - else: - split_at = int(len(x[0]) * (1. - validation_split)) - - x, val_x = (slice_arrays(x, 0, split_at), slice_arrays(x, split_at)) - y, val_y = (slice_arrays(y, 0, split_at), slice_arrays(y, split_at)) - sample_weights, val_sample_weights = ( - slice_arrays(sample_weights, 0, split_at), - slice_arrays(sample_weights, split_at) - ) - elif validation_steps: - val_x = [] - val_y = [] - val_sample_weights = [] - else: - val_x = None - val_y = None - val_sample_weights = None - - return x, y, val_x, val_y, val_sample_weights - - def predict(self, - x, - batch_size=None, - verbose=0, - steps=None, - max_queue_size=10, - workers=1, - use_multiprocessing=False): - with _tpu_session_context(): - return super(KerasTPUModel, self).predict( - x, - batch_size=batch_size, - verbose=verbose, - steps=steps, - max_queue_size=max_queue_size, - workers=workers, - use_multiprocessing=use_multiprocessing) - - @property - def optimizer(self): - if self._tpu_model: - return self._tpu_model.optimizer - return self._cpu_model.optimizer - - @optimizer.setter - def optimizer(self, optimizer): - self._optimizer = optimizer - - @property - def metrics(self): - if self._tpu_model: - return self._tpu_model.metrics - return self._stateful_metric_functions - - @metrics.setter - def metrics(self, metrics): - self._stateful_metric_functions = metrics - - def _make_train_function(self): - if not self.train_function: - self.train_function = TPUFunction( - self, - model_fn_lib.ModeKeys.TRAIN, - tpu_assignment=self._tpu_assignment) - - return self.train_function - - def _make_test_function(self): - if not self.test_function: - self.test_function = TPUFunction( - self, model_fn_lib.ModeKeys.EVAL, tpu_assignment=self._tpu_assignment) - return self.test_function - - def _make_predict_function(self): - if not self.predict_function: - self.predict_function = TPUFunction( - self, - model_fn_lib.ModeKeys.PREDICT, - tpu_assignment=self._tpu_assignment) - return self.predict_function - - def _initialize_weights(self, cloned_model): - """Initialize TPU weights. - - This is called on the first compile of the TPU model (first call to - fit/predict/evaluate). - - Args: - cloned_model: `keras.Model`, TPU model to initialize. - """ - if self._tpu_weights_initialized: - return - - self._tpu_model = cloned_model - self._tpu_weights_initialized = True - - weights = self._cpu_model.get_weights() - - if isinstance(self.cpu_optimizer, keras_optimizers.TFOptimizer): - cpu_optimizer_config = {} - else: - cpu_optimizer_config = self.cpu_optimizer.get_config() - - logging.info('Setting weights on TPU model.') - cloned_model.set_weights(weights) - if self._tpu_model.optimizer is None: - # tpu_model may not be compiled, e.g., loading weights and then predict. - return - for k, v in six.iteritems(cpu_optimizer_config): - if k == 'name': - continue - opt_var = getattr(self._tpu_model.optimizer, k) - if isinstance(opt_var, variables.Variable): - logging.info('CPU -> TPU %s: %s {%s}', k, v, K.get_value(opt_var)) - K.get_session().run(opt_var.assign(v)) - else: - logging.warning('Cannot update non-variable config: %s', k) - - @property - def cpu_optimizer(self): - return self._cpu_model.optimizer - - def sync_to_cpu(self): - """Copy weights from the CPU, returning a synchronized CPU model.""" - if not self._tpu_weights_initialized: - return self._cpu_model - - logging.info('Copying TPU weights to the CPU') - tpu_weights = self._tpu_model.get_weights() - - # TFOptimizers have no configurable options - if isinstance(self.cpu_optimizer, keras_optimizers.TFOptimizer): - tpu_optimizer_config = {} - else: - tpu_optimizer_config = self._tpu_model.optimizer.get_config() - - self._cpu_model.set_weights(tpu_weights) - for k, v in six.iteritems(tpu_optimizer_config): - logging.info('TPU -> CPU %s: %s', k, v) - if k == 'name': - continue - opt_var = getattr(self.cpu_optimizer, k) - if isinstance(opt_var, variables.Variable): - K.get_session().run(opt_var.assign(v)) - else: - logging.warning('Cannot update non-variable config: %s', k) - - return self._cpu_model - - def get_weights(self): - return self.sync_to_cpu().get_weights() - - def save_weights(self, *args, **kw): - return self.sync_to_cpu().save_weights(*args, **kw) - - def save(self, *args, **kw): - return self.sync_to_cpu().save(*args, **kw) - - def set_weights(self, weights): - # We may not have a TPU model available if we haven't run fit/predict, so - # we can't directly set the TPU weights here. - # Instead, reset CPU model weights and force TPU re-initialization at the - # next call. - self._cpu_model.set_weights(weights) - self._tpu_weights_initialized = False - - def load_weights(self, filepath, by_name=False): - self._cpu_model.load_weights(filepath, by_name) - self._tpu_weights_initialized = False - - -# pylint: disable=bad-continuation -def _validate_shapes(model): - """Validate that all layers in `model` have constant shape.""" - for layer in model.layers: - if isinstance(layer.input_shape, tuple): - input_shapes = [layer.input_shape] - else: - input_shapes = layer.input_shape - - if isinstance(layer.output_shape, tuple): - output_shapes = [layer.output_shape] - else: - output_shapes = layer.output_shape - - for shape in input_shapes + output_shapes: - for dim in shape[1:]: - if dim is None: - raise ValueError( - """ -Layer %(layer)s has a variable shape in a non-batch dimension. TPU models must -have constant shapes for all operations. - -You may have to specify `input_length` for RNN/TimeDistributed layers. - -Layer: %(layer)s -Input shape: %(input_shape)s -Output shape: %(output_shape)s - """ % { - 'layer': layer, - 'input_shape': layer.input_shape, - 'output_shape': layer.output_shape - }) - - -# pylint: enable=bad-continuation - - -@deprecated( - '2019-02-20', 'Switch to tf.contrib.distribute.TPUStrategy. ' - 'https://www.tensorflow.org/api_docs/python/tf/contrib/distribute/DistributionStrategy' -) -def tpu_model(model, strategy=None): - """Copy `model` along with weights to the TPU. - - Returns a TPU model. - - Usage: - ``` - a = Input(shape=(32,)) - b = Dense(32)(a) - model = Model(inputs=a, outputs=b) - - # If `num_cores_per_host` is greater than one, batch parallelism will be used - # to run on multiple TPU cores. - strategy = keras_support.TPUDistributionStrategy(tpu_cluster_resolver) - model = keras_support.tpu_model(model, strategy) - model.compile( - optimizer=tf.compat.v1.train.GradientDescentOptimizer(learning_rate=1.0), - ...) - ``` - - Args: - model: A `tf.keras.Model` instance. - strategy: `TPUDistributionStrategy`. The strategy to use for replicating - model across multiple TPU cores. - - Returns: - A new `KerasTPUModel` instance. - """ - _validate_shapes(model) - # TODO(xiejw): Validate TPU model. TPUModel only? - # TODO(xiejw): Validate replicas. Full or 1. Shall we allow subset? - # TODO(xiejw): Adds reduction option. - - if strategy is None: - strategy = TPUDistributionStrategy() - else: - if not isinstance(strategy, TPUDistributionStrategy): - raise TypeError( - '`strategy` must have type `tf.contrib.tpu.TPUDistributionStrategy`. ' - 'Got: {}'.format(type(strategy))) - - # If the model has already been initialized, grab the optimizer configuration - # and model weights before entering the TPU session. - if model.optimizer: - if (isinstance(model.optimizer, keras_optimizers.Optimizer) and not - isinstance(model.optimizer, keras_optimizers.TFOptimizer)): - optimizer_config = model.optimizer.get_config() - else: - optimizer_config = None - model_weights = model.get_weights() - else: - model_weights = None - - setup_tpu_session(strategy._tpu_cluster_resolver) - - # Force initialization of the CPU model in the TPU session. - cpu_model = models.clone_model(model) - if model.optimizer: - cpu_model.compile( - _clone_optimizer(model.optimizer, optimizer_config), - model.loss, - metrics_module.clone_metrics(model._compile_metrics), - model.loss_weights, - model.sample_weight_mode, - metrics_module.clone_metrics(model._compile_weighted_metrics), - ) - - if model_weights: - cpu_model.set_weights(model_weights) - cpu_model.reset_states() - - return KerasTPUModel(cpu_model=cpu_model, strategy=strategy) diff --git a/tensorflow/contrib/tpu/python/tpu/keras_tpu_variables.py b/tensorflow/contrib/tpu/python/tpu/keras_tpu_variables.py deleted file mode 100644 index de425626c81..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/keras_tpu_variables.py +++ /dev/null @@ -1,346 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Distributed variable implementation for TPUs. - -N.B. This is an experimental feature that should only be used for Keras support. - -It is unsupported and will be removed in favor of Distribution Strategy soon. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import contextlib - -import numpy as np - -from tensorflow.python.client import session as session_lib -from tensorflow.python.framework import dtypes as dtypes_module -from tensorflow.python.framework import ops -from tensorflow.python.keras import backend -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_resource_variable_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope - - -@contextlib.contextmanager -def _handle_graph(handle): - with handle.graph.as_default(): - yield - - -def _enclosing_tpu_context(): - # pylint: disable=protected-access - context = ops.get_default_graph()._get_control_flow_context() - # pylint: enable=protected-access - while context is not None and not isinstance( - context, control_flow_ops.XLAControlFlowContext): - context = context.outer_context - return context - - -class ReplicatedVariable(object): - """A replicated variable for use on TPUs. - - When accessed inside a tpu.replicate() context, this variable acts as if it - is a single variable whose handle is a replicated input to the computation. - - Outside a tpu.replicate() context currently this object has pretty murky - semantics, especially with respect to things such as - * initialization - * colocation. - """ - - def __init__(self, name, variables): - self._name = name - self._primary_var = variables[0] - self._common_name = self._primary_var.name.split(":")[0] - self._vars = variables - self._cached_value = None - self._dtype = variables[0].dtype - - @property - def handle(self): - tpu_context = _enclosing_tpu_context() - if tpu_context is None: - return self._primary_var.handle - - return tpu_context.get_replicated_var_handle(self._name, self._vars) - - @contextlib.contextmanager - def _assign_dependencies(self): - """Makes assignments depend on the cached value, if any. - - This prevents undefined behavior with reads not ordered wrt writes. - - Yields: - None. - """ - if self._cached_value is not None: - with ops.control_dependencies([self._cached_value]): - yield - else: - yield - - @property - def initializer(self): - return control_flow_ops.group([v.initializer for v in self._vars]) - - @property - def graph(self): - return self._primary_var.graph - - @property - def _shared_name(self): - return self._common_name - - @property - def _unique_id(self): - return self._primary_var._unique_id # pylint: disable=protected-access - - @property - def name(self): - return self._name - - @property - def dtype(self): - return self._primary_var.dtype - - @property - def shape(self): - return self._primary_var.shape - - def get_shape(self): - return self._primary_var.get_shape() - - def to_proto(self, export_scope=None): - return self._primary_var.to_proto(export_scope=export_scope) - - @property - def constraint(self): - return None - - @property - def op(self): - return self.get().op - - @property - def is_tensor_like(self): - return True - - def _read_variable_op(self): - if _enclosing_tpu_context() is None: - return self._primary_var.read_value() - v = gen_resource_variable_ops.read_variable_op(self.handle, self._dtype) - return v - - def read_value(self): - return self._read_variable_op() - - def is_initialized(self, name=None): - return self._vars[0].is_initialized(name=name) - - def __getitem__(self, *args): - return self.read_value().__getitem__(*args) - - def assign(self, value, use_locking=None, name=None, read_value=False): - """Assign `value` to all replicas. - - Outside of the tpu.rewrite context, assign explicitly to all replicas. - Inside of the tpu.rewrite context, assigns to the local replica. - - Arguments: - value: Tensor to assign - use_locking: ignored - name: ignored - read_value: return the value from the assignment - Returns: - Assignment operation, or new value of the variable if `read_value` is True - """ - del use_locking - if _enclosing_tpu_context() is None: - assign_ops = [] - with self._assign_dependencies(): - for var in self._vars: - assign_ops.append(var.assign(value, use_locking=None, name=name)) - - if read_value: - with ops.control_dependencies(assign_ops): - return self.read_value() - else: - return control_flow_ops.group(assign_ops) - - with _handle_graph(self.handle), self._assign_dependencies(): - value_tensor = ops.convert_to_tensor(value, dtype=self.dtype) - assign_op = gen_resource_variable_ops.assign_variable_op( - self.handle, value_tensor, name=name) - if read_value: - return self._read_variable_op() - return assign_op - - def assign_add(self, delta, use_locking=None, name=None, read_value=True): - del use_locking - with _handle_graph(self.handle), self._assign_dependencies(): - assign_add_op = gen_resource_variable_ops.assign_add_variable_op( - self.handle, - ops.convert_to_tensor(delta, dtype=self.dtype), - name=name) - if read_value: - return self._read_variable_op() - return assign_add_op - - def assign_sub(self, delta, use_locking=None, name=None, read_value=True): - del use_locking - with _handle_graph(self.handle), self._assign_dependencies(): - assign_sub_op = gen_resource_variable_ops.assign_sub_variable_op( - self.handle, - ops.convert_to_tensor(delta, dtype=self.dtype), - name=name) - if read_value: - return self._read_variable_op() - return assign_sub_op - - def get(self): - return self._primary_var - - @property - def _in_graph_mode(self): - return self._primary_var._in_graph_mode # pylint: disable=protected-access - - def _should_act_as_resource_variable(self): - """Pass resource_variable_ops.is_resource_variable check.""" - pass - - def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False): - """Converts a variable to a tensor.""" - # pylint: disable=protected-access - if _enclosing_tpu_context() is None: - return self._primary_var._dense_var_to_tensor(dtype, name, as_ref) - # pylint: enable=protected-access - if dtype is not None and dtype != self.dtype: - return math_ops.cast(self._read_variable_op(), dtype) - if as_ref: - return self.handle - else: - return self.read_value() - - -# Register a conversion function which reads the value of the variable, -# allowing instances of the class to be used as tensors. -def _tensor_conversion(var, dtype=None, name=None, as_ref=False): - return var._dense_var_to_tensor(dtype=dtype, name=name, as_ref=as_ref) # pylint: disable=protected-access - - -def replicated_fetch_function(var): - # pylint: disable=protected-access - return ([var._dense_var_to_tensor()], lambda v: v[0]) - # pylint: enable=protected-access - - -ops.register_tensor_conversion_function(ReplicatedVariable, _tensor_conversion) -ops.register_dense_tensor_like_type(ReplicatedVariable) -session_lib.register_session_run_conversion_functions( - ReplicatedVariable, replicated_fetch_function) - - -def replicated_scope(num_replicas): - """Variable scope for constructing replicated variables.""" - - def _replicated_variable_getter(getter, name, *args, **kwargs): - """Getter that constructs replicated variables.""" - collections = kwargs.pop("collections", None) - if collections is None: - collections = [ops.GraphKeys.GLOBAL_VARIABLES] - kwargs["collections"] = [] - - variables = [] - index = {} - for i in range(num_replicas): - replica_name = "{}/{}".format(name, i) - with ops.device("device:TPU:{}".format(i)): - v = getter(*args, name=replica_name, **kwargs) - variables.append(v) - index[i] = v - result = ReplicatedVariable(name, variables) - - g = ops.get_default_graph() - # If "trainable" is True, next_creator() will add the member variables - # to the TRAINABLE_VARIABLES collection, so we manually remove - # them and replace with the MirroredVariable. We can't set - # "trainable" to False for next_creator() since that causes functions - # like implicit_gradients to skip those variables. - if kwargs.get("trainable", True): - collections.append(ops.GraphKeys.TRAINABLE_VARIABLES) - l = g.get_collection_ref(ops.GraphKeys.TRAINABLE_VARIABLES) - for v in index.values(): - if v in l: - l.remove(v) - g.add_to_collections(collections, result) - - return result - - return variable_scope.variable_scope( - "", custom_getter=_replicated_variable_getter) - - -@contextlib.contextmanager -def replicated_variable_for_optimizer(num_replicas): - """Context manager for optimizer weights. Overrides K.variable.""" - if num_replicas == 1: - yield - return - - try: - old_v = backend.variable - - def opt_variable(value, dtype=None, name=None, constraint=None): - """Instantiates a variable and returns it.""" - if dtype is None: - dtype = backend.floatx() - - variables = [] - for i in range(num_replicas): - # Keras holds the variables in optimizer class instance , so the name - # does not matter here. ResourceVariable constructor will find a unique - # name (including name=None) for each replica. - with ops.device("device:TPU:{}".format(i)): - v = resource_variable_ops.ResourceVariable( - value, - dtype=dtypes_module.as_dtype(dtype), - name=name, - constraint=constraint) - variables.append(v) - name = "replicate_{}_{}".format("variable" if name is None else name, - ops.uid()) - v = ReplicatedVariable(name, variables) - - # pylint: disable=protected-access - - if isinstance(value, np.ndarray): - v._keras_shape = value.shape - elif hasattr(value, "shape"): - v._keras_shape = backend.int_shape(value) - v._uses_learning_phase = False - backend.track_variable(v) - return v - - backend.variable = opt_variable - yield - - finally: - backend.variable = old_v diff --git a/tensorflow/contrib/tpu/python/tpu/session_support.py b/tensorflow/contrib/tpu/python/tpu/session_support.py deleted file mode 100644 index ed8f9525c9b..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/session_support.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.session_support import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py deleted file mode 100644 index 73db253fd79..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.tensor_tracer import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/topology.py b/tensorflow/contrib/tpu/python/tpu/topology.py deleted file mode 100644 index 5bf805752cf..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/topology.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import,redefined-builtin -from tensorflow.python.tpu.topology import * -# pylint: enable=wildcard-import,unused-import,redefined-builtin diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py deleted file mode 100644 index 5364b20f231..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/tpu.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import,redefined-builtin -from tensorflow.python.tpu.tpu import * -# used by tests -from tensorflow.python.tpu.tpu import _TPU_REPLICATE_ATTR -# pylint: enable=wildcard-import,unused-import,redefined-builtin diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py deleted file mode 100644 index 2c9bce0bca2..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow_estimator.python.estimator.tpu.tpu_config import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py deleted file mode 100644 index 573f49b2b9b..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow_estimator.python.estimator.tpu.tpu_context import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py deleted file mode 100644 index cb38a8f1a6b..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.tpu_embedding import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_embedding_gradient.py b/tensorflow/contrib/tpu/python/tpu/tpu_embedding_gradient.py deleted file mode 100644 index 308adc77e9a..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/tpu_embedding_gradient.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.tpu_embedding_gradient import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py deleted file mode 100644 index 0ee490681e4..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import,redefined-builtin -from tensorflow_estimator.python.estimator.tpu.tpu_estimator import * -# used by tests -from tensorflow_estimator.python.estimator.tpu.tpu_estimator import _clone_export_output_with_tensors -from tensorflow_estimator.python.estimator.tpu.tpu_estimator import _create_global_step -from tensorflow_estimator.python.estimator.tpu.tpu_estimator import _export_output_to_tensors -from tensorflow_estimator.python.estimator.tpu.tpu_estimator import _get_scaffold -from tensorflow_estimator.python.estimator.tpu.tpu_estimator import _Inputs -from tensorflow_estimator.python.estimator.tpu.tpu_estimator import _ITERATIONS_PER_LOOP_VAR -from tensorflow_estimator.python.estimator.tpu.tpu_estimator import _TPU_ENQUEUE_OPS -from tensorflow_estimator.python.estimator.tpu.tpu_estimator import _TPU_ESTIMATOR -from tensorflow_estimator.python.estimator.tpu.tpu_estimator import _TPU_TRAIN_OP -# pylint: enable=wildcard-import,unused-import,redefined-builtin diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_feed.py b/tensorflow/contrib/tpu/python/tpu/tpu_feed.py deleted file mode 100644 index af2542ea852..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/tpu_feed.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import,redefined-builtin -from tensorflow.python.tpu.tpu_feed import * -# used by tests -from tensorflow.python.tpu.tpu_feed import _PartitionedInfeedQueue -# pylint: enable=wildcard-import,unused-import,redefined-builtin diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_function.py b/tensorflow/contrib/tpu/python/tpu/tpu_function.py deleted file mode 100644 index f2755c6979c..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/tpu_function.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.tpu_function import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_optimizer.py b/tensorflow/contrib/tpu/python/tpu/tpu_optimizer.py deleted file mode 100644 index ca58e78d7b3..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/tpu_optimizer.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.tpu_optimizer import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_sharding.py b/tensorflow/contrib/tpu/python/tpu/tpu_sharding.py deleted file mode 100644 index 93c52335a58..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/tpu_sharding.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import,redefined-builtin -from tensorflow.python.tpu.tpu_sharding import * -# pylint: enable=wildcard-import,unused-import,redefined-builtin diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py b/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py deleted file mode 100644 index 258d34ddaf5..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.tpu_system_metadata import * -# used by tests -from tensorflow.python.tpu.tpu_system_metadata import _query_tpu_system_metadata -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/training_loop.py b/tensorflow/contrib/tpu/python/tpu/training_loop.py deleted file mode 100644 index 673359b232d..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/training_loop.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.tpu.training_loop import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/python/tpu/util.py b/tensorflow/contrib/tpu/python/tpu/util.py deleted file mode 100644 index 6e0da240466..00000000000 --- a/tensorflow/contrib/tpu/python/tpu/util.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Stub file to maintain backwards compatibility.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=wildcard-import,unused-import -from tensorflow_estimator.python.estimator.tpu.util import * -# pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/contrib/tpu/tpu_estimator.md b/tensorflow/contrib/tpu/tpu_estimator.md deleted file mode 100644 index 552febd80bd..00000000000 --- a/tensorflow/contrib/tpu/tpu_estimator.md +++ /dev/null @@ -1,238 +0,0 @@ -# Using the Estimator API with TPUs - - -This document describes how to train a TensorFlow model on TPUs using the -Estimator API. If you are interested in the hardware itself, check out the -[Cloud TPU documentation](https://cloud.google.com/tpu/docs). - -The TPU Estimator simplifies running models on a Cloud TPU by automatically -handling numerous low-level hardware-specific details - -[TOC] - -## Introduction to Estimator - -[TensorFlow -tutorials](https://www.tensorflow.org/extend/estimators) cover the Estimator -API. At a high-level, the Estimator API provides: - -* `Estimator.train()` - train a model on a given input for a fixed number of - steps. -* `Estimator.evaluate()` - evaluate the model on a test set. -* `Estimator.predict()` - run inference using the trained model. -* `Estimator.export_savedmodel()` - export your model for serving. - -In addition, `Estimator` includes default behavior common to training jobs, -such as saving and restoring checkpoints, creating summaries for TensorBoard, -etc. - -`Estimator` requires you to write a `model_fn` and an `input_fn`, which -correspond to the model and input portions of your TensorFlow graph. - -The following code demonstrates using `TPUEstimator` with MNIST example to -handle training: - - def model_fn(features, labels, mode, params): - """A simple CNN.""" - del params # unused - - input_layer = tf.reshape(features, [-1, 28, 28, 1]) - conv1 = tf.layers.conv2d( - inputs=input_layer, filters=32, kernel_size=[5, 5], padding="same", - activation=tf.nn.relu) - pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2) - conv2 = tf.layers.conv2d( - inputs=pool1, filters=64, kernel_size=[5, 5], - padding="same", activation=tf.nn.relu) - pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2) - pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64]) - dense = tf.layers.dense(inputs=pool2_flat, units=128, activation=tf.nn.relu) - dropout = tf.layers.dropout( - inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN) - logits = tf.layers.dense(inputs=dropout, units=10) - onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=10) - - loss = tf.losses.softmax_cross_entropy( - onehot_labels=onehot_labels, logits=logits) - - learning_rate = tf.train.exponential_decay( - FLAGS.learning_rate, tf.train.get_global_step(), 100000, 0.96) - - optimizer = tpu_optimizer.CrossShardOptimizer( - tf.train.GradientDescentOptimizer(learning_rate=learning_rate)) - - train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step()) - return tpu_estimator.TPUEstimatorSpec(mode=mode, loss=loss, train_op=train_op) - - - def get_input_fn(filename): - """Returns an `input_fn` for train and eval.""" - - def input_fn(params): - """An input_fn to parse 28x28 images from filename using tf.data.""" - batch_size = params["batch_size"] - - def parser(serialized_example): - """Parses a single tf.Example into image and label tensors.""" - features = tf.parse_single_example( - serialized_example, - features={ - "image_raw": tf.FixedLenFeature([], tf.string), - "label": tf.FixedLenFeature([], tf.int64), - }) - image = tf.decode_raw(features["image_raw"], tf.uint8) - image.set_shape([28 * 28]) - # Normalize the values of the image from the range [0, 255] to [-0.5, 0.5] - image = tf.cast(image, tf.float32) * (1. / 255) - 0.5 - label = tf.cast(features["label"], tf.int32) - return image, label - - dataset = tf.data.TFRecordDataset( - filename, buffer_size=FLAGS.dataset_reader_buffer_size) - dataset = dataset.map(parser).cache().repeat().batch( - batch_size, drop_remainder=True) - return dataset - return input_fn - - - def main(unused_argv): - - tf.logging.set_verbosity(tf.logging.INFO) - - run_config = tpu_config.RunConfig( - master=FLAGS.master, - model_dir=FLAGS.model_dir, - session_config=tf.ConfigProto( - allow_soft_placement=True, log_device_placement=True), - tpu_config=tpu_config.TPUConfig(FLAGS.iterations, FLAGS.num_shards),) - - estimator = tpu_estimator.TPUEstimator( - model_fn=model_fn, - use_tpu=FLAGS.use_tpu, - train_batch_size=FLAGS.batch_size, - eval_batch_size=FLAGS.batch_size, - config=run_config) - - estimator.train(input_fn=get_input_fn(FLAGS.train_file), - max_steps=FLAGS.train_steps) - - -Although this code is quite simple by appearance, there are some new -concepts to learn for using `TPU`s. The next section will cover the most -important details. - -## New Concepts Related to TPU/TPUEstimator - -TF programs run with `TPU Estimator` use an [in-graph -replication](https://www.tensorflow.org/deploy/distributed) approach. - -In-graph replication (also known as single-session replication) differs from -the between-graph replication (also known as multi-session replication) -training typically used in distributed TensorFlow. The major -differences include: - -1. The TensorFlow Session master is not local anymore. The user python program - creates one single graph that is replicated across all the cores in the Cloud - TPU. The typical configuration today sets the TensorFlow session master to be - the first worker. - -1. The input pipeline is placed on remote hosts (instead of local) to ensure the - training examples can be fed as fast as possible to TPU system. All queue-based - input pipelines do not work effectively. Dataset (tf.data) is - required. - -1. Workers in the TPU system operate in synchronous fashion, and each perform - the same step at the same time. - -Regarding programming model, _"The programmer picks a (large) batch size B and -writes the program (and sets hyperparameters) based on that batch size. The -system distributes the computation across the available devices." - -To align these, `TPUEstimator` wraps the computation (the `model_fn`) and -distributes it to all available TPU chips. - -To summarize: - -- The `input_fn` models the input pipeline running on remote host CPU. Use - `tf.data` to program the input Ops. `input_fn` is expected to be invoked - multiple times when using TPU pods. Each handles one device's input of the - global batch. The shard batch size should be retrieved from - `params['batch_size']`. We plan to provide better abstraction about the - sharding mechanism for `tf.data` to remove the `params['batch_size']`. - -- The `model_fn` models the computation which will be replicated and distributed - to all TPU chips. It should only contains ops that are supported by TPUs. - -## Convert from Vanilla Estimator to TPUEstimator - -It is always recommended to port a small, simple model first to make sure that -you are familiar with the basic concepts of `TPUEstimator` and test end-to-end -behavior. Once your simple model runs, gradually add more functionality. -In addition, there are several sample models, available at -[github.com/tensorflow/tpu](https://github.com/tensorflow/tpu). - -To convert your code from the vanilla `Estimator` class to use TPUs, change the -following (note some of the details may change over time): - -- Switch from `tf.estimator.RunConfig` to `tf.contrib.tpu.RunConfig`. -- Set the `TPUConfig` (part of the `tf.contrib.tpu.RunConfig`) to specify the - `iterations_per_loop`, number of iterations to run on the TPU device for one - `session.run` call (per training loop), and `num_shards`, the number of shards - (typically the number of TPU cores you’re running on). TPUs run a number of - iterations of the training loop before returning to host. Until all iterations - on the TPU device are run, no checkpoints or summaries will be saved. In the - future, we’ll choose a reasonable default. -- In `model_fn`, use `tf.contrib.tpu.CrossShardOptimizer` to wrap your - optimizer. Example: - - optimizer = tpu_optimizer.CrossShardOptimizer( - tf.train.GradientDescentOptimizer(learning_rate=learning_rate)) - -- Switch from `tf.estimator.Estimator` to `tf.contrib.tpu.TPUEstimator`. - -The default `RunConfig` will save summaries for TensorBoard every 100 steps and -write checkpoints every 10 minutes. - - -## FAQ - -### Why `tf.data` is Required for the Input Pipeline - -There are two reasons: - -1. The user code runs on the client, while the TPU computation is executed on - the `worker`. Input pipeline ops must be placed on the remote worker for - good performance. Only `tf.data` (Dataset) supports this. - -1. In order to amortize the TPU launch cost, the model train step is wrapped in - a `tf.while_loop`, such that one `Session.run` actually runs many iterations - for one train loop. To remove network back and forth, the input pipeline - in the future will be wrapped in a `tf.while_loop` and be placed on the - corresponding `worker`. Withou this, unnecessary network latency becomes - the performance bottleneck for models with short training-step times, or in - environments where network latency is higher. Only `tf.data` can be wrapped - by a `tf.while_loop`. - - -### How to add other CPU Ops into Graph -As `model_fn` only allows TPU Ops for computation, the easier workaround to add -CPU Ops into Graph is: - -1. Create a [SessionRunHook](https://www.tensorflow.org/api_docs/python/tf/train/SessionRunHook). -1. Modify the graph in the `def begin(self)`, -1. Pass the hook to `TPUEstimator.train`. - -### Running On GCP Cloud TPUs -To run your models on GCP Cloud TPUs refer to the [Cloud Documentation](https://cloud.google.com/tpu/docs/tutorials/mnist). -Refer to this link for all [Cloud TPU documentation](https://cloud.google.com/tpu/docs). - - -### Profiling -You can profile the `worker` by using instructions as specified in the [Cloud TPU Tools](https://cloud.google.com/tpu/docs/cloud-tpu-tools). - - -### Is `int64` supported? -`int64` is not supported by TPU. Cast to int32 if applicable. The only exception -is global step, which relies on `assign_add`. `int64` support for global step -is added to ensure checkpoint compatibility between `TPUEstimator` and non-TPU -`Estimator`. diff --git a/tensorflow/contrib/training/BUILD b/tensorflow/contrib/training/BUILD deleted file mode 100644 index 1161f52cfbc..00000000000 --- a/tensorflow/contrib/training/BUILD +++ /dev/null @@ -1,306 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. -# Placeholder for Google-internal load statements. -load("//tensorflow/core/platform:default/build_config.bzl", "tf_proto_library") -load("//tensorflow:tensorflow.bzl", "py_test") - -package( - default_visibility = [ - "//tensorflow:internal", - ], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -py_library( - name = "training_py", - srcs = [ - "__init__.py", - "python/__init__.py", - "python/training/__init__.py", - "python/training/bucket_ops.py", - "python/training/device_setter.py", - "python/training/evaluation.py", - "python/training/feeding_queue_runner.py", - "python/training/hparam.py", - "python/training/resample.py", - "python/training/sampling_ops.py", - "python/training/sequence_queueing_state_saver.py", - "python/training/training.py", - "python/training/tuner.py", - ], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - ":protos_all_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:check_ops", - "//tensorflow/python:clip_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:layers_base", - "//tensorflow/python:logging_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:script_ops", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:state_ops", - "//tensorflow/python:string_ops", - "//tensorflow/python:summary", - "//tensorflow/python:tensor_array_ops", - "//tensorflow/python:tensor_util", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/data", - "//tensorflow/python/estimator:estimator_py", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_test( - name = "device_setter_test", - size = "small", - srcs = ["python/training/device_setter_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":training_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "sequence_queueing_state_saver_test", - size = "medium", - srcs = ["python/training/sequence_queueing_state_saver_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":training_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:string_ops", - "//third_party/py/numpy", - ], -) - -py_test( - name = "batch_sequences_with_states_test", - size = "medium", - srcs = ["python/training/batch_sequences_with_states_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["manual"], - deps = [ - ":training_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:sparse_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:string_ops", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "feeding_queue_runner_test", - size = "medium", - srcs = ["python/training/feeding_queue_runner_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_ops", - "//tensorflow/python:session", - "//tensorflow/python:training", - "//tensorflow/python/estimator:estimator_py", - "//third_party/py/numpy", - ], -) - -py_test( - name = "hparam_test", - size = "small", - srcs = ["python/training/hparam_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":training_py", - "//tensorflow/python:client_testlib", - ], -) - -py_test( - name = "resample_test", - size = "small", - srcs = ["python/training/resample_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":training_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "sampling_ops_test", - size = "medium", - srcs = ["python/training/sampling_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - deps = [ - ":training_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:logging_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "sampling_ops_threading_test", - size = "small", - srcs = ["python/training/sampling_ops_threading_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = [ - "manual", - "notsan", - ], - deps = [ - ":training_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:training", - "//tensorflow/python:variables", - ], -) - -py_test( - name = "bucket_ops_test", - size = "medium", - srcs = ["python/training/bucket_ops_test.py"], - python_version = "PY2", - srcs_version = "PY2AND3", - tags = ["manual"], - deps = [ - ":training_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) - -py_test( - name = "evaluation_test", - size = "small", - srcs = ["python/training/evaluation_test.py"], - python_version = "PY2", - shard_count = 3, - srcs_version = "PY2AND3", - tags = [ - "manual", - "notap", # Disabling until b/33000128 and b/33040312 are fixed. - ], - deps = [ - ":training_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/contrib/losses:losses_py", - "//tensorflow/contrib/metrics:metrics_py", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_seed", - "//tensorflow/python:session", - "//tensorflow/python:state_ops", - "//tensorflow/python:summary", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - -py_test( - name = "training_test", - size = "medium", - srcs = ["python/training/training_test.py"], - python_version = "PY2", - shard_count = 8, - srcs_version = "PY2AND3", - tags = ["notsan"], - deps = [ - ":training_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/contrib/layers:layers_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:gradients", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_seed", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//tensorflow/python/ops/losses", - "//third_party/py/numpy", - ], -) - -tf_proto_library( - name = "protos_all", - srcs = glob(["**/*.proto"]), - cc_api_version = 2, - visibility = ["//visibility:public"], -) diff --git a/tensorflow/contrib/training/__init__.py b/tensorflow/contrib/training/__init__.py deleted file mode 100644 index 87ce57ef060..00000000000 --- a/tensorflow/contrib/training/__init__.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Training and input utilities. - -See -[Contrib Training](https://tensorflow.org/api_guides/python/contrib.training) -guide. - -@@batch_sequences_with_states -@@NextQueuedSequenceBatch -@@SequenceQueueingStateSaver -@@rejection_sample -@@resample_at_rate -@@stratified_sample -@@weighted_resample -@@bucket -@@bucket_by_sequence_length -@@RandomStrategy -@@GreedyLoadBalancingStrategy -@@byte_size_load_fn -@@FailureTolerator -@@rejection_sample -@@stratified_sample -@@resample_at_rate -@@weighted_resample -@@HParams -@@HParamDef -@@parse_values -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import,wildcard-import -from tensorflow.contrib.training.python.training.bucket_ops import * -from tensorflow.contrib.training.python.training.device_setter import * -from tensorflow.contrib.training.python.training.evaluation import checkpoints_iterator -from tensorflow.contrib.training.python.training.evaluation import evaluate_once -from tensorflow.contrib.training.python.training.evaluation import evaluate_repeatedly -from tensorflow.contrib.training.python.training.evaluation import get_or_create_eval_step -from tensorflow.contrib.training.python.training.evaluation import StopAfterNEvalsHook -from tensorflow.contrib.training.python.training.evaluation import SummaryAtEndHook -from tensorflow.contrib.training.python.training.evaluation import wait_for_new_checkpoint -from tensorflow.contrib.training.python.training.feeding_queue_runner import FeedingQueueRunner -from tensorflow.contrib.training.python.training.hparam import * -from tensorflow.contrib.training.python.training.resample import * -from tensorflow.contrib.training.python.training.sampling_ops import * -from tensorflow.contrib.training.python.training.sequence_queueing_state_saver import * -from tensorflow.contrib.training.python.training.training import add_gradients_summaries -from tensorflow.contrib.training.python.training.training import clip_gradient_norms -from tensorflow.contrib.training.python.training.training import clip_gradient_norms_fn -from tensorflow.contrib.training.python.training.training import create_train_op -from tensorflow.contrib.training.python.training.training import multiply_gradients -from tensorflow.contrib.training.python.training.training import train -from tensorflow.contrib.training.python.training.tuner import Tuner -# pylint: enable=unused-import,wildcard-import - -from tensorflow.python.util.all_util import remove_undocumented - -# Allow explicitly imported symbols. Symbols imported with * must also be -# whitelisted here or in the module docstring above. -_allowed_symbols = [ - 'checkpoints_iterator', 'evaluate_once', 'evaluate_repeatedly', - 'FeedingQueueRunner', 'get_or_create_eval_step', 'StopAfterNEvalsHook', - 'SummaryAtEndHook', 'wait_for_new_checkpoint', 'add_gradients_summaries', - 'clip_gradient_norms', 'clip_gradient_norms_fn', 'create_train_op', - 'multiply_gradients', 'train'] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/training/python/__init__.py b/tensorflow/contrib/training/python/__init__.py deleted file mode 100644 index 52e83069cb0..00000000000 --- a/tensorflow/contrib/training/python/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/training/python/training/__init__.py b/tensorflow/contrib/training/python/training/__init__.py deleted file mode 100644 index 52e83069cb0..00000000000 --- a/tensorflow/contrib/training/python/training/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function diff --git a/tensorflow/contrib/training/python/training/batch_sequences_with_states_test.py b/tensorflow/contrib/training/python/training/batch_sequences_with_states_test.py deleted file mode 100644 index afeef978f31..00000000000 --- a/tensorflow/contrib/training/python/training/batch_sequences_with_states_test.py +++ /dev/null @@ -1,647 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.batch_sequences_with_states.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -import numpy as np - -from tensorflow.contrib.training.python.training import sequence_queueing_state_saver as sqss -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import string_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import coordinator -from tensorflow.python.training import queue_runner_impl -from tensorflow.python.training import saver - - -class BatchSequencesWithStatesTest(test.TestCase): - - def setUp(self): - super(BatchSequencesWithStatesTest, self).setUp() - self.value_length = 4 - ind1 = np.array([ - [0, 0], - [1, 0], [1, 3], [1, 4], - [3, 2], [3, 3]]) - val1 = np.array([0, 10, 13, 14, 32, 33]) - shape1 = np.array([self.value_length, 6]) - sp_tensor1 = sparse_tensor.SparseTensor( - array_ops.constant(ind1, dtypes.int64), - array_ops.constant(val1, dtypes.int64), - array_ops.placeholder_with_default(shape1, shape=[2])) - ind2 = np.array([ - [0, 0, 1], - [0, 1, 0], - [0, 1, 2], - [1, 0, 3], - [1, 1, 0], - [1, 1, 1], - [1, 1, 2], - [1, 2, 2]]) - val2 = np.array([1, 10, 12, 103, 150, 149, 150, 122]) - shape2 = np.array([self.value_length, 3, 4]) - sp_tensor2 = sparse_tensor.SparseTensor( - array_ops.constant(ind2, dtypes.int64), - array_ops.constant(val2, dtypes.int64), - array_ops.placeholder_with_default(shape2, shape=[3])) - sp_tensor3 = sparse_tensor.SparseTensor( - array_ops.constant([[1, 9], [2, 2], [2, 10]], dtypes.int64), - array_ops.constant([7, 15, 2], dtypes.int64), - array_ops.constant([5, 12], dtypes.int64) - ) - self.sp_tensor3_expected = sparse_tensor.SparseTensorValue( - [[0, 1, 9], [0, 2, 2], [0, 2, 10], [1, 1, 9], [1, 2, 2], [1, 2, 10]], - [7, 15, 2, 7, 15, 2], - [2, 5, 12] - ) - self.batch_size = 2 - self.key = string_ops.string_join([ - "key_", string_ops.as_string( - math_ops.cast(10000 * random_ops.random_uniform(()), dtypes.int32)) - ]) - self.sequences = { - "seq1": np.random.rand(self.value_length, 5), - "seq2": np.random.rand(self.value_length, 4, 2), - "seq3": sp_tensor1, - "seq4": sp_tensor2} - self.context = { - "context1": [3, 4], - "sp_context": sp_tensor3} - self.initial_states = { - "state1": np.random.rand(6, 7), - "state2": np.random.rand(8) - } - - def _prefix(self, key_value): - return set( - [s.decode("ascii").split(":")[0].encode("ascii") for s in key_value]) - - def _testBasics(self, num_unroll, length, pad, - expected_seq1_batch1, expected_seq2_batch1, - expected_seq1_batch2, expected_seq2_batch2, - expected_seq3_batch1, expected_seq3_batch2, - expected_seq4_batch1, expected_seq4_batch2, - key=None, make_keys_unique=False): - - with self.cached_session() as sess: - next_batch = sqss.batch_sequences_with_states( - input_key=key if key is not None else self.key, - input_sequences=self.sequences, - input_context=self.context, - input_length=length, - initial_states=self.initial_states, - num_unroll=num_unroll, - batch_size=self.batch_size, - num_threads=3, - # to enforce that we only move on to the next examples after finishing - # all segments of the first ones. - capacity=2, - pad=pad, - make_keys_unique=make_keys_unique, - make_keys_unique_seed=9) - - state1 = next_batch.state("state1") - state2 = next_batch.state("state2") - state1_update = next_batch.save_state("state1", state1 + 1) - state2_update = next_batch.save_state("state2", state2 - 1) - - # Make sure queue runner with SQSS is added properly to meta graph def. - # Saver requires at least one variable. - v0 = variables.Variable(10.0, name="v0") - ops.add_to_collection("variable_collection", v0) - variables.global_variables_initializer() - save = saver.Saver([v0]) - test_dir = os.path.join(test.get_temp_dir(), "sqss_test") - filename = os.path.join(test_dir, "metafile") - meta_graph_def = save.export_meta_graph(filename) - qr_saved = meta_graph_def.collection_def[ops.GraphKeys.QUEUE_RUNNERS] - self.assertTrue(qr_saved.bytes_list.value is not None) - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(coord=coord) - - # Step 1 - (key_value, next_key_value, seq1_value, seq2_value, seq3_value, - seq4_value, context1_value, context2_value, state1_value, state2_value, - length_value, _, _) = sess.run( - (next_batch.key, next_batch.next_key, next_batch.sequences["seq1"], - next_batch.sequences["seq2"], next_batch.sequences["seq3"], - next_batch.sequences["seq4"], next_batch.context["context1"], - next_batch.context["sp_context"], state1, state2, next_batch.length, - state1_update, state2_update)) - expected_first_keys = set([b"00000_of_00002"]) - expected_second_keys = set([b"00001_of_00002"]) - expected_final_keys = set([b"STOP"]) - - self.assertEqual(expected_first_keys, self._prefix(key_value)) - self.assertEqual(expected_second_keys, self._prefix(next_key_value)) - self.assertAllEqual( - np.tile(self.context["context1"], (self.batch_size, 1)), - context1_value) - self.assertAllEqual(self.sp_tensor3_expected.indices, - context2_value.indices) - self.assertAllEqual(self.sp_tensor3_expected.values, - context2_value.values) - self.assertAllEqual(self.sp_tensor3_expected.dense_shape, - context2_value.dense_shape) - self.assertAllEqual(expected_seq1_batch1, seq1_value) - self.assertAllEqual(expected_seq2_batch1, seq2_value) - self.assertAllEqual(expected_seq3_batch1.indices, seq3_value.indices) - self.assertAllEqual(expected_seq3_batch1.values, seq3_value.values) - self.assertAllEqual(expected_seq3_batch1.dense_shape, - seq3_value.dense_shape) - self.assertAllEqual(expected_seq4_batch1.indices, seq4_value.indices) - self.assertAllEqual(expected_seq4_batch1.values, seq4_value.values) - self.assertAllEqual(expected_seq4_batch1.dense_shape, - seq4_value.dense_shape) - self.assertAllEqual( - np.tile(self.initial_states["state1"], (self.batch_size, 1, 1)), - state1_value) - self.assertAllEqual( - np.tile(self.initial_states["state2"], (self.batch_size, 1)), - state2_value) - self.assertAllEqual(length_value, [num_unroll, num_unroll]) - - # Step 2 - (key_value, next_key_value, seq1_value, seq2_value, seq3_value, - seq4_value, context1_value, context2_value, state1_value, state2_value, - length_value, _, _) = sess.run( - (next_batch.key, next_batch.next_key, next_batch.sequences["seq1"], - next_batch.sequences["seq2"], next_batch.sequences["seq3"], - next_batch.sequences["seq4"], next_batch.context["context1"], - next_batch.context["sp_context"], state1, state2, next_batch.length, - state1_update, state2_update)) - - self.assertEqual(expected_second_keys, self._prefix(key_value)) - self.assertEqual(expected_final_keys, self._prefix(next_key_value)) - self.assertAllEqual( - np.tile(self.context["context1"], (self.batch_size, 1)), - context1_value) - self.assertAllEqual(self.sp_tensor3_expected.indices, - context2_value.indices) - self.assertAllEqual(self.sp_tensor3_expected.values, - context2_value.values) - self.assertAllEqual(self.sp_tensor3_expected.dense_shape, - context2_value.dense_shape) - self.assertAllEqual(expected_seq1_batch2, seq1_value) - self.assertAllEqual(expected_seq2_batch2, seq2_value) - self.assertAllEqual(expected_seq3_batch2.indices, seq3_value.indices) - self.assertAllEqual(expected_seq3_batch2.values, seq3_value.values) - self.assertAllEqual(expected_seq3_batch2.dense_shape, - seq3_value.dense_shape) - self.assertAllEqual(expected_seq4_batch2.indices, seq4_value.indices) - self.assertAllEqual(expected_seq4_batch2.values, seq4_value.values) - self.assertAllEqual(expected_seq4_batch2.dense_shape, - seq4_value.dense_shape) - self.assertAllEqual(1 + np.tile(self.initial_states["state1"], - (self.batch_size, 1, 1)), state1_value) - self.assertAllEqual(-1 + np.tile(self.initial_states["state2"], - (self.batch_size, 1)), state2_value) - self.assertAllEqual([1, 1], length_value) - - coord.request_stop() - coord.join(threads, stop_grace_period_secs=2) - - def _testBasicPadding(self, pad, key=None, make_keys_unique=False): - num_unroll = 2 # Divisor of value_length - so no padding necessary. - expected_seq1_batch1 = np.tile( - self.sequences["seq1"][np.newaxis, 0:num_unroll, :], - (self.batch_size, 1, 1)) - expected_seq2_batch1 = np.tile( - self.sequences["seq2"][np.newaxis, 0:num_unroll, :, :], - (self.batch_size, 1, 1, 1)) - expected_seq1_batch2 = np.tile( - self.sequences["seq1"][np.newaxis, num_unroll:self.value_length, :], - (self.batch_size, 1, 1)) - expected_seq2_batch2 = np.tile( - self.sequences["seq2"][np.newaxis, num_unroll:self.value_length, :, :], - (self.batch_size, 1, 1, 1)) - ind1_1 = np.array([ - # batch entry 1 - [0, 0, 0], - [0, 1, 0], [0, 1, 3], [0, 1, 4], - # batch entry 2 - [1, 0, 0], - [1, 1, 0], [1, 1, 3], [1, 1, 4]]) - ind1_2 = np.array([ - # batch entry 1 - [0, 1, 2], [0, 1, 3], - # batch entry 2 - [1, 1, 2], [1, 1, 3]]) - val1_1 = np.array([0, 10, 13, 14, - 0, 10, 13, 14]) - val1_2 = np.array([32, 33, - 32, 33]) - shape1 = np.array([self.batch_size, num_unroll, 6]) - - # For sp_tensor2 all values fall into the first segment. - ind2_1 = np.array([ - # batch entry 1 - [0, 0, 0, 1], - [0, 0, 1, 0], - [0, 0, 1, 2], - [0, 1, 0, 3], - [0, 1, 1, 0], - [0, 1, 1, 1], - [0, 1, 1, 2], - [0, 1, 2, 2], - # batch entry 2 - [1, 0, 0, 1], - [1, 0, 1, 0], - [1, 0, 1, 2], - [1, 1, 0, 3], - [1, 1, 1, 0], - [1, 1, 1, 1], - [1, 1, 1, 2], - [1, 1, 2, 2], - ]) - val2_1 = np.array([1, 10, 12, 103, 150, 149, 150, 122, - 1, 10, 12, 103, 150, 149, 150, 122]) - shape2 = np.array([self.batch_size, num_unroll, 3, 4]) - expected_seq3_batch1 = sparse_tensor.SparseTensorValue( - ind1_1, val1_1, shape1) - expected_seq3_batch2 = sparse_tensor.SparseTensorValue( - ind1_2, val1_2, shape1) - expected_seq4_batch1 = sparse_tensor.SparseTensorValue( - ind2_1, val2_1, shape2) - expected_seq4_batch2 = sparse_tensor.SparseTensorValue( - np.empty(shape=[0, 4], dtype=np.int64), np.array([]), shape2) - self._testBasics( - num_unroll=num_unroll, - length=3, - pad=pad, - expected_seq1_batch1=expected_seq1_batch1, - expected_seq1_batch2=expected_seq1_batch2, - expected_seq2_batch1=expected_seq2_batch1, - expected_seq2_batch2=expected_seq2_batch2, - expected_seq3_batch1=expected_seq3_batch1, - expected_seq3_batch2=expected_seq3_batch2, - expected_seq4_batch1=expected_seq4_batch1, - expected_seq4_batch2=expected_seq4_batch2, - key=key, - make_keys_unique=make_keys_unique) - - def testBasicPadding(self): - self._testBasicPadding(pad=True) - - def testBasicNoPadding(self): - self._testBasicPadding(pad=False) - - def testRandomKeyGen(self): - self._testBasicPadding(pad=False, - key=constant_op.constant("fixed_key"), - make_keys_unique=True) - - def testNotAMultiple(self): - num_unroll = 3 # Not a divisor of value_length - - # so padding would have been necessary. - - # Use placeholder_with_default in sequences to make sure we get runtime - # error instead of shape inference error - sequences = { - "seq1": array_ops.placeholder_with_default(self.sequences["seq1"], - shape=(None, 5)), - "seq2": array_ops.placeholder_with_default(self.sequences["seq2"], - shape=(None, 4, 2)), - "seq3": self.sequences["seq3"], - "seq4": self.sequences["seq4"], - } - - with self.cached_session() as sess: - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - ".*should be a multiple of: 3, but saw " - "value: 4. Consider setting pad=True."): - coord = coordinator.Coordinator() - threads = None - try: - with coord.stop_on_exception(): - next_batch = sqss.batch_sequences_with_states( - input_key=self.key, - input_sequences=sequences, - input_context=self.context, - input_length=3, - initial_states=self.initial_states, - num_unroll=num_unroll, - batch_size=self.batch_size, - num_threads=3, - # to enforce that we only move on to the next examples after - # finishing all segments of the first ones. - capacity=2, - pad=False) - threads = queue_runner_impl.start_queue_runners(coord=coord) - sess.run([next_batch.key]) - except errors_impl.OutOfRangeError: - pass - finally: - coord.request_stop() - if threads is not None: - coord.join(threads, stop_grace_period_secs=2) - - def testAdvancedPadding(self): - num_unroll = 3 # Not a divisor of value_length - so padding to 6 necessary. - expected_seq1_batch1 = np.tile( - self.sequences["seq1"][np.newaxis, 0:num_unroll, :], - (self.batch_size, 1, 1)) - expected_seq2_batch1 = np.tile( - self.sequences["seq2"][np.newaxis, 0:num_unroll, :, :], - (self.batch_size, 1, 1, 1)) - - padded_seq1 = np.concatenate( - [ - self.sequences["seq1"][np.newaxis, num_unroll:self.value_length, :], - np.zeros((1, 1, 5)), np.zeros((1, 1, 5)) - ], - axis=1) - expected_seq1_batch2 = np.concatenate( - [padded_seq1] * self.batch_size, axis=0) - - padded_seq2 = np.concatenate( - [ - self.sequences["seq2"][np.newaxis, num_unroll:self.value_length, :], - np.zeros((1, 1, 4, 2)), np.zeros((1, 1, 4, 2)) - ], - axis=1) - expected_seq2_batch2 = np.concatenate( - [padded_seq2] * self.batch_size, axis=0) - - ind1_1 = np.array([ - # batch entry 1 - [0, 0, 0], - [0, 1, 0], [0, 1, 3], [0, 1, 4], - # batch entry 2 - [1, 0, 0], - [1, 1, 0], [1, 1, 3], [1, 1, 4]]) - ind1_2 = np.array([ - # batch entry 1 - [0, 0, 2], [0, 0, 3], - # batch entry 2 - [1, 0, 2], [1, 0, 3]]) - val1_1 = np.array([0, 10, 13, 14, - 0, 10, 13, 14]) - val1_2 = np.array([32, 33, - 32, 33]) - shape1 = np.array([self.batch_size, num_unroll, 6]) - - # For sp_tensor2 all values fall into the first segment. - ind2_1 = np.array([ - # batch entry 1 - [0, 0, 0, 1], - [0, 0, 1, 0], - [0, 0, 1, 2], - [0, 1, 0, 3], - [0, 1, 1, 0], - [0, 1, 1, 1], - [0, 1, 1, 2], - [0, 1, 2, 2], - # batch entry 2 - [1, 0, 0, 1], - [1, 0, 1, 0], - [1, 0, 1, 2], - [1, 1, 0, 3], - [1, 1, 1, 0], - [1, 1, 1, 1], - [1, 1, 1, 2], - [1, 1, 2, 2], - ]) - val2_1 = np.array([1, 10, 12, 103, 150, 149, 150, 122, - 1, 10, 12, 103, 150, 149, 150, 122]) - shape2 = np.array([self.batch_size, num_unroll, 3, 4]) - expected_seq3_batch1 = sparse_tensor.SparseTensorValue( - ind1_1, val1_1, shape1) - expected_seq3_batch2 = sparse_tensor.SparseTensorValue( - ind1_2, val1_2, shape1) - expected_seq4_batch1 = sparse_tensor.SparseTensorValue( - ind2_1, val2_1, shape2) - expected_seq4_batch2 = sparse_tensor.SparseTensorValue( - np.empty(shape=[0, 4], dtype=np.int64), np.array([]), shape2) - - ind1_1 = np.array([ - # batch entry 1 - [0, 0, 0], - [0, 1, 0], [0, 1, 3], [0, 1, 4], - # batch entry 2 - [1, 0, 0], - [1, 1, 0], [1, 1, 3], [1, 1, 4]]) - ind1_2 = np.array([ - # batch entry 1 - [0, 0, 2], [0, 0, 3], - # batch entry 2 - [1, 0, 2], [1, 0, 3]]) - val1_1 = np.array([0, 10, 13, 14, - 0, 10, 13, 14]) - val1_2 = np.array([32, 33, - 32, 33]) - shape1 = np.array([self.batch_size, num_unroll, 6]) - - # For sp_tensor2 all values fall into the first segment. - ind2_1 = np.array([ - # batch entry 1 - [0, 0, 0, 1], - [0, 0, 1, 0], - [0, 0, 1, 2], - [0, 1, 0, 3], - [0, 1, 1, 0], - [0, 1, 1, 1], - [0, 1, 1, 2], - [0, 1, 2, 2], - # batch entry 2 - [1, 0, 0, 1], - [1, 0, 1, 0], - [1, 0, 1, 2], - [1, 1, 0, 3], - [1, 1, 1, 0], - [1, 1, 1, 1], - [1, 1, 1, 2], - [1, 1, 2, 2], - ]) - val2_1 = np.array([1, 10, 12, 103, 150, 149, 150, 122, - 1, 10, 12, 103, 150, 149, 150, 122]) - shape2 = np.array([self.batch_size, num_unroll, 3, 4]) - expected_seq3_batch1 = sparse_tensor.SparseTensorValue( - ind1_1, val1_1, shape1) - expected_seq3_batch2 = sparse_tensor.SparseTensorValue( - ind1_2, val1_2, shape1) - expected_seq4_batch1 = sparse_tensor.SparseTensorValue( - ind2_1, val2_1, shape2) - expected_seq4_batch2 = sparse_tensor.SparseTensorValue( - np.empty(shape=[0, 4], dtype=np.int64), np.array([]), shape2) - - self._testBasics( - num_unroll=num_unroll, - length=None, - pad=True, - expected_seq1_batch1=expected_seq1_batch1, - expected_seq1_batch2=expected_seq1_batch2, - expected_seq2_batch1=expected_seq2_batch1, - expected_seq2_batch2=expected_seq2_batch2, - expected_seq3_batch1=expected_seq3_batch1, - expected_seq3_batch2=expected_seq3_batch2, - expected_seq4_batch1=expected_seq4_batch1, - expected_seq4_batch2=expected_seq4_batch2) - - -class PaddingTest(test.TestCase): - - def testPaddingInvalidLengths(self): - with ops.Graph().as_default() as g, self.session(graph=g): - sequences = { - "key_1": constant_op.constant([1, 2, 3]), # length 3 - "key_2": constant_op.constant([1.5, 2.5]) # length 2 - } - - _, padded_seq = sqss._padding(sequences, 2) - with self.assertRaisesOpError( - ".*All sequence lengths must match, but received lengths.*"): - padded_seq["key_1"].eval() - - def testPadding(self): - with ops.Graph().as_default() as g, self.session(graph=g): - sequences = { - "key_1": constant_op.constant([1, 2]), - "key_2": constant_op.constant([0.5, -1.0]), - "key_3": constant_op.constant(["a", "b"]), # padding strings - "key_4": constant_op.constant([[1, 2, 3], [4, 5, 6]]) - } - _, padded_seq = sqss._padding(sequences, 5) - - expected_padded_seq = { - "key_1": [1, 2, 0, 0, 0], - "key_2": [0.5, -1.0, 0.0, 0.0, 0.0], - "key_3": ["a", "b", "", "", ""], - "key_4": [[1, 2, 3], [4, 5, 6], [0, 0, 0], [0, 0, 0], [0, 0, 0]] - } - - for key, val in expected_padded_seq.items(): - self.assertTrue( - math_ops.reduce_all(math_ops.equal(val, padded_seq[key])).eval()) - - def testPaddingOnlySparse(self): - ind1 = np.array([[0], [2]]) - val1 = np.array([3, 4]) - shape1 = np.array([4]) - - ind2 = np.array([[1], [2]]) - val2 = np.array([9, 12]) - shape2 = np.array([5]) - - with ops.Graph().as_default() as g, self.session(graph=g): - sp_tensor1 = sparse_tensor.SparseTensor( - indices=array_ops.constant(ind1, dtypes.int64), - values=array_ops.constant(val1, dtypes.int64), - dense_shape=array_ops.constant(shape1, dtypes.int64)) - sp_tensor2 = sparse_tensor.SparseTensor( - indices=array_ops.constant(ind2, dtypes.int64), - values=array_ops.constant(val2, dtypes.int64), - dense_shape=array_ops.constant(shape2, dtypes.int64)) - - sp_tensor1_expected = sparse_tensor.SparseTensor( - indices=sp_tensor1.indices, - values=sp_tensor1.values, - dense_shape=[8]) - sp_tensor2_expected = sparse_tensor.SparseTensor( - indices=sp_tensor2.indices, - values=sp_tensor2.values, - dense_shape=[8]) - - sequences = { - "key_1": sp_tensor1, - "key_2": sp_tensor2, - } - _, padded_seq = sqss._padding(sequences, 4) - - expected_padded_seq = { - "key_1": sp_tensor1_expected, - "key_2": sp_tensor2_expected, - } - - for key, val in expected_padded_seq.items(): - self.assertAllEqual( - sparse_ops.sparse_tensor_to_dense(val).eval(), - sparse_ops.sparse_tensor_to_dense(padded_seq[key]).eval()) - - -class SparseTensorReConstructionTest(test.TestCase): - - def testAddManyTakeManyRoundTripBatched(self): - with self.test_session(use_gpu=False) as sess: - # N == 4 because shape_value == [4, 5] - indices_value_1 = np.array([[0, 0], [0, 1], [2, 0]], dtype=np.int64) - values_value_1 = np.array([b"a", b"b", b"c"]) - shape_value_1 = np.array([4, 5], dtype=np.int64) - sparse_tensor_1 = sparse_tensor.SparseTensor( - array_ops.placeholder(dtypes.int64), - array_ops.placeholder(dtypes.string), - array_ops.placeholder(dtypes.int64)) - dict1 = {"key": sparse_tensor_1} - indices_value_2 = np.array([[1, 4], [2, 3]], dtype=np.int64) - values_value_2 = np.array([b"d", b"e"]) - shape_value_2 = np.array([4, 5], dtype=np.int64) - sparse_tensor_2 = sparse_tensor.SparseTensor( - array_ops.placeholder(dtypes.int64), - array_ops.placeholder(dtypes.string), - array_ops.placeholder(dtypes.int64)) - dict2 = {"key": sparse_tensor_2} - - input_seq1, keys1, tensor_list1 = sqss._deconstruct_sparse_tensor_seq( - dict1, shared_name="a") - handles_1 = input_seq1["key"] - input_seq2, _, _ = sqss._deconstruct_sparse_tensor_seq( - dict2, shared_name="a") - handles_2 = input_seq2["key"] - - combined_handles = array_ops.stack( - [handles_1[1], handles_1[2], handles_1[3], - handles_2[1], handles_2[2], handles_2[3]]) - batched_dict = {"key": combined_handles} - sqss._reconstruct_sparse_tensor_seq( - batched_dict, - keys1, - tensor_list1, - batch_size=2, - num_unroll=3) - - roundtrip_value, = sess.run( - [batched_dict["key"]], - feed_dict={sparse_tensor_1.indices: indices_value_1, - sparse_tensor_1.values: values_value_1, - sparse_tensor_1.dense_shape: shape_value_1, - sparse_tensor_2.indices: indices_value_2, - sparse_tensor_2.values: values_value_2, - sparse_tensor_2.dense_shape: shape_value_2}) - - self.assertAllEqual(roundtrip_value.indices, - np.array([[0, 1, 0], [1, 0, 4], [1, 1, 3]], - dtype=np.int64)) - self.assertAllEqual(roundtrip_value.values, - np.array([b"c", b"d", b"e"])) - self.assertAllEqual(roundtrip_value.dense_shape, - np.array([2, 3, 5], dtype=np.int64)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/training/python/training/bucket_ops.py b/tensorflow/contrib/training/python/training/bucket_ops.py deleted file mode 100644 index fddcf1e4f62..00000000000 --- a/tensorflow/contrib/training/python/training/bucket_ops.py +++ /dev/null @@ -1,425 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Operations for bucketing data into groups. - -The classes and functions in this module are used to queue up data into -buckets conditional on side information (e.g. sequence length). -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -import numpy as np - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.layers import utils -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.summary import summary -from tensorflow.python.training import input as input_py -from tensorflow.python.training import queue_runner - -# pylint: disable=protected-access -_as_original_type = input_py._as_original_type -_as_tensor_list = input_py._as_tensor_list -_restore_sparse_tensors = input_py._restore_sparse_tensors -_dtypes = input_py._dtypes -_store_sparse_tensors = input_py._store_sparse_tensors -_validate_keep_input = input_py._validate_keep_input -_shapes = input_py._shapes -_which_queue = input_py._which_queue - -# pylint: enable=protected-access - - -def _validate_bucket(tensor_list): - tensor_list = ops.convert_n_to_tensor_or_indexed_slices(tensor_list) - if not tensor_list: - raise ValueError("Expected at least one tensor in bucket().") - return tensor_list - - -def bucket(tensors, - which_bucket, - batch_size, - num_buckets, - num_threads=1, - capacity=32, - bucket_capacities=None, - shapes=None, - dynamic_pad=False, - allow_smaller_final_batch=False, - keep_input=True, - shared_name=None, - name=None): - """Lazy bucketing of input tensors according to `which_bucket`. - - The argument `tensors` can be a list or a dictionary of tensors. - The value returned by the function will be of the same type - as `tensors`. - - The tensors entering this function are put into the bucket given by - `which_bucket`. Each bucket has its own queue. When a bucket contains - `batch_size` elements, this minibatch is pushed onto a top queue. The - tensors returned from this function are a the result of dequeueing the - next minibatch from this top queue. - - This function is implemented using several queues. A `QueueRunner` for the - queues is added to the current `Graph`'s `QUEUE_RUNNER` collection. - - As the returned tensors are the result of a dequeue operation, evaluating - them will throw a `tf.errors.OutOfRangeError` when the input queue is - exhausted. If these tensors are feeding another input queue, its queue runner - will catch this exception, however, if they are used in your main thread - you are responsible for catching this yourself. - - *N.B.:* If `dynamic_pad` is `False`, you must ensure that either - (i) the `shapes` argument is passed, or (ii) all of the tensors in - `tensors` must have fully-defined shapes. `ValueError` will be - raised if neither of these conditions holds. - - If `dynamic_pad` is `True`, it is sufficient that the *rank* of the - tensors is known, but individual dimensions may have shape `None`. - In this case, for each enqueue the dimensions with value `None` - may have a variable length; upon dequeue, the output tensors will be padded - on the right to the maximum shape of the tensors in the current minibatch. - For numbers, this padding takes value 0. For strings, this padding is - the empty string. See `PaddingFIFOQueue` for more info. - - If `allow_smaller_final_batch` is `True`, a smaller batch value than - `batch_size` is returned when the queues are closed and there are not enough - elements to fill the batch, otherwise the pending elements are discarded. - In addition, all output tensors' static shapes, as accessed via the - `get_shape()` method will have a 0th `Dimension` value of `None`, and - operations that depend on fixed batch_size would fail. - - Args: - tensors: The list or dictionary of tensors, representing a single element, - to bucket. Nested lists are not supported. - which_bucket: An `int32` scalar Tensor taking a value in `[0, num_buckets)`. - batch_size: The new batch size pulled from the queue (all queues will have - the same size). If a list is passed in then each bucket will have a - different batch_size. - (python int, int32 scalar or iterable of integers of length num_buckets). - num_buckets: A python integer, the number of buckets. - num_threads: An integer. The number of threads enqueuing `tensors`. - capacity: An integer. The maximum number of minibatches in the top queue, - and also (by default) the maximum number of elements within each bucket. - bucket_capacities: (Optional) None or a list of integers, the capacities of - each bucket. If None, capacity is used (default). If specified, it must - be a list of integers of length num_buckets: the i-th element is used - as capacity for the i-th bucket queue. - shapes: (Optional) The shapes for each example. Defaults to the - inferred shapes for `tensors`. - dynamic_pad: Boolean. Allow variable dimensions in input shapes. - The given dimensions are padded upon dequeue so that tensors within a - batch have the same shapes. - allow_smaller_final_batch: (Optional) Boolean. If `True`, allow the final - batches to be smaller if there are insufficient items left in the queues. - keep_input: A `bool` scalar Tensor. If provided, this tensor controls - whether the input is added to the queue or not. If it evaluates `True`, - then `tensors` are added to the bucket; otherwise they are dropped. This - tensor essentially acts as a filtering mechanism. - shared_name: (Optional). If set, the queues will be shared under the given - name across multiple sessions. - name: (Optional) A name for the operations. - - Returns: - A tuple `(bucket, outputs)` where `bucket` is - a `int32` scalar tensor and `outputs` is a list or - dictionary of batched outputs corresponding to elements of `tensors`. - Every step will receive a new bucket of outputs. - - Raises: - ValueError: If the `shapes` are not specified, and cannot be - inferred from the elements of `tensors` or if batch_size is a sequence - but its length != num_buckets. Also if bucket_capacities is not None but - its length != num_buckets. - """ - batch_size_per_bucket = False - if isinstance(batch_size, (list, tuple)): - batch_size_per_bucket = True - if len(batch_size) != num_buckets: - raise ValueError( - "If batch_size is a list it must have num_buckets elements") - else: - batch_size = [batch_size] * num_buckets - - if bucket_capacities is None: - bucket_capacities = [capacity] * num_buckets - if len(bucket_capacities) != num_buckets: - raise ValueError( - "The list bucket_capacities (%s) must have exactly num_buckets (%d) " - "elements." % (str(bucket_capacities), num_buckets)) - - tensor_list = _as_tensor_list(tensors) - with ops.name_scope(name, "bucket", tensor_list) as name: - tensor_list = _validate_bucket(tensor_list) - keep_input = _validate_keep_input(keep_input, enqueue_many=False) - (tensor_list, sparse_info) = _store_sparse_tensors( - tensor_list, enqueue_many=False, keep_input=keep_input) - - # Round-trip batch_size to a tensor, and possibly back - for i, bucket_batch_size in enumerate(batch_size): - bucket_batch_size = ops.convert_to_tensor( - bucket_batch_size, dtype=dtypes.int32, name="batch_size") - static_batch_size = tensor_util.constant_value(bucket_batch_size) - batch_size[i] = (static_batch_size if static_batch_size is not None else - bucket_batch_size) - - types = _dtypes([tensor_list]) - shapes = _shapes([tensor_list], shapes, enqueue_many=False) - - which_bucket = ops.convert_to_tensor( - which_bucket, dtype=dtypes.int32, name="which_bucket") - - queue_creator = _which_queue(dynamic_pad) - bucket_queues = [] - for i in range(num_buckets): - shared_name_i = ("%s_%d" % (shared_name, i) if shared_name is not None - else None) - bucket_queues.append( - queue_creator( - capacity=bucket_capacities[i], - dtypes=types, - shapes=shapes, - shared_name=shared_name_i, - name="bucket_queue_%d" % i)) - - maybe_static_batch_size = ( - None if (allow_smaller_final_batch or batch_size_per_bucket) - else static_batch_size) - - bucket_shapes = [ - tensor_shape.TensorShape([maybe_static_batch_size]).concatenate(s) - for s in bucket_queues[0].shapes - ] - # top_queue is a PaddingFIFOQueue even if the bucket queues are regular FIFO - # queues because if we use allow_smaller_final_batch, shapes will - # contain Nones in their first entry; as a result, a regular - # FIFOQueue would die when being passed shapes that are not fully defined. - top_queue = data_flow_ops.PaddingFIFOQueue( - capacity=capacity, - dtypes=[dtypes.int32] + types, - shapes=[tensor_shape.TensorShape([])] + bucket_shapes, - shared_name=shared_name, - name="top_queue") - - def enqueue_which(): - """Return an op that enqueues conditionally in one of the queues.""" - def enqueue_single(i): - return bucket_queues[i].enqueue(tensor_list) - - enqueues = [ - control_flow_ops.cond( - math_ops.equal(which_bucket, i), - functools.partial(enqueue_single, i), control_flow_ops.no_op) - for i in range(num_buckets) - ] - return control_flow_ops.group(*enqueues, name="group_enqueues") - - maybe_enqueue = utils.smart_cond( - keep_input, - enqueue_which, - control_flow_ops.no_op) - - bucket_enqueue_ops = [maybe_enqueue] * num_threads - - if allow_smaller_final_batch: - which_dequeue = lambda q: q.dequeue_up_to - else: - which_dequeue = lambda q: q.dequeue_many - - def make_list(t): - if isinstance(t, (list, tuple)): - return t - else: - return [t] - - enqueues_to_top = [ - top_queue.enqueue( - [constant_op.constant(i)] + make_list(which_dequeue(q)( - bs, name="read_bucket_%d" % i)), - name="enqueue_from_bucket_%d" % i) - for i, (q, bs) in enumerate(zip(bucket_queues, batch_size)) - ] - - queue_runner.add_queue_runner( - queue_runner.QueueRunner( - bucket_queues[0], enqueues_to_top, - close_op=top_queue.close(), - cancel_op=top_queue.close(cancel_pending_enqueues=True), - queue_closed_exception_types=(errors.OutOfRangeError, - errors.CancelledError))) - queue_runner.add_queue_runner( - queue_runner.QueueRunner( - top_queue, - bucket_enqueue_ops, - close_op=control_flow_ops.group( - *[q.close() for q in bucket_queues]), - cancel_op=control_flow_ops.group( - *[q.close(cancel_pending_enqueues=True) - for q in bucket_queues]), - queue_closed_exception_types=(errors.OutOfRangeError, - errors.CancelledError))) - - for q in bucket_queues: - summary.scalar("bucket/%s/size" % q.name, - math_ops.cast(top_queue.size(), dtypes.float32)) - summary.scalar("bucket/%s/fraction_of_%d_full" % (top_queue.name, capacity), - math_ops.cast(top_queue.size(), dtypes.float32) * - (1. / capacity)) - - dequeued = top_queue.dequeue(name="dequeue_top") - which_bucket_dequeued = dequeued[0] - dequeued = dequeued[1:] - if len(dequeued) == 1: - dequeued = dequeued[0] - dequeued = _restore_sparse_tensors(dequeued, sparse_info) - return (which_bucket_dequeued, _as_original_type(tensors, dequeued)) - - -def bucket_by_sequence_length(input_length, - tensors, - batch_size, - bucket_boundaries, - num_threads=1, - capacity=32, - bucket_capacities=None, - shapes=None, - dynamic_pad=False, - allow_smaller_final_batch=False, - keep_input=True, - shared_name=None, - name=None): - """Lazy bucketing of inputs according to their length. - - This method calls `tf.contrib.training.bucket` under the hood, after first - subdividing the bucket boundaries into separate buckets and identifying which - bucket the given `input_length` belongs to. See the documentation for - `which_bucket` for details of the other arguments. - - Args: - input_length: `int32` scalar `Tensor`, the sequence length of tensors. - tensors: The list or dictionary of tensors, representing a single element, - to bucket. Nested lists are not supported. - batch_size: The new batch size pulled from the queue (all queues will have - the same size). If a list is passed in then each bucket will have a - different batch_size. - (python int, int32 scalar or iterable of integers of length num_buckets). - bucket_boundaries: int list, increasing non-negative numbers. - The edges of the buckets to use when bucketing tensors. Two extra buckets - are created, one for `input_length < bucket_boundaries[0]` and - one for `input_length >= bucket_boundaries[-1]`. - num_threads: An integer. The number of threads enqueuing `tensors`. - capacity: An integer. The maximum number of minibatches in the top queue, - and also the maximum number of elements within each bucket. - bucket_capacities: (Optional) None or a list of integers, the capacities of - each bucket. If None, capacity is used (default). If specified, it must - be a list of integers of length one larger than bucket_boundaries. - Its i-th element is used as capacity for the i-th bucket queue. - shapes: (Optional) The shapes for each example. Defaults to the - inferred shapes for `tensors`. - dynamic_pad: Boolean. Allow variable dimensions in input shapes. - The given dimensions are padded upon dequeue so that tensors within a - batch have the same shapes. - allow_smaller_final_batch: (Optional) Boolean. If `True`, allow the final - batches to be smaller if there are insufficient items left in the queues. - keep_input: A `bool` scalar Tensor. If provided, this tensor controls - whether the input is added to the queue or not. If it evaluates `True`, - then `tensors` are added to the bucket; otherwise they are dropped. This - tensor essentially acts as a filtering mechanism. - shared_name: (Optional). If set, the queues will be shared under the given - name across multiple sessions. - name: (Optional) A name for the operations. - - Returns: - A tuple `(sequence_length, outputs)` where `sequence_length` is - a 1-D `Tensor` of size `batch_size` and `outputs` is a list or dictionary - of batched, bucketed, outputs corresponding to elements of `tensors`. - - Raises: - TypeError: if `bucket_boundaries` is not a list of python integers. - ValueError: if `bucket_boundaries` is empty or contains non-increasing - values or if batch_size is a list and it's length doesn't equal the number - of buckets. - """ - tensor_list = _as_tensor_list(tensors) - if not isinstance(bucket_boundaries, (list, tuple)): - raise TypeError( - "bucket_boundaries must be a list or tuple, but received: %s" % - bucket_boundaries) - if not bucket_boundaries: - raise ValueError("bucket_boundaries must not be empty") - for (s, e) in zip(bucket_boundaries[:-1], bucket_boundaries[1:]): - if not isinstance(s, int) or not isinstance(e, int): - raise TypeError("bucket boundaries must be integers, but saw: %s and %s" % - (s, e)) - if s >= e: - raise ValueError( - "Buckets must contain sequential increasing lengths, but saw: " - "%d before %d" % (s, e)) - - with ops.name_scope(name, "bucket_by_sequence_length", - [input_length] + tensor_list) as name: - input_length = ops.convert_to_tensor( - input_length, dtype=dtypes.int32, name="input_length") - # Bucketing conditions are: - # l < b[0] - # b[0] <= l < b[1] - # b[1] <= l < b[2] - # ... - # b[N-2] <= l < b[N-1] - # b[N-1] <= l - # Equivalent to: - # [-inf, b[0], b[1], ..., b[N-1]] <= l < [b[0], b[1], ..., b[N-1], inf] - buckets_min = [np.iinfo(np.int32).min] + list(bucket_boundaries) - buckets_max = list(bucket_boundaries) + [np.iinfo(np.int32).max] - conditions_c = math_ops.logical_and( - math_ops.less_equal(buckets_min, input_length), - math_ops.less(input_length, buckets_max)) - which_bucket = math_ops.reduce_min(array_ops.where_v2(conditions_c)) - which_bucket = math_ops.cast(which_bucket, dtypes.int32) - - if shapes is not None: - shapes = [tensor_shape.TensorShape([])] + shapes - - _, dequeued = bucket( - tensors=[input_length] + tensor_list, - which_bucket=which_bucket, - batch_size=batch_size, - num_buckets=len(bucket_boundaries) + 1, - num_threads=num_threads, - capacity=capacity, - bucket_capacities=bucket_capacities, - shapes=shapes, - dynamic_pad=dynamic_pad, - allow_smaller_final_batch=allow_smaller_final_batch, - keep_input=keep_input, - shared_name=shared_name) - - return (dequeued[0], _as_original_type(tensors, dequeued[1:])) - - -__all__ = ["bucket", "bucket_by_sequence_length"] diff --git a/tensorflow/contrib/training/python/training/bucket_ops_test.py b/tensorflow/contrib/training/python/training/bucket_ops_test.py deleted file mode 100644 index b259e0ee83f..00000000000 --- a/tensorflow/contrib/training/python/training/bucket_ops_test.py +++ /dev/null @@ -1,450 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.contrib.training.bucket.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random - -import numpy as np -from tensorflow.contrib.training.python.training import bucket_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes as dtypes_lib -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test -from tensorflow.python.training import coordinator -from tensorflow.python.training import queue_runner_impl - - -def _which_bucket(bucket_edges, v): - """Identify which bucket v falls into. - - Args: - bucket_edges: int array, bucket edges - v: int scalar, index - Returns: - int scalar, the bucket. - If v < bucket_edges[0], return 0. - If bucket_edges[0] <= v < bucket_edges[1], return 1. - ... - If bucket_edges[-2] <= v < bucket_edges[-1], return len(bucket_edges). - If v >= bucket_edges[-1], return len(bucket_edges) + 1 - """ - v = np.asarray(v) - full = [0] + bucket_edges - found = np.where(np.logical_and(v >= full[:-1], v < full[1:]))[0] - if not found.size: - return len(full) - return found[0] - - -class BucketTest(test.TestCase): - - def setUp(self): - ops.reset_default_graph() - - self.scalar_int_feed = array_ops.placeholder(dtypes_lib.int32, ()) - self.unk_int64_feed = array_ops.placeholder(dtypes_lib.int64, (None,)) - self.vec3_str_feed = array_ops.placeholder(dtypes_lib.string, (3,)) - self.sparse_c = sparse_tensor.SparseTensor( - indices=[[0]], - values=[1.0], - dense_shape=[1]) - - self._coord = coordinator.Coordinator() - # Make capacity very large so we can feed all the inputs in the - # main thread without blocking - input_queue = data_flow_ops.PaddingFIFOQueue( - 5000, - dtypes=[dtypes_lib.int32, dtypes_lib.int64, dtypes_lib.string], - shapes=[(), (None,), (3,)]) - - self._input_enqueue_op = input_queue.enqueue( - (self.scalar_int_feed, self.unk_int64_feed, self.vec3_str_feed)) - self.scalar_int, self.unk_int64, self.vec3_str = input_queue.dequeue() - self._threads = None - self._close_op = input_queue.close() - self._sess = None - - def enqueue_inputs(self, sess, feed_dict): - sess.run(self._input_enqueue_op, feed_dict=feed_dict) - - def start_queue_runners(self, sess): - # Store session to be able to close inputs later - if self._sess is None: - self._sess = sess - self._threads = queue_runner_impl.start_queue_runners(coord=self._coord) - - def tearDown(self): - if self._sess is not None: - self._sess.run(self._close_op) - self._coord.request_stop() - self._coord.join(self._threads) - - def testSingleBucket(self): - bucketed_dynamic = bucket_ops.bucket( - tensors=[self.scalar_int, self.unk_int64, self.vec3_str, self.sparse_c], - which_bucket=constant_op.constant(0), - num_buckets=2, - batch_size=32, - num_threads=10, - dynamic_pad=True) - # Check shape inference on bucketing outputs - self.assertAllEqual( - [[32], [32, None], [32, 3], [None, None]], - [out.get_shape().as_list() for out in bucketed_dynamic[1]]) - with self.cached_session() as sess: - for v in range(32): - self.enqueue_inputs(sess, { - self.scalar_int_feed: v, - self.unk_int64_feed: v * [v], - self.vec3_str_feed: 3 * [str(v)] - }) - self.start_queue_runners(sess) - - # Get a single minibatch - bucketed_values = sess.run(bucketed_dynamic) - - # (which_bucket, bucket_tensors). - self.assertEqual(2, len(bucketed_values)) - - # Count number of bucket_tensors. - self.assertEqual(4, len(bucketed_values[1])) - - # Ensure bucket 0 was used for all minibatch entries. - self.assertAllEqual(0, bucketed_values[0]) - - expected_scalar_int = np.arange(32) - expected_unk_int64 = np.zeros((32, 31)).astype(np.int64) - for i in range(32): - expected_unk_int64[i, :i] = i - expected_vec3_str = np.vstack(3 * [np.arange(32).astype(bytes)]).T - - # Must resort the output because num_threads > 1 leads to - # sometimes-inconsistent insertion order. - resort = np.argsort(bucketed_values[1][0]) - self.assertAllEqual(expected_scalar_int, bucketed_values[1][0][resort]) - self.assertAllEqual(expected_unk_int64, bucketed_values[1][1][resort]) - self.assertAllEqual(expected_vec3_str, bucketed_values[1][2][resort]) - - def testBatchSizePerBucket(self): - which_bucket = control_flow_ops.cond(self.scalar_int < 5, - lambda: constant_op.constant(0), - lambda: constant_op.constant(1)) - batch_sizes = [5, 10] - bucketed_dynamic = bucket_ops.bucket( - tensors=[self.scalar_int, self.unk_int64, self.vec3_str, self.sparse_c], - which_bucket=which_bucket, - num_buckets=2, - batch_size=batch_sizes, - num_threads=1, - dynamic_pad=True) - # Check shape inference on bucketing outputs - self.assertAllEqual( - [[None], [None, None], [None, 3], [None, None]], - [out.get_shape().as_list() for out in bucketed_dynamic[1]]) - with self.cached_session() as sess: - for v in range(15): - self.enqueue_inputs(sess, { - self.scalar_int_feed: v, - self.unk_int64_feed: v * [v], - self.vec3_str_feed: 3 * [str(v)] - }) - self.start_queue_runners(sess) - - # Get two minibatches (one with small values, one with large). - bucketed_values_0 = sess.run(bucketed_dynamic) - bucketed_values_1 = sess.run(bucketed_dynamic) - - # Figure out which output has the small values - if bucketed_values_0[0] < 5: - bucketed_values_large, bucketed_values_small = (bucketed_values_1, - bucketed_values_0) - else: - bucketed_values_small, bucketed_values_large = (bucketed_values_0, - bucketed_values_1) - - # Ensure bucket 0 was used for all minibatch entries. - self.assertAllEqual(0, bucketed_values_small[0]) - self.assertAllEqual(1, bucketed_values_large[0]) - - # Check that the batch sizes differ per bucket - self.assertEqual(5, len(bucketed_values_small[1][0])) - self.assertEqual(10, len(bucketed_values_large[1][0])) - - def testEvenOddBuckets(self): - which_bucket = (self.scalar_int % 2) - bucketed_dynamic = bucket_ops.bucket( - tensors=[self.scalar_int, self.unk_int64, self.vec3_str, self.sparse_c], - which_bucket=which_bucket, - num_buckets=2, - batch_size=32, - num_threads=10, - dynamic_pad=True) - # Check shape inference on bucketing outputs - self.assertAllEqual( - [[32], [32, None], [32, 3], [None, None]], - [out.get_shape().as_list() for out in bucketed_dynamic[1]]) - with self.cached_session() as sess: - for v in range(64): - self.enqueue_inputs(sess, { - self.scalar_int_feed: v, - self.unk_int64_feed: v * [v], - self.vec3_str_feed: 3 * [str(v)] - }) - self.start_queue_runners(sess) - - # Get two minibatches (one containing even values, one containing odds) - bucketed_values_0 = sess.run(bucketed_dynamic) - bucketed_values_1 = sess.run(bucketed_dynamic) - - # (which_bucket, bucket_tensors). - self.assertEqual(2, len(bucketed_values_0)) - self.assertEqual(2, len(bucketed_values_1)) - - # Count number of bucket_tensors. - self.assertEqual(4, len(bucketed_values_0[1])) - self.assertEqual(4, len(bucketed_values_1[1])) - - # Figure out which output has the even values (there's - # randomness due to the multithreaded nature of bucketing) - if bucketed_values_0[0] % 2 == 1: - bucketed_values_even, bucketed_values_odd = (bucketed_values_1, - bucketed_values_0) - else: - bucketed_values_even, bucketed_values_odd = (bucketed_values_0, - bucketed_values_1) - - # Ensure bucket 0 was used for all minibatch entries. - self.assertAllEqual(0, bucketed_values_even[0]) - self.assertAllEqual(1, bucketed_values_odd[0]) - - # Test the first bucket outputted, the events starting at 0 - expected_scalar_int = np.arange(0, 32 * 2, 2) - expected_unk_int64 = np.zeros((32, 31 * 2)).astype(np.int64) - for i in range(0, 32): - expected_unk_int64[i, :2 * i] = 2 * i - expected_vec3_str = np.vstack(3 * - [np.arange(0, 32 * 2, 2).astype(bytes)]).T - - # Must resort the output because num_threads > 1 leads to - # sometimes-inconsistent insertion order. - resort = np.argsort(bucketed_values_even[1][0]) - self.assertAllEqual(expected_scalar_int, - bucketed_values_even[1][0][resort]) - self.assertAllEqual(expected_unk_int64, - bucketed_values_even[1][1][resort]) - self.assertAllEqual(expected_vec3_str, bucketed_values_even[1][2][resort]) - - # Test the second bucket outputted, the odds starting at 1 - expected_scalar_int = np.arange(1, 32 * 2 + 1, 2) - expected_unk_int64 = np.zeros((32, 31 * 2 + 1)).astype(np.int64) - for i in range(0, 32): - expected_unk_int64[i, :2 * i + 1] = 2 * i + 1 - expected_vec3_str = np.vstack( - 3 * [np.arange(1, 32 * 2 + 1, 2).astype(bytes)]).T - - # Must resort the output because num_threads > 1 leads to - # sometimes-inconsistent insertion order. - resort = np.argsort(bucketed_values_odd[1][0]) - self.assertAllEqual(expected_scalar_int, - bucketed_values_odd[1][0][resort]) - self.assertAllEqual(expected_unk_int64, bucketed_values_odd[1][1][resort]) - self.assertAllEqual(expected_vec3_str, bucketed_values_odd[1][2][resort]) - - def testEvenOddBucketsFilterOutAllOdd(self): - which_bucket = (self.scalar_int % 2) - keep_input = math_ops.equal(which_bucket, 0) - bucketed_dynamic = bucket_ops.bucket( - tensors=[self.scalar_int, self.unk_int64, self.vec3_str], - which_bucket=which_bucket, - num_buckets=2, - batch_size=32, - num_threads=10, - keep_input=keep_input, - dynamic_pad=True) - # Check shape inference on bucketing outputs - self.assertAllEqual( - [[32], [32, None], [32, 3]], - [out.get_shape().as_list() for out in bucketed_dynamic[1]]) - with self.cached_session() as sess: - for v in range(128): - self.enqueue_inputs(sess, { - self.scalar_int_feed: v, - self.unk_int64_feed: v * [v], - self.vec3_str_feed: 3 * [str(v)] - }) - self.start_queue_runners(sess) - - # Get two minibatches ([0, 2, ...] and [64, 66, ...]) - bucketed_values_even0 = sess.run(bucketed_dynamic) - bucketed_values_even1 = sess.run(bucketed_dynamic) - - # Ensure that bucket 1 was completely filtered out - self.assertAllEqual(0, bucketed_values_even0[0]) - self.assertAllEqual(0, bucketed_values_even1[0]) - - # Merge their output for sorting and comparison - bucketed_values_all_elem0 = np.concatenate((bucketed_values_even0[1][0], - bucketed_values_even1[1][0])) - - self.assertAllEqual( - np.arange(0, 128, 2), sorted(bucketed_values_all_elem0)) - - def testFailOnWrongBucketCapacities(self): - with self.assertRaisesRegexp(ValueError, r"must have exactly num_buckets"): - bucket_ops.bucket( # 2 buckets and 3 capacities raises ValueError. - tensors=[self.scalar_int, self.unk_int64, self.vec3_str], - which_bucket=constant_op.constant(0), num_buckets=2, - batch_size=32, bucket_capacities=[3, 4, 5]) - - -class BucketBySequenceLengthTest(test.TestCase): - - def _testBucketBySequenceLength(self, - allow_small_batch, - bucket_capacities=None, - drain_entire_queue=True): - ops.reset_default_graph() - - # All inputs must be identical lengths across tuple index. - # The input reader will get input_length from the first tuple - # entry. - data_len = 4 - labels_len = 3 - input_pairs = [(length, ([np.int64(length)] * data_len, - [str(length).encode("ascii")] * labels_len)) - for length in (1, 3, 4, 5, 6, 10)] - - lengths = array_ops.placeholder(dtypes_lib.int32, ()) - data = array_ops.placeholder(dtypes_lib.int64, (data_len,)) - labels = array_ops.placeholder(dtypes_lib.string, (labels_len,)) - - batch_size = 8 - bucket_boundaries = [3, 4, 5, 10] - num_pairs_to_enqueue = 50 * batch_size + 100 - - # Make capacity very large so we can feed all the inputs in the - # main thread without blocking - input_queue = data_flow_ops.FIFOQueue( - 5000, (dtypes_lib.int32, dtypes_lib.int64, dtypes_lib.string), ( - (), (data_len,), (labels_len,))) - input_enqueue_op = input_queue.enqueue((lengths, data, labels)) - lengths_t, data_t, labels_t = input_queue.dequeue() - close_input_op = input_queue.close() - - (out_lengths_t, data_and_labels_t) = (bucket_ops.bucket_by_sequence_length( - input_length=lengths_t, - tensors=[data_t, labels_t], - batch_size=batch_size, - bucket_boundaries=bucket_boundaries, - bucket_capacities=bucket_capacities, - allow_smaller_final_batch=allow_small_batch, - num_threads=10)) - - expected_batch_size = None if allow_small_batch else batch_size - self.assertEqual(out_lengths_t.get_shape().as_list(), [expected_batch_size]) - self.assertEqual(data_and_labels_t[0].get_shape().as_list(), - [expected_batch_size, data_len]) - self.assertEqual(data_and_labels_t[1].get_shape().as_list(), - [expected_batch_size, labels_len]) - - def _read_test(sess): - num_pairs_dequeued = 0 - try: - while drain_entire_queue or num_pairs_dequeued < 40 * batch_size: - (out_lengths, (data, labels)) = sess.run( - (out_lengths_t, data_and_labels_t)) - num_pairs_dequeued += out_lengths.shape[0] - if allow_small_batch: - self.assertEqual(data_len, data.shape[1]) - self.assertEqual(labels_len, labels.shape[1]) - self.assertGreaterEqual(batch_size, out_lengths.shape[0]) - self.assertGreaterEqual(batch_size, data.shape[0]) - self.assertGreaterEqual(batch_size, labels.shape[0]) - else: - self.assertEqual((batch_size, data_len), data.shape) - self.assertEqual((batch_size, labels_len), labels.shape) - self.assertEqual((batch_size,), out_lengths.shape) - for (lr, dr, tr) in zip(out_lengths, data, labels): - # Make sure length matches data (here it's the same value). - self.assertEqual(dr[0], lr) - # Make sure data & labels match. - self.assertEqual(dr[0], int(tr[0].decode("ascii"))) - # Make sure for each row, data came from the same bucket. - self.assertEqual( - _which_bucket(bucket_boundaries, dr[0]), - _which_bucket(bucket_boundaries, dr[1])) - except errors.OutOfRangeError: - if allow_small_batch: - self.assertEqual(num_pairs_to_enqueue, num_pairs_dequeued) - else: - # Maximum left over in the queues should be at most one less than the - # batch_size, for every bucket. - num_buckets = len(bucket_boundaries) + 2 - self.assertLessEqual( - num_pairs_to_enqueue - (batch_size - 1) * num_buckets, - num_pairs_dequeued) - - with self.cached_session() as sess: - coord = coordinator.Coordinator() - - # Feed the inputs, then close the input thread. - for _ in range(num_pairs_to_enqueue): - which = random.randint(0, len(input_pairs) - 1) - length, pair = input_pairs[which] - sess.run(input_enqueue_op, - feed_dict={lengths: length, - data: pair[0], - labels: pair[1]}) - sess.run(close_input_op) - - # Start the queue runners - threads = queue_runner_impl.start_queue_runners(coord=coord) - # Read off the top of the bucket and ensure correctness of output - _read_test(sess) - coord.request_stop() - coord.join(threads) - - def testBucketBySequenceLength(self): - self._testBucketBySequenceLength(allow_small_batch=False) - - def testBucketBySequenceLengthAllow(self): - self._testBucketBySequenceLength(allow_small_batch=True) - - def testBucketBySequenceLengthBucketCapacities(self): - # Above bucket_boundaries = [3, 4, 5, 10] so we need 5 capacities. - with self.assertRaisesRegexp(ValueError, r"must have exactly num_buckets"): - self._testBucketBySequenceLength(allow_small_batch=False, - bucket_capacities=[32, 32, 32, 32]) - # Test with different capacities. - capacities = [48, 40, 32, 24, 16] - self._testBucketBySequenceLength(allow_small_batch=True, - bucket_capacities=capacities) - - def testBucketBySequenceLengthShutdown(self): - self._testBucketBySequenceLength(allow_small_batch=True, - drain_entire_queue=False) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/training/python/training/device_setter.py b/tensorflow/contrib/training/python/training/device_setter.py deleted file mode 100644 index 8513aef02fd..00000000000 --- a/tensorflow/contrib/training/python/training/device_setter.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Strategies for placing variables on parameter servers. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import hashlib -import numpy as np - -from tensorflow.python.framework import tensor_shape - - -class RandomStrategy(object): - """Returns a random PS task for op placement. - - This may perform better than the default round-robin placement if you - have a large number of variables. Depending on your architecture and - number of parameter servers, round-robin can lead to situations where - all of one type of variable is placed on a single PS task, which may - lead to contention issues. - - This strategy uses a hash function on the name of each op for deterministic - placement. - """ - - def __init__(self, num_ps_tasks, seed=0): - """Creates a new `RandomStrategy`.""" - self._num_tasks = num_ps_tasks - self._seed = seed - - def __call__(self, op): - """Chooses a ps task index for the given `Operation`.""" - key = "%s_%d" % (op.name, self._seed) - key = key.encode("utf-8") - # Use MD5 instead of Python's built-in hash() to get consistent outputs - # between runs. - n = int(hashlib.md5(key).hexdigest(), 16) - return int(n % self._num_tasks) - - -class GreedyLoadBalancingStrategy(object): - """Returns the least-loaded ps task for op placement. - - The load is calculated by a user-specified load function passed in at - construction. There are no units for load, and the load function is - responsible for providing an internally consistent measure. - - Note that this strategy is very sensitive to the exact order in which - ps ops (typically variables) are created, as it greedily places ops - on the least-loaded ps at the point each op is processed. - - One reasonable heuristic is the `byte_size_load_fn`, which - estimates load as the number of bytes that would be used to store and - transmit the entire variable. More advanced load functions - could consider the difference in access patterns across ops, or trade - off CPU-intensive ops with RAM-intensive ops with network bandwidth. - - This class is intended to be used as a `ps_strategy` in - `tf.compat.v1.train.replica_device_setter`. - """ - - def __init__(self, num_tasks, load_fn): - """Create a new `LoadBalancingStrategy`. - - Args: - num_tasks: Number of ps tasks to cycle among. - load_fn: A callable that takes an `Operation` and returns a - numeric load value for that op. - """ - self._num_tasks = num_tasks - self._load_fn = load_fn - self._ps_loads = np.zeros(num_tasks) - - def __call__(self, op): - """Choose a ps task index for the given `Operation`. - - Args: - op: A `Operation` to be placed on ps. - - Returns: - The next ps task index to use for the `Operation`. Greedily - places the op on the least-loaded ps task so far, as determined - by the load function. - """ - task = np.argmin(self._ps_loads) - self._ps_loads[task] += self._load_fn(op) - return task - - -def byte_size_load_fn(op): - """Load function that computes the byte size of a single-output `Operation`. - - This is intended to be used with `"Variable"` ops, which have a single - `Tensor` output with the contents of the variable. However, it can also be - used for calculating the size of any op that has a single output. - - Intended to be used with `GreedyLoadBalancingStrategy`. - - Args: - op: An `Operation` with a single output, typically a "Variable" op. - - Returns: - The number of bytes in the output `Tensor`. - - Raises: - ValueError: if `op` does not have a single output, or if the shape of the - single output is not fully-defined. - """ - if len(op.outputs) != 1: - raise ValueError("Op %s must have a single output" % op) - output = op.outputs[0] - elem_size = output.dtype.size - shape = output.get_shape() - if not shape.is_fully_defined(): - # Due to legacy behavior, scalar "Variable" ops have output Tensors that - # have unknown shape when the op is created (and hence passed to this - # load function for placement), even though the scalar shape is set - # explicitly immediately afterward. - shape = tensor_shape.TensorShape(op.get_attr("shape")) - shape.assert_is_fully_defined() - return shape.num_elements() * elem_size diff --git a/tensorflow/contrib/training/python/training/device_setter_test.py b/tensorflow/contrib/training/python/training/device_setter_test.py deleted file mode 100644 index 3bb2dce83d5..00000000000 --- a/tensorflow/contrib/training/python/training/device_setter_test.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.contrib.training.device_setter.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -from tensorflow.contrib.training.python.training import device_setter as device_setter_lib -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import device_setter -from tensorflow.python.training import server_lib - -_CLUSTER_SPEC = server_lib.ClusterSpec({ - "ps": ["ps0:2222", "ps1:2222"], - "worker": ["worker0:2222", "worker1:2222", "worker2:2222"] -}) - -MockOperation = collections.namedtuple("MockOperation", "name") - - -class RandomStrategyTest(test.TestCase): - - def testBasic(self): - ps_strategy = device_setter_lib.RandomStrategy(2, seed=0) - with ops.device( - device_setter.replica_device_setter( - cluster=_CLUSTER_SPEC, - ps_strategy=ps_strategy)): - u = variables.Variable(array_ops.zeros([2, 2])) - v = variables.Variable(array_ops.zeros([2, 1])) - w = variables.Variable(array_ops.zeros([2, 2])) - x = variables.Variable(array_ops.zeros([1, 3])) - a = v + w - # Randomly distributed with seed 0. - self.assertDeviceEqual("/job:ps/task:1", u.device) - self.assertDeviceEqual("/job:ps/task:1", u.initializer.device) - self.assertDeviceEqual("/job:ps/task:0", v.device) - self.assertDeviceEqual("/job:ps/task:0", v.initializer.device) - self.assertDeviceEqual("/job:ps/task:1", w.device) - self.assertDeviceEqual("/job:ps/task:1", w.initializer.device) - self.assertDeviceEqual("/job:ps/task:1", x.device) - self.assertDeviceEqual("/job:ps/task:1", x.initializer.device) - self.assertDeviceEqual("/job:worker", a.device) - - def testHandlesUnicode(self): - op = MockOperation(u"A unicode \u018e string \xf1") - ps_strategy = device_setter_lib.RandomStrategy(2, seed=0) - ps_task = ps_strategy(op) - self.assertEqual(ps_task, 1) - - -class GreedyLoadBalancingStrategyTest(test.TestCase): - - def testUniformLoadEqualsRoundRobin(self): - - def _load_fn(unused_op): - return 1 - - with ops.device( - device_setter.replica_device_setter( - cluster=_CLUSTER_SPEC, - ps_strategy=device_setter_lib.GreedyLoadBalancingStrategy( - 2, _load_fn))): - u = variables.Variable(array_ops.zeros([2, 2])) - v = variables.Variable(array_ops.zeros([2, 1])) - w = variables.Variable(array_ops.zeros([2, 2])) - x = variables.Variable(array_ops.zeros([1, 3])) - a = v + w - self.assertDeviceEqual("/job:ps/task:0", u.device) - self.assertDeviceEqual("/job:ps/task:0", u.initializer.device) - self.assertDeviceEqual("/job:ps/task:1", v.device) - self.assertDeviceEqual("/job:ps/task:1", v.initializer.device) - self.assertDeviceEqual("/job:ps/task:0", w.device) - self.assertDeviceEqual("/job:ps/task:0", w.initializer.device) - self.assertDeviceEqual("/job:ps/task:1", x.device) - self.assertDeviceEqual("/job:ps/task:1", x.initializer.device) - self.assertDeviceEqual("/job:worker", a.device) - - def testByteSizeLoadFn(self): - with ops.device( - device_setter.replica_device_setter( - cluster=_CLUSTER_SPEC, - ps_strategy=device_setter_lib.GreedyLoadBalancingStrategy( - 2, device_setter_lib.byte_size_load_fn))): - u = variables.VariableV1(array_ops.zeros([2, 2])) - v = variables.VariableV1(array_ops.zeros([2, 1])) - w = variables.VariableV1(array_ops.zeros([2, 2])) - x = variables.VariableV1(array_ops.zeros([1, 3])) - a = v + w - self.assertDeviceEqual("/job:ps/task:0", u.device) - self.assertDeviceEqual("/job:ps/task:0", u.initializer.device) - self.assertDeviceEqual("/job:ps/task:1", v.device) - self.assertDeviceEqual("/job:ps/task:1", v.initializer.device) - self.assertDeviceEqual("/job:ps/task:1", w.device) - self.assertDeviceEqual("/job:ps/task:1", w.initializer.device) - self.assertDeviceEqual("/job:ps/task:0", x.device) - self.assertDeviceEqual("/job:ps/task:0", x.initializer.device) - self.assertDeviceEqual("/job:worker", a.device) - - def testByteSizeLoadFnWithScalar(self): - with ops.device( - device_setter.replica_device_setter( - cluster=_CLUSTER_SPEC, - ps_strategy=device_setter_lib.GreedyLoadBalancingStrategy( - 2, device_setter_lib.byte_size_load_fn))): - # Note: we must test the load function as part of the device function - # instead of passing u.op to the function directly, because the only - # time that the output Tensor has unknown shape for scalars is during - # Variable construction. - u = variables.Variable(0) - self.assertDeviceEqual("/job:ps/task:0", u.device) - self.assertDeviceEqual("/job:ps/task:0", u.initializer.device) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/training/python/training/evaluation.py b/tensorflow/contrib/training/python/training/evaluation.py deleted file mode 100644 index e51854ce159..00000000000 --- a/tensorflow/contrib/training/python/training/evaluation.py +++ /dev/null @@ -1,463 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains functions for evaluation and summarization of metrics. - -The evaluation.py module contains helper functions for evaluating TensorFlow -modules using a variety of metrics and summarizing the results. - -**************************************** -* Evaluating a Checkpointed Model Once * -**************************************** - -Once we've trained a model, we'll want to evaluate it. The simplest way to do -this is to evaluate the performance of a saved model a single time. In order -to do this, we can specify a number of metrics we'll want to evaluate as well -as specify the summaries we want to save to disk. Furthermore, we can print -out the metrics values to stdout: - - # Specify where the checkpoint is stored: - checkpoint_path = ... - - # Create model and obtain the predictions: - images, labels = LoadData(...) - predictions = MyModel(images) - - # Choose the metrics to compute: - names_to_values, names_to_updates = tf.contrib.metrics.aggregate_metric_map({ - "accuracy": tf.compat.v1.metrics.accuracy(labels, predictions), - "mse": tf.compat.v1.metrics.mean_squared_error(labels, predictions), - }) - - # Define the summaries to write: - for metric_name, metric_value in metrics_to_values.iteritems(): - tf.compat.v1.summary.scalar(metric_name, metric_value) - - checkpoint_dir = '/tmp/my_model_dir/' - log_dir = '/tmp/my_model_eval/' - - # We'll evaluate 1000 batches: - num_evals = 1000 - - names_to_values = evaluate_once( - checkpoint_path=checkpoint_path, - eval_ops=names_to_updates.values(), - final_ops=names_to_values, - hooks=[ - tf.contrib.training.StopAfterNEvalsHook(num_evals), - tf.contrib.training.SummaryAtEndHook(logdir), - ], - config=None) - - for name in names_to_values: - print('Metric %s has value %f.' % (name, names_to_values[name])) - - -************************************************ -* Evaluating a Checkpointed Model with Metrics * -************************************************ - -Often, one wants to evaluate a model checkpoint saved on disk. This can be -performed once or repeatedly on a set schedule. - -To evaluate a particular model, users define zero or more metrics and zero or -more summaries and call the evaluate_repeatedly method: - - # Create model and obtain the predictions: - images, labels = LoadData(...) - predictions = MyModel(images) - - # Choose the metrics to compute: - names_to_values, names_to_updates = tf.contrib.metrics.aggregate_metric_map({ - "accuracy": tf.compat.v1.metrics.accuracy(labels, predictions), - "mse": tf.compat.v1.metrics.mean_squared_error(labels, predictions), - }) - - # Define the summaries to write: - for metric_name, metric_value in metrics_to_values.iteritems(): - tf.compat.v1.summary.scalar(metric_name, metric_value) - - checkpoint_dir = '/tmp/my_model_dir/' - log_dir = '/tmp/my_model_eval/' - - # We'll evaluate 1000 batches: - num_evals = 1000 - - # Evaluate every 10 minutes: - tf.contrib.training.evaluate_repeatedly( - checkpoint_dir, - eval_ops=names_to_updates.values(), - hooks=[ - tf.contrib.training.StopAfterNEvalsHook(num_evals), - tf.contrib.training.SummaryAtEndHook(logdir), - ], - eval_interval_secs=600) - -******************************************************* -* Evaluating a Checkpointed Model with Summaries Only * -******************************************************* - -At times, an evaluation can be performed without metrics at all but rather -with only summaries. The user need only leave out the 'eval_ops' argument: - - # Create model and obtain the predictions: - images, labels = LoadData(...) - predictions = MyModel(images) - - # Define the summaries to write: - tf.compat.v1.summary.scalar(...) - tf.compat.v1.summary.histogram(...) - - checkpoint_dir = '/tmp/my_model_dir/' - log_dir = '/tmp/my_model_eval/' - - # Evaluate once every 10 minutes. - tf.contrib.training.evaluate_repeatedly( - checkpoint_dir, - hooks=[ - tf.contrib.training.SummaryAtEndHook(logdir), - ], - eval_interval_secs=600) - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time - -from tensorflow.python.ops import state_ops -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.summary import summary -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.training import checkpoint_management -from tensorflow.python.training import evaluation -from tensorflow.python.training import monitored_session -from tensorflow.python.training import session_run_hook -from tensorflow.python.training import training_util - -__all__ = [ - 'StopAfterNEvalsHook', - 'SummaryAtEndHook', - 'checkpoints_iterator', - 'evaluate_once', - 'evaluate_repeatedly', - 'get_or_create_eval_step', - 'wait_for_new_checkpoint', -] - -# pylint: disable=protected-access -# pylint: disable=invalid-name -StopAfterNEvalsHook = evaluation._StopAfterNEvalsHook -evaluate_once = evaluation._evaluate_once -get_or_create_eval_step = evaluation._get_or_create_eval_step - -# pylint: enable=invalid-name -# pylint: enable=protected-access - - -def wait_for_new_checkpoint(checkpoint_dir, - last_checkpoint=None, - seconds_to_sleep=1, - timeout=None): - """Waits until a new checkpoint file is found. - - Args: - checkpoint_dir: The directory in which checkpoints are saved. - last_checkpoint: The last checkpoint path used or `None` if we're expecting - a checkpoint for the first time. - seconds_to_sleep: The number of seconds to sleep for before looking for a - new checkpoint. - timeout: The maximum number of seconds to wait. If left as `None`, then the - process will wait indefinitely. - - Returns: - a new checkpoint path, or None if the timeout was reached. - """ - logging.info('Waiting for new checkpoint at %s', checkpoint_dir) - stop_time = time.time() + timeout if timeout is not None else None - while True: - checkpoint_path = checkpoint_management.latest_checkpoint(checkpoint_dir) - if checkpoint_path is None or checkpoint_path == last_checkpoint: - if stop_time is not None and time.time() + seconds_to_sleep > stop_time: - return None - time.sleep(seconds_to_sleep) - else: - logging.info('Found new checkpoint at %s', checkpoint_path) - return checkpoint_path - - -def checkpoints_iterator(checkpoint_dir, - min_interval_secs=0, - timeout=None, - timeout_fn=None): - """Continuously yield new checkpoint files as they appear. - - The iterator only checks for new checkpoints when control flow has been - reverted to it. This means it can miss checkpoints if your code takes longer - to run between iterations than `min_interval_secs` or the interval at which - new checkpoints are written. - - The `timeout` argument is the maximum number of seconds to block waiting for - a new checkpoint. It is used in combination with the `timeout_fn` as - follows: - - * If the timeout expires and no `timeout_fn` was specified, the iterator - stops yielding. - * If a `timeout_fn` was specified, that function is called and if it returns - a true boolean value the iterator stops yielding. - * If the function returns a false boolean value then the iterator resumes the - wait for new checkpoints. At this point the timeout logic applies again. - - This behavior gives control to callers on what to do if checkpoints do not - come fast enough or stop being generated. For example, if callers have a way - to detect that the training has stopped and know that no new checkpoints - will be generated, they can provide a `timeout_fn` that returns `True` when - the training has stopped. If they know that the training is still going on - they return `False` instead. - - Args: - checkpoint_dir: The directory in which checkpoints are saved. - min_interval_secs: The minimum number of seconds between yielding - checkpoints. - timeout: The maximum number of seconds to wait between checkpoints. If left - as `None`, then the process will wait indefinitely. - timeout_fn: Optional function to call after a timeout. If the function - returns True, then it means that no new checkpoints will be generated and - the iterator will exit. The function is called with no arguments. - - Yields: - String paths to latest checkpoint files as they arrive. - """ - checkpoint_path = None - while True: - new_checkpoint_path = wait_for_new_checkpoint( - checkpoint_dir, checkpoint_path, timeout=timeout) - if new_checkpoint_path is None: - if not timeout_fn: - # timed out - logging.info('Timed-out waiting for a checkpoint.') - return - if timeout_fn(): - # The timeout_fn indicated that we are truly done. - return - else: - # The timeout_fn indicated that more checkpoints may come. - continue - start = time.time() - checkpoint_path = new_checkpoint_path - yield checkpoint_path - time_to_next_eval = start + min_interval_secs - time.time() - if time_to_next_eval > 0: - time.sleep(time_to_next_eval) - - -class SummaryAtEndHook(session_run_hook.SessionRunHook): - """A run hook that saves a summary with the results of evaluation.""" - - def __init__(self, - log_dir=None, - summary_writer=None, - summary_op=None, - feed_dict=None): - """Constructs the Summary Hook. - - Args: - log_dir: The directory where the summary events are saved to. Used only - when `summary_writer` is not specified. - summary_writer: A `tf.compat.v1.summary.FileWriter` to write summary - events with. - summary_op: The summary op to run. If left as `None`, then all summaries - in the tf.GraphKeys.SUMMARIES collection are used. - feed_dict: An optional feed dictionary to use when evaluating the - summaries. - - Raises: - ValueError: If both `log_dir` and `summary_writer` are `None`. - """ - self._summary_op = summary_op - self._replace_summary_op = summary_op is None - self._feed_dict = feed_dict - self._summary_writer = summary_writer - self._log_dir = log_dir - if self._log_dir is None and self._summary_writer is None: - raise ValueError('One of log_dir or summary_writer should be used.') - - def begin(self): - if self._replace_summary_op: - # This can still remain None if there are no summaries. - self._summary_op = summary.merge_all() - self._global_step = training_util.get_or_create_global_step() - - def after_create_session(self, session, coord): - if self._summary_writer is None and self._log_dir: - self._summary_writer = summary.FileWriterCache.get(self._log_dir) - - def end(self, session): - if self._summary_op is not None: - global_step = training_util.global_step(session, self._global_step) - summary_str = session.run(self._summary_op, self._feed_dict) - if self._summary_writer: - self._summary_writer.add_summary(summary_str, global_step) - if self._summary_writer: - self._summary_writer.flush() - - -def _scaffold_with_init(scaffold, saver, checkpoint_path): - """Creates a scaffold that loads the given checkpoint using an init_fn. - - Args: - scaffold: The scaffold to copy. - saver: The saver to use when restoring the checkpoint. - checkpoint_path: An absolute path to a checkpoint. - - Returns: - A scaffold with an init_fn that loads the given checkpoint. If the scaffold - provided already has an init_fn, the scaffold is returned unchanged. - """ - - def restore_checkpoint(_, session): - saver.restore(session, checkpoint_path) - - if not scaffold.init_fn: - scaffold = monitored_session.Scaffold( - init_op=scaffold.init_op, - init_feed_dict=scaffold.init_feed_dict, - init_fn=restore_checkpoint, - ready_op=scaffold.ready_op, - local_init_op=scaffold.local_init_op, - summary_op=scaffold.summary_op, - saver=scaffold.saver) - return scaffold - - -def evaluate_repeatedly(checkpoint_dir, - master='', - scaffold=None, - eval_ops=None, - feed_dict=None, - final_ops=None, - final_ops_feed_dict=None, - eval_interval_secs=60, - hooks=None, - config=None, - max_number_of_evaluations=None, - timeout=None, - timeout_fn=None): - """Repeatedly searches for a checkpoint in `checkpoint_dir` and evaluates it. - - During a single evaluation, the `eval_ops` is run until the session is - interrupted or requested to finish. This is typically requested via a - `tf.contrib.training.StopAfterNEvalsHook` which results in `eval_ops` running - the requested number of times. - - Optionally, a user can pass in `final_ops`, a single `Tensor`, a list of - `Tensors` or a dictionary from names to `Tensors`. The `final_ops` is - evaluated a single time after `eval_ops` has finished running and the fetched - values of `final_ops` are returned. If `final_ops` is left as `None`, then - `None` is returned. - - One may also consider using a `tf.contrib.training.SummaryAtEndHook` to record - summaries after the `eval_ops` have run. If `eval_ops` is `None`, the - summaries run immediately after the model checkpoint has been restored. - - Note that `evaluate_once` creates a local variable used to track the number of - evaluations run via `tf.contrib.training.get_or_create_eval_step`. - Consequently, if a custom local init op is provided via a `scaffold`, the - caller should ensure that the local init op also initializes the eval step. - - Args: - checkpoint_dir: The directory where checkpoints are stored. - master: The address of the TensorFlow master. - scaffold: An tf.compat.v1.train.Scaffold instance for initializing variables - and restoring variables. Note that `scaffold.init_fn` is used by the - function to restore the checkpoint. If you supply a custom init_fn, then - it must also take care of restoring the model from its checkpoint. - eval_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names to - `Tensors`, which is run until the session is requested to stop, commonly - done by a `tf.contrib.training.StopAfterNEvalsHook`. - feed_dict: The feed dictionary to use when executing the `eval_ops`. - final_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names - to `Tensors`. - final_ops_feed_dict: A feed dictionary to use when evaluating `final_ops`. - eval_interval_secs: The minimum number of seconds between evaluations. - hooks: List of `tf.estimator.SessionRunHook` callbacks which are run inside - the evaluation loop. - config: An instance of `tf.compat.v1.ConfigProto` that will be used to - configure the `Session`. If left as `None`, the default will be used. - max_number_of_evaluations: The maximum times to run the evaluation. If left - as `None`, then evaluation runs indefinitely. - timeout: The maximum number of seconds to wait between checkpoints. If left - as `None`, then the process will wait indefinitely. - timeout_fn: Optional function to call after a timeout. If the function - returns True, then it means that no new checkpoints will be generated and - the iterator will exit. The function is called with no arguments. - - Returns: - The fetched values of `final_ops` or `None` if `final_ops` is `None`. - """ - eval_step = get_or_create_eval_step() - - # Prepare the run hooks. - hooks = hooks or [] - - if eval_ops is not None: - update_eval_step = state_ops.assign_add(eval_step, 1) - - for h in hooks: - if isinstance(h, StopAfterNEvalsHook): - h._set_evals_completed_tensor(update_eval_step) # pylint: disable=protected-access - - if isinstance(eval_ops, dict): - eval_ops['update_eval_step'] = update_eval_step - elif isinstance(eval_ops, (tuple, list)): - eval_ops = list(eval_ops) + [update_eval_step] - else: - eval_ops = [eval_ops, update_eval_step] - - final_ops_hook = basic_session_run_hooks.FinalOpsHook(final_ops, - final_ops_feed_dict) - hooks.append(final_ops_hook) - - num_evaluations = 0 - for checkpoint_path in checkpoints_iterator( - checkpoint_dir, - min_interval_secs=eval_interval_secs, - timeout=timeout, - timeout_fn=timeout_fn): - - session_creator = monitored_session.ChiefSessionCreator( - scaffold=scaffold, - checkpoint_filename_with_path=checkpoint_path, - master=master, - config=config) - - with monitored_session.MonitoredSession( - session_creator=session_creator, hooks=hooks) as session: - logging.info('Starting evaluation at ' + - time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime())) - if eval_ops is not None: - while not session.should_stop(): - session.run(eval_ops, feed_dict) - - logging.info('Finished evaluation at ' + - time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime())) - num_evaluations += 1 - - if (max_number_of_evaluations is not None and - num_evaluations >= max_number_of_evaluations): - return final_ops_hook.final_ops_values - - return final_ops_hook.final_ops_values diff --git a/tensorflow/contrib/training/python/training/evaluation_test.py b/tensorflow/contrib/training/python/training/evaluation_test.py deleted file mode 100644 index ddd135f0474..00000000000 --- a/tensorflow/contrib/training/python/training/evaluation_test.py +++ /dev/null @@ -1,507 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.contrib.training.evaluation.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import glob -import os -import time - -import numpy as np - -from tensorflow.contrib.framework.python.ops import variables -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.losses.python.losses import loss_ops -from tensorflow.contrib.training.python.training import evaluation -from tensorflow.contrib.training.python.training import training -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session as session_lib -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import metrics -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test -from tensorflow.python.summary import summary as summary_lib -from tensorflow.python.summary import summary_iterator -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.training import gradient_descent -from tensorflow.python.training import saver as saver_lib - - -class CheckpointIteratorTest(test.TestCase): - - def testReturnsEmptyIfNoCheckpointsFound(self): - checkpoint_dir = os.path.join(self.get_temp_dir(), 'no_checkpoints_found') - - num_found = 0 - for _ in evaluation.checkpoints_iterator(checkpoint_dir, timeout=0): - num_found += 1 - self.assertEqual(num_found, 0) - - def testReturnsSingleCheckpointIfOneCheckpointFound(self): - checkpoint_dir = os.path.join(self.get_temp_dir(), 'one_checkpoint_found') - if not gfile.Exists(checkpoint_dir): - gfile.MakeDirs(checkpoint_dir) - - global_step = variables.get_or_create_global_step() - saver = saver_lib.Saver() # Saves the global step. - - with self.cached_session() as session: - session.run(variables_lib.global_variables_initializer()) - save_path = os.path.join(checkpoint_dir, 'model.ckpt') - saver.save(session, save_path, global_step=global_step) - - num_found = 0 - for _ in evaluation.checkpoints_iterator(checkpoint_dir, timeout=0): - num_found += 1 - self.assertEqual(num_found, 1) - - def testReturnsSingleCheckpointIfOneShardedCheckpoint(self): - checkpoint_dir = os.path.join(self.get_temp_dir(), - 'one_checkpoint_found_sharded') - if not gfile.Exists(checkpoint_dir): - gfile.MakeDirs(checkpoint_dir) - - global_step = variables.get_or_create_global_step() - - # This will result in 3 different checkpoint shard files. - with ops.device('/cpu:0'): - variables_lib.Variable(10, name='v0') - with ops.device('/cpu:1'): - variables_lib.Variable(20, name='v1') - - saver = saver_lib.Saver(sharded=True) - - with session_lib.Session( - target='', - config=config_pb2.ConfigProto(device_count={'CPU': 2})) as session: - - session.run(variables_lib.global_variables_initializer()) - save_path = os.path.join(checkpoint_dir, 'model.ckpt') - saver.save(session, save_path, global_step=global_step) - - num_found = 0 - for _ in evaluation.checkpoints_iterator(checkpoint_dir, timeout=0): - num_found += 1 - self.assertEqual(num_found, 1) - - def testTimeoutFn(self): - timeout_fn_calls = [0] - def timeout_fn(): - timeout_fn_calls[0] += 1 - return timeout_fn_calls[0] > 3 - - results = list( - evaluation.checkpoints_iterator( - '/non-existent-dir', timeout=0.1, timeout_fn=timeout_fn)) - self.assertEqual([], results) - self.assertEqual(4, timeout_fn_calls[0]) - - -class WaitForNewCheckpointTest(test.TestCase): - - def testReturnsNoneAfterTimeout(self): - start = time.time() - ret = evaluation.wait_for_new_checkpoint( - '/non-existent-dir', 'foo', timeout=1.0, seconds_to_sleep=0.5) - end = time.time() - self.assertIsNone(ret) - - # We've waited one second. - self.assertGreater(end, start + 0.5) - - # The timeout kicked in. - self.assertLess(end, start + 1.1) - - -def logistic_classifier(inputs): - return layers.fully_connected(inputs, 1, activation_fn=math_ops.sigmoid) - - -class EvaluateOnceTest(test.TestCase): - - def setUp(self): - super(EvaluateOnceTest, self).setUp() - - # Create an easy training set: - np.random.seed(0) - - self._inputs = np.zeros((16, 4)) - self._labels = np.random.randint(0, 2, size=(16, 1)).astype(np.float32) - - for i in range(16): - j = int(2 * self._labels[i] + np.random.randint(0, 2)) - self._inputs[i, j] = 1 - - def _train_model(self, checkpoint_dir, num_steps): - """Trains a simple classification model. - - Note that the data has been configured such that after around 300 steps, - the model has memorized the dataset (e.g. we can expect %100 accuracy). - - Args: - checkpoint_dir: The directory where the checkpoint is written to. - num_steps: The number of steps to train for. - """ - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = logistic_classifier(tf_inputs) - loss = loss_ops.log_loss(tf_predictions, tf_labels) - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - train_op = training.create_train_op(loss, optimizer) - - loss = training.train( - train_op, - checkpoint_dir, - hooks=[basic_session_run_hooks.StopAtStepHook(num_steps)]) - - if num_steps >= 300: - assert loss < .015 - - def testEvaluatePerfectModel(self): - checkpoint_dir = os.path.join(self.get_temp_dir(), - 'evaluate_perfect_model_once') - - # Train a Model to completion: - self._train_model(checkpoint_dir, num_steps=300) - - # Run - inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - labels = constant_op.constant(self._labels, dtype=dtypes.float32) - logits = logistic_classifier(inputs) - predictions = math_ops.round(logits) - - accuracy, update_op = metrics.accuracy( - predictions=predictions, labels=labels) - - checkpoint_path = evaluation.wait_for_new_checkpoint(checkpoint_dir) - - final_ops_values = evaluation.evaluate_once( - checkpoint_path=checkpoint_path, - eval_ops=update_op, - final_ops={'accuracy': accuracy}, - hooks=[ - evaluation.StopAfterNEvalsHook(1), - ]) - self.assertTrue(final_ops_values['accuracy'] > .99) - - def testEvalOpAndFinalOp(self): - checkpoint_dir = os.path.join(self.get_temp_dir(), 'eval_ops_and_final_ops') - - # Train a model for a single step to get a checkpoint. - self._train_model(checkpoint_dir, num_steps=1) - checkpoint_path = evaluation.wait_for_new_checkpoint(checkpoint_dir) - - # Create the model so we have something to restore. - inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - logistic_classifier(inputs) - - num_evals = 5 - final_increment = 9.0 - - my_var = variables.local_variable(0.0, name='MyVar') - eval_ops = state_ops.assign_add(my_var, 1.0) - final_ops = array_ops.identity(my_var) + final_increment - - final_ops_values = evaluation.evaluate_once( - checkpoint_path=checkpoint_path, - eval_ops=eval_ops, - final_ops={'value': final_ops}, - hooks=[ - evaluation.StopAfterNEvalsHook(num_evals), - ]) - self.assertEqual(final_ops_values['value'], num_evals + final_increment) - - def testOnlyFinalOp(self): - checkpoint_dir = os.path.join(self.get_temp_dir(), 'only_final_ops') - - # Train a model for a single step to get a checkpoint. - self._train_model(checkpoint_dir, num_steps=1) - checkpoint_path = evaluation.wait_for_new_checkpoint(checkpoint_dir) - - # Create the model so we have something to restore. - inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - logistic_classifier(inputs) - - final_increment = 9.0 - - my_var = variables.local_variable(0.0, name='MyVar') - final_ops = array_ops.identity(my_var) + final_increment - - final_ops_values = evaluation.evaluate_once( - checkpoint_path=checkpoint_path, final_ops={'value': final_ops}) - self.assertEqual(final_ops_values['value'], final_increment) - - -class EvaluateRepeatedlyTest(test.TestCase): - - def setUp(self): - super(EvaluateRepeatedlyTest, self).setUp() - - # Create an easy training set: - np.random.seed(0) - - self._inputs = np.zeros((16, 4)) - self._labels = np.random.randint(0, 2, size=(16, 1)).astype(np.float32) - - for i in range(16): - j = int(2 * self._labels[i] + np.random.randint(0, 2)) - self._inputs[i, j] = 1 - - def _train_model(self, checkpoint_dir, num_steps): - """Trains a simple classification model. - - Note that the data has been configured such that after around 300 steps, - the model has memorized the dataset (e.g. we can expect %100 accuracy). - - Args: - checkpoint_dir: The directory where the checkpoint is written to. - num_steps: The number of steps to train for. - """ - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = logistic_classifier(tf_inputs) - loss = loss_ops.log_loss(tf_predictions, tf_labels) - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - train_op = training.create_train_op(loss, optimizer) - - loss = training.train( - train_op, - checkpoint_dir, - hooks=[basic_session_run_hooks.StopAtStepHook(num_steps)]) - - def testEvaluatePerfectModel(self): - checkpoint_dir = os.path.join(self.get_temp_dir(), - 'evaluate_perfect_model_repeated') - - # Train a Model to completion: - self._train_model(checkpoint_dir, num_steps=300) - - # Run - inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - labels = constant_op.constant(self._labels, dtype=dtypes.float32) - logits = logistic_classifier(inputs) - predictions = math_ops.round(logits) - - accuracy, update_op = metrics.accuracy( - predictions=predictions, labels=labels) - - final_values = evaluation.evaluate_repeatedly( - checkpoint_dir=checkpoint_dir, - eval_ops=update_op, - final_ops={'accuracy': accuracy}, - hooks=[ - evaluation.StopAfterNEvalsHook(1), - ], - max_number_of_evaluations=1) - self.assertTrue(final_values['accuracy'] > .99) - - def testEvaluationLoopTimeout(self): - checkpoint_dir = os.path.join(self.get_temp_dir(), - 'evaluation_loop_timeout') - if not gfile.Exists(checkpoint_dir): - gfile.MakeDirs(checkpoint_dir) - - # We need a variable that the saver will try to restore. - variables.get_or_create_global_step() - - # Run with placeholders. If we actually try to evaluate this, we'd fail - # since we're not using a feed_dict. - cant_run_op = array_ops.placeholder(dtype=dtypes.float32) - - start = time.time() - final_values = evaluation.evaluate_repeatedly( - checkpoint_dir=checkpoint_dir, - eval_ops=cant_run_op, - hooks=[evaluation.StopAfterNEvalsHook(10)], - timeout=6) - end = time.time() - self.assertFalse(final_values) - - # Assert that we've waited for the duration of the timeout (minus the sleep - # time). - self.assertGreater(end - start, 5.0) - - # Then the timeout kicked in and stops the loop. - self.assertLess(end - start, 7) - - def testEvaluationLoopTimeoutWithTimeoutFn(self): - checkpoint_dir = os.path.join(self.get_temp_dir(), - 'evaluation_loop_timeout_with_timeout_fn') - - # Train a Model to completion: - self._train_model(checkpoint_dir, num_steps=300) - - # Run - inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - labels = constant_op.constant(self._labels, dtype=dtypes.float32) - logits = logistic_classifier(inputs) - predictions = math_ops.round(logits) - - accuracy, update_op = metrics.accuracy( - predictions=predictions, labels=labels) - - timeout_fn_calls = [0] - def timeout_fn(): - timeout_fn_calls[0] += 1 - return timeout_fn_calls[0] > 3 - - final_values = evaluation.evaluate_repeatedly( - checkpoint_dir=checkpoint_dir, - eval_ops=update_op, - final_ops={'accuracy': accuracy}, - hooks=[ - evaluation.StopAfterNEvalsHook(1), - ], - eval_interval_secs=1, - max_number_of_evaluations=2, - timeout=0.1, - timeout_fn=timeout_fn) - # We should have evaluated once. - self.assertTrue(final_values['accuracy'] > .99) - # And called 4 times the timeout fn - self.assertEqual(4, timeout_fn_calls[0]) - - def testEvaluateWithEvalFeedDict(self): - # Create a checkpoint. - checkpoint_dir = os.path.join(self.get_temp_dir(), - 'evaluate_with_eval_feed_dict') - self._train_model(checkpoint_dir, num_steps=1) - - # We need a variable that the saver will try to restore. - variables.get_or_create_global_step() - - # Create a variable and an eval op that increments it with a placeholder. - my_var = variables.local_variable(0.0, name='my_var') - increment = array_ops.placeholder(dtype=dtypes.float32) - eval_ops = state_ops.assign_add(my_var, increment) - - increment_value = 3 - num_evals = 5 - expected_value = increment_value * num_evals - final_values = evaluation.evaluate_repeatedly( - checkpoint_dir=checkpoint_dir, - eval_ops=eval_ops, - feed_dict={increment: 3}, - final_ops={'my_var': array_ops.identity(my_var)}, - hooks=[ - evaluation.StopAfterNEvalsHook(num_evals), - ], - max_number_of_evaluations=1) - self.assertEqual(final_values['my_var'], expected_value) - - def _create_names_to_metrics(self, predictions, labels): - accuracy0, update_op0 = metrics.accuracy(labels, predictions) - accuracy1, update_op1 = metrics.accuracy(labels, predictions + 1) - - names_to_values = {'Accuracy': accuracy0, 'Another_accuracy': accuracy1} - names_to_updates = {'Accuracy': update_op0, 'Another_accuracy': update_op1} - return names_to_values, names_to_updates - - def _verify_events(self, output_dir, names_to_values): - """Verifies that the given `names_to_values` are found in the summaries. - - Also checks that a GraphDef was written out to the events file. - - Args: - output_dir: An existing directory where summaries are found. - names_to_values: A dictionary of strings to values. - """ - # Check that the results were saved. The events file may have additional - # entries, e.g. the event version stamp, so have to parse things a bit. - output_filepath = glob.glob(os.path.join(output_dir, '*')) - self.assertEqual(len(output_filepath), 1) - - events = summary_iterator.summary_iterator(output_filepath[0]) - summaries = [] - graph_def = None - for event in events: - if event.summary.value: - summaries.append(event.summary) - elif event.graph_def: - graph_def = event.graph_def - values = [] - for summary in summaries: - for value in summary.value: - values.append(value) - saved_results = {v.tag: v.simple_value for v in values} - for name in names_to_values: - self.assertAlmostEqual(names_to_values[name], saved_results[name], 5) - self.assertIsNotNone(graph_def) - - def testSummariesAreFlushedToDisk(self): - checkpoint_dir = os.path.join(self.get_temp_dir(), 'summaries_are_flushed') - logdir = os.path.join(self.get_temp_dir(), 'summaries_are_flushed_eval') - if gfile.Exists(logdir): - gfile.DeleteRecursively(logdir) - - # Train a Model to completion: - self._train_model(checkpoint_dir, num_steps=300) - - # Create the model (which can be restored). - inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - logistic_classifier(inputs) - - names_to_values = {'bread': 3.4, 'cheese': 4.5, 'tomato': 2.0} - - for k in names_to_values: - v = names_to_values[k] - summary_lib.scalar(k, v) - - evaluation.evaluate_repeatedly( - checkpoint_dir=checkpoint_dir, - hooks=[ - evaluation.SummaryAtEndHook(log_dir=logdir), - ], - max_number_of_evaluations=1) - - self._verify_events(logdir, names_to_values) - - def testSummaryAtEndHookWithoutSummaries(self): - logdir = os.path.join(self.get_temp_dir(), - 'summary_at_end_hook_without_summaires') - if gfile.Exists(logdir): - gfile.DeleteRecursively(logdir) - - with ops.Graph().as_default(): - # Purposefully don't add any summaries. The hook will just dump the - # GraphDef event. - hook = evaluation.SummaryAtEndHook(log_dir=logdir) - hook.begin() - with self.cached_session() as session: - hook.after_create_session(session, None) - hook.end(session) - self._verify_events(logdir, {}) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/training/python/training/feeding_queue_runner.py b/tensorflow/contrib/training/python/training/feeding_queue_runner.py deleted file mode 100644 index d055555b010..00000000000 --- a/tensorflow/contrib/training/python/training/feeding_queue_runner.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""A `QueueRunner` that takes a feed function as an argument.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.python.estimator.inputs.queues.feeding_queue_runner import _FeedingQueueRunner as FeedingQueueRunner -# pylint: enable=unused-import diff --git a/tensorflow/contrib/training/python/training/feeding_queue_runner_test.py b/tensorflow/contrib/training/python/training/feeding_queue_runner_test.py deleted file mode 100644 index e60e44eb53a..00000000000 --- a/tensorflow/contrib/training/python/training/feeding_queue_runner_test.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests `FeedingQueueRunner` using arrays and `DataFrames`.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.client import session -from tensorflow.python.estimator.inputs.queues.feeding_functions import _enqueue_data as enqueue_data -from tensorflow.python.framework import ops -from tensorflow.python.platform import test -from tensorflow.python.training import coordinator -from tensorflow.python.training import queue_runner_impl - -# pylint: disable=g-import-not-at-top -try: - import pandas as pd - HAS_PANDAS = True -except ImportError: - HAS_PANDAS = False - - -def get_rows(array, row_indices): - rows = [array[i] for i in row_indices] - return np.vstack(rows) - - -class FeedingQueueRunnerTestCase(test.TestCase): - """Tests for `FeedingQueueRunner`.""" - - def testArrayFeeding(self): - with ops.Graph().as_default(): - array = np.arange(32).reshape([16, 2]) - q = enqueue_data(array, capacity=100) - batch_size = 3 - dq_op = q.dequeue_many(batch_size) - with session.Session() as sess: - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(sess=sess, coord=coord) - for i in range(100): - indices = [ - j % array.shape[0] - for j in range(batch_size * i, batch_size * (i + 1)) - ] - expected_dq = get_rows(array, indices) - dq = sess.run(dq_op) - np.testing.assert_array_equal(indices, dq[0]) - np.testing.assert_array_equal(expected_dq, dq[1]) - coord.request_stop() - coord.join(threads) - - def testArrayFeedingMultiThread(self): - with ops.Graph().as_default(): - array = np.arange(256).reshape([128, 2]) - q = enqueue_data(array, capacity=128, num_threads=8, shuffle=True) - batch_size = 3 - dq_op = q.dequeue_many(batch_size) - with session.Session() as sess: - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(sess=sess, coord=coord) - for _ in range(100): - dq = sess.run(dq_op) - indices = dq[0] - expected_dq = get_rows(array, indices) - np.testing.assert_array_equal(expected_dq, dq[1]) - coord.request_stop() - coord.join(threads) - - def testPandasFeeding(self): - if not HAS_PANDAS: - return - with ops.Graph().as_default(): - array1 = np.arange(32) - array2 = np.arange(32, 64) - df = pd.DataFrame({"a": array1, "b": array2}, index=np.arange(64, 96)) - q = enqueue_data(df, capacity=100) - batch_size = 5 - dq_op = q.dequeue_many(5) - with session.Session() as sess: - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(sess=sess, coord=coord) - for i in range(100): - indices = [ - j % array1.shape[0] - for j in range(batch_size * i, batch_size * (i + 1)) - ] - expected_df_indices = df.index[indices] - expected_rows = df.iloc[indices] - dq = sess.run(dq_op) - np.testing.assert_array_equal(expected_df_indices, dq[0]) - for col_num, col in enumerate(df.columns): - np.testing.assert_array_equal(expected_rows[col].values, - dq[col_num + 1]) - coord.request_stop() - coord.join(threads) - - def testPandasFeedingMultiThread(self): - if not HAS_PANDAS: - return - with ops.Graph().as_default(): - array1 = np.arange(128, 256) - array2 = 2 * array1 - df = pd.DataFrame({"a": array1, "b": array2}, index=np.arange(128)) - q = enqueue_data(df, capacity=128, num_threads=8, shuffle=True) - batch_size = 5 - dq_op = q.dequeue_many(batch_size) - with session.Session() as sess: - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(sess=sess, coord=coord) - for _ in range(100): - dq = sess.run(dq_op) - indices = dq[0] - expected_rows = df.iloc[indices] - for col_num, col in enumerate(df.columns): - np.testing.assert_array_equal(expected_rows[col].values, - dq[col_num + 1]) - coord.request_stop() - coord.join(threads) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/training/python/training/hparam.proto b/tensorflow/contrib/training/python/training/hparam.proto deleted file mode 100644 index 67462cd9cfa..00000000000 --- a/tensorflow/contrib/training/python/training/hparam.proto +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2016 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ============================================================================= - -syntax = "proto3"; - -package tensorflow; -option cc_enable_arenas = true; - -// Protocol buffer holding hyper parameters. -// Examples of hyper parameters: -// learning_rate = 0.1, -// num_hidden_units = 100, -// activations = ['relu', 'tanh'] -message HParamDef { - message BytesList { - repeated bytes value = 1; - } - message FloatList { - repeated float value = 1 [packed = true]; - } - message Int64List { - repeated int64 value = 1 [packed = true]; - } - message BoolList { - repeated bool value = 1 [packed = true]; - } - message HParamType { - oneof kind { - int64 int64_value = 1; - float float_value = 2; - bytes bytes_value = 3; - bool bool_value = 7; - Int64List int64_list = 4; - FloatList float_list = 5; - BytesList bytes_list = 6; - BoolList bool_list = 8; - } - }; - map hparam = 1; -} diff --git a/tensorflow/contrib/training/python/training/hparam.py b/tensorflow/contrib/training/python/training/hparam.py deleted file mode 100644 index 7c9e10105af..00000000000 --- a/tensorflow/contrib/training/python/training/hparam.py +++ /dev/null @@ -1,746 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Hyperparameter values.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import json -import numbers -import re - -import six - -from tensorflow.contrib.training.python.training import hparam_pb2 -from tensorflow.python.framework import ops -from tensorflow.python.util import compat -from tensorflow.python.util import deprecation - -# Define the regular expression for parsing a single clause of the input -# (delimited by commas). A legal clause looks like: -# []? = -# where is either a single token or [] enclosed list of tokens. -# For example: "var[1] = a" or "x = [1,2,3]" -PARAM_RE = re.compile(r""" - (?P[a-zA-Z][\w\.]*) # variable name: "var" or "x" - (\[\s*(?P\d+)\s*\])? # (optional) index: "1" or None - \s*=\s* - ((?P[^,\[]*) # single value: "a" or None - | - \[(?P[^\]]*)\]) # list of values: None or "1,2,3" - ($|,\s*)""", re.VERBOSE) - - -def _parse_fail(name, var_type, value, values): - """Helper function for raising a value error for bad assignment.""" - raise ValueError( - 'Could not parse hparam \'%s\' of type \'%s\' with value \'%s\' in %s' % - (name, var_type.__name__, value, values)) - - -def _reuse_fail(name, values): - """Helper function for raising a value error for reuse of name.""" - raise ValueError('Multiple assignments to variable \'%s\' in %s' % (name, - values)) - - -def _process_scalar_value(name, parse_fn, var_type, m_dict, values, - results_dictionary): - """Update results_dictionary with a scalar value. - - Used to update the results_dictionary to be returned by parse_values when - encountering a clause with a scalar RHS (e.g. "s=5" or "arr[0]=5".) - - Mutates results_dictionary. - - Args: - name: Name of variable in assignment ("s" or "arr"). - parse_fn: Function for parsing the actual value. - var_type: Type of named variable. - m_dict: Dictionary constructed from regex parsing. - m_dict['val']: RHS value (scalar) - m_dict['index']: List index value (or None) - values: Full expression being parsed - results_dictionary: The dictionary being updated for return by the parsing - function. - - Raises: - ValueError: If the name has already been used. - """ - try: - parsed_value = parse_fn(m_dict['val']) - except ValueError: - _parse_fail(name, var_type, m_dict['val'], values) - - # If no index is provided - if not m_dict['index']: - if name in results_dictionary: - _reuse_fail(name, values) - results_dictionary[name] = parsed_value - else: - if name in results_dictionary: - # The name has already been used as a scalar, then it - # will be in this dictionary and map to a non-dictionary. - if not isinstance(results_dictionary.get(name), dict): - _reuse_fail(name, values) - else: - results_dictionary[name] = {} - - index = int(m_dict['index']) - # Make sure the index position hasn't already been assigned a value. - if index in results_dictionary[name]: - _reuse_fail('{}[{}]'.format(name, index), values) - results_dictionary[name][index] = parsed_value - - -def _process_list_value(name, parse_fn, var_type, m_dict, values, - results_dictionary): - """Update results_dictionary from a list of values. - - Used to update results_dictionary to be returned by parse_values when - encountering a clause with a list RHS (e.g. "arr=[1,2,3]".) - - Mutates results_dictionary. - - Args: - name: Name of variable in assignment ("arr"). - parse_fn: Function for parsing individual values. - var_type: Type of named variable. - m_dict: Dictionary constructed from regex parsing. - m_dict['val']: RHS value (scalar) - values: Full expression being parsed - results_dictionary: The dictionary being updated for return by the parsing - function. - - Raises: - ValueError: If the name has an index or the values cannot be parsed. - """ - if m_dict['index'] is not None: - raise ValueError('Assignment of a list to a list index.') - elements = filter(None, re.split('[ ,]', m_dict['vals'])) - # Make sure the name hasn't already been assigned a value - if name in results_dictionary: - raise _reuse_fail(name, values) - try: - results_dictionary[name] = [parse_fn(e) for e in elements] - except ValueError: - _parse_fail(name, var_type, m_dict['vals'], values) - - -def _cast_to_type_if_compatible(name, param_type, value): - """Cast hparam to the provided type, if compatible. - - Args: - name: Name of the hparam to be cast. - param_type: The type of the hparam. - value: The value to be cast, if compatible. - - Returns: - The result of casting `value` to `param_type`. - - Raises: - ValueError: If the type of `value` is not compatible with param_type. - * If `param_type` is a string type, but `value` is not. - * If `param_type` is a boolean, but `value` is not, or vice versa. - * If `param_type` is an integer type, but `value` is not. - * If `param_type` is a float type, but `value` is not a numeric type. - """ - fail_msg = ( - "Could not cast hparam '%s' of type '%s' from value %r" % - (name, param_type, value)) - - # If `value` is already of type `param_type`, return it directly. - # `isinstance` is too weak (e.g. isinstance(True, int) == True). - if type(value) == param_type: # pylint: disable=unidiomatic-typecheck - return value - - # Some callers use None, for which we can't do any casting/checking. :( - if issubclass(param_type, type(None)): - return value - - # Avoid converting a non-string type to a string. - if (issubclass(param_type, (six.string_types, six.binary_type)) and - not isinstance(value, (six.string_types, six.binary_type))): - raise ValueError(fail_msg) - - # Avoid converting a number or string type to a boolean or vice versa. - if issubclass(param_type, bool) != isinstance(value, bool): - raise ValueError(fail_msg) - - # Avoid converting float to an integer (the reverse is fine). - if (issubclass(param_type, numbers.Integral) and - not isinstance(value, numbers.Integral)): - raise ValueError(fail_msg) - - # Avoid converting a non-numeric type to a numeric type. - if (issubclass(param_type, numbers.Number) and - not isinstance(value, numbers.Number)): - raise ValueError(fail_msg) - - return param_type(value) - - -def parse_values(values, type_map, ignore_unknown=False): - """Parses hyperparameter values from a string into a python map. - - `values` is a string containing comma-separated `name=value` pairs. - For each pair, the value of the hyperparameter named `name` is set to - `value`. - - If a hyperparameter name appears multiple times in `values`, a ValueError - is raised (e.g. 'a=1,a=2', 'a[1]=1,a[1]=2'). - - If a hyperparameter name in both an index assignment and scalar assignment, - a ValueError is raised. (e.g. 'a=[1,2,3],a[0] = 1'). - - The hyperparameter name may contain '.' symbols, which will result in an - attribute name that is only accessible through the getattr and setattr - functions. (And must be first explicit added through add_hparam.) - - WARNING: Use of '.' in your variable names is allowed, but is not well - supported and not recommended. - - The `value` in `name=value` must follows the syntax according to the - type of the parameter: - - * Scalar integer: A Python-parsable integer point value. E.g.: 1, - 100, -12. - * Scalar float: A Python-parsable floating point value. E.g.: 1.0, - -.54e89. - * Boolean: Either true or false. - * Scalar string: A non-empty sequence of characters, excluding comma, - spaces, and square brackets. E.g.: foo, bar_1. - * List: A comma separated list of scalar values of the parameter type - enclosed in square brackets. E.g.: [1,2,3], [1.0,1e-12], [high,low]. - - When index assignment is used, the corresponding type_map key should be the - list name. E.g. for "arr[1]=0" the type_map must have the key "arr" (not - "arr[1]"). - - Args: - values: String. Comma separated list of `name=value` pairs where - 'value' must follow the syntax described above. - type_map: A dictionary mapping hyperparameter names to types. Note every - parameter name in values must be a key in type_map. The values must - conform to the types indicated, where a value V is said to conform to a - type T if either V has type T, or V is a list of elements of type T. - Hence, for a multidimensional parameter 'x' taking float values, - 'x=[0.1,0.2]' will parse successfully if type_map['x'] = float. - ignore_unknown: Bool. Whether values that are missing a type in type_map - should be ignored. If set to True, a ValueError will not be raised for - unknown hyperparameter type. - - Returns: - A python map mapping each name to either: - * A scalar value. - * A list of scalar values. - * A dictionary mapping index numbers to scalar values. - (e.g. "x=5,L=[1,2],arr[1]=3" results in {'x':5,'L':[1,2],'arr':{1:3}}") - - Raises: - ValueError: If there is a problem with input. - * If `values` cannot be parsed. - * If a list is assigned to a list index (e.g. 'a[1] = [1,2,3]'). - * If the same rvalue is assigned two different values (e.g. 'a=1,a=2', - 'a[1]=1,a[1]=2', or 'a=1,a=[1]') - """ - results_dictionary = {} - pos = 0 - while pos < len(values): - m = PARAM_RE.match(values, pos) - if not m: - raise ValueError('Malformed hyperparameter value: %s' % values[pos:]) - # Check that there is a comma between parameters and move past it. - pos = m.end() - # Parse the values. - m_dict = m.groupdict() - name = m_dict['name'] - if name not in type_map: - if ignore_unknown: - continue - raise ValueError('Unknown hyperparameter type for %s' % name) - type_ = type_map[name] - - # Set up correct parsing function (depending on whether type_ is a bool) - if type_ == bool: - - def parse_bool(value): - if value in ['true', 'True']: - return True - elif value in ['false', 'False']: - return False - else: - try: - return bool(int(value)) - except ValueError: - _parse_fail(name, type_, value, values) - - parse = parse_bool - else: - parse = type_ - - # If a singe value is provided - if m_dict['val'] is not None: - _process_scalar_value(name, parse, type_, m_dict, values, - results_dictionary) - - # If the assigned value is a list: - elif m_dict['vals'] is not None: - _process_list_value(name, parse, type_, m_dict, values, - results_dictionary) - - else: # Not assigned a list or value - _parse_fail(name, type_, '', values) - - return results_dictionary - - -class HParams(object): - """Class to hold a set of hyperparameters as name-value pairs. - - A `HParams` object holds hyperparameters used to build and train a model, - such as the number of hidden units in a neural net layer or the learning rate - to use when training. - - You first create a `HParams` object by specifying the names and values of the - hyperparameters. - - To make them easily accessible the parameter names are added as direct - attributes of the class. A typical usage is as follows: - - ```python - # Create a HParams object specifying names and values of the model - # hyperparameters: - hparams = HParams(learning_rate=0.1, num_hidden_units=100) - - # The hyperparameter are available as attributes of the HParams object: - hparams.learning_rate ==> 0.1 - hparams.num_hidden_units ==> 100 - ``` - - Hyperparameters have type, which is inferred from the type of their value - passed at construction type. The currently supported types are: integer, - float, boolean, string, and list of integer, float, boolean, or string. - - You can override hyperparameter values by calling the - [`parse()`](#HParams.parse) method, passing a string of comma separated - `name=value` pairs. This is intended to make it possible to override - any hyperparameter values from a single command-line flag to which - the user passes 'hyper-param=value' pairs. It avoids having to define - one flag for each hyperparameter. - - The syntax expected for each value depends on the type of the parameter. - See `parse()` for a description of the syntax. - - Example: - - ```python - # Define a command line flag to pass name=value pairs. - # For example using argparse: - import argparse - parser = argparse.ArgumentParser(description='Train my model.') - parser.add_argument('--hparams', type=str, - help='Comma separated list of "name=value" pairs.') - args = parser.parse_args() - ... - def my_program(): - # Create a HParams object specifying the names and values of the - # model hyperparameters: - hparams = tf.contrib.training.HParams( - learning_rate=0.1, - num_hidden_units=100, - activations=['relu', 'tanh']) - - # Override hyperparameters values by parsing the command line - hparams.parse(args.hparams) - - # If the user passed `--hparams=learning_rate=0.3` on the command line - # then 'hparams' has the following attributes: - hparams.learning_rate ==> 0.3 - hparams.num_hidden_units ==> 100 - hparams.activations ==> ['relu', 'tanh'] - - # If the hyperparameters are in json format use parse_json: - hparams.parse_json('{"learning_rate": 0.3, "activations": "relu"}') - ``` - """ - - _HAS_DYNAMIC_ATTRIBUTES = True # Required for pytype checks. - - def __init__(self, hparam_def=None, model_structure=None, **kwargs): - """Create an instance of `HParams` from keyword arguments. - - The keyword arguments specify name-values pairs for the hyperparameters. - The parameter types are inferred from the type of the values passed. - - The parameter names are added as attributes of `HParams` object, so they - can be accessed directly with the dot notation `hparams._name_`. - - Example: - - ```python - # Define 3 hyperparameters: 'learning_rate' is a float parameter, - # 'num_hidden_units' an integer parameter, and 'activation' a string - # parameter. - hparams = tf.contrib.training.HParams( - learning_rate=0.1, num_hidden_units=100, activation='relu') - - hparams.activation ==> 'relu' - ``` - - Note that a few names are reserved and cannot be used as hyperparameter - names. If you use one of the reserved name the constructor raises a - `ValueError`. - - Args: - hparam_def: Serialized hyperparameters, encoded as a hparam_pb2.HParamDef - protocol buffer. If provided, this object is initialized by - deserializing hparam_def. Otherwise **kwargs is used. - model_structure: An instance of ModelStructure, defining the feature - crosses to be used in the Trial. - **kwargs: Key-value pairs where the key is the hyperparameter name and - the value is the value for the parameter. - - Raises: - ValueError: If both `hparam_def` and initialization values are provided, - or if one of the arguments is invalid. - - """ - # Register the hyperparameters and their type in _hparam_types. - # This simplifies the implementation of parse(). - # _hparam_types maps the parameter name to a tuple (type, bool). - # The type value is the type of the parameter for scalar hyperparameters, - # or the type of the list elements for multidimensional hyperparameters. - # The bool value is True if the value is a list, False otherwise. - self._hparam_types = {} - self._model_structure = model_structure - if hparam_def: - self._init_from_proto(hparam_def) - if kwargs: - raise ValueError('hparam_def and initialization values are ' - 'mutually exclusive') - else: - for name, value in six.iteritems(kwargs): - self.add_hparam(name, value) - - def _init_from_proto(self, hparam_def): - """Creates a new HParams from `HParamDef` protocol buffer. - - Args: - hparam_def: `HParamDef` protocol buffer. - """ - assert isinstance(hparam_def, hparam_pb2.HParamDef) - for name, value in hparam_def.hparam.items(): - kind = value.WhichOneof('kind') - if kind.endswith('_value'): - # Single value. - if kind.startswith('int64'): - # Setting attribute value to be 'int' to ensure the type is compatible - # with both Python2 and Python3. - self.add_hparam(name, int(getattr(value, kind))) - elif kind.startswith('bytes'): - # Setting attribute value to be 'str' to ensure the type is compatible - # with both Python2 and Python3. UTF-8 encoding is assumed. - self.add_hparam(name, compat.as_str(getattr(value, kind))) - else: - self.add_hparam(name, getattr(value, kind)) - else: - # List of values. - if kind.startswith('int64'): - # Setting attribute value to be 'int' to ensure the type is compatible - # with both Python2 and Python3. - self.add_hparam(name, [int(v) for v in getattr(value, kind).value]) - elif kind.startswith('bytes'): - # Setting attribute value to be 'str' to ensure the type is compatible - # with both Python2 and Python3. UTF-8 encoding is assumed. - self.add_hparam( - name, [compat.as_str(v) for v in getattr(value, kind).value]) - else: - self.add_hparam(name, [v for v in getattr(value, kind).value]) - - def add_hparam(self, name, value): - """Adds {name, value} pair to hyperparameters. - - Args: - name: Name of the hyperparameter. - value: Value of the hyperparameter. Can be one of the following types: - int, float, string, int list, float list, or string list. - - Raises: - ValueError: if one of the arguments is invalid. - """ - # Keys in kwargs are unique, but 'name' could the name of a pre-existing - # attribute of this object. In that case we refuse to use it as a - # hyperparameter name. - if getattr(self, name, None) is not None: - raise ValueError('Hyperparameter name is reserved: %s' % name) - if isinstance(value, (list, tuple)): - if not value: - raise ValueError( - 'Multi-valued hyperparameters cannot be empty: %s' % name) - self._hparam_types[name] = (type(value[0]), True) - else: - self._hparam_types[name] = (type(value), False) - setattr(self, name, value) - - def set_hparam(self, name, value): - """Set the value of an existing hyperparameter. - - This function verifies that the type of the value matches the type of the - existing hyperparameter. - - Args: - name: Name of the hyperparameter. - value: New value of the hyperparameter. - - Raises: - KeyError: If the hyperparameter doesn't exist. - ValueError: If there is a type mismatch. - """ - param_type, is_list = self._hparam_types[name] - if isinstance(value, list): - if not is_list: - raise ValueError( - 'Must not pass a list for single-valued parameter: %s' % name) - setattr(self, name, [ - _cast_to_type_if_compatible(name, param_type, v) for v in value]) - else: - if is_list: - raise ValueError( - 'Must pass a list for multi-valued parameter: %s.' % name) - setattr(self, name, _cast_to_type_if_compatible(name, param_type, value)) - - def del_hparam(self, name): - """Removes the hyperparameter with key 'name'. - - Does nothing if it isn't present. - - Args: - name: Name of the hyperparameter. - """ - if hasattr(self, name): - delattr(self, name) - del self._hparam_types[name] - - def parse(self, values): - """Override existing hyperparameter values, parsing new values from a string. - - See parse_values for more detail on the allowed format for values. - - Args: - values: String. Comma separated list of `name=value` pairs where 'value' - must follow the syntax described above. - - Returns: - The `HParams` instance. - - Raises: - ValueError: If `values` cannot be parsed or a hyperparameter in `values` - doesn't exist. - """ - type_map = {} - for name, t in self._hparam_types.items(): - param_type, _ = t - type_map[name] = param_type - - values_map = parse_values(values, type_map) - return self.override_from_dict(values_map) - - def override_from_dict(self, values_dict): - """Override existing hyperparameter values, parsing new values from a dictionary. - - Args: - values_dict: Dictionary of name:value pairs. - - Returns: - The `HParams` instance. - - Raises: - KeyError: If a hyperparameter in `values_dict` doesn't exist. - ValueError: If `values_dict` cannot be parsed. - """ - for name, value in values_dict.items(): - self.set_hparam(name, value) - return self - - @deprecation.deprecated(None, 'Use `override_from_dict`.') - def set_from_map(self, values_map): - """DEPRECATED. Use override_from_dict.""" - return self.override_from_dict(values_dict=values_map) - - def set_model_structure(self, model_structure): - self._model_structure = model_structure - - def get_model_structure(self): - return self._model_structure - - def to_json(self, indent=None, separators=None, sort_keys=False): - """Serializes the hyperparameters into JSON. - - Args: - indent: If a non-negative integer, JSON array elements and object members - will be pretty-printed with that indent level. An indent level of 0, or - negative, will only insert newlines. `None` (the default) selects the - most compact representation. - separators: Optional `(item_separator, key_separator)` tuple. Default is - `(', ', ': ')`. - sort_keys: If `True`, the output dictionaries will be sorted by key. - - Returns: - A JSON string. - """ - return json.dumps( - self.values(), - indent=indent, - separators=separators, - sort_keys=sort_keys) - - def parse_json(self, values_json): - """Override existing hyperparameter values, parsing new values from a json object. - - Args: - values_json: String containing a json object of name:value pairs. - - Returns: - The `HParams` instance. - - Raises: - KeyError: If a hyperparameter in `values_json` doesn't exist. - ValueError: If `values_json` cannot be parsed. - """ - values_map = json.loads(values_json) - return self.override_from_dict(values_map) - - def values(self): - """Return the hyperparameter values as a Python dictionary. - - Returns: - A dictionary with hyperparameter names as keys. The values are the - hyperparameter values. - """ - return {n: getattr(self, n) for n in self._hparam_types.keys()} - - def get(self, key, default=None): - """Returns the value of `key` if it exists, else `default`.""" - if key in self._hparam_types: - # Ensure that default is compatible with the parameter type. - if default is not None: - param_type, is_param_list = self._hparam_types[key] - type_str = 'list<%s>' % param_type if is_param_list else str(param_type) - fail_msg = ("Hparam '%s' of type '%s' is incompatible with " - 'default=%s' % (key, type_str, default)) - - is_default_list = isinstance(default, list) - if is_param_list != is_default_list: - raise ValueError(fail_msg) - - try: - if is_default_list: - for value in default: - _cast_to_type_if_compatible(key, param_type, value) - else: - _cast_to_type_if_compatible(key, param_type, default) - except ValueError as e: - raise ValueError('%s. %s' % (fail_msg, e)) - - return getattr(self, key) - - return default - - def __contains__(self, key): - return key in self._hparam_types - - def __str__(self): - hpdict = self.values() - output_list = ['{}={}'.format(key, hpdict[key]) for key in hpdict] - return ','.join(output_list) - - def __repr__(self): - strval = str(sorted(self.values().items())) - return '%s(%s)' % (type(self).__name__, strval) - - @staticmethod - def _get_kind_name(param_type, is_list): - """Returns the field name given parameter type and is_list. - - Args: - param_type: Data type of the hparam. - is_list: Whether this is a list. - - Returns: - A string representation of the field name. - - Raises: - ValueError: If parameter type is not recognized. - """ - if issubclass(param_type, bool): - # This check must happen before issubclass(param_type, six.integer_types), - # since Python considers bool to be a subclass of int. - typename = 'bool' - elif issubclass(param_type, six.integer_types): - # Setting 'int' and 'long' types to be 'int64' to ensure the type is - # compatible with both Python2 and Python3. - typename = 'int64' - elif issubclass(param_type, (six.string_types, six.binary_type)): - # Setting 'string' and 'bytes' types to be 'bytes' to ensure the type is - # compatible with both Python2 and Python3. - typename = 'bytes' - elif issubclass(param_type, float): - typename = 'float' - else: - raise ValueError('Unsupported parameter type: %s' % str(param_type)) - - suffix = 'list' if is_list else 'value' - return '_'.join([typename, suffix]) - - def to_proto(self, export_scope=None): # pylint: disable=unused-argument - """Converts a `HParams` object to a `HParamDef` protocol buffer. - - Args: - export_scope: Optional `string`. Name scope to remove. - - Returns: - A `HParamDef` protocol buffer. - """ - hparam_proto = hparam_pb2.HParamDef() - for name in self._hparam_types: - # Parse the values. - param_type, is_list = self._hparam_types.get(name, (None, None)) - kind = HParams._get_kind_name(param_type, is_list) - - if is_list: - if kind.startswith('bytes'): - v_list = [compat.as_bytes(v) for v in getattr(self, name)] - else: - v_list = [v for v in getattr(self, name)] - getattr(hparam_proto.hparam[name], kind).value.extend(v_list) - else: - v = getattr(self, name) - if kind.startswith('bytes'): - v = compat.as_bytes(getattr(self, name)) - setattr(hparam_proto.hparam[name], kind, v) - - return hparam_proto - - @staticmethod - def from_proto(hparam_def, import_scope=None): # pylint: disable=unused-argument - return HParams(hparam_def=hparam_def) - - -ops.register_proto_function( - 'hparams', - proto_type=hparam_pb2.HParamDef, - to_proto=HParams.to_proto, - from_proto=HParams.from_proto) diff --git a/tensorflow/contrib/training/python/training/hparam_test.py b/tensorflow/contrib/training/python/training/hparam_test.py deleted file mode 100644 index 1877d2b38ac..00000000000 --- a/tensorflow/contrib/training/python/training/hparam_test.py +++ /dev/null @@ -1,615 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for hparam.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.training.python.training import hparam - -from tensorflow.python.platform import test - - -class HParamsTest(test.TestCase): - - def testEmpty(self): - hparams = hparam.HParams() - self.assertDictEqual({}, hparams.values()) - hparams.parse('') - self.assertDictEqual({}, hparams.values()) - with self.assertRaisesRegexp(ValueError, 'Unknown hyperparameter'): - hparams.parse('xyz=123') - - def testContains(self): - hparams = hparam.HParams(foo=1) - self.assertTrue('foo' in hparams) - self.assertFalse('bar' in hparams) - - def testSomeValues(self): - hparams = hparam.HParams(aaa=1, b=2.0, c_c='relu6', d='/a/b=c/d') - self.assertDictEqual({ - 'aaa': 1, - 'b': 2.0, - 'c_c': 'relu6', - 'd': '/a/b=c/d' - }, hparams.values()) - expected_str = ('HParams([(\'aaa\', 1), (\'b\', 2.0), (\'c_c\', \'relu6\'),' - ' (\'d\', \'/a/b=c/d\')])') - self.assertEqual(expected_str, repr(hparams)) - self.assertEqual(expected_str, repr(hparams)) - self.assertEqual(1, hparams.aaa) - self.assertEqual(2.0, hparams.b) - self.assertEqual('relu6', hparams.c_c) - self.assertEqual('/a/b=c/d', hparams.d) - hparams.parse('aaa=12') - self.assertDictEqual({ - 'aaa': 12, - 'b': 2.0, - 'c_c': 'relu6', - 'd': '/a/b=c/d' - }, hparams.values()) - self.assertEqual(12, hparams.aaa) - self.assertEqual(2.0, hparams.b) - self.assertEqual('relu6', hparams.c_c) - self.assertEqual('/a/b=c/d', hparams.d) - hparams.parse('c_c=relu4, b=-2.0e10') - self.assertDictEqual({ - 'aaa': 12, - 'b': -2.0e10, - 'c_c': 'relu4', - 'd': '/a/b=c/d' - }, hparams.values()) - self.assertEqual(12, hparams.aaa) - self.assertEqual(-2.0e10, hparams.b) - self.assertEqual('relu4', hparams.c_c) - self.assertEqual('/a/b=c/d', hparams.d) - hparams.parse('c_c=,b=0,') - self.assertDictEqual({ - 'aaa': 12, - 'b': 0, - 'c_c': '', - 'd': '/a/b=c/d' - }, hparams.values()) - self.assertEqual(12, hparams.aaa) - self.assertEqual(0.0, hparams.b) - self.assertEqual('', hparams.c_c) - self.assertEqual('/a/b=c/d', hparams.d) - hparams.parse('c_c=2.3",b=+2,') - self.assertEqual(2.0, hparams.b) - self.assertEqual('2.3"', hparams.c_c) - hparams.parse('d=/a/b/c/d,aaa=11,') - self.assertEqual(11, hparams.aaa) - self.assertEqual(2.0, hparams.b) - self.assertEqual('2.3"', hparams.c_c) - self.assertEqual('/a/b/c/d', hparams.d) - hparams.parse('b=1.5,d=/a=b/c/d,aaa=10,') - self.assertEqual(10, hparams.aaa) - self.assertEqual(1.5, hparams.b) - self.assertEqual('2.3"', hparams.c_c) - self.assertEqual('/a=b/c/d', hparams.d) - with self.assertRaisesRegexp(ValueError, 'Unknown hyperparameter'): - hparams.parse('x=123') - with self.assertRaisesRegexp(ValueError, 'Could not parse'): - hparams.parse('aaa=poipoi') - with self.assertRaisesRegexp(ValueError, 'Could not parse'): - hparams.parse('aaa=1.0') - with self.assertRaisesRegexp(ValueError, 'Could not parse'): - hparams.parse('b=12x') - with self.assertRaisesRegexp(ValueError, 'Could not parse'): - hparams.parse('b=relu') - with self.assertRaisesRegexp(ValueError, 'Must not pass a list'): - hparams.parse('aaa=[123]') - self.assertEqual(10, hparams.aaa) - self.assertEqual(1.5, hparams.b) - self.assertEqual('2.3"', hparams.c_c) - self.assertEqual('/a=b/c/d', hparams.d) - # Exports to proto. - hparam_def = hparams.to_proto() - # Imports from proto. - hparams2 = hparam.HParams(hparam_def=hparam_def) - # Verifies that all hparams are restored. - self.assertEqual(10, hparams2.aaa) - self.assertEqual(1.5, hparams2.b) - self.assertEqual('2.3"', hparams2.c_c) - self.assertEqual('/a=b/c/d', hparams2.d) - - def testWithPeriodInVariableName(self): - hparams = hparam.HParams() - hparams.add_hparam(name='a.b', value=0.0) - hparams.parse('a.b=1.0') - self.assertEqual(1.0, getattr(hparams, 'a.b')) - hparams.add_hparam(name='c.d', value=0.0) - with self.assertRaisesRegexp(ValueError, 'Could not parse'): - hparams.parse('c.d=abc') - hparams.add_hparam(name='e.f', value='') - hparams.parse('e.f=abc') - self.assertEqual('abc', getattr(hparams, 'e.f')) - hparams.add_hparam(name='d..', value=0.0) - hparams.parse('d..=10.0') - self.assertEqual(10.0, getattr(hparams, 'd..')) - - def testSetFromMap(self): - hparams = hparam.HParams(a=1, b=2.0, c='tanh') - hparams.override_from_dict({'a': -2, 'c': 'identity'}) - self.assertDictEqual({'a': -2, 'c': 'identity', 'b': 2.0}, hparams.values()) - - hparams = hparam.HParams(x=1, b=2.0, d=[0.5]) - hparams.override_from_dict({'d': [0.1, 0.2, 0.3]}) - self.assertDictEqual({ - 'd': [0.1, 0.2, 0.3], - 'x': 1, - 'b': 2.0 - }, hparams.values()) - - def testBoolParsing(self): - for value in 'true', 'false', 'True', 'False', '1', '0': - for initial in False, True: - hparams = hparam.HParams(use_gpu=initial) - hparams.parse('use_gpu=' + value) - self.assertEqual(hparams.use_gpu, value in ['True', 'true', '1']) - - # Exports to proto. - hparam_def = hparams.to_proto() - # Imports from proto. - hparams2 = hparam.HParams(hparam_def=hparam_def) - self.assertEqual(hparams.use_gpu, hparams2.use_gpu) - # Check that hparams2.use_gpu is a bool rather than an int. - # The assertEqual() call above won't catch this, since - # (0 == False) and (1 == True) in Python. - self.assertEqual(bool, type(hparams2.use_gpu)) - - def testBoolParsingFail(self): - hparams = hparam.HParams(use_gpu=True) - with self.assertRaisesRegexp(ValueError, r'Could not parse.*use_gpu'): - hparams.parse('use_gpu=yep') - - def testLists(self): - hparams = hparam.HParams(aaa=[1], b=[2.0, 3.0], c_c=['relu6']) - self.assertDictEqual({ - 'aaa': [1], - 'b': [2.0, 3.0], - 'c_c': ['relu6'] - }, hparams.values()) - self.assertEqual([1], hparams.aaa) - self.assertEqual([2.0, 3.0], hparams.b) - self.assertEqual(['relu6'], hparams.c_c) - hparams.parse('aaa=[12]') - self.assertEqual([12], hparams.aaa) - hparams.parse('aaa=[12,34,56]') - self.assertEqual([12, 34, 56], hparams.aaa) - hparams.parse('c_c=[relu4,relu12],b=[1.0]') - self.assertEqual(['relu4', 'relu12'], hparams.c_c) - self.assertEqual([1.0], hparams.b) - hparams.parse('c_c=[],aaa=[-34]') - self.assertEqual([-34], hparams.aaa) - self.assertEqual([], hparams.c_c) - hparams.parse('c_c=[_12,3\'4"],aaa=[+3]') - self.assertEqual([3], hparams.aaa) - self.assertEqual(['_12', '3\'4"'], hparams.c_c) - with self.assertRaisesRegexp(ValueError, 'Unknown hyperparameter'): - hparams.parse('x=[123]') - with self.assertRaisesRegexp(ValueError, 'Could not parse'): - hparams.parse('aaa=[poipoi]') - with self.assertRaisesRegexp(ValueError, 'Could not parse'): - hparams.parse('aaa=[1.0]') - with self.assertRaisesRegexp(ValueError, 'Could not parse'): - hparams.parse('b=[12x]') - with self.assertRaisesRegexp(ValueError, 'Could not parse'): - hparams.parse('b=[relu]') - with self.assertRaisesRegexp(ValueError, 'Must pass a list'): - hparams.parse('aaa=123') - # Exports to proto. - hparam_def = hparams.to_proto() - # Imports from proto. - hparams2 = hparam.HParams(hparam_def=hparam_def) - # Verifies that all hparams are restored. - self.assertEqual([3], hparams2.aaa) - self.assertEqual([1.0], hparams2.b) - self.assertEqual(['_12', '3\'4"'], hparams2.c_c) - - def testStr(self): - hparam1 = hparam.HParams(a=1, b=[2.0, 3.0], c='relu6') - hparam1_str = str(hparam1) - # Create the signature - hparam2 = hparam.HParams() - hparam2.add_hparam('a', 4) - hparam2.add_hparam('b', [5.0, 6.0]) - hparam2.add_hparam('c', 'relu10') - # Load from string - hparam2.parse(hparam1_str) - # Verifies all hparams are restored - self.assertEqual(hparam2.a, hparam1.a) - self.assertEqual(hparam2.b, hparam1.b) - self.assertEqual(hparam2.c, hparam1.c) - - def testParseValuesWithIndexAssigment1(self): - """Assignment to an index position.""" - parse_dict = hparam.parse_values('arr[1]=10', {'arr': int}) - self.assertEqual(len(parse_dict), 1) - self.assertTrue(isinstance(parse_dict['arr'], dict)) - self.assertDictEqual(parse_dict['arr'], {1: 10}) - - def testParseValuesWithIndexAssigment1_IgnoreUnknown(self): - """Assignment to an index position.""" - parse_dict = hparam.parse_values( - 'arr[1]=10,b=5', {'arr': int}, ignore_unknown=True) - self.assertEqual(len(parse_dict), 1) - self.assertTrue(isinstance(parse_dict['arr'], dict)) - self.assertDictEqual(parse_dict['arr'], {1: 10}) - - def testParseValuesWithIndexAssigment2(self): - """Assignment to multiple index positions.""" - parse_dict = hparam.parse_values('arr[0]=10,arr[5]=20', {'arr': int}) - self.assertEqual(len(parse_dict), 1) - self.assertTrue(isinstance(parse_dict['arr'], dict)) - self.assertDictEqual(parse_dict['arr'], {0: 10, 5: 20}) - - def testParseValuesWithIndexAssigment2_IgnoreUnknown(self): - """Assignment to multiple index positions.""" - parse_dict = hparam.parse_values( - 'arr[0]=10,arr[5]=20,foo=bar', {'arr': int}, ignore_unknown=True) - self.assertEqual(len(parse_dict), 1) - self.assertTrue(isinstance(parse_dict['arr'], dict)) - self.assertDictEqual(parse_dict['arr'], {0: 10, 5: 20}) - - def testParseValuesWithIndexAssigment3(self): - """Assignment to index positions in multiple names.""" - parse_dict = hparam.parse_values('arr[0]=10,arr[1]=20,L[5]=100,L[10]=200', { - 'arr': int, - 'L': int - }) - self.assertEqual(len(parse_dict), 2) - self.assertTrue(isinstance(parse_dict['arr'], dict)) - self.assertDictEqual(parse_dict['arr'], {0: 10, 1: 20}) - self.assertTrue(isinstance(parse_dict['L'], dict)) - self.assertDictEqual(parse_dict['L'], {5: 100, 10: 200}) - - def testParseValuesWithIndexAssigment3_IgnoreUnknown(self): - """Assignment to index positions in multiple names.""" - parse_dict = hparam.parse_values( - 'arr[0]=10,C=5,arr[1]=20,B[0]=kkk,L[5]=100,L[10]=200', { - 'arr': int, - 'L': int - }, - ignore_unknown=True) - self.assertEqual(len(parse_dict), 2) - self.assertTrue(isinstance(parse_dict['arr'], dict)) - self.assertDictEqual(parse_dict['arr'], {0: 10, 1: 20}) - self.assertTrue(isinstance(parse_dict['L'], dict)) - self.assertDictEqual(parse_dict['L'], {5: 100, 10: 200}) - - def testParseValuesWithIndexAssigment4(self): - """Assignment of index positions and scalars.""" - parse_dict = hparam.parse_values('x=10,arr[1]=20,y=30', { - 'x': int, - 'y': int, - 'arr': int - }) - self.assertEqual(len(parse_dict), 3) - self.assertTrue(isinstance(parse_dict['arr'], dict)) - self.assertDictEqual(parse_dict['arr'], {1: 20}) - self.assertEqual(parse_dict['x'], 10) - self.assertEqual(parse_dict['y'], 30) - - def testParseValuesWithIndexAssigment4_IgnoreUnknown(self): - """Assignment of index positions and scalars.""" - parse_dict = hparam.parse_values( - 'x=10,foo[0]=bar,arr[1]=20,zzz=78,y=30', { - 'x': int, - 'y': int, - 'arr': int - }, - ignore_unknown=True) - self.assertEqual(len(parse_dict), 3) - self.assertTrue(isinstance(parse_dict['arr'], dict)) - self.assertDictEqual(parse_dict['arr'], {1: 20}) - self.assertEqual(parse_dict['x'], 10) - self.assertEqual(parse_dict['y'], 30) - - def testParseValuesWithIndexAssigment5(self): - """Different variable types.""" - parse_dict = hparam.parse_values('a[0]=5,b[1]=true,c[2]=abc,d[3]=3.14', { - 'a': int, - 'b': bool, - 'c': str, - 'd': float - }) - self.assertEqual(set(parse_dict.keys()), {'a', 'b', 'c', 'd'}) - self.assertTrue(isinstance(parse_dict['a'], dict)) - self.assertDictEqual(parse_dict['a'], {0: 5}) - self.assertTrue(isinstance(parse_dict['b'], dict)) - self.assertDictEqual(parse_dict['b'], {1: True}) - self.assertTrue(isinstance(parse_dict['c'], dict)) - self.assertDictEqual(parse_dict['c'], {2: 'abc'}) - self.assertTrue(isinstance(parse_dict['d'], dict)) - self.assertDictEqual(parse_dict['d'], {3: 3.14}) - - def testParseValuesWithIndexAssigment5_IgnoreUnknown(self): - """Different variable types.""" - parse_dict = hparam.parse_values( - 'a[0]=5,cc=4,b[1]=true,c[2]=abc,mm=2,d[3]=3.14', { - 'a': int, - 'b': bool, - 'c': str, - 'd': float - }, - ignore_unknown=True) - self.assertEqual(set(parse_dict.keys()), {'a', 'b', 'c', 'd'}) - self.assertTrue(isinstance(parse_dict['a'], dict)) - self.assertDictEqual(parse_dict['a'], {0: 5}) - self.assertTrue(isinstance(parse_dict['b'], dict)) - self.assertDictEqual(parse_dict['b'], {1: True}) - self.assertTrue(isinstance(parse_dict['c'], dict)) - self.assertDictEqual(parse_dict['c'], {2: 'abc'}) - self.assertTrue(isinstance(parse_dict['d'], dict)) - self.assertDictEqual(parse_dict['d'], {3: 3.14}) - - def testParseValuesWithBadIndexAssigment1(self): - """Reject assignment of list to variable type.""" - with self.assertRaisesRegexp(ValueError, - r'Assignment of a list to a list index.'): - hparam.parse_values('arr[1]=[1,2,3]', {'arr': int}) - - def testParseValuesWithBadIndexAssigment1_IgnoreUnknown(self): - """Reject assignment of list to variable type.""" - with self.assertRaisesRegexp(ValueError, - r'Assignment of a list to a list index.'): - hparam.parse_values( - 'arr[1]=[1,2,3],c=8', {'arr': int}, ignore_unknown=True) - - def testParseValuesWithBadIndexAssigment2(self): - """Reject if type missing.""" - with self.assertRaisesRegexp(ValueError, - r'Unknown hyperparameter type for arr'): - hparam.parse_values('arr[1]=5', {}) - - def testParseValuesWithBadIndexAssigment2_IgnoreUnknown(self): - """Ignore missing type.""" - hparam.parse_values('arr[1]=5', {}, ignore_unknown=True) - - def testParseValuesWithBadIndexAssigment3(self): - """Reject type of the form name[index].""" - with self.assertRaisesRegexp(ValueError, - 'Unknown hyperparameter type for arr'): - hparam.parse_values('arr[1]=1', {'arr[1]': int}) - - def testParseValuesWithBadIndexAssigment3_IgnoreUnknown(self): - """Ignore type of the form name[index].""" - hparam.parse_values('arr[1]=1', {'arr[1]': int}, ignore_unknown=True) - - def testWithReusedVariables(self): - with self.assertRaisesRegexp(ValueError, - 'Multiple assignments to variable \'x\''): - hparam.parse_values('x=1,x=1', {'x': int}) - - with self.assertRaisesRegexp(ValueError, - 'Multiple assignments to variable \'arr\''): - hparam.parse_values('arr=[100,200],arr[0]=10', {'arr': int}) - - with self.assertRaisesRegexp( - ValueError, r'Multiple assignments to variable \'arr\[0\]\''): - hparam.parse_values('arr[0]=10,arr[0]=20', {'arr': int}) - - with self.assertRaisesRegexp(ValueError, - 'Multiple assignments to variable \'arr\''): - hparam.parse_values('arr[0]=10,arr=[100]', {'arr': int}) - - def testJson(self): - hparams = hparam.HParams(aaa=1, b=2.0, c_c='relu6', d=True) - self.assertDictEqual({ - 'aaa': 1, - 'b': 2.0, - 'c_c': 'relu6', - 'd': True - }, hparams.values()) - self.assertEqual(1, hparams.aaa) - self.assertEqual(2.0, hparams.b) - self.assertEqual('relu6', hparams.c_c) - hparams.parse_json('{"aaa": 12, "b": 3.0, "c_c": "relu4", "d": false}') - self.assertDictEqual({ - 'aaa': 12, - 'b': 3.0, - 'c_c': 'relu4', - 'd': False - }, hparams.values()) - self.assertEqual(12, hparams.aaa) - self.assertEqual(3.0, hparams.b) - self.assertEqual('relu4', hparams.c_c) - - json_str = hparams.to_json() - hparams2 = hparam.HParams(aaa=10, b=20.0, c_c='hello', d=False) - hparams2.parse_json(json_str) - self.assertEqual(12, hparams2.aaa) - self.assertEqual(3.0, hparams2.b) - self.assertEqual('relu4', hparams2.c_c) - self.assertEqual(False, hparams2.d) - - hparams3 = hparam.HParams(aaa=123) - self.assertEqual('{"aaa": 123}', hparams3.to_json()) - self.assertEqual('{\n "aaa": 123\n}', hparams3.to_json(indent=2)) - self.assertEqual('{"aaa"=123}', hparams3.to_json(separators=(';', '='))) - - hparams4 = hparam.HParams(aaa=123, b='hello', c_c=False) - self.assertEqual('{"aaa": 123, "b": "hello", "c_c": false}', - hparams4.to_json(sort_keys=True)) - - def testSetHParam(self): - hparams = hparam.HParams(aaa=1, b=2.0, c_c='relu6', d=True) - self.assertDictEqual({ - 'aaa': 1, - 'b': 2.0, - 'c_c': 'relu6', - 'd': True - }, hparams.values()) - self.assertEqual(1, hparams.aaa) - self.assertEqual(2.0, hparams.b) - self.assertEqual('relu6', hparams.c_c) - - hparams.set_hparam('aaa', 12) - hparams.set_hparam('b', 3.0) - hparams.set_hparam('c_c', 'relu4') - hparams.set_hparam('d', False) - self.assertDictEqual({ - 'aaa': 12, - 'b': 3.0, - 'c_c': 'relu4', - 'd': False - }, hparams.values()) - self.assertEqual(12, hparams.aaa) - self.assertEqual(3.0, hparams.b) - self.assertEqual('relu4', hparams.c_c) - - def testSetHParamListNonListMismatch(self): - hparams = hparam.HParams(a=1, b=[2.0, 3.0]) - with self.assertRaisesRegexp(ValueError, r'Must not pass a list'): - hparams.set_hparam('a', [1.0]) - with self.assertRaisesRegexp(ValueError, r'Must pass a list'): - hparams.set_hparam('b', 1.0) - - def testSetHParamTypeMismatch(self): - hparams = hparam.HParams( - int_=1, str_='str', bool_=True, float_=1.1, list_int=[1, 2], none=None) - - with self.assertRaises(ValueError): - hparams.set_hparam('str_', 2.2) - - with self.assertRaises(ValueError): - hparams.set_hparam('int_', False) - - with self.assertRaises(ValueError): - hparams.set_hparam('bool_', 1) - - # Unfortunately there is no automagic conversion of bool-like strings to - # bool. - with self.assertRaises(ValueError): - hparams.set_hparam('bool_', 'true') - - with self.assertRaises(ValueError): - hparams.set_hparam('bool_', 'True') - - with self.assertRaises(ValueError): - hparams.set_hparam('bool_', 'false') - - with self.assertRaises(ValueError): - hparams.set_hparam('bool_', 'False') - - with self.assertRaises(ValueError): - hparams.set_hparam('bool_', '0') - - with self.assertRaises(ValueError): - hparams.set_hparam('bool_', '1') - - with self.assertRaises(ValueError): - hparams.set_hparam('int_', 2.2) - - with self.assertRaises(ValueError): - hparams.set_hparam('list_int', [2, 3.3]) - - with self.assertRaises(ValueError): - hparams.set_hparam('int_', '2') - - # Casting int to float is OK - hparams.set_hparam('float_', 1) - - # Getting stuck with NoneType :( - hparams.set_hparam('none', '1') - self.assertEqual('1', hparams.none) - - def testSetHParamExactTypeMatch(self): - - class DummyContext(object): - - def __init__(self, a, b=0): - self.a = a - self.b = b - - hparams = hparam.HParams(x=DummyContext(a=100, b=100)) - # Verify x is assigned directly, without casting. - hparams.set_hparam('x', DummyContext(a=100, b=100)) - self.assertEqual(hparams.x.a, 100) - self.assertEqual(hparams.x.b, 100) - - def testNonProtoFails(self): - with self.assertRaisesRegexp(AssertionError, ''): - hparam.HParams(hparam_def=1) - with self.assertRaisesRegexp(AssertionError, ''): - hparam.HParams(hparam_def=1.0) - with self.assertRaisesRegexp(AssertionError, ''): - hparam.HParams(hparam_def='hello') - with self.assertRaisesRegexp(AssertionError, ''): - hparam.HParams(hparam_def=[1, 2, 3]) - - def testGet(self): - hparams = hparam.HParams(aaa=1, b=2.0, c_c='relu6', d=True, e=[5.0, 6.0]) - - # Existing parameters with default=None. - self.assertEqual(1, hparams.get('aaa')) - self.assertEqual(2.0, hparams.get('b')) - self.assertEqual('relu6', hparams.get('c_c')) - self.assertEqual(True, hparams.get('d')) - self.assertEqual([5.0, 6.0], hparams.get('e', None)) - - # Existing parameters with compatible defaults. - self.assertEqual(1, hparams.get('aaa', 2)) - self.assertEqual(2.0, hparams.get('b', 3.0)) - self.assertEqual(2.0, hparams.get('b', 3)) - self.assertEqual('relu6', hparams.get('c_c', 'default')) - self.assertEqual(True, hparams.get('d', True)) - self.assertEqual([5.0, 6.0], hparams.get('e', [1.0, 2.0, 3.0])) - self.assertEqual([5.0, 6.0], hparams.get('e', [1, 2, 3])) - - # Existing parameters with incompatible defaults. - with self.assertRaises(ValueError): - hparams.get('aaa', 2.0) - - with self.assertRaises(ValueError): - hparams.get('b', False) - - with self.assertRaises(ValueError): - hparams.get('c_c', [1, 2, 3]) - - with self.assertRaises(ValueError): - hparams.get('d', 'relu') - - with self.assertRaises(ValueError): - hparams.get('e', 123.0) - - with self.assertRaises(ValueError): - hparams.get('e', ['a', 'b', 'c']) - - # Nonexistent parameters. - self.assertEqual(None, hparams.get('unknown')) - self.assertEqual(123, hparams.get('unknown', 123)) - self.assertEqual([1, 2, 3], hparams.get('unknown', [1, 2, 3])) - - def testDel(self): - hparams = hparam.HParams(aaa=1, b=2.0) - - with self.assertRaises(ValueError): - hparams.set_hparam('aaa', 'will fail') - - with self.assertRaises(ValueError): - hparams.add_hparam('aaa', 'will fail') - - hparams.del_hparam('aaa') - hparams.add_hparam('aaa', 'will work') - self.assertEqual('will work', hparams.get('aaa')) - - hparams.set_hparam('aaa', 'still works') - self.assertEqual('still works', hparams.get('aaa')) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/training/python/training/resample.py b/tensorflow/contrib/training/python/training/resample.py deleted file mode 100644 index 7b8332b1d67..00000000000 --- a/tensorflow/contrib/training/python/training/resample.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Resampling methods for batches of tensors.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import tensor_array_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.training import moving_averages - - -def _repeat_range(counts, name=None): - """Repeat integers given by range(len(counts)) each the given number of times. - - Example behavior: - [0, 1, 2, 3] -> [1, 2, 2, 3, 3, 3] - - Args: - counts: 1D tensor with dtype=int32. - name: optional name for operation. - - Returns: - 1D tensor with dtype=int32 and dynamic length giving the repeated integers. - """ - with ops.name_scope(name, 'repeat_range', [counts]) as scope: - counts = ops.convert_to_tensor(counts, name='counts') - - def cond(unused_output, i): - return i < size - - def body(output, i): - value = array_ops.fill(counts[i:i+1], i) - return (output.write(i, value), i + 1) - - size = array_ops.shape(counts)[0] - init_output_array = tensor_array_ops.TensorArray( - dtype=dtypes.int32, size=size, infer_shape=False) - output_array, num_writes = control_flow_ops.while_loop( - cond, body, loop_vars=[init_output_array, 0]) - - return control_flow_ops.cond( - num_writes > 0, - output_array.concat, - lambda: array_ops.zeros(shape=[0], dtype=dtypes.int32), - name=scope) - - -def resample_at_rate(inputs, rates, scope=None, seed=None, back_prop=False): - """Given `inputs` tensors, stochastically resamples each at a given rate. - - For example, if the inputs are `[[a1, a2], [b1, b2]]` and the rates - tensor contains `[3, 1]`, then the return value may look like `[[a1, - a2, a1, a1], [b1, b2, b1, b1]]`. However, many other outputs are - possible, since this is stochastic -- averaged over many repeated - calls, each set of inputs should appear in the output `rate` times - the number of invocations. - - Args: - inputs: A list of tensors, each of which has a shape of `[batch_size, ...]` - rates: A tensor of shape `[batch_size]` containing the resampling rates - for each input. - scope: Scope for the op. - seed: Random seed to use. - back_prop: Whether to allow back-propagation through this op. - - Returns: - Selections from the input tensors. - """ - with ops.name_scope(scope, default_name='resample_at_rate', - values=list(inputs) + [rates]): - rates = ops.convert_to_tensor(rates, name='rates') - sample_counts = math_ops.cast( - random_ops.random_poisson(rates, (), rates.dtype, seed=seed), - dtypes.int32) - sample_indices = _repeat_range(sample_counts) - if not back_prop: - sample_indices = array_ops.stop_gradient(sample_indices) - return [array_ops.gather(x, sample_indices) for x in inputs] - - -def weighted_resample(inputs, weights, overall_rate, scope=None, - mean_decay=0.999, seed=None): - """Performs an approximate weighted resampling of `inputs`. - - This method chooses elements from `inputs` where each item's rate of - selection is proportional to its value in `weights`, and the average - rate of selection across all inputs (and many invocations!) is - `overall_rate`. - - Args: - inputs: A list of tensors whose first dimension is `batch_size`. - weights: A `[batch_size]`-shaped tensor with each batch member's weight. - overall_rate: Desired overall rate of resampling. - scope: Scope to use for the op. - mean_decay: How quickly to decay the running estimate of the mean weight. - seed: Random seed. - - Returns: - A list of tensors exactly like `inputs`, but with an unknown (and - possibly zero) first dimension. - A tensor containing the effective resampling rate used for each output. - """ - # Algorithm: Just compute rates as weights/mean_weight * - # overall_rate. This way the average weight corresponds to the - # overall rate, and a weight twice the average has twice the rate, - # etc. - with ops.name_scope(scope, 'weighted_resample', inputs) as opscope: - # First: Maintain a running estimated mean weight, with zero debiasing - # enabled (by default) to avoid throwing the average off. - - with variable_scope.variable_scope(scope, 'estimate_mean', inputs): - estimated_mean = variable_scope.get_local_variable( - 'estimated_mean', - initializer=math_ops.cast(0, weights.dtype), - dtype=weights.dtype) - - batch_mean = math_ops.reduce_mean(weights) - mean = moving_averages.assign_moving_average( - estimated_mean, batch_mean, mean_decay) - - # Then, normalize the weights into rates using the mean weight and - # overall target rate: - rates = weights * overall_rate / mean - - results = resample_at_rate([rates] + inputs, rates, - scope=opscope, seed=seed, back_prop=False) - - return (results[1:], results[0]) diff --git a/tensorflow/contrib/training/python/training/resample_test.py b/tensorflow/contrib/training/python/training/resample_test.py deleted file mode 100644 index 8665a24883b..00000000000 --- a/tensorflow/contrib/training/python/training/resample_test.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import math - -import numpy -from tensorflow.contrib.training.python.training import resample -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class ResampleTest(test.TestCase): - """Tests that resampling runs and outputs are close to expected values.""" - - def testRepeatRange(self): - cases = [ - ([], []), - ([0], []), - ([1], [0]), - ([1, 0], [0]), - ([0, 1], [1]), - ([3], [0, 0, 0]), - ([0, 1, 2, 3], [1, 2, 2, 3, 3, 3]), - ] - with self.cached_session() as sess: - for inputs, expected in cases: - array_inputs = numpy.array(inputs, dtype=numpy.int32) - actual = sess.run(resample._repeat_range(array_inputs)) - self.assertAllEqual(actual, expected) - - def testRoundtrip(self, rate=0.25, count=5, n=500): - """Tests `resample(x, weights)` and resample(resample(x, rate), 1/rate)`.""" - - foo = self.get_values(count) - bar = self.get_values(count) - weights = self.get_weights(count) - - resampled_in, rates = resample.weighted_resample( - [foo, bar], constant_op.constant(weights), rate, seed=123) - - resampled_back_out = resample.resample_at_rate( - resampled_in, 1.0 / rates, seed=456) - - init = control_flow_ops.group(variables.local_variables_initializer(), - variables.global_variables_initializer()) - with self.cached_session() as s: - s.run(init) # initialize - - # outputs - counts_resampled = collections.Counter() - counts_reresampled = collections.Counter() - for _ in range(n): - resampled_vs, reresampled_vs = s.run([resampled_in, resampled_back_out]) - - self.assertAllEqual(resampled_vs[0], resampled_vs[1]) - self.assertAllEqual(reresampled_vs[0], reresampled_vs[1]) - - for v in resampled_vs[0]: - counts_resampled[v] += 1 - for v in reresampled_vs[0]: - counts_reresampled[v] += 1 - - # assert that resampling worked as expected - self.assert_expected(weights, rate, counts_resampled, n) - - # and that re-resampling gives the approx identity. - self.assert_expected( - [1.0 for _ in weights], - 1.0, - counts_reresampled, - n, - abs_delta=0.1 * n * count) - - def testCorrectRates(self, rate=0.25, count=10, n=500, rtol=0.1): - """Tests that the rates returned by weighted_resample are correct.""" - - # The approach here is to verify that: - # - sum(1/rate) approximates the size of the original collection - # - sum(1/rate * value) approximates the sum of the original inputs, - # - sum(1/rate * value)/sum(1/rate) approximates the mean. - vals = self.get_values(count) - weights = self.get_weights(count) - - resampled, rates = resample.weighted_resample([vals], - constant_op.constant(weights), - rate) - - invrates = 1.0 / rates - - init = control_flow_ops.group(variables.local_variables_initializer(), - variables.global_variables_initializer()) - expected_sum_op = math_ops.reduce_sum(vals) - with self.cached_session() as s: - s.run(init) - expected_sum = n * s.run(expected_sum_op) - - weight_sum = 0.0 - weighted_value_sum = 0.0 - for _ in range(n): - val, inv_rate = s.run([resampled[0], invrates]) - weight_sum += sum(inv_rate) - weighted_value_sum += sum(val * inv_rate) - - # sum(inv_rate) ~= N*count: - expected_count = count * n - self.assertAlmostEqual( - expected_count, weight_sum, delta=(rtol * expected_count)) - - # sum(vals) * n ~= weighted_sum(resampled, 1.0/weights) - self.assertAlmostEqual( - expected_sum, weighted_value_sum, delta=(rtol * expected_sum)) - - # Mean ~= weighted mean: - expected_mean = expected_sum / float(n * count) - self.assertAlmostEqual( - expected_mean, - weighted_value_sum / weight_sum, - delta=(rtol * expected_mean)) - - def testZeroRateUnknownShapes(self, count=10): - """Tests that resampling runs with completely runtime shapes.""" - # Use placeholcers without shape set: - vals = array_ops.placeholder(dtype=dtypes.int32) - rates = array_ops.placeholder(dtype=dtypes.float32) - - resampled = resample.resample_at_rate([vals], rates) - - with self.cached_session() as s: - rs, = s.run(resampled, { - vals: list(range(count)), - rates: numpy.zeros( - shape=[count], dtype=numpy.float32) - }) - self.assertEqual(rs.shape, (0,)) - - def testDtypes(self, count=10): - """Test that we can define the ops with float64 weights.""" - - vals = self.get_values(count) - weights = math_ops.cast(self.get_weights(count), dtypes.float64) - - # should not error: - resample.resample_at_rate([vals], weights) - resample.weighted_resample( - [vals], weights, overall_rate=math_ops.cast(1.0, dtypes.float64)) - - def get_weights(self, n, mean=10.0, stddev=5): - """Returns random positive weight values.""" - assert mean > 0, 'Weights have to be positive.' - results = [] - while len(results) < n: - v = numpy.random.normal(mean, stddev) - if v > 0: - results.append(v) - return results - - def get_values(self, n): - return constant_op.constant(list(range(n))) - - def assert_expected(self, - weights, - overall_rate, - counts, - n, - tol=2.0, - abs_delta=0): - # Overall, we expect sum(counts) there to be `overall_rate` * n * - # len(weights)... with a stddev on that expectation equivalent to - # performing (n * len(weights)) trials each with probability of - # overall_rate. - expected_overall_count = len(weights) * n * overall_rate - actual_overall_count = sum(counts.values()) - - stddev = math.sqrt(len(weights) * n * overall_rate * (1 - overall_rate)) - - self.assertAlmostEqual( - expected_overall_count, - actual_overall_count, - delta=(stddev * tol + abs_delta)) - - # And we can form a similar expectation for each item -- it should - # appear in the results a number of time proportional to its - # weight, which is similar to performing `expected_overall_count` - # trials each with a probability of weight/weight_sum. - weight_sum = sum(weights) - fractions = [w / weight_sum for w in weights] - expected_counts = [expected_overall_count * f for f in fractions] - - stddevs = [ - math.sqrt(expected_overall_count * f * (1 - f)) for f in fractions - ] - - for i in range(len(expected_counts)): - expected_count = expected_counts[i] - actual_count = counts[i] - self.assertAlmostEqual( - expected_count, actual_count, delta=(stddevs[i] * tol + abs_delta)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/training/python/training/sampling_ops.py b/tensorflow/contrib/training/python/training/sampling_ops.py deleted file mode 100644 index 257cc4fce21..00000000000 --- a/tensorflow/contrib/training/python/training/sampling_ops.py +++ /dev/null @@ -1,425 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Sampling functions.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.training import input as input_ops - -__all__ = [ - 'rejection_sample', - 'stratified_sample', -] - - -def rejection_sample(tensors, - accept_prob_fn, - batch_size, - queue_threads=1, - enqueue_many=False, - prebatch_capacity=16, - prebatch_threads=1, - runtime_checks=False, - name=None): - """Stochastically creates batches by rejection sampling. - - Each list of non-batched tensors is evaluated by `accept_prob_fn`, to produce - a scalar tensor between 0 and 1. This tensor corresponds to the probability of - being accepted. When `batch_size` tensor groups have been accepted, the batch - queue will return a mini-batch. - - Args: - tensors: List of tensors for data. All tensors are either one item or a - batch, according to enqueue_many. - accept_prob_fn: A python lambda that takes a non-batch tensor from each - item in `tensors`, and produces a scalar tensor. - batch_size: Size of batch to be returned. - queue_threads: The number of threads for the queue that will hold the final - batch. - enqueue_many: Bool. If true, interpret input tensors as having a batch - dimension. - prebatch_capacity: Capacity for the large queue that is used to convert - batched tensors to single examples. - prebatch_threads: Number of threads for the large queue that is used to - convert batched tensors to single examples. - runtime_checks: Bool. If true, insert runtime checks on the output of - `accept_prob_fn`. Using `True` might have a performance impact. - name: Optional prefix for ops created by this function. - Raises: - ValueError: enqueue_many is True and labels doesn't have a batch - dimension, or if enqueue_many is False and labels isn't a scalar. - ValueError: enqueue_many is True, and batch dimension on data and labels - don't match. - ValueError: if a zero initial probability class has a nonzero target - probability. - Returns: - A list of tensors of the same length as `tensors`, with batch dimension - `batch_size`. - - Example: - # Get tensor for a single data and label example. - data, label = data_provider.Get(['data', 'label']) - - # Get stratified batch according to data tensor. - accept_prob_fn = lambda x: (tf.tanh(x[0]) + 1) / 2 - data_batch = tf.contrib.training.rejection_sample( - [data, label], accept_prob_fn, 16) - - # Run batch through network. - ... - """ - with variable_scope.variable_scope(name, 'rejection_sample', tensors): - tensor_list = ops.convert_n_to_tensor_or_indexed_slices(tensors) - # Reduce the case of a batched example to that of a batch of a single - # example by taking a batch of size one. - if enqueue_many: - # Validate that batch dimension of the input is consistent. - tensor_list = _verify_data_inputs(tensor_list) - - # Make a single queue to hold input examples. Reshape output so examples - # don't have singleton batch dimension. - batched = input_ops.batch( - tensor_list, - batch_size=1, - num_threads=prebatch_threads, - capacity=prebatch_capacity, - enqueue_many=True) - tensor_list = [array_ops.squeeze(x, [0]) for x in batched] - - # Set up a queue containing batches that have the distribution. - cur_prob = accept_prob_fn(tensor_list) - if runtime_checks: - cur_prob = array_ops.identity( - control_flow_ops.with_dependencies([ - check_ops.assert_less_equal(0.0, cur_prob), - check_ops.assert_less_equal(cur_prob, 1.0) - ], cur_prob), - name='prob_with_checks') - minibatch = input_ops.maybe_batch( - tensor_list, - keep_input=random_ops.random_uniform([]) < cur_prob, - batch_size=batch_size, - num_threads=queue_threads) - - # Queues return a single tensor if the list of enqueued tensors is one. Since - # we want the type to always be the same, always return a list. - if isinstance(minibatch, ops.Tensor): - minibatch = [minibatch] - - return minibatch - - -def stratified_sample(tensors, - labels, - target_probs, - batch_size, - init_probs=None, - enqueue_many=False, - queue_capacity=16, - threads_per_queue=1, - name=None): - """Stochastically creates batches based on per-class probabilities. - - This method discards examples. Internally, it creates one queue to amortize - the cost of disk reads, and one queue to hold the properly-proportioned - batch. - - Args: - tensors: List of tensors for data. All tensors are either one item or a - batch, according to enqueue_many. - labels: Tensor for label of data. Label is a single integer or a batch, - depending on `enqueue_many`. It is not a one-hot vector. - target_probs: Target class proportions in batch. An object whose type has a - registered Tensor conversion function. - batch_size: Size of batch to be returned. - init_probs: Class proportions in the data. An object whose type has a - registered Tensor conversion function, or `None` for estimating the - initial distribution. - enqueue_many: Bool. If true, interpret input tensors as having a batch - dimension. - queue_capacity: Capacity of the large queue that holds input examples. - threads_per_queue: Number of threads for the large queue that holds input - examples and for the final queue with the proper class proportions. - name: Optional prefix for ops created by this function. - Raises: - ValueError: If `tensors` isn't iterable. - ValueError: `enqueue_many` is True and labels doesn't have a batch - dimension, or if `enqueue_many` is False and labels isn't a scalar. - ValueError: `enqueue_many` is True, and batch dimension on data and labels - don't match. - ValueError: if probs don't sum to one. - ValueError: if a zero initial probability class has a nonzero target - probability. - TFAssertion: if labels aren't integers in [0, num classes). - Returns: - (data_batch, label_batch), where data_batch is a list of tensors of the same - length as `tensors` - - Example: - # Get tensor for a single data and label example. - data, label = data_provider.Get(['data', 'label']) - - # Get stratified batch according to per-class probabilities. - target_probs = [...distribution you want...] - [data_batch], labels = tf.contrib.training.stratified_sample( - [data], label, target_probs) - - # Run batch through network. - ... - """ - with ops.name_scope(name, 'stratified_sample', list(tensors) + [labels]): - tensor_list = ops.convert_n_to_tensor_or_indexed_slices(tensors) - labels = ops.convert_to_tensor(labels) - target_probs = ops.convert_to_tensor(target_probs, dtype=dtypes.float32) - # Reduce the case of a single example to that of a batch of size 1. - if not enqueue_many: - tensor_list = [array_ops.expand_dims(tensor, 0) for tensor in tensor_list] - labels = array_ops.expand_dims(labels, 0) - - # If `init_probs` is `None`, set up online estimation of data distribution. - if init_probs is None: - # We use `target_probs` to get the number of classes, so its shape must be - # fully defined at graph construction time. - target_probs.get_shape().assert_is_fully_defined() - init_probs = _estimate_data_distribution( - labels, target_probs.get_shape().num_elements()) - else: - init_probs = ops.convert_to_tensor(init_probs, dtype=dtypes.float32) - - # Validate that input is consistent. - tensor_list, labels, [init_probs, target_probs] = _verify_input( - tensor_list, labels, [init_probs, target_probs]) - - # Check that all zero initial probabilities also have zero target - # probabilities. - assert_op = control_flow_ops.Assert( - math_ops.reduce_all( - math_ops.logical_or( - math_ops.not_equal(init_probs, 0), - math_ops.equal(target_probs, 0))), - ['All classes with zero initial probability must also have zero target ' - 'probability: ', init_probs, target_probs - ]) - init_probs = control_flow_ops.with_dependencies([assert_op], init_probs) - - # Calculate acceptance sampling probabilities. - accept_probs = _calculate_acceptance_probabilities(init_probs, target_probs) - proportion_rejected = math_ops.reduce_sum((1 - accept_probs) * init_probs) - accept_probs = control_flow_ops.cond( - math_ops.less(proportion_rejected, .5), - lambda: accept_probs, - lambda: logging_ops.Print( # pylint: disable=g-long-lambda - accept_probs, [accept_probs], - message='Proportion of examples rejected by sampler is high.', - first_n=10)) - - # Make a single queue to hold input examples. Reshape output so examples - # don't have singleton batch dimension. - batched = input_ops.batch( - tensor_list + [labels], - batch_size=1, - num_threads=threads_per_queue, - capacity=queue_capacity, - enqueue_many=True) - val_list = [array_ops.squeeze(x, [0]) for x in batched[:-1]] - label = array_ops.squeeze(batched[-1], [0]) - - # Set up second queue containing batches that have the desired class - # proportions. - cur_prob = array_ops.gather(accept_probs, label) - batched = input_ops.maybe_batch( - val_list + [label], - keep_input=random_ops.random_uniform([]) < cur_prob, - batch_size=batch_size, - num_threads=threads_per_queue) - return batched[:-1], batched[-1] - - -def _estimate_data_distribution(labels, num_classes, smoothing_constant=10): - """Estimate data distribution as labels are seen.""" - # Variable to track running count of classes. Smooth by a nonzero value to - # avoid division-by-zero. Higher values provide more stability at the cost of - # slower convergence. - if smoothing_constant <= 0: - raise ValueError('smoothing_constant must be nonzero.') - num_examples_per_class_seen = variable_scope.variable( - initial_value=[smoothing_constant] * num_classes, - trainable=False, - name='class_count', - dtype=dtypes.int64) - - # Update the class-count based on what labels are seen in batch. - num_examples_per_class_seen = num_examples_per_class_seen.assign_add( - math_ops.reduce_sum( - array_ops.one_hot( - labels, num_classes, dtype=dtypes.int64), 0)) - - # Normalize count into a probability. - # NOTE: Without the `+= 0` line below, the test - # `testMultiThreadedEstimateDataDistribution` fails. The reason is that - # before this line, `num_examples_per_class_seen` is a Tensor that shares a - # buffer with an underlying `ref` object. When the `ref` is changed by another - # thread, `num_examples_per_class_seen` changes as well. Since this can happen - # in the middle of the normalization computation, we get probabilities that - # are very far from summing to one. Adding `+= 0` copies the contents of the - # tensor to a new buffer, which will be consistent from the start to the end - # of the normalization computation. - num_examples_per_class_seen += 0 - init_prob_estimate = math_ops.truediv( - num_examples_per_class_seen, - math_ops.reduce_sum(num_examples_per_class_seen)) - - # Must return float32 (not float64) to agree with downstream `_verify_input` - # checks. - return math_ops.cast(init_prob_estimate, dtypes.float32) - - -def _verify_data_inputs(tensor_list): - """Verify that batched data inputs are well-formed.""" - for tensor in tensor_list: - # Data tensor should have a batch dimension. - shape = tensor.get_shape().with_rank_at_least(1) - - # Data batch dimensions must be compatible. - tensor_shape.dimension_at_index(shape, 0).assert_is_compatible_with( - tensor_list[0].get_shape()[0]) - - return tensor_list - - -def _verify_input(tensor_list, labels, probs_list): - """Verify that batched inputs are well-formed.""" - checked_probs_list = [] - for probs in probs_list: - # Since number of classes shouldn't change at runtime, probabilities shape - # should be fully defined. - probs.get_shape().assert_is_fully_defined() - - # Probabilities must be 1D. - probs.get_shape().assert_has_rank(1) - - # Probabilities must be nonnegative and sum to one. - tol = 1e-6 - prob_sum = math_ops.reduce_sum(probs) - checked_probs = control_flow_ops.with_dependencies([ - check_ops.assert_non_negative(probs), - check_ops.assert_less(prob_sum, 1.0 + tol), - check_ops.assert_less(1.0 - tol, prob_sum) - ], probs) - checked_probs_list.append(checked_probs) - - # All probabilities should be the same length. - prob_length = checked_probs_list[0].get_shape().num_elements() - for checked_prob in checked_probs_list: - if checked_prob.get_shape().num_elements() != prob_length: - raise ValueError('Probability parameters must have the same length.') - - # Labels tensor should only have batch dimension. - labels.get_shape().assert_has_rank(1) - - for tensor in tensor_list: - # Data tensor should have a batch dimension. - shape = tensor.get_shape().with_rank_at_least(1) - - # Data and label batch dimensions must be compatible. - tensor_shape.dimension_at_index(shape, 0).assert_is_compatible_with( - labels.get_shape()[0]) - - # Data and labels must have the same, strictly positive batch size. Since we - # can't assume we know the batch size at graph creation, add runtime checks. - labels_batch_size = array_ops.shape(labels)[0] - lbl_assert = check_ops.assert_positive(labels_batch_size) - - # Make each tensor depend on its own checks. - labels = control_flow_ops.with_dependencies([lbl_assert], labels) - tensor_list = [ - control_flow_ops.with_dependencies([ - lbl_assert, - check_ops.assert_equal(array_ops.shape(x)[0], labels_batch_size) - ], x) for x in tensor_list - ] - - # Label's classes must be integers 0 <= x < num_classes. - labels = control_flow_ops.with_dependencies([ - check_ops.assert_integer(labels), check_ops.assert_non_negative(labels), - check_ops.assert_less(labels, math_ops.cast(prob_length, labels.dtype)) - ], labels) - - return tensor_list, labels, checked_probs_list - - -def _calculate_acceptance_probabilities(init_probs, target_probs): - """Calculate the per-class acceptance rates. - - Args: - init_probs: The class probabilities of the data. - target_probs: The desired class proportion in minibatches. - Returns: - A list of the per-class acceptance probabilities. - - This method is based on solving the following analysis: - - Let F be the probability of a rejection (on any example). - Let p_i be the proportion of examples in the data in class i (init_probs) - Let a_i is the rate the rejection sampler should *accept* class i - Let t_i is the target proportion in the minibatches for class i (target_probs) - - ``` - F = sum_i(p_i * (1-a_i)) - = 1 - sum_i(p_i * a_i) using sum_i(p_i) = 1 - ``` - - An example with class `i` will be accepted if `k` rejections occur, then an - example with class `i` is seen by the rejector, and it is accepted. This can - be written as follows: - - ``` - t_i = sum_k=0^inf(F^k * p_i * a_i) - = p_i * a_j / (1 - F) using geometric series identity, since 0 <= F < 1 - = p_i * a_i / sum_j(p_j * a_j) using F from above - ``` - - Note that the following constraints hold: - ``` - 0 <= p_i <= 1, sum_i(p_i) = 1 - 0 <= a_i <= 1 - 0 <= t_i <= 1, sum_i(t_i) = 1 - ``` - - - A solution for a_i in terms of the other variables is the following: - ```a_i = (t_i / p_i) / max_i[t_i / p_i]``` - """ - # Make list of t_i / p_i. - ratio_l = target_probs / init_probs - - # Replace NaNs with 0s. - ratio_l = array_ops.where_v2( - math_ops.is_nan(ratio_l), array_ops.zeros_like(ratio_l), ratio_l) - - # Calculate list of acceptance probabilities. - max_ratio = math_ops.reduce_max(ratio_l) - return ratio_l / max_ratio diff --git a/tensorflow/contrib/training/python/training/sampling_ops_test.py b/tensorflow/contrib/training/python/training/sampling_ops_test.py deleted file mode 100644 index 1aeff7dc80d..00000000000 --- a/tensorflow/contrib/training/python/training/sampling_ops_test.py +++ /dev/null @@ -1,409 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# pylint: disable=unused-import -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from tensorflow.contrib.training.python.training import sampling_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import coordinator -from tensorflow.python.training import queue_runner_impl - - -class StratifiedSampleTest(test.TestCase): - - def testGraphBuildAssertionFailures(self): - val = [array_ops.zeros([1, 3]), array_ops.ones([1, 5])] - label = constant_op.constant([1], shape=[1]) # must have batch dimension - probs = [.2] * 5 - init_probs = [.1, .3, .1, .3, .2] - batch_size = 16 - - # Label must have only batch dimension if enqueue_many is True. - with self.assertRaises(ValueError): - sampling_ops.stratified_sample( - val, - array_ops.zeros([]), - probs, - batch_size, - init_probs, - enqueue_many=True) - with self.assertRaises(ValueError): - sampling_ops.stratified_sample( - val, - array_ops.zeros([1, 1]), - probs, - batch_size, - init_probs, - enqueue_many=True) - - # Label must not be one-hot. - with self.assertRaises(ValueError): - sampling_ops.stratified_sample(val, - constant_op.constant([0, 1, 0, 0, 0]), - probs, batch_size, init_probs) - - # Data must be list, not singleton tensor. - with self.assertRaises(TypeError): - sampling_ops.stratified_sample( - array_ops.zeros([1, 3]), label, probs, batch_size, init_probs) - - # Data must have batch dimension if enqueue_many is True. - with self.assertRaises(ValueError): - sampling_ops.stratified_sample( - val, - constant_op.constant(1), - probs, - batch_size, - init_probs, - enqueue_many=True) - - # Batch dimensions on data and labels should be equal. - with self.assertRaises(ValueError): - sampling_ops.stratified_sample( - [array_ops.zeros([2, 1])], - label, - probs, - batch_size, - init_probs, - enqueue_many=True) - - # Probabilities must be numpy array, python list, or tensor. - with self.assertRaises(ValueError): - sampling_ops.stratified_sample(val, label, 1, batch_size, init_probs) - - # Probabilities shape must be fully defined. - with self.assertRaises(ValueError): - sampling_ops.stratified_sample( - val, - label, - array_ops.placeholder( - dtypes.float32, shape=[None]), - batch_size, - init_probs) - - # In the rejection sampling case, make sure that probability lengths are - # the same. - with self.assertRaises(ValueError): - sampling_ops.stratified_sample( - val, label, [.1] * 10, batch_size, init_probs=[.2] * 5) - - # In the rejection sampling case, make sure that zero initial probability - # classes also have zero target probability. - with self.assertRaises(ValueError): - sampling_ops.stratified_sample( - val, label, [.2, .4, .4], batch_size, init_probs=[0, .5, .5]) - - def testRuntimeAssertionFailures(self): - valid_probs = [.2] * 5 - valid_labels = [1, 2, 3] - vals = [array_ops.zeros([3, 1])] - - illegal_labels = [ - [0, -1, 1], # classes must be nonnegative - [5, 1, 1], # classes must be less than number of classes - [2, 3], # data and label batch size must be the same - ] - - illegal_probs = [ - [.1] * 5, # probabilities must sum to one - [-.5, .5, .5, .4, .1], # probabilities must be non-negative - ] - - # Set up graph with illegal label vector. - label_ph = array_ops.placeholder(dtypes.int32, shape=[None]) - probs_ph = array_ops.placeholder( - dtypes.float32, shape=[5]) # shape must be defined - val_tf, lbl_tf, prob_tf = sampling_ops._verify_input( # pylint: disable=protected-access - vals, label_ph, [probs_ph]) - - for illegal_label in illegal_labels: - # Run session that should fail. - with self.cached_session() as sess: - with self.assertRaises(errors_impl.InvalidArgumentError): - sess.run([val_tf, lbl_tf], - feed_dict={label_ph: illegal_label, - probs_ph: valid_probs}) - - for illegal_prob in illegal_probs: - # Run session that should fail. - with self.cached_session() as sess: - with self.assertRaises(errors_impl.InvalidArgumentError): - sess.run([prob_tf], - feed_dict={label_ph: valid_labels, - probs_ph: illegal_prob}) - - def testCanBeCalledMultipleTimes(self): - batch_size = 20 - val_input_batch = [array_ops.zeros([2, 3, 4])] - lbl_input_batch = array_ops.ones([], dtype=dtypes.int32) - probs = np.array([0, 1, 0, 0, 0]) - batches = sampling_ops.stratified_sample( - val_input_batch, lbl_input_batch, probs, batch_size, init_probs=probs) - batches += sampling_ops.stratified_sample( - val_input_batch, lbl_input_batch, probs, batch_size, init_probs=probs) - summary_op = logging_ops.merge_summary( - ops.get_collection(ops.GraphKeys.SUMMARIES)) - - with self.cached_session() as sess: - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(coord=coord) - - sess.run(batches + (summary_op,)) - - coord.request_stop() - coord.join(threads) - - def testRejectionBatchingBehavior(self): - batch_size = 20 - input_batch_size = 11 - val_input_batch = [array_ops.zeros([input_batch_size, 2, 3, 4])] - lbl_input_batch = control_flow_ops.cond( - math_ops.greater(.5, random_ops.random_uniform([])), - lambda: array_ops.ones([input_batch_size], dtype=dtypes.int32) * 1, - lambda: array_ops.ones([input_batch_size], dtype=dtypes.int32) * 3) - probs = np.array([0, .2, 0, .8, 0]) - data_batch, labels = sampling_ops.stratified_sample( - val_input_batch, - lbl_input_batch, - probs, - batch_size, - init_probs=[0, .3, 0, .7, 0], - enqueue_many=True) - with self.cached_session() as sess: - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(coord=coord) - - sess.run([data_batch, labels]) - - coord.request_stop() - coord.join(threads) - - def testBatchDimensionNotRequired(self): - classes = 5 - # Probs must be a tensor, since we pass it directly to _verify_input. - probs = constant_op.constant([1.0 / classes] * classes) - - # Make sure that these vals/labels pairs don't throw any runtime exceptions. - legal_input_pairs = [ - (np.zeros([2, 3]), [x % classes for x in range(2)]), # batch dim 2 - (np.zeros([4, 15]), [x % classes for x in range(4)]), # batch dim 4 - (np.zeros([10, 1]), [x % classes for x in range(10)]), # batch dim 10 - ] - - # Set up graph with placeholders. - vals_ph = array_ops.placeholder( - dtypes.float32) # completely undefined shape - labels_ph = array_ops.placeholder( - dtypes.int32) # completely undefined shape - val_tf, labels_tf, _ = sampling_ops._verify_input( # pylint: disable=protected-access - [vals_ph], labels_ph, [probs]) - - # Run graph to make sure there are no shape-related runtime errors. - for vals, labels in legal_input_pairs: - with self.cached_session() as sess: - sess.run([val_tf, labels_tf], - feed_dict={vals_ph: vals, - labels_ph: labels}) - - def testRejectionDataListInput(self): - batch_size = 20 - val_input_batch = [ - array_ops.zeros([2, 3, 4]), array_ops.ones([2, 4]), array_ops.ones(2) * - 3 - ] - lbl_input_batch = array_ops.ones([], dtype=dtypes.int32) - probs = np.array([0, 1, 0, 0, 0]) - val_list, lbls = sampling_ops.stratified_sample( - val_input_batch, - lbl_input_batch, - probs, - batch_size, - init_probs=[0, 1, 0, 0, 0]) - - # Check output shapes. - self.assertTrue(isinstance(val_list, list)) - self.assertEqual(len(val_list), len(val_input_batch)) - self.assertTrue(isinstance(lbls, ops.Tensor)) - - with self.cached_session() as sess: - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(coord=coord) - - out = sess.run(val_list + [lbls]) - - coord.request_stop() - coord.join(threads) - - # Check output shapes. - self.assertEqual(len(out), len(val_input_batch) + 1) - - def normalBehaviorHelper(self, sampler): - # Set up graph. - random_seed.set_random_seed(1234) - lbl1 = 0 - lbl2 = 3 - # This cond allows the necessary class queues to be populated. - label = control_flow_ops.cond( - math_ops.greater(.5, random_ops.random_uniform([])), - lambda: constant_op.constant(lbl1), lambda: constant_op.constant(lbl2)) - val = [np.array([1, 4]) * label] - probs = np.array([.8, 0, 0, .2, 0]) - batch_size = 16 - - data_batch, labels = sampler(val, label, probs, batch_size) - - # Run session and keep track of how frequently the labels and values appear. - data_l = [] - label_l = [] - with self.cached_session() as sess: - # Need to initialize variables that keep running total of classes seen. - variables.global_variables_initializer().run() - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(coord=coord) - - for _ in range(20): - [data], lbls = sess.run([data_batch, labels]) - data_l.append(data) - label_l.append(lbls) - - coord.request_stop() - coord.join(threads) - - # First check that the data matches the labels. - for lbl, data in zip(label_l, data_l): - for i in range(batch_size): - self.assertListEqual(list(np.array([1, 4]) * lbl[i]), list(data[i, :])) - - # Check that the labels are approximately correct. - expected_label = probs[0] * lbl1 + probs[3] * lbl2 - lbl_list = range(len(probs)) - lbl_std_dev = np.sqrt(np.sum((np.square(lbl_list - expected_label)))) - lbl_std_dev_of_mean = lbl_std_dev / np.sqrt(len(label_l)) # CLT - actual_lbl = np.mean(label_l) - # Tolerance is 3 standard deviations of the mean. According to the central - # limit theorem, this should cover 99.7% of cases. Note that since the seed - # is fixed, for a given implementation, this test will pass or fail 100% of - # the time. This use of assertNear is to cover cases where someone changes - # an implementation detail, which would cause the random behavior to differ. - self.assertNear(actual_lbl, expected_label, 3 * lbl_std_dev_of_mean) - - def testRejectionNormalBehavior(self): - initial_p = [.7, 0, 0, .3, 0] - - def curried_sampler(val, lbls, probs, batch, enqueue_many=False): - return sampling_ops.stratified_sample( - val, - lbls, - probs, - batch, - init_probs=initial_p, - enqueue_many=enqueue_many) - - self.normalBehaviorHelper(curried_sampler) - - def testRejectionNormalBehaviorWithOnlineInitPEstimate(self): - - def curried_sampler(val, lbls, probs, batch, enqueue_many=False): - return sampling_ops.stratified_sample( - val, lbls, probs, batch, init_probs=None, enqueue_many=enqueue_many) - - self.normalBehaviorHelper(curried_sampler) - - -class RejectionSampleTest(test.TestCase): - - def testGraphConstructionFailures(self): - accept_prob_fn = lambda _: constant_op.constant(1.0) - batch_size = 32 - # Data must have batch dimension if `enqueue_many` is `True`. - with self.assertRaises(ValueError): - sampling_ops.rejection_sample( - [array_ops.zeros([])], accept_prob_fn, batch_size, enqueue_many=True) - - # Batch dimensions should be equal if `enqueue_many` is `True`. - with self.assertRaises(ValueError): - sampling_ops.rejection_sample( - [array_ops.zeros([5, 1]), array_ops.zeros([4, 1])], - accept_prob_fn, - batch_size, - enqueue_many=True) - - def testRuntimeFailures(self): - prob_ph = array_ops.placeholder(dtypes.float32, []) - accept_prob_fn = lambda _: prob_ph - batch_size = 32 - - # Set up graph. - random_seed.set_random_seed(1234) - sampling_ops.rejection_sample( - [array_ops.zeros([])], - accept_prob_fn, - batch_size, - runtime_checks=True, - name='rejection_sample') - prob_tensor = ops.get_default_graph().get_tensor_by_name( - 'rejection_sample/prob_with_checks:0') - - # Run session that should fail. - with self.cached_session() as sess: - for illegal_prob in [-0.1, 1.1]: - with self.assertRaises(errors_impl.InvalidArgumentError): - sess.run(prob_tensor, feed_dict={prob_ph: illegal_prob}) - - def testNormalBehavior(self): - tensor_list = [ - control_flow_ops.cond( - math_ops.greater(.5, random_ops.random_uniform([])), - lambda: constant_op.constant(1.0), - lambda: constant_op.constant(2.0)) - ] - accept_prob_fn = lambda x: x[0] - 1.0 - batch_size = 10 - - # Set up graph. - sample = sampling_ops.rejection_sample(tensor_list, accept_prob_fn, - batch_size) - - with self.cached_session() as sess: - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(coord=coord) - - for _ in range(5): - sample_np = sess.run(sample)[0] - self.assertListEqual([2.0] * batch_size, list(sample_np)) - - coord.request_stop() - coord.join(threads) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/training/python/training/sampling_ops_threading_test.py b/tensorflow/contrib/training/python/training/sampling_ops_threading_test.py deleted file mode 100644 index 73ad859ab34..00000000000 --- a/tensorflow/contrib/training/python/training/sampling_ops_threading_test.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# pylint: disable=unused-import -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.training.python.training import sampling_ops -from tensorflow.python.framework import dtypes as dtypes_lib -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test -from tensorflow.python.training import coordinator -from tensorflow.python.training import queue_runner_impl - - -class SamplingOpsThreadingTest(test.TestCase): - - def testMultiThreadedEstimateDataDistribution(self): - num_classes = 10 - - # Set up graph. - random_seed.set_random_seed(1234) - label = math_ops.cast( - math_ops.round(random_ops.random_uniform([1]) * num_classes), - dtypes_lib.int32) - - prob_estimate = sampling_ops._estimate_data_distribution( # pylint: disable=protected-access - label, num_classes) - # Check that prob_estimate is well-behaved in a multithreaded context. - _, _, [prob_estimate] = sampling_ops._verify_input( # pylint: disable=protected-access - [], label, [prob_estimate]) - - # Use queues to run multiple threads over the graph, each of which - # fetches `prob_estimate`. - queue = data_flow_ops.FIFOQueue( - capacity=25, - dtypes=[prob_estimate.dtype], - shapes=[prob_estimate.get_shape()]) - enqueue_op = queue.enqueue([prob_estimate]) - queue_runner_impl.add_queue_runner( - queue_runner_impl.QueueRunner(queue, [enqueue_op] * 25)) - out_tensor = queue.dequeue() - - # Run the multi-threaded session. - with self.cached_session() as sess: - # Need to initialize variables that keep running total of classes seen. - variables.global_variables_initializer().run() - - coord = coordinator.Coordinator() - threads = queue_runner_impl.start_queue_runners(coord=coord) - - for _ in range(25): - sess.run([out_tensor]) - - coord.request_stop() - coord.join(threads) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/training/python/training/sequence_queueing_state_saver.py b/tensorflow/contrib/training/python/training/sequence_queueing_state_saver.py deleted file mode 100644 index 02baf4e071e..00000000000 --- a/tensorflow/contrib/training/python/training/sequence_queueing_state_saver.py +++ /dev/null @@ -1,1878 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""SequenceQueueingStateSaver and wrappers. - -Please see the reading data how-to for context. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import numbers - -import six - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import string_ops -from tensorflow.python.summary import summary -from tensorflow.python.training import queue_runner - -# pylint: disable=protected-access -_restore_sparse = sparse_ops._take_many_sparse_from_tensors_map -_store_sparse = sparse_ops._add_many_sparse_to_tensors_map -# pylint: enable=protected-access - - -class _SequenceInputWrapper(object): - """A wrapper object for storing sequence-related input. - - The SequenceInputWapper accepts four objects: - - length: A scalar int containing the length of the input sequence. - key: A scalar string containing the unique key of the input sequence. - sequences: A dict mapping labels, like `input`, to tensors - whose initial index dimension is at least size `length`. - context: A dict mapping labels, like `global_target`, to tensors - that represent data across the entire example. - """ - - def __init__(self, length, key, sequences, context): - length = ops.convert_to_tensor(length, name="length") - key = ops.convert_to_tensor(key, name="key") - if not isinstance(sequences, dict): - raise TypeError("sequences must be a dict") - if not isinstance(context, dict): - raise TypeError("context must be a dict") - if not sequences: - raise ValueError("must have at least one sequence tensor") - for k in sequences.keys(): - if not isinstance(k, six.string_types): - raise TypeError("sequence key must be string: %s" % k) - if ":" in k: - raise ValueError("sequence key may not have a colon: '%s'" % k) - for k in context.keys(): - if not isinstance(k, six.string_types): - raise TypeError("context key must be string: %s" % k) - if ":" in k: - raise ValueError("context key may not have a colon: '%s'" % k) - sequences = dict((k, ops.convert_to_tensor( - v, name="sequence_%s" % k)) for k, v in sequences.items()) - context = dict((k, ops.convert_to_tensor( - v, name="context_%s" % k)) for k, v in context.items()) - self._length = length - self._key = key - self._sequences = sequences - self._context = context - - @property - def length(self): - return self._length - - @property - def key(self): - return self._key - - @property - def sequences(self): - return self._sequences - - @property - def context(self): - return self._context - - -def _check_multiple_of(value, multiple_of): - """Checks that value `value` is a non-zero multiple of `multiple_of`. - - Args: - value: an int32 scalar Tensor. - multiple_of: an int or int32 scalar Tensor. - - Returns: - new_value: an int32 scalar Tensor matching `value`, but which includes an - assertion that `value` is a multiple of `multiple_of`. - """ - assert isinstance(value, ops.Tensor) - with ops.control_dependencies([ - control_flow_ops.Assert( - math_ops.logical_and( - math_ops.equal(math_ops.mod(value, multiple_of), 0), - math_ops.not_equal(value, 0)), [ - string_ops.string_join([ - "Tensor %s should be a multiple of: " % value.name, - string_ops.as_string(multiple_of), ", but saw value: ", - string_ops.as_string(value), - ". Consider setting pad=True." - ]) - ]) - ]): - new_value = array_ops.identity(value, name="multiple_of_checked") - return new_value - - -def _check_rank(value, expected_rank): - """Check the rank of Tensor `value`, via shape inference and assertions. - - Args: - value: A Tensor, possibly with shape associated shape information. - expected_rank: int32 scalar (optionally a `Tensor`). - - Returns: - new_value: A Tensor matching `value`. Accessing this tensor tests - assertions on its rank. If expected_rank is not a `Tensor`, then - new_value's shape's rank has been set. - - Raises: - ValueError: if `expected_rank` is not a `Tensor` and the rank of `value` - is known and is not equal to `expected_rank`. - """ - assert isinstance(value, ops.Tensor) - with ops.control_dependencies([ - control_flow_ops.Assert( - math_ops.equal(expected_rank, array_ops.rank(value)), [ - string_ops.string_join([ - "Rank of tensor %s should be: " % value.name, - string_ops.as_string(expected_rank), ", shape received:" - ]), array_ops.shape(value) - ]) - ]): - new_value = array_ops.identity(value, name="rank_checked") - if isinstance(expected_rank, ops.Tensor): - expected_rank_value = tensor_util.constant_value(expected_rank) - if expected_rank_value is not None: - expected_rank = int(expected_rank_value) - if not isinstance(expected_rank, ops.Tensor): - try: - new_value.set_shape(new_value.get_shape().with_rank(expected_rank)) - except ValueError as e: - raise ValueError("Rank check failed for %s: %s" % (value.name, str(e))) - return new_value - - -def _check_shape(value, expected_shape): - """Check the shape of Tensor `value`, via shape inference and assertions. - - Args: - value: A Tensor, possibly with shape associated shape information. - expected_shape: a `TensorShape`, list of `int32`, or a vector `Tensor`. - - Returns: - new_value: A Tensor matching `value`. Accessing this tensor tests - assertions on its shape. If expected_shape is not a `Tensor`, then - new_value's shape has been set. - - Raises: - ValueError: if `expected_shape` is not a `Tensor` and the shape of `value` - is known and is not equal to `expected_shape`. - """ - assert isinstance(value, ops.Tensor) - if isinstance(expected_shape, tensor_shape.TensorShape): - expected_shape = expected_shape.as_list() - if isinstance(expected_shape, ops.Tensor): - expected_shape_value = tensor_util.constant_value(expected_shape) - if expected_shape_value is not None: - expected_shape = [int(d) for d in expected_shape_value] - if isinstance(expected_shape, ops.Tensor): - value = _check_rank(value, array_ops.size(expected_shape)) - else: - value = _check_rank(value, len(expected_shape)) - with ops.control_dependencies([ - control_flow_ops.Assert( - math_ops.reduce_all( - math_ops.equal(expected_shape, array_ops.shape(value))), [ - string_ops.string_join([ - "Shape of tensor %s should be: " % value.name, - string_ops.as_string(expected_shape), - ", shape received: ", - string_ops.as_string(array_ops.shape(value)) - ]) - ]) - ]): - new_value = array_ops.identity(value, name="shape_checked") - if not isinstance(expected_shape, ops.Tensor): - try: - new_value.set_shape(new_value.get_shape().merge_with(expected_shape)) - except ValueError as e: - raise ValueError("Shape check failed for %s: %s" % (value.name, str(e))) - return new_value - - -def _check_dimensions(value, dimensions, expected_sizes, debug_prefix): - """Check the dimensions of Tensor `value`, via shape inference and assertions. - - Args: - value: A Tensor, with optional / partial shape associated shape information. - dimensions: An int list, the dimensions to check. - expected_sizes: list of mixed ints and int32 scalar tensors. - Optionally also a vector `Tensor`. - debug_prefix: A string, used for naming ops and printing debugging messages. - - Returns: - new_value: A Tensor matching `value`. Accessing this tensor tests - assertions on its shape. If expected_sizes is not a `Tensor`, then - new_value's shape has been set for all `dimensions[i]` where - `expected_sizes[i]` is not a `Tensor`. - - Raises: - TypeError: if any of the input contains invalid types: - if `value` is not a `Tensor`. - if `dimensions` is not a `list` or `tuple`. - ValueError: if input has incorrect sizes or inferred shapes do not match: - if `dimensions` contains repeated dimensions. - if `expected_sizes` is not a `Tensor` and its length does not match that - `dimensions`. - if `value`'s shape has a well-defined rank, and one of the values in - `dimensions` is equal to or above this rank. - if `value`'s shape is well defined for some `dimensions[i]`, and - `expected_sizes[i]` is not a `Tensor`, and these two values do - not match. - """ - - if not isinstance(dimensions, (list, tuple)): - raise TypeError("dimensions must be a list or tuple") - if len(set(dimensions)) != len(dimensions): - raise ValueError("dimensions are not unique: %s" % dimensions) - if not isinstance(value, ops.Tensor): - raise TypeError("value is not a Tensor: %s" % value) - value_shape = value.get_shape() - if not isinstance(expected_sizes, ops.Tensor): - if len(dimensions) != len(expected_sizes): - raise ValueError("len(dimensions) != len(expected_sizes): %d vs. %d" % - (len(dimensions), len(expected_sizes))) - if value_shape.ndims is not None: - if value_shape.ndims <= max(dimensions): - raise ValueError( - "%s: rank of input is not greater than max(dimensions): " - "%d vs. %d" % (debug_prefix, value.get_shape().ndims, - max(dimensions))) - value_dims = value_shape.as_list() - for d, s in zip(dimensions, expected_sizes): - if not isinstance(s, ops.Tensor): - value_dims[d] = s - try: - value.set_shape(value.get_shape().merge_with(value_dims)) - except ValueError as e: - raise ValueError("Dimensions check failed for %s: %s" % - (debug_prefix, str(e))) - with ops.control_dependencies([ - control_flow_ops.Assert( - math_ops.equal(expected_size, array_ops.shape(value)[dimension]), [ - string_ops.string_join([ - "Dimension %d of tensor labeled %s should be: " % - (dimension, debug_prefix), - string_ops.as_string(expected_size), ", shape received: ", - string_ops.as_string(array_ops.shape(value)) - ]) - ]) for (dimension, expected_size) in zip(dimensions, expected_sizes) - ]): - new_value = array_ops.identity(value, name="dims_checked_%s" % debug_prefix) - return new_value - - -def _prepare_sequence_inputs(inputs, states): - """Convert input to tensors and validate shape information. - - Args: - inputs: A `_SequenceInputWrapper` instance. - states: A dictionary mapping state names to input constants or tensors. - - Returns: - The tuple (length, key, sorted_states, sorted_sequences, sorted_context), - where each value has been checked for valid shape, and the sorted_* dicts - are instances of OrderedDict; with key-value pairs sorted by key. - - Raises: - ValueError: if the shapes of inputs.context.values(), states.values(), - or inputs.sequences.values() are not fully defined (with the exception - of the dimension of any `Tensor` in inputs.sequences.values()). - TypeError: if the dtype of length is not int32. - """ - # Convert state initial values to tensors - states = dict((k, ops.convert_to_tensor( - v, name="state_%s" % k)) for k, v in states.items()) - - def _assert_fully_defined(label, dict_, ignore_first_dimension=False): - start_dimension = 1 if ignore_first_dimension else 0 - for k, v in dict_.items(): - if not v.get_shape()[start_dimension:].is_fully_defined(): - raise ValueError("Shape for %s %s is not fully defined %s: %s" % - (label, k, "(ignoring first dimension)" if - ignore_first_dimension else "", v.get_shape())) - - _assert_fully_defined("state", states) - _assert_fully_defined("context", inputs.context) - # Sequences' first dimension (time) may be variable - _assert_fully_defined( - "sequence", inputs.sequences, ignore_first_dimension=True) - - # Get dictionaries' dtypes ordered by name - ordering is important - # when switching between dicts and tuples for passing to Barrier. - def _sort_by_name(d): - return collections.OrderedDict(sorted(d.items(), key=lambda k_v: k_v[0])) - - sorted_sequences = _sort_by_name(inputs.sequences) - sorted_context = _sort_by_name(inputs.context) - sorted_states = _sort_by_name(states) - - length = _check_rank(inputs.length, 0) - key = _check_rank(inputs.key, 0) - - if length.dtype != dtypes.int32: - raise TypeError("length dtype must be int32, but received: %s" % - length.dtype) - if key.dtype != dtypes.string: - raise TypeError("key dtype must be string, but received: %s" % key.dtype) - - return (length, key, sorted_states, sorted_sequences, sorted_context) - - -# NextQueuedSequenceBatch works closely with -# SequenceQueueingStateSaver and requires access to its private properties -# pylint: disable=protected-access -class NextQueuedSequenceBatch(object): - """NextQueuedSequenceBatch stores deferred SequenceQueueingStateSaver data. - - This class is instantiated by `SequenceQueueingStateSaver` and is accessible - via its `next_batch` property. - """ - - def __init__(self, state_saver): - self._state_saver = state_saver - - @property - def total_length(self): - """The lengths of the original (non-truncated) unrolled examples. - - Returns: - An integer vector of length `batch_size`, the total lengths. - """ - return self._state_saver._received_total_length - - @property - def length(self): - """The lengths of the given truncated unrolled examples. - - For initial iterations, for which `sequence * num_unroll < length`, - this number is `num_unroll`. For the remainder, - this number is between `0` and `num_unroll`. - - Returns: - An integer vector of length `batch_size`, the lengths. - """ - return self._state_saver._received_length - - @property - def batch_size(self): - """The batch_size of the given batch. - - Usually, this is the batch_size requested when initializing the SQSS, but - if allow_small_batch=True this will become smaller when inputs are - exhausted. - - Returns: - A scalar integer tensor, the batch_size - """ - return self._state_saver._received_batch_size - - @property - def insertion_index(self): - """The insertion indices of the examples (when they were first added). - - These indices start with the value -2**63 and increase with every - call to the prefetch op. Each whole example gets its own insertion - index, and this is used to prioritize the example so that its truncated - segments appear in adjacent iterations, even if new examples are inserted - by the prefetch op between iterations. - - Returns: - An int64 vector of length `batch_size`, the insertion indices. - """ - return self._state_saver._received_indices - - @property - def key(self): - """The key names of the given truncated unrolled examples. - - The format of the key is: - - ```python - "%05d_of_%05d:%s" % (sequence, sequence_count, original_key) - ``` - - where `original_key` is the unique key read in by the prefetcher. - - Returns: - A string vector of length `batch_size`, the keys. - """ - return self._state_saver._received_keys - - @property - def next_key(self): - """The key names of the next (in iteration) truncated unrolled examples. - - The format of the key is: - - ```python - "%05d_of_%05d:%s" % (sequence + 1, sequence_count, original_key) - ``` - - if `sequence + 1 < sequence_count`, otherwise: - - ```python - "STOP:%s" % original_key - ``` - - where `original_key` is the unique key read in by the prefetcher. - - Returns: - A string vector of length `batch_size`, the keys. - """ - return self._state_saver._received_next_key - - @property - def sequence(self): - """An int32 vector, length `batch_size`: the sequence index of each entry. - - When an input is split up, the sequence values - ``` - 0, 1, ..., sequence_count - 1 - ``` - are assigned to each split. - - Returns: - An int32 vector `Tensor`. - """ - return self._state_saver._received_sequence - - @property - def sequence_count(self): - """An int32 vector, length `batch_size`: the sequence count of each entry. - - When an input is split up, the number of splits is equal to: - `padded_length / num_unroll`. This is the sequence_count. - - Returns: - An int32 vector `Tensor`. - """ - return self._state_saver._received_sequence_count - - @property - def context(self): - """A dict mapping keys of `input_context` to batched context. - - Returns: - A dict mapping keys of `input_context` to tensors. - If we had at input: - - ```python - context["name"].get_shape() == [d1, d2, ...] - ``` - - then for this property: - - ```python - context["name"].get_shape() == [batch_size, d1, d2, ...] - ``` - - """ - return self._state_saver._received_context - - @property - def sequences(self): - """A dict mapping keys of `input_sequences` to split and rebatched data. - - Returns: - A dict mapping keys of `input_sequences` to tensors. - If we had at input: - - ```python - sequences["name"].get_shape() == [None, d1, d2, ...] - ``` - - where `None` meant the sequence time was dynamic, then for this property: - - ```python - sequences["name"].get_shape() == [batch_size, num_unroll, d1, d2, ...]. - ``` - - """ - return self._state_saver._received_sequences - - def state(self, state_name): - """Returns batched state tensors. - - Args: - state_name: string, matches a key provided in `initial_states`. - - Returns: - A `Tensor`: a batched set of states, either initial states (if this is - the first run of the given example), or a value as stored during - a previous iteration via `save_state` control flow. - Its type is the same as `initial_states["state_name"].dtype`. - If we had at input: - - ```python - initial_states[state_name].get_shape() == [d1, d2, ...], - ``` - - then - - ```python - state(state_name).get_shape() == [batch_size, d1, d2, ...] - ``` - - Raises: - KeyError: if `state_name` does not match any of the initial states - declared in `initial_states`. - """ - return self._state_saver._received_states[state_name] - - def save_state(self, state_name, value, name=None): - """Returns an op to save the current batch of state `state_name`. - - Args: - state_name: string, matches a key provided in `initial_states`. - value: A `Tensor`. - Its type must match that of `initial_states[state_name].dtype`. - If we had at input: - - ```python - initial_states[state_name].get_shape() == [d1, d2, ...] - ``` - - then the shape of `value` must match: - - ```python - tf.shape(value) == [batch_size, d1, d2, ...] - ``` - - name: string (optional). The name scope for newly created ops. - - Returns: - A control flow op that stores the new state of each entry into - the state saver. This op must be run for every iteration that - accesses data from the state saver (otherwise the state saver - will never progress through its states and run out of capacity). - - Raises: - KeyError: if `state_name` does not match any of the initial states - declared in `initial_states`. - """ - if state_name not in self._state_saver._received_states.keys(): - raise KeyError("state was not declared: %s" % state_name) - default_name = "InputQueueingStateSaver_SaveState" - with ops.name_scope(name, default_name, values=[value]): - # Place all operations on the CPU. Barriers and queues are only - # implemented for CPU, but all the other book-keeping operations - # (reshape, shape, range, ...) would be placed on GPUs if available, - # unless we explicitly tie them to CPU. - with ops.colocate_with(self._state_saver._capacity_queue.queue_ref): - indices_where_not_done = array_ops.reshape( - array_ops.where_v2( - math_ops.logical_not(self._state_saver._sequence_is_done)), - [-1]) - keeping_next_key = array_ops.gather( - self._state_saver._received_next_key, indices_where_not_done) - value = _check_shape( - array_ops.identity( - value, name="convert_%s" % state_name), - array_ops.shape(self._state_saver._received_states[state_name])) - keeping_state = array_ops.gather(value, indices_where_not_done) - return self._state_saver._barrier.insert_many( - self._state_saver._get_barrier_index("state", state_name), - keeping_next_key, - keeping_state, - name="BarrierInsertState_%s" % state_name) - - -# pylint: enable=protected-access - - -class SequenceQueueingStateSaver(object): - """SequenceQueueingStateSaver provides access to stateful values from input. - - This class is meant to be used instead of, e.g., a `Queue`, for splitting - variable-length sequence inputs into segments of sequences with fixed length - and batching them into mini-batches. It maintains contexts and state for a - sequence across the segments. It can be used in conjunction with a - `QueueRunner` (see the example below). - - The `SequenceQueueingStateSaver` (SQSS) accepts one example at a time via the - inputs `input_length`, `input_key`, `input_sequences` (a dict), - `input_context` (a dict), and `initial_states` (a dict). - The sequences, values in `input_sequences`, may have variable first dimension - (the `padded_length`), though this dimension must always be a multiple of - `num_unroll`. All other dimensions must be fixed and accessible via - `get_shape` calls. The length prior to padding can be recorded in - `input_length`. The context values in `input_context` must all have fixed and - well defined dimensions. The initial state values must all have fixed and - well defined dimensions. - - The SQSS splits the sequences of an input example into segments of length - `num_unroll`. Across examples minibatches of size `batch_size` are formed. - These minibatches contain a segment of the sequences, copy the context values, - and maintain state, length, and key information of the original input - examples. In the first segment of an example the state is still the initial - state. It can then be updated; and updated state values are accessible in - subsequent segments of the same example. After each segment - `batch.save_state()` must be called which is done by the state_saving_rnn. - Without this call, the dequeue op associated with the SQSS will not run. - Internally, SQSS has a queue for the input examples. Its `capacity` is - configurable. If set smaller than `batch_size` then the dequeue op will block - indefinitely. A small multiple of `batch_size` is a good rule of thumb to - prevent that queue from becoming a bottleneck and slowing down training. - If set too large (and note that it defaults to unbounded) memory consumption - goes up. Moreover, when iterating over the same input examples multiple times - reusing the same `key` the `capacity` must be smaller than the number of - examples. - - The prefetcher, which reads one unrolled, variable-length input sequence at - a time, is accessible via `prefetch_op`. The underlying `Barrier` object - is accessible via `barrier`. Processed minibatches, as well as - state read and write capabilities are accessible via `next_batch`. - Specifically, `next_batch` provides access to all of the minibatched - data, including the following, see `NextQueuedSequenceBatch` for details: - - * `total_length`, `length`, `insertion_index`, `key`, `next_key`, - * `sequence` (the index each minibatch entry's time segment index), - * `sequence_count` (the total time segment count for each minibatch entry), - * `context` (a dict of the copied minibatched context values), - * `sequences` (a dict of the split minibatched variable-length sequences), - * `state` (to access the states of the current segments of these entries) - * `save_state` (to save the states for the next segments of these entries) - - Example usage: - - ```python - batch_size = 32 - num_unroll = 20 - lstm_size = 8 - cell = tf.compat.v1.nn.rnn_cell.BasicLSTMCell(num_units=lstm_size) - initial_state_values = tf.zeros(cell.state_size, dtype=tf.float32) - - raw_data = get_single_input_from_input_reader() - length, key, sequences, context = my_parser(raw_data) - assert "input" in sequences.keys() - assert "label" in context.keys() - initial_states = {"lstm_state": initial_state_value} - - stateful_reader = tf.SequenceQueueingStateSaver( - batch_size, num_unroll, - length=length, input_key=key, input_sequences=sequences, - input_context=context, initial_states=initial_states, - capacity=batch_size*100) - - batch = stateful_reader.next_batch - inputs = batch.sequences["input"] - context_label = batch.context["label"] - - inputs_by_time = tf.split(value=inputs, num_or_size_splits=num_unroll, axis=1) - assert len(inputs_by_time) == num_unroll - - lstm_output, _ = tf.contrib.rnn.static_state_saving_rnn( - cell, - inputs_by_time, - state_saver=batch, - state_name="lstm_state") - - # Start a prefetcher in the background - sess = tf.compat.v1.Session() - num_threads = 3 - queue_runner = tf.compat.v1.train.QueueRunner( - stateful_reader, [stateful_reader.prefetch_op] * num_threads) - tf.compat.v1.train.add_queue_runner(queue_runner) - tf.compat.v1.train.start_queue_runners(sess=session) - - while True: - # Step through batches, perform training or inference... - session.run([lstm_output]) - ``` - - **Note**: Usually the barrier is given to a QueueRunner as in the - examples above. The QueueRunner will close the barrier if the prefetch_op - receives an OutOfRange Error from upstream input queues (i.e., reaches - the end of the input). If the barrier is closed no further new examples - are added to the SQSS. The underlying barrier might, however, still - contain further unroll-steps of examples that have not undergone all - iterations. To gracefully finish all examples, the flag - `allow_small_batch` must be set to true, which causes the SQSS to issue - progressively smaller mini-batches with the remaining examples. - """ - - def __init__(self, - batch_size, - num_unroll, - input_length, - input_key, - input_sequences, - input_context, - initial_states, - capacity=None, - allow_small_batch=False, - name=None): - """Creates the SequenceQueueingStateSaver. - - Args: - batch_size: int or int32 scalar `Tensor`, how large minibatches should - be when accessing the `state()` method and `context`, `sequences`, etc, - properties. - num_unroll: Python integer, how many time steps to unroll at a time. - The input sequences of length `k` are then split into `k / num_unroll` - many segments. - input_length: An int32 scalar `Tensor`, the length of the sequence prior - to padding. This value may be at most `padded_length` for any given - input (see below for the definition of `padded_length`). - Batched and total lengths of the current iteration are made accessible - via the `length` and `total_length` properties. The shape of - input_length (scalar) must be fully specified. - input_key: A string scalar `Tensor`, the **unique** key for the given - input. This is used to keep track of the split minibatch elements - of this input. Batched keys of the current iteration are made - accessible via the `key` property. The shape of `input_key` (scalar) - must be fully specified. - input_sequences: A dict mapping string names to `Tensor` values. The - values must all have matching first dimension, called `padded_length`. - The `SequenceQueueingStateSaver` will split these tensors along - this first dimension into minibatch elements of dimension - `num_unroll`. Batched and segmented sequences of the current iteration - are made accessible via the `sequences` property. - - **Note**: `padded_length` may be dynamic, and may vary from input - to input, but must always be a multiple of `num_unroll`. The remainder - of the shape (other than the first dimension) must be fully specified. - input_context: A dict mapping string names to `Tensor` values. The values - are treated as "global" across all time splits of the given input, - and will be copied across for all minibatch elements accordingly. - Batched and copied context of the current iteration are made - accessible via the `context` property. - - **Note**: All input_context values must have fully defined shapes. - initial_states: A dict mapping string state names to multi-dimensional - values (e.g. constants or tensors). This input defines the set of - states that will be kept track of during computing iterations, and - which can be accessed via the `state` and `save_state` methods. - - **Note**: All initial_state values must have fully defined shapes. - capacity: The max capacity of the SQSS in number of examples. Needs to be - at least `batch_size`. Defaults to unbounded. - allow_small_batch: If true, the SQSS will return smaller batches when - there aren't enough input examples to fill a whole batch and the end of - the input has been reached (i.e., the underlying barrier has been - closed). - name: An op name string (optional). - - Raises: - TypeError: if any of the inputs is not an expected type. - ValueError: if any of the input values is inconsistent, e.g. if - not enough shape information is available from inputs to build - the state saver. - """ - if capacity is not None and isinstance(batch_size, ops.Tensor): - with ops.control_dependencies([check_ops.assert_greater_equal( - math_ops.cast(capacity, dtype=dtypes.int64), - math_ops.cast(batch_size, dtype=dtypes.int64), - message="capacity needs to be >= batch_size.")]): - input_key = array_ops.identity(input_key) - elif capacity is not None and capacity < batch_size: - raise ValueError("capacity %d needs to be >= batch_size %d" % ( - capacity, batch_size)) - # The barrier is ignorant of the number of actual examples, since a long - # example that requires many iterations produces more elements in the - # barrier than a short example. Furthermore, we don't have an upper bound - # on the length of examples, and hence have to keep the capacity of the - # barrier at infinite to avoid dead-lock. Instead we have to keep track of - # the number of active examples in this class, and block the prefetch_op - # when capacity is reached. To this end, we employ a FIFOQueue in which we - # store one token (its value doesn't matter) for each input example, and - # dequeue a token for each completed example. Since the capacity of this - # queue is limited the enqueue operation will block if capacity is reached. - self._capacity_queue = data_flow_ops.FIFOQueue( - capacity=capacity, dtypes=[dtypes.int32], shapes=[[]]) - # Place all operations on the CPU. Barriers and queues are only implemented - # for CPU, but all the other book-keeping operations - # (reshape, shape, range, ...) would be placed on GPUs if available, - # unless we explicitly tie them to CPU. - with ops.colocate_with(self._capacity_queue.queue_ref): - if not isinstance(initial_states, dict): - raise TypeError("initial_states must be a dictionary") - if not initial_states: - raise ValueError( - "initial_states may not be empty: at least one state variable is " - "required to properly enqueue split sequences to run in separate " - "iterations") - for k in initial_states: - if not isinstance(k, six.string_types): - raise TypeError("state name must be a string: %s" % k) - if ":" in k: - raise ValueError("state name may not have a colon: '%s'" % k) - - op_vars = ([input_length, input_key] + list(input_sequences.values()) + - list(input_context.values())) - with ops.name_scope(name, "InputQueueingStateSaver", op_vars) as scope: - inputs = _SequenceInputWrapper(input_length, input_key, input_sequences, - input_context) - self._batch_size = batch_size - self._num_unroll = num_unroll - self._name = scope - - # This step makes sure all shapes are well defined. We can now - # use get_shape() on any tensor in the output of this function - # and get a fully-defined shape. - (self._length, self._key, self._sorted_states, self._sorted_sequences, - self._sorted_context) = _prepare_sequence_inputs(inputs, - initial_states) - self._padded_length = array_ops.identity( - array_ops.shape(six.next(six.itervalues(self._sorted_sequences)))[ - 0], - name="padded_length") # The name is useful for debugging - self._padded_length = _check_multiple_of(self._padded_length, - self._num_unroll) - - # sequences should have length == all matching - self._sorted_sequences = collections.OrderedDict( - (k, _check_dimensions( - v, [0], [self._padded_length], - debug_prefix="sorted_sequences_%s" % k)) - for k, v in self._sorted_sequences.items()) - self._uninitialized_states = self._sorted_states - - # Once this is set, self._get_barrier_*_index are available for use. - self._store_index_maps(self._sorted_sequences, self._sorted_context, - self._sorted_states) - - # Make sure that the length is <= the padded_length - with ops.control_dependencies([ - control_flow_ops.Assert( - math_ops.less_equal(self._length, self._padded_length), [ - "Input length should be <= than length from sequences:", - self._length, " vs. ", self._padded_length - ]) - ]): - self._length = array_ops.identity(self._length) - - # Only create barrier; enqueue and dequeue operations happen when you - # access prefetch_op and next_batch. - self._create_barrier() - self._scope = scope - self._allow_small_batch = allow_small_batch - self._prefetch_op = None - self._next_batch = None - - @property - def name(self): - return self._name - - @property - def barrier(self): - return self._barrier - - @property - def batch_size(self): - return self._batch_size - - @property - def num_unroll(self): - return self._num_unroll - - @property - def prefetch_op(self): - """The op used to prefetch new data into the state saver. - - Running it once enqueues one new input example into the state saver. - The first time this gets called, it additionally creates the prefetch_op. - Subsequent calls simply return the previously created `prefetch_op`. - - It should be run in a separate thread via e.g. a `QueueRunner`. - - Returns: - An `Operation` that performs prefetching. - """ - if not self._prefetch_op: - with ops.name_scope(None), ops.name_scope( - self._scope, values=[self._barrier.barrier_ref]): - self._create_prefetch_op() - return self._prefetch_op - - @property - def next_batch(self): - """The `NextQueuedSequenceBatch` providing access to batched output data. - - Also provides access to the `state` and `save_state` methods. - The first time this gets called, it additionally prepares barrier reads - and creates `NextQueuedSequenceBatch` / next_batch objects. Subsequent - calls simply return the previously created `next_batch`. - - In order to access data in `next_batch` without blocking, the `prefetch_op` - must have been run at least `batch_size` times (ideally in a separate - thread, or launched via a `QueueRunner`). After processing a segment in - `next_batch()`, `batch.save_state()` must be called which is done by the - state_saving_rnn. Without this call, the dequeue op associated with the SQSS - will not run. - - Returns: - A cached `NextQueuedSequenceBatch` instance. - """ - # This is needed to prevent errors if next_batch is called before - # prefetch_op is created. - if not self._prefetch_op: - with ops.name_scope(None), ops.name_scope( - self._scope, values=[self._barrier.barrier_ref]): - self._create_prefetch_op() - if not self._next_batch: - with ops.name_scope(None), ops.name_scope( - self._scope, values=[self._barrier.barrier_ref]): - self._prepare_barrier_reads() - return self._next_batch - - def close(self, cancel_pending_enqueues=False, name=None): - """Closes the barrier and the FIFOQueue. - - This operation signals that no more segments of new sequences will be - enqueued. New segments of already inserted sequences may still be enqueued - and dequeued if there is a sufficient number filling a batch or - allow_small_batch is true. Otherwise dequeue operations will fail - immediately. - - Args: - cancel_pending_enqueues: (Optional.) A boolean, defaulting to - `False`. If `True`, all pending enqueues to the underlying queues will - be cancelled, and completing already started sequences is not possible. - name: Optional name for the op. - - Returns: - The operation that closes the barrier and the FIFOQueue. - """ - with ops.name_scope(name, "SQSSClose", [self._prefetch_op]) as name: - barrier_close = self.barrier.close(cancel_pending_enqueues, - "BarrierClose") - fifo_queue_close = self._capacity_queue.close(cancel_pending_enqueues, - "FIFOClose") - return control_flow_ops.group(barrier_close, fifo_queue_close, name=name) - - def _store_index_maps(self, sequences, context, states): - """Prepares the internal dictionaries _name_to_index and _index_to_name. - - These dictionaries are used to keep track of indices into the barrier. - - Args: - sequences: `OrderedDict` of string, `Tensor` pairs. - context: `OrderedDict` of string, `Tensor` pairs. - states: `OrderedDict` of string, `Tensor` pairs. - """ - assert isinstance(sequences, dict) - assert isinstance(context, dict) - assert isinstance(states, dict) - self._name_to_index = { - name: ix - for (ix, name) in enumerate([ - "__length", "__total_length", "__next_key", "__sequence", - "__sequence_count" - ] + ["__sequence__%s" % k for k in sequences.keys()] + [ - "__context__%s" % k for k in context.keys() - ] + ["__state__%s" % k for k in states.keys()])} - self._index_to_name = [ - name - for (name, _) in sorted( - self._name_to_index.items(), key=lambda n_ix: n_ix[1]) - ] - - def _get_barrier_length_index(self): - return self._name_to_index["__length"] - - def _get_barrier_total_length_index(self): - return self._name_to_index["__total_length"] - - def _get_barrier_next_key_index(self): - return self._name_to_index["__next_key"] - - def _get_barrier_sequence_index(self): - return self._name_to_index["__sequence"] - - def _get_barrier_sequence_count_index(self): - return self._name_to_index["__sequence_count"] - - def _get_barrier_index(self, index_type, name): - assert index_type in ("sequence", "context", "state") - key = "__%s__%s" % (index_type, name) - assert key in self._name_to_index, ( - "Requested a name not in the value type %s: %s" % (index_type, name)) - return self._name_to_index[key] - - def _create_barrier(self): - """Create the barrier. - - This method initializes the Barrier object with the right types and shapes. - """ - # Create the barrier - sequence_dtypes = [v.dtype for k, v in self._sorted_sequences.items()] - context_dtypes = [v.dtype for k, v in self._sorted_context.items()] - state_dtypes = [v.dtype for k, v in self._sorted_states.items()] - types = ([ - dtypes.int32, # length - dtypes.int32, # total_length - dtypes.string, # next_keys - dtypes.int32, # sequence - dtypes.int32 - ] # expanded_sequence_count - + sequence_dtypes + context_dtypes + state_dtypes) - sequence_shapes = [ - [self._num_unroll] + self._sorted_sequences[k].get_shape().as_list()[1:] - for k in self._sorted_sequences.keys() - ] - context_shapes = [ - self._sorted_context[k].get_shape().as_list() - for k in self._sorted_context.keys() - ] - state_shapes = [ - self._sorted_states[k].get_shape().as_list() - for k in self._sorted_states.keys() - ] - shapes = ([ - (), # length - (), # total_length - (), # next_keys - (), # sequence - () - ] # expanded_sequence_count - + sequence_shapes + context_shapes + state_shapes) - - self._barrier = data_flow_ops.Barrier(types=types, shapes=shapes) - - def _create_prefetch_op(self): - """Group insert_many ops and create prefetch_op. - - This method implements the "meat" of the logic underlying the - `SequenceQueueingStateSaver`. It performs dynamic reshaping of - sequences, copying of context, and initial insertion of these values, - as well as the key, next_key, sequence, sequence_count, and initial - states into the barrier. - """ - # Step 1: identify how many barrier entries to split this input - # into, store the result as a scalar - sequence_count = math_ops.div(self._padded_length, self._num_unroll) - sequence_count_vec = array_ops.expand_dims(sequence_count, 0) - - # The final unrolled sequence's length is num_unroll only in - # the case that num_unroll divides it evenly. - ones = array_ops.ones(sequence_count_vec, dtype=dtypes.int32) - sequence = math_ops.range(sequence_count) - expanded_length = math_ops.maximum( - 0, self._length - self._num_unroll * sequence) - expanded_length = math_ops.minimum(self._num_unroll, expanded_length) - expanded_total_length = self._length * ones - expanded_sequence_count = sequence_count * ones - current_keys = string_ops.string_join( - [ - string_ops.as_string( - sequence, width=5, fill="0"), "_of_", string_ops.as_string( - sequence_count, width=5, fill="0"), ":", self._key - ], - name="StringJoinCurrentKeys") - next_keys = array_ops.concat( - [ - array_ops.slice(current_keys, [1], [-1]), array_ops.expand_dims( - string_ops.string_join( - ["STOP:", self._key], name="StringJoinStop"), - 0) - ], - 0, - name="concat_next_keys") - reshaped_sequences = collections.OrderedDict(( - k, - _check_dimensions( - # Reshape sequences to sequence_count rows - array_ops.reshape( - v, - array_ops.concat( - [ - array_ops.expand_dims(sequence_count, 0), - array_ops.expand_dims(self._num_unroll, 0), - v.get_shape().as_list()[1:] - ], - 0, - name="concat_sequences_%s" % k), - name="reshape_sequences_%s" % k), - [0, 1] + list(range(2, v.get_shape().ndims + 1)), - [sequence_count, self._num_unroll] + v.get_shape().as_list()[1:], - debug_prefix="reshaped_sequences_%s" % - k)) for k, v in self._sorted_sequences.items()) - expanded_context = collections.OrderedDict( - ( - k, - _check_dimensions( - # Copy context to be sequence_count rows - array_ops.tile( - array_ops.expand_dims(v, 0), - array_ops.concat( - [ - array_ops.expand_dims(sequence_count, 0), - [1] * v.get_shape().ndims - ], - 0, - name="concat_context_%s" % k), - name="tile_context_%s" % k), - [0] + list(range(1, v.get_shape().ndims + 1)), - [sequence_count] + v.get_shape().as_list(), - debug_prefix="expanded_context_%s" % k)) - for k, v in self._sorted_context.items()) - - # Storing into the barrier, for each current_key: - # sequence_ix, sequence_count, next_key, length, - # context... (copied), sequences... (truncated) - # Also storing into the barrier for the first key - # states (using initial_states). - insert_sequence_op = self._barrier.insert_many( - self._get_barrier_sequence_index(), - current_keys, - sequence, - name="BarrierInsertSequence") - insert_sequence_count_op = self._barrier.insert_many( - self._get_barrier_sequence_count_index(), - current_keys, - expanded_sequence_count, - name="BarrierInsertSequenceCount") - insert_next_key_op = self._barrier.insert_many( - self._get_barrier_next_key_index(), - current_keys, - next_keys, - name="BarrierInsertNextKey") - insert_length_op = self._barrier.insert_many( - self._get_barrier_length_index(), - current_keys, - expanded_length, - name="BarrierInsertLength") - insert_total_length_op = self._barrier.insert_many( - self._get_barrier_total_length_index(), - current_keys, - expanded_total_length, - name="BarrierInsertTotalLength") - insert_context_ops = dict((name, self._barrier.insert_many( - self._get_barrier_index("context", name), - current_keys, - value, - name="BarrierInsertContext_%s" % name)) - for (name, value) in expanded_context.items()) - insert_sequences_ops = dict((name, self._barrier.insert_many( - self._get_barrier_index("sequence", name), - current_keys, - value, - name="BarrierInsertSequences_%s" % name)) - for (name, value) in reshaped_sequences.items()) - - # An op that blocks if we reached capacity in number of active examples. - TOKEN_WITH_IGNORED_VALUE = 21051976 # pylint: disable=invalid-name - insert_capacity_token_op = self._capacity_queue.enqueue( - (TOKEN_WITH_IGNORED_VALUE,)) - - # Insert just the initial state. Specifically force this to run - # the insert sequence op *first* so that the Barrier receives - # an insert with *all* the segments and the segments all get the same index. - with ops.control_dependencies( - [insert_sequence_op, insert_capacity_token_op]): - insert_initial_state_ops = dict( - (name, self._barrier.insert_many( - self._get_barrier_index("state", name), - array_ops.stack([current_keys[0]]), - array_ops.stack([value]), - name="BarrierInitialInsertState_%s" % name)) - for (name, value) in self._uninitialized_states.items()) - - all_inserts = ([ - insert_capacity_token_op, insert_sequence_op, insert_sequence_count_op, - insert_next_key_op, insert_length_op, insert_total_length_op - ] + list(insert_initial_state_ops.values()) + - list(insert_context_ops.values()) + - list(insert_sequences_ops.values())) - - self._prefetch_op = control_flow_ops.group( - *all_inserts, name="StateSaverPrefetchGroup") - - def _prepare_barrier_reads(self): - """Creates ops for reading the barrier, as used by properties like `length`. - """ - # Ops for reading from the barrier. These ops must be run in a - # different thread than the prefetcher op to avoid blocking. - received = self._barrier.take_many( - self._batch_size, self._allow_small_batch, name="BarrierTakeMany") - - self._received_indices = received[0] - self._received_keys = received[1] - received_values = received[2] - - self._received_sequence = received_values[self._get_barrier_sequence_index( - )] - self._received_sequence_count = received_values[ - self._get_barrier_sequence_count_index()] - self._received_next_key = received_values[self._get_barrier_next_key_index( - )] - self._received_length = received_values[self._get_barrier_length_index()] - self._received_total_length = received_values[ - self._get_barrier_total_length_index()] - self._received_context = collections.OrderedDict( - (name, received_values[self._get_barrier_index("context", name)]) - for name in self._sorted_context.keys()) - self._received_sequences = collections.OrderedDict( - (name, received_values[self._get_barrier_index("sequence", name)]) - for name in self._sorted_sequences.keys()) - - self._received_batch_size = array_ops.squeeze( - array_ops.shape(self._received_length)) - - # Which examples are we done with? - self._sequence_is_done = ( - self._received_sequence + 1 >= self._received_sequence_count) - - # Compute the number of finished sequences and dequeue as many tokens from - # the capacity queue. - finished_sequences = (math_ops.reduce_sum( - math_ops.cast(self._sequence_is_done, dtypes.int32))) - # TODO(ebrevdo): convert to dequeue_up_to when FIFOQueue supports it. - dequeue_op = self._capacity_queue.dequeue_many(finished_sequences) - - # Tie the dequeue_op to the received_state, such that it is definitely - # carried out. - with ops.control_dependencies([dequeue_op]): - self._received_states = collections.OrderedDict( - (name, array_ops.identity(received_values[self._get_barrier_index( - "state", name)])) for name in self._sorted_states.keys()) - self._next_batch = NextQueuedSequenceBatch(self) - - -def batch_sequences_with_states(input_key, - input_sequences, - input_context, - input_length, - initial_states, - num_unroll, - batch_size, - num_threads=3, - capacity=1000, - allow_small_batch=True, - pad=True, - make_keys_unique=False, - make_keys_unique_seed=None, - name=None): - """Creates batches of segments of sequential input. - - This method creates a `SequenceQueueingStateSaver` (SQSS) and adds it to - the queuerunners. It returns a `NextQueuedSequenceBatch`. - - It accepts one example at a time identified by a unique `input_key`. - `input_sequence` is a dict with values that are tensors with time as first - dimension. This time dimension must be the same across those tensors of an - example. It can vary across examples. Although it always has to be a multiple - of `num_unroll`. Hence, padding may be necessary and it is turned on by - default by `pad=True`. - - `input_length` is a Tensor scalar or an int recording the time dimension prior - to padding. It should be between 0 and the time dimension. One reason we want - to keep track of it is so that we can take it into consideration when - computing the loss. If `pad=True` then `input_length` can be `None` and will - be inferred. - - This methods segments `input_sequence` into segments of length `num_unroll`. - It batches input sequences from `batch_size` many examples. These mini-batches - are available through the `sequence` property of the output. Moreover, for - each entry in the batch we can access its original `input_key` in `key` and - its input length in `total_length`. `length` records within this segment how - many non-padded time steps there are. - - Static features of an example that do not vary across time can be part of the - `input_context`, a dict with Tensor values. This method copies the context for - each segment and makes it available in the `context` of the output. - - This method can maintain and update a state for each example. It accepts some - initial_states as a dict with Tensor values. The first mini-batch an example - is contained has initial_states as entry of the `state`. If save_state is - called then the next segment will have the updated entry of the `state`. - See `NextQueuedSequenceBatch` for a complete list of properties and methods. - - Example usage: - - ```python - batch_size = 32 - num_unroll = 20 - num_enqueue_threads = 3 - lstm_size = 8 - cell = tf.compat.v1.nn.rnn_cell.BasicLSTMCell(num_units=lstm_size) - - key, sequences, context = my_parser(raw_data) - initial_state_values = tf.zeros((state_size,), dtype=tf.float32) - initial_states = {"lstm_state": initial_state_values} - batch = tf.batch_sequences_with_states( - input_key=key, - input_sequences=sequences, - input_context=context, - input_length=tf.shape(sequences["input"])[0], - initial_states=initial_states, - num_unroll=num_unroll, - batch_size=batch_size, - num_threads=num_enqueue_threads, - capacity=batch_size * num_enqueue_threads * 2) - - inputs = batch.sequences["input"] - context_label = batch.context["label"] - - inputs_by_time = tf.split(value=inputs, num_or_size_splits=num_unroll, axis=1) - assert len(inputs_by_time) == num_unroll - - lstm_output, _ = tf.contrib.rnn.static_state_saving_rnn( - cell, - inputs_by_time, - state_saver=batch, - state_name="lstm_state") - - # Start a prefetcher in the background - sess = tf.compat.v1.Session() - - tf.compat.v1.train.start_queue_runners(sess=session) - - while True: - # Step through batches, perform training or inference... - session.run([lstm_output]) - ``` - - Args: - input_key: A string scalar `Tensor`, the **unique** key for the given - input example. This is used to keep track of the split minibatch elements - of this input. Batched keys of the current iteration are made - accessible via the `key` property. The shape of `input_key` (scalar) must - be fully specified. Consider setting `make_keys_unique` to True when - iterating over the same input multiple times. - - **Note**: if `make_keys_unique=False` then `input_key`s must be unique. - input_sequences: A dict mapping string names to `Tensor` values. The values - must all have matching first dimension, called `value_length`. They may - vary from input to input. The remainder of the shape (other than the first - dimension) must be fully specified. - The `SequenceQueueingStateSaver` will split these tensors along - this first dimension into minibatch elements of dimension `num_unrolled`. - Batched and segmented sequences of the current iteration are made - accessible via the `sequences` property. - - **Note**: if `pad=False`, then `value_length` must always be a multiple - of `num_unroll`. - input_context: A dict mapping string names to `Tensor` values. The values - are treated as "global" across all time splits of the given input example, - and will be copied across for all minibatch elements accordingly. - Batched and copied context of the current iteration are made - accessible via the `context` property. - - **Note**: All input_context values must have fully defined shapes. - input_length: None or an int32 scalar `Tensor`, the length of the sequence - prior to padding. If `input_length=None` and `pad=True` then the length - will be inferred and will be equal to `value_length`. If `pad=False` then - `input_length` cannot be `None`: `input_length` must be specified. Its - shape of `input_length` (scalar) must be fully specified. Its value may be - at most `value_length` for any given input (see above for the definition - of `value_length`). Batched and total lengths of the current iteration are - made accessible via the `length` and `total_length` properties. - initial_states: A dict mapping string state names to multi-dimensional - values (e.g. constants or tensors). This input defines the set of - states that will be kept track of during computing iterations, and - which can be accessed via the `state` and `save_state` methods. - - **Note**: All initial_state values must have fully defined shapes. - num_unroll: Python integer, how many time steps to unroll at a time. - The input sequences of length k are then split into k / num_unroll many - segments. - batch_size: int or int32 scalar `Tensor`, how large minibatches should - be when accessing the `state()` method and `context`, `sequences`, etc, - properties. - num_threads: The int number of threads enqueuing input examples into a - queue. - capacity: The max capacity of the queue in number of examples. Needs to be - at least `batch_size`. Defaults to 1000. When iterating over the same - input example multiple times reusing their keys the `capacity` must be - smaller than the number of examples. - allow_small_batch: If true, the queue will return smaller batches when - there aren't enough input examples to fill a whole batch and the end of - the input has been reached. - pad: If `True`, `input_sequences` will be padded to multiple of - `num_unroll`. In that case `input_length` may be `None` and is assumed to - be the length of first dimension of values in `input_sequences` - (i.e. `value_length`). - make_keys_unique: Whether to append a random integer to the `input_key` in - an effort to make it unique. The seed can be set via - `make_keys_unique_seed`. - make_keys_unique_seed: If `make_keys_unique=True` this fixes the seed with - which a random postfix is generated. - name: An op name string (optional). - - Returns: - A NextQueuedSequenceBatch with segmented and batched inputs and their - states. - - Raises: - TypeError: if any of the inputs is not an expected type. - ValueError: if any of the input values is inconsistent, e.g. if - not enough shape information is available from inputs to build - the state saver. - """ - tensor_list = (list(input_sequences.values()) + list(input_context.values()) + - list(initial_states.values())) - with ops.name_scope(name, "batch_sequences_with_states", tensor_list) as name: - if pad: - length, input_sequences = _padding(input_sequences, num_unroll) - input_length = input_length if input_length is not None else length - elif input_sequences: - # Assert that value_length is a multiple of num_unroll. - checked_input_sequences = {} - for key, value in input_sequences.items(): - if (isinstance(value, sparse_tensor.SparseTensor) or - isinstance(value, sparse_tensor.SparseTensorValue)): - value_length = value.dense_shape[0] - with ops.control_dependencies([ - control_flow_ops.Assert( - math_ops.logical_and( - math_ops.equal(value_length % num_unroll, 0), - math_ops.not_equal(value_length, 0)), - [ - string_ops.string_join([ - "SparseTensor %s first dimension should be a " - "multiple of: " % key, - string_ops.as_string(num_unroll), - ", but saw value: ", - string_ops.as_string(value_length), - ". Consider setting pad=True."])])]): - checked_input_sequences[key] = sparse_tensor.SparseTensor( - indices=array_ops.identity( - value.indices, name="multiple_of_checked"), - values=array_ops.identity( - value.values, name="multiple_of_checked"), - dense_shape=array_ops.identity( - value.dense_shape, name="multiple_of_checked")) - else: - if not isinstance(value, ops.Tensor): - try: - value = ops.convert_to_tensor(value) - except TypeError: - raise TypeError( - "Unsupported input_sequences expected Tensor or SparseTensor " - "values, got: %s for key %s" % (str(type(value)), key)) - value_length = array_ops.shape(value)[0] - with ops.control_dependencies([ - control_flow_ops.Assert( - math_ops.logical_and( - math_ops.equal(value_length % num_unroll, 0), - math_ops.not_equal(value_length, 0)), - [ - string_ops.string_join([ - "Tensor %s first dimension should be a multiple " - "of: " % key, - string_ops.as_string(num_unroll), - ", but saw value: ", - string_ops.as_string(value_length), - ". Consider setting pad=True." - ]) - ]) - ]): - checked_input_sequences[key] = array_ops.identity( - value, name="multiple_of_checked") - input_sequences = checked_input_sequences - # Move SparseTensors in context into input_sequences. - _move_sparse_tensor_out_context(input_context, input_sequences, num_unroll) - # Deconstruct SparseTensors in sequence into a dense Tensor before inputting - # to SQSS. - (transformed_input_seq, - sparse_tensor_keys, - tensor_list) = _deconstruct_sparse_tensor_seq(input_sequences) - - if make_keys_unique: - input_key = string_ops.string_join([ - input_key, - string_ops.as_string( - random_ops.random_uniform( - (), minval=0, maxval=100000000, dtype=dtypes.int32, - seed=make_keys_unique_seed))]) - - # setup stateful queue reader - stateful_reader = SequenceQueueingStateSaver( - batch_size, - num_unroll, - input_length=input_length, - input_key=input_key, - input_sequences=transformed_input_seq, - input_context=input_context, - initial_states=initial_states, - capacity=capacity, - allow_small_batch=allow_small_batch) - - barrier = stateful_reader.barrier - summary.scalar("queue/%s/ready_segment_batches_" % barrier.name, - math_ops.cast(barrier.ready_size(), dtypes.float32)) - - q_runner = queue_runner.QueueRunner( - stateful_reader, [stateful_reader.prefetch_op] * num_threads, - queue_closed_exception_types=(errors.OutOfRangeError, - errors.CancelledError)) - queue_runner.add_queue_runner(q_runner) - batch = stateful_reader.next_batch - - # Reconstruct SparseTensors in sequence. - _reconstruct_sparse_tensor_seq( - batch.sequences, - sparse_tensor_keys, - tensor_list, - batch_size, - num_unroll) - # Move select SparseTensors back to context. - _move_sparse_tensor_in_context(batch.context, batch.sequences) - return batch - - -def _padding(sequences, num_unroll): - """For a dictionary of sequences, pads tensors to a multiple of `num_unroll`. - - Args: - sequences: dictionary with `Tensor` values. - num_unroll: int specifying to what multiple to pad sequences to. - Returns: - length: Scalar `Tensor` of dimension 0 of all the values in sequences. - padded_sequence: Dictionary of sequences that are padded to a multiple of - `num_unroll`. - Raises: - ValueError: If `num_unroll` not an int or sequences not a dictionary from - string to `Tensor`. - """ - if not isinstance(num_unroll, numbers.Integral): - raise ValueError("Unsupported num_unroll expected int, got: %s" % - str(num_unroll)) - if not isinstance(sequences, dict): - raise TypeError("Unsupported sequences expected dict, got: %s" % - str(sequences)) - for key, value in sequences.items(): - if not isinstance(key, six.string_types): - raise TypeError("Unsupported sequences key expected string, got: %s" % - str(key)) - if not sequences: - return 0, {} - - # Sort 'sequences_dict' so 'length' will have a predictable value below. - sequences_dict = collections.OrderedDict() - for key, value in sorted(sequences.items()): - if not (isinstance(value, sparse_tensor.SparseTensor) or - isinstance(value, sparse_tensor.SparseTensorValue)): - sequences_dict[key] = ops.convert_to_tensor(value) - else: - sequences_dict[key] = value - - lengths = [array_ops.shape(value)[0] for value in sequences_dict.values() - if isinstance(value, ops.Tensor)] - if lengths: - length = lengths[0] - all_lengths_equal = [ - control_flow_ops.Assert( - math_ops.equal(l, length), [string_ops.string_join( - ["All sequence lengths must match, but received lengths: ", - string_ops.as_string(lengths)])]) - for l in lengths] - length = control_flow_ops.with_dependencies(all_lengths_equal, length) - else: # Only have SparseTensors - sparse_lengths = [value.dense_shape[0] for value in sequences_dict.values() - if isinstance(value, sparse_tensor.SparseTensor)] - length = math_ops.reduce_max(math_ops.cast(sparse_lengths, dtypes.int32)) - - unroll = array_ops.constant(num_unroll) - padded_length = length + ((unroll - (length % unroll)) % unroll) - padded_sequences = {} - for key, value in sequences_dict.items(): - if isinstance(value, ops.Tensor): - # 1. create shape of paddings - # first dimension of value will be increased by num_paddings to - # padded_length - num_paddings = [padded_length - array_ops.shape(value)[0]] - # the shape of the paddings that we concat with the original value will be - # [num_paddings, tf.shape(value)[1], tf.shape(value)[2], ..., - # tf.shape(value)[tf.rank(value) - 1])] - padding_shape = array_ops.concat( - (num_paddings, array_ops.shape(value)[1:]), 0) - # 2. fill padding shape with dummies - dummy = array_ops.constant( - "" if value.dtype == dtypes.string else 0, dtype=value.dtype) - paddings = array_ops.fill(dims=padding_shape, value=dummy) - # 3. concat values with paddings - padded_sequences[key] = array_ops.concat([value, paddings], 0) - else: - padded_shape = array_ops.concat( - [[math_ops.cast(padded_length, dtypes.int64)], value.dense_shape[1:]], - 0) - padded_sequences[key] = sparse_tensor.SparseTensor( - indices=value.indices, - values=value.values, - dense_shape=padded_shape) - return length, padded_sequences - - -_SPARSE_CONTEXT_PREFIX_KEY = "_context_in_seq_" - - -def _move_sparse_tensor_out_context(input_context, input_sequences, num_unroll): - """Moves `SparseTensor`s from `input_context` into `input_sequences` as seq. - - For `key, value` pairs in `input_context` with `SparseTensor` `value` removes - them from `input_context` and transforms the `value` into a sequence and - then adding `key`, transformed `value` into `input_sequences`. - The transformation is done by adding a new first dimension of `value_length` - equal to that of the other values in input_sequences` and tiling the `value` - every `num_unroll` steps. - - Args: - input_context: dictionary with `Tensor` or `SparseTensor` values. To be - modified to take out `SparseTensor` values. - input_sequences: dictionary with `Tensor` or `SparseTensor` values. To be - modified to add transformed `SparseTensor` values from `input_context`. - num_unroll: int specifying to what multiple to pad sequences to. - """ - value_length = array_ops.constant(1) - if input_sequences: - seq = list(input_sequences.values())[0] - if isinstance(seq, ops.Tensor): - with ops.control_dependencies([seq]): - value_length = array_ops.shape(seq)[0] - else: - value_length = seq.dense_shape[0] - value_length = math_ops.cast(value_length, dtype=dtypes.int64) - def _copy_sparse_tensor(sp_tensor): - """Operation to tile a sparse tensor along a newly added 0 dimension. - - Adding a new first dimension of `value_length` and tiling the `sp_tensor` - every `num_unroll` steps. - - Args: - sp_tensor: `SparseTensor`. - Returns: - `SparseTensor` sequence with `sp_tensor` tiled. - """ - n = value_length // num_unroll - n = math_ops.cast(n, dtype=dtypes.int32) - values = array_ops.tile(sp_tensor.values, array_ops.expand_dims(n, 0)) - shape = array_ops.concat( - [array_ops.expand_dims(value_length, 0), sp_tensor.dense_shape], 0) - - # Construct new indices by multiplying old ones and prepending [0, n). - # First multiply indices n times along a newly created 0-dimension. - multiplied_indices = array_ops.tile( - array_ops.expand_dims(sp_tensor.indices, 0), - array_ops.stack([n, 1, 1])) - - # Construct indicator for [0, n). - # [ [ [0] [0] ... [0] ] - # [ [num_unroll] [num_unroll] ... [num_unroll] ] - # ... - # [ [num_unroll*(n-1)] [num_unroll*(n-1)] ... [num_unroll*(n-1)] ] ] - # of shape [n, shape(sp_tensor.indices)[0], 1] - # Get current dimensions of indices. - dim0 = array_ops.shape(sp_tensor.indices)[0] - dim1 = array_ops.shape(sp_tensor.indices)[1] - ind = math_ops.range(start=0, limit=value_length, delta=num_unroll) - - # ind.set_shape([n]) - ind = array_ops.expand_dims(ind, 1) - ind = array_ops.expand_dims(ind, 2) - ind = array_ops.tile(ind, [1, dim0, 1]) - - # Concatenate both and reshape. - indices = array_ops.concat([ind, multiplied_indices], 2) - indices = array_ops.reshape(indices, [dim0 * n, dim1 + 1]) - - return sparse_tensor.SparseTensor(indices=indices, - values=values, - dense_shape=shape) - - sparse_tensor_keys = [ - k for k in sorted(input_context.keys()) - if (isinstance(input_context[k], sparse_tensor.SparseTensor) or - isinstance(input_context[k], sparse_tensor.SparseTensorValue))] - for key in sparse_tensor_keys: - input_sequences[_SPARSE_CONTEXT_PREFIX_KEY + key] = _copy_sparse_tensor( - input_context[key]) - del input_context[key] - - -def _move_sparse_tensor_in_context(context, sequences): - sparse_tensor_keys = [ - k for k in sorted(sequences) if k.startswith(_SPARSE_CONTEXT_PREFIX_KEY)] - for key in sparse_tensor_keys: - new_key = key[len(_SPARSE_CONTEXT_PREFIX_KEY):] - sp_tensor = sequences[key] - # Take out time dimension. - sp_tensor = sparse_tensor.SparseTensor( - sp_tensor.indices, # with only 0s at column 1 representing time. - sp_tensor.values, - array_ops.concat( - [[sp_tensor.dense_shape[0]], # batch - [1], # time - sp_tensor.dense_shape[2:]], # SparseTensor shape prior to batching - 0)) - new_shape = array_ops.concat( - [[sp_tensor.dense_shape[0]], sp_tensor.dense_shape[2:]], 0) - context[new_key] = sparse_ops.sparse_reshape(sp_tensor, new_shape) - del sequences[key] - - -def _deconstruct_sparse_tensor_seq(input_sequence, shared_name=None): - """Converts `SparseTensor` values into `Tensors` of IDs and meta data. - - Given a dict of keys -> `Tensor` or `SparseTensor` transforms the - `SparseTensor` values into `Tensor` values of IDs by calling `_store_sparse`. - The IDs are pointers into and underlying `SparseTensorsMap` that is being - constructed. Additional meta data is returned in order to be able to - reconstruct `SparseTensor` values after batching and segmenting the IDs - `Tensor`. - - Args: - input_sequence: dictionary with `Tensor` or `SparseTensor` values. - shared_name: The shared name for the underlying `SparseTensorsMap` - (optional, defaults to the name of the newly created op). - Returns: - A tuple `(sequence, sparse_tensor_keys, tensor_list)` where `sequence` is - dictionary with the same keys as `input_sequence` but only `Tensor` values, - `sparse_tensor_keys` is a list of the keys of the `SparseTensor` values that - were converted, and `tensor_list` is a list of the same length with - `Tensor` objects. - """ - sparse_tensor_keys = [ - k for k in sorted(input_sequence.keys()) - if (isinstance(input_sequence[k], sparse_tensor.SparseTensor) or - isinstance(input_sequence[k], sparse_tensor.SparseTensorValue))] - if not sparse_tensor_keys: - return input_sequence, None, sparse_tensor_keys - sparse_tensor_list = [input_sequence[k] for k in sparse_tensor_keys] - tensor_list = [_store_sparse(sp_tensor, shared_name=shared_name) - for sp_tensor in sparse_tensor_list] - transformed_input_seq = dict(input_sequence) - tensor_op_list = [] - for i, k in enumerate(sparse_tensor_keys): - transformed_input_seq[k] = tensor_list[i] - tensor_op_list += [tensor_list[i].op] - return transformed_input_seq, sparse_tensor_keys, tensor_op_list - - -def _reconstruct_sparse_tensor_seq(sequence, - sparse_tensor_keys, - tensor_op_list, - batch_size, - num_unroll): - """Inverse of _deconstruct_sparse_tensor_seq. - - Given a dict of keys -> `Tensor` reconstructs `SparseTensor` values for keys - in `sparse_tensor_keys`. Their `Tensor` values are assumed to be IDs into the - underlying `SparseTensorsMap`. The `dense_shape` of the `SparseTensor`s is - `[batch_size, num_unroll, d_0, d_1, ..., d_n]` when the original - `SparseTensor` that got deconstructed with `_deconstruct_sparse_tensor_seq` - has a `dense_shape` of `[None, d_0, d_1, ..., d_n]`. - - Args: - sequence: dictionary with only `Tensor` values that is being updated. - sparse_tensor_keys: list of the keys present in `sequence` identifying - `SparseTensor` values that should be reconstructed. - tensor_op_list: list of the same length as `sparse_tensor_keys` with - `Tensor` objects. - batch_size: int or int32 scalar `Tensor`, how large minibatches should - be. - num_unroll: Python integer, how many time steps were unrolled at a time. - """ - def _flatten_tensor(tensor): - """Flattens `Tensor` of `shape [batch_size, num_unroll]` into 1D `Tensor`. - - The main use of this function is to work around the limitation of - `_restore_sparse` to only accept 1D handles. - - Args: - tensor: 2D `Tensor` of `shape [batch_size, num_unroll]` - Returns: - 1D `Tensor`. - """ - return array_ops.reshape(tensor, [-1]) - - def _unflatten_sparse_tensor(sp_tensor): - """Recreates `[batch_size, num_unroll]` dimensions in the `SparseTensor`. - - Counter-part of `_flatten_tensor` which is called on the input of - `_restore_sparse` while this method is called on the output of it. - Together they work around the limitation of `_restore_sparse` to only - accept 1D handles. - - The `indices` in `sp_tensor` is a 2D `Tensor` of `shape [N, ndims]`, where - `N` is the number of `values` and `ndims` is the number of dimension in its - dense counterpart. Among `ndims` the first entry corresponds to the batch - dimension `[0, num_unroll * batch_size)` from which we need to recreate the - 2 dimensions `batch_size` and `num_unroll`. - - The reason this reconstruction works is because the output of - `_restore_sparse` despite being a `SparseTensor` is actually dense w.r.t. - that first entry. - - Args: - sp_tensor: A SparseTensor. - Returns: - A SparseTensor with a +1 higher rank than the input. - """ - idx_batch = math_ops.cast( - math_ops.floor(sp_tensor.indices[:, 0] / num_unroll), dtypes.int64) - idx_time = math_ops.mod(sp_tensor.indices[:, 0], num_unroll) - indices = array_ops.concat( - [ - array_ops.expand_dims(idx_batch, 1), - array_ops.expand_dims(idx_time, 1), sp_tensor.indices[:, 1:] - ], - axis=1) - dense_shape = array_ops.concat( - [[math_ops.cast(batch_size, dtype=dtypes.int64)], - [math_ops.cast(num_unroll, dtype=dtypes.int64)], - sp_tensor.dense_shape[1:]], axis=0) - return sparse_tensor.SparseTensor( - indices=indices, - values=sp_tensor.values, - dense_shape=dense_shape) - - if not sparse_tensor_keys: - return - tensor_list = [sequence[k] for k in sparse_tensor_keys] - sp_tensors = [ - _restore_sparse(sparse_map_op=i, - # Flatten the 2D Tensor [batch_size, num_unroll] of - # handles to a 1D Tensor. - # Reconstruct the dimensions later. - # TODO(b/34247140): Remove this workaround. - sparse_handles=_flatten_tensor(s), rank=None) - for i, s in zip(tensor_op_list, tensor_list)] - num_unroll = ops.convert_to_tensor(num_unroll, dtype=dtypes.int64, - name="num_unroll_int64") - - # Recreate the [batch_size, num_unroll] dimensions in the SparseTensors. - # The dense_shape will have a +1 higher rank. - # TODO(b/34247140): Remove this workaround. - sp_tensors_higher_dim = [_unflatten_sparse_tensor(s) for s in sp_tensors] - - # Set values to SparseTensors for sparse_tensor_keys. - for i, key in enumerate(sparse_tensor_keys): - sequence[key] = sp_tensors_higher_dim[i] - return diff --git a/tensorflow/contrib/training/python/training/sequence_queueing_state_saver_test.py b/tensorflow/contrib/training/python/training/sequence_queueing_state_saver_test.py deleted file mode 100644 index 15dc1622054..00000000000 --- a/tensorflow/contrib/training/python/training/sequence_queueing_state_saver_test.py +++ /dev/null @@ -1,667 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.SequenceQueueingStateSaver.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time - -import numpy as np - -from tensorflow.contrib.training.python.training import sequence_queueing_state_saver as sqss -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops -from tensorflow.python.ops import string_ops -from tensorflow.python.platform import test - - -class SequenceQueueingStateSaverTest(test.TestCase): - - def testSequenceInputWrapper(self): - with self.cached_session(): - length = 3 - key = "key" - padded_length = 4 - sequences = { - "seq1": np.random.rand(padded_length, 5), - "seq2": np.random.rand(padded_length, 4, 2) - } - context = {"context1": [3, 4]} - input_wrapper = sqss._SequenceInputWrapper(length, key, sequences, - context) - self.assertTrue(isinstance(input_wrapper.length, ops.Tensor)) - self.assertTrue(isinstance(input_wrapper.key, ops.Tensor)) - self.assertTrue(isinstance(input_wrapper.sequences["seq1"], ops.Tensor)) - self.assertTrue(isinstance(input_wrapper.sequences["seq2"], ops.Tensor)) - self.assertTrue(isinstance(input_wrapper.context["context1"], ops.Tensor)) - - def testStateSaverWithTwoSimpleSteps(self): - with self.cached_session() as sess: - batch_size_value = 2 - batch_size = constant_op.constant(batch_size_value) - num_unroll = 2 - length = 3 - key = string_ops.string_join([ - "key_", string_ops.as_string( - math_ops.cast(10000 * random_ops.random_uniform(()), - dtypes.int32)) - ]) - padded_length = 4 - sequences = { - "seq1": np.random.rand(padded_length, 5), - "seq2": np.random.rand(padded_length, 4, 2) - } - context = {"context1": [3, 4]} - initial_states = { - "state1": np.random.rand(6, 7), - "state2": np.random.rand(8) - } - state_saver = sqss.SequenceQueueingStateSaver( - batch_size=batch_size, - num_unroll=num_unroll, - input_length=length, - input_key=key, - input_sequences=sequences, - input_context=context, - initial_states=initial_states, - capacity=100) - - initial_key_value_0, _ = sess.run((key, state_saver.prefetch_op)) - initial_key_value_1, _ = sess.run((key, state_saver.prefetch_op)) - - initial_key_value_0 = initial_key_value_0.decode("ascii") - initial_key_value_1 = initial_key_value_1.decode("ascii") - - # Step 1 - next_batch = state_saver.next_batch - (key_value, next_key_value, seq1_value, seq2_value, context1_value, - state1_value, state2_value, length_value, _, _) = sess.run( - (next_batch.key, next_batch.next_key, next_batch.sequences["seq1"], - next_batch.sequences["seq2"], next_batch.context["context1"], - next_batch.state("state1"), next_batch.state("state2"), - next_batch.length, - next_batch.save_state("state1", next_batch.state("state1") + 1), - next_batch.save_state("state2", next_batch.state("state2") - 1))) - - expected_first_keys = set( - ("00000_of_00002:%s" % x).encode("ascii") - for x in (initial_key_value_0, initial_key_value_1)) - expected_second_keys = set( - ("00001_of_00002:%s" % x).encode("ascii") - for x in (initial_key_value_0, initial_key_value_1)) - expected_final_keys = set( - ("STOP:%s" % x).encode("ascii") - for x in (initial_key_value_0, initial_key_value_1)) - - self.assertEqual(set(key_value), expected_first_keys) - self.assertEqual(set(next_key_value), expected_second_keys) - self.assertAllEqual(context1_value, - np.tile(context["context1"], (batch_size_value, 1))) - self.assertAllEqual(seq1_value, - np.tile(sequences["seq1"][np.newaxis, 0:2, :], - (batch_size_value, 1, 1))) - self.assertAllEqual(seq2_value, - np.tile(sequences["seq2"][np.newaxis, 0:2, :, :], - (batch_size_value, 1, 1, 1))) - self.assertAllEqual(state1_value, - np.tile(initial_states["state1"], - (batch_size_value, 1, 1))) - self.assertAllEqual(state2_value, - np.tile(initial_states["state2"], - (batch_size_value, 1))) - self.assertAllEqual(length_value, [2, 2]) - - # Step 2 - (key_value, next_key_value, seq1_value, seq2_value, context1_value, - state1_value, state2_value, length_value, _, _) = sess.run( - (next_batch.key, next_batch.next_key, next_batch.sequences["seq1"], - next_batch.sequences["seq2"], next_batch.context["context1"], - next_batch.state("state1"), next_batch.state("state2"), - next_batch.length, - next_batch.save_state("state1", next_batch.state("state1") + 1), - next_batch.save_state("state2", next_batch.state("state2") - 1))) - - self.assertEqual(set(key_value), expected_second_keys) - self.assertEqual(set(next_key_value), expected_final_keys) - self.assertAllEqual(context1_value, - np.tile(context["context1"], (batch_size_value, 1))) - self.assertAllEqual(seq1_value, - np.tile(sequences["seq1"][np.newaxis, 2:4, :], - (batch_size_value, 1, 1))) - self.assertAllEqual(seq2_value, - np.tile(sequences["seq2"][np.newaxis, 2:4, :, :], - (batch_size_value, 1, 1, 1))) - self.assertAllEqual(state1_value, 1 + np.tile(initial_states["state1"], - (batch_size_value, 1, 1))) - self.assertAllEqual(state2_value, -1 + np.tile(initial_states["state2"], - (batch_size_value, 1))) - self.assertAllEqual(length_value, [1, 1]) - - # Finished. Let's make sure there's nothing left in the barrier. - self.assertEqual(0, state_saver.barrier.ready_size().eval()) - - def testStateSaverFailsIfPaddedLengthIsNotMultipleOfNumUnroll(self): - with self.cached_session() as sess: - batch_size = constant_op.constant(32) - num_unroll = 17 - bad_padded_length = 3 - length = array_ops.placeholder(dtypes.int32) - key = array_ops.placeholder(dtypes.string) - sequences = { - "seq1": array_ops.placeholder( - dtypes.float32, shape=(None, 5)) - } - context = {} - initial_states = { - "state1": array_ops.placeholder( - dtypes.float32, shape=()) - } - state_saver = sqss.SequenceQueueingStateSaver( - batch_size=batch_size, - num_unroll=num_unroll, - input_length=length, - input_key=key, - input_sequences=sequences, - input_context=context, - initial_states=initial_states) - - with self.assertRaisesOpError( - "should be a multiple of: 17, but saw value: %d" % bad_padded_length): - sess.run([state_saver.prefetch_op], - feed_dict={ - length: 1, - key: "key", - sequences["seq1"]: np.random.rand(bad_padded_length, 5), - initial_states["state1"]: 1.0 - }) - - def _testStateSaverFailsIfCapacityTooSmall(self, batch_size): - with self.cached_session() as sess: - num_unroll = 2 - length = array_ops.placeholder(dtypes.int32) - key = array_ops.placeholder(dtypes.string) - sequences = { - "seq1": array_ops.placeholder( - dtypes.float32, shape=(None, 5)), - "seq2": array_ops.placeholder( - dtypes.float32, shape=(None,)) - } - context = {} - initial_states = { - "state1": array_ops.placeholder( - dtypes.float32, shape=()) - } - state_saver = sqss.SequenceQueueingStateSaver( - batch_size=batch_size, - num_unroll=num_unroll, - input_length=length, - input_key=key, - input_sequences=sequences, - input_context=context, - initial_states=initial_states, - capacity=10) - - sess.run([state_saver.prefetch_op], - feed_dict={ - length: 1, - key: "key", - sequences["seq1"]: np.random.rand(num_unroll, 5), - sequences["seq2"]: np.random.rand(num_unroll), - initial_states["state1"]: 1.0 - }) - - def testStateSaverFailsIfCapacityTooSmallTensor(self): - batch_size_value = 32 - batch_size = constant_op.constant(batch_size_value) - with self.assertRaisesOpError( - ".*capacity needs to be >= batch_size.*"): - self._testStateSaverFailsIfCapacityTooSmall(batch_size) - - def testStateSaverFailsIfCapacityTooSmallInt(self): - batch_size = 32 - with self.assertRaisesRegexp( - ValueError, - "capacity %d needs to be >= batch_size %d" % (10, batch_size)): - self._testStateSaverFailsIfCapacityTooSmall(batch_size) - - def testStateSaverFailsIfInconsistentPaddedLength(self): - with self.cached_session() as sess: - batch_size = constant_op.constant(32) - num_unroll = 17 - length = array_ops.placeholder(dtypes.int32) - key = array_ops.placeholder(dtypes.string) - sequences = { - "seq1": array_ops.placeholder( - dtypes.float32, shape=(None, 5)), - "seq2": array_ops.placeholder( - dtypes.float32, shape=(None,)) - } - context = {} - initial_states = { - "state1": array_ops.placeholder( - dtypes.float32, shape=()) - } - state_saver = sqss.SequenceQueueingStateSaver( - batch_size=batch_size, - num_unroll=num_unroll, - input_length=length, - input_key=key, - input_sequences=sequences, - input_context=context, - initial_states=initial_states) - - with self.assertRaisesOpError( - "Dimension 0 of tensor labeled sorted_sequences_seq2 " - "should be: %d, shape received: %d" % (num_unroll, 2 * num_unroll)): - sess.run([state_saver.prefetch_op], - feed_dict={ - length: 1, - key: "key", - sequences["seq1"]: np.random.rand(num_unroll, 5), - sequences["seq2"]: np.random.rand(2 * num_unroll), - initial_states["state1"]: 1.0 - }) - - def testStateSaverFailsIfInconsistentWriteState(self): - # TODO(b/26910386): Identify why this infrequently causes timeouts. - with self.cached_session() as sess: - batch_size = constant_op.constant(1) - num_unroll = 17 - length = array_ops.placeholder(dtypes.int32) - key = array_ops.placeholder(dtypes.string) - sequences = { - "seq1": array_ops.placeholder( - dtypes.float32, shape=(None, 5)) - } - context = {} - initial_states = { - "state1": array_ops.placeholder( - dtypes.float32, shape=()) - } - state_saver = sqss.SequenceQueueingStateSaver( - batch_size=batch_size, - num_unroll=num_unroll, - input_length=length, - input_key=key, - input_sequences=sequences, - input_context=context, - initial_states=initial_states) - next_batch = state_saver.next_batch - with self.assertRaisesRegexp(KeyError, "state was not declared: state2"): - save_op = next_batch.save_state("state2", None) - with self.assertRaisesRegexp(ValueError, "Rank check failed for.*state1"): - save_op = next_batch.save_state("state1", np.random.rand(1, 1)) - with self.assertRaisesOpError( - r"convert_state1:0 should be: 1, shape received:\] \[1 1\]"): - state_input = array_ops.placeholder(dtypes.float32) - with ops.control_dependencies([state_saver.prefetch_op]): - save_op = next_batch.save_state("state1", state_input) - sess.run([save_op], - feed_dict={ - length: 1, - key: "key", - sequences["seq1"]: np.random.rand(num_unroll, 5), - initial_states["state1"]: 1.0, - state_input: np.random.rand(1, 1) - }) - - def testStateSaverWithManyInputsReadWriteThread(self): - batch_size_value = 32 - num_proc_threads = 100 - with self.cached_session() as sess: - batch_size = constant_op.constant(batch_size_value) - num_unroll = 17 - length = array_ops.placeholder(dtypes.int32) - key = array_ops.placeholder(dtypes.string) - sequences = { - "seq1": array_ops.placeholder( - dtypes.float32, shape=(None, 5)), - "seq2": array_ops.placeholder( - dtypes.float32, shape=(None, 4, 2)), - "seq3": array_ops.placeholder( - dtypes.float64, shape=(None,)) - } - context = { - "context1": array_ops.placeholder( - dtypes.string, shape=(3, 4)), - "context2": array_ops.placeholder( - dtypes.int64, shape=()) - } - initial_states = { - "state1": array_ops.placeholder( - dtypes.float32, shape=(6, 7)), - "state2": array_ops.placeholder( - dtypes.int32, shape=()) - } - state_saver = sqss.SequenceQueueingStateSaver( - batch_size=batch_size, - num_unroll=num_unroll, - input_length=length, - input_key=key, - input_sequences=sequences, - input_context=context, - initial_states=initial_states) - next_batch = state_saver.next_batch - cancel_op = state_saver.close(cancel_pending_enqueues=True) - - update_1 = next_batch.save_state("state1", 1 + next_batch.state("state1")) - update_2 = next_batch.save_state("state2", - -1 + next_batch.state("state2")) - - original_values = {} - - def insert(which): - for i in range(20): - # Insert varying length inputs - pad_i = num_unroll * (1 + (i % 10)) - length_i = int(np.random.rand() * pad_i) - key_value = "key_%02d_%04d" % (which, i) - stored_state = { - "length": length_i, - "seq1": np.random.rand(pad_i, 5), - "seq2": np.random.rand(pad_i, 4, 2), - "seq3": np.random.rand(pad_i), - "context1": np.random.rand(3, 4).astype(np.str), - "context2": np.asarray( - 100 * np.random.rand(), dtype=np.int32), - "state1": np.random.rand(6, 7), - "state2": np.asarray( - 100 * np.random.rand(), dtype=np.int32) - } - original_values[key_value] = stored_state - sess.run([state_saver.prefetch_op], - feed_dict={ - length: stored_state["length"], - key: key_value, - sequences["seq1"]: stored_state["seq1"], - sequences["seq2"]: stored_state["seq2"], - sequences["seq3"]: stored_state["seq3"], - context["context1"]: stored_state["context1"], - context["context2"]: stored_state["context2"], - initial_states["state1"]: stored_state["state1"], - initial_states["state2"]: stored_state["state2"] - }) - - processed_count = [0] - - def process_and_check_state(): - next_batch = state_saver.next_batch - while True: - try: - (got_key, next_key, length, total_length, sequence, sequence_count, - context1, context2, seq1, seq2, seq3, state1, state2, _, - _) = (sess.run([ - next_batch.key, next_batch.next_key, next_batch.length, - next_batch.total_length, next_batch.sequence, - next_batch.sequence_count, next_batch.context["context1"], - next_batch.context["context2"], next_batch.sequences["seq1"], - next_batch.sequences["seq2"], next_batch.sequences["seq3"], - next_batch.state("state1"), next_batch.state("state2"), - update_1, update_2 - ])) - - except errors_impl.OutOfRangeError: - # SQSS has been closed - break - - self.assertEqual(len(got_key), batch_size_value) - - processed_count[0] += len(got_key) - - for i in range(batch_size_value): - key_name = got_key[i].decode("ascii").split(":")[1] - # We really saved this unique key - self.assertTrue(key_name in original_values) - # The unique key matches next_key - self.assertEqual(key_name, - next_key[i].decode("ascii").split(":")[1]) - # Pull out the random values we used to create this example - stored_state = original_values[key_name] - self.assertEqual(total_length[i], stored_state["length"]) - self.assertEqual("%05d_of_%05d:%s" % - (sequence[i], sequence_count[i], key_name), - got_key[i].decode("ascii")) - expected_length = max( - 0, - min(num_unroll, - stored_state["length"] - sequence[i] * num_unroll)) - self.assertEqual(length[i], expected_length) - expected_state1 = stored_state["state1"] + sequence[i] - expected_state2 = stored_state["state2"] - sequence[i] - expected_sequence1 = stored_state["seq1"][sequence[i] * num_unroll:( - sequence[i] + 1) * num_unroll] - expected_sequence2 = stored_state["seq2"][sequence[i] * num_unroll:( - sequence[i] + 1) * num_unroll] - expected_sequence3 = stored_state["seq3"][sequence[i] * num_unroll:( - sequence[i] + 1) * num_unroll] - - self.assertAllClose(state1[i], expected_state1) - self.assertAllEqual(state2[i], expected_state2) - # context1 is strings, which come back as bytes - self.assertAllEqual(context1[i].astype(np.str), - stored_state["context1"]) - self.assertAllEqual(context2[i], stored_state["context2"]) - self.assertAllClose(seq1[i], expected_sequence1) - self.assertAllClose(seq2[i], expected_sequence2) - self.assertAllClose(seq3[i], expected_sequence3) - - # Total number of inserts will be a multiple of batch_size - insert_threads = [ - self.checkedThread( - insert, args=(which,)) for which in range(batch_size_value) - ] - process_threads = [ - self.checkedThread(process_and_check_state) - for _ in range(num_proc_threads) - ] - - for t in insert_threads: - t.start() - for t in process_threads: - t.start() - for t in insert_threads: - t.join() - - time.sleep(3) # Allow the threads to run and process for a while - cancel_op.run() - - for t in process_threads: - t.join() - - # Each thread processed at least 2 sequence segments - self.assertGreater(processed_count[0], 2 * 20 * batch_size_value) - - def testStateSaverProcessesExamplesInOrder(self): - with self.cached_session() as sess: - batch_size_value = 32 - batch_size = constant_op.constant(batch_size_value) - num_unroll = 17 - length = array_ops.placeholder(dtypes.int32) - key = array_ops.placeholder(dtypes.string) - sequences = { - "seq1": array_ops.placeholder( - dtypes.float32, shape=(None, 5)) - } - context = {"context1": array_ops.placeholder(dtypes.string, shape=(3, 4))} - initial_states = { - "state1": array_ops.placeholder( - dtypes.float32, shape=()) - } - state_saver = sqss.SequenceQueueingStateSaver( - batch_size=batch_size, - num_unroll=num_unroll, - input_length=length, - input_key=key, - input_sequences=sequences, - input_context=context, - initial_states=initial_states) - next_batch = state_saver.next_batch - - update = next_batch.save_state("state1", 1 + next_batch.state("state1")) - get_ready_size = state_saver.barrier.ready_size() - get_incomplete_size = state_saver.barrier.incomplete_size() - - global_insert_key = [0] - - def insert(insert_key): - # Insert varying length inputs - sess.run([state_saver.prefetch_op], - feed_dict={ - length: np.random.randint(2 * num_unroll), - key: "%05d" % insert_key[0], - sequences["seq1"]: np.random.rand(2 * num_unroll, 5), - context["context1"]: np.random.rand(3, 4).astype(np.str), - initial_states["state1"]: 0.0 - }) - insert_key[0] += 1 - - for _ in range(batch_size_value * 100): - insert(global_insert_key) - - def process_and_validate(check_key): - true_step = int(check_key[0] / 2) # Each entry has two slices - check_key[0] += 1 - got_keys, input_index, _ = sess.run( - [next_batch.key, next_batch.insertion_index, update]) - decoded_keys = [int(x.decode("ascii").split(":")[-1]) for x in got_keys] - min_key = min(decoded_keys) - min_index = int(min(input_index)) # numpy scalar - max_key = max(decoded_keys) - max_index = int(max(input_index)) # numpy scalar - # The current min key should be above the previous min - self.assertEqual(min_key, true_step * batch_size_value) - self.assertEqual(max_key, (true_step + 1) * batch_size_value - 1) - self.assertEqual(2**63 + min_index, true_step * batch_size_value) - self.assertEqual(2**63 + max_index, - (true_step + 1) * batch_size_value - 1) - - # There are now (batch_size * 100 * 2) / batch_size = 200 full steps - global_step_key = [0] - for _ in range(200): - process_and_validate(global_step_key) - - # Processed everything in the queue - self.assertEqual(get_incomplete_size.eval(), 0) - self.assertEqual(get_ready_size.eval(), 0) - - def testStateSaverCanHandleVariableBatchsize(self): - with self.cached_session() as sess: - batch_size = array_ops.placeholder(dtypes.int32) - num_unroll = 17 - length = array_ops.placeholder(dtypes.int32) - key = array_ops.placeholder(dtypes.string) - sequences = { - "seq1": array_ops.placeholder( - dtypes.float32, shape=(None, 5)) - } - context = {"context1": array_ops.placeholder(dtypes.string, shape=(3, 4))} - initial_states = { - "state1": array_ops.placeholder( - dtypes.float32, shape=()) - } - state_saver = sqss.SequenceQueueingStateSaver( - batch_size=batch_size, - num_unroll=num_unroll, - input_length=length, - input_key=key, - input_sequences=sequences, - input_context=context, - initial_states=initial_states) - next_batch = state_saver.next_batch - - update = next_batch.save_state("state1", 1 + next_batch.state("state1")) - - for insert_key in range(128): - # Insert varying length inputs - sess.run([state_saver.prefetch_op], - feed_dict={ - length: np.random.randint(2 * num_unroll), - key: "%05d" % insert_key, - sequences["seq1"]: np.random.rand(2 * num_unroll, 5), - context["context1"]: np.random.rand(3, 4).astype(np.str), - initial_states["state1"]: 0.0 - }) - - all_received_indices = [] - # Pull out and validate batch sizes 0, 1, ..., 7 - for batch_size_value in range(8): - got_keys, input_index, context1, seq1, state1, _ = sess.run( - [ - next_batch.key, next_batch.insertion_index, - next_batch.context["context1"], next_batch.sequences["seq1"], - next_batch.state("state1"), update - ], - feed_dict={batch_size: batch_size_value}) - # Indices may have come in out of order within the batch - all_received_indices.append(input_index.tolist()) - self.assertEqual(got_keys.size, batch_size_value) - self.assertEqual(input_index.size, batch_size_value) - self.assertEqual(context1.shape, (batch_size_value, 3, 4)) - self.assertEqual(seq1.shape, (batch_size_value, num_unroll, 5)) - self.assertEqual(state1.shape, (batch_size_value,)) - - # Each input was split into 2 iterations (sequences size == 2*num_unroll) - expected_indices = [[], [0], [0, 1], [1, 2, 3], [2, 3, 4, 5], - [4, 5, 6, 7, 8], [6, 7, 8, 9, 10, 11], - [9, 10, 11, 12, 13, 14, 15]] - self.assertEqual(len(all_received_indices), len(expected_indices)) - for received, expected in zip(all_received_indices, expected_indices): - self.assertAllEqual([x + 2**63 for x in received], expected) - - def testStateSaverScopeNames(self): - batch_size = constant_op.constant(2) - sqss_scope_name = "unique_scope_name_for_sqss" - num_unroll = 2 - length = 3 - key = string_ops.string_join([ - "key_", string_ops.as_string( - math_ops.cast(10000 * random_ops.random_uniform(()), dtypes.int32)) - ]) - padded_length = 4 - sequences = { - "seq1": np.random.rand(padded_length, 5), - "seq2": np.random.rand(padded_length, 4, 2) - } - context = {"context1": [3, 4]} - initial_states = { - "state1": np.random.rand(6, 7), - "state2": np.random.rand(8) - } - state_saver = sqss.SequenceQueueingStateSaver( - batch_size=batch_size, - num_unroll=num_unroll, - input_length=length, - input_key=key, - input_sequences=sequences, - input_context=context, - initial_states=initial_states, - name=sqss_scope_name) - prefetch_op = state_saver.prefetch_op - next_batch = state_saver.next_batch - self.assertTrue( - state_saver.barrier.barrier_ref.name.startswith("%s/" % - sqss_scope_name)) - self.assertTrue(prefetch_op.name.startswith("%s/" % sqss_scope_name)) - self.assertTrue(next_batch.key.name.startswith("%s/" % sqss_scope_name)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/training/python/training/sgdr_learning_rate_decay.py b/tensorflow/contrib/training/python/training/sgdr_learning_rate_decay.py deleted file mode 100644 index 7f1cf3c8d33..00000000000 --- a/tensorflow/contrib/training/python/training/sgdr_learning_rate_decay.py +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""SGDR learning rate decay function.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.ops import math_ops, control_flow_ops - - -def sgdr_decay(learning_rate, global_step, initial_period_steps, - t_mul=2.0, m_mul=1.0, name=None): - """Implements Stochastic Gradient Descent with Warm Restarts (SGDR). - - As described in "SGDR: Stochastic Gradient Descent - with Warm Restarts" by Ilya Loshchilov & Frank Hutter, Proceedings of - ICLR'2017, available at https://arxiv.org/pdf/1608.03983.pdf - - The learning rate decreases according to cosine annealing: - - ```python - learning_rate * 0.5 * (1 + cos(x_val * pi)) # for x_val defined in [0, 1] - ``` - - Thus, at the beginning (when the restart index i = 0), - the learning rate decreases for `initial_period_steps` steps from the initial - learning rate `learning_rate` (when `x_val=0`, we get `cos(0)=1`) to - 0 (when `x_val=1`, we get `cos(pi)=-1`). - - The decrease within the i-th period takes `t_i` steps, - where `t_0` = `initial_period_steps` is the user-defined number of batch - iterations (not epochs as in the paper) to be performed before the first - restart is launched. - - Then, we perform the first restart (i=1) by setting the learning rate to - `learning_rate*(m_mul^i)`, where `m_mul in [0,1]` (set to 1 by default). - The i-th restart runs for `t_i=t_0*(t_mul^i)` steps, i.e., every new - restart runs `t_mul` times longer than the previous one. - - Importantly, when one has no access to a validation set, SGDR suggests - to report the best expected / recommended solution in the following way: - When we are within our initial run (i=0), every new solution represents - SGDR's recommended solution. Instead, when i>0, the recommended solution is - the one obtained at the end of each restart. - - Note that the minimum learning rate is set to 0 for simplicity, - you can adjust the code to deal with any positive minimum learning rate - as defined in the paper. - - `initial_period_steps` is the duration of the first period measured in terms - of number of minibatch updates. If one wants to use epochs, one should compute - the number of updates required for an epoch. - - For example, assume the following parameters and intention: - Minibatch size: 100 - Training dataset size: 10000 - If the user wants the first decay period to span across 5 epochs, then - `initial_period_steps` = 5 * 10000/100 = 500 - - Train for 10000 batch iterations with the initial learning rate set to - 0.1, then restart to run 2 times longer, i.e, for 20000 batch iterations - and with the initial learning rate 0.05, then restart again and again, - doubling the runtime of each new period and with two times smaller - initial learning rate. - - To accomplish the above, one would write: - - ```python - ... - global_step = tf.Variable(0, trainable=False) - starter_learning_rate = 0.1 - learning_rate = sgdr_decay(starter_learning_rate, global_step, - initial_period_steps=10000, t_mul=2, m_mul=0.5) - # Passing global_step to minimize() will increment it at each step. - learning_step = ( - tf.compat.v1.train.GradientDescentOptimizer(learning_rate) - .minimize(...my loss..., global_step=global_step) - ) - - # Step | 0 | 1000 | 5000 | 9000 | 9999 | 10000 | 11000 | - # LR | 0.1 | 0.097 | 0.05 | 0.002 | 0.00 | 0.05 | 0.0496 | - - # Step | 20000 | 29000 | 29999 | 30000 | - # LR | 0.025 | 0.0003 | 0.00 | 0.025 | - ``` - - Args: - learning_rate: A scalar `float32` or `float64` `Tensor` or a - Python number. The initial learning rate. - global_step: A scalar `int32` or `int64` `Tensor` or a Python number. - Global step to use for the decay computation. Must not be negative. - initial_period_steps: Duration of the first period measured as the number - of minibatch updates, if one wants to use epochs, one should compute - the number of updates required for an epoch. - t_mul: A scalar `float32` or `float64` `Tensor` or a Python number. - Must be positive. - Used to derive the number of iterations in the i-th period: - `initial_period_steps * (t_mul^i)`. Defaults to 2.0. - m_mul: A scalar `float32` or `float64` `Tensor` or a Python number. - Must be positive. - Used to derive the initial learning rate of the i-th period: - `learning_rate * (m_mul^i)`. Defaults to 1.0 - - Returns: - A scalar `Tensor` of the same type as `learning_rate`. - The learning rate for a provided global_step. - Raises: - ValueError: if `global_step` is not supplied. - """ - - if global_step is None: - raise ValueError("global_step is required for sgdr_decay.") - with ops.name_scope(name, "SGDRDecay", - [learning_rate, global_step, - initial_period_steps, t_mul, m_mul]) as name: - learning_rate = ops.convert_to_tensor(learning_rate, - name="initial_learning_rate") - dtype = learning_rate.dtype - global_step = math_ops.cast(global_step, dtype) - t_0 = math_ops.cast(initial_period_steps, dtype) - t_mul = math_ops.cast(t_mul, dtype) - m_mul = math_ops.cast(m_mul, dtype) - - c_one = math_ops.cast(constant_op.constant(1.0), dtype) - c_half = math_ops.cast(constant_op.constant(0.5), dtype) - c_pi = math_ops.cast(constant_op.constant(math.pi), dtype) - - # Find normalized value of the current step - x_val = math_ops.div(global_step, t_0) - - def compute_step(x_val, geometric=False): - if geometric: - # Consider geometric series where t_mul != 1 - # 1 + t_mul + t_mul^2 ... = (1 - t_mul^i_restart) / (1 - t_mul) - - # First find how many restarts were performed for a given x_val - # Find maximal integer i_restart value for which this equation holds - # x_val >= (1 - t_mul^i_restart) / (1 - t_mul) - # x_val * (1 - t_mul) <= (1 - t_mul^i_restart) - # t_mul^i_restart <= (1 - x_val * (1 - t_mul)) - - # tensorflow allows only log with base e - # i_restart <= log(1 - x_val * (1 - t_mul) / log(t_mul) - # Find how many restarts were performed - - i_restart = math_ops.floor( - math_ops.log(c_one - x_val * (c_one - t_mul)) / math_ops.log(t_mul)) - # Compute the sum of all restarts before the current one - sum_r = (c_one - t_mul ** i_restart) / (c_one - t_mul) - # Compute our position within the current restart - x_val = (x_val - sum_r) / t_mul ** i_restart - - else: - # Find how many restarts were performed - i_restart = math_ops.floor(x_val) - # Compute our position within the current restart - x_val = x_val - i_restart - return i_restart, x_val - - i_restart, x_val = control_flow_ops.cond( - math_ops.equal(t_mul, c_one), - lambda: compute_step(x_val, geometric=False), - lambda: compute_step(x_val, geometric=True)) - - # If m_mul < 1, then the initial learning rate of every new restart will be - # smaller, i.e., by a factor of m_mul ** i_restart at i_restart-th restart - m_fac = learning_rate * (m_mul ** i_restart) - - return math_ops.multiply(c_half * m_fac, - (math_ops.cos(x_val * c_pi) + c_one), name=name) diff --git a/tensorflow/contrib/training/python/training/sgdr_learning_rate_decay_test.py b/tensorflow/contrib/training/python/training/sgdr_learning_rate_decay_test.py deleted file mode 100644 index 3269d5fef20..00000000000 --- a/tensorflow/contrib/training/python/training/sgdr_learning_rate_decay_test.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Functional test for sgdr learning rate decay.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -from sgdr_learning_rate_decay import sgdr_decay -from tensorflow.python.platform import googletest -from tensorflow.python.framework import test_util -from tensorflow.python.framework import dtypes -from tensorflow import placeholder - - -class SGDRDecayTest(test_util.TensorFlowTestCase): - """Unit tests for SGDR learning rate decay.""" - - def get_original_values(self, lr, t_e, mult_factor, iter_per_epoch, epochs): - """Get an array with learning rate values from the consecutive steps using - the original implementation - (https://github.com/loshchil/SGDR/blob/master/SGDR_WRNs.py).""" - t0 = math.pi / 2.0 - tt = 0 - te_next = t_e - - lr_values = [] - sh_lr = lr - for epoch in range(epochs): - for _ in range(iter_per_epoch): - # In the original approach training function is executed here - lr_values.append(sh_lr) - dt = 2.0 * math.pi / float(2.0 * t_e) - tt = tt + float(dt) / iter_per_epoch - if tt >= math.pi: - tt = tt - math.pi - cur_t = t0 + tt - new_lr = lr * (1.0 + math.sin(cur_t)) / 2.0 # lr_min = 0, lr_max = lr - sh_lr = new_lr - if (epoch + 1) == te_next: # time to restart - sh_lr = lr - tt = 0 # by setting to 0 we set lr to lr_max, see above - t_e = t_e * mult_factor # change the period of restarts - te_next = te_next + t_e # note the next restart's epoch - - return lr_values - - def get_sgdr_values(self, lr, initial_period_steps, t_mul, iters): - """Get an array with learning rate values from the consecutive steps - using current tensorflow implementation.""" - with self.cached_session(): - step = placeholder(dtypes.int32) - - decay = sgdr_decay(lr, step, initial_period_steps, t_mul) - lr_values = [] - for i in range(iters): - lr_values.append(decay.eval(feed_dict={step: i})) - - return lr_values - - def testCompareToOriginal(self): - """Compare values generated by tensorflow implementation to the values - generated by the original implementation - (https://github.com/loshchil/SGDR/blob/master/SGDR_WRNs.py).""" - with self.cached_session(): - lr = 10.0 - init_steps = 2 - t_mul = 3 - iters = 10 - epochs = 50 - - org_lr = self.get_original_values(lr, init_steps, t_mul, iters, epochs) - sgdr_lr = self.get_sgdr_values(lr, init_steps*iters, t_mul, iters*epochs) - - for org, sgdr in zip(org_lr, sgdr_lr): - self.assertAllClose(org, sgdr) - - def testMDecay(self): - """Test m_mul argument. Check values for learning rate at the beginning - of the first, second, third and fourth period. """ - with self.cached_session(): - step = placeholder(dtypes.int32) - - lr = 0.1 - t_e = 10 - t_mul = 3 - m_mul = 0.9 - - decay = sgdr_decay(lr, step, t_e, t_mul, m_mul) - - test_step = 0 - self.assertAllClose(decay.eval(feed_dict={step: test_step}), - lr) - - test_step = t_e - self.assertAllClose(decay.eval(feed_dict={step: test_step}), - lr * m_mul) - - test_step = t_e + t_e*t_mul - self.assertAllClose(decay.eval(feed_dict={step: test_step}), - lr * m_mul**2) - - test_step = t_e + t_e*t_mul + t_e * (t_mul**2) - self.assertAllClose(decay.eval(feed_dict={step: test_step}), - lr * (m_mul**3)) - - def testCos(self): - """Check learning rate values at the beginning, in the middle - and at the end of the period.""" - with self.cached_session(): - step = placeholder(dtypes.int32) - lr = 0.2 - t_e = 1000 - t_mul = 1 - - decay = sgdr_decay(lr, step, t_e, t_mul) - - test_step = 0 - self.assertAllClose(decay.eval(feed_dict={step: test_step}), lr) - - test_step = t_e//2 - self.assertAllClose(decay.eval(feed_dict={step: test_step}), lr/2) - - test_step = t_e - self.assertAllClose(decay.eval(feed_dict={step: test_step}), lr) - - test_step = t_e*3//2 - self.assertAllClose(decay.eval(feed_dict={step: test_step}), lr/2) - -if __name__ == "__main__": - googletest.main() diff --git a/tensorflow/contrib/training/python/training/training.py b/tensorflow/contrib/training/python/training/training.py deleted file mode 100644 index 36d6e828476..00000000000 --- a/tensorflow/contrib/training/python/training/training.py +++ /dev/null @@ -1,550 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains various routines and helper functions for training models. - -This script contains various functions for training models. These include -manipulating gradients, creating a `train_op` (an operation that computes the -loss and applies the gradients) and a training loop function. The training loop -allows the user to pass in the `train_op` and runs the optimization according -to user-specified arguments. - -************************************ -* A simple working training script * -************************************ - - # Load data and create the model: - images, labels = LoadData(...) - predictions = MyModel(images) - - # Define the loss: - tf.contrib.losses.log_loss(predictions, labels) - total_loss = tf.contrib.losses.get_total_loss() - - # Define the optimizer: - optimizer = tf.compat.v1.train.MomentumOptimizer(FLAGS.learning_rate, - FLAGS.momentum) - - # Create the train_op - train_op = tf.contrib.training.create_train_op(total_loss, optimizer) - - # Run training. - tf.contrib.training.train(train_op, my_log_dir) - -************************* -* Creating the train_op * -************************* - -In order to use the `train` function, one needs a train_op: an `Operation` that -(a) computes the loss, (b) applies the gradients to update the weights and -(c) returns the value of the loss. tf.contrib.training.create_train_op creates -such an `Operation`. This function also provides the ability to manipulate -the gradients using a few arguments: - - # Create the train_op and clip the gradient norms: - train_op = tf.contrib.training.create_train_op( - total_loss, - optimizer, - transform_grads_fn=clip_gradient_norms_fn(3)) - - # Create the train_op and scale the gradients by providing a map from variable - # name (or variable) to a scaling coefficient: - def transform_grads_fn(grads): - gradient_multipliers = { - 'conv0/weights': 1.2, - 'fc8/weights': 3.4, - } - return tf.contrib.training.multiply_gradients( - grads, gradient_multipliers) - - train_op = tf.contrib.training.create_train_op( - total_loss, - optimizer, - transform_grads_fn=transform_grads_fn) - -**************************************************************** -* Performing additional (non-gradient) updates during training * -**************************************************************** - -Many networks utilize modules, like BatchNorm, that require performing a series -of non-gradient updates during training. tf.contrib.training.create_train_op -allows a user to pass in a list of update_ops to call along with the gradient -updates. - - train_op = tf.contrib.training.create_train_op( - total_loss, optimizer, update_ops) - -By default, tf.contrib.training.create_train_op includes all update ops that are -part of the `tf.GraphKeys.UPDATE_OPS` collection. Additionally, the -tf.contrib.layers.batch_norm function adds the moving mean and moving variance -updates to this collection. Consequently, users who want to use -tf.contrib.layers.batch_norm will not need to take any additional steps in order -to have the moving mean and moving variance updates be computed. - -However, users with additional, specialized updates can either override the -default update ops or simply add additional update ops to the -`tf.GraphKeys.UPDATE_OPS` collection: - - # Force `create_train_op` to NOT use ANY update_ops: - train_op = tf.contrib.training.create_train_op( - total_loss, - optimizer, - update_ops=[]) - - # Use an alternative set of update ops: - train_op = tf.contrib.training.create_train_op( - total_loss, - optimizer, - update_ops=my_other_update_ops) - - # Use a set of update ops in addition to the default updates: - tf.compat.v1.add_to_collection(tf.GraphKeys.UPDATE_OPS, my_update0) - tf.compat.v1.add_to_collection(tf.GraphKeys.UPDATE_OPS, my_update1) - - train_op = tf.contrib.training.create_train_op( - total_loss, - optimizer) - - # Which is the same as: - train_op = tf.contrib.training.create_train_op( - total_loss, - optimizer, - update_ops=tf.compat.v1.get_collection(tf.GraphKeys.UPDATE_OPS)) - -****************************************** -* Initializing a model from a checkpoint * -****************************************** - -It is common to want to 'warm-start' a model from a pre-trained checkpoint. -One can use a tf.Scaffold and an initializing function to do so. - - ... - - # Create the train_op - train_op = tf.contrib.training.create_train_op(total_loss, optimizer) - - # Create the initial assignment op - checkpoint_path = '/path/to/old_model_checkpoint' - variables_to_restore = tf.contrib.framework.get_model_variables() - init_fn = tf.contrib.framework.assign_from_checkpoint_fn( - checkpoint_path, variables_to_restore) - - # Run training. - scaffold = tf.Scaffold(init_fn=init_fn) - tf.contrib.training.train(train_op, my_log_dir, scaffold=scaffold) - -*************************************************************************** -* Initializing a model from a checkpoint whose variable names don't match * -*************************************************************************** - -At times, a user may want to initialize a new model with values from a -checkpoint whose variable names do not match those of the current model. In this -case, one needs to create a mapping from the checkpoint variable names to the -current model variables. This requires only a small modification of the code -above: - ... - # Creates a model with two variables, var0 and var1 - predictions = MyModel(images) - ... - - # Create the train_op - train_op = tf.contrib.training.create_train_op(total_loss, optimizer) - - checkpoint_path = '/path/to/old_model_checkpoint' - - # Create the mapping: - variables_to_restore = { - 'name_var_0_in_checkpoint': - tf.contrib.framework.get_unique_variable('var0'), - 'name_var_1_in_checkpoint': - tf.contrib.framework.get_unique_variable('var1') - } - init_fn = tf.contrib.framework.assign_from_checkpoint_fn( - checkpoint_path, variables_to_restore) - scaffold = tf.Scaffold(init_fn=init_fn) - - # Run training. - tf.contrib.training.train(train_op, my_log_dir, scaffold=scaffold) - - -************************************************* -* Fine-Tuning Part of a model from a checkpoint * -************************************************* - -Rather than initializing all of the weights of a given model, we sometimes -only want to restore some of the weights from a checkpoint. To do this, one -need only filter those variables to initialize as follows: - - ... - - # Create the train_op - train_op = tf.contrib.training.create_train_op(total_loss, optimizer) - - checkpoint_path = '/path/to/old_model_checkpoint' - - # Specify the variables to restore via a list of inclusion or exclusion - # patterns: - variables_to_restore = tf.contrib.framework.get_variables_to_restore( - include=["conv"], exclude=["fc8", "fc9]) - # or - variables_to_restore = tf.contrib.framework.get_variables_to_restore( - exclude=["conv"]) - - init_fn = tf.contrib.framework.assign_from_checkpoint_fn( - checkpoint_path, variables_to_restore) - scaffold = tf.Scaffold(init_fn=init_fn) - - # Run training. - tf.contrib.training.train(train_op, my_log_dir, scaffold=scaffold) - -****************************************************** -* Initializing model variables from values in memory * -****************************************************** - -One may want to initialize the weights of a model from values coming from an -arbitrary source (a text document, matlab file, etc). While this is technically -feasible using assign operations, this strategy results in the values of your -weights being stored in the graph. For large models, this becomes prohibitively -large. However, it's possible to perform this initial assignment without having -to store the values of the initial model in the graph itself by using -placeholders and a feed dictionary: - - ... - - # Create the train_op - train_op = tf.contrib.training.create_train_op(total_loss, optimizer) - - # Create the mapping from variable names to values: - var0_initial_value = ReadFromDisk(...) - var1_initial_value = ReadFromDisk(...) - - var_names_to_values = { - 'var0': var0_initial_value, - 'var1': var1_initial_value, - } - - init_fn = tf.contrib.framework.assign_from_values_fn(var_names_to_values) - scaffold = tf.Scaffold(init_fn=init_fn) - - # Run training. - tf.contrib.training.train(train_op, my_log_dir, scaffold=scaffold) -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import variables as tf_variables -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.summary import summary -from tensorflow.python.training import monitored_session -from tensorflow.python.training import optimizer as tf_optimizer -from tensorflow.python.training import training_util - -# TODO(nsilberman): move add_gradients_summaries, clip_gradient_norms and -# multiply_gradients into contrib/summaries and contrib/optimizers.py -__all__ = [ - 'add_gradients_summaries', - 'clip_gradient_norms', - 'clip_gradient_norms_fn', - 'create_train_op', - 'multiply_gradients', - 'train', -] - - -def add_gradients_summaries(grads_and_vars): - """Add summaries to gradients. - - Args: - grads_and_vars: A list of gradient to variable pairs (tuples). - - Returns: - The list of created summaries. - """ - summaries = [] - for grad, var in grads_and_vars: - if grad is not None: - if isinstance(grad, ops.IndexedSlices): - grad_values = grad.values - else: - grad_values = grad - summaries.append( - summary.histogram(var.op.name + '_gradient', grad_values)) - summaries.append( - summary.scalar(var.op.name + '_gradient_norm', - clip_ops.global_norm([grad_values]))) - else: - logging.info('Var %s has no gradient', var.op.name) - - return summaries - - -def clip_gradient_norms(gradients_to_variables, max_norm): - """Clips the gradients by the given value. - - Args: - gradients_to_variables: A list of gradient to variable pairs (tuples). - max_norm: the maximum norm value. - - Returns: - A list of clipped gradient to variable pairs. - """ - clipped_grads_and_vars = [] - for grad, var in gradients_to_variables: - if grad is not None: - if isinstance(grad, ops.IndexedSlices): - tmp = clip_ops.clip_by_norm(grad.values, max_norm) - grad = ops.IndexedSlices(tmp, grad.indices, grad.dense_shape) - else: - grad = clip_ops.clip_by_norm(grad, max_norm) - clipped_grads_and_vars.append((grad, var)) - return clipped_grads_and_vars - - -def clip_gradient_norms_fn(max_norm): - """Returns a `transform_grads_fn` function for gradient clipping.""" - - def clip_norms(gradients_to_variables): - return clip_gradient_norms(gradients_to_variables, max_norm) - - return clip_norms - - -def multiply_gradients(grads_and_vars, gradient_multipliers): - """Multiply specified gradients. - - Args: - grads_and_vars: A list of gradient to variable pairs (tuples). - gradient_multipliers: A map from either `Variables` or `Variable` op names - to the coefficient by which the associated gradient should be scaled. - - Returns: - The updated list of gradient to variable pairs. - - Raises: - ValueError: If `grads_and_vars` is not a list or if `gradient_multipliers` - is empty or None or if `gradient_multipliers` is not a dictionary. - """ - if not isinstance(grads_and_vars, list): - raise ValueError('`grads_and_vars` must be a list.') - if not gradient_multipliers: - raise ValueError('`gradient_multipliers` is empty.') - if not isinstance(gradient_multipliers, dict): - raise ValueError('`gradient_multipliers` must be a dict.') - - multiplied_grads_and_vars = [] - for grad, var in grads_and_vars: - if var in gradient_multipliers or var.op.name in gradient_multipliers: - key = var if var in gradient_multipliers else var.op.name - if grad is None: - raise ValueError('Requested multiple of `None` gradient.') - - if isinstance(grad, ops.IndexedSlices): - tmp = grad.values * ops.convert_to_tensor( - gradient_multipliers[key], dtype=grad.dtype) - grad = ops.IndexedSlices(tmp, grad.indices, grad.dense_shape) - else: - grad *= ops.convert_to_tensor( - gradient_multipliers[key], dtype=grad.dtype) - multiplied_grads_and_vars.append((grad, var)) - return multiplied_grads_and_vars - - -_USE_GLOBAL_STEP = 0 - - -def create_train_op(total_loss, - optimizer, - global_step=_USE_GLOBAL_STEP, - update_ops=None, - variables_to_train=None, - transform_grads_fn=None, - summarize_gradients=False, - gate_gradients=tf_optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - check_numerics=True): - """Creates an `Operation` that evaluates the gradients and returns the loss. - - Args: - total_loss: A `Tensor` representing the total loss. - optimizer: A tf.Optimizer to use for computing the gradients. - global_step: A `Tensor` representing the global step variable. If left as - `_USE_GLOBAL_STEP`, then tf.contrib.framework.global_step() is used. - update_ops: An optional list of updates to execute. If `update_ops` is - `None`, then the update ops are set to the contents of the - `tf.GraphKeys.UPDATE_OPS` collection. If `update_ops` is not `None`, but - it doesn't contain all of the update ops in `tf.GraphKeys.UPDATE_OPS`, a - warning will be displayed. - variables_to_train: an optional list of variables to train. If None, it will - default to all tf.compat.v1.trainable_variables(). - transform_grads_fn: A function which takes a single argument, a list of - gradient to variable pairs (tuples), performs any requested gradient - updates, such as gradient clipping or multipliers, and returns the updated - list. - summarize_gradients: Whether or not add summaries for each gradient. - gate_gradients: How to gate the computation of gradients. See tf.Optimizer. - aggregation_method: Specifies the method used to combine gradient terms. - Valid values are defined in the class `AggregationMethod`. - colocate_gradients_with_ops: Whether or not to try colocating the gradients - with the ops that generated them. - check_numerics: Whether or not we apply check_numerics. - - Returns: - A `Tensor` that when evaluated, computes the gradients and returns the total - loss value. - """ - if global_step is _USE_GLOBAL_STEP: - global_step = training_util.get_or_create_global_step() - - # Update ops use GraphKeys.UPDATE_OPS collection if update_ops is None. - global_update_ops = set(ops.get_collection(ops.GraphKeys.UPDATE_OPS)) - if update_ops is None: - update_ops = global_update_ops - else: - update_ops = set(update_ops) - if not global_update_ops.issubset(update_ops): - logging.warning('update_ops in create_train_op does not contain all the ' - 'update_ops in GraphKeys.UPDATE_OPS') - - # Make sure update_ops are computed before total_loss. - if update_ops: - with ops.control_dependencies(update_ops): - barrier = control_flow_ops.no_op(name='update_barrier') - total_loss = control_flow_ops.with_dependencies([barrier], total_loss) - - if variables_to_train is None: - # Default to tf.compat.v1.trainable_variables() - variables_to_train = tf_variables.trainable_variables() - else: - # Make sure that variables_to_train are in - # tf.compat.v1.trainable_variables() - for v in variables_to_train: - assert v.trainable or v in tf_variables.trainable_variables() - - assert variables_to_train - - # Create the gradients. Note that apply_gradients adds the gradient - # computation to the current graph. - grads = optimizer.compute_gradients( - total_loss, - variables_to_train, - gate_gradients=gate_gradients, - aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops) - - if transform_grads_fn: - grads = transform_grads_fn(grads) - - # Summarize gradients. - if summarize_gradients: - with ops.name_scope('summarize_grads'): - add_gradients_summaries(grads) - - # Create gradient updates. - grad_updates = optimizer.apply_gradients(grads, global_step=global_step) - - with ops.name_scope('train_op'): - # Make sure total_loss is valid. - if check_numerics: - total_loss = array_ops.check_numerics(total_loss, - 'LossTensor is inf or nan') - - # Ensure the train_tensor computes grad_updates. - train_op = control_flow_ops.with_dependencies([grad_updates], total_loss) - - # Add the operation used for training to the 'train_op' collection - train_ops = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) - if train_op not in train_ops: - train_ops.append(train_op) - - return train_op - - -def train(train_op, - logdir, - master='', - is_chief=True, - scaffold=None, - hooks=None, - chief_only_hooks=None, - save_checkpoint_secs=600, - save_summaries_steps=100, - config=None, - max_wait_secs=7200, - run_metadata=None): - """Runs the training loop. - - Args: - train_op: A `Tensor` that, when executed, will apply the gradients and - return the loss value. - logdir: The directory where the graph and checkpoints are saved. - master: The URL of the master. - is_chief: Specifies whether or not the training is being run by the primary - replica during replica training. - scaffold: An tf.compat.v1.train.Scaffold instance. - hooks: List of `tf.estimator.SessionRunHook` callbacks which are run inside - the training loop. - chief_only_hooks: List of `tf.estimator.SessionRunHook` instances which are - run inside the training loop for the chief trainer only. - save_checkpoint_secs: The frequency, in seconds, that a checkpoint is saved - using a default checkpoint saver. If `save_checkpoint_secs` is set to - `None`, then the default checkpoint saver isn't used. - save_summaries_steps: The frequency, in number of global steps, that the - summaries are written to disk using a default summary saver. If - `save_summaries_steps` is set to `None`, then the default summary saver - isn't used. - config: An instance of `tf.compat.v1.ConfigProto`. - max_wait_secs: Maximum time workers should wait for the session to become - available. This should be kept relatively short to help detect incorrect - code, but sometimes may need to be increased if the chief takes a while to - start up. - run_metadata: A [`RunMetadata`] protocol buffer. - - Returns: - the value of the loss function after training. - - Raises: - ValueError: if `logdir` is `None` and either `save_checkpoint_secs` or - `save_summaries_steps` are `None. - """ - if logdir is None and is_chief: - if save_summaries_steps: - raise ValueError( - 'logdir cannot be None when save_summaries_steps is not None') - - if save_checkpoint_secs: - raise ValueError( - 'logdir cannot be None when save_checkpoint_secs is not None') - - with monitored_session.MonitoredTrainingSession( - master=master, - is_chief=is_chief, - checkpoint_dir=logdir, - scaffold=scaffold, - hooks=hooks, - chief_only_hooks=chief_only_hooks, - save_checkpoint_secs=save_checkpoint_secs, - save_summaries_steps=save_summaries_steps, - config=config, - max_wait_secs=max_wait_secs) as session: - loss = None - while not session.should_stop(): - loss = session.run(train_op, run_metadata=run_metadata) - return loss diff --git a/tensorflow/contrib/training/python/training/training_test.py b/tensorflow/contrib/training/python/training/training_test.py deleted file mode 100644 index 3b524ac8c76..00000000000 --- a/tensorflow/contrib/training/python/training/training_test.py +++ /dev/null @@ -1,626 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tf.contrib.training.training.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -import numpy as np - -from tensorflow.contrib.framework.python.ops import variables as variables_lib -from tensorflow.contrib.layers.python.layers import layers -from tensorflow.contrib.training.python.training import training -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variables as variables_lib2 -from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import gfile -from tensorflow.python.platform import test -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.training import checkpoint_management -from tensorflow.python.training import gradient_descent -from tensorflow.python.training import monitored_session -from tensorflow.python.training import saver as saver_lib -# pylint: enable=g-import-not-at-top - - -def logistic_classifier(inputs): - return layers.fully_connected(inputs, 1, activation_fn=math_ops.sigmoid) - - -def batchnorm_classifier(inputs): - inputs = layers.batch_norm(inputs, decay=0.1, fused=False) - return layers.fully_connected(inputs, 1, activation_fn=math_ops.sigmoid) - - -class ClipGradsTest(test.TestCase): - - def testClipGrads(self): - xs = variables_lib2.Variable(0.0) - ys = xs * 4.0 - grads = gradients_impl.gradients([ys], [xs]) - gradients_to_variables = list(zip(grads, [xs])) - clipped_gradients_to_variables = training.clip_gradient_norms( - gradients_to_variables, 3.0) - - with self.cached_session() as session: - session.run(variables_lib2.global_variables_initializer()) - self.assertAlmostEqual(4.0, gradients_to_variables[0][0].eval()) - self.assertAlmostEqual(3.0, clipped_gradients_to_variables[0][0].eval()) - - def testClipGradsFn(self): - xs = variables_lib2.Variable(0.0) - ys = xs * 4.0 - grads = gradients_impl.gradients([ys], [xs]) - gradients_to_variables = list(zip(grads, [xs])) - clipped_gradients_to_variables = training.clip_gradient_norms_fn(3.0)( - gradients_to_variables) - - with self.cached_session() as session: - session.run(variables_lib2.global_variables_initializer()) - self.assertAlmostEqual(4.0, gradients_to_variables[0][0].eval()) - self.assertAlmostEqual(3.0, clipped_gradients_to_variables[0][0].eval()) - - -class CreateTrainOpTest(test.TestCase): - - def setUp(self): - np.random.seed(0) - - # Create an easy training set: - self._inputs = np.random.rand(16, 4).astype(np.float32) - self._labels = np.random.randint(0, 2, size=(16, 1)).astype(np.float32) - - def testTrainOpInCollection(self): - with ops.Graph().as_default(): - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = batchnorm_classifier(tf_inputs) - loss = losses.log_loss(tf_labels, tf_predictions) - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - train_op = training.create_train_op(loss, optimizer) - - # Make sure the training op was recorded in the proper collection - self.assertTrue(train_op in ops.get_collection(ops.GraphKeys.TRAIN_OP)) - - def testUseUpdateOps(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - expected_mean = np.mean(self._inputs, axis=(0)) - expected_var = np.var(self._inputs, axis=(0)) - - tf_predictions = batchnorm_classifier(tf_inputs) - loss = losses.log_loss(tf_labels, tf_predictions) - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = training.create_train_op(loss, optimizer) - - moving_mean = variables_lib.get_variables_by_name('moving_mean')[0] - moving_variance = variables_lib.get_variables_by_name('moving_variance')[ - 0] - - with self.cached_session() as session: - # Initialize all variables - session.run(variables_lib2.global_variables_initializer()) - mean, variance = session.run([moving_mean, moving_variance]) - # After initialization moving_mean == 0 and moving_variance == 1. - self.assertAllClose(mean, [0] * 4) - self.assertAllClose(variance, [1] * 4) - - for _ in range(10): - session.run(train_op) - - mean = moving_mean.eval() - variance = moving_variance.eval() - # After 10 updates with decay 0.1 moving_mean == expected_mean and - # moving_variance == expected_var. - self.assertAllClose(mean, expected_mean) - self.assertAllClose(variance, expected_var) - - def testEmptyUpdateOps(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = batchnorm_classifier(tf_inputs) - loss = losses.log_loss(tf_labels, tf_predictions) - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - train_op = training.create_train_op(loss, optimizer, update_ops=[]) - - moving_mean = variables_lib.get_variables_by_name('moving_mean')[0] - moving_variance = variables_lib.get_variables_by_name('moving_variance')[ - 0] - - with self.cached_session() as session: - # Initialize all variables - session.run(variables_lib2.global_variables_initializer()) - mean, variance = session.run([moving_mean, moving_variance]) - # After initialization moving_mean == 0 and moving_variance == 1. - self.assertAllClose(mean, [0] * 4) - self.assertAllClose(variance, [1] * 4) - - for _ in range(10): - session.run(train_op) - - mean = moving_mean.eval() - variance = moving_variance.eval() - - # Since we skip update_ops the moving_vars are not updated. - self.assertAllClose(mean, [0] * 4) - self.assertAllClose(variance, [1] * 4) - - def testGlobalStepIsIncrementedByDefault(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = batchnorm_classifier(tf_inputs) - loss = losses.log_loss(tf_labels, tf_predictions) - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - train_op = training.create_train_op(loss, optimizer) - - global_step = variables_lib.get_or_create_global_step() - - with self.cached_session() as session: - # Initialize all variables - session.run(variables_lib2.global_variables_initializer()) - - for _ in range(10): - session.run(train_op) - - # After 10 updates global_step should be 10. - self.assertAllClose(global_step.eval(), 10) - - def testGlobalStepNotIncrementedWhenSetToNone(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = batchnorm_classifier(tf_inputs) - loss = losses.log_loss(tf_labels, tf_predictions) - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - train_op = training.create_train_op(loss, optimizer, global_step=None) - - global_step = variables_lib.get_or_create_global_step() - - with self.cached_session() as session: - # Initialize all variables - session.run(variables_lib2.global_variables_initializer()) - - for _ in range(10): - session.run(train_op) - - # Since train_op don't use global_step it shouldn't change. - self.assertAllClose(global_step.eval(), 0) - - -class TrainBatchNormClassifierTest(test.TestCase): - - def setUp(self): - # Create an easy training set: - np.random.seed(0) - - self._inputs = np.zeros((16, 4)) - self._labels = np.random.randint(0, 2, size=(16, 1)).astype(np.float32) - - for i in range(16): - j = int(2 * self._labels[i] + np.random.randint(0, 2)) - self._inputs[i, j] = 1 - - def testTrainWithNoInitAssignCanAchieveZeroLoss(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = batchnorm_classifier(tf_inputs) - losses.log_loss(tf_labels, tf_predictions) - total_loss = losses.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = training.create_train_op(total_loss, optimizer) - - loss = training.train( - train_op, - None, - hooks=[basic_session_run_hooks.StopAtStepHook(num_steps=300)], - save_summaries_steps=None, - save_checkpoint_secs=None) - self.assertLess(loss, .1) - - -class TrainTest(test.TestCase): - - def setUp(self): - # Create an easy training set: - np.random.seed(0) - - self._inputs = np.zeros((16, 4)) - self._labels = np.random.randint(0, 2, size=(16, 1)).astype(np.float32) - - for i in range(16): - j = int(2 * self._labels[i] + np.random.randint(0, 2)) - self._inputs[i, j] = 1 - - def testCanAchieveZeroLoss(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = logistic_classifier(tf_inputs) - losses.log_loss(tf_labels, tf_predictions) - total_loss = losses.get_total_loss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - train_op = training.create_train_op(total_loss, optimizer) - - loss = training.train( - train_op, - None, - hooks=[basic_session_run_hooks.StopAtStepHook(num_steps=300)], - save_summaries_steps=None, - save_checkpoint_secs=None) - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - - def testTrainWithLocalVariable(self): - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - local_multiplier = variables_lib.local_variable(1.0) - - tf_predictions = logistic_classifier(tf_inputs) * local_multiplier - losses.log_loss(tf_labels, tf_predictions) - total_loss = losses.get_total_loss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - train_op = training.create_train_op(total_loss, optimizer) - - loss = training.train( - train_op, - None, - hooks=[basic_session_run_hooks.StopAtStepHook(num_steps=300)], - save_summaries_steps=None, - save_checkpoint_secs=None) - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - - def testResumeTrainAchievesRoughlyTheSameLoss(self): - number_of_steps = [300, 1, 5] - logdir = os.path.join(self.get_temp_dir(), 'resume_train_same_loss') - - for i in range(len(number_of_steps)): - with ops.Graph().as_default(): - random_seed.set_random_seed(i) - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = logistic_classifier(tf_inputs) - losses.log_loss(tf_labels, tf_predictions) - total_loss = losses.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = training.create_train_op(total_loss, optimizer) - - saver = saver_lib.Saver() - - loss = training.train( - train_op, - logdir, - hooks=[ - basic_session_run_hooks.StopAtStepHook( - num_steps=number_of_steps[i]), - basic_session_run_hooks.CheckpointSaverHook( - logdir, save_steps=50, saver=saver), - ], - save_checkpoint_secs=None, - save_summaries_steps=None) - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - - def create_train_op(self, learning_rate=1.0, gradient_multiplier=1.0): - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = logistic_classifier(tf_inputs) - losses.log_loss(tf_labels, tf_predictions) - total_loss = losses.get_total_loss() - - optimizer = gradient_descent.GradientDescentOptimizer( - learning_rate=learning_rate) - - def transform_grads_fn(grads): - if gradient_multiplier != 1.0: - variables = variables_lib2.trainable_variables() - gradient_multipliers = {var: gradient_multiplier for var in variables} - - with ops.name_scope('multiply_grads'): - return training.multiply_gradients(grads, gradient_multipliers) - else: - return grads - - return training.create_train_op( - total_loss, optimizer, transform_grads_fn=transform_grads_fn) - - def testTrainWithInitFromCheckpoint(self): - logdir1 = os.path.join(self.get_temp_dir(), 'tmp_logs1/') - logdir2 = os.path.join(self.get_temp_dir(), 'tmp_logs2/') - - if gfile.Exists(logdir1): # For running on jenkins. - gfile.DeleteRecursively(logdir1) - if gfile.Exists(logdir2): # For running on jenkins. - gfile.DeleteRecursively(logdir2) - - # First, train the model one step (make sure the error is high). - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - train_op = self.create_train_op() - saver = saver_lib.Saver() - loss = training.train( - train_op, - logdir1, - hooks=[ - basic_session_run_hooks.CheckpointSaverHook( - logdir1, save_steps=1, saver=saver), - basic_session_run_hooks.StopAtStepHook(num_steps=1), - ], - save_checkpoint_secs=None, - save_summaries_steps=None) - self.assertGreater(loss, .5) - - # Next, train the model to convergence. - with ops.Graph().as_default(): - random_seed.set_random_seed(1) - train_op = self.create_train_op() - saver = saver_lib.Saver() - loss = training.train( - train_op, - logdir1, - hooks=[ - basic_session_run_hooks.CheckpointSaverHook( - logdir1, save_steps=300, saver=saver), - basic_session_run_hooks.StopAtStepHook(num_steps=300), - ], - save_checkpoint_secs=None, - save_summaries_steps=None) - self.assertIsNotNone(loss) - self.assertLess(loss, .02) - - # Finally, advance the model a single step and validate that the loss is - # still low. - with ops.Graph().as_default(): - random_seed.set_random_seed(2) - train_op = self.create_train_op() - - model_variables = variables_lib2.global_variables() - model_path = checkpoint_management.latest_checkpoint(logdir1) - - assign_fn = variables_lib.assign_from_checkpoint_fn( - model_path, model_variables) - - def init_fn(_, session): - assign_fn(session) - - loss = training.train( - train_op, - None, - scaffold=monitored_session.Scaffold(init_fn=init_fn), - hooks=[basic_session_run_hooks.StopAtStepHook(num_steps=1)], - save_checkpoint_secs=None, - save_summaries_steps=None) - - self.assertIsNotNone(loss) - self.assertLess(loss, .02) - - def ModelLoss(self): - tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) - tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) - - tf_predictions = logistic_classifier(tf_inputs) - losses.log_loss(tf_labels, tf_predictions) - return losses.get_total_loss() - - def testTrainAllVarsHasLowerLossThanTrainSubsetOfVars(self): - logdir = os.path.join(self.get_temp_dir(), 'tmp_logs3/') - if gfile.Exists(logdir): # For running on jenkins. - gfile.DeleteRecursively(logdir) - - # First, train only the weights of the model. - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - total_loss = self.ModelLoss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - weights = variables_lib.get_variables_by_name('weights') - - train_op = training.create_train_op( - total_loss, optimizer, variables_to_train=weights) - - saver = saver_lib.Saver() - loss = training.train( - train_op, - logdir, - hooks=[ - basic_session_run_hooks.CheckpointSaverHook( - logdir, save_steps=200, saver=saver), - basic_session_run_hooks.StopAtStepHook(num_steps=200), - ], - save_checkpoint_secs=None, - save_summaries_steps=None) - self.assertGreater(loss, .015) - self.assertLess(loss, .05) - - # Next, train the biases of the model. - with ops.Graph().as_default(): - random_seed.set_random_seed(1) - total_loss = self.ModelLoss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - biases = variables_lib.get_variables_by_name('biases') - - train_op = training.create_train_op( - total_loss, optimizer, variables_to_train=biases) - - saver = saver_lib.Saver() - loss = training.train( - train_op, - logdir, - hooks=[ - basic_session_run_hooks.CheckpointSaverHook( - logdir, save_steps=300, saver=saver), - basic_session_run_hooks.StopAtStepHook(num_steps=300), - ], - save_checkpoint_secs=None, - save_summaries_steps=None) - self.assertGreater(loss, .015) - self.assertLess(loss, .05) - - # Finally, train both weights and bias to get lower loss. - with ops.Graph().as_default(): - random_seed.set_random_seed(2) - total_loss = self.ModelLoss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - - train_op = training.create_train_op(total_loss, optimizer) - saver = saver_lib.Saver() - loss = training.train( - train_op, - logdir, - hooks=[ - basic_session_run_hooks.StopAtStepHook(num_steps=400), - ], - save_checkpoint_secs=None, - save_summaries_steps=None) - self.assertIsNotNone(loss) - self.assertLess(loss, .015) - - def testTrainingSubsetsOfVariablesOnlyUpdatesThoseVariables(self): - # First, train only the weights of the model. - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - total_loss = self.ModelLoss() - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) - weights, biases = variables_lib.get_variables() - - train_op = training.create_train_op(total_loss, optimizer) - train_weights = training.create_train_op( - total_loss, optimizer, variables_to_train=[weights]) - train_biases = training.create_train_op( - total_loss, optimizer, variables_to_train=[biases]) - - with self.cached_session() as session: - # Initialize the variables. - session.run(variables_lib2.global_variables_initializer()) - - # Get the initial weights and biases values. - weights_values, biases_values = session.run([weights, biases]) - self.assertGreater(np.linalg.norm(weights_values), 0) - self.assertAlmostEqual(np.linalg.norm(biases_values), 0) - - # Update weights and biases. - loss = session.run(train_op) - self.assertGreater(loss, .5) - new_weights, new_biases = session.run([weights, biases]) - - # Check that the weights and biases have been updated. - self.assertGreater(np.linalg.norm(weights_values - new_weights), 0) - self.assertGreater(np.linalg.norm(biases_values - new_biases), 0) - - weights_values, biases_values = new_weights, new_biases - - # Update only weights. - loss = session.run(train_weights) - self.assertGreater(loss, .5) - new_weights, new_biases = session.run([weights, biases]) - - # Check that the weights have been updated, but biases have not. - self.assertGreater(np.linalg.norm(weights_values - new_weights), 0) - self.assertAlmostEqual(np.linalg.norm(biases_values - new_biases), 0) - weights_values = new_weights - - # Update only biases. - loss = session.run(train_biases) - self.assertGreater(loss, .5) - new_weights, new_biases = session.run([weights, biases]) - - # Check that the biases have been updated, but weights have not. - self.assertAlmostEqual(np.linalg.norm(weights_values - new_weights), 0) - self.assertGreater(np.linalg.norm(biases_values - new_biases), 0) - - def testTrainWithAlteredGradients(self): - # Use the same learning rate but different gradient multipliers - # to train two models. Model with equivalently larger learning - # rate (i.e., learning_rate * gradient_multiplier) has smaller - # training loss. - multipliers = [1., 1000.] - number_of_steps = 10 - learning_rate = 0.001 - - # First, train the model with equivalently smaller learning rate. - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - train_op = self.create_train_op( - learning_rate=learning_rate, gradient_multiplier=multipliers[0]) - - loss0 = training.train( - train_op, - None, - hooks=[ - basic_session_run_hooks.StopAtStepHook(num_steps=number_of_steps), - ], - save_checkpoint_secs=None, - save_summaries_steps=None) - self.assertIsNotNone(loss0) - self.assertGreater(loss0, .5) - - # Second, train the model with equivalently larger learning rate. - with ops.Graph().as_default(): - random_seed.set_random_seed(0) - train_op = self.create_train_op( - learning_rate=learning_rate, gradient_multiplier=multipliers[1]) - - loss1 = training.train( - train_op, - None, - hooks=[ - basic_session_run_hooks.StopAtStepHook(num_steps=number_of_steps), - ], - save_checkpoint_secs=None, - save_summaries_steps=None) - self.assertIsNotNone(loss1) - self.assertLess(loss1, .5) - - # The loss of the model trained with larger learning rate should - # be smaller. - self.assertGreater(loss0, loss1) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/training/python/training/tuner.py b/tensorflow/contrib/training/python/training/tuner.py deleted file mode 100644 index ad647a61da7..00000000000 --- a/tensorflow/contrib/training/python/training/tuner.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""The Tuner interface for hyper-parameters tuning.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import abc - -import six - -from tensorflow.contrib.framework.python.framework import experimental - - -@six.add_metaclass(abc.ABCMeta) -class Tuner(object): - """Tuner class is the interface for Experiment hyper-parameters tuning. - - Example: - ``` - def _create_my_experiment(run_config, hparams): - hidden_units = [hparams.unit_per_layer] * hparams.num_hidden_layers - - return tf.contrib.learn.Experiment( - estimator=DNNClassifier(config=run_config, hidden_units=hidden_units), - train_input_fn=my_train_input, - eval_input_fn=my_eval_input) - - tuner = create_tuner(study_configuration, objective_key) - - learn_runner.tune(experiment_fn=_create_my_experiment, tuner) - """ - - @experimental - @abc.abstractmethod - def next_trial(self): - """Switch to the next trial. - - Ask the tuning service for a new trial for hyper-parameters tuning. - - Returns: - A boolean indicating if a trial was assigned to the tuner. - - Raises: - RuntimeError: If the tuner is initialized correctly. - """ - raise NotImplementedError("Calling an abstract method.") - - @experimental - @abc.abstractmethod - def run_experiment(self, experiment_fn): - """Creates an Experiment by calling `experiment_fn` and executes it. - - It creates a `RunConfig`, which captures the current execution environment - configuration and retrieves the hyper-parameters for current trial from the - tuning service. Both are passed to the `experiment_fn` and used to create - the Experiment for current trial execution. When finished, the measure will - be reported to the tuning service. - - - If the `RunConfig` does not include a task type, then an exception is - raised. The task type should be one of the types supported by the tuner. If - tuner does not support the task type directly, it could delegate the task to - Experiment, which is usually a function of Experiment. An exception would be - raised, if neither tuner nor Experiment could support the task type. - - Args: - experiment_fn: A function that creates an `Experiment`. It should accept - an argument `run_config` which should be used to create the `Estimator` - (passed as `config` to its constructor), and an argument `hparams`, - which should be used for hyper-parameters tuning. It must return an - `Experiment`. - """ - raise NotImplementedError("Calling an abstract method.") diff --git a/tensorflow/contrib/util/BUILD b/tensorflow/contrib/util/BUILD deleted file mode 100644 index f0579cd3d8b..00000000000 --- a/tensorflow/contrib/util/BUILD +++ /dev/null @@ -1,83 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -load("//tensorflow:tensorflow.bzl", "tf_cc_binary") -load("//tensorflow:tensorflow.bzl", "tf_cc_test") - -package( - default_visibility = ["//tensorflow:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) - -exports_files(["LICENSE"]) - -# Convertor of a frozen graph definition into the memmapped format. -cc_library( - name = "convert_graphdef_memmapped_format_lib", - srcs = ["convert_graphdef_memmapped_format_lib.cc"], - hdrs = ["convert_graphdef_memmapped_format_lib.h"], - deps = [ - "//tensorflow/core:array_ops_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:framework_internal", - "//tensorflow/core:functional_ops_op_lib", - "//tensorflow/core:lib", - "//tensorflow/core:nn_ops_op_lib", - "//tensorflow/core:no_op_op_lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:sendrecv_ops_op_lib", - "//tensorflow/core:tensorflow", - "//tensorflow/core/kernels:immutable_constant_op", - ], -) - -tf_cc_binary( - name = "convert_graphdef_memmapped_format", - srcs = ["convert_graphdef_memmapped_format.cc"], - deps = [ - ":convert_graphdef_memmapped_format_lib", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - ], -) - -tf_cc_test( - name = "convert_graphdef_memmapped_format_test", - srcs = ["convert_graphdef_memmapped_format_test.cc"], - linkopts = select({ - "//tensorflow:macos": ["-headerpad_max_install_names"], - "//conditions:default": [], - }), - deps = [ - ":convert_graphdef_memmapped_format_lib", - "//tensorflow/cc:cc_ops", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:tensor_testutil", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -tf_cc_binary( - name = "inspect_checkpoint", - srcs = ["inspect_checkpoint.cc"], - deps = [ - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:tensorflow", - ], -) - -py_library( - name = "util_py", - srcs = glob(["**/*.py"]), - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:framework", - "//tensorflow/python:platform", - "//tensorflow/python:tensor_util", - "//tensorflow/python:util", - ], -) diff --git a/tensorflow/contrib/util/__init__.py b/tensorflow/contrib/util/__init__.py deleted file mode 100644 index acc5a049aa8..00000000000 --- a/tensorflow/contrib/util/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Utilities for dealing with Tensors. - -@@constant_value -@@make_tensor_proto -@@make_ndarray -@@ops_used_by_graph_def -@@stripped_op_list_for_graph - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.python.framework.meta_graph import ops_used_by_graph_def -from tensorflow.python.framework.meta_graph import stripped_op_list_for_graph -from tensorflow.python.framework.tensor_util import constant_value -from tensorflow.python.framework.tensor_util import make_tensor_proto -from tensorflow.python.framework.tensor_util import MakeNdarray as make_ndarray -# pylint: disable=unused_import -from tensorflow.python.util.all_util import remove_undocumented -remove_undocumented(__name__) diff --git a/tensorflow/contrib/util/convert_graphdef_memmapped_format.cc b/tensorflow/contrib/util/convert_graphdef_memmapped_format.cc deleted file mode 100644 index 29b124e2a88..00000000000 --- a/tensorflow/contrib/util/convert_graphdef_memmapped_format.cc +++ /dev/null @@ -1,90 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Utility that converts a "frozen" inference graph (output from the -// freeze_graph utility) into a format in which large Const ops are converted to -// ImmutableConst ops which are memmapped when the graph is executed by -// TensorFlow. -// -// tensorflow/contrib/util/convert_graphdef_memmapped_format -// --in_graph=frozen.model --out_graph=memmapped.mmodel -// -// Parameters: -// in_graph - name of a file with a frozen GraphDef proto in binary format -// out_graph - name of the output file, where the graph in memmapped format will -// be saved. -// min_conversion_size_bytes - tensors with fewer than this many bytes of data -// will not be converted to ImmutableConst format, and kept in the graph. - -#include - -#include "tensorflow/contrib/util/convert_graphdef_memmapped_format_lib.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/util/command_line_flags.h" - -namespace tensorflow { -namespace { - -int ParseFlagsAndConvertGraph(int argc, char* argv[]) { - string in_graph = ""; - string out_graph = ""; - int min_conversion_tensor_size = 10000; - std::vector flag_list = { - Flag("in_graph", &in_graph, "input graph"), - Flag("out_graph", &out_graph, "output graph"), - Flag("min_conversion_tensor_size", &min_conversion_tensor_size, - "constants with tensors that have less than this number elements " - "won't be converted into ImmutableConst (be memmapped)"), - }; - string usage = Flags::Usage(argv[0], flag_list); - const bool parse_result = Flags::Parse(&argc, argv, flag_list); - // We need to call this to set up global state for TensorFlow. - port::InitMain(usage.c_str(), &argc, &argv); - if (!parse_result) { - LOG(ERROR) << "\n" << usage; - return -1; - } - if (argc > 1) { - LOG(ERROR) << "Unknown argument " << argv[1] << "\n" << usage; - return -1; - } - if (in_graph.empty()) { - LOG(ERROR) << "in_graph graph can't be empty"; - return -1; - } - if (out_graph.empty()) { - LOG(ERROR) << "out_graph graph can't be empty"; - return -1; - } - if (min_conversion_tensor_size <= 0) { - LOG(ERROR) << "min_conversion_tensor_size must be > 0"; - return -1; - } - const auto result = ConvertConstantsToImmutable(in_graph, out_graph, - min_conversion_tensor_size); - if (!result.ok()) { - LOG(ERROR) << "Conversion failed " << result.error_message(); - return -1; - } - return 0; -} - -} // namespace -} // namespace tensorflow - -int main(int argc, char* argv[]) { - return tensorflow::ParseFlagsAndConvertGraph(argc, argv); -} diff --git a/tensorflow/contrib/util/convert_graphdef_memmapped_format_lib.cc b/tensorflow/contrib/util/convert_graphdef_memmapped_format_lib.cc deleted file mode 100644 index 9675428e56e..00000000000 --- a/tensorflow/contrib/util/convert_graphdef_memmapped_format_lib.cc +++ /dev/null @@ -1,175 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include "tensorflow/contrib/util/convert_graphdef_memmapped_format_lib.h" - -#include -#include "tensorflow/core/framework/attr_value.pb.h" -#include "tensorflow/core/framework/graph.pb.h" -#include "tensorflow/core/framework/node_def.pb.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor.pb.h" -#include "tensorflow/core/framework/tensor_shape.pb.h" -#include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/kernels/immutable_constant_op.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/util/memmapped_file_system_writer.h" - -namespace tensorflow { -namespace { -class NodeConverter { - public: - // Converts one node. In-place updates node_def, writes the tensor in - // memmapped - // format, using writer. If the conversion has been done, convert_counter is - // increased. - Status ConvertConstantsToImmutable(NodeDef* node_def, - MemmappedFileSystemWriter* writer, - int* convert_counter, - int min_conversion_size_bytes) { - // Check the size. - const AttrValue& value = node_def->attr().at("value"); - const TensorProto& tensor_proto = value.tensor(); - - // Create copies of tensor datatype and shape, to put into the operator - // after - // the tensor is destroyed. - const DataType tensor_data_type = tensor_proto.dtype(); - const TensorShapeProto tensor_shape = tensor_proto.tensor_shape(); - - // Check that the tensor type is POD, only these types are supported for - // memmapping. - // DataType enum is explicitly converted to int to avoid errors with passing - // enum type are a parameter type to std::unordered_set. - static std::unordered_set supported_types{ -#define TYPE_FOR_SET(type) static_cast(DataTypeToEnum::value), - TF_CALL_POD_TYPES(TYPE_FOR_SET) -#undef ADD_TYPE - }; - - if (supported_types.count(static_cast(tensor_data_type)) == 0) { - return Status::OK(); - } - - // Create Tensor from value and write it in memmapped format. - Tensor parsed(tensor_proto.dtype()); - if (!parsed.FromProto(cpu_allocator(), tensor_proto)) { - return errors::InvalidArgument("Cannot parse tensor from proto: ", - tensor_proto.DebugString()); - } - if (parsed.TotalBytes() < static_cast(min_conversion_size_bytes)) { - return Status::OK(); - } - - const string memmapped_region_name = - MemmappedFileSystem::kMemmappedPackagePrefix + - ConvertVariableNameToUniqueRegionName(node_def->name()); - - TF_RETURN_IF_ERROR(writer->SaveTensor(parsed, memmapped_region_name)); - - node_def->set_op("ImmutableConst"); - - // Erase all attributes and leave only attributes that can be understood by - // ImmutableConst. - auto* mutable_attr = node_def->mutable_attr(); - mutable_attr->clear(); - - { - AttrValue attr_value; - attr_value.set_type(tensor_data_type); - mutable_attr->insert({ImmutableConstantOp::kDTypeAttr, attr_value}); - } - { - AttrValue attr_value; - *(attr_value.mutable_shape()) = tensor_shape; - mutable_attr->insert({ImmutableConstantOp::kShapeAttr, attr_value}); - } - { - AttrValue attr_value; - attr_value.set_s(memmapped_region_name); - mutable_attr->insert( - {ImmutableConstantOp::kMemoryRegionNameAttr, attr_value}); - } - ++*convert_counter; - return Status::OK(); - } - - private: - string ConvertVariableNameToUniqueRegionName(const string& variable_name) { - string region_name = SanitizeVariableName(variable_name); - while (!used_names_.insert(region_name).second) { - region_name += '_'; - } - return region_name; - } - - static string SanitizeVariableName(const string& variable_name) { - string result; - for (char c : variable_name) { - if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || - (c >= '0' && c <= '9') || c == '_' || c == '.') { - result += c; - } else { - result += '_'; - } - } - return result; - } - std::unordered_set used_names_; -}; - -} // namespace - -// Loads the graph, replaces operators, and writes it out. -Status ConvertConstantsToImmutable(const string& in_graph_filename, - const string& out_graph_filename, - int min_conversion_size_bytes) { - Env* default_env = Env::Default(); - GraphDef graph_def; - const auto load_graph_status = - ReadBinaryProto(default_env, in_graph_filename, &graph_def); - if (!load_graph_status.ok()) { - return tensorflow::errors::NotFound( - "Failed to load graph at '", in_graph_filename, - "' : ", load_graph_status.error_message()); - } - - NodeConverter node_converter; - - // Create output writer. - MemmappedFileSystemWriter writer; - TF_RETURN_IF_ERROR(writer.InitializeToFile(default_env, out_graph_filename)); - - // Iterate over graph nodes, looking for Const and replacing it with - // ImmutableConst. - int convert_counter = 0; - for (int i = 0; i < graph_def.node_size(); ++i) { - const NodeDef& node = graph_def.node(i); - if (node.op() == "Const") { - // Try to convert to ImmutableConst - TF_RETURN_IF_ERROR(node_converter.ConvertConstantsToImmutable( - graph_def.mutable_node(i), &writer, &convert_counter, - min_conversion_size_bytes)); - } - } - TF_RETURN_IF_ERROR(writer.SaveProtobuf( - graph_def, MemmappedFileSystem::kMemmappedPackageDefaultGraphDef)); - TF_RETURN_IF_ERROR(writer.FlushAndClose()); - LOG(INFO) << "Converted " << convert_counter << " nodes"; - return Status::OK(); -} - -} // namespace tensorflow diff --git a/tensorflow/contrib/util/convert_graphdef_memmapped_format_lib.h b/tensorflow/contrib/util/convert_graphdef_memmapped_format_lib.h deleted file mode 100644 index 61fc6f36f7e..00000000000 --- a/tensorflow/contrib/util/convert_graphdef_memmapped_format_lib.h +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_UTIL_CONVERT_GRAPHDEF_MEMMAPPED_FORMAT_LIB_H_ -#define TENSORFLOW_CONTRIB_UTIL_CONVERT_GRAPHDEF_MEMMAPPED_FORMAT_LIB_H_ - -#include - -#include "tensorflow/core/lib/core/status.h" - -namespace tensorflow { - -// Converts a "frozen" inference graph (output from the freeze_graph utility) -// into a format in which large Const ops are converted to ImmutableConst ops -// which are memmapped when the graph is executed by TensorFlow. -Status ConvertConstantsToImmutable(const string& in_graph_filename, - const string& out_graph_filename, - int min_conversion_size_bytes); - -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_UTIL_CONVERT_GRAPHDEF_MEMMAPPED_FORMAT_LIB_H_ diff --git a/tensorflow/contrib/util/convert_graphdef_memmapped_format_test.cc b/tensorflow/contrib/util/convert_graphdef_memmapped_format_test.cc deleted file mode 100644 index 1207a338f39..00000000000 --- a/tensorflow/contrib/util/convert_graphdef_memmapped_format_test.cc +++ /dev/null @@ -1,136 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include "tensorflow/cc/ops/standard_ops.h" -#include "tensorflow/contrib/util/convert_graphdef_memmapped_format_lib.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/graph/graph_def_builder.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/platform/test_benchmark.h" -#include "tensorflow/core/public/session.h" -#include "tensorflow/core/util/memmapped_file_system.h" - -namespace tensorflow { -namespace { - -bool GraphHasImmutableConstNodes(const GraphDef& graph_def) { - for (const auto& node : graph_def.node()) { - if (node.op() == "ImmutableConst") { - return true; - } - } - return false; -} - -TEST(ConvertGraphdefMemmappedFormatTest, ConvertModel) { - const string dir = testing::TmpDir(); - const string filename_pb = io::JoinPath(dir, "graphdef.pb"); - - // Create a simple graph and write it to filename_pb. - constexpr int kTensorWidth = 4000; - constexpr int kTensorHeight = 100; - const TensorShape kTestTensorShape({kTensorWidth, kTensorHeight}); - const TensorShape kTestTensorShapeT({kTensorHeight, kTensorWidth}); - - Tensor test_tensor1(DT_FLOAT, kTestTensorShape); - test::FillFn(&test_tensor1, [](int) -> float { return 2.0; }); - - Tensor test_tensor2(DT_FLOAT, kTestTensorShapeT); - test::FillFn(&test_tensor2, [](int) -> float { return 3.0; }); - - auto root = Scope::NewRootScope().ExitOnError(); - Output m = ops::MatMul(root, test_tensor1, test_tensor2); - const string result_name = m.node()->name(); - - GraphDef graph_def; - TF_ASSERT_OK(root.ToGraphDef(&graph_def)); - string graph_def_serialized; - graph_def.SerializeToString(&graph_def_serialized); - TF_ASSERT_OK( - WriteStringToFile(Env::Default(), filename_pb, graph_def_serialized)); - - const string filename_mmap = io::JoinPath(dir, "graphdef.mmap"); - TF_ASSERT_OK(ConvertConstantsToImmutable(filename_pb, filename_mmap, 10000)); - - // Create and initialize MemmappedEnv from the converted file. - MemmappedEnv memmapped_env(Env::Default()); - TF_ASSERT_OK(memmapped_env.InitializeFromFile(filename_mmap)); - - // Load the graph and run calculations. - SessionOptions session_options; - session_options.env = &memmapped_env; - std::unique_ptr session(NewSession(session_options)); - ASSERT_TRUE(session != nullptr) << "Failed to create session"; - GraphDef loaded_graph_def; - TF_ASSERT_OK(ReadBinaryProto( - &memmapped_env, MemmappedFileSystem::kMemmappedPackageDefaultGraphDef, - &loaded_graph_def)); - ASSERT_TRUE(GraphHasImmutableConstNodes(loaded_graph_def)); - - TF_ASSERT_OK(session->Create(loaded_graph_def)) << "Can't create test graph"; - std::vector outputs; - TF_ASSERT_OK(session->Run({}, {result_name + ":0"}, {}, &outputs)); - ASSERT_EQ(outputs.size(), 1); - EXPECT_EQ(outputs.front().flat()(0), 2.0f * 3.0f * kTensorHeight); - EXPECT_EQ(outputs.front().flat()(1), 2.0f * 3.0f * kTensorHeight); - EXPECT_EQ(outputs.front().flat()(2), 2.0f * 3.0f * kTensorHeight); -} - -TEST(ConvertGraphdefMemmappedFormatTest, NotSupportedTypesConvert) { - // Create a graph with strings. - const string dir = testing::TmpDir(); - const string filename_pb = io::JoinPath(dir, "string_graphdef.pb"); - - constexpr int kTensorWidth = 4000; - constexpr int kTensorHeight = 100; - const TensorShape kTestTensorShape({kTensorWidth, kTensorHeight}); - Tensor test_tensor1(DT_STRING, kTestTensorShape); - test::FillFn(&test_tensor1, [](int) -> string { return "ABC"; }); - - Tensor test_tensor2(DT_STRING, kTestTensorShape); - test::FillFn(&test_tensor2, [](int) -> string { return "XYZ"; }); - auto root = Scope::NewRootScope().ExitOnError(); - Output m = ops::Add(root, test_tensor1, test_tensor2); - const string result_name = m.node()->name(); - - GraphDef graph_def; - TF_ASSERT_OK(root.ToGraphDef(&graph_def)); - string graph_def_serialized; - graph_def.SerializeToString(&graph_def_serialized); - TF_ASSERT_OK( - WriteStringToFile(Env::Default(), filename_pb, graph_def_serialized)); - - const string filename_mmap = io::JoinPath(dir, "string_graphdef.mmap"); - TF_ASSERT_OK(ConvertConstantsToImmutable(filename_pb, filename_mmap, 1000)); - - // Create and initialize MemmappedEnv from the converted file. - MemmappedEnv memmapped_env(Env::Default()); - TF_ASSERT_OK(memmapped_env.InitializeFromFile(filename_mmap)); - - // Load the graph and run calculations. - SessionOptions session_options; - session_options.env = &memmapped_env; - std::unique_ptr session(NewSession(session_options)); - ASSERT_TRUE(session != nullptr) << "Failed to create session"; - GraphDef loaded_graph_def; - TF_ASSERT_OK(ReadBinaryProto( - &memmapped_env, MemmappedFileSystem::kMemmappedPackageDefaultGraphDef, - &loaded_graph_def)); - ASSERT_FALSE(GraphHasImmutableConstNodes(loaded_graph_def)); -} - -} // namespace -} // namespace tensorflow diff --git a/tensorflow/contrib/util/inspect_checkpoint.cc b/tensorflow/contrib/util/inspect_checkpoint.cc deleted file mode 100644 index 9b578ceb075..00000000000 --- a/tensorflow/contrib/util/inspect_checkpoint.cc +++ /dev/null @@ -1,50 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/gtl/array_slice.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/util/tensor_slice_reader.h" - -namespace tensorflow { -namespace { - -int InspectCheckpoint(const string& in) { - tensorflow::checkpoint::TensorSliceReader reader( - in, tensorflow::checkpoint::OpenTableTensorSliceReader); - Status s = reader.status(); - if (!s.ok()) { - fprintf(stderr, "Unable to open the checkpoint file\n"); - return -1; - } - for (auto e : reader.Tensors()) { - fprintf(stdout, "%s %s\n", e.first.c_str(), - e.second->shape().DebugString().c_str()); - } - return 0; -} - -} // namespace -} // namespace tensorflow - -int main(int argc, char** argv) { - tensorflow::port::InitMain(argv[0], &argc, &argv); - if (argc != 2) { - fprintf(stderr, "Usage: %s checkpoint_file\n", argv[0]); - exit(1); - } - return tensorflow::InspectCheckpoint(argv[1]); -} diff --git a/tensorflow/contrib/util/loader.py b/tensorflow/contrib/util/loader.py deleted file mode 100644 index dca01d26f4c..00000000000 --- a/tensorflow/contrib/util/loader.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities for loading op libraries. - -@@load_op_library -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import re - -from tensorflow.python.framework import load_library -from tensorflow.python.platform import resource_loader - - -def load_op_library(path): - """Loads a contrib op library from the given path. - - NOTE(mrry): On Windows, we currently assume that some contrib op - libraries are statically linked into the main TensorFlow Python - extension DLL - use dynamically linked ops if the .so is present. - - Args: - path: An absolute path to a shared object file. - - Returns: - A Python module containing the Python wrappers for Ops defined in the - plugin. - """ - if os.name == 'nt': - # To avoid making every user_ops aware of windows, re-write - # the file extension from .so to .dll if .so file doesn't exist. - if not os.path.exists(path): - path = re.sub(r'\.so$', '.dll', path) - - # Currently we have only some user_ops as dlls on windows - don't try - # to load them if the dll is not found. - # TODO(mrry): Once we have all of them this check should be removed. - if not os.path.exists(path): - return None - path = resource_loader.get_path_to_datafile(path) - ret = load_library.load_op_library(path) - assert ret, 'Could not load %s' % path - return ret diff --git a/tensorflow/examples/speech_commands/train.py b/tensorflow/examples/speech_commands/train.py index 00cf7b03c1e..406c4ec0d5d 100644 --- a/tensorflow/examples/speech_commands/train.py +++ b/tensorflow/examples/speech_commands/train.py @@ -86,17 +86,6 @@ FLAGS = None def main(_): - if FLAGS.quantize: - try: - _ = tf.contrib - except AttributeError as e: - msg = e.args[0] - msg += ('\n\n The --quantize option still requires contrib, which is not ' - 'part of TensorFlow 2.0. Please install a previous version:' - '\n `pip install tensorflow<=1.15`') - e.args = (msg,) - raise e - # Set the verbosity based on flags (default is INFO, so we see all messages) tf.compat.v1.logging.set_verbosity(FLAGS.verbosity) @@ -164,8 +153,18 @@ def main(_): with tf.compat.v1.name_scope('cross_entropy'): cross_entropy_mean = tf.compat.v1.losses.sparse_softmax_cross_entropy( labels=ground_truth_input, logits=logits) + if FLAGS.quantize: - tf.contrib.quantize.create_training_graph(quant_delay=0) + try: + tf.contrib.quantize.create_training_graph(quant_delay=0) + except ImportError as e: + msg = e.args[0] + msg += ('\n\n The --quantize option still requires contrib, which is not ' + 'part of TensorFlow 2.0. Please install a previous version:' + '\n `pip install tensorflow<=1.15`') + e.args = (msg,) + raise e + with tf.compat.v1.name_scope('train'), tf.control_dependencies( control_dependencies): learning_rate_input = tf.compat.v1.placeholder( diff --git a/tensorflow/examples/speech_commands/train_test.py b/tensorflow/examples/speech_commands/train_test.py index 6e72853d114..cd5c47cea17 100644 --- a/tensorflow/examples/speech_commands/train_test.py +++ b/tensorflow/examples/speech_commands/train_test.py @@ -132,26 +132,6 @@ class TrainTest(test.TestCase): os.path.join(train.FLAGS.train_dir, train.FLAGS.model_architecture + '.ckpt-1.meta'))) - @requires_contrib - @test_util.run_deprecated_v1 - def testQuantizedTrain(self): - train.FLAGS = self._getDefaultFlags() - train.FLAGS.quantize = True - train.FLAGS.model_architecture = 'tiny_conv' - train.main('') - self.assertTrue( - gfile.Exists( - os.path.join(train.FLAGS.train_dir, - train.FLAGS.model_architecture + '.pbtxt'))) - self.assertTrue( - gfile.Exists( - os.path.join(train.FLAGS.train_dir, - train.FLAGS.model_architecture + '_labels.txt'))) - self.assertTrue( - gfile.Exists( - os.path.join(train.FLAGS.train_dir, - train.FLAGS.model_architecture + '.ckpt-1.meta'))) - if __name__ == '__main__': test.main() diff --git a/tensorflow/examples/tutorials/mnist/input_data.py b/tensorflow/examples/tutorials/mnist/input_data.py index dfe0174409a..c203c7b5341 100644 --- a/tensorflow/examples/tutorials/mnist/input_data.py +++ b/tensorflow/examples/tutorials/mnist/input_data.py @@ -14,9 +14,7 @@ # ============================================================================== """Functions for downloading and reading MNIST data (deprecated). -This module and all its submodules are deprecated. See -[contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) -for migration instructions. +This module and all its submodules are deprecated. """ from __future__ import absolute_import @@ -118,9 +116,7 @@ def _extract_labels(f, one_hot=False, num_classes=10): class _DataSet(object): """Container class for a _DataSet (deprecated). - THIS CLASS IS DEPRECATED. See - [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) - for general migration instructions. + THIS CLASS IS DEPRECATED. """ @deprecated(None, 'Please use alternatives such as official/mnist/_DataSet.py' diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD index 884c52e6da2..3bd836f5e4e 100644 --- a/tensorflow/java/BUILD +++ b/tensorflow/java/BUILD @@ -39,7 +39,6 @@ filegroup( "src/main/java/org/tensorflow/types/*.java", ]), visibility = [ - "//tensorflow/contrib/android:__pkg__", "//tensorflow/java:__pkg__", "//tensorflow/tools/android/inference_interface:__pkg__", ], diff --git a/tensorflow/java/src/main/native/BUILD b/tensorflow/java/src/main/native/BUILD index 4c3077e0d16..0b363ff577e 100644 --- a/tensorflow/java/src/main/native/BUILD +++ b/tensorflow/java/src/main/native/BUILD @@ -4,12 +4,6 @@ package(default_visibility = [ "//tensorflow/java:__pkg__", - # TODO(ashankar): Temporary hack for the Java API and - # //third_party/tensorflow/tools/android/inference_interface:android_tensorflow_inference_jni - # to co-exist in a single shared library. However, the hope is that - # //third_party/tensorflow/tools/android/inference_interface:android_tensorflow_jni can be - # removed once the Java API provides feature parity with it. - "//tensorflow/contrib/android:__pkg__", "//tensorflow/tools/android/inference_interface:__pkg__", ]) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index c3a4753bf0b..3feb216adfd 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -2336,29 +2336,17 @@ tf_gen_op_wrapper_private_py( tf_gen_op_wrapper_private_py( name = "ragged_array_ops_gen", - visibility = [ - "//learning/brain/contrib/text:__pkg__", - "//learning/brain/contrib/text/python/ragged:__pkg__", - "//tensorflow/python/ops/ragged:__pkg__", - ], + visibility = ["//tensorflow/python/ops/ragged:__pkg__"], ) tf_gen_op_wrapper_private_py( name = "ragged_math_ops_gen", - visibility = [ - "//learning/brain/contrib/text:__pkg__", - "//learning/brain/contrib/text/python/ragged:__pkg__", - "//tensorflow/python/ops/ragged:__pkg__", - ], + visibility = ["//tensorflow/python/ops/ragged:__pkg__"], ) tf_gen_op_wrapper_private_py( name = "ragged_conversion_ops_gen", - visibility = [ - "//learning/brain/contrib/text:__pkg__", - "//learning/brain/contrib/text/python/ragged:__pkg__", - "//tensorflow/python/ops/ragged:__pkg__", - ], + visibility = ["//tensorflow/python/ops/ragged:__pkg__"], ) tf_gen_op_wrapper_private_py( diff --git a/tensorflow/tools/android/inference_interface/java/org/tensorflow/contrib/android/RunStats.java b/tensorflow/tools/android/inference_interface/java/org/tensorflow/contrib/android/RunStats.java deleted file mode 100644 index 39996f6ab03..00000000000 --- a/tensorflow/tools/android/inference_interface/java/org/tensorflow/contrib/android/RunStats.java +++ /dev/null @@ -1,63 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -package org.tensorflow.contrib.android; - -/** Accumulate and analyze stats from metadata obtained from Session.Runner.run. */ -public class RunStats implements AutoCloseable { - - /** - * Options to be provided to a {@link org.tensorflow.Session.Runner} to enable stats accumulation. - */ - public static byte[] runOptions() { - return fullTraceRunOptions; - } - - public RunStats() { - nativeHandle = allocate(); - } - - @Override - public void close() { - if (nativeHandle != 0) { - delete(nativeHandle); - } - nativeHandle = 0; - } - - /** Accumulate stats obtained when executing a graph. */ - public synchronized void add(byte[] runMetadata) { - add(nativeHandle, runMetadata); - } - - /** Summary of the accumulated runtime stats. */ - public synchronized String summary() { - return summary(nativeHandle); - } - - private long nativeHandle; - - // Hack: This is what a serialized RunOptions protocol buffer with trace_level: FULL_TRACE ends - // up as. - private static byte[] fullTraceRunOptions = new byte[] {0x08, 0x03}; - - private static native long allocate(); - - private static native void delete(long handle); - - private static native void add(long handle, byte[] runMetadata); - - private static native String summary(long handle); -} diff --git a/tensorflow/tools/android/inference_interface/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java b/tensorflow/tools/android/inference_interface/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java deleted file mode 100644 index abddadac5bc..00000000000 --- a/tensorflow/tools/android/inference_interface/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java +++ /dev/null @@ -1,650 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -package org.tensorflow.contrib.android; - -import android.content.res.AssetManager; -import android.os.Build.VERSION; -import android.os.Trace; -import android.text.TextUtils; -import android.util.Log; -import java.io.ByteArrayOutputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.DoubleBuffer; -import java.nio.FloatBuffer; -import java.nio.IntBuffer; -import java.nio.LongBuffer; -import java.util.ArrayList; -import java.util.List; -import org.tensorflow.Graph; -import org.tensorflow.Operation; -import org.tensorflow.Session; -import org.tensorflow.Tensor; -import org.tensorflow.TensorFlow; -import org.tensorflow.Tensors; -import org.tensorflow.types.UInt8; - -/** - * Wrapper over the TensorFlow API ({@link Graph}, {@link Session}) providing a smaller API surface - * for inference. - * - *

See tensorflow/examples/android/src/org/tensorflow/demo/TensorFlowImageClassifier.java for an - * example usage. - */ -public class TensorFlowInferenceInterface { - private static final String TAG = "TensorFlowInferenceInterface"; - private static final String ASSET_FILE_PREFIX = "file:///android_asset/"; - - /* - * Load a TensorFlow model from the AssetManager or from disk if it is not an asset file. - * - * @param assetManager The AssetManager to use to load the model file. - * @param model The filepath to the GraphDef proto representing the model. - */ - public TensorFlowInferenceInterface(AssetManager assetManager, String model) { - prepareNativeRuntime(); - - this.modelName = model; - this.g = new Graph(); - this.sess = new Session(g); - this.runner = sess.runner(); - - final boolean hasAssetPrefix = model.startsWith(ASSET_FILE_PREFIX); - InputStream is = null; - try { - String aname = hasAssetPrefix ? model.split(ASSET_FILE_PREFIX)[1] : model; - is = assetManager.open(aname); - } catch (IOException e) { - if (hasAssetPrefix) { - throw new RuntimeException("Failed to load model from '" + model + "'", e); - } - // Perhaps the model file is not an asset but is on disk. - try { - is = new FileInputStream(model); - } catch (IOException e2) { - throw new RuntimeException("Failed to load model from '" + model + "'", e); - } - } - - try { - if (VERSION.SDK_INT >= 18) { - Trace.beginSection("initializeTensorFlow"); - Trace.beginSection("readGraphDef"); - } - - // TODO(ashankar): Can we somehow mmap the contents instead of copying them? - byte[] graphDef = new byte[is.available()]; - final int numBytesRead = is.read(graphDef); - if (numBytesRead != graphDef.length) { - throw new IOException( - "read error: read only " - + numBytesRead - + " of the graph, expected to read " - + graphDef.length); - } - - if (VERSION.SDK_INT >= 18) { - Trace.endSection(); // readGraphDef. - } - - loadGraph(graphDef, g); - is.close(); - Log.i(TAG, "Successfully loaded model from '" + model + "'"); - - if (VERSION.SDK_INT >= 18) { - Trace.endSection(); // initializeTensorFlow. - } - } catch (IOException e) { - throw new RuntimeException("Failed to load model from '" + model + "'", e); - } - } - - /* - * Load a TensorFlow model from provided InputStream. - * Note: The InputStream will not be closed after loading model, users need to - * close it themselves. - * - * @param is The InputStream to use to load the model. - */ - public TensorFlowInferenceInterface(InputStream is) { - prepareNativeRuntime(); - - // modelName is redundant for model loading from input stream, here is for - // avoiding error in initialization as modelName is marked final. - this.modelName = ""; - this.g = new Graph(); - this.sess = new Session(g); - this.runner = sess.runner(); - - try { - if (VERSION.SDK_INT >= 18) { - Trace.beginSection("initializeTensorFlow"); - Trace.beginSection("readGraphDef"); - } - - int baosInitSize = is.available() > 16384 ? is.available() : 16384; - ByteArrayOutputStream baos = new ByteArrayOutputStream(baosInitSize); - int numBytesRead; - byte[] buf = new byte[16384]; - while ((numBytesRead = is.read(buf, 0, buf.length)) != -1) { - baos.write(buf, 0, numBytesRead); - } - byte[] graphDef = baos.toByteArray(); - - if (VERSION.SDK_INT >= 18) { - Trace.endSection(); // readGraphDef. - } - - loadGraph(graphDef, g); - Log.i(TAG, "Successfully loaded model from the input stream"); - - if (VERSION.SDK_INT >= 18) { - Trace.endSection(); // initializeTensorFlow. - } - } catch (IOException e) { - throw new RuntimeException("Failed to load model from the input stream", e); - } - } - - /* - * Construct a TensorFlowInferenceInterface with provided Graph - * - * @param g The Graph to use to construct this interface. - */ - public TensorFlowInferenceInterface(Graph g) { - prepareNativeRuntime(); - - // modelName is redundant here, here is for - // avoiding error in initialization as modelName is marked final. - this.modelName = ""; - this.g = g; - this.sess = new Session(g); - this.runner = sess.runner(); - } - - /** - * Runs inference between the previously registered input nodes (via feed*) and the requested - * output nodes. Output nodes can then be queried with the fetch* methods. - * - * @param outputNames A list of output nodes which should be filled by the inference pass. - */ - public void run(String[] outputNames) { - run(outputNames, false); - } - - /** - * Runs inference between the previously registered input nodes (via feed*) and the requested - * output nodes. Output nodes can then be queried with the fetch* methods. - * - * @param outputNames A list of output nodes which should be filled by the inference pass. - */ - public void run(String[] outputNames, boolean enableStats) { - run(outputNames, enableStats, new String[] {}); - } - - /** An overloaded version of runInference that allows supplying targetNodeNames as well */ - public void run(String[] outputNames, boolean enableStats, String[] targetNodeNames) { - // Release any Tensors from the previous run calls. - closeFetches(); - - // Add fetches. - for (String o : outputNames) { - fetchNames.add(o); - TensorId tid = TensorId.parse(o); - runner.fetch(tid.name, tid.outputIndex); - } - - // Add targets. - for (String t : targetNodeNames) { - runner.addTarget(t); - } - - // Run the session. - try { - if (enableStats) { - Session.Run r = runner.setOptions(RunStats.runOptions()).runAndFetchMetadata(); - fetchTensors = r.outputs; - - if (runStats == null) { - runStats = new RunStats(); - } - runStats.add(r.metadata); - } else { - fetchTensors = runner.run(); - } - } catch (RuntimeException e) { - // Ideally the exception would have been let through, but since this interface predates the - // TensorFlow Java API, must return -1. - Log.e( - TAG, - "Failed to run TensorFlow inference with inputs:[" - + TextUtils.join(", ", feedNames) - + "], outputs:[" - + TextUtils.join(", ", fetchNames) - + "]"); - throw e; - } finally { - // Always release the feeds (to save resources) and reset the runner, this run is - // over. - closeFeeds(); - runner = sess.runner(); - } - } - - /** Returns a reference to the Graph describing the computation run during inference. */ - public Graph graph() { - return g; - } - - public Operation graphOperation(String operationName) { - final Operation operation = g.operation(operationName); - if (operation == null) { - throw new RuntimeException( - "Node '" + operationName + "' does not exist in model '" + modelName + "'"); - } - return operation; - } - - /** Returns the last stat summary string if logging is enabled. */ - public String getStatString() { - return (runStats == null) ? "" : runStats.summary(); - } - - /** - * Cleans up the state associated with this Object. - * - *

The TenosrFlowInferenceInterface object is no longer usable after this method returns. - */ - public void close() { - closeFeeds(); - closeFetches(); - sess.close(); - g.close(); - if (runStats != null) { - runStats.close(); - } - runStats = null; - } - - @Override - protected void finalize() throws Throwable { - try { - close(); - } finally { - super.finalize(); - } - } - - // Methods for taking a native Tensor and filling it with values from Java arrays. - - /** - * Given a source array with shape {@link dims} and content {@link src}, copy the contents into - * the input Tensor with name {@link inputName}. The source array {@link src} must have at least - * as many elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, boolean[] src, long... dims) { - byte[] b = new byte[src.length]; - - for (int i = 0; i < src.length; i++) { - b[i] = src[i] ? (byte) 1 : (byte) 0; - } - - addFeed(inputName, Tensor.create(Boolean.class, dims, ByteBuffer.wrap(b))); - } - - /** - * Given a source array with shape {@link dims} and content {@link src}, copy the contents into - * the input Tensor with name {@link inputName}. The source array {@link src} must have at least - * as many elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, float[] src, long... dims) { - addFeed(inputName, Tensor.create(dims, FloatBuffer.wrap(src))); - } - - /** - * Given a source array with shape {@link dims} and content {@link src}, copy the contents into - * the input Tensor with name {@link inputName}. The source array {@link src} must have at least - * as many elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, int[] src, long... dims) { - addFeed(inputName, Tensor.create(dims, IntBuffer.wrap(src))); - } - - /** - * Given a source array with shape {@link dims} and content {@link src}, copy the contents into - * the input Tensor with name {@link inputName}. The source array {@link src} must have at least - * as many elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, long[] src, long... dims) { - addFeed(inputName, Tensor.create(dims, LongBuffer.wrap(src))); - } - - /** - * Given a source array with shape {@link dims} and content {@link src}, copy the contents into - * the input Tensor with name {@link inputName}. The source array {@link src} must have at least - * as many elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, double[] src, long... dims) { - addFeed(inputName, Tensor.create(dims, DoubleBuffer.wrap(src))); - } - - /** - * Given a source array with shape {@link dims} and content {@link src}, copy the contents into - * the input Tensor with name {@link inputName}. The source array {@link src} must have at least - * as many elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, byte[] src, long... dims) { - addFeed(inputName, Tensor.create(UInt8.class, dims, ByteBuffer.wrap(src))); - } - - /** - * Copy a byte sequence into the input Tensor with name {@link inputName} as a string-valued - * scalar tensor. In the TensorFlow type system, a "string" is an arbitrary sequence of bytes, not - * a Java {@code String} (which is a sequence of characters). - */ - public void feedString(String inputName, byte[] src) { - addFeed(inputName, Tensors.create(src)); - } - - /** - * Copy an array of byte sequences into the input Tensor with name {@link inputName} as a - * string-valued one-dimensional tensor (vector). In the TensorFlow type system, a "string" is an - * arbitrary sequence of bytes, not a Java {@code String} (which is a sequence of characters). - */ - public void feedString(String inputName, byte[][] src) { - addFeed(inputName, Tensors.create(src)); - } - - // Methods for taking a native Tensor and filling it with src from Java native IO buffers. - - /** - * Given a source buffer with shape {@link dims} and content {@link src}, both stored as - * direct and native ordered java.nio buffers, copy the contents into the input - * Tensor with name {@link inputName}. The source buffer {@link src} must have at least as many - * elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, FloatBuffer src, long... dims) { - addFeed(inputName, Tensor.create(dims, src)); - } - - /** - * Given a source buffer with shape {@link dims} and content {@link src}, both stored as - * direct and native ordered java.nio buffers, copy the contents into the input - * Tensor with name {@link inputName}. The source buffer {@link src} must have at least as many - * elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, IntBuffer src, long... dims) { - addFeed(inputName, Tensor.create(dims, src)); - } - - /** - * Given a source buffer with shape {@link dims} and content {@link src}, both stored as - * direct and native ordered java.nio buffers, copy the contents into the input - * Tensor with name {@link inputName}. The source buffer {@link src} must have at least as many - * elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, LongBuffer src, long... dims) { - addFeed(inputName, Tensor.create(dims, src)); - } - - /** - * Given a source buffer with shape {@link dims} and content {@link src}, both stored as - * direct and native ordered java.nio buffers, copy the contents into the input - * Tensor with name {@link inputName}. The source buffer {@link src} must have at least as many - * elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, DoubleBuffer src, long... dims) { - addFeed(inputName, Tensor.create(dims, src)); - } - - /** - * Given a source buffer with shape {@link dims} and content {@link src}, both stored as - * direct and native ordered java.nio buffers, copy the contents into the input - * Tensor with name {@link inputName}. The source buffer {@link src} must have at least as many - * elements as that of the destination Tensor. If {@link src} has more elements than the - * destination has capacity, the copy is truncated. - */ - public void feed(String inputName, ByteBuffer src, long... dims) { - addFeed(inputName, Tensor.create(UInt8.class, dims, src)); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into a Java array. {@link - * dst} must have length greater than or equal to that of the source Tensor. This operation will - * not affect dst's content past the source Tensor's size. - */ - public void fetch(String outputName, float[] dst) { - fetch(outputName, FloatBuffer.wrap(dst)); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into a Java array. {@link - * dst} must have length greater than or equal to that of the source Tensor. This operation will - * not affect dst's content past the source Tensor's size. - */ - public void fetch(String outputName, int[] dst) { - fetch(outputName, IntBuffer.wrap(dst)); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into a Java array. {@link - * dst} must have length greater than or equal to that of the source Tensor. This operation will - * not affect dst's content past the source Tensor's size. - */ - public void fetch(String outputName, long[] dst) { - fetch(outputName, LongBuffer.wrap(dst)); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into a Java array. {@link - * dst} must have length greater than or equal to that of the source Tensor. This operation will - * not affect dst's content past the source Tensor's size. - */ - public void fetch(String outputName, double[] dst) { - fetch(outputName, DoubleBuffer.wrap(dst)); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into a Java array. {@link - * dst} must have length greater than or equal to that of the source Tensor. This operation will - * not affect dst's content past the source Tensor's size. - */ - public void fetch(String outputName, byte[] dst) { - fetch(outputName, ByteBuffer.wrap(dst)); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into the direct and - * native ordered java.nio buffer {@link dst}. {@link dst} must have capacity greater than - * or equal to that of the source Tensor. This operation will not affect dst's content past the - * source Tensor's size. - */ - public void fetch(String outputName, FloatBuffer dst) { - getTensor(outputName).writeTo(dst); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into the direct and - * native ordered java.nio buffer {@link dst}. {@link dst} must have capacity greater than - * or equal to that of the source Tensor. This operation will not affect dst's content past the - * source Tensor's size. - */ - public void fetch(String outputName, IntBuffer dst) { - getTensor(outputName).writeTo(dst); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into the direct and - * native ordered java.nio buffer {@link dst}. {@link dst} must have capacity greater than - * or equal to that of the source Tensor. This operation will not affect dst's content past the - * source Tensor's size. - */ - public void fetch(String outputName, LongBuffer dst) { - getTensor(outputName).writeTo(dst); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into the direct and - * native ordered java.nio buffer {@link dst}. {@link dst} must have capacity greater than - * or equal to that of the source Tensor. This operation will not affect dst's content past the - * source Tensor's size. - */ - public void fetch(String outputName, DoubleBuffer dst) { - getTensor(outputName).writeTo(dst); - } - - /** - * Read from a Tensor named {@link outputName} and copy the contents into the direct and - * native ordered java.nio buffer {@link dst}. {@link dst} must have capacity greater than - * or equal to that of the source Tensor. This operation will not affect dst's content past the - * source Tensor's size. - */ - public void fetch(String outputName, ByteBuffer dst) { - getTensor(outputName).writeTo(dst); - } - - private void prepareNativeRuntime() { - Log.i(TAG, "Checking to see if TensorFlow native methods are already loaded"); - try { - // Hack to see if the native libraries have been loaded. - new RunStats(); - Log.i(TAG, "TensorFlow native methods already loaded"); - } catch (UnsatisfiedLinkError e1) { - Log.i( - TAG, "TensorFlow native methods not found, attempting to load via tensorflow_inference"); - try { - System.loadLibrary("tensorflow_inference"); - Log.i(TAG, "Successfully loaded TensorFlow native methods (RunStats error may be ignored)"); - } catch (UnsatisfiedLinkError e2) { - throw new RuntimeException( - "Native TF methods not found; check that the correct native" - + " libraries are present in the APK."); - } - } - } - - private void loadGraph(byte[] graphDef, Graph g) throws IOException { - final long startMs = System.currentTimeMillis(); - - if (VERSION.SDK_INT >= 18) { - Trace.beginSection("importGraphDef"); - } - - try { - g.importGraphDef(graphDef); - } catch (IllegalArgumentException e) { - throw new IOException("Not a valid TensorFlow Graph serialization: " + e.getMessage()); - } - - if (VERSION.SDK_INT >= 18) { - Trace.endSection(); // importGraphDef. - } - - final long endMs = System.currentTimeMillis(); - Log.i( - TAG, - "Model load took " + (endMs - startMs) + "ms, TensorFlow version: " + TensorFlow.version()); - } - - private void addFeed(String inputName, Tensor t) { - // The string format accepted by TensorFlowInferenceInterface is node_name[:output_index]. - TensorId tid = TensorId.parse(inputName); - runner.feed(tid.name, tid.outputIndex, t); - feedNames.add(inputName); - feedTensors.add(t); - } - - private static class TensorId { - String name; - int outputIndex; - - // Parse output names into a TensorId. - // - // E.g., "foo" --> ("foo", 0), while "foo:1" --> ("foo", 1) - public static TensorId parse(String name) { - TensorId tid = new TensorId(); - int colonIndex = name.lastIndexOf(':'); - if (colonIndex < 0) { - tid.outputIndex = 0; - tid.name = name; - return tid; - } - try { - tid.outputIndex = Integer.parseInt(name.substring(colonIndex + 1)); - tid.name = name.substring(0, colonIndex); - } catch (NumberFormatException e) { - tid.outputIndex = 0; - tid.name = name; - } - return tid; - } - } - - private Tensor getTensor(String outputName) { - int i = 0; - for (String n : fetchNames) { - if (n.equals(outputName)) { - return fetchTensors.get(i); - } - ++i; - } - throw new RuntimeException( - "Node '" + outputName + "' was not provided to run(), so it cannot be read"); - } - - private void closeFeeds() { - for (Tensor t : feedTensors) { - t.close(); - } - feedTensors.clear(); - feedNames.clear(); - } - - private void closeFetches() { - for (Tensor t : fetchTensors) { - t.close(); - } - fetchTensors.clear(); - fetchNames.clear(); - } - - // Immutable state. - private final String modelName; - private final Graph g; - private final Session sess; - - // State reset on every call to run. - private Session.Runner runner; - private List feedNames = new ArrayList(); - private List> feedTensors = new ArrayList>(); - private List fetchNames = new ArrayList(); - private List> fetchTensors = new ArrayList>(); - - // Mutable state. - private RunStats runStats; -} diff --git a/tensorflow/tools/ci_build/builds/android.sh b/tensorflow/tools/ci_build/builds/android.sh index b5d37b4f117..a15764a6e93 100755 --- a/tensorflow/tools/ci_build/builds/android.sh +++ b/tensorflow/tools/ci_build/builds/android.sh @@ -27,7 +27,10 @@ configure_android_workspace echo "========== TensorFlow Demo Build Test ==========" TARGETS= -TARGETS+=" //tensorflow/examples/android:tensorflow_demo" + +# TODO(aselle): Reenable once this stops referencing contrib and back enabled. +# TARGETS+=" //tensorflow/examples/android:tensorflow_demo" + # Also build the Eager Runtime so it remains compatible with Android for the # benefits of clients like TensorFlow Lite. For now it is enough to build only # :execute, which what TF Lite needs. diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index 0a8871700e2..13ab92f97d2 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -94,6 +94,7 @@ COMMON_PIP_DEPS = [ "//tensorflow/python/saved_model:saved_model", "//tensorflow/python/tools:tools_pip", "//tensorflow/python/tools/api/generator:create_python_api", + "//tensorflow/python/tpu", "//tensorflow/python:test_ops", "//tensorflow/python:while_v2", "//tensorflow/tools/common:public_api", @@ -179,7 +180,6 @@ filegroup( "//tensorflow:windows": [], "//tensorflow:no_gcp_support": [], "//conditions:default": [ - "@com_github_googleapis_googleapis//:LICENSE", "@com_github_googlecloudplatform_google_cloud_cpp//:LICENSE", ], }) + select({ diff --git a/third_party/examples/eager/spinn/BUILD b/third_party/examples/eager/spinn/BUILD deleted file mode 100644 index 640bcb230c8..00000000000 --- a/third_party/examples/eager/spinn/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -licenses(["notice"]) # 3-clause BSD. - -py_library( - name = "spinn", - srcs = ["spinn.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/eager/python:tfe", - "//tensorflow/contrib/eager/python/examples/spinn:data", - "@six_archive//:six", - ], -) diff --git a/third_party/examples/eager/spinn/LICENSE b/third_party/examples/eager/spinn/LICENSE deleted file mode 100644 index 09d493bf1fc..00000000000 --- a/third_party/examples/eager/spinn/LICENSE +++ /dev/null @@ -1,29 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2017, -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/examples/eager/spinn/README.md b/third_party/examples/eager/spinn/README.md deleted file mode 100644 index e2fd8009a05..00000000000 --- a/third_party/examples/eager/spinn/README.md +++ /dev/null @@ -1,109 +0,0 @@ -# SPINN with TensorFlow eager execution - -SPINN, or Stack-Augmented Parser-Interpreter Neural Network, is a recursive -neural network that utilizes syntactic parse information for natural language -understanding. - -SPINN was originally described by: -Bowman, S.R., Gauthier, J., Rastogi A., Gupta, R., Manning, C.D., & Potts, C. - (2016). A Fast Unified Model for Parsing and Sentence Understanding. - https://arxiv.org/abs/1603.06021 - -Our implementation is based on @jekbradbury's PyTorch implementation at: -https://github.com/jekbradbury/examples/blob/spinn/snli/spinn.py, - -which was released under the BSD 3-Clause License at: -https://github.com/jekbradbury/examples/blob/spinn/LICENSE - -Other eager execution examples can be found under [tensorflow/contrib/eager/python/examples](../../../../tensorflow/contrib/eager/python/examples). - -## Content - -- [`data.py`](../../../../tensorflow/contrib/eager/python/examples/spinn/data.py): Pipeline for loading and preprocessing the - [SNLI](https://nlp.stanford.edu/projects/snli/) data and - [GloVe](https://nlp.stanford.edu/projects/glove/) word embedding, written - using the [`tf.data`](https://www.tensorflow.org/guide/datasets) - API. -- [`spinn.py`](./spinn.py): Model definition and training routines. - This example illustrates how one might perform the following actions with - eager execution enabled: - * defining a model consisting of a dynamic computation graph, - * assigning operations to the CPU or GPU dependending on device availability, - * training the model using the data from the `tf.data`-based pipeline, - * obtaining metrics such as mean accuracy during training, - * saving and loading checkpoints, - * writing summaries for monitoring and visualization in TensorBoard. - -## To run - -- Make sure you have installed TensorFlow release 1.5 or higher. Alternatively, - you can use the latest `tf-nightly` or `tf-nightly-gpu` pip - package to access the eager execution feature. - -- Download and extract the raw SNLI data and GloVe embedding vectors. - For example: - - ```bash - curl -fSsL https://nlp.stanford.edu/projects/snli/snli_1.0.zip --create-dirs -o /tmp/spinn-data/snli/snli_1.0.zip - unzip -d /tmp/spinn-data/snli /tmp/spinn-data/snli/snli_1.0.zip - curl -fSsL http://nlp.stanford.edu/data/glove.42B.300d.zip --create-dirs -o /tmp/spinn-data/glove/glove.42B.300d.zip - unzip -d /tmp/spinn-data/glove /tmp/spinn-data/glove/glove.42B.300d.zip - ``` - -- Train model. E.g., - - ```bash - python spinn.py --data_root /tmp/spinn-data --logdir /tmp/spinn-logs - ``` - - During training, model checkpoints and TensorBoard summaries will be written - periodically to the directory specified with the `--logdir` flag. - The training script will reload a saved checkpoint from the directory if it - can find one there. - - To view the summaries with TensorBoard: - - ```bash - tensorboard --logdir /tmp/spinn-logs - ``` - -- After training, you may use the model to perform inference on input data in - the SNLI data format. The premise and hypotheses sentences are specified with - the command-line flags `--inference_premise` and `--inference_hypothesis`, - respectively. Each sentence should include the words, as well as parentheses - representing a binary parsing of the sentence. The words and parentheses - should all be separated by spaces. For instance, - - ```bash - python spinn.py --data_root /tmp/spinn-data --logdir /tmp/spinn-logs \ - --inference_premise '( ( The dog ) ( ( is running ) . ) )' \ - --inference_hypothesis '( ( The dog ) ( moves . ) )' - ``` - - which will generate an output like the following, due to the semantic - consistency of the two sentences. - - ```none - Inference logits: - entailment: 1.101249 (winner) - contradiction: -2.374171 - neutral: -0.296733 - ``` - - By contrast, the following sentence pair: - - ```bash - python spinn.py --data_root /tmp/spinn-data --logdir /tmp/spinn-logs \ - --inference_premise '( ( The dog ) ( ( is running ) . ) )' \ - --inference_hypothesis '( ( The dog ) ( rests . ) )' - ``` - - will give you an output like the following, due to the semantic - contradiction of the two sentences. - - ```none - Inference logits: - entailment: -1.070098 - contradiction: 2.798695 (winner) - neutral: -1.402287 - ``` diff --git a/third_party/examples/eager/spinn/spinn.py b/third_party/examples/eager/spinn/spinn.py deleted file mode 100644 index de63ebe9e67..00000000000 --- a/third_party/examples/eager/spinn/spinn.py +++ /dev/null @@ -1,813 +0,0 @@ -r"""Implementation of SPINN in TensorFlow eager execution. - -SPINN: Stack-Augmented Parser-Interpreter Neural Network. - -Ths file contains model definition and code for training the model. - -The model definition is based on PyTorch implementation at: - https://github.com/jekbradbury/examples/tree/spinn/snli - -which was released under a BSD 3-Clause License at: -https://github.com/jekbradbury/examples/blob/spinn/LICENSE: - -Copyright (c) 2017, -All rights reserved. - -See ./LICENSE for more details. - -Instructions for use: -* See `README.md` for details on how to prepare the SNLI and GloVe data. -* Suppose you have prepared the data at "/tmp/spinn-data", use the folloing - command to train the model: - - ```bash - python spinn.py --data_root /tmp/spinn-data --logdir /tmp/spinn-logs - ``` - - Checkpoints and TensorBoard summaries will be written to "/tmp/spinn-logs". - -References: -* Bowman, S.R., Gauthier, J., Rastogi A., Gupta, R., Manning, C.D., & Potts, C. - (2016). A Fast Unified Model for Parsing and Sentence Understanding. - https://arxiv.org/abs/1603.06021 -* Bradbury, J. (2017). Recursive Neural Networks with PyTorch. - https://devblogs.nvidia.com/parallelforall/recursive-neural-networks-pytorch/ -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import itertools -import os -import sys -import time - -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow as tf - -import tensorflow.contrib.eager as tfe -from tensorflow.contrib.eager.python.examples.spinn import data - - -layers = tf.keras.layers - - -def _bundle(lstm_iter): - """Concatenate a list of Tensors along 1st axis and split result into two. - - Args: - lstm_iter: A `list` of `N` dense `Tensor`s, each of which has the shape - (R, 2 * M). - - Returns: - A `list` of two dense `Tensor`s, each of which has the shape (N * R, M). - """ - return tf.split(tf.concat(lstm_iter, 0), 2, axis=1) - - -def _unbundle(state): - """Concatenate a list of Tensors along 2nd axis and split result. - - This is the inverse of `_bundle`. - - Args: - state: A `list` of two dense `Tensor`s, each of which has the shape (R, M). - - Returns: - A `list` of `R` dense `Tensors`, each of which has the shape (1, 2 * M). - """ - return tf.split(tf.concat(state, 1), state[0].shape[0], axis=0) - - -# pylint: disable=not-callable -class Reducer(tf.keras.Model): - """A module that applies reduce operation on left and right vectors.""" - - def __init__(self, size, tracker_size=None): - super(Reducer, self).__init__() - self.left = layers.Dense(5 * size, activation=None) - self.right = layers.Dense(5 * size, activation=None, use_bias=False) - if tracker_size is not None: - self.track = layers.Dense(5 * size, activation=None, use_bias=False) - else: - self.track = None - - def call(self, left_in, right_in, tracking=None): - """Invoke forward pass of the Reduce module. - - This method feeds a linear combination of `left_in`, `right_in` and - `tracking` into a Tree LSTM and returns the output of the Tree LSTM. - - Args: - left_in: A list of length L. Each item is a dense `Tensor` with - the shape (1, n_dims). n_dims is the size of the embedding vector. - right_in: A list of the same length as `left_in`. Each item should have - the same shape as the items of `left_in`. - tracking: Optional list of the same length as `left_in`. Each item is a - dense `Tensor` with shape (1, tracker_size * 2). tracker_size is the - size of the Tracker's state vector. - - Returns: - Output: A list of length batch_size. Each item has the shape (1, n_dims). - """ - left, right = _bundle(left_in), _bundle(right_in) - lstm_in = self.left(left[0]) + self.right(right[0]) - if self.track and tracking: - lstm_in += self.track(_bundle(tracking)[0]) - return _unbundle(self._tree_lstm(left[1], right[1], lstm_in)) - - def _tree_lstm(self, c1, c2, lstm_in): - a, i, f1, f2, o = tf.split(lstm_in, 5, axis=1) - c = tf.tanh(a) * tf.sigmoid(i) + tf.sigmoid(f1) * c1 + tf.sigmoid(f2) * c2 - h = tf.sigmoid(o) * tf.tanh(c) - return h, c - - -class Tracker(tf.keras.Model): - """A module that tracks the history of the sentence with an LSTM.""" - - def __init__(self, tracker_size, predict): - """Constructor of Tracker. - - Args: - tracker_size: Number of dimensions of the underlying `LSTMCell`. - predict: (`bool`) Whether prediction mode is enabled. - """ - super(Tracker, self).__init__() - self._rnn = tf.nn.rnn_cell.LSTMCell(tracker_size) - self._state_size = tracker_size - if predict: - self._transition = layers.Dense(4) - else: - self._transition = None - - def reset_state(self): - self.state = None - - def call(self, bufs, stacks): - """Invoke the forward pass of the Tracker module. - - This method feeds the concatenation of the top two elements of the stacks - into an LSTM cell and returns the resultant state of the LSTM cell. - - Args: - bufs: A `list` of length batch_size. Each item is a `list` of - max_sequence_len (maximum sequence length of the batch). Each item - of the nested list is a dense `Tensor` of shape (1, d_proj), where - d_proj is the size of the word embedding vector or the size of the - vector space that the word embedding vector is projected to. - stacks: A `list` of size batch_size. Each item is a `list` of - variable length corresponding to the current height of the stack. - Each item of the nested list is a dense `Tensor` of shape (1, d_proj). - - Returns: - 1. A list of length batch_size. Each item is a dense `Tensor` of shape - (1, d_tracker * 2). - 2. If under predict mode, result of applying a Dense layer on the - first state vector of the RNN. Else, `None`. - """ - buf = _bundle([buf[-1] for buf in bufs])[0] - stack1 = _bundle([stack[-1] for stack in stacks])[0] - stack2 = _bundle([stack[-2] for stack in stacks])[0] - x = tf.concat([buf, stack1, stack2], 1) - if self.state is None: - batch_size = int(x.shape[0]) - zeros = tf.zeros((batch_size, self._state_size), dtype=tf.float32) - self.state = [zeros, zeros] - _, self.state = self._rnn(x, self.state) - unbundled = _unbundle(self.state) - if self._transition: - return unbundled, self._transition(self.state[0]) - else: - return unbundled, None - - -class SPINN(tf.keras.Model): - """Stack-augmented Parser-Interpreter Neural Network. - - See https://arxiv.org/abs/1603.06021 for more details. - """ - - def __init__(self, config): - """Constructor of SPINN. - - Args: - config: A `namedtupled` with the following attributes. - d_proj - (`int`) number of dimensions of the vector space to project the - word embeddings to. - d_tracker - (`int`) number of dimensions of the Tracker's state vector. - d_hidden - (`int`) number of the dimensions of the hidden state, for the - Reducer module. - n_mlp_layers - (`int`) number of multi-layer perceptron layers to use to - convert the output of the `Feature` module to logits. - predict - (`bool`) Whether the Tracker will enabled predictions. - """ - super(SPINN, self).__init__() - self.config = config - self.reducer = Reducer(config.d_hidden, config.d_tracker) - if config.d_tracker is not None: - self.tracker = Tracker(config.d_tracker, config.predict) - else: - self.tracker = None - - def call(self, buffers, transitions, training=False): - """Invoke the forward pass of the SPINN model. - - Args: - buffers: Dense `Tensor` of shape - (max_sequence_len, batch_size, config.d_proj). - transitions: Dense `Tensor` with integer values that represent the parse - trees of the sentences. A value of 2 indicates "reduce"; a value of 3 - indicates "shift". Shape: (max_sequence_len * 2 - 3, batch_size). - training: Whether the invocation is under training mode. - - Returns: - Output `Tensor` of shape (batch_size, config.d_embed). - """ - max_sequence_len, batch_size, d_proj = (int(x) for x in buffers.shape) - - # Split the buffers into left and right word items and put the initial - # items in a stack. - splitted = tf.split( - tf.reshape(tf.transpose(buffers, [1, 0, 2]), [-1, d_proj]), - max_sequence_len * batch_size, axis=0) - buffers = [splitted[k:k + max_sequence_len] - for k in xrange(0, len(splitted), max_sequence_len)] - stacks = [[buf[0], buf[0]] for buf in buffers] - - if self.tracker: - # Reset tracker state for new batch. - self.tracker.reset_state() - - num_transitions = transitions.shape[0] - - # Iterate through transitions and perform the appropriate stack-pop, reduce - # and stack-push operations. - transitions = transitions.numpy() - for i in xrange(num_transitions): - trans = transitions[i] - if self.tracker: - # Invoke tracker to obtain the current tracker states for the sentences. - tracker_states, trans_hypothesis = self.tracker(buffers, stacks=stacks) - if trans_hypothesis: - trans = tf.argmax(trans_hypothesis, axis=-1) - else: - tracker_states = itertools.repeat(None) - lefts, rights, trackings = [], [], [] - for transition, buf, stack, tracking in zip( - trans, buffers, stacks, tracker_states): - if int(transition) == 3: # Shift. - stack.append(buf.pop()) - elif int(transition) == 2: # Reduce. - rights.append(stack.pop()) - lefts.append(stack.pop()) - trackings.append(tracking) - - if rights: - reducer_output = self.reducer(lefts, rights, trackings) - reduced = iter(reducer_output) - - for transition, stack in zip(trans, stacks): - if int(transition) == 2: # Reduce. - stack.append(next(reduced)) - return _bundle([stack.pop() for stack in stacks])[0] - - -class Perceptron(tf.keras.Model): - """One layer of the SNLIClassifier multi-layer perceptron.""" - - def __init__(self, dimension, dropout_rate, previous_layer): - """Configure the Perceptron.""" - super(Perceptron, self).__init__() - self.dense = tf.keras.layers.Dense(dimension, activation=tf.nn.elu) - self.batchnorm = layers.BatchNormalization() - self.dropout = layers.Dropout(rate=dropout_rate) - self.previous_layer = previous_layer - - def call(self, x, training): - """Run previous Perceptron layers, then this one.""" - x = self.previous_layer(x, training=training) - x = self.dense(x) - x = self.batchnorm(x, training=training) - x = self.dropout(x, training=training) - return x - - -class SNLIClassifier(tf.keras.Model): - """SNLI Classifier Model. - - A model aimed at solving the SNLI (Standford Natural Language Inference) - task, using the SPINN model from above. For details of the task, see: - https://nlp.stanford.edu/projects/snli/ - """ - - def __init__(self, config, embed): - """Constructor of SNLICLassifier. - - Args: - config: A namedtuple containing required configurations for the model. It - needs to have the following attributes. - projection - (`bool`) whether the word vectors are to be projected onto - another vector space (of `d_proj` dimensions). - d_proj - (`int`) number of dimensions of the vector space to project the - word embeddings to. - embed_dropout - (`float`) dropout rate for the word embedding vectors. - n_mlp_layers - (`int`) number of multi-layer perceptron (MLP) layers to - use to convert the output of the `Feature` module to logits. - mlp_dropout - (`float`) dropout rate of the MLP layers. - d_out - (`int`) number of dimensions of the final output of the MLP - layers. - lr - (`float`) learning rate. - embed: A embedding matrix of shape (vocab_size, d_embed). - """ - super(SNLIClassifier, self).__init__() - self.config = config - self.embed = tf.constant(embed) - - self.projection = layers.Dense(config.d_proj) - self.embed_bn = layers.BatchNormalization() - self.embed_dropout = layers.Dropout(rate=config.embed_dropout) - self.encoder = SPINN(config) - - self.feature_bn = layers.BatchNormalization() - self.feature_dropout = layers.Dropout(rate=config.mlp_dropout) - - current_mlp = lambda result, training: result - for _ in range(config.n_mlp_layers): - current_mlp = Perceptron(dimension=config.d_mlp, - dropout_rate=config.mlp_dropout, - previous_layer=current_mlp) - self.mlp = current_mlp - self.mlp_output = layers.Dense( - config.d_out, - kernel_initializer=tf.random_uniform_initializer(minval=-5e-3, - maxval=5e-3)) - - def call(self, - premise, - premise_transition, - hypothesis, - hypothesis_transition, - training=False): - """Invoke the forward pass the SNLIClassifier model. - - Args: - premise: The word indices of the premise sentences, with shape - (max_prem_seq_len, batch_size). - premise_transition: The transitions for the premise sentences, with shape - (max_prem_seq_len * 2 - 3, batch_size). - hypothesis: The word indices of the hypothesis sentences, with shape - (max_hypo_seq_len, batch_size). - hypothesis_transition: The transitions for the hypothesis sentences, with - shape (max_hypo_seq_len * 2 - 3, batch_size). - training: Whether the invocation is under training mode. - - Returns: - The logits, as a dense `Tensor` of shape (batch_size, d_out), where d_out - is the size of the output vector. - """ - # Perform embedding lookup on the premise and hypothesis inputs, which have - # the word-index format. - premise_embed = tf.nn.embedding_lookup(self.embed, premise) - hypothesis_embed = tf.nn.embedding_lookup(self.embed, hypothesis) - - if self.config.projection: - # Project the embedding vectors to another vector space. - premise_embed = self.projection(premise_embed) - hypothesis_embed = self.projection(hypothesis_embed) - - # Perform batch normalization and dropout on the possibly projected word - # vectors. - premise_embed = self.embed_bn(premise_embed, training=training) - hypothesis_embed = self.embed_bn(hypothesis_embed, training=training) - premise_embed = self.embed_dropout(premise_embed, training=training) - hypothesis_embed = self.embed_dropout(hypothesis_embed, training=training) - - # Run the batch-normalized and dropout-processed word vectors through the - # SPINN encoder. - premise = self.encoder(premise_embed, premise_transition, - training=training) - hypothesis = self.encoder(hypothesis_embed, hypothesis_transition, - training=training) - - # Combine encoder outputs for premises and hypotheses into logits. - # Then apply batch normalization and dropuout on the logits. - logits = tf.concat( - [premise, hypothesis, premise - hypothesis, premise * hypothesis], 1) - logits = self.feature_dropout( - self.feature_bn(logits, training=training), training=training) - - # Apply the multi-layer perceptron on the logits. - logits = self.mlp(logits, training=training) - logits = self.mlp_output(logits) - return logits - - -class SNLIClassifierTrainer(tfe.Checkpointable): - """A class that coordinates the training of an SNLIClassifier.""" - - def __init__(self, snli_classifier, lr): - """Constructor of SNLIClassifierTrainer. - - Args: - snli_classifier: An instance of `SNLIClassifier`. - lr: Learning rate. - """ - self._model = snli_classifier - # Create a custom learning rate Variable for the RMSProp optimizer, because - # the learning rate needs to be manually decayed later (see - # decay_learning_rate()). - self._learning_rate = tf.Variable(lr, name="learning_rate") - self._optimizer = tf.train.RMSPropOptimizer(self._learning_rate, - epsilon=1e-6) - - def loss(self, labels, logits): - """Calculate the loss given a batch of data. - - Args: - labels: The truth labels, with shape (batch_size,). - logits: The logits output from the forward pass of the SNLIClassifier - model, with shape (batch_size, d_out), where d_out is the output - dimension size of the SNLIClassifier. - - Returns: - The loss value, as a scalar `Tensor`. - """ - return tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits( - labels=labels, logits=logits)) - - def train_batch(self, - labels, - premise, - premise_transition, - hypothesis, - hypothesis_transition): - """Train model on batch of data. - - Args: - labels: The truth labels, with shape (batch_size,). - premise: The word indices of the premise sentences, with shape - (max_prem_seq_len, batch_size). - premise_transition: The transitions for the premise sentences, with shape - (max_prem_seq_len * 2 - 3, batch_size). - hypothesis: The word indices of the hypothesis sentences, with shape - (max_hypo_seq_len, batch_size). - hypothesis_transition: The transitions for the hypothesis sentences, with - shape (max_hypo_seq_len * 2 - 3, batch_size). - - Returns: - 1. loss value as a scalar `Tensor`. - 2. logits as a dense `Tensor` of shape (batch_size, d_out), where d_out is - the output dimension size of the SNLIClassifier. - """ - with tf.GradientTape() as tape: - tape.watch(self._model.variables) - logits = self._model(premise, - premise_transition, - hypothesis, - hypothesis_transition, - training=True) - loss = self.loss(labels, logits) - gradients = tape.gradient(loss, self._model.variables) - self._optimizer.apply_gradients(zip(gradients, self._model.variables), - global_step=tf.train.get_global_step()) - return loss, logits - - def decay_learning_rate(self, decay_by): - """Decay learning rate of the optimizer by factor decay_by.""" - self._learning_rate.assign(self._learning_rate * decay_by) - print("Decayed learning rate of optimizer to: %s" % - self._learning_rate.numpy()) - - @property - def learning_rate(self): - return self._learning_rate - - @property - def model(self): - return self._model - - @property - def variables(self): - return (self._model.variables + [self.learning_rate] + - self._optimizer.variables()) - - -def _batch_n_correct(logits, label): - """Calculate number of correct predictions in a batch. - - Args: - logits: A logits Tensor of shape `(batch_size, num_categories)` and dtype - `float32`. - label: A labels Tensor of shape `(batch_size,)` and dtype `int64` - - Returns: - Number of correct predictions. - """ - return tf.reduce_sum( - tf.cast((tf.equal( - tf.argmax(logits, axis=1), label)), tf.float32)).numpy() - - -def _evaluate_on_dataset(snli_data, batch_size, trainer, use_gpu): - """Run evaluation on a dataset. - - Args: - snli_data: The `data.SnliData` to use in this evaluation. - batch_size: The batch size to use during this evaluation. - trainer: An instance of `SNLIClassifierTrainer to use for this - evaluation. - use_gpu: Whether GPU is being used. - - Returns: - 1. Average loss across all examples of the dataset. - 2. Average accuracy rate across all examples of the dataset. - """ - mean_loss = tfe.metrics.Mean() - accuracy = tfe.metrics.Accuracy() - for label, prem, prem_trans, hypo, hypo_trans in _get_dataset_iterator( - snli_data, batch_size): - if use_gpu: - label, prem, hypo = label.gpu(), prem.gpu(), hypo.gpu() - logits = trainer.model(prem, prem_trans, hypo, hypo_trans, training=False) - loss_val = trainer.loss(label, logits) - batch_size = tf.shape(label)[0] - mean_loss(loss_val, weights=batch_size.gpu() if use_gpu else batch_size) - accuracy(tf.argmax(logits, axis=1), label) - return mean_loss.result().numpy(), accuracy.result().numpy() - - -def _get_dataset_iterator(snli_data, batch_size): - """Get a data iterator for a split of SNLI data. - - Args: - snli_data: A `data.SnliData` object. - batch_size: The desired batch size. - - Returns: - A dataset iterator. - """ - with tf.device("/device:CPU:0"): - # Some tf.data ops, such as ShuffleDataset, are available only on CPU. - dataset = tf.data.Dataset.from_generator( - snli_data.get_generator(batch_size), - (tf.int64, tf.int64, tf.int64, tf.int64, tf.int64)) - dataset = dataset.shuffle(snli_data.num_batches(batch_size)) - return tfe.Iterator(dataset) - - -def train_or_infer_spinn(embed, - word2index, - train_data, - dev_data, - test_data, - config): - """Perform Training or Inference on a SPINN model. - - Args: - embed: The embedding matrix as a float32 numpy array with shape - [vocabulary_size, word_vector_len]. word_vector_len is the length of a - word embedding vector. - word2index: A `dict` mapping word to word index. - train_data: An instance of `data.SnliData`, for the train split. - dev_data: Same as above, for the dev split. - test_data: Same as above, for the test split. - config: A configuration object. See the argument to this Python binary for - details. - - Returns: - If `config.inference_premise ` and `config.inference_hypothesis` are not - `None`, i.e., inference mode: the logits for the possible labels of the - SNLI data set, as a `Tensor` of three floats. - else: - The trainer object. - Raises: - ValueError: if only one of config.inference_premise and - config.inference_hypothesis is specified. - """ - # TODO(cais): Refactor this function into separate one for training and - # inference. - use_gpu = tfe.num_gpus() > 0 and not config.force_cpu - device = "gpu:0" if use_gpu else "cpu:0" - print("Using device: %s" % device) - - if ((config.inference_premise and not config.inference_hypothesis) or - (not config.inference_premise and config.inference_hypothesis)): - raise ValueError( - "--inference_premise and --inference_hypothesis must be both " - "specified or both unspecified, but only one is specified.") - - if config.inference_premise: - # Inference mode. - inference_sentence_pair = [ - data.encode_sentence(config.inference_premise, word2index), - data.encode_sentence(config.inference_hypothesis, word2index)] - else: - inference_sentence_pair = None - - log_header = ( - " Time Epoch Iteration Progress (%Epoch) Loss Dev/Loss" - " Accuracy Dev/Accuracy") - log_template = ( - "{:>6.0f} {:>5.0f} {:>9.0f} {:>5.0f}/{:<5.0f} {:>7.0f}% {:>8.6f} {} " - "{:12.4f} {}") - dev_log_template = ( - "{:>6.0f} {:>5.0f} {:>9.0f} {:>5.0f}/{:<5.0f} {:>7.0f}% {:>8.6f} " - "{:8.6f} {:12.4f} {:12.4f}") - - summary_writer = tf.contrib.summary.create_file_writer( - config.logdir, flush_millis=10000) - - with tf.device(device), \ - summary_writer.as_default(), \ - tf.contrib.summary.always_record_summaries(): - model = SNLIClassifier(config, embed) - global_step = tf.train.get_or_create_global_step() - trainer = SNLIClassifierTrainer(model, config.lr) - checkpoint = tf.train.Checkpoint(trainer=trainer, global_step=global_step) - checkpoint.restore(tf.train.latest_checkpoint(config.logdir)) - - if inference_sentence_pair: - # Inference mode. - prem, prem_trans = inference_sentence_pair[0] - hypo, hypo_trans = inference_sentence_pair[1] - hypo_trans = inference_sentence_pair[1][1] - inference_logits = model( - tf.constant(prem), tf.constant(prem_trans), - tf.constant(hypo), tf.constant(hypo_trans), training=False) - inference_logits = inference_logits[0][1:] - max_index = tf.argmax(inference_logits) - print("\nInference logits:") - for i, (label, logit) in enumerate( - zip(data.POSSIBLE_LABELS, inference_logits)): - winner_tag = " (winner)" if max_index == i else "" - print(" {0:<16}{1:.6f}{2}".format(label + ":", logit, winner_tag)) - return inference_logits - - train_len = train_data.num_batches(config.batch_size) - start = time.time() - iterations = 0 - mean_loss = tfe.metrics.Mean() - accuracy = tfe.metrics.Accuracy() - print(log_header) - for epoch in xrange(config.epochs): - batch_idx = 0 - for label, prem, prem_trans, hypo, hypo_trans in _get_dataset_iterator( - train_data, config.batch_size): - if use_gpu: - label, prem, hypo = label.gpu(), prem.gpu(), hypo.gpu() - # prem_trans and hypo_trans are used for dynamic control flow and can - # remain on CPU. Same in _evaluate_on_dataset(). - - iterations += 1 - batch_train_loss, batch_train_logits = trainer.train_batch( - label, prem, prem_trans, hypo, hypo_trans) - batch_size = tf.shape(label)[0] - mean_loss(batch_train_loss.numpy(), - weights=batch_size.gpu() if use_gpu else batch_size) - accuracy(tf.argmax(batch_train_logits, axis=1), label) - - if iterations % config.save_every == 0: - checkpoint.save(os.path.join(config.logdir, "ckpt")) - - if iterations % config.dev_every == 0: - dev_loss, dev_frac_correct = _evaluate_on_dataset( - dev_data, config.batch_size, trainer, use_gpu) - print(dev_log_template.format( - time.time() - start, - epoch, iterations, 1 + batch_idx, train_len, - 100.0 * (1 + batch_idx) / train_len, - mean_loss.result(), dev_loss, - accuracy.result() * 100.0, dev_frac_correct * 100.0)) - tf.contrib.summary.scalar("dev/loss", dev_loss) - tf.contrib.summary.scalar("dev/accuracy", dev_frac_correct) - elif iterations % config.log_every == 0: - mean_loss_val = mean_loss.result() - accuracy_val = accuracy.result() - print(log_template.format( - time.time() - start, - epoch, iterations, 1 + batch_idx, train_len, - 100.0 * (1 + batch_idx) / train_len, - mean_loss_val, " " * 8, accuracy_val * 100.0, " " * 12)) - tf.contrib.summary.scalar("train/loss", mean_loss_val) - tf.contrib.summary.scalar("train/accuracy", accuracy_val) - # Reset metrics. - mean_loss = tfe.metrics.Mean() - accuracy = tfe.metrics.Accuracy() - - batch_idx += 1 - if (epoch + 1) % config.lr_decay_every == 0: - trainer.decay_learning_rate(config.lr_decay_by) - - test_loss, test_frac_correct = _evaluate_on_dataset( - test_data, config.batch_size, trainer, use_gpu) - print("Final test loss: %g; accuracy: %g%%" % - (test_loss, test_frac_correct * 100.0)) - - return trainer - - -def main(_): - config = FLAGS - - # Load embedding vectors. - vocab = data.load_vocabulary(FLAGS.data_root) - word2index, embed = data.load_word_vectors(FLAGS.data_root, vocab) - - if not (config.inference_premise or config.inference_hypothesis): - print("Loading train, dev and test data...") - train_data = data.SnliData( - os.path.join(FLAGS.data_root, "snli/snli_1.0/snli_1.0_train.txt"), - word2index, sentence_len_limit=FLAGS.sentence_len_limit) - dev_data = data.SnliData( - os.path.join(FLAGS.data_root, "snli/snli_1.0/snli_1.0_dev.txt"), - word2index, sentence_len_limit=FLAGS.sentence_len_limit) - test_data = data.SnliData( - os.path.join(FLAGS.data_root, "snli/snli_1.0/snli_1.0_test.txt"), - word2index, sentence_len_limit=FLAGS.sentence_len_limit) - else: - train_data = None - dev_data = None - test_data = None - - train_or_infer_spinn( - embed, word2index, train_data, dev_data, test_data, config) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description= - "TensorFlow eager implementation of the SPINN SNLI classifier.") - parser.add_argument("--data_root", type=str, default="/tmp/spinn-data", - help="Root directory in which the training data and " - "embedding matrix are found. See README.md for how to " - "generate such a directory.") - parser.add_argument("--sentence_len_limit", type=int, default=-1, - help="Maximum allowed sentence length (# of words). " - "The default of -1 means unlimited.") - parser.add_argument("--logdir", type=str, default="/tmp/spinn-logs", - help="Directory in which summaries will be written for " - "TensorBoard.") - parser.add_argument("--inference_premise", type=str, default=None, - help="Premise sentence for inference. Must be " - "accompanied by --inference_hypothesis. If specified, " - "will override all training parameters and perform " - "inference.") - parser.add_argument("--inference_hypothesis", type=str, default=None, - help="Hypothesis sentence for inference. Must be " - "accompanied by --inference_premise. If specified, will " - "override all training parameters and perform inference.") - parser.add_argument("--epochs", type=int, default=50, - help="Number of epochs to train.") - parser.add_argument("--batch_size", type=int, default=128, - help="Batch size to use during training.") - parser.add_argument("--d_proj", type=int, default=600, - help="Dimensions to project the word embedding vectors " - "to.") - parser.add_argument("--d_hidden", type=int, default=300, - help="Size of the hidden layer of the Tracker.") - parser.add_argument("--d_out", type=int, default=4, - help="Output dimensions of the SNLIClassifier.") - parser.add_argument("--d_mlp", type=int, default=1024, - help="Size of each layer of the multi-layer perceptron " - "of the SNLICLassifier.") - parser.add_argument("--n_mlp_layers", type=int, default=2, - help="Number of layers in the multi-layer perceptron " - "of the SNLICLassifier.") - parser.add_argument("--d_tracker", type=int, default=64, - help="Size of the tracker LSTM.") - parser.add_argument("--log_every", type=int, default=50, - help="Print log and write TensorBoard summary every _ " - "training batches.") - parser.add_argument("--lr", type=float, default=2e-3, - help="Initial learning rate.") - parser.add_argument("--lr_decay_by", type=float, default=0.75, - help="The ratio to multiply the learning rate by every " - "time the learning rate is decayed.") - parser.add_argument("--lr_decay_every", type=float, default=1, - help="Decay the learning rate every _ epoch(s).") - parser.add_argument("--dev_every", type=int, default=1000, - help="Run evaluation on the dev split every _ training " - "batches.") - parser.add_argument("--save_every", type=int, default=1000, - help="Save checkpoint every _ training batches.") - parser.add_argument("--embed_dropout", type=float, default=0.08, - help="Word embedding dropout rate.") - parser.add_argument("--mlp_dropout", type=float, default=0.07, - help="SNLIClassifier multi-layer perceptron dropout " - "rate.") - parser.add_argument("--no-projection", action="store_false", - dest="projection", - help="Whether word embedding vectors are projected to " - "another set of vectors (see d_proj).") - parser.add_argument("--predict_transitions", action="store_true", - dest="predict", - help="Whether the Tracker will perform prediction.") - parser.add_argument("--force_cpu", action="store_true", dest="force_cpu", - help="Force use CPU-only regardless of whether a GPU is " - "available.") - FLAGS, unparsed = parser.parse_known_args() - - tfe.run(main=main, argv=[sys.argv[0]] + unparsed)